From 998e529f8d9adda6db834a5d796b0fe76a7809d1 Mon Sep 17 00:00:00 2001
From: Robin Eklind <rnd0x00@gmail.com>
Date: Thu, 26 Mar 2020 19:15:58 +0100
Subject: [PATCH] llvm: update test cases to LLVM 10.0

Steps taken:

1. Update VER in testdata/llvm/Makefile from 9.0.0 to 10.0.0
2. Run `make` from testdata/llvm
3. Run `./skip.sh` from testdata/llvm
4. Run `git add .` from testdata/llvm

This is just the first step to update the test cases to LLVM 10.0.

A follow-up commit will update `*.ll.golden` test cases.

And another follow-up commit will update the test cases of Coreutils
and SQLite, which requires Clang 10.0 being installed on the system.

Updates llir/llvm#132.
---
 llvm/Makefile                                 |     4 +-
 .../Analysis/BasicAA/assume-index-positive.ll |   116 +
 llvm/test/Analysis/BasicAA/cs-cs.ll           |    40 +-
 llvm/test/Analysis/BasicAA/dereferenceable.ll |   149 +
 llvm/test/Analysis/BasicAA/gep-alias.ll       |     2 +-
 llvm/test/Analysis/BasicAA/intrinsics.ll      |     4 +-
 llvm/test/Analysis/BasicAA/ptrmask.ll         |    29 +
 llvm/test/Analysis/BasicAA/store-promote.ll   |    10 +-
 .../Analysis/BranchProbabilityInfo/basic.ll   |    18 +
 .../Analysis/BranchProbabilityInfo/fcmp.ll    |    41 +
 .../BranchProbabilityInfo/noreturn.ll         |    26 +
 .../Analysis/BranchProbabilityInfo/pr22718.ll |     6 +-
 .../ConstantFolding/binop-identity-undef.ll   |    50 +
 .../test/Analysis/ConstantFolding/copysign.ll |    53 +
 .../Analysis/ConstantFolding/gep-alias.ll     |    17 +
 .../ConstantFolding/gep-zeroinit-vector.ll    |     2 +-
 .../Analysis/ConstantFolding/insertelement.ll |    19 +
 llvm/test/Analysis/ConstantFolding/math-1.ll  |   195 +
 llvm/test/Analysis/ConstantFolding/math-2.ll  |    48 +
 llvm/test/Analysis/ConstantFolding/rint.ll    |   109 +
 llvm/test/Analysis/ConstantFolding/round.ll   |    92 +
 .../Analysis/ConstantFolding/shufflevector.ll |    11 +
 llvm/test/Analysis/ConstantFolding/trunc.ll   |   105 +
 .../Analysis/CostModel/AArch64/aggregates.ll  |   142 +
 .../test/Analysis/CostModel/AMDGPU/add-sub.ll |    73 +-
 .../CostModel/AMDGPU/addrspacecast.ll         |     1 +
 .../test/Analysis/CostModel/AMDGPU/bit-ops.ll |    57 +-
 .../CostModel/AMDGPU/extractelement.ll        |    11 +-
 llvm/test/Analysis/CostModel/AMDGPU/fabs.ll   |    21 +-
 llvm/test/Analysis/CostModel/AMDGPU/fadd.ll   |    30 +-
 llvm/test/Analysis/CostModel/AMDGPU/fdiv.ll   |   205 +-
 llvm/test/Analysis/CostModel/AMDGPU/fma.ll    |   120 +
 llvm/test/Analysis/CostModel/AMDGPU/fmul.ll   |    30 +-
 llvm/test/Analysis/CostModel/AMDGPU/fsub.ll   |    22 +-
 .../CostModel/AMDGPU/insertelement.ll         |    10 +-
 llvm/test/Analysis/CostModel/AMDGPU/mul.ll    |    74 +-
 llvm/test/Analysis/CostModel/AMDGPU/shifts.ll |    63 +-
 .../CostModel/AMDGPU/shufflevector.ll         |    27 +-
 llvm/test/Analysis/CostModel/ARM/arith.ll     |   733 +
 llvm/test/Analysis/CostModel/ARM/cast.ll      |   924 +-
 llvm/test/Analysis/CostModel/ARM/divrem.ll    |  1284 +-
 llvm/test/Analysis/CostModel/ARM/fparith.ll   |   172 +
 llvm/test/Analysis/CostModel/ARM/freeshift.ll |    96 +
 .../test/Analysis/CostModel/ARM/load_store.ll |   278 +
 llvm/test/Analysis/CostModel/ARM/select.ll    |   101 +-
 llvm/test/Analysis/CostModel/ARM/shuffle.ll   |   137 +-
 .../CostModel/PowerPC/future-cost-model.ll    |    16 +
 .../CostModel/PowerPC/insert_extract.ll       |    48 +-
 .../Analysis/CostModel/SystemZ/fp-cast.ll     |    36 +-
 .../CostModel/SystemZ/intrinsic-cost-crash.ll |     2 +-
 .../Analysis/CostModel/SystemZ/intrinsics.ll  |    22 +-
 .../CostModel/SystemZ/logic-miscext3.ll       |    24 +-
 .../test/Analysis/CostModel/X86/aggregates.ll |   142 +
 .../CostModel/X86/alternate-shuffle-cost.ll   |    36 +-
 llvm/test/Analysis/CostModel/X86/arith-fix.ll |    48 +-
 .../Analysis/CostModel/X86/arith-overflow.ll  |    84 +-
 .../test/Analysis/CostModel/X86/arith-ssat.ll |    12 +-
 .../test/Analysis/CostModel/X86/arith-usat.ll |    12 +-
 llvm/test/Analysis/CostModel/X86/arith.ll     |    32 +-
 llvm/test/Analysis/CostModel/X86/cast.ll      |    64 +-
 llvm/test/Analysis/CostModel/X86/ctlz.ll      |    95 +-
 llvm/test/Analysis/CostModel/X86/ctpop.ll     |     8 +-
 llvm/test/Analysis/CostModel/X86/extend.ll    |   226 +-
 llvm/test/Analysis/CostModel/X86/fptosi.ll    |   129 +-
 llvm/test/Analysis/CostModel/X86/fptoui.ll    |   134 +-
 llvm/test/Analysis/CostModel/X86/fshl.ll      |    24 +-
 llvm/test/Analysis/CostModel/X86/fshr.ll      |    24 +-
 llvm/test/Analysis/CostModel/X86/icmp.ll      |   232 +-
 .../CostModel/X86/masked-intrinsic-cost.ll    |    80 +-
 .../CostModel/X86/min-legal-vector-width.ll   |    18 +-
 .../test/Analysis/CostModel/X86/reduce-add.ll |   288 +-
 .../test/Analysis/CostModel/X86/reduce-and.ll |    52 +-
 .../test/Analysis/CostModel/X86/reduce-mul.ll |   112 +-
 llvm/test/Analysis/CostModel/X86/reduce-or.ll |    52 +-
 .../Analysis/CostModel/X86/reduce-smax.ll     |    72 +-
 .../Analysis/CostModel/X86/reduce-smin.ll     |    72 +-
 .../Analysis/CostModel/X86/reduce-umax.ll     |    76 +-
 .../Analysis/CostModel/X86/reduce-umin.ll     |    76 +-
 .../test/Analysis/CostModel/X86/reduce-xor.ll |    10 +-
 llvm/test/Analysis/CostModel/X86/reduction.ll |   455 +-
 .../X86/shuffle-extract_subvector.ll          |  1753 ++-
 .../CostModel/X86/shuffle-transpose.ll        |     8 +-
 llvm/test/Analysis/CostModel/X86/sitofp.ll    |    26 +-
 .../Analysis/CostModel/X86/slm-arith-costs.ll |    24 +-
 .../Analysis/CostModel/X86/testshiftashr.ll   |    40 +-
 .../Analysis/CostModel/X86/testshiftlshr.ll   |    38 +-
 .../Analysis/CostModel/X86/testshiftshl.ll    |    40 +-
 llvm/test/Analysis/CostModel/X86/trunc.ll     |   200 +-
 llvm/test/Analysis/CostModel/X86/uitofp.ll    |    38 +-
 .../Analysis/CostModel/X86/vector-extract.ll  |   728 +-
 .../Analysis/CostModel/X86/vector-insert.ll   |    74 +
 llvm/test/Analysis/DDG/basic-a.ll             |   202 +
 llvm/test/Analysis/DDG/basic-b.ll             |   233 +
 llvm/test/Analysis/DDG/basic-loopnest.ll      |   456 +
 llvm/test/Analysis/DDG/root-node.ll           |    49 +
 .../constant_functions_multi_dim.ll           |     4 +-
 llvm/test/Analysis/DependenceAnalysis/Dump.ll |    50 +
 .../DependenceAnalysis/MIVCheckConst.ll       |     2 +-
 .../NonCanonicalizedSubscript.ll              |     2 +-
 .../SimpleSIVNoValidityCheck.ll               |     1 +
 .../AMDGPU/b42473-r1-crash.ll                 |   111 +
 .../intrinsic_addressnottaken1.ll             |    44 +
 .../intrinsic_addressnottaken2.ll             |    50 +
 .../GlobalsModRef/intrinsic_addresstaken.ll   |    52 +
 .../lvi-after-jumpthreading.ll                |    10 +-
 .../LoopCacheAnalysis/PowerPC/compute-cost.ll |    35 +
 .../LoopCacheAnalysis/PowerPC/loads-store.ll  |    88 +
 .../LoopCacheAnalysis/PowerPC/matmul.ll       |    81 +
 .../LoopCacheAnalysis/PowerPC/matvecmul.ll    |   185 +
 .../LoopCacheAnalysis/PowerPC/single-store.ll |    77 +
 .../LoopCacheAnalysis/PowerPC/stencil.ll      |    98 +
 .../invariant.group-bug.ll                    |     4 +-
 llvm/test/Analysis/MemorySSA/debugvalue.ll    |    44 +
 llvm/test/Analysis/MemorySSA/debugvalue2.ll   |    54 +
 .../MemorySSA/loop-rotate-disablebasicaa.ll   |    26 +
 .../MemorySSA/loop-rotate-simplified-clone.ll |    29 +
 llvm/test/Analysis/MemorySSA/loop-unswitch.ll |    26 +
 llvm/test/Analysis/MemorySSA/pr28880.ll       |     2 +-
 llvm/test/Analysis/MemorySSA/pr39197.ll       |     2 +-
 llvm/test/Analysis/MemorySSA/pr40038.ll       |     2 +-
 llvm/test/Analysis/MemorySSA/pr40754.ll       |    57 +-
 llvm/test/Analysis/MemorySSA/pr41640.ll       |     5 +-
 llvm/test/Analysis/MemorySSA/pr42940.ll       |   189 +
 llvm/test/Analysis/MemorySSA/pr43044.ll       |    52 +
 llvm/test/Analysis/MemorySSA/pr43317.ll       |    35 +
 llvm/test/Analysis/MemorySSA/pr43320.ll       |    33 +
 llvm/test/Analysis/MemorySSA/pr43426.ll       |    40 +
 llvm/test/Analysis/MemorySSA/pr43427.ll       |    42 +
 llvm/test/Analysis/MemorySSA/pr43438.ll       |   100 +
 llvm/test/Analysis/MemorySSA/pr43493.ll       |    27 +
 llvm/test/Analysis/MemorySSA/pr43540.ll       |    34 +
 llvm/test/Analysis/MemorySSA/pr43541.ll       |    50 +
 llvm/test/Analysis/MemorySSA/pr43569.ll       |    49 +
 llvm/test/Analysis/MemorySSA/pr43641.ll       |    22 +
 llvm/test/Analysis/MemorySSA/pr44027.ll       |    27 +
 llvm/test/Analysis/MemorySSA/pr44029.ll       |    63 +
 llvm/test/Analysis/MemorySSA/renamephis.ll    |    51 +
 llvm/test/Analysis/MemorySSA/unreachable.ll   |    31 +
 .../MustExecute/must_be_executed_context.ll   |   399 +
 .../Analysis/ScalarEvolution/limit-depth.ll   |     2 +-
 .../ScalarEvolution/max-expr-cache.ll         |     4 +-
 .../max-trip-count-address-space.ll           |     2 +-
 .../ScalarEvolution/max-trip-count.ll         |     2 +-
 .../multiple-max-iterations.ll                |     2 +
 llvm/test/Analysis/ScalarEvolution/nsw.ll     |     2 +-
 llvm/test/Analysis/ScalarEvolution/pr22674.ll |     2 +-
 .../Analysis/ScalarEvolution/range_nw_flag.ll |   121 +
 .../ScalarEvolution/scev-canonical-mode.ll    |     2 +-
 .../test/Analysis/ScalarEvolution/sext-mul.ll |     4 +-
 .../ScalarEvolution/trip-count-andor.ll       |   365 +
 .../Analysis/ScalarEvolution/trip-count.ll    |    70 +-
 .../Analysis/ScalarEvolution/trip-count10.ll  |    78 +-
 .../Analysis/ScalarEvolution/trip-count12.ll  |     2 +-
 .../Analysis/ScalarEvolution/trip-count15.ll  |   121 +
 .../Analysis/ScalarEvolution/trip-count2.ll   |    50 +-
 .../Analysis/ScalarEvolution/trip-count3.ll   |    14 +-
 .../Analysis/ScalarEvolution/trip-count4.ll   |    37 +-
 .../Analysis/ScalarEvolution/trip-count6.ll   |    17 +-
 .../Analysis/ScalarEvolution/trip-count7.ll   |   133 +-
 .../Analysis/ScalarEvolution/trip-count8.ll   |    41 +-
 .../Analysis/ScalarEvolution/trip-count9.ll   |   263 +-
 .../ScalarEvolution/umin-umax-folds.ll        |   317 +
 .../ScalarEvolution/widenable-condition.ll    |    45 +
 .../TypeBasedAliasAnalysis/PR17620.ll         |     4 +-
 .../TypeBasedAliasAnalysis/functionattrs.ll   |     2 +-
 .../TypeBasedAliasAnalysis/intrinsics.ll      |     4 +-
 .../TypeBasedAliasAnalysis/memcpyopt.ll       |     5 +-
 .../TypeBasedAliasAnalysis/tbaa-path.ll       |     2 +-
 .../ValueTracking/known-nonnull-at.ll         |   122 +-
 .../ValueTracking/memory-dereferenceable.ll   |     8 +
 .../ValueTracking/non-negative-phi-bits.ll    |     2 +-
 .../Assembler/2003-11-11-ImplicitRename.ll    |     7 +
 llvm/test/Assembler/asm-path-writer.ll        |     4 +-
 .../test/Assembler/auto_upgrade_intrinsics.ll |    15 +
 llvm/test/Assembler/block-labels.ll           |     4 +-
 llvm/test/Assembler/byval-type-attr.ll        |    20 +-
 .../call-nonzero-program-addrspace-2.ll.x     |     4 +-
 ...talayout-invalid-function-ptr-alignment.ll |     5 +
 ...alayout-invalid-stack-natural-alignment.ll |     5 +
 llvm/test/Assembler/debug-info.ll             |     9 +-
 llvm/test/Assembler/dimodule.ll               |     4 +-
 .../export-symbol-anonymous-class.ll          |    38 +
 .../Assembler/incorrect-tdep-attrs-parsing.ll |     2 +-
 llvm/test/Assembler/invalid-arg-num-1.ll      |     6 +
 llvm/test/Assembler/invalid-arg-num-2.ll      |     6 +
 llvm/test/Assembler/invalid-arg-num-3.ll      |     6 +
 .../invoke-nonzero-program-addrspace.ll.x     |     5 +-
 llvm/test/Assembler/multi-mod-disassemble.ll  |    15 +
 .../Assembler/multi-summary-disassemble.ll    |    18 +
 .../Assembler/source-filename-backslash.ll    |     5 +-
 llvm/test/Assembler/thinlto-summary.ll        |    10 +-
 llvm/test/Assembler/thinlto-vtable-summary.ll |     2 +-
 llvm/test/Bindings/llvm-c/debug_info.ll       |    92 +-
 llvm/test/Bindings/llvm-c/echo.ll             |    18 +
 llvm/test/Bindings/llvm-c/freeze.ll           |    23 +
 llvm/test/Bitcode/DILocation-implicit-code.ll |     4 +-
 llvm/test/Bitcode/aarch64-addp-upgrade.ll     |    18 +
 llvm/test/Bitcode/attributes-3.3.ll           |    36 +-
 llvm/test/Bitcode/attributes.ll               |    72 +-
 llvm/test/Bitcode/avr-calling-conventions.ll  |    12 +-
 llvm/test/Bitcode/calling-conventions.3.2.ll  |     9 +
 llvm/test/Bitcode/compatibility-3.6.ll.x      |    15 +-
 llvm/test/Bitcode/compatibility-3.7.ll.x      |    15 +-
 llvm/test/Bitcode/compatibility-3.8.ll        |    15 +-
 llvm/test/Bitcode/compatibility-3.9.ll        |    15 +-
 llvm/test/Bitcode/compatibility-4.0.ll        |    15 +-
 llvm/test/Bitcode/compatibility-5.0.ll        |    19 +-
 llvm/test/Bitcode/compatibility-6.0.ll        |    19 +-
 llvm/test/Bitcode/compatibility.ll            |   167 +-
 llvm/test/Bitcode/drop-debug-info.3.5.ll.x    |     2 +-
 .../test/Bitcode/invalid-functionptr-align.ll |     5 +
 .../Bitcode/invalid-type-for-null-constant.ll |     6 +
 llvm/test/Bitcode/multi-module.ll             |     5 -
 .../Bitcode/operand-bundles-bc-analyzer.ll    |     1 +
 llvm/test/Bitcode/summary_version.ll          |     2 +-
 llvm/test/Bitcode/thinlto-alias3.ll           |    11 +
 llvm/test/Bitcode/thinlto-deadstrip-flag.ll   |     4 +-
 .../Bitcode/thinlto-synthetic-count-flag.ll   |     4 +-
 .../upgrade-arc-runtime-calls-bitcast.ll      |    21 +
 .../test/Bitcode/upgrade-arc-runtime-calls.ll |   125 +
 llvm/test/Bitcode/upgrade-clang-arc-use.ll    |     4 +-
 llvm/test/Bitcode/upgrade-datalayout.ll       |     9 +
 llvm/test/Bitcode/upgrade-datalayout2.ll      |    10 +
 llvm/test/Bitcode/upgrade-datalayout3.ll      |     8 +
 llvm/test/Bitcode/upgrade-frame-pointer.ll    |    33 +
 .../test/Bitcode/upgrade-memory-intrinsics.ll |     2 +-
 llvm/test/Bitcode/upgrade-tbaa.ll             |     2 +-
 llvm/test/BugPoint/attr-crash.ll              |    16 +
 llvm/test/BugPoint/func-attrs-keyval.ll       |     2 +-
 llvm/test/BugPoint/func-attrs.ll              |    12 +-
 llvm/test/BugPoint/metadata.ll                |    16 +-
 .../test/BugPoint/retain-crashing-metadata.ll |    22 +
 .../AArch64/GlobalISel/arm64-callingconv.ll   |    37 +-
 .../AArch64/GlobalISel/arm64-fallback.ll      |    68 +-
 .../GlobalISel/arm64-irtranslator-gep.ll      |    51 +
 .../GlobalISel/arm64-irtranslator-switch.ll   |    46 +-
 .../AArch64/GlobalISel/arm64-irtranslator.ll  |   245 +-
 .../GlobalISel/call-lowering-i128-on-stack.ll |    12 +
 .../AArch64/GlobalISel/call-translator-cse.ll |     6 +-
 .../AArch64/GlobalISel/call-translator-ios.ll |    10 +-
 .../GlobalISel/call-translator-musttail.ll    |    13 +
 .../call-translator-tail-call-weak.ll         |    15 +
 .../GlobalISel/call-translator-tail-call.ll   |   269 +
 .../call-translator-variadic-musttail.ll      |   223 +
 .../AArch64/GlobalISel/call-translator.ll     |    27 +-
 .../combiner-load-store-indexing.ll           |   182 +
 .../CodeGen/AArch64/GlobalISel/const-0.ll     |    25 +
 .../AArch64/GlobalISel/dynamic-alloca.ll      |    75 +-
 .../GlobalISel/gisel-commandline-option.ll    |    11 +-
 .../gisel-fail-intermediate-legalizer.ll      |     6 +-
 .../GlobalISel/integration-shuffle-vector.ll  |    25 +
 .../GlobalISel/irtranslator-exceptions.ll     |     4 +-
 .../GlobalISel/irtranslator-extends.ll        |    30 +
 .../irtranslator-split-vector-arg.ll          |    22 +
 .../AArch64/GlobalISel/irtranslator-tbaa.ll   |    19 +
 .../AArch64/GlobalISel/legalize-sext-128.ll   |     9 +
 .../AArch64/GlobalISel/memcpy_chk_no_tail.ll  |    30 +
 .../AArch64/GlobalISel/no-neon-no-fp.ll       |    13 +
 .../CodeGen/AArch64/GlobalISel/swifterror.ll  |    92 +-
 .../CodeGen/AArch64/GlobalISel/swiftself.ll   |    62 +
 .../GlobalISel/tail-call-no-save-fp-lr.ll     |    23 +
 .../AArch64/GlobalISel/translate-gep.ll       |    16 +-
 llvm/test/CodeGen/AArch64/O0-pipeline.ll      |    16 +-
 llvm/test/CodeGen/AArch64/O3-pipeline.ll      |    27 +-
 ...aarch64-2014-08-11-MachineCombinerCrash.ll |     2 +-
 .../AArch64/aarch64-a57-fp-load-balancing.ll  |     4 +-
 .../AArch64/aarch64-dynamic-stack-layout.ll   |   124 +-
 .../AArch64/aarch64-fix-cortex-a53-835769.ll  |    16 +-
 llvm/test/CodeGen/AArch64/aarch64-smull.ll    |    60 +
 .../CodeGen/AArch64/aarch64-stp-cluster.ll    |    20 +-
 .../AArch64/aarch64-sve-asm-negative.ll       |    12 +
 llvm/test/CodeGen/AArch64/aarch64-sve-asm.ll  |    70 +
 .../AArch64/addsub-constant-folding.ll        |    72 +-
 llvm/test/CodeGen/AArch64/addsub-shifted.ll   |    41 +-
 llvm/test/CodeGen/AArch64/addsub_ext.ll       |    53 +-
 llvm/test/CodeGen/AArch64/align-down.ll       |   149 +
 llvm/test/CodeGen/AArch64/alloca.ll           |    13 +-
 llvm/test/CodeGen/AArch64/arm64-aapcs.ll      |     2 +-
 .../test/CodeGen/AArch64/arm64-abi-varargs.ll |   147 +-
 llvm/test/CodeGen/AArch64/arm64-abi_align.ll  |    16 +-
 .../arm64-alloca-frame-pointer-offset.ll      |     4 +-
 .../CodeGen/AArch64/arm64-blockaddress.ll     |     8 +-
 .../CodeGen/AArch64/arm64-call-tailcalls.ll   |     1 +
 .../AArch64/arm64-code-model-large-darwin.ll  |    16 +
 .../arm64-collect-loh-garbage-crash.ll        |     1 +
 .../CodeGen/AArch64/arm64-collect-loh-str.ll  |     1 +
 .../test/CodeGen/AArch64/arm64-collect-loh.ll |   115 +-
 .../CodeGen/AArch64/arm64-detect-vec-redux.ll |     2 +-
 .../AArch64/arm64-fast-isel-addr-offset.ll    |     2 +-
 .../arm64-fastisel-gep-promote-before-add.ll  |     2 +-
 .../arm64-fma-combine-with-fpfusion.ll        |     2 +-
 llvm/test/CodeGen/AArch64/arm64-fmadd.ll      |    18 +
 llvm/test/CodeGen/AArch64/arm64-fp.ll         |    47 +-
 .../CodeGen/AArch64/arm64-indexed-memory.ll   |     1 +
 .../AArch64/arm64-indexed-vector-ldst-2.ll    |     2 +-
 llvm/test/CodeGen/AArch64/arm64-inline-asm.ll |    47 +-
 .../test/CodeGen/AArch64/arm64-large-frame.ll |     2 +-
 .../test/CodeGen/AArch64/arm64-ldp-cluster.ll |    36 +-
 llvm/test/CodeGen/AArch64/arm64-ldxr-stxr.ll  |    73 +
 .../CodeGen/AArch64/arm64-memset-inline.ll    |     4 +-
 .../AArch64/arm64-memset-to-bzero-pgso.ll     |   128 +
 .../AArch64/arm64-misched-basic-A53.ll        |     2 +-
 .../AArch64/arm64-misched-basic-A57.ll        |     2 +-
 .../test/CodeGen/AArch64/arm64-neon-2velem.ll |  1142 +-
 .../arm64-neon-vector-shuffle-extract.ll      |    26 +
 .../CodeGen/AArch64/arm64-preserve-most.ll    |    38 +
 llvm/test/CodeGen/AArch64/arm64-rev.ll        |   243 +-
 .../CodeGen/AArch64/arm64-shrink-wrapping.ll  |     4 +-
 llvm/test/CodeGen/AArch64/arm64-st1.ll        |     4 +-
 llvm/test/CodeGen/AArch64/arm64-stacksave.ll  |     4 +-
 .../CodeGen/AArch64/arm64-storebytesmerge.ll  |     4 +-
 llvm/test/CodeGen/AArch64/arm64-tls-darwin.ll |     1 +
 .../CodeGen/AArch64/arm64-tls-initial-exec.ll |    51 +
 .../CodeGen/AArch64/arm64-tls-local-exec.ll   |   106 +
 .../AArch64/arm64-triv-disjoint-mem-access.ll |     2 +-
 .../CodeGen/AArch64/arm64-variadic-aapcs.ll   |     9 +-
 llvm/test/CodeGen/AArch64/arm64-vcvt_f.ll     |   145 +
 llvm/test/CodeGen/AArch64/arm64-vmul.ll       |  1440 +-
 llvm/test/CodeGen/AArch64/arm64-vshift.ll     |   209 +-
 .../AArch64/arm64-zero-cycle-zeroing.ll       |     2 +-
 llvm/test/CodeGen/AArch64/arm64_32-addrs.ll   |    44 +
 llvm/test/CodeGen/AArch64/arm64_32-atomics.ll |   261 +
 .../test/CodeGen/AArch64/arm64_32-fastisel.ll |    49 +
 .../AArch64/arm64_32-frame-pointers.ll        |    26 +
 .../test/CodeGen/AArch64/arm64_32-gep-sink.ll |    61 +
 llvm/test/CodeGen/AArch64/arm64_32-memcpy.ll  |    66 +
 llvm/test/CodeGen/AArch64/arm64_32-neon.ll    |   198 +
 llvm/test/CodeGen/AArch64/arm64_32-null.ll    |    28 +
 .../AArch64/arm64_32-pointer-extend.ll        |    49 +
 .../AArch64/arm64_32-stack-pointers.ll        |    13 +
 llvm/test/CodeGen/AArch64/arm64_32-tls.ll     |    22 +
 llvm/test/CodeGen/AArch64/arm64_32-va.ll      |    56 +
 llvm/test/CodeGen/AArch64/arm64_32.ll         |   730 +
 llvm/test/CodeGen/AArch64/basic-pic.ll        |    10 +-
 llvm/test/CodeGen/AArch64/bitfield-insert.ll  |    12 +-
 ...ranch-target-enforcement-indirect-calls.ll |     3 +
 .../CodeGen/AArch64/bti-branch-relaxation.ll  |    64 +
 .../CodeGen/AArch64/callbr-asm-obj-file.ll    |    30 +-
 llvm/test/CodeGen/AArch64/cfguard-checks.ll   |   147 +
 .../CodeGen/AArch64/cfguard-module-flag.ll    |    25 +
 llvm/test/CodeGen/AArch64/cgp-usubo.ll        |    12 +-
 llvm/test/CodeGen/AArch64/cls.ll              |    20 +
 llvm/test/CodeGen/AArch64/cmpxchg-O0.ll       |     6 +-
 .../CodeGen/AArch64/code-model-large-abs.ll   |    11 +
 llvm/test/CodeGen/AArch64/consthoist-gep.ll   |     2 +-
 llvm/test/CodeGen/AArch64/cpus.ll             |     6 +-
 llvm/test/CodeGen/AArch64/csr-split.ll        |   224 +
 llvm/test/CodeGen/AArch64/cxx-tlscc.ll        |     2 +-
 .../CodeGen/AArch64/dag-combine-invaraints.ll |     4 +-
 .../CodeGen/AArch64/dbg-declare-tag-offset.ll |    19 +-
 .../CodeGen/AArch64/dbg-value-tag-offset.ll   |    68 +
 .../div-rem-pair-recomposition-signed.ll      |   319 +
 .../div-rem-pair-recomposition-unsigned.ll    |   319 +
 llvm/test/CodeGen/AArch64/dllimport.ll        |     4 +-
 llvm/test/CodeGen/AArch64/eon.ll              |     3 +
 llvm/test/CodeGen/AArch64/expand-select.ll    |    40 +-
 llvm/test/CodeGen/AArch64/f16-instructions.ll |     8 +-
 llvm/test/CodeGen/AArch64/fadd-combines.ll    |    51 +-
 .../AArch64/fast-isel-branch-uncond-debug.ll  |    44 +
 .../CodeGen/AArch64/fast-isel-sp-adjust.ll    |     1 +
 llvm/test/CodeGen/AArch64/fastcc-reserved.ll  |     9 +-
 llvm/test/CodeGen/AArch64/fastcc.ll           |    28 +-
 llvm/test/CodeGen/AArch64/fp-intrinsics.ll    |  1568 +++
 llvm/test/CodeGen/AArch64/fp16-fmla.ll        |   220 +
 .../CodeGen/AArch64/fp16_intrinsic_lane.ll    |     8 +-
 .../AArch64/fp16_intrinsic_scalar_3op.ll      |    85 +-
 .../fpconv-vector-op-scalarize-strict.ll      |    33 +
 .../AArch64/global-merge-group-by-use.ll      |     4 +-
 .../AArch64/global-merge-hidden-minsize.ll    |    25 +
 ...st-and-by-const-from-lshr-in-eqcmp-zero.ll |    92 +-
 ...ist-and-by-const-from-shl-in-eqcmp-zero.ll |    82 +-
 .../CodeGen/AArch64/hwasan-check-memaccess.ll |    50 +-
 .../AArch64/i128_volatile_load_store.ll       |   117 +
 .../test/CodeGen/AArch64/illegal-float-ops.ll |    72 +-
 llvm/test/CodeGen/AArch64/irg_sp_tagp.ll      |     2 +-
 llvm/test/CodeGen/AArch64/jump-table-32.ll    |    42 +
 .../test/CodeGen/AArch64/jump-table-exynos.ll |     2 -
 llvm/test/CodeGen/AArch64/large-stack.ll      |    49 +
 .../CodeGen/AArch64/ldst-paired-aliasing.ll   |    31 +-
 .../AArch64/load-combine-big-endian.ll        |   158 +-
 llvm/test/CodeGen/AArch64/load-combine.ll     |   142 +-
 .../AArch64/loopvectorize_pr33804_double.ll   |     2 +-
 llvm/test/CodeGen/AArch64/lower-ptrmask.ll    |    29 +
 .../CodeGen/AArch64/machine-combiner-madd.ll  |     2 -
 .../AArch64/machine-outliner-remarks.ll       |     2 +-
 .../machine-outliner-retaddr-sign-cfi.ll      |    91 +
 ...tliner-retaddr-sign-diff-scope-same-key.ll |    77 +
 .../machine-outliner-retaddr-sign-non-leaf.ll |    79 +
 ...tliner-retaddr-sign-same-scope-diff-key.ll |    78 +
 ...iner-retaddr-sign-same-scope-same-key-a.ll |    75 +
 ...iner-retaddr-sign-same-scope-same-key-b.ll |    81 +
 .../machine-outliner-retaddr-sign-sp-mod.ll   |    44 +
 ...machine-outliner-retaddr-sign-subtarget.ll |    87 +
 .../machine-outliner-retaddr-sign-thunk.ll    |    76 +
 .../machine-outliner-retaddr-sign-v8-3.ll     |    83 +
 llvm/test/CodeGen/AArch64/machine-outliner.ll |    29 +-
 llvm/test/CodeGen/AArch64/macro-fusion.ll     |    20 +
 llvm/test/CodeGen/AArch64/max-jump-table.ll   |   145 +-
 .../CodeGen/AArch64/merge-store-dependency.ll |    46 +-
 llvm/test/CodeGen/AArch64/merge-store.ll      |    20 +-
 .../CodeGen/AArch64/misched-fusion-aes.ll     |     2 -
 llvm/test/CodeGen/AArch64/neon-dot-product.ll |   148 +-
 llvm/test/CodeGen/AArch64/neon-mla-mls.ll     |   145 +-
 llvm/test/CodeGen/AArch64/neon-vcadd.ll       |    67 +
 llvm/test/CodeGen/AArch64/no-quad-ldp-stp.ll  |     1 -
 llvm/test/CodeGen/AArch64/no_cfi.ll           |    13 +
 .../AArch64/note-gnu-property-pac-bti-0.ll    |    14 +
 .../AArch64/note-gnu-property-pac-bti-1.ll    |    18 +
 .../AArch64/note-gnu-property-pac-bti-2.ll    |    18 +
 .../AArch64/note-gnu-property-pac-bti-3.ll    |    18 +
 .../AArch64/note-gnu-property-pac-bti-4.ll    |    25 +
 .../AArch64/note-gnu-property-pac-bti-5.ll    |    26 +
 .../AArch64/note-gnu-property-pac-bti-6.ll    |    22 +
 .../AArch64/note-gnu-property-pac-bti-7.ll    |    23 +
 .../AArch64/note-gnu-property-pac-bti-8.ll    |    21 +
 .../CodeGen/AArch64/overeager_mla_fusing.ll   |    59 +
 .../AArch64/patchable-function-entry-bti.ll   |    86 +
 .../AArch64/patchable-function-entry.ll       |   105 +
 llvm/test/CodeGen/AArch64/powi-windows.ll     |    46 +
 .../CodeGen/AArch64/preferred-alignment.ll    |     4 +-
 .../AArch64/preferred-function-alignment.ll   |    16 +-
 .../AArch64/ragreedy-local-interval-cost.ll   |   339 +
 .../AArch64/regress-w29-reserved-with-fp.ll   |     2 +-
 llvm/test/CodeGen/AArch64/remat.ll            |    10 +-
 llvm/test/CodeGen/AArch64/sadd_sat.ll         |    73 +-
 llvm/test/CodeGen/AArch64/sadd_sat_plus.ll    |    94 +
 llvm/test/CodeGen/AArch64/sadd_sat_vec.ll     |   621 +-
 .../CodeGen/AArch64/sched-past-vector-ldst.ll |    26 +-
 llvm/test/CodeGen/AArch64/sdivpow2.ll         |    19 +
 llvm/test/CodeGen/AArch64/seh-finally.ll      |     8 +-
 llvm/test/CodeGen/AArch64/select_const.ll     |   625 +
 .../CodeGen/AArch64/selectcc-to-shiftand.ll   |    95 +
 .../test/CodeGen/AArch64/shadow-call-stack.ll |     2 +-
 llvm/test/CodeGen/AArch64/shift-amount-mod.ll |    12 +
 llvm/test/CodeGen/AArch64/shift-by-signext.ll |   122 +
 llvm/test/CodeGen/AArch64/shift-logic.ll      |   153 +
 llvm/test/CodeGen/AArch64/shift-mod.ll        |    77 +
 llvm/test/CodeGen/AArch64/shift_minsize.ll    |     5 +
 llvm/test/CodeGen/AArch64/sibling-call.ll     |    24 +-
 .../CodeGen/AArch64/sign-return-address.ll    |    75 +-
 .../AArch64/sink-copy-for-shrink-wrap.ll      |     2 +-
 llvm/test/CodeGen/AArch64/space.ll            |    16 +
 .../AArch64/speculation-hardening-dagisel.ll  |     2 +-
 .../AArch64/speculation-hardening-loads.ll    |     2 +-
 .../CodeGen/AArch64/speculation-hardening.ll  |    37 +-
 llvm/test/CodeGen/AArch64/sponentry.ll        |     4 +-
 llvm/test/CodeGen/AArch64/srem-lkk.ll         |   149 +
 .../CodeGen/AArch64/srem-seteq-optsize.ll     |    39 +
 .../AArch64/srem-seteq-vec-nonsplat.ll        |   847 ++
 .../CodeGen/AArch64/srem-seteq-vec-splat.ll   |   226 +
 llvm/test/CodeGen/AArch64/srem-seteq.ll       |   286 +
 llvm/test/CodeGen/AArch64/srem-vector-lkk.ll  |   324 +
 llvm/test/CodeGen/AArch64/ssub_sat.ll         |    74 +-
 llvm/test/CodeGen/AArch64/ssub_sat_plus.ll    |    94 +
 llvm/test/CodeGen/AArch64/ssub_sat_vec.ll     |   662 +-
 .../CodeGen/AArch64/stack-guard-reassign.ll   |     7 +-
 .../test/CodeGen/AArch64/stack-guard-vaarg.ll |     2 +-
 .../CodeGen/AArch64/stack-tagging-ex-1.ll     |    69 +
 .../CodeGen/AArch64/stack-tagging-ex-2.ll     |   183 +
 .../stack-tagging-initializer-merge.ll        |   308 +
 .../AArch64/stack-tagging-unchecked-ld-st.ll  |   260 +
 .../AArch64/stack-tagging-untag-placement.ll  |    82 +
 .../test/CodeGen/AArch64/stack_guard_remat.ll |     2 +-
 llvm/test/CodeGen/AArch64/stackmap.ll         |   492 +
 llvm/test/CodeGen/AArch64/stgp.ll             |     2 +-
 .../CodeGen/AArch64/strict-fp-int-promote.ll  |    67 +
 llvm/test/CodeGen/AArch64/strqu.ll            |     4 +-
 llvm/test/CodeGen/AArch64/sub-of-bias.ll      |   103 +
 .../CodeGen/AArch64/sve-alloca-stackid.ll     |    17 +
 .../CodeGen/AArch64/sve-calling-convention.ll |   121 +
 llvm/test/CodeGen/AArch64/sve-fp.ll           |   129 +
 .../AArch64/sve-gather-scatter-dag-combine.ll |    72 +
 .../test/CodeGen/AArch64/sve-int-arith-imm.ll |   448 +
 .../CodeGen/AArch64/sve-int-arith-pred.ll     |   406 +
 llvm/test/CodeGen/AArch64/sve-int-arith.ll    |   216 +
 llvm/test/CodeGen/AArch64/sve-int-div-pred.ll |    91 +
 llvm/test/CodeGen/AArch64/sve-int-imm.ll      |   519 +
 llvm/test/CodeGen/AArch64/sve-int-log-imm.ll  |   128 +
 llvm/test/CodeGen/AArch64/sve-int-log-pred.ll |   179 +
 llvm/test/CodeGen/AArch64/sve-int-log.ll      |    96 +
 llvm/test/CodeGen/AArch64/sve-int-mad-pred.ll |   199 +
 llvm/test/CodeGen/AArch64/sve-int-mul-pred.ll |   134 +
 .../CodeGen/AArch64/sve-int-reduce-pred.ll    |   400 +
 .../AArch64/sve-intrinsics-conversion.ll      |   159 +
 .../AArch64/sve-intrinsics-counting-bits.ll   |   181 +
 .../AArch64/sve-intrinsics-counting-elems.ll  |   171 +
 .../AArch64/sve-intrinsics-fp-arith.ll        |  1681 +++
 .../AArch64/sve-intrinsics-fp-compares.ll     |   267 +
 .../AArch64/sve-intrinsics-fp-converts.ll     |   400 +
 .../AArch64/sve-intrinsics-fp-reduce.ll       |   214 +
 ...nsics-gather-loads-32bit-scaled-offsets.ll |   255 +
 ...ics-gather-loads-32bit-unscaled-offsets.ll |   348 +
 ...insics-gather-loads-64bit-scaled-offset.ll |    80 +
 ...sics-gather-loads-64bit-unscaled-offset.ll |   103 +
 ...sve-intrinsics-gather-loads-vector-base.ll |   186 +
 .../AArch64/sve-intrinsics-int-arith.ll       |   192 +
 .../sve-intrinsics-int-compares-with-imm.ll   |  1353 ++
 .../AArch64/sve-intrinsics-int-compares.ll    |   645 +
 .../CodeGen/AArch64/sve-intrinsics-loads.ll   |    88 +
 .../CodeGen/AArch64/sve-intrinsics-logical.ll |    99 +
 .../AArch64/sve-intrinsics-perm-select.ll     |  1765 +++
 .../AArch64/sve-intrinsics-pred-creation.ll   |    42 +
 .../AArch64/sve-intrinsics-pred-operations.ll |   130 +
 .../AArch64/sve-intrinsics-pred-testing.ll    |    36 +
 .../AArch64/sve-intrinsics-reversal.ll        |   166 +
 ...ics-scatter-stores-32bit-scaled-offsets.ll |   193 +
 ...s-scatter-stores-32bit-unscaled-offsets.ll |   248 +
 ...sics-scatter-stores-64bit-scaled-offset.ll |    58 +
 ...cs-scatter-stores-64bit-unscaled-offset.ll |    70 +
 ...e-intrinsics-scatter-stores-vector-base.ll |   133 +
 .../CodeGen/AArch64/sve-intrinsics-shifts.ll  |   367 +
 .../CodeGen/AArch64/sve-intrinsics-sqdec.ll   |   337 +
 .../CodeGen/AArch64/sve-intrinsics-sqinc.ll   |   337 +
 .../CodeGen/AArch64/sve-intrinsics-stores.ll  |    95 +
 .../CodeGen/AArch64/sve-intrinsics-uqdec.ll   |   257 +
 .../CodeGen/AArch64/sve-intrinsics-uqinc.ll   |   257 +
 .../CodeGen/AArch64/sve-intrinsics-while.ll   |   309 +
 .../CodeGen/AArch64/sve-masked-ldst-nonext.ll |   193 +
 .../CodeGen/AArch64/sve-masked-ldst-sext.ll   |    66 +
 .../CodeGen/AArch64/sve-masked-ldst-trunc.ll  |    66 +
 .../CodeGen/AArch64/sve-masked-ldst-zext.ll   |    72 +
 .../AArch64/sve-neg-int-arith-imm-2.ll        |    12 +
 .../CodeGen/AArch64/sve-neg-int-arith-imm.ll  |    11 +
 llvm/test/CodeGen/AArch64/sve-pred-log.ll     |   286 +
 llvm/test/CodeGen/AArch64/sve-select.ll       |    85 +
 llvm/test/CodeGen/AArch64/sve-setcc.ll        |    26 +
 llvm/test/CodeGen/AArch64/sve-vector-splat.ll |   135 +
 ...ve2-intrinsics-binary-narrowing-add-sub.ll |   278 +
 .../sve2-intrinsics-binary-narrowing-shr.ll   |   512 +
 .../AArch64/sve2-intrinsics-fp-converts.ll    |    84 +
 ...sve2-intrinsics-fp-int-binary-logarithm.ll |    39 +
 .../sve2-intrinsics-fp-widening-mul-acc.ll    |   127 +
 ...-intrinsics-non-widening-pairwise-arith.ll |   191 +
 .../sve2-intrinsics-unary-narrowing.ll        |   202 +
 .../CodeGen/AArch64/sve2-intrinsics-while.ll  |   309 +
 llvm/test/CodeGen/AArch64/swift-return.ll     |     4 +-
 llvm/test/CodeGen/AArch64/swiftcc.ll          |     2 +
 llvm/test/CodeGen/AArch64/swifterror.ll       |   131 +-
 llvm/test/CodeGen/AArch64/swiftself.ll        |    29 +-
 llvm/test/CodeGen/AArch64/tagged-globals.ll   |    32 +
 llvm/test/CodeGen/AArch64/tail-call.ll        |   120 +-
 .../AArch64/tailcall-bitcast-memcpy.ll        |    18 +
 .../CodeGen/AArch64/tailcall-ccmismatch.ll    |     1 +
 .../AArch64/tailcall-mem-intrinsics.ll        |     1 +
 .../CodeGen/AArch64/tailcall-string-rvo.ll    |     1 +
 .../CodeGen/AArch64/tailcall_misched_graph.ll |    46 +-
 llvm/test/CodeGen/AArch64/tbz-tbnz.ll         |     2 +-
 llvm/test/CodeGen/AArch64/tme.ll              |    44 +
 llvm/test/CodeGen/AArch64/uadd_sat.ll         |    44 +-
 llvm/test/CodeGen/AArch64/uadd_sat_plus.ll    |    77 +
 llvm/test/CodeGen/AArch64/uadd_sat_vec.ll     |   226 +-
 .../umulo-128-legalisation-lowering.ll        |     4 +-
 ...asked-merge-scalar-constmask-innerouter.ll |    16 +-
 ...-merge-scalar-constmask-interleavedbits.ll |    16 +-
 ...-scalar-constmask-interleavedbytehalves.ll |    16 +-
 ...d-masked-merge-scalar-constmask-lowhigh.ll |    16 +-
 ...unfold-masked-merge-scalar-variablemask.ll |    16 +-
 llvm/test/CodeGen/AArch64/urem-lkk.ll         |   103 +
 .../CodeGen/AArch64/urem-seteq-nonzero.ll     |   243 +
 .../CodeGen/AArch64/urem-seteq-optsize.ll     |     3 -
 .../AArch64/urem-seteq-vec-nonsplat.ll        |   278 +-
 .../CodeGen/AArch64/urem-seteq-vec-nonzero.ll |   115 +
 .../CodeGen/AArch64/urem-seteq-vec-splat.ll   |   107 +-
 .../AArch64/urem-seteq-vec-tautological.ll    |    97 +
 llvm/test/CodeGen/AArch64/urem-seteq.ll       |    79 +-
 llvm/test/CodeGen/AArch64/urem-vector-lkk.ll  |   267 +
 .../AArch64/use-cr-result-of-dom-icmp-st.ll   |   547 +
 llvm/test/CodeGen/AArch64/usub_sat.ll         |    43 +-
 llvm/test/CodeGen/AArch64/usub_sat_plus.ll    |    76 +
 llvm/test/CodeGen/AArch64/usub_sat_vec.ll     |   136 +-
 llvm/test/CodeGen/AArch64/vararg-tallcall.ll  |     2 +
 .../AArch64/vecreduce-and-legalization.ll     |    18 +-
 .../vecreduce-fadd-legalization-strict.ll     |   128 +
 .../vecreduce-fmax-legalization-nan.ll        |    88 +
 .../AArch64/vecreduce-fmax-legalization.ll    |     2 +-
 .../vecreduce-fmul-legalization-strict.ll     |   114 +
 .../CodeGen/AArch64/vector_merge_dep_check.ll |     2 +-
 .../test/CodeGen/AArch64/vselect-constants.ll |   195 +
 llvm/test/CodeGen/AArch64/win64-no-uwtable.ll |    35 +
 llvm/test/CodeGen/AArch64/win64_vararg.ll     |    74 +-
 .../CodeGen/AArch64/windows-extern-weak.ll    |    27 +
 .../AArch64/wineh-try-catch-realign.ll        |    12 +-
 llvm/test/CodeGen/AArch64/wineh-try-catch.ll  |    36 +-
 .../AMDGPU/GlobalISel/bool-legalization.ll    |   105 +
 .../GlobalISel/divergent-control-flow.ll      |    61 +
 .../AMDGPU/GlobalISel/extractelement.ll       |  1289 ++
 .../AMDGPU/GlobalISel/function-returns.ll     |  1063 ++
 .../CodeGen/AMDGPU/GlobalISel/global-value.ll |   156 +
 .../GlobalISel/irtranslator-amdgcn-sendmsg.ll |    15 +
 .../GlobalISel/irtranslator-amdgpu_kernel.ll  |   380 +-
 .../GlobalISel/irtranslator-amdgpu_ps.ll      |    31 +-
 .../GlobalISel/irtranslator-amdgpu_vs.ll      |    25 +-
 .../GlobalISel/irtranslator-atomicrmw.ll      |    48 +
 .../GlobalISel/irtranslator-function-args.ll  |  2227 +++
 .../irtranslator-struct-return-intrinsics.ll  |     5 +-
 .../GlobalISel/lds-global-non-entry-func.ll   |    13 +
 .../AMDGPU/GlobalISel/lds-global-value.ll     |    35 +
 .../CodeGen/AMDGPU/GlobalISel/lds-size.ll     |     1 +
 .../AMDGPU/GlobalISel/lds-zero-initializer.ll |     5 +
 .../llvm.amdgcn.ds.ordered.add.gfx10.ll       |     1 +
 .../GlobalISel/llvm.amdgcn.ds.ordered.add.ll  |     5 +
 .../GlobalISel/llvm.amdgcn.ds.ordered.swap.ll |     5 +
 .../GlobalISel/llvm.amdgcn.end.cf.i32.ll      |     3 +
 .../GlobalISel/llvm.amdgcn.end.cf.i64.ll      |     3 +
 .../GlobalISel/llvm.amdgcn.if.break.i32.ll    |     3 +-
 .../GlobalISel/llvm.amdgcn.if.break.i64.ll    |     6 +-
 .../GlobalISel/llvm.amdgcn.init.exec.ll       |     2 +
 .../llvm.amdgcn.init.exec.wave32.ll           |     2 +
 .../GlobalISel/llvm.amdgcn.is.private.ll      |   109 +
 .../GlobalISel/llvm.amdgcn.is.shared.ll       |   109 +
 .../llvm.amdgcn.kernarg.segment.ptr.ll        |     2 +-
 ...llvm.amdgcn.raw.buffer.store.format.f16.ll |   504 +
 ...llvm.amdgcn.raw.buffer.store.format.f32.ll |   306 +
 .../llvm.amdgcn.raw.buffer.store.ll           |   776 ++
 .../AMDGPU/GlobalISel/llvm.amdgcn.s.sleep.ll  |    45 +
 .../GlobalISel/llvm.amdgcn.workgroup.id.ll    |     4 +-
 .../GlobalISel/llvm.amdgcn.workitem.id.ll     |    31 +-
 .../AMDGPU/GlobalISel/llvm.amdgcn.wqm.vote.ll |     3 +
 .../AMDGPU/GlobalISel/read_register.ll        |     2 +
 .../AMDGPU/GlobalISel/readcyclecounter.ll     |     3 +
 .../regbankselect-amdgcn.image.load.1d.ll     |   181 +
 .../regbankselect-amdgcn.image.sample.1d.ll   |   268 +
 .../regbankselect-amdgcn.raw.buffer.load.ll   |   173 +
 ...regbankselect-amdgcn.struct.buffer.load.ll |   179 +
 ...egbankselect-amdgcn.struct.buffer.store.ll |   174 +
 llvm/test/CodeGen/AMDGPU/GlobalISel/ret.ll    |     3 +
 llvm/test/CodeGen/AMDGPU/add.i16.ll           |     4 +-
 .../AMDGPU/addrspacecast-constantexpr.ll      |     2 +-
 .../CodeGen/AMDGPU/agpr-register-count.ll     |   139 +-
 .../CodeGen/AMDGPU/amdgcn.private-memory.ll   |     9 +-
 .../AMDGPU/amdgpu-codegenprepare-idiv.ll      |   104 +-
 .../AMDGPU/amdgpu-codegenprepare-mul24.ll     |   219 +-
 llvm/test/CodeGen/AMDGPU/amdgpu-inline.ll     |     7 +
 .../CodeGen/AMDGPU/amdgpu-mul24-knownbits.ll  |    34 +
 .../CodeGen/AMDGPU/amdgpu-unroll-threshold.ll |    52 +
 .../CodeGen/AMDGPU/amdgpu.private-memory.ll   |     5 +-
 .../AMDGPU/annotate-kernel-features-hsa.ll    |    19 +
 .../test/CodeGen/AMDGPU/array-ptr-calc-i32.ll |     2 +-
 .../AMDGPU/atomic_optimizations_buffer.ll     |    92 +-
 .../atomic_optimizations_global_pointer.ll    |   118 +-
 .../atomic_optimizations_local_pointer.ll     |  5120 ++++++-
 .../atomic_optimizations_pixelshader.ll       |    36 +-
 .../AMDGPU/atomic_optimizations_raw_buffer.ll |    70 +-
 .../atomic_optimizations_struct_buffer.ll     |    70 +-
 llvm/test/CodeGen/AMDGPU/atomicrmw-nand.ll    |    12 +-
 .../attr-amdgpu-flat-work-group-size-v3.ll    |    18 +-
 .../attr-amdgpu-flat-work-group-size.ll       |    18 +-
 .../CodeGen/AMDGPU/attr-amdgpu-num-sgpr.ll    |    17 +-
 llvm/test/CodeGen/AMDGPU/basic-branch.ll      |     4 +-
 .../AMDGPU/bitcast-constant-to-vector.ll      |     4 +-
 llvm/test/CodeGen/AMDGPU/bitreverse.ll        |   329 +-
 .../test/CodeGen/AMDGPU/branch-relax-spill.ll |     6 +-
 llvm/test/CodeGen/AMDGPU/branch-relaxation.ll |     3 +-
 llvm/test/CodeGen/AMDGPU/branch-uniformity.ll |     4 +-
 llvm/test/CodeGen/AMDGPU/bswap.ll             |   197 +-
 .../AMDGPU/buffer-intrinsics-mmo-offsets.ll   |   414 +
 llvm/test/CodeGen/AMDGPU/byval-frame-setup.ll |     3 +-
 .../CodeGen/AMDGPU/call-argument-types.ll     |    60 +-
 llvm/test/CodeGen/AMDGPU/call-constant.ll     |    45 +
 .../AMDGPU/call-graph-register-usage.ll       |    28 +
 .../AMDGPU/callee-special-input-vgprs.ll      |    86 +-
 .../CodeGen/AMDGPU/calling-conventions.ll     |    31 +
 llvm/test/CodeGen/AMDGPU/cc-sgpr-limit.ll     |   138 +
 .../test/CodeGen/AMDGPU/cc-sgpr-over-limit.ll |   101 +
 .../CodeGen/AMDGPU/cgp-addressing-modes.ll    |     5 +-
 llvm/test/CodeGen/AMDGPU/chain-hi-to-lo.ll    |   260 +-
 .../CodeGen/AMDGPU/combine-cond-add-sub.ll    |   136 +-
 llvm/test/CodeGen/AMDGPU/commute-shifts.ll    |    16 +-
 .../CodeGen/AMDGPU/computeNumSignBits-mul.ll  |   172 +
 .../AMDGPU/control-flow-fastregalloc.ll       |     7 +-
 .../CodeGen/AMDGPU/control-flow-optnone.ll    |     4 +-
 llvm/test/CodeGen/AMDGPU/copy-illegal-type.ll |   612 +-
 .../AMDGPU/cross-block-use-is-not-abi-copy.ll |    89 +
 .../CodeGen/AMDGPU/cse-phi-incoming-val.ll    |    42 +
 llvm/test/CodeGen/AMDGPU/ctpop.ll             |     2 +-
 llvm/test/CodeGen/AMDGPU/ctpop16.ll           |     2 +-
 llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll     |    75 +-
 .../CodeGen/AMDGPU/dagcombine-setcc-select.ll |    73 +-
 .../dead-machine-elim-after-dead-lane.ll      |    28 +
 llvm/test/CodeGen/AMDGPU/debug-value2.ll      |     8 +-
 llvm/test/CodeGen/AMDGPU/div_i128.ll          |     2 +-
 llvm/test/CodeGen/AMDGPU/divergence-at-use.ll |    20 +
 .../divergent-branch-uniform-condition.ll     |    36 +-
 llvm/test/CodeGen/AMDGPU/divrem24-assume.ll   |     2 +-
 llvm/test/CodeGen/AMDGPU/dpp_combine.ll       |    53 +
 ...ds-negative-offset-addressing-mode-loop.ll |     6 +-
 llvm/test/CodeGen/AMDGPU/ds_read2.ll          |     2 +-
 .../AMDGPU/enable-no-signed-zeros-fp-math.ll  |    26 +-
 .../AMDGPU/extract-subvector-equal-length.ll  |     4 +-
 llvm/test/CodeGen/AMDGPU/extract-subvector.ll |    40 +
 .../AMDGPU/extract_subvector_vec4_vec3.ll     |    21 +-
 .../CodeGen/AMDGPU/extract_vector_dynelt.ll   |     4 +-
 llvm/test/CodeGen/AMDGPU/fabs.ll              |    12 +-
 .../CodeGen/AMDGPU/fadd-fma-fmul-combine.ll   |   118 +-
 llvm/test/CodeGen/AMDGPU/fadd.f16.ll          |     2 +-
 llvm/test/CodeGen/AMDGPU/fdiv.ll              |    77 +-
 .../CodeGen/AMDGPU/fdiv32-to-rcp-folding.ll   |    58 +-
 llvm/test/CodeGen/AMDGPU/fence-barrier.ll     |     3 +-
 llvm/test/CodeGen/AMDGPU/fexp.ll              |     1 -
 llvm/test/CodeGen/AMDGPU/ffloor.f64.ll        |    28 +-
 .../test/CodeGen/AMDGPU/flat-address-space.ll |    27 +-
 llvm/test/CodeGen/AMDGPU/fma.f64.ll           |   155 +-
 llvm/test/CodeGen/AMDGPU/fmac.sdwa.ll         |   138 +-
 llvm/test/CodeGen/AMDGPU/fmin_legacy.ll       |     8 +-
 .../AMDGPU/fmul-2-combine-multi-use.ll        |    12 +-
 llvm/test/CodeGen/AMDGPU/fmuladd.f16.ll       |     6 +-
 llvm/test/CodeGen/AMDGPU/fneg-combines.ll     |    15 +-
 llvm/test/CodeGen/AMDGPU/fneg-fabs.ll         |    16 +-
 .../fneg-fold-legalize-dag-increase-insts.ll  |    24 +
 llvm/test/CodeGen/AMDGPU/fneg.ll              |    29 +-
 llvm/test/CodeGen/AMDGPU/fpow.ll              |   562 +
 llvm/test/CodeGen/AMDGPU/fptrunc.ll           |    10 +
 .../CodeGen/AMDGPU/frame-index-elimination.ll |    44 +-
 llvm/test/CodeGen/AMDGPU/fsqrt.ll             |    38 +-
 llvm/test/CodeGen/AMDGPU/fsub.ll              |    12 +-
 llvm/test/CodeGen/AMDGPU/function-returns.ll  |    20 +
 llvm/test/CodeGen/AMDGPU/global-atomics-fp.ll |    29 +
 llvm/test/CodeGen/AMDGPU/global-constant.ll   |    48 +-
 llvm/test/CodeGen/AMDGPU/global-saddr.ll      |     2 +-
 llvm/test/CodeGen/AMDGPU/global_atomics.ll    |     9 +-
 .../test/CodeGen/AMDGPU/global_atomics_i64.ll |     4 +-
 llvm/test/CodeGen/AMDGPU/global_smrd.ll       |     2 +-
 .../hsa-metadata-from-llvm-ir-full-v3.ll      |     1 +
 .../AMDGPU/hsa-metadata-from-llvm-ir-full.ll  |     1 +
 .../AMDGPU/hsa-metadata-hostcall-absent-v3.ll |    55 +
 .../AMDGPU/hsa-metadata-hostcall-absent.ll    |    52 +
 .../hsa-metadata-hostcall-present-v3.ll       |    59 +
 .../AMDGPU/hsa-metadata-hostcall-present.ll   |    58 +
 .../hsa-metadata-kernel-code-props-v3.ll      |    19 +-
 .../AMDGPU/hsa-metadata-kernel-code-props.ll  |    26 +-
 llvm/test/CodeGen/AMDGPU/i1-copy-from-loop.ll |    27 +-
 .../AMDGPU/i1-copy-phi-uniform-branch.ll      |     1 -
 llvm/test/CodeGen/AMDGPU/idiv-licm.ll         |   439 +-
 llvm/test/CodeGen/AMDGPU/idot2.ll             |   938 +-
 llvm/test/CodeGen/AMDGPU/idot4s.ll            |   700 +-
 llvm/test/CodeGen/AMDGPU/idot4u.ll            |  1032 +-
 llvm/test/CodeGen/AMDGPU/idot8s.ll            |  2368 ++--
 llvm/test/CodeGen/AMDGPU/idot8u.ll            |  3019 ++--
 .../AMDGPU/illegal-sgpr-to-vgpr-copy.ll       |     9 +-
 llvm/test/CodeGen/AMDGPU/implicit-def-muse.ll |     7 +-
 .../AMDGPU/indirect-addressing-si-noopt.ll    |     4 -
 .../AMDGPU/indirect-addressing-term.ll        |    18 +-
 llvm/test/CodeGen/AMDGPU/inline-asm.ll        |    23 +-
 .../test/CodeGen/AMDGPU/inline-constraints.ll |     5 +-
 .../AMDGPU/insert-subvector-unused-scratch.ll |     4 +-
 llvm/test/CodeGen/AMDGPU/insert_vector_elt.ll |  1738 ++-
 .../CodeGen/AMDGPU/insert_vector_elt.v2i16.ll |  1894 ++-
 llvm/test/CodeGen/AMDGPU/kernel-args.ll       |     2 +-
 .../AMDGPU/kernel-argument-dag-lowering.ll    |     2 +-
 .../test/CodeGen/AMDGPU/kill-infinite-loop.ll |    68 +
 .../AMDGPU/large-work-group-promote-alloca.ll |     7 +-
 llvm/test/CodeGen/AMDGPU/lcssa-optnone.ll     |    26 +
 .../CodeGen/AMDGPU/llvm.amdgcn.cvt.pkrtz.ll   |   524 +-
 .../CodeGen/AMDGPU/llvm.amdgcn.div.scale.ll   |     2 +-
 .../AMDGPU/llvm.amdgcn.ds.gws.barrier.ll      |    91 +-
 .../CodeGen/AMDGPU/llvm.amdgcn.ds.gws.init.ll |    48 +-
 .../AMDGPU/llvm.amdgcn.ds.gws.sema.br.ll      |     7 +-
 .../AMDGPU/llvm.amdgcn.ds.gws.sema.p.ll       |     7 +-
 .../llvm.amdgcn.ds.gws.sema.release.all.ll    |     7 +-
 .../AMDGPU/llvm.amdgcn.ds.gws.sema.v.ll       |     7 +-
 .../AMDGPU/llvm.amdgcn.ds.ordered.swap.ll     |     4 +-
 llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fmed3.ll |     8 +-
 .../CodeGen/AMDGPU/llvm.amdgcn.init.exec.ll   |     7 +-
 .../AMDGPU/llvm.amdgcn.init.exec.wave32.ll    |    31 +
 .../CodeGen/AMDGPU/llvm.amdgcn.interp.f16.ll  |     8 +-
 .../CodeGen/AMDGPU/llvm.amdgcn.is.private.ll  |    50 +
 .../CodeGen/AMDGPU/llvm.amdgcn.is.shared.ll   |    51 +
 llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll  |   118 +-
 .../CodeGen/AMDGPU/llvm.amdgcn.mov.dpp.ll     |    25 +
 .../AMDGPU/llvm.amdgcn.mqsad.pk.u16.u8.ll     |     2 +-
 .../AMDGPU/llvm.amdgcn.qsad.pk.u16.u8.ll      |     2 +-
 .../AMDGPU/llvm.amdgcn.raw.buffer.atomic.ll   |    10 +-
 .../AMDGPU/llvm.amdgcn.raw.buffer.load.ll     |   109 +-
 .../AMDGPU/llvm.amdgcn.raw.buffer.store.ll    |    97 +
 .../AMDGPU/llvm.amdgcn.readfirstlane.ll       |     4 +-
 .../CodeGen/AMDGPU/llvm.amdgcn.readlane.ll    |     3 +-
 .../CodeGen/AMDGPU/llvm.amdgcn.s.barrier.ll   |    56 +-
 .../AMDGPU/llvm.amdgcn.s.buffer.load.ll       |   160 +-
 .../CodeGen/AMDGPU/llvm.amdgcn.softwqm.ll     |   188 +
 .../llvm.amdgcn.struct.buffer.atomic.ll       |    10 +-
 .../AMDGPU/llvm.amdgcn.struct.buffer.load.ll  |    79 +
 .../AMDGPU/llvm.amdgcn.struct.buffer.store.ll |    53 +-
 .../CodeGen/AMDGPU/llvm.amdgcn.update.dpp.ll  |    65 +-
 .../CodeGen/AMDGPU/llvm.amdgcn.wqm.vote.ll    |    21 +-
 .../CodeGen/AMDGPU/llvm.amdgcn.writelane.ll   |    20 +-
 llvm/test/CodeGen/AMDGPU/llvm.fma.f16.ll      |   132 +-
 llvm/test/CodeGen/AMDGPU/llvm.maxnum.f16.ll   |    27 +-
 llvm/test/CodeGen/AMDGPU/llvm.minnum.f16.ll   |    27 +-
 llvm/test/CodeGen/AMDGPU/llvm.round.f64.ll    |   762 +-
 llvm/test/CodeGen/AMDGPU/load-lo16.ll         |  2038 ++-
 .../CodeGen/AMDGPU/local-memory.amdgcn.ll     |     8 +-
 llvm/test/CodeGen/AMDGPU/loop_break.ll        |    26 +-
 .../lower-range-metadata-intrinsic-call.ll    |     2 +-
 llvm/test/CodeGen/AMDGPU/lshr.v2i16.ll        |   554 +-
 llvm/test/CodeGen/AMDGPU/mad_int24.ll         |   105 +-
 llvm/test/CodeGen/AMDGPU/madak.ll             |    16 +-
 llvm/test/CodeGen/AMDGPU/max.i16.ll           |   122 +-
 llvm/test/CodeGen/AMDGPU/max.ll               |    30 +-
 llvm/test/CodeGen/AMDGPU/memory_clause.ll     |   233 +-
 llvm/test/CodeGen/AMDGPU/mfma-loop.ll         |   491 +
 llvm/test/CodeGen/AMDGPU/min.ll               |     2 +-
 llvm/test/CodeGen/AMDGPU/movreld-bug.ll       |    18 +-
 .../CodeGen/AMDGPU/mul24-pass-ordering.ll     |   261 +
 llvm/test/CodeGen/AMDGPU/mul_uint24-amdgcn.ll |     2 +-
 llvm/test/CodeGen/AMDGPU/multilevel-break.ll  |    72 +-
 .../no-initializer-constant-addrspace.ll      |     8 +-
 llvm/test/CodeGen/AMDGPU/occupancy-levels.ll  |   288 +
 llvm/test/CodeGen/AMDGPU/offset-split-flat.ll |  1470 ++
 .../CodeGen/AMDGPU/offset-split-global.ll     |  1408 ++
 .../AMDGPU/opencl-printf-no-hostcall.ll       |    18 +
 llvm/test/CodeGen/AMDGPU/opencl-printf.ll     |    34 +
 llvm/test/CodeGen/AMDGPU/operand-folding.ll   |    24 +
 llvm/test/CodeGen/AMDGPU/packed-op-sel.ll     |     6 +-
 llvm/test/CodeGen/AMDGPU/preserve-hi16.ll     |   154 +-
 .../CodeGen/AMDGPU/private-memory-r600.ll     |     2 +-
 .../AMDGPU/promote-alloca-addrspacecast.ll    |     2 +-
 .../AMDGPU/promote-alloca-to-lds-icmp.ll      |     2 +-
 .../AMDGPU/promote-alloca-to-lds-phi.ll       |     2 +-
 .../AMDGPU/promote-alloca-to-lds-select.ll    |     2 +-
 .../AMDGPU/promote-constOffset-to-imm.ll      |   133 +-
 .../AMDGPU/r600-constant-array-fixup.ll       |     2 +-
 llvm/test/CodeGen/AMDGPU/r600-export-fix.ll   |    51 +-
 llvm/test/CodeGen/AMDGPU/read_register.ll     |    20 +-
 .../reduce-build-vec-ext-to-ext-build-vec.ll  |     6 +-
 llvm/test/CodeGen/AMDGPU/ret.ll               |    12 +-
 .../rewrite-out-arguments-address-space.ll    |     4 +-
 .../CodeGen/AMDGPU/rewrite-out-arguments.ll   |    56 +-
 llvm/test/CodeGen/AMDGPU/rsq.ll               |     2 +-
 llvm/test/CodeGen/AMDGPU/s_code_end.ll        |    37 +-
 llvm/test/CodeGen/AMDGPU/saddo.ll             |   463 +-
 llvm/test/CodeGen/AMDGPU/scalar_to_vector.ll  |   201 +-
 llvm/test/CodeGen/AMDGPU/scratch-simple.ll    |     2 +-
 llvm/test/CodeGen/AMDGPU/sdiv.ll              |    64 +-
 llvm/test/CodeGen/AMDGPU/sdwa-peephole.ll     |    31 +-
 llvm/test/CodeGen/AMDGPU/select-opt.ll        |     4 +-
 llvm/test/CodeGen/AMDGPU/select.f16.ll        |    30 +-
 llvm/test/CodeGen/AMDGPU/selectcc-opt.ll      |     2 +-
 llvm/test/CodeGen/AMDGPU/setcc-opt.ll         |    16 +-
 llvm/test/CodeGen/AMDGPU/sgpr-control-flow.ll |     3 +-
 llvm/test/CodeGen/AMDGPU/sgpr-copy.ll         |     3 +-
 llvm/test/CodeGen/AMDGPU/shift-i128.ll        |   207 +-
 llvm/test/CodeGen/AMDGPU/shl.ll               |  1712 ++-
 llvm/test/CodeGen/AMDGPU/shl.v2i16.ll         |   578 +-
 llvm/test/CodeGen/AMDGPU/shl_add_ptr.ll       |     2 +-
 .../CodeGen/AMDGPU/shrink-add-sub-constant.ll |   603 +-
 llvm/test/CodeGen/AMDGPU/si-annotate-cf.ll    |     4 +-
 llvm/test/CodeGen/AMDGPU/si-scheduler.ll      |     2 +-
 .../CodeGen/AMDGPU/si-spill-sgpr-stack.ll     |    12 +-
 llvm/test/CodeGen/AMDGPU/simplify-libcalls.ll |   268 +-
 llvm/test/CodeGen/AMDGPU/sint_to_fp.f64.ll    |    69 +-
 llvm/test/CodeGen/AMDGPU/sint_to_fp.ll        |     2 +-
 llvm/test/CodeGen/AMDGPU/sitofp.f16.ll        |    14 +-
 llvm/test/CodeGen/AMDGPU/sminmax.ll           |    29 +-
 llvm/test/CodeGen/AMDGPU/sminmax.v2i16.ll     |    18 +-
 llvm/test/CodeGen/AMDGPU/smrd-vccz-bug.ll     |     6 +-
 llvm/test/CodeGen/AMDGPU/smrd.ll              |     1 -
 llvm/test/CodeGen/AMDGPU/smrd_vmem_war.ll     |    29 +
 llvm/test/CodeGen/AMDGPU/spill-m0.ll          |   132 +-
 .../test/CodeGen/AMDGPU/spill-vgpr-to-agpr.ll |    24 +-
 llvm/test/CodeGen/AMDGPU/spill-wide-sgpr.ll   |    80 +-
 .../CodeGen/AMDGPU/split-arg-dbg-value.ll     |   223 +
 ...tack-pointer-offset-relative-frameindex.ll |    66 +
 .../CodeGen/AMDGPU/stack-realign-kernel.ll    |     1 -
 llvm/test/CodeGen/AMDGPU/store-hi16.ll        |    32 +-
 .../CodeGen/AMDGPU/sub-zext-cc-zext-cc.ll     |    34 +
 llvm/test/CodeGen/AMDGPU/sub.i16.ll           |     8 +-
 llvm/test/CodeGen/AMDGPU/sub.v2i16.ll         |    42 +-
 .../AMDGPU/subreg-coalescer-undef-use.ll      |    49 +-
 llvm/test/CodeGen/AMDGPU/trunc-combine.ll     |   134 +-
 llvm/test/CodeGen/AMDGPU/uint_to_fp.f64.ll    |    97 +-
 llvm/test/CodeGen/AMDGPU/uint_to_fp.ll        |     2 +-
 llvm/test/CodeGen/AMDGPU/uitofp.f16.ll        |    13 +-
 llvm/test/CodeGen/AMDGPU/uniform-cfg.ll       |     5 +-
 .../AMDGPU/uniform-loop-inside-nonuniform.ll  |     5 +-
 llvm/test/CodeGen/AMDGPU/unsupported-calls.ll |    10 +
 .../CodeGen/AMDGPU/use-sgpr-multiple-times.ll |     9 +-
 llvm/test/CodeGen/AMDGPU/v_mac_f16.ll         |    17 +-
 llvm/test/CodeGen/AMDGPU/valu-i1.ll           |    11 +-
 .../CodeGen/AMDGPU/vector_shuffle.packed.ll   |   122 +-
 ...r-descriptor-waterfall-loop-idom-update.ll |    47 +
 ...vgpr-spill-emergency-stack-slot-compute.ll |     1 +
 llvm/test/CodeGen/AMDGPU/wait.ll              |     8 +-
 llvm/test/CodeGen/AMDGPU/waitcnt-vscnt.ll     |    50 +-
 llvm/test/CodeGen/AMDGPU/wave32.ll            |    64 +-
 llvm/test/CodeGen/AMDGPU/wqm.ll               |     3 +
 llvm/test/CodeGen/AMDGPU/write_register.ll    |    20 +-
 llvm/test/CodeGen/AMDGPU/wwm-reserved.ll      |     6 +-
 llvm/test/CodeGen/AMDGPU/zero_extend.ll       |     6 +-
 .../CodeGen/ARM/2007-01-19-InfiniteLoop.ll    |     2 +-
 .../CodeGen/ARM/2009-07-18-RewriterBug.ll     |    26 +-
 .../CodeGen/ARM/2010-11-29-PrologueBug.ll     |     2 +-
 llvm/test/CodeGen/ARM/2010-12-07-PEIBug.ll    |     2 +-
 llvm/test/CodeGen/ARM/2011-08-25-ldmia_ret.ll |     2 +-
 .../CodeGen/ARM/2014-05-14-DwarfEHCrash.ll    |     2 +-
 .../ARM/2016-05-01-RegScavengerAssert.ll      |     2 +-
 .../ARM/GlobalISel/arm-irtranslator.ll        |    30 +-
 .../ARM/GlobalISel/arm-param-lowering.ll      |    36 +-
 .../irtranslator-varargs-lowering.ll          |     8 +-
 llvm/test/CodeGen/ARM/O3-pipeline.ll          |    32 +-
 llvm/test/CodeGen/ARM/ParallelDSP/blocks.ll   |   295 +
 .../ARM/ParallelDSP/complex_dot_prod.ll       |   150 +
 llvm/test/CodeGen/ARM/ParallelDSP/exchange.ll |   452 +
 .../ARM/ParallelDSP/inner-full-unroll.ll      |     4 +-
 .../ARM/ParallelDSP/multi-use-loads.ll        |   351 +-
 .../CodeGen/ARM/ParallelDSP/overlapping.ll    |   220 +
 llvm/test/CodeGen/ARM/ParallelDSP/pr42729.ll  |    84 +
 llvm/test/CodeGen/ARM/ParallelDSP/pr43073.ll  |   294 +
 llvm/test/CodeGen/ARM/ParallelDSP/sext-acc.ll |   186 +
 llvm/test/CodeGen/ARM/ParallelDSP/smlad11.ll  |     4 +-
 llvm/test/CodeGen/ARM/ParallelDSP/smlad12.ll  |     2 +-
 llvm/test/CodeGen/ARM/ParallelDSP/smladx-1.ll |     9 +-
 .../test/CodeGen/ARM/ParallelDSP/smlaldx-1.ll |     9 +-
 .../test/CodeGen/ARM/ParallelDSP/smlaldx-2.ll |     9 +-
 .../ARM/ParallelDSP/unroll-n-jam-smlad.ll     |    14 +-
 llvm/test/CodeGen/ARM/Windows/wineh-basic.ll  |     4 +-
 llvm/test/CodeGen/ARM/a15-partial-update.ll   |    71 +-
 .../test/CodeGen/ARM/addsubcarry-promotion.ll |    26 +-
 llvm/test/CodeGen/ARM/addsubo-legalization.ll |   143 +-
 llvm/test/CodeGen/ARM/arm-shrink-wrapping.ll  |    28 +-
 llvm/test/CodeGen/ARM/arm-storebytesmerge.ll  |     2 +-
 llvm/test/CodeGen/ARM/bfi.ll                  |   101 +-
 llvm/test/CodeGen/ARM/bswap16.ll              |    37 +-
 .../CodeGen/ARM/build-attributes-fn-attr6.ll  |     1 +
 .../build-attributes-optimization-minsize.ll  |     6 +-
 .../build-attributes-optimization-mixed.ll    |     6 +-
 .../build-attributes-optimization-optnone.ll  |     6 +-
 .../build-attributes-optimization-optsize.ll  |     6 +-
 .../ARM/build-attributes-optimization.ll      |     6 +-
 llvm/test/CodeGen/ARM/build-attributes.ll     |    96 +-
 llvm/test/CodeGen/ARM/byval_load_align.ll     |     4 +-
 llvm/test/CodeGen/ARM/call-tc.ll              |    20 +-
 llvm/test/CodeGen/ARM/cfguard-checks.ll       |   151 +
 llvm/test/CodeGen/ARM/cfguard-module-flag.ll  |    26 +
 llvm/test/CodeGen/ARM/clang-section.ll        |     8 +-
 llvm/test/CodeGen/ARM/cls.ll                  |    27 +
 llvm/test/CodeGen/ARM/cmov_fp16.ll            |   261 +
 llvm/test/CodeGen/ARM/coalesce-dbgvalue.ll    |     4 +-
 llvm/test/CodeGen/ARM/combine-vmovdrr.ll      |     4 +-
 llvm/test/CodeGen/ARM/compare-call.ll         |     2 +-
 llvm/test/CodeGen/ARM/constantpool-align.ll   |    25 +
 .../CodeGen/ARM/constantpool-promote-dbg.ll   |     2 +-
 llvm/test/CodeGen/ARM/constantpool-promote.ll |     4 +-
 .../ARM/cortex-a57-misched-ldm-wrback.ll      |     2 +-
 .../CodeGen/ARM/cortex-a57-misched-ldm.ll     |     2 +-
 .../ARM/cortex-a57-misched-stm-wrback.ll      |     2 +-
 .../CodeGen/ARM/cortex-a57-misched-stm.ll     |     2 +-
 .../CodeGen/ARM/cortex-a57-misched-vfma.ll    |    18 +-
 .../ARM/cortex-a57-misched-vldm-wrback.ll     |     2 +-
 .../CodeGen/ARM/cortex-a57-misched-vldm.ll    |     2 +-
 .../ARM/cortex-a57-misched-vstm-wrback.ll     |     2 +-
 .../CodeGen/ARM/cortex-a57-misched-vstm.ll    |     2 +-
 llvm/test/CodeGen/ARM/csr-split.ll            |   137 +
 llvm/test/CodeGen/ARM/cxx-tlscc.ll            |    14 +-
 .../CodeGen/ARM/dagcombine-anyexttozeroext.ll |     9 +-
 llvm/test/CodeGen/ARM/debug-info-arg.ll       |     2 +-
 llvm/test/CodeGen/ARM/debug-info-blocks.ll    |     2 +-
 llvm/test/CodeGen/ARM/debug-info-sreg2.ll     |     2 +-
 .../CodeGen/ARM/debuginfo-split-carryexpr.ll  |    51 +
 llvm/test/CodeGen/ARM/disable-fp-elim.ll      |     4 +-
 llvm/test/CodeGen/ARM/dsp-mlal.ll             |   223 +-
 llvm/test/CodeGen/ARM/dwarf-frame.ll          |    38 +
 llvm/test/CodeGen/ARM/dwarf-unwind.ll         |     2 +-
 llvm/test/CodeGen/ARM/early-cfi-sections.ll   |     2 +-
 llvm/test/CodeGen/ARM/fcmp-xo.ll              |    12 +-
 llvm/test/CodeGen/ARM/fold-stack-adjust.ll    |    13 +
 llvm/test/CodeGen/ARM/fp-intrinsics.ll        |  1083 ++
 llvm/test/CodeGen/ARM/fp16-fullfp16.ll        |     2 +-
 llvm/test/CodeGen/ARM/fp16-fusedMAC.ll        |   429 +
 llvm/test/CodeGen/ARM/fp16-instructions.ll    |    64 +-
 llvm/test/CodeGen/ARM/fp16-promote.ll         |    17 +-
 llvm/test/CodeGen/ARM/fpcmp.ll                |    10 +-
 .../ARM/fragmented-args-multiple-regs.ll      |    72 +
 llvm/test/CodeGen/ARM/fusedMAC.ll             |    50 +-
 llvm/test/CodeGen/ARM/global-merge-1.ll       |     6 +-
 llvm/test/CodeGen/ARM/gnu_mcount_nc.ll        |    41 +
 llvm/test/CodeGen/ARM/hello.ll                |     2 +-
 ...st-and-by-const-from-lshr-in-eqcmp-zero.ll |   875 +-
 ...ist-and-by-const-from-shl-in-eqcmp-zero.ll |   973 +-
 llvm/test/CodeGen/ARM/ifcvt-iter-indbr.ll     |     2 +-
 llvm/test/CodeGen/ARM/ifcvt10.ll              |     2 +-
 llvm/test/CodeGen/ARM/ifcvt11.ll              |     6 +-
 llvm/test/CodeGen/ARM/ifcvt5.ll               |     4 +-
 .../CodeGen/ARM/inlineasm-X-allocation.ll     |     2 +-
 llvm/test/CodeGen/ARM/insn-sched1.ll          |     2 +-
 llvm/test/CodeGen/ARM/intrinsics-cmse.ll      |    45 +
 .../test/CodeGen/ARM/ipra-exact-definition.ll |    44 +
 llvm/test/CodeGen/ARM/ipra-no-csr.ll          |    22 +
 llvm/test/CodeGen/ARM/ipra-r0-returned.ll     |    18 +
 llvm/test/CodeGen/ARM/ipra-reg-usage.ll       |     2 +-
 llvm/test/CodeGen/ARM/ipra.ll                 |   202 +
 llvm/test/CodeGen/ARM/isel-v8i32-crash.ll     |     2 +-
 llvm/test/CodeGen/ARM/ldrd.ll                 |    18 +-
 llvm/test/CodeGen/ARM/legalize-bitcast.ll     |    59 +
 .../CodeGen/ARM/load-combine-big-endian.ll    |   761 +-
 llvm/test/CodeGen/ARM/load-combine.ll         |   662 +-
 .../test/CodeGen/ARM/loopvectorize_pr33804.ll |     2 +-
 llvm/test/CodeGen/ARM/lsr-unfolded-offset.ll  |     2 +-
 llvm/test/CodeGen/ARM/memcpy-ldm-stm.ll       |    12 +-
 llvm/test/CodeGen/ARM/memfunc.ll              |    18 +-
 llvm/test/CodeGen/ARM/mul_const.ll            |   191 +-
 llvm/test/CodeGen/ARM/neon-v8.1a.ll           |    48 +-
 llvm/test/CodeGen/ARM/neon-vcadd.ll           |    54 +
 .../test/CodeGen/ARM/neon-vqaddsub-upgrade.ll |   330 +
 llvm/test/CodeGen/ARM/noreturn.ll             |    10 +-
 llvm/test/CodeGen/ARM/out-of-registers.ll     |     2 +-
 llvm/test/CodeGen/ARM/postrasched.ll          |    30 +
 llvm/test/CodeGen/ARM/pow.ll                  |    16 +
 llvm/test/CodeGen/ARM/prefetch.ll             |    49 +
 llvm/test/CodeGen/ARM/qdadd.ll                |   186 +
 llvm/test/CodeGen/ARM/rev.ll                  |    57 +-
 llvm/test/CodeGen/ARM/ror.ll                  |    13 +-
 llvm/test/CodeGen/ARM/sadd_sat.ll             |   428 +
 llvm/test/CodeGen/ARM/sadd_sat_plus.ll        |   379 +
 llvm/test/CodeGen/ARM/select.ll               |     4 +-
 llvm/test/CodeGen/ARM/select_const.ll         |   763 +-
 llvm/test/CodeGen/ARM/shift_minsize.ll        |     8 +-
 llvm/test/CodeGen/ARM/shift_parts.ll          |    65 +-
 llvm/test/CodeGen/ARM/signext-inreg.ll        |    17 +
 llvm/test/CodeGen/ARM/smml.ll                 |     7 +
 .../CodeGen/ARM/softfp-constant-comparison.ll |    46 +
 llvm/test/CodeGen/ARM/ssub_sat.ll             |   678 +
 llvm/test/CodeGen/ARM/ssub_sat_plus.ll        |   384 +
 llvm/test/CodeGen/ARM/stack-guard-reassign.ll |     7 +-
 .../ARM/stack-protector-bmovpcb_call.ll       |     4 +-
 llvm/test/CodeGen/ARM/stack-size-section.ll   |     2 +-
 llvm/test/CodeGen/ARM/stack_guard_remat.ll    |     2 +-
 .../CodeGen/ARM/struct-byval-frame-index.ll   |     2 +-
 llvm/test/CodeGen/ARM/swifterror.ll           |     2 +-
 llvm/test/CodeGen/ARM/swiftself.ll            |    12 +-
 llvm/test/CodeGen/ARM/tail-call-weak.ll       |     9 +-
 .../CodeGen/ARM/thumb1_return_sequence.ll     |     4 +-
 llvm/test/CodeGen/ARM/uadd_sat.ll             |   185 +
 llvm/test/CodeGen/ARM/uadd_sat_plus.ll        |   232 +
 .../test/CodeGen/ARM/unschedule-first-call.ll |     2 +-
 llvm/test/CodeGen/ARM/urem-opt-size.ll        |     2 +-
 llvm/test/CodeGen/ARM/useaa.ll                |     2 +-
 llvm/test/CodeGen/ARM/usub_sat.ll             |   188 +
 llvm/test/CodeGen/ARM/usub_sat_plus.ll        |   220 +
 llvm/test/CodeGen/ARM/uxtb.ll                 |    75 +-
 llvm/test/CodeGen/ARM/v7k-abi-align.ll        |    22 +-
 llvm/test/CodeGen/ARM/va_arg.ll               |    51 +-
 .../vecreduce-fadd-legalization-soft-float.ll |    63 +
 .../ARM/vecreduce-fadd-legalization-strict.ll |   166 +
 .../ARM/vecreduce-fmul-legalization-strict.ll |   166 +
 llvm/test/CodeGen/ARM/vector-spilling.ll      |     2 +-
 llvm/test/CodeGen/ARM/vfp.ll                  |     2 +-
 llvm/test/CodeGen/ARM/vldm-sched-a9.ll        |     2 +-
 llvm/test/CodeGen/ARM/vmul.ll                 |     4 +-
 llvm/test/CodeGen/ARM/vqadd.ll                |    64 +-
 llvm/test/CodeGen/ARM/vqdmul.ll               |    24 +-
 llvm/test/CodeGen/ARM/vqsub.ll                |    64 +-
 llvm/test/CodeGen/ARM/vrev.ll                 |   243 +-
 llvm/test/CodeGen/ARM/vsel-fp16.ll            |    40 +-
 llvm/test/CodeGen/ARM/vsel.ll                 |    80 +-
 llvm/test/CodeGen/ARM/vstlane.ll              |   383 +-
 llvm/test/CodeGen/ARM/warn-stack.ll           |     4 +-
 llvm/test/CodeGen/AVR/rot.ll                  |     8 +-
 llvm/test/CodeGen/BPF/32-bit-subreg-alu.ll    |     1 +
 .../CodeGen/BPF/32-bit-subreg-cond-select.ll  |    17 +-
 .../BPF/32-bit-subreg-peephole-phi-1.ll       |    34 +
 .../BPF/32-bit-subreg-peephole-phi-2.ll       |    34 +
 .../BPF/32-bit-subreg-peephole-phi-3.ll       |    52 +
 .../CodeGen/BPF/32-bit-subreg-peephole.ll     |    27 +-
 llvm/test/CodeGen/BPF/BTF/array-size-0.ll     |     2 +-
 llvm/test/CodeGen/BPF/BTF/binary-format.ll    |    28 +-
 .../test/CodeGen/BPF/BTF/char-no-debuginfo.ll |     2 +-
 llvm/test/CodeGen/BPF/BTF/extern-builtin.ll   |    89 +
 llvm/test/CodeGen/BPF/BTF/extern-func-arg.ll  |    79 +
 .../test/CodeGen/BPF/BTF/extern-global-var.ll |     4 +-
 .../BPF/BTF/extern-var-func-weak-section.ll   |    90 +
 .../CodeGen/BPF/BTF/extern-var-func-weak.ll   |    90 +
 llvm/test/CodeGen/BPF/BTF/extern-var-func.ll  |    91 +
 .../CodeGen/BPF/BTF/extern-var-section.ll     |   119 +
 .../CodeGen/BPF/BTF/extern-var-struct-weak.ll |   109 +
 .../test/CodeGen/BPF/BTF/extern-var-struct.ll |   110 +
 .../BPF/BTF/extern-var-weak-section.ll        |   117 +
 llvm/test/CodeGen/BPF/BTF/filename.ll         |     8 +-
 llvm/test/CodeGen/BPF/BTF/func-func-ptr.ll    |     8 +-
 llvm/test/CodeGen/BPF/BTF/func-non-void.ll    |     8 +-
 llvm/test/CodeGen/BPF/BTF/func-source.ll      |     8 +-
 llvm/test/CodeGen/BPF/BTF/func-typedef.ll     |     8 +-
 llvm/test/CodeGen/BPF/BTF/func-unused-arg.ll  |     8 +-
 llvm/test/CodeGen/BPF/BTF/func-void.ll        |     8 +-
 .../test/CodeGen/BPF/BTF/global-var-inited.ll |    20 +-
 llvm/test/CodeGen/BPF/BTF/local-var.ll        |     4 +-
 llvm/test/CodeGen/BPF/BTF/static-func.ll      |    96 +
 .../BPF/BTF/static-var-derived-type.ll        |     4 +-
 .../CodeGen/BPF/BTF/static-var-inited-sec.ll  |     4 +-
 .../test/CodeGen/BPF/BTF/static-var-inited.ll |     4 +-
 .../BPF/BTF/static-var-readonly-sec.ll        |     4 +-
 .../CodeGen/BPF/BTF/static-var-readonly.ll    |     4 +-
 llvm/test/CodeGen/BPF/BTF/static-var-sec.ll   |     4 +-
 .../BPF/BTF/static-var-zerolen-array.ll       |     4 +-
 llvm/test/CodeGen/BPF/BTF/static-var.ll       |     4 +-
 llvm/test/CodeGen/BPF/BTF/weak-global-2.ll    |    66 +
 llvm/test/CodeGen/BPF/BTF/weak-global.ll      |    65 +
 .../CodeGen/BPF/CORE/field-reloc-alu32.ll     |    73 +
 .../BPF/CORE/field-reloc-bitfield-1.ll        |   126 +
 .../BPF/CORE/field-reloc-bitfield-2.ll        |   124 +
 llvm/test/CodeGen/BPF/CORE/intrinsic-array.ll |    12 +-
 .../CORE/intrinsic-fieldinfo-byte-size-1.ll   |   153 +
 .../CORE/intrinsic-fieldinfo-byte-size-2.ll   |   142 +
 .../CORE/intrinsic-fieldinfo-byte-size-3.ll   |   133 +
 .../CORE/intrinsic-fieldinfo-byte-size-4.ll   |    86 +
 .../CORE/intrinsic-fieldinfo-existence-1.ll   |   167 +
 .../CORE/intrinsic-fieldinfo-existence-2.ll   |   124 +
 .../CORE/intrinsic-fieldinfo-existence-3.ll   |   132 +
 .../BPF/CORE/intrinsic-fieldinfo-lshift-1.ll  |   158 +
 .../BPF/CORE/intrinsic-fieldinfo-lshift-2.ll  |   125 +
 .../BPF/CORE/intrinsic-fieldinfo-rshift-1.ll  |   153 +
 .../BPF/CORE/intrinsic-fieldinfo-rshift-2.ll  |   124 +
 .../BPF/CORE/intrinsic-fieldinfo-rshift-3.ll  |   134 +
 .../CORE/intrinsic-fieldinfo-signedness-1.ll  |   167 +
 .../CORE/intrinsic-fieldinfo-signedness-2.ll  |   155 +
 .../CORE/intrinsic-fieldinfo-signedness-3.ll  |   152 +
 .../test/CodeGen/BPF/CORE/intrinsic-struct.ll |    10 +-
 .../CodeGen/BPF/CORE/intrinsic-transforms.ll  |   120 +
 llvm/test/CodeGen/BPF/CORE/intrinsic-union.ll |    10 +-
 .../CodeGen/BPF/CORE/no-elf-ama-symbol.ll     |    65 +
 llvm/test/CodeGen/BPF/CORE/no-narrow-load.ll  |   156 +
 .../BPF/CORE/offset-reloc-access-str.ll       |     8 +-
 .../CodeGen/BPF/CORE/offset-reloc-basic.ll    |    19 +-
 .../BPF/CORE/offset-reloc-cast-array-1.ll     |   128 +
 .../BPF/CORE/offset-reloc-cast-array-2.ll     |   135 +
 .../BPF/CORE/offset-reloc-cast-struct-1.ll    |   116 +
 .../BPF/CORE/offset-reloc-cast-struct-2.ll    |   121 +
 .../BPF/CORE/offset-reloc-cast-struct-3.ll    |   120 +
 .../BPF/CORE/offset-reloc-cast-union-1.ll     |   121 +
 .../BPF/CORE/offset-reloc-cast-union-2.ll     |   122 +
 .../CodeGen/BPF/CORE/offset-reloc-end-load.ll |    83 +
 .../CodeGen/BPF/CORE/offset-reloc-end-ret.ll  |    78 +
 .../BPF/CORE/offset-reloc-fieldinfo-1.ll      |   195 +
 .../BPF/CORE/offset-reloc-fieldinfo-2.ll      |   258 +
 .../CodeGen/BPF/CORE/offset-reloc-global-1.ll |    82 +
 .../CodeGen/BPF/CORE/offset-reloc-global-2.ll |    98 +
 .../CodeGen/BPF/CORE/offset-reloc-global-3.ll |    87 +
 .../CodeGen/BPF/CORE/offset-reloc-ignore.ll   |    64 +
 .../BPF/CORE/offset-reloc-middle-chain.ll     |   132 +
 .../BPF/CORE/offset-reloc-multi-array-1.ll    |   104 +
 .../BPF/CORE/offset-reloc-multi-array-2.ll    |   110 +
 .../BPF/CORE/offset-reloc-multilevel.ll       |    23 +-
 .../BPF/CORE/offset-reloc-pointer-1.ll        |    86 +
 .../BPF/CORE/offset-reloc-pointer-2.ll        |    88 +
 .../BPF/CORE/offset-reloc-struct-anonymous.ll |    25 +-
 .../BPF/CORE/offset-reloc-struct-array.ll     |    25 +-
 .../BPF/CORE/offset-reloc-typedef-array.ll    |     9 +-
 .../BPF/CORE/offset-reloc-typedef-struct.ll   |     7 +-
 .../BPF/CORE/offset-reloc-typedef-union.ll    |     7 +-
 .../CodeGen/BPF/CORE/offset-reloc-typedef.ll  |     9 +-
 .../CodeGen/BPF/CORE/offset-reloc-union.ll    |    23 +-
 llvm/test/CodeGen/BPF/callx.ll                |    20 +
 llvm/test/CodeGen/BPF/dwarfdump.ll            |     2 +-
 llvm/test/CodeGen/BPF/objdump_two_funcs.ll    |     2 +-
 llvm/test/CodeGen/BPF/optnone-1.ll            |    52 +
 llvm/test/CodeGen/BPF/reloc-btf-2.ll          |     2 +-
 llvm/test/CodeGen/BPF/reloc-btf.ll            |     2 +-
 llvm/test/CodeGen/BPF/remove_truncate_6.ll    |    80 +
 llvm/test/CodeGen/BPF/shifts.ll               |     2 +-
 llvm/test/CodeGen/BPF/warn-stack.ll           |     4 +-
 llvm/test/CodeGen/BPF/xadd.ll                 |     2 +-
 .../test/CodeGen/Generic/DbgValueAggregate.ll |    36 +
 llvm/test/CodeGen/Generic/cfi-sections.ll     |     4 +-
 .../Generic/expand-experimental-reductions.ll |    13 +
 llvm/test/CodeGen/Hexagon/64bit_tstbit.ll     |    34 +
 llvm/test/CodeGen/Hexagon/NVJumpCmp.ll        |     2 +-
 llvm/test/CodeGen/Hexagon/addh-sext-trunc.ll  |     2 +-
 llvm/test/CodeGen/Hexagon/alu64.ll            |     2 +-
 .../Hexagon/autohvx/isel-setcc-v256i1.ll      |    15 +
 .../CodeGen/Hexagon/autohvx/minmax-128b.ll    |   344 +
 .../CodeGen/Hexagon/autohvx/minmax-64b.ll     |   344 +
 .../Hexagon/autohvx/vector-compare-128b.ll    |   180 +-
 .../Hexagon/autohvx/vector-compare-64b.ll     |   180 +-
 llvm/test/CodeGen/Hexagon/base-offset-addr.ll |     2 +-
 llvm/test/CodeGen/Hexagon/base-offset-post.ll |     2 +-
 .../CodeGen/Hexagon/bit-loop-rc-mismatch.ll   |     4 +-
 llvm/test/CodeGen/Hexagon/builtin-prefetch.ll |     2 +-
 llvm/test/CodeGen/Hexagon/cfi-offset.ll       |     2 +-
 llvm/test/CodeGen/Hexagon/cmp-extend.ll       |     2 +-
 llvm/test/CodeGen/Hexagon/cmp.ll              |     2 +-
 .../CodeGen/Hexagon/constp-combine-neg.ll     |     2 +-
 llvm/test/CodeGen/Hexagon/constp-extract.ll   |     2 +-
 llvm/test/CodeGen/Hexagon/constp-physreg.ll   |     2 +-
 llvm/test/CodeGen/Hexagon/dead-store-stack.ll |     4 +-
 llvm/test/CodeGen/Hexagon/dealloc_return.ll   |     2 +-
 .../Hexagon/early-if-conversion-bug1.ll       |     4 +-
 llvm/test/CodeGen/Hexagon/ehabi.ll            |     2 +-
 llvm/test/CodeGen/Hexagon/fminmax.ll          |     2 +-
 llvm/test/CodeGen/Hexagon/fpelim-basic.ll     |     4 +-
 llvm/test/CodeGen/Hexagon/hasfp-crash1.ll     |     2 +-
 llvm/test/CodeGen/Hexagon/hasfp-crash2.ll     |     2 +-
 .../hexagon_vector_loop_carried_reuse.ll      |     2 +-
 ...agon_vector_loop_carried_reuse_constant.ll |     2 +-
 llvm/test/CodeGen/Hexagon/hwloop-crit-edge.ll |     2 +-
 llvm/test/CodeGen/Hexagon/hwloop-preheader.ll |     2 +-
 llvm/test/CodeGen/Hexagon/insert4.ll          |     2 +-
 .../CodeGen/Hexagon/isel-bitcast-v8i1-i8.ll   |    18 +
 .../Hexagon/isel-bitcast-v8i8-v4i16.ll        |    13 +
 .../CodeGen/Hexagon/isel-minmax-v64bit.ll     |   202 +
 llvm/test/CodeGen/Hexagon/isel-prefer.ll      |     4 +-
 .../test/CodeGen/Hexagon/isel-vselect-v4i8.ll |     9 +
 llvm/test/CodeGen/Hexagon/jt-in-text.ll       |     2 +-
 llvm/test/CodeGen/Hexagon/mem-fi-add.ll       |     4 +-
 llvm/test/CodeGen/Hexagon/memops-stack.ll     |     4 +-
 llvm/test/CodeGen/Hexagon/muxii-bug.ll        |    30 +
 .../CodeGen/Hexagon/packetizer-resources.ll   |    29 +
 llvm/test/CodeGen/Hexagon/pic-regusage.ll     |     2 +-
 llvm/test/CodeGen/Hexagon/postinc-offset.ll   |     2 +-
 llvm/test/CodeGen/Hexagon/rdf-dead-loop.ll    |     2 +-
 .../CodeGen/Hexagon/rdf-inline-asm-fixed.ll   |     2 +-
 llvm/test/CodeGen/Hexagon/rdf-inline-asm.ll   |     2 +-
 llvm/test/CodeGen/Hexagon/reg-by-name.ll      |   780 ++
 .../test/CodeGen/Hexagon/reg-scavengebug-3.ll |     4 +-
 llvm/test/CodeGen/Hexagon/runtime-stkchk.ll   |     4 +-
 llvm/test/CodeGen/Hexagon/sdr-shr32.ll        |     2 +-
 llvm/test/CodeGen/Hexagon/signext-inreg.ll    |   272 +
 .../Hexagon/switch-lut-explicit-section.ll    |     2 +-
 .../Hexagon/switch-lut-function-section.ll    |     2 +-
 .../Hexagon/switch-lut-multiple-functions.ll  |     2 +-
 .../Hexagon/switch-lut-text-section.ll        |     2 +-
 llvm/test/CodeGen/Hexagon/swp-art-deps-rec.ll |     2 +-
 llvm/test/CodeGen/Hexagon/swp-bad-sched.ll    |     2 +-
 llvm/test/CodeGen/Hexagon/swp-carried-1.ll    |     2 +-
 llvm/test/CodeGen/Hexagon/swp-chain-refs.ll   |     2 +-
 llvm/test/CodeGen/Hexagon/swp-change-dep1.ll  |     2 +-
 llvm/test/CodeGen/Hexagon/swp-change-deps.ll  |     2 +-
 llvm/test/CodeGen/Hexagon/swp-check-offset.ll |     6 +-
 llvm/test/CodeGen/Hexagon/swp-const-tc1.ll    |     2 +-
 llvm/test/CodeGen/Hexagon/swp-const-tc2.ll    |     2 +-
 llvm/test/CodeGen/Hexagon/swp-const-tc3.ll    |     2 +-
 .../CodeGen/Hexagon/swp-conv3x3-nested.ll     |     4 +-
 .../test/CodeGen/Hexagon/swp-copytophi-dag.ll |     2 +-
 llvm/test/CodeGen/Hexagon/swp-crash-iter.ll   |    32 +
 .../CodeGen/Hexagon/swp-dep-neg-offset.ll     |     2 +-
 llvm/test/CodeGen/Hexagon/swp-disable-Os.ll   |     2 +-
 .../CodeGen/Hexagon/swp-epilog-numphis.ll     |     2 +-
 llvm/test/CodeGen/Hexagon/swp-epilog-phi10.ll |     2 +-
 llvm/test/CodeGen/Hexagon/swp-epilog-phi12.ll |    54 +
 llvm/test/CodeGen/Hexagon/swp-epilog-phi2.ll  |     2 +-
 llvm/test/CodeGen/Hexagon/swp-epilog-phi4.ll  |     2 +-
 llvm/test/CodeGen/Hexagon/swp-epilog-phi5.ll  |     2 +-
 llvm/test/CodeGen/Hexagon/swp-epilog-phi7.ll  |     4 +-
 llvm/test/CodeGen/Hexagon/swp-epilog-phi8.ll  |     2 +-
 llvm/test/CodeGen/Hexagon/swp-kernel-phi1.ll  |     2 +-
 llvm/test/CodeGen/Hexagon/swp-large-rec.ll    |     2 +-
 llvm/test/CodeGen/Hexagon/swp-listen-loop3.ll |     2 +-
 .../Hexagon/swp-loop-carried-unknown.ll       |     2 +-
 llvm/test/CodeGen/Hexagon/swp-lots-deps.ll    |     2 +-
 llvm/test/CodeGen/Hexagon/swp-max.ll          |     2 +-
 llvm/test/CodeGen/Hexagon/swp-maxstart.ll     |     2 +-
 .../CodeGen/Hexagon/swp-memrefs-epilog.ll     |     2 +-
 llvm/test/CodeGen/Hexagon/swp-multi-loops.ll  |     2 +-
 llvm/test/CodeGen/Hexagon/swp-new-phi.ll      |     2 +-
 llvm/test/CodeGen/Hexagon/swp-order-copies.ll |     2 +-
 llvm/test/CodeGen/Hexagon/swp-order-deps7.ll  |     2 +-
 llvm/test/CodeGen/Hexagon/swp-order.ll        |     2 +-
 .../test/CodeGen/Hexagon/swp-phi-ch-offset.ll |     2 +-
 llvm/test/CodeGen/Hexagon/swp-phi-chains.ll   |     2 +-
 llvm/test/CodeGen/Hexagon/swp-phi-dep.ll      |     2 +-
 llvm/test/CodeGen/Hexagon/swp-phi-ref.ll      |     2 +-
 llvm/test/CodeGen/Hexagon/swp-prolog-phi.ll   |     2 +-
 llvm/test/CodeGen/Hexagon/swp-prolog-phi4.ll  |     2 +-
 llvm/test/CodeGen/Hexagon/swp-rename.ll       |     2 +-
 llvm/test/CodeGen/Hexagon/swp-resmii-1.ll     |     2 +-
 llvm/test/CodeGen/Hexagon/swp-resmii.ll       |     2 +-
 llvm/test/CodeGen/Hexagon/swp-reuse-phi-6.ll  |     2 +-
 llvm/test/CodeGen/Hexagon/swp-sigma.ll        |     2 +-
 llvm/test/CodeGen/Hexagon/swp-stages4.ll      |     6 +-
 llvm/test/CodeGen/Hexagon/swp-subreg.ll       |     2 +-
 llvm/test/CodeGen/Hexagon/swp-swap.ll         |     2 +-
 llvm/test/CodeGen/Hexagon/swp-tfri.ll         |     2 +-
 llvm/test/CodeGen/Hexagon/swp-vect-dotprod.ll |     6 +-
 llvm/test/CodeGen/Hexagon/swp-vmult.ll        |     2 +-
 llvm/test/CodeGen/Hexagon/swp-vsum.ll         |     4 +-
 llvm/test/CodeGen/Hexagon/tstbit.ll           |    85 +-
 llvm/test/CodeGen/Hexagon/usr-ovf-dep.ll      |     2 +-
 .../CodeGen/Hexagon/v6-unaligned-spill.ll     |     2 +-
 llvm/test/CodeGen/Hexagon/v60Intrins.ll       |     2 +-
 llvm/test/CodeGen/Hexagon/v60small.ll         |     2 +-
 llvm/test/CodeGen/Hexagon/v6vec-vprint.ll     |     2 +-
 llvm/test/CodeGen/Hexagon/vec-pred-spill1.ll  |     4 +-
 .../CodeGen/Hexagon/vect/vect-bad-bitcast.ll  |     4 +-
 .../test/CodeGen/Hexagon/vect/vect-extract.ll |     2 +-
 .../test/CodeGen/Hexagon/vect/vect-shuffle.ll |     2 +-
 llvm/test/CodeGen/Hexagon/vect/vect-v4i16.ll  |     2 +-
 .../test/CodeGen/Hexagon/vect/vect-vshifts.ll |     2 +-
 llvm/test/CodeGen/Hexagon/vect/vect-xor.ll    |     2 +-
 .../MIR/AMDGPU/machine-function-info.ll       |    33 +-
 .../CodeGen/MIR/Mips/setRegClassOrRegBank.ll  |    27 +
 llvm/test/CodeGen/MSP430/cc_args.ll           |     2 +-
 llvm/test/CodeGen/MSP430/cc_ret.ll            |     2 +-
 llvm/test/CodeGen/MSP430/interrupt.ll         |     9 +
 llvm/test/CodeGen/MSP430/selectcc.ll          |    46 +
 .../MSP430/shift-amount-threshold-b.ll        |    50 +
 .../CodeGen/MSP430/shift-amount-threshold.ll  |   210 +
 .../MSP430/transient-stack-alignment.ll       |     2 +-
 llvm/test/CodeGen/Mips/Fast-ISel/br1.ll       |     2 +-
 .../test/CodeGen/Mips/Fast-ISel/loadstore2.ll |     2 +-
 .../CodeGen/Mips/Fast-ISel/loadstrconst.ll    |     2 +-
 llvm/test/CodeGen/Mips/Fast-ISel/logopm.ll    |     4 +-
 .../CodeGen/Mips/Fast-ISel/simplestorefp1.ll  |     2 +-
 .../CodeGen/Mips/Fast-ISel/simplestorei.ll    |     2 +-
 .../irtranslator/aggregate_struct_return.ll   |   132 +
 .../Mips/GlobalISel/irtranslator/call.ll      |    36 +-
 .../GlobalISel/irtranslator/extend_args.ll    |    12 +-
 .../GlobalISel/irtranslator/sret_pointer.ll   |    39 +
 .../GlobalISel/irtranslator/stack_args.ll     |     2 +-
 .../Mips/GlobalISel/irtranslator/var_arg.ll   |    66 +
 .../CodeGen/Mips/GlobalISel/llvm-ir/add.ll    |    79 +-
 .../Mips/GlobalISel/llvm-ir/add_vec.ll        |    70 +
 .../GlobalISel/llvm-ir/add_vec_builtin.ll     |   138 +
 .../llvm-ir/aggregate_struct_return.ll        |   114 +
 .../Mips/GlobalISel/llvm-ir/bitreverse.ll     |   184 +
 .../Mips/GlobalISel/llvm-ir/bitwise.ll        |   198 +
 .../CodeGen/Mips/GlobalISel/llvm-ir/branch.ll |     3 +-
 .../Mips/GlobalISel/llvm-ir/brindirect.ll     |    33 +
 .../CodeGen/Mips/GlobalISel/llvm-ir/bswap.ll  |    68 +
 .../Mips/GlobalISel/llvm-ir/constants.ll      |    14 +-
 .../Mips/GlobalISel/llvm-ir/dyn_stackalloc.ll |    64 +
 .../Mips/GlobalISel/llvm-ir/fabs_vec.ll       |    34 +
 .../GlobalISel/llvm-ir/fabs_vec_builtin.ll    |    35 +
 .../CodeGen/Mips/GlobalISel/llvm-ir/fcmp.ll   |     4 +-
 .../CodeGen/Mips/GlobalISel/llvm-ir/fence.ll  |    13 +
 ...loating_point_vec_arithmetic_operations.ll |   145 +
 ...point_vec_arithmetic_operations_builtin.ll |   146 +
 .../GlobalISel/llvm-ir/fptosi_and_fptoui.ll   |   219 +
 .../Mips/GlobalISel/llvm-ir/fsqrt_vec.ll      |    34 +
 .../GlobalISel/llvm-ir/fsqrt_vec_builtin.ll   |    35 +
 .../CodeGen/Mips/GlobalISel/llvm-ir/icmp.ll   |    35 +-
 .../Mips/GlobalISel/llvm-ir/implicit_def.ll   |    83 +
 .../llvm-ir/inttoptr_and_ptrtoint.ll          |    24 +
 .../GlobalISel/llvm-ir/jump_table_and_brjt.ll |   287 +
 .../CodeGen/Mips/GlobalISel/llvm-ir/load.ll   |     4 +-
 .../GlobalISel/llvm-ir/load_store_fold.ll     |   105 +
 .../Mips/GlobalISel/llvm-ir/load_store_vec.ll |    80 +
 .../llvm-ir/long_ambiguous_chain_s32.ll       |   210 +-
 .../llvm-ir/long_ambiguous_chain_s64.ll       |   266 +-
 .../CodeGen/Mips/GlobalISel/llvm-ir/mul.ll    |    50 +-
 .../Mips/GlobalISel/llvm-ir/mul_vec.ll        |    70 +
 .../GlobalISel/llvm-ir/mul_vec_builtin.ll     |    74 +
 .../CodeGen/Mips/GlobalISel/llvm-ir/phi.ll    |    27 +-
 .../Mips/GlobalISel/llvm-ir/rem_and_div.ll    |    36 +-
 .../GlobalISel/llvm-ir/rem_and_div_vec.ll     |   274 +
 .../llvm-ir/rem_and_div_vec_builtin.ll        |   290 +
 .../CodeGen/Mips/GlobalISel/llvm-ir/select.ll |    35 +-
 .../GlobalISel/llvm-ir/sitofp_and_uitofp.ll   |   201 +
 .../Mips/GlobalISel/llvm-ir/sret_pointer.ll   |    38 +
 .../Mips/GlobalISel/llvm-ir/stack_args.ll     |     4 +-
 .../CodeGen/Mips/GlobalISel/llvm-ir/store.ll  |     4 +-
 .../CodeGen/Mips/GlobalISel/llvm-ir/sub.ll    |    26 +-
 .../Mips/GlobalISel/llvm-ir/sub_vec.ll        |    70 +
 .../GlobalISel/llvm-ir/sub_vec_builtin.ll     |   138 +
 .../GlobalISel/llvm-ir/test_TypeInfoforMF.ll  |    12 +-
 .../CodeGen/Mips/GlobalISel/llvm-ir/trap.ll   |    14 +
 .../CodeGen/Mips/GlobalISel/llvm-ir/trunc.ll  |    13 +
 .../llvm-ir/truncStore_and_aExtLoad.ll        |     3 +-
 .../Mips/GlobalISel/llvm-ir/var_arg.ll        |    67 +
 .../llvm-ir/zextLoad_and_sextLoad.ll          |    28 +
 .../Mips/GlobalISel/llvm-ir/zext_and_sext.ll  |    27 +
 llvm/test/CodeGen/Mips/address-selection.ll   |     6 +-
 llvm/test/CodeGen/Mips/atomic-min-max-64.ll   |   158 +
 llvm/test/CodeGen/Mips/atomic-min-max.ll      |  4674 +++++++
 llvm/test/CodeGen/Mips/atomic.ll              |     2 +-
 llvm/test/CodeGen/Mips/beqzc.ll               |     2 +-
 llvm/test/CodeGen/Mips/beqzc1.ll              |     2 +-
 llvm/test/CodeGen/Mips/biggot.ll              |    14 +-
 llvm/test/CodeGen/Mips/blockaddr.ll           |    39 +-
 .../Mips/branch-relaxation-with-hazard.ll     |     2 +
 llvm/test/CodeGen/Mips/brsize3.ll             |     2 +-
 llvm/test/CodeGen/Mips/brsize3a.ll            |     2 +-
 llvm/test/CodeGen/Mips/cconv/vector.ll        |  1068 +-
 llvm/test/CodeGen/Mips/ci2.ll                 |     2 +-
 llvm/test/CodeGen/Mips/cmplarge.ll            |     4 +-
 .../Mips/compactbranches/no-beqzc-bnezc.ll    |     4 +-
 llvm/test/CodeGen/Mips/const1.ll              |     2 +-
 llvm/test/CodeGen/Mips/const4a.ll             |     4 +-
 llvm/test/CodeGen/Mips/const6.ll              |     4 +-
 llvm/test/CodeGen/Mips/const6a.ll             |     2 +-
 llvm/test/CodeGen/Mips/constraint-empty.ll    |    14 +
 llvm/test/CodeGen/Mips/copy-fp64.ll           |    29 +
 llvm/test/CodeGen/Mips/cpus-no-mips64.ll      |    16 +
 llvm/test/CodeGen/Mips/cpus.ll                |    69 +
 llvm/test/CodeGen/Mips/ctlz.ll                |     2 +-
 .../CodeGen/Mips/delay-slot-fill-forward.ll   |     2 +-
 .../CodeGen/Mips/dynamic-stack-realignment.ll |    20 +-
 llvm/test/CodeGen/Mips/f16abs.ll              |     4 +-
 llvm/test/CodeGen/Mips/fp16instrinsmc.ll      |     4 +-
 llvm/test/CodeGen/Mips/fpneeded.ll            |     2 +-
 llvm/test/CodeGen/Mips/fpnotneeded.ll         |     2 +-
 llvm/test/CodeGen/Mips/frame-address.ll       |    25 +-
 llvm/test/CodeGen/Mips/global-address.ll      |    19 +-
 llvm/test/CodeGen/Mips/hf16call32.ll          |     4 +-
 llvm/test/CodeGen/Mips/hf16call32_body.ll     |     2 +-
 llvm/test/CodeGen/Mips/hfptrcall.ll           |     4 +-
 .../Mips/indirect-jump-hazard/long-calls.ll   |    25 +-
 .../Mips/inlineasm-constraint-reg64.ll        |    19 +-
 llvm/test/CodeGen/Mips/l3mc.ll                |     2 +-
 llvm/test/CodeGen/Mips/lcb2.ll                |     2 +-
 llvm/test/CodeGen/Mips/lcb3c.ll               |     2 +-
 llvm/test/CodeGen/Mips/lcb4a.ll               |     2 +-
 llvm/test/CodeGen/Mips/lcb5.ll                |     2 +-
 llvm/test/CodeGen/Mips/llvm-ir/lshr.ll        |   258 +-
 llvm/test/CodeGen/Mips/llvm-ir/shl.ll         |   246 +-
 llvm/test/CodeGen/Mips/long-calls.ll          |    14 +-
 llvm/test/CodeGen/Mips/mbrsize4a.ll           |     4 +-
 llvm/test/CodeGen/Mips/mcount.ll              |   123 +
 .../Mips/micromips-ase-function-attribute.ll  |     2 +-
 llvm/test/CodeGen/Mips/micromips-attr.ll      |     4 +-
 llvm/test/CodeGen/Mips/mips16-hf-attr-2.ll    |    12 +-
 llvm/test/CodeGen/Mips/mips16-hf-attr.ll      |    12 +-
 llvm/test/CodeGen/Mips/mips16_32_1.ll         |     2 +-
 llvm/test/CodeGen/Mips/mips16_32_10.ll        |     6 +-
 llvm/test/CodeGen/Mips/mips16_32_3.ll         |     6 +-
 llvm/test/CodeGen/Mips/mips16_32_4.ll         |     6 +-
 llvm/test/CodeGen/Mips/mips16_32_5.ll         |     6 +-
 llvm/test/CodeGen/Mips/mips16_32_6.ll         |     6 +-
 llvm/test/CodeGen/Mips/mips16_32_7.ll         |     6 +-
 llvm/test/CodeGen/Mips/mips16_32_8.ll         |     8 +-
 llvm/test/CodeGen/Mips/mips16_32_9.ll         |     6 +-
 .../test/CodeGen/Mips/msa/2r_vector_scalar.ll |     5 +-
 llvm/test/CodeGen/Mips/msa/arithmetic.ll      |    64 +
 llvm/test/CodeGen/Mips/msa/nori.b.ll          |    26 +
 llvm/test/CodeGen/Mips/msa/remat-ldi.ll       |   107 +
 .../CodeGen/Mips/no-frame-pointer-elim.ll     |    37 +
 llvm/test/CodeGen/Mips/nomips16.ll            |     4 +-
 .../CodeGen/Mips/pbqp-reserved-physreg.ll     |     2 +-
 llvm/test/CodeGen/Mips/powif64_16.ll          |     2 +-
 llvm/test/CodeGen/Mips/pr42736.ll             |    28 +
 llvm/test/CodeGen/Mips/reloc-jalr.ll          |   143 +-
 llvm/test/CodeGen/Mips/s2rem.ll               |     4 +-
 llvm/test/CodeGen/Mips/sel1c.ll               |     2 +-
 llvm/test/CodeGen/Mips/sel2c.ll               |     2 +-
 llvm/test/CodeGen/Mips/simplebr.ll            |     4 +-
 llvm/test/CodeGen/Mips/sr1.ll                 |     4 +-
 llvm/test/CodeGen/Mips/stack-alignment.ll     |     2 +-
 llvm/test/CodeGen/Mips/start-asm-file.ll      |    83 +-
 llvm/test/CodeGen/Mips/tnaked.ll              |     6 +-
 llvm/test/CodeGen/Mips/v2i16tof32.ll          |     9 +-
 llvm/test/CodeGen/NVPTX/bug21465.ll           |     2 +-
 llvm/test/CodeGen/NVPTX/bug22322.ll           |     2 +-
 llvm/test/CodeGen/NVPTX/bug41651.ll           |    13 +
 llvm/test/CodeGen/NVPTX/pr17529.ll            |     2 +-
 llvm/test/CodeGen/NVPTX/shfl-p.ll             |   172 +
 llvm/test/CodeGen/NVPTX/shfl-sync-p.ll        |   180 +
 .../CodeGen/PowerPC/2007-09-08-unaligned.ll   |     3 +-
 .../CodeGen/PowerPC/2008-10-28-f128-i32.ll    |    53 +-
 .../test/CodeGen/PowerPC/2010-02-12-saveCR.ll |     2 +-
 llvm/test/CodeGen/PowerPC/CSR-fit.ll          |    46 +-
 .../PowerPC/CompareEliminationSpillIssue.ll   |     2 +-
 .../PowerPC/MCSE-caller-preserved-reg.ll      |    19 +-
 .../CodeGen/PowerPC/PR35812-neg-cmpxchg.ll    |     6 +-
 .../test/CodeGen/PowerPC/VSX-XForm-Scalars.ll |     6 +-
 .../PowerPC/aantidep-inline-asm-use.ll        |     2 +-
 .../PowerPC/absol-jump-table-enabled.ll       |   100 +
 llvm/test/CodeGen/PowerPC/add_cmp.ll          |    44 +-
 llvm/test/CodeGen/PowerPC/addi-licm.ll        |     2 +-
 llvm/test/CodeGen/PowerPC/addi-offset-fold.ll |     2 +-
 llvm/test/CodeGen/PowerPC/addrfuncstr.ll      |     4 +-
 llvm/test/CodeGen/PowerPC/aix-byval-param.ll  |    16 +
 llvm/test/CodeGen/PowerPC/aix-cc-abi.ll       |   880 ++
 llvm/test/CodeGen/PowerPC/aix-cc-altivec.ll   |    23 +
 .../aix-external-sym-sdnode-lowering.ll       |   128 +
 llvm/test/CodeGen/PowerPC/aix-func-dsc-gen.ll |   112 +
 .../test/CodeGen/PowerPC/aix-indirect-call.ll |   144 +
 .../PowerPC/aix-lower-block-address.ll        |    72 +
 .../PowerPC/aix-lower-constant-pool-index.ll  |    87 +
 .../CodeGen/PowerPC/aix-lower-jump-table.ll   |   188 +
 llvm/test/CodeGen/PowerPC/aix-lr.ll           |    32 +
 llvm/test/CodeGen/PowerPC/aix-nest-param.ll   |    13 +
 .../PowerPC/aix-readonly-with-relocation.ll   |    19 +
 .../PowerPC/aix-reference-func-addr-const.ll  |    32 +
 llvm/test/CodeGen/PowerPC/aix-return55.ll     |    11 +
 llvm/test/CodeGen/PowerPC/aix-space.ll        |    17 +
 llvm/test/CodeGen/PowerPC/aix-sret-param.ll   |    98 +
 llvm/test/CodeGen/PowerPC/aix-stackargs.ll    |    12 +
 llvm/test/CodeGen/PowerPC/aix-trampoline.ll   |    14 +
 .../CodeGen/PowerPC/aix-undef-func-call.ll    |    29 +
 .../PowerPC/aix-user-defined-memcpy.ll        |    58 +
 .../PowerPC/aix-weak-undef-func-call.ll       |    30 +
 .../PowerPC/aix-xcoff-data-only-notoc.ll      |    19 +
 llvm/test/CodeGen/PowerPC/aix-xcoff-data.ll   |   595 +
 llvm/test/CodeGen/PowerPC/aix-xcoff-lcomm.ll  |   112 +
 .../PowerPC/aix-xcoff-mergeable-const.ll      |   166 +
 .../PowerPC/aix-xcoff-mergeable-str.ll        |    62 +
 llvm/test/CodeGen/PowerPC/aix-xcoff-rodata.ll |   322 +
 .../PowerPC/aix-xcoff-textdisassembly.ll      |    22 +
 llvm/test/CodeGen/PowerPC/aix-xcoff-toc.ll    |   338 +
 llvm/test/CodeGen/PowerPC/and-mask.ll         |    88 +
 llvm/test/CodeGen/PowerPC/asm-constraints.ll  |     2 +-
 .../CodeGen/PowerPC/asym-regclass-copy.ll     |     2 +-
 .../CodeGen/PowerPC/available-externally.ll   |     3 +-
 .../CodeGen/PowerPC/bitcasts-direct-move.ll   |     4 +-
 llvm/test/CodeGen/PowerPC/branch-opt.ll       |    10 +-
 llvm/test/CodeGen/PowerPC/brcond.ll           |     6 +-
 .../CodeGen/PowerPC/build-vector-allones.ll   |   109 +
 .../CodeGen/PowerPC/build-vector-tests.ll     |   765 +-
 .../CodeGen/PowerPC/builtins-ppc-p9-f128.ll   |     2 +-
 llvm/test/CodeGen/PowerPC/check-cpu.ll        |    11 +
 llvm/test/CodeGen/PowerPC/combine-fneg.ll     |    11 +-
 .../test/CodeGen/PowerPC/constant-combines.ll |     8 +-
 llvm/test/CodeGen/PowerPC/cr-spills.ll        |     2 +-
 llvm/test/CodeGen/PowerPC/crypto_bifs.ll      |     2 +-
 llvm/test/CodeGen/PowerPC/csr-split.ll        |   263 +
 llvm/test/CodeGen/PowerPC/ctr-cleanup.ll      |     2 +-
 llvm/test/CodeGen/PowerPC/ctr-minmaxnum.ll    |    40 +-
 llvm/test/CodeGen/PowerPC/ctrloop-intrin.ll   |     2 +-
 llvm/test/CodeGen/PowerPC/ctrloop-le.ll       |     3 -
 llvm/test/CodeGen/PowerPC/ctrloop-lt.ll       |     5 -
 .../CodeGen/PowerPC/ctrloop-shortLoops.ll     |    14 +-
 llvm/test/CodeGen/PowerPC/cxx_tlscc64.ll      |     2 +-
 llvm/test/CodeGen/PowerPC/dform-adjust.ll     |   125 +
 .../CodeGen/PowerPC/direct-move-profit.ll     |     2 +-
 llvm/test/CodeGen/PowerPC/div-e-32.ll         |     2 +-
 llvm/test/CodeGen/PowerPC/div-e-all.ll        |     2 +-
 llvm/test/CodeGen/PowerPC/ec-input.ll         |     4 +-
 llvm/test/CodeGen/PowerPC/eh-dwarf-cfa.ll     |     2 +-
 llvm/test/CodeGen/PowerPC/elf-common.ll       |    45 +
 .../CodeGen/PowerPC/expand-contiguous-isel.ll |     1 +
 llvm/test/CodeGen/PowerPC/expand-isel.ll      |     4 +-
 .../CodeGen/PowerPC/extra-toc-reg-deps.ll     |     4 +-
 .../test/CodeGen/PowerPC/extract-and-store.ll |    36 +-
 llvm/test/CodeGen/PowerPC/f128-aggregates.ll  |     6 +-
 llvm/test/CodeGen/PowerPC/f128-fma.ll         |     2 +-
 llvm/test/CodeGen/PowerPC/fast-isel-call.ll   |     6 +-
 llvm/test/CodeGen/PowerPC/fast-isel-const.ll  |     2 +-
 .../CodeGen/PowerPC/fast-isel-load-store.ll   |     9 +-
 llvm/test/CodeGen/PowerPC/fast-isel-ret.ll    |     6 +-
 llvm/test/CodeGen/PowerPC/fdiv.ll             |    15 +
 .../CodeGen/PowerPC/float-load-store-pair.ll  |    34 +-
 .../CodeGen/PowerPC/float-vector-gather.ll    |    48 +
 llvm/test/CodeGen/PowerPC/fma-assoc.ll        |     1 -
 llvm/test/CodeGen/PowerPC/fma-combine.ll      |   139 +
 llvm/test/CodeGen/PowerPC/fma-ext.ll          |     1 -
 .../PowerPC/fma-mutate-duplicate-vreg.ll      |     2 +-
 llvm/test/CodeGen/PowerPC/fma-mutate.ll       |    19 +-
 llvm/test/CodeGen/PowerPC/fmf-propagation.ll  |     2 +-
 llvm/test/CodeGen/PowerPC/fneg.ll             |    43 +-
 llvm/test/CodeGen/PowerPC/fold-rlwinm-1.ll    |    44 +
 .../fp-int-conversions-direct-moves.ll        |   307 +-
 .../CodeGen/PowerPC/fp-int128-fp-combine.ll   |     6 +-
 .../PowerPC/fp-intrinsics-fptosi-legal.ll     |    20 +
 llvm/test/CodeGen/PowerPC/fp64-to-int16.ll    |     2 +-
 llvm/test/CodeGen/PowerPC/frameaddr.ll        |     4 +-
 llvm/test/CodeGen/PowerPC/funnel-shift-rot.ll |    24 +-
 .../CodeGen/PowerPC/glob-comp-aa-crash.ll     |     6 +-
 llvm/test/CodeGen/PowerPC/gpr-vsr-spill.ll    |     2 +-
 .../PowerPC/ifcvt-forked-bug-2016-08-08.ll    |     2 +-
 llvm/test/CodeGen/PowerPC/ifunc.ll            |    25 +
 llvm/test/CodeGen/PowerPC/inc-of-add.ll       |    14 +-
 .../PowerPC/inline-asm-vsx-clobbers.ll        |    32 +
 .../CodeGen/PowerPC/inlineasm-extendedmne.ll  |    72 +
 llvm/test/CodeGen/PowerPC/instr-properties.ll |     9 +
 llvm/test/CodeGen/PowerPC/isel-rc-nox0.ll     |     2 +-
 .../PowerPC/jump-tables-collapse-rotate.ll    |     4 +-
 llvm/test/CodeGen/PowerPC/knowCRBitSpill.ll   |     7 +-
 llvm/test/CodeGen/PowerPC/licm-tocReg.ll      |    10 +-
 llvm/test/CodeGen/PowerPC/load-and-splat.ll   |   264 +
 .../PowerPC/load-shuffle-and-shuffle-store.ll |   792 ++
 .../PowerPC/loop-instr-form-prepare.ll        |   827 ++
 .../PowerPC/lower-globaladdr32-aix-asm.ll     |    45 +
 .../CodeGen/PowerPC/lower-globaladdr32-aix.ll |    38 +
 .../PowerPC/lower-globaladdr64-aix-asm.ll     |    45 +
 .../CodeGen/PowerPC/lower-globaladdr64-aix.ll |    38 +
 llvm/test/CodeGen/PowerPC/lower-massv-attr.ll |    29 +
 llvm/test/CodeGen/PowerPC/lower-massv.ll      |   603 +
 llvm/test/CodeGen/PowerPC/lsr-insns-cost.ll   |    66 +
 llvm/test/CodeGen/PowerPC/machine-combiner.ll |     6 +-
 llvm/test/CodeGen/PowerPC/machine-pre.ll      |    10 +-
 .../memCmpUsedInZeroEqualityComparison.ll     |     4 +-
 llvm/test/CodeGen/PowerPC/memcmpIR.ll         |     8 +-
 llvm/test/CodeGen/PowerPC/memcpy-vec.ll       |     8 +-
 .../test/CodeGen/PowerPC/mi-peephole-splat.ll |   136 +
 llvm/test/CodeGen/PowerPC/negctr.ll           |     2 +-
 llvm/test/CodeGen/PowerPC/no-dup-spill-fp.ll  |     2 +-
 llvm/test/CodeGen/PowerPC/no-duplicate.ll     |    91 +
 .../CodeGen/PowerPC/not-fixed-frame-object.ll |     2 +-
 .../CodeGen/PowerPC/opt-cmp-inst-cr0-live.ll  |    20 +-
 llvm/test/CodeGen/PowerPC/popcnt-zext.ll      |   339 +
 llvm/test/CodeGen/PowerPC/popcnt.ll           |     8 +-
 .../PowerPC/power9-moves-and-splats.ll        |    36 +-
 llvm/test/CodeGen/PowerPC/ppc-empty-fs.ll     |     2 +-
 llvm/test/CodeGen/PowerPC/ppc-passname.ll     |    20 +-
 llvm/test/CodeGen/PowerPC/ppc32-lshrti3.ll    |     2 +-
 llvm/test/CodeGen/PowerPC/ppc64-P9-mod.ll     |    16 +-
 .../PowerPC/ppc64-align-long-double.ll        |    78 +-
 llvm/test/CodeGen/PowerPC/ppc64-elf-abi.ll    |     2 -
 .../PowerPC/ppc64-pre-inc-no-extra-phi.ll     |     2 +-
 llvm/test/CodeGen/PowerPC/ppc64-smallarg.ll   |     2 +-
 llvm/test/CodeGen/PowerPC/ppc64le-smallarg.ll |     2 +-
 .../ppcf128-constrained-fp-intrinsics.ll      |  1563 +++
 llvm/test/CodeGen/PowerPC/pr17168.ll          |     2 +-
 llvm/test/CodeGen/PowerPC/pr17354.ll          |     2 +-
 llvm/test/CodeGen/PowerPC/pr18663-2.ll        |     6 +-
 llvm/test/CodeGen/PowerPC/pr24546.ll          |     4 +-
 llvm/test/CodeGen/PowerPC/pr25080.ll          |   123 +
 llvm/test/CodeGen/PowerPC/pr26180.ll          |     4 +-
 llvm/test/CodeGen/PowerPC/pr27350.ll          |     2 +-
 llvm/test/CodeGen/PowerPC/pr28130.ll          |     2 +-
 llvm/test/CodeGen/PowerPC/pr31144.ll          |     4 +-
 llvm/test/CodeGen/PowerPC/pr32140.ll          |     8 +-
 llvm/test/CodeGen/PowerPC/pr36068.ll          |     2 +-
 llvm/test/CodeGen/PowerPC/pr36292.ll          |    14 +-
 llvm/test/CodeGen/PowerPC/pr38087.ll          |    14 +-
 llvm/test/CodeGen/PowerPC/pr39815.ll          |    11 +-
 llvm/test/CodeGen/PowerPC/pr41088.ll          |   141 +
 llvm/test/CodeGen/PowerPC/pr42492.ll          |    28 +-
 llvm/test/CodeGen/PowerPC/pr43527.ll          |    75 +
 llvm/test/CodeGen/PowerPC/pr44183.ll          |    56 +
 llvm/test/CodeGen/PowerPC/pr44239.ll          |    40 +
 llvm/test/CodeGen/PowerPC/pre-inc-disable.ll  |   223 +-
 llvm/test/CodeGen/PowerPC/pwr7-gt-nop.ll      |    12 +-
 llvm/test/CodeGen/PowerPC/qpx-load-splat.ll   |     7 +-
 llvm/test/CodeGen/PowerPC/qpx-recipest.ll     |   483 +-
 llvm/test/CodeGen/PowerPC/recipest.ll         |   295 +-
 .../CodeGen/PowerPC/reduce_scalarization02.ll |    87 +
 .../PowerPC/redundant-copy-after-tail-dup.ll  |     3 +-
 llvm/test/CodeGen/PowerPC/reloc-align.ll      |     2 +-
 .../PowerPC/remove-redundant-load-imm.ll      |    51 +
 .../CodeGen/PowerPC/repeated-fp-divisors.ll   |     4 +-
 llvm/test/CodeGen/PowerPC/rounding-ops.ll     |     8 +-
 llvm/test/CodeGen/PowerPC/sat-add.ll          |    68 +-
 llvm/test/CodeGen/PowerPC/save-bp.ll          |     2 +-
 .../test/CodeGen/PowerPC/save-cr-ppc32svr4.ll |     2 +-
 .../CodeGen/PowerPC/save-crbp-ppc32svr4.ll    |     2 +-
 llvm/test/CodeGen/PowerPC/scalar-min-max.ll   |   203 +
 .../CodeGen/PowerPC/scalar-rounding-ops.ll    |   561 +
 .../PowerPC/scheduling-mem-dependency.ll      |    47 +-
 .../CodeGen/PowerPC/select-addrRegRegOnly.ll  |     2 +-
 llvm/test/CodeGen/PowerPC/setcc-logic.ll      |     8 +-
 llvm/test/CodeGen/PowerPC/setrnd.ll           |     2 +-
 .../test/CodeGen/PowerPC/sext-vector-inreg.ll |    22 +
 llvm/test/CodeGen/PowerPC/shift-cmp.ll        |    16 +-
 llvm/test/CodeGen/PowerPC/shrink-wrap.ll      |    54 +
 llvm/test/CodeGen/PowerPC/signbit-shift.ll    |     4 +-
 llvm/test/CodeGen/PowerPC/sjlj.ll             |     4 +-
 llvm/test/CodeGen/PowerPC/sms-cpy-1.ll        |   113 +
 llvm/test/CodeGen/PowerPC/sms-grp-order.ll    |    37 +-
 llvm/test/CodeGen/PowerPC/sms-phi-1.ll        |    64 +
 llvm/test/CodeGen/PowerPC/sms-phi-2.ll        |    69 +
 llvm/test/CodeGen/PowerPC/sms-phi-3.ll        |    85 +
 llvm/test/CodeGen/PowerPC/sms-phi-5.ll        |    56 +
 llvm/test/CodeGen/PowerPC/sms-simple.ll       |     8 +-
 llvm/test/CodeGen/PowerPC/smulfixsat.ll       |    52 +
 llvm/test/CodeGen/PowerPC/spe.ll              |  1267 +-
 llvm/test/CodeGen/PowerPC/spill_p9_setb.ll    |    56 +
 llvm/test/CodeGen/PowerPC/srem-lkk.ll         |   149 +
 llvm/test/CodeGen/PowerPC/srem-vector-lkk.ll  |  1675 +++
 .../CodeGen/PowerPC/stack-guard-reassign.ll   |    11 +-
 llvm/test/CodeGen/PowerPC/store-combine.ll    |   124 +-
 llvm/test/CodeGen/PowerPC/store-constant.ll   |   203 +-
 llvm/test/CodeGen/PowerPC/store_fptoi.ll      |    32 +-
 llvm/test/CodeGen/PowerPC/stubs.ll            |     2 +-
 llvm/test/CodeGen/PowerPC/stwu-sched.ll       |     2 +-
 llvm/test/CodeGen/PowerPC/sub-of-not.ll       |    14 +-
 llvm/test/CodeGen/PowerPC/swaps-le-1.ll       |    40 +-
 llvm/test/CodeGen/PowerPC/swaps-le-7.ll       |     4 +-
 .../CodeGen/PowerPC/testComparesi32gtu.ll     |    72 +-
 .../CodeGen/PowerPC/testComparesi32ltu.ll     |    72 +-
 .../test/CodeGen/PowerPC/testComparesieqsc.ll |     2 +-
 .../test/CodeGen/PowerPC/testComparesieqsi.ll |     2 +-
 .../CodeGen/PowerPC/testComparesieqsll.ll     |     2 +-
 .../test/CodeGen/PowerPC/testComparesieqss.ll |     2 +-
 .../test/CodeGen/PowerPC/testComparesiequc.ll |     2 +-
 .../test/CodeGen/PowerPC/testComparesiequi.ll |     2 +-
 .../CodeGen/PowerPC/testComparesiequll.ll     |     2 +-
 .../test/CodeGen/PowerPC/testComparesiequs.ll |     2 +-
 .../test/CodeGen/PowerPC/testComparesigesc.ll |     2 +-
 .../test/CodeGen/PowerPC/testComparesigesi.ll |     2 +-
 .../CodeGen/PowerPC/testComparesigesll.ll     |     2 +-
 .../test/CodeGen/PowerPC/testComparesigess.ll |     2 +-
 .../test/CodeGen/PowerPC/testComparesigeuc.ll |   125 +-
 .../test/CodeGen/PowerPC/testComparesigeui.ll |   129 +-
 .../CodeGen/PowerPC/testComparesigeull.ll     |   128 +-
 .../test/CodeGen/PowerPC/testComparesigeus.ll |   127 +-
 .../test/CodeGen/PowerPC/testComparesigtsc.ll |    43 +-
 .../test/CodeGen/PowerPC/testComparesigtsi.ll |    43 +-
 .../CodeGen/PowerPC/testComparesigtsll.ll     |    73 +-
 .../test/CodeGen/PowerPC/testComparesigtss.ll |    40 +-
 .../test/CodeGen/PowerPC/testComparesigtuc.ll |    61 +-
 .../test/CodeGen/PowerPC/testComparesigtui.ll |    61 +-
 .../test/CodeGen/PowerPC/testComparesigtus.ll |    63 +-
 .../test/CodeGen/PowerPC/testComparesilesc.ll |     2 +-
 .../test/CodeGen/PowerPC/testComparesilesi.ll |     2 +-
 .../CodeGen/PowerPC/testComparesilesll.ll     |     2 +-
 .../test/CodeGen/PowerPC/testComparesiless.ll |     2 +-
 .../test/CodeGen/PowerPC/testComparesileuc.ll |   142 +-
 .../test/CodeGen/PowerPC/testComparesileui.ll |   142 +-
 .../CodeGen/PowerPC/testComparesileull.ll     |   137 +-
 .../test/CodeGen/PowerPC/testComparesileus.ll |   142 +-
 .../test/CodeGen/PowerPC/testComparesiltsc.ll |    79 +-
 .../test/CodeGen/PowerPC/testComparesiltsi.ll |    75 +-
 .../CodeGen/PowerPC/testComparesiltsll.ll     |   108 +-
 .../test/CodeGen/PowerPC/testComparesiltss.ll |    79 +-
 .../test/CodeGen/PowerPC/testComparesiltuc.ll |    57 +-
 .../test/CodeGen/PowerPC/testComparesiltui.ll |    57 +-
 .../test/CodeGen/PowerPC/testComparesiltus.ll |    57 +-
 .../test/CodeGen/PowerPC/testComparesinesc.ll |     2 +-
 .../test/CodeGen/PowerPC/testComparesinesi.ll |     2 +-
 .../CodeGen/PowerPC/testComparesinesll.ll     |     2 +-
 .../test/CodeGen/PowerPC/testComparesiness.ll |     2 +-
 .../test/CodeGen/PowerPC/testComparesineuc.ll |     2 +-
 .../test/CodeGen/PowerPC/testComparesineui.ll |     2 +-
 .../CodeGen/PowerPC/testComparesineull.ll     |     2 +-
 .../test/CodeGen/PowerPC/testComparesineus.ll |     2 +-
 .../CodeGen/PowerPC/testCompareslleqsc.ll     |     2 +-
 .../CodeGen/PowerPC/testCompareslleqsi.ll     |     2 +-
 .../CodeGen/PowerPC/testCompareslleqsll.ll    |     2 +-
 .../CodeGen/PowerPC/testCompareslleqss.ll     |     2 +-
 .../CodeGen/PowerPC/testComparesllequc.ll     |     2 +-
 .../CodeGen/PowerPC/testComparesllequi.ll     |     2 +-
 .../CodeGen/PowerPC/testComparesllequll.ll    |     2 +-
 .../CodeGen/PowerPC/testComparesllequs.ll     |     2 +-
 .../CodeGen/PowerPC/testComparesllgesc.ll     |     2 +-
 .../CodeGen/PowerPC/testComparesllgesi.ll     |     2 +-
 .../CodeGen/PowerPC/testComparesllgesll.ll    |     2 +-
 .../CodeGen/PowerPC/testComparesllgess.ll     |     2 +-
 .../CodeGen/PowerPC/testComparesllgeuc.ll     |   127 +-
 .../CodeGen/PowerPC/testComparesllgeui.ll     |   127 +-
 .../CodeGen/PowerPC/testComparesllgeull.ll    |   128 +-
 .../CodeGen/PowerPC/testComparesllgeus.ll     |   127 +-
 .../CodeGen/PowerPC/testComparesllgtsll.ll    |    73 +-
 .../CodeGen/PowerPC/testComparesllgtuc.ll     |    71 +-
 .../CodeGen/PowerPC/testComparesllgtui.ll     |    63 +-
 .../CodeGen/PowerPC/testComparesllgtus.ll     |    27 +-
 .../CodeGen/PowerPC/testCompareslllesc.ll     |     2 +-
 .../CodeGen/PowerPC/testCompareslllesi.ll     |     2 +-
 .../CodeGen/PowerPC/testCompareslllesll.ll    |     2 +-
 .../CodeGen/PowerPC/testComparesllless.ll     |     2 +-
 .../CodeGen/PowerPC/testComparesllleuc.ll     |   142 +-
 .../CodeGen/PowerPC/testComparesllleui.ll     |   142 +-
 .../CodeGen/PowerPC/testComparesllleull.ll    |   137 +-
 .../CodeGen/PowerPC/testComparesllleus.ll     |   142 +-
 .../CodeGen/PowerPC/testComparesllltsll.ll    |   108 +-
 .../CodeGen/PowerPC/testComparesllltuc.ll     |    57 +-
 .../CodeGen/PowerPC/testComparesllltui.ll     |    42 +-
 .../CodeGen/PowerPC/testComparesllltus.ll     |    56 +-
 .../CodeGen/PowerPC/testComparesllnesll.ll    |     2 +-
 .../CodeGen/PowerPC/testComparesllneull.ll    |     2 +-
 llvm/test/CodeGen/PowerPC/test_call_aix.ll    |    11 +
 llvm/test/CodeGen/PowerPC/test_func_desc.ll   |    74 +
 llvm/test/CodeGen/PowerPC/tls.ll              |     8 +-
 .../CodeGen/PowerPC/toc-load-sched-bug.ll     |    12 +-
 .../test/CodeGen/PowerPC/tocSaveInPrologue.ll |    17 +-
 llvm/test/CodeGen/PowerPC/trunc-srl-load.ll   |    22 +-
 llvm/test/CodeGen/PowerPC/uint-to-fp-v4i32.ll |    32 +-
 .../CodeGen/PowerPC/uint-to-ppcfp128-crash.ll |     2 +-
 llvm/test/CodeGen/PowerPC/umulfixsat.ll       |    36 +
 .../umulo-128-legalisation-lowering.ll        |   303 +-
 llvm/test/CodeGen/PowerPC/unal-vec-ldst.ll    |   573 +-
 llvm/test/CodeGen/PowerPC/unal4-std.ll        |     2 +-
 llvm/test/CodeGen/PowerPC/unaligned-floats.ll |    43 +
 llvm/test/CodeGen/PowerPC/urem-lkk.ll         |   106 +
 llvm/test/CodeGen/PowerPC/urem-vector-lkk.ll  |  1338 ++
 .../PowerPC/use-cr-result-of-dom-icmp-st.ll   |   480 +-
 llvm/test/CodeGen/PowerPC/uwtables.ll         |     2 +-
 llvm/test/CodeGen/PowerPC/vavg.ll             |   240 +
 llvm/test/CodeGen/PowerPC/vec-bswap.ll        |   115 +
 llvm/test/CodeGen/PowerPC/vec-min-max.ll      |    49 +
 .../CodeGen/PowerPC/vec_add_sub_doubleword.ll |     2 +-
 .../PowerPC/vec_buildvector_loadstore.ll      |    18 +-
 .../PowerPC/vec_conv_fp32_to_i64_elts.ll      |    32 +-
 .../PowerPC/vec_conv_fp64_to_i32_elts.ll      |    16 +-
 .../PowerPC/vec_conv_i16_to_fp32_elts.ll      |    24 +-
 .../PowerPC/vec_conv_i16_to_fp64_elts.ll      |     8 +-
 .../PowerPC/vec_conv_i64_to_fp32_elts.ll      |    24 +-
 .../PowerPC/vec_conv_i8_to_fp32_elts.ll       |    24 +-
 llvm/test/CodeGen/PowerPC/vec_revb.ll         |    17 +
 llvm/test/CodeGen/PowerPC/vec_shuffle.ll      |   126 +-
 llvm/test/CodeGen/PowerPC/vec_shuffle_le.ll   |   172 +-
 .../CodeGen/PowerPC/vec_shuffle_p8vector.ll   |    50 +-
 .../PowerPC/vec_shuffle_p8vector_le.ll        |    21 +-
 .../vector-constrained-fp-intrinsics.ll       |  1834 ++-
 .../CodeGen/PowerPC/vector-extend-sign.ll     |   178 +
 llvm/test/CodeGen/PowerPC/vector-rotates.ll   |   136 +
 .../test/CodeGen/PowerPC/vselect-constants.ll |     4 +-
 .../CodeGen/PowerPC/vsx-fma-mutate-undef.ll   |     4 +-
 .../CodeGen/PowerPC/vsx_insert_extract_le.ll  |    29 +-
 llvm/test/CodeGen/PowerPC/vsx_shuffle_le.ll   |     6 +-
 llvm/test/CodeGen/PowerPC/zero-not-run.ll     |     2 +-
 llvm/test/CodeGen/PowerPC/zext-and-cmp.ll     |     2 +-
 .../RISCV/GlobalISel/calllowering-ret.ll      |    17 +
 .../GlobalISel/irtranslator-calllowering.ll   |    17 +
 llvm/test/CodeGen/RISCV/add-before-shl.ll     |    34 +-
 .../test/CodeGen/RISCV/addc-adde-sube-subc.ll |     4 +-
 llvm/test/CodeGen/RISCV/addcarry.ll           |    28 +-
 llvm/test/CodeGen/RISCV/alloca.ll             |     4 +-
 llvm/test/CodeGen/RISCV/alu32.ll              |     4 +-
 llvm/test/CodeGen/RISCV/alu64.ll              |    16 +-
 .../test/CodeGen/RISCV/arith-with-overflow.ll |    32 +-
 .../test/CodeGen/RISCV/atomic-cmpxchg-flag.ll |     2 +-
 llvm/test/CodeGen/RISCV/atomic-cmpxchg.ll     |  1290 +-
 llvm/test/CodeGen/RISCV/atomic-rmw.ll         | 11356 ++++++++--------
 llvm/test/CodeGen/RISCV/bare-select.ll        |    16 +-
 llvm/test/CodeGen/RISCV/blockaddress.ll       |    10 +-
 .../CodeGen/RISCV/bswap-ctlz-cttz-ctpop.ll    |   112 +-
 .../test/CodeGen/RISCV/callee-saved-fpr32s.ll |    20 +-
 .../test/CodeGen/RISCV/callee-saved-fpr64s.ll |    19 +-
 llvm/test/CodeGen/RISCV/callee-saved-gprs.ll  |   173 +-
 .../RISCV/calling-conv-ilp32-ilp32f-common.ll |    44 +-
 ...calling-conv-ilp32-ilp32f-ilp32d-common.ll |   312 +-
 llvm/test/CodeGen/RISCV/calling-conv-ilp32.ll |    20 +-
 .../test/CodeGen/RISCV/calling-conv-ilp32d.ll |   144 +-
 .../calling-conv-ilp32f-ilp32d-common.ll      |    98 +-
 .../calling-conv-lp64-lp64f-lp64d-common.ll   |   168 +-
 llvm/test/CodeGen/RISCV/calling-conv-lp64.ll  |    26 +-
 .../CodeGen/RISCV/calling-conv-rv32f-ilp32.ll |    28 +-
 llvm/test/CodeGen/RISCV/codemodel-lowering.ll |    18 +-
 .../test/CodeGen/RISCV/compress-inline-asm.ll |     2 +-
 llvm/test/CodeGen/RISCV/compress.ll           |     6 +-
 llvm/test/CodeGen/RISCV/copysign-casts.ll     |   110 +
 llvm/test/CodeGen/RISCV/disjoint.ll           |    26 +
 llvm/test/CodeGen/RISCV/div.ll                |     2 +-
 llvm/test/CodeGen/RISCV/double-arith.ll       |    46 +-
 .../RISCV/double-bitmanip-dagcombines.ll      |     2 +-
 llvm/test/CodeGen/RISCV/double-br-fcmp.ll     |    16 +-
 .../test/CodeGen/RISCV/double-calling-conv.ll |    20 +-
 llvm/test/CodeGen/RISCV/double-convert.ll     |     6 +-
 llvm/test/CodeGen/RISCV/double-fcmp.ll        |    16 +-
 llvm/test/CodeGen/RISCV/double-imm.ll         |    10 +-
 llvm/test/CodeGen/RISCV/double-intrinsics.ll  |    55 +-
 llvm/test/CodeGen/RISCV/double-mem.ll         |    73 +-
 .../CodeGen/RISCV/double-previous-failure.ll  |     2 +-
 llvm/test/CodeGen/RISCV/double-select-fcmp.ll |    50 +-
 .../RISCV/double-stack-spill-restore.ll       |     6 +-
 .../RISCV/exception-pointer-register.ll       |    10 -
 llvm/test/CodeGen/RISCV/fastcc-float.ll       |    71 +
 llvm/test/CodeGen/RISCV/fastcc-int.ll         |    85 +
 llvm/test/CodeGen/RISCV/fixups-relax-diff.ll  |     2 +-
 llvm/test/CodeGen/RISCV/float-arith.ll        |    76 +-
 llvm/test/CodeGen/RISCV/float-br-fcmp.ll      |    32 +-
 llvm/test/CodeGen/RISCV/float-convert.ll      |    12 +-
 llvm/test/CodeGen/RISCV/float-fcmp.ll         |    32 +-
 llvm/test/CodeGen/RISCV/float-imm.ll          |    20 +-
 llvm/test/CodeGen/RISCV/float-intrinsics.ll   |    34 +-
 llvm/test/CodeGen/RISCV/float-mem.ll          |    84 +-
 llvm/test/CodeGen/RISCV/float-select-fcmp.ll  |    44 +-
 llvm/test/CodeGen/RISCV/fp128.ll              |    86 +-
 llvm/test/CodeGen/RISCV/fp16-promote.ll       |   142 +
 llvm/test/CodeGen/RISCV/frame-info.ll         |   215 +-
 llvm/test/CodeGen/RISCV/frame.ll              |     2 +-
 .../CodeGen/RISCV/get-register-invalid.ll     |    12 +
 .../CodeGen/RISCV/get-register-noreserve.ll   |    38 +
 .../CodeGen/RISCV/get-register-reserve.ll     |    34 +
 .../CodeGen/RISCV/get-setcc-result-type.ll    |    24 +-
 .../CodeGen/RISCV/hoist-global-addr-base.ll   |     4 +-
 llvm/test/CodeGen/RISCV/imm-cse.ll            |    24 +-
 llvm/test/CodeGen/RISCV/imm.ll                |     8 +-
 llvm/test/CodeGen/RISCV/indirectbr.ll         |     4 +-
 .../CodeGen/RISCV/inline-asm-abi-names.ll     |  1590 +++
 .../test/CodeGen/RISCV/inline-asm-clobbers.ll |    61 +
 .../CodeGen/RISCV/inline-asm-d-abi-names.ll   |  1509 ++
 .../RISCV/inline-asm-d-constraint-f.ll        |    41 +-
 .../CodeGen/RISCV/inline-asm-f-abi-names.ll   |  1509 ++
 .../RISCV/inline-asm-f-constraint-f.ll        |    43 +-
 llvm/test/CodeGen/RISCV/inline-asm.ll         |    75 +-
 .../CodeGen/RISCV/interrupt-attr-callee.ll    |    70 +
 .../CodeGen/RISCV/interrupt-attr-nocall.ll    |   716 +-
 llvm/test/CodeGen/RISCV/interrupt-attr.ll     |     2 +-
 llvm/test/CodeGen/RISCV/intrinsics/trap.ll    |    38 +
 llvm/test/CodeGen/RISCV/large-stack.ll        |   133 +-
 llvm/test/CodeGen/RISCV/legalize-fneg.ll      |    38 +-
 llvm/test/CodeGen/RISCV/lsr-legaladdimm.ll    |    24 +-
 llvm/test/CodeGen/RISCV/mem.ll                |    45 +-
 llvm/test/CodeGen/RISCV/mem64.ll              |    58 +-
 llvm/test/CodeGen/RISCV/mir-target-flags.ll   |    74 +
 llvm/test/CodeGen/RISCV/module-target-abi.ll  |    24 +
 llvm/test/CodeGen/RISCV/module-target-abi2.ll |    27 +
 llvm/test/CodeGen/RISCV/mul.ll                |     2 +-
 llvm/test/CodeGen/RISCV/option-norvc.ll       |     2 +-
 llvm/test/CodeGen/RISCV/option-rvc.ll         |     2 +-
 llvm/test/CodeGen/RISCV/remat.ll              |    87 +-
 .../test/CodeGen/RISCV/reserved-reg-errors.ll |    36 +
 llvm/test/CodeGen/RISCV/reserved-regs.ll      |   130 +
 .../CodeGen/RISCV/rv32i-rv64i-float-double.ll |     6 +-
 llvm/test/CodeGen/RISCV/rv64-large-stack.ll   |    34 +
 .../test/CodeGen/RISCV/rv64f-float-convert.ll |     6 +-
 .../test/CodeGen/RISCV/rv64i-complex-float.ll |    53 +
 .../CodeGen/RISCV/rv64i-exhaustive-w-insts.ll |    36 +-
 .../CodeGen/RISCV/rv64i-single-softfloat.ll   |   712 +
 .../RISCV/rv64i-w-insts-legalization.ll       |    97 +
 .../CodeGen/RISCV/rv64m-exhaustive-w-insts.ll |    18 +-
 .../RISCV/rv64m-w-insts-legalization.ll       |    34 +
 .../CodeGen/RISCV/select-optimize-multiple.ll |   154 +-
 llvm/test/CodeGen/RISCV/setcc-logic.ll        |    28 +-
 llvm/test/CodeGen/RISCV/shifts.ll             |   102 +-
 llvm/test/CodeGen/RISCV/shrinkwrap.ll         |    97 +
 llvm/test/CodeGen/RISCV/split-offsets.ll      |    62 +-
 llvm/test/CodeGen/RISCV/split-sp-adjust.ll    |    45 +
 llvm/test/CodeGen/RISCV/srem-lkk.ll           |   503 +
 llvm/test/CodeGen/RISCV/srem-vector-lkk.ll    |  1432 ++
 ...realignment-with-variable-sized-objects.ll |    72 +
 llvm/test/CodeGen/RISCV/stack-realignment.ll  |   204 +-
 .../RISCV/subtarget-features-std-ext.ll       |    20 +
 llvm/test/CodeGen/RISCV/tail-calls.ll         |    42 +-
 llvm/test/CodeGen/RISCV/tls-models.ll         |   103 +-
 .../RISCV/umulo-128-legalisation-lowering.ll  |   200 +-
 llvm/test/CodeGen/RISCV/urem-lkk.ll           |   300 +
 llvm/test/CodeGen/RISCV/urem-vector-lkk.ll    |  1187 ++
 llvm/test/CodeGen/RISCV/vararg.ll             |   500 +-
 .../CodeGen/RISCV/zext-with-load-is-free.ll   |     8 +-
 llvm/test/CodeGen/SPARC/64atomics.ll          |    60 +
 llvm/test/CodeGen/SPARC/atomics.ll            |    60 +-
 llvm/test/CodeGen/SPARC/exception.ll          |     2 +-
 llvm/test/CodeGen/SPARC/fp128.ll              |    11 +-
 llvm/test/CodeGen/SPARC/inlineasm.ll          |     2 +-
 llvm/test/CodeGen/SystemZ/Large/branch-01.ll  |     4 +-
 llvm/test/CodeGen/SystemZ/anyregcc-novec.ll   |    17 +-
 llvm/test/CodeGen/SystemZ/call-05.ll          |     3 +-
 llvm/test/CodeGen/SystemZ/cond-move-01.ll     |     2 +-
 llvm/test/CodeGen/SystemZ/cond-move-02.ll     |     2 +-
 llvm/test/CodeGen/SystemZ/cond-move-03.ll     |     2 +-
 llvm/test/CodeGen/SystemZ/cond-move-06.ll     |     2 +-
 llvm/test/CodeGen/SystemZ/cond-move-07.ll     |     2 +-
 llvm/test/CodeGen/SystemZ/ctpop-02.ll         |     4 +-
 llvm/test/CodeGen/SystemZ/fentry-insertion.ll |    29 +
 llvm/test/CodeGen/SystemZ/fp-abs-02.ll        |     6 +-
 llvm/test/CodeGen/SystemZ/fp-abs-04.ll        |     6 +-
 llvm/test/CodeGen/SystemZ/fp-cmp-04.ll        |     8 +-
 llvm/test/CodeGen/SystemZ/fp-cmp-05.ll        |     8 +-
 llvm/test/CodeGen/SystemZ/fp-libcall.ll       |    62 +
 llvm/test/CodeGen/SystemZ/fp-mul-08.ll        |    16 +-
 llvm/test/CodeGen/SystemZ/fp-mul-09.ll        |    16 +-
 llvm/test/CodeGen/SystemZ/fp-mul-10.ll        |    12 +-
 llvm/test/CodeGen/SystemZ/fp-mul-13.ll        |    16 +
 llvm/test/CodeGen/SystemZ/fp-neg-01.ll        |     6 +-
 llvm/test/CodeGen/SystemZ/fp-neg-02.ll        |     6 +-
 llvm/test/CodeGen/SystemZ/fp-strict-add-01.ll |    52 +-
 llvm/test/CodeGen/SystemZ/fp-strict-add-02.ll |    52 +-
 llvm/test/CodeGen/SystemZ/fp-strict-add-03.ll |    10 +-
 llvm/test/CodeGen/SystemZ/fp-strict-add-04.ll |     4 +-
 llvm/test/CodeGen/SystemZ/fp-strict-alias.ll  |   242 +-
 llvm/test/CodeGen/SystemZ/fp-strict-cmp-01.ll |   435 +
 llvm/test/CodeGen/SystemZ/fp-strict-cmp-02.ll |   249 +
 llvm/test/CodeGen/SystemZ/fp-strict-cmp-03.ll |    47 +
 llvm/test/CodeGen/SystemZ/fp-strict-cmp-04.ll |   524 +
 llvm/test/CodeGen/SystemZ/fp-strict-cmp-05.ll |   103 +
 llvm/test/CodeGen/SystemZ/fp-strict-cmp-06.ll |    44 +
 .../test/CodeGen/SystemZ/fp-strict-cmps-01.ll |   436 +
 .../test/CodeGen/SystemZ/fp-strict-cmps-02.ll |   249 +
 .../test/CodeGen/SystemZ/fp-strict-cmps-03.ll |    48 +
 .../test/CodeGen/SystemZ/fp-strict-cmps-04.ll |   148 +
 .../test/CodeGen/SystemZ/fp-strict-cmps-05.ll |   103 +
 .../test/CodeGen/SystemZ/fp-strict-cmps-06.ll |    44 +
 .../test/CodeGen/SystemZ/fp-strict-conv-01.ll |    26 +-
 .../test/CodeGen/SystemZ/fp-strict-conv-02.ll |    25 +-
 .../test/CodeGen/SystemZ/fp-strict-conv-03.ll |    25 +-
 .../test/CodeGen/SystemZ/fp-strict-conv-04.ll |    25 +-
 .../test/CodeGen/SystemZ/fp-strict-conv-05.ll |    45 +
 .../test/CodeGen/SystemZ/fp-strict-conv-06.ll |    49 +
 .../test/CodeGen/SystemZ/fp-strict-conv-07.ll |    45 +
 .../test/CodeGen/SystemZ/fp-strict-conv-08.ll |    47 +
 .../test/CodeGen/SystemZ/fp-strict-conv-09.ll |    42 +
 .../test/CodeGen/SystemZ/fp-strict-conv-10.ll |    91 +
 .../test/CodeGen/SystemZ/fp-strict-conv-11.ll |    42 +
 .../test/CodeGen/SystemZ/fp-strict-conv-12.ll |    90 +
 .../test/CodeGen/SystemZ/fp-strict-conv-13.ll |    86 +
 .../test/CodeGen/SystemZ/fp-strict-conv-14.ll |    79 +
 .../test/CodeGen/SystemZ/fp-strict-conv-15.ll |    17 +-
 .../test/CodeGen/SystemZ/fp-strict-conv-16.ll |   125 +
 llvm/test/CodeGen/SystemZ/fp-strict-div-01.ll |    52 +-
 llvm/test/CodeGen/SystemZ/fp-strict-div-02.ll |    52 +-
 llvm/test/CodeGen/SystemZ/fp-strict-div-03.ll |    10 +-
 llvm/test/CodeGen/SystemZ/fp-strict-div-04.ll |     4 +-
 llvm/test/CodeGen/SystemZ/fp-strict-mul-01.ll |    52 +-
 llvm/test/CodeGen/SystemZ/fp-strict-mul-02.ll |   307 +-
 llvm/test/CodeGen/SystemZ/fp-strict-mul-03.ll |    52 +-
 llvm/test/CodeGen/SystemZ/fp-strict-mul-04.ll |   337 +-
 llvm/test/CodeGen/SystemZ/fp-strict-mul-05.ll |    10 +-
 llvm/test/CodeGen/SystemZ/fp-strict-mul-06.ll |    34 +-
 llvm/test/CodeGen/SystemZ/fp-strict-mul-07.ll |    34 +-
 llvm/test/CodeGen/SystemZ/fp-strict-mul-08.ll |    50 +-
 llvm/test/CodeGen/SystemZ/fp-strict-mul-09.ll |    50 +-
 llvm/test/CodeGen/SystemZ/fp-strict-mul-10.ll |    29 +-
 llvm/test/CodeGen/SystemZ/fp-strict-mul-11.ll |    16 +-
 llvm/test/CodeGen/SystemZ/fp-strict-mul-12.ll |    86 +
 llvm/test/CodeGen/SystemZ/fp-strict-mul-13.ll |    21 +
 .../CodeGen/SystemZ/fp-strict-round-01.ll     |   109 +-
 .../CodeGen/SystemZ/fp-strict-round-02.ll     |   109 +-
 .../CodeGen/SystemZ/fp-strict-round-03.ll     |   109 +-
 .../test/CodeGen/SystemZ/fp-strict-sqrt-01.ll |    25 +-
 .../test/CodeGen/SystemZ/fp-strict-sqrt-02.ll |    25 +-
 .../test/CodeGen/SystemZ/fp-strict-sqrt-03.ll |     4 +-
 .../test/CodeGen/SystemZ/fp-strict-sqrt-04.ll |     4 +-
 llvm/test/CodeGen/SystemZ/fp-strict-sub-01.ll |    52 +-
 llvm/test/CodeGen/SystemZ/fp-strict-sub-02.ll |    52 +-
 llvm/test/CodeGen/SystemZ/fp-strict-sub-03.ll |    10 +-
 llvm/test/CodeGen/SystemZ/fp-strict-sub-04.ll |     4 +-
 llvm/test/CodeGen/SystemZ/frame-02.ll         |    86 +-
 llvm/test/CodeGen/SystemZ/frame-03.ll         |    86 +-
 llvm/test/CodeGen/SystemZ/frame-04.ll         |    86 +-
 llvm/test/CodeGen/SystemZ/frame-19.ll         |    11 +-
 llvm/test/CodeGen/SystemZ/frame-20.ll         |    86 +-
 llvm/test/CodeGen/SystemZ/frame-22.ll         |    87 +
 llvm/test/CodeGen/SystemZ/ghc-cc-01.ll        |   103 +
 llvm/test/CodeGen/SystemZ/ghc-cc-02.ll        |    14 +
 llvm/test/CodeGen/SystemZ/ghc-cc-03.ll        |    11 +
 llvm/test/CodeGen/SystemZ/ghc-cc-04.ll        |    16 +
 llvm/test/CodeGen/SystemZ/ghc-cc-05.ll        |    16 +
 llvm/test/CodeGen/SystemZ/ghc-cc-06.ll        |    12 +
 llvm/test/CodeGen/SystemZ/ghc-cc-07.ll        |    12 +
 llvm/test/CodeGen/SystemZ/int-cmp-44.ll       |    59 +-
 llvm/test/CodeGen/SystemZ/int-cmp-45.ll       |    27 +-
 llvm/test/CodeGen/SystemZ/int-cmp-56.ll       |   163 +
 llvm/test/CodeGen/SystemZ/int-cmp-57.ll       |   103 +
 llvm/test/CodeGen/SystemZ/int-uadd-03.ll      |     2 +-
 llvm/test/CodeGen/SystemZ/int-usub-03.ll      |     2 +-
 llvm/test/CodeGen/SystemZ/ipra-04.ll          |     2 +-
 llvm/test/CodeGen/SystemZ/la-05.ll            |    31 +
 .../test/CodeGen/SystemZ/mature-mc-support.ll |     8 +-
 llvm/test/CodeGen/SystemZ/mnop-mcount-01.ll   |    26 +
 llvm/test/CodeGen/SystemZ/mnop-mcount-02.ll   |    10 +
 .../test/CodeGen/SystemZ/mrecord-mcount-01.ll |    32 +
 .../test/CodeGen/SystemZ/mrecord-mcount-02.ll |    10 +
 llvm/test/CodeGen/SystemZ/multiselect.ll      |    55 +-
 llvm/test/CodeGen/SystemZ/not-01.ll           |    30 +-
 .../CodeGen/SystemZ/stack-size-section.ll     |     2 +-
 llvm/test/CodeGen/SystemZ/stackmap-nops.ll    |   153 +-
 .../SystemZ/store_nonbytesized_vecs.ll        |    77 +-
 .../test/CodeGen/SystemZ/subregliveness-01.ll |     2 +-
 .../test/CodeGen/SystemZ/subregliveness-02.ll |     2 +-
 .../test/CodeGen/SystemZ/subregliveness-03.ll |     4 +-
 .../test/CodeGen/SystemZ/subregliveness-04.ll |     2 +-
 .../test/CodeGen/SystemZ/subregliveness-05.ll |     2 +-
 .../SystemZ/tail-call-mem-intrinsics.ll       |     2 +-
 llvm/test/CodeGen/SystemZ/vec-abs-05.ll       |     4 +-
 llvm/test/CodeGen/SystemZ/vec-abs-06.ll       |     5 +-
 llvm/test/CodeGen/SystemZ/vec-bswap-01.ll     |     2 +-
 llvm/test/CodeGen/SystemZ/vec-bswap-02.ll     |     2 +-
 llvm/test/CodeGen/SystemZ/vec-bswap-03.ll     |     2 +-
 llvm/test/CodeGen/SystemZ/vec-bswap-04.ll     |     2 +-
 llvm/test/CodeGen/SystemZ/vec-bswap-05.ll     |     2 +-
 llvm/test/CodeGen/SystemZ/vec-bswap-06.ll     |     2 +-
 llvm/test/CodeGen/SystemZ/vec-bswap-07.ll     |     2 +-
 llvm/test/CodeGen/SystemZ/vec-conv-03.ll      |     4 +-
 llvm/test/CodeGen/SystemZ/vec-eswap-01.ll     |     2 +-
 llvm/test/CodeGen/SystemZ/vec-eswap-02.ll     |     2 +-
 .../test/CodeGen/SystemZ/vec-intrinsics-03.ll |     4 +-
 llvm/test/CodeGen/SystemZ/vec-max-05.ll       |    90 +-
 llvm/test/CodeGen/SystemZ/vec-min-05.ll       |    90 +-
 llvm/test/CodeGen/SystemZ/vec-mul-02.ll       |     2 +-
 llvm/test/CodeGen/SystemZ/vec-mul-04.ll       |     3 +-
 llvm/test/CodeGen/SystemZ/vec-mul-05.ll       |    15 +-
 llvm/test/CodeGen/SystemZ/vec-neg-01.ll       |     4 +-
 llvm/test/CodeGen/SystemZ/vec-neg-02.ll       |     5 +-
 .../test/CodeGen/SystemZ/vec-strict-add-01.ll |     8 +-
 .../test/CodeGen/SystemZ/vec-strict-add-02.ll |     8 +-
 .../test/CodeGen/SystemZ/vec-strict-cmp-01.ll |   560 +
 .../test/CodeGen/SystemZ/vec-strict-cmp-02.ll |   442 +
 .../test/CodeGen/SystemZ/vec-strict-cmp-03.ll |   442 +
 .../CodeGen/SystemZ/vec-strict-cmps-01.ll     |   442 +
 .../CodeGen/SystemZ/vec-strict-cmps-02.ll     |   442 +
 .../CodeGen/SystemZ/vec-strict-cmps-03.ll     |    56 +
 .../CodeGen/SystemZ/vec-strict-conv-01.ll     |   130 +
 .../CodeGen/SystemZ/vec-strict-conv-02.ll     |    61 +
 .../CodeGen/SystemZ/vec-strict-conv-03.ll     |    52 +
 .../test/CodeGen/SystemZ/vec-strict-div-01.ll |    10 +-
 .../test/CodeGen/SystemZ/vec-strict-div-02.ll |    10 +-
 .../test/CodeGen/SystemZ/vec-strict-max-01.ll |   104 +-
 .../test/CodeGen/SystemZ/vec-strict-min-01.ll |   104 +-
 .../test/CodeGen/SystemZ/vec-strict-mul-01.ll |    10 +-
 .../test/CodeGen/SystemZ/vec-strict-mul-02.ll |    12 +-
 .../test/CodeGen/SystemZ/vec-strict-mul-03.ll |    10 +-
 .../test/CodeGen/SystemZ/vec-strict-mul-04.ll |    13 +-
 .../test/CodeGen/SystemZ/vec-strict-mul-05.ll |    33 +-
 .../CodeGen/SystemZ/vec-strict-round-01.ll    |    74 +-
 .../CodeGen/SystemZ/vec-strict-round-02.ll    |    74 +-
 .../CodeGen/SystemZ/vec-strict-sqrt-01.ll     |    10 +-
 .../CodeGen/SystemZ/vec-strict-sqrt-02.ll     |    10 +-
 .../test/CodeGen/SystemZ/vec-strict-sub-01.ll |     9 +-
 .../test/CodeGen/SystemZ/vec-strict-sub-02.ll |    10 +-
 .../vector-constrained-fp-intrinsics.ll       |  1172 +-
 llvm/test/CodeGen/Thumb/PR17309.ll            |     4 +-
 llvm/test/CodeGen/Thumb/PR35481.ll            |     2 +-
 llvm/test/CodeGen/Thumb/callee_save.ll        |     6 +-
 llvm/test/CodeGen/Thumb/cmp-and-fold.ll       |   208 +
 llvm/test/CodeGen/Thumb/fastcc.ll             |     2 +-
 llvm/test/CodeGen/Thumb/large-stack.ll        |     6 +-
 llvm/test/CodeGen/Thumb/ldm-merge-call.ll     |     4 +-
 llvm/test/CodeGen/Thumb/rev.ll                |    25 +-
 .../CodeGen/Thumb/scheduler-clone-cpsr-def.ll |    41 +
 llvm/test/CodeGen/Thumb/stack_guard_remat.ll  |    41 +-
 llvm/test/CodeGen/Thumb/stm-merge.ll          |     2 +-
 llvm/test/CodeGen/Thumb/stm-scavenging.ll     |     4 +-
 .../test/CodeGen/Thumb2/2009-07-21-ISelBug.ll |     2 +-
 .../CodeGen/Thumb2/2010-02-11-phi-cycle.ll    |     9 +-
 .../CodeGen/Thumb2/2010-03-15-AsmCCClobber.ll |     8 +-
 .../CodeGen/Thumb2/2010-11-22-EpilogueBug.ll  |     2 +-
 .../LowOverheadLoops/basic-tail-pred.ll       |   337 +
 .../Thumb2/LowOverheadLoops/branch-targets.ll |   512 +
 .../cond-vector-reduce-mve-codegen.ll         |   520 +
 .../Thumb2/LowOverheadLoops/fast-fp-loops.ll  |   592 +
 .../LowOverheadLoops/mve-float-loops.ll       |  2008 +++
 .../LowOverheadLoops/mve-tail-data-types.ll   |  1230 ++
 .../CodeGen/Thumb2/LowOverheadLoops/nested.ll |   241 +
 .../LowOverheadLoops/no-dec-le-simple.ll      |   158 +
 .../LowOverheadLoops/tail-pred-narrow.ll      |    54 +
 .../tail-pred-pattern-fail.ll                 |   505 +
 .../LowOverheadLoops/tail-pred-widen.ll       |   173 +
 .../Thumb2/LowOverheadLoops/tail-reduce.ll    |   118 +
 .../LowOverheadLoops/vector-arith-codegen.ll  |   411 +
 .../vector-reduce-mve-tail.ll                 |    75 +
 .../Thumb2/LowOverheadLoops/vector-unroll.ll  |   118 +
 llvm/test/CodeGen/Thumb2/aligned-spill.ll     |     6 +-
 llvm/test/CodeGen/Thumb2/bug-subw.ll          |    74 +
 llvm/test/CodeGen/Thumb2/cbnz.ll              |     4 +-
 llvm/test/CodeGen/Thumb2/constant-islands.ll  |     2 +-
 llvm/test/CodeGen/Thumb2/csel.ll              |   340 +
 llvm/test/CodeGen/Thumb2/emit-unwinding.ll    |    15 +
 llvm/test/CodeGen/Thumb2/float-cmp.ll         |    52 +-
 .../CodeGen/Thumb2/float-intrinsics-double.ll |     2 +-
 .../CodeGen/Thumb2/float-intrinsics-float.ll  |     2 +-
 llvm/test/CodeGen/Thumb2/float-ops.ll         |     2 +-
 llvm/test/CodeGen/Thumb2/frame-pointer.ll     |    16 +-
 llvm/test/CodeGen/Thumb2/large-call.ll        |     2 +-
 llvm/test/CodeGen/Thumb2/lsll0.ll             |    47 +
 llvm/test/CodeGen/Thumb2/mve-abs.ll           |    63 +-
 llvm/test/CodeGen/Thumb2/mve-be.ll            |   297 +
 llvm/test/CodeGen/Thumb2/mve-bitreverse.ll    |    52 +
 llvm/test/CodeGen/Thumb2/mve-bswap.ll         |    37 +
 llvm/test/CodeGen/Thumb2/mve-ctlz.ll          |   140 +
 llvm/test/CodeGen/Thumb2/mve-ctpop.ll         |   151 +
 llvm/test/CodeGen/Thumb2/mve-cttz.ll          |   160 +
 llvm/test/CodeGen/Thumb2/mve-div-expand.ll    |   150 +-
 llvm/test/CodeGen/Thumb2/mve-extractelt.ll    |    86 +
 llvm/test/CodeGen/Thumb2/mve-fmas.ll          |   271 +
 llvm/test/CodeGen/Thumb2/mve-fmath.ll         |   594 +-
 llvm/test/CodeGen/Thumb2/mve-fp-negabs.ll     |    68 +-
 llvm/test/CodeGen/Thumb2/mve-frint.ll         |   204 +-
 .../CodeGen/Thumb2/mve-gather-ind16-scaled.ll |   271 +
 .../Thumb2/mve-gather-ind16-unscaled.ll       |   242 +
 .../CodeGen/Thumb2/mve-gather-ind32-scaled.ll |   441 +
 .../Thumb2/mve-gather-ind32-unscaled.ll       |   757 +
 .../Thumb2/mve-gather-ind8-unscaled.ll        |   370 +
 llvm/test/CodeGen/Thumb2/mve-gather-ptrs.ll   |   863 ++
 .../CodeGen/Thumb2/mve-gather-scatter-opt.ll  |    46 +
 .../Thumb2/mve-intrinsics/load-store.ll       |  1208 ++
 .../Thumb2/mve-intrinsics/predicates.ll       |   219 +
 .../Thumb2/mve-intrinsics/scalar-shifts.ll    |   280 +
 .../Thumb2/mve-intrinsics/scatter-gather.ll   |  2018 +++
 .../CodeGen/Thumb2/mve-intrinsics/vabavq.ll   |   158 +
 .../CodeGen/Thumb2/mve-intrinsics/vabdq.ll    |   139 +
 .../Thumb2/mve-intrinsics/vadc-multiple.ll    |    87 +
 .../CodeGen/Thumb2/mve-intrinsics/vadc.ll     |    98 +
 .../CodeGen/Thumb2/mve-intrinsics/vaddq.ll    |    93 +
 .../CodeGen/Thumb2/mve-intrinsics/vandq.ll    |   177 +
 .../CodeGen/Thumb2/mve-intrinsics/vbicq.ll    |   181 +
 .../CodeGen/Thumb2/mve-intrinsics/vcaddq.ll   |   870 ++
 .../CodeGen/Thumb2/mve-intrinsics/vcmlaq.ll   |   205 +
 .../CodeGen/Thumb2/mve-intrinsics/vcmulq.ll   |   323 +
 .../CodeGen/Thumb2/mve-intrinsics/vcvt.ll     |    56 +
 .../mve-intrinsics/vector-shift-imm-dyadic.ll |  1270 ++
 .../Thumb2/mve-intrinsics/vector-shift-imm.ll |  1476 ++
 .../Thumb2/mve-intrinsics/vector-shift-var.ll |  1338 ++
 .../CodeGen/Thumb2/mve-intrinsics/veorq.ll    |   177 +
 .../CodeGen/Thumb2/mve-intrinsics/vhaddq.ll   |   135 +
 .../CodeGen/Thumb2/mve-intrinsics/vhsubq.ll   |    92 +
 .../CodeGen/Thumb2/mve-intrinsics/vld24.ll    |   109 +
 .../CodeGen/Thumb2/mve-intrinsics/vldr.ll     |    62 +
 .../CodeGen/Thumb2/mve-intrinsics/vmaxnmq.ll  |    91 +
 .../CodeGen/Thumb2/mve-intrinsics/vmaxq.ll    |   132 +
 .../CodeGen/Thumb2/mve-intrinsics/vminnmq.ll  |    91 +
 .../CodeGen/Thumb2/mve-intrinsics/vminq.ll    |   132 +
 .../CodeGen/Thumb2/mve-intrinsics/vminvq.ll   |    36 +
 .../CodeGen/Thumb2/mve-intrinsics/vmldav.ll   |   734 +
 .../CodeGen/Thumb2/mve-intrinsics/vmlldav.ll  |  1183 ++
 .../CodeGen/Thumb2/mve-intrinsics/vmulhq.ll   |   135 +
 .../CodeGen/Thumb2/mve-intrinsics/vmullbq.ll  |   181 +
 .../CodeGen/Thumb2/mve-intrinsics/vmulltq.ll  |   179 +
 .../CodeGen/Thumb2/mve-intrinsics/vmulq.ll    |   171 +
 .../CodeGen/Thumb2/mve-intrinsics/vornq.ll    |   181 +
 .../CodeGen/Thumb2/mve-intrinsics/vorrq.ll    |   177 +
 .../CodeGen/Thumb2/mve-intrinsics/vqaddq.ll   |    92 +
 .../CodeGen/Thumb2/mve-intrinsics/vqdmulhq.ll |    92 +
 .../Thumb2/mve-intrinsics/vqrdmulhq.ll        |    92 +
 .../CodeGen/Thumb2/mve-intrinsics/vqsubq.ll   |    92 +
 .../CodeGen/Thumb2/mve-intrinsics/vrhaddq.ll  |   135 +
 .../CodeGen/Thumb2/mve-intrinsics/vrmulhq.ll  |   135 +
 .../CodeGen/Thumb2/mve-intrinsics/vsubq.ll    |    93 +
 llvm/test/CodeGen/Thumb2/mve-ldst-offset.ll   |  1454 +-
 llvm/test/CodeGen/Thumb2/mve-ldst-postinc.ll  |  1621 ++-
 llvm/test/CodeGen/Thumb2/mve-ldst-preinc.ll   |  1620 ++-
 llvm/test/CodeGen/Thumb2/mve-ldst-regimm.ll   |    10 +-
 llvm/test/CodeGen/Thumb2/mve-loadstore.ll     |   276 +-
 .../CodeGen/Thumb2/mve-masked-ldst-offset.ll  |  2646 ++++
 .../CodeGen/Thumb2/mve-masked-ldst-postinc.ll |  2646 ++++
 .../CodeGen/Thumb2/mve-masked-ldst-preinc.ll  |  2646 ++++
 llvm/test/CodeGen/Thumb2/mve-masked-ldst.ll   |  1151 ++
 llvm/test/CodeGen/Thumb2/mve-masked-load.ll   |  2196 +++
 llvm/test/CodeGen/Thumb2/mve-masked-store.ll  |  1819 +++
 llvm/test/CodeGen/Thumb2/mve-minmax.ll        |    94 +-
 .../test/CodeGen/Thumb2/mve-multivec-spill.ll |   103 +
 llvm/test/CodeGen/Thumb2/mve-phireg.ll        |   285 +
 llvm/test/CodeGen/Thumb2/mve-pred-and.ll      |   714 +
 llvm/test/CodeGen/Thumb2/mve-pred-bitcast.ll  |   431 +
 .../CodeGen/Thumb2/mve-pred-build-const.ll    |   196 +
 .../test/CodeGen/Thumb2/mve-pred-build-var.ll |   246 +
 llvm/test/CodeGen/Thumb2/mve-pred-ext.ll      |   335 +
 .../test/CodeGen/Thumb2/mve-pred-loadstore.ll |   355 +
 llvm/test/CodeGen/Thumb2/mve-pred-not.ll      |   402 +
 llvm/test/CodeGen/Thumb2/mve-pred-or.ll       |   508 +
 llvm/test/CodeGen/Thumb2/mve-pred-shuffle.ll  |   563 +
 llvm/test/CodeGen/Thumb2/mve-pred-spill.ll    |   165 +
 .../test/CodeGen/Thumb2/mve-pred-threshold.ll |   615 +
 llvm/test/CodeGen/Thumb2/mve-pred-xor.ll      |   561 +
 .../CodeGen/Thumb2/mve-saturating-arith.ll    |   408 +
 llvm/test/CodeGen/Thumb2/mve-sext.ll          |   462 +-
 llvm/test/CodeGen/Thumb2/mve-shifts-scalar.ll |   422 +
 llvm/test/CodeGen/Thumb2/mve-shifts.ll        |    30 +-
 llvm/test/CodeGen/Thumb2/mve-shuffle.ll       |    73 +-
 llvm/test/CodeGen/Thumb2/mve-shuffleext.ll    |    94 +
 llvm/test/CodeGen/Thumb2/mve-shufflemov.ll    |   442 +
 llvm/test/CodeGen/Thumb2/mve-simple-arith.ll  |   162 +-
 llvm/test/CodeGen/Thumb2/mve-stack.ll         |   278 +
 llvm/test/CodeGen/Thumb2/mve-vaddqr.ll        |    74 +
 llvm/test/CodeGen/Thumb2/mve-vaddv.ll         |   105 +
 llvm/test/CodeGen/Thumb2/mve-vcmp.ll          |   537 +
 llvm/test/CodeGen/Thumb2/mve-vcmpf.ll         |  2541 ++++
 llvm/test/CodeGen/Thumb2/mve-vcmpfr.ll        |  5222 +++++++
 llvm/test/CodeGen/Thumb2/mve-vcmpfz.ll        |  4970 +++++++
 llvm/test/CodeGen/Thumb2/mve-vcmpr.ll         |  1186 ++
 llvm/test/CodeGen/Thumb2/mve-vcmpz.ll         |   835 ++
 llvm/test/CodeGen/Thumb2/mve-vctp.ll          |    54 +
 llvm/test/CodeGen/Thumb2/mve-vcvt.ll          |    64 +-
 .../CodeGen/Thumb2/mve-vecreduce-fminmax.ll   |  2264 +++
 llvm/test/CodeGen/Thumb2/mve-vector-spill.ll  |   163 +
 llvm/test/CodeGen/Thumb2/mve-vfma.ll          |    25 +
 llvm/test/CodeGen/Thumb2/mve-vhaddsub.ll      |   305 +
 llvm/test/CodeGen/Thumb2/mve-vld2.ll          |   615 +
 llvm/test/CodeGen/Thumb2/mve-vld3.ll          |  1745 +++
 llvm/test/CodeGen/Thumb2/mve-vld4.ll          |  1235 ++
 llvm/test/CodeGen/Thumb2/mve-vldst4.ll        |   284 +
 llvm/test/CodeGen/Thumb2/mve-vmaxv.ll         |   136 +
 llvm/test/CodeGen/Thumb2/mve-vmla.ll          |   200 +
 llvm/test/CodeGen/Thumb2/mve-vmovn.ll         |   422 +
 llvm/test/CodeGen/Thumb2/mve-vmulqr.ll        |    74 +
 llvm/test/CodeGen/Thumb2/mve-vpsel.ll         |    77 +
 .../CodeGen/Thumb2/mve-vpt-from-intrinsics.ll |    45 +
 llvm/test/CodeGen/Thumb2/mve-vst2.ll          |   606 +
 llvm/test/CodeGen/Thumb2/mve-vst3.ll          |  1967 +++
 llvm/test/CodeGen/Thumb2/mve-vst4.ll          |  1126 ++
 llvm/test/CodeGen/Thumb2/mve-vsubqr.ll        |    77 +
 llvm/test/CodeGen/Thumb2/mve-widen-narrow.ll  |   312 +-
 llvm/test/CodeGen/Thumb2/segmented-stacks.ll  |    86 +-
 llvm/test/CodeGen/Thumb2/stack_guard_remat.ll |     2 +-
 .../Thumb2/t2peephole-t2ADDrr-to-t2ADDri.ll   |    10 +
 llvm/test/CodeGen/Thumb2/thumb2-ldm.ll        |     8 +-
 llvm/test/CodeGen/Thumb2/thumb2-uxtb.ll       |   163 +-
 llvm/test/CodeGen/Thumb2/vqabs.ll             |    50 +
 llvm/test/CodeGen/Thumb2/vqneg.ll             |    44 +
 .../CodeGen/VE/simple_prologue_epilogue.ll    |    32 +
 llvm/test/CodeGen/VE/target_support.ll        |     2 +
 llvm/test/CodeGen/WebAssembly/atomic-fence.ll |    23 +-
 llvm/test/CodeGen/WebAssembly/bulk-memory.ll  |    40 +-
 .../CodeGen/WebAssembly/cfg-stackify-eh.ll    |   135 +-
 llvm/test/CodeGen/WebAssembly/conv-trap.ll    |    80 +
 llvm/test/CodeGen/WebAssembly/cpus.ll         |     9 +-
 llvm/test/CodeGen/WebAssembly/exception.ll    |     3 +-
 llvm/test/CodeGen/WebAssembly/export-name.ll  |    17 +
 .../CodeGen/WebAssembly/function-bitcasts.ll  |    19 +-
 .../CodeGen/WebAssembly/indirect-import.ll    |     4 +-
 .../CodeGen/WebAssembly/llround-conv-i32.ll   |     4 +-
 .../lower-em-exceptions-resume-only.ll        |    17 +
 .../lower-em-exceptions-whitelist.ll          |     2 +-
 .../WebAssembly/lower-em-exceptions.ll        |     6 +-
 .../WebAssembly/lower-em-sjlj-alias.ll        |    43 +
 .../CodeGen/WebAssembly/lower-em-sjlj-sret.ll |    27 +
 .../test/CodeGen/WebAssembly/lower-em-sjlj.ll |    55 +-
 llvm/test/CodeGen/WebAssembly/multivalue.ll   |    49 +-
 .../CodeGen/WebAssembly/negative-base-reg.ll  |     2 +-
 llvm/test/CodeGen/WebAssembly/reg-stackify.ll |     2 +-
 llvm/test/CodeGen/WebAssembly/select.ll       |    75 +-
 llvm/test/CodeGen/WebAssembly/simd-arith.ll   |   357 +-
 .../CodeGen/WebAssembly/simd-build-vector.ll  |   187 +-
 .../CodeGen/WebAssembly/simd-comparisons.ll   |   150 +
 .../WebAssembly/simd-extended-extract.ll      |    83 +-
 .../CodeGen/WebAssembly/simd-intrinsics.ll    |   218 +-
 .../CodeGen/WebAssembly/simd-load-splat.ll    |    21 +
 llvm/test/CodeGen/WebAssembly/simd-offset.ll  |  1670 ++-
 llvm/test/CodeGen/WebAssembly/simd.ll         |    80 +-
 llvm/test/CodeGen/WebAssembly/tailcall.ll     |    90 +-
 .../WebAssembly/tls-general-dynamic.ll        |    31 +
 llvm/test/CodeGen/WinCFGuard/cfguard-cast.ll  |    35 +
 llvm/test/CodeGen/WinCFGuard/cfguard.ll       |     9 +-
 llvm/test/CodeGen/WinEH/wineh-comdat.ll       |     2 +-
 .../CodeGen/WinEH/wineh-noret-cleanup.ll.x    |     7 +-
 .../CodeGen/WinEH/wineh-statenumbering.ll     |     2 +-
 .../CodeGen/X86/2006-04-27-ISelFoldingBug.ll  |    30 +-
 .../X86/2006-05-08-CoalesceSubRegClass.ll     |    13 +-
 .../test/CodeGen/X86/2006-05-08-InstrSched.ll |    16 +-
 .../2006-07-28-AsmPrint-Long-As-Pointer.ll    |     3 +-
 .../CodeGen/X86/2006-08-21-ExtraMovInst.ll    |    17 +-
 .../X86/2006-10-10-FindModifiedNodeSlotBug.ll |    16 +-
 .../CodeGen/X86/2007-01-13-StackPtrIndex.ll   |   249 +-
 .../test/CodeGen/X86/2007-02-16-BranchFold.ll |    79 +-
 .../CodeGen/X86/2007-03-01-SpillerCrash.ll    |     9 +-
 .../X86/2007-03-24-InlineAsmPModifier.ll      |     9 +-
 .../X86/2007-04-27-InlineAsm-IntMemInput.ll   |     9 +-
 .../X86/2007-08-09-IllegalX86-64Asm.ll        |   172 +-
 .../CodeGen/X86/2007-08-10-SignExtSubreg.ll   |     9 +-
 .../CodeGen/X86/2007-10-04-AvoidEFLAGSCopy.ll |    19 +-
 .../X86/2007-10-12-CoalesceExtSubReg.ll       |    25 +-
 .../CodeGen/X86/2007-10-12-SpillerUnfold1.ll  |    32 +-
 .../CodeGen/X86/2007-10-12-SpillerUnfold2.ll  |    18 +-
 .../CodeGen/X86/2007-10-29-ExtendSetCC.ll     |     9 +-
 .../test/CodeGen/X86/2007-11-06-InstrSched.ll |    34 +-
 llvm/test/CodeGen/X86/2007-11-07-MulBy4.ll    |    16 +-
 .../test/CodeGen/X86/2007-12-18-LoadCSEBug.ll |    26 +-
 .../X86/2008-02-22-LocalRegAllocBug.ll        |    47 +-
 llvm/test/CodeGen/X86/2008-03-07-APIntBug.ll  |    13 +-
 .../X86/2008-03-31-SpillerFoldingBug.ll       |    40 +-
 .../CodeGen/X86/2008-04-09-BranchFolding.ll   |    20 +-
 llvm/test/CodeGen/X86/2008-04-16-ReMatBug.ll  |    90 +-
 .../CodeGen/X86/2008-04-17-CoalescerBug.ll    |   184 +-
 llvm/test/CodeGen/X86/2008-04-24-MemCpyBug.ll |    25 +-
 .../CodeGen/X86/2008-04-28-CoalescerBug.ll    |    58 +-
 .../X86/2008-05-01-InvalidOrdCompare.ll       |    18 +-
 .../CodeGen/X86/2008-05-12-tailmerge-5.ll     |    62 +-
 .../CodeGen/X86/2008-05-21-CoalescerBug.ll    |    30 +-
 .../X86/2008-06-13-VolatileLoadStore.ll       |    18 +-
 .../test/CodeGen/X86/2008-06-16-SubregsBug.ll |    17 +-
 .../test/CodeGen/X86/2008-06-25-VecISelBug.ll |     7 +-
 .../X86/2008-07-09-ELFSectionAttributes.ll    |     4 +-
 llvm/test/CodeGen/X86/2008-07-11-SHLBy1.ll    |    10 +-
 llvm/test/CodeGen/X86/2008-08-06-CmpStride.ll |    21 +-
 .../CodeGen/X86/2008-08-17-UComiCodeGenBug.ll |     9 +-
 .../CodeGen/X86/2008-09-05-sinttofp-2xi32.ll  |     2 -
 .../CodeGen/X86/2008-09-11-CoalescerBug2.ll   |     1 -
 .../CodeGen/X86/2008-09-25-sseregparm-1.ll    |    20 +-
 .../CodeGen/X86/2008-10-06-x87ld-nan-1.ll     |    15 +-
 .../CodeGen/X86/2008-10-06-x87ld-nan-2.ll     |    22 +-
 .../CodeGen/X86/2008-10-24-FlippedCompare.ll  |    14 +-
 llvm/test/CodeGen/X86/2008-11-29-ULT-Sign.ll  |    13 +-
 .../2008-12-01-loop-iv-used-outside-loop.ll   |    23 +-
 .../CodeGen/X86/2008-12-02-dagcombine-1.ll    |    10 +-
 .../CodeGen/X86/2008-12-02-dagcombine-2.ll    |     8 +-
 .../CodeGen/X86/2008-12-02-dagcombine-3.ll    |    13 +-
 .../CodeGen/X86/2008-12-16-dagcombine-4.ll    |     8 +-
 .../CodeGen/X86/2008-12-22-dagcombine-5.ll    |     8 +-
 .../CodeGen/X86/2008-12-23-crazy-address.ll   |    31 +-
 .../CodeGen/X86/2008-12-23-dagcombine-6.ll    |    13 +-
 llvm/test/CodeGen/X86/2009-01-31-BigShift.ll  |    17 +-
 llvm/test/CodeGen/X86/2009-01-31-BigShift2.ll |    12 +-
 .../X86/2009-02-21-ExtWeakInitializer.ll      |     5 +-
 .../CodeGen/X86/2009-02-26-MachineLICMBug.ll  |     4 +-
 .../CodeGen/X86/2009-03-23-MultiUseSched.ll   |   229 +-
 llvm/test/CodeGen/X86/2009-04-12-picrel.ll    |     9 +-
 .../CodeGen/X86/2009-04-14-IllegalRegs.ll     |    55 +-
 .../CodeGen/X86/2009-04-25-CoalescerBug.ll    |    14 +-
 .../X86/2009-04-29-IndirectDestOperands.ll    |    22 +-
 llvm/test/CodeGen/X86/2009-05-30-ISelBug.ll   |    21 +-
 .../CodeGen/X86/2009-06-05-VZextByteShort.ll  |     3 +-
 ...2-x86_64-tail-call-conv-out-of-sync-bug.ll |    19 +-
 .../CodeGen/X86/2009-06-15-not-a-tail-call.ll |    17 +-
 llvm/test/CodeGen/X86/2009-08-12-badswitch.ll |   150 +-
 llvm/test/CodeGen/X86/20090313-signext.ll     |    16 +-
 .../X86/2010-02-19-TailCallRetAddrBug.ll      |     2 +-
 llvm/test/CodeGen/X86/2011-05-09-loaduse.ll   |    30 +-
 .../CodeGen/X86/2011-10-19-LegelizeLoad.ll    |    28 +-
 llvm/test/CodeGen/X86/2011-12-28-vselecti8.ll |     9 +-
 .../CodeGen/X86/2011-12-8-bitcastintprom.ll   |     6 +-
 llvm/test/CodeGen/X86/2012-01-18-vbitcast.ll  |     5 +-
 .../CodeGen/X86/2012-03-15-build_vector_wl.ll |     1 -
 llvm/test/CodeGen/X86/2012-04-26-sdglue.ll    |     4 +-
 llvm/test/CodeGen/X86/2012-07-10-extload64.ll |     2 +-
 .../test/CodeGen/X86/2012-08-07-CmpISelBug.ll |    13 +-
 .../CodeGen/X86/2013-03-13-VEX-DestReg.ll     |     2 +-
 .../CodeGen/X86/2014-08-29-CompactUnwind.ll   |     2 +-
 llvm/test/CodeGen/X86/3dnow-intrinsics.ll     |    33 +-
 llvm/test/CodeGen/X86/4char-promote.ll        |     7 +-
 .../X86/DynamicCalleeSavedRegisters.ll        |    58 +-
 llvm/test/CodeGen/X86/GlobalISel/add-ext.ll   |    10 +-
 .../test/CodeGen/X86/GlobalISel/add-scalar.ll |     4 +-
 .../CodeGen/X86/GlobalISel/ashr-scalar.ll     |    11 +-
 .../test/CodeGen/X86/GlobalISel/ext-x86-64.ll |    12 +-
 llvm/test/CodeGen/X86/GlobalISel/ext.ll       |    12 +-
 .../GlobalISel/irtranslator-callingconv.ll    |    40 +-
 .../CodeGen/X86/GlobalISel/lshr-scalar.ll     |    11 +-
 llvm/test/CodeGen/X86/GlobalISel/ptr-add.ll   |   140 +
 .../test/CodeGen/X86/GlobalISel/shl-scalar.ll |    11 +-
 .../x86_64-irtranslator-struct-return.ll      |    75 +-
 llvm/test/CodeGen/X86/O0-pipeline.ll          |    12 +-
 llvm/test/CodeGen/X86/O3-pipeline.ll          |    22 +-
 llvm/test/CodeGen/X86/SwitchLowering.ll       |    38 +-
 llvm/test/CodeGen/X86/SwizzleShuff.ll         |     2 +-
 llvm/test/CodeGen/X86/absolute-constant.ll    |    23 +-
 llvm/test/CodeGen/X86/addcarry.ll             |   622 +
 .../test/CodeGen/X86/addr-label-difference.ll |     5 +-
 llvm/test/CodeGen/X86/alias-static-alloca.ll  |    28 +-
 .../X86/align-branch-boundary-default.ll      |    37 +
 .../align-branch-boundary-noautopadding.ll    |    36 +
 .../X86/align-branch-boundary-suppressions.ll |    85 +
 llvm/test/CodeGen/X86/align-down-const.ll     |   152 +
 llvm/test/CodeGen/X86/align-down.ll           |   272 +
 llvm/test/CodeGen/X86/aligned-comm.ll         |    11 +-
 llvm/test/CodeGen/X86/all-ones-vector.ll      |    56 +-
 llvm/test/CodeGen/X86/and-load-fold.ll        |     4 +-
 llvm/test/CodeGen/X86/anyext.ll               |     2 -
 llvm/test/CodeGen/X86/arg-cast.ll             |    10 +-
 llvm/test/CodeGen/X86/atom-fixup-lea4.ll      |     2 +-
 .../CodeGen/X86/atom-pad-short-functions.ll   |    23 +
 llvm/test/CodeGen/X86/atomic-monotonic.ll     |    69 +-
 .../CodeGen/X86/atomic-non-integer-fp128.ll   |    35 +
 llvm/test/CodeGen/X86/atomic-non-integer.ll   |    74 +-
 llvm/test/CodeGen/X86/atomic-unordered.ll     |  1165 +-
 llvm/test/CodeGen/X86/atomic32.ll             |   175 +
 llvm/test/CodeGen/X86/atomicf128.ll           |    32 +-
 llvm/test/CodeGen/X86/avg-mask.ll             |   114 +-
 llvm/test/CodeGen/X86/avg.ll                  |  1500 +-
 llvm/test/CodeGen/X86/avoid-sfb-overlaps.ll   |     9 +-
 llvm/test/CodeGen/X86/avoid-sfb.ll            |    13 +-
 llvm/test/CodeGen/X86/avx-basic.ll            |     2 +-
 llvm/test/CodeGen/X86/avx-bitcast.ll          |     4 +-
 llvm/test/CodeGen/X86/avx-cvt-2.ll            |     2 +-
 llvm/test/CodeGen/X86/avx-cvt.ll              |    36 +
 llvm/test/CodeGen/X86/avx-fp2int.ll           |     2 +
 llvm/test/CodeGen/X86/avx-intel-ocl.ll        |     4 +-
 llvm/test/CodeGen/X86/avx-isa-check.ll        |     2 +-
 llvm/test/CodeGen/X86/avx-load-store.ll       |    23 +
 llvm/test/CodeGen/X86/avx-minmax.ll           |    51 +-
 llvm/test/CodeGen/X86/avx-splat.ll            |   165 +-
 llvm/test/CodeGen/X86/avx-unpack.ll           |     8 +-
 llvm/test/CodeGen/X86/avx-vbroadcast.ll       |    47 +-
 llvm/test/CodeGen/X86/avx-vpclmulqdq.ll       |     1 +
 llvm/test/CodeGen/X86/avx-vzeroupper.ll       |   109 +-
 llvm/test/CodeGen/X86/avx2-conversions.ll     |     6 +-
 llvm/test/CodeGen/X86/avx2-gather.ll          |     4 +-
 llvm/test/CodeGen/X86/avx2-intrinsics-x86.ll  |    32 +
 llvm/test/CodeGen/X86/avx2-masked-gather.ll   |   594 +-
 llvm/test/CodeGen/X86/avx2-shift.ll           |    10 +-
 llvm/test/CodeGen/X86/avx2-vbroadcast.ll      |     4 +-
 llvm/test/CodeGen/X86/avx2-vector-shifts.ll   |    10 +-
 .../CodeGen/X86/avx512-any_extend_load.ll     |    38 +-
 llvm/test/CodeGen/X86/avx512-arith.ll         |    94 +
 .../CodeGen/X86/avx512-broadcast-unfold.ll    |  4635 +++++++
 llvm/test/CodeGen/X86/avx512-bugfix-25270.ll  |     4 +-
 llvm/test/CodeGen/X86/avx512-calling-conv.ll  |  2315 +++-
 .../CodeGen/X86/avx512-cmp-kor-sequence.ll    |     2 +-
 llvm/test/CodeGen/X86/avx512-cmp-mask.ll      |    51 +
 llvm/test/CodeGen/X86/avx512-cmp.ll           |    39 +-
 llvm/test/CodeGen/X86/avx512-cvt.ll           |   143 +-
 llvm/test/CodeGen/X86/avx512-ext.ll           |  1127 +-
 llvm/test/CodeGen/X86/avx512-fma-commute.ll   |    16 +-
 llvm/test/CodeGen/X86/avx512-inc-dec.ll       |    12 +-
 .../test/CodeGen/X86/avx512-insert-extract.ll |   106 +-
 llvm/test/CodeGen/X86/avx512-intel-ocl.ll     |   328 +-
 .../X86/avx512-intrinsics-fast-isel.ll        |    25 +-
 .../CodeGen/X86/avx512-intrinsics-upgrade.ll  |   129 +-
 llvm/test/CodeGen/X86/avx512-intrinsics.ll    |    58 +-
 llvm/test/CodeGen/X86/avx512-logic.ll         |    32 +-
 llvm/test/CodeGen/X86/avx512-mask-op.ll       |  1825 ++-
 llvm/test/CodeGen/X86/avx512-mask-spills.ll   |    16 +-
 .../CodeGen/X86/avx512-masked_memop-16-8.ll   |   451 +-
 llvm/test/CodeGen/X86/avx512-memfold.ll       |     2 +-
 llvm/test/CodeGen/X86/avx512-nontemporal.ll   |    42 +-
 llvm/test/CodeGen/X86/avx512-regcall-Mask.ll  |    88 +-
 .../test/CodeGen/X86/avx512-regcall-NoMask.ll |   125 +-
 llvm/test/CodeGen/X86/avx512-scalar_mask.ll   |    12 +-
 llvm/test/CodeGen/X86/avx512-select.ll        |    84 +-
 .../X86/avx512-shuffles/partial_permute.ll    |   114 +-
 .../CodeGen/X86/avx512-skx-insert-subvec.ll   |    50 +
 llvm/test/CodeGen/X86/avx512-trunc.ll         |   147 +-
 .../test/CodeGen/X86/avx512-unsafe-fp-math.ll |     5 +-
 llvm/test/CodeGen/X86/avx512-vbroadcast.ll    |     4 +-
 .../test/CodeGen/X86/avx512-vbroadcasti128.ll |    28 +-
 .../test/CodeGen/X86/avx512-vbroadcasti256.ll |    28 +-
 llvm/test/CodeGen/X86/avx512-vec-cmp.ll       |   523 +-
 llvm/test/CodeGen/X86/avx512-vec3-crash.ll    |    18 +-
 llvm/test/CodeGen/X86/avx512-vselect.ll       |   156 +
 .../X86/avx512bw-intrinsics-upgrade.ll        |    14 +-
 llvm/test/CodeGen/X86/avx512bw-mask-op.ll     |    48 +
 .../X86/avx512bwvl-intrinsics-upgrade.ll      |   120 +-
 .../X86/avx512dqvl-intrinsics-upgrade.ll      |    27 +-
 .../CodeGen/X86/avx512f-vec-test-testn.ll     |   131 +-
 .../X86/avx512ifma-intrinsics-upgrade.ll      |    34 +-
 .../test/CodeGen/X86/avx512ifma-intrinsics.ll |    34 +-
 .../CodeGen/X86/avx512vbmi2-funnel-shifts.ll  |   104 +
 .../X86/avx512vbmi2vl-funnel-shifts.ll        |   203 +
 .../X86/avx512vl-intrinsics-fast-isel.ll      |     4 +
 .../X86/avx512vl-intrinsics-upgrade.ll        |   655 +-
 llvm/test/CodeGen/X86/avx512vl-nontemporal.ll |    34 +-
 llvm/test/CodeGen/X86/avx512vl-vec-cmp.ll     |    24 +-
 .../CodeGen/X86/avx512vl-vec-masked-cmp.ll    |   734 +-
 .../X86/avx512vl_vnni-intrinsics-upgrade.ll   |    64 +-
 .../CodeGen/X86/avx512vl_vnni-intrinsics.ll   |    64 +-
 .../X86/avx512vlcd-intrinsics-fast-isel.ll    |     2 -
 .../X86/avx512vlvp2intersect-intrinsics.ll    |     7 +-
 .../X86/avx512vnni-intrinsics-upgrade.ll      |    32 +-
 .../test/CodeGen/X86/avx512vnni-intrinsics.ll |    32 +-
 llvm/test/CodeGen/X86/avx512vnni.ll           |   198 +
 .../X86/avx512vp2intersect-intrinsics.ll      |     3 +-
 llvm/test/CodeGen/X86/bit-piece-comment.ll    |     2 +-
 llvm/test/CodeGen/X86/bit-test-shift.ll       |    11 +-
 .../test/CodeGen/X86/bitcast-and-setcc-128.ll |   653 +-
 .../test/CodeGen/X86/bitcast-and-setcc-256.ll |   161 +-
 .../test/CodeGen/X86/bitcast-and-setcc-512.ll |   174 +-
 .../X86/bitcast-int-to-vector-bool-zext.ll    |    43 +-
 llvm/test/CodeGen/X86/bitcast-mmx.ll          |    38 +-
 llvm/test/CodeGen/X86/bitcast-setcc-128.ll    |   340 +-
 llvm/test/CodeGen/X86/bitcast-setcc-512.ll    |    18 +-
 llvm/test/CodeGen/X86/bitcast-vector-bool.ll  |   220 +-
 llvm/test/CodeGen/X86/bitcast2.ll             |    12 +-
 llvm/test/CodeGen/X86/bitreverse.ll           |    13 +-
 llvm/test/CodeGen/X86/block-placement.ll      |     4 +-
 llvm/test/CodeGen/X86/bmi-x86_64.ll           |    32 +-
 llvm/test/CodeGen/X86/bmi.ll                  |     4 +-
 llvm/test/CodeGen/X86/br-fold.ll              |     6 +-
 .../CodeGen/X86/break-anti-dependencies.ll    |    56 +-
 .../X86/broadcast-elm-cross-splat-vec.ll      |   296 +-
 llvm/test/CodeGen/X86/bswap-vector.ll         |    85 +-
 llvm/test/CodeGen/X86/bswap_tree.ll           |    29 +
 llvm/test/CodeGen/X86/bug26810.ll             |    12 +-
 llvm/test/CodeGen/X86/build-vector-512.ll     |   648 +-
 llvm/test/CodeGen/X86/buildvec-insertvec.ll   |    25 +-
 .../CodeGen/X86/bypass-slow-division-32.ll    |     7 -
 .../CodeGen/X86/bypass-slow-division-64.ll    |    10 +
 .../CodeGen/X86/bypass-slow-division-tune.ll  |    50 +-
 llvm/test/CodeGen/X86/byval6.ll               |    41 +-
 llvm/test/CodeGen/X86/callbr-asm.ll           |    30 +-
 llvm/test/CodeGen/X86/catchpad-lifetime.ll    |     4 +
 .../CodeGen/X86/catchpad-realign-savexmm.ll   |    21 +-
 llvm/test/CodeGen/X86/catchpad-regmask.ll     |     6 +-
 llvm/test/CodeGen/X86/catchpad-weight.ll      |     6 +-
 llvm/test/CodeGen/X86/catchret-regmask.ll     |     7 +-
 llvm/test/CodeGen/X86/cfguard-checks.ll       |   262 +
 llvm/test/CodeGen/X86/cfguard-module-flag.ll  |    26 +
 .../CodeGen/X86/cfguard-x86-64-vectorcall.ll  |    38 +
 .../CodeGen/X86/cfguard-x86-vectorcall.ll     |    43 +
 llvm/test/CodeGen/X86/cleanuppad-inalloca.ll  |     2 +-
 llvm/test/CodeGen/X86/cleanuppad-realign.ll   |    10 +-
 .../X86/clear_upper_vector_element_bits.ll    |   300 +-
 llvm/test/CodeGen/X86/cmov-into-branch.ll     |    80 +-
 llvm/test/CodeGen/X86/cmov.ll                 |     2 -
 llvm/test/CodeGen/X86/cmp.ll                  |     4 +-
 .../X86/cmpxchg8b_alloca_regalloc_handling.ll |    61 +
 llvm/test/CodeGen/X86/coalesce-esp.ll         |    30 +-
 llvm/test/CodeGen/X86/coalescer-commute1.ll   |    22 +-
 llvm/test/CodeGen/X86/coalescer-commute3.ll   |    29 +-
 llvm/test/CodeGen/X86/coalescer-commute4.ll   |    34 +-
 llvm/test/CodeGen/X86/coalescer-commute5.ll   |     9 +-
 llvm/test/CodeGen/X86/coalescer-remat.ll      |    16 +-
 .../CodeGen/X86/codegen-prepare-collapse.ll   |    18 +
 llvm/test/CodeGen/X86/coff-fp-section-name.ll |    97 +
 .../CodeGen/X86/combine-64bit-vec-binop.ll    |     6 +-
 llvm/test/CodeGen/X86/combine-and.ll          |    12 +
 llvm/test/CodeGen/X86/combine-bitselect.ll    |    24 +-
 llvm/test/CodeGen/X86/combine-bswap.ll        |     4 +-
 llvm/test/CodeGen/X86/combine-fabs.ll         |    13 +-
 llvm/test/CodeGen/X86/combine-lds.ll          |     7 +-
 llvm/test/CodeGen/X86/combine-mulo.ll         |    29 +-
 llvm/test/CodeGen/X86/combine-or.ll           |     2 +-
 llvm/test/CodeGen/X86/combine-sdiv.ll         |   358 +-
 llvm/test/CodeGen/X86/combine-shl.ll          |   103 +
 llvm/test/CodeGen/X86/combine-srem.ll         |   139 +-
 .../CodeGen/X86/combine-sse41-intrinsics.ll   |    47 +-
 .../CodeGen/X86/combineIncDecVector-crash.ll  |    51 +
 llvm/test/CodeGen/X86/commute-fcmp.ll         |   522 +-
 llvm/test/CodeGen/X86/compare-add.ll          |     8 +-
 llvm/test/CodeGen/X86/compare_folding.ll      |    14 +-
 llvm/test/CodeGen/X86/complex-fastmath.ll     |    12 +-
 .../CodeGen/X86/conditional-tailcall-pgso.ll  |   242 +
 llvm/test/CodeGen/X86/conditional-tailcall.ll |     2 +-
 llvm/test/CodeGen/X86/constant-combines.ll    |     3 +-
 .../CodeGen/X86/constrained-fp80-trunc-ext.ll |    18 +-
 .../X86/copysign-constant-magnitude.ll        |    33 +-
 llvm/test/CodeGen/X86/cpus-intel.ll           |     1 +
 .../X86/crash-lre-eliminate-dead-def.ll       |     4 +-
 llvm/test/CodeGen/X86/csr-split.ll            |   230 +
 llvm/test/CodeGen/X86/cstring.ll              |     3 +-
 llvm/test/CodeGen/X86/ctpop-combine.ll        |     4 +-
 llvm/test/CodeGen/X86/cvtv2f32.ll             |    44 +-
 llvm/test/CodeGen/X86/cxx_tlscc64.ll          |     2 +-
 .../CodeGen/X86/dagcombine-unsafe-math.ll     |     7 +-
 .../X86/dbg-changes-codegen-branch-folding.ll |     4 +-
 llvm/test/CodeGen/X86/dbg-changes-codegen.ll  |     4 +-
 llvm/test/CodeGen/X86/dbg-combine.ll          |     2 +-
 llvm/test/CodeGen/X86/dbg-value-func-arg.ll   |    86 +
 llvm/test/CodeGen/X86/debug-loclists-lto.ll   |    66 +
 llvm/test/CodeGen/X86/debug-loclists.ll       |   224 +-
 llvm/test/CodeGen/X86/debugloc-argsize.ll     |     2 +-
 .../X86/div-rem-pair-recomposition-signed.ll  |   916 ++
 .../div-rem-pair-recomposition-unsigned.ll    |   916 ++
 llvm/test/CodeGen/X86/divrem.ll               |     9 +-
 llvm/test/CodeGen/X86/divrem8_ext.ll          |    38 +-
 .../X86/dont-trunc-store-double-to-float.ll   |    21 +-
 llvm/test/CodeGen/X86/dtor-priority-coff.ll   |    70 +
 llvm/test/CodeGen/X86/dwarf-headers.ll        |     2 +-
 llvm/test/CodeGen/X86/early-cfi-sections.ll   |     2 +-
 llvm/test/CodeGen/X86/eh-frame-unreachable.ll |     2 +-
 llvm/test/CodeGen/X86/elf-associated.ll       |     8 +-
 llvm/test/CodeGen/X86/empty-function.ll       |     2 +-
 .../CodeGen/X86/empty-struct-return-type.ll   |    11 +-
 llvm/test/CodeGen/X86/epilogue-cfi-fp.ll      |     2 +-
 llvm/test/CodeGen/X86/exedepsfix-broadcast.ll |     2 +-
 llvm/test/CodeGen/X86/extend.ll               |    14 +-
 llvm/test/CodeGen/X86/extmul128.ll            |    13 +-
 llvm/test/CodeGen/X86/extmul64.ll             |    13 +-
 llvm/test/CodeGen/X86/extract-combine.ll      |     8 +-
 llvm/test/CodeGen/X86/extract-concat.ll       |   168 +-
 llvm/test/CodeGen/X86/extract-extract.ll      |     6 +-
 llvm/test/CodeGen/X86/extract-insert.ll       |     2 -
 llvm/test/CodeGen/X86/extract-store.ll        |    99 +-
 llvm/test/CodeGen/X86/extractelement-index.ll |    24 +
 llvm/test/CodeGen/X86/extractelement-load.ll  |   182 +-
 llvm/test/CodeGen/X86/extractps.ll            |    23 +-
 llvm/test/CodeGen/X86/f16c-intrinsics.ll      |    10 +-
 llvm/test/CodeGen/X86/fadd-combines.ll        |    28 +
 llvm/test/CodeGen/X86/fast-isel-divrem.ll     |     8 +-
 .../test/CodeGen/X86/fast-isel-nontemporal.ll |    48 +-
 .../X86/fast-isel-select-pseudo-cmov.ll       |    24 +-
 llvm/test/CodeGen/X86/fast-isel-select-sse.ll |   180 +-
 llvm/test/CodeGen/X86/fast-isel-vecload.ll    |    56 +-
 .../CodeGen/X86/field-extract-use-trunc.ll    |    75 +-
 llvm/test/CodeGen/X86/fildll.ll               |    39 +-
 llvm/test/CodeGen/X86/finite-libcalls.ll      |    36 +-
 llvm/test/CodeGen/X86/fixup-bw-copy.ll        |     2 +-
 llvm/test/CodeGen/X86/fixup-lea.ll            |   112 +-
 llvm/test/CodeGen/X86/float-conv-elim.ll      |     2 +-
 llvm/test/CodeGen/X86/fma-fneg-combine-2.ll   |   109 +
 llvm/test/CodeGen/X86/fmaxnum.ll              |     8 +-
 llvm/test/CodeGen/X86/fmf-reduction.ll        |    26 +
 llvm/test/CodeGen/X86/fminnum.ll              |     8 +-
 llvm/test/CodeGen/X86/fmul-combines.ll        |    54 +-
 llvm/test/CodeGen/X86/fold-call-2.ll          |    11 +-
 llvm/test/CodeGen/X86/fold-call-3.ll          |    62 +-
 llvm/test/CodeGen/X86/fold-call-oper.ll       |     2 +-
 llvm/test/CodeGen/X86/fold-load-unops.ll      |   183 +
 llvm/test/CodeGen/X86/fold-load-vec.ll        |     2 +-
 llvm/test/CodeGen/X86/fold-sext-trunc.ll      |     3 +-
 llvm/test/CodeGen/X86/fold-tied-op.ll         |     2 +-
 .../test/CodeGen/X86/fold-vector-sext-zext.ll |    16 +-
 llvm/test/CodeGen/X86/fp-arith.ll             |    66 +-
 llvm/test/CodeGen/X86/fp-cvt.ll               |    89 +-
 .../CodeGen/X86/fp-elim-and-no-fp-elim.ll     |     4 +-
 llvm/test/CodeGen/X86/fp-elim.ll              |     4 +-
 llvm/test/CodeGen/X86/fp-fast.ll              |    76 +-
 llvm/test/CodeGen/X86/fp-fold.ll              |   242 +-
 .../CodeGen/X86/fp-intrinsics-flags-x86_64.ll |    18 +
 llvm/test/CodeGen/X86/fp-intrinsics-flags.ll  |   110 +
 llvm/test/CodeGen/X86/fp-intrinsics-fma.ll    |    69 +
 llvm/test/CodeGen/X86/fp-intrinsics.ll        |  2640 +++-
 llvm/test/CodeGen/X86/fp-stack-2results.ll    |   150 +-
 llvm/test/CodeGen/X86/fp-stack-direct-ret.ll  |     8 +-
 llvm/test/CodeGen/X86/fp-stack-ret-conv.ll    |    21 +-
 llvm/test/CodeGen/X86/fp-stack-ret-store.ll   |    30 +-
 llvm/test/CodeGen/X86/fp-stack-retcopy.ll     |     6 +-
 llvm/test/CodeGen/X86/fp-stack-set-st1.ll     |    19 +-
 .../CodeGen/X86/fp-strict-libcalls-msvc32.ll  |   155 +
 llvm/test/CodeGen/X86/fp-strict-scalar-cmp.ll |  4032 ++++++
 .../CodeGen/X86/fp-strict-scalar-fptoint.ll   |  1336 ++
 .../CodeGen/X86/fp-strict-scalar-inttofp.ll   |  1299 ++
 .../CodeGen/X86/fp-strict-scalar-round.ll     |   474 +
 llvm/test/CodeGen/X86/fp-strict-scalar.ll     |   698 +
 llvm/test/CodeGen/X86/fp128-cast-strict.ll    |   975 ++
 llvm/test/CodeGen/X86/fp128-cast.ll           |  1223 +-
 llvm/test/CodeGen/X86/fp128-compare.ll        |    77 +-
 llvm/test/CodeGen/X86/fp128-extract.ll        |     7 +-
 llvm/test/CodeGen/X86/fp128-g.ll              |    12 +-
 llvm/test/CodeGen/X86/fp128-i128.ll           |    67 +-
 .../test/CodeGen/X86/fp128-libcalls-strict.ll |  1420 ++
 llvm/test/CodeGen/X86/fp128-libcalls.ll       |   836 +-
 llvm/test/CodeGen/X86/fp128-select.ll         |   158 +-
 llvm/test/CodeGen/X86/fp2sint.ll              |     9 +-
 .../CodeGen/X86/fp80-strict-scalar-cmp.ll     |   992 ++
 llvm/test/CodeGen/X86/fp80-strict-scalar.ll   |   872 ++
 llvm/test/CodeGen/X86/fp_load_fold.ll         |    40 +-
 llvm/test/CodeGen/X86/fpcmp-soft-fp.ll        |    10 +-
 llvm/test/CodeGen/X86/frame-order.ll          |     4 +-
 llvm/test/CodeGen/X86/frameregister.ll        |     2 +-
 llvm/test/CodeGen/X86/fshl.ll                 |    81 +-
 llvm/test/CodeGen/X86/fshr.ll                 |    78 +-
 llvm/test/CodeGen/X86/fsxor-alignment.ll      |    16 +-
 llvm/test/CodeGen/X86/ftrunc.ll               |    14 +-
 llvm/test/CodeGen/X86/funclet-layout.ll       |    16 +-
 llvm/test/CodeGen/X86/gcc_except_table.ll     |     2 +-
 llvm/test/CodeGen/X86/gpr-to-mask.ll          |     6 +-
 .../greedy_regalloc_bad_eviction_sequence.ll  |     4 +-
 llvm/test/CodeGen/X86/h-registers-0.ll        |   174 +-
 llvm/test/CodeGen/X86/h-registers-2.ll        |    13 +-
 llvm/test/CodeGen/X86/h-registers-3.ll        |    49 +-
 llvm/test/CodeGen/X86/haddsub-3.ll            |    89 +-
 llvm/test/CodeGen/X86/haddsub-broadcast.ll    |    21 +
 llvm/test/CodeGen/X86/haddsub.ll              |   243 +-
 llvm/test/CodeGen/X86/half.ll                 |     5 +-
 llvm/test/CodeGen/X86/hidden-vis-2.ll         |    14 +-
 ...st-and-by-const-from-lshr-in-eqcmp-zero.ll |   466 +-
 ...ist-and-by-const-from-shl-in-eqcmp-zero.ll |   246 +-
 .../CodeGen/X86/horizontal-reduce-smax.ll     |   414 +
 .../CodeGen/X86/horizontal-reduce-smin.ll     |   414 +
 .../CodeGen/X86/horizontal-reduce-umax.ll     |   494 +
 .../CodeGen/X86/horizontal-reduce-umin.ll     |   354 +
 llvm/test/CodeGen/X86/i128-add.ll             |     8 +-
 llvm/test/CodeGen/X86/i128-and-beyond.ll      |    15 +-
 llvm/test/CodeGen/X86/i128-immediate.ll       |     8 +-
 llvm/test/CodeGen/X86/i128-mul.ll             |   135 +-
 llvm/test/CodeGen/X86/i386-shrink-wrapping.ll |     2 +-
 llvm/test/CodeGen/X86/ifunc-asm.ll            |     3 +-
 .../CodeGen/X86/illegal-vector-args-return.ll |    16 +-
 llvm/test/CodeGen/X86/immediate_merging.ll    |   106 +
 llvm/test/CodeGen/X86/immediate_merging64.ll  |    41 +
 llvm/test/CodeGen/X86/implicit-null-check.ll  |    68 +-
 llvm/test/CodeGen/X86/imul.ll                 |     9 +-
 .../CodeGen/X86/inline-asm-A-constraint.ll    |     2 +-
 .../test/CodeGen/X86/inline-asm-modifier-n.ll |     9 +-
 llvm/test/CodeGen/X86/inline-asm-mrv.ll       |    30 +-
 llvm/test/CodeGen/X86/inline-asm-pic.ll       |    14 +-
 llvm/test/CodeGen/X86/inlineasm-sched-bug.ll  |     4 +-
 .../test/CodeGen/X86/ins_subreg_coalesce-2.ll |     9 +-
 .../test/CodeGen/X86/ins_subreg_coalesce-3.ll |    41 +-
 .../CodeGen/X86/insertelement-copytoregs.ll   |     9 +-
 .../CodeGen/X86/insertelement-legalize.ll     |    23 +-
 .../test/CodeGen/X86/insertelement-shuffle.ll |    14 +-
 .../CodeGen/X86/insertelement-var-index.ll    |    44 +
 llvm/test/CodeGen/X86/insertps-combine.ll     |     9 +-
 llvm/test/CodeGen/X86/ipra-reg-usage.ll       |     2 +-
 .../X86/isel-blendi-gettargetconstant.ll      |    19 +
 llvm/test/CodeGen/X86/isel-sink2.ll           |    16 +-
 llvm/test/CodeGen/X86/isnan.ll                |    12 +-
 llvm/test/CodeGen/X86/isnan2.ll               |    10 +-
 llvm/test/CodeGen/X86/ispositive.ll           |     8 +-
 llvm/test/CodeGen/X86/known-bits-vector.ll    |    30 +-
 llvm/test/CodeGen/X86/known-bits.ll           |    98 +-
 .../test/CodeGen/X86/known-signbits-vector.ll |   574 +-
 llvm/test/CodeGen/X86/kshift.ll               |   227 +-
 llvm/test/CodeGen/X86/label-annotation.ll     |     4 +-
 llvm/test/CodeGen/X86/label-heapallocsite.ll  |    43 +-
 llvm/test/CodeGen/X86/large-constants.ll      |    51 +-
 .../test/CodeGen/X86/lea-opt-memop-check-1.ll |     2 +-
 llvm/test/CodeGen/X86/lea-recursion.ll        |    34 +-
 llvm/test/CodeGen/X86/lea.ll                  |     2 +-
 llvm/test/CodeGen/X86/leaf-fp-elim.ll         |     2 +-
 llvm/test/CodeGen/X86/lifetime-alias.ll       |    16 +-
 llvm/test/CodeGen/X86/limited-prec.ll         |   528 +-
 .../CodeGen/X86/llc-override-mcpu-mattr.ll    |     6 +-
 llvm/test/CodeGen/X86/load-combine-dbg.ll     |     2 +-
 llvm/test/CodeGen/X86/load-combine.ll         |    28 +-
 llvm/test/CodeGen/X86/load-local-v3i1.ll      |   132 +-
 llvm/test/CodeGen/X86/load-partial.ll         |   314 +-
 llvm/test/CodeGen/X86/localescape.ll          |     2 +-
 llvm/test/CodeGen/X86/loop-blocks.ll          |    46 +
 .../test/CodeGen/X86/loop-strength-reduce5.ll |    22 +-
 .../test/CodeGen/X86/loop-strength-reduce6.ll |     7 +-
 .../test/CodeGen/X86/loop-strength-reduce7.ll |    27 +-
 llvm/test/CodeGen/X86/lower-bitcast.ll        |    58 -
 llvm/test/CodeGen/X86/lower-ptrmask.ll        |    31 +
 llvm/test/CodeGen/X86/lrshrink.ll             |    49 +-
 llvm/test/CodeGen/X86/lsr-negative-stride.ll  |    42 +-
 llvm/test/CodeGen/X86/lsr-sort.ll             |    19 +-
 llvm/test/CodeGen/X86/lzcnt-zext-cmp.ll       |     2 +-
 llvm/test/CodeGen/X86/machine-combiner.ll     |    12 +-
 .../CodeGen/X86/machine-outliner-debuginfo.ll |     2 +-
 .../X86/machine-outliner-disubprogram.ll      |     2 +-
 .../CodeGen/X86/machine-outliner-noredzone.ll |     2 +-
 .../CodeGen/X86/machine-outliner-tailcalls.ll |     2 +-
 llvm/test/CodeGen/X86/machine-outliner.ll     |     2 +-
 llvm/test/CodeGen/X86/madd.ll                 |   292 +-
 llvm/test/CodeGen/X86/masked-iv-safe.ll       |   199 +-
 llvm/test/CodeGen/X86/masked-iv-unsafe.ll     |   367 +-
 llvm/test/CodeGen/X86/masked_compressstore.ll |  5471 ++++----
 llvm/test/CodeGen/X86/masked_expandload.ll    |  5599 ++++----
 llvm/test/CodeGen/X86/masked_gather.ll        |  1793 ++-
 .../test/CodeGen/X86/masked_gather_scatter.ll |   879 +-
 .../X86/masked_gather_scatter_widen.ll        |   307 +-
 llvm/test/CodeGen/X86/masked_load.ll          |  8894 ++++++------
 llvm/test/CodeGen/X86/masked_store.ll         |  6314 ++++-----
 llvm/test/CodeGen/X86/masked_store_trunc.ll   |  9312 ++++++-------
 .../CodeGen/X86/masked_store_trunc_ssat.ll    | 10534 +++++++-------
 .../CodeGen/X86/masked_store_trunc_usat.ll    | 10110 +++++++-------
 llvm/test/CodeGen/X86/maskmovdqu.ll           |    36 +-
 llvm/test/CodeGen/X86/materialize.ll          |   115 +
 llvm/test/CodeGen/X86/memcmp-minsize.ll       |    61 +-
 .../CodeGen/X86/memcmp-more-load-pairs.ll     |  6241 +++++++++
 llvm/test/CodeGen/X86/memcmp-optsize.ll       |   147 +-
 llvm/test/CodeGen/X86/memcmp-pgso.ll          |  1064 ++
 llvm/test/CodeGen/X86/memcmp.ll               |  3407 ++++-
 llvm/test/CodeGen/X86/memcpy.ll               |   443 +-
 llvm/test/CodeGen/X86/memset-3.ll             |     7 +-
 llvm/test/CodeGen/X86/memset-nonzero.ll       |   221 +-
 .../X86/memset-sse-stack-realignment.ll       |   192 +-
 llvm/test/CodeGen/X86/memset-zero.ll          |   227 +
 .../X86/merge-consecutive-loads-256.ll        |    29 +
 .../X86/merge-consecutive-loads-512.ll        |    16 +-
 llvm/test/CodeGen/X86/merge-sp-update-lea.ll  |     2 +-
 .../merge-vector-stores-scale-idx-crash.ll    |     2 +-
 llvm/test/CodeGen/X86/midpoint-int-vec-128.ll |    59 +-
 llvm/test/CodeGen/X86/midpoint-int-vec-256.ll |    60 +-
 llvm/test/CodeGen/X86/midpoint-int-vec-512.ll |   716 +-
 llvm/test/CodeGen/X86/midpoint-int.ll         |   605 +-
 .../CodeGen/X86/min-legal-vector-width.ll     |   975 +-
 llvm/test/CodeGen/X86/misched-copy.ll         |     2 +-
 llvm/test/CodeGen/X86/misched-matmul.ll       |     2 +-
 .../X86/misched_phys_reg_assign_order.ll      |     2 +-
 llvm/test/CodeGen/X86/mixed-ptr-sizes.ll      |   128 +
 .../CodeGen/X86/mmx-arg-passing-x86-64.ll     |     8 +-
 llvm/test/CodeGen/X86/mmx-arith.ll            |   337 +-
 llvm/test/CodeGen/X86/mmx-coalescing.ll       |     7 +-
 llvm/test/CodeGen/X86/mmx-cvt.ll              |    19 +-
 llvm/test/CodeGen/X86/mmx-fold-load.ll        |    16 +-
 llvm/test/CodeGen/X86/mmx-intrinsics.ll       |    65 +
 llvm/test/CodeGen/X86/movfs.ll                |     8 +-
 llvm/test/CodeGen/X86/movmsk-cmp.ll           |   268 +-
 llvm/test/CodeGen/X86/movpc32-check.ll        |     4 +-
 .../test/CodeGen/X86/ms-inline-asm-PR44272.ll |    18 +
 llvm/test/CodeGen/X86/ms-inline-asm-avx512.ll |     2 +-
 llvm/test/CodeGen/X86/ms-inline-asm.ll        |     2 +-
 llvm/test/CodeGen/X86/mul-i1024.ll            | 10822 ++++++---------
 llvm/test/CodeGen/X86/mul-i256.ll             |   335 +-
 llvm/test/CodeGen/X86/mul-i512.ll             |  2201 ++-
 llvm/test/CodeGen/X86/mul-remat.ll            |    10 +-
 llvm/test/CodeGen/X86/mul-shift-reassoc.ll    |    11 +-
 llvm/test/CodeGen/X86/mul128.ll               |    36 +-
 llvm/test/CodeGen/X86/mulfix_combine.ll       |   156 +
 llvm/test/CodeGen/X86/mulo-pow2.ll            |    13 +-
 llvm/test/CodeGen/X86/mulvi32.ll              |    41 +-
 llvm/test/CodeGen/X86/musttail-fastcall.ll    |    90 +-
 llvm/test/CodeGen/X86/musttail-inalloca.ll    |    38 +
 llvm/test/CodeGen/X86/musttail-tailcc.ll      |    92 +
 llvm/test/CodeGen/X86/musttail-varargs.ll     |    96 +-
 llvm/test/CodeGen/X86/mxcsr-reg-usage.ll      |    52 +
 llvm/test/CodeGen/X86/neg-shl-add.ll          |    20 +-
 llvm/test/CodeGen/X86/neg_fp.ll               |    50 +-
 llvm/test/CodeGen/X86/negate-add-zero.ll      |    49 +-
 .../X86/negative-stride-fptosi-user.ll        |    27 +-
 llvm/test/CodeGen/X86/no-plt-libcalls.ll      |     6 +-
 llvm/test/CodeGen/X86/no-sse-win64.ll         |   129 +
 llvm/test/CodeGen/X86/no-sse-x86.ll           |    76 +
 llvm/test/CodeGen/X86/nobt.ll                 |    39 +-
 llvm/test/CodeGen/X86/nocf_check.ll           |     2 +-
 llvm/test/CodeGen/X86/nomovtopush.ll          |    59 +
 llvm/test/CodeGen/X86/nontemporal-3.ll        |   149 +-
 llvm/test/CodeGen/X86/nontemporal-loads-2.ll  |    34 +-
 llvm/test/CodeGen/X86/nontemporal-loads.ll    |   129 +-
 llvm/test/CodeGen/X86/noreturn-call-linux.ll  |    59 +
 llvm/test/CodeGen/X86/noreturn-call-win64.ll  |    53 +
 llvm/test/CodeGen/X86/noreturn-call.ll        |    56 +
 llvm/test/CodeGen/X86/not-of-dec.ll           |   117 +
 llvm/test/CodeGen/X86/oddshuffles.ll          |   453 +-
 llvm/test/CodeGen/X86/oddsubvector.ll         |    51 +-
 llvm/test/CodeGen/X86/offset-operator.ll      |    15 +
 llvm/test/CodeGen/X86/optimize-max-0.ll       |   418 +-
 llvm/test/CodeGen/X86/overflow.ll             |    49 +-
 llvm/test/CodeGen/X86/overlap-shift.ll        |    12 +-
 llvm/test/CodeGen/X86/packed_struct.ll        |    19 +-
 llvm/test/CodeGen/X86/packss.ll               |   153 +-
 llvm/test/CodeGen/X86/paddus.ll               |   163 +-
 .../X86/patchable-function-entry-ibt.ll       |    88 +
 .../CodeGen/X86/patchable-function-entry.ll   |   102 +
 llvm/test/CodeGen/X86/patchable-prologue.ll   |     2 +-
 llvm/test/CodeGen/X86/peep-test-0.ll          |    22 +-
 llvm/test/CodeGen/X86/peep-test-1.ll          |    20 +-
 llvm/test/CodeGen/X86/phaddsub-extract.ll     |   143 +-
 llvm/test/CodeGen/X86/pic-load-remat.ll       |    27 +-
 llvm/test/CodeGen/X86/pmaddubsw.ll            |   168 +-
 llvm/test/CodeGen/X86/pmul.ll                 |    64 +-
 llvm/test/CodeGen/X86/pmulh.ll                |   100 +-
 llvm/test/CodeGen/X86/pointer-vector.ll       |     2 +-
 llvm/test/CodeGen/X86/postalloc-coalescing.ll |    19 +-
 llvm/test/CodeGen/X86/powi-windows.ll         |    46 +
 llvm/test/CodeGen/X86/powi.ll                 |    49 +
 llvm/test/CodeGen/X86/pr14161.ll              |     2 -
 llvm/test/CodeGen/X86/pr1489.ll               |    97 +-
 llvm/test/CodeGen/X86/pr1505.ll               |     9 +-
 llvm/test/CodeGen/X86/pr15309.ll              |    12 +-
 llvm/test/CodeGen/X86/pr15705.ll              |     2 +-
 llvm/test/CodeGen/X86/pr18846.ll              |     2 +-
 llvm/test/CodeGen/X86/pr20020.ll              |     2 +-
 llvm/test/CodeGen/X86/pr2326.ll               |    13 +-
 llvm/test/CodeGen/X86/pr24374.ll              |     2 +-
 llvm/test/CodeGen/X86/pr28472.ll              |     7 +-
 llvm/test/CodeGen/X86/pr29112.ll              |    32 +-
 llvm/test/CodeGen/X86/pr31045.ll              |     2 +-
 llvm/test/CodeGen/X86/pr32610.ll              |     2 +-
 llvm/test/CodeGen/X86/pr3366.ll               |    10 +-
 llvm/test/CodeGen/X86/pr34080-2.ll            |     4 +-
 llvm/test/CodeGen/X86/pr34080.ll              |     2 +-
 llvm/test/CodeGen/X86/pr34177.ll              |   121 +-
 llvm/test/CodeGen/X86/pr3457.ll               |    24 +-
 llvm/test/CodeGen/X86/pr34629.ll              |     2 +-
 llvm/test/CodeGen/X86/pr34634.ll              |     2 +-
 llvm/test/CodeGen/X86/pr34657.ll              |    13 +-
 llvm/test/CodeGen/X86/pr35918.ll              |    96 +-
 llvm/test/CodeGen/X86/pr37063.ll              |     6 +-
 llvm/test/CodeGen/X86/pr37916.ll              |     1 -
 llvm/test/CodeGen/X86/pr38795.ll              |    27 +-
 llvm/test/CodeGen/X86/pr38803.ll              |     5 +-
 llvm/test/CodeGen/X86/pr39666.ll              |    21 +-
 .../CodeGen/X86/pr40631_deadstore_elision.ll  |     3 +-
 llvm/test/CodeGen/X86/pr40994.ll              |     4 +-
 llvm/test/CodeGen/X86/pr42727.ll              |    34 +
 llvm/test/CodeGen/X86/pr42870.ll              |    31 +
 llvm/test/CodeGen/X86/pr42905.ll              |    26 +
 llvm/test/CodeGen/X86/pr42909.ll              |    21 +
 llvm/test/CodeGen/X86/pr42998.ll              |    63 +
 llvm/test/CodeGen/X86/pr43157.ll              |    20 +
 llvm/test/CodeGen/X86/pr43507.ll              |    18 +
 llvm/test/CodeGen/X86/pr43509.ll              |    25 +
 llvm/test/CodeGen/X86/pr43529.ll              |    39 +
 llvm/test/CodeGen/X86/pr43575.ll              |    14 +
 llvm/test/CodeGen/X86/pr43820.ll              |   383 +
 llvm/test/CodeGen/X86/pr43866.ll              |    37 +
 llvm/test/CodeGen/X86/pr43952.ll              |    16 +
 llvm/test/CodeGen/X86/pr44140.ll              |    97 +
 llvm/test/CodeGen/X86/pr44396.ll              |    45 +
 llvm/test/CodeGen/X86/pr44412.ll              |    68 +
 llvm/test/CodeGen/X86/pr44812.ll              |    24 +
 .../CodeGen/X86/prefer-avx256-mask-shuffle.ll |     4 +-
 llvm/test/CodeGen/X86/prolog-push-seq.ll      |     2 +-
 llvm/test/CodeGen/X86/promote-vec3.ll         |    82 +-
 llvm/test/CodeGen/X86/promote.ll              |    23 +-
 llvm/test/CodeGen/X86/psadbw.ll               |    43 +
 llvm/test/CodeGen/X86/psubus.ll               |   262 +-
 llvm/test/CodeGen/X86/ptr-rotate.ll           |     7 +-
 llvm/test/CodeGen/X86/push-cfi-obj.ll         |     2 +-
 llvm/test/CodeGen/X86/push-cfi.ll             |     2 +-
 llvm/test/CodeGen/X86/ragreedy-hoist-spill.ll |   180 +-
 .../X86/ragreedy-last-chance-recoloring.ll    |     2 +-
 llvm/test/CodeGen/X86/recip-fastmath.ll       |   196 +-
 llvm/test/CodeGen/X86/recip-fastmath2.ll      |  1181 +-
 .../X86/regalloc-advanced-split-cost.ll       |     2 +-
 llvm/test/CodeGen/X86/regparm.ll              |     2 +-
 llvm/test/CodeGen/X86/relptr-rodata.ll        |    21 +
 llvm/test/CodeGen/X86/rem.ll                  |     9 +-
 llvm/test/CodeGen/X86/remarks-section.ll      |    93 +-
 llvm/test/CodeGen/X86/remat-constant.ll       |    15 +-
 llvm/test/CodeGen/X86/ret-addr.ll             |    56 +-
 llvm/test/CodeGen/X86/ret-i64-0.ll            |     8 +-
 llvm/test/CodeGen/X86/ret-mmx.ll              |     2 +-
 llvm/test/CodeGen/X86/reverse_branches.ll     |    54 +-
 llvm/test/CodeGen/X86/rot16.ll                |    53 +
 .../test/CodeGen/X86/rotate-extract-vector.ll |    42 +
 llvm/test/CodeGen/X86/rotate-extract.ll       |    59 +
 llvm/test/CodeGen/X86/rounding-ops.ll         |    57 +
 llvm/test/CodeGen/X86/sad.ll                  |    60 +-
 llvm/test/CodeGen/X86/sadd_sat.ll             |   117 +-
 llvm/test/CodeGen/X86/sadd_sat_plus.ll        |   207 +
 llvm/test/CodeGen/X86/sadd_sat_vec.ll         |  2966 ++--
 llvm/test/CodeGen/X86/saddo-redundant-add.ll  |    22 +-
 llvm/test/CodeGen/X86/sat-add.ll              |    22 +-
 llvm/test/CodeGen/X86/scalar-extract.ll       |    11 +-
 llvm/test/CodeGen/X86/scalar-fp-to-i32.ll     |  1024 ++
 llvm/test/CodeGen/X86/scalar-fp-to-i64.ll     |   330 +-
 llvm/test/CodeGen/X86/scalar-int-to-fp.ll     |   217 +-
 llvm/test/CodeGen/X86/scalar_widen_div.ll     |   102 +-
 llvm/test/CodeGen/X86/scalarize-fp.ll         |    23 +
 llvm/test/CodeGen/X86/sdiv_fix.ll             |   713 +
 llvm/test/CodeGen/X86/segmented-stacks.ll     |    41 +-
 llvm/test/CodeGen/X86/seh-catchpad.ll         |    14 +-
 llvm/test/CodeGen/X86/seh-except-finally.ll   |     8 +-
 llvm/test/CodeGen/X86/seh-no-invokes.ll       |     4 +-
 llvm/test/CodeGen/X86/select-1-or-neg1.ll     |     4 +-
 .../CodeGen/X86/select-of-fp-constants.ll     |     5 +-
 llvm/test/CodeGen/X86/select-sra.ll           |   196 +
 .../CodeGen/X86/select-testb-volatile-load.ll |    33 +
 llvm/test/CodeGen/X86/select.ll               |    32 +-
 llvm/test/CodeGen/X86/selectcc-to-shiftand.ll |   209 +-
 llvm/test/CodeGen/X86/setcc-wide-types.ll     |   597 +-
 llvm/test/CodeGen/X86/setcc.ll                |    14 +
 llvm/test/CodeGen/X86/setuge.ll               |    20 +-
 llvm/test/CodeGen/X86/shadow-stack.ll         |    14 +-
 llvm/test/CodeGen/X86/shift-amount-mod.ll     |    22 +
 llvm/test/CodeGen/X86/shift-avx2-crash.ll     |     3 +-
 llvm/test/CodeGen/X86/shift-by-signext.ll     |   148 +
 llvm/test/CodeGen/X86/shift-coalesce.ll       |    27 +-
 llvm/test/CodeGen/X86/shift-combine-crash.ll  |     1 +
 llvm/test/CodeGen/X86/shift-combine.ll        |   270 +
 llvm/test/CodeGen/X86/shift-i128.ll           |  1471 +-
 llvm/test/CodeGen/X86/shift-logic.ll          |   189 +
 llvm/test/CodeGen/X86/shift-one.ll            |     8 +-
 llvm/test/CodeGen/X86/shift-parts.ll          |    21 +-
 llvm/test/CodeGen/X86/shl_elim.ll             |    12 +-
 llvm/test/CodeGen/X86/shrink-compare-pgso.ll  |   321 +
 llvm/test/CodeGen/X86/shrink-fp-const1.ll     |     7 +-
 llvm/test/CodeGen/X86/shrink-fp-const2.ll     |     7 +-
 llvm/test/CodeGen/X86/shrink_vmul.ll          |   526 +-
 llvm/test/CodeGen/X86/shrinkwrap-hang.ll      |     2 +-
 .../test/CodeGen/X86/shuffle-combine-crash.ll |    15 +-
 .../X86/shuffle-strided-with-offset-128.ll    |   433 +-
 .../X86/shuffle-strided-with-offset-256.ll    |   550 +-
 .../X86/shuffle-strided-with-offset-512.ll    |   589 +-
 llvm/test/CodeGen/X86/shuffle-vs-trunc-128.ll |   159 +-
 llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll |   352 +-
 llvm/test/CodeGen/X86/shuffle-vs-trunc-512.ll |    80 +-
 llvm/test/CodeGen/X86/sincos-opt.ll           |    27 +
 llvm/test/CodeGen/X86/sjlj-baseptr.ll         |     2 +-
 llvm/test/CodeGen/X86/slow-incdec.ll          |    52 +-
 llvm/test/CodeGen/X86/slow-pmulld.ll          |   252 +-
 llvm/test/CodeGen/X86/smul_fix.ll             |    55 +-
 llvm/test/CodeGen/X86/smul_fix_sat.ll         |   168 +-
 .../CodeGen/X86/smul_fix_sat_constants.ll     |    20 +-
 llvm/test/CodeGen/X86/soft-fp.ll              |     5 +-
 .../CodeGen/X86/speculative-load-hardening.ll |    24 +-
 llvm/test/CodeGen/X86/splat-for-size.ll       |   198 +
 llvm/test/CodeGen/X86/split-eh-lpad-edges.ll  |    33 +-
 llvm/test/CodeGen/X86/split-vector-bitcast.ll |    24 +-
 llvm/test/CodeGen/X86/split-vector-rem.ll     |   196 +-
 llvm/test/CodeGen/X86/sqrt-fastmath-mir.ll    |    34 +-
 llvm/test/CodeGen/X86/sqrt-fastmath.ll        |     4 +-
 llvm/test/CodeGen/X86/sqrt-partial.ll         |   110 +-
 llvm/test/CodeGen/X86/sqrt.ll                 |    27 +-
 llvm/test/CodeGen/X86/srem-lkk.ll             |   159 +
 llvm/test/CodeGen/X86/srem-seteq-optsize.ll   |    73 +
 .../CodeGen/X86/srem-seteq-vec-nonsplat.ll    |  3370 +++++
 llvm/test/CodeGen/X86/srem-seteq-vec-splat.ll |   752 +
 llvm/test/CodeGen/X86/srem-seteq.ll           |   388 +
 llvm/test/CodeGen/X86/srem-vector-lkk.ll      |   556 +
 llvm/test/CodeGen/X86/sse-align-1.ll          |    11 +-
 llvm/test/CodeGen/X86/sse-align-11.ll         |    22 +-
 llvm/test/CodeGen/X86/sse-align-4.ll          |    11 +-
 llvm/test/CodeGen/X86/sse-align-5.ll          |     7 +-
 llvm/test/CodeGen/X86/sse-align-6.ll          |    17 +-
 llvm/test/CodeGen/X86/sse-align-8.ll          |     7 +-
 llvm/test/CodeGen/X86/sse-align-9.ll          |    11 +-
 llvm/test/CodeGen/X86/sse-load-ret.ll         |    45 +-
 llvm/test/CodeGen/X86/sse-minmax.ll           |     6 +-
 llvm/test/CodeGen/X86/sse-varargs.ll          |    20 +-
 .../CodeGen/X86/sse2-intrinsics-canonical.ll  |   106 +-
 llvm/test/CodeGen/X86/sse2-vector-shifts.ll   |     7 +-
 llvm/test/CodeGen/X86/sse41.ll                |    29 +-
 llvm/test/CodeGen/X86/ssub_sat.ll             |   162 +-
 llvm/test/CodeGen/X86/ssub_sat_plus.ll        |   197 +
 llvm/test/CodeGen/X86/ssub_sat_vec.ll         |  3167 ++---
 llvm/test/CodeGen/X86/stack-align.ll          |     2 +-
 llvm/test/CodeGen/X86/stack-folding-3dnow.ll  |   266 +-
 .../CodeGen/X86/stack-folding-adx-x86_64.ll   |   295 +-
 .../CodeGen/X86/stack-folding-avx512bf16.ll   |   264 +-
 .../X86/stack-folding-avx512vp2intersect.ll   |    81 +-
 llvm/test/CodeGen/X86/stack-folding-bmi.ll    |   491 +-
 llvm/test/CodeGen/X86/stack-folding-bmi2.ll   |   253 +-
 .../test/CodeGen/X86/stack-folding-fp-avx1.ll |  2512 +++-
 .../CodeGen/X86/stack-folding-fp-avx512.ll    |  1366 +-
 .../CodeGen/X86/stack-folding-fp-avx512vl.ll  |   916 +-
 .../CodeGen/X86/stack-folding-fp-sse42.ll     |  1739 ++-
 .../CodeGen/X86/stack-folding-int-avx1.ll     |  1484 +-
 .../CodeGen/X86/stack-folding-int-avx2.ll     |  1396 +-
 .../CodeGen/X86/stack-folding-int-avx512.ll   |  4858 +++++--
 .../CodeGen/X86/stack-folding-int-avx512vl.ll |  2910 +++-
 .../X86/stack-folding-int-avx512vnni.ll       |   216 +
 .../CodeGen/X86/stack-folding-int-sse42.ll    |  1806 ++-
 llvm/test/CodeGen/X86/stack-folding-lwp.ll    |   171 +-
 llvm/test/CodeGen/X86/stack-folding-mmx.ll    |   928 +-
 llvm/test/CodeGen/X86/stack-folding-sha.ll    |    88 +-
 llvm/test/CodeGen/X86/stack-folding-tbm.ll    |   805 +-
 llvm/test/CodeGen/X86/stack-folding-x86_64.ll |   163 +-
 llvm/test/CodeGen/X86/stack-folding-xop.ll    |   855 +-
 llvm/test/CodeGen/X86/stack-protector-2.ll    |   165 +
 .../stack-protector-strong-macho-win32-xor.ll |    57 +
 .../X86/stack-protector-vreg-to-vreg-copy.ll  |     4 +-
 llvm/test/CodeGen/X86/stack-protector.ll      |     4 +-
 llvm/test/CodeGen/X86/stack-size-section.ll   |     2 +-
 llvm/test/CodeGen/X86/stack_guard_remat.ll    |     2 +-
 llvm/test/CodeGen/X86/stackmap-nops.ll        |   488 +-
 .../X86/statepoint-no-realign-stack.ll        |   107 +
 llvm/test/CodeGen/X86/store-global-address.ll |     7 +-
 llvm/test/CodeGen/X86/store-narrow.ll         |   197 +-
 .../CodeGen/X86/store-zero-and-minus-one.ll   |    50 +
 llvm/test/CodeGen/X86/stores-merging.ll       |    25 +-
 llvm/test/CodeGen/X86/storetrunc-fp.ll        |    11 +-
 llvm/test/CodeGen/X86/sub-of-bias.ll          |   229 +
 llvm/test/CodeGen/X86/subcarry.ll             |   273 +-
 llvm/test/CodeGen/X86/subreg-to-reg-0.ll      |     8 +-
 llvm/test/CodeGen/X86/subreg-to-reg-2.ll      |    10 +-
 llvm/test/CodeGen/X86/subreg-to-reg-4.ll      |    97 +-
 llvm/test/CodeGen/X86/subvector-broadcast.ll  |   364 +-
 llvm/test/CodeGen/X86/swifterror.ll           |     2 +-
 llvm/test/CodeGen/X86/switch-bt.ll            |    39 +
 llvm/test/CodeGen/X86/switch-density.ll       |    69 +
 llvm/test/CodeGen/X86/switch-zextload.ll      |    14 +-
 llvm/test/CodeGen/X86/tail-call-deref.ll      |    36 +
 .../X86/tail-dup-merge-loop-headers.ll        |   116 +-
 llvm/test/CodeGen/X86/tail-dup-repeat.ll      |    11 +-
 llvm/test/CodeGen/X86/tail-merge-wineh.ll     |     2 +-
 llvm/test/CodeGen/X86/tail-opts.ll            |   210 +-
 llvm/test/CodeGen/X86/tailcall-assume.ll      |    15 +
 llvm/test/CodeGen/X86/tailcall-tailcc.ll      |   155 +
 llvm/test/CodeGen/X86/tailcall.ll             |    58 +-
 llvm/test/CodeGen/X86/tailcc-calleesave.ll    |    19 +
 .../CodeGen/X86/tailcc-disable-tail-calls.ll  |    40 +
 llvm/test/CodeGen/X86/tailcc-fastcc.ll        |    49 +
 llvm/test/CodeGen/X86/tailcc-fastisel.ll      |    18 +
 llvm/test/CodeGen/X86/tailcc-largecode.ll     |    71 +
 llvm/test/CodeGen/X86/tailcc-stackalign.ll    |    23 +
 llvm/test/CodeGen/X86/tailcc-structret.ll     |     7 +
 llvm/test/CodeGen/X86/tailccbyval.ll          |    21 +
 llvm/test/CodeGen/X86/tailccbyval64.ll        |    42 +
 llvm/test/CodeGen/X86/tailccfp.ll             |     6 +
 llvm/test/CodeGen/X86/tailccfp2.ll            |    27 +
 llvm/test/CodeGen/X86/tailccpic1.ll           |    16 +
 llvm/test/CodeGen/X86/tailccpic2.ll           |    15 +
 llvm/test/CodeGen/X86/tailccstack64.ll        |    28 +
 .../test/CodeGen/X86/taildup-heapallocsite.ll |    90 +
 llvm/test/CodeGen/X86/test-shrink-bug.ll      |     8 +-
 llvm/test/CodeGen/X86/test-vs-bittest.ll      |   362 +-
 llvm/test/CodeGen/X86/throws-cfi-fp.ll        |     2 +-
 .../CodeGen/X86/tls-addr-non-leaf-function.ll |     4 +-
 llvm/test/CodeGen/X86/tls-shrink-wrapping.ll  |     4 +-
 llvm/test/CodeGen/X86/trap.ll                 |     9 +
 llvm/test/CodeGen/X86/trunc-and.ll            |    24 +
 llvm/test/CodeGen/X86/trunc-ext-ld-st.ll      |   111 +-
 llvm/test/CodeGen/X86/trunc-subvector.ll      |   134 +-
 llvm/test/CodeGen/X86/twoaddr-coalesce-3.ll   |     2 +-
 llvm/test/CodeGen/X86/twoaddr-coalesce.ll     |    26 +-
 llvm/test/CodeGen/X86/typeid-alias.ll         |    18 +
 llvm/test/CodeGen/X86/uadd_sat.ll             |    87 +-
 llvm/test/CodeGen/X86/uadd_sat_plus.ll        |   138 +
 llvm/test/CodeGen/X86/uadd_sat_vec.ll         |   329 +-
 llvm/test/CodeGen/X86/udiv_fix.ll             |   344 +
 llvm/test/CodeGen/X86/uint64-to-float.ll      |     5 +-
 llvm/test/CodeGen/X86/umul-with-carry.ll      |    22 +-
 llvm/test/CodeGen/X86/umul_fix.ll             |    64 +-
 llvm/test/CodeGen/X86/umul_fix_sat.ll         |   542 +
 .../X86/umulo-128-legalisation-lowering.ll    |    85 +-
 llvm/test/CodeGen/X86/undef-label.ll          |     2 +-
 ...unfold-masked-merge-vector-variablemask.ll |    64 +-
 llvm/test/CodeGen/X86/unreachable-trap.ll     |    11 +-
 llvm/test/CodeGen/X86/unused_stackslots.ll    |     4 +-
 llvm/test/CodeGen/X86/urem-lkk.ll             |   108 +
 llvm/test/CodeGen/X86/urem-power-of-two.ll    |     2 -
 llvm/test/CodeGen/X86/urem-seteq-nonzero.ll   |   324 +
 .../CodeGen/X86/urem-seteq-vec-nonsplat.ll    |  1579 +--
 .../CodeGen/X86/urem-seteq-vec-nonzero.ll     |   369 +
 llvm/test/CodeGen/X86/urem-seteq-vec-splat.ll |   261 +-
 .../X86/urem-seteq-vec-tautological.ll        |   275 +
 llvm/test/CodeGen/X86/urem-seteq.ll           |   117 +-
 llvm/test/CodeGen/X86/urem-vector-lkk.ll      |   378 +
 .../X86/use-cr-result-of-dom-icmp-st.ll       |   615 +
 llvm/test/CodeGen/X86/usub_sat.ll             |    95 +-
 llvm/test/CodeGen/X86/usub_sat_plus.ll        |   149 +
 llvm/test/CodeGen/X86/usub_sat_vec.ll         |   340 +-
 llvm/test/CodeGen/X86/uwtables.ll             |     2 +-
 llvm/test/CodeGen/X86/v8i1-masks.ll           |  1293 ++
 llvm/test/CodeGen/X86/var-permute-512.ll      |   508 +-
 .../X86/variable-sized-darwin-bzero.ll        |    14 +-
 llvm/test/CodeGen/X86/vec-copysign-avx512.ll  |    89 +-
 llvm/test/CodeGen/X86/vec-strict-128.ll       |   438 +
 llvm/test/CodeGen/X86/vec-strict-256.ll       |   298 +
 llvm/test/CodeGen/X86/vec-strict-512.ll       |   283 +
 llvm/test/CodeGen/X86/vec-strict-cmp-128.ll   |  4462 ++++++
 llvm/test/CodeGen/X86/vec-strict-cmp-256.ll   |  2436 ++++
 llvm/test/CodeGen/X86/vec-strict-cmp-512.ll   |  1410 ++
 .../test/CodeGen/X86/vec-strict-cmp-sub128.ll |   308 +
 .../CodeGen/X86/vec-strict-fptoint-128.ll     |  2813 ++++
 .../CodeGen/X86/vec-strict-fptoint-256.ll     |  1761 +++
 .../CodeGen/X86/vec-strict-fptoint-512.ll     |   908 ++
 .../CodeGen/X86/vec-strict-inttofp-128.ll     |  1377 ++
 .../CodeGen/X86/vec-strict-inttofp-256.ll     |  1147 ++
 .../CodeGen/X86/vec-strict-inttofp-512.ll     |   606 +
 llvm/test/CodeGen/X86/vec-strict-round-128.ll |   174 +
 llvm/test/CodeGen/X86/vec_align.ll            |    13 +-
 llvm/test/CodeGen/X86/vec_align_i256.ll       |    12 +-
 llvm/test/CodeGen/X86/vec_anyext.ll           |     1 +
 llvm/test/CodeGen/X86/vec_call.ll             |    26 +-
 llvm/test/CodeGen/X86/vec_cast2.ll            |   158 +-
 llvm/test/CodeGen/X86/vec_cast3.ll            |   156 +-
 llvm/test/CodeGen/X86/vec_ctbits.ll           |    74 +-
 llvm/test/CodeGen/X86/vec_extract-mmx.ll      |    16 +-
 llvm/test/CodeGen/X86/vec_extract.ll          |    10 +-
 llvm/test/CodeGen/X86/vec_fp_to_int.ll        |   776 +-
 llvm/test/CodeGen/X86/vec_insert-5.ll         |     3 +-
 llvm/test/CodeGen/X86/vec_insert-7.ll         |     9 +-
 llvm/test/CodeGen/X86/vec_insert-mmx.ll       |    14 +-
 llvm/test/CodeGen/X86/vec_int_to_fp.ll        |  2061 ++-
 llvm/test/CodeGen/X86/vec_minmax_sint.ll      |   170 +-
 llvm/test/CodeGen/X86/vec_minmax_uint.ll      |   170 +-
 llvm/test/CodeGen/X86/vec_round.ll            |    14 +-
 llvm/test/CodeGen/X86/vec_saddo.ll            |  1389 +-
 llvm/test/CodeGen/X86/vec_setcc.ll            |    45 +
 llvm/test/CodeGen/X86/vec_shift6.ll           |    24 +-
 llvm/test/CodeGen/X86/vec_shuf-insert.ll      |    19 +-
 llvm/test/CodeGen/X86/vec_smulo.ll            |  1003 +-
 llvm/test/CodeGen/X86/vec_split.ll            |   115 +-
 llvm/test/CodeGen/X86/vec_ss_load_fold.ll     |     4 +-
 llvm/test/CodeGen/X86/vec_ssubo.ll            |  1425 +-
 llvm/test/CodeGen/X86/vec_trunc_sext.ll       |     1 +
 llvm/test/CodeGen/X86/vec_uaddo.ll            |   175 +-
 .../CodeGen/X86/vec_uint_to_fp-fastmath.ll    |   240 +-
 llvm/test/CodeGen/X86/vec_uint_to_fp.ll       |    38 +-
 llvm/test/CodeGen/X86/vec_umulo.ll            |  1254 +-
 llvm/test/CodeGen/X86/vec_usubo.ll            |   168 +-
 llvm/test/CodeGen/X86/vec_zero-2.ll           |     1 +
 llvm/test/CodeGen/X86/vector-bitreverse.ll    |   142 +-
 llvm/test/CodeGen/X86/vector-blend.ll         |    74 +-
 .../test/CodeGen/X86/vector-compare-all_of.ll |     6 -
 .../test/CodeGen/X86/vector-compare-any_of.ll |    12 +-
 .../CodeGen/X86/vector-compare-results.ll     |   132 +-
 .../vector-constrained-fp-intrinsics-flags.ll |    59 +
 .../vector-constrained-fp-intrinsics-fma.ll   |    30 +-
 .../X86/vector-constrained-fp-intrinsics.ll   |  4807 +++++--
 llvm/test/CodeGen/X86/vector-ext-logic.ll     |   113 +-
 llvm/test/CodeGen/X86/vector-fshl-128.ll      |   225 +-
 llvm/test/CodeGen/X86/vector-fshl-256.ll      |   123 +-
 llvm/test/CodeGen/X86/vector-fshl-512.ll      |   632 +-
 llvm/test/CodeGen/X86/vector-fshl-rot-128.ll  |   110 +-
 llvm/test/CodeGen/X86/vector-fshl-rot-256.ll  |    96 +-
 llvm/test/CodeGen/X86/vector-fshl-rot-512.ll  |   500 +-
 llvm/test/CodeGen/X86/vector-fshr-128.ll      |   342 +-
 llvm/test/CodeGen/X86/vector-fshr-256.ll      |   123 +-
 llvm/test/CodeGen/X86/vector-fshr-512.ll      |   638 +-
 llvm/test/CodeGen/X86/vector-fshr-rot-128.ll  |   123 +-
 llvm/test/CodeGen/X86/vector-fshr-rot-256.ll  |    75 +-
 llvm/test/CodeGen/X86/vector-fshr-rot-512.ll  |   522 +-
 llvm/test/CodeGen/X86/vector-gep.ll           |     3 +-
 .../CodeGen/X86/vector-half-conversions.ll    |   199 +-
 llvm/test/CodeGen/X86/vector-idiv-sdiv-128.ll |   293 +-
 llvm/test/CodeGen/X86/vector-idiv-sdiv-256.ll |     4 +-
 llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll |   204 +-
 llvm/test/CodeGen/X86/vector-idiv-udiv-128.ll |    82 +-
 llvm/test/CodeGen/X86/vector-idiv-udiv-256.ll |     2 -
 llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll |   220 +-
 llvm/test/CodeGen/X86/vector-idiv-v2i32.ll    |  1102 +-
 llvm/test/CodeGen/X86/vector-interleave.ll    |   190 +
 llvm/test/CodeGen/X86/vector-intrinsics.ll    |   355 +-
 llvm/test/CodeGen/X86/vector-lzcnt-512.ll     |   172 +-
 llvm/test/CodeGen/X86/vector-lzcnt-sub128.ll  |    85 +
 llvm/test/CodeGen/X86/vector-mul.ll           |   916 +-
 .../CodeGen/X86/vector-mulfix-legalize.ll     |   146 +
 llvm/test/CodeGen/X86/vector-narrow-binop.ll  |    41 +-
 llvm/test/CodeGen/X86/vector-pcmp.ll          |    17 +-
 llvm/test/CodeGen/X86/vector-popcnt-512.ll    |    66 +-
 llvm/test/CodeGen/X86/vector-reduce-add.ll    |   564 +-
 .../CodeGen/X86/vector-reduce-and-bool.ll     |   252 +-
 llvm/test/CodeGen/X86/vector-reduce-and.ll    |    60 +-
 .../CodeGen/X86/vector-reduce-fadd-fast.ll    |    30 +-
 llvm/test/CodeGen/X86/vector-reduce-mul.ll    |  1347 +-
 .../test/CodeGen/X86/vector-reduce-or-bool.ll |   264 +-
 llvm/test/CodeGen/X86/vector-reduce-or.ll     |    60 +-
 llvm/test/CodeGen/X86/vector-reduce-smax.ll   |   789 +-
 llvm/test/CodeGen/X86/vector-reduce-smin.ll   |   802 +-
 llvm/test/CodeGen/X86/vector-reduce-umax.ll   |   697 +-
 llvm/test/CodeGen/X86/vector-reduce-umin.ll   |   718 +-
 .../CodeGen/X86/vector-reduce-xor-bool.ll     |   383 +-
 llvm/test/CodeGen/X86/vector-reduce-xor.ll    |    60 +-
 llvm/test/CodeGen/X86/vector-rem.ll           |    20 +-
 llvm/test/CodeGen/X86/vector-rotate-128.ll    |   253 +-
 llvm/test/CodeGen/X86/vector-rotate-256.ll    |    77 +-
 llvm/test/CodeGen/X86/vector-rotate-512.ll    |   520 +-
 llvm/test/CodeGen/X86/vector-sext.ll          |   307 +-
 .../test/CodeGen/X86/vector-shift-ashr-512.ll |   160 +-
 .../CodeGen/X86/vector-shift-ashr-sub128.ll   |  3806 +++---
 .../X86/vector-shift-by-select-loop.ll        |   168 +-
 .../test/CodeGen/X86/vector-shift-lshr-128.ll |     5 +-
 .../test/CodeGen/X86/vector-shift-lshr-256.ll |    83 +
 .../test/CodeGen/X86/vector-shift-lshr-512.ll |   118 +-
 .../CodeGen/X86/vector-shift-lshr-sub128.ll   |  2817 ++--
 llvm/test/CodeGen/X86/vector-shift-shl-128.ll |    50 +-
 llvm/test/CodeGen/X86/vector-shift-shl-256.ll |    50 +-
 llvm/test/CodeGen/X86/vector-shift-shl-512.ll |   112 +-
 .../CodeGen/X86/vector-shift-shl-sub128.ll    |  2084 +--
 .../CodeGen/X86/vector-shuffle-128-v16.ll     |    26 +-
 .../test/CodeGen/X86/vector-shuffle-128-v2.ll |    15 +-
 .../test/CodeGen/X86/vector-shuffle-128-v4.ll |    35 +-
 .../test/CodeGen/X86/vector-shuffle-128-v8.ll |     4 +-
 .../CodeGen/X86/vector-shuffle-256-v32.ll     |    16 +-
 .../test/CodeGen/X86/vector-shuffle-256-v4.ll |   317 +-
 .../test/CodeGen/X86/vector-shuffle-256-v8.ll |   162 +-
 .../CodeGen/X86/vector-shuffle-512-v16.ll     |     2 +-
 .../CodeGen/X86/vector-shuffle-512-v32.ll     |   119 +-
 .../CodeGen/X86/vector-shuffle-512-v64.ll     |   232 +-
 .../test/CodeGen/X86/vector-shuffle-512-v8.ll |    26 +-
 .../test/CodeGen/X86/vector-shuffle-avx512.ll |  1053 +-
 .../X86/vector-shuffle-combining-avx.ll       |    57 +
 .../X86/vector-shuffle-combining-ssse3.ll     |     6 +-
 .../CodeGen/X86/vector-shuffle-combining.ll   |   403 +-
 llvm/test/CodeGen/X86/vector-shuffle-v1.ll    |    50 +-
 llvm/test/CodeGen/X86/vector-trunc-math.ll    |    32 +-
 llvm/test/CodeGen/X86/vector-trunc-packus.ll  |  7151 +++++++---
 llvm/test/CodeGen/X86/vector-trunc-ssat.ll    |  6973 +++++++---
 llvm/test/CodeGen/X86/vector-trunc-usat.ll    |  5617 +++++---
 llvm/test/CodeGen/X86/vector-trunc.ll         |   276 +-
 .../CodeGen/X86/vector-truncate-combine.ll    |     4 -
 llvm/test/CodeGen/X86/vector-tzcnt-512.ll     |   208 +-
 llvm/test/CodeGen/X86/vector-variable-idx.ll  |    18 +-
 .../CodeGen/X86/vector-width-store-merge.ll   |    49 +-
 llvm/test/CodeGen/X86/vector-zext.ll          |   130 +-
 llvm/test/CodeGen/X86/viabs.ll                |     8 +-
 llvm/test/CodeGen/X86/volatile.ll             |    26 +-
 .../X86/vp2intersect_multiple_pairs.ll        |    37 +-
 llvm/test/CodeGen/X86/vsel-cmp-load.ll        |    82 +-
 llvm/test/CodeGen/X86/vselect-avx.ll          |    48 +-
 llvm/test/CodeGen/X86/vselect.ll              |    94 +-
 llvm/test/CodeGen/X86/vshift-4.ll             |    12 +-
 .../X86/vshli-simplify-demanded-bits.ll       |    58 +
 llvm/test/CodeGen/X86/widen_arith-1.ll        |    14 +-
 llvm/test/CodeGen/X86/widen_arith-2.ll        |     7 +-
 llvm/test/CodeGen/X86/widen_arith-3.ll        |    13 +-
 llvm/test/CodeGen/X86/widen_bitops-0.ll       |    72 +-
 llvm/test/CodeGen/X86/widen_cast-1.ll         |    17 +-
 llvm/test/CodeGen/X86/widen_cast-2.ll         |     3 +-
 llvm/test/CodeGen/X86/widen_cast-3.ll         |     3 +-
 llvm/test/CodeGen/X86/widen_cast-4.ll         |    37 +-
 llvm/test/CodeGen/X86/widen_cast-5.ll         |     9 +-
 llvm/test/CodeGen/X86/widen_cast-6.ll         |     2 -
 llvm/test/CodeGen/X86/widen_compare-1.ll      |    22 +-
 llvm/test/CodeGen/X86/widen_conv-1.ll         |    41 +-
 llvm/test/CodeGen/X86/widen_conv-2.ll         |     8 +-
 llvm/test/CodeGen/X86/widen_conv-3.ll         |   123 +-
 llvm/test/CodeGen/X86/widen_conv-4.ll         |    96 +-
 llvm/test/CodeGen/X86/widen_conversions.ll    |     4 +-
 llvm/test/CodeGen/X86/widen_load-2.ll         |    91 +-
 llvm/test/CodeGen/X86/widen_mul.ll            |    10 +-
 llvm/test/CodeGen/X86/widen_shuffle-1.ll      |    10 +-
 llvm/test/CodeGen/X86/win-catchpad-csrs.ll    |    34 +-
 llvm/test/CodeGen/X86/win-catchpad.ll         |    14 +-
 llvm/test/CodeGen/X86/win-cleanuppad.ll       |     4 +-
 llvm/test/CodeGen/X86/win-funclet-cfi.ll      |     4 +-
 .../X86/win32-eh-available-externally.ll      |     2 +-
 llvm/test/CodeGen/X86/win32-seh-catchpad.ll   |     2 +-
 .../CodeGen/X86/win32-seh-nested-finally.ll   |     4 +-
 llvm/test/CodeGen/X86/win64-eh-empty-block.ll |   107 +
 .../test/CodeGen/X86/win64-funclet-savexmm.ll |   115 +
 .../CodeGen/X86/win64-stackprobe-overflow.ll  |    14 +
 llvm/test/CodeGen/X86/win64_call_epi.ll       |     5 +-
 llvm/test/CodeGen/X86/win64_eh.ll             |    20 +-
 llvm/test/CodeGen/X86/win64_frame.ll          |    66 +-
 llvm/test/CodeGen/X86/win_coreclr_chkstk.ll   |     2 +-
 llvm/test/CodeGen/X86/wineh-coreclr.ll        |     4 +
 llvm/test/CodeGen/X86/x86-32-intrcc.ll        |     2 +-
 llvm/test/CodeGen/X86/x86-64-arg.ll           |     8 +-
 llvm/test/CodeGen/X86/x86-64-baseptr.ll       |     2 +-
 llvm/test/CodeGen/X86/x86-64-bittest-logic.ll |   137 +
 llvm/test/CodeGen/X86/x86-64-disp.ll          |     8 +-
 .../X86/x86-64-double-shifts-Oz-Os-O2.ll      |    35 +-
 .../CodeGen/X86/x86-64-flags-intrinsics.ll    |     8 +-
 llvm/test/CodeGen/X86/x86-64-gv-offset.ll     |    14 +-
 llvm/test/CodeGen/X86/x86-64-intrcc.ll        |     2 +-
 llvm/test/CodeGen/X86/x86-64-mem.ll           |    45 +-
 llvm/test/CodeGen/X86/x86-64-pic-1.ll         |    12 +-
 llvm/test/CodeGen/X86/x86-64-pic-10.ll        |    16 +-
 llvm/test/CodeGen/X86/x86-64-pic-11.ll        |    12 +-
 llvm/test/CodeGen/X86/x86-64-pic-2.ll         |    13 +-
 llvm/test/CodeGen/X86/x86-64-pic-4.ll         |     9 +-
 llvm/test/CodeGen/X86/x86-64-pic-5.ll         |     9 +-
 llvm/test/CodeGen/X86/x86-64-pic-6.ll         |     9 +-
 llvm/test/CodeGen/X86/x86-64-pic-7.ll         |     8 +-
 llvm/test/CodeGen/X86/x86-64-pic-8.ll         |     9 +-
 llvm/test/CodeGen/X86/x86-64-pic-9.ll         |    12 +-
 llvm/test/CodeGen/X86/x86-64-ret0.ll          |    13 +-
 .../CodeGen/X86/x86-64-stack-and-frame-ptr.ll |     2 +-
 llvm/test/CodeGen/X86/x86-64-varargs.ll       |    23 +-
 llvm/test/CodeGen/X86/x86-cmov-converter.ll   |    26 +-
 .../CodeGen/X86/x86-interleaved-access.ll     |   226 +-
 llvm/test/CodeGen/X86/x86-interrupt_cc.ll     |    12 +-
 .../X86/x86-mixed-alignment-dagcombine.ll     |    31 +-
 .../CodeGen/X86/x86-repmov-copy-eflags.ll     |    53 +
 .../X86/x86-sanitizer-shrink-wrapping.ll      |     2 +-
 llvm/test/CodeGen/X86/x86-shifts.ll           |    18 +-
 .../CodeGen/X86/x86-shrink-wrap-unwind.ll     |     6 +-
 llvm/test/CodeGen/X86/x86-shrink-wrapping.ll  |     4 +-
 llvm/test/CodeGen/X86/x86-store-gv-addr.ll    |    15 +-
 llvm/test/CodeGen/X86/xmulo.ll                |    12 +-
 llvm/test/CodeGen/X86/xop-ifma.ll             |    12 +-
 llvm/test/CodeGen/X86/xor.ll                  |     7 +-
 llvm/test/CodeGen/X86/xray-custom-log.ll      |     8 +-
 .../xray-selective-instrumentation-miss.ll    |     7 +-
 .../X86/xray-selective-instrumentation.ll     |    14 +-
 llvm/test/CodeGen/X86/xray-typed-event-log.ll |    12 +-
 llvm/test/CodeGen/X86/zext-inreg-1.ll         |    19 +-
 .../AArch64/call-site-info-output.ll          |    41 +
 llvm/test/DebugInfo/AArch64/line-header.ll.x  |     2 +-
 .../DebugInfo/AArch64/machine-outliner.ll     |     2 +-
 llvm/test/DebugInfo/ARM/PR16736.ll            |     2 +-
 llvm/test/DebugInfo/ARM/PR26163.ll            |     4 +-
 .../DebugInfo/ARM/call-site-info-output.ll    |    41 +
 .../ARM/entry-value-multi-byte-expr.ll        |    91 +
 .../test/DebugInfo/ARM/lowerbdgdeclare_vla.ll |     2 +-
 .../multiple-constant-uses-drops-dbgloc.ll    |     2 +-
 .../DebugInfo/COFF/array-odr-violation.ll     |    39 +-
 llvm/test/DebugInfo/COFF/asan-module-ctor.ll  |     2 +-
 llvm/test/DebugInfo/COFF/asm.ll               |     4 +-
 .../DebugInfo/COFF/class-options-common.ll    |    14 +-
 llvm/test/DebugInfo/COFF/comdat.ll            |    10 +-
 llvm/test/DebugInfo/COFF/cpp-mangling.ll      |     2 +-
 .../DebugInfo/COFF/defer-complete-type.ll     |     2 +-
 llvm/test/DebugInfo/COFF/enum-co.ll           |     2 +-
 llvm/test/DebugInfo/COFF/fp-stack.ll          |     2 +-
 llvm/test/DebugInfo/COFF/fpo-argsize.ll       |     2 +-
 llvm/test/DebugInfo/COFF/fpo-csrs.ll          |     4 +-
 llvm/test/DebugInfo/COFF/fpo-funclet.ll       |     4 +-
 .../test/DebugInfo/COFF/fpo-realign-alloca.ll |     4 +-
 .../test/DebugInfo/COFF/fpo-realign-vframe.ll |     4 +-
 llvm/test/DebugInfo/COFF/fpo-shrink-wrap.ll   |     4 +-
 llvm/test/DebugInfo/COFF/fpo-stack-protect.ll |     4 +-
 llvm/test/DebugInfo/COFF/frameproc-flags.ll   |    14 +-
 llvm/test/DebugInfo/COFF/function-options.ll  |     2 +-
 llvm/test/DebugInfo/COFF/global-constants.ll  |     4 +-
 .../test/DebugInfo/COFF/global-type-hashes.ll |     4 +-
 llvm/test/DebugInfo/COFF/global_visibility.ll |     4 +-
 llvm/test/DebugInfo/COFF/inheritance.ll       |     4 +-
 llvm/test/DebugInfo/COFF/inlining-files.ll    |     2 +-
 llvm/test/DebugInfo/COFF/inlining-header.ll   |     2 +-
 llvm/test/DebugInfo/COFF/inlining-levels.ll   |     2 +-
 llvm/test/DebugInfo/COFF/inlining-padding.ll  |     2 +-
 llvm/test/DebugInfo/COFF/inlining.ll          |    53 +-
 llvm/test/DebugInfo/COFF/lambda.ll            |     4 +-
 llvm/test/DebugInfo/COFF/lexicalblock.ll      |     2 +-
 llvm/test/DebugInfo/COFF/line-zero.ll         |    77 +
 llvm/test/DebugInfo/COFF/lines-difile.ll      |     6 +-
 llvm/test/DebugInfo/COFF/local-constant.ll    |     4 +-
 .../test/DebugInfo/COFF/local-variable-gap.ll |    10 +-
 llvm/test/DebugInfo/COFF/local-variables.ll   |    16 +-
 llvm/test/DebugInfo/COFF/long-name.ll         |     2 +-
 llvm/test/DebugInfo/COFF/multifile.ll         |     4 +-
 llvm/test/DebugInfo/COFF/multifunction.ll     |     4 +-
 llvm/test/DebugInfo/COFF/nrvo.ll              |     6 +-
 llvm/test/DebugInfo/COFF/parameter-order.ll   |     2 +-
 llvm/test/DebugInfo/COFF/pieces.ll            |    29 +-
 llvm/test/DebugInfo/COFF/pr37492.ll           |    30 +-
 .../test/DebugInfo/COFF/purge-typedef-udts.ll |     2 +-
 .../test/DebugInfo/COFF/register-variables.ll |    52 +-
 llvm/test/DebugInfo/COFF/retained-types.ll    |     2 +-
 llvm/test/DebugInfo/COFF/scopes.ll            |     2 +-
 llvm/test/DebugInfo/COFF/simple.ll            |     4 +-
 llvm/test/DebugInfo/COFF/static-methods.ll    |     4 +-
 llvm/test/DebugInfo/COFF/synthetic.ll         |    26 +-
 .../COFF/tail-call-without-lexical-scopes.ll  |     4 +-
 llvm/test/DebugInfo/COFF/thunk.ll             |    10 +-
 llvm/test/DebugInfo/COFF/type-quals.ll        |     2 +-
 llvm/test/DebugInfo/COFF/types-array.ll       |     4 +-
 llvm/test/DebugInfo/COFF/types-basic.ll       |   343 +-
 .../test/DebugInfo/COFF/types-calling-conv.ll |     4 +-
 llvm/test/DebugInfo/COFF/types-cvarargs.ll    |     4 +-
 .../test/DebugInfo/COFF/types-data-members.ll |   354 +-
 .../COFF/types-method-ref-qualifiers.ll       |     2 +-
 .../DebugInfo/COFF/types-recursive-struct.ll  |     2 +-
 .../DebugInfo/COFF/types-recursive-unnamed.ll |     2 +-
 llvm/test/DebugInfo/COFF/udts.ll              |     2 +-
 llvm/test/DebugInfo/COFF/unnamed.ll           |     2 +-
 llvm/test/DebugInfo/COFF/vframe-csr.ll        |     4 +-
 llvm/test/DebugInfo/COFF/vframe-fpo.ll        |     4 +-
 llvm/test/DebugInfo/COFF/vftables.ll          |     8 +-
 .../DebugInfo/COFF/virtual-method-kinds.ll    |     8 +-
 llvm/test/DebugInfo/COFF/virtual-methods.ll   |     8 +-
 .../test/DebugInfo/COFF/vtable-optzn-array.ll |     4 +-
 .../Generic/2010-05-03-DisableFramePtr.ll     |     2 +-
 llvm/test/DebugInfo/Generic/PR20038.ll        |     4 +-
 llvm/test/DebugInfo/Generic/block-asan.ll     |     6 +-
 .../DebugInfo/Generic/constant-pointers.ll    |     2 +-
 .../DebugInfo/Generic/cross-cu-inlining.ll    |     4 +-
 .../DebugInfo/Generic/cross-cu-linkonce.ll    |     2 +-
 llvm/test/DebugInfo/Generic/cu-range-hole.ll  |     2 +-
 llvm/test/DebugInfo/Generic/cu-ranges.ll      |     2 +-
 .../DebugInfo/Generic/dead-argument-order.ll  |     2 +-
 .../Generic/debug-info-always-inline.ll       |     6 +-
 llvm/test/DebugInfo/Generic/debug-label.ll    |     3 +-
 llvm/test/DebugInfo/Generic/def-line.ll       |     4 +-
 .../test/DebugInfo/Generic/directives-only.ll |     4 +-
 llvm/test/DebugInfo/Generic/discriminator.ll  |     4 +-
 llvm/test/DebugInfo/Generic/enum-types.ll     |     2 +-
 llvm/test/DebugInfo/Generic/enum.ll           |     2 +-
 .../Generic/export-symbol-anonymous-class.ll  |    43 +
 .../Generic/extended-loc-directive.ll         |     4 +-
 .../DebugInfo/Generic/global-sra-array.ll     |     2 +-
 llvm/test/DebugInfo/Generic/global.ll         |     2 +-
 .../Generic/incorrect-variable-debugloc.ll    |     6 +-
 .../Generic/incorrect-variable-debugloc1.ll   |     4 +-
 .../DebugInfo/Generic/inline-no-debug-info.ll |     2 +-
 llvm/test/DebugInfo/Generic/inline-scopes.ll  |     4 +-
 .../DebugInfo/Generic/inlined-arguments.ll    |     4 +-
 .../test/DebugInfo/Generic/inlined-strings.ll |     2 +-
 llvm/test/DebugInfo/Generic/lto-comp-dir.ll   |     4 +-
 llvm/test/DebugInfo/Generic/mainsubprogram.ll |     2 +-
 llvm/test/DebugInfo/Generic/member-order.ll   |     2 +-
 .../Generic/missing-abstract-variable.ll      |     4 +-
 llvm/test/DebugInfo/Generic/multiline.ll      |     4 +-
 llvm/test/DebugInfo/Generic/namespace.ll      |     2 +-
 .../Generic/namespace_function_definition.ll  |     2 +-
 .../namespace_inline_function_definition.ll   |     4 +-
 llvm/test/DebugInfo/Generic/piece-verifier.ll |     2 +-
 .../DebugInfo/Generic/recursive_inlining.ll   |     4 +-
 llvm/test/DebugInfo/Generic/restrict.ll       |     2 +-
 llvm/test/DebugInfo/Generic/tu-composite.ll   |     2 +-
 .../DebugInfo/Generic/unconditional-branch.ll |     2 +-
 llvm/test/DebugInfo/Generic/version.ll        |     2 +-
 llvm/test/DebugInfo/Inputs/gmlt.ll            |     4 +-
 llvm/test/DebugInfo/Inputs/line.ll            |     2 +-
 llvm/test/DebugInfo/Mips/dsr-fixed-objects.ll |     8 +-
 llvm/test/DebugInfo/Mips/eh_frame.ll          |     7 +-
 llvm/test/DebugInfo/Mips/fn-call-line.ll      |     4 +-
 llvm/test/DebugInfo/NVPTX/cu-range-hole.ll    |     8 +-
 .../DebugInfo/NVPTX/dbg-declare-alloca.ll     |    22 +-
 llvm/test/DebugInfo/NVPTX/debug-addr-class.ll |     6 +-
 llvm/test/DebugInfo/NVPTX/debug-file-loc.ll   |     6 +-
 llvm/test/DebugInfo/NVPTX/debug-info.ll       |    12 +-
 llvm/test/DebugInfo/NVPTX/debug-loc-offset.ll |    14 +-
 .../DebugInfo/RISCV/dwarf-riscv-relocs.ll     |    20 +-
 .../test/DebugInfo/RISCV/relax-debug-frame.ll |    58 +
 .../Sparc/entry-value-complex-reg-expr.ll     |    79 +
 llvm/test/DebugInfo/Sparc/gnu-window-save.ll  |     4 +-
 llvm/test/DebugInfo/Sparc/subreg.ll           |     2 +-
 .../DebugInfo/WebAssembly/dbg-loop-loc.ll     |     2 +-
 .../WebAssembly/dbg-value-dwarfdump.ll        |    70 +
 .../DebugInfo/WebAssembly/dbg-value-ti.ll     |    74 +
 llvm/test/DebugInfo/X86/DIModule.ll           |     4 +-
 llvm/test/DebugInfo/X86/DIModuleContext.ll    |     2 +-
 .../DebugInfo/X86/DW_AT_calling-convention.ll |     2 +-
 llvm/test/DebugInfo/X86/DW_AT_deleted.ll      |   110 +
 .../DebugInfo/X86/DW_AT_location-reference.ll |    11 +-
 .../X86/DW_AT_stmt_list_sec_offset.ll         |     2 +-
 llvm/test/DebugInfo/X86/PR26148.ll            |     2 +-
 llvm/test/DebugInfo/X86/PR37234.ll            |     3 +-
 llvm/test/DebugInfo/X86/addr_comments.ll      |     2 +-
 llvm/test/DebugInfo/X86/arguments.ll          |     2 +-
 llvm/test/DebugInfo/X86/clang-module.ll       |     2 +-
 .../DebugInfo/X86/codegenprep-addrsink.ll     |    67 +
 .../DebugInfo/X86/coff_debug_info_type.ll     |     2 +-
 .../test/DebugInfo/X86/coff_relative_names.ll |     2 +-
 llvm/test/DebugInfo/X86/constant-loclist.ll   |    10 +-
 .../X86/containing-type-extension-rust.ll     |     6 +-
 llvm/test/DebugInfo/X86/cu-ranges-odr.ll      |     2 +-
 llvm/test/DebugInfo/X86/cu-ranges.ll          |     2 +-
 llvm/test/DebugInfo/X86/dbg-addr-dse.ll       |     4 +-
 .../X86/dbg-value-dropped-instcombine.ll      |    76 +
 .../DebugInfo/X86/dbg-value-frame-index.ll    |     4 +-
 .../X86/dbg-value-regmask-clobber.ll          |     6 +-
 llvm/test/DebugInfo/X86/dbg_value_direct.ll   |     4 +-
 .../DebugInfo/X86/dbgcall-site-64-bit-imms.ll |    56 +
 .../X86/dbgcall-site-zero-valued-imms.ll      |    68 +
 .../DebugInfo/X86/debug-dead-local-var.ll     |     2 +-
 llvm/test/DebugInfo/X86/debug-info-blocks.ll  |     2 +-
 .../X86/debug-info-template-align.ll          |    63 +
 llvm/test/DebugInfo/X86/debug-loc-frame.ll    |     6 +-
 .../X86/debug-macinfo-split-dwarf.ll          |    85 +
 llvm/test/DebugInfo/X86/debug-macro.ll        |     5 +-
 .../test/DebugInfo/X86/debug-ranges-offset.ll |     4 +-
 llvm/test/DebugInfo/X86/debug_addr.ll         |     4 +-
 .../test/DebugInfo/X86/decl-derived-member.ll |     6 +-
 llvm/test/DebugInfo/X86/discriminator.ll      |     2 +-
 llvm/test/DebugInfo/X86/discriminator2.ll     |     4 +-
 llvm/test/DebugInfo/X86/discriminator3.ll     |     4 +-
 llvm/test/DebugInfo/X86/dw_op_minus_direct.ll |     2 +-
 .../X86/dwarf-aranges-no-dwarf-labels.ll      |     4 +-
 .../X86/dwarf-callsite-related-attrs.ll       |    16 +-
 .../test/DebugInfo/X86/dwarf-linkage-names.ll |     2 +-
 .../DebugInfo/X86/dwarf-pubnames-split.ll     |     2 +-
 llvm/test/DebugInfo/X86/empty_macinfo.ll      |     6 +-
 llvm/test/DebugInfo/X86/fission-inline.ll     |     4 +-
 .../test/DebugInfo/X86/fission-no-inlining.ll |     4 +-
 llvm/test/DebugInfo/X86/fission-ranges.ll     |    32 +-
 llvm/test/DebugInfo/X86/generate-odr-hash.ll  |     2 +-
 .../DebugInfo/X86/ghost-sdnode-dbgvalues.ll   |     2 +-
 .../DebugInfo/X86/gmlt-empty-base-address.ll  |    36 +
 .../DebugInfo/X86/gnu-public-names-gmlt.ll    |     4 +-
 llvm/test/DebugInfo/X86/gnu-public-names.ll   |     4 +-
 .../DebugInfo/X86/inline-member-function.ll   |     2 +-
 llvm/test/DebugInfo/X86/inline-seldag-test.ll |     2 +-
 .../DebugInfo/X86/length_symbol_difference.ll |     1 -
 llvm/test/DebugInfo/X86/lexical_block.ll      |     2 +-
 llvm/test/DebugInfo/X86/line-info.ll          |     2 +-
 .../X86/live-debug-values-expr-conflict.ll    |    75 +
 .../X86/live-debug-values-remove-range.ll     |   114 +
 .../DebugInfo/X86/live-debug-variables.ll     |     2 +-
 llvm/test/DebugInfo/X86/loclists-dwp.ll       |     4 +-
 llvm/test/DebugInfo/X86/low-pc-cu.ll          |     2 +-
 llvm/test/DebugInfo/X86/mi-print.ll           |     2 +-
 llvm/test/DebugInfo/X86/no_debug_ranges.ll    |     2 +-
 llvm/test/DebugInfo/X86/nodebug.ll            |     2 +-
 .../DebugInfo/X86/nodebug_with_debug_loc.ll   |     4 +-
 llvm/test/DebugInfo/X86/noreturn_cpp11.ll     |    96 +-
 llvm/test/DebugInfo/X86/objc-property-void.ll |     2 +-
 llvm/test/DebugInfo/X86/objc_direct.ll        |    54 +
 llvm/test/DebugInfo/X86/pieces-1.ll           |     4 +-
 llvm/test/DebugInfo/X86/pieces-3.ll           |     7 +-
 llvm/test/DebugInfo/X86/pieces-4.ll           |     4 +-
 llvm/test/DebugInfo/X86/pr28270.ll            |     4 +-
 llvm/test/DebugInfo/X86/safestack-byval.ll    |     2 +-
 .../DebugInfo/X86/sdag-transfer-dbgvalue.ll   |    64 +
 llvm/test/DebugInfo/X86/single-dbg_value.ll   |     2 +-
 llvm/test/DebugInfo/X86/spill-nospill.ll      |     4 +-
 .../X86/split-dwarf-multiple-cu-hash.ll       |     2 +-
 .../DebugInfo/X86/split-dwarf-v5-ranges.ll    |     2 +-
 llvm/test/DebugInfo/X86/sret.ll               |     6 +-
 .../test/DebugInfo/X86/sroa-after-inlining.ll |   139 +
 llvm/test/DebugInfo/X86/sroasplit-2.ll        |     2 +-
 llvm/test/DebugInfo/X86/stack-value-dwarf2.ll |     2 +-
 llvm/test/DebugInfo/X86/stack-value-piece.ll  |    18 +-
 .../X86/string-offsets-table-order.ll         |     6 +-
 .../DebugInfo/X86/string-offsets-table.ll     |     4 +-
 llvm/test/DebugInfo/X86/template.ll           |     4 +
 llvm/test/DebugInfo/X86/tls.ll                |     2 +-
 .../DebugInfo/X86/tu-to-non-named-type.ll     |     2 +-
 llvm/test/DebugInfo/X86/void-typedef.ll       |     2 +-
 .../test/DebugInfo/salvage-cast-debug-info.ll |    25 +
 .../tut-simplify-cfg-blockaddress.ll          |    23 +
 .../SimplifyCFG/tut-simplify-cfg1.ll          |    90 +
 .../tut-simplify-cfg2-dead-block-order.ll     |   109 +
 .../SimplifyCFG/tut-simplify-cfg3-phis.ll     |    70 +
 ...ify-cfg4-multiple-duplicate-cfg-updates.ll |    40 +
 ...t-simplify-cfg5-del-phis-for-dead-block.ll |   122 +
 .../tut-simplify-cfg6-dead-self-loop.ll       |    25 +
 .../ExecutionEngine/OrcLazy/emulated-tls.ll   |    23 +
 .../test/ExecutionEngine/OrcLazy/printargv.ll |    81 +
 .../OrcLazy/static-library-support.ll         |    11 +
 llvm/test/Feature/fp-intrinsics.ll            |   223 +-
 llvm/test/Feature/intrinsics.ll               |     2 +-
 llvm/test/Feature/load_extension.ll           |    13 +
 llvm/test/Feature/optnone-opt.ll              |     6 -
 llvm/test/Feature/strip_names.ll              |     2 +-
 llvm/test/Feature/undefined.ll                |    13 +
 .../AddressSanitizer/debug-info-alloca.ll     |    75 +
 .../AddressSanitizer/global_addrspace.ll      |    31 +
 .../AddressSanitizer/global_metadata.ll       |     2 +-
 .../AddressSanitizer/global_metadata_array.ll |     2 +-
 .../global_metadata_windows.ll                |     4 +-
 .../AddressSanitizer/local_stack_base.ll      |     4 +-
 .../version-mismatch-check.ll                 |    12 +
 .../DataFlowSanitizer/abilist.ll              |    10 +-
 .../DataFlowSanitizer/debug-nonzero-labels.ll |     2 +-
 .../DataFlowSanitizer/debug.ll                |     2 +-
 .../DataFlowSanitizer/memset.ll               |     2 +-
 .../HWAddressSanitizer/alloca-compat.ll       |    17 +
 .../HWAddressSanitizer/alloca.ll              |     4 +-
 .../HWAddressSanitizer/basic-compat.ll        |    13 +
 .../HWAddressSanitizer/basic.ll               |    42 +-
 .../dbg-value-tag-offset.ll                   |    61 +
 .../HWAddressSanitizer/globals.ll             |    37 +
 .../HWAddressSanitizer/kernel-alloca.ll       |     2 +-
 .../HWAddressSanitizer/landingpad.ll          |    10 +-
 .../HWAddressSanitizer/personality.ll         |    90 +
 .../Instrumentation/InstrProfiling/PR23499.ll |     8 +-
 .../Instrumentation/InstrProfiling/comdat.ll  |    16 +-
 .../Instrumentation/InstrProfiling/linkage.ll |     8 +-
 .../MemorySanitizer/attributes.ll             |    47 +
 .../Instrumentation/MemorySanitizer/clmul.ll  |    72 +
 .../MemorySanitizer/msan_basic.ll             |    13 +-
 .../msan_llvm_launder_invariant.ll            |    38 +
 .../msan_llvm_strip_invariant.ll              |    21 +
 .../MemorySanitizer/store-origin.ll           |     2 +-
 .../SanitizerCoverage/abort-in-entry-block.ll |     1 +
 .../SanitizerCoverage/backedge-pruning.ll     |     2 +
 .../SanitizerCoverage/chains.ll               |     1 +
 .../cmp-tracing-api-x86_32.ll                 |    25 +-
 .../cmp-tracing-api-x86_64.ll                 |    25 +-
 .../SanitizerCoverage/cmp-tracing.ll          |     1 +
 .../SanitizerCoverage/coff-comdat.ll          |     1 +
 .../coff-pc-table-inline-8bit-counters.ll     |     1 +
 .../SanitizerCoverage/coff-used-ctor.ll       |     3 +-
 .../SanitizerCoverage/const-cmp-tracing.ll    |     1 +
 .../SanitizerCoverage/coverage-dbg.ll         |     3 +-
 .../SanitizerCoverage/coverage.ll             |     4 +-
 .../SanitizerCoverage/coverage2-dbg.ll        |     3 +-
 .../SanitizerCoverage/div-tracing.ll          |     1 +
 .../SanitizerCoverage/gep-tracing.ll          |     1 +
 .../SanitizerCoverage/inline-8bit-counters.ll |     1 +
 .../interposable-symbol-nocomdat.ll           |     2 +
 .../SanitizerCoverage/no-func.ll              |     1 +
 .../SanitizerCoverage/pc-table.ll             |     2 +
 .../SanitizerCoverage/postdominator_check.ll  |     2 +
 .../Instrumentation/SanitizerCoverage/seh.ll  |     3 +
 ...stack-depth-variable-declared-by-user.ll.x |     2 +
 .../SanitizerCoverage/stack-depth.ll          |     9 +-
 .../SanitizerCoverage/switch-tracing.ll       |     1 +
 .../trace-pc-guard-comdat.ll                  |     1 +
 .../trace-pc-guard-inline-8bit-counters.ll    |     1 +
 .../trace-pc-guard-nocomdat.ll                |     1 +
 .../SanitizerCoverage/tracing-comdat.ll       |     3 +
 .../SanitizerCoverage/tracing.ll              |     4 +
 .../SanitizerCoverage/unreachable-critedge.ll |     1 +
 .../SanitizerCoverage/wineh.ll                |     7 +-
 .../ThreadSanitizer/tsan_basic.ll             |     2 +-
 llvm/test/JitListener/multiple.ll             |     2 +-
 llvm/test/JitListener/simple.ll               |     2 +-
 llvm/test/LTO/ARM/lto-linking-metadata.ll     |    19 +
 .../test/LTO/Resolution/X86/Inputs/alias-1.ll |     2 +-
 .../Resolution/X86/Inputs/alias-alias-1.ll    |     2 +-
 .../Resolution/X86/Inputs/appending-var-2.ll  |     2 +-
 .../Resolution/X86/Inputs/comdat-mixed-lto.ll |     2 +-
 llvm/test/LTO/Resolution/X86/Inputs/comdat.ll |     2 +-
 .../test/LTO/Resolution/X86/Inputs/common2.ll |     2 +-
 .../test/LTO/Resolution/X86/Inputs/commons.ll |     2 +-
 .../Resolution/X86/Inputs/dead-strip-alias.ll |     2 +-
 .../X86/Inputs/dead-strip-fulllto.ll          |     2 +-
 llvm/test/LTO/Resolution/X86/Inputs/ifunc2.ll |     6 +
 .../LTO/Resolution/X86/Inputs/intrinsic.ll    |     2 +-
 .../X86/Inputs/link-odr-availextern-ae.ll     |     2 +-
 .../X86/Inputs/link-odr-availextern-odr.ll    |     2 +-
 .../LTO/Resolution/X86/Inputs/mixed_lto.ll    |     2 +-
 .../LTO/Resolution/X86/Inputs/mod-asm-used.ll |     2 +-
 .../Resolution/X86/Inputs/not-prevailing.ll   |     2 +-
 llvm/test/LTO/Resolution/X86/alias-alias.ll   |     2 +-
 llvm/test/LTO/Resolution/X86/alias.ll         |     2 +-
 llvm/test/LTO/Resolution/X86/appending-var.ll |     2 +-
 llvm/test/LTO/Resolution/X86/asm-output.ll    |     2 +-
 .../Resolution/X86/available-externally.ll    |     2 +-
 .../LTO/Resolution/X86/cache-dso-local.ll     |     2 +-
 .../LTO/Resolution/X86/cache-dso-local2.ll    |     2 +-
 .../LTO/Resolution/X86/cache-prevailing.ll    |     2 +-
 .../LTO/Resolution/X86/comdat-mixed-lto.ll    |     2 +-
 llvm/test/LTO/Resolution/X86/comdat.ll        |     4 +-
 llvm/test/LTO/Resolution/X86/common2.ll       |     2 +-
 llvm/test/LTO/Resolution/X86/commons.ll       |     2 +-
 .../LTO/Resolution/X86/dead-strip-alias.ll    |     2 +-
 .../LTO/Resolution/X86/dead-strip-fulllto.ll  |     2 +-
 ...diagnostic-handler-remarks-with-hotness.ll |     2 +-
 .../X86/diagnostic-handler-remarks.ll         |     2 +-
 .../LTO/Resolution/X86/export-jumptable.ll    |     2 +-
 .../X86/function-alias-non-prevailing.ll      |     2 +-
 llvm/test/LTO/Resolution/X86/ifunc.ll         |     4 +-
 llvm/test/LTO/Resolution/X86/ifunc2.ll        |    19 +
 llvm/test/LTO/Resolution/X86/intrinsic.ll     |     2 +-
 .../Resolution/X86/link-odr-availextern.ll    |     2 +-
 .../LTO/Resolution/X86/linker-redef-thin.ll   |     2 +-
 llvm/test/LTO/Resolution/X86/linker-redef.ll  |     2 +-
 llvm/test/LTO/Resolution/X86/linkonce.ll      |     2 +-
 .../Resolution/X86/load-sample-prof-icp.ll    |     2 +-
 .../Resolution/X86/load-sample-prof-lto.ll    |     2 +-
 .../LTO/Resolution/X86/load-sample-prof.ll    |     2 +-
 .../LTO/Resolution/X86/local-def-dllimport.ll |     2 +-
 .../test/LTO/Resolution/X86/lowertypetests.ll |     2 +-
 .../test/LTO/Resolution/X86/lto-unit-check.ll |     2 +-
 llvm/test/LTO/Resolution/X86/mixed_lto.ll     |     2 +-
 llvm/test/LTO/Resolution/X86/mod-asm-used.ll  |     2 +-
 llvm/test/LTO/Resolution/X86/multi-thinlto.ll |     2 +-
 .../Resolution/X86/not-prevailing-alias.ll    |     2 +-
 .../X86/not-prevailing-variables.ll           |     2 +-
 .../X86/not-prevailing-weak-aliasee.ll        |    33 +
 .../test/LTO/Resolution/X86/not-prevailing.ll |     2 +-
 .../LTO/Resolution/X86/setting-dso-local.ll   |     2 +-
 llvm/test/LTO/Resolution/X86/symtab-elf.ll    |     2 +-
 .../LTO/Resolution/X86/type-checked-load.ll   |     2 +-
 llvm/test/LTO/X86/Inputs/codemodel-3.ll       |     2 +-
 llvm/test/LTO/X86/Inputs/dllimport.ll         |     2 +-
 .../X86/Inputs/remangle_intrinsics_tbaa.ll    |     2 +-
 llvm/test/LTO/X86/Inputs/start-lib1.ll        |     8 +
 llvm/test/LTO/X86/Inputs/start-lib2.ll        |     6 +
 .../LTO/X86/Inputs/strip-debug-info-bar.ll    |     2 +-
 llvm/test/LTO/X86/Inputs/type-mapping-bug2.ll |     2 +-
 llvm/test/LTO/X86/Inputs/type-mapping-bug3.ll |    26 +
 llvm/test/LTO/X86/codemodel-1.ll              |     2 +-
 llvm/test/LTO/X86/codemodel-2.ll              |     2 +-
 llvm/test/LTO/X86/codemodel-3.ll              |     2 +-
 llvm/test/LTO/X86/current-section.ll          |     2 +-
 ...diagnostic-handler-remarks-with-hotness.ll |     2 +-
 .../LTO/X86/diagnostic-handler-remarks.ll     |     2 +-
 llvm/test/LTO/X86/disable-verify.ll           |     2 +-
 llvm/test/LTO/X86/dllimport.ll                |     2 +-
 llvm/test/LTO/X86/embed-bitcode.ll            |    28 +
 llvm/test/LTO/X86/internalize.ll              |     2 +-
 .../X86/no-undefined-puts-when-implemented.ll |     2 +-
 llvm/test/LTO/X86/objc-detection-i386.ll      |     2 +-
 llvm/test/LTO/X86/objc-detection.ll           |     4 +-
 llvm/test/LTO/X86/parallel.ll                 |     2 +-
 llvm/test/LTO/X86/pr38046.ll                  |     2 +-
 llvm/test/LTO/X86/private-symbol.ll           |     2 +-
 llvm/test/LTO/X86/remangle_intrinsics_tbaa.ll |     2 +-
 llvm/test/LTO/X86/set-merged.ll               |     2 +-
 .../LTO/X86/strip-debug-info-no-call-loc.ll   |     2 +-
 llvm/test/LTO/X86/strip-debug-info.ll         |     2 +-
 llvm/test/LTO/X86/symver-asm.ll               |     2 +-
 llvm/test/LTO/X86/symver-asm2.ll              |     2 +-
 llvm/test/LTO/X86/symver-asm3.ll              |     2 +-
 llvm/test/LTO/X86/tailcallelim.ll             |     2 +-
 llvm/test/LTO/X86/triple-init.ll              |     2 +-
 llvm/test/LTO/X86/triple-init2.ll             |     2 +-
 llvm/test/LTO/X86/type-mapping-bug2.ll        |     2 +-
 llvm/test/LTO/X86/type-mapping-bug3.ll        |    47 +
 llvm/test/Linker/Inputs/module-max-warn.ll    |     3 +
 .../Linker/Inputs/thinlto_funcimport_debug.ll |     4 +-
 .../Inputs/type-unique-inheritance-a.ll       |     2 +-
 .../Inputs/type-unique-inheritance-b.ll       |     6 +-
 .../Linker/Inputs/type-unique-simple2-a.ll    |     2 +-
 .../Linker/Inputs/type-unique-simple2-b.ll    |     6 +-
 llvm/test/Linker/addrspace.ll                 |    12 +
 llvm/test/Linker/comdat16.ll                  |     2 +-
 llvm/test/Linker/comdat8.ll                   |     4 +-
 llvm/test/Linker/datalayout.ll                |     7 +-
 llvm/test/Linker/module-max-warn.ll           |    12 +
 llvm/test/Linker/thinlto_funcimport_debug.ll  |     2 +-
 llvm/test/Linker/type-unique-simple-a.ll      |     2 +-
 llvm/test/Linker/type-unique-simple-b.ll      |     6 +-
 llvm/test/Linker/type-unique-type-array-a.ll  |     4 +-
 llvm/test/Linker/type-unique-type-array-b.ll  |     4 +-
 llvm/test/MC/AArch64/coff-debug.ll            |     2 +-
 llvm/test/MC/ARM/elf-execute-only-section.ll  |     4 +-
 llvm/test/MC/ARM/inline-asm-srcloc.ll         |     2 +-
 llvm/test/MC/COFF/cv-compiler-info.ll         |     2 +-
 llvm/test/MC/ELF/cfi-version.ll               |     4 +-
 llvm/test/MC/ELF/section-relro.ll             |    65 +
 llvm/test/MC/Hexagon/extended_relocations.ll  |     4 +-
 llvm/test/MC/MachO/cstexpr-gotpcrel-32.ll     |    37 +-
 llvm/test/MC/MachO/tlv-bss.ll                 |     2 +-
 llvm/test/MC/WebAssembly/assembler-binary.ll  |     4 +-
 llvm/test/MC/WebAssembly/comdat.ll            |     5 +-
 .../data-symbol-in-text-section.ll            |    13 +
 llvm/test/MC/WebAssembly/debug-info.ll        |    36 +-
 llvm/test/MC/WebAssembly/dwarfdump.ll         |     2 +-
 llvm/test/MC/WebAssembly/event-section.ll     |     5 +-
 .../MC/WebAssembly/external-func-address.ll   |     5 +-
 llvm/test/MC/WebAssembly/libcall.ll           |     5 +-
 llvm/test/MC/WebAssembly/no-dead-strip.ll     |     4 +-
 llvm/test/MC/WebAssembly/types.ll             |    23 +-
 llvm/test/MC/X86/stackmap-nops.ll             |     4 +-
 llvm/test/Other/2010-05-06-Printer.ll         |     1 -
 llvm/test/Other/attribute-comment.ll          |     2 +-
 llvm/test/Other/cgscc-devirt-iteration.ll     |     2 +-
 llvm/test/Other/cgscc-libcall-update.ll       |     6 +-
 llvm/test/Other/invariant.group.ll            |     4 +-
 llvm/test/Other/lint.ll                       |     5 +-
 llvm/test/Other/module-pass-printer.ll        |    18 +
 llvm/test/Other/new-pass-manager.ll           |     2 +
 llvm/test/Other/new-pm-defaults.ll            |    98 +-
 llvm/test/Other/new-pm-lto-defaults.ll        |    11 +-
 llvm/test/Other/new-pm-pgo-O0.ll              |    21 +
 llvm/test/Other/new-pm-pr42726-cgscc.ll       |    57 +
 llvm/test/Other/new-pm-thinlto-defaults.ll    |    57 +-
 llvm/test/Other/opt-O0-pipeline.ll            |     3 +-
 llvm/test/Other/opt-O2-pipeline.ll            |    30 +-
 llvm/test/Other/opt-O3-pipeline.ll            |    30 +-
 llvm/test/Other/opt-Os-pipeline.ll            |    30 +-
 llvm/test/Other/opt-override-mcpu-mattr.ll    |    16 +-
 llvm/test/Other/pass-pipelines.ll             |     3 +-
 llvm/test/Other/print-module-scope.ll         |     4 +-
 llvm/test/Other/print-slotindexes.ll          |    12 +
 llvm/test/Other/printer.ll                    |    35 +-
 llvm/test/Other/scalable-vectors-core-ir.ll   |   393 +
 llvm/test/Other/unroll-sroa.ll                |    61 +
 llvm/test/Reduce/remove-args.ll               |    13 +
 llvm/test/Reduce/remove-bbs.ll                |    29 +
 llvm/test/Reduce/remove-funcs.ll              |    27 +
 llvm/test/Reduce/remove-global-vars.ll        |    25 +
 llvm/test/Reduce/remove-instructions.ll       |    23 +
 llvm/test/Reduce/remove-metadata.ll           |    20 +
 .../test/ThinLTO/X86/Inputs/alias_internal.ll |     2 +-
 llvm/test/ThinLTO/X86/Inputs/cache-icall.ll   |     2 +-
 .../ThinLTO/X86/Inputs/cache-import-lists1.ll |     2 +-
 .../ThinLTO/X86/Inputs/cache-import-lists2.ll |     2 +-
 .../Inputs/cache-typeid-resolutions-import.ll |     2 +-
 .../X86/Inputs/cache-typeid-resolutions1.ll   |     2 +-
 .../X86/Inputs/cache-typeid-resolutions2.ll   |     2 +-
 .../X86/Inputs/cache-typeid-resolutions3.ll   |     2 +-
 llvm/test/ThinLTO/X86/Inputs/cache.ll         |     2 +-
 .../ThinLTO/X86/Inputs/callees-metadata.ll    |     2 +-
 .../ThinLTO/X86/Inputs/cfi-distributed.ll     |     2 +-
 .../X86/Inputs/cfi-icall-only-bazqux.ll       |     2 +-
 .../ThinLTO/X86/Inputs/crash_debuginfo.ll     |     2 +-
 llvm/test/ThinLTO/X86/Inputs/deadstrip.ll     |     2 +-
 .../Inputs/debuginfo-compositetype-import.ll  |     2 +-
 .../ThinLTO/X86/Inputs/debuginfo-cu-import.ll |     2 +-
 llvm/test/ThinLTO/X86/Inputs/devirt2.ll       |    59 +
 llvm/test/ThinLTO/X86/Inputs/devirt_alias.ll  |    15 +
 .../X86/Inputs/devirt_available_externally.ll |    14 +
 .../devirt_external_comdat_same_guid.ll       |    43 +
 .../X86/Inputs/devirt_local_same_guid.ll      |    41 +
 .../test/ThinLTO/X86/Inputs/devirt_promote.ll |    39 +
 .../X86/Inputs/devirt_single_hybrid_bar.ll    |    58 +
 .../X86/Inputs/devirt_single_hybrid_foo.ll    |    35 +
 .../X86/Inputs/diagnostic-handler-remarks.ll  |     2 +-
 .../Inputs/dicompositetype-unique-alias.ll    |     2 +-
 .../X86/Inputs/dicompositetype-unique.ll      |     2 +-
 .../X86/Inputs/dicompositetype-unique2.ll     |     2 +-
 llvm/test/ThinLTO/X86/Inputs/dot-dumper.ll    |     2 +-
 llvm/test/ThinLTO/X86/Inputs/dot-dumper2.ll   |     2 +-
 .../ThinLTO/X86/Inputs/drop-debug-info.ll     |     2 +-
 .../ThinLTO/X86/Inputs/dsolocal_dllimport.ll  |     2 +-
 llvm/test/ThinLTO/X86/Inputs/emit_imports.ll  |     2 +-
 llvm/test/ThinLTO/X86/Inputs/emit_imports2.ll |     2 +-
 llvm/test/ThinLTO/X86/Inputs/empty.ll         |     2 +-
 .../X86/Inputs/empty_module_with_cache.ll     |     2 +-
 llvm/test/ThinLTO/X86/Inputs/export.ll        |     2 +-
 .../ThinLTO/X86/Inputs/funcimport-debug.ll    |     2 +-
 .../ThinLTO/X86/Inputs/funcimport-tbaa.ll     |     2 +-
 llvm/test/ThinLTO/X86/Inputs/funcimport.ll    |     2 +-
 llvm/test/ThinLTO/X86/Inputs/funcimport2.ll   |     2 +-
 .../X86/Inputs/funcimport_alwaysinline.ll     |    10 +
 .../X86/Inputs/function_entry_count.ll        |     2 +-
 .../X86/Inputs/globals-import-blockaddr.ll    |     2 +-
 .../X86/Inputs/globals-import-cf-baz.ll       |     2 +-
 .../test/ThinLTO/X86/Inputs/globals-import.ll |     2 +-
 .../test/ThinLTO/X86/Inputs/guid_collision.ll |    15 +
 .../ThinLTO/X86/Inputs/import_opaque_type.ll  |     2 +-
 .../X86/Inputs/index-const-prop-alias.ll      |     2 +-
 .../X86/Inputs/index-const-prop-cache-foo.ll  |     2 +-
 .../Inputs/index-const-prop-cache-test1.ll    |     2 +-
 .../Inputs/index-const-prop-cache-test2.ll    |     2 +-
 .../X86/Inputs/index-const-prop-comdat.ll     |     2 +-
 .../X86/Inputs/index-const-prop-define-g.ll   |     2 +-
 .../X86/Inputs/index-const-prop-full-lto.ll   |     2 +-
 .../X86/Inputs/index-const-prop-gvref.ll      |     2 +-
 .../X86/Inputs/index-const-prop-linkage.ll    |     2 +-
 .../ThinLTO/X86/Inputs/index-const-prop.ll    |     2 +-
 llvm/test/ThinLTO/X86/Inputs/internalize.ll   |     6 +
 .../ThinLTO/X86/Inputs/lazyload_metadata.ll   |     2 +-
 .../X86/Inputs/linkonce_aliasee_ref_import.ll |     2 +-
 .../X86/Inputs/linkonce_odr_unnamed_addr.ll   |     2 +-
 .../X86/Inputs/linkonce_resolution_comdat.ll  |     2 +-
 llvm/test/ThinLTO/X86/Inputs/llvm.used.ll     |     2 +-
 .../X86/Inputs/local_name_conflict1.ll        |     2 +-
 .../X86/Inputs/local_name_conflict2.ll        |     2 +-
 .../X86/Inputs/local_name_conflict_var1.ll    |     2 +-
 .../X86/Inputs/local_name_conflict_var2.ll    |     2 +-
 llvm/test/ThinLTO/X86/Inputs/merge-triple.ll  |     2 +-
 llvm/test/ThinLTO/X86/Inputs/module_asm.ll    |     2 +-
 llvm/test/ThinLTO/X86/Inputs/module_asm2.ll   |     2 +-
 llvm/test/ThinLTO/X86/Inputs/noinline.ll      |     2 +-
 .../ThinLTO/X86/Inputs/personality-local.ll   |     2 +-
 llvm/test/ThinLTO/X86/Inputs/personality.ll   |     2 +-
 llvm/test/ThinLTO/X86/Inputs/pr35472.ll       |     2 +-
 .../X86/Inputs/reference_non_importable.ll    |     2 +-
 .../X86/Inputs/referenced_by_constant.ll      |     2 +-
 .../X86/Inputs/strong_non_prevailing.ll       |     2 +-
 .../thinlto-internalize-doublepromoted.ll     |    10 +
 .../X86/Inputs/thinlto-internalize-used2.ll   |     2 +-
 .../ThinLTO/X86/Inputs/weak_resolution.ll     |     2 +-
 .../ThinLTO/X86/Inputs/writeonly-with-refs.ll |    17 +
 llvm/test/ThinLTO/X86/alias_internal.ll       |     2 +-
 llvm/test/ThinLTO/X86/autoupgrade.ll          |     2 +-
 llvm/test/ThinLTO/X86/builtin-nostrip.ll      |     2 +-
 llvm/test/ThinLTO/X86/cache-config.ll         |     2 +-
 llvm/test/ThinLTO/X86/cache-icall.ll          |     2 +-
 llvm/test/ThinLTO/X86/cache-import-lists.ll   |     2 +-
 .../ThinLTO/X86/cache-typeid-resolutions.ll   |     2 +-
 llvm/test/ThinLTO/X86/cache.ll                |     2 +-
 llvm/test/ThinLTO/X86/callees-metadata.ll     |     2 +-
 llvm/test/ThinLTO/X86/cfi-devirt.ll           |     2 +-
 llvm/test/ThinLTO/X86/cfi-distributed.ll      |     2 +-
 .../test/ThinLTO/X86/cfi-icall-only-defuse.ll |     2 +-
 llvm/test/ThinLTO/X86/cfi-icall.ll            |     2 +-
 llvm/test/ThinLTO/X86/cfi.ll                  |     2 +-
 llvm/test/ThinLTO/X86/crash_debuginfo.ll      |     2 +-
 llvm/test/ThinLTO/X86/deadstrip.ll            |     2 +-
 .../X86/debuginfo-compositetype-import.ll     |     2 +-
 llvm/test/ThinLTO/X86/debuginfo-cu-import.ll  |     2 +-
 llvm/test/ThinLTO/X86/devirt-after-icp.ll     |     2 +-
 llvm/test/ThinLTO/X86/devirt.ll               |    58 +-
 llvm/test/ThinLTO/X86/devirt2.ll              |   278 +
 llvm/test/ThinLTO/X86/devirt_alias.ll         |    58 +
 .../X86/devirt_available_externally.ll        |    72 +
 .../X86/devirt_external_comdat_same_guid.ll   |    87 +
 .../ThinLTO/X86/devirt_local_same_guid.ll     |    74 +
 llvm/test/ThinLTO/X86/devirt_promote.ll       |    72 +
 .../test/ThinLTO/X86/devirt_promote_legacy.ll |    57 +
 llvm/test/ThinLTO/X86/devirt_single_hybrid.ll |    55 +
 ...diagnostic-handler-remarks-with-hotness.ll |     2 +-
 .../ThinLTO/X86/diagnostic-handler-remarks.ll |    15 +-
 .../X86/dicompositetype-unique-alias.ll       |     2 +-
 .../ThinLTO/X86/dicompositetype-unique.ll     |     2 +-
 .../ThinLTO/X86/dicompositetype-unique2.ll    |     2 +-
 llvm/test/ThinLTO/X86/distributed_import.ll   |     2 +-
 llvm/test/ThinLTO/X86/dot-dumper-full-lto.ll  |     2 +-
 llvm/test/ThinLTO/X86/dot-dumper.ll           |    10 +-
 llvm/test/ThinLTO/X86/dot-dumper2.ll          |     4 +-
 llvm/test/ThinLTO/X86/drop-debug-info.ll      |     2 +-
 llvm/test/ThinLTO/X86/dsolocal_dllimport.ll   |     2 +-
 llvm/test/ThinLTO/X86/emit_imports.ll         |     2 +-
 llvm/test/ThinLTO/X86/empty-module.ll         |     2 +-
 .../ThinLTO/X86/empty_module_with_cache.ll    |     2 +-
 llvm/test/ThinLTO/X86/export.ll               |     2 +-
 llvm/test/ThinLTO/X86/funcimport-debug.ll     |     2 +-
 llvm/test/ThinLTO/X86/funcimport-stats.ll     |    21 +
 llvm/test/ThinLTO/X86/funcimport-tbaa.ll      |     2 +-
 llvm/test/ThinLTO/X86/funcimport.ll           |     2 +-
 llvm/test/ThinLTO/X86/funcimport2.ll          |     2 +-
 .../ThinLTO/X86/funcimport_alwaysinline.ll    |    26 +
 llvm/test/ThinLTO/X86/function_entry_count.ll |     2 +-
 .../ThinLTO/X86/globals-import-blockaddr.ll   |     2 +-
 .../ThinLTO/X86/globals-import-const-fold.ll  |     2 +-
 llvm/test/ThinLTO/X86/globals-import.ll       |     4 +-
 llvm/test/ThinLTO/X86/guid_collision.ll       |    37 +
 llvm/test/ThinLTO/X86/import_opaque_type.ll   |     2 +-
 llvm/test/ThinLTO/X86/index-const-prop-O0.ll  |     2 +-
 .../ThinLTO/X86/index-const-prop-alias.ll     |     2 +-
 .../ThinLTO/X86/index-const-prop-cache.ll     |     2 +-
 .../ThinLTO/X86/index-const-prop-comdat.ll    |     2 +-
 .../test/ThinLTO/X86/index-const-prop-dead.ll |     2 +-
 .../ThinLTO/X86/index-const-prop-full-lto.ll  |     2 +-
 .../ThinLTO/X86/index-const-prop-gvref.ll     |     2 +-
 .../test/ThinLTO/X86/index-const-prop-ldst.ll |     2 +-
 .../ThinLTO/X86/index-const-prop-linkage.ll   |     2 +-
 llvm/test/ThinLTO/X86/index-const-prop.ll     |     2 +-
 llvm/test/ThinLTO/X86/index-const-prop2.ll    |    10 +-
 llvm/test/ThinLTO/X86/internalize.ll          |    37 +-
 llvm/test/ThinLTO/X86/lazyload_metadata.ll    |    21 +-
 .../X86/linkonce_aliasee_ref_import.ll        |     2 +-
 .../ThinLTO/X86/linkonce_odr_unnamed_addr.ll  |     2 +-
 .../ThinLTO/X86/linkonce_resolution_comdat.ll |     6 +-
 llvm/test/ThinLTO/X86/llvm.used.ll            |     2 +-
 llvm/test/ThinLTO/X86/load-store-caching.ll   |     2 +-
 llvm/test/ThinLTO/X86/local_name_conflict.ll  |     4 +-
 .../ThinLTO/X86/local_name_conflict_var.ll    |     2 +-
 llvm/test/ThinLTO/X86/merge-triple.ll         |     2 +-
 llvm/test/ThinLTO/X86/module_asm2.ll          |     2 +-
 llvm/test/ThinLTO/X86/module_asm_glob.ll      |     2 +-
 .../X86/module_summary_graph_traits.ll        |     2 +-
 llvm/test/ThinLTO/X86/newpm-basic.ll          |     2 +-
 .../X86/nodevirt-nonpromoted-typeid.ll        |    66 +
 llvm/test/ThinLTO/X86/noinline.ll             |     2 +-
 llvm/test/ThinLTO/X86/not-internalized.ll     |    34 +
 llvm/test/ThinLTO/X86/personality-local.ll    |     2 +-
 llvm/test/ThinLTO/X86/personality.ll          |     2 +-
 llvm/test/ThinLTO/X86/pr35472.ll              |     2 +-
 llvm/test/ThinLTO/X86/printer.ll              |    12 +
 .../ThinLTO/X86/reference_non_importable.ll   |     2 +-
 .../ThinLTO/X86/referenced_by_constant.ll     |     2 +-
 llvm/test/ThinLTO/X86/save_objects.ll         |     2 +-
 .../test/ThinLTO/X86/strong_non_prevailing.ll |     2 +-
 .../X86/thinlto-internalize-doublepromoted.ll |    33 +
 .../ThinLTO/X86/thinlto-internalize-used.ll   |     2 +-
 llvm/test/ThinLTO/X86/tli-nobuiltin.ll        |     2 +-
 llvm/test/ThinLTO/X86/weak_externals.ll       |     6 +-
 llvm/test/ThinLTO/X86/weak_resolution.ll      |     2 +-
 llvm/test/ThinLTO/X86/writeonly-with-refs.ll  |    38 +
 llvm/test/ThinLTO/X86/writeonly.ll            |    13 +-
 llvm/test/ThinLTO/X86/writeonly2.ll           |     6 +-
 llvm/test/Transforms/ADCE/2016-09-06.ll       |     2 +-
 .../blocks-with-dead-term-nondeterministic.ll |     2 +-
 .../Transforms/AddDiscriminators/basic.ll     |     2 +-
 .../AddDiscriminators/call-nested.ll          |     4 +-
 .../test/Transforms/AddDiscriminators/call.ll |     4 +-
 .../Transforms/AddDiscriminators/diamond.ll   |     4 +-
 .../AddDiscriminators/first-only.ll           |     2 +-
 .../Transforms/AddDiscriminators/invoke.ll    |     6 +-
 .../Transforms/AddDiscriminators/multiple.ll  |     2 +-
 .../AddDiscriminators/no-discriminators.ll    |     2 +-
 .../Transforms/AddDiscriminators/oneline.ll   |     2 +-
 .../AggressiveInstCombine/popcount.ll         |   193 +
 .../AlignmentFromAssumptions/simple.ll        |    55 +
 .../2008-02-01-ReturnAttrs.ll                 |    19 +-
 .../2008-07-02-array-indexing.ll              |    27 +-
 .../ArgumentPromotion/2008-09-07-CGUpdate.ll  |     7 +-
 .../2008-09-08-CGUpdateSelfEdge.ll            |    17 +-
 .../ArgumentPromotion/X86/attributes.ll       |    41 +-
 .../X86/min-legal-vector-width.ll             |   161 +-
 .../ArgumentPromotion/X86/thiscall.ll         |    43 +-
 .../ArgumentPromotion/aggregate-promote.ll    |    26 +-
 .../Transforms/ArgumentPromotion/attrs.ll     |    52 +-
 .../Transforms/ArgumentPromotion/basictest.ll |    20 +-
 .../Transforms/ArgumentPromotion/byval-2.ll   |    36 +-
 .../Transforms/ArgumentPromotion/byval.ll     |    60 +-
 .../Transforms/ArgumentPromotion/chained.ll   |    21 +-
 .../ArgumentPromotion/control-flow.ll         |    21 +-
 .../ArgumentPromotion/control-flow2.ll        |    33 +-
 .../Transforms/ArgumentPromotion/crash.ll     |    51 +-
 llvm/test/Transforms/ArgumentPromotion/dbg.ll |    41 +-
 .../test/Transforms/ArgumentPromotion/fp80.ll |    50 +-
 .../Transforms/ArgumentPromotion/inalloca.ll  |    32 +-
 .../ArgumentPromotion/invalidation.ll         |    33 +-
 .../Transforms/ArgumentPromotion/musttail.ll  |    38 +-
 .../ArgumentPromotion/naked_functions.ll      |    15 +-
 .../nonzero-address-spaces.ll                 |    15 +-
 .../Transforms/ArgumentPromotion/pr27568.ll   |    23 +-
 .../Transforms/ArgumentPromotion/pr3085.ll    |  1329 +-
 .../Transforms/ArgumentPromotion/pr32917.ll   |    18 +-
 .../pr33641_remove_arg_dbgvalue.ll            |    18 +-
 .../Transforms/ArgumentPromotion/profile.ll   |    12 +-
 .../ArgumentPromotion/reserve-tbaa.ll         |    29 +-
 .../test/Transforms/ArgumentPromotion/sret.ll |    23 +-
 .../test/Transforms/ArgumentPromotion/tail.ll |    24 +-
 .../Transforms/ArgumentPromotion/variadic.ll  |    14 +-
 .../AMDGPU/expand-atomic-rmw-fadd.ll          |   145 +
 .../AMDGPU/expand-atomic-rmw-fsub.ll          |     4 +-
 .../2008-02-01-ReturnAttrs.ll                 |    30 +
 .../2008-07-02-array-indexing.ll              |    43 +
 .../ArgumentPromotion/2008-09-07-CGUpdate.ll  |    13 +
 .../2008-09-08-CGUpdateSelfEdge.ll            |    26 +
 .../ArgumentPromotion/X86/attributes.ll       |    89 +
 .../X86/min-legal-vector-width.ll             |   328 +
 .../ArgumentPromotion/X86/thiscall.ll         |    73 +
 .../ArgumentPromotion/aggregate-promote.ll    |    36 +
 .../Attributor/ArgumentPromotion/attrs.ll     |    52 +
 .../Attributor/ArgumentPromotion/basictest.ll |    45 +
 .../Attributor/ArgumentPromotion/byval-2.ll   |    47 +
 .../Attributor/ArgumentPromotion/byval.ll     |    69 +
 .../Attributor/ArgumentPromotion/chained.ll   |    31 +
 .../ArgumentPromotion/control-flow.ll         |    38 +
 .../ArgumentPromotion/control-flow2.ll        |    38 +
 .../Attributor/ArgumentPromotion/crash.ll     |    94 +
 .../Attributor/ArgumentPromotion/dbg.ll       |    51 +
 .../Attributor/ArgumentPromotion/fp80.ll      |    72 +
 .../Attributor/ArgumentPromotion/inalloca.ll  |    91 +
 .../ArgumentPromotion/invalidation.ll         |    50 +
 .../live_called_from_dead.ll                  |    55 +
 .../Attributor/ArgumentPromotion/musttail.ll  |    70 +
 .../ArgumentPromotion/naked_functions.ll      |    34 +
 .../nonzero-address-spaces.ll                 |    35 +
 .../Attributor/ArgumentPromotion/pr27568.ll   |    44 +
 .../Attributor/ArgumentPromotion/pr3085.ll    |  1945 +++
 .../Attributor/ArgumentPromotion/pr32917.ll   |    35 +
 .../pr33641_remove_arg_dbgvalue.ll            |    45 +
 .../Attributor/ArgumentPromotion/profile.ll   |    34 +
 .../ArgumentPromotion/reserve-tbaa.ll         |    65 +
 .../Attributor/ArgumentPromotion/sret.ll      |    39 +
 .../Attributor/ArgumentPromotion/tail.ll      |    29 +
 .../Attributor/ArgumentPromotion/variadic.ll  |    38 +
 .../IPConstantProp/2008-06-09-WeakProp.ll     |    25 +
 .../IPConstantProp/2009-09-24-byval-ptr.ll    |   102 +
 .../Attributor/IPConstantProp/PR16052.ll      |    34 +
 .../Attributor/IPConstantProp/PR26044.ll      |    88 +
 .../Attributor/IPConstantProp/PR43857.ll      |    30 +
 .../IPConstantProp/arg-count-mismatch.ll      |    85 +
 .../IPConstantProp/arg-type-mismatch.ll       |    25 +
 .../Attributor/IPConstantProp/comdat-ipo.ll   |    34 +
 .../IPConstantProp/dangling-block-address.ll  |    43 +
 .../Attributor/IPConstantProp/deadarg.ll      |     7 +
 .../IPConstantProp/fp-bc-icmp-const-fold.ll   |    78 +
 .../Attributor/IPConstantProp/global.ll       |    40 +
 .../IPConstantProp/multiple_callbacks.ll      |   117 +
 .../IPConstantProp/musttail-call.ll           |    79 +
 .../Attributor/IPConstantProp/naked-return.ll |    47 +
 .../IPConstantProp/openmp_parallel_for.ll     |   132 +
 .../Attributor/IPConstantProp/pthreads.ll     |    96 +
 .../Attributor/IPConstantProp/recursion.ll    |    19 +
 .../IPConstantProp/remove-call-inst.ll        |    45 +
 .../IPConstantProp/return-argument.ll         |    96 +
 .../IPConstantProp/return-constant.ll         |    55 +
 .../IPConstantProp/return-constants.ll        |    69 +
 ...fter-each-resolving-undefs-for-function.ll |    65 +
 .../IPConstantProp/thread_local_acs.ll        |    53 +
 llvm/test/Transforms/Attributor/align.ll      |   412 +
 llvm/test/Transforms/Attributor/callbacks.ll  |    66 +
 .../Attributor/dereferenceable-1.ll           |   312 +
 .../Attributor/dereferenceable-2.ll           |   356 +
 .../Transforms/Attributor/heap_to_stack.ll    |   402 +
 .../Transforms/Attributor/internal-noalias.ll |    48 +
 llvm/test/Transforms/Attributor/liveness.ll   |   948 ++
 .../Attributor/lvi-after-jumpthreading.ll     |   181 +
 .../Transforms/Attributor/lvi-for-ashr.ll     |    46 +
 llvm/test/Transforms/Attributor/misc.ll       |   105 +
 .../Transforms/Attributor/new_attributes.ll   |    43 +
 llvm/test/Transforms/Attributor/noalias.ll    |   284 +
 .../test/Transforms/Attributor/nocapture-1.ll |   346 +
 .../test/Transforms/Attributor/nocapture-2.ll |   471 +
 llvm/test/Transforms/Attributor/nofree.ll     |   243 +
 llvm/test/Transforms/Attributor/nonnull.ll    |   840 ++
 llvm/test/Transforms/Attributor/norecurse.ll  |   147 +
 llvm/test/Transforms/Attributor/noreturn.ll   |   149 +
 .../Transforms/Attributor/noreturn_async.ll   |   142 +
 .../Transforms/Attributor/noreturn_sync.ll    |   138 +
 llvm/test/Transforms/Attributor/nosync.ll     |   319 +
 llvm/test/Transforms/Attributor/nounwind.ll   |    98 +
 llvm/test/Transforms/Attributor/range.ll      |   504 +
 .../read_write_returned_arguments_scc.ll      |   165 +
 llvm/test/Transforms/Attributor/readattrs.ll  |   197 +
 llvm/test/Transforms/Attributor/returned.ll   |   812 ++
 .../Attributor/undefined_behavior.ll          |   305 +
 .../Transforms/Attributor/value-simplify.ll   |   316 +
 llvm/test/Transforms/Attributor/willreturn.ll |   482 +
 llvm/test/Transforms/BDCE/pr41925.ll          |   126 +
 .../Transforms/BlockExtractor/invalid-line.ll |     9 +
 .../ARM/bitreverse-recognize.ll               |     2 +-
 .../ARM/sink-add-mul-shufflevector.ll         |   219 +
 .../CodeGenPrepare/ARM/sinkchain.ll           |   107 +
 .../PowerPC/split-store-alignment.ll          |   111 +
 .../X86/memset_chk-simplify-nobuiltin.ll      |     7 +-
 .../CodeGenPrepare/X86/overflow-intrinsics.ll |    20 -
 .../CodeGenPrepare/X86/sink-addrmode.ll       |    41 +
 .../X86/split-store-alignment.ll              |    74 +
 .../CodeGenPrepare/bitreverse-hang.ll         |     2 +-
 .../Transforms/CodeGenPrepare/dom-tree.ll     |     2 +-
 .../CodeGenPrepare/sink-shift-and-trunc.ll    |     2 +-
 .../Transforms/ConstProp/InsertElement.ll     |    33 +-
 llvm/test/Transforms/ConstProp/fma.ll         |   246 +
 .../test/Transforms/ConstProp/overflow-ops.ll |    13 +-
 .../AArch64/const-hoist-intrinsics.ll         |    89 +
 .../AArch64/consthoist-unreachable.ll         |    64 +
 .../Transforms/ConstantHoisting/X86/ehpad.ll  |     4 +-
 .../X86/pr43903-not-all-uses-rebased.ll       |    51 +
 .../Coroutines/coro-alloc-with-param.ll       |    96 +
 llvm/test/Transforms/Coroutines/coro-debug.ll |    10 +-
 llvm/test/Transforms/Coroutines/coro-early.ll |     2 +-
 .../Transforms/Coroutines/coro-param-copy.ll  |    69 +
 .../Coroutines/coro-retcon-alloca.ll          |   247 +
 .../Coroutines/coro-retcon-once-value.ll      |   114 +
 .../Coroutines/coro-retcon-once-value2.ll     |    71 +
 .../Coroutines/coro-retcon-resume-values.ll   |    85 +
 .../Coroutines/coro-retcon-resume-values2.ll  |   100 +
 .../Coroutines/coro-retcon-value.ll           |   103 +
 .../test/Transforms/Coroutines/coro-retcon.ll |   113 +
 .../Transforms/Coroutines/coro-split-dbg.ll   |     8 +-
 .../Coroutines/coro-split-musttail.ll         |    27 +-
 .../Coroutines/coro-split-musttail1.ll        |   104 +
 .../Transforms/Coroutines/coro-swifterror.ll  |   143 +
 .../CorrelatedValuePropagation/and.ll         |   127 +
 .../CorrelatedValuePropagation/icmp.ll        |   381 +-
 .../CorrelatedValuePropagation/mul.ll         |   175 +
 .../CorrelatedValuePropagation/overflows.ll   |   587 +-
 .../phi-common-val.ll                         |    26 +
 .../CorrelatedValuePropagation/range.ll       |     3 +-
 .../CorrelatedValuePropagation/select.ll      |   163 +-
 .../CorrelatedValuePropagation/sext.ll        |   135 +
 .../CorrelatedValuePropagation/shl.ll         |   378 +
 llvm/test/Transforms/DCE/calls-errno.ll       |    17 +-
 llvm/test/Transforms/DCE/dbg-value-removal.ll |   112 +
 .../DeadArgElim/2010-04-30-DbgInfo.ll         |     2 +-
 llvm/test/Transforms/DeadArgElim/dbginfo.ll   |     4 +-
 .../Transforms/DeadArgElim/naked_functions.ll |     2 +-
 .../DeleteThrowableInst.ll                    |    41 +
 .../DeadStoreElimination/libcalls.ll          |    10 +-
 .../DeadStoreElimination/libcalls2.ll         |    14 +
 .../mda-with-dbg-values.ll                    |     2 +-
 .../Transforms/DeadStoreElimination/simple.ll |     4 +-
 .../DivRemPairs/X86/div-expanded-rem-pair.ll  |    60 +-
 .../DivRemPairs/X86/div-rem-pairs.ll          |     8 +-
 llvm/test/Transforms/EarlyCSE/commute.ll      |    85 +-
 .../Transforms/EarlyCSE/debug-info-undef.ll   |     2 +-
 .../test/Transforms/EarlyCSE/floatingpoint.ll |    12 +
 llvm/test/Transforms/EarlyCSE/writeonly.ll    |    15 +
 .../EntryExitInstrumenter/mcount.ll           |     4 +-
 .../Transforms/ExpandMemCmp/X86/memcmp.ll     |   140 +-
 llvm/test/Transforms/Float2Int/basic.ll       |   412 +-
 .../Transforms/FunctionAttrs/arg_returned.ll  |   176 +-
 .../Transforms/FunctionAttrs/comdat-ipo.ll    |     4 +-
 .../FunctionAttrs/naked_functions.ll          |     2 +-
 .../Transforms/FunctionAttrs/nocapture.ll     |   122 +-
 .../FunctionAttrs/nofree-attributor.ll        |    75 +-
 llvm/test/Transforms/FunctionAttrs/nonnull.ll |   559 +-
 .../Transforms/FunctionAttrs/norecurse.ll     |     4 +-
 .../test/Transforms/FunctionAttrs/nounwind.ll |    20 +-
 .../Transforms/FunctionAttrs/readattrs.ll     |    12 +-
 .../test/Transforms/FunctionAttrs/readnone.ll |     8 +-
 .../Transforms/FunctionAttrs/writeonly.ll     |    30 +
 .../FunctionImport/Inputs/comdat.ll           |     2 +-
 .../FunctionImport/Inputs/funcimport_debug.ll |     4 +-
 .../Inputs/funcimport_resolved1.ll            |     2 +-
 .../Inputs/funcimport_resolved2.ll            |     2 +-
 .../FunctionImport/Inputs/funcimport_var2.ll  |     2 +-
 .../Inputs/hotness_based_import.ll            |     2 +-
 .../FunctionImport/Inputs/import_stats.ll     |     2 +-
 .../FunctionImport/Inputs/not-prevailing.ll   |     2 +-
 llvm/test/Transforms/FunctionImport/comdat.ll |     2 +-
 .../FunctionImport/funcimport_debug.ll        |     6 +-
 .../FunctionImport/funcimport_resolved.ll     |     2 +-
 .../FunctionImport/funcimport_var.ll          |     2 +-
 .../FunctionImport/hotness_based_import.ll    |     2 +-
 .../Transforms/FunctionImport/import_stats.ll |     2 +-
 .../FunctionImport/not-prevailing.ll          |     2 +-
 .../Transforms/GCOVProfiling/global-ctor.ll.x |     2 +-
 .../test/Transforms/GCOVProfiling/linezero.ll |     4 +-
 .../GCOVProfiling/return-block.ll.x           |     4 +-
 .../2011-06-01-NonLocalMemdepMiscompile.ll    |    10 +-
 llvm/test/Transforms/GVN/PRE/rle.ll           |   125 +
 llvm/test/Transforms/GVN/cond_br2.ll          |     6 +-
 llvm/test/Transforms/GVN/edge.ll              |    69 +
 llvm/test/Transforms/GVN/equality-assume.ll   |   168 +
 llvm/test/Transforms/GVN/pr42605.ll           |    87 +
 llvm/test/Transforms/GVN/preserve-analysis.ll |    56 +
 .../GVNHoist/hoist-unsafe-pr31729.ll          |     2 +-
 llvm/test/Transforms/GVNHoist/pr30499.ll      |     4 +-
 .../GlobalDCE/virtual-functions-base-call.ll  |    78 +
 .../virtual-functions-base-pointer-call.ll    |   118 +
 .../virtual-functions-derived-call.ll         |    78 +
 .../virtual-functions-derived-pointer-call.ll |   120 +
 .../virtual-functions-visibility-post-lto.ll  |    95 +
 .../virtual-functions-visibility-pre-lto.ll   |    94 +
 .../Transforms/GlobalDCE/virtual-functions.ll |    55 +
 llvm/test/Transforms/GlobalDCE/vtable-rtti.ll |    47 +
 llvm/test/Transforms/GlobalOpt/atomic.ll      |     8 +-
 .../Transforms/GlobalOpt/large-int-crash.ll   |    23 +
 .../GlobalOpt/long-compilation-global-sra.ll  |    61 +
 .../Transforms/GlobalOpt/naked_functions.ll   |     2 +-
 .../basic_widenable_condition_guards.ll       |   143 +-
 .../Transforms/GuardWidening/mixed_guards.ll  |     3 +-
 .../HardwareLoops/ARM/calls-codegen.ll        |     5 +-
 .../Transforms/HardwareLoops/ARM/calls.ll     |    13 +-
 .../Transforms/HardwareLoops/ARM/counter.ll   |     2 +-
 .../Transforms/HardwareLoops/ARM/do-rem.ll    |     2 +-
 .../HardwareLoops/ARM/fp-emulation.ll         |     4 +-
 .../Transforms/HardwareLoops/ARM/simple-do.ll |     5 +-
 .../Transforms/HardwareLoops/ARM/structure.ll |    25 +-
 .../assumption-cache-invalidation.ll          |    50 +
 llvm/test/Transforms/HotColdSplit/noreturn.ll |    20 +
 .../Transforms/HotColdSplit/retain-section.ll |    28 +
 .../IPConstantProp/2008-06-09-WeakProp.ll     |     4 +-
 .../IPConstantProp/2009-09-24-byval-ptr.ll    |     6 +-
 .../test/Transforms/IPConstantProp/PR16052.ll |    19 +-
 .../test/Transforms/IPConstantProp/PR26044.ll |    58 +-
 .../test/Transforms/IPConstantProp/PR43857.ll |    29 +
 .../IPConstantProp/arg-count-mismatch.ll      |     5 +-
 .../IPConstantProp/multiple_callbacks.ll      |     1 +
 .../Transforms/IPConstantProp/naked-return.ll |     2 +-
 .../Transforms/IPConstantProp/recursion.ll    |     5 +-
 .../IPConstantProp/return-argument.ll         |     7 +-
 .../IPConstantProp/return-constant.ll         |     7 +-
 .../IPConstantProp/return-constants.ll        |     7 +-
 ...fter-each-resolving-undefs-for-function.ll |    17 +-
 .../IPConstantProp/user-with-multiple-uses.ll |     3 -
 .../IRCE/ranges_of_different_types.ll         |     8 +-
 .../test/Transforms/IRCE/rc-negative-bound.ll |    56 +-
 .../IndVarSimplify/eliminate-exit-no-dl.ll    |    45 +
 .../IndVarSimplify/eliminate-exit.ll          |    34 +
 .../IndVarSimplify/eliminate-trunc.ll         |     2 +-
 .../IndVarSimplify/exit_value_tests.ll        |     1 -
 .../IndVarSimplify/floating-point-iv.ll       |   231 +
 .../IndVarSimplify/indvar-debug-value.ll      |    92 +
 .../IndVarSimplify/indvar-debug-value2.ll     |    90 +
 .../IndVarSimplify/lftr-dead-ivs.ll           |     2 +-
 .../Transforms/IndVarSimplify/lftr-pr20680.ll |     6 +-
 .../Transforms/IndVarSimplify/lftr-reuse.ll   |     2 +-
 llvm/test/Transforms/IndVarSimplify/lftr.ll   |    31 +-
 .../loop-invariant-conditions.ll              |     2 +-
 .../IndVarSimplify/loop-predication.ll        |   858 ++
 .../IndVarSimplify/loop_evaluate_1.ll         |    41 +-
 .../IndVarSimplify/no-iv-rewrite.ll           |     2 +-
 .../test/Transforms/IndVarSimplify/pr24356.ll |     2 +-
 .../test/Transforms/IndVarSimplify/pr38674.ll |    11 +-
 .../test/Transforms/IndVarSimplify/pr39673.ll |     2 +-
 .../IndVarSimplify/replace-loop-exit-folds.ll |     2 +-
 .../Transforms/IndVarSimplify/rlev-add-me.ll  |   228 +
 .../IndVarSimplify/sink-from-preheader.ll     |     2 +-
 .../Transforms/IndVarSimplify/widen-nsw.ll    |     2 +-
 .../AMDGPU/address-space-id-funcs.ll          |    55 +
 .../Transforms/InferFunctionAttrs/annotate.ll |    16 +-
 .../InferFunctionAttrs/dereferenceable.ll     |    79 +
 llvm/test/Transforms/Inline/X86/switch.ll     |   160 +
 .../Inline/alloca-dbgdeclare-merge.ll         |     4 +-
 .../Inline/debug-info-duplicate-calls.ll      |     6 +-
 .../Inline/inline-indirect-chain.ll           |    55 +
 .../Inline/inline-skip-use-empty-alloca.ll    |    71 +
 llvm/test/Transforms/Inline/inline-vla.ll     |     4 +-
 llvm/test/Transforms/Inline/inline_invoke.ll  |     2 +-
 .../test/Transforms/Inline/inline_minisize.ll |     2 +-
 .../Inline/no-inline-line-tables.ll           |    99 +
 llvm/test/Transforms/Inline/noalias-calls.ll  |     2 +-
 .../optimization-remarks-hotness-threshold.ll |     2 +-
 .../optimization-remarks-passed-yaml.ll       |     2 +-
 .../Transforms/Inline/optimization-remarks.ll |     6 +-
 ...004-11-27-SetCCForCastLargerAndConstant.ll |   233 +-
 .../InstCombine/2008-12-17-SRemNegConstVec.ll |     7 +-
 .../2009-06-16-SRemDemandedBits.ll            |    15 +-
 .../InstCombine/2010-01-28-NegativeSRem.ll    |    15 +-
 .../AMDGPU/amdgcn-demanded-vector-elts.ll     |   803 +-
 .../InstCombine/AMDGPU/amdgcn-intrinsics.ll   |    10 +-
 .../test/Transforms/InstCombine/AMDGPU/tan.ll |    21 +
 .../Transforms/InstCombine/ARM/mve-v2i2v.ll   |   330 +
 .../test/Transforms/InstCombine/ARM/strcmp.ll |    63 +-
 llvm/test/Transforms/InstCombine/PR37526.ll   |     7 +-
 .../InstCombine/X86/2009-03-23-i80-fp80.ll    |    20 +
 .../Transforms/InstCombine/X86/x86-avx2.ll    |    21 +-
 .../Transforms/InstCombine/X86/x86-avx512.ll  |   154 +-
 .../Transforms/InstCombine/X86/x86-bmi-tbm.ll |   132 +
 .../Transforms/InstCombine/X86/x86-f16c.ll    |     4 +-
 .../Transforms/InstCombine/X86/x86-fma.ll     |     4 +-
 .../Transforms/InstCombine/X86/x86-pack.ll    |     9 +-
 .../Transforms/InstCombine/X86/x86-pshufb.ll  |    95 +-
 .../Transforms/InstCombine/X86/x86-sse.ll     |    80 +-
 .../Transforms/InstCombine/X86/x86-sse4a.ll   |    20 +-
 .../InstCombine/X86/x86-vector-shifts.ll      |   570 +-
 .../Transforms/InstCombine/X86/x86-vpermil.ll |    67 +-
 .../Transforms/InstCombine/X86/x86-xop.ll     |    32 +-
 llvm/test/Transforms/InstCombine/abs-1.ll     |   156 +-
 llvm/test/Transforms/InstCombine/add4.ll      |    33 +-
 .../Transforms/InstCombine/addrspacecast.ll   |     2 +-
 .../InstCombine/adjust-for-minmax.ll          |   164 +-
 .../test/Transforms/InstCombine/align-addr.ll |    64 +-
 .../Transforms/InstCombine/and-or-icmps.ll    |   145 +-
 llvm/test/Transforms/InstCombine/and.ll       |     3 +-
 llvm/test/Transforms/InstCombine/and2.ll      |    38 +-
 .../Transforms/InstCombine/apint-shift.ll     |     7 +-
 llvm/test/Transforms/InstCombine/apint-sub.ll |    38 +-
 llvm/test/Transforms/InstCombine/assume.ll    |   155 +-
 .../InstCombine/assume_inevitable.ll          |     4 +-
 llvm/test/Transforms/InstCombine/bcopy.ll     |    25 +
 .../InstCombine/bitcast-function.ll           |   206 +
 .../InstCombine/bitcast-phi-uselistorder.ll   |    33 +
 .../InstCombine/bitcast-vec-canon.ll          |    31 +
 .../Transforms/InstCombine/bitreverse-hang.ll |     2 +-
 llvm/test/Transforms/InstCombine/bswap.ll     |   115 +-
 .../builtin-dynamic-object-size.ll            |     8 +-
 .../builtin-object-size-custom-dl.ll          |    32 +
 .../InstCombine/builtin-object-size-ptr.ll    |    61 +-
 .../test/Transforms/InstCombine/call-guard.ll |     3 +-
 ...etween-negative-and-positive-thresholds.ll |   479 +
 ...ern-between-zero-and-positive-threshold.ll |   399 +
 ...th-select-of-constant-threshold-pattern.ll |   134 +
 ...nt-low-bit-mask-and-icmp-eq-to-icmp-ule.ll |     2 +-
 ...nt-low-bit-mask-and-icmp-ne-to-icmp-ugt.ll |    12 +-
 ...t-low-bit-mask-and-icmp-sge-to-icmp-sle.ll |     2 +-
 ...t-low-bit-mask-and-icmp-sgt-to-icmp-sgt.ll |    14 +-
 ...t-low-bit-mask-and-icmp-sle-to-icmp-sle.ll |     2 +-
 ...t-low-bit-mask-and-icmp-slt-to-icmp-sgt.ll |    12 +-
 ...t-low-bit-mask-and-icmp-uge-to-icmp-ule.ll |     2 +-
 ...t-low-bit-mask-and-icmp-ugt-to-icmp-ugt.ll |    14 +-
 ...t-low-bit-mask-and-icmp-ule-to-icmp-ule.ll |     2 +-
 ...t-low-bit-mask-and-icmp-ult-to-icmp-ugt.ll |    12 +-
 .../Transforms/InstCombine/cast-mul-select.ll |     2 +
 llvm/test/Transforms/InstCombine/cast.ll      |  1006 +-
 .../Transforms/InstCombine/clamp-to-minmax.ll |   157 +-
 .../InstCombine/compare-unescaped.ll          |   108 +-
 ...e-length-signext-after-high-bit-extract.ll |  1139 ++
 .../InstCombine/constant-fold-libfunc.ll      |     2 +-
 llvm/test/Transforms/InstCombine/copysign.ll  |    94 +-
 llvm/test/Transforms/InstCombine/cos-1.ll     |    20 +-
 .../Transforms/InstCombine/debuginfo-dce.ll   |     2 +-
 .../Transforms/InstCombine/deref-alloc-fns.ll |   229 +
 .../Transforms/InstCombine/disable-builtin.ll |    21 +
 .../Transforms/InstCombine/div-shift-crash.ll |     2 +-
 llvm/test/Transforms/InstCombine/div.ll       |     9 +-
 .../InstCombine/do-not-clone-dbg-declare.ll   |   144 +
 .../InstCombine/element-atomic-memintrins.ll  |    66 +-
 .../InstCombine/expensive-combines.ll         |    28 +
 .../Transforms/InstCombine/fabs-copysign.ll   |   119 +
 .../InstCombine/fadd-fsub-factor.ll           |   162 +
 llvm/test/Transforms/InstCombine/fadd.ll      |   362 +
 llvm/test/Transforms/InstCombine/fast-math.ll |     2 +-
 llvm/test/Transforms/InstCombine/fdiv.ll      |    71 +
 llvm/test/Transforms/InstCombine/fma.ll       |   351 +-
 llvm/test/Transforms/InstCombine/fmul.ll      |   180 +-
 .../Transforms/InstCombine/fortify-folding.ll |   131 +-
 llvm/test/Transforms/InstCombine/fpextend.ll  |   302 +-
 llvm/test/Transforms/InstCombine/fptrunc.ll   |   138 +
 llvm/test/Transforms/InstCombine/freeze.ll    |    20 +
 llvm/test/Transforms/InstCombine/fsub.ll      |   372 +
 .../Transforms/InstCombine/gc.relocate.ll     |    11 +
 llvm/test/Transforms/InstCombine/gep-alias.ll |    15 +
 .../Transforms/InstCombine/gep-custom-dl.ll   |    33 +-
 .../InstCombine/gep-inbounds-null.ll          |   236 +
 .../test/Transforms/InstCombine/gep-vector.ll |    80 +-
 .../Transforms/InstCombine/getelementptr.ll   |   865 +-
 .../high-bit-signmask-with-trunc.ll           |   148 +
 .../InstCombine/high-bit-signmask.ll          |   126 +
 ...n-out-of-bias-calculation-with-constant.ll |    98 +
 .../hoist-negation-out-of-bias-calculation.ll |   158 +
 llvm/test/Transforms/InstCombine/icmp-add.ll  |   164 +-
 .../Transforms/InstCombine/icmp-custom-dl.ll  |    10 +-
 .../InstCombine/icmp-div-constant.ll          |   136 +-
 .../Transforms/InstCombine/icmp-shr-lt-gt.ll  |   177 +-
 llvm/test/Transforms/InstCombine/icmp-sub.ll  |    59 +
 llvm/test/Transforms/InstCombine/icmp-vec.ll  |    19 +-
 llvm/test/Transforms/InstCombine/icmp.ll      |   213 +-
 .../InstCombine/insert-extract-shuffle.ll     |   161 +
 llvm/test/Transforms/InstCombine/intptr8.ll   |    77 +
 .../Transforms/InstCombine/known-non-zero.ll  |    46 +-
 .../InstCombine/lifetime-sanitizer.ll         |    64 +
 .../InstCombine/limit-max-iterations.ll       |    44 +
 .../InstCombine/load-bitcast-vec.ll           |    46 +
 .../Transforms/InstCombine/load-bitcast64.ll  |    63 +-
 llvm/test/Transforms/InstCombine/load-cmp.ll  |    50 +-
 .../InstCombine/load-insert-store.ll          |    98 +
 llvm/test/Transforms/InstCombine/load.ll      |    48 +
 llvm/test/Transforms/InstCombine/log-pow.ll   |   143 +-
 .../Transforms/InstCombine/logical-select.ll  |     4 +-
 .../InstCombine/lower-dbg-declare.ll          |   239 +-
 .../InstCombine/malloc-free-delete.ll         |   157 +-
 .../InstCombine/mem-deref-bytes-addrspaces.ll |    22 +
 .../Transforms/InstCombine/mem-deref-bytes.ll |   163 +
 .../InstCombine/mem-par-metadata-memcpy.ll    |     2 +-
 llvm/test/Transforms/InstCombine/memccpy.ll   |   214 +
 llvm/test/Transforms/InstCombine/memchr.ll    |    46 +-
 .../InstCombine/memcmp-constant-fold.ll       |    13 +-
 llvm/test/Transforms/InstCombine/memcpy-1.ll  |    12 +-
 .../InstCombine/memcpy-from-global.ll         |   213 +-
 .../Transforms/InstCombine/memcpy-to-load.ll  |     6 +-
 llvm/test/Transforms/InstCombine/memcpy.ll    |     2 +-
 .../Transforms/InstCombine/memcpy_chk-1.ll    |    34 +-
 .../Transforms/InstCombine/memmove_chk-1.ll   |    21 +-
 llvm/test/Transforms/InstCombine/mempcpy.ll   |    56 +
 llvm/test/Transforms/InstCombine/memrchr.ll   |    55 +
 llvm/test/Transforms/InstCombine/memset-1.ll  |   103 +-
 llvm/test/Transforms/InstCombine/memset.ll    |     2 +-
 .../Transforms/InstCombine/memset_chk-1.ll    |    73 +-
 .../Transforms/InstCombine/minmax-fold.ll     |   196 +-
 llvm/test/Transforms/InstCombine/minmax-fp.ll |    90 +-
 .../InstCombine/minmax-of-minmax.ll           |   326 +
 llvm/test/Transforms/InstCombine/mul.ll       |    90 +
 llvm/test/Transforms/InstCombine/not.ll       |   153 +
 .../test/Transforms/InstCombine/objsize-64.ll |    29 +-
 llvm/test/Transforms/InstCombine/objsize.ll   |   181 +-
 .../Transforms/InstCombine/overflow_to_sat.ll |   710 +
 ...nput-masking-after-truncation-variant-a.ll |   273 +
 ...nput-masking-after-truncation-variant-b.ll |   310 +
 ...nput-masking-after-truncation-variant-c.ll |   219 +
 ...nput-masking-after-truncation-variant-d.ll |   247 +
 ...nput-masking-after-truncation-variant-e.ll |   219 +
 ...dant-left-shift-input-masking-variant-a.ll |   157 +
 ...dant-left-shift-input-masking-variant-b.ll |   157 +
 ...dant-left-shift-input-masking-variant-c.ll |   117 +
 ...dant-left-shift-input-masking-variant-d.ll |   137 +
 ...dant-left-shift-input-masking-variant-e.ll |   117 +
 .../phi-equal-incoming-pointers.ll            |   616 +
 .../phi-known-bits-operand-order.ll           |    99 +
 llvm/test/Transforms/InstCombine/pow-exp.ll   |     1 +
 llvm/test/Transforms/InstCombine/pow-sqrt.ll  |   140 +-
 .../test/Transforms/InstCombine/pow_fp_int.ll |    64 +
 llvm/test/Transforms/InstCombine/pr26992.ll   |    31 +-
 llvm/test/Transforms/InstCombine/pr39177.ll   |     2 +-
 llvm/test/Transforms/InstCombine/pr43081.ll   |    15 +
 ...edStrictnessPredicateAndConstant-assert.ll |    36 +
 llvm/test/Transforms/InstCombine/pr43893.ll   |    55 +
 llvm/test/Transforms/InstCombine/pr44242.ll   |   190 +
 llvm/test/Transforms/InstCombine/pr44245.ll   |   192 +
 llvm/test/Transforms/InstCombine/pr44541.ll   |    25 +
 llvm/test/Transforms/InstCombine/pr44552.ll   |    59 +
 llvm/test/Transforms/InstCombine/pr44835.ll   |    29 +
 llvm/test/Transforms/InstCombine/printf-1.ll  |    97 +-
 llvm/test/Transforms/InstCombine/printf-2.ll  |    21 +-
 llvm/test/Transforms/InstCombine/puts-1.ll    |    11 +-
 llvm/test/Transforms/InstCombine/realloc.ll   |     4 +-
 ...nput-masking-after-truncation-variant-a.ll |   199 +
 ...nput-masking-after-truncation-variant-b.ll |   238 +
 ...nput-masking-after-truncation-variant-c.ll |   169 +
 ...nput-masking-after-truncation-variant-d.ll |   189 +
 ...nput-masking-after-truncation-variant-e.ll |   169 +
 ...nput-masking-after-truncation-variant-f.ll |   198 +
 ...dant-left-shift-input-masking-variant-a.ll |    47 +-
 ...dant-left-shift-input-masking-variant-b.ll |    47 +-
 ...dant-left-shift-input-masking-variant-c.ll |    58 +-
 ...dant-left-shift-input-masking-variant-d.ll |    81 +-
 ...dant-left-shift-input-masking-variant-e.ll |    70 +-
 ...dant-left-shift-input-masking-variant-f.ll |    61 +-
 llvm/test/Transforms/InstCombine/rem.ll       |    94 +-
 ...f-negative-is-non-zero-and-no-underflow.ll |   247 +
 ...ve-or-zero-is-non-zero-and-no-underflow.ll |   161 +
 ...ult-of-usub-is-non-zero-and-no-overflow.ll |   430 +
 .../reuse-constant-from-select-in-icmp.ll     |   336 +
 llvm/test/Transforms/InstCombine/sadd_sat.ll  |   503 +
 .../InstCombine/saturating-add-sub.ll         |    86 +
 .../InstCombine/select-bitext-bitwise-ops.ll  |    36 +-
 .../Transforms/InstCombine/select-crash.ll    |     4 +-
 .../InstCombine/select-ctlz-to-cttz.ll        |   219 +
 .../InstCombine/select-extractelement.ll      |   111 +-
 .../InstCombine/select-imm-canon.ll           |    70 +
 .../InstCombine/select-obo-peo-ops.ll         |   466 +-
 llvm/test/Transforms/InstCombine/select.ll    |   218 +-
 .../InstCombine/select_arithmetic.ll          |    45 +
 ...ciation-in-bittest-with-truncation-lshr.ll |   468 +
 ...ociation-in-bittest-with-truncation-shl.ll |   463 +
 .../shift-amount-reassociation-in-bittest.ll  |    28 +
 ...ount-reassociation-with-truncation-ashr.ll |   200 +
 ...ount-reassociation-with-truncation-lshr.ll |   200 +
 ...mount-reassociation-with-truncation-shl.ll |   212 +
 .../InstCombine/shift-amount-reassociation.ll |   134 +
 .../InstCombine/shift-by-signext.ll           |   173 +
 .../Transforms/InstCombine/shift-logic.ll     |   188 +
 llvm/test/Transforms/InstCombine/shift.ll     |   391 +-
 .../Transforms/InstCombine/shuffle_select.ll  |     3 +-
 .../InstCombine/shufflevector-div-rem.ll      |   122 +
 ...-test-via-right-shifting-all-other-bits.ll |   362 +
 .../Transforms/InstCombine/sink-alloca.ll     |     2 +-
 llvm/test/Transforms/InstCombine/snprintf.ll  |    10 +-
 llvm/test/Transforms/InstCombine/sprintf-1.ll |    85 +-
 llvm/test/Transforms/InstCombine/sqrt.ll      |    12 +
 .../InstCombine/srem-via-sdiv-mul-sub.ll      |   117 +
 .../InstCombine/stacksave-debuginfo.ll        |     9 +-
 .../Transforms/InstCombine/stdio-custom-dl.ll |    22 +
 llvm/test/Transforms/InstCombine/stpcpy-1.ll  |    28 +-
 .../Transforms/InstCombine/stpcpy_chk-1.ll    |    40 +-
 llvm/test/Transforms/InstCombine/strchr-1.ll  |    87 +-
 llvm/test/Transforms/InstCombine/strcmp-1.ll  |    75 +-
 .../Transforms/InstCombine/strcmp-memcmp.ll   |    72 +-
 llvm/test/Transforms/InstCombine/strcpy-1.ll  |    25 +-
 .../Transforms/InstCombine/strcpy_chk-1.ll    |    40 +-
 llvm/test/Transforms/InstCombine/strcspn-1.ll |    17 +-
 ...low-check-to-comparison-of-sub-operands.ll |   111 +
 llvm/test/Transforms/InstCombine/strlen-1.ll  |    34 +-
 llvm/test/Transforms/InstCombine/strncat-2.ll |    74 +-
 llvm/test/Transforms/InstCombine/strncat-3.ll |     6 +-
 llvm/test/Transforms/InstCombine/strncmp-1.ll |    87 +-
 llvm/test/Transforms/InstCombine/strncmp-2.ll |     6 +-
 llvm/test/Transforms/InstCombine/strncpy-1.ll |    90 +-
 llvm/test/Transforms/InstCombine/strncpy-2.ll |     5 +-
 .../Transforms/InstCombine/strncpy_chk-1.ll   |    26 +-
 llvm/test/Transforms/InstCombine/strndup.ll   |    67 +
 llvm/test/Transforms/InstCombine/strpbrk-1.ll |    20 +-
 llvm/test/Transforms/InstCombine/strrchr-1.ll |    55 +-
 llvm/test/Transforms/InstCombine/strstr-1.ll  |    41 +-
 .../InstCombine/sub-and-or-neg-xor.ll         |   115 +
 .../sub-ashr-and-to-icmp-select.ll            |   213 +
 .../InstCombine/sub-ashr-or-to-icmp-select.ll |   239 +
 llvm/test/Transforms/InstCombine/sub-gep.ll   |   101 +
 .../test/Transforms/InstCombine/sub-minmax.ll |    52 +-
 .../InstCombine/sub-of-negatible.ll           |   377 +
 .../Transforms/InstCombine/sub-or-and-xor.ll  |   104 +
 .../InstCombine/sub-xor-or-neg-and.ll         |   115 +
 llvm/test/Transforms/InstCombine/sub.ll       |    18 +-
 .../subtract-from-one-hand-of-select.ll       |    78 +
 .../subtract-of-one-hand-of-select.ll         |    78 +
 .../test/Transforms/InstCombine/trunc-load.ll |    50 +-
 llvm/test/Transforms/InstCombine/trunc.ll     |    18 +
 .../InstCombine/unavailable-debug.ll          |     2 +-
 .../InstCombine/unordered-fcmp-select.ll      |   157 +-
 .../test/Transforms/InstCombine/unpack-fca.ll |    18 +-
 .../unrecognized_three-way-comparison.ll      |   222 +
 ...gned-add-lack-of-overflow-check-via-add.ll |   236 +
 ...gned-add-lack-of-overflow-check-via-xor.ll |   189 +
 .../unsigned-add-lack-of-overflow-check.ll    |   199 +
 .../unsigned-add-overflow-check-via-add.ll    |   232 +
 .../unsigned-add-overflow-check-via-xor.ll    |   189 +
 .../unsigned-add-overflow-check.ll            |   199 +
 ...mul-lack-of-overflow-check-via-mul-udiv.ll |   173 +
 ...k-of-overflow-check-via-udiv-of-allones.ll |   109 +
 ...nsigned-mul-overflow-check-via-mul-udiv.ll |   167 +
 ...-mul-overflow-check-via-udiv-of-allones.ll |   105 +
 .../unsigned-sub-lack-of-overflow-check.ll    |   155 +
 .../unsigned-sub-overflow-check.ll            |   155 +
 .../InstCombine/unsigned_saturated_sub.ll     |   293 +
 .../Transforms/InstCombine/unused-nonnull.ll  |    45 +
 .../InstCombine/urem-via-udiv-mul-sub.ll      |   117 +
 ...signext-of-variable-high-bit-extraction.ll |   575 +
 .../InstCombine/vec_demanded_elts.ll          |    35 +-
 .../Transforms/InstCombine/vec_shuffle.ll     |   204 +-
 .../InstCombine/vec_udiv_to_shift.ll          |    16 +
 .../Transforms/InstCombine/vector-urem.ll     |     2 +-
 .../InstCombine/widenable-conditions.ll       |   156 +
 .../Transforms/InstCombine/with_overflow.ll   |    21 +-
 .../xor-of-icmps-with-extra-uses.ll           |   163 +
 .../InstCombine/zext-bool-add-sub.ll          |    20 +-
 .../Transforms/InstCombine/zext-or-icmp.ll    |     2 +-
 llvm/test/Transforms/InstCombine/zext.ll      |   268 +-
 .../Transforms/InstMerge/st_sink_split_bb.ll  |    94 +
 .../InstSimplify/assume-non-zero.ll           |   233 +
 llvm/test/Transforms/InstSimplify/assume.ll   |     8 +-
 llvm/test/Transforms/InstSimplify/call.ll     |   235 +
 .../div-by-0-guard-before-smul_ov-not.ll      |   102 +
 .../div-by-0-guard-before-smul_ov.ll          |    90 +
 .../div-by-0-guard-before-umul_ov-not.ll      |   102 +
 .../div-by-0-guard-before-umul_ov.ll          |    90 +
 .../Transforms/InstSimplify/fcmp-select.ll    |   168 +
 llvm/test/Transforms/InstSimplify/freeze.ll   |    20 +
 .../Transforms/InstSimplify/icmp-abs-nabs.ll  |    17 +
 .../Transforms/InstSimplify/insertelement.ll  |    23 +
 .../InstSimplify/known-never-nan.ll           |    39 +-
 .../Transforms/InstSimplify/known-non-zero.ll |    57 +-
 ...ck-in-uadd_with_overflow-of-nonnull-ptr.ll |   265 +
 .../InstSimplify/remove-dead-call.ll          |    28 +
 ...f-negative-is-non-zero-and-no-underflow.ll |   125 +
 ...-by-nonzero-is-non-zero-and-no-overflow.ll |    86 +
 ...ult-of-usub-is-non-zero-and-no-overflow.ll |   339 +
 llvm/test/Transforms/InstSimplify/select.ll   |   169 +-
 .../Transforms/InstSimplify/shufflevector.ll  |    27 +
 .../InstSimplify/unsigned-range-checks.ll     |   130 +
 .../ARM/interleaved-accesses.ll               |  1552 ++-
 .../Internalize/vcall-visibility.ll           |    64 +
 .../bb-unreachable-from-entry.ll              |     9 +-
 .../Transforms/JumpThreading/ddt-crash3.ll    |     2 +-
 .../Transforms/JumpThreading/header-succ.ll   |   130 +-
 .../Transforms/JumpThreading/induction.ll     |    17 +-
 .../Transforms/JumpThreading/thread-loads.ll  |     2 +-
 .../JumpThreading/unreachable-loops.ll        |    63 +
 .../JumpThreading/update-edge-weight.ll       |     1 +
 llvm/test/Transforms/LCSSA/pr44058.ll         |    37 +
 llvm/test/Transforms/LICM/AMDGPU/bitcast.ll   |    21 +
 llvm/test/Transforms/LICM/argmemonly-call.ll  |     6 +-
 llvm/test/Transforms/LICM/atomics.ll          |    11 +-
 llvm/test/Transforms/LICM/callbr-crash.ll     |    18 +
 llvm/test/Transforms/LICM/explicit_guards.ll  |    49 +-
 llvm/test/Transforms/LICM/guards.ll           |    33 +-
 llvm/test/Transforms/LICM/hoist-deref-load.ll |     2 +-
 llvm/test/Transforms/LICM/hoist-mustexec.ll   |     4 +-
 llvm/test/Transforms/LICM/hoist-phi.ll        |     4 +-
 llvm/test/Transforms/LICM/hoisting.ll         |    26 +-
 llvm/test/Transforms/LICM/pr38513.ll          |    27 +
 llvm/test/Transforms/LICM/pr42969.ll          |    70 +
 .../Transforms/LICM/pragma-licm-disable.ll    |    33 +
 llvm/test/Transforms/LICM/promote-order.ll    |     9 +-
 llvm/test/Transforms/LICM/read-only-calls.ll  |     5 +-
 llvm/test/Transforms/LICM/store-hoisting.ll   |     3 +-
 llvm/test/Transforms/LICM/volatile-alias.ll   |     2 +-
 .../AMDGPU/vect-ptr-ptr-size-mismatch.ll      |    96 +
 .../test/Transforms/LoopFusion/cannot_fuse.ll |   307 +-
 .../LoopFusion/diagnostics_analysis.ll        |   130 +
 .../LoopFusion/diagnostics_missed.ll          |   269 +
 llvm/test/Transforms/LoopFusion/four_loops.ll |   146 +-
 llvm/test/Transforms/LoopFusion/guarded.ll    |    65 +
 llvm/test/Transforms/LoopFusion/loop_nest.ll  |   101 +-
 llvm/test/Transforms/LoopFusion/simple.ll     |   365 +-
 .../test/Transforms/LoopInstSimplify/basic.ll |     2 +-
 .../LoopInterchange/interchangeable.ll        |    58 +-
 .../LoopInterchange/perserve-lcssa.ll         |    33 +-
 .../LoopInterchange/phi-ordering.ll           |    10 +-
 .../pr43176-move-to-new-latch.ll              |   140 +
 ...r43473-invalid-lcssa-phis-in-inner-exit.ll |   108 +
 ...97-lcssa-for-multiple-outer-loop-blocks.ll |   150 +
 .../reductions-across-inner-and-outer-loop.ll |     2 +
 .../update-condbranch-duplicate-successors.ll |   145 +
 .../basic_widenable_branch_guards.ll          |    66 +
 .../LoopPredication/predicate-exits.ll        |  1110 ++
 .../Transforms/LoopReroll/reroll_with_dbg.ll  |     2 +-
 llvm/test/Transforms/LoopRotate/basic.ll      |     2 +-
 .../LoopRotate/dbg-value-duplicates-2.ll      |    77 +
 llvm/test/Transforms/LoopRotate/pr35210.ll    |     2 +-
 llvm/test/Transforms/LoopRotate/switch.ll     |   166 +
 .../test/Transforms/LoopSimplify/basictest.ll |   240 +-
 ...fferent-addrspace-addressing-mode-loops.ll |     2 +-
 .../LoopStrengthReduce/X86/pr17473.ll         |     4 +-
 .../Transforms/LoopStrengthReduce/pr18165.ll  |     4 +-
 .../two-combinations-bug.ll                   |     2 +-
 .../LoopUnroll/AArch64/unroll-optsize.ll      |     4 +-
 .../LoopUnroll/AMDGPU/unroll-for-private.ll   |     2 +-
 .../LoopUnroll/ARM/dont-unroll-loopdec.ll     |    40 +
 .../Transforms/LoopUnroll/ARM/mve-nounroll.ll |   176 +
 .../LoopUnroll/disable-full-unroll-by-opt.ll  |    35 +
 .../LoopUnroll/optsize-loop-size.ll           |    58 +
 .../LoopUnroll/peel-loop-conditions-pgo-1.ll  |    43 +
 .../LoopUnroll/peel-loop-conditions-pgo-2.ll  |    43 +
 .../LoopUnroll/peel-loop-conditions.ll        |   598 +-
 .../Transforms/LoopUnroll/peel-loop-inner.ll  |    50 +
 .../LoopUnroll/peel-loop-pgo-deopt-idom-2.ll  |    46 +
 .../LoopUnroll/peel-loop-pgo-deopt.ll         |    19 +-
 .../Transforms/LoopUnroll/peel-loop-pgo.ll    |     8 +-
 .../LoopUnroll/runtime-epilog-debuginfo.ll    |     2 +-
 .../LoopUnroll/runtime-loop-multiple-exits.ll |     6 +-
 .../LoopUnroll/runtime-small-upperbound.ll    |    70 +
 llvm/test/Transforms/LoopUnroll/scevunroll.ll |    10 +-
 .../LoopUnroll/unroll-preserve-scev-lcssa.ll  |    71 +
 .../LoopUnrollAndJam/dependencies.ll          |     1 +
 .../Transforms/LoopUnrollAndJam/disable.ll    |     1 +
 .../LoopUnrollAndJam/pragma-explicit.ll       |     1 +
 .../LoopUnrollAndJam/unroll-and-jam.ll        |     1 +
 .../AArch64/arbitrary-induction-step.ll       |    18 +-
 .../LoopVectorize/AArch64/pr33053.ll          |     2 +-
 .../LoopVectorize/ARM/arm-ieee-vectorize.ll   |     9 +-
 .../LoopVectorize/ARM/mve-interleaved-cost.ll |  1111 ++
 .../LoopVectorize/ARM/mve-maskedldst.ll       |   176 +
 .../LoopVectorize/ARM/mve-shiftcost.ll        |    87 +
 .../Transforms/LoopVectorize/ARM/mve-vldn.ll  |    87 +
 .../ARM/prefer-tail-loop-folding.ll           |   655 +
 .../ARM/tail-folding-counting-down.ll         |    47 +
 .../LoopVectorize/ARM/tail-loop-folding.ll    |   120 +
 .../LoopVectorize/PowerPC/reg-usage.ll        |   281 +
 .../LoopVectorize/PowerPC/vectorize-bswap.ll  |    97 +
 .../predicated-first-order-recurrence.ll      |   103 +
 .../X86/consecutive-ptr-uniforms.ll           |    83 +
 .../LoopVectorize/X86/constant-fold.ll        |     2 +-
 .../LoopVectorize/X86/cost-model-assert.ll    |   127 +
 .../illegal-parallel-loop-uniform-write.ll    |     2 +-
 .../LoopVectorize/X86/int128_no_gather.ll     |     4 +-
 .../LoopVectorize/X86/interleave_short_tc.ll  |    59 +
 .../interleaved-accesses-waw-dependency.ll    |   110 +
 .../LoopVectorize/X86/interleaving.ll         |    12 +-
 .../LoopVectorize/X86/load-deref-pred.ll      |  2128 +++
 .../X86/no_fpmath_with_hotness.ll             |     2 +-
 .../Transforms/LoopVectorize/X86/pr42674.ll   |    61 +
 .../LoopVectorize/X86/reg-usage-debug.ll      |    12 +-
 .../Transforms/LoopVectorize/X86/reg-usage.ll |    34 +-
 .../LoopVectorize/X86/scatter_crash.ll        |     2 +-
 .../X86/tail_folding_and_assume_safety.ll     |   166 +
 .../LoopVectorize/X86/tail_loop_folding.ll    |   152 +
 .../X86/vect.omp.force.small-tc.ll            |    22 +-
 .../X86/vectorization-remarks-profitable.ll   |     2 +-
 .../Transforms/LoopVectorize/dbg.value.ll     |     2 +-
 .../test/Transforms/LoopVectorize/debugloc.ll |     2 +-
 .../diag-missing-instr-debug-loc.ll           |     2 +-
 .../LoopVectorize/diag-with-hotness-info-2.ll |     2 +-
 .../LoopVectorize/diag-with-hotness-info.ll   |     2 +-
 .../first-order-recurrence-complex.ll         |   275 +
 ...t-order-recurrence-multiply-recurrences.ll |    45 +
 .../LoopVectorize/first-order-recurrence.ll   |    73 +
 .../Transforms/LoopVectorize/hoist-loads.ll   |     4 +-
 .../LoopVectorize/if-conversion-nest.ll       |    54 +-
 .../LoopVectorize/if-pred-stores.ll           |     4 +-
 .../LoopVectorize/incorrect-dom-info.ll       |     2 +-
 .../interleaved-accesses-uniform-load.ll      |    49 +
 .../LoopVectorize/interleaved-accesses.ll     |    10 +-
 .../LoopVectorize/metadata-width.ll           |     2 +-
 .../LoopVectorize/multiple-address-spaces.ll  |     2 +-
 .../LoopVectorize/no_int_induction.ll         |     4 +-
 .../LoopVectorize/nofloat-report.ll           |    27 +
 llvm/test/Transforms/LoopVectorize/nuw.ll     |    58 +
 llvm/test/Transforms/LoopVectorize/optsize.ll |    37 +
 .../LoopVectorize/pr44488-predication.ll      |    79 +
 .../LoopVectorize/reduction-small-size.ll     |     2 +-
 .../Transforms/LoopVectorize/reduction.ll     |     2 +-
 .../runtime-check-address-space.ll            |     2 +-
 .../runtime-check-readonly-address-space.ll   |     2 +-
 .../Transforms/LoopVectorize/runtime-check.ll |    64 +-
 .../LoopVectorize/scev-exitlim-crash.ll       |     4 +-
 .../tail-folding-counting-down.ll             |    42 +
 .../LoopVectorize/vectorize-once.ll           |     2 +-
 .../constant-intrinsics.ll                    |   114 +
 .../crash-on-large-allocas.ll                 |    16 +
 .../objectsize_basic.ll                       |    83 +
 .../Transforms/LowerExpectIntrinsic/basic.ll  |    31 +-
 .../bigger-expressions-double.ll              |   513 +
 .../multiply-double-contraction-fmf.ll        |    69 +
 .../multiply-double-contraction.ll            |    69 +
 .../LowerMatrixIntrinsics/multiply-double.ll  |   254 +
 .../multiply-float-contraction-fmf.ll         |    69 +
 .../multiply-float-contraction.ll             |    69 +
 .../LowerMatrixIntrinsics/multiply-float.ll   |   254 +
 .../LowerMatrixIntrinsics/multiply-i32.ll     |   254 +
 .../propagate-backward.ll                     |    96 +
 .../propagate-backwards-unsupported.ll        |   135 +
 .../propagate-forward.ll                      |   116 +
 .../propagate-mixed-users.ll                  |    53 +
 .../propagate-multiple-iterations.ll          |    84 +
 .../strided-load-double.ll                    |    68 +
 .../strided-load-float.ll                     |    68 +
 .../LowerMatrixIntrinsics/strided-load-i32.ll |    68 +
 .../strided-store-double.ll                   |    68 +
 .../strided-store-float.ll                    |    68 +
 .../strided-store-i32.ll                      |    68 +
 .../LowerMatrixIntrinsics/transpose-double.ll |   114 +
 .../LowerMatrixIntrinsics/transpose-float.ll  |   114 +
 .../LowerMatrixIntrinsics/transpose-i32.ll    |   114 +
 llvm/test/Transforms/LowerTypeTests/align.ll  |    16 +
 .../LowerTypeTests/export-rename-local.ll     |    15 +
 .../Transforms/LowerTypeTests/import-icall.ll |    18 +-
 .../MemCpyOpt/aggregate-type-crash.ll         |    30 +
 llvm/test/Transforms/MemCpyOpt/lifetime.ll    |     2 +-
 .../memcpy-to-memset-with-lifetimes.ll        |    37 +-
 llvm/test/Transforms/MemCpyOpt/memcpy.ll      |     2 +-
 .../Transforms/MemCpyOpt/store-to-memset.ll   |    77 +
 .../MergeFunc/byval-attr-congruent-type.ll    |    30 +
 llvm/test/Transforms/MergeFunc/comdat.ll      |     2 +-
 .../MergeFunc/merge-block-address.ll          |     1 +
 .../Transforms/MergeFunc/phi-check-blocks.ll  |     8 +-
 llvm/test/Transforms/MergeFunc/weak-small.ll  |     8 +-
 .../MergeICmps/X86/int64-and-ptr.ll           |     2 +-
 .../test/Transforms/MergeICmps/X86/pr41917.ll |     4 +-
 .../Transforms/NewGVN/basic-cyclic-opt.ll     |     2 +-
 llvm/test/Transforms/NewGVN/cond_br2.ll       |     6 +-
 llvm/test/Transforms/NewGVN/equivalent-phi.ll |     2 +-
 .../test/Transforms/NewGVN/memory-handling.ll |     6 +-
 .../Transforms/NewGVN/phi-edge-handling.ll    |     2 +-
 llvm/test/Transforms/NewGVN/pr31483.ll        |     4 +-
 llvm/test/Transforms/NewGVN/pr31501.ll        |     2 +-
 llvm/test/Transforms/NewGVN/pr33187.ll        |     2 +-
 llvm/test/Transforms/NewGVN/pr33305.ll        |     4 +-
 llvm/test/Transforms/NewGVN/pr34430.ll        |     2 +-
 llvm/test/Transforms/NewGVN/pr34452.ll        |     2 +-
 llvm/test/Transforms/NewGVN/pr43441.ll        |    42 +
 llvm/test/Transforms/ObjCARC/basic.ll         |     2 +-
 llvm/test/Transforms/ObjCARC/code-motion.ll   |    39 +
 ...e-that-exception-unwind-path-is-visited.ll |     2 +-
 .../inlined-autorelease-return-value.ll       |   292 +
 llvm/test/Transforms/ObjCARC/nested.ll        |     2 +-
 llvm/test/Transforms/ObjCARC/post-inlining.ll |    64 +
 .../Transforms/ObjCARC/unsafe-claim-rv.ll     |     3 +-
 .../Inputs/thinlto_cspgo_bar_gen.ll           |     2 +-
 .../Inputs/thinlto_cspgo_bar_use.ll           |     2 +-
 .../Inputs/thinlto_indirect_call_promotion.ll |     2 +-
 .../Inputs/thinlto_samplepgo_icp.ll           |     2 +-
 .../Inputs/thinlto_samplepgo_icp2a.ll         |     2 +-
 .../Inputs/thinlto_samplepgo_icp2b.ll         |     2 +-
 .../Inputs/thinlto_samplepgo_icp3.ll          |     2 +-
 llvm/test/Transforms/PGOProfile/X86/macho.ll  |     2 +-
 llvm/test/Transforms/PGOProfile/branch1.ll    |     2 +-
 llvm/test/Transforms/PGOProfile/branch2.ll    |     2 +-
 llvm/test/Transforms/PGOProfile/chr.ll        |   205 +-
 .../Transforms/PGOProfile/comdat_internal.ll  |     7 +-
 .../Transforms/PGOProfile/criticaledge.ll     |     2 +-
 .../PGOProfile/cspgo_profile_summary.ll       |     2 +-
 .../PGOProfile/diag_FE_profile.ll.x           |     2 +-
 .../Transforms/PGOProfile/diag_mismatch.ll    |     2 +-
 .../PGOProfile/diag_no_funcprofdata.ll        |     2 +-
 .../PGOProfile/diag_no_profile.ll.x           |     2 +-
 .../PGOProfile/diag_no_value_sites.ll         |     2 +-
 .../PGOProfile/do-not-instrument.ll           |     2 +-
 llvm/test/Transforms/PGOProfile/func_entry.ll |    29 +-
 .../PGOProfile/icp_covariant_call_return.ll   |     2 +-
 .../PGOProfile/icp_covariant_invoke_return.ll |     2 +-
 llvm/test/Transforms/PGOProfile/icp_invoke.ll |     2 +-
 .../Transforms/PGOProfile/icp_invoke_nouse.ll |     2 +-
 .../Transforms/PGOProfile/icp_mismatch_msg.ll |     2 +-
 llvm/test/Transforms/PGOProfile/icp_vararg.ll |     2 +-
 .../PGOProfile/indirect_call_annotation.ll    |     2 +-
 .../PGOProfile/indirect_call_profile.ll       |     2 +-
 .../PGOProfile/indirect_call_promotion.ll     |     2 +-
 .../PGOProfile/indirect_call_promotion_vla.ll |     2 +-
 llvm/test/Transforms/PGOProfile/indirectbr.ll |     2 +-
 llvm/test/Transforms/PGOProfile/landingpad.ll |     2 +-
 .../PGOProfile/large_count_remarks.ll         |     2 +-
 llvm/test/Transforms/PGOProfile/loop1.ll      |     2 +-
 llvm/test/Transforms/PGOProfile/loop2.ll      |     2 +-
 llvm/test/Transforms/PGOProfile/memcpy.ll     |     2 +-
 .../PGOProfile/memop_size_annotation.ll       |     2 +-
 .../Transforms/PGOProfile/memop_size_opt.ll   |     2 +-
 .../PGOProfile/memop_size_opt_zero.ll         |     2 +-
 .../PGOProfile/misexpect-branch-correct.ll    |    94 +
 .../PGOProfile/misexpect-branch-stripped.ll   |   115 +
 .../misexpect-branch-unpredictable.ll         |    89 +
 .../Transforms/PGOProfile/misexpect-branch.ll |   130 +
 .../PGOProfile/misexpect-switch-default.ll    |   196 +
 .../Transforms/PGOProfile/misexpect-switch.ll |   293 +
 .../PGOProfile/multiple_hash_profile.ll       |     2 +-
 .../Transforms/PGOProfile/noreturncall.ll     |     2 +-
 llvm/test/Transforms/PGOProfile/preinline.ll  |     2 +-
 llvm/test/Transforms/PGOProfile/remap.ll      |     2 +-
 llvm/test/Transforms/PGOProfile/select1.ll    |     2 +-
 llvm/test/Transforms/PGOProfile/select2.ll    |     2 +-
 .../PGOProfile/select_hash_conflict.ll        |     2 +-
 llvm/test/Transforms/PGOProfile/single_bb.ll  |     2 +-
 .../PGOProfile/statics_counter_naming.ll      |     2 +-
 llvm/test/Transforms/PGOProfile/switch.ll     |     2 +-
 .../PGOProfile/thinlto_cspgo_gen.ll           |     2 +-
 .../PGOProfile/thinlto_cspgo_use.ll           |     2 +-
 .../thinlto_indirect_call_promotion.ll        |     2 +-
 .../PGOProfile/thinlto_samplepgo_icp.ll       |     2 +-
 .../PGOProfile/thinlto_samplepgo_icp2.ll      |     2 +-
 .../PGOProfile/thinlto_samplepgo_icp3.ll      |     2 +-
 .../thinlto_samplepgo_icp_droppeddead.ll      |     2 +-
 .../Transforms/PGOProfile/unreachable_bb.ll   |     2 +-
 .../PhaseOrdering/lifetime-sanitizer.ll       |    71 +
 .../PhaseOrdering/min-max-abs-cse.ll          |   114 +
 .../PhaseOrdering/scev-custom-dl.ll           |    70 +
 .../PhaseOrdering/simplifycfg-options.ll      |     8 +-
 .../PhaseOrdering/two-shifts-by-sext.ll       |   125 +
 .../unsigned-multiply-overflow-check.ll       |   161 +
 .../PreISelIntrinsicLowering/objc-arc.ll      |    10 +-
 .../Reassociate/2002-05-15-MissedTree.ll      |    16 +-
 .../Reassociate/2002-05-15-SubReassociate.ll  |     9 +-
 .../Reassociate/2019-08-22-FNegAssert.ll      |    24 +
 .../Reassociate/canonicalize-neg-const.ll     |   385 +-
 llvm/test/Transforms/Reassociate/commute.ll   |    15 +-
 .../Reassociate/fast-ReassociateVector.ll     |    19 +-
 .../Reassociate/fast-SubReassociate.ll        |    39 +-
 .../Transforms/Reassociate/fast-basictest.ll  |    21 +-
 .../Transforms/Reassociate/fast-fp-commute.ll |    11 +-
 .../Transforms/Reassociate/fast-multistep.ll  |    13 +-
 llvm/test/Transforms/Reassociate/looptest.ll  |    74 +-
 .../Reassociate/mixed-fast-nonfast-fp.ll      |    17 +-
 llvm/test/Transforms/Reassociate/mulfactor.ll |    39 +-
 llvm/test/Transforms/Reassociate/multistep.ll |    13 +-
 llvm/test/Transforms/Reassociate/no-op.ll     |    24 +-
 .../Transforms/Reassociate/optional-flags.ll  |    25 +-
 .../Transforms/Reassociate/propagate-flags.ll |     3 +-
 .../Reassociate/reassoc-intermediate-fnegs.ll |    10 +-
 .../Reassociate/reassociate-landingpad.ll     |     2 +-
 .../reassociate_salvages_debug_info.ll        |    50 +
 .../Transforms/Reassociate/shift-factor.ll    |     3 +-
 ...f_intrinsics_when_deleting_instructions.ll |    95 +
 .../Transforms/Reassociate/vaarg_movable.ll   |    21 +-
 .../test/Transforms/Reassociate/wrap-flags.ll |    43 +-
 .../Transforms/Reassociate/xor_reassoc.ll     |   198 +-
 llvm/test/Transforms/SCCP/apint-basictest3.ll |     2 +-
 llvm/test/Transforms/SCCP/indirectbr.ll       |    34 +-
 .../Transforms/SCCP/ipsccp-predinfo-order.ll  |    76 +
 .../SCCP/struct-arg-resolve-undefs.ll         |    49 +
 llvm/test/Transforms/SCCP/ubsan_overflow.ll   |    13 +
 .../SLPVectorizer/AArch64/ext-trunc.ll        |    30 +-
 .../SLPVectorizer/AArch64/gather-root.ll      |   102 +-
 .../SLPVectorizer/AArch64/getelementptr.ll    |    57 +-
 .../SLPVectorizer/AArch64/horizontal.ll       |    16 -
 .../SLPVectorizer/AArch64/spillcost-di.ll     |     4 +-
 .../SLPVectorizer/AArch64/transpose.ll        |    99 +-
 .../address-space-ptr-sze-gep-index-assert.ll |    13 +
 .../NVPTX/non-vectorizable-intrinsic.ll       |     2 +-
 .../Transforms/SLPVectorizer/X86/PR31847.ll   |   153 +
 .../Transforms/SLPVectorizer/X86/PR34635.ll   |   175 +-
 .../Transforms/SLPVectorizer/X86/PR35628_1.ll |    13 +-
 .../Transforms/SLPVectorizer/X86/PR35628_2.ll |     5 -
 .../Transforms/SLPVectorizer/X86/PR35777.ll   |    46 +-
 .../Transforms/SLPVectorizer/X86/PR39774.ll   |    72 +-
 .../Transforms/SLPVectorizer/X86/PR40310.ll   |    16 -
 .../Transforms/SLPVectorizer/X86/align.ll     |     2 +-
 .../SLPVectorizer/X86/alternate-cast.ll       |    98 +-
 .../SLPVectorizer/X86/alternate-int.ll        |    41 +-
 .../SLPVectorizer/X86/arith-add-ssat.ll       |    85 +-
 .../SLPVectorizer/X86/arith-add-usat.ll       |   127 +-
 .../Transforms/SLPVectorizer/X86/arith-add.ll |    91 +-
 .../Transforms/SLPVectorizer/X86/arith-fix.ll |   250 +-
 .../Transforms/SLPVectorizer/X86/arith-fp.ll  |    48 +-
 .../Transforms/SLPVectorizer/X86/arith-mul.ll |    91 +-
 .../SLPVectorizer/X86/arith-sub-ssat.ll       |    85 +-
 .../SLPVectorizer/X86/arith-sub-usat.ll       |   127 +-
 .../Transforms/SLPVectorizer/X86/arith-sub.ll |    91 +-
 .../SLPVectorizer/X86/bad-reduction.ll        |   358 +
 .../SLPVectorizer/X86/bitreverse.ll           |    28 +-
 .../SLPVectorizer/X86/blending-shuffle.ll     |    47 +-
 .../test/Transforms/SLPVectorizer/X86/call.ll |     2 +-
 .../test/Transforms/SLPVectorizer/X86/cast.ll |    43 +-
 .../SLPVectorizer/X86/consecutive-access.ll   |    34 +-
 .../SLPVectorizer/X86/crash_bullet3.ll        |     2 +-
 .../SLPVectorizer/X86/crash_flop7.ll          |     2 +-
 .../Transforms/SLPVectorizer/X86/crash_gep.ll |    23 +
 .../SLPVectorizer/X86/crash_smallpt.ll        |     2 +-
 .../test/Transforms/SLPVectorizer/X86/ctlz.ll |   346 +-
 .../Transforms/SLPVectorizer/X86/ctpop.ll     |    90 +-
 .../test/Transforms/SLPVectorizer/X86/cttz.ll |    44 +-
 .../SLPVectorizer/X86/debug_info.ll           |     2 +-
 .../SLPVectorizer/X86/different-vec-widths.ll |   104 +
 .../Transforms/SLPVectorizer/X86/fptosi.ll    |    37 +-
 .../Transforms/SLPVectorizer/X86/fptoui.ll    |    43 +-
 .../test/Transforms/SLPVectorizer/X86/hadd.ll |    57 +-
 .../SLPVectorizer/X86/horizontal-list.ll      |   354 +-
 .../SLPVectorizer/X86/horizontal-minmax.ll    |   800 +-
 .../SLPVectorizer/X86/horizontal.ll           |   148 -
 .../test/Transforms/SLPVectorizer/X86/hsub.ll |    57 +-
 .../X86/insert-element-build-vector.ll        |   626 +-
 .../SLPVectorizer/X86/jumbled_store_crash.ll  |   104 +
 .../SLPVectorizer/X86/load-merge.ll           |   158 +
 .../SLPVectorizer/X86/long_chains.ll          |     8 +-
 .../Transforms/SLPVectorizer/X86/lookahead.ll |   406 +-
 .../SLPVectorizer/X86/no_alternate_divrem.ll  |   129 +
 .../Transforms/SLPVectorizer/X86/pr16899.ll   |     2 +-
 .../Transforms/SLPVectorizer/X86/pr19657.ll   |   104 +-
 .../Transforms/SLPVectorizer/X86/pr35497.ll   |    13 +-
 .../Transforms/SLPVectorizer/X86/pr42022.ll   |   278 +
 .../SLPVectorizer/X86/reassociated-loads.ll   |    31 -
 .../Transforms/SLPVectorizer/X86/reduction.ll |    84 +
 .../SLPVectorizer/X86/reduction2.ll           |   117 +-
 .../SLPVectorizer/X86/reduction_loads.ll      |    24 -
 .../SLPVectorizer/X86/reduction_unrolled.ll   |    35 +-
 .../SLPVectorizer/X86/remark_horcost.ll       |     6 +-
 .../SLPVectorizer/X86/reorder_repeated_ops.ll |   122 +-
 .../X86/reuse-extracts-in-wider-vect.ll       |    67 +
 .../test/Transforms/SLPVectorizer/X86/sext.ll |  1173 +-
 .../SLPVectorizer/X86/shift-ashr.ll           |    77 +-
 .../SLPVectorizer/X86/shift-lshr.ll           |    77 +-
 .../Transforms/SLPVectorizer/X86/shift-shl.ll |    77 +-
 .../Transforms/SLPVectorizer/X86/sitofp.ll    |   115 +-
 .../test/Transforms/SLPVectorizer/X86/sqrt.ll |     7 +-
 .../SLPVectorizer/X86/store-jumbled.ll        |     7 +-
 .../SLPVectorizer/X86/stores_vectorize.ll     |    45 +-
 .../Transforms/SLPVectorizer/X86/uitofp.ll    |   115 +-
 .../SLPVectorizer/X86/undef_vect.ll           |    10 -
 .../SLPVectorizer/X86/used-reduced-op.ll      |   529 +
 .../SLPVectorizer/X86/vector_gep.ll           |     2 +-
 .../X86/vectorize-reorder-reuse.ll            |    42 -
 .../test/Transforms/SLPVectorizer/X86/zext.ll |  1098 +-
 llvm/test/Transforms/SROA/addrspacecast.ll    |    11 +-
 llvm/test/Transforms/SROA/alignment.ll        |    15 +
 .../Transforms/SROA/mem-par-metadata-sroa.ll  |     2 +-
 llvm/test/Transforms/SROA/tbaa-struct.ll      |    32 +
 .../SROA/vector-promotion-different-size.ll   |    24 +
 llvm/test/Transforms/SafeStack/ARM/debug.ll   |     6 +-
 .../Transforms/SafeStack/X86/debug-loc.ll     |     4 +-
 .../Transforms/SafeStack/X86/debug-loc2.ll    |    13 +-
 .../Inputs/profile-symbol-list.ll             |   134 +
 llvm/test/Transforms/SampleProfile/branch.ll  |     6 +-
 .../compressed-profile-symbol-list.ll         |     5 +
 .../SampleProfile/entry_counts_cold.ll        |     4 +-
 .../entry_counts_missing_dbginfo.ll           |     4 +-
 .../Transforms/SampleProfile/gcc-simple.ll    |     4 +-
 .../Transforms/SampleProfile/inline-act.ll    |     2 +-
 .../SampleProfile/inline-callee-update.ll     |    73 +
 .../Transforms/SampleProfile/inline-cold.ll   |   102 +
 .../SampleProfile/inline-coverage.ll          |     2 +-
 .../SampleProfile/inline-mergeprof.ll         |    97 +
 .../Transforms/SampleProfile/inline-stats.ll  |   104 +
 .../SampleProfile/inline-topdown.ll           |   123 +
 llvm/test/Transforms/SampleProfile/offset.ll  |     2 +-
 .../SampleProfile/profile-format-compress.ll  |   123 +
 .../SampleProfile/profile-format.ll           |   123 +
 .../SampleProfile/profile-sample-accurate.ll  |   157 +
 .../Transforms/SampleProfile/propagate.ll     |     6 +-
 llvm/test/Transforms/SampleProfile/remap.ll   |     6 +-
 llvm/test/Transforms/SampleProfile/remarks.ll |    12 +-
 .../uncompressed-profile-symbol-list.ll       |     4 +
 .../X86/expand-masked-compressstore.ll        |    72 +-
 .../X86/expand-masked-expandload.ll           |    90 +-
 .../X86/expand-masked-gather.ll               |    11 +-
 .../X86/expand-masked-load.ll                 |    81 +-
 .../X86/expand-masked-scatter.ll              |    64 +
 .../X86/expand-masked-store.ll                |    23 +-
 llvm/test/Transforms/Scalarizer/dbginfo.ll    |     2 +-
 llvm/test/Transforms/Scalarizer/intrinsics.ll |    12 +
 .../Scalarizer/phi-unreachable-pred.ll        |    98 +
 .../SimpleLoopUnswitch/basictest-profmd.ll    |     2 +-
 .../SimpleLoopUnswitch/basictest.ll           |     2 +-
 .../SimpleLoopUnswitch/delete-dead-blocks.ll  |     1 +
 .../exponential-nontrivial-unswitch-nested.ll |    20 +
 ...exponential-nontrivial-unswitch-nested2.ll |    22 +
 .../exponential-nontrivial-unswitch.ll        |    20 +
 .../exponential-nontrivial-unswitch2.ll       |    20 +
 .../exponential-switch-unswitch.ll            |    20 +
 .../Transforms/SimpleLoopUnswitch/guards.ll   |     2 +-
 .../Transforms/SimpleLoopUnswitch/msan.ll     |     2 +-
 .../nontrivial-unswitch-cost.ll               |     1 +
 .../nontrivial-unswitch-redundant-switch.ll   |     1 +
 .../SimpleLoopUnswitch/nontrivial-unswitch.ll |     1 +
 .../preserve-scev-exiting-multiple-loops.ll   |    63 +
 .../trivial-unswitch-iteration.ll             |     2 +-
 .../trivial-unswitch-profmd.ll                |     2 +-
 .../SimpleLoopUnswitch/trivial-unswitch.ll    |     2 +-
 .../SimpleLoopUnswitch/update-scev.ll         |     2 +-
 .../Hexagon/switch-to-lookup-table.ll         |     2 +-
 llvm/test/Transforms/SimplifyCFG/HoistCode.ll |    39 +-
 .../Transforms/SimplifyCFG/PhiEliminate3.ll   |    59 +-
 .../Transforms/SimplifyCFG/SpeculativeExec.ll |   135 +-
 .../SimplifyCFG/X86/merge-cond-stores-cost.ll |    43 +
 .../SimplifyCFG/X86/speculate-cttz-ctlz.ll    |   304 +-
 .../SimplifyCFG/X86/switch_to_lookup_table.ll |     6 +-
 .../dce-cond-after-folding-terminator.ll      |     5 +-
 .../SimplifyCFG/hoist-dbgvalue-inlined.ll     |     1 -
 .../Transforms/SimplifyCFG/invalidate-dom.ll  |    90 +
 .../SimplifyCFG/merge-cleanuppads.ll          |     4 +-
 .../SimplifyCFG/merge-cond-stores-2.ll        |    92 +-
 .../SimplifyCFG/merge-cond-stores.ll          |    28 +-
 llvm/test/Transforms/SimplifyCFG/safe-abs.ll  |    30 +
 .../SimplifyCFG/safe-low-bit-extract.ll       |    31 +
 .../signbit-like-value-extension.ll           |    35 +
 .../SimplifyCFG/sink-common-code.ll           |    27 +
 .../Transforms/SimplifyCFG/speculate-math.ll  |   112 +-
 .../SimplifyCFG/switch-range-to-icmp.ll       |    38 +
 .../SimplifyCFG/switch_create-custom-dl.ll    |    16 +-
 .../Transforms/SimplifyCFG/switch_msan.ll     |   102 +
 .../unsigned-multiplication-will-overflow.ll  |    36 +
 .../SimplifyCFG/volatile-phioper.ll           |     4 +-
 .../Transforms/SimplifyCFG/wc-widen-block.ll  |   451 +
 .../AMDGPU/pr23975.ll                         |     2 +-
 .../ThinLTOBitcodeWriter/associated.ll        |    14 +
 .../cfi-functions-canonical-jump-tables.ll    |    23 +
 .../Transforms/TypePromotion/ARM/calls.ll     |   342 +
 .../Transforms/TypePromotion/ARM/casts.ll     |  1072 ++
 .../TypePromotion/ARM/clear-structures.ll     |   124 +
 .../Transforms/TypePromotion/ARM/icmps.ll     |   349 +
 .../Transforms/TypePromotion/ARM/large-int.ll |    66 +
 .../Transforms/TypePromotion/ARM/phis-ret.ll  |   344 +
 .../Transforms/TypePromotion/ARM/pointers.ll  |   240 +
 .../TypePromotion/ARM/signed-icmps.ll         |   103 +
 .../Transforms/TypePromotion/ARM/signed.ll    |   123 +
 .../Transforms/TypePromotion/ARM/switch.ll    |   291 +
 .../Transforms/TypePromotion/ARM/wrapping.ll  |   356 +
 .../Transforms/Util/PredicateInfo/condprop.ll |     6 +-
 .../Util/PredicateInfo/testandor.ll           |    26 +-
 llvm/test/Transforms/Util/add-TLI-mappings.ll |    61 +
 llvm/test/Transforms/Util/dbg-call-bitcast.ll |    48 +
 llvm/test/Transforms/Util/dbg-user-of-aext.ll |     2 +-
 llvm/test/Transforms/Util/flattencfg.ll       |    60 +
 .../Util/libcalls-fast-math-inf-loop.ll       |     4 +-
 .../Util/simplify-dbg-declare-load.ll         |     1 +
 .../WholeProgramDevirt/branch-funnel.ll       |     2 +-
 .../WholeProgramDevirt/export-single-impl.ll  |     6 +-
 .../virtual-const-prop-begin.ll               |    14 +-
 .../virtual-const-prop-end.ll                 |    16 +-
 llvm/test/Verifier/blockbyref.ll              |     4 +-
 llvm/test/Verifier/callbr.ll                  |    50 +
 llvm/test/Verifier/comdat2.ll.x               |     4 +-
 .../Verifier/dereferenceable-md-inttoptr.ll   |     6 +
 llvm/test/Verifier/dereferenceable-md.ll.x    |    48 +-
 .../diexpression-dwarf-entry-value.ll         |     8 +
 .../diexpression-entry-value-llvm-ir.ll       |    31 +
 .../Verifier/diexpression-entry-value.ll.x    |    11 +-
 .../diexpression-valid-entry-value.ll         |     2 +-
 llvm/test/Verifier/fp-intrinsics.ll           |    28 +-
 llvm/test/Verifier/intrinsic-bad-arg-type.ll  |    10 +
 llvm/test/Verifier/intrinsic-immarg.ll.x      |    22 +-
 .../invalid-frame-pointer-attr-empty.ll       |     9 +
 .../invalid-frame-pointer-attr-no-value.ll    |     9 +
 .../Verifier/invalid-frame-pointer-attr.ll    |     9 +
 .../invalid-patchable-function-entry.ll       |    21 +
 .../update_llc_test_checks/Inputs/basic.ll    |    32 +
 .../Inputs/argument_name_reuse.ll             |     7 +
 .../update_test_checks/Inputs/basic.ll        |    49 +
 .../update_test_checks/Inputs/scrub_attrs.ll  |     8 +
 .../Inputs/sometimes_deleted_function.ll      |    12 +
 llvm/test/tools/dsymutil/Inputs/frame-dw2.ll  |     4 +-
 llvm/test/tools/dsymutil/Inputs/frame-dw4.ll  |     4 +-
 .../gold/X86/linkonce_odr_unnamed_addr.ll     |     2 +
 llvm/test/tools/gold/X86/slp-vectorize-pm.ll  |    79 +
 llvm/test/tools/gold/X86/split-dwarf.ll       |     2 +-
 llvm/test/tools/gold/X86/strip_names.ll       |     2 +-
 .../X86/v1.12/thinlto_emit_linked_objects.ll  |     2 +-
 .../test/tools/llvm-dwarfdump/AArch64/arch.ll |     1 -
 .../test/tools/llvm-dwarfdump/X86/locstats.ll |   211 +
 .../tools/llvm-dwarfdump/X86/statistics.ll    |     2 +-
 .../X86/stats-dbg-callsite-info.ll            |    76 +
 .../X86/valid-call-site-GNU-extensions.ll     |   103 +
 llvm/test/tools/llvm-lib/Inputs/arm64.ll      |     2 +-
 llvm/test/tools/llvm-lib/Inputs/i386.ll       |     2 +-
 llvm/test/tools/llvm-lib/Inputs/x86_64.ll     |     2 +-
 llvm/test/tools/llvm-locstats/locstats.ll     |   175 +
 .../tools/llvm-locstats/no_scope_bytes.ll     |    39 +
 llvm/test/tools/llvm-lto2/X86/pipeline.ll     |     2 +-
 .../tools/llvm-lto2/X86/slp-vectorize-pm.ll   |    51 +
 .../tools/llvm-lto2/X86/stats-file-option.ll  |     2 +-
 .../tools/llvm-objdump/AMDGPU/source-lines.ll |     2 +-
 .../Hexagon/source-interleave-hexagon.ll      |     2 +-
 ...rce-interleave-same-line-different-file.ll |     2 +-
 .../X86/Inputs/source-interleave.ll           |     2 +-
 .../X86/disassemble-archive-with-source.ll    |     2 +-
 .../llvm-objdump/X86/macho-data-in-code.ll    |    45 +
 .../tools/llvm-readobj/ELF/Inputs/trivial.ll  |    20 +
 5149 files changed, 561647 insertions(+), 147823 deletions(-)
 create mode 100644 llvm/test/Analysis/BasicAA/assume-index-positive.ll
 create mode 100644 llvm/test/Analysis/BasicAA/dereferenceable.ll
 create mode 100644 llvm/test/Analysis/BasicAA/ptrmask.ll
 create mode 100644 llvm/test/Analysis/BranchProbabilityInfo/fcmp.ll
 create mode 100644 llvm/test/Analysis/ConstantFolding/binop-identity-undef.ll
 create mode 100644 llvm/test/Analysis/ConstantFolding/copysign.ll
 create mode 100644 llvm/test/Analysis/ConstantFolding/gep-alias.ll
 create mode 100644 llvm/test/Analysis/ConstantFolding/insertelement.ll
 create mode 100644 llvm/test/Analysis/ConstantFolding/math-1.ll
 create mode 100644 llvm/test/Analysis/ConstantFolding/math-2.ll
 create mode 100644 llvm/test/Analysis/ConstantFolding/rint.ll
 create mode 100644 llvm/test/Analysis/ConstantFolding/round.ll
 create mode 100644 llvm/test/Analysis/ConstantFolding/shufflevector.ll
 create mode 100644 llvm/test/Analysis/ConstantFolding/trunc.ll
 create mode 100644 llvm/test/Analysis/CostModel/AArch64/aggregates.ll
 create mode 100644 llvm/test/Analysis/CostModel/AMDGPU/fma.ll
 create mode 100644 llvm/test/Analysis/CostModel/ARM/arith.ll
 create mode 100644 llvm/test/Analysis/CostModel/ARM/fparith.ll
 create mode 100644 llvm/test/Analysis/CostModel/ARM/freeshift.ll
 create mode 100644 llvm/test/Analysis/CostModel/ARM/load_store.ll
 create mode 100644 llvm/test/Analysis/CostModel/PowerPC/future-cost-model.ll
 create mode 100644 llvm/test/Analysis/CostModel/X86/aggregates.ll
 create mode 100644 llvm/test/Analysis/DDG/basic-a.ll
 create mode 100644 llvm/test/Analysis/DDG/basic-b.ll
 create mode 100644 llvm/test/Analysis/DDG/basic-loopnest.ll
 create mode 100644 llvm/test/Analysis/DDG/root-node.ll
 create mode 100644 llvm/test/Analysis/DependenceAnalysis/Dump.ll
 create mode 100644 llvm/test/Analysis/DivergenceAnalysis/AMDGPU/b42473-r1-crash.ll
 create mode 100644 llvm/test/Analysis/GlobalsModRef/intrinsic_addressnottaken1.ll
 create mode 100644 llvm/test/Analysis/GlobalsModRef/intrinsic_addressnottaken2.ll
 create mode 100644 llvm/test/Analysis/GlobalsModRef/intrinsic_addresstaken.ll
 create mode 100644 llvm/test/Analysis/LoopCacheAnalysis/PowerPC/compute-cost.ll
 create mode 100644 llvm/test/Analysis/LoopCacheAnalysis/PowerPC/loads-store.ll
 create mode 100644 llvm/test/Analysis/LoopCacheAnalysis/PowerPC/matmul.ll
 create mode 100644 llvm/test/Analysis/LoopCacheAnalysis/PowerPC/matvecmul.ll
 create mode 100644 llvm/test/Analysis/LoopCacheAnalysis/PowerPC/single-store.ll
 create mode 100644 llvm/test/Analysis/LoopCacheAnalysis/PowerPC/stencil.ll
 create mode 100644 llvm/test/Analysis/MemorySSA/debugvalue.ll
 create mode 100644 llvm/test/Analysis/MemorySSA/debugvalue2.ll
 create mode 100644 llvm/test/Analysis/MemorySSA/loop-rotate-disablebasicaa.ll
 create mode 100644 llvm/test/Analysis/MemorySSA/loop-rotate-simplified-clone.ll
 create mode 100644 llvm/test/Analysis/MemorySSA/loop-unswitch.ll
 create mode 100644 llvm/test/Analysis/MemorySSA/pr42940.ll
 create mode 100644 llvm/test/Analysis/MemorySSA/pr43044.ll
 create mode 100644 llvm/test/Analysis/MemorySSA/pr43317.ll
 create mode 100644 llvm/test/Analysis/MemorySSA/pr43320.ll
 create mode 100644 llvm/test/Analysis/MemorySSA/pr43426.ll
 create mode 100644 llvm/test/Analysis/MemorySSA/pr43427.ll
 create mode 100644 llvm/test/Analysis/MemorySSA/pr43438.ll
 create mode 100644 llvm/test/Analysis/MemorySSA/pr43493.ll
 create mode 100644 llvm/test/Analysis/MemorySSA/pr43540.ll
 create mode 100644 llvm/test/Analysis/MemorySSA/pr43541.ll
 create mode 100644 llvm/test/Analysis/MemorySSA/pr43569.ll
 create mode 100644 llvm/test/Analysis/MemorySSA/pr43641.ll
 create mode 100644 llvm/test/Analysis/MemorySSA/pr44027.ll
 create mode 100644 llvm/test/Analysis/MemorySSA/pr44029.ll
 create mode 100644 llvm/test/Analysis/MemorySSA/renamephis.ll
 create mode 100644 llvm/test/Analysis/MemorySSA/unreachable.ll
 create mode 100644 llvm/test/Analysis/MustExecute/must_be_executed_context.ll
 create mode 100644 llvm/test/Analysis/ScalarEvolution/multiple-max-iterations.ll
 create mode 100644 llvm/test/Analysis/ScalarEvolution/range_nw_flag.ll
 create mode 100644 llvm/test/Analysis/ScalarEvolution/trip-count-andor.ll
 create mode 100644 llvm/test/Analysis/ScalarEvolution/trip-count15.ll
 create mode 100644 llvm/test/Analysis/ScalarEvolution/umin-umax-folds.ll
 create mode 100644 llvm/test/Analysis/ScalarEvolution/widenable-condition.ll
 create mode 100644 llvm/test/Assembler/2003-11-11-ImplicitRename.ll
 create mode 100644 llvm/test/Assembler/datalayout-invalid-function-ptr-alignment.ll
 create mode 100644 llvm/test/Assembler/datalayout-invalid-stack-natural-alignment.ll
 create mode 100644 llvm/test/Assembler/export-symbol-anonymous-class.ll
 create mode 100644 llvm/test/Assembler/invalid-arg-num-1.ll
 create mode 100644 llvm/test/Assembler/invalid-arg-num-2.ll
 create mode 100644 llvm/test/Assembler/invalid-arg-num-3.ll
 create mode 100644 llvm/test/Assembler/multi-mod-disassemble.ll
 create mode 100644 llvm/test/Assembler/multi-summary-disassemble.ll
 create mode 100644 llvm/test/Bindings/llvm-c/freeze.ll
 create mode 100644 llvm/test/Bitcode/aarch64-addp-upgrade.ll
 create mode 100644 llvm/test/Bitcode/invalid-functionptr-align.ll
 create mode 100644 llvm/test/Bitcode/invalid-type-for-null-constant.ll
 create mode 100644 llvm/test/Bitcode/thinlto-alias3.ll
 create mode 100644 llvm/test/Bitcode/upgrade-arc-runtime-calls-bitcast.ll
 create mode 100644 llvm/test/Bitcode/upgrade-arc-runtime-calls.ll
 create mode 100644 llvm/test/Bitcode/upgrade-datalayout.ll
 create mode 100644 llvm/test/Bitcode/upgrade-datalayout2.ll
 create mode 100644 llvm/test/Bitcode/upgrade-datalayout3.ll
 create mode 100644 llvm/test/Bitcode/upgrade-frame-pointer.ll
 create mode 100644 llvm/test/BugPoint/attr-crash.ll
 create mode 100644 llvm/test/BugPoint/retain-crashing-metadata.ll
 create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator-gep.ll
 create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/call-lowering-i128-on-stack.ll
 create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/call-translator-musttail.ll
 create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/call-translator-tail-call-weak.ll
 create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/call-translator-tail-call.ll
 create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/call-translator-variadic-musttail.ll
 create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/combiner-load-store-indexing.ll
 create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/const-0.ll
 create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/integration-shuffle-vector.ll
 create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-extends.ll
 create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-split-vector-arg.ll
 create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-tbaa.ll
 create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/legalize-sext-128.ll
 create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/memcpy_chk_no_tail.ll
 create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/no-neon-no-fp.ll
 create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/swiftself.ll
 create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/tail-call-no-save-fp-lr.ll
 create mode 100644 llvm/test/CodeGen/AArch64/aarch64-sve-asm-negative.ll
 create mode 100644 llvm/test/CodeGen/AArch64/aarch64-sve-asm.ll
 create mode 100644 llvm/test/CodeGen/AArch64/align-down.ll
 create mode 100644 llvm/test/CodeGen/AArch64/arm64-code-model-large-darwin.ll
 create mode 100644 llvm/test/CodeGen/AArch64/arm64-memset-to-bzero-pgso.ll
 create mode 100644 llvm/test/CodeGen/AArch64/arm64-preserve-most.ll
 create mode 100644 llvm/test/CodeGen/AArch64/arm64-tls-initial-exec.ll
 create mode 100644 llvm/test/CodeGen/AArch64/arm64-tls-local-exec.ll
 create mode 100644 llvm/test/CodeGen/AArch64/arm64_32-addrs.ll
 create mode 100644 llvm/test/CodeGen/AArch64/arm64_32-atomics.ll
 create mode 100644 llvm/test/CodeGen/AArch64/arm64_32-fastisel.ll
 create mode 100644 llvm/test/CodeGen/AArch64/arm64_32-frame-pointers.ll
 create mode 100644 llvm/test/CodeGen/AArch64/arm64_32-gep-sink.ll
 create mode 100644 llvm/test/CodeGen/AArch64/arm64_32-memcpy.ll
 create mode 100644 llvm/test/CodeGen/AArch64/arm64_32-neon.ll
 create mode 100644 llvm/test/CodeGen/AArch64/arm64_32-null.ll
 create mode 100644 llvm/test/CodeGen/AArch64/arm64_32-pointer-extend.ll
 create mode 100644 llvm/test/CodeGen/AArch64/arm64_32-stack-pointers.ll
 create mode 100644 llvm/test/CodeGen/AArch64/arm64_32-tls.ll
 create mode 100644 llvm/test/CodeGen/AArch64/arm64_32-va.ll
 create mode 100644 llvm/test/CodeGen/AArch64/arm64_32.ll
 create mode 100644 llvm/test/CodeGen/AArch64/bti-branch-relaxation.ll
 create mode 100644 llvm/test/CodeGen/AArch64/cfguard-checks.ll
 create mode 100644 llvm/test/CodeGen/AArch64/cfguard-module-flag.ll
 create mode 100644 llvm/test/CodeGen/AArch64/cls.ll
 create mode 100644 llvm/test/CodeGen/AArch64/csr-split.ll
 create mode 100644 llvm/test/CodeGen/AArch64/dbg-value-tag-offset.ll
 create mode 100644 llvm/test/CodeGen/AArch64/div-rem-pair-recomposition-signed.ll
 create mode 100644 llvm/test/CodeGen/AArch64/div-rem-pair-recomposition-unsigned.ll
 create mode 100644 llvm/test/CodeGen/AArch64/fast-isel-branch-uncond-debug.ll
 create mode 100644 llvm/test/CodeGen/AArch64/fp-intrinsics.ll
 create mode 100644 llvm/test/CodeGen/AArch64/fp16-fmla.ll
 create mode 100644 llvm/test/CodeGen/AArch64/fpconv-vector-op-scalarize-strict.ll
 create mode 100644 llvm/test/CodeGen/AArch64/global-merge-hidden-minsize.ll
 create mode 100644 llvm/test/CodeGen/AArch64/i128_volatile_load_store.ll
 create mode 100644 llvm/test/CodeGen/AArch64/jump-table-32.ll
 create mode 100644 llvm/test/CodeGen/AArch64/large-stack.ll
 create mode 100644 llvm/test/CodeGen/AArch64/lower-ptrmask.ll
 create mode 100644 llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-cfi.ll
 create mode 100644 llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-diff-scope-same-key.ll
 create mode 100644 llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-non-leaf.ll
 create mode 100644 llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-same-scope-diff-key.ll
 create mode 100644 llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-same-scope-same-key-a.ll
 create mode 100644 llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-same-scope-same-key-b.ll
 create mode 100644 llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-sp-mod.ll
 create mode 100644 llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-subtarget.ll
 create mode 100644 llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-thunk.ll
 create mode 100644 llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-v8-3.ll
 create mode 100644 llvm/test/CodeGen/AArch64/macro-fusion.ll
 create mode 100644 llvm/test/CodeGen/AArch64/neon-vcadd.ll
 create mode 100644 llvm/test/CodeGen/AArch64/no_cfi.ll
 create mode 100644 llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-0.ll
 create mode 100644 llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-1.ll
 create mode 100644 llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-2.ll
 create mode 100644 llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-3.ll
 create mode 100644 llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-4.ll
 create mode 100644 llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-5.ll
 create mode 100644 llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-6.ll
 create mode 100644 llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-7.ll
 create mode 100644 llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-8.ll
 create mode 100644 llvm/test/CodeGen/AArch64/overeager_mla_fusing.ll
 create mode 100644 llvm/test/CodeGen/AArch64/patchable-function-entry-bti.ll
 create mode 100644 llvm/test/CodeGen/AArch64/patchable-function-entry.ll
 create mode 100644 llvm/test/CodeGen/AArch64/powi-windows.ll
 create mode 100644 llvm/test/CodeGen/AArch64/ragreedy-local-interval-cost.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sadd_sat_plus.ll
 create mode 100644 llvm/test/CodeGen/AArch64/select_const.ll
 create mode 100644 llvm/test/CodeGen/AArch64/shift-by-signext.ll
 create mode 100644 llvm/test/CodeGen/AArch64/shift-logic.ll
 create mode 100644 llvm/test/CodeGen/AArch64/space.ll
 create mode 100644 llvm/test/CodeGen/AArch64/srem-lkk.ll
 create mode 100644 llvm/test/CodeGen/AArch64/srem-seteq-optsize.ll
 create mode 100644 llvm/test/CodeGen/AArch64/srem-seteq-vec-nonsplat.ll
 create mode 100644 llvm/test/CodeGen/AArch64/srem-seteq-vec-splat.ll
 create mode 100644 llvm/test/CodeGen/AArch64/srem-seteq.ll
 create mode 100644 llvm/test/CodeGen/AArch64/srem-vector-lkk.ll
 create mode 100644 llvm/test/CodeGen/AArch64/ssub_sat_plus.ll
 create mode 100644 llvm/test/CodeGen/AArch64/stack-tagging-ex-1.ll
 create mode 100644 llvm/test/CodeGen/AArch64/stack-tagging-ex-2.ll
 create mode 100644 llvm/test/CodeGen/AArch64/stack-tagging-initializer-merge.ll
 create mode 100644 llvm/test/CodeGen/AArch64/stack-tagging-unchecked-ld-st.ll
 create mode 100644 llvm/test/CodeGen/AArch64/stack-tagging-untag-placement.ll
 create mode 100644 llvm/test/CodeGen/AArch64/stackmap.ll
 create mode 100644 llvm/test/CodeGen/AArch64/strict-fp-int-promote.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sub-of-bias.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve-alloca-stackid.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve-calling-convention.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve-fp.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve-gather-scatter-dag-combine.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve-int-arith-imm.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve-int-arith-pred.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve-int-arith.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve-int-div-pred.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve-int-imm.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve-int-log-imm.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve-int-log-pred.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve-int-log.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve-int-mad-pred.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve-int-mul-pred.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve-int-reduce-pred.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve-intrinsics-conversion.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve-intrinsics-counting-bits.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve-intrinsics-counting-elems.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve-intrinsics-fp-arith.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve-intrinsics-fp-compares.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve-intrinsics-fp-converts.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve-intrinsics-fp-reduce.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve-intrinsics-gather-loads-32bit-scaled-offsets.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve-intrinsics-gather-loads-32bit-unscaled-offsets.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve-intrinsics-gather-loads-64bit-scaled-offset.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve-intrinsics-gather-loads-64bit-unscaled-offset.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve-intrinsics-gather-loads-vector-base.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve-intrinsics-int-arith.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve-intrinsics-int-compares-with-imm.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve-intrinsics-int-compares.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve-intrinsics-loads.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve-intrinsics-logical.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve-intrinsics-perm-select.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve-intrinsics-pred-creation.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve-intrinsics-pred-operations.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve-intrinsics-pred-testing.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve-intrinsics-reversal.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve-intrinsics-scatter-stores-32bit-scaled-offsets.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve-intrinsics-scatter-stores-32bit-unscaled-offsets.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve-intrinsics-scatter-stores-64bit-scaled-offset.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve-intrinsics-scatter-stores-64bit-unscaled-offset.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve-intrinsics-scatter-stores-vector-base.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve-intrinsics-shifts.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve-intrinsics-sqdec.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve-intrinsics-sqinc.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve-intrinsics-stores.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve-intrinsics-uqdec.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve-intrinsics-uqinc.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve-intrinsics-while.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve-masked-ldst-nonext.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve-masked-ldst-sext.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve-masked-ldst-trunc.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve-masked-ldst-zext.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve-neg-int-arith-imm-2.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve-neg-int-arith-imm.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve-pred-log.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve-select.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve-setcc.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve-vector-splat.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve2-intrinsics-binary-narrowing-add-sub.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve2-intrinsics-binary-narrowing-shr.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve2-intrinsics-fp-converts.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve2-intrinsics-fp-int-binary-logarithm.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve2-intrinsics-fp-widening-mul-acc.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve2-intrinsics-non-widening-pairwise-arith.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve2-intrinsics-unary-narrowing.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve2-intrinsics-while.ll
 create mode 100644 llvm/test/CodeGen/AArch64/tagged-globals.ll
 create mode 100644 llvm/test/CodeGen/AArch64/tailcall-bitcast-memcpy.ll
 create mode 100644 llvm/test/CodeGen/AArch64/tme.ll
 create mode 100644 llvm/test/CodeGen/AArch64/uadd_sat_plus.ll
 create mode 100644 llvm/test/CodeGen/AArch64/urem-lkk.ll
 create mode 100644 llvm/test/CodeGen/AArch64/urem-seteq-nonzero.ll
 create mode 100644 llvm/test/CodeGen/AArch64/urem-seteq-vec-nonzero.ll
 create mode 100644 llvm/test/CodeGen/AArch64/urem-seteq-vec-tautological.ll
 create mode 100644 llvm/test/CodeGen/AArch64/urem-vector-lkk.ll
 create mode 100644 llvm/test/CodeGen/AArch64/use-cr-result-of-dom-icmp-st.ll
 create mode 100644 llvm/test/CodeGen/AArch64/usub_sat_plus.ll
 create mode 100644 llvm/test/CodeGen/AArch64/vecreduce-fadd-legalization-strict.ll
 create mode 100644 llvm/test/CodeGen/AArch64/vecreduce-fmax-legalization-nan.ll
 create mode 100644 llvm/test/CodeGen/AArch64/vecreduce-fmul-legalization-strict.ll
 create mode 100644 llvm/test/CodeGen/AArch64/vselect-constants.ll
 create mode 100644 llvm/test/CodeGen/AArch64/win64-no-uwtable.ll
 create mode 100644 llvm/test/CodeGen/AArch64/windows-extern-weak.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/GlobalISel/bool-legalization.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/GlobalISel/divergent-control-flow.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/GlobalISel/function-returns.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/GlobalISel/global-value.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgcn-sendmsg.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-atomicrmw.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-function-args.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/GlobalISel/lds-global-non-entry-func.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/GlobalISel/lds-global-value.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/GlobalISel/lds-size.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/GlobalISel/lds-zero-initializer.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ds.ordered.add.gfx10.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ds.ordered.add.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ds.ordered.swap.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.init.exec.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.init.exec.wave32.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.is.private.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.is.shared.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.format.f16.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.format.f32.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.s.sleep.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.wqm.vote.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/GlobalISel/read_register.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/GlobalISel/readcyclecounter.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.image.load.1d.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.image.sample.1d.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.raw.buffer.load.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.struct.buffer.load.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.struct.buffer.store.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/GlobalISel/ret.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/amdgpu-mul24-knownbits.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/amdgpu-unroll-threshold.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/buffer-intrinsics-mmo-offsets.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/call-constant.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/cc-sgpr-limit.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/cc-sgpr-over-limit.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/computeNumSignBits-mul.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/cse-phi-incoming-val.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/dead-machine-elim-after-dead-lane.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/divergence-at-use.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/dpp_combine.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/extract-subvector.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/fneg-fold-legalize-dag-increase-insts.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/fpow.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/global-atomics-fp.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-absent-v3.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-absent.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-present-v3.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-present.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/kill-infinite-loop.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/lcssa-optnone.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/llvm.amdgcn.init.exec.wave32.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/llvm.amdgcn.is.private.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/llvm.amdgcn.is.shared.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/llvm.amdgcn.softwqm.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/mfma-loop.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/mul24-pass-ordering.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/occupancy-levels.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/offset-split-flat.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/offset-split-global.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/opencl-printf-no-hostcall.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/opencl-printf.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/smrd_vmem_war.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/split-arg-dbg-value.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/stack-pointer-offset-relative-frameindex.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/sub-zext-cc-zext-cc.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/vgpr-descriptor-waterfall-loop-idom-update.ll
 create mode 100644 llvm/test/CodeGen/ARM/ParallelDSP/blocks.ll
 create mode 100644 llvm/test/CodeGen/ARM/ParallelDSP/complex_dot_prod.ll
 create mode 100644 llvm/test/CodeGen/ARM/ParallelDSP/exchange.ll
 create mode 100644 llvm/test/CodeGen/ARM/ParallelDSP/overlapping.ll
 create mode 100644 llvm/test/CodeGen/ARM/ParallelDSP/pr42729.ll
 create mode 100644 llvm/test/CodeGen/ARM/ParallelDSP/pr43073.ll
 create mode 100644 llvm/test/CodeGen/ARM/ParallelDSP/sext-acc.ll
 create mode 100644 llvm/test/CodeGen/ARM/cfguard-checks.ll
 create mode 100644 llvm/test/CodeGen/ARM/cfguard-module-flag.ll
 create mode 100644 llvm/test/CodeGen/ARM/cls.ll
 create mode 100644 llvm/test/CodeGen/ARM/cmov_fp16.ll
 create mode 100644 llvm/test/CodeGen/ARM/csr-split.ll
 create mode 100644 llvm/test/CodeGen/ARM/debuginfo-split-carryexpr.ll
 create mode 100644 llvm/test/CodeGen/ARM/dwarf-frame.ll
 create mode 100644 llvm/test/CodeGen/ARM/fp-intrinsics.ll
 create mode 100644 llvm/test/CodeGen/ARM/fp16-fusedMAC.ll
 create mode 100644 llvm/test/CodeGen/ARM/fragmented-args-multiple-regs.ll
 create mode 100644 llvm/test/CodeGen/ARM/gnu_mcount_nc.ll
 create mode 100644 llvm/test/CodeGen/ARM/intrinsics-cmse.ll
 create mode 100644 llvm/test/CodeGen/ARM/ipra-exact-definition.ll
 create mode 100644 llvm/test/CodeGen/ARM/ipra-no-csr.ll
 create mode 100644 llvm/test/CodeGen/ARM/ipra-r0-returned.ll
 create mode 100644 llvm/test/CodeGen/ARM/ipra.ll
 create mode 100644 llvm/test/CodeGen/ARM/legalize-bitcast.ll
 create mode 100644 llvm/test/CodeGen/ARM/neon-vcadd.ll
 create mode 100644 llvm/test/CodeGen/ARM/neon-vqaddsub-upgrade.ll
 create mode 100644 llvm/test/CodeGen/ARM/postrasched.ll
 create mode 100644 llvm/test/CodeGen/ARM/qdadd.ll
 create mode 100644 llvm/test/CodeGen/ARM/sadd_sat.ll
 create mode 100644 llvm/test/CodeGen/ARM/sadd_sat_plus.ll
 create mode 100644 llvm/test/CodeGen/ARM/signext-inreg.ll
 create mode 100644 llvm/test/CodeGen/ARM/softfp-constant-comparison.ll
 create mode 100644 llvm/test/CodeGen/ARM/ssub_sat.ll
 create mode 100644 llvm/test/CodeGen/ARM/ssub_sat_plus.ll
 create mode 100644 llvm/test/CodeGen/ARM/uadd_sat.ll
 create mode 100644 llvm/test/CodeGen/ARM/uadd_sat_plus.ll
 create mode 100644 llvm/test/CodeGen/ARM/usub_sat.ll
 create mode 100644 llvm/test/CodeGen/ARM/usub_sat_plus.ll
 create mode 100644 llvm/test/CodeGen/ARM/vecreduce-fadd-legalization-soft-float.ll
 create mode 100644 llvm/test/CodeGen/ARM/vecreduce-fadd-legalization-strict.ll
 create mode 100644 llvm/test/CodeGen/ARM/vecreduce-fmul-legalization-strict.ll
 create mode 100644 llvm/test/CodeGen/BPF/32-bit-subreg-peephole-phi-1.ll
 create mode 100644 llvm/test/CodeGen/BPF/32-bit-subreg-peephole-phi-2.ll
 create mode 100644 llvm/test/CodeGen/BPF/32-bit-subreg-peephole-phi-3.ll
 create mode 100644 llvm/test/CodeGen/BPF/BTF/extern-builtin.ll
 create mode 100644 llvm/test/CodeGen/BPF/BTF/extern-func-arg.ll
 create mode 100644 llvm/test/CodeGen/BPF/BTF/extern-var-func-weak-section.ll
 create mode 100644 llvm/test/CodeGen/BPF/BTF/extern-var-func-weak.ll
 create mode 100644 llvm/test/CodeGen/BPF/BTF/extern-var-func.ll
 create mode 100644 llvm/test/CodeGen/BPF/BTF/extern-var-section.ll
 create mode 100644 llvm/test/CodeGen/BPF/BTF/extern-var-struct-weak.ll
 create mode 100644 llvm/test/CodeGen/BPF/BTF/extern-var-struct.ll
 create mode 100644 llvm/test/CodeGen/BPF/BTF/extern-var-weak-section.ll
 create mode 100644 llvm/test/CodeGen/BPF/BTF/static-func.ll
 create mode 100644 llvm/test/CodeGen/BPF/BTF/weak-global-2.ll
 create mode 100644 llvm/test/CodeGen/BPF/BTF/weak-global.ll
 create mode 100644 llvm/test/CodeGen/BPF/CORE/field-reloc-alu32.ll
 create mode 100644 llvm/test/CodeGen/BPF/CORE/field-reloc-bitfield-1.ll
 create mode 100644 llvm/test/CodeGen/BPF/CORE/field-reloc-bitfield-2.ll
 create mode 100644 llvm/test/CodeGen/BPF/CORE/intrinsic-fieldinfo-byte-size-1.ll
 create mode 100644 llvm/test/CodeGen/BPF/CORE/intrinsic-fieldinfo-byte-size-2.ll
 create mode 100644 llvm/test/CodeGen/BPF/CORE/intrinsic-fieldinfo-byte-size-3.ll
 create mode 100644 llvm/test/CodeGen/BPF/CORE/intrinsic-fieldinfo-byte-size-4.ll
 create mode 100644 llvm/test/CodeGen/BPF/CORE/intrinsic-fieldinfo-existence-1.ll
 create mode 100644 llvm/test/CodeGen/BPF/CORE/intrinsic-fieldinfo-existence-2.ll
 create mode 100644 llvm/test/CodeGen/BPF/CORE/intrinsic-fieldinfo-existence-3.ll
 create mode 100644 llvm/test/CodeGen/BPF/CORE/intrinsic-fieldinfo-lshift-1.ll
 create mode 100644 llvm/test/CodeGen/BPF/CORE/intrinsic-fieldinfo-lshift-2.ll
 create mode 100644 llvm/test/CodeGen/BPF/CORE/intrinsic-fieldinfo-rshift-1.ll
 create mode 100644 llvm/test/CodeGen/BPF/CORE/intrinsic-fieldinfo-rshift-2.ll
 create mode 100644 llvm/test/CodeGen/BPF/CORE/intrinsic-fieldinfo-rshift-3.ll
 create mode 100644 llvm/test/CodeGen/BPF/CORE/intrinsic-fieldinfo-signedness-1.ll
 create mode 100644 llvm/test/CodeGen/BPF/CORE/intrinsic-fieldinfo-signedness-2.ll
 create mode 100644 llvm/test/CodeGen/BPF/CORE/intrinsic-fieldinfo-signedness-3.ll
 create mode 100644 llvm/test/CodeGen/BPF/CORE/intrinsic-transforms.ll
 create mode 100644 llvm/test/CodeGen/BPF/CORE/no-elf-ama-symbol.ll
 create mode 100644 llvm/test/CodeGen/BPF/CORE/no-narrow-load.ll
 create mode 100644 llvm/test/CodeGen/BPF/CORE/offset-reloc-cast-array-1.ll
 create mode 100644 llvm/test/CodeGen/BPF/CORE/offset-reloc-cast-array-2.ll
 create mode 100644 llvm/test/CodeGen/BPF/CORE/offset-reloc-cast-struct-1.ll
 create mode 100644 llvm/test/CodeGen/BPF/CORE/offset-reloc-cast-struct-2.ll
 create mode 100644 llvm/test/CodeGen/BPF/CORE/offset-reloc-cast-struct-3.ll
 create mode 100644 llvm/test/CodeGen/BPF/CORE/offset-reloc-cast-union-1.ll
 create mode 100644 llvm/test/CodeGen/BPF/CORE/offset-reloc-cast-union-2.ll
 create mode 100644 llvm/test/CodeGen/BPF/CORE/offset-reloc-end-load.ll
 create mode 100644 llvm/test/CodeGen/BPF/CORE/offset-reloc-end-ret.ll
 create mode 100644 llvm/test/CodeGen/BPF/CORE/offset-reloc-fieldinfo-1.ll
 create mode 100644 llvm/test/CodeGen/BPF/CORE/offset-reloc-fieldinfo-2.ll
 create mode 100644 llvm/test/CodeGen/BPF/CORE/offset-reloc-global-1.ll
 create mode 100644 llvm/test/CodeGen/BPF/CORE/offset-reloc-global-2.ll
 create mode 100644 llvm/test/CodeGen/BPF/CORE/offset-reloc-global-3.ll
 create mode 100644 llvm/test/CodeGen/BPF/CORE/offset-reloc-ignore.ll
 create mode 100644 llvm/test/CodeGen/BPF/CORE/offset-reloc-middle-chain.ll
 create mode 100644 llvm/test/CodeGen/BPF/CORE/offset-reloc-multi-array-1.ll
 create mode 100644 llvm/test/CodeGen/BPF/CORE/offset-reloc-multi-array-2.ll
 create mode 100644 llvm/test/CodeGen/BPF/CORE/offset-reloc-pointer-1.ll
 create mode 100644 llvm/test/CodeGen/BPF/CORE/offset-reloc-pointer-2.ll
 create mode 100644 llvm/test/CodeGen/BPF/callx.ll
 create mode 100644 llvm/test/CodeGen/BPF/optnone-1.ll
 create mode 100644 llvm/test/CodeGen/BPF/remove_truncate_6.ll
 create mode 100644 llvm/test/CodeGen/Generic/DbgValueAggregate.ll
 create mode 100644 llvm/test/CodeGen/Hexagon/64bit_tstbit.ll
 create mode 100644 llvm/test/CodeGen/Hexagon/autohvx/isel-setcc-v256i1.ll
 create mode 100644 llvm/test/CodeGen/Hexagon/autohvx/minmax-128b.ll
 create mode 100644 llvm/test/CodeGen/Hexagon/autohvx/minmax-64b.ll
 create mode 100644 llvm/test/CodeGen/Hexagon/isel-bitcast-v8i1-i8.ll
 create mode 100644 llvm/test/CodeGen/Hexagon/isel-bitcast-v8i8-v4i16.ll
 create mode 100644 llvm/test/CodeGen/Hexagon/isel-minmax-v64bit.ll
 create mode 100644 llvm/test/CodeGen/Hexagon/isel-vselect-v4i8.ll
 create mode 100644 llvm/test/CodeGen/Hexagon/muxii-bug.ll
 create mode 100644 llvm/test/CodeGen/Hexagon/packetizer-resources.ll
 create mode 100644 llvm/test/CodeGen/Hexagon/reg-by-name.ll
 create mode 100644 llvm/test/CodeGen/Hexagon/signext-inreg.ll
 create mode 100644 llvm/test/CodeGen/Hexagon/swp-crash-iter.ll
 create mode 100644 llvm/test/CodeGen/Hexagon/swp-epilog-phi12.ll
 create mode 100644 llvm/test/CodeGen/MIR/Mips/setRegClassOrRegBank.ll
 create mode 100644 llvm/test/CodeGen/MSP430/selectcc.ll
 create mode 100644 llvm/test/CodeGen/MSP430/shift-amount-threshold-b.ll
 create mode 100644 llvm/test/CodeGen/MSP430/shift-amount-threshold.ll
 create mode 100644 llvm/test/CodeGen/Mips/GlobalISel/irtranslator/aggregate_struct_return.ll
 create mode 100644 llvm/test/CodeGen/Mips/GlobalISel/irtranslator/sret_pointer.ll
 create mode 100644 llvm/test/CodeGen/Mips/GlobalISel/irtranslator/var_arg.ll
 create mode 100644 llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/add_vec.ll
 create mode 100644 llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/add_vec_builtin.ll
 create mode 100644 llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/aggregate_struct_return.ll
 create mode 100644 llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/bitreverse.ll
 create mode 100644 llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/brindirect.ll
 create mode 100644 llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/bswap.ll
 create mode 100644 llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/dyn_stackalloc.ll
 create mode 100644 llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/fabs_vec.ll
 create mode 100644 llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/fabs_vec_builtin.ll
 create mode 100644 llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/fence.ll
 create mode 100644 llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/floating_point_vec_arithmetic_operations.ll
 create mode 100644 llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/floating_point_vec_arithmetic_operations_builtin.ll
 create mode 100644 llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/fsqrt_vec.ll
 create mode 100644 llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/fsqrt_vec_builtin.ll
 create mode 100644 llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/implicit_def.ll
 create mode 100644 llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/inttoptr_and_ptrtoint.ll
 create mode 100644 llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/jump_table_and_brjt.ll
 create mode 100644 llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/load_store_fold.ll
 create mode 100644 llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/load_store_vec.ll
 create mode 100644 llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/mul_vec.ll
 create mode 100644 llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/mul_vec_builtin.ll
 create mode 100644 llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/rem_and_div_vec.ll
 create mode 100644 llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/rem_and_div_vec_builtin.ll
 create mode 100644 llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/sret_pointer.ll
 create mode 100644 llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/sub_vec.ll
 create mode 100644 llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/sub_vec_builtin.ll
 create mode 100644 llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/trap.ll
 create mode 100644 llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/trunc.ll
 create mode 100644 llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/var_arg.ll
 create mode 100644 llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/zext_and_sext.ll
 create mode 100644 llvm/test/CodeGen/Mips/atomic-min-max-64.ll
 create mode 100644 llvm/test/CodeGen/Mips/atomic-min-max.ll
 create mode 100644 llvm/test/CodeGen/Mips/constraint-empty.ll
 create mode 100644 llvm/test/CodeGen/Mips/copy-fp64.ll
 create mode 100644 llvm/test/CodeGen/Mips/cpus-no-mips64.ll
 create mode 100644 llvm/test/CodeGen/Mips/cpus.ll
 create mode 100644 llvm/test/CodeGen/Mips/mcount.ll
 create mode 100644 llvm/test/CodeGen/Mips/msa/nori.b.ll
 create mode 100644 llvm/test/CodeGen/Mips/msa/remat-ldi.ll
 create mode 100644 llvm/test/CodeGen/Mips/no-frame-pointer-elim.ll
 create mode 100644 llvm/test/CodeGen/Mips/pr42736.ll
 create mode 100644 llvm/test/CodeGen/NVPTX/bug41651.ll
 create mode 100644 llvm/test/CodeGen/NVPTX/shfl-p.ll
 create mode 100644 llvm/test/CodeGen/NVPTX/shfl-sync-p.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/absol-jump-table-enabled.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/aix-byval-param.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/aix-cc-abi.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/aix-cc-altivec.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/aix-external-sym-sdnode-lowering.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/aix-func-dsc-gen.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/aix-indirect-call.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/aix-lower-block-address.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/aix-lower-constant-pool-index.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/aix-lower-jump-table.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/aix-lr.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/aix-nest-param.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/aix-readonly-with-relocation.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/aix-reference-func-addr-const.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/aix-return55.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/aix-space.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/aix-sret-param.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/aix-stackargs.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/aix-trampoline.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/aix-undef-func-call.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/aix-user-defined-memcpy.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/aix-weak-undef-func-call.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/aix-xcoff-data-only-notoc.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/aix-xcoff-data.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/aix-xcoff-lcomm.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/aix-xcoff-mergeable-const.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/aix-xcoff-mergeable-str.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/aix-xcoff-rodata.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/aix-xcoff-textdisassembly.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/aix-xcoff-toc.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/and-mask.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/build-vector-allones.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/check-cpu.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/csr-split.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/dform-adjust.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/elf-common.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/fdiv.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/float-vector-gather.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/fma-combine.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/fold-rlwinm-1.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/fp-intrinsics-fptosi-legal.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/ifunc.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/inline-asm-vsx-clobbers.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/inlineasm-extendedmne.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/instr-properties.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/load-and-splat.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/load-shuffle-and-shuffle-store.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/loop-instr-form-prepare.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/lower-globaladdr32-aix-asm.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/lower-globaladdr32-aix.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/lower-globaladdr64-aix-asm.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/lower-globaladdr64-aix.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/lower-massv-attr.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/lower-massv.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/lsr-insns-cost.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/mi-peephole-splat.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/no-duplicate.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/popcnt-zext.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/ppcf128-constrained-fp-intrinsics.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/pr25080.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/pr41088.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/pr43527.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/pr44183.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/pr44239.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/reduce_scalarization02.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/remove-redundant-load-imm.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/scalar-min-max.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/scalar-rounding-ops.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/sext-vector-inreg.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/shrink-wrap.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/sms-cpy-1.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/sms-phi-1.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/sms-phi-2.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/sms-phi-3.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/sms-phi-5.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/smulfixsat.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/spill_p9_setb.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/srem-lkk.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/srem-vector-lkk.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/test_func_desc.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/umulfixsat.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/unaligned-floats.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/urem-lkk.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/urem-vector-lkk.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/vavg.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/vec-bswap.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/vector-extend-sign.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/vector-rotates.ll
 create mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/calllowering-ret.ll
 create mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/irtranslator-calllowering.ll
 create mode 100644 llvm/test/CodeGen/RISCV/copysign-casts.ll
 create mode 100644 llvm/test/CodeGen/RISCV/disjoint.ll
 create mode 100644 llvm/test/CodeGen/RISCV/fastcc-float.ll
 create mode 100644 llvm/test/CodeGen/RISCV/fastcc-int.ll
 create mode 100644 llvm/test/CodeGen/RISCV/fp16-promote.ll
 create mode 100644 llvm/test/CodeGen/RISCV/get-register-invalid.ll
 create mode 100644 llvm/test/CodeGen/RISCV/get-register-noreserve.ll
 create mode 100644 llvm/test/CodeGen/RISCV/get-register-reserve.ll
 create mode 100644 llvm/test/CodeGen/RISCV/inline-asm-abi-names.ll
 create mode 100644 llvm/test/CodeGen/RISCV/inline-asm-clobbers.ll
 create mode 100644 llvm/test/CodeGen/RISCV/inline-asm-d-abi-names.ll
 create mode 100644 llvm/test/CodeGen/RISCV/inline-asm-f-abi-names.ll
 create mode 100644 llvm/test/CodeGen/RISCV/interrupt-attr-callee.ll
 create mode 100644 llvm/test/CodeGen/RISCV/intrinsics/trap.ll
 create mode 100644 llvm/test/CodeGen/RISCV/mir-target-flags.ll
 create mode 100644 llvm/test/CodeGen/RISCV/module-target-abi.ll
 create mode 100644 llvm/test/CodeGen/RISCV/module-target-abi2.ll
 create mode 100644 llvm/test/CodeGen/RISCV/reserved-reg-errors.ll
 create mode 100644 llvm/test/CodeGen/RISCV/reserved-regs.ll
 create mode 100644 llvm/test/CodeGen/RISCV/rv64-large-stack.ll
 create mode 100644 llvm/test/CodeGen/RISCV/rv64i-complex-float.ll
 create mode 100644 llvm/test/CodeGen/RISCV/rv64i-single-softfloat.ll
 create mode 100644 llvm/test/CodeGen/RISCV/rv64i-w-insts-legalization.ll
 create mode 100644 llvm/test/CodeGen/RISCV/rv64m-w-insts-legalization.ll
 create mode 100644 llvm/test/CodeGen/RISCV/shrinkwrap.ll
 create mode 100644 llvm/test/CodeGen/RISCV/split-sp-adjust.ll
 create mode 100644 llvm/test/CodeGen/RISCV/srem-lkk.ll
 create mode 100644 llvm/test/CodeGen/RISCV/srem-vector-lkk.ll
 create mode 100644 llvm/test/CodeGen/RISCV/stack-realignment-with-variable-sized-objects.ll
 create mode 100644 llvm/test/CodeGen/RISCV/subtarget-features-std-ext.ll
 create mode 100644 llvm/test/CodeGen/RISCV/urem-lkk.ll
 create mode 100644 llvm/test/CodeGen/RISCV/urem-vector-lkk.ll
 create mode 100644 llvm/test/CodeGen/SPARC/64atomics.ll
 create mode 100644 llvm/test/CodeGen/SystemZ/fentry-insertion.ll
 create mode 100644 llvm/test/CodeGen/SystemZ/fp-mul-13.ll
 create mode 100644 llvm/test/CodeGen/SystemZ/fp-strict-cmp-01.ll
 create mode 100644 llvm/test/CodeGen/SystemZ/fp-strict-cmp-02.ll
 create mode 100644 llvm/test/CodeGen/SystemZ/fp-strict-cmp-03.ll
 create mode 100644 llvm/test/CodeGen/SystemZ/fp-strict-cmp-04.ll
 create mode 100644 llvm/test/CodeGen/SystemZ/fp-strict-cmp-05.ll
 create mode 100644 llvm/test/CodeGen/SystemZ/fp-strict-cmp-06.ll
 create mode 100644 llvm/test/CodeGen/SystemZ/fp-strict-cmps-01.ll
 create mode 100644 llvm/test/CodeGen/SystemZ/fp-strict-cmps-02.ll
 create mode 100644 llvm/test/CodeGen/SystemZ/fp-strict-cmps-03.ll
 create mode 100644 llvm/test/CodeGen/SystemZ/fp-strict-cmps-04.ll
 create mode 100644 llvm/test/CodeGen/SystemZ/fp-strict-cmps-05.ll
 create mode 100644 llvm/test/CodeGen/SystemZ/fp-strict-cmps-06.ll
 create mode 100644 llvm/test/CodeGen/SystemZ/fp-strict-conv-05.ll
 create mode 100644 llvm/test/CodeGen/SystemZ/fp-strict-conv-06.ll
 create mode 100644 llvm/test/CodeGen/SystemZ/fp-strict-conv-07.ll
 create mode 100644 llvm/test/CodeGen/SystemZ/fp-strict-conv-08.ll
 create mode 100644 llvm/test/CodeGen/SystemZ/fp-strict-conv-09.ll
 create mode 100644 llvm/test/CodeGen/SystemZ/fp-strict-conv-10.ll
 create mode 100644 llvm/test/CodeGen/SystemZ/fp-strict-conv-11.ll
 create mode 100644 llvm/test/CodeGen/SystemZ/fp-strict-conv-12.ll
 create mode 100644 llvm/test/CodeGen/SystemZ/fp-strict-conv-13.ll
 create mode 100644 llvm/test/CodeGen/SystemZ/fp-strict-conv-14.ll
 create mode 100644 llvm/test/CodeGen/SystemZ/fp-strict-conv-16.ll
 create mode 100644 llvm/test/CodeGen/SystemZ/fp-strict-mul-12.ll
 create mode 100644 llvm/test/CodeGen/SystemZ/fp-strict-mul-13.ll
 create mode 100644 llvm/test/CodeGen/SystemZ/frame-22.ll
 create mode 100644 llvm/test/CodeGen/SystemZ/ghc-cc-01.ll
 create mode 100644 llvm/test/CodeGen/SystemZ/ghc-cc-02.ll
 create mode 100644 llvm/test/CodeGen/SystemZ/ghc-cc-03.ll
 create mode 100644 llvm/test/CodeGen/SystemZ/ghc-cc-04.ll
 create mode 100644 llvm/test/CodeGen/SystemZ/ghc-cc-05.ll
 create mode 100644 llvm/test/CodeGen/SystemZ/ghc-cc-06.ll
 create mode 100644 llvm/test/CodeGen/SystemZ/ghc-cc-07.ll
 create mode 100644 llvm/test/CodeGen/SystemZ/int-cmp-56.ll
 create mode 100644 llvm/test/CodeGen/SystemZ/int-cmp-57.ll
 create mode 100644 llvm/test/CodeGen/SystemZ/la-05.ll
 create mode 100644 llvm/test/CodeGen/SystemZ/mnop-mcount-01.ll
 create mode 100644 llvm/test/CodeGen/SystemZ/mnop-mcount-02.ll
 create mode 100644 llvm/test/CodeGen/SystemZ/mrecord-mcount-01.ll
 create mode 100644 llvm/test/CodeGen/SystemZ/mrecord-mcount-02.ll
 create mode 100644 llvm/test/CodeGen/SystemZ/vec-strict-cmp-01.ll
 create mode 100644 llvm/test/CodeGen/SystemZ/vec-strict-cmp-02.ll
 create mode 100644 llvm/test/CodeGen/SystemZ/vec-strict-cmp-03.ll
 create mode 100644 llvm/test/CodeGen/SystemZ/vec-strict-cmps-01.ll
 create mode 100644 llvm/test/CodeGen/SystemZ/vec-strict-cmps-02.ll
 create mode 100644 llvm/test/CodeGen/SystemZ/vec-strict-cmps-03.ll
 create mode 100644 llvm/test/CodeGen/SystemZ/vec-strict-conv-01.ll
 create mode 100644 llvm/test/CodeGen/SystemZ/vec-strict-conv-02.ll
 create mode 100644 llvm/test/CodeGen/SystemZ/vec-strict-conv-03.ll
 create mode 100644 llvm/test/CodeGen/Thumb/cmp-and-fold.ll
 create mode 100644 llvm/test/CodeGen/Thumb/scheduler-clone-cpsr-def.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/LowOverheadLoops/basic-tail-pred.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/LowOverheadLoops/branch-targets.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/LowOverheadLoops/cond-vector-reduce-mve-codegen.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/LowOverheadLoops/fast-fp-loops.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/LowOverheadLoops/mve-float-loops.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/LowOverheadLoops/mve-tail-data-types.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/LowOverheadLoops/nested.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/LowOverheadLoops/no-dec-le-simple.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-pred-narrow.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-pred-pattern-fail.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-pred-widen.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-reduce.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/LowOverheadLoops/vector-arith-codegen.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/LowOverheadLoops/vector-reduce-mve-tail.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/LowOverheadLoops/vector-unroll.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/bug-subw.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/csel.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/lsll0.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-be.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-bitreverse.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-bswap.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-ctlz.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-ctpop.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-cttz.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-extractelt.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-fmas.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-gather-ind16-scaled.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-gather-ind16-unscaled.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-gather-ind32-scaled.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-gather-ind32-unscaled.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-gather-ind8-unscaled.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-gather-ptrs.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-gather-scatter-opt.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-intrinsics/load-store.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-intrinsics/predicates.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-intrinsics/scalar-shifts.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-intrinsics/scatter-gather.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-intrinsics/vabavq.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-intrinsics/vabdq.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-intrinsics/vadc-multiple.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-intrinsics/vadc.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-intrinsics/vaddq.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-intrinsics/vandq.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-intrinsics/vbicq.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-intrinsics/vcaddq.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-intrinsics/vcmlaq.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-intrinsics/vcmulq.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-intrinsics/vcvt.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-intrinsics/vector-shift-imm-dyadic.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-intrinsics/vector-shift-imm.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-intrinsics/vector-shift-var.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-intrinsics/veorq.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-intrinsics/vhaddq.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-intrinsics/vhsubq.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-intrinsics/vld24.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-intrinsics/vldr.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-intrinsics/vmaxnmq.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-intrinsics/vmaxq.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-intrinsics/vminnmq.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-intrinsics/vminq.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-intrinsics/vminvq.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-intrinsics/vmldav.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-intrinsics/vmlldav.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-intrinsics/vmulhq.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-intrinsics/vmullbq.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-intrinsics/vmulltq.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-intrinsics/vmulq.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-intrinsics/vornq.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-intrinsics/vorrq.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-intrinsics/vqaddq.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-intrinsics/vqdmulhq.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-intrinsics/vqrdmulhq.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-intrinsics/vqsubq.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-intrinsics/vrhaddq.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-intrinsics/vrmulhq.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-intrinsics/vsubq.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-masked-ldst-offset.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-masked-ldst-postinc.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-masked-ldst-preinc.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-masked-ldst.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-masked-load.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-masked-store.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-multivec-spill.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-phireg.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-pred-and.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-pred-bitcast.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-pred-build-const.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-pred-build-var.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-pred-ext.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-pred-loadstore.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-pred-not.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-pred-or.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-pred-shuffle.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-pred-spill.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-pred-threshold.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-pred-xor.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-saturating-arith.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-shifts-scalar.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-shuffleext.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-shufflemov.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-stack.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-vaddqr.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-vaddv.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-vcmp.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-vcmpf.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-vcmpfr.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-vcmpfz.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-vcmpr.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-vcmpz.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-vctp.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-vecreduce-fminmax.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-vector-spill.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-vfma.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-vhaddsub.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-vld2.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-vld3.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-vld4.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-vldst4.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-vmaxv.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-vmla.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-vmovn.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-vmulqr.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-vpsel.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-vpt-from-intrinsics.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-vst2.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-vst3.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-vst4.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-vsubqr.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/t2peephole-t2ADDrr-to-t2ADDri.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/vqabs.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/vqneg.ll
 create mode 100644 llvm/test/CodeGen/VE/simple_prologue_epilogue.ll
 create mode 100644 llvm/test/CodeGen/VE/target_support.ll
 create mode 100644 llvm/test/CodeGen/WebAssembly/export-name.ll
 create mode 100644 llvm/test/CodeGen/WebAssembly/lower-em-exceptions-resume-only.ll
 create mode 100644 llvm/test/CodeGen/WebAssembly/lower-em-sjlj-alias.ll
 create mode 100644 llvm/test/CodeGen/WebAssembly/lower-em-sjlj-sret.ll
 create mode 100644 llvm/test/CodeGen/WebAssembly/simd-load-splat.ll
 create mode 100644 llvm/test/CodeGen/WinCFGuard/cfguard-cast.ll
 create mode 100644 llvm/test/CodeGen/X86/GlobalISel/ptr-add.ll
 create mode 100644 llvm/test/CodeGen/X86/align-branch-boundary-default.ll
 create mode 100644 llvm/test/CodeGen/X86/align-branch-boundary-noautopadding.ll
 create mode 100644 llvm/test/CodeGen/X86/align-branch-boundary-suppressions.ll
 create mode 100644 llvm/test/CodeGen/X86/align-down-const.ll
 create mode 100644 llvm/test/CodeGen/X86/align-down.ll
 create mode 100644 llvm/test/CodeGen/X86/atomic-non-integer-fp128.ll
 create mode 100644 llvm/test/CodeGen/X86/avx512-broadcast-unfold.ll
 create mode 100644 llvm/test/CodeGen/X86/avx512-cmp-mask.ll
 create mode 100644 llvm/test/CodeGen/X86/avx512vbmi2-funnel-shifts.ll
 create mode 100644 llvm/test/CodeGen/X86/avx512vbmi2vl-funnel-shifts.ll
 create mode 100644 llvm/test/CodeGen/X86/avx512vnni.ll
 create mode 100644 llvm/test/CodeGen/X86/cfguard-checks.ll
 create mode 100644 llvm/test/CodeGen/X86/cfguard-module-flag.ll
 create mode 100644 llvm/test/CodeGen/X86/cfguard-x86-64-vectorcall.ll
 create mode 100644 llvm/test/CodeGen/X86/cfguard-x86-vectorcall.ll
 create mode 100644 llvm/test/CodeGen/X86/codegen-prepare-collapse.ll
 create mode 100644 llvm/test/CodeGen/X86/coff-fp-section-name.ll
 create mode 100644 llvm/test/CodeGen/X86/combineIncDecVector-crash.ll
 create mode 100644 llvm/test/CodeGen/X86/conditional-tailcall-pgso.ll
 create mode 100644 llvm/test/CodeGen/X86/csr-split.ll
 create mode 100644 llvm/test/CodeGen/X86/dbg-value-func-arg.ll
 create mode 100644 llvm/test/CodeGen/X86/debug-loclists-lto.ll
 create mode 100644 llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll
 create mode 100644 llvm/test/CodeGen/X86/div-rem-pair-recomposition-unsigned.ll
 create mode 100644 llvm/test/CodeGen/X86/dtor-priority-coff.ll
 create mode 100644 llvm/test/CodeGen/X86/fma-fneg-combine-2.ll
 create mode 100644 llvm/test/CodeGen/X86/fmf-reduction.ll
 create mode 100644 llvm/test/CodeGen/X86/fp-intrinsics-flags-x86_64.ll
 create mode 100644 llvm/test/CodeGen/X86/fp-intrinsics-flags.ll
 create mode 100644 llvm/test/CodeGen/X86/fp-intrinsics-fma.ll
 create mode 100644 llvm/test/CodeGen/X86/fp-strict-libcalls-msvc32.ll
 create mode 100644 llvm/test/CodeGen/X86/fp-strict-scalar-cmp.ll
 create mode 100644 llvm/test/CodeGen/X86/fp-strict-scalar-fptoint.ll
 create mode 100644 llvm/test/CodeGen/X86/fp-strict-scalar-inttofp.ll
 create mode 100644 llvm/test/CodeGen/X86/fp-strict-scalar-round.ll
 create mode 100644 llvm/test/CodeGen/X86/fp-strict-scalar.ll
 create mode 100644 llvm/test/CodeGen/X86/fp128-cast-strict.ll
 create mode 100644 llvm/test/CodeGen/X86/fp128-libcalls-strict.ll
 create mode 100644 llvm/test/CodeGen/X86/fp80-strict-scalar-cmp.ll
 create mode 100644 llvm/test/CodeGen/X86/fp80-strict-scalar.ll
 create mode 100644 llvm/test/CodeGen/X86/haddsub-broadcast.ll
 create mode 100644 llvm/test/CodeGen/X86/isel-blendi-gettargetconstant.ll
 create mode 100644 llvm/test/CodeGen/X86/lower-ptrmask.ll
 create mode 100644 llvm/test/CodeGen/X86/memcmp-more-load-pairs.ll
 create mode 100644 llvm/test/CodeGen/X86/memcmp-pgso.ll
 create mode 100644 llvm/test/CodeGen/X86/mixed-ptr-sizes.ll
 create mode 100644 llvm/test/CodeGen/X86/ms-inline-asm-PR44272.ll
 create mode 100644 llvm/test/CodeGen/X86/mulfix_combine.ll
 create mode 100644 llvm/test/CodeGen/X86/musttail-inalloca.ll
 create mode 100644 llvm/test/CodeGen/X86/musttail-tailcc.ll
 create mode 100644 llvm/test/CodeGen/X86/mxcsr-reg-usage.ll
 create mode 100644 llvm/test/CodeGen/X86/no-sse-win64.ll
 create mode 100644 llvm/test/CodeGen/X86/no-sse-x86.ll
 create mode 100644 llvm/test/CodeGen/X86/nomovtopush.ll
 create mode 100644 llvm/test/CodeGen/X86/noreturn-call-linux.ll
 create mode 100644 llvm/test/CodeGen/X86/noreturn-call-win64.ll
 create mode 100644 llvm/test/CodeGen/X86/not-of-dec.ll
 create mode 100644 llvm/test/CodeGen/X86/offset-operator.ll
 create mode 100644 llvm/test/CodeGen/X86/patchable-function-entry-ibt.ll
 create mode 100644 llvm/test/CodeGen/X86/patchable-function-entry.ll
 create mode 100644 llvm/test/CodeGen/X86/powi-windows.ll
 create mode 100644 llvm/test/CodeGen/X86/pr42727.ll
 create mode 100644 llvm/test/CodeGen/X86/pr42870.ll
 create mode 100644 llvm/test/CodeGen/X86/pr42905.ll
 create mode 100644 llvm/test/CodeGen/X86/pr42909.ll
 create mode 100644 llvm/test/CodeGen/X86/pr42998.ll
 create mode 100644 llvm/test/CodeGen/X86/pr43157.ll
 create mode 100644 llvm/test/CodeGen/X86/pr43507.ll
 create mode 100644 llvm/test/CodeGen/X86/pr43509.ll
 create mode 100644 llvm/test/CodeGen/X86/pr43529.ll
 create mode 100644 llvm/test/CodeGen/X86/pr43575.ll
 create mode 100644 llvm/test/CodeGen/X86/pr43820.ll
 create mode 100644 llvm/test/CodeGen/X86/pr43866.ll
 create mode 100644 llvm/test/CodeGen/X86/pr43952.ll
 create mode 100644 llvm/test/CodeGen/X86/pr44140.ll
 create mode 100644 llvm/test/CodeGen/X86/pr44396.ll
 create mode 100644 llvm/test/CodeGen/X86/pr44412.ll
 create mode 100644 llvm/test/CodeGen/X86/pr44812.ll
 create mode 100644 llvm/test/CodeGen/X86/psadbw.ll
 create mode 100644 llvm/test/CodeGen/X86/relptr-rodata.ll
 create mode 100644 llvm/test/CodeGen/X86/sadd_sat_plus.ll
 create mode 100644 llvm/test/CodeGen/X86/scalar-fp-to-i32.ll
 create mode 100644 llvm/test/CodeGen/X86/sdiv_fix.ll
 create mode 100644 llvm/test/CodeGen/X86/select-sra.ll
 create mode 100644 llvm/test/CodeGen/X86/select-testb-volatile-load.ll
 create mode 100644 llvm/test/CodeGen/X86/shift-by-signext.ll
 create mode 100644 llvm/test/CodeGen/X86/shift-logic.ll
 create mode 100644 llvm/test/CodeGen/X86/shrink-compare-pgso.ll
 create mode 100644 llvm/test/CodeGen/X86/srem-lkk.ll
 create mode 100644 llvm/test/CodeGen/X86/srem-seteq-optsize.ll
 create mode 100644 llvm/test/CodeGen/X86/srem-seteq-vec-nonsplat.ll
 create mode 100644 llvm/test/CodeGen/X86/srem-seteq-vec-splat.ll
 create mode 100644 llvm/test/CodeGen/X86/srem-seteq.ll
 create mode 100644 llvm/test/CodeGen/X86/srem-vector-lkk.ll
 create mode 100644 llvm/test/CodeGen/X86/ssub_sat_plus.ll
 create mode 100644 llvm/test/CodeGen/X86/stack-folding-int-avx512vnni.ll
 create mode 100644 llvm/test/CodeGen/X86/stack-protector-2.ll
 create mode 100644 llvm/test/CodeGen/X86/stack-protector-strong-macho-win32-xor.ll
 create mode 100644 llvm/test/CodeGen/X86/statepoint-no-realign-stack.ll
 create mode 100644 llvm/test/CodeGen/X86/sub-of-bias.ll
 create mode 100644 llvm/test/CodeGen/X86/tail-call-deref.ll
 create mode 100644 llvm/test/CodeGen/X86/tailcall-assume.ll
 create mode 100644 llvm/test/CodeGen/X86/tailcall-tailcc.ll
 create mode 100644 llvm/test/CodeGen/X86/tailcc-calleesave.ll
 create mode 100644 llvm/test/CodeGen/X86/tailcc-disable-tail-calls.ll
 create mode 100644 llvm/test/CodeGen/X86/tailcc-fastcc.ll
 create mode 100644 llvm/test/CodeGen/X86/tailcc-fastisel.ll
 create mode 100644 llvm/test/CodeGen/X86/tailcc-largecode.ll
 create mode 100644 llvm/test/CodeGen/X86/tailcc-stackalign.ll
 create mode 100644 llvm/test/CodeGen/X86/tailcc-structret.ll
 create mode 100644 llvm/test/CodeGen/X86/tailccbyval.ll
 create mode 100644 llvm/test/CodeGen/X86/tailccbyval64.ll
 create mode 100644 llvm/test/CodeGen/X86/tailccfp.ll
 create mode 100644 llvm/test/CodeGen/X86/tailccfp2.ll
 create mode 100644 llvm/test/CodeGen/X86/tailccpic1.ll
 create mode 100644 llvm/test/CodeGen/X86/tailccpic2.ll
 create mode 100644 llvm/test/CodeGen/X86/tailccstack64.ll
 create mode 100644 llvm/test/CodeGen/X86/taildup-heapallocsite.ll
 create mode 100644 llvm/test/CodeGen/X86/typeid-alias.ll
 create mode 100644 llvm/test/CodeGen/X86/uadd_sat_plus.ll
 create mode 100644 llvm/test/CodeGen/X86/udiv_fix.ll
 create mode 100644 llvm/test/CodeGen/X86/umul_fix_sat.ll
 create mode 100644 llvm/test/CodeGen/X86/urem-lkk.ll
 create mode 100644 llvm/test/CodeGen/X86/urem-seteq-nonzero.ll
 create mode 100644 llvm/test/CodeGen/X86/urem-seteq-vec-nonzero.ll
 create mode 100644 llvm/test/CodeGen/X86/urem-seteq-vec-tautological.ll
 create mode 100644 llvm/test/CodeGen/X86/urem-vector-lkk.ll
 create mode 100644 llvm/test/CodeGen/X86/use-cr-result-of-dom-icmp-st.ll
 create mode 100644 llvm/test/CodeGen/X86/usub_sat_plus.ll
 create mode 100644 llvm/test/CodeGen/X86/vec-strict-128.ll
 create mode 100644 llvm/test/CodeGen/X86/vec-strict-256.ll
 create mode 100644 llvm/test/CodeGen/X86/vec-strict-512.ll
 create mode 100644 llvm/test/CodeGen/X86/vec-strict-cmp-128.ll
 create mode 100644 llvm/test/CodeGen/X86/vec-strict-cmp-256.ll
 create mode 100644 llvm/test/CodeGen/X86/vec-strict-cmp-512.ll
 create mode 100644 llvm/test/CodeGen/X86/vec-strict-cmp-sub128.ll
 create mode 100644 llvm/test/CodeGen/X86/vec-strict-fptoint-128.ll
 create mode 100644 llvm/test/CodeGen/X86/vec-strict-fptoint-256.ll
 create mode 100644 llvm/test/CodeGen/X86/vec-strict-fptoint-512.ll
 create mode 100644 llvm/test/CodeGen/X86/vec-strict-inttofp-128.ll
 create mode 100644 llvm/test/CodeGen/X86/vec-strict-inttofp-256.ll
 create mode 100644 llvm/test/CodeGen/X86/vec-strict-inttofp-512.ll
 create mode 100644 llvm/test/CodeGen/X86/vec-strict-round-128.ll
 create mode 100644 llvm/test/CodeGen/X86/vector-constrained-fp-intrinsics-flags.ll
 create mode 100644 llvm/test/CodeGen/X86/vector-lzcnt-sub128.ll
 create mode 100644 llvm/test/CodeGen/X86/vector-mulfix-legalize.ll
 create mode 100644 llvm/test/CodeGen/X86/vshli-simplify-demanded-bits.ll
 create mode 100644 llvm/test/CodeGen/X86/win64-eh-empty-block.ll
 create mode 100644 llvm/test/CodeGen/X86/win64-funclet-savexmm.ll
 create mode 100644 llvm/test/CodeGen/X86/win64-stackprobe-overflow.ll
 create mode 100644 llvm/test/DebugInfo/AArch64/call-site-info-output.ll
 create mode 100644 llvm/test/DebugInfo/ARM/call-site-info-output.ll
 create mode 100644 llvm/test/DebugInfo/ARM/entry-value-multi-byte-expr.ll
 create mode 100644 llvm/test/DebugInfo/COFF/line-zero.ll
 create mode 100644 llvm/test/DebugInfo/Generic/export-symbol-anonymous-class.ll
 create mode 100644 llvm/test/DebugInfo/RISCV/relax-debug-frame.ll
 create mode 100644 llvm/test/DebugInfo/Sparc/entry-value-complex-reg-expr.ll
 create mode 100644 llvm/test/DebugInfo/WebAssembly/dbg-value-dwarfdump.ll
 create mode 100644 llvm/test/DebugInfo/WebAssembly/dbg-value-ti.ll
 create mode 100644 llvm/test/DebugInfo/X86/DW_AT_deleted.ll
 create mode 100644 llvm/test/DebugInfo/X86/codegenprep-addrsink.ll
 create mode 100644 llvm/test/DebugInfo/X86/dbg-value-dropped-instcombine.ll
 create mode 100644 llvm/test/DebugInfo/X86/dbgcall-site-64-bit-imms.ll
 create mode 100644 llvm/test/DebugInfo/X86/dbgcall-site-zero-valued-imms.ll
 create mode 100644 llvm/test/DebugInfo/X86/debug-info-template-align.ll
 create mode 100644 llvm/test/DebugInfo/X86/debug-macinfo-split-dwarf.ll
 create mode 100644 llvm/test/DebugInfo/X86/gmlt-empty-base-address.ll
 create mode 100644 llvm/test/DebugInfo/X86/live-debug-values-expr-conflict.ll
 create mode 100644 llvm/test/DebugInfo/X86/live-debug-values-remove-range.ll
 create mode 100644 llvm/test/DebugInfo/X86/objc_direct.ll
 create mode 100644 llvm/test/DebugInfo/X86/sdag-transfer-dbgvalue.ll
 create mode 100644 llvm/test/DebugInfo/X86/sroa-after-inlining.ll
 create mode 100644 llvm/test/DebugInfo/salvage-cast-debug-info.ll
 create mode 100644 llvm/test/Examples/IRTransforms/SimplifyCFG/tut-simplify-cfg-blockaddress.ll
 create mode 100644 llvm/test/Examples/IRTransforms/SimplifyCFG/tut-simplify-cfg1.ll
 create mode 100644 llvm/test/Examples/IRTransforms/SimplifyCFG/tut-simplify-cfg2-dead-block-order.ll
 create mode 100644 llvm/test/Examples/IRTransforms/SimplifyCFG/tut-simplify-cfg3-phis.ll
 create mode 100644 llvm/test/Examples/IRTransforms/SimplifyCFG/tut-simplify-cfg4-multiple-duplicate-cfg-updates.ll
 create mode 100644 llvm/test/Examples/IRTransforms/SimplifyCFG/tut-simplify-cfg5-del-phis-for-dead-block.ll
 create mode 100644 llvm/test/Examples/IRTransforms/SimplifyCFG/tut-simplify-cfg6-dead-self-loop.ll
 create mode 100644 llvm/test/ExecutionEngine/OrcLazy/emulated-tls.ll
 create mode 100644 llvm/test/ExecutionEngine/OrcLazy/printargv.ll
 create mode 100644 llvm/test/ExecutionEngine/OrcLazy/static-library-support.ll
 create mode 100644 llvm/test/Feature/load_extension.ll
 create mode 100644 llvm/test/Instrumentation/AddressSanitizer/debug-info-alloca.ll
 create mode 100644 llvm/test/Instrumentation/AddressSanitizer/global_addrspace.ll
 create mode 100644 llvm/test/Instrumentation/AddressSanitizer/version-mismatch-check.ll
 create mode 100644 llvm/test/Instrumentation/HWAddressSanitizer/alloca-compat.ll
 create mode 100644 llvm/test/Instrumentation/HWAddressSanitizer/basic-compat.ll
 create mode 100644 llvm/test/Instrumentation/HWAddressSanitizer/dbg-value-tag-offset.ll
 create mode 100644 llvm/test/Instrumentation/HWAddressSanitizer/globals.ll
 create mode 100644 llvm/test/Instrumentation/HWAddressSanitizer/personality.ll
 create mode 100644 llvm/test/Instrumentation/MemorySanitizer/attributes.ll
 create mode 100644 llvm/test/Instrumentation/MemorySanitizer/clmul.ll
 create mode 100644 llvm/test/Instrumentation/MemorySanitizer/msan_llvm_launder_invariant.ll
 create mode 100644 llvm/test/Instrumentation/MemorySanitizer/msan_llvm_strip_invariant.ll
 create mode 100644 llvm/test/LTO/ARM/lto-linking-metadata.ll
 create mode 100644 llvm/test/LTO/Resolution/X86/Inputs/ifunc2.ll
 create mode 100644 llvm/test/LTO/Resolution/X86/ifunc2.ll
 create mode 100644 llvm/test/LTO/Resolution/X86/not-prevailing-weak-aliasee.ll
 create mode 100644 llvm/test/LTO/X86/Inputs/start-lib1.ll
 create mode 100644 llvm/test/LTO/X86/Inputs/start-lib2.ll
 create mode 100644 llvm/test/LTO/X86/Inputs/type-mapping-bug3.ll
 create mode 100644 llvm/test/LTO/X86/embed-bitcode.ll
 create mode 100644 llvm/test/LTO/X86/type-mapping-bug3.ll
 create mode 100644 llvm/test/Linker/Inputs/module-max-warn.ll
 create mode 100644 llvm/test/Linker/addrspace.ll
 create mode 100644 llvm/test/Linker/module-max-warn.ll
 create mode 100644 llvm/test/MC/ELF/section-relro.ll
 create mode 100644 llvm/test/MC/WebAssembly/data-symbol-in-text-section.ll
 create mode 100644 llvm/test/Other/module-pass-printer.ll
 create mode 100644 llvm/test/Other/new-pm-pgo-O0.ll
 create mode 100644 llvm/test/Other/new-pm-pr42726-cgscc.ll
 create mode 100644 llvm/test/Other/print-slotindexes.ll
 create mode 100644 llvm/test/Other/scalable-vectors-core-ir.ll
 create mode 100644 llvm/test/Other/unroll-sroa.ll
 create mode 100644 llvm/test/Reduce/remove-args.ll
 create mode 100644 llvm/test/Reduce/remove-bbs.ll
 create mode 100644 llvm/test/Reduce/remove-funcs.ll
 create mode 100644 llvm/test/Reduce/remove-global-vars.ll
 create mode 100644 llvm/test/Reduce/remove-instructions.ll
 create mode 100644 llvm/test/Reduce/remove-metadata.ll
 create mode 100644 llvm/test/ThinLTO/X86/Inputs/devirt2.ll
 create mode 100644 llvm/test/ThinLTO/X86/Inputs/devirt_alias.ll
 create mode 100644 llvm/test/ThinLTO/X86/Inputs/devirt_available_externally.ll
 create mode 100644 llvm/test/ThinLTO/X86/Inputs/devirt_external_comdat_same_guid.ll
 create mode 100644 llvm/test/ThinLTO/X86/Inputs/devirt_local_same_guid.ll
 create mode 100644 llvm/test/ThinLTO/X86/Inputs/devirt_promote.ll
 create mode 100644 llvm/test/ThinLTO/X86/Inputs/devirt_single_hybrid_bar.ll
 create mode 100644 llvm/test/ThinLTO/X86/Inputs/devirt_single_hybrid_foo.ll
 create mode 100644 llvm/test/ThinLTO/X86/Inputs/funcimport_alwaysinline.ll
 create mode 100644 llvm/test/ThinLTO/X86/Inputs/guid_collision.ll
 create mode 100644 llvm/test/ThinLTO/X86/Inputs/internalize.ll
 create mode 100644 llvm/test/ThinLTO/X86/Inputs/thinlto-internalize-doublepromoted.ll
 create mode 100644 llvm/test/ThinLTO/X86/Inputs/writeonly-with-refs.ll
 create mode 100644 llvm/test/ThinLTO/X86/devirt2.ll
 create mode 100644 llvm/test/ThinLTO/X86/devirt_alias.ll
 create mode 100644 llvm/test/ThinLTO/X86/devirt_available_externally.ll
 create mode 100644 llvm/test/ThinLTO/X86/devirt_external_comdat_same_guid.ll
 create mode 100644 llvm/test/ThinLTO/X86/devirt_local_same_guid.ll
 create mode 100644 llvm/test/ThinLTO/X86/devirt_promote.ll
 create mode 100644 llvm/test/ThinLTO/X86/devirt_promote_legacy.ll
 create mode 100644 llvm/test/ThinLTO/X86/devirt_single_hybrid.ll
 create mode 100644 llvm/test/ThinLTO/X86/funcimport-stats.ll
 create mode 100644 llvm/test/ThinLTO/X86/funcimport_alwaysinline.ll
 create mode 100644 llvm/test/ThinLTO/X86/guid_collision.ll
 create mode 100644 llvm/test/ThinLTO/X86/nodevirt-nonpromoted-typeid.ll
 create mode 100644 llvm/test/ThinLTO/X86/not-internalized.ll
 create mode 100644 llvm/test/ThinLTO/X86/printer.ll
 create mode 100644 llvm/test/ThinLTO/X86/thinlto-internalize-doublepromoted.ll
 create mode 100644 llvm/test/ThinLTO/X86/writeonly-with-refs.ll
 create mode 100644 llvm/test/Transforms/AggressiveInstCombine/popcount.ll
 create mode 100644 llvm/test/Transforms/Attributor/ArgumentPromotion/2008-02-01-ReturnAttrs.ll
 create mode 100644 llvm/test/Transforms/Attributor/ArgumentPromotion/2008-07-02-array-indexing.ll
 create mode 100644 llvm/test/Transforms/Attributor/ArgumentPromotion/2008-09-07-CGUpdate.ll
 create mode 100644 llvm/test/Transforms/Attributor/ArgumentPromotion/2008-09-08-CGUpdateSelfEdge.ll
 create mode 100644 llvm/test/Transforms/Attributor/ArgumentPromotion/X86/attributes.ll
 create mode 100644 llvm/test/Transforms/Attributor/ArgumentPromotion/X86/min-legal-vector-width.ll
 create mode 100644 llvm/test/Transforms/Attributor/ArgumentPromotion/X86/thiscall.ll
 create mode 100644 llvm/test/Transforms/Attributor/ArgumentPromotion/aggregate-promote.ll
 create mode 100644 llvm/test/Transforms/Attributor/ArgumentPromotion/attrs.ll
 create mode 100644 llvm/test/Transforms/Attributor/ArgumentPromotion/basictest.ll
 create mode 100644 llvm/test/Transforms/Attributor/ArgumentPromotion/byval-2.ll
 create mode 100644 llvm/test/Transforms/Attributor/ArgumentPromotion/byval.ll
 create mode 100644 llvm/test/Transforms/Attributor/ArgumentPromotion/chained.ll
 create mode 100644 llvm/test/Transforms/Attributor/ArgumentPromotion/control-flow.ll
 create mode 100644 llvm/test/Transforms/Attributor/ArgumentPromotion/control-flow2.ll
 create mode 100644 llvm/test/Transforms/Attributor/ArgumentPromotion/crash.ll
 create mode 100644 llvm/test/Transforms/Attributor/ArgumentPromotion/dbg.ll
 create mode 100644 llvm/test/Transforms/Attributor/ArgumentPromotion/fp80.ll
 create mode 100644 llvm/test/Transforms/Attributor/ArgumentPromotion/inalloca.ll
 create mode 100644 llvm/test/Transforms/Attributor/ArgumentPromotion/invalidation.ll
 create mode 100644 llvm/test/Transforms/Attributor/ArgumentPromotion/live_called_from_dead.ll
 create mode 100644 llvm/test/Transforms/Attributor/ArgumentPromotion/musttail.ll
 create mode 100644 llvm/test/Transforms/Attributor/ArgumentPromotion/naked_functions.ll
 create mode 100644 llvm/test/Transforms/Attributor/ArgumentPromotion/nonzero-address-spaces.ll
 create mode 100644 llvm/test/Transforms/Attributor/ArgumentPromotion/pr27568.ll
 create mode 100644 llvm/test/Transforms/Attributor/ArgumentPromotion/pr3085.ll
 create mode 100644 llvm/test/Transforms/Attributor/ArgumentPromotion/pr32917.ll
 create mode 100644 llvm/test/Transforms/Attributor/ArgumentPromotion/pr33641_remove_arg_dbgvalue.ll
 create mode 100644 llvm/test/Transforms/Attributor/ArgumentPromotion/profile.ll
 create mode 100644 llvm/test/Transforms/Attributor/ArgumentPromotion/reserve-tbaa.ll
 create mode 100644 llvm/test/Transforms/Attributor/ArgumentPromotion/sret.ll
 create mode 100644 llvm/test/Transforms/Attributor/ArgumentPromotion/tail.ll
 create mode 100644 llvm/test/Transforms/Attributor/ArgumentPromotion/variadic.ll
 create mode 100644 llvm/test/Transforms/Attributor/IPConstantProp/2008-06-09-WeakProp.ll
 create mode 100644 llvm/test/Transforms/Attributor/IPConstantProp/2009-09-24-byval-ptr.ll
 create mode 100644 llvm/test/Transforms/Attributor/IPConstantProp/PR16052.ll
 create mode 100644 llvm/test/Transforms/Attributor/IPConstantProp/PR26044.ll
 create mode 100644 llvm/test/Transforms/Attributor/IPConstantProp/PR43857.ll
 create mode 100644 llvm/test/Transforms/Attributor/IPConstantProp/arg-count-mismatch.ll
 create mode 100644 llvm/test/Transforms/Attributor/IPConstantProp/arg-type-mismatch.ll
 create mode 100644 llvm/test/Transforms/Attributor/IPConstantProp/comdat-ipo.ll
 create mode 100644 llvm/test/Transforms/Attributor/IPConstantProp/dangling-block-address.ll
 create mode 100644 llvm/test/Transforms/Attributor/IPConstantProp/deadarg.ll
 create mode 100644 llvm/test/Transforms/Attributor/IPConstantProp/fp-bc-icmp-const-fold.ll
 create mode 100644 llvm/test/Transforms/Attributor/IPConstantProp/global.ll
 create mode 100644 llvm/test/Transforms/Attributor/IPConstantProp/multiple_callbacks.ll
 create mode 100644 llvm/test/Transforms/Attributor/IPConstantProp/musttail-call.ll
 create mode 100644 llvm/test/Transforms/Attributor/IPConstantProp/naked-return.ll
 create mode 100644 llvm/test/Transforms/Attributor/IPConstantProp/openmp_parallel_for.ll
 create mode 100644 llvm/test/Transforms/Attributor/IPConstantProp/pthreads.ll
 create mode 100644 llvm/test/Transforms/Attributor/IPConstantProp/recursion.ll
 create mode 100644 llvm/test/Transforms/Attributor/IPConstantProp/remove-call-inst.ll
 create mode 100644 llvm/test/Transforms/Attributor/IPConstantProp/return-argument.ll
 create mode 100644 llvm/test/Transforms/Attributor/IPConstantProp/return-constant.ll
 create mode 100644 llvm/test/Transforms/Attributor/IPConstantProp/return-constants.ll
 create mode 100644 llvm/test/Transforms/Attributor/IPConstantProp/solve-after-each-resolving-undefs-for-function.ll
 create mode 100644 llvm/test/Transforms/Attributor/IPConstantProp/thread_local_acs.ll
 create mode 100644 llvm/test/Transforms/Attributor/align.ll
 create mode 100644 llvm/test/Transforms/Attributor/callbacks.ll
 create mode 100644 llvm/test/Transforms/Attributor/dereferenceable-1.ll
 create mode 100644 llvm/test/Transforms/Attributor/dereferenceable-2.ll
 create mode 100644 llvm/test/Transforms/Attributor/heap_to_stack.ll
 create mode 100644 llvm/test/Transforms/Attributor/internal-noalias.ll
 create mode 100644 llvm/test/Transforms/Attributor/liveness.ll
 create mode 100644 llvm/test/Transforms/Attributor/lvi-after-jumpthreading.ll
 create mode 100644 llvm/test/Transforms/Attributor/lvi-for-ashr.ll
 create mode 100644 llvm/test/Transforms/Attributor/misc.ll
 create mode 100644 llvm/test/Transforms/Attributor/new_attributes.ll
 create mode 100644 llvm/test/Transforms/Attributor/noalias.ll
 create mode 100644 llvm/test/Transforms/Attributor/nocapture-1.ll
 create mode 100644 llvm/test/Transforms/Attributor/nocapture-2.ll
 create mode 100644 llvm/test/Transforms/Attributor/nofree.ll
 create mode 100644 llvm/test/Transforms/Attributor/nonnull.ll
 create mode 100644 llvm/test/Transforms/Attributor/norecurse.ll
 create mode 100644 llvm/test/Transforms/Attributor/noreturn.ll
 create mode 100644 llvm/test/Transforms/Attributor/noreturn_async.ll
 create mode 100644 llvm/test/Transforms/Attributor/noreturn_sync.ll
 create mode 100644 llvm/test/Transforms/Attributor/nosync.ll
 create mode 100644 llvm/test/Transforms/Attributor/nounwind.ll
 create mode 100644 llvm/test/Transforms/Attributor/range.ll
 create mode 100644 llvm/test/Transforms/Attributor/read_write_returned_arguments_scc.ll
 create mode 100644 llvm/test/Transforms/Attributor/readattrs.ll
 create mode 100644 llvm/test/Transforms/Attributor/returned.ll
 create mode 100644 llvm/test/Transforms/Attributor/undefined_behavior.ll
 create mode 100644 llvm/test/Transforms/Attributor/value-simplify.ll
 create mode 100644 llvm/test/Transforms/Attributor/willreturn.ll
 create mode 100644 llvm/test/Transforms/BDCE/pr41925.ll
 create mode 100644 llvm/test/Transforms/BlockExtractor/invalid-line.ll
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/ARM/sink-add-mul-shufflevector.ll
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/ARM/sinkchain.ll
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/PowerPC/split-store-alignment.ll
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/X86/split-store-alignment.ll
 create mode 100644 llvm/test/Transforms/ConstProp/fma.ll
 create mode 100644 llvm/test/Transforms/ConstantHoisting/AArch64/const-hoist-intrinsics.ll
 create mode 100755 llvm/test/Transforms/ConstantHoisting/AArch64/consthoist-unreachable.ll
 create mode 100644 llvm/test/Transforms/ConstantHoisting/X86/pr43903-not-all-uses-rebased.ll
 create mode 100644 llvm/test/Transforms/Coroutines/coro-alloc-with-param.ll
 create mode 100644 llvm/test/Transforms/Coroutines/coro-param-copy.ll
 create mode 100644 llvm/test/Transforms/Coroutines/coro-retcon-alloca.ll
 create mode 100644 llvm/test/Transforms/Coroutines/coro-retcon-once-value.ll
 create mode 100644 llvm/test/Transforms/Coroutines/coro-retcon-once-value2.ll
 create mode 100644 llvm/test/Transforms/Coroutines/coro-retcon-resume-values.ll
 create mode 100644 llvm/test/Transforms/Coroutines/coro-retcon-resume-values2.ll
 create mode 100644 llvm/test/Transforms/Coroutines/coro-retcon-value.ll
 create mode 100644 llvm/test/Transforms/Coroutines/coro-retcon.ll
 create mode 100644 llvm/test/Transforms/Coroutines/coro-split-musttail1.ll
 create mode 100644 llvm/test/Transforms/Coroutines/coro-swifterror.ll
 create mode 100644 llvm/test/Transforms/CorrelatedValuePropagation/and.ll
 create mode 100644 llvm/test/Transforms/CorrelatedValuePropagation/mul.ll
 create mode 100644 llvm/test/Transforms/CorrelatedValuePropagation/sext.ll
 create mode 100644 llvm/test/Transforms/CorrelatedValuePropagation/shl.ll
 create mode 100644 llvm/test/Transforms/DCE/dbg-value-removal.ll
 create mode 100644 llvm/test/Transforms/DeadStoreElimination/DeleteThrowableInst.ll
 create mode 100644 llvm/test/Transforms/DeadStoreElimination/libcalls2.ll
 create mode 100644 llvm/test/Transforms/EarlyCSE/writeonly.ll
 create mode 100644 llvm/test/Transforms/FunctionAttrs/writeonly.ll
 create mode 100644 llvm/test/Transforms/GVN/equality-assume.ll
 create mode 100644 llvm/test/Transforms/GVN/pr42605.ll
 create mode 100644 llvm/test/Transforms/GVN/preserve-analysis.ll
 create mode 100644 llvm/test/Transforms/GlobalDCE/virtual-functions-base-call.ll
 create mode 100644 llvm/test/Transforms/GlobalDCE/virtual-functions-base-pointer-call.ll
 create mode 100644 llvm/test/Transforms/GlobalDCE/virtual-functions-derived-call.ll
 create mode 100644 llvm/test/Transforms/GlobalDCE/virtual-functions-derived-pointer-call.ll
 create mode 100644 llvm/test/Transforms/GlobalDCE/virtual-functions-visibility-post-lto.ll
 create mode 100644 llvm/test/Transforms/GlobalDCE/virtual-functions-visibility-pre-lto.ll
 create mode 100644 llvm/test/Transforms/GlobalDCE/virtual-functions.ll
 create mode 100644 llvm/test/Transforms/GlobalDCE/vtable-rtti.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/large-int-crash.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/long-compilation-global-sra.ll
 create mode 100644 llvm/test/Transforms/HotColdSplit/assumption-cache-invalidation.ll
 create mode 100644 llvm/test/Transforms/HotColdSplit/retain-section.ll
 create mode 100644 llvm/test/Transforms/IPConstantProp/PR43857.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/eliminate-exit-no-dl.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/indvar-debug-value.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/indvar-debug-value2.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/loop-predication.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/rlev-add-me.ll
 create mode 100644 llvm/test/Transforms/InferAddressSpaces/AMDGPU/address-space-id-funcs.ll
 create mode 100644 llvm/test/Transforms/Inline/X86/switch.ll
 create mode 100644 llvm/test/Transforms/Inline/inline-indirect-chain.ll
 create mode 100644 llvm/test/Transforms/Inline/inline-skip-use-empty-alloca.ll
 create mode 100644 llvm/test/Transforms/Inline/no-inline-line-tables.ll
 create mode 100644 llvm/test/Transforms/InstCombine/AMDGPU/tan.ll
 create mode 100644 llvm/test/Transforms/InstCombine/ARM/mve-v2i2v.ll
 create mode 100644 llvm/test/Transforms/InstCombine/X86/2009-03-23-i80-fp80.ll
 create mode 100644 llvm/test/Transforms/InstCombine/bcopy.ll
 create mode 100644 llvm/test/Transforms/InstCombine/bitcast-function.ll
 create mode 100644 llvm/test/Transforms/InstCombine/bitcast-phi-uselistorder.ll
 create mode 100644 llvm/test/Transforms/InstCombine/builtin-object-size-custom-dl.ll
 create mode 100644 llvm/test/Transforms/InstCombine/canonicalize-clamp-like-pattern-between-negative-and-positive-thresholds.ll
 create mode 100644 llvm/test/Transforms/InstCombine/canonicalize-clamp-like-pattern-between-zero-and-positive-threshold.ll
 create mode 100644 llvm/test/Transforms/InstCombine/canonicalize-clamp-with-select-of-constant-threshold-pattern.ll
 create mode 100644 llvm/test/Transforms/InstCombine/conditional-variable-length-signext-after-high-bit-extract.ll
 create mode 100644 llvm/test/Transforms/InstCombine/deref-alloc-fns.ll
 create mode 100644 llvm/test/Transforms/InstCombine/disable-builtin.ll
 create mode 100644 llvm/test/Transforms/InstCombine/do-not-clone-dbg-declare.ll
 create mode 100644 llvm/test/Transforms/InstCombine/expensive-combines.ll
 create mode 100644 llvm/test/Transforms/InstCombine/fabs-copysign.ll
 create mode 100644 llvm/test/Transforms/InstCombine/fptrunc.ll
 create mode 100644 llvm/test/Transforms/InstCombine/freeze.ll
 create mode 100644 llvm/test/Transforms/InstCombine/gep-alias.ll
 create mode 100644 llvm/test/Transforms/InstCombine/gep-inbounds-null.ll
 create mode 100644 llvm/test/Transforms/InstCombine/high-bit-signmask-with-trunc.ll
 create mode 100644 llvm/test/Transforms/InstCombine/high-bit-signmask.ll
 create mode 100644 llvm/test/Transforms/InstCombine/hoist-negation-out-of-bias-calculation-with-constant.ll
 create mode 100644 llvm/test/Transforms/InstCombine/hoist-negation-out-of-bias-calculation.ll
 create mode 100644 llvm/test/Transforms/InstCombine/intptr8.ll
 create mode 100644 llvm/test/Transforms/InstCombine/lifetime-sanitizer.ll
 create mode 100644 llvm/test/Transforms/InstCombine/limit-max-iterations.ll
 create mode 100644 llvm/test/Transforms/InstCombine/load-insert-store.ll
 create mode 100644 llvm/test/Transforms/InstCombine/mem-deref-bytes-addrspaces.ll
 create mode 100644 llvm/test/Transforms/InstCombine/mem-deref-bytes.ll
 create mode 100644 llvm/test/Transforms/InstCombine/memccpy.ll
 create mode 100644 llvm/test/Transforms/InstCombine/mempcpy.ll
 create mode 100644 llvm/test/Transforms/InstCombine/memrchr.ll
 create mode 100644 llvm/test/Transforms/InstCombine/minmax-of-minmax.ll
 create mode 100644 llvm/test/Transforms/InstCombine/overflow_to_sat.ll
 create mode 100644 llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-after-truncation-variant-a.ll
 create mode 100644 llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-after-truncation-variant-b.ll
 create mode 100644 llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-after-truncation-variant-c.ll
 create mode 100644 llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-after-truncation-variant-d.ll
 create mode 100644 llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-after-truncation-variant-e.ll
 create mode 100644 llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-variant-a.ll
 create mode 100644 llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-variant-b.ll
 create mode 100644 llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-variant-c.ll
 create mode 100644 llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-variant-d.ll
 create mode 100644 llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-variant-e.ll
 create mode 100644 llvm/test/Transforms/InstCombine/phi-equal-incoming-pointers.ll
 create mode 100644 llvm/test/Transforms/InstCombine/phi-known-bits-operand-order.ll
 create mode 100644 llvm/test/Transforms/InstCombine/pr43081.ll
 create mode 100644 llvm/test/Transforms/InstCombine/pr43376-getFlippedStrictnessPredicateAndConstant-assert.ll
 create mode 100644 llvm/test/Transforms/InstCombine/pr43893.ll
 create mode 100644 llvm/test/Transforms/InstCombine/pr44242.ll
 create mode 100644 llvm/test/Transforms/InstCombine/pr44245.ll
 create mode 100644 llvm/test/Transforms/InstCombine/pr44541.ll
 create mode 100644 llvm/test/Transforms/InstCombine/pr44552.ll
 create mode 100644 llvm/test/Transforms/InstCombine/pr44835.ll
 create mode 100644 llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-after-truncation-variant-a.ll
 create mode 100644 llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-after-truncation-variant-b.ll
 create mode 100644 llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-after-truncation-variant-c.ll
 create mode 100644 llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-after-truncation-variant-d.ll
 create mode 100644 llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-after-truncation-variant-e.ll
 create mode 100644 llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-after-truncation-variant-f.ll
 create mode 100644 llvm/test/Transforms/InstCombine/result-of-add-of-negative-is-non-zero-and-no-underflow.ll
 create mode 100644 llvm/test/Transforms/InstCombine/result-of-add-of-negative-or-zero-is-non-zero-and-no-underflow.ll
 create mode 100644 llvm/test/Transforms/InstCombine/result-of-usub-is-non-zero-and-no-overflow.ll
 create mode 100644 llvm/test/Transforms/InstCombine/reuse-constant-from-select-in-icmp.ll
 create mode 100644 llvm/test/Transforms/InstCombine/sadd_sat.ll
 create mode 100644 llvm/test/Transforms/InstCombine/select-ctlz-to-cttz.ll
 create mode 100644 llvm/test/Transforms/InstCombine/select-imm-canon.ll
 create mode 100644 llvm/test/Transforms/InstCombine/shift-amount-reassociation-in-bittest-with-truncation-lshr.ll
 create mode 100644 llvm/test/Transforms/InstCombine/shift-amount-reassociation-in-bittest-with-truncation-shl.ll
 create mode 100644 llvm/test/Transforms/InstCombine/shift-amount-reassociation-with-truncation-ashr.ll
 create mode 100644 llvm/test/Transforms/InstCombine/shift-amount-reassociation-with-truncation-lshr.ll
 create mode 100644 llvm/test/Transforms/InstCombine/shift-amount-reassociation-with-truncation-shl.ll
 create mode 100644 llvm/test/Transforms/InstCombine/shift-by-signext.ll
 create mode 100644 llvm/test/Transforms/InstCombine/shift-logic.ll
 create mode 100644 llvm/test/Transforms/InstCombine/shufflevector-div-rem.ll
 create mode 100644 llvm/test/Transforms/InstCombine/sign-bit-test-via-right-shifting-all-other-bits.ll
 create mode 100644 llvm/test/Transforms/InstCombine/srem-via-sdiv-mul-sub.ll
 create mode 100644 llvm/test/Transforms/InstCombine/stdio-custom-dl.ll
 create mode 100644 llvm/test/Transforms/InstCombine/strict-sub-underflow-check-to-comparison-of-sub-operands.ll
 create mode 100644 llvm/test/Transforms/InstCombine/strndup.ll
 create mode 100644 llvm/test/Transforms/InstCombine/sub-and-or-neg-xor.ll
 create mode 100644 llvm/test/Transforms/InstCombine/sub-ashr-and-to-icmp-select.ll
 create mode 100644 llvm/test/Transforms/InstCombine/sub-ashr-or-to-icmp-select.ll
 create mode 100644 llvm/test/Transforms/InstCombine/sub-gep.ll
 create mode 100644 llvm/test/Transforms/InstCombine/sub-of-negatible.ll
 create mode 100644 llvm/test/Transforms/InstCombine/sub-or-and-xor.ll
 create mode 100644 llvm/test/Transforms/InstCombine/sub-xor-or-neg-and.ll
 create mode 100644 llvm/test/Transforms/InstCombine/subtract-from-one-hand-of-select.ll
 create mode 100644 llvm/test/Transforms/InstCombine/subtract-of-one-hand-of-select.ll
 create mode 100644 llvm/test/Transforms/InstCombine/unsigned-add-lack-of-overflow-check-via-add.ll
 create mode 100644 llvm/test/Transforms/InstCombine/unsigned-add-lack-of-overflow-check-via-xor.ll
 create mode 100644 llvm/test/Transforms/InstCombine/unsigned-add-lack-of-overflow-check.ll
 create mode 100644 llvm/test/Transforms/InstCombine/unsigned-add-overflow-check-via-add.ll
 create mode 100644 llvm/test/Transforms/InstCombine/unsigned-add-overflow-check-via-xor.ll
 create mode 100644 llvm/test/Transforms/InstCombine/unsigned-add-overflow-check.ll
 create mode 100644 llvm/test/Transforms/InstCombine/unsigned-mul-lack-of-overflow-check-via-mul-udiv.ll
 create mode 100644 llvm/test/Transforms/InstCombine/unsigned-mul-lack-of-overflow-check-via-udiv-of-allones.ll
 create mode 100644 llvm/test/Transforms/InstCombine/unsigned-mul-overflow-check-via-mul-udiv.ll
 create mode 100644 llvm/test/Transforms/InstCombine/unsigned-mul-overflow-check-via-udiv-of-allones.ll
 create mode 100644 llvm/test/Transforms/InstCombine/unsigned-sub-lack-of-overflow-check.ll
 create mode 100644 llvm/test/Transforms/InstCombine/unsigned-sub-overflow-check.ll
 create mode 100644 llvm/test/Transforms/InstCombine/unused-nonnull.ll
 create mode 100644 llvm/test/Transforms/InstCombine/urem-via-udiv-mul-sub.ll
 create mode 100644 llvm/test/Transforms/InstCombine/variable-signext-of-variable-high-bit-extraction.ll
 create mode 100644 llvm/test/Transforms/InstCombine/vec_udiv_to_shift.ll
 create mode 100644 llvm/test/Transforms/InstCombine/widenable-conditions.ll
 create mode 100644 llvm/test/Transforms/InstCombine/xor-of-icmps-with-extra-uses.ll
 create mode 100644 llvm/test/Transforms/InstMerge/st_sink_split_bb.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/assume-non-zero.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/div-by-0-guard-before-smul_ov-not.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/div-by-0-guard-before-smul_ov.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/div-by-0-guard-before-umul_ov-not.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/div-by-0-guard-before-umul_ov.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/freeze.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/redundant-null-check-in-uadd_with_overflow-of-nonnull-ptr.ll
 create mode 100755 llvm/test/Transforms/InstSimplify/remove-dead-call.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/result-of-add-of-negative-is-non-zero-and-no-underflow.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/result-of-usub-by-nonzero-is-non-zero-and-no-overflow.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/result-of-usub-is-non-zero-and-no-overflow.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/unsigned-range-checks.ll
 create mode 100644 llvm/test/Transforms/Internalize/vcall-visibility.ll
 create mode 100644 llvm/test/Transforms/JumpThreading/unreachable-loops.ll
 create mode 100644 llvm/test/Transforms/LCSSA/pr44058.ll
 create mode 100644 llvm/test/Transforms/LICM/AMDGPU/bitcast.ll
 create mode 100644 llvm/test/Transforms/LICM/callbr-crash.ll
 create mode 100644 llvm/test/Transforms/LICM/pr38513.ll
 create mode 100644 llvm/test/Transforms/LICM/pr42969.ll
 create mode 100644 llvm/test/Transforms/LICM/pragma-licm-disable.ll
 create mode 100644 llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/vect-ptr-ptr-size-mismatch.ll
 create mode 100644 llvm/test/Transforms/LoopFusion/diagnostics_analysis.ll
 create mode 100644 llvm/test/Transforms/LoopFusion/diagnostics_missed.ll
 create mode 100644 llvm/test/Transforms/LoopFusion/guarded.ll
 create mode 100644 llvm/test/Transforms/LoopInterchange/pr43176-move-to-new-latch.ll
 create mode 100644 llvm/test/Transforms/LoopInterchange/pr43473-invalid-lcssa-phis-in-inner-exit.ll
 create mode 100644 llvm/test/Transforms/LoopInterchange/pr43797-lcssa-for-multiple-outer-loop-blocks.ll
 create mode 100644 llvm/test/Transforms/LoopInterchange/update-condbranch-duplicate-successors.ll
 create mode 100644 llvm/test/Transforms/LoopPredication/predicate-exits.ll
 create mode 100644 llvm/test/Transforms/LoopRotate/dbg-value-duplicates-2.ll
 create mode 100644 llvm/test/Transforms/LoopRotate/switch.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/ARM/dont-unroll-loopdec.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/ARM/mve-nounroll.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/disable-full-unroll-by-opt.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/optsize-loop-size.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/peel-loop-conditions-pgo-1.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/peel-loop-conditions-pgo-2.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/peel-loop-inner.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/peel-loop-pgo-deopt-idom-2.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/runtime-small-upperbound.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/unroll-preserve-scev-lcssa.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/ARM/mve-interleaved-cost.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/ARM/mve-maskedldst.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/ARM/mve-shiftcost.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/ARM/mve-vldn.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/ARM/prefer-tail-loop-folding.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/ARM/tail-folding-counting-down.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/ARM/tail-loop-folding.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/PowerPC/reg-usage.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/PowerPC/vectorize-bswap.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/SystemZ/predicated-first-order-recurrence.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/cost-model-assert.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/interleave_short_tc.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/interleaved-accesses-waw-dependency.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/load-deref-pred.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/pr42674.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/tail_folding_and_assume_safety.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/tail_loop_folding.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/first-order-recurrence-complex.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/first-order-recurrence-multiply-recurrences.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/interleaved-accesses-uniform-load.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/nofloat-report.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/nuw.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/pr44488-predication.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/tail-folding-counting-down.ll
 create mode 100644 llvm/test/Transforms/LowerConstantIntrinsics/constant-intrinsics.ll
 create mode 100644 llvm/test/Transforms/LowerConstantIntrinsics/crash-on-large-allocas.ll
 create mode 100644 llvm/test/Transforms/LowerConstantIntrinsics/objectsize_basic.ll
 create mode 100644 llvm/test/Transforms/LowerMatrixIntrinsics/bigger-expressions-double.ll
 create mode 100644 llvm/test/Transforms/LowerMatrixIntrinsics/multiply-double-contraction-fmf.ll
 create mode 100644 llvm/test/Transforms/LowerMatrixIntrinsics/multiply-double-contraction.ll
 create mode 100644 llvm/test/Transforms/LowerMatrixIntrinsics/multiply-double.ll
 create mode 100644 llvm/test/Transforms/LowerMatrixIntrinsics/multiply-float-contraction-fmf.ll
 create mode 100644 llvm/test/Transforms/LowerMatrixIntrinsics/multiply-float-contraction.ll
 create mode 100644 llvm/test/Transforms/LowerMatrixIntrinsics/multiply-float.ll
 create mode 100644 llvm/test/Transforms/LowerMatrixIntrinsics/multiply-i32.ll
 create mode 100644 llvm/test/Transforms/LowerMatrixIntrinsics/propagate-backward.ll
 create mode 100644 llvm/test/Transforms/LowerMatrixIntrinsics/propagate-backwards-unsupported.ll
 create mode 100644 llvm/test/Transforms/LowerMatrixIntrinsics/propagate-forward.ll
 create mode 100644 llvm/test/Transforms/LowerMatrixIntrinsics/propagate-mixed-users.ll
 create mode 100644 llvm/test/Transforms/LowerMatrixIntrinsics/propagate-multiple-iterations.ll
 create mode 100644 llvm/test/Transforms/LowerMatrixIntrinsics/strided-load-double.ll
 create mode 100644 llvm/test/Transforms/LowerMatrixIntrinsics/strided-load-float.ll
 create mode 100644 llvm/test/Transforms/LowerMatrixIntrinsics/strided-load-i32.ll
 create mode 100644 llvm/test/Transforms/LowerMatrixIntrinsics/strided-store-double.ll
 create mode 100644 llvm/test/Transforms/LowerMatrixIntrinsics/strided-store-float.ll
 create mode 100644 llvm/test/Transforms/LowerMatrixIntrinsics/strided-store-i32.ll
 create mode 100644 llvm/test/Transforms/LowerMatrixIntrinsics/transpose-double.ll
 create mode 100644 llvm/test/Transforms/LowerMatrixIntrinsics/transpose-float.ll
 create mode 100644 llvm/test/Transforms/LowerMatrixIntrinsics/transpose-i32.ll
 create mode 100644 llvm/test/Transforms/LowerTypeTests/align.ll
 create mode 100644 llvm/test/Transforms/LowerTypeTests/export-rename-local.ll
 create mode 100644 llvm/test/Transforms/MemCpyOpt/aggregate-type-crash.ll
 create mode 100644 llvm/test/Transforms/MemCpyOpt/store-to-memset.ll
 create mode 100644 llvm/test/Transforms/MergeFunc/byval-attr-congruent-type.ll
 create mode 100644 llvm/test/Transforms/NewGVN/pr43441.ll
 create mode 100644 llvm/test/Transforms/ObjCARC/code-motion.ll
 create mode 100644 llvm/test/Transforms/ObjCARC/inlined-autorelease-return-value.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/misexpect-branch-correct.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/misexpect-branch-stripped.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/misexpect-branch-unpredictable.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/misexpect-branch.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/misexpect-switch-default.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/misexpect-switch.ll
 create mode 100644 llvm/test/Transforms/PhaseOrdering/lifetime-sanitizer.ll
 create mode 100644 llvm/test/Transforms/PhaseOrdering/min-max-abs-cse.ll
 create mode 100644 llvm/test/Transforms/PhaseOrdering/two-shifts-by-sext.ll
 create mode 100644 llvm/test/Transforms/PhaseOrdering/unsigned-multiply-overflow-check.ll
 create mode 100644 llvm/test/Transforms/Reassociate/2019-08-22-FNegAssert.ll
 create mode 100644 llvm/test/Transforms/Reassociate/reassociate_salvages_debug_info.ll
 create mode 100644 llvm/test/Transforms/Reassociate/undef_intrinsics_when_deleting_instructions.ll
 create mode 100644 llvm/test/Transforms/SCCP/ipsccp-predinfo-order.ll
 create mode 100644 llvm/test/Transforms/SCCP/struct-arg-resolve-undefs.ll
 create mode 100644 llvm/test/Transforms/SCCP/ubsan_overflow.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/PR31847.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/bad-reduction.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/different-vec-widths.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/jumbled_store_crash.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/no_alternate_divrem.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/pr42022.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/reuse-extracts-in-wider-vect.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/used-reduced-op.ll
 create mode 100644 llvm/test/Transforms/SROA/tbaa-struct.ll
 create mode 100644 llvm/test/Transforms/SROA/vector-promotion-different-size.ll
 create mode 100644 llvm/test/Transforms/SampleProfile/Inputs/profile-symbol-list.ll
 create mode 100644 llvm/test/Transforms/SampleProfile/compressed-profile-symbol-list.ll
 create mode 100644 llvm/test/Transforms/SampleProfile/inline-callee-update.ll
 create mode 100644 llvm/test/Transforms/SampleProfile/inline-cold.ll
 create mode 100644 llvm/test/Transforms/SampleProfile/inline-mergeprof.ll
 create mode 100644 llvm/test/Transforms/SampleProfile/inline-stats.ll
 create mode 100644 llvm/test/Transforms/SampleProfile/inline-topdown.ll
 create mode 100644 llvm/test/Transforms/SampleProfile/profile-format-compress.ll
 create mode 100644 llvm/test/Transforms/SampleProfile/profile-format.ll
 create mode 100644 llvm/test/Transforms/SampleProfile/profile-sample-accurate.ll
 create mode 100644 llvm/test/Transforms/SampleProfile/uncompressed-profile-symbol-list.ll
 create mode 100644 llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-scatter.ll
 create mode 100644 llvm/test/Transforms/Scalarizer/phi-unreachable-pred.ll
 create mode 100644 llvm/test/Transforms/SimpleLoopUnswitch/preserve-scev-exiting-multiple-loops.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/X86/merge-cond-stores-cost.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/invalidate-dom.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/safe-abs.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/safe-low-bit-extract.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/signbit-like-value-extension.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/switch_msan.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/unsigned-multiplication-will-overflow.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/wc-widen-block.ll
 create mode 100644 llvm/test/Transforms/ThinLTOBitcodeWriter/associated.ll
 create mode 100644 llvm/test/Transforms/ThinLTOBitcodeWriter/cfi-functions-canonical-jump-tables.ll
 create mode 100644 llvm/test/Transforms/TypePromotion/ARM/calls.ll
 create mode 100644 llvm/test/Transforms/TypePromotion/ARM/casts.ll
 create mode 100644 llvm/test/Transforms/TypePromotion/ARM/clear-structures.ll
 create mode 100644 llvm/test/Transforms/TypePromotion/ARM/icmps.ll
 create mode 100644 llvm/test/Transforms/TypePromotion/ARM/large-int.ll
 create mode 100644 llvm/test/Transforms/TypePromotion/ARM/phis-ret.ll
 create mode 100644 llvm/test/Transforms/TypePromotion/ARM/pointers.ll
 create mode 100644 llvm/test/Transforms/TypePromotion/ARM/signed-icmps.ll
 create mode 100644 llvm/test/Transforms/TypePromotion/ARM/signed.ll
 create mode 100644 llvm/test/Transforms/TypePromotion/ARM/switch.ll
 create mode 100644 llvm/test/Transforms/TypePromotion/ARM/wrapping.ll
 create mode 100644 llvm/test/Transforms/Util/add-TLI-mappings.ll
 create mode 100644 llvm/test/Transforms/Util/dbg-call-bitcast.ll
 create mode 100644 llvm/test/Verifier/callbr.ll
 create mode 100644 llvm/test/Verifier/dereferenceable-md-inttoptr.ll
 create mode 100644 llvm/test/Verifier/diexpression-dwarf-entry-value.ll
 create mode 100644 llvm/test/Verifier/diexpression-entry-value-llvm-ir.ll
 create mode 100644 llvm/test/Verifier/intrinsic-bad-arg-type.ll
 create mode 100644 llvm/test/Verifier/invalid-frame-pointer-attr-empty.ll
 create mode 100644 llvm/test/Verifier/invalid-frame-pointer-attr-no-value.ll
 create mode 100644 llvm/test/Verifier/invalid-frame-pointer-attr.ll
 create mode 100644 llvm/test/Verifier/invalid-patchable-function-entry.ll
 create mode 100644 llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/basic.ll
 create mode 100644 llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/argument_name_reuse.ll
 create mode 100644 llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/basic.ll
 create mode 100644 llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/scrub_attrs.ll
 create mode 100644 llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/sometimes_deleted_function.ll
 create mode 100644 llvm/test/tools/gold/X86/slp-vectorize-pm.ll
 create mode 100644 llvm/test/tools/llvm-dwarfdump/X86/locstats.ll
 create mode 100644 llvm/test/tools/llvm-dwarfdump/X86/stats-dbg-callsite-info.ll
 create mode 100644 llvm/test/tools/llvm-dwarfdump/X86/valid-call-site-GNU-extensions.ll
 create mode 100644 llvm/test/tools/llvm-locstats/locstats.ll
 create mode 100644 llvm/test/tools/llvm-locstats/no_scope_bytes.ll
 create mode 100644 llvm/test/tools/llvm-lto2/X86/slp-vectorize-pm.ll
 create mode 100644 llvm/test/tools/llvm-objdump/X86/macho-data-in-code.ll
 create mode 100644 llvm/test/tools/llvm-readobj/ELF/Inputs/trivial.ll

diff --git a/llvm/Makefile b/llvm/Makefile
index 86e5208e9..e9ab6e4af 100644
--- a/llvm/Makefile
+++ b/llvm/Makefile
@@ -1,5 +1,5 @@
 # LLVM version.
-VER=9.0.0
+VER=10.0.0
 
 ROOT_DIR=$(shell pwd)
 
@@ -21,7 +21,7 @@ llvm-${VER}.src: | llvm-${VER}.src.tar.xz
 	tar -xJf llvm-${VER}.src.tar.xz
 
 llvm-${VER}.src.tar.xz:
-	wget -O $@ "https://releases.llvm.org/${VER}/llvm-${VER}.src.tar.xz"
+	wget -O $@ "https://github.com/llvm/llvm-project/releases/download/llvmorg-${VER}/llvm-${VER}.src.tar.xz"
 	touch $@
 
 clean:
diff --git a/llvm/test/Analysis/BasicAA/assume-index-positive.ll b/llvm/test/Analysis/BasicAA/assume-index-positive.ll
new file mode 100644
index 000000000..d89738a23
--- /dev/null
+++ b/llvm/test/Analysis/BasicAA/assume-index-positive.ll
@@ -0,0 +1,116 @@
+; RUN: opt -basicaa -aa-eval -print-all-alias-modref-info %s 2>&1 | FileCheck %s
+
+; %col.ptr.1 and %col.ptr.2 do not alias, if we know that %skip >= 0, because
+; the distance between %col.ptr.1 and %col.ptr.2 is %skip + 6 and we load 6
+; elements.
+define void @test1(double* %ptr, i32 %skip) {
+; CHECK-LABEL: Function: test1: 4 pointers, 1 call sites
+; CHECK-NEXT:  MustAlias:   <6 x double>* %col.ptr.1, double* %ptr
+; CHECK-NEXT:  NoAlias: double* %col.ptr.2, double* %ptr
+; CHECK-NEXT:  NoAlias: <6 x double>* %col.ptr.1, double* %col.ptr.2
+; CHECK-NEXT:  NoAlias: <6 x double>* %col.ptr.2.cast, double* %ptr
+; CHECK-NEXT:  NoAlias: <6 x double>* %col.ptr.1, <6 x double>* %col.ptr.2.cast
+; CHECK-NEXT:  MustAlias:   <6 x double>* %col.ptr.2.cast, double* %col.ptr.2
+; CHECK-NEXT:  NoModRef:  Ptr: double* %ptr <->  call void @llvm.assume(i1 %gt)
+; CHECK-NEXT:  NoModRef:  Ptr: <6 x double>* %col.ptr.1 <->  call void @llvm.assume(i1 %gt)
+; CHECK-NEXT:  NoModRef:  Ptr: double* %col.ptr.2   <->  call void @llvm.assume(i1 %gt)
+; CHECK-NEXT:  NoModRef:  Ptr: <6 x double>* %col.ptr.2.cast    <->  call void @llvm.assume(i1 %gt)
+;
+  %gt = icmp sgt i32 %skip, -1
+  call void @llvm.assume(i1 %gt)
+  %stride = add nsw nuw i32 %skip, 6
+  %col.ptr.1 = bitcast double* %ptr to <6 x double>*
+  %lv.1 = load <6 x double>, <6 x double>* %col.ptr.1, align 8
+  %col.ptr.2= getelementptr double, double* %ptr, i32 %stride
+  %col.ptr.2.cast = bitcast double* %col.ptr.2 to <6 x double>*
+  %lv.2 = load <6 x double>, <6 x double>* %col.ptr.2.cast, align 8
+  %res.1 = fadd <6 x double> %lv.1, %lv.1
+  %res.2 = fadd <6 x double> %lv.2, %lv.2
+  store <6 x double> %res.1, <6 x double>* %col.ptr.1, align 8
+  store <6 x double> %res.2, <6 x double>* %col.ptr.2.cast, align 8
+  ret void
+}
+
+; Same as @test1, but now we do not have an assume guaranteeing %skip >= 0.
+define void @test2(double* %ptr, i32 %skip) {
+; CHECK-LABEL: Function: test2: 4 pointers, 0 call sites
+; CHECK-NEXT:  MustAlias:   <6 x double>* %col.ptr.1, double* %ptr
+; CHECK-NEXT:  MayAlias:    double* %col.ptr.2, double* %ptr
+; CHECK-NEXT:  MayAlias:    <6 x double>* %col.ptr.1, double* %col.ptr.2
+; CHECK-NEXT:  MayAlias:    <6 x double>* %col.ptr.2.cast, double* %ptr
+; CHECK-NEXT:  MayAlias:    <6 x double>* %col.ptr.1, <6 x double>* %col.ptr.2.cast
+; CHECK-NEXT:  MustAlias:   <6 x double>* %col.ptr.2.cast, double* %col.ptr.2
+;
+  %stride = add nsw nuw i32 %skip, 6
+  %col.ptr.1 = bitcast double* %ptr to <6 x double>*
+  %lv.1 = load <6 x double>, <6 x double>* %col.ptr.1, align 8
+  %col.ptr.2= getelementptr double, double* %ptr, i32 %stride
+  %col.ptr.2.cast = bitcast double* %col.ptr.2 to <6 x double>*
+  %lv.2 = load <6 x double>, <6 x double>* %col.ptr.2.cast, align 8
+  %res.1 = fadd <6 x double> %lv.1, %lv.1
+  %res.2 = fadd <6 x double> %lv.2, %lv.2
+  store <6 x double> %res.1, <6 x double>* %col.ptr.1, align 8
+  store <6 x double> %res.2, <6 x double>* %col.ptr.2.cast, align 8
+  ret void
+}
+
+; Same as @test1, but the assume just guarantees %skip > -3, which is not
+; enough to derive NoAlias
+define void @test3(double* %ptr, i32 %skip) {
+; CHECK-LABEL: Function: test3: 4 pointers, 1 call sites
+; CHECK-NEXT:  MustAlias:   <6 x double>* %col.ptr.1, double* %ptr
+; CHECK-NEXT:  MayAlias:    double* %col.ptr.2, double* %ptr
+; CHECK-NEXT:  MayAlias:    <6 x double>* %col.ptr.1, double* %col.ptr.2
+; CHECK-NEXT:  MayAlias:    <6 x double>* %col.ptr.2.cast, double* %ptr
+; CHECK-NEXT:  MayAlias:    <6 x double>* %col.ptr.1, <6 x double>* %col.ptr.2.cast
+; CHECK-NEXT:  MustAlias:   <6 x double>* %col.ptr.2.cast, double* %col.ptr.2
+; CHECK-NEXT:  NoModRef:  Ptr: double* %ptr <->  call void @llvm.assume(i1 %gt)
+; CHECK-NEXT:  NoModRef:  Ptr: <6 x double>* %col.ptr.1 <->  call void @llvm.assume(i1 %gt)
+; CHECK-NEXT:  NoModRef:  Ptr: double* %col.ptr.2   <->  call void @llvm.assume(i1 %gt)
+; CHECK-NEXT:  NoModRef:  Ptr: <6 x double>* %col.ptr.2.cast    <->  call void @llvm.assume(i1 %gt)
+;
+  %gt = icmp sgt i32 %skip, -3
+  call void @llvm.assume(i1 %gt)
+  %stride = add nsw nuw i32 %skip, 6
+  %col.ptr.1 = bitcast double* %ptr to <6 x double>*
+  %lv.1 = load <6 x double>, <6 x double>* %col.ptr.1, align 8
+  %col.ptr.2= getelementptr double, double* %ptr, i32 %stride
+  %col.ptr.2.cast = bitcast double* %col.ptr.2 to <6 x double>*
+  %lv.2 = load <6 x double>, <6 x double>* %col.ptr.2.cast, align 8
+  %res.1 = fadd <6 x double> %lv.1, %lv.1
+  %res.2 = fadd <6 x double> %lv.2, %lv.2
+  store <6 x double> %res.1, <6 x double>* %col.ptr.1, align 8
+  store <6 x double> %res.2, <6 x double>* %col.ptr.2.cast, align 8
+  ret void
+}
+
+; Same as @test1, but the assume uses the sge predicate for %skip >= 0.
+define void @test4(double* %ptr, i32 %skip) {
+; CHECK-LABEL: Function: test4: 4 pointers, 1 call sites
+; CHECK-NEXT:  MustAlias:   <6 x double>* %col.ptr.1, double* %ptr
+; CHECK-NEXT:  NoAlias:     double* %col.ptr.2, double* %ptr
+; CHECK-NEXT:  NoAlias:     <6 x double>* %col.ptr.1, double* %col.ptr.2
+; CHECK-NEXT:  NoAlias:     <6 x double>* %col.ptr.2.cast, double* %ptr
+; CHECK-NEXT:  NoAlias:     <6 x double>* %col.ptr.1, <6 x double>* %col.ptr.2.cast
+; CHECK-NEXT:  MustAlias:   <6 x double>* %col.ptr.2.cast, double* %col.ptr.2
+; CHECK-NEXT:  NoModRef:  Ptr: double* %ptr <->  call void @llvm.assume(i1 %gt)
+; CHECK-NEXT:  NoModRef:  Ptr: <6 x double>* %col.ptr.1 <->  call void @llvm.assume(i1 %gt)
+; CHECK-NEXT:  NoModRef:  Ptr: double* %col.ptr.2   <->  call void @llvm.assume(i1 %gt)
+; CHECK-NEXT:  NoModRef:  Ptr: <6 x double>* %col.ptr.2.cast    <->  call void @llvm.assume(i1 %gt)
+;
+  %gt = icmp sge i32 %skip, 0
+  call void @llvm.assume(i1 %gt)
+  %stride = add nsw nuw i32 %skip, 6
+  %col.ptr.1 = bitcast double* %ptr to <6 x double>*
+  %lv.1 = load <6 x double>, <6 x double>* %col.ptr.1, align 8
+  %col.ptr.2= getelementptr double, double* %ptr, i32 %stride
+  %col.ptr.2.cast = bitcast double* %col.ptr.2 to <6 x double>*
+  %lv.2 = load <6 x double>, <6 x double>* %col.ptr.2.cast, align 8
+  %res.1 = fadd <6 x double> %lv.1, %lv.1
+  %res.2 = fadd <6 x double> %lv.2, %lv.2
+  store <6 x double> %res.1, <6 x double>* %col.ptr.1, align 8
+  store <6 x double> %res.2, <6 x double>* %col.ptr.2.cast, align 8
+  ret void
+}
+
+declare void @llvm.assume(i1 %cond)
diff --git a/llvm/test/Analysis/BasicAA/cs-cs.ll b/llvm/test/Analysis/BasicAA/cs-cs.ll
index 98899993d..beb9eaa83 100644
--- a/llvm/test/Analysis/BasicAA/cs-cs.ll
+++ b/llvm/test/Analysis/BasicAA/cs-cs.ll
@@ -364,26 +364,26 @@ entry:
   call void @an_argmemonly_func(i8* %q) #9 [ "unknown"() ]
   ret void
 
-; CHECK: Just Ref:  Ptr: i8* %p        <->  call void @a_readonly_func(i8* %p) #6 [ "unknown"() ]
-; CHECK: Just Ref:  Ptr: i8* %q        <->  call void @a_readonly_func(i8* %p) #6 [ "unknown"() ]
-; CHECK: NoModRef:  Ptr: i8* %p        <->  call void @an_inaccessiblememonly_func() #7 [ "unknown"() ]
-; CHECK: NoModRef:  Ptr: i8* %q        <->  call void @an_inaccessiblememonly_func() #7 [ "unknown"() ]
-; CHECK: NoModRef:  Ptr: i8* %p        <->  call void @an_inaccessibleorargmemonly_func(i8* %q) #8 [ "unknown"() ]
-; CHECK: Both ModRef (MustAlias):  Ptr: i8* %q     <->  call void @an_inaccessibleorargmemonly_func(i8* %q) #8 [ "unknown"() ]
-; CHECK: NoModRef:  Ptr: i8* %p        <->  call void @an_argmemonly_func(i8* %q) #9 [ "unknown"() ]
-; CHECK: Both ModRef (MustAlias):  Ptr: i8* %q     <->  call void @an_argmemonly_func(i8* %q) #9 [ "unknown"() ]
-; CHECK: Just Ref:   call void @a_readonly_func(i8* %p) #6 [ "unknown"() ] <->   call void @an_inaccessiblememonly_func() #7 [ "unknown"() ]
-; CHECK: Just Ref:   call void @a_readonly_func(i8* %p) #6 [ "unknown"() ] <->   call void @an_inaccessibleorargmemonly_func(i8* %q) #8 [ "unknown"() ]
-; CHECK: Just Ref:   call void @a_readonly_func(i8* %p) #6 [ "unknown"() ] <->   call void @an_argmemonly_func(i8* %q) #9 [ "unknown"() ]
-; CHECK: Both ModRef:   call void @an_inaccessiblememonly_func() #7 [ "unknown"() ] <->   call void @a_readonly_func(i8* %p) #6 [ "unknown"() ]
-; CHECK: Both ModRef:   call void @an_inaccessiblememonly_func() #7 [ "unknown"() ] <->   call void @an_inaccessibleorargmemonly_func(i8* %q) #8 [ "unknown"() ]
-; CHECK: NoModRef:   call void @an_inaccessiblememonly_func() #7 [ "unknown"() ] <->   call void @an_argmemonly_func(i8* %q) #9 [ "unknown"() ]
-; CHECK: Both ModRef:   call void @an_inaccessibleorargmemonly_func(i8* %q) #8 [ "unknown"() ] <->   call void @a_readonly_func(i8* %p) #6 [ "unknown"() ]
-; CHECK: Both ModRef:   call void @an_inaccessibleorargmemonly_func(i8* %q) #8 [ "unknown"() ] <->   call void @an_inaccessiblememonly_func() #7 [ "unknown"() ]
-; CHECK: Both ModRef (MustAlias):   call void @an_inaccessibleorargmemonly_func(i8* %q) #8 [ "unknown"() ] <->   call void @an_argmemonly_func(i8* %q) #9 [ "unknown"() ]
-; CHECK: Both ModRef:   call void @an_argmemonly_func(i8* %q) #9 [ "unknown"() ] <->   call void @a_readonly_func(i8* %p) #6 [ "unknown"() ]
-; CHECK: NoModRef:   call void @an_argmemonly_func(i8* %q) #9 [ "unknown"() ] <->   call void @an_inaccessiblememonly_func() #7 [ "unknown"() ]
-; CHECK: Both ModRef (MustAlias):   call void @an_argmemonly_func(i8* %q) #9 [ "unknown"() ] <->   call void @an_inaccessibleorargmemonly_func(i8* %q) #8 [ "unknown"() ]
+; CHECK: Just Ref:  Ptr: i8* %p        <->  call void @a_readonly_func(i8* %p) #7 [ "unknown"() ]
+; CHECK: Just Ref:  Ptr: i8* %q        <->  call void @a_readonly_func(i8* %p) #7 [ "unknown"() ]
+; CHECK: NoModRef:  Ptr: i8* %p        <->  call void @an_inaccessiblememonly_func() #8 [ "unknown"() ]
+; CHECK: NoModRef:  Ptr: i8* %q        <->  call void @an_inaccessiblememonly_func() #8 [ "unknown"() ]
+; CHECK: NoModRef:  Ptr: i8* %p        <->  call void @an_inaccessibleorargmemonly_func(i8* %q) #9 [ "unknown"() ]
+; CHECK: Both ModRef (MustAlias):  Ptr: i8* %q     <->  call void @an_inaccessibleorargmemonly_func(i8* %q) #9 [ "unknown"() ]
+; CHECK: NoModRef:  Ptr: i8* %p        <->  call void @an_argmemonly_func(i8* %q) #10 [ "unknown"() ]
+; CHECK: Both ModRef (MustAlias):  Ptr: i8* %q     <->  call void @an_argmemonly_func(i8* %q) #10 [ "unknown"() ]
+; CHECK: Just Ref:   call void @a_readonly_func(i8* %p) #7 [ "unknown"() ] <->   call void @an_inaccessiblememonly_func() #8 [ "unknown"() ]
+; CHECK: Just Ref:   call void @a_readonly_func(i8* %p) #7 [ "unknown"() ] <->   call void @an_inaccessibleorargmemonly_func(i8* %q) #9 [ "unknown"() ]
+; CHECK: Just Ref:   call void @a_readonly_func(i8* %p) #7 [ "unknown"() ] <->   call void @an_argmemonly_func(i8* %q) #10 [ "unknown"() ]
+; CHECK: Both ModRef:   call void @an_inaccessiblememonly_func() #8 [ "unknown"() ] <->   call void @a_readonly_func(i8* %p) #7 [ "unknown"() ]
+; CHECK: Both ModRef:   call void @an_inaccessiblememonly_func() #8 [ "unknown"() ] <->   call void @an_inaccessibleorargmemonly_func(i8* %q) #9 [ "unknown"() ]
+; CHECK: NoModRef:   call void @an_inaccessiblememonly_func() #8 [ "unknown"() ] <->   call void @an_argmemonly_func(i8* %q) #10 [ "unknown"() ]
+; CHECK: Both ModRef:   call void @an_inaccessibleorargmemonly_func(i8* %q) #9 [ "unknown"() ] <->   call void @a_readonly_func(i8* %p) #7 [ "unknown"() ]
+; CHECK: Both ModRef:   call void @an_inaccessibleorargmemonly_func(i8* %q) #9 [ "unknown"() ] <->   call void @an_inaccessiblememonly_func() #8 [ "unknown"() ]
+; CHECK: Both ModRef (MustAlias):   call void @an_inaccessibleorargmemonly_func(i8* %q) #9 [ "unknown"() ] <->   call void @an_argmemonly_func(i8* %q) #10 [ "unknown"() ]
+; CHECK: Both ModRef:   call void @an_argmemonly_func(i8* %q) #10 [ "unknown"() ] <->   call void @a_readonly_func(i8* %p) #7 [ "unknown"() ]
+; CHECK: NoModRef:   call void @an_argmemonly_func(i8* %q) #10 [ "unknown"() ] <->   call void @an_inaccessiblememonly_func() #8 [ "unknown"() ]
+; CHECK: Both ModRef (MustAlias):   call void @an_argmemonly_func(i8* %q) #10 [ "unknown"() ] <->   call void @an_inaccessibleorargmemonly_func(i8* %q) #9 [ "unknown"() ]
 }
 
 attributes #0 = { argmemonly nounwind }
diff --git a/llvm/test/Analysis/BasicAA/dereferenceable.ll b/llvm/test/Analysis/BasicAA/dereferenceable.ll
new file mode 100644
index 000000000..efc9addbe
--- /dev/null
+++ b/llvm/test/Analysis/BasicAA/dereferenceable.ll
@@ -0,0 +1,149 @@
+; RUN: opt -basicaa -print-all-alias-modref-info -aa-eval -analyze < %s 2>&1 | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+@G = global i32 0, align 4
+
+define i64 @global_and_deref_arg_1(i64* dereferenceable(8) %arg) {
+; CHECK:     Function: global_and_deref_arg_1: 2 pointers, 0 call sites
+; CHECK-NEXT:  NoAlias:	i32* @G, i64* %arg
+bb:
+  store i64 1, i64* %arg, align 8
+  store i32 0, i32* @G, align 4
+  %tmp = load i64, i64* %arg, align 8
+  ret i64 %tmp
+}
+
+define i32 @global_and_deref_arg_2(i32* dereferenceable(8) %arg) {
+; CHECK:     Function: global_and_deref_arg_2: 2 pointers, 0 call sites
+; CHECK-NEXT:  NoAlias:	i32* %arg, i32* @G
+bb:
+  store i32 1, i32* %arg, align 8
+  store i32 0, i32* @G, align 4
+  %tmp = load i32, i32* %arg, align 8
+  ret i32 %tmp
+}
+
+define i32 @byval_and_deref_arg_1(i32* byval %obj, i64* dereferenceable(8) %arg) {
+; CHECK:     Function: byval_and_deref_arg_1: 2 pointers, 0 call sites
+; CHECK-NEXT:  NoAlias:	i32* %obj, i64* %arg
+bb:
+  store i32 1, i32* %obj, align 4
+  store i64 0, i64* %arg, align 8
+  %tmp = load i32, i32* %obj, align 4
+  ret i32 %tmp
+}
+
+define i32 @byval_and_deref_arg_2(i32* byval %obj, i32* dereferenceable(8) %arg) {
+; CHECK:     Function: byval_and_deref_arg_2: 2 pointers, 0 call sites
+; CHECK-NEXT:  NoAlias:	i32* %arg, i32* %obj
+bb:
+  store i32 1, i32* %obj, align 4
+  store i32 0, i32* %arg, align 8
+  %tmp = load i32, i32* %obj, align 4
+  ret i32 %tmp
+}
+
+declare dereferenceable(8) i32* @get_i32_deref8()
+declare dereferenceable(8) i64* @get_i64_deref8()
+declare void @unknown(i32*)
+
+define i32 @local_and_deref_ret_1() {
+; CHECK:     Function: local_and_deref_ret_1: 2 pointers, 2 call sites
+; CHECK-NEXT:  NoAlias:	i32* %obj, i64* %ret
+bb:
+  %obj = alloca i32
+  call void @unknown(i32* %obj)
+  %ret = call dereferenceable(8) i64* @get_i64_deref8()
+  store i32 1, i32* %obj, align 4
+  store i64 0, i64* %ret, align 8
+  %tmp = load i32, i32* %obj, align 4
+  ret i32 %tmp
+}
+
+define i32 @local_and_deref_ret_2() {
+; CHECK:     Function: local_and_deref_ret_2: 2 pointers, 2 call sites
+; CHECK-NEXT:  NoAlias:	i32* %obj, i32* %ret
+bb:
+  %obj = alloca i32
+  call void @unknown(i32* %obj)
+  %ret = call dereferenceable(8) i32* @get_i32_deref8()
+  store i32 1, i32* %obj, align 4
+  store i32 0, i32* %ret, align 8
+  %tmp = load i32, i32* %obj, align 4
+  ret i32 %tmp
+}
+
+
+; Baseline tests, same as above but with 2 instead of 8 dereferenceable bytes.
+
+define i64 @global_and_deref_arg_non_deref_1(i64* dereferenceable(2) %arg) {
+; CHECK:     Function: global_and_deref_arg_non_deref_1: 2 pointers, 0 call sites
+; CHECK-NEXT:  NoAlias:	i32* @G, i64* %arg
+bb:
+  store i64 1, i64* %arg, align 8
+  store i32 0, i32* @G, align 4
+  %tmp = load i64, i64* %arg, align 8
+  ret i64 %tmp
+}
+
+define i32 @global_and_deref_arg_non_deref_2(i32* dereferenceable(2) %arg) {
+; CHECK:     Function: global_and_deref_arg_non_deref_2: 2 pointers, 0 call sites
+; Different result than above (see @global_and_deref_arg_2).
+; CHECK-NEXT:  MayAlias:	i32* %arg, i32* @G
+bb:
+  store i32 1, i32* %arg, align 8
+  store i32 0, i32* @G, align 4
+  %tmp = load i32, i32* %arg, align 8
+  ret i32 %tmp
+}
+
+define i32 @byval_and_deref_arg_non_deref_1(i32* byval %obj, i64* dereferenceable(2) %arg) {
+; CHECK:     Function: byval_and_deref_arg_non_deref_1: 2 pointers, 0 call sites
+; CHECK-NEXT:  NoAlias:	i32* %obj, i64* %arg
+bb:
+  store i32 1, i32* %obj, align 4
+  store i64 0, i64* %arg, align 8
+  %tmp = load i32, i32* %obj, align 4
+  ret i32 %tmp
+}
+
+define i32 @byval_and_deref_arg_non_deref_2(i32* byval %obj, i32* dereferenceable(2) %arg) {
+; CHECK:     Function: byval_and_deref_arg_non_deref_2: 2 pointers, 0 call sites
+; CHECK-NEXT:  NoAlias:	i32* %arg, i32* %obj
+bb:
+  store i32 1, i32* %obj, align 4
+  store i32 0, i32* %arg, align 8
+  %tmp = load i32, i32* %obj, align 4
+  ret i32 %tmp
+}
+
+declare dereferenceable(2) i32* @get_i32_deref2()
+declare dereferenceable(2) i64* @get_i64_deref2()
+
+define i32 @local_and_deref_ret_non_deref_1() {
+; CHECK:     Function: local_and_deref_ret_non_deref_1: 2 pointers, 2 call sites
+; CHECK-NEXT:  NoAlias:	i32* %obj, i64* %ret
+bb:
+  %obj = alloca i32
+  call void @unknown(i32* %obj)
+  %ret = call dereferenceable(2) i64* @get_i64_deref2()
+  store i32 1, i32* %obj, align 4
+  store i64 0, i64* %ret, align 8
+  %tmp = load i32, i32* %obj, align 4
+  ret i32 %tmp
+}
+
+define i32 @local_and_deref_ret_non_deref_2() {
+; CHECK:     Function: local_and_deref_ret_non_deref_2: 2 pointers, 2 call sites
+; Different result than above (see @local_and_deref_ret_2).
+; CHECK-NEXT:  MayAlias:	i32* %obj, i32* %ret
+bb:
+  %obj = alloca i32
+  call void @unknown(i32* %obj)
+  %ret = call dereferenceable(2) i32* @get_i32_deref2()
+  store i32 1, i32* %obj, align 4
+  store i32 0, i32* %ret, align 8
+  %tmp = load i32, i32* %obj, align 4
+  ret i32 %tmp
+}
diff --git a/llvm/test/Analysis/BasicAA/gep-alias.ll b/llvm/test/Analysis/BasicAA/gep-alias.ll
index 1e435af2f..5fd77e19e 100644
--- a/llvm/test/Analysis/BasicAA/gep-alias.ll
+++ b/llvm/test/Analysis/BasicAA/gep-alias.ll
@@ -247,7 +247,7 @@ define i32 @test12(i32 %x, i32 %y, i8* %p) nounwind {
 ; CHECK: [[U0ADDR:%[a-zA-Z0-9_]+]] = getelementptr inbounds [3 x i8], [3 x i8]* %u, i32 0, i32 0
 ; CHECK: [[U0:%[a-zA-Z0-9_]+]] = load i8, i8* [[U0ADDR]], align 1
 ; CHECK: [[U0ARG:%[a-zA-Z0-9_]+]] = zext i8 [[U0]] to i32
-; CHECK: call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([7 x i8], [7 x i8]* @.str, i32 0, i32 0), i32 [[T0ARG]], i32 [[U0ARG]])
+; CHECK: call i32 (i8*, ...) @printf(i8* nonnull dereferenceable(1) getelementptr inbounds ([7 x i8], [7 x i8]* @.str, i32 0, i32 0), i32 [[T0ARG]], i32 [[U0ARG]])
 ; CHECK: ret
 define void @test13() {
 entry:
diff --git a/llvm/test/Analysis/BasicAA/intrinsics.ll b/llvm/test/Analysis/BasicAA/intrinsics.ll
index 68e59862b..cf792e8c6 100644
--- a/llvm/test/Analysis/BasicAA/intrinsics.ll
+++ b/llvm/test/Analysis/BasicAA/intrinsics.ll
@@ -22,6 +22,6 @@ entry:
 declare <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>*, i32, <8 x i1>, <8 x i16>) nounwind readonly
 declare void @llvm.masked.store.v8i16.p0v8i16(<8 x i16>, <8 x i16>*, i32, <8 x i1>) nounwind
 
-; CHECK: attributes #0 = { argmemonly nounwind readonly }
-; CHECK: attributes #1 = { argmemonly nounwind }
+; CHECK: attributes #0 = { argmemonly nounwind readonly willreturn }
+; CHECK: attributes #1 = { argmemonly nounwind willreturn }
 ; CHECK: attributes [[ATTR]] = { nounwind }
diff --git a/llvm/test/Analysis/BasicAA/ptrmask.ll b/llvm/test/Analysis/BasicAA/ptrmask.ll
new file mode 100644
index 000000000..27c14ebb7
--- /dev/null
+++ b/llvm/test/Analysis/BasicAA/ptrmask.ll
@@ -0,0 +1,29 @@
+; RUN: opt -basicaa -aa-eval -print-no-aliases -disable-output %s 2>&1 | FileCheck %s
+
+%struct = type <{ [20 x i64] }>
+
+; CHECK-LABEL: Function: test_noalias: 4 pointers, 1 call sites
+; CHECK-NEXT:  NoAlias:	%struct* %ptr1, i64* %ptr2
+; CHECK-NEXT:  NoAlias:	%struct* %addr.ptr, i64* %ptr2
+; CHECK-NEXT:  NoAlias:	i64* %gep, i64* %ptr2
+define void @test_noalias(%struct* noalias %ptr1, i64* %ptr2, i64 %offset) {
+entry:
+  %addr.ptr = call %struct* @llvm.ptrmask.p0s_struct.p0s.struct.i64(%struct* %ptr1, i64 72057594037927928)
+  store i64 10, i64* %ptr2
+  %gep = getelementptr inbounds %struct, %struct* %addr.ptr, i64 0, i32 0, i64 %offset
+  store i64 1, i64* %gep, align 8
+  ret void
+}
+
+; CHECK-NEXT: Function: test_alias: 4 pointers, 1 call sites
+; CHECK-NOT: NoAlias
+define void @test_alias(%struct* %ptr1, i64* %ptr2, i64 %offset) {
+entry:
+  %addr.ptr = call %struct* @llvm.ptrmask.p0s_struct.p0s.struct.i64(%struct* %ptr1, i64 72057594037927928)
+  store i64 10, i64* %ptr2
+  %gep = getelementptr inbounds %struct, %struct* %addr.ptr, i64 0, i32 0, i64 %offset
+  store i64 1, i64* %gep, align 8
+  ret void
+}
+
+declare %struct* @llvm.ptrmask.p0s_struct.p0s.struct.i64(%struct*, i64)
diff --git a/llvm/test/Analysis/BasicAA/store-promote.ll b/llvm/test/Analysis/BasicAA/store-promote.ll
index afe11c2a1..23b74bc1a 100644
--- a/llvm/test/Analysis/BasicAA/store-promote.ll
+++ b/llvm/test/Analysis/BasicAA/store-promote.ll
@@ -1,8 +1,9 @@
-; Test that LICM uses basicaa to do alias analysis, which is capable of 
+; Test that LICM uses basicaa to do alias analysis, which is capable of
 ; disambiguating some obvious cases.  If LICM is able to disambiguate the
 ; two pointers, then the load should be hoisted, and the store sunk.
 
-; RUN: opt < %s -basicaa -licm -S | FileCheck %s
+; RUN: opt < %s -basicaa -licm -enable-mssa-loop-dependency=false -S | FileCheck %s -check-prefixes=CHECK,AST
+; RUN: opt < %s -basicaa -licm -enable-mssa-loop-dependency=true  -S | FileCheck %s -check-prefixes=CHECK,MSSA
 target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
 
 @A = global i32 7               ; <i32*> [#uses=3]
@@ -25,10 +26,13 @@ Out:            ; preds = %Loop
 ; The Loop block should be empty after the load/store are promoted.
 ; CHECK:     @test1
 ; CHECK:        load i32, i32* @A
+; MSSA:         load i32, i32* @A
+; MSSA:         store i32 %Atmp, i32* @B
 ; CHECK:      Loop:
 ; CHECK-NEXT:   br i1 %c, label %Out, label %Loop
 ; CHECK:      Out:
-; CHECK:        store i32 %Atmp, i32* @B
+; AST:          store i32 %Atmp, i32* @B
+; AST:          load i32, i32* @A
 }
 
 define i32 @test2(i1 %c) {
diff --git a/llvm/test/Analysis/BranchProbabilityInfo/basic.ll b/llvm/test/Analysis/BranchProbabilityInfo/basic.ll
index 64e0a8245..8212cc476 100644
--- a/llvm/test/Analysis/BranchProbabilityInfo/basic.ll
+++ b/llvm/test/Analysis/BranchProbabilityInfo/basic.ll
@@ -141,6 +141,24 @@ exit:
   ret i32 %result
 }
 
+define i32 @test_cold_loop(i32 %a, i32 %b) {
+entry:
+  %cond1 = icmp eq i32 %a, 42
+  br i1 %cond1, label %header, label %exit
+
+header:
+  br label %body
+
+body:
+  %cond2 = icmp eq i32 %b, 42
+  br i1 %cond2, label %header, label %exit
+; CHECK: edge body -> header probability is 0x40000000 / 0x80000000 = 50.00%
+
+exit:
+  call void @coldfunc()
+  ret i32 %b
+}
+
 declare i32 @regular_function(i32 %i)
 
 define i32 @test_cold_call_sites_with_prof(i32 %a, i32 %b, i1 %flag, i1 %flag2) {
diff --git a/llvm/test/Analysis/BranchProbabilityInfo/fcmp.ll b/llvm/test/Analysis/BranchProbabilityInfo/fcmp.ll
new file mode 100644
index 000000000..8089916fb
--- /dev/null
+++ b/llvm/test/Analysis/BranchProbabilityInfo/fcmp.ll
@@ -0,0 +1,41 @@
+; RUN: opt < %s -analyze -branch-prob | FileCheck %s
+
+; This function tests the floating point unorder comparison. The probability
+; of NaN should be extremely small.
+; CHECK: Printing analysis 'Branch Probability Analysis' for function 'uno'
+; CHECK:  edge  -> a probability is 0x00000800 / 0x80000000 = 0.00%
+; CHECK:  edge  -> b probability is 0x7ffff800 / 0x80000000 = 100.00% [HOT edge]
+
+define void @uno(float %val1, float %val2) {
+  %cond = fcmp uno float %val1, %val2
+  br i1 %cond, label %a, label %b
+
+a:
+  call void @fa()
+  ret void
+
+b:
+  call void @fb()
+  ret void
+}
+
+; This function tests the floating point order comparison.
+; CHECK: Printing analysis 'Branch Probability Analysis' for function 'ord'
+; CHECK:  edge  -> a probability is 0x7ffff800 / 0x80000000 = 100.00% [HOT edge]
+; CHECK:  edge  -> b probability is 0x00000800 / 0x80000000 = 0.00%
+
+define void @ord(float %val1, float %val2) {
+  %cond = fcmp ord float %val1, %val2
+  br i1 %cond, label %a, label %b
+
+a:
+  call void @fa()
+  ret void
+
+b:
+  call void @fb()
+  ret void
+}
+
+declare void @fa()
+declare void @fb()
diff --git a/llvm/test/Analysis/BranchProbabilityInfo/noreturn.ll b/llvm/test/Analysis/BranchProbabilityInfo/noreturn.ll
index 0566ca16c..6e01afd2c 100644
--- a/llvm/test/Analysis/BranchProbabilityInfo/noreturn.ll
+++ b/llvm/test/Analysis/BranchProbabilityInfo/noreturn.ll
@@ -79,6 +79,32 @@ exit:
   ret i32 %b
 }
 
+define i32 @test4(i32 %a, i32 %b) {
+; CHECK: Printing analysis {{.*}} for function 'test4'
+; Make sure we handle loops post-dominated by unreachables.
+entry:
+  %cond1 = icmp eq i32 %a, 42
+  br i1 %cond1, label %header, label %exit
+; CHECK: edge entry -> header probability is 0x00000001 / 0x80000000 = 0.00%
+; CHECK: edge entry -> exit probability is 0x7fffffff / 0x80000000 = 100.00% [HOT edge]
+
+header:
+  br label %body
+
+body:
+  %cond2 = icmp eq i32 %a, 42
+  br i1 %cond2, label %header, label %abort
+; CHECK: edge body -> header probability is 0x40000000 / 0x80000000 = 50.00%
+; CHECK: edge body -> abort probability is 0x40000000 / 0x80000000 = 50.00%
+
+abort:
+  call void @abort() noreturn
+  unreachable
+
+exit:
+  ret i32 %b
+}
+
 @_ZTIi = external global i8*
 
 ; CHECK-LABEL: throwSmallException
diff --git a/llvm/test/Analysis/BranchProbabilityInfo/pr22718.ll b/llvm/test/Analysis/BranchProbabilityInfo/pr22718.ll
index 66ac89793..8674f9e5a 100644
--- a/llvm/test/Analysis/BranchProbabilityInfo/pr22718.ll
+++ b/llvm/test/Analysis/BranchProbabilityInfo/pr22718.ll
@@ -73,9 +73,9 @@ declare i32 @rand() #1
 
 declare i32 @printf(i8*, ...) #2
 
-attributes #0 = { inlinehint nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #2 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { inlinehint nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #3 = { nounwind }
 
 !llvm.ident = !{!0}
diff --git a/llvm/test/Analysis/ConstantFolding/binop-identity-undef.ll b/llvm/test/Analysis/ConstantFolding/binop-identity-undef.ll
new file mode 100644
index 000000000..683078921
--- /dev/null
+++ b/llvm/test/Analysis/ConstantFolding/binop-identity-undef.ll
@@ -0,0 +1,50 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -constprop -S %s | FileCheck %s
+
+define i32 @and1() {
+; CHECK-LABEL: @and1(
+; CHECK-NEXT:    ret i32 undef
+;
+  %r = and i32 undef, -1
+  ret i32 %r
+}
+
+define i32 @and2() {
+; CHECK-LABEL: @and2(
+; CHECK-NEXT:    ret i32 undef
+;
+  %r = and i32 -1, undef
+  ret i32 %r
+}
+
+define i32 @and3_no_identity() {
+; CHECK-LABEL: @and3_no_identity(
+; CHECK-NEXT:    ret i32 0
+;
+  %r = and i32 10, undef
+  ret i32 %r
+}
+
+define i32 @or1() {
+; CHECK-LABEL: @or1(
+; CHECK-NEXT:    ret i32 undef
+;
+  %r = or i32 0, undef
+  ret i32 %r
+}
+
+define i32 @or2() {
+; CHECK-LABEL: @or2(
+; CHECK-NEXT:    ret i32 undef
+;
+  %r = or i32 undef, 0
+  ret i32 %r
+}
+
+define i32 @or3_no_identity() {
+; CHECK-LABEL: @or3_no_identity(
+; CHECK-NEXT:    ret i32 -1
+;
+  %r = or i32 undef, 10
+  ret i32 %r
+}
diff --git a/llvm/test/Analysis/ConstantFolding/copysign.ll b/llvm/test/Analysis/ConstantFolding/copysign.ll
new file mode 100644
index 000000000..228ffcb47
--- /dev/null
+++ b/llvm/test/Analysis/ConstantFolding/copysign.ll
@@ -0,0 +1,53 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -constprop < %s | FileCheck %s
+
+declare float @llvm.copysign.f32(float, float)
+declare double @llvm.copysign.f64(double, double)
+
+define float @f32_01() {
+; CHECK-LABEL: @f32_01(
+; CHECK-NEXT:    ret float -1.000000e+00
+;
+  %x = call float @llvm.copysign.f32(float 1.0, float -2.0)
+  ret float %x
+}
+
+define float @f32_02() {
+; CHECK-LABEL: @f32_02(
+; CHECK-NEXT:    ret float 2.000000e+00
+;
+  %x = call float @llvm.copysign.f32(float -2.0, float 1.0)
+  ret float %x
+}
+
+define float @f32_03() {
+; CHECK-LABEL: @f32_03(
+; CHECK-NEXT:    ret float -2.000000e+00
+;
+  %x = call float @llvm.copysign.f32(float -2.0, float -1.0)
+  ret float %x
+}
+
+define double @f64_01() {
+; CHECK-LABEL: @f64_01(
+; CHECK-NEXT:    ret double -1.000000e+00
+;
+  %x = call double @llvm.copysign.f64(double 1.0, double -2.0)
+  ret double %x
+}
+
+define double @f64_02() {
+; CHECK-LABEL: @f64_02(
+; CHECK-NEXT:    ret double 1.000000e+00
+;
+  %x = call double @llvm.copysign.f64(double -1.0, double 2.0)
+  ret double %x
+}
+
+define double @f64_03() {
+; CHECK-LABEL: @f64_03(
+; CHECK-NEXT:    ret double -1.000000e+00
+;
+  %x = call double @llvm.copysign.f64(double -1.0, double -2.0)
+  ret double %x
+}
diff --git a/llvm/test/Analysis/ConstantFolding/gep-alias.ll b/llvm/test/Analysis/ConstantFolding/gep-alias.ll
new file mode 100644
index 000000000..0fcc778a4
--- /dev/null
+++ b/llvm/test/Analysis/ConstantFolding/gep-alias.ll
@@ -0,0 +1,17 @@
+; RUN: opt -instcombine -S -o - %s | FileCheck %s
+; Test that we don't replace an alias with its aliasee when simplifying GEPs.
+; In this test case the transformation is invalid because it replaces the
+; reference to the symbol "b" (which refers to whichever instance of "b"
+; was chosen by the linker) with a reference to "a" (which refers to the
+; specific instance of "b" in this module).
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@a = internal global [3 x i8*] zeroinitializer
+@b = linkonce_odr alias [3 x i8*], [3 x i8*]* @a
+
+define i8** @f() {
+  ; CHECK: ret i8** getelementptr ([3 x i8*], [3 x i8*]* @b, i64 0, i64 1)
+  ret i8** getelementptr ([3 x i8*], [3 x i8*]* @b, i64 0, i64 1)
+}
diff --git a/llvm/test/Analysis/ConstantFolding/gep-zeroinit-vector.ll b/llvm/test/Analysis/ConstantFolding/gep-zeroinit-vector.ll
index bb5fcbdb8..03d27e9fb 100644
--- a/llvm/test/Analysis/ConstantFolding/gep-zeroinit-vector.ll
+++ b/llvm/test/Analysis/ConstantFolding/gep-zeroinit-vector.ll
@@ -9,7 +9,7 @@
 
 define <2 x i16*> @test_gep() {
 ; CHECK-LABEL: @test_gep(
-; CHECK-NEXT:    ret <2 x i16*> <i16* getelementptr inbounds (%rec8, %rec8* extractelement (<2 x %rec8*> getelementptr ([1 x %rec8], [1 x %rec8]* @a, <2 x i64> zeroinitializer, <2 x i64> zeroinitializer), i32 0), i32 0, i32 0), i16* getelementptr inbounds (%rec8, %rec8* extractelement (<2 x %rec8*> getelementptr ([1 x %rec8], [1 x %rec8]* @a, <2 x i64> zeroinitializer, <2 x i64> zeroinitializer), i32 1), i32 0, i32 0)>
+; CHECK-NEXT:    ret <2 x i16*> <i16* getelementptr inbounds ([1 x %rec8], [1 x %rec8]* @a, i32 0, i32 0, i32 0), i16* getelementptr inbounds ([1 x %rec8], [1 x %rec8]* @a, i32 0, i32 0, i32 0)>
 ;
   %A = getelementptr [1 x %rec8], [1 x %rec8]* @a, <2 x i16> zeroinitializer, <2 x i64> zeroinitializer
   %B = bitcast <2 x %rec8*> %A to <2 x i16*>
diff --git a/llvm/test/Analysis/ConstantFolding/insertelement.ll b/llvm/test/Analysis/ConstantFolding/insertelement.ll
new file mode 100644
index 000000000..960042acf
--- /dev/null
+++ b/llvm/test/Analysis/ConstantFolding/insertelement.ll
@@ -0,0 +1,19 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -constprop -S | FileCheck %s
+
+
+define <4 x i32> @insertelement_fixedlength_constant() {
+; CHECK-LABEL: @insertelement_fixedlength_constant(
+; CHECK-NEXT:    ret <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+;
+  %i = insertelement <4 x i32> undef, i32 1, i32 0
+  ret <4 x i32> %i
+}
+
+define <vscale x 4 x i32> @insertelement_scalable_constant() {
+; CHECK-LABEL: @insertelement_scalable_constant(
+; CHECK-NEXT:    ret <vscale x 4 x i32> insertelement (<vscale x 4 x i32> undef, i32 1, i32 0)
+;
+  %i = insertelement <vscale x 4 x i32> undef, i32 1, i32 0
+  ret <vscale x 4 x i32> %i
+}
diff --git a/llvm/test/Analysis/ConstantFolding/math-1.ll b/llvm/test/Analysis/ConstantFolding/math-1.ll
new file mode 100644
index 000000000..595095017
--- /dev/null
+++ b/llvm/test/Analysis/ConstantFolding/math-1.ll
@@ -0,0 +1,195 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -early-cse -S -o - %s | FileCheck %s
+
+declare double @acos(double)
+define double @f_acos() {
+; CHECK-LABEL: @f_acos(
+; CHECK-NEXT:    ret double 0.000000e+00
+;
+  %res = tail call fast double @acos(double 1.0)
+  ret double %res
+}
+
+declare float @asinf(float)
+define float @f_asinf() {
+; CHECK-LABEL: @f_asinf(
+; CHECK-NEXT:    ret float 0x3FF921FB{{.+}}
+;
+  %res = tail call fast float @asinf(float 1.0)
+  ret float %res
+}
+
+declare double @atan(double)
+define double @f_atan() {
+; CHECK-LABEL: @f_atan(
+; CHECK-NEXT:    [[RES:%.*]] = tail call fast double @atan(double 1.000000e+00)
+; CHECK-NEXT:    ret double 0x3FE921FB
+;
+  %res = tail call fast double @atan(double 1.0)
+  ret double %res
+}
+
+declare float @cosf(float)
+define float @f_cosf() {
+; CHECK-LABEL: @f_cosf(
+; CHECK-NEXT:    ret float 0x3FE14A2{{.+}}
+;
+  %res = tail call fast float @cosf(float 1.0)
+  ret float %res
+}
+
+declare float @llvm.cos.f32(float)
+define float @i_cosf() {
+; CHECK-LABEL: @i_cosf(
+; CHECK-NEXT:    ret float 0x3FE14A2
+;
+  %res = tail call fast float @llvm.cos.f32(float 1.0)
+  ret float %res
+}
+
+declare double @cosh(double)
+define double @f_cosh() {
+; CHECK-LABEL: @f_cosh(
+; CHECK-NEXT:    ret double 0x3FF8B075{{.+}}
+;
+  %res = tail call fast double @cosh(double 1.0)
+  ret double %res
+}
+
+declare float @expf(float)
+define float @f_expf() {
+; CHECK-LABEL: @f_expf(
+; CHECK-NEXT:    ret float 0x4005BF0A{{.+}}
+;
+  %res = tail call fast float @expf(float 1.0)
+  ret float %res
+}
+
+declare float @llvm.exp.f32(float)
+define float @i_expf() {
+; CHECK-LABEL: @i_expf(
+; CHECK-NEXT:    ret float 0x4005BF0A{{.+}}
+;
+  %res = tail call fast float @llvm.exp.f32(float 1.0)
+  ret float %res
+}
+
+declare double @exp2(double)
+define double @f_exp2() {
+; CHECK-LABEL: @f_exp2(
+; CHECK-NEXT:    ret double 2.000000e+00
+;
+  %res = tail call fast double @exp2(double 1.0)
+  ret double %res
+}
+
+declare double @llvm.exp2.f64(double)
+define double @i_exp2() {
+; CHECK-LABEL: @i_exp2(
+; CHECK-NEXT:    ret double 2.000000e+00
+;
+  %res = tail call fast double @llvm.exp2.f64(double 1.0)
+  ret double %res
+}
+
+; FIXME: exp10() is not widely supported.
+declare float @exp10f(float)
+define float @f_exp10f() {
+; CHECK-LABEL: @f_exp10f(
+; CHECK-NEXT:    [[RES:%.*]] = tail call float @exp10f(float 1.000000e+00)
+; CHECK-NEXT:    ret float [[RES]]
+;
+  %res = tail call float @exp10f(float 1.0)
+  ret float %res
+}
+
+declare double @log(double)
+define double @f_log() {
+; CHECK-LABEL: @f_log(
+; CHECK-NEXT:    ret double 0.000000e+00
+;
+  %res = tail call fast double @log(double 1.0)
+  ret double %res
+}
+
+declare double @llvm.log.f64(double)
+define double @i_log() {
+; CHECK-LABEL: @i_log(
+; CHECK-NEXT:    ret double 0.000000e+00
+;
+  %res = tail call fast double @llvm.log.f64(double 1.0)
+  ret double %res
+}
+
+declare float @log2f(float)
+define float @f_log2f() {
+; CHECK-LABEL: @f_log2f(
+; CHECK-NEXT:    ret float 0.000000e+00
+;
+  %res = tail call fast float @log2f(float 1.0)
+  ret float %res
+}
+
+declare float @llvm.log2.f32(float)
+define float @i_log2f() {
+; CHECK-LABEL: @i_log2f(
+; CHECK-NEXT:    ret float 0.000000e+00
+;
+  %res = tail call fast float @llvm.log2.f32(float 1.0)
+  ret float %res
+}
+
+declare double @log10(double)
+define double @f_log10() {
+; CHECK-LABEL: @f_log10(
+; CHECK-NEXT:    ret double 0.000000e+00
+;
+  %res = tail call fast double @log10(double 1.0)
+  ret double %res
+}
+
+declare float @sinf(float)
+define float @f_sinf() {
+; CHECK-LABEL: @f_sinf(
+; CHECK-NEXT:    ret float 0x3FEAED54{{.+}}
+;
+  %res = tail call fast float @sinf(float 1.0)
+  ret float %res
+}
+
+declare double @sinh(double)
+define double @f_sinh() {
+; CHECK-LABEL: @f_sinh(
+; CHECK-NEXT:    ret double 0x3FF2CD9F{{.+}}
+;
+  %res = tail call fast double @sinh(double 1.0)
+  ret double %res
+}
+
+declare float @sqrtf(float)
+define float @f_sqrtf() {
+; CHECK-LABEL: @f_sqrtf(
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %res = tail call fast float @sqrtf(float 1.0)
+  ret float %res
+}
+
+declare double @tan(double)
+define double @f_tan() {
+; CHECK-LABEL: @f_tan(
+; CHECK-NEXT:    ret double 0x3FF8EB24{{.+}}
+;
+  %res = tail call fast double @tan(double 1.0)
+  ret double %res
+}
+
+declare float @tanhf(float)
+define float @f_tanhf() {
+; CHECK-LABEL: @f_tanhf(
+; CHECK-NEXT:    [[RES:%.*]] = tail call fast float @tanhf(float 1.000000e+00)
+; CHECK-NEXT:    ret float 0x3FE85EFA{{.+}}
+;
+  %res = tail call fast float @tanhf(float 1.0)
+  ret float %res
+}
diff --git a/llvm/test/Analysis/ConstantFolding/math-2.ll b/llvm/test/Analysis/ConstantFolding/math-2.ll
new file mode 100644
index 000000000..90b64797f
--- /dev/null
+++ b/llvm/test/Analysis/ConstantFolding/math-2.ll
@@ -0,0 +1,48 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -early-cse -S -o - %s | FileCheck %s
+
+declare double @atan2(double, double)
+define double @f_atan2() {
+; CHECK-LABEL: @f_atan2(
+; CHECK-NEXT:    [[RES:%.*]] = tail call fast double @atan2(double 1.000000e+00, double 2.000000e+00)
+; CHECK-NEXT:    ret double 0x3FDDAC6{{.+}}
+;
+  %res = tail call fast double @atan2(double 1.0, double 2.0)
+  ret double %res
+}
+
+declare float @fmodf(float, float)
+define float @f_fmodf() {
+; CHECK-LABEL: @f_fmodf(
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %res = tail call fast float @fmodf(float 1.0, float 2.0)
+  ret float %res
+}
+
+declare double @pow(double, double)
+define double @f_pow() {
+; CHECK-LABEL: @f_pow(
+; CHECK-NEXT:    ret double 1.000000e+00
+;
+  %res = tail call fast double @pow(double 1.0, double 2.0)
+  ret double %res
+}
+
+declare float @llvm.pow.f32(float, float)
+define float @i_powf() {
+; CHECK-LABEL: @i_powf(
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %res = tail call fast float @llvm.pow.f32(float 1.0, float 2.0)
+  ret float %res
+}
+
+declare double @llvm.powi.f64(double, i32)
+define double @i_powi() {
+; CHECK-LABEL: @i_powi(
+; CHECK-NEXT:    ret double 1.000000e+00
+;
+  %res = tail call fast double @llvm.powi.f64(double 1.0, i32 2)
+  ret double %res
+}
diff --git a/llvm/test/Analysis/ConstantFolding/rint.ll b/llvm/test/Analysis/ConstantFolding/rint.ll
new file mode 100644
index 000000000..9ad794d60
--- /dev/null
+++ b/llvm/test/Analysis/ConstantFolding/rint.ll
@@ -0,0 +1,109 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -early-cse < %s | FileCheck %s
+
+declare float @nearbyintf(float) #0
+declare float @llvm.nearbyint.f32(float) #0
+declare double @nearbyint(double) #0
+declare double @llvm.nearbyint.f64(double) #0
+declare float @rintf(float) #0
+declare float @llvm.rint.f32(float) #0
+declare double @rint(double) #0
+declare double @llvm.rint.f64(double) #0
+
+define float @constant_fold_rint_f32_01() #0 {
+; CHECK-LABEL: @constant_fold_rint_f32_01(
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %x = call float @nearbyintf(float 1.25) #0
+  ret float %x
+}
+
+define float @constant_fold_rint_f32_02() #0 {
+; CHECK-LABEL: @constant_fold_rint_f32_02(
+; CHECK-NEXT:    ret float -1.000000e+00
+;
+  %x = call float @llvm.nearbyint.f32(float -1.25) #0
+  ret float %x
+}
+
+define float @constant_fold_rint_f32_03() #0 {
+; CHECK-LABEL: @constant_fold_rint_f32_03(
+; CHECK-NEXT:    ret float 2.000000e+00
+;
+  %x = call float @rintf(float 1.5) #0
+  ret float %x
+}
+
+define float @constant_fold_rint_f32_04() #0 {
+; CHECK-LABEL: @constant_fold_rint_f32_04(
+; CHECK-NEXT:    ret float -2.000000e+00
+;
+  %x = call float @llvm.rint.f32(float -1.5) #0
+  ret float %x
+}
+
+define float @constant_fold_rint_f32_05() #0 {
+; CHECK-LABEL: @constant_fold_rint_f32_05(
+; CHECK-NEXT:    ret float 3.000000e+00
+;
+  %x = call float @nearbyintf(float 2.75) #0
+  ret float %x
+}
+
+define float @constant_fold_rint_f32_06() #0 {
+; CHECK-LABEL: @constant_fold_rint_f32_06(
+; CHECK-NEXT:    ret float -3.000000e+00
+;
+  %x = call float @llvm.nearbyint.f32(float -2.75) #0
+  ret float %x
+}
+
+define double @constant_fold_rint_f64_01() #0 {
+; CHECK-LABEL: @constant_fold_rint_f64_01(
+; CHECK-NEXT:    ret double 1.000000e+00
+;
+  %x = call double @rint(double 1.3) #0
+  ret double %x
+}
+
+define double @constant_fold_rint_f64_02() #0 {
+; CHECK-LABEL: @constant_fold_rint_f64_02(
+; CHECK-NEXT:    ret double -1.000000e+00
+;
+  %x = call double @llvm.rint.f64(double -1.3) #0
+  ret double %x
+}
+
+define double @constant_fold_rint_f64_03() #0 {
+; CHECK-LABEL: @constant_fold_rint_f64_03(
+; CHECK-NEXT:    ret double 2.000000e+00
+;
+  %x = call double @nearbyint(double 1.5) #0
+  ret double %x
+}
+
+define double @constant_fold_rint_f64_04() #0 {
+; CHECK-LABEL: @constant_fold_rint_f64_04(
+; CHECK-NEXT:    ret double -2.000000e+00
+;
+  %x = call double @llvm.nearbyint.f64(double -1.5) #0
+  ret double %x
+}
+
+define double @constant_fold_rint_f64_05() #0 {
+; CHECK-LABEL: @constant_fold_rint_f64_05(
+; CHECK-NEXT:    ret double 3.000000e+00
+;
+  %x = call double @rint(double 2.7) #0
+  ret double %x
+}
+
+define double @constant_fold_rint_f64_06() #0 {
+; CHECK-LABEL: @constant_fold_rint_f64_06(
+; CHECK-NEXT:    ret double -3.000000e+00
+;
+  %x = call double @llvm.rint.f64(double -2.7) #0
+  ret double %x
+}
+
+attributes #0 = { nounwind readnone }
diff --git a/llvm/test/Analysis/ConstantFolding/round.ll b/llvm/test/Analysis/ConstantFolding/round.ll
new file mode 100644
index 000000000..d5b847810
--- /dev/null
+++ b/llvm/test/Analysis/ConstantFolding/round.ll
@@ -0,0 +1,92 @@
+; RUN: opt -S -early-cse < %s | FileCheck %s
+
+declare float @roundf(float) #0
+declare float @llvm.round.f32(float) #0
+declare double @round(double) #0
+declare double @llvm.round.f64(double) #0
+
+; CHECK-LABEL: @constant_fold_round_f32_01
+; CHECK-NEXT: ret float 1.000000e+00
+define float @constant_fold_round_f32_01() #0 {
+  %x = call float @roundf(float 1.25) #0
+  ret float %x
+}
+
+; CHECK-LABEL: @constant_fold_round_f32_02
+; CHECK-NEXT: ret float -1.000000e+00
+define float @constant_fold_round_f32_02() #0 {
+  %x = call float @llvm.round.f32(float -1.25) #0
+  ret float %x
+}
+
+; CHECK-LABEL: @constant_fold_round_f32_03
+; CHECK-NEXT: ret float 2.000000e+00
+define float @constant_fold_round_f32_03() #0 {
+  %x = call float @roundf(float 1.5) #0
+  ret float %x
+}
+
+; CHECK-LABEL: @constant_fold_round_f32_04
+; CHECK-NEXT: ret float -2.000000e+00
+define float @constant_fold_round_f32_04() #0 {
+  %x = call float @llvm.round.f32(float -1.5) #0
+  ret float %x
+}
+
+; CHECK-LABEL: @constant_fold_round_f32_05
+; CHECK-NEXT: ret float 3.000000e+00
+define float @constant_fold_round_f32_05() #0 {
+  %x = call float @roundf(float 2.75) #0
+  ret float %x
+}
+
+; CHECK-LABEL: @constant_fold_round_f32_06
+; CHECK-NEXT: ret float -3.000000e+00
+define float @constant_fold_round_f32_06() #0 {
+  %x = call float @llvm.round.f32(float -2.75) #0
+  ret float %x
+}
+
+; CHECK-LABEL: @constant_fold_round_f64_01
+; CHECK-NEXT: ret double 1.000000e+00
+define double @constant_fold_round_f64_01() #0 {
+  %x = call double @round(double 1.3) #0
+  ret double %x
+}
+
+; CHECK-LABEL: @constant_fold_round_f64_02
+; CHECK-NEXT: ret double -1.000000e+00
+define double @constant_fold_round_f64_02() #0 {
+  %x = call double @llvm.round.f64(double -1.3) #0
+  ret double %x
+}
+
+; CHECK-LABEL: @constant_fold_round_f64_03
+; CHECK-NEXT: ret double 2.000000e+00
+define double @constant_fold_round_f64_03() #0 {
+  %x = call double @round(double 1.5) #0
+  ret double %x
+}
+
+; CHECK-LABEL: @constant_fold_round_f64_04
+; CHECK-NEXT: ret double -2.000000e+00
+define double @constant_fold_round_f64_04() #0 {
+  %x = call double @llvm.round.f64(double -1.5) #0
+  ret double %x
+}
+
+; CHECK-LABEL: @constant_fold_round_f64_05
+; CHECK-NEXT: ret double 3.000000e+00
+define double @constant_fold_round_f64_05() #0 {
+  %x = call double @round(double 2.7) #0
+  ret double %x
+}
+
+; CHECK-LABEL: @constant_fold_round_f64_06
+; CHECK-NEXT: ret double -3.000000e+00
+define double @constant_fold_round_f64_06() #0 {
+  %x = call double @llvm.round.f64(double -2.7) #0
+  ret double %x
+}
+
+attributes #0 = { nounwind readnone }
diff --git a/llvm/test/Analysis/ConstantFolding/shufflevector.ll b/llvm/test/Analysis/ConstantFolding/shufflevector.ll
new file mode 100644
index 000000000..d69c2caec
--- /dev/null
+++ b/llvm/test/Analysis/ConstantFolding/shufflevector.ll
@@ -0,0 +1,11 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -constprop -S | FileCheck %s
+
+define <vscale x 4 x i32> @shufflevector_scalable_constant() {
+; CHECK-LABEL: @shufflevector_scalable_constant(
+; CHECK-NEXT:    ret <vscale x 4 x i32> shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> undef, i32 1, i32 0), <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer)
+;
+  %i = insertelement <vscale x 4 x i32> undef, i32 1, i32 0
+  %i2 = shufflevector <vscale x 4 x i32> %i, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  ret <vscale x 4 x i32> %i2
+}
diff --git a/llvm/test/Analysis/ConstantFolding/trunc.ll b/llvm/test/Analysis/ConstantFolding/trunc.ll
new file mode 100644
index 000000000..df1380917
--- /dev/null
+++ b/llvm/test/Analysis/ConstantFolding/trunc.ll
@@ -0,0 +1,105 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -early-cse < %s | FileCheck %s
+
+declare float @truncf(float) #0
+declare float @llvm.trunc.f32(float) #0
+declare double @trunc(double) #0
+declare double @llvm.trunc.f64(double) #0
+
+define float @constant_fold_trunc_f32_01() #0 {
+; CHECK-LABEL: @constant_fold_trunc_f32_01(
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %x = call float @truncf(float 1.25) #0
+  ret float %x
+}
+
+define float @constant_fold_trunc_f32_02() #0 {
+; CHECK-LABEL: @constant_fold_trunc_f32_02(
+; CHECK-NEXT:    ret float -1.000000e+00
+;
+  %x = call float @llvm.trunc.f32(float -1.25) #0
+  ret float %x
+}
+
+define float @constant_fold_trunc_f32_03() #0 {
+; CHECK-LABEL: @constant_fold_trunc_f32_03(
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %x = call float @truncf(float 1.5) #0
+  ret float %x
+}
+
+define float @constant_fold_trunc_f32_04() #0 {
+; CHECK-LABEL: @constant_fold_trunc_f32_04(
+; CHECK-NEXT:    ret float -1.000000e+00
+;
+  %x = call float @llvm.trunc.f32(float -1.5) #0
+  ret float %x
+}
+
+define float @constant_fold_trunc_f32_05() #0 {
+; CHECK-LABEL: @constant_fold_trunc_f32_05(
+; CHECK-NEXT:    ret float 2.000000e+00
+;
+  %x = call float @truncf(float 2.75) #0
+  ret float %x
+}
+
+define float @constant_fold_trunc_f32_06() #0 {
+; CHECK-LABEL: @constant_fold_trunc_f32_06(
+; CHECK-NEXT:    ret float -2.000000e+00
+;
+  %x = call float @llvm.trunc.f32(float -2.75) #0
+  ret float %x
+}
+
+define double @constant_fold_trunc_f64_01() #0 {
+; CHECK-LABEL: @constant_fold_trunc_f64_01(
+; CHECK-NEXT:    ret double 1.000000e+00
+;
+  %x = call double @trunc(double 1.3) #0
+  ret double %x
+}
+
+define double @constant_fold_trunc_f64_02() #0 {
+; CHECK-LABEL: @constant_fold_trunc_f64_02(
+; CHECK-NEXT:    ret double -1.000000e+00
+;
+  %x = call double @llvm.trunc.f64(double -1.3) #0
+  ret double %x
+}
+
+define double @constant_fold_trunc_f64_03() #0 {
+; CHECK-LABEL: @constant_fold_trunc_f64_03(
+; CHECK-NEXT:    ret double 1.000000e+00
+;
+  %x = call double @trunc(double 1.5) #0
+  ret double %x
+}
+
+define double @constant_fold_trunc_f64_04() #0 {
+; CHECK-LABEL: @constant_fold_trunc_f64_04(
+; CHECK-NEXT:    ret double -1.000000e+00
+;
+  %x = call double @llvm.trunc.f64(double -1.5) #0
+  ret double %x
+}
+
+define double @constant_fold_trunc_f64_05() #0 {
+; CHECK-LABEL: @constant_fold_trunc_f64_05(
+; CHECK-NEXT:    ret double 2.000000e+00
+;
+  %x = call double @trunc(double 2.7) #0
+  ret double %x
+}
+
+define double @constant_fold_trunc_f64_06() #0 {
+; CHECK-LABEL: @constant_fold_trunc_f64_06(
+; CHECK-NEXT:    ret double -2.000000e+00
+;
+  %x = call double @llvm.trunc.f64(double -2.7) #0
+  ret double %x
+}
+
+attributes #0 = { nounwind readnone }
diff --git a/llvm/test/Analysis/CostModel/AArch64/aggregates.ll b/llvm/test/Analysis/CostModel/AArch64/aggregates.ll
new file mode 100644
index 000000000..35d232b3b
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/AArch64/aggregates.ll
@@ -0,0 +1,142 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt < %s -mtriple=aarch64-unknown-linux-gnu -cost-model -cost-kind=throughput -analyze | FileCheck %s --check-prefixes=ALL,THROUGHPUT
+; RUN: opt < %s -mtriple=aarch64-unknown-linux-gnu -cost-model -cost-kind=latency -analyze | FileCheck %s --check-prefixes=ALL,LATENCY
+; RUN: opt < %s -mtriple=aarch64-unknown-linux-gnu -cost-model -cost-kind=code-size -analyze | FileCheck %s --check-prefixes=ALL,CODESIZE
+
+define i32 @extract_first_i32({i32, i32} %agg) {
+; THROUGHPUT-LABEL: 'extract_first_i32'
+; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, i32 } %agg, 0
+; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %r
+;
+; LATENCY-LABEL: 'extract_first_i32'
+; LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, i32 } %agg, 0
+; LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %r
+;
+; CODESIZE-LABEL: 'extract_first_i32'
+; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, i32 } %agg, 0
+; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %r
+;
+  %r = extractvalue {i32, i32} %agg, 0
+  ret i32 %r
+}
+
+define i32 @extract_second_i32({i32, i32} %agg) {
+; THROUGHPUT-LABEL: 'extract_second_i32'
+; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, i32 } %agg, 1
+; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %r
+;
+; LATENCY-LABEL: 'extract_second_i32'
+; LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, i32 } %agg, 1
+; LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %r
+;
+; CODESIZE-LABEL: 'extract_second_i32'
+; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, i32 } %agg, 1
+; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %r
+;
+  %r = extractvalue {i32, i32} %agg, 1
+  ret i32 %r
+}
+
+define i32 @extract_i32({i32, i1} %agg) {
+; THROUGHPUT-LABEL: 'extract_i32'
+; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, i1 } %agg, 0
+; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %r
+;
+; LATENCY-LABEL: 'extract_i32'
+; LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, i1 } %agg, 0
+; LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %r
+;
+; CODESIZE-LABEL: 'extract_i32'
+; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, i1 } %agg, 0
+; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %r
+;
+  %r = extractvalue {i32, i1} %agg, 0
+  ret i32 %r
+}
+
+define i1 @extract_i1({i32, i1} %agg) {
+; THROUGHPUT-LABEL: 'extract_i1'
+; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, i1 } %agg, 1
+; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i1 %r
+;
+; LATENCY-LABEL: 'extract_i1'
+; LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, i1 } %agg, 1
+; LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i1 %r
+;
+; CODESIZE-LABEL: 'extract_i1'
+; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, i1 } %agg, 1
+; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i1 %r
+;
+  %r = extractvalue {i32, i1} %agg, 1
+  ret i1 %r
+}
+
+define float @extract_float({i32, float} %agg) {
+; THROUGHPUT-LABEL: 'extract_float'
+; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, float } %agg, 1
+; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r
+;
+; LATENCY-LABEL: 'extract_float'
+; LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, float } %agg, 1
+; LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret float %r
+;
+; CODESIZE-LABEL: 'extract_float'
+; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, float } %agg, 1
+; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret float %r
+;
+  %r = extractvalue {i32, float} %agg, 1
+  ret float %r
+}
+
+define [42 x i42] @extract_array({i32, [42 x i42]} %agg) {
+; THROUGHPUT-LABEL: 'extract_array'
+; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, [42 x i42] } %agg, 1
+; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret [42 x i42] %r
+;
+; LATENCY-LABEL: 'extract_array'
+; LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, [42 x i42] } %agg, 1
+; LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret [42 x i42] %r
+;
+; CODESIZE-LABEL: 'extract_array'
+; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, [42 x i42] } %agg, 1
+; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret [42 x i42] %r
+;
+  %r = extractvalue {i32, [42 x i42]} %agg, 1
+  ret [42 x i42] %r
+}
+
+define <42 x i42> @extract_vector({i32, <42 x i42>} %agg) {
+; THROUGHPUT-LABEL: 'extract_vector'
+; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, <42 x i42> } %agg, 1
+; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <42 x i42> %r
+;
+; LATENCY-LABEL: 'extract_vector'
+; LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, <42 x i42> } %agg, 1
+; LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <42 x i42> %r
+;
+; CODESIZE-LABEL: 'extract_vector'
+; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, <42 x i42> } %agg, 1
+; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <42 x i42> %r
+;
+  %r = extractvalue {i32, <42 x i42>} %agg, 1
+  ret <42 x i42> %r
+}
+
+%T1 = type { i32, float, <4 x i1> }
+
+define %T1 @extract_struct({i32, %T1} %agg) {
+; THROUGHPUT-LABEL: 'extract_struct'
+; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, %T1 } %agg, 1
+; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret %T1 %r
+;
+; LATENCY-LABEL: 'extract_struct'
+; LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, %T1 } %agg, 1
+; LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret %T1 %r
+;
+; CODESIZE-LABEL: 'extract_struct'
+; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, %T1 } %agg, 1
+; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret %T1 %r
+;
+  %r = extractvalue {i32, %T1} %agg, 1
+  ret %T1 %r
+}
diff --git a/llvm/test/Analysis/CostModel/AMDGPU/add-sub.ll b/llvm/test/Analysis/CostModel/AMDGPU/add-sub.ll
index b538b12f0..9a2c01058 100644
--- a/llvm/test/Analysis/CostModel/AMDGPU/add-sub.ll
+++ b/llvm/test/Analysis/CostModel/AMDGPU/add-sub.ll
@@ -1,8 +1,11 @@
-; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa -mattr=+half-rate-64-ops < %s | FileCheck %s
-; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa -mattr=-half-rate-64-ops < %s | FileCheck %s
+; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa -mcpu=gfx900 -mattr=+half-rate-64-ops < %s | FileCheck -check-prefixes=FAST16,ALL %s
+; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa -mattr=-half-rate-64-ops < %s | FileCheck -check-prefixes=SLOW16,ALL %s
+; RUN: opt -cost-model -cost-kind=code-size -analyze -mtriple=amdgcn-unknown-amdhsa -mcpu=gfx900 -mattr=+half-rate-64-ops < %s | FileCheck -check-prefixes=FAST16,ALL %s
+; RUN: opt -cost-model -cost-kind=code-size -analyze -mtriple=amdgcn-unknown-amdhsa -mattr=-half-rate-64-ops < %s | FileCheck -check-prefixes=SLOW16,ALL %s
 
-; CHECK: 'add_i32'
-; CHECK: estimated cost of 1 for {{.*}} add i32
+
+; ALL: 'add_i32'
+; ALL: estimated cost of 1 for {{.*}} add i32
 define amdgpu_kernel void @add_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %vaddr, i32 %b) #0 {
   %vec = load i32, i32 addrspace(1)* %vaddr
   %add = add i32 %vec, %b
@@ -10,8 +13,8 @@ define amdgpu_kernel void @add_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %va
   ret void
 }
 
-; CHECK: 'add_v2i32'
-; CHECK: estimated cost of 2 for {{.*}} add <2 x i32>
+; ALL: 'add_v2i32'
+; ALL: estimated cost of 2 for {{.*}} add <2 x i32>
 define amdgpu_kernel void @add_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32> addrspace(1)* %vaddr, <2 x i32> %b) #0 {
   %vec = load <2 x i32>, <2 x i32> addrspace(1)* %vaddr
   %add = add <2 x i32> %vec, %b
@@ -19,10 +22,10 @@ define amdgpu_kernel void @add_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32> add
   ret void
 }
 
-; CHECK: 'add_v3i32'
+; ALL: 'add_v3i32'
 ; Allow for 4 when v3i32 is illegal and TargetLowering thinks it needs widening,
 ; and 3 when it is legal.
-; CHECK: estimated cost of {{[34]}} for {{.*}} add <3 x i32>
+; ALL: estimated cost of {{[34]}} for {{.*}} add <3 x i32>
 define amdgpu_kernel void @add_v3i32(<3 x i32> addrspace(1)* %out, <3 x i32> addrspace(1)* %vaddr, <3 x i32> %b) #0 {
   %vec = load <3 x i32>, <3 x i32> addrspace(1)* %vaddr
   %add = add <3 x i32> %vec, %b
@@ -30,8 +33,8 @@ define amdgpu_kernel void @add_v3i32(<3 x i32> addrspace(1)* %out, <3 x i32> add
   ret void
 }
 
-; CHECK: 'add_v4i32'
-; CHECK: estimated cost of 4 for {{.*}} add <4 x i32>
+; ALL: 'add_v4i32'
+; ALL: estimated cost of 4 for {{.*}} add <4 x i32>
 define amdgpu_kernel void @add_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> addrspace(1)* %vaddr, <4 x i32> %b) #0 {
   %vec = load <4 x i32>, <4 x i32> addrspace(1)* %vaddr
   %add = add <4 x i32> %vec, %b
@@ -39,10 +42,10 @@ define amdgpu_kernel void @add_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> add
   ret void
 }
 
-; CHECK: 'add_v5i32'
+; ALL: 'add_v5i32'
 ; Allow for 8 when v3i32 is illegal and TargetLowering thinks it needs widening,
 ; and 5 when it is legal.
-; CHECK: estimated cost of {{[58]}} for {{.*}} add <5 x i32>
+; ALL: estimated cost of {{[58]}} for {{.*}} add <5 x i32>
 define amdgpu_kernel void @add_v5i32(<5 x i32> addrspace(1)* %out, <5 x i32> addrspace(1)* %vaddr, <5 x i32> %b) #0 {
   %vec = load <5 x i32>, <5 x i32> addrspace(1)* %vaddr
   %add = add <5 x i32> %vec, %b
@@ -50,8 +53,8 @@ define amdgpu_kernel void @add_v5i32(<5 x i32> addrspace(1)* %out, <5 x i32> add
   ret void
 }
 
-; CHECK: 'add_i64'
-; CHECK: estimated cost of 2 for {{.*}} add i64
+; ALL: 'add_i64'
+; ALL: estimated cost of 2 for {{.*}} add i64
 define amdgpu_kernel void @add_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %vaddr, i64 %b) #0 {
   %vec = load i64, i64 addrspace(1)* %vaddr
   %add = add i64 %vec, %b
@@ -59,8 +62,8 @@ define amdgpu_kernel void @add_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %va
   ret void
 }
 
-; CHECK: 'add_v2i64'
-; CHECK: estimated cost of 4 for {{.*}} add <2 x i64>
+; ALL: 'add_v2i64'
+; ALL: estimated cost of 4 for {{.*}} add <2 x i64>
 define amdgpu_kernel void @add_v2i64(<2 x i64> addrspace(1)* %out, <2 x i64> addrspace(1)* %vaddr, <2 x i64> %b) #0 {
   %vec = load <2 x i64>, <2 x i64> addrspace(1)* %vaddr
   %add = add <2 x i64> %vec, %b
@@ -68,8 +71,8 @@ define amdgpu_kernel void @add_v2i64(<2 x i64> addrspace(1)* %out, <2 x i64> add
   ret void
 }
 
-; CHECK: 'add_v3i64'
-; CHECK: estimated cost of 6 for {{.*}} add <3 x i64>
+; ALL: 'add_v3i64'
+; ALL: estimated cost of 6 for {{.*}} add <3 x i64>
 define amdgpu_kernel void @add_v3i64(<3 x i64> addrspace(1)* %out, <3 x i64> addrspace(1)* %vaddr, <3 x i64> %b) #0 {
   %vec = load <3 x i64>, <3 x i64> addrspace(1)* %vaddr
   %add = add <3 x i64> %vec, %b
@@ -77,8 +80,8 @@ define amdgpu_kernel void @add_v3i64(<3 x i64> addrspace(1)* %out, <3 x i64> add
   ret void
 }
 
-; CHECK: 'add_v4i64'
-; CHECK: estimated cost of 8 for {{.*}} add <4 x i64>
+; ALL: 'add_v4i64'
+; ALL: estimated cost of 8 for {{.*}} add <4 x i64>
 define amdgpu_kernel void @add_v4i64(<4 x i64> addrspace(1)* %out, <4 x i64> addrspace(1)* %vaddr, <4 x i64> %b) #0 {
   %vec = load <4 x i64>, <4 x i64> addrspace(1)* %vaddr
   %add = add <4 x i64> %vec, %b
@@ -86,8 +89,8 @@ define amdgpu_kernel void @add_v4i64(<4 x i64> addrspace(1)* %out, <4 x i64> add
   ret void
 }
 
-; CHECK: 'add_v16i64'
-; CHECK: estimated cost of 32 for {{.*}} add <16 x i64>
+; ALL: 'add_v16i64'
+; ALL: estimated cost of 32 for {{.*}} add <16 x i64>
 define amdgpu_kernel void @add_v16i64(<16 x i64> addrspace(1)* %out, <16 x i64> addrspace(1)* %vaddr, <16 x i64> %b) #0 {
   %vec = load <16 x i64>, <16 x i64> addrspace(1)* %vaddr
   %add = add <16 x i64> %vec, %b
@@ -95,8 +98,8 @@ define amdgpu_kernel void @add_v16i64(<16 x i64> addrspace(1)* %out, <16 x i64>
   ret void
 }
 
-; CHECK: 'add_i16'
-; CHECK: estimated cost of 1 for {{.*}} add i16
+; ALL: 'add_i16'
+; ALL: estimated cost of 1 for {{.*}} add i16
 define amdgpu_kernel void @add_i16(i16 addrspace(1)* %out, i16 addrspace(1)* %vaddr, i16 %b) #0 {
   %vec = load i16, i16 addrspace(1)* %vaddr
   %add = add i16 %vec, %b
@@ -104,8 +107,9 @@ define amdgpu_kernel void @add_i16(i16 addrspace(1)* %out, i16 addrspace(1)* %va
   ret void
 }
 
-; CHECK: 'add_v2i16'
-; CHECK: estimated cost of 2 for {{.*}} add <2 x i16>
+; ALL: 'add_v2i16'
+; SLOW16: estimated cost of 2 for {{.*}} add <2 x i16>
+; FAST16: estimated cost of 1 for {{.*}} add <2 x i16>
 define amdgpu_kernel void @add_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %vaddr, <2 x i16> %b) #0 {
   %vec = load <2 x i16>, <2 x i16> addrspace(1)* %vaddr
   %add = add <2 x i16> %vec, %b
@@ -113,8 +117,8 @@ define amdgpu_kernel void @add_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> add
   ret void
 }
 
-; CHECK: 'sub_i32'
-; CHECK: estimated cost of 1 for {{.*}} sub i32
+; ALL: 'sub_i32'
+; ALL: estimated cost of 1 for {{.*}} sub i32
 define amdgpu_kernel void @sub_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %vaddr, i32 %b) #0 {
   %vec = load i32, i32 addrspace(1)* %vaddr
   %sub = sub i32 %vec, %b
@@ -122,16 +126,16 @@ define amdgpu_kernel void @sub_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %va
   ret void
 }
 
-; CHECK: 'sub_i64'
-; CHECK: estimated cost of 2 for {{.*}} sub i64
+; ALL: 'sub_i64'
+; ALL: estimated cost of 2 for {{.*}} sub i64
 define amdgpu_kernel void @sub_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %vaddr, i64 %b) #0 {
   %vec = load i64, i64 addrspace(1)* %vaddr
   %sub = sub i64 %vec, %b
   store i64 %sub, i64 addrspace(1)* %out
   ret void
 }
-; CHECK: 'sub_i16'
-; CHECK: estimated cost of 1 for {{.*}} sub i16
+; ALL: 'sub_i16'
+; ALL: estimated cost of 1 for {{.*}} sub i16
 define amdgpu_kernel void @sub_i16(i16 addrspace(1)* %out, i16 addrspace(1)* %vaddr, i16 %b) #0 {
   %vec = load i16, i16 addrspace(1)* %vaddr
   %sub = sub i16 %vec, %b
@@ -139,8 +143,9 @@ define amdgpu_kernel void @sub_i16(i16 addrspace(1)* %out, i16 addrspace(1)* %va
   ret void
 }
 
-; CHECK: 'sub_v2i16'
-; CHECK: estimated cost of 2 for {{.*}} sub <2 x i16>
+; ALL: 'sub_v2i16'
+; SLOW16: estimated cost of 2 for {{.*}} sub <2 x i16>
+; FAST16: estimated cost of 1 for {{.*}} sub <2 x i16>
 define amdgpu_kernel void @sub_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %vaddr, <2 x i16> %b) #0 {
   %vec = load <2 x i16>, <2 x i16> addrspace(1)* %vaddr
   %sub = sub <2 x i16> %vec, %b
diff --git a/llvm/test/Analysis/CostModel/AMDGPU/addrspacecast.ll b/llvm/test/Analysis/CostModel/AMDGPU/addrspacecast.ll
index f15ab500a..a87a965c6 100644
--- a/llvm/test/Analysis/CostModel/AMDGPU/addrspacecast.ll
+++ b/llvm/test/Analysis/CostModel/AMDGPU/addrspacecast.ll
@@ -1,4 +1,5 @@
 ; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa -mcpu=kaveri < %s | FileCheck %s
+; RUN: opt -cost-model -cost-kind=code-size -analyze -mtriple=amdgcn-unknown-amdhsa -mcpu=kaveri < %s | FileCheck %s
 
 ; CHECK-LABEL: 'addrspacecast_global_to_flat'
 ; CHECK: estimated cost of 0 for {{.*}} addrspacecast i8 addrspace(1)* %ptr to i8*
diff --git a/llvm/test/Analysis/CostModel/AMDGPU/bit-ops.ll b/llvm/test/Analysis/CostModel/AMDGPU/bit-ops.ll
index aa70f5032..2dec5f350 100644
--- a/llvm/test/Analysis/CostModel/AMDGPU/bit-ops.ll
+++ b/llvm/test/Analysis/CostModel/AMDGPU/bit-ops.ll
@@ -1,7 +1,10 @@
-; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa < %s | FileCheck %s
+; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa < %s | FileCheck -check-prefixes=ALL,SLOW16 %s
+; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa -mcpu=gfx900 < %s | FileCheck -check-prefixes=ALL,FAST16 %s
+; RUN: opt -cost-model -cost-kind=code-size -analyze -mtriple=amdgcn-unknown-amdhsa < %s | FileCheck -check-prefixes=ALL,SLOW16 %s
+; RUN: opt -cost-model -cost-kind=code-size -analyze -mtriple=amdgcn-unknown-amdhsa -mcpu=gfx900 < %s | FileCheck -check-prefixes=ALL,FAST16 %s
 
-; CHECK: 'or_i32'
-; CHECK: estimated cost of 1 for {{.*}} or i32
+; ALL: 'or_i32'
+; ALL: estimated cost of 1 for {{.*}} or i32
 define amdgpu_kernel void @or_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %vaddr, i32 %b) #0 {
   %vec = load i32, i32 addrspace(1)* %vaddr
   %or = or i32 %vec, %b
@@ -9,8 +12,8 @@ define amdgpu_kernel void @or_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %vad
   ret void
 }
 
-; CHECK: 'or_i64'
-; CHECK: estimated cost of 2 for {{.*}} or i64
+; ALL: 'or_i64'
+; ALL: estimated cost of 2 for {{.*}} or i64
 define amdgpu_kernel void @or_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %vaddr, i64 %b) #0 {
   %vec = load i64, i64 addrspace(1)* %vaddr
   %or = or i64 %vec, %b
@@ -18,8 +21,18 @@ define amdgpu_kernel void @or_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %vad
   ret void
 }
 
-; CHECK: 'xor_i32'
-; CHECK: estimated cost of 1 for {{.*}} xor i32
+; ALL: 'or_v2i16'
+; SLOW16: estimated cost of 2 for {{.*}} or <2 x i16>
+; FAST16: estimated cost of 1 for {{.*}} or <2 x i16>
+define amdgpu_kernel void @or_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %vaddr, <2 x i16> %b) #0 {
+  %vec = load <2 x i16>, <2 x i16> addrspace(1)* %vaddr
+  %or = or <2 x i16> %vec, %b
+  store <2 x i16> %or, <2 x i16> addrspace(1)* %out
+  ret void
+}
+
+; ALL: 'xor_i32'
+; ALL: estimated cost of 1 for {{.*}} xor i32
 define amdgpu_kernel void @xor_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %vaddr, i32 %b) #0 {
   %vec = load i32, i32 addrspace(1)* %vaddr
   %or = xor i32 %vec, %b
@@ -27,8 +40,8 @@ define amdgpu_kernel void @xor_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %va
   ret void
 }
 
-; CHECK: 'xor_i64'
-; CHECK: estimated cost of 2 for {{.*}} xor i64
+; ALL: 'xor_i64'
+; ALL: estimated cost of 2 for {{.*}} xor i64
 define amdgpu_kernel void @xor_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %vaddr, i64 %b) #0 {
   %vec = load i64, i64 addrspace(1)* %vaddr
   %or = xor i64 %vec, %b
@@ -36,9 +49,18 @@ define amdgpu_kernel void @xor_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %va
   ret void
 }
 
+; ALL: 'xor_v2i16'
+; SLOW16: estimated cost of 2 for {{.*}} xor <2 x i16>
+; FAST16: estimated cost of 1 for {{.*}} xor <2 x i16>
+define amdgpu_kernel void @xor_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %vaddr, <2 x i16> %b) #0 {
+  %vec = load <2 x i16>, <2 x i16> addrspace(1)* %vaddr
+  %xor = xor <2 x i16> %vec, %b
+  store <2 x i16> %xor, <2 x i16> addrspace(1)* %out
+  ret void
+}
 
-; CHECK: 'and_i32'
-; CHECK: estimated cost of 1 for {{.*}} and i32
+; ALL: 'and_i32'
+; ALL: estimated cost of 1 for {{.*}} and i32
 define amdgpu_kernel void @and_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %vaddr, i32 %b) #0 {
   %vec = load i32, i32 addrspace(1)* %vaddr
   %or = and i32 %vec, %b
@@ -46,8 +68,8 @@ define amdgpu_kernel void @and_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %va
   ret void
 }
 
-; CHECK: 'and_i64'
-; CHECK: estimated cost of 2 for {{.*}} and i64
+; ALL: 'and_i64'
+; ALL: estimated cost of 2 for {{.*}} and i64
 define amdgpu_kernel void @and_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %vaddr, i64 %b) #0 {
   %vec = load i64, i64 addrspace(1)* %vaddr
   %or = and i64 %vec, %b
@@ -55,5 +77,14 @@ define amdgpu_kernel void @and_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %va
   ret void
 }
 
+; ALL: 'and_v2i16'
+; SLOW16: estimated cost of 2 for {{.*}} and <2 x i16>
+; FAST16: estimated cost of 1 for {{.*}} and <2 x i16>
+define amdgpu_kernel void @and_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %vaddr, <2 x i16> %b) #0 {
+  %vec = load <2 x i16>, <2 x i16> addrspace(1)* %vaddr
+  %and = and <2 x i16> %vec, %b
+  store <2 x i16> %and, <2 x i16> addrspace(1)* %out
+  ret void
+}
 
 attributes #0 = { nounwind }
diff --git a/llvm/test/Analysis/CostModel/AMDGPU/extractelement.ll b/llvm/test/Analysis/CostModel/AMDGPU/extractelement.ll
index 889cc8bb8..67ce8ffba 100644
--- a/llvm/test/Analysis/CostModel/AMDGPU/extractelement.ll
+++ b/llvm/test/Analysis/CostModel/AMDGPU/extractelement.ll
@@ -1,6 +1,10 @@
 ; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa %s | FileCheck -check-prefixes=GCN,CI %s
-; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa -mcpu=fiji %s | FileCheck -check-prefixes=GCN,VI %s
-; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa -mcpu=gfx900 %s | FileCheck -check-prefixes=GCN,GFX9 %s
+; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa -mcpu=fiji %s | FileCheck -check-prefixes=GCN,GFX89 %s
+; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa -mcpu=gfx900 %s | FileCheck -check-prefixes=GCN,GFX89 %s
+; RUN: opt -cost-model -cost-kind=code-size -analyze -mtriple=amdgcn-unknown-amdhsa %s | FileCheck -check-prefixes=GCN,CI %s
+; RUN: opt -cost-model -cost-kind=code-size -analyze -mtriple=amdgcn-unknown-amdhsa -mcpu=fiji %s | FileCheck -check-prefixes=GCN,GFX89 %s
+; RUN: opt -cost-model -cost-kind=code-size -analyze -mtriple=amdgcn-unknown-amdhsa -mcpu=gfx900 %s | FileCheck -check-prefixes=GCN,GFX89 %s
+
 
 ; GCN: 'extractelement_v2i32'
 ; GCN: estimated cost of 0 for {{.*}} extractelement <2 x i32>
@@ -113,8 +117,7 @@ define amdgpu_kernel void @extractelement_v4i8(i8 addrspace(1)* %out, <4 x i8> a
 
 ; GCN: 'extractelement_0_v2i16':
 ; CI: estimated cost of 1 for {{.*}} extractelement <2 x i16> %vec, i16 0
-; VI: estimated cost of 0 for {{.*}} extractelement <2 x i16>
-; GFX9: estimated cost of 0 for {{.*}} extractelement <2 x i16>
+; GFX89: estimated cost of 0 for {{.*}} extractelement <2 x i16>
 define amdgpu_kernel void @extractelement_0_v2i16(i16 addrspace(1)* %out, <2 x i16> addrspace(1)* %vaddr) {
   %vec = load <2 x i16>, <2 x i16> addrspace(1)* %vaddr
   %elt = extractelement <2 x i16> %vec, i16 0
diff --git a/llvm/test/Analysis/CostModel/AMDGPU/fabs.ll b/llvm/test/Analysis/CostModel/AMDGPU/fabs.ll
index 51e65fe91..de5381c21 100644
--- a/llvm/test/Analysis/CostModel/AMDGPU/fabs.ll
+++ b/llvm/test/Analysis/CostModel/AMDGPU/fabs.ll
@@ -1,6 +1,7 @@
 ; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa < %s | FileCheck %s
+; RUN: opt -cost-model -cost-kind=code-size -analyze -mtriple=amdgcn-unknown-amdhsa < %s | FileCheck %s
 
-; CHECK: 'fabs_f32'
+; CHECK-LABEL: 'fabs_f32'
 ; CHECK: estimated cost of 0 for {{.*}} call float @llvm.fabs.f32
 define amdgpu_kernel void @fabs_f32(float addrspace(1)* %out, float addrspace(1)* %vaddr) #0 {
   %vec = load float, float addrspace(1)* %vaddr
@@ -9,7 +10,7 @@ define amdgpu_kernel void @fabs_f32(float addrspace(1)* %out, float addrspace(1)
   ret void
 }
 
-; CHECK: 'fabs_v2f32'
+; CHECK-LABEL: 'fabs_v2f32'
 ; CHECK: estimated cost of 0 for {{.*}} call <2 x float> @llvm.fabs.v2f32
 define amdgpu_kernel void @fabs_v2f32(<2 x float> addrspace(1)* %out, <2 x float> addrspace(1)* %vaddr) #0 {
   %vec = load <2 x float>, <2 x float> addrspace(1)* %vaddr
@@ -18,7 +19,7 @@ define amdgpu_kernel void @fabs_v2f32(<2 x float> addrspace(1)* %out, <2 x float
   ret void
 }
 
-; CHECK: 'fabs_v3f32'
+; CHECK-LABEL: 'fabs_v3f32'
 ; CHECK: estimated cost of 0 for {{.*}} call <3 x float> @llvm.fabs.v3f32
 define amdgpu_kernel void @fabs_v3f32(<3 x float> addrspace(1)* %out, <3 x float> addrspace(1)* %vaddr) #0 {
   %vec = load <3 x float>, <3 x float> addrspace(1)* %vaddr
@@ -27,7 +28,7 @@ define amdgpu_kernel void @fabs_v3f32(<3 x float> addrspace(1)* %out, <3 x float
   ret void
 }
 
-; CHECK: 'fabs_v5f32'
+; CHECK-LABEL: 'fabs_v5f32'
 ; CHECK: estimated cost of 0 for {{.*}} call <5 x float> @llvm.fabs.v5f32
 define amdgpu_kernel void @fabs_v5f32(<5 x float> addrspace(1)* %out, <5 x float> addrspace(1)* %vaddr) #0 {
   %vec = load <5 x float>, <5 x float> addrspace(1)* %vaddr
@@ -36,7 +37,7 @@ define amdgpu_kernel void @fabs_v5f32(<5 x float> addrspace(1)* %out, <5 x float
   ret void
 }
 
-; CHECK: 'fabs_f64'
+; CHECK-LABEL: 'fabs_f64'
 ; CHECK: estimated cost of 0 for {{.*}} call double @llvm.fabs.f64
 define amdgpu_kernel void @fabs_f64(double addrspace(1)* %out, double addrspace(1)* %vaddr) #0 {
   %vec = load double, double addrspace(1)* %vaddr
@@ -45,7 +46,7 @@ define amdgpu_kernel void @fabs_f64(double addrspace(1)* %out, double addrspace(
   ret void
 }
 
-; CHECK: 'fabs_v2f64'
+; CHECK-LABEL: 'fabs_v2f64'
 ; CHECK: estimated cost of 0 for {{.*}} call <2 x double> @llvm.fabs.v2f64
 define amdgpu_kernel void @fabs_v2f64(<2 x double> addrspace(1)* %out, <2 x double> addrspace(1)* %vaddr) #0 {
   %vec = load <2 x double>, <2 x double> addrspace(1)* %vaddr
@@ -54,7 +55,7 @@ define amdgpu_kernel void @fabs_v2f64(<2 x double> addrspace(1)* %out, <2 x doub
   ret void
 }
 
-; CHECK: 'fabs_v3f64'
+; CHECK-LABEL: 'fabs_v3f64'
 ; CHECK: estimated cost of 0 for {{.*}} call <3 x double> @llvm.fabs.v3f64
 define amdgpu_kernel void @fabs_v3f64(<3 x double> addrspace(1)* %out, <3 x double> addrspace(1)* %vaddr) #0 {
   %vec = load <3 x double>, <3 x double> addrspace(1)* %vaddr
@@ -63,7 +64,7 @@ define amdgpu_kernel void @fabs_v3f64(<3 x double> addrspace(1)* %out, <3 x doub
   ret void
 }
 
-; CHECK: 'fabs_f16'
+; CHECK-LABEL: 'fabs_f16'
 ; CHECK: estimated cost of 0 for {{.*}} call half @llvm.fabs.f16
 define amdgpu_kernel void @fabs_f16(half addrspace(1)* %out, half addrspace(1)* %vaddr) #0 {
   %vec = load half, half addrspace(1)* %vaddr
@@ -72,7 +73,7 @@ define amdgpu_kernel void @fabs_f16(half addrspace(1)* %out, half addrspace(1)*
   ret void
 }
 
-; CHECK: 'fabs_v2f16'
+; CHECK-LABEL: 'fabs_v2f16'
 ; CHECK: estimated cost of 0 for {{.*}} call <2 x half> @llvm.fabs.v2f16
 define amdgpu_kernel void @fabs_v2f16(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %vaddr) #0 {
   %vec = load <2 x half>, <2 x half> addrspace(1)* %vaddr
@@ -81,7 +82,7 @@ define amdgpu_kernel void @fabs_v2f16(<2 x half> addrspace(1)* %out, <2 x half>
   ret void
 }
 
-; CHECK: 'fabs_v3f16'
+; CHECK-LABEL: 'fabs_v3f16'
 ; CHECK: estimated cost of 0 for {{.*}} call <3 x half> @llvm.fabs.v3f16
 define amdgpu_kernel void @fabs_v3f16(<3 x half> addrspace(1)* %out, <3 x half> addrspace(1)* %vaddr) #0 {
   %vec = load <3 x half>, <3 x half> addrspace(1)* %vaddr
diff --git a/llvm/test/Analysis/CostModel/AMDGPU/fadd.ll b/llvm/test/Analysis/CostModel/AMDGPU/fadd.ll
index 5a5a09419..1203182a8 100644
--- a/llvm/test/Analysis/CostModel/AMDGPU/fadd.ll
+++ b/llvm/test/Analysis/CostModel/AMDGPU/fadd.ll
@@ -1,5 +1,7 @@
-; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa -mattr=+half-rate-64-ops < %s | FileCheck -check-prefix=FASTF64 -check-prefix=ALL %s
-; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa -mattr=-half-rate-64-ops < %s | FileCheck -check-prefix=SLOWF64 -check-prefix=ALL %s
+; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa -mcpu=gfx900  -mattr=+half-rate-64-ops < %s | FileCheck -check-prefixes=FASTF64,FASTF16,ALL %s
+; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa -mattr=-half-rate-64-ops < %s | FileCheck -check-prefixes=SLOWF64,SLOWF16,ALL %s
+; RUN: opt -cost-model -cost-kind=code-size -analyze -mtriple=amdgcn-unknown-amdhsa -mcpu=gfx900 -mattr=+half-rate-64-ops < %s | FileCheck -check-prefixes=FASTF64,FASTF16,ALL %s
+; RUN: opt -cost-model -cost-kind=code-size -analyze -mtriple=amdgcn-unknown-amdhsa -mattr=-half-rate-64-ops < %s | FileCheck -check-prefixes=SLOWF64,SLOWF16,ALL %s
 
 ; ALL: 'fadd_f32'
 ; ALL: estimated cost of 1 for {{.*}} fadd float
@@ -71,8 +73,8 @@ define amdgpu_kernel void @fadd_v3f64(<3 x double> addrspace(1)* %out, <3 x doub
   ret void
 }
 
-; ALL 'fadd_f16'
-; ALL estimated cost of 1 for {{.*}} fadd half
+; ALL: 'fadd_f16'
+; ALL: estimated cost of 1 for {{.*}} fadd half
 define amdgpu_kernel void @fadd_f16(half addrspace(1)* %out, half addrspace(1)* %vaddr, half %b) #0 {
   %vec = load half, half addrspace(1)* %vaddr
   %add = fadd half %vec, %b
@@ -80,8 +82,9 @@ define amdgpu_kernel void @fadd_f16(half addrspace(1)* %out, half addrspace(1)*
   ret void
 }
 
-; ALL 'fadd_v2f16'
-; ALL estimated cost of 2 for {{.*}} fadd <2 x half>
+; ALL: 'fadd_v2f16'
+; SLOWF16: estimated cost of 2 for {{.*}} fadd <2 x half>
+; FASTF16: estimated cost of 1 for {{.*}} fadd <2 x half>
 define amdgpu_kernel void @fadd_v2f16(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %vaddr, <2 x half> %b) #0 {
   %vec = load <2 x half>, <2 x half> addrspace(1)* %vaddr
   %add = fadd <2 x half> %vec, %b
@@ -89,8 +92,19 @@ define amdgpu_kernel void @fadd_v2f16(<2 x half> addrspace(1)* %out, <2 x half>
   ret void
 }
 
-; ALL 'fadd_v4f16'
-; ALL estimated cost of 4 for {{.*}} fadd <4 x half>
+; ALL: 'fadd_v3f16'
+; SLOWF16: estimated cost of 4 for {{.*}} fadd <3 x half>
+; FASTF16: estimated cost of 2 for {{.*}} fadd <3 x half>
+define amdgpu_kernel void @fadd_v3f16(<3 x half> addrspace(1)* %out, <3 x half> addrspace(1)* %vaddr, <3 x half> %b) #0 {
+  %vec = load <3 x half>, <3 x half> addrspace(1)* %vaddr
+  %add = fadd <3 x half> %vec, %b
+  store <3 x half> %add, <3 x half> addrspace(1)* %out
+  ret void
+}
+
+; ALL: 'fadd_v4f16'
+; SLOWF16: estimated cost of 4 for {{.*}} fadd <4 x half>
+; FASTF16: estimated cost of 2 for {{.*}} fadd <4 x half>
 define amdgpu_kernel void @fadd_v4f16(<4 x half> addrspace(1)* %out, <4 x half> addrspace(1)* %vaddr, <4 x half> %b) #0 {
   %vec = load <4 x half>, <4 x half> addrspace(1)* %vaddr
   %add = fadd <4 x half> %vec, %b
diff --git a/llvm/test/Analysis/CostModel/AMDGPU/fdiv.ll b/llvm/test/Analysis/CostModel/AMDGPU/fdiv.ll
index 325960ac9..6986a3158 100644
--- a/llvm/test/Analysis/CostModel/AMDGPU/fdiv.ll
+++ b/llvm/test/Analysis/CostModel/AMDGPU/fdiv.ll
@@ -1,48 +1,91 @@
-; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa -mcpu=hawaii -mattr=+half-rate-64-ops < %s | FileCheck -check-prefixes=ALL,CIFASTF64,NOFP32DENORM,NOFP16,NOFP16-NOFP32DENORM %s
-; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa -mcpu=kaveri -mattr=-half-rate-64-ops < %s | FileCheck -check-prefixes=ALL,CISLOWF64,NOFP32DENORM,NOFP16,NOFP16-NOFP32DENORM  %s
-; RUN: opt -cost-model -analyze -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -mattr=+half-rate-64-ops < %s | FileCheck -check-prefixes=ALL,SIFASTF64,NOFP32DENORM,NOFP16,NOFP16-NOFP32DENORM  %s
-; RUN: opt -cost-model -analyze -mtriple=amdgcn-mesa-mesa3d -mcpu=verde -mattr=-half-rate-64-ops < %s | FileCheck -check-prefixes=ALL,SISLOWF64,NOFP32DENORM,NOFP16,NOFP16-NOFP32DENORM  %s
-; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa -mcpu=hawaii -mattr=+fp32-denormals < %s | FileCheck -check-prefixes=ALL,FP32DENORMS,SLOWFP32DENORMS,NOFP16,NOFP16-FP32DENORM %s
-; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa -mcpu=gfx900 -mattr=+fp32-denormals < %s | FileCheck -check-prefixes=ALL,FP32DENORMS,FASTFP32DENORMS,FP16 %s
-
-; ALL: 'fdiv_f32'
-; NOFP32DENORM: estimated cost of 12 for {{.*}} fdiv float
-; FP32DENORMS: estimated cost of 10 for {{.*}} fdiv float
-define amdgpu_kernel void @fdiv_f32(float addrspace(1)* %out, float addrspace(1)* %vaddr, float %b) #0 {
+; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa -mcpu=hawaii < %s | FileCheck -check-prefixes=ALL,CIFASTF64,NOFP16,NOFP16-NOFP32DENORM,SLOWFP32DENORMS %s
+; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa -mcpu=kaveri < %s | FileCheck -check-prefixes=ALL,CISLOWF64,NOFP16,NOFP16-NOFP32DENORM,SLOWFP32DENORMS  %s
+; RUN: opt -cost-model -analyze -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti < %s | FileCheck -check-prefixes=ALL,SIFASTF64,NOFP32DENORM,NOFP16,NOFP16-NOFP32DENORM,SLOWFP32DENORMS  %s
+; RUN: opt -cost-model -analyze -mtriple=amdgcn-mesa-mesa3d -mcpu=verde < %s | FileCheck -check-prefixes=ALL,SISLOWF64,NOFP32DENORM,NOFP16,NOFP16-NOFP32DENORM,SLOWFP32DENORMS  %s
+; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa -mcpu=hawaii < %s | FileCheck -check-prefixes=ALL,NOFP16,NOFP16-FP32DENORM,SLOWFP32DENORMS %s
+; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa -mcpu=gfx900 < %s | FileCheck -check-prefixes=ALL,FASTFP32DENORMS,FP16 %s
+
+; RUN: opt -cost-model -cost-kind=code-size -analyze -mtriple=amdgcn-unknown-amdhsa -mcpu=hawaii < %s | FileCheck -check-prefixes=ALL,CIFASTF64,NOFP16,NOFP16-NOFP32DENORM %s
+; RUN: opt -cost-model -cost-kind=code-size -analyze -mtriple=amdgcn-unknown-amdhsa -mcpu=kaveri < %s | FileCheck -check-prefixes=ALL,CISLOWF64,NOFP16,NOFP16-NOFP32DENORM  %s
+; RUN: opt -cost-model -cost-kind=code-size -analyze -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti < %s | FileCheck -check-prefixes=ALL,SIFASTF64,NOFP16,NOFP16-NOFP32DENORM  %s
+; RUN: opt -cost-model -cost-kind=code-size -analyze -mtriple=amdgcn-mesa-mesa3d -mcpu=verde < %s | FileCheck -check-prefixes=ALL,SISLOWF64,NOFP16,NOFP16-NOFP32DENORM  %s
+; RUN: opt -cost-model -cost-kind=code-size -analyze -mtriple=amdgcn-unknown-amdhsa -mcpu=hawaii < %s | FileCheck -check-prefixes=ALL,SLOWFP32DENORMS,NOFP16,NOFP16-FP32DENORM %s
+; RUN: opt -cost-model -cost-kind=code-size -analyze -mtriple=amdgcn-unknown-amdhsa -mcpu=gfx900 < %s | FileCheck -check-prefixes=ALL,FASTFP32DENORMS,FP16 %s
+
+; ALL: 'fdiv_f32_ieee'
+; ALL: estimated cost of 10 for {{.*}} fdiv float
+define amdgpu_kernel void @fdiv_f32_ieee(float addrspace(1)* %out, float addrspace(1)* %vaddr, float %b) #0 {
+  %vec = load float, float addrspace(1)* %vaddr
+  %add = fdiv float %vec, %b
+  store float %add, float addrspace(1)* %out
+  ret void
+}
+
+; ALL: 'fdiv_f32_ftzdaz'
+; ALL: estimated cost of 12 for {{.*}} fdiv float
+define amdgpu_kernel void @fdiv_f32_ftzdaz(float addrspace(1)* %out, float addrspace(1)* %vaddr, float %b) #1 {
   %vec = load float, float addrspace(1)* %vaddr
   %add = fdiv float %vec, %b
   store float %add, float addrspace(1)* %out
   ret void
 }
 
-; ALL: 'fdiv_v2f32'
-; NOFP32DENORM: estimated cost of 24 for {{.*}} fdiv <2 x float>
-; FP32DENORMS: estimated cost of 20 for {{.*}} fdiv <2 x float>
-define amdgpu_kernel void @fdiv_v2f32(<2 x float> addrspace(1)* %out, <2 x float> addrspace(1)* %vaddr, <2 x float> %b) #0 {
+; ALL: 'fdiv_v2f32_ieee'
+; ALL: estimated cost of 20 for {{.*}} fdiv <2 x float>
+define amdgpu_kernel void @fdiv_v2f32_ieee(<2 x float> addrspace(1)* %out, <2 x float> addrspace(1)* %vaddr, <2 x float> %b) #0 {
+  %vec = load <2 x float>, <2 x float> addrspace(1)* %vaddr
+  %add = fdiv <2 x float> %vec, %b
+  store <2 x float> %add, <2 x float> addrspace(1)* %out
+  ret void
+}
+
+; ALL: 'fdiv_v2f32_ftzdaz'
+; ALL: estimated cost of 24 for {{.*}} fdiv <2 x float>
+define amdgpu_kernel void @fdiv_v2f32_ftzdaz(<2 x float> addrspace(1)* %out, <2 x float> addrspace(1)* %vaddr, <2 x float> %b) #1 {
   %vec = load <2 x float>, <2 x float> addrspace(1)* %vaddr
   %add = fdiv <2 x float> %vec, %b
   store <2 x float> %add, <2 x float> addrspace(1)* %out
   ret void
 }
 
-; ALL: 'fdiv_v3f32'
+; ALL: 'fdiv_v3f32_ieee'
 ; Allow for 48/40 when v3f32 is illegal and TargetLowering thinks it needs widening,
 ; and 36/30 when it is legal.
-; NOFP32DENORM: estimated cost of {{36|48}} for {{.*}} fdiv <3 x float>
-; FP32DENORMS: estimated cost of {{30|40}} for {{.*}} fdiv <3 x float>
-define amdgpu_kernel void @fdiv_v3f32(<3 x float> addrspace(1)* %out, <3 x float> addrspace(1)* %vaddr, <3 x float> %b) #0 {
+; ALL: estimated cost of {{30|40}} for {{.*}} fdiv <3 x float>
+define amdgpu_kernel void @fdiv_v3f32_ieee(<3 x float> addrspace(1)* %out, <3 x float> addrspace(1)* %vaddr, <3 x float> %b) #0 {
   %vec = load <3 x float>, <3 x float> addrspace(1)* %vaddr
   %add = fdiv <3 x float> %vec, %b
   store <3 x float> %add, <3 x float> addrspace(1)* %out
   ret void
 }
 
-; ALL: 'fdiv_v5f32'
+; ALL: 'fdiv_v3f32_ftzdaz'
+; Allow for 48/40 when v3f32 is illegal and TargetLowering thinks it needs widening,
+; and 36/30 when it is legal.
+; ALL: estimated cost of {{36|48}} for {{.*}} fdiv <3 x float>
+define amdgpu_kernel void @fdiv_v3f32_ftzdaz(<3 x float> addrspace(1)* %out, <3 x float> addrspace(1)* %vaddr, <3 x float> %b) #1 {
+  %vec = load <3 x float>, <3 x float> addrspace(1)* %vaddr
+  %add = fdiv <3 x float> %vec, %b
+  store <3 x float> %add, <3 x float> addrspace(1)* %out
+  ret void
+}
+
+; ALL: 'fdiv_v5f32_ieee'
+; Allow for 96/80 when v5f32 is illegal and TargetLowering thinks it needs widening,
+; and 60/50 when it is legal.
+; ALL: estimated cost of {{80|50}} for {{.*}} fdiv <5 x float>
+define amdgpu_kernel void @fdiv_v5f32_ieee(<5 x float> addrspace(1)* %out, <5 x float> addrspace(1)* %vaddr, <5 x float> %b) #0 {
+  %vec = load <5 x float>, <5 x float> addrspace(1)* %vaddr
+  %add = fdiv <5 x float> %vec, %b
+  store <5 x float> %add, <5 x float> addrspace(1)* %out
+  ret void
+}
+
+; ALL: 'fdiv_v5f32_ftzdaz'
 ; Allow for 96/80 when v5f32 is illegal and TargetLowering thinks it needs widening,
 ; and 60/50 when it is legal.
-; NOFP32DENORM: estimated cost of {{96|60}} for {{.*}} fdiv <5 x float>
-; FP32DENORMS: estimated cost of {{80|50}} for {{.*}} fdiv <5 x float>
-define amdgpu_kernel void @fdiv_v5f32(<5 x float> addrspace(1)* %out, <5 x float> addrspace(1)* %vaddr, <5 x float> %b) #0 {
+; ALL: estimated cost of {{96|60}} for {{.*}} fdiv <5 x float>
+define amdgpu_kernel void @fdiv_v5f32_ftzdaz(<5 x float> addrspace(1)* %out, <5 x float> addrspace(1)* %vaddr, <5 x float> %b) #1 {
   %vec = load <5 x float>, <5 x float> addrspace(1)* %vaddr
   %add = fdiv <5 x float> %vec, %b
   store <5 x float> %add, <5 x float> addrspace(1)* %out
@@ -85,55 +128,99 @@ define amdgpu_kernel void @fdiv_v3f64(<3 x double> addrspace(1)* %out, <3 x doub
   ret void
 }
 
-; ALL: 'fdiv_f16'
-; NOFP16-NOFP32DENORM: estimated cost of 12 for {{.*}} fdiv half
-; NOFP16-FP32DENORM: estimated cost of 10 for {{.*}} fdiv half
+; ALL: 'fdiv_f16_f32_ieee'
+; NOFP16: estimated cost of 10 for {{.*}} fdiv half
 ; FP16: estimated cost of 10 for {{.*}} fdiv half
-define amdgpu_kernel void @fdiv_f16(half addrspace(1)* %out, half addrspace(1)* %vaddr, half %b) #0 {
+define amdgpu_kernel void @fdiv_f16_f32_ieee(half addrspace(1)* %out, half addrspace(1)* %vaddr, half %b) #0 {
   %vec = load half, half addrspace(1)* %vaddr
   %add = fdiv half %vec, %b
   store half %add, half addrspace(1)* %out
   ret void
 }
 
-; ALL: 'fdiv_v2f16'
-; NOFP16-NOFP32DENORM: estimated cost of 24 for {{.*}} fdiv <2 x half>
-; NOFP16-FP32DENORM: estimated cost of 20 for {{.*}} fdiv <2 x half>
+; ALL: 'fdiv_f16_f32_ftzdaz'
+; NOFP16: estimated cost of 12 for {{.*}} fdiv half
+; FP16: estimated cost of 10 for {{.*}} fdiv half
+define amdgpu_kernel void @fdiv_f16_f32_ftzdaz(half addrspace(1)* %out, half addrspace(1)* %vaddr, half %b) #1 {
+  %vec = load half, half addrspace(1)* %vaddr
+  %add = fdiv half %vec, %b
+  store half %add, half addrspace(1)* %out
+  ret void
+}
+
+; ALL: 'fdiv_v2f16_f32_ieee'
+; NOFP16: estimated cost of 20 for {{.*}} fdiv <2 x half>
 ; FP16: estimated cost of 20 for {{.*}} fdiv <2 x half>
-define amdgpu_kernel void @fdiv_v2f16(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %vaddr, <2 x half> %b) #0 {
+define amdgpu_kernel void @fdiv_v2f16_f32_ieee(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %vaddr, <2 x half> %b) #0 {
   %vec = load <2 x half>, <2 x half> addrspace(1)* %vaddr
   %add = fdiv <2 x half> %vec, %b
   store <2 x half> %add, <2 x half> addrspace(1)* %out
   ret void
 }
 
-; ALL: 'fdiv_v4f16'
-; NOFP16-NOFP32DENORM: estimated cost of 48 for {{.*}} fdiv <4 x half>
-; NOFP16-FP32DENORM: estimated cost of 40 for {{.*}} fdiv <4 x half>
+; ALL: 'fdiv_v2f16_f32_ftzdaz'
+; NOFP16: estimated cost of 24 for {{.*}} fdiv <2 x half>
+; FP16: estimated cost of 20 for {{.*}} fdiv <2 x half>
+define amdgpu_kernel void @fdiv_v2f16_f32_ftzdaz(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %vaddr, <2 x half> %b) #1 {
+  %vec = load <2 x half>, <2 x half> addrspace(1)* %vaddr
+  %add = fdiv <2 x half> %vec, %b
+  store <2 x half> %add, <2 x half> addrspace(1)* %out
+  ret void
+}
+
+; ALL: 'fdiv_v4f16_f32_ieee'
+; NOFP16: estimated cost of 40 for {{.*}} fdiv <4 x half>
 ; FP16: estimated cost of 40 for {{.*}} fdiv <4 x half>
-define amdgpu_kernel void @fdiv_v4f16(<4 x half> addrspace(1)* %out, <4 x half> addrspace(1)* %vaddr, <4 x half> %b) #0 {
+define amdgpu_kernel void @fdiv_v4f16_f32_ieee(<4 x half> addrspace(1)* %out, <4 x half> addrspace(1)* %vaddr, <4 x half> %b) #0 {
   %vec = load <4 x half>, <4 x half> addrspace(1)* %vaddr
   %add = fdiv <4 x half> %vec, %b
   store <4 x half> %add, <4 x half> addrspace(1)* %out
   ret void
 }
 
-; ALL: 'rcp_f32'
-; NOFP32DENORM: estimated cost of 3 for {{.*}} fdiv float
+; ALL: 'fdiv_v4f16_f32_ftzdaz'
+; NOFP16: estimated cost of 48 for {{.*}} fdiv <4 x half>
+; FP16: estimated cost of 40 for {{.*}} fdiv <4 x half>
+define amdgpu_kernel void @fdiv_v4f16_f32_ftzdaz(<4 x half> addrspace(1)* %out, <4 x half> addrspace(1)* %vaddr, <4 x half> %b) #1 {
+  %vec = load <4 x half>, <4 x half> addrspace(1)* %vaddr
+  %add = fdiv <4 x half> %vec, %b
+  store <4 x half> %add, <4 x half> addrspace(1)* %out
+  ret void
+}
+
+; ALL: 'rcp_f32_ieee'
 ; SLOWFP32DENORMS: estimated cost of 10 for {{.*}} fdiv float
 ; FASTFP32DENORMS: estimated cost of 10 for {{.*}} fdiv float
-define amdgpu_kernel void @rcp_f32(float addrspace(1)* %out, float addrspace(1)* %vaddr) #0 {
+define amdgpu_kernel void @rcp_f32_ieee(float addrspace(1)* %out, float addrspace(1)* %vaddr) #0 {
   %vec = load float, float addrspace(1)* %vaddr
   %add = fdiv float 1.0, %vec
   store float %add, float addrspace(1)* %out
   ret void
 }
 
-; ALL: 'rcp_f16'
-; NOFP16-NOFP32DENORM: estimated cost of 3 for {{.*}} fdiv half
-; NOFP16-FP32DENORM: estimated cost of 10 for {{.*}} fdiv half
+; ALL: 'rcp_f32_ftzdaz'
+; ALL: estimated cost of 3 for {{.*}} fdiv float
+define amdgpu_kernel void @rcp_f32_ftzdaz(float addrspace(1)* %out, float addrspace(1)* %vaddr) #1 {
+  %vec = load float, float addrspace(1)* %vaddr
+  %add = fdiv float 1.0, %vec
+  store float %add, float addrspace(1)* %out
+  ret void
+}
+
+; ALL: 'rcp_f16_f32_ieee'
+; NOFP16: estimated cost of 10 for {{.*}} fdiv half
+; FP16: estimated cost of 3 for {{.*}} fdiv half
+define amdgpu_kernel void @rcp_f16_f32_ieee(half addrspace(1)* %out, half addrspace(1)* %vaddr) #0 {
+  %vec = load half, half addrspace(1)* %vaddr
+  %add = fdiv half 1.0, %vec
+  store half %add, half addrspace(1)* %out
+  ret void
+}
+
+; ALL: 'rcp_f16_f32_ftzdaz'
+; NOFP16: estimated cost of 3 for {{.*}} fdiv half
 ; FP16: estimated cost of 3 for {{.*}} fdiv half
-define amdgpu_kernel void @rcp_f16(half addrspace(1)* %out, half addrspace(1)* %vaddr) #0 {
+define amdgpu_kernel void @rcp_f16_f32_ftzdaz(half addrspace(1)* %out, half addrspace(1)* %vaddr) #1 {
   %vec = load half, half addrspace(1)* %vaddr
   %add = fdiv half 1.0, %vec
   store half %add, half addrspace(1)* %out
@@ -152,26 +239,44 @@ define amdgpu_kernel void @rcp_f64(double addrspace(1)* %out, double addrspace(1
   ret void
 }
 
-; ALL: 'rcp_v2f32'
-; NOFP32DENORM: estimated cost of 6 for {{.*}} fdiv <2 x float>
+; ALL: 'rcp_v2f32_ieee'
 ; SLOWFP32DENORMS: estimated cost of 20 for {{.*}} fdiv <2 x float>
 ; FASTFP32DENORMS: estimated cost of 20 for {{.*}} fdiv <2 x float>
-define amdgpu_kernel void @rcp_v2f32(<2 x float> addrspace(1)* %out, <2 x float> addrspace(1)* %vaddr) #0 {
+define amdgpu_kernel void @rcp_v2f32_ieee(<2 x float> addrspace(1)* %out, <2 x float> addrspace(1)* %vaddr) #0 {
   %vec = load <2 x float>, <2 x float> addrspace(1)* %vaddr
   %add = fdiv <2 x float> <float 1.0, float 1.0>, %vec
   store <2 x float> %add, <2 x float> addrspace(1)* %out
   ret void
 }
 
-; ALL: 'rcp_v2f16'
-; NOFP16-NOFP32DENORM: estimated cost of 6 for {{.*}} fdiv <2 x half>
-; NOFP16-FP32DENORM: estimated cost of 20 for {{.*}} fdiv <2 x half>
+; ALL: 'rcp_v2f32_ftzdaz'
+; ALL: estimated cost of 6 for {{.*}} fdiv <2 x float>
+define amdgpu_kernel void @rcp_v2f32_ftzdaz(<2 x float> addrspace(1)* %out, <2 x float> addrspace(1)* %vaddr) #1 {
+  %vec = load <2 x float>, <2 x float> addrspace(1)* %vaddr
+  %add = fdiv <2 x float> <float 1.0, float 1.0>, %vec
+  store <2 x float> %add, <2 x float> addrspace(1)* %out
+  ret void
+}
+
+; ALL: 'rcp_v2f16_f32_ieee'
+; NOFP16: estimated cost of 20 for {{.*}} fdiv <2 x half>
+; FP16: estimated cost of 6 for {{.*}} fdiv <2 x half>
+define amdgpu_kernel void @rcp_v2f16_f32_ieee(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %vaddr) #0 {
+  %vec = load <2 x half>, <2 x half> addrspace(1)* %vaddr
+  %add = fdiv <2 x half> <half 1.0, half 1.0>, %vec
+  store <2 x half> %add, <2 x half> addrspace(1)* %out
+  ret void
+}
+
+; ALL: 'rcp_v2f16_f32_ftzdaz'
+; NOFP16: estimated cost of 6 for {{.*}} fdiv <2 x half>
 ; FP16: estimated cost of 6 for {{.*}} fdiv <2 x half>
-define amdgpu_kernel void @rcp_v2f16(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %vaddr) #0 {
+define amdgpu_kernel void @rcp_v2f16_f32_ftzdaz(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %vaddr) #1 {
   %vec = load <2 x half>, <2 x half> addrspace(1)* %vaddr
   %add = fdiv <2 x half> <half 1.0, half 1.0>, %vec
   store <2 x half> %add, <2 x half> addrspace(1)* %out
   ret void
 }
 
-attributes #0 = { nounwind }
+attributes #0 = { nounwind "target-features"="+fp32-denormals" }
+attributes #1 = { nounwind "target-features"="-fp32-denormals" }
diff --git a/llvm/test/Analysis/CostModel/AMDGPU/fma.ll b/llvm/test/Analysis/CostModel/AMDGPU/fma.ll
new file mode 100644
index 000000000..462163d2f
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/AMDGPU/fma.ll
@@ -0,0 +1,120 @@
+; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa -mcpu=gfx900  -mattr=+half-rate-64-ops < %s | FileCheck -check-prefixes=FASTF64,FAST32,FASTF16,ALL %s
+; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa -mattr=-half-rate-64-ops < %s | FileCheck -check-prefixes=SLOWF64,SLOW32,SLOWF16,ALL %s
+; RUN: opt -cost-model -cost-kind=code-size -analyze -mtriple=amdgcn-unknown-amdhsa -mcpu=gfx900 -mattr=+half-rate-64-ops < %s | FileCheck -check-prefixes=FASTF64,FAST32,FASTF16,ALL %s
+; RUN: opt -cost-model -cost-kind=code-size -analyze -mtriple=amdgcn-unknown-amdhsa -mattr=-half-rate-64-ops < %s | FileCheck -check-prefixes=SLOWF64,SLOW32,SLOWF16,ALL %s
+
+; ALL-LABEL: 'fma_f32'
+; SLOW32: estimated cost of 3 for {{.*}} call float @llvm.fma.f32
+; FAST32: estimated cost of 2 for {{.*}} call float @llvm.fma.f32
+define amdgpu_kernel void @fma_f32(float addrspace(1)* %out, float addrspace(1)* %vaddr) #0 {
+  %vec = load float, float addrspace(1)* %vaddr
+  %fma = call float @llvm.fma.f32(float %vec, float %vec, float %vec) #1
+  store float %fma, float addrspace(1)* %out
+  ret void
+}
+
+; ALL-LABEL: 'fma_v2f32'
+; SLOW32: estimated cost of 6 for {{.*}} call <2 x float> @llvm.fma.v2f32
+; FAST32: estimated cost of 4 for {{.*}} call <2 x float> @llvm.fma.v2f32
+define amdgpu_kernel void @fma_v2f32(<2 x float> addrspace(1)* %out, <2 x float> addrspace(1)* %vaddr) #0 {
+  %vec = load <2 x float>, <2 x float> addrspace(1)* %vaddr
+  %fma = call <2 x float> @llvm.fma.v2f32(<2 x float> %vec, <2 x float> %vec, <2 x float> %vec) #1
+  store <2 x float> %fma, <2 x float> addrspace(1)* %out
+  ret void
+}
+
+; ALL-LABEL: 'fma_v3f32'
+; SLOW32: estimated cost of 9 for {{.*}} call <3 x float> @llvm.fma.v3f32
+; FAST32: estimated cost of 6 for {{.*}} call <3 x float> @llvm.fma.v3f32
+define amdgpu_kernel void @fma_v3f32(<3 x float> addrspace(1)* %out, <3 x float> addrspace(1)* %vaddr) #0 {
+  %vec = load <3 x float>, <3 x float> addrspace(1)* %vaddr
+  %fma = call <3 x float> @llvm.fma.v3f32(<3 x float> %vec, <3 x float> %vec, <3 x float> %vec) #1
+  store <3 x float> %fma, <3 x float> addrspace(1)* %out
+  ret void
+}
+
+; ALL-LABEL: 'fma_v5f32'
+; SLOW32: estimated cost of 15 for {{.*}} call <5 x float> @llvm.fma.v5f32
+; FAST32: estimated cost of 10 for {{.*}} call <5 x float> @llvm.fma.v5f32
+define amdgpu_kernel void @fma_v5f32(<5 x float> addrspace(1)* %out, <5 x float> addrspace(1)* %vaddr) #0 {
+  %vec = load <5 x float>, <5 x float> addrspace(1)* %vaddr
+  %fma = call <5 x float> @llvm.fma.v5f32(<5 x float> %vec, <5 x float> %vec, <5 x float> %vec) #1
+  store <5 x float> %fma, <5 x float> addrspace(1)* %out
+  ret void
+}
+
+; ALL-LABEL: 'fma_f64'
+; SLOW64: estimated cost of 3 for {{.*}} call double @llvm.fma.f64
+; FAST64: estimated cost of 2 for {{.*}} call double @llvm.fma.f64
+define amdgpu_kernel void @fma_f64(double addrspace(1)* %out, double addrspace(1)* %vaddr) #0 {
+  %vec = load double, double addrspace(1)* %vaddr
+  %fma = call double @llvm.fma.f64(double %vec, double %vec, double %vec) #1
+  store double %fma, double addrspace(1)* %out
+  ret void
+}
+
+; ALL-LABEL: 'fma_v2f64'
+; SLOW64: estimated cost of 6 for {{.*}} call <2 x double> @llvm.fma.v2f64
+; FAST64: estimated cost of 4 for {{.*}} call <2 x double> @llvm.fma.v2f64
+define amdgpu_kernel void @fma_v2f64(<2 x double> addrspace(1)* %out, <2 x double> addrspace(1)* %vaddr) #0 {
+  %vec = load <2 x double>, <2 x double> addrspace(1)* %vaddr
+  %fma = call <2 x double> @llvm.fma.v2f64(<2 x double> %vec, <2 x double> %vec, <2 x double> %vec) #1
+  store <2 x double> %fma, <2 x double> addrspace(1)* %out
+  ret void
+}
+
+; ALL-LABEL: 'fma_v3f64'
+; SLOW64: estimated cost of 9 for {{.*}} call <3 x double> @llvm.fma.v3f64
+; FAST64: estimated cost of 6 for {{.*}} call <3 x double> @llvm.fma.v3f64
+define amdgpu_kernel void @fma_v3f64(<3 x double> addrspace(1)* %out, <3 x double> addrspace(1)* %vaddr) #0 {
+  %vec = load <3 x double>, <3 x double> addrspace(1)* %vaddr
+  %fma = call <3 x double> @llvm.fma.v3f64(<3 x double> %vec, <3 x double> %vec, <3 x double> %vec) #1
+  store <3 x double> %fma, <3 x double> addrspace(1)* %out
+  ret void
+}
+
+; ALL-LABEL: 'fma_f16'
+; SLOW16: estimated cost of 3 for {{.*}} call half @llvm.fma.f16
+; FAST16: estimated cost of 2 for {{.*}} call half @llvm.fma.f16
+define amdgpu_kernel void @fma_f16(half addrspace(1)* %out, half addrspace(1)* %vaddr) #0 {
+  %vec = load half, half addrspace(1)* %vaddr
+  %fma = call half @llvm.fma.f16(half %vec, half %vec, half %vec) #1
+  store half %fma, half addrspace(1)* %out
+  ret void
+}
+
+; ALL-LABEL: 'fma_v2f16'
+; SLOW16: estimated cost of 6 for {{.*}} call <2 x half> @llvm.fma.v2f16
+; FAST16: estimated cost of 2 for {{.*}} call <2 x half> @llvm.fma.v2f16
+define amdgpu_kernel void @fma_v2f16(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %vaddr) #0 {
+  %vec = load <2 x half>, <2 x half> addrspace(1)* %vaddr
+  %fma = call <2 x half> @llvm.fma.v2f16(<2 x half> %vec, <2 x half> %vec, <2 x half> %vec) #1
+  store <2 x half> %fma, <2 x half> addrspace(1)* %out
+  ret void
+}
+
+; ALL-LABEL: 'fma_v3f16'
+; SLOW16: estimated cost of 12 for {{.*}} call <3 x half> @llvm.fma.v3f16
+; FAST16: estimated cost of 4 for {{.*}} call <3 x half> @llvm.fma.v3f16
+define amdgpu_kernel void @fma_v3f16(<3 x half> addrspace(1)* %out, <3 x half> addrspace(1)* %vaddr) #0 {
+  %vec = load <3 x half>, <3 x half> addrspace(1)* %vaddr
+  %fma = call <3 x half> @llvm.fma.v3f16(<3 x half> %vec, <3 x half> %vec, <3 x half> %vec) #1
+  store <3 x half> %fma, <3 x half> addrspace(1)* %out
+  ret void
+}
+
+declare float @llvm.fma.f32(float, float, float) #1
+declare <2 x float> @llvm.fma.v2f32(<2 x float>, <2 x float>, <2 x float>) #1
+declare <3 x float> @llvm.fma.v3f32(<3 x float>, <3 x float>, <3 x float>) #1
+declare <5 x float> @llvm.fma.v5f32(<5 x float>, <5 x float>, <5 x float>) #1
+
+declare double @llvm.fma.f64(double, double, double) #1
+declare <2 x double> @llvm.fma.v2f64(<2 x double>, <2 x double>, <2 x double>) #1
+declare <3 x double> @llvm.fma.v3f64(<3 x double>, <3 x double>, <3 x double>) #1
+
+declare half @llvm.fma.f16(half, half, half) #1
+declare <2 x half> @llvm.fma.v2f16(<2 x half>, <2 x half>, <2 x half>) #1
+declare <3 x half> @llvm.fma.v3f16(<3 x half>, <3 x half>, <3 x half>) #1
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind readnone }
diff --git a/llvm/test/Analysis/CostModel/AMDGPU/fmul.ll b/llvm/test/Analysis/CostModel/AMDGPU/fmul.ll
index b1e2b51dd..9de2072f5 100644
--- a/llvm/test/Analysis/CostModel/AMDGPU/fmul.ll
+++ b/llvm/test/Analysis/CostModel/AMDGPU/fmul.ll
@@ -1,5 +1,7 @@
-; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa -mattr=+half-rate-64-ops < %s | FileCheck -check-prefix=FASTF64 -check-prefix=ALL %s
-; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa -mattr=-half-rate-64-ops < %s | FileCheck -check-prefix=SLOWF64 -check-prefix=ALL %s
+; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa -mcpu=gfx900 -mattr=+half-rate-64-ops < %s | FileCheck -check-prefixes=FASTF64,FASTF16,ALL %s
+; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa -mattr=-half-rate-64-ops < %s | FileCheck -check-prefixes=SLOWF64,SLOWF16,ALL %s
+; RUN: opt -cost-model -cost-kind=code-size -analyze -mtriple=amdgcn-unknown-amdhsa -mcpu=gfx900 -mattr=+half-rate-64-ops < %s | FileCheck -check-prefixes=FASTF64,FASTF16,ALL %s
+; RUN: opt -cost-model -cost-kind=code-size -analyze -mtriple=amdgcn-unknown-amdhsa -mattr=-half-rate-64-ops < %s | FileCheck -check-prefixes=SLOWF64,SLOWF16,ALL %s
 
 ; ALL: 'fmul_f32'
 ; ALL: estimated cost of 1 for {{.*}} fmul float
@@ -71,8 +73,8 @@ define amdgpu_kernel void @fmul_v3f64(<3 x double> addrspace(1)* %out, <3 x doub
   ret void
 }
 
-; ALL 'fmul_f16'
-; ALL estimated cost of 1 for {{.*}} fmul half
+; ALL: 'fmul_f16'
+; ALL: estimated cost of 1 for {{.*}} fmul half
 define amdgpu_kernel void @fmul_f16(half addrspace(1)* %out, half addrspace(1)* %vaddr, half %b) #0 {
   %vec = load half, half addrspace(1)* %vaddr
   %add = fmul half %vec, %b
@@ -80,8 +82,9 @@ define amdgpu_kernel void @fmul_f16(half addrspace(1)* %out, half addrspace(1)*
   ret void
 }
 
-; ALL 'fmul_v2f16'
-; ALL estimated cost of 2 for {{.*}} fmul <2 x half>
+; ALL: 'fmul_v2f16'
+; SLOWF16 estimated cost of 2 for {{.*}} fmul <2 x half>
+; FASTF16 estimated cost of 1 for {{.*}} fmul <2 x half>
 define amdgpu_kernel void @fmul_v2f16(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %vaddr, <2 x half> %b) #0 {
   %vec = load <2 x half>, <2 x half> addrspace(1)* %vaddr
   %add = fmul <2 x half> %vec, %b
@@ -89,8 +92,19 @@ define amdgpu_kernel void @fmul_v2f16(<2 x half> addrspace(1)* %out, <2 x half>
   ret void
 }
 
-; ALL 'fmul_v4f16'
-; ALL estimated cost of 4 for {{.*}} fmul <4 x half>
+; ALL: 'fmul_v3f16'
+; SLOWF16 estimated cost of 4 for {{.*}} fmul <3 x half>
+; FASTF16 estimated cost of 2 for {{.*}} fmul <3 x half>
+define amdgpu_kernel void @fmul_v3f16(<3 x half> addrspace(1)* %out, <3 x half> addrspace(1)* %vaddr, <3 x half> %b) #0 {
+  %vec = load <3 x half>, <3 x half> addrspace(1)* %vaddr
+  %add = fmul <3 x half> %vec, %b
+  store <3 x half> %add, <3 x half> addrspace(1)* %out
+  ret void
+}
+
+; ALL: 'fmul_v4f16'
+; SLOWF16: estimated cost of 4 for {{.*}} fmul <4 x half>
+; FASTF16: estimated cost of 2 for {{.*}} fmul <4 x half>
 define amdgpu_kernel void @fmul_v4f16(<4 x half> addrspace(1)* %out, <4 x half> addrspace(1)* %vaddr, <4 x half> %b) #0 {
   %vec = load <4 x half>, <4 x half> addrspace(1)* %vaddr
   %add = fmul <4 x half> %vec, %b
diff --git a/llvm/test/Analysis/CostModel/AMDGPU/fsub.ll b/llvm/test/Analysis/CostModel/AMDGPU/fsub.ll
index 9ab518423..8bc6ebcd7 100644
--- a/llvm/test/Analysis/CostModel/AMDGPU/fsub.ll
+++ b/llvm/test/Analysis/CostModel/AMDGPU/fsub.ll
@@ -1,5 +1,7 @@
-; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa -mattr=+half-rate-64-ops < %s | FileCheck -check-prefix=FASTF64 -check-prefix=ALL %s
-; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa -mattr=-half-rate-64-ops < %s | FileCheck -check-prefix=SLOWF64 -check-prefix=ALL %s
+; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa -mcpu=gfx900 -mattr=+half-rate-64-ops < %s | FileCheck -check-prefixes=FASTF64,FASTF16,ALL %s
+; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa -mattr=-half-rate-64-ops < %s | FileCheck -check-prefixes=SLOWF64,SLOWF16,ALL %s
+; RUN: opt -cost-model -analyze -cost-kind=code-size -mtriple=amdgcn-unknown-amdhsa -mcpu=gfx900 -mattr=+half-rate-64-ops < %s | FileCheck -check-prefixes=FASTF64,FASTF16,ALL %s
+; RUN: opt -cost-model -analyze -cost-kind=code-size -mtriple=amdgcn-unknown-amdhsa -mattr=-half-rate-64-ops < %s | FileCheck -check-prefixes=SLOWF64,SLOWF16,ALL %s
 
 ; ALL: 'fsub_f32'
 ; ALL: estimated cost of 1 for {{.*}} fsub float
@@ -81,7 +83,8 @@ define amdgpu_kernel void @fsub_f16(half addrspace(1)* %out, half addrspace(1)*
 }
 
 ; ALL: 'fsub_v2f16'
-; ALL: estimated cost of 2 for {{.*}} fsub <2 x half>
+; SLOWF16: estimated cost of 2 for {{.*}} fsub <2 x half>
+; FASTF16: estimated cost of 1 for {{.*}} fsub <2 x half>
 define amdgpu_kernel void @fsub_v2f16(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %vaddr, <2 x half> %b) #0 {
   %vec = load <2 x half>, <2 x half> addrspace(1)* %vaddr
   %add = fsub <2 x half> %vec, %b
@@ -89,8 +92,19 @@ define amdgpu_kernel void @fsub_v2f16(<2 x half> addrspace(1)* %out, <2 x half>
   ret void
 }
 
+; ALL: 'fsub_v3f16'
+; SLOWF16: estimated cost of 4 for {{.*}} fsub <3 x half>
+; FASTF16: estimated cost of 2 for {{.*}} fsub <3 x half>
+define amdgpu_kernel void @fsub_v3f16(<3 x half> addrspace(1)* %out, <3 x half> addrspace(1)* %vaddr, <3 x half> %b) #0 {
+  %vec = load <3 x half>, <3 x half> addrspace(1)* %vaddr
+  %add = fsub <3 x half> %vec, %b
+  store <3 x half> %add, <3 x half> addrspace(1)* %out
+  ret void
+}
+
 ; ALL: 'fsub_v4f16'
-; ALL: estimated cost of 4 for {{.*}} fsub <4 x half>
+; SLOWF16: estimated cost of 4 for {{.*}} fsub <4 x half>
+; FASTF16: estimated cost of 2 for {{.*}} fsub <4 x half>
 define amdgpu_kernel void @fsub_v4f16(<4 x half> addrspace(1)* %out, <4 x half> addrspace(1)* %vaddr, <4 x half> %b) #0 {
   %vec = load <4 x half>, <4 x half> addrspace(1)* %vaddr
   %add = fsub <4 x half> %vec, %b
diff --git a/llvm/test/Analysis/CostModel/AMDGPU/insertelement.ll b/llvm/test/Analysis/CostModel/AMDGPU/insertelement.ll
index 67ab2607a..7bd86db27 100644
--- a/llvm/test/Analysis/CostModel/AMDGPU/insertelement.ll
+++ b/llvm/test/Analysis/CostModel/AMDGPU/insertelement.ll
@@ -1,6 +1,9 @@
 ; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa %s | FileCheck -check-prefixes=GCN,CI %s
-; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa -mcpu=fiji %s | FileCheck -check-prefixes=GCN,VI %s
-; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa -mcpu=gfx900 %s | FileCheck -check-prefixes=GCN,GFX9 %s
+; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa -mcpu=fiji %s | FileCheck -check-prefixes=GCN,GFX89 %s
+; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa -mcpu=gfx900 %s | FileCheck -check-prefixes=GCN,GFX89 %s
+; RUN: opt -cost-model -cost-kind=code-size -analyze -mtriple=amdgcn-unknown-amdhsa %s | FileCheck -check-prefixes=GCN,CI %s
+; RUN: opt -cost-model -cost-kind=code-size -analyze -mtriple=amdgcn-unknown-amdhsa -mcpu=fiji %s | FileCheck -check-prefixes=GCN,GFX89 %s
+; RUN: opt -cost-model -cost-kind=code-size -analyze -mtriple=amdgcn-unknown-amdhsa -mcpu=gfx900 %s | FileCheck -check-prefixes=GCN,GFX89 %s
 
 ; GCN-LABEL: 'insertelement_v2i32'
 ; GCN: estimated cost of 0 for {{.*}} insertelement <2 x i32>
@@ -22,8 +25,7 @@ define amdgpu_kernel void @insertelement_v2i64(<2 x i64> addrspace(1)* %out, <2
 
 ; GCN-LABEL: 'insertelement_0_v2i16'
 ; CI: estimated cost of 1 for {{.*}} insertelement <2 x i16>
-; VI: estimated cost of 0 for {{.*}} insertelement <2 x i16>
-; GFX9: estimated cost of 0 for {{.*}} insertelement <2 x i16>
+; GFX89: estimated cost of 0 for {{.*}} insertelement <2 x i16>
 define amdgpu_kernel void @insertelement_0_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %vaddr) {
   %vec = load <2 x i16>, <2 x i16> addrspace(1)* %vaddr
   %insert = insertelement <2 x i16> %vec, i16 123, i16 0
diff --git a/llvm/test/Analysis/CostModel/AMDGPU/mul.ll b/llvm/test/Analysis/CostModel/AMDGPU/mul.ll
index 85a377041..4d8a66ecd 100644
--- a/llvm/test/Analysis/CostModel/AMDGPU/mul.ll
+++ b/llvm/test/Analysis/CostModel/AMDGPU/mul.ll
@@ -1,7 +1,10 @@
-; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa < %s | FileCheck %s
+; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa < %s | FileCheck -check-prefixes=SLOW16,ALL %s
+; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa -mcpu=gfx900 < %s | FileCheck -check-prefixes=FAST16,ALL %s
+; RUN: opt -cost-model -cost-kind=code-size -analyze -mtriple=amdgcn-unknown-amdhsa < %s | FileCheck -check-prefixes=SLOW16,ALL %s
+; RUN: opt -cost-model -cost-kind=code-size -analyze -mtriple=amdgcn-unknown-amdhsa -mcpu=gfx900 < %s | FileCheck -check-prefixes=FAST16,ALL %s
 
-; CHECK: 'mul_i32'
-; CHECK: estimated cost of 3 for {{.*}} mul i32
+; ALL: 'mul_i32'
+; ALL: estimated cost of 3 for {{.*}} mul i32
 define amdgpu_kernel void @mul_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %vaddr, i32 %b) #0 {
   %vec = load i32, i32 addrspace(1)* %vaddr
   %mul = mul i32 %vec, %b
@@ -9,8 +12,8 @@ define amdgpu_kernel void @mul_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %va
   ret void
 }
 
-; CHECK: 'mul_v2i32'
-; CHECK: estimated cost of 6 for {{.*}} mul <2 x i32>
+; ALL: 'mul_v2i32'
+; ALL: estimated cost of 6 for {{.*}} mul <2 x i32>
 define amdgpu_kernel void @mul_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32> addrspace(1)* %vaddr, <2 x i32> %b) #0 {
   %vec = load <2 x i32>, <2 x i32> addrspace(1)* %vaddr
   %mul = mul <2 x i32> %vec, %b
@@ -18,10 +21,10 @@ define amdgpu_kernel void @mul_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32> add
   ret void
 }
 
-; CHECK: 'mul_v3i32'
+; ALL: 'mul_v3i32'
 ; Allow for 12 when v3i32 is illegal and TargetLowering thinks it needs widening,
 ; and 9 when it is legal.
-; CHECK: estimated cost of {{9|12}} for {{.*}} mul <3 x i32>
+; ALL: estimated cost of {{9|12}} for {{.*}} mul <3 x i32>
 define amdgpu_kernel void @mul_v3i32(<3 x i32> addrspace(1)* %out, <3 x i32> addrspace(1)* %vaddr, <3 x i32> %b) #0 {
   %vec = load <3 x i32>, <3 x i32> addrspace(1)* %vaddr
   %mul = mul <3 x i32> %vec, %b
@@ -29,10 +32,10 @@ define amdgpu_kernel void @mul_v3i32(<3 x i32> addrspace(1)* %out, <3 x i32> add
   ret void
 }
 
-; CHECK: 'mul_v5i32'
+; ALL: 'mul_v5i32'
 ; Allow for 24 when v5i32 is illegal and TargetLowering thinks it needs widening,
 ; and 15 when it is legal.
-; CHECK: estimated cost of {{15|24}} for {{.*}} mul <5 x i32>
+; ALL: estimated cost of {{15|24}} for {{.*}} mul <5 x i32>
 define amdgpu_kernel void @mul_v5i32(<5 x i32> addrspace(1)* %out, <5 x i32> addrspace(1)* %vaddr, <5 x i32> %b) #0 {
   %vec = load <5 x i32>, <5 x i32> addrspace(1)* %vaddr
   %mul = mul <5 x i32> %vec, %b
@@ -40,8 +43,8 @@ define amdgpu_kernel void @mul_v5i32(<5 x i32> addrspace(1)* %out, <5 x i32> add
   ret void
 }
 
-; CHECK: 'mul_v4i32'
-; CHECK: estimated cost of 12 for {{.*}} mul <4 x i32>
+; ALL: 'mul_v4i32'
+; ALL: estimated cost of 12 for {{.*}} mul <4 x i32>
 define amdgpu_kernel void @mul_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> addrspace(1)* %vaddr, <4 x i32> %b) #0 {
   %vec = load <4 x i32>, <4 x i32> addrspace(1)* %vaddr
   %mul = mul <4 x i32> %vec, %b
@@ -49,8 +52,8 @@ define amdgpu_kernel void @mul_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> add
   ret void
 }
 
-; CHECK: 'mul_i64'
-; CHECK: estimated cost of 16 for {{.*}} mul i64
+; ALL: 'mul_i64'
+; ALL: estimated cost of 16 for {{.*}} mul i64
 define amdgpu_kernel void @mul_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %vaddr, i64 %b) #0 {
   %vec = load i64, i64 addrspace(1)* %vaddr
   %mul = mul i64 %vec, %b
@@ -58,8 +61,8 @@ define amdgpu_kernel void @mul_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %va
   ret void
 }
 
-; CHECK: 'mul_v2i64'
-; CHECK: estimated cost of 32 for {{.*}} mul <2 x i64>
+; ALL: 'mul_v2i64'
+; ALL: estimated cost of 32 for {{.*}} mul <2 x i64>
 define amdgpu_kernel void @mul_v2i64(<2 x i64> addrspace(1)* %out, <2 x i64> addrspace(1)* %vaddr, <2 x i64> %b) #0 {
   %vec = load <2 x i64>, <2 x i64> addrspace(1)* %vaddr
   %mul = mul <2 x i64> %vec, %b
@@ -67,8 +70,8 @@ define amdgpu_kernel void @mul_v2i64(<2 x i64> addrspace(1)* %out, <2 x i64> add
   ret void
 }
 
-; CHECK: 'mul_v3i64'
-; CHECK: estimated cost of 48 for {{.*}} mul <3 x i64>
+; ALL: 'mul_v3i64'
+; ALL: estimated cost of 48 for {{.*}} mul <3 x i64>
 define amdgpu_kernel void @mul_v3i64(<3 x i64> addrspace(1)* %out, <3 x i64> addrspace(1)* %vaddr, <3 x i64> %b) #0 {
   %vec = load <3 x i64>, <3 x i64> addrspace(1)* %vaddr
   %mul = mul <3 x i64> %vec, %b
@@ -76,8 +79,8 @@ define amdgpu_kernel void @mul_v3i64(<3 x i64> addrspace(1)* %out, <3 x i64> add
   ret void
 }
 
-; CHECK: 'mul_v4i64'
-; CHECK: estimated cost of 64 for {{.*}} mul <4 x i64>
+; ALL: 'mul_v4i64'
+; ALL: estimated cost of 64 for {{.*}} mul <4 x i64>
 define amdgpu_kernel void @mul_v4i64(<4 x i64> addrspace(1)* %out, <4 x i64> addrspace(1)* %vaddr, <4 x i64> %b) #0 {
   %vec = load <4 x i64>, <4 x i64> addrspace(1)* %vaddr
   %mul = mul <4 x i64> %vec, %b
@@ -86,8 +89,8 @@ define amdgpu_kernel void @mul_v4i64(<4 x i64> addrspace(1)* %out, <4 x i64> add
 }
 
 
-; CHECK: 'mul_v8i64'
-; CHECK: estimated cost of 128 for {{.*}} mul <8 x i64>
+; ALL: 'mul_v8i64'
+; ALL: estimated cost of 128 for {{.*}} mul <8 x i64>
 define amdgpu_kernel void @mul_v8i64(<8 x i64> addrspace(1)* %out, <8 x i64> addrspace(1)* %vaddr, <8 x i64> %b) #0 {
   %vec = load <8 x i64>, <8 x i64> addrspace(1)* %vaddr
   %mul = mul <8 x i64> %vec, %b
@@ -95,4 +98,33 @@ define amdgpu_kernel void @mul_v8i64(<8 x i64> addrspace(1)* %out, <8 x i64> add
   ret void
 }
 
+; ALL: 'mul_i16'
+; ALL: estimated cost of 3 for {{.*}} mul i16
+define amdgpu_kernel void @mul_i16(i16 addrspace(1)* %out, i16 addrspace(1)* %vaddr, i16 %b) #0 {
+  %vec = load i16, i16 addrspace(1)* %vaddr
+  %mul = mul i16 %vec, %b
+  store i16 %mul, i16 addrspace(1)* %out
+  ret void
+}
+
+; ALL: 'mul_v2i16'
+; SLOW16: estimated cost of 6 for {{.*}} mul <2 x i16>
+; FAST16: estimated cost of 3 for {{.*}} mul <2 x i16>
+define amdgpu_kernel void @mul_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %vaddr, <2 x i16> %b) #0 {
+  %vec = load <2 x i16>, <2 x i16> addrspace(1)* %vaddr
+  %mul = mul <2 x i16> %vec, %b
+  store <2 x i16> %mul, <2 x i16> addrspace(1)* %out
+  ret void
+}
+
+; ALL: 'mul_v3i16'
+; SLOW16: estimated cost of 12 for {{.*}} mul <3 x i16>
+; FAST16: estimated cost of 6 for {{.*}} mul <3 x i16>
+define amdgpu_kernel void @mul_v3i16(<3 x i16> addrspace(1)* %out, <3 x i16> addrspace(1)* %vaddr, <3 x i16> %b) #0 {
+  %vec = load <3 x i16>, <3 x i16> addrspace(1)* %vaddr
+  %mul = mul <3 x i16> %vec, %b
+  store <3 x i16> %mul, <3 x i16> addrspace(1)* %out
+  ret void
+}
+
 attributes #0 = { nounwind }
diff --git a/llvm/test/Analysis/CostModel/AMDGPU/shifts.ll b/llvm/test/Analysis/CostModel/AMDGPU/shifts.ll
index 85fb0ebe1..55f547fe3 100644
--- a/llvm/test/Analysis/CostModel/AMDGPU/shifts.ll
+++ b/llvm/test/Analysis/CostModel/AMDGPU/shifts.ll
@@ -1,5 +1,7 @@
-; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa -mattr=+half-rate-64-ops < %s | FileCheck -check-prefix=ALL -check-prefix=FAST64 %s
-; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa -mattr=-half-rate-64-ops < %s | FileCheck -check-prefix=ALL -check-prefix=SLOW64 %s
+; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa -mcpu=gfx900 -mattr=+half-rate-64-ops < %s | FileCheck -check-prefixes=ALL,FAST64,FAST16 %s
+; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa -mattr=-half-rate-64-ops < %s | FileCheck -check-prefixes=ALL,SLOW64,SLOW16 %s
+; RUN: opt -cost-model -cost-kind=code-size -analyze -mtriple=amdgcn-unknown-amdhsa -mcpu=gfx900 -mattr=+half-rate-64-ops < %s | FileCheck -check-prefixes=ALL,FAST64,FAST16 %s
+; RUN: opt -cost-model -cost-kind=code-size -analyze -mtriple=amdgcn-unknown-amdhsa -mattr=-half-rate-64-ops < %s | FileCheck -check-prefixes=ALL,SLOW64,SLOW16 %s
 
 ; ALL: 'shl_i32'
 ; ALL: estimated cost of 1 for {{.*}} shl i32
@@ -20,6 +22,25 @@ define amdgpu_kernel void @shl_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %va
   ret void
 }
 
+; ALL: 'shl_i16'
+; ALL: estimated cost of 1 for {{.*}} shl i16
+define amdgpu_kernel void @shl_i16(i16 addrspace(1)* %out, i16 addrspace(1)* %vaddr, i16 %b) #0 {
+  %vec = load i16, i16 addrspace(1)* %vaddr
+  %or = shl i16 %vec, %b
+  store i16 %or, i16 addrspace(1)* %out
+  ret void
+}
+
+; ALL: 'shl_v2i16'
+; SLOW16: estimated cost of 2 for {{.*}} shl <2 x i16>
+; FAST16: estimated cost of 1 for {{.*}} shl <2 x i16>
+define amdgpu_kernel void @shl_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %vaddr, <2 x i16> %b) #0 {
+  %vec = load <2 x i16>, <2 x i16> addrspace(1)* %vaddr
+  %or = shl <2 x i16> %vec, %b
+  store <2 x i16> %or, <2 x i16> addrspace(1)* %out
+  ret void
+}
+
 ; ALL: 'lshr_i32'
 ; ALL: estimated cost of 1 for {{.*}} lshr i32
 define amdgpu_kernel void @lshr_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %vaddr, i32 %b) #0 {
@@ -39,6 +60,25 @@ define amdgpu_kernel void @lshr_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %v
   ret void
 }
 
+; ALL: 'lshr_i16'
+; ALL: estimated cost of 1 for {{.*}} lshr i16
+define amdgpu_kernel void @lshr_i16(i16 addrspace(1)* %out, i16 addrspace(1)* %vaddr, i16 %b) #0 {
+  %vec = load i16, i16 addrspace(1)* %vaddr
+  %or = lshr i16 %vec, %b
+  store i16 %or, i16 addrspace(1)* %out
+  ret void
+}
+
+; ALL: 'lshr_v2i16'
+; SLOW16: estimated cost of 2 for {{.*}} lshr <2 x i16>
+; FAST16: estimated cost of 1 for {{.*}} lshr <2 x i16>
+define amdgpu_kernel void @lshr_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %vaddr, <2 x i16> %b) #0 {
+  %vec = load <2 x i16>, <2 x i16> addrspace(1)* %vaddr
+  %or = lshr <2 x i16> %vec, %b
+  store <2 x i16> %or, <2 x i16> addrspace(1)* %out
+  ret void
+}
+
 ; ALL: 'ashr_i32'
 ; ALL: estimated cost of 1 for {{.*}} ashr i32
 define amdgpu_kernel void @ashr_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %vaddr, i32 %b) #0 {
@@ -58,4 +98,23 @@ define amdgpu_kernel void @ashr_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %v
   ret void
 }
 
+; ALL: 'ashr_i16'
+; ALL: estimated cost of 1 for {{.*}} ashr i16
+define amdgpu_kernel void @ashr_i16(i16 addrspace(1)* %out, i16 addrspace(1)* %vaddr, i16 %b) #0 {
+  %vec = load i16, i16 addrspace(1)* %vaddr
+  %or = ashr i16 %vec, %b
+  store i16 %or, i16 addrspace(1)* %out
+  ret void
+}
+
+; ALL: 'ashr_v2i16'
+; SLOW16: estimated cost of 2 for {{.*}} ashr <2 x i16>
+; FAST16: estimated cost of 1 for {{.*}} ashr <2 x i16>
+define amdgpu_kernel void @ashr_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %vaddr, <2 x i16> %b) #0 {
+  %vec = load <2 x i16>, <2 x i16> addrspace(1)* %vaddr
+  %or = ashr <2 x i16> %vec, %b
+  store <2 x i16> %or, <2 x i16> addrspace(1)* %out
+  ret void
+}
+
 attributes #0 = { nounwind }
diff --git a/llvm/test/Analysis/CostModel/AMDGPU/shufflevector.ll b/llvm/test/Analysis/CostModel/AMDGPU/shufflevector.ll
index cc756c82f..ff3abcaab 100644
--- a/llvm/test/Analysis/CostModel/AMDGPU/shufflevector.ll
+++ b/llvm/test/Analysis/CostModel/AMDGPU/shufflevector.ll
@@ -1,7 +1,11 @@
-; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa -mcpu=gfx900 %s | FileCheck -check-prefixes=GFX9,GCN %s
-; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa -mcpu=fiji %s | FileCheck -check-prefixes=VI,GCN %s
+; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa -mcpu=gfx900 %s | FileCheck -check-prefixes=GFX9,GCN,TPT %s
+; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa -mcpu=fiji %s | FileCheck -check-prefixes=VI,GCN,TPT %s
+; RUN: opt -cost-model -cost-kind=code-size -analyze -mtriple=amdgcn-unknown-amdhsa -mcpu=gfx900 %s | FileCheck -check-prefixes=GFX9,GCN,CS %s
+; RUN: opt -cost-model -cost-kind=code-size -analyze -mtriple=amdgcn-unknown-amdhsa -mcpu=fiji %s | FileCheck -check-prefixes=VI,GCN,CS %s
 
+; GCN-LABEL: 'shufflevector_00_v2i16'
 ; GFX9: estimated cost of 0 for {{.*}} shufflevector <2 x i16> %vec, <2 x i16> undef, <2 x i32> zeroinitializer
+; VI: estimated cost of 1 for {{.*}} shufflevector <2 x i16> %vec, <2 x i16> undef, <2 x i32> zeroinitializer
 define amdgpu_kernel void @shufflevector_00_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %vaddr) {
   %vec = load <2 x i16>, <2 x i16> addrspace(1)* %vaddr
   %shuf = shufflevector <2 x i16> %vec, <2 x i16> undef, <2 x i32> zeroinitializer
@@ -9,7 +13,8 @@ define amdgpu_kernel void @shufflevector_00_v2i16(<2 x i16> addrspace(1)* %out,
   ret void
 }
 
-; GFX9: estimated cost of 0 for {{.*}} shufflevector <2 x i16> %vec, <2 x i16> undef, <2 x i32> <i32 0, i32 1>
+; GCN-LABEL: 'shufflevector_01_v2i16'
+; GCN: estimated cost of 0 for {{.*}} shufflevector <2 x i16> %vec, <2 x i16> undef, <2 x i32> <i32 0, i32 1>
 define amdgpu_kernel void @shufflevector_01_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %vaddr) {
   %vec = load <2 x i16>, <2 x i16> addrspace(1)* %vaddr
   %shuf = shufflevector <2 x i16> %vec, <2 x i16> undef, <2 x i32> <i32 0, i32 1>
@@ -17,7 +22,9 @@ define amdgpu_kernel void @shufflevector_01_v2i16(<2 x i16> addrspace(1)* %out,
   ret void
 }
 
+; GCN-LABEL: 'shufflevector_10_v2i16'
 ; GFX9: estimated cost of 0 for {{.*}} shufflevector <2 x i16> %vec, <2 x i16> undef, <2 x i32> <i32 1, i32 0>
+; VI: estimated cost of 2 for {{.*}} shufflevector <2 x i16> %vec, <2 x i16> undef, <2 x i32> <i32 1, i32 0>
 define amdgpu_kernel void @shufflevector_10_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %vaddr) {
   %vec = load <2 x i16>, <2 x i16> addrspace(1)* %vaddr
   %shuf = shufflevector <2 x i16> %vec, <2 x i16> undef, <2 x i32> <i32 1, i32 0>
@@ -25,7 +32,9 @@ define amdgpu_kernel void @shufflevector_10_v2i16(<2 x i16> addrspace(1)* %out,
   ret void
 }
 
+; GCN-LABEL: 'shufflevector_11_v2i16'
 ; GFX9: estimated cost of 0 for {{.*}} shufflevector <2 x i16> %vec, <2 x i16> undef, <2 x i32> <i32 1, i32 1>
+; VI: estimated cost of 2 for {{.*}} shufflevector <2 x i16> %vec, <2 x i16> undef, <2 x i32> <i32 1, i32 1>
 define amdgpu_kernel void @shufflevector_11_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %vaddr) {
   %vec = load <2 x i16>, <2 x i16> addrspace(1)* %vaddr
   %shuf = shufflevector <2 x i16> %vec, <2 x i16> undef, <2 x i32> <i32 1, i32 1>
@@ -33,6 +42,7 @@ define amdgpu_kernel void @shufflevector_11_v2i16(<2 x i16> addrspace(1)* %out,
   ret void
 }
 
+; GCN-LABEL: 'shufflevector_02_v2i16'
 ; GCN: estimated cost of 2 for {{.*}} shufflevector <2 x i16> %vec0, <2 x i16> %vec1, <2 x i32> <i32 0, i32 2>
 define amdgpu_kernel void @shufflevector_02_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %vaddr0, <2 x i16> addrspace(1)* %vaddr1) {
   %vec0 = load <2 x i16>, <2 x i16> addrspace(1)* %vaddr0
@@ -41,3 +51,14 @@ define amdgpu_kernel void @shufflevector_02_v2i16(<2 x i16> addrspace(1)* %out,
   store <2 x i16> %shuf, <2 x i16> addrspace(1)* %out
   ret void
 }
+
+; GCN-LABEL: 'shufflevector_xxx'
+; TPT: Unknown cost for {{.*}} shufflevector <2 x i8> %vec, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CS: estimated cost of 1 for {{.*}} shufflevector <2 x i8> %vec, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; Should not assert
+define amdgpu_kernel void @shufflevector_xxx(<4 x i8> addrspace(1)* %out, <2 x i8> addrspace(1)* %vaddr) {
+  %vec = load <2 x i8>, <2 x i8> addrspace(1)* %vaddr
+  %shuf = shufflevector <2 x i8> %vec, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+  store <4 x i8> %shuf, <4 x i8> addrspace(1)* %out
+  ret void
+}
diff --git a/llvm/test/Analysis/CostModel/ARM/arith.ll b/llvm/test/Analysis/CostModel/ARM/arith.ll
new file mode 100644
index 000000000..ad9a3d5a0
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/ARM/arith.ll
@@ -0,0 +1,733 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt -cost-model -analyze -mtriple=thumbv8.1-m.main-none-eabi -mattr=+mve,+mve1beat < %s | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-MVE1
+; RUN: opt -cost-model -analyze -mtriple=thumbv8.1-m.main-none-eabi -mattr=+mve < %s | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-MVE2
+; RUN: opt -cost-model -analyze -mtriple=thumbv8.1-m.main-none-eabi -mattr=+mve,+mve4beat < %s | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-MVE4
+
+define void @i8() {
+; CHECK-LABEL: 'i8'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c = add i8 undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %d = sub i8 undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e = mul i8 undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f = ashr i8 undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %g = lshr i8 undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %h = shl i8 undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i = and i8 undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %j = or i8 undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %k = xor i8 undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %c = add i8 undef, undef
+  %d = sub i8 undef, undef
+  %e = mul i8 undef, undef
+  %f = ashr i8 undef, undef
+  %g = lshr i8 undef, undef
+  %h = shl i8 undef, undef
+  %i = and i8 undef, undef
+  %j = or i8 undef, undef
+  %k = xor i8 undef, undef
+  ret void
+}
+
+define void @i16() {
+; CHECK-LABEL: 'i16'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c = add i16 undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %d = sub i16 undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e = mul i16 undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f = ashr i16 undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %g = lshr i16 undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %h = shl i16 undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i = and i16 undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %j = or i16 undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %k = xor i16 undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %c = add i16 undef, undef
+  %d = sub i16 undef, undef
+  %e = mul i16 undef, undef
+  %f = ashr i16 undef, undef
+  %g = lshr i16 undef, undef
+  %h = shl i16 undef, undef
+  %i = and i16 undef, undef
+  %j = or i16 undef, undef
+  %k = xor i16 undef, undef
+  ret void
+}
+
+define void @i32() {
+; CHECK-LABEL: 'i32'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c = add i32 undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %d = sub i32 undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e = mul i32 undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f = ashr i32 undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %g = lshr i32 undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %h = shl i32 undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i = and i32 undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %j = or i32 undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %k = xor i32 undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %c = add i32 undef, undef
+  %d = sub i32 undef, undef
+  %e = mul i32 undef, undef
+  %f = ashr i32 undef, undef
+  %g = lshr i32 undef, undef
+  %h = shl i32 undef, undef
+  %i = and i32 undef, undef
+  %j = or i32 undef, undef
+  %k = xor i32 undef, undef
+  ret void
+}
+
+define void @i64() {
+; CHECK-LABEL: 'i64'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %c = add i64 undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %d = sub i64 undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %e = mul i64 undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %f = ashr i64 undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %g = lshr i64 undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %h = shl i64 undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %i = and i64 undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %j = or i64 undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %k = xor i64 undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %c = add i64 undef, undef
+  %d = sub i64 undef, undef
+  %e = mul i64 undef, undef
+  %f = ashr i64 undef, undef
+  %g = lshr i64 undef, undef
+  %h = shl i64 undef, undef
+  %i = and i64 undef, undef
+  %j = or i64 undef, undef
+  %k = xor i64 undef, undef
+  ret void
+}
+
+
+define void @vi8() {
+; CHECK-MVE1-LABEL: 'vi8'
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %c2 = add <2 x i8> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %d2 = sub <2 x i8> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %e2 = mul <2 x i8> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %f2 = ashr <2 x i8> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %g2 = lshr <2 x i8> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %h2 = shl <2 x i8> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i2 = and <2 x i8> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %j2 = or <2 x i8> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %k2 = xor <2 x i8> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c4 = add <4 x i8> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %d4 = sub <4 x i8> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e4 = mul <4 x i8> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f4 = ashr <4 x i8> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %g4 = lshr <4 x i8> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %h4 = shl <4 x i8> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i4 = and <4 x i8> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %j4 = or <4 x i8> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %k4 = xor <4 x i8> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c8 = add <8 x i8> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %d8 = sub <8 x i8> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e8 = mul <8 x i8> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f8 = ashr <8 x i8> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %g8 = lshr <8 x i8> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %h8 = shl <8 x i8> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i8 = and <8 x i8> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %j8 = or <8 x i8> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %k8 = xor <8 x i8> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c16 = add <16 x i8> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %d16 = sub <16 x i8> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e16 = mul <16 x i8> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f16 = ashr <16 x i8> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %g16 = lshr <16 x i8> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %h16 = shl <16 x i8> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i16 = and <16 x i8> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %j16 = or <16 x i8> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %k16 = xor <16 x i8> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-MVE2-LABEL: 'vi8'
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %c2 = add <2 x i8> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %d2 = sub <2 x i8> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %e2 = mul <2 x i8> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %f2 = ashr <2 x i8> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %g2 = lshr <2 x i8> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %h2 = shl <2 x i8> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %i2 = and <2 x i8> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %j2 = or <2 x i8> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %k2 = xor <2 x i8> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %c4 = add <4 x i8> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %d4 = sub <4 x i8> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %e4 = mul <4 x i8> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %f4 = ashr <4 x i8> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %g4 = lshr <4 x i8> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %h4 = shl <4 x i8> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %i4 = and <4 x i8> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %j4 = or <4 x i8> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %k4 = xor <4 x i8> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %c8 = add <8 x i8> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %d8 = sub <8 x i8> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %e8 = mul <8 x i8> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %f8 = ashr <8 x i8> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %g8 = lshr <8 x i8> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %h8 = shl <8 x i8> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %i8 = and <8 x i8> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %j8 = or <8 x i8> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %k8 = xor <8 x i8> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %c16 = add <16 x i8> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %d16 = sub <16 x i8> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %e16 = mul <16 x i8> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %f16 = ashr <16 x i8> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %g16 = lshr <16 x i8> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %h16 = shl <16 x i8> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %i16 = and <16 x i8> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %j16 = or <16 x i8> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %k16 = xor <16 x i8> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-MVE4-LABEL: 'vi8'
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %c2 = add <2 x i8> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %d2 = sub <2 x i8> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %e2 = mul <2 x i8> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %f2 = ashr <2 x i8> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %g2 = lshr <2 x i8> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %h2 = shl <2 x i8> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %i2 = and <2 x i8> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %j2 = or <2 x i8> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %k2 = xor <2 x i8> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %c4 = add <4 x i8> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %d4 = sub <4 x i8> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %e4 = mul <4 x i8> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %f4 = ashr <4 x i8> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %g4 = lshr <4 x i8> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %h4 = shl <4 x i8> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %i4 = and <4 x i8> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %j4 = or <4 x i8> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %k4 = xor <4 x i8> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %c8 = add <8 x i8> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %d8 = sub <8 x i8> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %e8 = mul <8 x i8> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %f8 = ashr <8 x i8> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %g8 = lshr <8 x i8> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %h8 = shl <8 x i8> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %i8 = and <8 x i8> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %j8 = or <8 x i8> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %k8 = xor <8 x i8> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %c16 = add <16 x i8> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %d16 = sub <16 x i8> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %e16 = mul <16 x i8> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %f16 = ashr <16 x i8> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %g16 = lshr <16 x i8> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %h16 = shl <16 x i8> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %i16 = and <16 x i8> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %j16 = or <16 x i8> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %k16 = xor <16 x i8> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %c2 = add <2 x i8> undef, undef
+  %d2 = sub <2 x i8> undef, undef
+  %e2 = mul <2 x i8> undef, undef
+  %f2 = ashr <2 x i8> undef, undef
+  %g2 = lshr <2 x i8> undef, undef
+  %h2 = shl <2 x i8> undef, undef
+  %i2 = and <2 x i8> undef, undef
+  %j2 = or <2 x i8> undef, undef
+  %k2 = xor <2 x i8> undef, undef
+  %c4 = add <4 x i8> undef, undef
+  %d4 = sub <4 x i8> undef, undef
+  %e4 = mul <4 x i8> undef, undef
+  %f4 = ashr <4 x i8> undef, undef
+  %g4 = lshr <4 x i8> undef, undef
+  %h4 = shl <4 x i8> undef, undef
+  %i4 = and <4 x i8> undef, undef
+  %j4 = or <4 x i8> undef, undef
+  %k4 = xor <4 x i8> undef, undef
+  %c8 = add <8 x i8> undef, undef
+  %d8 = sub <8 x i8> undef, undef
+  %e8 = mul <8 x i8> undef, undef
+  %f8 = ashr <8 x i8> undef, undef
+  %g8 = lshr <8 x i8> undef, undef
+  %h8 = shl <8 x i8> undef, undef
+  %i8 = and <8 x i8> undef, undef
+  %j8 = or <8 x i8> undef, undef
+  %k8 = xor <8 x i8> undef, undef
+  %c16 = add <16 x i8> undef, undef
+  %d16 = sub <16 x i8> undef, undef
+  %e16 = mul <16 x i8> undef, undef
+  %f16 = ashr <16 x i8> undef, undef
+  %g16 = lshr <16 x i8> undef, undef
+  %h16 = shl <16 x i8> undef, undef
+  %i16 = and <16 x i8> undef, undef
+  %j16 = or <16 x i8> undef, undef
+  %k16 = xor <16 x i8> undef, undef
+  ret void
+}
+
+define void @vi16() {
+; CHECK-MVE1-LABEL: 'vi16'
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %c2 = add <2 x i16> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %d2 = sub <2 x i16> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %e2 = mul <2 x i16> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %f2 = ashr <2 x i16> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %g2 = lshr <2 x i16> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %h2 = shl <2 x i16> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i2 = and <2 x i16> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %j2 = or <2 x i16> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %k2 = xor <2 x i16> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c4 = add <4 x i16> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %d4 = sub <4 x i16> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e4 = mul <4 x i16> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f4 = ashr <4 x i16> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %g4 = lshr <4 x i16> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %h4 = shl <4 x i16> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i4 = and <4 x i16> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %j4 = or <4 x i16> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %k4 = xor <4 x i16> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c8 = add <8 x i16> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %d8 = sub <8 x i16> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e8 = mul <8 x i16> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f8 = ashr <8 x i16> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %g8 = lshr <8 x i16> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %h8 = shl <8 x i16> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i8 = and <8 x i16> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %j8 = or <8 x i16> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %k8 = xor <8 x i16> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %c16 = add <16 x i16> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %d16 = sub <16 x i16> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %e16 = mul <16 x i16> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %f16 = ashr <16 x i16> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %g16 = lshr <16 x i16> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %h16 = shl <16 x i16> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %i16 = and <16 x i16> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %j16 = or <16 x i16> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %k16 = xor <16 x i16> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-MVE2-LABEL: 'vi16'
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %c2 = add <2 x i16> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %d2 = sub <2 x i16> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %e2 = mul <2 x i16> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %f2 = ashr <2 x i16> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %g2 = lshr <2 x i16> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %h2 = shl <2 x i16> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %i2 = and <2 x i16> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %j2 = or <2 x i16> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %k2 = xor <2 x i16> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %c4 = add <4 x i16> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %d4 = sub <4 x i16> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %e4 = mul <4 x i16> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %f4 = ashr <4 x i16> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %g4 = lshr <4 x i16> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %h4 = shl <4 x i16> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %i4 = and <4 x i16> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %j4 = or <4 x i16> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %k4 = xor <4 x i16> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %c8 = add <8 x i16> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %d8 = sub <8 x i16> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %e8 = mul <8 x i16> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %f8 = ashr <8 x i16> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %g8 = lshr <8 x i16> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %h8 = shl <8 x i16> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %i8 = and <8 x i16> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %j8 = or <8 x i16> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %k8 = xor <8 x i16> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %c16 = add <16 x i16> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %d16 = sub <16 x i16> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %e16 = mul <16 x i16> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %f16 = ashr <16 x i16> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %g16 = lshr <16 x i16> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %h16 = shl <16 x i16> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %i16 = and <16 x i16> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %j16 = or <16 x i16> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %k16 = xor <16 x i16> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-MVE4-LABEL: 'vi16'
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %c2 = add <2 x i16> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %d2 = sub <2 x i16> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %e2 = mul <2 x i16> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %f2 = ashr <2 x i16> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %g2 = lshr <2 x i16> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %h2 = shl <2 x i16> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %i2 = and <2 x i16> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %j2 = or <2 x i16> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %k2 = xor <2 x i16> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %c4 = add <4 x i16> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %d4 = sub <4 x i16> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %e4 = mul <4 x i16> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %f4 = ashr <4 x i16> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %g4 = lshr <4 x i16> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %h4 = shl <4 x i16> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %i4 = and <4 x i16> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %j4 = or <4 x i16> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %k4 = xor <4 x i16> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %c8 = add <8 x i16> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %d8 = sub <8 x i16> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %e8 = mul <8 x i16> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %f8 = ashr <8 x i16> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %g8 = lshr <8 x i16> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %h8 = shl <8 x i16> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %i8 = and <8 x i16> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %j8 = or <8 x i16> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %k8 = xor <8 x i16> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %c16 = add <16 x i16> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %d16 = sub <16 x i16> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %e16 = mul <16 x i16> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %f16 = ashr <16 x i16> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %g16 = lshr <16 x i16> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %h16 = shl <16 x i16> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %i16 = and <16 x i16> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %j16 = or <16 x i16> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %k16 = xor <16 x i16> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %c2 = add <2 x i16> undef, undef
+  %d2 = sub <2 x i16> undef, undef
+  %e2 = mul <2 x i16> undef, undef
+  %f2 = ashr <2 x i16> undef, undef
+  %g2 = lshr <2 x i16> undef, undef
+  %h2 = shl <2 x i16> undef, undef
+  %i2 = and <2 x i16> undef, undef
+  %j2 = or <2 x i16> undef, undef
+  %k2 = xor <2 x i16> undef, undef
+  %c4 = add <4 x i16> undef, undef
+  %d4 = sub <4 x i16> undef, undef
+  %e4 = mul <4 x i16> undef, undef
+  %f4 = ashr <4 x i16> undef, undef
+  %g4 = lshr <4 x i16> undef, undef
+  %h4 = shl <4 x i16> undef, undef
+  %i4 = and <4 x i16> undef, undef
+  %j4 = or <4 x i16> undef, undef
+  %k4 = xor <4 x i16> undef, undef
+  %c8 = add <8 x i16> undef, undef
+  %d8 = sub <8 x i16> undef, undef
+  %e8 = mul <8 x i16> undef, undef
+  %f8 = ashr <8 x i16> undef, undef
+  %g8 = lshr <8 x i16> undef, undef
+  %h8 = shl <8 x i16> undef, undef
+  %i8 = and <8 x i16> undef, undef
+  %j8 = or <8 x i16> undef, undef
+  %k8 = xor <8 x i16> undef, undef
+  %c16 = add <16 x i16> undef, undef
+  %d16 = sub <16 x i16> undef, undef
+  %e16 = mul <16 x i16> undef, undef
+  %f16 = ashr <16 x i16> undef, undef
+  %g16 = lshr <16 x i16> undef, undef
+  %h16 = shl <16 x i16> undef, undef
+  %i16 = and <16 x i16> undef, undef
+  %j16 = or <16 x i16> undef, undef
+  %k16 = xor <16 x i16> undef, undef
+  ret void
+}
+
+define void @vi32() {
+; CHECK-MVE1-LABEL: 'vi32'
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %c2 = add <2 x i32> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %d2 = sub <2 x i32> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %e2 = mul <2 x i32> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %f2 = ashr <2 x i32> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %g2 = lshr <2 x i32> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %h2 = shl <2 x i32> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i2 = and <2 x i32> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %j2 = or <2 x i32> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %k2 = xor <2 x i32> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c4 = add <4 x i32> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %d4 = sub <4 x i32> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e4 = mul <4 x i32> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f4 = ashr <4 x i32> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %g4 = lshr <4 x i32> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %h4 = shl <4 x i32> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i4 = and <4 x i32> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %j4 = or <4 x i32> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %k4 = xor <4 x i32> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %c8 = add <8 x i32> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %d8 = sub <8 x i32> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %e8 = mul <8 x i32> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %f8 = ashr <8 x i32> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %g8 = lshr <8 x i32> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %h8 = shl <8 x i32> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %i8 = and <8 x i32> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %j8 = or <8 x i32> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %k8 = xor <8 x i32> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %c16 = add <16 x i32> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %d16 = sub <16 x i32> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %e16 = mul <16 x i32> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %f16 = ashr <16 x i32> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %g16 = lshr <16 x i32> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %h16 = shl <16 x i32> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %i16 = and <16 x i32> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %j16 = or <16 x i32> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %k16 = xor <16 x i32> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-MVE2-LABEL: 'vi32'
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %c2 = add <2 x i32> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %d2 = sub <2 x i32> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %e2 = mul <2 x i32> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %f2 = ashr <2 x i32> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %g2 = lshr <2 x i32> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %h2 = shl <2 x i32> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %i2 = and <2 x i32> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %j2 = or <2 x i32> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %k2 = xor <2 x i32> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %c4 = add <4 x i32> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %d4 = sub <4 x i32> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %e4 = mul <4 x i32> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %f4 = ashr <4 x i32> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %g4 = lshr <4 x i32> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %h4 = shl <4 x i32> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %i4 = and <4 x i32> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %j4 = or <4 x i32> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %k4 = xor <4 x i32> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %c8 = add <8 x i32> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %d8 = sub <8 x i32> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %e8 = mul <8 x i32> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %f8 = ashr <8 x i32> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %g8 = lshr <8 x i32> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %h8 = shl <8 x i32> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %i8 = and <8 x i32> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %j8 = or <8 x i32> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %k8 = xor <8 x i32> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %c16 = add <16 x i32> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %d16 = sub <16 x i32> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %e16 = mul <16 x i32> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %f16 = ashr <16 x i32> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %g16 = lshr <16 x i32> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %h16 = shl <16 x i32> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %i16 = and <16 x i32> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %j16 = or <16 x i32> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %k16 = xor <16 x i32> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-MVE4-LABEL: 'vi32'
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %c2 = add <2 x i32> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %d2 = sub <2 x i32> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %e2 = mul <2 x i32> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %f2 = ashr <2 x i32> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %g2 = lshr <2 x i32> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %h2 = shl <2 x i32> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %i2 = and <2 x i32> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %j2 = or <2 x i32> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %k2 = xor <2 x i32> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %c4 = add <4 x i32> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %d4 = sub <4 x i32> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %e4 = mul <4 x i32> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %f4 = ashr <4 x i32> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %g4 = lshr <4 x i32> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %h4 = shl <4 x i32> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %i4 = and <4 x i32> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %j4 = or <4 x i32> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %k4 = xor <4 x i32> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %c8 = add <8 x i32> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %d8 = sub <8 x i32> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %e8 = mul <8 x i32> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %f8 = ashr <8 x i32> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %g8 = lshr <8 x i32> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %h8 = shl <8 x i32> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %i8 = and <8 x i32> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %j8 = or <8 x i32> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %k8 = xor <8 x i32> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %c16 = add <16 x i32> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %d16 = sub <16 x i32> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %e16 = mul <16 x i32> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %f16 = ashr <16 x i32> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %g16 = lshr <16 x i32> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %h16 = shl <16 x i32> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %i16 = and <16 x i32> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %j16 = or <16 x i32> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %k16 = xor <16 x i32> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %c2 = add <2 x i32> undef, undef
+  %d2 = sub <2 x i32> undef, undef
+  %e2 = mul <2 x i32> undef, undef
+  %f2 = ashr <2 x i32> undef, undef
+  %g2 = lshr <2 x i32> undef, undef
+  %h2 = shl <2 x i32> undef, undef
+  %i2 = and <2 x i32> undef, undef
+  %j2 = or <2 x i32> undef, undef
+  %k2 = xor <2 x i32> undef, undef
+  %c4 = add <4 x i32> undef, undef
+  %d4 = sub <4 x i32> undef, undef
+  %e4 = mul <4 x i32> undef, undef
+  %f4 = ashr <4 x i32> undef, undef
+  %g4 = lshr <4 x i32> undef, undef
+  %h4 = shl <4 x i32> undef, undef
+  %i4 = and <4 x i32> undef, undef
+  %j4 = or <4 x i32> undef, undef
+  %k4 = xor <4 x i32> undef, undef
+  %c8 = add <8 x i32> undef, undef
+  %d8 = sub <8 x i32> undef, undef
+  %e8 = mul <8 x i32> undef, undef
+  %f8 = ashr <8 x i32> undef, undef
+  %g8 = lshr <8 x i32> undef, undef
+  %h8 = shl <8 x i32> undef, undef
+  %i8 = and <8 x i32> undef, undef
+  %j8 = or <8 x i32> undef, undef
+  %k8 = xor <8 x i32> undef, undef
+  %c16 = add <16 x i32> undef, undef
+  %d16 = sub <16 x i32> undef, undef
+  %e16 = mul <16 x i32> undef, undef
+  %f16 = ashr <16 x i32> undef, undef
+  %g16 = lshr <16 x i32> undef, undef
+  %h16 = shl <16 x i32> undef, undef
+  %i16 = and <16 x i32> undef, undef
+  %j16 = or <16 x i32> undef, undef
+  %k16 = xor <16 x i32> undef, undef
+  ret void
+}
+
+define void @vi64() {
+; CHECK-MVE1-LABEL: 'vi64'
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %c2 = add <2 x i64> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %d2 = sub <2 x i64> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %e2 = mul <2 x i64> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %f2 = ashr <2 x i64> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %g2 = lshr <2 x i64> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %h2 = shl <2 x i64> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i2 = and <2 x i64> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %j2 = or <2 x i64> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %k2 = xor <2 x i64> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %c4 = add <4 x i64> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %d4 = sub <4 x i64> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %e4 = mul <4 x i64> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %f4 = ashr <4 x i64> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %g4 = lshr <4 x i64> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %h4 = shl <4 x i64> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %i4 = and <4 x i64> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %j4 = or <4 x i64> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %k4 = xor <4 x i64> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %c8 = add <8 x i64> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %d8 = sub <8 x i64> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %e8 = mul <8 x i64> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f8 = ashr <8 x i64> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %g8 = lshr <8 x i64> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %h8 = shl <8 x i64> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %i8 = and <8 x i64> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %j8 = or <8 x i64> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %k8 = xor <8 x i64> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 288 for instruction: %c16 = add <16 x i64> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 288 for instruction: %d16 = sub <16 x i64> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 288 for instruction: %e16 = mul <16 x i64> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 288 for instruction: %f16 = ashr <16 x i64> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 288 for instruction: %g16 = lshr <16 x i64> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 288 for instruction: %h16 = shl <16 x i64> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %i16 = and <16 x i64> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %j16 = or <16 x i64> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %k16 = xor <16 x i64> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-MVE2-LABEL: 'vi64'
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %c2 = add <2 x i64> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %d2 = sub <2 x i64> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %e2 = mul <2 x i64> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %f2 = ashr <2 x i64> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %g2 = lshr <2 x i64> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %h2 = shl <2 x i64> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %i2 = and <2 x i64> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %j2 = or <2 x i64> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %k2 = xor <2 x i64> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %c4 = add <4 x i64> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %d4 = sub <4 x i64> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %e4 = mul <4 x i64> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %f4 = ashr <4 x i64> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %g4 = lshr <4 x i64> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %h4 = shl <4 x i64> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %i4 = and <4 x i64> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %j4 = or <4 x i64> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %k4 = xor <4 x i64> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %c8 = add <8 x i64> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %d8 = sub <8 x i64> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %e8 = mul <8 x i64> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f8 = ashr <8 x i64> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %g8 = lshr <8 x i64> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %h8 = shl <8 x i64> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %i8 = and <8 x i64> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %j8 = or <8 x i64> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %k8 = xor <8 x i64> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 288 for instruction: %c16 = add <16 x i64> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 288 for instruction: %d16 = sub <16 x i64> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 288 for instruction: %e16 = mul <16 x i64> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 288 for instruction: %f16 = ashr <16 x i64> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 288 for instruction: %g16 = lshr <16 x i64> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 288 for instruction: %h16 = shl <16 x i64> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %i16 = and <16 x i64> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %j16 = or <16 x i64> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %k16 = xor <16 x i64> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-MVE4-LABEL: 'vi64'
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %c2 = add <2 x i64> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %d2 = sub <2 x i64> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %e2 = mul <2 x i64> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %f2 = ashr <2 x i64> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %g2 = lshr <2 x i64> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %h2 = shl <2 x i64> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %i2 = and <2 x i64> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %j2 = or <2 x i64> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %k2 = xor <2 x i64> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %c4 = add <4 x i64> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %d4 = sub <4 x i64> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %e4 = mul <4 x i64> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %f4 = ashr <4 x i64> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %g4 = lshr <4 x i64> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %h4 = shl <4 x i64> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %i4 = and <4 x i64> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %j4 = or <4 x i64> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %k4 = xor <4 x i64> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 144 for instruction: %c8 = add <8 x i64> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 144 for instruction: %d8 = sub <8 x i64> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 144 for instruction: %e8 = mul <8 x i64> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 144 for instruction: %f8 = ashr <8 x i64> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 144 for instruction: %g8 = lshr <8 x i64> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 144 for instruction: %h8 = shl <8 x i64> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %i8 = and <8 x i64> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %j8 = or <8 x i64> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %k8 = xor <8 x i64> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 544 for instruction: %c16 = add <16 x i64> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 544 for instruction: %d16 = sub <16 x i64> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 544 for instruction: %e16 = mul <16 x i64> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 544 for instruction: %f16 = ashr <16 x i64> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 544 for instruction: %g16 = lshr <16 x i64> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 544 for instruction: %h16 = shl <16 x i64> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %i16 = and <16 x i64> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %j16 = or <16 x i64> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %k16 = xor <16 x i64> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %c2 = add <2 x i64> undef, undef
+  %d2 = sub <2 x i64> undef, undef
+  %e2 = mul <2 x i64> undef, undef
+  %f2 = ashr <2 x i64> undef, undef
+  %g2 = lshr <2 x i64> undef, undef
+  %h2 = shl <2 x i64> undef, undef
+  %i2 = and <2 x i64> undef, undef
+  %j2 = or <2 x i64> undef, undef
+  %k2 = xor <2 x i64> undef, undef
+  %c4 = add <4 x i64> undef, undef
+  %d4 = sub <4 x i64> undef, undef
+  %e4 = mul <4 x i64> undef, undef
+  %f4 = ashr <4 x i64> undef, undef
+  %g4 = lshr <4 x i64> undef, undef
+  %h4 = shl <4 x i64> undef, undef
+  %i4 = and <4 x i64> undef, undef
+  %j4 = or <4 x i64> undef, undef
+  %k4 = xor <4 x i64> undef, undef
+  %c8 = add <8 x i64> undef, undef
+  %d8 = sub <8 x i64> undef, undef
+  %e8 = mul <8 x i64> undef, undef
+  %f8 = ashr <8 x i64> undef, undef
+  %g8 = lshr <8 x i64> undef, undef
+  %h8 = shl <8 x i64> undef, undef
+  %i8 = and <8 x i64> undef, undef
+  %j8 = or <8 x i64> undef, undef
+  %k8 = xor <8 x i64> undef, undef
+  %c16 = add <16 x i64> undef, undef
+  %d16 = sub <16 x i64> undef, undef
+  %e16 = mul <16 x i64> undef, undef
+  %f16 = ashr <16 x i64> undef, undef
+  %g16 = lshr <16 x i64> undef, undef
+  %h16 = shl <16 x i64> undef, undef
+  %i16 = and <16 x i64> undef, undef
+  %j16 = or <16 x i64> undef, undef
+  %k16 = xor <16 x i64> undef, undef
+  ret void
+}
diff --git a/llvm/test/Analysis/CostModel/ARM/cast.ll b/llvm/test/Analysis/CostModel/ARM/cast.ll
index 72308834c..0b068b5e0 100644
--- a/llvm/test/Analysis/CostModel/ARM/cast.ll
+++ b/llvm/test/Analysis/CostModel/ARM/cast.ll
@@ -1,548 +1,954 @@
-; RUN: opt < %s  -cost-model -analyze -mtriple=thumbv7-apple-ios6.0.0 -mcpu=cortex-a8 | FileCheck %s
-target datalayout = "e-p:32:32:32-i1:8:32-i8:8:32-i16:16:32-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:32:64-v128:32:128-a0:0:32-n32-S32"
-target triple = "thumbv7-apple-ios6.0.0"
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt < %s  -cost-model -analyze -mtriple=thumbv7-apple-ios6.0.0 -mcpu=cortex-a9 | FileCheck %s --check-prefix=CHECK-NEON
+; RUN: opt -cost-model -analyze -mtriple=thumbv8.1-m.main-none-eabi -mattr=+mve.fp < %s | FileCheck %s --check-prefix=CHECK-MVE
 
 define i32 @casts() {
-
-    ; -- scalars --
-  ; CHECK: Found an estimated cost of 1 for instruction:   %r0 = sext i1 undef to i8
+; CHECK-NEON-LABEL: 'casts'
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r0 = sext i1 undef to i8
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r1 = zext i1 undef to i8
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r2 = sext i1 undef to i16
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r3 = zext i1 undef to i16
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r4 = sext i1 undef to i32
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r5 = zext i1 undef to i32
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r6 = sext i1 undef to i64
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r7 = zext i1 undef to i64
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r8 = trunc i8 undef to i1
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r9 = sext i8 undef to i16
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r10 = zext i8 undef to i16
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r11 = sext i8 undef to i32
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r12 = zext i8 undef to i32
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r13 = sext i8 undef to i64
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r14 = zext i8 undef to i64
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r15 = trunc i16 undef to i1
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r16 = trunc i16 undef to i8
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r17 = sext i16 undef to i32
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r18 = zext i16 undef to i32
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r19 = sext i16 undef to i64
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r20 = zext i16 undef to i64
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r21 = trunc i32 undef to i1
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r22 = trunc i32 undef to i8
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r23 = trunc i32 undef to i16
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r24 = sext i32 undef to i64
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r25 = zext i32 undef to i64
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r26 = trunc i64 undef to i1
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r27 = trunc i64 undef to i8
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r28 = trunc i64 undef to i16
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r29 = trunc i64 undef to i32
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r30 = fptoui float undef to i1
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r31 = fptosi float undef to i1
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r32 = fptoui float undef to i8
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r33 = fptosi float undef to i8
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r34 = fptoui float undef to i16
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r35 = fptosi float undef to i16
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r36 = fptoui float undef to i32
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r37 = fptosi float undef to i32
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r38 = fptoui float undef to i64
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r39 = fptosi float undef to i64
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r40 = fptoui double undef to i1
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r41 = fptosi double undef to i1
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r42 = fptoui double undef to i8
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r43 = fptosi double undef to i8
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r44 = fptoui double undef to i16
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r45 = fptosi double undef to i16
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r46 = fptoui double undef to i32
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r47 = fptosi double undef to i32
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r48 = fptoui double undef to i64
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r49 = fptosi double undef to i64
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r50 = sitofp i1 undef to float
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r51 = uitofp i1 undef to float
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r52 = sitofp i1 undef to double
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r53 = uitofp i1 undef to double
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r54 = sitofp i8 undef to float
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r55 = uitofp i8 undef to float
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r56 = sitofp i8 undef to double
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r57 = uitofp i8 undef to double
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r58 = sitofp i16 undef to float
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r59 = uitofp i16 undef to float
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r60 = sitofp i16 undef to double
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r61 = uitofp i16 undef to double
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r62 = sitofp i32 undef to float
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r63 = uitofp i32 undef to float
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r64 = sitofp i32 undef to double
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r65 = uitofp i32 undef to double
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r66 = sitofp i64 undef to float
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r67 = uitofp i64 undef to float
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r68 = sitofp i64 undef to double
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r69 = uitofp i64 undef to double
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %q70 = sext <4 x i8> undef to <4 x i32>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %q71 = sext <8 x i8> undef to <8 x i16>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s70 = sext <4 x i8> undef to <4 x i32>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r70 = sext <8 x i8> undef to <8 x i32>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %r71 = sext <16 x i8> undef to <16 x i32>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %q72 = zext <4 x i8> undef to <4 x i32>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %q73 = zext <8 x i8> undef to <8 x i16>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s72 = zext <4 x i8> undef to <4 x i32>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r72 = zext <8 x i8> undef to <8 x i32>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %r73 = zext <16 x i8> undef to <16 x i32>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %rext_0 = sext <8 x i8> undef to <8 x i64>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %rext_1 = zext <8 x i8> undef to <8 x i64>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %rext_2 = sext <8 x i16> undef to <8 x i64>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %rext_3 = zext <8 x i16> undef to <8 x i64>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %rext_4 = sext <4 x i16> undef to <4 x i64>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %rext_5 = zext <4 x i16> undef to <4 x i64>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rext_6 = sext <2 x i8> undef to <2 x i64>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rext_7 = zext <2 x i8> undef to <2 x i64>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rext_8 = sext <2 x i16> undef to <2 x i64>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rext_9 = zext <2 x i16> undef to <2 x i64>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rext_a = sext <2 x i32> undef to <2 x i64>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rext_b = zext <2 x i32> undef to <2 x i64>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r74 = trunc <8 x i32> undef to <8 x i8>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %r75 = trunc <16 x i32> undef to <16 x i8>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r80 = fptrunc double undef to float
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r81 = fptrunc <2 x double> undef to <2 x float>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r82 = fptrunc <4 x double> undef to <4 x float>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r83 = fptrunc <8 x double> undef to <8 x float>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %r84 = fptrunc <16 x double> undef to <16 x float>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r85 = fpext float undef to double
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r86 = fpext <2 x float> undef to <2 x double>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r87 = fpext <4 x float> undef to <4 x double>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r88 = fpext <8 x float> undef to <8 x double>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %r89 = fpext <16 x float> undef to <16 x double>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r90 = fptoui <2 x float> undef to <2 x i1>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r91 = fptosi <2 x float> undef to <2 x i1>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r92 = fptoui <2 x float> undef to <2 x i8>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r93 = fptosi <2 x float> undef to <2 x i8>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r94 = fptoui <2 x float> undef to <2 x i16>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r95 = fptosi <2 x float> undef to <2 x i16>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r96 = fptoui <2 x float> undef to <2 x i32>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r97 = fptosi <2 x float> undef to <2 x i32>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %r98 = fptoui <2 x float> undef to <2 x i64>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %r99 = fptosi <2 x float> undef to <2 x i64>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %r100 = fptoui <2 x double> undef to <2 x i1>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %r101 = fptosi <2 x double> undef to <2 x i1>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %r102 = fptoui <2 x double> undef to <2 x i8>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %r103 = fptosi <2 x double> undef to <2 x i8>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %r104 = fptoui <2 x double> undef to <2 x i16>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %r105 = fptosi <2 x double> undef to <2 x i16>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r106 = fptoui <2 x double> undef to <2 x i32>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r107 = fptosi <2 x double> undef to <2 x i32>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %r108 = fptoui <2 x double> undef to <2 x i64>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %r109 = fptosi <2 x double> undef to <2 x i64>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %r110 = fptoui <4 x float> undef to <4 x i1>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %r111 = fptosi <4 x float> undef to <4 x i1>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r112 = fptoui <4 x float> undef to <4 x i8>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r113 = fptosi <4 x float> undef to <4 x i8>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r114 = fptoui <4 x float> undef to <4 x i16>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r115 = fptosi <4 x float> undef to <4 x i16>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r116 = fptoui <4 x float> undef to <4 x i32>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r117 = fptosi <4 x float> undef to <4 x i32>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: %r118 = fptoui <4 x float> undef to <4 x i64>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: %r119 = fptosi <4 x float> undef to <4 x i64>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %r120 = fptoui <4 x double> undef to <4 x i1>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %r121 = fptosi <4 x double> undef to <4 x i1>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %r122 = fptoui <4 x double> undef to <4 x i8>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %r123 = fptosi <4 x double> undef to <4 x i8>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %r124 = fptoui <4 x double> undef to <4 x i16>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %r125 = fptosi <4 x double> undef to <4 x i16>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %r126 = fptoui <4 x double> undef to <4 x i32>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %r127 = fptosi <4 x double> undef to <4 x i32>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: %r128 = fptoui <4 x double> undef to <4 x i64>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: %r129 = fptosi <4 x double> undef to <4 x i64>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: %r130 = fptoui <8 x float> undef to <8 x i1>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: %r131 = fptosi <8 x float> undef to <8 x i1>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %r132 = fptoui <8 x float> undef to <8 x i8>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %r133 = fptosi <8 x float> undef to <8 x i8>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r134 = fptoui <8 x float> undef to <8 x i16>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r135 = fptosi <8 x float> undef to <8 x i16>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r136 = fptoui <8 x float> undef to <8 x i32>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r137 = fptosi <8 x float> undef to <8 x i32>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 131 for instruction: %r138 = fptoui <8 x float> undef to <8 x i64>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 131 for instruction: %r139 = fptosi <8 x float> undef to <8 x i64>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %r140 = fptoui <8 x double> undef to <8 x i1>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %r141 = fptosi <8 x double> undef to <8 x i1>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %r142 = fptoui <8 x double> undef to <8 x i8>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %r143 = fptosi <8 x double> undef to <8 x i8>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %r144 = fptoui <8 x double> undef to <8 x i16>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %r145 = fptosi <8 x double> undef to <8 x i16>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %r146 = fptoui <8 x double> undef to <8 x i32>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %r147 = fptosi <8 x double> undef to <8 x i32>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 131 for instruction: %r148 = fptoui <8 x double> undef to <8 x i64>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 131 for instruction: %r149 = fptosi <8 x double> undef to <8 x i64>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 131 for instruction: %r150 = fptoui <16 x float> undef to <16 x i1>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 131 for instruction: %r151 = fptosi <16 x float> undef to <16 x i1>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %r152 = fptoui <16 x float> undef to <16 x i8>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %r153 = fptosi <16 x float> undef to <16 x i8>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r154 = fptoui <16 x float> undef to <16 x i16>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r155 = fptosi <16 x float> undef to <16 x i16>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r156 = fptoui <16 x float> undef to <16 x i32>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r157 = fptosi <16 x float> undef to <16 x i32>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 263 for instruction: %r158 = fptoui <16 x float> undef to <16 x i64>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 263 for instruction: %r159 = fptosi <16 x float> undef to <16 x i64>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 135 for instruction: %r160 = fptoui <16 x double> undef to <16 x i1>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 135 for instruction: %r161 = fptosi <16 x double> undef to <16 x i1>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 135 for instruction: %r162 = fptoui <16 x double> undef to <16 x i8>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 135 for instruction: %r163 = fptosi <16 x double> undef to <16 x i8>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 135 for instruction: %r164 = fptoui <16 x double> undef to <16 x i16>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 135 for instruction: %r165 = fptosi <16 x double> undef to <16 x i16>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %r166 = fptoui <16 x double> undef to <16 x i32>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %r167 = fptosi <16 x double> undef to <16 x i32>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 263 for instruction: %r168 = fptoui <16 x double> undef to <16 x i64>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 263 for instruction: %r169 = fptosi <16 x double> undef to <16 x i64>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %r170 = uitofp <2 x i1> undef to <2 x float>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %r171 = sitofp <2 x i1> undef to <2 x float>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r172 = uitofp <2 x i8> undef to <2 x float>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r173 = sitofp <2 x i8> undef to <2 x float>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r174 = uitofp <2 x i16> undef to <2 x float>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r175 = sitofp <2 x i16> undef to <2 x float>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r176 = uitofp <2 x i32> undef to <2 x float>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r177 = sitofp <2 x i32> undef to <2 x float>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %r178 = uitofp <2 x i64> undef to <2 x float>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %r179 = sitofp <2 x i64> undef to <2 x float>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r180 = uitofp <2 x i1> undef to <2 x double>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r181 = sitofp <2 x i1> undef to <2 x double>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r182 = uitofp <2 x i8> undef to <2 x double>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r183 = sitofp <2 x i8> undef to <2 x double>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r184 = uitofp <2 x i16> undef to <2 x double>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r185 = sitofp <2 x i16> undef to <2 x double>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r186 = uitofp <2 x i32> undef to <2 x double>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r187 = sitofp <2 x i32> undef to <2 x double>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %r188 = uitofp <2 x i64> undef to <2 x double>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %r189 = sitofp <2 x i64> undef to <2 x double>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r190 = uitofp <4 x i1> undef to <4 x float>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r191 = sitofp <4 x i1> undef to <4 x float>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r192 = uitofp <4 x i8> undef to <4 x float>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r193 = sitofp <4 x i8> undef to <4 x float>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r194 = uitofp <4 x i16> undef to <4 x float>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r195 = sitofp <4 x i16> undef to <4 x float>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r196 = uitofp <4 x i32> undef to <4 x float>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r197 = sitofp <4 x i32> undef to <4 x float>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 57 for instruction: %r198 = uitofp <4 x i64> undef to <4 x float>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 57 for instruction: %r199 = sitofp <4 x i64> undef to <4 x float>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %r200 = uitofp <4 x i1> undef to <4 x double>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %r201 = sitofp <4 x i1> undef to <4 x double>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %r202 = uitofp <4 x i8> undef to <4 x double>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %r203 = sitofp <4 x i8> undef to <4 x double>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %r204 = uitofp <4 x i16> undef to <4 x double>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %r205 = sitofp <4 x i16> undef to <4 x double>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %r206 = uitofp <4 x i32> undef to <4 x double>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %r207 = sitofp <4 x i32> undef to <4 x double>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 49 for instruction: %r208 = uitofp <4 x i64> undef to <4 x double>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 49 for instruction: %r209 = sitofp <4 x i64> undef to <4 x double>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %r210 = uitofp <8 x i1> undef to <8 x float>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %r211 = sitofp <8 x i1> undef to <8 x float>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %r212 = uitofp <8 x i8> undef to <8 x float>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %r213 = sitofp <8 x i8> undef to <8 x float>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r214 = uitofp <8 x i16> undef to <8 x float>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r215 = sitofp <8 x i16> undef to <8 x float>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r216 = uitofp <8 x i32> undef to <8 x float>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r217 = sitofp <8 x i32> undef to <8 x float>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 115 for instruction: %r218 = uitofp <8 x i64> undef to <8 x float>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 115 for instruction: %r219 = sitofp <8 x i64> undef to <8 x float>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %r220 = uitofp <8 x i1> undef to <8 x double>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %r221 = sitofp <8 x i1> undef to <8 x double>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %r222 = uitofp <8 x i8> undef to <8 x double>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %r223 = sitofp <8 x i8> undef to <8 x double>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %r224 = uitofp <8 x i16> undef to <8 x double>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %r225 = sitofp <8 x i16> undef to <8 x double>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %r226 = uitofp <8 x i16> undef to <8 x double>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %r227 = sitofp <8 x i16> undef to <8 x double>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 99 for instruction: %r228 = uitofp <8 x i64> undef to <8 x double>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 99 for instruction: %r229 = sitofp <8 x i64> undef to <8 x double>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %r230 = uitofp <16 x i1> undef to <16 x float>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %r231 = sitofp <16 x i1> undef to <16 x float>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %r232 = uitofp <16 x i8> undef to <16 x float>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %r233 = sitofp <16 x i8> undef to <16 x float>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r234 = uitofp <16 x i16> undef to <16 x float>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r235 = sitofp <16 x i16> undef to <16 x float>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r236 = uitofp <16 x i32> undef to <16 x float>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r237 = sitofp <16 x i32> undef to <16 x float>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 231 for instruction: %r238 = uitofp <16 x i64> undef to <16 x float>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 231 for instruction: %r239 = sitofp <16 x i64> undef to <16 x float>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %r240 = uitofp <16 x i1> undef to <16 x double>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %r241 = sitofp <16 x i1> undef to <16 x double>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: %r242 = uitofp <16 x i8> undef to <16 x double>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: %r243 = sitofp <16 x i8> undef to <16 x double>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %r244 = uitofp <16 x i16> undef to <16 x double>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %r245 = sitofp <16 x i16> undef to <16 x double>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %r246 = uitofp <16 x i16> undef to <16 x double>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %r247 = sitofp <16 x i16> undef to <16 x double>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 199 for instruction: %r248 = uitofp <16 x i64> undef to <16 x double>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 199 for instruction: %r249 = sitofp <16 x i64> undef to <16 x double>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; CHECK-MVE-LABEL: 'casts'
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r0 = sext i1 undef to i8
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r1 = zext i1 undef to i8
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r2 = sext i1 undef to i16
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r3 = zext i1 undef to i16
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r4 = sext i1 undef to i32
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r5 = zext i1 undef to i32
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r6 = sext i1 undef to i64
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r7 = zext i1 undef to i64
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r8 = trunc i8 undef to i1
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r9 = sext i8 undef to i16
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r10 = zext i8 undef to i16
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r11 = sext i8 undef to i32
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r12 = zext i8 undef to i32
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r13 = sext i8 undef to i64
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r14 = zext i8 undef to i64
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r15 = trunc i16 undef to i1
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r16 = trunc i16 undef to i8
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r17 = sext i16 undef to i32
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r18 = zext i16 undef to i32
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r19 = sext i16 undef to i64
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r20 = zext i16 undef to i64
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r21 = trunc i32 undef to i1
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r22 = trunc i32 undef to i8
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r23 = trunc i32 undef to i16
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r24 = sext i32 undef to i64
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r25 = zext i32 undef to i64
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r26 = trunc i64 undef to i1
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r27 = trunc i64 undef to i8
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r28 = trunc i64 undef to i16
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r29 = trunc i64 undef to i32
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r30 = fptoui float undef to i1
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r31 = fptosi float undef to i1
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r32 = fptoui float undef to i8
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r33 = fptosi float undef to i8
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r34 = fptoui float undef to i16
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r35 = fptosi float undef to i16
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r36 = fptoui float undef to i32
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r37 = fptosi float undef to i32
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r38 = fptoui float undef to i64
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r39 = fptosi float undef to i64
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r40 = fptoui double undef to i1
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r41 = fptosi double undef to i1
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r42 = fptoui double undef to i8
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r43 = fptosi double undef to i8
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r44 = fptoui double undef to i16
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r45 = fptosi double undef to i16
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r46 = fptoui double undef to i32
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r47 = fptosi double undef to i32
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r48 = fptoui double undef to i64
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r49 = fptosi double undef to i64
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r50 = sitofp i1 undef to float
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r51 = uitofp i1 undef to float
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r52 = sitofp i1 undef to double
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r53 = uitofp i1 undef to double
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r54 = sitofp i8 undef to float
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r55 = uitofp i8 undef to float
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r56 = sitofp i8 undef to double
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r57 = uitofp i8 undef to double
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r58 = sitofp i16 undef to float
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r59 = uitofp i16 undef to float
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r60 = sitofp i16 undef to double
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r61 = uitofp i16 undef to double
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r62 = sitofp i32 undef to float
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r63 = uitofp i32 undef to float
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r64 = sitofp i32 undef to double
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r65 = uitofp i32 undef to double
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r66 = sitofp i64 undef to float
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r67 = uitofp i64 undef to float
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r68 = sitofp i64 undef to double
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r69 = uitofp i64 undef to double
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %q70 = sext <4 x i8> undef to <4 x i32>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %q71 = sext <8 x i8> undef to <8 x i16>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %s70 = sext <4 x i8> undef to <4 x i32>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %r70 = sext <8 x i8> undef to <8 x i32>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 74 for instruction: %r71 = sext <16 x i8> undef to <16 x i32>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %q72 = zext <4 x i8> undef to <4 x i32>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %q73 = zext <8 x i8> undef to <8 x i16>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %s72 = zext <4 x i8> undef to <4 x i32>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %r72 = zext <8 x i8> undef to <8 x i32>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 74 for instruction: %r73 = zext <16 x i8> undef to <16 x i32>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 330 for instruction: %rext_0 = sext <8 x i8> undef to <8 x i64>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 74 for instruction: %rext_1 = zext <8 x i8> undef to <8 x i64>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 330 for instruction: %rext_2 = sext <8 x i16> undef to <8 x i64>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 74 for instruction: %rext_3 = zext <8 x i16> undef to <8 x i64>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 82 for instruction: %rext_4 = sext <4 x i16> undef to <4 x i64>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %rext_5 = zext <4 x i16> undef to <4 x i64>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %rext_6 = sext <2 x i8> undef to <2 x i64>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rext_7 = zext <2 x i8> undef to <2 x i64>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %rext_8 = sext <2 x i16> undef to <2 x i64>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rext_9 = zext <2 x i16> undef to <2 x i64>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %rext_a = sext <2 x i32> undef to <2 x i64>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rext_b = zext <2 x i32> undef to <2 x i64>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r74 = trunc <8 x i32> undef to <8 x i8>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r75 = trunc <16 x i32> undef to <16 x i8>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r80 = fptrunc double undef to float
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r81 = fptrunc <2 x double> undef to <2 x float>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r82 = fptrunc <4 x double> undef to <4 x float>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %r83 = fptrunc <8 x double> undef to <8 x float>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 85 for instruction: %r84 = fptrunc <16 x double> undef to <16 x float>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r85 = fpext float undef to double
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r86 = fpext <2 x float> undef to <2 x double>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 82 for instruction: %r87 = fpext <4 x float> undef to <4 x double>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 330 for instruction: %r88 = fpext <8 x float> undef to <8 x double>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 661 for instruction: %r89 = fpext <16 x float> undef to <16 x double>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r90 = fptoui <2 x float> undef to <2 x i1>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r91 = fptosi <2 x float> undef to <2 x i1>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r92 = fptoui <2 x float> undef to <2 x i8>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r93 = fptosi <2 x float> undef to <2 x i8>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r94 = fptoui <2 x float> undef to <2 x i16>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r95 = fptosi <2 x float> undef to <2 x i16>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r96 = fptoui <2 x float> undef to <2 x i32>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r97 = fptosi <2 x float> undef to <2 x i32>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r98 = fptoui <2 x float> undef to <2 x i64>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r99 = fptosi <2 x float> undef to <2 x i64>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r100 = fptoui <2 x double> undef to <2 x i1>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r101 = fptosi <2 x double> undef to <2 x i1>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r102 = fptoui <2 x double> undef to <2 x i8>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r103 = fptosi <2 x double> undef to <2 x i8>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r104 = fptoui <2 x double> undef to <2 x i16>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r105 = fptosi <2 x double> undef to <2 x i16>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r106 = fptoui <2 x double> undef to <2 x i32>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r107 = fptosi <2 x double> undef to <2 x i32>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r108 = fptoui <2 x double> undef to <2 x i64>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r109 = fptosi <2 x double> undef to <2 x i64>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r110 = fptoui <4 x float> undef to <4 x i1>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r111 = fptosi <4 x float> undef to <4 x i1>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r112 = fptoui <4 x float> undef to <4 x i8>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r113 = fptosi <4 x float> undef to <4 x i8>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r114 = fptoui <4 x float> undef to <4 x i16>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r115 = fptosi <4 x float> undef to <4 x i16>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r116 = fptoui <4 x float> undef to <4 x i32>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r117 = fptosi <4 x float> undef to <4 x i32>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 82 for instruction: %r118 = fptoui <4 x float> undef to <4 x i64>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 82 for instruction: %r119 = fptosi <4 x float> undef to <4 x i64>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 82 for instruction: %r120 = fptoui <4 x double> undef to <4 x i1>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 82 for instruction: %r121 = fptosi <4 x double> undef to <4 x i1>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 82 for instruction: %r122 = fptoui <4 x double> undef to <4 x i8>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 82 for instruction: %r123 = fptosi <4 x double> undef to <4 x i8>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 82 for instruction: %r124 = fptoui <4 x double> undef to <4 x i16>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 82 for instruction: %r125 = fptosi <4 x double> undef to <4 x i16>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 82 for instruction: %r126 = fptoui <4 x double> undef to <4 x i32>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 82 for instruction: %r127 = fptosi <4 x double> undef to <4 x i32>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 82 for instruction: %r128 = fptoui <4 x double> undef to <4 x i64>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 82 for instruction: %r129 = fptosi <4 x double> undef to <4 x i64>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r130 = fptoui <8 x float> undef to <8 x i1>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r131 = fptosi <8 x float> undef to <8 x i1>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r132 = fptoui <8 x float> undef to <8 x i8>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r133 = fptosi <8 x float> undef to <8 x i8>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r134 = fptoui <8 x float> undef to <8 x i16>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r135 = fptosi <8 x float> undef to <8 x i16>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r136 = fptoui <8 x float> undef to <8 x i32>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r137 = fptosi <8 x float> undef to <8 x i32>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 330 for instruction: %r138 = fptoui <8 x float> undef to <8 x i64>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 330 for instruction: %r139 = fptosi <8 x float> undef to <8 x i64>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 330 for instruction: %r140 = fptoui <8 x double> undef to <8 x i1>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 330 for instruction: %r141 = fptosi <8 x double> undef to <8 x i1>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 330 for instruction: %r142 = fptoui <8 x double> undef to <8 x i8>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 330 for instruction: %r143 = fptosi <8 x double> undef to <8 x i8>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 330 for instruction: %r144 = fptoui <8 x double> undef to <8 x i16>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 330 for instruction: %r145 = fptosi <8 x double> undef to <8 x i16>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 330 for instruction: %r146 = fptoui <8 x double> undef to <8 x i32>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 330 for instruction: %r147 = fptosi <8 x double> undef to <8 x i32>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 330 for instruction: %r148 = fptoui <8 x double> undef to <8 x i64>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 330 for instruction: %r149 = fptosi <8 x double> undef to <8 x i64>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %r150 = fptoui <16 x float> undef to <16 x i1>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %r151 = fptosi <16 x float> undef to <16 x i1>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %r152 = fptoui <16 x float> undef to <16 x i8>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %r153 = fptosi <16 x float> undef to <16 x i8>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %r154 = fptoui <16 x float> undef to <16 x i16>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %r155 = fptosi <16 x float> undef to <16 x i16>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r156 = fptoui <16 x float> undef to <16 x i32>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r157 = fptosi <16 x float> undef to <16 x i32>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1322 for instruction: %r158 = fptoui <16 x float> undef to <16 x i64>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1322 for instruction: %r159 = fptosi <16 x float> undef to <16 x i64>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 661 for instruction: %r160 = fptoui <16 x double> undef to <16 x i1>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 661 for instruction: %r161 = fptosi <16 x double> undef to <16 x i1>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 661 for instruction: %r162 = fptoui <16 x double> undef to <16 x i8>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 661 for instruction: %r163 = fptosi <16 x double> undef to <16 x i8>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 661 for instruction: %r164 = fptoui <16 x double> undef to <16 x i16>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 661 for instruction: %r165 = fptosi <16 x double> undef to <16 x i16>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 661 for instruction: %r166 = fptoui <16 x double> undef to <16 x i32>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 661 for instruction: %r167 = fptosi <16 x double> undef to <16 x i32>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 661 for instruction: %r168 = fptoui <16 x double> undef to <16 x i64>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 661 for instruction: %r169 = fptosi <16 x double> undef to <16 x i64>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r170 = uitofp <2 x i1> undef to <2 x float>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r171 = sitofp <2 x i1> undef to <2 x float>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r172 = uitofp <2 x i8> undef to <2 x float>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r173 = sitofp <2 x i8> undef to <2 x float>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r174 = uitofp <2 x i16> undef to <2 x float>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r175 = sitofp <2 x i16> undef to <2 x float>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r176 = uitofp <2 x i32> undef to <2 x float>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r177 = sitofp <2 x i32> undef to <2 x float>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r178 = uitofp <2 x i64> undef to <2 x float>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r179 = sitofp <2 x i64> undef to <2 x float>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %r180 = uitofp <2 x i1> undef to <2 x double>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %r181 = sitofp <2 x i1> undef to <2 x double>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %r182 = uitofp <2 x i8> undef to <2 x double>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %r183 = sitofp <2 x i8> undef to <2 x double>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %r184 = uitofp <2 x i16> undef to <2 x double>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %r185 = sitofp <2 x i16> undef to <2 x double>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %r186 = uitofp <2 x i32> undef to <2 x double>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %r187 = sitofp <2 x i32> undef to <2 x double>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %r188 = uitofp <2 x i64> undef to <2 x double>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %r189 = sitofp <2 x i64> undef to <2 x double>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r190 = uitofp <4 x i1> undef to <4 x float>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r191 = sitofp <4 x i1> undef to <4 x float>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r192 = uitofp <4 x i8> undef to <4 x float>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r193 = sitofp <4 x i8> undef to <4 x float>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r194 = uitofp <4 x i16> undef to <4 x float>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r195 = sitofp <4 x i16> undef to <4 x float>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r196 = uitofp <4 x i32> undef to <4 x float>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r197 = sitofp <4 x i32> undef to <4 x float>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r198 = uitofp <4 x i64> undef to <4 x float>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r199 = sitofp <4 x i64> undef to <4 x float>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: %r200 = uitofp <4 x i1> undef to <4 x double>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: %r201 = sitofp <4 x i1> undef to <4 x double>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: %r202 = uitofp <4 x i8> undef to <4 x double>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: %r203 = sitofp <4 x i8> undef to <4 x double>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: %r204 = uitofp <4 x i16> undef to <4 x double>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: %r205 = sitofp <4 x i16> undef to <4 x double>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: %r206 = uitofp <4 x i32> undef to <4 x double>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: %r207 = sitofp <4 x i32> undef to <4 x double>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: %r208 = uitofp <4 x i64> undef to <4 x double>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: %r209 = sitofp <4 x i64> undef to <4 x double>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r210 = uitofp <8 x i1> undef to <8 x float>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r211 = sitofp <8 x i1> undef to <8 x float>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r212 = uitofp <8 x i8> undef to <8 x float>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r213 = sitofp <8 x i8> undef to <8 x float>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r214 = uitofp <8 x i16> undef to <8 x float>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r215 = sitofp <8 x i16> undef to <8 x float>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r216 = uitofp <8 x i32> undef to <8 x float>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r217 = sitofp <8 x i32> undef to <8 x float>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %r218 = uitofp <8 x i64> undef to <8 x float>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %r219 = sitofp <8 x i64> undef to <8 x float>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 522 for instruction: %r220 = uitofp <8 x i1> undef to <8 x double>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 522 for instruction: %r221 = sitofp <8 x i1> undef to <8 x double>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 522 for instruction: %r222 = uitofp <8 x i8> undef to <8 x double>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 522 for instruction: %r223 = sitofp <8 x i8> undef to <8 x double>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 522 for instruction: %r224 = uitofp <8 x i16> undef to <8 x double>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 522 for instruction: %r225 = sitofp <8 x i16> undef to <8 x double>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 522 for instruction: %r226 = uitofp <8 x i16> undef to <8 x double>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 522 for instruction: %r227 = sitofp <8 x i16> undef to <8 x double>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 522 for instruction: %r228 = uitofp <8 x i64> undef to <8 x double>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 522 for instruction: %r229 = sitofp <8 x i64> undef to <8 x double>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %r230 = uitofp <16 x i1> undef to <16 x float>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %r231 = sitofp <16 x i1> undef to <16 x float>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %r232 = uitofp <16 x i8> undef to <16 x float>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %r233 = sitofp <16 x i8> undef to <16 x float>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %r234 = uitofp <16 x i16> undef to <16 x float>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %r235 = sitofp <16 x i16> undef to <16 x float>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r236 = uitofp <16 x i32> undef to <16 x float>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r237 = sitofp <16 x i32> undef to <16 x float>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 170 for instruction: %r238 = uitofp <16 x i64> undef to <16 x float>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 170 for instruction: %r239 = sitofp <16 x i64> undef to <16 x float>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1045 for instruction: %r240 = uitofp <16 x i1> undef to <16 x double>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1045 for instruction: %r241 = sitofp <16 x i1> undef to <16 x double>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1045 for instruction: %r242 = uitofp <16 x i8> undef to <16 x double>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1045 for instruction: %r243 = sitofp <16 x i8> undef to <16 x double>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1045 for instruction: %r244 = uitofp <16 x i16> undef to <16 x double>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1045 for instruction: %r245 = sitofp <16 x i16> undef to <16 x double>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1045 for instruction: %r246 = uitofp <16 x i16> undef to <16 x double>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1045 for instruction: %r247 = sitofp <16 x i16> undef to <16 x double>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1045 for instruction: %r248 = uitofp <16 x i64> undef to <16 x double>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1045 for instruction: %r249 = sitofp <16 x i64> undef to <16 x double>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  ; -- scalars --
   %r0 = sext i1 undef to i8
-  ; CHECK: Found an estimated cost of 1 for instruction:   %r1 = zext i1 undef to i8
   %r1 = zext i1 undef to i8
-  ; CHECK: Found an estimated cost of 1 for instruction:   %r2 = sext i1 undef to i16
   %r2 = sext i1 undef to i16
-  ; CHECK: Found an estimated cost of 1 for instruction:   %r3 = zext i1 undef to i16
   %r3 = zext i1 undef to i16
-  ; CHECK: Found an estimated cost of 1 for instruction:   %r4 = sext i1 undef to i32
   %r4 = sext i1 undef to i32
-  ; CHECK: Found an estimated cost of 1 for instruction:   %r5 = zext i1 undef to i32
   %r5 = zext i1 undef to i32
-  ; CHECK: Found an estimated cost of 1 for instruction:   %r6 = sext i1 undef to i64
   %r6 = sext i1 undef to i64
-  ; CHECK: Found an estimated cost of 1 for instruction:   %r7 = zext i1 undef to i64
   %r7 = zext i1 undef to i64
-  ; CHECK: Found an estimated cost of 0 for instruction:   %r8 = trunc i8 undef to i1
   %r8 = trunc i8 undef to i1
-  ; CHECK: Found an estimated cost of 1 for instruction:   %r9 = sext i8 undef to i16
   %r9 = sext i8 undef to i16
-  ; CHECK: Found an estimated cost of 1 for instruction:   %r10 = zext i8 undef to i16
   %r10 = zext i8 undef to i16
-  ; CHECK: Found an estimated cost of 1 for instruction:   %r11 = sext i8 undef to i32
   %r11 = sext i8 undef to i32
-  ; CHECK: Found an estimated cost of 1 for instruction:   %r12 = zext i8 undef to i32
   %r12 = zext i8 undef to i32
-  ; CHECK: Found an estimated cost of 1 for instruction:   %r13 = sext i8 undef to i64
   %r13 = sext i8 undef to i64
-  ; CHECK: Found an estimated cost of 1 for instruction:   %r14 = zext i8 undef to i64
   %r14 = zext i8 undef to i64
-  ; CHECK: Found an estimated cost of 0 for instruction:   %r15 = trunc i16 undef to i1
   %r15 = trunc i16 undef to i1
-  ; CHECK: Found an estimated cost of 0 for instruction:   %r16 = trunc i16 undef to i8
   %r16 = trunc i16 undef to i8
-  ; CHECK: Found an estimated cost of 1 for instruction:   %r17 = sext i16 undef to i32
   %r17 = sext i16 undef to i32
-  ; CHECK: Found an estimated cost of 1 for instruction:   %r18 = zext i16 undef to i32
   %r18 = zext i16 undef to i32
-  ; CHECK: Found an estimated cost of 2 for instruction:   %r19 = sext i16 undef to i64
   %r19 = sext i16 undef to i64
-  ; CHECK: Found an estimated cost of 1 for instruction:   %r20 = zext i16 undef to i64
   %r20 = zext i16 undef to i64
-  ; CHECK: Found an estimated cost of 0 for instruction:   %r21 = trunc i32 undef to i1
   %r21 = trunc i32 undef to i1
-  ; CHECK: Found an estimated cost of 0 for instruction:   %r22 = trunc i32 undef to i8
   %r22 = trunc i32 undef to i8
-  ; CHECK: Found an estimated cost of 0 for instruction:   %r23 = trunc i32 undef to i16
   %r23 = trunc i32 undef to i16
-  ; CHECK: Found an estimated cost of 1 for instruction:   %r24 = sext i32 undef to i64
   %r24 = sext i32 undef to i64
-  ; CHECK: Found an estimated cost of 1 for instruction:   %r25 = zext i32 undef to i64
   %r25 = zext i32 undef to i64
-  ; CHECK: Found an estimated cost of 0 for instruction:   %r26 = trunc i64 undef to i1
   %r26 = trunc i64 undef to i1
-  ; CHECK: Found an estimated cost of 0 for instruction:   %r27 = trunc i64 undef to i8
   %r27 = trunc i64 undef to i8
-  ; CHECK: Found an estimated cost of 0 for instruction:   %r28 = trunc i64 undef to i16
   %r28 = trunc i64 undef to i16
-  ; CHECK: Found an estimated cost of 0 for instruction:   %r29 = trunc i64 undef to i32
   %r29 = trunc i64 undef to i32
 
-    ; -- floating point conversions --
+  ; -- floating point conversions --
   ; Moves between scalar and NEON registers.
-  ; CHECK: Found an estimated cost of 2 for instruction:   %r30 = fptoui float undef to i1
   %r30 = fptoui float undef to i1
-  ; CHECK: Found an estimated cost of 2 for instruction:   %r31 = fptosi float undef to i1
   %r31 = fptosi float undef to i1
-  ; CHECK: Found an estimated cost of 2 for instruction:   %r32 = fptoui float undef to i8
   %r32 = fptoui float undef to i8
-  ; CHECK: Found an estimated cost of 2 for instruction:   %r33 = fptosi float undef to i8
   %r33 = fptosi float undef to i8
-  ; CHECK:  Found an estimated cost of 2 for instruction:   %r34 = fptoui float undef to i16
   %r34 = fptoui float undef to i16
-  ; CHECK:  Found an estimated cost of 2 for instruction:   %r35 = fptosi float undef to i16
   %r35 = fptosi float undef to i16
-  ; CHECK:  Found an estimated cost of 2 for instruction:   %r36 = fptoui float undef to i32
   %r36 = fptoui float undef to i32
-  ; CHECK:  Found an estimated cost of 2 for instruction:   %r37 = fptosi float undef to i32
   %r37 = fptosi float undef to i32
-  ; CHECK:  Found an estimated cost of 10 for instruction:   %r38 = fptoui float undef to i64
   %r38 = fptoui float undef to i64
-  ; CHECK:  Found an estimated cost of 10 for instruction:   %r39 = fptosi float undef to i64
   %r39 = fptosi float undef to i64
-  ; CHECK:  Found an estimated cost of 2 for instruction:   %r40 = fptoui double undef to i1
   %r40 = fptoui double undef to i1
-  ; CHECK:  Found an estimated cost of 2 for instruction:   %r41 = fptosi double undef to i1
   %r41 = fptosi double undef to i1
-  ; CHECK:  Found an estimated cost of 2 for instruction:   %r42 = fptoui double undef to i8
   %r42 = fptoui double undef to i8
-  ; CHECK:  Found an estimated cost of 2 for instruction:   %r43 = fptosi double undef to i8
   %r43 = fptosi double undef to i8
-  ; CHECK:  Found an estimated cost of 2 for instruction:   %r44 = fptoui double undef to i16
   %r44 = fptoui double undef to i16
-  ; CHECK:  Found an estimated cost of 2 for instruction:   %r45 = fptosi double undef to i16
   %r45 = fptosi double undef to i16
-  ; CHECK:  Found an estimated cost of 2 for instruction:   %r46 = fptoui double undef to i32
   %r46 = fptoui double undef to i32
-  ; CHECK:  Found an estimated cost of 2 for instruction:   %r47 = fptosi double undef to i32
   %r47 = fptosi double undef to i32
   ; Function call
-  ; CHECK:  Found an estimated cost of 10 for instruction:   %r48 = fptoui double undef to i64
   %r48 = fptoui double undef to i64
-  ; CHECK:  Found an estimated cost of 10 for instruction:   %r49 = fptosi double undef to i64
   %r49 = fptosi double undef to i64
 
-  ; CHECK:  Found an estimated cost of 2 for instruction:   %r50 = sitofp i1 undef to float
   %r50 = sitofp i1 undef to float
-  ; CHECK:  Found an estimated cost of 2 for instruction:   %r51 = uitofp i1 undef to float
   %r51 = uitofp i1 undef to float
-  ; CHECK:  Found an estimated cost of 2 for instruction:   %r52 = sitofp i1 undef to double
   %r52 = sitofp i1 undef to double
-  ; CHECK:  Found an estimated cost of 2 for instruction:   %r53 = uitofp i1 undef to double
   %r53 = uitofp i1 undef to double
-  ; CHECK:  Found an estimated cost of 2 for instruction:   %r54 = sitofp i8 undef to float
   %r54 = sitofp i8 undef to float
-  ; CHECK:  Found an estimated cost of 2 for instruction:   %r55 = uitofp i8 undef to float
   %r55 = uitofp i8 undef to float
-  ; CHECK:  Found an estimated cost of 2 for instruction:   %r56 = sitofp i8 undef to double
   %r56 = sitofp i8 undef to double
-  ; CHECK:  Found an estimated cost of 2 for instruction:   %r57 = uitofp i8 undef to double
   %r57 = uitofp i8 undef to double
-  ; CHECK:  Found an estimated cost of 2 for instruction:   %r58 = sitofp i16 undef to float
   %r58 = sitofp i16 undef to float
-  ; CHECK:  Found an estimated cost of 2 for instruction:   %r59 = uitofp i16 undef to float
   %r59 = uitofp i16 undef to float
-  ; CHECK:  Found an estimated cost of 2 for instruction:   %r60 = sitofp i16 undef to double
   %r60 = sitofp i16 undef to double
-  ; CHECK:  Found an estimated cost of 2 for instruction:   %r61 = uitofp i16 undef to double
   %r61 = uitofp i16 undef to double
-  ; CHECK:  Found an estimated cost of 2 for instruction:   %r62 = sitofp i32 undef to float
   %r62 = sitofp i32 undef to float
-  ; CHECK:  Found an estimated cost of 2 for instruction:   %r63 = uitofp i32 undef to float
   %r63 = uitofp i32 undef to float
-  ; CHECK:  Found an estimated cost of 2 for instruction:   %r64 = sitofp i32 undef to double
   %r64 = sitofp i32 undef to double
-  ; CHECK:  Found an estimated cost of 2 for instruction:   %r65 = uitofp i32 undef to double
   %r65 = uitofp i32 undef to double
   ; Function call
-  ; CHECK:  Found an estimated cost of 10 for instruction:   %r66 = sitofp i64 undef to float
   %r66 = sitofp i64 undef to float
-  ; CHECK:  Found an estimated cost of 10 for instruction:   %r67 = uitofp i64 undef to float
   %r67 = uitofp i64 undef to float
-  ; CHECK:  Found an estimated cost of 10 for instruction:   %r68 = sitofp i64 undef to double
   %r68 = sitofp i64 undef to double
-  ; CHECK:  Found an estimated cost of 10 for instruction:   %r69 = uitofp i64 undef to double
   %r69 = uitofp i64 undef to double
 
-  ; CHECK:  Found an estimated cost of 3 for instruction:   %r70 = sext <8 x i8> undef to <8 x i32>
+  %q70 = sext <4 x i8> undef to <4 x i32>
+  %q71 = sext <8 x i8> undef to <8 x i16>
+  %s70 = sext <4 x i8> undef to <4 x i32>
   %r70 = sext <8 x i8> undef to <8 x i32>
-  ; CHECK:  Found an estimated cost of 6 for instruction:   %r71 = sext <16 x i8> undef to <16 x i32>
   %r71 = sext <16 x i8> undef to <16 x i32>
-  ; CHECK:  Found an estimated cost of 3 for instruction:   %r72 = zext <8 x i8> undef to <8 x i32>
+  %q72 = zext <4 x i8> undef to <4 x i32>
+  %q73 = zext <8 x i8> undef to <8 x i16>
+  %s72 = zext <4 x i8> undef to <4 x i32>
   %r72 = zext <8 x i8> undef to <8 x i32>
-  ; CHECK:  Found an estimated cost of 6 for instruction:   %r73 = zext <16 x i8> undef to <16 x i32>
   %r73 = zext <16 x i8> undef to <16 x i32>
 
-  ; CHECK:  Found an estimated cost of 7 for instruction:   %rext_0 = sext <8 x i8> undef to <8 x i64>
   %rext_0 = sext <8 x i8> undef to <8 x i64>
-  ; CHECK:  Found an estimated cost of 7 for instruction:   %rext_1 = zext <8 x i8> undef to <8 x i64>
   %rext_1 = zext <8 x i8> undef to <8 x i64>
-  ; CHECK:  Found an estimated cost of 6 for instruction:   %rext_2 = sext <8 x i16> undef to <8 x i64>
   %rext_2 = sext <8 x i16> undef to <8 x i64>
-  ; CHECK:  Found an estimated cost of 6 for instruction:   %rext_3 = zext <8 x i16> undef to <8 x i64>
   %rext_3 = zext <8 x i16> undef to <8 x i64>
-  ; CHECK:  Found an estimated cost of 3 for instruction:   %rext_4 = sext <4 x i16> undef to <4 x i64>
   %rext_4 = sext <4 x i16> undef to <4 x i64>
-  ; CHECK:  Found an estimated cost of 3 for instruction:   %rext_5 = zext <4 x i16> undef to <4 x i64>
   %rext_5 = zext <4 x i16> undef to <4 x i64>
+  %rext_6 = sext <2 x i8> undef to <2 x i64>
+  %rext_7 = zext <2 x i8> undef to <2 x i64>
+  %rext_8 = sext <2 x i16> undef to <2 x i64>
+  %rext_9 = zext <2 x i16> undef to <2 x i64>
+  %rext_a = sext <2 x i32> undef to <2 x i64>
+  %rext_b = zext <2 x i32> undef to <2 x i64>
 
   ; Vector cast cost of instructions lowering the cast to the stack.
-  ; CHECK:  Found an estimated cost of 3 for instruction:   %r74 = trunc <8 x i32> undef to <8 x i8>
   %r74 = trunc <8 x i32> undef to <8 x i8>
-  ; CHECK:  Found an estimated cost of 6 for instruction:   %r75 = trunc <16 x i32> undef to <16 x i8>
   %r75 = trunc <16 x i32> undef to <16 x i8>
 
   ; Floating point truncation costs.
-  ; CHECK:  Found an estimated cost of 1 for instruction:   %r80 = fptrunc double undef to float
   %r80 = fptrunc double undef to float
-  ; CHECK:  Found an estimated cost of 2 for instruction:   %r81 = fptrunc <2 x double> undef to <2 x float>
   %r81 = fptrunc <2 x double> undef to <2 x float>
-  ; CHECK:  Found an estimated cost of 4 for instruction:   %r82 = fptrunc <4 x double> undef to <4 x float>
   %r82 = fptrunc <4 x double> undef to <4 x float>
-  ; CHECK:  Found an estimated cost of 8 for instruction:   %r83 = fptrunc <8 x double> undef to <8 x float>
   %r83 = fptrunc <8 x double> undef to <8 x float>
-  ; CHECK:  Found an estimated cost of 16 for instruction:   %r84 = fptrunc <16 x double> undef to <16 x float>
   %r84 = fptrunc <16 x double> undef to <16 x float>
 
   ; Floating point extension costs.
-  ; CHECK:  Found an estimated cost of 1 for instruction:   %r85 = fpext float undef to double
   %r85 = fpext float undef to double
-  ; CHECK:  Found an estimated cost of 2 for instruction:   %r86 = fpext <2 x float> undef to <2 x double>
   %r86 = fpext <2 x float> undef to <2 x double>
-  ; CHECK:  Found an estimated cost of 4 for instruction:   %r87 = fpext <4 x float> undef to <4 x double>
   %r87 = fpext <4 x float> undef to <4 x double>
-  ; CHECK:  Found an estimated cost of 8 for instruction:   %r88 = fpext <8 x float> undef to <8 x double>
   %r88 = fpext <8 x float> undef to <8 x double>
-  ; CHECK:  Found an estimated cost of 16 for instruction:   %r89 = fpext <16 x float> undef to <16 x double>
   %r89 = fpext <16 x float> undef to <16 x double>
 
   ;; Floating point to integer vector casts.
-  ; CHECK:  Found an estimated cost of 1 for instruction:   %r90 = fptoui <2 x float> undef to <2 x i1>
   %r90 = fptoui <2 x float> undef to <2 x i1>
-  ; CHECK:  Found an estimated cost of 1 for instruction:   %r91 = fptosi <2 x float> undef to <2 x i1>
   %r91 = fptosi <2 x float> undef to <2 x i1>
-  ; CHECK:  Found an estimated cost of 1 for instruction:   %r92 = fptoui <2 x float> undef to <2 x i8>
   %r92 = fptoui <2 x float> undef to <2 x i8>
-  ; CHECK:  Found an estimated cost of 1 for instruction:   %r93 = fptosi <2 x float> undef to <2 x i8>
   %r93 = fptosi <2 x float> undef to <2 x i8>
-  ; CHECK:  Found an estimated cost of 1 for instruction:   %r94 = fptoui <2 x float> undef to <2 x i16>
   %r94 = fptoui <2 x float> undef to <2 x i16>
-  ; CHECK:  Found an estimated cost of 1 for instruction:   %r95 = fptosi <2 x float> undef to <2 x i16>
   %r95 = fptosi <2 x float> undef to <2 x i16>
-  ; CHECK:  Found an estimated cost of 1 for instruction:   %r96 = fptoui <2 x float> undef to <2 x i32>
   %r96 = fptoui <2 x float> undef to <2 x i32>
-  ; CHECK:  Found an estimated cost of 1 for instruction:   %r97 = fptosi <2 x float> undef to <2 x i32>
   %r97 = fptosi <2 x float> undef to <2 x i32>
-  ; CHECK:  Found an estimated cost of 32 for instruction:   %r98 = fptoui <2 x float> undef to <2 x i64>
   %r98 = fptoui <2 x float> undef to <2 x i64>
-  ; CHECK:  Found an estimated cost of 32 for instruction:   %r99 = fptosi <2 x float> undef to <2 x i64>
   %r99 = fptosi <2 x float> undef to <2 x i64>
 
-  ; CHECK:  Found an estimated cost of 16 for instruction:   %r100 = fptoui <2 x double> undef to <2 x i1>
   %r100 = fptoui <2 x double> undef to <2 x i1>
-  ; CHECK:  Found an estimated cost of 16 for instruction:   %r101 = fptosi <2 x double> undef to <2 x i1>
   %r101 = fptosi <2 x double> undef to <2 x i1>
-  ; CHECK:  Found an estimated cost of 16 for instruction:   %r102 = fptoui <2 x double> undef to <2 x i8>
   %r102 = fptoui <2 x double> undef to <2 x i8>
-  ; CHECK:  Found an estimated cost of 16 for instruction:   %r103 = fptosi <2 x double> undef to <2 x i8>
   %r103 = fptosi <2 x double> undef to <2 x i8>
-  ; CHECK:  Found an estimated cost of 16 for instruction:   %r104 = fptoui <2 x double> undef to <2 x i16>
   %r104 = fptoui <2 x double> undef to <2 x i16>
-  ; CHECK:  Found an estimated cost of 16 for instruction:   %r105 = fptosi <2 x double> undef to <2 x i16>
   %r105 = fptosi <2 x double> undef to <2 x i16>
-  ; CHECK:  Found an estimated cost of 2 for instruction:   %r106 = fptoui <2 x double> undef to <2 x i32>
   %r106 = fptoui <2 x double> undef to <2 x i32>
-  ; CHECK:  Found an estimated cost of 2 for instruction:   %r107 = fptosi <2 x double> undef to <2 x i32>
   %r107 = fptosi <2 x double> undef to <2 x i32>
-  ; CHECK:  Found an estimated cost of 32 for instruction:   %r108 = fptoui <2 x double> undef to <2 x i64>
   %r108 = fptoui <2 x double> undef to <2 x i64>
-  ; CHECK:  Found an estimated cost of 32 for instruction:   %r109 = fptosi <2 x double> undef to <2 x i64>
   %r109 = fptosi <2 x double> undef to <2 x i64>
 
-  ; CHECK:  Found an estimated cost of 32 for instruction:   %r110 = fptoui <4 x float> undef to <4 x i1>
   %r110 = fptoui <4 x float> undef to <4 x i1>
-  ; CHECK:  Found an estimated cost of 32 for instruction:   %r111 = fptosi <4 x float> undef to <4 x i1>
   %r111 = fptosi <4 x float> undef to <4 x i1>
-  ; CHECK:  Found an estimated cost of 3 for instruction:   %r112 = fptoui <4 x float> undef to <4 x i8>
   %r112 = fptoui <4 x float> undef to <4 x i8>
-  ; CHECK:  Found an estimated cost of 3 for instruction:   %r113 = fptosi <4 x float> undef to <4 x i8>
   %r113 = fptosi <4 x float> undef to <4 x i8>
 
-  ; CHECK:  Found an estimated cost of 2 for instruction:   %r114 = fptoui <4 x float> undef to <4 x i16>
   %r114 = fptoui <4 x float> undef to <4 x i16>
-  ; CHECK:  Found an estimated cost of 2 for instruction:   %r115 = fptosi <4 x float> undef to <4 x i16>
   %r115 = fptosi <4 x float> undef to <4 x i16>
-  ; CHECK:  Found an estimated cost of 1 for instruction:   %r116 = fptoui <4 x float> undef to <4 x i32>
   %r116 = fptoui <4 x float> undef to <4 x i32>
-  ; CHECK:  Found an estimated cost of 1 for instruction:   %r117 = fptosi <4 x float> undef to <4 x i32>
   %r117 = fptosi <4 x float> undef to <4 x i32>
-  ; CHECK:  Found an estimated cost of 65 for instruction:   %r118 = fptoui <4 x float> undef to <4 x i64>
   %r118 = fptoui <4 x float> undef to <4 x i64>
-  ; CHECK:  Found an estimated cost of 65 for instruction:   %r119 = fptosi <4 x float> undef to <4 x i64>
   %r119 = fptosi <4 x float> undef to <4 x i64>
 
-  ; CHECK:  Found an estimated cost of 33 for instruction:   %r120 = fptoui <4 x double> undef to <4 x i1>
   %r120 = fptoui <4 x double> undef to <4 x i1>
-  ; CHECK:  Found an estimated cost of 33 for instruction:   %r121 = fptosi <4 x double> undef to <4 x i1>
   %r121 = fptosi <4 x double> undef to <4 x i1>
-  ; CHECK:  Found an estimated cost of 33 for instruction:   %r122 = fptoui <4 x double> undef to <4 x i8>
   %r122 = fptoui <4 x double> undef to <4 x i8>
-  ; CHECK:  Found an estimated cost of 33 for instruction:   %r123 = fptosi <4 x double> undef to <4 x i8>
   %r123 = fptosi <4 x double> undef to <4 x i8>
-  ; CHECK:  Found an estimated cost of 33 for instruction:   %r124 = fptoui <4 x double> undef to <4 x i16>
   %r124 = fptoui <4 x double> undef to <4 x i16>
-  ; CHECK:  Found an estimated cost of 33 for instruction:   %r125 = fptosi <4 x double> undef to <4 x i16>
   %r125 = fptosi <4 x double> undef to <4 x i16>
-  ; CHECK:  Found an estimated cost of 5 for instruction:   %r126 = fptoui <4 x double> undef to <4 x i32>
   %r126 = fptoui <4 x double> undef to <4 x i32>
-  ; CHECK:  Found an estimated cost of 5 for instruction:   %r127 = fptosi <4 x double> undef to <4 x i32>
   %r127 = fptosi <4 x double> undef to <4 x i32>
-  ; CHECK:  Found an estimated cost of 65 for instruction:   %r128 = fptoui <4 x double> undef to <4 x i64>
   %r128 = fptoui <4 x double> undef to <4 x i64>
-  ; CHECK:  Found an estimated cost of 65 for instruction:   %r129 = fptosi <4 x double> undef to <4 x i64>
   %r129 = fptosi <4 x double> undef to <4 x i64>
 
-  ; CHECK:  Found an estimated cost of 65 for instruction:   %r130 = fptoui <8 x float> undef to <8 x i1>
   %r130 = fptoui <8 x float> undef to <8 x i1>
-  ; CHECK:  Found an estimated cost of 65 for instruction:   %r131 = fptosi <8 x float> undef to <8 x i1>
   %r131 = fptosi <8 x float> undef to <8 x i1>
-  ; CHECK:  Found an estimated cost of 7 for instruction:   %r132 = fptoui <8 x float> undef to <8 x i8>
   %r132 = fptoui <8 x float> undef to <8 x i8>
-  ; CHECK:  Found an estimated cost of 7 for instruction:   %r133 = fptosi <8 x float> undef to <8 x i8>
   %r133 = fptosi <8 x float> undef to <8 x i8>
-  ; CHECK:  Found an estimated cost of 4 for instruction:   %r134 = fptoui <8 x float> undef to <8 x i16>
   %r134 = fptoui <8 x float> undef to <8 x i16>
-  ; CHECK:  Found an estimated cost of 4 for instruction:   %r135 = fptosi <8 x float> undef to <8 x i16>
   %r135 = fptosi <8 x float> undef to <8 x i16>
-  ; CHECK:  Found an estimated cost of 2 for instruction:   %r136 = fptoui <8 x float> undef to <8 x i32>
   %r136 = fptoui <8 x float> undef to <8 x i32>
-  ; CHECK:  Found an estimated cost of 2 for instruction:   %r137 = fptosi <8 x float> undef to <8 x i32>
   %r137 = fptosi <8 x float> undef to <8 x i32>
-  ; CHECK:  Found an estimated cost of 131 for instruction:   %r138 = fptoui <8 x float> undef to <8 x i64>
   %r138 = fptoui <8 x float> undef to <8 x i64>
-  ; CHECK:  Found an estimated cost of 131 for instruction:   %r139 = fptosi <8 x float> undef to <8 x i64>
   %r139 = fptosi <8 x float> undef to <8 x i64>
 
-  ; CHECK:  Found an estimated cost of 67 for instruction:   %r140 = fptoui <8 x double> undef to <8 x i1>
   %r140 = fptoui <8 x double> undef to <8 x i1>
-  ; CHECK:  Found an estimated cost of 67 for instruction:   %r141 = fptosi <8 x double> undef to <8 x i1>
   %r141 = fptosi <8 x double> undef to <8 x i1>
-  ; CHECK:  Found an estimated cost of 67 for instruction:   %r142 = fptoui <8 x double> undef to <8 x i8>
   %r142 = fptoui <8 x double> undef to <8 x i8>
-  ; CHECK:  Found an estimated cost of 67 for instruction:   %r143 = fptosi <8 x double> undef to <8 x i8>
   %r143 = fptosi <8 x double> undef to <8 x i8>
-  ; CHECK:  Found an estimated cost of 67 for instruction:   %r144 = fptoui <8 x double> undef to <8 x i16>
   %r144 = fptoui <8 x double> undef to <8 x i16>
-  ; CHECK:  Found an estimated cost of 67 for instruction:   %r145 = fptosi <8 x double> undef to <8 x i16>
   %r145 = fptosi <8 x double> undef to <8 x i16>
-  ; CHECK:  Found an estimated cost of 11 for instruction:   %r146 = fptoui <8 x double> undef to <8 x i32>
   %r146 = fptoui <8 x double> undef to <8 x i32>
-  ; CHECK:  Found an estimated cost of 11 for instruction:   %r147 = fptosi <8 x double> undef to <8 x i32>
   %r147 = fptosi <8 x double> undef to <8 x i32>
-  ; CHECK:  Found an estimated cost of 131 for instruction:   %r148 = fptoui <8 x double> undef to <8 x i64>
   %r148 = fptoui <8 x double> undef to <8 x i64>
-  ; CHECK:  Found an estimated cost of 131 for instruction:   %r149 = fptosi <8 x double> undef to <8 x i64>
   %r149 = fptosi <8 x double> undef to <8 x i64>
 
-  ; CHECK:  Found an estimated cost of 131 for instruction:   %r150 = fptoui <16 x float> undef to <16 x i1>
   %r150 = fptoui <16 x float> undef to <16 x i1>
-  ; CHECK:  Found an estimated cost of 131 for instruction:   %r151 = fptosi <16 x float> undef to <16 x i1>
   %r151 = fptosi <16 x float> undef to <16 x i1>
- ; CHECK:  Found an estimated cost of 15 for instruction:   %r152 = fptoui <16 x float> undef to <16 x i8>
   %r152 = fptoui <16 x float> undef to <16 x i8>
-  ; CHECK:  Found an estimated cost of 15 for instruction:   %r153 = fptosi <16 x float> undef to <16 x i8>
   %r153 = fptosi <16 x float> undef to <16 x i8>
-  ; CHECK:  Found an estimated cost of 8 for instruction:   %r154 = fptoui <16 x float> undef to <16 x i16>
   %r154 = fptoui <16 x float> undef to <16 x i16>
-  ; CHECK:  Found an estimated cost of 8 for instruction:   %r155 = fptosi <16 x float> undef to <16 x i16>
   %r155 = fptosi <16 x float> undef to <16 x i16>
-  ; CHECK:  Found an estimated cost of 4 for instruction:   %r156 = fptoui <16 x float> undef to <16 x i32>
   %r156 = fptoui <16 x float> undef to <16 x i32>
-  ; CHECK:  Found an estimated cost of 4 for instruction:   %r157 = fptosi <16 x float> undef to <16 x i32>
   %r157 = fptosi <16 x float> undef to <16 x i32>
-  ; CHECK:  Found an estimated cost of 263 for instruction:   %r158 = fptoui <16 x float> undef to <16 x i64>
   %r158 = fptoui <16 x float> undef to <16 x i64>
-  ; CHECK:  Found an estimated cost of 263 for instruction:   %r159 = fptosi <16 x float> undef to <16 x i64>
   %r159 = fptosi <16 x float> undef to <16 x i64>
 
-  ; CHECK:  Found an estimated cost of 135 for instruction:   %r160 = fptoui <16 x double> undef to <16 x i1>
   %r160 = fptoui <16 x double> undef to <16 x i1>
-  ; CHECK:  Found an estimated cost of 135 for instruction:   %r161 = fptosi <16 x double> undef to <16 x i1>
   %r161 = fptosi <16 x double> undef to <16 x i1>
-  ; CHECK:  Found an estimated cost of 135 for instruction:   %r162 = fptoui <16 x double> undef to <16 x i8>
   %r162 = fptoui <16 x double> undef to <16 x i8>
-  ; CHECK:  Found an estimated cost of 135 for instruction:   %r163 = fptosi <16 x double> undef to <16 x i8>
   %r163 = fptosi <16 x double> undef to <16 x i8>
-  ; CHECK:  Found an estimated cost of 135 for instruction:   %r164 = fptoui <16 x double> undef to <16 x i16>
   %r164 = fptoui <16 x double> undef to <16 x i16>
-  ; CHECK:  Found an estimated cost of 135 for instruction:   %r165 = fptosi <16 x double> undef to <16 x i16>
   %r165 = fptosi <16 x double> undef to <16 x i16>
-  ; CHECK:  Found an estimated cost of 23 for instruction:   %r166 = fptoui <16 x double> undef to <16 x i32>
   %r166 = fptoui <16 x double> undef to <16 x i32>
-  ; CHECK:  Found an estimated cost of 23 for instruction:   %r167 = fptosi <16 x double> undef to <16 x i32>
   %r167 = fptosi <16 x double> undef to <16 x i32>
-  ; CHECK:  Found an estimated cost of 263 for instruction:   %r168 = fptoui <16 x double> undef to <16 x i64>
   %r168 = fptoui <16 x double> undef to <16 x i64>
-  ; CHECK:  Found an estimated cost of 263 for instruction:   %r169 = fptosi <16 x double> undef to <16 x i64>
   %r169 = fptosi <16 x double> undef to <16 x i64>
 
-  ; CHECK:  Found an estimated cost of 12 for instruction:   %r170 = uitofp <2 x i1> undef to <2 x float>
   %r170 = uitofp <2 x i1> undef to <2 x float>
-  ; CHECK:  Found an estimated cost of 12 for instruction:   %r171 = sitofp <2 x i1> undef to <2 x float>
   %r171 = sitofp <2 x i1> undef to <2 x float>
-  ; CHECK:  Found an estimated cost of 3 for instruction:   %r172 = uitofp <2 x i8> undef to <2 x float>
   %r172 = uitofp <2 x i8> undef to <2 x float>
-  ; CHECK:  Found an estimated cost of 3 for instruction:   %r173 = sitofp <2 x i8> undef to <2 x float>
   %r173 = sitofp <2 x i8> undef to <2 x float>
-  ; CHECK:  Found an estimated cost of 2 for instruction:   %r174 = uitofp <2 x i16> undef to <2 x float>
   %r174 = uitofp <2 x i16> undef to <2 x float>
-  ; CHECK:  Found an estimated cost of 2 for instruction:   %r175 = sitofp <2 x i16> undef to <2 x float>
   %r175 = sitofp <2 x i16> undef to <2 x float>
-  ; CHECK:  Found an estimated cost of 1 for instruction:   %r176 = uitofp <2 x i32> undef to <2 x float>
   %r176 = uitofp <2 x i32> undef to <2 x float>
-  ; CHECK:  Found an estimated cost of 1 for instruction:   %r177 = sitofp <2 x i32> undef to <2 x float>
   %r177 = sitofp <2 x i32> undef to <2 x float>
-  ; CHECK:  Found an estimated cost of 28 for instruction:   %r178 = uitofp <2 x i64> undef to <2 x float>
   %r178 = uitofp <2 x i64> undef to <2 x float>
-  ; CHECK:  Found an estimated cost of 28 for instruction:   %r179 = sitofp <2 x i64> undef to <2 x float>
   %r179 = sitofp <2 x i64> undef to <2 x float>
 
-  ; CHECK:  Found an estimated cost of 8 for instruction:   %r180 = uitofp <2 x i1> undef to <2 x double>
   %r180 = uitofp <2 x i1> undef to <2 x double>
-  ; CHECK:  Found an estimated cost of 8 for instruction:   %r181 = sitofp <2 x i1> undef to <2 x double>
   %r181 = sitofp <2 x i1> undef to <2 x double>
-  ; CHECK:  Found an estimated cost of 4 for instruction:   %r182 = uitofp <2 x i8> undef to <2 x double>
   %r182 = uitofp <2 x i8> undef to <2 x double>
-  ; CHECK:  Found an estimated cost of 4 for instruction:   %r183 = sitofp <2 x i8> undef to <2 x double>
   %r183 = sitofp <2 x i8> undef to <2 x double>
-  ; CHECK:  Found an estimated cost of 3 for instruction:   %r184 = uitofp <2 x i16> undef to <2 x double>
   %r184 = uitofp <2 x i16> undef to <2 x double>
-  ; CHECK:  Found an estimated cost of 3 for instruction:   %r185 = sitofp <2 x i16> undef to <2 x double>
   %r185 = sitofp <2 x i16> undef to <2 x double>
-  ; CHECK:  Found an estimated cost of 2 for instruction:   %r186 = uitofp <2 x i32> undef to <2 x double>
   %r186 = uitofp <2 x i32> undef to <2 x double>
-  ; CHECK:  Found an estimated cost of 2 for instruction:   %r187 = sitofp <2 x i32> undef to <2 x double>
   %r187 = sitofp <2 x i32> undef to <2 x double>
-  ; CHECK:  Found an estimated cost of 24 for instruction:   %r188 = uitofp <2 x i64> undef to <2 x double>
   %r188 = uitofp <2 x i64> undef to <2 x double>
-  ; CHECK:  Found an estimated cost of 24 for instruction:   %r189 = sitofp <2 x i64> undef to <2 x double>
   %r189 = sitofp <2 x i64> undef to <2 x double>
 
-  ; CHECK:  Found an estimated cost of 3 for instruction:   %r190 = uitofp <4 x i1> undef to <4 x float>
   %r190 = uitofp <4 x i1> undef to <4 x float>
-  ; CHECK:  Found an estimated cost of 3 for instruction:   %r191 = sitofp <4 x i1> undef to <4 x float>
   %r191 = sitofp <4 x i1> undef to <4 x float>
-  ; CHECK:  Found an estimated cost of 3 for instruction:   %r192 = uitofp <4 x i8> undef to <4 x float>
   %r192 = uitofp <4 x i8> undef to <4 x float>
-  ; CHECK:  Found an estimated cost of 3 for instruction:   %r193 = sitofp <4 x i8> undef to <4 x float>
   %r193 = sitofp <4 x i8> undef to <4 x float>
-  ; CHECK:  Found an estimated cost of 2 for instruction:   %r194 = uitofp <4 x i16> undef to <4 x float>
   %r194 = uitofp <4 x i16> undef to <4 x float>
-  ; CHECK:  Found an estimated cost of 2 for instruction:   %r195 = sitofp <4 x i16> undef to <4 x float>
   %r195 = sitofp <4 x i16> undef to <4 x float>
-  ; CHECK:  Found an estimated cost of 1 for instruction:   %r196 = uitofp <4 x i32> undef to <4 x float>
   %r196 = uitofp <4 x i32> undef to <4 x float>
-  ; CHECK:  Found an estimated cost of 1 for instruction:   %r197 = sitofp <4 x i32> undef to <4 x float>
   %r197 = sitofp <4 x i32> undef to <4 x float>
-  ; CHECK:  Found an estimated cost of 57 for instruction:   %r198 = uitofp <4 x i64> undef to <4 x float>
   %r198 = uitofp <4 x i64> undef to <4 x float>
-  ; CHECK:  Found an estimated cost of 57 for instruction:   %r199 = sitofp <4 x i64> undef to <4 x float>
   %r199 = sitofp <4 x i64> undef to <4 x float>
 
-  ; CHECK:  Found an estimated cost of 17 for instruction:   %r200 = uitofp <4 x i1> undef to <4 x double>
   %r200 = uitofp <4 x i1> undef to <4 x double>
-  ; CHECK:  Found an estimated cost of 17 for instruction:   %r201 = sitofp <4 x i1> undef to <4 x double>
   %r201 = sitofp <4 x i1> undef to <4 x double>
-  ; CHECK:  Found an estimated cost of 9 for instruction:   %r202 = uitofp <4 x i8> undef to <4 x double>
   %r202 = uitofp <4 x i8> undef to <4 x double>
-  ; CHECK:  Found an estimated cost of 9 for instruction:   %r203 = sitofp <4 x i8> undef to <4 x double>
   %r203 = sitofp <4 x i8> undef to <4 x double>
-  ; CHECK:  Found an estimated cost of 7 for instruction:   %r204 = uitofp <4 x i16> undef to <4 x double>
   %r204 = uitofp <4 x i16> undef to <4 x double>
-  ; CHECK:  Found an estimated cost of 7 for instruction:   %r205 = sitofp <4 x i16> undef to <4 x double>
   %r205 = sitofp <4 x i16> undef to <4 x double>
-  ; CHECK:  Found an estimated cost of 5 for instruction:   %r206 = uitofp <4 x i32> undef to <4 x double>
   %r206 = uitofp <4 x i32> undef to <4 x double>
-  ; CHECK:  Found an estimated cost of 5 for instruction:   %r207 = sitofp <4 x i32> undef to <4 x double>
   %r207 = sitofp <4 x i32> undef to <4 x double>
-  ; CHECK:  Found an estimated cost of 49 for instruction:   %r208 = uitofp <4 x i64> undef to <4 x double>
   %r208 = uitofp <4 x i64> undef to <4 x double>
-  ; CHECK:  Found an estimated cost of 49 for instruction:   %r209 = sitofp <4 x i64> undef to <4 x double>
   %r209 = sitofp <4 x i64> undef to <4 x double>
 
-  ; CHECK:  Found an estimated cost of 7 for instruction:   %r210 = uitofp <8 x i1> undef to <8 x float>
   %r210 = uitofp <8 x i1> undef to <8 x float>
-  ; CHECK:  Found an estimated cost of 7 for instruction:   %r211 = sitofp <8 x i1> undef to <8 x float>
   %r211 = sitofp <8 x i1> undef to <8 x float>
-  ; CHECK:  Found an estimated cost of 7 for instruction:   %r212 = uitofp <8 x i8> undef to <8 x float>
   %r212 = uitofp <8 x i8> undef to <8 x float>
-  ; CHECK:  Found an estimated cost of 7 for instruction:   %r213 = sitofp <8 x i8> undef to <8 x float>
   %r213 = sitofp <8 x i8> undef to <8 x float>
-  ; CHECK:  Found an estimated cost of 4 for instruction:   %r214 = uitofp <8 x i16> undef to <8 x float>
   %r214 = uitofp <8 x i16> undef to <8 x float>
-  ; CHECK:  Found an estimated cost of 4 for instruction:   %r215 = sitofp <8 x i16> undef to <8 x float>
   %r215 = sitofp <8 x i16> undef to <8 x float>
-  ; CHECK:  Found an estimated cost of 2 for instruction:   %r216 = uitofp <8 x i32> undef to <8 x float>
   %r216 = uitofp <8 x i32> undef to <8 x float>
-  ; CHECK:  Found an estimated cost of 2 for instruction:   %r217 = sitofp <8 x i32> undef to <8 x float>
   %r217 = sitofp <8 x i32> undef to <8 x float>
-  ; CHECK:  Found an estimated cost of 115 for instruction:   %r218 = uitofp <8 x i64> undef to <8 x float>
   %r218 = uitofp <8 x i64> undef to <8 x float>
-  ; CHECK:  Found an estimated cost of 115 for instruction:   %r219 = sitofp <8 x i64> undef to <8 x float>
   %r219 = sitofp <8 x i64> undef to <8 x float>
 
-  ; CHECK:  Found an estimated cost of 35 for instruction:   %r220 = uitofp <8 x i1> undef to <8 x double>
   %r220 = uitofp <8 x i1> undef to <8 x double>
-  ; CHECK:  Found an estimated cost of 35 for instruction:   %r221 = sitofp <8 x i1> undef to <8 x double>
   %r221 = sitofp <8 x i1> undef to <8 x double>
-  ; CHECK:  Found an estimated cost of 19 for instruction:   %r222 = uitofp <8 x i8> undef to <8 x double>
   %r222 = uitofp <8 x i8> undef to <8 x double>
-  ; CHECK:  Found an estimated cost of 19 for instruction:   %r223 = sitofp <8 x i8> undef to <8 x double>
   %r223 = sitofp <8 x i8> undef to <8 x double>
-  ; CHECK:  Found an estimated cost of 15 for instruction:   %r224 = uitofp <8 x i16> undef to <8 x double>
   %r224 = uitofp <8 x i16> undef to <8 x double>
-  ; CHECK:  Found an estimated cost of 15 for instruction:   %r225 = sitofp <8 x i16> undef to <8 x double>
   %r225 = sitofp <8 x i16> undef to <8 x double>
-  ; CHECK:  Found an estimated cost of 15 for instruction:   %r226 = uitofp <8 x i16> undef to <8 x double>
   %r226 = uitofp <8 x i16> undef to <8 x double>
-  ; CHECK:  Found an estimated cost of 15 for instruction:   %r227 = sitofp <8 x i16> undef to <8 x double>
   %r227 = sitofp <8 x i16> undef to <8 x double>
-  ; CHECK:  Found an estimated cost of 99 for instruction:   %r228 = uitofp <8 x i64> undef to <8 x double>
   %r228 = uitofp <8 x i64> undef to <8 x double>
-  ; CHECK:  Found an estimated cost of 99 for instruction:   %r229 = sitofp <8 x i64> undef to <8 x double>
   %r229 = sitofp <8 x i64> undef to <8 x double>
 
-  ; CHECK:  Found an estimated cost of 15 for instruction:   %r230 = uitofp <16 x i1> undef to <16 x float>
   %r230 = uitofp <16 x i1> undef to <16 x float>
-  ; CHECK:  Found an estimated cost of 15 for instruction:   %r231 = sitofp <16 x i1> undef to <16 x float>
   %r231 = sitofp <16 x i1> undef to <16 x float>
-  ; CHECK:  Found an estimated cost of 15 for instruction:   %r232 = uitofp <16 x i8> undef to <16 x float>
   %r232 = uitofp <16 x i8> undef to <16 x float>
-  ; CHECK:  Found an estimated cost of 15 for instruction:   %r233 = sitofp <16 x i8> undef to <16 x float>
   %r233 = sitofp <16 x i8> undef to <16 x float>
-  ; CHECK:  Found an estimated cost of 8 for instruction:   %r234 = uitofp <16 x i16> undef to <16 x float>
   %r234 = uitofp <16 x i16> undef to <16 x float>
-  ; CHECK:  Found an estimated cost of 8 for instruction:   %r235 = sitofp <16 x i16> undef to <16 x float>
   %r235 = sitofp <16 x i16> undef to <16 x float>
-  ; CHECK:  Found an estimated cost of 4 for instruction:   %r236 = uitofp <16 x i32> undef to <16 x float>
   %r236 = uitofp <16 x i32> undef to <16 x float>
-  ; CHECK:  Found an estimated cost of 4 for instruction:   %r237 = sitofp <16 x i32> undef to <16 x float>
   %r237 = sitofp <16 x i32> undef to <16 x float>
-  ; CHECK:  Found an estimated cost of 231 for instruction:   %r238 = uitofp <16 x i64> undef to <16 x float>
   %r238 = uitofp <16 x i64> undef to <16 x float>
-  ; CHECK:  Found an estimated cost of 231 for instruction:   %r239 = sitofp <16 x i64> undef to <16 x float>
   %r239 = sitofp <16 x i64> undef to <16 x float>
 
-  ; CHECK:  Found an estimated cost of 71 for instruction:   %r240 = uitofp <16 x i1> undef to <16 x double>
   %r240 = uitofp <16 x i1> undef to <16 x double>
-  ; CHECK:  Found an estimated cost of 71 for instruction:   %r241 = sitofp <16 x i1> undef to <16 x double>
   %r241 = sitofp <16 x i1> undef to <16 x double>
-  ; CHECK:  Found an estimated cost of 39 for instruction:   %r242 = uitofp <16 x i8> undef to <16 x double>
   %r242 = uitofp <16 x i8> undef to <16 x double>
-  ; CHECK:  Found an estimated cost of 39 for instruction:   %r243 = sitofp <16 x i8> undef to <16 x double>
   %r243 = sitofp <16 x i8> undef to <16 x double>
-  ; CHECK:  Found an estimated cost of 31 for instruction:   %r244 = uitofp <16 x i16> undef to <16 x double>
   %r244 = uitofp <16 x i16> undef to <16 x double>
-  ; CHECK:  Found an estimated cost of 31 for instruction:   %r245 = sitofp <16 x i16> undef to <16 x double>
   %r245 = sitofp <16 x i16> undef to <16 x double>
-  ; CHECK:  Found an estimated cost of 31 for instruction:   %r246 = uitofp <16 x i16> undef to <16 x double>
   %r246 = uitofp <16 x i16> undef to <16 x double>
-  ; CHECK:  Found an estimated cost of 31 for instruction:   %r247 = sitofp <16 x i16> undef to <16 x double>
   %r247 = sitofp <16 x i16> undef to <16 x double>
-  ; CHECK:   Found an estimated cost of 199 for instruction:   %r248 = uitofp <16 x i64> undef to <16 x double>
   %r248 = uitofp <16 x i64> undef to <16 x double>
-  ; CHECK:   Found an estimated cost of 199 for instruction:   %r249 = sitofp <16 x i64> undef to <16 x double>
   %r249 = sitofp <16 x i64> undef to <16 x double>
 
-  ; CHECK:   Found an estimated cost of 0 for instruction:   ret i32 undef
+  ret i32 undef
+}
+
+
+define i32 @load_extends() {
+; CHECK-NEON-LABEL: 'load_extends'
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi8 = load i8, i8* undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi16 = load i16, i16* undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi32 = load i32, i32* undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i8 = load <2 x i8>, <2 x i8>* undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i8 = load <4 x i8>, <4 x i8>* undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i8 = load <8 x i8>, <8 x i8>* undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i16 = load <2 x i16>, <2 x i16>* undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i16 = load <4 x i16>, <4 x i16>* undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i32 = load <2 x i32>, <2 x i32>* undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r0 = sext i8 %loadi8 to i16
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r1 = zext i8 %loadi8 to i16
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r2 = sext i8 %loadi8 to i32
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r3 = zext i8 %loadi8 to i32
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r4 = sext i8 %loadi8 to i64
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r5 = zext i8 %loadi8 to i64
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r6 = sext i16 %loadi16 to i32
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r7 = zext i16 %loadi16 to i32
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r8 = sext i16 %loadi16 to i64
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r9 = zext i16 %loadi16 to i64
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r10 = sext i32 %loadi32 to i64
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r11 = zext i32 %loadi32 to i64
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v0 = sext <8 x i8> %loadv8i8 to <8 x i16>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v1 = zext <8 x i8> %loadv8i8 to <8 x i16>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2 = sext <4 x i8> %loadv4i8 to <4 x i32>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v3 = zext <4 x i8> %loadv4i8 to <4 x i32>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4 = sext <2 x i8> %loadv2i8 to <2 x i64>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v5 = zext <2 x i8> %loadv2i8 to <2 x i64>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v6 = sext <4 x i16> %loadv4i16 to <4 x i32>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v7 = zext <4 x i16> %loadv4i16 to <4 x i32>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8 = sext <2 x i16> %loadv2i16 to <2 x i64>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v9 = zext <2 x i16> %loadv2i16 to <2 x i64>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v10 = sext <2 x i32> %loadv2i32 to <2 x i64>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v11 = zext <2 x i32> %loadv2i32 to <2 x i64>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; CHECK-MVE-LABEL: 'load_extends'
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi8 = load i8, i8* undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi16 = load i16, i16* undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi32 = load i32, i32* undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv2i8 = load <2 x i8>, <2 x i8>* undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv4i8 = load <4 x i8>, <4 x i8>* undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv8i8 = load <8 x i8>, <8 x i8>* undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv2i16 = load <2 x i16>, <2 x i16>* undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv4i16 = load <4 x i16>, <4 x i16>* undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv2i32 = load <2 x i32>, <2 x i32>* undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r0 = sext i8 %loadi8 to i16
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r1 = zext i8 %loadi8 to i16
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r2 = sext i8 %loadi8 to i32
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r3 = zext i8 %loadi8 to i32
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r4 = sext i8 %loadi8 to i64
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r5 = zext i8 %loadi8 to i64
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r6 = sext i16 %loadi16 to i32
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r7 = zext i16 %loadi16 to i32
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r8 = sext i16 %loadi16 to i64
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r9 = zext i16 %loadi16 to i64
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r10 = sext i32 %loadi32 to i64
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r11 = zext i32 %loadi32 to i64
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v0 = sext <8 x i8> %loadv8i8 to <8 x i16>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v1 = zext <8 x i8> %loadv8i8 to <8 x i16>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2 = sext <4 x i8> %loadv4i8 to <4 x i32>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v3 = zext <4 x i8> %loadv4i8 to <4 x i32>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v4 = sext <2 x i8> %loadv2i8 to <2 x i64>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v5 = zext <2 x i8> %loadv2i8 to <2 x i64>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v6 = sext <4 x i16> %loadv4i16 to <4 x i32>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v7 = zext <4 x i16> %loadv4i16 to <4 x i32>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v8 = sext <2 x i16> %loadv2i16 to <2 x i64>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v9 = zext <2 x i16> %loadv2i16 to <2 x i64>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v10 = sext <2 x i32> %loadv2i32 to <2 x i64>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v11 = zext <2 x i32> %loadv2i32 to <2 x i64>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+
+  %loadi8 = load i8, i8* undef
+  %loadi16 = load i16, i16* undef
+  %loadi32 = load i32, i32* undef
+  %loadv2i8 = load <2 x i8>, <2 x i8>* undef
+  %loadv4i8 = load <4 x i8>, <4 x i8>* undef
+  %loadv8i8 = load <8 x i8>, <8 x i8>* undef
+  %loadv2i16 = load <2 x i16>, <2 x i16>* undef
+  %loadv4i16 = load <4 x i16>, <4 x i16>* undef
+  %loadv2i32 = load <2 x i32>, <2 x i32>* undef
+
+  %r0 = sext i8 %loadi8 to i16
+  %r1 = zext i8 %loadi8 to i16
+  %r2 = sext i8 %loadi8 to i32
+  %r3 = zext i8 %loadi8 to i32
+  %r4 = sext i8 %loadi8 to i64
+  %r5 = zext i8 %loadi8 to i64
+  %r6 = sext i16 %loadi16 to i32
+  %r7 = zext i16 %loadi16 to i32
+  %r8 = sext i16 %loadi16 to i64
+  %r9 = zext i16 %loadi16 to i64
+  %r10 = sext i32 %loadi32 to i64
+  %r11 = zext i32 %loadi32 to i64
+
+  %v0 = sext <8 x i8> %loadv8i8 to <8 x i16>
+  %v1 = zext <8 x i8> %loadv8i8 to <8 x i16>
+  %v2 = sext <4 x i8> %loadv4i8 to <4 x i32>
+  %v3 = zext <4 x i8> %loadv4i8 to <4 x i32>
+  %v4 = sext <2 x i8> %loadv2i8 to <2 x i64>
+  %v5 = zext <2 x i8> %loadv2i8 to <2 x i64>
+  %v6 = sext <4 x i16> %loadv4i16 to <4 x i32>
+  %v7 = zext <4 x i16> %loadv4i16 to <4 x i32>
+  %v8 = sext <2 x i16> %loadv2i16 to <2 x i64>
+  %v9 = zext <2 x i16> %loadv2i16 to <2 x i64>
+  %v10 = sext <2 x i32> %loadv2i32 to <2 x i64>
+  %v11 = zext <2 x i32> %loadv2i32 to <2 x i64>
+
   ret i32 undef
 }
 
diff --git a/llvm/test/Analysis/CostModel/ARM/divrem.ll b/llvm/test/Analysis/CostModel/ARM/divrem.ll
index c4ac59b59..d20fcb522 100644
--- a/llvm/test/Analysis/CostModel/ARM/divrem.ll
+++ b/llvm/test/Analysis/CostModel/ARM/divrem.ll
@@ -1,450 +1,838 @@
-; RUN: opt < %s  -cost-model -analyze -mtriple=thumbv7-apple-ios6.0.0 -mcpu=cortex-a9 | FileCheck %s
-
-define <2 x i8> @sdiv_v2_i8(<2 x i8>  %a, <2 x i8> %b) {
-  ; CHECK: sdiv_v2_i8
-  ; CHECK: cost of 40 {{.*}} sdiv
-
-  %1 = sdiv <2 x i8>  %a, %b
-  ret <2 x i8> %1
-}
-define <2 x i16> @sdiv_v2_i16(<2 x i16>  %a, <2 x i16> %b) {
-  ; CHECK: sdiv_v2_i16
-  ; CHECK: cost of 40 {{.*}} sdiv
-
-  %1 = sdiv <2 x i16>  %a, %b
-  ret <2 x i16> %1
-}
-define <2 x i32> @sdiv_v2_i32(<2 x i32>  %a, <2 x i32> %b) {
-  ; CHECK: sdiv_v2_i32
-  ; CHECK: cost of 40 {{.*}} sdiv
-
-  %1 = sdiv <2 x i32>  %a, %b
-  ret <2 x i32> %1
-}
-define <2 x i64> @sdiv_v2_i64(<2 x i64>  %a, <2 x i64> %b) {
-  ; CHECK: sdiv_v2_i64
-  ; CHECK: cost of 40 {{.*}} sdiv
-
-  %1 = sdiv <2 x i64>  %a, %b
-  ret <2 x i64> %1
-}
-define <4 x i8> @sdiv_v4_i8(<4 x i8>  %a, <4 x i8> %b) {
-  ; CHECK: sdiv_v4_i8
-  ; CHECK: cost of 10 {{.*}} sdiv
-
-  %1 = sdiv <4 x i8>  %a, %b
-  ret <4 x i8> %1
-}
-define <4 x i16> @sdiv_v4_i16(<4 x i16>  %a, <4 x i16> %b) {
-  ; CHECK: sdiv_v4_i16
-  ; CHECK: cost of 10 {{.*}} sdiv
-
-  %1 = sdiv <4 x i16>  %a, %b
-  ret <4 x i16> %1
-}
-define <4 x i32> @sdiv_v4_i32(<4 x i32>  %a, <4 x i32> %b) {
-  ; CHECK: sdiv_v4_i32
-  ; CHECK: cost of 80 {{.*}} sdiv
-
-  %1 = sdiv <4 x i32>  %a, %b
-  ret <4 x i32> %1
-}
-define <4 x i64> @sdiv_v4_i64(<4 x i64>  %a, <4 x i64> %b) {
-  ; CHECK: sdiv_v4_i64
-  ; CHECK: cost of 80 {{.*}} sdiv
-
-  %1 = sdiv <4 x i64>  %a, %b
-  ret <4 x i64> %1
-}
-define <8 x i8> @sdiv_v8_i8(<8 x i8>  %a, <8 x i8> %b) {
-  ; CHECK: sdiv_v8_i8
-  ; CHECK: cost of 10 {{.*}} sdiv
-
-  %1 = sdiv <8 x i8>  %a, %b
-  ret <8 x i8> %1
-}
-define <8 x i16> @sdiv_v8_i16(<8 x i16>  %a, <8 x i16> %b) {
-  ; CHECK: sdiv_v8_i16
-  ; CHECK: cost of 160 {{.*}} sdiv
-
-  %1 = sdiv <8 x i16>  %a, %b
-  ret <8 x i16> %1
-}
-define <8 x i32> @sdiv_v8_i32(<8 x i32>  %a, <8 x i32> %b) {
-  ; CHECK: sdiv_v8_i32
-  ; CHECK: cost of 160 {{.*}} sdiv
-
-  %1 = sdiv <8 x i32>  %a, %b
-  ret <8 x i32> %1
-}
-define <8 x i64> @sdiv_v8_i64(<8 x i64>  %a, <8 x i64> %b) {
-  ; CHECK: sdiv_v8_i64
-  ; CHECK: cost of 160 {{.*}} sdiv
-
-  %1 = sdiv <8 x i64>  %a, %b
-  ret <8 x i64> %1
-}
-define <16 x i8> @sdiv_v16_i8(<16 x i8>  %a, <16 x i8> %b) {
-  ; CHECK: sdiv_v16_i8
-  ; CHECK: cost of 320 {{.*}} sdiv
-
-  %1 = sdiv <16 x i8>  %a, %b
-  ret <16 x i8> %1
-}
-define <16 x i16> @sdiv_v16_i16(<16 x i16>  %a, <16 x i16> %b) {
-  ; CHECK: sdiv_v16_i16
-  ; CHECK: cost of 320 {{.*}} sdiv
-
-  %1 = sdiv <16 x i16>  %a, %b
-  ret <16 x i16> %1
-}
-define <16 x i32> @sdiv_v16_i32(<16 x i32>  %a, <16 x i32> %b) {
-  ; CHECK: sdiv_v16_i32
-  ; CHECK: cost of 320 {{.*}} sdiv
-
-  %1 = sdiv <16 x i32>  %a, %b
-  ret <16 x i32> %1
-}
-define <16 x i64> @sdiv_v16_i64(<16 x i64>  %a, <16 x i64> %b) {
-  ; CHECK: sdiv_v16_i64
-  ; CHECK: cost of 320 {{.*}} sdiv
-
-  %1 = sdiv <16 x i64>  %a, %b
-  ret <16 x i64> %1
-}
-define <2 x i8> @udiv_v2_i8(<2 x i8>  %a, <2 x i8> %b) {
-  ; CHECK: udiv_v2_i8
-  ; CHECK: cost of 40 {{.*}} udiv
-
-  %1 = udiv <2 x i8>  %a, %b
-  ret <2 x i8> %1
-}
-define <2 x i16> @udiv_v2_i16(<2 x i16>  %a, <2 x i16> %b) {
-  ; CHECK: udiv_v2_i16
-  ; CHECK: cost of 40 {{.*}} udiv
-
-  %1 = udiv <2 x i16>  %a, %b
-  ret <2 x i16> %1
-}
-define <2 x i32> @udiv_v2_i32(<2 x i32>  %a, <2 x i32> %b) {
-  ; CHECK: udiv_v2_i32
-  ; CHECK: cost of 40 {{.*}} udiv
-
-  %1 = udiv <2 x i32>  %a, %b
-  ret <2 x i32> %1
-}
-define <2 x i64> @udiv_v2_i64(<2 x i64>  %a, <2 x i64> %b) {
-  ; CHECK: udiv_v2_i64
-  ; CHECK: cost of 40 {{.*}} udiv
-
-  %1 = udiv <2 x i64>  %a, %b
-  ret <2 x i64> %1
-}
-define <4 x i8> @udiv_v4_i8(<4 x i8>  %a, <4 x i8> %b) {
-  ; CHECK: udiv_v4_i8
-  ; CHECK: cost of 10 {{.*}} udiv
-
-  %1 = udiv <4 x i8>  %a, %b
-  ret <4 x i8> %1
-}
-define <4 x i16> @udiv_v4_i16(<4 x i16>  %a, <4 x i16> %b) {
-  ; CHECK: udiv_v4_i16
-  ; CHECK: cost of 10 {{.*}} udiv
-
-  %1 = udiv <4 x i16>  %a, %b
-  ret <4 x i16> %1
-}
-define <4 x i32> @udiv_v4_i32(<4 x i32>  %a, <4 x i32> %b) {
-  ; CHECK: udiv_v4_i32
-  ; CHECK: cost of 80 {{.*}} udiv
-
-  %1 = udiv <4 x i32>  %a, %b
-  ret <4 x i32> %1
-}
-define <4 x i64> @udiv_v4_i64(<4 x i64>  %a, <4 x i64> %b) {
-  ; CHECK: udiv_v4_i64
-  ; CHECK: cost of 80 {{.*}} udiv
-
-  %1 = udiv <4 x i64>  %a, %b
-  ret <4 x i64> %1
-}
-define <8 x i8> @udiv_v8_i8(<8 x i8>  %a, <8 x i8> %b) {
-  ; CHECK: udiv_v8_i8
-  ; CHECK: cost of 10 {{.*}} udiv
-
-  %1 = udiv <8 x i8>  %a, %b
-  ret <8 x i8> %1
-}
-define <8 x i16> @udiv_v8_i16(<8 x i16>  %a, <8 x i16> %b) {
-  ; CHECK: udiv_v8_i16
-  ; CHECK: cost of 160 {{.*}} udiv
-
-  %1 = udiv <8 x i16>  %a, %b
-  ret <8 x i16> %1
-}
-define <8 x i32> @udiv_v8_i32(<8 x i32>  %a, <8 x i32> %b) {
-  ; CHECK: udiv_v8_i32
-  ; CHECK: cost of 160 {{.*}} udiv
-
-  %1 = udiv <8 x i32>  %a, %b
-  ret <8 x i32> %1
-}
-define <8 x i64> @udiv_v8_i64(<8 x i64>  %a, <8 x i64> %b) {
-  ; CHECK: udiv_v8_i64
-  ; CHECK: cost of 160 {{.*}} udiv
-
-  %1 = udiv <8 x i64>  %a, %b
-  ret <8 x i64> %1
-}
-define <16 x i8> @udiv_v16_i8(<16 x i8>  %a, <16 x i8> %b) {
-  ; CHECK: udiv_v16_i8
-  ; CHECK: cost of 320 {{.*}} udiv
-
-  %1 = udiv <16 x i8>  %a, %b
-  ret <16 x i8> %1
-}
-define <16 x i16> @udiv_v16_i16(<16 x i16>  %a, <16 x i16> %b) {
-  ; CHECK: udiv_v16_i16
-  ; CHECK: cost of 320 {{.*}} udiv
-
-  %1 = udiv <16 x i16>  %a, %b
-  ret <16 x i16> %1
-}
-define <16 x i32> @udiv_v16_i32(<16 x i32>  %a, <16 x i32> %b) {
-  ; CHECK: udiv_v16_i32
-  ; CHECK: cost of 320 {{.*}} udiv
-
-  %1 = udiv <16 x i32>  %a, %b
-  ret <16 x i32> %1
-}
-define <16 x i64> @udiv_v16_i64(<16 x i64>  %a, <16 x i64> %b) {
-  ; CHECK: udiv_v16_i64
-  ; CHECK: cost of 320 {{.*}} udiv
-
-  %1 = udiv <16 x i64>  %a, %b
-  ret <16 x i64> %1
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt < %s  -cost-model -analyze -mtriple=thumbv7-apple-ios6.0.0 -mcpu=cortex-a9 | FileCheck %s --check-prefix=CHECK-NEON
+; RUN: opt -cost-model -analyze -mtriple=thumbv8.1-m.main-none-eabi -mattr=+mve.fp < %s | FileCheck %s --check-prefix=CHECK-MVE
+
+define void @i8() {
+; CHECK-NEON-LABEL: 'i8'
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %1 = sdiv i8 undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %2 = udiv i8 undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = srem i8 undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = urem i8 undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %5 = sdiv i8 undef, 2
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %6 = udiv i8 undef, 2
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = srem i8 undef, 2
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = urem i8 undef, 2
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-MVE-LABEL: 'i8'
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = sdiv i8 undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = udiv i8 undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = srem i8 undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = urem i8 undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = sdiv i8 undef, 2
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = udiv i8 undef, 2
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = srem i8 undef, 2
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = urem i8 undef, 2
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %1 = sdiv i8 undef, undef
+  %2 = udiv i8 undef, undef
+  %3 = srem i8 undef, undef
+  %4 = urem i8 undef, undef
+  %5 = sdiv i8 undef, 2
+  %6 = udiv i8 undef, 2
+  %7 = srem i8 undef, 2
+  %8 = urem i8 undef, 2
+  ret void
+}
+
+define void @i16() {
+; CHECK-NEON-LABEL: 'i16'
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %1 = sdiv i16 undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %2 = udiv i16 undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = srem i16 undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = urem i16 undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %5 = sdiv i16 undef, 2
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %6 = udiv i16 undef, 2
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = srem i16 undef, 2
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = urem i16 undef, 2
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-MVE-LABEL: 'i16'
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = sdiv i16 undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = udiv i16 undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = srem i16 undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = urem i16 undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = sdiv i16 undef, 2
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = udiv i16 undef, 2
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = srem i16 undef, 2
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = urem i16 undef, 2
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %1 = sdiv i16 undef, undef
+  %2 = udiv i16 undef, undef
+  %3 = srem i16 undef, undef
+  %4 = urem i16 undef, undef
+  %5 = sdiv i16 undef, 2
+  %6 = udiv i16 undef, 2
+  %7 = srem i16 undef, 2
+  %8 = urem i16 undef, 2
+  ret void
+}
+
+define void @i32() {
+; CHECK-NEON-LABEL: 'i32'
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %1 = sdiv i32 undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %2 = udiv i32 undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = srem i32 undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = urem i32 undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %5 = sdiv i32 undef, 2
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %6 = udiv i32 undef, 2
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = srem i32 undef, 2
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = urem i32 undef, 2
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-MVE-LABEL: 'i32'
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = sdiv i32 undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = udiv i32 undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = srem i32 undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = urem i32 undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = sdiv i32 undef, 2
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = udiv i32 undef, 2
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = srem i32 undef, 2
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = urem i32 undef, 2
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %1 = sdiv i32 undef, undef
+  %2 = udiv i32 undef, undef
+  %3 = srem i32 undef, undef
+  %4 = urem i32 undef, undef
+  %5 = sdiv i32 undef, 2
+  %6 = udiv i32 undef, 2
+  %7 = srem i32 undef, 2
+  %8 = urem i32 undef, 2
+  ret void
+}
+
+define void @i64() {
+; CHECK-NEON-LABEL: 'i64'
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %1 = sdiv i64 undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %2 = udiv i64 undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = srem i64 undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = urem i64 undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %5 = sdiv i64 undef, 2
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %6 = udiv i64 undef, 2
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = srem i64 undef, 2
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = urem i64 undef, 2
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-MVE-LABEL: 'i64'
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = sdiv i64 undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = udiv i64 undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = srem i64 undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = urem i64 undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = sdiv i64 undef, 2
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = udiv i64 undef, 2
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = srem i64 undef, 2
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = urem i64 undef, 2
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %1 = sdiv i64 undef, undef
+  %2 = udiv i64 undef, undef
+  %3 = srem i64 undef, undef
+  %4 = urem i64 undef, undef
+  %5 = sdiv i64 undef, 2
+  %6 = udiv i64 undef, 2
+  %7 = srem i64 undef, 2
+  %8 = urem i64 undef, 2
+  ret void
+}
+
+define void @f16() {
+; CHECK-NEON-LABEL: 'f16'
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %1 = fdiv half undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %2 = frem half undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %3 = fdiv half undef, 0xH4000
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %4 = frem half undef, 0xH4000
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-MVE-LABEL: 'f16'
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = fdiv half undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = frem half undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = fdiv half undef, 0xH4000
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = frem half undef, 0xH4000
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %1 = fdiv half undef, undef
+  %2 = frem half undef, undef
+  %3 = fdiv half undef, 2.
+  %4 = frem half undef, 2.
+  ret void
+}
+
+define void @f32() {
+; CHECK-NEON-LABEL: 'f32'
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %1 = fdiv float undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %2 = frem float undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %3 = fdiv float undef, 2.000000e+00
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %4 = frem float undef, 2.000000e+00
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-MVE-LABEL: 'f32'
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = fdiv float undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = frem float undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = fdiv float undef, 2.000000e+00
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = frem float undef, 2.000000e+00
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %1 = fdiv float undef, undef
+  %2 = frem float undef, undef
+  %3 = fdiv float undef, 2.
+  %4 = frem float undef, 2.
+  ret void
+}
+
+define void @f64() {
+; CHECK-NEON-LABEL: 'f64'
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %1 = fdiv double undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %2 = frem double undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %3 = fdiv double undef, 2.000000e+00
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %4 = frem double undef, 2.000000e+00
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-MVE-LABEL: 'f64'
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = fdiv double undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = frem double undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = fdiv double undef, 2.000000e+00
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = frem double undef, 2.000000e+00
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %1 = fdiv double undef, undef
+  %2 = frem double undef, undef
+  %3 = fdiv double undef, 2.
+  %4 = frem double undef, 2.
+  ret void
+}
+
+define void @vi8() {
+; CHECK-NEON-LABEL: 'vi8'
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t1 = sdiv <2 x i8> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t2 = udiv <2 x i8> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t3 = srem <2 x i8> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t4 = urem <2 x i8> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %f1 = sdiv <4 x i8> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %f2 = udiv <4 x i8> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f3 = srem <4 x i8> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f4 = urem <4 x i8> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %e1 = sdiv <8 x i8> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %e2 = udiv <8 x i8> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e3 = srem <8 x i8> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e4 = urem <8 x i8> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s1 = sdiv <16 x i8> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s2 = udiv <16 x i8> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s3 = srem <16 x i8> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s4 = urem <16 x i8> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-MVE-LABEL: 'vi8'
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %t1 = sdiv <2 x i8> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %t2 = udiv <2 x i8> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %t3 = srem <2 x i8> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %t4 = urem <2 x i8> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f1 = sdiv <4 x i8> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f2 = udiv <4 x i8> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f3 = srem <4 x i8> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f4 = urem <4 x i8> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e1 = sdiv <8 x i8> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e2 = udiv <8 x i8> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e3 = srem <8 x i8> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e4 = urem <8 x i8> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %s1 = sdiv <16 x i8> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %s2 = udiv <16 x i8> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %s3 = srem <16 x i8> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %s4 = urem <16 x i8> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %t1 = sdiv <2 x i8> undef, undef
+  %t2 = udiv <2 x i8> undef, undef
+  %t3 = srem <2 x i8> undef, undef
+  %t4 = urem <2 x i8> undef, undef
+  %f1 = sdiv <4 x i8> undef, undef
+  %f2 = udiv <4 x i8> undef, undef
+  %f3 = srem <4 x i8> undef, undef
+  %f4 = urem <4 x i8> undef, undef
+  %e1 = sdiv <8 x i8> undef, undef
+  %e2 = udiv <8 x i8> undef, undef
+  %e3 = srem <8 x i8> undef, undef
+  %e4 = urem <8 x i8> undef, undef
+  %s1 = sdiv <16 x i8> undef, undef
+  %s2 = udiv <16 x i8> undef, undef
+  %s3 = srem <16 x i8> undef, undef
+  %s4 = urem <16 x i8> undef, undef
+  ret void
+}
+
+define void @vi16() {
+; CHECK-NEON-LABEL: 'vi16'
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t1 = sdiv <2 x i16> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t2 = udiv <2 x i16> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t3 = srem <2 x i16> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t4 = urem <2 x i16> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %f1 = sdiv <4 x i16> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %f2 = udiv <4 x i16> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f3 = srem <4 x i16> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f4 = urem <4 x i16> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e1 = sdiv <8 x i16> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e2 = udiv <8 x i16> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e3 = srem <8 x i16> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e4 = urem <8 x i16> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s1 = sdiv <16 x i16> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s2 = udiv <16 x i16> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s3 = srem <16 x i16> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s4 = urem <16 x i16> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-MVE-LABEL: 'vi16'
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %t1 = sdiv <2 x i16> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %t2 = udiv <2 x i16> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %t3 = srem <2 x i16> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %t4 = urem <2 x i16> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f1 = sdiv <4 x i16> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f2 = udiv <4 x i16> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f3 = srem <4 x i16> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f4 = urem <4 x i16> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e1 = sdiv <8 x i16> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e2 = udiv <8 x i16> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e3 = srem <8 x i16> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e4 = urem <8 x i16> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %s1 = sdiv <16 x i16> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %s2 = udiv <16 x i16> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %s3 = srem <16 x i16> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %s4 = urem <16 x i16> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %t1 = sdiv <2 x i16> undef, undef
+  %t2 = udiv <2 x i16> undef, undef
+  %t3 = srem <2 x i16> undef, undef
+  %t4 = urem <2 x i16> undef, undef
+  %f1 = sdiv <4 x i16> undef, undef
+  %f2 = udiv <4 x i16> undef, undef
+  %f3 = srem <4 x i16> undef, undef
+  %f4 = urem <4 x i16> undef, undef
+  %e1 = sdiv <8 x i16> undef, undef
+  %e2 = udiv <8 x i16> undef, undef
+  %e3 = srem <8 x i16> undef, undef
+  %e4 = urem <8 x i16> undef, undef
+  %s1 = sdiv <16 x i16> undef, undef
+  %s2 = udiv <16 x i16> undef, undef
+  %s3 = srem <16 x i16> undef, undef
+  %s4 = urem <16 x i16> undef, undef
+  ret void
+}
+
+define void @vi32() {
+; CHECK-NEON-LABEL: 'vi32'
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t1 = sdiv <2 x i32> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t2 = udiv <2 x i32> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t3 = srem <2 x i32> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t4 = urem <2 x i32> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f1 = sdiv <4 x i32> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f2 = udiv <4 x i32> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f3 = srem <4 x i32> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f4 = urem <4 x i32> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e1 = sdiv <8 x i32> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e2 = udiv <8 x i32> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e3 = srem <8 x i32> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e4 = urem <8 x i32> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s1 = sdiv <16 x i32> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s2 = udiv <16 x i32> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s3 = srem <16 x i32> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s4 = urem <16 x i32> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-MVE-LABEL: 'vi32'
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %t1 = sdiv <2 x i32> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %t2 = udiv <2 x i32> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %t3 = srem <2 x i32> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %t4 = urem <2 x i32> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f1 = sdiv <4 x i32> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f2 = udiv <4 x i32> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f3 = srem <4 x i32> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f4 = urem <4 x i32> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e1 = sdiv <8 x i32> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e2 = udiv <8 x i32> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e3 = srem <8 x i32> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e4 = urem <8 x i32> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %s1 = sdiv <16 x i32> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %s2 = udiv <16 x i32> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %s3 = srem <16 x i32> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %s4 = urem <16 x i32> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %t1 = sdiv <2 x i32> undef, undef
+  %t2 = udiv <2 x i32> undef, undef
+  %t3 = srem <2 x i32> undef, undef
+  %t4 = urem <2 x i32> undef, undef
+  %f1 = sdiv <4 x i32> undef, undef
+  %f2 = udiv <4 x i32> undef, undef
+  %f3 = srem <4 x i32> undef, undef
+  %f4 = urem <4 x i32> undef, undef
+  %e1 = sdiv <8 x i32> undef, undef
+  %e2 = udiv <8 x i32> undef, undef
+  %e3 = srem <8 x i32> undef, undef
+  %e4 = urem <8 x i32> undef, undef
+  %s1 = sdiv <16 x i32> undef, undef
+  %s2 = udiv <16 x i32> undef, undef
+  %s3 = srem <16 x i32> undef, undef
+  %s4 = urem <16 x i32> undef, undef
+  ret void
+}
+
+define void @vi64() {
+; CHECK-NEON-LABEL: 'vi64'
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t1 = sdiv <2 x i64> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t2 = udiv <2 x i64> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t3 = srem <2 x i64> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t4 = urem <2 x i64> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f1 = sdiv <4 x i64> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f2 = udiv <4 x i64> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f3 = srem <4 x i64> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f4 = urem <4 x i64> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e1 = sdiv <8 x i64> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e2 = udiv <8 x i64> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e3 = srem <8 x i64> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e4 = urem <8 x i64> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s1 = sdiv <16 x i64> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s2 = udiv <16 x i64> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s3 = srem <16 x i64> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s4 = urem <16 x i64> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-MVE-LABEL: 'vi64'
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %t1 = sdiv <2 x i64> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %t2 = udiv <2 x i64> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %t3 = srem <2 x i64> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %t4 = urem <2 x i64> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f1 = sdiv <4 x i64> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f2 = udiv <4 x i64> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f3 = srem <4 x i64> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f4 = urem <4 x i64> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e1 = sdiv <8 x i64> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e2 = udiv <8 x i64> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e3 = srem <8 x i64> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e4 = urem <8 x i64> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %s1 = sdiv <16 x i64> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %s2 = udiv <16 x i64> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %s3 = srem <16 x i64> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %s4 = urem <16 x i64> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %t1 = sdiv <2 x i64> undef, undef
+  %t2 = udiv <2 x i64> undef, undef
+  %t3 = srem <2 x i64> undef, undef
+  %t4 = urem <2 x i64> undef, undef
+  %f1 = sdiv <4 x i64> undef, undef
+  %f2 = udiv <4 x i64> undef, undef
+  %f3 = srem <4 x i64> undef, undef
+  %f4 = urem <4 x i64> undef, undef
+  %e1 = sdiv <8 x i64> undef, undef
+  %e2 = udiv <8 x i64> undef, undef
+  %e3 = srem <8 x i64> undef, undef
+  %e4 = urem <8 x i64> undef, undef
+  %s1 = sdiv <16 x i64> undef, undef
+  %s2 = udiv <16 x i64> undef, undef
+  %s3 = srem <16 x i64> undef, undef
+  %s4 = urem <16 x i64> undef, undef
+  ret void
+}
+
+define void @vf16() {
+; CHECK-NEON-LABEL: 'vf16'
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %1 = fdiv <2 x half> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %2 = fdiv <2 x half> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %3 = fdiv <4 x half> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %4 = fdiv <4 x half> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %5 = fdiv <8 x half> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %6 = fdiv <8 x half> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-MVE-LABEL: 'vf16'
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %1 = fdiv <2 x half> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %2 = fdiv <2 x half> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %3 = fdiv <4 x half> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %4 = fdiv <4 x half> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %5 = fdiv <8 x half> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %6 = fdiv <8 x half> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %1 = fdiv <2 x half> undef, undef
+  %2 = fdiv <2 x half> undef, undef
+  %3 = fdiv <4 x half> undef, undef
+  %4 = fdiv <4 x half> undef, undef
+  %5 = fdiv <8 x half> undef, undef
+  %6 = fdiv <8 x half> undef, undef
+  ret void
+}
+
+define void @vf32() {
+; CHECK-NEON-LABEL: 'vf32'
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %1 = fdiv <2 x float> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %2 = fdiv <2 x float> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %3 = fdiv <4 x float> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %4 = fdiv <4 x float> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %5 = fdiv <8 x float> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %6 = fdiv <8 x float> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-MVE-LABEL: 'vf32'
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %1 = fdiv <2 x float> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %2 = fdiv <2 x float> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %3 = fdiv <4 x float> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %4 = fdiv <4 x float> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %5 = fdiv <8 x float> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %6 = fdiv <8 x float> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %1 = fdiv <2 x float> undef, undef
+  %2 = fdiv <2 x float> undef, undef
+  %3 = fdiv <4 x float> undef, undef
+  %4 = fdiv <4 x float> undef, undef
+  %5 = fdiv <8 x float> undef, undef
+  %6 = fdiv <8 x float> undef, undef
+  ret void
+}
+
+define void @vf64() {
+; CHECK-NEON-LABEL: 'vf64'
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %1 = fdiv <2 x double> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %2 = fdiv <2 x double> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %3 = fdiv <4 x double> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %4 = fdiv <4 x double> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %5 = fdiv <8 x double> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %6 = fdiv <8 x double> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-MVE-LABEL: 'vf64'
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %1 = fdiv <2 x double> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %2 = fdiv <2 x double> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %3 = fdiv <4 x double> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %4 = fdiv <4 x double> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %5 = fdiv <8 x double> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %6 = fdiv <8 x double> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %1 = fdiv <2 x double> undef, undef
+  %2 = fdiv <2 x double> undef, undef
+  %3 = fdiv <4 x double> undef, undef
+  %4 = fdiv <4 x double> undef, undef
+  %5 = fdiv <8 x double> undef, undef
+  %6 = fdiv <8 x double> undef, undef
+  ret void
+}
+
+define void @vi8_2() {
+; CHECK-NEON-LABEL: 'vi8_2'
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t1 = sdiv <2 x i8> undef, <i8 2, i8 2>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t2 = udiv <2 x i8> undef, <i8 2, i8 2>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t3 = srem <2 x i8> undef, <i8 2, i8 2>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t4 = urem <2 x i8> undef, <i8 2, i8 2>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %f1 = sdiv <4 x i8> undef, <i8 2, i8 2, i8 2, i8 2>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %f2 = udiv <4 x i8> undef, <i8 2, i8 2, i8 2, i8 2>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f3 = srem <4 x i8> undef, <i8 2, i8 2, i8 2, i8 2>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f4 = urem <4 x i8> undef, <i8 2, i8 2, i8 2, i8 2>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %e1 = sdiv <8 x i8> undef, <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %e2 = udiv <8 x i8> undef, <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e3 = srem <8 x i8> undef, <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e4 = urem <8 x i8> undef, <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s1 = sdiv <16 x i8> undef, <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s2 = udiv <16 x i8> undef, <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s3 = srem <16 x i8> undef, <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s4 = urem <16 x i8> undef, <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-MVE-LABEL: 'vi8_2'
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %t1 = sdiv <2 x i8> undef, <i8 2, i8 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %t2 = udiv <2 x i8> undef, <i8 2, i8 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %t3 = srem <2 x i8> undef, <i8 2, i8 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %t4 = urem <2 x i8> undef, <i8 2, i8 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f1 = sdiv <4 x i8> undef, <i8 2, i8 2, i8 2, i8 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f2 = udiv <4 x i8> undef, <i8 2, i8 2, i8 2, i8 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f3 = srem <4 x i8> undef, <i8 2, i8 2, i8 2, i8 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f4 = urem <4 x i8> undef, <i8 2, i8 2, i8 2, i8 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e1 = sdiv <8 x i8> undef, <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e2 = udiv <8 x i8> undef, <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e3 = srem <8 x i8> undef, <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e4 = urem <8 x i8> undef, <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %s1 = sdiv <16 x i8> undef, <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %s2 = udiv <16 x i8> undef, <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %s3 = srem <16 x i8> undef, <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %s4 = urem <16 x i8> undef, <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %t1 = sdiv <2 x i8> undef, <i8 2, i8 2>
+  %t2 = udiv <2 x i8> undef, <i8 2, i8 2>
+  %t3 = srem <2 x i8> undef, <i8 2, i8 2>
+  %t4 = urem <2 x i8> undef, <i8 2, i8 2>
+  %f1 = sdiv <4 x i8> undef, <i8 2, i8 2, i8 2, i8 2>
+  %f2 = udiv <4 x i8> undef, <i8 2, i8 2, i8 2, i8 2>
+  %f3 = srem <4 x i8> undef, <i8 2, i8 2, i8 2, i8 2>
+  %f4 = urem <4 x i8> undef, <i8 2, i8 2, i8 2, i8 2>
+  %e1 = sdiv <8 x i8> undef, <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>
+  %e2 = udiv <8 x i8> undef, <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>
+  %e3 = srem <8 x i8> undef, <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>
+  %e4 = urem <8 x i8> undef, <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>
+  %s1 = sdiv <16 x i8> undef, <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>
+  %s2 = udiv <16 x i8> undef, <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>
+  %s3 = srem <16 x i8> undef, <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>
+  %s4 = urem <16 x i8> undef, <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>
+  ret void
+}
+
+define void @vi16_2() {
+; CHECK-NEON-LABEL: 'vi16_2'
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t1 = sdiv <2 x i16> undef, <i16 2, i16 2>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t2 = udiv <2 x i16> undef, <i16 2, i16 2>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t3 = srem <2 x i16> undef, <i16 2, i16 2>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t4 = urem <2 x i16> undef, <i16 2, i16 2>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %f1 = sdiv <4 x i16> undef, <i16 2, i16 2, i16 2, i16 2>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %f2 = udiv <4 x i16> undef, <i16 2, i16 2, i16 2, i16 2>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f3 = srem <4 x i16> undef, <i16 2, i16 2, i16 2, i16 2>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f4 = urem <4 x i16> undef, <i16 2, i16 2, i16 2, i16 2>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e1 = sdiv <8 x i16> undef, <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e2 = udiv <8 x i16> undef, <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e3 = srem <8 x i16> undef, <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e4 = urem <8 x i16> undef, <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s1 = sdiv <16 x i16> undef, <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s2 = udiv <16 x i16> undef, <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s3 = srem <16 x i16> undef, <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s4 = urem <16 x i16> undef, <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-MVE-LABEL: 'vi16_2'
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %t1 = sdiv <2 x i16> undef, <i16 2, i16 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %t2 = udiv <2 x i16> undef, <i16 2, i16 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %t3 = srem <2 x i16> undef, <i16 2, i16 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %t4 = urem <2 x i16> undef, <i16 2, i16 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f1 = sdiv <4 x i16> undef, <i16 2, i16 2, i16 2, i16 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f2 = udiv <4 x i16> undef, <i16 2, i16 2, i16 2, i16 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f3 = srem <4 x i16> undef, <i16 2, i16 2, i16 2, i16 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f4 = urem <4 x i16> undef, <i16 2, i16 2, i16 2, i16 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e1 = sdiv <8 x i16> undef, <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e2 = udiv <8 x i16> undef, <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e3 = srem <8 x i16> undef, <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e4 = urem <8 x i16> undef, <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %s1 = sdiv <16 x i16> undef, <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %s2 = udiv <16 x i16> undef, <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %s3 = srem <16 x i16> undef, <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %s4 = urem <16 x i16> undef, <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %t1 = sdiv <2 x i16> undef, <i16 2, i16 2>
+  %t2 = udiv <2 x i16> undef, <i16 2, i16 2>
+  %t3 = srem <2 x i16> undef, <i16 2, i16 2>
+  %t4 = urem <2 x i16> undef, <i16 2, i16 2>
+  %f1 = sdiv <4 x i16> undef, <i16 2, i16 2, i16 2, i16 2>
+  %f2 = udiv <4 x i16> undef, <i16 2, i16 2, i16 2, i16 2>
+  %f3 = srem <4 x i16> undef, <i16 2, i16 2, i16 2, i16 2>
+  %f4 = urem <4 x i16> undef, <i16 2, i16 2, i16 2, i16 2>
+  %e1 = sdiv <8 x i16> undef, <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
+  %e2 = udiv <8 x i16> undef, <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
+  %e3 = srem <8 x i16> undef, <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
+  %e4 = urem <8 x i16> undef, <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
+  %s1 = sdiv <16 x i16> undef, <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
+  %s2 = udiv <16 x i16> undef, <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
+  %s3 = srem <16 x i16> undef, <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
+  %s4 = urem <16 x i16> undef, <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
+  ret void
+}
+
+define void @vi32_2() {
+; CHECK-NEON-LABEL: 'vi32_2'
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t1 = sdiv <2 x i32> undef, <i32 2, i32 2>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t2 = udiv <2 x i32> undef, <i32 2, i32 2>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t3 = srem <2 x i32> undef, <i32 2, i32 2>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t4 = urem <2 x i32> undef, <i32 2, i32 2>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f1 = sdiv <4 x i32> undef, <i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f2 = udiv <4 x i32> undef, <i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f3 = srem <4 x i32> undef, <i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f4 = urem <4 x i32> undef, <i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e1 = sdiv <8 x i32> undef, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e2 = udiv <8 x i32> undef, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e3 = srem <8 x i32> undef, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e4 = urem <8 x i32> undef, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s1 = sdiv <16 x i32> undef, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s2 = udiv <16 x i32> undef, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s3 = srem <16 x i32> undef, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s4 = urem <16 x i32> undef, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-MVE-LABEL: 'vi32_2'
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %t1 = sdiv <2 x i32> undef, <i32 2, i32 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %t2 = udiv <2 x i32> undef, <i32 2, i32 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %t3 = srem <2 x i32> undef, <i32 2, i32 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %t4 = urem <2 x i32> undef, <i32 2, i32 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f1 = sdiv <4 x i32> undef, <i32 2, i32 2, i32 2, i32 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f2 = udiv <4 x i32> undef, <i32 2, i32 2, i32 2, i32 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f3 = srem <4 x i32> undef, <i32 2, i32 2, i32 2, i32 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f4 = urem <4 x i32> undef, <i32 2, i32 2, i32 2, i32 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e1 = sdiv <8 x i32> undef, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e2 = udiv <8 x i32> undef, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e3 = srem <8 x i32> undef, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e4 = urem <8 x i32> undef, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %s1 = sdiv <16 x i32> undef, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %s2 = udiv <16 x i32> undef, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %s3 = srem <16 x i32> undef, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %s4 = urem <16 x i32> undef, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %t1 = sdiv <2 x i32> undef, <i32 2, i32 2>
+  %t2 = udiv <2 x i32> undef, <i32 2, i32 2>
+  %t3 = srem <2 x i32> undef, <i32 2, i32 2>
+  %t4 = urem <2 x i32> undef, <i32 2, i32 2>
+  %f1 = sdiv <4 x i32> undef, <i32 2, i32 2, i32 2, i32 2>
+  %f2 = udiv <4 x i32> undef, <i32 2, i32 2, i32 2, i32 2>
+  %f3 = srem <4 x i32> undef, <i32 2, i32 2, i32 2, i32 2>
+  %f4 = urem <4 x i32> undef, <i32 2, i32 2, i32 2, i32 2>
+  %e1 = sdiv <8 x i32> undef, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  %e2 = udiv <8 x i32> undef, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  %e3 = srem <8 x i32> undef, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  %e4 = urem <8 x i32> undef, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  %s1 = sdiv <16 x i32> undef, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  %s2 = udiv <16 x i32> undef, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  %s3 = srem <16 x i32> undef, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  %s4 = urem <16 x i32> undef, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  ret void
+}
+
+define void @vi64_2() {
+; CHECK-NEON-LABEL: 'vi64_2'
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t1 = sdiv <2 x i64> undef, <i64 2, i64 2>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t2 = udiv <2 x i64> undef, <i64 2, i64 2>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t3 = srem <2 x i64> undef, <i64 2, i64 2>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t4 = urem <2 x i64> undef, <i64 2, i64 2>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f1 = sdiv <4 x i64> undef, <i64 2, i64 2, i64 2, i64 2>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f2 = udiv <4 x i64> undef, <i64 2, i64 2, i64 2, i64 2>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f3 = srem <4 x i64> undef, <i64 2, i64 2, i64 2, i64 2>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f4 = urem <4 x i64> undef, <i64 2, i64 2, i64 2, i64 2>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e1 = sdiv <8 x i64> undef, <i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e2 = udiv <8 x i64> undef, <i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e3 = srem <8 x i64> undef, <i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e4 = urem <8 x i64> undef, <i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s1 = sdiv <16 x i64> undef, <i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s2 = udiv <16 x i64> undef, <i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s3 = srem <16 x i64> undef, <i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s4 = urem <16 x i64> undef, <i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-MVE-LABEL: 'vi64_2'
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %t1 = sdiv <2 x i64> undef, <i64 2, i64 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %t2 = udiv <2 x i64> undef, <i64 2, i64 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %t3 = srem <2 x i64> undef, <i64 2, i64 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %t4 = urem <2 x i64> undef, <i64 2, i64 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f1 = sdiv <4 x i64> undef, <i64 2, i64 2, i64 2, i64 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f2 = udiv <4 x i64> undef, <i64 2, i64 2, i64 2, i64 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f3 = srem <4 x i64> undef, <i64 2, i64 2, i64 2, i64 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f4 = urem <4 x i64> undef, <i64 2, i64 2, i64 2, i64 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e1 = sdiv <8 x i64> undef, <i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e2 = udiv <8 x i64> undef, <i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e3 = srem <8 x i64> undef, <i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e4 = urem <8 x i64> undef, <i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %s1 = sdiv <16 x i64> undef, <i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %s2 = udiv <16 x i64> undef, <i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %s3 = srem <16 x i64> undef, <i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %s4 = urem <16 x i64> undef, <i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %t1 = sdiv <2 x i64> undef, <i64 2, i64 2>
+  %t2 = udiv <2 x i64> undef, <i64 2, i64 2>
+  %t3 = srem <2 x i64> undef, <i64 2, i64 2>
+  %t4 = urem <2 x i64> undef, <i64 2, i64 2>
+  %f1 = sdiv <4 x i64> undef, <i64 2, i64 2, i64 2, i64 2>
+  %f2 = udiv <4 x i64> undef, <i64 2, i64 2, i64 2, i64 2>
+  %f3 = srem <4 x i64> undef, <i64 2, i64 2, i64 2, i64 2>
+  %f4 = urem <4 x i64> undef, <i64 2, i64 2, i64 2, i64 2>
+  %e1 = sdiv <8 x i64> undef, <i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2>
+  %e2 = udiv <8 x i64> undef, <i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2>
+  %e3 = srem <8 x i64> undef, <i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2>
+  %e4 = urem <8 x i64> undef, <i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2>
+  %s1 = sdiv <16 x i64> undef, <i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2>
+  %s2 = udiv <16 x i64> undef, <i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2>
+  %s3 = srem <16 x i64> undef, <i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2>
+  %s4 = urem <16 x i64> undef, <i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2>
+  ret void
+}
+
+define void @vf16_2() {
+; CHECK-NEON-LABEL: 'vf16_2'
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %1 = fdiv <2 x half> undef, <half 0xH4000, half 0xH4000>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %2 = fdiv <2 x half> undef, <half 0xH4000, half 0xH4000>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %3 = fdiv <4 x half> undef, <half 0xH4000, half 0xH4000, half 0xH4000, half 0xH4000>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %4 = fdiv <4 x half> undef, <half 0xH4000, half 0xH4000, half 0xH4000, half 0xH4000>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %5 = fdiv <8 x half> undef, <half 0xH4000, half 0xH4000, half 0xH4000, half 0xH4000, half 0xH4000, half 0xH4000, half 0xH4000, half 0xH4000>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %6 = fdiv <8 x half> undef, <half 0xH4000, half 0xH4000, half 0xH4000, half 0xH4000, half 0xH4000, half 0xH4000, half 0xH4000, half 0xH4000>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-MVE-LABEL: 'vf16_2'
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %1 = fdiv <2 x half> undef, <half 0xH4000, half 0xH4000>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %2 = fdiv <2 x half> undef, <half 0xH4000, half 0xH4000>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %3 = fdiv <4 x half> undef, <half 0xH4000, half 0xH4000, half 0xH4000, half 0xH4000>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %4 = fdiv <4 x half> undef, <half 0xH4000, half 0xH4000, half 0xH4000, half 0xH4000>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %5 = fdiv <8 x half> undef, <half 0xH4000, half 0xH4000, half 0xH4000, half 0xH4000, half 0xH4000, half 0xH4000, half 0xH4000, half 0xH4000>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %6 = fdiv <8 x half> undef, <half 0xH4000, half 0xH4000, half 0xH4000, half 0xH4000, half 0xH4000, half 0xH4000, half 0xH4000, half 0xH4000>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %1 = fdiv <2 x half> undef, <half 2., half 2.>
+  %2 = fdiv <2 x half> undef, <half 2., half 2.>
+  %3 = fdiv <4 x half> undef, <half 2., half 2., half 2., half 2.>
+  %4 = fdiv <4 x half> undef, <half 2., half 2., half 2., half 2.>
+  %5 = fdiv <8 x half> undef, <half 2., half 2., half 2., half 2., half 2., half 2., half 2., half 2.>
+  %6 = fdiv <8 x half> undef, <half 2., half 2., half 2., half 2., half 2., half 2., half 2., half 2.>
+  ret void
+}
+
+define void @vf32_2() {
+; CHECK-NEON-LABEL: 'vf32_2'
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %1 = fdiv <2 x float> undef, <float 2.000000e+00, float 2.000000e+00>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %2 = fdiv <2 x float> undef, <float 2.000000e+00, float 2.000000e+00>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %3 = fdiv <4 x float> undef, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %4 = fdiv <4 x float> undef, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %5 = fdiv <8 x float> undef, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %6 = fdiv <8 x float> undef, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-MVE-LABEL: 'vf32_2'
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %1 = fdiv <2 x float> undef, <float 2.000000e+00, float 2.000000e+00>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %2 = fdiv <2 x float> undef, <float 2.000000e+00, float 2.000000e+00>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %3 = fdiv <4 x float> undef, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %4 = fdiv <4 x float> undef, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %5 = fdiv <8 x float> undef, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %6 = fdiv <8 x float> undef, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %1 = fdiv <2 x float> undef, <float 2., float 2.>
+  %2 = fdiv <2 x float> undef, <float 2., float 2.>
+  %3 = fdiv <4 x float> undef, <float 2., float 2., float 2., float 2.>
+  %4 = fdiv <4 x float> undef, <float 2., float 2., float 2., float 2.>
+  %5 = fdiv <8 x float> undef, <float 2., float 2., float 2., float 2., float 2., float 2., float 2., float 2.>
+  %6 = fdiv <8 x float> undef, <float 2., float 2., float 2., float 2., float 2., float 2., float 2., float 2.>
+  ret void
+}
+
+define void @vf64_2() {
+; CHECK-NEON-LABEL: 'vf64_2'
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %1 = fdiv <2 x double> undef, <double 2.000000e+00, double 2.000000e+00>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %2 = fdiv <2 x double> undef, <double 2.000000e+00, double 2.000000e+00>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %3 = fdiv <4 x double> undef, <double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %4 = fdiv <4 x double> undef, <double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %5 = fdiv <8 x double> undef, <double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %6 = fdiv <8 x double> undef, <double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-MVE-LABEL: 'vf64_2'
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %1 = fdiv <2 x double> undef, <double 2.000000e+00, double 2.000000e+00>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %2 = fdiv <2 x double> undef, <double 2.000000e+00, double 2.000000e+00>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %3 = fdiv <4 x double> undef, <double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %4 = fdiv <4 x double> undef, <double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %5 = fdiv <8 x double> undef, <double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %6 = fdiv <8 x double> undef, <double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %1 = fdiv <2 x double> undef, <double 2., double 2.>
+  %2 = fdiv <2 x double> undef, <double 2., double 2.>
+  %3 = fdiv <4 x double> undef, <double 2., double 2., double 2., double 2.>
+  %4 = fdiv <4 x double> undef, <double 2., double 2., double 2., double 2.>
+  %5 = fdiv <8 x double> undef, <double 2., double 2., double 2., double 2., double 2., double 2., double 2., double 2.>
+  %6 = fdiv <8 x double> undef, <double 2., double 2., double 2., double 2., double 2., double 2., double 2., double 2.>
+  ret void
 }
-define <2 x i8> @srem_v2_i8(<2 x i8>  %a, <2 x i8> %b) {
-  ; CHECK: srem_v2_i8
-  ; CHECK: cost of 40 {{.*}} srem
 
-  %1 = srem <2 x i8>  %a, %b
-  ret <2 x i8> %1
-}
-define <2 x i16> @srem_v2_i16(<2 x i16>  %a, <2 x i16> %b) {
-  ; CHECK: srem_v2_i16
-  ; CHECK: cost of 40 {{.*}} srem
-
-  %1 = srem <2 x i16>  %a, %b
-  ret <2 x i16> %1
-}
-define <2 x i32> @srem_v2_i32(<2 x i32>  %a, <2 x i32> %b) {
-  ; CHECK: srem_v2_i32
-  ; CHECK: cost of 40 {{.*}} srem
-
-  %1 = srem <2 x i32>  %a, %b
-  ret <2 x i32> %1
-}
-define <2 x i64> @srem_v2_i64(<2 x i64>  %a, <2 x i64> %b) {
-  ; CHECK: srem_v2_i64
-  ; CHECK: cost of 40 {{.*}} srem
-
-  %1 = srem <2 x i64>  %a, %b
-  ret <2 x i64> %1
-}
-define <4 x i8> @srem_v4_i8(<4 x i8>  %a, <4 x i8> %b) {
-  ; CHECK: srem_v4_i8
-  ; CHECK: cost of 80 {{.*}} srem
-
-  %1 = srem <4 x i8>  %a, %b
-  ret <4 x i8> %1
-}
-define <4 x i16> @srem_v4_i16(<4 x i16>  %a, <4 x i16> %b) {
-  ; CHECK: srem_v4_i16
-  ; CHECK: cost of 80 {{.*}} srem
-
-  %1 = srem <4 x i16>  %a, %b
-  ret <4 x i16> %1
-}
-define <4 x i32> @srem_v4_i32(<4 x i32>  %a, <4 x i32> %b) {
-  ; CHECK: srem_v4_i32
-  ; CHECK: cost of 80 {{.*}} srem
-
-  %1 = srem <4 x i32>  %a, %b
-  ret <4 x i32> %1
-}
-define <4 x i64> @srem_v4_i64(<4 x i64>  %a, <4 x i64> %b) {
-  ; CHECK: srem_v4_i64
-  ; CHECK: cost of 80 {{.*}} srem
-
-  %1 = srem <4 x i64>  %a, %b
-  ret <4 x i64> %1
-}
-define <8 x i8> @srem_v8_i8(<8 x i8>  %a, <8 x i8> %b) {
-  ; CHECK: srem_v8_i8
-  ; CHECK: cost of 160 {{.*}} srem
-
-  %1 = srem <8 x i8>  %a, %b
-  ret <8 x i8> %1
-}
-define <8 x i16> @srem_v8_i16(<8 x i16>  %a, <8 x i16> %b) {
-  ; CHECK: srem_v8_i16
-  ; CHECK: cost of 160 {{.*}} srem
-
-  %1 = srem <8 x i16>  %a, %b
-  ret <8 x i16> %1
-}
-define <8 x i32> @srem_v8_i32(<8 x i32>  %a, <8 x i32> %b) {
-  ; CHECK: srem_v8_i32
-  ; CHECK: cost of 160 {{.*}} srem
-
-  %1 = srem <8 x i32>  %a, %b
-  ret <8 x i32> %1
-}
-define <8 x i64> @srem_v8_i64(<8 x i64>  %a, <8 x i64> %b) {
-  ; CHECK: srem_v8_i64
-  ; CHECK: cost of 160 {{.*}} srem
-
-  %1 = srem <8 x i64>  %a, %b
-  ret <8 x i64> %1
-}
-define <16 x i8> @srem_v16_i8(<16 x i8>  %a, <16 x i8> %b) {
-  ; CHECK: srem_v16_i8
-  ; CHECK: cost of 320 {{.*}} srem
-
-  %1 = srem <16 x i8>  %a, %b
-  ret <16 x i8> %1
-}
-define <16 x i16> @srem_v16_i16(<16 x i16>  %a, <16 x i16> %b) {
-  ; CHECK: srem_v16_i16
-  ; CHECK: cost of 320 {{.*}} srem
-
-  %1 = srem <16 x i16>  %a, %b
-  ret <16 x i16> %1
-}
-define <16 x i32> @srem_v16_i32(<16 x i32>  %a, <16 x i32> %b) {
-  ; CHECK: srem_v16_i32
-  ; CHECK: cost of 320 {{.*}} srem
-
-  %1 = srem <16 x i32>  %a, %b
-  ret <16 x i32> %1
-}
-define <16 x i64> @srem_v16_i64(<16 x i64>  %a, <16 x i64> %b) {
-  ; CHECK: srem_v16_i64
-  ; CHECK: cost of 320 {{.*}} srem
-
-  %1 = srem <16 x i64>  %a, %b
-  ret <16 x i64> %1
-}
-define <2 x i8> @urem_v2_i8(<2 x i8>  %a, <2 x i8> %b) {
-  ; CHECK: urem_v2_i8
-  ; CHECK: cost of 40 {{.*}} urem
-
-  %1 = urem <2 x i8>  %a, %b
-  ret <2 x i8> %1
-}
-define <2 x i16> @urem_v2_i16(<2 x i16>  %a, <2 x i16> %b) {
-  ; CHECK: urem_v2_i16
-  ; CHECK: cost of 40 {{.*}} urem
-
-  %1 = urem <2 x i16>  %a, %b
-  ret <2 x i16> %1
-}
-define <2 x i32> @urem_v2_i32(<2 x i32>  %a, <2 x i32> %b) {
-  ; CHECK: urem_v2_i32
-  ; CHECK: cost of 40 {{.*}} urem
-
-  %1 = urem <2 x i32>  %a, %b
-  ret <2 x i32> %1
-}
-define <2 x i64> @urem_v2_i64(<2 x i64>  %a, <2 x i64> %b) {
-  ; CHECK: urem_v2_i64
-  ; CHECK: cost of 40 {{.*}} urem
-
-  %1 = urem <2 x i64>  %a, %b
-  ret <2 x i64> %1
-}
-define <4 x i8> @urem_v4_i8(<4 x i8>  %a, <4 x i8> %b) {
-  ; CHECK: urem_v4_i8
-  ; CHECK: cost of 80 {{.*}} urem
-
-  %1 = urem <4 x i8>  %a, %b
-  ret <4 x i8> %1
-}
-define <4 x i16> @urem_v4_i16(<4 x i16>  %a, <4 x i16> %b) {
-  ; CHECK: urem_v4_i16
-  ; CHECK: cost of 80 {{.*}} urem
-
-  %1 = urem <4 x i16>  %a, %b
-  ret <4 x i16> %1
-}
-define <4 x i32> @urem_v4_i32(<4 x i32>  %a, <4 x i32> %b) {
-  ; CHECK: urem_v4_i32
-  ; CHECK: cost of 80 {{.*}} urem
-
-  %1 = urem <4 x i32>  %a, %b
-  ret <4 x i32> %1
-}
-define <4 x i64> @urem_v4_i64(<4 x i64>  %a, <4 x i64> %b) {
-  ; CHECK: urem_v4_i64
-  ; CHECK: cost of 80 {{.*}} urem
-
-  %1 = urem <4 x i64>  %a, %b
-  ret <4 x i64> %1
-}
-define <8 x i8> @urem_v8_i8(<8 x i8>  %a, <8 x i8> %b) {
-  ; CHECK: urem_v8_i8
-  ; CHECK: cost of 160 {{.*}} urem
-
-  %1 = urem <8 x i8>  %a, %b
-  ret <8 x i8> %1
-}
-define <8 x i16> @urem_v8_i16(<8 x i16>  %a, <8 x i16> %b) {
-  ; CHECK: urem_v8_i16
-  ; CHECK: cost of 160 {{.*}} urem
-
-  %1 = urem <8 x i16>  %a, %b
-  ret <8 x i16> %1
-}
-define <8 x i32> @urem_v8_i32(<8 x i32>  %a, <8 x i32> %b) {
-  ; CHECK: urem_v8_i32
-  ; CHECK: cost of 160 {{.*}} urem
-
-  %1 = urem <8 x i32>  %a, %b
-  ret <8 x i32> %1
-}
-define <8 x i64> @urem_v8_i64(<8 x i64>  %a, <8 x i64> %b) {
-  ; CHECK: urem_v8_i64
-  ; CHECK: cost of 160 {{.*}} urem
-
-  %1 = urem <8 x i64>  %a, %b
-  ret <8 x i64> %1
-}
-define <16 x i8> @urem_v16_i8(<16 x i8>  %a, <16 x i8> %b) {
-  ; CHECK: urem_v16_i8
-  ; CHECK: cost of 320 {{.*}} urem
-
-  %1 = urem <16 x i8>  %a, %b
-  ret <16 x i8> %1
-}
-define <16 x i16> @urem_v16_i16(<16 x i16>  %a, <16 x i16> %b) {
-  ; CHECK: urem_v16_i16
-  ; CHECK: cost of 320 {{.*}} urem
-
-  %1 = urem <16 x i16>  %a, %b
-  ret <16 x i16> %1
-}
-define <16 x i32> @urem_v16_i32(<16 x i32>  %a, <16 x i32> %b) {
-  ; CHECK: urem_v16_i32
-  ; CHECK: cost of 320 {{.*}} urem
-
-  %1 = urem <16 x i32>  %a, %b
-  ret <16 x i32> %1
-}
-define <16 x i64> @urem_v16_i64(<16 x i64>  %a, <16 x i64> %b) {
-  ; CHECK: urem_v16_i64
-  ; CHECK: cost of 320 {{.*}} urem
-
-  %1 = urem <16 x i64>  %a, %b
-  ret <16 x i64> %1
-}
diff --git a/llvm/test/Analysis/CostModel/ARM/fparith.ll b/llvm/test/Analysis/CostModel/ARM/fparith.ll
new file mode 100644
index 000000000..bc2cd104a
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/ARM/fparith.ll
@@ -0,0 +1,172 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt -cost-model -analyze -mtriple=thumbv8.1-m.main-none-eabi -mattr=+mve < %s | FileCheck %s --check-prefix=CHECK-MVE
+; RUN: opt -cost-model -analyze -mtriple=thumbv8.1-m.main-none-eabi -mattr=+mve.fp < %s | FileCheck %s --check-prefix=CHECK-MVEFP
+
+define void @f32() {
+; CHECK-MVE-LABEL: 'f32'
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c = fadd float undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %d = fsub float undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e = fmul float undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-MVEFP-LABEL: 'f32'
+; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c = fadd float undef, undef
+; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %d = fsub float undef, undef
+; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e = fmul float undef, undef
+; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %c = fadd float undef, undef
+  %d = fsub float undef, undef
+  %e = fmul float undef, undef
+  ret void
+}
+
+define void @f16() {
+; CHECK-MVE-LABEL: 'f16'
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c = fadd half undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %d = fsub half undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e = fmul half undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-MVEFP-LABEL: 'f16'
+; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c = fadd half undef, undef
+; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %d = fsub half undef, undef
+; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e = fmul half undef, undef
+; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %c = fadd half undef, undef
+  %d = fsub half undef, undef
+  %e = fmul half undef, undef
+  ret void
+}
+
+define void @f64() {
+; CHECK-MVE-LABEL: 'f64'
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c = fadd double undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %d = fsub double undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e = fmul double undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-MVEFP-LABEL: 'f64'
+; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c = fadd double undef, undef
+; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %d = fsub double undef, undef
+; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e = fmul double undef, undef
+; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %c = fadd double undef, undef
+  %d = fsub double undef, undef
+  %e = fmul double undef, undef
+  ret void
+}
+
+define void @vf32() {
+; CHECK-MVE-LABEL: 'vf32'
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %c2 = fadd <2 x float> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %d2 = fsub <2 x float> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %e2 = fmul <2 x float> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %c4 = fadd <4 x float> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %d4 = fsub <4 x float> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %e4 = fmul <4 x float> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %c8 = fadd <8 x float> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %d8 = fsub <8 x float> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e8 = fmul <8 x float> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-MVEFP-LABEL: 'vf32'
+; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %c2 = fadd <2 x float> undef, undef
+; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %d2 = fsub <2 x float> undef, undef
+; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %e2 = fmul <2 x float> undef, undef
+; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %c4 = fadd <4 x float> undef, undef
+; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %d4 = fsub <4 x float> undef, undef
+; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %e4 = fmul <4 x float> undef, undef
+; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %c8 = fadd <8 x float> undef, undef
+; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %d8 = fsub <8 x float> undef, undef
+; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %e8 = fmul <8 x float> undef, undef
+; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %c2 = fadd <2 x float> undef, undef
+  %d2 = fsub <2 x float> undef, undef
+  %e2 = fmul <2 x float> undef, undef
+  %c4 = fadd <4 x float> undef, undef
+  %d4 = fsub <4 x float> undef, undef
+  %e4 = fmul <4 x float> undef, undef
+  %c8 = fadd <8 x float> undef, undef
+  %d8 = fsub <8 x float> undef, undef
+  %e8 = fmul <8 x float> undef, undef
+  ret void
+}
+
+define void @vf16() {
+; CHECK-MVE-LABEL: 'vf16'
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %c2 = fadd <2 x half> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %d2 = fsub <2 x half> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %e2 = fmul <2 x half> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %c4 = fadd <4 x half> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %d4 = fsub <4 x half> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %e4 = fmul <4 x half> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %c8 = fadd <8 x half> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %d8 = fsub <8 x half> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e8 = fmul <8 x half> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-MVEFP-LABEL: 'vf16'
+; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %c2 = fadd <2 x half> undef, undef
+; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %d2 = fsub <2 x half> undef, undef
+; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %e2 = fmul <2 x half> undef, undef
+; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %c4 = fadd <4 x half> undef, undef
+; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %d4 = fsub <4 x half> undef, undef
+; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %e4 = fmul <4 x half> undef, undef
+; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %c8 = fadd <8 x half> undef, undef
+; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %d8 = fsub <8 x half> undef, undef
+; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %e8 = fmul <8 x half> undef, undef
+; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %c2 = fadd <2 x half> undef, undef
+  %d2 = fsub <2 x half> undef, undef
+  %e2 = fmul <2 x half> undef, undef
+  %c4 = fadd <4 x half> undef, undef
+  %d4 = fsub <4 x half> undef, undef
+  %e4 = fmul <4 x half> undef, undef
+  %c8 = fadd <8 x half> undef, undef
+  %d8 = fsub <8 x half> undef, undef
+  %e8 = fmul <8 x half> undef, undef
+  ret void
+}
+
+define void @vf64() {
+; CHECK-MVE-LABEL: 'vf64'
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %c2 = fadd <2 x double> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %d2 = fsub <2 x double> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %e2 = fmul <2 x double> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %c4 = fadd <4 x double> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %d4 = fsub <4 x double> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %e4 = fmul <4 x double> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %c8 = fadd <8 x double> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %d8 = fsub <8 x double> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e8 = fmul <8 x double> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-MVEFP-LABEL: 'vf64'
+; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %c2 = fadd <2 x double> undef, undef
+; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %d2 = fsub <2 x double> undef, undef
+; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %e2 = fmul <2 x double> undef, undef
+; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %c4 = fadd <4 x double> undef, undef
+; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %d4 = fsub <4 x double> undef, undef
+; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %e4 = fmul <4 x double> undef, undef
+; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %c8 = fadd <8 x double> undef, undef
+; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %d8 = fsub <8 x double> undef, undef
+; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e8 = fmul <8 x double> undef, undef
+; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %c2 = fadd <2 x double> undef, undef
+  %d2 = fsub <2 x double> undef, undef
+  %e2 = fmul <2 x double> undef, undef
+  %c4 = fadd <4 x double> undef, undef
+  %d4 = fsub <4 x double> undef, undef
+  %e4 = fmul <4 x double> undef, undef
+  %c8 = fadd <8 x double> undef, undef
+  %d8 = fsub <8 x double> undef, undef
+  %e8 = fmul <8 x double> undef, undef
+  ret void
+}
+
diff --git a/llvm/test/Analysis/CostModel/ARM/freeshift.ll b/llvm/test/Analysis/CostModel/ARM/freeshift.ll
new file mode 100644
index 000000000..464b55225
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/ARM/freeshift.ll
@@ -0,0 +1,96 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt -cost-model -analyze -mtriple=thumbv8.1m.main-none-eabi < %s | FileCheck %s
+
+define void @shl(i32 %a, i32 %b) {
+; CHECK-LABEL: 'shl'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %as = shl i32 %a, 3
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %ac = add i32 %b, %as
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %ss = shl i32 %a, 3
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sc = sub i32 %b, %ss
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %xs = shl i32 %a, 3
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %xc = xor i32 %b, %xs
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %ns = shl i32 %a, 3
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nc = and i32 %b, %ns
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %os = shl i32 %a, 3
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %oc = or i32 %b, %os
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %is = shl i32 %a, 3
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %ic = icmp eq i32 %b, %is
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %as = shl i32 %a, 3
+  %ac = add i32 %b, %as
+  %ss = shl i32 %a, 3
+  %sc = sub i32 %b, %ss
+  %xs = shl i32 %a, 3
+  %xc = xor i32 %b, %xs
+  %ns = shl i32 %a, 3
+  %nc = and i32 %b, %ns
+  %os = shl i32 %a, 3
+  %oc = or i32 %b, %os
+  %is = shl i32 %a, 3
+  %ic = icmp eq i32 %b, %is
+  ret void
+}
+
+define void @ashr(i32 %a, i32 %b) {
+; CHECK-LABEL: 'ashr'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %as = ashr i32 %a, 3
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %ac = add i32 %b, %as
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %ss = ashr i32 %a, 3
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sc = sub i32 %b, %ss
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %xs = ashr i32 %a, 3
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %xc = xor i32 %b, %xs
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %ns = ashr i32 %a, 3
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nc = and i32 %b, %ns
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %os = ashr i32 %a, 3
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %oc = or i32 %b, %os
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %is = ashr i32 %a, 3
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %ic = icmp eq i32 %b, %is
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %as = ashr i32 %a, 3
+  %ac = add i32 %b, %as
+  %ss = ashr i32 %a, 3
+  %sc = sub i32 %b, %ss
+  %xs = ashr i32 %a, 3
+  %xc = xor i32 %b, %xs
+  %ns = ashr i32 %a, 3
+  %nc = and i32 %b, %ns
+  %os = ashr i32 %a, 3
+  %oc = or i32 %b, %os
+  %is = ashr i32 %a, 3
+  %ic = icmp eq i32 %b, %is
+  ret void
+}
+
+define void @lshr(i32 %a, i32 %b) {
+; CHECK-LABEL: 'lshr'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %as = lshr i32 %a, 3
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %ac = add i32 %b, %as
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %ss = lshr i32 %a, 3
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sc = sub i32 %b, %ss
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %xs = lshr i32 %a, 3
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %xc = xor i32 %b, %xs
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %ns = lshr i32 %a, 3
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nc = and i32 %b, %ns
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %os = lshr i32 %a, 3
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %oc = or i32 %b, %os
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %is = lshr i32 %a, 3
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %ic = icmp eq i32 %b, %is
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %as = lshr i32 %a, 3
+  %ac = add i32 %b, %as
+  %ss = lshr i32 %a, 3
+  %sc = sub i32 %b, %ss
+  %xs = lshr i32 %a, 3
+  %xc = xor i32 %b, %xs
+  %ns = lshr i32 %a, 3
+  %nc = and i32 %b, %ns
+  %os = lshr i32 %a, 3
+  %oc = or i32 %b, %os
+  %is = lshr i32 %a, 3
+  %ic = icmp eq i32 %b, %is
+  ret void
+}
+
diff --git a/llvm/test/Analysis/CostModel/ARM/load_store.ll b/llvm/test/Analysis/CostModel/ARM/load_store.ll
new file mode 100644
index 000000000..89b18d90f
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/ARM/load_store.ll
@@ -0,0 +1,278 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt -cost-model -analyze -mtriple=thumbv6m-none-eabi < %s | FileCheck %s --check-prefix=CHECK-NOVEC
+; RUN: opt -cost-model -analyze -mtriple=thumbv7m-none-eabi -mcpu=cortex-m3 < %s | FileCheck %s --check-prefix=CHECK-NOVEC
+; RUN: opt -cost-model -analyze -mtriple=thumbv7m-none-eabi -mcpu=cortex-m4 < %s | FileCheck %s --check-prefix=CHECK-FP
+; RUN: opt -cost-model -analyze -mtriple=thumbv8.1-m.main-none-eabi -mattr=+mve < %s | FileCheck %s --check-prefix=CHECK-MVE
+; RUN: opt -cost-model -analyze -mtriple=thumbv7-apple-ios6.0.0 -mcpu=swift < %s | FileCheck %s --check-prefix=CHECK-NEON
+; RUN: opt -cost-model -analyze -mtriple=arm-none-eabi -mcpu=cortex-a53 < %s | FileCheck %s --check-prefix=CHECK-NEON
+
+define void @stores() {
+; CHECK-NOVEC-LABEL: 'stores'
+; CHECK-NOVEC-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 undef, i8* undef, align 4
+; CHECK-NOVEC-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i16 undef, i16* undef, align 4
+; CHECK-NOVEC-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i32 undef, i32* undef, align 4
+; CHECK-NOVEC-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store i64 undef, i64* undef, align 4
+; CHECK-NOVEC-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store i128 undef, i128* undef, align 4
+; CHECK-NOVEC-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store float undef, float* undef, align 4
+; CHECK-NOVEC-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store double undef, double* undef, align 4
+; CHECK-NOVEC-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x i8> undef, <2 x i8>* undef, align 1
+; CHECK-NOVEC-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x i16> undef, <2 x i16>* undef, align 2
+; CHECK-NOVEC-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x i32> undef, <2 x i32>* undef, align 4
+; CHECK-NOVEC-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <2 x i64> undef, <2 x i64>* undef, align 4
+; CHECK-NOVEC-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <4 x i32> undef, <4 x i32>* undef, align 4
+; CHECK-NOVEC-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: store <8 x i16> undef, <8 x i16>* undef, align 2
+; CHECK-NOVEC-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: store <16 x i8> undef, <16 x i8>* undef, align 1
+; CHECK-NOVEC-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <4 x float> undef, <4 x float>* undef, align 4
+; CHECK-NOVEC-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: store <4 x double> undef, <4 x double>* undef, align 4
+; CHECK-NOVEC-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x float> undef, <2 x float>* undef, align 4
+; CHECK-NOVEC-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <2 x double> undef, <2 x double>* undef, align 4
+; CHECK-NOVEC-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <2 x i64> undef, <2 x i64>* undef, align 1
+; CHECK-NOVEC-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <4 x i32> undef, <4 x i32>* undef, align 1
+; CHECK-NOVEC-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: store <8 x i16> undef, <8 x i16>* undef, align 1
+; CHECK-NOVEC-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <4 x float> undef, <4 x float>* undef, align 1
+; CHECK-NOVEC-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <2 x double> undef, <2 x double>* undef, align 1
+; CHECK-NOVEC-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-FP-LABEL: 'stores'
+; CHECK-FP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 undef, i8* undef, align 4
+; CHECK-FP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i16 undef, i16* undef, align 4
+; CHECK-FP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i32 undef, i32* undef, align 4
+; CHECK-FP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store i64 undef, i64* undef, align 4
+; CHECK-FP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store i128 undef, i128* undef, align 4
+; CHECK-FP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store float undef, float* undef, align 4
+; CHECK-FP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store double undef, double* undef, align 4
+; CHECK-FP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x i8> undef, <2 x i8>* undef, align 1
+; CHECK-FP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x i16> undef, <2 x i16>* undef, align 2
+; CHECK-FP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x i32> undef, <2 x i32>* undef, align 4
+; CHECK-FP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <2 x i64> undef, <2 x i64>* undef, align 4
+; CHECK-FP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <4 x i32> undef, <4 x i32>* undef, align 4
+; CHECK-FP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: store <8 x i16> undef, <8 x i16>* undef, align 2
+; CHECK-FP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: store <16 x i8> undef, <16 x i8>* undef, align 1
+; CHECK-FP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <4 x float> undef, <4 x float>* undef, align 4
+; CHECK-FP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <4 x double> undef, <4 x double>* undef, align 4
+; CHECK-FP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x float> undef, <2 x float>* undef, align 4
+; CHECK-FP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x double> undef, <2 x double>* undef, align 4
+; CHECK-FP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <2 x i64> undef, <2 x i64>* undef, align 1
+; CHECK-FP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <4 x i32> undef, <4 x i32>* undef, align 1
+; CHECK-FP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: store <8 x i16> undef, <8 x i16>* undef, align 1
+; CHECK-FP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <4 x float> undef, <4 x float>* undef, align 1
+; CHECK-FP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x double> undef, <2 x double>* undef, align 1
+; CHECK-FP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-MVE-LABEL: 'stores'
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 undef, i8* undef, align 4
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i16 undef, i16* undef, align 4
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i32 undef, i32* undef, align 4
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store i64 undef, i64* undef, align 4
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store i128 undef, i128* undef, align 4
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store float undef, float* undef, align 4
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store double undef, double* undef, align 4
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x i8> undef, <2 x i8>* undef, align 1
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x i16> undef, <2 x i16>* undef, align 2
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x i32> undef, <2 x i32>* undef, align 4
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x i64> undef, <2 x i64>* undef, align 4
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <4 x i32> undef, <4 x i32>* undef, align 4
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <8 x i16> undef, <8 x i16>* undef, align 2
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <16 x i8> undef, <16 x i8>* undef, align 1
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <4 x float> undef, <4 x float>* undef, align 4
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <4 x double> undef, <4 x double>* undef, align 4
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x float> undef, <2 x float>* undef, align 4
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x double> undef, <2 x double>* undef, align 4
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x i64> undef, <2 x i64>* undef, align 1
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <4 x i32> undef, <4 x i32>* undef, align 1
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <8 x i16> undef, <8 x i16>* undef, align 1
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <4 x float> undef, <4 x float>* undef, align 1
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x double> undef, <2 x double>* undef, align 1
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-NEON-LABEL: 'stores'
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 undef, i8* undef, align 4
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i16 undef, i16* undef, align 4
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i32 undef, i32* undef, align 4
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store i64 undef, i64* undef, align 4
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store i128 undef, i128* undef, align 4
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store float undef, float* undef, align 4
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store double undef, double* undef, align 4
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> undef, <2 x i8>* undef, align 1
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i16> undef, <2 x i16>* undef, align 2
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i32> undef, <2 x i32>* undef, align 4
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i64> undef, <2 x i64>* undef, align 4
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> undef, <4 x i32>* undef, align 4
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i16> undef, <8 x i16>* undef, align 2
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> undef, <16 x i8>* undef, align 1
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x float> undef, <4 x float>* undef, align 4
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: store <4 x double> undef, <4 x double>* undef, align 4
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x float> undef, <2 x float>* undef, align 4
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <2 x double> undef, <2 x double>* undef, align 4
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i64> undef, <2 x i64>* undef, align 1
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> undef, <4 x i32>* undef, align 1
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i16> undef, <8 x i16>* undef, align 1
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x float> undef, <4 x float>* undef, align 1
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <2 x double> undef, <2 x double>* undef, align 1
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  store i8 undef, i8* undef, align 4
+  store i16 undef, i16* undef, align 4
+  store i32 undef, i32* undef, align 4
+  store i64 undef, i64* undef, align 4
+  store i128 undef, i128* undef, align 4
+  store float undef, float* undef, align 4
+  store double undef, double* undef, align 4
+
+  store <2 x i8> undef, <2 x i8>* undef, align 1
+  store <2 x i16> undef, <2 x i16>* undef, align 2
+  store <2 x i32> undef, <2 x i32>* undef, align 4
+  store <2 x i64> undef, <2 x i64>* undef, align 4
+  store <4 x i32> undef, <4 x i32>* undef, align 4
+  store <8 x i16> undef, <8 x i16>* undef, align 2
+  store <16 x i8> undef, <16 x i8>* undef, align 1
+
+  store <4 x float> undef, <4 x float>* undef, align 4
+  store <4 x double> undef, <4 x double>* undef, align 4
+  store <2 x float> undef, <2 x float>* undef, align 4
+  store <2 x double> undef, <2 x double>* undef, align 4
+
+  store <2 x i64> undef, <2 x i64>* undef, align 1
+  store <4 x i32> undef, <4 x i32>* undef, align 1
+  store <8 x i16> undef, <8 x i16>* undef, align 1
+  store <4 x float> undef, <4 x float>* undef, align 1
+  store <2 x double> undef, <2 x double>* undef, align 1
+
+  ret void
+}
+
+define void @loads() {
+; CHECK-NOVEC-LABEL: 'loads'
+; CHECK-NOVEC-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = load i8, i8* undef, align 4
+; CHECK-NOVEC-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = load i16, i16* undef, align 4
+; CHECK-NOVEC-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = load i32, i32* undef, align 4
+; CHECK-NOVEC-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %4 = load i64, i64* undef, align 4
+; CHECK-NOVEC-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %5 = load i128, i128* undef, align 4
+; CHECK-NOVEC-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = load float, float* undef, align 4
+; CHECK-NOVEC-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %7 = load double, double* undef, align 4
+; CHECK-NOVEC-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %8 = load <2 x i8>, <2 x i8>* undef, align 1
+; CHECK-NOVEC-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %9 = load <2 x i16>, <2 x i16>* undef, align 2
+; CHECK-NOVEC-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %10 = load <2 x i32>, <2 x i32>* undef, align 4
+; CHECK-NOVEC-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %11 = load <2 x i64>, <2 x i64>* undef, align 4
+; CHECK-NOVEC-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %12 = load <4 x i32>, <4 x i32>* undef, align 4
+; CHECK-NOVEC-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %13 = load <8 x i16>, <8 x i16>* undef, align 2
+; CHECK-NOVEC-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %14 = load <16 x i8>, <16 x i8>* undef, align 1
+; CHECK-NOVEC-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %15 = load <4 x float>, <4 x float>* undef, align 4
+; CHECK-NOVEC-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %16 = load <4 x double>, <4 x double>* undef, align 4
+; CHECK-NOVEC-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %17 = load <2 x float>, <2 x float>* undef, align 4
+; CHECK-NOVEC-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %18 = load <2 x double>, <2 x double>* undef, align 4
+; CHECK-NOVEC-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %19 = load <2 x i64>, <2 x i64>* undef, align 1
+; CHECK-NOVEC-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %20 = load <4 x i32>, <4 x i32>* undef, align 1
+; CHECK-NOVEC-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %21 = load <8 x i16>, <8 x i16>* undef, align 1
+; CHECK-NOVEC-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %22 = load <4 x float>, <4 x float>* undef, align 1
+; CHECK-NOVEC-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %23 = load <2 x double>, <2 x double>* undef, align 1
+; CHECK-NOVEC-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-FP-LABEL: 'loads'
+; CHECK-FP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = load i8, i8* undef, align 4
+; CHECK-FP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = load i16, i16* undef, align 4
+; CHECK-FP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = load i32, i32* undef, align 4
+; CHECK-FP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %4 = load i64, i64* undef, align 4
+; CHECK-FP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %5 = load i128, i128* undef, align 4
+; CHECK-FP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = load float, float* undef, align 4
+; CHECK-FP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = load double, double* undef, align 4
+; CHECK-FP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %8 = load <2 x i8>, <2 x i8>* undef, align 1
+; CHECK-FP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %9 = load <2 x i16>, <2 x i16>* undef, align 2
+; CHECK-FP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %10 = load <2 x i32>, <2 x i32>* undef, align 4
+; CHECK-FP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %11 = load <2 x i64>, <2 x i64>* undef, align 4
+; CHECK-FP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %12 = load <4 x i32>, <4 x i32>* undef, align 4
+; CHECK-FP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %13 = load <8 x i16>, <8 x i16>* undef, align 2
+; CHECK-FP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %14 = load <16 x i8>, <16 x i8>* undef, align 1
+; CHECK-FP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %15 = load <4 x float>, <4 x float>* undef, align 4
+; CHECK-FP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %16 = load <4 x double>, <4 x double>* undef, align 4
+; CHECK-FP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %17 = load <2 x float>, <2 x float>* undef, align 4
+; CHECK-FP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %18 = load <2 x double>, <2 x double>* undef, align 4
+; CHECK-FP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %19 = load <2 x i64>, <2 x i64>* undef, align 1
+; CHECK-FP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %20 = load <4 x i32>, <4 x i32>* undef, align 1
+; CHECK-FP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %21 = load <8 x i16>, <8 x i16>* undef, align 1
+; CHECK-FP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %22 = load <4 x float>, <4 x float>* undef, align 1
+; CHECK-FP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %23 = load <2 x double>, <2 x double>* undef, align 1
+; CHECK-FP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-MVE-LABEL: 'loads'
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = load i8, i8* undef, align 4
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = load i16, i16* undef, align 4
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = load i32, i32* undef, align 4
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %4 = load i64, i64* undef, align 4
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %5 = load i128, i128* undef, align 4
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = load float, float* undef, align 4
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = load double, double* undef, align 4
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %8 = load <2 x i8>, <2 x i8>* undef, align 1
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %9 = load <2 x i16>, <2 x i16>* undef, align 2
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %10 = load <2 x i32>, <2 x i32>* undef, align 4
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %11 = load <2 x i64>, <2 x i64>* undef, align 4
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %12 = load <4 x i32>, <4 x i32>* undef, align 4
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %13 = load <8 x i16>, <8 x i16>* undef, align 2
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %14 = load <16 x i8>, <16 x i8>* undef, align 1
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %15 = load <4 x float>, <4 x float>* undef, align 4
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %16 = load <4 x double>, <4 x double>* undef, align 4
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %17 = load <2 x float>, <2 x float>* undef, align 4
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %18 = load <2 x double>, <2 x double>* undef, align 4
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %19 = load <2 x i64>, <2 x i64>* undef, align 1
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %20 = load <4 x i32>, <4 x i32>* undef, align 1
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %21 = load <8 x i16>, <8 x i16>* undef, align 1
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %22 = load <4 x float>, <4 x float>* undef, align 1
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %23 = load <2 x double>, <2 x double>* undef, align 1
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-NEON-LABEL: 'loads'
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = load i8, i8* undef, align 4
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = load i16, i16* undef, align 4
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = load i32, i32* undef, align 4
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %4 = load i64, i64* undef, align 4
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %5 = load i128, i128* undef, align 4
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = load float, float* undef, align 4
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = load double, double* undef, align 4
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = load <2 x i8>, <2 x i8>* undef, align 1
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %9 = load <2 x i16>, <2 x i16>* undef, align 2
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %10 = load <2 x i32>, <2 x i32>* undef, align 4
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %11 = load <2 x i64>, <2 x i64>* undef, align 4
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %12 = load <4 x i32>, <4 x i32>* undef, align 4
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %13 = load <8 x i16>, <8 x i16>* undef, align 2
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %14 = load <16 x i8>, <16 x i8>* undef, align 1
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %15 = load <4 x float>, <4 x float>* undef, align 4
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %16 = load <4 x double>, <4 x double>* undef, align 4
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %17 = load <2 x float>, <2 x float>* undef, align 4
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %18 = load <2 x double>, <2 x double>* undef, align 4
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %19 = load <2 x i64>, <2 x i64>* undef, align 1
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %20 = load <4 x i32>, <4 x i32>* undef, align 1
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %21 = load <8 x i16>, <8 x i16>* undef, align 1
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %22 = load <4 x float>, <4 x float>* undef, align 1
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %23 = load <2 x double>, <2 x double>* undef, align 1
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  load i8, i8* undef, align 4
+  load i16, i16* undef, align 4
+  load i32, i32* undef, align 4
+  load i64, i64* undef, align 4
+  load i128, i128* undef, align 4
+  load float, float* undef, align 4
+  load double, double* undef, align 4
+
+  load <2 x i8>, <2 x i8>* undef, align 1
+  load <2 x i16>, <2 x i16>* undef, align 2
+  load <2 x i32>, <2 x i32>* undef, align 4
+  load <2 x i64>, <2 x i64>* undef, align 4
+  load <4 x i32>, <4 x i32>* undef, align 4
+  load <8 x i16>, <8 x i16>* undef, align 2
+  load <16 x i8>, <16 x i8>* undef, align 1
+
+  load <4 x float>, <4 x float>* undef, align 4
+  load <4 x double>, <4 x double>* undef, align 4
+  load <2 x float>, <2 x float>* undef, align 4
+  load <2 x double>, <2 x double>* undef, align 4
+
+  load <2 x i64>, <2 x i64>* undef, align 1
+  load <4 x i32>, <4 x i32>* undef, align 1
+  load <8 x i16>, <8 x i16>* undef, align 1
+  load <4 x float>, <4 x float>* undef, align 1
+  load <2 x double>, <2 x double>* undef, align 1
+
+  ret void
+}
+
diff --git a/llvm/test/Analysis/CostModel/ARM/select.ll b/llvm/test/Analysis/CostModel/ARM/select.ll
index 57e1418a3..4821438e8 100644
--- a/llvm/test/Analysis/CostModel/ARM/select.ll
+++ b/llvm/test/Analysis/CostModel/ARM/select.ll
@@ -1,74 +1,107 @@
-; RUN: opt < %s  -cost-model -analyze -mtriple=thumbv7-apple-ios6.0.0 -mcpu=swift | FileCheck %s
-target datalayout = "e-p:32:32:32-i1:8:32-i8:8:32-i16:16:32-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:32:64-v128:32:128-a0:0:32-n32-S32"
-target triple = "thumbv7-apple-ios6.0.0"
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt < %s  -cost-model -analyze -mtriple=thumbv8.1-m.main-none-eabi -mattr=+mve.fp | FileCheck %s --check-prefix=CHECK-MVE
+; RUN: opt < %s  -cost-model -analyze -mtriple=thumbv7-apple-ios6.0.0 -mcpu=swift | FileCheck %s --check-prefix=CHECK-NEON
 
-; CHECK: casts
 define void @casts() {
-    ; Scalar values
-  ; CHECK: cost of 1 {{.*}} select
+; CHECK-MVE-LABEL: 'casts'
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v1 = select i1 undef, i8 undef, i8 undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2 = select i1 undef, i16 undef, i16 undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v3 = select i1 undef, i32 undef, i32 undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4 = select i1 undef, i64 undef, i64 undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v5 = select i1 undef, float undef, float undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v6 = select i1 undef, double undef, double undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v7 = select <2 x i1> undef, <2 x i8> undef, <2 x i8> undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8 = select <4 x i1> undef, <4 x i8> undef, <4 x i8> undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v9 = select <8 x i1> undef, <8 x i8> undef, <8 x i8> undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v10 = select <16 x i1> undef, <16 x i8> undef, <16 x i8> undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v11 = select <2 x i1> undef, <2 x i16> undef, <2 x i16> undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v12 = select <4 x i1> undef, <4 x i16> undef, <4 x i16> undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v13 = select <8 x i1> undef, <8 x i16> undef, <8 x i16> undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v13b = select <16 x i1> undef, <16 x i16> undef, <16 x i16> undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v14 = select <2 x i1> undef, <2 x i32> undef, <2 x i32> undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v15 = select <4 x i1> undef, <4 x i32> undef, <4 x i32> undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v15b = select <8 x i1> undef, <8 x i32> undef, <8 x i32> undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v15c = select <16 x i1> undef, <16 x i32> undef, <16 x i32> undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v16 = select <2 x i1> undef, <2 x i64> undef, <2 x i64> undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %v16a = select <4 x i1> undef, <4 x i64> undef, <4 x i64> undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %v16b = select <8 x i1> undef, <8 x i64> undef, <8 x i64> undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 576 for instruction: %v16c = select <16 x i1> undef, <16 x i64> undef, <16 x i64> undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v17 = select <2 x i1> undef, <2 x float> undef, <2 x float> undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v18 = select <4 x i1> undef, <4 x float> undef, <4 x float> undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v19 = select <2 x i1> undef, <2 x double> undef, <2 x double> undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v20 = select <1 x i1> undef, <1 x i32> undef, <1 x i32> undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v21 = select <3 x i1> undef, <3 x float> undef, <3 x float> undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 60 for instruction: %v22 = select <5 x i1> undef, <5 x double> undef, <5 x double> undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-NEON-LABEL: 'casts'
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v1 = select i1 undef, i8 undef, i8 undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2 = select i1 undef, i16 undef, i16 undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v3 = select i1 undef, i32 undef, i32 undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4 = select i1 undef, i64 undef, i64 undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v5 = select i1 undef, float undef, float undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v6 = select i1 undef, double undef, double undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v7 = select <2 x i1> undef, <2 x i8> undef, <2 x i8> undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8 = select <4 x i1> undef, <4 x i8> undef, <4 x i8> undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v9 = select <8 x i1> undef, <8 x i8> undef, <8 x i8> undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v10 = select <16 x i1> undef, <16 x i8> undef, <16 x i8> undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v11 = select <2 x i1> undef, <2 x i16> undef, <2 x i16> undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v12 = select <4 x i1> undef, <4 x i16> undef, <4 x i16> undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v13 = select <8 x i1> undef, <8 x i16> undef, <8 x i16> undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v13b = select <16 x i1> undef, <16 x i16> undef, <16 x i16> undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v14 = select <2 x i1> undef, <2 x i32> undef, <2 x i32> undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v15 = select <4 x i1> undef, <4 x i32> undef, <4 x i32> undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v15b = select <8 x i1> undef, <8 x i32> undef, <8 x i32> undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v15c = select <16 x i1> undef, <16 x i32> undef, <16 x i32> undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16 = select <2 x i1> undef, <2 x i64> undef, <2 x i64> undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %v16a = select <4 x i1> undef, <4 x i64> undef, <4 x i64> undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %v16b = select <8 x i1> undef, <8 x i64> undef, <8 x i64> undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %v16c = select <16 x i1> undef, <16 x i64> undef, <16 x i64> undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v17 = select <2 x i1> undef, <2 x float> undef, <2 x float> undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v18 = select <4 x i1> undef, <4 x float> undef, <4 x float> undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v19 = select <2 x i1> undef, <2 x double> undef, <2 x double> undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v20 = select <1 x i1> undef, <1 x i32> undef, <1 x i32> undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21 = select <3 x i1> undef, <3 x float> undef, <3 x float> undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v22 = select <5 x i1> undef, <5 x double> undef, <5 x double> undef
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  ; Scalar values
   %v1 = select i1 undef, i8 undef, i8 undef
-  ; CHECK: cost of 1 {{.*}} select
   %v2 = select i1 undef, i16 undef, i16 undef
-  ; CHECK: cost of 1 {{.*}} select
   %v3 = select i1 undef, i32 undef, i32 undef
-  ; CHECK: cost of 2 {{.*}} select
   %v4 = select i1 undef, i64 undef, i64 undef
-  ; CHECK: cost of 1 {{.*}} select
   %v5 = select i1 undef, float undef, float undef
-  ; CHECK: cost of 1 {{.*}} select
   %v6 = select i1 undef, double undef, double undef
 
-    ; Vector values
-  ; CHECK: cost of 1 {{.*}} select
+  ; Vector values
   %v7 = select <2 x i1> undef, <2 x i8> undef, <2 x i8> undef
-  ; CHECK: cost of 1 {{.*}} select
   %v8 = select <4 x i1>  undef, <4 x i8> undef, <4 x i8> undef
-  ; CHECK: cost of 1 {{.*}} select
   %v9 = select <8 x i1>  undef, <8 x i8> undef, <8 x i8> undef
-  ; CHECK: cost of 1 {{.*}} select
   %v10 = select <16 x i1>  undef, <16 x i8> undef, <16 x i8> undef
 
-  ; CHECK: cost of 1 {{.*}} select
   %v11 = select <2 x i1> undef, <2 x i16> undef, <2 x i16> undef
-  ; CHECK: cost of 1 {{.*}} select
   %v12 = select <4 x i1>  undef, <4 x i16> undef, <4 x i16> undef
-  ; CHECK: cost of 1 {{.*}} select
   %v13 = select <8 x i1>  undef, <8 x i16> undef, <8 x i16> undef
-  ; CHECK: cost of 2 {{.*}} select
   %v13b = select <16 x i1>  undef, <16 x i16> undef, <16 x i16> undef
 
-  ; CHECK: cost of 1 {{.*}} select
   %v14 = select <2 x i1> undef, <2 x i32> undef, <2 x i32> undef
-  ; CHECK: cost of 1 {{.*}} select
   %v15 = select <4 x i1>  undef, <4 x i32> undef, <4 x i32> undef
-  ; CHECK: cost of 2 {{.*}} select
   %v15b = select <8 x i1>  undef, <8 x i32> undef, <8 x i32> undef
-  ; CHECK: cost of 4 {{.*}} select
   %v15c = select <16 x i1>  undef, <16 x i32> undef, <16 x i32> undef
 
-  ; CHECK: cost of 1 {{.*}} select
   %v16 = select <2 x i1> undef, <2 x i64> undef, <2 x i64> undef
-  ; CHECK: cost of 19 {{.*}} select
   %v16a = select <4 x i1> undef, <4 x i64> undef, <4 x i64> undef
-  ; CHECK: cost of 50 {{.*}} select
   %v16b = select <8 x i1> undef, <8 x i64> undef, <8 x i64> undef
-  ; CHECK: cost of 100 {{.*}} select
   %v16c = select <16 x i1> undef, <16 x i64> undef, <16 x i64> undef
 
-  ; CHECK: cost of 1 {{.*}} select
   %v17 = select <2 x i1> undef, <2 x float> undef, <2 x float> undef
-  ; CHECK: cost of 1 {{.*}} select
   %v18 = select <4 x i1>  undef, <4 x float> undef, <4 x float> undef
 
-  ; CHECK: cost of 1 {{.*}} select
   %v19 = select <2 x i1>  undef, <2 x double> undef, <2 x double> undef
 
   ; odd vectors get legalized and should have similar costs
-  ; CHECK: cost of 1 {{.*}} select
   %v20 = select <1 x i1>  undef, <1 x i32> undef, <1 x i32> undef
-  ; CHECK: cost of 1 {{.*}} select
   %v21 = select <3 x i1>  undef, <3 x float> undef, <3 x float> undef
-  ; CHECK: cost of 4 {{.*}} select
   %v22 = select <5 x i1>  undef, <5 x double> undef, <5 x double> undef
 
   ret void
diff --git a/llvm/test/Analysis/CostModel/ARM/shuffle.ll b/llvm/test/Analysis/CostModel/ARM/shuffle.ll
index a6a423525..ce2098653 100644
--- a/llvm/test/Analysis/CostModel/ARM/shuffle.ll
+++ b/llvm/test/Analysis/CostModel/ARM/shuffle.ll
@@ -1,71 +1,104 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
-; RUN: opt < %s  -cost-model -analyze -mtriple=thumbv7-apple-ios6.0.0 -mcpu=swift | FileCheck %s
-target datalayout = "e-p:32:32:32-i1:8:32-i8:8:32-i16:16:32-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:32:64-v128:32:128-a0:0:32-n32-S32"
-target triple = "thumbv7-apple-ios6.0.0"
+; RUN: opt < %s  -cost-model -analyze -mtriple=thumbv8.1-m.main-none-eabi -mattr=+mve.fp | FileCheck %s --check-prefix=CHECK-MVE
+; RUN: opt < %s  -cost-model -analyze -mtriple=thumbv7-apple-ios6.0.0 -mcpu=swift | FileCheck %s --check-prefix=CHECK-NEON
 
 define void @broadcast() {
-; CHECK-LABEL: 'broadcast'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v7 = shufflevector <2 x i8> undef, <2 x i8> undef, <2 x i32> zeroinitializer
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8 = shufflevector <4 x i8> undef, <4 x i8> undef, <4 x i32> zeroinitializer
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v9 = shufflevector <8 x i8> undef, <8 x i8> undef, <8 x i32> zeroinitializer
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v10 = shufflevector <16 x i8> undef, <16 x i8> undef, <16 x i32> zeroinitializer
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v11 = shufflevector <2 x i16> undef, <2 x i16> undef, <2 x i32> zeroinitializer
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v12 = shufflevector <4 x i16> undef, <4 x i16> undef, <4 x i32> zeroinitializer
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v13 = shufflevector <8 x i16> undef, <8 x i16> undef, <8 x i32> zeroinitializer
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v14 = shufflevector <2 x i32> undef, <2 x i32> undef, <2 x i32> zeroinitializer
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v15 = shufflevector <4 x i32> undef, <4 x i32> undef, <4 x i32> zeroinitializer
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16 = shufflevector <2 x float> undef, <2 x float> undef, <2 x i32> zeroinitializer
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v17 = shufflevector <4 x float> undef, <4 x float> undef, <4 x i32> zeroinitializer
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-MVE-LABEL: 'broadcast'
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v7 = shufflevector <2 x i8> undef, <2 x i8> undef, <2 x i32> zeroinitializer
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8 = shufflevector <4 x i8> undef, <4 x i8> undef, <4 x i32> zeroinitializer
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v9 = shufflevector <8 x i8> undef, <8 x i8> undef, <8 x i32> zeroinitializer
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v10 = shufflevector <16 x i8> undef, <16 x i8> undef, <16 x i32> zeroinitializer
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v11 = shufflevector <2 x i16> undef, <2 x i16> undef, <2 x i32> zeroinitializer
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v12 = shufflevector <4 x i16> undef, <4 x i16> undef, <4 x i32> zeroinitializer
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v13 = shufflevector <8 x i16> undef, <8 x i16> undef, <8 x i32> zeroinitializer
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v14 = shufflevector <2 x i32> undef, <2 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v15 = shufflevector <4 x i32> undef, <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16 = shufflevector <2 x float> undef, <2 x float> undef, <2 x i32> zeroinitializer
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v17 = shufflevector <4 x float> undef, <4 x float> undef, <4 x i32> zeroinitializer
+; CHECK-MVE-NEXT:  Cost Model: Unknown cost for instruction: %v18 = shufflevector <8 x half> undef, <8 x half> undef, <4 x i32> zeroinitializer
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
-  %v7 = shufflevector <2 x i8> undef, <2 x i8>undef, <2 x i32> zeroinitializer
-  %v8 = shufflevector <4 x i8> undef, <4 x i8>undef, <4 x i32> zeroinitializer
-  %v9 = shufflevector <8 x i8> undef, <8 x i8>undef, <8 x i32> zeroinitializer
-  %v10 = shufflevector <16 x i8> undef, <16 x i8>undef, <16 x i32> zeroinitializer
+; CHECK-NEON-LABEL: 'broadcast'
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v7 = shufflevector <2 x i8> undef, <2 x i8> undef, <2 x i32> zeroinitializer
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8 = shufflevector <4 x i8> undef, <4 x i8> undef, <4 x i32> zeroinitializer
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v9 = shufflevector <8 x i8> undef, <8 x i8> undef, <8 x i32> zeroinitializer
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v10 = shufflevector <16 x i8> undef, <16 x i8> undef, <16 x i32> zeroinitializer
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v11 = shufflevector <2 x i16> undef, <2 x i16> undef, <2 x i32> zeroinitializer
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v12 = shufflevector <4 x i16> undef, <4 x i16> undef, <4 x i32> zeroinitializer
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v13 = shufflevector <8 x i16> undef, <8 x i16> undef, <8 x i32> zeroinitializer
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v14 = shufflevector <2 x i32> undef, <2 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v15 = shufflevector <4 x i32> undef, <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16 = shufflevector <2 x float> undef, <2 x float> undef, <2 x i32> zeroinitializer
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v17 = shufflevector <4 x float> undef, <4 x float> undef, <4 x i32> zeroinitializer
+; CHECK-NEON-NEXT:  Cost Model: Unknown cost for instruction: %v18 = shufflevector <8 x half> undef, <8 x half> undef, <4 x i32> zeroinitializer
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %v7 = shufflevector <2 x i8> undef, <2 x i8> undef, <2 x i32> zeroinitializer
+  %v8 = shufflevector <4 x i8> undef, <4 x i8> undef, <4 x i32> zeroinitializer
+  %v9 = shufflevector <8 x i8> undef, <8 x i8> undef, <8 x i32> zeroinitializer
+  %v10 = shufflevector <16 x i8> undef, <16 x i8> undef, <16 x i32> zeroinitializer
 
-  %v11 = shufflevector <2 x i16> undef, <2 x i16>undef, <2 x i32> zeroinitializer
-  %v12 = shufflevector <4 x i16> undef, <4 x i16>undef, <4 x i32> zeroinitializer
-  %v13 = shufflevector <8 x i16> undef, <8 x i16>undef, <8 x i32> zeroinitializer
+  %v11 = shufflevector <2 x i16> undef, <2 x i16> undef, <2 x i32> zeroinitializer
+  %v12 = shufflevector <4 x i16> undef, <4 x i16> undef, <4 x i32> zeroinitializer
+  %v13 = shufflevector <8 x i16> undef, <8 x i16> undef, <8 x i32> zeroinitializer
 
-  %v14 = shufflevector <2 x i32> undef, <2 x i32>undef, <2 x i32> zeroinitializer
-  %v15 = shufflevector <4 x i32> undef, <4 x i32>undef, <4 x i32> zeroinitializer
+  %v14 = shufflevector <2 x i32> undef, <2 x i32> undef, <2 x i32> zeroinitializer
+  %v15 = shufflevector <4 x i32> undef, <4 x i32> undef, <4 x i32> zeroinitializer
 
-  %v16 = shufflevector <2 x float> undef, <2 x float>undef, <2 x i32> zeroinitializer
-  %v17 = shufflevector <4 x float> undef, <4 x float>undef, <4 x i32> zeroinitializer
+  %v16 = shufflevector <2 x float> undef, <2 x float> undef, <2 x i32> zeroinitializer
+  %v17 = shufflevector <4 x float> undef, <4 x float> undef, <4 x i32> zeroinitializer
+  %v18 = shufflevector <8 x half> undef, <8 x half> undef, <4 x i32> zeroinitializer
 
   ret void
 }
 
-;; Reverse shuffles should be lowered to vrev and possibly a vext (for quadwords)
+;; Reverse shuffles should be lowered to vrev and possibly a vext (for quadwords, on neon)
 define void @reverse() {
-; CHECK-LABEL: 'reverse'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v7 = shufflevector <2 x i8> undef, <2 x i8> undef, <2 x i32> <i32 1, i32 0>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8 = shufflevector <4 x i8> undef, <4 x i8> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v9 = shufflevector <8 x i8> undef, <8 x i8> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v10 = shufflevector <16 x i8> undef, <16 x i8> undef, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v11 = shufflevector <2 x i16> undef, <2 x i16> undef, <2 x i32> <i32 1, i32 0>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v12 = shufflevector <4 x i16> undef, <4 x i16> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v13 = shufflevector <8 x i16> undef, <8 x i16> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v14 = shufflevector <2 x i32> undef, <2 x i32> undef, <2 x i32> <i32 1, i32 0>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v15 = shufflevector <4 x i32> undef, <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16 = shufflevector <2 x float> undef, <2 x float> undef, <2 x i32> <i32 1, i32 0>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v17 = shufflevector <4 x float> undef, <4 x float> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-MVE-LABEL: 'reverse'
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v7 = shufflevector <2 x i8> undef, <2 x i8> undef, <2 x i32> <i32 1, i32 0>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v8 = shufflevector <4 x i8> undef, <4 x i8> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 256 for instruction: %v9 = shufflevector <8 x i8> undef, <8 x i8> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1024 for instruction: %v10 = shufflevector <16 x i8> undef, <16 x i8> undef, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v11 = shufflevector <2 x i16> undef, <2 x i16> undef, <2 x i32> <i32 1, i32 0>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v12 = shufflevector <4 x i16> undef, <4 x i16> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 256 for instruction: %v13 = shufflevector <8 x i16> undef, <8 x i16> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v14 = shufflevector <2 x i32> undef, <2 x i32> undef, <2 x i32> <i32 1, i32 0>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v15 = shufflevector <4 x i32> undef, <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v16 = shufflevector <2 x float> undef, <2 x float> undef, <2 x i32> <i32 1, i32 0>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v17 = shufflevector <4 x float> undef, <4 x float> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 256 for instruction: %v18 = shufflevector <8 x half> undef, <8 x half> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-NEON-LABEL: 'reverse'
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v7 = shufflevector <2 x i8> undef, <2 x i8> undef, <2 x i32> <i32 1, i32 0>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8 = shufflevector <4 x i8> undef, <4 x i8> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v9 = shufflevector <8 x i8> undef, <8 x i8> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v10 = shufflevector <16 x i8> undef, <16 x i8> undef, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v11 = shufflevector <2 x i16> undef, <2 x i16> undef, <2 x i32> <i32 1, i32 0>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v12 = shufflevector <4 x i16> undef, <4 x i16> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v13 = shufflevector <8 x i16> undef, <8 x i16> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v14 = shufflevector <2 x i32> undef, <2 x i32> undef, <2 x i32> <i32 1, i32 0>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v15 = shufflevector <4 x i32> undef, <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16 = shufflevector <2 x float> undef, <2 x float> undef, <2 x i32> <i32 1, i32 0>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v17 = shufflevector <4 x float> undef, <4 x float> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v18 = shufflevector <8 x half> undef, <8 x half> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
-  %v7 = shufflevector <2 x i8> undef, <2 x i8>undef, <2 x i32> <i32 1, i32 0>
-  %v8 = shufflevector <4 x i8> undef, <4 x i8>undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-  %v9 = shufflevector <8 x i8> undef, <8 x i8>undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-  %v10 = shufflevector <16 x i8> undef, <16 x i8>undef, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  %v7 = shufflevector <2 x i8> undef, <2 x i8> undef, <2 x i32> <i32 1, i32 0>
+  %v8 = shufflevector <4 x i8> undef, <4 x i8> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %v9 = shufflevector <8 x i8> undef, <8 x i8> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  %v10 = shufflevector <16 x i8> undef, <16 x i8> undef, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 
-  %v11 = shufflevector <2 x i16> undef, <2 x i16>undef, <2 x i32> <i32 1, i32 0>
-  %v12 = shufflevector <4 x i16> undef, <4 x i16>undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-  %v13 = shufflevector <8 x i16> undef, <8 x i16>undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  %v11 = shufflevector <2 x i16> undef, <2 x i16> undef, <2 x i32> <i32 1, i32 0>
+  %v12 = shufflevector <4 x i16> undef, <4 x i16> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %v13 = shufflevector <8 x i16> undef, <8 x i16> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 
-  %v14 = shufflevector <2 x i32> undef, <2 x i32>undef, <2 x i32> <i32 1, i32 0>
-  %v15 = shufflevector <4 x i32> undef, <4 x i32>undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %v14 = shufflevector <2 x i32> undef, <2 x i32> undef, <2 x i32> <i32 1, i32 0>
+  %v15 = shufflevector <4 x i32> undef, <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
 
-  %v16 = shufflevector <2 x float> undef, <2 x float>undef, <2 x i32> <i32 1, i32 0>
-  %v17 = shufflevector <4 x float> undef, <4 x float>undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %v16 = shufflevector <2 x float> undef, <2 x float> undef, <2 x i32> <i32 1, i32 0>
+  %v17 = shufflevector <4 x float> undef, <4 x float> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %v18 = shufflevector <8 x half> undef, <8 x half> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 
   ret void
 }
diff --git a/llvm/test/Analysis/CostModel/PowerPC/future-cost-model.ll b/llvm/test/Analysis/CostModel/PowerPC/future-cost-model.ll
new file mode 100644
index 000000000..3e4fb82e6
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/PowerPC/future-cost-model.ll
@@ -0,0 +1,16 @@
+; RUN: opt < %s -cost-model -analyze -mtriple=powerpc64le-unknown-linux-gnu \
+; RUN:       -mcpu=future | FileCheck %s --check-prefix=FUTURE
+; RUN: opt < %s -cost-model -analyze -mtriple=powerpc64le-unknown-linux-gnu \
+; RUN:       -mcpu=pwr9 | FileCheck %s --check-prefix=PWR9
+
+define void @test(i16 %p1, i16 %p2, <4 x i16> %p3, <4 x i16> %p4) {
+  %i1 = add i16 %p1, %p2
+  %v1 = add <4 x i16> %p3, %p4
+  ret void
+  ; FUTURE: cost of 1 {{.*}} add
+  ; FUTURE: cost of 1 {{.*}} add
+
+  ; PWR9: cost of 1 {{.*}} add
+  ; PWR9: cost of 2 {{.*}} add
+}
+
diff --git a/llvm/test/Analysis/CostModel/PowerPC/insert_extract.ll b/llvm/test/Analysis/CostModel/PowerPC/insert_extract.ll
index 030af67d6..59dbd9411 100644
--- a/llvm/test/Analysis/CostModel/PowerPC/insert_extract.ll
+++ b/llvm/test/Analysis/CostModel/PowerPC/insert_extract.ll
@@ -14,15 +14,15 @@ define i32 @insert(i32 %arg) {
 ; CHECK-P7-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; CHECK-P8LE-LABEL: 'insert'
-; CHECK-P8LE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %x = insertelement <4 x i32> undef, i32 %arg, i32 0
+; CHECK-P8LE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %x = insertelement <4 x i32> undef, i32 %arg, i32 0
 ; CHECK-P8LE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; CHECK-P9BE-LABEL: 'insert'
-; CHECK-P9BE-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %x = insertelement <4 x i32> undef, i32 %arg, i32 0
+; CHECK-P9BE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %x = insertelement <4 x i32> undef, i32 %arg, i32 0
 ; CHECK-P9BE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; CHECK-P9LE-LABEL: 'insert'
-; CHECK-P9LE-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %x = insertelement <4 x i32> undef, i32 %arg, i32 0
+; CHECK-P9LE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %x = insertelement <4 x i32> undef, i32 %arg, i32 0
 ; CHECK-P9LE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %x = insertelement <4 x i32> undef, i32 %arg, i32 0
@@ -40,11 +40,11 @@ define i32 @extract(<4 x i32> %arg) {
 ; CHECK-P8LE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %x
 ;
 ; CHECK-P9BE-LABEL: 'extract'
-; CHECK-P9BE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %x = extractelement <4 x i32> %arg, i32 0
+; CHECK-P9BE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %x = extractelement <4 x i32> %arg, i32 0
 ; CHECK-P9BE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %x
 ;
 ; CHECK-P9LE-LABEL: 'extract'
-; CHECK-P9LE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %x = extractelement <4 x i32> %arg, i32 0
+; CHECK-P9LE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %x = extractelement <4 x i32> %arg, i32 0
 ; CHECK-P9LE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %x
 ;
   %x = extractelement <4 x i32> %arg, i32 0
@@ -83,15 +83,15 @@ define void @test4xi32(<4 x i32> %v1, i32 %x1) {
 ; CHECK-P7-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-P8LE-LABEL: 'test4xi32'
-; CHECK-P8LE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v2 = insertelement <4 x i32> %v1, i32 %x1, i32 2
+; CHECK-P8LE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2 = insertelement <4 x i32> %v1, i32 %x1, i32 2
 ; CHECK-P8LE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-P9BE-LABEL: 'test4xi32'
-; CHECK-P9BE-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %v2 = insertelement <4 x i32> %v1, i32 %x1, i32 2
+; CHECK-P9BE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2 = insertelement <4 x i32> %v1, i32 %x1, i32 2
 ; CHECK-P9BE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-P9LE-LABEL: 'test4xi32'
-; CHECK-P9LE-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %v2 = insertelement <4 x i32> %v1, i32 %x1, i32 2
+; CHECK-P9LE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2 = insertelement <4 x i32> %v1, i32 %x1, i32 2
 ; CHECK-P9LE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %v2 = insertelement <4 x i32> %v1, i32 %x1, i32 2
@@ -114,17 +114,17 @@ define void @vexti32(<4 x i32> %p1) {
 ; CHECK-P8LE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-P9BE-LABEL: 'vexti32'
-; CHECK-P9BE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %i1 = extractelement <4 x i32> %p1, i32 0
-; CHECK-P9BE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %i2 = extractelement <4 x i32> %p1, i32 1
-; CHECK-P9BE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %i3 = extractelement <4 x i32> %p1, i32 2
-; CHECK-P9BE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %i4 = extractelement <4 x i32> %p1, i32 3
+; CHECK-P9BE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %i1 = extractelement <4 x i32> %p1, i32 0
+; CHECK-P9BE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i2 = extractelement <4 x i32> %p1, i32 1
+; CHECK-P9BE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %i3 = extractelement <4 x i32> %p1, i32 2
+; CHECK-P9BE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %i4 = extractelement <4 x i32> %p1, i32 3
 ; CHECK-P9BE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-P9LE-LABEL: 'vexti32'
-; CHECK-P9LE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %i1 = extractelement <4 x i32> %p1, i32 0
-; CHECK-P9LE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %i2 = extractelement <4 x i32> %p1, i32 1
-; CHECK-P9LE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %i3 = extractelement <4 x i32> %p1, i32 2
-; CHECK-P9LE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %i4 = extractelement <4 x i32> %p1, i32 3
+; CHECK-P9LE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %i1 = extractelement <4 x i32> %p1, i32 0
+; CHECK-P9LE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %i2 = extractelement <4 x i32> %p1, i32 1
+; CHECK-P9LE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i3 = extractelement <4 x i32> %p1, i32 2
+; CHECK-P9LE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %i4 = extractelement <4 x i32> %p1, i32 3
 ; CHECK-P9LE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %i1 = extractelement <4 x i32> %p1, i32 0
@@ -146,13 +146,13 @@ define void @vexti64(<2 x i64> %p1) {
 ; CHECK-P8LE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-P9BE-LABEL: 'vexti64'
-; CHECK-P9BE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %i1 = extractelement <2 x i64> %p1, i32 0
-; CHECK-P9BE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %i2 = extractelement <2 x i64> %p1, i32 1
+; CHECK-P9BE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i1 = extractelement <2 x i64> %p1, i32 0
+; CHECK-P9BE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %i2 = extractelement <2 x i64> %p1, i32 1
 ; CHECK-P9BE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-P9LE-LABEL: 'vexti64'
-; CHECK-P9LE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %i1 = extractelement <2 x i64> %p1, i32 0
-; CHECK-P9LE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %i2 = extractelement <2 x i64> %p1, i32 1
+; CHECK-P9LE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %i1 = extractelement <2 x i64> %p1, i32 0
+; CHECK-P9LE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i2 = extractelement <2 x i64> %p1, i32 1
 ; CHECK-P9LE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %i1 = extractelement <2 x i64> %p1, i32 0
@@ -172,13 +172,13 @@ define void @vext(<8 x i16> %p1, <16 x i8> %p2) {
 ; CHECK-P8LE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-P9BE-LABEL: 'vext'
-; CHECK-P9BE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %i1 = extractelement <8 x i16> %p1, i32 0
-; CHECK-P9BE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %i2 = extractelement <16 x i8> %p2, i32 0
+; CHECK-P9BE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %i1 = extractelement <8 x i16> %p1, i32 0
+; CHECK-P9BE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %i2 = extractelement <16 x i8> %p2, i32 0
 ; CHECK-P9BE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-P9LE-LABEL: 'vext'
-; CHECK-P9LE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %i1 = extractelement <8 x i16> %p1, i32 0
-; CHECK-P9LE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %i2 = extractelement <16 x i8> %p2, i32 0
+; CHECK-P9LE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %i1 = extractelement <8 x i16> %p1, i32 0
+; CHECK-P9LE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %i2 = extractelement <16 x i8> %p2, i32 0
 ; CHECK-P9LE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %i1 = extractelement <8 x i16> %p1, i32 0
diff --git a/llvm/test/Analysis/CostModel/SystemZ/fp-cast.ll b/llvm/test/Analysis/CostModel/SystemZ/fp-cast.ll
index 9a43ca318..0c95b5373 100644
--- a/llvm/test/Analysis/CostModel/SystemZ/fp-cast.ll
+++ b/llvm/test/Analysis/CostModel/SystemZ/fp-cast.ll
@@ -1,7 +1,7 @@
 ; RUN: opt < %s -cost-model -analyze -mtriple=systemz-unknown -mcpu=z13 \
 ; RUN:  | FileCheck %s -check-prefixes=CHECK,Z13
-; RUN: opt < %s -cost-model -analyze -mtriple=systemz-unknown -mcpu=arch13 \
-; RUN:  | FileCheck %s -check-prefixes=CHECK,AR13
+; RUN: opt < %s -cost-model -analyze -mtriple=systemz-unknown -mcpu=z15 \
+; RUN:  | FileCheck %s -check-prefixes=CHECK,Z15
 ;
 ; Note: The scalarized vector instructions costs are not including any
 ; extracts, due to the undef operands.
@@ -118,7 +118,7 @@ define void @fptosi() {
 ; CHECK: Cost Model: Found an estimated cost of 6 for instruction:   %v19 = fptosi <2 x double> undef to <2 x i8>
 ; CHECK: Cost Model: Found an estimated cost of 5 for instruction:   %v20 = fptosi <2 x float> undef to <2 x i64>
 ; Z13:   Cost Model: Found an estimated cost of 12 for instruction:   %v21 = fptosi <2 x float> undef to <2 x i32>
-; AR13:  Cost Model: Found an estimated cost of 1 for instruction:   %v21 = fptosi <2 x float> undef to <2 x i32>
+; Z15:   Cost Model: Found an estimated cost of 1 for instruction:   %v21 = fptosi <2 x float> undef to <2 x i32>
 ; CHECK: Cost Model: Found an estimated cost of 6 for instruction:   %v22 = fptosi <2 x float> undef to <2 x i16>
 ; CHECK: Cost Model: Found an estimated cost of 6 for instruction:   %v23 = fptosi <2 x float> undef to <2 x i8>
 ; CHECK: Cost Model: Found an estimated cost of 6 for instruction:   %v24 = fptosi <4 x fp128> undef to <4 x i64>
@@ -131,7 +131,7 @@ define void @fptosi() {
 ; CHECK: Cost Model: Found an estimated cost of 12 for instruction:   %v31 = fptosi <4 x double> undef to <4 x i8>
 ; CHECK: Cost Model: Found an estimated cost of 10 for instruction:   %v32 = fptosi <4 x float> undef to <4 x i64>
 ; Z13:   Cost Model: Found an estimated cost of 12 for instruction:   %v33 = fptosi <4 x float> undef to <4 x i32>
-; AR13:  Cost Model: Found an estimated cost of 1 for instruction:   %v33 = fptosi <4 x float> undef to <4 x i32>
+; Z15:   Cost Model: Found an estimated cost of 1 for instruction:   %v33 = fptosi <4 x float> undef to <4 x i32>
 ; CHECK: Cost Model: Found an estimated cost of 12 for instruction:   %v34 = fptosi <4 x float> undef to <4 x i16>
 ; CHECK: Cost Model: Found an estimated cost of 12 for instruction:   %v35 = fptosi <4 x float> undef to <4 x i8>
 ; CHECK: Cost Model: Found an estimated cost of 12 for instruction:   %v36 = fptosi <8 x fp128> undef to <8 x i64>
@@ -144,7 +144,7 @@ define void @fptosi() {
 ; CHECK: Cost Model: Found an estimated cost of 24 for instruction:   %v43 = fptosi <8 x double> undef to <8 x i8>
 ; CHECK: Cost Model: Found an estimated cost of 20 for instruction:   %v44 = fptosi <8 x float> undef to <8 x i64>
 ; Z13:   Cost Model: Found an estimated cost of 24 for instruction:   %v45 = fptosi <8 x float> undef to <8 x i32>
-; AR13:  Cost Model: Found an estimated cost of 2 for instruction:   %v45 = fptosi <8 x float> undef to <8 x i32>
+; Z15:   Cost Model: Found an estimated cost of 2 for instruction:   %v45 = fptosi <8 x float> undef to <8 x i32>
 ; CHECK: Cost Model: Found an estimated cost of 24 for instruction:   %v46 = fptosi <8 x float> undef to <8 x i16>
 ; CHECK: Cost Model: Found an estimated cost of 24 for instruction:   %v47 = fptosi <8 x float> undef to <8 x i8>
 ; CHECK: Cost Model: Found an estimated cost of 8 for instruction:   %v48 = fptosi <16 x double> undef to <16 x i64>
@@ -153,7 +153,7 @@ define void @fptosi() {
 ; CHECK: Cost Model: Found an estimated cost of 48 for instruction:   %v51 = fptosi <16 x double> undef to <16 x i8>
 ; CHECK: Cost Model: Found an estimated cost of 40 for instruction:   %v52 = fptosi <16 x float> undef to <16 x i64>
 ; Z13:   Cost Model: Found an estimated cost of 48 for instruction:   %v53 = fptosi <16 x float> undef to <16 x i32>
-; AR13:  Cost Model: Found an estimated cost of 4 for instruction:   %v53 = fptosi <16 x float> undef to <16 x i32>
+; Z15:   Cost Model: Found an estimated cost of 4 for instruction:   %v53 = fptosi <16 x float> undef to <16 x i32>
 ; CHECK: Cost Model: Found an estimated cost of 48 for instruction:   %v54 = fptosi <16 x float> undef to <16 x i16>
 ; CHECK: Cost Model: Found an estimated cost of 48 for instruction:   %v55 = fptosi <16 x float> undef to <16 x i8>
 
@@ -241,7 +241,7 @@ define void @fptoui() {
 ; CHECK: Cost Model: Found an estimated cost of 6 for instruction:   %v19 = fptoui <2 x double> undef to <2 x i8>
 ; CHECK: Cost Model: Found an estimated cost of 5 for instruction:   %v20 = fptoui <2 x float> undef to <2 x i64>
 ; Z13:   Cost Model: Found an estimated cost of 12 for instruction:   %v21 = fptoui <2 x float> undef to <2 x i32>
-; AR13:  Cost Model: Found an estimated cost of 1 for instruction:   %v21 = fptoui <2 x float> undef to <2 x i32>
+; Z15:   Cost Model: Found an estimated cost of 1 for instruction:   %v21 = fptoui <2 x float> undef to <2 x i32>
 ; CHECK: Cost Model: Found an estimated cost of 6 for instruction:   %v22 = fptoui <2 x float> undef to <2 x i16>
 ; CHECK: Cost Model: Found an estimated cost of 6 for instruction:   %v23 = fptoui <2 x float> undef to <2 x i8>
 ; CHECK: Cost Model: Found an estimated cost of 6 for instruction:   %v24 = fptoui <4 x fp128> undef to <4 x i64>
@@ -254,7 +254,7 @@ define void @fptoui() {
 ; CHECK: Cost Model: Found an estimated cost of 12 for instruction:   %v31 = fptoui <4 x double> undef to <4 x i8>
 ; CHECK: Cost Model: Found an estimated cost of 10 for instruction:   %v32 = fptoui <4 x float> undef to <4 x i64>
 ; Z13:   Cost Model: Found an estimated cost of 12 for instruction:   %v33 = fptoui <4 x float> undef to <4 x i32>
-; AR13:  Cost Model: Found an estimated cost of 1 for instruction:   %v33 = fptoui <4 x float> undef to <4 x i32>
+; Z15:   Cost Model: Found an estimated cost of 1 for instruction:   %v33 = fptoui <4 x float> undef to <4 x i32>
 ; CHECK: Cost Model: Found an estimated cost of 12 for instruction:   %v34 = fptoui <4 x float> undef to <4 x i16>
 ; CHECK: Cost Model: Found an estimated cost of 12 for instruction:   %v35 = fptoui <4 x float> undef to <4 x i8>
 ; CHECK: Cost Model: Found an estimated cost of 12 for instruction:   %v36 = fptoui <8 x fp128> undef to <8 x i64>
@@ -267,7 +267,7 @@ define void @fptoui() {
 ; CHECK: Cost Model: Found an estimated cost of 24 for instruction:   %v43 = fptoui <8 x double> undef to <8 x i8>
 ; CHECK: Cost Model: Found an estimated cost of 20 for instruction:   %v44 = fptoui <8 x float> undef to <8 x i64>
 ; Z13:   Cost Model: Found an estimated cost of 24 for instruction:   %v45 = fptoui <8 x float> undef to <8 x i32>
-; AR13:  Cost Model: Found an estimated cost of 2 for instruction:   %v45 = fptoui <8 x float> undef to <8 x i32>
+; Z15:   Cost Model: Found an estimated cost of 2 for instruction:   %v45 = fptoui <8 x float> undef to <8 x i32>
 ; CHECK: Cost Model: Found an estimated cost of 24 for instruction:   %v46 = fptoui <8 x float> undef to <8 x i16>
 ; CHECK: Cost Model: Found an estimated cost of 24 for instruction:   %v47 = fptoui <8 x float> undef to <8 x i8>
 ; CHECK: Cost Model: Found an estimated cost of 8 for instruction:   %v48 = fptoui <16 x double> undef to <16 x i64>
@@ -276,7 +276,7 @@ define void @fptoui() {
 ; CHECK: Cost Model: Found an estimated cost of 48 for instruction:   %v51 = fptoui <16 x double> undef to <16 x i8>
 ; CHECK: Cost Model: Found an estimated cost of 40 for instruction:   %v52 = fptoui <16 x float> undef to <16 x i64>
 ; Z13:   Cost Model: Found an estimated cost of 48 for instruction:   %v53 = fptoui <16 x float> undef to <16 x i32>
-; AR13:  Cost Model: Found an estimated cost of 4 for instruction:   %v53 = fptoui <16 x float> undef to <16 x i32>
+; Z15:   Cost Model: Found an estimated cost of 4 for instruction:   %v53 = fptoui <16 x float> undef to <16 x i32>
 ; CHECK: Cost Model: Found an estimated cost of 48 for instruction:   %v54 = fptoui <16 x float> undef to <16 x i16>
 ; CHECK: Cost Model: Found an estimated cost of 48 for instruction:   %v55 = fptoui <16 x float> undef to <16 x i8>
 
@@ -391,7 +391,7 @@ define void @sitofp() {
 ; CHECK: Cost Model: Found an estimated cost of 5 for instruction:   %v15 = sitofp <2 x i32> undef to <2 x fp128>
 ; CHECK: Cost Model: Found an estimated cost of 7 for instruction:   %v16 = sitofp <2 x i32> undef to <2 x double>
 ; Z13:   Cost Model: Found an estimated cost of 14 for instruction:   %v17 = sitofp <2 x i32> undef to <2 x float>
-; AR13:  Cost Model: Found an estimated cost of 1 for instruction:   %v17 = sitofp <2 x i32> undef to <2 x float>
+; Z15:   Cost Model: Found an estimated cost of 1 for instruction:   %v17 = sitofp <2 x i32> undef to <2 x float>
 ; CHECK: Cost Model: Found an estimated cost of 7 for instruction:   %v18 = sitofp <2 x i16> undef to <2 x fp128>
 ; CHECK: Cost Model: Found an estimated cost of 9 for instruction:   %v19 = sitofp <2 x i16> undef to <2 x double>
 ; CHECK: Cost Model: Found an estimated cost of 9 for instruction:   %v20 = sitofp <2 x i16> undef to <2 x float>
@@ -404,7 +404,7 @@ define void @sitofp() {
 ; CHECK: Cost Model: Found an estimated cost of 9 for instruction:   %v27 = sitofp <4 x i32> undef to <4 x fp128>
 ; CHECK: Cost Model: Found an estimated cost of 13 for instruction:   %v28 = sitofp <4 x i32> undef to <4 x double>
 ; Z13:   Cost Model: Found an estimated cost of 13 for instruction:   %v29 = sitofp <4 x i32> undef to <4 x float>
-; AR13:  Cost Model: Found an estimated cost of 1 for instruction:   %v29 = sitofp <4 x i32> undef to <4 x float>
+; Z15:   Cost Model: Found an estimated cost of 1 for instruction:   %v29 = sitofp <4 x i32> undef to <4 x float>
 ; CHECK: Cost Model: Found an estimated cost of 13 for instruction:   %v30 = sitofp <4 x i16> undef to <4 x fp128>
 ; CHECK: Cost Model: Found an estimated cost of 17 for instruction:   %v31 = sitofp <4 x i16> undef to <4 x double>
 ; CHECK: Cost Model: Found an estimated cost of 17 for instruction:   %v32 = sitofp <4 x i16> undef to <4 x float>
@@ -417,7 +417,7 @@ define void @sitofp() {
 ; CHECK: Cost Model: Found an estimated cost of 17 for instruction:   %v39 = sitofp <8 x i32> undef to <8 x fp128>
 ; CHECK: Cost Model: Found an estimated cost of 25 for instruction:   %v40 = sitofp <8 x i32> undef to <8 x double>
 ; Z13:   Cost Model: Found an estimated cost of 25 for instruction:   %v41 = sitofp <8 x i32> undef to <8 x float>
-; AR13:  Cost Model: Found an estimated cost of 2 for instruction:   %v41 = sitofp <8 x i32> undef to <8 x float>
+; Z15:   Cost Model: Found an estimated cost of 2 for instruction:   %v41 = sitofp <8 x i32> undef to <8 x float>
 ; CHECK: Cost Model: Found an estimated cost of 25 for instruction:   %v42 = sitofp <8 x i16> undef to <8 x fp128>
 ; CHECK: Cost Model: Found an estimated cost of 33 for instruction:   %v43 = sitofp <8 x i16> undef to <8 x double>
 ; CHECK: Cost Model: Found an estimated cost of 33 for instruction:   %v44 = sitofp <8 x i16> undef to <8 x float>
@@ -428,7 +428,7 @@ define void @sitofp() {
 ; CHECK: Cost Model: Found an estimated cost of 49 for instruction:   %v49 = sitofp <16 x i64> undef to <16 x float>
 ; CHECK: Cost Model: Found an estimated cost of 49 for instruction:   %v50 = sitofp <16 x i32> undef to <16 x double>
 ; Z13:   Cost Model: Found an estimated cost of 49 for instruction:   %v51 = sitofp <16 x i32> undef to <16 x float>
-; AR13:  Cost Model: Found an estimated cost of 4 for instruction:   %v51 = sitofp <16 x i32> undef to <16 x float>
+; Z15:   Cost Model: Found an estimated cost of 4 for instruction:   %v51 = sitofp <16 x i32> undef to <16 x float>
 ; CHECK: Cost Model: Found an estimated cost of 65 for instruction:   %v52 = sitofp <16 x i16> undef to <16 x double>
 ; CHECK: Cost Model: Found an estimated cost of 65 for instruction:   %v53 = sitofp <16 x i16> undef to <16 x float>
 ; CHECK: Cost Model: Found an estimated cost of 65 for instruction:   %v54 = sitofp <16 x i8> undef to <16 x double>
@@ -513,7 +513,7 @@ define void @uitofp() {
 ; CHECK: Cost Model: Found an estimated cost of 5 for instruction:   %v15 = uitofp <2 x i32> undef to <2 x fp128>
 ; CHECK: Cost Model: Found an estimated cost of 7 for instruction:   %v16 = uitofp <2 x i32> undef to <2 x double>
 ; Z13:   Cost Model: Found an estimated cost of 14 for instruction:   %v17 = uitofp <2 x i32> undef to <2 x float>
-; AR13:  Cost Model: Found an estimated cost of 1 for instruction:   %v17 = uitofp <2 x i32> undef to <2 x float>
+; Z15:   Cost Model: Found an estimated cost of 1 for instruction:   %v17 = uitofp <2 x i32> undef to <2 x float>
 ; CHECK: Cost Model: Found an estimated cost of 7 for instruction:   %v18 = uitofp <2 x i16> undef to <2 x fp128>
 ; CHECK: Cost Model: Found an estimated cost of 9 for instruction:   %v19 = uitofp <2 x i16> undef to <2 x double>
 ; CHECK: Cost Model: Found an estimated cost of 9 for instruction:   %v20 = uitofp <2 x i16> undef to <2 x float>
@@ -526,7 +526,7 @@ define void @uitofp() {
 ; CHECK: Cost Model: Found an estimated cost of 9 for instruction:   %v27 = uitofp <4 x i32> undef to <4 x fp128>
 ; CHECK: Cost Model: Found an estimated cost of 13 for instruction:   %v28 = uitofp <4 x i32> undef to <4 x double>
 ; Z13:   Cost Model: Found an estimated cost of 13 for instruction:   %v29 = uitofp <4 x i32> undef to <4 x float>
-; AR13:  Cost Model: Found an estimated cost of 1 for instruction:   %v29 = uitofp <4 x i32> undef to <4 x float>
+; Z15:   Cost Model: Found an estimated cost of 1 for instruction:   %v29 = uitofp <4 x i32> undef to <4 x float>
 ; CHECK: Cost Model: Found an estimated cost of 13 for instruction:   %v30 = uitofp <4 x i16> undef to <4 x fp128>
 ; CHECK: Cost Model: Found an estimated cost of 17 for instruction:   %v31 = uitofp <4 x i16> undef to <4 x double>
 ; CHECK: Cost Model: Found an estimated cost of 17 for instruction:   %v32 = uitofp <4 x i16> undef to <4 x float>
@@ -539,7 +539,7 @@ define void @uitofp() {
 ; CHECK: Cost Model: Found an estimated cost of 17 for instruction:   %v39 = uitofp <8 x i32> undef to <8 x fp128>
 ; CHECK: Cost Model: Found an estimated cost of 25 for instruction:   %v40 = uitofp <8 x i32> undef to <8 x double>
 ; Z13:   Cost Model: Found an estimated cost of 25 for instruction:   %v41 = uitofp <8 x i32> undef to <8 x float>
-; AR13:  Cost Model: Found an estimated cost of 2 for instruction:   %v41 = uitofp <8 x i32> undef to <8 x float>
+; Z15:   Cost Model: Found an estimated cost of 2 for instruction:   %v41 = uitofp <8 x i32> undef to <8 x float>
 ; CHECK: Cost Model: Found an estimated cost of 25 for instruction:   %v42 = uitofp <8 x i16> undef to <8 x fp128>
 ; CHECK: Cost Model: Found an estimated cost of 33 for instruction:   %v43 = uitofp <8 x i16> undef to <8 x double>
 ; CHECK: Cost Model: Found an estimated cost of 33 for instruction:   %v44 = uitofp <8 x i16> undef to <8 x float>
@@ -550,7 +550,7 @@ define void @uitofp() {
 ; CHECK: Cost Model: Found an estimated cost of 49 for instruction:   %v49 = uitofp <16 x i64> undef to <16 x float>
 ; CHECK: Cost Model: Found an estimated cost of 49 for instruction:   %v50 = uitofp <16 x i32> undef to <16 x double>
 ; Z13:   Cost Model: Found an estimated cost of 49 for instruction:   %v51 = uitofp <16 x i32> undef to <16 x float>
-; AR13:  Cost Model: Found an estimated cost of 4 for instruction:   %v51 = uitofp <16 x i32> undef to <16 x float>
+; Z15:   Cost Model: Found an estimated cost of 4 for instruction:   %v51 = uitofp <16 x i32> undef to <16 x float>
 ; CHECK: Cost Model: Found an estimated cost of 65 for instruction:   %v52 = uitofp <16 x i16> undef to <16 x double>
 ; CHECK: Cost Model: Found an estimated cost of 65 for instruction:   %v53 = uitofp <16 x i16> undef to <16 x float>
 ; CHECK: Cost Model: Found an estimated cost of 65 for instruction:   %v54 = uitofp <16 x i8> undef to <16 x double>
diff --git a/llvm/test/Analysis/CostModel/SystemZ/intrinsic-cost-crash.ll b/llvm/test/Analysis/CostModel/SystemZ/intrinsic-cost-crash.ll
index ff5b2a205..866bd9e4b 100644
--- a/llvm/test/Analysis/CostModel/SystemZ/intrinsic-cost-crash.ll
+++ b/llvm/test/Analysis/CostModel/SystemZ/intrinsic-cost-crash.ll
@@ -49,7 +49,7 @@ for.body:                                         ; preds = %for.body, %for.body
 }
 
 attributes #0 = { argmemonly nounwind }
-attributes #1 = { nounwind ssp uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+fxsr,+mmx,+sse,+sse2,+sse3,+sse4.1,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind ssp uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+fxsr,+mmx,+sse,+sse2,+sse3,+sse4.1,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.ident = !{!0}
 
diff --git a/llvm/test/Analysis/CostModel/SystemZ/intrinsics.ll b/llvm/test/Analysis/CostModel/SystemZ/intrinsics.ll
index bbde62783..4bc050830 100644
--- a/llvm/test/Analysis/CostModel/SystemZ/intrinsics.ll
+++ b/llvm/test/Analysis/CostModel/SystemZ/intrinsics.ll
@@ -1,7 +1,7 @@
 ; RUN: opt < %s -cost-model -analyze -mtriple=systemz-unknown -mcpu=z13 \
 ; RUN:  | FileCheck %s -check-prefixes=CHECK,Z13
-; RUN: opt < %s -cost-model -analyze -mtriple=systemz-unknown -mcpu=arch13 \
-; RUN:  | FileCheck %s -check-prefixes=CHECK,AR13
+; RUN: opt < %s -cost-model -analyze -mtriple=systemz-unknown -mcpu=z15 \
+; RUN:  | FileCheck %s -check-prefixes=CHECK,Z15
 
 define void @bswap_i64(i64 %arg, <2 x i64> %arg2) {
 ; CHECK: Printing analysis 'Cost Model Analysis' for function 'bswap_i64':
@@ -69,15 +69,15 @@ define void @bswap_i64_mem(i64* %src, i64 %arg, i64* %dst) {
 define void @bswap_v2i64_mem(<2 x i64>* %src, <2 x i64> %arg, <2 x i64>* %dst) {
 ; CHECK:Printing analysis 'Cost Model Analysis' for function 'bswap_v2i64_mem':
 ; Z13:   Cost Model: Found an estimated cost of 1 for instruction:   %Ld1 = load <2 x i64>, <2 x i64>* %src
-; AR13:  Cost Model: Found an estimated cost of 0 for instruction:   %Ld1 = load <2 x i64>, <2 x i64>* %src
+; Z15:   Cost Model: Found an estimated cost of 0 for instruction:   %Ld1 = load <2 x i64>, <2 x i64>* %src
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %swp1 = tail call <2 x i64> @llvm.bswap.v2i64(<2 x i64> %Ld1)
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %swp2 = tail call <2 x i64> @llvm.bswap.v2i64(<2 x i64> %arg)
 ; Z13:   Cost Model: Found an estimated cost of 1 for instruction:   store <2 x i64> %swp2, <2 x i64>* %dst
-; AR13:  Cost Model: Found an estimated cost of 0 for instruction:   store <2 x i64> %swp2, <2 x i64>* %dst
+; Z15:   Cost Model: Found an estimated cost of 0 for instruction:   store <2 x i64> %swp2, <2 x i64>* %dst
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %Ld2 = load <2 x i64>, <2 x i64>* %src
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %swp3 = tail call <2 x i64> @llvm.bswap.v2i64(<2 x i64> %Ld2)
 ; Z13:   Cost Model: Found an estimated cost of 1 for instruction:   store <2 x i64> %swp3, <2 x i64>* %dst
-; AR13:  Cost Model: Found an estimated cost of 0 for instruction:   store <2 x i64> %swp3, <2 x i64>* %dst
+; Z15:   Cost Model: Found an estimated cost of 0 for instruction:   store <2 x i64> %swp3, <2 x i64>* %dst
 
   %Ld1  = load <2 x i64>, <2 x i64>* %src
   %swp1 = tail call <2 x i64> @llvm.bswap.v2i64(<2 x i64> %Ld1)
@@ -117,15 +117,15 @@ define void @bswap_i32_mem(i32* %src, i32 %arg, i32* %dst) {
 define void @bswap_v4i32_mem(<4 x i32>* %src, <4 x i32> %arg, <4 x i32>* %dst) {
 ; CHECK: Printing analysis 'Cost Model Analysis' for function 'bswap_v4i32_mem':
 ; Z13:   Cost Model: Found an estimated cost of 1 for instruction:   %Ld1 = load <4 x i32>, <4 x i32>* %src
-; AR13:  Cost Model: Found an estimated cost of 0 for instruction:   %Ld1 = load <4 x i32>, <4 x i32>* %src
+; Z15:   Cost Model: Found an estimated cost of 0 for instruction:   %Ld1 = load <4 x i32>, <4 x i32>* %src
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %swp1 = tail call <4 x i32> @llvm.bswap.v4i32(<4 x i32> %Ld1)
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %swp2 = tail call <4 x i32> @llvm.bswap.v4i32(<4 x i32> %arg)
 ; Z13:   Cost Model: Found an estimated cost of 1 for instruction:   store <4 x i32> %swp2, <4 x i32>* %dst
-; AR13:  Cost Model: Found an estimated cost of 0 for instruction:   store <4 x i32> %swp2, <4 x i32>* %dst
+; Z15:   Cost Model: Found an estimated cost of 0 for instruction:   store <4 x i32> %swp2, <4 x i32>* %dst
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %Ld2 = load <4 x i32>, <4 x i32>* %src
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %swp3 = tail call <4 x i32> @llvm.bswap.v4i32(<4 x i32> %Ld2)
 ; Z13:   Cost Model: Found an estimated cost of 1 for instruction:   store <4 x i32> %swp3, <4 x i32>* %dst
-; AR13:  Cost Model: Found an estimated cost of 0 for instruction:   store <4 x i32> %swp3, <4 x i32>* %dst
+; Z15:   Cost Model: Found an estimated cost of 0 for instruction:   store <4 x i32> %swp3, <4 x i32>* %dst
 %Ld1  = load <4 x i32>, <4 x i32>* %src
   %swp1 = tail call <4 x i32> @llvm.bswap.v4i32(<4 x i32> %Ld1)
 
@@ -164,15 +164,15 @@ define void @bswap_i16_mem(i16* %src, i16 %arg, i16* %dst) {
 define void @bswap_v8i16_mem(<8 x i16>* %src, <8 x i16> %arg, <8 x i16>* %dst) {
 ; CHECK: Printing analysis 'Cost Model Analysis' for function 'bswap_v8i16_mem':
 ; Z13:   Cost Model: Found an estimated cost of 1 for instruction:   %Ld1 = load <8 x i16>, <8 x i16>* %src
-; AR13:  Cost Model: Found an estimated cost of 0 for instruction:   %Ld1 = load <8 x i16>, <8 x i16>* %src
+; Z15:   Cost Model: Found an estimated cost of 0 for instruction:   %Ld1 = load <8 x i16>, <8 x i16>* %src
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %swp1 = tail call <8 x i16> @llvm.bswap.v8i16(<8 x i16> %Ld1)
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %swp2 = tail call <8 x i16> @llvm.bswap.v8i16(<8 x i16> %arg)
 ; Z13:   Cost Model: Found an estimated cost of 1 for instruction:   store <8 x i16> %swp2, <8 x i16>* %dst
-; AR13:  Cost Model: Found an estimated cost of 0 for instruction:   store <8 x i16> %swp2, <8 x i16>* %dst
+; Z15:   Cost Model: Found an estimated cost of 0 for instruction:   store <8 x i16> %swp2, <8 x i16>* %dst
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %Ld2 = load <8 x i16>, <8 x i16>* %src
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %swp3 = tail call <8 x i16> @llvm.bswap.v8i16(<8 x i16> %Ld2)
 ; Z13:   Cost Model: Found an estimated cost of 1 for instruction:   store <8 x i16> %swp3, <8 x i16>* %dst
-; AR13:  Cost Model: Found an estimated cost of 0 for instruction:   store <8 x i16> %swp3, <8 x i16>* %dst
+; Z15:   Cost Model: Found an estimated cost of 0 for instruction:   store <8 x i16> %swp3, <8 x i16>* %dst
 %Ld1  = load <8 x i16>, <8 x i16>* %src
   %swp1 = tail call <8 x i16> @llvm.bswap.v8i16(<8 x i16> %Ld1)
 
diff --git a/llvm/test/Analysis/CostModel/SystemZ/logic-miscext3.ll b/llvm/test/Analysis/CostModel/SystemZ/logic-miscext3.ll
index 86706309e..f3b915d0a 100644
--- a/llvm/test/Analysis/CostModel/SystemZ/logic-miscext3.ll
+++ b/llvm/test/Analysis/CostModel/SystemZ/logic-miscext3.ll
@@ -1,25 +1,25 @@
 ; RUN: opt < %s -cost-model -analyze -mtriple=systemz-unknown -mcpu=z13 \
 ; RUN:  | FileCheck %s -check-prefixes=CHECK,Z13
-; RUN: opt < %s -cost-model -analyze -mtriple=systemz-unknown -mcpu=arch13 \
-; RUN:  | FileCheck %s -check-prefixes=CHECK,AR13
+; RUN: opt < %s -cost-model -analyze -mtriple=systemz-unknown -mcpu=z15 \
+; RUN:  | FileCheck %s -check-prefixes=CHECK,Z15
 
 define void @fun0(i32 %a)  {
 ; CHECK-LABEL: Printing analysis 'Cost Model Analysis' for function 'fun0':
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %c0 = xor i32 %l0, -1
 ; Z13:   Cost Model: Found an estimated cost of 1 for instruction:   %res0 = or i32 %a, %c0
-; AR13:  Cost Model: Found an estimated cost of 0 for instruction:   %res0 = or i32 %a, %c0
+; Z15:   Cost Model: Found an estimated cost of 0 for instruction:   %res0 = or i32 %a, %c0
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %c1 = xor i32 %l1, -1
 ; Z13:   Cost Model: Found an estimated cost of 1 for instruction:   %res1 = and i32 %a, %c1
-; AR13:  Cost Model: Found an estimated cost of 0 for instruction:   %res1 = and i32 %a, %c1
+; Z15:   Cost Model: Found an estimated cost of 0 for instruction:   %res1 = and i32 %a, %c1
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %c2 = and i32 %l2, %a
 ; Z13:   Cost Model: Found an estimated cost of 1 for instruction:   %res2 = xor i32 %c2, -1
-; AR13:  Cost Model: Found an estimated cost of 0 for instruction:   %res2 = xor i32 %c2, -1
+; Z15:   Cost Model: Found an estimated cost of 0 for instruction:   %res2 = xor i32 %c2, -1
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %c3 = or i32 %l3, %a
 ; Z13:   Cost Model: Found an estimated cost of 1 for instruction:   %res3 = xor i32 %c3, -1
-; AR13:  Cost Model: Found an estimated cost of 0 for instruction:   %res3 = xor i32 %c3, -1
+; Z15:   Cost Model: Found an estimated cost of 0 for instruction:   %res3 = xor i32 %c3, -1
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %c4 = xor i32 %l4, %a
 ; Z13:   Cost Model: Found an estimated cost of 1 for instruction:   %res4 = xor i32 %c4, -1
-; AR13:  Cost Model: Found an estimated cost of 0 for instruction:   %res4 = xor i32 %c4, -1
+; Z15:   Cost Model: Found an estimated cost of 0 for instruction:   %res4 = xor i32 %c4, -1
 
 entry:
   %l0 = load i32, i32* undef
@@ -54,19 +54,19 @@ define void @fun1(i64 %a)  {
 ; CHECK-LABEL: Printing analysis 'Cost Model Analysis' for function 'fun1':
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %c0 = xor i64 %l0, -1
 ; Z13:   Cost Model: Found an estimated cost of 1 for instruction:   %res0 = or i64 %a, %c0
-; AR13:  Cost Model: Found an estimated cost of 0 for instruction:   %res0 = or i64 %a, %c0
+; Z15:   Cost Model: Found an estimated cost of 0 for instruction:   %res0 = or i64 %a, %c0
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %c1 = xor i64 %l1, -1
 ; Z13:   Cost Model: Found an estimated cost of 1 for instruction:   %res1 = and i64 %a, %c1
-; AR13:  Cost Model: Found an estimated cost of 0 for instruction:   %res1 = and i64 %a, %c1
+; Z15:   Cost Model: Found an estimated cost of 0 for instruction:   %res1 = and i64 %a, %c1
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %c2 = and i64 %l2, %a
 ; Z13:   Cost Model: Found an estimated cost of 1 for instruction:   %res2 = xor i64 %c2, -1
-; AR13:  Cost Model: Found an estimated cost of 0 for instruction:   %res2 = xor i64 %c2, -1
+; Z15:   Cost Model: Found an estimated cost of 0 for instruction:   %res2 = xor i64 %c2, -1
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %c3 = or i64 %l3, %a
 ; Z13:   Cost Model: Found an estimated cost of 1 for instruction:   %res3 = xor i64 %c3, -1
-; AR13:  Cost Model: Found an estimated cost of 0 for instruction:   %res3 = xor i64 %c3, -1
+; Z15:   Cost Model: Found an estimated cost of 0 for instruction:   %res3 = xor i64 %c3, -1
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %c4 = xor i64 %l4, %a
 ; Z13:   Cost Model: Found an estimated cost of 1 for instruction:   %res4 = xor i64 %c4, -1
-; AR13:  Cost Model: Found an estimated cost of 0 for instruction:   %res4 = xor i64 %c4, -1
+; Z15:   Cost Model: Found an estimated cost of 0 for instruction:   %res4 = xor i64 %c4, -1
 entry:
   %l0 = load i64, i64* undef
   %c0 = xor i64 %l0, -1
diff --git a/llvm/test/Analysis/CostModel/X86/aggregates.ll b/llvm/test/Analysis/CostModel/X86/aggregates.ll
new file mode 100644
index 000000000..3fd97d8bf
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/X86/aggregates.ll
@@ -0,0 +1,142 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -cost-kind=throughput -analyze | FileCheck %s --check-prefixes=ALL,THROUGHPUT
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -cost-kind=latency -analyze | FileCheck %s --check-prefixes=ALL,LATENCY
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -cost-kind=code-size -analyze | FileCheck %s --check-prefixes=ALL,CODESIZE
+
+define i32 @extract_first_i32({i32, i32} %agg) {
+; THROUGHPUT-LABEL: 'extract_first_i32'
+; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, i32 } %agg, 0
+; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %r
+;
+; LATENCY-LABEL: 'extract_first_i32'
+; LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, i32 } %agg, 0
+; LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %r
+;
+; CODESIZE-LABEL: 'extract_first_i32'
+; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, i32 } %agg, 0
+; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %r
+;
+  %r = extractvalue {i32, i32} %agg, 0
+  ret i32 %r
+}
+
+define i32 @extract_second_i32({i32, i32} %agg) {
+; THROUGHPUT-LABEL: 'extract_second_i32'
+; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, i32 } %agg, 1
+; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %r
+;
+; LATENCY-LABEL: 'extract_second_i32'
+; LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, i32 } %agg, 1
+; LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %r
+;
+; CODESIZE-LABEL: 'extract_second_i32'
+; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, i32 } %agg, 1
+; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %r
+;
+  %r = extractvalue {i32, i32} %agg, 1
+  ret i32 %r
+}
+
+define i32 @extract_i32({i32, i1} %agg) {
+; THROUGHPUT-LABEL: 'extract_i32'
+; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, i1 } %agg, 0
+; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %r
+;
+; LATENCY-LABEL: 'extract_i32'
+; LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, i1 } %agg, 0
+; LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %r
+;
+; CODESIZE-LABEL: 'extract_i32'
+; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, i1 } %agg, 0
+; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %r
+;
+  %r = extractvalue {i32, i1} %agg, 0
+  ret i32 %r
+}
+
+define i1 @extract_i1({i32, i1} %agg) {
+; THROUGHPUT-LABEL: 'extract_i1'
+; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, i1 } %agg, 1
+; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i1 %r
+;
+; LATENCY-LABEL: 'extract_i1'
+; LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, i1 } %agg, 1
+; LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i1 %r
+;
+; CODESIZE-LABEL: 'extract_i1'
+; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, i1 } %agg, 1
+; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i1 %r
+;
+  %r = extractvalue {i32, i1} %agg, 1
+  ret i1 %r
+}
+
+define float @extract_float({i32, float} %agg) {
+; THROUGHPUT-LABEL: 'extract_float'
+; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, float } %agg, 1
+; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r
+;
+; LATENCY-LABEL: 'extract_float'
+; LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, float } %agg, 1
+; LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret float %r
+;
+; CODESIZE-LABEL: 'extract_float'
+; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, float } %agg, 1
+; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret float %r
+;
+  %r = extractvalue {i32, float} %agg, 1
+  ret float %r
+}
+
+define [42 x i42] @extract_array({i32, [42 x i42]} %agg) {
+; THROUGHPUT-LABEL: 'extract_array'
+; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, [42 x i42] } %agg, 1
+; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret [42 x i42] %r
+;
+; LATENCY-LABEL: 'extract_array'
+; LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, [42 x i42] } %agg, 1
+; LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret [42 x i42] %r
+;
+; CODESIZE-LABEL: 'extract_array'
+; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, [42 x i42] } %agg, 1
+; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret [42 x i42] %r
+;
+  %r = extractvalue {i32, [42 x i42]} %agg, 1
+  ret [42 x i42] %r
+}
+
+define <42 x i42> @extract_vector({i32, <42 x i42>} %agg) {
+; THROUGHPUT-LABEL: 'extract_vector'
+; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, <42 x i42> } %agg, 1
+; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <42 x i42> %r
+;
+; LATENCY-LABEL: 'extract_vector'
+; LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, <42 x i42> } %agg, 1
+; LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <42 x i42> %r
+;
+; CODESIZE-LABEL: 'extract_vector'
+; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, <42 x i42> } %agg, 1
+; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <42 x i42> %r
+;
+  %r = extractvalue {i32, <42 x i42>} %agg, 1
+  ret <42 x i42> %r
+}
+
+%T1 = type { i32, float, <4 x i1> }
+
+define %T1 @extract_struct({i32, %T1} %agg) {
+; THROUGHPUT-LABEL: 'extract_struct'
+; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, %T1 } %agg, 1
+; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret %T1 %r
+;
+; LATENCY-LABEL: 'extract_struct'
+; LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, %T1 } %agg, 1
+; LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret %T1 %r
+;
+; CODESIZE-LABEL: 'extract_struct'
+; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, %T1 } %agg, 1
+; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret %T1 %r
+;
+  %r = extractvalue {i32, %T1} %agg, 1
+  ret %T1 %r
+}
diff --git a/llvm/test/Analysis/CostModel/X86/alternate-shuffle-cost.ll b/llvm/test/Analysis/CostModel/X86/alternate-shuffle-cost.ll
index 42204e1ee..9a8fc259e 100644
--- a/llvm/test/Analysis/CostModel/X86/alternate-shuffle-cost.ll
+++ b/llvm/test/Analysis/CostModel/X86/alternate-shuffle-cost.ll
@@ -18,9 +18,21 @@
 ; 64-bit packed float vectors (v2f32) are widened to type v4f32.
 
 define <2 x i32> @test_v2i32(<2 x i32> %a, <2 x i32> %b) {
-; CHECK-LABEL: 'test_v2i32'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = shufflevector <2 x i32> %a, <2 x i32> %b, <2 x i32> <i32 0, i32 3>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i32> %1
+; SSE2-LABEL: 'test_v2i32'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %1 = shufflevector <2 x i32> %a, <2 x i32> %b, <2 x i32> <i32 0, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i32> %1
+;
+; SSSE3-LABEL: 'test_v2i32'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %1 = shufflevector <2 x i32> %a, <2 x i32> %b, <2 x i32> <i32 0, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i32> %1
+;
+; SSE42-LABEL: 'test_v2i32'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = shufflevector <2 x i32> %a, <2 x i32> %b, <2 x i32> <i32 0, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i32> %1
+;
+; AVX-LABEL: 'test_v2i32'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = shufflevector <2 x i32> %a, <2 x i32> %b, <2 x i32> <i32 0, i32 3>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i32> %1
 ;
 ; BTVER2-LABEL: 'test_v2i32'
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = shufflevector <2 x i32> %a, <2 x i32> %b, <2 x i32> <i32 0, i32 3>
@@ -56,9 +68,21 @@ define <2 x float> @test_v2f32(<2 x float> %a, <2 x float> %b) {
 }
 
 define <2 x i32> @test_v2i32_2(<2 x i32> %a, <2 x i32> %b) {
-; CHECK-LABEL: 'test_v2i32_2'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = shufflevector <2 x i32> %a, <2 x i32> %b, <2 x i32> <i32 2, i32 1>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i32> %1
+; SSE2-LABEL: 'test_v2i32_2'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %1 = shufflevector <2 x i32> %a, <2 x i32> %b, <2 x i32> <i32 2, i32 1>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i32> %1
+;
+; SSSE3-LABEL: 'test_v2i32_2'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %1 = shufflevector <2 x i32> %a, <2 x i32> %b, <2 x i32> <i32 2, i32 1>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i32> %1
+;
+; SSE42-LABEL: 'test_v2i32_2'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = shufflevector <2 x i32> %a, <2 x i32> %b, <2 x i32> <i32 2, i32 1>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i32> %1
+;
+; AVX-LABEL: 'test_v2i32_2'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = shufflevector <2 x i32> %a, <2 x i32> %b, <2 x i32> <i32 2, i32 1>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i32> %1
 ;
 ; BTVER2-LABEL: 'test_v2i32_2'
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = shufflevector <2 x i32> %a, <2 x i32> %b, <2 x i32> <i32 2, i32 1>
diff --git a/llvm/test/Analysis/CostModel/X86/arith-fix.ll b/llvm/test/Analysis/CostModel/X86/arith-fix.ll
index b65ddb630..b72ca1c1e 100644
--- a/llvm/test/Analysis/CostModel/X86/arith-fix.ll
+++ b/llvm/test/Analysis/CostModel/X86/arith-fix.ll
@@ -38,9 +38,9 @@ define i32 @smul(i32 %arg) {
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V4I64 = call <4 x i64> @llvm.smul.fix.v4i64(<4 x i64> undef, <4 x i64> undef, i32 3)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 88 for instruction: %V8I64 = call <8 x i64> @llvm.smul.fix.v8i64(<8 x i64> undef, <8 x i64> undef, i32 3)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %I32 = call i32 @llvm.smul.fix.i32(i32 undef, i32 undef, i32 3)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V4I32 = call <4 x i32> @llvm.smul.fix.v4i32(<4 x i32> undef, <4 x i32> undef, i32 3)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %V8I32 = call <8 x i32> @llvm.smul.fix.v8i32(<8 x i32> undef, <8 x i32> undef, i32 3)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 136 for instruction: %V16I32 = call <16 x i32> @llvm.smul.fix.v16i32(<16 x i32> undef, <16 x i32> undef, i32 3)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V4I32 = call <4 x i32> @llvm.smul.fix.v4i32(<4 x i32> undef, <4 x i32> undef, i32 3)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 74 for instruction: %V8I32 = call <8 x i32> @llvm.smul.fix.v8i32(<8 x i32> undef, <8 x i32> undef, i32 3)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 152 for instruction: %V16I32 = call <16 x i32> @llvm.smul.fix.v16i32(<16 x i32> undef, <16 x i32> undef, i32 3)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %I16 = call i16 @llvm.smul.fix.i16(i16 undef, i16 undef, i32 3)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V8I16 = call <8 x i16> @llvm.smul.fix.v8i16(<8 x i16> undef, <8 x i16> undef, i32 3)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %V16I16 = call <16 x i16> @llvm.smul.fix.v16i16(<16 x i16> undef, <16 x i16> undef, i32 3)
@@ -57,9 +57,9 @@ define i32 @smul(i32 %arg) {
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V4I64 = call <4 x i64> @llvm.smul.fix.v4i64(<4 x i64> undef, <4 x i64> undef, i32 3)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 88 for instruction: %V8I64 = call <8 x i64> @llvm.smul.fix.v8i64(<8 x i64> undef, <8 x i64> undef, i32 3)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %I32 = call i32 @llvm.smul.fix.i32(i32 undef, i32 undef, i32 3)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V4I32 = call <4 x i32> @llvm.smul.fix.v4i32(<4 x i32> undef, <4 x i32> undef, i32 3)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V8I32 = call <8 x i32> @llvm.smul.fix.v8i32(<8 x i32> undef, <8 x i32> undef, i32 3)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 112 for instruction: %V16I32 = call <16 x i32> @llvm.smul.fix.v16i32(<16 x i32> undef, <16 x i32> undef, i32 3)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V4I32 = call <4 x i32> @llvm.smul.fix.v4i32(<4 x i32> undef, <4 x i32> undef, i32 3)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 62 for instruction: %V8I32 = call <8 x i32> @llvm.smul.fix.v8i32(<8 x i32> undef, <8 x i32> undef, i32 3)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %V16I32 = call <16 x i32> @llvm.smul.fix.v16i32(<16 x i32> undef, <16 x i32> undef, i32 3)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %I16 = call i16 @llvm.smul.fix.i16(i16 undef, i16 undef, i32 3)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I16 = call <8 x i16> @llvm.smul.fix.v8i16(<8 x i16> undef, <8 x i16> undef, i32 3)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V16I16 = call <16 x i16> @llvm.smul.fix.v16i16(<16 x i16> undef, <16 x i16> undef, i32 3)
@@ -171,9 +171,9 @@ define i32 @smul(i32 %arg) {
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V4I64 = call <4 x i64> @llvm.smul.fix.v4i64(<4 x i64> undef, <4 x i64> undef, i32 3)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 88 for instruction: %V8I64 = call <8 x i64> @llvm.smul.fix.v8i64(<8 x i64> undef, <8 x i64> undef, i32 3)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %I32 = call i32 @llvm.smul.fix.i32(i32 undef, i32 undef, i32 3)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %V4I32 = call <4 x i32> @llvm.smul.fix.v4i32(<4 x i32> undef, <4 x i32> undef, i32 3)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 90 for instruction: %V8I32 = call <8 x i32> @llvm.smul.fix.v8i32(<8 x i32> undef, <8 x i32> undef, i32 3)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 184 for instruction: %V16I32 = call <16 x i32> @llvm.smul.fix.v16i32(<16 x i32> undef, <16 x i32> undef, i32 3)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %V4I32 = call <4 x i32> @llvm.smul.fix.v4i32(<4 x i32> undef, <4 x i32> undef, i32 3)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V8I32 = call <8 x i32> @llvm.smul.fix.v8i32(<8 x i32> undef, <8 x i32> undef, i32 3)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V16I32 = call <16 x i32> @llvm.smul.fix.v16i32(<16 x i32> undef, <16 x i32> undef, i32 3)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %I16 = call i16 @llvm.smul.fix.i16(i16 undef, i16 undef, i32 3)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V8I16 = call <8 x i16> @llvm.smul.fix.v8i16(<8 x i16> undef, <8 x i16> undef, i32 3)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %V16I16 = call <16 x i16> @llvm.smul.fix.v16i16(<16 x i16> undef, <16 x i16> undef, i32 3)
@@ -190,9 +190,9 @@ define i32 @smul(i32 %arg) {
 ; GLM-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V4I64 = call <4 x i64> @llvm.smul.fix.v4i64(<4 x i64> undef, <4 x i64> undef, i32 3)
 ; GLM-NEXT:  Cost Model: Found an estimated cost of 88 for instruction: %V8I64 = call <8 x i64> @llvm.smul.fix.v8i64(<8 x i64> undef, <8 x i64> undef, i32 3)
 ; GLM-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %I32 = call i32 @llvm.smul.fix.i32(i32 undef, i32 undef, i32 3)
-; GLM-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V4I32 = call <4 x i32> @llvm.smul.fix.v4i32(<4 x i32> undef, <4 x i32> undef, i32 3)
-; GLM-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V8I32 = call <8 x i32> @llvm.smul.fix.v8i32(<8 x i32> undef, <8 x i32> undef, i32 3)
-; GLM-NEXT:  Cost Model: Found an estimated cost of 112 for instruction: %V16I32 = call <16 x i32> @llvm.smul.fix.v16i32(<16 x i32> undef, <16 x i32> undef, i32 3)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V4I32 = call <4 x i32> @llvm.smul.fix.v4i32(<4 x i32> undef, <4 x i32> undef, i32 3)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 62 for instruction: %V8I32 = call <8 x i32> @llvm.smul.fix.v8i32(<8 x i32> undef, <8 x i32> undef, i32 3)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %V16I32 = call <16 x i32> @llvm.smul.fix.v16i32(<16 x i32> undef, <16 x i32> undef, i32 3)
 ; GLM-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %I16 = call i16 @llvm.smul.fix.i16(i16 undef, i16 undef, i32 3)
 ; GLM-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I16 = call <8 x i16> @llvm.smul.fix.v8i16(<8 x i16> undef, <8 x i16> undef, i32 3)
 ; GLM-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V16I16 = call <16 x i16> @llvm.smul.fix.v16i16(<16 x i16> undef, <16 x i16> undef, i32 3)
@@ -272,9 +272,9 @@ define i32 @umul(i32 %arg) {
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V4I64 = call <4 x i64> @llvm.umul.fix.v4i64(<4 x i64> undef, <4 x i64> undef, i32 3)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 88 for instruction: %V8I64 = call <8 x i64> @llvm.umul.fix.v8i64(<8 x i64> undef, <8 x i64> undef, i32 3)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.umul.fix.i32(i32 undef, i32 undef, i32 3)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %V4I32 = call <4 x i32> @llvm.umul.fix.v4i32(<4 x i32> undef, <4 x i32> undef, i32 3)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 58 for instruction: %V8I32 = call <8 x i32> @llvm.umul.fix.v8i32(<8 x i32> undef, <8 x i32> undef, i32 3)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 120 for instruction: %V16I32 = call <16 x i32> @llvm.umul.fix.v16i32(<16 x i32> undef, <16 x i32> undef, i32 3)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V4I32 = call <4 x i32> @llvm.umul.fix.v4i32(<4 x i32> undef, <4 x i32> undef, i32 3)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %V8I32 = call <8 x i32> @llvm.umul.fix.v8i32(<8 x i32> undef, <8 x i32> undef, i32 3)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 136 for instruction: %V16I32 = call <16 x i32> @llvm.umul.fix.v16i32(<16 x i32> undef, <16 x i32> undef, i32 3)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %I16 = call i16 @llvm.umul.fix.i16(i16 undef, i16 undef, i32 3)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V8I16 = call <8 x i16> @llvm.umul.fix.v8i16(<8 x i16> undef, <8 x i16> undef, i32 3)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 62 for instruction: %V16I16 = call <16 x i16> @llvm.umul.fix.v16i16(<16 x i16> undef, <16 x i16> undef, i32 3)
@@ -291,9 +291,9 @@ define i32 @umul(i32 %arg) {
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V4I64 = call <4 x i64> @llvm.umul.fix.v4i64(<4 x i64> undef, <4 x i64> undef, i32 3)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 88 for instruction: %V8I64 = call <8 x i64> @llvm.umul.fix.v8i64(<8 x i64> undef, <8 x i64> undef, i32 3)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.umul.fix.i32(i32 undef, i32 undef, i32 3)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V4I32 = call <4 x i32> @llvm.umul.fix.v4i32(<4 x i32> undef, <4 x i32> undef, i32 3)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V8I32 = call <8 x i32> @llvm.umul.fix.v8i32(<8 x i32> undef, <8 x i32> undef, i32 3)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 112 for instruction: %V16I32 = call <16 x i32> @llvm.umul.fix.v16i32(<16 x i32> undef, <16 x i32> undef, i32 3)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V4I32 = call <4 x i32> @llvm.umul.fix.v4i32(<4 x i32> undef, <4 x i32> undef, i32 3)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 62 for instruction: %V8I32 = call <8 x i32> @llvm.umul.fix.v8i32(<8 x i32> undef, <8 x i32> undef, i32 3)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %V16I32 = call <16 x i32> @llvm.umul.fix.v16i32(<16 x i32> undef, <16 x i32> undef, i32 3)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %I16 = call i16 @llvm.umul.fix.i16(i16 undef, i16 undef, i32 3)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I16 = call <8 x i16> @llvm.umul.fix.v8i16(<8 x i16> undef, <8 x i16> undef, i32 3)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V16I16 = call <16 x i16> @llvm.umul.fix.v16i16(<16 x i16> undef, <16 x i16> undef, i32 3)
@@ -405,9 +405,9 @@ define i32 @umul(i32 %arg) {
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V4I64 = call <4 x i64> @llvm.umul.fix.v4i64(<4 x i64> undef, <4 x i64> undef, i32 3)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 88 for instruction: %V8I64 = call <8 x i64> @llvm.umul.fix.v8i64(<8 x i64> undef, <8 x i64> undef, i32 3)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.umul.fix.i32(i32 undef, i32 undef, i32 3)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %V4I32 = call <4 x i32> @llvm.umul.fix.v4i32(<4 x i32> undef, <4 x i32> undef, i32 3)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 90 for instruction: %V8I32 = call <8 x i32> @llvm.umul.fix.v8i32(<8 x i32> undef, <8 x i32> undef, i32 3)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 184 for instruction: %V16I32 = call <16 x i32> @llvm.umul.fix.v16i32(<16 x i32> undef, <16 x i32> undef, i32 3)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %V4I32 = call <4 x i32> @llvm.umul.fix.v4i32(<4 x i32> undef, <4 x i32> undef, i32 3)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V8I32 = call <8 x i32> @llvm.umul.fix.v8i32(<8 x i32> undef, <8 x i32> undef, i32 3)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V16I32 = call <16 x i32> @llvm.umul.fix.v16i32(<16 x i32> undef, <16 x i32> undef, i32 3)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %I16 = call i16 @llvm.umul.fix.i16(i16 undef, i16 undef, i32 3)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V8I16 = call <8 x i16> @llvm.umul.fix.v8i16(<8 x i16> undef, <8 x i16> undef, i32 3)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %V16I16 = call <16 x i16> @llvm.umul.fix.v16i16(<16 x i16> undef, <16 x i16> undef, i32 3)
@@ -424,9 +424,9 @@ define i32 @umul(i32 %arg) {
 ; GLM-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V4I64 = call <4 x i64> @llvm.umul.fix.v4i64(<4 x i64> undef, <4 x i64> undef, i32 3)
 ; GLM-NEXT:  Cost Model: Found an estimated cost of 88 for instruction: %V8I64 = call <8 x i64> @llvm.umul.fix.v8i64(<8 x i64> undef, <8 x i64> undef, i32 3)
 ; GLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.umul.fix.i32(i32 undef, i32 undef, i32 3)
-; GLM-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V4I32 = call <4 x i32> @llvm.umul.fix.v4i32(<4 x i32> undef, <4 x i32> undef, i32 3)
-; GLM-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V8I32 = call <8 x i32> @llvm.umul.fix.v8i32(<8 x i32> undef, <8 x i32> undef, i32 3)
-; GLM-NEXT:  Cost Model: Found an estimated cost of 112 for instruction: %V16I32 = call <16 x i32> @llvm.umul.fix.v16i32(<16 x i32> undef, <16 x i32> undef, i32 3)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V4I32 = call <4 x i32> @llvm.umul.fix.v4i32(<4 x i32> undef, <4 x i32> undef, i32 3)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 62 for instruction: %V8I32 = call <8 x i32> @llvm.umul.fix.v8i32(<8 x i32> undef, <8 x i32> undef, i32 3)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %V16I32 = call <16 x i32> @llvm.umul.fix.v16i32(<16 x i32> undef, <16 x i32> undef, i32 3)
 ; GLM-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %I16 = call i16 @llvm.umul.fix.i16(i16 undef, i16 undef, i32 3)
 ; GLM-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I16 = call <8 x i16> @llvm.umul.fix.v8i16(<8 x i16> undef, <8 x i16> undef, i32 3)
 ; GLM-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V16I16 = call <16 x i16> @llvm.umul.fix.v16i16(<16 x i16> undef, <16 x i16> undef, i32 3)
diff --git a/llvm/test/Analysis/CostModel/X86/arith-overflow.ll b/llvm/test/Analysis/CostModel/X86/arith-overflow.ll
index b705ed901..5ab9b845e 100644
--- a/llvm/test/Analysis/CostModel/X86/arith-overflow.ll
+++ b/llvm/test/Analysis/CostModel/X86/arith-overflow.ll
@@ -171,9 +171,9 @@ define i32 @sadd(i32 %arg) {
 ;
 ; SLM-LABEL: 'sadd'
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call { i64, i1 } @llvm.sadd.with.overflow.i64(i64 undef, i64 undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.sadd.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.sadd.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.sadd.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.sadd.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.sadd.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.sadd.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 undef, i32 undef)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.sadd.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.sadd.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
@@ -409,9 +409,9 @@ define i32 @uadd(i32 %arg) {
 ;
 ; SLM-LABEL: 'uadd'
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 undef, i64 undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.uadd.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.uadd.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.uadd.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.uadd.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.uadd.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.uadd.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 undef, i32 undef)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.uadd.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.uadd.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
@@ -647,9 +647,9 @@ define i32 @ssub(i32 %arg) {
 ;
 ; SLM-LABEL: 'ssub'
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call { i64, i1 } @llvm.ssub.with.overflow.i64(i64 undef, i64 undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.ssub.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.ssub.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.ssub.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.ssub.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.ssub.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.ssub.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 undef, i32 undef)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.ssub.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.ssub.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
@@ -885,9 +885,9 @@ define i32 @usub(i32 %arg) {
 ;
 ; SLM-LABEL: 'usub'
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call { i64, i1 } @llvm.usub.with.overflow.i64(i64 undef, i64 undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.usub.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.usub.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.usub.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.usub.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.usub.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.usub.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call { i32, i1 } @llvm.usub.with.overflow.i32(i32 undef, i32 undef)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.usub.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.usub.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
@@ -994,9 +994,9 @@ define i32 @smul(i32 %arg) {
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 62 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.smul.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.smul.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = call { i32, i1 } @llvm.smul.with.overflow.i32(i32 undef, i32 undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.smul.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 58 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.smul.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 120 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.smul.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.smul.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.smul.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 136 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.smul.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %I16 = call { i16, i1 } @llvm.smul.with.overflow.i16(i16 undef, i16 undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.smul.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 62 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.smul.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
@@ -1013,9 +1013,9 @@ define i32 @smul(i32 %arg) {
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.smul.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.smul.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = call { i32, i1 } @llvm.smul.with.overflow.i32(i32 undef, i32 undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.smul.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.smul.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 112 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.smul.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.smul.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 62 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.smul.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.smul.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %I16 = call { i16, i1 } @llvm.smul.with.overflow.i16(i16 undef, i16 undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.smul.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.smul.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
@@ -1123,13 +1123,13 @@ define i32 @smul(i32 %arg) {
 ;
 ; SLM-LABEL: 'smul'
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %I64 = call { i64, i1 } @llvm.smul.with.overflow.i64(i64 undef, i64 undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.smul.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.smul.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.smul.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.smul.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.smul.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.smul.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = call { i32, i1 } @llvm.smul.with.overflow.i32(i32 undef, i32 undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.smul.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 90 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.smul.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 184 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.smul.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.smul.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.smul.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.smul.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %I16 = call { i16, i1 } @llvm.smul.with.overflow.i16(i16 undef, i16 undef)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.smul.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.smul.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
@@ -1146,9 +1146,9 @@ define i32 @smul(i32 %arg) {
 ; GLM-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.smul.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
 ; GLM-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.smul.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; GLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = call { i32, i1 } @llvm.smul.with.overflow.i32(i32 undef, i32 undef)
-; GLM-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.smul.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
-; GLM-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.smul.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
-; GLM-NEXT:  Cost Model: Found an estimated cost of 112 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.smul.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.smul.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 62 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.smul.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.smul.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; GLM-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %I16 = call { i16, i1 } @llvm.smul.with.overflow.i16(i16 undef, i16 undef)
 ; GLM-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.smul.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
 ; GLM-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.smul.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
@@ -1232,9 +1232,9 @@ define i32 @umul(i32 %arg) {
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.umul.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 112 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.umul.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I32 = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 undef, i32 undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.umul.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.umul.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 116 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.umul.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.umul.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.umul.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 132 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.umul.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %I16 = call { i16, i1 } @llvm.umul.with.overflow.i16(i16 undef, i16 undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.umul.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 60 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.umul.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
@@ -1251,9 +1251,9 @@ define i32 @umul(i32 %arg) {
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.umul.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 84 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.umul.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I32 = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 undef, i32 undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.umul.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.umul.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.umul.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.umul.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 60 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.umul.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 124 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.umul.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %I16 = call { i16, i1 } @llvm.umul.with.overflow.i16(i16 undef, i16 undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.umul.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.umul.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
@@ -1361,13 +1361,13 @@ define i32 @umul(i32 %arg) {
 ;
 ; SLM-LABEL: 'umul'
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %I64 = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 undef, i64 undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.umul.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.umul.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 84 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.umul.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.umul.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.umul.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 88 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.umul.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I32 = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 undef, i32 undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.umul.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 88 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.umul.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 180 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.umul.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 46 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.umul.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.umul.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.umul.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %I16 = call { i16, i1 } @llvm.umul.with.overflow.i16(i16 undef, i16 undef)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.umul.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.umul.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
@@ -1384,9 +1384,9 @@ define i32 @umul(i32 %arg) {
 ; GLM-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.umul.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
 ; GLM-NEXT:  Cost Model: Found an estimated cost of 84 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.umul.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; GLM-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I32 = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 undef, i32 undef)
-; GLM-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.umul.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
-; GLM-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.umul.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
-; GLM-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.umul.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.umul.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 60 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.umul.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 124 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.umul.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; GLM-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %I16 = call { i16, i1 } @llvm.umul.with.overflow.i16(i16 undef, i16 undef)
 ; GLM-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.umul.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
 ; GLM-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.umul.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
diff --git a/llvm/test/Analysis/CostModel/X86/arith-ssat.ll b/llvm/test/Analysis/CostModel/X86/arith-ssat.ll
index d99bba326..c951330bc 100644
--- a/llvm/test/Analysis/CostModel/X86/arith-ssat.ll
+++ b/llvm/test/Analysis/CostModel/X86/arith-ssat.ll
@@ -170,9 +170,9 @@ define i32 @add(i32 %arg) {
 ;
 ; SLM-LABEL: 'add'
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.sadd.sat.i64(i64 undef, i64 undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V2I64 = call <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V4I64 = call <4 x i64> @llvm.sadd.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %V8I64 = call <8 x i64> @llvm.sadd.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V2I64 = call <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V4I64 = call <4 x i64> @llvm.sadd.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 59 for instruction: %V8I64 = call <8 x i64> @llvm.sadd.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.sadd.sat.i32(i32 undef, i32 undef)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I32 = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I32 = call <8 x i32> @llvm.sadd.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
@@ -404,9 +404,9 @@ define i32 @sub(i32 %arg) {
 ;
 ; SLM-LABEL: 'sub'
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.ssub.sat.i64(i64 undef, i64 undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V2I64 = call <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V4I64 = call <4 x i64> @llvm.ssub.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %V8I64 = call <8 x i64> @llvm.ssub.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V2I64 = call <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V4I64 = call <4 x i64> @llvm.ssub.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 59 for instruction: %V8I64 = call <8 x i64> @llvm.ssub.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.ssub.sat.i32(i32 undef, i32 undef)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I32 = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I32 = call <8 x i32> @llvm.ssub.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
diff --git a/llvm/test/Analysis/CostModel/X86/arith-usat.ll b/llvm/test/Analysis/CostModel/X86/arith-usat.ll
index e7aaba8c0..a83c16e84 100644
--- a/llvm/test/Analysis/CostModel/X86/arith-usat.ll
+++ b/llvm/test/Analysis/CostModel/X86/arith-usat.ll
@@ -170,9 +170,9 @@ define i32 @add(i32 %arg) {
 ;
 ; SLM-LABEL: 'add'
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.uadd.sat.i64(i64 undef, i64 undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I64 = call <4 x i64> @llvm.uadd.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I64 = call <8 x i64> @llvm.uadd.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2I64 = call <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.uadd.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I64 = call <8 x i64> @llvm.uadd.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.uadd.sat.i32(i32 undef, i32 undef)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4I32 = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8I32 = call <8 x i32> @llvm.uadd.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
@@ -404,9 +404,9 @@ define i32 @sub(i32 %arg) {
 ;
 ; SLM-LABEL: 'sub'
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.usub.sat.i64(i64 undef, i64 undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.usub.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I64 = call <4 x i64> @llvm.usub.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I64 = call <8 x i64> @llvm.usub.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2I64 = call <2 x i64> @llvm.usub.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.usub.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I64 = call <8 x i64> @llvm.usub.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.usub.sat.i32(i32 undef, i32 undef)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = call <8 x i32> @llvm.usub.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
diff --git a/llvm/test/Analysis/CostModel/X86/arith.ll b/llvm/test/Analysis/CostModel/X86/arith.ll
index f52cbf88b..a2ab5e847 100644
--- a/llvm/test/Analysis/CostModel/X86/arith.ll
+++ b/llvm/test/Analysis/CostModel/X86/arith.ll
@@ -1342,36 +1342,32 @@ define i32 @mul(i32 %arg) {
 ; A <2 x i64> vector multiply is implemented using
 ; 3 PMULUDQ and 2 PADDS and 4 shifts.
 define void @mul_2i32() {
-; SSE-LABEL: 'mul_2i32'
-; SSE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %A0 = mul <2 x i32> undef, undef
-; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; SSSE3-LABEL: 'mul_2i32'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %A0 = mul <2 x i32> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; SSE42-LABEL: 'mul_2i32'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %A0 = mul <2 x i32> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX-LABEL: 'mul_2i32'
-; AVX-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %A0 = mul <2 x i32> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %A0 = mul <2 x i32> undef, undef
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
-; AVX512F-LABEL: 'mul_2i32'
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %A0 = mul <2 x i32> undef, undef
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; AVX512BW-LABEL: 'mul_2i32'
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %A0 = mul <2 x i32> undef, undef
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; AVX512DQ-LABEL: 'mul_2i32'
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %A0 = mul <2 x i32> undef, undef
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; AVX512-LABEL: 'mul_2i32'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %A0 = mul <2 x i32> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SLM-LABEL: 'mul_2i32'
-; SLM-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %A0 = mul <2 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %A0 = mul <2 x i32> undef, undef
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; GLM-LABEL: 'mul_2i32'
-; GLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %A0 = mul <2 x i32> undef, undef
+; GLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %A0 = mul <2 x i32> undef, undef
 ; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; BTVER2-LABEL: 'mul_2i32'
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %A0 = mul <2 x i32> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %A0 = mul <2 x i32> undef, undef
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %A0 = mul <2 x i32> undef, undef
diff --git a/llvm/test/Analysis/CostModel/X86/cast.ll b/llvm/test/Analysis/CostModel/X86/cast.ll
index 7072313f5..b43e871dd 100644
--- a/llvm/test/Analysis/CostModel/X86/cast.ll
+++ b/llvm/test/Analysis/CostModel/X86/cast.ll
@@ -90,12 +90,12 @@ define i32 @zext_sext(<8 x i1> %in) {
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %D3 = zext <16 x i16> undef to <16 x i32>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %D4 = zext <16 x i8> undef to <16 x i32>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %D5 = zext <16 x i1> undef to <16 x i32>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %E = trunc <4 x i64> undef to <4 x i32>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %E = trunc <4 x i64> undef to <4 x i32>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %F = trunc <8 x i32> undef to <8 x i16>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %F1 = trunc <16 x i16> undef to <16 x i8>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %F2 = trunc <8 x i32> undef to <8 x i8>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F3 = trunc <4 x i64> undef to <4 x i8>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %G = trunc <8 x i64> undef to <8 x i32>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %F3 = trunc <4 x i64> undef to <4 x i8>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %G = trunc <8 x i64> undef to <8 x i32>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %G1 = trunc <16 x i32> undef to <16 x i16>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %G2 = trunc <16 x i32> undef to <16 x i8>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
@@ -120,12 +120,12 @@ define i32 @zext_sext(<8 x i1> %in) {
 ; SSE41-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %D3 = zext <16 x i16> undef to <16 x i32>
 ; SSE41-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %D4 = zext <16 x i8> undef to <16 x i32>
 ; SSE41-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %D5 = zext <16 x i1> undef to <16 x i32>
-; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %E = trunc <4 x i64> undef to <4 x i32>
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %E = trunc <4 x i64> undef to <4 x i32>
 ; SSE41-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %F = trunc <8 x i32> undef to <8 x i16>
 ; SSE41-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %F1 = trunc <16 x i16> undef to <16 x i8>
 ; SSE41-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %F2 = trunc <8 x i32> undef to <8 x i8>
-; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F3 = trunc <4 x i64> undef to <4 x i8>
-; SSE41-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %G = trunc <8 x i64> undef to <8 x i32>
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %F3 = trunc <4 x i64> undef to <4 x i8>
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %G = trunc <8 x i64> undef to <8 x i32>
 ; SSE41-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %G1 = trunc <16 x i32> undef to <16 x i16>
 ; SSE41-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %G2 = trunc <16 x i32> undef to <16 x i8>
 ; SSE41-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
@@ -139,11 +139,11 @@ define i32 @zext_sext(<8 x i1> %in) {
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %B = zext <8 x i16> undef to <8 x i32>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %C = sext <4 x i32> undef to <4 x i64>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %C.v8i8.z = zext <8 x i8> undef to <8 x i32>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %C.v8i8.s = sext <8 x i8> undef to <8 x i32>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %C.v8i8.s = sext <8 x i8> undef to <8 x i32>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %C.v4i16.z = zext <4 x i16> undef to <4 x i64>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %C.v4i16.s = sext <4 x i16> undef to <4 x i64>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %C.v4i16.s = sext <4 x i16> undef to <4 x i64>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %C.v4i8.z = zext <4 x i8> undef to <4 x i64>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %C.v4i8.s = sext <4 x i8> undef to <4 x i64>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %C.v4i8.s = sext <4 x i8> undef to <4 x i64>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %D = zext <4 x i32> undef to <4 x i64>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %D1 = zext <8 x i32> undef to <8 x i64>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %D2 = sext <8 x i32> undef to <8 x i64>
@@ -168,12 +168,12 @@ define i32 @zext_sext(<8 x i1> %in) {
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %A = sext <8 x i16> undef to <8 x i32>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %B = zext <8 x i16> undef to <8 x i32>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %C = sext <4 x i32> undef to <4 x i64>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %C.v8i8.z = zext <8 x i8> undef to <8 x i32>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %C.v8i8.s = sext <8 x i8> undef to <8 x i32>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %C.v4i16.z = zext <4 x i16> undef to <4 x i64>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %C.v4i16.s = sext <4 x i16> undef to <4 x i64>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %C.v4i8.z = zext <4 x i8> undef to <4 x i64>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %C.v4i8.s = sext <4 x i8> undef to <4 x i64>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %C.v8i8.z = zext <8 x i8> undef to <8 x i32>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %C.v8i8.s = sext <8 x i8> undef to <8 x i32>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %C.v4i16.z = zext <4 x i16> undef to <4 x i64>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %C.v4i16.s = sext <4 x i16> undef to <4 x i64>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %C.v4i8.z = zext <4 x i8> undef to <4 x i64>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %C.v4i8.s = sext <4 x i8> undef to <4 x i64>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %D = zext <4 x i32> undef to <4 x i64>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %D1 = zext <8 x i32> undef to <8 x i64>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %D2 = sext <8 x i32> undef to <8 x i64>
@@ -198,12 +198,12 @@ define i32 @zext_sext(<8 x i1> %in) {
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %A = sext <8 x i16> undef to <8 x i32>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %B = zext <8 x i16> undef to <8 x i32>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %C = sext <4 x i32> undef to <4 x i64>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %C.v8i8.z = zext <8 x i8> undef to <8 x i32>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %C.v8i8.s = sext <8 x i8> undef to <8 x i32>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %C.v4i16.z = zext <4 x i16> undef to <4 x i64>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %C.v4i16.s = sext <4 x i16> undef to <4 x i64>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %C.v4i8.z = zext <4 x i8> undef to <4 x i64>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %C.v4i8.s = sext <4 x i8> undef to <4 x i64>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %C.v8i8.z = zext <8 x i8> undef to <8 x i32>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %C.v8i8.s = sext <8 x i8> undef to <8 x i32>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %C.v4i16.z = zext <4 x i16> undef to <4 x i64>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %C.v4i16.s = sext <4 x i16> undef to <4 x i64>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %C.v4i8.z = zext <4 x i8> undef to <4 x i64>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %C.v4i8.s = sext <4 x i8> undef to <4 x i64>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %D = zext <4 x i32> undef to <4 x i64>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %D1 = zext <8 x i32> undef to <8 x i64>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %D2 = sext <8 x i32> undef to <8 x i64>
@@ -314,13 +314,13 @@ define i32 @masks4(<4 x i1> %in) {
 define void @sitofp4(<4 x i1> %a, <4 x i8> %b, <4 x i16> %c, <4 x i32> %d) {
 ; SSE-LABEL: 'sitofp4'
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %A1 = sitofp <4 x i1> %a to <4 x float>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %A2 = sitofp <4 x i1> %a to <4 x double>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %B1 = sitofp <4 x i8> %b to <4 x float>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %B2 = sitofp <4 x i8> %b to <4 x double>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %C1 = sitofp <4 x i16> %c to <4 x float>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %C2 = sitofp <4 x i16> %c to <4 x double>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %A2 = sitofp <4 x i1> %a to <4 x double>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %B1 = sitofp <4 x i8> %b to <4 x float>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %B2 = sitofp <4 x i8> %b to <4 x double>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %C1 = sitofp <4 x i16> %c to <4 x float>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %C2 = sitofp <4 x i16> %c to <4 x double>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %D1 = sitofp <4 x i32> %d to <4 x float>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %D2 = sitofp <4 x i32> %d to <4 x double>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %D2 = sitofp <4 x i32> %d to <4 x double>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX-LABEL: 'sitofp4'
@@ -359,7 +359,7 @@ define void @sitofp4(<4 x i1> %a, <4 x i8> %b, <4 x i16> %c, <4 x i32> %d) {
 define void @sitofp8(<8 x i1> %a, <8 x i8> %b, <8 x i16> %c, <8 x i32> %d) {
 ; SSE-LABEL: 'sitofp8'
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %A1 = sitofp <8 x i1> %a to <8 x float>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %B1 = sitofp <8 x i8> %b to <8 x float>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %B1 = sitofp <8 x i8> %b to <8 x float>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %C1 = sitofp <8 x i16> %c to <8 x float>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %D1 = sitofp <8 x i32> %d to <8 x float>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
@@ -390,9 +390,9 @@ define void @uitofp4(<4 x i1> %a, <4 x i8> %b, <4 x i16> %c, <4 x i32> %d) {
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %A1 = uitofp <4 x i1> %a to <4 x float>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %A2 = uitofp <4 x i1> %a to <4 x double>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %B1 = uitofp <4 x i8> %b to <4 x float>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %B2 = uitofp <4 x i8> %b to <4 x double>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %C1 = uitofp <4 x i16> %c to <4 x float>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %C2 = uitofp <4 x i16> %c to <4 x double>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %B2 = uitofp <4 x i8> %b to <4 x double>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %C1 = uitofp <4 x i16> %c to <4 x float>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %C2 = uitofp <4 x i16> %c to <4 x double>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %D1 = uitofp <4 x i32> %d to <4 x float>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %D2 = uitofp <4 x i32> %d to <4 x double>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
@@ -433,7 +433,7 @@ define void @uitofp4(<4 x i1> %a, <4 x i8> %b, <4 x i16> %c, <4 x i32> %d) {
 define void @uitofp8(<8 x i1> %a, <8 x i8> %b, <8 x i16> %c, <8 x i32> %d) {
 ; SSE-LABEL: 'uitofp8'
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %A1 = uitofp <8 x i1> %a to <8 x float>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %B1 = uitofp <8 x i8> %b to <8 x float>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %B1 = uitofp <8 x i8> %b to <8 x float>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %C1 = uitofp <8 x i16> %c to <8 x float>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %D1 = uitofp <8 x i32> %d to <8 x float>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
diff --git a/llvm/test/Analysis/CostModel/X86/ctlz.ll b/llvm/test/Analysis/CostModel/X86/ctlz.ll
index e2a6bc521..6da6cb5f9 100644
--- a/llvm/test/Analysis/CostModel/X86/ctlz.ll
+++ b/llvm/test/Analysis/CostModel/X86/ctlz.ll
@@ -1,11 +1,12 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mattr=+sse2 | FileCheck %s -check-prefixes=CHECK,SSE,SSE2
-; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mattr=+sse4.2 | FileCheck %s -check-prefixes=CHECK,SSE,SSE42
-; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mattr=+avx | FileCheck %s -check-prefixes=CHECK,AVX,AVX1
-; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mattr=+avx2 | FileCheck %s -check-prefixes=CHECK,AVX,AVX2
-; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mattr=+avx512f | FileCheck %s -check-prefixes=CHECK,AVX512,AVX512F
-; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mattr=+avx512vl,+avx512bw,+avx512dq | FileCheck %s -check-prefixes=CHECK,AVX512,AVX512BW
-; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mattr=+avx512vl,+avx512bw,+avx512dq,+avx512cd | FileCheck %s -check-prefixes=CHECK,AVX512CD
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mattr=-lzcnt,+sse2 | FileCheck %s -check-prefixes=CHECK,SSE,SSE2,NOLZCNT
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mattr=+lzcnt,+sse2 | FileCheck %s -check-prefixes=CHECK,SSE,SSE2,LZCNT
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mattr=+lzcnt,+sse4.2 | FileCheck %s -check-prefixes=CHECK,LZCNT,SSE,SSE42
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mattr=+lzcnt,+avx | FileCheck %s -check-prefixes=CHECK,LZCNT,AVX,AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mattr=+lzcnt,+avx2 | FileCheck %s -check-prefixes=CHECK,LZCNT,AVX,AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mattr=+lzcnt,+avx512f | FileCheck %s -check-prefixes=CHECK,LZCNT,AVX512,AVX512F
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mattr=+lzcnt,+avx512vl,+avx512bw,+avx512dq | FileCheck %s -check-prefixes=CHECK,LZCNT,AVX512,AVX512BW
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mattr=+lzcnt,+avx512vl,+avx512bw,+avx512dq,+avx512cd | FileCheck %s -check-prefixes=CHECK,LZCNT,AVX512CD
 
 ; Verify the cost of scalar leading zero count instructions.
 
@@ -15,72 +16,104 @@ declare i16 @llvm.ctlz.i16(i16, i1)
 declare  i8 @llvm.ctlz.i8(i8, i1)
 
 define i64 @var_ctlz_i64(i64 %a) {
-; CHECK-LABEL: 'var_ctlz_i64'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %ctlz = call i64 @llvm.ctlz.i64(i64 %a, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %ctlz
+; NOLZCNT-LABEL: 'var_ctlz_i64'
+; NOLZCNT-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %ctlz = call i64 @llvm.ctlz.i64(i64 %a, i1 false)
+; NOLZCNT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %ctlz
+;
+; LZCNT-LABEL: 'var_ctlz_i64'
+; LZCNT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %ctlz = call i64 @llvm.ctlz.i64(i64 %a, i1 false)
+; LZCNT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %ctlz
 ;
   %ctlz = call i64 @llvm.ctlz.i64(i64 %a, i1 0)
   ret i64 %ctlz
 }
 
 define i64 @var_ctlz_i64u(i64 %a) {
-; CHECK-LABEL: 'var_ctlz_i64u'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %ctlz = call i64 @llvm.ctlz.i64(i64 %a, i1 true)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %ctlz
+; NOLZCNT-LABEL: 'var_ctlz_i64u'
+; NOLZCNT-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %ctlz = call i64 @llvm.ctlz.i64(i64 %a, i1 true)
+; NOLZCNT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %ctlz
+;
+; LZCNT-LABEL: 'var_ctlz_i64u'
+; LZCNT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %ctlz = call i64 @llvm.ctlz.i64(i64 %a, i1 true)
+; LZCNT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %ctlz
 ;
   %ctlz = call i64 @llvm.ctlz.i64(i64 %a, i1 1)
   ret i64 %ctlz
 }
 
 define i32 @var_ctlz_i32(i32 %a) {
-; CHECK-LABEL: 'var_ctlz_i32'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %ctlz = call i32 @llvm.ctlz.i32(i32 %a, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %ctlz
+; NOLZCNT-LABEL: 'var_ctlz_i32'
+; NOLZCNT-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %ctlz = call i32 @llvm.ctlz.i32(i32 %a, i1 false)
+; NOLZCNT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %ctlz
+;
+; LZCNT-LABEL: 'var_ctlz_i32'
+; LZCNT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %ctlz = call i32 @llvm.ctlz.i32(i32 %a, i1 false)
+; LZCNT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %ctlz
 ;
   %ctlz = call i32 @llvm.ctlz.i32(i32 %a, i1 0)
   ret i32 %ctlz
 }
 
 define i32 @var_ctlz_i32u(i32 %a) {
-; CHECK-LABEL: 'var_ctlz_i32u'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %ctlz = call i32 @llvm.ctlz.i32(i32 %a, i1 true)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %ctlz
+; NOLZCNT-LABEL: 'var_ctlz_i32u'
+; NOLZCNT-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %ctlz = call i32 @llvm.ctlz.i32(i32 %a, i1 true)
+; NOLZCNT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %ctlz
+;
+; LZCNT-LABEL: 'var_ctlz_i32u'
+; LZCNT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %ctlz = call i32 @llvm.ctlz.i32(i32 %a, i1 true)
+; LZCNT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %ctlz
 ;
   %ctlz = call i32 @llvm.ctlz.i32(i32 %a, i1 1)
   ret i32 %ctlz
 }
 
 define i16 @var_ctlz_i16(i16 %a) {
-; CHECK-LABEL: 'var_ctlz_i16'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %ctlz = call i16 @llvm.ctlz.i16(i16 %a, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i16 %ctlz
+; NOLZCNT-LABEL: 'var_ctlz_i16'
+; NOLZCNT-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %ctlz = call i16 @llvm.ctlz.i16(i16 %a, i1 false)
+; NOLZCNT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i16 %ctlz
+;
+; LZCNT-LABEL: 'var_ctlz_i16'
+; LZCNT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %ctlz = call i16 @llvm.ctlz.i16(i16 %a, i1 false)
+; LZCNT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i16 %ctlz
 ;
   %ctlz = call i16 @llvm.ctlz.i16(i16 %a, i1 0)
   ret i16 %ctlz
 }
 
 define i16 @var_ctlz_i16u(i16 %a) {
-; CHECK-LABEL: 'var_ctlz_i16u'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %ctlz = call i16 @llvm.ctlz.i16(i16 %a, i1 true)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i16 %ctlz
+; NOLZCNT-LABEL: 'var_ctlz_i16u'
+; NOLZCNT-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %ctlz = call i16 @llvm.ctlz.i16(i16 %a, i1 true)
+; NOLZCNT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i16 %ctlz
+;
+; LZCNT-LABEL: 'var_ctlz_i16u'
+; LZCNT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %ctlz = call i16 @llvm.ctlz.i16(i16 %a, i1 true)
+; LZCNT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i16 %ctlz
 ;
   %ctlz = call i16 @llvm.ctlz.i16(i16 %a, i1 1)
   ret i16 %ctlz
 }
 
 define i8 @var_ctlz_i8(i8 %a) {
-; CHECK-LABEL: 'var_ctlz_i8'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %ctlz = call i8 @llvm.ctlz.i8(i8 %a, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i8 %ctlz
+; NOLZCNT-LABEL: 'var_ctlz_i8'
+; NOLZCNT-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %ctlz = call i8 @llvm.ctlz.i8(i8 %a, i1 false)
+; NOLZCNT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i8 %ctlz
+;
+; LZCNT-LABEL: 'var_ctlz_i8'
+; LZCNT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %ctlz = call i8 @llvm.ctlz.i8(i8 %a, i1 false)
+; LZCNT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i8 %ctlz
 ;
   %ctlz = call i8 @llvm.ctlz.i8(i8 %a, i1 0)
   ret i8 %ctlz
 }
 
 define i8 @var_ctlz_i8u(i8 %a) {
-; CHECK-LABEL: 'var_ctlz_i8u'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %ctlz = call i8 @llvm.ctlz.i8(i8 %a, i1 true)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i8 %ctlz
+; NOLZCNT-LABEL: 'var_ctlz_i8u'
+; NOLZCNT-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %ctlz = call i8 @llvm.ctlz.i8(i8 %a, i1 true)
+; NOLZCNT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i8 %ctlz
+;
+; LZCNT-LABEL: 'var_ctlz_i8u'
+; LZCNT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %ctlz = call i8 @llvm.ctlz.i8(i8 %a, i1 true)
+; LZCNT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i8 %ctlz
 ;
   %ctlz = call i8 @llvm.ctlz.i8(i8 %a, i1 1)
   ret i8 %ctlz
diff --git a/llvm/test/Analysis/CostModel/X86/ctpop.ll b/llvm/test/Analysis/CostModel/X86/ctpop.ll
index bb59558e4..e9cf666af 100644
--- a/llvm/test/Analysis/CostModel/X86/ctpop.ll
+++ b/llvm/test/Analysis/CostModel/X86/ctpop.ll
@@ -16,7 +16,7 @@ declare  i8 @llvm.ctpop.i8(i8)
 
 define i64 @var_ctpop_i64(i64 %a) {
 ; NOPOPCNT-LABEL: 'var_ctpop_i64'
-; NOPOPCNT-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %ctpop = call i64 @llvm.ctpop.i64(i64 %a)
+; NOPOPCNT-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %ctpop = call i64 @llvm.ctpop.i64(i64 %a)
 ; NOPOPCNT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %ctpop
 ;
 ; POPCNT-LABEL: 'var_ctpop_i64'
@@ -29,7 +29,7 @@ define i64 @var_ctpop_i64(i64 %a) {
 
 define i32 @var_ctpop_i32(i32 %a) {
 ; NOPOPCNT-LABEL: 'var_ctpop_i32'
-; NOPOPCNT-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %ctpop = call i32 @llvm.ctpop.i32(i32 %a)
+; NOPOPCNT-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %ctpop = call i32 @llvm.ctpop.i32(i32 %a)
 ; NOPOPCNT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %ctpop
 ;
 ; POPCNT-LABEL: 'var_ctpop_i32'
@@ -42,7 +42,7 @@ define i32 @var_ctpop_i32(i32 %a) {
 
 define i16 @var_ctpop_i16(i16 %a) {
 ; NOPOPCNT-LABEL: 'var_ctpop_i16'
-; NOPOPCNT-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %ctpop = call i16 @llvm.ctpop.i16(i16 %a)
+; NOPOPCNT-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %ctpop = call i16 @llvm.ctpop.i16(i16 %a)
 ; NOPOPCNT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i16 %ctpop
 ;
 ; POPCNT-LABEL: 'var_ctpop_i16'
@@ -55,7 +55,7 @@ define i16 @var_ctpop_i16(i16 %a) {
 
 define i8 @var_ctpop_i8(i8 %a) {
 ; NOPOPCNT-LABEL: 'var_ctpop_i8'
-; NOPOPCNT-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %ctpop = call i8 @llvm.ctpop.i8(i8 %a)
+; NOPOPCNT-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %ctpop = call i8 @llvm.ctpop.i8(i8 %a)
 ; NOPOPCNT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i8 %ctpop
 ;
 ; POPCNT-LABEL: 'var_ctpop_i8'
diff --git a/llvm/test/Analysis/CostModel/X86/extend.ll b/llvm/test/Analysis/CostModel/X86/extend.ll
index 1de0b0b20..93831165e 100644
--- a/llvm/test/Analysis/CostModel/X86/extend.ll
+++ b/llvm/test/Analysis/CostModel/X86/extend.ll
@@ -13,47 +13,55 @@
 
 define i32 @zext_vXi32() {
 ; SSE2-LABEL: 'zext_vXi32'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %I64 = zext i32 undef to i64
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i64 = zext <2 x i32> undef to <2 x i64>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4i64 = zext <4 x i32> undef to <4 x i64>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8i64 = zext <8 x i32> undef to <8 x i64>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'zext_vXi32'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %I64 = zext i32 undef to i64
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i64 = zext <2 x i32> undef to <2 x i64>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4i64 = zext <4 x i32> undef to <4 x i64>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8i64 = zext <8 x i32> undef to <8 x i64>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE42-LABEL: 'zext_vXi32'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %I64 = zext i32 undef to i64
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i64 = zext <2 x i32> undef to <2 x i64>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i64 = zext <4 x i32> undef to <4 x i64>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8i64 = zext <8 x i32> undef to <8 x i64>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX1-LABEL: 'zext_vXi32'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %I64 = zext i32 undef to i64
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i64 = zext <2 x i32> undef to <2 x i64>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4i64 = zext <4 x i32> undef to <4 x i64>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V8i64 = zext <8 x i32> undef to <8 x i64>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX2-LABEL: 'zext_vXi32'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %I64 = zext i32 undef to i64
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i64 = zext <2 x i32> undef to <2 x i64>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i64 = zext <4 x i32> undef to <4 x i64>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8i64 = zext <8 x i32> undef to <8 x i64>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512-LABEL: 'zext_vXi32'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %I64 = zext i32 undef to i64
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i64 = zext <2 x i32> undef to <2 x i64>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i64 = zext <4 x i32> undef to <4 x i64>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i64 = zext <8 x i32> undef to <8 x i64>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; BTVER2-LABEL: 'zext_vXi32'
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %I64 = zext i32 undef to i64
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i64 = zext <2 x i32> undef to <2 x i64>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4i64 = zext <4 x i32> undef to <4 x i64>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V8i64 = zext <8 x i32> undef to <8 x i64>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
+  %I64 = zext i32 undef to i64
   %V2i64 = zext <2 x i32> undef to <2 x i64>
   %V4i64 = zext <4 x i32> undef to <4 x i64>
   %V8i64 = zext <8 x i32> undef to <8 x i64>
@@ -63,9 +71,11 @@ define i32 @zext_vXi32() {
 
 define i32 @zext_vXi16() {
 ; SSE2-LABEL: 'zext_vXi16'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = zext i16 undef to i64
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i64 = zext <2 x i16> undef to <2 x i64>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4i64 = zext <4 x i16> undef to <4 x i64>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8i64 = zext <8 x i16> undef to <8 x i64>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = zext i16 undef to i32
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i32 = zext <2 x i16> undef to <2 x i32>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = zext <4 x i16> undef to <4 x i32>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8i32 = zext <8 x i16> undef to <8 x i32>
@@ -73,9 +83,11 @@ define i32 @zext_vXi16() {
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'zext_vXi16'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = zext i16 undef to i64
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i64 = zext <2 x i16> undef to <2 x i64>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4i64 = zext <4 x i16> undef to <4 x i64>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8i64 = zext <8 x i16> undef to <8 x i64>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = zext i16 undef to i32
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i32 = zext <2 x i16> undef to <2 x i32>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = zext <4 x i16> undef to <4 x i32>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8i32 = zext <8 x i16> undef to <8 x i32>
@@ -83,9 +95,11 @@ define i32 @zext_vXi16() {
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE42-LABEL: 'zext_vXi16'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = zext i16 undef to i64
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i64 = zext <2 x i16> undef to <2 x i64>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i64 = zext <4 x i16> undef to <4 x i64>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8i64 = zext <8 x i16> undef to <8 x i64>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = zext i16 undef to i32
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i32 = zext <2 x i16> undef to <2 x i32>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = zext <4 x i16> undef to <4 x i32>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8i32 = zext <8 x i16> undef to <8 x i32>
@@ -93,9 +107,11 @@ define i32 @zext_vXi16() {
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX1-LABEL: 'zext_vXi16'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = zext i16 undef to i64
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i64 = zext <2 x i16> undef to <2 x i64>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4i64 = zext <4 x i16> undef to <4 x i64>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8i64 = zext <8 x i16> undef to <8 x i64>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = zext i16 undef to i32
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i32 = zext <2 x i16> undef to <2 x i32>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = zext <4 x i16> undef to <4 x i32>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8i32 = zext <8 x i16> undef to <8 x i32>
@@ -103,9 +119,11 @@ define i32 @zext_vXi16() {
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX2-LABEL: 'zext_vXi16'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = zext i16 undef to i64
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i64 = zext <2 x i16> undef to <2 x i64>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4i64 = zext <4 x i16> undef to <4 x i64>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8i64 = zext <8 x i16> undef to <8 x i64>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i64 = zext <4 x i16> undef to <4 x i64>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8i64 = zext <8 x i16> undef to <8 x i64>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = zext i16 undef to i32
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i32 = zext <2 x i16> undef to <2 x i32>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = zext <4 x i16> undef to <4 x i32>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i32 = zext <8 x i16> undef to <8 x i32>
@@ -113,9 +131,11 @@ define i32 @zext_vXi16() {
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512-LABEL: 'zext_vXi16'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = zext i16 undef to i64
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i64 = zext <2 x i16> undef to <2 x i64>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4i64 = zext <4 x i16> undef to <4 x i64>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i64 = zext <4 x i16> undef to <4 x i64>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i64 = zext <8 x i16> undef to <8 x i64>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = zext i16 undef to i32
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i32 = zext <2 x i16> undef to <2 x i32>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = zext <4 x i16> undef to <4 x i32>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i32 = zext <8 x i16> undef to <8 x i32>
@@ -123,19 +143,23 @@ define i32 @zext_vXi16() {
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; BTVER2-LABEL: 'zext_vXi16'
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = zext i16 undef to i64
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i64 = zext <2 x i16> undef to <2 x i64>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4i64 = zext <4 x i16> undef to <4 x i64>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8i64 = zext <8 x i16> undef to <8 x i64>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = zext i16 undef to i32
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i32 = zext <2 x i16> undef to <2 x i32>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = zext <4 x i16> undef to <4 x i32>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8i32 = zext <8 x i16> undef to <8 x i32>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16i32 = zext <16 x i16> undef to <16 x i32>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
+  %I64 = zext i16 undef to i64
   %V2i64 = zext <2 x i16> undef to <2 x i64>
   %V4i64 = zext <4 x i16> undef to <4 x i64>
   %V8i64 = zext <8 x i16> undef to <8 x i64>
 
+  %I32 = zext i16 undef to i32
   %V2i32 = zext <2 x i16> undef to <2 x i32>
   %V4i32 = zext <4 x i16> undef to <4 x i32>
   %V8i32 = zext <8 x i16> undef to <8 x i32>
@@ -146,13 +170,16 @@ define i32 @zext_vXi16() {
 
 define i32 @zext_vXi8() {
 ; SSE2-LABEL: 'zext_vXi8'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = zext i8 undef to i64
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i64 = zext <2 x i8> undef to <2 x i64>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4i64 = zext <4 x i8> undef to <4 x i64>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V8i64 = zext <8 x i8> undef to <8 x i64>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = zext i8 undef to i32
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i32 = zext <2 x i8> undef to <2 x i32>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i32 = zext <4 x i8> undef to <4 x i32>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8i32 = zext <8 x i8> undef to <8 x i32>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V16i32 = zext <16 x i8> undef to <16 x i32>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = zext i8 undef to i16
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i16 = zext <2 x i8> undef to <2 x i16>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i16 = zext <4 x i8> undef to <4 x i16>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i16 = zext <8 x i8> undef to <8 x i16>
@@ -161,13 +188,16 @@ define i32 @zext_vXi8() {
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'zext_vXi8'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = zext i8 undef to i64
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i64 = zext <2 x i8> undef to <2 x i64>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4i64 = zext <4 x i8> undef to <4 x i64>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V8i64 = zext <8 x i8> undef to <8 x i64>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = zext i8 undef to i32
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i32 = zext <2 x i8> undef to <2 x i32>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i32 = zext <4 x i8> undef to <4 x i32>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8i32 = zext <8 x i8> undef to <8 x i32>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V16i32 = zext <16 x i8> undef to <16 x i32>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = zext i8 undef to i16
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i16 = zext <2 x i8> undef to <2 x i16>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i16 = zext <4 x i8> undef to <4 x i16>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i16 = zext <8 x i8> undef to <8 x i16>
@@ -176,13 +206,16 @@ define i32 @zext_vXi8() {
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE42-LABEL: 'zext_vXi8'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = zext i8 undef to i64
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i64 = zext <2 x i8> undef to <2 x i64>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i64 = zext <4 x i8> undef to <4 x i64>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8i64 = zext <8 x i8> undef to <8 x i64>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = zext i8 undef to i32
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i32 = zext <2 x i8> undef to <2 x i32>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = zext <4 x i8> undef to <4 x i32>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8i32 = zext <8 x i8> undef to <8 x i32>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16i32 = zext <16 x i8> undef to <16 x i32>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = zext i8 undef to i16
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i16 = zext <2 x i8> undef to <2 x i16>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i16 = zext <4 x i8> undef to <4 x i16>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i16 = zext <8 x i8> undef to <8 x i16>
@@ -191,13 +224,16 @@ define i32 @zext_vXi8() {
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX1-LABEL: 'zext_vXi8'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = zext i8 undef to i64
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i64 = zext <2 x i8> undef to <2 x i64>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4i64 = zext <4 x i8> undef to <4 x i64>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V8i64 = zext <8 x i8> undef to <8 x i64>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = zext i8 undef to i32
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i32 = zext <2 x i8> undef to <2 x i32>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = zext <4 x i8> undef to <4 x i32>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8i32 = zext <8 x i8> undef to <8 x i32>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16i32 = zext <16 x i8> undef to <16 x i32>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = zext i8 undef to i16
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i16 = zext <2 x i8> undef to <2 x i16>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i16 = zext <4 x i8> undef to <4 x i16>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i16 = zext <8 x i8> undef to <8 x i16>
@@ -206,13 +242,16 @@ define i32 @zext_vXi8() {
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX2-LABEL: 'zext_vXi8'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = zext i8 undef to i64
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i64 = zext <2 x i8> undef to <2 x i64>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4i64 = zext <4 x i8> undef to <4 x i64>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8i64 = zext <8 x i8> undef to <8 x i64>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i64 = zext <4 x i8> undef to <4 x i64>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8i64 = zext <8 x i8> undef to <8 x i64>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = zext i8 undef to i32
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i32 = zext <2 x i8> undef to <2 x i32>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = zext <4 x i8> undef to <4 x i32>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8i32 = zext <8 x i8> undef to <8 x i32>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i32 = zext <8 x i8> undef to <8 x i32>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16i32 = zext <16 x i8> undef to <16 x i32>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = zext i8 undef to i16
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i16 = zext <2 x i8> undef to <2 x i16>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i16 = zext <4 x i8> undef to <4 x i16>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i16 = zext <8 x i8> undef to <8 x i16>
@@ -221,13 +260,16 @@ define i32 @zext_vXi8() {
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512F-LABEL: 'zext_vXi8'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = zext i8 undef to i64
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i64 = zext <2 x i8> undef to <2 x i64>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4i64 = zext <4 x i8> undef to <4 x i64>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8i64 = zext <8 x i8> undef to <8 x i64>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i64 = zext <4 x i8> undef to <4 x i64>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i64 = zext <8 x i8> undef to <8 x i64>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = zext i8 undef to i32
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i32 = zext <2 x i8> undef to <2 x i32>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = zext <4 x i8> undef to <4 x i32>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8i32 = zext <8 x i8> undef to <8 x i32>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i32 = zext <8 x i8> undef to <8 x i32>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16i32 = zext <16 x i8> undef to <16 x i32>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = zext i8 undef to i16
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i16 = zext <2 x i8> undef to <2 x i16>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i16 = zext <4 x i8> undef to <4 x i16>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i16 = zext <8 x i8> undef to <8 x i16>
@@ -236,13 +278,16 @@ define i32 @zext_vXi8() {
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512BW-LABEL: 'zext_vXi8'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = zext i8 undef to i64
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i64 = zext <2 x i8> undef to <2 x i64>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4i64 = zext <4 x i8> undef to <4 x i64>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8i64 = zext <8 x i8> undef to <8 x i64>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i64 = zext <4 x i8> undef to <4 x i64>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i64 = zext <8 x i8> undef to <8 x i64>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = zext i8 undef to i32
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i32 = zext <2 x i8> undef to <2 x i32>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = zext <4 x i8> undef to <4 x i32>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8i32 = zext <8 x i8> undef to <8 x i32>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i32 = zext <8 x i8> undef to <8 x i32>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16i32 = zext <16 x i8> undef to <16 x i32>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = zext i8 undef to i16
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i16 = zext <2 x i8> undef to <2 x i16>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i16 = zext <4 x i8> undef to <4 x i16>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i16 = zext <8 x i8> undef to <8 x i16>
@@ -251,13 +296,16 @@ define i32 @zext_vXi8() {
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; BTVER2-LABEL: 'zext_vXi8'
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = zext i8 undef to i64
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i64 = zext <2 x i8> undef to <2 x i64>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4i64 = zext <4 x i8> undef to <4 x i64>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V8i64 = zext <8 x i8> undef to <8 x i64>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = zext i8 undef to i32
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i32 = zext <2 x i8> undef to <2 x i32>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = zext <4 x i8> undef to <4 x i32>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8i32 = zext <8 x i8> undef to <8 x i32>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16i32 = zext <16 x i8> undef to <16 x i32>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = zext i8 undef to i16
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i16 = zext <2 x i8> undef to <2 x i16>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i16 = zext <4 x i8> undef to <4 x i16>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i16 = zext <8 x i8> undef to <8 x i16>
@@ -265,15 +313,18 @@ define i32 @zext_vXi8() {
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V32i16 = zext <32 x i8> undef to <32 x i16>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
+  %I64 = zext i8 undef to i64
   %V2i64 = zext <2 x i8> undef to <2 x i64>
   %V4i64 = zext <4 x i8> undef to <4 x i64>
   %V8i64 = zext <8 x i8> undef to <8 x i64>
 
+  %I32 = zext i8 undef to i32
   %V2i32 = zext <2 x i8> undef to <2 x i32>
   %V4i32 = zext <4 x i8> undef to <4 x i32>
   %V8i32 = zext <8 x i8> undef to <8 x i32>
   %V16i32 = zext <16 x i8> undef to <16 x i32>
 
+  %I16 = zext i8 undef to i16
   %V2i16 = zext <2 x i8> undef to <2 x i16>
   %V4i16 = zext <4 x i8> undef to <4 x i16>
   %V8i16 = zext <8 x i8> undef to <8 x i16>
@@ -285,18 +336,22 @@ define i32 @zext_vXi8() {
 
 define i32 @zext_vXi1() {
 ; SSE-LABEL: 'zext_vXi1'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = zext i1 undef to i64
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i64 = zext <2 x i1> undef to <2 x i64>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4i64 = zext <4 x i1> undef to <4 x i64>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8i64 = zext <8 x i1> undef to <8 x i64>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = zext i1 undef to i32
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i32 = zext <2 x i1> undef to <2 x i32>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = zext <4 x i1> undef to <4 x i32>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8i32 = zext <8 x i1> undef to <8 x i32>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16i32 = zext <16 x i1> undef to <16 x i32>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = zext i1 undef to i16
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i16 = zext <2 x i1> undef to <2 x i16>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i16 = zext <4 x i1> undef to <4 x i16>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i16 = zext <8 x i1> undef to <8 x i16>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16i16 = zext <16 x i1> undef to <16 x i16>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V32i16 = zext <32 x i1> undef to <32 x i16>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = zext i1 undef to i8
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i8 = zext <2 x i1> undef to <2 x i8>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i8 = zext <4 x i1> undef to <4 x i8>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i8 = zext <8 x i1> undef to <8 x i8>
@@ -306,18 +361,22 @@ define i32 @zext_vXi1() {
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX1-LABEL: 'zext_vXi1'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = zext i1 undef to i64
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i64 = zext <2 x i1> undef to <2 x i64>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4i64 = zext <4 x i1> undef to <4 x i64>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V8i64 = zext <8 x i1> undef to <8 x i64>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = zext i1 undef to i32
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i32 = zext <2 x i1> undef to <2 x i32>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = zext <4 x i1> undef to <4 x i32>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8i32 = zext <8 x i1> undef to <8 x i32>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V16i32 = zext <16 x i1> undef to <16 x i32>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = zext i1 undef to i16
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i16 = zext <2 x i1> undef to <2 x i16>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i16 = zext <4 x i1> undef to <4 x i16>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i16 = zext <8 x i1> undef to <8 x i16>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16i16 = zext <16 x i1> undef to <16 x i16>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 97 for instruction: %V32i16 = zext <32 x i1> undef to <32 x i16>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = zext i1 undef to i8
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i8 = zext <2 x i1> undef to <2 x i8>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i8 = zext <4 x i1> undef to <4 x i8>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i8 = zext <8 x i1> undef to <8 x i8>
@@ -327,18 +386,22 @@ define i32 @zext_vXi1() {
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX2-LABEL: 'zext_vXi1'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = zext i1 undef to i64
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i64 = zext <2 x i1> undef to <2 x i64>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4i64 = zext <4 x i1> undef to <4 x i64>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8i64 = zext <8 x i1> undef to <8 x i64>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = zext i1 undef to i32
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i32 = zext <2 x i1> undef to <2 x i32>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = zext <4 x i1> undef to <4 x i32>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8i32 = zext <8 x i1> undef to <8 x i32>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16i32 = zext <16 x i1> undef to <16 x i32>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = zext i1 undef to i16
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i16 = zext <2 x i1> undef to <2 x i16>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i16 = zext <4 x i1> undef to <4 x i16>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i16 = zext <8 x i1> undef to <8 x i16>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16i16 = zext <16 x i1> undef to <16 x i16>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 97 for instruction: %V32i16 = zext <32 x i1> undef to <32 x i16>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = zext i1 undef to i8
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i8 = zext <2 x i1> undef to <2 x i8>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i8 = zext <4 x i1> undef to <4 x i8>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i8 = zext <8 x i1> undef to <8 x i8>
@@ -348,18 +411,22 @@ define i32 @zext_vXi1() {
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512F-LABEL: 'zext_vXi1'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = zext i1 undef to i64
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2i64 = zext <2 x i1> undef to <2 x i64>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4i64 = zext <4 x i1> undef to <4 x i64>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8i64 = zext <8 x i1> undef to <8 x i64>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = zext i1 undef to i32
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2i32 = zext <2 x i1> undef to <2 x i32>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4i32 = zext <4 x i1> undef to <4 x i32>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8i32 = zext <8 x i1> undef to <8 x i32>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16i32 = zext <16 x i1> undef to <16 x i32>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = zext i1 undef to i16
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2i16 = zext <2 x i1> undef to <2 x i16>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4i16 = zext <4 x i1> undef to <4 x i16>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8i16 = zext <8 x i1> undef to <8 x i16>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16i16 = zext <16 x i1> undef to <16 x i16>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 97 for instruction: %V32i16 = zext <32 x i1> undef to <32 x i16>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = zext i1 undef to i8
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2i8 = zext <2 x i1> undef to <2 x i8>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4i8 = zext <4 x i1> undef to <4 x i8>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8i8 = zext <8 x i1> undef to <8 x i8>
@@ -369,18 +436,22 @@ define i32 @zext_vXi1() {
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512BW-LABEL: 'zext_vXi1'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = zext i1 undef to i64
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2i64 = zext <2 x i1> undef to <2 x i64>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4i64 = zext <4 x i1> undef to <4 x i64>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8i64 = zext <8 x i1> undef to <8 x i64>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = zext i1 undef to i32
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2i32 = zext <2 x i1> undef to <2 x i32>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4i32 = zext <4 x i1> undef to <4 x i32>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8i32 = zext <8 x i1> undef to <8 x i32>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16i32 = zext <16 x i1> undef to <16 x i32>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = zext i1 undef to i16
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2i16 = zext <2 x i1> undef to <2 x i16>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4i16 = zext <4 x i1> undef to <4 x i16>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8i16 = zext <8 x i1> undef to <8 x i16>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16i16 = zext <16 x i1> undef to <16 x i16>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32i16 = zext <32 x i1> undef to <32 x i16>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = zext i1 undef to i8
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2i8 = zext <2 x i1> undef to <2 x i8>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4i8 = zext <4 x i1> undef to <4 x i8>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8i8 = zext <8 x i1> undef to <8 x i8>
@@ -390,18 +461,22 @@ define i32 @zext_vXi1() {
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; BTVER2-LABEL: 'zext_vXi1'
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = zext i1 undef to i64
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i64 = zext <2 x i1> undef to <2 x i64>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4i64 = zext <4 x i1> undef to <4 x i64>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V8i64 = zext <8 x i1> undef to <8 x i64>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = zext i1 undef to i32
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i32 = zext <2 x i1> undef to <2 x i32>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = zext <4 x i1> undef to <4 x i32>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8i32 = zext <8 x i1> undef to <8 x i32>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V16i32 = zext <16 x i1> undef to <16 x i32>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = zext i1 undef to i16
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i16 = zext <2 x i1> undef to <2 x i16>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i16 = zext <4 x i1> undef to <4 x i16>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i16 = zext <8 x i1> undef to <8 x i16>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16i16 = zext <16 x i1> undef to <16 x i16>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 97 for instruction: %V32i16 = zext <32 x i1> undef to <32 x i16>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = zext i1 undef to i8
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i8 = zext <2 x i1> undef to <2 x i8>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i8 = zext <4 x i1> undef to <4 x i8>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i8 = zext <8 x i1> undef to <8 x i8>
@@ -410,21 +485,25 @@ define i32 @zext_vXi1() {
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64i8 = zext <64 x i1> undef to <64 x i8>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
+  %I64 = zext i1 undef to i64
   %V2i64 = zext <2 x i1> undef to <2 x i64>
   %V4i64 = zext <4 x i1> undef to <4 x i64>
   %V8i64 = zext <8 x i1> undef to <8 x i64>
 
+  %I32 = zext i1 undef to i32
   %V2i32 = zext <2 x i1> undef to <2 x i32>
   %V4i32 = zext <4 x i1> undef to <4 x i32>
   %V8i32 = zext <8 x i1> undef to <8 x i32>
   %V16i32 = zext <16 x i1> undef to <16 x i32>
 
+  %I16 = zext i1 undef to i16
   %V2i16 = zext <2 x i1> undef to <2 x i16>
   %V4i16 = zext <4 x i1> undef to <4 x i16>
   %V8i16 = zext <8 x i1> undef to <8 x i16>
   %V16i16 = zext <16 x i1> undef to <16 x i16>
   %V32i16 = zext <32 x i1> undef to <32 x i16>
 
+  %I8 = zext i1 undef to i8
   %V2i8 = zext <2 x i1> undef to <2 x i8>
   %V4i8 = zext <4 x i1> undef to <4 x i8>
   %V8i8 = zext <8 x i1> undef to <8 x i8>
@@ -437,47 +516,55 @@ define i32 @zext_vXi1() {
 
 define i32 @sext_vXi32() {
 ; SSE2-LABEL: 'sext_vXi32'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = sext i32 undef to i64
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i64 = sext <2 x i32> undef to <2 x i64>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4i64 = sext <4 x i32> undef to <4 x i64>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V8i64 = sext <8 x i32> undef to <8 x i64>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'sext_vXi32'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = sext i32 undef to i64
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i64 = sext <2 x i32> undef to <2 x i64>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4i64 = sext <4 x i32> undef to <4 x i64>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V8i64 = sext <8 x i32> undef to <8 x i64>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE42-LABEL: 'sext_vXi32'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = sext i32 undef to i64
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i64 = sext <2 x i32> undef to <2 x i64>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i64 = sext <4 x i32> undef to <4 x i64>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8i64 = sext <8 x i32> undef to <8 x i64>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX1-LABEL: 'sext_vXi32'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = sext i32 undef to i64
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i64 = sext <2 x i32> undef to <2 x i64>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4i64 = sext <4 x i32> undef to <4 x i64>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V8i64 = sext <8 x i32> undef to <8 x i64>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX2-LABEL: 'sext_vXi32'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = sext i32 undef to i64
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i64 = sext <2 x i32> undef to <2 x i64>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i64 = sext <4 x i32> undef to <4 x i64>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8i64 = sext <8 x i32> undef to <8 x i64>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512-LABEL: 'sext_vXi32'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = sext i32 undef to i64
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i64 = sext <2 x i32> undef to <2 x i64>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i64 = sext <4 x i32> undef to <4 x i64>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i64 = sext <8 x i32> undef to <8 x i64>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; BTVER2-LABEL: 'sext_vXi32'
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = sext i32 undef to i64
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i64 = sext <2 x i32> undef to <2 x i64>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4i64 = sext <4 x i32> undef to <4 x i64>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V8i64 = sext <8 x i32> undef to <8 x i64>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
+  %I64 = sext i32 undef to i64
   %V2i64 = sext <2 x i32> undef to <2 x i64>
   %V4i64 = sext <4 x i32> undef to <4 x i64>
   %V8i64 = sext <8 x i32> undef to <8 x i64>
@@ -487,9 +574,11 @@ define i32 @sext_vXi32() {
 
 define i32 @sext_vXi16() {
 ; SSE2-LABEL: 'sext_vXi16'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = sext i16 undef to i64
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i64 = sext <2 x i16> undef to <2 x i64>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4i64 = sext <4 x i16> undef to <4 x i64>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V8i64 = sext <8 x i16> undef to <8 x i64>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = sext i16 undef to i32
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i32 = sext <2 x i16> undef to <2 x i32>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i32 = sext <4 x i16> undef to <4 x i32>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8i32 = sext <8 x i16> undef to <8 x i32>
@@ -497,9 +586,11 @@ define i32 @sext_vXi16() {
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'sext_vXi16'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = sext i16 undef to i64
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i64 = sext <2 x i16> undef to <2 x i64>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4i64 = sext <4 x i16> undef to <4 x i64>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V8i64 = sext <8 x i16> undef to <8 x i64>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = sext i16 undef to i32
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i32 = sext <2 x i16> undef to <2 x i32>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i32 = sext <4 x i16> undef to <4 x i32>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8i32 = sext <8 x i16> undef to <8 x i32>
@@ -507,9 +598,11 @@ define i32 @sext_vXi16() {
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE42-LABEL: 'sext_vXi16'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = sext i16 undef to i64
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i64 = sext <2 x i16> undef to <2 x i64>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i64 = sext <4 x i16> undef to <4 x i64>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8i64 = sext <8 x i16> undef to <8 x i64>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = sext i16 undef to i32
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i32 = sext <2 x i16> undef to <2 x i32>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = sext <4 x i16> undef to <4 x i32>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8i32 = sext <8 x i16> undef to <8 x i32>
@@ -517,9 +610,11 @@ define i32 @sext_vXi16() {
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX1-LABEL: 'sext_vXi16'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = sext i16 undef to i64
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i64 = sext <2 x i16> undef to <2 x i64>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4i64 = sext <4 x i16> undef to <4 x i64>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V8i64 = sext <8 x i16> undef to <8 x i64>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4i64 = sext <4 x i16> undef to <4 x i64>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V8i64 = sext <8 x i16> undef to <8 x i64>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = sext i16 undef to i32
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i32 = sext <2 x i16> undef to <2 x i32>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = sext <4 x i16> undef to <4 x i32>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8i32 = sext <8 x i16> undef to <8 x i32>
@@ -527,9 +622,11 @@ define i32 @sext_vXi16() {
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX2-LABEL: 'sext_vXi16'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = sext i16 undef to i64
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i64 = sext <2 x i16> undef to <2 x i64>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4i64 = sext <4 x i16> undef to <4 x i64>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8i64 = sext <8 x i16> undef to <8 x i64>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i64 = sext <4 x i16> undef to <4 x i64>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8i64 = sext <8 x i16> undef to <8 x i64>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = sext i16 undef to i32
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i32 = sext <2 x i16> undef to <2 x i32>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = sext <4 x i16> undef to <4 x i32>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i32 = sext <8 x i16> undef to <8 x i32>
@@ -537,9 +634,11 @@ define i32 @sext_vXi16() {
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512-LABEL: 'sext_vXi16'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = sext i16 undef to i64
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i64 = sext <2 x i16> undef to <2 x i64>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4i64 = sext <4 x i16> undef to <4 x i64>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i64 = sext <4 x i16> undef to <4 x i64>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i64 = sext <8 x i16> undef to <8 x i64>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = sext i16 undef to i32
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i32 = sext <2 x i16> undef to <2 x i32>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = sext <4 x i16> undef to <4 x i32>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i32 = sext <8 x i16> undef to <8 x i32>
@@ -547,19 +646,23 @@ define i32 @sext_vXi16() {
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; BTVER2-LABEL: 'sext_vXi16'
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = sext i16 undef to i64
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i64 = sext <2 x i16> undef to <2 x i64>
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4i64 = sext <4 x i16> undef to <4 x i64>
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V8i64 = sext <8 x i16> undef to <8 x i64>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4i64 = sext <4 x i16> undef to <4 x i64>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V8i64 = sext <8 x i16> undef to <8 x i64>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = sext i16 undef to i32
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i32 = sext <2 x i16> undef to <2 x i32>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = sext <4 x i16> undef to <4 x i32>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8i32 = sext <8 x i16> undef to <8 x i32>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16i32 = sext <16 x i16> undef to <16 x i32>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
+  %I64 = sext i16 undef to i64
   %V2i64 = sext <2 x i16> undef to <2 x i64>
   %V4i64 = sext <4 x i16> undef to <4 x i64>
   %V8i64 = sext <8 x i16> undef to <8 x i64>
 
+  %I32 = sext i16 undef to i32
   %V2i32 = sext <2 x i16> undef to <2 x i32>
   %V4i32 = sext <4 x i16> undef to <4 x i32>
   %V8i32 = sext <8 x i16> undef to <8 x i32>
@@ -570,13 +673,16 @@ define i32 @sext_vXi16() {
 
 define i32 @sext_vXi8() {
 ; SSE2-LABEL: 'sext_vXi8'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = sext i8 undef to i64
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i64 = sext <2 x i8> undef to <2 x i64>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4i64 = sext <4 x i8> undef to <4 x i64>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8i64 = sext <8 x i8> undef to <8 x i64>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = sext i8 undef to i32
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i32 = sext <2 x i8> undef to <2 x i32>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4i32 = sext <4 x i8> undef to <4 x i32>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8i32 = sext <8 x i8> undef to <8 x i32>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16i32 = sext <16 x i8> undef to <16 x i32>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = sext i8 undef to i16
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i16 = sext <2 x i8> undef to <2 x i16>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4i16 = sext <4 x i8> undef to <4 x i16>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8i16 = sext <8 x i8> undef to <8 x i16>
@@ -585,13 +691,16 @@ define i32 @sext_vXi8() {
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'sext_vXi8'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = sext i8 undef to i64
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i64 = sext <2 x i8> undef to <2 x i64>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4i64 = sext <4 x i8> undef to <4 x i64>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8i64 = sext <8 x i8> undef to <8 x i64>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = sext i8 undef to i32
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i32 = sext <2 x i8> undef to <2 x i32>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4i32 = sext <4 x i8> undef to <4 x i32>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8i32 = sext <8 x i8> undef to <8 x i32>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16i32 = sext <16 x i8> undef to <16 x i32>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = sext i8 undef to i16
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i16 = sext <2 x i8> undef to <2 x i16>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4i16 = sext <4 x i8> undef to <4 x i16>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8i16 = sext <8 x i8> undef to <8 x i16>
@@ -600,13 +709,16 @@ define i32 @sext_vXi8() {
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE42-LABEL: 'sext_vXi8'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = sext i8 undef to i64
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i64 = sext <2 x i8> undef to <2 x i64>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i64 = sext <4 x i8> undef to <4 x i64>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8i64 = sext <8 x i8> undef to <8 x i64>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = sext i8 undef to i32
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i32 = sext <2 x i8> undef to <2 x i32>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = sext <4 x i8> undef to <4 x i32>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8i32 = sext <8 x i8> undef to <8 x i32>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16i32 = sext <16 x i8> undef to <16 x i32>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = sext i8 undef to i16
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i16 = sext <2 x i8> undef to <2 x i16>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i16 = sext <4 x i8> undef to <4 x i16>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i16 = sext <8 x i8> undef to <8 x i16>
@@ -615,13 +727,16 @@ define i32 @sext_vXi8() {
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX1-LABEL: 'sext_vXi8'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = sext i8 undef to i64
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i64 = sext <2 x i8> undef to <2 x i64>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4i64 = sext <4 x i8> undef to <4 x i64>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V8i64 = sext <8 x i8> undef to <8 x i64>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4i64 = sext <4 x i8> undef to <4 x i64>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V8i64 = sext <8 x i8> undef to <8 x i64>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = sext i8 undef to i32
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i32 = sext <2 x i8> undef to <2 x i32>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = sext <4 x i8> undef to <4 x i32>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8i32 = sext <8 x i8> undef to <8 x i32>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8i32 = sext <8 x i8> undef to <8 x i32>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16i32 = sext <16 x i8> undef to <16 x i32>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = sext i8 undef to i16
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i16 = sext <2 x i8> undef to <2 x i16>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i16 = sext <4 x i8> undef to <4 x i16>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i16 = sext <8 x i8> undef to <8 x i16>
@@ -630,13 +745,16 @@ define i32 @sext_vXi8() {
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX2-LABEL: 'sext_vXi8'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = sext i8 undef to i64
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i64 = sext <2 x i8> undef to <2 x i64>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4i64 = sext <4 x i8> undef to <4 x i64>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8i64 = sext <8 x i8> undef to <8 x i64>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i64 = sext <4 x i8> undef to <4 x i64>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8i64 = sext <8 x i8> undef to <8 x i64>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = sext i8 undef to i32
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i32 = sext <2 x i8> undef to <2 x i32>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = sext <4 x i8> undef to <4 x i32>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8i32 = sext <8 x i8> undef to <8 x i32>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i32 = sext <8 x i8> undef to <8 x i32>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16i32 = sext <16 x i8> undef to <16 x i32>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = sext i8 undef to i16
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i16 = sext <2 x i8> undef to <2 x i16>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i16 = sext <4 x i8> undef to <4 x i16>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i16 = sext <8 x i8> undef to <8 x i16>
@@ -645,13 +763,16 @@ define i32 @sext_vXi8() {
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512F-LABEL: 'sext_vXi8'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = sext i8 undef to i64
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i64 = sext <2 x i8> undef to <2 x i64>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4i64 = sext <4 x i8> undef to <4 x i64>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8i64 = sext <8 x i8> undef to <8 x i64>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i64 = sext <4 x i8> undef to <4 x i64>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i64 = sext <8 x i8> undef to <8 x i64>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = sext i8 undef to i32
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i32 = sext <2 x i8> undef to <2 x i32>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = sext <4 x i8> undef to <4 x i32>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8i32 = sext <8 x i8> undef to <8 x i32>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i32 = sext <8 x i8> undef to <8 x i32>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16i32 = sext <16 x i8> undef to <16 x i32>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = sext i8 undef to i16
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i16 = sext <2 x i8> undef to <2 x i16>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i16 = sext <4 x i8> undef to <4 x i16>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i16 = sext <8 x i8> undef to <8 x i16>
@@ -660,13 +781,16 @@ define i32 @sext_vXi8() {
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512BW-LABEL: 'sext_vXi8'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = sext i8 undef to i64
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i64 = sext <2 x i8> undef to <2 x i64>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4i64 = sext <4 x i8> undef to <4 x i64>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8i64 = sext <8 x i8> undef to <8 x i64>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i64 = sext <4 x i8> undef to <4 x i64>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i64 = sext <8 x i8> undef to <8 x i64>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = sext i8 undef to i32
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i32 = sext <2 x i8> undef to <2 x i32>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = sext <4 x i8> undef to <4 x i32>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8i32 = sext <8 x i8> undef to <8 x i32>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i32 = sext <8 x i8> undef to <8 x i32>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16i32 = sext <16 x i8> undef to <16 x i32>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = sext i8 undef to i16
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i16 = sext <2 x i8> undef to <2 x i16>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i16 = sext <4 x i8> undef to <4 x i16>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i16 = sext <8 x i8> undef to <8 x i16>
@@ -675,13 +799,16 @@ define i32 @sext_vXi8() {
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; BTVER2-LABEL: 'sext_vXi8'
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = sext i8 undef to i64
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i64 = sext <2 x i8> undef to <2 x i64>
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4i64 = sext <4 x i8> undef to <4 x i64>
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V8i64 = sext <8 x i8> undef to <8 x i64>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4i64 = sext <4 x i8> undef to <4 x i64>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V8i64 = sext <8 x i8> undef to <8 x i64>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = sext i8 undef to i32
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i32 = sext <2 x i8> undef to <2 x i32>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = sext <4 x i8> undef to <4 x i32>
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8i32 = sext <8 x i8> undef to <8 x i32>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8i32 = sext <8 x i8> undef to <8 x i32>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16i32 = sext <16 x i8> undef to <16 x i32>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = sext i8 undef to i16
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i16 = sext <2 x i8> undef to <2 x i16>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i16 = sext <4 x i8> undef to <4 x i16>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i16 = sext <8 x i8> undef to <8 x i16>
@@ -689,15 +816,18 @@ define i32 @sext_vXi8() {
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V32i16 = sext <32 x i8> undef to <32 x i16>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
+  %I64 = sext i8 undef to i64
   %V2i64 = sext <2 x i8> undef to <2 x i64>
   %V4i64 = sext <4 x i8> undef to <4 x i64>
   %V8i64 = sext <8 x i8> undef to <8 x i64>
 
+  %I32 = sext i8 undef to i32
   %V2i32 = sext <2 x i8> undef to <2 x i32>
   %V4i32 = sext <4 x i8> undef to <4 x i32>
   %V8i32 = sext <8 x i8> undef to <8 x i32>
   %V16i32 = sext <16 x i8> undef to <16 x i32>
 
+  %I16 = sext i8 undef to i16
   %V2i16 = sext <2 x i8> undef to <2 x i16>
   %V4i16 = sext <4 x i8> undef to <4 x i16>
   %V8i16 = sext <8 x i8> undef to <8 x i16>
@@ -709,18 +839,22 @@ define i32 @sext_vXi8() {
 
 define i32 @sext_vXi1() {
 ; SSE-LABEL: 'sext_vXi1'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = sext i1 undef to i64
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i64 = sext <2 x i1> undef to <2 x i64>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4i64 = sext <4 x i1> undef to <4 x i64>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V8i64 = sext <8 x i1> undef to <8 x i64>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = sext i1 undef to i32
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i32 = sext <2 x i1> undef to <2 x i32>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i32 = sext <4 x i1> undef to <4 x i32>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8i32 = sext <8 x i1> undef to <8 x i32>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V16i32 = sext <16 x i1> undef to <16 x i32>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = sext i1 undef to i16
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i16 = sext <2 x i1> undef to <2 x i16>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i16 = sext <4 x i1> undef to <4 x i16>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8i16 = sext <8 x i1> undef to <8 x i16>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V16i16 = sext <16 x i1> undef to <16 x i16>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V32i16 = sext <32 x i1> undef to <32 x i16>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = sext i1 undef to i8
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i8 = sext <2 x i1> undef to <2 x i8>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i8 = sext <4 x i1> undef to <4 x i8>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8i8 = sext <8 x i1> undef to <8 x i8>
@@ -730,18 +864,22 @@ define i32 @sext_vXi1() {
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX1-LABEL: 'sext_vXi1'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = sext i1 undef to i64
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i64 = sext <2 x i1> undef to <2 x i64>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4i64 = sext <4 x i1> undef to <4 x i64>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V8i64 = sext <8 x i1> undef to <8 x i64>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = sext i1 undef to i32
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i32 = sext <2 x i1> undef to <2 x i32>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i32 = sext <4 x i1> undef to <4 x i32>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8i32 = sext <8 x i1> undef to <8 x i32>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V16i32 = sext <16 x i1> undef to <16 x i32>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = sext i1 undef to i16
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i16 = sext <2 x i1> undef to <2 x i16>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i16 = sext <4 x i1> undef to <4 x i16>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8i16 = sext <8 x i1> undef to <8 x i16>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16i16 = sext <16 x i1> undef to <16 x i16>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 97 for instruction: %V32i16 = sext <32 x i1> undef to <32 x i16>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = sext i1 undef to i8
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i8 = sext <2 x i1> undef to <2 x i8>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i8 = sext <4 x i1> undef to <4 x i8>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8i8 = sext <8 x i1> undef to <8 x i8>
@@ -751,18 +889,22 @@ define i32 @sext_vXi1() {
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX2-LABEL: 'sext_vXi1'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = sext i1 undef to i64
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i64 = sext <2 x i1> undef to <2 x i64>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4i64 = sext <4 x i1> undef to <4 x i64>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8i64 = sext <8 x i1> undef to <8 x i64>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = sext i1 undef to i32
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i32 = sext <2 x i1> undef to <2 x i32>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i32 = sext <4 x i1> undef to <4 x i32>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8i32 = sext <8 x i1> undef to <8 x i32>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16i32 = sext <16 x i1> undef to <16 x i32>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = sext i1 undef to i16
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i16 = sext <2 x i1> undef to <2 x i16>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i16 = sext <4 x i1> undef to <4 x i16>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8i16 = sext <8 x i1> undef to <8 x i16>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16i16 = sext <16 x i1> undef to <16 x i16>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 97 for instruction: %V32i16 = sext <32 x i1> undef to <32 x i16>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = sext i1 undef to i8
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i8 = sext <2 x i1> undef to <2 x i8>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i8 = sext <4 x i1> undef to <4 x i8>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8i8 = sext <8 x i1> undef to <8 x i8>
@@ -772,18 +914,22 @@ define i32 @sext_vXi1() {
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512F-LABEL: 'sext_vXi1'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = sext i1 undef to i64
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2i64 = sext <2 x i1> undef to <2 x i64>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4i64 = sext <4 x i1> undef to <4 x i64>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8i64 = sext <8 x i1> undef to <8 x i64>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = sext i1 undef to i32
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2i32 = sext <2 x i1> undef to <2 x i32>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4i32 = sext <4 x i1> undef to <4 x i32>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8i32 = sext <8 x i1> undef to <8 x i32>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16i32 = sext <16 x i1> undef to <16 x i32>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = sext i1 undef to i16
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2i16 = sext <2 x i1> undef to <2 x i16>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4i16 = sext <4 x i1> undef to <4 x i16>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8i16 = sext <8 x i1> undef to <8 x i16>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16i16 = sext <16 x i1> undef to <16 x i16>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 97 for instruction: %V32i16 = sext <32 x i1> undef to <32 x i16>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = sext i1 undef to i8
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2i8 = sext <2 x i1> undef to <2 x i8>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4i8 = sext <4 x i1> undef to <4 x i8>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8i8 = sext <8 x i1> undef to <8 x i8>
@@ -793,18 +939,22 @@ define i32 @sext_vXi1() {
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512BW-LABEL: 'sext_vXi1'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = sext i1 undef to i64
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2i64 = sext <2 x i1> undef to <2 x i64>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4i64 = sext <4 x i1> undef to <4 x i64>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8i64 = sext <8 x i1> undef to <8 x i64>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = sext i1 undef to i32
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2i32 = sext <2 x i1> undef to <2 x i32>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4i32 = sext <4 x i1> undef to <4 x i32>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8i32 = sext <8 x i1> undef to <8 x i32>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16i32 = sext <16 x i1> undef to <16 x i32>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = sext i1 undef to i16
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2i16 = sext <2 x i1> undef to <2 x i16>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4i16 = sext <4 x i1> undef to <4 x i16>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i16 = sext <8 x i1> undef to <8 x i16>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16i16 = sext <16 x i1> undef to <16 x i16>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32i16 = sext <32 x i1> undef to <32 x i16>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = sext i1 undef to i8
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2i8 = sext <2 x i1> undef to <2 x i8>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4i8 = sext <4 x i1> undef to <4 x i8>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8i8 = sext <8 x i1> undef to <8 x i8>
@@ -814,18 +964,22 @@ define i32 @sext_vXi1() {
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; BTVER2-LABEL: 'sext_vXi1'
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = sext i1 undef to i64
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i64 = sext <2 x i1> undef to <2 x i64>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4i64 = sext <4 x i1> undef to <4 x i64>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V8i64 = sext <8 x i1> undef to <8 x i64>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = sext i1 undef to i32
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i32 = sext <2 x i1> undef to <2 x i32>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i32 = sext <4 x i1> undef to <4 x i32>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8i32 = sext <8 x i1> undef to <8 x i32>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V16i32 = sext <16 x i1> undef to <16 x i32>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = sext i1 undef to i16
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i16 = sext <2 x i1> undef to <2 x i16>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i16 = sext <4 x i1> undef to <4 x i16>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8i16 = sext <8 x i1> undef to <8 x i16>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16i16 = sext <16 x i1> undef to <16 x i16>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 97 for instruction: %V32i16 = sext <32 x i1> undef to <32 x i16>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = sext i1 undef to i8
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i8 = sext <2 x i1> undef to <2 x i8>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i8 = sext <4 x i1> undef to <4 x i8>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8i8 = sext <8 x i1> undef to <8 x i8>
@@ -834,21 +988,25 @@ define i32 @sext_vXi1() {
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64i8 = sext <64 x i1> undef to <64 x i8>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
+  %I64 = sext i1 undef to i64
   %V2i64 = sext <2 x i1> undef to <2 x i64>
   %V4i64 = sext <4 x i1> undef to <4 x i64>
   %V8i64 = sext <8 x i1> undef to <8 x i64>
 
+  %I32 = sext i1 undef to i32
   %V2i32 = sext <2 x i1> undef to <2 x i32>
   %V4i32 = sext <4 x i1> undef to <4 x i32>
   %V8i32 = sext <8 x i1> undef to <8 x i32>
   %V16i32 = sext <16 x i1> undef to <16 x i32>
 
+  %I16 = sext i1 undef to i16
   %V2i16 = sext <2 x i1> undef to <2 x i16>
   %V4i16 = sext <4 x i1> undef to <4 x i16>
   %V8i16 = sext <8 x i1> undef to <8 x i16>
   %V16i16 = sext <16 x i1> undef to <16 x i16>
   %V32i16 = sext <32 x i1> undef to <32 x i16>
 
+  %I8 = sext i1 undef to i8
   %V2i8 = sext <2 x i1> undef to <2 x i8>
   %V4i8 = sext <4 x i1> undef to <4 x i8>
   %V8i8 = sext <8 x i1> undef to <8 x i8>
diff --git a/llvm/test/Analysis/CostModel/X86/fptosi.ll b/llvm/test/Analysis/CostModel/X86/fptosi.ll
index ff4050872..35a86fb13 100644
--- a/llvm/test/Analysis/CostModel/X86/fptosi.ll
+++ b/llvm/test/Analysis/CostModel/X86/fptosi.ll
@@ -6,7 +6,7 @@
 ; RUN: opt < %s -mtriple=x86_64-apple-darwin -cost-model -analyze -mattr=+avx512f | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F
 ; RUN: opt < %s -mtriple=x86_64-apple-darwin -cost-model -analyze -mattr=+avx512f,+avx512dq | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512DQ
 ;
-; RUN: opt < %s -mtriple=x86_64-apple-darwin -cost-model -analyze -mcpu=slm | FileCheck %s --check-prefixes=CHECK,SSE,SSE42
+; RUN: opt < %s -mtriple=x86_64-apple-darwin -cost-model -analyze -mcpu=slm | FileCheck %s --check-prefixes=SLM
 ; RUN: opt < %s -mtriple=x86_64-apple-darwin -cost-model -analyze -mcpu=goldmont | FileCheck %s --check-prefixes=CHECK,SSE,SSE42
 ; RUN: opt < %s -mtriple=x86_64-apple-darwin -cost-model -analyze -mcpu=btver2 | FileCheck %s --check-prefixes=BTVER2
 
@@ -39,6 +39,13 @@ define i32 @fptosi_double_i64(i32 %arg) {
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = fptosi <8 x double> undef to <8 x i64>
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
+; SLM-LABEL: 'fptosi_double_i64'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = fptosi double undef to i64
+; SLM-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2I64 = fptosi <2 x double> undef to <2 x i64>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V4I64 = fptosi <4 x double> undef to <4 x i64>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %V8I64 = fptosi <8 x double> undef to <8 x i64>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
 ; BTVER2-LABEL: 'fptosi_double_i64'
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = fptosi double undef to i64
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = fptosi <2 x double> undef to <2 x i64>
@@ -75,6 +82,13 @@ define i32 @fptosi_double_i32(i32 %arg) {
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = fptosi <8 x double> undef to <8 x i32>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
+; SLM-LABEL: 'fptosi_double_i32'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = fptosi double undef to i32
+; SLM-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2I32 = fptosi <2 x double> undef to <2 x i32>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4I32 = fptosi <4 x double> undef to <4 x i32>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V8I32 = fptosi <8 x double> undef to <8 x i32>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
 ; BTVER2-LABEL: 'fptosi_double_i32'
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = fptosi double undef to i32
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2I32 = fptosi <2 x double> undef to <2 x i32>
@@ -92,35 +106,35 @@ define i32 @fptosi_double_i32(i32 %arg) {
 define i32 @fptosi_double_i16(i32 %arg) {
 ; SSE-LABEL: 'fptosi_double_i16'
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = fptosi double undef to i16
-; SSE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I16 = fptosi <2 x double> undef to <2 x i16>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V4I16 = fptosi <4 x double> undef to <4 x i16>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %V8I16 = fptosi <8 x double> undef to <8 x i16>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I16 = fptosi <2 x double> undef to <2 x i16>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4I16 = fptosi <4 x double> undef to <4 x i16>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V8I16 = fptosi <8 x double> undef to <8 x i16>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX-LABEL: 'fptosi_double_i16'
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = fptosi double undef to i16
-; AVX-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I16 = fptosi <2 x double> undef to <2 x i16>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I16 = fptosi <2 x double> undef to <2 x i16>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I16 = fptosi <4 x double> undef to <4 x i16>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I16 = fptosi <8 x double> undef to <8 x i16>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
-; AVX512F-LABEL: 'fptosi_double_i16'
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = fptosi double undef to i16
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I16 = fptosi <2 x double> undef to <2 x i16>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I16 = fptosi <4 x double> undef to <4 x i16>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = fptosi <8 x double> undef to <8 x i16>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+; AVX512-LABEL: 'fptosi_double_i16'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = fptosi double undef to i16
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I16 = fptosi <2 x double> undef to <2 x i16>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I16 = fptosi <4 x double> undef to <4 x i16>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = fptosi <8 x double> undef to <8 x i16>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
-; AVX512DQ-LABEL: 'fptosi_double_i16'
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = fptosi double undef to i16
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I16 = fptosi <2 x double> undef to <2 x i16>
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I16 = fptosi <4 x double> undef to <4 x i16>
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = fptosi <8 x double> undef to <8 x i16>
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+; SLM-LABEL: 'fptosi_double_i16'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = fptosi double undef to i16
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I16 = fptosi <2 x double> undef to <2 x i16>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4I16 = fptosi <4 x double> undef to <4 x i16>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V8I16 = fptosi <8 x double> undef to <8 x i16>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; BTVER2-LABEL: 'fptosi_double_i16'
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = fptosi double undef to i16
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I16 = fptosi <2 x double> undef to <2 x i16>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I16 = fptosi <2 x double> undef to <2 x i16>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I16 = fptosi <4 x double> undef to <4 x i16>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I16 = fptosi <8 x double> undef to <8 x i16>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
@@ -143,29 +157,29 @@ define i32 @fptosi_double_i8(i32 %arg) {
 ; AVX-LABEL: 'fptosi_double_i8'
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = fptosi double undef to i8
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I8 = fptosi <2 x double> undef to <2 x i8>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I8 = fptosi <4 x double> undef to <4 x i8>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I8 = fptosi <8 x double> undef to <8 x i8>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I8 = fptosi <4 x double> undef to <4 x i8>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V8I8 = fptosi <8 x double> undef to <8 x i8>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
-; AVX512F-LABEL: 'fptosi_double_i8'
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = fptosi double undef to i8
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I8 = fptosi <2 x double> undef to <2 x i8>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I8 = fptosi <4 x double> undef to <4 x i8>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I8 = fptosi <8 x double> undef to <8 x i8>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+; AVX512-LABEL: 'fptosi_double_i8'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = fptosi double undef to i8
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I8 = fptosi <2 x double> undef to <2 x i8>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I8 = fptosi <4 x double> undef to <4 x i8>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I8 = fptosi <8 x double> undef to <8 x i8>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
-; AVX512DQ-LABEL: 'fptosi_double_i8'
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = fptosi double undef to i8
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I8 = fptosi <2 x double> undef to <2 x i8>
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I8 = fptosi <4 x double> undef to <4 x i8>
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I8 = fptosi <8 x double> undef to <8 x i8>
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+; SLM-LABEL: 'fptosi_double_i8'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = fptosi double undef to i8
+; SLM-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2I8 = fptosi <2 x double> undef to <2 x i8>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V4I8 = fptosi <4 x double> undef to <4 x i8>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: %V8I8 = fptosi <8 x double> undef to <8 x i8>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; BTVER2-LABEL: 'fptosi_double_i8'
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = fptosi double undef to i8
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I8 = fptosi <2 x double> undef to <2 x i8>
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I8 = fptosi <4 x double> undef to <4 x i8>
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I8 = fptosi <8 x double> undef to <8 x i8>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I8 = fptosi <4 x double> undef to <4 x i8>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V8I8 = fptosi <8 x double> undef to <8 x i8>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %I8 = fptosi double undef to i8
@@ -208,6 +222,14 @@ define i32 @fptosi_float_i64(i32 %arg) {
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I64 = fptosi <16 x float> undef to <16 x i64>
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
+; SLM-LABEL: 'fptosi_float_i64'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = fptosi float undef to i64
+; SLM-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2I64 = fptosi <2 x float> undef to <2 x i64>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V4I64 = fptosi <4 x float> undef to <4 x i64>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %V8I64 = fptosi <8 x float> undef to <8 x i64>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 103 for instruction: %V16I64 = fptosi <16 x float> undef to <16 x i64>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
 ; BTVER2-LABEL: 'fptosi_float_i64'
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = fptosi float undef to i64
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = fptosi <2 x float> undef to <2 x i64>
@@ -232,6 +254,13 @@ define i32 @fptosi_float_i32(i32 %arg) {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I32 = fptosi <16 x float> undef to <16 x i32>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
+; SLM-LABEL: 'fptosi_float_i32'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = fptosi float undef to i32
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = fptosi <4 x float> undef to <4 x i32>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = fptosi <8 x float> undef to <8 x i32>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I32 = fptosi <16 x float> undef to <16 x i32>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
 ; BTVER2-LABEL: 'fptosi_float_i32'
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = fptosi float undef to i32
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = fptosi <4 x float> undef to <4 x i32>
@@ -249,28 +278,35 @@ define i32 @fptosi_float_i32(i32 %arg) {
 define i32 @fptosi_float_i16(i32 %arg) {
 ; SSE-LABEL: 'fptosi_float_i16'
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = fptosi float undef to i16
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I16 = fptosi <4 x float> undef to <4 x i16>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I16 = fptosi <8 x float> undef to <8 x i16>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16I16 = fptosi <16 x float> undef to <16 x i16>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I16 = fptosi <4 x float> undef to <4 x i16>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8I16 = fptosi <8 x float> undef to <8 x i16>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V16I16 = fptosi <16 x float> undef to <16 x i16>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX-LABEL: 'fptosi_float_i16'
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = fptosi float undef to i16
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I16 = fptosi <4 x float> undef to <4 x i16>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I16 = fptosi <4 x float> undef to <4 x i16>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = fptosi <8 x float> undef to <8 x i16>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I16 = fptosi <16 x float> undef to <16 x i16>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512-LABEL: 'fptosi_float_i16'
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = fptosi float undef to i16
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I16 = fptosi <4 x float> undef to <4 x i16>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I16 = fptosi <4 x float> undef to <4 x i16>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = fptosi <8 x float> undef to <8 x i16>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = fptosi <16 x float> undef to <16 x i16>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
+; SLM-LABEL: 'fptosi_float_i16'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = fptosi float undef to i16
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I16 = fptosi <4 x float> undef to <4 x i16>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8I16 = fptosi <8 x float> undef to <8 x i16>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V16I16 = fptosi <16 x float> undef to <16 x i16>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
 ; BTVER2-LABEL: 'fptosi_float_i16'
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = fptosi float undef to i16
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I16 = fptosi <4 x float> undef to <4 x i16>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I16 = fptosi <4 x float> undef to <4 x i16>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = fptosi <8 x float> undef to <8 x i16>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I16 = fptosi <16 x float> undef to <16 x i16>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
@@ -285,9 +321,9 @@ define i32 @fptosi_float_i16(i32 %arg) {
 define i32 @fptosi_float_i8(i32 %arg) {
 ; SSE-LABEL: 'fptosi_float_i8'
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = fptosi float undef to i8
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I8 = fptosi <4 x float> undef to <4 x i8>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I8 = fptosi <8 x float> undef to <8 x i8>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16I8 = fptosi <16 x float> undef to <16 x i8>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I8 = fptosi <4 x float> undef to <4 x i8>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V8I8 = fptosi <8 x float> undef to <8 x i8>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %V16I8 = fptosi <16 x float> undef to <16 x i8>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX-LABEL: 'fptosi_float_i8'
@@ -304,6 +340,13 @@ define i32 @fptosi_float_i8(i32 %arg) {
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = fptosi <16 x float> undef to <16 x i8>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
+; SLM-LABEL: 'fptosi_float_i8'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = fptosi float undef to i8
+; SLM-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V4I8 = fptosi <4 x float> undef to <4 x i8>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %V8I8 = fptosi <8 x float> undef to <8 x i8>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 87 for instruction: %V16I8 = fptosi <16 x float> undef to <16 x i8>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
 ; BTVER2-LABEL: 'fptosi_float_i8'
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = fptosi float undef to i8
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I8 = fptosi <4 x float> undef to <4 x i8>
diff --git a/llvm/test/Analysis/CostModel/X86/fptoui.ll b/llvm/test/Analysis/CostModel/X86/fptoui.ll
index f13781b4e..c3b2cb513 100644
--- a/llvm/test/Analysis/CostModel/X86/fptoui.ll
+++ b/llvm/test/Analysis/CostModel/X86/fptoui.ll
@@ -6,7 +6,7 @@
 ; RUN: opt < %s -mtriple=x86_64-apple-darwin -cost-model -analyze -mattr=+avx512f | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F
 ; RUN: opt < %s -mtriple=x86_64-apple-darwin -cost-model -analyze -mattr=+avx512f,+avx512dq | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512DQ
 ;
-; RUN: opt < %s -mtriple=x86_64-apple-darwin -cost-model -analyze -mcpu=slm | FileCheck %s --check-prefixes=CHECK,SSE,SSE42
+; RUN: opt < %s -mtriple=x86_64-apple-darwin -cost-model -analyze -mcpu=slm | FileCheck %s --check-prefixes=SLM
 ; RUN: opt < %s -mtriple=x86_64-apple-darwin -cost-model -analyze -mcpu=goldmont | FileCheck %s --check-prefixes=CHECK,SSE,SSE42
 ; RUN: opt < %s -mtriple=x86_64-apple-darwin -cost-model -analyze -mcpu=btver2 | FileCheck %s --check-prefixes=BTVER2
 
@@ -39,6 +39,13 @@ define i32 @fptoui_double_i64(i32 %arg) {
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = fptoui <8 x double> undef to <8 x i64>
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
+; SLM-LABEL: 'fptoui_double_i64'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I64 = fptoui double undef to i64
+; SLM-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V2I64 = fptoui <2 x double> undef to <2 x i64>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 37 for instruction: %V4I64 = fptoui <4 x double> undef to <4 x i64>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 75 for instruction: %V8I64 = fptoui <8 x double> undef to <8 x i64>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
 ; BTVER2-LABEL: 'fptoui_double_i64'
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I64 = fptoui double undef to i64
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2I64 = fptoui <2 x double> undef to <2 x i64>
@@ -68,19 +75,19 @@ define i32 @fptoui_double_i32(i32 %arg) {
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V8I32 = fptoui <8 x double> undef to <8 x i32>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
-; AVX512F-LABEL: 'fptoui_double_i32'
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = fptoui double undef to i32
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I32 = fptoui <2 x double> undef to <2 x i32>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = fptoui <4 x double> undef to <4 x i32>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = fptoui <8 x double> undef to <8 x i32>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+; AVX512-LABEL: 'fptoui_double_i32'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = fptoui double undef to i32
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = fptoui <2 x double> undef to <2 x i32>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = fptoui <4 x double> undef to <4 x i32>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = fptoui <8 x double> undef to <8 x i32>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
-; AVX512DQ-LABEL: 'fptoui_double_i32'
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = fptoui double undef to i32
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = fptoui <2 x double> undef to <2 x i32>
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = fptoui <4 x double> undef to <4 x i32>
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = fptoui <8 x double> undef to <8 x i32>
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+; SLM-LABEL: 'fptoui_double_i32'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = fptoui double undef to i32
+; SLM-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2I32 = fptoui <2 x double> undef to <2 x i32>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V4I32 = fptoui <4 x double> undef to <4 x i32>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: %V8I32 = fptoui <8 x double> undef to <8 x i32>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; BTVER2-LABEL: 'fptoui_double_i32'
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = fptoui double undef to i32
@@ -106,30 +113,30 @@ define i32 @fptoui_double_i16(i32 %arg) {
 ;
 ; AVX-LABEL: 'fptoui_double_i16'
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = fptoui double undef to i16
-; AVX-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I16 = fptoui <2 x double> undef to <2 x i16>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I16 = fptoui <4 x double> undef to <4 x i16>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V8I16 = fptoui <8 x double> undef to <8 x i16>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I16 = fptoui <2 x double> undef to <2 x i16>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I16 = fptoui <4 x double> undef to <4 x i16>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I16 = fptoui <8 x double> undef to <8 x i16>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
-; AVX512F-LABEL: 'fptoui_double_i16'
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = fptoui double undef to i16
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I16 = fptoui <2 x double> undef to <2 x i16>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I16 = fptoui <4 x double> undef to <4 x i16>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = fptoui <8 x double> undef to <8 x i16>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+; AVX512-LABEL: 'fptoui_double_i16'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = fptoui double undef to i16
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I16 = fptoui <2 x double> undef to <2 x i16>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I16 = fptoui <4 x double> undef to <4 x i16>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = fptoui <8 x double> undef to <8 x i16>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
-; AVX512DQ-LABEL: 'fptoui_double_i16'
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = fptoui double undef to i16
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I16 = fptoui <2 x double> undef to <2 x i16>
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I16 = fptoui <4 x double> undef to <4 x i16>
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = fptoui <8 x double> undef to <8 x i16>
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+; SLM-LABEL: 'fptoui_double_i16'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = fptoui double undef to i16
+; SLM-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2I16 = fptoui <2 x double> undef to <2 x i16>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V4I16 = fptoui <4 x double> undef to <4 x i16>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: %V8I16 = fptoui <8 x double> undef to <8 x i16>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; BTVER2-LABEL: 'fptoui_double_i16'
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = fptoui double undef to i16
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I16 = fptoui <2 x double> undef to <2 x i16>
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I16 = fptoui <4 x double> undef to <4 x i16>
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V8I16 = fptoui <8 x double> undef to <8 x i16>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I16 = fptoui <2 x double> undef to <2 x i16>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I16 = fptoui <4 x double> undef to <4 x i16>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I16 = fptoui <8 x double> undef to <8 x i16>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %I16 = fptoui double undef to i16
@@ -154,19 +161,19 @@ define i32 @fptoui_double_i8(i32 %arg) {
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V8I8 = fptoui <8 x double> undef to <8 x i8>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
-; AVX512F-LABEL: 'fptoui_double_i8'
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = fptoui double undef to i8
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I8 = fptoui <2 x double> undef to <2 x i8>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I8 = fptoui <4 x double> undef to <4 x i8>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I8 = fptoui <8 x double> undef to <8 x i8>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+; AVX512-LABEL: 'fptoui_double_i8'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = fptoui double undef to i8
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I8 = fptoui <2 x double> undef to <2 x i8>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I8 = fptoui <4 x double> undef to <4 x i8>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I8 = fptoui <8 x double> undef to <8 x i8>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
-; AVX512DQ-LABEL: 'fptoui_double_i8'
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = fptoui double undef to i8
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I8 = fptoui <2 x double> undef to <2 x i8>
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I8 = fptoui <4 x double> undef to <4 x i8>
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I8 = fptoui <8 x double> undef to <8 x i8>
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+; SLM-LABEL: 'fptoui_double_i8'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = fptoui double undef to i8
+; SLM-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2I8 = fptoui <2 x double> undef to <2 x i8>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V4I8 = fptoui <4 x double> undef to <4 x i8>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: %V8I8 = fptoui <8 x double> undef to <8 x i8>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; BTVER2-LABEL: 'fptoui_double_i8'
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = fptoui double undef to i8
@@ -215,6 +222,14 @@ define i32 @fptoui_float_i64(i32 %arg) {
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I64 = fptoui <16 x float> undef to <16 x i64>
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
+; SLM-LABEL: 'fptoui_float_i64'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I64 = fptoui float undef to i64
+; SLM-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V2I64 = fptoui <2 x float> undef to <2 x i64>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 37 for instruction: %V4I64 = fptoui <4 x float> undef to <4 x i64>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 75 for instruction: %V8I64 = fptoui <8 x float> undef to <8 x i64>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 151 for instruction: %V16I64 = fptoui <16 x float> undef to <16 x i64>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
 ; BTVER2-LABEL: 'fptoui_float_i64'
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I64 = fptoui float undef to i64
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2I64 = fptoui <2 x float> undef to <2 x i64>
@@ -253,6 +268,13 @@ define i32 @fptoui_float_i32(i32 %arg) {
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I32 = fptoui <16 x float> undef to <16 x i32>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
+; SLM-LABEL: 'fptoui_float_i32'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = fptoui float undef to i32
+; SLM-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V4I32 = fptoui <4 x float> undef to <4 x i32>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %V8I32 = fptoui <8 x float> undef to <8 x i32>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 87 for instruction: %V16I32 = fptoui <16 x float> undef to <16 x i32>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
 ; BTVER2-LABEL: 'fptoui_float_i32'
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = fptoui float undef to i32
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = fptoui <4 x float> undef to <4 x i32>
@@ -277,7 +299,7 @@ define i32 @fptoui_float_i16(i32 %arg) {
 ;
 ; AVX-LABEL: 'fptoui_float_i16'
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = fptoui float undef to i16
-; AVX-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I16 = fptoui <4 x float> undef to <4 x i16>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I16 = fptoui <4 x float> undef to <4 x i16>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = fptoui <8 x float> undef to <8 x i16>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I16 = fptoui <16 x float> undef to <16 x i16>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
@@ -289,9 +311,16 @@ define i32 @fptoui_float_i16(i32 %arg) {
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = fptoui <16 x float> undef to <16 x i16>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
+; SLM-LABEL: 'fptoui_float_i16'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = fptoui float undef to i16
+; SLM-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V4I16 = fptoui <4 x float> undef to <4 x i16>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %V8I16 = fptoui <8 x float> undef to <8 x i16>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 87 for instruction: %V16I16 = fptoui <16 x float> undef to <16 x i16>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
 ; BTVER2-LABEL: 'fptoui_float_i16'
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = fptoui float undef to i16
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I16 = fptoui <4 x float> undef to <4 x i16>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I16 = fptoui <4 x float> undef to <4 x i16>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = fptoui <8 x float> undef to <8 x i16>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I16 = fptoui <16 x float> undef to <16 x i16>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
@@ -314,8 +343,8 @@ define i32 @fptoui_float_i8(i32 %arg) {
 ; AVX-LABEL: 'fptoui_float_i8'
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = fptoui float undef to i8
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I8 = fptoui <4 x float> undef to <4 x i8>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I8 = fptoui <8 x float> undef to <8 x i8>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = fptoui <16 x float> undef to <16 x i8>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I8 = fptoui <8 x float> undef to <8 x i8>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 49 for instruction: %V16I8 = fptoui <16 x float> undef to <16 x i8>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512-LABEL: 'fptoui_float_i8'
@@ -325,11 +354,18 @@ define i32 @fptoui_float_i8(i32 %arg) {
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = fptoui <16 x float> undef to <16 x i8>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
+; SLM-LABEL: 'fptoui_float_i8'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = fptoui float undef to i8
+; SLM-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V4I8 = fptoui <4 x float> undef to <4 x i8>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %V8I8 = fptoui <8 x float> undef to <8 x i8>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 87 for instruction: %V16I8 = fptoui <16 x float> undef to <16 x i8>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
 ; BTVER2-LABEL: 'fptoui_float_i8'
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = fptoui float undef to i8
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I8 = fptoui <4 x float> undef to <4 x i8>
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I8 = fptoui <8 x float> undef to <8 x i8>
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = fptoui <16 x float> undef to <16 x i8>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I8 = fptoui <8 x float> undef to <8 x i8>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 49 for instruction: %V16I8 = fptoui <16 x float> undef to <16 x i8>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %I8 = fptoui float undef to i8
diff --git a/llvm/test/Analysis/CostModel/X86/fshl.ll b/llvm/test/Analysis/CostModel/X86/fshl.ll
index 38621c35a..976fa750f 100644
--- a/llvm/test/Analysis/CostModel/X86/fshl.ll
+++ b/llvm/test/Analysis/CostModel/X86/fshl.ll
@@ -57,9 +57,9 @@ define void @var_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 x i64
 ;
 ; SLM-LABEL: 'var_funnel_i64'
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %b64, i64 %c64)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %c128)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %c256)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %c512)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %c128)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %c256)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %c512)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; GLM-LABEL: 'var_funnel_i64'
@@ -385,9 +385,9 @@ define void @splatvar_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <2 x i64> %c128, <2 x i64> undef, <2 x i32> zeroinitializer
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <4 x i64> %c256, <4 x i64> undef, <4 x i32> zeroinitializer
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <8 x i64> %c512, <8 x i64> undef, <8 x i32> zeroinitializer
-; SLM-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %u128)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %u256)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %u512)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %u128)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %u256)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %u512)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; GLM-LABEL: 'splatvar_funnel_i64'
@@ -777,9 +777,9 @@ define void @constant_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8
 ;
 ; SLM-LABEL: 'constant_funnel_i64'
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %b64, i64 7)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> <i64 1, i64 7>)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> <i64 1, i64 7, i64 15, i64 31>)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 60 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> <i64 1, i64 7, i64 15, i64 31, i64 1, i64 7, i64 15, i64 31>)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> <i64 1, i64 7>)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> <i64 1, i64 7, i64 15, i64 31>)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> <i64 1, i64 7, i64 15, i64 31, i64 1, i64 7, i64 15, i64 31>)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; GLM-LABEL: 'constant_funnel_i64'
@@ -1093,9 +1093,9 @@ define void @splatconstant_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256
 ;
 ; SLM-LABEL: 'splatconstant_funnel_i64'
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %b64, i64 7)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> <i64 7, i64 7>)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> <i64 7, i64 7, i64 7, i64 7>)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> <i64 7, i64 7, i64 7, i64 7, i64 7, i64 7, i64 7, i64 7>)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> <i64 7, i64 7>)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> <i64 7, i64 7, i64 7, i64 7>)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> <i64 7, i64 7, i64 7, i64 7, i64 7, i64 7, i64 7, i64 7>)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; GLM-LABEL: 'splatconstant_funnel_i64'
diff --git a/llvm/test/Analysis/CostModel/X86/fshr.ll b/llvm/test/Analysis/CostModel/X86/fshr.ll
index 31e409cc9..ef74f9d71 100644
--- a/llvm/test/Analysis/CostModel/X86/fshr.ll
+++ b/llvm/test/Analysis/CostModel/X86/fshr.ll
@@ -57,9 +57,9 @@ define void @var_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 x i64
 ;
 ; SLM-LABEL: 'var_funnel_i64'
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %b64, i64 %c64)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %c128)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %c256)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %c512)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %c128)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %c256)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %c512)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; GLM-LABEL: 'var_funnel_i64'
@@ -385,9 +385,9 @@ define void @splatvar_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <2 x i64> %c128, <2 x i64> undef, <2 x i32> zeroinitializer
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <4 x i64> %c256, <4 x i64> undef, <4 x i32> zeroinitializer
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <8 x i64> %c512, <8 x i64> undef, <8 x i32> zeroinitializer
-; SLM-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %u128)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %u256)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %u512)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %u128)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %u256)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %u512)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; GLM-LABEL: 'splatvar_funnel_i64'
@@ -777,9 +777,9 @@ define void @constant_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8
 ;
 ; SLM-LABEL: 'constant_funnel_i64'
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %b64, i64 7)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> <i64 1, i64 7>)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> <i64 1, i64 7, i64 15, i64 31>)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 60 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> <i64 1, i64 7, i64 15, i64 31, i64 1, i64 7, i64 15, i64 31>)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> <i64 1, i64 7>)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> <i64 1, i64 7, i64 15, i64 31>)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> <i64 1, i64 7, i64 15, i64 31, i64 1, i64 7, i64 15, i64 31>)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; GLM-LABEL: 'constant_funnel_i64'
@@ -1093,9 +1093,9 @@ define void @splatconstant_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256
 ;
 ; SLM-LABEL: 'splatconstant_funnel_i64'
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %b64, i64 7)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> <i64 7, i64 7>)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> <i64 7, i64 7, i64 7, i64 7>)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> <i64 7, i64 7, i64 7, i64 7, i64 7, i64 7, i64 7, i64 7>)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> <i64 7, i64 7>)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> <i64 7, i64 7, i64 7, i64 7>)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> <i64 7, i64 7, i64 7, i64 7, i64 7, i64 7, i64 7, i64 7>)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; GLM-LABEL: 'splatconstant_funnel_i64'
diff --git a/llvm/test/Analysis/CostModel/X86/icmp.ll b/llvm/test/Analysis/CostModel/X86/icmp.ll
index caf20d1b9..af84cf9fc 100644
--- a/llvm/test/Analysis/CostModel/X86/icmp.ll
+++ b/llvm/test/Analysis/CostModel/X86/icmp.ll
@@ -11,7 +11,7 @@
 ; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -cost-model -analyze -mattr=+xop,+avx | FileCheck %s -check-prefixes=CHECK,AVX,XOPAVX1
 ; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -cost-model -analyze -mattr=+xop,+avx2 | FileCheck %s -check-prefixes=CHECK,AVX,XOPAVX2
 ;
-; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -cost-model -analyze -mcpu=slm | FileCheck %s --check-prefixes=CHECK,SSE,SSE42
+; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -cost-model -analyze -mcpu=slm | FileCheck %s --check-prefixes=CHECK,SSE,SLM
 ; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -cost-model -analyze -mcpu=goldmont | FileCheck %s --check-prefixes=CHECK,SSE,SSE42
 ; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -cost-model -analyze -mcpu=btver2 | FileCheck %s --check-prefixes=BTVER2
 
@@ -269,6 +269,29 @@ define i32 @cmp_int_eq(i32 %arg) {
 ; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I64 = icmp eq <16 x i64> undef, undef
 ; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
+; SLM-LABEL: 'cmp_int_eq'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp eq i8 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp eq <16 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = icmp eq <32 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = icmp eq <64 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128I8 = icmp eq <128 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp eq i16 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp eq <8 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = icmp eq <16 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = icmp eq <32 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I16 = icmp eq <64 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp eq i32 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp eq <4 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = icmp eq <8 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = icmp eq <16 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I32 = icmp eq <32 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp eq i64 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = icmp eq <2 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I64 = icmp eq <4 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I64 = icmp eq <8 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16I64 = icmp eq <16 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
 ; BTVER2-LABEL: 'cmp_int_eq'
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp eq i8 undef, undef
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp eq <16 x i8> undef, undef
@@ -573,6 +596,29 @@ define i32 @cmp_int_ne(i32 %arg) {
 ; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I64 = icmp ne <16 x i64> undef, undef
 ; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
+; SLM-LABEL: 'cmp_int_ne'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp ne i8 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = icmp ne <16 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = icmp ne <32 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = icmp ne <64 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V128I8 = icmp ne <128 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp ne i16 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = icmp ne <8 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = icmp ne <16 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = icmp ne <32 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I16 = icmp ne <64 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp ne i32 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = icmp ne <4 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = icmp ne <8 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = icmp ne <16 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I32 = icmp ne <32 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp ne i64 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2I64 = icmp ne <2 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4I64 = icmp ne <4 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V8I64 = icmp ne <8 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V16I64 = icmp ne <16 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
 ; BTVER2-LABEL: 'cmp_int_ne'
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp ne i8 undef, undef
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = icmp ne <16 x i8> undef, undef
@@ -877,6 +923,29 @@ define i32 @cmp_int_sge(i32 %arg) {
 ; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I64 = icmp sge <16 x i64> undef, undef
 ; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
+; SLM-LABEL: 'cmp_int_sge'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp sge i8 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = icmp sge <16 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = icmp sge <32 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = icmp sge <64 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V128I8 = icmp sge <128 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp sge i16 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = icmp sge <8 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = icmp sge <16 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = icmp sge <32 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I16 = icmp sge <64 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp sge i32 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = icmp sge <4 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = icmp sge <8 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = icmp sge <16 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I32 = icmp sge <32 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp sge i64 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2I64 = icmp sge <2 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4I64 = icmp sge <4 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V8I64 = icmp sge <8 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V16I64 = icmp sge <16 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
 ; BTVER2-LABEL: 'cmp_int_sge'
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp sge i8 undef, undef
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = icmp sge <16 x i8> undef, undef
@@ -1181,6 +1250,29 @@ define i32 @cmp_int_uge(i32 %arg) {
 ; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16I64 = icmp uge <16 x i64> undef, undef
 ; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
+; SLM-LABEL: 'cmp_int_uge'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp uge i8 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = icmp uge <16 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = icmp uge <32 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = icmp uge <64 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V128I8 = icmp uge <128 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp uge i16 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = icmp uge <8 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = icmp uge <16 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = icmp uge <32 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I16 = icmp uge <64 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp uge i32 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = icmp uge <4 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = icmp uge <8 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = icmp uge <16 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I32 = icmp uge <32 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp uge i64 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I64 = icmp uge <2 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I64 = icmp uge <4 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V8I64 = icmp uge <8 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V16I64 = icmp uge <16 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
 ; BTVER2-LABEL: 'cmp_int_uge'
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp uge i8 undef, undef
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = icmp uge <16 x i8> undef, undef
@@ -1485,6 +1577,29 @@ define i32 @cmp_int_sgt(i32 %arg) {
 ; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I64 = icmp sgt <16 x i64> undef, undef
 ; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
+; SLM-LABEL: 'cmp_int_sgt'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp sgt i8 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp sgt <16 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = icmp sgt <32 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = icmp sgt <64 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128I8 = icmp sgt <128 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp sgt i16 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp sgt <8 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = icmp sgt <16 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = icmp sgt <32 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I16 = icmp sgt <64 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp sgt i32 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp sgt <4 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = icmp sgt <8 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = icmp sgt <16 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I32 = icmp sgt <32 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp sgt i64 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = icmp sgt <2 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I64 = icmp sgt <4 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I64 = icmp sgt <8 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16I64 = icmp sgt <16 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
 ; BTVER2-LABEL: 'cmp_int_sgt'
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp sgt i8 undef, undef
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp sgt <16 x i8> undef, undef
@@ -1789,6 +1904,29 @@ define i32 @cmp_int_ugt(i32 %arg) {
 ; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16I64 = icmp ugt <16 x i64> undef, undef
 ; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
+; SLM-LABEL: 'cmp_int_ugt'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp ugt i8 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = icmp ugt <16 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32I8 = icmp ugt <32 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V64I8 = icmp ugt <64 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V128I8 = icmp ugt <128 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp ugt i16 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I16 = icmp ugt <8 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16I16 = icmp ugt <16 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V32I16 = icmp ugt <32 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V64I16 = icmp ugt <64 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp ugt i32 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4I32 = icmp ugt <4 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8I32 = icmp ugt <8 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16I32 = icmp ugt <16 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V32I32 = icmp ugt <32 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp ugt i64 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I64 = icmp ugt <2 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4I64 = icmp ugt <4 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I64 = icmp ugt <8 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I64 = icmp ugt <16 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
 ; BTVER2-LABEL: 'cmp_int_ugt'
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp ugt i8 undef, undef
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = icmp ugt <16 x i8> undef, undef
@@ -2093,6 +2231,29 @@ define i32 @cmp_int_sle(i32 %arg) {
 ; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I64 = icmp sle <16 x i64> undef, undef
 ; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
+; SLM-LABEL: 'cmp_int_sle'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp sle i8 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = icmp sle <16 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = icmp sle <32 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = icmp sle <64 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V128I8 = icmp sle <128 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp sle i16 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = icmp sle <8 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = icmp sle <16 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = icmp sle <32 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I16 = icmp sle <64 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp sle i32 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = icmp sle <4 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = icmp sle <8 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = icmp sle <16 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I32 = icmp sle <32 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp sle i64 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2I64 = icmp sle <2 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4I64 = icmp sle <4 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V8I64 = icmp sle <8 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V16I64 = icmp sle <16 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
 ; BTVER2-LABEL: 'cmp_int_sle'
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp sle i8 undef, undef
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = icmp sle <16 x i8> undef, undef
@@ -2397,6 +2558,29 @@ define i32 @cmp_int_ule(i32 %arg) {
 ; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16I64 = icmp ule <16 x i64> undef, undef
 ; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
+; SLM-LABEL: 'cmp_int_ule'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp ule i8 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = icmp ule <16 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = icmp ule <32 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = icmp ule <64 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V128I8 = icmp ule <128 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp ule i16 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = icmp ule <8 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = icmp ule <16 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = icmp ule <32 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I16 = icmp ule <64 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp ule i32 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = icmp ule <4 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = icmp ule <8 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = icmp ule <16 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I32 = icmp ule <32 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp ule i64 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I64 = icmp ule <2 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I64 = icmp ule <4 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V8I64 = icmp ule <8 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V16I64 = icmp ule <16 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
 ; BTVER2-LABEL: 'cmp_int_ule'
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp ule i8 undef, undef
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = icmp ule <16 x i8> undef, undef
@@ -2701,6 +2885,29 @@ define i32 @cmp_int_slt(i32 %arg) {
 ; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I64 = icmp slt <16 x i64> undef, undef
 ; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
+; SLM-LABEL: 'cmp_int_slt'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp slt i8 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp slt <16 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = icmp slt <32 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = icmp slt <64 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128I8 = icmp slt <128 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp slt i16 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp slt <8 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = icmp slt <16 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = icmp slt <32 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I16 = icmp slt <64 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp slt i32 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp slt <4 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = icmp slt <8 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = icmp slt <16 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I32 = icmp slt <32 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp slt i64 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = icmp slt <2 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I64 = icmp slt <4 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I64 = icmp slt <8 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16I64 = icmp slt <16 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
 ; BTVER2-LABEL: 'cmp_int_slt'
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp slt i8 undef, undef
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp slt <16 x i8> undef, undef
@@ -3005,6 +3212,29 @@ define i32 @cmp_int_ult(i32 %arg) {
 ; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16I64 = icmp ult <16 x i64> undef, undef
 ; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
+; SLM-LABEL: 'cmp_int_ult'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp ult i8 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = icmp ult <16 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32I8 = icmp ult <32 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V64I8 = icmp ult <64 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V128I8 = icmp ult <128 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp ult i16 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I16 = icmp ult <8 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16I16 = icmp ult <16 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V32I16 = icmp ult <32 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V64I16 = icmp ult <64 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp ult i32 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4I32 = icmp ult <4 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8I32 = icmp ult <8 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16I32 = icmp ult <16 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V32I32 = icmp ult <32 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp ult i64 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I64 = icmp ult <2 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4I64 = icmp ult <4 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I64 = icmp ult <8 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I64 = icmp ult <16 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
 ; BTVER2-LABEL: 'cmp_int_ult'
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp ult i8 undef, undef
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = icmp ult <16 x i8> undef, undef
diff --git a/llvm/test/Analysis/CostModel/X86/masked-intrinsic-cost.ll b/llvm/test/Analysis/CostModel/X86/masked-intrinsic-cost.ll
index 050a83364..076bfa3c7 100644
--- a/llvm/test/Analysis/CostModel/X86/masked-intrinsic-cost.ll
+++ b/llvm/test/Analysis/CostModel/X86/masked-intrinsic-cost.ll
@@ -52,7 +52,7 @@ define i32 @masked_load() {
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = call <16 x i32> @llvm.masked.load.v16i32.p0v16i32(<16 x i32>* undef, i32 1, <16 x i1> undef, <16 x i32> undef)
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call <8 x i32> @llvm.masked.load.v8i32.p0v8i32(<8 x i32>* undef, i32 1, <8 x i1> undef, <8 x i32> undef)
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* undef, i32 1, <4 x i1> undef, <4 x i32> undef)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I32 = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* undef, i32 1, <2 x i1> undef, <2 x i32> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I32 = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* undef, i32 1, <2 x i1> undef, <2 x i32> undef)
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %V32I16 = call <32 x i16> @llvm.masked.load.v32i16.p0v32i16(<32 x i16>* undef, i32 1, <32 x i1> undef, <32 x i16> undef)
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V16I16 = call <16 x i16> @llvm.masked.load.v16i16.p0v16i16(<16 x i16>* undef, i32 1, <16 x i1> undef, <16 x i16> undef)
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V8I16 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* undef, i32 1, <8 x i1> undef, <8 x i16> undef)
@@ -79,7 +79,7 @@ define i32 @masked_load() {
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I32 = call <16 x i32> @llvm.masked.load.v16i32.p0v16i32(<16 x i32>* undef, i32 1, <16 x i1> undef, <16 x i32> undef)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <8 x i32> @llvm.masked.load.v8i32.p0v8i32(<8 x i32>* undef, i32 1, <8 x i1> undef, <8 x i32> undef)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* undef, i32 1, <4 x i1> undef, <4 x i32> undef)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2I32 = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* undef, i32 1, <2 x i1> undef, <2 x i32> undef)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I32 = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* undef, i32 1, <2 x i1> undef, <2 x i32> undef)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %V32I16 = call <32 x i16> @llvm.masked.load.v32i16.p0v32i16(<32 x i16>* undef, i32 1, <32 x i1> undef, <32 x i16> undef)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V16I16 = call <16 x i16> @llvm.masked.load.v16i16.p0v16i16(<16 x i16>* undef, i32 1, <16 x i1> undef, <16 x i16> undef)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V8I16 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* undef, i32 1, <8 x i1> undef, <8 x i16> undef)
@@ -106,15 +106,15 @@ define i32 @masked_load() {
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I32 = call <16 x i32> @llvm.masked.load.v16i32.p0v16i32(<16 x i32>* undef, i32 1, <16 x i1> undef, <16 x i32> undef)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <8 x i32> @llvm.masked.load.v8i32.p0v8i32(<8 x i32>* undef, i32 1, <8 x i1> undef, <8 x i32> undef)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* undef, i32 1, <4 x i1> undef, <4 x i32> undef)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2I32 = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* undef, i32 1, <2 x i1> undef, <2 x i32> undef)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I32 = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* undef, i32 1, <2 x i1> undef, <2 x i32> undef)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.masked.load.v32i16.p0v32i16(<32 x i16>* undef, i32 1, <32 x i1> undef, <32 x i16> undef)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.masked.load.v16i16.p0v16i16(<16 x i16>* undef, i32 1, <16 x i1> undef, <16 x i16> undef)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* undef, i32 1, <8 x i1> undef, <8 x i16> undef)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4I16 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* undef, i32 1, <4 x i1> undef, <4 x i16> undef)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I16 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* undef, i32 1, <4 x i1> undef, <4 x i16> undef)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64I8 = call <64 x i8> @llvm.masked.load.v64i8.p0v64i8(<64 x i8>* undef, i32 1, <64 x i1> undef, <64 x i8> undef)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = call <32 x i8> @llvm.masked.load.v32i8.p0v32i8(<32 x i8>* undef, i32 1, <32 x i1> undef, <32 x i8> undef)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* undef, i32 1, <16 x i1> undef, <16 x i8> undef)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I8 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* undef, i32 1, <8 x i1> undef, <8 x i8> undef)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I8 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* undef, i32 1, <8 x i1> undef, <8 x i8> undef)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 0
 ;
   %V8F64 = call <8 x double> @llvm.masked.load.v8f64.p0v8f64(<8 x double>* undef, i32 1, <8 x i1> undef, <8 x double> undef)
@@ -194,7 +194,7 @@ define i32 @masked_store() {
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v16i32.p0v16i32(<16 x i32> undef, <16 x i32>* undef, i32 1, <16 x i1> undef)
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.store.v8i32.p0v8i32(<8 x i32> undef, <8 x i32>* undef, i32 1, <8 x i1> undef)
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> undef, <4 x i32>* undef, i32 1, <4 x i1> undef)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.store.v2i32.p0v2i32(<2 x i32> undef, <2 x i32>* undef, i32 1, <2 x i1> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: call void @llvm.masked.store.v2i32.p0v2i32(<2 x i32> undef, <2 x i32>* undef, i32 1, <2 x i1> undef)
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: call void @llvm.masked.store.v32i16.p0v32i16(<32 x i16> undef, <32 x i16>* undef, i32 1, <32 x i1> undef)
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.masked.store.v16i16.p0v16i16(<16 x i16> undef, <16 x i16>* undef, i32 1, <16 x i1> undef)
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> undef, <8 x i16>* undef, i32 1, <8 x i1> undef)
@@ -221,7 +221,7 @@ define i32 @masked_store() {
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v16i32.p0v16i32(<16 x i32> undef, <16 x i32>* undef, i32 1, <16 x i1> undef)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i32.p0v8i32(<8 x i32> undef, <8 x i32>* undef, i32 1, <8 x i1> undef)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> undef, <4 x i32>* undef, i32 1, <4 x i1> undef)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.store.v2i32.p0v2i32(<2 x i32> undef, <2 x i32>* undef, i32 1, <2 x i1> undef)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: call void @llvm.masked.store.v2i32.p0v2i32(<2 x i32> undef, <2 x i32>* undef, i32 1, <2 x i1> undef)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: call void @llvm.masked.store.v32i16.p0v32i16(<32 x i16> undef, <32 x i16>* undef, i32 1, <32 x i1> undef)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.masked.store.v16i16.p0v16i16(<16 x i16> undef, <16 x i16>* undef, i32 1, <16 x i1> undef)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> undef, <8 x i16>* undef, i32 1, <8 x i1> undef)
@@ -248,15 +248,15 @@ define i32 @masked_store() {
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v16i32.p0v16i32(<16 x i32> undef, <16 x i32>* undef, i32 1, <16 x i1> undef)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i32.p0v8i32(<8 x i32> undef, <8 x i32>* undef, i32 1, <8 x i1> undef)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> undef, <4 x i32>* undef, i32 1, <4 x i1> undef)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.store.v2i32.p0v2i32(<2 x i32> undef, <2 x i32>* undef, i32 1, <2 x i1> undef)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: call void @llvm.masked.store.v2i32.p0v2i32(<2 x i32> undef, <2 x i32>* undef, i32 1, <2 x i1> undef)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v32i16.p0v32i16(<32 x i16> undef, <32 x i16>* undef, i32 1, <32 x i1> undef)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v16i16.p0v16i16(<16 x i16> undef, <16 x i16>* undef, i32 1, <16 x i1> undef)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> undef, <8 x i16>* undef, i32 1, <8 x i1> undef)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> undef, <4 x i16>* undef, i32 1, <4 x i1> undef)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> undef, <4 x i16>* undef, i32 1, <4 x i1> undef)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v64i8.p0v64i8(<64 x i8> undef, <64 x i8>* undef, i32 1, <64 x i1> undef)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v32i8.p0v32i8(<32 x i8> undef, <32 x i8>* undef, i32 1, <32 x i1> undef)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> undef, <16 x i8>* undef, i32 1, <16 x i1> undef)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> undef, <8 x i8>* undef, i32 1, <8 x i1> undef)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> undef, <8 x i8>* undef, i32 1, <8 x i1> undef)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 0
 ;
   call void @llvm.masked.store.v8f64.p0v8f64(<8 x double> undef, <8 x double>* undef, i32 1, <8 x i1> undef)
@@ -960,15 +960,10 @@ define <8 x float> @test4(<8 x i32> %trigger, <8 x float>* %addr, <8 x float> %d
 }
 
 define void @test5(<2 x i32> %trigger, <2 x float>* %addr, <2 x float> %val) {
-; SSE2-LABEL: 'test5'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: call void @llvm.masked.store.v2f32.p0v2f32(<2 x float> %val, <2 x float>* %addr, i32 4, <2 x i1> %mask)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; SSE42-LABEL: 'test5'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: call void @llvm.masked.store.v2f32.p0v2f32(<2 x float> %val, <2 x float>* %addr, i32 4, <2 x i1> %mask)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; SSE-LABEL: 'test5'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
+; SSE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: call void @llvm.masked.store.v2f32.p0v2f32(<2 x float> %val, <2 x float>* %addr, i32 4, <2 x i1> %mask)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX-LABEL: 'test5'
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
@@ -986,24 +981,19 @@ define void @test5(<2 x i32> %trigger, <2 x float>* %addr, <2 x float> %val) {
 }
 
 define void @test6(<2 x i32> %trigger, <2 x i32>* %addr, <2 x i32> %val) {
-; SSE2-LABEL: 'test6'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.store.v2i32.p0v2i32(<2 x i32> %val, <2 x i32>* %addr, i32 4, <2 x i1> %mask)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; SSE42-LABEL: 'test6'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.store.v2i32.p0v2i32(<2 x i32> %val, <2 x i32>* %addr, i32 4, <2 x i1> %mask)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; SSE-LABEL: 'test6'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
+; SSE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.store.v2i32.p0v2i32(<2 x i32> %val, <2 x i32>* %addr, i32 4, <2 x i1> %mask)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX-LABEL: 'test6'
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
-; AVX-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.store.v2i32.p0v2i32(<2 x i32> %val, <2 x i32>* %addr, i32 4, <2 x i1> %mask)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: call void @llvm.masked.store.v2i32.p0v2i32(<2 x i32> %val, <2 x i32>* %addr, i32 4, <2 x i1> %mask)
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test6'
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.store.v2i32.p0v2i32(<2 x i32> %val, <2 x i32>* %addr, i32 4, <2 x i1> %mask)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: call void @llvm.masked.store.v2i32.p0v2i32(<2 x i32> %val, <2 x i32>* %addr, i32 4, <2 x i1> %mask)
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %mask = icmp eq <2 x i32> %trigger, zeroinitializer
@@ -1012,15 +1002,10 @@ define void @test6(<2 x i32> %trigger, <2 x i32>* %addr, <2 x i32> %val) {
 }
 
 define <2 x float> @test7(<2 x i32> %trigger, <2 x float>* %addr, <2 x float> %dst) {
-; SSE2-LABEL: 'test7'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res = call <2 x float> @llvm.masked.load.v2f32.p0v2f32(<2 x float>* %addr, i32 4, <2 x i1> %mask, <2 x float> %dst)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x float> %res
-;
-; SSE42-LABEL: 'test7'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res = call <2 x float> @llvm.masked.load.v2f32.p0v2f32(<2 x float>* %addr, i32 4, <2 x i1> %mask, <2 x float> %dst)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x float> %res
+; SSE-LABEL: 'test7'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
+; SSE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res = call <2 x float> @llvm.masked.load.v2f32.p0v2f32(<2 x float>* %addr, i32 4, <2 x i1> %mask, <2 x float> %dst)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x float> %res
 ;
 ; AVX-LABEL: 'test7'
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
@@ -1038,24 +1023,19 @@ define <2 x float> @test7(<2 x i32> %trigger, <2 x float>* %addr, <2 x float> %d
 }
 
 define <2 x i32> @test8(<2 x i32> %trigger, <2 x i32>* %addr, <2 x i32> %dst) {
-; SSE2-LABEL: 'test8'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %res = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* %addr, i32 4, <2 x i1> %mask, <2 x i32> %dst)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i32> %res
-;
-; SSE42-LABEL: 'test8'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %res = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* %addr, i32 4, <2 x i1> %mask, <2 x i32> %dst)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i32> %res
+; SSE-LABEL: 'test8'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
+; SSE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %res = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* %addr, i32 4, <2 x i1> %mask, <2 x i32> %dst)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i32> %res
 ;
 ; AVX-LABEL: 'test8'
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
-; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* %addr, i32 4, <2 x i1> %mask, <2 x i32> %dst)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %res = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* %addr, i32 4, <2 x i1> %mask, <2 x i32> %dst)
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i32> %res
 ;
 ; AVX512-LABEL: 'test8'
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* %addr, i32 4, <2 x i1> %mask, <2 x i32> %dst)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %res = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* %addr, i32 4, <2 x i1> %mask, <2 x i32> %dst)
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i32> %res
 ;
   %mask = icmp eq <2 x i32> %trigger, zeroinitializer
diff --git a/llvm/test/Analysis/CostModel/X86/min-legal-vector-width.ll b/llvm/test/Analysis/CostModel/X86/min-legal-vector-width.ll
index 028ed87a9..1f354e2cf 100644
--- a/llvm/test/Analysis/CostModel/X86/min-legal-vector-width.ll
+++ b/llvm/test/Analysis/CostModel/X86/min-legal-vector-width.ll
@@ -5,7 +5,7 @@
 
 define void @zext256() "min-legal-vector-width"="256" {
 ; VEC256-LABEL: 'zext256'
-; VEC256-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %A = zext <8 x i16> undef to <8 x i64>
+; VEC256-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %A = zext <8 x i16> undef to <8 x i64>
 ; VEC256-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %B = zext <8 x i32> undef to <8 x i64>
 ; VEC256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %C = zext <16 x i8> undef to <16 x i32>
 ; VEC256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %D = zext <16 x i16> undef to <16 x i32>
@@ -30,7 +30,7 @@ define void @zext256() "min-legal-vector-width"="256" {
 
 define void @zext512() "min-legal-vector-width"="512" {
 ; AVX-LABEL: 'zext512'
-; AVX-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %A = zext <8 x i16> undef to <8 x i64>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %A = zext <8 x i16> undef to <8 x i64>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %B = zext <8 x i32> undef to <8 x i64>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %C = zext <16 x i8> undef to <16 x i32>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %D = zext <16 x i16> undef to <16 x i32>
@@ -63,8 +63,8 @@ define void @zext512() "min-legal-vector-width"="512" {
 
 define void @sext256() "min-legal-vector-width"="256" {
 ; VEC256-LABEL: 'sext256'
-; VEC256-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %A = sext <8 x i8> undef to <8 x i64>
-; VEC256-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %B = sext <8 x i16> undef to <8 x i64>
+; VEC256-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %A = sext <8 x i8> undef to <8 x i64>
+; VEC256-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %B = sext <8 x i16> undef to <8 x i64>
 ; VEC256-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %C = sext <8 x i32> undef to <8 x i64>
 ; VEC256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %D = sext <16 x i8> undef to <16 x i32>
 ; VEC256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %E = sext <16 x i16> undef to <16 x i32>
@@ -72,7 +72,7 @@ define void @sext256() "min-legal-vector-width"="256" {
 ; VEC256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; VEC512-LABEL: 'sext256'
-; VEC512-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %A = sext <8 x i8> undef to <8 x i64>
+; VEC512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %A = sext <8 x i8> undef to <8 x i64>
 ; VEC512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %B = sext <8 x i16> undef to <8 x i64>
 ; VEC512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %C = sext <8 x i32> undef to <8 x i64>
 ; VEC512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %D = sext <16 x i8> undef to <16 x i32>
@@ -91,8 +91,8 @@ define void @sext256() "min-legal-vector-width"="256" {
 
 define void @sext512() "min-legal-vector-width"="512" {
 ; AVX-LABEL: 'sext512'
-; AVX-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %A = sext <8 x i8> undef to <8 x i64>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %B = sext <8 x i16> undef to <8 x i64>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %A = sext <8 x i8> undef to <8 x i64>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %B = sext <8 x i16> undef to <8 x i64>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %C = sext <8 x i32> undef to <8 x i64>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %D = sext <16 x i8> undef to <16 x i32>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %E = sext <16 x i16> undef to <16 x i32>
@@ -100,7 +100,7 @@ define void @sext512() "min-legal-vector-width"="512" {
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SKX256-LABEL: 'sext512'
-; SKX256-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %A = sext <8 x i8> undef to <8 x i64>
+; SKX256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %A = sext <8 x i8> undef to <8 x i64>
 ; SKX256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %B = sext <8 x i16> undef to <8 x i64>
 ; SKX256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %C = sext <8 x i32> undef to <8 x i64>
 ; SKX256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %D = sext <16 x i8> undef to <16 x i32>
@@ -109,7 +109,7 @@ define void @sext512() "min-legal-vector-width"="512" {
 ; SKX256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; VEC512-LABEL: 'sext512'
-; VEC512-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %A = sext <8 x i8> undef to <8 x i64>
+; VEC512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %A = sext <8 x i8> undef to <8 x i64>
 ; VEC512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %B = sext <8 x i16> undef to <8 x i64>
 ; VEC512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %C = sext <8 x i32> undef to <8 x i64>
 ; VEC512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %D = sext <16 x i8> undef to <16 x i32>
diff --git a/llvm/test/Analysis/CostModel/X86/reduce-add.ll b/llvm/test/Analysis/CostModel/X86/reduce-add.ll
index 066ddb77a..b5729eac4 100644
--- a/llvm/test/Analysis/CostModel/X86/reduce-add.ll
+++ b/llvm/test/Analysis/CostModel/X86/reduce-add.ll
@@ -8,30 +8,16 @@
 ; RUN: opt < %s -cost-model -mtriple=x86_64-apple-darwin -analyze -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512BW
 ; RUN: opt < %s -cost-model -mtriple=x86_64-apple-darwin -analyze -mattr=+avx512f,+avx512dq | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512DQ
 
+; RUN: opt < %s -cost-model -mtriple=x86_64-apple-darwin -analyze -mcpu=slm | FileCheck %s --check-prefixes=SLM
+
 define i32 @reduce_i64(i32 %arg) {
-; SSE2-LABEL: 'reduce_i64'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1 = call i64 @llvm.experimental.vector.reduce.add.v1i64(<1 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i64 @llvm.experimental.vector.reduce.add.v2i64(<2 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4 = call i64 @llvm.experimental.vector.reduce.add.v4i64(<4 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8 = call i64 @llvm.experimental.vector.reduce.add.v8i64(<8 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V16 = call i64 @llvm.experimental.vector.reduce.add.v16i64(<16 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-; SSSE3-LABEL: 'reduce_i64'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1 = call i64 @llvm.experimental.vector.reduce.add.v1i64(<1 x i64> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i64 @llvm.experimental.vector.reduce.add.v2i64(<2 x i64> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4 = call i64 @llvm.experimental.vector.reduce.add.v4i64(<4 x i64> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8 = call i64 @llvm.experimental.vector.reduce.add.v8i64(<8 x i64> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V16 = call i64 @llvm.experimental.vector.reduce.add.v16i64(<16 x i64> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-; SSE42-LABEL: 'reduce_i64'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1 = call i64 @llvm.experimental.vector.reduce.add.v1i64(<1 x i64> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i64 @llvm.experimental.vector.reduce.add.v2i64(<2 x i64> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4 = call i64 @llvm.experimental.vector.reduce.add.v4i64(<4 x i64> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8 = call i64 @llvm.experimental.vector.reduce.add.v8i64(<8 x i64> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16 = call i64 @llvm.experimental.vector.reduce.add.v16i64(<16 x i64> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+; SSE-LABEL: 'reduce_i64'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1 = call i64 @llvm.experimental.vector.reduce.add.v1i64(<1 x i64> undef)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i64 @llvm.experimental.vector.reduce.add.v2i64(<2 x i64> undef)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4 = call i64 @llvm.experimental.vector.reduce.add.v4i64(<4 x i64> undef)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8 = call i64 @llvm.experimental.vector.reduce.add.v8i64(<8 x i64> undef)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16 = call i64 @llvm.experimental.vector.reduce.add.v16i64(<16 x i64> undef)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX-LABEL: 'reduce_i64'
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1 = call i64 @llvm.experimental.vector.reduce.add.v1i64(<1 x i64> undef)
@@ -48,6 +34,14 @@ define i32 @reduce_i64(i32 %arg) {
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8 = call i64 @llvm.experimental.vector.reduce.add.v8i64(<8 x i64> undef)
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16 = call i64 @llvm.experimental.vector.reduce.add.v16i64(<16 x i64> undef)
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SLM-LABEL: 'reduce_i64'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1 = call i64 @llvm.experimental.vector.reduce.add.v1i64(<1 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2 = call i64 @llvm.experimental.vector.reduce.add.v2i64(<2 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4 = call i64 @llvm.experimental.vector.reduce.add.v4i64(<4 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V8 = call i64 @llvm.experimental.vector.reduce.add.v8i64(<8 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V16 = call i64 @llvm.experimental.vector.reduce.add.v16i64(<16 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %V1  = call i64 @llvm.experimental.vector.reduce.add.v1i64(<1 x i64> undef)
   %V2  = call i64 @llvm.experimental.vector.reduce.add.v2i64(<2 x i64> undef)
@@ -58,32 +52,16 @@ define i32 @reduce_i64(i32 %arg) {
 }
 
 define i32 @reduce_i32(i32 %arg) {
-; SSE2-LABEL: 'reduce_i32'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.add.v2i32(<2 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.add.v8i32(<8 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.add.v16i32(<16 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V32 = call i32 @llvm.experimental.vector.reduce.add.v32i32(<32 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-; SSSE3-LABEL: 'reduce_i32'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.add.v2i32(<2 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.add.v8i32(<8 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.add.v16i32(<16 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V32 = call i32 @llvm.experimental.vector.reduce.add.v32i32(<32 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-; SSE42-LABEL: 'reduce_i32'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.add.v2i32(<2 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.add.v8i32(<8 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.add.v16i32(<16 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V32 = call i32 @llvm.experimental.vector.reduce.add.v32i32(<32 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+; SSE-LABEL: 'reduce_i32'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.add.v2i32(<2 x i32> undef)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> undef)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.add.v8i32(<8 x i32> undef)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.add.v16i32(<16 x i32> undef)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V32 = call i32 @llvm.experimental.vector.reduce.add.v32i32(<32 x i32> undef)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX-LABEL: 'reduce_i32'
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.add.v2i32(<2 x i32> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.add.v2i32(<2 x i32> undef)
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> undef)
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.add.v8i32(<8 x i32> undef)
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.add.v16i32(<16 x i32> undef)
@@ -91,12 +69,20 @@ define i32 @reduce_i32(i32 %arg) {
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512-LABEL: 'reduce_i32'
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.add.v2i32(<2 x i32> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.add.v2i32(<2 x i32> undef)
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> undef)
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.add.v8i32(<8 x i32> undef)
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.add.v16i32(<16 x i32> undef)
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V32 = call i32 @llvm.experimental.vector.reduce.add.v32i32(<32 x i32> undef)
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SLM-LABEL: 'reduce_i32'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.add.v2i32(<2 x i32> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.add.v8i32(<8 x i32> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.add.v16i32(<16 x i32> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V32 = call i32 @llvm.experimental.vector.reduce.add.v32i32(<32 x i32> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %V2  = call i32 @llvm.experimental.vector.reduce.add.v2i32(<2 x i32> undef)
   %V4  = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> undef)
@@ -107,77 +93,59 @@ define i32 @reduce_i32(i32 %arg) {
 }
 
 define i32 @reduce_i16(i32 %arg) {
-; SSE2-LABEL: 'reduce_i16'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.add.v2i16(<2 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.add.v4i16(<4 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.add.v8i16(<8 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.add.v16i16(<16 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.add.v32i16(<32 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.add.v64i16(<64 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-; SSSE3-LABEL: 'reduce_i16'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.add.v2i16(<2 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.add.v4i16(<4 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.add.v8i16(<8 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.add.v16i16(<16 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.add.v32i16(<32 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.add.v64i16(<64 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-; SSE42-LABEL: 'reduce_i16'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.add.v2i16(<2 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.add.v4i16(<4 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.add.v8i16(<8 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.add.v16i16(<16 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.add.v32i16(<32 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.add.v64i16(<64 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-; AVX1-LABEL: 'reduce_i16'
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.add.v2i16(<2 x i16> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.add.v4i16(<4 x i16> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.add.v8i16(<8 x i16> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 49 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.add.v16i16(<16 x i16> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 53 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.add.v32i16(<32 x i16> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 61 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.add.v64i16(<64 x i16> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-; AVX2-LABEL: 'reduce_i16'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.add.v2i16(<2 x i16> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.add.v4i16(<4 x i16> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.add.v8i16(<8 x i16> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.add.v16i16(<16 x i16> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.add.v32i16(<32 x i16> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.add.v64i16(<64 x i16> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+; SSE-LABEL: 'reduce_i16'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.add.v2i16(<2 x i16> undef)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.add.v4i16(<4 x i16> undef)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.add.v8i16(<8 x i16> undef)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.add.v16i16(<16 x i16> undef)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.add.v32i16(<32 x i16> undef)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.add.v64i16(<64 x i16> undef)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX-LABEL: 'reduce_i16'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.add.v2i16(<2 x i16> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.add.v4i16(<4 x i16> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.add.v8i16(<8 x i16> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.add.v16i16(<16 x i16> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.add.v32i16(<32 x i16> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.add.v64i16(<64 x i16> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512F-LABEL: 'reduce_i16'
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.add.v2i16(<2 x i16> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.add.v2i16(<2 x i16> undef)
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.add.v4i16(<4 x i16> undef)
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.add.v8i16(<8 x i16> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.add.v16i16(<16 x i16> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.add.v32i16(<32 x i16> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.add.v64i16(<64 x i16> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.add.v16i16(<16 x i16> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.add.v32i16(<32 x i16> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.add.v64i16(<64 x i16> undef)
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512BW-LABEL: 'reduce_i16'
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.add.v2i16(<2 x i16> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.add.v2i16(<2 x i16> undef)
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.add.v4i16(<4 x i16> undef)
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.add.v8i16(<8 x i16> undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.add.v16i16(<16 x i16> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.add.v16i16(<16 x i16> undef)
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.add.v32i16(<32 x i16> undef)
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.add.v64i16(<64 x i16> undef)
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512DQ-LABEL: 'reduce_i16'
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.add.v2i16(<2 x i16> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.add.v2i16(<2 x i16> undef)
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.add.v4i16(<4 x i16> undef)
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.add.v8i16(<8 x i16> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.add.v16i16(<16 x i16> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.add.v32i16(<32 x i16> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.add.v64i16(<64 x i16> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.add.v16i16(<16 x i16> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.add.v32i16(<32 x i16> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.add.v64i16(<64 x i16> undef)
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SLM-LABEL: 'reduce_i16'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.add.v2i16(<2 x i16> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.add.v4i16(<4 x i16> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.add.v8i16(<8 x i16> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.add.v16i16(<16 x i16> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.add.v32i16(<32 x i16> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.add.v64i16(<64 x i16> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %V2  = call i16 @llvm.experimental.vector.reduce.add.v2i16(<2 x i16> undef)
   %V4  = call i16 @llvm.experimental.vector.reduce.add.v4i16(<4 x i16> undef)
@@ -189,85 +157,65 @@ define i32 @reduce_i16(i32 %arg) {
 }
 
 define i32 @reduce_i8(i32 %arg) {
-; SSE2-LABEL: 'reduce_i8'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.add.v2i8(<2 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.add.v4i8(<4 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.add.v8i8(<8 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 45 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.add.v16i8(<16 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 46 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.add.v32i8(<32 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.add.v64i8(<64 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.add.v128i8(<128 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-; SSSE3-LABEL: 'reduce_i8'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.add.v2i8(<2 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.add.v4i8(<4 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.add.v8i8(<8 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.add.v16i8(<16 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.add.v32i8(<32 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.add.v64i8(<64 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.add.v128i8(<128 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-; SSE42-LABEL: 'reduce_i8'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.add.v2i8(<2 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.add.v4i8(<4 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.add.v8i8(<8 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.add.v16i8(<16 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.add.v32i8(<32 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.add.v64i8(<64 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.add.v128i8(<128 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-; AVX1-LABEL: 'reduce_i8'
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.add.v2i8(<2 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.add.v4i8(<4 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.add.v8i8(<8 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.add.v16i8(<16 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 61 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.add.v32i8(<32 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.add.v64i8(<64 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 73 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.add.v128i8(<128 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-; AVX2-LABEL: 'reduce_i8'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.add.v2i8(<2 x i8> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.add.v4i8(<4 x i8> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.add.v8i8(<8 x i8> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.add.v16i8(<16 x i8> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.add.v32i8(<32 x i8> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.add.v64i8(<64 x i8> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.add.v128i8(<128 x i8> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+; SSE-LABEL: 'reduce_i8'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.add.v2i8(<2 x i8> undef)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.add.v4i8(<4 x i8> undef)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.add.v8i8(<8 x i8> undef)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.add.v16i8(<16 x i8> undef)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.add.v32i8(<32 x i8> undef)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.add.v64i8(<64 x i8> undef)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.add.v128i8(<128 x i8> undef)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX-LABEL: 'reduce_i8'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.add.v2i8(<2 x i8> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.add.v4i8(<4 x i8> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.add.v8i8(<8 x i8> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.add.v16i8(<16 x i8> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.add.v32i8(<32 x i8> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.add.v64i8(<64 x i8> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.add.v128i8(<128 x i8> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512F-LABEL: 'reduce_i8'
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.add.v2i8(<2 x i8> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.add.v4i8(<4 x i8> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.add.v8i8(<8 x i8> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.add.v16i8(<16 x i8> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.add.v32i8(<32 x i8> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.add.v64i8(<64 x i8> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.add.v128i8(<128 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.add.v2i8(<2 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.add.v4i8(<4 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.add.v8i8(<8 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.add.v16i8(<16 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.add.v32i8(<32 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.add.v64i8(<64 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.add.v128i8(<128 x i8> undef)
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512BW-LABEL: 'reduce_i8'
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.add.v2i8(<2 x i8> undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.add.v4i8(<4 x i8> undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.add.v8i8(<8 x i8> undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.add.v16i8(<16 x i8> undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.add.v32i8(<32 x i8> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.add.v2i8(<2 x i8> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.add.v4i8(<4 x i8> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.add.v8i8(<8 x i8> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.add.v16i8(<16 x i8> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.add.v32i8(<32 x i8> undef)
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 55 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.add.v64i8(<64 x i8> undef)
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.add.v128i8(<128 x i8> undef)
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512DQ-LABEL: 'reduce_i8'
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.add.v2i8(<2 x i8> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.add.v4i8(<4 x i8> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.add.v8i8(<8 x i8> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.add.v16i8(<16 x i8> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.add.v32i8(<32 x i8> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.add.v64i8(<64 x i8> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.add.v128i8(<128 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.add.v2i8(<2 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.add.v4i8(<4 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.add.v8i8(<8 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.add.v16i8(<16 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.add.v32i8(<32 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.add.v64i8(<64 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.add.v128i8(<128 x i8> undef)
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SLM-LABEL: 'reduce_i8'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.add.v2i8(<2 x i8> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.add.v4i8(<4 x i8> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.add.v8i8(<8 x i8> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.add.v16i8(<16 x i8> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.add.v32i8(<32 x i8> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.add.v64i8(<64 x i8> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.add.v128i8(<128 x i8> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %V2   = call i8 @llvm.experimental.vector.reduce.add.v2i8(<2 x i8> undef)
   %V4   = call i8 @llvm.experimental.vector.reduce.add.v4i8(<4 x i8> undef)
diff --git a/llvm/test/Analysis/CostModel/X86/reduce-and.ll b/llvm/test/Analysis/CostModel/X86/reduce-and.ll
index 77e8ee57e..455f7326f 100644
--- a/llvm/test/Analysis/CostModel/X86/reduce-and.ll
+++ b/llvm/test/Analysis/CostModel/X86/reduce-and.ll
@@ -92,8 +92,8 @@ define i32 @reduce_i32(i32 %arg) {
 
 define i32 @reduce_i16(i32 %arg) {
 ; SSE2-LABEL: 'reduce_i16'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.and.v2i16(<2 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.and.v4i16(<4 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.and.v2i16(<2 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.and.v4i16(<4 x i16> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.and.v8i16(<8 x i16> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.and.v16i16(<16 x i16> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.and.v32i16(<32 x i16> undef)
@@ -174,9 +174,9 @@ define i32 @reduce_i16(i32 %arg) {
 
 define i32 @reduce_i8(i32 %arg) {
 ; SSE2-LABEL: 'reduce_i8'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.and.v2i8(<2 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.and.v4i8(<4 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.and.v8i8(<8 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.and.v2i8(<2 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.and.v4i8(<4 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.and.v8i8(<8 x i8> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 45 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.and.v16i8(<16 x i8> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 46 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.and.v32i8(<32 x i8> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.and.v64i8(<64 x i8> undef)
@@ -299,35 +299,35 @@ define i32 @reduce_i1(i32 %arg) {
 ;
 ; AVX512F-LABEL: 'reduce_i1'
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1 = call i1 @llvm.experimental.vector.reduce.and.v1i1(<1 x i1> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i1 @llvm.experimental.vector.reduce.and.v2i1(<2 x i1> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V4 = call i1 @llvm.experimental.vector.reduce.and.v4i1(<4 x i1> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V8 = call i1 @llvm.experimental.vector.reduce.and.v8i1(<8 x i1> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 133 for instruction: %V16 = call i1 @llvm.experimental.vector.reduce.and.v16i1(<16 x i1> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 134 for instruction: %V32 = call i1 @llvm.experimental.vector.reduce.and.v32i1(<32 x i1> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 136 for instruction: %V64 = call i1 @llvm.experimental.vector.reduce.and.v64i1(<64 x i1> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 140 for instruction: %V128 = call i1 @llvm.experimental.vector.reduce.and.v128i1(<128 x i1> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i1 @llvm.experimental.vector.reduce.and.v2i1(<2 x i1> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4 = call i1 @llvm.experimental.vector.reduce.and.v4i1(<4 x i1> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8 = call i1 @llvm.experimental.vector.reduce.and.v8i1(<8 x i1> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V16 = call i1 @llvm.experimental.vector.reduce.and.v16i1(<16 x i1> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V32 = call i1 @llvm.experimental.vector.reduce.and.v32i1(<32 x i1> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V64 = call i1 @llvm.experimental.vector.reduce.and.v64i1(<64 x i1> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %V128 = call i1 @llvm.experimental.vector.reduce.and.v128i1(<128 x i1> undef)
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512BW-LABEL: 'reduce_i1'
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1 = call i1 @llvm.experimental.vector.reduce.and.v1i1(<1 x i1> undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i1 @llvm.experimental.vector.reduce.and.v2i1(<2 x i1> undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V4 = call i1 @llvm.experimental.vector.reduce.and.v4i1(<4 x i1> undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V8 = call i1 @llvm.experimental.vector.reduce.and.v8i1(<8 x i1> undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 133 for instruction: %V16 = call i1 @llvm.experimental.vector.reduce.and.v16i1(<16 x i1> undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 326 for instruction: %V32 = call i1 @llvm.experimental.vector.reduce.and.v32i1(<32 x i1> undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 775 for instruction: %V64 = call i1 @llvm.experimental.vector.reduce.and.v64i1(<64 x i1> undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 776 for instruction: %V128 = call i1 @llvm.experimental.vector.reduce.and.v128i1(<128 x i1> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i1 @llvm.experimental.vector.reduce.and.v2i1(<2 x i1> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4 = call i1 @llvm.experimental.vector.reduce.and.v4i1(<4 x i1> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8 = call i1 @llvm.experimental.vector.reduce.and.v8i1(<8 x i1> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V16 = call i1 @llvm.experimental.vector.reduce.and.v16i1(<16 x i1> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V32 = call i1 @llvm.experimental.vector.reduce.and.v32i1(<32 x i1> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V64 = call i1 @llvm.experimental.vector.reduce.and.v64i1(<64 x i1> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V128 = call i1 @llvm.experimental.vector.reduce.and.v128i1(<128 x i1> undef)
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512DQ-LABEL: 'reduce_i1'
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1 = call i1 @llvm.experimental.vector.reduce.and.v1i1(<1 x i1> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i1 @llvm.experimental.vector.reduce.and.v2i1(<2 x i1> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V4 = call i1 @llvm.experimental.vector.reduce.and.v4i1(<4 x i1> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V8 = call i1 @llvm.experimental.vector.reduce.and.v8i1(<8 x i1> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 133 for instruction: %V16 = call i1 @llvm.experimental.vector.reduce.and.v16i1(<16 x i1> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 134 for instruction: %V32 = call i1 @llvm.experimental.vector.reduce.and.v32i1(<32 x i1> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 136 for instruction: %V64 = call i1 @llvm.experimental.vector.reduce.and.v64i1(<64 x i1> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 140 for instruction: %V128 = call i1 @llvm.experimental.vector.reduce.and.v128i1(<128 x i1> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i1 @llvm.experimental.vector.reduce.and.v2i1(<2 x i1> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4 = call i1 @llvm.experimental.vector.reduce.and.v4i1(<4 x i1> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8 = call i1 @llvm.experimental.vector.reduce.and.v8i1(<8 x i1> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V16 = call i1 @llvm.experimental.vector.reduce.and.v16i1(<16 x i1> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V32 = call i1 @llvm.experimental.vector.reduce.and.v32i1(<32 x i1> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V64 = call i1 @llvm.experimental.vector.reduce.and.v64i1(<64 x i1> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %V128 = call i1 @llvm.experimental.vector.reduce.and.v128i1(<128 x i1> undef)
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %V1   = call i1 @llvm.experimental.vector.reduce.and.v1i1(<1 x i1> undef)
diff --git a/llvm/test/Analysis/CostModel/X86/reduce-mul.ll b/llvm/test/Analysis/CostModel/X86/reduce-mul.ll
index ce94216f4..42c9b5ced 100644
--- a/llvm/test/Analysis/CostModel/X86/reduce-mul.ll
+++ b/llvm/test/Analysis/CostModel/X86/reduce-mul.ll
@@ -67,7 +67,7 @@ define i32 @reduce_i64(i32 %arg) {
 
 define i32 @reduce_i32(i32 %arg) {
 ; SSE2-LABEL: 'reduce_i32'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.mul.v2i32(<2 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.mul.v2i32(<2 x i32> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.mul.v4i32(<4 x i32> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.mul.v8i32(<8 x i32> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.mul.v16i32(<16 x i32> undef)
@@ -75,7 +75,7 @@ define i32 @reduce_i32(i32 %arg) {
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'reduce_i32'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.mul.v2i32(<2 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.mul.v2i32(<2 x i32> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.mul.v4i32(<4 x i32> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.mul.v8i32(<8 x i32> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.mul.v16i32(<16 x i32> undef)
@@ -83,7 +83,7 @@ define i32 @reduce_i32(i32 %arg) {
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE42-LABEL: 'reduce_i32'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.mul.v2i32(<2 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.mul.v2i32(<2 x i32> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.mul.v4i32(<4 x i32> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.mul.v8i32(<8 x i32> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.mul.v16i32(<16 x i32> undef)
@@ -91,7 +91,7 @@ define i32 @reduce_i32(i32 %arg) {
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX1-LABEL: 'reduce_i32'
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.mul.v2i32(<2 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.mul.v2i32(<2 x i32> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.mul.v4i32(<4 x i32> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.mul.v8i32(<8 x i32> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.mul.v16i32(<16 x i32> undef)
@@ -99,36 +99,20 @@ define i32 @reduce_i32(i32 %arg) {
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX2-LABEL: 'reduce_i32'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.mul.v2i32(<2 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.mul.v2i32(<2 x i32> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.mul.v4i32(<4 x i32> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.mul.v8i32(<8 x i32> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.mul.v16i32(<16 x i32> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32 = call i32 @llvm.experimental.vector.reduce.mul.v32i32(<32 x i32> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
-; AVX512F-LABEL: 'reduce_i32'
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.mul.v2i32(<2 x i32> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.mul.v4i32(<4 x i32> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.mul.v8i32(<8 x i32> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.mul.v16i32(<16 x i32> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V32 = call i32 @llvm.experimental.vector.reduce.mul.v32i32(<32 x i32> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-; AVX512BW-LABEL: 'reduce_i32'
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.mul.v2i32(<2 x i32> undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.mul.v4i32(<4 x i32> undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.mul.v8i32(<8 x i32> undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.mul.v16i32(<16 x i32> undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V32 = call i32 @llvm.experimental.vector.reduce.mul.v32i32(<32 x i32> undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-; AVX512DQ-LABEL: 'reduce_i32'
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.mul.v2i32(<2 x i32> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.mul.v4i32(<4 x i32> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.mul.v8i32(<8 x i32> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.mul.v16i32(<16 x i32> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V32 = call i32 @llvm.experimental.vector.reduce.mul.v32i32(<32 x i32> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+; AVX512-LABEL: 'reduce_i32'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.mul.v2i32(<2 x i32> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.mul.v4i32(<4 x i32> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.mul.v8i32(<8 x i32> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.mul.v16i32(<16 x i32> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V32 = call i32 @llvm.experimental.vector.reduce.mul.v32i32(<32 x i32> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %V2  = call i32 @llvm.experimental.vector.reduce.mul.v2i32(<2 x i32> undef)
   %V4  = call i32 @llvm.experimental.vector.reduce.mul.v4i32(<4 x i32> undef)
@@ -140,8 +124,8 @@ define i32 @reduce_i32(i32 %arg) {
 
 define i32 @reduce_i16(i32 %arg) {
 ; SSE2-LABEL: 'reduce_i16'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.mul.v2i16(<2 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.mul.v4i16(<4 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.mul.v2i16(<2 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.mul.v4i16(<4 x i16> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.mul.v8i16(<8 x i16> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.mul.v16i16(<16 x i16> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.mul.v32i16(<32 x i16> undef)
@@ -149,8 +133,8 @@ define i32 @reduce_i16(i32 %arg) {
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'reduce_i16'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.mul.v2i16(<2 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.mul.v4i16(<4 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.mul.v2i16(<2 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.mul.v4i16(<4 x i16> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.mul.v8i16(<8 x i16> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.mul.v16i16(<16 x i16> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.mul.v32i16(<32 x i16> undef)
@@ -158,8 +142,8 @@ define i32 @reduce_i16(i32 %arg) {
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE42-LABEL: 'reduce_i16'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.mul.v2i16(<2 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.mul.v4i16(<4 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.mul.v2i16(<2 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.mul.v4i16(<4 x i16> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.mul.v8i16(<8 x i16> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.mul.v16i16(<16 x i16> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.mul.v32i16(<32 x i16> undef)
@@ -167,8 +151,8 @@ define i32 @reduce_i16(i32 %arg) {
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX1-LABEL: 'reduce_i16'
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.mul.v2i16(<2 x i16> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.mul.v4i16(<4 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.mul.v2i16(<2 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.mul.v4i16(<4 x i16> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.mul.v8i16(<8 x i16> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 49 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.mul.v16i16(<16 x i16> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 53 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.mul.v32i16(<32 x i16> undef)
@@ -176,8 +160,8 @@ define i32 @reduce_i16(i32 %arg) {
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX2-LABEL: 'reduce_i16'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.mul.v2i16(<2 x i16> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.mul.v4i16(<4 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.mul.v2i16(<2 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.mul.v4i16(<4 x i16> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.mul.v8i16(<8 x i16> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.mul.v16i16(<16 x i16> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.mul.v32i16(<32 x i16> undef)
@@ -185,7 +169,7 @@ define i32 @reduce_i16(i32 %arg) {
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512F-LABEL: 'reduce_i16'
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.mul.v2i16(<2 x i16> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.mul.v2i16(<2 x i16> undef)
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.mul.v4i16(<4 x i16> undef)
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.mul.v8i16(<8 x i16> undef)
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.mul.v16i16(<16 x i16> undef)
@@ -194,7 +178,7 @@ define i32 @reduce_i16(i32 %arg) {
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512BW-LABEL: 'reduce_i16'
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.mul.v2i16(<2 x i16> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.mul.v2i16(<2 x i16> undef)
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.mul.v4i16(<4 x i16> undef)
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.mul.v8i16(<8 x i16> undef)
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.mul.v16i16(<16 x i16> undef)
@@ -222,9 +206,9 @@ define i32 @reduce_i16(i32 %arg) {
 
 define i32 @reduce_i8(i32 %arg) {
 ; SSE2-LABEL: 'reduce_i8'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.mul.v2i8(<2 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.mul.v4i8(<4 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.mul.v8i8(<8 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.mul.v2i8(<2 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 45 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.mul.v4i8(<4 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.mul.v8i8(<8 x i8> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 89 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.mul.v16i8(<16 x i8> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 101 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.mul.v32i8(<32 x i8> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 125 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.mul.v64i8(<64 x i8> undef)
@@ -232,9 +216,9 @@ define i32 @reduce_i8(i32 %arg) {
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'reduce_i8'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.mul.v2i8(<2 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.mul.v4i8(<4 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.mul.v8i8(<8 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.mul.v2i8(<2 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.mul.v4i8(<4 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.mul.v8i8(<8 x i8> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 53 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.mul.v16i8(<16 x i8> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.mul.v32i8(<32 x i8> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 89 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.mul.v64i8(<64 x i8> undef)
@@ -242,9 +226,9 @@ define i32 @reduce_i8(i32 %arg) {
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE42-LABEL: 'reduce_i8'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.mul.v2i8(<2 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.mul.v4i8(<4 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.mul.v8i8(<8 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.mul.v2i8(<2 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.mul.v4i8(<4 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.mul.v8i8(<8 x i8> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 53 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.mul.v16i8(<16 x i8> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.mul.v32i8(<32 x i8> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 89 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.mul.v64i8(<64 x i8> undef)
@@ -252,9 +236,9 @@ define i32 @reduce_i8(i32 %arg) {
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX1-LABEL: 'reduce_i8'
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.mul.v2i8(<2 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.mul.v4i8(<4 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.mul.v8i8(<8 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.mul.v2i8(<2 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.mul.v4i8(<4 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.mul.v8i8(<8 x i8> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 53 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.mul.v16i8(<16 x i8> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 171 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.mul.v32i8(<32 x i8> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 197 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.mul.v64i8(<64 x i8> undef)
@@ -262,9 +246,9 @@ define i32 @reduce_i8(i32 %arg) {
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX2-LABEL: 'reduce_i8'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.mul.v2i8(<2 x i8> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.mul.v4i8(<4 x i8> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.mul.v8i8(<8 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.mul.v2i8(<2 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.mul.v4i8(<4 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.mul.v8i8(<8 x i8> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.mul.v16i8(<16 x i8> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 106 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.mul.v32i8(<32 x i8> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 123 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.mul.v64i8(<64 x i8> undef)
@@ -272,9 +256,9 @@ define i32 @reduce_i8(i32 %arg) {
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512F-LABEL: 'reduce_i8'
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.mul.v2i8(<2 x i8> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.mul.v4i8(<4 x i8> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.mul.v8i8(<8 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.mul.v2i8(<2 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.mul.v4i8(<4 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.mul.v8i8(<8 x i8> undef)
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.mul.v16i8(<16 x i8> undef)
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 86 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.mul.v32i8(<32 x i8> undef)
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 99 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.mul.v64i8(<64 x i8> undef)
@@ -282,9 +266,9 @@ define i32 @reduce_i8(i32 %arg) {
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512BW-LABEL: 'reduce_i8'
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.mul.v2i8(<2 x i8> undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.mul.v4i8(<4 x i8> undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.mul.v8i8(<8 x i8> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.mul.v2i8(<2 x i8> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.mul.v4i8(<4 x i8> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.mul.v8i8(<8 x i8> undef)
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.mul.v16i8(<16 x i8> undef)
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.mul.v32i8(<32 x i8> undef)
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 115 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.mul.v64i8(<64 x i8> undef)
@@ -292,9 +276,9 @@ define i32 @reduce_i8(i32 %arg) {
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512DQ-LABEL: 'reduce_i8'
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.mul.v2i8(<2 x i8> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.mul.v4i8(<4 x i8> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.mul.v8i8(<8 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.mul.v2i8(<2 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.mul.v4i8(<4 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.mul.v8i8(<8 x i8> undef)
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.mul.v16i8(<16 x i8> undef)
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 86 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.mul.v32i8(<32 x i8> undef)
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 99 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.mul.v64i8(<64 x i8> undef)
diff --git a/llvm/test/Analysis/CostModel/X86/reduce-or.ll b/llvm/test/Analysis/CostModel/X86/reduce-or.ll
index 16978e518..ee05562dc 100644
--- a/llvm/test/Analysis/CostModel/X86/reduce-or.ll
+++ b/llvm/test/Analysis/CostModel/X86/reduce-or.ll
@@ -92,8 +92,8 @@ define i32 @reduce_i32(i32 %arg) {
 
 define i32 @reduce_i16(i32 %arg) {
 ; SSE2-LABEL: 'reduce_i16'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.or.v2i16(<2 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.or.v4i16(<4 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.or.v2i16(<2 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.or.v4i16(<4 x i16> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.or.v8i16(<8 x i16> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.or.v16i16(<16 x i16> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.or.v32i16(<32 x i16> undef)
@@ -174,9 +174,9 @@ define i32 @reduce_i16(i32 %arg) {
 
 define i32 @reduce_i8(i32 %arg) {
 ; SSE2-LABEL: 'reduce_i8'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.or.v2i8(<2 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.or.v4i8(<4 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.or.v8i8(<8 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.or.v2i8(<2 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.or.v4i8(<4 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.or.v8i8(<8 x i8> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 45 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.or.v16i8(<16 x i8> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 46 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.or.v32i8(<32 x i8> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.or.v64i8(<64 x i8> undef)
@@ -299,35 +299,35 @@ define i32 @reduce_i1(i32 %arg) {
 ;
 ; AVX512F-LABEL: 'reduce_i1'
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1 = call i1 @llvm.experimental.vector.reduce.or.v1i1(<1 x i1> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i1 @llvm.experimental.vector.reduce.or.v2i1(<2 x i1> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V4 = call i1 @llvm.experimental.vector.reduce.or.v4i1(<4 x i1> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V8 = call i1 @llvm.experimental.vector.reduce.or.v8i1(<8 x i1> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 133 for instruction: %V16 = call i1 @llvm.experimental.vector.reduce.or.v16i1(<16 x i1> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 134 for instruction: %V32 = call i1 @llvm.experimental.vector.reduce.or.v32i1(<32 x i1> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 136 for instruction: %V64 = call i1 @llvm.experimental.vector.reduce.or.v64i1(<64 x i1> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 140 for instruction: %V128 = call i1 @llvm.experimental.vector.reduce.or.v128i1(<128 x i1> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i1 @llvm.experimental.vector.reduce.or.v2i1(<2 x i1> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4 = call i1 @llvm.experimental.vector.reduce.or.v4i1(<4 x i1> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8 = call i1 @llvm.experimental.vector.reduce.or.v8i1(<8 x i1> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V16 = call i1 @llvm.experimental.vector.reduce.or.v16i1(<16 x i1> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V32 = call i1 @llvm.experimental.vector.reduce.or.v32i1(<32 x i1> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V64 = call i1 @llvm.experimental.vector.reduce.or.v64i1(<64 x i1> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %V128 = call i1 @llvm.experimental.vector.reduce.or.v128i1(<128 x i1> undef)
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512BW-LABEL: 'reduce_i1'
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1 = call i1 @llvm.experimental.vector.reduce.or.v1i1(<1 x i1> undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i1 @llvm.experimental.vector.reduce.or.v2i1(<2 x i1> undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V4 = call i1 @llvm.experimental.vector.reduce.or.v4i1(<4 x i1> undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V8 = call i1 @llvm.experimental.vector.reduce.or.v8i1(<8 x i1> undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 133 for instruction: %V16 = call i1 @llvm.experimental.vector.reduce.or.v16i1(<16 x i1> undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 326 for instruction: %V32 = call i1 @llvm.experimental.vector.reduce.or.v32i1(<32 x i1> undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 775 for instruction: %V64 = call i1 @llvm.experimental.vector.reduce.or.v64i1(<64 x i1> undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 776 for instruction: %V128 = call i1 @llvm.experimental.vector.reduce.or.v128i1(<128 x i1> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i1 @llvm.experimental.vector.reduce.or.v2i1(<2 x i1> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4 = call i1 @llvm.experimental.vector.reduce.or.v4i1(<4 x i1> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8 = call i1 @llvm.experimental.vector.reduce.or.v8i1(<8 x i1> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V16 = call i1 @llvm.experimental.vector.reduce.or.v16i1(<16 x i1> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V32 = call i1 @llvm.experimental.vector.reduce.or.v32i1(<32 x i1> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V64 = call i1 @llvm.experimental.vector.reduce.or.v64i1(<64 x i1> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V128 = call i1 @llvm.experimental.vector.reduce.or.v128i1(<128 x i1> undef)
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512DQ-LABEL: 'reduce_i1'
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1 = call i1 @llvm.experimental.vector.reduce.or.v1i1(<1 x i1> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i1 @llvm.experimental.vector.reduce.or.v2i1(<2 x i1> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V4 = call i1 @llvm.experimental.vector.reduce.or.v4i1(<4 x i1> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V8 = call i1 @llvm.experimental.vector.reduce.or.v8i1(<8 x i1> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 133 for instruction: %V16 = call i1 @llvm.experimental.vector.reduce.or.v16i1(<16 x i1> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 134 for instruction: %V32 = call i1 @llvm.experimental.vector.reduce.or.v32i1(<32 x i1> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 136 for instruction: %V64 = call i1 @llvm.experimental.vector.reduce.or.v64i1(<64 x i1> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 140 for instruction: %V128 = call i1 @llvm.experimental.vector.reduce.or.v128i1(<128 x i1> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i1 @llvm.experimental.vector.reduce.or.v2i1(<2 x i1> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4 = call i1 @llvm.experimental.vector.reduce.or.v4i1(<4 x i1> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8 = call i1 @llvm.experimental.vector.reduce.or.v8i1(<8 x i1> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V16 = call i1 @llvm.experimental.vector.reduce.or.v16i1(<16 x i1> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V32 = call i1 @llvm.experimental.vector.reduce.or.v32i1(<32 x i1> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V64 = call i1 @llvm.experimental.vector.reduce.or.v64i1(<64 x i1> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %V128 = call i1 @llvm.experimental.vector.reduce.or.v128i1(<128 x i1> undef)
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %V1   = call i1 @llvm.experimental.vector.reduce.or.v1i1(<1 x i1> undef)
diff --git a/llvm/test/Analysis/CostModel/X86/reduce-smax.ll b/llvm/test/Analysis/CostModel/X86/reduce-smax.ll
index 54613791e..a578ad67f 100644
--- a/llvm/test/Analysis/CostModel/X86/reduce-smax.ll
+++ b/llvm/test/Analysis/CostModel/X86/reduce-smax.ll
@@ -83,7 +83,7 @@ define i32 @reduce_i32(i32 %arg) {
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE42-LABEL: 'reduce_i32'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.smax.v2i32(<2 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.smax.v2i32(<2 x i32> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.smax.v4i32(<4 x i32> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.smax.v8i32(<8 x i32> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.smax.v16i32(<16 x i32> undef)
@@ -91,7 +91,7 @@ define i32 @reduce_i32(i32 %arg) {
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX1-LABEL: 'reduce_i32'
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.smax.v2i32(<2 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.smax.v2i32(<2 x i32> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.smax.v4i32(<4 x i32> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.smax.v8i32(<8 x i32> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.smax.v16i32(<16 x i32> undef)
@@ -99,7 +99,7 @@ define i32 @reduce_i32(i32 %arg) {
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX2-LABEL: 'reduce_i32'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.smax.v2i32(<2 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.smax.v2i32(<2 x i32> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.smax.v4i32(<4 x i32> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.smax.v8i32(<8 x i32> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.smax.v16i32(<16 x i32> undef)
@@ -107,7 +107,7 @@ define i32 @reduce_i32(i32 %arg) {
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512-LABEL: 'reduce_i32'
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.smax.v2i32(<2 x i32> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.smax.v2i32(<2 x i32> undef)
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.smax.v4i32(<4 x i32> undef)
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.smax.v8i32(<8 x i32> undef)
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.smax.v16i32(<16 x i32> undef)
@@ -124,8 +124,8 @@ define i32 @reduce_i32(i32 %arg) {
 
 define i32 @reduce_i16(i32 %arg) {
 ; SSE2-LABEL: 'reduce_i16'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.smax.v2i16(<2 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.smax.v4i16(<4 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.smax.v2i16(<2 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.smax.v4i16(<4 x i16> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.smax.v8i16(<8 x i16> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.smax.v16i16(<16 x i16> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.smax.v32i16(<32 x i16> undef)
@@ -133,8 +133,8 @@ define i32 @reduce_i16(i32 %arg) {
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'reduce_i16'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.smax.v2i16(<2 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.smax.v4i16(<4 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.smax.v2i16(<2 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.smax.v4i16(<4 x i16> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.smax.v8i16(<8 x i16> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.smax.v16i16(<16 x i16> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.smax.v32i16(<32 x i16> undef)
@@ -142,7 +142,7 @@ define i32 @reduce_i16(i32 %arg) {
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE42-LABEL: 'reduce_i16'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.smax.v2i16(<2 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.smax.v2i16(<2 x i16> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.smax.v4i16(<4 x i16> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.smax.v8i16(<8 x i16> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.smax.v16i16(<16 x i16> undef)
@@ -151,7 +151,7 @@ define i32 @reduce_i16(i32 %arg) {
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX1-LABEL: 'reduce_i16'
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.smax.v2i16(<2 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.smax.v2i16(<2 x i16> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.smax.v4i16(<4 x i16> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.smax.v8i16(<8 x i16> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.smax.v16i16(<16 x i16> undef)
@@ -160,7 +160,7 @@ define i32 @reduce_i16(i32 %arg) {
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX2-LABEL: 'reduce_i16'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.smax.v2i16(<2 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.smax.v2i16(<2 x i16> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.smax.v4i16(<4 x i16> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.smax.v8i16(<8 x i16> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.smax.v16i16(<16 x i16> undef)
@@ -169,7 +169,7 @@ define i32 @reduce_i16(i32 %arg) {
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512F-LABEL: 'reduce_i16'
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.smax.v2i16(<2 x i16> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.smax.v2i16(<2 x i16> undef)
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.smax.v4i16(<4 x i16> undef)
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.smax.v8i16(<8 x i16> undef)
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.smax.v16i16(<16 x i16> undef)
@@ -178,7 +178,7 @@ define i32 @reduce_i16(i32 %arg) {
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512BW-LABEL: 'reduce_i16'
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.smax.v2i16(<2 x i16> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.smax.v2i16(<2 x i16> undef)
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.smax.v4i16(<4 x i16> undef)
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.smax.v8i16(<8 x i16> undef)
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.smax.v16i16(<16 x i16> undef)
@@ -187,7 +187,7 @@ define i32 @reduce_i16(i32 %arg) {
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512DQ-LABEL: 'reduce_i16'
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.smax.v2i16(<2 x i16> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.smax.v2i16(<2 x i16> undef)
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.smax.v4i16(<4 x i16> undef)
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.smax.v8i16(<8 x i16> undef)
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.smax.v16i16(<16 x i16> undef)
@@ -206,8 +206,8 @@ define i32 @reduce_i16(i32 %arg) {
 
 define i32 @reduce_i8(i32 %arg) {
 ; SSE2-LABEL: 'reduce_i8'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.smax.v2i8(<2 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.smax.v4i8(<4 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.smax.v2i8(<2 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.smax.v4i8(<4 x i8> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.smax.v8i8(<8 x i8> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.smax.v16i8(<16 x i8> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.smax.v32i8(<32 x i8> undef)
@@ -216,8 +216,8 @@ define i32 @reduce_i8(i32 %arg) {
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'reduce_i8'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.smax.v2i8(<2 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.smax.v4i8(<4 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.smax.v2i8(<2 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.smax.v4i8(<4 x i8> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.smax.v8i8(<8 x i8> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.smax.v16i8(<16 x i8> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.smax.v32i8(<32 x i8> undef)
@@ -226,9 +226,9 @@ define i32 @reduce_i8(i32 %arg) {
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE42-LABEL: 'reduce_i8'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.smax.v2i8(<2 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.smax.v4i8(<4 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.smax.v8i8(<8 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.smax.v2i8(<2 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.smax.v4i8(<4 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.smax.v8i8(<8 x i8> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.smax.v16i8(<16 x i8> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.smax.v32i8(<32 x i8> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.smax.v64i8(<64 x i8> undef)
@@ -236,9 +236,9 @@ define i32 @reduce_i8(i32 %arg) {
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX1-LABEL: 'reduce_i8'
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.smax.v2i8(<2 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.smax.v4i8(<4 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.smax.v8i8(<8 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.smax.v2i8(<2 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.smax.v4i8(<4 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.smax.v8i8(<8 x i8> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.smax.v16i8(<16 x i8> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.smax.v32i8(<32 x i8> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.smax.v64i8(<64 x i8> undef)
@@ -246,9 +246,9 @@ define i32 @reduce_i8(i32 %arg) {
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX2-LABEL: 'reduce_i8'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.smax.v2i8(<2 x i8> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.smax.v4i8(<4 x i8> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.smax.v8i8(<8 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.smax.v2i8(<2 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.smax.v4i8(<4 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.smax.v8i8(<8 x i8> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.smax.v16i8(<16 x i8> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.smax.v32i8(<32 x i8> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.smax.v64i8(<64 x i8> undef)
@@ -256,9 +256,9 @@ define i32 @reduce_i8(i32 %arg) {
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512F-LABEL: 'reduce_i8'
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.smax.v2i8(<2 x i8> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.smax.v4i8(<4 x i8> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.smax.v8i8(<8 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.smax.v2i8(<2 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.smax.v4i8(<4 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.smax.v8i8(<8 x i8> undef)
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.smax.v16i8(<16 x i8> undef)
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.smax.v32i8(<32 x i8> undef)
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.smax.v64i8(<64 x i8> undef)
@@ -266,9 +266,9 @@ define i32 @reduce_i8(i32 %arg) {
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512BW-LABEL: 'reduce_i8'
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.smax.v2i8(<2 x i8> undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.smax.v4i8(<4 x i8> undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.smax.v8i8(<8 x i8> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.smax.v2i8(<2 x i8> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.smax.v4i8(<4 x i8> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.smax.v8i8(<8 x i8> undef)
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.smax.v16i8(<16 x i8> undef)
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.smax.v32i8(<32 x i8> undef)
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 61 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.smax.v64i8(<64 x i8> undef)
@@ -276,9 +276,9 @@ define i32 @reduce_i8(i32 %arg) {
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512DQ-LABEL: 'reduce_i8'
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.smax.v2i8(<2 x i8> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.smax.v4i8(<4 x i8> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.smax.v8i8(<8 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.smax.v2i8(<2 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.smax.v4i8(<4 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.smax.v8i8(<8 x i8> undef)
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.smax.v16i8(<16 x i8> undef)
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.smax.v32i8(<32 x i8> undef)
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.smax.v64i8(<64 x i8> undef)
diff --git a/llvm/test/Analysis/CostModel/X86/reduce-smin.ll b/llvm/test/Analysis/CostModel/X86/reduce-smin.ll
index 777ac088e..01263daea 100644
--- a/llvm/test/Analysis/CostModel/X86/reduce-smin.ll
+++ b/llvm/test/Analysis/CostModel/X86/reduce-smin.ll
@@ -83,7 +83,7 @@ define i32 @reduce_i32(i32 %arg) {
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE42-LABEL: 'reduce_i32'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.smin.v2i32(<2 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.smin.v2i32(<2 x i32> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.smin.v4i32(<4 x i32> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.smin.v8i32(<8 x i32> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.smin.v16i32(<16 x i32> undef)
@@ -91,7 +91,7 @@ define i32 @reduce_i32(i32 %arg) {
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX1-LABEL: 'reduce_i32'
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.smin.v2i32(<2 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.smin.v2i32(<2 x i32> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.smin.v4i32(<4 x i32> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.smin.v8i32(<8 x i32> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.smin.v16i32(<16 x i32> undef)
@@ -99,7 +99,7 @@ define i32 @reduce_i32(i32 %arg) {
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX2-LABEL: 'reduce_i32'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.smin.v2i32(<2 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.smin.v2i32(<2 x i32> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.smin.v4i32(<4 x i32> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.smin.v8i32(<8 x i32> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.smin.v16i32(<16 x i32> undef)
@@ -107,7 +107,7 @@ define i32 @reduce_i32(i32 %arg) {
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512-LABEL: 'reduce_i32'
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.smin.v2i32(<2 x i32> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.smin.v2i32(<2 x i32> undef)
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.smin.v4i32(<4 x i32> undef)
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.smin.v8i32(<8 x i32> undef)
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.smin.v16i32(<16 x i32> undef)
@@ -124,8 +124,8 @@ define i32 @reduce_i32(i32 %arg) {
 
 define i32 @reduce_i16(i32 %arg) {
 ; SSE2-LABEL: 'reduce_i16'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.smin.v2i16(<2 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.smin.v4i16(<4 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.smin.v2i16(<2 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.smin.v4i16(<4 x i16> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.smin.v8i16(<8 x i16> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.smin.v16i16(<16 x i16> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.smin.v32i16(<32 x i16> undef)
@@ -133,8 +133,8 @@ define i32 @reduce_i16(i32 %arg) {
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'reduce_i16'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.smin.v2i16(<2 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.smin.v4i16(<4 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.smin.v2i16(<2 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.smin.v4i16(<4 x i16> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.smin.v8i16(<8 x i16> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.smin.v16i16(<16 x i16> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.smin.v32i16(<32 x i16> undef)
@@ -142,7 +142,7 @@ define i32 @reduce_i16(i32 %arg) {
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE42-LABEL: 'reduce_i16'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.smin.v2i16(<2 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.smin.v2i16(<2 x i16> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.smin.v4i16(<4 x i16> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.smin.v8i16(<8 x i16> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.smin.v16i16(<16 x i16> undef)
@@ -151,7 +151,7 @@ define i32 @reduce_i16(i32 %arg) {
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX1-LABEL: 'reduce_i16'
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.smin.v2i16(<2 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.smin.v2i16(<2 x i16> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.smin.v4i16(<4 x i16> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.smin.v8i16(<8 x i16> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.smin.v16i16(<16 x i16> undef)
@@ -160,7 +160,7 @@ define i32 @reduce_i16(i32 %arg) {
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX2-LABEL: 'reduce_i16'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.smin.v2i16(<2 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.smin.v2i16(<2 x i16> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.smin.v4i16(<4 x i16> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.smin.v8i16(<8 x i16> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.smin.v16i16(<16 x i16> undef)
@@ -169,7 +169,7 @@ define i32 @reduce_i16(i32 %arg) {
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512F-LABEL: 'reduce_i16'
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.smin.v2i16(<2 x i16> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.smin.v2i16(<2 x i16> undef)
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.smin.v4i16(<4 x i16> undef)
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.smin.v8i16(<8 x i16> undef)
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.smin.v16i16(<16 x i16> undef)
@@ -178,7 +178,7 @@ define i32 @reduce_i16(i32 %arg) {
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512BW-LABEL: 'reduce_i16'
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.smin.v2i16(<2 x i16> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.smin.v2i16(<2 x i16> undef)
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.smin.v4i16(<4 x i16> undef)
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.smin.v8i16(<8 x i16> undef)
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.smin.v16i16(<16 x i16> undef)
@@ -187,7 +187,7 @@ define i32 @reduce_i16(i32 %arg) {
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512DQ-LABEL: 'reduce_i16'
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.smin.v2i16(<2 x i16> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.smin.v2i16(<2 x i16> undef)
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.smin.v4i16(<4 x i16> undef)
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.smin.v8i16(<8 x i16> undef)
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.smin.v16i16(<16 x i16> undef)
@@ -206,8 +206,8 @@ define i32 @reduce_i16(i32 %arg) {
 
 define i32 @reduce_i8(i32 %arg) {
 ; SSE2-LABEL: 'reduce_i8'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.smin.v2i8(<2 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.smin.v4i8(<4 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.smin.v2i8(<2 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.smin.v4i8(<4 x i8> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.smin.v8i8(<8 x i8> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.smin.v16i8(<16 x i8> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.smin.v32i8(<32 x i8> undef)
@@ -216,8 +216,8 @@ define i32 @reduce_i8(i32 %arg) {
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'reduce_i8'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.smin.v2i8(<2 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.smin.v4i8(<4 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.smin.v2i8(<2 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.smin.v4i8(<4 x i8> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.smin.v8i8(<8 x i8> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.smin.v16i8(<16 x i8> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.smin.v32i8(<32 x i8> undef)
@@ -226,9 +226,9 @@ define i32 @reduce_i8(i32 %arg) {
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE42-LABEL: 'reduce_i8'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.smin.v2i8(<2 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.smin.v4i8(<4 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.smin.v8i8(<8 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.smin.v2i8(<2 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.smin.v4i8(<4 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.smin.v8i8(<8 x i8> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.smin.v16i8(<16 x i8> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.smin.v32i8(<32 x i8> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.smin.v64i8(<64 x i8> undef)
@@ -236,9 +236,9 @@ define i32 @reduce_i8(i32 %arg) {
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX1-LABEL: 'reduce_i8'
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.smin.v2i8(<2 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.smin.v4i8(<4 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.smin.v8i8(<8 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.smin.v2i8(<2 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.smin.v4i8(<4 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.smin.v8i8(<8 x i8> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.smin.v16i8(<16 x i8> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.smin.v32i8(<32 x i8> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.smin.v64i8(<64 x i8> undef)
@@ -246,9 +246,9 @@ define i32 @reduce_i8(i32 %arg) {
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX2-LABEL: 'reduce_i8'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.smin.v2i8(<2 x i8> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.smin.v4i8(<4 x i8> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.smin.v8i8(<8 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.smin.v2i8(<2 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.smin.v4i8(<4 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.smin.v8i8(<8 x i8> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.smin.v16i8(<16 x i8> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.smin.v32i8(<32 x i8> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.smin.v64i8(<64 x i8> undef)
@@ -256,9 +256,9 @@ define i32 @reduce_i8(i32 %arg) {
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512F-LABEL: 'reduce_i8'
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.smin.v2i8(<2 x i8> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.smin.v4i8(<4 x i8> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.smin.v8i8(<8 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.smin.v2i8(<2 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.smin.v4i8(<4 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.smin.v8i8(<8 x i8> undef)
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.smin.v16i8(<16 x i8> undef)
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.smin.v32i8(<32 x i8> undef)
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.smin.v64i8(<64 x i8> undef)
@@ -266,9 +266,9 @@ define i32 @reduce_i8(i32 %arg) {
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512BW-LABEL: 'reduce_i8'
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.smin.v2i8(<2 x i8> undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.smin.v4i8(<4 x i8> undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.smin.v8i8(<8 x i8> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.smin.v2i8(<2 x i8> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.smin.v4i8(<4 x i8> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.smin.v8i8(<8 x i8> undef)
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.smin.v16i8(<16 x i8> undef)
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.smin.v32i8(<32 x i8> undef)
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 61 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.smin.v64i8(<64 x i8> undef)
@@ -276,9 +276,9 @@ define i32 @reduce_i8(i32 %arg) {
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512DQ-LABEL: 'reduce_i8'
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.smin.v2i8(<2 x i8> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.smin.v4i8(<4 x i8> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.smin.v8i8(<8 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.smin.v2i8(<2 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.smin.v4i8(<4 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.smin.v8i8(<8 x i8> undef)
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.smin.v16i8(<16 x i8> undef)
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.smin.v32i8(<32 x i8> undef)
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.smin.v64i8(<64 x i8> undef)
diff --git a/llvm/test/Analysis/CostModel/X86/reduce-umax.ll b/llvm/test/Analysis/CostModel/X86/reduce-umax.ll
index 1259bdf8c..f77915e1b 100644
--- a/llvm/test/Analysis/CostModel/X86/reduce-umax.ll
+++ b/llvm/test/Analysis/CostModel/X86/reduce-umax.ll
@@ -83,7 +83,7 @@ define i32 @reduce_i32(i32 %arg) {
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE42-LABEL: 'reduce_i32'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.umax.v2i32(<2 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.umax.v2i32(<2 x i32> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.umax.v4i32(<4 x i32> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.umax.v8i32(<8 x i32> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.umax.v16i32(<16 x i32> undef)
@@ -91,7 +91,7 @@ define i32 @reduce_i32(i32 %arg) {
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX1-LABEL: 'reduce_i32'
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.umax.v2i32(<2 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.umax.v2i32(<2 x i32> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.umax.v4i32(<4 x i32> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.umax.v8i32(<8 x i32> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.umax.v16i32(<16 x i32> undef)
@@ -99,7 +99,7 @@ define i32 @reduce_i32(i32 %arg) {
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX2-LABEL: 'reduce_i32'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.umax.v2i32(<2 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.umax.v2i32(<2 x i32> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.umax.v4i32(<4 x i32> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.umax.v8i32(<8 x i32> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.umax.v16i32(<16 x i32> undef)
@@ -107,7 +107,7 @@ define i32 @reduce_i32(i32 %arg) {
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512-LABEL: 'reduce_i32'
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.umax.v2i32(<2 x i32> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.umax.v2i32(<2 x i32> undef)
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.umax.v4i32(<4 x i32> undef)
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.umax.v8i32(<8 x i32> undef)
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.umax.v16i32(<16 x i32> undef)
@@ -124,8 +124,8 @@ define i32 @reduce_i32(i32 %arg) {
 
 define i32 @reduce_i16(i32 %arg) {
 ; SSE2-LABEL: 'reduce_i16'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.umax.v2i16(<2 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.umax.v4i16(<4 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.umax.v2i16(<2 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.umax.v4i16(<4 x i16> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.umax.v8i16(<8 x i16> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.umax.v16i16(<16 x i16> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.umax.v32i16(<32 x i16> undef)
@@ -133,8 +133,8 @@ define i32 @reduce_i16(i32 %arg) {
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'reduce_i16'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.umax.v2i16(<2 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.umax.v4i16(<4 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.umax.v2i16(<2 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.umax.v4i16(<4 x i16> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.umax.v8i16(<8 x i16> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.umax.v16i16(<16 x i16> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.umax.v32i16(<32 x i16> undef)
@@ -142,7 +142,7 @@ define i32 @reduce_i16(i32 %arg) {
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE42-LABEL: 'reduce_i16'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.umax.v2i16(<2 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.umax.v2i16(<2 x i16> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.umax.v4i16(<4 x i16> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.umax.v8i16(<8 x i16> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.umax.v16i16(<16 x i16> undef)
@@ -151,7 +151,7 @@ define i32 @reduce_i16(i32 %arg) {
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX1-LABEL: 'reduce_i16'
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.umax.v2i16(<2 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.umax.v2i16(<2 x i16> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.umax.v4i16(<4 x i16> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.umax.v8i16(<8 x i16> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.umax.v16i16(<16 x i16> undef)
@@ -160,7 +160,7 @@ define i32 @reduce_i16(i32 %arg) {
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX2-LABEL: 'reduce_i16'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.umax.v2i16(<2 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.umax.v2i16(<2 x i16> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.umax.v4i16(<4 x i16> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.umax.v8i16(<8 x i16> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.umax.v16i16(<16 x i16> undef)
@@ -169,7 +169,7 @@ define i32 @reduce_i16(i32 %arg) {
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512F-LABEL: 'reduce_i16'
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.umax.v2i16(<2 x i16> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.umax.v2i16(<2 x i16> undef)
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.umax.v4i16(<4 x i16> undef)
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.umax.v8i16(<8 x i16> undef)
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.umax.v16i16(<16 x i16> undef)
@@ -178,7 +178,7 @@ define i32 @reduce_i16(i32 %arg) {
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512BW-LABEL: 'reduce_i16'
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.umax.v2i16(<2 x i16> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.umax.v2i16(<2 x i16> undef)
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.umax.v4i16(<4 x i16> undef)
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.umax.v8i16(<8 x i16> undef)
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.umax.v16i16(<16 x i16> undef)
@@ -187,7 +187,7 @@ define i32 @reduce_i16(i32 %arg) {
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512DQ-LABEL: 'reduce_i16'
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.umax.v2i16(<2 x i16> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.umax.v2i16(<2 x i16> undef)
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.umax.v4i16(<4 x i16> undef)
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.umax.v8i16(<8 x i16> undef)
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.umax.v16i16(<16 x i16> undef)
@@ -206,9 +206,9 @@ define i32 @reduce_i16(i32 %arg) {
 
 define i32 @reduce_i8(i32 %arg) {
 ; SSE2-LABEL: 'reduce_i8'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.umax.v2i8(<2 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.umax.v4i8(<4 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.umax.v8i8(<8 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.umax.v2i8(<2 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.umax.v4i8(<4 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.umax.v8i8(<8 x i8> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.umax.v16i8(<16 x i8> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.umax.v32i8(<32 x i8> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.umax.v64i8(<64 x i8> undef)
@@ -216,9 +216,9 @@ define i32 @reduce_i8(i32 %arg) {
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'reduce_i8'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.umax.v2i8(<2 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.umax.v4i8(<4 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.umax.v8i8(<8 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.umax.v2i8(<2 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.umax.v4i8(<4 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.umax.v8i8(<8 x i8> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.umax.v16i8(<16 x i8> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.umax.v32i8(<32 x i8> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.umax.v64i8(<64 x i8> undef)
@@ -226,9 +226,9 @@ define i32 @reduce_i8(i32 %arg) {
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE42-LABEL: 'reduce_i8'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.umax.v2i8(<2 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.umax.v4i8(<4 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.umax.v8i8(<8 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.umax.v2i8(<2 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.umax.v4i8(<4 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.umax.v8i8(<8 x i8> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.umax.v16i8(<16 x i8> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.umax.v32i8(<32 x i8> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.umax.v64i8(<64 x i8> undef)
@@ -236,9 +236,9 @@ define i32 @reduce_i8(i32 %arg) {
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX1-LABEL: 'reduce_i8'
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.umax.v2i8(<2 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.umax.v4i8(<4 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.umax.v8i8(<8 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.umax.v2i8(<2 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.umax.v4i8(<4 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.umax.v8i8(<8 x i8> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.umax.v16i8(<16 x i8> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.umax.v32i8(<32 x i8> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.umax.v64i8(<64 x i8> undef)
@@ -246,9 +246,9 @@ define i32 @reduce_i8(i32 %arg) {
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX2-LABEL: 'reduce_i8'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.umax.v2i8(<2 x i8> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.umax.v4i8(<4 x i8> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.umax.v8i8(<8 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.umax.v2i8(<2 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.umax.v4i8(<4 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.umax.v8i8(<8 x i8> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.umax.v16i8(<16 x i8> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.umax.v32i8(<32 x i8> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.umax.v64i8(<64 x i8> undef)
@@ -256,9 +256,9 @@ define i32 @reduce_i8(i32 %arg) {
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512F-LABEL: 'reduce_i8'
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.umax.v2i8(<2 x i8> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.umax.v4i8(<4 x i8> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.umax.v8i8(<8 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.umax.v2i8(<2 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.umax.v4i8(<4 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.umax.v8i8(<8 x i8> undef)
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.umax.v16i8(<16 x i8> undef)
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.umax.v32i8(<32 x i8> undef)
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.umax.v64i8(<64 x i8> undef)
@@ -266,9 +266,9 @@ define i32 @reduce_i8(i32 %arg) {
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512BW-LABEL: 'reduce_i8'
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.umax.v2i8(<2 x i8> undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.umax.v4i8(<4 x i8> undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.umax.v8i8(<8 x i8> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.umax.v2i8(<2 x i8> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.umax.v4i8(<4 x i8> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.umax.v8i8(<8 x i8> undef)
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.umax.v16i8(<16 x i8> undef)
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.umax.v32i8(<32 x i8> undef)
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 61 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.umax.v64i8(<64 x i8> undef)
@@ -276,9 +276,9 @@ define i32 @reduce_i8(i32 %arg) {
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512DQ-LABEL: 'reduce_i8'
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.umax.v2i8(<2 x i8> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.umax.v4i8(<4 x i8> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.umax.v8i8(<8 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.umax.v2i8(<2 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.umax.v4i8(<4 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.umax.v8i8(<8 x i8> undef)
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.umax.v16i8(<16 x i8> undef)
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.umax.v32i8(<32 x i8> undef)
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.umax.v64i8(<64 x i8> undef)
diff --git a/llvm/test/Analysis/CostModel/X86/reduce-umin.ll b/llvm/test/Analysis/CostModel/X86/reduce-umin.ll
index b946310e8..ce35e8b24 100644
--- a/llvm/test/Analysis/CostModel/X86/reduce-umin.ll
+++ b/llvm/test/Analysis/CostModel/X86/reduce-umin.ll
@@ -83,7 +83,7 @@ define i32 @reduce_i32(i32 %arg) {
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE42-LABEL: 'reduce_i32'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.umin.v2i32(<2 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.umin.v2i32(<2 x i32> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.umin.v4i32(<4 x i32> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.umin.v8i32(<8 x i32> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.umin.v16i32(<16 x i32> undef)
@@ -91,7 +91,7 @@ define i32 @reduce_i32(i32 %arg) {
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX1-LABEL: 'reduce_i32'
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.umin.v2i32(<2 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.umin.v2i32(<2 x i32> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.umin.v4i32(<4 x i32> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.umin.v8i32(<8 x i32> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.umin.v16i32(<16 x i32> undef)
@@ -99,7 +99,7 @@ define i32 @reduce_i32(i32 %arg) {
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX2-LABEL: 'reduce_i32'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.umin.v2i32(<2 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.umin.v2i32(<2 x i32> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.umin.v4i32(<4 x i32> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.umin.v8i32(<8 x i32> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.umin.v16i32(<16 x i32> undef)
@@ -107,7 +107,7 @@ define i32 @reduce_i32(i32 %arg) {
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512-LABEL: 'reduce_i32'
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.umin.v2i32(<2 x i32> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.umin.v2i32(<2 x i32> undef)
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.umin.v4i32(<4 x i32> undef)
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.umin.v8i32(<8 x i32> undef)
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.umin.v16i32(<16 x i32> undef)
@@ -124,8 +124,8 @@ define i32 @reduce_i32(i32 %arg) {
 
 define i32 @reduce_i16(i32 %arg) {
 ; SSE2-LABEL: 'reduce_i16'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.umin.v2i16(<2 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.umin.v4i16(<4 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.umin.v2i16(<2 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.umin.v4i16(<4 x i16> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.umin.v8i16(<8 x i16> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.umin.v16i16(<16 x i16> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.umin.v32i16(<32 x i16> undef)
@@ -133,8 +133,8 @@ define i32 @reduce_i16(i32 %arg) {
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'reduce_i16'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.umin.v2i16(<2 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.umin.v4i16(<4 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.umin.v2i16(<2 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.umin.v4i16(<4 x i16> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.umin.v8i16(<8 x i16> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.umin.v16i16(<16 x i16> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.umin.v32i16(<32 x i16> undef)
@@ -142,7 +142,7 @@ define i32 @reduce_i16(i32 %arg) {
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE42-LABEL: 'reduce_i16'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.umin.v2i16(<2 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.umin.v2i16(<2 x i16> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.umin.v4i16(<4 x i16> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.umin.v8i16(<8 x i16> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.umin.v16i16(<16 x i16> undef)
@@ -151,7 +151,7 @@ define i32 @reduce_i16(i32 %arg) {
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX1-LABEL: 'reduce_i16'
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.umin.v2i16(<2 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.umin.v2i16(<2 x i16> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.umin.v4i16(<4 x i16> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.umin.v8i16(<8 x i16> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.umin.v16i16(<16 x i16> undef)
@@ -160,7 +160,7 @@ define i32 @reduce_i16(i32 %arg) {
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX2-LABEL: 'reduce_i16'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.umin.v2i16(<2 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.umin.v2i16(<2 x i16> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.umin.v4i16(<4 x i16> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.umin.v8i16(<8 x i16> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.umin.v16i16(<16 x i16> undef)
@@ -169,7 +169,7 @@ define i32 @reduce_i16(i32 %arg) {
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512F-LABEL: 'reduce_i16'
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.umin.v2i16(<2 x i16> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.umin.v2i16(<2 x i16> undef)
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.umin.v4i16(<4 x i16> undef)
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.umin.v8i16(<8 x i16> undef)
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.umin.v16i16(<16 x i16> undef)
@@ -178,7 +178,7 @@ define i32 @reduce_i16(i32 %arg) {
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512BW-LABEL: 'reduce_i16'
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.umin.v2i16(<2 x i16> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.umin.v2i16(<2 x i16> undef)
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.umin.v4i16(<4 x i16> undef)
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.umin.v8i16(<8 x i16> undef)
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.umin.v16i16(<16 x i16> undef)
@@ -187,7 +187,7 @@ define i32 @reduce_i16(i32 %arg) {
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512DQ-LABEL: 'reduce_i16'
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.umin.v2i16(<2 x i16> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.umin.v2i16(<2 x i16> undef)
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.umin.v4i16(<4 x i16> undef)
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.umin.v8i16(<8 x i16> undef)
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.umin.v16i16(<16 x i16> undef)
@@ -206,9 +206,9 @@ define i32 @reduce_i16(i32 %arg) {
 
 define i32 @reduce_i8(i32 %arg) {
 ; SSE2-LABEL: 'reduce_i8'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.umin.v2i8(<2 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.umin.v4i8(<4 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.umin.v8i8(<8 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.umin.v2i8(<2 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.umin.v4i8(<4 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.umin.v8i8(<8 x i8> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.umin.v16i8(<16 x i8> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.umin.v32i8(<32 x i8> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.umin.v64i8(<64 x i8> undef)
@@ -216,9 +216,9 @@ define i32 @reduce_i8(i32 %arg) {
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'reduce_i8'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.umin.v2i8(<2 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.umin.v4i8(<4 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.umin.v8i8(<8 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.umin.v2i8(<2 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.umin.v4i8(<4 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.umin.v8i8(<8 x i8> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.umin.v16i8(<16 x i8> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.umin.v32i8(<32 x i8> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.umin.v64i8(<64 x i8> undef)
@@ -226,9 +226,9 @@ define i32 @reduce_i8(i32 %arg) {
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE42-LABEL: 'reduce_i8'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.umin.v2i8(<2 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.umin.v4i8(<4 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.umin.v8i8(<8 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.umin.v2i8(<2 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.umin.v4i8(<4 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.umin.v8i8(<8 x i8> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.umin.v16i8(<16 x i8> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.umin.v32i8(<32 x i8> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.umin.v64i8(<64 x i8> undef)
@@ -236,9 +236,9 @@ define i32 @reduce_i8(i32 %arg) {
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX1-LABEL: 'reduce_i8'
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.umin.v2i8(<2 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.umin.v4i8(<4 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.umin.v8i8(<8 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.umin.v2i8(<2 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.umin.v4i8(<4 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.umin.v8i8(<8 x i8> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.umin.v16i8(<16 x i8> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.umin.v32i8(<32 x i8> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.umin.v64i8(<64 x i8> undef)
@@ -246,9 +246,9 @@ define i32 @reduce_i8(i32 %arg) {
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX2-LABEL: 'reduce_i8'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.umin.v2i8(<2 x i8> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.umin.v4i8(<4 x i8> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.umin.v8i8(<8 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.umin.v2i8(<2 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.umin.v4i8(<4 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.umin.v8i8(<8 x i8> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.umin.v16i8(<16 x i8> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.umin.v32i8(<32 x i8> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.umin.v64i8(<64 x i8> undef)
@@ -256,9 +256,9 @@ define i32 @reduce_i8(i32 %arg) {
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512F-LABEL: 'reduce_i8'
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.umin.v2i8(<2 x i8> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.umin.v4i8(<4 x i8> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.umin.v8i8(<8 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.umin.v2i8(<2 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.umin.v4i8(<4 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.umin.v8i8(<8 x i8> undef)
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.umin.v16i8(<16 x i8> undef)
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.umin.v32i8(<32 x i8> undef)
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.umin.v64i8(<64 x i8> undef)
@@ -266,9 +266,9 @@ define i32 @reduce_i8(i32 %arg) {
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512BW-LABEL: 'reduce_i8'
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.umin.v2i8(<2 x i8> undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.umin.v4i8(<4 x i8> undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.umin.v8i8(<8 x i8> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.umin.v2i8(<2 x i8> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.umin.v4i8(<4 x i8> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.umin.v8i8(<8 x i8> undef)
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.umin.v16i8(<16 x i8> undef)
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.umin.v32i8(<32 x i8> undef)
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 61 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.umin.v64i8(<64 x i8> undef)
@@ -276,9 +276,9 @@ define i32 @reduce_i8(i32 %arg) {
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512DQ-LABEL: 'reduce_i8'
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.umin.v2i8(<2 x i8> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.umin.v4i8(<4 x i8> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.umin.v8i8(<8 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.umin.v2i8(<2 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.umin.v4i8(<4 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.umin.v8i8(<8 x i8> undef)
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.umin.v16i8(<16 x i8> undef)
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.umin.v32i8(<32 x i8> undef)
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.umin.v64i8(<64 x i8> undef)
diff --git a/llvm/test/Analysis/CostModel/X86/reduce-xor.ll b/llvm/test/Analysis/CostModel/X86/reduce-xor.ll
index a97fc21f3..df927b436 100644
--- a/llvm/test/Analysis/CostModel/X86/reduce-xor.ll
+++ b/llvm/test/Analysis/CostModel/X86/reduce-xor.ll
@@ -92,8 +92,8 @@ define i32 @reduce_i32(i32 %arg) {
 
 define i32 @reduce_i16(i32 %arg) {
 ; SSE2-LABEL: 'reduce_i16'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.xor.v2i16(<2 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.xor.v4i16(<4 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.xor.v2i16(<2 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.xor.v4i16(<4 x i16> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.xor.v8i16(<8 x i16> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.xor.v16i16(<16 x i16> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.xor.v32i16(<32 x i16> undef)
@@ -174,9 +174,9 @@ define i32 @reduce_i16(i32 %arg) {
 
 define i32 @reduce_i8(i32 %arg) {
 ; SSE2-LABEL: 'reduce_i8'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.xor.v2i8(<2 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.xor.v4i8(<4 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.xor.v8i8(<8 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.xor.v2i8(<2 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.xor.v4i8(<4 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.xor.v8i8(<8 x i8> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 45 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.xor.v16i8(<16 x i8> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 46 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.xor.v32i8(<32 x i8> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.xor.v64i8(<64 x i8> undef)
diff --git a/llvm/test/Analysis/CostModel/X86/reduction.ll b/llvm/test/Analysis/CostModel/X86/reduction.ll
index 0acb16c0c..ac3730205 100644
--- a/llvm/test/Analysis/CostModel/X86/reduction.ll
+++ b/llvm/test/Analysis/CostModel/X86/reduction.ll
@@ -5,6 +5,8 @@
 ; RUN: opt < %s -cost-model -costmodel-reduxcost=true -mtriple=x86_64-apple-darwin -analyze -mattr=+avx | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
 ; RUN: opt < %s -cost-model -costmodel-reduxcost=true -mtriple=x86_64-apple-darwin -analyze -mattr=+avx2 | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
 
+; RUN: opt < %s -cost-model -costmodel-reduxcost=true -mtriple=x86_64-apple-darwin -analyze -mcpu=slm | FileCheck %s --check-prefixes=SLM
+
 ; Check that we recognize the tree starting at the extractelement as a
 ; reduction.
 ; NOTE: We're only really interested in the extractelement cost, which represents the entire reduction.
@@ -15,7 +17,7 @@ define fastcc float @reduction_cost_float(<4 x float> %rdx) {
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx = fadd <4 x float> %rdx, %rdx.shuf
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <4 x float> %bin.rdx, <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx8 = fadd <4 x float> %bin.rdx, %rdx.shuf7
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %r = extractelement <4 x float> %bin.rdx8, i32 0
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r = extractelement <4 x float> %bin.rdx8, i32 0
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r
 ;
 ; SSSE3-LABEL: 'reduction_cost_float'
@@ -23,7 +25,7 @@ define fastcc float @reduction_cost_float(<4 x float> %rdx) {
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx = fadd <4 x float> %rdx, %rdx.shuf
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <4 x float> %bin.rdx, <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx8 = fadd <4 x float> %bin.rdx, %rdx.shuf7
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %r = extractelement <4 x float> %bin.rdx8, i32 0
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r = extractelement <4 x float> %bin.rdx8, i32 0
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r
 ;
 ; SSE42-LABEL: 'reduction_cost_float'
@@ -41,6 +43,14 @@ define fastcc float @reduction_cost_float(<4 x float> %rdx) {
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = fadd <4 x float> %bin.rdx, %rdx.shuf7
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r = extractelement <4 x float> %bin.rdx8, i32 0
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r
+;
+; SLM-LABEL: 'reduction_cost_float'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx = fadd <4 x float> %rdx, %rdx.shuf
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <4 x float> %bin.rdx, <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = fadd <4 x float> %bin.rdx, %rdx.shuf7
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r = extractelement <4 x float> %bin.rdx8, i32 0
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r
 ;
   %rdx.shuf = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
   %bin.rdx = fadd <4 x float> %rdx, %rdx.shuf
@@ -81,6 +91,16 @@ define fastcc i32 @reduction_cost_int(<8 x i32> %rdx) {
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx.3 = add <8 x i32> %bin.rdx.2, %rdx.shuf.3
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %r = extractelement <8 x i32> %bin.rdx.3, i32 0
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %r
+;
+; SLM-LABEL: 'reduction_cost_int'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf = shufflevector <8 x i32> %rdx, <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx = add <8 x i32> %rdx, %rdx.shuf
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf.2 = shufflevector <8 x i32> %bin.rdx, <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx.2 = add <8 x i32> %bin.rdx, %rdx.shuf.2
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf.3 = shufflevector <8 x i32> %bin.rdx.2, <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx.3 = add <8 x i32> %bin.rdx.2, %rdx.shuf.3
+; SLM-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %r = extractelement <8 x i32> %bin.rdx.3, i32 0
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %r
 ;
   %rdx.shuf = shufflevector <8 x i32> %rdx, <8 x i32> undef,
   <8 x i32> <i32 4    , i32     5, i32     6, i32     7,
@@ -107,7 +127,7 @@ define fastcc float @pairwise_hadd(<4 x float> %rdx, float %f1) {
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx.1 = fadd <4 x float> %rdx.shuf.1.0, %rdx.shuf.1.1
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %r = extractelement <4 x float> %bin.rdx.1, i32 0
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r = extractelement <4 x float> %bin.rdx.1, i32 0
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r2 = fadd float %r, %f1
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r2
 ;
@@ -118,7 +138,7 @@ define fastcc float @pairwise_hadd(<4 x float> %rdx, float %f1) {
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx.1 = fadd <4 x float> %rdx.shuf.1.0, %rdx.shuf.1.1
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %r = extractelement <4 x float> %bin.rdx.1, i32 0
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r = extractelement <4 x float> %bin.rdx.1, i32 0
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r2 = fadd float %r, %f1
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r2
 ;
@@ -143,6 +163,17 @@ define fastcc float @pairwise_hadd(<4 x float> %rdx, float %f1) {
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r = extractelement <4 x float> %bin.rdx.1, i32 0
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r2 = fadd float %r, %f1
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r2
+;
+; SLM-LABEL: 'pairwise_hadd'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx.0 = fadd <4 x float> %rdx.shuf.0.0, %rdx.shuf.0.1
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx.1 = fadd <4 x float> %rdx.shuf.1.0, %rdx.shuf.1.1
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r = extractelement <4 x float> %bin.rdx.1, i32 0
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r2 = fadd float %r, %f1
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r2
 ;
   %rdx.shuf.0.0 = shufflevector <4 x float> %rdx, <4 x float> undef,
   <4 x i32> <i32 0, i32 2 , i32 undef, i32 undef>
@@ -168,7 +199,7 @@ define fastcc float @pairwise_hadd_assoc(<4 x float> %rdx, float %f1) {
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx.1 = fadd <4 x float> %rdx.shuf.1.0, %rdx.shuf.1.1
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %r = extractelement <4 x float> %bin.rdx.1, i32 0
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r = extractelement <4 x float> %bin.rdx.1, i32 0
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r2 = fadd float %r, %f1
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r2
 ;
@@ -179,7 +210,7 @@ define fastcc float @pairwise_hadd_assoc(<4 x float> %rdx, float %f1) {
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx.1 = fadd <4 x float> %rdx.shuf.1.0, %rdx.shuf.1.1
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %r = extractelement <4 x float> %bin.rdx.1, i32 0
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r = extractelement <4 x float> %bin.rdx.1, i32 0
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r2 = fadd float %r, %f1
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r2
 ;
@@ -204,6 +235,17 @@ define fastcc float @pairwise_hadd_assoc(<4 x float> %rdx, float %f1) {
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r = extractelement <4 x float> %bin.rdx.1, i32 0
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r2 = fadd float %r, %f1
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r2
+;
+; SLM-LABEL: 'pairwise_hadd_assoc'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx.0 = fadd <4 x float> %rdx.shuf.0.1, %rdx.shuf.0.0
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx.1 = fadd <4 x float> %rdx.shuf.1.0, %rdx.shuf.1.1
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r = extractelement <4 x float> %bin.rdx.1, i32 0
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r2 = fadd float %r, %f1
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r2
 ;
   %rdx.shuf.0.0 = shufflevector <4 x float> %rdx, <4 x float> undef,
   <4 x i32> <i32 0, i32 2 , i32 undef, i32 undef>
@@ -228,7 +270,7 @@ define fastcc float @pairwise_hadd_skip_first(<4 x float> %rdx, float %f1) {
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx.0 = fadd <4 x float> %rdx.shuf.0.0, %rdx.shuf.0.1
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx.1 = fadd <4 x float> %bin.rdx.0, %rdx.shuf.1.1
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %r = extractelement <4 x float> %bin.rdx.1, i32 0
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r = extractelement <4 x float> %bin.rdx.1, i32 0
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r2 = fadd float %r, %f1
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r2
 ;
@@ -238,7 +280,7 @@ define fastcc float @pairwise_hadd_skip_first(<4 x float> %rdx, float %f1) {
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx.0 = fadd <4 x float> %rdx.shuf.0.0, %rdx.shuf.0.1
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx.1 = fadd <4 x float> %bin.rdx.0, %rdx.shuf.1.1
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %r = extractelement <4 x float> %bin.rdx.1, i32 0
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r = extractelement <4 x float> %bin.rdx.1, i32 0
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r2 = fadd float %r, %f1
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r2
 ;
@@ -261,6 +303,16 @@ define fastcc float @pairwise_hadd_skip_first(<4 x float> %rdx, float %f1) {
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r = extractelement <4 x float> %bin.rdx.1, i32 0
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r2 = fadd float %r, %f1
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r2
+;
+; SLM-LABEL: 'pairwise_hadd_skip_first'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx.0 = fadd <4 x float> %rdx.shuf.0.0, %rdx.shuf.0.1
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx.1 = fadd <4 x float> %bin.rdx.0, %rdx.shuf.1.1
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r = extractelement <4 x float> %bin.rdx.1, i32 0
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r2 = fadd float %r, %f1
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r2
 ;
   %rdx.shuf.0.0 = shufflevector <4 x float> %rdx, <4 x float> undef,
   <4 x i32> <i32 0, i32 2 , i32 undef, i32 undef>
@@ -280,13 +332,13 @@ define fastcc double @no_pairwise_reduction2double(<2 x double> %rdx, double %f1
 ; SSE2-LABEL: 'no_pairwise_reduction2double'
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <2 x double> %rdx, <2 x double> undef, <2 x i32> <i32 1, i32 undef>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx = fadd <2 x double> %rdx, %rdx.shuf
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r = extractelement <2 x double> %bin.rdx, i32 0
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r = extractelement <2 x double> %bin.rdx, i32 0
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %r
 ;
 ; SSSE3-LABEL: 'no_pairwise_reduction2double'
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <2 x double> %rdx, <2 x double> undef, <2 x i32> <i32 1, i32 undef>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx = fadd <2 x double> %rdx, %rdx.shuf
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r = extractelement <2 x double> %bin.rdx, i32 0
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r = extractelement <2 x double> %bin.rdx, i32 0
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %r
 ;
 ; SSE42-LABEL: 'no_pairwise_reduction2double'
@@ -300,6 +352,12 @@ define fastcc double @no_pairwise_reduction2double(<2 x double> %rdx, double %f1
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx = fadd <2 x double> %rdx, %rdx.shuf
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r = extractelement <2 x double> %bin.rdx, i32 0
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %r
+;
+; SLM-LABEL: 'no_pairwise_reduction2double'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <2 x double> %rdx, <2 x double> undef, <2 x i32> <i32 1, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx = fadd <2 x double> %rdx, %rdx.shuf
+; SLM-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r = extractelement <2 x double> %bin.rdx, i32 0
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %r
 ;
   %rdx.shuf = shufflevector <2 x double> %rdx, <2 x double> undef, <2 x i32> <i32 1, i32 undef>
   %bin.rdx = fadd <2 x double> %rdx, %rdx.shuf
@@ -314,7 +372,7 @@ define fastcc float @no_pairwise_reduction4float(<4 x float> %rdx, float %f1) {
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx = fadd <4 x float> %rdx, %rdx.shuf
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <4 x float> %bin.rdx, <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx8 = fadd <4 x float> %bin.rdx, %rdx.shuf7
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %r = extractelement <4 x float> %bin.rdx8, i32 0
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r = extractelement <4 x float> %bin.rdx8, i32 0
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r
 ;
 ; SSSE3-LABEL: 'no_pairwise_reduction4float'
@@ -322,7 +380,7 @@ define fastcc float @no_pairwise_reduction4float(<4 x float> %rdx, float %f1) {
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx = fadd <4 x float> %rdx, %rdx.shuf
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <4 x float> %bin.rdx, <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx8 = fadd <4 x float> %bin.rdx, %rdx.shuf7
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %r = extractelement <4 x float> %bin.rdx8, i32 0
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r = extractelement <4 x float> %bin.rdx8, i32 0
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r
 ;
 ; SSE42-LABEL: 'no_pairwise_reduction4float'
@@ -340,6 +398,14 @@ define fastcc float @no_pairwise_reduction4float(<4 x float> %rdx, float %f1) {
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = fadd <4 x float> %bin.rdx, %rdx.shuf7
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r = extractelement <4 x float> %bin.rdx8, i32 0
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r
+;
+; SLM-LABEL: 'no_pairwise_reduction4float'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx = fadd <4 x float> %rdx, %rdx.shuf
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <4 x float> %bin.rdx, <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = fadd <4 x float> %bin.rdx, %rdx.shuf7
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r = extractelement <4 x float> %bin.rdx8, i32 0
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r
 ;
   %rdx.shuf = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
   %bin.rdx = fadd <4 x float> %rdx, %rdx.shuf
@@ -356,7 +422,7 @@ define fastcc double @no_pairwise_reduction4double(<4 x double> %rdx, double %f1
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %bin.rdx = fadd <4 x double> %rdx, %rdx.shuf
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %rdx.shuf7 = shufflevector <4 x double> %bin.rdx, <4 x double> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %bin.rdx8 = fadd <4 x double> %bin.rdx, %rdx.shuf7
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %r = extractelement <4 x double> %bin.rdx8, i32 0
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r = extractelement <4 x double> %bin.rdx8, i32 0
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %r
 ;
 ; SSSE3-LABEL: 'no_pairwise_reduction4double'
@@ -364,7 +430,7 @@ define fastcc double @no_pairwise_reduction4double(<4 x double> %rdx, double %f1
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %bin.rdx = fadd <4 x double> %rdx, %rdx.shuf
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %rdx.shuf7 = shufflevector <4 x double> %bin.rdx, <4 x double> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %bin.rdx8 = fadd <4 x double> %bin.rdx, %rdx.shuf7
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %r = extractelement <4 x double> %bin.rdx8, i32 0
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r = extractelement <4 x double> %bin.rdx8, i32 0
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %r
 ;
 ; SSE42-LABEL: 'no_pairwise_reduction4double'
@@ -390,6 +456,14 @@ define fastcc double @no_pairwise_reduction4double(<4 x double> %rdx, double %f1
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = fadd <4 x double> %bin.rdx, %rdx.shuf7
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r = extractelement <4 x double> %bin.rdx8, i32 0
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %r
+;
+; SLM-LABEL: 'no_pairwise_reduction4double'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %rdx.shuf = shufflevector <4 x double> %rdx, <4 x double> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %bin.rdx = fadd <4 x double> %rdx, %rdx.shuf
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %rdx.shuf7 = shufflevector <4 x double> %bin.rdx, <4 x double> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %bin.rdx8 = fadd <4 x double> %bin.rdx, %rdx.shuf7
+; SLM-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %r = extractelement <4 x double> %bin.rdx8, i32 0
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %r
 ;
   %rdx.shuf = shufflevector <4 x double> %rdx, <4 x double> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
   %bin.rdx = fadd <4 x double> %rdx, %rdx.shuf
@@ -450,6 +524,16 @@ define fastcc float @no_pairwise_reduction8float(<8 x float> %rdx, float %f1) {
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = fadd <8 x float> %bin.rdx, %rdx.shuf7
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r = extractelement <8 x float> %bin.rdx8, i32 0
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r
+;
+; SLM-LABEL: 'no_pairwise_reduction8float'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf3 = shufflevector <8 x float> %rdx, <8 x float> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx4 = fadd <8 x float> %rdx, %rdx.shuf3
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf = shufflevector <8 x float> %bin.rdx4, <8 x float> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx = fadd <8 x float> %bin.rdx4, %rdx.shuf
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf7 = shufflevector <8 x float> %bin.rdx, <8 x float> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx8 = fadd <8 x float> %bin.rdx, %rdx.shuf7
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r = extractelement <8 x float> %bin.rdx8, i32 0
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r
 ;
   %rdx.shuf3 = shufflevector <8 x float> %rdx, <8 x float> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7,i32 undef, i32 undef, i32 undef, i32 undef>
   %bin.rdx4 = fadd <8 x float> %rdx, %rdx.shuf3
@@ -463,29 +547,23 @@ define fastcc float @no_pairwise_reduction8float(<8 x float> %rdx, float %f1) {
 }
 
 define fastcc i64 @no_pairwise_reduction2i64(<2 x i64> %rdx, i64 %f1) {
-; SSE2-LABEL: 'no_pairwise_reduction2i64'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <2 x i64> %rdx, <2 x i64> undef, <2 x i32> <i32 1, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx = add <2 x i64> %rdx, %rdx.shuf
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r = extractelement <2 x i64> %bin.rdx, i32 0
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %r
-;
-; SSSE3-LABEL: 'no_pairwise_reduction2i64'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <2 x i64> %rdx, <2 x i64> undef, <2 x i32> <i32 1, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx = add <2 x i64> %rdx, %rdx.shuf
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r = extractelement <2 x i64> %bin.rdx, i32 0
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %r
-;
-; SSE42-LABEL: 'no_pairwise_reduction2i64'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <2 x i64> %rdx, <2 x i64> undef, <2 x i32> <i32 1, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx = add <2 x i64> %rdx, %rdx.shuf
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r = extractelement <2 x i64> %bin.rdx, i32 0
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %r
+; SSE-LABEL: 'no_pairwise_reduction2i64'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <2 x i64> %rdx, <2 x i64> undef, <2 x i32> <i32 1, i32 undef>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx = add <2 x i64> %rdx, %rdx.shuf
+; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r = extractelement <2 x i64> %bin.rdx, i32 0
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %r
 ;
 ; AVX-LABEL: 'no_pairwise_reduction2i64'
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <2 x i64> %rdx, <2 x i64> undef, <2 x i32> <i32 1, i32 undef>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx = add <2 x i64> %rdx, %rdx.shuf
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r = extractelement <2 x i64> %bin.rdx, i32 0
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %r
+;
+; SLM-LABEL: 'no_pairwise_reduction2i64'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <2 x i64> %rdx, <2 x i64> undef, <2 x i32> <i32 1, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %bin.rdx = add <2 x i64> %rdx, %rdx.shuf
+; SLM-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %r = extractelement <2 x i64> %bin.rdx, i32 0
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %r
 ;
   %rdx.shuf = shufflevector <2 x i64> %rdx, <2 x i64> undef, <2 x i32> <i32 1, i32 undef>
   %bin.rdx = add <2 x i64> %rdx, %rdx.shuf
@@ -495,37 +573,21 @@ define fastcc i64 @no_pairwise_reduction2i64(<2 x i64> %rdx, i64 %f1) {
 }
 
 define fastcc i32 @no_pairwise_reduction4i32(<4 x i32> %rdx, i32 %f1) {
-; SSE2-LABEL: 'no_pairwise_reduction4i32'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <4 x i32> %rdx, <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx = add <4 x i32> %rdx, %rdx.shuf
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <4 x i32> %bin.rdx, <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = add <4 x i32> %bin.rdx, %rdx.shuf7
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %r = extractelement <4 x i32> %bin.rdx8, i32 0
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %r
-;
-; SSSE3-LABEL: 'no_pairwise_reduction4i32'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <4 x i32> %rdx, <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx = add <4 x i32> %rdx, %rdx.shuf
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <4 x i32> %bin.rdx, <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = add <4 x i32> %bin.rdx, %rdx.shuf7
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %r = extractelement <4 x i32> %bin.rdx8, i32 0
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %r
-;
-; SSE42-LABEL: 'no_pairwise_reduction4i32'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <4 x i32> %rdx, <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx = add <4 x i32> %rdx, %rdx.shuf
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <4 x i32> %bin.rdx, <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = add <4 x i32> %bin.rdx, %rdx.shuf7
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r = extractelement <4 x i32> %bin.rdx8, i32 0
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %r
-;
-; AVX-LABEL: 'no_pairwise_reduction4i32'
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <4 x i32> %rdx, <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx = add <4 x i32> %rdx, %rdx.shuf
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <4 x i32> %bin.rdx, <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = add <4 x i32> %bin.rdx, %rdx.shuf7
-; AVX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r = extractelement <4 x i32> %bin.rdx8, i32 0
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %r
+; CHECK-LABEL: 'no_pairwise_reduction4i32'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <4 x i32> %rdx, <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx = add <4 x i32> %rdx, %rdx.shuf
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <4 x i32> %bin.rdx, <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = add <4 x i32> %bin.rdx, %rdx.shuf7
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r = extractelement <4 x i32> %bin.rdx8, i32 0
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %r
+;
+; SLM-LABEL: 'no_pairwise_reduction4i32'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <4 x i32> %rdx, <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx = add <4 x i32> %rdx, %rdx.shuf
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <4 x i32> %bin.rdx, <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = add <4 x i32> %bin.rdx, %rdx.shuf7
+; SLM-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r = extractelement <4 x i32> %bin.rdx8, i32 0
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %r
 ;
   %rdx.shuf = shufflevector <4 x i32> %rdx, <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
   %bin.rdx = add <4 x i32> %rdx, %rdx.shuf
@@ -560,6 +622,14 @@ define fastcc i64 @no_pairwise_reduction4i64(<4 x i64> %rdx, i64 %f1) {
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = add <4 x i64> %bin.rdx, %rdx.shuf7
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r = extractelement <4 x i64> %bin.rdx8, i32 0
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %r
+;
+; SLM-LABEL: 'no_pairwise_reduction4i64'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %rdx.shuf = shufflevector <4 x i64> %rdx, <4 x i64> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %bin.rdx = add <4 x i64> %rdx, %rdx.shuf
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %rdx.shuf7 = shufflevector <4 x i64> %bin.rdx, <4 x i64> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %bin.rdx8 = add <4 x i64> %bin.rdx, %rdx.shuf7
+; SLM-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r = extractelement <4 x i64> %bin.rdx8, i32 0
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %r
 ;
   %rdx.shuf = shufflevector <4 x i64> %rdx, <4 x i64> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
   %bin.rdx = add <4 x i64> %rdx, %rdx.shuf
@@ -578,7 +648,7 @@ define fastcc i16 @no_pairwise_reduction8i16(<8 x i16> %rdx, i16 %f1) {
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx = add <8 x i16> %bin.rdx4, %rdx.shuf
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %rdx.shuf7 = shufflevector <8 x i16> %bin.rdx, <8 x i16> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = add <8 x i16> %bin.rdx, %rdx.shuf7
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %r = extractelement <8 x i16> %bin.rdx8, i32 0
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r = extractelement <8 x i16> %bin.rdx8, i32 0
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i16 %r
 ;
 ; SSSE3-LABEL: 'no_pairwise_reduction8i16'
@@ -588,7 +658,7 @@ define fastcc i16 @no_pairwise_reduction8i16(<8 x i16> %rdx, i16 %f1) {
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx = add <8 x i16> %bin.rdx4, %rdx.shuf
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <8 x i16> %bin.rdx, <8 x i16> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = add <8 x i16> %bin.rdx, %rdx.shuf7
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %r = extractelement <8 x i16> %bin.rdx8, i32 0
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r = extractelement <8 x i16> %bin.rdx8, i32 0
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i16 %r
 ;
 ; SSE42-LABEL: 'no_pairwise_reduction8i16'
@@ -610,6 +680,16 @@ define fastcc i16 @no_pairwise_reduction8i16(<8 x i16> %rdx, i16 %f1) {
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = add <8 x i16> %bin.rdx, %rdx.shuf7
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r = extractelement <8 x i16> %bin.rdx8, i32 0
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i16 %r
+;
+; SLM-LABEL: 'no_pairwise_reduction8i16'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf3 = shufflevector <8 x i16> %rdx, <8 x i16> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx4 = add <8 x i16> %rdx, %rdx.shuf3
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <8 x i16> %bin.rdx4, <8 x i16> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx = add <8 x i16> %bin.rdx4, %rdx.shuf
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <8 x i16> %bin.rdx, <8 x i16> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = add <8 x i16> %bin.rdx, %rdx.shuf7
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r = extractelement <8 x i16> %bin.rdx8, i32 0
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i16 %r
 ;
   %rdx.shuf3 = shufflevector <8 x i16> %rdx, <8 x i16> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7,i32 undef, i32 undef, i32 undef, i32 undef>
   %bin.rdx4 = add <8 x i16> %rdx, %rdx.shuf3
@@ -652,6 +732,16 @@ define fastcc i32 @no_pairwise_reduction8i32(<8 x i32> %rdx, i32 %f1) {
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = add <8 x i32> %bin.rdx, %rdx.shuf7
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %r = extractelement <8 x i32> %bin.rdx8, i32 0
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %r
+;
+; SLM-LABEL: 'no_pairwise_reduction8i32'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf3 = shufflevector <8 x i32> %rdx, <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx4 = add <8 x i32> %rdx, %rdx.shuf3
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf = shufflevector <8 x i32> %bin.rdx4, <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx = add <8 x i32> %bin.rdx4, %rdx.shuf
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf7 = shufflevector <8 x i32> %bin.rdx, <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx8 = add <8 x i32> %bin.rdx, %rdx.shuf7
+; SLM-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %r = extractelement <8 x i32> %bin.rdx8, i32 0
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %r
 ;
   %rdx.shuf3 = shufflevector <8 x i32> %rdx, <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7,i32 undef, i32 undef, i32 undef, i32 undef>
   %bin.rdx4 = add <8 x i32> %rdx, %rdx.shuf3
@@ -669,14 +759,14 @@ define fastcc double @pairwise_reduction2double(<2 x double> %rdx, double %f1) {
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <2 x double> %rdx, <2 x double> undef, <2 x i32> <i32 0, i32 undef>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <2 x double> %rdx, <2 x double> undef, <2 x i32> <i32 1, i32 undef>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx8 = fadd <2 x double> %rdx.shuf.1.0, %rdx.shuf.1.1
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r = extractelement <2 x double> %bin.rdx8, i32 0
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r = extractelement <2 x double> %bin.rdx8, i32 0
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %r
 ;
 ; SSSE3-LABEL: 'pairwise_reduction2double'
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <2 x double> %rdx, <2 x double> undef, <2 x i32> <i32 0, i32 undef>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <2 x double> %rdx, <2 x double> undef, <2 x i32> <i32 1, i32 undef>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx8 = fadd <2 x double> %rdx.shuf.1.0, %rdx.shuf.1.1
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r = extractelement <2 x double> %bin.rdx8, i32 0
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r = extractelement <2 x double> %bin.rdx8, i32 0
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %r
 ;
 ; SSE42-LABEL: 'pairwise_reduction2double'
@@ -692,6 +782,13 @@ define fastcc double @pairwise_reduction2double(<2 x double> %rdx, double %f1) {
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = fadd <2 x double> %rdx.shuf.1.0, %rdx.shuf.1.1
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r = extractelement <2 x double> %bin.rdx8, i32 0
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %r
+;
+; SLM-LABEL: 'pairwise_reduction2double'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <2 x double> %rdx, <2 x double> undef, <2 x i32> <i32 0, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <2 x double> %rdx, <2 x double> undef, <2 x i32> <i32 1, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx8 = fadd <2 x double> %rdx.shuf.1.0, %rdx.shuf.1.1
+; SLM-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r = extractelement <2 x double> %bin.rdx8, i32 0
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %r
 ;
   %rdx.shuf.1.0 = shufflevector <2 x double> %rdx, <2 x double> undef, <2 x i32> <i32 0, i32 undef>
   %rdx.shuf.1.1 = shufflevector <2 x double> %rdx, <2 x double> undef, <2 x i32> <i32 1, i32 undef>
@@ -709,7 +806,7 @@ define fastcc float @pairwise_reduction4float(<4 x float> %rdx, float %f1) {
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x float> %bin.rdx, <4 x float> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x float> %bin.rdx, <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx8 = fadd <4 x float> %rdx.shuf.1.0, %rdx.shuf.1.1
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %r = extractelement <4 x float> %bin.rdx8, i32 0
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r = extractelement <4 x float> %bin.rdx8, i32 0
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r
 ;
 ; SSSE3-LABEL: 'pairwise_reduction4float'
@@ -719,7 +816,7 @@ define fastcc float @pairwise_reduction4float(<4 x float> %rdx, float %f1) {
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x float> %bin.rdx, <4 x float> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x float> %bin.rdx, <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx8 = fadd <4 x float> %rdx.shuf.1.0, %rdx.shuf.1.1
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %r = extractelement <4 x float> %bin.rdx8, i32 0
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r = extractelement <4 x float> %bin.rdx8, i32 0
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r
 ;
 ; SSE42-LABEL: 'pairwise_reduction4float'
@@ -741,6 +838,16 @@ define fastcc float @pairwise_reduction4float(<4 x float> %rdx, float %f1) {
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = fadd <4 x float> %rdx.shuf.1.0, %rdx.shuf.1.1
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r = extractelement <4 x float> %bin.rdx8, i32 0
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r
+;
+; SLM-LABEL: 'pairwise_reduction4float'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx = fadd <4 x float> %rdx.shuf.0.0, %rdx.shuf.0.1
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x float> %bin.rdx, <4 x float> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x float> %bin.rdx, <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = fadd <4 x float> %rdx.shuf.1.0, %rdx.shuf.1.1
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r = extractelement <4 x float> %bin.rdx8, i32 0
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r
 ;
   %rdx.shuf.0.0 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
   %rdx.shuf.0.1 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
@@ -761,7 +868,7 @@ define fastcc double @pairwise_reduction4double(<4 x double> %rdx, double %f1) {
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x double> %bin.rdx, <4 x double> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %rdx.shuf.1.1 = shufflevector <4 x double> %bin.rdx, <4 x double> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %bin.rdx8 = fadd <4 x double> %rdx.shuf.1.0, %rdx.shuf.1.1
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %r = extractelement <4 x double> %bin.rdx8, i32 0
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r = extractelement <4 x double> %bin.rdx8, i32 0
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %r
 ;
 ; SSSE3-LABEL: 'pairwise_reduction4double'
@@ -771,7 +878,7 @@ define fastcc double @pairwise_reduction4double(<4 x double> %rdx, double %f1) {
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x double> %bin.rdx, <4 x double> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %rdx.shuf.1.1 = shufflevector <4 x double> %bin.rdx, <4 x double> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %bin.rdx8 = fadd <4 x double> %rdx.shuf.1.0, %rdx.shuf.1.1
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %r = extractelement <4 x double> %bin.rdx8, i32 0
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r = extractelement <4 x double> %bin.rdx8, i32 0
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %r
 ;
 ; SSE42-LABEL: 'pairwise_reduction4double'
@@ -803,6 +910,16 @@ define fastcc double @pairwise_reduction4double(<4 x double> %rdx, double %f1) {
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = fadd <4 x double> %rdx.shuf.1.0, %rdx.shuf.1.1
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %r = extractelement <4 x double> %bin.rdx8, i32 0
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %r
+;
+; SLM-LABEL: 'pairwise_reduction4double'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %rdx.shuf.0.0 = shufflevector <4 x double> %rdx, <4 x double> undef, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %rdx.shuf.0.1 = shufflevector <4 x double> %rdx, <4 x double> undef, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %bin.rdx = fadd <4 x double> %rdx.shuf.0.0, %rdx.shuf.0.1
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x double> %bin.rdx, <4 x double> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %rdx.shuf.1.1 = shufflevector <4 x double> %bin.rdx, <4 x double> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %bin.rdx8 = fadd <4 x double> %rdx.shuf.1.0, %rdx.shuf.1.1
+; SLM-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %r = extractelement <4 x double> %bin.rdx8, i32 0
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %r
 ;
   %rdx.shuf.0.0 = shufflevector <4 x double> %rdx, <4 x double> undef, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
   %rdx.shuf.0.1 = shufflevector <4 x double> %rdx, <4 x double> undef, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
@@ -826,7 +943,7 @@ define fastcc float @pairwise_reduction8float(<8 x float> %rdx, float %f1) {
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.2.0 = shufflevector <8 x float> %bin.rdx8, <8 x float> undef, <8 x i32> <i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf.2.1 = shufflevector <8 x float> %bin.rdx8, <8 x float> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %bin.rdx9 = fadd <8 x float> %rdx.shuf.2.0, %rdx.shuf.2.1
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %r = extractelement <8 x float> %bin.rdx9, i32 0
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r = extractelement <8 x float> %bin.rdx9, i32 0
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r
 ;
 ; SSSE3-LABEL: 'pairwise_reduction8float'
@@ -839,7 +956,7 @@ define fastcc float @pairwise_reduction8float(<8 x float> %rdx, float %f1) {
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.2.0 = shufflevector <8 x float> %bin.rdx8, <8 x float> undef, <8 x i32> <i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf.2.1 = shufflevector <8 x float> %bin.rdx8, <8 x float> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %bin.rdx9 = fadd <8 x float> %rdx.shuf.2.0, %rdx.shuf.2.1
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %r = extractelement <8 x float> %bin.rdx9, i32 0
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r = extractelement <8 x float> %bin.rdx9, i32 0
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r
 ;
 ; SSE42-LABEL: 'pairwise_reduction8float'
@@ -880,6 +997,19 @@ define fastcc float @pairwise_reduction8float(<8 x float> %rdx, float %f1) {
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx9 = fadd <8 x float> %rdx.shuf.2.0, %rdx.shuf.2.1
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %r = extractelement <8 x float> %bin.rdx9, i32 0
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r
+;
+; SLM-LABEL: 'pairwise_reduction8float'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf.0.0 = shufflevector <8 x float> %rdx, <8 x float> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 undef, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf.0.1 = shufflevector <8 x float> %rdx, <8 x float> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx = fadd <8 x float> %rdx.shuf.0.0, %rdx.shuf.0.1
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf.1.0 = shufflevector <8 x float> %bin.rdx, <8 x float> undef, <8 x i32> <i32 0, i32 2, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf.1.1 = shufflevector <8 x float> %bin.rdx, <8 x float> undef, <8 x i32> <i32 1, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx8 = fadd <8 x float> %rdx.shuf.1.0, %rdx.shuf.1.1
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.2.0 = shufflevector <8 x float> %bin.rdx8, <8 x float> undef, <8 x i32> <i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf.2.1 = shufflevector <8 x float> %bin.rdx8, <8 x float> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx9 = fadd <8 x float> %rdx.shuf.2.0, %rdx.shuf.2.1
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r = extractelement <8 x float> %bin.rdx9, i32 0
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r
 ;
   %rdx.shuf.0.0 = shufflevector <8 x float> %rdx, <8 x float> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6,i32 undef, i32 undef, i32 undef, i32 undef>
   %rdx.shuf.0.1 = shufflevector <8 x float> %rdx, <8 x float> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7,i32 undef, i32 undef, i32 undef, i32 undef>
@@ -896,26 +1026,12 @@ define fastcc float @pairwise_reduction8float(<8 x float> %rdx, float %f1) {
 }
 
 define fastcc i64 @pairwise_reduction2i64(<2 x i64> %rdx, i64 %f1) {
-; SSE2-LABEL: 'pairwise_reduction2i64'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <2 x i64> %rdx, <2 x i64> undef, <2 x i32> <i32 0, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <2 x i64> %rdx, <2 x i64> undef, <2 x i32> <i32 1, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = add <2 x i64> %rdx.shuf.1.0, %rdx.shuf.1.1
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r = extractelement <2 x i64> %bin.rdx8, i32 0
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %r
-;
-; SSSE3-LABEL: 'pairwise_reduction2i64'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <2 x i64> %rdx, <2 x i64> undef, <2 x i32> <i32 0, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <2 x i64> %rdx, <2 x i64> undef, <2 x i32> <i32 1, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = add <2 x i64> %rdx.shuf.1.0, %rdx.shuf.1.1
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r = extractelement <2 x i64> %bin.rdx8, i32 0
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %r
-;
-; SSE42-LABEL: 'pairwise_reduction2i64'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <2 x i64> %rdx, <2 x i64> undef, <2 x i32> <i32 0, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <2 x i64> %rdx, <2 x i64> undef, <2 x i32> <i32 1, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = add <2 x i64> %rdx.shuf.1.0, %rdx.shuf.1.1
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r = extractelement <2 x i64> %bin.rdx8, i32 0
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %r
+; SSE-LABEL: 'pairwise_reduction2i64'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <2 x i64> %rdx, <2 x i64> undef, <2 x i32> <i32 0, i32 undef>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <2 x i64> %rdx, <2 x i64> undef, <2 x i32> <i32 1, i32 undef>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = add <2 x i64> %rdx.shuf.1.0, %rdx.shuf.1.1
+; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r = extractelement <2 x i64> %bin.rdx8, i32 0
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %r
 ;
 ; AVX-LABEL: 'pairwise_reduction2i64'
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <2 x i64> %rdx, <2 x i64> undef, <2 x i32> <i32 0, i32 undef>
@@ -923,6 +1039,13 @@ define fastcc i64 @pairwise_reduction2i64(<2 x i64> %rdx, i64 %f1) {
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = add <2 x i64> %rdx.shuf.1.0, %rdx.shuf.1.1
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r = extractelement <2 x i64> %bin.rdx8, i32 0
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %r
+;
+; SLM-LABEL: 'pairwise_reduction2i64'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <2 x i64> %rdx, <2 x i64> undef, <2 x i32> <i32 0, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <2 x i64> %rdx, <2 x i64> undef, <2 x i32> <i32 1, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %bin.rdx8 = add <2 x i64> %rdx.shuf.1.0, %rdx.shuf.1.1
+; SLM-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %r = extractelement <2 x i64> %bin.rdx8, i32 0
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %r
 ;
   %rdx.shuf.1.0 = shufflevector <2 x i64> %rdx, <2 x i64> undef, <2 x i32> <i32 0, i32 undef>
   %rdx.shuf.1.1 = shufflevector <2 x i64> %rdx, <2 x i64> undef, <2 x i32> <i32 1, i32 undef>
@@ -933,45 +1056,25 @@ define fastcc i64 @pairwise_reduction2i64(<2 x i64> %rdx, i64 %f1) {
 }
 
 define fastcc i32 @pairwise_reduction4i32(<4 x i32> %rdx, i32 %f1) {
-; SSE2-LABEL: 'pairwise_reduction4i32'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x i32> %rdx, <4 x i32> undef, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x i32> %rdx, <4 x i32> undef, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx = add <4 x i32> %rdx.shuf.0.0, %rdx.shuf.0.1
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x i32> %bin.rdx, <4 x i32> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x i32> %bin.rdx, <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = add <4 x i32> %rdx.shuf.1.0, %rdx.shuf.1.1
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %r = extractelement <4 x i32> %bin.rdx8, i32 0
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %r
-;
-; SSSE3-LABEL: 'pairwise_reduction4i32'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x i32> %rdx, <4 x i32> undef, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x i32> %rdx, <4 x i32> undef, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx = add <4 x i32> %rdx.shuf.0.0, %rdx.shuf.0.1
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x i32> %bin.rdx, <4 x i32> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x i32> %bin.rdx, <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = add <4 x i32> %rdx.shuf.1.0, %rdx.shuf.1.1
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %r = extractelement <4 x i32> %bin.rdx8, i32 0
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %r
-;
-; SSE42-LABEL: 'pairwise_reduction4i32'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x i32> %rdx, <4 x i32> undef, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x i32> %rdx, <4 x i32> undef, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx = add <4 x i32> %rdx.shuf.0.0, %rdx.shuf.0.1
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x i32> %bin.rdx, <4 x i32> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x i32> %bin.rdx, <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = add <4 x i32> %rdx.shuf.1.0, %rdx.shuf.1.1
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r = extractelement <4 x i32> %bin.rdx8, i32 0
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %r
-;
-; AVX-LABEL: 'pairwise_reduction4i32'
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x i32> %rdx, <4 x i32> undef, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x i32> %rdx, <4 x i32> undef, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx = add <4 x i32> %rdx.shuf.0.0, %rdx.shuf.0.1
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x i32> %bin.rdx, <4 x i32> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x i32> %bin.rdx, <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = add <4 x i32> %rdx.shuf.1.0, %rdx.shuf.1.1
-; AVX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r = extractelement <4 x i32> %bin.rdx8, i32 0
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %r
+; CHECK-LABEL: 'pairwise_reduction4i32'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x i32> %rdx, <4 x i32> undef, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x i32> %rdx, <4 x i32> undef, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx = add <4 x i32> %rdx.shuf.0.0, %rdx.shuf.0.1
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x i32> %bin.rdx, <4 x i32> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x i32> %bin.rdx, <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = add <4 x i32> %rdx.shuf.1.0, %rdx.shuf.1.1
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r = extractelement <4 x i32> %bin.rdx8, i32 0
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %r
+;
+; SLM-LABEL: 'pairwise_reduction4i32'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x i32> %rdx, <4 x i32> undef, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x i32> %rdx, <4 x i32> undef, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx = add <4 x i32> %rdx.shuf.0.0, %rdx.shuf.0.1
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x i32> %bin.rdx, <4 x i32> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x i32> %bin.rdx, <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = add <4 x i32> %rdx.shuf.1.0, %rdx.shuf.1.1
+; SLM-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r = extractelement <4 x i32> %bin.rdx8, i32 0
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %r
 ;
   %rdx.shuf.0.0 = shufflevector <4 x i32> %rdx, <4 x i32> undef, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
   %rdx.shuf.0.1 = shufflevector <4 x i32> %rdx, <4 x i32> undef, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
@@ -1014,6 +1117,16 @@ define fastcc i64 @pairwise_reduction4i64(<4 x i64> %rdx, i64 %f1) {
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = add <4 x i64> %rdx.shuf.1.0, %rdx.shuf.1.1
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %r = extractelement <4 x i64> %bin.rdx8, i32 0
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %r
+;
+; SLM-LABEL: 'pairwise_reduction4i64'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %rdx.shuf.0.0 = shufflevector <4 x i64> %rdx, <4 x i64> undef, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %rdx.shuf.0.1 = shufflevector <4 x i64> %rdx, <4 x i64> undef, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %bin.rdx = add <4 x i64> %rdx.shuf.0.0, %rdx.shuf.0.1
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x i64> %bin.rdx, <4 x i64> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %rdx.shuf.1.1 = shufflevector <4 x i64> %bin.rdx, <4 x i64> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %bin.rdx8 = add <4 x i64> %rdx.shuf.1.0, %rdx.shuf.1.1
+; SLM-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r = extractelement <4 x i64> %bin.rdx8, i32 0
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %r
 ;
   %rdx.shuf.0.0 = shufflevector <4 x i64> %rdx, <4 x i64> undef, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
   %rdx.shuf.0.1 = shufflevector <4 x i64> %rdx, <4 x i64> undef, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
@@ -1037,7 +1150,7 @@ define fastcc i16 @pairwise_reduction8i16(<8 x i16> %rdx, i16 %f1) {
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.2.0 = shufflevector <8 x i16> %bin.rdx8, <8 x i16> undef, <8 x i32> <i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %rdx.shuf.2.1 = shufflevector <8 x i16> %bin.rdx8, <8 x i16> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx9 = add <8 x i16> %rdx.shuf.2.0, %rdx.shuf.2.1
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %r = extractelement <8 x i16> %bin.rdx9, i32 0
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %r = extractelement <8 x i16> %bin.rdx9, i32 0
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i16 %r
 ;
 ; SSSE3-LABEL: 'pairwise_reduction8i16'
@@ -1050,7 +1163,7 @@ define fastcc i16 @pairwise_reduction8i16(<8 x i16> %rdx, i16 %f1) {
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.2.0 = shufflevector <8 x i16> %bin.rdx8, <8 x i16> undef, <8 x i32> <i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.2.1 = shufflevector <8 x i16> %bin.rdx8, <8 x i16> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx9 = add <8 x i16> %rdx.shuf.2.0, %rdx.shuf.2.1
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %r = extractelement <8 x i16> %bin.rdx9, i32 0
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %r = extractelement <8 x i16> %bin.rdx9, i32 0
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i16 %r
 ;
 ; SSE42-LABEL: 'pairwise_reduction8i16'
@@ -1078,6 +1191,19 @@ define fastcc i16 @pairwise_reduction8i16(<8 x i16> %rdx, i16 %f1) {
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx9 = add <8 x i16> %rdx.shuf.2.0, %rdx.shuf.2.1
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %r = extractelement <8 x i16> %bin.rdx9, i32 0
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i16 %r
+;
+; SLM-LABEL: 'pairwise_reduction8i16'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <8 x i16> %rdx, <8 x i16> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 undef, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <8 x i16> %rdx, <8 x i16> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx = add <8 x i16> %rdx.shuf.0.0, %rdx.shuf.0.1
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.0 = shufflevector <8 x i16> %bin.rdx, <8 x i16> undef, <8 x i32> <i32 0, i32 2, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <8 x i16> %bin.rdx, <8 x i16> undef, <8 x i32> <i32 1, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = add <8 x i16> %rdx.shuf.1.0, %rdx.shuf.1.1
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.2.0 = shufflevector <8 x i16> %bin.rdx8, <8 x i16> undef, <8 x i32> <i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.2.1 = shufflevector <8 x i16> %bin.rdx8, <8 x i16> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx9 = add <8 x i16> %rdx.shuf.2.0, %rdx.shuf.2.1
+; SLM-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %r = extractelement <8 x i16> %bin.rdx9, i32 0
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i16 %r
 ;
   %rdx.shuf.0.0 = shufflevector <8 x i16> %rdx, <8 x i16> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6,i32 undef, i32 undef, i32 undef, i32 undef>
   %rdx.shuf.0.1 = shufflevector <8 x i16> %rdx, <8 x i16> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7,i32 undef, i32 undef, i32 undef, i32 undef>
@@ -1094,44 +1220,18 @@ define fastcc i16 @pairwise_reduction8i16(<8 x i16> %rdx, i16 %f1) {
 }
 
 define fastcc i32 @pairwise_reduction8i32(<8 x i32> %rdx, i32 %f1) {
-; SSE2-LABEL: 'pairwise_reduction8i32'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf.0.0 = shufflevector <8 x i32> %rdx, <8 x i32> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf.0.1 = shufflevector <8 x i32> %rdx, <8 x i32> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx = add <8 x i32> %rdx.shuf.0.0, %rdx.shuf.0.1
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf.1.0 = shufflevector <8 x i32> %bin.rdx, <8 x i32> undef, <8 x i32> <i32 0, i32 2, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf.1.1 = shufflevector <8 x i32> %bin.rdx, <8 x i32> undef, <8 x i32> <i32 1, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx8 = add <8 x i32> %rdx.shuf.1.0, %rdx.shuf.1.1
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.2.0 = shufflevector <8 x i32> %bin.rdx8, <8 x i32> undef, <8 x i32> <i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf.2.1 = shufflevector <8 x i32> %bin.rdx8, <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx9 = add <8 x i32> %rdx.shuf.2.0, %rdx.shuf.2.1
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %r = extractelement <8 x i32> %bin.rdx9, i32 0
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %r
-;
-; SSSE3-LABEL: 'pairwise_reduction8i32'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf.0.0 = shufflevector <8 x i32> %rdx, <8 x i32> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf.0.1 = shufflevector <8 x i32> %rdx, <8 x i32> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx = add <8 x i32> %rdx.shuf.0.0, %rdx.shuf.0.1
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf.1.0 = shufflevector <8 x i32> %bin.rdx, <8 x i32> undef, <8 x i32> <i32 0, i32 2, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf.1.1 = shufflevector <8 x i32> %bin.rdx, <8 x i32> undef, <8 x i32> <i32 1, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx8 = add <8 x i32> %rdx.shuf.1.0, %rdx.shuf.1.1
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.2.0 = shufflevector <8 x i32> %bin.rdx8, <8 x i32> undef, <8 x i32> <i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf.2.1 = shufflevector <8 x i32> %bin.rdx8, <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx9 = add <8 x i32> %rdx.shuf.2.0, %rdx.shuf.2.1
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %r = extractelement <8 x i32> %bin.rdx9, i32 0
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %r
-;
-; SSE42-LABEL: 'pairwise_reduction8i32'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf.0.0 = shufflevector <8 x i32> %rdx, <8 x i32> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf.0.1 = shufflevector <8 x i32> %rdx, <8 x i32> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx = add <8 x i32> %rdx.shuf.0.0, %rdx.shuf.0.1
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf.1.0 = shufflevector <8 x i32> %bin.rdx, <8 x i32> undef, <8 x i32> <i32 0, i32 2, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf.1.1 = shufflevector <8 x i32> %bin.rdx, <8 x i32> undef, <8 x i32> <i32 1, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx8 = add <8 x i32> %rdx.shuf.1.0, %rdx.shuf.1.1
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.2.0 = shufflevector <8 x i32> %bin.rdx8, <8 x i32> undef, <8 x i32> <i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf.2.1 = shufflevector <8 x i32> %bin.rdx8, <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx9 = add <8 x i32> %rdx.shuf.2.0, %rdx.shuf.2.1
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %r = extractelement <8 x i32> %bin.rdx9, i32 0
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %r
+; SSE-LABEL: 'pairwise_reduction8i32'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf.0.0 = shufflevector <8 x i32> %rdx, <8 x i32> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf.0.1 = shufflevector <8 x i32> %rdx, <8 x i32> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx = add <8 x i32> %rdx.shuf.0.0, %rdx.shuf.0.1
+; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf.1.0 = shufflevector <8 x i32> %bin.rdx, <8 x i32> undef, <8 x i32> <i32 0, i32 2, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf.1.1 = shufflevector <8 x i32> %bin.rdx, <8 x i32> undef, <8 x i32> <i32 1, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx8 = add <8 x i32> %rdx.shuf.1.0, %rdx.shuf.1.1
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.2.0 = shufflevector <8 x i32> %bin.rdx8, <8 x i32> undef, <8 x i32> <i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf.2.1 = shufflevector <8 x i32> %bin.rdx8, <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx9 = add <8 x i32> %rdx.shuf.2.0, %rdx.shuf.2.1
+; SSE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %r = extractelement <8 x i32> %bin.rdx9, i32 0
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %r
 ;
 ; AVX1-LABEL: 'pairwise_reduction8i32'
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf.0.0 = shufflevector <8 x i32> %rdx, <8 x i32> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -1158,6 +1258,19 @@ define fastcc i32 @pairwise_reduction8i32(<8 x i32> %rdx, i32 %f1) {
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx9 = add <8 x i32> %rdx.shuf.2.0, %rdx.shuf.2.1
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %r = extractelement <8 x i32> %bin.rdx9, i32 0
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %r
+;
+; SLM-LABEL: 'pairwise_reduction8i32'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf.0.0 = shufflevector <8 x i32> %rdx, <8 x i32> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 undef, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf.0.1 = shufflevector <8 x i32> %rdx, <8 x i32> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx = add <8 x i32> %rdx.shuf.0.0, %rdx.shuf.0.1
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf.1.0 = shufflevector <8 x i32> %bin.rdx, <8 x i32> undef, <8 x i32> <i32 0, i32 2, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf.1.1 = shufflevector <8 x i32> %bin.rdx, <8 x i32> undef, <8 x i32> <i32 1, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx8 = add <8 x i32> %rdx.shuf.1.0, %rdx.shuf.1.1
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.2.0 = shufflevector <8 x i32> %bin.rdx8, <8 x i32> undef, <8 x i32> <i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf.2.1 = shufflevector <8 x i32> %bin.rdx8, <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx9 = add <8 x i32> %rdx.shuf.2.0, %rdx.shuf.2.1
+; SLM-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %r = extractelement <8 x i32> %bin.rdx9, i32 0
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %r
 ;
   %rdx.shuf.0.0 = shufflevector <8 x i32> %rdx, <8 x i32> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6,i32 undef, i32 undef, i32 undef, i32 undef>
   %rdx.shuf.0.1 = shufflevector <8 x i32> %rdx, <8 x i32> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7,i32 undef, i32 undef, i32 undef, i32 undef>
diff --git a/llvm/test/Analysis/CostModel/X86/shuffle-extract_subvector.ll b/llvm/test/Analysis/CostModel/X86/shuffle-extract_subvector.ll
index f74b30cf1..a794a9a3d 100644
--- a/llvm/test/Analysis/CostModel/X86/shuffle-extract_subvector.ll
+++ b/llvm/test/Analysis/CostModel/X86/shuffle-extract_subvector.ll
@@ -5,11 +5,11 @@
 ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mattr=+avx | FileCheck %s -check-prefixes=CHECK,AVX,AVX1
 ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mattr=+avx2 | FileCheck %s -check-prefixes=CHECK,AVX,AVX2
 ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mattr=+avx512f | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F
-; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=CHECK,AVX512
-; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mattr=+avx512f,+avx512bw,+avx512vbmi | FileCheck %s --check-prefixes=CHECK,AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512BW
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mattr=+avx512f,+avx512bw,+avx512vbmi | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512BW
 ;
-; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mcpu=slm | FileCheck %s --check-prefixes=CHECK,SSE,SSE42
-; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mcpu=goldmont | FileCheck %s --check-prefixes=CHECK,SSE,SSE42
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mcpu=slm | FileCheck %s --check-prefixes=CHECK,SSE,SSE42,SLM
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mcpu=goldmont | FileCheck %s --check-prefixes=CHECK,SSE,SSE42,GLM
 ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mcpu=btver2 | FileCheck %s --check-prefixes=BTVER2
 
 ;
@@ -27,6 +27,7 @@ define void @test_vXf64(<4 x double> %src256, <8 x double> %src512) {
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_0123 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_2345 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:  Cost Model: Unknown cost for instruction: %V512_567u = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 5, i32 6, i32 7, i32 undef>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX-LABEL: 'test_vXf64'
@@ -39,6 +40,7 @@ define void @test_vXf64(<4 x double> %src256, <8 x double> %src512) {
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_0123 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V512_2345 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; AVX-NEXT:  Cost Model: Unknown cost for instruction: %V512_567u = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 5, i32 6, i32 7, i32 undef>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXf64'
@@ -51,6 +53,7 @@ define void @test_vXf64(<4 x double> %src256, <8 x double> %src512) {
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_0123 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V512_2345 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_4567 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; AVX512-NEXT:  Cost Model: Unknown cost for instruction: %V512_567u = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 5, i32 6, i32 7, i32 undef>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; BTVER2-LABEL: 'test_vXf64'
@@ -63,6 +66,7 @@ define void @test_vXf64(<4 x double> %src256, <8 x double> %src512) {
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_0123 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V512_2345 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; BTVER2-NEXT:  Cost Model: Unknown cost for instruction: %V512_567u = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 5, i32 6, i32 7, i32 undef>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %V256_01 = shufflevector <4 x double> %src256, <4 x double> undef, <2 x i32> <i32 0, i32 1>
@@ -74,11 +78,12 @@ define void @test_vXf64(<4 x double> %src256, <8 x double> %src512) {
   %V512_0123 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %V512_2345 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
   %V512_4567 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %V512_567u = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 5, i32 6, i32 7, i32 undef>
   ret void
 }
 
-define void @test_vXfi64(<4 x i64> %src256, <8 x i64> %src512) {
-; SSE-LABEL: 'test_vXfi64'
+define void @test_vXi64(<4 x i64> %src256, <8 x i64> %src512) {
+; SSE-LABEL: 'test_vXi64'
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_01 = shufflevector <4 x i64> %src256, <4 x i64> undef, <2 x i32> <i32 0, i32 1>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_23 = shufflevector <4 x i64> %src256, <4 x i64> undef, <2 x i32> <i32 2, i32 3>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_01 = shufflevector <8 x i64> %src512, <8 x i64> undef, <2 x i32> <i32 0, i32 1>
@@ -90,7 +95,7 @@ define void @test_vXfi64(<4 x i64> %src256, <8 x i64> %src512) {
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <8 x i64> %src512, <8 x i64> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
-; AVX-LABEL: 'test_vXfi64'
+; AVX-LABEL: 'test_vXi64'
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_01 = shufflevector <4 x i64> %src256, <4 x i64> undef, <2 x i32> <i32 0, i32 1>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_23 = shufflevector <4 x i64> %src256, <4 x i64> undef, <2 x i32> <i32 2, i32 3>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_01 = shufflevector <8 x i64> %src512, <8 x i64> undef, <2 x i32> <i32 0, i32 1>
@@ -102,7 +107,7 @@ define void @test_vXfi64(<4 x i64> %src256, <8 x i64> %src512) {
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <8 x i64> %src512, <8 x i64> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
-; AVX512-LABEL: 'test_vXfi64'
+; AVX512-LABEL: 'test_vXi64'
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_01 = shufflevector <4 x i64> %src256, <4 x i64> undef, <2 x i32> <i32 0, i32 1>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_23 = shufflevector <4 x i64> %src256, <4 x i64> undef, <2 x i32> <i32 2, i32 3>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_01 = shufflevector <8 x i64> %src512, <8 x i64> undef, <2 x i32> <i32 0, i32 1>
@@ -114,7 +119,7 @@ define void @test_vXfi64(<4 x i64> %src256, <8 x i64> %src512) {
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_4567 = shufflevector <8 x i64> %src512, <8 x i64> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
-; BTVER2-LABEL: 'test_vXfi64'
+; BTVER2-LABEL: 'test_vXi64'
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_01 = shufflevector <4 x i64> %src256, <4 x i64> undef, <2 x i32> <i32 0, i32 1>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_23 = shufflevector <4 x i64> %src256, <4 x i64> undef, <2 x i32> <i32 2, i32 3>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_01 = shufflevector <8 x i64> %src512, <8 x i64> undef, <2 x i32> <i32 0, i32 1>
@@ -137,3 +142,1733 @@ define void @test_vXfi64(<4 x i64> %src256, <8 x i64> %src512) {
   %V512_4567 = shufflevector <8 x i64> %src512, <8 x i64> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
   ret void
 }
+
+define void @test_vXi32(<4 x i32> %src128, <8 x i32> %src256, <16 x i32> %src512) {
+; SSE-LABEL: 'test_vXi32'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128_01 = shufflevector <4 x i32> %src128, <4 x i32> undef, <2 x i32> <i32 0, i32 1>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_23 = shufflevector <4 x i32> %src128, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_01 = shufflevector <8 x i32> %src256, <8 x i32> undef, <2 x i32> <i32 0, i32 1>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_23 = shufflevector <8 x i32> %src256, <8 x i32> undef, <2 x i32> <i32 2, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_45 = shufflevector <8 x i32> %src256, <8 x i32> undef, <2 x i32> <i32 4, i32 5>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_67 = shufflevector <8 x i32> %src256, <8 x i32> undef, <2 x i32> <i32 6, i32 7>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_0123 = shufflevector <8 x i32> %src256, <8 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_4567 = shufflevector <8 x i32> %src256, <8 x i32> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_01 = shufflevector <16 x i32> %src512, <16 x i32> undef, <2 x i32> <i32 0, i32 1>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_23 = shufflevector <16 x i32> %src512, <16 x i32> undef, <2 x i32> <i32 2, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_45 = shufflevector <16 x i32> %src512, <16 x i32> undef, <2 x i32> <i32 4, i32 5>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_67 = shufflevector <16 x i32> %src512, <16 x i32> undef, <2 x i32> <i32 6, i32 7>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89 = shufflevector <16 x i32> %src512, <16 x i32> undef, <2 x i32> <i32 8, i32 9>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_AB = shufflevector <16 x i32> %src512, <16 x i32> undef, <2 x i32> <i32 10, i32 11>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_CD = shufflevector <16 x i32> %src512, <16 x i32> undef, <2 x i32> <i32 12, i32 13>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_EF = shufflevector <16 x i32> %src512, <16 x i32> undef, <2 x i32> <i32 14, i32 15>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_0123 = shufflevector <16 x i32> %src512, <16 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <16 x i32> %src512, <16 x i32> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89AB = shufflevector <16 x i32> %src512, <16 x i32> undef, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_CDEF = shufflevector <16 x i32> %src512, <16 x i32> undef, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_01234567 = shufflevector <16 x i32> %src512, <16 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89ABCDEF = shufflevector <16 x i32> %src512, <16 x i32> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX-LABEL: 'test_vXi32'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128_01 = shufflevector <4 x i32> %src128, <4 x i32> undef, <2 x i32> <i32 0, i32 1>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_23 = shufflevector <4 x i32> %src128, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_01 = shufflevector <8 x i32> %src256, <8 x i32> undef, <2 x i32> <i32 0, i32 1>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_23 = shufflevector <8 x i32> %src256, <8 x i32> undef, <2 x i32> <i32 2, i32 3>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_45 = shufflevector <8 x i32> %src256, <8 x i32> undef, <2 x i32> <i32 4, i32 5>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_67 = shufflevector <8 x i32> %src256, <8 x i32> undef, <2 x i32> <i32 6, i32 7>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_0123 = shufflevector <8 x i32> %src256, <8 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_4567 = shufflevector <8 x i32> %src256, <8 x i32> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_01 = shufflevector <16 x i32> %src512, <16 x i32> undef, <2 x i32> <i32 0, i32 1>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_23 = shufflevector <16 x i32> %src512, <16 x i32> undef, <2 x i32> <i32 2, i32 3>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_45 = shufflevector <16 x i32> %src512, <16 x i32> undef, <2 x i32> <i32 4, i32 5>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_67 = shufflevector <16 x i32> %src512, <16 x i32> undef, <2 x i32> <i32 6, i32 7>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89 = shufflevector <16 x i32> %src512, <16 x i32> undef, <2 x i32> <i32 8, i32 9>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_AB = shufflevector <16 x i32> %src512, <16 x i32> undef, <2 x i32> <i32 10, i32 11>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CD = shufflevector <16 x i32> %src512, <16 x i32> undef, <2 x i32> <i32 12, i32 13>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_EF = shufflevector <16 x i32> %src512, <16 x i32> undef, <2 x i32> <i32 14, i32 15>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_0123 = shufflevector <16 x i32> %src512, <16 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_4567 = shufflevector <16 x i32> %src512, <16 x i32> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89AB = shufflevector <16 x i32> %src512, <16 x i32> undef, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CDEF = shufflevector <16 x i32> %src512, <16 x i32> undef, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_01234567 = shufflevector <16 x i32> %src512, <16 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89ABCDEF = shufflevector <16 x i32> %src512, <16 x i32> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX512-LABEL: 'test_vXi32'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128_01 = shufflevector <4 x i32> %src128, <4 x i32> undef, <2 x i32> <i32 0, i32 1>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_23 = shufflevector <4 x i32> %src128, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_01 = shufflevector <8 x i32> %src256, <8 x i32> undef, <2 x i32> <i32 0, i32 1>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_23 = shufflevector <8 x i32> %src256, <8 x i32> undef, <2 x i32> <i32 2, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_45 = shufflevector <8 x i32> %src256, <8 x i32> undef, <2 x i32> <i32 4, i32 5>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_67 = shufflevector <8 x i32> %src256, <8 x i32> undef, <2 x i32> <i32 6, i32 7>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_0123 = shufflevector <8 x i32> %src256, <8 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_4567 = shufflevector <8 x i32> %src256, <8 x i32> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_01 = shufflevector <16 x i32> %src512, <16 x i32> undef, <2 x i32> <i32 0, i32 1>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_23 = shufflevector <16 x i32> %src512, <16 x i32> undef, <2 x i32> <i32 2, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_45 = shufflevector <16 x i32> %src512, <16 x i32> undef, <2 x i32> <i32 4, i32 5>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_67 = shufflevector <16 x i32> %src512, <16 x i32> undef, <2 x i32> <i32 6, i32 7>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89 = shufflevector <16 x i32> %src512, <16 x i32> undef, <2 x i32> <i32 8, i32 9>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_AB = shufflevector <16 x i32> %src512, <16 x i32> undef, <2 x i32> <i32 10, i32 11>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CD = shufflevector <16 x i32> %src512, <16 x i32> undef, <2 x i32> <i32 12, i32 13>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_EF = shufflevector <16 x i32> %src512, <16 x i32> undef, <2 x i32> <i32 14, i32 15>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_0123 = shufflevector <16 x i32> %src512, <16 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_4567 = shufflevector <16 x i32> %src512, <16 x i32> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89AB = shufflevector <16 x i32> %src512, <16 x i32> undef, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CDEF = shufflevector <16 x i32> %src512, <16 x i32> undef, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_01234567 = shufflevector <16 x i32> %src512, <16 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89ABCDEF = shufflevector <16 x i32> %src512, <16 x i32> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; BTVER2-LABEL: 'test_vXi32'
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128_01 = shufflevector <4 x i32> %src128, <4 x i32> undef, <2 x i32> <i32 0, i32 1>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_23 = shufflevector <4 x i32> %src128, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_01 = shufflevector <8 x i32> %src256, <8 x i32> undef, <2 x i32> <i32 0, i32 1>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_23 = shufflevector <8 x i32> %src256, <8 x i32> undef, <2 x i32> <i32 2, i32 3>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_45 = shufflevector <8 x i32> %src256, <8 x i32> undef, <2 x i32> <i32 4, i32 5>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_67 = shufflevector <8 x i32> %src256, <8 x i32> undef, <2 x i32> <i32 6, i32 7>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_0123 = shufflevector <8 x i32> %src256, <8 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_4567 = shufflevector <8 x i32> %src256, <8 x i32> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_01 = shufflevector <16 x i32> %src512, <16 x i32> undef, <2 x i32> <i32 0, i32 1>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_23 = shufflevector <16 x i32> %src512, <16 x i32> undef, <2 x i32> <i32 2, i32 3>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_45 = shufflevector <16 x i32> %src512, <16 x i32> undef, <2 x i32> <i32 4, i32 5>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_67 = shufflevector <16 x i32> %src512, <16 x i32> undef, <2 x i32> <i32 6, i32 7>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89 = shufflevector <16 x i32> %src512, <16 x i32> undef, <2 x i32> <i32 8, i32 9>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_AB = shufflevector <16 x i32> %src512, <16 x i32> undef, <2 x i32> <i32 10, i32 11>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CD = shufflevector <16 x i32> %src512, <16 x i32> undef, <2 x i32> <i32 12, i32 13>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_EF = shufflevector <16 x i32> %src512, <16 x i32> undef, <2 x i32> <i32 14, i32 15>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_0123 = shufflevector <16 x i32> %src512, <16 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_4567 = shufflevector <16 x i32> %src512, <16 x i32> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89AB = shufflevector <16 x i32> %src512, <16 x i32> undef, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CDEF = shufflevector <16 x i32> %src512, <16 x i32> undef, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_01234567 = shufflevector <16 x i32> %src512, <16 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89ABCDEF = shufflevector <16 x i32> %src512, <16 x i32> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %V128_01 = shufflevector <4 x i32> %src128, <4 x i32> undef, <2 x i32> <i32 0, i32 1>
+  %V128_23 = shufflevector <4 x i32> %src128, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
+  %V256_01 = shufflevector <8 x i32> %src256, <8 x i32> undef, <2 x i32> <i32 0, i32 1>
+  %V256_23 = shufflevector <8 x i32> %src256, <8 x i32> undef, <2 x i32> <i32 2, i32 3>
+  %V256_45 = shufflevector <8 x i32> %src256, <8 x i32> undef, <2 x i32> <i32 4, i32 5>
+  %V256_67 = shufflevector <8 x i32> %src256, <8 x i32> undef, <2 x i32> <i32 6, i32 7>
+  %V256_0123 = shufflevector <8 x i32> %src256, <8 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %V256_4567 = shufflevector <8 x i32> %src256, <8 x i32> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %V512_01 = shufflevector <16 x i32> %src512, <16 x i32> undef, <2 x i32> <i32 0, i32 1>
+  %V512_23 = shufflevector <16 x i32> %src512, <16 x i32> undef, <2 x i32> <i32 2, i32 3>
+  %V512_45 = shufflevector <16 x i32> %src512, <16 x i32> undef, <2 x i32> <i32 4, i32 5>
+  %V512_67 = shufflevector <16 x i32> %src512, <16 x i32> undef, <2 x i32> <i32 6, i32 7>
+  %V512_89 = shufflevector <16 x i32> %src512, <16 x i32> undef, <2 x i32> <i32 8, i32 9>
+  %V512_AB = shufflevector <16 x i32> %src512, <16 x i32> undef, <2 x i32> <i32 10, i32 11>
+  %V512_CD = shufflevector <16 x i32> %src512, <16 x i32> undef, <2 x i32> <i32 12, i32 13>
+  %V512_EF = shufflevector <16 x i32> %src512, <16 x i32> undef, <2 x i32> <i32 14, i32 15>
+  %V512_0123 = shufflevector <16 x i32> %src512, <16 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %V512_4567 = shufflevector <16 x i32> %src512, <16 x i32> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %V512_89AB = shufflevector <16 x i32> %src512, <16 x i32> undef, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+  %V512_CDEF = shufflevector <16 x i32> %src512, <16 x i32> undef, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+  %V512_01234567 = shufflevector <16 x i32> %src512, <16 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %V512_89ABCDEF = shufflevector <16 x i32> %src512, <16 x i32> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  ret void
+}
+
+define void @test_vXi16(<4 x i16> %src64, <8 x i16> %src128, <16 x i16> %src256, <32 x i16> %src512) {
+; SSE2-LABEL: 'test_vXi16'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64_01 = shufflevector <4 x i16> %src64, <4 x i16> undef, <2 x i32> <i32 0, i32 1>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_23 = shufflevector <4 x i16> %src64, <4 x i16> undef, <2 x i32> <i32 2, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128_01 = shufflevector <8 x i16> %src128, <8 x i16> undef, <2 x i32> <i32 0, i32 1>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_23 = shufflevector <8 x i16> %src128, <8 x i16> undef, <2 x i32> <i32 2, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_45 = shufflevector <8 x i16> %src128, <8 x i16> undef, <2 x i32> <i32 4, i32 5>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_67 = shufflevector <8 x i16> %src128, <8 x i16> undef, <2 x i32> <i32 6, i32 7>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128_0123 = shufflevector <8 x i16> %src128, <8 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_4567 = shufflevector <8 x i16> %src128, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_01 = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 0, i32 1>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_23 = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 2, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_45 = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 4, i32 5>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_67 = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 6, i32 7>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_89 = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 8, i32 9>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_AB = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 10, i32 11>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_CD = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 12, i32 13>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_EF = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_0123 = shufflevector <16 x i16> %src256, <16 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V256_2345 = shufflevector <16 x i16> %src256, <16 x i16> undef, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_4567 = shufflevector <16 x i16> %src256, <16 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V256_6789 = shufflevector <16 x i16> %src256, <16 x i16> undef, <4 x i32> <i32 6, i32 7, i32 8, i32 9>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_89AB = shufflevector <16 x i16> %src256, <16 x i16> undef, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_CDEF = shufflevector <16 x i16> %src256, <16 x i16> undef, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_01234567 = shufflevector <16 x i16> %src256, <16 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_89ABCDEF = shufflevector <16 x i16> %src256, <16 x i16> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 0, i32 1>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_02_03 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 2, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_04_05 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 4, i32 5>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_06_07 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 6, i32 7>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_08_09 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 8, i32 9>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_0A_0B = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 10, i32 11>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_0C_0D = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 12, i32 13>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_0E_0F = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_10_11 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 16, i32 17>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_12_13 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 18, i32 19>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_14_15 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 20, i32 21>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_16_17 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 22, i32 23>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_18_19 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 24, i32 25>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_1A_1B = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 26, i32 27>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_1C_1D = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 28, i32 29>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_1E_1F = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 30, i32 31>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01_02_03 = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_02_03_04_05 = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_04_05_06_07 = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_06_07_08_09 = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 6, i32 7, i32 8, i32 9>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_08_09_0A_0B = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_0C_0D_0E_0F = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_10_11_12_13 = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_14_15_16_17 = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 20, i32 21, i32 22, i32 23>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_18_19_1A_1B = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 24, i32 25, i32 26, i32 27>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_1C_1D_1E_1F = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 28, i32 29, i32 30, i32 31>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01_02_03_04_05_06_07 = shufflevector <32 x i16> %src512, <32 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_08_09_0A_0B_0C_0D_0E_0F = shufflevector <32 x i16> %src512, <32 x i16> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_10_11_12_13_14_15_16_17 = shufflevector <32 x i16> %src512, <32 x i16> undef, <8 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_18_19_1A_1B_1C_1D_1E_1F = shufflevector <32 x i16> %src512, <32 x i16> undef, <8 x i32> <i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01_02_03_04_05_06_07_08_09_0A_0B_0C_0D_0E_0F = shufflevector <32 x i16> %src512, <32 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_10_11_12_13_14_15_16_17_18_19_1A_1B_1C_1D_1E_1F = shufflevector <32 x i16> %src512, <32 x i16> undef, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; SSSE3-LABEL: 'test_vXi16'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64_01 = shufflevector <4 x i16> %src64, <4 x i16> undef, <2 x i32> <i32 0, i32 1>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_23 = shufflevector <4 x i16> %src64, <4 x i16> undef, <2 x i32> <i32 2, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128_01 = shufflevector <8 x i16> %src128, <8 x i16> undef, <2 x i32> <i32 0, i32 1>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_23 = shufflevector <8 x i16> %src128, <8 x i16> undef, <2 x i32> <i32 2, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_45 = shufflevector <8 x i16> %src128, <8 x i16> undef, <2 x i32> <i32 4, i32 5>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_67 = shufflevector <8 x i16> %src128, <8 x i16> undef, <2 x i32> <i32 6, i32 7>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128_0123 = shufflevector <8 x i16> %src128, <8 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_4567 = shufflevector <8 x i16> %src128, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_01 = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 0, i32 1>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_23 = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 2, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_45 = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 4, i32 5>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_67 = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 6, i32 7>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_89 = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 8, i32 9>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_AB = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 10, i32 11>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_CD = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 12, i32 13>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_EF = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 14, i32 15>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_0123 = shufflevector <16 x i16> %src256, <16 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V256_2345 = shufflevector <16 x i16> %src256, <16 x i16> undef, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_4567 = shufflevector <16 x i16> %src256, <16 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V256_6789 = shufflevector <16 x i16> %src256, <16 x i16> undef, <4 x i32> <i32 6, i32 7, i32 8, i32 9>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_89AB = shufflevector <16 x i16> %src256, <16 x i16> undef, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_CDEF = shufflevector <16 x i16> %src256, <16 x i16> undef, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_01234567 = shufflevector <16 x i16> %src256, <16 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_89ABCDEF = shufflevector <16 x i16> %src256, <16 x i16> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 0, i32 1>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_02_03 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 2, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_04_05 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 4, i32 5>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_06_07 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 6, i32 7>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_08_09 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 8, i32 9>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_0A_0B = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 10, i32 11>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_0C_0D = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 12, i32 13>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_0E_0F = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 14, i32 15>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_10_11 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 16, i32 17>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_12_13 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 18, i32 19>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_14_15 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 20, i32 21>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_16_17 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 22, i32 23>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_18_19 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 24, i32 25>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_1A_1B = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 26, i32 27>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_1C_1D = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 28, i32 29>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_1E_1F = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 30, i32 31>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01_02_03 = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_02_03_04_05 = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_04_05_06_07 = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_06_07_08_09 = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 6, i32 7, i32 8, i32 9>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_08_09_0A_0B = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_0C_0D_0E_0F = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_10_11_12_13 = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_14_15_16_17 = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 20, i32 21, i32 22, i32 23>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_18_19_1A_1B = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 24, i32 25, i32 26, i32 27>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_1C_1D_1E_1F = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 28, i32 29, i32 30, i32 31>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01_02_03_04_05_06_07 = shufflevector <32 x i16> %src512, <32 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_08_09_0A_0B_0C_0D_0E_0F = shufflevector <32 x i16> %src512, <32 x i16> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_10_11_12_13_14_15_16_17 = shufflevector <32 x i16> %src512, <32 x i16> undef, <8 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_18_19_1A_1B_1C_1D_1E_1F = shufflevector <32 x i16> %src512, <32 x i16> undef, <8 x i32> <i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01_02_03_04_05_06_07_08_09_0A_0B_0C_0D_0E_0F = shufflevector <32 x i16> %src512, <32 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_10_11_12_13_14_15_16_17_18_19_1A_1B_1C_1D_1E_1F = shufflevector <32 x i16> %src512, <32 x i16> undef, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX-LABEL: 'test_vXi16'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64_01 = shufflevector <4 x i16> %src64, <4 x i16> undef, <2 x i32> <i32 0, i32 1>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_23 = shufflevector <4 x i16> %src64, <4 x i16> undef, <2 x i32> <i32 2, i32 3>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128_01 = shufflevector <8 x i16> %src128, <8 x i16> undef, <2 x i32> <i32 0, i32 1>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_23 = shufflevector <8 x i16> %src128, <8 x i16> undef, <2 x i32> <i32 2, i32 3>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_45 = shufflevector <8 x i16> %src128, <8 x i16> undef, <2 x i32> <i32 4, i32 5>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_67 = shufflevector <8 x i16> %src128, <8 x i16> undef, <2 x i32> <i32 6, i32 7>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128_0123 = shufflevector <8 x i16> %src128, <8 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_4567 = shufflevector <8 x i16> %src128, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_01 = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 0, i32 1>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_23 = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 2, i32 3>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_45 = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 4, i32 5>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_67 = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 6, i32 7>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_89 = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 8, i32 9>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_AB = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 10, i32 11>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_CD = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 12, i32 13>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_EF = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 14, i32 15>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_0123 = shufflevector <16 x i16> %src256, <16 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V256_2345 = shufflevector <16 x i16> %src256, <16 x i16> undef, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_4567 = shufflevector <16 x i16> %src256, <16 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V256_6789 = shufflevector <16 x i16> %src256, <16 x i16> undef, <4 x i32> <i32 6, i32 7, i32 8, i32 9>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_89AB = shufflevector <16 x i16> %src256, <16 x i16> undef, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_CDEF = shufflevector <16 x i16> %src256, <16 x i16> undef, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_01234567 = shufflevector <16 x i16> %src256, <16 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_89ABCDEF = shufflevector <16 x i16> %src256, <16 x i16> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 0, i32 1>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_02_03 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 2, i32 3>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_04_05 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 4, i32 5>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_06_07 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 6, i32 7>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_08_09 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 8, i32 9>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_0A_0B = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 10, i32 11>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_0C_0D = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 12, i32 13>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_0E_0F = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 14, i32 15>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_10_11 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 16, i32 17>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_12_13 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 18, i32 19>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_14_15 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 20, i32 21>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_16_17 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 22, i32 23>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_18_19 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 24, i32 25>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_1A_1B = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 26, i32 27>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_1C_1D = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 28, i32 29>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_1E_1F = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 30, i32 31>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01_02_03 = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_02_03_04_05 = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_04_05_06_07 = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_06_07_08_09 = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 6, i32 7, i32 8, i32 9>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_08_09_0A_0B = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_0C_0D_0E_0F = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_10_11_12_13 = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_14_15_16_17 = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 20, i32 21, i32 22, i32 23>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_18_19_1A_1B = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 24, i32 25, i32 26, i32 27>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_1C_1D_1E_1F = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 28, i32 29, i32 30, i32 31>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01_02_03_04_05_06_07 = shufflevector <32 x i16> %src512, <32 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_08_09_0A_0B_0C_0D_0E_0F = shufflevector <32 x i16> %src512, <32 x i16> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_10_11_12_13_14_15_16_17 = shufflevector <32 x i16> %src512, <32 x i16> undef, <8 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_18_19_1A_1B_1C_1D_1E_1F = shufflevector <32 x i16> %src512, <32 x i16> undef, <8 x i32> <i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01_02_03_04_05_06_07_08_09_0A_0B_0C_0D_0E_0F = shufflevector <32 x i16> %src512, <32 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_10_11_12_13_14_15_16_17_18_19_1A_1B_1C_1D_1E_1F = shufflevector <32 x i16> %src512, <32 x i16> undef, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX512F-LABEL: 'test_vXi16'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64_01 = shufflevector <4 x i16> %src64, <4 x i16> undef, <2 x i32> <i32 0, i32 1>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_23 = shufflevector <4 x i16> %src64, <4 x i16> undef, <2 x i32> <i32 2, i32 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128_01 = shufflevector <8 x i16> %src128, <8 x i16> undef, <2 x i32> <i32 0, i32 1>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_23 = shufflevector <8 x i16> %src128, <8 x i16> undef, <2 x i32> <i32 2, i32 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_45 = shufflevector <8 x i16> %src128, <8 x i16> undef, <2 x i32> <i32 4, i32 5>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_67 = shufflevector <8 x i16> %src128, <8 x i16> undef, <2 x i32> <i32 6, i32 7>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128_0123 = shufflevector <8 x i16> %src128, <8 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_4567 = shufflevector <8 x i16> %src128, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_01 = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 0, i32 1>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_23 = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 2, i32 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_45 = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 4, i32 5>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_67 = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 6, i32 7>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_89 = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 8, i32 9>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_AB = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 10, i32 11>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_CD = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 12, i32 13>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_EF = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 14, i32 15>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_0123 = shufflevector <16 x i16> %src256, <16 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V256_2345 = shufflevector <16 x i16> %src256, <16 x i16> undef, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_4567 = shufflevector <16 x i16> %src256, <16 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V256_6789 = shufflevector <16 x i16> %src256, <16 x i16> undef, <4 x i32> <i32 6, i32 7, i32 8, i32 9>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_89AB = shufflevector <16 x i16> %src256, <16 x i16> undef, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_CDEF = shufflevector <16 x i16> %src256, <16 x i16> undef, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_01234567 = shufflevector <16 x i16> %src256, <16 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_89ABCDEF = shufflevector <16 x i16> %src256, <16 x i16> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 0, i32 1>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_02_03 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 2, i32 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_04_05 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 4, i32 5>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_06_07 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 6, i32 7>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_08_09 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 8, i32 9>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_0A_0B = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 10, i32 11>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_0C_0D = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 12, i32 13>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_0E_0F = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 14, i32 15>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_10_11 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 16, i32 17>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_12_13 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 18, i32 19>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_14_15 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 20, i32 21>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_16_17 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 22, i32 23>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_18_19 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 24, i32 25>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_1A_1B = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 26, i32 27>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_1C_1D = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 28, i32 29>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_1E_1F = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 30, i32 31>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01_02_03 = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_02_03_04_05 = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_04_05_06_07 = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_06_07_08_09 = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 6, i32 7, i32 8, i32 9>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_08_09_0A_0B = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_0C_0D_0E_0F = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_10_11_12_13 = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_14_15_16_17 = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 20, i32 21, i32 22, i32 23>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_18_19_1A_1B = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 24, i32 25, i32 26, i32 27>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_1C_1D_1E_1F = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 28, i32 29, i32 30, i32 31>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01_02_03_04_05_06_07 = shufflevector <32 x i16> %src512, <32 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_08_09_0A_0B_0C_0D_0E_0F = shufflevector <32 x i16> %src512, <32 x i16> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_10_11_12_13_14_15_16_17 = shufflevector <32 x i16> %src512, <32 x i16> undef, <8 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_18_19_1A_1B_1C_1D_1E_1F = shufflevector <32 x i16> %src512, <32 x i16> undef, <8 x i32> <i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01_02_03_04_05_06_07_08_09_0A_0B_0C_0D_0E_0F = shufflevector <32 x i16> %src512, <32 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_10_11_12_13_14_15_16_17_18_19_1A_1B_1C_1D_1E_1F = shufflevector <32 x i16> %src512, <32 x i16> undef, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX512BW-LABEL: 'test_vXi16'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64_01 = shufflevector <4 x i16> %src64, <4 x i16> undef, <2 x i32> <i32 0, i32 1>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_23 = shufflevector <4 x i16> %src64, <4 x i16> undef, <2 x i32> <i32 2, i32 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128_01 = shufflevector <8 x i16> %src128, <8 x i16> undef, <2 x i32> <i32 0, i32 1>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_23 = shufflevector <8 x i16> %src128, <8 x i16> undef, <2 x i32> <i32 2, i32 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_45 = shufflevector <8 x i16> %src128, <8 x i16> undef, <2 x i32> <i32 4, i32 5>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_67 = shufflevector <8 x i16> %src128, <8 x i16> undef, <2 x i32> <i32 6, i32 7>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128_0123 = shufflevector <8 x i16> %src128, <8 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_4567 = shufflevector <8 x i16> %src128, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_01 = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 0, i32 1>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_23 = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 2, i32 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_45 = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 4, i32 5>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_67 = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 6, i32 7>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_89 = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 8, i32 9>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_AB = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 10, i32 11>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_CD = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 12, i32 13>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_EF = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 14, i32 15>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_0123 = shufflevector <16 x i16> %src256, <16 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V256_2345 = shufflevector <16 x i16> %src256, <16 x i16> undef, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_4567 = shufflevector <16 x i16> %src256, <16 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V256_6789 = shufflevector <16 x i16> %src256, <16 x i16> undef, <4 x i32> <i32 6, i32 7, i32 8, i32 9>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_89AB = shufflevector <16 x i16> %src256, <16 x i16> undef, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_CDEF = shufflevector <16 x i16> %src256, <16 x i16> undef, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_01234567 = shufflevector <16 x i16> %src256, <16 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_89ABCDEF = shufflevector <16 x i16> %src256, <16 x i16> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 0, i32 1>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_02_03 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 2, i32 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_04_05 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 4, i32 5>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_06_07 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 6, i32 7>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_08_09 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 8, i32 9>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_0A_0B = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 10, i32 11>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_0C_0D = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 12, i32 13>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_0E_0F = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 14, i32 15>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_10_11 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 16, i32 17>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_12_13 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 18, i32 19>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_14_15 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 20, i32 21>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_16_17 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 22, i32 23>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_18_19 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 24, i32 25>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_1A_1B = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 26, i32 27>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_1C_1D = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 28, i32 29>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_1E_1F = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 30, i32 31>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01_02_03 = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_02_03_04_05 = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_04_05_06_07 = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_06_07_08_09 = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 6, i32 7, i32 8, i32 9>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_08_09_0A_0B = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_0C_0D_0E_0F = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_10_11_12_13 = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_14_15_16_17 = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 20, i32 21, i32 22, i32 23>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_18_19_1A_1B = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 24, i32 25, i32 26, i32 27>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_1C_1D_1E_1F = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 28, i32 29, i32 30, i32 31>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01_02_03_04_05_06_07 = shufflevector <32 x i16> %src512, <32 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_08_09_0A_0B_0C_0D_0E_0F = shufflevector <32 x i16> %src512, <32 x i16> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_10_11_12_13_14_15_16_17 = shufflevector <32 x i16> %src512, <32 x i16> undef, <8 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_18_19_1A_1B_1C_1D_1E_1F = shufflevector <32 x i16> %src512, <32 x i16> undef, <8 x i32> <i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01_02_03_04_05_06_07_08_09_0A_0B_0C_0D_0E_0F = shufflevector <32 x i16> %src512, <32 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_10_11_12_13_14_15_16_17_18_19_1A_1B_1C_1D_1E_1F = shufflevector <32 x i16> %src512, <32 x i16> undef, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; SLM-LABEL: 'test_vXi16'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64_01 = shufflevector <4 x i16> %src64, <4 x i16> undef, <2 x i32> <i32 0, i32 1>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_23 = shufflevector <4 x i16> %src64, <4 x i16> undef, <2 x i32> <i32 2, i32 3>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128_01 = shufflevector <8 x i16> %src128, <8 x i16> undef, <2 x i32> <i32 0, i32 1>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_23 = shufflevector <8 x i16> %src128, <8 x i16> undef, <2 x i32> <i32 2, i32 3>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_45 = shufflevector <8 x i16> %src128, <8 x i16> undef, <2 x i32> <i32 4, i32 5>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_67 = shufflevector <8 x i16> %src128, <8 x i16> undef, <2 x i32> <i32 6, i32 7>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128_0123 = shufflevector <8 x i16> %src128, <8 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_4567 = shufflevector <8 x i16> %src128, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_01 = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 0, i32 1>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_23 = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 2, i32 3>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_45 = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 4, i32 5>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_67 = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 6, i32 7>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_89 = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 8, i32 9>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_AB = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 10, i32 11>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_CD = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 12, i32 13>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_EF = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 14, i32 15>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_0123 = shufflevector <16 x i16> %src256, <16 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V256_2345 = shufflevector <16 x i16> %src256, <16 x i16> undef, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_4567 = shufflevector <16 x i16> %src256, <16 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V256_6789 = shufflevector <16 x i16> %src256, <16 x i16> undef, <4 x i32> <i32 6, i32 7, i32 8, i32 9>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_89AB = shufflevector <16 x i16> %src256, <16 x i16> undef, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_CDEF = shufflevector <16 x i16> %src256, <16 x i16> undef, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_01234567 = shufflevector <16 x i16> %src256, <16 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_89ABCDEF = shufflevector <16 x i16> %src256, <16 x i16> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 0, i32 1>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_02_03 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 2, i32 3>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_04_05 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 4, i32 5>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_06_07 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 6, i32 7>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_08_09 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 8, i32 9>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_0A_0B = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 10, i32 11>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_0C_0D = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 12, i32 13>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_0E_0F = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 14, i32 15>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_10_11 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 16, i32 17>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_12_13 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 18, i32 19>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_14_15 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 20, i32 21>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_16_17 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 22, i32 23>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_18_19 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 24, i32 25>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_1A_1B = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 26, i32 27>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_1C_1D = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 28, i32 29>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_1E_1F = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 30, i32 31>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01_02_03 = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %V512_02_03_04_05 = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_04_05_06_07 = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 53 for instruction: %V512_06_07_08_09 = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 6, i32 7, i32 8, i32 9>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_08_09_0A_0B = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_0C_0D_0E_0F = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_10_11_12_13 = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_14_15_16_17 = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 20, i32 21, i32 22, i32 23>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_18_19_1A_1B = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 24, i32 25, i32 26, i32 27>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_1C_1D_1E_1F = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 28, i32 29, i32 30, i32 31>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01_02_03_04_05_06_07 = shufflevector <32 x i16> %src512, <32 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_08_09_0A_0B_0C_0D_0E_0F = shufflevector <32 x i16> %src512, <32 x i16> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_10_11_12_13_14_15_16_17 = shufflevector <32 x i16> %src512, <32 x i16> undef, <8 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_18_19_1A_1B_1C_1D_1E_1F = shufflevector <32 x i16> %src512, <32 x i16> undef, <8 x i32> <i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01_02_03_04_05_06_07_08_09_0A_0B_0C_0D_0E_0F = shufflevector <32 x i16> %src512, <32 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_10_11_12_13_14_15_16_17_18_19_1A_1B_1C_1D_1E_1F = shufflevector <32 x i16> %src512, <32 x i16> undef, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; GLM-LABEL: 'test_vXi16'
+; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64_01 = shufflevector <4 x i16> %src64, <4 x i16> undef, <2 x i32> <i32 0, i32 1>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_23 = shufflevector <4 x i16> %src64, <4 x i16> undef, <2 x i32> <i32 2, i32 3>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128_01 = shufflevector <8 x i16> %src128, <8 x i16> undef, <2 x i32> <i32 0, i32 1>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_23 = shufflevector <8 x i16> %src128, <8 x i16> undef, <2 x i32> <i32 2, i32 3>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_45 = shufflevector <8 x i16> %src128, <8 x i16> undef, <2 x i32> <i32 4, i32 5>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_67 = shufflevector <8 x i16> %src128, <8 x i16> undef, <2 x i32> <i32 6, i32 7>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128_0123 = shufflevector <8 x i16> %src128, <8 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_4567 = shufflevector <8 x i16> %src128, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_01 = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 0, i32 1>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_23 = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 2, i32 3>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_45 = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 4, i32 5>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_67 = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 6, i32 7>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_89 = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 8, i32 9>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_AB = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 10, i32 11>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_CD = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 12, i32 13>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_EF = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 14, i32 15>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_0123 = shufflevector <16 x i16> %src256, <16 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V256_2345 = shufflevector <16 x i16> %src256, <16 x i16> undef, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_4567 = shufflevector <16 x i16> %src256, <16 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V256_6789 = shufflevector <16 x i16> %src256, <16 x i16> undef, <4 x i32> <i32 6, i32 7, i32 8, i32 9>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_89AB = shufflevector <16 x i16> %src256, <16 x i16> undef, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_CDEF = shufflevector <16 x i16> %src256, <16 x i16> undef, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_01234567 = shufflevector <16 x i16> %src256, <16 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_89ABCDEF = shufflevector <16 x i16> %src256, <16 x i16> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 0, i32 1>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_02_03 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 2, i32 3>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_04_05 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 4, i32 5>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_06_07 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 6, i32 7>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_08_09 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 8, i32 9>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_0A_0B = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 10, i32 11>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_0C_0D = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 12, i32 13>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_0E_0F = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 14, i32 15>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_10_11 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 16, i32 17>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_12_13 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 18, i32 19>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_14_15 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 20, i32 21>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_16_17 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 22, i32 23>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_18_19 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 24, i32 25>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_1A_1B = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 26, i32 27>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_1C_1D = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 28, i32 29>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_1E_1F = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 30, i32 31>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01_02_03 = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_02_03_04_05 = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_04_05_06_07 = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_06_07_08_09 = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 6, i32 7, i32 8, i32 9>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_08_09_0A_0B = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_0C_0D_0E_0F = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_10_11_12_13 = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_14_15_16_17 = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 20, i32 21, i32 22, i32 23>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_18_19_1A_1B = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 24, i32 25, i32 26, i32 27>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_1C_1D_1E_1F = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 28, i32 29, i32 30, i32 31>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01_02_03_04_05_06_07 = shufflevector <32 x i16> %src512, <32 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_08_09_0A_0B_0C_0D_0E_0F = shufflevector <32 x i16> %src512, <32 x i16> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_10_11_12_13_14_15_16_17 = shufflevector <32 x i16> %src512, <32 x i16> undef, <8 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_18_19_1A_1B_1C_1D_1E_1F = shufflevector <32 x i16> %src512, <32 x i16> undef, <8 x i32> <i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01_02_03_04_05_06_07_08_09_0A_0B_0C_0D_0E_0F = shufflevector <32 x i16> %src512, <32 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_10_11_12_13_14_15_16_17_18_19_1A_1B_1C_1D_1E_1F = shufflevector <32 x i16> %src512, <32 x i16> undef, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; BTVER2-LABEL: 'test_vXi16'
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64_01 = shufflevector <4 x i16> %src64, <4 x i16> undef, <2 x i32> <i32 0, i32 1>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_23 = shufflevector <4 x i16> %src64, <4 x i16> undef, <2 x i32> <i32 2, i32 3>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128_01 = shufflevector <8 x i16> %src128, <8 x i16> undef, <2 x i32> <i32 0, i32 1>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_23 = shufflevector <8 x i16> %src128, <8 x i16> undef, <2 x i32> <i32 2, i32 3>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_45 = shufflevector <8 x i16> %src128, <8 x i16> undef, <2 x i32> <i32 4, i32 5>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_67 = shufflevector <8 x i16> %src128, <8 x i16> undef, <2 x i32> <i32 6, i32 7>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128_0123 = shufflevector <8 x i16> %src128, <8 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_4567 = shufflevector <8 x i16> %src128, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_01 = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 0, i32 1>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_23 = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 2, i32 3>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_45 = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 4, i32 5>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_67 = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 6, i32 7>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_89 = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 8, i32 9>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_AB = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 10, i32 11>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_CD = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 12, i32 13>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_EF = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 14, i32 15>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_0123 = shufflevector <16 x i16> %src256, <16 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V256_2345 = shufflevector <16 x i16> %src256, <16 x i16> undef, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_4567 = shufflevector <16 x i16> %src256, <16 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V256_6789 = shufflevector <16 x i16> %src256, <16 x i16> undef, <4 x i32> <i32 6, i32 7, i32 8, i32 9>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_89AB = shufflevector <16 x i16> %src256, <16 x i16> undef, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_CDEF = shufflevector <16 x i16> %src256, <16 x i16> undef, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_01234567 = shufflevector <16 x i16> %src256, <16 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_89ABCDEF = shufflevector <16 x i16> %src256, <16 x i16> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 0, i32 1>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_02_03 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 2, i32 3>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_04_05 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 4, i32 5>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_06_07 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 6, i32 7>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_08_09 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 8, i32 9>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_0A_0B = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 10, i32 11>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_0C_0D = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 12, i32 13>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_0E_0F = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 14, i32 15>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_10_11 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 16, i32 17>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_12_13 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 18, i32 19>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_14_15 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 20, i32 21>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_16_17 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 22, i32 23>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_18_19 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 24, i32 25>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_1A_1B = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 26, i32 27>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_1C_1D = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 28, i32 29>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_1E_1F = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 30, i32 31>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01_02_03 = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_02_03_04_05 = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_04_05_06_07 = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_06_07_08_09 = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 6, i32 7, i32 8, i32 9>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_08_09_0A_0B = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_0C_0D_0E_0F = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_10_11_12_13 = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_14_15_16_17 = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 20, i32 21, i32 22, i32 23>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_18_19_1A_1B = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 24, i32 25, i32 26, i32 27>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_1C_1D_1E_1F = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 28, i32 29, i32 30, i32 31>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01_02_03_04_05_06_07 = shufflevector <32 x i16> %src512, <32 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_08_09_0A_0B_0C_0D_0E_0F = shufflevector <32 x i16> %src512, <32 x i16> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_10_11_12_13_14_15_16_17 = shufflevector <32 x i16> %src512, <32 x i16> undef, <8 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_18_19_1A_1B_1C_1D_1E_1F = shufflevector <32 x i16> %src512, <32 x i16> undef, <8 x i32> <i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01_02_03_04_05_06_07_08_09_0A_0B_0C_0D_0E_0F = shufflevector <32 x i16> %src512, <32 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_10_11_12_13_14_15_16_17_18_19_1A_1B_1C_1D_1E_1F = shufflevector <32 x i16> %src512, <32 x i16> undef, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %V64_01 = shufflevector <4 x i16> %src64, <4 x i16> undef, <2 x i32> <i32 0, i32 1>
+  %V64_23 = shufflevector <4 x i16> %src64, <4 x i16> undef, <2 x i32> <i32 2, i32 3>
+  %V128_01 = shufflevector <8 x i16> %src128, <8 x i16> undef, <2 x i32> <i32 0, i32 1>
+  %V128_23 = shufflevector <8 x i16> %src128, <8 x i16> undef, <2 x i32> <i32 2, i32 3>
+  %V128_45 = shufflevector <8 x i16> %src128, <8 x i16> undef, <2 x i32> <i32 4, i32 5>
+  %V128_67 = shufflevector <8 x i16> %src128, <8 x i16> undef, <2 x i32> <i32 6, i32 7>
+  %V128_0123 = shufflevector <8 x i16> %src128, <8 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %V128_4567 = shufflevector <8 x i16> %src128, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %V256_01 = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 0, i32 1>
+  %V256_23 = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 2, i32 3>
+  %V256_45 = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 4, i32 5>
+  %V256_67 = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 6, i32 7>
+  %V256_89 = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 8, i32 9>
+  %V256_AB = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 10, i32 11>
+  %V256_CD = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 12, i32 13>
+  %V256_EF = shufflevector <16 x i16> %src256, <16 x i16> undef, <2 x i32> <i32 14, i32 15>
+  %V256_0123 = shufflevector <16 x i16> %src256, <16 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %V256_2345 = shufflevector <16 x i16> %src256, <16 x i16> undef, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
+  %V256_4567 = shufflevector <16 x i16> %src256, <16 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %V256_6789 = shufflevector <16 x i16> %src256, <16 x i16> undef, <4 x i32> <i32 6, i32 7, i32 8, i32 9>
+  %V256_89AB = shufflevector <16 x i16> %src256, <16 x i16> undef, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+  %V256_CDEF = shufflevector <16 x i16> %src256, <16 x i16> undef, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+  %V256_01234567 = shufflevector <16 x i16> %src256, <16 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %V256_89ABCDEF = shufflevector <16 x i16> %src256, <16 x i16> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %V512_00_01 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 0, i32 1>
+  %V512_02_03 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 2, i32 3>
+  %V512_04_05 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 4, i32 5>
+  %V512_06_07 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 6, i32 7>
+  %V512_08_09 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 8, i32 9>
+  %V512_0A_0B = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 10, i32 11>
+  %V512_0C_0D = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 12, i32 13>
+  %V512_0E_0F = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 14, i32 15>
+  %V512_10_11 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 16, i32 17>
+  %V512_12_13 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 18, i32 19>
+  %V512_14_15 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 20, i32 21>
+  %V512_16_17 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 22, i32 23>
+  %V512_18_19 = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 24, i32 25>
+  %V512_1A_1B = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 26, i32 27>
+  %V512_1C_1D = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 28, i32 29>
+  %V512_1E_1F = shufflevector <32 x i16> %src512, <32 x i16> undef, <2 x i32> <i32 30, i32 31>
+  %V512_00_01_02_03 = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %V512_02_03_04_05 = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
+  %V512_04_05_06_07 = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %V512_06_07_08_09 = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 6, i32 7, i32 8, i32 9>
+  %V512_08_09_0A_0B = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+  %V512_0C_0D_0E_0F = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+  %V512_10_11_12_13 = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
+  %V512_14_15_16_17 = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 20, i32 21, i32 22, i32 23>
+  %V512_18_19_1A_1B = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 24, i32 25, i32 26, i32 27>
+  %V512_1C_1D_1E_1F = shufflevector <32 x i16> %src512, <32 x i16> undef, <4 x i32> <i32 28, i32 29, i32 30, i32 31>
+  %V512_00_01_02_03_04_05_06_07 = shufflevector <32 x i16> %src512, <32 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %V512_08_09_0A_0B_0C_0D_0E_0F = shufflevector <32 x i16> %src512, <32 x i16> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %V512_10_11_12_13_14_15_16_17 = shufflevector <32 x i16> %src512, <32 x i16> undef, <8 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+  %V512_18_19_1A_1B_1C_1D_1E_1F = shufflevector <32 x i16> %src512, <32 x i16> undef, <8 x i32> <i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+  %V512_00_01_02_03_04_05_06_07_08_09_0A_0B_0C_0D_0E_0F = shufflevector <32 x i16> %src512, <32 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %V512_10_11_12_13_14_15_16_17_18_19_1A_1B_1C_1D_1E_1F = shufflevector <32 x i16> %src512, <32 x i16> undef, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+  ret void
+}
+
+define void @test_vXi8(<8 x i8> %src64, <16 x i8> %src128, <32 x i8> %src256, <64 x i8> %src512) {
+; SSE2-LABEL: 'test_vXi8'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64_01 = shufflevector <8 x i8> %src64, <8 x i8> undef, <2 x i32> <i32 0, i32 1>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64_23 = shufflevector <8 x i8> %src64, <8 x i8> undef, <2 x i32> <i32 2, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64_45 = shufflevector <8 x i8> %src64, <8 x i8> undef, <2 x i32> <i32 4, i32 5>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64_67 = shufflevector <8 x i8> %src64, <8 x i8> undef, <2 x i32> <i32 6, i32 7>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64_0123 = shufflevector <8 x i8> %src64, <8 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_4567 = shufflevector <8 x i8> %src64, <8 x i8> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128_01 = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 0, i32 1>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_23 = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 2, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_45 = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 4, i32 5>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_67 = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 6, i32 7>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_89 = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 8, i32 9>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_AB = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 10, i32 11>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_CD = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 12, i32 13>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_EF = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128_0123 = shufflevector <16 x i8> %src128, <16 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128_2345 = shufflevector <16 x i8> %src128, <16 x i8> undef, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_4567 = shufflevector <16 x i8> %src128, <16 x i8> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128_6789 = shufflevector <16 x i8> %src128, <16 x i8> undef, <4 x i32> <i32 6, i32 7, i32 8, i32 9>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_89AB = shufflevector <16 x i8> %src128, <16 x i8> undef, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_CDEF = shufflevector <16 x i8> %src128, <16 x i8> undef, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128_01234567 = shufflevector <16 x i8> %src128, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_89ABCDEF = shufflevector <16 x i8> %src128, <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_00_01 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 0, i32 1>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_02_03 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 2, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_04_05 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 4, i32 5>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_06_07 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 6, i32 7>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_08_09 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 8, i32 9>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_0A_0B = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 10, i32 11>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_0C_0D = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 12, i32 13>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_0E_0F = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_10_11 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 16, i32 17>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_12_13 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 18, i32 19>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_14_15 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 20, i32 21>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_16_17 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 22, i32 23>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_18_19 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 24, i32 25>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_1A_1B = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 26, i32 27>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_1C_1D = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 28, i32 29>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_1E_1F = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 30, i32 31>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_00_01_02_03 = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V256_02_03_04_05 = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_04_05_06_07 = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V256_06_07_08_09 = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 6, i32 7, i32 8, i32 9>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_08_09_0A_0B = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_0C_0D_0E_0F = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_10_11_12_13 = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_14_15_16_17 = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 20, i32 21, i32 22, i32 23>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_18_19_1A_1B = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 24, i32 25, i32 26, i32 27>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_1C_1D_1E_1F = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 28, i32 29, i32 30, i32 31>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_00_01_02_03_04_05_06_07 = shufflevector <32 x i8> %src256, <32 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_08_09_0A_0B_0C_0D_0E_0F = shufflevector <32 x i8> %src256, <32 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_10_11_12_13_14_15_16_17 = shufflevector <32 x i8> %src256, <32 x i8> undef, <8 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_18_19_1A_1B_1C_1D_1E_1F = shufflevector <32 x i8> %src256, <32 x i8> undef, <8 x i32> <i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_00_01_02_03_04_05_06_07_08_09_0A_0B_0C_0D_0E_0F = shufflevector <32 x i8> %src256, <32 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_10_11_12_13_14_15_16_17_18_19_1A_1B_1C_1D_1E_1F = shufflevector <32 x i8> %src256, <32 x i8> undef, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 0, i32 1>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_02_03 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 2, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_04_05 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 4, i32 5>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_06_07 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 6, i32 7>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_08_09 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 8, i32 9>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_0A_0B = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 10, i32 11>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_0C_0D = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 12, i32 13>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_0E_0F = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_10_11 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 16, i32 17>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_12_13 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 18, i32 19>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_14_15 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 20, i32 21>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_16_17 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 22, i32 23>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_18_19 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 24, i32 25>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_1A_1B = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 26, i32 27>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_1C_1D = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 28, i32 29>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_1E_1F = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 30, i32 31>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_20_21 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 32, i32 33>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_22_23 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 34, i32 35>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_24_25 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 36, i32 37>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_26_27 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 38, i32 39>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_28_29 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 40, i32 41>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_2A_2B = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 42, i32 43>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_2C_2D = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 44, i32 45>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_2E_2F = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 46, i32 47>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_30_31 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 48, i32 49>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_32_33 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 50, i32 51>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_34_35 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 52, i32 53>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_36_37 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 54, i32 55>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_38_39 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 56, i32 57>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_3A_3B = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 58, i32 59>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_3C_3D = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 60, i32 61>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_3E_3F = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 62, i32 63>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01_02_03 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_04_05_06_07 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_08_09_0A_0B = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_0C_0D_0E_0F = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_10_11_12_13 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_14_15_16_17 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 20, i32 21, i32 22, i32 23>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_18_19_1A_1B = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 24, i32 25, i32 26, i32 27>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_1C_1D_1E_1F = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 28, i32 29, i32 30, i32 31>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_20_21_22_23 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 32, i32 33, i32 34, i32 35>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_24_25_26_27 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 36, i32 37, i32 38, i32 39>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_28_29_2A_2B = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 40, i32 41, i32 42, i32 43>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_2C_2D_2E_2F = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 44, i32 45, i32 46, i32 47>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_30_31_32_33 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 48, i32 49, i32 50, i32 51>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_34_35_36_37 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 52, i32 53, i32 54, i32 55>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_38_39_3A_3B = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 56, i32 57, i32 58, i32 59>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_3C_3D_3E_3F = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 60, i32 61, i32 62, i32 63>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01_02_03_04_05_06_07 = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_08_09_0A_0B_0C_0D_0E_0F = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_10_11_12_13_14_15_16_17 = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_18_19_1A_1B_1C_1D_1E_1F = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_20_21_22_23_24_25_26_27 = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_28_29_2A_2B_2C_2D_2E_2F = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_30_31_32_33_34_35_36_37 = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_38_39_3A_3B_3C_3D_3E_3F = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01_02_03_04_05_06_07_08_09_0A_0B_0C_0D_0E_0F = shufflevector <64 x i8> %src512, <64 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_10_11_12_13_14_15_16_17_18_19_1A_1B_1C_1D_1E_1F = shufflevector <64 x i8> %src512, <64 x i8> undef, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_20_21_22_23_24_25_26_27_28_29_2A_2B_2C_2D_2E_2F = shufflevector <64 x i8> %src512, <64 x i8> undef, <16 x i32> <i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_30_31_32_33_34_35_36_37_38_39_3A_3B_3C_3D_3E_3F = shufflevector <64 x i8> %src512, <64 x i8> undef, <16 x i32> <i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01_02_03_04_05_06_07_08_09_0A_0B_0C_0D_0E_0F_10_11_12_13_14_15_16_17_18_19_1A_1B_1C_1D_1E_1F = shufflevector <64 x i8> %src512, <64 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_20_21_22_23_24_25_26_27_28_29_2A_2B_2C_2D_2E_2F_30_31_32_33_34_35_36_37_38_39_3A_3B_3C_3D_3E_3F = shufflevector <64 x i8> %src512, <64 x i8> undef, <32 x i32> <i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; SSSE3-LABEL: 'test_vXi8'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64_01 = shufflevector <8 x i8> %src64, <8 x i8> undef, <2 x i32> <i32 0, i32 1>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_23 = shufflevector <8 x i8> %src64, <8 x i8> undef, <2 x i32> <i32 2, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_45 = shufflevector <8 x i8> %src64, <8 x i8> undef, <2 x i32> <i32 4, i32 5>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_67 = shufflevector <8 x i8> %src64, <8 x i8> undef, <2 x i32> <i32 6, i32 7>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64_0123 = shufflevector <8 x i8> %src64, <8 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_4567 = shufflevector <8 x i8> %src64, <8 x i8> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128_01 = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 0, i32 1>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_23 = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 2, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_45 = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 4, i32 5>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_67 = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 6, i32 7>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_89 = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 8, i32 9>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_AB = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 10, i32 11>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_CD = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 12, i32 13>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_EF = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 14, i32 15>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128_0123 = shufflevector <16 x i8> %src128, <16 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128_2345 = shufflevector <16 x i8> %src128, <16 x i8> undef, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_4567 = shufflevector <16 x i8> %src128, <16 x i8> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128_6789 = shufflevector <16 x i8> %src128, <16 x i8> undef, <4 x i32> <i32 6, i32 7, i32 8, i32 9>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_89AB = shufflevector <16 x i8> %src128, <16 x i8> undef, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_CDEF = shufflevector <16 x i8> %src128, <16 x i8> undef, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128_01234567 = shufflevector <16 x i8> %src128, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_89ABCDEF = shufflevector <16 x i8> %src128, <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_00_01 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 0, i32 1>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_02_03 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 2, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_04_05 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 4, i32 5>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_06_07 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 6, i32 7>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_08_09 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 8, i32 9>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_0A_0B = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 10, i32 11>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_0C_0D = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 12, i32 13>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_0E_0F = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 14, i32 15>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_10_11 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 16, i32 17>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_12_13 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 18, i32 19>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_14_15 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 20, i32 21>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_16_17 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 22, i32 23>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_18_19 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 24, i32 25>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_1A_1B = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 26, i32 27>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_1C_1D = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 28, i32 29>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_1E_1F = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 30, i32 31>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_00_01_02_03 = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V256_02_03_04_05 = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_04_05_06_07 = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V256_06_07_08_09 = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 6, i32 7, i32 8, i32 9>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_08_09_0A_0B = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_0C_0D_0E_0F = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_10_11_12_13 = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_14_15_16_17 = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 20, i32 21, i32 22, i32 23>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_18_19_1A_1B = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 24, i32 25, i32 26, i32 27>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_1C_1D_1E_1F = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 28, i32 29, i32 30, i32 31>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_00_01_02_03_04_05_06_07 = shufflevector <32 x i8> %src256, <32 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_08_09_0A_0B_0C_0D_0E_0F = shufflevector <32 x i8> %src256, <32 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_10_11_12_13_14_15_16_17 = shufflevector <32 x i8> %src256, <32 x i8> undef, <8 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_18_19_1A_1B_1C_1D_1E_1F = shufflevector <32 x i8> %src256, <32 x i8> undef, <8 x i32> <i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_00_01_02_03_04_05_06_07_08_09_0A_0B_0C_0D_0E_0F = shufflevector <32 x i8> %src256, <32 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_10_11_12_13_14_15_16_17_18_19_1A_1B_1C_1D_1E_1F = shufflevector <32 x i8> %src256, <32 x i8> undef, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 0, i32 1>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_02_03 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 2, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_04_05 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 4, i32 5>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_06_07 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 6, i32 7>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_08_09 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 8, i32 9>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_0A_0B = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 10, i32 11>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_0C_0D = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 12, i32 13>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_0E_0F = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 14, i32 15>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_10_11 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 16, i32 17>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_12_13 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 18, i32 19>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_14_15 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 20, i32 21>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_16_17 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 22, i32 23>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_18_19 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 24, i32 25>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_1A_1B = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 26, i32 27>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_1C_1D = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 28, i32 29>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_1E_1F = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 30, i32 31>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_20_21 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 32, i32 33>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_22_23 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 34, i32 35>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_24_25 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 36, i32 37>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_26_27 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 38, i32 39>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_28_29 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 40, i32 41>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_2A_2B = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 42, i32 43>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_2C_2D = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 44, i32 45>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_2E_2F = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 46, i32 47>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_30_31 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 48, i32 49>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_32_33 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 50, i32 51>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_34_35 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 52, i32 53>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_36_37 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 54, i32 55>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_38_39 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 56, i32 57>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_3A_3B = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 58, i32 59>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_3C_3D = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 60, i32 61>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_3E_3F = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 62, i32 63>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01_02_03 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_04_05_06_07 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_08_09_0A_0B = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_0C_0D_0E_0F = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_10_11_12_13 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_14_15_16_17 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 20, i32 21, i32 22, i32 23>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_18_19_1A_1B = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 24, i32 25, i32 26, i32 27>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_1C_1D_1E_1F = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 28, i32 29, i32 30, i32 31>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_20_21_22_23 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 32, i32 33, i32 34, i32 35>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_24_25_26_27 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 36, i32 37, i32 38, i32 39>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_28_29_2A_2B = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 40, i32 41, i32 42, i32 43>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_2C_2D_2E_2F = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 44, i32 45, i32 46, i32 47>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_30_31_32_33 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 48, i32 49, i32 50, i32 51>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_34_35_36_37 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 52, i32 53, i32 54, i32 55>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_38_39_3A_3B = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 56, i32 57, i32 58, i32 59>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_3C_3D_3E_3F = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 60, i32 61, i32 62, i32 63>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01_02_03_04_05_06_07 = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_08_09_0A_0B_0C_0D_0E_0F = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_10_11_12_13_14_15_16_17 = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_18_19_1A_1B_1C_1D_1E_1F = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_20_21_22_23_24_25_26_27 = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_28_29_2A_2B_2C_2D_2E_2F = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_30_31_32_33_34_35_36_37 = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_38_39_3A_3B_3C_3D_3E_3F = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01_02_03_04_05_06_07_08_09_0A_0B_0C_0D_0E_0F = shufflevector <64 x i8> %src512, <64 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_10_11_12_13_14_15_16_17_18_19_1A_1B_1C_1D_1E_1F = shufflevector <64 x i8> %src512, <64 x i8> undef, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_20_21_22_23_24_25_26_27_28_29_2A_2B_2C_2D_2E_2F = shufflevector <64 x i8> %src512, <64 x i8> undef, <16 x i32> <i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_30_31_32_33_34_35_36_37_38_39_3A_3B_3C_3D_3E_3F = shufflevector <64 x i8> %src512, <64 x i8> undef, <16 x i32> <i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01_02_03_04_05_06_07_08_09_0A_0B_0C_0D_0E_0F_10_11_12_13_14_15_16_17_18_19_1A_1B_1C_1D_1E_1F = shufflevector <64 x i8> %src512, <64 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_20_21_22_23_24_25_26_27_28_29_2A_2B_2C_2D_2E_2F_30_31_32_33_34_35_36_37_38_39_3A_3B_3C_3D_3E_3F = shufflevector <64 x i8> %src512, <64 x i8> undef, <32 x i32> <i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX-LABEL: 'test_vXi8'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64_01 = shufflevector <8 x i8> %src64, <8 x i8> undef, <2 x i32> <i32 0, i32 1>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_23 = shufflevector <8 x i8> %src64, <8 x i8> undef, <2 x i32> <i32 2, i32 3>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_45 = shufflevector <8 x i8> %src64, <8 x i8> undef, <2 x i32> <i32 4, i32 5>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_67 = shufflevector <8 x i8> %src64, <8 x i8> undef, <2 x i32> <i32 6, i32 7>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64_0123 = shufflevector <8 x i8> %src64, <8 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_4567 = shufflevector <8 x i8> %src64, <8 x i8> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128_01 = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 0, i32 1>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_23 = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 2, i32 3>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_45 = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 4, i32 5>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_67 = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 6, i32 7>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_89 = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 8, i32 9>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_AB = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 10, i32 11>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_CD = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 12, i32 13>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_EF = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 14, i32 15>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128_0123 = shufflevector <16 x i8> %src128, <16 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128_2345 = shufflevector <16 x i8> %src128, <16 x i8> undef, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_4567 = shufflevector <16 x i8> %src128, <16 x i8> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128_6789 = shufflevector <16 x i8> %src128, <16 x i8> undef, <4 x i32> <i32 6, i32 7, i32 8, i32 9>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_89AB = shufflevector <16 x i8> %src128, <16 x i8> undef, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_CDEF = shufflevector <16 x i8> %src128, <16 x i8> undef, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128_01234567 = shufflevector <16 x i8> %src128, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_89ABCDEF = shufflevector <16 x i8> %src128, <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_00_01 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 0, i32 1>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_02_03 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 2, i32 3>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_04_05 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 4, i32 5>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_06_07 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 6, i32 7>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_08_09 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 8, i32 9>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_0A_0B = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 10, i32 11>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_0C_0D = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 12, i32 13>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_0E_0F = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 14, i32 15>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_10_11 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 16, i32 17>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_12_13 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 18, i32 19>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_14_15 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 20, i32 21>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_16_17 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 22, i32 23>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_18_19 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 24, i32 25>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_1A_1B = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 26, i32 27>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_1C_1D = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 28, i32 29>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_1E_1F = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 30, i32 31>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_00_01_02_03 = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V256_02_03_04_05 = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_04_05_06_07 = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V256_06_07_08_09 = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 6, i32 7, i32 8, i32 9>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_08_09_0A_0B = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_0C_0D_0E_0F = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_10_11_12_13 = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_14_15_16_17 = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 20, i32 21, i32 22, i32 23>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_18_19_1A_1B = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 24, i32 25, i32 26, i32 27>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_1C_1D_1E_1F = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 28, i32 29, i32 30, i32 31>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_00_01_02_03_04_05_06_07 = shufflevector <32 x i8> %src256, <32 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_08_09_0A_0B_0C_0D_0E_0F = shufflevector <32 x i8> %src256, <32 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_10_11_12_13_14_15_16_17 = shufflevector <32 x i8> %src256, <32 x i8> undef, <8 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_18_19_1A_1B_1C_1D_1E_1F = shufflevector <32 x i8> %src256, <32 x i8> undef, <8 x i32> <i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_00_01_02_03_04_05_06_07_08_09_0A_0B_0C_0D_0E_0F = shufflevector <32 x i8> %src256, <32 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_10_11_12_13_14_15_16_17_18_19_1A_1B_1C_1D_1E_1F = shufflevector <32 x i8> %src256, <32 x i8> undef, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 0, i32 1>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_02_03 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 2, i32 3>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_04_05 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 4, i32 5>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_06_07 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 6, i32 7>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_08_09 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 8, i32 9>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_0A_0B = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 10, i32 11>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_0C_0D = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 12, i32 13>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_0E_0F = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 14, i32 15>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_10_11 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 16, i32 17>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_12_13 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 18, i32 19>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_14_15 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 20, i32 21>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_16_17 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 22, i32 23>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_18_19 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 24, i32 25>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_1A_1B = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 26, i32 27>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_1C_1D = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 28, i32 29>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_1E_1F = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 30, i32 31>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_20_21 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 32, i32 33>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_22_23 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 34, i32 35>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_24_25 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 36, i32 37>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_26_27 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 38, i32 39>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_28_29 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 40, i32 41>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_2A_2B = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 42, i32 43>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_2C_2D = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 44, i32 45>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_2E_2F = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 46, i32 47>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_30_31 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 48, i32 49>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_32_33 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 50, i32 51>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_34_35 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 52, i32 53>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_36_37 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 54, i32 55>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_38_39 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 56, i32 57>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_3A_3B = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 58, i32 59>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_3C_3D = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 60, i32 61>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_3E_3F = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 62, i32 63>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01_02_03 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_04_05_06_07 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_08_09_0A_0B = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_0C_0D_0E_0F = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_10_11_12_13 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_14_15_16_17 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 20, i32 21, i32 22, i32 23>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_18_19_1A_1B = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 24, i32 25, i32 26, i32 27>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_1C_1D_1E_1F = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 28, i32 29, i32 30, i32 31>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_20_21_22_23 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 32, i32 33, i32 34, i32 35>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_24_25_26_27 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 36, i32 37, i32 38, i32 39>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_28_29_2A_2B = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 40, i32 41, i32 42, i32 43>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_2C_2D_2E_2F = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 44, i32 45, i32 46, i32 47>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_30_31_32_33 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 48, i32 49, i32 50, i32 51>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_34_35_36_37 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 52, i32 53, i32 54, i32 55>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_38_39_3A_3B = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 56, i32 57, i32 58, i32 59>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_3C_3D_3E_3F = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 60, i32 61, i32 62, i32 63>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01_02_03_04_05_06_07 = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_08_09_0A_0B_0C_0D_0E_0F = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_10_11_12_13_14_15_16_17 = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_18_19_1A_1B_1C_1D_1E_1F = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_20_21_22_23_24_25_26_27 = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_28_29_2A_2B_2C_2D_2E_2F = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_30_31_32_33_34_35_36_37 = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_38_39_3A_3B_3C_3D_3E_3F = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01_02_03_04_05_06_07_08_09_0A_0B_0C_0D_0E_0F = shufflevector <64 x i8> %src512, <64 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_10_11_12_13_14_15_16_17_18_19_1A_1B_1C_1D_1E_1F = shufflevector <64 x i8> %src512, <64 x i8> undef, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_20_21_22_23_24_25_26_27_28_29_2A_2B_2C_2D_2E_2F = shufflevector <64 x i8> %src512, <64 x i8> undef, <16 x i32> <i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_30_31_32_33_34_35_36_37_38_39_3A_3B_3C_3D_3E_3F = shufflevector <64 x i8> %src512, <64 x i8> undef, <16 x i32> <i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01_02_03_04_05_06_07_08_09_0A_0B_0C_0D_0E_0F_10_11_12_13_14_15_16_17_18_19_1A_1B_1C_1D_1E_1F = shufflevector <64 x i8> %src512, <64 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_20_21_22_23_24_25_26_27_28_29_2A_2B_2C_2D_2E_2F_30_31_32_33_34_35_36_37_38_39_3A_3B_3C_3D_3E_3F = shufflevector <64 x i8> %src512, <64 x i8> undef, <32 x i32> <i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX512F-LABEL: 'test_vXi8'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64_01 = shufflevector <8 x i8> %src64, <8 x i8> undef, <2 x i32> <i32 0, i32 1>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_23 = shufflevector <8 x i8> %src64, <8 x i8> undef, <2 x i32> <i32 2, i32 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_45 = shufflevector <8 x i8> %src64, <8 x i8> undef, <2 x i32> <i32 4, i32 5>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_67 = shufflevector <8 x i8> %src64, <8 x i8> undef, <2 x i32> <i32 6, i32 7>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64_0123 = shufflevector <8 x i8> %src64, <8 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_4567 = shufflevector <8 x i8> %src64, <8 x i8> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128_01 = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 0, i32 1>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_23 = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 2, i32 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_45 = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 4, i32 5>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_67 = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 6, i32 7>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_89 = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 8, i32 9>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_AB = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 10, i32 11>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_CD = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 12, i32 13>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_EF = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 14, i32 15>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128_0123 = shufflevector <16 x i8> %src128, <16 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128_2345 = shufflevector <16 x i8> %src128, <16 x i8> undef, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_4567 = shufflevector <16 x i8> %src128, <16 x i8> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128_6789 = shufflevector <16 x i8> %src128, <16 x i8> undef, <4 x i32> <i32 6, i32 7, i32 8, i32 9>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_89AB = shufflevector <16 x i8> %src128, <16 x i8> undef, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_CDEF = shufflevector <16 x i8> %src128, <16 x i8> undef, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128_01234567 = shufflevector <16 x i8> %src128, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_89ABCDEF = shufflevector <16 x i8> %src128, <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_00_01 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 0, i32 1>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_02_03 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 2, i32 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_04_05 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 4, i32 5>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_06_07 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 6, i32 7>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_08_09 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 8, i32 9>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_0A_0B = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 10, i32 11>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_0C_0D = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 12, i32 13>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_0E_0F = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 14, i32 15>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_10_11 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 16, i32 17>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_12_13 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 18, i32 19>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_14_15 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 20, i32 21>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_16_17 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 22, i32 23>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_18_19 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 24, i32 25>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_1A_1B = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 26, i32 27>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_1C_1D = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 28, i32 29>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_1E_1F = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 30, i32 31>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_00_01_02_03 = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V256_02_03_04_05 = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_04_05_06_07 = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V256_06_07_08_09 = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 6, i32 7, i32 8, i32 9>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_08_09_0A_0B = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_0C_0D_0E_0F = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_10_11_12_13 = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_14_15_16_17 = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 20, i32 21, i32 22, i32 23>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_18_19_1A_1B = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 24, i32 25, i32 26, i32 27>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_1C_1D_1E_1F = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 28, i32 29, i32 30, i32 31>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_00_01_02_03_04_05_06_07 = shufflevector <32 x i8> %src256, <32 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_08_09_0A_0B_0C_0D_0E_0F = shufflevector <32 x i8> %src256, <32 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_10_11_12_13_14_15_16_17 = shufflevector <32 x i8> %src256, <32 x i8> undef, <8 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_18_19_1A_1B_1C_1D_1E_1F = shufflevector <32 x i8> %src256, <32 x i8> undef, <8 x i32> <i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_00_01_02_03_04_05_06_07_08_09_0A_0B_0C_0D_0E_0F = shufflevector <32 x i8> %src256, <32 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_10_11_12_13_14_15_16_17_18_19_1A_1B_1C_1D_1E_1F = shufflevector <32 x i8> %src256, <32 x i8> undef, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 0, i32 1>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_02_03 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 2, i32 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_04_05 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 4, i32 5>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_06_07 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 6, i32 7>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_08_09 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 8, i32 9>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_0A_0B = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 10, i32 11>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_0C_0D = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 12, i32 13>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_0E_0F = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 14, i32 15>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_10_11 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 16, i32 17>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_12_13 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 18, i32 19>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_14_15 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 20, i32 21>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_16_17 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 22, i32 23>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_18_19 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 24, i32 25>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_1A_1B = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 26, i32 27>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_1C_1D = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 28, i32 29>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_1E_1F = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 30, i32 31>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_20_21 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 32, i32 33>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_22_23 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 34, i32 35>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_24_25 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 36, i32 37>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_26_27 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 38, i32 39>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_28_29 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 40, i32 41>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_2A_2B = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 42, i32 43>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_2C_2D = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 44, i32 45>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_2E_2F = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 46, i32 47>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_30_31 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 48, i32 49>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_32_33 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 50, i32 51>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_34_35 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 52, i32 53>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_36_37 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 54, i32 55>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_38_39 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 56, i32 57>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_3A_3B = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 58, i32 59>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_3C_3D = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 60, i32 61>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_3E_3F = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 62, i32 63>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01_02_03 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_04_05_06_07 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_08_09_0A_0B = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_0C_0D_0E_0F = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_10_11_12_13 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_14_15_16_17 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 20, i32 21, i32 22, i32 23>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_18_19_1A_1B = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 24, i32 25, i32 26, i32 27>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_1C_1D_1E_1F = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 28, i32 29, i32 30, i32 31>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_20_21_22_23 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 32, i32 33, i32 34, i32 35>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_24_25_26_27 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 36, i32 37, i32 38, i32 39>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_28_29_2A_2B = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 40, i32 41, i32 42, i32 43>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_2C_2D_2E_2F = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 44, i32 45, i32 46, i32 47>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_30_31_32_33 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 48, i32 49, i32 50, i32 51>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_34_35_36_37 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 52, i32 53, i32 54, i32 55>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_38_39_3A_3B = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 56, i32 57, i32 58, i32 59>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_3C_3D_3E_3F = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 60, i32 61, i32 62, i32 63>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01_02_03_04_05_06_07 = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_08_09_0A_0B_0C_0D_0E_0F = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_10_11_12_13_14_15_16_17 = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_18_19_1A_1B_1C_1D_1E_1F = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_20_21_22_23_24_25_26_27 = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_28_29_2A_2B_2C_2D_2E_2F = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_30_31_32_33_34_35_36_37 = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_38_39_3A_3B_3C_3D_3E_3F = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01_02_03_04_05_06_07_08_09_0A_0B_0C_0D_0E_0F = shufflevector <64 x i8> %src512, <64 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_10_11_12_13_14_15_16_17_18_19_1A_1B_1C_1D_1E_1F = shufflevector <64 x i8> %src512, <64 x i8> undef, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_20_21_22_23_24_25_26_27_28_29_2A_2B_2C_2D_2E_2F = shufflevector <64 x i8> %src512, <64 x i8> undef, <16 x i32> <i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_30_31_32_33_34_35_36_37_38_39_3A_3B_3C_3D_3E_3F = shufflevector <64 x i8> %src512, <64 x i8> undef, <16 x i32> <i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01_02_03_04_05_06_07_08_09_0A_0B_0C_0D_0E_0F_10_11_12_13_14_15_16_17_18_19_1A_1B_1C_1D_1E_1F = shufflevector <64 x i8> %src512, <64 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_20_21_22_23_24_25_26_27_28_29_2A_2B_2C_2D_2E_2F_30_31_32_33_34_35_36_37_38_39_3A_3B_3C_3D_3E_3F = shufflevector <64 x i8> %src512, <64 x i8> undef, <32 x i32> <i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX512BW-LABEL: 'test_vXi8'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64_01 = shufflevector <8 x i8> %src64, <8 x i8> undef, <2 x i32> <i32 0, i32 1>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_23 = shufflevector <8 x i8> %src64, <8 x i8> undef, <2 x i32> <i32 2, i32 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_45 = shufflevector <8 x i8> %src64, <8 x i8> undef, <2 x i32> <i32 4, i32 5>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_67 = shufflevector <8 x i8> %src64, <8 x i8> undef, <2 x i32> <i32 6, i32 7>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64_0123 = shufflevector <8 x i8> %src64, <8 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_4567 = shufflevector <8 x i8> %src64, <8 x i8> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128_01 = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 0, i32 1>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_23 = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 2, i32 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_45 = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 4, i32 5>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_67 = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 6, i32 7>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_89 = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 8, i32 9>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_AB = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 10, i32 11>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_CD = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 12, i32 13>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_EF = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 14, i32 15>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128_0123 = shufflevector <16 x i8> %src128, <16 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128_2345 = shufflevector <16 x i8> %src128, <16 x i8> undef, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_4567 = shufflevector <16 x i8> %src128, <16 x i8> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128_6789 = shufflevector <16 x i8> %src128, <16 x i8> undef, <4 x i32> <i32 6, i32 7, i32 8, i32 9>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_89AB = shufflevector <16 x i8> %src128, <16 x i8> undef, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_CDEF = shufflevector <16 x i8> %src128, <16 x i8> undef, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128_01234567 = shufflevector <16 x i8> %src128, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_89ABCDEF = shufflevector <16 x i8> %src128, <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_00_01 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 0, i32 1>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_02_03 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 2, i32 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_04_05 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 4, i32 5>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_06_07 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 6, i32 7>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_08_09 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 8, i32 9>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_0A_0B = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 10, i32 11>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_0C_0D = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 12, i32 13>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_0E_0F = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 14, i32 15>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_10_11 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 16, i32 17>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_12_13 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 18, i32 19>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_14_15 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 20, i32 21>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_16_17 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 22, i32 23>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_18_19 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 24, i32 25>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_1A_1B = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 26, i32 27>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_1C_1D = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 28, i32 29>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_1E_1F = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 30, i32 31>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_00_01_02_03 = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V256_02_03_04_05 = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_04_05_06_07 = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V256_06_07_08_09 = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 6, i32 7, i32 8, i32 9>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_08_09_0A_0B = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_0C_0D_0E_0F = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_10_11_12_13 = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_14_15_16_17 = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 20, i32 21, i32 22, i32 23>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_18_19_1A_1B = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 24, i32 25, i32 26, i32 27>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_1C_1D_1E_1F = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 28, i32 29, i32 30, i32 31>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_00_01_02_03_04_05_06_07 = shufflevector <32 x i8> %src256, <32 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_08_09_0A_0B_0C_0D_0E_0F = shufflevector <32 x i8> %src256, <32 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_10_11_12_13_14_15_16_17 = shufflevector <32 x i8> %src256, <32 x i8> undef, <8 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_18_19_1A_1B_1C_1D_1E_1F = shufflevector <32 x i8> %src256, <32 x i8> undef, <8 x i32> <i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_00_01_02_03_04_05_06_07_08_09_0A_0B_0C_0D_0E_0F = shufflevector <32 x i8> %src256, <32 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_10_11_12_13_14_15_16_17_18_19_1A_1B_1C_1D_1E_1F = shufflevector <32 x i8> %src256, <32 x i8> undef, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 0, i32 1>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_02_03 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 2, i32 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_04_05 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 4, i32 5>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_06_07 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 6, i32 7>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_08_09 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 8, i32 9>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_0A_0B = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 10, i32 11>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_0C_0D = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 12, i32 13>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_0E_0F = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 14, i32 15>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_10_11 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 16, i32 17>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_12_13 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 18, i32 19>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_14_15 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 20, i32 21>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_16_17 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 22, i32 23>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_18_19 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 24, i32 25>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_1A_1B = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 26, i32 27>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_1C_1D = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 28, i32 29>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_1E_1F = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 30, i32 31>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_20_21 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 32, i32 33>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_22_23 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 34, i32 35>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_24_25 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 36, i32 37>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_26_27 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 38, i32 39>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_28_29 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 40, i32 41>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_2A_2B = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 42, i32 43>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_2C_2D = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 44, i32 45>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_2E_2F = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 46, i32 47>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_30_31 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 48, i32 49>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_32_33 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 50, i32 51>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_34_35 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 52, i32 53>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_36_37 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 54, i32 55>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_38_39 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 56, i32 57>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_3A_3B = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 58, i32 59>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_3C_3D = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 60, i32 61>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_3E_3F = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 62, i32 63>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01_02_03 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_04_05_06_07 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_08_09_0A_0B = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_0C_0D_0E_0F = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_10_11_12_13 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_14_15_16_17 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 20, i32 21, i32 22, i32 23>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_18_19_1A_1B = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 24, i32 25, i32 26, i32 27>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_1C_1D_1E_1F = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 28, i32 29, i32 30, i32 31>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_20_21_22_23 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 32, i32 33, i32 34, i32 35>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_24_25_26_27 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 36, i32 37, i32 38, i32 39>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_28_29_2A_2B = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 40, i32 41, i32 42, i32 43>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_2C_2D_2E_2F = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 44, i32 45, i32 46, i32 47>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_30_31_32_33 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 48, i32 49, i32 50, i32 51>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_34_35_36_37 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 52, i32 53, i32 54, i32 55>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_38_39_3A_3B = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 56, i32 57, i32 58, i32 59>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_3C_3D_3E_3F = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 60, i32 61, i32 62, i32 63>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01_02_03_04_05_06_07 = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_08_09_0A_0B_0C_0D_0E_0F = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_10_11_12_13_14_15_16_17 = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_18_19_1A_1B_1C_1D_1E_1F = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_20_21_22_23_24_25_26_27 = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_28_29_2A_2B_2C_2D_2E_2F = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_30_31_32_33_34_35_36_37 = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_38_39_3A_3B_3C_3D_3E_3F = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01_02_03_04_05_06_07_08_09_0A_0B_0C_0D_0E_0F = shufflevector <64 x i8> %src512, <64 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_10_11_12_13_14_15_16_17_18_19_1A_1B_1C_1D_1E_1F = shufflevector <64 x i8> %src512, <64 x i8> undef, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_20_21_22_23_24_25_26_27_28_29_2A_2B_2C_2D_2E_2F = shufflevector <64 x i8> %src512, <64 x i8> undef, <16 x i32> <i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_30_31_32_33_34_35_36_37_38_39_3A_3B_3C_3D_3E_3F = shufflevector <64 x i8> %src512, <64 x i8> undef, <16 x i32> <i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01_02_03_04_05_06_07_08_09_0A_0B_0C_0D_0E_0F_10_11_12_13_14_15_16_17_18_19_1A_1B_1C_1D_1E_1F = shufflevector <64 x i8> %src512, <64 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_20_21_22_23_24_25_26_27_28_29_2A_2B_2C_2D_2E_2F_30_31_32_33_34_35_36_37_38_39_3A_3B_3C_3D_3E_3F = shufflevector <64 x i8> %src512, <64 x i8> undef, <32 x i32> <i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; SLM-LABEL: 'test_vXi8'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64_01 = shufflevector <8 x i8> %src64, <8 x i8> undef, <2 x i32> <i32 0, i32 1>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_23 = shufflevector <8 x i8> %src64, <8 x i8> undef, <2 x i32> <i32 2, i32 3>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_45 = shufflevector <8 x i8> %src64, <8 x i8> undef, <2 x i32> <i32 4, i32 5>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_67 = shufflevector <8 x i8> %src64, <8 x i8> undef, <2 x i32> <i32 6, i32 7>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64_0123 = shufflevector <8 x i8> %src64, <8 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_4567 = shufflevector <8 x i8> %src64, <8 x i8> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128_01 = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 0, i32 1>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_23 = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 2, i32 3>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_45 = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 4, i32 5>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_67 = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 6, i32 7>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_89 = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 8, i32 9>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_AB = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 10, i32 11>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_CD = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 12, i32 13>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_EF = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 14, i32 15>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128_0123 = shufflevector <16 x i8> %src128, <16 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V128_2345 = shufflevector <16 x i8> %src128, <16 x i8> undef, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_4567 = shufflevector <16 x i8> %src128, <16 x i8> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V128_6789 = shufflevector <16 x i8> %src128, <16 x i8> undef, <4 x i32> <i32 6, i32 7, i32 8, i32 9>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_89AB = shufflevector <16 x i8> %src128, <16 x i8> undef, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_CDEF = shufflevector <16 x i8> %src128, <16 x i8> undef, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128_01234567 = shufflevector <16 x i8> %src128, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_89ABCDEF = shufflevector <16 x i8> %src128, <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_00_01 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 0, i32 1>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_02_03 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 2, i32 3>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_04_05 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 4, i32 5>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_06_07 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 6, i32 7>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_08_09 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 8, i32 9>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_0A_0B = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 10, i32 11>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_0C_0D = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 12, i32 13>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_0E_0F = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 14, i32 15>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_10_11 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 16, i32 17>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_12_13 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 18, i32 19>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_14_15 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 20, i32 21>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_16_17 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 22, i32 23>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_18_19 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 24, i32 25>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_1A_1B = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 26, i32 27>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_1C_1D = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 28, i32 29>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_1E_1F = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 30, i32 31>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_00_01_02_03 = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V256_02_03_04_05 = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_04_05_06_07 = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V256_06_07_08_09 = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 6, i32 7, i32 8, i32 9>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_08_09_0A_0B = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_0C_0D_0E_0F = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_10_11_12_13 = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_14_15_16_17 = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 20, i32 21, i32 22, i32 23>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_18_19_1A_1B = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 24, i32 25, i32 26, i32 27>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_1C_1D_1E_1F = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 28, i32 29, i32 30, i32 31>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_00_01_02_03_04_05_06_07 = shufflevector <32 x i8> %src256, <32 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_08_09_0A_0B_0C_0D_0E_0F = shufflevector <32 x i8> %src256, <32 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_10_11_12_13_14_15_16_17 = shufflevector <32 x i8> %src256, <32 x i8> undef, <8 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_18_19_1A_1B_1C_1D_1E_1F = shufflevector <32 x i8> %src256, <32 x i8> undef, <8 x i32> <i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_00_01_02_03_04_05_06_07_08_09_0A_0B_0C_0D_0E_0F = shufflevector <32 x i8> %src256, <32 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_10_11_12_13_14_15_16_17_18_19_1A_1B_1C_1D_1E_1F = shufflevector <32 x i8> %src256, <32 x i8> undef, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 0, i32 1>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_02_03 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 2, i32 3>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_04_05 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 4, i32 5>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_06_07 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 6, i32 7>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_08_09 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 8, i32 9>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_0A_0B = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 10, i32 11>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_0C_0D = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 12, i32 13>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_0E_0F = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 14, i32 15>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_10_11 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 16, i32 17>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_12_13 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 18, i32 19>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_14_15 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 20, i32 21>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_16_17 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 22, i32 23>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_18_19 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 24, i32 25>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_1A_1B = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 26, i32 27>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_1C_1D = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 28, i32 29>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_1E_1F = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 30, i32 31>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_20_21 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 32, i32 33>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_22_23 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 34, i32 35>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_24_25 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 36, i32 37>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_26_27 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 38, i32 39>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_28_29 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 40, i32 41>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_2A_2B = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 42, i32 43>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_2C_2D = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 44, i32 45>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_2E_2F = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 46, i32 47>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_30_31 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 48, i32 49>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_32_33 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 50, i32 51>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_34_35 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 52, i32 53>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_36_37 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 54, i32 55>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_38_39 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 56, i32 57>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_3A_3B = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 58, i32 59>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_3C_3D = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 60, i32 61>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_3E_3F = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 62, i32 63>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01_02_03 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_04_05_06_07 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_08_09_0A_0B = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_0C_0D_0E_0F = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_10_11_12_13 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_14_15_16_17 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 20, i32 21, i32 22, i32 23>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_18_19_1A_1B = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 24, i32 25, i32 26, i32 27>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_1C_1D_1E_1F = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 28, i32 29, i32 30, i32 31>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_20_21_22_23 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 32, i32 33, i32 34, i32 35>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_24_25_26_27 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 36, i32 37, i32 38, i32 39>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_28_29_2A_2B = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 40, i32 41, i32 42, i32 43>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_2C_2D_2E_2F = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 44, i32 45, i32 46, i32 47>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_30_31_32_33 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 48, i32 49, i32 50, i32 51>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_34_35_36_37 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 52, i32 53, i32 54, i32 55>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_38_39_3A_3B = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 56, i32 57, i32 58, i32 59>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_3C_3D_3E_3F = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 60, i32 61, i32 62, i32 63>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01_02_03_04_05_06_07 = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_08_09_0A_0B_0C_0D_0E_0F = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_10_11_12_13_14_15_16_17 = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_18_19_1A_1B_1C_1D_1E_1F = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_20_21_22_23_24_25_26_27 = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_28_29_2A_2B_2C_2D_2E_2F = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_30_31_32_33_34_35_36_37 = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_38_39_3A_3B_3C_3D_3E_3F = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01_02_03_04_05_06_07_08_09_0A_0B_0C_0D_0E_0F = shufflevector <64 x i8> %src512, <64 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_10_11_12_13_14_15_16_17_18_19_1A_1B_1C_1D_1E_1F = shufflevector <64 x i8> %src512, <64 x i8> undef, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_20_21_22_23_24_25_26_27_28_29_2A_2B_2C_2D_2E_2F = shufflevector <64 x i8> %src512, <64 x i8> undef, <16 x i32> <i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_30_31_32_33_34_35_36_37_38_39_3A_3B_3C_3D_3E_3F = shufflevector <64 x i8> %src512, <64 x i8> undef, <16 x i32> <i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01_02_03_04_05_06_07_08_09_0A_0B_0C_0D_0E_0F_10_11_12_13_14_15_16_17_18_19_1A_1B_1C_1D_1E_1F = shufflevector <64 x i8> %src512, <64 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_20_21_22_23_24_25_26_27_28_29_2A_2B_2C_2D_2E_2F_30_31_32_33_34_35_36_37_38_39_3A_3B_3C_3D_3E_3F = shufflevector <64 x i8> %src512, <64 x i8> undef, <32 x i32> <i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; GLM-LABEL: 'test_vXi8'
+; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64_01 = shufflevector <8 x i8> %src64, <8 x i8> undef, <2 x i32> <i32 0, i32 1>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_23 = shufflevector <8 x i8> %src64, <8 x i8> undef, <2 x i32> <i32 2, i32 3>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_45 = shufflevector <8 x i8> %src64, <8 x i8> undef, <2 x i32> <i32 4, i32 5>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_67 = shufflevector <8 x i8> %src64, <8 x i8> undef, <2 x i32> <i32 6, i32 7>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64_0123 = shufflevector <8 x i8> %src64, <8 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_4567 = shufflevector <8 x i8> %src64, <8 x i8> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128_01 = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 0, i32 1>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_23 = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 2, i32 3>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_45 = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 4, i32 5>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_67 = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 6, i32 7>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_89 = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 8, i32 9>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_AB = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 10, i32 11>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_CD = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 12, i32 13>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_EF = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 14, i32 15>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128_0123 = shufflevector <16 x i8> %src128, <16 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128_2345 = shufflevector <16 x i8> %src128, <16 x i8> undef, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_4567 = shufflevector <16 x i8> %src128, <16 x i8> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128_6789 = shufflevector <16 x i8> %src128, <16 x i8> undef, <4 x i32> <i32 6, i32 7, i32 8, i32 9>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_89AB = shufflevector <16 x i8> %src128, <16 x i8> undef, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_CDEF = shufflevector <16 x i8> %src128, <16 x i8> undef, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128_01234567 = shufflevector <16 x i8> %src128, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_89ABCDEF = shufflevector <16 x i8> %src128, <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_00_01 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 0, i32 1>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_02_03 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 2, i32 3>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_04_05 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 4, i32 5>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_06_07 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 6, i32 7>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_08_09 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 8, i32 9>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_0A_0B = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 10, i32 11>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_0C_0D = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 12, i32 13>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_0E_0F = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 14, i32 15>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_10_11 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 16, i32 17>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_12_13 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 18, i32 19>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_14_15 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 20, i32 21>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_16_17 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 22, i32 23>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_18_19 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 24, i32 25>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_1A_1B = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 26, i32 27>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_1C_1D = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 28, i32 29>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_1E_1F = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 30, i32 31>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_00_01_02_03 = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V256_02_03_04_05 = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_04_05_06_07 = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V256_06_07_08_09 = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 6, i32 7, i32 8, i32 9>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_08_09_0A_0B = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_0C_0D_0E_0F = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_10_11_12_13 = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_14_15_16_17 = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 20, i32 21, i32 22, i32 23>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_18_19_1A_1B = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 24, i32 25, i32 26, i32 27>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_1C_1D_1E_1F = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 28, i32 29, i32 30, i32 31>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_00_01_02_03_04_05_06_07 = shufflevector <32 x i8> %src256, <32 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_08_09_0A_0B_0C_0D_0E_0F = shufflevector <32 x i8> %src256, <32 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_10_11_12_13_14_15_16_17 = shufflevector <32 x i8> %src256, <32 x i8> undef, <8 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_18_19_1A_1B_1C_1D_1E_1F = shufflevector <32 x i8> %src256, <32 x i8> undef, <8 x i32> <i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_00_01_02_03_04_05_06_07_08_09_0A_0B_0C_0D_0E_0F = shufflevector <32 x i8> %src256, <32 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_10_11_12_13_14_15_16_17_18_19_1A_1B_1C_1D_1E_1F = shufflevector <32 x i8> %src256, <32 x i8> undef, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 0, i32 1>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_02_03 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 2, i32 3>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_04_05 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 4, i32 5>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_06_07 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 6, i32 7>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_08_09 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 8, i32 9>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_0A_0B = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 10, i32 11>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_0C_0D = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 12, i32 13>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_0E_0F = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 14, i32 15>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_10_11 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 16, i32 17>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_12_13 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 18, i32 19>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_14_15 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 20, i32 21>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_16_17 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 22, i32 23>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_18_19 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 24, i32 25>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_1A_1B = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 26, i32 27>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_1C_1D = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 28, i32 29>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_1E_1F = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 30, i32 31>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_20_21 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 32, i32 33>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_22_23 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 34, i32 35>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_24_25 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 36, i32 37>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_26_27 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 38, i32 39>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_28_29 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 40, i32 41>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_2A_2B = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 42, i32 43>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_2C_2D = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 44, i32 45>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_2E_2F = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 46, i32 47>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_30_31 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 48, i32 49>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_32_33 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 50, i32 51>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_34_35 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 52, i32 53>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_36_37 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 54, i32 55>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_38_39 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 56, i32 57>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_3A_3B = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 58, i32 59>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_3C_3D = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 60, i32 61>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_3E_3F = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 62, i32 63>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01_02_03 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_04_05_06_07 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_08_09_0A_0B = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_0C_0D_0E_0F = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_10_11_12_13 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_14_15_16_17 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 20, i32 21, i32 22, i32 23>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_18_19_1A_1B = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 24, i32 25, i32 26, i32 27>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_1C_1D_1E_1F = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 28, i32 29, i32 30, i32 31>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_20_21_22_23 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 32, i32 33, i32 34, i32 35>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_24_25_26_27 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 36, i32 37, i32 38, i32 39>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_28_29_2A_2B = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 40, i32 41, i32 42, i32 43>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_2C_2D_2E_2F = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 44, i32 45, i32 46, i32 47>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_30_31_32_33 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 48, i32 49, i32 50, i32 51>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_34_35_36_37 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 52, i32 53, i32 54, i32 55>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_38_39_3A_3B = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 56, i32 57, i32 58, i32 59>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_3C_3D_3E_3F = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 60, i32 61, i32 62, i32 63>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01_02_03_04_05_06_07 = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_08_09_0A_0B_0C_0D_0E_0F = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_10_11_12_13_14_15_16_17 = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_18_19_1A_1B_1C_1D_1E_1F = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_20_21_22_23_24_25_26_27 = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_28_29_2A_2B_2C_2D_2E_2F = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_30_31_32_33_34_35_36_37 = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_38_39_3A_3B_3C_3D_3E_3F = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01_02_03_04_05_06_07_08_09_0A_0B_0C_0D_0E_0F = shufflevector <64 x i8> %src512, <64 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_10_11_12_13_14_15_16_17_18_19_1A_1B_1C_1D_1E_1F = shufflevector <64 x i8> %src512, <64 x i8> undef, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_20_21_22_23_24_25_26_27_28_29_2A_2B_2C_2D_2E_2F = shufflevector <64 x i8> %src512, <64 x i8> undef, <16 x i32> <i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_30_31_32_33_34_35_36_37_38_39_3A_3B_3C_3D_3E_3F = shufflevector <64 x i8> %src512, <64 x i8> undef, <16 x i32> <i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01_02_03_04_05_06_07_08_09_0A_0B_0C_0D_0E_0F_10_11_12_13_14_15_16_17_18_19_1A_1B_1C_1D_1E_1F = shufflevector <64 x i8> %src512, <64 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_20_21_22_23_24_25_26_27_28_29_2A_2B_2C_2D_2E_2F_30_31_32_33_34_35_36_37_38_39_3A_3B_3C_3D_3E_3F = shufflevector <64 x i8> %src512, <64 x i8> undef, <32 x i32> <i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; BTVER2-LABEL: 'test_vXi8'
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64_01 = shufflevector <8 x i8> %src64, <8 x i8> undef, <2 x i32> <i32 0, i32 1>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_23 = shufflevector <8 x i8> %src64, <8 x i8> undef, <2 x i32> <i32 2, i32 3>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_45 = shufflevector <8 x i8> %src64, <8 x i8> undef, <2 x i32> <i32 4, i32 5>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_67 = shufflevector <8 x i8> %src64, <8 x i8> undef, <2 x i32> <i32 6, i32 7>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64_0123 = shufflevector <8 x i8> %src64, <8 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_4567 = shufflevector <8 x i8> %src64, <8 x i8> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128_01 = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 0, i32 1>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_23 = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 2, i32 3>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_45 = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 4, i32 5>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_67 = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 6, i32 7>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_89 = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 8, i32 9>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_AB = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 10, i32 11>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_CD = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 12, i32 13>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_EF = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 14, i32 15>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128_0123 = shufflevector <16 x i8> %src128, <16 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128_2345 = shufflevector <16 x i8> %src128, <16 x i8> undef, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_4567 = shufflevector <16 x i8> %src128, <16 x i8> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128_6789 = shufflevector <16 x i8> %src128, <16 x i8> undef, <4 x i32> <i32 6, i32 7, i32 8, i32 9>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_89AB = shufflevector <16 x i8> %src128, <16 x i8> undef, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_CDEF = shufflevector <16 x i8> %src128, <16 x i8> undef, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128_01234567 = shufflevector <16 x i8> %src128, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_89ABCDEF = shufflevector <16 x i8> %src128, <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_00_01 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 0, i32 1>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_02_03 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 2, i32 3>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_04_05 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 4, i32 5>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_06_07 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 6, i32 7>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_08_09 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 8, i32 9>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_0A_0B = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 10, i32 11>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_0C_0D = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 12, i32 13>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_0E_0F = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 14, i32 15>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_10_11 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 16, i32 17>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_12_13 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 18, i32 19>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_14_15 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 20, i32 21>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_16_17 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 22, i32 23>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_18_19 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 24, i32 25>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_1A_1B = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 26, i32 27>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_1C_1D = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 28, i32 29>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_1E_1F = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 30, i32 31>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_00_01_02_03 = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V256_02_03_04_05 = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_04_05_06_07 = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V256_06_07_08_09 = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 6, i32 7, i32 8, i32 9>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_08_09_0A_0B = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_0C_0D_0E_0F = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_10_11_12_13 = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_14_15_16_17 = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 20, i32 21, i32 22, i32 23>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_18_19_1A_1B = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 24, i32 25, i32 26, i32 27>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_1C_1D_1E_1F = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 28, i32 29, i32 30, i32 31>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_00_01_02_03_04_05_06_07 = shufflevector <32 x i8> %src256, <32 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_08_09_0A_0B_0C_0D_0E_0F = shufflevector <32 x i8> %src256, <32 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_10_11_12_13_14_15_16_17 = shufflevector <32 x i8> %src256, <32 x i8> undef, <8 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_18_19_1A_1B_1C_1D_1E_1F = shufflevector <32 x i8> %src256, <32 x i8> undef, <8 x i32> <i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_00_01_02_03_04_05_06_07_08_09_0A_0B_0C_0D_0E_0F = shufflevector <32 x i8> %src256, <32 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_10_11_12_13_14_15_16_17_18_19_1A_1B_1C_1D_1E_1F = shufflevector <32 x i8> %src256, <32 x i8> undef, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 0, i32 1>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_02_03 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 2, i32 3>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_04_05 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 4, i32 5>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_06_07 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 6, i32 7>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_08_09 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 8, i32 9>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_0A_0B = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 10, i32 11>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_0C_0D = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 12, i32 13>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_0E_0F = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 14, i32 15>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_10_11 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 16, i32 17>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_12_13 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 18, i32 19>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_14_15 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 20, i32 21>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_16_17 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 22, i32 23>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_18_19 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 24, i32 25>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_1A_1B = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 26, i32 27>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_1C_1D = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 28, i32 29>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_1E_1F = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 30, i32 31>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_20_21 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 32, i32 33>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_22_23 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 34, i32 35>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_24_25 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 36, i32 37>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_26_27 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 38, i32 39>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_28_29 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 40, i32 41>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_2A_2B = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 42, i32 43>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_2C_2D = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 44, i32 45>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_2E_2F = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 46, i32 47>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_30_31 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 48, i32 49>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_32_33 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 50, i32 51>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_34_35 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 52, i32 53>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_36_37 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 54, i32 55>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_38_39 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 56, i32 57>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_3A_3B = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 58, i32 59>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_3C_3D = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 60, i32 61>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_3E_3F = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 62, i32 63>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01_02_03 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_04_05_06_07 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_08_09_0A_0B = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_0C_0D_0E_0F = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_10_11_12_13 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_14_15_16_17 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 20, i32 21, i32 22, i32 23>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_18_19_1A_1B = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 24, i32 25, i32 26, i32 27>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_1C_1D_1E_1F = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 28, i32 29, i32 30, i32 31>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_20_21_22_23 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 32, i32 33, i32 34, i32 35>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_24_25_26_27 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 36, i32 37, i32 38, i32 39>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_28_29_2A_2B = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 40, i32 41, i32 42, i32 43>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_2C_2D_2E_2F = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 44, i32 45, i32 46, i32 47>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_30_31_32_33 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 48, i32 49, i32 50, i32 51>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_34_35_36_37 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 52, i32 53, i32 54, i32 55>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_38_39_3A_3B = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 56, i32 57, i32 58, i32 59>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_3C_3D_3E_3F = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 60, i32 61, i32 62, i32 63>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01_02_03_04_05_06_07 = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_08_09_0A_0B_0C_0D_0E_0F = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_10_11_12_13_14_15_16_17 = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_18_19_1A_1B_1C_1D_1E_1F = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_20_21_22_23_24_25_26_27 = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_28_29_2A_2B_2C_2D_2E_2F = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_30_31_32_33_34_35_36_37 = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_38_39_3A_3B_3C_3D_3E_3F = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01_02_03_04_05_06_07_08_09_0A_0B_0C_0D_0E_0F = shufflevector <64 x i8> %src512, <64 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_10_11_12_13_14_15_16_17_18_19_1A_1B_1C_1D_1E_1F = shufflevector <64 x i8> %src512, <64 x i8> undef, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_20_21_22_23_24_25_26_27_28_29_2A_2B_2C_2D_2E_2F = shufflevector <64 x i8> %src512, <64 x i8> undef, <16 x i32> <i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_30_31_32_33_34_35_36_37_38_39_3A_3B_3C_3D_3E_3F = shufflevector <64 x i8> %src512, <64 x i8> undef, <16 x i32> <i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_00_01_02_03_04_05_06_07_08_09_0A_0B_0C_0D_0E_0F_10_11_12_13_14_15_16_17_18_19_1A_1B_1C_1D_1E_1F = shufflevector <64 x i8> %src512, <64 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_20_21_22_23_24_25_26_27_28_29_2A_2B_2C_2D_2E_2F_30_31_32_33_34_35_36_37_38_39_3A_3B_3C_3D_3E_3F = shufflevector <64 x i8> %src512, <64 x i8> undef, <32 x i32> <i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %V64_01 = shufflevector <8 x i8> %src64, <8 x i8> undef, <2 x i32> <i32 0, i32 1>
+  %V64_23 = shufflevector <8 x i8> %src64, <8 x i8> undef, <2 x i32> <i32 2, i32 3>
+  %V64_45 = shufflevector <8 x i8> %src64, <8 x i8> undef, <2 x i32> <i32 4, i32 5>
+  %V64_67 = shufflevector <8 x i8> %src64, <8 x i8> undef, <2 x i32> <i32 6, i32 7>
+  %V64_0123 = shufflevector <8 x i8> %src64, <8 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %V64_4567 = shufflevector <8 x i8> %src64, <8 x i8> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %V128_01 = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 0, i32 1>
+  %V128_23 = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 2, i32 3>
+  %V128_45 = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 4, i32 5>
+  %V128_67 = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 6, i32 7>
+  %V128_89 = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 8, i32 9>
+  %V128_AB = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 10, i32 11>
+  %V128_CD = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 12, i32 13>
+  %V128_EF = shufflevector <16 x i8> %src128, <16 x i8> undef, <2 x i32> <i32 14, i32 15>
+  %V128_0123 = shufflevector <16 x i8> %src128, <16 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %V128_2345 = shufflevector <16 x i8> %src128, <16 x i8> undef, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
+  %V128_4567 = shufflevector <16 x i8> %src128, <16 x i8> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %V128_6789 = shufflevector <16 x i8> %src128, <16 x i8> undef, <4 x i32> <i32 6, i32 7, i32 8, i32 9>
+  %V128_89AB = shufflevector <16 x i8> %src128, <16 x i8> undef, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+  %V128_CDEF = shufflevector <16 x i8> %src128, <16 x i8> undef, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+  %V128_01234567 = shufflevector <16 x i8> %src128, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %V128_89ABCDEF = shufflevector <16 x i8> %src128, <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %V256_00_01 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 0, i32 1>
+  %V256_02_03 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 2, i32 3>
+  %V256_04_05 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 4, i32 5>
+  %V256_06_07 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 6, i32 7>
+  %V256_08_09 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 8, i32 9>
+  %V256_0A_0B = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 10, i32 11>
+  %V256_0C_0D = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 12, i32 13>
+  %V256_0E_0F = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 14, i32 15>
+  %V256_10_11 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 16, i32 17>
+  %V256_12_13 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 18, i32 19>
+  %V256_14_15 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 20, i32 21>
+  %V256_16_17 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 22, i32 23>
+  %V256_18_19 = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 24, i32 25>
+  %V256_1A_1B = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 26, i32 27>
+  %V256_1C_1D = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 28, i32 29>
+  %V256_1E_1F = shufflevector <32 x i8> %src256, <32 x i8> undef, <2 x i32> <i32 30, i32 31>
+  %V256_00_01_02_03 = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %V256_02_03_04_05 = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
+  %V256_04_05_06_07 = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %V256_06_07_08_09 = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 6, i32 7, i32 8, i32 9>
+  %V256_08_09_0A_0B = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+  %V256_0C_0D_0E_0F = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+  %V256_10_11_12_13 = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
+  %V256_14_15_16_17 = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 20, i32 21, i32 22, i32 23>
+  %V256_18_19_1A_1B = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 24, i32 25, i32 26, i32 27>
+  %V256_1C_1D_1E_1F = shufflevector <32 x i8> %src256, <32 x i8> undef, <4 x i32> <i32 28, i32 29, i32 30, i32 31>
+  %V256_00_01_02_03_04_05_06_07 = shufflevector <32 x i8> %src256, <32 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %V256_08_09_0A_0B_0C_0D_0E_0F = shufflevector <32 x i8> %src256, <32 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %V256_10_11_12_13_14_15_16_17 = shufflevector <32 x i8> %src256, <32 x i8> undef, <8 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+  %V256_18_19_1A_1B_1C_1D_1E_1F = shufflevector <32 x i8> %src256, <32 x i8> undef, <8 x i32> <i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+  %V256_00_01_02_03_04_05_06_07_08_09_0A_0B_0C_0D_0E_0F = shufflevector <32 x i8> %src256, <32 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %V256_10_11_12_13_14_15_16_17_18_19_1A_1B_1C_1D_1E_1F = shufflevector <32 x i8> %src256, <32 x i8> undef, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+  %V512_00_01 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 0, i32 1>
+  %V512_02_03 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 2, i32 3>
+  %V512_04_05 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 4, i32 5>
+  %V512_06_07 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 6, i32 7>
+  %V512_08_09 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 8, i32 9>
+  %V512_0A_0B = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 10, i32 11>
+  %V512_0C_0D = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 12, i32 13>
+  %V512_0E_0F = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 14, i32 15>
+  %V512_10_11 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 16, i32 17>
+  %V512_12_13 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 18, i32 19>
+  %V512_14_15 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 20, i32 21>
+  %V512_16_17 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 22, i32 23>
+  %V512_18_19 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 24, i32 25>
+  %V512_1A_1B = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 26, i32 27>
+  %V512_1C_1D = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 28, i32 29>
+  %V512_1E_1F = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 30, i32 31>
+  %V512_20_21 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 32, i32 33>
+  %V512_22_23 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 34, i32 35>
+  %V512_24_25 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 36, i32 37>
+  %V512_26_27 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 38, i32 39>
+  %V512_28_29 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 40, i32 41>
+  %V512_2A_2B = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 42, i32 43>
+  %V512_2C_2D = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 44, i32 45>
+  %V512_2E_2F = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 46, i32 47>
+  %V512_30_31 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 48, i32 49>
+  %V512_32_33 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 50, i32 51>
+  %V512_34_35 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 52, i32 53>
+  %V512_36_37 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 54, i32 55>
+  %V512_38_39 = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 56, i32 57>
+  %V512_3A_3B = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 58, i32 59>
+  %V512_3C_3D = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 60, i32 61>
+  %V512_3E_3F = shufflevector <64 x i8> %src512, <64 x i8> undef, <2 x i32> <i32 62, i32 63>
+  %V512_00_01_02_03 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %V512_04_05_06_07 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %V512_08_09_0A_0B = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 8, i32 9,i32 10, i32 11>
+  %V512_0C_0D_0E_0F = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+  %V512_10_11_12_13 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
+  %V512_14_15_16_17 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 20, i32 21, i32 22, i32 23>
+  %V512_18_19_1A_1B = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 24, i32 25, i32 26, i32 27>
+  %V512_1C_1D_1E_1F = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 28, i32 29, i32 30, i32 31>
+  %V512_20_21_22_23 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 32, i32 33, i32 34, i32 35>
+  %V512_24_25_26_27 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 36, i32 37, i32 38, i32 39>
+  %V512_28_29_2A_2B = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 40, i32 41, i32 42, i32 43>
+  %V512_2C_2D_2E_2F = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 44, i32 45, i32 46, i32 47>
+  %V512_30_31_32_33 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 48, i32 49, i32 50, i32 51>
+  %V512_34_35_36_37 = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 52, i32 53, i32 54, i32 55>
+  %V512_38_39_3A_3B = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 56, i32 57, i32 58, i32 59>
+  %V512_3C_3D_3E_3F = shufflevector <64 x i8> %src512, <64 x i8> undef, <4 x i32> <i32 60, i32 61, i32 62, i32 63>
+  %V512_00_01_02_03_04_05_06_07 = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %V512_08_09_0A_0B_0C_0D_0E_0F = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %V512_10_11_12_13_14_15_16_17 = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+  %V512_18_19_1A_1B_1C_1D_1E_1F = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+  %V512_20_21_22_23_24_25_26_27 = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39>
+  %V512_28_29_2A_2B_2C_2D_2E_2F = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47>
+  %V512_30_31_32_33_34_35_36_37 = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55>
+  %V512_38_39_3A_3B_3C_3D_3E_3F = shufflevector <64 x i8> %src512, <64 x i8> undef, <8 x i32> <i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+  %V512_00_01_02_03_04_05_06_07_08_09_0A_0B_0C_0D_0E_0F = shufflevector <64 x i8> %src512, <64 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %V512_10_11_12_13_14_15_16_17_18_19_1A_1B_1C_1D_1E_1F = shufflevector <64 x i8> %src512, <64 x i8> undef, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23,i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+  %V512_20_21_22_23_24_25_26_27_28_29_2A_2B_2C_2D_2E_2F = shufflevector <64 x i8> %src512, <64 x i8> undef, <16 x i32> <i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47>
+  %V512_30_31_32_33_34_35_36_37_38_39_3A_3B_3C_3D_3E_3F = shufflevector <64 x i8> %src512, <64 x i8> undef, <16 x i32> <i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+  %V512_00_01_02_03_04_05_06_07_08_09_0A_0B_0C_0D_0E_0F_10_11_12_13_14_15_16_17_18_19_1A_1B_1C_1D_1E_1F = shufflevector <64 x i8> %src512, <64 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23,i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+  %V512_20_21_22_23_24_25_26_27_28_29_2A_2B_2C_2D_2E_2F_30_31_32_33_34_35_36_37_38_39_3A_3B_3C_3D_3E_3F = shufflevector <64 x i8> %src512, <64 x i8> undef, <32 x i32> <i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+  ret void
+}
diff --git a/llvm/test/Analysis/CostModel/X86/shuffle-transpose.ll b/llvm/test/Analysis/CostModel/X86/shuffle-transpose.ll
index 25a887604..80ea1c980 100644
--- a/llvm/test/Analysis/CostModel/X86/shuffle-transpose.ll
+++ b/llvm/test/Analysis/CostModel/X86/shuffle-transpose.ll
@@ -123,21 +123,21 @@ define void @test_vXf32(<2 x float> %a64, <2 x float> %b64, <4 x float> %a128, <
 
 define void @test_vXi32(<2 x i32> %a64, <2 x i32> %b64, <4 x i32> %a128, <4 x i32> %b128, <8 x i32> %a256, <8 x i32> %b256, <16 x i32> %a512, <16 x i32> %b512) {
 ; SSE-LABEL: 'test_vXi32'
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <2 x i32> %a64, <2 x i32> %b64, <2 x i32> <i32 0, i32 2>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64 = shufflevector <2 x i32> %a64, <2 x i32> %b64, <2 x i32> <i32 0, i32 2>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX1-LABEL: 'test_vXi32'
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <2 x i32> %a64, <2 x i32> %b64, <2 x i32> <i32 0, i32 2>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64 = shufflevector <2 x i32> %a64, <2 x i32> %b64, <2 x i32> <i32 0, i32 2>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX2-LABEL: 'test_vXi32'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <2 x i32> %a64, <2 x i32> %b64, <2 x i32> <i32 0, i32 2>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64 = shufflevector <2 x i32> %a64, <2 x i32> %b64, <2 x i32> <i32 0, i32 2>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
@@ -151,7 +151,7 @@ define void @test_vXi32(<2 x i32> %a64, <2 x i32> %b64, <4 x i32> %a128, <4 x i3
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; BTVER2-LABEL: 'test_vXi32'
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <2 x i32> %a64, <2 x i32> %b64, <2 x i32> <i32 0, i32 2>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64 = shufflevector <2 x i32> %a64, <2 x i32> %b64, <2 x i32> <i32 0, i32 2>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
diff --git a/llvm/test/Analysis/CostModel/X86/sitofp.ll b/llvm/test/Analysis/CostModel/X86/sitofp.ll
index c39988a4f..8ac8050da 100644
--- a/llvm/test/Analysis/CostModel/X86/sitofp.ll
+++ b/llvm/test/Analysis/CostModel/X86/sitofp.ll
@@ -13,9 +13,9 @@
 define i32 @sitofp_i8_double() {
 ; SSE-LABEL: 'sitofp_i8_double'
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cvt_i8_f64 = sitofp i8 undef to double
-; SSE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %cvt_v2i8_v2f64 = sitofp <2 x i8> undef to <2 x double>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %cvt_v4i8_v4f64 = sitofp <4 x i8> undef to <4 x double>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %cvt_v8i8_v8f64 = sitofp <8 x i8> undef to <8 x double>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %cvt_v2i8_v2f64 = sitofp <2 x i8> undef to <2 x double>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %cvt_v4i8_v4f64 = sitofp <4 x i8> undef to <4 x double>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %cvt_v8i8_v8f64 = sitofp <8 x i8> undef to <8 x double>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX-LABEL: 'sitofp_i8_double'
@@ -49,8 +49,8 @@ define i32 @sitofp_i8_double() {
 define i32 @sitofp_i16_double() {
 ; SSE-LABEL: 'sitofp_i16_double'
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cvt_i16_f64 = sitofp i16 undef to double
-; SSE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %cvt_v2i16_v2f64 = sitofp <2 x i16> undef to <2 x double>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %cvt_v4i16_v4f64 = sitofp <4 x i16> undef to <4 x double>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %cvt_v2i16_v2f64 = sitofp <2 x i16> undef to <2 x double>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %cvt_v4i16_v4f64 = sitofp <4 x i16> undef to <4 x double>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %cvt_v8i16_v8f64 = sitofp <8 x i16> undef to <8 x double>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
@@ -86,27 +86,27 @@ define i32 @sitofp_i32_double() {
 ; SSE-LABEL: 'sitofp_i32_double'
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cvt_i32_f64 = sitofp i32 undef to double
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %cvt_v2i32_v2f64 = sitofp <2 x i32> undef to <2 x double>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %cvt_v4i32_v4f64 = sitofp <4 x i32> undef to <4 x double>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %cvt_v8i32_v8f64 = sitofp <8 x i32> undef to <8 x double>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %cvt_v4i32_v4f64 = sitofp <4 x i32> undef to <4 x double>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %cvt_v8i32_v8f64 = sitofp <8 x i32> undef to <8 x double>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX-LABEL: 'sitofp_i32_double'
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cvt_i32_f64 = sitofp i32 undef to double
-; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %cvt_v2i32_v2f64 = sitofp <2 x i32> undef to <2 x double>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %cvt_v2i32_v2f64 = sitofp <2 x i32> undef to <2 x double>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cvt_v4i32_v4f64 = sitofp <4 x i32> undef to <4 x double>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %cvt_v8i32_v8f64 = sitofp <8 x i32> undef to <8 x double>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512-LABEL: 'sitofp_i32_double'
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cvt_i32_f64 = sitofp i32 undef to double
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %cvt_v2i32_v2f64 = sitofp <2 x i32> undef to <2 x double>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %cvt_v2i32_v2f64 = sitofp <2 x i32> undef to <2 x double>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cvt_v4i32_v4f64 = sitofp <4 x i32> undef to <4 x double>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cvt_v8i32_v8f64 = sitofp <8 x i32> undef to <8 x double>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; BTVER2-LABEL: 'sitofp_i32_double'
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cvt_i32_f64 = sitofp i32 undef to double
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %cvt_v2i32_v2f64 = sitofp <2 x i32> undef to <2 x double>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %cvt_v2i32_v2f64 = sitofp <2 x i32> undef to <2 x double>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cvt_v4i32_v4f64 = sitofp <4 x i32> undef to <4 x double>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %cvt_v8i32_v8f64 = sitofp <8 x i32> undef to <8 x double>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
@@ -164,8 +164,8 @@ define i32 @sitofp_i64_double() {
 define i32 @sitofp_i8_float() {
 ; SSE-LABEL: 'sitofp_i8_float'
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cvt_i8_f32 = sitofp i8 undef to float
-; SSE-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %cvt_v4i8_v4f32 = sitofp <4 x i8> undef to <4 x float>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %cvt_v8i8_v8f32 = sitofp <8 x i8> undef to <8 x float>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %cvt_v4i8_v4f32 = sitofp <4 x i8> undef to <4 x float>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %cvt_v8i8_v8f32 = sitofp <8 x i8> undef to <8 x float>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %cvt_v16i8_v16f32 = sitofp <16 x i8> undef to <16 x float>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
@@ -200,7 +200,7 @@ define i32 @sitofp_i8_float() {
 define i32 @sitofp_i16_float() {
 ; SSE-LABEL: 'sitofp_i16_float'
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cvt_i16_f32 = sitofp i16 undef to float
-; SSE-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %cvt_v4i16_v4f32 = sitofp <4 x i16> undef to <4 x float>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %cvt_v4i16_v4f32 = sitofp <4 x i16> undef to <4 x float>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %cvt_v8i16_v8f32 = sitofp <8 x i16> undef to <8 x float>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %cvt_v16i16_v16f32 = sitofp <16 x i16> undef to <16 x float>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
diff --git a/llvm/test/Analysis/CostModel/X86/slm-arith-costs.ll b/llvm/test/Analysis/CostModel/X86/slm-arith-costs.ll
index ab2eedc67..723cf35aa 100644
--- a/llvm/test/Analysis/CostModel/X86/slm-arith-costs.ll
+++ b/llvm/test/Analysis/CostModel/X86/slm-arith-costs.ll
@@ -47,11 +47,11 @@ entry:
 
 define <2 x i8> @slm-costs_8_v2_mul(<2 x i8> %a, <2 x i8> %b)  {
 ; SLM-LABEL: 'slm-costs_8_v2_mul'
-; SLM-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %res = mul nsw <2 x i8> %a, %b
+; SLM-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %res = mul nsw <2 x i8> %a, %b
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i8> %res
 ;
 ; GLM-LABEL: 'slm-costs_8_v2_mul'
-; GLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %res = mul nsw <2 x i8> %a, %b
+; GLM-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %res = mul nsw <2 x i8> %a, %b
 ; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i8> %res
 ;
 entry:
@@ -61,11 +61,11 @@ entry:
 
 define <4 x i8> @slm-costs_8_v4_mul(<4 x i8> %a, <4 x i8> %b)  {
 ; SLM-LABEL: 'slm-costs_8_v4_mul'
-; SLM-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res = mul nsw <4 x i8> %a, %b
+; SLM-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %res = mul nsw <4 x i8> %a, %b
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i8> %res
 ;
 ; GLM-LABEL: 'slm-costs_8_v4_mul'
-; GLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %res = mul nsw <4 x i8> %a, %b
+; GLM-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %res = mul nsw <4 x i8> %a, %b
 ; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i8> %res
 ;
 entry:
@@ -177,11 +177,11 @@ entry:
 
 define <8 x i8> @slm-costs_8_v8_mul(<8 x i8> %a, <8 x i8> %b)  {
 ; SLM-LABEL: 'slm-costs_8_v8_mul'
-; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %res = mul nsw <8 x i8> %a, %b
+; SLM-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %res = mul nsw <8 x i8> %a, %b
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i8> %res
 ;
 ; GLM-LABEL: 'slm-costs_8_v8_mul'
-; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = mul nsw <8 x i8> %a, %b
+; GLM-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %res = mul nsw <8 x i8> %a, %b
 ; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i8> %res
 ;
 entry:
@@ -216,11 +216,11 @@ entry:
 
 define <2 x i16> @slm-costs_16_v2_mul(<2 x i16> %a, <2 x i16> %b)  {
 ; SLM-LABEL: 'slm-costs_16_v2_mul'
-; SLM-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %res = mul nsw <2 x i16> %a, %b
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %res = mul nsw <2 x i16> %a, %b
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i16> %res
 ;
 ; GLM-LABEL: 'slm-costs_16_v2_mul'
-; GLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %res = mul nsw <2 x i16> %a, %b
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = mul nsw <2 x i16> %a, %b
 ; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i16> %res
 ;
 entry:
@@ -230,11 +230,11 @@ entry:
 
 define <4 x i16> @slm-costs_16_v4_mul(<4 x i16> %a, <4 x i16> %b)  {
 ; SLM-LABEL: 'slm-costs_16_v4_mul'
-; SLM-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %res = mul nsw <4 x i16> %a, %b
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %res = mul nsw <4 x i16> %a, %b
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i16> %res
 ;
 ; GLM-LABEL: 'slm-costs_16_v4_mul'
-; GLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %res = mul nsw <4 x i16> %a, %b
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = mul nsw <4 x i16> %a, %b
 ; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i16> %res
 ;
 entry:
@@ -385,11 +385,11 @@ entry:
 
 define <2 x i32> @slm-costs_32_v2_mul(<2 x i32> %a, <2 x i32> %b)  {
 ; SLM-LABEL: 'slm-costs_32_v2_mul'
-; SLM-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %res = mul nsw <2 x i32> %a, %b
+; SLM-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %res = mul nsw <2 x i32> %a, %b
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i32> %res
 ;
 ; GLM-LABEL: 'slm-costs_32_v2_mul'
-; GLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %res = mul nsw <2 x i32> %a, %b
+; GLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %res = mul nsw <2 x i32> %a, %b
 ; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i32> %res
 ;
 entry:
diff --git a/llvm/test/Analysis/CostModel/X86/testshiftashr.ll b/llvm/test/Analysis/CostModel/X86/testshiftashr.ll
index 7f588a748..270f13a3e 100644
--- a/llvm/test/Analysis/CostModel/X86/testshiftashr.ll
+++ b/llvm/test/Analysis/CostModel/X86/testshiftashr.ll
@@ -5,9 +5,9 @@
 define %shifttype @shift2i16(%shifttype %a, %shifttype %b) {
 entry:
   ; SSE2-LABEL: shift2i16
-  ; SSE2: cost of 12 {{.*}} ashr
+  ; SSE2: cost of 32 {{.*}} ashr
   ; SSE2-CODEGEN-LABEL: shift2i16
-  ; SSE2-CODEGEN: psrlq
+  ; SSE2-CODEGEN: psraw
 
   %0 = ashr %shifttype %a , %b
   ret %shifttype %0
@@ -17,9 +17,9 @@ entry:
 define %shifttype4i16 @shift4i16(%shifttype4i16 %a, %shifttype4i16 %b) {
 entry:
   ; SSE2-LABEL: shift4i16
-  ; SSE2: cost of 16 {{.*}} ashr
+  ; SSE2: cost of 32 {{.*}} ashr
   ; SSE2-CODEGEN-LABEL: shift4i16
-  ; SSE2-CODEGEN: psrad
+  ; SSE2-CODEGEN: psraw
 
   %0 = ashr %shifttype4i16 %a , %b
   ret %shifttype4i16 %0
@@ -65,9 +65,9 @@ entry:
 define %shifttype2i32 @shift2i32(%shifttype2i32 %a, %shifttype2i32 %b) {
 entry:
   ; SSE2-LABEL: shift2i32
-  ; SSE2: cost of 12 {{.*}} ashr
+  ; SSE2: cost of 16 {{.*}} ashr
   ; SSE2-CODEGEN-LABEL: shift2i32
-  ; SSE2-CODEGEN: psrlq
+  ; SSE2-CODEGEN: psrad
 
   %0 = ashr %shifttype2i32 %a , %b
   ret %shifttype2i32 %0
@@ -185,9 +185,9 @@ entry:
 define %shifttype2i8 @shift2i8(%shifttype2i8 %a, %shifttype2i8 %b) {
 entry:
   ; SSE2-LABEL: shift2i8
-  ; SSE2: cost of 12 {{.*}} ashr
+  ; SSE2: cost of 54 {{.*}} ashr
   ; SSE2-CODEGEN-LABEL: shift2i8
-  ; SSE2-CODEGEN: psrlq
+  ; SSE2-CODEGEN: psrlw
 
   %0 = ashr %shifttype2i8 %a , %b
   ret %shifttype2i8 %0
@@ -197,9 +197,9 @@ entry:
 define %shifttype4i8 @shift4i8(%shifttype4i8 %a, %shifttype4i8 %b) {
 entry:
   ; SSE2-LABEL: shift4i8
-  ; SSE2: cost of 16 {{.*}} ashr
+  ; SSE2: cost of 54 {{.*}} ashr
   ; SSE2-CODEGEN-LABEL: shift4i8
-  ; SSE2-CODEGEN: psrad
+  ; SSE2-CODEGEN: psraw
 
   %0 = ashr %shifttype4i8 %a , %b
   ret %shifttype4i8 %0
@@ -209,7 +209,7 @@ entry:
 define %shifttype8i8 @shift8i8(%shifttype8i8 %a, %shifttype8i8 %b) {
 entry:
   ; SSE2-LABEL: shift8i8
-  ; SSE2: cost of 32 {{.*}} ashr
+  ; SSE2: cost of 54 {{.*}} ashr
   ; SSE2-CODEGEN-LABEL: shift8i8
   ; SSE2-CODEGEN: psraw
 
@@ -247,9 +247,9 @@ entry:
 define %shifttypec @shift2i16const(%shifttypec %a, %shifttypec %b) {
 entry:
   ; SSE2-LABEL: shift2i16const
-  ; SSE2: cost of 4 {{.*}} ashr
+  ; SSE2: cost of 1 {{.*}} ashr
   ; SSE2-CODEGEN-LABEL: shift2i16const
-  ; SSE2-CODEGEN: psrad $3
+  ; SSE2-CODEGEN: psraw $3
 
   %0 = ashr %shifttypec %a , <i16 3, i16 3>
   ret %shifttypec %0
@@ -261,7 +261,7 @@ entry:
   ; SSE2-LABEL: shift4i16const
   ; SSE2: cost of 1 {{.*}} ashr
   ; SSE2-CODEGEN-LABEL: shift4i16const
-  ; SSE2-CODEGEN: psrad $19
+  ; SSE2-CODEGEN: psraw $3
 
   %0 = ashr %shifttypec4i16 %a , <i16 3, i16 3, i16 3, i16 3>
   ret %shifttypec4i16 %0
@@ -320,7 +320,7 @@ entry:
 define %shifttypec2i32 @shift2i32c(%shifttypec2i32 %a, %shifttypec2i32 %b) {
 entry:
   ; SSE2-LABEL: shift2i32c
-  ; SSE2: cost of 4 {{.*}} ashr
+  ; SSE2: cost of 1 {{.*}} ashr
   ; SSE2-CODEGEN-LABEL: shift2i32c
   ; SSE2-CODEGEN: psrad $3
 
@@ -464,7 +464,7 @@ entry:
   ; SSE2-LABEL: shift2i8c
   ; SSE2: cost of 4 {{.*}} ashr
   ; SSE2-CODEGEN-LABEL: shift2i8c
-  ; SSE2-CODEGEN: psrad $3
+  ; SSE2-CODEGEN: psrlw $3
 
   %0 = ashr %shifttypec2i8 %a , <i8 3, i8 3>
   ret %shifttypec2i8 %0
@@ -474,9 +474,9 @@ entry:
 define %shifttypec4i8 @shift4i8c(%shifttypec4i8 %a, %shifttypec4i8 %b) {
 entry:
   ; SSE2-LABEL: shift4i8c
-  ; SSE2: cost of 1 {{.*}} ashr
+  ; SSE2: cost of 4 {{.*}} ashr
   ; SSE2-CODEGEN-LABEL: shift4i8c
-  ; SSE2-CODEGEN: psrad $27
+  ; SSE2-CODEGEN: psrlw $3
 
   %0 = ashr %shifttypec4i8 %a , <i8 3, i8 3, i8 3, i8 3>
   ret %shifttypec4i8 %0
@@ -486,9 +486,9 @@ entry:
 define %shifttypec8i8 @shift8i8c(%shifttypec8i8 %a, %shifttypec8i8 %b) {
 entry:
   ; SSE2-LABEL: shift8i8c
-  ; SSE2: cost of 1 {{.*}} ashr
+  ; SSE2: cost of 4 {{.*}} ashr
   ; SSE2-CODEGEN-LABEL: shift8i8c
-  ; SSE2-CODEGEN: psraw $11
+  ; SSE2-CODEGEN: psrlw $3
 
   %0 = ashr %shifttypec8i8 %a , <i8 3, i8 3, i8 3, i8 3,
                                  i8 3, i8 3, i8 3, i8 3>
diff --git a/llvm/test/Analysis/CostModel/X86/testshiftlshr.ll b/llvm/test/Analysis/CostModel/X86/testshiftlshr.ll
index 3e30614e1..d60f6895c 100644
--- a/llvm/test/Analysis/CostModel/X86/testshiftlshr.ll
+++ b/llvm/test/Analysis/CostModel/X86/testshiftlshr.ll
@@ -5,9 +5,9 @@
 define %shifttype @shift2i16(%shifttype %a, %shifttype %b) {
 entry:
   ; SSE2-LABEL: shift2i16
-  ; SSE2: cost of 4 {{.*}} lshr
+  ; SSE2: cost of 32 {{.*}} lshr
   ; SSE2-CODEGEN-LABEL: shift2i16
-  ; SSE2-CODEGEN: psrlq
+  ; SSE2-CODEGEN: psrlw
 
   %0 = lshr %shifttype %a , %b
   ret %shifttype %0
@@ -17,9 +17,9 @@ entry:
 define %shifttype4i16 @shift4i16(%shifttype4i16 %a, %shifttype4i16 %b) {
 entry:
   ; SSE2-LABEL: shift4i16
-  ; SSE2: cost of 16 {{.*}} lshr
+  ; SSE2: cost of 32 {{.*}} lshr
   ; SSE2-CODEGEN-LABEL: shift4i16
-  ; SSE2-CODEGEN: psrld
+  ; SSE2-CODEGEN: psrlw
 
   %0 = lshr %shifttype4i16 %a , %b
   ret %shifttype4i16 %0
@@ -65,9 +65,9 @@ entry:
 define %shifttype2i32 @shift2i32(%shifttype2i32 %a, %shifttype2i32 %b) {
 entry:
   ; SSE2-LABEL: shift2i32
-  ; SSE2: cost of 4 {{.*}} lshr
+  ; SSE2: cost of 16 {{.*}} lshr
   ; SSE2-CODEGEN-LABEL: shift2i32
-  ; SSE2-CODEGEN: psrlq
+  ; SSE2-CODEGEN: psrld
 
   %0 = lshr %shifttype2i32 %a , %b
   ret %shifttype2i32 %0
@@ -185,9 +185,9 @@ entry:
 define %shifttype2i8 @shift2i8(%shifttype2i8 %a, %shifttype2i8 %b) {
 entry:
   ; SSE2-LABEL: shift2i8
-  ; SSE2: cost of 4 {{.*}} lshr
+  ; SSE2: cost of 26 {{.*}} lshr
   ; SSE2-CODEGEN-LABEL: shift2i8
-  ; SSE2-CODEGEN: psrlq
+  ; SSE2-CODEGEN: psrlw
 
   %0 = lshr %shifttype2i8 %a , %b
   ret %shifttype2i8 %0
@@ -197,9 +197,9 @@ entry:
 define %shifttype4i8 @shift4i8(%shifttype4i8 %a, %shifttype4i8 %b) {
 entry:
   ; SSE2-LABEL: shift4i8
-  ; SSE2: cost of 16 {{.*}} lshr
+  ; SSE2: cost of 26 {{.*}} lshr
   ; SSE2-CODEGEN-LABEL: shift4i8
-  ; SSE2-CODEGEN: psrld
+  ; SSE2-CODEGEN: psrlw
 
   %0 = lshr %shifttype4i8 %a , %b
   ret %shifttype4i8 %0
@@ -209,7 +209,7 @@ entry:
 define %shifttype8i8 @shift8i8(%shifttype8i8 %a, %shifttype8i8 %b) {
 entry:
   ; SSE2-LABEL: shift8i8
-  ; SSE2: cost of 32 {{.*}} lshr
+  ; SSE2: cost of 26 {{.*}} lshr
   ; SSE2-CODEGEN-LABEL: shift8i8
   ; SSE2-CODEGEN: psrlw
 
@@ -249,7 +249,7 @@ entry:
   ; SSE2-LABEL: shift2i16const
   ; SSE2: cost of 1 {{.*}} lshr
   ; SSE2-CODEGEN-LABEL: shift2i16const
-  ; SSE2-CODEGEN: psrlq $3
+  ; SSE2-CODEGEN: psrlw $3
 
   %0 = lshr %shifttypec %a , <i16 3, i16 3>
   ret %shifttypec %0
@@ -261,7 +261,7 @@ entry:
   ; SSE2-LABEL: shift4i16const
   ; SSE2: cost of 1 {{.*}} lshr
   ; SSE2-CODEGEN-LABEL: shift4i16const
-  ; SSE2-CODEGEN: psrld $3
+  ; SSE2-CODEGEN: psrlw $3
 
   %0 = lshr %shifttypec4i16 %a , <i16 3, i16 3, i16 3, i16 3>
   ret %shifttypec4i16 %0
@@ -322,7 +322,7 @@ entry:
   ; SSE2-LABEL: shift2i32c
   ; SSE2: cost of 1 {{.*}} lshr
   ; SSE2-CODEGEN-LABEL: shift2i32c
-  ; SSE2-CODEGEN: psrlq $3
+  ; SSE2-CODEGEN: psrld $3
 
   %0 = lshr %shifttypec2i32 %a , <i32 3, i32 3>
   ret %shifttypec2i32 %0
@@ -461,9 +461,9 @@ entry:
 define %shifttypec2i8 @shift2i8c(%shifttypec2i8 %a, %shifttypec2i8 %b) {
 entry:
   ; SSE2-LABEL: shift2i8c
-  ; SSE2: cost of 1 {{.*}} lshr
+  ; SSE2: cost of 2 {{.*}} lshr
   ; SSE2-CODEGEN-LABEL: shift2i8c
-  ; SSE2-CODEGEN: psrlq $3
+  ; SSE2-CODEGEN: psrlw $3
 
   %0 = lshr %shifttypec2i8 %a , <i8 3, i8 3>
   ret %shifttypec2i8 %0
@@ -473,9 +473,9 @@ entry:
 define %shifttypec4i8 @shift4i8c(%shifttypec4i8 %a, %shifttypec4i8 %b) {
 entry:
   ; SSE2-LABEL: shift4i8c
-  ; SSE2: cost of 1 {{.*}} lshr
+  ; SSE2: cost of 2 {{.*}} lshr
   ; SSE2-CODEGEN-LABEL: shift4i8c
-  ; SSE2-CODEGEN: psrld $3
+  ; SSE2-CODEGEN: psrlw $3
 
   %0 = lshr %shifttypec4i8 %a , <i8 3, i8 3, i8 3, i8 3>
   ret %shifttypec4i8 %0
@@ -485,7 +485,7 @@ entry:
 define %shifttypec8i8 @shift8i8c(%shifttypec8i8 %a, %shifttypec8i8 %b) {
 entry:
   ; SSE2-LABEL: shift8i8c
-  ; SSE2: cost of 1 {{.*}} lshr
+  ; SSE2: cost of 2 {{.*}} lshr
   ; SSE2-CODEGEN-LABEL: shift8i8c
   ; SSE2-CODEGEN: psrlw $3
 
diff --git a/llvm/test/Analysis/CostModel/X86/testshiftshl.ll b/llvm/test/Analysis/CostModel/X86/testshiftshl.ll
index 7db82b9fa..359ef1d5b 100644
--- a/llvm/test/Analysis/CostModel/X86/testshiftshl.ll
+++ b/llvm/test/Analysis/CostModel/X86/testshiftshl.ll
@@ -5,9 +5,9 @@
 define %shifttype @shift2i16(%shifttype %a, %shifttype %b) {
 entry:
   ; SSE2-LABEL: shift2i16
-  ; SSE2: cost of 4 {{.*}} shl
+  ; SSE2: cost of 32 {{.*}} shl
   ; SSE2-CODEGEN-LABEL: shift2i16
-  ; SSE2-CODEGEN: psllq
+  ; SSE2-CODEGEN: pmullw
 
   %0 = shl %shifttype %a , %b
   ret %shifttype %0
@@ -17,9 +17,9 @@ entry:
 define %shifttype4i16 @shift4i16(%shifttype4i16 %a, %shifttype4i16 %b) {
 entry:
   ; SSE2-LABEL: shift4i16
-  ; SSE2: cost of 10 {{.*}} shl
+  ; SSE2: cost of 32 {{.*}} shl
   ; SSE2-CODEGEN-LABEL: shift4i16
-  ; SSE2-CODEGEN: pmuludq
+  ; SSE2-CODEGEN: pmullw
 
   %0 = shl %shifttype4i16 %a , %b
   ret %shifttype4i16 %0
@@ -65,9 +65,9 @@ entry:
 define %shifttype2i32 @shift2i32(%shifttype2i32 %a, %shifttype2i32 %b) {
 entry:
   ; SSE2-LABEL: shift2i32
-  ; SSE2: cost of 4 {{.*}} shl
+  ; SSE2: cost of 10 {{.*}} shl
   ; SSE2-CODEGEN-LABEL: shift2i32
-  ; SSE2-CODEGEN: psllq
+  ; SSE2-CODEGEN: pmuludq
 
   %0 = shl %shifttype2i32 %a , %b
   ret %shifttype2i32 %0
@@ -185,9 +185,9 @@ entry:
 define %shifttype2i8 @shift2i8(%shifttype2i8 %a, %shifttype2i8 %b) {
 entry:
   ; SSE2-LABEL: shift2i8
-  ; SSE2: cost of 4 {{.*}} shl
+  ; SSE2: cost of 26 {{.*}} shl
   ; SSE2-CODEGEN-LABEL: shift2i8
-  ; SSE2-CODEGEN: psllq
+  ; SSE2-CODEGEN: psllw
 
   %0 = shl %shifttype2i8 %a , %b
   ret %shifttype2i8 %0
@@ -197,9 +197,9 @@ entry:
 define %shifttype4i8 @shift4i8(%shifttype4i8 %a, %shifttype4i8 %b) {
 entry:
   ; SSE2-LABEL: shift4i8
-  ; SSE2: cost of 10 {{.*}} shl
+  ; SSE2: cost of 26 {{.*}} shl
   ; SSE2-CODEGEN-LABEL: shift4i8
-  ; SSE2-CODEGEN: pmuludq
+  ; SSE2-CODEGEN: psllw
 
   %0 = shl %shifttype4i8 %a , %b
   ret %shifttype4i8 %0
@@ -209,9 +209,9 @@ entry:
 define %shifttype8i8 @shift8i8(%shifttype8i8 %a, %shifttype8i8 %b) {
 entry:
   ; SSE2-LABEL: shift8i8
-  ; SSE2: cost of 32 {{.*}} shl
+  ; SSE2: cost of 26 {{.*}} shl
   ; SSE2-CODEGEN-LABEL: shift8i8
-  ; SSE2-CODEGEN: pmullw
+  ; SSE2-CODEGEN: psllw
 
   %0 = shl %shifttype8i8 %a , %b
   ret %shifttype8i8 %0
@@ -249,7 +249,7 @@ entry:
   ; SSE2-LABEL: shift2i16const
   ; SSE2: cost of 1 {{.*}} shl
   ; SSE2-CODEGEN-LABEL: shift2i16const
-  ; SSE2-CODEGEN: psllq $3
+  ; SSE2-CODEGEN: psllw $3
 
   %0 = shl %shifttypec %a , <i16 3, i16 3>
   ret %shifttypec %0
@@ -261,7 +261,7 @@ entry:
   ; SSE2-LABEL: shift4i16const
   ; SSE2: cost of 1 {{.*}} shl
   ; SSE2-CODEGEN-LABEL: shift4i16const
-  ; SSE2-CODEGEN: pslld $3
+  ; SSE2-CODEGEN: psllw $3
 
   %0 = shl %shifttypec4i16 %a , <i16 3, i16 3, i16 3, i16 3>
   ret %shifttypec4i16 %0
@@ -322,7 +322,7 @@ entry:
   ; SSE2-LABEL: shift2i32c
   ; SSE2: cost of 1 {{.*}} shl
   ; SSE2-CODEGEN-LABEL: shift2i32c
-  ; SSE2-CODEGEN: psllq $3
+  ; SSE2-CODEGEN: pslld $3
 
   %0 = shl %shifttypec2i32 %a , <i32 3, i32 3>
   ret %shifttypec2i32 %0
@@ -461,9 +461,9 @@ entry:
 define %shifttypec2i8 @shift2i8c(%shifttypec2i8 %a, %shifttypec2i8 %b) {
 entry:
   ; SSE2-LABEL: shift2i8c
-  ; SSE2: cost of 1 {{.*}} shl
+  ; SSE2: cost of 2 {{.*}} shl
   ; SSE2-CODEGEN-LABEL: shift2i8c
-  ; SSE2-CODEGEN: psllq $3
+  ; SSE2-CODEGEN: psllw $3
 
   %0 = shl %shifttypec2i8 %a , <i8 3, i8 3>
   ret %shifttypec2i8 %0
@@ -473,9 +473,9 @@ entry:
 define %shifttypec4i8 @shift4i8c(%shifttypec4i8 %a, %shifttypec4i8 %b) {
 entry:
   ; SSE2-LABEL: shift4i8c
-  ; SSE2: cost of 1 {{.*}} shl
+  ; SSE2: cost of 2 {{.*}} shl
   ; SSE2-CODEGEN-LABEL: shift4i8c
-  ; SSE2-CODEGEN: pslld $3
+  ; SSE2-CODEGEN: psllw $3
 
   %0 = shl %shifttypec4i8 %a , <i8 3, i8 3, i8 3, i8 3>
   ret %shifttypec4i8 %0
@@ -485,7 +485,7 @@ entry:
 define %shifttypec8i8 @shift8i8c(%shifttypec8i8 %a, %shifttypec8i8 %b) {
 entry:
   ; SSE2-LABEL: shift8i8c
-  ; SSE2: cost of 1 {{.*}} shl
+  ; SSE2: cost of 2 {{.*}} shl
   ; SSE2-CODEGEN-LABEL: shift8i8c
   ; SSE2-CODEGEN: psllw $3
 
diff --git a/llvm/test/Analysis/CostModel/X86/trunc.ll b/llvm/test/Analysis/CostModel/X86/trunc.ll
index 9668da564..37d18c3e3 100644
--- a/llvm/test/Analysis/CostModel/X86/trunc.ll
+++ b/llvm/test/Analysis/CostModel/X86/trunc.ll
@@ -13,35 +13,35 @@
 
 define i32 @trunc_vXi32() {
 ; SSE-LABEL: 'trunc_vXi32'
-; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i32>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i32>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i32>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i32>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i32>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i32>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i32>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i32>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX1-LABEL: 'trunc_vXi32'
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i32>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i32>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i32>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i32>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i32>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX2-LABEL: 'trunc_vXi32'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i32>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i32>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i32>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i32>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i32>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512-LABEL: 'trunc_vXi32'
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i32>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i32>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i32>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i32>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i32>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; BTVER2-LABEL: 'trunc_vXi32'
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i32>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i32>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i32>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i32>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i32>
@@ -56,12 +56,12 @@ define i32 @trunc_vXi32() {
 
 define i32 @trunc_vXi16() {
 ; SSE2-LABEL: 'trunc_vXi16'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i16>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i16>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i16>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i16>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i16>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i16>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i16>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i16>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i16>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i16>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i16>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i16>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4i32 = trunc <4 x i32> undef to <4 x i16>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8i32 = trunc <8 x i32> undef to <8 x i16>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V16i32 = trunc <16 x i32> undef to <16 x i16>
@@ -69,12 +69,12 @@ define i32 @trunc_vXi16() {
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'trunc_vXi16'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i16>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i16>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i16>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i16>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i16>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i16>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i16>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i16>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i16>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i16>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i16>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i16>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4i32 = trunc <4 x i32> undef to <4 x i16>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8i32 = trunc <8 x i32> undef to <8 x i16>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V16i32 = trunc <16 x i32> undef to <16 x i16>
@@ -82,12 +82,12 @@ define i32 @trunc_vXi16() {
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE42-LABEL: 'trunc_vXi16'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i16>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i16>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i16>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i16>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i16>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i16>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i16>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i16>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i16>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i16>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i16>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i16>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = trunc <4 x i32> undef to <4 x i16>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8i32 = trunc <8 x i32> undef to <8 x i16>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16i32 = trunc <16 x i32> undef to <16 x i16>
@@ -95,12 +95,12 @@ define i32 @trunc_vXi16() {
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX1-LABEL: 'trunc_vXi16'
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i16>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i16>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i16>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i16>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i16>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i16>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i16>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i16>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i16>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i16>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i16>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = trunc <4 x i32> undef to <4 x i16>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8i32 = trunc <8 x i32> undef to <8 x i16>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16i32 = trunc <16 x i32> undef to <16 x i16>
@@ -108,12 +108,12 @@ define i32 @trunc_vXi16() {
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX2-LABEL: 'trunc_vXi16'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i16>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i16>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i16>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i16>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i16>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i16>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i16>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i16>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i16>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i16>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i16>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = trunc <4 x i32> undef to <4 x i16>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8i32 = trunc <8 x i32> undef to <8 x i16>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16i32 = trunc <16 x i32> undef to <16 x i16>
@@ -121,12 +121,12 @@ define i32 @trunc_vXi16() {
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512F-LABEL: 'trunc_vXi16'
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i16>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i16>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i16>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i16>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i16>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i16>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i16>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i16>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = trunc <4 x i32> undef to <4 x i16>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8i32 = trunc <8 x i32> undef to <8 x i16>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16i32 = trunc <16 x i32> undef to <16 x i16>
@@ -134,12 +134,12 @@ define i32 @trunc_vXi16() {
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512BW-LABEL: 'trunc_vXi16'
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i16>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i16>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i16>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i16>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i16>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i16>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i16>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i16>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = trunc <4 x i32> undef to <4 x i16>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8i32 = trunc <8 x i32> undef to <8 x i16>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16i32 = trunc <16 x i32> undef to <16 x i16>
@@ -147,12 +147,12 @@ define i32 @trunc_vXi16() {
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; BTVER2-LABEL: 'trunc_vXi16'
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i16>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i16>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i16>
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i16>
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i16>
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i16>
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i16>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i16>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i16>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i16>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i16>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = trunc <4 x i32> undef to <4 x i16>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8i32 = trunc <8 x i32> undef to <8 x i16>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16i32 = trunc <16 x i32> undef to <16 x i16>
@@ -175,19 +175,19 @@ define i32 @trunc_vXi16() {
 
 define i32 @trunc_vXi8() {
 ; SSE2-LABEL: 'trunc_vXi8'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i8>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i8>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i8>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i8>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i8>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V64i64 = trunc <64 x i64> undef to <64 x i8>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i8>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i8>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i8>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i8>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i8>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 79 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i8>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 159 for instruction: %V64i64 = trunc <64 x i64> undef to <64 x i8>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i8>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4i32 = trunc <4 x i32> undef to <4 x i8>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8i32 = trunc <8 x i32> undef to <8 x i8>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16i32 = trunc <16 x i32> undef to <16 x i8>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V32i32 = trunc <32 x i32> undef to <32 x i8>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V64i32 = trunc <64 x i32> undef to <64 x i8>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i16 = trunc <2 x i16> undef to <2 x i8>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i16 = trunc <2 x i16> undef to <2 x i8>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4i16 = trunc <4 x i16> undef to <4 x i8>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8i16 = trunc <8 x i16> undef to <8 x i8>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16i16 = trunc <16 x i16> undef to <16 x i8>
@@ -196,19 +196,19 @@ define i32 @trunc_vXi8() {
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'trunc_vXi8'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i8>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i8>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i8>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i8>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i8>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V64i64 = trunc <64 x i64> undef to <64 x i8>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i8>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i8>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i8>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i8>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i8>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 79 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i8>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 159 for instruction: %V64i64 = trunc <64 x i64> undef to <64 x i8>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i8>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4i32 = trunc <4 x i32> undef to <4 x i8>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8i32 = trunc <8 x i32> undef to <8 x i8>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16i32 = trunc <16 x i32> undef to <16 x i8>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V32i32 = trunc <32 x i32> undef to <32 x i8>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V64i32 = trunc <64 x i32> undef to <64 x i8>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i16 = trunc <2 x i16> undef to <2 x i8>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i16 = trunc <2 x i16> undef to <2 x i8>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4i16 = trunc <4 x i16> undef to <4 x i8>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8i16 = trunc <8 x i16> undef to <8 x i8>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16i16 = trunc <16 x i16> undef to <16 x i8>
@@ -217,19 +217,19 @@ define i32 @trunc_vXi8() {
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE42-LABEL: 'trunc_vXi8'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i8>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i8>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i8>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i8>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i8>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V64i64 = trunc <64 x i64> undef to <64 x i8>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i8>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i8>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i8>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i8>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i8>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i8>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %V64i64 = trunc <64 x i64> undef to <64 x i8>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i8>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = trunc <4 x i32> undef to <4 x i8>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8i32 = trunc <8 x i32> undef to <8 x i8>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16i32 = trunc <16 x i32> undef to <16 x i8>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V32i32 = trunc <32 x i32> undef to <32 x i8>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V64i32 = trunc <64 x i32> undef to <64 x i8>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i16 = trunc <2 x i16> undef to <2 x i8>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i16 = trunc <2 x i16> undef to <2 x i8>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i16 = trunc <4 x i16> undef to <4 x i8>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i16 = trunc <8 x i16> undef to <8 x i8>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16i16 = trunc <16 x i16> undef to <16 x i8>
@@ -238,19 +238,19 @@ define i32 @trunc_vXi8() {
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX1-LABEL: 'trunc_vXi8'
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i8>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i8>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i8>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i8>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i8>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i8>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64i64 = trunc <64 x i64> undef to <64 x i8>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i8>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i8>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i8>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i8>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %V64i64 = trunc <64 x i64> undef to <64 x i8>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i8>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = trunc <4 x i32> undef to <4 x i8>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8i32 = trunc <8 x i32> undef to <8 x i8>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16i32 = trunc <16 x i32> undef to <16 x i8>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V32i32 = trunc <32 x i32> undef to <32 x i8>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V64i32 = trunc <64 x i32> undef to <64 x i8>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i16 = trunc <2 x i16> undef to <2 x i8>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i16 = trunc <2 x i16> undef to <2 x i8>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i16 = trunc <4 x i16> undef to <4 x i8>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i16 = trunc <8 x i16> undef to <8 x i8>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16i16 = trunc <16 x i16> undef to <16 x i8>
@@ -259,19 +259,19 @@ define i32 @trunc_vXi8() {
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX2-LABEL: 'trunc_vXi8'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i8>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i8>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i8>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i8>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i8>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i8>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64i64 = trunc <64 x i64> undef to <64 x i8>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i8>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i8>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i8>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i8>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %V64i64 = trunc <64 x i64> undef to <64 x i8>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i8>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = trunc <4 x i32> undef to <4 x i8>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8i32 = trunc <8 x i32> undef to <8 x i8>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16i32 = trunc <16 x i32> undef to <16 x i8>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V32i32 = trunc <32 x i32> undef to <32 x i8>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V64i32 = trunc <64 x i32> undef to <64 x i8>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i16 = trunc <2 x i16> undef to <2 x i8>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i16 = trunc <2 x i16> undef to <2 x i8>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i16 = trunc <4 x i16> undef to <4 x i8>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i16 = trunc <8 x i16> undef to <8 x i8>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16i16 = trunc <16 x i16> undef to <16 x i8>
@@ -280,19 +280,19 @@ define i32 @trunc_vXi8() {
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512F-LABEL: 'trunc_vXi8'
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i8>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i8>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i8>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i8>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i8>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i8>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i8>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i8>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64i64 = trunc <64 x i64> undef to <64 x i8>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i8>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i8>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = trunc <4 x i32> undef to <4 x i8>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8i32 = trunc <8 x i32> undef to <8 x i8>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16i32 = trunc <16 x i32> undef to <16 x i8>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V32i32 = trunc <32 x i32> undef to <32 x i8>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64i32 = trunc <64 x i32> undef to <64 x i8>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i16 = trunc <2 x i16> undef to <2 x i8>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i16 = trunc <2 x i16> undef to <2 x i8>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i16 = trunc <4 x i16> undef to <4 x i8>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i16 = trunc <8 x i16> undef to <8 x i8>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16i16 = trunc <16 x i16> undef to <16 x i8>
@@ -301,19 +301,19 @@ define i32 @trunc_vXi8() {
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512BW-LABEL: 'trunc_vXi8'
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i8>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i8>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i8>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i8>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i8>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i8>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i8>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i8>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64i64 = trunc <64 x i64> undef to <64 x i8>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i8>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i8>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = trunc <4 x i32> undef to <4 x i8>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8i32 = trunc <8 x i32> undef to <8 x i8>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16i32 = trunc <16 x i32> undef to <16 x i8>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V32i32 = trunc <32 x i32> undef to <32 x i8>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64i32 = trunc <64 x i32> undef to <64 x i8>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i16 = trunc <2 x i16> undef to <2 x i8>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i16 = trunc <2 x i16> undef to <2 x i8>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i16 = trunc <4 x i16> undef to <4 x i8>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i16 = trunc <8 x i16> undef to <8 x i8>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16i16 = trunc <16 x i16> undef to <16 x i8>
@@ -322,19 +322,19 @@ define i32 @trunc_vXi8() {
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; BTVER2-LABEL: 'trunc_vXi8'
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i8>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i8>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i8>
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i8>
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i8>
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i8>
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64i64 = trunc <64 x i64> undef to <64 x i8>
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i8>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i8>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i8>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i8>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %V64i64 = trunc <64 x i64> undef to <64 x i8>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i8>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = trunc <4 x i32> undef to <4 x i8>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8i32 = trunc <8 x i32> undef to <8 x i8>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16i32 = trunc <16 x i32> undef to <16 x i8>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V32i32 = trunc <32 x i32> undef to <32 x i8>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V64i32 = trunc <64 x i32> undef to <64 x i8>
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i16 = trunc <2 x i16> undef to <2 x i8>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i16 = trunc <2 x i16> undef to <2 x i8>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i16 = trunc <4 x i16> undef to <4 x i8>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i16 = trunc <8 x i16> undef to <8 x i8>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16i16 = trunc <16 x i16> undef to <16 x i8>
diff --git a/llvm/test/Analysis/CostModel/X86/uitofp.ll b/llvm/test/Analysis/CostModel/X86/uitofp.ll
index c76ac14a6..3379bcde7 100644
--- a/llvm/test/Analysis/CostModel/X86/uitofp.ll
+++ b/llvm/test/Analysis/CostModel/X86/uitofp.ll
@@ -13,9 +13,9 @@
 define i32 @uitofp_i8_double() {
 ; SSE-LABEL: 'uitofp_i8_double'
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cvt_i8_f64 = uitofp i8 undef to double
-; SSE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %cvt_v2i8_v2f64 = uitofp <2 x i8> undef to <2 x double>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %cvt_v4i8_v4f64 = uitofp <4 x i8> undef to <4 x double>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %cvt_v8i8_v8f64 = uitofp <8 x i8> undef to <8 x double>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %cvt_v2i8_v2f64 = uitofp <2 x i8> undef to <2 x double>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %cvt_v4i8_v4f64 = uitofp <4 x i8> undef to <4 x double>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %cvt_v8i8_v8f64 = uitofp <8 x i8> undef to <8 x double>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX-LABEL: 'uitofp_i8_double'
@@ -49,8 +49,8 @@ define i32 @uitofp_i8_double() {
 define i32 @uitofp_i16_double() {
 ; SSE-LABEL: 'uitofp_i16_double'
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cvt_i16_f64 = uitofp i16 undef to double
-; SSE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %cvt_v2i16_v2f64 = uitofp <2 x i16> undef to <2 x double>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %cvt_v4i16_v4f64 = uitofp <4 x i16> undef to <4 x double>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %cvt_v2i16_v2f64 = uitofp <2 x i16> undef to <2 x double>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %cvt_v4i16_v4f64 = uitofp <4 x i16> undef to <4 x double>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %cvt_v8i16_v8f64 = uitofp <8 x i16> undef to <8 x double>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
@@ -85,7 +85,7 @@ define i32 @uitofp_i16_double() {
 define i32 @uitofp_i32_double() {
 ; SSE-LABEL: 'uitofp_i32_double'
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cvt_i32_f64 = uitofp i32 undef to double
-; SSE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %cvt_v2i32_v2f64 = uitofp <2 x i32> undef to <2 x double>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %cvt_v2i32_v2f64 = uitofp <2 x i32> undef to <2 x double>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %cvt_v4i32_v4f64 = uitofp <4 x i32> undef to <4 x double>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %cvt_v8i32_v8f64 = uitofp <8 x i32> undef to <8 x double>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
@@ -165,7 +165,7 @@ define i32 @uitofp_i8_float() {
 ; SSE-LABEL: 'uitofp_i8_float'
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cvt_i8_f32 = uitofp i8 undef to float
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %cvt_v4i8_v4f32 = uitofp <4 x i8> undef to <4 x float>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %cvt_v8i8_v8f32 = uitofp <8 x i8> undef to <8 x float>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %cvt_v8i8_v8f32 = uitofp <8 x i8> undef to <8 x float>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %cvt_v16i8_v16f32 = uitofp <16 x i8> undef to <16 x float>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
@@ -200,7 +200,7 @@ define i32 @uitofp_i8_float() {
 define i32 @uitofp_i16_float() {
 ; SSE-LABEL: 'uitofp_i16_float'
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cvt_i16_f32 = uitofp i16 undef to float
-; SSE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %cvt_v4i16_v4f32 = uitofp <4 x i16> undef to <4 x float>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %cvt_v4i16_v4f32 = uitofp <4 x i16> undef to <4 x float>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %cvt_v8i16_v8f32 = uitofp <8 x i16> undef to <8 x float>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %cvt_v16i16_v16f32 = uitofp <16 x i16> undef to <16 x float>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
@@ -278,7 +278,7 @@ define i32 @uitofp_i32_float() {
 
 define i32 @uitofp_i64_float() {
 ; SSE-LABEL: 'uitofp_i64_float'
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cvt_i64_f32 = uitofp i64 undef to float
+; SSE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %cvt_i64_f32 = uitofp i64 undef to float
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %cvt_v2i64_v2f32 = uitofp <2 x i64> undef to <2 x float>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %cvt_v4i64_v4f32 = uitofp <4 x i64> undef to <4 x float>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 60 for instruction: %cvt_v8i64_v8f32 = uitofp <8 x i64> undef to <8 x float>
@@ -286,11 +286,11 @@ define i32 @uitofp_i64_float() {
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX-LABEL: 'uitofp_i64_float'
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cvt_i64_f32 = uitofp i64 undef to float
-; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %cvt_v2i64_v2f32 = uitofp <2 x i64> undef to <2 x float>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %cvt_v4i64_v4f32 = uitofp <4 x i64> undef to <4 x float>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %cvt_v8i64_v8f32 = uitofp <8 x i64> undef to <8 x float>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %cvt_v16i64_v16f32 = uitofp <16 x i64> undef to <16 x float>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %cvt_i64_f32 = uitofp i64 undef to float
+; AVX-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %cvt_v2i64_v2f32 = uitofp <2 x i64> undef to <2 x float>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %cvt_v4i64_v4f32 = uitofp <4 x i64> undef to <4 x float>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 45 for instruction: %cvt_v8i64_v8f32 = uitofp <8 x i64> undef to <8 x float>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 91 for instruction: %cvt_v16i64_v16f32 = uitofp <16 x i64> undef to <16 x float>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512F-LABEL: 'uitofp_i64_float'
@@ -310,11 +310,11 @@ define i32 @uitofp_i64_float() {
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; BTVER2-LABEL: 'uitofp_i64_float'
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cvt_i64_f32 = uitofp i64 undef to float
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %cvt_v2i64_v2f32 = uitofp <2 x i64> undef to <2 x float>
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %cvt_v4i64_v4f32 = uitofp <4 x i64> undef to <4 x float>
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %cvt_v8i64_v8f32 = uitofp <8 x i64> undef to <8 x float>
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %cvt_v16i64_v16f32 = uitofp <16 x i64> undef to <16 x float>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %cvt_i64_f32 = uitofp i64 undef to float
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %cvt_v2i64_v2f32 = uitofp <2 x i64> undef to <2 x float>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %cvt_v4i64_v4f32 = uitofp <4 x i64> undef to <4 x float>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 45 for instruction: %cvt_v8i64_v8f32 = uitofp <8 x i64> undef to <8 x float>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 91 for instruction: %cvt_v16i64_v16f32 = uitofp <16 x i64> undef to <16 x float>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %cvt_i64_f32 = uitofp i64 undef to float
diff --git a/llvm/test/Analysis/CostModel/X86/vector-extract.ll b/llvm/test/Analysis/CostModel/X86/vector-extract.ll
index c8a90aa72..974bcc634 100644
--- a/llvm/test/Analysis/CostModel/X86/vector-extract.ll
+++ b/llvm/test/Analysis/CostModel/X86/vector-extract.ll
@@ -9,16 +9,19 @@
 ; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -cost-model -analyze -mattr=+avx512f | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F
 ; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -cost-model -analyze -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512BW
 ;
-; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -cost-model -analyze -mcpu=slm | FileCheck %s --check-prefixes=CHECK,SSE,SSE42
-; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -cost-model -analyze -mcpu=goldmont | FileCheck %s --check-prefixes=CHECK,SSE,SSE42
+; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -cost-model -analyze -mcpu=slm | FileCheck %s --check-prefixes=CHECK,SSE,SSE42,SLM
+; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -cost-model -analyze -mcpu=goldmont | FileCheck %s --check-prefixes=CHECK,SSE,SSE42,GLM
 ; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -cost-model -analyze -mcpu=btver2 | FileCheck %s --check-prefixes=BTVER2
 
 define i32 @extract_double(i32 %arg) {
 ; SSE-LABEL: 'extract_double'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2f64_a = extractelement <2 x double> undef, i32 %arg
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2f64_0 = extractelement <2 x double> undef, i32 0
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2f64_1 = extractelement <2 x double> undef, i32 1
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4f64_a = extractelement <4 x double> undef, i32 %arg
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4f64_0 = extractelement <4 x double> undef, i32 0
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4f64_3 = extractelement <4 x double> undef, i32 3
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8f64_a = extractelement <8 x double> undef, i32 %arg
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8f64_0 = extractelement <8 x double> undef, i32 0
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8f64_3 = extractelement <8 x double> undef, i32 3
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8f64_4 = extractelement <8 x double> undef, i32 4
@@ -26,10 +29,13 @@ define i32 @extract_double(i32 %arg) {
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX-LABEL: 'extract_double'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2f64_a = extractelement <2 x double> undef, i32 %arg
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2f64_0 = extractelement <2 x double> undef, i32 0
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2f64_1 = extractelement <2 x double> undef, i32 1
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4f64_a = extractelement <4 x double> undef, i32 %arg
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4f64_0 = extractelement <4 x double> undef, i32 0
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4f64_3 = extractelement <4 x double> undef, i32 3
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8f64_a = extractelement <8 x double> undef, i32 %arg
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8f64_0 = extractelement <8 x double> undef, i32 0
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8f64_3 = extractelement <8 x double> undef, i32 3
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8f64_4 = extractelement <8 x double> undef, i32 4
@@ -37,10 +43,13 @@ define i32 @extract_double(i32 %arg) {
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512-LABEL: 'extract_double'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2f64_a = extractelement <2 x double> undef, i32 %arg
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2f64_0 = extractelement <2 x double> undef, i32 0
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2f64_1 = extractelement <2 x double> undef, i32 1
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4f64_a = extractelement <4 x double> undef, i32 %arg
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4f64_0 = extractelement <4 x double> undef, i32 0
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4f64_3 = extractelement <4 x double> undef, i32 3
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8f64_a = extractelement <8 x double> undef, i32 %arg
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8f64_0 = extractelement <8 x double> undef, i32 0
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8f64_3 = extractelement <8 x double> undef, i32 3
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8f64_4 = extractelement <8 x double> undef, i32 4
@@ -48,22 +57,28 @@ define i32 @extract_double(i32 %arg) {
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; BTVER2-LABEL: 'extract_double'
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2f64_a = extractelement <2 x double> undef, i32 %arg
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2f64_0 = extractelement <2 x double> undef, i32 0
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2f64_1 = extractelement <2 x double> undef, i32 1
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4f64_a = extractelement <4 x double> undef, i32 %arg
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4f64_0 = extractelement <4 x double> undef, i32 0
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4f64_3 = extractelement <4 x double> undef, i32 3
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8f64_a = extractelement <8 x double> undef, i32 %arg
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8f64_0 = extractelement <8 x double> undef, i32 0
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8f64_3 = extractelement <8 x double> undef, i32 3
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8f64_4 = extractelement <8 x double> undef, i32 4
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8f64_7 = extractelement <8 x double> undef, i32 7
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
+  %v2f64_a = extractelement <2 x double> undef, i32 %arg
   %v2f64_0 = extractelement <2 x double> undef, i32 0
   %v2f64_1 = extractelement <2 x double> undef, i32 1
 
+  %v4f64_a = extractelement <4 x double> undef, i32 %arg
   %v4f64_0 = extractelement <4 x double> undef, i32 0
   %v4f64_3 = extractelement <4 x double> undef, i32 3
 
+  %v8f64_a = extractelement <8 x double> undef, i32 %arg
   %v8f64_0 = extractelement <8 x double> undef, i32 0
   %v8f64_3 = extractelement <8 x double> undef, i32 3
   %v8f64_4 = extractelement <8 x double> undef, i32 4
@@ -74,14 +89,18 @@ define i32 @extract_double(i32 %arg) {
 
 define i32 @extract_float(i32 %arg) {
 ; SSE-LABEL: 'extract_float'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2f32_a = extractelement <2 x float> undef, i32 %arg
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2f32_0 = extractelement <2 x float> undef, i32 0
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2f32_1 = extractelement <2 x float> undef, i32 1
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4f32_a = extractelement <4 x float> undef, i32 %arg
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4f32_0 = extractelement <4 x float> undef, i32 0
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4f32_3 = extractelement <4 x float> undef, i32 3
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8f32_a = extractelement <8 x float> undef, i32 %arg
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8f32_0 = extractelement <8 x float> undef, i32 0
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8f32_3 = extractelement <8 x float> undef, i32 3
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8f32_4 = extractelement <8 x float> undef, i32 4
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8f32_7 = extractelement <8 x float> undef, i32 7
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16f32_a = extractelement <16 x float> undef, i32 %arg
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v16f32_0 = extractelement <16 x float> undef, i32 0
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16f32_3 = extractelement <16 x float> undef, i32 3
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v16f32_8 = extractelement <16 x float> undef, i32 8
@@ -89,14 +108,18 @@ define i32 @extract_float(i32 %arg) {
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX-LABEL: 'extract_float'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2f32_a = extractelement <2 x float> undef, i32 %arg
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2f32_0 = extractelement <2 x float> undef, i32 0
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2f32_1 = extractelement <2 x float> undef, i32 1
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4f32_a = extractelement <4 x float> undef, i32 %arg
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4f32_0 = extractelement <4 x float> undef, i32 0
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4f32_3 = extractelement <4 x float> undef, i32 3
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8f32_a = extractelement <8 x float> undef, i32 %arg
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8f32_0 = extractelement <8 x float> undef, i32 0
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8f32_3 = extractelement <8 x float> undef, i32 3
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8f32_4 = extractelement <8 x float> undef, i32 4
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8f32_7 = extractelement <8 x float> undef, i32 7
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16f32_a = extractelement <16 x float> undef, i32 %arg
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v16f32_0 = extractelement <16 x float> undef, i32 0
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16f32_3 = extractelement <16 x float> undef, i32 3
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v16f32_8 = extractelement <16 x float> undef, i32 8
@@ -104,14 +127,18 @@ define i32 @extract_float(i32 %arg) {
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512-LABEL: 'extract_float'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2f32_a = extractelement <2 x float> undef, i32 %arg
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2f32_0 = extractelement <2 x float> undef, i32 0
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2f32_1 = extractelement <2 x float> undef, i32 1
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4f32_a = extractelement <4 x float> undef, i32 %arg
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4f32_0 = extractelement <4 x float> undef, i32 0
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4f32_3 = extractelement <4 x float> undef, i32 3
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8f32_a = extractelement <8 x float> undef, i32 %arg
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8f32_0 = extractelement <8 x float> undef, i32 0
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8f32_3 = extractelement <8 x float> undef, i32 3
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8f32_4 = extractelement <8 x float> undef, i32 4
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8f32_7 = extractelement <8 x float> undef, i32 7
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16f32_a = extractelement <16 x float> undef, i32 %arg
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v16f32_0 = extractelement <16 x float> undef, i32 0
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16f32_3 = extractelement <16 x float> undef, i32 3
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16f32_8 = extractelement <16 x float> undef, i32 8
@@ -119,31 +146,39 @@ define i32 @extract_float(i32 %arg) {
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; BTVER2-LABEL: 'extract_float'
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2f32_a = extractelement <2 x float> undef, i32 %arg
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2f32_0 = extractelement <2 x float> undef, i32 0
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2f32_1 = extractelement <2 x float> undef, i32 1
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4f32_a = extractelement <4 x float> undef, i32 %arg
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4f32_0 = extractelement <4 x float> undef, i32 0
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4f32_3 = extractelement <4 x float> undef, i32 3
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8f32_a = extractelement <8 x float> undef, i32 %arg
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8f32_0 = extractelement <8 x float> undef, i32 0
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8f32_3 = extractelement <8 x float> undef, i32 3
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8f32_4 = extractelement <8 x float> undef, i32 4
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8f32_7 = extractelement <8 x float> undef, i32 7
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16f32_a = extractelement <16 x float> undef, i32 %arg
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v16f32_0 = extractelement <16 x float> undef, i32 0
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16f32_3 = extractelement <16 x float> undef, i32 3
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v16f32_8 = extractelement <16 x float> undef, i32 8
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16f32_15 = extractelement <16 x float> undef, i32 15
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
+  %v2f32_a = extractelement <2 x float> undef, i32 %arg
   %v2f32_0 = extractelement <2 x float> undef, i32 0
   %v2f32_1 = extractelement <2 x float> undef, i32 1
 
+  %v4f32_a = extractelement <4 x float> undef, i32 %arg
   %v4f32_0 = extractelement <4 x float> undef, i32 0
   %v4f32_3 = extractelement <4 x float> undef, i32 3
 
+  %v8f32_a = extractelement <8 x float> undef, i32 %arg
   %v8f32_0 = extractelement <8 x float> undef, i32 0
   %v8f32_3 = extractelement <8 x float> undef, i32 3
   %v8f32_4 = extractelement <8 x float> undef, i32 4
   %v8f32_7 = extractelement <8 x float> undef, i32 7
 
+  %v16f32_a = extractelement <16 x float> undef, i32 %arg
   %v16f32_0 = extractelement <16 x float> undef, i32 0
   %v16f32_3 = extractelement <16 x float> undef, i32 3
   %v16f32_8 = extractelement <16 x float> undef, i32 8
@@ -153,34 +188,141 @@ define i32 @extract_float(i32 %arg) {
 }
 
 define i32 @extract_i64(i32 %arg) {
-; CHECK-LABEL: 'extract_i64'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64_0 = extractelement <2 x i64> undef, i32 0
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64_1 = extractelement <2 x i64> undef, i32 1
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i64_0 = extractelement <4 x i64> undef, i32 0
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i64_3 = extractelement <4 x i64> undef, i32 3
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i64_0 = extractelement <8 x i64> undef, i32 0
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i64_3 = extractelement <8 x i64> undef, i32 3
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i64_4 = extractelement <8 x i64> undef, i32 4
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i64_7 = extractelement <8 x i64> undef, i32 7
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+; SSE2-LABEL: 'extract_i64'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64_a = extractelement <2 x i64> undef, i32 %arg
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64_0 = extractelement <2 x i64> undef, i32 0
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64_1 = extractelement <2 x i64> undef, i32 1
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i64_a = extractelement <4 x i64> undef, i32 %arg
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i64_0 = extractelement <4 x i64> undef, i32 0
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i64_3 = extractelement <4 x i64> undef, i32 3
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i64_a = extractelement <8 x i64> undef, i32 %arg
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i64_0 = extractelement <8 x i64> undef, i32 0
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i64_3 = extractelement <8 x i64> undef, i32 3
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i64_4 = extractelement <8 x i64> undef, i32 4
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i64_7 = extractelement <8 x i64> undef, i32 7
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE3-LABEL: 'extract_i64'
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64_a = extractelement <2 x i64> undef, i32 %arg
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64_0 = extractelement <2 x i64> undef, i32 0
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64_1 = extractelement <2 x i64> undef, i32 1
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i64_a = extractelement <4 x i64> undef, i32 %arg
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i64_0 = extractelement <4 x i64> undef, i32 0
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i64_3 = extractelement <4 x i64> undef, i32 3
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i64_a = extractelement <8 x i64> undef, i32 %arg
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i64_0 = extractelement <8 x i64> undef, i32 0
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i64_3 = extractelement <8 x i64> undef, i32 3
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i64_4 = extractelement <8 x i64> undef, i32 4
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i64_7 = extractelement <8 x i64> undef, i32 7
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSSE3-LABEL: 'extract_i64'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64_a = extractelement <2 x i64> undef, i32 %arg
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64_0 = extractelement <2 x i64> undef, i32 0
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64_1 = extractelement <2 x i64> undef, i32 1
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i64_a = extractelement <4 x i64> undef, i32 %arg
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i64_0 = extractelement <4 x i64> undef, i32 0
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i64_3 = extractelement <4 x i64> undef, i32 3
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i64_a = extractelement <8 x i64> undef, i32 %arg
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i64_0 = extractelement <8 x i64> undef, i32 0
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i64_3 = extractelement <8 x i64> undef, i32 3
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i64_4 = extractelement <8 x i64> undef, i32 4
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i64_7 = extractelement <8 x i64> undef, i32 7
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE41-LABEL: 'extract_i64'
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64_a = extractelement <2 x i64> undef, i32 %arg
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64_0 = extractelement <2 x i64> undef, i32 0
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64_1 = extractelement <2 x i64> undef, i32 1
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i64_a = extractelement <4 x i64> undef, i32 %arg
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i64_0 = extractelement <4 x i64> undef, i32 0
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i64_3 = extractelement <4 x i64> undef, i32 3
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i64_a = extractelement <8 x i64> undef, i32 %arg
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i64_0 = extractelement <8 x i64> undef, i32 0
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i64_3 = extractelement <8 x i64> undef, i32 3
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i64_4 = extractelement <8 x i64> undef, i32 4
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i64_7 = extractelement <8 x i64> undef, i32 7
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX-LABEL: 'extract_i64'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64_a = extractelement <2 x i64> undef, i32 %arg
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64_0 = extractelement <2 x i64> undef, i32 0
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64_1 = extractelement <2 x i64> undef, i32 1
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i64_a = extractelement <4 x i64> undef, i32 %arg
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i64_0 = extractelement <4 x i64> undef, i32 0
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i64_3 = extractelement <4 x i64> undef, i32 3
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i64_a = extractelement <8 x i64> undef, i32 %arg
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i64_0 = extractelement <8 x i64> undef, i32 0
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i64_3 = extractelement <8 x i64> undef, i32 3
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i64_4 = extractelement <8 x i64> undef, i32 4
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i64_7 = extractelement <8 x i64> undef, i32 7
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512-LABEL: 'extract_i64'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64_a = extractelement <2 x i64> undef, i32 %arg
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64_0 = extractelement <2 x i64> undef, i32 0
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64_1 = extractelement <2 x i64> undef, i32 1
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i64_a = extractelement <4 x i64> undef, i32 %arg
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i64_0 = extractelement <4 x i64> undef, i32 0
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i64_3 = extractelement <4 x i64> undef, i32 3
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i64_a = extractelement <8 x i64> undef, i32 %arg
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i64_0 = extractelement <8 x i64> undef, i32 0
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i64_3 = extractelement <8 x i64> undef, i32 3
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i64_4 = extractelement <8 x i64> undef, i32 4
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i64_7 = extractelement <8 x i64> undef, i32 7
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SLM-LABEL: 'extract_i64'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64_a = extractelement <2 x i64> undef, i32 %arg
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64_0 = extractelement <2 x i64> undef, i32 0
+; SLM-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v2i64_1 = extractelement <2 x i64> undef, i32 1
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i64_a = extractelement <4 x i64> undef, i32 %arg
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i64_0 = extractelement <4 x i64> undef, i32 0
+; SLM-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v4i64_3 = extractelement <4 x i64> undef, i32 3
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i64_a = extractelement <8 x i64> undef, i32 %arg
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i64_0 = extractelement <8 x i64> undef, i32 0
+; SLM-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %v8i64_3 = extractelement <8 x i64> undef, i32 3
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i64_4 = extractelement <8 x i64> undef, i32 4
+; SLM-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %v8i64_7 = extractelement <8 x i64> undef, i32 7
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; GLM-LABEL: 'extract_i64'
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64_a = extractelement <2 x i64> undef, i32 %arg
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64_0 = extractelement <2 x i64> undef, i32 0
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64_1 = extractelement <2 x i64> undef, i32 1
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i64_a = extractelement <4 x i64> undef, i32 %arg
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i64_0 = extractelement <4 x i64> undef, i32 0
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i64_3 = extractelement <4 x i64> undef, i32 3
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i64_a = extractelement <8 x i64> undef, i32 %arg
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i64_0 = extractelement <8 x i64> undef, i32 0
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i64_3 = extractelement <8 x i64> undef, i32 3
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i64_4 = extractelement <8 x i64> undef, i32 4
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i64_7 = extractelement <8 x i64> undef, i32 7
+; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; BTVER2-LABEL: 'extract_i64'
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64_a = extractelement <2 x i64> undef, i32 %arg
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64_0 = extractelement <2 x i64> undef, i32 0
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64_1 = extractelement <2 x i64> undef, i32 1
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i64_a = extractelement <4 x i64> undef, i32 %arg
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i64_0 = extractelement <4 x i64> undef, i32 0
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i64_3 = extractelement <4 x i64> undef, i32 3
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i64_a = extractelement <8 x i64> undef, i32 %arg
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i64_0 = extractelement <8 x i64> undef, i32 0
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i64_3 = extractelement <8 x i64> undef, i32 3
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i64_4 = extractelement <8 x i64> undef, i32 4
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i64_7 = extractelement <8 x i64> undef, i32 7
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
+  %v2i64_a = extractelement <2 x i64> undef, i32 %arg
   %v2i64_0 = extractelement <2 x i64> undef, i32 0
   %v2i64_1 = extractelement <2 x i64> undef, i32 1
 
+  %v4i64_a = extractelement <4 x i64> undef, i32 %arg
   %v4i64_0 = extractelement <4 x i64> undef, i32 0
   %v4i64_3 = extractelement <4 x i64> undef, i32 3
 
+  %v8i64_a = extractelement <8 x i64> undef, i32 %arg
   %v8i64_0 = extractelement <8 x i64> undef, i32 0
   %v8i64_3 = extractelement <8 x i64> undef, i32 3
   %v8i64_4 = extractelement <8 x i64> undef, i32 4
@@ -190,47 +332,192 @@ define i32 @extract_i64(i32 %arg) {
 }
 
 define i32 @extract_i32(i32 %arg) {
-; CHECK-LABEL: 'extract_i32'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32_0 = extractelement <2 x i32> undef, i32 0
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32_1 = extractelement <2 x i32> undef, i32 1
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_0 = extractelement <4 x i32> undef, i32 0
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_3 = extractelement <4 x i32> undef, i32 3
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_0 = extractelement <8 x i32> undef, i32 0
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_3 = extractelement <8 x i32> undef, i32 3
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_4 = extractelement <8 x i32> undef, i32 4
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_7 = extractelement <8 x i32> undef, i32 7
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_0 = extractelement <16 x i32> undef, i32 0
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_3 = extractelement <16 x i32> undef, i32 3
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_8 = extractelement <16 x i32> undef, i32 8
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_15 = extractelement <16 x i32> undef, i32 15
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+; SSE2-LABEL: 'extract_i32'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32_a = extractelement <2 x i32> undef, i32 %arg
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32_0 = extractelement <2 x i32> undef, i32 0
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32_1 = extractelement <2 x i32> undef, i32 1
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_a = extractelement <4 x i32> undef, i32 %arg
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_0 = extractelement <4 x i32> undef, i32 0
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_3 = extractelement <4 x i32> undef, i32 3
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_a = extractelement <8 x i32> undef, i32 %arg
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_0 = extractelement <8 x i32> undef, i32 0
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_3 = extractelement <8 x i32> undef, i32 3
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_4 = extractelement <8 x i32> undef, i32 4
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_7 = extractelement <8 x i32> undef, i32 7
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_a = extractelement <16 x i32> undef, i32 %arg
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_0 = extractelement <16 x i32> undef, i32 0
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_3 = extractelement <16 x i32> undef, i32 3
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_8 = extractelement <16 x i32> undef, i32 8
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_15 = extractelement <16 x i32> undef, i32 15
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE3-LABEL: 'extract_i32'
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32_a = extractelement <2 x i32> undef, i32 %arg
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32_0 = extractelement <2 x i32> undef, i32 0
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32_1 = extractelement <2 x i32> undef, i32 1
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_a = extractelement <4 x i32> undef, i32 %arg
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_0 = extractelement <4 x i32> undef, i32 0
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_3 = extractelement <4 x i32> undef, i32 3
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_a = extractelement <8 x i32> undef, i32 %arg
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_0 = extractelement <8 x i32> undef, i32 0
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_3 = extractelement <8 x i32> undef, i32 3
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_4 = extractelement <8 x i32> undef, i32 4
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_7 = extractelement <8 x i32> undef, i32 7
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_a = extractelement <16 x i32> undef, i32 %arg
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_0 = extractelement <16 x i32> undef, i32 0
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_3 = extractelement <16 x i32> undef, i32 3
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_8 = extractelement <16 x i32> undef, i32 8
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_15 = extractelement <16 x i32> undef, i32 15
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSSE3-LABEL: 'extract_i32'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32_a = extractelement <2 x i32> undef, i32 %arg
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32_0 = extractelement <2 x i32> undef, i32 0
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32_1 = extractelement <2 x i32> undef, i32 1
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_a = extractelement <4 x i32> undef, i32 %arg
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_0 = extractelement <4 x i32> undef, i32 0
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_3 = extractelement <4 x i32> undef, i32 3
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_a = extractelement <8 x i32> undef, i32 %arg
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_0 = extractelement <8 x i32> undef, i32 0
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_3 = extractelement <8 x i32> undef, i32 3
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_4 = extractelement <8 x i32> undef, i32 4
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_7 = extractelement <8 x i32> undef, i32 7
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_a = extractelement <16 x i32> undef, i32 %arg
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_0 = extractelement <16 x i32> undef, i32 0
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_3 = extractelement <16 x i32> undef, i32 3
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_8 = extractelement <16 x i32> undef, i32 8
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_15 = extractelement <16 x i32> undef, i32 15
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE41-LABEL: 'extract_i32'
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32_a = extractelement <2 x i32> undef, i32 %arg
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32_0 = extractelement <2 x i32> undef, i32 0
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32_1 = extractelement <2 x i32> undef, i32 1
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_a = extractelement <4 x i32> undef, i32 %arg
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_0 = extractelement <4 x i32> undef, i32 0
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_3 = extractelement <4 x i32> undef, i32 3
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_a = extractelement <8 x i32> undef, i32 %arg
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_0 = extractelement <8 x i32> undef, i32 0
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_3 = extractelement <8 x i32> undef, i32 3
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_4 = extractelement <8 x i32> undef, i32 4
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_7 = extractelement <8 x i32> undef, i32 7
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_a = extractelement <16 x i32> undef, i32 %arg
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_0 = extractelement <16 x i32> undef, i32 0
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_3 = extractelement <16 x i32> undef, i32 3
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_8 = extractelement <16 x i32> undef, i32 8
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_15 = extractelement <16 x i32> undef, i32 15
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX-LABEL: 'extract_i32'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32_a = extractelement <2 x i32> undef, i32 %arg
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32_0 = extractelement <2 x i32> undef, i32 0
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32_1 = extractelement <2 x i32> undef, i32 1
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_a = extractelement <4 x i32> undef, i32 %arg
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_0 = extractelement <4 x i32> undef, i32 0
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_3 = extractelement <4 x i32> undef, i32 3
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_a = extractelement <8 x i32> undef, i32 %arg
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_0 = extractelement <8 x i32> undef, i32 0
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_3 = extractelement <8 x i32> undef, i32 3
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_4 = extractelement <8 x i32> undef, i32 4
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_7 = extractelement <8 x i32> undef, i32 7
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_a = extractelement <16 x i32> undef, i32 %arg
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_0 = extractelement <16 x i32> undef, i32 0
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_3 = extractelement <16 x i32> undef, i32 3
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_8 = extractelement <16 x i32> undef, i32 8
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_15 = extractelement <16 x i32> undef, i32 15
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512-LABEL: 'extract_i32'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32_a = extractelement <2 x i32> undef, i32 %arg
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32_0 = extractelement <2 x i32> undef, i32 0
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32_1 = extractelement <2 x i32> undef, i32 1
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_a = extractelement <4 x i32> undef, i32 %arg
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_0 = extractelement <4 x i32> undef, i32 0
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_3 = extractelement <4 x i32> undef, i32 3
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_a = extractelement <8 x i32> undef, i32 %arg
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_0 = extractelement <8 x i32> undef, i32 0
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_3 = extractelement <8 x i32> undef, i32 3
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_4 = extractelement <8 x i32> undef, i32 4
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_7 = extractelement <8 x i32> undef, i32 7
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_a = extractelement <16 x i32> undef, i32 %arg
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_0 = extractelement <16 x i32> undef, i32 0
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_3 = extractelement <16 x i32> undef, i32 3
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_8 = extractelement <16 x i32> undef, i32 8
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_15 = extractelement <16 x i32> undef, i32 15
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SLM-LABEL: 'extract_i32'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32_a = extractelement <2 x i32> undef, i32 %arg
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32_0 = extractelement <2 x i32> undef, i32 0
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2i32_1 = extractelement <2 x i32> undef, i32 1
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_a = extractelement <4 x i32> undef, i32 %arg
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_0 = extractelement <4 x i32> undef, i32 0
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4i32_3 = extractelement <4 x i32> undef, i32 3
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_a = extractelement <8 x i32> undef, i32 %arg
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_0 = extractelement <8 x i32> undef, i32 0
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8i32_3 = extractelement <8 x i32> undef, i32 3
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_4 = extractelement <8 x i32> undef, i32 4
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8i32_7 = extractelement <8 x i32> undef, i32 7
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_a = extractelement <16 x i32> undef, i32 %arg
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_0 = extractelement <16 x i32> undef, i32 0
+; SLM-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v16i32_3 = extractelement <16 x i32> undef, i32 3
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_8 = extractelement <16 x i32> undef, i32 8
+; SLM-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v16i32_15 = extractelement <16 x i32> undef, i32 15
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; GLM-LABEL: 'extract_i32'
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32_a = extractelement <2 x i32> undef, i32 %arg
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32_0 = extractelement <2 x i32> undef, i32 0
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32_1 = extractelement <2 x i32> undef, i32 1
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_a = extractelement <4 x i32> undef, i32 %arg
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_0 = extractelement <4 x i32> undef, i32 0
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_3 = extractelement <4 x i32> undef, i32 3
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_a = extractelement <8 x i32> undef, i32 %arg
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_0 = extractelement <8 x i32> undef, i32 0
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_3 = extractelement <8 x i32> undef, i32 3
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_4 = extractelement <8 x i32> undef, i32 4
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_7 = extractelement <8 x i32> undef, i32 7
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_a = extractelement <16 x i32> undef, i32 %arg
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_0 = extractelement <16 x i32> undef, i32 0
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_3 = extractelement <16 x i32> undef, i32 3
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_8 = extractelement <16 x i32> undef, i32 8
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_15 = extractelement <16 x i32> undef, i32 15
+; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; BTVER2-LABEL: 'extract_i32'
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32_a = extractelement <2 x i32> undef, i32 %arg
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32_0 = extractelement <2 x i32> undef, i32 0
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32_1 = extractelement <2 x i32> undef, i32 1
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_a = extractelement <4 x i32> undef, i32 %arg
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_0 = extractelement <4 x i32> undef, i32 0
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_3 = extractelement <4 x i32> undef, i32 3
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_a = extractelement <8 x i32> undef, i32 %arg
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_0 = extractelement <8 x i32> undef, i32 0
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_3 = extractelement <8 x i32> undef, i32 3
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_4 = extractelement <8 x i32> undef, i32 4
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_7 = extractelement <8 x i32> undef, i32 7
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_a = extractelement <16 x i32> undef, i32 %arg
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_0 = extractelement <16 x i32> undef, i32 0
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_3 = extractelement <16 x i32> undef, i32 3
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_8 = extractelement <16 x i32> undef, i32 8
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_15 = extractelement <16 x i32> undef, i32 15
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
+  %v2i32_a = extractelement <2 x i32> undef, i32 %arg
   %v2i32_0 = extractelement <2 x i32> undef, i32 0
   %v2i32_1 = extractelement <2 x i32> undef, i32 1
 
+  %v4i32_a = extractelement <4 x i32> undef, i32 %arg
   %v4i32_0 = extractelement <4 x i32> undef, i32 0
   %v4i32_3 = extractelement <4 x i32> undef, i32 3
 
+  %v8i32_a = extractelement <8 x i32> undef, i32 %arg
   %v8i32_0 = extractelement <8 x i32> undef, i32 0
   %v8i32_3 = extractelement <8 x i32> undef, i32 3
   %v8i32_4 = extractelement <8 x i32> undef, i32 4
   %v8i32_7 = extractelement <8 x i32> undef, i32 7
 
+  %v16i32_a = extractelement <16 x i32> undef, i32 %arg
   %v16i32_0 = extractelement <16 x i32> undef, i32 0
   %v16i32_3 = extractelement <16 x i32> undef, i32 3
   %v16i32_8 = extractelement <16 x i32> undef, i32 8
@@ -240,29 +527,168 @@ define i32 @extract_i32(i32 %arg) {
 }
 
 define i32 @extract_i16(i32 %arg) {
-; CHECK-LABEL: 'extract_i16'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_0 = extractelement <8 x i16> undef, i32 0
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_7 = extractelement <8 x i16> undef, i32 7
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_0 = extractelement <16 x i16> undef, i32 0
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_7 = extractelement <16 x i16> undef, i32 7
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_8 = extractelement <16 x i16> undef, i32 8
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_15 = extractelement <16 x i16> undef, i32 15
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_0 = extractelement <32 x i16> undef, i32 0
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_7 = extractelement <32 x i16> undef, i32 7
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_8 = extractelement <32 x i16> undef, i32 8
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_15 = extractelement <32 x i16> undef, i32 15
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_16 = extractelement <32 x i16> undef, i32 16
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_24 = extractelement <32 x i16> undef, i32 24
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_31 = extractelement <32 x i16> undef, i32 31
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+; SSE2-LABEL: 'extract_i16'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_a = extractelement <8 x i16> undef, i32 %arg
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_0 = extractelement <8 x i16> undef, i32 0
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_7 = extractelement <8 x i16> undef, i32 7
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_a = extractelement <16 x i16> undef, i32 %arg
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_0 = extractelement <16 x i16> undef, i32 0
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_7 = extractelement <16 x i16> undef, i32 7
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_8 = extractelement <16 x i16> undef, i32 8
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_15 = extractelement <16 x i16> undef, i32 15
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_a = extractelement <32 x i16> undef, i32 %arg
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_0 = extractelement <32 x i16> undef, i32 0
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_7 = extractelement <32 x i16> undef, i32 7
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_8 = extractelement <32 x i16> undef, i32 8
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_15 = extractelement <32 x i16> undef, i32 15
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_16 = extractelement <32 x i16> undef, i32 16
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_24 = extractelement <32 x i16> undef, i32 24
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_31 = extractelement <32 x i16> undef, i32 31
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE3-LABEL: 'extract_i16'
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_a = extractelement <8 x i16> undef, i32 %arg
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_0 = extractelement <8 x i16> undef, i32 0
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_7 = extractelement <8 x i16> undef, i32 7
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_a = extractelement <16 x i16> undef, i32 %arg
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_0 = extractelement <16 x i16> undef, i32 0
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_7 = extractelement <16 x i16> undef, i32 7
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_8 = extractelement <16 x i16> undef, i32 8
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_15 = extractelement <16 x i16> undef, i32 15
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_a = extractelement <32 x i16> undef, i32 %arg
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_0 = extractelement <32 x i16> undef, i32 0
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_7 = extractelement <32 x i16> undef, i32 7
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_8 = extractelement <32 x i16> undef, i32 8
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_15 = extractelement <32 x i16> undef, i32 15
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_16 = extractelement <32 x i16> undef, i32 16
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_24 = extractelement <32 x i16> undef, i32 24
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_31 = extractelement <32 x i16> undef, i32 31
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSSE3-LABEL: 'extract_i16'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_a = extractelement <8 x i16> undef, i32 %arg
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_0 = extractelement <8 x i16> undef, i32 0
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_7 = extractelement <8 x i16> undef, i32 7
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_a = extractelement <16 x i16> undef, i32 %arg
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_0 = extractelement <16 x i16> undef, i32 0
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_7 = extractelement <16 x i16> undef, i32 7
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_8 = extractelement <16 x i16> undef, i32 8
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_15 = extractelement <16 x i16> undef, i32 15
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_a = extractelement <32 x i16> undef, i32 %arg
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_0 = extractelement <32 x i16> undef, i32 0
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_7 = extractelement <32 x i16> undef, i32 7
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_8 = extractelement <32 x i16> undef, i32 8
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_15 = extractelement <32 x i16> undef, i32 15
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_16 = extractelement <32 x i16> undef, i32 16
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_24 = extractelement <32 x i16> undef, i32 24
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_31 = extractelement <32 x i16> undef, i32 31
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE41-LABEL: 'extract_i16'
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_a = extractelement <8 x i16> undef, i32 %arg
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_0 = extractelement <8 x i16> undef, i32 0
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_7 = extractelement <8 x i16> undef, i32 7
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_a = extractelement <16 x i16> undef, i32 %arg
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_0 = extractelement <16 x i16> undef, i32 0
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_7 = extractelement <16 x i16> undef, i32 7
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_8 = extractelement <16 x i16> undef, i32 8
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_15 = extractelement <16 x i16> undef, i32 15
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_a = extractelement <32 x i16> undef, i32 %arg
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_0 = extractelement <32 x i16> undef, i32 0
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_7 = extractelement <32 x i16> undef, i32 7
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_8 = extractelement <32 x i16> undef, i32 8
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_15 = extractelement <32 x i16> undef, i32 15
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_16 = extractelement <32 x i16> undef, i32 16
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_24 = extractelement <32 x i16> undef, i32 24
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_31 = extractelement <32 x i16> undef, i32 31
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX-LABEL: 'extract_i16'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_a = extractelement <8 x i16> undef, i32 %arg
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_0 = extractelement <8 x i16> undef, i32 0
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_7 = extractelement <8 x i16> undef, i32 7
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_a = extractelement <16 x i16> undef, i32 %arg
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_0 = extractelement <16 x i16> undef, i32 0
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_7 = extractelement <16 x i16> undef, i32 7
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_8 = extractelement <16 x i16> undef, i32 8
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_15 = extractelement <16 x i16> undef, i32 15
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_a = extractelement <32 x i16> undef, i32 %arg
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_0 = extractelement <32 x i16> undef, i32 0
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_7 = extractelement <32 x i16> undef, i32 7
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_8 = extractelement <32 x i16> undef, i32 8
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_15 = extractelement <32 x i16> undef, i32 15
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_16 = extractelement <32 x i16> undef, i32 16
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_24 = extractelement <32 x i16> undef, i32 24
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_31 = extractelement <32 x i16> undef, i32 31
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512-LABEL: 'extract_i16'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_a = extractelement <8 x i16> undef, i32 %arg
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_0 = extractelement <8 x i16> undef, i32 0
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_7 = extractelement <8 x i16> undef, i32 7
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_a = extractelement <16 x i16> undef, i32 %arg
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_0 = extractelement <16 x i16> undef, i32 0
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_7 = extractelement <16 x i16> undef, i32 7
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_8 = extractelement <16 x i16> undef, i32 8
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_15 = extractelement <16 x i16> undef, i32 15
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_a = extractelement <32 x i16> undef, i32 %arg
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_0 = extractelement <32 x i16> undef, i32 0
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_7 = extractelement <32 x i16> undef, i32 7
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_8 = extractelement <32 x i16> undef, i32 8
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_15 = extractelement <32 x i16> undef, i32 15
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_16 = extractelement <32 x i16> undef, i32 16
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_24 = extractelement <32 x i16> undef, i32 24
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_31 = extractelement <32 x i16> undef, i32 31
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SLM-LABEL: 'extract_i16'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_a = extractelement <8 x i16> undef, i32 %arg
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_0 = extractelement <8 x i16> undef, i32 0
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i16_7 = extractelement <8 x i16> undef, i32 7
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_a = extractelement <16 x i16> undef, i32 %arg
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_0 = extractelement <16 x i16> undef, i32 0
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i16_7 = extractelement <16 x i16> undef, i32 7
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_8 = extractelement <16 x i16> undef, i32 8
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i16_15 = extractelement <16 x i16> undef, i32 15
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_a = extractelement <32 x i16> undef, i32 %arg
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_0 = extractelement <32 x i16> undef, i32 0
+; SLM-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v32i16_7 = extractelement <32 x i16> undef, i32 7
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_8 = extractelement <32 x i16> undef, i32 8
+; SLM-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v32i16_15 = extractelement <32 x i16> undef, i32 15
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_16 = extractelement <32 x i16> undef, i32 16
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_24 = extractelement <32 x i16> undef, i32 24
+; SLM-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v32i16_31 = extractelement <32 x i16> undef, i32 31
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; GLM-LABEL: 'extract_i16'
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_a = extractelement <8 x i16> undef, i32 %arg
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_0 = extractelement <8 x i16> undef, i32 0
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_7 = extractelement <8 x i16> undef, i32 7
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_a = extractelement <16 x i16> undef, i32 %arg
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_0 = extractelement <16 x i16> undef, i32 0
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_7 = extractelement <16 x i16> undef, i32 7
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_8 = extractelement <16 x i16> undef, i32 8
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_15 = extractelement <16 x i16> undef, i32 15
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_a = extractelement <32 x i16> undef, i32 %arg
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_0 = extractelement <32 x i16> undef, i32 0
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_7 = extractelement <32 x i16> undef, i32 7
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_8 = extractelement <32 x i16> undef, i32 8
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_15 = extractelement <32 x i16> undef, i32 15
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_16 = extractelement <32 x i16> undef, i32 16
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_24 = extractelement <32 x i16> undef, i32 24
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_31 = extractelement <32 x i16> undef, i32 31
+; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; BTVER2-LABEL: 'extract_i16'
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_a = extractelement <8 x i16> undef, i32 %arg
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_0 = extractelement <8 x i16> undef, i32 0
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_7 = extractelement <8 x i16> undef, i32 7
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_a = extractelement <16 x i16> undef, i32 %arg
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_0 = extractelement <16 x i16> undef, i32 0
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_7 = extractelement <16 x i16> undef, i32 7
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_8 = extractelement <16 x i16> undef, i32 8
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_15 = extractelement <16 x i16> undef, i32 15
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_a = extractelement <32 x i16> undef, i32 %arg
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_0 = extractelement <32 x i16> undef, i32 0
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_7 = extractelement <32 x i16> undef, i32 7
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_8 = extractelement <32 x i16> undef, i32 8
@@ -272,14 +698,17 @@ define i32 @extract_i16(i32 %arg) {
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_31 = extractelement <32 x i16> undef, i32 31
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
+  %v8i16_a = extractelement <8 x i16> undef, i32 %arg
   %v8i16_0 = extractelement <8 x i16> undef, i32 0
   %v8i16_7 = extractelement <8 x i16> undef, i32 7
 
+  %v16i16_a = extractelement <16 x i16> undef, i32 %arg
   %v16i16_0 = extractelement <16 x i16> undef, i32 0
   %v16i16_7 = extractelement <16 x i16> undef, i32 7
   %v16i16_8 = extractelement <16 x i16> undef, i32 8
   %v16i16_15 = extractelement <16 x i16> undef, i32 15
 
+  %v32i16_a = extractelement <32 x i16> undef, i32 %arg
   %v32i16_0 = extractelement <32 x i16> undef, i32 0
   %v32i16_7 = extractelement <32 x i16> undef, i32 7
   %v32i16_8 = extractelement <32 x i16> undef, i32 8
@@ -292,37 +721,211 @@ define i32 @extract_i16(i32 %arg) {
 }
 
 define i32 @extract_i8(i32 %arg) {
-; CHECK-LABEL: 'extract_i8'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_0 = extractelement <16 x i8> undef, i32 0
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_8 = extractelement <16 x i8> undef, i32 8
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_15 = extractelement <16 x i8> undef, i32 15
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_0 = extractelement <32 x i8> undef, i32 0
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_7 = extractelement <32 x i8> undef, i32 7
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_8 = extractelement <32 x i8> undef, i32 8
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_15 = extractelement <32 x i8> undef, i32 15
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_24 = extractelement <32 x i8> undef, i32 24
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_31 = extractelement <32 x i8> undef, i32 31
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_0 = extractelement <64 x i8> undef, i32 0
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_7 = extractelement <64 x i8> undef, i32 7
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_8 = extractelement <64 x i8> undef, i32 8
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_15 = extractelement <64 x i8> undef, i32 15
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_24 = extractelement <64 x i8> undef, i32 24
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_31 = extractelement <64 x i8> undef, i32 31
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_32 = extractelement <64 x i8> undef, i32 32
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_48 = extractelement <64 x i8> undef, i32 48
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_63 = extractelement <64 x i8> undef, i32 63
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+; SSE2-LABEL: 'extract_i8'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_a = extractelement <16 x i8> undef, i32 %arg
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_0 = extractelement <16 x i8> undef, i32 0
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_8 = extractelement <16 x i8> undef, i32 8
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_15 = extractelement <16 x i8> undef, i32 15
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_a = extractelement <32 x i8> undef, i32 %arg
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_0 = extractelement <32 x i8> undef, i32 0
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_7 = extractelement <32 x i8> undef, i32 7
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_8 = extractelement <32 x i8> undef, i32 8
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_15 = extractelement <32 x i8> undef, i32 15
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_24 = extractelement <32 x i8> undef, i32 24
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_31 = extractelement <32 x i8> undef, i32 31
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_a = extractelement <64 x i8> undef, i32 %arg
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_0 = extractelement <64 x i8> undef, i32 0
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_7 = extractelement <64 x i8> undef, i32 7
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_8 = extractelement <64 x i8> undef, i32 8
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_15 = extractelement <64 x i8> undef, i32 15
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_24 = extractelement <64 x i8> undef, i32 24
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_31 = extractelement <64 x i8> undef, i32 31
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_32 = extractelement <64 x i8> undef, i32 32
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_48 = extractelement <64 x i8> undef, i32 48
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_63 = extractelement <64 x i8> undef, i32 63
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE3-LABEL: 'extract_i8'
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_a = extractelement <16 x i8> undef, i32 %arg
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_0 = extractelement <16 x i8> undef, i32 0
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_8 = extractelement <16 x i8> undef, i32 8
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_15 = extractelement <16 x i8> undef, i32 15
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_a = extractelement <32 x i8> undef, i32 %arg
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_0 = extractelement <32 x i8> undef, i32 0
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_7 = extractelement <32 x i8> undef, i32 7
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_8 = extractelement <32 x i8> undef, i32 8
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_15 = extractelement <32 x i8> undef, i32 15
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_24 = extractelement <32 x i8> undef, i32 24
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_31 = extractelement <32 x i8> undef, i32 31
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_a = extractelement <64 x i8> undef, i32 %arg
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_0 = extractelement <64 x i8> undef, i32 0
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_7 = extractelement <64 x i8> undef, i32 7
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_8 = extractelement <64 x i8> undef, i32 8
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_15 = extractelement <64 x i8> undef, i32 15
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_24 = extractelement <64 x i8> undef, i32 24
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_31 = extractelement <64 x i8> undef, i32 31
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_32 = extractelement <64 x i8> undef, i32 32
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_48 = extractelement <64 x i8> undef, i32 48
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_63 = extractelement <64 x i8> undef, i32 63
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSSE3-LABEL: 'extract_i8'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_a = extractelement <16 x i8> undef, i32 %arg
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_0 = extractelement <16 x i8> undef, i32 0
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_8 = extractelement <16 x i8> undef, i32 8
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_15 = extractelement <16 x i8> undef, i32 15
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_a = extractelement <32 x i8> undef, i32 %arg
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_0 = extractelement <32 x i8> undef, i32 0
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_7 = extractelement <32 x i8> undef, i32 7
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_8 = extractelement <32 x i8> undef, i32 8
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_15 = extractelement <32 x i8> undef, i32 15
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_24 = extractelement <32 x i8> undef, i32 24
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_31 = extractelement <32 x i8> undef, i32 31
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_a = extractelement <64 x i8> undef, i32 %arg
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_0 = extractelement <64 x i8> undef, i32 0
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_7 = extractelement <64 x i8> undef, i32 7
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_8 = extractelement <64 x i8> undef, i32 8
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_15 = extractelement <64 x i8> undef, i32 15
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_24 = extractelement <64 x i8> undef, i32 24
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_31 = extractelement <64 x i8> undef, i32 31
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_32 = extractelement <64 x i8> undef, i32 32
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_48 = extractelement <64 x i8> undef, i32 48
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_63 = extractelement <64 x i8> undef, i32 63
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE41-LABEL: 'extract_i8'
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_a = extractelement <16 x i8> undef, i32 %arg
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_0 = extractelement <16 x i8> undef, i32 0
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_8 = extractelement <16 x i8> undef, i32 8
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_15 = extractelement <16 x i8> undef, i32 15
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_a = extractelement <32 x i8> undef, i32 %arg
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_0 = extractelement <32 x i8> undef, i32 0
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_7 = extractelement <32 x i8> undef, i32 7
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_8 = extractelement <32 x i8> undef, i32 8
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_15 = extractelement <32 x i8> undef, i32 15
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_24 = extractelement <32 x i8> undef, i32 24
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_31 = extractelement <32 x i8> undef, i32 31
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_a = extractelement <64 x i8> undef, i32 %arg
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_0 = extractelement <64 x i8> undef, i32 0
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_7 = extractelement <64 x i8> undef, i32 7
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_8 = extractelement <64 x i8> undef, i32 8
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_15 = extractelement <64 x i8> undef, i32 15
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_24 = extractelement <64 x i8> undef, i32 24
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_31 = extractelement <64 x i8> undef, i32 31
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_32 = extractelement <64 x i8> undef, i32 32
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_48 = extractelement <64 x i8> undef, i32 48
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_63 = extractelement <64 x i8> undef, i32 63
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX-LABEL: 'extract_i8'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_a = extractelement <16 x i8> undef, i32 %arg
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_0 = extractelement <16 x i8> undef, i32 0
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_8 = extractelement <16 x i8> undef, i32 8
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_15 = extractelement <16 x i8> undef, i32 15
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_a = extractelement <32 x i8> undef, i32 %arg
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_0 = extractelement <32 x i8> undef, i32 0
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_7 = extractelement <32 x i8> undef, i32 7
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_8 = extractelement <32 x i8> undef, i32 8
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_15 = extractelement <32 x i8> undef, i32 15
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_24 = extractelement <32 x i8> undef, i32 24
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_31 = extractelement <32 x i8> undef, i32 31
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_a = extractelement <64 x i8> undef, i32 %arg
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_0 = extractelement <64 x i8> undef, i32 0
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_7 = extractelement <64 x i8> undef, i32 7
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_8 = extractelement <64 x i8> undef, i32 8
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_15 = extractelement <64 x i8> undef, i32 15
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_24 = extractelement <64 x i8> undef, i32 24
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_31 = extractelement <64 x i8> undef, i32 31
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_32 = extractelement <64 x i8> undef, i32 32
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_48 = extractelement <64 x i8> undef, i32 48
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_63 = extractelement <64 x i8> undef, i32 63
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512-LABEL: 'extract_i8'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_a = extractelement <16 x i8> undef, i32 %arg
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_0 = extractelement <16 x i8> undef, i32 0
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_8 = extractelement <16 x i8> undef, i32 8
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_15 = extractelement <16 x i8> undef, i32 15
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_a = extractelement <32 x i8> undef, i32 %arg
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_0 = extractelement <32 x i8> undef, i32 0
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_7 = extractelement <32 x i8> undef, i32 7
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_8 = extractelement <32 x i8> undef, i32 8
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_15 = extractelement <32 x i8> undef, i32 15
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_24 = extractelement <32 x i8> undef, i32 24
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_31 = extractelement <32 x i8> undef, i32 31
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_a = extractelement <64 x i8> undef, i32 %arg
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_0 = extractelement <64 x i8> undef, i32 0
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_7 = extractelement <64 x i8> undef, i32 7
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_8 = extractelement <64 x i8> undef, i32 8
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_15 = extractelement <64 x i8> undef, i32 15
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_24 = extractelement <64 x i8> undef, i32 24
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_31 = extractelement <64 x i8> undef, i32 31
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_32 = extractelement <64 x i8> undef, i32 32
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_48 = extractelement <64 x i8> undef, i32 48
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_63 = extractelement <64 x i8> undef, i32 63
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SLM-LABEL: 'extract_i8'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_a = extractelement <16 x i8> undef, i32 %arg
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_0 = extractelement <16 x i8> undef, i32 0
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i8_8 = extractelement <16 x i8> undef, i32 8
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i8_15 = extractelement <16 x i8> undef, i32 15
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_a = extractelement <32 x i8> undef, i32 %arg
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_0 = extractelement <32 x i8> undef, i32 0
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32i8_7 = extractelement <32 x i8> undef, i32 7
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32i8_8 = extractelement <32 x i8> undef, i32 8
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32i8_15 = extractelement <32 x i8> undef, i32 15
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32i8_24 = extractelement <32 x i8> undef, i32 24
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32i8_31 = extractelement <32 x i8> undef, i32 31
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_a = extractelement <64 x i8> undef, i32 %arg
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_0 = extractelement <64 x i8> undef, i32 0
+; SLM-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v64i8_7 = extractelement <64 x i8> undef, i32 7
+; SLM-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v64i8_8 = extractelement <64 x i8> undef, i32 8
+; SLM-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v64i8_15 = extractelement <64 x i8> undef, i32 15
+; SLM-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v64i8_24 = extractelement <64 x i8> undef, i32 24
+; SLM-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v64i8_31 = extractelement <64 x i8> undef, i32 31
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_32 = extractelement <64 x i8> undef, i32 32
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_48 = extractelement <64 x i8> undef, i32 48
+; SLM-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v64i8_63 = extractelement <64 x i8> undef, i32 63
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; GLM-LABEL: 'extract_i8'
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_a = extractelement <16 x i8> undef, i32 %arg
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_0 = extractelement <16 x i8> undef, i32 0
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_8 = extractelement <16 x i8> undef, i32 8
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_15 = extractelement <16 x i8> undef, i32 15
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_a = extractelement <32 x i8> undef, i32 %arg
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_0 = extractelement <32 x i8> undef, i32 0
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_7 = extractelement <32 x i8> undef, i32 7
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_8 = extractelement <32 x i8> undef, i32 8
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_15 = extractelement <32 x i8> undef, i32 15
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_24 = extractelement <32 x i8> undef, i32 24
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_31 = extractelement <32 x i8> undef, i32 31
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_a = extractelement <64 x i8> undef, i32 %arg
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_0 = extractelement <64 x i8> undef, i32 0
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_7 = extractelement <64 x i8> undef, i32 7
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_8 = extractelement <64 x i8> undef, i32 8
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_15 = extractelement <64 x i8> undef, i32 15
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_24 = extractelement <64 x i8> undef, i32 24
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_31 = extractelement <64 x i8> undef, i32 31
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_32 = extractelement <64 x i8> undef, i32 32
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_48 = extractelement <64 x i8> undef, i32 48
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_63 = extractelement <64 x i8> undef, i32 63
+; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; BTVER2-LABEL: 'extract_i8'
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_a = extractelement <16 x i8> undef, i32 %arg
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_0 = extractelement <16 x i8> undef, i32 0
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_8 = extractelement <16 x i8> undef, i32 8
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_15 = extractelement <16 x i8> undef, i32 15
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_a = extractelement <32 x i8> undef, i32 %arg
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_0 = extractelement <32 x i8> undef, i32 0
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_7 = extractelement <32 x i8> undef, i32 7
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_8 = extractelement <32 x i8> undef, i32 8
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_15 = extractelement <32 x i8> undef, i32 15
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_24 = extractelement <32 x i8> undef, i32 24
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_31 = extractelement <32 x i8> undef, i32 31
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_a = extractelement <64 x i8> undef, i32 %arg
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_0 = extractelement <64 x i8> undef, i32 0
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_7 = extractelement <64 x i8> undef, i32 7
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_8 = extractelement <64 x i8> undef, i32 8
@@ -334,10 +937,12 @@ define i32 @extract_i8(i32 %arg) {
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_63 = extractelement <64 x i8> undef, i32 63
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
+  %v16i8_a = extractelement <16 x i8> undef, i32 %arg
   %v16i8_0 = extractelement <16 x i8> undef, i32 0
   %v16i8_8 = extractelement <16 x i8> undef, i32 8
   %v16i8_15 = extractelement <16 x i8> undef, i32 15
 
+  %v32i8_a = extractelement <32 x i8> undef, i32 %arg
   %v32i8_0 = extractelement <32 x i8> undef, i32 0
   %v32i8_7 = extractelement <32 x i8> undef, i32 7
   %v32i8_8 = extractelement <32 x i8> undef, i32 8
@@ -345,6 +950,7 @@ define i32 @extract_i8(i32 %arg) {
   %v32i8_24 = extractelement <32 x i8> undef, i32 24
   %v32i8_31 = extractelement <32 x i8> undef, i32 31
 
+  %v64i8_a = extractelement <64 x i8> undef, i32 %arg
   %v64i8_0 = extractelement <64 x i8> undef, i32 0
   %v64i8_7 = extractelement <64 x i8> undef, i32 7
   %v64i8_8 = extractelement <64 x i8> undef, i32 8
diff --git a/llvm/test/Analysis/CostModel/X86/vector-insert.ll b/llvm/test/Analysis/CostModel/X86/vector-insert.ll
index 2df0137d6..68b1af571 100644
--- a/llvm/test/Analysis/CostModel/X86/vector-insert.ll
+++ b/llvm/test/Analysis/CostModel/X86/vector-insert.ll
@@ -15,10 +15,13 @@
 
 define i32 @insert_double(i32 %arg) {
 ; SSE-LABEL: 'insert_double'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2f64_a = insertelement <2 x double> undef, double undef, i32 %arg
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2f64_0 = insertelement <2 x double> undef, double undef, i32 0
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2f64_1 = insertelement <2 x double> undef, double undef, i32 1
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4f64_a = insertelement <4 x double> undef, double undef, i32 %arg
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4f64_0 = insertelement <4 x double> undef, double undef, i32 0
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4f64_3 = insertelement <4 x double> undef, double undef, i32 3
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8f64_a = insertelement <8 x double> undef, double undef, i32 %arg
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8f64_0 = insertelement <8 x double> undef, double undef, i32 0
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8f64_3 = insertelement <8 x double> undef, double undef, i32 3
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8f64_4 = insertelement <8 x double> undef, double undef, i32 4
@@ -26,10 +29,13 @@ define i32 @insert_double(i32 %arg) {
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX-LABEL: 'insert_double'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2f64_a = insertelement <2 x double> undef, double undef, i32 %arg
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2f64_0 = insertelement <2 x double> undef, double undef, i32 0
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2f64_1 = insertelement <2 x double> undef, double undef, i32 1
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4f64_a = insertelement <4 x double> undef, double undef, i32 %arg
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4f64_0 = insertelement <4 x double> undef, double undef, i32 0
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4f64_3 = insertelement <4 x double> undef, double undef, i32 3
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8f64_a = insertelement <8 x double> undef, double undef, i32 %arg
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8f64_0 = insertelement <8 x double> undef, double undef, i32 0
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8f64_3 = insertelement <8 x double> undef, double undef, i32 3
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8f64_4 = insertelement <8 x double> undef, double undef, i32 4
@@ -37,10 +43,13 @@ define i32 @insert_double(i32 %arg) {
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512-LABEL: 'insert_double'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2f64_a = insertelement <2 x double> undef, double undef, i32 %arg
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2f64_0 = insertelement <2 x double> undef, double undef, i32 0
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2f64_1 = insertelement <2 x double> undef, double undef, i32 1
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4f64_a = insertelement <4 x double> undef, double undef, i32 %arg
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4f64_0 = insertelement <4 x double> undef, double undef, i32 0
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4f64_3 = insertelement <4 x double> undef, double undef, i32 3
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8f64_a = insertelement <8 x double> undef, double undef, i32 %arg
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8f64_0 = insertelement <8 x double> undef, double undef, i32 0
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8f64_3 = insertelement <8 x double> undef, double undef, i32 3
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8f64_4 = insertelement <8 x double> undef, double undef, i32 4
@@ -48,22 +57,28 @@ define i32 @insert_double(i32 %arg) {
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; BTVER2-LABEL: 'insert_double'
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2f64_a = insertelement <2 x double> undef, double undef, i32 %arg
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2f64_0 = insertelement <2 x double> undef, double undef, i32 0
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2f64_1 = insertelement <2 x double> undef, double undef, i32 1
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4f64_a = insertelement <4 x double> undef, double undef, i32 %arg
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4f64_0 = insertelement <4 x double> undef, double undef, i32 0
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4f64_3 = insertelement <4 x double> undef, double undef, i32 3
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8f64_a = insertelement <8 x double> undef, double undef, i32 %arg
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8f64_0 = insertelement <8 x double> undef, double undef, i32 0
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8f64_3 = insertelement <8 x double> undef, double undef, i32 3
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8f64_4 = insertelement <8 x double> undef, double undef, i32 4
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8f64_7 = insertelement <8 x double> undef, double undef, i32 7
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
+  %v2f64_a = insertelement <2 x double> undef, double undef, i32 %arg
   %v2f64_0 = insertelement <2 x double> undef, double undef, i32 0
   %v2f64_1 = insertelement <2 x double> undef, double undef, i32 1
 
+  %v4f64_a = insertelement <4 x double> undef, double undef, i32 %arg
   %v4f64_0 = insertelement <4 x double> undef, double undef, i32 0
   %v4f64_3 = insertelement <4 x double> undef, double undef, i32 3
 
+  %v8f64_a = insertelement <8 x double> undef, double undef, i32 %arg
   %v8f64_0 = insertelement <8 x double> undef, double undef, i32 0
   %v8f64_3 = insertelement <8 x double> undef, double undef, i32 3
   %v8f64_4 = insertelement <8 x double> undef, double undef, i32 4
@@ -74,14 +89,18 @@ define i32 @insert_double(i32 %arg) {
 
 define i32 @insert_float(i32 %arg) {
 ; SSE-LABEL: 'insert_float'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2f32_a = insertelement <2 x float> undef, float undef, i32 %arg
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2f32_0 = insertelement <2 x float> undef, float undef, i32 0
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2f32_1 = insertelement <2 x float> undef, float undef, i32 1
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4f32_a = insertelement <4 x float> undef, float undef, i32 %arg
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4f32_0 = insertelement <4 x float> undef, float undef, i32 0
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4f32_3 = insertelement <4 x float> undef, float undef, i32 3
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8f32_a = insertelement <8 x float> undef, float undef, i32 %arg
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8f32_0 = insertelement <8 x float> undef, float undef, i32 0
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8f32_3 = insertelement <8 x float> undef, float undef, i32 3
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8f32_4 = insertelement <8 x float> undef, float undef, i32 4
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8f32_7 = insertelement <8 x float> undef, float undef, i32 7
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16f32_a = insertelement <16 x float> undef, float undef, i32 %arg
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v16f32_0 = insertelement <16 x float> undef, float undef, i32 0
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16f32_3 = insertelement <16 x float> undef, float undef, i32 3
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v16f32_8 = insertelement <16 x float> undef, float undef, i32 8
@@ -89,14 +108,18 @@ define i32 @insert_float(i32 %arg) {
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX-LABEL: 'insert_float'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2f32_a = insertelement <2 x float> undef, float undef, i32 %arg
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2f32_0 = insertelement <2 x float> undef, float undef, i32 0
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2f32_1 = insertelement <2 x float> undef, float undef, i32 1
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4f32_a = insertelement <4 x float> undef, float undef, i32 %arg
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4f32_0 = insertelement <4 x float> undef, float undef, i32 0
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4f32_3 = insertelement <4 x float> undef, float undef, i32 3
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8f32_a = insertelement <8 x float> undef, float undef, i32 %arg
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8f32_0 = insertelement <8 x float> undef, float undef, i32 0
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8f32_3 = insertelement <8 x float> undef, float undef, i32 3
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8f32_4 = insertelement <8 x float> undef, float undef, i32 4
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8f32_7 = insertelement <8 x float> undef, float undef, i32 7
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16f32_a = insertelement <16 x float> undef, float undef, i32 %arg
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v16f32_0 = insertelement <16 x float> undef, float undef, i32 0
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16f32_3 = insertelement <16 x float> undef, float undef, i32 3
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v16f32_8 = insertelement <16 x float> undef, float undef, i32 8
@@ -104,14 +127,18 @@ define i32 @insert_float(i32 %arg) {
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512-LABEL: 'insert_float'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2f32_a = insertelement <2 x float> undef, float undef, i32 %arg
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2f32_0 = insertelement <2 x float> undef, float undef, i32 0
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2f32_1 = insertelement <2 x float> undef, float undef, i32 1
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4f32_a = insertelement <4 x float> undef, float undef, i32 %arg
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4f32_0 = insertelement <4 x float> undef, float undef, i32 0
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4f32_3 = insertelement <4 x float> undef, float undef, i32 3
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8f32_a = insertelement <8 x float> undef, float undef, i32 %arg
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8f32_0 = insertelement <8 x float> undef, float undef, i32 0
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8f32_3 = insertelement <8 x float> undef, float undef, i32 3
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8f32_4 = insertelement <8 x float> undef, float undef, i32 4
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8f32_7 = insertelement <8 x float> undef, float undef, i32 7
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16f32_a = insertelement <16 x float> undef, float undef, i32 %arg
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v16f32_0 = insertelement <16 x float> undef, float undef, i32 0
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16f32_3 = insertelement <16 x float> undef, float undef, i32 3
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16f32_8 = insertelement <16 x float> undef, float undef, i32 8
@@ -119,31 +146,39 @@ define i32 @insert_float(i32 %arg) {
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; BTVER2-LABEL: 'insert_float'
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2f32_a = insertelement <2 x float> undef, float undef, i32 %arg
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2f32_0 = insertelement <2 x float> undef, float undef, i32 0
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2f32_1 = insertelement <2 x float> undef, float undef, i32 1
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4f32_a = insertelement <4 x float> undef, float undef, i32 %arg
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4f32_0 = insertelement <4 x float> undef, float undef, i32 0
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4f32_3 = insertelement <4 x float> undef, float undef, i32 3
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8f32_a = insertelement <8 x float> undef, float undef, i32 %arg
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8f32_0 = insertelement <8 x float> undef, float undef, i32 0
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8f32_3 = insertelement <8 x float> undef, float undef, i32 3
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8f32_4 = insertelement <8 x float> undef, float undef, i32 4
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8f32_7 = insertelement <8 x float> undef, float undef, i32 7
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16f32_a = insertelement <16 x float> undef, float undef, i32 %arg
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v16f32_0 = insertelement <16 x float> undef, float undef, i32 0
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16f32_3 = insertelement <16 x float> undef, float undef, i32 3
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v16f32_8 = insertelement <16 x float> undef, float undef, i32 8
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16f32_15 = insertelement <16 x float> undef, float undef, i32 15
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
+  %v2f32_a = insertelement <2 x float> undef, float undef, i32 %arg
   %v2f32_0 = insertelement <2 x float> undef, float undef, i32 0
   %v2f32_1 = insertelement <2 x float> undef, float undef, i32 1
 
+  %v4f32_a = insertelement <4 x float> undef, float undef, i32 %arg
   %v4f32_0 = insertelement <4 x float> undef, float undef, i32 0
   %v4f32_3 = insertelement <4 x float> undef, float undef, i32 3
 
+  %v8f32_a = insertelement <8 x float> undef, float undef, i32 %arg
   %v8f32_0 = insertelement <8 x float> undef, float undef, i32 0
   %v8f32_3 = insertelement <8 x float> undef, float undef, i32 3
   %v8f32_4 = insertelement <8 x float> undef, float undef, i32 4
   %v8f32_7 = insertelement <8 x float> undef, float undef, i32 7
 
+  %v16f32_a  = insertelement <16 x float> undef, float undef, i32 %arg
   %v16f32_0  = insertelement <16 x float> undef, float undef, i32 0
   %v16f32_3  = insertelement <16 x float> undef, float undef, i32 3
   %v16f32_8  = insertelement <16 x float> undef, float undef, i32 8
@@ -154,10 +189,13 @@ define i32 @insert_float(i32 %arg) {
 
 define i32 @insert_i64(i32 %arg) {
 ; CHECK-LABEL: 'insert_i64'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64_a = insertelement <2 x i64> undef, i64 undef, i32 %arg
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64_0 = insertelement <2 x i64> undef, i64 undef, i32 0
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64_1 = insertelement <2 x i64> undef, i64 undef, i32 1
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i64_a = insertelement <4 x i64> undef, i64 undef, i32 %arg
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i64_0 = insertelement <4 x i64> undef, i64 undef, i32 0
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i64_3 = insertelement <4 x i64> undef, i64 undef, i32 3
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i64_a = insertelement <8 x i64> undef, i64 undef, i32 %arg
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i64_0 = insertelement <8 x i64> undef, i64 undef, i32 0
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i64_3 = insertelement <8 x i64> undef, i64 undef, i32 3
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i64_4 = insertelement <8 x i64> undef, i64 undef, i32 4
@@ -165,22 +203,28 @@ define i32 @insert_i64(i32 %arg) {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; BTVER2-LABEL: 'insert_i64'
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64_a = insertelement <2 x i64> undef, i64 undef, i32 %arg
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64_0 = insertelement <2 x i64> undef, i64 undef, i32 0
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64_1 = insertelement <2 x i64> undef, i64 undef, i32 1
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i64_a = insertelement <4 x i64> undef, i64 undef, i32 %arg
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i64_0 = insertelement <4 x i64> undef, i64 undef, i32 0
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i64_3 = insertelement <4 x i64> undef, i64 undef, i32 3
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i64_a = insertelement <8 x i64> undef, i64 undef, i32 %arg
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i64_0 = insertelement <8 x i64> undef, i64 undef, i32 0
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i64_3 = insertelement <8 x i64> undef, i64 undef, i32 3
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i64_4 = insertelement <8 x i64> undef, i64 undef, i32 4
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i64_7 = insertelement <8 x i64> undef, i64 undef, i32 7
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
+  %v2i64_a = insertelement <2 x i64> undef, i64 undef, i32 %arg
   %v2i64_0 = insertelement <2 x i64> undef, i64 undef, i32 0
   %v2i64_1 = insertelement <2 x i64> undef, i64 undef, i32 1
 
+  %v4i64_a = insertelement <4 x i64> undef, i64 undef, i32 %arg
   %v4i64_0 = insertelement <4 x i64> undef, i64 undef, i32 0
   %v4i64_3 = insertelement <4 x i64> undef, i64 undef, i32 3
 
+  %v8i64_a = insertelement <8 x i64> undef, i64 undef, i32 %arg
   %v8i64_0 = insertelement <8 x i64> undef, i64 undef, i32 0
   %v8i64_3 = insertelement <8 x i64> undef, i64 undef, i32 3
   %v8i64_4 = insertelement <8 x i64> undef, i64 undef, i32 4
@@ -191,14 +235,18 @@ define i32 @insert_i64(i32 %arg) {
 
 define i32 @insert_i32(i32 %arg) {
 ; CHECK-LABEL: 'insert_i32'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32_a = insertelement <2 x i32> undef, i32 undef, i32 %arg
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32_0 = insertelement <2 x i32> undef, i32 undef, i32 0
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32_1 = insertelement <2 x i32> undef, i32 undef, i32 1
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_a = insertelement <4 x i32> undef, i32 undef, i32 %arg
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_0 = insertelement <4 x i32> undef, i32 undef, i32 0
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_3 = insertelement <4 x i32> undef, i32 undef, i32 3
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_a = insertelement <8 x i32> undef, i32 undef, i32 %arg
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_0 = insertelement <8 x i32> undef, i32 undef, i32 0
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_3 = insertelement <8 x i32> undef, i32 undef, i32 3
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_4 = insertelement <8 x i32> undef, i32 undef, i32 4
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_7 = insertelement <8 x i32> undef, i32 undef, i32 7
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_a = insertelement <16 x i32> undef, i32 undef, i32 %arg
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_0 = insertelement <16 x i32> undef, i32 undef, i32 0
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_3 = insertelement <16 x i32> undef, i32 undef, i32 3
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_8 = insertelement <16 x i32> undef, i32 undef, i32 8
@@ -206,31 +254,39 @@ define i32 @insert_i32(i32 %arg) {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; BTVER2-LABEL: 'insert_i32'
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32_a = insertelement <2 x i32> undef, i32 undef, i32 %arg
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32_0 = insertelement <2 x i32> undef, i32 undef, i32 0
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32_1 = insertelement <2 x i32> undef, i32 undef, i32 1
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_a = insertelement <4 x i32> undef, i32 undef, i32 %arg
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_0 = insertelement <4 x i32> undef, i32 undef, i32 0
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_3 = insertelement <4 x i32> undef, i32 undef, i32 3
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_a = insertelement <8 x i32> undef, i32 undef, i32 %arg
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_0 = insertelement <8 x i32> undef, i32 undef, i32 0
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_3 = insertelement <8 x i32> undef, i32 undef, i32 3
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_4 = insertelement <8 x i32> undef, i32 undef, i32 4
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_7 = insertelement <8 x i32> undef, i32 undef, i32 7
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_a = insertelement <16 x i32> undef, i32 undef, i32 %arg
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_0 = insertelement <16 x i32> undef, i32 undef, i32 0
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_3 = insertelement <16 x i32> undef, i32 undef, i32 3
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_8 = insertelement <16 x i32> undef, i32 undef, i32 8
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_15 = insertelement <16 x i32> undef, i32 undef, i32 15
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
+  %v2i32_a = insertelement <2 x i32> undef, i32 undef, i32 %arg
   %v2i32_0 = insertelement <2 x i32> undef, i32 undef, i32 0
   %v2i32_1 = insertelement <2 x i32> undef, i32 undef, i32 1
 
+  %v4i32_a = insertelement <4 x i32> undef, i32 undef, i32 %arg
   %v4i32_0 = insertelement <4 x i32> undef, i32 undef, i32 0
   %v4i32_3 = insertelement <4 x i32> undef, i32 undef, i32 3
 
+  %v8i32_a = insertelement <8 x i32> undef, i32 undef, i32 %arg
   %v8i32_0 = insertelement <8 x i32> undef, i32 undef, i32 0
   %v8i32_3 = insertelement <8 x i32> undef, i32 undef, i32 3
   %v8i32_4 = insertelement <8 x i32> undef, i32 undef, i32 4
   %v8i32_7 = insertelement <8 x i32> undef, i32 undef, i32 7
 
+  %v16i32_a  = insertelement <16 x i32> undef, i32 undef, i32 %arg
   %v16i32_0  = insertelement <16 x i32> undef, i32 undef, i32 0
   %v16i32_3  = insertelement <16 x i32> undef, i32 undef, i32 3
   %v16i32_8  = insertelement <16 x i32> undef, i32 undef, i32 8
@@ -241,12 +297,15 @@ define i32 @insert_i32(i32 %arg) {
 
 define i32 @insert_i16(i32 %arg) {
 ; CHECK-LABEL: 'insert_i16'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_a = insertelement <8 x i16> undef, i16 undef, i32 %arg
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_0 = insertelement <8 x i16> undef, i16 undef, i32 0
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_7 = insertelement <8 x i16> undef, i16 undef, i32 7
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_a = insertelement <16 x i16> undef, i16 undef, i32 %arg
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_0 = insertelement <16 x i16> undef, i16 undef, i32 0
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_7 = insertelement <16 x i16> undef, i16 undef, i32 7
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_8 = insertelement <16 x i16> undef, i16 undef, i32 8
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_15 = insertelement <16 x i16> undef, i16 undef, i32 15
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_a = insertelement <32 x i16> undef, i16 undef, i32 %arg
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_0 = insertelement <32 x i16> undef, i16 undef, i32 0
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_7 = insertelement <32 x i16> undef, i16 undef, i32 7
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_8 = insertelement <32 x i16> undef, i16 undef, i32 8
@@ -257,12 +316,15 @@ define i32 @insert_i16(i32 %arg) {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; BTVER2-LABEL: 'insert_i16'
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_a = insertelement <8 x i16> undef, i16 undef, i32 %arg
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_0 = insertelement <8 x i16> undef, i16 undef, i32 0
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_7 = insertelement <8 x i16> undef, i16 undef, i32 7
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_a = insertelement <16 x i16> undef, i16 undef, i32 %arg
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_0 = insertelement <16 x i16> undef, i16 undef, i32 0
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_7 = insertelement <16 x i16> undef, i16 undef, i32 7
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_8 = insertelement <16 x i16> undef, i16 undef, i32 8
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_15 = insertelement <16 x i16> undef, i16 undef, i32 15
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_a = insertelement <32 x i16> undef, i16 undef, i32 %arg
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_0 = insertelement <32 x i16> undef, i16 undef, i32 0
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_7 = insertelement <32 x i16> undef, i16 undef, i32 7
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_8 = insertelement <32 x i16> undef, i16 undef, i32 8
@@ -272,14 +334,17 @@ define i32 @insert_i16(i32 %arg) {
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_31 = insertelement <32 x i16> undef, i16 undef, i32 31
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
+  %v8i16_a = insertelement <8 x i16> undef, i16 undef, i32 %arg
   %v8i16_0 = insertelement <8 x i16> undef, i16 undef, i32 0
   %v8i16_7 = insertelement <8 x i16> undef, i16 undef, i32 7
 
+  %v16i16_a  = insertelement <16 x i16> undef, i16 undef, i32 %arg
   %v16i16_0  = insertelement <16 x i16> undef, i16 undef, i32 0
   %v16i16_7  = insertelement <16 x i16> undef, i16 undef, i32 7
   %v16i16_8  = insertelement <16 x i16> undef, i16 undef, i32 8
   %v16i16_15 = insertelement <16 x i16> undef, i16 undef, i32 15
 
+  %v32i16_a  = insertelement <32 x i16> undef, i16 undef, i32 %arg
   %v32i16_0  = insertelement <32 x i16> undef, i16 undef, i32 0
   %v32i16_7  = insertelement <32 x i16> undef, i16 undef, i32 7
   %v32i16_8  = insertelement <32 x i16> undef, i16 undef, i32 8
@@ -293,15 +358,18 @@ define i32 @insert_i16(i32 %arg) {
 
 define i32 @insert_i8(i32 %arg) {
 ; CHECK-LABEL: 'insert_i8'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_a = insertelement <16 x i8> undef, i8 undef, i32 %arg
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_0 = insertelement <16 x i8> undef, i8 undef, i32 0
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_8 = insertelement <16 x i8> undef, i8 undef, i32 8
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_15 = insertelement <16 x i8> undef, i8 undef, i32 15
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_a = insertelement <32 x i8> undef, i8 undef, i32 %arg
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_0 = insertelement <32 x i8> undef, i8 undef, i32 0
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_7 = insertelement <32 x i8> undef, i8 undef, i32 7
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_8 = insertelement <32 x i8> undef, i8 undef, i32 8
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_15 = insertelement <32 x i8> undef, i8 undef, i32 15
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_24 = insertelement <32 x i8> undef, i8 undef, i32 24
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_31 = insertelement <32 x i8> undef, i8 undef, i32 31
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_a = insertelement <64 x i8> undef, i8 undef, i32 %arg
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_0 = insertelement <64 x i8> undef, i8 undef, i32 0
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_7 = insertelement <64 x i8> undef, i8 undef, i32 7
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_8 = insertelement <64 x i8> undef, i8 undef, i32 8
@@ -314,15 +382,18 @@ define i32 @insert_i8(i32 %arg) {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; BTVER2-LABEL: 'insert_i8'
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_a = insertelement <16 x i8> undef, i8 undef, i32 %arg
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_0 = insertelement <16 x i8> undef, i8 undef, i32 0
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_8 = insertelement <16 x i8> undef, i8 undef, i32 8
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_15 = insertelement <16 x i8> undef, i8 undef, i32 15
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_a = insertelement <32 x i8> undef, i8 undef, i32 %arg
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_0 = insertelement <32 x i8> undef, i8 undef, i32 0
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_7 = insertelement <32 x i8> undef, i8 undef, i32 7
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_8 = insertelement <32 x i8> undef, i8 undef, i32 8
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_15 = insertelement <32 x i8> undef, i8 undef, i32 15
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_24 = insertelement <32 x i8> undef, i8 undef, i32 24
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_31 = insertelement <32 x i8> undef, i8 undef, i32 31
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_a = insertelement <64 x i8> undef, i8 undef, i32 %arg
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_0 = insertelement <64 x i8> undef, i8 undef, i32 0
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_7 = insertelement <64 x i8> undef, i8 undef, i32 7
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_8 = insertelement <64 x i8> undef, i8 undef, i32 8
@@ -334,10 +405,12 @@ define i32 @insert_i8(i32 %arg) {
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_63 = insertelement <64 x i8> undef, i8 undef, i32 63
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
+  %v16i8_a  = insertelement <16 x i8> undef, i8 undef, i32 %arg
   %v16i8_0  = insertelement <16 x i8> undef, i8 undef, i32 0
   %v16i8_8  = insertelement <16 x i8> undef, i8 undef, i32 8
   %v16i8_15 = insertelement <16 x i8> undef, i8 undef, i32 15
 
+  %v32i8_a  = insertelement <32 x i8> undef, i8 undef, i32 %arg
   %v32i8_0  = insertelement <32 x i8> undef, i8 undef, i32 0
   %v32i8_7  = insertelement <32 x i8> undef, i8 undef, i32 7
   %v32i8_8  = insertelement <32 x i8> undef, i8 undef, i32 8
@@ -345,6 +418,7 @@ define i32 @insert_i8(i32 %arg) {
   %v32i8_24 = insertelement <32 x i8> undef, i8 undef, i32 24
   %v32i8_31 = insertelement <32 x i8> undef, i8 undef, i32 31
 
+  %v64i8_a  = insertelement <64 x i8> undef, i8 undef, i32 %arg
   %v64i8_0  = insertelement <64 x i8> undef, i8 undef, i32 0
   %v64i8_7  = insertelement <64 x i8> undef, i8 undef, i32 7
   %v64i8_8  = insertelement <64 x i8> undef, i8 undef, i32 8
diff --git a/llvm/test/Analysis/DDG/basic-a.ll b/llvm/test/Analysis/DDG/basic-a.ll
new file mode 100644
index 000000000..cebca668b
--- /dev/null
+++ b/llvm/test/Analysis/DDG/basic-a.ll
@@ -0,0 +1,202 @@
+; RUN: opt < %s -disable-output "-passes=print<ddg>" 2>&1 | FileCheck %s
+
+; CHECK-LABEL: 'DDG' for loop 'test1.for.body':
+
+; CHECK: Node Address:[[PI:0x[0-9a-f]*]]:pi-block
+; CHECK-NEXT: --- start of nodes in pi-block ---
+; CHECK: Node Address:[[N1:0x[0-9a-f]*]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %i.02 = phi i64 [ %inc, %test1.for.body ], [ 0, %test1.for.body.preheader ]
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N2:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N2]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %inc = add i64 %i.02, 1
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N1]]
+; CHECK-NEXT: --- end of nodes in pi-block ---
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N3:0x[0-9a-f]*]]
+; CHECK-NEXT:  [def-use] to [[N4:0x[0-9a-f]*]]
+; CHECK-NEXT:  [def-use] to [[N5:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N5]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %exitcond = icmp ne i64 %inc, %n
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N6:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N6]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    br i1 %exitcond, label %test1.for.body, label %for.end.loopexit
+; CHECK-NEXT: Edges:none!
+
+; CHECK: Node Address:[[N4]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %arrayidx1 = getelementptr inbounds float, float* %a, i64 %i.02
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N7:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N3]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %arrayidx = getelementptr inbounds float, float* %b, i64 %i.02
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N8:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N8]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %0 = load float, float* %arrayidx, align 4
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N9:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N10:0x[0-9a-f]*]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %conv = uitofp i64 %n to float
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N9]]
+
+; CHECK: Node Address:[[N9]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %add = fadd float %0, %conv
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N7]]
+
+; CHECK: Node Address:[[N7]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    store float %add, float* %arrayidx1, align 4
+; CHECK-NEXT: Edges:none!
+
+
+;; No memory dependencies.
+;; void test1(unsigned long n, float * restrict a, float * restrict b) {
+;;  for (unsigned long i = 0; i < n; i++)
+;;    a[i] = b[i] + n;
+;; }
+
+define void @test1(i64 %n, float* noalias %a, float* noalias %b) {
+entry:
+  %exitcond1 = icmp ne i64 0, %n
+  br i1 %exitcond1, label %test1.for.body, label %for.end
+
+test1.for.body:                                         ; preds = %entry, %test1.for.body
+  %i.02 = phi i64 [ %inc, %test1.for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %b, i64 %i.02
+  %0 = load float, float* %arrayidx, align 4
+  %conv = uitofp i64 %n to float
+  %add = fadd float %0, %conv
+  %arrayidx1 = getelementptr inbounds float, float* %a, i64 %i.02
+  store float %add, float* %arrayidx1, align 4
+  %inc = add i64 %i.02, 1
+  %exitcond = icmp ne i64 %inc, %n
+  br i1 %exitcond, label %test1.for.body, label %for.end
+
+for.end:                                          ; preds = %test1.for.body, %entry
+  ret void
+}
+
+
+; CHECK-LABEL: 'DDG' for loop 'test2.for.body':
+
+; CHECK: Node Address:[[PI:0x[0-9a-f]*]]:pi-block
+; CHECK-NEXT: --- start of nodes in pi-block ---
+; CHECK: Node Address:[[N1:0x[0-9a-f]*]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %i.02 = phi i64 [ %inc, %test2.for.body ], [ 0, %test2.for.body.preheader ]
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N2:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N2]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %inc = add i64 %i.02, 1
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N1]]
+; CHECK-NEXT: --- end of nodes in pi-block ---
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N3:0x[0-9a-f]*]]
+; CHECK-NEXT:  [def-use] to [[N4:0x[0-9a-f]*]]
+; CHECK-NEXT:  [def-use] to [[N5:0x[0-9a-f]*]]
+; CHECK-NEXT:  [def-use] to [[N6:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N6]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %exitcond = icmp ne i64 %inc, %n
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N7:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N7]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    br i1 %exitcond, label %test2.for.body, label %for.end.loopexit
+; CHECK-NEXT: Edges:none!
+
+; CHECK: Node Address:[[N5]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %arrayidx2 = getelementptr inbounds float, float* %a, i64 %i.02
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N8:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N4]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %arrayidx1 = getelementptr inbounds float, float* %a, i64 %i.02
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N9:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N9]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %1 = load float, float* %arrayidx1, align 4
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N20:0x[0-9a-f]*]]
+; CHECK-NEXT:  [memory] to [[N8]]
+
+; CHECK: Node Address:[[N3]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %arrayidx = getelementptr inbounds float, float* %b, i64 %i.02
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N10:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N10]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %0 = load float, float* %arrayidx, align 4
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N20]]
+
+; CHECK: Node Address:[[N20]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %add = fadd float %0, %1
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N8]]
+
+; CHECK: Node Address:[[N8]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    store float %add, float* %arrayidx2, align 4
+; CHECK-NEXT: Edges:none!
+
+
+
+;; Loop-independent memory dependencies.
+;; void test2(unsigned long n, float * restrict a, float * restrict b) {
+;;  for (unsigned long i = 0; i < n; i++)
+;;    a[i] = b[i] + a[i];
+;; }
+
+define void @test2(i64 %n, float* noalias %a, float* noalias %b) {
+entry:
+  %exitcond1 = icmp ne i64 0, %n
+  br i1 %exitcond1, label %test2.for.body, label %for.end
+
+test2.for.body:                                         ; preds = %entry, %test2.for.body
+  %i.02 = phi i64 [ %inc, %test2.for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %b, i64 %i.02
+  %0 = load float, float* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds float, float* %a, i64 %i.02
+  %1 = load float, float* %arrayidx1, align 4
+  %add = fadd float %0, %1
+  %arrayidx2 = getelementptr inbounds float, float* %a, i64 %i.02
+  store float %add, float* %arrayidx2, align 4
+  %inc = add i64 %i.02, 1
+  %exitcond = icmp ne i64 %inc, %n
+  br i1 %exitcond, label %test2.for.body, label %for.end
+
+for.end:                                          ; preds = %test2.for.body, %entry
+  ret void
+}
\ No newline at end of file
diff --git a/llvm/test/Analysis/DDG/basic-b.ll b/llvm/test/Analysis/DDG/basic-b.ll
new file mode 100644
index 000000000..50803d466
--- /dev/null
+++ b/llvm/test/Analysis/DDG/basic-b.ll
@@ -0,0 +1,233 @@
+; RUN: opt < %s -disable-output "-passes=print<ddg>" 2>&1 | FileCheck %s
+
+; CHECK-LABEL: 'DDG' for loop 'test1.for.body':
+
+; CHECK: Node Address:[[N1:0x[0-9a-f]*]]:pi-block
+; CHECK-NEXT:--- start of nodes in pi-block ---
+; CHECK: Node Address:[[N2:0x[0-9a-f]*]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %i.02 = phi i64 [ %inc, %test1.for.body ], [ 1, %test1.for.body.preheader ]
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N3:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N3]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %inc = add i64 %i.02, 1
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N2]]
+; CHECK-NEXT:--- end of nodes in pi-block ---
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N4:0x[0-9a-f]*]]
+; CHECK-NEXT:  [def-use] to [[N5:0x[0-9a-f]*]]
+; CHECK-NEXT:  [def-use] to [[N6:0x[0-9a-f]*]]
+; CHECK-NEXT:  [def-use] to [[N7:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N7]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %cmp = icmp ult i64 %inc, %sub
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N8:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N8]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    br i1 %cmp, label %test1.for.body, label %for.end.loopexit
+; CHECK-NEXT: Edges:none!
+
+; CHECK: Node Address:[[N6]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %arrayidx3 = getelementptr inbounds float, float* %a, i64 %i.02
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N9:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N5]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %sub1 = add i64 %i.02, -1
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N10:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N10]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %arrayidx2 = getelementptr inbounds float, float* %a, i64 %sub1
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N9]]
+
+; CHECK: Node Address:[[N4]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %arrayidx = getelementptr inbounds float, float* %b, i64 %i.02
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N11:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N11]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %0 = load float, float* %arrayidx, align 4
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N9]]
+
+; CHECK: Node Address:[[N9]]:pi-block
+; CHECK-NEXT: --- start of nodes in pi-block ---
+; CHECK: Node Address:[[N12:0x[0-9a-f]*]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %1 = load float, float* %arrayidx2, align 4
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N13:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N13]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %add = fadd float %0, %1
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N14:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N14]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    store float %add, float* %arrayidx3, align 4
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [memory] to [[N12]]
+; CHECK-NEXT:--- end of nodes in pi-block ---
+; CHECK-NEXT: Edges:none!
+
+
+
+;; Loop-carried dependence requiring edge-reversal to expose a cycle
+;; in the graph.
+;; void test(unsigned long n, float * restrict a, float * restrict b) {
+;;  for (unsigned long i = 1; i < n-1; i++)
+;;    a[i] = b[i] + a[i-1];
+;; }
+
+define void @test1(i64 %n, float* noalias %a, float* noalias %b) {
+entry:
+  %sub = add i64 %n, -1
+  %cmp1 = icmp ult i64 1, %sub
+  br i1 %cmp1, label %test1.for.body, label %for.end
+
+test1.for.body:                                         ; preds = %entry, %test1.for.body
+  %i.02 = phi i64 [ %inc, %test1.for.body ], [ 1, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %b, i64 %i.02
+  %0 = load float, float* %arrayidx, align 4
+  %sub1 = add i64 %i.02, -1
+  %arrayidx2 = getelementptr inbounds float, float* %a, i64 %sub1
+  %1 = load float, float* %arrayidx2, align 4
+  %add = fadd float %0, %1
+  %arrayidx3 = getelementptr inbounds float, float* %a, i64 %i.02
+  store float %add, float* %arrayidx3, align 4
+  %inc = add i64 %i.02, 1
+  %cmp = icmp ult i64 %inc, %sub
+  br i1 %cmp, label %test1.for.body, label %for.end
+
+for.end:                                          ; preds = %test1.for.body, %entry
+  ret void
+}
+
+; CHECK-LABEL: 'DDG' for loop 'test2.for.body':
+
+; CHECK: Node Address:[[N1:0x[0-9a-f]*]]:pi-block
+; CHECK-NEXT:--- start of nodes in pi-block ---
+
+; CHECK: Node Address:[[N2:0x[0-9a-f]*]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %i.02 = phi i64 [ %inc, %test2.for.body ], [ 1, %test2.for.body.preheader ]
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N3:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N3]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %inc = add i64 %i.02, 1
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N2]]
+; CHECK-NEXT:--- end of nodes in pi-block ---
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N4:0x[0-9a-f]*]]
+; CHECK-NEXT:  [def-use] to [[N5:0x[0-9a-f]*]]
+; CHECK-NEXT:  [def-use] to [[N6:0x[0-9a-f]*]]
+; CHECK-NEXT:  [def-use] to [[N7:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N7]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %cmp = icmp ult i64 %inc, %sub
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N8:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N8]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    br i1 %cmp, label %test2.for.body, label %for.end.loopexit
+; CHECK-NEXT: Edges:none!
+
+; CHECK: Node Address:[[N6]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %arrayidx3 = getelementptr inbounds float, float* %a, i64 %i.02
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N9:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N5]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %add1 = add i64 %i.02, 1
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N10:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N10]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %arrayidx2 = getelementptr inbounds float, float* %a, i64 %add1
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N11:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N11]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %1 = load float, float* %arrayidx2, align 4
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N12:0x[0-9a-f]*]]
+; CHECK-NEXT:  [memory] to [[N9]]
+
+; CHECK: Node Address:[[N4]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %arrayidx = getelementptr inbounds float, float* %b, i64 %i.02
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N13:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N13]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %0 = load float, float* %arrayidx, align 4
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N12]]
+
+
+; CHECK: Node Address:[[N12]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %add = fadd float %0, %1
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N9]]
+
+; CHECK: Node Address:[[N9]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    store float %add, float* %arrayidx3, align 4
+; CHECK-NEXT: Edges:none!
+
+
+;; Forward loop-carried dependence *not* causing a cycle.
+;; void test2(unsigned long n, float * restrict a, float * restrict b) {
+;;  for (unsigned long i = 1; i < n-1; i++)
+;;    a[i] = b[i] + a[i+1];
+;; }
+
+define void @test2(i64 %n, float* noalias %a, float* noalias %b) {
+entry:
+  %sub = add i64 %n, -1
+  %cmp1 = icmp ult i64 1, %sub
+  br i1 %cmp1, label %test2.for.body, label %for.end
+
+test2.for.body:                                         ; preds = %entry, %test2.for.body
+  %i.02 = phi i64 [ %inc, %test2.for.body ], [ 1, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %b, i64 %i.02
+  %0 = load float, float* %arrayidx, align 4
+  %add1 = add i64 %i.02, 1
+  %arrayidx2 = getelementptr inbounds float, float* %a, i64 %add1
+  %1 = load float, float* %arrayidx2, align 4
+  %add = fadd float %0, %1
+  %arrayidx3 = getelementptr inbounds float, float* %a, i64 %i.02
+  store float %add, float* %arrayidx3, align 4
+  %inc = add i64 %i.02, 1
+  %cmp = icmp ult i64 %inc, %sub
+  br i1 %cmp, label %test2.for.body, label %for.end
+
+for.end:                                          ; preds = %test2.for.body, %entry
+  ret void
+}
diff --git a/llvm/test/Analysis/DDG/basic-loopnest.ll b/llvm/test/Analysis/DDG/basic-loopnest.ll
new file mode 100644
index 000000000..3581c7a68
--- /dev/null
+++ b/llvm/test/Analysis/DDG/basic-loopnest.ll
@@ -0,0 +1,456 @@
+; RUN: opt < %s -disable-output "-passes=print<ddg>" 2>&1 | FileCheck %s
+
+
+; CHECK-LABEL: 'DDG' for loop 'test1.for.cond1.preheader':
+
+; CHECK: Node Address:[[N1:0x[0-9a-f]*]]:pi-block
+; CHECK-NEXT:--- start of nodes in pi-block ---
+; CHECK: Node Address:[[N2:0x[0-9a-f]*]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %j.02 = phi i64 [ %inc, %for.body4 ], [ 1, %for.body4.preheader ]
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N3:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N3]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %inc = add i64 %j.02, 1
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N2]]
+; CHECK-NEXT:--- end of nodes in pi-block ---
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N4:0x[0-9a-f]*]]
+; CHECK-NEXT:  [def-use] to [[N5:0x[0-9a-f]*]]
+; CHECK-NEXT:  [def-use] to [[N6:0x[0-9a-f]*]]
+; CHECK-NEXT:  [def-use] to [[N7:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N5]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %sub7 = add i64 %j.02, -1
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N8:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N9:0x[0-9a-f]*]]:pi-block
+; CHECK-NEXT:--- start of nodes in pi-block ---
+; CHECK: Node Address:[[N10:0x[0-9a-f]*]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %i.04 = phi i64 [ %inc13, %for.inc12 ], [ 0, %test1.for.cond1.preheader.preheader ]
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N11:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N11]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %inc13 = add i64 %i.04, 1
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N10]]
+; CHECK-NEXT:--- end of nodes in pi-block ---
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N12:0x[0-9a-f]*]]
+; CHECK-NEXT:  [def-use] to [[N13:0x[0-9a-f]*]]
+; CHECK-NEXT:  [def-use] to [[N14:0x[0-9a-f]*]]
+; CHECK-NEXT:  [def-use] to [[N15:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N15]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %exitcond = icmp ne i64 %inc13, %n
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N16:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N16]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    br i1 %exitcond, label %test1.for.cond1.preheader, label %for.end14.loopexit
+; CHECK-NEXT: Edges:none!
+
+; CHECK: Node Address:[[N14]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %4 = mul nsw i64 %i.04, %n
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N17:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N17]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %arrayidx10 = getelementptr inbounds float, float* %a, i64 %4
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N6]]
+
+; CHECK: Node Address:[[N6]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %arrayidx11 = getelementptr inbounds float, float* %arrayidx10, i64 %j.02
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N18:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N13]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %2 = mul nsw i64 %i.04, %n
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N19:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N19]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %arrayidx6 = getelementptr inbounds float, float* %a, i64 %2
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N8]]
+
+; CHECK: Node Address:[[N8]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %arrayidx8 = getelementptr inbounds float, float* %arrayidx6, i64 %sub7
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N18]]
+
+; CHECK: Node Address:[[N12]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %0 = mul nsw i64 %i.04, %n
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N20:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N20]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %arrayidx = getelementptr inbounds float, float* %b, i64 %0
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N4]]
+
+; CHECK: Node Address:[[N4]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %arrayidx5 = getelementptr inbounds float, float* %arrayidx, i64 %j.02
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N21:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N21]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %1 = load float, float* %arrayidx5, align 4
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N18]]
+
+; CHECK: Node Address:[[N18]]:pi-block
+; CHECK-NEXT:--- start of nodes in pi-block ---
+; CHECK: Node Address:[[N22:0x[0-9a-f]*]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %3 = load float, float* %arrayidx8, align 4
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N23:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N23]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %add = fadd float %1, %3
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N24:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N24]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    store float %add, float* %arrayidx11, align 4
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [memory] to [[N22]]
+; CHECK-NEXT:--- end of nodes in pi-block ---
+; CHECK-NEXT: Edges:none!
+
+; CHECK: Node Address:[[N25:0x[0-9a-f]*]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    br label %for.inc12
+; CHECK-NEXT: Edges:none!
+
+; CHECK: Node Address:[[N26:0x[0-9a-f]*]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    br label %for.body4
+; CHECK-NEXT: Edges:none!
+
+; CHECK: Node Address:[[N27:0x[0-9a-f]*]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %sub = add i64 %n, -1
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N7]]
+; CHECK-NEXT:  [def-use] to [[N28:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N28]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %cmp21 = icmp ult i64 1, %sub
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N29:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N29]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    br i1 %cmp21, label %for.body4.preheader, label %for.inc12
+; CHECK-NEXT: Edges:none!
+
+; CHECK: Node Address:[[N7]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %cmp2 = icmp ult i64 %inc, %sub
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N30:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N30]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    br i1 %cmp2, label %for.body4, label %for.inc12.loopexit
+; CHECK-NEXT: Edges:none!
+
+
+;; This test has a cycle.
+;; void test1(unsigned long n, float a[][n], float b[][n]) {
+;;  for (unsigned long i = 0; i < n; i++)
+;;    for (unsigned long j = 1; j < n-1; j++)
+;;      a[i][j] = b[i][j] + a[i][j-1];
+;; }
+
+define void @test1(i64 %n, float* noalias %a, float* noalias %b) {
+entry:
+  %exitcond3 = icmp ne i64 0, %n
+  br i1 %exitcond3, label %test1.for.cond1.preheader, label %for.end14
+
+test1.for.cond1.preheader:                              ; preds = %entry, %for.inc12
+  %i.04 = phi i64 [ %inc13, %for.inc12 ], [ 0, %entry ]
+  %sub = add i64 %n, -1
+  %cmp21 = icmp ult i64 1, %sub
+  br i1 %cmp21, label %for.body4, label %for.inc12
+
+for.body4:                                        ; preds = %test1.for.cond1.preheader, %for.body4
+  %j.02 = phi i64 [ %inc, %for.body4 ], [ 1, %test1.for.cond1.preheader ]
+  %0 = mul nsw i64 %i.04, %n
+  %arrayidx = getelementptr inbounds float, float* %b, i64 %0
+  %arrayidx5 = getelementptr inbounds float, float* %arrayidx, i64 %j.02
+  %1 = load float, float* %arrayidx5, align 4
+  %2 = mul nsw i64 %i.04, %n
+  %arrayidx6 = getelementptr inbounds float, float* %a, i64 %2
+  %sub7 = add i64 %j.02, -1
+  %arrayidx8 = getelementptr inbounds float, float* %arrayidx6, i64 %sub7
+  %3 = load float, float* %arrayidx8, align 4
+  %add = fadd float %1, %3
+  %4 = mul nsw i64 %i.04, %n
+  %arrayidx10 = getelementptr inbounds float, float* %a, i64 %4
+  %arrayidx11 = getelementptr inbounds float, float* %arrayidx10, i64 %j.02
+  store float %add, float* %arrayidx11, align 4
+  %inc = add i64 %j.02, 1
+  %cmp2 = icmp ult i64 %inc, %sub
+  br i1 %cmp2, label %for.body4, label %for.inc12
+
+for.inc12:                                        ; preds = %for.body4, %test1.for.cond1.preheader
+  %inc13 = add i64 %i.04, 1
+  %exitcond = icmp ne i64 %inc13, %n
+  br i1 %exitcond, label %test1.for.cond1.preheader, label %for.end14
+
+for.end14:                                        ; preds = %for.inc12, %entry
+  ret void
+}
+
+
+
+; CHECK-LABEL: 'DDG' for loop 'test2.for.cond1.preheader':
+
+; CHECK: Node Address:[[PI1:0x[0-9a-f]*]]:pi-block
+; CHECK-NEXT:--- start of nodes in pi-block ---
+; CHECK: Node Address:[[N1:0x[0-9a-f]*]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %j.02 = phi i64 [ %inc, %for.body4 ], [ 1, %for.body4.preheader ]
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N2:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N2]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %inc = add i64 %j.02, 1
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N1]]
+; CHECK-NEXT:--- end of nodes in pi-block ---
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N3:0x[0-9a-f]*]]
+; CHECK-NEXT:  [def-use] to [[N4:0x[0-9a-f]*]]
+; CHECK-NEXT:  [def-use] to [[N5:0x[0-9a-f]*]]
+; CHECK-NEXT:  [def-use] to [[N6:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N4]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %add7 = add i64 %j.02, 1
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N7:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N8:0x[0-9a-f]*]]:pi-block
+; CHECK-NEXT:--- start of nodes in pi-block ---
+; CHECK: Node Address:[[N9:0x[0-9a-f]*]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %i.04 = phi i64 [ %inc13, %for.inc12 ], [ 0, %test2.for.cond1.preheader.preheader ]
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N10:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N10]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %inc13 = add i64 %i.04, 1
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N9]]
+; CHECK-NEXT:--- end of nodes in pi-block ---
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N11:0x[0-9a-f]*]]
+; CHECK-NEXT:  [def-use] to [[N12:0x[0-9a-f]*]]
+; CHECK-NEXT:  [def-use] to [[N13:0x[0-9a-f]*]]
+; CHECK-NEXT:  [def-use] to [[N14:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N14]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %exitcond = icmp ne i64 %inc13, %n
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N15:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N15]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    br i1 %exitcond, label %test2.for.cond1.preheader, label %for.end14.loopexit
+; CHECK-NEXT: Edges:none!
+
+; CHECK: Node Address:[[N13]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %4 = mul nsw i64 %i.04, %n
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N16:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N16]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %arrayidx10 = getelementptr inbounds float, float* %a, i64 %4
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N5]]
+
+; CHECK: Node Address:[[N5]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %arrayidx11 = getelementptr inbounds float, float* %arrayidx10, i64 %j.02
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N17:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N12]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %2 = mul nsw i64 %i.04, %n
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N18:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N18]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %arrayidx6 = getelementptr inbounds float, float* %a, i64 %2
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N7]]
+
+; CHECK: Node Address:[[N7]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %arrayidx8 = getelementptr inbounds float, float* %arrayidx6, i64 %add7
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N19:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N19]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %3 = load float, float* %arrayidx8, align 4
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N20:0x[0-9a-f]*]]
+; CHECK-NEXT:  [memory] to [[N17]]
+
+; CHECK: Node Address:[[N11]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %0 = mul nsw i64 %i.04, %n
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N21:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N21]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %arrayidx = getelementptr inbounds float, float* %b, i64 %0
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N3]]
+
+; CHECK: Node Address:[[N3]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %arrayidx5 = getelementptr inbounds float, float* %arrayidx, i64 %j.02
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N22:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N22]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %1 = load float, float* %arrayidx5, align 4
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N20]]
+
+; CHECK: Node Address:[[N20]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %add = fadd float %1, %3
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N17]]
+
+; CHECK: Node Address:[[N17]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    store float %add, float* %arrayidx11, align 4
+; CHECK-NEXT: Edges:none!
+
+; CHECK: Node Address:[[N23:0x[0-9a-f]*]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    br label %for.inc12
+; CHECK-NEXT: Edges:none!
+
+; CHECK: Node Address:[[N24:0x[0-9a-f]*]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    br label %for.body4
+; CHECK-NEXT: Edges:none!
+
+; CHECK: Node Address:[[N25:0x[0-9a-f]*]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %sub = add i64 %n, -1
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N6]]
+; CHECK-NEXT:  [def-use] to [[N26:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N26]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %cmp21 = icmp ult i64 1, %sub
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N27:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N27]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    br i1 %cmp21, label %for.body4.preheader, label %for.inc12
+; CHECK-NEXT: Edges:none!
+
+; CHECK: Node Address:[[N6]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %cmp2 = icmp ult i64 %inc, %sub
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N28:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N28]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    br i1 %cmp2, label %for.body4, label %for.inc12.loopexit
+; CHECK-NEXT: Edges:none!
+
+
+;; This test has no cycles.
+;; void test2(unsigned long n, float a[][n], float b[][n]) {
+;;  for (unsigned long i = 0; i < n; i++)
+;;    for (unsigned long j = 1; j < n-1; j++)
+;;      a[i][j] = b[i][j] + a[i][j+1];
+;; }
+
+define void @test2(i64 %n, float* noalias %a, float* noalias %b) {
+entry:
+  %exitcond3 = icmp ne i64 0, %n
+  br i1 %exitcond3, label %test2.for.cond1.preheader, label %for.end14
+
+test2.for.cond1.preheader:                              ; preds = %entry, %for.inc12
+  %i.04 = phi i64 [ %inc13, %for.inc12 ], [ 0, %entry ]
+  %sub = add i64 %n, -1
+  %cmp21 = icmp ult i64 1, %sub
+  br i1 %cmp21, label %for.body4, label %for.inc12
+
+for.body4:                                        ; preds = %test2.for.cond1.preheader, %for.body4
+  %j.02 = phi i64 [ %inc, %for.body4 ], [ 1, %test2.for.cond1.preheader ]
+  %0 = mul nsw i64 %i.04, %n
+  %arrayidx = getelementptr inbounds float, float* %b, i64 %0
+  %arrayidx5 = getelementptr inbounds float, float* %arrayidx, i64 %j.02
+  %1 = load float, float* %arrayidx5, align 4
+  %2 = mul nsw i64 %i.04, %n
+  %arrayidx6 = getelementptr inbounds float, float* %a, i64 %2
+  %add7 = add i64 %j.02, 1
+  %arrayidx8 = getelementptr inbounds float, float* %arrayidx6, i64 %add7
+  %3 = load float, float* %arrayidx8, align 4
+  %add = fadd float %1, %3
+  %4 = mul nsw i64 %i.04, %n
+  %arrayidx10 = getelementptr inbounds float, float* %a, i64 %4
+  %arrayidx11 = getelementptr inbounds float, float* %arrayidx10, i64 %j.02
+  store float %add, float* %arrayidx11, align 4
+  %inc = add i64 %j.02, 1
+  %cmp2 = icmp ult i64 %inc, %sub
+  br i1 %cmp2, label %for.body4, label %for.inc12
+
+for.inc12:                                        ; preds = %for.body4, %test2.for.cond1.preheader
+  %inc13 = add i64 %i.04, 1
+  %exitcond = icmp ne i64 %inc13, %n
+  br i1 %exitcond, label %test2.for.cond1.preheader, label %for.end14
+
+for.end14:                                        ; preds = %for.inc12, %entry
+  ret void
+}
\ No newline at end of file
diff --git a/llvm/test/Analysis/DDG/root-node.ll b/llvm/test/Analysis/DDG/root-node.ll
new file mode 100644
index 000000000..868fb72d4
--- /dev/null
+++ b/llvm/test/Analysis/DDG/root-node.ll
@@ -0,0 +1,49 @@
+; RUN: opt < %s -disable-output "-passes=print<ddg>" 2>&1 | FileCheck %s
+
+; CHECK-LABEL: 'DDG' for loop 'test1.for.body':
+
+; CHECK: Node Address:[[ROOT:0x[0-9a-f]*]]:root
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [rooted] to [[N1:0x[0-9a-f]*]]
+; CHECK-NEXT:  [rooted] to [[N2:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N2]]:pi-block
+; CHECK: %i1.02 = phi i64 [ 0, %for.body.lr.ph ], [ %inc, %test1.for.body ]
+
+; CHECK: Node Address:[[N1]]:pi-block
+; CHECK: %i2.03 = phi i64 [ 0, %for.body.lr.ph ], [ %inc2, %test1.for.body ]
+
+;; // Two separate components in the graph. Root node must link to both.
+;; void test1(unsigned long n, float * restrict a, float * restrict b) {
+;;   for (unsigned long i1 = 0, i2 = 0; i1 < n; i1++, i2++) {
+;;     a[i1] = 1;
+;;     b[i2] = -1;
+;;   }
+;; }
+
+define void @test1(i64 %n, float* noalias %a, float* noalias %b) {
+entry:
+  %cmp1 = icmp ult i64 0, %n
+  br i1 %cmp1, label %for.body.lr.ph, label %for.end
+
+for.body.lr.ph:                                   ; preds = %entry
+  br label %test1.for.body
+
+test1.for.body:                                         ; preds = %for.body.lr.ph, %test1.for.body
+  %i2.03 = phi i64 [ 0, %for.body.lr.ph ], [ %inc2, %test1.for.body ]
+  %i1.02 = phi i64 [ 0, %for.body.lr.ph ], [ %inc, %test1.for.body ]
+  %arrayidx = getelementptr inbounds float, float* %a, i64 %i1.02
+  store float 1.000000e+00, float* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds float, float* %b, i64 %i2.03
+  store float -1.000000e+00, float* %arrayidx1, align 4
+  %inc = add i64 %i1.02, 1
+  %inc2 = add i64 %i2.03, 1
+  %cmp = icmp ult i64 %inc, %n
+  br i1 %cmp, label %test1.for.body, label %for.cond.for.end_crit_edge
+
+for.cond.for.end_crit_edge:                       ; preds = %test1.for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.cond.for.end_crit_edge, %entry
+  ret void
+}
diff --git a/llvm/test/Analysis/Delinearization/constant_functions_multi_dim.ll b/llvm/test/Analysis/Delinearization/constant_functions_multi_dim.ll
index b44b900d3..18fbdcc60 100644
--- a/llvm/test/Analysis/Delinearization/constant_functions_multi_dim.ll
+++ b/llvm/test/Analysis/Delinearization/constant_functions_multi_dim.ll
@@ -64,8 +64,8 @@ declare i64 @_Z13get_global_idj(i32) #1
 ; Function Attrs: nounwind readnone speculatable
 declare float @llvm.fmuladd.f32(float, float, float) #2
 
-attributes #0 = { noinline nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noinline nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind readnone speculatable }
 attributes #3 = { nounwind readnone }
 
diff --git a/llvm/test/Analysis/DependenceAnalysis/Dump.ll b/llvm/test/Analysis/DependenceAnalysis/Dump.ll
new file mode 100644
index 000000000..d86221d85
--- /dev/null
+++ b/llvm/test/Analysis/DependenceAnalysis/Dump.ll
@@ -0,0 +1,50 @@
+; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1 \
+; RUN: | FileCheck %s
+
+;; Test to make sure the dump shows the src and dst
+;; instructions (including call instructions).
+;;
+;; void bar(float * restrict A);
+;; void foo(float * restrict A, int n) {
+;;   for (int i = 0; i < n; i++) {
+;;     A[i] = i;
+;;     bar(A);
+;;   }
+;; }
+
+; CHECK-LABEL: foo
+
+; CHECK: Src:  store float %conv, float* %arrayidx, align 4 --> Dst:  store float %conv, float* %arrayidx, align 4
+; CHECK-NEXT:   da analyze - none!
+; CHECK-NEXT: Src:  store float %conv, float* %arrayidx, align 4 --> Dst:  call void @bar(float* %A)
+; CHECK-NEXT:   da analyze - confused!
+; CHECK-NEXT: Src:  call void @bar(float* %A) --> Dst:  call void @bar(float* %A)
+; CHECK-NEXT:   da analyze - confused!
+
+define void @foo(float* noalias %A, i32 signext %n) {
+entry:
+  %cmp1 = icmp slt i32 0, %n
+  br i1 %cmp1, label %for.body.lr.ph, label %for.end
+
+for.body.lr.ph:                                   ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.lr.ph, %for.body
+  %i.02 = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %for.body ]
+  %conv = sitofp i32 %i.02 to float
+  %idxprom = zext i32 %i.02 to i64
+  %arrayidx = getelementptr inbounds float, float* %A, i64 %idxprom
+  store float %conv, float* %arrayidx, align 4
+  call void @bar(float* %A) #3
+  %inc = add nuw nsw i32 %i.02, 1
+  %cmp = icmp slt i32 %inc, %n
+  br i1 %cmp, label %for.body, label %for.cond.for.end_crit_edge
+
+for.cond.for.end_crit_edge:                       ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.cond.for.end_crit_edge, %entry
+  ret void
+}
+
+declare void @bar(float*)
diff --git a/llvm/test/Analysis/DependenceAnalysis/MIVCheckConst.ll b/llvm/test/Analysis/DependenceAnalysis/MIVCheckConst.ll
index d5dd1050b..9a3692d61 100644
--- a/llvm/test/Analysis/DependenceAnalysis/MIVCheckConst.ll
+++ b/llvm/test/Analysis/DependenceAnalysis/MIVCheckConst.ll
@@ -71,4 +71,4 @@ bb40:
   ret void
 }
 
-attributes #0 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/Analysis/DependenceAnalysis/NonCanonicalizedSubscript.ll b/llvm/test/Analysis/DependenceAnalysis/NonCanonicalizedSubscript.ll
index 2aba99243..1da2672ff 100644
--- a/llvm/test/Analysis/DependenceAnalysis/NonCanonicalizedSubscript.ll
+++ b/llvm/test/Analysis/DependenceAnalysis/NonCanonicalizedSubscript.ll
@@ -96,7 +96,7 @@ for.end13:                                        ; preds = %for.cond
   ret void
 }
 
-attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.ident = !{!0}
 
diff --git a/llvm/test/Analysis/DependenceAnalysis/SimpleSIVNoValidityCheck.ll b/llvm/test/Analysis/DependenceAnalysis/SimpleSIVNoValidityCheck.ll
index 57ccf85b8..7063f20cd 100644
--- a/llvm/test/Analysis/DependenceAnalysis/SimpleSIVNoValidityCheck.ll
+++ b/llvm/test/Analysis/DependenceAnalysis/SimpleSIVNoValidityCheck.ll
@@ -235,3 +235,4 @@ for.end11.loopexit:                               ; preds = %for.inc9
 for.end11:                                        ; preds = %for.end11.loopexit, %entry
   ret void
 }
+
diff --git a/llvm/test/Analysis/DivergenceAnalysis/AMDGPU/b42473-r1-crash.ll b/llvm/test/Analysis/DivergenceAnalysis/AMDGPU/b42473-r1-crash.ll
new file mode 100644
index 000000000..cb3e42de3
--- /dev/null
+++ b/llvm/test/Analysis/DivergenceAnalysis/AMDGPU/b42473-r1-crash.ll
@@ -0,0 +1,111 @@
+; RUN: opt -mtriple amdgcn-unknown-amdhsa -analyze -divergence -use-gpu-divergence-analysis %s | FileCheck %s
+
+declare i32 @gf2(i32)
+declare i32 @gf1(i32)
+
+define  void @tw1(i32 addrspace(4)* noalias nocapture readonly %A, i32 addrspace(4)* noalias nocapture %B) local_unnamed_addr #2 {
+; CHECK: Printing analysis 'Legacy Divergence Analysis' for function 'tw1':
+; CHECK: DIVERGENT: i32 addrspace(4)* %A
+; CHECK: DIVERGENT: i32 addrspace(4)* %B
+entry:
+; CHECK: DIVERGENT:       %call = tail call i32 @gf2(i32 0) #0
+; CHECK: DIVERGENT:       %cmp = icmp ult i32 %call, 16
+; CHECK: DIVERGENT:       br i1 %cmp, label %if.then, label %new_exit
+  %call = tail call  i32 @gf2(i32 0) #3
+  %cmp = icmp ult i32 %call, 16
+  br i1 %cmp, label %if.then, label %new_exit
+
+if.then:
+; CHECK: DIVERGENT:       %call1 = tail call i32 @gf1(i32 0) #0
+; CHECK: DIVERGENT:       %arrayidx = getelementptr inbounds i32, i32 addrspace(4)* %A, i32 %call1
+; CHECK: DIVERGENT:       %0 = load i32, i32 addrspace(4)* %arrayidx, align 4
+; CHECK: DIVERGENT:       %cmp225 = icmp sgt i32 %0, 0
+; CHECK: DIVERGENT:       %arrayidx10 = getelementptr inbounds i32, i32 addrspace(4)* %B, i32 %call1
+; CHECK: DIVERGENT:       br i1 %cmp225, label %while.body.preheader, label %if.then.while.end_crit_edge
+  %call1 = tail call  i32 @gf1(i32 0) #4
+  %arrayidx = getelementptr inbounds i32, i32 addrspace(4)* %A, i32 %call1
+  %0 = load i32, i32 addrspace(4)* %arrayidx, align 4
+  %cmp225 = icmp sgt i32 %0, 0
+  %arrayidx10 = getelementptr inbounds i32, i32 addrspace(4)* %B, i32 %call1
+  br i1 %cmp225, label %while.body.preheader, label %if.then.while.end_crit_edge
+
+while.body.preheader:
+  br label %while.body
+
+if.then.while.end_crit_edge:
+; CHECK: DIVERGENT:       %.pre = load i32, i32 addrspace(4)* %arrayidx10, align 4
+  %.pre = load i32, i32 addrspace(4)* %arrayidx10, align 4
+  br label %while.end
+
+while.body:
+; CHECK-NOT: DIVERGENT:                  %i.026 = phi i32 [ %inc, %if.end.while.body_crit_edge ], [ 0, %while.body.preheader ]
+; CHECK: DIVERGENT:       %call3 = tail call i32 @gf1(i32 0) #0
+; CHECK: DIVERGENT:       %cmp4 = icmp ult i32 %call3, 10
+; CHECK: DIVERGENT:       %arrayidx6 = getelementptr inbounds i32, i32 addrspace(4)* %A, i32 %i.026
+; CHECK: DIVERGENT:       %1 = load i32, i32 addrspace(4)* %arrayidx6, align 4
+; CHECK: DIVERGENT:       br i1 %cmp4, label %if.then5, label %if.else
+  %i.026 = phi i32 [ %inc, %if.end.while.body_crit_edge ], [ 0, %while.body.preheader ]
+  %call3 = tail call  i32 @gf1(i32 0) #4
+  %cmp4 = icmp ult i32 %call3, 10
+  %arrayidx6 = getelementptr inbounds i32, i32 addrspace(4)* %A, i32 %i.026
+  %1 = load i32, i32 addrspace(4)* %arrayidx6, align 4
+  br i1 %cmp4, label %if.then5, label %if.else
+
+if.then5:
+; CHECK: DIVERGENT:       %mul = shl i32 %1, 1
+; CHECK: DIVERGENT:       %2 = load i32, i32 addrspace(4)* %arrayidx10, align 4
+; CHECK: DIVERGENT:       %add = add nsw i32 %2, %mul
+  %mul = shl i32 %1, 1
+  %2 = load i32, i32 addrspace(4)* %arrayidx10, align 4
+  %add = add nsw i32 %2, %mul
+  br label %if.end
+
+if.else:
+; CHECK: DIVERGENT:       %mul9 = shl i32 %1, 2
+; CHECK: DIVERGENT:       %3 = load i32, i32 addrspace(4)* %arrayidx10, align 4
+; CHECK: DIVERGENT:       %add11 = add nsw i32 %3, %mul9
+  %mul9 = shl i32 %1, 2
+  %3 = load i32, i32 addrspace(4)* %arrayidx10, align 4
+  %add11 = add nsw i32 %3, %mul9
+  br label %if.end
+
+if.end:
+; CHECK: DIVERGENT:       %storemerge = phi i32 [ %add11, %if.else ], [ %add, %if.then5 ]
+; CHECK: DIVERGENT:       store i32 %storemerge, i32 addrspace(4)* %arrayidx10, align 4
+; CHECK-NOT: DIVERGENT:                  %inc = add nuw nsw i32 %i.026, 1
+; CHECK: DIVERGENT:       %exitcond = icmp ne i32 %inc, %0
+; CHECK: DIVERGENT:       br i1 %exitcond, label %if.end.while.body_crit_edge, label %while.end.loopexit
+  %storemerge = phi i32 [ %add11, %if.else ], [ %add, %if.then5 ]
+  store i32 %storemerge, i32 addrspace(4)* %arrayidx10, align 4
+  %inc = add nuw nsw i32 %i.026, 1
+  %exitcond = icmp ne i32 %inc, %0
+  br i1 %exitcond, label %if.end.while.body_crit_edge, label %while.end.loopexit
+
+if.end.while.body_crit_edge:
+  br label %while.body
+
+while.end.loopexit:
+; CHECK: DIVERGENT:       %storemerge.lcssa = phi i32 [ %storemerge, %if.end ]
+  %storemerge.lcssa = phi i32 [ %storemerge, %if.end ]
+  br label %while.end
+
+while.end:
+; CHECK: DIVERGENT:       %4 = phi i32 [ %.pre, %if.then.while.end_crit_edge ], [ %storemerge.lcssa, %while.end.loopexit ]
+; CHECK: DIVERGENT:       %i.0.lcssa = phi i32 [ 0, %if.then.while.end_crit_edge ], [ %0, %while.end.loopexit ]
+; CHECK: DIVERGENT:       %sub = sub nsw i32 %4, %i.0.lcssa
+; CHECK: DIVERGENT:       store i32 %sub, i32 addrspace(4)* %arrayidx10, align 4
+  %4 = phi i32 [ %.pre, %if.then.while.end_crit_edge ], [ %storemerge.lcssa, %while.end.loopexit ]
+  %i.0.lcssa = phi i32 [ 0, %if.then.while.end_crit_edge ], [ %0, %while.end.loopexit ]
+  %sub = sub nsw i32 %4, %i.0.lcssa
+  store i32 %sub, i32 addrspace(4)* %arrayidx10, align 4
+  br label %new_exit
+
+new_exit:
+  ret void
+}
+
+attributes #0 = { nounwind readnone }
+attributes #1 = { nounwind readnone }
+attributes #2 = { nounwind readnone }
+attributes #3 = { nounwind readnone }
+attributes #4 = { nounwind readnone }
diff --git a/llvm/test/Analysis/GlobalsModRef/intrinsic_addressnottaken1.ll b/llvm/test/Analysis/GlobalsModRef/intrinsic_addressnottaken1.ll
new file mode 100644
index 000000000..934534b5b
--- /dev/null
+++ b/llvm/test/Analysis/GlobalsModRef/intrinsic_addressnottaken1.ll
@@ -0,0 +1,44 @@
+; RUN: opt -globals-aa -gvn -S < %s | FileCheck %s
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@deallocCalled = internal global i8 0, align 1
+
+; CHECK-LABEL: @main()
+define dso_local i32 @main() {
+entry:
+  %tmp0 = call i8* @llvm.objc.autoreleasePoolPush() #1
+  %tmp6 = load i8, i8* @deallocCalled, align 1
+  %tobool = icmp ne i8 %tmp6, 0
+  br i1 %tobool, label %if.else, label %if.end
+
+if.else:                                          ; preds = %entry
+  call void @__assert_fail() #0
+  unreachable
+
+; CHECK-LABEL: if.end:
+; CHECK-NEXT: call void @llvm.objc.autoreleasePoolPop
+; CHECK-NOT: load i8, i8* @deallocCalled
+if.end:                                           ; preds = %entry
+  call void @llvm.objc.autoreleasePoolPop(i8* %tmp0)
+  %tmp7 = load i8, i8* @deallocCalled, align 1
+  %tobool3 = icmp ne i8 %tmp7, 0
+  br i1 %tobool3, label %if.end6, label %if.else5
+
+if.else5:                                         ; preds = %if.end
+  call void @__assert_fail() #0
+  unreachable
+
+if.end6:                                          ; preds = %if.end
+  store i8 0, i8* @deallocCalled, align 1
+  ret i32 0
+}
+
+declare i8* @llvm.objc.autoreleasePoolPush() #1
+declare void @llvm.objc.autoreleasePoolPop(i8*) #1
+declare dso_local void @__assert_fail() #0
+
+attributes #0 = { noreturn nounwind }
+attributes #1 = { nounwind }
+
diff --git a/llvm/test/Analysis/GlobalsModRef/intrinsic_addressnottaken2.ll b/llvm/test/Analysis/GlobalsModRef/intrinsic_addressnottaken2.ll
new file mode 100644
index 000000000..d9874d714
--- /dev/null
+++ b/llvm/test/Analysis/GlobalsModRef/intrinsic_addressnottaken2.ll
@@ -0,0 +1,50 @@
+; RUN: opt -globals-aa -gvn -S < %s | FileCheck %s
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@deallocCalled = internal global i8 0, align 1
+
+define internal i8* @_i_Associated__dealloc() {
+entry:
+  store i8 1, i8* @deallocCalled, align 1
+  ret i8* null
+}
+
+; CHECK-LABEL: @main()
+define dso_local i32 @main() {
+entry:
+  %tmp0 = call i8* @llvm.objc.autoreleasePoolPush() #1
+  %tmp6 = load i8, i8* @deallocCalled, align 1
+  %tobool = icmp ne i8 %tmp6, 0
+  br i1 %tobool, label %if.else, label %if.end
+
+if.else:                                          ; preds = %entry
+  call void @__assert_fail() #0
+  unreachable
+
+; CHECK-LABEL: if.end:
+; CHECK-NEXT: call void @llvm.objc.autoreleasePoolPop
+; CHECK-NOT: load i8, i8* @deallocCalled
+if.end:                                           ; preds = %entry
+  call void @llvm.objc.autoreleasePoolPop(i8* %tmp0)
+  %tmp7 = load i8, i8* @deallocCalled, align 1
+  %tobool3 = icmp ne i8 %tmp7, 0
+  br i1 %tobool3, label %if.end6, label %if.else5
+
+if.else5:                                         ; preds = %if.end
+  call void @__assert_fail() #0
+  unreachable
+
+if.end6:                                          ; preds = %if.end
+  store i8 0, i8* @deallocCalled, align 1
+  ret i32 0
+}
+
+declare i8* @llvm.objc.autoreleasePoolPush() #1
+declare void @llvm.objc.autoreleasePoolPop(i8*) #1
+declare dso_local void @__assert_fail() #0
+
+attributes #0 = { noreturn nounwind }
+attributes #1 = { nounwind }
+
diff --git a/llvm/test/Analysis/GlobalsModRef/intrinsic_addresstaken.ll b/llvm/test/Analysis/GlobalsModRef/intrinsic_addresstaken.ll
new file mode 100644
index 000000000..613533254
--- /dev/null
+++ b/llvm/test/Analysis/GlobalsModRef/intrinsic_addresstaken.ll
@@ -0,0 +1,52 @@
+; RUN: opt -globals-aa -gvn -S < %s | FileCheck %s
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@deallocCalled = internal global i8 0, align 1
+@.objc_method_list = internal global { i8* ()* } { i8* ()*  @_i_Associated__dealloc }, align 8
+@._OBJC_CLASS_Associated = global { i8* } { i8* bitcast ({ i8* ()* }* @.objc_method_list to i8*) }, align 8
+@._OBJC_INIT_CLASS_Associated = global { i8* }* @._OBJC_CLASS_Associated
+@llvm.used = appending global [1 x i8*] [i8* bitcast ({ i8* }** @._OBJC_INIT_CLASS_Associated to i8*)]
+
+define internal i8* @_i_Associated__dealloc() {
+entry:
+  store i8 1, i8* @deallocCalled, align 1
+  ret i8* null
+}
+
+; CHECK-LABEL: @main()
+define dso_local i32 @main() {
+entry:
+  %tmp0 = call i8* @llvm.objc.autoreleasePoolPush() #1
+  %tmp6 = load i8, i8* @deallocCalled, align 1
+  %tobool = icmp ne i8 %tmp6, 0
+  br i1 %tobool, label %if.else, label %if.end
+
+if.else:                                          ; preds = %entry
+  call void @__assert_fail() #0
+  unreachable
+
+; CHECK-LABEL: if.end:
+; CHECK-NEXT: call void @llvm.objc.autoreleasePoolPop
+if.end:                                           ; preds = %entry
+  call void @llvm.objc.autoreleasePoolPop(i8* %tmp0)
+  %tmp7 = load i8, i8* @deallocCalled, align 1
+  %tobool3 = icmp ne i8 %tmp7, 0
+  br i1 %tobool3, label %if.end6, label %if.else5
+
+if.else5:                                         ; preds = %if.end
+  call void @__assert_fail() #0
+  unreachable
+
+if.end6:                                          ; preds = %if.end
+  store i8 0, i8* @deallocCalled, align 1
+  ret i32 0
+}
+
+declare i8* @llvm.objc.autoreleasePoolPush() #1
+declare void @llvm.objc.autoreleasePoolPop(i8*) #1
+declare dso_local void @__assert_fail() #0
+
+attributes #0 = { noreturn nounwind }
+attributes #1 = { nounwind }
diff --git a/llvm/test/Analysis/LazyValueAnalysis/lvi-after-jumpthreading.ll b/llvm/test/Analysis/LazyValueAnalysis/lvi-after-jumpthreading.ll
index fbc2eb135..303c7c4c1 100644
--- a/llvm/test/Analysis/LazyValueAnalysis/lvi-after-jumpthreading.ll
+++ b/llvm/test/Analysis/LazyValueAnalysis/lvi-after-jumpthreading.ll
@@ -53,9 +53,9 @@ entry:
 
 ; CHECK-LABEL: loop:
 ; CHECK-NEXT:    ; LatticeVal for: 'i32 %n' is: overdefined
-; CHECK-NEXT:    ; LatticeVal for: '  %iv = phi i32 [ 0, %entry ], [ %iv.next, %backedge ]' in BB: '%loop' is: constantrange<0, -2147483647>
+; CHECK-NEXT:    ; LatticeVal for: '  %iv = phi i32 [ 0, %entry ], [ %iv.next, %backedge ]' in BB: '%loop' is: constantrange<0, 400>
 ; CHECK-DAG:     ; LatticeVal for: '  %iv = phi i32 [ 0, %entry ], [ %iv.next, %backedge ]' in BB: '%backedge' is: constantrange<0, -2147483648>
-; CHECK-DAG:     ; LatticeVal for: '  %iv = phi i32 [ 0, %entry ], [ %iv.next, %backedge ]' in BB: '%exit' is: constantrange<0, -2147483647>
+; CHECK-DAG:     ; LatticeVal for: '  %iv = phi i32 [ 0, %entry ], [ %iv.next, %backedge ]' in BB: '%exit' is: constantrange<0, -2147483648>
 ; CHECK-NEXT:  %iv = phi i32 [ 0, %entry ], [ %iv.next, %backedge ]
 loop:
   %iv = phi i32 [0, %entry], [%iv.next, %backedge]
@@ -81,7 +81,7 @@ loop:
 
 ; CHECK-LABEL: backedge:
 ; CHECK-NEXT:    ; LatticeVal for: 'i32 %n' is: overdefined
-; CHECK-NEXT:    ; LatticeVal for: '  %iv.next = add nsw i32 %iv, 1' in BB: '%backedge' is: constantrange<1, -2147483647>
+; CHECK-NEXT:    ; LatticeVal for: '  %iv.next = add nsw i32 %iv, 1' in BB: '%backedge' is: constantrange<1, -2147483648>
 ; CHECK-NEXT:  %iv.next = add nsw i32 %iv, 1
 backedge:
   %iv.next = add nsw i32 %iv, 1
@@ -108,7 +108,7 @@ define i32 @test3(i32 %i, i1 %f, i32 %n) {
 ; CHECK-LABEL: LVI for function 'test3':
 ; CHECK-LABEL: entry
 ; CHECK:  ; LatticeVal for: 'i32 %i' is: overdefined
-; CHECK: %c = icmp ne i32 %i, -2134 
+; CHECK: %c = icmp ne i32 %i, -2134
 ; CHECK: br i1 %c, label %cont, label %exit
 entry:
   %c = icmp ne i32 %i, -2134
@@ -141,7 +141,7 @@ do:
   call void (i1, ...) @llvm.experimental.guard(i1 %cond) [ "deopt"() ]
   %cond.2 = icmp sgt i32 %i, 0
   br i1 %cond.2, label %exit, label %cont
-  
+
 exit2:
 ; CHECK-LABEL: exit2:
 ; LatticeVal for: 'i32 %i' is: constantrange<-2134, 1>
diff --git a/llvm/test/Analysis/LoopCacheAnalysis/PowerPC/compute-cost.ll b/llvm/test/Analysis/LoopCacheAnalysis/PowerPC/compute-cost.ll
new file mode 100644
index 000000000..8afde3697
--- /dev/null
+++ b/llvm/test/Analysis/LoopCacheAnalysis/PowerPC/compute-cost.ll
@@ -0,0 +1,35 @@
+; RUN: opt < %s -passes='print<loop-cache-cost>' -disable-output 2>&1 | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-n32:64"
+target triple = "powerpc64le-unknown-linux-gnu"
+
+; Check IndexedReference::computeRefCost can handle type differences between
+; Stride and TripCount
+
+; CHECK: Loop 'for.cond' has cost = 64
+
+%struct._Handleitem = type { %struct._Handleitem* }
+
+define void @handle_to_ptr(%struct._Handleitem** %blocks) {
+; Preheader:
+entry:
+  br label %for.cond
+
+; Loop:
+for.cond:                                         ; preds = %for.body, %entry
+  %i.0 = phi i32 [ 1, %entry ], [ %inc, %for.body ]
+  %cmp = icmp ult i32 %i.0, 1024
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %idxprom = zext i32 %i.0 to i64
+  %arrayidx = getelementptr inbounds %struct._Handleitem*, %struct._Handleitem** %blocks, i64 %idxprom
+  store %struct._Handleitem* null, %struct._Handleitem** %arrayidx, align 8
+  %inc = add nuw nsw i32 %i.0, 1
+  br label %for.cond
+
+; Exit blocks
+for.end:                                          ; preds = %for.cond
+  ret void
+
+}
diff --git a/llvm/test/Analysis/LoopCacheAnalysis/PowerPC/loads-store.ll b/llvm/test/Analysis/LoopCacheAnalysis/PowerPC/loads-store.ll
new file mode 100644
index 000000000..b73abcac9
--- /dev/null
+++ b/llvm/test/Analysis/LoopCacheAnalysis/PowerPC/loads-store.ll
@@ -0,0 +1,88 @@
+; RUN: opt < %s -passes='print<loop-cache-cost>' -disable-output 2>&1 | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-n32:64"
+target triple = "powerpc64le-unknown-linux-gnu"
+
+; void foo(long n, long m, long o, int A[n][m][o], int B[n][m][o], int C[n][m][o]) {
+;   for (long i = 0; i < n; i++)
+;     for (long j = 0; j < m; j++)
+;       for (long k = 0; k < o; k++)
+;         A[i][k][j] += B[i][k][j] + C[i][j][k];
+; }
+
+; CHECK-DAG: Loop 'for.i' has cost = 3000000
+; CHECK-DAG: Loop 'for.k' has cost = 2030000
+; CHECK-DAG: Loop 'for.j' has cost = 1060000
+
+define void @foo(i64 %n, i64 %m, i64 %o, i32* %A, i32* %B, i32* %C) {
+entry:
+  %cmp32 = icmp sgt i64 %n, 0
+  %cmp230 = icmp sgt i64 %m, 0
+  %cmp528 = icmp sgt i64 %o, 0
+  br i1 %cmp32, label %for.cond1.preheader.lr.ph, label %for.end
+
+for.cond1.preheader.lr.ph:                        ; preds = %entry
+  br i1 %cmp230, label %for.i.preheader, label %for.end
+
+for.i.preheader:                                  ; preds = %for.cond1.preheader.lr.ph
+  br i1 %cmp528, label %for.i.preheader.split, label %for.end
+
+for.i.preheader.split:                            ; preds = %for.i.preheader
+  br label %for.i
+
+for.i:                                            ; preds = %for.inci, %for.i.preheader.split
+  %i = phi i64 [ %inci, %for.inci ], [ 0, %for.i.preheader.split ]
+  %muli = mul i64 %i, %m
+  br label %for.j
+
+for.j:                                            ; preds = %for.incj, %for.i
+  %j = phi i64 [ %incj, %for.incj ], [ 0, %for.i ]
+  %addj = add i64 %muli, %j
+  %mulj = mul i64 %addj, %o
+  br label %for.k
+
+for.k:                                            ; preds = %for.k, %for.j
+  %k = phi i64 [ 0, %for.j ], [ %inck, %for.k ]
+
+  ; B[i][k][j]
+  %addk = add i64 %muli, %k
+  %mulk = mul i64 %addk, %o
+  %arrayidx1 = add i64 %j, %mulk
+  %arrayidx2 = getelementptr inbounds i32, i32* %B, i64 %arrayidx1
+  %elem_B = load i32, i32* %arrayidx2, align 4
+
+  ; C[i][j][k]
+  %arrayidx3 = add i64 %k, %mulj
+  %arrayidx4 = getelementptr inbounds i32, i32* %C, i64 %arrayidx3
+  %elem_C = load i32, i32* %arrayidx4, align 4
+
+  ; A[i][k][j]
+  %arrayidx5 = getelementptr inbounds i32, i32* %A, i64 %arrayidx1
+  %elem_A = load i32, i32* %arrayidx5, align 4
+
+  ; A[i][k][j] += B[i][k][j] + C[i][j][k]
+  %add1 = add i32 %elem_B, %elem_C
+  %add2 = add i32 %add1, %elem_A
+  %arrayidx6 = getelementptr inbounds i32, i32* %A, i64 %arrayidx1
+  store i32 %add2, i32* %arrayidx6, align 4
+
+  %inck = add nsw i64 %k, 1
+  %exitcond.us = icmp eq i64 %inck, %o
+  br i1 %exitcond.us, label %for.incj, label %for.k
+
+for.incj:                                         ; preds = %for.k
+  %incj = add nsw i64 %j, 1
+  %exitcond54.us = icmp eq i64 %incj, %m
+  br i1 %exitcond54.us, label %for.inci, label %for.j
+
+for.inci:                                         ; preds = %for.incj
+  %inci = add nsw i64 %i, 1
+  %exitcond55.us = icmp eq i64 %inci, %n
+  br i1 %exitcond55.us, label %for.end.loopexit, label %for.i
+
+for.end.loopexit:                                 ; preds = %for.inci
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %for.cond1.preheader.lr.ph, %entry
+  ret void
+}
diff --git a/llvm/test/Analysis/LoopCacheAnalysis/PowerPC/matmul.ll b/llvm/test/Analysis/LoopCacheAnalysis/PowerPC/matmul.ll
new file mode 100644
index 000000000..c7342fe32
--- /dev/null
+++ b/llvm/test/Analysis/LoopCacheAnalysis/PowerPC/matmul.ll
@@ -0,0 +1,81 @@
+; RUN: opt < %s -passes='print<loop-cache-cost>' -disable-output 2>&1 | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-n32:64"
+target triple = "powerpc64le-unknown-linux-gnu"
+
+; void matmul(long n, long m, long o, int A[n][m], int B[n][m], int C[n]) {
+;   for (long i = 0; i < n; i++) 
+;     for (long j = 0; j < m; j++) 
+;       for (long k = 0; k < o; k++) 
+;         C[i][j] = C[i][j] + A[i][k] * B[k][j];
+; }
+
+; CHECK-DAG:Loop 'for.i' has cost = 2010000
+; CHECK-DAG:Loop 'for.k' has cost = 1040000
+; CHECK-DAG:Loop 'for.j' has cost = 70000
+    
+define void @matmul(i64 %n, i64 %m, i64 %o, i32* %A, i32* %B, i32* %C) {
+entry:
+  br label %for.i
+
+for.i:                                         ; preds = %entry, %for.inc.i
+  %i = phi i64 [ 0, %entry ], [ %i.next, %for.inc.i ]
+  %muli = mul i64 %i, %m
+  br label %for.j
+
+for.j:                                        ; preds = %for.i, %for.inc.j
+  %j = phi i64 [ 0, %for.i ], [ %j.next, %for.inc.j ]
+  %addj = add i64 %muli, %j
+  %mulj = mul i64 %addj, %o
+  br label %for.k
+
+for.k:                                        ; preds = %for.j, %for.inc.k
+  %k = phi i64 [ 0, %for.j ], [ %k.next, %for.inc.k ]
+
+  ; A[i][k]
+  %arrayidx3 = add i64 %k, %muli
+  %arrayidx4 = getelementptr inbounds i32, i32* %A, i64 %arrayidx3
+  %elem_A = load i32, i32* %arrayidx4, align 4
+
+  ; B[k][j]
+  %mulk = mul i64 %k, %o
+  %arrayidx5 = add i64 %j, %mulk
+  %arrayidx6 = getelementptr inbounds i32, i32* %B, i64 %arrayidx5
+  %elem_B = load i32, i32* %arrayidx6, align 4
+
+  ; C[i][k]
+  %arrayidx7 = add i64 %j, %muli
+  %arrayidx8 = getelementptr inbounds i32, i32* %C, i64 %arrayidx7
+  %elem_C = load i32, i32* %arrayidx8, align 4
+
+  ; C[i][j] = C[i][j] + A[i][k] * B[k][j];
+  %mul = mul nsw i32 %elem_A, %elem_B
+  %add = add nsw i32 %elem_C, %mul
+  store i32 %add, i32* %arrayidx8, align 4
+
+  br label %for.inc.k
+
+for.inc.k:                                          ; preds = %for.k
+  %k.next = add nuw nsw i64 %k, 1
+  %exitcond = icmp ne i64 %k.next, %o
+  br i1 %exitcond, label %for.k, label %for.end
+
+for.end:                                          ; preds = %for.inc
+  br label %for.inc.j
+
+for.inc.j:                                        ; preds = %for.end
+  %j.next = add nuw nsw i64 %j, 1
+  %exitcond5 = icmp ne i64 %j.next, %m
+  br i1 %exitcond5, label %for.j, label %for.end23
+
+for.end23:                                        ; preds = %for.inc.j
+  br label %for.inc.i
+
+for.inc.i:                                        ; preds = %for.end23
+  %i.next = add nuw nsw i64 %i, 1
+  %exitcond8 = icmp ne i64 %i.next, %n
+  br i1 %exitcond8, label %for.i, label %for.end26
+
+for.end26:                                        ; preds = %for.inc.i
+  ret void
+}
diff --git a/llvm/test/Analysis/LoopCacheAnalysis/PowerPC/matvecmul.ll b/llvm/test/Analysis/LoopCacheAnalysis/PowerPC/matvecmul.ll
new file mode 100644
index 000000000..313795251
--- /dev/null
+++ b/llvm/test/Analysis/LoopCacheAnalysis/PowerPC/matvecmul.ll
@@ -0,0 +1,185 @@
+; RUN: opt < %s -passes='print<loop-cache-cost>' -disable-output 2>&1 | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-n32:64"
+target triple = "powerpc64le-unknown-linux-gnu"
+
+; void matvecmul(const double *__restrict y, const double * __restrict x, const double * __restrict b,
+;    const int * __restrict nb, const int * __restrict nx, const int * __restrict ny, const int * __restrict nz) {
+;
+;   for (int k=1;k<nz,++k) 
+;      for (int j=1;j<ny,++j)
+;        for (int i=1;i<nx,++i)
+;          for (int l=1;l<nb,++l)
+;            for (int m=1;m<nb,++m)
+;                 y[k+1][j][i][l] = y[k+1][j][i][l] + b[k][j][i][m][l]*x[k][j][i][m]
+; }
+
+; CHECK-DAG: Loop 'k_loop' has cost = 30000000000
+; CHECK-DAG: Loop 'j_loop' has cost = 30000000000
+; CHECK-DAG: Loop 'i_loop' has cost = 30000000000
+; CHECK-DAG: Loop 'm_loop' has cost = 10700000000
+; CHECK-DAG: Loop 'l_loop' has cost = 1300000000
+
+%_elem_type_of_double = type <{ double }>
+
+; Function Attrs: norecurse nounwind
+define void @mat_vec_mpy([0 x %_elem_type_of_double]* noalias %y, [0 x %_elem_type_of_double]* noalias readonly %x,
+    [0 x %_elem_type_of_double]* noalias readonly %b, i32* noalias readonly %nb, i32* noalias readonly %nx, 
+    i32* noalias readonly %ny, i32* noalias readonly %nz) {
+mat_times_vec_entry:
+  %_ind_val = load i32, i32* %nb, align 4
+  %_conv = sext i32 %_ind_val to i64
+  %_grt_tmp.i = icmp sgt i64 %_conv, 0
+  %a_b.i = select i1 %_grt_tmp.i, i64 %_conv, i64 0
+  %_ind_val1 = load i32, i32* %nx, align 4
+  %_conv2 = sext i32 %_ind_val1 to i64
+  %_grt_tmp.i266 = icmp sgt i64 %_conv2, 0
+  %a_b.i267 = select i1 %_grt_tmp.i266, i64 %_conv2, i64 0
+  %_ind_val3 = load i32, i32* %ny, align 4
+  %_conv4 = sext i32 %_ind_val3 to i64
+  %_grt_tmp.i264 = icmp sgt i64 %_conv4, 0
+  %a_b.i265 = select i1 %_grt_tmp.i264, i64 %_conv4, i64 0
+  %_ind_val5 = load i32, i32* %nz, align 4
+  %_mult_tmp = shl nsw i64 %a_b.i, 3
+  %_mult_tmp7 = mul i64 %_mult_tmp, %a_b.i267
+  %_mult_tmp8 = mul i64 %_mult_tmp7, %a_b.i265
+  %_sub_tmp = sub nuw nsw i64 -8, %_mult_tmp
+  %_sub_tmp21 = sub i64 %_sub_tmp, %_mult_tmp7
+  %_sub_tmp23 = sub i64 %_sub_tmp21, %_mult_tmp8
+  %_mult_tmp73 = mul i64 %_mult_tmp, %a_b.i
+  %_mult_tmp74 = mul i64 %_mult_tmp73, %a_b.i267
+  %_mult_tmp75 = mul i64 %_mult_tmp74, %a_b.i265
+  %_sub_tmp93 = sub i64 %_sub_tmp, %_mult_tmp73
+  %_sub_tmp95 = sub i64 %_sub_tmp93, %_mult_tmp74
+  %_sub_tmp97 = sub i64 %_sub_tmp95, %_mult_tmp75
+  %_grt_tmp853288 = icmp slt i32 %_ind_val5, 1
+  br i1 %_grt_tmp853288, label %_return_bb, label %k_loop.lr.ph
+
+k_loop.lr.ph:                                     ; preds = %mat_times_vec_entry
+  %_grt_tmp851279 = icmp slt i32 %_ind_val3, 1
+  %_grt_tmp847270 = icmp slt i32 %_ind_val, 1
+  %_aa_conv = bitcast [0 x %_elem_type_of_double]* %y to i8*
+  %_adda_ = getelementptr inbounds i8, i8* %_aa_conv, i64 %_sub_tmp23
+  %_aa_conv434 = bitcast [0 x %_elem_type_of_double]* %x to i8*
+  %_adda_435 = getelementptr inbounds i8, i8* %_aa_conv434, i64 %_sub_tmp23
+  %_aa_conv785 = bitcast [0 x %_elem_type_of_double]* %b to i8*
+  %_adda_786 = getelementptr inbounds i8, i8* %_aa_conv785, i64 %_sub_tmp97
+  br i1 %_grt_tmp851279, label %k_loop.us.preheader, label %k_loop.lr.ph.split
+
+k_loop.us.preheader:                              ; preds = %k_loop.lr.ph
+  br label %_return_bb.loopexit
+
+k_loop.lr.ph.split:                               ; preds = %k_loop.lr.ph
+  %_grt_tmp849273 = icmp slt i32 %_ind_val1, 1
+  br i1 %_grt_tmp849273, label %k_loop.us291.preheader, label %k_loop.lr.ph.split.split
+
+k_loop.us291.preheader:                           ; preds = %k_loop.lr.ph.split
+  br label %_return_bb.loopexit300
+
+k_loop.lr.ph.split.split:                         ; preds = %k_loop.lr.ph.split
+  br i1 %_grt_tmp847270, label %k_loop.us294.preheader, label %k_loop.preheader
+
+k_loop.preheader:                                 ; preds = %k_loop.lr.ph.split.split
+  %0 = add i32 %_ind_val, 1
+  %1 = add i32 %_ind_val1, 1
+  %2 = add i32 %_ind_val3, 1
+  %3 = add i32 %_ind_val5, 1
+  br label %k_loop
+
+k_loop.us294.preheader:                           ; preds = %k_loop.lr.ph.split.split
+  br label %_return_bb.loopexit301
+
+k_loop:                                           ; preds = %k_loop._label_18_crit_edge.split.split.split, %k_loop.preheader
+  %indvars.iv316 = phi i64 [ 1, %k_loop.preheader ], [ %indvars.iv.next317, %k_loop._label_18_crit_edge.split.split.split ]
+  %indvars.iv.next317 = add nuw nsw i64 %indvars.iv316, 1
+  %_ix_x_len = mul i64 %_mult_tmp8, %indvars.iv.next317
+  %_ix_x_len410 = mul i64 %_mult_tmp75, %indvars.iv316
+  %_ix_x_len822 = mul i64 %_mult_tmp8, %indvars.iv316
+  br label %j_loop
+
+j_loop:                                           ; preds = %j_loop._label_15_crit_edge.split.split, %k_loop
+  %indvars.iv312 = phi i64 [ %indvars.iv.next313, %j_loop._label_15_crit_edge.split.split ], [ 1, %k_loop ]
+  %_ix_x_len371 = mul i64 %_mult_tmp7, %indvars.iv312
+  %_ix_x_len415 = mul i64 %_mult_tmp74, %indvars.iv312
+  br label %i_loop
+
+i_loop:                                           ; preds = %i_loop._label_12_crit_edge.split, %j_loop
+  %indvars.iv307 = phi i64 [ %indvars.iv.next308, %i_loop._label_12_crit_edge.split ], [ 1, %j_loop ]
+  %_ix_x_len375 = mul i64 %_mult_tmp, %indvars.iv307
+  %_ix_x_len420 = mul i64 %_mult_tmp73, %indvars.iv307
+  br label %l_loop
+
+l_loop:                                           ; preds = %l_loop._label_9_crit_edge, %i_loop
+  %indvars.iv303 = phi i64 [ %indvars.iv.next304, %l_loop._label_9_crit_edge ], [ 1, %i_loop ]
+  %_ix_x_len378 = shl nuw nsw i64 %indvars.iv303, 3
+  br label %m_loop
+
+m_loop:                                           ; preds = %m_loop, %l_loop
+  %indvars.iv = phi i64 [ %indvars.iv.next, %m_loop ], [ 1, %l_loop ]
+  %_ix_x_len424 = mul i64 %_mult_tmp, %indvars.iv
+  %_ix_x_len454 = shl nuw nsw i64 %indvars.iv, 3
+  %_ixa_gep = getelementptr inbounds i8, i8* %_adda_, i64 %_ix_x_len
+  %_ixa_gep791 = getelementptr inbounds i8, i8* %_adda_786, i64 %_ix_x_len410
+  %_ixa_gep823 = getelementptr inbounds i8, i8* %_adda_435, i64 %_ix_x_len822
+  %_ixa_gep372 = getelementptr inbounds i8, i8* %_ixa_gep, i64 %_ix_x_len371
+  %_ixa_gep376 = getelementptr inbounds i8, i8* %_ixa_gep372, i64 %_ix_x_len375
+  %_ixa_gep796 = getelementptr inbounds i8, i8* %_ixa_gep791, i64 %_ix_x_len415
+  %_ixa_gep828 = getelementptr inbounds i8, i8* %_ixa_gep823, i64 %_ix_x_len371
+  %_ixa_gep379 = getelementptr inbounds i8, i8* %_ixa_gep376, i64 %_ix_x_len378
+  %_ixa_gep801 = getelementptr inbounds i8, i8* %_ixa_gep796, i64 %_ix_x_len420
+  %_ixa_gep833 = getelementptr inbounds i8, i8* %_ixa_gep828, i64 %_ix_x_len375
+  %_ixa_gep806 = getelementptr inbounds i8, i8* %_ixa_gep801, i64 %_ix_x_len378
+  %_ixa_gep810 = getelementptr inbounds i8, i8* %_ixa_gep806, i64 %_ix_x_len424
+  %_gepp = bitcast i8* %_ixa_gep379 to double*
+  %_gepp813 = bitcast i8* %_ixa_gep810 to double*
+  %_ind_val814 = load double, double* %_gepp813, align 8
+  %_ixa_gep837 = getelementptr inbounds i8, i8* %_ixa_gep833, i64 %_ix_x_len454
+  %_gepp840 = bitcast i8* %_ixa_gep837 to double*
+  %_ind_val841 = load double, double* %_gepp840, align 8
+  %_mult_tmp842 = fmul double %_ind_val814, %_ind_val841
+  store double %_mult_tmp842, double* %_gepp, align 8
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %wide.trip.count = zext i32 %0 to i64
+  %wide.trip.count305 = zext i32 %0 to i64
+  %wide.trip.count309 = zext i32 %1 to i64
+  %wide.trip.count314 = zext i32 %2 to i64
+  %wide.trip.count319 = zext i32 %3 to i64
+  %exitcond = icmp ne i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond, label %m_loop, label %l_loop._label_9_crit_edge
+
+l_loop._label_9_crit_edge:                        ; preds = %m_loop
+  %indvars.iv.next304 = add nuw nsw i64 %indvars.iv303, 1
+  %exitcond306 = icmp ne i64 %indvars.iv.next304, %wide.trip.count305
+  br i1 %exitcond306, label %l_loop, label %i_loop._label_12_crit_edge.split
+
+i_loop._label_12_crit_edge.split:                 ; preds = %l_loop._label_9_crit_edge
+  %indvars.iv.next308 = add nuw nsw i64 %indvars.iv307, 1
+  %exitcond310 = icmp ne i64 %indvars.iv.next308, %wide.trip.count309
+  br i1 %exitcond310, label %i_loop, label %j_loop._label_15_crit_edge.split.split
+
+j_loop._label_15_crit_edge.split.split:           ; preds = %i_loop._label_12_crit_edge.split
+  %indvars.iv.next313 = add nuw nsw i64 %indvars.iv312, 1
+  %exitcond315 = icmp ne i64 %indvars.iv.next313, %wide.trip.count314
+  br i1 %exitcond315, label %j_loop, label %k_loop._label_18_crit_edge.split.split.split
+
+k_loop._label_18_crit_edge.split.split.split:     ; preds = %j_loop._label_15_crit_edge.split.split
+  %exitcond320 = icmp ne i64 %indvars.iv.next317, %wide.trip.count319
+  br i1 %exitcond320, label %k_loop, label %_return_bb.loopexit302
+
+_return_bb.loopexit:                              ; preds = %k_loop.us.preheader
+  br label %_return_bb
+
+_return_bb.loopexit300:                           ; preds = %k_loop.us291.preheader
+  br label %_return_bb
+
+_return_bb.loopexit301:                           ; preds = %k_loop.us294.preheader
+  br label %_return_bb
+
+_return_bb.loopexit302:                           ; preds = %k_loop._label_18_crit_edge.split.split.split
+  br label %_return_bb
+
+_return_bb:                                       ; preds = %_return_bb.loopexit302, %_return_bb.loopexit301, %_return_bb.loopexit300, %_return_bb.loopexit, %mat_times_vec_entry
+  ret void
+}
+
+
diff --git a/llvm/test/Analysis/LoopCacheAnalysis/PowerPC/single-store.ll b/llvm/test/Analysis/LoopCacheAnalysis/PowerPC/single-store.ll
new file mode 100644
index 000000000..cce6731fb
--- /dev/null
+++ b/llvm/test/Analysis/LoopCacheAnalysis/PowerPC/single-store.ll
@@ -0,0 +1,77 @@
+; RUN: opt < %s -passes='print<loop-cache-cost>' -disable-output 2>&1 | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-n32:64"
+target triple = "powerpc64le-unknown-linux-gnu"
+
+; void foo(long n, long m, long o, int A[n][m][o]) {
+;   for (long i = 0; i < n; i++)
+;     for (long j = 0; j < m; j++)
+;       for (long k = 0; k < o; k++)
+;         A[2*i+3][3*j-4][2*k+7] = 1;
+; }
+
+; CHECK-DAG: Loop 'for.i' has cost = 1000000
+; CHECK-DAG: Loop 'for.j' has cost = 1000000
+; CHECK-DAG: Loop 'for.k' has cost = 60000
+
+define void @foo(i64 %n, i64 %m, i64 %o, i32* %A) {
+entry:
+  %cmp32 = icmp sgt i64 %n, 0
+  %cmp230 = icmp sgt i64 %m, 0
+  %cmp528 = icmp sgt i64 %o, 0
+  br i1 %cmp32, label %for.cond1.preheader.lr.ph, label %for.end
+
+for.cond1.preheader.lr.ph:                        ; preds = %entry
+  br i1 %cmp230, label %for.i.preheader, label %for.end
+
+for.i.preheader:                                  ; preds = %for.cond1.preheader.lr.ph
+  br i1 %cmp528, label %for.i.preheader.split, label %for.end
+
+for.i.preheader.split:                            ; preds = %for.i.preheader
+  br label %for.i
+
+for.i:                                            ; preds = %for.inci, %for.i.preheader.split
+  %i = phi i64 [ %inci, %for.inci ], [ 0, %for.i.preheader.split ]
+  %mul8 = shl i64 %i, 1
+  %add9 = add nsw i64 %mul8, 3
+  %0 = mul i64 %add9, %m
+  %sub = add i64 %0, -4
+  br label %for.j
+
+for.j:                                            ; preds = %for.incj, %for.i
+  %j = phi i64 [ %incj, %for.incj ], [ 0, %for.i ]
+  %mul7 = mul nsw i64 %j, 3
+  %tmp = add i64 %sub, %mul7
+  %tmp27 = mul i64 %tmp, %o
+  br label %for.k
+
+for.k:                                            ; preds = %for.k, %for.j.us
+  %k = phi i64 [ 0, %for.j ], [ %inck, %for.k ]
+
+  %mul = mul nsw i64 %k, 2
+  %arrayidx.sum = add i64 %mul, 7
+  %arrayidx10.sum = add i64 %arrayidx.sum, %tmp27
+  %arrayidx11 = getelementptr inbounds i32, i32* %A, i64 %arrayidx10.sum
+  store i32 1, i32* %arrayidx11, align 4
+
+  %inck = add nsw i64 %k, 1
+  %exitcond.us = icmp eq i64 %inck, %o
+  br i1 %exitcond.us, label %for.incj, label %for.k
+
+for.incj:                                         ; preds = %for.k
+  %incj = add nsw i64 %j, 1
+  %exitcond54.us = icmp eq i64 %incj, %m
+  br i1 %exitcond54.us, label %for.inci, label %for.j
+
+for.inci:                                         ; preds = %for.incj
+  %inci = add nsw i64 %i, 1
+  %exitcond55.us = icmp eq i64 %inci, %n
+  br i1 %exitcond55.us, label %for.end.loopexit, label %for.i
+
+for.end.loopexit:                                 ; preds = %for.inci
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %for.cond1.preheader.lr.ph, %entry
+  ret void
+}
+
diff --git a/llvm/test/Analysis/LoopCacheAnalysis/PowerPC/stencil.ll b/llvm/test/Analysis/LoopCacheAnalysis/PowerPC/stencil.ll
new file mode 100644
index 000000000..1f1515435
--- /dev/null
+++ b/llvm/test/Analysis/LoopCacheAnalysis/PowerPC/stencil.ll
@@ -0,0 +1,98 @@
+; RUN: opt < %s -passes='print<loop-cache-cost>' -disable-output 2>&1 | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-n32:64"
+target triple = "powerpc64le-unknown-linux-gnu"
+
+; void foo(long n, long m, long o, int A[n][m], int B[n][m], int C[n]) {
+;   for (long i = 0; i < n; i++)
+;     for (long j = 0; j < m; j++) {
+;         A[i][j] = A[i][j+1] + B[i-1][j] + B[i+1][j+1] + C[i];
+;         A[i][j] += B[i][i];
+;     }   
+; }
+
+; CHECK-DAG: Loop 'for.i' has cost = 20600
+; CHECK-DAG: Loop 'for.j' has cost = 800
+
+define void @foo(i64 %n, i64 %m, i32* %A, i32* %B, i32* %C) {
+entry:
+  %cmp32 = icmp sgt i64 %n, 0
+  %cmp230 = icmp sgt i64 %m, 0
+  br i1 %cmp32, label %for.cond1.preheader.lr.ph, label %for.end
+
+for.cond1.preheader.lr.ph:                        ; preds = %entry
+  br i1 %cmp230, label %for.i.preheader, label %for.end
+
+for.i.preheader:                                  ; preds = %for.cond1.preheader.lr.ph
+  br label %for.i
+
+for.i:                                            ; preds = %for.inci, %for.i.preheader.split
+  %i = phi i64 [ %inci, %for.inci ], [ 0, %for.i.preheader ]
+  %subione = sub i64 %i, 1
+  %addione = add i64 %i, 1
+  %muli = mul i64 %i, %m
+  %muliminusone = mul i64 %subione, %m
+  %muliplusone = mul i64 %addione, %m
+  br label %for.j
+
+for.j:                                            ; preds = %for.incj, %for.i
+  %j = phi i64 [ %incj, %for.incj ], [ 0, %for.i ]
+  %addj = add i64 %muli, %j
+
+  ; B[i-1][j]
+  %arrayidx1 = add i64 %j, %muliminusone
+  %arrayidx2 = getelementptr inbounds i32, i32* %B, i64 %arrayidx1
+  %elem_B1 = load i32, i32* %arrayidx2, align 4
+
+  ; B[i-1][j+1]
+  %addjone = add i64 %j, 1
+  %arrayidx3 = add i64 %addjone, %muliminusone
+  %arrayidx4 = getelementptr inbounds i32, i32* %B, i64 %arrayidx3
+  %elem_B2 = load i32, i32* %arrayidx4, align 4
+
+  ; C[i]
+  %arrayidx6 = getelementptr inbounds i32, i32* %C, i64 %i
+  %elem_C = load i32, i32* %arrayidx6, align 4
+
+  ; A[i][j+1]
+  %arrayidx7 = add i64 %addjone, %muli
+  %arrayidx8 = getelementptr inbounds i32, i32* %A, i64 %arrayidx7
+  %elem_A = load i32, i32* %arrayidx8, align 4
+
+  ; A[i][j] = A[i][j+1] + B[i-1][j] + B[i-1][j+1] + C[i]
+  %addB = add i32 %elem_B1, %elem_B2
+  %addC = add i32 %addB, %elem_C
+  %addA = add i32 %elem_A, %elem_C
+  %arrayidx9 = add i64 %j, %muli
+  %arrayidx10 = getelementptr inbounds i32, i32* %A, i64 %arrayidx9
+  store i32 %addA, i32* %arrayidx10, align 4
+
+  ; A[i][j] += B[i][i];
+  %arrayidx11 = add i64 %j, %muli
+  %arrayidx12 = getelementptr inbounds i32, i32* %A, i64 %arrayidx11
+  %elem_A1 = load i32, i32* %arrayidx12, align 4
+  %arrayidx13 = add i64 %i, %muli
+  %arrayidx14 = getelementptr inbounds i32, i32* %B, i64 %arrayidx13
+  %elem_B3 = load i32, i32* %arrayidx14, align 4
+  %addA1 = add i32 %elem_A1, %elem_B3
+  store i32 %addA1, i32* %arrayidx12, align 4
+
+  br label %for.incj
+
+for.incj:                                         ; preds = %for.j
+  %incj = add nsw i64 %j, 1
+  %exitcond54.us = icmp eq i64 %incj, %m
+  br i1 %exitcond54.us, label %for.inci, label %for.j
+
+for.inci:                                         ; preds = %for.incj
+  %inci = add nsw i64 %i, 1
+  %exitcond55.us = icmp eq i64 %inci, %n
+  br i1 %exitcond55.us, label %for.end.loopexit, label %for.i
+
+for.end.loopexit:                                 ; preds = %for.inci
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %for.cond1.preheader.lr.ph, %entry
+  ret void
+}
+
diff --git a/llvm/test/Analysis/MemoryDependenceAnalysis/invariant.group-bug.ll b/llvm/test/Analysis/MemoryDependenceAnalysis/invariant.group-bug.ll
index 00ed8d6ed..5ef98b888 100644
--- a/llvm/test/Analysis/MemoryDependenceAnalysis/invariant.group-bug.ll
+++ b/llvm/test/Analysis/MemoryDependenceAnalysis/invariant.group-bug.ll
@@ -96,8 +96,8 @@ define void @fail(i1* noalias sret, %0*, %1*, i8*) local_unnamed_addr #0 {
   ret void
 }
 
-attributes #0 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+popcnt,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+popcnt,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+popcnt,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+popcnt,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.linker.options = !{}
 !llvm.module.flags = !{!0, !1, !3, !4, !5}
diff --git a/llvm/test/Analysis/MemorySSA/debugvalue.ll b/llvm/test/Analysis/MemorySSA/debugvalue.ll
new file mode 100644
index 000000000..2d7055627
--- /dev/null
+++ b/llvm/test/Analysis/MemorySSA/debugvalue.ll
@@ -0,0 +1,44 @@
+; RUN: opt -disable-basicaa -loop-rotate -enable-mssa-loop-dependency -verify-memoryssa -S %s | FileCheck %s
+; REQUIRES: asserts
+
+; CHECK-LABEL: @f_w4_i2
+define void @f_w4_i2() {
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.body, %entry
+  %i.0 = phi i16 [ 0, %entry ], [ %inc, %for.body ]
+  call void @llvm.dbg.value(metadata i16 %i.0, metadata !32, metadata !DIExpression()), !dbg !31
+  br i1 undef, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.cond
+  ret void
+
+for.body:                                         ; preds = %for.cond
+  store i32 undef, i32* undef, align 1
+  %inc = add i16 %i.0, 1
+  call void @llvm.dbg.value(metadata i16 %inc, metadata !32, metadata !DIExpression()), !dbg !31
+  br label %for.cond
+}
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.value(metadata, metadata, metadata) #1
+
+attributes #1 = { nounwind readnone speculatable willreturn }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!14}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 10.0.0s", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !3, nameTableKind: None)
+!1 = !DIFile(filename: "vec.c", directory: "test")
+!2 = !{}
+!3 = !{!4}
+!4 = !DIBasicType(name: "int", size: 16, encoding: DW_ATE_signed)
+!14 = !{i32 2, !"Debug Info Version", i32 3}
+!15 = distinct !DISubprogram(name: "f_w4_i2", scope: !1, file: !1, line: 36, type: !16, scopeLine: 38, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !23)
+!16 = !DISubroutineType(types: !17)
+!17 = !{null}
+!23 = !{}
+!30 = distinct !DILexicalBlock(scope: !15, file: !1, line: 43, column: 5)
+!31 = !DILocation(line: 0, scope: !30)
+!32 = !DILocalVariable(name: "i", scope: !30, file: !1, line: 43, type: !4)
diff --git a/llvm/test/Analysis/MemorySSA/debugvalue2.ll b/llvm/test/Analysis/MemorySSA/debugvalue2.ll
new file mode 100644
index 000000000..63144523f
--- /dev/null
+++ b/llvm/test/Analysis/MemorySSA/debugvalue2.ll
@@ -0,0 +1,54 @@
+; RUN: opt -disable-basicaa -print-memoryssa -disable-output %s 2>&1 | FileCheck %s
+
+; Note that the test crashes the MemorySSA verification when doing loop-rotate,
+; if debuginfo is modelled in MemorySSA, due to the fact that MemorySSA is not
+; updated when adding/removing debuginfo intrinsics.
+
+target triple = "x86_64-unknown-linux-gnu"
+
+; CHECK-LABEL: @overflow_iter_var
+; CHECK-NOT: MemoryDef
+define void @overflow_iter_var() !dbg !11 {
+entry:
+  call void @llvm.dbg.value(metadata i16 0, metadata !16, metadata !DIExpression()), !dbg !18
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.body, %entry
+  call void @llvm.dbg.value(metadata i16 0, metadata !16, metadata !DIExpression()), !dbg !18
+  call void @llvm.dbg.value(metadata i16 undef, metadata !20, metadata !DIExpression()), !dbg !21
+  br i1 undef, label %for.end, label %for.body
+
+for.body:                                         ; preds = %for.cond
+  %0 = load i16, i16* undef, align 1
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  ret void
+}
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.value(metadata, metadata, metadata) #1
+
+attributes #1 = { nounwind readnone speculatable willreturn }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!10}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 10.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, globals: !3, nameTableKind: None)
+!1 = !DIFile(filename: "2_loops.c", directory: "/")
+!2 = !{}
+!3 = !{}
+!6 = !DICompositeType(tag: DW_TAG_array_type, baseType: !7, size: 4096, elements: !8)
+!7 = !DIBasicType(name: "int", size: 16, encoding: DW_ATE_signed)
+!8 = !{!9}
+!9 = !DISubrange(count: 256)
+!10 = !{i32 2, !"Debug Info Version", i32 3}
+!11 = distinct !DISubprogram(name: "overflow_iter_var", scope: !1, file: !1, line: 20, type: !12, scopeLine: 21, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !2)
+!12 = !DISubroutineType(types: !13)
+!13 = !{null, !14, !14}
+!14 = !DIBasicType(name: "unsigned int", size: 16, encoding: DW_ATE_unsigned)
+!16 = !DILocalVariable(name: "i", scope: !17, file: !1, line: 23, type: !14)
+!17 = distinct !DILexicalBlock(scope: !11, file: !1, line: 23, column: 3)
+!18 = !DILocation(line: 0, scope: !17)
+!20 = !DILocalVariable(name: "stop1", arg: 1, scope: !11, file: !1, line: 20, type: !14)
+!21 = !DILocation(line: 0, scope: !11)
diff --git a/llvm/test/Analysis/MemorySSA/loop-rotate-disablebasicaa.ll b/llvm/test/Analysis/MemorySSA/loop-rotate-disablebasicaa.ll
new file mode 100644
index 000000000..f601d48f4
--- /dev/null
+++ b/llvm/test/Analysis/MemorySSA/loop-rotate-disablebasicaa.ll
@@ -0,0 +1,26 @@
+; RUN: opt -disable-basicaa -print-memoryssa -disable-output %s 2>&1 | FileCheck %s
+
+; Note: if @foo is modelled as a MemoryDef, this test will assert with -loop-rotate, due to MemorySSA not
+; being preserved when moving instructions that may not read from or write to memory.
+
+; CHECK-LABEL: @main
+; CHECK-NOT: MemoryDef
+define void @main() {
+entry:
+  br label %for.cond120
+
+for.cond120:                                      ; preds = %for.body127, %entry
+  call void @foo()
+  br i1 undef, label %for.body127, label %for.cond.cleanup126
+
+for.cond.cleanup126:                              ; preds = %for.cond120
+  unreachable
+
+for.body127:                                      ; preds = %for.cond120
+  %0 = load i16**, i16*** undef, align 1
+  br label %for.cond120
+}
+
+declare void @foo() readnone
+
+
diff --git a/llvm/test/Analysis/MemorySSA/loop-rotate-simplified-clone.ll b/llvm/test/Analysis/MemorySSA/loop-rotate-simplified-clone.ll
new file mode 100644
index 000000000..3d0efc6f6
--- /dev/null
+++ b/llvm/test/Analysis/MemorySSA/loop-rotate-simplified-clone.ll
@@ -0,0 +1,29 @@
+; RUN: opt -verify-memoryssa -enable-mssa-loop-dependency -loop-rotate %s -S | FileCheck %s 
+; REQUIRES: asserts
+
+; CHECK-LABEL: @test()
+define dso_local void @test() {
+entry:
+  br label %preheader
+
+preheader:
+  br label %l39
+
+l39:
+  %v40 = phi float (float)* [ @foo, %preheader ], [ %v43, %crit_edge ]
+  %v41 = call float %v40(float undef)
+  %v42 = load i32, i32* undef, align 8
+  br i1 undef, label %crit_edge, label %loopexit
+
+crit_edge:
+  %v43 = load float (float)*, float (float)** undef, align 8
+  br label %l39
+
+loopexit:
+  unreachable
+}
+
+; Function Attrs: readnone
+declare dso_local float @foo(float) #0 align 32
+
+attributes #0 = { readnone }
diff --git a/llvm/test/Analysis/MemorySSA/loop-unswitch.ll b/llvm/test/Analysis/MemorySSA/loop-unswitch.ll
new file mode 100644
index 000000000..f42a9ca15
--- /dev/null
+++ b/llvm/test/Analysis/MemorySSA/loop-unswitch.ll
@@ -0,0 +1,26 @@
+; RUN: opt -S -loop-unswitch -disable-basicaa -enable-mssa-loop-dependency -verify-memoryssa < %s | FileCheck %s
+; REQUIRES: asserts
+
+target triple = "x86_64-unknown-linux-gnu"
+
+; CHECK-LABEL:  @foo()
+; Function Attrs: readnone speculatable
+declare i32 @foo() #0
+
+define void @main() {
+entry:
+  br label %for.cond2682
+
+for.cond2682:                                     ; preds = %if.then2712, %entry
+  %mul2708 = call i32 @foo()
+  %tobool2709 = icmp ne i32 %mul2708, 0
+  br i1 %tobool2709, label %if.then2712, label %lor.lhs.false2710
+
+lor.lhs.false2710:                                ; preds = %for.cond2682
+  unreachable
+
+if.then2712:                                      ; preds = %for.cond2682
+  br label %for.cond2682
+}
+
+attributes #0 = { readnone speculatable }
diff --git a/llvm/test/Analysis/MemorySSA/pr28880.ll b/llvm/test/Analysis/MemorySSA/pr28880.ll
index ae64c0c5d..80c0440f8 100644
--- a/llvm/test/Analysis/MemorySSA/pr28880.ll
+++ b/llvm/test/Analysis/MemorySSA/pr28880.ll
@@ -47,5 +47,5 @@ bb6:                                              ; preds = %bb3
   unreachable
 }
 
-attributes #0 = { nounwind ssp uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="core2" "target-features"="+cx16,+fxsr,+mmx,+sse,+sse2,+sse3,+ssse3" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind ssp uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="core2" "target-features"="+cx16,+fxsr,+mmx,+sse,+sse2,+sse3,+ssse3" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
diff --git a/llvm/test/Analysis/MemorySSA/pr39197.ll b/llvm/test/Analysis/MemorySSA/pr39197.ll
index e384c060c..16a321a81 100644
--- a/llvm/test/Analysis/MemorySSA/pr39197.ll
+++ b/llvm/test/Analysis/MemorySSA/pr39197.ll
@@ -133,7 +133,7 @@ define dso_local void @safe_add_func_uint8_t_u_u() #0 {
   ret void
 }
 
-attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="z13" "target-features"="+transactional-execution,+vector" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="z13" "target-features"="+transactional-execution,+vector" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { argmemonly nounwind }
 attributes #2 = { nounwind }
 
diff --git a/llvm/test/Analysis/MemorySSA/pr40038.ll b/llvm/test/Analysis/MemorySSA/pr40038.ll
index b46b52e27..844ede33c 100644
--- a/llvm/test/Analysis/MemorySSA/pr40038.ll
+++ b/llvm/test/Analysis/MemorySSA/pr40038.ll
@@ -70,7 +70,7 @@ bb18:                                             ; preds = %bb12, %bb1
 ; Function Attrs: cold noreturn nounwind
 declare void @llvm.trap() #1
 
-attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="z13" "target-features"="+transactional-execution,+vector" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="z13" "target-features"="+transactional-execution,+vector" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { cold noreturn nounwind }
 
 !llvm.ident = !{!0}
diff --git a/llvm/test/Analysis/MemorySSA/pr40754.ll b/llvm/test/Analysis/MemorySSA/pr40754.ll
index 8db320d80..3262a0cdd 100644
--- a/llvm/test/Analysis/MemorySSA/pr40754.ll
+++ b/llvm/test/Analysis/MemorySSA/pr40754.ll
@@ -11,44 +11,45 @@ target triple = "systemz-unknown"
 ; Function Attrs: norecurse noreturn nounwind
 define dso_local void @func_65() local_unnamed_addr {
 ; CHECK-LABEL: @func_65()
-  br label %1
+label0:
+  br label %label1
 
-; <label>:1:                                      ; preds = %.thread, %0
-  br label %2
+label1:                                      ; preds = %.thread, %label0
+  br label %label2
 
-; <label>:2:                                      ; preds = %.critedge, %1
-  br label %3
+label2:                                      ; preds = %.critedge, %label1
+  br label %label3
 
-; <label>:3:                                      ; preds = %5, %2
-  %storemerge = phi i32 [ 0, %2 ], [ %6, %5 ]
+label3:                                      ; preds = %label5, %label2
+  %storemerge = phi i32 [ 0, %label2 ], [ %tmp6, %label5 ]
   store i32 %storemerge, i32* @g_185, align 4
-  %4 = icmp ult i32 %storemerge, 2
-  br i1 %4, label %5, label %.thread.loopexit
-
-; <label>:5:                                      ; preds = %3
-  %6 = add i32 %storemerge, 1
-  %7 = zext i32 %6 to i64
-  %8 = getelementptr [8 x [4 x [6 x i32]]], [8 x [4 x [6 x i32]]]* @g_120, i64 0, i64 undef, i64 %7, i64 undef
-  %9 = load i32, i32* %8, align 4
-  %10 = icmp eq i32 %9, 0
-  br i1 %10, label %3, label %11
-
-; <label>:11:                                     ; preds = %5
-  %storemerge.lcssa4 = phi i32 [ %storemerge, %5 ]
-  %12 = icmp eq i32 %storemerge.lcssa4, 0
-  br i1 %12, label %.critedge, label %.thread.loopexit3
-
-.critedge:                                        ; preds = %11
+  %tmp4 = icmp ult i32 %storemerge, 2
+  br i1 %tmp4, label %label5, label %.thread.loopexit
+
+label5:                                      ; preds = %label3
+  %tmp6 = add i32 %storemerge, 1
+  %tmp7 = zext i32 %tmp6 to i64
+  %tmp8 = getelementptr [8 x [4 x [6 x i32]]], [8 x [4 x [6 x i32]]]* @g_120, i64 0, i64 undef, i64 %tmp7, i64 undef
+  %tmp9 = load i32, i32* %tmp8, align 4
+  %tmp10 = icmp eq i32 %tmp9, 0
+  br i1 %tmp10, label %label3, label %label11
+
+label11:                                     ; preds = %label5
+  %storemerge.lcssa4 = phi i32 [ %storemerge, %label5 ]
+  %tmp12 = icmp eq i32 %storemerge.lcssa4, 0
+  br i1 %tmp12, label %.critedge, label %.thread.loopexit3
+
+.critedge:                                        ; preds = %label11
   store i16 0, i16* @g_329, align 2
-  br label %2
+  br label %label2
 
-.thread.loopexit:                                 ; preds = %3
+.thread.loopexit:                                 ; preds = %label3
   br label %.thread
 
-.thread.loopexit3:                                ; preds = %11
+.thread.loopexit3:                                ; preds = %label11
   br label %.thread
 
 .thread:                                          ; preds = %.thread.loopexit3, %.thread.loopexit
-  br label %1
+  br label %label1
 }
 
diff --git a/llvm/test/Analysis/MemorySSA/pr41640.ll b/llvm/test/Analysis/MemorySSA/pr41640.ll
index 5a7654424..5d2d52320 100644
--- a/llvm/test/Analysis/MemorySSA/pr41640.ll
+++ b/llvm/test/Analysis/MemorySSA/pr41640.ll
@@ -1,5 +1,4 @@
-; RUN: opt -S -licm -enable-mssa-loop-dependency %s | FileCheck %s
-; REQUIRES: asserts
+; RUN: opt -disable-output -licm -print-memoryssa -enable-mssa-loop-dependency=true < %s 2>&1 | FileCheck %s
 target datalayout = "E-m:e-i1:8:16-i8:8:16-i64:64-f128:64-v128:64-a:8:16-n32:64"
 target triple = "s390x-ibm-linux"
 
@@ -8,6 +7,8 @@ target triple = "s390x-ibm-linux"
 @g_1087 = external dso_local global i32**, align 8
 
 ; CHECK-LABEL: @f1()
+; CHECK: 5 = MemoryPhi(
+; CHECK-NOT: 7 = MemoryPhi(
 define dso_local fastcc void @f1() unnamed_addr #0 {
 label0:
   br i1 undef, label %thread-pre-split.i.preheader, label %label5
diff --git a/llvm/test/Analysis/MemorySSA/pr42940.ll b/llvm/test/Analysis/MemorySSA/pr42940.ll
new file mode 100644
index 000000000..ccd3007c4
--- /dev/null
+++ b/llvm/test/Analysis/MemorySSA/pr42940.ll
@@ -0,0 +1,189 @@
+; RUN: opt -licm -enable-mssa-loop-dependency -verify-memoryssa  -S %s | FileCheck %s
+; REQUIRES: asserts
+
+target datalayout = "E-m:e-i1:8:16-i8:8:16-i64:64-f128:64-v128:64-a:8:16-n32:64"
+target triple = "s390x-ibm-linux"
+
+@g_77 = external dso_local global i16, align 2
+
+; CHECK-LABEL: @f1()
+define void @f1() {
+entry:
+  store i16 undef, i16* @g_77, align 2
+  br label %loop_pre
+
+unreachablelabel: ; No predecessors
+  br label %loop_pre
+
+loop_pre:
+  br label %for.cond.header
+
+for.cond.header:
+  store i32 0, i32* undef, align 4
+  br i1 undef, label %for.body, label %for.end
+
+for.body:
+  %tmp1 = load volatile i16, i16* undef, align 2
+  br label %for.end
+
+for.end:
+  br i1 undef, label %func.exit, label %for.cond.header
+
+func.exit:
+  ret void
+}
+
+@g_159 = external dso_local global i32, align 4
+
+; CHECK-LABEL: @f2()
+define void @f2() {
+entry:
+  br label %for.header.first
+
+for.header.first:
+  br label %for.body.first
+
+for.body.first:
+  store i32 0, i32* @g_159, align 4
+  br i1 undef, label %for.body.first, label %for.end.first
+
+for.end.first:
+  br i1 undef, label %lor.end, label %for.header.first
+
+lor.end:
+  br label %for.pre
+
+unreachablelabel: ; No predecessors
+  br label %for.pre
+
+for.pre:
+  br label %for.header.second
+
+for.header.second:
+  store i32 undef, i32* undef, align 4
+  br label %for.header.second
+}
+
+@g_271 = external dso_local global i8, align 2
+@g_427 = external dso_local unnamed_addr global [9 x i16], align 2
+
+; CHECK-LABEL: @f3()
+define  void @f3() {
+entry:
+  br label %for.preheader
+
+for.preheader:
+  store volatile i8 undef, i8* @g_271, align 2
+  br i1 undef, label %for.preheader, label %for.end
+
+for.end:
+  br label %lbl_1058.i
+
+unreachablelabel: ; No predecessors
+  br label %lbl_1058.i
+
+lbl_1058.i:
+  br label %for.cond3.preheader.i
+
+for.cond3.preheader.i:
+  %tmp1 = load i16, i16* getelementptr inbounds ([9 x i16], [9 x i16]* @g_427, i64 0, i64 2), align 2
+  %conv620.i129 = zext i16 %tmp1 to i32
+  %cmp621.i130 = icmp ugt i32 undef, %conv620.i129
+  %conv622.i131 = zext i1 %cmp621.i130 to i32
+  store i32 %conv622.i131, i32* undef, align 4
+  br i1 undef, label %func.exit, label %for.cond3.preheader.i
+
+func.exit:
+  ret void
+}
+
+@g_6 = external dso_local unnamed_addr global [3 x i32], align 4
+@g_244 = external dso_local global i64, align 8
+@g_1164 = external dso_local global i64, align 8
+
+; CHECK-LABEL: @f4()
+define void @f4() {
+entry:
+  br label %for.cond8.preheader
+
+for.cond8.preheader:
+  store i32 0, i32* getelementptr inbounds ([3 x i32], [3 x i32]* @g_6, i64 0, i64 2), align 4
+  br i1 undef, label %if.end, label %for.cond8.preheader
+
+if.end:
+  br i1 undef, label %cleanup1270, label %for.cond504.preheader
+
+for.cond504.preheader:
+  store i64 undef, i64* @g_244, align 8
+  br label %cleanup1270
+
+for.cond559.preheader:
+  store i64 undef, i64* @g_1164, align 8
+  br i1 undef, label %for.cond559.preheader, label %cleanup1270
+
+cleanup1270:
+  ret void
+}
+
+@g_1504 = external dso_local local_unnamed_addr global i16****, align 8
+
+define void @f5() {
+bb:
+  tail call fastcc void @f21()
+  br label %bb12.outer
+
+bb12.outer.loopexit:                              ; No predecessors!
+  br label %bb12.outer
+
+bb12.outer:                                       ; preds = %bb12.outer.loopexit, %bb
+  br i1 undef, label %bb12.outer.split.us, label %bb12.preheader
+
+bb12.preheader:                                   ; preds = %bb12.outer
+  br label %bb12
+
+bb12.outer.split.us:                              ; preds = %bb12.outer
+  br label %bb16.us.us
+
+bb16.us.us:                                       ; preds = %bb16.us.us, %bb12.outer.split.us
+  br label %bb16.us.us
+
+bb12:                                             ; preds = %bb77.1, %bb12.preheader
+  br i1 undef, label %bb25.preheader, label %bb77
+
+bb25.preheader:                                   ; preds = %bb12.1, %bb12
+  br label %bb25
+
+bb25:                                             ; preds = %l0, %bb25.preheader
+  br i1 undef, label %bb62, label %bb71.thread
+
+bb62:                                             ; preds = %bb25
+  br i1 undef, label %bb92.loopexit, label %l0
+
+l0:                                                ; preds = %bb62
+  br label %bb25
+
+bb71.thread:                                      ; preds = %bb25
+  br label %bb92
+
+bb77:                                             ; preds = %bb12
+  %tmp78 = load i16****, i16***** @g_1504, align 8
+  %tmp79 = load volatile i16***, i16**** %tmp78, align 8
+  br i1 undef, label %bb91, label %bb12.1
+
+bb91:                                             ; preds = %bb77.1, %bb77
+  unreachable
+
+bb92.loopexit:                                    ; preds = %bb62
+  br label %bb92
+
+bb92:                                             ; preds = %bb92.loopexit, %bb71.thread
+  ret void
+
+bb12.1:                                           ; preds = %bb77
+  br i1 undef, label %bb25.preheader, label %bb77.1
+
+bb77.1:                                           ; preds = %bb12.1
+  br i1 undef, label %bb91, label %bb12
+}
+
+declare void @f21()
diff --git a/llvm/test/Analysis/MemorySSA/pr43044.ll b/llvm/test/Analysis/MemorySSA/pr43044.ll
new file mode 100644
index 000000000..a0b2cf7b9
--- /dev/null
+++ b/llvm/test/Analysis/MemorySSA/pr43044.ll
@@ -0,0 +1,52 @@
+; RUN: opt -loop-rotate -licm -enable-mssa-loop-dependency -verify-memoryssa %s -S | FileCheck %s
+; REQUIRES: asserts
+
+target datalayout = "E-m:e-i1:8:16-i8:8:16-i64:64-f128:64-v128:64-a:8:16-n32:64"
+target triple = "s390x-ibm-linux"
+
+declare void @llvm.lifetime.end.p0i8(i64 immarg, i8* nocapture)
+
+; CHECK-LABEL: @func_42()
+define void @func_42() {
+entry:
+  br label %for.cond1050
+
+for.cond1050.loopexit:                            ; preds = %for.cond1373
+  br label %for.cond1050
+
+for.cond1050:                                     ; preds = %for.cond1050.loopexit, %entry
+  %storemerge6 = phi i32 [ 2, %entry ], [ 0, %for.cond1050.loopexit ]
+  %cmp1051 = icmp sgt i32 %storemerge6, -1
+  br i1 %cmp1051, label %for.cond1055.preheader, label %cleanup1400.loopexit1
+
+for.cond1055.preheader:                           ; preds = %for.cond1050
+  store i64 0, i64* null, align 8
+  %0 = load i64, i64* null, align 8
+  %tobool1383 = icmp eq i64 %0, 0
+  br i1 %tobool1383, label %for.cond1055.preheader.cleanup1400.loopexit.split_crit_edge, label %for.cond1055.preheader.for.cond1055.preheader.split_crit_edge
+
+for.cond1055.preheader.for.cond1055.preheader.split_crit_edge: ; preds = %for.cond1055.preheader
+  br label %for.body1376
+
+for.cond1055.preheader.cleanup1400.loopexit.split_crit_edge: ; preds = %for.cond1055.preheader
+  br label %cleanup1400.loopexit.split
+
+for.cond1373:                                     ; preds = %for.body1376
+  br i1 true, label %for.body1376, label %for.cond1050.loopexit
+
+for.body1376:                                     ; preds = %for.cond1373, %for.cond1055.preheader.for.cond1055.preheader.split_crit_edge
+  br i1 false, label %cleanup1400.loopexit, label %for.cond1373
+
+cleanup1400.loopexit:                             ; preds = %for.body1376
+  br label %cleanup1400.loopexit.split
+
+cleanup1400.loopexit.split:                       ; preds = %cleanup1400.loopexit, %for.cond1055.preheader.cleanup1400.loopexit.split_crit_edge
+  br label %cleanup1400
+
+cleanup1400.loopexit1:                            ; preds = %for.cond1050
+  br label %cleanup1400
+
+cleanup1400:                                      ; preds = %cleanup1400.loopexit1, %cleanup1400.loopexit.split
+  call void @llvm.lifetime.end.p0i8(i64 4, i8* nonnull undef)
+  unreachable
+}
diff --git a/llvm/test/Analysis/MemorySSA/pr43317.ll b/llvm/test/Analysis/MemorySSA/pr43317.ll
new file mode 100644
index 000000000..b1fbfdbaa
--- /dev/null
+++ b/llvm/test/Analysis/MemorySSA/pr43317.ll
@@ -0,0 +1,35 @@
+; RUN: opt -disable-output -licm -print-memoryssa -enable-mssa-loop-dependency=true < %s 2>&1 | FileCheck %s
+@v_274 = external dso_local global i64, align 1
+@v_295 = external dso_local global i16, align 1
+@v_335 = external dso_local global i32, align 1
+
+; CHECK-LABEL: @main()
+; CHECK-NOT: 5 = MemoryPhi(
+; CHECK-NOT: 6 = MemoryPhi(
+; CHECK: 4 = MemoryPhi(
+; CHECK-NOT: 7 = MemoryPhi(
+define dso_local void @main() {
+entry:
+  store i32 undef, i32* @v_335, align 1
+  br i1 undef, label %gate, label %exit
+
+nopredentry1:                                     ; No predecessors!
+  br label %preinfiniteloop
+
+nopredentry2:                                     ; No predecessors!
+  br label %gate
+
+gate:                                             ; preds = %nopredentry2, %entry
+  br i1 undef, label %preinfiniteloop, label %exit
+
+preinfiniteloop:                                  ; preds = %gate, %nopredentry1
+  br label %infiniteloop
+
+infiniteloop:                                     ; preds = %infiniteloop, %preinfiniteloop
+  store i16 undef, i16* @v_295, align 1
+  br label %infiniteloop
+
+exit:                                             ; preds = %gate, %entry
+  store i64 undef, i64* @v_274, align 1
+  ret void
+}
diff --git a/llvm/test/Analysis/MemorySSA/pr43320.ll b/llvm/test/Analysis/MemorySSA/pr43320.ll
new file mode 100644
index 000000000..6aca3f9ee
--- /dev/null
+++ b/llvm/test/Analysis/MemorySSA/pr43320.ll
@@ -0,0 +1,33 @@
+; RUN: opt -licm -enable-mssa-loop-dependency -verify-memoryssa -S < %s | FileCheck %s
+; REQUIRES: asserts
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64-unknown-none-eabi"
+
+; CHECK-LABEL: @e()
+define void @e() {
+entry:
+  br label %g
+
+g:                                                ; preds = %cleanup, %entry
+  %0 = load i32, i32* null, align 4
+  %and = and i32 %0, undef
+  store i32 %and, i32* null, align 4
+  br i1 undef, label %if.end8, label %if.then
+
+if.then:                                          ; preds = %g
+  br i1 undef, label %k, label %cleanup
+
+k:                                                ; preds = %if.end8, %if.then
+  br i1 undef, label %if.end8, label %cleanup
+
+if.end8:                                          ; preds = %k, %g
+  br i1 undef, label %for.cond.preheader, label %k
+
+for.cond.preheader:                               ; preds = %if.end8
+  unreachable
+
+cleanup:                                          ; preds = %k, %if.then
+  br label %g
+}
+
diff --git a/llvm/test/Analysis/MemorySSA/pr43426.ll b/llvm/test/Analysis/MemorySSA/pr43426.ll
new file mode 100644
index 000000000..f603c32cf
--- /dev/null
+++ b/llvm/test/Analysis/MemorySSA/pr43426.ll
@@ -0,0 +1,40 @@
+; RUN: opt -licm -enable-mssa-loop-dependency -S %s | FileCheck %s
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; CHECK-LABEL: @d()
+define dso_local void @d() {
+entry:
+  br label %header
+
+header:
+  store i32 1, i32* null, align 4
+  br i1 true, label %cleanup53, label %body
+
+body:
+  br i1 undef, label %cleanup31, label %for.cond11
+
+for.cond11: ; Needs branch as is
+  br i1 undef, label %unreachable, label %latch
+
+cleanup31:
+  br label %unreachable
+
+deadblock:
+  br i1 undef, label %unreachable, label %deadblock
+
+cleanup53:
+  %val = load i32, i32* null, align 4
+  %cmpv = icmp eq i32 %val, 0
+  br i1 %cmpv, label %cleanup63, label %latch
+
+latch:
+  br label %header
+
+cleanup63:
+  ret void
+
+unreachable:
+  unreachable
+}
+
diff --git a/llvm/test/Analysis/MemorySSA/pr43427.ll b/llvm/test/Analysis/MemorySSA/pr43427.ll
new file mode 100644
index 000000000..f70887822
--- /dev/null
+++ b/llvm/test/Analysis/MemorySSA/pr43427.ll
@@ -0,0 +1,42 @@
+; RUN: opt -disable-output -licm -print-memoryssa -enable-mssa-loop-dependency=true < %s 2>&1 | FileCheck %s
+
+; CHECK-LABEL: @f()
+; CHECK: 8 = MemoryPhi(
+; CHECK: 7 = MemoryPhi(
+; CHECK: 9 = MemoryPhi(
+define void @f() {
+entry:
+  %e = alloca i16, align 1
+  br label %lbl1
+
+lbl1:                                             ; preds = %if.else, %cleanup, %entry
+  store i16 undef, i16* %e, align 1
+  call void @g()
+  br i1 undef, label %for.end, label %if.else
+
+for.end:                                          ; preds = %lbl1
+  br i1 undef, label %lbl3, label %lbl2
+
+lbl2:                                             ; preds = %lbl3, %for.end
+  br label %lbl3
+
+lbl3:                                             ; preds = %lbl2, %for.end
+  br i1 undef, label %lbl2, label %cleanup
+
+cleanup:                                          ; preds = %lbl3
+  %cleanup.dest = load i32, i32* undef, align 1
+  %switch = icmp ult i32 %cleanup.dest, 1
+  br i1 %switch, label %cleanup.cont, label %lbl1
+
+cleanup.cont:                                     ; preds = %cleanup
+  call void @llvm.lifetime.end.p0i8(i64 1, i8* null)
+  ret void
+
+if.else:                                          ; preds = %lbl1
+  br label %lbl1
+}
+
+declare void @g()
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.lifetime.end.p0i8(i64 immarg, i8* nocapture)
diff --git a/llvm/test/Analysis/MemorySSA/pr43438.ll b/llvm/test/Analysis/MemorySSA/pr43438.ll
new file mode 100644
index 000000000..69bbcedde
--- /dev/null
+++ b/llvm/test/Analysis/MemorySSA/pr43438.ll
@@ -0,0 +1,100 @@
+; RUN: opt -disable-output -licm -print-memoryssa -enable-mssa-loop-dependency=true < %s 2>&1 | FileCheck %s
+target triple = "x86_64-unknown-linux-gnu"
+
+; CHECK-LABEL: @main()
+; CHECK: 5 = MemoryPhi(
+; CHECK-NOT: 7 = MemoryPhi(
+@v_67 = external dso_local global i32, align 1
+@v_76 = external dso_local global i16, align 1
+@v_86 = external dso_local global i16 *, align 1
+
+define dso_local void @main() {
+entry:
+  %v_59 = alloca i16, align 2
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.body, %entry
+  br i1 undef, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  store i16 undef, i16* %v_59, align 2
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  br i1 undef, label %if.else568, label %cond.end82
+
+cond.false69:                                     ; No predecessors!
+  br label %cond.end82
+
+cond.end82:                                       ; preds = %cond.false69, %cond.true55
+  br i1 undef, label %if.else568, label %land.lhs.true87
+
+land.lhs.true87:                                  ; preds = %cond.end82
+  br i1 undef, label %if.then88, label %if.else568
+
+if.then88:                                        ; preds = %land.lhs.true87
+  store i16 * @v_76, i16 ** @v_86, align 1
+  br label %if.end569
+
+if.else568:                                       ; preds = %land.lhs.true87, %cond.end82, %for.end
+  store volatile i32 undef, i32 * @v_67, align 1
+  br label %if.end569
+
+if.end569:                                        ; preds = %if.else568, %if.then88
+  ret void
+}
+
+
+; CHECK-LABEL: @f()
+; CHECK: 8 = MemoryPhi(
+; CHECK: 7 = MemoryPhi(
+; CHECK: 11 = MemoryPhi(
+; CHECK: 10 = MemoryPhi(
+; CHECK: 9 = MemoryPhi(
+define void @f() {
+entry:
+  %e = alloca i16, align 1
+  br label %lbl1
+
+lbl1:                                             ; preds = %if.else, %for.end5, %entry
+  store i16 undef, i16* %e, align 1
+  %0 = load i16, i16* %e, align 1
+  %call = call i16 @g(i16 %0)
+  br i1 undef, label %for.end, label %if.else
+
+for.end:                                          ; preds = %if.then
+  br i1 true, label %for.cond2, label %lbl2
+
+lbl2:                                             ; preds = %for.body4, %if.end
+  br label %for.cond2
+
+for.cond2:                                        ; preds = %lbl3
+  br i1 undef, label %for.body4, label %for.end5
+
+for.body4:                                        ; preds = %for.cond2
+  br label %lbl2
+
+for.end5:                                         ; preds = %for.cond2
+  switch i32 undef, label %unreachable [
+    i32 0, label %if.end12
+    i32 2, label %lbl1
+  ]
+
+if.else:                                          ; preds = %lbl1
+  switch i32 undef, label %unreachable [
+    i32 0, label %if.end12
+    i32 2, label %lbl1
+  ]
+
+if.end12:                                         ; preds = %cleanup.cont11s, %cleanup.cont
+  call void @llvm.lifetime.end.p0i8(i64 1, i8* undef)
+  ret void
+
+unreachable:                                      ; preds = %if.else, %for.end5
+  unreachable
+}
+
+declare i16 @g(i16)
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.lifetime.end.p0i8(i64 immarg, i8* nocapture)
diff --git a/llvm/test/Analysis/MemorySSA/pr43493.ll b/llvm/test/Analysis/MemorySSA/pr43493.ll
new file mode 100644
index 000000000..69506e8c0
--- /dev/null
+++ b/llvm/test/Analysis/MemorySSA/pr43493.ll
@@ -0,0 +1,27 @@
+; RUN: opt -enable-mssa-loop-dependency=true -loop-rotate -verify-memoryssa -S %s | FileCheck %s
+; REQUIRES: asserts
+
+; CHECK-LABEL: @func_35()
+define void @func_35() {
+entry:
+  br i1 undef, label %for.cond1704.preheader, label %return
+
+for.cond1704.preheader:                           ; preds = %entry
+  br label %for.cond1704
+
+for.cond1704:                                     ; preds = %for.cond1704.preheader, %for.body1707
+  br i1 false, label %for.body1707, label %return.loopexit
+
+for.body1707:                                     ; preds = %for.cond1704
+  store i32 1712, i32* undef, align 1
+  br label %for.cond1704
+
+for.body1102:                                     ; preds = %for.body1102
+  br i1 undef, label %for.body1102, label %return
+
+return.loopexit:                                  ; preds = %for.cond1704
+  br label %return
+
+return:                                           ; preds = %return.loopexit, %for.body1102, %entry
+  ret void
+}
diff --git a/llvm/test/Analysis/MemorySSA/pr43540.ll b/llvm/test/Analysis/MemorySSA/pr43540.ll
new file mode 100644
index 000000000..325e6bc0a
--- /dev/null
+++ b/llvm/test/Analysis/MemorySSA/pr43540.ll
@@ -0,0 +1,34 @@
+; RUN: opt -S -licm -enable-mssa-loop-dependency=true %s | FileCheck %s
+@v_1 = global i8 0, align 1
+@v_2 =  global i8 0, align 1
+
+; CHECK-LABEL: @foo()
+; CHECK: for.cond:
+; CHECK-NOT: store
+; CHECK: for.body:
+; CHECK: call void @llvm.memcpy.p0i8.p0i8.i64
+; CHECK: store
+define void @foo() {
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.body, %entry
+  %0 = phi i16 [ %inc, %for.body ], [ 0, %entry ]
+  %cmp = icmp slt i16 %0, 1
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* @v_1, i8 * @v_2, i64 1, i1 false)
+  store i8 1, i8 * @v_2, align 1
+  %inc = add nsw i16 %0, 1
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  ret void
+}
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* noalias nocapture writeonly, i8 * noalias nocapture readonly, i64, i1 immarg) #2
+
+attributes #2 = { argmemonly nounwind willreturn }
+
diff --git a/llvm/test/Analysis/MemorySSA/pr43541.ll b/llvm/test/Analysis/MemorySSA/pr43541.ll
new file mode 100644
index 000000000..3f6b2e26b
--- /dev/null
+++ b/llvm/test/Analysis/MemorySSA/pr43541.ll
@@ -0,0 +1,50 @@
+; RUN: opt -gvn-hoist -enable-mssa-loop-dependency -S < %s | FileCheck %s
+; REQUIRES: asserts
+%struct.job_pool.6.7 = type { i32 }
+
+; CHECK-LABEL: @f()
+define dso_local void @f() {
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.end, %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.cond
+  br label %if.end
+
+if.then:                                          ; No predecessors!
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %for.body
+  br i1 false, label %for.body12.lr.ph, label %for.end
+
+for.body12.lr.ph:                                 ; preds = %if.end
+  br label %for.body12
+
+for.body12:                                       ; preds = %if.end40, %for.body12.lr.ph
+  br label %if.then23
+
+if.then23:                                        ; preds = %for.body12
+  br i1 undef, label %if.then24, label %if.else
+
+if.then24:                                        ; preds = %if.then23
+  %0 = load %struct.job_pool.6.7*, %struct.job_pool.6.7** undef, align 8
+  br label %if.end40
+
+if.else:                                          ; preds = %if.then23
+  %1 = load %struct.job_pool.6.7*, %struct.job_pool.6.7** undef, align 8
+  br label %if.end40
+
+if.end40:                                         ; preds = %if.else, %if.then24
+  br i1 false, label %for.body12, label %for.cond9.for.end_crit_edge
+
+for.cond9.for.end_crit_edge:                      ; preds = %if.end40
+  br label %for.end
+
+for.end:                                          ; preds = %for.cond9.for.end_crit_edge, %if.end
+  br i1 true, label %if.then45, label %for.cond
+
+if.then45:                                        ; preds = %for.end
+  ret void
+}
diff --git a/llvm/test/Analysis/MemorySSA/pr43569.ll b/llvm/test/Analysis/MemorySSA/pr43569.ll
new file mode 100644
index 000000000..c9c68451e
--- /dev/null
+++ b/llvm/test/Analysis/MemorySSA/pr43569.ll
@@ -0,0 +1,49 @@
+; RUN: opt -pgo-kind=pgo-instr-gen-pipeline -aa-pipeline=default -passes="default<O3>" -enable-nontrivial-unswitch -S < %s | FileCheck %s
+; REQUIRES: asserts
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@__profn_c = private constant [1 x i8] c"c"
+@b = common dso_local global i32 0, align 4
+@a = common dso_local global i16 0, align 2
+
+; CHECK-LABEL: @c()
+; Function Attrs: nounwind uwtable
+define dso_local void @c() #0 {
+entry:
+  call void @llvm.instrprof.increment(i8* getelementptr inbounds ([1 x i8], [1 x i8]* @__profn_c, i32 0, i32 0), i64 68269137, i32 3, i32 0)
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.end, %entry
+  call void @llvm.instrprof.increment(i8* getelementptr inbounds ([1 x i8], [1 x i8]* @__profn_c, i32 0, i32 0), i64 68269137, i32 3, i32 1)
+  store i32 0, i32* @b, align 4
+  br label %for.cond1
+
+for.cond1:                                        ; preds = %for.inc, %for.cond
+  %0 = load i32, i32* @b, align 4
+  %1 = load i16, i16* @a, align 2
+  %conv = sext i16 %1 to i32
+  %cmp = icmp slt i32 %0, %conv
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond1
+  call void @llvm.instrprof.increment(i8* getelementptr inbounds ([1 x i8], [1 x i8]* @__profn_c, i32 0, i32 0), i64 68269137, i32 3, i32 2)
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body
+  %2 = load i32, i32* @b, align 4
+  %inc = add nsw i32 %2, 1
+  store i32 %inc, i32* @b, align 4
+  br label %for.cond1
+
+for.end:                                          ; preds = %for.cond1
+  br label %for.cond
+}
+
+; Function Attrs: nounwind
+declare void @llvm.instrprof.increment(i8*, i64, i32, i32) #1
+
+attributes #0 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind }
+
diff --git a/llvm/test/Analysis/MemorySSA/pr43641.ll b/llvm/test/Analysis/MemorySSA/pr43641.ll
new file mode 100644
index 000000000..06a6b5255
--- /dev/null
+++ b/llvm/test/Analysis/MemorySSA/pr43641.ll
@@ -0,0 +1,22 @@
+; RUN: opt -simple-loop-unswitch -enable-nontrivial-unswitch -enable-mssa-loop-dependency -verify-memoryssa -S < %s | FileCheck %s
+; REQUIRES: asserts
+
+; CHECK-LABEL: @c
+define dso_local void @c(i32 signext %d) local_unnamed_addr {
+entry:
+  br i1 undef, label %while.end, label %while.body.lr.ph
+
+while.body.lr.ph:                                 ; preds = %entry
+  %tobool1 = icmp ne i32 %d, 0
+  br label %while.body
+
+while.body:                                       ; preds = %while.body, %while.body.lr.ph
+  %call = tail call signext i32 bitcast (i32 (...)* @e to i32 ()*)()
+  %0 = and i1 %tobool1, undef
+  br i1 %0, label %while.body, label %while.end
+
+while.end:                                        ; preds = %while.body, %entry
+  ret void
+}
+
+declare signext i32 @e(...) local_unnamed_addr
diff --git a/llvm/test/Analysis/MemorySSA/pr44027.ll b/llvm/test/Analysis/MemorySSA/pr44027.ll
new file mode 100644
index 000000000..3c0f9266c
--- /dev/null
+++ b/llvm/test/Analysis/MemorySSA/pr44027.ll
@@ -0,0 +1,27 @@
+; RUN: opt -gvn-hoist -verify-memoryssa -S < %s | FileCheck %s
+; REQUIRES: asserts
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare i32 @wobble(...)
+
+declare void @spam() align 2
+
+; CHECK-LABEL: @f()
+define void @f() personality i8* bitcast (i32 (...)* @wobble to i8*) {
+bb:
+  %tmp = alloca i32*, align 8
+  invoke void @spam()
+          to label %bb16 unwind label %bb23
+
+bb16:                                             ; preds = %bb
+  %tmp17 = load i32*, i32** %tmp, align 8
+  %tmp18 = load i32*, i32** %tmp, align 8
+  unreachable
+
+bb23:                                             ; preds = %bb
+  %tmp24 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+}
diff --git a/llvm/test/Analysis/MemorySSA/pr44029.ll b/llvm/test/Analysis/MemorySSA/pr44029.ll
new file mode 100644
index 000000000..e3aab4d0c
--- /dev/null
+++ b/llvm/test/Analysis/MemorySSA/pr44029.ll
@@ -0,0 +1,63 @@
+; RUN: opt -loop-simplifycfg -verify-memoryssa -S < %s | FileCheck %s
+; REQUIRES: asserts
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare i32 @eggs(...)
+
+declare void @spam()
+
+; CHECK-LABEL: @f()
+define void @f() personality i8* bitcast (i32 (...)* @eggs to i8*) {
+bb:
+  invoke void @spam()
+          to label %bb2 unwind label %bb4
+
+bb2:                                              ; preds = %bb
+  invoke void @spam()
+          to label %bb8 unwind label %bb5
+
+bb4:                                              ; preds = %bb
+  %tmp = landingpad { i8*, i32 }
+          cleanup
+  resume { i8*, i32 } undef
+
+bb5:                                              ; preds = %bb2
+  %tmp6 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+bb8:                                              ; preds = %bb13, %bb2
+  br label %bb10
+
+bb10:                                             ; preds = %bb8
+  invoke void @spam()
+          to label %bb11 unwind label %bb20
+
+bb11:                                             ; preds = %bb10
+  invoke void @spam()
+          to label %bb12 unwind label %bb22
+
+bb12:                                             ; preds = %bb11
+  invoke void @spam()
+          to label %bb13 unwind label %bb24
+
+bb13:                                             ; preds = %bb12
+  br label %bb8
+
+bb20:                                             ; preds = %bb10
+  %tmp21 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+bb22:                                             ; preds = %bb11
+  %tmp23 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+bb24:                                             ; preds = %bb12
+  %tmp25 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+}
diff --git a/llvm/test/Analysis/MemorySSA/renamephis.ll b/llvm/test/Analysis/MemorySSA/renamephis.ll
new file mode 100644
index 000000000..7d240d823
--- /dev/null
+++ b/llvm/test/Analysis/MemorySSA/renamephis.ll
@@ -0,0 +1,51 @@
+; RUN: opt -licm -enable-mssa-loop-dependency -verify-memoryssa -S %s | FileCheck %s
+; REQUIRES: asserts
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@0 = external global { { [86 x i8]*, i32, i32 }, { i16, i16, [20 x i8] }*, i8, i8 }
+
+declare void @g()
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.lifetime.end.p0i8(i64 immarg, i8* nocapture) #0
+
+; CHECK-LABEL: @f
+define void @f() align 2 {
+entry:
+  %P = alloca i32*, align 8
+  br label %cond.end.i.i.i.i
+
+cond.end.i.i.i.i:                                 ; preds = %cont20, %entry
+  br i1 undef, label %cont20, label %if.end
+
+cont20:                                           ; preds = %cond.end.i.i.i.i, %cond.end.i.i.i.i, %cond.end.i.i.i.i
+  store i32* undef, i32** %P, align 8
+  br label %cond.end.i.i.i.i
+
+if.end:                                           ; preds = %cond.end.i.i.i.i
+  br i1 undef, label %cond.exit, label %handler.type_mismatch2.i
+
+handler.type_mismatch2.i:                         ; preds = %if.end
+  tail call void @g()
+  unreachable
+
+cond.exit:             ; preds = %if.end
+  switch i8 undef, label %block.exit [
+    i8 81, label %sw.bb94
+    i8 12, label %cleanup
+    i8 74, label %cleanup
+  ]
+
+block.exit: ; preds = %cond.exit
+  unreachable
+
+sw.bb94:                                          ; preds = %cond.exit
+  call void @llvm.lifetime.end.p0i8(i64 8, i8* nonnull undef)
+  br label %cleanup
+
+cleanup:                                          ; preds = %sw.bb94, %cond.exit, %cond.exit
+  ret void
+}
+
+attributes #0 = { argmemonly nounwind willreturn }
diff --git a/llvm/test/Analysis/MemorySSA/unreachable.ll b/llvm/test/Analysis/MemorySSA/unreachable.ll
new file mode 100644
index 000000000..6a9364353
--- /dev/null
+++ b/llvm/test/Analysis/MemorySSA/unreachable.ll
@@ -0,0 +1,31 @@
+; RUN: opt -licm -enable-mssa-loop-dependency -verify-memoryssa %s -S | FileCheck %s
+; REQUIRES: asserts
+; Ensure verification doesn't fail with unreachable blocks.
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-grtev4-linux-gnu"
+
+declare dso_local void @f()
+
+; CHECK-LABEL: @foo
+define dso_local void @foo() {
+entry:
+  br i1 undef, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  br label %try.cont
+
+if.end:                                           ; preds = %entry
+; 1 = MemoryDef(liveOnEntry)
+  call void @f()
+  br label %try.cont
+
+catch:                                            ; No predecessors!
+; 2 = MemoryDef(liveOnEntry)
+  call void @f()
+  br label %try.cont
+
+try.cont:                                         ; preds = %if.end, %catch, %if.then
+; 3 = MemoryPhi({if.then,liveOnEntry},{if.end,1},{catch,liveOnEntry})
+  ret void
+}
diff --git a/llvm/test/Analysis/MustExecute/must_be_executed_context.ll b/llvm/test/Analysis/MustExecute/must_be_executed_context.ll
new file mode 100644
index 000000000..fd650872d
--- /dev/null
+++ b/llvm/test/Analysis/MustExecute/must_be_executed_context.ll
@@ -0,0 +1,399 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -print-mustexecute               -analyze 2>&1 | FileCheck %s --check-prefix=ME
+; RUN: opt < %s -print-must-be-executed-contexts -analyze 2>&1 | FileCheck %s --check-prefix=MBEC
+;
+;    void simple_conditional(int c) {
+;      A();
+;      B();
+;      if (c) {
+;        C();
+;        D();
+;      }
+;      E();
+;      F();
+;      G();
+;    }
+;
+; Best result:
+; Start Instruction   | Visit Set
+; A                   | A, B,       E, F
+;    B                | A, B,       E, F
+;       C             | A, B, C, D, E, F
+;          D          | A, B, C, D, E, F
+;             E       | A, B,       E, F
+;                F    | A, B,       E, F
+;                   G | A, B,       E, F, G
+;
+; FIXME: We miss the B -> E and backward exploration.
+;
+; There are no loops so print-mustexec will not do anything.
+; ME-NOT: mustexec
+;
+define void @simple_conditional(i32 %arg) {
+bb:
+  call void @A()
+; MBEC:      -- Explore context of:   call void @A()
+; MBEC-NEXT:   [F: simple_conditional]   call void @A()
+; MBEC-NEXT:   [F: simple_conditional]   call void @B()
+; MBEC-NEXT:   [F: simple_conditional]   %tmp = icmp eq i32 %arg, 0
+; MBEC-NEXT:   [F: simple_conditional]   br i1 %tmp, label %bb2, label %bb1
+; MBEC-NEXT:   [F: simple_conditional]   call void @E()
+; MBEC-NEXT:   [F: simple_conditional]   call void @F()
+; MBEC-NOT:    call
+
+  call void @B()
+; MBEC:      -- Explore context of:   call void @B()
+; MBEC-NEXT:   [F: simple_conditional]   call void @B()
+; MBEC-NEXT:   [F: simple_conditional]   %tmp = icmp eq i32 %arg, 0
+; MBEC-NEXT:   [F: simple_conditional]   br i1 %tmp, label %bb2, label %bb1
+; MBEC-NEXT:   [F: simple_conditional]   call void @E()
+; MBEC-NEXT:   [F: simple_conditional]   call void @F()
+; MBEC-NOT:    call
+; MBEC:      -- Explore context of: %tmp
+
+  %tmp = icmp eq i32 %arg, 0
+  br i1 %tmp, label %bb2, label %bb1
+
+bb1:                                              ; preds = %bb
+  call void @C()
+; MBEC:      -- Explore context of:   call void @C()
+; MBEC-NEXT:   [F: simple_conditional]   call void @C()
+; MBEC-NEXT:   [F: simple_conditional]   call void @D()
+; MBEC-NEXT:   [F: simple_conditional]   br label %bb2
+; MBEC-NEXT:   [F: simple_conditional]   call void @E()
+; MBEC-NEXT:   [F: simple_conditional]   call void @F()
+; MBEC-NOT:    call
+
+  call void @D()
+; MBEC:      -- Explore context of:   call void @D()
+; MBEC-NEXT:   [F: simple_conditional]   call void @D()
+; MBEC-NEXT:   [F: simple_conditional]   br label %bb2
+; MBEC-NEXT:   [F: simple_conditional]   call void @E()
+; MBEC-NEXT:   [F: simple_conditional]   call void @F()
+; MBEC-NOT:    call
+; MBEC:      -- Explore context of: br
+
+  br label %bb2
+
+bb2:                                              ; preds = %bb, %bb1
+  call void @E()
+; MBEC:      -- Explore context of:   call void @E()
+; MBEC-NEXT:   [F: simple_conditional]   call void @E()
+; MBEC-NEXT:   [F: simple_conditional]   call void @F()
+; MBEC-NOT:    call
+
+  call void @F() ; might not return!
+; MBEC:      -- Explore context of:   call void @F()
+; MBEC-NEXT:   [F: simple_conditional]   call void @F()
+; MBEC-NOT:    call
+
+  call void @G()
+; MBEC:      -- Explore context of:   call void @G()
+; MBEC-NEXT:   [F: simple_conditional]   call void @G()
+; MBEC-NEXT:   [F: simple_conditional]   ret void
+; MBEC-NOT:    call
+; MBEC:      -- Explore context of: ret
+
+  ret void
+}
+
+
+;    void complex_loops_and_control(int c, int d) {
+;      A();
+;      while (1) {
+;        B();
+;        if (++c == d)
+;          C();
+;        if (++c == d)
+;          continue;
+;        D();
+;        if (++c == d)
+;          break;
+;        do {
+;          if (++c == d)
+;            continue;
+;          E();
+;        } while (++c == d);
+;        F();
+;      }
+;      G();
+;    }
+;
+; Best result:
+; Start Instruction    | Visit Set
+; A                    | A, B
+;    B                 | A, B
+;       C              | A, B, C
+;          D           | A, B,    D
+;             E        | A, B,    D, E, F
+;                F     | A, B,    D,    F
+;                   G  | A, B,    D,       G
+;
+;
+; ME: define void @complex_loops_and_control
+define void @complex_loops_and_control(i32 %arg, i32 %arg1) {
+bb:
+  call void @A()
+; ME:     call void @A()
+; ME-NOT: mustexec
+; ME-NEXT: br
+; MBEC:      -- Explore context of:   call void @A()
+; MBEC-NEXT:   [F: complex_loops_and_control]   call void @A()
+; MBEC-NEXT:   [F: complex_loops_and_control]   br label %bb2
+; MBEC-NEXT:   [F: complex_loops_and_control]   %.0 = phi i32 [ %arg, %bb ], [ %.0.be, %.backedge ]
+; MBEC-NEXT:   [F: complex_loops_and_control]   call void @B()
+; MBEC-NEXT:   [F: complex_loops_and_control]   %tmp = add nsw i32 %.0, 1
+; MBEC-NEXT:   [F: complex_loops_and_control]   %tmp3 = icmp eq i32 %tmp, %arg1
+; MBEC-NEXT:   [F: complex_loops_and_control]   br i1 %tmp3, label %bb4, label %bb5
+; MBEC-NOT:    call
+; MBEC:      -- Explore context of: br
+  br label %bb2
+
+bb2:                                              ; preds = %.backedge, %bb
+  %.0 = phi i32 [ %arg, %bb ], [ %.0.be, %.backedge ]
+  call void @B()
+; ME: call void @B() ; (mustexec in: bb2)
+; MBEC:      -- Explore context of:   call void @B()
+; MBEC-NEXT:   [F: complex_loops_and_control]   call void @B()
+; MBEC-NEXT:   [F: complex_loops_and_control]   %tmp = add nsw i32 %.0, 1
+; MBEC-NEXT:   [F: complex_loops_and_control]   %tmp3 = icmp eq i32 %tmp, %arg1
+; MBEC-NEXT:   [F: complex_loops_and_control]   br i1 %tmp3, label %bb4, label %bb5
+; MBEC-NOT:    call
+; MBEC:      -- Explore context of: %tmp
+  %tmp = add nsw i32 %.0, 1
+  %tmp3 = icmp eq i32 %tmp, %arg1
+  br i1 %tmp3, label %bb4, label %bb5
+
+bb4:                                              ; preds = %bb2
+  call void @C()
+; ME: call void @C()
+; ME-NOT: mustexec
+; ME-NEXT: br
+; FIXME: Missing A and B (backward)
+; MBEC:      -- Explore context of:   call void @C()
+; MBEC-NEXT:   [F: complex_loops_and_control]   call void @C()
+; MBEC-NEXT:   [F: complex_loops_and_control]   br label %bb5
+; MBEC-NEXT:   [F: complex_loops_and_control]   %tmp6 = add nsw i32 %.0, 2
+; MBEC-NEXT:   [F: complex_loops_and_control]   %tmp7 = icmp eq i32 %tmp6, %arg1
+; MBEC-NEXT:   [F: complex_loops_and_control]   br i1 %tmp7, label %bb8, label %bb9
+; MBEC-NOT:    call
+; MBEC:      -- Explore context of: br
+  br label %bb5
+
+bb5:                                              ; preds = %bb4, %bb2
+  %tmp6 = add nsw i32 %.0, 2
+  %tmp7 = icmp eq i32 %tmp6, %arg1
+  br i1 %tmp7, label %bb8, label %bb9
+
+bb8:                                              ; preds = %bb5
+  br label %.backedge
+
+.backedge:                                        ; preds = %bb8, %bb22
+  %.0.be = phi i32 [ %tmp6, %bb8 ], [ %.lcssa, %bb22 ]
+  br label %bb2
+
+bb9:                                              ; preds = %bb5
+  call void @D()
+; ME: call void @D()
+; ME-NOT: mustexec
+; ME-NEXT: %tmp10
+; FIXME: Missing A and B (backward)
+; MBEC:      -- Explore context of:   call void @D()
+; MBEC-NEXT:   [F: complex_loops_and_control]   call void @D()
+; MBEC-NEXT:   [F: complex_loops_and_control]   %tmp10 = add nsw i32 %.0, 3
+; MBEC-NEXT:   [F: complex_loops_and_control]   %tmp11 = icmp eq i32 %tmp10, %arg1
+; MBEC-NEXT:   [F: complex_loops_and_control]   br i1 %tmp11, label %bb12, label %bb13
+; MBEC-NOT:    call
+; MBEC:      -- Explore context of: %tmp10
+  %tmp10 = add nsw i32 %.0, 3
+  %tmp11 = icmp eq i32 %tmp10, %arg1
+  br i1 %tmp11, label %bb12, label %bb13
+
+bb12:                                             ; preds = %bb9
+  br label %bb23
+
+bb13:                                             ; preds = %bb9
+  br label %bb14
+
+bb14:                                             ; preds = %bb19, %bb13
+  %.1 = phi i32 [ %tmp10, %bb13 ], [ %tmp20, %bb19 ]
+  %tmp15 = add nsw i32 %.1, 1
+  %tmp16 = icmp eq i32 %tmp15, %arg1
+  br i1 %tmp16, label %bb17, label %bb18
+
+bb17:                                             ; preds = %bb14
+  br label %bb19
+
+bb18:                                             ; preds = %bb14
+  call void @E()
+; ME: call void @E()
+; ME-NOT: mustexec
+; ME-NEXT: br
+; FIXME: Missing A, B, and D (backward), as well as F
+; MBEC:      -- Explore context of:   call void @E()
+; MBEC-NEXT:   [F: complex_loops_and_control]   call void @E()
+; MBEC-NEXT:   [F: complex_loops_and_control]   br label %bb19
+; MBEC-NEXT:   [F: complex_loops_and_control]   %tmp20 = add nsw i32 %.1, 2
+; MBEC-NEXT:   [F: complex_loops_and_control]   %tmp21 = icmp eq i32 %tmp20, %arg1
+; MBEC-NEXT:   [F: complex_loops_and_control]   br i1 %tmp21, label %bb14, label %bb22
+; MBEC-NOT:    call
+; MBEC:      -- Explore context of: br
+  br label %bb19
+
+bb19:                                             ; preds = %bb18, %bb17
+  %tmp20 = add nsw i32 %.1, 2
+  %tmp21 = icmp eq i32 %tmp20, %arg1
+  br i1 %tmp21, label %bb14, label %bb22
+
+bb22:                                             ; preds = %bb19
+  %.lcssa = phi i32 [ %tmp20, %bb19 ]
+  call void @F()
+; ME: call void @F()
+; ME-NOT: mustexec
+; ME-NEXT: br
+; FIXME: Missing A, B, and D (backward)
+; MBEC:      -- Explore context of:   call void @F()
+; MBEC-NEXT:   [F: complex_loops_and_control]   call void @F()
+; MBEC-NOT:    call
+; MBEC:      -- Explore context of: br
+  br label %.backedge
+
+bb23:                                             ; preds = %bb12
+  call void @G()
+; ME: call void @G()
+; ME-NOT: mustexec
+; ME-NEXT: ret
+; FIXME: Missing A, B, and D (backward)
+; MBEC:      -- Explore context of:   call void @G()
+; MBEC-NEXT:   [F: complex_loops_and_control]   call void @G()
+; MBEC-NEXT:   [F: complex_loops_and_control]   ret void
+; MBEC-NOT:    call
+; MBEC:      -- Explore context of: ret
+  ret void
+}
+
+declare void @A() nounwind willreturn
+
+declare void @B() nounwind willreturn
+
+declare void @C() nounwind willreturn
+
+declare void @D() nounwind willreturn
+
+declare void @E() nounwind willreturn
+
+declare void @F() nounwind
+
+declare void @G() nounwind willreturn
+
+declare i32 @g(i32*) nounwind willreturn
+
+declare void @h(i32*) nounwind willreturn
+
+define i32 @nonnull_exec_ctx_1(i32* %a, i32 %b) {
+; MBEC:      -- Explore context of:   %tmp3 = icmp eq i32 %b, 0
+; MBEC-NEXT:   [F: nonnull_exec_ctx_1]   %tmp3 = icmp eq i32 %b, 0
+; MBEC-NEXT:   [F: nonnull_exec_ctx_1]   br i1 %tmp3, label %ex, label %hd
+; MBEC-NEXT: -- Explore context of:   br i1 %tmp3, label %ex, label %hd
+; MBEC-NEXT:   [F: nonnull_exec_ctx_1]   br i1 %tmp3, label %ex, label %hd
+; MBEC-NEXT: -- Explore context of:   %tmp5 = tail call i32 @g(i32* nonnull %a)
+; MBEC-NEXT:   [F: nonnull_exec_ctx_1]   %tmp5 = tail call i32 @g(i32* nonnull %a)
+; MBEC-NEXT:   [F: nonnull_exec_ctx_1]   ret i32 %tmp5
+; MBEC-NEXT: -- Explore context of:   ret i32 %tmp5
+; MBEC-NEXT:   [F: nonnull_exec_ctx_1]   ret i32 %tmp5
+; MBEC-NEXT: -- Explore context of:   %tmp7 = phi i32 [ %tmp8, %hd ], [ 0, %en ]
+; MBEC-NEXT:   [F: nonnull_exec_ctx_1]   %tmp7 = phi i32 [ %tmp8, %hd ], [ 0, %en ]
+; MBEC-NEXT:   [F: nonnull_exec_ctx_1]   tail call void @h(i32* %a)
+; MBEC-NEXT:   [F: nonnull_exec_ctx_1]   %tmp8 = add nuw i32 %tmp7, 1
+; MBEC-NEXT:   [F: nonnull_exec_ctx_1]   %tmp9 = icmp eq i32 %tmp8, %b
+; MBEC-NEXT:   [F: nonnull_exec_ctx_1]   br i1 %tmp9, label %ex, label %hd
+; MBEC-NEXT: -- Explore context of:   tail call void @h(i32* %a)
+; MBEC-NEXT:   [F: nonnull_exec_ctx_1]   tail call void @h(i32* %a)
+; MBEC-NEXT:   [F: nonnull_exec_ctx_1]   %tmp8 = add nuw i32 %tmp7, 1
+; MBEC-NEXT:   [F: nonnull_exec_ctx_1]   %tmp9 = icmp eq i32 %tmp8, %b
+; MBEC-NEXT:   [F: nonnull_exec_ctx_1]   br i1 %tmp9, label %ex, label %hd
+; MBEC-NEXT: -- Explore context of:   %tmp8 = add nuw i32 %tmp7, 1
+; MBEC-NEXT:   [F: nonnull_exec_ctx_1]   %tmp8 = add nuw i32 %tmp7, 1
+; MBEC-NEXT:   [F: nonnull_exec_ctx_1]   %tmp9 = icmp eq i32 %tmp8, %b
+; MBEC-NEXT:   [F: nonnull_exec_ctx_1]   br i1 %tmp9, label %ex, label %hd
+; MBEC-NEXT: -- Explore context of:   %tmp9 = icmp eq i32 %tmp8, %b
+; MBEC-NEXT:   [F: nonnull_exec_ctx_1]   %tmp9 = icmp eq i32 %tmp8, %b
+; MBEC-NEXT:   [F: nonnull_exec_ctx_1]   br i1 %tmp9, label %ex, label %hd
+; MBEC-NEXT: -- Explore context of:   br i1 %tmp9, label %ex, label %hd
+; MBEC-NEXT:   [F: nonnull_exec_ctx_1]   br i1 %tmp9, label %ex, label %hd
+en:
+  %tmp3 = icmp eq i32 %b, 0
+  br i1 %tmp3, label %ex, label %hd
+
+ex:
+  %tmp5 = tail call i32 @g(i32* nonnull %a)
+  ret i32 %tmp5
+
+hd:
+  %tmp7 = phi i32 [ %tmp8, %hd ], [ 0, %en ]
+  tail call void @h(i32* %a)
+  %tmp8 = add nuw i32 %tmp7, 1
+  %tmp9 = icmp eq i32 %tmp8, %b
+  br i1 %tmp9, label %ex, label %hd
+}
+
+define i32 @nonnull_exec_ctx_2(i32* %a, i32 %b) nounwind willreturn {
+; MBEC:      -- Explore context of:   %tmp3 = icmp eq i32 %b, 0
+; MBEC-NEXT:   [F: nonnull_exec_ctx_2]   %tmp3 = icmp eq i32 %b, 0
+; MBEC-NEXT:   [F: nonnull_exec_ctx_2]   br i1 %tmp3, label %ex, label %hd
+; MBEC-NEXT:   [F: nonnull_exec_ctx_2]   %tmp5 = tail call i32 @g(i32* nonnull %a)
+; MBEC-NEXT:   [F: nonnull_exec_ctx_2]   ret i32 %tmp5
+; MBEC-NEXT: -- Explore context of:   br i1 %tmp3, label %ex, label %hd
+; MBEC-NEXT:   [F: nonnull_exec_ctx_2]   br i1 %tmp3, label %ex, label %hd
+; MBEC-NEXT:   [F: nonnull_exec_ctx_2]   %tmp5 = tail call i32 @g(i32* nonnull %a)
+; MBEC-NEXT:   [F: nonnull_exec_ctx_2]   ret i32 %tmp5
+; MBEC-NEXT: -- Explore context of:   %tmp5 = tail call i32 @g(i32* nonnull %a)
+; MBEC-NEXT:   [F: nonnull_exec_ctx_2]   %tmp5 = tail call i32 @g(i32* nonnull %a)
+; MBEC-NEXT:   [F: nonnull_exec_ctx_2]   ret i32 %tmp5
+; MBEC-NEXT: -- Explore context of:   ret i32 %tmp5
+; MBEC-NEXT:   [F: nonnull_exec_ctx_2]   ret i32 %tmp5
+; MBEC-NEXT: -- Explore context of:   %tmp7 = phi i32 [ %tmp8, %hd ], [ 0, %en ]
+; MBEC-NEXT:   [F: nonnull_exec_ctx_2]   %tmp7 = phi i32 [ %tmp8, %hd ], [ 0, %en ]
+; MBEC-NEXT:   [F: nonnull_exec_ctx_2]   tail call void @h(i32* %a)
+; MBEC-NEXT:   [F: nonnull_exec_ctx_2]   %tmp8 = add nuw i32 %tmp7, 1
+; MBEC-NEXT:   [F: nonnull_exec_ctx_2]   %tmp9 = icmp eq i32 %tmp8, %b
+; MBEC-NEXT:   [F: nonnull_exec_ctx_2]   br i1 %tmp9, label %ex, label %hd
+; MBEC-NEXT:   [F: nonnull_exec_ctx_2]   %tmp5 = tail call i32 @g(i32* nonnull %a)
+; MBEC-NEXT:   [F: nonnull_exec_ctx_2]   ret i32 %tmp5
+; MBEC-NEXT: -- Explore context of:   tail call void @h(i32* %a)
+; MBEC-NEXT:   [F: nonnull_exec_ctx_2]   tail call void @h(i32* %a)
+; MBEC-NEXT:   [F: nonnull_exec_ctx_2]   %tmp8 = add nuw i32 %tmp7, 1
+; MBEC-NEXT:   [F: nonnull_exec_ctx_2]   %tmp9 = icmp eq i32 %tmp8, %b
+; MBEC-NEXT:   [F: nonnull_exec_ctx_2]   br i1 %tmp9, label %ex, label %hd
+; MBEC-NEXT:   [F: nonnull_exec_ctx_2]   %tmp5 = tail call i32 @g(i32* nonnull %a)
+; MBEC-NEXT:   [F: nonnull_exec_ctx_2]   ret i32 %tmp5
+; MBEC-NEXT: -- Explore context of:   %tmp8 = add nuw i32 %tmp7, 1
+; MBEC-NEXT:   [F: nonnull_exec_ctx_2]   %tmp8 = add nuw i32 %tmp7, 1
+; MBEC-NEXT:   [F: nonnull_exec_ctx_2]   %tmp9 = icmp eq i32 %tmp8, %b
+; MBEC-NEXT:   [F: nonnull_exec_ctx_2]   br i1 %tmp9, label %ex, label %hd
+; MBEC-NEXT:   [F: nonnull_exec_ctx_2]   %tmp5 = tail call i32 @g(i32* nonnull %a)
+; MBEC-NEXT:   [F: nonnull_exec_ctx_2]   ret i32 %tmp5
+; MBEC-NEXT: -- Explore context of:   %tmp9 = icmp eq i32 %tmp8, %b
+; MBEC-NEXT:   [F: nonnull_exec_ctx_2]   %tmp9 = icmp eq i32 %tmp8, %b
+; MBEC-NEXT:   [F: nonnull_exec_ctx_2]   br i1 %tmp9, label %ex, label %hd
+; MBEC-NEXT:   [F: nonnull_exec_ctx_2]   %tmp5 = tail call i32 @g(i32* nonnull %a)
+; MBEC-NEXT:   [F: nonnull_exec_ctx_2]   ret i32 %tmp5
+; MBEC-NEXT: -- Explore context of:   br i1 %tmp9, label %ex, label %hd
+; MBEC-NEXT:   [F: nonnull_exec_ctx_2]   br i1 %tmp9, label %ex, label %hd
+; MBEC-NEXT:   [F: nonnull_exec_ctx_2]   %tmp5 = tail call i32 @g(i32* nonnull %a)
+; MBEC-NEXT:   [F: nonnull_exec_ctx_2]   ret i32 %tmp5
+en:
+  %tmp3 = icmp eq i32 %b, 0
+  br i1 %tmp3, label %ex, label %hd
+
+ex:
+  %tmp5 = tail call i32 @g(i32* nonnull %a)
+  ret i32 %tmp5
+
+hd:
+  %tmp7 = phi i32 [ %tmp8, %hd ], [ 0, %en ]
+  tail call void @h(i32* %a)
+  %tmp8 = add nuw i32 %tmp7, 1
+  %tmp9 = icmp eq i32 %tmp8, %b
+  br i1 %tmp9, label %ex, label %hd
+}
diff --git a/llvm/test/Analysis/ScalarEvolution/limit-depth.ll b/llvm/test/Analysis/ScalarEvolution/limit-depth.ll
index db68a4f84..6fdf8c5df 100644
--- a/llvm/test/Analysis/ScalarEvolution/limit-depth.ll
+++ b/llvm/test/Analysis/ScalarEvolution/limit-depth.ll
@@ -46,7 +46,7 @@ define void @test_mul(i32 %a, i32 %b, i32 %c, i32 %d, i32 %e, i32 %f) {
 define void @test_sext(i32 %a, i32 %b, i32 %c, i32 %d, i32 %e, i32 %f) {
 ; CHECK-LABEL: @test_sext
 ; CHECK:        %se2 = sext i64 %iv2.inc to i128
-; CHECK-NEXT:   -->  {(1 + (sext i64 {(sext i32 (1 + %a)<nsw> to i64),+,1}<nsw><%loop> to i128))<nsw>,+,1}<nsw><%loop2>
+; CHECK-NEXT:   -->  {(1 + (sext i64 {(sext i32 (1 + %a) to i64),+,1}<nsw><%loop> to i128))<nsw>,+,1}<nsw><%loop2>
 entry:
   br label %loop
 
diff --git a/llvm/test/Analysis/ScalarEvolution/max-expr-cache.ll b/llvm/test/Analysis/ScalarEvolution/max-expr-cache.ll
index e30345eb9..a0fa4a9d2 100644
--- a/llvm/test/Analysis/ScalarEvolution/max-expr-cache.ll
+++ b/llvm/test/Analysis/ScalarEvolution/max-expr-cache.ll
@@ -55,7 +55,7 @@ bb4:
   %tmp45 = icmp slt i32 %tmp43, 256
   %tmp46 = select i1 %tmp45, i32 %tmp43, i32 256
 ; CHECK:  %tmp46 = select i1 %tmp45, i32 %tmp43, i32 256
-; CHECK-NEXT:  -->  (256 smin (1 + (256 smin (1 + (256 smin (1 + (256 smin (1 + (256 smin (1 + (256 smin (1 + (256 smin (1 + (256 smin {%tmp3,+,-256}<%bb4>)) smin {%tmp3,+,-256}<%bb4>)) smin {%tmp3,+,-256}<%bb4>)) smin {%tmp3,+,-256}<%bb4>)) smin {%tmp3,+,-256}<%bb4>)) smin {%tmp3,+,-256}<%bb4>)) smin {%tmp3,+,-256}<%bb4>)) smin {%tmp3,+,-256}<%bb4>)
+; CHECK-NEXT:  -->  (256 smin (1 + (256 smin (1 + (256 smin (1 + (256 smin (1 + (256 smin (1 + (256 smin (1 + (256 smin (1 + (256 smin {%tmp3,+,-256}<%bb4>))<nsw> smin {%tmp3,+,-256}<%bb4>))<nsw> smin {%tmp3,+,-256}<%bb4>))<nsw> smin {%tmp3,+,-256}<%bb4>))<nsw> smin {%tmp3,+,-256}<%bb4>))<nsw> smin {%tmp3,+,-256}<%bb4>))<nsw> smin {%tmp3,+,-256}<%bb4>))<nsw> smin {%tmp3,+,-256}<%bb4>) U: [-2147483648,257) S: [-2147483648,257)
   %tmp47 = icmp sgt i32 %tmp44, %tmp46
   %tmp48 = select i1 %tmp47, i32 %tmp44, i32 %tmp46
   %tmp49 = ashr i32 %tmp48, 3
@@ -130,7 +130,7 @@ bb4:
   %tmp45 = icmp ult i32 %tmp43, 256
   %tmp46 = select i1 %tmp45, i32 %tmp43, i32 256
 ; CHECK:  %tmp46 = select i1 %tmp45, i32 %tmp43, i32 256
-; CHECK-NEXT:  --> (256 umin (1 + (256 umin (1 + (256 umin (1 + (256 umin (1 + (256 umin (1 + (256 umin (1 + (256 umin (1 + (256 umin {%tmp3,+,-256}<%bb4>)) umin {%tmp3,+,-256}<%bb4>)) umin {%tmp3,+,-256}<%bb4>)) umin {%tmp3,+,-256}<%bb4>)) umin {%tmp3,+,-256}<%bb4>)) umin {%tmp3,+,-256}<%bb4>)) umin {%tmp3,+,-256}<%bb4>)) umin {%tmp3,+,-256}<%bb4>) 
+; CHECK-NEXT:  --> (256 umin (1 + (256 umin (1 + (256 umin (1 + (256 umin (1 + (256 umin (1 + (256 umin (1 + (256 umin (1 + (256 umin {%tmp3,+,-256}<%bb4>))<nuw><nsw> umin {%tmp3,+,-256}<%bb4>))<nuw><nsw> umin {%tmp3,+,-256}<%bb4>))<nuw><nsw> umin {%tmp3,+,-256}<%bb4>))<nuw><nsw> umin {%tmp3,+,-256}<%bb4>))<nuw><nsw> umin {%tmp3,+,-256}<%bb4>))<nuw><nsw> umin {%tmp3,+,-256}<%bb4>))<nuw><nsw> umin {%tmp3,+,-256}<%bb4>) U: [0,257) S: [0,257)
   %tmp47 = icmp ugt i32 %tmp44, %tmp46
   %tmp48 = select i1 %tmp47, i32 %tmp44, i32 %tmp46
   %tmp49 = ashr i32 %tmp48, 3
diff --git a/llvm/test/Analysis/ScalarEvolution/max-trip-count-address-space.ll b/llvm/test/Analysis/ScalarEvolution/max-trip-count-address-space.ll
index 1c72440a7..5260fe90e 100644
--- a/llvm/test/Analysis/ScalarEvolution/max-trip-count-address-space.ll
+++ b/llvm/test/Analysis/ScalarEvolution/max-trip-count-address-space.ll
@@ -65,4 +65,4 @@ for.end:                                          ; preds = %for.cond.for.end_cr
 
 ; CHECK: Determining loop execution counts for: @test
 ; CHECK-NEXT: backedge-taken count is
-; CHECK-NEXT: max backedge-taken count is -1
+; CHECK-NEXT: max backedge-taken count is 4294967294
diff --git a/llvm/test/Analysis/ScalarEvolution/max-trip-count.ll b/llvm/test/Analysis/ScalarEvolution/max-trip-count.ll
index 53b882bdc..e3ba313a6 100644
--- a/llvm/test/Analysis/ScalarEvolution/max-trip-count.ll
+++ b/llvm/test/Analysis/ScalarEvolution/max-trip-count.ll
@@ -97,7 +97,7 @@ for.end:                                          ; preds = %for.cond.for.end_cr
 
 ; CHECK: Determining loop execution counts for: @test
 ; CHECK-NEXT: backedge-taken count is
-; CHECK-NEXT: max backedge-taken count is -1
+; CHECK-NEXT: max backedge-taken count is 4294967294
 
 ; PR19799: Indvars miscompile due to an incorrect max backedge taken count from SCEV.
 ; CHECK-LABEL: @pr19799
diff --git a/llvm/test/Analysis/ScalarEvolution/multiple-max-iterations.ll b/llvm/test/Analysis/ScalarEvolution/multiple-max-iterations.ll
new file mode 100644
index 000000000..4686a2024
--- /dev/null
+++ b/llvm/test/Analysis/ScalarEvolution/multiple-max-iterations.ll
@@ -0,0 +1,2 @@
+; Ensure we can pass -scalar-evolution-max-iterations multiple times
+; RUN: opt -S -scalar-evolution -scalar-evolution-max-iterations=42 -scalar-evolution-max-iterations=42 < %s
diff --git a/llvm/test/Analysis/ScalarEvolution/nsw.ll b/llvm/test/Analysis/ScalarEvolution/nsw.ll
index 694273686..ca24f9d4a 100644
--- a/llvm/test/Analysis/ScalarEvolution/nsw.ll
+++ b/llvm/test/Analysis/ScalarEvolution/nsw.ll
@@ -163,7 +163,7 @@ bb5:                                              ; preds = %bb2
 declare void @f(i32)
 
 ; CHECK-LABEL: nswnowrap
-; CHECK: --> {(1 + %v)<nsw>,+,1}<nsw><%for.body>{{ U: [^ ]+ S: [^ ]+}}{{ *}}Exits: (2 + %v)
+; CHECK: --> {(1 + %v)<nsw>,+,1}<nsw><%for.body>{{ U: [^ ]+ S: [^ ]+}}{{ *}}Exits: (1 + ((1 + %v)<nsw> smax %v))
 define void @nswnowrap(i32 %v, i32* %buf) {
 entry:
   %add = add nsw i32 %v, 1
diff --git a/llvm/test/Analysis/ScalarEvolution/pr22674.ll b/llvm/test/Analysis/ScalarEvolution/pr22674.ll
index 1bc7fd327..7b382d86b 100644
--- a/llvm/test/Analysis/ScalarEvolution/pr22674.ll
+++ b/llvm/test/Analysis/ScalarEvolution/pr22674.ll
@@ -83,7 +83,7 @@ return:                                           ; preds = %_ZNK4llvm9Attribute
   ret void
 }
 
-attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.module.flags = !{!0}
 !llvm.ident = !{!1}
diff --git a/llvm/test/Analysis/ScalarEvolution/range_nw_flag.ll b/llvm/test/Analysis/ScalarEvolution/range_nw_flag.ll
new file mode 100644
index 000000000..0a0b5fb5e
--- /dev/null
+++ b/llvm/test/Analysis/ScalarEvolution/range_nw_flag.ll
@@ -0,0 +1,121 @@
+; RUN: opt < %s -S -analyze -scalar-evolution | FileCheck %s
+
+; copied from flags-from-poison.ll
+; CHECK-LABEL: @test-add-nuw
+; CHECK: -->  {(1 + %offset)<nuw>,+,1}<nuw><%loop> U: [1,0) S: [1,0)
+define void @test-add-nuw(float* %input, i32 %offset, i32 %numIterations) {
+entry:
+  br label %loop
+loop:
+  %i = phi i32 [ %nexti, %loop ], [ 0, %entry ]
+  %nexti = add nuw i32 %i, 1
+  %index32 = add nuw i32 %nexti, %offset
+  %ptr = getelementptr inbounds float, float* %input, i32 %index32
+  %f = load float, float* %ptr, align 4
+  %exitcond = icmp eq i32 %nexti, %numIterations
+  br i1 %exitcond, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+; CHECK-LABEL: @test-addrec-nuw
+; CHECK: -->  {(1 + (10 smax %offset))<nuw>,+,1}<nuw><%loop> U: [11,0) S: [11,0)
+define void @test-addrec-nuw(float* %input, i32 %offset, i32 %numIterations) {
+entry:
+  %cmp = icmp sgt i32 %offset, 10
+  %min.10 = select i1 %cmp, i32 %offset, i32 10
+  br label %loop
+loop:
+  %i = phi i32 [ %nexti, %loop ], [ 0, %entry ]
+  %nexti = add nuw i32 %i, 1
+  %index32 = add nuw i32 %nexti, %min.10
+  %ptr = getelementptr inbounds float, float* %input, i32 %index32
+  %f = load float, float* %ptr, align 4
+  %exitcond = icmp eq i32 %nexti, %numIterations
+  br i1 %exitcond, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+; CHECK-LABEL: @test-addrec-nsw-start-neg-strip-neg
+; CHECK: -->  {(-1 + (-10 smin %offset))<nsw>,+,-1}<nsw><%loop> U: [-2147483648,-10) S: [-2147483648,-10)
+define void @test-addrec-nsw-start-neg-strip-neg(float* %input, i32 %offset, i32 %numIterations) {
+entry:
+  %cmp = icmp slt i32 %offset, -10
+  %max = select i1 %cmp, i32 %offset, i32 -10
+  br label %loop
+loop:
+  %i = phi i32 [ %nexti, %loop ], [ 0, %entry ]
+  %nexti = add nsw i32 %i, -1
+  %index32 = add nsw i32 %nexti, %max
+  %ptr = getelementptr inbounds float, float* %input, i32 %index32
+  %f = load float, float* %ptr, align 4
+  %exitcond = icmp eq i32 %nexti, %numIterations
+  br i1 %exitcond, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+; CHECK-LABEL: @test-addrec-nsw-start-pos-strip-neg
+; CHECK: -->  {(-1 + (10 smin %offset))<nsw>,+,-1}<nsw><%loop> U: [-2147483648,10) S: [-2147483648,10)
+define void @test-addrec-nsw-start-pos-strip-neg(float* %input, i32 %offset, i32 %numIterations) {
+entry:
+  %cmp = icmp slt i32 %offset, 10
+  %max = select i1 %cmp, i32 %offset, i32  10
+  br label %loop
+loop:
+  %i = phi i32 [ %nexti, %loop ], [ 0, %entry ]
+  %nexti = add nsw i32 %i, -1
+  %index32 = add nsw i32 %nexti, %max
+  %ptr = getelementptr inbounds float, float* %input, i32 %index32
+  %f = load float, float* %ptr, align 4
+  %exitcond = icmp eq i32 %nexti, %numIterations
+  br i1 %exitcond, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+; CHECK-LABEL: @test-addrec-nsw-start-pos-strip-pos
+; CHECK: -->  {(1 + (10 smax %offset))<nuw><nsw>,+,1}<nuw><nsw><%loop> U: [11,-2147483648) S: [11,-2147483648)
+define void @test-addrec-nsw-start-pos-strip-pos(float* %input, i32 %offset, i32 %numIterations) {
+entry:
+  %cmp = icmp sgt i32 %offset, 10
+  %min = select i1 %cmp, i32 %offset, i32  10
+  br label %loop
+loop:
+  %i = phi i32 [ %nexti, %loop ], [ 0, %entry ]
+  %nexti = add nsw i32 %i, 1
+  %index32 = add nsw i32 %nexti, %min
+  %ptr = getelementptr inbounds float, float* %input, i32 %index32
+  %f = load float, float* %ptr, align 4
+  %exitcond = icmp eq i32 %nexti, %numIterations
+  br i1 %exitcond, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+; CHECK-LABEL: @test-addrec-nsw-start-neg-strip-pos
+; CHECK: -->  {(1 + (-10 smax %offset))<nsw>,+,1}<nsw><%loop> U: [-9,-2147483648) S: [-9,-2147483648)
+define void @test-addrec-nsw-start-neg-strip-pos(float* %input, i32 %offset, i32 %numIterations) {
+entry:
+  %cmp = icmp sgt i32 %offset, -10
+  %min = select i1 %cmp, i32 %offset, i32  -10
+  br label %loop
+loop:
+  %i = phi i32 [ %nexti, %loop ], [ 0, %entry ]
+  %nexti = add nsw i32 %i, 1
+  %index32 = add nsw i32 %nexti, %min
+  %ptr = getelementptr inbounds float, float* %input, i32 %index32
+  %f = load float, float* %ptr, align 4
+  %exitcond = icmp eq i32 %nexti, %numIterations
+  br i1 %exitcond, label %exit, label %loop
+
+exit:
+  ret void
+}
+
diff --git a/llvm/test/Analysis/ScalarEvolution/scev-canonical-mode.ll b/llvm/test/Analysis/ScalarEvolution/scev-canonical-mode.ll
index 0aa7ff08f..c78bec323 100644
--- a/llvm/test/Analysis/ScalarEvolution/scev-canonical-mode.ll
+++ b/llvm/test/Analysis/ScalarEvolution/scev-canonical-mode.ll
@@ -29,4 +29,4 @@ hup.exit:                                         ; preds = %for.body.i, %if.the
   ret void
 }
 
-attributes #0 = { norecurse nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { norecurse nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/Analysis/ScalarEvolution/sext-mul.ll b/llvm/test/Analysis/ScalarEvolution/sext-mul.ll
index 42810be6e..4a1074981 100644
--- a/llvm/test/Analysis/ScalarEvolution/sext-mul.ll
+++ b/llvm/test/Analysis/ScalarEvolution/sext-mul.ll
@@ -11,7 +11,7 @@
 ; CHECK: %tmp15 = getelementptr inbounds i32, i32* %arg, i64 %tmp14
 ; CHECK-NEXT: --> {{.*}} Exits: (4 + (4 * (sext i32 (-2 + (2 * %arg2)) to i64))<nsw> + %arg)
 ; CHECK:Loop %bb7: backedge-taken count is (-1 + (zext i32 %arg2 to i64))<nsw>
-; CHECK-NEXT:Loop %bb7: max backedge-taken count is -1
+; CHECK-NEXT:Loop %bb7: max backedge-taken count is 4294967294
 ; CHECK-NEXT:Loop %bb7: Predicated backedge-taken count is (-1 + (zext i32 %arg2 to i64))<nsw>
 
 define void @foo(i32* nocapture %arg, i32 %arg1, i32 %arg2) {
@@ -52,7 +52,7 @@ bb7:                                              ; preds = %bb7, %bb3
 ; CHECK: %t14 = or i128 %t10, 1
 ; CHECK-NEXT: --> {{.*}} Exits: (1 + (sext i127 (-633825300114114700748351602688 + (633825300114114700748351602688 * (zext i32 %arg5 to i127))) to i128))<nuw><nsw>
 ; CHECK: Loop %bb7: backedge-taken count is (-1 + (zext i32 %arg5 to i128))<nsw>
-; CHECK-NEXT: Loop %bb7: max backedge-taken count is -1
+; CHECK-NEXT: Loop %bb7: max backedge-taken count is 4294967294
 ; CHECK-NEXT: Loop %bb7: Predicated backedge-taken count is (-1 + (zext i32 %arg5 to i128))<nsw>
 
 define void @goo(i32* nocapture %arg3, i32 %arg4, i32 %arg5) {
diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count-andor.ll b/llvm/test/Analysis/ScalarEvolution/trip-count-andor.ll
new file mode 100644
index 000000000..7ffc423e0
--- /dev/null
+++ b/llvm/test/Analysis/ScalarEvolution/trip-count-andor.ll
@@ -0,0 +1,365 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt < %s -analyze -scalar-evolution -scalar-evolution-classify-expressions=0 2>&1 | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @unsimplified_and1(i32 %n) {
+; CHECK-LABEL: 'unsimplified_and1'
+; CHECK-NEXT:  Determining loop execution counts for: @unsimplified_and1
+; CHECK-NEXT:  Loop %loop: backedge-taken count is %n
+; CHECK-NEXT:  Loop %loop: max backedge-taken count is -1
+; CHECK-NEXT:  Loop %loop: Predicated backedge-taken count is %n
+; CHECK-NEXT:   Predicates:
+; CHECK:       Loop %loop: Trip multiple is 1
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.inc, %loop ]
+  %iv.inc = add nsw i32 %iv, 1
+  %becond = icmp ule i32 %iv.inc, %n
+  %and = and i1 %becond, true
+  br i1 %and, label %loop, label %leave
+
+leave:
+  ret void
+}
+
+define void @unsimplified_and2(i32 %n) {
+; CHECK-LABEL: 'unsimplified_and2'
+; CHECK-NEXT:  Determining loop execution counts for: @unsimplified_and2
+; CHECK-NEXT:  Loop %loop: backedge-taken count is %n
+; CHECK-NEXT:  Loop %loop: max backedge-taken count is -1
+; CHECK-NEXT:  Loop %loop: Predicated backedge-taken count is %n
+; CHECK-NEXT:   Predicates:
+; CHECK:       Loop %loop: Trip multiple is 1
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.inc, %loop ]
+  %iv.inc = add nsw i32 %iv, 1
+  %becond = icmp ule i32 %iv.inc, %n
+  %and = and i1 true, %becond
+  br i1 %and, label %loop, label %leave
+
+leave:
+  ret void
+}
+
+define void @unsimplified_and3(i32 %n) {
+; CHECK-LABEL: 'unsimplified_and3'
+; CHECK-NEXT:  Determining loop execution counts for: @unsimplified_and3
+; CHECK-NEXT:  Loop %loop: backedge-taken count is false
+; CHECK-NEXT:  Loop %loop: max backedge-taken count is false
+; CHECK-NEXT:  Loop %loop: Predicated backedge-taken count is false
+; CHECK-NEXT:   Predicates:
+; CHECK:       Loop %loop: Trip multiple is 1
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.inc, %loop ]
+  %iv.inc = add nsw i32 %iv, 1
+  %becond = icmp ule i32 %iv.inc, %n
+  %and = and i1 false, %becond
+  br i1 %and, label %loop, label %leave
+
+leave:
+  ret void
+}
+
+define void @unsimplified_and4(i32 %n) {
+; CHECK-LABEL: 'unsimplified_and4'
+; CHECK-NEXT:  Determining loop execution counts for: @unsimplified_and4
+; CHECK-NEXT:  Loop %loop: backedge-taken count is false
+; CHECK-NEXT:  Loop %loop: max backedge-taken count is false
+; CHECK-NEXT:  Loop %loop: Predicated backedge-taken count is false
+; CHECK-NEXT:   Predicates:
+; CHECK:       Loop %loop: Trip multiple is 1
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.inc, %loop ]
+  %iv.inc = add nsw i32 %iv, 1
+  %becond = icmp ule i32 %iv.inc, %n
+  %and = and i1 %becond, false
+  br i1 %and, label %loop, label %leave
+
+leave:
+  ret void
+}
+
+define void @unsimplified_or1(i32 %n) {
+; CHECK-LABEL: 'unsimplified_or1'
+; CHECK-NEXT:  Determining loop execution counts for: @unsimplified_or1
+; CHECK-NEXT:  Loop %loop: Unpredictable backedge-taken count.
+; CHECK-NEXT:  Loop %loop: Unpredictable max backedge-taken count.
+; CHECK-NEXT:  Loop %loop: Unpredictable predicated backedge-taken count.
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.inc, %loop ]
+  %iv.inc = add nsw i32 %iv, 1
+  %becond = icmp ule i32 %iv.inc, %n
+  %or = or i1 %becond, true
+  br i1 %or, label %loop, label %leave
+
+leave:
+  ret void
+}
+
+define void @unsimplified_or2(i32 %n) {
+; CHECK-LABEL: 'unsimplified_or2'
+; CHECK-NEXT:  Determining loop execution counts for: @unsimplified_or2
+; CHECK-NEXT:  Loop %loop: Unpredictable backedge-taken count.
+; CHECK-NEXT:  Loop %loop: Unpredictable max backedge-taken count.
+; CHECK-NEXT:  Loop %loop: Unpredictable predicated backedge-taken count.
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.inc, %loop ]
+  %iv.inc = add nsw i32 %iv, 1
+  %becond = icmp ule i32 %iv.inc, %n
+  %or = or i1 true, %becond
+  br i1 %or, label %loop, label %leave
+
+leave:
+  ret void
+}
+
+define void @unsimplified_or3(i32 %n) {
+; CHECK-LABEL: 'unsimplified_or3'
+; CHECK-NEXT:  Determining loop execution counts for: @unsimplified_or3
+; CHECK-NEXT:  Loop %loop: backedge-taken count is %n
+; CHECK-NEXT:  Loop %loop: max backedge-taken count is -1
+; CHECK-NEXT:  Loop %loop: Predicated backedge-taken count is %n
+; CHECK-NEXT:   Predicates:
+; CHECK:       Loop %loop: Trip multiple is 1
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.inc, %loop ]
+  %iv.inc = add nsw i32 %iv, 1
+  %becond = icmp ule i32 %iv.inc, %n
+  %or = or i1 false, %becond
+  br i1 %or, label %loop, label %leave
+
+leave:
+  ret void
+}
+
+define void @unsimplified_or4(i32 %n) {
+; CHECK-LABEL: 'unsimplified_or4'
+; CHECK-NEXT:  Determining loop execution counts for: @unsimplified_or4
+; CHECK-NEXT:  Loop %loop: backedge-taken count is %n
+; CHECK-NEXT:  Loop %loop: max backedge-taken count is -1
+; CHECK-NEXT:  Loop %loop: Predicated backedge-taken count is %n
+; CHECK-NEXT:   Predicates:
+; CHECK:       Loop %loop: Trip multiple is 1
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.inc, %loop ]
+  %iv.inc = add nsw i32 %iv, 1
+  %becond = icmp ule i32 %iv.inc, %n
+  %or = or i1 %becond, false
+  br i1 %or, label %loop, label %leave
+
+leave:
+  ret void
+}
+
+define void @reversed_and1(i32 %n) {
+; CHECK-LABEL: 'reversed_and1'
+; CHECK-NEXT:  Determining loop execution counts for: @reversed_and1
+; CHECK-NEXT:  Loop %loop: backedge-taken count is %n
+; CHECK-NEXT:  Loop %loop: max backedge-taken count is -1
+; CHECK-NEXT:  Loop %loop: Predicated backedge-taken count is %n
+; CHECK-NEXT:   Predicates:
+; CHECK:       Loop %loop: Trip multiple is 1
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.inc, %loop ]
+  %iv.inc = add nsw i32 %iv, 1
+  %becond = icmp ugt i32 %iv.inc, %n
+  %and = and i1 %becond, true
+  br i1 %and, label %leave, label %loop
+
+leave:
+  ret void
+}
+
+define void @reversed_and2(i32 %n) {
+; CHECK-LABEL: 'reversed_and2'
+; CHECK-NEXT:  Determining loop execution counts for: @reversed_and2
+; CHECK-NEXT:  Loop %loop: backedge-taken count is %n
+; CHECK-NEXT:  Loop %loop: max backedge-taken count is -1
+; CHECK-NEXT:  Loop %loop: Predicated backedge-taken count is %n
+; CHECK-NEXT:   Predicates:
+; CHECK:       Loop %loop: Trip multiple is 1
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.inc, %loop ]
+  %iv.inc = add nsw i32 %iv, 1
+  %becond = icmp ugt i32 %iv.inc, %n
+  %and = and i1 true, %becond
+  br i1 %and, label %leave, label %loop
+
+leave:
+  ret void
+}
+
+define void @reversed_and3(i32 %n) {
+; CHECK-LABEL: 'reversed_and3'
+; CHECK-NEXT:  Determining loop execution counts for: @reversed_and3
+; CHECK-NEXT:  Loop %loop: Unpredictable backedge-taken count.
+; CHECK-NEXT:  Loop %loop: Unpredictable max backedge-taken count.
+; CHECK-NEXT:  Loop %loop: Unpredictable predicated backedge-taken count.
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.inc, %loop ]
+  %iv.inc = add nsw i32 %iv, 1
+  %becond = icmp ugt i32 %iv.inc, %n
+  %and = and i1 false, %becond
+  br i1 %and, label %leave, label %loop
+
+leave:
+  ret void
+}
+
+define void @reversed_and4(i32 %n) {
+; CHECK-LABEL: 'reversed_and4'
+; CHECK-NEXT:  Determining loop execution counts for: @reversed_and4
+; CHECK-NEXT:  Loop %loop: Unpredictable backedge-taken count.
+; CHECK-NEXT:  Loop %loop: Unpredictable max backedge-taken count.
+; CHECK-NEXT:  Loop %loop: Unpredictable predicated backedge-taken count.
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.inc, %loop ]
+  %iv.inc = add nsw i32 %iv, 1
+  %becond = icmp ugt i32 %iv.inc, %n
+  %and = and i1 %becond, false
+  br i1 %and, label %leave, label %loop
+
+leave:
+  ret void
+}
+
+define void @reversed_or1(i32 %n) {
+; CHECK-LABEL: 'reversed_or1'
+; CHECK-NEXT:  Determining loop execution counts for: @reversed_or1
+; CHECK-NEXT:  Loop %loop: backedge-taken count is false
+; CHECK-NEXT:  Loop %loop: max backedge-taken count is false
+; CHECK-NEXT:  Loop %loop: Predicated backedge-taken count is false
+; CHECK-NEXT:   Predicates:
+; CHECK:       Loop %loop: Trip multiple is 1
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.inc, %loop ]
+  %iv.inc = add nsw i32 %iv, 1
+  %becond = icmp ugt i32 %iv.inc, %n
+  %or = or i1 %becond, true
+  br i1 %or, label %leave, label %loop
+
+leave:
+  ret void
+}
+
+define void @reversed_or2(i32 %n) {
+; CHECK-LABEL: 'reversed_or2'
+; CHECK-NEXT:  Determining loop execution counts for: @reversed_or2
+; CHECK-NEXT:  Loop %loop: backedge-taken count is false
+; CHECK-NEXT:  Loop %loop: max backedge-taken count is false
+; CHECK-NEXT:  Loop %loop: Predicated backedge-taken count is false
+; CHECK-NEXT:   Predicates:
+; CHECK:       Loop %loop: Trip multiple is 1
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.inc, %loop ]
+  %iv.inc = add nsw i32 %iv, 1
+  %becond = icmp ugt i32 %iv.inc, %n
+  %or = or i1 true, %becond
+  br i1 %or, label %leave, label %loop
+
+leave:
+  ret void
+}
+
+define void @reversed_or3(i32 %n) {
+; CHECK-LABEL: 'reversed_or3'
+; CHECK-NEXT:  Determining loop execution counts for: @reversed_or3
+; CHECK-NEXT:  Loop %loop: backedge-taken count is %n
+; CHECK-NEXT:  Loop %loop: max backedge-taken count is -1
+; CHECK-NEXT:  Loop %loop: Predicated backedge-taken count is %n
+; CHECK-NEXT:   Predicates:
+; CHECK:       Loop %loop: Trip multiple is 1
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.inc, %loop ]
+  %iv.inc = add nsw i32 %iv, 1
+  %becond = icmp ugt i32 %iv.inc, %n
+  %or = or i1 false, %becond
+  br i1 %or, label %leave, label %loop
+
+leave:
+  ret void
+}
+
+define void @reversed_or4(i32 %n) {
+; CHECK-LABEL: 'reversed_or4'
+; CHECK-NEXT:  Determining loop execution counts for: @reversed_or4
+; CHECK-NEXT:  Loop %loop: backedge-taken count is %n
+; CHECK-NEXT:  Loop %loop: max backedge-taken count is -1
+; CHECK-NEXT:  Loop %loop: Predicated backedge-taken count is %n
+; CHECK-NEXT:   Predicates:
+; CHECK:       Loop %loop: Trip multiple is 1
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.inc, %loop ]
+  %iv.inc = add nsw i32 %iv, 1
+  %becond = icmp ugt i32 %iv.inc, %n
+  %or = or i1 %becond, false
+  br i1 %or, label %leave, label %loop
+
+leave:
+  ret void
+}
diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count.ll b/llvm/test/Analysis/ScalarEvolution/trip-count.ll
index b5ff1c3d8..aef7f1b9b 100644
--- a/llvm/test/Analysis/ScalarEvolution/trip-count.ll
+++ b/llvm/test/Analysis/ScalarEvolution/trip-count.ll
@@ -1,43 +1,50 @@
-; RUN: opt < %s -analyze -scalar-evolution -scalar-evolution-max-iterations=0 | FileCheck %s
-; RUN: opt < %s -passes='print<scalar-evolution>' -disable-output 2>&1 | FileCheck %s
-; PR1101
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt < %s -analyze -scalar-evolution -scalar-evolution-max-iterations=0  -scalar-evolution-classify-expressions=0  | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
-@A = weak global [1000 x i32] zeroinitializer, align 32         
-
-; CHECK-LABEL: Determining loop execution counts for: @test1
-; CHECK: backedge-taken count is 10000
-
-define void @test1(i32 %N) {
+@A = weak global [1000 x i32] zeroinitializer, align 32
+
+define void @PR1101(i32 %N) {
+; CHECK-LABEL: 'PR1101'
+; CHECK-NEXT:  Determining loop execution counts for: @PR1101
+; CHECK-NEXT:  Loop %bb3: backedge-taken count is 10000
+; CHECK-NEXT:  Loop %bb3: max backedge-taken count is 10000
+; CHECK-NEXT:  Loop %bb3: Predicated backedge-taken count is 10000
+; CHECK-NEXT:   Predicates:
+; CHECK:       Loop %bb3: Trip multiple is 10001
+;
 entry:
-        br label %bb3
+  br label %bb3
 
 bb:             ; preds = %bb3
-        %tmp = getelementptr [1000 x i32], [1000 x i32]* @A, i32 0, i32 %i.0          ; <i32*> [#uses=1]
-        store i32 123, i32* %tmp
-        %tmp2 = add i32 %i.0, 1         ; <i32> [#uses=1]
-        br label %bb3
+  %tmp = getelementptr [1000 x i32], [1000 x i32]* @A, i32 0, i32 %i.0          ; <i32*> [#uses=1]
+  store i32 123, i32* %tmp
+  %tmp2 = add i32 %i.0, 1         ; <i32> [#uses=1]
+  br label %bb3
 
 bb3:            ; preds = %bb, %entry
-        %i.0 = phi i32 [ 0, %entry ], [ %tmp2, %bb ]            ; <i32> [#uses=3]
-        %tmp3 = icmp sle i32 %i.0, 9999          ; <i1> [#uses=1]
-        br i1 %tmp3, label %bb, label %bb5
+  %i.0 = phi i32 [ 0, %entry ], [ %tmp2, %bb ]            ; <i32> [#uses=3]
+  %tmp3 = icmp sle i32 %i.0, 9999          ; <i1> [#uses=1]
+  br i1 %tmp3, label %bb, label %bb5
 
 bb5:            ; preds = %bb3
-        br label %return
+  br label %return
 
 return:         ; preds = %bb5
-        ret void
+  ret void
 }
 
-; PR22795
-; CHECK-LABEL: Classifying expressions for: @test2
-; CHECK:   %iv = phi i32 [ -1, %entry ], [ %next.1, %for.inc.1 ]
-; CHECK-NEXT:  -->  {-1,+,2}<%preheader> U: full-set S: full-set             Exits: 13
-
-define i32 @test2() {
+define i32 @PR22795() {
+; CHECK-LABEL: 'PR22795'
+; CHECK-NEXT:  Determining loop execution counts for: @PR22795
+; CHECK-NEXT:  Loop %preheader: backedge-taken count is 7
+; CHECK-NEXT:  Loop %preheader: max backedge-taken count is 7
+; CHECK-NEXT:  Loop %preheader: Predicated backedge-taken count is 7
+; CHECK-NEXT:   Predicates:
+; CHECK:       Loop %preheader: Trip multiple is 8
+;
 entry:
   %bins = alloca [16 x i64], align 16
   %0 = bitcast [16 x i64]* %bins to i8*
@@ -93,11 +100,14 @@ declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i1) #0
 declare void @may_exit() nounwind
 
 define void @pr28012(i32 %n) {
-; CHECK-LABEL: Classifying expressions for: @pr28012
-; CHECK: Loop %loop: backedge-taken count is -1431655751
-; CHECK: Loop %loop: max backedge-taken count is -1431655751
-; CHECK: Loop %loop: Predicated backedge-taken count is -1431655751
-
+; CHECK-LABEL: 'pr28012'
+; CHECK-NEXT:  Determining loop execution counts for: @pr28012
+; CHECK-NEXT:  Loop %loop: backedge-taken count is -1431655751
+; CHECK-NEXT:  Loop %loop: max backedge-taken count is -1431655751
+; CHECK-NEXT:  Loop %loop: Predicated backedge-taken count is -1431655751
+; CHECK-NEXT:   Predicates:
+; CHECK:       Loop %loop: Trip multiple is 2863311546
+;
 entry:
   br label %loop
 
diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count10.ll b/llvm/test/Analysis/ScalarEvolution/trip-count10.ll
index ead80b950..5540e3e6a 100644
--- a/llvm/test/Analysis/ScalarEvolution/trip-count10.ll
+++ b/llvm/test/Analysis/ScalarEvolution/trip-count10.ll
@@ -1,24 +1,15 @@
-; RUN: opt < %s -analyze -scalar-evolution | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt < %s -analyze -scalar-evolution -scalar-evolution-classify-expressions=0 | FileCheck %s
 
 ; Trip counts with trivial exit conditions.
 
-; CHECK: Determining loop execution counts for: @a
-; CHECK: Loop %loop: Unpredictable backedge-taken count.
-; CHECK: Loop %loop: Unpredictable max backedge-taken count.
-
-; CHECK: Determining loop execution counts for: @b
-; CHECK: Loop %loop: backedge-taken count is false
-; CHECK: Loop %loop: max backedge-taken count is false
-
-; CHECK: Determining loop execution counts for: @c
-; CHECK: Loop %loop: backedge-taken count is false
-; CHECK: Loop %loop: max backedge-taken count is false
-
-; CHECK: Determining loop execution counts for: @d
-; CHECK: Loop %loop: Unpredictable backedge-taken count.
-; CHECK: Loop %loop: Unpredictable max backedge-taken count.
-
 define void @a(i64 %n) nounwind {
+; CHECK-LABEL: 'a'
+; CHECK-NEXT:  Determining loop execution counts for: @a
+; CHECK-NEXT:  Loop %loop: Unpredictable backedge-taken count.
+; CHECK-NEXT:  Loop %loop: Unpredictable max backedge-taken count.
+; CHECK-NEXT:  Loop %loop: Unpredictable predicated backedge-taken count.
+;
 entry:
   %t0 = icmp sgt i64 %n, 0
   br i1 %t0, label %loop, label %return
@@ -33,6 +24,14 @@ return:
   ret void
 }
 define void @b(i64 %n) nounwind {
+; CHECK-LABEL: 'b'
+; CHECK-NEXT:  Determining loop execution counts for: @b
+; CHECK-NEXT:  Loop %loop: backedge-taken count is false
+; CHECK-NEXT:  Loop %loop: max backedge-taken count is false
+; CHECK-NEXT:  Loop %loop: Predicated backedge-taken count is false
+; CHECK-NEXT:   Predicates:
+; CHECK:       Loop %loop: Trip multiple is 1
+;
 entry:
   %t0 = icmp sgt i64 %n, 0
   br i1 %t0, label %loop, label %return
@@ -47,6 +46,14 @@ return:
   ret void
 }
 define void @c(i64 %n) nounwind {
+; CHECK-LABEL: 'c'
+; CHECK-NEXT:  Determining loop execution counts for: @c
+; CHECK-NEXT:  Loop %loop: backedge-taken count is false
+; CHECK-NEXT:  Loop %loop: max backedge-taken count is false
+; CHECK-NEXT:  Loop %loop: Predicated backedge-taken count is false
+; CHECK-NEXT:   Predicates:
+; CHECK:       Loop %loop: Trip multiple is 1
+;
 entry:
   %t0 = icmp sgt i64 %n, 0
   br i1 %t0, label %loop, label %return
@@ -61,6 +68,12 @@ return:
   ret void
 }
 define void @d(i64 %n) nounwind {
+; CHECK-LABEL: 'd'
+; CHECK-NEXT:  Determining loop execution counts for: @d
+; CHECK-NEXT:  Loop %loop: Unpredictable backedge-taken count.
+; CHECK-NEXT:  Loop %loop: Unpredictable max backedge-taken count.
+; CHECK-NEXT:  Loop %loop: Unpredictable predicated backedge-taken count.
+;
 entry:
   %t0 = icmp sgt i64 %n, 0
   br i1 %t0, label %loop, label %return
@@ -80,13 +93,15 @@ return:
 ; should return unknown.
 
 ; PR7416
-; CHECK: Determining loop execution counts for: @nonpolynomial
-; CHECK-NEXT: Loop %loophead: Unpredictable backedge-taken count
-; CHECK-NEXT: Loop %loophead: Unpredictable max backedge-taken count
-
 declare i1 @g() nounwind
 
 define void @nonpolynomial() {
+; CHECK-LABEL: 'nonpolynomial'
+; CHECK-NEXT:  Determining loop execution counts for: @nonpolynomial
+; CHECK-NEXT:  Loop %loophead: Unpredictable backedge-taken count.
+; CHECK-NEXT:  Loop %loophead: Unpredictable max backedge-taken count.
+; CHECK-NEXT:  Loop %loophead: Unpredictable predicated backedge-taken count.
+;
 entry:
   br label %loophead
 loophead:
@@ -108,11 +123,15 @@ retbb:
 
 ; PHI nodes with all constant operands.
 
-; CHECK: Determining loop execution counts for: @constant_phi_operands
-; CHECK: Loop %loop: backedge-taken count is 1
-; CHECK: Loop %loop: max backedge-taken count is 1
-
 define void @constant_phi_operands() nounwind {
+; CHECK-LABEL: 'constant_phi_operands'
+; CHECK-NEXT:  Determining loop execution counts for: @constant_phi_operands
+; CHECK-NEXT:  Loop %loop: backedge-taken count is 1
+; CHECK-NEXT:  Loop %loop: max backedge-taken count is 1
+; CHECK-NEXT:  Loop %loop: Predicated backedge-taken count is 1
+; CHECK-NEXT:   Predicates:
+; CHECK:       Loop %loop: Trip multiple is 2
+;
 entry:
   br label %loop
 
@@ -129,10 +148,13 @@ return:
 ; One side of the expression test against a value that will be skipped.
 ; We can't assume undefined behavior just because we have an NSW flag.
 ;
-; CHECK: Determining loop execution counts for: @exit_orcond_nsw
-; CHECK: Loop %for.body.i: Unpredictable backedge-taken count.
-; CHECK: Loop %for.body.i: max backedge-taken count is 1
 define void @exit_orcond_nsw(i32 *%a) nounwind {
+; CHECK-LABEL: 'exit_orcond_nsw'
+; CHECK-NEXT:  Determining loop execution counts for: @exit_orcond_nsw
+; CHECK-NEXT:  Loop %for.body.i: Unpredictable backedge-taken count.
+; CHECK-NEXT:  Loop %for.body.i: max backedge-taken count is 1
+; CHECK-NEXT:  Loop %for.body.i: Unpredictable predicated backedge-taken count.
+;
 entry:
   br label %for.body.i
 
diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count12.ll b/llvm/test/Analysis/ScalarEvolution/trip-count12.ll
index 5e7d72d5e..d0086ee2e 100644
--- a/llvm/test/Analysis/ScalarEvolution/trip-count12.ll
+++ b/llvm/test/Analysis/ScalarEvolution/trip-count12.ll
@@ -1,7 +1,7 @@
 ; RUN: opt < %s -analyze -scalar-evolution | FileCheck %s
 
 ; CHECK: Determining loop execution counts for: @test
-; CHECK: Loop %for.body: backedge-taken count is ((-2 + %len)<nsw> /u 2)
+; CHECK: Loop %for.body: backedge-taken count is ((-2 + %len) /u 2)
 ; CHECK: Loop %for.body: max backedge-taken count is 1073741823
 
 define zeroext i16 @test(i16* nocapture %p, i32 %len) nounwind readonly {
diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count15.ll b/llvm/test/Analysis/ScalarEvolution/trip-count15.ll
new file mode 100644
index 000000000..3ad83776b
--- /dev/null
+++ b/llvm/test/Analysis/ScalarEvolution/trip-count15.ll
@@ -0,0 +1,121 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt -S -analyze -scalar-evolution < %s | FileCheck %s
+
+define void @umin_unsigned_check(i64 %n) {
+; CHECK-LABEL: 'umin_unsigned_check'
+; CHECK-NEXT:  Classifying expressions for: @umin_unsigned_check
+; CHECK-NEXT:    %min.n = select i1 %min.cmp, i64 4096, i64 %n
+; CHECK-NEXT:    --> (4096 umin %n) U: [0,4097) S: [0,4097)
+; CHECK-NEXT:    %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+; CHECK-NEXT:    --> {0,+,1}<%loop> U: [0,4098) S: [0,4098) Exits: (1 + (4096 umin %n))<nuw><nsw> LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:    %iv.next = add i64 %iv, 1
+; CHECK-NEXT:    --> {1,+,1}<%loop> U: [1,4099) S: [1,4099) Exits: (2 + (4096 umin %n)) LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:  Determining loop execution counts for: @umin_unsigned_check
+; CHECK-NEXT:  Loop %loop: backedge-taken count is (1 + (4096 umin %n))<nuw><nsw>
+; CHECK-NEXT:  Loop %loop: max backedge-taken count is 4097
+; CHECK-NEXT:  Loop %loop: Predicated backedge-taken count is (1 + (4096 umin %n))<nuw><nsw>
+; CHECK-NEXT:   Predicates:
+; CHECK:       Loop %loop: Trip multiple is 1
+;
+entry:
+  %min.cmp = icmp ult i64 4096, %n
+  %min.n = select i1 %min.cmp, i64 4096, i64 %n
+  br label %loop
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %iv.next = add i64 %iv, 1
+  %exit = icmp ugt i64 %iv, %min.n
+  br i1 %exit, label %loop_exit, label %loop
+
+loop_exit:
+  ret void
+}
+
+define void @umin_signed_check(i64 %n) {
+; CHECK-LABEL: 'umin_signed_check'
+; CHECK-NEXT:  Classifying expressions for: @umin_signed_check
+; CHECK-NEXT:    %min.n = select i1 %min.cmp, i64 4096, i64 %n
+; CHECK-NEXT:    --> (4096 umin %n) U: [0,4097) S: [0,4097)
+; CHECK-NEXT:    %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+; CHECK-NEXT:    --> {0,+,1}<%loop> U: [0,4098) S: [0,4098) Exits: (1 + (4096 umin %n))<nuw><nsw> LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:    %iv.next = add i64 %iv, 1
+; CHECK-NEXT:    --> {1,+,1}<%loop> U: [1,4099) S: [1,4099) Exits: (2 + (4096 umin %n)) LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:  Determining loop execution counts for: @umin_signed_check
+; CHECK-NEXT:  Loop %loop: backedge-taken count is (1 + (4096 umin %n))<nuw><nsw>
+; CHECK-NEXT:  Loop %loop: max backedge-taken count is 4097
+; CHECK-NEXT:  Loop %loop: Predicated backedge-taken count is (1 + (4096 umin %n))<nuw><nsw>
+; CHECK-NEXT:   Predicates:
+; CHECK:       Loop %loop: Trip multiple is 1
+;
+entry:
+  %min.cmp = icmp ult i64 4096, %n
+  %min.n = select i1 %min.cmp, i64 4096, i64 %n
+  br label %loop
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %iv.next = add i64 %iv, 1
+  %exit = icmp sgt i64 %iv, %min.n
+  br i1 %exit, label %loop_exit, label %loop
+
+loop_exit:
+  ret void
+}
+
+define void @smin_signed_check(i64 %n) {
+; CHECK-LABEL: 'smin_signed_check'
+; CHECK-NEXT:  Classifying expressions for: @smin_signed_check
+; CHECK-NEXT:    %min.n = select i1 %min.cmp, i64 4096, i64 %n
+; CHECK-NEXT:    --> (4096 smin %n) U: [-9223372036854775808,4097) S: [-9223372036854775808,4097)
+; CHECK-NEXT:    %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+; CHECK-NEXT:    --> {0,+,1}<%loop> U: [0,4098) S: [0,4098) Exits: (0 smax (1 + (4096 smin %n))<nsw>) LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:    %iv.next = add i64 %iv, 1
+; CHECK-NEXT:    --> {1,+,1}<%loop> U: [1,4099) S: [1,4099) Exits: (1 + (0 smax (1 + (4096 smin %n))<nsw>))<nuw><nsw> LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:  Determining loop execution counts for: @smin_signed_check
+; CHECK-NEXT:  Loop %loop: backedge-taken count is (0 smax (1 + (4096 smin %n))<nsw>)
+; CHECK-NEXT:  Loop %loop: max backedge-taken count is 4097
+; CHECK-NEXT:  Loop %loop: Predicated backedge-taken count is (0 smax (1 + (4096 smin %n))<nsw>)
+; CHECK-NEXT:   Predicates:
+; CHECK:       Loop %loop: Trip multiple is 1
+;
+entry:
+  %min.cmp = icmp slt i64 4096, %n
+  %min.n = select i1 %min.cmp, i64 4096, i64 %n
+  br label %loop
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %iv.next = add i64 %iv, 1
+  %exit = icmp sgt i64 %iv, %min.n
+  br i1 %exit, label %loop_exit, label %loop
+
+loop_exit:
+  ret void
+}
+
+define void @smin_unsigned_check(i64 %n) {
+; CHECK-LABEL: 'smin_unsigned_check'
+; CHECK-NEXT:  Classifying expressions for: @smin_unsigned_check
+; CHECK-NEXT:    %min.n = select i1 %min.cmp, i64 4096, i64 %n
+; CHECK-NEXT:    --> (4096 smin %n) U: [-9223372036854775808,4097) S: [-9223372036854775808,4097)
+; CHECK-NEXT:    %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+; CHECK-NEXT:    --> {0,+,1}<%loop> U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:    %iv.next = add i64 %iv, 1
+; CHECK-NEXT:    --> {1,+,1}<%loop> U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:  Determining loop execution counts for: @smin_unsigned_check
+; CHECK-NEXT:  Loop %loop: Unpredictable backedge-taken count.
+; CHECK-NEXT:  Loop %loop: Unpredictable max backedge-taken count.
+; CHECK-NEXT:  Loop %loop: Unpredictable predicated backedge-taken count.
+;
+entry:
+  %min.cmp = icmp slt i64 4096, %n
+  %min.n = select i1 %min.cmp, i64 4096, i64 %n
+  br label %loop
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %iv.next = add i64 %iv, 1
+  %exit = icmp ugt i64 %iv, %min.n
+  br i1 %exit, label %loop_exit, label %loop
+
+loop_exit:
+  ret void
+}
+
diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count2.ll b/llvm/test/Analysis/ScalarEvolution/trip-count2.ll
index d988eff7c..7f4552723 100644
--- a/llvm/test/Analysis/ScalarEvolution/trip-count2.ll
+++ b/llvm/test/Analysis/ScalarEvolution/trip-count2.ll
@@ -1,34 +1,40 @@
-; RUN: opt < %s -analyze -scalar-evolution | FileCheck %s
-; PR1101
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt < %s -analyze -scalar-evolution  -scalar-evolution-classify-expressions=0  | FileCheck %s
 
-@A = weak global [1000 x i32] zeroinitializer, align 32         
+@A = weak global [1000 x i32] zeroinitializer, align 32
 
-; CHECK: backedge-taken count is 4
-
-define void @test(i32 %N) {
+define void @PR1101(i32 %N) {
+; CHECK-LABEL: 'PR1101'
+; CHECK-NEXT:  Determining loop execution counts for: @PR1101
+; CHECK-NEXT:  Loop %bb3: backedge-taken count is 4
+; CHECK-NEXT:  Loop %bb3: max backedge-taken count is 4
+; CHECK-NEXT:  Loop %bb3: Predicated backedge-taken count is 4
+; CHECK-NEXT:   Predicates:
+; CHECK:       Loop %bb3: Trip multiple is 5
+;
 entry:
-        br label %bb3
+  br label %bb3
 
 bb:             ; preds = %bb3
-        %tmp = getelementptr [1000 x i32], [1000 x i32]* @A, i32 0, i32 %i.0          ; <i32*> [#uses=1]
-        store i32 123, i32* %tmp
-        %tmp4 = mul i32 %i.0, 4         ; <i32> [#uses=1]
-        %tmp5 = or i32 %tmp4, 1
-        %tmp61 = xor i32 %tmp5, -2147483648
-        %tmp6 = trunc i32 %tmp61 to i16
-        %tmp71 = shl i16 %tmp6, 2
-        %tmp7 = zext i16 %tmp71 to i32
-        %tmp2 = add i32 %tmp7, %i.0
-        br label %bb3
+  %tmp = getelementptr [1000 x i32], [1000 x i32]* @A, i32 0, i32 %i.0          ; <i32*> [#uses=1]
+  store i32 123, i32* %tmp
+  %tmp4 = mul i32 %i.0, 4         ; <i32> [#uses=1]
+  %tmp5 = or i32 %tmp4, 1
+  %tmp61 = xor i32 %tmp5, -2147483648
+  %tmp6 = trunc i32 %tmp61 to i16
+  %tmp71 = shl i16 %tmp6, 2
+  %tmp7 = zext i16 %tmp71 to i32
+  %tmp2 = add i32 %tmp7, %i.0
+  br label %bb3
 
 bb3:            ; preds = %bb, %entry
-        %i.0 = phi i32 [ 0, %entry ], [ %tmp2, %bb ]            ; <i32> [#uses=3]
-        %tmp3 = icmp sle i32 %i.0, 9999          ; <i1> [#uses=1]
-        br i1 %tmp3, label %bb, label %bb5
+  %i.0 = phi i32 [ 0, %entry ], [ %tmp2, %bb ]            ; <i32> [#uses=3]
+  %tmp3 = icmp sle i32 %i.0, 9999          ; <i1> [#uses=1]
+  br i1 %tmp3, label %bb, label %bb5
 
 bb5:            ; preds = %bb3
-        br label %return
+  br label %return
 
 return:         ; preds = %bb5
-        ret void
+  ret void
 }
diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count3.ll b/llvm/test/Analysis/ScalarEvolution/trip-count3.ll
index e10012c0c..a50886be3 100644
--- a/llvm/test/Analysis/ScalarEvolution/trip-count3.ll
+++ b/llvm/test/Analysis/ScalarEvolution/trip-count3.ll
@@ -1,12 +1,10 @@
-; RUN: opt < %s -scalar-evolution -analyze | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt < %s -scalar-evolution -analyze -scalar-evolution-classify-expressions=0 | FileCheck %s
 
 ; ScalarEvolution can't compute a trip count because it doesn't know if
 ; dividing by the stride will have a remainder. This could theoretically
 ; be teaching it how to use a more elaborate trip count computation.
 
-; CHECK: Loop %bb3.i: backedge-taken count is ((63 + (-1 * (63 smin %0)) + %0) /u 64)
-; CHECK: Loop %bb3.i: max backedge-taken count is 33554431
-
 %struct.FILE = type { i32, i8*, i8*, i8*, i8*, i8*, i8*, i8*, i8*, i8*, i8*, i8*, %struct._IO_marker*, %struct.FILE*, i32, i32, i64, i16, i8, [1 x i8], i8*, i64, i8*, i8*, i8*, i8*, i64, i32, [20 x i8] }
 %struct.SHA_INFO = type { [5 x i32], i32, i32, [16 x i32] }
 %struct._IO_marker = type { %struct._IO_marker*, %struct.FILE*, i32 }
@@ -39,6 +37,14 @@ declare i32 @fclose(%struct.FILE* nocapture) nounwind
 declare void @sha_stream(%struct.SHA_INFO* nocapture, %struct.FILE* nocapture) nounwind
 
 define void @sha_stream_bb3_2E_i(%struct.SHA_INFO* %sha_info, i8* %data1, i32, i8** %buffer_addr.0.i.out, i32* %count_addr.0.i.out) nounwind {
+; CHECK-LABEL: 'sha_stream_bb3_2E_i'
+; CHECK-NEXT:  Determining loop execution counts for: @sha_stream_bb3_2E_i
+; CHECK-NEXT:  Loop %bb3.i: backedge-taken count is ((63 + (-1 * (63 smin %0)) + %0) /u 64)
+; CHECK-NEXT:  Loop %bb3.i: max backedge-taken count is 33554431
+; CHECK-NEXT:  Loop %bb3.i: Predicated backedge-taken count is ((63 + (-1 * (63 smin %0)) + %0) /u 64)
+; CHECK-NEXT:   Predicates:
+; CHECK:       Loop %bb3.i: Trip multiple is 1
+;
 newFuncRoot:
   br label %bb3.i
 
diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count4.ll b/llvm/test/Analysis/ScalarEvolution/trip-count4.ll
index 966ffd26f..4c2d079e1 100644
--- a/llvm/test/Analysis/ScalarEvolution/trip-count4.ll
+++ b/llvm/test/Analysis/ScalarEvolution/trip-count4.ll
@@ -1,25 +1,32 @@
-; RUN: opt < %s -analyze -scalar-evolution | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt < %s -analyze -scalar-evolution -scalar-evolution-classify-expressions=0 | FileCheck %s
 
 ; ScalarEvolution should be able to compute a loop exit value for %indvar.i8.
 
-; CHECK: sext{{.*}}trunc{{.*}}Exits: 11
-
 define void @another_count_down_signed(double* %d, i64 %n) nounwind {
+; CHECK-LABEL: 'another_count_down_signed'
+; CHECK-NEXT:  Determining loop execution counts for: @another_count_down_signed
+; CHECK-NEXT:  Loop %loop: backedge-taken count is (-11 + %n)
+; CHECK-NEXT:  Loop %loop: max backedge-taken count is -1
+; CHECK-NEXT:  Loop %loop: Predicated backedge-taken count is (-11 + %n)
+; CHECK-NEXT:   Predicates:
+; CHECK:       Loop %loop: Trip multiple is 1
+;
 entry:
-	br label %loop
+  br label %loop
 
 loop:		; preds = %loop, %entry
-	%indvar = phi i64 [ %n, %entry ], [ %indvar.next, %loop ]		; <i64> [#uses=4]
-	%s0 = shl i64 %indvar, 8		; <i64> [#uses=1]
-	%indvar.i8 = ashr i64 %s0, 8		; <i64> [#uses=1]
-	%t0 = getelementptr double, double* %d, i64 %indvar.i8		; <double*> [#uses=2]
-	%t1 = load double, double* %t0		; <double> [#uses=1]
-	%t2 = fmul double %t1, 1.000000e-01		; <double> [#uses=1]
-	store double %t2, double* %t0
-	%indvar.next = sub i64 %indvar, 1		; <i64> [#uses=2]
-	%exitcond = icmp eq i64 %indvar.next, 10		; <i1> [#uses=1]
-	br i1 %exitcond, label %return, label %loop
+  %indvar = phi i64 [ %n, %entry ], [ %indvar.next, %loop ]		; <i64> [#uses=4]
+  %s0 = shl i64 %indvar, 8		; <i64> [#uses=1]
+  %indvar.i8 = ashr i64 %s0, 8		; <i64> [#uses=1]
+  %t0 = getelementptr double, double* %d, i64 %indvar.i8		; <double*> [#uses=2]
+  %t1 = load double, double* %t0		; <double> [#uses=1]
+  %t2 = fmul double %t1, 1.000000e-01		; <double> [#uses=1]
+  store double %t2, double* %t0
+  %indvar.next = sub i64 %indvar, 1		; <i64> [#uses=2]
+  %exitcond = icmp eq i64 %indvar.next, 10		; <i1> [#uses=1]
+  br i1 %exitcond, label %return, label %loop
 
 return:		; preds = %loop
-	ret void
+  ret void
 }
diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count6.ll b/llvm/test/Analysis/ScalarEvolution/trip-count6.ll
index 7980bbdcb..103b097e0 100644
--- a/llvm/test/Analysis/ScalarEvolution/trip-count6.ll
+++ b/llvm/test/Analysis/ScalarEvolution/trip-count6.ll
@@ -1,10 +1,17 @@
-; RUN: opt < %s -analyze -scalar-evolution | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt < %s -analyze -scalar-evolution -scalar-evolution-classify-expressions=0 | FileCheck %s
 
 @mode_table = global [4 x i32] zeroinitializer          ; <[4 x i32]*> [#uses=1]
 
-; CHECK: max backedge-taken count is 1{{$}}
-
 define i8 @f() {
+; CHECK-LABEL: 'f'
+; CHECK-NEXT:  Determining loop execution counts for: @f
+; CHECK-NEXT:  Loop %bb: <multiple exits> Unpredictable backedge-taken count.
+; CHECK-NEXT:    exit count for bb: ***COULDNOTCOMPUTE***
+; CHECK-NEXT:    exit count for bb2: 1
+; CHECK-NEXT:  Loop %bb: max backedge-taken count is 1
+; CHECK-NEXT:  Loop %bb: Unpredictable predicated backedge-taken count.
+;
 entry:
   tail call i32 @fegetround( )          ; <i32>:0 [#uses=1]
   br label %bb
@@ -29,10 +36,10 @@ bb4:            ; preds = %bb2
   br label %bb
 
 bb5:            ; preds = %bb2
-  tail call void @raise_exception( ) noreturn 
+  tail call void @raise_exception( ) noreturn
   unreachable
 }
 
 declare i32 @fegetround()
 
-declare void @raise_exception() noreturn 
+declare void @raise_exception() noreturn
diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count7.ll b/llvm/test/Analysis/ScalarEvolution/trip-count7.ll
index bbe76c490..8b92bf71c 100644
--- a/llvm/test/Analysis/ScalarEvolution/trip-count7.ll
+++ b/llvm/test/Analysis/ScalarEvolution/trip-count7.ll
@@ -1,9 +1,8 @@
-; RUN: opt < %s -analyze -scalar-evolution | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt < %s -analyze -scalar-evolution -scalar-evolution-classify-expressions=0 | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
 
-; CHECK: Loop %bb7.i: Unpredictable backedge-taken count.
-
 	%struct.complex = type { float, float }
 	%struct.element = type { i32, i32 }
 	%struct.node = type { %struct.node*, %struct.node*, i32 }
@@ -62,90 +61,96 @@ declare void @Doit() nounwind
 declare void @Doit_bb7([15 x i32]*, [17 x i32]*, [9 x i32]*) nounwind
 
 define void @Doit_bb7_2E_i([9 x i32]* %x1, [15 x i32]* %c, [17 x i32]* %b, [9 x i32]* %a, i32* %q, i32* %x1.sub, i32* %b9, i32* %a10, i32* %c11) nounwind {
+; CHECK-LABEL: 'Doit_bb7_2E_i'
+; CHECK-NEXT:  Determining loop execution counts for: @Doit_bb7_2E_i
+; CHECK-NEXT:  Loop %bb7.i: Unpredictable backedge-taken count.
+; CHECK-NEXT:  Loop %bb7.i: max backedge-taken count is 8
+; CHECK-NEXT:  Loop %bb7.i: Unpredictable predicated backedge-taken count.
+;
 newFuncRoot:
-	br label %bb7.i
+  br label %bb7.i
 
 Try.exit.exitStub:		; preds = %bb7.i
-	ret void
+  ret void
 
 bb.i:		; preds = %bb7.i
-	%tmp = add i32 %j.0.i, 1		; <i32> [#uses=5]
-	store i32 0, i32* %q, align 4
-	%tmp1 = sext i32 %tmp to i64		; <i64> [#uses=1]
-	%tmp2 = getelementptr [9 x i32], [9 x i32]* %a, i64 0, i64 %tmp1		; <i32*> [#uses=1]
-	%tmp3 = load i32, i32* %tmp2, align 4		; <i32> [#uses=1]
-	%tmp4 = icmp eq i32 %tmp3, 0		; <i1> [#uses=1]
-	br i1 %tmp4, label %bb.i.bb7.i.backedge_crit_edge, label %bb1.i
+  %tmp = add i32 %j.0.i, 1		; <i32> [#uses=5]
+  store i32 0, i32* %q, align 4
+  %tmp1 = sext i32 %tmp to i64		; <i64> [#uses=1]
+  %tmp2 = getelementptr [9 x i32], [9 x i32]* %a, i64 0, i64 %tmp1		; <i32*> [#uses=1]
+  %tmp3 = load i32, i32* %tmp2, align 4		; <i32> [#uses=1]
+  %tmp4 = icmp eq i32 %tmp3, 0		; <i1> [#uses=1]
+  br i1 %tmp4, label %bb.i.bb7.i.backedge_crit_edge, label %bb1.i
 
 bb1.i:		; preds = %bb.i
-	%tmp5 = add i32 %j.0.i, 2		; <i32> [#uses=1]
-	%tmp6 = sext i32 %tmp5 to i64		; <i64> [#uses=1]
-	%tmp7 = getelementptr [17 x i32], [17 x i32]* %b, i64 0, i64 %tmp6		; <i32*> [#uses=1]
-	%tmp8 = load i32, i32* %tmp7, align 4		; <i32> [#uses=1]
-	%tmp9 = icmp eq i32 %tmp8, 0		; <i1> [#uses=1]
-	br i1 %tmp9, label %bb1.i.bb7.i.backedge_crit_edge, label %bb2.i
+  %tmp5 = add i32 %j.0.i, 2		; <i32> [#uses=1]
+  %tmp6 = sext i32 %tmp5 to i64		; <i64> [#uses=1]
+  %tmp7 = getelementptr [17 x i32], [17 x i32]* %b, i64 0, i64 %tmp6		; <i32*> [#uses=1]
+  %tmp8 = load i32, i32* %tmp7, align 4		; <i32> [#uses=1]
+  %tmp9 = icmp eq i32 %tmp8, 0		; <i1> [#uses=1]
+  br i1 %tmp9, label %bb1.i.bb7.i.backedge_crit_edge, label %bb2.i
 
 bb2.i:		; preds = %bb1.i
-	%tmp10 = sub i32 7, %j.0.i		; <i32> [#uses=1]
-	%tmp11 = sext i32 %tmp10 to i64		; <i64> [#uses=1]
-	%tmp12 = getelementptr [15 x i32], [15 x i32]* %c, i64 0, i64 %tmp11		; <i32*> [#uses=1]
-	%tmp13 = load i32, i32* %tmp12, align 4		; <i32> [#uses=1]
-	%tmp14 = icmp eq i32 %tmp13, 0		; <i1> [#uses=1]
-	br i1 %tmp14, label %bb2.i.bb7.i.backedge_crit_edge, label %bb3.i
+  %tmp10 = sub i32 7, %j.0.i		; <i32> [#uses=1]
+  %tmp11 = sext i32 %tmp10 to i64		; <i64> [#uses=1]
+  %tmp12 = getelementptr [15 x i32], [15 x i32]* %c, i64 0, i64 %tmp11		; <i32*> [#uses=1]
+  %tmp13 = load i32, i32* %tmp12, align 4		; <i32> [#uses=1]
+  %tmp14 = icmp eq i32 %tmp13, 0		; <i1> [#uses=1]
+  br i1 %tmp14, label %bb2.i.bb7.i.backedge_crit_edge, label %bb3.i
 
 bb3.i:		; preds = %bb2.i
-	%tmp15 = getelementptr [9 x i32], [9 x i32]* %x1, i64 0, i64 1		; <i32*> [#uses=1]
-	store i32 %tmp, i32* %tmp15, align 4
-	%tmp16 = sext i32 %tmp to i64		; <i64> [#uses=1]
-	%tmp17 = getelementptr [9 x i32], [9 x i32]* %a, i64 0, i64 %tmp16		; <i32*> [#uses=1]
-	store i32 0, i32* %tmp17, align 4
-	%tmp18 = add i32 %j.0.i, 2		; <i32> [#uses=1]
-	%tmp19 = sext i32 %tmp18 to i64		; <i64> [#uses=1]
-	%tmp20 = getelementptr [17 x i32], [17 x i32]* %b, i64 0, i64 %tmp19		; <i32*> [#uses=1]
-	store i32 0, i32* %tmp20, align 4
-	%tmp21 = sub i32 7, %j.0.i		; <i32> [#uses=1]
-	%tmp22 = sext i32 %tmp21 to i64		; <i64> [#uses=1]
-	%tmp23 = getelementptr [15 x i32], [15 x i32]* %c, i64 0, i64 %tmp22		; <i32*> [#uses=1]
-	store i32 0, i32* %tmp23, align 4
-	call void @Try(i32 2, i32* %q, i32* %b9, i32* %a10, i32* %c11, i32* %x1.sub) nounwind
-	%tmp24 = load i32, i32* %q, align 4		; <i32> [#uses=1]
-	%tmp25 = icmp eq i32 %tmp24, 0		; <i1> [#uses=1]
-	br i1 %tmp25, label %bb5.i, label %bb3.i.bb7.i.backedge_crit_edge
+  %tmp15 = getelementptr [9 x i32], [9 x i32]* %x1, i64 0, i64 1		; <i32*> [#uses=1]
+  store i32 %tmp, i32* %tmp15, align 4
+  %tmp16 = sext i32 %tmp to i64		; <i64> [#uses=1]
+  %tmp17 = getelementptr [9 x i32], [9 x i32]* %a, i64 0, i64 %tmp16		; <i32*> [#uses=1]
+  store i32 0, i32* %tmp17, align 4
+  %tmp18 = add i32 %j.0.i, 2		; <i32> [#uses=1]
+  %tmp19 = sext i32 %tmp18 to i64		; <i64> [#uses=1]
+  %tmp20 = getelementptr [17 x i32], [17 x i32]* %b, i64 0, i64 %tmp19		; <i32*> [#uses=1]
+  store i32 0, i32* %tmp20, align 4
+  %tmp21 = sub i32 7, %j.0.i		; <i32> [#uses=1]
+  %tmp22 = sext i32 %tmp21 to i64		; <i64> [#uses=1]
+  %tmp23 = getelementptr [15 x i32], [15 x i32]* %c, i64 0, i64 %tmp22		; <i32*> [#uses=1]
+  store i32 0, i32* %tmp23, align 4
+  call void @Try(i32 2, i32* %q, i32* %b9, i32* %a10, i32* %c11, i32* %x1.sub) nounwind
+  %tmp24 = load i32, i32* %q, align 4		; <i32> [#uses=1]
+  %tmp25 = icmp eq i32 %tmp24, 0		; <i1> [#uses=1]
+  br i1 %tmp25, label %bb5.i, label %bb3.i.bb7.i.backedge_crit_edge
 
 bb5.i:		; preds = %bb3.i
-	%tmp26 = sext i32 %tmp to i64		; <i64> [#uses=1]
-	%tmp27 = getelementptr [9 x i32], [9 x i32]* %a, i64 0, i64 %tmp26		; <i32*> [#uses=1]
-	store i32 1, i32* %tmp27, align 4
-	%tmp28 = add i32 %j.0.i, 2		; <i32> [#uses=1]
-	%tmp29 = sext i32 %tmp28 to i64		; <i64> [#uses=1]
-	%tmp30 = getelementptr [17 x i32], [17 x i32]* %b, i64 0, i64 %tmp29		; <i32*> [#uses=1]
-	store i32 1, i32* %tmp30, align 4
-	%tmp31 = sub i32 7, %j.0.i		; <i32> [#uses=1]
-	%tmp32 = sext i32 %tmp31 to i64		; <i64> [#uses=1]
-	%tmp33 = getelementptr [15 x i32], [15 x i32]* %c, i64 0, i64 %tmp32		; <i32*> [#uses=1]
-	store i32 1, i32* %tmp33, align 4
-	br label %bb7.i.backedge
+  %tmp26 = sext i32 %tmp to i64		; <i64> [#uses=1]
+  %tmp27 = getelementptr [9 x i32], [9 x i32]* %a, i64 0, i64 %tmp26		; <i32*> [#uses=1]
+  store i32 1, i32* %tmp27, align 4
+  %tmp28 = add i32 %j.0.i, 2		; <i32> [#uses=1]
+  %tmp29 = sext i32 %tmp28 to i64		; <i64> [#uses=1]
+  %tmp30 = getelementptr [17 x i32], [17 x i32]* %b, i64 0, i64 %tmp29		; <i32*> [#uses=1]
+  store i32 1, i32* %tmp30, align 4
+  %tmp31 = sub i32 7, %j.0.i		; <i32> [#uses=1]
+  %tmp32 = sext i32 %tmp31 to i64		; <i64> [#uses=1]
+  %tmp33 = getelementptr [15 x i32], [15 x i32]* %c, i64 0, i64 %tmp32		; <i32*> [#uses=1]
+  store i32 1, i32* %tmp33, align 4
+  br label %bb7.i.backedge
 
 bb7.i.backedge:		; preds = %bb3.i.bb7.i.backedge_crit_edge, %bb2.i.bb7.i.backedge_crit_edge, %bb1.i.bb7.i.backedge_crit_edge, %bb.i.bb7.i.backedge_crit_edge, %bb5.i
-	br label %bb7.i
+  br label %bb7.i
 
 bb7.i:		; preds = %bb7.i.backedge, %newFuncRoot
-	%j.0.i = phi i32 [ 0, %newFuncRoot ], [ %tmp, %bb7.i.backedge ]		; <i32> [#uses=8]
-	%tmp34 = load i32, i32* %q, align 4		; <i32> [#uses=1]
-	%tmp35 = icmp eq i32 %tmp34, 0		; <i1> [#uses=1]
-	%tmp36 = icmp ne i32 %j.0.i, 8		; <i1> [#uses=1]
-	%tmp37 = and i1 %tmp35, %tmp36		; <i1> [#uses=1]
-	br i1 %tmp37, label %bb.i, label %Try.exit.exitStub
+  %j.0.i = phi i32 [ 0, %newFuncRoot ], [ %tmp, %bb7.i.backedge ]		; <i32> [#uses=8]
+  %tmp34 = load i32, i32* %q, align 4		; <i32> [#uses=1]
+  %tmp35 = icmp eq i32 %tmp34, 0		; <i1> [#uses=1]
+  %tmp36 = icmp ne i32 %j.0.i, 8		; <i1> [#uses=1]
+  %tmp37 = and i1 %tmp35, %tmp36		; <i1> [#uses=1]
+  br i1 %tmp37, label %bb.i, label %Try.exit.exitStub
 
 bb.i.bb7.i.backedge_crit_edge:		; preds = %bb.i
-	br label %bb7.i.backedge
+  br label %bb7.i.backedge
 
 bb1.i.bb7.i.backedge_crit_edge:		; preds = %bb1.i
-	br label %bb7.i.backedge
+  br label %bb7.i.backedge
 
 bb2.i.bb7.i.backedge_crit_edge:		; preds = %bb2.i
-	br label %bb7.i.backedge
+  br label %bb7.i.backedge
 
 bb3.i.bb7.i.backedge_crit_edge:		; preds = %bb3.i
-	br label %bb7.i.backedge
+  br label %bb7.i.backedge
 }
diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count8.ll b/llvm/test/Analysis/ScalarEvolution/trip-count8.ll
index a1777bc96..ac06fbf5d 100644
--- a/llvm/test/Analysis/ScalarEvolution/trip-count8.ll
+++ b/llvm/test/Analysis/ScalarEvolution/trip-count8.ll
@@ -1,38 +1,45 @@
-; RUN: opt < %s -analyze -scalar-evolution | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt < %s -analyze -scalar-evolution -scalar-evolution-classify-expressions=0 | FileCheck %s
 ; PR4599
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
 
-; CHECK: Loop %for.body: backedge-taken count is (-1 + {{%?}}ecx)
-
 define i32 @foo(i32 %ecx) nounwind {
+; CHECK-LABEL: 'foo'
+; CHECK-NEXT:  Determining loop execution counts for: @foo
+; CHECK-NEXT:  Loop %for.body: backedge-taken count is (-1 + %ecx)
+; CHECK-NEXT:  Loop %for.body: max backedge-taken count is -2
+; CHECK-NEXT:  Loop %for.body: Predicated backedge-taken count is (-1 + %ecx)
+; CHECK-NEXT:   Predicates:
+; CHECK:       Loop %for.body: Trip multiple is 1
+;
 entry:
-	%cmp2 = icmp eq i32 %ecx, 0		; <i1> [#uses=1]
-	br i1 %cmp2, label %for.end, label %bb.nph
+  %cmp2 = icmp eq i32 %ecx, 0		; <i1> [#uses=1]
+  br i1 %cmp2, label %for.end, label %bb.nph
 
 for.cond:		; preds = %for.inc
-	%cmp = icmp ult i32 %inc, %ecx		; <i1> [#uses=1]
-	br i1 %cmp, label %for.body, label %for.cond.for.end_crit_edge
+  %cmp = icmp ult i32 %inc, %ecx		; <i1> [#uses=1]
+  br i1 %cmp, label %for.body, label %for.cond.for.end_crit_edge
 
 for.cond.for.end_crit_edge:		; preds = %for.cond
-	%phitmp = add i32 %i.01, 2		; <i32> [#uses=1]
-	br label %for.end
+  %phitmp = add i32 %i.01, 2		; <i32> [#uses=1]
+  br label %for.end
 
 bb.nph:		; preds = %entry
-	br label %for.body
+  br label %for.body
 
 for.body:		; preds = %bb.nph, %for.cond
-	%i.01 = phi i32 [ %inc, %for.cond ], [ 0, %bb.nph ]		; <i32> [#uses=3]
-	%call = call i32 @bar(i32 %i.01) nounwind		; <i32> [#uses=0]
-	br label %for.inc
+  %i.01 = phi i32 [ %inc, %for.cond ], [ 0, %bb.nph ]		; <i32> [#uses=3]
+  %call = call i32 @bar(i32 %i.01) nounwind		; <i32> [#uses=0]
+  br label %for.inc
 
 for.inc:		; preds = %for.body
-	%inc = add i32 %i.01, 1		; <i32> [#uses=2]
-	br label %for.cond
+  %inc = add i32 %i.01, 1		; <i32> [#uses=2]
+  br label %for.cond
 
 for.end:		; preds = %for.cond.for.end_crit_edge, %entry
-	%i.0.lcssa = phi i32 [ %phitmp, %for.cond.for.end_crit_edge ], [ 1, %entry ]		; <i32> [#uses=1]
-	ret i32 %i.0.lcssa
+  %i.0.lcssa = phi i32 [ %phitmp, %for.cond.for.end_crit_edge ], [ 1, %entry ]		; <i32> [#uses=1]
+  ret i32 %i.0.lcssa
 }
 
 declare i32 @bar(i32)
diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count9.ll b/llvm/test/Analysis/ScalarEvolution/trip-count9.ll
index c0a1d12fa..d0fb51a1e 100644
--- a/llvm/test/Analysis/ScalarEvolution/trip-count9.ll
+++ b/llvm/test/Analysis/ScalarEvolution/trip-count9.ll
@@ -1,4 +1,5 @@
-; RUN: opt -analyze -scalar-evolution -S < %s | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt -analyze -scalar-evolution -S -scalar-evolution-classify-expressions=0 < %s | FileCheck %s
 
 ; Every combination of
 ;  - starting at 0, 1, or %x
@@ -8,10 +9,15 @@
 
 ; Some of these represent missed opportunities.
 
-; CHECK: Determining loop execution counts for: @foo
-; CHECK: Loop %loop: backedge-taken count is (-1 + %n)
-; CHECK: Loop %loop: max backedge-taken count is 6
 define void @foo(i4 %n) {
+; CHECK-LABEL: 'foo'
+; CHECK-NEXT:  Determining loop execution counts for: @foo
+; CHECK-NEXT:  Loop %loop: backedge-taken count is (-1 + %n)
+; CHECK-NEXT:  Loop %loop: max backedge-taken count is 6
+; CHECK-NEXT:  Loop %loop: Predicated backedge-taken count is (-1 + %n)
+; CHECK-NEXT:   Predicates:
+; CHECK:       Loop %loop: Trip multiple is 1
+;
 entry:
   %s = icmp sgt i4 %n, 0
   br i1 %s, label %loop, label %exit
@@ -24,10 +30,13 @@ exit:
   ret void
 }
 
-; CHECK: Determining loop execution counts for: @step2
-; CHECK: Loop %loop: Unpredictable backedge-taken count.
-; CHECK: Loop %loop: Unpredictable max backedge-taken count.
 define void @step2(i4 %n) {
+; CHECK-LABEL: 'step2'
+; CHECK-NEXT:  Determining loop execution counts for: @step2
+; CHECK-NEXT:  Loop %loop: Unpredictable backedge-taken count.
+; CHECK-NEXT:  Loop %loop: Unpredictable max backedge-taken count.
+; CHECK-NEXT:  Loop %loop: Unpredictable predicated backedge-taken count.
+;
 entry:
   %s = icmp sgt i4 %n, 0
   br i1 %s, label %loop, label %exit
@@ -40,10 +49,15 @@ exit:
   ret void
 }
 
-; CHECK: Determining loop execution counts for: @start1
-; CHECK: Loop %loop: backedge-taken count is (-2 + (2 smax %n))
-; CHECK: Loop %loop: max backedge-taken count is 5
 define void @start1(i4 %n) {
+; CHECK-LABEL: 'start1'
+; CHECK-NEXT:  Determining loop execution counts for: @start1
+; CHECK-NEXT:  Loop %loop: backedge-taken count is (-2 + (2 smax %n))<nsw>
+; CHECK-NEXT:  Loop %loop: max backedge-taken count is 5
+; CHECK-NEXT:  Loop %loop: Predicated backedge-taken count is (-2 + (2 smax %n))<nsw>
+; CHECK-NEXT:   Predicates:
+; CHECK:       Loop %loop: Trip multiple is 1
+;
 entry:
   %s = icmp sgt i4 %n, 0
   br i1 %s, label %loop, label %exit
@@ -56,10 +70,13 @@ exit:
   ret void
 }
 
-; CHECK: Determining loop execution counts for: @start1_step2
-; CHECK: Loop %loop: Unpredictable backedge-taken count.
-; CHECK: Loop %loop: Unpredictable max backedge-taken count.
 define void @start1_step2(i4 %n) {
+; CHECK-LABEL: 'start1_step2'
+; CHECK-NEXT:  Determining loop execution counts for: @start1_step2
+; CHECK-NEXT:  Loop %loop: Unpredictable backedge-taken count.
+; CHECK-NEXT:  Loop %loop: Unpredictable max backedge-taken count.
+; CHECK-NEXT:  Loop %loop: Unpredictable predicated backedge-taken count.
+;
 entry:
   %s = icmp sgt i4 %n, 0
   br i1 %s, label %loop, label %exit
@@ -72,10 +89,15 @@ exit:
   ret void
 }
 
-; CHECK: Determining loop execution counts for: @startx
-; CHECK: Loop %loop: backedge-taken count is (-1 + (-1 * %x) + ((1 + %x) smax %n))
-; CHECK: Loop %loop: max backedge-taken count is -1
 define void @startx(i4 %n, i4 %x) {
+; CHECK-LABEL: 'startx'
+; CHECK-NEXT:  Determining loop execution counts for: @startx
+; CHECK-NEXT:  Loop %loop: backedge-taken count is (-1 + (-1 * %x) + ((1 + %x) smax %n))
+; CHECK-NEXT:  Loop %loop: max backedge-taken count is -1
+; CHECK-NEXT:  Loop %loop: Predicated backedge-taken count is (-1 + (-1 * %x) + ((1 + %x) smax %n))
+; CHECK-NEXT:   Predicates:
+; CHECK:       Loop %loop: Trip multiple is 1
+;
 entry:
   %s = icmp sgt i4 %n, 0
   br i1 %s, label %loop, label %exit
@@ -88,10 +110,13 @@ exit:
   ret void
 }
 
-; CHECK: Determining loop execution counts for: @startx_step2
-; CHECK: Loop %loop: Unpredictable backedge-taken count.
-; CHECK: Loop %loop: Unpredictable max backedge-taken count.
 define void @startx_step2(i4 %n, i4 %x) {
+; CHECK-LABEL: 'startx_step2'
+; CHECK-NEXT:  Determining loop execution counts for: @startx_step2
+; CHECK-NEXT:  Loop %loop: Unpredictable backedge-taken count.
+; CHECK-NEXT:  Loop %loop: Unpredictable max backedge-taken count.
+; CHECK-NEXT:  Loop %loop: Unpredictable predicated backedge-taken count.
+;
 entry:
   %s = icmp sgt i4 %n, 0
   br i1 %s, label %loop, label %exit
@@ -104,10 +129,15 @@ exit:
   ret void
 }
 
-; CHECK: Determining loop execution counts for: @nsw
-; CHECK: Loop %loop: backedge-taken count is (-1 + %n)
-; CHECK: Loop %loop: max backedge-taken count is 6
 define void @nsw(i4 %n) {
+; CHECK-LABEL: 'nsw'
+; CHECK-NEXT:  Determining loop execution counts for: @nsw
+; CHECK-NEXT:  Loop %loop: backedge-taken count is (-1 + %n)
+; CHECK-NEXT:  Loop %loop: max backedge-taken count is 6
+; CHECK-NEXT:  Loop %loop: Predicated backedge-taken count is (-1 + %n)
+; CHECK-NEXT:   Predicates:
+; CHECK:       Loop %loop: Trip multiple is 1
+;
 entry:
   %s = icmp sgt i4 %n, 0
   br i1 %s, label %loop, label %exit
@@ -128,11 +158,15 @@ exit:
 ; (b) the return instruction enacts a side effect that is control
 ; dependent on the poison value.
 ;
-; CHECK-LABEL: nsw_step2
-; CHECK: Determining loop execution counts for: @nsw_step2
-; CHECK: Loop %loop: backedge-taken count is ((-1 + %n) /u 2)
-; CHECK: Loop %loop: max backedge-taken count is 2
 define void @nsw_step2(i4 %n) {
+; CHECK-LABEL: 'nsw_step2'
+; CHECK-NEXT:  Determining loop execution counts for: @nsw_step2
+; CHECK-NEXT:  Loop %loop: backedge-taken count is ((-1 + %n) /u 2)
+; CHECK-NEXT:  Loop %loop: max backedge-taken count is 2
+; CHECK-NEXT:  Loop %loop: Predicated backedge-taken count is ((-1 + %n) /u 2)
+; CHECK-NEXT:   Predicates:
+; CHECK:       Loop %loop: Trip multiple is 1
+;
 entry:
   %s = icmp sgt i4 %n, 0
   br i1 %s, label %loop, label %exit
@@ -145,11 +179,15 @@ exit:
   ret void
 }
 
-; CHECK-LABEL: nsw_start1
-; CHECK: Determining loop execution counts for: @nsw_start1
-; CHECK: Loop %loop: backedge-taken count is (-2 + (2 smax %n))
-; CHECK: Loop %loop: max backedge-taken count is 5
 define void @nsw_start1(i4 %n) {
+; CHECK-LABEL: 'nsw_start1'
+; CHECK-NEXT:  Determining loop execution counts for: @nsw_start1
+; CHECK-NEXT:  Loop %loop: backedge-taken count is (-2 + (2 smax %n))<nsw>
+; CHECK-NEXT:  Loop %loop: max backedge-taken count is 5
+; CHECK-NEXT:  Loop %loop: Predicated backedge-taken count is (-2 + (2 smax %n))<nsw>
+; CHECK-NEXT:   Predicates:
+; CHECK:       Loop %loop: Trip multiple is 1
+;
 entry:
   %s = icmp sgt i4 %n, 0
   br i1 %s, label %loop, label %exit
@@ -162,10 +200,15 @@ exit:
   ret void
 }
 
-; CHECK: Determining loop execution counts for: @nsw_start1_step2
-; CHECK: Loop %loop: backedge-taken count is ((-2 + (3 smax %n)) /u 2)
-; CHECK: Loop %loop: max backedge-taken count is 2
 define void @nsw_start1_step2(i4 %n) {
+; CHECK-LABEL: 'nsw_start1_step2'
+; CHECK-NEXT:  Determining loop execution counts for: @nsw_start1_step2
+; CHECK-NEXT:  Loop %loop: backedge-taken count is ((-2 + (3 smax %n)) /u 2)
+; CHECK-NEXT:  Loop %loop: max backedge-taken count is 2
+; CHECK-NEXT:  Loop %loop: Predicated backedge-taken count is ((-2 + (3 smax %n)) /u 2)
+; CHECK-NEXT:   Predicates:
+; CHECK:       Loop %loop: Trip multiple is 1
+;
 entry:
   %s = icmp sgt i4 %n, 0
   br i1 %s, label %loop, label %exit
@@ -178,10 +221,15 @@ exit:
   ret void
 }
 
-; CHECK: Determining loop execution counts for: @nsw_startx
-; CHECK: Loop %loop: backedge-taken count is (-1 + (-1 * %x) + ((1 + %x)<nsw> smax %n))
-; CHECK: Loop %loop: max backedge-taken count is -1
 define void @nsw_startx(i4 %n, i4 %x) {
+; CHECK-LABEL: 'nsw_startx'
+; CHECK-NEXT:  Determining loop execution counts for: @nsw_startx
+; CHECK-NEXT:  Loop %loop: backedge-taken count is (-1 + (-1 * %x) + ((1 + %x) smax %n))
+; CHECK-NEXT:  Loop %loop: max backedge-taken count is -1
+; CHECK-NEXT:  Loop %loop: Predicated backedge-taken count is (-1 + (-1 * %x) + ((1 + %x) smax %n))
+; CHECK-NEXT:   Predicates:
+; CHECK:       Loop %loop: Trip multiple is 1
+;
 entry:
   %s = icmp sgt i4 %n, 0
   br i1 %s, label %loop, label %exit
@@ -194,10 +242,15 @@ exit:
   ret void
 }
 
-; CHECK: Determining loop execution counts for: @nsw_startx_step2
-; CHECK: Loop %loop: backedge-taken count is ((-1 + (-1 * %x) + ((2 + %x)<nsw> smax %n)) /u 2)
-; CHECK: Loop %loop: max backedge-taken count is 7
 define void @nsw_startx_step2(i4 %n, i4 %x) {
+; CHECK-LABEL: 'nsw_startx_step2'
+; CHECK-NEXT:  Determining loop execution counts for: @nsw_startx_step2
+; CHECK-NEXT:  Loop %loop: backedge-taken count is ((-1 + (-1 * %x) + ((2 + %x) smax %n)) /u 2)
+; CHECK-NEXT:  Loop %loop: max backedge-taken count is 7
+; CHECK-NEXT:  Loop %loop: Predicated backedge-taken count is ((-1 + (-1 * %x) + ((2 + %x) smax %n)) /u 2)
+; CHECK-NEXT:   Predicates:
+; CHECK:       Loop %loop: Trip multiple is 1
+;
 entry:
   %s = icmp sgt i4 %n, 0
   br i1 %s, label %loop, label %exit
@@ -210,10 +263,15 @@ exit:
   ret void
 }
 
-; CHECK: Determining loop execution counts for: @even
-; CHECK: Loop %loop: backedge-taken count is (-1 + (2 * %n))
-; CHECK: Loop %loop: max backedge-taken count is 5
 define void @even(i4 %n) {
+; CHECK-LABEL: 'even'
+; CHECK-NEXT:  Determining loop execution counts for: @even
+; CHECK-NEXT:  Loop %loop: backedge-taken count is (-1 + (2 * %n))
+; CHECK-NEXT:  Loop %loop: max backedge-taken count is 5
+; CHECK-NEXT:  Loop %loop: Predicated backedge-taken count is (-1 + (2 * %n))
+; CHECK-NEXT:   Predicates:
+; CHECK:       Loop %loop: Trip multiple is 2
+;
 entry:
   %m = shl i4 %n, 1
   %s = icmp sgt i4 %m, 0
@@ -227,10 +285,15 @@ exit:
   ret void
 }
 
-; CHECK: Determining loop execution counts for: @even_step2
-; CHECK: Loop %loop: backedge-taken count is ((-1 + (2 * %n)) /u 2)
-; CHECK: Loop %loop: max backedge-taken count is 2
 define void @even_step2(i4 %n) {
+; CHECK-LABEL: 'even_step2'
+; CHECK-NEXT:  Determining loop execution counts for: @even_step2
+; CHECK-NEXT:  Loop %loop: backedge-taken count is ((-1 + (2 * %n)) /u 2)
+; CHECK-NEXT:  Loop %loop: max backedge-taken count is 2
+; CHECK-NEXT:  Loop %loop: Predicated backedge-taken count is ((-1 + (2 * %n)) /u 2)
+; CHECK-NEXT:   Predicates:
+; CHECK:       Loop %loop: Trip multiple is 1
+;
 entry:
   %m = shl i4 %n, 1
   %s = icmp sgt i4 %m, 0
@@ -244,10 +307,15 @@ exit:
   ret void
 }
 
-; CHECK: Determining loop execution counts for: @even_start1
-; CHECK: Loop %loop: backedge-taken count is (-2 + (2 smax (2 * %n)))
-; CHECK: Loop %loop: max backedge-taken count is 4
 define void @even_start1(i4 %n) {
+; CHECK-LABEL: 'even_start1'
+; CHECK-NEXT:  Determining loop execution counts for: @even_start1
+; CHECK-NEXT:  Loop %loop: backedge-taken count is (-2 + (2 smax (2 * %n)))<nsw>
+; CHECK-NEXT:  Loop %loop: max backedge-taken count is 4
+; CHECK-NEXT:  Loop %loop: Predicated backedge-taken count is (-2 + (2 smax (2 * %n)))<nsw>
+; CHECK-NEXT:   Predicates:
+; CHECK:       Loop %loop: Trip multiple is 1
+;
 entry:
   %m = shl i4 %n, 1
   %s = icmp sgt i4 %m, 0
@@ -261,10 +329,15 @@ exit:
   ret void
 }
 
-; CHECK: Determining loop execution counts for: @even_start1_step2
-; CHECK: Loop %loop: backedge-taken count is ((-2 + (3 smax (2 * %n))) /u 2)
-; CHECK: Loop %loop: max backedge-taken count is 2
 define void @even_start1_step2(i4 %n) {
+; CHECK-LABEL: 'even_start1_step2'
+; CHECK-NEXT:  Determining loop execution counts for: @even_start1_step2
+; CHECK-NEXT:  Loop %loop: backedge-taken count is ((-2 + (3 smax (2 * %n))) /u 2)
+; CHECK-NEXT:  Loop %loop: max backedge-taken count is 2
+; CHECK-NEXT:  Loop %loop: Predicated backedge-taken count is ((-2 + (3 smax (2 * %n))) /u 2)
+; CHECK-NEXT:   Predicates:
+; CHECK:       Loop %loop: Trip multiple is 1
+;
 entry:
   %m = shl i4 %n, 1
   %s = icmp sgt i4 %m, 0
@@ -278,10 +351,15 @@ exit:
   ret void
 }
 
-; CHECK: Determining loop execution counts for: @even_startx
-; CHECK: Loop %loop: backedge-taken count is (-1 + (-1 * %x) + ((1 + %x) smax (2 * %n)))
-; CHECK: Loop %loop: max backedge-taken count is -2
 define void @even_startx(i4 %n, i4 %x) {
+; CHECK-LABEL: 'even_startx'
+; CHECK-NEXT:  Determining loop execution counts for: @even_startx
+; CHECK-NEXT:  Loop %loop: backedge-taken count is (-1 + (-1 * %x) + ((1 + %x) smax (2 * %n)))
+; CHECK-NEXT:  Loop %loop: max backedge-taken count is -2
+; CHECK-NEXT:  Loop %loop: Predicated backedge-taken count is (-1 + (-1 * %x) + ((1 + %x) smax (2 * %n)))
+; CHECK-NEXT:   Predicates:
+; CHECK:       Loop %loop: Trip multiple is 1
+;
 entry:
   %m = shl i4 %n, 1
   %s = icmp sgt i4 %m, 0
@@ -295,10 +373,15 @@ exit:
   ret void
 }
 
-; CHECK: Determining loop execution counts for: @even_startx_step2
-; CHECK: Loop %loop: backedge-taken count is ((-1 + (-1 * %x) + ((2 + %x) smax (2 * %n))) /u 2)
-; CHECK: Loop %loop: max backedge-taken count is 7
 define void @even_startx_step2(i4 %n, i4 %x) {
+; CHECK-LABEL: 'even_startx_step2'
+; CHECK-NEXT:  Determining loop execution counts for: @even_startx_step2
+; CHECK-NEXT:  Loop %loop: backedge-taken count is ((-1 + (-1 * %x) + ((2 + %x) smax (2 * %n))) /u 2)
+; CHECK-NEXT:  Loop %loop: max backedge-taken count is 7
+; CHECK-NEXT:  Loop %loop: Predicated backedge-taken count is ((-1 + (-1 * %x) + ((2 + %x) smax (2 * %n))) /u 2)
+; CHECK-NEXT:   Predicates:
+; CHECK:       Loop %loop: Trip multiple is 1
+;
 entry:
   %m = shl i4 %n, 1
   %s = icmp sgt i4 %m, 0
@@ -312,10 +395,15 @@ exit:
   ret void
 }
 
-; CHECK: Determining loop execution counts for: @even_nsw
-; CHECK: Loop %loop: backedge-taken count is (-1 + (2 * %n))
-; CHECK: Loop %loop: max backedge-taken count is 5
 define void @even_nsw(i4 %n) {
+; CHECK-LABEL: 'even_nsw'
+; CHECK-NEXT:  Determining loop execution counts for: @even_nsw
+; CHECK-NEXT:  Loop %loop: backedge-taken count is (-1 + (2 * %n))
+; CHECK-NEXT:  Loop %loop: max backedge-taken count is 5
+; CHECK-NEXT:  Loop %loop: Predicated backedge-taken count is (-1 + (2 * %n))
+; CHECK-NEXT:   Predicates:
+; CHECK:       Loop %loop: Trip multiple is 2
+;
 entry:
   %m = shl i4 %n, 1
   %s = icmp sgt i4 %m, 0
@@ -329,10 +417,15 @@ exit:
   ret void
 }
 
-; CHECK: Determining loop execution counts for: @even_nsw_step2
-; CHECK: Loop %loop: backedge-taken count is ((-1 + (2 * %n)) /u 2)
-; CHECK: Loop %loop: max backedge-taken count is 2
 define void @even_nsw_step2(i4 %n) {
+; CHECK-LABEL: 'even_nsw_step2'
+; CHECK-NEXT:  Determining loop execution counts for: @even_nsw_step2
+; CHECK-NEXT:  Loop %loop: backedge-taken count is ((-1 + (2 * %n)) /u 2)
+; CHECK-NEXT:  Loop %loop: max backedge-taken count is 2
+; CHECK-NEXT:  Loop %loop: Predicated backedge-taken count is ((-1 + (2 * %n)) /u 2)
+; CHECK-NEXT:   Predicates:
+; CHECK:       Loop %loop: Trip multiple is 1
+;
 entry:
   %m = shl i4 %n, 1
   %s = icmp sgt i4 %m, 0
@@ -346,10 +439,15 @@ exit:
   ret void
 }
 
-; CHECK: Determining loop execution counts for: @even_nsw_start1
-; CHECK: Loop %loop: backedge-taken count is (-2 + (2 smax (2 * %n)))
-; CHECK: Loop %loop: max backedge-taken count is 4
 define void @even_nsw_start1(i4 %n) {
+; CHECK-LABEL: 'even_nsw_start1'
+; CHECK-NEXT:  Determining loop execution counts for: @even_nsw_start1
+; CHECK-NEXT:  Loop %loop: backedge-taken count is (-2 + (2 smax (2 * %n)))<nsw>
+; CHECK-NEXT:  Loop %loop: max backedge-taken count is 4
+; CHECK-NEXT:  Loop %loop: Predicated backedge-taken count is (-2 + (2 smax (2 * %n)))<nsw>
+; CHECK-NEXT:   Predicates:
+; CHECK:       Loop %loop: Trip multiple is 1
+;
 entry:
   %m = shl i4 %n, 1
   %s = icmp sgt i4 %m, 0
@@ -363,10 +461,15 @@ exit:
   ret void
 }
 
-; CHECK: Determining loop execution counts for: @even_nsw_start1_step2
-; CHECK: Loop %loop: backedge-taken count is ((-2 + (3 smax (2 * %n))) /u 2)
-; CHECK: Loop %loop: max backedge-taken count is 2
 define void @even_nsw_start1_step2(i4 %n) {
+; CHECK-LABEL: 'even_nsw_start1_step2'
+; CHECK-NEXT:  Determining loop execution counts for: @even_nsw_start1_step2
+; CHECK-NEXT:  Loop %loop: backedge-taken count is ((-2 + (3 smax (2 * %n))) /u 2)
+; CHECK-NEXT:  Loop %loop: max backedge-taken count is 2
+; CHECK-NEXT:  Loop %loop: Predicated backedge-taken count is ((-2 + (3 smax (2 * %n))) /u 2)
+; CHECK-NEXT:   Predicates:
+; CHECK:       Loop %loop: Trip multiple is 1
+;
 entry:
   %m = shl i4 %n, 1
   %s = icmp sgt i4 %m, 0
@@ -380,10 +483,15 @@ exit:
   ret void
 }
 
-; CHECK: Determining loop execution counts for: @even_nsw_startx
-; CHECK: Loop %loop: backedge-taken count is (-1 + (-1 * %x) + ((1 + %x)<nsw> smax (2 * %n)))
-; CHECK: Loop %loop: max backedge-taken count is -2
 define void @even_nsw_startx(i4 %n, i4 %x) {
+; CHECK-LABEL: 'even_nsw_startx'
+; CHECK-NEXT:  Determining loop execution counts for: @even_nsw_startx
+; CHECK-NEXT:  Loop %loop: backedge-taken count is (-1 + (-1 * %x) + ((1 + %x) smax (2 * %n)))
+; CHECK-NEXT:  Loop %loop: max backedge-taken count is -2
+; CHECK-NEXT:  Loop %loop: Predicated backedge-taken count is (-1 + (-1 * %x) + ((1 + %x) smax (2 * %n)))
+; CHECK-NEXT:   Predicates:
+; CHECK:       Loop %loop: Trip multiple is 1
+;
 entry:
   %m = shl i4 %n, 1
   %s = icmp sgt i4 %m, 0
@@ -397,10 +505,15 @@ exit:
   ret void
 }
 
-; CHECK: Determining loop execution counts for: @even_nsw_startx_step2
-; CHECK: Loop %loop: backedge-taken count is ((-1 + (-1 * %x) + ((2 + %x)<nsw> smax (2 * %n))) /u 2)
-; CHECK: Loop %loop: max backedge-taken count is 7
 define void @even_nsw_startx_step2(i4 %n, i4 %x) {
+; CHECK-LABEL: 'even_nsw_startx_step2'
+; CHECK-NEXT:  Determining loop execution counts for: @even_nsw_startx_step2
+; CHECK-NEXT:  Loop %loop: backedge-taken count is ((-1 + (-1 * %x) + ((2 + %x) smax (2 * %n))) /u 2)
+; CHECK-NEXT:  Loop %loop: max backedge-taken count is 7
+; CHECK-NEXT:  Loop %loop: Predicated backedge-taken count is ((-1 + (-1 * %x) + ((2 + %x) smax (2 * %n))) /u 2)
+; CHECK-NEXT:   Predicates:
+; CHECK:       Loop %loop: Trip multiple is 1
+;
 entry:
   %m = shl i4 %n, 1
   %s = icmp sgt i4 %m, 0
diff --git a/llvm/test/Analysis/ScalarEvolution/umin-umax-folds.ll b/llvm/test/Analysis/ScalarEvolution/umin-umax-folds.ll
new file mode 100644
index 000000000..fb8f59fe4
--- /dev/null
+++ b/llvm/test/Analysis/ScalarEvolution/umin-umax-folds.ll
@@ -0,0 +1,317 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt -analyze -scalar-evolution < %s | FileCheck %s
+
+define void @umin_sext_x_zext_x(i32 %len) {
+; CHECK-LABEL: 'umin_sext_x_zext_x'
+; CHECK-NEXT:  Classifying expressions for: @umin_sext_x_zext_x
+; CHECK-NEXT:    %len.zext = zext i32 %len to i64
+; CHECK-NEXT:    --> (zext i32 %len to i64) U: [0,4294967296) S: [0,4294967296)
+; CHECK-NEXT:    %len.sext = sext i32 %len to i64
+; CHECK-NEXT:    --> (sext i32 %len to i64) U: [-2147483648,2147483648) S: [-2147483648,2147483648)
+; CHECK-NEXT:    %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+; CHECK-NEXT:    --> {0,+,1}<%loop> U: [0,4294967296) S: [0,4294967296) Exits: (zext i32 %len to i64) LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:    %iv.next = add i64 %iv, 1
+; CHECK-NEXT:    --> {1,+,1}<%loop> U: [1,4294967297) S: [1,4294967297) Exits: (1 + (zext i32 %len to i64))<nuw><nsw> LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:    %and = and i1 %cmp1, %cmp2
+; CHECK-NEXT:    --> %and U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %loop: Variant }
+; CHECK-NEXT:  Determining loop execution counts for: @umin_sext_x_zext_x
+; CHECK-NEXT:  Loop %loop: backedge-taken count is (zext i32 %len to i64)
+; CHECK-NEXT:  Loop %loop: max backedge-taken count is 4294967295
+; CHECK-NEXT:  Loop %loop: Predicated backedge-taken count is (zext i32 %len to i64)
+; CHECK-NEXT:   Predicates:
+; CHECK:       Loop %loop: Trip multiple is 1
+;
+entry:
+  %len.zext = zext i32 %len to i64
+  %len.sext = sext i32 %len to i64
+  br label %loop
+loop:
+  %iv = phi i64 [0, %entry], [%iv.next, %loop]
+  %iv.next = add i64 %iv, 1
+  %cmp1 = icmp ult i64 %iv, %len.zext
+  %cmp2 = icmp ult i64 %iv, %len.sext
+  %and = and i1 %cmp1, %cmp2
+  br i1 %and, label %loop, label %exit
+exit:
+  ret void
+}
+
+define void @ule_sext_x_zext_x(i32 %len) {
+; CHECK-LABEL: 'ule_sext_x_zext_x'
+; CHECK-NEXT:  Classifying expressions for: @ule_sext_x_zext_x
+; CHECK-NEXT:    %len.zext = zext i32 %len to i64
+; CHECK-NEXT:    --> (zext i32 %len to i64) U: [0,4294967296) S: [0,4294967296)
+; CHECK-NEXT:    %len.sext = sext i32 %len to i64
+; CHECK-NEXT:    --> (sext i32 %len to i64) U: [-2147483648,2147483648) S: [-2147483648,2147483648)
+; CHECK-NEXT:    %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+; CHECK-NEXT:    --> {0,+,1}<%loop> U: [0,4294967296) S: [0,4294967296) Exits: (zext i32 %len to i64) LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:    %iv.next = add i64 %iv, 1
+; CHECK-NEXT:    --> {1,+,1}<%loop> U: [1,4294967297) S: [1,4294967297) Exits: (1 + (zext i32 %len to i64))<nuw><nsw> LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:    %sel = select i1 %cmp1, i64 %len.zext, i64 %len.sext
+; CHECK-NEXT:    --> (zext i32 %len to i64) U: [0,4294967296) S: [0,4294967296) Exits: (zext i32 %len to i64) LoopDispositions: { %loop: Invariant }
+; CHECK-NEXT:  Determining loop execution counts for: @ule_sext_x_zext_x
+; CHECK-NEXT:  Loop %loop: backedge-taken count is (zext i32 %len to i64)
+; CHECK-NEXT:  Loop %loop: max backedge-taken count is 4294967295
+; CHECK-NEXT:  Loop %loop: Predicated backedge-taken count is (zext i32 %len to i64)
+; CHECK-NEXT:   Predicates:
+; CHECK:       Loop %loop: Trip multiple is 1
+;
+entry:
+  %len.zext = zext i32 %len to i64
+  %len.sext = sext i32 %len to i64
+  br label %loop
+loop:
+  %iv = phi i64 [0, %entry], [%iv.next, %loop]
+  %iv.next = add i64 %iv, 1
+  %cmp1 = icmp ule i64 %len.zext, %len.sext
+  %sel = select i1 %cmp1, i64 %len.zext, i64 %len.sext
+  %cmp2 = icmp ult i64 %iv, %sel
+  br i1 %cmp2, label %loop, label %exit
+exit:
+  ret void
+}
+
+define void @uge_sext_x_zext_x(i32 %len) {
+; CHECK-LABEL: 'uge_sext_x_zext_x'
+; CHECK-NEXT:  Classifying expressions for: @uge_sext_x_zext_x
+; CHECK-NEXT:    %len.zext = zext i32 %len to i64
+; CHECK-NEXT:    --> (zext i32 %len to i64) U: [0,4294967296) S: [0,4294967296)
+; CHECK-NEXT:    %len.sext = sext i32 %len to i64
+; CHECK-NEXT:    --> (sext i32 %len to i64) U: [-2147483648,2147483648) S: [-2147483648,2147483648)
+; CHECK-NEXT:    %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+; CHECK-NEXT:    --> {0,+,1}<%loop> U: full-set S: full-set Exits: (sext i32 %len to i64) LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:    %iv.next = add i64 %iv, 1
+; CHECK-NEXT:    --> {1,+,1}<%loop> U: full-set S: full-set Exits: (1 + (sext i32 %len to i64))<nsw> LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:    %sel = select i1 %cmp1, i64 %len.zext, i64 %len.sext
+; CHECK-NEXT:    --> (sext i32 %len to i64) U: [-2147483648,2147483648) S: [-2147483648,2147483648) Exits: (sext i32 %len to i64) LoopDispositions: { %loop: Invariant }
+; CHECK-NEXT:  Determining loop execution counts for: @uge_sext_x_zext_x
+; CHECK-NEXT:  Loop %loop: backedge-taken count is (sext i32 %len to i64)
+; CHECK-NEXT:  Loop %loop: max backedge-taken count is -1
+; CHECK-NEXT:  Loop %loop: Predicated backedge-taken count is (sext i32 %len to i64)
+; CHECK-NEXT:   Predicates:
+; CHECK:       Loop %loop: Trip multiple is 1
+;
+entry:
+  %len.zext = zext i32 %len to i64
+  %len.sext = sext i32 %len to i64
+  br label %loop
+loop:
+  %iv = phi i64 [0, %entry], [%iv.next, %loop]
+  %iv.next = add i64 %iv, 1
+  %cmp1 = icmp uge i64 %len.zext, %len.sext
+  %sel = select i1 %cmp1, i64 %len.zext, i64 %len.sext
+  %cmp2 = icmp ult i64 %iv, %sel
+  br i1 %cmp2, label %loop, label %exit
+exit:
+  ret void
+}
+
+define void @ult_sext_x_zext_x(i32 %len) {
+; CHECK-LABEL: 'ult_sext_x_zext_x'
+; CHECK-NEXT:  Classifying expressions for: @ult_sext_x_zext_x
+; CHECK-NEXT:    %len.zext = zext i32 %len to i64
+; CHECK-NEXT:    --> (zext i32 %len to i64) U: [0,4294967296) S: [0,4294967296)
+; CHECK-NEXT:    %len.sext = sext i32 %len to i64
+; CHECK-NEXT:    --> (sext i32 %len to i64) U: [-2147483648,2147483648) S: [-2147483648,2147483648)
+; CHECK-NEXT:    %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+; CHECK-NEXT:    --> {0,+,1}<%loop> U: [0,4294967296) S: [0,4294967296) Exits: (zext i32 %len to i64) LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:    %iv.next = add i64 %iv, 1
+; CHECK-NEXT:    --> {1,+,1}<%loop> U: [1,4294967297) S: [1,4294967297) Exits: (1 + (zext i32 %len to i64))<nuw><nsw> LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:    %umin = select i1 %cmp1, i64 %len.zext, i64 %len.sext
+; CHECK-NEXT:    --> (zext i32 %len to i64) U: [0,4294967296) S: [0,4294967296) Exits: (zext i32 %len to i64) LoopDispositions: { %loop: Invariant }
+; CHECK-NEXT:  Determining loop execution counts for: @ult_sext_x_zext_x
+; CHECK-NEXT:  Loop %loop: backedge-taken count is (zext i32 %len to i64)
+; CHECK-NEXT:  Loop %loop: max backedge-taken count is 4294967295
+; CHECK-NEXT:  Loop %loop: Predicated backedge-taken count is (zext i32 %len to i64)
+; CHECK-NEXT:   Predicates:
+; CHECK:       Loop %loop: Trip multiple is 1
+;
+entry:
+  %len.zext = zext i32 %len to i64
+  %len.sext = sext i32 %len to i64
+  br label %loop
+loop:
+  %iv = phi i64 [0, %entry], [%iv.next, %loop]
+  %iv.next = add i64 %iv, 1
+  %cmp1 = icmp ult i64 %len.zext, %len.sext
+  %umin = select i1 %cmp1, i64 %len.zext, i64 %len.sext
+  %cmp2 = icmp ult i64 %iv, %umin
+  br i1 %cmp2, label %loop, label %exit
+exit:
+  ret void
+}
+
+define void @ugt_sext_x_zext_x(i32 %len) {
+; CHECK-LABEL: 'ugt_sext_x_zext_x'
+; CHECK-NEXT:  Classifying expressions for: @ugt_sext_x_zext_x
+; CHECK-NEXT:    %len.zext = zext i32 %len to i64
+; CHECK-NEXT:    --> (zext i32 %len to i64) U: [0,4294967296) S: [0,4294967296)
+; CHECK-NEXT:    %len.sext = sext i32 %len to i64
+; CHECK-NEXT:    --> (sext i32 %len to i64) U: [-2147483648,2147483648) S: [-2147483648,2147483648)
+; CHECK-NEXT:    %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+; CHECK-NEXT:    --> {0,+,1}<%loop> U: full-set S: full-set Exits: (sext i32 %len to i64) LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:    %iv.next = add i64 %iv, 1
+; CHECK-NEXT:    --> {1,+,1}<%loop> U: full-set S: full-set Exits: (1 + (sext i32 %len to i64))<nsw> LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:    %umax = select i1 %cmp1, i64 %len.zext, i64 %len.sext
+; CHECK-NEXT:    --> (sext i32 %len to i64) U: [-2147483648,2147483648) S: [-2147483648,2147483648) Exits: (sext i32 %len to i64) LoopDispositions: { %loop: Invariant }
+; CHECK-NEXT:  Determining loop execution counts for: @ugt_sext_x_zext_x
+; CHECK-NEXT:  Loop %loop: backedge-taken count is (sext i32 %len to i64)
+; CHECK-NEXT:  Loop %loop: max backedge-taken count is -1
+; CHECK-NEXT:  Loop %loop: Predicated backedge-taken count is (sext i32 %len to i64)
+; CHECK-NEXT:   Predicates:
+; CHECK:       Loop %loop: Trip multiple is 1
+;
+entry:
+  %len.zext = zext i32 %len to i64
+  %len.sext = sext i32 %len to i64
+  br label %loop
+loop:
+  %iv = phi i64 [0, %entry], [%iv.next, %loop]
+  %iv.next = add i64 %iv, 1
+  %cmp1 = icmp ugt i64 %len.zext, %len.sext
+  %umax = select i1 %cmp1, i64 %len.zext, i64 %len.sext
+  %cmp2 = icmp ult i64 %iv, %umax
+  br i1 %cmp2, label %loop, label %exit
+exit:
+  ret void
+}
+
+define void @sle_sext_x_zext_x(i32 %len) {
+; CHECK-LABEL: 'sle_sext_x_zext_x'
+; CHECK-NEXT:  Classifying expressions for: @sle_sext_x_zext_x
+; CHECK-NEXT:    %len.zext = zext i32 %len to i64
+; CHECK-NEXT:    --> (zext i32 %len to i64) U: [0,4294967296) S: [0,4294967296)
+; CHECK-NEXT:    %len.sext = sext i32 %len to i64
+; CHECK-NEXT:    --> (sext i32 %len to i64) U: [-2147483648,2147483648) S: [-2147483648,2147483648)
+; CHECK-NEXT:    %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+; CHECK-NEXT:    --> {0,+,1}<%loop> U: [0,4294967296) S: [0,4294967296) Exits: (zext i32 %len to i64) LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:    %iv.next = add i64 %iv, 1
+; CHECK-NEXT:    --> {1,+,1}<%loop> U: [1,4294967297) S: [1,4294967297) Exits: (1 + (zext i32 %len to i64))<nuw><nsw> LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:    %sel = select i1 %cmp1, i64 %len.zext, i64 %len.sext
+; CHECK-NEXT:    --> (zext i32 %len to i64) U: [0,4294967296) S: [0,4294967296) Exits: (zext i32 %len to i64) LoopDispositions: { %loop: Invariant }
+; CHECK-NEXT:  Determining loop execution counts for: @sle_sext_x_zext_x
+; CHECK-NEXT:  Loop %loop: backedge-taken count is (zext i32 %len to i64)
+; CHECK-NEXT:  Loop %loop: max backedge-taken count is 4294967295
+; CHECK-NEXT:  Loop %loop: Predicated backedge-taken count is (zext i32 %len to i64)
+; CHECK-NEXT:   Predicates:
+; CHECK:       Loop %loop: Trip multiple is 1
+;
+entry:
+  %len.zext = zext i32 %len to i64
+  %len.sext = sext i32 %len to i64
+  br label %loop
+loop:
+  %iv = phi i64 [0, %entry], [%iv.next, %loop]
+  %iv.next = add i64 %iv, 1
+  %cmp1 = icmp ule i64 %len.zext, %len.sext
+  %sel = select i1 %cmp1, i64 %len.zext, i64 %len.sext
+  %cmp2 = icmp ult i64 %iv, %sel
+  br i1 %cmp2, label %loop, label %exit
+exit:
+  ret void
+}
+
+define void @sge_sext_x_zext_x(i32 %len) {
+; CHECK-LABEL: 'sge_sext_x_zext_x'
+; CHECK-NEXT:  Classifying expressions for: @sge_sext_x_zext_x
+; CHECK-NEXT:    %len.zext = zext i32 %len to i64
+; CHECK-NEXT:    --> (zext i32 %len to i64) U: [0,4294967296) S: [0,4294967296)
+; CHECK-NEXT:    %len.sext = sext i32 %len to i64
+; CHECK-NEXT:    --> (sext i32 %len to i64) U: [-2147483648,2147483648) S: [-2147483648,2147483648)
+; CHECK-NEXT:    %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+; CHECK-NEXT:    --> {0,+,1}<%loop> U: [0,4294967296) S: [0,4294967296) Exits: (zext i32 %len to i64) LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:    %iv.next = add i64 %iv, 1
+; CHECK-NEXT:    --> {1,+,1}<%loop> U: [1,4294967297) S: [1,4294967297) Exits: (1 + (zext i32 %len to i64))<nuw><nsw> LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:    %sel = select i1 %cmp1, i64 %len.zext, i64 %len.sext
+; CHECK-NEXT:    --> (zext i32 %len to i64) U: [0,4294967296) S: [0,4294967296) Exits: (zext i32 %len to i64) LoopDispositions: { %loop: Invariant }
+; CHECK-NEXT:  Determining loop execution counts for: @sge_sext_x_zext_x
+; CHECK-NEXT:  Loop %loop: backedge-taken count is (zext i32 %len to i64)
+; CHECK-NEXT:  Loop %loop: max backedge-taken count is 4294967295
+; CHECK-NEXT:  Loop %loop: Predicated backedge-taken count is (zext i32 %len to i64)
+; CHECK-NEXT:   Predicates:
+; CHECK:       Loop %loop: Trip multiple is 1
+;
+entry:
+  %len.zext = zext i32 %len to i64
+  %len.sext = sext i32 %len to i64
+  br label %loop
+loop:
+  %iv = phi i64 [0, %entry], [%iv.next, %loop]
+  %iv.next = add i64 %iv, 1
+  %cmp1 = icmp sge i64 %len.zext, %len.sext
+  %sel = select i1 %cmp1, i64 %len.zext, i64 %len.sext
+  %cmp2 = icmp ult i64 %iv, %sel
+  br i1 %cmp2, label %loop, label %exit
+exit:
+  ret void
+}
+
+define void @slt_sext_x_zext_x(i32 %len) {
+; CHECK-LABEL: 'slt_sext_x_zext_x'
+; CHECK-NEXT:  Classifying expressions for: @slt_sext_x_zext_x
+; CHECK-NEXT:    %len.zext = zext i32 %len to i64
+; CHECK-NEXT:    --> (zext i32 %len to i64) U: [0,4294967296) S: [0,4294967296)
+; CHECK-NEXT:    %len.sext = sext i32 %len to i64
+; CHECK-NEXT:    --> (sext i32 %len to i64) U: [-2147483648,2147483648) S: [-2147483648,2147483648)
+; CHECK-NEXT:    %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+; CHECK-NEXT:    --> {0,+,1}<%loop> U: full-set S: full-set Exits: (sext i32 %len to i64) LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:    %iv.next = add i64 %iv, 1
+; CHECK-NEXT:    --> {1,+,1}<%loop> U: full-set S: full-set Exits: (1 + (sext i32 %len to i64))<nsw> LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:    %umin = select i1 %cmp1, i64 %len.zext, i64 %len.sext
+; CHECK-NEXT:    --> (sext i32 %len to i64) U: [-2147483648,2147483648) S: [-2147483648,2147483648) Exits: (sext i32 %len to i64) LoopDispositions: { %loop: Invariant }
+; CHECK-NEXT:  Determining loop execution counts for: @slt_sext_x_zext_x
+; CHECK-NEXT:  Loop %loop: backedge-taken count is (sext i32 %len to i64)
+; CHECK-NEXT:  Loop %loop: max backedge-taken count is -1
+; CHECK-NEXT:  Loop %loop: Predicated backedge-taken count is (sext i32 %len to i64)
+; CHECK-NEXT:   Predicates:
+; CHECK:       Loop %loop: Trip multiple is 1
+;
+entry:
+  %len.zext = zext i32 %len to i64
+  %len.sext = sext i32 %len to i64
+  br label %loop
+loop:
+  %iv = phi i64 [0, %entry], [%iv.next, %loop]
+  %iv.next = add i64 %iv, 1
+  %cmp1 = icmp slt i64 %len.zext, %len.sext
+  %umin = select i1 %cmp1, i64 %len.zext, i64 %len.sext
+  %cmp2 = icmp ult i64 %iv, %umin
+  br i1 %cmp2, label %loop, label %exit
+exit:
+  ret void
+}
+
+define void @sgt_sext_x_zext_x(i32 %len) {
+; CHECK-LABEL: 'sgt_sext_x_zext_x'
+; CHECK-NEXT:  Classifying expressions for: @sgt_sext_x_zext_x
+; CHECK-NEXT:    %len.zext = zext i32 %len to i64
+; CHECK-NEXT:    --> (zext i32 %len to i64) U: [0,4294967296) S: [0,4294967296)
+; CHECK-NEXT:    %len.sext = sext i32 %len to i64
+; CHECK-NEXT:    --> (sext i32 %len to i64) U: [-2147483648,2147483648) S: [-2147483648,2147483648)
+; CHECK-NEXT:    %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+; CHECK-NEXT:    --> {0,+,1}<%loop> U: [0,4294967296) S: [0,4294967296) Exits: (zext i32 %len to i64) LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:    %iv.next = add i64 %iv, 1
+; CHECK-NEXT:    --> {1,+,1}<%loop> U: [1,4294967297) S: [1,4294967297) Exits: (1 + (zext i32 %len to i64))<nuw><nsw> LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:    %umax = select i1 %cmp1, i64 %len.zext, i64 %len.sext
+; CHECK-NEXT:    --> (zext i32 %len to i64) U: [0,4294967296) S: [0,4294967296) Exits: (zext i32 %len to i64) LoopDispositions: { %loop: Invariant }
+; CHECK-NEXT:  Determining loop execution counts for: @sgt_sext_x_zext_x
+; CHECK-NEXT:  Loop %loop: backedge-taken count is (zext i32 %len to i64)
+; CHECK-NEXT:  Loop %loop: max backedge-taken count is 4294967295
+; CHECK-NEXT:  Loop %loop: Predicated backedge-taken count is (zext i32 %len to i64)
+; CHECK-NEXT:   Predicates:
+; CHECK:       Loop %loop: Trip multiple is 1
+;
+entry:
+  %len.zext = zext i32 %len to i64
+  %len.sext = sext i32 %len to i64
+  br label %loop
+loop:
+  %iv = phi i64 [0, %entry], [%iv.next, %loop]
+  %iv.next = add i64 %iv, 1
+  %cmp1 = icmp sgt i64 %len.zext, %len.sext
+  %umax = select i1 %cmp1, i64 %len.zext, i64 %len.sext
+  %cmp2 = icmp ult i64 %iv, %umax
+  br i1 %cmp2, label %loop, label %exit
+exit:
+  ret void
+}
diff --git a/llvm/test/Analysis/ScalarEvolution/widenable-condition.ll b/llvm/test/Analysis/ScalarEvolution/widenable-condition.ll
new file mode 100644
index 000000000..b7b5f7154
--- /dev/null
+++ b/llvm/test/Analysis/ScalarEvolution/widenable-condition.ll
@@ -0,0 +1,45 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt -analyze -scalar-evolution < %s | FileCheck %s
+
+; The semanics of this example are a bit subtle.  The loop is required
+; execute some number of times up to 1999.  The compiler is free to reduce
+; the number of said iterations to zero (or any value in between) if desired,
+; but if it does so, the return value and the last value stored to G must
+; agree.  For SCEV, this translates as widenable conditions preventing exact
+; exit counts from being computed, but not restricting max exit counts.
+; It's tempting to say that SCEV should return a precise exit count here, but
+; would result in miscompiles if transformations such as RLEV ran before
+; widening of the WC.
+define i32 @wc_max() {
+; CHECK-LABEL: 'wc_max'
+; CHECK-NEXT:  Classifying expressions for: @wc_max
+; CHECK-NEXT:    %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+; CHECK-NEXT:    --> {0,+,1}<%loop> U: [0,2000) S: [0,2000) Exits: <<Unknown>> LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:    %iv.next = add i32 %iv, 1
+; CHECK-NEXT:    --> {1,+,1}<%loop> U: [1,2001) S: [1,2001) Exits: <<Unknown>> LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:    %widenable_cond3 = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    --> %widenable_cond3 U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %loop: Variant }
+; CHECK-NEXT:    %exiplicit_guard_cond4 = and i1 %cond_1, %widenable_cond3
+; CHECK-NEXT:    --> %exiplicit_guard_cond4 U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %loop: Variant }
+; CHECK-NEXT:  Determining loop execution counts for: @wc_max
+; CHECK-NEXT:  Loop %loop: Unpredictable backedge-taken count.
+; CHECK-NEXT:  Loop %loop: max backedge-taken count is 1999
+; CHECK-NEXT:  Loop %loop: Unpredictable predicated backedge-taken count.
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i32 [0, %entry], [%iv.next, %loop]
+  %iv.next = add i32 %iv, 1
+  store i32 %iv, i32 *@G
+  %cond_1 = icmp slt i32 %iv.next, 2000
+  %widenable_cond3 = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond4 = and i1 %cond_1, %widenable_cond3
+  br i1 %exiplicit_guard_cond4, label %loop, label %exit
+
+exit:
+  ret i32 %iv
+}
+
+@G = external global i32
+declare i1 @llvm.experimental.widenable.condition()
diff --git a/llvm/test/Analysis/TypeBasedAliasAnalysis/PR17620.ll b/llvm/test/Analysis/TypeBasedAliasAnalysis/PR17620.ll
index e78529c07..3e1bf5c47 100644
--- a/llvm/test/Analysis/TypeBasedAliasAnalysis/PR17620.ll
+++ b/llvm/test/Analysis/TypeBasedAliasAnalysis/PR17620.ll
@@ -27,8 +27,8 @@ entry:
 
 declare void @callee(i32*, i32*) #1
 
-attributes #0 = { ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { ssp uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.ident = !{!0}
 
diff --git a/llvm/test/Analysis/TypeBasedAliasAnalysis/functionattrs.ll b/llvm/test/Analysis/TypeBasedAliasAnalysis/functionattrs.ll
index 63df6bd84..28dc4342d 100644
--- a/llvm/test/Analysis/TypeBasedAliasAnalysis/functionattrs.ll
+++ b/llvm/test/Analysis/TypeBasedAliasAnalysis/functionattrs.ll
@@ -78,7 +78,7 @@ declare void @llvm.memcpy.p0i8.p0i8.i64(i8*, i8*, i64, i1) nounwind
 ; CHECK: attributes #3 = { nounwind }
 ; CHECK: attributes #4 = { nounwind readnone }
 ; CHECK: attributes #5 = { nofree norecurse nounwind }
-; CHECK: attributes #6 = { argmemonly nounwind }
+; CHECK: attributes #6 = { argmemonly nounwind willreturn }
 
 ; Root note.
 !0 = !{ }
diff --git a/llvm/test/Analysis/TypeBasedAliasAnalysis/intrinsics.ll b/llvm/test/Analysis/TypeBasedAliasAnalysis/intrinsics.ll
index 655d4558a..61b92b5bd 100644
--- a/llvm/test/Analysis/TypeBasedAliasAnalysis/intrinsics.ll
+++ b/llvm/test/Analysis/TypeBasedAliasAnalysis/intrinsics.ll
@@ -22,8 +22,8 @@ entry:
 declare <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>*, i32, <8 x i1>, <8 x i16>) nounwind readonly
 declare void @llvm.masked.store.v8i16.p0v8i16(<8 x i16>, <8 x i16>*, i32, <8 x i1>) nounwind
 
-; CHECK: attributes #0 = { argmemonly nounwind readonly }
-; CHECK: attributes #1 = { argmemonly nounwind }
+; CHECK: attributes #0 = { argmemonly nounwind readonly willreturn }
+; CHECK: attributes #1 = { argmemonly nounwind willreturn }
 ; CHECK: attributes [[NUW]] = { nounwind }
 
 !0 = !{!"tbaa root"}
diff --git a/llvm/test/Analysis/TypeBasedAliasAnalysis/memcpyopt.ll b/llvm/test/Analysis/TypeBasedAliasAnalysis/memcpyopt.ll
index ecc737c77..c1f6f18f6 100644
--- a/llvm/test/Analysis/TypeBasedAliasAnalysis/memcpyopt.ll
+++ b/llvm/test/Analysis/TypeBasedAliasAnalysis/memcpyopt.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -S -tbaa -basicaa -memcpyopt -instcombine < %s | FileCheck %s
 
 target datalayout = "e-p:64:64:64"
@@ -5,11 +6,11 @@ target datalayout = "e-p:64:64:64"
 ; The second memcpy is redundant and can be deleted. There's an intervening store, but
 ; it has a TBAA tag which declares that it is unrelated.
 
+define void @foo(i8* nocapture %p, i8* nocapture %q, i8* nocapture %s) nounwind {
 ; CHECK: @foo
-; CHECK-NEXT: tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 1 %p, i8* align 1 %q, i64 16, i1 false), !tbaa !0
+; CHECK-NEXT: tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 1 dereferenceable(16) %p, i8* nonnull align 1 dereferenceable(16) %q, i64 16, i1 false), !tbaa !0
 ; CHECK-NEXT: store i8 2, i8* %s, align 1, !tbaa [[TAGA:!.*]]
 ; CHECK-NEXT: ret void
-define void @foo(i8* nocapture %p, i8* nocapture %q, i8* nocapture %s) nounwind {
   tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %p, i8* %q, i64 16, i1 false), !tbaa !2
   store i8 2, i8* %s, align 1, !tbaa !1
   tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %q, i8* %p, i64 16, i1 false), !tbaa !2
diff --git a/llvm/test/Analysis/TypeBasedAliasAnalysis/tbaa-path.ll b/llvm/test/Analysis/TypeBasedAliasAnalysis/tbaa-path.ll
index a2e4dc6dd..0b6e8245e 100644
--- a/llvm/test/Analysis/TypeBasedAliasAnalysis/tbaa-path.ll
+++ b/llvm/test/Analysis/TypeBasedAliasAnalysis/tbaa-path.ll
@@ -361,7 +361,7 @@ entry:
   ret i32 %5
 }
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !0 = !{!1, !1, i64 0}
 !1 = !{!"any pointer", !2}
diff --git a/llvm/test/Analysis/ValueTracking/known-nonnull-at.ll b/llvm/test/Analysis/ValueTracking/known-nonnull-at.ll
index 00eea73ca..9df8c2855 100644
--- a/llvm/test/Analysis/ValueTracking/known-nonnull-at.ll
+++ b/llvm/test/Analysis/ValueTracking/known-nonnull-at.ll
@@ -7,7 +7,7 @@ declare void @bar(i8* %a, i8* nonnull %b)
 
 define i1 @caller1(i8* %x, i8* %y) {
 ; CHECK-LABEL: @caller1(
-; CHECK-NEXT:    call void @bar(i8* %x, i8* %y)
+; CHECK-NEXT:    call void @bar(i8* [[X:%.*]], i8* [[Y:%.*]])
 ; CHECK-NEXT:    ret i1 false
 ;
   call void @bar(i8* %x, i8* %y)
@@ -19,8 +19,8 @@ define i1 @caller1(i8* %x, i8* %y) {
 
 define i1 @caller2(i8* %x, i8* %y) {
 ; CHECK-LABEL: @caller2(
-; CHECK-NEXT:    call void @bar(i8* %y, i8* %x)
-; CHECK-NEXT:    [[NULL_CHECK:%.*]] = icmp eq i8* %y, null
+; CHECK-NEXT:    call void @bar(i8* [[Y:%.*]], i8* [[X:%.*]])
+; CHECK-NEXT:    [[NULL_CHECK:%.*]] = icmp eq i8* [[Y]], null
 ; CHECK-NEXT:    ret i1 [[NULL_CHECK]]
 ;
   call void @bar(i8* %y, i8* %x)
@@ -32,7 +32,7 @@ define i1 @caller2(i8* %x, i8* %y) {
 
 define i1 @caller3(i8* %x, i8* %y) {
 ; CHECK-LABEL: @caller3(
-; CHECK-NEXT:    call void @bar(i8* %x, i8* %y)
+; CHECK-NEXT:    call void @bar(i8* [[X:%.*]], i8* [[Y:%.*]])
 ; CHECK-NEXT:    ret i1 true
 ;
   call void @bar(i8* %x, i8* %y)
@@ -44,8 +44,8 @@ define i1 @caller3(i8* %x, i8* %y) {
 
 define i1 @caller4(i8* %x, i8* %y) {
 ; CHECK-LABEL: @caller4(
-; CHECK-NEXT:    [[NULL_CHECK:%.*]] = icmp ne i8* %y, null
-; CHECK-NEXT:    call void @bar(i8* %x, i8* %y)
+; CHECK-NEXT:    [[NULL_CHECK:%.*]] = icmp ne i8* [[Y:%.*]], null
+; CHECK-NEXT:    call void @bar(i8* [[X:%.*]], i8* [[Y]])
 ; CHECK-NEXT:    ret i1 [[NULL_CHECK]]
 ;
   %null_check = icmp ne i8* %y, null
@@ -57,12 +57,12 @@ define i1 @caller4(i8* %x, i8* %y) {
 
 define i1 @caller5(i8* %x, i8* %y) {
 ; CHECK-LABEL: @caller5(
-; CHECK-NEXT:    [[NULL_CHECK:%.*]] = icmp eq i8* %y, null
-; CHECK-NEXT:    br i1 [[NULL_CHECK]], label %t, label %f
+; CHECK-NEXT:    [[NULL_CHECK:%.*]] = icmp eq i8* [[Y:%.*]], null
+; CHECK-NEXT:    br i1 [[NULL_CHECK]], label [[T:%.*]], label [[F:%.*]]
 ; CHECK:       t:
 ; CHECK-NEXT:    ret i1 [[NULL_CHECK]]
 ; CHECK:       f:
-; CHECK-NEXT:    call void @bar(i8* %x, i8* %y)
+; CHECK-NEXT:    call void @bar(i8* [[X:%.*]], i8* [[Y]])
 ; CHECK-NEXT:    ret i1 [[NULL_CHECK]]
 ;
   %null_check = icmp eq i8* %y, null
@@ -80,13 +80,17 @@ declare i32 @esfp(...)
 
 define i1 @caller6(i8* %x, i8* %y) personality i8* bitcast (i32 (...)* @esfp to i8*){
 ; CHECK-LABEL: @caller6(
-; CHECK-NEXT:    invoke void @bar(i8* %x, i8* nonnull %y)
-; CHECK-NEXT:    to label %cont unwind label %exc
+; CHECK-NEXT:    invoke void @bar(i8* [[X:%.*]], i8* nonnull [[Y:%.*]])
+; CHECK-NEXT:    to label [[CONT:%.*]] unwind label [[EXC:%.*]]
 ; CHECK:       cont:
 ; CHECK-NEXT:    ret i1 false
+; CHECK:       exc:
+; CHECK-NEXT:    [[LP:%.*]] = landingpad { i8*, i32 }
+; CHECK-NEXT:    filter [0 x i8*] zeroinitializer
+; CHECK-NEXT:    unreachable
 ;
   invoke void @bar(i8* %x, i8* nonnull %y)
-    to label %cont unwind label %exc
+  to label %cont unwind label %exc
 
 cont:
   %null_check = icmp eq i8* %y, null
@@ -94,7 +98,7 @@ cont:
 
 exc:
   %lp = landingpad { i8*, i32 }
-    filter [0 x i8*] zeroinitializer
+  filter [0 x i8*] zeroinitializer
   unreachable
 }
 
@@ -102,8 +106,9 @@ declare i8* @returningPtr(i8* returned %p)
 
 define i1 @nonnullReturnTest(i8* nonnull %x) {
 ; CHECK-LABEL: @nonnullReturnTest(
-; CHECK-NEXT:    %x2 = call i8* @returningPtr(i8* %x)
+; CHECK-NEXT:    [[X2:%.*]] = call i8* @returningPtr(i8* [[X:%.*]])
 ; CHECK-NEXT:    ret i1 false
+;
   %x2 = call i8* @returningPtr(i8* %x)
   %null_check = icmp eq i8* %x2, null
   ret i1 %null_check
@@ -111,11 +116,94 @@ define i1 @nonnullReturnTest(i8* nonnull %x) {
 
 define i1 @unknownReturnTest(i8* %x) {
 ; CHECK-LABEL: @unknownReturnTest(
-; CHECK-NEXT:    %x2 = call i8* @returningPtr(i8* %x)
-; CHECK-NEXT:    %null_check = icmp eq i8* %x2, null
-; CHECK-NEXT:    ret i1 %null_check
+; CHECK-NEXT:    [[X2:%.*]] = call i8* @returningPtr(i8* [[X:%.*]])
+; CHECK-NEXT:    [[NULL_CHECK:%.*]] = icmp eq i8* [[X2]], null
+; CHECK-NEXT:    ret i1 [[NULL_CHECK]]
+;
   %x2 = call i8* @returningPtr(i8* %x)
   %null_check = icmp eq i8* %x2, null
   ret i1 %null_check
 }
 
+; Make sure that if load/store happened, the pointer is nonnull.
+
+define i32 @test_null_after_store(i32* %0) {
+; CHECK-LABEL: @test_null_after_store(
+; CHECK-NEXT:    store i32 123, i32* [[TMP0:%.*]], align 4
+; CHECK-NEXT:    ret i32 2
+;
+  store i32 123, i32* %0, align 4
+  %2 = icmp eq i32* %0, null
+  %3 = select i1 %2, i32 1, i32 2
+  ret i32 %3
+}
+
+define i32 @test_null_after_load(i32* %0) {
+; CHECK-LABEL: @test_null_after_load(
+; CHECK-NEXT:    ret i32 1
+;
+  %2 = load i32, i32* %0, align 4
+  %3 = icmp eq i32* %0, null
+  %4 = select i1 %3, i32 %2, i32 1
+  ret i32 %4
+}
+
+; Make sure that different address space does not affect null pointer check.
+
+define i32 @test_null_after_store_addrspace(i32 addrspace(1)* %0) {
+; CHECK-LABEL: @test_null_after_store_addrspace(
+; CHECK-NEXT:    store i32 123, i32 addrspace(1)* [[TMP0:%.*]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 addrspace(1)* [[TMP0]], null
+; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[TMP2]], i32 1, i32 2
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  store i32 123, i32 addrspace(1)* %0, align 4
+  %2 = icmp eq i32 addrspace(1)* %0, null
+  %3 = select i1 %2, i32 1, i32 2
+  ret i32 %3
+}
+
+define i32 @test_null_after_load_addrspace(i32 addrspace(1)* %0) {
+; CHECK-LABEL: @test_null_after_load_addrspace(
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32 addrspace(1)* [[TMP0:%.*]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 addrspace(1)* [[TMP0]], null
+; CHECK-NEXT:    [[TMP4:%.*]] = select i1 [[TMP3]], i32 [[TMP2]], i32 1
+; CHECK-NEXT:    ret i32 [[TMP4]]
+;
+; CHECK-NEXT     ret i32 %4
+  %2 = load i32, i32 addrspace(1)* %0, align 4
+  %3 = icmp eq i32 addrspace(1)* %0, null
+  %4 = select i1 %3, i32 %2, i32 1
+  ret i32 %4
+}
+
+; Make sure if store happened after the check, nullptr check is not removed.
+
+declare i8* @func(i64)
+
+define i8* @test_load_store_after_check(i8* %0) {
+; CHECK-LABEL: @test_load_store_after_check(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8* @func(i64 0)
+; CHECK-NEXT:    [[NULL_CHECK:%.*]] = icmp eq i8* [[TMP1]], null
+; CHECK-NEXT:    br i1 [[NULL_CHECK]], label [[RETURN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.end:
+; CHECK-NEXT:    store i8 7, i8* [[TMP1]]
+; CHECK-NEXT:    br label [[RETURN]]
+; CHECK:       return:
+; CHECK-NEXT:    [[RETVAL_0:%.*]] = phi i8* [ [[TMP1]], [[IF_END]] ], [ null, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    ret i8* [[RETVAL_0]]
+;
+entry:
+  %1 = call i8* @func(i64 0)
+  %null_check = icmp eq i8* %1, null
+  br i1 %null_check, label %return, label %if.end
+
+if.end:
+  store i8 7, i8* %1
+  br label %return
+
+return:
+  %retval.0 = phi i8* [ %1, %if.end ], [ null, %entry ]
+  ret i8* %retval.0
+}
diff --git a/llvm/test/Analysis/ValueTracking/memory-dereferenceable.ll b/llvm/test/Analysis/ValueTracking/memory-dereferenceable.ll
index 2e9453f67..80bc72d0d 100644
--- a/llvm/test/Analysis/ValueTracking/memory-dereferenceable.ll
+++ b/llvm/test/Analysis/ValueTracking/memory-dereferenceable.ll
@@ -172,6 +172,14 @@ entry:
     ret void
 }
 
+; CHECK: The following are dereferenceable:
+; CHECK: %ptr = inttoptr i32 %val to i32*, !dereferenceable !0
+define i32 @f_0(i32 %val) {
+  %ptr = inttoptr i32 %val to i32*, !dereferenceable !0
+  %load29 = load i32, i32* %ptr, align 8
+  ret i32 %load29 
+}
+
 ; Just check that we don't crash.
 ; CHECK-LABEL: 'opaque_type_crasher'
 define void @opaque_type_crasher(%TypeOpaque* dereferenceable(16) %a) {
diff --git a/llvm/test/Analysis/ValueTracking/non-negative-phi-bits.ll b/llvm/test/Analysis/ValueTracking/non-negative-phi-bits.ll
index 059bbaa3c..3b1c43df5 100755
--- a/llvm/test/Analysis/ValueTracking/non-negative-phi-bits.ll
+++ b/llvm/test/Analysis/ValueTracking/non-negative-phi-bits.ll
@@ -8,7 +8,7 @@ define void @test() #0 {
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
-; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ult i64 [[INDVARS_IV_NEXT]], 40
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ult i64 [[INDVARS_IV]], 39
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]]
 ; CHECK:       for.end:
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Assembler/2003-11-11-ImplicitRename.ll b/llvm/test/Assembler/2003-11-11-ImplicitRename.ll
new file mode 100644
index 000000000..84065a178
--- /dev/null
+++ b/llvm/test/Assembler/2003-11-11-ImplicitRename.ll
@@ -0,0 +1,7 @@
+; RUN: not llvm-as < %s > /dev/null 2>&1
+
+void %test() {
+  %X = add int 0, 1
+  %X = add int 1, 2
+  ret void
+}
diff --git a/llvm/test/Assembler/asm-path-writer.ll b/llvm/test/Assembler/asm-path-writer.ll
index f22639566..a0e63ef20 100644
--- a/llvm/test/Assembler/asm-path-writer.ll
+++ b/llvm/test/Assembler/asm-path-writer.ll
@@ -1,6 +1,6 @@
 ; RUN: llvm-as < %s | llvm-dis | FileCheck %s
 
-; CHECK: ^0 = module: (path: ".\5Cf4folder\5Cabc.o", hash: (0, 0, 0, 0, 0))
+; CHECK: ^0 = module: (path: ".\\f4folder\\abc.o", hash: (0, 0, 0, 0, 0))
 
-^0 = module: (path: ".\5Cf4folder\5Cabc.o", hash: (0, 0, 0, 0, 0))
+^0 = module: (path: ".\5Cf4folder\\abc.o", hash: (0, 0, 0, 0, 0))
 ^1 = gv: (guid: 15822663052811949562, summaries: (function: (module: ^0, flags: (linkage: external, notEligibleToImport: 0, live: 0, dsoLocal: 0), insts: 2)))
diff --git a/llvm/test/Assembler/auto_upgrade_intrinsics.ll b/llvm/test/Assembler/auto_upgrade_intrinsics.ll
index 9ca4c4855..f75206d61 100644
--- a/llvm/test/Assembler/auto_upgrade_intrinsics.ll
+++ b/llvm/test/Assembler/auto_upgrade_intrinsics.ll
@@ -140,6 +140,21 @@ define void @tests.lifetime.start.end() {
   ret void
 }
 
+declare void @llvm.prefetch(i8*, i32, i32, i32)
+define void @test.prefetch(i8* %ptr) {
+; CHECK-LABEL: @test.prefetch(
+; CHECK: @llvm.prefetch.p0i8(i8* %ptr, i32 0, i32 3, i32 2)
+  call void @llvm.prefetch(i8* %ptr, i32 0, i32 3, i32 2)
+  ret void
+}
+
+declare void @llvm.prefetch.p0i8(i8*, i32, i32, i32)
+define void @test.prefetch.2(i8* %ptr) {
+; CHECK-LABEL: @test.prefetch.2(
+; CHECK: @llvm.prefetch.p0i8(i8* %ptr, i32 0, i32 3, i32 2)
+  call void @llvm.prefetch(i8* %ptr, i32 0, i32 3, i32 2)
+  ret void
+}
 
 ; This is part of @test.objectsize(), since llvm.objectsize declaration gets
 ; emitted at the end.
diff --git a/llvm/test/Assembler/block-labels.ll b/llvm/test/Assembler/block-labels.ll
index 1dcf496ad..9ab55e00c 100644
--- a/llvm/test/Assembler/block-labels.ll
+++ b/llvm/test/Assembler/block-labels.ll
@@ -38,11 +38,11 @@ $N:
 ; CHECK-NEXT:   ret i32 %4
 ; CHECK-NEXT: }
 
-define void @test2(i32, i32) {
+define void @test2(i32 %0, i32 %1) {
 ; entry label id still not printed on output
 2:
   ret void
 }
 
-; CHECK-LABEL: define void @test2(i32, i32) {
+; CHECK-LABEL: define void @test2(i32 %0, i32 %1) {
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Assembler/byval-type-attr.ll b/llvm/test/Assembler/byval-type-attr.ll
index eb17a30db..dd195a396 100644
--- a/llvm/test/Assembler/byval-type-attr.ll
+++ b/llvm/test/Assembler/byval-type-attr.ll
@@ -1,12 +1,12 @@
 ; RUN: llvm-as < %s | llvm-dis | llvm-as | llvm-dis | FileCheck %s
 
-; CHECK: define void @foo(i32* byval(i32) align 4)
-define void @foo(i32* byval(i32) align 4) {
+; CHECK: define void @foo(i32* byval(i32) align 4 %0)
+define void @foo(i32* byval(i32) align 4 %0) {
   ret void
 }
 
-; CHECK: define void @bar({ i32*, i8 }* byval({ i32*, i8 }) align 4)
-define void @bar({i32*, i8}* byval({i32*, i8}) align 4) {
+; CHECK: define void @bar({ i32*, i8 }* byval({ i32*, i8 }) align 4 %0)
+define void @bar({i32*, i8}* byval({i32*, i8}) align 4 %0) {
   ret void
 }
 
@@ -29,3 +29,15 @@ fail:
 declare void @baz(%named_type* byval(%named_type))
 
 declare i32 @__gxx_personality_v0(...)
+
+%0 = type opaque
+
+; CHECK: define void @anon({ %0* }* byval({ %0* }) %arg)
+; CHECK:   call void @anon_callee({ %0* }* byval({ %0* }) %arg)
+define void @anon({ %0* }* byval({ %0* }) %arg) {
+  call void @anon_callee({ %0* }* byval({ %0* }) %arg)
+  ret void
+}
+
+; CHECK: declare void @anon_callee({ %0* }* byval({ %0* }))
+declare void @anon_callee({ %0* }* byval({ %0* }))
diff --git a/llvm/test/Assembler/call-nonzero-program-addrspace-2.ll.x b/llvm/test/Assembler/call-nonzero-program-addrspace-2.ll.x
index cde546e0c..6ec1fd0e2 100644
--- a/llvm/test/Assembler/call-nonzero-program-addrspace-2.ll.x
+++ b/llvm/test/Assembler/call-nonzero-program-addrspace-2.ll.x
@@ -3,7 +3,7 @@
 
 ; Check that numbered variables in a nonzero program address space 200 can be used in a call instruction
 
-define i8 @test_unnamed(i8(i32)*, i8(i32) addrspace(42)*) {
+define i8 @test_unnamed(i8(i32)*, i8(i32) addrspace(42)* %0) {
   ; Calls with explicit address spaces are fine:
   call addrspace(0) i8 %0(i32 0)
   call addrspace(42) i8 %1(i32 0)
@@ -14,7 +14,7 @@ define i8 @test_unnamed(i8(i32)*, i8(i32) addrspace(42)*) {
 }
 
 ; PROGAS42:       target datalayout = "P42"
-; PROGAS42:       define i8 @test_unnamed(i8 (i32)*, i8 (i32) addrspace(42)*) addrspace(42) {
+; PROGAS42:       define i8 @test_unnamed(i8 (i32)* %0, i8 (i32) addrspace(42)* %1) addrspace(42) {
 ; PROGAS42-NEXT:    %3 = call addrspace(0) i8 %0(i32 0)
 ; PROGAS42-NEXT:    %4 = call addrspace(42) i8 %1(i32 0)
 ; PROGAS42-NEXT:    %5 = call addrspace(42) i8 %1(i32 0)
diff --git a/llvm/test/Assembler/datalayout-invalid-function-ptr-alignment.ll b/llvm/test/Assembler/datalayout-invalid-function-ptr-alignment.ll
new file mode 100644
index 000000000..21cd6a6dc
--- /dev/null
+++ b/llvm/test/Assembler/datalayout-invalid-function-ptr-alignment.ll
@@ -0,0 +1,5 @@
+; RUN: not llvm-as %s 2>&1 | FileCheck %s
+
+; CHECK: LLVM ERROR: Alignment is neither 0 nor a power of 2
+
+target datalayout = "Fi24"
diff --git a/llvm/test/Assembler/datalayout-invalid-stack-natural-alignment.ll b/llvm/test/Assembler/datalayout-invalid-stack-natural-alignment.ll
new file mode 100644
index 000000000..c8d7ba62a
--- /dev/null
+++ b/llvm/test/Assembler/datalayout-invalid-stack-natural-alignment.ll
@@ -0,0 +1,5 @@
+; RUN: not llvm-as %s 2>&1 | FileCheck %s
+
+; CHECK: LLVM ERROR: Alignment is neither 0 nor a power of 2
+
+target datalayout = "S24"
diff --git a/llvm/test/Assembler/debug-info.ll b/llvm/test/Assembler/debug-info.ll
index 8c3922ae6..d54dba07a 100644
--- a/llvm/test/Assembler/debug-info.ll
+++ b/llvm/test/Assembler/debug-info.ll
@@ -1,8 +1,8 @@
 ; RUN: llvm-as < %s | llvm-dis | llvm-as | llvm-dis | FileCheck %s
 ; RUN: verify-uselistorder %s
 
-; CHECK: !named = !{!0, !0, !1, !2, !3, !4, !5, !6, !7, !8, !8, !9, !10, !11, !12, !13, !14, !15, !16, !17, !18, !19, !20, !21, !22, !23, !24, !25, !26, !27, !27, !28, !29, !30, !31, !32, !33, !34, !35, !36, !37, !38, !39, !40, !41}
-!named = !{!0, !1, !2, !3, !4, !5, !6, !7, !8, !9, !10, !11, !12, !13, !14, !15, !16, !17, !18, !19, !20, !21, !22, !23, !24, !25, !26, !27, !28, !29, !30, !31, !32, !33, !34, !35, !36, !37, !38, !39, !40, !41, !42, !43, !44}
+; CHECK: !named = !{!0, !0, !1, !2, !3, !4, !5, !6, !7, !8, !8, !9, !10, !11, !12, !13, !14, !15, !16, !17, !18, !19, !20, !21, !22, !23, !24, !25, !26, !27, !27, !28, !29, !30, !31, !32, !33, !34, !35, !36, !37, !38, !39}
+!named = !{!0, !1, !2, !3, !4, !5, !6, !7, !8, !9, !10, !11, !12, !13, !14, !15, !16, !17, !18, !19, !20, !21, !22, !23, !24, !25, !26, !27, !28, !29, !30, !31, !32, !33, !34, !35, !36, !37, !38, !39, !40, !41, !42}
 
 ; CHECK:      !0 = !DISubrange(count: 3)
 ; CHECK-NEXT: !1 = !DISubrange(count: 3, lowerBound: 4)
@@ -99,8 +99,3 @@
 ; CHECK-NEXT: !39 = !DIBasicType(name: "u64.le", size: 64, align: 1, encoding: DW_ATE_unsigned, flags: DIFlagLittleEndian)
 !41 = !DIBasicType(name: "u64.be", size: 64, align: 1, encoding: DW_ATE_unsigned, flags: DIFlagBigEndian)
 !42 = !DIBasicType(name: "u64.le", size: 64, align: 1, encoding: DW_ATE_unsigned, flags: DIFlagLittleEndian)
-
-; CHECK-NEXT: !40 = distinct !DISubprogram(name: "fn", scope: {{.*}}, file: {{.*}}, spFlags: 0)
-; CHECK-NEXT: !41 = !DILocalVariable(name: "Name", arg: 1, scope: {{.*}}, file: {{.*}}, line: 13, type: {{.*}}, flags: DIFlagArgumentNotModified)
-!43 = distinct !DISubprogram(name: "fn", scope: !12, file: !12, spFlags: 0)
-!44 = !DILocalVariable(name: "Name", arg: 1, scope: !43, file: !12, line: 13, type: !7, flags: DIFlagArgumentNotModified)
diff --git a/llvm/test/Assembler/dimodule.ll b/llvm/test/Assembler/dimodule.ll
index 994bc12b6..6337bffb3 100644
--- a/llvm/test/Assembler/dimodule.ll
+++ b/llvm/test/Assembler/dimodule.ll
@@ -9,7 +9,7 @@
 ; CHECK: !1 = !DIModule(scope: !0, name: "Module")
 !1 = !DIModule(scope: !0, name: "Module")
 
-; CHECK: !2 = !DIModule(scope: !0, name: "Module", configMacros: "-DNDEBUG", includePath: "/usr/include", isysroot: "/")
-!2 = !DIModule(scope: !0, name: "Module", configMacros: "-DNDEBUG", includePath: "/usr/include", isysroot: "/")
+; CHECK: !2 = !DIModule(scope: !0, name: "Module", configMacros: "-DNDEBUG", includePath: "/usr/include", sysroot: "/")
+!2 = !DIModule(scope: !0, name: "Module", configMacros: "-DNDEBUG", includePath: "/usr/include", sysroot: "/")
 
 !3 = !DIModule(scope: !0, name: "Module", configMacros: "")
diff --git a/llvm/test/Assembler/export-symbol-anonymous-class.ll b/llvm/test/Assembler/export-symbol-anonymous-class.ll
new file mode 100644
index 000000000..29fedbd88
--- /dev/null
+++ b/llvm/test/Assembler/export-symbol-anonymous-class.ll
@@ -0,0 +1,38 @@
+; Round trip for the following:
+; ```
+; struct A {
+;  struct {
+;      int y;
+;  };
+; } a;
+; ```
+; This is the minimum code to generate export symbols flag due to anonymous class in A.
+
+; RUN: llvm-as %s -o - | llvm-dis | llvm-as | llvm-dis | FileCheck %s
+
+%struct.A = type { %struct.anon }
+%struct.anon = type { i32 }
+
+@a = global %struct.A zeroinitializer, align 4, !dbg !0
+
+!llvm.module.flags = !{!14, !15}
+!llvm.dbg.cu = !{!2}
+!llvm.ident = !{!16}
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "a", scope: !2, file: !3, line: 5, type: !6, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !3, producer: "clang version 10.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5, nameTableKind: GNU)
+!3 = !DIFile(filename: "simple_anon_class.cpp", directory: "/dir")
+!4 = !{}
+!5 = !{!0}
+!6 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "A", file: !3, line: 1, size: 32, flags: DIFlagTypePassByValue, elements: !7, identifier: "_ZTS1A")
+!7 = !{!8}
+!8 = !DIDerivedType(tag: DW_TAG_member, scope: !6, file: !3, line: 2, baseType: !9, size: 32)
+!9 = distinct !DICompositeType(tag: DW_TAG_structure_type, scope: !6, file: !3, line: 2, size: 32, flags: DIFlagExportSymbols | DIFlagTypePassByValue, elements: !10, identifier: "_ZTSN1AUt_E")
+; CHECK: DIFlagExportSymbols | DIFlagTypePassByValue
+!10 = !{!11}
+!11 = !DIDerivedType(tag: DW_TAG_member, name: "y", scope: !9, file: !3, line: 3, baseType: !12, size: 32)
+!12 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!14 = !{i32 2, !"Dwarf Version", i32 4}
+!15 = !{i32 2, !"Debug Info Version", i32 3}
+!16 = !{!"clang version 10.0.0"}
diff --git a/llvm/test/Assembler/incorrect-tdep-attrs-parsing.ll b/llvm/test/Assembler/incorrect-tdep-attrs-parsing.ll
index bf8152b69..7105eac51 100644
--- a/llvm/test/Assembler/incorrect-tdep-attrs-parsing.ll
+++ b/llvm/test/Assembler/incorrect-tdep-attrs-parsing.ll
@@ -1,6 +1,6 @@
 ; RUN: llvm-as < %s | llvm-dis | FileCheck %s
 
-; CHECK: define i32 @f(i64 "foo bar", i64, i64, i64 "xyz") {
+; CHECK: define i32 @f(i64 "foo bar" %0, i64 %1, i64 %2, i64 "xyz" %3) {
 define i32 @f(i64 "foo bar", i64, i64, i64 "xyz") {
   ret i32 41
 }
diff --git a/llvm/test/Assembler/invalid-arg-num-1.ll b/llvm/test/Assembler/invalid-arg-num-1.ll
new file mode 100644
index 000000000..ee13c339d
--- /dev/null
+++ b/llvm/test/Assembler/invalid-arg-num-1.ll
@@ -0,0 +1,6 @@
+; RUN: not llvm-as < %s -o /dev/null 2>&1 | FileCheck %s
+
+; CHECK: error: argument expected to be numbered '%1'
+define void @foo(i32 %0, i32 %5) {
+  ret void
+}
diff --git a/llvm/test/Assembler/invalid-arg-num-2.ll b/llvm/test/Assembler/invalid-arg-num-2.ll
new file mode 100644
index 000000000..6ecb00335
--- /dev/null
+++ b/llvm/test/Assembler/invalid-arg-num-2.ll
@@ -0,0 +1,6 @@
+; RUN: not llvm-as < %s -o /dev/null 2>&1 | FileCheck %s
+
+; CHECK: error: argument expected to be numbered '%1'
+define void @foo(i8 %0, i32 %named, i32 %2) {
+  ret void
+}
diff --git a/llvm/test/Assembler/invalid-arg-num-3.ll b/llvm/test/Assembler/invalid-arg-num-3.ll
new file mode 100644
index 000000000..f16383656
--- /dev/null
+++ b/llvm/test/Assembler/invalid-arg-num-3.ll
@@ -0,0 +1,6 @@
+; RUN: not llvm-as < %s -o /dev/null 2>&1 | FileCheck %s
+
+; CHECK: error: argument expected to be numbered '%0'
+define void @foo(i8 %1) {
+  ret void
+}
diff --git a/llvm/test/Assembler/invoke-nonzero-program-addrspace.ll.x b/llvm/test/Assembler/invoke-nonzero-program-addrspace.ll.x
index f13973ab7..a3f70aec3 100644
--- a/llvm/test/Assembler/invoke-nonzero-program-addrspace.ll.x
+++ b/llvm/test/Assembler/invoke-nonzero-program-addrspace.ll.x
@@ -18,10 +18,7 @@ lpad:
     unreachable
 }
 
-; Note: we add addrspace(200) to the @__gxx_personality_v0 function declaration,
-; as it is used in the personality function of @test_invoke within an address
-; space of 200.
-declare i32 @__gxx_personality_v0(...) addrspace(200)
+declare i32 @__gxx_personality_v0(...)
 
 
 ; PROGAS200:  target datalayout = "P200"
diff --git a/llvm/test/Assembler/multi-mod-disassemble.ll b/llvm/test/Assembler/multi-mod-disassemble.ll
new file mode 100644
index 000000000..58773b644
--- /dev/null
+++ b/llvm/test/Assembler/multi-mod-disassemble.ll
@@ -0,0 +1,15 @@
+; RUN: llvm-as %s -o %t.o
+; RUN: llvm-cat -b -o %t2.o %t.o %t.o
+; RUN: llvm-dis -o %t3 %t2.o
+; RUN: llvm-as -o %t4.o %t3.0
+; RUN: llvm-as -o %t5.o %t3.1
+; RUN: cmp %t4.o %t5.o
+; RUN: llvm-cat -b -o %t6.o %t5.o %t4.o
+; RUN: llvm-dis -o %t7.o %t6.o
+; RUN: diff %t7.o.0 %t7.o.1
+; RUN: FileCheck < %t7.o.0 %s
+; RUN: FileCheck < %t7.o.1 %s
+
+; CHECK: source_filename = "{{.*}}multi-mod-disassemble.ll"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/Assembler/multi-summary-disassemble.ll b/llvm/test/Assembler/multi-summary-disassemble.ll
new file mode 100644
index 000000000..d49ca35be
--- /dev/null
+++ b/llvm/test/Assembler/multi-summary-disassemble.ll
@@ -0,0 +1,18 @@
+; RUN: llvm-as %s -o %t.o
+; RUN: llvm-cat -b -o %t2.o %t.o %t.o
+; RUN: llvm-dis -o %t3 %t2.o
+; RUN: llvm-as -o %t4.o %t3.0
+; RUN: llvm-as -o %t5.o %t3.1
+; RUN: cmp %t4.o %t5.o
+; RUN: llvm-cat -b -o %t6.o %t5.o %t4.o
+; RUN: llvm-dis -o %t7.o %t6.o
+; RUN: diff %t7.o.0 %t7.o.1
+; RUN: FileCheck < %t7.o.0 %s
+; RUN: FileCheck < %t7.o.1 %s
+
+; ModuleID = 'multi-summary-disassemble.o'
+
+^0 = module: (path: "multi-summary-disassemble.ll", hash: (1369602428, 2747878711, 259090915, 2507395659, 1141468049))
+^1 = gv: (guid: 3, summaries: (function: (module: ^0, flags: (linkage: internal, notEligibleToImport: 0, live: 0, dsoLocal: 1), insts: 1)))
+; CHECK: ^0 = module: (path:
+; CHECK: ^1 = gv: (guid: 3, summaries: (function: (module: ^0,
diff --git a/llvm/test/Assembler/source-filename-backslash.ll b/llvm/test/Assembler/source-filename-backslash.ll
index 8669401cf..c4b25c605 100644
--- a/llvm/test/Assembler/source-filename-backslash.ll
+++ b/llvm/test/Assembler/source-filename-backslash.ll
@@ -1,8 +1,7 @@
-
 ; Make sure that llvm-as/llvm-dis properly assemble/disassemble the
 ; source_filename.
 
 ; RUN: llvm-as < %s | llvm-dis | FileCheck %s
 
-; CHECK: source_filename = "C:\5Cpath\5Cwith\5Cbackslashes\5Ctest.cc"
-source_filename = "C:\5Cpath\5Cwith\5Cbackslashes\5Ctest.cc"
+; CHECK: source_filename = "C:\\path\\with\\backslashes\\test.cc"
+source_filename = "C:\\path\\with\5Cbackslashes\\test.cc"
diff --git a/llvm/test/Assembler/thinlto-summary.ll b/llvm/test/Assembler/thinlto-summary.ll
index 23d83e3a2..dea535cc4 100644
--- a/llvm/test/Assembler/thinlto-summary.ll
+++ b/llvm/test/Assembler/thinlto-summary.ll
@@ -37,8 +37,8 @@
 
 ; Functions with various flag combinations (notEligibleToImport, Live,
 ; combinations of optional function flags).
-^15 = gv: (guid: 14, summaries: (function: (module: ^1, flags: (linkage: external, notEligibleToImport: 1, live: 1, dsoLocal: 0), insts: 1)))
-^16 = gv: (guid: 15, summaries: (function: (module: ^1, flags: (linkage: external, notEligibleToImport: 0, live: 0, dsoLocal: 0), insts: 1, funcFlags: (readNone: 1, noRecurse: 1))))
+^15 = gv: (guid: 14, summaries: (function: (module: ^1, flags: (linkage: external, notEligibleToImport: 1, live: 1, dsoLocal: 0), insts: 1, funcFlags: (noInline: 1))))
+^16 = gv: (guid: 15, summaries: (function: (module: ^1, flags: (linkage: external, notEligibleToImport: 0, live: 0, dsoLocal: 0), insts: 1, funcFlags: (readNone: 1, noRecurse: 1, alwaysInline: 1))))
 ; This one also tests backwards reference in calls.
 ^17 = gv: (guid: 16, summaries: (function: (module: ^1, flags: (linkage: external, notEligibleToImport: 0, live: 0, dsoLocal: 0), insts: 1, funcFlags: (readOnly: 1, returnDoesNotAlias: 1), calls: ((callee: ^15)))))
 
@@ -80,9 +80,9 @@
 ; CHECK: ^12 = gv: (guid: 11, summaries: (variable: (module: ^0, flags: (linkage: appending, notEligibleToImport: 0, live: 0, dsoLocal: 0, canAutoHide: 0), varFlags: (readonly: 0, writeonly: 0), refs: (^4))))
 ; CHECK: ^13 = gv: (guid: 12, summaries: (variable: (module: ^0, flags: (linkage: external, notEligibleToImport: 0, live: 0, dsoLocal: 0, canAutoHide: 0), varFlags: (readonly: 1, writeonly: 0))))
 ; CHECK: ^14 = gv: (guid: 13, summaries: (variable: (module: ^0, flags: (linkage: external, notEligibleToImport: 0, live: 0, dsoLocal: 1, canAutoHide: 0), varFlags: (readonly: 0, writeonly: 0))))
-; CHECK: ^15 = gv: (guid: 14, summaries: (function: (module: ^1, flags: (linkage: external, notEligibleToImport: 1, live: 1, dsoLocal: 0, canAutoHide: 0), insts: 1)))
-; CHECK: ^16 = gv: (guid: 15, summaries: (function: (module: ^1, flags: (linkage: external, notEligibleToImport: 0, live: 0, dsoLocal: 0, canAutoHide: 0), insts: 1, funcFlags: (readNone: 1, readOnly: 0, noRecurse: 1, returnDoesNotAlias: 0, noInline: 0))))
-; CHECK: ^17 = gv: (guid: 16, summaries: (function: (module: ^1, flags: (linkage: external, notEligibleToImport: 0, live: 0, dsoLocal: 0, canAutoHide: 0), insts: 1, funcFlags: (readNone: 0, readOnly: 1, noRecurse: 0, returnDoesNotAlias: 1, noInline: 0), calls: ((callee: ^15)))))
+; CHECK: ^15 = gv: (guid: 14, summaries: (function: (module: ^1, flags: (linkage: external, notEligibleToImport: 1, live: 1, dsoLocal: 0, canAutoHide: 0), insts: 1, funcFlags: (readNone: 0, readOnly: 0, noRecurse: 0, returnDoesNotAlias: 0, noInline: 1, alwaysInline: 0))))
+; CHECK: ^16 = gv: (guid: 15, summaries: (function: (module: ^1, flags: (linkage: external, notEligibleToImport: 0, live: 0, dsoLocal: 0, canAutoHide: 0), insts: 1, funcFlags: (readNone: 1, readOnly: 0, noRecurse: 1, returnDoesNotAlias: 0, noInline: 0, alwaysInline: 1))))
+; CHECK: ^17 = gv: (guid: 16, summaries: (function: (module: ^1, flags: (linkage: external, notEligibleToImport: 0, live: 0, dsoLocal: 0, canAutoHide: 0), insts: 1, funcFlags: (readNone: 0, readOnly: 1, noRecurse: 0, returnDoesNotAlias: 1, noInline: 0, alwaysInline: 0), calls: ((callee: ^15)))))
 ; CHECK: ^18 = gv: (guid: 17, summaries: (alias: (module: ^0, flags: (linkage: external, notEligibleToImport: 0, live: 0, dsoLocal: 1, canAutoHide: 0), aliasee: ^14)))
 ; CHECK: ^19 = gv: (guid: 18, summaries: (function: (module: ^0, flags: (linkage: external, notEligibleToImport: 0, live: 0, dsoLocal: 0, canAutoHide: 0), insts: 4, typeIdInfo: (typeTests: (^24, ^26)))))
 ; CHECK: ^20 = gv: (guid: 19, summaries: (function: (module: ^0, flags: (linkage: external, notEligibleToImport: 0, live: 0, dsoLocal: 0, canAutoHide: 0), insts: 8, typeIdInfo: (typeTestAssumeVCalls: (vFuncId: (^27, offset: 16))))))
diff --git a/llvm/test/Assembler/thinlto-vtable-summary.ll b/llvm/test/Assembler/thinlto-vtable-summary.ll
index ac4c703c3..e4d2a6846 100644
--- a/llvm/test/Assembler/thinlto-vtable-summary.ll
+++ b/llvm/test/Assembler/thinlto-vtable-summary.ll
@@ -4,7 +4,7 @@
 ; RUN: grep "^\^" %t.ll >%t3
 ; Expect that the summary information is the same after round-trip through
 ; llvm-as and llvm-dis.
-; RUN: diff %t2 %t3
+; RUN: diff -b %t2 %t3
 
 source_filename = "thinlto-vtable-summary.ll"
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Bindings/llvm-c/debug_info.ll b/llvm/test/Bindings/llvm-c/debug_info.ll
index 84635f041..af682fdf1 100644
--- a/llvm/test/Bindings/llvm-c/debug_info.ll
+++ b/llvm/test/Bindings/llvm-c/debug_info.ll
@@ -3,13 +3,13 @@
 ; CHECK: ; ModuleID = 'debuginfo.c'
 ; CHECK-NEXT: source_filename = "debuginfo.c"
 
-; CHECK:      define i64 @foo(i64, i64, <10 x i64>) !dbg !20 {
+; CHECK:      define i64 @foo(i64 %0, i64 %1, <10 x i64> %2) !dbg !31 {
 ; CHECK-NEXT: entry:
-; CHECK-NEXT:   call void @llvm.dbg.declare(metadata i64 0, metadata !27, metadata !DIExpression()), !dbg !32
-; CHECK-NEXT:   call void @llvm.dbg.declare(metadata i64 0, metadata !28, metadata !DIExpression()), !dbg !32
-; CHECK-NEXT:   call void @llvm.dbg.declare(metadata i64 0, metadata !29, metadata !DIExpression()), !dbg !32
+; CHECK-NEXT:   call void @llvm.dbg.declare(metadata i64 0, metadata !38, metadata !DIExpression()), !dbg !43
+; CHECK-NEXT:   call void @llvm.dbg.declare(metadata i64 0, metadata !39, metadata !DIExpression()), !dbg !43
+; CHECK-NEXT:   call void @llvm.dbg.declare(metadata i64 0, metadata !40, metadata !DIExpression()), !dbg !43
 ; CHECK:      vars:                                             ; No predecessors!
-; CHECK-NEXT:   call void @llvm.dbg.value(metadata i64 0, metadata !30, metadata !DIExpression(DW_OP_constu, 0, DW_OP_stack_value)), !dbg !33
+; CHECK-NEXT:   call void @llvm.dbg.value(metadata i64 0, metadata !41, metadata !DIExpression(DW_OP_constu, 0, DW_OP_stack_value)), !dbg !44
 ; CHECK-NEXT: }
 
 ; CHECK:      ; Function Attrs: nounwind readnone speculatable
@@ -18,42 +18,54 @@
 ; CHECK:      ; Function Attrs: nounwind readnone speculatable
 ; CHECK-NEXT: declare void @llvm.dbg.value(metadata, metadata, metadata) #0
 
-; CHECK:      attributes #0 = { nounwind readnone speculatable }
+; CHECK:      attributes #0 = { nounwind readnone speculatable willreturn }
 
 ; CHECK:      !llvm.dbg.cu = !{!0}
-; CHECK-NEXT: !FooType = !{!16}
+; CHECK-NEXT: !FooType = !{!28}
+; CHECK-NEXT: !EnumTest = !{!3}
 
-; CHECK:      !0 = distinct !DICompileUnit(language: DW_LANG_C, file: !1, producer: "llvm-c-test", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, globals: !3, imports: !12, splitDebugInlining: false)
+; CHECK:      !0 = distinct !DICompileUnit(language: DW_LANG_C, file: !1, producer: "llvm-c-test", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, globals: !11, imports: !19, macros: !23, splitDebugInlining: false)
 ; CHECK-NEXT: !1 = !DIFile(filename: "debuginfo.c", directory: ".")
-; CHECK-NEXT: !2 = !{}
-; CHECK-NEXT: !3 = !{!4, !8}
-; CHECK-NEXT: !4 = !DIGlobalVariableExpression(var: !5, expr: !DIExpression(DW_OP_constu, 0, DW_OP_stack_value))
-; CHECK-NEXT: !5 = distinct !DIGlobalVariable(name: "globalClass", scope: !6, file: !1, line: 1, type: !7, isLocal: true, isDefinition: true)
-; CHECK-NEXT: !6 = !DIModule(scope: null, name: "llvm-c-test", includePath: "/test/include/llvm-c-test.h")
-; CHECK-NEXT: !7 = !DICompositeType(tag: DW_TAG_structure_type, name: "TestClass", scope: !1, file: !1, line: 42, size: 64, flags: DIFlagObjcClassComplete, elements: !2)
-; CHECK-NEXT: !8 = !DIGlobalVariableExpression(var: !9, expr: !DIExpression(DW_OP_constu, 0, DW_OP_stack_value))
-; CHECK-NEXT: !9 = distinct !DIGlobalVariable(name: "global", scope: !6, file: !1, line: 1, type: !10, isLocal: true, isDefinition: true)
-; CHECK-NEXT: !10 = !DIDerivedType(tag: DW_TAG_typedef, name: "int64_t", scope: !1, file: !1, line: 42, baseType: !11)
-; CHECK-NEXT: !11 = !DIBasicType(name: "Int64", size: 64)
-; CHECK-NEXT: !12 = !{!13, !15}
-; CHECK-NEXT: !13 = !DIImportedEntity(tag: DW_TAG_imported_module, scope: !6, entity: !14, file: !1, line: 42)
-; CHECK-NEXT: !14 = !DIModule(scope: null, name: "llvm-c-test-import", includePath: "/test/include/llvm-c-test-import.h")
-; CHECK-NEXT: !15 = !DIImportedEntity(tag: DW_TAG_imported_module, scope: !6, entity: !13, file: !1, line: 42)
-; CHECK-NEXT: !16 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !17, size: 192, dwarfAddressSpace: 0)
-; CHECK-NEXT: !17 = !DICompositeType(tag: DW_TAG_structure_type, name: "MyStruct", scope: !18, file: !1, size: 192, elements: !19, runtimeLang: DW_LANG_C89, identifier: "MyStruct")
-; CHECK-NEXT: !18 = !DINamespace(name: "NameSpace", scope: !6)
-; CHECK-NEXT: !19 = !{!11, !11, !11}
-; CHECK-NEXT: !20 = distinct !DISubprogram(name: "foo", linkageName: "foo", scope: !1, file: !1, line: 42, type: !21, scopeLine: 42, spFlags: DISPFlagLocalToUnit | DISPFlagDefinition, unit: !0, retainedNodes: !26)
-; CHECK-NEXT: !21 = !DISubroutineType(types: !22)
-; CHECK-NEXT: !22 = !{!11, !11, !23}
-; CHECK-NEXT: !23 = !DICompositeType(tag: DW_TAG_array_type, baseType: !11, size: 640, flags: DIFlagVector, elements: !24)
-; CHECK-NEXT: !24 = !{!25}
-; CHECK-NEXT: !25 = !DISubrange(count: 10)
-; CHECK-NEXT: !26 = !{!27, !28, !29, !30}
-; CHECK-NEXT: !27 = !DILocalVariable(name: "a", arg: 1, scope: !20, file: !1, line: 42, type: !11)
-; CHECK-NEXT: !28 = !DILocalVariable(name: "b", arg: 2, scope: !20, file: !1, line: 42, type: !11)
-; CHECK-NEXT: !29 = !DILocalVariable(name: "c", arg: 3, scope: !20, file: !1, line: 42, type: !23)
-; CHECK-NEXT: !30 = !DILocalVariable(name: "d", scope: !31, file: !1, line: 43, type: !11)
-; CHECK-NEXT: !31 = distinct !DILexicalBlock(scope: !20, file: !1, line: 42)
-; CHECK-NEXT: !32 = !DILocation(line: 42, scope: !20)
-; CHECK-NEXT: !33 = !DILocation(line: 43, scope: !20)
+; CHECK-NEXT: !2 = !{!3}
+; CHECK-NEXT: !3 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "EnumTest", scope: !4, file: !1, baseType: !6, size: 64, elements: !7)
+; CHECK-NEXT: !4 = !DINamespace(name: "NameSpace", scope: !5)
+; CHECK-NEXT: !5 = !DIModule(scope: null, name: "llvm-c-test", includePath: "/test/include/llvm-c-test.h")
+; CHECK-NEXT: !6 = !DIBasicType(name: "Int64", size: 64)
+; CHECK-NEXT: !7 = !{!8, !9, !10}
+; CHECK-NEXT: !8 = !DIEnumerator(name: "Test_A", value: 0, isUnsigned: true)
+; CHECK-NEXT: !9 = !DIEnumerator(name: "Test_B", value: 1, isUnsigned: true)
+; CHECK-NEXT: !10 = !DIEnumerator(name: "Test_B", value: 2, isUnsigned: true)
+; CHECK-NEXT: !11 = !{!12, !16}
+; CHECK-NEXT: !12 = !DIGlobalVariableExpression(var: !13, expr: !DIExpression(DW_OP_constu, 0, DW_OP_stack_value))
+; CHECK-NEXT: !13 = distinct !DIGlobalVariable(name: "globalClass", scope: !5, file: !1, line: 1, type: !14, isLocal: true, isDefinition: true)
+; CHECK-NEXT: !14 = !DICompositeType(tag: DW_TAG_structure_type, name: "TestClass", scope: !1, file: !1, line: 42, size: 64, flags: DIFlagObjcClassComplete, elements: !15)
+; CHECK-NEXT: !15 = !{}
+; CHECK-NEXT: !16 = !DIGlobalVariableExpression(var: !17, expr: !DIExpression(DW_OP_constu, 0, DW_OP_stack_value))
+; CHECK-NEXT: !17 = distinct !DIGlobalVariable(name: "global", scope: !5, file: !1, line: 1, type: !18, isLocal: true, isDefinition: true)
+; CHECK-NEXT: !18 = !DIDerivedType(tag: DW_TAG_typedef, name: "int64_t", scope: !1, file: !1, line: 42, baseType: !6)
+; CHECK-NEXT: !19 = !{!20, !22}
+; CHECK-NEXT: !20 = !DIImportedEntity(tag: DW_TAG_imported_module, scope: !5, entity: !21, file: !1, line: 42)
+; CHECK-NEXT: !21 = !DIModule(scope: null, name: "llvm-c-test-import", includePath: "/test/include/llvm-c-test-import.h")
+; CHECK-NEXT: !22 = !DIImportedEntity(tag: DW_TAG_imported_module, scope: !5, entity: !20, file: !1, line: 42)
+; CHECK-NEXT: !23 = !{!24}
+; CHECK-NEXT: !24 = !DIMacroFile(file: !1, nodes: !25)
+; CHECK-NEXT: !25 = !{!26, !27}
+; CHECK-NEXT: !26 = !DIMacro(type: DW_MACINFO_define, name: "SIMPLE_DEFINE")
+; CHECK-NEXT: !27 = !DIMacro(type: DW_MACINFO_define, name: "VALUE_DEFINE", value: "1")
+; CHECK-NEXT: !28 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !29, size: 192, dwarfAddressSpace: 0)
+; CHECK-NEXT: !29 = !DICompositeType(tag: DW_TAG_structure_type, name: "MyStruct", scope: !4, file: !1, size: 192, elements: !30, runtimeLang: DW_LANG_C89, identifier: "MyStruct")
+; CHECK-NEXT: !30 = !{!6, !6, !6}
+; CHECK-NEXT: !31 = distinct !DISubprogram(name: "foo", linkageName: "foo", scope: !1, file: !1, line: 42, type: !32, scopeLine: 42, spFlags: DISPFlagLocalToUnit | DISPFlagDefinition, unit: !0, retainedNodes: !37)
+; CHECK-NEXT: !32 = !DISubroutineType(types: !33)
+; CHECK-NEXT: !33 = !{!6, !6, !34}
+; CHECK-NEXT: !34 = !DICompositeType(tag: DW_TAG_array_type, baseType: !6, size: 640, flags: DIFlagVector, elements: !35)
+; CHECK-NEXT: !35 = !{!36}
+; CHECK-NEXT: !36 = !DISubrange(count: 10)
+; CHECK-NEXT: !37 = !{!38, !39, !40, !41}
+; CHECK-NEXT: !38 = !DILocalVariable(name: "a", arg: 1, scope: !31, file: !1, line: 42, type: !6)
+; CHECK-NEXT: !39 = !DILocalVariable(name: "b", arg: 2, scope: !31, file: !1, line: 42, type: !6)
+; CHECK-NEXT: !40 = !DILocalVariable(name: "c", arg: 3, scope: !31, file: !1, line: 42, type: !34)
+; CHECK-NEXT: !41 = !DILocalVariable(name: "d", scope: !42, file: !1, line: 43, type: !6)
+; CHECK-NEXT: !42 = distinct !DILexicalBlock(scope: !31, file: !1, line: 42)
+; CHECK-NEXT: !43 = !DILocation(line: 42, scope: !31)
+; CHECK-NEXT: !44 = !DILocation(line: 43, scope: !31)
diff --git a/llvm/test/Bindings/llvm-c/echo.ll b/llvm/test/Bindings/llvm-c/echo.ll
index bb6556e76..a244a65d9 100644
--- a/llvm/test/Bindings/llvm-c/echo.ll
+++ b/llvm/test/Bindings/llvm-c/echo.ll
@@ -21,6 +21,7 @@ module asm "classical GAS"
 @protected = protected global i32 23
 @section = global i32 27, section ".custom"
 @align = global i32 31, align 4
+@nullptr = global i32* null
 
 @aliased1 = alias i32, i32* @var
 @aliased2 = internal alias i32, i32* @var
@@ -138,6 +139,23 @@ done:
   ret i32 %p
 }
 
+define void @memops(i8* %ptr) {
+  %a = load i8, i8* %ptr
+  %b = load volatile i8, i8* %ptr
+  %c = load i8, i8* %ptr, align 8
+  %d = load atomic i8, i8* %ptr acquire, align 32
+  store i8 0, i8* %ptr
+  store volatile i8 0, i8* %ptr
+  store i8 0, i8* %ptr, align 8
+  store atomic i8 0, i8* %ptr release, align 32
+  %e = atomicrmw add i8* %ptr, i8 0 monotonic
+  %f = atomicrmw volatile xchg i8* %ptr, i8 0 acq_rel
+  %g = cmpxchg i8* %ptr, i8 1, i8 2 seq_cst acquire
+  %h = cmpxchg weak i8* %ptr, i8 1, i8 2 seq_cst acquire
+  %i = cmpxchg volatile i8* %ptr, i8 1, i8 2 monotonic monotonic
+  ret void
+}
+
 declare void @personalityFn()
 
 define void @exn() personality void ()* @personalityFn {
diff --git a/llvm/test/Bindings/llvm-c/freeze.ll b/llvm/test/Bindings/llvm-c/freeze.ll
new file mode 100644
index 000000000..4f3086f4c
--- /dev/null
+++ b/llvm/test/Bindings/llvm-c/freeze.ll
@@ -0,0 +1,23 @@
+; RUN: llvm-as < %s | llvm-dis > %t.orig
+; RUN: llvm-as < %s | llvm-c-test --echo > %t.echo
+; RUN: diff -w %t.orig %t.echo
+
+%struct.T = type { i32, i32 }
+
+define i32 @f(i32 %arg, <2 x i32> %arg2, float %arg3, <2 x float> %arg4,
+              i8* %arg5, %struct.T %arg6, [2 x i32] %arg7, { i32, i32 } %arg8) {
+  %1 = freeze i32 %arg
+  %2 = freeze i32 10
+  %3 = freeze i32 %1
+  %4 = freeze i32 undef
+  %5 = freeze i666 11
+  %6 = freeze <2 x i32> %arg2
+  %7 = freeze float %arg3
+  %8 = freeze <2 x float> %arg4
+  %9 = freeze i8* %arg5
+  %10 = freeze %struct.T %arg6
+  %11 = freeze [2 x i32] %arg7
+  %12 = freeze { i32, i32 } %arg8
+  %13 = freeze i8* null
+  ret i32 %1
+}
diff --git a/llvm/test/Bitcode/DILocation-implicit-code.ll b/llvm/test/Bitcode/DILocation-implicit-code.ll
index 03daa0feb..159cb8ae8 100644
--- a/llvm/test/Bitcode/DILocation-implicit-code.ll
+++ b/llvm/test/Bitcode/DILocation-implicit-code.ll
@@ -143,8 +143,8 @@ declare dso_local i8* @__cxa_allocate_exception(i64)
 
 declare dso_local void @__cxa_throw(i8*, i8*, i8*)
 
-attributes #0 = { noinline optnone uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { noinline nounwind optnone uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noinline optnone uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { noinline nounwind optnone uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind readnone }
 attributes #3 = { noreturn nounwind }
 attributes #4 = { nounwind }
diff --git a/llvm/test/Bitcode/aarch64-addp-upgrade.ll b/llvm/test/Bitcode/aarch64-addp-upgrade.ll
new file mode 100644
index 000000000..4e78996aa
--- /dev/null
+++ b/llvm/test/Bitcode/aarch64-addp-upgrade.ll
@@ -0,0 +1,18 @@
+; RUN: llvm-dis %p/aarch64-addp-upgrade.bc -o - | FileCheck %s
+
+; Bitcode was generated from file below, which may or may not even assemble any
+; more.
+
+; CHECK: call <2 x float> @llvm.aarch64.neon.faddp.v2f32(<2 x float> %lhs, <2 x float> %rhs)
+define <2 x float> @test_addp(<2 x float> %lhs, <2 x float> %rhs) {
+  %res = call <2 x float> @llvm.aarch64.neon.addp.v2f32(<2 x float> %lhs, <2 x float> %rhs)
+  ret <2 x float> %res
+}
+
+; CHECK: call <2 x float> @llvm.aarch64.neon.faddp.v2f32(<2 x float> %lhs, <2 x float> %rhs)
+define <2 x float> @test_addp1(<2 x float> %lhs, <2 x float> %rhs) {
+  %res = call <2 x float> @llvm.aarch64.neon.addp.v2f32(<2 x float> %lhs, <2 x float> %rhs)
+  ret <2 x float> %res
+}
+
+declare <2 x float> @llvm.aarch64.neon.addp.v2f32(<2 x float>, <2 x float>)
diff --git a/llvm/test/Bitcode/attributes-3.3.ll b/llvm/test/Bitcode/attributes-3.3.ll
index f1b37d199..dc7834eaa 100644
--- a/llvm/test/Bitcode/attributes-3.3.ll
+++ b/llvm/test/Bitcode/attributes-3.3.ll
@@ -5,14 +5,14 @@
 ; The test checks that LLVM does not silently misread attributes of
 ; older bitcode files.
 
-define void @f1(i8 zeroext)
-; CHECK: define void @f1(i8 zeroext)
+define void @f1(i8 zeroext %0)
+; CHECK: define void @f1(i8 zeroext %0)
 {
         ret void;
 }
 
-define void @f2(i8 signext)
-; CHECK: define void @f2(i8 signext)
+define void @f2(i8 signext %0)
+; CHECK: define void @f2(i8 signext %0)
 {
         ret void;
 }
@@ -23,14 +23,14 @@ define void @f3() noreturn
         ret void;
 }
 
-define void @f4(i8 inreg)
-; CHECK: define void @f4(i8 inreg)
+define void @f4(i8 inreg %0)
+; CHECK: define void @f4(i8 inreg %0)
 {
         ret void;
 }
 
-define void @f5(i8* sret)
-; CHECK: define void @f5(i8* sret)
+define void @f5(i8* sret %0)
+; CHECK: define void @f5(i8* sret %0)
 {
         ret void;
 }
@@ -41,20 +41,20 @@ define void @f6() nounwind
         ret void;
 }
 
-define void @f7(i8* noalias)
-; CHECK: define void @f7(i8* noalias)
+define void @f7(i8* noalias %0)
+; CHECK: define void @f7(i8* noalias %0)
 {
         ret void;
 }
 
-define void @f8(i8* byval)
-; CHECK: define void @f8(i8* byval(i8))
+define void @f8(i8* byval %0)
+; CHECK: define void @f8(i8* byval(i8) %0)
 {
         ret void;
 }
 
-define void @f9(i8* nest)
-; CHECK: define void @f9(i8* nest)
+define void @f9(i8* nest %0)
+; CHECK: define void @f9(i8* nest %0)
 {
         ret void;
 }
@@ -101,14 +101,14 @@ define void @f16() sspreq
         ret void;
 }
 
-define void @f17(i8 align 4)
-; CHECK: define void @f17(i8 align 4)
+define void @f17(i8 align 4 %0)
+; CHECK: define void @f17(i8 align 4 %0)
 {
         ret void;
 }
 
-define void @f18(i8* nocapture)
-; CHECK: define void @f18(i8* nocapture)
+define void @f18(i8* nocapture %0)
+; CHECK: define void @f18(i8* nocapture %0)
 {
         ret void;
 }
diff --git a/llvm/test/Bitcode/attributes.ll b/llvm/test/Bitcode/attributes.ll
index 03a98a58e..cc3d07608 100644
--- a/llvm/test/Bitcode/attributes.ll
+++ b/llvm/test/Bitcode/attributes.ll
@@ -2,14 +2,14 @@
 ; RUN: verify-uselistorder < %s
 ; PR12696
 
-define void @f1(i8 zeroext)
-; CHECK: define void @f1(i8 zeroext)
+define void @f1(i8 zeroext %0)
+; CHECK: define void @f1(i8 zeroext %0)
 {
         ret void;
 }
 
-define void @f2(i8 signext)
-; CHECK: define void @f2(i8 signext)
+define void @f2(i8 signext %0)
+; CHECK: define void @f2(i8 signext %0)
 {
         ret void;
 }
@@ -20,14 +20,14 @@ define void @f3() noreturn
         ret void;
 }
 
-define void @f4(i8 inreg)
-; CHECK: define void @f4(i8 inreg)
+define void @f4(i8 inreg %0)
+; CHECK: define void @f4(i8 inreg %0)
 {
         ret void;
 }
 
-define void @f5(i8* sret)
-; CHECK: define void @f5(i8* sret)
+define void @f5(i8* sret %0)
+; CHECK: define void @f5(i8* sret %0)
 {
         ret void;
 }
@@ -38,20 +38,20 @@ define void @f6() nounwind
         ret void;
 }
 
-define void @f7(i8* noalias)
-; CHECK: define void @f7(i8* noalias)
+define void @f7(i8* noalias %0)
+; CHECK: define void @f7(i8* noalias %0)
 {
         ret void;
 }
 
-define void @f8(i8* byval)
-; CHECK: define void @f8(i8* byval(i8))
+define void @f8(i8* byval %0)
+; CHECK: define void @f8(i8* byval(i8) %0)
 {
         ret void;
 }
 
-define void @f9(i8* nest)
-; CHECK: define void @f9(i8* nest)
+define void @f9(i8* nest %0)
+; CHECK: define void @f9(i8* nest %0)
 {
         ret void;
 }
@@ -98,14 +98,14 @@ define void @f16() sspreq
         ret void;
 }
 
-define void @f17(i8 align 4)
-; CHECK: define void @f17(i8 align 4)
+define void @f17(i8 align 4 %0)
+; CHECK: define void @f17(i8 align 4 %0)
 {
         ret void;
 }
 
-define void @f18(i8* nocapture)
-; CHECK: define void @f18(i8* nocapture)
+define void @f18(i8* nocapture %0)
+; CHECK: define void @f18(i8* nocapture %0)
 {
         ret void;
 }
@@ -214,8 +214,8 @@ define void @f35() optnone noinline
         ret void;
 }
 
-define void @f36(i8* inalloca) {
-; CHECK: define void @f36(i8* inalloca) {
+define void @f36(i8* inalloca %0) {
+; CHECK: define void @f36(i8* inalloca %0) {
         ret void
 }
 
@@ -240,8 +240,8 @@ define dereferenceable(18446744073709551606) i8* @f40(i8* dereferenceable(184467
         ret i8* %a
 }
 
-define void @f41(i8* align 32, double* align 64) {
-; CHECK: define void @f41(i8* align 32, double* align 64) {
+define void @f41(i8* align 32 %0, double* align 64 %1) {
+; CHECK: define void @f41(i8* align 32 %0, double* align 64 %1) {
         ret void
 }
 
@@ -262,13 +262,13 @@ define void @f44() argmemonly
         ret void;
 }
 
-; CHECK: define "string_attribute" void @f45(i32 "string_attribute")
-define "string_attribute" void @f45(i32 "string_attribute") {
+; CHECK: define "string_attribute" void @f45(i32 "string_attribute" %0)
+define "string_attribute" void @f45(i32 "string_attribute" %0) {
   ret void
 }
 
-; CHECK: define "string_attribute_with_value"="value" void @f46(i32 "string_attribute_with_value"="value")
-define "string_attribute_with_value"="value" void @f46(i32 "string_attribute_with_value"="value") {
+; CHECK: define "string_attribute_with_value"="value" void @f46(i32 "string_attribute_with_value"="value" %0)
+define "string_attribute_with_value"="value" void @f46(i32 "string_attribute_with_value"="value" %0) {
   ret void
 }
 
@@ -287,20 +287,20 @@ define void @f49() inaccessiblemem_or_argmemonly {
   ret void
 }
 
-; CHECK: define void @f50(i8* swiftself)
-define void @f50(i8* swiftself)
+; CHECK: define void @f50(i8* swiftself %0)
+define void @f50(i8* swiftself %0)
 {
   ret void;
 }
 
-; CHECK: define i32 @f51(i8** swifterror)
-define i32 @f51(i8** swifterror)
+; CHECK: define i32 @f51(i8** swifterror %0)
+define i32 @f51(i8** swifterror %0)
 {
   ret i32 0
 }
 
-; CHECK: define i32 @f52(i32, i8** swifterror)
-define i32 @f52(i32, i8** swifterror)
+; CHECK: define i32 @f52(i32 %0, i8** swifterror %1)
+define i32 @f52(i32 %0, i8** swifterror %1)
 {
   ret i32 0
 }
@@ -318,13 +318,13 @@ entry:
   ret float 1.0
 }
 
-; CHECK: define i8* @f54(i32) #30
-define i8* @f54(i32) allocsize(0) {
+; CHECK: define i8* @f54(i32 %0) #30
+define i8* @f54(i32 %0) allocsize(0) {
   ret i8* null
 }
 
-; CHECK: define i8* @f55(i32, i32) #31
-define i8* @f55(i32, i32) allocsize(0, 1) {
+; CHECK: define i8* @f55(i32 %0, i32 %1) #31
+define i8* @f55(i32 %0, i32 %1) allocsize(0, 1) {
   ret i8* null
 }
 
diff --git a/llvm/test/Bitcode/avr-calling-conventions.ll b/llvm/test/Bitcode/avr-calling-conventions.ll
index ea1005d14..1743b3324 100644
--- a/llvm/test/Bitcode/avr-calling-conventions.ll
+++ b/llvm/test/Bitcode/avr-calling-conventions.ll
@@ -1,16 +1,16 @@
 ; RUN: llvm-dis < %s.bc | FileCheck %s
 
-; CHECK: define avr_intrcc void @foo(i8)
-define avr_intrcc void @foo(i8) {
+; CHECK: define avr_intrcc void @foo(i8 %0)
+define avr_intrcc void @foo(i8 %0) {
   ret void
 }
 
-; CHECK: define avr_signalcc void @bar(i8)
-define avr_signalcc void @bar(i8) {
+; CHECK: define avr_signalcc void @bar(i8 %0)
+define avr_signalcc void @bar(i8 %0) {
   ret void
 }
 
-; CHECK: define void @baz(i8)
-define void @baz(i8) {
+; CHECK: define void @baz(i8 %0)
+define void @baz(i8 %0) {
   ret void
 }
diff --git a/llvm/test/Bitcode/calling-conventions.3.2.ll b/llvm/test/Bitcode/calling-conventions.3.2.ll
index b60f1d70c..d968d802b 100644
--- a/llvm/test/Bitcode/calling-conventions.3.2.ll
+++ b/llvm/test/Bitcode/calling-conventions.3.2.ll
@@ -26,6 +26,9 @@ declare spir_func void @spir_func()
 declare intel_ocl_bicc void @intel_ocl_bicc()
 ; CHECK: declare intel_ocl_bicc void @intel_ocl_bicc
 
+declare cfguard_checkcc void @cfguard_checkcc()
+; CHECK: declare cfguard_checkcc void @cfguard_checkcc
+
 declare x86_stdcallcc void @x86_stdcallcc()
 ; CHECK: declare x86_stdcallcc void @x86_stdcallcc 
 
@@ -95,6 +98,12 @@ define void @call_intel_ocl_bicc() {
   ret void
 }
 
+define void @call_cfguard_checkcc() {
+; CHECK: call cfguard_checkcc void @cfguard_checkcc
+  call cfguard_checkcc void @cfguard_checkcc()
+  ret void
+}
+
 define void @call_x86_stdcallcc() { 
 ; CHECK: call x86_stdcallcc void @x86_stdcallcc
   call x86_stdcallcc void @x86_stdcallcc()
diff --git a/llvm/test/Bitcode/compatibility-3.6.ll.x b/llvm/test/Bitcode/compatibility-3.6.ll.x
index 322c95b02..6bfd4e5a7 100644
--- a/llvm/test/Bitcode/compatibility-3.6.ll.x
+++ b/llvm/test/Bitcode/compatibility-3.6.ll.x
@@ -981,7 +981,7 @@ exit:
   ; CHECK: select <2 x i1> <i1 true, i1 false>, <2 x i8> <i8 2, i8 3>, <2 x i8> <i8 3, i8 2>
 
   call void @f.nobuiltin() builtin
-  ; CHECK: call void @f.nobuiltin() #34
+  ; CHECK: call void @f.nobuiltin() #35
 
   call fastcc noalias i32* @f.noalias() noinline
   ; CHECK: call fastcc noalias i32* @f.noalias() #11
@@ -1116,7 +1116,7 @@ define void @intrinsics.codegen() {
   call i8* @llvm.returnaddress(i32 1)
   ; CHECK: call i8* @llvm.returnaddress(i32 1)
   call i8* @llvm.frameaddress(i32 1)
-  ; CHECK: call i8* @llvm.frameaddress(i32 1)
+  ; CHECK: call i8* @llvm.frameaddress.p0i8(i32 1)
 
   call i32 @llvm.read_register.i32(metadata !10)
   ; CHECK: call i32 @llvm.read_register.i32(metadata !10)
@@ -1133,7 +1133,7 @@ define void @intrinsics.codegen() {
   ; CHECK: call void @llvm.stackrestore(i8* %stack)
 
   call void @llvm.prefetch(i8* %stack, i32 0, i32 3, i32 0)
-  ; CHECK: call void @llvm.prefetch(i8* %stack, i32 0, i32 3, i32 0)
+  ; CHECK: call void @llvm.prefetch.p0i8(i8* %stack, i32 0, i32 3, i32 0)
 
   call void @llvm.pcmarker(i32 1)
   ; CHECK: call void @llvm.pcmarker(i32 1)
@@ -1179,12 +1179,13 @@ define void @intrinsics.codegen() {
 ; CHECK: attributes #26 = { sspstrong }
 ; CHECK: attributes #27 = { uwtable }
 ; CHECK: attributes #28 = { "cpu"="cortex-a8" }
-; CHECK: attributes #29 = { nounwind readnone }
+; CHECK: attributes #29 = { nounwind readnone willreturn }
 ; CHECK: attributes #30 = { argmemonly nounwind readonly }
 ; CHECK: attributes #31 = { argmemonly nounwind }
-; CHECK: attributes #32 = { nounwind readonly }
-; CHECK: attributes #33 = { inaccessiblemem_or_argmemonly nounwind }
-; CHECK: attributes #34 = { builtin }
+; CHECK: attributes #32 = { nounwind readnone }
+; CHECK: attributes #33 = { nounwind readonly }
+; CHECK: attributes #34 = { inaccessiblemem_or_argmemonly nounwind willreturn }
+; CHECK: attributes #35 = { builtin }
 
 ;; Metadata
 
diff --git a/llvm/test/Bitcode/compatibility-3.7.ll.x b/llvm/test/Bitcode/compatibility-3.7.ll.x
index e82607413..9117b1ce5 100644
--- a/llvm/test/Bitcode/compatibility-3.7.ll.x
+++ b/llvm/test/Bitcode/compatibility-3.7.ll.x
@@ -1022,7 +1022,7 @@ exit:
   ; CHECK: select <2 x i1> <i1 true, i1 false>, <2 x i8> <i8 2, i8 3>, <2 x i8> <i8 3, i8 2>
 
   call void @f.nobuiltin() builtin
-  ; CHECK: call void @f.nobuiltin() #37
+  ; CHECK: call void @f.nobuiltin() #38
 
   call fastcc noalias i32* @f.noalias() noinline
   ; CHECK: call fastcc noalias i32* @f.noalias() #12
@@ -1147,7 +1147,7 @@ define void @intrinsics.codegen() {
   call i8* @llvm.returnaddress(i32 1)
   ; CHECK: call i8* @llvm.returnaddress(i32 1)
   call i8* @llvm.frameaddress(i32 1)
-  ; CHECK: call i8* @llvm.frameaddress(i32 1)
+  ; CHECK: call i8* @llvm.frameaddress.p0i8(i32 1)
 
   call i32 @llvm.read_register.i32(metadata !10)
   ; CHECK: call i32 @llvm.read_register.i32(metadata !10)
@@ -1164,7 +1164,7 @@ define void @intrinsics.codegen() {
   ; CHECK: call void @llvm.stackrestore(i8* %stack)
 
   call void @llvm.prefetch(i8* %stack, i32 0, i32 3, i32 0)
-  ; CHECK: call void @llvm.prefetch(i8* %stack, i32 0, i32 3, i32 0)
+  ; CHECK: call void @llvm.prefetch.p0i8(i8* %stack, i32 0, i32 3, i32 0)
 
   call void @llvm.pcmarker(i32 1)
   ; CHECK: call void @llvm.pcmarker(i32 1)
@@ -1242,12 +1242,13 @@ define void @misc.metadata() {
 ; CHECK: attributes #29 = { "thunk" }
 ; CHECK: attributes #30 = { uwtable }
 ; CHECK: attributes #31 = { "cpu"="cortex-a8" }
-; CHECK: attributes #32 = { nounwind readnone }
+; CHECK: attributes #32 = { nounwind readnone willreturn }
 ; CHECK: attributes #33 = { argmemonly nounwind readonly }
 ; CHECK: attributes #34 = { argmemonly nounwind }
-; CHECK: attributes #35 = { nounwind readonly }
-; CHECK: attributes #36 = { inaccessiblemem_or_argmemonly nounwind }
-; CHECK: attributes #37 = { builtin }
+; CHECK: attributes #35 = { nounwind readnone }
+; CHECK: attributes #36 = { nounwind readonly }
+; CHECK: attributes #37 = { inaccessiblemem_or_argmemonly nounwind willreturn }
+; CHECK: attributes #38 = { builtin }
 
 ;; Metadata
 
diff --git a/llvm/test/Bitcode/compatibility-3.8.ll b/llvm/test/Bitcode/compatibility-3.8.ll
index c8d826191..c8f195940 100644
--- a/llvm/test/Bitcode/compatibility-3.8.ll
+++ b/llvm/test/Bitcode/compatibility-3.8.ll
@@ -1170,7 +1170,7 @@ exit:
   ; CHECK: select <2 x i1> <i1 true, i1 false>, <2 x i8> <i8 2, i8 3>, <2 x i8> <i8 3, i8 2>
 
   call void @f.nobuiltin() builtin
-  ; CHECK: call void @f.nobuiltin() #40
+  ; CHECK: call void @f.nobuiltin() #41
 
   call fastcc noalias i32* @f.noalias() noinline
   ; CHECK: call fastcc noalias i32* @f.noalias() #12
@@ -1302,7 +1302,7 @@ define void @intrinsics.codegen() {
   call i8* @llvm.returnaddress(i32 1)
   ; CHECK: call i8* @llvm.returnaddress(i32 1)
   call i8* @llvm.frameaddress(i32 1)
-  ; CHECK: call i8* @llvm.frameaddress(i32 1)
+  ; CHECK: call i8* @llvm.frameaddress.p0i8(i32 1)
 
   call i32 @llvm.read_register.i32(metadata !10)
   ; CHECK: call i32 @llvm.read_register.i32(metadata !10)
@@ -1319,7 +1319,7 @@ define void @intrinsics.codegen() {
   ; CHECK: call void @llvm.stackrestore(i8* %stack)
 
   call void @llvm.prefetch(i8* %stack, i32 0, i32 3, i32 0)
-  ; CHECK: call void @llvm.prefetch(i8* %stack, i32 0, i32 3, i32 0)
+  ; CHECK: call void @llvm.prefetch.p0i8(i8* %stack, i32 0, i32 3, i32 0)
 
   call void @llvm.pcmarker(i32 1)
   ; CHECK: call void @llvm.pcmarker(i32 1)
@@ -1552,12 +1552,13 @@ normal:
 ; CHECK: attributes #32 = { norecurse }
 ; CHECK: attributes #33 = { inaccessiblememonly }
 ; CHECK: attributes #34 = { inaccessiblemem_or_argmemonly }
-; CHECK: attributes #35 = { nounwind readnone }
+; CHECK: attributes #35 = { nounwind readnone willreturn }
 ; CHECK: attributes #36 = { argmemonly nounwind readonly }
 ; CHECK: attributes #37 = { argmemonly nounwind }
-; CHECK: attributes #38 = { nounwind readonly }
-; CHECK: attributes #39 = { inaccessiblemem_or_argmemonly nounwind }
-; CHECK: attributes #40 = { builtin }
+; CHECK: attributes #38 = { nounwind readnone }
+; CHECK: attributes #39 = { nounwind readonly }
+; CHECK: attributes #40 = { inaccessiblemem_or_argmemonly nounwind willreturn }
+; CHECK: attributes #41 = { builtin }
 
 ;; Metadata
 
diff --git a/llvm/test/Bitcode/compatibility-3.9.ll b/llvm/test/Bitcode/compatibility-3.9.ll
index f031e54a9..b251aaec7 100644
--- a/llvm/test/Bitcode/compatibility-3.9.ll
+++ b/llvm/test/Bitcode/compatibility-3.9.ll
@@ -1241,7 +1241,7 @@ exit:
   ; CHECK: select <2 x i1> <i1 true, i1 false>, <2 x i8> <i8 2, i8 3>, <2 x i8> <i8 3, i8 2>
 
   call void @f.nobuiltin() builtin
-  ; CHECK: call void @f.nobuiltin() #41
+  ; CHECK: call void @f.nobuiltin() #42
 
   call fastcc noalias i32* @f.noalias() noinline
   ; CHECK: call fastcc noalias i32* @f.noalias() #12
@@ -1373,7 +1373,7 @@ define void @intrinsics.codegen() {
   call i8* @llvm.returnaddress(i32 1)
   ; CHECK: call i8* @llvm.returnaddress(i32 1)
   call i8* @llvm.frameaddress(i32 1)
-  ; CHECK: call i8* @llvm.frameaddress(i32 1)
+  ; CHECK: call i8* @llvm.frameaddress.p0i8(i32 1)
 
   call i32 @llvm.read_register.i32(metadata !10)
   ; CHECK: call i32 @llvm.read_register.i32(metadata !10)
@@ -1390,7 +1390,7 @@ define void @intrinsics.codegen() {
   ; CHECK: call void @llvm.stackrestore(i8* %stack)
 
   call void @llvm.prefetch(i8* %stack, i32 0, i32 3, i32 0)
-  ; CHECK: call void @llvm.prefetch(i8* %stack, i32 0, i32 3, i32 0)
+  ; CHECK: call void @llvm.prefetch.p0i8(i8* %stack, i32 0, i32 3, i32 0)
 
   call void @llvm.pcmarker(i32 1)
   ; CHECK: call void @llvm.pcmarker(i32 1)
@@ -1625,13 +1625,14 @@ declare void @f.writeonly() writeonly
 ; CHECK: attributes #32 = { norecurse }
 ; CHECK: attributes #33 = { inaccessiblememonly }
 ; CHECK: attributes #34 = { inaccessiblemem_or_argmemonly }
-; CHECK: attributes #35 = { nounwind readnone }
+; CHECK: attributes #35 = { nounwind readnone willreturn }
 ; CHECK: attributes #36 = { argmemonly nounwind readonly }
 ; CHECK: attributes #37 = { argmemonly nounwind }
-; CHECK: attributes #38 = { nounwind readonly }
-; CHECK: attributes #39 = { inaccessiblemem_or_argmemonly nounwind }
+; CHECK: attributes #38 = { nounwind readnone }
+; CHECK: attributes #39 = { nounwind readonly }
 ; CHECK: attributes #40 = { writeonly }
-; CHECK: attributes #41 = { builtin }
+; CHECK: attributes #41 = { inaccessiblemem_or_argmemonly nounwind willreturn }
+; CHECK: attributes #42 = { builtin }
 
 ;; Metadata
 
diff --git a/llvm/test/Bitcode/compatibility-4.0.ll b/llvm/test/Bitcode/compatibility-4.0.ll
index 8020df45d..ba4b196ac 100644
--- a/llvm/test/Bitcode/compatibility-4.0.ll
+++ b/llvm/test/Bitcode/compatibility-4.0.ll
@@ -1241,7 +1241,7 @@ exit:
   ; CHECK: select <2 x i1> <i1 true, i1 false>, <2 x i8> <i8 2, i8 3>, <2 x i8> <i8 3, i8 2>
 
   call void @f.nobuiltin() builtin
-  ; CHECK: call void @f.nobuiltin() #41
+  ; CHECK: call void @f.nobuiltin() #42
 
   call fastcc noalias i32* @f.noalias() noinline
   ; CHECK: call fastcc noalias i32* @f.noalias() #12
@@ -1373,7 +1373,7 @@ define void @intrinsics.codegen() {
   call i8* @llvm.returnaddress(i32 1)
   ; CHECK: call i8* @llvm.returnaddress(i32 1)
   call i8* @llvm.frameaddress(i32 1)
-  ; CHECK: call i8* @llvm.frameaddress(i32 1)
+  ; CHECK: call i8* @llvm.frameaddress.p0i8(i32 1)
 
   call i32 @llvm.read_register.i32(metadata !10)
   ; CHECK: call i32 @llvm.read_register.i32(metadata !10)
@@ -1390,7 +1390,7 @@ define void @intrinsics.codegen() {
   ; CHECK: call void @llvm.stackrestore(i8* %stack)
 
   call void @llvm.prefetch(i8* %stack, i32 0, i32 3, i32 0)
-  ; CHECK: call void @llvm.prefetch(i8* %stack, i32 0, i32 3, i32 0)
+  ; CHECK: call void @llvm.prefetch.p0i8(i8* %stack, i32 0, i32 3, i32 0)
 
   call void @llvm.pcmarker(i32 1)
   ; CHECK: call void @llvm.pcmarker(i32 1)
@@ -1650,13 +1650,14 @@ define i8** @constexpr() {
 ; CHECK: attributes #32 = { norecurse }
 ; CHECK: attributes #33 = { inaccessiblememonly }
 ; CHECK: attributes #34 = { inaccessiblemem_or_argmemonly }
-; CHECK: attributes #35 = { nounwind readnone }
+; CHECK: attributes #35 = { nounwind readnone willreturn }
 ; CHECK: attributes #36 = { argmemonly nounwind readonly }
 ; CHECK: attributes #37 = { argmemonly nounwind }
-; CHECK: attributes #38 = { nounwind readonly }
-; CHECK: attributes #39 = { inaccessiblemem_or_argmemonly nounwind }
+; CHECK: attributes #38 = { nounwind readnone }
+; CHECK: attributes #39 = { nounwind readonly }
 ; CHECK: attributes #40 = { writeonly }
-; CHECK: attributes #41 = { builtin }
+; CHECK: attributes #41 = { inaccessiblemem_or_argmemonly nounwind willreturn }
+; CHECK: attributes #42 = { builtin }
 
 ;; Metadata
 
diff --git a/llvm/test/Bitcode/compatibility-5.0.ll b/llvm/test/Bitcode/compatibility-5.0.ll
index 3b32d668a..dffe593ac 100644
--- a/llvm/test/Bitcode/compatibility-5.0.ll
+++ b/llvm/test/Bitcode/compatibility-5.0.ll
@@ -1248,10 +1248,10 @@ exit:
   ; CHECK: select <2 x i1> <i1 true, i1 false>, <2 x i8> <i8 2, i8 3>, <2 x i8> <i8 3, i8 2>
 
   call void @f.nobuiltin() builtin
-  ; CHECK: call void @f.nobuiltin() #42
+  ; CHECK: call void @f.nobuiltin() #43
 
   call void @f.strictfp() strictfp
-  ; CHECK: call void @f.strictfp() #43
+  ; CHECK: call void @f.strictfp() #44
 
   call fastcc noalias i32* @f.noalias() noinline
   ; CHECK: call fastcc noalias i32* @f.noalias() #12
@@ -1383,7 +1383,7 @@ define void @intrinsics.codegen() {
   call i8* @llvm.returnaddress(i32 1)
   ; CHECK: call i8* @llvm.returnaddress(i32 1)
   call i8* @llvm.frameaddress(i32 1)
-  ; CHECK: call i8* @llvm.frameaddress(i32 1)
+  ; CHECK: call i8* @llvm.frameaddress.p0i8(i32 1)
 
   call i32 @llvm.read_register.i32(metadata !10)
   ; CHECK: call i32 @llvm.read_register.i32(metadata !10)
@@ -1400,7 +1400,7 @@ define void @intrinsics.codegen() {
   ; CHECK: call void @llvm.stackrestore(i8* %stack)
 
   call void @llvm.prefetch(i8* %stack, i32 0, i32 3, i32 0)
-  ; CHECK: call void @llvm.prefetch(i8* %stack, i32 0, i32 3, i32 0)
+  ; CHECK: call void @llvm.prefetch.p0i8(i8* %stack, i32 0, i32 3, i32 0)
 
   call void @llvm.pcmarker(i32 1)
   ; CHECK: call void @llvm.pcmarker(i32 1)
@@ -1663,15 +1663,16 @@ define i8** @constexpr() {
 ; CHECK: attributes #32 = { norecurse }
 ; CHECK: attributes #33 = { inaccessiblememonly }
 ; CHECK: attributes #34 = { inaccessiblemem_or_argmemonly }
-; CHECK: attributes #35 = { nounwind readnone }
+; CHECK: attributes #35 = { nounwind readnone willreturn }
 ; CHECK: attributes #36 = { argmemonly nounwind readonly }
 ; CHECK: attributes #37 = { argmemonly nounwind }
-; CHECK: attributes #38 = { nounwind readonly }
-; CHECK: attributes #39 = { inaccessiblemem_or_argmemonly nounwind }
+; CHECK: attributes #38 = { nounwind readnone }
+; CHECK: attributes #39 = { nounwind readonly }
 ; CHECK: attributes #40 = { writeonly }
 ; CHECK: attributes #41 = { speculatable }
-; CHECK: attributes #42 = { builtin }
-; CHECK: attributes #43 = { strictfp }
+; CHECK: attributes #42 = { inaccessiblemem_or_argmemonly nounwind willreturn }
+; CHECK: attributes #43 = { builtin }
+; CHECK: attributes #44 = { strictfp }
 
 ;; Metadata
 
diff --git a/llvm/test/Bitcode/compatibility-6.0.ll b/llvm/test/Bitcode/compatibility-6.0.ll
index 3b08c4a72..2b160558d 100644
--- a/llvm/test/Bitcode/compatibility-6.0.ll
+++ b/llvm/test/Bitcode/compatibility-6.0.ll
@@ -1259,10 +1259,10 @@ exit:
   ; CHECK: select <2 x i1> <i1 true, i1 false>, <2 x i8> <i8 2, i8 3>, <2 x i8> <i8 3, i8 2>
 
   call void @f.nobuiltin() builtin
-  ; CHECK: call void @f.nobuiltin() #42
+  ; CHECK: call void @f.nobuiltin() #43
 
   call void @f.strictfp() strictfp
-  ; CHECK: call void @f.strictfp() #43
+  ; CHECK: call void @f.strictfp() #44
 
   call fastcc noalias i32* @f.noalias() noinline
   ; CHECK: call fastcc noalias i32* @f.noalias() #12
@@ -1394,7 +1394,7 @@ define void @intrinsics.codegen() {
   call i8* @llvm.returnaddress(i32 1)
   ; CHECK: call i8* @llvm.returnaddress(i32 1)
   call i8* @llvm.frameaddress(i32 1)
-  ; CHECK: call i8* @llvm.frameaddress(i32 1)
+  ; CHECK: call i8* @llvm.frameaddress.p0i8(i32 1)
 
   call i32 @llvm.read_register.i32(metadata !10)
   ; CHECK: call i32 @llvm.read_register.i32(metadata !10)
@@ -1411,7 +1411,7 @@ define void @intrinsics.codegen() {
   ; CHECK: call void @llvm.stackrestore(i8* %stack)
 
   call void @llvm.prefetch(i8* %stack, i32 0, i32 3, i32 0)
-  ; CHECK: call void @llvm.prefetch(i8* %stack, i32 0, i32 3, i32 0)
+  ; CHECK: call void @llvm.prefetch.p0i8(i8* %stack, i32 0, i32 3, i32 0)
 
   call void @llvm.pcmarker(i32 1)
   ; CHECK: call void @llvm.pcmarker(i32 1)
@@ -1674,15 +1674,16 @@ define i8** @constexpr() {
 ; CHECK: attributes #32 = { norecurse }
 ; CHECK: attributes #33 = { inaccessiblememonly }
 ; CHECK: attributes #34 = { inaccessiblemem_or_argmemonly }
-; CHECK: attributes #35 = { nounwind readnone }
+; CHECK: attributes #35 = { nounwind readnone willreturn }
 ; CHECK: attributes #36 = { argmemonly nounwind readonly }
 ; CHECK: attributes #37 = { argmemonly nounwind }
-; CHECK: attributes #38 = { nounwind readonly }
-; CHECK: attributes #39 = { inaccessiblemem_or_argmemonly nounwind }
+; CHECK: attributes #38 = { nounwind readnone }
+; CHECK: attributes #39 = { nounwind readonly }
 ; CHECK: attributes #40 = { writeonly }
 ; CHECK: attributes #41 = { speculatable }
-; CHECK: attributes #42 = { builtin }
-; CHECK: attributes #43 = { strictfp }
+; CHECK: attributes #42 = { inaccessiblemem_or_argmemonly nounwind willreturn }
+; CHECK: attributes #43 = { builtin }
+; CHECK: attributes #44 = { strictfp }
 
 ;; Metadata
 
diff --git a/llvm/test/Bitcode/compatibility.ll b/llvm/test/Bitcode/compatibility.ll
index 154786e08..055397f12 100644
--- a/llvm/test/Bitcode/compatibility.ll
+++ b/llvm/test/Bitcode/compatibility.ll
@@ -665,15 +665,12 @@ entry:
   ret void
 }
 
-; Note: we skip align on FuncHeader for now, since the grammar contains a LR
-; conflict.
-
 ; Functions -- align
-;declare void @f.align2() align 2
+declare void @f.align2() align 2
 ; CHECK: declare void @f.align2() align 2
-;declare void @f.align4() align 4
+declare void @f.align4() align 4
 ; CHECK: declare void @f.align4() align 4
-;declare void @f.align8() align 8
+declare void @f.align8() align 8
 ; CHECK: declare void @f.align8() align 8
 
 ; Functions -- GC
@@ -864,24 +861,115 @@ define void @fastmathflags_vector_select(<2 x i1> %cond, <2 x double> %op1, <2 x
   ret void
 }
 
+define void @fastmathflags_array_select(i1 %cond, [2 x double] %op1, [2 x double] %op2) {
+  %f.nnan.nsz = select nnan nsz i1 %cond, [2 x double] %op1, [2 x double] %op2
+  ; CHECK: %f.nnan.nsz = select nnan nsz i1 %cond, [2 x double] %op1, [2 x double] %op2
+  %f.fast = select fast i1 %cond, [2 x double] %op1, [2 x double] %op2
+  ; CHECK: %f.fast = select fast i1 %cond, [2 x double] %op1, [2 x double] %op2
+  ret void
+}
+
+define void @fastmathflags_phi(i1 %cond, float %f1, float %f2, double %d1, double %d2, half %h1, half %h2) {
+entry:
+  br i1 %cond, label %L1, label %L2
+L1:
+  br label %exit
+L2:
+  br label %exit
+exit:
+  %p.nnan = phi nnan float [ %f1, %L1 ], [ %f2, %L2 ]
+  ; CHECK: %p.nnan = phi nnan float [ %f1, %L1 ], [ %f2, %L2 ]
+  %p.ninf = phi ninf double [ %d1, %L1 ], [ %d2, %L2 ]
+  ; CHECK: %p.ninf = phi ninf double [ %d1, %L1 ], [ %d2, %L2 ]
+  %p.contract = phi contract half [ %h1, %L1 ], [ %h2, %L2 ]
+  ; CHECK: %p.contract = phi contract half [ %h1, %L1 ], [ %h2, %L2 ]
+  %p.nsz.reassoc = phi reassoc nsz float [ %f1, %L1 ], [ %f2, %L2 ]
+  ; CHECK: %p.nsz.reassoc = phi reassoc nsz float [ %f1, %L1 ], [ %f2, %L2 ]
+  %p.fast = phi fast half [ %h2, %L1 ], [ %h1, %L2 ]
+  ; CHECK: %p.fast = phi fast half [ %h2, %L1 ], [ %h1, %L2 ]
+  ret void
+}
+
+define void @fastmathflags_vector_phi(i1 %cond, <4 x float> %f1, <4 x float> %f2, <2 x double> %d1, <2 x double> %d2, <8 x half> %h1, <8 x half> %h2) {
+entry:
+  br i1 %cond, label %L1, label %L2
+L1:
+  br label %exit
+L2:
+  br label %exit
+exit:
+  %p.nnan = phi nnan <4 x float> [ %f1, %L1 ], [ %f2, %L2 ]
+  ; CHECK: %p.nnan = phi nnan <4 x float> [ %f1, %L1 ], [ %f2, %L2 ]
+  %p.ninf = phi ninf <2 x double> [ %d1, %L1 ], [ %d2, %L2 ]
+  ; CHECK: %p.ninf = phi ninf <2 x double> [ %d1, %L1 ], [ %d2, %L2 ]
+  %p.contract = phi contract <8 x half> [ %h1, %L1 ], [ %h2, %L2 ]
+  ; CHECK: %p.contract = phi contract <8 x half> [ %h1, %L1 ], [ %h2, %L2 ]
+  %p.nsz.reassoc = phi reassoc nsz <4 x float> [ %f1, %L1 ], [ %f2, %L2 ]
+  ; CHECK: %p.nsz.reassoc = phi reassoc nsz <4 x float> [ %f1, %L1 ], [ %f2, %L2 ]
+  %p.fast = phi fast <8 x half> [ %h2, %L1 ], [ %h1, %L2 ]
+  ; CHECK: %p.fast = phi fast <8 x half> [ %h2, %L1 ], [ %h1, %L2 ]
+  ret void
+}
+
+define void @fastmathflags_array_phi(i1 %cond, [4 x float] %f1, [4 x float] %f2, [2 x double] %d1, [2 x double] %d2, [8 x half] %h1, [8 x half] %h2) {
+entry:
+  br i1 %cond, label %L1, label %L2
+L1:
+  br label %exit
+L2:
+  br label %exit
+exit:
+  %p.nnan = phi nnan [4 x float] [ %f1, %L1 ], [ %f2, %L2 ]
+  ; CHECK: %p.nnan = phi nnan [4 x float] [ %f1, %L1 ], [ %f2, %L2 ]
+  %p.ninf = phi ninf [2 x double] [ %d1, %L1 ], [ %d2, %L2 ]
+  ; CHECK: %p.ninf = phi ninf [2 x double] [ %d1, %L1 ], [ %d2, %L2 ]
+  %p.contract = phi contract [8 x half] [ %h1, %L1 ], [ %h2, %L2 ]
+  ; CHECK: %p.contract = phi contract [8 x half] [ %h1, %L1 ], [ %h2, %L2 ]
+  %p.nsz.reassoc = phi reassoc nsz [4 x float] [ %f1, %L1 ], [ %f2, %L2 ]
+  ; CHECK: %p.nsz.reassoc = phi reassoc nsz [4 x float] [ %f1, %L1 ], [ %f2, %L2 ]
+  %p.fast = phi fast [8 x half] [ %h2, %L1 ], [ %h1, %L2 ]
+  ; CHECK: %p.fast = phi fast [8 x half] [ %h2, %L1 ], [ %h1, %L2 ]
+  ret void
+}
+
 ; Check various fast math flags and floating-point types on calls.
 
-declare float @fmf1()
-declare double @fmf2()
-declare <4 x double> @fmf3()
+declare float @fmf_f32()
+declare double @fmf_f64()
+declare <4 x double> @fmf_v4f64()
 
 ; CHECK-LABEL: fastMathFlagsForCalls(
 define void @fastMathFlagsForCalls(float %f, double %d1, <4 x double> %d2) {
-  %call.fast = call fast float @fmf1()
-  ; CHECK: %call.fast = call fast float @fmf1()
+  %call.fast = call fast float @fmf_f32()
+  ; CHECK: %call.fast = call fast float @fmf_f32()
+
+  ; Throw in some other attributes to make sure those stay in the right places.
+
+  %call.nsz.arcp = notail call nsz arcp double @fmf_f64()
+  ; CHECK: %call.nsz.arcp = notail call nsz arcp double @fmf_f64()
+
+  %call.nnan.ninf = tail call nnan ninf fastcc <4 x double> @fmf_v4f64()
+  ; CHECK: %call.nnan.ninf = tail call nnan ninf fastcc <4 x double> @fmf_v4f64()
+
+  ret void
+}
+
+declare [2 x float] @fmf_a2f32()
+declare [2 x double] @fmf_a2f64()
+declare [2 x <4 x double>] @fmf_a2v4f64()
+
+; CHECK-LABEL: fastMathFlagsForArrayCalls(
+define void @fastMathFlagsForArrayCalls([2 x float] %f, [2 x double] %d1, [2 x <4 x double>] %d2) {
+  %call.fast = call fast [2 x float] @fmf_a2f32()
+  ; CHECK: %call.fast = call fast [2 x float] @fmf_a2f32()
 
   ; Throw in some other attributes to make sure those stay in the right places.
 
-  %call.nsz.arcp = notail call nsz arcp double @fmf2()
-  ; CHECK: %call.nsz.arcp = notail call nsz arcp double @fmf2()
+  %call.nsz.arcp = notail call nsz arcp [2 x double] @fmf_a2f64()
+  ; CHECK: %call.nsz.arcp = notail call nsz arcp [2 x double] @fmf_a2f64()
 
-  %call.nnan.ninf = tail call nnan ninf fastcc <4 x double> @fmf3()
-  ; CHECK: %call.nnan.ninf = tail call nnan ninf fastcc <4 x double> @fmf3()
+  %call.nnan.ninf = tail call nnan ninf fastcc [2 x <4 x double>] @fmf_a2v4f64()
+  ; CHECK: %call.nnan.ninf = tail call nnan ninf fastcc [2 x <4 x double>] @fmf_a2v4f64()
 
   ret void
 }
@@ -1281,7 +1369,7 @@ define void @instructions.conversions() {
 }
 
 ; Instructions -- Other Operations
-define void @instructions.other(i32 %op1, i32 %op2, half %fop1, half %fop2) {
+define void @instructions.other(i32 %op1, i32 %op2, half %fop1, half %fop2, <2 x i32> %vop, i8* %pop) {
 entry:
   icmp eq  i32 %op1, %op2
   ; CHECK: icmp eq  i32 %op1, %op2
@@ -1354,16 +1442,23 @@ exit:
   ; CHECK: select <2 x i1> <i1 true, i1 false>, <2 x i8> <i8 2, i8 3>, <2 x i8> <i8 3, i8 2>
 
   call void @f.nobuiltin() builtin
-  ; CHECK: call void @f.nobuiltin() #42
-
-  call void @f.strictfp() strictfp
-  ; CHECK: call void @f.strictfp() #43
+  ; CHECK: call void @f.nobuiltin() #44
 
   call fastcc noalias i32* @f.noalias() noinline
   ; CHECK: call fastcc noalias i32* @f.noalias() #12
   tail call ghccc nonnull i32* @f.nonnull() minsize
   ; CHECK: tail call ghccc nonnull i32* @f.nonnull() #7
 
+  freeze i32 %op1
+  ; CHECK: freeze i32 %op1
+  freeze i32 10
+  ; CHECK: freeze i32 10
+  freeze half %fop1
+  ; CHECK: freeze half %fop1
+  freeze <2 x i32> %vop
+  ; CHECK: freeze <2 x i32> %vop
+  freeze i8* %pop
+  ; CHECK: freeze i8* %pop
   ret void
 }
 
@@ -1478,7 +1573,7 @@ declare void @llvm.write_register.i32(metadata, i32)
 declare void @llvm.write_register.i64(metadata, i64)
 declare i8* @llvm.stacksave()
 declare void @llvm.stackrestore(i8*)
-declare void @llvm.prefetch(i8*, i32, i32, i32)
+declare void @llvm.prefetch.p0i8(i8*, i32, i32, i32)
 declare void @llvm.pcmarker(i32)
 declare i64 @llvm.readcyclecounter()
 declare void @llvm.clear_cache(i8*, i8*)
@@ -1489,7 +1584,7 @@ define void @intrinsics.codegen() {
   call i8* @llvm.returnaddress(i32 1)
   ; CHECK: call i8* @llvm.returnaddress(i32 1)
   call i8* @llvm.frameaddress(i32 1)
-  ; CHECK: call i8* @llvm.frameaddress(i32 1)
+  ; CHECK: call i8* @llvm.frameaddress.p0i8(i32 1)
 
   call i32 @llvm.read_register.i32(metadata !10)
   ; CHECK: call i32 @llvm.read_register.i32(metadata !10)
@@ -1505,8 +1600,8 @@ define void @intrinsics.codegen() {
   call void @llvm.stackrestore(i8* %stack)
   ; CHECK: call void @llvm.stackrestore(i8* %stack)
 
-  call void @llvm.prefetch(i8* %stack, i32 0, i32 3, i32 0)
-  ; CHECK: call void @llvm.prefetch(i8* %stack, i32 0, i32 3, i32 0)
+  call void @llvm.prefetch.p0i8(i8* %stack, i32 0, i32 3, i32 0)
+  ; CHECK: call void @llvm.prefetch.p0i8(i8* %stack, i32 0, i32 3, i32 0)
 
   call void @llvm.pcmarker(i32 1)
   ; CHECK: call void @llvm.pcmarker(i32 1)
@@ -1722,10 +1817,10 @@ normal:
 
 
 declare void @f.writeonly() writeonly
-; CHECK: declare void @f.writeonly() #40
+; CHECK: declare void @f.writeonly() #41
 
 declare void @f.speculatable() speculatable
-; CHECK: declare void @f.speculatable() #41
+; CHECK: declare void @f.speculatable() #42
 
 ;; Constant Expressions
 
@@ -1734,6 +1829,13 @@ define i8** @constexpr() {
   ret i8** getelementptr inbounds ({ [4 x i8*], [4 x i8*] }, { [4 x i8*], [4 x i8*] }* null, i32 0, inrange i32 1, i32 2)
 }
 
+define void @instructions.strictfp() strictfp {
+  call void @f.strictfp() strictfp
+  ; CHECK: call void @f.strictfp() #43
+
+  ret void
+}
+
 ; immarg attribute
 declare void @llvm.test.immarg.intrinsic(i32 immarg)
 ; CHECK: declare void @llvm.test.immarg.intrinsic(i32 immarg)
@@ -1782,15 +1884,16 @@ declare void @byval_named_type(%named_type* byval(%named_type))
 ; CHECK: attributes #32 = { norecurse }
 ; CHECK: attributes #33 = { inaccessiblememonly }
 ; CHECK: attributes #34 = { inaccessiblemem_or_argmemonly }
-; CHECK: attributes #35 = { nounwind readnone }
+; CHECK: attributes #35 = { nounwind readnone willreturn }
 ; CHECK: attributes #36 = { argmemonly nounwind readonly }
 ; CHECK: attributes #37 = { argmemonly nounwind }
-; CHECK: attributes #38 = { nounwind readonly }
-; CHECK: attributes #39 = { inaccessiblemem_or_argmemonly nounwind }
-; CHECK: attributes #40 = { writeonly }
-; CHECK: attributes #41 = { speculatable }
-; CHECK: attributes #42 = { builtin }
+; CHECK: attributes #38 = { nounwind readnone }
+; CHECK: attributes #39 = { nounwind readonly }
+; CHECK: attributes #40 = { inaccessiblemem_or_argmemonly nounwind willreturn }
+; CHECK: attributes #41 = { writeonly }
+; CHECK: attributes #42 = { speculatable }
 ; CHECK: attributes #43 = { strictfp }
+; CHECK: attributes #44 = { builtin }
 
 ;; Metadata
 
diff --git a/llvm/test/Bitcode/drop-debug-info.3.5.ll.x b/llvm/test/Bitcode/drop-debug-info.3.5.ll.x
index fde136d1c..35d39583c 100644
--- a/llvm/test/Bitcode/drop-debug-info.3.5.ll.x
+++ b/llvm/test/Bitcode/drop-debug-info.3.5.ll.x
@@ -19,7 +19,7 @@ entry:
   ret i32 0, !dbg !12
 }
 
-attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!9, !10}
diff --git a/llvm/test/Bitcode/invalid-functionptr-align.ll b/llvm/test/Bitcode/invalid-functionptr-align.ll
new file mode 100644
index 000000000..4ff797a4b
--- /dev/null
+++ b/llvm/test/Bitcode/invalid-functionptr-align.ll
@@ -0,0 +1,5 @@
+; Bitcode with invalid function pointer alignment.
+
+; RUN: not llvm-dis %s.bc -o - 2>&1 | FileCheck %s
+
+CHECK: LLVM ERROR: Alignment is neither 0 nor a power of 2
diff --git a/llvm/test/Bitcode/invalid-type-for-null-constant.ll b/llvm/test/Bitcode/invalid-type-for-null-constant.ll
new file mode 100644
index 000000000..28756fa18
--- /dev/null
+++ b/llvm/test/Bitcode/invalid-type-for-null-constant.ll
@@ -0,0 +1,6 @@
+; Bitcode with a CST_CODE_NULL with void type.
+
+; RUN: not llvm-dis %s.bc -o - 2>&1 | FileCheck %s
+
+; CHECK: error: Invalid type for a constant null value
+
diff --git a/llvm/test/Bitcode/multi-module.ll b/llvm/test/Bitcode/multi-module.ll
index 5f73240e8..b320bf1bf 100644
--- a/llvm/test/Bitcode/multi-module.ll
+++ b/llvm/test/Bitcode/multi-module.ll
@@ -1,6 +1,4 @@
 ; RUN: llvm-cat -o %t %s %S/Inputs/multi-module.ll
-; RUN: not llvm-dis -o - %t 2>&1 | FileCheck --check-prefix=ERROR %s
-; ERROR: Expected a single module
 
 ; RUN: llvm-bcanalyzer -dump %t | FileCheck --check-prefix=BCA %s
 
@@ -10,18 +8,15 @@
 ; RUN: llvm-as -o %t1 %s
 ; RUN: llvm-as -o %t2 %S/Inputs/multi-module.ll
 ; RUN: llvm-cat -o %t %t1 %t2
-; RUN: not llvm-dis -o - %t 2>&1 | FileCheck --check-prefix=ERROR %s
 ; RUN: llvm-bcanalyzer -dump %t | FileCheck --check-prefix=BCA %s
 
 ; RUN: llvm-cat -b -o %t %t1 %t2
-; RUN: not llvm-dis -o - %t 2>&1 | FileCheck --check-prefix=ERROR %s
 ; RUN: llvm-bcanalyzer -dump %t | FileCheck --check-prefix=BCA %s
 
 ; RUN: llvm-modextract -n 0 -o - %t | llvm-dis | FileCheck --check-prefix=IR1 %s
 ; RUN: llvm-modextract -n 1 -o - %t | llvm-dis | FileCheck --check-prefix=IR2 %s
 
 ; RUN: llvm-cat -b -o %t3 %t %t
-; RUN: not llvm-dis -o - %t3 2>&1 | FileCheck --check-prefix=ERROR %s
 ; RUN: llvm-bcanalyzer -dump %t3 | FileCheck --check-prefix=BCA4 %s
 
 ; RUN: llvm-modextract -n 0 -o - %t3 | llvm-dis | FileCheck --check-prefix=IR1 %s
diff --git a/llvm/test/Bitcode/operand-bundles-bc-analyzer.ll b/llvm/test/Bitcode/operand-bundles-bc-analyzer.ll
index 8f7aa37cb..ac7feec77 100644
--- a/llvm/test/Bitcode/operand-bundles-bc-analyzer.ll
+++ b/llvm/test/Bitcode/operand-bundles-bc-analyzer.ll
@@ -6,6 +6,7 @@
 ; CHECK-NEXT:    <OPERAND_BUNDLE_TAG
 ; CHECK-NEXT:    <OPERAND_BUNDLE_TAG
 ; CHECK-NEXT:    <OPERAND_BUNDLE_TAG
+; CHECK-NEXT:    <OPERAND_BUNDLE_TAG
 ; CHECK-NEXT:  </OPERAND_BUNDLE_TAGS_BLOCK
 
 ; CHECK:   <FUNCTION_BLOCK
diff --git a/llvm/test/Bitcode/summary_version.ll b/llvm/test/Bitcode/summary_version.ll
index e531a078d..2a6707371 100644
--- a/llvm/test/Bitcode/summary_version.ll
+++ b/llvm/test/Bitcode/summary_version.ll
@@ -2,7 +2,7 @@
 ; RUN: opt  -module-summary  %s -o - | llvm-bcanalyzer -dump | FileCheck %s
 
 ; CHECK: <GLOBALVAL_SUMMARY_BLOCK
-; CHECK: <VERSION op0=7/>
+; CHECK: <VERSION op0=8/>
 
 
 
diff --git a/llvm/test/Bitcode/thinlto-alias3.ll b/llvm/test/Bitcode/thinlto-alias3.ll
new file mode 100644
index 000000000..274b4a6dc
--- /dev/null
+++ b/llvm/test/Bitcode/thinlto-alias3.ll
@@ -0,0 +1,11 @@
+; Test that inttoptr, add and ptrtoint don't cause problems in alias summaries.
+; RUN: opt -module-summary %s -o - | llvm-dis | FileCheck %s
+
+; CHECK: ^1 = gv: (name: "a", {{.*}} aliasee: ^2
+; CHECK: ^2 = gv: (name: "b",
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@a = alias i32, i32* inttoptr (i64 add (i64 ptrtoint (i32* @b to i64), i64 1297036692682702848) to i32*)
+@b = global i32 1
diff --git a/llvm/test/Bitcode/thinlto-deadstrip-flag.ll b/llvm/test/Bitcode/thinlto-deadstrip-flag.ll
index 5330a25db..acde6e943 100644
--- a/llvm/test/Bitcode/thinlto-deadstrip-flag.ll
+++ b/llvm/test/Bitcode/thinlto-deadstrip-flag.ll
@@ -5,14 +5,14 @@
 ; RUN: llvm-lto2 run %t.o -o %t.out -thinlto-distributed-indexes \
 ; RUN:		-r %t.o,glob,plx
 ; RUN: llvm-bcanalyzer -dump %t.o.thinlto.bc | FileCheck %s --check-prefix=WITHDEAD
-; WITHDEAD: <FLAGS op0=1/>
+; WITHDEAD: <FLAGS op0=33/>
 
 ; Ensure dead stripping performed flag is not set on distributed index
 ; when option used to disable dead stripping computation.
 ; RUN: llvm-lto2 run %t.o -o %t.out -thinlto-distributed-indexes \
 ; RUN:		-r %t.o,glob,plx -compute-dead=false
 ; RUN: llvm-bcanalyzer -dump %t.o.thinlto.bc | FileCheck %s --check-prefix=NODEAD
-; NODEAD: <FLAGS op0=0/>
+; NODEAD: <FLAGS op0=32/>
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/Bitcode/thinlto-synthetic-count-flag.ll b/llvm/test/Bitcode/thinlto-synthetic-count-flag.ll
index eb18a025b..2174335f7 100644
--- a/llvm/test/Bitcode/thinlto-synthetic-count-flag.ll
+++ b/llvm/test/Bitcode/thinlto-synthetic-count-flag.ll
@@ -5,7 +5,7 @@
 ; RUN: llvm-lto2 run %t.o -o %t.out -thinlto-distributed-indexes \
 ; RUN:		-r %t.o,glob,plx -compute-dead=false
 ; RUN: llvm-bcanalyzer -dump %t.o.thinlto.bc | FileCheck %s --check-prefix=NOSYNTHETIC
-; NOSYNTHETIC: <FLAGS op0=0/>
+; NOSYNTHETIC: <FLAGS op0=32/>
 
 ; Ensure synthetic entry count flag is set on distributed index
 ; when option used to enable synthetic count propagation
@@ -13,7 +13,7 @@
 ; RUN:		-r %t.o,glob,plx -thinlto-synthesize-entry-counts \
 ; RUN:          -compute-dead=false
 ; RUN: llvm-bcanalyzer -dump %t.o.thinlto.bc | FileCheck %s --check-prefix=HASSYNTHETIC
-; HASSYNTHETIC: <FLAGS op0=4/>
+; HASSYNTHETIC: <FLAGS op0=36/>
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/Bitcode/upgrade-arc-runtime-calls-bitcast.ll b/llvm/test/Bitcode/upgrade-arc-runtime-calls-bitcast.ll
new file mode 100644
index 000000000..deac12d36
--- /dev/null
+++ b/llvm/test/Bitcode/upgrade-arc-runtime-calls-bitcast.ll
@@ -0,0 +1,21 @@
+target triple = "arm64-apple-ios7.0"
+
+; RUN: llvm-dis < %S/upgrade-arc-runtime-calls-bitcast.bc | FileCheck %s
+
+; CHECK: tail call i8* @objc_retain(i32 1)
+; CHECK: tail call i8* @objc_storeStrong(
+
+define void @testRuntimeCalls(i8* %a, i8** %b) {
+  %v6 = tail call i8* @objc_retain(i32 1)
+  %1 = tail call i8* @objc_storeStrong(i8** %b, i8* %a)
+  ret void
+}
+
+declare i8* @objc_retain(i32)
+declare i8* @objc_storeStrong(i8**, i8*)
+
+attributes #0 = { nounwind }
+
+!llvm.module.flags = !{!0}
+
+!0 = !{i32 1, !"clang.arc.retainAutoreleasedReturnValueMarker", !"mov\09fp, fp\09\09; marker for objc_retainAutoreleaseReturnValue"}
diff --git a/llvm/test/Bitcode/upgrade-arc-runtime-calls.ll b/llvm/test/Bitcode/upgrade-arc-runtime-calls.ll
new file mode 100644
index 000000000..d42c776dd
--- /dev/null
+++ b/llvm/test/Bitcode/upgrade-arc-runtime-calls.ll
@@ -0,0 +1,125 @@
+; Test that calls to ARC runtime functions are converted to intrinsic calls if
+; the bitcode has the arm64 retainAutoreleasedReturnValueMarker metadata.
+
+; upgrade-arc-runtime-calls.bc and upgrade-mrr-runtime-calls.bc are identical
+; except that the former has the arm64 retainAutoreleasedReturnValueMarker
+; metadata. upgrade-arc-runtime-calls-new.bc has the new module flag format of
+; marker, it should not be upgraded.
+
+; RUN: llvm-dis < %S/upgrade-arc-runtime-calls.bc | FileCheck -check-prefixes=ARC %s
+; RUN: llvm-dis < %S/upgrade-mrr-runtime-calls.bc | FileCheck -check-prefixes=NOUPGRADE %s
+; RUN: llvm-dis < %S/upgrade-arc-runtime-calls-new.bc | FileCheck -check-prefixes=NOUPGRADE %s
+
+define void @testRuntimeCalls(i8* %a, i8** %b, i8** %c, i32* %d, i32** %e) personality i32 (...)* @__gxx_personality_v0 {
+entry:
+  %v0 = tail call i8* @objc_autorelease(i8* %a) #0
+  tail call void @objc_autoreleasePoolPop(i8* %a) #0
+  %v1 = tail call i8* @objc_autoreleasePoolPush() #0
+  %v2 = tail call i8* @objc_autoreleaseReturnValue(i8* %a) #0
+  tail call void @objc_copyWeak(i8** %b, i8** %c) #0
+  tail call void @objc_destroyWeak(i8** %b) #0
+  %v3 = tail call i32* @objc_initWeak(i32** %e, i32* %d) #0
+  %v4 = tail call i8* @objc_loadWeak(i8** %b) #0
+  %v5 = tail call i8* @objc_loadWeakRetained(i8** %b) #0
+  tail call void @objc_moveWeak(i8** %b, i8** %c) #0
+  tail call void @objc_release(i8* %a) #0
+  %v6 = tail call i8* @objc_retain(i8* %a) #0
+  %v7 = tail call i8* @objc_retainAutorelease(i8* %a) #0
+  %v8 = tail call i8* @objc_retainAutoreleaseReturnValue(i8* %a) #0
+  %v9 = tail call i8* @objc_retainAutoreleasedReturnValue(i8* %a) #0
+  %v10 = tail call i8* @objc_retainBlock(i8* %a) #0
+  tail call void @objc_storeStrong(i8** %b, i8* %a) #0
+  %v11 = tail call i8* @objc_storeWeak(i8** %b, i8* %a) #0
+  tail call void (...) @clang.arc.use(i8* %a) #0
+  %v12 = tail call i8* @objc_unsafeClaimAutoreleasedReturnValue(i8* %a) #0
+  %v13 = tail call i8* @objc_retainedObject(i8* %a) #0
+  %v14 = tail call i8* @objc_unretainedObject(i8* %a) #0
+  %v15 = tail call i8* @objc_unretainedPointer(i8* %a) #0
+  %v16 = tail call i8* @objc_retain.autorelease(i8* %a) #0
+  %v17 = tail call i32 @objc_sync.enter(i8* %a) #0
+  %v18 = tail call i32 @objc_sync.exit(i8* %a) #0
+  tail call void @objc_arc_annotation_topdown_bbstart(i8** %b, i8** %c) #0
+  tail call void @objc_arc_annotation_topdown_bbend(i8** %b, i8** %c) #0
+  tail call void @objc_arc_annotation_bottomup_bbstart(i8** %b, i8** %c) #0
+  tail call void @objc_arc_annotation_bottomup_bbend(i8** %b, i8** %c) #0
+  invoke void @objc_autoreleasePoolPop(i8* %a)
+          to label %normalBlock unwind label %unwindBlock
+normalBlock:
+  ret void
+unwindBlock:
+  %ll = landingpad { i8*, i32 }
+          cleanup
+  ret void
+}
+
+// Check that auto-upgrader converts function calls to intrinsic calls. Note that
+// the auto-upgrader doesn't touch invoke instructions.
+
+// ARC: define void @testRuntimeCalls(i8* %[[A:.*]], i8** %[[B:.*]], i8** %[[C:.*]], i32* %[[D:.*]], i32** %[[E:.*]]) personality
+// ARC: %[[V0:.*]] = tail call i8* @llvm.objc.autorelease(i8* %[[A]])
+// ARC-NEXT: tail call void @llvm.objc.autoreleasePoolPop(i8* %[[A]])
+// ARC-NEXT: %[[V1:.*]] = tail call i8* @llvm.objc.autoreleasePoolPush()
+// ARC-NEXT: %[[V2:.*]] = tail call i8* @llvm.objc.autoreleaseReturnValue(i8* %[[A]])
+// ARC-NEXT: tail call void @llvm.objc.copyWeak(i8** %[[B]], i8** %[[C]])
+// ARC-NEXT: tail call void @llvm.objc.destroyWeak(i8** %[[B]])
+// ARC-NEXT: %[[V100:.*]] = bitcast i32** %[[E]] to i8**
+// ARC-NEXT: %[[V101:.*]] = bitcast i32* %[[D]] to i8*
+// ARC-NEXT: %[[V102:.*]] = tail call i8* @llvm.objc.initWeak(i8** %[[V100]], i8* %[[V101]])
+// ARC-NEXT: %[[V103:.*]] = bitcast i8* %[[V102]] to i32*
+// ARC-NEXT: %[[V4:.*]] = tail call i8* @llvm.objc.loadWeak(i8** %[[B]])
+// ARC-NEXT: %[[V5:.*]] = tail call i8* @llvm.objc.loadWeakRetained(i8** %[[B]])
+// ARC-NEXT: tail call void @llvm.objc.moveWeak(i8** %[[B]], i8** %[[C]])
+// ARC-NEXT: tail call void @llvm.objc.release(i8* %[[A]])
+// ARC-NEXT: %[[V6:.*]] = tail call i8* @llvm.objc.retain(i8* %[[A]])
+// ARC-NEXT: %[[V7:.*]] = tail call i8* @llvm.objc.retainAutorelease(i8* %[[A]])
+// ARC-NEXT: %[[V8:.*]] = tail call i8* @llvm.objc.retainAutoreleaseReturnValue(i8* %[[A]])
+// ARC-NEXT: %[[V9:.*]] = tail call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %[[A]])
+// ARC-NEXT: %[[V10:.*]] = tail call i8* @llvm.objc.retainBlock(i8* %[[A]])
+// ARC-NEXT: tail call void @llvm.objc.storeStrong(i8** %[[B]], i8* %[[A]])
+// ARC-NEXT: %[[V11:.*]] = tail call i8* @llvm.objc.storeWeak(i8** %[[B]], i8* %[[A]])
+// ARC-NEXT: tail call void (...) @llvm.objc.clang.arc.use(i8* %[[A]])
+// ARC-NEXT: %[[V12:.*]] = tail call i8* @llvm.objc.unsafeClaimAutoreleasedReturnValue(i8* %[[A]])
+// ARC-NEXT: %[[V13:.*]] = tail call i8* @llvm.objc.retainedObject(i8* %[[A]])
+// ARC-NEXT: %[[V14:.*]] = tail call i8* @llvm.objc.unretainedObject(i8* %[[A]])
+// ARC-NEXT: %[[V15:.*]] = tail call i8* @llvm.objc.unretainedPointer(i8* %[[A]])
+// ARC-NEXT: %[[V16:.*]] = tail call i8* @objc_retain.autorelease(i8* %[[A]])
+// ARC-NEXT: %[[V17:.*]] = tail call i32 @objc_sync.enter(i8* %[[A]])
+// ARC-NEXT: %[[V18:.*]] = tail call i32 @objc_sync.exit(i8* %[[A]])
+// ARC-NEXT: tail call void @llvm.objc.arc.annotation.topdown.bbstart(i8** %[[B]], i8** %[[C]])
+// ARC-NEXT: tail call void @llvm.objc.arc.annotation.topdown.bbend(i8** %[[B]], i8** %[[C]])
+// ARC-NEXT: tail call void @llvm.objc.arc.annotation.bottomup.bbstart(i8** %[[B]], i8** %[[C]])
+// ARC-NEXT: tail call void @llvm.objc.arc.annotation.bottomup.bbend(i8** %[[B]], i8** %[[C]])
+// ARC-NEXT: invoke void @objc_autoreleasePoolPop(i8* %[[A]])
+
+// NOUPGRADE: define void @testRuntimeCalls(i8* %[[A:.*]], i8** %[[B:.*]], i8** %[[C:.*]], i32* %[[D:.*]], i32** %[[E:.*]]) personality
+// NOUPGRADE: %[[V0:.*]] = tail call i8* @objc_autorelease(i8* %[[A]])
+// NOUPGRADE-NEXT: tail call void @objc_autoreleasePoolPop(i8* %[[A]])
+// NOUPGRADE-NEXT: %[[V1:.*]] = tail call i8* @objc_autoreleasePoolPush()
+// NOUPGRADE-NEXT: %[[V2:.*]] = tail call i8* @objc_autoreleaseReturnValue(i8* %[[A]])
+// NOUPGRADE-NEXT: tail call void @objc_copyWeak(i8** %[[B]], i8** %[[C]])
+// NOUPGRADE-NEXT: tail call void @objc_destroyWeak(i8** %[[B]])
+// NOUPGRADE-NEXT: %[[V3:.*]] = tail call i32* @objc_initWeak(i32** %[[E]], i32* %[[D]])
+// NOUPGRADE-NEXT: %[[V4:.*]] = tail call i8* @objc_loadWeak(i8** %[[B]])
+// NOUPGRADE-NEXT: %[[V5:.*]] = tail call i8* @objc_loadWeakRetained(i8** %[[B]])
+// NOUPGRADE-NEXT: tail call void @objc_moveWeak(i8** %[[B]], i8** %[[C]])
+// NOUPGRADE-NEXT: tail call void @objc_release(i8* %[[A]])
+// NOUPGRADE-NEXT: %[[V6:.*]] = tail call i8* @objc_retain(i8* %[[A]])
+// NOUPGRADE-NEXT: %[[V7:.*]] = tail call i8* @objc_retainAutorelease(i8* %[[A]])
+// NOUPGRADE-NEXT: %[[V8:.*]] = tail call i8* @objc_retainAutoreleaseReturnValue(i8* %[[A]])
+// NOUPGRADE-NEXT: %[[V9:.*]] = tail call i8* @objc_retainAutoreleasedReturnValue(i8* %[[A]])
+// NOUPGRADE-NEXT: %[[V10:.*]] = tail call i8* @objc_retainBlock(i8* %[[A]])
+// NOUPGRADE-NEXT: tail call void @objc_storeStrong(i8** %[[B]], i8* %[[A]])
+// NOUPGRADE-NEXT: %[[V11:.*]] = tail call i8* @objc_storeWeak(i8** %[[B]], i8* %[[A]])
+// NOUPGRADE-NEXT: tail call void (...) @llvm.objc.clang.arc.use(i8* %[[A]])
+// NOUPGRADE-NEXT: %[[V12:.*]] = tail call i8* @objc_unsafeClaimAutoreleasedReturnValue(i8* %[[A]])
+// NOUPGRADE-NEXT: %[[V13:.*]] = tail call i8* @objc_retainedObject(i8* %[[A]])
+// NOUPGRADE-NEXT: %[[V14:.*]] = tail call i8* @objc_unretainedObject(i8* %[[A]])
+// NOUPGRADE-NEXT: %[[V15:.*]] = tail call i8* @objc_unretainedPointer(i8* %[[A]])
+// NOUPGRADE-NEXT: %[[V16:.*]] = tail call i8* @objc_retain.autorelease(i8* %[[A]])
+// NOUPGRADE-NEXT: %[[V17:.*]] = tail call i32 @objc_sync.enter(i8* %[[A]])
+// NOUPGRADE-NEXT: %[[V18:.*]] = tail call i32 @objc_sync.exit(i8* %[[A]])
+// NOUPGRADE-NEXT: tail call void @objc_arc_annotation_topdown_bbstart(i8** %[[B]], i8** %[[C]])
+// NOUPGRADE-NEXT: tail call void @objc_arc_annotation_topdown_bbend(i8** %[[B]], i8** %[[C]])
+// NOUPGRADE-NEXT: tail call void @objc_arc_annotation_bottomup_bbstart(i8** %[[B]], i8** %[[C]])
+// NOUPGRADE-NEXT: tail call void @objc_arc_annotation_bottomup_bbend(i8** %[[B]], i8** %[[C]])
+// NOUPGRADE-NEXT: invoke void @objc_autoreleasePoolPop(i8* %[[A]])
diff --git a/llvm/test/Bitcode/upgrade-clang-arc-use.ll b/llvm/test/Bitcode/upgrade-clang-arc-use.ll
index db5481f06..ac8dd0ddd 100644
--- a/llvm/test/Bitcode/upgrade-clang-arc-use.ll
+++ b/llvm/test/Bitcode/upgrade-clang-arc-use.ll
@@ -1,4 +1,4 @@
-; Test upgrade of clang.arc.use by removing it.
+; Test upgrade of clang.arc.use by upgrading to llvm.objc.clang.arc.use.
 ; Bitcode input generated from llvm 6.0
 
 ; RUN: llvm-dis %s.bc -o - | FileCheck %s
@@ -6,7 +6,7 @@
 %0 = type opaque
 define void @foo() {
   %1 = tail call %0* @foo0()
-; CHECK-NOT: clang.arc.use
+; CHECK: call void (...) @llvm.objc.clang.arc.use(
   call void (...) @clang.arc.use(%0* %1)
   ret void
 }
diff --git a/llvm/test/Bitcode/upgrade-datalayout.ll b/llvm/test/Bitcode/upgrade-datalayout.ll
new file mode 100644
index 000000000..8c00294f2
--- /dev/null
+++ b/llvm/test/Bitcode/upgrade-datalayout.ll
@@ -0,0 +1,9 @@
+; Test to make sure datalayout is automatically upgraded.
+;
+; RUN: llvm-as %s -o - | llvm-dis - | FileCheck %s
+
+target datalayout = "e-m:e-p:32:32-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; CHECK: target datalayout = "e-m:e-p:32:32-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+
diff --git a/llvm/test/Bitcode/upgrade-datalayout2.ll b/llvm/test/Bitcode/upgrade-datalayout2.ll
new file mode 100644
index 000000000..21de5b8a6
--- /dev/null
+++ b/llvm/test/Bitcode/upgrade-datalayout2.ll
@@ -0,0 +1,10 @@
+; Test to make sure datalayout is not automatically upgraded if it does not
+; match a possible x86 datalayout.
+;
+; RUN: llvm-as %s -o - | llvm-dis - | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; CHECK: target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
diff --git a/llvm/test/Bitcode/upgrade-datalayout3.ll b/llvm/test/Bitcode/upgrade-datalayout3.ll
new file mode 100644
index 000000000..526ba6069
--- /dev/null
+++ b/llvm/test/Bitcode/upgrade-datalayout3.ll
@@ -0,0 +1,8 @@
+; Test to make sure datalayout is automatically upgraded.
+;
+; RUN: llvm-as %s -o - | llvm-dis - | FileCheck %s
+
+target datalayout = "e-m:w-p:32:32-i64:64-f80:32-n8:16:32-S32"
+target triple = "i686-pc-windows-msvc"
+
+; CHECK: target datalayout = "e-m:w-p:32:32-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:32-n8:16:32-S32"
diff --git a/llvm/test/Bitcode/upgrade-frame-pointer.ll b/llvm/test/Bitcode/upgrade-frame-pointer.ll
new file mode 100644
index 000000000..5251db9ec
--- /dev/null
+++ b/llvm/test/Bitcode/upgrade-frame-pointer.ll
@@ -0,0 +1,33 @@
+;; Test that function attributes "no-frame-pointer-elim" ("true" or "false") and
+;; "no-frame-pointer-elim-non-leaf" (value is ignored) can be upgraded to
+;; "frame-pointer".
+
+; RUN: llvm-as %s -o - | llvm-dis - | FileCheck %s
+
+; CHECK: define void @all0() #0
+define void @all0() "no-frame-pointer-elim"="true" { ret void }
+; CHECK: define void @all1() #1
+define void @all1() #0 { ret void }
+
+; CHECK: define void @non_leaf0() #2
+define void @non_leaf0() "no-frame-pointer-elim-non-leaf" { ret void }
+; CHECK: define void @non_leaf1() #3
+define void @non_leaf1() #1 { ret void }
+
+; CHECK: define void @none() #4
+define void @none() "no-frame-pointer-elim"="false" { ret void }
+
+;; Don't add "frame-pointer" if neither "no-frame-pointer-elim" nor
+;; "no-frame-pointer-elim-non-leaf" is present.
+; CHECK: define void @no_attr() {
+define void @no_attr() { ret void }
+
+attributes #0 = { readnone "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" }
+attributes #1 = { readnone "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf" }
+
+;; Other attributes (e.g. readnone) are unaffected.
+; CHECK: attributes #0 = { "frame-pointer"="all" }
+; CHECK: attributes #1 = { readnone "frame-pointer"="all" }
+; CHECK: attributes #2 = { "frame-pointer"="non-leaf" }
+; CHECK: attributes #3 = { readnone "frame-pointer"="non-leaf" }
+; CHECK: attributes #4 = { "frame-pointer"="none" }
diff --git a/llvm/test/Bitcode/upgrade-memory-intrinsics.ll b/llvm/test/Bitcode/upgrade-memory-intrinsics.ll
index 1bdee928c..c53431317 100644
--- a/llvm/test/Bitcode/upgrade-memory-intrinsics.ll
+++ b/llvm/test/Bitcode/upgrade-memory-intrinsics.ll
@@ -28,7 +28,7 @@ define void @test2(i8* %p1, i8* %p2, i8* %p3) {
 }
 
 ; CHECK: declare void @llvm.memset.p0i8.i64(i8* nocapture writeonly, i8, i64, i1 immarg)
-; CHECK: declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture writeonly, i8* nocapture readonly, i64, i1 immarg)
+; CHECK: declare void @llvm.memcpy.p0i8.p0i8.i64(i8* noalias nocapture writeonly, i8* noalias nocapture readonly, i64, i1 immarg)
 ; CHECK: declare void @llvm.memmove.p0i8.p0i8.i64(i8* nocapture, i8* nocapture readonly, i64, i1 immarg)
 declare void @llvm.memset.p0i8.i64(i8* nocapture writeonly, i8, i64, i32, i1)
 declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture writeonly , i8* nocapture readonly, i64, i32, i1)
diff --git a/llvm/test/Bitcode/upgrade-tbaa.ll b/llvm/test/Bitcode/upgrade-tbaa.ll
index c20c66aad..7abfff868 100644
--- a/llvm/test/Bitcode/upgrade-tbaa.ll
+++ b/llvm/test/Bitcode/upgrade-tbaa.ll
@@ -11,7 +11,7 @@ entry:
   ret void
 }
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !0 = !{!"omnipotent char", !1}
 !1 = !{!"Simple C/C++ TBAA"}
diff --git a/llvm/test/BugPoint/attr-crash.ll b/llvm/test/BugPoint/attr-crash.ll
new file mode 100644
index 000000000..7032e86a5
--- /dev/null
+++ b/llvm/test/BugPoint/attr-crash.ll
@@ -0,0 +1,16 @@
+; RUN: bugpoint -load %llvmshlibdir/BugpointPasses%shlibext %s -output-prefix %t -bugpoint-crashfuncattr 2>&1 | FileCheck %s
+; REQUIRES: plugins
+;
+; ModuleID = 'attr-crash.ll'
+source_filename = "test.c"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+
+; Function Attrs: noinline nounwind optnone uwtable
+define dso_local i32 @main(i32 %argc, i8** %argv) local_unnamed_addr #0 {
+  ret i32 0
+}
+
+; CHECK-NOT: Attribute 'optnone' requires 'noinline'!
+attributes #0 = { noinline nounwind optnone uwtable "bugpoint-crash" }
diff --git a/llvm/test/BugPoint/func-attrs-keyval.ll b/llvm/test/BugPoint/func-attrs-keyval.ll
index aaf8541e5..0de5db9e3 100644
--- a/llvm/test/BugPoint/func-attrs-keyval.ll
+++ b/llvm/test/BugPoint/func-attrs-keyval.ll
@@ -8,4 +8,4 @@ define void @f() #0 {
 }
 
 ; CHECK: attributes #[[ATTRS]] = { "bugpoint-crash"="sure" }
-attributes #0 = { "bugpoint-crash"="sure" noreturn "no-frame-pointer-elim-non-leaf" }
+attributes #0 = { "bugpoint-crash"="sure" noreturn "frame-pointer"="non-leaf" }
diff --git a/llvm/test/BugPoint/func-attrs.ll b/llvm/test/BugPoint/func-attrs.ll
index 8f742ee93..234477927 100644
--- a/llvm/test/BugPoint/func-attrs.ll
+++ b/llvm/test/BugPoint/func-attrs.ll
@@ -1,11 +1,15 @@
 ; RUN: bugpoint -load %llvmshlibdir/BugpointPasses%shlibext %s -output-prefix %t -bugpoint-crashfuncattr -silence-passes
-; RUN: llvm-dis %t-reduced-simplified.bc -o - | FileCheck %s
+; RUN: llvm-dis %t-reduced-simplified.bc -o - | FileCheck -check-prefixes=ALL,ENABLED %s
+; RUN: bugpoint -disable-attribute-remove -load %llvmshlibdir/BugpointPasses%shlibext %s -output-prefix %t -bugpoint-crashfuncattr -silence-passes
+; RUN: llvm-dis %t-reduced-simplified.bc -o - | FileCheck -check-prefixes=ALL,DISABLED %s
+
 ; REQUIRES: plugins
 
-; CHECK: f() #[[ATTRS:[0-9]+]]
+; ALL: f() #[[ATTRS:[0-9]+]]
 define void @f() #0 {
   ret void
 }
 
-; CHECK: attributes #[[ATTRS]] = { "bugpoint-crash" }
-attributes #0 = { noinline "bugpoint-crash" "no-frame-pointer-elim-non-leaf" }
+; ENABLED: attributes #[[ATTRS]] = { "bugpoint-crash" }
+; DISABLED: attributes #[[ATTRS]] = { noinline "bugpoint-crash" "frame-pointer"="non-leaf" }
+attributes #0 = { noinline "bugpoint-crash" "frame-pointer"="non-leaf" }
\ No newline at end of file
diff --git a/llvm/test/BugPoint/metadata.ll b/llvm/test/BugPoint/metadata.ll
index a090bbde1..9cc4659b5 100644
--- a/llvm/test/BugPoint/metadata.ll
+++ b/llvm/test/BugPoint/metadata.ll
@@ -8,22 +8,18 @@
 ; RUN: bugpoint -load %llvmshlibdir/BugpointPasses%shlibext %s -output-prefix %t-notype -bugpoint-crashcalls -silence-passes -disable-namedmd-remove -disable-strip-debuginfo > /dev/null
 ; RUN: llvm-dis %t-notype-reduced-simplified.bc -o - | FileCheck %s --check-prefix=NOTYPE
 ;
-; Bugpoint should keep the call's metadata attached to the call.
+; Bugpoint can drop the metadata on the call, as it does not contrinute to the crash.
 
-; CHECK: call void @foo(), !dbg ![[LOC:[0-9]+]], !attach ![[CALL:[0-9]+]]
-; NODEBUG: call void @foo(), !attach ![[CALL:[0-9]+]]
-; NOTYPE: call void @foo(), !dbg ![[LOC:[0-9]+]], !attach ![[CALL:[0-9]+]]
-; NODEBUG-NOT: call void @foo(), !attach ![[CALL:[0-9]+]]
+; CHECK: call void @foo()
+; NODEBUG: call void @foo()
+; NOTYPE: call void @foo()
+; NODEBUG-NOT: call void @foo()
 ; NOTYPE-NOT: !DIBasicType
 ; NOTYPE: !DICompileUnit
 ; NOTYPE-NOT: !DIBasicType
-; CHECK-DAG: ![[LOC]] = !DILocation(line: 104, column: 105, scope: ![[SCOPE:[0-9]+]])
-; CHECK-DAG: ![[SCOPE]] = distinct !DISubprogram(name: "test",{{.*}}file: ![[FILE:[0-9]+]]
-; CHECK-DAG: ![[FILE]] = !DIFile(filename: "source.c", directory: "/dir")
-; CHECK-DAG: ![[CALL]] = !{!"the call to foo"}
 
 %rust_task = type {}
-define void @test(i32* %a, i8* %b) {
+define void @test(i32* %a, i8* %b) !dbg !9 {
     %s = mul i8 22, 9, !attach !0, !dbg !10
     store i8 %s, i8* %b, !attach !1, !dbg !11
     call void @foo(), !attach !2, !dbg !12
diff --git a/llvm/test/BugPoint/retain-crashing-metadata.ll b/llvm/test/BugPoint/retain-crashing-metadata.ll
new file mode 100644
index 000000000..ddceb8078
--- /dev/null
+++ b/llvm/test/BugPoint/retain-crashing-metadata.ll
@@ -0,0 +1,22 @@
+; REQUIRES: plugins
+; RUN: bugpoint -load %llvmshlibdir/BugpointPasses%shlibext %s -output-prefix %t-notype -bugpoint-crashmetadata -silence-passes > /dev/null
+; RUN: llvm-dis %t-notype-reduced-simplified.bc -o - | FileCheck %s
+;
+; Make sure BugPoint retains metadata contributing to a crash.
+
+; CHECK-LABEL: define void @test2(float %f) {
+; CHECK-NEXT: %arg = fadd float %f, 1.000000e+01
+; CHECK-NOT: !fpmath
+; CHECK-NEXT: %x = call float @llvm.fabs.f32(float %arg), !fpmath [[FPMATH:![0-9]+]]
+; CHECK-NEXT: ret void
+
+; CHECK: [[FPMATH]] = !{float 2.500000e+00}
+define void @test2(float %f) {
+    %arg = fadd float %f, 1.000000e+01, !fpmath !0
+    %x = call float @llvm.fabs.f32(float %arg), !fpmath !0
+    ret void
+}
+
+declare float @llvm.fabs.f32(float)
+
+!0 = !{float 2.500000e+00}
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-callingconv.ll b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-callingconv.ll
index 6bdcc4801..21cc3f82a 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-callingconv.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-callingconv.ll
@@ -1,4 +1,4 @@
-; RUN: llc -O0 -stop-after=irtranslator -global-isel -verify-machineinstrs %s -o - 2>&1 | FileCheck %s
+; RUN: llc -O0 -stop-after=irtranslator -global-isel -global-isel-abort=1 -verify-machineinstrs %s -o - 2>&1 | FileCheck %s
 
 target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64-linux-gnu"
@@ -96,3 +96,38 @@ define void @test_stack_ext_needed() {
   call void @stack_ext_needed([8 x i64] undef, i8 signext 42)
   ret void
 }
+
+; Check that we can lower incoming i128 types into constituent s64 gprs.
+; CHECK-LABEL: name: callee_s128
+; CHECK: liveins: $x0, $x1, $x2, $x3, $x4
+; CHECK: [[A1_P1:%[0-9]+]]:_(s64) = COPY $x0
+; CHECK: [[A1_P2:%[0-9]+]]:_(s64) = COPY $x1
+; CHECK: [[A1_MERGE:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[A1_P1]](s64), [[A1_P2]](s64)
+; CHECK: [[A2_P1:%[0-9]+]]:_(s64) = COPY $x2
+; CHECK: [[A2_P2:%[0-9]+]]:_(s64) = COPY $x3
+; CHECK: [[A2_MERGE:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[A2_P1]](s64), [[A2_P2]](s64)
+; CHECK: G_STORE [[A2_MERGE]](s128)
+define void @callee_s128(i128 %a, i128 %b, i128 *%ptr) {
+  store i128 %b, i128 *%ptr
+  ret void
+}
+
+; Check we can lower outgoing s128 arguments into s64 gprs.
+; CHECK-LABEL: name: caller_s128
+; CHECK: [[PTR:%[0-9]+]]:_(p0) = COPY $x0
+; CHECK: [[LARGE_VAL:%[0-9]+]]:_(s128) = G_LOAD [[PTR]](p0)
+; CHECK: ADJCALLSTACKDOWN 0, 0, implicit-def $sp, implicit $sp
+; CHECK: [[A1_P1:%[0-9]+]]:_(s64), [[A1_P2:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[LARGE_VAL]](s128)
+; CHECK: [[A2_P1:%[0-9]+]]:_(s64), [[A2_P2:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES %1(s128)
+; CHECK: $x0 = COPY [[A1_P1]](s64)
+; CHECK: $x1 = COPY [[A1_P2]](s64)
+; CHECK: $x2 = COPY [[A2_P1]](s64)
+; CHECK: $x3 = COPY [[A2_P2]](s64)
+; CHECK: $x4 = COPY [[PTR]](p0)
+; CHECK: BL @callee_s128, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $x0, implicit $x1, implicit $x2, implicit $x3, implicit $x4
+; CHECK: ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp
+define void @caller_s128(i128 *%ptr) {
+  %v = load i128, i128 *%ptr
+  call void @callee_s128(i128 %v, i128 %v, i128 *%ptr)
+  ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-fallback.ll b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-fallback.ll
index fa6f19a5c..78ae99d98 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-fallback.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-fallback.ll
@@ -17,11 +17,11 @@ target triple = "aarch64--"
 
 ; We use __fixunstfti as the common denominator for __fixunstfti on Linux and
 ; ___fixunstfti on iOS
-; ERROR: unable to lower arguments: i128 (i128)* (in function: ABIi128)
+; ERROR: unable to translate instruction: ret
 ; FALLBACK: ldr q0,
 ; FALLBACK-NEXT: bl __fixunstfti
 ;
-; FALLBACK-WITH-REPORT-ERR: remark: <unknown>:0:0: unable to lower arguments: i128 (i128)* (in function: ABIi128)
+; FALLBACK-WITH-REPORT-ERR: unable to translate instruction: ret
 ; FALLBACK-WITH-REPORT-ERR: warning: Instruction selection used fallback path for ABIi128
 ; FALLBACK-WITH-REPORT-OUT-LABEL: ABIi128:
 ; FALLBACK-WITH-REPORT-OUT: ldr q0,
@@ -54,26 +54,6 @@ false:
 
 }
 
-; FALLBACK-WITH-REPORT-ERR: remark: <unknown>:0:0: unable to legalize instruction: %3:_(s32) = G_LOAD %1:_(p0) :: (load 3 from `i24* undef`, align 1) (in function: odd_type_load)
-; FALLBACK-WITH-REPORT-ERR: warning: Instruction selection used fallback path for odd_type_load
-; FALLBACK-WITH-REPORT-OUT-LABEL: odd_type_load
-define i32 @odd_type_load() {
-entry:
-  %ld = load i24, i24* undef, align 1
-  %cst = zext i24 %ld to i32
-  ret i32 %cst
-}
-
-  ; General legalizer inability to handle types whose size wasn't a power of 2.
-; FALLBACK-WITH-REPORT-ERR: remark: <unknown>:0:0: unable to legalize instruction: G_STORE %1:_(s42), %0:_(p0) :: (store 6 into %ir.addr, align 8) (in function: odd_type)
-; FALLBACK-WITH-REPORT-ERR: warning: Instruction selection used fallback path for odd_type
-; FALLBACK-WITH-REPORT-OUT-LABEL: odd_type:
-define void @odd_type(i42* %addr) {
-  %val42 = load i42, i42* %addr
-  store i42 %val42, i42* %addr
-  ret void
-}
-
 ; FALLBACK-WITH-REPORT-ERR: remark: <unknown>:0:0: unable to legalize instruction: G_STORE %1:_(<7 x s32>), %0:_(p0) :: (store 28 into %ir.addr, align 32) (in function: odd_vector)
 ; FALLBACK-WITH-REPORT-ERR: warning: Instruction selection used fallback path for odd_vector
 ; FALLBACK-WITH-REPORT-OUT-LABEL: odd_vector:
@@ -93,7 +73,7 @@ define i128 @sequence_sizes([8 x i8] %in) {
 }
 
 ; Just to make sure we don't accidentally emit a normal load/store.
-; FALLBACK-WITH-REPORT-ERR: remark: <unknown>:0:0: cannot select: %2:gpr(s64) = G_LOAD %0:gpr(p0) :: (load seq_cst 8 from %ir.addr)  (in function: atomic_ops)
+; FALLBACK-WITH-REPORT-ERR: cannot select: G_STORE %1:gpr(s64), %0:gpr64sp(p0) :: (store unordered 8 into %ir.addr) (in function: atomic_ops)
 ; FALLBACK-WITH-REPORT-ERR: warning: Instruction selection used fallback path for atomic_ops
 ; FALLBACK-WITH-REPORT-LABEL: atomic_ops:
 define i64 @atomic_ops(i64* %addr) {
@@ -116,24 +96,6 @@ define void @test_write_register_intrin() {
 @_ZTIi = external global i8*
 declare i32 @__gxx_personality_v0(...)
 
-; Check that we fallback on invoke translation failures.
-; FALLBACK-WITH-REPORT-ERR: remark: <unknown>:0:0: unable to translate instruction: invoke: '  invoke void %callee(i128 0)
-; FALLBACK-WITH-REPORT-NEXT:   to label %continue unwind label %broken' (in function: invoke_weird_type)
-; FALLBACK-WITH-REPORT-ERR: warning: Instruction selection used fallback path for invoke_weird_type
-; FALLBACK-WITH-REPORT-OUT-LABEL: invoke_weird_type:
-define void @invoke_weird_type(void(i128)* %callee) personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
-  invoke void %callee(i128 0)
-    to label %continue unwind label %broken
-
-broken:
-  landingpad { i8*, i32 } catch i8* bitcast(i8** @_ZTIi to i8*)
-  ret void
-
-continue:
-  ret void
-}
-
-; Check that we fallback on invoke translation failures.
 ; FALLBACK-WITH-REPORT-ERR: remark: <unknown>:0:0: unable to legalize instruction: %0:_(s128) = G_FCONSTANT fp128 0xL00000000000000004000000000000000
 ; FALLBACK-WITH-REPORT-ERR: warning: Instruction selection used fallback path for test_quad_dump
 ; FALLBACK-WITH-REPORT-OUT-LABEL: test_quad_dump:
@@ -205,7 +167,7 @@ define void @nonpow2_load_narrowing() {
   ret void
 }
 
-; FALLBACK-WITH-REPORT-ERR: remark: <unknown>:0:0: unable to legalize instruction: %4:_(s64) = G_EXTRACT %3:_(s96), 0 (in function: nonpow2_store_narrowing)
+; FALLBACK-WITH-REPORT-ERR: remark: <unknown>:0:0: cannot select: %14:gpr64(s64), %15:gpr(s1) = G_UADDE %17:gpr, %17:gpr, %13:gpr (in function: nonpow2_store_narrowing)
 ; FALLBACK-WITH-REPORT-ERR: warning: Instruction selection used fallback path for nonpow2_store_narrowing
 ; FALLBACK-WITH-REPORT-OUT-LABEL: nonpow2_store_narrowing:
 define void @nonpow2_store_narrowing(i96* %c) {
@@ -228,10 +190,22 @@ define void @nonpow2_vector_add_fewerelements() {
   ret void
 }
 
-%swift_error = type {i64, i8}
+; Currently can't handle dealing with a split type (s128 -> 2 x s64) on the stack yet.
+declare void @use_s128(i128 %a, i128 %b)
+; FALLBACK-WITH-REPORT-ERR: remark: <unknown>:0:0: unable to lower arguments: i32 (i32, i128, i32, i32, i32, i128, i32)* (in function: fn1)
+; FALLBACK-WITH-REPORT-ERR: warning: Instruction selection used fallback path for fn1
+; FALLBACK-WITH-REPORT-OUT-LABEL: fn1:
+define i32 @fn1(i32 %p1, i128 %p2, i32 %p3, i32 %p4, i32 %p5, i128 %p6, i32 %p7) {
+entry:
+  call void @use_s128(i128 %p2, i128 %p6)
+  ret i32 0
+}
 
-; FALLBACK-WITH-REPORT-ERR: remark: <unknown>:0:0: unable to lower arguments due to swiftself: void (%swift_error**)* (in function: swiftself_param)
-; FALLBACK-WITH-REPORT-ERR: warning: Instruction selection used fallback path for swiftself_param
-define void @swiftself_param(%swift_error** swiftself %error_ptr_ref) {
-  ret void
+; FALLBACK-WITH-REPORT-ERR: remark: <unknown>:0:0: cannot select: %2:fpr(<4 x s16>) = G_ZEXT %0:fpr(<4 x s8>) (in function: zext_v4s8)
+; FALLBACK-WITH-REPORT-ERR: warning: Instruction selection used fallback path for zext_v4s8
+; FALLBACK-WITH-REPORT-OUT-LABEL: zext_v4s8
+define <4 x i16> @zext_v4s8(<4 x i8> %in) {
+  %ext = zext <4 x i8> %in to <4 x i16>
+  ret <4 x i16> %ext
 }
+
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator-gep.ll b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator-gep.ll
new file mode 100644
index 000000000..457a5337d
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator-gep.ll
@@ -0,0 +1,51 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; RUN: llc -O0 -stop-after=irtranslator -global-isel -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefix=O0
+; RUN: llc -O3 -stop-after=irtranslator -global-isel -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefix=O3
+target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64--"
+
+define i32 @cse_gep([4 x i32]* %ptr, i32 %idx) {
+  ; O0-LABEL: name: cse_gep
+  ; O0: bb.1 (%ir-block.0):
+  ; O0:   liveins: $w1, $x0
+  ; O0:   [[COPY:%[0-9]+]]:_(p0) = COPY $x0
+  ; O0:   [[COPY1:%[0-9]+]]:_(s32) = COPY $w1
+  ; O0:   [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[COPY1]](s32)
+  ; O0:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
+  ; O0:   [[MUL:%[0-9]+]]:_(s64) = G_MUL [[C]], [[SEXT]]
+  ; O0:   [[GEP:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[MUL]](s64)
+  ; O0:   [[COPY2:%[0-9]+]]:_(p0) = COPY [[GEP]](p0)
+  ; O0:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY2]](p0) :: (load 4 from %ir.gep1)
+  ; O0:   [[MUL1:%[0-9]+]]:_(s64) = G_MUL [[C]], [[SEXT]]
+  ; O0:   [[GEP1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[MUL1]](s64)
+  ; O0:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
+  ; O0:   [[GEP2:%[0-9]+]]:_(p0) = G_PTR_ADD [[GEP1]], [[C1]](s64)
+  ; O0:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[GEP2]](p0) :: (load 4 from %ir.gep2)
+  ; O0:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[LOAD1]], [[LOAD1]]
+  ; O0:   $w0 = COPY [[ADD]](s32)
+  ; O0:   RET_ReallyLR implicit $w0
+  ; O3-LABEL: name: cse_gep
+  ; O3: bb.1 (%ir-block.0):
+  ; O3:   liveins: $w1, $x0
+  ; O3:   [[COPY:%[0-9]+]]:_(p0) = COPY $x0
+  ; O3:   [[COPY1:%[0-9]+]]:_(s32) = COPY $w1
+  ; O3:   [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[COPY1]](s32)
+  ; O3:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
+  ; O3:   [[MUL:%[0-9]+]]:_(s64) = G_MUL [[C]], [[SEXT]]
+  ; O3:   [[GEP:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[MUL]](s64)
+  ; O3:   [[COPY2:%[0-9]+]]:_(p0) = COPY [[GEP]](p0)
+  ; O3:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY2]](p0) :: (load 4 from %ir.gep1)
+  ; O3:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
+  ; O3:   [[GEP1:%[0-9]+]]:_(p0) = G_PTR_ADD [[GEP]], [[C1]](s64)
+  ; O3:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[GEP1]](p0) :: (load 4 from %ir.gep2)
+  ; O3:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[LOAD1]], [[LOAD1]]
+  ; O3:   $w0 = COPY [[ADD]](s32)
+  ; O3:   RET_ReallyLR implicit $w0
+  %sidx = sext i32 %idx to i64
+  %gep1 = getelementptr inbounds [4 x i32], [4 x i32]* %ptr, i64 %sidx, i64 0
+  %v1 = load i32, i32* %gep1
+  %gep2 = getelementptr inbounds [4 x i32], [4 x i32]* %ptr, i64 %sidx, i64 1
+  %v2 = load i32, i32* %gep2
+  %res = add i32 %v2, %v2
+  ret i32 %res
+}
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator-switch.ll b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator-switch.ll
index f50752163..55bb11b75 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator-switch.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator-switch.ll
@@ -781,9 +781,9 @@ define void @jt_multiple_jump_tables(%1* %arg, i32 %arg1, i32* %arg2) {
   ; CHECK:   [[PHI:%[0-9]+]]:_(s64) = G_PHI [[C55]](s64), %bb.1, [[C56]](s64), %bb.2, [[C57]](s64), %bb.3, [[C58]](s64), %bb.4, [[C59]](s64), %bb.5, [[C60]](s64), %bb.6, [[C61]](s64), %bb.7, [[C62]](s64), %bb.8, [[C63]](s64), %bb.9, [[C64]](s64), %bb.10, [[C65]](s64), %bb.11, [[C66]](s64), %bb.12, [[C67]](s64), %bb.13, [[C68]](s64), %bb.14, [[C69]](s64), %bb.15, [[C70]](s64), %bb.16, [[C71]](s64), %bb.17, [[C72]](s64), %bb.18, [[C73]](s64), %bb.19, [[C74]](s64), %bb.20, [[C75]](s64), %bb.21, [[C76]](s64), %bb.22, [[C77]](s64), %bb.23, [[C78]](s64), %bb.24, [[C79]](s64), %bb.25, [[C80]](s64), %bb.26, [[C81]](s64), %bb.27, [[C82]](s64), %bb.28, [[C83]](s64), %bb.29, [[C84]](s64), %bb.30, [[C85]](s64), %bb.31, [[C86]](s64), %bb.32, [[C87]](s64), %bb.33, [[C88]](s64), %bb.34, [[C89]](s64), %bb.35, [[C90]](s64), %bb.36, [[C91]](s64), %bb.37, [[C92]](s64), %bb.38, [[C93]](s64), %bb.39, [[C94]](s64), %bb.40, [[C95]](s64), %bb.41, [[C96]](s64), %bb.42, [[C97]](s64), %bb.43, [[C98]](s64), %bb.44, [[C99]](s64), %bb.45, [[C100]](s64), %bb.46, [[C101]](s64), %bb.47, [[C102]](s64), %bb.48, [[C103]](s64), %bb.49, [[C104]](s64), %bb.50, [[C105]](s64), %bb.51, [[C106]](s64), %bb.52, [[C107]](s64), %bb.53, [[C108]](s64), %bb.54, [[C109]](s64), %bb.55
   ; CHECK:   [[C110:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
   ; CHECK:   [[MUL:%[0-9]+]]:_(s64) = G_MUL [[C110]], [[PHI]]
-  ; CHECK:   [[GEP:%[0-9]+]]:_(p0) = G_GEP [[GV]], [[MUL]](s64)
+  ; CHECK:   [[GEP:%[0-9]+]]:_(p0) = G_PTR_ADD [[GV]], [[MUL]](s64)
   ; CHECK:   [[C111:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-  ; CHECK:   [[GEP1:%[0-9]+]]:_(p0) = G_GEP [[GEP]], [[C111]](s64)
+  ; CHECK:   [[GEP1:%[0-9]+]]:_(p0) = G_PTR_ADD [[GEP]], [[C111]](s64)
   ; CHECK:   [[LOAD:%[0-9]+]]:_(p0) = G_LOAD [[GEP1]](p0) :: (load 8 from %ir.tmp59)
   ; CHECK:   ADJCALLSTACKDOWN 0, 0, implicit-def $sp, implicit $sp
   ; CHECK:   $x0 = COPY [[COPY]](p0)
@@ -1412,3 +1412,45 @@ bb4:                                              ; preds = %bb1
 
 declare i64* @ham(i32)
 
+define internal void @bar() unnamed_addr #1 {
+  ; CHECK-LABEL: name: bar
+  ; CHECK: bb.1 (%ir-block.0):
+    unreachable
+}
+
+define i1 @i1_value_cmp_is_signed(i1) {
+  ; CHECK-LABEL: name: i1_value_cmp_is_signed
+  ; CHECK: bb.1.Entry:
+  ; CHECK:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   liveins: $w0
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $w0
+  ; CHECK:   [[TRUNC:%[0-9]+]]:_(s1) = G_TRUNC [[COPY]](s32)
+  ; CHECK:   [[C:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
+  ; CHECK:   [[C1:%[0-9]+]]:_(s1) = G_CONSTANT i1 false
+  ; CHECK:   [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(sle), [[TRUNC]](s1), [[C1]]
+  ; CHECK:   G_BRCOND [[ICMP]](s1), %bb.3
+  ; CHECK:   G_BR %bb.2
+  ; CHECK: bb.2.BadValue:
+  ; CHECK:   successors:
+  ; CHECK:   ADJCALLSTACKDOWN 0, 0, implicit-def $sp, implicit $sp
+  ; CHECK:   BL @bar, csr_aarch64_aapcs, implicit-def $lr, implicit $sp
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp
+  ; CHECK: bb.3.OkValue:
+  ; CHECK:   [[ZEXT:%[0-9]+]]:_(s8) = G_ZEXT [[TRUNC]](s1)
+  ; CHECK:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ZEXT]](s8)
+  ; CHECK:   $w0 = COPY [[ANYEXT]](s32)
+  ; CHECK:   RET_ReallyLR implicit $w0
+Entry:
+  switch i1 %0, label %BadValue [
+    i1 false, label %OkValue
+    i1 true, label %OkValue
+  ]
+
+BadValue:
+  call fastcc void @bar()
+  unreachable
+
+OkValue:
+  ret i1 %0
+}
+
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll
index 6891adc54..f200cf2f0 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll
@@ -351,7 +351,9 @@ define void @trunc(i64 %a) {
 ; CHECK: [[SUM2:%.*]]:_(s64) = G_ADD [[VAL1]], [[VAL2]]
 ; CHECK: [[VAL3:%[0-9]+]]:_(s64) = G_LOAD [[ADDR]](p0) :: (volatile load 8 from %ir.addr)
 ; CHECK: [[SUM3:%[0-9]+]]:_(s64) = G_ADD [[SUM2]], [[VAL3]]
-; CHECK: $x0 = COPY [[SUM3]]
+; CHECK: [[VAL4:%[0-9]+]]:_(s64) = G_LOAD [[ADDR]](p0) :: (load 8 from %ir.addr, !range !0)
+; CHECK: [[SUM4:%[0-9]+]]:_(s64) = G_ADD [[SUM3]], [[VAL4]]
+; CHECK: $x0 = COPY [[SUM4]]
 ; CHECK: RET_ReallyLR implicit $x0
 define i64 @load(i64* %addr, i64 addrspace(42)* %addr42) {
   %val1 = load i64, i64* %addr, align 16
@@ -361,7 +363,10 @@ define i64 @load(i64* %addr, i64 addrspace(42)* %addr42) {
 
   %val3 = load volatile i64, i64* %addr
   %sum3 = add i64 %sum2, %val3
-  ret i64 %sum3
+
+  %val4 = load i64, i64* %addr, !range !0
+  %sum4 = add i64 %sum3, %val4
+  ret i64 %sum4
 }
 
 ; CHECK-LABEL: name: store
@@ -384,8 +389,7 @@ define void @store(i64* %addr, i64 addrspace(42)* %addr42, i64 %val1, i64 %val2)
 ; CHECK-LABEL: name: intrinsics
 ; CHECK: [[CUR:%[0-9]+]]:_(s32) = COPY $w0
 ; CHECK: [[BITS:%[0-9]+]]:_(s32) = COPY $w1
-; CHECK: [[CREG:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-; CHECK: [[PTR:%[0-9]+]]:_(p0) = G_INTRINSIC intrinsic(@llvm.returnaddress), [[CREG]]
+; CHECK: [[PTR:%[0-9]+]]:_(p0) = G_INTRINSIC intrinsic(@llvm.returnaddress), 0
 ; CHECK: [[PTR_VEC:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.ptr.vec
 ; CHECK: [[VEC:%[0-9]+]]:_(<8 x s8>) = G_LOAD [[PTR_VEC]]
 ; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.aarch64.neon.st2), [[VEC]](<8 x s8>), [[VEC]](<8 x s8>), [[PTR]](p0)
@@ -601,10 +605,10 @@ define i8* @test_constant_null() {
 ; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0
 ; CHECK: [[VAL1:%[0-9]+]]:_(s8) = G_LOAD %0(p0) :: (load 1 from %ir.addr, align 4)
 ; CHECK: [[CST1:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-; CHECK: [[GEP1:%[0-9]+]]:_(p0) = G_GEP [[ADDR]], [[CST1]](s64)
+; CHECK: [[GEP1:%[0-9]+]]:_(p0) = G_PTR_ADD [[ADDR]], [[CST1]](s64)
 ; CHECK: [[VAL2:%[0-9]+]]:_(s32) = G_LOAD [[GEP1]](p0) :: (load 4 from %ir.addr + 4)
 ; CHECK: G_STORE [[VAL1]](s8), [[ADDR]](p0) :: (store 1 into %ir.addr, align 4)
-; CHECK: [[GEP2:%[0-9]+]]:_(p0) = G_GEP [[ADDR]], [[CST1]](s64)
+; CHECK: [[GEP2:%[0-9]+]]:_(p0) = G_PTR_ADD [[ADDR]], [[CST1]](s64)
 ; CHECK: G_STORE [[VAL2]](s32), [[GEP2]](p0) :: (store 4 into %ir.addr + 4)
 define void @test_struct_memops({ i8, i32 }* %addr) {
   %val = load { i8, i32 }, { i8, i32 }* %addr
@@ -728,7 +732,7 @@ define float @test_fneg_fmf(float %arg1) {
 ; CHECK: [[VAL:%[0-9]+]]:_(s32), [[OVERFLOW:%[0-9]+]]:_(s1) = G_SADDO [[LHS]], [[RHS]]
 ; CHECK: G_STORE [[VAL]](s32), [[ADDR]](p0) :: (store 4 into %ir.addr)
 ; CHECK: [[CST:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-; CHECK: [[GEP:%[0-9]+]]:_(p0) = G_GEP [[ADDR]], [[CST]](s64)
+; CHECK: [[GEP:%[0-9]+]]:_(p0) = G_PTR_ADD [[ADDR]], [[CST]](s64)
 ; CHECK: G_STORE [[OVERFLOW]](s1), [[GEP]](p0) :: (store 1 into %ir.addr + 4, align 4)
 declare { i32, i1 } @llvm.sadd.with.overflow.i32(i32, i32)
 define void @test_sadd_overflow(i32 %lhs, i32 %rhs, { i32, i1 }* %addr) {
@@ -744,7 +748,7 @@ define void @test_sadd_overflow(i32 %lhs, i32 %rhs, { i32, i1 }* %addr) {
 ; CHECK: [[VAL:%[0-9]+]]:_(s32), [[OVERFLOW:%[0-9]+]]:_(s1) = G_UADDO [[LHS]], [[RHS]]
 ; CHECK: G_STORE [[VAL]](s32), [[ADDR]](p0) :: (store 4 into %ir.addr)
 ; CHECK: [[CST:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-; CHECK: [[GEP:%[0-9]+]]:_(p0) = G_GEP [[ADDR]], [[CST]](s64)
+; CHECK: [[GEP:%[0-9]+]]:_(p0) = G_PTR_ADD [[ADDR]], [[CST]](s64)
 ; CHECK: G_STORE [[OVERFLOW]](s1), [[GEP]](p0) :: (store 1 into %ir.addr + 4, align 4)
 declare { i32, i1 } @llvm.uadd.with.overflow.i32(i32, i32)
 define void @test_uadd_overflow(i32 %lhs, i32 %rhs, { i32, i1 }* %addr) {
@@ -760,7 +764,7 @@ define void @test_uadd_overflow(i32 %lhs, i32 %rhs, { i32, i1 }* %addr) {
 ; CHECK: [[VAL:%[0-9]+]]:_(s32), [[OVERFLOW:%[0-9]+]]:_(s1) = G_SSUBO [[LHS]], [[RHS]]
 ; CHECK: G_STORE [[VAL]](s32), [[ADDR]](p0) :: (store 4 into %ir.subr)
 ; CHECK: [[CST:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-; CHECK: [[GEP:%[0-9]+]]:_(p0) = G_GEP [[ADDR]], [[CST]](s64)
+; CHECK: [[GEP:%[0-9]+]]:_(p0) = G_PTR_ADD [[ADDR]], [[CST]](s64)
 ; CHECK: G_STORE [[OVERFLOW]](s1), [[GEP]](p0) :: (store 1 into %ir.subr + 4, align 4)
 declare { i32, i1 } @llvm.ssub.with.overflow.i32(i32, i32)
 define void @test_ssub_overflow(i32 %lhs, i32 %rhs, { i32, i1 }* %subr) {
@@ -776,7 +780,7 @@ define void @test_ssub_overflow(i32 %lhs, i32 %rhs, { i32, i1 }* %subr) {
 ; CHECK: [[VAL:%[0-9]+]]:_(s32), [[OVERFLOW:%[0-9]+]]:_(s1) = G_USUBO [[LHS]], [[RHS]]
 ; CHECK: G_STORE [[VAL]](s32), [[ADDR]](p0) :: (store 4 into %ir.subr)
 ; CHECK: [[CST:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-; CHECK: [[GEP:%[0-9]+]]:_(p0) = G_GEP [[ADDR]], [[CST]](s64)
+; CHECK: [[GEP:%[0-9]+]]:_(p0) = G_PTR_ADD [[ADDR]], [[CST]](s64)
 ; CHECK: G_STORE [[OVERFLOW]](s1), [[GEP]](p0) :: (store 1 into %ir.subr + 4, align 4)
 declare { i32, i1 } @llvm.usub.with.overflow.i32(i32, i32)
 define void @test_usub_overflow(i32 %lhs, i32 %rhs, { i32, i1 }* %subr) {
@@ -792,7 +796,7 @@ define void @test_usub_overflow(i32 %lhs, i32 %rhs, { i32, i1 }* %subr) {
 ; CHECK: [[VAL:%[0-9]+]]:_(s32), [[OVERFLOW:%[0-9]+]]:_(s1) = G_SMULO [[LHS]], [[RHS]]
 ; CHECK: G_STORE [[VAL]](s32), [[ADDR]](p0) :: (store 4 into %ir.addr)
 ; CHECK: [[CST:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-; CHECK: [[GEP:%[0-9]+]]:_(p0) = G_GEP [[ADDR]], [[CST]](s64)
+; CHECK: [[GEP:%[0-9]+]]:_(p0) = G_PTR_ADD [[ADDR]], [[CST]](s64)
 ; CHECK: G_STORE [[OVERFLOW]](s1), [[GEP]](p0) :: (store 1 into %ir.addr + 4, align 4)
 declare { i32, i1 } @llvm.smul.with.overflow.i32(i32, i32)
 define void @test_smul_overflow(i32 %lhs, i32 %rhs, { i32, i1 }* %addr) {
@@ -808,7 +812,7 @@ define void @test_smul_overflow(i32 %lhs, i32 %rhs, { i32, i1 }* %addr) {
 ; CHECK: [[VAL:%[0-9]+]]:_(s32), [[OVERFLOW:%[0-9]+]]:_(s1) = G_UMULO [[LHS]], [[RHS]]
 ; CHECK: G_STORE [[VAL]](s32), [[ADDR]](p0) :: (store 4 into %ir.addr)
 ; CHECK: [[CST:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-; CHECK: [[GEP:%[0-9]+]]:_(p0) = G_GEP [[ADDR]], [[CST]](s64)
+; CHECK: [[GEP:%[0-9]+]]:_(p0) = G_PTR_ADD [[ADDR]], [[CST]](s64)
 ; CHECK: G_STORE [[OVERFLOW]](s1), [[GEP]](p0) :: (store 1 into %ir.addr + 4, align 4)
 declare { i32, i1 } @llvm.umul.with.overflow.i32(i32, i32)
 define void @test_umul_overflow(i32 %lhs, i32 %rhs, { i32, i1 }* %addr) {
@@ -821,13 +825,13 @@ define void @test_umul_overflow(i32 %lhs, i32 %rhs, { i32, i1 }* %addr) {
 ; CHECK: %0:_(p0) = COPY $x0
 ; CHECK: [[LD1:%[0-9]+]]:_(s8) = G_LOAD %0(p0) :: (load 1 from %ir.addr, align 4)
 ; CHECK: [[CST1:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-; CHECK: [[GEP1:%[0-9]+]]:_(p0) = G_GEP %0, [[CST1]](s64)
+; CHECK: [[GEP1:%[0-9]+]]:_(p0) = G_PTR_ADD %0, [[CST1]](s64)
 ; CHECK: [[LD2:%[0-9]+]]:_(s8) = G_LOAD [[GEP1]](p0) :: (load 1 from %ir.addr + 4, align 4)
 ; CHECK: [[CST2:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-; CHECK: [[GEP2:%[0-9]+]]:_(p0) = G_GEP %0, [[CST2]](s64)
+; CHECK: [[GEP2:%[0-9]+]]:_(p0) = G_PTR_ADD %0, [[CST2]](s64)
 ; CHECK: [[LD3:%[0-9]+]]:_(s32) = G_LOAD [[GEP2]](p0) :: (load 4 from %ir.addr + 8)
 ; CHECK: [[CST3:%[0-9]+]]:_(s64) = G_CONSTANT i64 12
-; CHECK: [[GEP3:%[0-9]+]]:_(p0) = G_GEP %0, [[CST3]](s64)
+; CHECK: [[GEP3:%[0-9]+]]:_(p0) = G_PTR_ADD %0, [[CST3]](s64)
 ; CHECK: [[LD4:%[0-9]+]]:_(s32) = G_LOAD [[GEP3]](p0) :: (load 4 from %ir.addr + 12)
 ; CHECK: $w0 = COPY [[LD3]](s32)
 %struct.nested = type {i8, { i8, i32 }, i32}
@@ -842,16 +846,16 @@ define i32 @test_extractvalue(%struct.nested* %addr) {
 ; CHECK: %1:_(p0) = COPY $x1
 ; CHECK: [[LD1:%[0-9]+]]:_(s8) = G_LOAD %0(p0) :: (load 1 from %ir.addr, align 4)
 ; CHECK: [[CST1:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-; CHECK: [[GEP1:%[0-9]+]]:_(p0) = G_GEP %0, [[CST1]](s64)
+; CHECK: [[GEP1:%[0-9]+]]:_(p0) = G_PTR_ADD %0, [[CST1]](s64)
 ; CHECK: [[LD2:%[0-9]+]]:_(s8) = G_LOAD [[GEP1]](p0) :: (load 1 from %ir.addr + 4, align 4)
 ; CHECK: [[CST2:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-; CHECK: [[GEP2:%[0-9]+]]:_(p0) = G_GEP %0, [[CST2]](s64)
+; CHECK: [[GEP2:%[0-9]+]]:_(p0) = G_PTR_ADD %0, [[CST2]](s64)
 ; CHECK: [[LD3:%[0-9]+]]:_(s32) = G_LOAD [[GEP2]](p0) :: (load 4 from %ir.addr + 8)
 ; CHECK: [[CST3:%[0-9]+]]:_(s64) = G_CONSTANT i64 12
-; CHECK: [[GEP3:%[0-9]+]]:_(p0) = G_GEP %0, [[CST3]](s64)
+; CHECK: [[GEP3:%[0-9]+]]:_(p0) = G_PTR_ADD %0, [[CST3]](s64)
 ; CHECK: [[LD4:%[0-9]+]]:_(s32) = G_LOAD [[GEP3]](p0) :: (load 4 from %ir.addr + 12)
 ; CHECK: G_STORE [[LD2]](s8), %1(p0) :: (store 1 into %ir.addr2, align 4)
-; CHECK: [[GEP4:%[0-9]+]]:_(p0) = G_GEP %1, [[CST1]](s64)
+; CHECK: [[GEP4:%[0-9]+]]:_(p0) = G_PTR_ADD %1, [[CST1]](s64)
 ; CHECK: G_STORE [[LD3]](s32), [[GEP4]](p0) :: (store 4 into %ir.addr2 + 4)
 define void @test_extractvalue_agg(%struct.nested* %addr, {i8, i32}* %addr2) {
   %struct = load %struct.nested, %struct.nested* %addr
@@ -876,20 +880,20 @@ define void @test_trivial_extract_ptr([1 x i8*] %s, i8 %val) {
 ; CHECK: %1:_(s32) = COPY $w1
 ; CHECK: [[LD1:%[0-9]+]]:_(s8) = G_LOAD %0(p0) :: (load 1 from %ir.addr, align 4)
 ; CHECK: [[CST1:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-; CHECK: [[GEP1:%[0-9]+]]:_(p0) = G_GEP %0, [[CST1]](s64)
+; CHECK: [[GEP1:%[0-9]+]]:_(p0) = G_PTR_ADD %0, [[CST1]](s64)
 ; CHECK: [[LD2:%[0-9]+]]:_(s8) = G_LOAD [[GEP1]](p0) :: (load 1 from %ir.addr + 4, align 4)
 ; CHECK: [[CST2:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-; CHECK: [[GEP2:%[0-9]+]]:_(p0) = G_GEP %0, [[CST2]](s64)
+; CHECK: [[GEP2:%[0-9]+]]:_(p0) = G_PTR_ADD %0, [[CST2]](s64)
 ; CHECK: [[LD3:%[0-9]+]]:_(s32) = G_LOAD [[GEP2]](p0) :: (load 4 from %ir.addr + 8)
 ; CHECK: [[CST3:%[0-9]+]]:_(s64) = G_CONSTANT i64 12
-; CHECK: [[GEP3:%[0-9]+]]:_(p0) = G_GEP %0, [[CST3]](s64)
+; CHECK: [[GEP3:%[0-9]+]]:_(p0) = G_PTR_ADD %0, [[CST3]](s64)
 ; CHECK: [[LD4:%[0-9]+]]:_(s32) = G_LOAD [[GEP3]](p0) :: (load 4 from %ir.addr + 12)
 ; CHECK: G_STORE [[LD1]](s8), %0(p0) :: (store 1 into %ir.addr, align 4)
-; CHECK: [[GEP4:%[0-9]+]]:_(p0) = G_GEP %0, [[CST1]](s64)
+; CHECK: [[GEP4:%[0-9]+]]:_(p0) = G_PTR_ADD %0, [[CST1]](s64)
 ; CHECK: G_STORE [[LD2]](s8), [[GEP4]](p0) :: (store 1 into %ir.addr + 4, align 4)
-; CHECK: [[GEP5:%[0-9]+]]:_(p0) = G_GEP %0, [[CST2]](s64)
+; CHECK: [[GEP5:%[0-9]+]]:_(p0) = G_PTR_ADD %0, [[CST2]](s64)
 ; CHECK: G_STORE %1(s32), [[GEP5]](p0) :: (store 4 into %ir.addr + 8)
-; CHECK: [[GEP6:%[0-9]+]]:_(p0) = G_GEP %0, [[CST3]](s64)
+; CHECK: [[GEP6:%[0-9]+]]:_(p0) = G_PTR_ADD %0, [[CST3]](s64)
 ; CHECK: G_STORE [[LD4]](s32), [[GEP6]](p0) :: (store 4 into %ir.addr + 12)
 define void @test_insertvalue(%struct.nested* %addr, i32 %val) {
   %struct = load %struct.nested, %struct.nested* %addr
@@ -921,23 +925,23 @@ define [1 x i8*] @test_trivial_insert_ptr([1 x i8*] %s, i8* %val) {
 ; CHECK: %1:_(p0) = COPY $x1
 ; CHECK: [[LD1:%[0-9]+]]:_(s8) = G_LOAD %1(p0) :: (load 1 from %ir.addr2, align 4)
 ; CHECK: [[CST1:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-; CHECK: [[GEP1:%[0-9]+]]:_(p0) = G_GEP %1, [[CST1]](s64)
+; CHECK: [[GEP1:%[0-9]+]]:_(p0) = G_PTR_ADD %1, [[CST1]](s64)
 ; CHECK: [[LD2:%[0-9]+]]:_(s32) = G_LOAD [[GEP1]](p0) :: (load 4 from %ir.addr2 + 4)
 ; CHECK: [[LD3:%[0-9]+]]:_(s8) = G_LOAD %0(p0) :: (load 1 from %ir.addr, align 4)
-; CHECK: [[GEP2:%[0-9]+]]:_(p0) = G_GEP %0, [[CST1]](s64)
+; CHECK: [[GEP2:%[0-9]+]]:_(p0) = G_PTR_ADD %0, [[CST1]](s64)
 ; CHECK: [[LD4:%[0-9]+]]:_(s8) = G_LOAD [[GEP2]](p0) :: (load 1 from %ir.addr + 4, align 4)
 ; CHECK: [[CST3:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-; CHECK: [[GEP3:%[0-9]+]]:_(p0) = G_GEP %0, [[CST3]](s64)
+; CHECK: [[GEP3:%[0-9]+]]:_(p0) = G_PTR_ADD %0, [[CST3]](s64)
 ; CHECK: [[LD5:%[0-9]+]]:_(s32) = G_LOAD [[GEP3]](p0) :: (load 4 from %ir.addr + 8)
 ; CHECK: [[CST4:%[0-9]+]]:_(s64) = G_CONSTANT i64 12
-; CHECK: [[GEP4:%[0-9]+]]:_(p0) = G_GEP %0, [[CST4]](s64)
+; CHECK: [[GEP4:%[0-9]+]]:_(p0) = G_PTR_ADD %0, [[CST4]](s64)
 ; CHECK: [[LD6:%[0-9]+]]:_(s32) = G_LOAD [[GEP4]](p0) :: (load 4 from %ir.addr + 12)
 ; CHECK: G_STORE [[LD3]](s8), %0(p0) :: (store 1 into %ir.addr, align 4)
-; CHECK: [[GEP5:%[0-9]+]]:_(p0) = G_GEP %0, [[CST1]](s64)
+; CHECK: [[GEP5:%[0-9]+]]:_(p0) = G_PTR_ADD %0, [[CST1]](s64)
 ; CHECK: G_STORE [[LD1]](s8), [[GEP5]](p0) :: (store 1 into %ir.addr + 4, align 4)
-; CHECK: [[GEP6:%[0-9]+]]:_(p0) = G_GEP %0, [[CST3]](s64)
+; CHECK: [[GEP6:%[0-9]+]]:_(p0) = G_PTR_ADD %0, [[CST3]](s64)
 ; CHECK: G_STORE [[LD2]](s32), [[GEP6]](p0) :: (store 4 into %ir.addr + 8)
-; CHECK: [[GEP7:%[0-9]+]]:_(p0) = G_GEP %0, [[CST4]](s64)
+; CHECK: [[GEP7:%[0-9]+]]:_(p0) = G_PTR_ADD %0, [[CST4]](s64)
 ; CHECK: G_STORE [[LD6]](s32), [[GEP7]](p0) :: (store 4 into %ir.addr + 12)
 define void @test_insertvalue_agg(%struct.nested* %addr, {i8, i32}* %addr2) {
   %smallstruct = load {i8, i32}, {i8, i32}* %addr2
@@ -1130,24 +1134,39 @@ define void @test_memcpy(i8* %dst, i8* %src, i64 %size) {
 ; CHECK: [[DST:%[0-9]+]]:_(p0) = COPY $x0
 ; CHECK: [[SRC:%[0-9]+]]:_(p0) = COPY $x1
 ; CHECK: [[SIZE:%[0-9]+]]:_(s64) = COPY $x2
-; CHECK: $x0 = COPY [[DST]]
-; CHECK: $x1 = COPY [[SRC]]
-; CHECK: $x2 = COPY [[SIZE]]
-; CHECK: BL &memcpy, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $x0, implicit $x1, implicit $x2
+; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[DST]](p0), [[SRC]](p0), [[SIZE]](s64), 0 :: (store 1 into %ir.dst), (load 1 from %ir.src)
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 %size, i1 0)
   ret void
 }
 
+define void @test_memcpy_tail(i8* %dst, i8* %src, i64 %size) {
+; CHECK-LABEL: name: test_memcpy_tail
+; CHECK: [[DST:%[0-9]+]]:_(p0) = COPY $x0
+; CHECK: [[SRC:%[0-9]+]]:_(p0) = COPY $x1
+; CHECK: [[SIZE:%[0-9]+]]:_(s64) = COPY $x2
+; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[DST]](p0), [[SRC]](p0), [[SIZE]](s64), 1 :: (store 1 into %ir.dst), (load 1 from %ir.src)
+  tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 %size, i1 0)
+  ret void
+}
+
+declare void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)*, i8 addrspace(1)*, i64, i1)
+define void @test_memcpy_nonzero_as(i8 addrspace(1)* %dst, i8 addrspace(1) * %src, i64 %size) {
+; CHECK-LABEL: name: test_memcpy_nonzero_as
+; CHECK: [[DST:%[0-9]+]]:_(p1) = COPY $x0
+; CHECK: [[SRC:%[0-9]+]]:_(p1) = COPY $x1
+; CHECK: [[SIZE:%[0-9]+]]:_(s64) = COPY $x2
+; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[DST]](p1), [[SRC]](p1), [[SIZE]](s64), 0 :: (store 1 into %ir.dst, addrspace 1), (load 1 from %ir.src, addrspace 1)
+  call void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)* %dst, i8 addrspace(1)* %src, i64 %size, i1 0)
+  ret void
+}
+
 declare void @llvm.memmove.p0i8.p0i8.i64(i8*, i8*, i64, i1)
 define void @test_memmove(i8* %dst, i8* %src, i64 %size) {
 ; CHECK-LABEL: name: test_memmove
 ; CHECK: [[DST:%[0-9]+]]:_(p0) = COPY $x0
 ; CHECK: [[SRC:%[0-9]+]]:_(p0) = COPY $x1
 ; CHECK: [[SIZE:%[0-9]+]]:_(s64) = COPY $x2
-; CHECK: $x0 = COPY [[DST]]
-; CHECK: $x1 = COPY [[SRC]]
-; CHECK: $x2 = COPY [[SIZE]]
-; CHECK: BL &memmove, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $x0, implicit $x1, implicit $x2
+; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memmove), [[DST]](p0), [[SRC]](p0), [[SIZE]](s64), 0 :: (store 1 into %ir.dst), (load 1 from %ir.src)
   call void @llvm.memmove.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 %size, i1 0)
   ret void
 }
@@ -1159,32 +1178,11 @@ define void @test_memset(i8* %dst, i8 %val, i64 %size) {
 ; CHECK: [[SRC_C:%[0-9]+]]:_(s32) = COPY $w1
 ; CHECK: [[SRC:%[0-9]+]]:_(s8) = G_TRUNC [[SRC_C]]
 ; CHECK: [[SIZE:%[0-9]+]]:_(s64) = COPY $x2
-; CHECK: $x0 = COPY [[DST]]
-; CHECK: [[SRC_TMP:%[0-9]+]]:_(s32) = G_ANYEXT [[SRC]]
-; CHECK: $w1 = COPY [[SRC_TMP]]
-; CHECK: $x2 = COPY [[SIZE]]
-; CHECK: BL &memset, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $x0, implicit $w1, implicit $x2
+; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memset), [[DST]](p0), [[SRC]](s8), [[SIZE]](s64), 0 :: (store 1 into %ir.dst)
   call void @llvm.memset.p0i8.i64(i8* %dst, i8 %val, i64 %size, i1 0)
   ret void
 }
 
-declare i64 @llvm.objectsize.i64(i8*, i1)
-declare i32 @llvm.objectsize.i32(i8*, i1)
-define void @test_objectsize(i8* %addr0, i8* %addr1) {
-; CHECK-LABEL: name: test_objectsize
-; CHECK: [[ADDR0:%[0-9]+]]:_(p0) = COPY $x0
-; CHECK: [[ADDR1:%[0-9]+]]:_(p0) = COPY $x1
-; CHECK: {{%[0-9]+}}:_(s64) = G_CONSTANT i64 -1
-; CHECK: {{%[0-9]+}}:_(s64) = G_CONSTANT i64 0
-; CHECK: {{%[0-9]+}}:_(s32) = G_CONSTANT i32 -1
-; CHECK: {{%[0-9]+}}:_(s32) = G_CONSTANT i32 0
-  %size64.0 = call i64 @llvm.objectsize.i64(i8* %addr0, i1 0)
-  %size64.intmin = call i64 @llvm.objectsize.i64(i8* %addr0, i1 1)
-  %size32.0 = call i32 @llvm.objectsize.i32(i8* %addr0, i1 0)
-  %size32.intmin = call i32 @llvm.objectsize.i32(i8* %addr0, i1 1)
-  ret void
-}
-
 define void @test_large_const(i128* %addr) {
 ; CHECK-LABEL: name: test_large_const
 ; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0
@@ -1388,6 +1386,16 @@ define i32 @test_ctpop_intrinsic(i32 %a) {
   ret i32 %res
 }
 
+declare i32 @llvm.bitreverse.i32(i32)
+define i32 @test_bitreverse_intrinsic(i32 %a) {
+; CHECK-LABEL: name: test_bitreverse
+; CHECK: [[A:%[0-9]+]]:_(s32) = COPY $w0
+; CHECK: [[RES:%[0-9]+]]:_(s32) = G_BITREVERSE [[A]]
+; CHECK: $w0 = COPY [[RES]]
+  %res = call i32 @llvm.bitreverse.i32(i32 %a)
+  ret i32 %res
+}
+
 declare void @llvm.lifetime.start.p0i8(i64, i8*)
 declare void @llvm.lifetime.end.p0i8(i64, i8*)
 define void @test_lifetime_intrin() {
@@ -1607,7 +1615,7 @@ define float @test_different_call_conv_target(float %x) {
 ; CHECK-LABEL: name: test_different_call_conv
 ; CHECK: [[X:%[0-9]+]]:_(s32) = COPY $s0
 ; CHECK: $s8 = COPY [[X]]
-; CHECK: BL @different_call_conv_target, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $s8, implicit-def $s0
+; CHECK: BL @different_call_conv_target, csr_aarch64_noregs, implicit-def $lr, implicit $sp, implicit $s8, implicit-def $s0
   %res = call ghccc float @different_call_conv_target(float %x)
   ret float %res
 }
@@ -1616,9 +1624,7 @@ define <2 x i32> @test_shufflevector_s32_v2s32(i32 %arg) {
 ; CHECK-LABEL: name: test_shufflevector_s32_v2s32
 ; CHECK: [[ARG:%[0-9]+]]:_(s32) = COPY $w0
 ; CHECK-DAG: [[UNDEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-; CHECK-DAG: [[C0:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-; CHECK-DAG: [[MASK:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[C0]](s32), [[C0]](s32)
-; CHECK: [[VEC:%[0-9]+]]:_(<2 x s32>) = G_SHUFFLE_VECTOR [[ARG]](s32), [[UNDEF]], [[MASK]](<2 x s32>)
+; CHECK: [[VEC:%[0-9]+]]:_(<2 x s32>) = G_SHUFFLE_VECTOR [[ARG]](s32), [[UNDEF]], shufflemask(0, 0)
 ; CHECK: $d0 = COPY [[VEC]](<2 x s32>)
   %vec = insertelement <1 x i32> undef, i32 %arg, i32 0
   %res = shufflevector <1 x i32> %vec, <1 x i32> undef, <2 x i32> zeroinitializer
@@ -1628,25 +1634,40 @@ define <2 x i32> @test_shufflevector_s32_v2s32(i32 %arg) {
 define i32 @test_shufflevector_v2s32_s32(<2 x i32> %arg) {
 ; CHECK-LABEL: name: test_shufflevector_v2s32_s32
 ; CHECK: [[ARG:%[0-9]+]]:_(<2 x s32>) = COPY $d0
-; CHECK-DAG: [[UNDEF:%[0-9]+]]:_(<2 x s32>) = G_IMPLICIT_DEF
-; CHECK-DAG: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-; CHECK: [[RES:%[0-9]+]]:_(s32) = G_SHUFFLE_VECTOR [[ARG]](<2 x s32>), [[UNDEF]], [[C1]](s32)
+; CHECK: [[RES:%[0-9]+]]:_(s32) = G_SHUFFLE_VECTOR [[ARG]](<2 x s32>), [[UNDEF]], shufflemask(1)
 ; CHECK: $w0 = COPY [[RES]](s32)
   %vec = shufflevector <2 x i32> %arg, <2 x i32> undef, <1 x i32> <i32 1>
   %res = extractelement <1 x i32> %vec, i32 0
   ret i32 %res
 }
 
-define <2 x i32> @test_shufflevector_v2s32_v2s32(<2 x i32> %arg) {
-; CHECK-LABEL: name: test_shufflevector_v2s32_v2s32
+define <2 x i32> @test_shufflevector_v2s32_v2s32_undef(<2 x i32> %arg) {
+; CHECK-LABEL: name: test_shufflevector_v2s32_v2s32_undef
 ; CHECK: [[ARG:%[0-9]+]]:_(<2 x s32>) = COPY $d0
 ; CHECK-DAG: [[UNDEF:%[0-9]+]]:_(<2 x s32>) = G_IMPLICIT_DEF
-; CHECK-DAG: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-; CHECK-DAG: [[C0:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-; CHECK-DAG: [[MASK:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[C1]](s32), [[C0]](s32)
-; CHECK: [[VEC:%[0-9]+]]:_(<2 x s32>) = G_SHUFFLE_VECTOR [[ARG]](<2 x s32>), [[UNDEF]], [[MASK]](<2 x s32>)
+; CHECK: [[VEC:%[0-9]+]]:_(<2 x s32>) = G_SHUFFLE_VECTOR [[ARG]](<2 x s32>), [[UNDEF]], shufflemask(undef, undef)
 ; CHECK: $d0 = COPY [[VEC]](<2 x s32>)
-  %res = shufflevector <2 x i32> %arg, <2 x i32> undef, <2 x i32> <i32 1, i32 0>
+  %res = shufflevector <2 x i32> %arg, <2 x i32> undef, <2 x i32> undef
+  ret <2 x i32> %res
+}
+
+define <2 x i32> @test_shufflevector_v2s32_v2s32_undef_0(<2 x i32> %arg) {
+; CHECK-LABEL: name: test_shufflevector_v2s32_v2s32_undef_0
+; CHECK: [[ARG:%[0-9]+]]:_(<2 x s32>) = COPY $d0
+; CHECK-DAG: [[UNDEF:%[0-9]+]]:_(<2 x s32>) = G_IMPLICIT_DEF
+; CHECK: [[VEC:%[0-9]+]]:_(<2 x s32>) = G_SHUFFLE_VECTOR [[ARG]](<2 x s32>), [[UNDEF]], shufflemask(undef, 0)
+; CHECK: $d0 = COPY [[VEC]](<2 x s32>)
+  %res = shufflevector <2 x i32> %arg, <2 x i32> undef, <2 x i32> <i32 undef, i32 0>
+  ret <2 x i32> %res
+}
+
+define <2 x i32> @test_shufflevector_v2s32_v2s32_0_undef(<2 x i32> %arg) {
+; CHECK-LABEL: name: test_shufflevector_v2s32_v2s32_0_undef
+; CHECK: [[ARG:%[0-9]+]]:_(<2 x s32>) = COPY $d0
+; CHECK-DAG: [[UNDEF:%[0-9]+]]:_(<2 x s32>) = G_IMPLICIT_DEF
+; CHECK: [[VEC:%[0-9]+]]:_(<2 x s32>) = G_SHUFFLE_VECTOR [[ARG]](<2 x s32>), [[UNDEF]], shufflemask(0, undef)
+; CHECK: $d0 = COPY [[VEC]](<2 x s32>)
+  %res = shufflevector <2 x i32> %arg, <2 x i32> undef, <2 x i32> <i32 0, i32 undef>
   ret <2 x i32> %res
 }
 
@@ -1654,10 +1675,7 @@ define i32 @test_shufflevector_v2s32_v3s32(<2 x i32> %arg) {
 ; CHECK-LABEL: name: test_shufflevector_v2s32_v3s32
 ; CHECK: [[ARG:%[0-9]+]]:_(<2 x s32>) = COPY $d0
 ; CHECK-DAG: [[UNDEF:%[0-9]+]]:_(<2 x s32>) = G_IMPLICIT_DEF
-; CHECK-DAG: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-; CHECK-DAG: [[C0:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-; CHECK-DAG: [[MASK:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[C1]](s32), [[C0]](s32), [[C1]](s32)
-; CHECK: [[VEC:%[0-9]+]]:_(<3 x s32>) = G_SHUFFLE_VECTOR [[ARG]](<2 x s32>), [[UNDEF]], [[MASK]](<3 x s32>)
+; CHECK: [[VEC:%[0-9]+]]:_(<3 x s32>) = G_SHUFFLE_VECTOR [[ARG]](<2 x s32>), [[UNDEF]], shufflemask(1, 0, 1)
 ; CHECK: G_EXTRACT_VECTOR_ELT [[VEC]](<3 x s32>)
   %vec = shufflevector <2 x i32> %arg, <2 x i32> undef, <3 x i32> <i32 1, i32 0, i32 1>
   %res = extractelement <3 x i32> %vec, i32 0
@@ -1668,12 +1686,7 @@ define <4 x i32> @test_shufflevector_v2s32_v4s32(<2 x i32> %arg1, <2 x i32> %arg
 ; CHECK-LABEL: name: test_shufflevector_v2s32_v4s32
 ; CHECK: [[ARG1:%[0-9]+]]:_(<2 x s32>) = COPY $d0
 ; CHECK: [[ARG2:%[0-9]+]]:_(<2 x s32>) = COPY $d1
-; CHECK: [[C0:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-; CHECK: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-; CHECK: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
-; CHECK: [[MASK:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[C0]](s32), [[C1]](s32), [[C2]](s32), [[C3]](s32)
-; CHECK: [[VEC:%[0-9]+]]:_(<4 x s32>) = G_SHUFFLE_VECTOR [[ARG1]](<2 x s32>), [[ARG2]], [[MASK]](<4 x s32>)
+; CHECK: [[VEC:%[0-9]+]]:_(<4 x s32>) = G_SHUFFLE_VECTOR [[ARG1]](<2 x s32>), [[ARG2]], shufflemask(0, 1, 2, 3)
 ; CHECK: $q0 = COPY [[VEC]](<4 x s32>)
   %res = shufflevector <2 x i32> %arg1, <2 x i32> %arg2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   ret <4 x i32> %res
@@ -1683,10 +1696,7 @@ define <2 x i32> @test_shufflevector_v4s32_v2s32(<4 x i32> %arg) {
 ; CHECK-LABEL: name: test_shufflevector_v4s32_v2s32
 ; CHECK: [[ARG:%[0-9]+]]:_(<4 x s32>) = COPY $q0
 ; CHECK-DAG: [[UNDEF:%[0-9]+]]:_(<4 x s32>) = G_IMPLICIT_DEF
-; CHECK-DAG: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-; CHECK-DAG: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
-; CHECK-DAG: [[MASK:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[C1]](s32), [[C3]](s32)
-; CHECK: [[VEC:%[0-9]+]]:_(<2 x s32>) = G_SHUFFLE_VECTOR [[ARG]](<4 x s32>), [[UNDEF]], [[MASK]](<2 x s32>)
+; CHECK: [[VEC:%[0-9]+]]:_(<2 x s32>) = G_SHUFFLE_VECTOR [[ARG]](<4 x s32>), [[UNDEF]], shufflemask(1, 3)
 ; CHECK: $d0 = COPY [[VEC]](<2 x s32>)
   %res = shufflevector <4 x i32> %arg, <4 x i32> undef, <2 x i32> <i32 1, i32 3>
   ret <2 x i32> %res
@@ -1697,24 +1707,7 @@ define <16 x i8> @test_shufflevector_v8s8_v16s8(<8 x i8> %arg1, <8 x i8> %arg2)
 ; CHECK-LABEL: name: test_shufflevector_v8s8_v16s8
 ; CHECK: [[ARG1:%[0-9]+]]:_(<8 x s8>) = COPY $d0
 ; CHECK: [[ARG2:%[0-9]+]]:_(<8 x s8>) = COPY $d1
-; CHECK: [[C0:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-; CHECK: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-; CHECK: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 9
-; CHECK: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-; CHECK: [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
-; CHECK: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
-; CHECK: [[C11:%[0-9]+]]:_(s32) = G_CONSTANT i32 11
-; CHECK: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-; CHECK: [[C12:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
-; CHECK: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 5
-; CHECK: [[C13:%[0-9]+]]:_(s32) = G_CONSTANT i32 13
-; CHECK: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 6
-; CHECK: [[C14:%[0-9]+]]:_(s32) = G_CONSTANT i32 14
-; CHECK: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 7
-; CHECK: [[C15:%[0-9]+]]:_(s32) = G_CONSTANT i32 15
-; CHECK: [[MASK:%[0-9]+]]:_(<16 x s32>) = G_BUILD_VECTOR [[C0]](s32), [[C8]](s32), [[C1]](s32), [[C9]](s32), [[C2]](s32), [[C10]](s32), [[C3]](s32), [[C11]](s32), [[C4]](s32), [[C12]](s32), [[C5]](s32), [[C13]](s32), [[C6]](s32), [[C14]](s32), [[C7]](s32), [[C15]](s32)
-; CHECK: [[VEC:%[0-9]+]]:_(<16 x s8>) = G_SHUFFLE_VECTOR [[ARG1]](<8 x s8>), [[ARG2]], [[MASK]](<16 x s32>)
+; CHECK: [[VEC:%[0-9]+]]:_(<16 x s8>) = G_SHUFFLE_VECTOR [[ARG1]](<8 x s8>), [[ARG2]], shufflemask(0, 8, 1, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15)
 ; CHECK: $q0 = COPY [[VEC]](<16 x s8>)
   %res = shufflevector <8 x i8> %arg1, <8 x i8> %arg2, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
   ret <16 x i8> %res
@@ -1806,19 +1799,19 @@ define void @test_phi_diamond({ i8, i16, i32 }* %a.ptr, { i8, i16, i32 }* %b.ptr
 
 ; CHECK: [[LD1:%[0-9]+]]:_(s8) = G_LOAD [[ARG1]](p0) :: (load 1 from %ir.a.ptr, align 4)
 ; CHECK: [[CST1:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-; CHECK: [[GEP1:%[0-9]+]]:_(p0) = G_GEP [[ARG1]], [[CST1]](s64)
+; CHECK: [[GEP1:%[0-9]+]]:_(p0) = G_PTR_ADD [[ARG1]], [[CST1]](s64)
 ; CHECK: [[LD2:%[0-9]+]]:_(s16) = G_LOAD [[GEP1]](p0) :: (load 2 from %ir.a.ptr + 2)
 ; CHECK: [[CST2:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-; CHECK: [[GEP2:%[0-9]+]]:_(p0) = G_GEP [[ARG1]], [[CST2]](s64)
+; CHECK: [[GEP2:%[0-9]+]]:_(p0) = G_PTR_ADD [[ARG1]], [[CST2]](s64)
 ; CHECK: [[LD3:%[0-9]+]]:_(s32) = G_LOAD [[GEP2]](p0) :: (load 4 from %ir.a.ptr + 4)
 ; CHECK: G_BR %bb.4
 
 ; CHECK: [[LD4:%[0-9]+]]:_(s8) = G_LOAD [[ARG2]](p0) :: (load 1 from %ir.b.ptr, align 4)
 ; CHECK: [[CST3:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-; CHECK: [[GEP3:%[0-9]+]]:_(p0) = G_GEP [[ARG2]], [[CST3]](s64)
+; CHECK: [[GEP3:%[0-9]+]]:_(p0) = G_PTR_ADD [[ARG2]], [[CST3]](s64)
 ; CHECK: [[LD5:%[0-9]+]]:_(s16) = G_LOAD [[GEP3]](p0) :: (load 2 from %ir.b.ptr + 2)
 ; CHECK: [[CST4:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-; CHECK: [[GEP4:%[0-9]+]]:_(p0) = G_GEP [[ARG2]], [[CST4]](s64)
+; CHECK: [[GEP4:%[0-9]+]]:_(p0) = G_PTR_ADD [[ARG2]], [[CST4]](s64)
 ; CHECK: [[LD6:%[0-9]+]]:_(s32) = G_LOAD [[GEP4]](p0) :: (load 4 from %ir.b.ptr + 4)
 
 ; CHECK: [[PN1:%[0-9]+]]:_(s8) = G_PHI [[LD1]](s8), %bb.2, [[LD4]](s8), %bb.3
@@ -1826,10 +1819,10 @@ define void @test_phi_diamond({ i8, i16, i32 }* %a.ptr, { i8, i16, i32 }* %b.ptr
 ; CHECK: [[PN3:%[0-9]+]]:_(s32) = G_PHI [[LD3]](s32), %bb.2, [[LD6]](s32), %bb.3
 ; CHECK: G_STORE [[PN1]](s8), [[ARG4]](p0) :: (store 1 into %ir.dst, align 4)
 ; CHECK: [[CST5:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-; CHECK: [[GEP5:%[0-9]+]]:_(p0) = G_GEP [[ARG4]], [[CST5]](s64)
+; CHECK: [[GEP5:%[0-9]+]]:_(p0) = G_PTR_ADD [[ARG4]], [[CST5]](s64)
 ; CHECK: G_STORE [[PN2]](s16), [[GEP5]](p0) :: (store 2 into %ir.dst + 2)
 ; CHECK: [[CST6:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-; CHECK: [[GEP6:%[0-9]+]]:_(p0) = G_GEP [[ARG4]], [[CST6]](s64)
+; CHECK: [[GEP6:%[0-9]+]]:_(p0) = G_PTR_ADD [[ARG4]], [[CST6]](s64)
 ; CHECK: G_STORE [[PN3]](s32), [[GEP6]](p0) :: (store 4 into %ir.dst + 4)
 ; CHECK: RET_ReallyLR
 
@@ -1865,22 +1858,22 @@ define void @test_nested_aggregate_const(%agg.nested *%ptr) {
 ; CHECK: [[CST6:%[0-9]+]]:_(s32) = G_CONSTANT i32 13
 ; CHECK: G_STORE [[CST1]](s32), [[BASE]](p0) :: (store 4 into %ir.ptr, align 8)
 ; CHECK: [[CST7:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-; CHECK: [[GEP1:%[0-9]+]]:_(p0) = G_GEP [[BASE]], [[CST7]](s64)
+; CHECK: [[GEP1:%[0-9]+]]:_(p0) = G_PTR_ADD [[BASE]], [[CST7]](s64)
 ; CHECK: G_STORE [[CST1]](s32), [[GEP1]](p0) :: (store 4 into %ir.ptr + 4)
 ; CHECK: [[CST8:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-; CHECK: [[GEP2:%[0-9]+]]:_(p0) = G_GEP [[BASE]], [[CST8]](s64)
+; CHECK: [[GEP2:%[0-9]+]]:_(p0) = G_PTR_ADD [[BASE]], [[CST8]](s64)
 ; CHECK: G_STORE [[CST2]](s16), [[GEP2]](p0) :: (store 2 into %ir.ptr + 8, align 8)
 ; CHECK: [[CST9:%[0-9]+]]:_(s64) = G_CONSTANT i64 10
-; CHECK: [[GEP3:%[0-9]+]]:_(p0) = G_GEP [[BASE]], [[CST9]](s64)
+; CHECK: [[GEP3:%[0-9]+]]:_(p0) = G_PTR_ADD [[BASE]], [[CST9]](s64)
 ; CHECK: G_STORE [[CST3]](s8), [[GEP3]](p0) :: (store 1 into %ir.ptr + 10, align 2)
 ; CHECK: [[CST10:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
-; CHECK: [[GEP4:%[0-9]+]]:_(p0) = G_GEP [[BASE]], [[CST10]](s64)
+; CHECK: [[GEP4:%[0-9]+]]:_(p0) = G_PTR_ADD [[BASE]], [[CST10]](s64)
 ; CHECK: G_STORE [[CST4]](s64), [[GEP4]](p0) :: (store 8 into %ir.ptr + 16)
 ; CHECK: [[CST11:%[0-9]+]]:_(s64) = G_CONSTANT i64 24
-; CHECK: [[GEP5:%[0-9]+]]:_(p0) = G_GEP [[BASE]], [[CST11]](s64)
+; CHECK: [[GEP5:%[0-9]+]]:_(p0) = G_PTR_ADD [[BASE]], [[CST11]](s64)
 ; CHECK: G_STORE [[CST5]](s64), [[GEP5]](p0) :: (store 8 into %ir.ptr + 24)
 ; CHECK: [[CST12:%[0-9]+]]:_(s64) = G_CONSTANT i64 32
-; CHECK: [[GEP6:%[0-9]+]]:_(p0) = G_GEP [[BASE]], [[CST12]](s64)
+; CHECK: [[GEP6:%[0-9]+]]:_(p0) = G_PTR_ADD [[BASE]], [[CST12]](s64)
 ; CHECK: G_STORE [[CST6]](s32), [[GEP6]](p0) :: (store 4 into %ir.ptr + 32, align 8)
   store %agg.nested { i32 1, i32 1, %agg.inner { i16 2, i8 3, %agg.inner.inner {i64 5, i64 8} }, i32 13}, %agg.nested *%ptr
   ret void
@@ -2333,3 +2326,15 @@ define void @test_var_annotation(i8*, i8*, i8*, i32) {
   call void @llvm.var.annotation(i8* %0, i8* %1, i8* %2, i32 %3)
   ret void
 }
+
+declare i64 @llvm.readcyclecounter()
+define i64 @test_readcyclecounter() {
+  ; CHECK-LABEL: name:            test_readcyclecounter
+  ; CHECK: [[RES:%[0-9]+]]:_(s64) = G_READCYCLECOUNTER{{$}}
+  ; CHECK-NEXT: $x0 = COPY [[RES]]
+  ; CHECK-NEXT: RET_ReallyLR implicit $x0
+  %res = call i64 @llvm.readcyclecounter()
+  ret i64 %res
+}
+
+!0 = !{ i64 0, i64 2 }
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/call-lowering-i128-on-stack.ll b/llvm/test/CodeGen/AArch64/GlobalISel/call-lowering-i128-on-stack.ll
new file mode 100644
index 000000000..b99f89bb6
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/call-lowering-i128-on-stack.ll
@@ -0,0 +1,12 @@
+; RUN: llc -O0 -global-isel -verify-machineinstrs %s -o - 2>&1 | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64-linux-gnu"
+
+; Check we don't assert when handling an i128 split arg on the stack.
+; CHECK-LABEL: fn1
+; CHECK: ret
+define i32 @fn1(i32 %p1, i128 %p2.coerce, i32 %p3, i32 %p4, i32 %p5, i128 %p6.coerce, i32 %p7) {
+entry:
+  ret i32 undef
+}
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/call-translator-cse.ll b/llvm/test/CodeGen/AArch64/GlobalISel/call-translator-cse.ll
index 74169cfa4..1223c544a 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/call-translator-cse.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/call-translator-cse.ll
@@ -4,16 +4,16 @@
 ; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0
 ; CHECK: [[LO:%[0-9]+]]:_(s64) = G_LOAD %0(p0) :: (load 8 from %ir.ptr)
 ; CHECK: [[CST:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-; CHECK: [[GEP:%[0-9]+]]:_(p0) = G_GEP [[ADDR]], [[CST]](s64)
+; CHECK: [[GEP:%[0-9]+]]:_(p0) = G_PTR_ADD [[ADDR]], [[CST]](s64)
 ; CHECK: [[HI:%[0-9]+]]:_(s64) = G_LOAD [[GEP]](p0) :: (load 8 from %ir.ptr + 8)
 
 ; CHECK: [[SP:%[0-9]+]]:_(p0) = COPY $sp
 ; CHECK: [[CST2:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-; CHECK: [[GEP2:%[0-9]+]]:_(p0) = G_GEP [[SP]], [[CST2]](s64)
+; CHECK: [[GEP2:%[0-9]+]]:_(p0) = G_PTR_ADD [[SP]], [[CST2]](s64)
 ; CHECK: G_STORE [[LO]](s64), [[GEP2]](p0) :: (store 8 into stack, align 1)
 ; CHECK: [[SP:%[0-9]+]]:_(p0) = COPY $sp
 ; CHECK: [[CST3:%[0-9]+]]:_(s64) = COPY [[CST]]
-; CHECK: [[GEP3:%[0-9]+]]:_(p0) = G_GEP [[SP]], [[CST3]](s64)
+; CHECK: [[GEP3:%[0-9]+]]:_(p0) = G_PTR_ADD [[SP]], [[CST3]](s64)
 ; CHECK: G_STORE [[HI]](s64), [[GEP3]](p0) :: (store 8 into stack + 8, align 1)
 define void @test_split_struct([2 x i64]* %ptr) {
   %struct = load [2 x i64], [2 x i64]* %ptr
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/call-translator-ios.ll b/llvm/test/CodeGen/AArch64/GlobalISel/call-translator-ios.ll
index 754bfdb95..fa67f49f9 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/call-translator-ios.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/call-translator-ios.ll
@@ -22,11 +22,11 @@ define signext i8 @test_stack_slots([8 x i64], i8 signext %lhs, i8 signext %rhs)
 ; CHECK: [[C12:%[0-9]+]]:_(s8) = G_CONSTANT i8 12
 ; CHECK: [[SP:%[0-9]+]]:_(p0) = COPY $sp
 ; CHECK: [[C42_OFFS:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-; CHECK: [[C42_LOC:%[0-9]+]]:_(p0) = G_GEP [[SP]], [[C42_OFFS]](s64)
+; CHECK: [[C42_LOC:%[0-9]+]]:_(p0) = G_PTR_ADD [[SP]], [[C42_OFFS]](s64)
 ; CHECK: G_STORE [[C42]](s8), [[C42_LOC]](p0) :: (store 1 into stack)
 ; CHECK: [[SP:%[0-9]+]]:_(p0) = COPY $sp
 ; CHECK: [[C12_OFFS:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-; CHECK: [[C12_LOC:%[0-9]+]]:_(p0) = G_GEP [[SP]], [[C12_OFFS]](s64)
+; CHECK: [[C12_LOC:%[0-9]+]]:_(p0) = G_PTR_ADD [[SP]], [[C12_OFFS]](s64)
 ; CHECK: G_STORE [[C12]](s8), [[C12_LOC]](p0) :: (store 1 into stack + 1)
 ; CHECK: BL @test_stack_slots
 define void @test_call_stack() {
@@ -57,17 +57,17 @@ define void @take_128bit_struct([2 x i64]* %ptr, [2 x i64] %in) {
 ; CHECK-LABEL: name: test_split_struct
 ; CHECK: [[LD1:%[0-9]+]]:_(s64) = G_LOAD %0(p0) :: (load 8 from %ir.ptr)
 ; CHECK: [[CST:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-; CHECK: [[GEP:%[0-9]+]]:_(p0) = G_GEP %0, [[CST]](s64)
+; CHECK: [[GEP:%[0-9]+]]:_(p0) = G_PTR_ADD %0, [[CST]](s64)
 ; CHECK: [[LD2:%[0-9]+]]:_(s64) = G_LOAD %3(p0) :: (load 8 from %ir.ptr + 8)
 
 ; CHECK: [[SP:%[0-9]+]]:_(p0) = COPY $sp
 ; CHECK: [[OFF:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-; CHECK: [[ADDR:%[0-9]+]]:_(p0) = G_GEP [[SP]], [[OFF]](s64)
+; CHECK: [[ADDR:%[0-9]+]]:_(p0) = G_PTR_ADD [[SP]], [[OFF]](s64)
 ; CHECK: G_STORE [[LD1]](s64), [[ADDR]](p0) :: (store 8 into stack, align 1)
 
 ; CHECK: [[SP:%[0-9]+]]:_(p0) = COPY $sp
 ; CHECK: [[OFF:%[0-9]+]]:_(s64) = COPY [[CST]]
-; CHECK: [[ADDR:%[0-9]+]]:_(p0) = G_GEP [[SP]], [[OFF]]
+; CHECK: [[ADDR:%[0-9]+]]:_(p0) = G_PTR_ADD [[SP]], [[OFF]]
 ; CHECK: G_STORE [[LD2]](s64), [[ADDR]](p0) :: (store 8 into stack + 8, align 1)
 define void @test_split_struct([2 x i64]* %ptr) {
   %struct = load [2 x i64], [2 x i64]* %ptr
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/call-translator-musttail.ll b/llvm/test/CodeGen/AArch64/GlobalISel/call-translator-musttail.ll
new file mode 100644
index 000000000..72af098a8
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/call-translator-musttail.ll
@@ -0,0 +1,13 @@
+; RUN: not llc %s -mtriple aarch64-apple-darwin -debug-only=aarch64-call-lowering -global-isel -global-isel-abort=2 -o - 2>&1 | FileCheck %s
+; REQUIRES: asserts
+
+; Verify that we fall back to SelectionDAG, and error out when we can't tail call musttail functions
+; CHECK: ... Cannot tail call externally-defined function with weak linkage for this OS.
+; CHECK-NEXT: Failed to lower musttail call as tail call
+; CHECK-NEXT: warning: Instruction selection used fallback path for caller_weak
+; CHECK-NEXT: LLVM ERROR: failed to perform tail call elimination on a call site marked musttail
+declare extern_weak void @callee_weak()
+define void @caller_weak() {
+  musttail call void @callee_weak()
+  ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/call-translator-tail-call-weak.ll b/llvm/test/CodeGen/AArch64/GlobalISel/call-translator-tail-call-weak.ll
new file mode 100644
index 000000000..bea5961ef
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/call-translator-tail-call-weak.ll
@@ -0,0 +1,15 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; RUN: llc %s -stop-after=irtranslator -verify-machineinstrs -mtriple aarch64-apple-darwin -global-isel -o - 2>&1 | FileCheck %s --check-prefixes=DARWIN,COMMON
+
+; Shouldn't tail call when the OS doesn't support it.
+declare extern_weak void @extern_weak_fn()
+define void @test_extern_weak() {
+  ; DARWIN-LABEL: name: test_extern_weak
+  ; DARWIN: bb.1 (%ir-block.0):
+  ; DARWIN:   ADJCALLSTACKDOWN 0, 0, implicit-def $sp, implicit $sp
+  ; DARWIN:   BL @extern_weak_fn, csr_aarch64_aapcs, implicit-def $lr, implicit $sp
+  ; DARWIN:   ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp
+  ; DARWIN:   RET_ReallyLR
+  tail call void @extern_weak_fn()
+  ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/call-translator-tail-call.ll b/llvm/test/CodeGen/AArch64/GlobalISel/call-translator-tail-call.ll
new file mode 100644
index 000000000..174c6ecb8
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/call-translator-tail-call.ll
@@ -0,0 +1,269 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; RUN: llc %s -stop-after=irtranslator -verify-machineinstrs -mtriple aarch64-apple-darwin -global-isel -o - 2>&1 | FileCheck %s --check-prefixes=DARWIN,COMMON
+; RUN: llc %s -stop-after=irtranslator -verify-machineinstrs -mtriple aarch64-windows -global-isel -o - 2>&1 | FileCheck %s --check-prefixes=WINDOWS,COMMON
+
+declare void @simple_fn()
+define void @tail_call() {
+  ; COMMON-LABEL: name: tail_call
+  ; COMMON: bb.1 (%ir-block.0):
+  ; COMMON:   TCRETURNdi @simple_fn, 0, csr_aarch64_aapcs, implicit $sp
+  tail call void @simple_fn()
+  ret void
+}
+
+; We should get a TCRETURNri here.
+; FIXME: We don't need the COPY.
+define void @indirect_tail_call(void()* %func) {
+  ; COMMON-LABEL: name: indirect_tail_call
+  ; COMMON: bb.1 (%ir-block.0):
+  ; COMMON:   liveins: $x0
+  ; COMMON:   [[COPY:%[0-9]+]]:tcgpr64(p0) = COPY $x0
+  ; COMMON:   TCRETURNri [[COPY]](p0), 0, csr_aarch64_aapcs, implicit $sp
+  tail call void %func()
+  ret void
+}
+
+declare void @outgoing_args_fn(i32)
+define void @test_outgoing_args(i32 %a) {
+  ; COMMON-LABEL: name: test_outgoing_args
+  ; COMMON: bb.1 (%ir-block.0):
+  ; COMMON:   liveins: $w0
+  ; COMMON:   [[COPY:%[0-9]+]]:_(s32) = COPY $w0
+  ; COMMON:   $w0 = COPY [[COPY]](s32)
+  ; COMMON:   TCRETURNdi @outgoing_args_fn, 0, csr_aarch64_aapcs, implicit $sp, implicit $w0
+  tail call void @outgoing_args_fn(i32 %a)
+  ret void
+}
+
+; Verify that we create frame indices for memory arguments in tail calls.
+; We get a bunch of copies here which are unused and thus eliminated. So, let's
+; just focus on what matters, which is that we get a G_FRAME_INDEX.
+declare void @outgoing_stack_args_fn(<4 x half>)
+define void @test_outgoing_stack_args([8 x <2 x double>], <4 x half> %arg) {
+  ; COMMON-LABEL: name: test_outgoing_stack_args
+  ; COMMON:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0
+  ; COMMON:   [[LOAD:%[0-9]+]]:_(<4 x s16>) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load 8 from %fixed-stack.0, align 1)
+  ; COMMON:   $d0 = COPY [[LOAD]](<4 x s16>)
+  ; COMMON:   TCRETURNdi @outgoing_stack_args_fn, 0, csr_aarch64_aapcs, implicit $sp, implicit $d0
+  tail call void @outgoing_stack_args_fn(<4 x half> %arg)
+  ret void
+}
+
+; Verify that we don't tail call when we cannot fit arguments on the caller's
+; stack.
+declare i32 @too_big_stack(i64 %x0, i64 %x1, i64 %x2, i64 %x3, i64 %x4, i64 %x5, i64 %x6, i64 %x7, i8 %c, i16 %s)
+define i32 @test_too_big_stack() {
+  ; COMMON-LABEL: name: test_too_big_stack
+  ; COMMON-NOT: TCRETURNdi
+  ; COMMON-NOT: TCRETURNri
+  ; COMMON: BL @too_big_stack
+  ; COMMON-DAG: RET_ReallyLR
+entry:
+  %call = tail call i32 @too_big_stack(i64 undef, i64 undef, i64 undef, i64 undef, i64 undef, i64 undef, i64 undef, i64 undef, i8 8, i16 9)
+  ret i32 %call
+}
+
+; Right now, we don't want to tail call callees with nonvoid return types, since
+; call lowering will insert COPYs after the call.
+; TODO: Support this.
+declare i32 @nonvoid_ret()
+define i32 @test_nonvoid_ret() {
+  ; COMMON-LABEL: name: test_nonvoid_ret
+  ; COMMON: bb.1 (%ir-block.0):
+  ; COMMON:   TCRETURNdi @nonvoid_ret, 0, csr_aarch64_aapcs, implicit $sp
+  %call = tail call i32 @nonvoid_ret()
+  ret i32 %call
+}
+
+declare void @varargs(i32, double, i64, ...)
+define void @test_varargs() {
+  ; COMMON-LABEL: name: test_varargs
+  ; COMMON: bb.1 (%ir-block.0):
+  ; COMMON:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 42
+  ; COMMON:   [[C1:%[0-9]+]]:_(s64) = G_FCONSTANT double 1.000000e+00
+  ; COMMON:   [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 12
+  ; COMMON:   $w0 = COPY [[C]](s32)
+  ; COMMON:   $d0 = COPY [[C1]](s64)
+  ; COMMON:   $x1 = COPY [[C2]](s64)
+  ; COMMON:   TCRETURNdi @varargs, 0, csr_aarch64_aapcs, implicit $sp, implicit $w0, implicit $d0, implicit $x1
+  tail call void(i32, double, i64, ...) @varargs(i32 42, double 1.0, i64 12)
+  ret void
+}
+
+; Darwin should not tail call here, because the last parameter to @varargs is
+; not fixed. So, it's passed on the stack, which will make us not fit. On
+; Windows, it's passed in a register, so it's safe to tail call.
+define void @test_varargs_2() {
+  ; DARWIN-LABEL: name: test_varargs_2
+  ; DARWIN-NOT: TCRETURNdi @varargs
+  ; DARWIN:   BL @varargs, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $w0, implicit $d0, implicit $x1
+  ; DARWIN:   ADJCALLSTACKUP 8, 0, implicit-def $sp, implicit $sp
+  ; DARWIN:   RET_ReallyLR
+
+  ; WINDOWS-LABEL: name: test_varargs_2
+  ; WINDOWS: bb.1 (%ir-block.0):
+  ; WINDOWS:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 42
+  ; WINDOWS:   [[C1:%[0-9]+]]:_(s64) = G_FCONSTANT double 1.000000e+00
+  ; WINDOWS:   [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 12
+  ; WINDOWS:   [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 314
+  ; WINDOWS:   $w0 = COPY [[C]](s32)
+  ; WINDOWS:   $d0 = COPY [[C1]](s64)
+  ; WINDOWS:   $x1 = COPY [[C2]](s64)
+  ; WINDOWS:   $x2 = COPY [[C3]](s64)
+  ; WINDOWS:   TCRETURNdi @varargs, 0, csr_aarch64_aapcs, implicit $sp, implicit $w0, implicit $d0, implicit $x1, implicit $x2
+  tail call void(i32, double, i64, ...) @varargs(i32 42, double 1.0, i64 12, i64 314)
+  ret void
+}
+
+; Same deal here, even though we have enough room to fit. On Darwin, we'll pass
+; the last argument to @varargs on the stack. We don't allow tail calling
+; varargs arguments that are on the stack.
+define void @test_varargs_3([8 x <2 x double>], <4 x half> %arg) {
+  ; DARWIN-LABEL: name: test_varargs_3
+  ; DARWIN-NOT: TCRETURNdi @varargs
+  ; DARWIN:   BL @varargs, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $w0, implicit $d0, implicit $x1
+  ; DARWIN:   ADJCALLSTACKUP 8, 0, implicit-def $sp, implicit $sp
+  ; DARWIN:   RET_ReallyLR
+
+  ; WINDOWS-LABEL: name: test_varargs_3
+  ; WINDOWS: bb.1 (%ir-block.1):
+  ; WINDOWS:   liveins: $q0, $q1, $q2, $q3, $q4, $q5, $q6, $q7
+  ; WINDOWS:   [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $q0
+  ; WINDOWS:   [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $q1
+  ; WINDOWS:   [[COPY2:%[0-9]+]]:_(<2 x s64>) = COPY $q2
+  ; WINDOWS:   [[COPY3:%[0-9]+]]:_(<2 x s64>) = COPY $q3
+  ; WINDOWS:   [[COPY4:%[0-9]+]]:_(<2 x s64>) = COPY $q4
+  ; WINDOWS:   [[COPY5:%[0-9]+]]:_(<2 x s64>) = COPY $q5
+  ; WINDOWS:   [[COPY6:%[0-9]+]]:_(<2 x s64>) = COPY $q6
+  ; WINDOWS:   [[COPY7:%[0-9]+]]:_(<2 x s64>) = COPY $q7
+  ; WINDOWS:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0
+  ; WINDOWS:   [[LOAD:%[0-9]+]]:_(<4 x s16>) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load 8 from %fixed-stack.0, align 1)
+  ; WINDOWS:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 42
+  ; WINDOWS:   [[C1:%[0-9]+]]:_(s64) = G_FCONSTANT double 1.000000e+00
+  ; WINDOWS:   [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 12
+  ; WINDOWS:   [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 314
+  ; WINDOWS:   $w0 = COPY [[C]](s32)
+  ; WINDOWS:   $d0 = COPY [[C1]](s64)
+  ; WINDOWS:   $x1 = COPY [[C2]](s64)
+  ; WINDOWS:   $x2 = COPY [[C3]](s64)
+  ; WINDOWS:   TCRETURNdi @varargs, 0, csr_aarch64_aapcs, implicit $sp, implicit $w0, implicit $d0, implicit $x1, implicit $x2
+  tail call void(i32, double, i64, ...) @varargs(i32 42, double 1.0, i64 12, i64 314)
+  ret void
+}
+
+; Unsupported calling convention for tail calls. Make sure we never tail call
+; it.
+declare ghccc void @bad_call_conv_fn()
+define void @test_bad_call_conv() {
+  ; COMMON-LABEL: name: test_bad_call_conv
+  ; COMMON: bb.1 (%ir-block.0):
+  ; COMMON:   ADJCALLSTACKDOWN 0, 0, implicit-def $sp, implicit $sp
+  ; COMMON:   BL @bad_call_conv_fn, csr_aarch64_noregs, implicit-def $lr, implicit $sp
+  ; COMMON:   ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp
+  ; COMMON:   RET_ReallyLR
+  tail call ghccc void @bad_call_conv_fn()
+  ret void
+}
+
+; Shouldn't tail call when the caller has byval arguments.
+define void @test_byval(i8* byval %ptr) {
+  ; COMMON-LABEL: name: test_byval
+  ; COMMON: bb.1 (%ir-block.0):
+  ; COMMON:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0
+  ; COMMON:   [[LOAD:%[0-9]+]]:_(p0) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load 8 from %fixed-stack.0, align 1)
+  ; COMMON:   ADJCALLSTACKDOWN 0, 0, implicit-def $sp, implicit $sp
+  ; COMMON:   BL @simple_fn, csr_aarch64_aapcs, implicit-def $lr, implicit $sp
+  ; COMMON:   ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp
+  ; COMMON:   RET_ReallyLR
+  tail call void @simple_fn()
+  ret void
+}
+
+; Shouldn't tail call when the caller has inreg arguments.
+define void @test_inreg(i8* inreg %ptr) {
+  ; COMMON-LABEL: name: test_inreg
+  ; COMMON: bb.1 (%ir-block.0):
+  ; COMMON:   liveins: $x0
+  ; COMMON:   [[COPY:%[0-9]+]]:_(p0) = COPY $x0
+  ; COMMON:   ADJCALLSTACKDOWN 0, 0, implicit-def $sp, implicit $sp
+  ; COMMON:   BL @simple_fn, csr_aarch64_aapcs, implicit-def $lr, implicit $sp
+  ; COMMON:   ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp
+  ; COMMON:   RET_ReallyLR
+  tail call void @simple_fn()
+  ret void
+}
+
+declare fastcc void @fast_fn()
+define void @test_mismatched_caller() {
+  ; COMMON-LABEL: name: test_mismatched_caller
+  ; COMMON: bb.1 (%ir-block.0):
+  ; COMMON:   TCRETURNdi @fast_fn, 0, csr_aarch64_aapcs, implicit $sp
+  tail call fastcc void @fast_fn()
+  ret void
+}
+
+; Verify that lifetime markers and llvm.assume don't impact tail calling.
+declare void @llvm.assume(i1)
+define void @test_assume() local_unnamed_addr {
+  ; COMMON-LABEL: name: test_assume
+  ; COMMON: bb.1.entry:
+  ; COMMON:   TCRETURNdi @nonvoid_ret, 0, csr_aarch64_aapcs, implicit $sp
+entry:
+  %x = tail call i32 @nonvoid_ret()
+  %y = icmp ne i32 %x, 0
+  tail call void @llvm.assume(i1 %y)
+  ret void
+}
+
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture)
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture)
+define void @test_lifetime() local_unnamed_addr {
+  ; COMMON-LABEL: name: test_lifetime
+  ; COMMON: bb.1.entry:
+  ; COMMON:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.t
+  ; COMMON:   LIFETIME_START %stack.0.t
+  ; COMMON:   TCRETURNdi @nonvoid_ret, 0, csr_aarch64_aapcs, implicit $sp
+entry:
+  %t = alloca i8, align 1
+  call void @llvm.lifetime.start.p0i8(i64 1, i8* %t)
+  %x = tail call i32 @nonvoid_ret()
+  %y = icmp ne i32 %x, 0
+  tail call void @llvm.lifetime.end.p0i8(i64 1, i8* %t)
+  ret void
+}
+
+; We can tail call when the callee swiftself is the same as the caller one.
+; It would be nice to move this to swiftself.ll, but it's important to verify
+; that we get the COPY that makes this safe in the first place.
+declare i8* @pluto()
+define hidden swiftcc i64 @swiftself_indirect_tail(i64* swiftself %arg) {
+  ; COMMON-LABEL: name: swiftself_indirect_tail
+  ; COMMON: bb.1 (%ir-block.0):
+  ; COMMON:   liveins: $x20
+  ; COMMON:   [[COPY:%[0-9]+]]:_(p0) = COPY $x20
+  ; COMMON:   ADJCALLSTACKDOWN 0, 0, implicit-def $sp, implicit $sp
+  ; COMMON:   BL @pluto, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit-def $x0
+  ; COMMON:   [[COPY1:%[0-9]+]]:tcgpr64(p0) = COPY $x0
+  ; COMMON:   ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp
+  ; COMMON:   $x20 = COPY [[COPY]](p0)
+  ; COMMON:   TCRETURNri [[COPY1]](p0), 0, csr_aarch64_aapcs, implicit $sp, implicit $x20
+  %tmp = call i8* @pluto()
+  %tmp1 = bitcast i8* %tmp to i64 (i64*)*
+  %tmp2 = tail call swiftcc i64 %tmp1(i64* swiftself %arg)
+  ret i64 %tmp2
+}
+
+; Verify that we can tail call musttail callees.
+declare void @must_callee(i8*)
+define void @foo(i32*) {
+  ; COMMON-LABEL: name: foo
+  ; COMMON: bb.1 (%ir-block.1):
+  ; COMMON:   liveins: $x0
+  ; COMMON:   [[COPY:%[0-9]+]]:_(p0) = COPY $x0
+  ; COMMON:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; COMMON:   [[INTTOPTR:%[0-9]+]]:_(p0) = G_INTTOPTR [[C]](s64)
+  ; COMMON:   $x0 = COPY [[INTTOPTR]](p0)
+  ; COMMON:   TCRETURNdi @must_callee, 0, csr_aarch64_aapcs, implicit $sp, implicit $x0
+  musttail call void @must_callee(i8* null)
+  ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/call-translator-variadic-musttail.ll b/llvm/test/CodeGen/AArch64/GlobalISel/call-translator-variadic-musttail.ll
new file mode 100644
index 000000000..0a0f94aa0
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/call-translator-variadic-musttail.ll
@@ -0,0 +1,223 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc %s -verify-machineinstrs -mtriple aarch64-apple-darwin -global-isel -o - 2>&1 | FileCheck %s
+
+; There are two things we want to test here:
+;  (1) We can tail call musttail calls.
+;  (2) We spill and reload all of the arguments around a normal call.
+
+declare i32 @musttail_variadic_callee(i32, ...)
+define i32 @test_musttail_variadic(i32 %arg0, ...) {
+; CHECK-LABEL: test_musttail_variadic:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    b _musttail_variadic_callee
+  %r = musttail call i32 (i32, ...) @musttail_variadic_callee(i32 %arg0, ...)
+  ret i32 %r
+}
+
+declare [2 x i64] @musttail_variadic_aggret_callee(i32 %arg0, ...)
+define [2 x i64] @test_musttail_variadic_aggret(i32 %arg0, ...) {
+; CHECK-LABEL: test_musttail_variadic_aggret:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    b _musttail_variadic_aggret_callee
+  %r = musttail call [2 x i64] (i32, ...) @musttail_variadic_aggret_callee(i32 %arg0, ...)
+  ret [2 x i64] %r
+}
+
+; Test musttailing with a normal call in the block. Test that we spill and
+; restore, as a normal call will clobber all argument registers.
+@asdf = internal constant [4 x i8] c"asdf"
+declare void @puts(i8*)
+define i32 @test_musttail_variadic_spill(i32 %arg0, ...) {
+; CHECK-LABEL: test_musttail_variadic_spill:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    sub sp, sp, #224 ; =224
+; CHECK-NEXT:    stp x28, x27, [sp, #128] ; 16-byte Folded Spill
+; CHECK-NEXT:    stp x26, x25, [sp, #144] ; 16-byte Folded Spill
+; CHECK-NEXT:    stp x24, x23, [sp, #160] ; 16-byte Folded Spill
+; CHECK-NEXT:    stp x22, x21, [sp, #176] ; 16-byte Folded Spill
+; CHECK-NEXT:    stp x20, x19, [sp, #192] ; 16-byte Folded Spill
+; CHECK-NEXT:    stp x29, x30, [sp, #208] ; 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 224
+; CHECK-NEXT:    .cfi_offset w30, -8
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    .cfi_offset w19, -24
+; CHECK-NEXT:    .cfi_offset w20, -32
+; CHECK-NEXT:    .cfi_offset w21, -40
+; CHECK-NEXT:    .cfi_offset w22, -48
+; CHECK-NEXT:    .cfi_offset w23, -56
+; CHECK-NEXT:    .cfi_offset w24, -64
+; CHECK-NEXT:    .cfi_offset w25, -72
+; CHECK-NEXT:    .cfi_offset w26, -80
+; CHECK-NEXT:    .cfi_offset w27, -88
+; CHECK-NEXT:    .cfi_offset w28, -96
+; CHECK-NEXT:    mov w19, w0
+; CHECK-NEXT:  Lloh0:
+; CHECK-NEXT:    adrp x0, _asdf@PAGE
+; CHECK-NEXT:  Lloh1:
+; CHECK-NEXT:    add x0, x0, _asdf@PAGEOFF
+; CHECK-NEXT:    mov x20, x1
+; CHECK-NEXT:    mov x21, x2
+; CHECK-NEXT:    mov x22, x3
+; CHECK-NEXT:    mov x23, x4
+; CHECK-NEXT:    mov x24, x5
+; CHECK-NEXT:    mov x25, x6
+; CHECK-NEXT:    mov x26, x7
+; CHECK-NEXT:    stp q1, q0, [sp, #96] ; 32-byte Folded Spill
+; CHECK-NEXT:    stp q3, q2, [sp, #64] ; 32-byte Folded Spill
+; CHECK-NEXT:    stp q5, q4, [sp, #32] ; 32-byte Folded Spill
+; CHECK-NEXT:    stp q7, q6, [sp] ; 32-byte Folded Spill
+; CHECK-NEXT:    mov x27, x8
+; CHECK-NEXT:    bl _puts
+; CHECK-NEXT:    ldp q1, q0, [sp, #96] ; 32-byte Folded Reload
+; CHECK-NEXT:    ldp q3, q2, [sp, #64] ; 32-byte Folded Reload
+; CHECK-NEXT:    ldp q5, q4, [sp, #32] ; 32-byte Folded Reload
+; CHECK-NEXT:    ldp q7, q6, [sp] ; 32-byte Folded Reload
+; CHECK-NEXT:    mov w0, w19
+; CHECK-NEXT:    mov x1, x20
+; CHECK-NEXT:    mov x2, x21
+; CHECK-NEXT:    mov x3, x22
+; CHECK-NEXT:    mov x4, x23
+; CHECK-NEXT:    mov x5, x24
+; CHECK-NEXT:    mov x6, x25
+; CHECK-NEXT:    mov x7, x26
+; CHECK-NEXT:    mov x8, x27
+; CHECK-NEXT:    ldp x29, x30, [sp, #208] ; 16-byte Folded Reload
+; CHECK-NEXT:    ldp x20, x19, [sp, #192] ; 16-byte Folded Reload
+; CHECK-NEXT:    ldp x22, x21, [sp, #176] ; 16-byte Folded Reload
+; CHECK-NEXT:    ldp x24, x23, [sp, #160] ; 16-byte Folded Reload
+; CHECK-NEXT:    ldp x26, x25, [sp, #144] ; 16-byte Folded Reload
+; CHECK-NEXT:    ldp x28, x27, [sp, #128] ; 16-byte Folded Reload
+; CHECK-NEXT:    add sp, sp, #224 ; =224
+; CHECK-NEXT:    b _musttail_variadic_callee
+; CHECK-NEXT:    .loh AdrpAdd Lloh0, Lloh1
+  call void @puts(i8* getelementptr ([4 x i8], [4 x i8]* @asdf, i32 0, i32 0))
+  %r = musttail call i32 (i32, ...) @musttail_variadic_callee(i32 %arg0, ...)
+  ret i32 %r
+}
+
+; Test musttailing with a varargs call in the block. Test that we spill and
+; reload all arguments in the variadic argument pack.
+declare void @llvm.va_start(i8*) nounwind
+declare void(i8*, ...)* @get_f(i8* %this)
+define void @f_thunk(i8* %this, ...) {
+; CHECK-LABEL: f_thunk:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    sub sp, sp, #256 ; =256
+; CHECK-NEXT:    stp x28, x27, [sp, #160] ; 16-byte Folded Spill
+; CHECK-NEXT:    stp x26, x25, [sp, #176] ; 16-byte Folded Spill
+; CHECK-NEXT:    stp x24, x23, [sp, #192] ; 16-byte Folded Spill
+; CHECK-NEXT:    stp x22, x21, [sp, #208] ; 16-byte Folded Spill
+; CHECK-NEXT:    stp x20, x19, [sp, #224] ; 16-byte Folded Spill
+; CHECK-NEXT:    stp x29, x30, [sp, #240] ; 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 256
+; CHECK-NEXT:    .cfi_offset w30, -8
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    .cfi_offset w19, -24
+; CHECK-NEXT:    .cfi_offset w20, -32
+; CHECK-NEXT:    .cfi_offset w21, -40
+; CHECK-NEXT:    .cfi_offset w22, -48
+; CHECK-NEXT:    .cfi_offset w23, -56
+; CHECK-NEXT:    .cfi_offset w24, -64
+; CHECK-NEXT:    .cfi_offset w25, -72
+; CHECK-NEXT:    .cfi_offset w26, -80
+; CHECK-NEXT:    .cfi_offset w27, -88
+; CHECK-NEXT:    .cfi_offset w28, -96
+; CHECK-NEXT:    mov x27, x8
+; CHECK-NEXT:    add x8, sp, #128 ; =128
+; CHECK-NEXT:    add x9, sp, #256 ; =256
+; CHECK-NEXT:    mov x19, x0
+; CHECK-NEXT:    mov x20, x1
+; CHECK-NEXT:    mov x21, x2
+; CHECK-NEXT:    mov x22, x3
+; CHECK-NEXT:    mov x23, x4
+; CHECK-NEXT:    mov x24, x5
+; CHECK-NEXT:    mov x25, x6
+; CHECK-NEXT:    mov x26, x7
+; CHECK-NEXT:    stp q1, q0, [sp, #96] ; 32-byte Folded Spill
+; CHECK-NEXT:    stp q3, q2, [sp, #64] ; 32-byte Folded Spill
+; CHECK-NEXT:    stp q5, q4, [sp, #32] ; 32-byte Folded Spill
+; CHECK-NEXT:    stp q7, q6, [sp] ; 32-byte Folded Spill
+; CHECK-NEXT:    str x9, [x8]
+; CHECK-NEXT:    bl _get_f
+; CHECK-NEXT:    mov x9, x0
+; CHECK-NEXT:    ldp q1, q0, [sp, #96] ; 32-byte Folded Reload
+; CHECK-NEXT:    ldp q3, q2, [sp, #64] ; 32-byte Folded Reload
+; CHECK-NEXT:    ldp q5, q4, [sp, #32] ; 32-byte Folded Reload
+; CHECK-NEXT:    ldp q7, q6, [sp] ; 32-byte Folded Reload
+; CHECK-NEXT:    mov x0, x19
+; CHECK-NEXT:    mov x1, x20
+; CHECK-NEXT:    mov x2, x21
+; CHECK-NEXT:    mov x3, x22
+; CHECK-NEXT:    mov x4, x23
+; CHECK-NEXT:    mov x5, x24
+; CHECK-NEXT:    mov x6, x25
+; CHECK-NEXT:    mov x7, x26
+; CHECK-NEXT:    mov x8, x27
+; CHECK-NEXT:    ldp x29, x30, [sp, #240] ; 16-byte Folded Reload
+; CHECK-NEXT:    ldp x20, x19, [sp, #224] ; 16-byte Folded Reload
+; CHECK-NEXT:    ldp x22, x21, [sp, #208] ; 16-byte Folded Reload
+; CHECK-NEXT:    ldp x24, x23, [sp, #192] ; 16-byte Folded Reload
+; CHECK-NEXT:    ldp x26, x25, [sp, #176] ; 16-byte Folded Reload
+; CHECK-NEXT:    ldp x28, x27, [sp, #160] ; 16-byte Folded Reload
+; CHECK-NEXT:    add sp, sp, #256 ; =256
+; CHECK-NEXT:    br x9
+  %ap = alloca [4 x i8*], align 16
+  %ap_i8 = bitcast [4 x i8*]* %ap to i8*
+  call void @llvm.va_start(i8* %ap_i8)
+  %fptr = call void(i8*, ...)*(i8*) @get_f(i8* %this)
+  musttail call void (i8*, ...) %fptr(i8* %this, ...)
+  ret void
+}
+
+; We don't need any spills and reloads here, but we should still emit the
+; copies in call lowering.
+define void @g_thunk(i8* %fptr_i8, ...) {
+; CHECK-LABEL: g_thunk:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    br x0
+  %fptr = bitcast i8* %fptr_i8 to void (i8*, ...)*
+  musttail call void (i8*, ...) %fptr(i8* %fptr_i8, ...)
+  ret void
+}
+
+; Test that this works with multiple exits and basic blocks.
+%struct.Foo = type { i1, i8*, i8* }
+@g = external global i32
+define void @h_thunk(%struct.Foo* %this, ...) {
+; CHECK-LABEL: h_thunk:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    ldrb w9, [x0]
+; CHECK-NEXT:    tbz w9, #0, LBB5_2
+; CHECK-NEXT:  ; %bb.1: ; %then
+; CHECK-NEXT:    ldr x9, [x0, #8]
+; CHECK-NEXT:    br x9
+; CHECK-NEXT:  LBB5_2: ; %else
+; CHECK-NEXT:  Lloh2:
+; CHECK-NEXT:    adrp x10, _g@GOTPAGE
+; CHECK-NEXT:    ldr x9, [x0, #16]
+; CHECK-NEXT:  Lloh3:
+; CHECK-NEXT:    ldr x10, [x10, _g@GOTPAGEOFF]
+; CHECK-NEXT:    mov w11, #42
+; CHECK-NEXT:  Lloh4:
+; CHECK-NEXT:    str w11, [x10]
+; CHECK-NEXT:    br x9
+; CHECK-NEXT:    .loh AdrpLdrGotStr Lloh2, Lloh3, Lloh4
+  %cond_p = getelementptr %struct.Foo, %struct.Foo* %this, i32 0, i32 0
+  %cond = load i1, i1* %cond_p
+  br i1 %cond, label %then, label %else
+
+then:
+  %a_p = getelementptr %struct.Foo, %struct.Foo* %this, i32 0, i32 1
+  %a_i8 = load i8*, i8** %a_p
+  %a = bitcast i8* %a_i8 to void (%struct.Foo*, ...)*
+  musttail call void (%struct.Foo*, ...) %a(%struct.Foo* %this, ...)
+  ret void
+
+else:
+  %b_p = getelementptr %struct.Foo, %struct.Foo* %this, i32 0, i32 2
+  %b_i8 = load i8*, i8** %b_p
+  %b = bitcast i8* %b_i8 to void (%struct.Foo*, ...)*
+  store i32 42, i32* @g
+  musttail call void (%struct.Foo*, ...) %b(%struct.Foo* %this, ...)
+  ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/call-translator.ll b/llvm/test/CodeGen/AArch64/GlobalISel/call-translator.ll
index b5e66d69c..622fd5ca6 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/call-translator.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/call-translator.ll
@@ -67,10 +67,10 @@ define void @test_multiple_args(i64 %in) {
 
 ; CHECK: G_STORE [[DBL]](s64), [[ADDR]](p0) :: (store 8 into %ir.addr)
 ; CHECK: [[CST1:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-; CHECK: [[GEP1:%[0-9]+]]:_(p0) = G_GEP [[ADDR]], [[CST1]](s64)
+; CHECK: [[GEP1:%[0-9]+]]:_(p0) = G_PTR_ADD [[ADDR]], [[CST1]](s64)
 ; CHECK: G_STORE [[I64]](s64), [[GEP1]](p0) :: (store 8 into %ir.addr + 8)
 ; CHECK: [[CST2:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
-; CHECK: [[GEP2:%[0-9]+]]:_(p0) = G_GEP [[ADDR]], [[CST2]](s64)
+; CHECK: [[GEP2:%[0-9]+]]:_(p0) = G_PTR_ADD [[ADDR]], [[CST2]](s64)
 ; CHECK: G_STORE [[I8]](s8), [[GEP2]](p0) :: (store 1 into %ir.addr + 16, align 8)
 ; CHECK: RET_ReallyLR
 define void @test_struct_formal({double, i64, i8} %in, {double, i64, i8}* %addr) {
@@ -84,10 +84,10 @@ define void @test_struct_formal({double, i64, i8} %in, {double, i64, i8}* %addr)
 
 ; CHECK: [[LD1:%[0-9]+]]:_(s64) = G_LOAD [[ADDR]](p0) :: (load 8 from %ir.addr)
 ; CHECK: [[CST1:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-; CHECK: [[GEP1:%[0-9]+]]:_(p0) = G_GEP [[ADDR]], [[CST1]](s64)
+; CHECK: [[GEP1:%[0-9]+]]:_(p0) = G_PTR_ADD [[ADDR]], [[CST1]](s64)
 ; CHECK: [[LD2:%[0-9]+]]:_(s64) = G_LOAD [[GEP1]](p0) :: (load 8 from %ir.addr + 8)
 ; CHECK: [[CST2:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
-; CHECK: [[GEP2:%[0-9]+]]:_(p0) = G_GEP [[ADDR]], [[CST2]](s64)
+; CHECK: [[GEP2:%[0-9]+]]:_(p0) = G_PTR_ADD [[ADDR]], [[CST2]](s64)
 ; CHECK: [[LD3:%[0-9]+]]:_(s32) = G_LOAD [[GEP2]](p0) :: (load 4 from %ir.addr + 16, align 8)
 
 ; CHECK: $d0 = COPY [[LD1]](s64)
@@ -100,17 +100,16 @@ define {double, i64, i32} @test_struct_return({double, i64, i32}* %addr) {
 }
 
 ; CHECK-LABEL: name: test_arr_call
-; CHECK: hasCalls: true
 ; CHECK: %0:_(p0) = COPY $x0
 ; CHECK: [[LD1:%[0-9]+]]:_(s64) = G_LOAD %0(p0) :: (load 8 from %ir.addr)
 ; CHECK: [[CST1:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-; CHECK: [[GEP1:%[0-9]+]]:_(p0) = G_GEP %0, [[CST1]](s64)
+; CHECK: [[GEP1:%[0-9]+]]:_(p0) = G_PTR_ADD %0, [[CST1]](s64)
 ; CHECK: [[LD2:%[0-9]+]]:_(s64) = G_LOAD [[GEP1]](p0) :: (load 8 from %ir.addr + 8)
 ; CHECK: [[CST2:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
-; CHECK: [[GEP2:%[0-9]+]]:_(p0) = G_GEP %0, [[CST2]](s64)
+; CHECK: [[GEP2:%[0-9]+]]:_(p0) = G_PTR_ADD %0, [[CST2]](s64)
 ; CHECK: [[LD3:%[0-9]+]]:_(s64) = G_LOAD [[GEP2]](p0) :: (load 8 from %ir.addr + 16)
 ; CHECK: [[CST3:%[0-9]+]]:_(s64) = G_CONSTANT i64 24
-; CHECK: [[GEP3:%[0-9]+]]:_(p0) = G_GEP %0, [[CST3]](s64)
+; CHECK: [[GEP3:%[0-9]+]]:_(p0) = G_PTR_ADD %0, [[CST3]](s64)
 ; CHECK: [[LD4:%[0-9]+]]:_(s64) = G_LOAD [[GEP3]](p0) :: (load 8 from %ir.addr + 24)
 
 ; CHECK: $x0 = COPY [[LD1]](s64)
@@ -199,15 +198,15 @@ define void @test_stack_slots([8 x i64], i64 %lhs, i64 %rhs, i64* %addr) {
 ; CHECK: ADJCALLSTACKDOWN 24, 0, implicit-def $sp, implicit $sp
 ; CHECK: [[SP:%[0-9]+]]:_(p0) = COPY $sp
 ; CHECK: [[C42_OFFS:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-; CHECK: [[C42_LOC:%[0-9]+]]:_(p0) = G_GEP [[SP]], [[C42_OFFS]](s64)
+; CHECK: [[C42_LOC:%[0-9]+]]:_(p0) = G_PTR_ADD [[SP]], [[C42_OFFS]](s64)
 ; CHECK: G_STORE [[C42]](s64), [[C42_LOC]](p0) :: (store 8 into stack, align 1)
 ; CHECK: [[SP:%[0-9]+]]:_(p0) = COPY $sp
 ; CHECK: [[C12_OFFS:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-; CHECK: [[C12_LOC:%[0-9]+]]:_(p0) = G_GEP [[SP]], [[C12_OFFS]](s64)
+; CHECK: [[C12_LOC:%[0-9]+]]:_(p0) = G_PTR_ADD [[SP]], [[C12_OFFS]](s64)
 ; CHECK: G_STORE [[C12]](s64), [[C12_LOC]](p0) :: (store 8 into stack + 8, align 1)
 ; CHECK: [[SP:%[0-9]+]]:_(p0) = COPY $sp
 ; CHECK: [[PTR_OFFS:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
-; CHECK: [[PTR_LOC:%[0-9]+]]:_(p0) = G_GEP [[SP]], [[PTR_OFFS]](s64)
+; CHECK: [[PTR_LOC:%[0-9]+]]:_(p0) = G_PTR_ADD [[SP]], [[PTR_OFFS]](s64)
 ; CHECK: G_STORE [[PTR]](p0), [[PTR_LOC]](p0) :: (store 8 into stack + 16, align 1)
 ; CHECK: BL @test_stack_slots
 ; CHECK: ADJCALLSTACKUP 24, 0, implicit-def $sp, implicit $sp
@@ -250,16 +249,16 @@ define void @take_128bit_struct([2 x i64]* %ptr, [2 x i64] %in) {
 ; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0
 ; CHECK: [[LO:%[0-9]+]]:_(s64) = G_LOAD %0(p0) :: (load 8 from %ir.ptr)
 ; CHECK: [[CST:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-; CHECK: [[GEP:%[0-9]+]]:_(p0) = G_GEP [[ADDR]], [[CST]](s64)
+; CHECK: [[GEP:%[0-9]+]]:_(p0) = G_PTR_ADD [[ADDR]], [[CST]](s64)
 ; CHECK: [[HI:%[0-9]+]]:_(s64) = G_LOAD [[GEP]](p0) :: (load 8 from %ir.ptr + 8)
 
 ; CHECK: [[SP:%[0-9]+]]:_(p0) = COPY $sp
 ; CHECK: [[CST2:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-; CHECK: [[GEP2:%[0-9]+]]:_(p0) = G_GEP [[SP]], [[CST2]](s64)
+; CHECK: [[GEP2:%[0-9]+]]:_(p0) = G_PTR_ADD [[SP]], [[CST2]](s64)
 ; CHECK: G_STORE [[LO]](s64), [[GEP2]](p0) :: (store 8 into stack, align 1)
 ; CHECK: [[SP:%[0-9]+]]:_(p0) = COPY $sp
 ; CHECK: [[CST3:%[0-9]+]]:_(s64) = COPY [[CST]]
-; CHECK: [[GEP3:%[0-9]+]]:_(p0) = G_GEP [[SP]], [[CST3]](s64)
+; CHECK: [[GEP3:%[0-9]+]]:_(p0) = G_PTR_ADD [[SP]], [[CST3]](s64)
 ; CHECK: G_STORE [[HI]](s64), [[GEP3]](p0) :: (store 8 into stack + 8, align 1)
 define void @test_split_struct([2 x i64]* %ptr) {
   %struct = load [2 x i64], [2 x i64]* %ptr
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/combiner-load-store-indexing.ll b/llvm/test/CodeGen/AArch64/GlobalISel/combiner-load-store-indexing.ll
new file mode 100644
index 000000000..a084e543c
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/combiner-load-store-indexing.ll
@@ -0,0 +1,182 @@
+; RUN: llc -mtriple=arm64-apple-ios -global-isel -global-isel-abort=1 -verify-machineinstrs -stop-after=aarch64-prelegalizer-combiner -force-legal-indexing %s -o - | FileCheck %s
+
+define i8* @test_simple_load_pre(i8* %ptr) {
+; CHECK-LABEL: name: test_simple_load_pre
+; CHECK: [[BASE:%.*]]:_(p0) = COPY $x0
+; CHECK: [[OFFSET:%.*]]:_(s64) = G_CONSTANT i64 42
+; CHECK-NOT: G_PTR_ADD
+; CHECK: {{%.*}}:_(s8), [[NEXT:%.*]]:_(p0) = G_INDEXED_LOAD [[BASE]], [[OFFSET]](s64), 1
+; CHECK: $x0 = COPY [[NEXT]](p0)
+
+  %next = getelementptr i8, i8* %ptr, i32 42
+  load volatile i8, i8* %next
+  ret i8* %next
+}
+
+define void @test_load_multiple_dominated(i8* %ptr, i1 %tst, i1 %tst2) {
+; CHECK-LABEL: name: test_load_multiple_dominated
+; CHECK: [[BASE:%.*]]:_(p0) = COPY $x0
+; CHECK: [[OFFSET:%.*]]:_(s64) = G_CONSTANT i64 42
+; CHECK-NOT: G_PTR_ADD
+; CHECK: {{%.*}}:_(s8), [[NEXT:%.*]]:_(p0) = G_INDEXED_LOAD [[BASE]], [[OFFSET]](s64), 1
+; CHECK: $x0 = COPY [[NEXT]](p0)
+  %next = getelementptr i8, i8* %ptr, i32 42
+  br i1 %tst, label %do_load, label %end
+
+do_load:
+  load volatile i8, i8* %next
+  br i1 %tst2, label %bb1, label %bb2
+
+bb1:
+  store volatile i8* %next, i8** undef
+  ret void
+
+bb2:
+  call void @bar(i8* %next)
+  ret void
+
+end:
+  ret void
+}
+
+define i8* @test_simple_store_pre(i8* %ptr) {
+; CHECK-LABEL: name: test_simple_store_pre
+; CHECK: [[BASE:%.*]]:_(p0) = COPY $x0
+; CHECK: [[VAL:%.*]]:_(s8) = G_CONSTANT i8 0
+; CHECK: [[OFFSET:%.*]]:_(s64) = G_CONSTANT i64 42
+; CHECK-NOT: G_PTR_ADD
+; CHECK: [[NEXT:%.*]]:_(p0) = G_INDEXED_STORE [[VAL]](s8), [[BASE]], [[OFFSET]](s64), 1
+; CHECK: $x0 = COPY [[NEXT]](p0)
+
+  %next = getelementptr i8, i8* %ptr, i32 42
+  store volatile i8 0, i8* %next
+  ret i8* %next
+}
+
+; The potentially pre-indexed address is used as the value stored. Converting
+; would produce the value too late but only by one instruction.
+define i64** @test_store_pre_val_loop(i64** %ptr) {
+; CHECK-LABEL: name: test_store_pre_val_loop
+; CHECK: G_PTR_ADD
+; CHECK: G_STORE %
+
+  %next = getelementptr i64*, i64** %ptr, i32 42
+  %next.p0 = bitcast i64** %next to i64*
+  store volatile i64* %next.p0, i64** %next
+  ret i64** %next
+}
+
+; Potentially pre-indexed address is used between GEP computing it and load.
+define i8* @test_load_pre_before(i8* %ptr) {
+; CHECK-LABEL: name: test_load_pre_before
+; CHECK: G_PTR_ADD
+; CHECK: BL @bar
+; CHECK: G_LOAD %
+
+  %next = getelementptr i8, i8* %ptr, i32 42
+  call void @bar(i8* %next)
+  load volatile i8, i8* %next
+  ret i8* %next
+}
+
+; Materializing the base into a writable register (from sp/fp) would be just as
+; bad as the original GEP.
+define i8* @test_alloca_load_pre() {
+; CHECK-LABEL: name: test_alloca_load_pre
+; CHECK: G_PTR_ADD
+; CHECK: G_LOAD %
+
+  %ptr = alloca i8, i32 128
+  %next = getelementptr i8, i8* %ptr, i32 42
+  load volatile i8, i8* %next
+  ret i8* %next
+}
+
+; Load does not dominate use of its address. No indexing.
+define i8* @test_pre_nodom(i8* %in, i1 %tst) {
+; CHECK-LABEL: name: test_pre_nodom
+; CHECK: G_PTR_ADD
+; CHECK: G_LOAD %
+
+  %next = getelementptr i8, i8* %in, i32 16
+  br i1 %tst, label %do_indexed, label %use_addr
+
+do_indexed:
+  %val = load i8, i8* %next
+  store i8 %val, i8* @var
+  store i8* %next, i8** @varp8
+  br label %use_addr
+
+use_addr:
+  ret i8* %next
+}
+
+define i8* @test_simple_load_post(i8* %ptr) {
+; CHECK-LABEL: name: test_simple_load_post
+; CHECK: [[BASE:%.*]]:_(p0) = COPY $x0
+; CHECK: [[OFFSET:%.*]]:_(s64) = G_CONSTANT i64 42
+; CHECK-NOT: G_PTR_ADD
+; CHECK: {{%.*}}:_(s8), [[NEXT:%.*]]:_(p0) = G_INDEXED_LOAD [[BASE]], [[OFFSET]](s64), 0
+; CHECK: $x0 = COPY [[NEXT]](p0)
+
+  %next = getelementptr i8, i8* %ptr, i32 42
+  load volatile i8, i8* %ptr
+  ret i8* %next
+}
+
+define i8* @test_simple_load_post_gep_after(i8* %ptr) {
+; CHECK-LABEL: name: test_simple_load_post_gep_after
+; CHECK: [[BASE:%.*]]:_(p0) = COPY $x0
+; CHECK: BL @get_offset
+; CHECK: [[OFFSET:%.*]]:_(s64) = COPY $x0
+; CHECK: {{%.*}}:_(s8), [[ADDR:%.*]]:_(p0) = G_INDEXED_LOAD [[BASE]], [[OFFSET]](s64), 0
+; CHECK: $x0 = COPY [[ADDR]](p0)
+
+  %offset = call i64 @get_offset()
+  load volatile i8, i8* %ptr
+  %next = getelementptr i8, i8* %ptr, i64 %offset
+  ret i8* %next
+}
+
+define i8* @test_load_post_keep_looking(i8* %ptr) {
+; CHECK: name: test_load_post_keep_looking
+; CHECK: G_INDEXED_LOAD
+
+  %offset = call i64 @get_offset()
+  load volatile i8, i8* %ptr
+  %intval = ptrtoint i8* %ptr to i8
+  store i8 %intval, i8* @var
+
+  %next = getelementptr i8, i8* %ptr, i64 %offset
+  ret i8* %next
+}
+
+; Base is frame index. Using indexing would need copy anyway.
+define i8* @test_load_post_alloca() {
+; CHECK-LABEL: name: test_load_post_alloca
+; CHECK: G_PTR_ADD
+; CHECK: G_LOAD %
+
+  %ptr = alloca i8, i32 128
+  %next = getelementptr i8, i8* %ptr, i32 42
+  load volatile i8, i8* %ptr
+  ret i8* %next
+}
+
+; Offset computation does not dominate the load we might be indexing.
+define i8* @test_load_post_gep_offset_after(i8* %ptr) {
+; CHECK-LABEL: name: test_load_post_gep_offset_after
+; CHECK: G_LOAD %
+; CHECK: BL @get_offset
+; CHECK: G_PTR_ADD
+
+  load volatile i8, i8* %ptr
+  %offset = call i64 @get_offset()
+  %next = getelementptr i8, i8* %ptr, i64 %offset
+  ret i8* %next
+}
+
+declare void @bar(i8*)
+declare i64 @get_offset()
+@var = global i8 0
+@varp8 = global i8* null
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/const-0.ll b/llvm/test/CodeGen/AArch64/GlobalISel/const-0.ll
new file mode 100644
index 000000000..89d1ee29b
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/const-0.ll
@@ -0,0 +1,25 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -global-isel -O0 -o - %s | FileCheck %s
+
+%struct.comp = type { i8*, i32, i8*, [3 x i8], i32 }
+
+define void @regbranch() {
+; CHECK-LABEL: regbranch:
+; CHECK: mov {{w[0-9]+}}, #0
+cond_next240.i:
+  br i1 false, label %cond_true251.i, label %cond_next272.i
+
+cond_true251.i:
+  switch i8 0, label %cond_next272.i [
+      i8 42, label %bb268.i
+      i8 43, label %bb268.i
+      i8 63, label %bb268.i
+  ]
+
+bb268.i:
+  br label %cond_next272.i
+
+cond_next272.i:
+  %len.2.i = phi i32 [ 0, %bb268.i ], [ 0, %cond_next240.i ], [ 0, %cond_true251.i ]
+  %tmp278.i = icmp eq i32 %len.2.i, 1
+  ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/dynamic-alloca.ll b/llvm/test/CodeGen/AArch64/GlobalISel/dynamic-alloca.ll
index baf0fa9aa..47b387817 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/dynamic-alloca.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/dynamic-alloca.ll
@@ -1,48 +1,59 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 ; RUN: llc -mtriple=aarch64 -global-isel %s -o - -stop-after=irtranslator | FileCheck %s
 
-; CHECK-LABEL: name: test_simple_alloca
-; CHECK: [[NUMELTS:%[0-9]+]]:_(s32) = COPY $w0
-; CHECK: [[TYPE_SIZE:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
-; CHECK: [[NUMELTS_64:%[0-9]+]]:_(s64) = G_ZEXT [[NUMELTS]](s32)
-; CHECK: [[NUMBYTES:%[0-9]+]]:_(s64) = G_MUL [[NUMELTS_64]], [[TYPE_SIZE]]
-; CHECK: [[SP_TMP:%[0-9]+]]:_(p0) = COPY $sp
-; CHECK: [[ALLOC:%[0-9]+]]:_(p0) = G_GEP [[SP_TMP]], [[NUMBYTES]]
-; CHECK: [[ALIGNED_ALLOC:%[0-9]+]]:_(p0) = G_PTR_MASK [[ALLOC]], 4
-; CHECK: $sp = COPY [[ALIGNED_ALLOC]]
-; CHECK: [[ALLOC:%[0-9]+]]:_(p0) = COPY [[ALIGNED_ALLOC]]
-; CHECK: $x0 = COPY [[ALLOC]]
 define i8* @test_simple_alloca(i32 %numelts) {
+  ; CHECK-LABEL: name: test_simple_alloca
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $w0
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $w0
+  ; CHECK:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+  ; CHECK:   [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[COPY]](s32)
+  ; CHECK:   [[MUL:%[0-9]+]]:_(s64) = G_MUL [[ZEXT]], [[C]]
+  ; CHECK:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 15
+  ; CHECK:   [[ADD:%[0-9]+]]:_(s64) = nuw G_ADD [[MUL]], [[C1]]
+  ; CHECK:   [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 -16
+  ; CHECK:   [[AND:%[0-9]+]]:_(s64) = G_AND [[ADD]], [[C2]]
+  ; CHECK:   [[DYN_STACKALLOC:%[0-9]+]]:_(p0) = G_DYN_STACKALLOC [[AND]](s64), 0
+  ; CHECK:   $x0 = COPY [[DYN_STACKALLOC]](p0)
+  ; CHECK:   RET_ReallyLR implicit $x0
   %addr = alloca i8, i32 %numelts
   ret i8* %addr
 }
 
-; CHECK-LABEL: name: test_aligned_alloca
-; CHECK: [[NUMELTS:%[0-9]+]]:_(s32) = COPY $w0
-; CHECK: [[TYPE_SIZE:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
-; CHECK: [[NUMELTS_64:%[0-9]+]]:_(s64) = G_ZEXT [[NUMELTS]](s32)
-; CHECK: [[NUMBYTES:%[0-9]+]]:_(s64) = G_MUL [[NUMELTS_64]], [[TYPE_SIZE]]
-; CHECK: [[SP_TMP:%[0-9]+]]:_(p0) = COPY $sp
-; CHECK: [[ALLOC:%[0-9]+]]:_(p0) = G_GEP [[SP_TMP]], [[NUMBYTES]]
-; CHECK: [[ALIGNED_ALLOC:%[0-9]+]]:_(p0) = G_PTR_MASK [[ALLOC]], 5
-; CHECK: $sp = COPY [[ALIGNED_ALLOC]]
-; CHECK: [[ALLOC:%[0-9]+]]:_(p0) = COPY [[ALIGNED_ALLOC]]
-; CHECK: $x0 = COPY [[ALLOC]]
 define i8* @test_aligned_alloca(i32 %numelts) {
+  ; CHECK-LABEL: name: test_aligned_alloca
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $w0
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $w0
+  ; CHECK:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+  ; CHECK:   [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[COPY]](s32)
+  ; CHECK:   [[MUL:%[0-9]+]]:_(s64) = G_MUL [[ZEXT]], [[C]]
+  ; CHECK:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 15
+  ; CHECK:   [[ADD:%[0-9]+]]:_(s64) = nuw G_ADD [[MUL]], [[C1]]
+  ; CHECK:   [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 -16
+  ; CHECK:   [[AND:%[0-9]+]]:_(s64) = G_AND [[ADD]], [[C2]]
+  ; CHECK:   [[DYN_STACKALLOC:%[0-9]+]]:_(p0) = G_DYN_STACKALLOC [[AND]](s64), 32
+  ; CHECK:   $x0 = COPY [[DYN_STACKALLOC]](p0)
+  ; CHECK:   RET_ReallyLR implicit $x0
   %addr = alloca i8, i32 %numelts, align 32
   ret i8* %addr
 }
 
-; CHECK-LABEL: name: test_natural_alloca
-; CHECK: [[NUMELTS:%[0-9]+]]:_(s32) = COPY $w0
-; CHECK: [[TYPE_SIZE:%[0-9]+]]:_(s64) = G_CONSTANT i64 -16
-; CHECK: [[NUMELTS_64:%[0-9]+]]:_(s64) = G_ZEXT [[NUMELTS]](s32)
-; CHECK: [[NUMBYTES:%[0-9]+]]:_(s64) = G_MUL [[NUMELTS_64]], [[TYPE_SIZE]]
-; CHECK: [[SP_TMP:%[0-9]+]]:_(p0) = COPY $sp
-; CHECK: [[ALLOC:%[0-9]+]]:_(p0) = G_GEP [[SP_TMP]], [[NUMBYTES]]
-; CHECK: $sp = COPY [[ALLOC]]
-; CHECK: [[ALLOC_TMP:%[0-9]+]]:_(p0) = COPY [[ALLOC]]
-; CHECK: $x0 = COPY [[ALLOC_TMP]]
 define i128* @test_natural_alloca(i32 %numelts) {
+  ; CHECK-LABEL: name: test_natural_alloca
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $w0
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $w0
+  ; CHECK:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
+  ; CHECK:   [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[COPY]](s32)
+  ; CHECK:   [[MUL:%[0-9]+]]:_(s64) = G_MUL [[ZEXT]], [[C]]
+  ; CHECK:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 15
+  ; CHECK:   [[ADD:%[0-9]+]]:_(s64) = nuw G_ADD [[MUL]], [[C1]]
+  ; CHECK:   [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 -16
+  ; CHECK:   [[AND:%[0-9]+]]:_(s64) = G_AND [[ADD]], [[C2]]
+  ; CHECK:   [[DYN_STACKALLOC:%[0-9]+]]:_(p0) = G_DYN_STACKALLOC [[AND]](s64), 0
+  ; CHECK:   $x0 = COPY [[DYN_STACKALLOC]](p0)
+  ; CHECK:   RET_ReallyLR implicit $x0
   %addr = alloca i128, i32 %numelts
   ret i128* %addr
 }
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/gisel-commandline-option.ll b/llvm/test/CodeGen/AArch64/GlobalISel/gisel-commandline-option.ll
index 54361eab6..f7cc409ea 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/gisel-commandline-option.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/gisel-commandline-option.ll
@@ -16,15 +16,15 @@
 
 ; RUN: llc -mtriple=aarch64-- -debug-pass=Structure %s -o /dev/null 2>&1 \
 ; RUN:   -verify-machineinstrs=0 -global-isel \
-; RUN:   | FileCheck %s --check-prefix ENABLED --check-prefix NOFALLBACK
+; RUN:   | FileCheck %s --check-prefix ENABLED --check-prefix NOFALLBACK --check-prefix ENABLED-O1
 
 ; RUN: llc -mtriple=aarch64-- -debug-pass=Structure %s -o /dev/null 2>&1 \
 ; RUN:   -verify-machineinstrs=0 -global-isel -global-isel-abort=2 \
-; RUN:   | FileCheck %s --check-prefix ENABLED --check-prefix FALLBACK
+; RUN:   | FileCheck %s --check-prefix ENABLED --check-prefix FALLBACK  --check-prefix ENABLED-O1
 
 ; RUN: llc -mtriple=aarch64-- -debug-pass=Structure %s -o /dev/null 2>&1 \
 ; RUN:   -verify-machineinstrs=0 -O1 -aarch64-enable-global-isel-at-O=3 \
-; RUN:   | FileCheck %s --check-prefix ENABLED
+; RUN:   | FileCheck %s --check-prefix ENABLED  --check-prefix ENABLED-O1
 
 ; RUN: llc -mtriple=aarch64-- -debug-pass=Structure %s -o /dev/null 2>&1 \
 ; RUN:   -verify-machineinstrs=0 -O1 -aarch64-enable-global-isel-at-O=0 \
@@ -43,6 +43,8 @@
 
 ; ENABLED:       IRTranslator
 ; VERIFY-NEXT:   Verify generated machine code
+; ENABLED-NEXT:  Analysis for ComputingKnownBits
+; ENABLED-O1-NEXT:  MachineDominator Tree Construction
 ; ENABLED-NEXT:  PreLegalizerCombiner
 ; VERIFY-NEXT:   Verify generated machine code
 ; ENABLED-NEXT:  Analysis containing CSE Info
@@ -50,8 +52,9 @@
 ; VERIFY-NEXT:   Verify generated machine code
 ; ENABLED-NEXT:  RegBankSelect
 ; VERIFY-NEXT:   Verify generated machine code
-; ENABLED-O0-NEXT:  Localizer
+; ENABLED-NEXT:  Localizer
 ; VERIFY-O0-NEXT:   Verify generated machine code
+; ENABLED-NEXT: Analysis for ComputingKnownBits
 ; ENABLED-NEXT:  InstructionSelect
 ; VERIFY-NEXT:   Verify generated machine code
 ; ENABLED-NEXT:  ResetMachineFunction
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/gisel-fail-intermediate-legalizer.ll b/llvm/test/CodeGen/AArch64/GlobalISel/gisel-fail-intermediate-legalizer.ll
index e333f742e..03794caf8 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/gisel-fail-intermediate-legalizer.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/gisel-fail-intermediate-legalizer.ll
@@ -1,8 +1,8 @@
 ;RUN: llc -mtriple=aarch64-unknown-unknown -o - -global-isel -global-isel-abort=2 %s 2>&1 | FileCheck %s
 ; CHECK: fallback
 ; CHECK-LABEL: foo
-define i16 @foo(half* %p) {
-  %tmp0 = load half, half* %p
-  %tmp1 = fptoui half %tmp0 to i16
+define i16 @foo(fp128* %p) {
+  %tmp0 = load fp128, fp128* %p
+  %tmp1 = fptoui fp128 %tmp0 to i16
   ret i16 %tmp1
 }
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/integration-shuffle-vector.ll b/llvm/test/CodeGen/AArch64/GlobalISel/integration-shuffle-vector.ll
new file mode 100644
index 000000000..a30c85349
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/integration-shuffle-vector.ll
@@ -0,0 +1,25 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; RUN: llc -global-isel -mtriple aarch64-apple-ios -stop-after=instruction-select %s -o - | FileCheck %s
+
+; Check that packing incoming arguments into a big vector type
+; and unpacking them in registers for the call to @bar gets selected as just
+; simple copies. I.e., we don't artificial try to keep the big
+; vector (%vec) alive.
+define void @shuffle_to_concat_vector(<2 x i64> %a, <2 x i64> %b) {
+  ; CHECK-LABEL: name: shuffle_to_concat_vector
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $q0, $q1
+  ; CHECK:   [[COPY:%[0-9]+]]:fpr128 = COPY $q0
+  ; CHECK:   [[COPY1:%[0-9]+]]:fpr128 = COPY $q1
+  ; CHECK:   ADJCALLSTACKDOWN 0, 0, implicit-def $sp, implicit $sp
+  ; CHECK:   $q0 = COPY [[COPY]]
+  ; CHECK:   $q1 = COPY [[COPY1]]
+  ; CHECK:   BL @bar, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $q0, implicit $q1
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp
+  ; CHECK:   RET_ReallyLR
+  %vec = shufflevector <2 x i64> %a, <2 x i64> %b, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  call void @bar(<4 x i64> %vec)
+  ret void
+}
+
+declare void @bar(<4 x i64> %vec)
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-exceptions.ll b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-exceptions.ll
index 5edf9d96c..bb393e08d 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-exceptions.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-exceptions.ll
@@ -68,13 +68,13 @@ continue:
 
 ; CHECK: [[SP:%[0-9]+]]:_(p0) = COPY $sp
 ; CHECK: [[OFFSET:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-; CHECK: [[SLOT:%[0-9]+]]:_(p0) = G_GEP [[SP]], [[OFFSET]](s64)
+; CHECK: [[SLOT:%[0-9]+]]:_(p0) = G_PTR_ADD [[SP]], [[OFFSET]](s64)
 ; CHECK: [[ANSWER_EXT:%[0-9]+]]:_(s64) = G_ANYEXT [[ANSWER]]
 ; CHECK: G_STORE [[ANSWER_EXT]](s64), [[SLOT]]
 
 ; CHECK: [[SP:%[0-9]+]]:_(p0) = COPY $sp
 ; CHECK: [[OFFSET:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-; CHECK: [[SLOT:%[0-9]+]]:_(p0) = G_GEP [[SP]], [[OFFSET]](s64)
+; CHECK: [[SLOT:%[0-9]+]]:_(p0) = G_PTR_ADD [[SP]], [[OFFSET]](s64)
 ; CHECK: G_STORE [[ONE]](s32), [[SLOT]]
 
 ; CHECK: BL @printf
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-extends.ll b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-extends.ll
new file mode 100644
index 000000000..b5658155f
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-extends.ll
@@ -0,0 +1,30 @@
+; RUN: llc -O0 -mtriple=aarch64-apple-ios -global-isel -stop-after=irtranslator %s -o - | FileCheck %s
+
+; Test that extends correctly translate to G_[ZS]EXT. The translator will never
+; emit a G_SEXT_INREG.
+
+define i32 @test_zext(i32 %a) {
+  ; CHECK-LABEL: name: test_zext
+  ; CHECK: %0:_(s32) = COPY $w0
+  ; CHECK: %1:_(s8) = G_TRUNC %0(s32)
+  ; CHECK: %2:_(s16) = G_ZEXT %1(s8)
+  ; CHECK: %3:_(s32) = G_ZEXT %2(s16)
+  ; CHECK: $w0 = COPY %3(s32)
+  %tmp0 = trunc i32 %a to i8
+  %tmp1 = zext i8 %tmp0 to i16
+  %tmp2 = zext i16 %tmp1 to i32
+  ret i32 %tmp2
+}
+
+define i32 @test_sext(i32 %a) {
+  ; CHECK-LABEL: name: test_sext
+  ; CHECK: %0:_(s32) = COPY $w0
+  ; CHECK: %1:_(s8) = G_TRUNC %0(s32)
+  ; CHECK: %2:_(s16) = G_SEXT %1(s8)
+  ; CHECK: %3:_(s32) = G_SEXT %2(s16)
+  ; CHECK: $w0 = COPY %3(s32)
+  %tmp0 = trunc i32 %a to i8
+  %tmp1 = sext i8 %tmp0 to i16
+  %tmp2 = sext i16 %tmp1 to i32
+  ret i32 %tmp2
+}
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-split-vector-arg.ll b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-split-vector-arg.ll
new file mode 100644
index 000000000..ef74cdb4a
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-split-vector-arg.ll
@@ -0,0 +1,22 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; RUN: llc -global-isel -global-isel-abort=1 %s -stop-after=irtranslator -o - | FileCheck %s
+target triple = "aarch64-apple-ios"
+
+; Check that we correctly split %arg into two vector registers of
+; size <2 x i64>.
+define hidden fastcc <4 x float> @foo(<4 x i64> %arg) unnamed_addr #0 {
+  ; CHECK-LABEL: name: foo
+  ; CHECK: bb.1.bb:
+  ; CHECK:   liveins: $q0, $q1
+  ; CHECK:   [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $q0
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $q1
+  ; CHECK:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s64>) = G_CONCAT_VECTORS [[COPY]](<2 x s64>), [[COPY1]](<2 x s64>)
+  ; CHECK:   [[UITOFP:%[0-9]+]]:_(<4 x s32>) = G_UITOFP [[CONCAT_VECTORS]](<4 x s64>)
+  ; CHECK:   $q0 = COPY [[UITOFP]](<4 x s32>)
+  ; CHECK:   RET_ReallyLR implicit $q0
+bb:
+  %tmp = uitofp <4 x i64> %arg to <4 x float>
+  ret <4 x float> %tmp
+}
+
+attributes #0 = { nounwind readnone }
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-tbaa.ll b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-tbaa.ll
new file mode 100644
index 000000000..bbf574703
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-tbaa.ll
@@ -0,0 +1,19 @@
+; RUN: llc -O0 -mtriple=aarch64-unknown-unknown -stop-after=irtranslator -o - %s | FileCheck %s
+
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+
+define void @snork() {
+bb:
+  %tmp1 = getelementptr i16, i16* null, i64 0
+  %tmp5 = getelementptr i16, i16* null, i64 2
+  %tmp6 = load i16, i16* %tmp1, align 2, !tbaa !0
+  store i16 %tmp6, i16* %tmp5, align 2, !tbaa !0
+  ; CHECK: %5:_(s16) = G_LOAD %2(p0) :: (load 2 from %ir.tmp1, !tbaa !0)
+  ; CHECK: G_STORE %5(s16), %4(p0) :: (store 2 into %ir.tmp5, !tbaa !0)
+  ret void
+}
+
+!0 = !{!1, !1, i64 0}
+!1 = !{!"short", !2, i64 0}
+!2 = !{!"omnipotent char", !3, i64 0}
+!3 = !{!"Simple C/C++ TBAA"}
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-sext-128.ll b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-sext-128.ll
new file mode 100644
index 000000000..aff5245d9
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-sext-128.ll
@@ -0,0 +1,9 @@
+; RUN: llc -O0 --global-isel=1 %s -o - -verify-machineinstrs
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64-unknown-linux-gnu"
+
+define i1 @foo(i64) {
+    %a = sext i64 %0 to i128
+    %b = icmp sle i128 %a, 0
+    ret i1 %b
+}
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/memcpy_chk_no_tail.ll b/llvm/test/CodeGen/AArch64/GlobalISel/memcpy_chk_no_tail.ll
new file mode 100644
index 000000000..68d833338
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/memcpy_chk_no_tail.ll
@@ -0,0 +1,30 @@
+; RUN: llc -global-isel -verify-machineinstrs %s -o - | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+target triple = "arm64-apple-ios13.0.0"
+
+%struct.int_sqrt = type { i32, i32 }
+
+; Function Attrs: nounwind optsize ssp uwtable
+; CHECK-LABEL: @usqrt
+; CHECK-NOT: b memcpy
+; CHECK: bl _memcpy
+define void @usqrt(i32 %x, %struct.int_sqrt* %q) local_unnamed_addr #0 {
+  %a = alloca i32, align 4
+  %bc = bitcast i32* %a to i8*
+  %bc2 = bitcast %struct.int_sqrt* %q to i8*
+  %obj = tail call i64 @llvm.objectsize.i64.p0i8(i8* %bc2, i1 false, i1 true, i1 false)
+  %call = call i8* @__memcpy_chk(i8* %bc2, i8* nonnull %bc, i64 1000, i64 %obj) #4
+  ret void
+}
+
+; Function Attrs: nofree nounwind optsize
+declare i8* @__memcpy_chk(i8*, i8*, i64, i64) local_unnamed_addr #2
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare i64 @llvm.objectsize.i64.p0i8(i8*, i1 immarg, i1 immarg, i1 immarg) #3
+attributes #0 = { optsize "disable-tail-calls"="false" "frame-pointer"="all" }
+attributes #2 = { nofree nounwind "disable-tail-calls"="false" "frame-pointer"="all" }
+attributes #3 = { nounwind readnone speculatable willreturn }
+attributes #4 = { nounwind optsize }
+
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/no-neon-no-fp.ll b/llvm/test/CodeGen/AArch64/GlobalISel/no-neon-no-fp.ll
new file mode 100644
index 000000000..f2e6fbce0
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/no-neon-no-fp.ll
@@ -0,0 +1,13 @@
+; RUN: not llc -o - -verify-machineinstrs -global-isel -global-isel-abort=1 -stop-after=legalizer %s 2>&1 | FileCheck %s
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64-unknown-unknown"
+
+; CHECK: unable to legalize instruction: G_STORE %1:_(s128), %0:_(p0) :: (store 16 into %ir.ptr) (in function: foo)
+define void @foo(i128 *%ptr) #0 align 2 {
+entry:
+  store i128 0, i128* %ptr, align 16
+  ret void
+}
+
+attributes #0 = { "use-soft-float"="false" "target-features"="-fp-armv8,-neon" }
+
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/swifterror.ll b/llvm/test/CodeGen/AArch64/GlobalISel/swifterror.ll
index 386771fa9..329828f5e 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/swifterror.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/swifterror.ll
@@ -9,7 +9,7 @@ declare void @free(i8*)
 define float @foo(%swift_error** swifterror %error_ptr_ref) {
 ; CHECK-LABEL: foo:
 ; CHECK: mov [[ID:w[0-9]+]], #1
-; CHECK: mov x0, #16
+; CHECK: mov w0, #16
 ; CHECK: malloc
 ; CHECK: strb [[ID]], [x0, #8]
 ; CHECK: mov x21, x0
@@ -59,7 +59,7 @@ define float @caller2(i8* %error_ref) {
 ; CHECK-LABEL: caller2:
 ; CHECK: mov [[ID:x[0-9]+]], x0
 ; CHECK: fmov [[CMP:s[0-9]+]], #1.0
-; CHECK: mov x21, #0
+; CHECK: mov x21, xzr
 ; CHECK: bl {{.*}}foo
 ; CHECK: cbnz x21
 ; CHECK: fcmp s0, [[CMP]]
@@ -98,9 +98,9 @@ handler:
 define float @foo_if(%swift_error** swifterror %error_ptr_ref, i32 %cc) {
 ; CHECK-LABEL: foo_if:
 ; CHECK: cbz w0
-; CHECK: mov [[ID:w[0-9]+]], #1
-; CHECK: mov x0, #16
+; CHECK: mov w0, #16
 ; CHECK: malloc
+; CHECK: mov [[ID:w[0-9]+]], #1
 ; CHECK: strb [[ID]], [x0, #8]
 ; CHECK: mov x21, x0
 ; CHECK-NOT: x21
@@ -127,7 +127,7 @@ normal:
 define float @foo_loop(%swift_error** swifterror %error_ptr_ref, i32 %cc, float %cc2) {
 ; CHECK-LABEL: foo_loop:
 ; CHECK: cbz
-; CHECK: mov x0, #16
+; CHECK: mov w0, #16
 ; CHECK: malloc
 ; CHECK: mov x21, x0
 ; CHECK: strb w{{.*}}, [x0, #8]
@@ -165,7 +165,7 @@ define void @foo_sret(%struct.S* sret %agg.result, i32 %val1, %swift_error** swi
 ; CHECK-LABEL: foo_sret:
 ; CHECK: mov [[SRET:x[0-9]+]], x8
 ; CHECK: mov [[ID:w[0-9]+]], #1
-; CHECK: mov x0, #16
+; CHECK: mov w0, #16
 ; CHECK: malloc
 ; CHECK: strb [[ID]], [x0, #8]
 ; CHECK: str w{{.*}}, [{{.*}}[[SRET]], #4]
@@ -187,7 +187,7 @@ entry:
 define float @caller3(i8* %error_ref) {
 ; CHECK-LABEL: caller3:
 ; CHECK: mov [[ID:x[0-9]+]], x0
-; CHECK: mov [[ZERO:x[0-9]+]], #0
+; CHECK: mov [[ZERO:x[0-9]+]], xzr
 ; CHECK: bl {{.*}}foo_sret
 ; CHECK: mov x0, x21
 ; CHECK: cbnz x21
@@ -221,19 +221,16 @@ declare void @llvm.va_start(i8*) nounwind
 define float @foo_vararg(%swift_error** swifterror %error_ptr_ref, ...) {
 ; CHECK-LABEL: foo_vararg:
 ; CHECK-DAG: mov [[ID:w[0-9]+]], #1
-; CHECK: mov x0, #16
+; CHECK: mov w0, #16
 ; CHECK: malloc
 ; CHECK-DAG: strb [[ID]], [x0, #8]
 
 ; First vararg
-; CHECK: ldr {{w[0-9]+}}, [x[[ARG1:[0-9]+]]]
+; CHECK: ldr {{w[0-9]+}}, [x[[ARG1:[0-9]+]]], #8
 ; Second vararg
-; CHECK: mov [[EIGHT:x[0-9]+]], #8
-; CHECK: add x[[ARG2:[0-9]+]], x[[ARG1]], [[EIGHT]]
-; CHECK: ldr {{w[0-9]+}}, [x[[ARG2]]]
+; CHECK: ldr {{w[0-9]+}}, [x[[ARG1]]], #8
 ; Third vararg
-; CHECK: add x[[ARG3:[0-9]+]], x[[ARG2]], [[EIGHT]]
-; CHECK: ldr {{w[0-9]+}}, [x[[ARG3]]]
+; CHECK: ldr {{w[0-9]+}}, [x[[ARG1]]], #8
 
 ; CHECK: mov x21, x0
 ; CHECK-NOT: x21
@@ -266,8 +263,8 @@ define float @caller4(i8* %error_ref) {
 
 ; CHECK: mov [[ID:x[0-9]+]], x0
 ; CHECK: stp {{x[0-9]+}}, {{x[0-9]+}}, [sp]
-; CHECK: mov x21, #0
 ; CHECK: str {{x[0-9]+}}, [sp, #16]
+; CHECK: mov x21, xzr
 
 ; CHECK: bl {{.*}}foo_vararg
 ; CHECK: mov x0, x21
@@ -336,16 +333,16 @@ entry:
 ; CHECK:  mov      x27, x7
 ; CHECK:  mov      x28, x21
 ; Setup call.
-; CHECK:  mov     x8, #0
-; CHECK:  mov     x0, #1
-; CHECK:  mov     x1, #2
-; CHECK:  mov     x2, #3
-; CHECK:  mov     x3, #4
-; CHECK:  mov     x4, #5
-; CHECK:  mov     x5, #6
-; CHECK:  mov     x6, #7
-; CHECK:  mov     x7, #8
-; CHECK:  mov      x21, #0
+; CHECK:  mov     w0, #1
+; CHECK:  mov     w1, #2
+; CHECK:  mov     w2, #3
+; CHECK:  mov     w3, #4
+; CHECK:  mov     w4, #5
+; CHECK:  mov     w5, #6
+; CHECK:  mov     w6, #7
+; CHECK:  mov     w7, #8
+; CHECK:  str     xzr, [sp]
+; CHECK:  mov      x21, xzr
 ; CHECK:  bl      _params_in_reg2
 ; Restore original arguments for next call.
 ; CHECK:  ldr      x0, [sp
@@ -401,15 +398,15 @@ declare swiftcc void @params_in_reg2(i64, i64, i64, i64, i64, i64, i64, i64, i8*
 ; CHECK:  mov      x27, x7
 ; CHECK:  mov      x28, x21
 ; Setup call arguments.
-; CHECK:  mov     x0, #1
-; CHECK:  mov     x1, #2
-; CHECK:  mov     x2, #3
-; CHECK:  mov     x3, #4
-; CHECK:  mov     x4, #5
-; CHECK:  mov     x5, #6
-; CHECK:  mov     x6, #7
-; CHECK:  mov     x7, #8
-; CHECK:  mov      x21, #0
+; CHECK:  mov     w0, #1
+; CHECK:  mov     w1, #2
+; CHECK:  mov     w2, #3
+; CHECK:  mov     w3, #4
+; CHECK:  mov     w4, #5
+; CHECK:  mov     w5, #6
+; CHECK:  mov     w6, #7
+; CHECK:  mov     w7, #8
+; CHECK:  mov      x21, xzr
 ; CHECK:  bl      _params_in_reg2
 ; Store swifterror %error_ptr_ref.
 ; CHECK:  stp     {{x[0-9]+}}, x21, [sp]
@@ -436,14 +433,14 @@ declare swiftcc void @params_in_reg2(i64, i64, i64, i64, i64, i64, i64, i64, i8*
 ; Save swifterror %err.
 ; CHECK:  mov      x19, x21
 ; Setup call.
-; CHECK:  mov     x0, #1
-; CHECK:  mov     x1, #2
-; CHECK:  mov     x2, #3
-; CHECK:  mov     x3, #4
-; CHECK:  mov     x4, #5
-; CHECK:  mov     x5, #6
-; CHECK:  mov     x6, #7
-; CHECK:  mov     x7, #8
+; CHECK:  mov     w0, #1
+; CHECK:  mov     w1, #2
+; CHECK:  mov     w2, #3
+; CHECK:  mov     w3, #4
+; CHECK:  mov     w4, #5
+; CHECK:  mov     w5, #6
+; CHECK:  mov     w6, #7
+; CHECK:  mov     w7, #8
 ; ... setup call with swiferror %error_ptr_ref.
 ; CHECK:  ldr     x21, [sp, #8]
 ; CHECK:  bl      _params_in_reg2
@@ -490,9 +487,18 @@ entry:
   ret void
 }
 
+; CHECK: tailcall_from_swifterror2
+; CHECK-NOT: b _simple_fn
+; CHECK: bl _simple_fn
+declare void @simple_fn()
+define swiftcc void @tailcall_from_swifterror2(%swift_error** swifterror %error_ptr_ref) {
+  tail call void @simple_fn()
+  ret void
+}
+
 declare swiftcc void @foo2(%swift_error** swifterror)
 ; CHECK-LABEL: testAssign
-; CHECK: mov      x21, #0
+; CHECK: mov      x21, xzr
 ; CHECK: bl      _foo2
 ; CHECK: mov      x0, x21
 
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/swiftself.ll b/llvm/test/CodeGen/AArch64/GlobalISel/swiftself.ll
new file mode 100644
index 000000000..8ed06f233
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/swiftself.ll
@@ -0,0 +1,62 @@
+; RUN: llc -verify-machineinstrs -mtriple=aarch64-apple-ios -o - %s -global-isel | FileCheck %s
+
+; Parameter with swiftself should be allocated to x20.
+; CHECK-LABEL: swiftself_param:
+; CHECK: mov x0, x20
+; CHECK-NEXT: ret
+define i8* @swiftself_param(i8* swiftself %addr0) {
+  ret i8 *%addr0
+}
+
+; Check that x20 is used to pass a swiftself argument.
+; CHECK-LABEL: call_swiftself:
+; CHECK: mov x20, x0
+; CHECK: bl {{_?}}swiftself_param
+; CHECK: ret
+define i8 *@call_swiftself(i8* %arg) {
+  %res = call i8 *@swiftself_param(i8* swiftself %arg)
+  ret i8 *%res
+}
+
+; Demonstrate that we do not need any movs when calling multiple functions
+; with swiftself argument.
+; CHECK-LABEL: swiftself_passthrough:
+; CHECK-NOT: mov{{.*}}x20
+; CHECK: bl {{_?}}swiftself_param
+; CHECK-NOT: mov{{.*}}x20
+; CHECK-NEXT: bl {{_?}}swiftself_param
+; CHECK: ret
+define void @swiftself_passthrough(i8* swiftself %addr0) {
+  call i8 *@swiftself_param(i8* swiftself %addr0)
+  call i8 *@swiftself_param(i8* swiftself %addr0)
+  ret void
+}
+
+; We can not use a tail call if the callee swiftself is not the same as the
+; caller one.
+; CHECK-LABEL: swiftself_notail:
+; CHECK: mov x20, x0
+; CHECK: bl {{_?}}swiftself_param
+; CHECK: ret
+define i8* @swiftself_notail(i8* swiftself %addr0, i8* %addr1) nounwind {
+  %res = tail call i8* @swiftself_param(i8* swiftself %addr1)
+  ret i8* %res
+}
+
+; We cannot pretend that 'x0' is alive across the thisreturn_attribute call as
+; we normally would. We marked the first parameter with swiftself which means it
+; will no longer be passed in x0.
+declare swiftcc i8* @thisreturn_attribute(i8* returned swiftself)
+; CHECK-LABEL: swiftself_nothisreturn:
+; CHECK-DAG: ldr  x20, [x20]
+; CHECK-DAG: mov [[CSREG:x[1-9].*]], x8
+; CHECK: bl {{_?}}thisreturn_attribute
+; CHECK: str x0, {{\[}}[[CSREG]]
+; CHECK: ret
+define hidden swiftcc void @swiftself_nothisreturn(i8** noalias nocapture sret, i8** noalias nocapture readonly swiftself) {
+entry:
+  %2 = load i8*, i8** %1, align 8
+  %3 = tail call swiftcc i8* @thisreturn_attribute(i8* swiftself %2)
+  store i8* %3, i8** %0, align 8
+  ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/tail-call-no-save-fp-lr.ll b/llvm/test/CodeGen/AArch64/GlobalISel/tail-call-no-save-fp-lr.ll
new file mode 100644
index 000000000..1c3630f42
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/tail-call-no-save-fp-lr.ll
@@ -0,0 +1,23 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc %s -verify-machineinstrs -mtriple aarch64-apple-darwin -global-isel -global-isel-abort=1 -frame-pointer=non-leaf -o - 2>&1 | FileCheck %s
+
+; Check that we get a tail call to foo without saving fp/lr.
+define void @bar(i32 %a) #1 {
+; CHECK-LABEL: bar:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    b _zoo
+entry:
+  tail call void @zoo(i32 undef)
+  ret void
+}
+
+define void @zoo(i32 %a) {
+; CHECK-LABEL: zoo:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    ret
+entry:
+  ret void
+}
+
+attributes #1 = { "frame-pointer"="all" }
+
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/translate-gep.ll b/llvm/test/CodeGen/AArch64/GlobalISel/translate-gep.ll
index 6412ac589..aefc20683 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/translate-gep.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/translate-gep.ll
@@ -10,7 +10,7 @@ define i8*  @translate_element_size1(i64 %arg) {
   ; CHECK:   [[COPY:%[0-9]+]]:_(s64) = COPY $x0
   ; CHECK:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
   ; CHECK:   [[INTTOPTR:%[0-9]+]]:_(p0) = G_INTTOPTR [[C]](s64)
-  ; CHECK:   [[GEP:%[0-9]+]]:_(p0) = G_GEP [[INTTOPTR]], [[COPY]](s64)
+  ; CHECK:   [[GEP:%[0-9]+]]:_(p0) = G_PTR_ADD [[INTTOPTR]], [[COPY]](s64)
   ; CHECK:   [[COPY1:%[0-9]+]]:_(p0) = COPY [[GEP]](p0)
   ; CHECK:   $x0 = COPY [[COPY1]](p0)
   ; CHECK:   RET_ReallyLR implicit $x0
@@ -25,7 +25,7 @@ define %type* @first_offset_const(%type* %addr) {
   ; CHECK:   liveins: $x0
   ; CHECK:   [[COPY:%[0-9]+]]:_(p0) = COPY $x0
   ; CHECK:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 32
-  ; CHECK:   [[GEP:%[0-9]+]]:_(p0) = G_GEP [[COPY]], [[C]](s64)
+  ; CHECK:   [[GEP:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
   ; CHECK:   $x0 = COPY [[GEP]](p0)
   ; CHECK:   RET_ReallyLR implicit $x0
   %res = getelementptr %type, %type* %addr, i32 1
@@ -54,7 +54,7 @@ define %type* @first_offset_variable(%type* %addr, i64 %idx) {
   ; CHECK:   [[COPY1:%[0-9]+]]:_(s64) = COPY $x1
   ; CHECK:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 32
   ; CHECK:   [[MUL:%[0-9]+]]:_(s64) = G_MUL [[C]], [[COPY1]]
-  ; CHECK:   [[GEP:%[0-9]+]]:_(p0) = G_GEP [[COPY]], [[MUL]](s64)
+  ; CHECK:   [[GEP:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[MUL]](s64)
   ; CHECK:   [[COPY2:%[0-9]+]]:_(p0) = COPY [[GEP]](p0)
   ; CHECK:   $x0 = COPY [[COPY2]](p0)
   ; CHECK:   RET_ReallyLR implicit $x0
@@ -72,7 +72,7 @@ define %type* @first_offset_ext(%type* %addr, i32 %idx) {
   ; CHECK:   [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[COPY1]](s32)
   ; CHECK:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 32
   ; CHECK:   [[MUL:%[0-9]+]]:_(s64) = G_MUL [[C]], [[SEXT]]
-  ; CHECK:   [[GEP:%[0-9]+]]:_(p0) = G_GEP [[COPY]], [[MUL]](s64)
+  ; CHECK:   [[GEP:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[MUL]](s64)
   ; CHECK:   [[COPY2:%[0-9]+]]:_(p0) = COPY [[GEP]](p0)
   ; CHECK:   $x0 = COPY [[COPY2]](p0)
   ; CHECK:   RET_ReallyLR implicit $x0
@@ -89,10 +89,10 @@ define i32* @const_then_var(%type1* %addr, i64 %idx) {
   ; CHECK:   [[COPY:%[0-9]+]]:_(p0) = COPY $x0
   ; CHECK:   [[COPY1:%[0-9]+]]:_(s64) = COPY $x1
   ; CHECK:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 272
-  ; CHECK:   [[GEP:%[0-9]+]]:_(p0) = G_GEP [[COPY]], [[C]](s64)
+  ; CHECK:   [[GEP:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
   ; CHECK:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
   ; CHECK:   [[MUL:%[0-9]+]]:_(s64) = G_MUL [[C1]], [[COPY1]]
-  ; CHECK:   [[GEP1:%[0-9]+]]:_(p0) = G_GEP [[GEP]], [[MUL]](s64)
+  ; CHECK:   [[GEP1:%[0-9]+]]:_(p0) = G_PTR_ADD [[GEP]], [[MUL]](s64)
   ; CHECK:   [[COPY2:%[0-9]+]]:_(p0) = COPY [[GEP1]](p0)
   ; CHECK:   $x0 = COPY [[COPY2]](p0)
   ; CHECK:   RET_ReallyLR implicit $x0
@@ -109,9 +109,9 @@ define i32* @var_then_const(%type1* %addr, i64 %idx) {
   ; CHECK:   [[COPY1:%[0-9]+]]:_(s64) = COPY $x1
   ; CHECK:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 64
   ; CHECK:   [[MUL:%[0-9]+]]:_(s64) = G_MUL [[C]], [[COPY1]]
-  ; CHECK:   [[GEP:%[0-9]+]]:_(p0) = G_GEP [[COPY]], [[MUL]](s64)
+  ; CHECK:   [[GEP:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[MUL]](s64)
   ; CHECK:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 40
-  ; CHECK:   [[GEP1:%[0-9]+]]:_(p0) = G_GEP [[GEP]], [[C1]](s64)
+  ; CHECK:   [[GEP1:%[0-9]+]]:_(p0) = G_PTR_ADD [[GEP]], [[C1]](s64)
   ; CHECK:   $x0 = COPY [[GEP1]](p0)
   ; CHECK:   RET_ReallyLR implicit $x0
   %res = getelementptr %type1, %type1* %addr, i64 %idx, i32 2, i32 2
diff --git a/llvm/test/CodeGen/AArch64/O0-pipeline.ll b/llvm/test/CodeGen/AArch64/O0-pipeline.ll
index e8f831f61..9b7e2f6dd 100644
--- a/llvm/test/CodeGen/AArch64/O0-pipeline.ll
+++ b/llvm/test/CodeGen/AArch64/O0-pipeline.ll
@@ -11,6 +11,7 @@
 ; CHECK-NEXT: Scoped NoAlias Alias Analysis
 ; CHECK-NEXT: Assumption Cache Tracker
 ; CHECK-NEXT: Create Garbage Collector Module Metadata
+; CHECK-NEXT: Profile summary info
 ; CHECK-NEXT: Machine Branch Probability Analysis
 ; CHECK-NEXT:   ModulePass Manager
 ; CHECK-NEXT:     Pre-ISel Intrinsic Lowering
@@ -21,6 +22,7 @@
 ; CHECK-NEXT:       Module Verifier
 ; CHECK-NEXT:       Lower Garbage Collection Instructions
 ; CHECK-NEXT:       Shadow Stack GC Lowering
+; CHECK-NEXT:       Lower constant intrinsics
 ; CHECK-NEXT:       Remove unreachable blocks from the CFG
 ; CHECK-NEXT:       Instrument function entry/exit with calls to e.g. mcount() (post inlining)
 ; CHECK-NEXT:       Scalarize Masked Memory Intrinsics
@@ -35,13 +37,19 @@
 ; CHECK-NEXT:       Module Verifier
 ; CHECK-NEXT:       Analysis containing CSE Info
 ; CHECK-NEXT:       IRTranslator
+; CHECK-NEXT:       Analysis for ComputingKnownBits
 ; CHECK-NEXT:       AArch64PreLegalizerCombiner
 ; CHECK-NEXT:       Analysis containing CSE Info
 ; CHECK-NEXT:       Legalizer
 ; CHECK-NEXT:       RegBankSelect
 ; CHECK-NEXT:       Localizer
+; CHECK-NEXT:       Analysis for ComputingKnownBits
 ; CHECK-NEXT:       InstructionSelect
 ; CHECK-NEXT:       ResetMachineFunction
+; CHECK-NEXT:       Dominator Tree Construction
+; CHECK-NEXT:       Natural Loop Information
+; CHECK-NEXT:       Lazy Branch Probability Analysis
+; CHECK-NEXT:       Lazy Block Frequency Analysis
 ; CHECK-NEXT:       AArch64 Instruction Selection
 ; CHECK-NEXT:       Finalize ISel and expand pseudo-instructions
 ; CHECK-NEXT:       Local Stack Slot Allocation
@@ -55,14 +63,14 @@
 ; CHECK-NEXT:       AArch64 pseudo instruction expansion pass
 ; CHECK-NEXT:       AArch64 speculation hardening pass
 ; CHECK-NEXT:       Analyze Machine Code For Garbage Collection
-; CHECK-NEXT:       Branch relaxation pass
+; CHECK-NEXT:       Insert fentry calls
+; CHECK-NEXT:       Insert XRay ops
+; CHECK-NEXT:       Implement the 'patchable-function' attribute
 ; CHECK-NEXT:       AArch64 Branch Targets
+; CHECK-NEXT:       Branch relaxation pass
 ; CHECK-NEXT:       Contiguously Lay Out Funclets
 ; CHECK-NEXT:       StackMap Liveness Analysis
 ; CHECK-NEXT:       Live DEBUG_VALUE analysis
-; CHECK-NEXT:       Insert fentry calls
-; CHECK-NEXT:       Insert XRay ops
-; CHECK-NEXT:       Implement the 'patchable-function' attribute
 ; CHECK-NEXT:       Lazy Machine Block Frequency Analysis
 ; CHECK-NEXT:       Machine Optimization Remark Emitter
 ; CHECK-NEXT:       AArch64 Assembly Printer
diff --git a/llvm/test/CodeGen/AArch64/O3-pipeline.ll b/llvm/test/CodeGen/AArch64/O3-pipeline.ll
index 4aabcf3c7..bf489cbba 100644
--- a/llvm/test/CodeGen/AArch64/O3-pipeline.ll
+++ b/llvm/test/CodeGen/AArch64/O3-pipeline.ll
@@ -10,8 +10,8 @@
 ; CHECK-NEXT: Assumption Cache Tracker
 ; CHECK-NEXT: Type-Based Alias Analysis
 ; CHECK-NEXT: Scoped NoAlias Alias Analysis
-; CHECK-NEXT: Create Garbage Collector Module Metadata
 ; CHECK-NEXT: Profile summary info
+; CHECK-NEXT: Create Garbage Collector Module Metadata
 ; CHECK-NEXT: Machine Branch Probability Analysis
 ; CHECK-NEXT:   ModulePass Manager
 ; CHECK-NEXT:     Pre-ISel Intrinsic Lowering
@@ -35,9 +35,13 @@
 ; CHECK-NEXT:       Basic Alias Analysis (stateless AA impl)
 ; CHECK-NEXT:         Function Alias Analysis Results
 ; CHECK-NEXT:       Merge contiguous icmps into a memcmp
+; CHECK-NEXT:       Natural Loop Information
+; CHECK-NEXT:       Lazy Branch Probability Analysis
+; CHECK-NEXT:       Lazy Block Frequency Analysis
 ; CHECK-NEXT:       Expand memcmp() to load/stores
 ; CHECK-NEXT:       Lower Garbage Collection Instructions
 ; CHECK-NEXT:       Shadow Stack GC Lowering
+; CHECK-NEXT:       Lower constant intrinsics
 ; CHECK-NEXT:       Remove unreachable blocks from the CFG
 ; CHECK-NEXT:       Dominator Tree Construction
 ; CHECK-NEXT:       Natural Loop Information
@@ -55,6 +59,8 @@
 ; CHECK-NEXT:       Interleaved Load Combine Pass
 ; CHECK-NEXT:       Dominator Tree Construction
 ; CHECK-NEXT:       Interleaved Access Pass
+; CHECK-NEXT:       Basic Alias Analysis (stateless AA impl)
+; CHECK-NEXT:       Function Alias Analysis Results
 ; CHECK-NEXT:       AArch64 Stack Tagging
 ; CHECK-NEXT:       Natural Loop Information
 ; CHECK-NEXT:       CodeGen Prepare
@@ -63,7 +69,8 @@
 ; CHECK-NEXT:       Dominator Tree Construction
 ; CHECK-NEXT:       Exception handling preparation
 ; CHECK-NEXT:     AArch64 Promote Constant
-; CHECK-NEXT:       Unnamed pass: implement Pass::getPassName()
+; CHECK-NEXT:       FunctionPass Manager
+; CHECK-NEXT:         Dominator Tree Construction
 ; CHECK-NEXT:     FunctionPass Manager
 ; CHECK-NEXT:       Merge internal globals
 ; CHECK-NEXT:       Safe Stack instrumentation pass
@@ -74,10 +81,13 @@
 ; CHECK-NEXT:       Function Alias Analysis Results
 ; CHECK-NEXT:       Natural Loop Information
 ; CHECK-NEXT:       Branch Probability Analysis
+; CHECK-NEXT:       Lazy Branch Probability Analysis
+; CHECK-NEXT:       Lazy Block Frequency Analysis
 ; CHECK-NEXT:       AArch64 Instruction Selection
 ; CHECK-NEXT:       MachineDominator Tree Construction
 ; CHECK-NEXT:       AArch64 Local Dynamic TLS Access Clean-up
 ; CHECK-NEXT:       Finalize ISel and expand pseudo-instructions
+; CHECK-NEXT:       Lazy Machine Block Frequency Analysis
 ; CHECK-NEXT:       Early Tail Duplication
 ; CHECK-NEXT:       Optimize machine instruction PHIs
 ; CHECK-NEXT:       Slot index numbering
@@ -89,15 +99,18 @@
 ; CHECK-NEXT:       Machine Natural Loop Construction
 ; CHECK-NEXT:       Machine Trace Metrics
 ; CHECK-NEXT:       AArch64 Conditional Compares
+; CHECK-NEXT:       Lazy Machine Block Frequency Analysis
 ; CHECK-NEXT:       Machine InstCombiner
 ; CHECK-NEXT:       AArch64 Conditional Branch Tuning
 ; CHECK-NEXT:       Machine Trace Metrics
 ; CHECK-NEXT:       Early If-Conversion
 ; CHECK-NEXT:       AArch64 Store Pair Suppression
 ; CHECK-NEXT:       AArch64 SIMD instructions optimization pass
+; CHECK-NEXT:       AArch64 Stack Tagging PreRA
 ; CHECK-NEXT:       MachineDominator Tree Construction
 ; CHECK-NEXT:       Machine Natural Loop Construction
 ; CHECK-NEXT:       Early Machine Loop Invariant Code Motion
+; CHECK-NEXT:       MachineDominator Tree Construction
 ; CHECK-NEXT:       Machine Block Frequency Analysis
 ; CHECK-NEXT:       Machine Common Subexpression Elimination
 ; CHECK-NEXT:       MachinePostDominator Tree Construction
@@ -111,6 +124,7 @@
 ; CHECK-NEXT:       Live Variable Analysis
 ; CHECK-NEXT:       Eliminate PHI nodes for register allocation
 ; CHECK-NEXT:       Two-Address instruction pass
+; CHECK-NEXT:       MachineDominator Tree Construction
 ; CHECK-NEXT:       Slot index numbering
 ; CHECK-NEXT:       Live Interval Analysis
 ; CHECK-NEXT:       Simple Register Coalescing
@@ -142,6 +156,7 @@
 ; CHECK-NEXT:       Shrink Wrapping analysis
 ; CHECK-NEXT:       Prologue/Epilogue Insertion & Frame Finalization
 ; CHECK-NEXT:       Control Flow Optimizer
+; CHECK-NEXT:       Lazy Machine Block Frequency Analysis
 ; CHECK-NEXT:       Tail Duplication
 ; CHECK-NEXT:       Machine Copy Propagation Pass
 ; CHECK-NEXT:       Post-RA pseudo instruction expansion pass
@@ -156,16 +171,16 @@
 ; CHECK-NEXT:       Machine Block Frequency Analysis
 ; CHECK-NEXT:       MachinePostDominator Tree Construction
 ; CHECK-NEXT:       Branch Probability Basic Block Placement
+; CHECK-NEXT:       Insert fentry calls
+; CHECK-NEXT:       Insert XRay ops
+; CHECK-NEXT:       Implement the 'patchable-function' attribute
 ; CHECK-NEXT:       AArch64 load / store optimization pass
-; CHECK-NEXT:       Branch relaxation pass
 ; CHECK-NEXT:       AArch64 Branch Targets
+; CHECK-NEXT:       Branch relaxation pass
 ; CHECK-NEXT:       AArch64 Compress Jump Tables
 ; CHECK-NEXT:       Contiguously Lay Out Funclets
 ; CHECK-NEXT:       StackMap Liveness Analysis
 ; CHECK-NEXT:       Live DEBUG_VALUE analysis
-; CHECK-NEXT:       Insert fentry calls
-; CHECK-NEXT:       Insert XRay ops
-; CHECK-NEXT:       Implement the 'patchable-function' attribute
 ; CHECK-NEXT:     Machine Outliner
 ; CHECK-NEXT:     FunctionPass Manager
 ; CHECK-NEXT:       Lazy Machine Block Frequency Analysis
diff --git a/llvm/test/CodeGen/AArch64/aarch64-2014-08-11-MachineCombinerCrash.ll b/llvm/test/CodeGen/AArch64/aarch64-2014-08-11-MachineCombinerCrash.ll
index d242aab08..0b48bb628 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-2014-08-11-MachineCombinerCrash.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-2014-08-11-MachineCombinerCrash.ll
@@ -37,7 +37,7 @@ for.body:                                         ; preds = %for.body, %entry
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.value(metadata, i64, metadata, metadata) #1
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/CodeGen/AArch64/aarch64-a57-fp-load-balancing.ll b/llvm/test/CodeGen/AArch64/aarch64-a57-fp-load-balancing.ll
index 55f6c01cb..4a126113d 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-a57-fp-load-balancing.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-a57-fp-load-balancing.ll
@@ -329,7 +329,7 @@ entry:
 
 declare void @hhh(double, double)
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="true" "no-nans-fp-math"="true" "stack-protector-buffer-size"="8" "unsafe-fp-math"="true" "use-soft-float"="false" }
-attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="true" "no-nans-fp-math"="true" "stack-protector-buffer-size"="8" "unsafe-fp-math"="true" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="true" "no-nans-fp-math"="true" "stack-protector-buffer-size"="8" "unsafe-fp-math"="true" "use-soft-float"="false" }
+attributes #1 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="true" "no-nans-fp-math"="true" "stack-protector-buffer-size"="8" "unsafe-fp-math"="true" "use-soft-float"="false" }
 attributes #2 = { nounwind }
 
diff --git a/llvm/test/CodeGen/AArch64/aarch64-dynamic-stack-layout.ll b/llvm/test/CodeGen/AArch64/aarch64-dynamic-stack-layout.ll
index 09eb5fe42..d980bc095 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-dynamic-stack-layout.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-dynamic-stack-layout.ll
@@ -99,18 +99,18 @@ entry:
 ; CHECK: .cfi_startproc
 ;   Check that used callee-saved registers are saved
 ; CHECK: sub	sp, sp, #32
-; CHECK: stp	x19, x30, [sp, #16]
+; CHECK: stp	x30, x19, [sp, #16]
 ;   Check correctness of cfi pseudo-instructions
 ; CHECK: .cfi_def_cfa_offset 32
-; CHECK: .cfi_offset w30, -8
-; CHECK: .cfi_offset w19, -16
+; CHECK: .cfi_offset w19, -8
+; CHECK: .cfi_offset w30, -16
 ;   Check correct access to arguments passed on the stack, through stack pointer
 ; CHECK: ldr	d[[DARG:[0-9]+]], [sp, #56]
 ; CHECK: ldr	w[[IARG:[0-9]+]], [sp, #40]
 ;   Check correct access to local variable on the stack, through stack pointer
 ; CHECK: ldr	w[[ILOC:[0-9]+]], [sp, #12]
 ;   Check epilogue:
-; CHECK: ldp	x19, x30, [sp, #16]
+; CHECK: ldp	x30, x19, [sp, #16]
 ; CHECK: ret
 ; CHECK: .cfi_endproc
 
@@ -180,28 +180,28 @@ entry:
 ; CHECK-LABEL: novla_dynamicrealign_call
 ; CHECK: .cfi_startproc
 ;   Check that used callee-saved registers are saved
-; CHECK: str	x19, [sp, #-32]!
+; CHECK: stp	x29, x30, [sp, #-32]!
 ;   Check that the frame pointer is created:
-; CHECK: stp	x29, x30, [sp, #16]
-; CHECK: add	x29, sp, #16
+; CHECK: str	x19, [sp, #16]
+; CHECK: mov	x29, sp
 ;   Check the dynamic realignment of the stack pointer to a 128-byte boundary
 ; CHECK: sub	x9, sp, #96
 ; CHECK: and	sp, x9, #0xffffffffffffff80
 ;   Check correctness of cfi pseudo-instructions
-; CHECK: .cfi_def_cfa w29, 16
-; CHECK: .cfi_offset w30, -8
-; CHECK: .cfi_offset w29, -16
-; CHECK: .cfi_offset w19, -32
+; CHECK: .cfi_def_cfa w29, 32
+; CHECK: .cfi_offset w19, -16
+; CHECK: .cfi_offset w30, -24
+; CHECK: .cfi_offset w29, -32
 ;   Check correct access to arguments passed on the stack, through frame pointer
-; CHECK: ldr	d[[DARG:[0-9]+]], [x29, #40]
-; CHECK: ldr	w[[IARG:[0-9]+]], [x29, #24]
+; CHECK: ldr	d[[DARG:[0-9]+]], [x29, #56]
+; CHECK: ldr	w[[IARG:[0-9]+]], [x29, #40]
 ;   Check correct access to local variable on the stack, through re-aligned stack pointer
 ; CHECK: ldr	w[[ILOC:[0-9]+]], [sp]
 ;   Check epilogue:
 ;     Check that stack pointer get restored from frame pointer.
-; CHECK: sub	sp, x29, #16            // =16
-; CHECK: ldp	x29, x30, [sp, #16]
-; CHECK: ldr	x19, [sp], #32
+; CHECK: mov	sp, x29
+; CHECK: ldr	x19, [sp, #16]
+; CHECK: ldp	x29, x30, [sp], #32
 ; CHECK: ret
 ; CHECK: .cfi_endproc
 
@@ -284,22 +284,22 @@ entry:
 ; CHECK-LABEL: vla_nodynamicrealign_call
 ; CHECK: .cfi_startproc
 ;   Check that used callee-saved registers are saved
-; CHECK: stp	x20, x19, [sp, #-32]!
+; CHECK: stp	x29, x30, [sp, #-32]!
 ;   Check that the frame pointer is created:
-; CHECK: stp	x29, x30, [sp, #16]
-; CHECK: add	x29, sp, #16
+; CHECK: stp	x20, x19, [sp, #16]
+; CHECK: mov	x29, sp
 ;   Check that space is reserved on the stack for the local variable,
 ;   rounded up to a multiple of 16 to keep the stack pointer 16-byte aligned.
 ; CHECK: sub	sp, sp, #16
 ;   Check correctness of cfi pseudo-instructions
-; CHECK: .cfi_def_cfa w29, 16
-; CHECK: .cfi_offset w30, -8
-; CHECK: .cfi_offset w29, -16
-; CHECK: .cfi_offset w19, -24
-; CHECK: .cfi_offset w20, -32
+; CHECK: .cfi_def_cfa w29, 32
+; CHECK: .cfi_offset w19, -8
+; CHECK: .cfi_offset w20, -16
+; CHECK: .cfi_offset w30, -24
+; CHECK: .cfi_offset w29, -32
 ;   Check correct access to arguments passed on the stack, through frame pointer
-; CHECK: ldr	w[[IARG:[0-9]+]], [x29, #24]
-; CHECK: ldr	d[[DARG:[0-9]+]], [x29, #40]
+; CHECK: ldr	w[[IARG:[0-9]+]], [x29, #40]
+; CHECK: ldr	d[[DARG:[0-9]+]], [x29, #56]
 ;   Check correct reservation of 16-byte aligned VLA (size in w0) on stack
 ; CHECK: mov	w9, w0
 ; CHECK: mov	 x10, sp
@@ -309,14 +309,14 @@ entry:
 ; CHECK: sub	 x[[VLASPTMP:[0-9]+]], x10, x9
 ; CHECK: mov	 sp, x[[VLASPTMP]]
 ;   Check correct access to local variable, through frame pointer
-; CHECK: ldur	w[[ILOC:[0-9]+]], [x29, #-20]
+; CHECK: ldur	w[[ILOC:[0-9]+]], [x29, #-4]
 ;   Check correct accessing of the VLA variable through the base pointer
 ; CHECK: ldr	w[[VLA:[0-9]+]], [x[[VLASPTMP]]]
 ;   Check epilogue:
 ;     Check that stack pointer get restored from frame pointer.
-; CHECK: sub	sp, x29, #16            // =16
-; CHECK: ldp	x29, x30, [sp, #16]
-; CHECK: ldp	x20, x19, [sp], #32
+; CHECK: mov	sp, x29
+; CHECK: ldp	x20, x19, [sp, #16]
+; CHECK: ldp	x29, x30, [sp], #32
 ; CHECK: ret
 ; CHECK: .cfi_endproc
 
@@ -385,11 +385,11 @@ entry:
 ; CHECK-LABEL: vla_dynamicrealign_call
 ; CHECK: .cfi_startproc
 ;   Check that used callee-saved registers are saved
-; CHECK: str	x21, [sp, #-48]!
-; CHECK: stp	x20, x19, [sp, #16]
+; CHECK: stp	x29, x30, [sp, #-48]!
+; CHECK: str	x21, [sp, #16]
+; CHECK: stp	x20, x19, [sp, #32]
 ;   Check that the frame pointer is created:
-; CHECK: stp	x29, x30, [sp, #32]
-; CHECK: add	x29, sp, #32
+; CHECK: mov	x29, sp
 ;   Check that the stack pointer gets re-aligned to 128
 ;   bytes & the base pointer (x19) gets initialized to
 ;   this 128-byte aligned area for local variables &
@@ -398,15 +398,15 @@ entry:
 ; CHECK: and	sp, x9, #0xffffffffffffff80
 ; CHECK: mov    x19, sp
 ;   Check correctness of cfi pseudo-instructions
-; CHECK: .cfi_def_cfa w29, 16
-; CHECK: .cfi_offset w30, -8
-; CHECK: .cfi_offset w29, -16
-; CHECK: .cfi_offset w19, -24
-; CHECK: .cfi_offset w20, -32
-; CHECK: .cfi_offset w21, -48
+; CHECK: .cfi_def_cfa w29, 48
+; CHECK: .cfi_offset w19, -8
+; CHECK: .cfi_offset w20, -16
+; CHECK: .cfi_offset w21, -32
+; CHECK: .cfi_offset w30, -40
+; CHECK: .cfi_offset w29, -48
 ;   Check correct access to arguments passed on the stack, through frame pointer
-; CHECK: ldr	w[[IARG:[0-9]+]], [x29, #24]
-; CHECK: ldr	d[[DARG:[0-9]+]], [x29, #40]
+; CHECK: ldr	w[[IARG:[0-9]+]], [x29, #56]
+; CHECK: ldr	d[[DARG:[0-9]+]], [x29, #72]
 ;   Check correct reservation of 16-byte aligned VLA (size in w0) on stack
 ;   and set-up of base pointer (x19).
 ; CHECK: mov	w9, w0
@@ -421,10 +421,10 @@ entry:
 ; CHECK: ldr	 w[[VLA:[0-9]+]], [x[[VLASPTMP]]]
 ;   Check epilogue:
 ;     Check that stack pointer get restored from frame pointer.
-; CHECK: sub	sp, x29, #32
-; CHECK: ldp	x29, x30, [sp, #32]
-; CHECK: ldp	x20, x19, [sp, #16]
-; CHECK: ldr	x21, [sp], #48
+; CHECK: mov	sp, x29
+; CHECK: ldp	x20, x19, [sp, #32]
+; CHECK: ldr	x21, [sp, #16]
+; CHECK: ldp	x29, x30, [sp], #48
 ; CHECK: ret
 ; CHECK: .cfi_endproc
 
@@ -493,10 +493,10 @@ entry:
 
 ; CHECK-LABEL: vla_dynamicrealign_nocall
 ;   Check that used callee-saved registers are saved
-; CHECK: str	x19, [sp, #-32]!
+; CHECK: stp	x29, x30, [sp, #-32]!
+; CHECK: str	x19, [sp, #16]
 ;   Check that the frame pointer is created:
-; CHECK: stp	x29, x30, [sp, #16]
-; CHECK: add	x29, sp, #16
+; CHECK: mov	x29, sp
 ;   Check that the stack pointer gets re-aligned to 128
 ;   bytes & the base pointer (x19) gets initialized to
 ;   this 128-byte aligned area for local variables &
@@ -505,8 +505,8 @@ entry:
 ; CHECK: and	sp, x9, #0xffffffffffffff80
 ; CHECK: mov    x19, sp
 ;   Check correct access to arguments passed on the stack, through frame pointer
-; CHECK: ldr	w[[IARG:[0-9]+]], [x29, #24]
-; CHECK: ldr	d[[DARG:[0-9]+]], [x29, #40]
+; CHECK: ldr	w[[IARG:[0-9]+]], [x29, #40]
+; CHECK: ldr	d[[DARG:[0-9]+]], [x29, #56]
 ;   Check correct reservation of 16-byte aligned VLA (size in w0) on stack
 ;   and set-up of base pointer (x19).
 ; CHECK: mov	w9, w0
@@ -521,9 +521,9 @@ entry:
 ; CHECK: ldr	 w[[VLA:[0-9]+]], [x[[VLASPTMP]]]
 ;   Check epilogue:
 ;     Check that stack pointer get restored from frame pointer.
-; CHECK: sub	sp, x29, #16
-; CHECK: ldp	x29, x30, [sp, #16]
-; CHECK: ldr	x19, [sp], #32
+; CHECK: mov	sp, x29
+; CHECK: ldr	x19, [sp, #16]
+; CHECK: ldp	x29, x30, [sp], #32
 ; CHECK: ret
 
 ; CHECK-MACHO-LABEL: _vla_dynamicrealign_nocall:
@@ -579,10 +579,10 @@ entry:
 
 ; CHECK-LABEL: vla_dynamicrealign_nocall_large_align
 ;   Check that used callee-saved registers are saved
-; CHECK: stp	x28, x19, [sp, #-32]!
+; CHECK: stp	x29, x30, [sp, #-32]!
+; CHECK: str  x19, [sp, #16]
 ;   Check that the frame pointer is created:
-; CHECK: stp	x29, x30, [sp, #16]
-; CHECK: add	x29, sp, #16
+; CHECK: mov	x29, sp
 ;   Check that the stack pointer gets re-aligned to 128
 ;   bytes & the base pointer (x19) gets initialized to
 ;   this 128-byte aligned area for local variables &
@@ -591,8 +591,8 @@ entry:
 ; CHECK: and	sp, x9, #0xffffffffffff8000
 ; CHECK: mov    x19, sp
 ;   Check correct access to arguments passed on the stack, through frame pointer
-; CHECK: ldr	w[[IARG:[0-9]+]], [x29, #24]
-; CHECK: ldr	d[[DARG:[0-9]+]], [x29, #40]
+; CHECK: ldr	w[[IARG:[0-9]+]], [x29, #40]
+; CHECK: ldr	d[[DARG:[0-9]+]], [x29, #56]
 ;   Check correct reservation of 16-byte aligned VLA (size in w0) on stack
 ;   and set-up of base pointer (x19).
 ; CHECK: mov	w9, w0
@@ -607,9 +607,9 @@ entry:
 ; CHECK: ldr	 w[[VLA:[0-9]+]], [x[[VLASPTMP]]]
 ;   Check epilogue:
 ;     Check that stack pointer get restored from frame pointer.
-; CHECK: sub	sp, x29, #16
-; CHECK: ldp	x29, x30, [sp, #16]
-; CHECK: ldp	x28, x19, [sp], #32
+; CHECK: mov	sp, x29
+; CHECK: ldr  x19, [sp, #16]
+; CHECK: ldp	x29, x30, [sp], #32
 ; CHECK: ret
 
 ; CHECK-MACHO-LABEL: _vla_dynamicrealign_nocall_large_align:
diff --git a/llvm/test/CodeGen/AArch64/aarch64-fix-cortex-a53-835769.ll b/llvm/test/CodeGen/AArch64/aarch64-fix-cortex-a53-835769.ll
index eafb41268..14c163a6b 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-fix-cortex-a53-835769.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-fix-cortex-a53-835769.ll
@@ -3,19 +3,19 @@
 ; therefore, the tests are a bit fragile/reliant on instruction scheduling. The
 ; test cases have been minimized as much as possible, but still most of the test
 ; cases could break if instruction scheduling heuristics for cortex-a53 change
-; RUN: llc < %s -mcpu=cortex-a53 -aarch64-fix-cortex-a53-835769=1 -stats 2>&1 \
+; RUN: llc < %s -mcpu=cortex-a53 -aarch64-fix-cortex-a53-835769=1 -frame-pointer=non-leaf -stats 2>&1 \
 ; RUN:  | FileCheck %s
-; RUN: llc < %s -mcpu=cortex-a53 -aarch64-fix-cortex-a53-835769=0 -stats 2>&1 \
+; RUN: llc < %s -mcpu=cortex-a53 -aarch64-fix-cortex-a53-835769=0 -frame-pointer=non-leaf -stats 2>&1 \
 ; RUN:  | FileCheck %s --check-prefix CHECK-NOWORKAROUND
 ; The following run lines are just to verify whether or not this pass runs by
 ; default for given CPUs. Given the fragility of the tests, this is only run on
 ; a test case where the scheduler has not freedom at all to reschedule the
 ; instructions, so the potentially massively different scheduling heuristics
 ; will not break the test case.
-; RUN: llc < %s -mcpu=generic    | FileCheck %s --check-prefix CHECK-BASIC-PASS-DISABLED
-; RUN: llc < %s -mcpu=cortex-a53 | FileCheck %s --check-prefix CHECK-BASIC-PASS-DISABLED
-; RUN: llc < %s -mcpu=cortex-a57 | FileCheck %s --check-prefix CHECK-BASIC-PASS-DISABLED
-; RUN: llc < %s -mcpu=cyclone    | FileCheck %s --check-prefix CHECK-BASIC-PASS-DISABLED
+; RUN: llc < %s -mcpu=generic    -frame-pointer=non-leaf | FileCheck %s --check-prefix CHECK-BASIC-PASS-DISABLED
+; RUN: llc < %s -mcpu=cortex-a53 -frame-pointer=non-leaf | FileCheck %s --check-prefix CHECK-BASIC-PASS-DISABLED
+; RUN: llc < %s -mcpu=cortex-a57 -frame-pointer=non-leaf | FileCheck %s --check-prefix CHECK-BASIC-PASS-DISABLED
+; RUN: llc < %s -mcpu=cyclone    -frame-pointer=non-leaf | FileCheck %s --check-prefix CHECK-BASIC-PASS-DISABLED
 
 target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64--linux-gnu"
@@ -525,8 +525,8 @@ while.cond:
   br label %while.cond
 }
 
-attributes #0 = { nounwind readonly "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind readonly "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind }
 
 
diff --git a/llvm/test/CodeGen/AArch64/aarch64-smull.ll b/llvm/test/CodeGen/AArch64/aarch64-smull.ll
index 8922ae971..582810635 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-smull.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-smull.ll
@@ -327,6 +327,66 @@ entry:
   ret void
 }
 
+define <16 x i16> @umull2_i8(<16 x i8> %arg1, <16 x i8> %arg2) {
+; CHECK-LABEL: umull2_i8:
+; CHECK-DAG: umull2 {{v[0-9]+}}.8h, {{v[0-9]+}}.16b, {{v[0-9]+}}.16b
+; CHECK-DAG: umull {{v[0-9]+}}.8h, {{v[0-9]+}}.8b, {{v[0-9]+}}.8b
+  %arg1_ext = zext <16 x i8> %arg1 to <16 x i16>
+  %arg2_ext = zext <16 x i8> %arg2 to <16 x i16>
+  %mul = mul <16 x i16> %arg1_ext, %arg2_ext
+  ret <16 x i16> %mul
+}
+
+define <16 x i16> @smull2_i8(<16 x i8> %arg1, <16 x i8> %arg2) {
+; CHECK-LABEL: smull2_i8:
+; CHECK-DAG: smull2 {{v[0-9]+}}.8h, {{v[0-9]+}}.16b, {{v[0-9]+}}.16b
+; CHECK-DAG: smull {{v[0-9]+}}.8h, {{v[0-9]+}}.8b, {{v[0-9]+}}.8b
+  %arg1_ext = sext <16 x i8> %arg1 to <16 x i16>
+  %arg2_ext = sext <16 x i8> %arg2 to <16 x i16>
+  %mul = mul <16 x i16> %arg1_ext, %arg2_ext
+  ret <16 x i16> %mul
+}
+
+define <8 x i32> @umull2_i16(<8 x i16> %arg1, <8 x i16> %arg2) {
+; CHECK-LABEL: umull2_i16:
+; CHECK-DAG: umull2 {{v[0-9]+}}.4s, {{v[0-9]+}}.8h, {{v[0-9]+}}.8h
+; CHECK-DAG: umull {{v[0-9]+}}.4s, {{v[0-9]+}}.4h, {{v[0-9]+}}.4h
+  %arg1_ext = zext <8 x i16> %arg1 to <8 x i32>
+  %arg2_ext = zext <8 x i16> %arg2 to <8 x i32>
+  %mul = mul <8 x i32> %arg1_ext, %arg2_ext
+  ret <8 x i32> %mul
+}
+
+define <8 x i32> @smull2_i16(<8 x i16> %arg1, <8 x i16> %arg2) {
+; CHECK-LABEL: smull2_i16:
+; CHECK-DAG: smull2 {{v[0-9]+}}.4s, {{v[0-9]+}}.8h, {{v[0-9]+}}.8h
+; CHECK-DAG: smull {{v[0-9]+}}.4s, {{v[0-9]+}}.4h, {{v[0-9]+}}.4h
+  %arg1_ext = sext <8 x i16> %arg1 to <8 x i32>
+  %arg2_ext = sext <8 x i16> %arg2 to <8 x i32>
+  %mul = mul <8 x i32> %arg1_ext, %arg2_ext
+  ret <8 x i32> %mul
+}
+
+define <4 x i64> @umull2_i32(<4 x i32> %arg1, <4 x i32> %arg2) {
+; CHECK-LABEL: umull2_i32:
+; CHECK-DAG: umull2 {{v[0-9]+}}.2d, {{v[0-9]+}}.4s, {{v[0-9]+}}.4s
+; CHECK-DAG: umull {{v[0-9]+}}.2d, {{v[0-9]+}}.2s, {{v[0-9]+}}.2s
+  %arg1_ext = zext <4 x i32> %arg1 to <4 x i64>
+  %arg2_ext = zext <4 x i32> %arg2 to <4 x i64>
+  %mul = mul <4 x i64> %arg1_ext, %arg2_ext
+  ret <4 x i64> %mul
+}
+
+define <4 x i64> @smull2_i32(<4 x i32> %arg1, <4 x i32> %arg2) {
+; CHECK-LABEL: smull2_i32:
+; CHECK-DAG: smull2 {{v[0-9]+}}.2d, {{v[0-9]+}}.4s, {{v[0-9]+}}.4s
+; CHECK-DAG: smull {{v[0-9]+}}.2d, {{v[0-9]+}}.2s, {{v[0-9]+}}.2s
+  %arg1_ext = sext <4 x i32> %arg1 to <4 x i64>
+  %arg2_ext = sext <4 x i32> %arg2 to <4 x i64>
+  %mul = mul <4 x i64> %arg1_ext, %arg2_ext
+  ret <4 x i64> %mul
+}
+
 declare <16 x i8> @llvm.aarch64.neon.vld1.v16i8(i8*, i32) nounwind readonly
 
 declare void @llvm.aarch64.neon.vst1.v8i16(i8*, <8 x i16>, i32) nounwind
diff --git a/llvm/test/CodeGen/AArch64/aarch64-stp-cluster.ll b/llvm/test/CodeGen/AArch64/aarch64-stp-cluster.ll
index 6d9b7d4bb..5f75b4ef9 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-stp-cluster.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-stp-cluster.ll
@@ -3,7 +3,7 @@
 
 ; CHECK: ********** MI Scheduling **********
 ; CHECK-LABEL: stp_i64_scale:%bb.0
-; CHECK:Cluster ld/st SU(4) - SU(3)
+; CHECK:Cluster ld/st SU(3) - SU(4)
 ; CHECK:Cluster ld/st SU(2) - SU(5)
 ; CHECK:SU(4):   STRXui %1:gpr64, %0:gpr64common, 1
 ; CHECK:SU(3):   STRXui %1:gpr64, %0:gpr64common, 2
@@ -24,7 +24,7 @@ entry:
 
 ; CHECK: ********** MI Scheduling **********
 ; CHECK-LABEL: stp_i32_scale:%bb.0
-; CHECK:Cluster ld/st SU(4) - SU(3)
+; CHECK:Cluster ld/st SU(3) - SU(4)
 ; CHECK:Cluster ld/st SU(2) - SU(5)
 ; CHECK:SU(4):   STRWui %1:gpr32, %0:gpr64common, 1
 ; CHECK:SU(3):   STRWui %1:gpr32, %0:gpr64common, 2
@@ -45,12 +45,12 @@ entry:
 
 ; CHECK:********** MI Scheduling **********
 ; CHECK-LABEL:stp_i64_unscale:%bb.0 entry
-; CHECK:Cluster ld/st SU(5) - SU(2)
-; CHECK:Cluster ld/st SU(4) - SU(3)
-; CHECK:SU(5):   STURXi %1:gpr64, %0:gpr64common, -32
+; CHECK:Cluster ld/st SU(2) - SU(5)
+; CHECK:Cluster ld/st SU(3) - SU(4)
 ; CHECK:SU(2):   STURXi %1:gpr64, %0:gpr64common, -24
-; CHECK:SU(4):   STURXi %1:gpr64, %0:gpr64common, -16
 ; CHECK:SU(3):   STURXi %1:gpr64, %0:gpr64common, -8
+; CHECK:SU(4):   STURXi %1:gpr64, %0:gpr64common, -16
+; CHECK:SU(5):   STURXi %1:gpr64, %0:gpr64common, -32
 define void @stp_i64_unscale(i64* nocapture %P, i64 %v) #0 {
 entry:
   %arrayidx = getelementptr inbounds i64, i64* %P, i64 -3
@@ -66,12 +66,12 @@ entry:
 
 ; CHECK:********** MI Scheduling **********
 ; CHECK-LABEL:stp_i32_unscale:%bb.0 entry
-; CHECK:Cluster ld/st SU(5) - SU(2)
-; CHECK:Cluster ld/st SU(4) - SU(3)
-; CHECK:SU(5):   STURWi %1:gpr32, %0:gpr64common, -16
+; CHECK:Cluster ld/st SU(2) - SU(5)
+; CHECK:Cluster ld/st SU(3) - SU(4)
 ; CHECK:SU(2):   STURWi %1:gpr32, %0:gpr64common, -12
-; CHECK:SU(4):   STURWi %1:gpr32, %0:gpr64common, -8
 ; CHECK:SU(3):   STURWi %1:gpr32, %0:gpr64common, -4
+; CHECK:SU(4):   STURWi %1:gpr32, %0:gpr64common, -8
+; CHECK:SU(5):   STURWi %1:gpr32, %0:gpr64common, -16
 define void @stp_i32_unscale(i32* nocapture %P, i32 %v) #0 {
 entry:
   %arrayidx = getelementptr inbounds i32, i32* %P, i32 -3
diff --git a/llvm/test/CodeGen/AArch64/aarch64-sve-asm-negative.ll b/llvm/test/CodeGen/AArch64/aarch64-sve-asm-negative.ll
new file mode 100644
index 000000000..ad483f40e
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/aarch64-sve-asm-negative.ll
@@ -0,0 +1,12 @@
+; RUN: not llc -mtriple aarch64-none-linux-gnu -mattr=+neon -o %t.s -filetype=asm %s 2>&1 | FileCheck %s
+
+; The 'y' constraint only applies to SVE vector registers (Z0-Z7)
+; The test below ensures that we get an appropriate error should the
+; constraint be used with a Neon register.
+
+; Function Attrs: nounwind readnone
+; CHECK: error: couldn't allocate input reg for constraint 'y'
+define <4 x i32> @test_neon(<4 x i32> %in1, <4 x i32> %in2) {
+  %1 = tail call <4 x i32> asm "add $0.4s, $1.4s, $2.4s", "=w,w,y"(<4 x i32> %in1, <4 x i32> %in2)
+  ret <4 x i32> %1
+}
diff --git a/llvm/test/CodeGen/AArch64/aarch64-sve-asm.ll b/llvm/test/CodeGen/AArch64/aarch64-sve-asm.ll
new file mode 100644
index 000000000..ad1093028
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/aarch64-sve-asm.ll
@@ -0,0 +1,70 @@
+; RUN: llc < %s -mtriple aarch64-none-linux-gnu -mattr=+sve -stop-after=finalize-isel | FileCheck %s --check-prefix=CHECK
+
+target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64-none-linux-gnu"
+
+; Function Attrs: nounwind readnone
+; CHECK: [[ARG1:%[0-9]+]]:zpr = COPY $z1
+; CHECK: [[ARG2:%[0-9]+]]:zpr = COPY $z0
+; CHECK: [[ARG3:%[0-9]+]]:zpr = COPY [[ARG2]]
+; CHECK: [[ARG4:%[0-9]+]]:zpr_3b = COPY [[ARG1]]
+; CHECK: INLINEASM {{.*}} [[ARG4]]
+define <vscale x 16 x i8> @test_svadd_i8(<vscale x 16 x i8> %Zn, <vscale x 16 x i8> %Zm) {
+  %1 = tail call <vscale x 16 x i8> asm "add $0.b, $1.b, $2.b", "=w,w,y"(<vscale x 16 x i8> %Zn, <vscale x 16 x i8> %Zm)
+  ret <vscale x 16 x i8> %1
+}
+
+; Function Attrs: nounwind readnone
+; CHECK: [[ARG1:%[0-9]+]]:zpr = COPY $z1
+; CHECK: [[ARG2:%[0-9]+]]:zpr = COPY $z0
+; CHECK: [[ARG3:%[0-9]+]]:zpr = COPY [[ARG2]]
+; CHECK: [[ARG4:%[0-9]+]]:zpr_4b = COPY [[ARG1]]
+; CHECK: INLINEASM {{.*}} [[ARG4]]
+define <vscale x 2 x i64> @test_svsub_i64(<vscale x 2 x i64> %Zn, <vscale x 2 x i64> %Zm) {
+  %1 = tail call <vscale x 2 x i64> asm "sub $0.d, $1.d, $2.d", "=w,w,x"(<vscale x 2 x i64> %Zn, <vscale x 2 x i64> %Zm)
+  ret <vscale x 2 x i64> %1
+}
+
+; Function Attrs: nounwind readnone
+; CHECK: [[ARG1:%[0-9]+]]:zpr = COPY $z1
+; CHECK: [[ARG2:%[0-9]+]]:zpr = COPY $z0
+; CHECK: [[ARG3:%[0-9]+]]:zpr = COPY [[ARG2]]
+; CHECK: [[ARG4:%[0-9]+]]:zpr_3b = COPY [[ARG1]]
+; CHECK: INLINEASM {{.*}} [[ARG4]]
+define <vscale x 8 x half> @test_svfmul_f16(<vscale x 8 x half> %Zn, <vscale x 8 x half> %Zm) {
+  %1 = tail call <vscale x 8 x half> asm "fmul $0.h, $1.h, $2.h", "=w,w,y"(<vscale x 8 x half> %Zn, <vscale x 8 x half> %Zm)
+  ret <vscale x 8 x half> %1
+}
+
+; Function Attrs: nounwind readnone
+; CHECK: [[ARG1:%[0-9]+]]:zpr = COPY $z1
+; CHECK: [[ARG2:%[0-9]+]]:zpr = COPY $z0
+; CHECK: [[ARG3:%[0-9]+]]:zpr = COPY [[ARG2]]
+; CHECK: [[ARG4:%[0-9]+]]:zpr_4b = COPY [[ARG1]]
+; CHECK: INLINEASM {{.*}} [[ARG4]]
+define <vscale x 4 x float> @test_svfmul_f(<vscale x 4 x float> %Zn, <vscale x 4 x float> %Zm) {
+  %1 = tail call <vscale x 4 x float> asm "fmul $0.s, $1.s, $2.s", "=w,w,x"(<vscale x 4 x float> %Zn, <vscale x 4 x float> %Zm)
+  ret <vscale x 4 x float> %1
+}
+
+; Function Attrs: nounwind readnone
+; CHECK: [[ARG1:%[0-9]+]]:zpr = COPY $z1
+; CHECK: [[ARG2:%[0-9]+]]:zpr = COPY $z0
+; CHECK: [[ARG3:%[0-9]+]]:ppr = COPY $p0
+; CHECK: [[ARG4:%[0-9]+]]:ppr_3b = COPY [[ARG3]]
+; CHECK: INLINEASM {{.*}} [[ARG4]]
+define <vscale x 8 x half> @test_svfadd_f16(<vscale x 16 x i1> %Pg, <vscale x 8 x half> %Zn, <vscale x 8 x half> %Zm) {
+  %1 = tail call <vscale x 8 x half> asm "fadd $0.h, $1/m, $2.h, $3.h", "=w,@3Upl,w,w"(<vscale x 16 x i1> %Pg, <vscale x 8 x half> %Zn, <vscale x 8 x half> %Zm)
+  ret <vscale x 8 x half> %1
+}
+
+; Function Attrs: nounwind readnone
+; CHECK: [[ARG1:%[0-9]+]]:zpr = COPY $z0
+; CHECK: [[ARG2:%[0-9]+]]:ppr = COPY $p0
+; CHECK: [[ARG3:%[0-9]+]]:ppr = COPY [[ARG2]]
+; CHECK: [[ARG4:%[0-9]+]]:zpr = COPY [[ARG1]]
+; CHECK: INLINEASM {{.*}} [[ARG3]]
+define <vscale x 4 x i32> @test_incp(<vscale x 16 x i1> %Pg, <vscale x 4 x i32> %Zn) {
+  %1 = tail call <vscale x 4 x i32> asm "incp $0.s, $1", "=w,@3Upa,0"(<vscale x 16 x i1> %Pg, <vscale x 4 x i32> %Zn)
+  ret <vscale x 4 x i32> %1
+}
diff --git a/llvm/test/CodeGen/AArch64/addsub-constant-folding.ll b/llvm/test/CodeGen/AArch64/addsub-constant-folding.ll
index bb3b2ebe8..43c98e818 100644
--- a/llvm/test/CodeGen/AArch64/addsub-constant-folding.ll
+++ b/llvm/test/CodeGen/AArch64/addsub-constant-folding.ll
@@ -19,15 +19,15 @@ define i32 @add_const_add_const(i32 %arg) {
 define i32 @add_const_add_const_extrause(i32 %arg) {
 ; CHECK-LABEL: add_const_add_const_extrause:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    stp x19, x30, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #-16]! // 16-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-NEXT:    .cfi_offset w30, -8
-; CHECK-NEXT:    .cfi_offset w19, -16
+; CHECK-NEXT:    .cfi_offset w19, -8
+; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    mov w19, w0
 ; CHECK-NEXT:    add w0, w0, #8 // =8
 ; CHECK-NEXT:    bl use
 ; CHECK-NEXT:    add w0, w19, #10 // =10
-; CHECK-NEXT:    ldp x19, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
 ; CHECK-NEXT:    ret
   %t0 = add i32 %arg, 8
   call void @use(i32 %t0)
@@ -96,15 +96,15 @@ define i32 @add_const_sub_const(i32 %arg) {
 define i32 @add_const_sub_const_extrause(i32 %arg) {
 ; CHECK-LABEL: add_const_sub_const_extrause:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    stp x19, x30, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #-16]! // 16-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-NEXT:    .cfi_offset w30, -8
-; CHECK-NEXT:    .cfi_offset w19, -16
+; CHECK-NEXT:    .cfi_offset w19, -8
+; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    mov w19, w0
 ; CHECK-NEXT:    add w0, w0, #8 // =8
 ; CHECK-NEXT:    bl use
 ; CHECK-NEXT:    add w0, w19, #6 // =6
-; CHECK-NEXT:    ldp x19, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
 ; CHECK-NEXT:    ret
   %t0 = add i32 %arg, 8
   call void @use(i32 %t0)
@@ -174,16 +174,16 @@ define i32 @add_const_const_sub(i32 %arg) {
 define i32 @add_const_const_sub_extrause(i32 %arg) {
 ; CHECK-LABEL: add_const_const_sub_extrause:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    stp x19, x30, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #-16]! // 16-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-NEXT:    .cfi_offset w30, -8
-; CHECK-NEXT:    .cfi_offset w19, -16
+; CHECK-NEXT:    .cfi_offset w19, -8
+; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    mov w19, w0
 ; CHECK-NEXT:    add w0, w0, #8 // =8
 ; CHECK-NEXT:    bl use
 ; CHECK-NEXT:    mov w8, #-6
 ; CHECK-NEXT:    sub w0, w8, w19
-; CHECK-NEXT:    ldp x19, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
 ; CHECK-NEXT:    ret
   %t0 = add i32 %arg, 8
   call void @use(i32 %t0)
@@ -252,15 +252,15 @@ define i32 @sub_const_add_const(i32 %arg) {
 define i32 @sub_const_add_const_extrause(i32 %arg) {
 ; CHECK-LABEL: sub_const_add_const_extrause:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    stp x19, x30, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #-16]! // 16-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-NEXT:    .cfi_offset w30, -8
-; CHECK-NEXT:    .cfi_offset w19, -16
+; CHECK-NEXT:    .cfi_offset w19, -8
+; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    mov w19, w0
 ; CHECK-NEXT:    sub w0, w0, #8 // =8
 ; CHECK-NEXT:    bl use
 ; CHECK-NEXT:    sub w0, w19, #6 // =6
-; CHECK-NEXT:    ldp x19, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
 ; CHECK-NEXT:    ret
   %t0 = sub i32 %arg, 8
   call void @use(i32 %t0)
@@ -329,15 +329,15 @@ define i32 @sub_const_sub_const(i32 %arg) {
 define i32 @sub_const_sub_const_extrause(i32 %arg) {
 ; CHECK-LABEL: sub_const_sub_const_extrause:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    stp x19, x30, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #-16]! // 16-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-NEXT:    .cfi_offset w30, -8
-; CHECK-NEXT:    .cfi_offset w19, -16
+; CHECK-NEXT:    .cfi_offset w19, -8
+; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    mov w19, w0
 ; CHECK-NEXT:    sub w0, w0, #8 // =8
 ; CHECK-NEXT:    bl use
 ; CHECK-NEXT:    sub w0, w19, #10 // =10
-; CHECK-NEXT:    ldp x19, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
 ; CHECK-NEXT:    ret
   %t0 = sub i32 %arg, 8
   call void @use(i32 %t0)
@@ -407,16 +407,16 @@ define i32 @sub_const_const_sub(i32 %arg) {
 define i32 @sub_const_const_sub_extrause(i32 %arg) {
 ; CHECK-LABEL: sub_const_const_sub_extrause:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    stp x19, x30, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #-16]! // 16-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-NEXT:    .cfi_offset w30, -8
-; CHECK-NEXT:    .cfi_offset w19, -16
+; CHECK-NEXT:    .cfi_offset w19, -8
+; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    mov w19, w0
 ; CHECK-NEXT:    sub w0, w0, #8 // =8
 ; CHECK-NEXT:    bl use
 ; CHECK-NEXT:    mov w8, #10
 ; CHECK-NEXT:    sub w0, w8, w19
-; CHECK-NEXT:    ldp x19, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
 ; CHECK-NEXT:    ret
   %t0 = sub i32 %arg, 8
   call void @use(i32 %t0)
@@ -486,17 +486,17 @@ define i32 @const_sub_add_const(i32 %arg) {
 define i32 @const_sub_add_const_extrause(i32 %arg) {
 ; CHECK-LABEL: const_sub_add_const_extrause:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    stp x19, x30, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #-16]! // 16-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-NEXT:    .cfi_offset w30, -8
-; CHECK-NEXT:    .cfi_offset w19, -16
+; CHECK-NEXT:    .cfi_offset w19, -8
+; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    mov w8, #8
 ; CHECK-NEXT:    mov w19, w0
 ; CHECK-NEXT:    sub w0, w8, w0
 ; CHECK-NEXT:    bl use
 ; CHECK-NEXT:    mov w8, #10
 ; CHECK-NEXT:    sub w0, w8, w19
-; CHECK-NEXT:    ldp x19, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
 ; CHECK-NEXT:    ret
   %t0 = sub i32 8, %arg
   call void @use(i32 %t0)
@@ -566,17 +566,17 @@ define i32 @const_sub_sub_const(i32 %arg) {
 define i32 @const_sub_sub_const_extrause(i32 %arg) {
 ; CHECK-LABEL: const_sub_sub_const_extrause:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    stp x19, x30, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #-16]! // 16-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-NEXT:    .cfi_offset w30, -8
-; CHECK-NEXT:    .cfi_offset w19, -16
+; CHECK-NEXT:    .cfi_offset w19, -8
+; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    mov w8, #8
 ; CHECK-NEXT:    mov w19, w0
 ; CHECK-NEXT:    sub w0, w8, w0
 ; CHECK-NEXT:    bl use
 ; CHECK-NEXT:    mov w8, #6
 ; CHECK-NEXT:    sub w0, w8, w19
-; CHECK-NEXT:    ldp x19, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
 ; CHECK-NEXT:    ret
   %t0 = sub i32 8, %arg
   call void @use(i32 %t0)
@@ -645,17 +645,17 @@ define i32 @const_sub_const_sub(i32 %arg) {
 define i32 @const_sub_const_sub_extrause(i32 %arg) {
 ; CHECK-LABEL: const_sub_const_sub_extrause:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    stp x19, x30, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #-16]! // 16-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-NEXT:    .cfi_offset w30, -8
-; CHECK-NEXT:    .cfi_offset w19, -16
+; CHECK-NEXT:    .cfi_offset w19, -8
+; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    mov w8, #8
 ; CHECK-NEXT:    sub w19, w8, w0
 ; CHECK-NEXT:    mov w0, w19
 ; CHECK-NEXT:    bl use
 ; CHECK-NEXT:    mov w8, #2
 ; CHECK-NEXT:    sub w0, w8, w19
-; CHECK-NEXT:    ldp x19, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
 ; CHECK-NEXT:    ret
   %t0 = sub i32 8, %arg
   call void @use(i32 %t0)
diff --git a/llvm/test/CodeGen/AArch64/addsub-shifted.ll b/llvm/test/CodeGen/AArch64/addsub-shifted.ll
index 7c7d65459..c00340249 100644
--- a/llvm/test/CodeGen/AArch64/addsub-shifted.ll
+++ b/llvm/test/CodeGen/AArch64/addsub-shifted.ll
@@ -1,28 +1,35 @@
 ; RUN: llc -verify-machineinstrs %s -o - -mtriple=arm64-apple-ios7.0 | FileCheck %s
+; RUN: llc -verify-machineinstrs %s -o - -mtriple=arm64-apple-ios7.0 -global-isel -pass-remarks-missed=gisel* 2>&1 | FileCheck %s --check-prefixes=GISEL,FALLBACK
+
+; FALLBACK-NOT: remark
 
 @var32 = global i32 0
 @var64 = global i64 0
 
 define void @test_lsl_arith(i32 %lhs32, i32 %rhs32, i64 %lhs64, i64 %rhs64) {
 ; CHECK-LABEL: test_lsl_arith:
+; GISEL-LABEL: test_lsl_arith:
 
   %rhs1 = load volatile i32, i32* @var32
   %shift1 = shl i32 %rhs1, 18
   %val1 = add i32 %lhs32, %shift1
   store volatile i32 %val1, i32* @var32
 ; CHECK: add {{w[0-9]+}}, {{w[0-9]+}}, {{w[0-9]+}}, lsl #18
+; GISEL: add {{w[0-9]+}}, {{w[0-9]+}}, {{w[0-9]+}}, lsl #18
 
   %rhs2 = load volatile i32, i32* @var32
   %shift2 = shl i32 %rhs2, 31
   %val2 = add i32 %shift2, %lhs32
   store volatile i32 %val2, i32* @var32
 ; CHECK: add {{w[0-9]+}}, {{w[0-9]+}}, {{w[0-9]+}}, lsl #31
+; GISEL: add {{w[0-9]+}}, {{w[0-9]+}}, {{w[0-9]+}}, lsl #31
 
   %rhs3 = load volatile i32, i32* @var32
   %shift3 = shl i32 %rhs3, 5
   %val3 = sub i32 %lhs32, %shift3
   store volatile i32 %val3, i32* @var32
 ; CHECK: sub {{w[0-9]+}}, {{w[0-9]+}}, {{w[0-9]+}}, lsl #5
+; GISEL: subs {{w[0-9]+}}, {{w[0-9]+}}, {{w[0-9]+}}, lsl #5
 
 ; Subtraction is not commutative!
   %rhs4 = load volatile i32, i32* @var32
@@ -30,30 +37,35 @@ define void @test_lsl_arith(i32 %lhs32, i32 %rhs32, i64 %lhs64, i64 %rhs64) {
   %val4 = sub i32 %shift4, %lhs32
   store volatile i32 %val4, i32* @var32
 ; CHECK-NOT: sub {{w[0-9]+}}, {{w[0-9]+}}, {{w[0-9]+}}, lsl #19
+; GISEL-NOT: sub{{[s]?}} {{w[0-9]+}}, {{w[0-9]+}}, {{w[0-9]+}}, lsl #19
 
   %lhs4a = load volatile i32, i32* @var32
   %shift4a = shl i32 %lhs4a, 15
   %val4a = sub i32 0, %shift4a
   store volatile i32 %val4a, i32* @var32
 ; CHECK: neg {{w[0-9]+}}, {{w[0-9]+}}, lsl #15
+; GISEL: negs {{w[0-9]+}}, {{w[0-9]+}}, lsl #15
 
   %rhs5 = load volatile i64, i64* @var64
   %shift5 = shl i64 %rhs5, 18
   %val5 = add i64 %lhs64, %shift5
   store volatile i64 %val5, i64* @var64
 ; CHECK: add {{x[0-9]+}}, {{x[0-9]+}}, {{x[0-9]+}}, lsl #18
+; GISEL: add {{x[0-9]+}}, {{x[0-9]+}}, {{x[0-9]+}}, lsl #18
 
   %rhs6 = load volatile i64, i64* @var64
   %shift6 = shl i64 %rhs6, 31
   %val6 = add i64 %shift6, %lhs64
   store volatile i64 %val6, i64* @var64
 ; CHECK: add {{x[0-9]+}}, {{x[0-9]+}}, {{x[0-9]+}}, lsl #31
+; GISEL: add {{x[0-9]+}}, {{x[0-9]+}}, {{x[0-9]+}}, lsl #31
 
   %rhs7 = load volatile i64, i64* @var64
   %shift7 = shl i64 %rhs7, 5
   %val7 = sub i64 %lhs64, %shift7
   store volatile i64 %val7, i64* @var64
 ; CHECK: sub {{x[0-9]+}}, {{x[0-9]+}}, {{x[0-9]+}}, lsl #5
+; GISEL: subs {{x[0-9]+}}, {{x[0-9]+}}, {{x[0-9]+}}, lsl #5
 
 ; Subtraction is not commutative!
   %rhs8 = load volatile i64, i64* @var64
@@ -61,12 +73,14 @@ define void @test_lsl_arith(i32 %lhs32, i32 %rhs32, i64 %lhs64, i64 %rhs64) {
   %val8 = sub i64 %shift8, %lhs64
   store volatile i64 %val8, i64* @var64
 ; CHECK-NOT: sub {{x[0-9]+}}, {{x[0-9]+}}, {{x[0-9]+}}, lsl #19
+; GISEL-NOT: sub{{[s]?}} {{x[0-9]+}}, {{x[0-9]+}}, {{x[0-9]+}}, lsl #19
 
   %lhs8a = load volatile i64, i64* @var64
   %shift8a = shl i64 %lhs8a, 60
   %val8a = sub i64 0, %shift8a
   store volatile i64 %val8a, i64* @var64
 ; CHECK: neg {{x[0-9]+}}, {{x[0-9]+}}, lsl #60
+; GISEL: negs {{x[0-9]+}}, {{x[0-9]+}}, lsl #60
 
   ret void
 ; CHECK: ret
@@ -79,56 +93,67 @@ define void @test_lsr_arith(i32 %lhs32, i32 %rhs32, i64 %lhs64, i64 %rhs64) {
   %val1 = add i32 %lhs32, %shift1
   store volatile i32 %val1, i32* @var32
 ; CHECK: add {{w[0-9]+}}, {{w[0-9]+}}, {{w[0-9]+}}, lsr #18
+; GISEL: add {{w[0-9]+}}, {{w[0-9]+}}, {{w[0-9]+}}, lsr #18
 
   %shift2 = lshr i32 %rhs32, 31
   %val2 = add i32 %shift2, %lhs32
   store volatile i32 %val2, i32* @var32
 ; CHECK: add {{w[0-9]+}}, {{w[0-9]+}}, {{w[0-9]+}}, lsr #31
+; GISEL: add {{w[0-9]+}}, {{w[0-9]+}}, {{w[0-9]+}}, lsr #31
 
   %shift3 = lshr i32 %rhs32, 5
   %val3 = sub i32 %lhs32, %shift3
   store volatile i32 %val3, i32* @var32
 ; CHECK: sub {{w[0-9]+}}, {{w[0-9]+}}, {{w[0-9]+}}, lsr #5
+; GISEL: subs {{w[0-9]+}}, {{w[0-9]+}}, {{w[0-9]+}}, lsr #5
 
 ; Subtraction is not commutative!
   %shift4 = lshr i32 %rhs32, 19
   %val4 = sub i32 %shift4, %lhs32
   store volatile i32 %val4, i32* @var32
 ; CHECK-NOT: sub {{w[0-9]+}}, {{w[0-9]+}}, {{w[0-9]+}}, lsr #19
+; GISEL-NOT: sub{{[s]?}} {{w[0-9]+}}, {{w[0-9]+}}, {{w[0-9]+}}, lsr #19
 
   %shift4a = lshr i32 %lhs32, 15
   %val4a = sub i32 0, %shift4a
   store volatile i32 %val4a, i32* @var32
 ; CHECK: neg {{w[0-9]+}}, {{w[0-9]+}}, lsr #15
+; GISEL: negs {{w[0-9]+}}, {{w[0-9]+}}, lsr #15
 
   %shift5 = lshr i64 %rhs64, 18
   %val5 = add i64 %lhs64, %shift5
   store volatile i64 %val5, i64* @var64
 ; CHECK: add {{x[0-9]+}}, {{x[0-9]+}}, {{x[0-9]+}}, lsr #18
+; GISEL: add {{x[0-9]+}}, {{x[0-9]+}}, {{x[0-9]+}}, lsr #18
 
   %shift6 = lshr i64 %rhs64, 31
   %val6 = add i64 %shift6, %lhs64
   store volatile i64 %val6, i64* @var64
 ; CHECK: add {{x[0-9]+}}, {{x[0-9]+}}, {{x[0-9]+}}, lsr #31
+; GISEL: add {{x[0-9]+}}, {{x[0-9]+}}, {{x[0-9]+}}, lsr #31
 
   %shift7 = lshr i64 %rhs64, 5
   %val7 = sub i64 %lhs64, %shift7
   store volatile i64 %val7, i64* @var64
 ; CHECK: sub {{x[0-9]+}}, {{x[0-9]+}}, {{x[0-9]+}}, lsr #5
+; GISEL: subs {{x[0-9]+}}, {{x[0-9]+}}, {{x[0-9]+}}, lsr #5
 
 ; Subtraction is not commutative!
   %shift8 = lshr i64 %rhs64, 19
   %val8 = sub i64 %shift8, %lhs64
   store volatile i64 %val8, i64* @var64
 ; CHECK-NOT: sub {{x[0-9]+}}, {{x[0-9]+}}, {{x[0-9]+}}, lsr #19
+; GISEL-NOT: sub{{[s]?}} {{x[0-9]+}}, {{x[0-9]+}}, {{x[0-9]+}}, lsr #19
 
   %shift8a = lshr i64 %lhs64, 45
   %val8a = sub i64 0, %shift8a
   store volatile i64 %val8a, i64* @var64
 ; CHECK: neg {{x[0-9]+}}, {{x[0-9]+}}, lsr #45
+; GISEL: negs {{x[0-9]+}}, {{x[0-9]+}}, lsr #45
 
   ret void
 ; CHECK: ret
+; GISEL: ret
 }
 
 define void @test_asr_arith(i32 %lhs32, i32 %rhs32, i64 %lhs64, i64 %rhs64) {
@@ -138,53 +163,63 @@ define void @test_asr_arith(i32 %lhs32, i32 %rhs32, i64 %lhs64, i64 %rhs64) {
   %val1 = add i32 %lhs32, %shift1
   store volatile i32 %val1, i32* @var32
 ; CHECK: add {{w[0-9]+}}, {{w[0-9]+}}, {{w[0-9]+}}, asr #18
+; GISEL: add {{w[0-9]+}}, {{w[0-9]+}}, {{w[0-9]+}}, asr #18
 
   %shift2 = ashr i32 %rhs32, 31
   %val2 = add i32 %shift2, %lhs32
   store volatile i32 %val2, i32* @var32
 ; CHECK: add {{w[0-9]+}}, {{w[0-9]+}}, {{w[0-9]+}}, asr #31
+; GISEL: add {{w[0-9]+}}, {{w[0-9]+}}, {{w[0-9]+}}, asr #31
 
   %shift3 = ashr i32 %rhs32, 5
   %val3 = sub i32 %lhs32, %shift3
   store volatile i32 %val3, i32* @var32
 ; CHECK: sub {{w[0-9]+}}, {{w[0-9]+}}, {{w[0-9]+}}, asr #5
+; GISEL: subs {{w[0-9]+}}, {{w[0-9]+}}, {{w[0-9]+}}, asr #5
 
 ; Subtraction is not commutative!
   %shift4 = ashr i32 %rhs32, 19
   %val4 = sub i32 %shift4, %lhs32
   store volatile i32 %val4, i32* @var32
 ; CHECK-NOT: sub {{w[0-9]+}}, {{w[0-9]+}}, {{w[0-9]+}}, asr #19
+; GISEL-NOT: sub{{[s]?}}  {{w[0-9]+}}, {{w[0-9]+}}, {{w[0-9]+}}, asr #19
 
   %shift4a = ashr i32 %lhs32, 15
   %val4a = sub i32 0, %shift4a
   store volatile i32 %val4a, i32* @var32
 ; CHECK: neg {{w[0-9]+}}, {{w[0-9]+}}, asr #15
+; GISEL: negs {{w[0-9]+}}, {{w[0-9]+}}, asr #15
 
   %shift5 = ashr i64 %rhs64, 18
   %val5 = add i64 %lhs64, %shift5
   store volatile i64 %val5, i64* @var64
 ; CHECK: add {{x[0-9]+}}, {{x[0-9]+}}, {{x[0-9]+}}, asr #18
+; GISEL: add {{x[0-9]+}}, {{x[0-9]+}}, {{x[0-9]+}}, asr #18
 
   %shift6 = ashr i64 %rhs64, 31
   %val6 = add i64 %shift6, %lhs64
   store volatile i64 %val6, i64* @var64
 ; CHECK: add {{x[0-9]+}}, {{x[0-9]+}}, {{x[0-9]+}}, asr #31
+; GISEL: add {{x[0-9]+}}, {{x[0-9]+}}, {{x[0-9]+}}, asr #31
 
   %shift7 = ashr i64 %rhs64, 5
   %val7 = sub i64 %lhs64, %shift7
   store volatile i64 %val7, i64* @var64
 ; CHECK: sub {{x[0-9]+}}, {{x[0-9]+}}, {{x[0-9]+}}, asr #5
+; GISEL: subs {{x[0-9]+}}, {{x[0-9]+}}, {{x[0-9]+}}, asr #5
 
 ; Subtraction is not commutative!
   %shift8 = ashr i64 %rhs64, 19
   %val8 = sub i64 %shift8, %lhs64
   store volatile i64 %val8, i64* @var64
 ; CHECK-NOT: sub {{x[0-9]+}}, {{x[0-9]+}}, {{x[0-9]+}}, asr #19
+; GISEL-NOT: sub{{[s]?}}  {{x[0-9]+}}, {{x[0-9]+}}, {{x[0-9]+}}, asr #19
 
   %shift8a = ashr i64 %lhs64, 45
   %val8a = sub i64 0, %shift8a
   store volatile i64 %val8a, i64* @var64
 ; CHECK: neg {{x[0-9]+}}, {{x[0-9]+}}, asr #45
+; GISEL: negs {{x[0-9]+}}, {{x[0-9]+}}, asr #45
 
   ret void
 ; CHECK: ret
@@ -253,6 +288,8 @@ define i32 @test_cmn(i32 %lhs32, i32 %rhs32, i64 %lhs64, i64 %rhs64) {
   ; 0 then the results will differ.
 ; CHECK: neg [[RHS:w[0-9]+]], {{w[0-9]+}}, lsl #13
 ; CHECK: cmp {{w[0-9]+}}, [[RHS]]
+; GISEL: negs [[RHS:w[0-9]+]], {{w[0-9]+}}, lsl #13
+; GISEL: cmp {{w[0-9]+}}, [[RHS]]
 
 t2:
   %shift2 = lshr i32 %rhs32, 20
@@ -276,6 +313,8 @@ t4:
   ; Again, it's important that cmn isn't used here in case %rhs64 == 0.
 ; CHECK: neg [[RHS:x[0-9]+]], {{x[0-9]+}}, lsl #43
 ; CHECK: cmp {{x[0-9]+}}, [[RHS]]
+; GISEL: negs [[RHS:x[0-9]+]], {{x[0-9]+}}, lsl #43
+; GISEL: cmp {{x[0-9]+}}, [[RHS]]
 
 t5:
   %shift5 = lshr i64 %rhs64, 20
@@ -297,5 +336,5 @@ end:
 
   ret i32 0
 ; CHECK: ret
+; GISEL: ret
 }
-
diff --git a/llvm/test/CodeGen/AArch64/addsub_ext.ll b/llvm/test/CodeGen/AArch64/addsub_ext.ll
index df1b9fe78..573ce9b6f 100644
--- a/llvm/test/CodeGen/AArch64/addsub_ext.ll
+++ b/llvm/test/CodeGen/AArch64/addsub_ext.ll
@@ -1,4 +1,9 @@
-; RUN: llc -verify-machineinstrs %s -o - -mtriple=aarch64-linux-gnu -aarch64-enable-atomic-cfg-tidy=0 | FileCheck %s
+; RUN: llc -enable-machine-outliner=never -verify-machineinstrs %s -o - -mtriple=aarch64-linux-gnu -aarch64-enable-atomic-cfg-tidy=0 | FileCheck %s
+; RUN: llc -global-isel -enable-machine-outliner=never -verify-machineinstrs %s -o - -mtriple=aarch64-linux-gnu -aarch64-enable-atomic-cfg-tidy=0 | FileCheck %s --check-prefix=GISEL
+
+; FIXME: GISel only knows how to handle explicit G_SEXT instructions. So when
+; G_SEXT is lowered to anything else, it won't fold in a stx*.
+; FIXME: GISel doesn't currently handle folding the addressing mode into a cmp.
 
 @var8 = global i8 0
 @var16 = global i16 0
@@ -7,6 +12,7 @@
 
 define void @addsub_i8rhs() minsize {
 ; CHECK-LABEL: addsub_i8rhs:
+; GISEL-LABEL: addsub_i8rhs:
     %val8_tmp = load i8, i8* @var8
     %lhs32 = load i32, i32* @var32
     %lhs64 = load i64, i64* @var64
@@ -20,23 +26,26 @@ define void @addsub_i8rhs() minsize {
     %res32_zext = add i32 %lhs32, %rhs32_zext
     store volatile i32 %res32_zext, i32* @var32
 ; CHECK: add {{w[0-9]+}}, {{w[0-9]+}}, {{w[0-9]+}}, uxtb
+; GISEL: add {{w[0-9]+}}, {{w[0-9]+}}, {{w[0-9]+}}, uxtb
 
    %rhs32_zext_shift = shl i32 %rhs32_zext, 3
    %res32_zext_shift = add i32 %lhs32, %rhs32_zext_shift
    store volatile i32 %res32_zext_shift, i32* @var32
 ; CHECK: add {{w[0-9]+}}, {{w[0-9]+}}, {{w[0-9]+}}, uxtb #3
-
+; GISEL: add {{w[0-9]+}}, {{w[0-9]+}}, {{w[0-9]+}}, uxtb #3
 
 ; Zero-extending to 64-bits
     %rhs64_zext = zext i8 %val8 to i64
     %res64_zext = add i64 %lhs64, %rhs64_zext
     store volatile i64 %res64_zext, i64* @var64
 ; CHECK: add {{x[0-9]+}}, {{x[0-9]+}}, {{w[0-9]+}}, uxtb
+; GISEL: add {{x[0-9]+}}, {{x[0-9]+}}, {{w[0-9]+}}, uxtb
 
    %rhs64_zext_shift = shl i64 %rhs64_zext, 1
    %res64_zext_shift = add i64 %lhs64, %rhs64_zext_shift
    store volatile i64 %res64_zext_shift, i64* @var64
 ; CHECK: add {{x[0-9]+}}, {{x[0-9]+}}, {{w[0-9]+}}, uxtb #1
+; GISEL: add {{x[0-9]+}}, {{x[0-9]+}}, {{w[0-9]+}}, uxtb #1
 
 ; Sign-extending to 32-bits
     %rhs32_sext = sext i8 %val8 to i32
@@ -95,23 +104,26 @@ define void @sub_i8rhs() minsize {
     %res32_zext = sub i32 %lhs32, %rhs32_zext
     store volatile i32 %res32_zext, i32* @var32
 ; CHECK: sub {{w[0-9]+}}, {{w[0-9]+}}, {{w[0-9]+}}, uxtb
+; GISEL: subs {{w[0-9]+}}, {{w[0-9]+}}, {{w[0-9]+}}, uxtb
 
    %rhs32_zext_shift = shl i32 %rhs32_zext, 3
    %res32_zext_shift = sub i32 %lhs32, %rhs32_zext_shift
    store volatile i32 %res32_zext_shift, i32* @var32
 ; CHECK: sub {{w[0-9]+}}, {{w[0-9]+}}, {{w[0-9]+}}, uxtb #3
-
+; GISEL: subs {{w[0-9]+}}, {{w[0-9]+}}, {{w[0-9]+}}, uxtb #3
 
 ; Zero-extending to 64-bits
     %rhs64_zext = zext i8 %val8 to i64
     %res64_zext = sub i64 %lhs64, %rhs64_zext
     store volatile i64 %res64_zext, i64* @var64
 ; CHECK: sub {{x[0-9]+}}, {{x[0-9]+}}, {{w[0-9]+}}, uxtb
+; GISEL: subs {{x[0-9]+}}, {{x[0-9]+}}, {{w[0-9]+}}, uxtb
 
    %rhs64_zext_shift = shl i64 %rhs64_zext, 1
    %res64_zext_shift = sub i64 %lhs64, %rhs64_zext_shift
    store volatile i64 %res64_zext_shift, i64* @var64
 ; CHECK: sub {{x[0-9]+}}, {{x[0-9]+}}, {{w[0-9]+}}, uxtb #1
+; GISEL: subs {{x[0-9]+}}, {{x[0-9]+}}, {{w[0-9]+}}, uxtb #1
 
 ; Sign-extending to 32-bits
     %rhs32_sext = sext i8 %val8 to i32
@@ -140,6 +152,7 @@ define void @sub_i8rhs() minsize {
 
 define void @addsub_i16rhs() minsize {
 ; CHECK-LABEL: addsub_i16rhs:
+; GISEL-LABEL: addsub_i16rhs:
     %val16_tmp = load i16, i16* @var16
     %lhs32 = load i32, i32* @var32
     %lhs64 = load i64, i64* @var64
@@ -153,23 +166,26 @@ define void @addsub_i16rhs() minsize {
     %res32_zext = add i32 %lhs32, %rhs32_zext
     store volatile i32 %res32_zext, i32* @var32
 ; CHECK: add {{w[0-9]+}}, {{w[0-9]+}}, {{w[0-9]+}}, uxth
+; GISEL: add {{w[0-9]+}}, {{w[0-9]+}}, {{w[0-9]+}}, uxth
 
    %rhs32_zext_shift = shl i32 %rhs32_zext, 3
    %res32_zext_shift = add i32 %lhs32, %rhs32_zext_shift
    store volatile i32 %res32_zext_shift, i32* @var32
 ; CHECK: add {{w[0-9]+}}, {{w[0-9]+}}, {{w[0-9]+}}, uxth #3
-
+; GISEL: add {{w[0-9]+}}, {{w[0-9]+}}, {{w[0-9]+}}, uxth #3
 
 ; Zero-extending to 64-bits
     %rhs64_zext = zext i16 %val16 to i64
     %res64_zext = add i64 %lhs64, %rhs64_zext
     store volatile i64 %res64_zext, i64* @var64
 ; CHECK: add {{x[0-9]+}}, {{x[0-9]+}}, {{w[0-9]+}}, uxth
+; GISEL: add {{x[0-9]+}}, {{x[0-9]+}}, {{w[0-9]+}}, uxth
 
    %rhs64_zext_shift = shl i64 %rhs64_zext, 1
    %res64_zext_shift = add i64 %lhs64, %rhs64_zext_shift
    store volatile i64 %res64_zext_shift, i64* @var64
 ; CHECK: add {{x[0-9]+}}, {{x[0-9]+}}, {{w[0-9]+}}, uxth #1
+; GISEL: add {{x[0-9]+}}, {{x[0-9]+}}, {{w[0-9]+}}, uxth #1
 
 ; Sign-extending to 32-bits
     %rhs32_sext = sext i16 %val16 to i32
@@ -215,6 +231,7 @@ end:
 
 define void @sub_i16rhs() minsize {
 ; CHECK-LABEL: sub_i16rhs:
+; GISEL-LABEL: sub_i16rhs:
     %val16_tmp = load i16, i16* @var16
     %lhs32 = load i32, i32* @var32
     %lhs64 = load i64, i64* @var64
@@ -228,23 +245,26 @@ define void @sub_i16rhs() minsize {
     %res32_zext = sub i32 %lhs32, %rhs32_zext
     store volatile i32 %res32_zext, i32* @var32
 ; CHECK: sub {{w[0-9]+}}, {{w[0-9]+}}, {{w[0-9]+}}, uxth
+; GISEL: subs {{w[0-9]+}}, {{w[0-9]+}}, {{w[0-9]+}}, uxth
 
    %rhs32_zext_shift = shl i32 %rhs32_zext, 3
    %res32_zext_shift = sub i32 %lhs32, %rhs32_zext_shift
    store volatile i32 %res32_zext_shift, i32* @var32
 ; CHECK: sub {{w[0-9]+}}, {{w[0-9]+}}, {{w[0-9]+}}, uxth #3
-
+; GISEL: subs {{w[0-9]+}}, {{w[0-9]+}}, {{w[0-9]+}}, uxth #3
 
 ; Zero-extending to 64-bits
     %rhs64_zext = zext i16 %val16 to i64
     %res64_zext = sub i64 %lhs64, %rhs64_zext
     store volatile i64 %res64_zext, i64* @var64
 ; CHECK: sub {{x[0-9]+}}, {{x[0-9]+}}, {{w[0-9]+}}, uxth
+; GISEL: subs {{x[0-9]+}}, {{x[0-9]+}}, {{w[0-9]+}}, uxth
 
    %rhs64_zext_shift = shl i64 %rhs64_zext, 1
    %res64_zext_shift = sub i64 %lhs64, %rhs64_zext_shift
    store volatile i64 %res64_zext_shift, i64* @var64
 ; CHECK: sub {{x[0-9]+}}, {{x[0-9]+}}, {{w[0-9]+}}, uxth #1
+; GISEL: subs {{x[0-9]+}}, {{x[0-9]+}}, {{w[0-9]+}}, uxth #1
 
 ; Sign-extending to 32-bits
     %rhs32_sext = sext i16 %val16 to i32
@@ -276,6 +296,7 @@ define void @sub_i16rhs() minsize {
 ; in the face of "add/sub (shifted register)" so I don't intend to.
 define void @addsub_i32rhs(i32 %in32) minsize {
 ; CHECK-LABEL: addsub_i32rhs:
+; GISEL-LABEL: addsub_i32rhs:
     %val32_tmp = load i32, i32* @var32
     %lhs64 = load i64, i64* @var64
 
@@ -285,22 +306,26 @@ define void @addsub_i32rhs(i32 %in32) minsize {
     %res64_zext = add i64 %lhs64, %rhs64_zext
     store volatile i64 %res64_zext, i64* @var64
 ; CHECK: add {{x[0-9]+}}, {{x[0-9]+}}, {{w[0-9]+}}, uxtw
+; GISEL: add {{x[0-9]+}}, {{x[0-9]+}}, {{w[0-9]+}}, uxtw
 
     %rhs64_zext2 = zext i32 %val32 to i64
     %rhs64_zext_shift = shl i64 %rhs64_zext2, 2
     %res64_zext_shift = add i64 %lhs64, %rhs64_zext_shift
     store volatile i64 %res64_zext_shift, i64* @var64
 ; CHECK: add {{x[0-9]+}}, {{x[0-9]+}}, {{w[0-9]+}}, uxtw #2
+; GISEL: add {{x[0-9]+}}, {{x[0-9]+}}, {{w[0-9]+}}, uxtw #2
 
     %rhs64_sext = sext i32 %val32 to i64
     %res64_sext = add i64 %lhs64, %rhs64_sext
     store volatile i64 %res64_sext, i64* @var64
 ; CHECK: add {{x[0-9]+}}, {{x[0-9]+}}, {{w[0-9]+}}, sxtw
+; GISEL: add {{x[0-9]+}}, {{x[0-9]+}}, {{w[0-9]+}}, sxtw
 
     %rhs64_sext_shift = shl i64 %rhs64_sext, 2
     %res64_sext_shift = add i64 %lhs64, %rhs64_sext_shift
     store volatile i64 %res64_sext_shift, i64* @var64
 ; CHECK: add {{x[0-9]+}}, {{x[0-9]+}}, {{w[0-9]+}}, sxtw #2
+; GISEL: add {{x[0-9]+}}, {{x[0-9]+}}, {{w[0-9]+}}, sxtw #2
 
     ret void
 }
@@ -316,12 +341,14 @@ define void @sub_i32rhs(i32 %in32) minsize {
     %res64_zext = sub i64 %lhs64, %rhs64_zext
     store volatile i64 %res64_zext, i64* @var64
 ; CHECK: sub {{x[0-9]+}}, {{x[0-9]+}}, {{w[0-9]+}}, uxtw
+; GISEL: subs {{x[0-9]+}}, {{x[0-9]+}}, {{w[0-9]+}}, uxtw
 
     %rhs64_zext2 = zext i32 %val32 to i64
     %rhs64_zext_shift = shl i64 %rhs64_zext2, 2
     %res64_zext_shift = sub i64 %lhs64, %rhs64_zext_shift
     store volatile i64 %res64_zext_shift, i64* @var64
 ; CHECK: sub {{x[0-9]+}}, {{x[0-9]+}}, {{w[0-9]+}}, uxtw #2
+; GISEL: subs {{x[0-9]+}}, {{x[0-9]+}}, {{w[0-9]+}}, uxtw #2
 
     %rhs64_sext = sext i32 %val32 to i64
     %res64_sext = sub i64 %lhs64, %rhs64_sext
@@ -339,11 +366,15 @@ define void @sub_i32rhs(i32 %in32) minsize {
 ; Check that implicit zext from w reg write is used instead of uxtw form of add.
 define i64 @add_fold_uxtw(i32 %x, i64 %y) {
 ; CHECK-LABEL: add_fold_uxtw:
+; GISEL-LABEL: add_fold_uxtw:
 entry:
 ; CHECK: and w[[TMP:[0-9]+]], w0, #0x3
+; GISEL: and w[[TMP:[0-9]+]], w0, #0x3
+; FIXME: Global ISel produces an unncessary ubfx here.
   %m = and i32 %x, 3
   %ext = zext i32 %m to i64
 ; CHECK-NEXT: add x0, x1, x[[TMP]]
+; GISEL: add x0, x1, x[[TMP]]
   %ret = add i64 %y, %ext
   ret i64 %ret
 }
@@ -352,11 +383,14 @@ entry:
 ; form of sub and that mov WZR is folded to form a neg instruction.
 define i64 @sub_fold_uxtw_xzr(i32 %x)  {
 ; CHECK-LABEL: sub_fold_uxtw_xzr:
+; GISEL-LABEL: sub_fold_uxtw_xzr:
 entry:
 ; CHECK: and w[[TMP:[0-9]+]], w0, #0x3
+; GISEL: and w[[TMP:[0-9]+]], w0, #0x3
   %m = and i32 %x, 3
   %ext = zext i32 %m to i64
 ; CHECK-NEXT: neg x0, x[[TMP]]
+; GISEL: negs x0, x[[TMP]]
   %ret = sub i64 0, %ext
   ret i64 %ret
 }
@@ -378,10 +412,13 @@ entry:
 ; form of add, leading to madd selection.
 define i64 @madd_fold_uxtw(i32 %x, i64 %y) {
 ; CHECK-LABEL: madd_fold_uxtw:
+; GISEL-LABEL: madd_fold_uxtw:
 entry:
 ; CHECK: and w[[TMP:[0-9]+]], w0, #0x3
+; GISEL: and w[[TMP:[0-9]+]], w0, #0x3
   %m = and i32 %x, 3
   %ext = zext i32 %m to i64
+; GISEL: madd x0, x1, x1, x[[TMP]]
 ; CHECK-NEXT: madd x0, x1, x1, x[[TMP]]
   %mul = mul i64 %y, %y
   %ret = add i64 %mul, %ext
@@ -408,11 +445,14 @@ entry:
 ; form of add and add of -1 gets selected as sub.
 define i64 @add_imm_fold_uxtw(i32 %x) {
 ; CHECK-LABEL: add_imm_fold_uxtw:
+; GISEL-LABEL: add_imm_fold_uxtw:
 entry:
 ; CHECK: and w[[TMP:[0-9]+]], w0, #0x3
+; GISEL: and w[[TMP:[0-9]+]], w0, #0x3
   %m = and i32 %x, 3
   %ext = zext i32 %m to i64
 ; CHECK-NEXT: sub x0, x[[TMP]], #1
+; GISEL: subs x0, x[[TMP]], #1
   %ret = add i64 %ext, -1
   ret i64 %ret
 }
@@ -421,12 +461,15 @@ entry:
 ; form of add and add lsl form gets selected.
 define i64 @add_lsl_fold_uxtw(i32 %x, i64 %y) {
 ; CHECK-LABEL: add_lsl_fold_uxtw:
+; GISEL-LABEL: add_lsl_fold_uxtw:
 entry:
 ; CHECK: orr w[[TMP:[0-9]+]], w0, #0x3
+; GISEL: orr w[[TMP:[0-9]+]], w0, #0x3
   %m = or i32 %x, 3
   %ext = zext i32 %m to i64
   %shift = shl i64 %y, 3
 ; CHECK-NEXT: add x0, x[[TMP]], x1, lsl #3
+; GISEL: add x0, x[[TMP]], x1, lsl #3
   %ret = add i64 %ext, %shift
   ret i64 %ret
 }
diff --git a/llvm/test/CodeGen/AArch64/align-down.ll b/llvm/test/CodeGen/AArch64/align-down.ll
new file mode 100644
index 000000000..659b69ca2
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/align-down.ll
@@ -0,0 +1,149 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64-unknown-linux-gnu < %s | FileCheck %s
+
+; Fold
+;   ptr - (ptr & (alignment-1))
+; To
+;   ptr & (0 - alignment)
+;
+; This needs to be a backend-level fold because only by now pointers
+; are just registers; in middle-end IR this can only be done via @llvm.ptrmask()
+; intrinsic which is not sufficiently widely-spread yet.
+;
+; https://bugs.llvm.org/show_bug.cgi?id=44448
+
+; The basic positive tests
+
+define i32 @t0_32(i32 %ptr, i32 %alignment) nounwind {
+; CHECK-LABEL: t0_32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    neg w8, w1
+; CHECK-NEXT:    and w0, w0, w8
+; CHECK-NEXT:    ret
+  %mask = add i32 %alignment, -1
+  %bias = and i32 %ptr, %mask
+  %r = sub i32 %ptr, %bias
+  ret i32 %r
+}
+define i64 @t1_64(i64 %ptr, i64 %alignment) nounwind {
+; CHECK-LABEL: t1_64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    neg x8, x1
+; CHECK-NEXT:    and x0, x0, x8
+; CHECK-NEXT:    ret
+  %mask = add i64 %alignment, -1
+  %bias = and i64 %ptr, %mask
+  %r = sub i64 %ptr, %bias
+  ret i64 %r
+}
+
+define i32 @t2_commutative(i32 %ptr, i32 %alignment) nounwind {
+; CHECK-LABEL: t2_commutative:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    neg w8, w1
+; CHECK-NEXT:    and w0, w0, w8
+; CHECK-NEXT:    ret
+  %mask = add i32 %alignment, -1
+  %bias = and i32 %mask, %ptr ; swapped
+  %r = sub i32 %ptr, %bias
+  ret i32 %r
+}
+
+; Extra use tests
+
+define i32 @t3_extrause0(i32 %ptr, i32 %alignment, i32* %mask_storage) nounwind {
+; CHECK-LABEL: t3_extrause0:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    neg w9, w1
+; CHECK-NEXT:    sub w8, w1, #1 // =1
+; CHECK-NEXT:    and w0, w0, w9
+; CHECK-NEXT:    str w8, [x2]
+; CHECK-NEXT:    ret
+  %mask = add i32 %alignment, -1
+  store i32 %mask, i32* %mask_storage
+  %bias = and i32 %ptr, %mask
+  %r = sub i32 %ptr, %bias
+  ret i32 %r
+}
+define i32 @n4_extrause1(i32 %ptr, i32 %alignment, i32* %bias_storage) nounwind {
+; CHECK-LABEL: n4_extrause1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub w8, w1, #1 // =1
+; CHECK-NEXT:    and w8, w0, w8
+; CHECK-NEXT:    sub w0, w0, w8
+; CHECK-NEXT:    str w8, [x2]
+; CHECK-NEXT:    ret
+  %mask = add i32 %alignment, -1
+  %bias = and i32 %ptr, %mask ; has extra uses, can't fold
+  store i32 %bias, i32* %bias_storage
+  %r = sub i32 %ptr, %bias
+  ret i32 %r
+}
+define i32 @n5_extrause2(i32 %ptr, i32 %alignment, i32* %mask_storage, i32* %bias_storage) nounwind {
+; CHECK-LABEL: n5_extrause2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub w8, w1, #1 // =1
+; CHECK-NEXT:    str w8, [x2]
+; CHECK-NEXT:    and w8, w0, w8
+; CHECK-NEXT:    sub w0, w0, w8
+; CHECK-NEXT:    str w8, [x3]
+; CHECK-NEXT:    ret
+  %mask = add i32 %alignment, -1
+  store i32 %mask, i32* %mask_storage
+  %bias = and i32 %ptr, %mask ; has extra uses, can't fold
+  store i32 %bias, i32* %bias_storage
+  %r = sub i32 %ptr, %bias
+  ret i32 %r
+}
+
+; Negative tests
+
+define i32 @n6_different_ptrs(i32 %ptr0, i32 %ptr1, i32 %alignment) nounwind {
+; CHECK-LABEL: n6_different_ptrs:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub w8, w2, #1 // =1
+; CHECK-NEXT:    and w8, w1, w8
+; CHECK-NEXT:    sub w0, w0, w8
+; CHECK-NEXT:    ret
+  %mask = add i32 %alignment, -1
+  %bias = and i32 %ptr1, %mask ; not %ptr0
+  %r = sub i32 %ptr0, %bias ; not %ptr1
+  ret i32 %r
+}
+define i32 @n7_different_ptrs_commutative(i32 %ptr0, i32 %ptr1, i32 %alignment) nounwind {
+; CHECK-LABEL: n7_different_ptrs_commutative:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub w8, w2, #1 // =1
+; CHECK-NEXT:    and w8, w8, w1
+; CHECK-NEXT:    sub w0, w0, w8
+; CHECK-NEXT:    ret
+  %mask = add i32 %alignment, -1
+  %bias = and i32 %mask, %ptr1 ; swapped, not %ptr0
+  %r = sub i32 %ptr0, %bias ; not %ptr1
+  ret i32 %r
+}
+
+define i32 @n8_not_lowbit_mask(i32 %ptr, i32 %alignment) nounwind {
+; CHECK-LABEL: n8_not_lowbit_mask:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    add w8, w1, #1 // =1
+; CHECK-NEXT:    bic w0, w0, w8
+; CHECK-NEXT:    ret
+  %mask = add i32 %alignment, 1 ; not -1
+  %bias = and i32 %ptr, %mask
+  %r = sub i32 %ptr, %bias
+  ret i32 %r
+}
+
+define i32 @n9_sub_is_not_commutative(i32 %ptr, i32 %alignment) nounwind {
+; CHECK-LABEL: n9_sub_is_not_commutative:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub w8, w1, #1 // =1
+; CHECK-NEXT:    and w8, w0, w8
+; CHECK-NEXT:    sub w0, w8, w0
+; CHECK-NEXT:    ret
+  %mask = add i32 %alignment, -1
+  %bias = and i32 %ptr, %mask
+  %r = sub i32 %bias, %ptr ; wrong order
+  ret i32 %r
+}
diff --git a/llvm/test/CodeGen/AArch64/alloca.ll b/llvm/test/CodeGen/AArch64/alloca.ll
index 25bb3c8ba..e7906a1e9 100644
--- a/llvm/test/CodeGen/AArch64/alloca.ll
+++ b/llvm/test/CodeGen/AArch64/alloca.ll
@@ -117,9 +117,9 @@ define void @test_alloca_large_frame(i64 %n) {
 ; CHECK-MACHO-LABEL: test_alloca_large_frame:
 
 
-; CHECK: stp     x28, x19, [sp, #-32]!
-; CHECK: stp     x29, x30, [sp, #16]
-; CHECK: add     x29, sp, #16
+; CHECK: stp     x29, x30, [sp, #-32]!
+; CHECK: stp     x28, x19, [sp, #16]
+; CHECK: mov     x29, sp
 ; CHECK: sub     sp, sp, #1953, lsl #12
 ; CHECK: sub     sp, sp, #512
 
@@ -136,9 +136,9 @@ define void @test_alloca_large_frame(i64 %n) {
 
   ret void
 
-; CHECK: sub     sp, x29, #16
-; CHECK: ldp     x29, x30, [sp, #16]
-; CHECK: ldp     x28, x19, [sp], #32
+; CHECK: mov     sp, x29
+; CHECK: ldp     x28, x19, [sp, #16]
+; CHECK: ldp     x29, x30, [sp], #32
 
 ; CHECK-MACHO: sub     sp, x29, #16
 ; CHECK-MACHO: ldp     x29, x30, [sp, #16]
@@ -152,6 +152,7 @@ define void @test_scoped_alloca(i64 %n) {
 ; CHECK-LABEL: test_scoped_alloca:
 
   %sp = call i8* @llvm.stacksave()
+; CHECK: mov x29, sp
 ; CHECK: mov [[SAVED_SP:x[0-9]+]], sp
 ; CHECK: mov [[OLDSP:x[0-9]+]], sp
 
diff --git a/llvm/test/CodeGen/AArch64/arm64-aapcs.ll b/llvm/test/CodeGen/AArch64/arm64-aapcs.ll
index 79a25c1e3..7887facb9 100644
--- a/llvm/test/CodeGen/AArch64/arm64-aapcs.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-aapcs.ll
@@ -25,7 +25,7 @@ define [2 x i64] @test_i64x2_align(i32, [2 x i64] %arg, i32 %after) {
 @var64 = global i64 0, align 8
 
 ; Check stack slots are 64-bit at all times.
-define void @test_stack_slots([8 x i32], i1 %bool, i8 %char, i16 %short,
+define void @test_stack_slots([8 x i64], i1 %bool, i8 %char, i16 %short,
                                 i32 %int, i64 %long) {
 ; CHECK-LABEL: test_stack_slots:
 ; CHECK-DAG: ldr w[[ext1:[0-9]+]], [sp, #24]
diff --git a/llvm/test/CodeGen/AArch64/arm64-abi-varargs.ll b/llvm/test/CodeGen/AArch64/arm64-abi-varargs.ll
index b0a425655..e904e86d8 100644
--- a/llvm/test/CodeGen/AArch64/arm64-abi-varargs.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-abi-varargs.ll
@@ -1,19 +1,29 @@
-; RUN: llc < %s -mtriple=arm64-apple-ios7.0.0 -mcpu=cyclone -enable-misched=false | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -aarch64-load-store-renaming=true < %s -mtriple=arm64-apple-ios7.0.0 -mcpu=cyclone -enable-misched=false | FileCheck %s
 
 ; rdar://13625505
 ; Here we have 9 fixed integer arguments the 9th argument in on stack, the
 ; varargs start right after at 8-byte alignment.
 define void @fn9(i32* %a1, i32 %a2, i32 %a3, i32 %a4, i32 %a5, i32 %a6, i32 %a7, i32 %a8, i32 %a9, ...) nounwind noinline ssp {
 ; CHECK-LABEL: fn9:
-; 9th fixed argument
-; CHECK: ldr {{w[0-9]+}}, [sp, #64]
-; CHECK-DAG: add [[ARGS:x[0-9]+]], sp, #72
-; First vararg
-; CHECK-DAG: ldr {{w[0-9]+}}, [sp, #72]
-; Second vararg
-; CHECK-DAG: ldr {{w[0-9]+}}, [sp, #80]
-; Third vararg
-; CHECK-DAG: ldr {{w[0-9]+}}, [sp, #88]
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    sub sp, sp, #64 ; =64
+; CHECK-NEXT:    ldr w8, [sp, #64]
+; CHECK-NEXT:    stp w2, w1, [sp, #52]
+; CHECK-NEXT:    stp w4, w3, [sp, #44]
+; CHECK-NEXT:    stp w6, w5, [sp, #36]
+; CHECK-NEXT:    str w7, [sp, #32]
+; CHECK-NEXT:    str w8, [x0]
+; CHECK-NEXT:    ldr w9, [sp, #72]
+; CHECK-NEXT:    ldr w8, [sp, #80]
+; CHECK-NEXT:    stp w8, w9, [sp, #16]
+; CHECK-NEXT:    add x8, sp, #72 ; =72
+; CHECK-NEXT:    add x8, x8, #24 ; =24
+; CHECK-NEXT:    str x8, [sp, #24]
+; CHECK-NEXT:    ldr w8, [sp, #88]
+; CHECK-NEXT:    str w8, [sp, #12]
+; CHECK-NEXT:    add sp, sp, #64 ; =64
+; CHECK-NEXT:    ret
   %1 = alloca i32, align 4
   %2 = alloca i32, align 4
   %3 = alloca i32, align 4
@@ -51,9 +61,43 @@ declare void @llvm.va_start(i8*) nounwind
 
 define i32 @main() nounwind ssp {
 ; CHECK-LABEL: main:
-; CHECK: stp {{x[0-9]+}}, {{x[0-9]+}}, [sp, #16]
-; CHECK: str {{x[0-9]+}}, [sp, #8]
-; CHECK: str {{w[0-9]+}}, [sp]
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    sub sp, sp, #96 ; =96
+; CHECK-NEXT:    stp x29, x30, [sp, #80] ; 16-byte Folded Spill
+; CHECK-NEXT:    mov w9, #1
+; CHECK-NEXT:    mov w8, #2
+; CHECK-NEXT:    stp w8, w9, [sp, #72]
+; CHECK-NEXT:    mov w9, #3
+; CHECK-NEXT:    mov w8, #4
+; CHECK-NEXT:    stp w8, w9, [sp, #64]
+; CHECK-NEXT:    mov w9, #5
+; CHECK-NEXT:    mov w8, #6
+; CHECK-NEXT:    stp w8, w9, [sp, #56]
+; CHECK-NEXT:    mov w9, #7
+; CHECK-NEXT:    mov w8, #8
+; CHECK-NEXT:    stp w8, w9, [sp, #48]
+; CHECK-NEXT:    mov w8, #9
+; CHECK-NEXT:    mov w9, #10
+; CHECK-NEXT:    stp w9, w8, [sp, #40]
+; CHECK-NEXT:    mov w10, #11
+; CHECK-NEXT:    mov w11, #12
+; CHECK-NEXT:    stp w11, w10, [sp, #32]
+; CHECK-NEXT:    stp x10, x11, [sp, #16]
+; CHECK-NEXT:    str x9, [sp, #8]
+; CHECK-NEXT:    str w8, [sp]
+; CHECK-NEXT:    add x0, sp, #76 ; =76
+; CHECK-NEXT:    mov w1, #2
+; CHECK-NEXT:    mov w2, #3
+; CHECK-NEXT:    mov w3, #4
+; CHECK-NEXT:    mov w4, #5
+; CHECK-NEXT:    mov w5, #6
+; CHECK-NEXT:    mov w6, #7
+; CHECK-NEXT:    mov w7, #8
+; CHECK-NEXT:    bl _fn9
+; CHECK-NEXT:    mov w0, #0
+; CHECK-NEXT:    ldp x29, x30, [sp, #80] ; 16-byte Folded Reload
+; CHECK-NEXT:    add sp, sp, #96 ; =96
+; CHECK-NEXT:    ret
   %a1 = alloca i32, align 4
   %a2 = alloca i32, align 4
   %a3 = alloca i32, align 4
@@ -97,12 +141,20 @@ define i32 @main() nounwind ssp {
 ;rdar://13668483
 @.str = private unnamed_addr constant [4 x i8] c"fmt\00", align 1
 define void @foo(i8* %fmt, ...) nounwind {
-entry:
 ; CHECK-LABEL: foo:
-; CHECK: ldr {{w[0-9]+}}, [sp, #48]
-; CHECK: add {{x[0-9]+}}, {{x[0-9]+}}, #23
-; CHECK: and x[[ADDR:[0-9]+]], {{x[0-9]+}}, #0xfffffffffffffff0
-; CHECK: ldr {{q[0-9]+}}, [x[[ADDR]]]
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    sub sp, sp, #48 ; =48
+; CHECK-NEXT:    ldr w8, [sp, #48]
+; CHECK-NEXT:    str w8, [sp, #28]
+; CHECK-NEXT:    add x8, sp, #48 ; =48
+; CHECK-NEXT:    add x8, x8, #23 ; =23
+; CHECK-NEXT:    and x8, x8, #0xfffffffffffffff0
+; CHECK-NEXT:    add x9, x8, #16 ; =16
+; CHECK-NEXT:    stp x9, x0, [sp, #32]
+; CHECK-NEXT:    ldr q0, [x8]
+; CHECK-NEXT:    str q0, [sp], #48
+; CHECK-NEXT:    ret
+entry:
   %fmt.addr = alloca i8*, align 8
   %args = alloca i8*, align 8
   %vc = alloca i32, align 4
@@ -118,10 +170,24 @@ entry:
 }
 
 define void @bar(i32 %x, <4 x i32> %y) nounwind {
-entry:
 ; CHECK-LABEL: bar:
-; CHECK: stp {{q[0-9]+}}, {{q[0-9]+}}, [sp, #16]
-; CHECK: str {{x[0-9]+}}, [sp]
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    sub sp, sp, #80 ; =80
+; CHECK-NEXT:    stp x29, x30, [sp, #64] ; 16-byte Folded Spill
+; CHECK-NEXT:    ; kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT:    str w0, [sp, #60]
+; CHECK-NEXT:    stp q0, q0, [sp, #16]
+; CHECK-NEXT:    str x0, [sp]
+; CHECK-NEXT:  Lloh0:
+; CHECK-NEXT:    adrp x0, l_.str@PAGE
+; CHECK-NEXT:  Lloh1:
+; CHECK-NEXT:    add x0, x0, l_.str@PAGEOFF
+; CHECK-NEXT:    bl _foo
+; CHECK-NEXT:    ldp x29, x30, [sp, #64] ; 16-byte Folded Reload
+; CHECK-NEXT:    add sp, sp, #80 ; =80
+; CHECK-NEXT:    ret
+; CHECK-NEXT:    .loh AdrpAdd Lloh0, Lloh1
+entry:
   %x.addr = alloca i32, align 4
   %y.addr = alloca <4 x i32>, align 16
   store i32 %x, i32* %x.addr, align 4
@@ -137,12 +203,20 @@ entry:
 ; side is 16-byte aligned on stack.
 %struct.s41 = type { i32, i16, i32, i16 }
 define void @foo2(i8* %fmt, ...) nounwind {
-entry:
 ; CHECK-LABEL: foo2:
-; CHECK: ldr {{w[0-9]+}}, [sp, #48]
-; CHECK: add {{x[0-9]+}}, {{x[0-9]+}}, #23
-; CHECK: and x[[ADDR:[0-9]+]], {{x[0-9]+}}, #0xfffffffffffffff0
-; CHECK: ldr {{q[0-9]+}}, [x[[ADDR]]]
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    sub sp, sp, #48 ; =48
+; CHECK-NEXT:    ldr w8, [sp, #48]
+; CHECK-NEXT:    str w8, [sp, #28]
+; CHECK-NEXT:    add x8, sp, #48 ; =48
+; CHECK-NEXT:    add x8, x8, #23 ; =23
+; CHECK-NEXT:    and x8, x8, #0xfffffffffffffff0
+; CHECK-NEXT:    add x9, x8, #16 ; =16
+; CHECK-NEXT:    stp x9, x0, [sp, #32]
+; CHECK-NEXT:    ldr q0, [x8]
+; CHECK-NEXT:    str q0, [sp], #48
+; CHECK-NEXT:    ret
+entry:
   %fmt.addr = alloca i8*, align 8
   %args = alloca i8*, align 8
   %vc = alloca i32, align 4
@@ -168,10 +242,25 @@ entry:
 declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i1) nounwind
 
 define void @bar2(i32 %x, i128 %s41.coerce) nounwind {
-entry:
 ; CHECK-LABEL: bar2:
-; CHECK: stp {{x[0-9]+}}, {{x[0-9]+}}, [sp, #16]
-; CHECK: str {{x[0-9]+}}, [sp]
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    sub sp, sp, #80 ; =80
+; CHECK-NEXT:    stp x29, x30, [sp, #64] ; 16-byte Folded Spill
+; CHECK-NEXT:    ; kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT:    str w0, [sp, #60]
+; CHECK-NEXT:    stp x1, x2, [sp, #32]
+; CHECK-NEXT:    stp x1, x2, [sp, #16]
+; CHECK-NEXT:    str x0, [sp]
+; CHECK-NEXT:  Lloh2:
+; CHECK-NEXT:    adrp x0, l_.str@PAGE
+; CHECK-NEXT:  Lloh3:
+; CHECK-NEXT:    add x0, x0, l_.str@PAGEOFF
+; CHECK-NEXT:    bl _foo2
+; CHECK-NEXT:    ldp x29, x30, [sp, #64] ; 16-byte Folded Reload
+; CHECK-NEXT:    add sp, sp, #80 ; =80
+; CHECK-NEXT:    ret
+; CHECK-NEXT:    .loh AdrpAdd Lloh2, Lloh3
+entry:
   %x.addr = alloca i32, align 4
   %s41 = alloca %struct.s41, align 16
   store i32 %x, i32* %x.addr, align 4
diff --git a/llvm/test/CodeGen/AArch64/arm64-abi_align.ll b/llvm/test/CodeGen/AArch64/arm64-abi_align.ll
index 7db3ea76d..b76d453c6 100644
--- a/llvm/test/CodeGen/AArch64/arm64-abi_align.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-abi_align.ll
@@ -1,5 +1,5 @@
-; RUN: llc -fast-isel-sink-local-values < %s -mtriple=arm64-apple-darwin -mcpu=cyclone -enable-misched=false -frame-pointer=all | FileCheck %s
-; RUN: llc -fast-isel-sink-local-values < %s -mtriple=arm64-apple-darwin -O0 -frame-pointer=all -fast-isel | FileCheck -check-prefix=FAST %s
+; RUN: llc -fast-isel-sink-local-values -aarch64-load-store-renaming=true < %s -mtriple=arm64-apple-darwin -mcpu=cyclone -enable-misched=false -frame-pointer=all | FileCheck %s
+; RUN: llc -fast-isel-sink-local-values -aarch64-load-store-renaming=true  < %s -mtriple=arm64-apple-darwin -O0 -frame-pointer=all -fast-isel | FileCheck -check-prefix=FAST %s
 
 ; rdar://12648441
 ; Generated from arm64-arguments.c with -O2.
@@ -392,10 +392,8 @@ entry:
 define i32 @caller43() #3 {
 entry:
 ; CHECK-LABEL: caller43
-; CHECK-DAG: str {{q[0-9]+}}, [sp, #48]
-; CHECK-DAG: str {{q[0-9]+}}, [sp, #32]
-; CHECK-DAG: str {{q[0-9]+}}, [sp, #16]
-; CHECK-DAG: str {{q[0-9]+}}, [sp]
+; CHECK-DAG: stp q1, q0, [sp, #32]
+; CHECK-DAG: stp q1, q0, [sp]
 ; CHECK: add x1, sp, #32
 ; CHECK: mov x2, sp
 ; Space for s1 is allocated at sp+32
@@ -434,10 +432,8 @@ entry:
 ; CHECK-LABEL: caller43_stack
 ; CHECK: sub sp, sp, #112
 ; CHECK: add x29, sp, #96
-; CHECK-DAG: stur {{q[0-9]+}}, [x29, #-16]
-; CHECK-DAG: stur {{q[0-9]+}}, [x29, #-32]
-; CHECK-DAG: str {{q[0-9]+}}, [sp, #48]
-; CHECK-DAG: str {{q[0-9]+}}, [sp, #32]
+; CHECK-DAG: stp q1, q0, [x29, #-32]
+; CHECK-DAG: stp q1, q0, [sp, #32]
 ; Space for s1 is allocated at x29-32 = sp+64
 ; Space for s2 is allocated at sp+32
 ; CHECK: add x[[B:[0-9]+]], sp, #32
diff --git a/llvm/test/CodeGen/AArch64/arm64-alloca-frame-pointer-offset.ll b/llvm/test/CodeGen/AArch64/arm64-alloca-frame-pointer-offset.ll
index 45e32c8aa..0267acdda 100644
--- a/llvm/test/CodeGen/AArch64/arm64-alloca-frame-pointer-offset.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-alloca-frame-pointer-offset.ll
@@ -1,8 +1,8 @@
 ; RUN: llc -mtriple=arm64-eabi -mcpu=cyclone < %s | FileCheck %s
 
 ; CHECK: foo
-; CHECK-DAG: stur w[[REG0:[0-9]+]], [x29, #-24]
-; CHECK-DAG: stur w[[REG0]], [x29, #-20]
+; CHECK-DAG: str w[[REG0:[0-9]+]], [x29, #24]
+; CHECK-DAG: str w[[REG0]], [x29, #28]
 define i32 @foo(i32 %a) nounwind {
   %retval = alloca i32, align 4
   %a.addr = alloca i32, align 4
diff --git a/llvm/test/CodeGen/AArch64/arm64-blockaddress.ll b/llvm/test/CodeGen/AArch64/arm64-blockaddress.ll
index b50ffdef5..b35cb28b4 100644
--- a/llvm/test/CodeGen/AArch64/arm64-blockaddress.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-blockaddress.ll
@@ -7,12 +7,12 @@
 define i64 @t() nounwind ssp {
 entry:
 ; CHECK-LABEL: t:
-; CHECK: adrp [[REG:x[0-9]+]], Ltmp1@PAGE
-; CHECK: add {{x[0-9]+}}, [[REG]], Ltmp1@PAGEOFF
+; CHECK: adrp [[REG:x[0-9]+]], Ltmp0@PAGE
+; CHECK: add {{x[0-9]+}}, [[REG]], Ltmp0@PAGEOFF
 
 ; CHECK-LINUX-LABEL: t:
-; CHECK-LINUX: adrp [[REG:x[0-9]+]], .Ltmp1
-; CHECK-LINUX: add {{x[0-9]+}}, [[REG]], :lo12:.Ltmp1
+; CHECK-LINUX: adrp [[REG:x[0-9]+]], .Ltmp0
+; CHECK-LINUX: add {{x[0-9]+}}, [[REG]], :lo12:.Ltmp0
 
 ; CHECK-LARGE-LABEL: t:
 ; CHECK-LARGE: movz [[ADDR_REG:x[0-9]+]], #:abs_g0_nc:[[DEST_LBL:.Ltmp[0-9]+]]
diff --git a/llvm/test/CodeGen/AArch64/arm64-call-tailcalls.ll b/llvm/test/CodeGen/AArch64/arm64-call-tailcalls.ll
index 7a91f05b8..9b6d1f3a1 100644
--- a/llvm/test/CodeGen/AArch64/arm64-call-tailcalls.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-call-tailcalls.ll
@@ -1,4 +1,5 @@
 ; RUN: llc < %s -mtriple=arm64-apple-ios7.0 | FileCheck %s
+; RUN: llc -global-isel < %s -mtriple=arm64-apple-ios7.0 | FileCheck %s
 
 @t = weak global i32 ()* null
 @x = external global i32, align 4
diff --git a/llvm/test/CodeGen/AArch64/arm64-code-model-large-darwin.ll b/llvm/test/CodeGen/AArch64/arm64-code-model-large-darwin.ll
new file mode 100644
index 000000000..8ad93af8e
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/arm64-code-model-large-darwin.ll
@@ -0,0 +1,16 @@
+; Check that AArch64 is honoring code-model=large at -O0 and -O2.
+;
+; RUN: llc -mtriple=arm64-apple-darwin19 -code-model=large -O0 -o - %s | FileCheck %s
+; RUN: llc -mtriple=arm64-apple-darwin19 -code-model=large -O2 -o - %s | FileCheck %s
+
+; CHECK: adrp    [[REG1:x[0-9]+]], _bar@GOTPAGE
+; CHECK: ldr     [[REG1]], {{\[}}[[REG1]], _bar@GOTPAGEOFF]
+; CHECK: blr     [[REG1]]
+
+declare void @bar()
+
+define void @foo() {
+  call void @bar()
+  ret void
+}
+
diff --git a/llvm/test/CodeGen/AArch64/arm64-collect-loh-garbage-crash.ll b/llvm/test/CodeGen/AArch64/arm64-collect-loh-garbage-crash.ll
index 727c18972..05f467e19 100644
--- a/llvm/test/CodeGen/AArch64/arm64-collect-loh-garbage-crash.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-collect-loh-garbage-crash.ll
@@ -1,4 +1,5 @@
 ; RUN: llc -o - %s -mtriple=arm64-apple-ios -O3 -aarch64-enable-collect-loh | FileCheck %s
+; RUN: llc -o - %s -mtriple=arm64_32-apple-watchos -O3 -aarch64-enable-collect-loh | FileCheck %s
 ; Check that the LOH analysis does not crash when the analysed chained
 ; contains instructions that are filtered out.
 ;
diff --git a/llvm/test/CodeGen/AArch64/arm64-collect-loh-str.ll b/llvm/test/CodeGen/AArch64/arm64-collect-loh-str.ll
index 773286ef1..962e36ddb 100644
--- a/llvm/test/CodeGen/AArch64/arm64-collect-loh-str.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-collect-loh-str.ll
@@ -1,4 +1,5 @@
 ; RUN: llc -o - %s -mtriple=arm64-apple-ios -O2 | FileCheck %s
+; RUN: llc -o - %s -mtriple=arm64_32-apple-ios -O2 | FileCheck %s
 ; Test case for <rdar://problem/15942912>.
 ; AdrpAddStr cannot be used when the store uses same
 ; register as address and value. Indeed, the related
diff --git a/llvm/test/CodeGen/AArch64/arm64-collect-loh.ll b/llvm/test/CodeGen/AArch64/arm64-collect-loh.ll
index eb3607dd4..816e5a7cc 100644
--- a/llvm/test/CodeGen/AArch64/arm64-collect-loh.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-collect-loh.ll
@@ -1,4 +1,5 @@
 ; RUN: llc -o - %s -mtriple=arm64-apple-ios -O2 | FileCheck %s
+; RUN: llc -o - %s -mtriple=arm64_32-apple-watchos -O2 | FileCheck %s
 ; RUN: llc -o - %s -mtriple=arm64-linux-gnu -O2 | FileCheck %s --check-prefix=CHECK-ELF
 
 ; CHECK-ELF-NOT: .loh
@@ -60,9 +61,9 @@ if.end4:                                          ; preds = %if.then2, %if.then,
 ; CHECK: [[ADRP_LABEL:Lloh[0-9]+]]:
 ; CHECK-NEXT: adrp [[ADRP_REG:x[0-9]+]], _C@GOTPAGE
 ; CHECK-NEXT: [[LDRGOT_LABEL:Lloh[0-9]+]]:
-; CHECK-NEXT: ldr [[LDRGOT_REG:x[0-9]+]], {{\[}}[[ADRP_REG]], _C@GOTPAGEOFF]
+; CHECK-NEXT: ldr {{[xw]}}[[LDRGOT_REG:[0-9]+]], {{\[}}[[ADRP_REG]], _C@GOTPAGEOFF]
 ; CHECK-NEXT: [[LDR_LABEL:Lloh[0-9]+]]:
-; CHECK-NEXT: ldr w0, {{\[}}[[LDRGOT_REG]]]
+; CHECK-NEXT: ldr w0, [x[[LDRGOT_REG]]]
 ; CHECK-NEXT: ret
 ; CHECK: .loh AdrpLdrGotLdr [[ADRP_LABEL]], [[LDRGOT_LABEL]], [[LDR_LABEL]]
 define i32 @getC() {
@@ -76,9 +77,9 @@ define i32 @getC() {
 ; CHECK: [[ADRP_LABEL:Lloh[0-9]+]]:
 ; CHECK-NEXT: adrp [[ADRP_REG:x[0-9]+]], _C@GOTPAGE
 ; CHECK-NEXT: [[LDRGOT_LABEL:Lloh[0-9]+]]:
-; CHECK-NEXT: ldr [[LDRGOT_REG:x[0-9]+]], {{\[}}[[ADRP_REG]], _C@GOTPAGEOFF]
+; CHECK-NEXT: ldr {{[xw]}}[[LDRGOT_REG:[0-9]+]], {{\[}}[[ADRP_REG]], _C@GOTPAGEOFF]
 ; CHECK-NEXT: [[LDR_LABEL:Lloh[0-9]+]]:
-; CHECK-NEXT: ldrsw x0, {{\[}}[[LDRGOT_REG]]]
+; CHECK-NEXT: ldrsw x0, [x[[LDRGOT_REG]]]
 ; CHECK-NEXT: ret
 ; CHECK: .loh AdrpLdrGotLdr [[ADRP_LABEL]], [[LDRGOT_LABEL]], [[LDR_LABEL]]
 define i64 @getSExtC() {
@@ -94,10 +95,10 @@ define i64 @getSExtC() {
 ; CHECK: [[ADRP_LABEL:Lloh[0-9]+]]:
 ; CHECK-NEXT: adrp [[ADRP_REG:x[0-9]+]], _C@GOTPAGE
 ; CHECK-NEXT: [[LDRGOT_LABEL:Lloh[0-9]+]]:
-; CHECK-NEXT: ldr [[LDRGOT_REG:x[0-9]+]], {{\[}}[[ADRP_REG]], _C@GOTPAGEOFF]
-; CHECK-NEXT: ldr [[LOAD:w[0-9]+]], {{\[}}[[LDRGOT_REG]]]
+; CHECK-NEXT: ldr {{[xw]}}[[LDRGOT_REG:[0-9]+]], {{\[}}[[ADRP_REG]], _C@GOTPAGEOFF]
+; CHECK-NEXT: ldr [[LOAD:w[0-9]+]], [x[[LDRGOT_REG]]]
 ; CHECK-NEXT: add [[ADD:w[0-9]+]], [[LOAD]], w0
-; CHECK-NEXT: str [[ADD]], {{\[}}[[LDRGOT_REG]]]
+; CHECK-NEXT: str [[ADD]], [x[[LDRGOT_REG]]]
 ; CHECK-NEXT: ret
 ; CHECK: .loh AdrpLdrGot [[ADRP_LABEL]], [[LDRGOT_LABEL]]
 define void @getSeveralC(i32 %t) {
@@ -114,9 +115,9 @@ entry:
 ; CHECK: [[ADRP_LABEL:Lloh[0-9]+]]:
 ; CHECK-NEXT: adrp [[ADRP_REG:x[0-9]+]], _C@GOTPAGE
 ; CHECK-NEXT: [[LDRGOT_LABEL:Lloh[0-9]+]]:
-; CHECK-NEXT: ldr [[LDRGOT_REG:x[0-9]+]], {{\[}}[[ADRP_REG]], _C@GOTPAGEOFF]
+; CHECK-NEXT: ldr {{[xw]}}[[LDRGOT_REG:[0-9]+]], {{\[}}[[ADRP_REG]], _C@GOTPAGEOFF]
 ; CHECK-NEXT: [[LDR_LABEL:Lloh[0-9]+]]:
-; CHECK-NEXT: str w0, {{\[}}[[LDRGOT_REG]]]
+; CHECK-NEXT: str w0, [x[[LDRGOT_REG]]]
 ; CHECK-NEXT: ret
 ; CHECK: .loh AdrpLdrGotStr [[ADRP_LABEL]], [[LDRGOT_LABEL]], [[LDR_LABEL]]
 define void @setC(i32 %t) {
@@ -142,7 +143,7 @@ entry:
 ; CHECK-NEXT: ret
 ; CHECK: .loh AdrpAddLdr [[ADRP_LABEL]], [[ADDGOT_LABEL]], [[LDR_LABEL]]
 define i32 @getInternalCPlus4() {
-  %addr = getelementptr i32, i32* @InternalC, i32 4
+  %addr = getelementptr inbounds i32, i32* @InternalC, i32 4
   %res = load i32, i32* %addr, align 4
   ret i32 %res
 }
@@ -159,7 +160,7 @@ define i32 @getInternalCPlus4() {
 ; CHECK-NEXT: ret
 ; CHECK: .loh AdrpAddLdr [[ADRP_LABEL]], [[ADDGOT_LABEL]], [[LDR_LABEL]]
 define i64 @getSExtInternalCPlus4() {
-  %addr = getelementptr i32, i32* @InternalC, i32 4
+  %addr = getelementptr inbounds i32, i32* @InternalC, i32 4
   %res = load i32, i32* %addr, align 4
   %sextres = sext i32 %res to i64
   ret i64 %sextres
@@ -180,7 +181,7 @@ define i64 @getSExtInternalCPlus4() {
 ; CHECK: .loh AdrpAdd [[ADRP_LABEL]], [[ADDGOT_LABEL]]
 define void @getSeveralInternalCPlus4(i32 %t) {
 entry:
-  %addr = getelementptr i32, i32* @InternalC, i32 4
+  %addr = getelementptr inbounds i32, i32* @InternalC, i32 4
   %tmp = load i32, i32* %addr, align 4
   %add = add nsw i32 %tmp, %t
   store i32 %add, i32* %addr, align 4
@@ -200,7 +201,7 @@ entry:
 ; CHECK: .loh AdrpAddStr [[ADRP_LABEL]], [[ADDGOT_LABEL]], [[LDR_LABEL]]
 define void @setInternalCPlus4(i32 %t) {
 entry:
-  %addr = getelementptr i32, i32* @InternalC, i32 4
+  %addr = getelementptr inbounds i32, i32* @InternalC, i32 4
   store i32 %t, i32* %addr, align 4
   ret void
 }
@@ -276,8 +277,8 @@ entry:
 ; CHECK: [[ADRP_LABEL:Lloh[0-9]+]]:
 ; CHECK-NEXT: adrp [[ADRP_REG:x[0-9]+]], _D@GOTPAGE
 ; CHECK-NEXT: [[LDRGOT_LABEL:Lloh[0-9]+]]:
-; CHECK-NEXT: ldr [[LDRGOT_REG:x[0-9]+]], {{\[}}[[ADRP_REG]], _D@GOTPAGEOFF]
-; CHECK-NEXT: ldrb w0, {{\[}}[[LDRGOT_REG]]]
+; CHECK-NEXT: ldr {{[xw]}}[[LDRGOT_REG:[0-9]+]], {{\[}}[[ADRP_REG]], _D@GOTPAGEOFF]
+; CHECK-NEXT: ldrb w0, [x[[LDRGOT_REG]]]
 ; CHECK-NEXT: ret
 ; CHECK: .loh AdrpLdrGot [[ADRP_LABEL]], [[LDRGOT_LABEL]]
 define i8 @getD() {
@@ -289,9 +290,9 @@ define i8 @getD() {
 ; CHECK: [[ADRP_LABEL:Lloh[0-9]+]]:
 ; CHECK-NEXT: adrp [[ADRP_REG:x[0-9]+]], _D@GOTPAGE
 ; CHECK-NEXT: [[LDRGOT_LABEL:Lloh[0-9]+]]:
-; CHECK-NEXT: ldr [[LDRGOT_REG:x[0-9]+]], {{\[}}[[ADRP_REG]], _D@GOTPAGEOFF]
+; CHECK-NEXT: ldr {{[xw]}}[[LDRGOT_REG:[0-9]+]], {{\[}}[[ADRP_REG]], _D@GOTPAGEOFF]
 ; CHECK-NEXT: [[STR_LABEL:Lloh[0-9]+]]:
-; CHECK-NEXT: strb w0, {{\[}}[[LDRGOT_REG]]]
+; CHECK-NEXT: strb w0, [x[[LDRGOT_REG]]]
 ; CHECK-NEXT: ret
 ; CHECK: .loh AdrpLdrGotStr [[ADRP_LABEL]], [[LDRGOT_LABEL]], [[STR_LABEL]]
 define void @setD(i8 %t) {
@@ -305,9 +306,9 @@ define void @setD(i8 %t) {
 ; CHECK: [[ADRP_LABEL:Lloh[0-9]+]]:
 ; CHECK-NEXT: adrp [[ADRP_REG:x[0-9]+]], _D@GOTPAGE
 ; CHECK-NEXT: [[LDRGOT_LABEL:Lloh[0-9]+]]:
-; CHECK-NEXT: ldr [[LDRGOT_REG:x[0-9]+]], {{\[}}[[ADRP_REG]], _D@GOTPAGEOFF]
+; CHECK-NEXT: ldr {{[xw]}}[[LDRGOT_REG:[0-9]+]], {{\[}}[[ADRP_REG]], _D@GOTPAGEOFF]
 ; CHECK-NEXT: [[LDR_LABEL:Lloh[0-9]+]]:
-; CHECK-NEXT: ldrsb w0, {{\[}}[[LDRGOT_REG]]]
+; CHECK-NEXT: ldrsb w0, [x[[LDRGOT_REG]]]
 ; CHECK-NEXT: ret
 ; CHECK: .loh AdrpLdrGotLdr [[ADRP_LABEL]], [[LDRGOT_LABEL]], [[LDR_LABEL]]
 define i32 @getSExtD() {
@@ -322,9 +323,9 @@ define i32 @getSExtD() {
 ; CHECK: [[ADRP_LABEL:Lloh[0-9]+]]:
 ; CHECK-NEXT: adrp [[ADRP_REG:x[0-9]+]], _D@GOTPAGE
 ; CHECK-NEXT: [[LDRGOT_LABEL:Lloh[0-9]+]]:
-; CHECK-NEXT: ldr [[LDRGOT_REG:x[0-9]+]], {{\[}}[[ADRP_REG]], _D@GOTPAGEOFF]
+; CHECK-NEXT: ldr {{[xw]}}[[LDRGOT_REG:[0-9]+]], {{\[}}[[ADRP_REG]], _D@GOTPAGEOFF]
 ; CHECK-NEXT: [[LDR_LABEL:Lloh[0-9]+]]:
-; CHECK-NEXT: ldrsb x0, {{\[}}[[LDRGOT_REG]]]
+; CHECK-NEXT: ldrsb x0, [x[[LDRGOT_REG]]]
 ; CHECK-NEXT: ret
 ; CHECK: .loh AdrpLdrGotLdr [[ADRP_LABEL]], [[LDRGOT_LABEL]], [[LDR_LABEL]]
 define i64 @getSExt64D() {
@@ -341,8 +342,8 @@ define i64 @getSExt64D() {
 ; CHECK: [[ADRP_LABEL:Lloh[0-9]+]]:
 ; CHECK-NEXT: adrp [[ADRP_REG:x[0-9]+]], _E@GOTPAGE
 ; CHECK-NEXT: [[LDRGOT_LABEL:Lloh[0-9]+]]:
-; CHECK-NEXT: ldr [[LDRGOT_REG:x[0-9]+]], {{\[}}[[ADRP_REG]], _E@GOTPAGEOFF]
-; CHECK-NEXT: ldrh w0, {{\[}}[[LDRGOT_REG]]]
+; CHECK-NEXT: ldr {{[xw]}}[[LDRGOT_REG:[0-9]+]], {{\[}}[[ADRP_REG]], _E@GOTPAGEOFF]
+; CHECK-NEXT: ldrh w0, [x[[LDRGOT_REG]]]
 ; CHECK-NEXT: ret
 ; CHECK: .loh AdrpLdrGot [[ADRP_LABEL]], [[LDRGOT_LABEL]]
 define i16 @getE() {
@@ -356,9 +357,9 @@ define i16 @getE() {
 ; CHECK: [[ADRP_LABEL:Lloh[0-9]+]]:
 ; CHECK-NEXT: adrp [[ADRP_REG:x[0-9]+]], _E@GOTPAGE
 ; CHECK-NEXT: [[LDRGOT_LABEL:Lloh[0-9]+]]:
-; CHECK-NEXT: ldr [[LDRGOT_REG:x[0-9]+]], {{\[}}[[ADRP_REG]], _E@GOTPAGEOFF]
+; CHECK-NEXT: ldr {{[xw]}}[[LDRGOT_REG:[0-9]+]], {{\[}}[[ADRP_REG]], _E@GOTPAGEOFF]
 ; CHECK-NEXT: [[LDR_LABEL:Lloh[0-9]+]]:
-; CHECK-NEXT: ldrsh w0, {{\[}}[[LDRGOT_REG]]]
+; CHECK-NEXT: ldrsh w0, [x[[LDRGOT_REG]]]
 ; CHECK-NEXT: ret
 ; CHECK: .loh AdrpLdrGotLdr [[ADRP_LABEL]], [[LDRGOT_LABEL]], [[LDR_LABEL]]
 define i32 @getSExtE() {
@@ -371,9 +372,9 @@ define i32 @getSExtE() {
 ; CHECK: [[ADRP_LABEL:Lloh[0-9]+]]:
 ; CHECK-NEXT: adrp [[ADRP_REG:x[0-9]+]], _E@GOTPAGE
 ; CHECK-NEXT: [[LDRGOT_LABEL:Lloh[0-9]+]]:
-; CHECK-NEXT: ldr [[LDRGOT_REG:x[0-9]+]], {{\[}}[[ADRP_REG]], _E@GOTPAGEOFF]
+; CHECK-NEXT: ldr {{[xw]}}[[LDRGOT_REG:[0-9]+]], {{\[}}[[ADRP_REG]], _E@GOTPAGEOFF]
 ; CHECK-NEXT: [[STR_LABEL:Lloh[0-9]+]]:
-; CHECK-NEXT: strh w0, {{\[}}[[LDRGOT_REG]]]
+; CHECK-NEXT: strh w0, [x[[LDRGOT_REG]]]
 ; CHECK-NEXT: ret
 ; CHECK: .loh AdrpLdrGotStr [[ADRP_LABEL]], [[LDRGOT_LABEL]], [[STR_LABEL]]
 define void @setE(i16 %t) {
@@ -387,9 +388,9 @@ define void @setE(i16 %t) {
 ; CHECK: [[ADRP_LABEL:Lloh[0-9]+]]:
 ; CHECK-NEXT: adrp [[ADRP_REG:x[0-9]+]], _E@GOTPAGE
 ; CHECK-NEXT: [[LDRGOT_LABEL:Lloh[0-9]+]]:
-; CHECK-NEXT: ldr [[LDRGOT_REG:x[0-9]+]], {{\[}}[[ADRP_REG]], _E@GOTPAGEOFF]
+; CHECK-NEXT: ldr {{[xw]}}[[LDRGOT_REG:[0-9]+]], {{\[}}[[ADRP_REG]], _E@GOTPAGEOFF]
 ; CHECK-NEXT: [[LDR_LABEL:Lloh[0-9]+]]:
-; CHECK-NEXT: ldrsh x0, {{\[}}[[LDRGOT_REG]]]
+; CHECK-NEXT: ldrsh x0, [x[[LDRGOT_REG]]]
 ; CHECK-NEXT: ret
 ; CHECK: .loh AdrpLdrGotLdr [[ADRP_LABEL]], [[LDRGOT_LABEL]], [[LDR_LABEL]]
 define i64 @getSExt64E() {
@@ -406,9 +407,9 @@ define i64 @getSExt64E() {
 ; CHECK: [[ADRP_LABEL:Lloh[0-9]+]]:
 ; CHECK-NEXT: adrp [[ADRP_REG:x[0-9]+]], _F@GOTPAGE
 ; CHECK-NEXT: [[LDRGOT_LABEL:Lloh[0-9]+]]:
-; CHECK-NEXT: ldr [[LDRGOT_REG:x[0-9]+]], {{\[}}[[ADRP_REG]], _F@GOTPAGEOFF]
+; CHECK-NEXT: ldr {{[xw]}}[[LDRGOT_REG:[0-9]+]], {{\[}}[[ADRP_REG]], _F@GOTPAGEOFF]
 ; CHECK-NEXT: [[LDR_LABEL:Lloh[0-9]+]]:
-; CHECK-NEXT: ldr x0, {{\[}}[[LDRGOT_REG]]]
+; CHECK-NEXT: ldr x0, [x[[LDRGOT_REG]]]
 ; CHECK-NEXT: ret
 ; CHECK: .loh AdrpLdrGotLdr [[ADRP_LABEL]], [[LDRGOT_LABEL]], [[LDR_LABEL]]
 define i64 @getF() {
@@ -420,9 +421,9 @@ define i64 @getF() {
 ; CHECK: [[ADRP_LABEL:Lloh[0-9]+]]:
 ; CHECK-NEXT: adrp [[ADRP_REG:x[0-9]+]], _F@GOTPAGE
 ; CHECK-NEXT: [[LDRGOT_LABEL:Lloh[0-9]+]]:
-; CHECK-NEXT: ldr [[LDRGOT_REG:x[0-9]+]], {{\[}}[[ADRP_REG]], _F@GOTPAGEOFF]
+; CHECK-NEXT: ldr {{[xw]}}[[LDRGOT_REG:[0-9]+]], {{\[}}[[ADRP_REG]], _F@GOTPAGEOFF]
 ; CHECK-NEXT: [[STR_LABEL:Lloh[0-9]+]]:
-; CHECK-NEXT: str x0, {{\[}}[[LDRGOT_REG]]]
+; CHECK-NEXT: str x0, [x[[LDRGOT_REG]]]
 ; CHECK-NEXT: ret
 ; CHECK: .loh AdrpLdrGotStr [[ADRP_LABEL]], [[LDRGOT_LABEL]], [[STR_LABEL]]
 define void @setF(i64 %t) {
@@ -438,9 +439,9 @@ define void @setF(i64 %t) {
 ; CHECK: [[ADRP_LABEL:Lloh[0-9]+]]:
 ; CHECK-NEXT: adrp [[ADRP_REG:x[0-9]+]], _G@GOTPAGE
 ; CHECK-NEXT: [[LDRGOT_LABEL:Lloh[0-9]+]]:
-; CHECK-NEXT: ldr [[LDRGOT_REG:x[0-9]+]], {{\[}}[[ADRP_REG]], _G@GOTPAGEOFF]
+; CHECK-NEXT: ldr {{[xw]}}[[LDRGOT_REG:[0-9]+]], {{\[}}[[ADRP_REG]], _G@GOTPAGEOFF]
 ; CHECK-NEXT: [[LDR_LABEL:Lloh[0-9]+]]:
-; CHECK-NEXT: ldr s0, {{\[}}[[LDRGOT_REG]]]
+; CHECK-NEXT: ldr s0, [x[[LDRGOT_REG]]]
 ; CHECK-NEXT: ret
 ; CHECK: .loh AdrpLdrGotLdr [[ADRP_LABEL]], [[LDRGOT_LABEL]], [[LDR_LABEL]]
 define float @getG() {
@@ -452,9 +453,9 @@ define float @getG() {
 ; CHECK: [[ADRP_LABEL:Lloh[0-9]+]]:
 ; CHECK-NEXT: adrp [[ADRP_REG:x[0-9]+]], _G@GOTPAGE
 ; CHECK-NEXT: [[LDRGOT_LABEL:Lloh[0-9]+]]:
-; CHECK-NEXT: ldr [[LDRGOT_REG:x[0-9]+]], {{\[}}[[ADRP_REG]], _G@GOTPAGEOFF]
+; CHECK-NEXT: ldr {{[xw]}}[[LDRGOT_REG:[0-9]+]], {{\[}}[[ADRP_REG]], _G@GOTPAGEOFF]
 ; CHECK-NEXT: [[STR_LABEL:Lloh[0-9]+]]:
-; CHECK-NEXT: str s0, {{\[}}[[LDRGOT_REG]]]
+; CHECK-NEXT: str s0, [x[[LDRGOT_REG]]]
 ; CHECK-NEXT: ret
 ; CHECK: .loh AdrpLdrGotStr [[ADRP_LABEL]], [[LDRGOT_LABEL]], [[STR_LABEL]]
 define void @setG(float %t) {
@@ -470,9 +471,9 @@ define void @setG(float %t) {
 ; CHECK: [[ADRP_LABEL:Lloh[0-9]+]]:
 ; CHECK-NEXT: adrp [[ADRP_REG:x[0-9]+]], _H@GOTPAGE
 ; CHECK-NEXT: [[LDRGOT_LABEL:Lloh[0-9]+]]:
-; CHECK-NEXT: ldr [[LDRGOT_REG:x[0-9]+]], {{\[}}[[ADRP_REG]], _H@GOTPAGEOFF]
+; CHECK-NEXT: ldr {{[xw]}}[[LDRGOT_REG:[0-9]+]], {{\[}}[[ADRP_REG]], _H@GOTPAGEOFF]
 ; CHECK-NEXT: [[LDR_LABEL:Lloh[0-9]+]]:
-; CHECK-NEXT: ldr h0, {{\[}}[[LDRGOT_REG]]]
+; CHECK-NEXT: ldr h0, [x[[LDRGOT_REG]]]
 ; CHECK-NEXT: ret
 ; CHECK: .loh AdrpLdrGotLdr [[ADRP_LABEL]], [[LDRGOT_LABEL]], [[LDR_LABEL]]
 define half @getH() {
@@ -484,9 +485,9 @@ define half @getH() {
 ; CHECK: [[ADRP_LABEL:Lloh[0-9]+]]:
 ; CHECK-NEXT: adrp [[ADRP_REG:x[0-9]+]], _H@GOTPAGE
 ; CHECK-NEXT: [[LDRGOT_LABEL:Lloh[0-9]+]]:
-; CHECK-NEXT: ldr [[LDRGOT_REG:x[0-9]+]], {{\[}}[[ADRP_REG]], _H@GOTPAGEOFF]
+; CHECK-NEXT: ldr {{[xw]}}[[LDRGOT_REG:[0-9]+]], {{\[}}[[ADRP_REG]], _H@GOTPAGEOFF]
 ; CHECK-NEXT: [[STR_LABEL:Lloh[0-9]+]]:
-; CHECK-NEXT: str h0, {{\[}}[[LDRGOT_REG]]]
+; CHECK-NEXT: str h0, [x[[LDRGOT_REG]]]
 ; CHECK-NEXT: ret
 ; CHECK: .loh AdrpLdrGotStr [[ADRP_LABEL]], [[LDRGOT_LABEL]], [[STR_LABEL]]
 define void @setH(half %t) {
@@ -502,9 +503,9 @@ define void @setH(half %t) {
 ; CHECK: [[ADRP_LABEL:Lloh[0-9]+]]:
 ; CHECK-NEXT: adrp [[ADRP_REG:x[0-9]+]], _I@GOTPAGE
 ; CHECK-NEXT: [[LDRGOT_LABEL:Lloh[0-9]+]]:
-; CHECK-NEXT: ldr [[LDRGOT_REG:x[0-9]+]], {{\[}}[[ADRP_REG]], _I@GOTPAGEOFF]
+; CHECK-NEXT: ldr {{[xw]}}[[LDRGOT_REG:[0-9]+]], {{\[}}[[ADRP_REG]], _I@GOTPAGEOFF]
 ; CHECK-NEXT: [[LDR_LABEL:Lloh[0-9]+]]:
-; CHECK-NEXT: ldr d0, {{\[}}[[LDRGOT_REG]]]
+; CHECK-NEXT: ldr d0, [x[[LDRGOT_REG]]]
 ; CHECK-NEXT: ret
 ; CHECK: .loh AdrpLdrGotLdr [[ADRP_LABEL]], [[LDRGOT_LABEL]], [[LDR_LABEL]]
 define double @getI() {
@@ -516,9 +517,9 @@ define double @getI() {
 ; CHECK: [[ADRP_LABEL:Lloh[0-9]+]]:
 ; CHECK-NEXT: adrp [[ADRP_REG:x[0-9]+]], _I@GOTPAGE
 ; CHECK-NEXT: [[LDRGOT_LABEL:Lloh[0-9]+]]:
-; CHECK-NEXT: ldr [[LDRGOT_REG:x[0-9]+]], {{\[}}[[ADRP_REG]], _I@GOTPAGEOFF]
+; CHECK-NEXT: ldr {{[xw]}}[[LDRGOT_REG:[0-9]+]], {{\[}}[[ADRP_REG]], _I@GOTPAGEOFF]
 ; CHECK-NEXT: [[STR_LABEL:Lloh[0-9]+]]:
-; CHECK-NEXT: str d0, {{\[}}[[LDRGOT_REG]]]
+; CHECK-NEXT: str d0, [x[[LDRGOT_REG]]]
 ; CHECK-NEXT: ret
 ; CHECK: .loh AdrpLdrGotStr [[ADRP_LABEL]], [[LDRGOT_LABEL]], [[STR_LABEL]]
 define void @setI(double %t) {
@@ -534,9 +535,9 @@ define void @setI(double %t) {
 ; CHECK: [[ADRP_LABEL:Lloh[0-9]+]]:
 ; CHECK-NEXT: adrp [[ADRP_REG:x[0-9]+]], _J@GOTPAGE
 ; CHECK-NEXT: [[LDRGOT_LABEL:Lloh[0-9]+]]:
-; CHECK-NEXT: ldr [[LDRGOT_REG:x[0-9]+]], {{\[}}[[ADRP_REG]], _J@GOTPAGEOFF]
+; CHECK-NEXT: ldr {{[xw]}}[[LDRGOT_REG:[0-9]+]], {{\[}}[[ADRP_REG]], _J@GOTPAGEOFF]
 ; CHECK-NEXT: [[LDR_LABEL:Lloh[0-9]+]]:
-; CHECK-NEXT: ldr d0, {{\[}}[[LDRGOT_REG]]]
+; CHECK-NEXT: ldr d0, [x[[LDRGOT_REG]]]
 ; CHECK-NEXT: ret
 ; CHECK: .loh AdrpLdrGotLdr [[ADRP_LABEL]], [[LDRGOT_LABEL]], [[LDR_LABEL]]
 define <2 x i32> @getJ() {
@@ -548,9 +549,9 @@ define <2 x i32> @getJ() {
 ; CHECK: [[ADRP_LABEL:Lloh[0-9]+]]:
 ; CHECK-NEXT: adrp [[ADRP_REG:x[0-9]+]], _J@GOTPAGE
 ; CHECK-NEXT: [[LDRGOT_LABEL:Lloh[0-9]+]]:
-; CHECK-NEXT: ldr [[LDRGOT_REG:x[0-9]+]], {{\[}}[[ADRP_REG]], _J@GOTPAGEOFF]
+; CHECK-NEXT: ldr {{[xw]}}[[LDRGOT_REG:[0-9]+]], {{\[}}[[ADRP_REG]], _J@GOTPAGEOFF]
 ; CHECK-NEXT: [[STR_LABEL:Lloh[0-9]+]]:
-; CHECK-NEXT: str d0, {{\[}}[[LDRGOT_REG]]]
+; CHECK-NEXT: str d0, [x[[LDRGOT_REG]]]
 ; CHECK-NEXT: ret
 ; CHECK: .loh AdrpLdrGotStr [[ADRP_LABEL]], [[LDRGOT_LABEL]], [[STR_LABEL]]
 define void @setJ(<2 x i32> %t) {
@@ -566,9 +567,9 @@ define void @setJ(<2 x i32> %t) {
 ; CHECK: [[ADRP_LABEL:Lloh[0-9]+]]:
 ; CHECK-NEXT: adrp [[ADRP_REG:x[0-9]+]], _K@GOTPAGE
 ; CHECK-NEXT: [[LDRGOT_LABEL:Lloh[0-9]+]]:
-; CHECK-NEXT: ldr [[LDRGOT_REG:x[0-9]+]], {{\[}}[[ADRP_REG]], _K@GOTPAGEOFF]
+; CHECK-NEXT: ldr {{[xw]}}[[LDRGOT_REG:[0-9]+]], {{\[}}[[ADRP_REG]], _K@GOTPAGEOFF]
 ; CHECK-NEXT: [[LDR_LABEL:Lloh[0-9]+]]:
-; CHECK-NEXT: ldr q0, {{\[}}[[LDRGOT_REG]]]
+; CHECK-NEXT: ldr q0, [x[[LDRGOT_REG]]]
 ; CHECK-NEXT: ret
 ; CHECK: .loh AdrpLdrGotLdr [[ADRP_LABEL]], [[LDRGOT_LABEL]], [[LDR_LABEL]]
 define <4 x i32> @getK() {
@@ -580,9 +581,9 @@ define <4 x i32> @getK() {
 ; CHECK: [[ADRP_LABEL:Lloh[0-9]+]]:
 ; CHECK-NEXT: adrp [[ADRP_REG:x[0-9]+]], _K@GOTPAGE
 ; CHECK-NEXT: [[LDRGOT_LABEL:Lloh[0-9]+]]:
-; CHECK-NEXT: ldr [[LDRGOT_REG:x[0-9]+]], {{\[}}[[ADRP_REG]], _K@GOTPAGEOFF]
+; CHECK-NEXT: ldr {{[xw]}}[[LDRGOT_REG:[0-9]+]], {{\[}}[[ADRP_REG]], _K@GOTPAGEOFF]
 ; CHECK-NEXT: [[STR_LABEL:Lloh[0-9]+]]:
-; CHECK-NEXT: str q0, {{\[}}[[LDRGOT_REG]]]
+; CHECK-NEXT: str q0, [x[[LDRGOT_REG]]]
 ; CHECK-NEXT: ret
 ; CHECK: .loh AdrpLdrGotStr [[ADRP_LABEL]], [[LDRGOT_LABEL]], [[STR_LABEL]]
 define void @setK(<4 x i32> %t) {
@@ -598,9 +599,9 @@ define void @setK(<4 x i32> %t) {
 ; CHECK: [[ADRP_LABEL:Lloh[0-9]+]]:
 ; CHECK-NEXT: adrp [[ADRP_REG:x[0-9]+]], _L@GOTPAGE
 ; CHECK-NEXT: [[LDRGOT_LABEL:Lloh[0-9]+]]:
-; CHECK-NEXT: ldr [[LDRGOT_REG:x[0-9]+]], {{\[}}[[ADRP_REG]], _L@GOTPAGEOFF]
+; CHECK-NEXT: ldr {{[xw]}}[[LDRGOT_REG:[0-9]+]], {{\[}}[[ADRP_REG]], _L@GOTPAGEOFF]
 ; CHECK-NEXT: [[LDR_LABEL:Lloh[0-9]+]]:
-; CHECK-NEXT: ldr b0, {{\[}}[[LDRGOT_REG]]]
+; CHECK-NEXT: ldr b0, [x[[LDRGOT_REG]]]
 ; CHECK-NEXT: ret
 ; CHECK: .loh AdrpLdrGotLdr [[ADRP_LABEL]], [[LDRGOT_LABEL]], [[LDR_LABEL]]
 define <1 x i8> @getL() {
@@ -612,11 +613,11 @@ define <1 x i8> @getL() {
 ; CHECK: [[ADRP_LABEL:Lloh[0-9]+]]:
 ; CHECK-NEXT: adrp [[ADRP_REG:x[0-9]+]], _L@GOTPAGE
 ; CHECK-NEXT: [[LDRGOT_LABEL:Lloh[0-9]+]]:
-; CHECK-NEXT: ldr [[LDRGOT_REG:x[0-9]+]], {{\[}}[[ADRP_REG]], _L@GOTPAGEOFF]
+; CHECK-NEXT: ldr {{[xw]}}[[LDRGOT_REG:[0-9]+]], {{\[}}[[ADRP_REG]], _L@GOTPAGEOFF]
 ; CHECK-NEXT: ; kill
 ; Ultimately we should generate str b0, but right now, we match the vector
 ; variant which does not allow to fold the immediate into the store.
-; CHECK-NEXT: st1.b { v0 }[0], {{\[}}[[LDRGOT_REG]]]
+; CHECK-NEXT: st1.b { v0 }[0], [x[[LDRGOT_REG]]]
 ; CHECK-NEXT: ret
 ; CHECK: .loh AdrpLdrGot [[ADRP_LABEL]], [[LDRGOT_LABEL]]
 define void @setL(<1 x i8> %t) {
diff --git a/llvm/test/CodeGen/AArch64/arm64-detect-vec-redux.ll b/llvm/test/CodeGen/AArch64/arm64-detect-vec-redux.ll
index 68130f1c9..4e86f5258 100644
--- a/llvm/test/CodeGen/AArch64/arm64-detect-vec-redux.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-detect-vec-redux.ll
@@ -47,6 +47,6 @@ declare <2 x i64> @llvm.aarch64.neon.addp.v2i64(<2 x i64>, <2 x i64>) #1
 ; Function Attrs: nounwind readnone
 declare <2 x i32> @llvm.aarch64.neon.sqdmulh.v2i32(<2 x i32>, <2 x i32>) #1
 
-attributes #0 = { nounwind readnone "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="generic" "target-features"="+neon" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind readnone "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="generic" "target-features"="+neon" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 attributes #2 = { nounwind }
diff --git a/llvm/test/CodeGen/AArch64/arm64-fast-isel-addr-offset.ll b/llvm/test/CodeGen/AArch64/arm64-fast-isel-addr-offset.ll
index 4aa10da72..1985a72b6 100644
--- a/llvm/test/CodeGen/AArch64/arm64-fast-isel-addr-offset.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-fast-isel-addr-offset.ll
@@ -1,4 +1,4 @@
-; RUN: llc -O0 -fast-isel-abort=1 -verify-machineinstrs -mtriple=arm64-apple-darwin < %s | FileCheck %s
+; RUN: llc -O0 -fast-isel -fast-isel-abort=1 -verify-machineinstrs -mtriple=arm64-apple-darwin < %s | FileCheck %s
 
 @sortlist = common global [5001 x i32] zeroinitializer, align 16
 @sortlist2 = common global [5001 x i64] zeroinitializer, align 16
diff --git a/llvm/test/CodeGen/AArch64/arm64-fastisel-gep-promote-before-add.ll b/llvm/test/CodeGen/AArch64/arm64-fastisel-gep-promote-before-add.ll
index 8268bcff1..da92c6da6 100644
--- a/llvm/test/CodeGen/AArch64/arm64-fastisel-gep-promote-before-add.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-fastisel-gep-promote-before-add.ll
@@ -1,6 +1,6 @@
 ; fastisel should not fold add with non-pointer bitwidth
 ; sext(a) + sext(b) != sext(a + b)
-; RUN: llc -mtriple=arm64-apple-darwin %s -O0 -o - | FileCheck %s
+; RUN: llc -fast-isel -mtriple=arm64-apple-darwin %s -O0 -o - | FileCheck %s
 
 define zeroext i8 @gep_promotion(i8* %ptr) nounwind uwtable ssp {
 entry:
diff --git a/llvm/test/CodeGen/AArch64/arm64-fma-combine-with-fpfusion.ll b/llvm/test/CodeGen/AArch64/arm64-fma-combine-with-fpfusion.ll
index 095a0b0ed..9b3d53911 100644
--- a/llvm/test/CodeGen/AArch64/arm64-fma-combine-with-fpfusion.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-fma-combine-with-fpfusion.ll
@@ -8,5 +8,5 @@ define float @mul_add(float %a, float %b, float %c) local_unnamed_addr #0 {
     ret float %add
 }
 
-attributes #0 = { norecurse nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="generic" "target-features"="+neon" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { norecurse nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="generic" "target-features"="+neon" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
diff --git a/llvm/test/CodeGen/AArch64/arm64-fmadd.ll b/llvm/test/CodeGen/AArch64/arm64-fmadd.ll
index 203ce6236..dffa83aa1 100644
--- a/llvm/test/CodeGen/AArch64/arm64-fmadd.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-fmadd.ll
@@ -88,5 +88,23 @@ entry:
   ret double %0
 }
 
+; This would crash while trying getNegatedExpression().
+
+define float @negated_constant(float %x) {
+; CHECK-LABEL: negated_constant:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #-1037565952
+; CHECK-NEXT:    mov w9, #1109917696
+; CHECK-NEXT:    fmov s1, w8
+; CHECK-NEXT:    fmul s1, s0, s1
+; CHECK-NEXT:    fmov s2, w9
+; CHECK-NEXT:    fmadd s0, s0, s2, s1
+; CHECK-NEXT:    ret
+  %m = fmul float %x, 42.0
+  %fma = call nsz float @llvm.fma.f32(float %x, float -42.0, float %m)
+  %nfma = fneg float %fma
+  ret float %nfma
+}
+
 declare float @llvm.fma.f32(float, float, float) nounwind readnone
 declare double @llvm.fma.f64(double, double, double) nounwind readnone
diff --git a/llvm/test/CodeGen/AArch64/arm64-fp.ll b/llvm/test/CodeGen/AArch64/arm64-fp.ll
index 1c88b3d90..85c5f25fa 100644
--- a/llvm/test/CodeGen/AArch64/arm64-fp.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-fp.ll
@@ -1,8 +1,51 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=arm64-eabi | FileCheck %s
 
 define float @t1(i1 %a, float %b, float %c) nounwind {
-; CHECK: t1
-; CHECK: fcsel	s0, s0, s1, ne
+; CHECK-LABEL: t1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    tst w0, #0x1
+; CHECK-NEXT:    fcsel s0, s0, s1, ne
+; CHECK-NEXT:    ret
   %sel = select i1 %a, float %b, float %c
   ret float %sel
 }
+
+; This may infinite loop if isNegatibleForFree and getNegatedExpression are conflicted.
+
+define double @negation_propagation(double* %arg, double %arg1, double %arg2) {
+; CHECK-LABEL: negation_propagation:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmov d2, #1.00000000
+; CHECK-NEXT:    fdiv d0, d2, d0
+; CHECK-NEXT:    fmul d2, d0, d0
+; CHECK-NEXT:    fmul d1, d0, d1
+; CHECK-NEXT:    fmul d0, d0, d2
+; CHECK-NEXT:    fsub d0, d1, d0
+; CHECK-NEXT:    ret
+  %t = fdiv double 1.0, %arg1
+  %t7 = fmul double %t, %arg2
+  %t10 = fneg double %t7
+  %t11 = fmul double %t, %t
+  %t13 = fsub double %t11, %t
+  %t14 = fneg double %t
+  %t15 = fmul double %t, %t14
+  %t16 = fmul double %t, %t15
+  %t18 = fadd double %t16, %t7
+  ret double %t18
+}
+
+define { double, double } @testfn(double %x, double %y) #0 {
+; CHECK-LABEL: testfn:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fsub d0, d0, d1
+; CHECK-NEXT:    fneg d1, d0
+; CHECK-NEXT:    ret
+  %sub = fsub fast double %x, %y
+  %neg = fneg fast double %sub
+  %r0 = insertvalue { double, double } undef, double %sub, 0
+  %r1 = insertvalue { double, double } %r0, double %neg, 1
+  ret { double, double } %r1
+}
+
+attributes #0 = { "no-signed-zeros-fp-math"="true" }
diff --git a/llvm/test/CodeGen/AArch64/arm64-indexed-memory.ll b/llvm/test/CodeGen/AArch64/arm64-indexed-memory.ll
index 7dcd6e25a..018a1143f 100644
--- a/llvm/test/CodeGen/AArch64/arm64-indexed-memory.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-indexed-memory.ll
@@ -1,4 +1,5 @@
 ; RUN: llc < %s -mtriple=arm64-eabi -aarch64-redzone | FileCheck %s
+; RUN: llc < %s -mtriple=arm64_32-apple-ios -aarch64-redzone | FileCheck %s
 
 define i64* @store64(i64* %ptr, i64 %index, i64 %spacing) {
 ; CHECK-LABEL: store64:
diff --git a/llvm/test/CodeGen/AArch64/arm64-indexed-vector-ldst-2.ll b/llvm/test/CodeGen/AArch64/arm64-indexed-vector-ldst-2.ll
index 1032a6d62..74f78e49f 100644
--- a/llvm/test/CodeGen/AArch64/arm64-indexed-vector-ldst-2.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-indexed-vector-ldst-2.ll
@@ -53,7 +53,7 @@ define <4 x i32> @f3(i32 *%p, <4 x i1> %m, <4 x i32> %v1, <4 x i32> %v2) {
 ; Function Attrs: nounwind readnone
 declare i64 @llvm.objectsize.i64.p0i8(i8*, i1) #1
 
-attributes #0 = { nounwind ssp "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind ssp "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !1 = !{!2, !2, i64 0}
diff --git a/llvm/test/CodeGen/AArch64/arm64-inline-asm.ll b/llvm/test/CodeGen/AArch64/arm64-inline-asm.ll
index b1ec988c5..3b8b4d848 100644
--- a/llvm/test/CodeGen/AArch64/arm64-inline-asm.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-inline-asm.ll
@@ -81,9 +81,9 @@ define i32 @constraint_I(i32 %i, i32 %j) nounwind {
 entry:
   ; CHECK-LABEL: constraint_I:
   %0 = tail call i32 asm sideeffect "add ${0:w}, ${1:w}, $2", "=r,r,I"(i32 %i, i32 16773120) nounwind
-  ; CHECK: add   {{w[0-9]+}}, {{w[0-9]+}}, #16773120
+  ; CHECK: add   {{w[0-9]+}}, {{w[0-9]+}}, 16773120
   %1 = tail call i32 asm sideeffect "add ${0:w}, ${1:w}, $2", "=r,r,I"(i32 %i, i32 4096) nounwind
-  ; CHECK: add   {{w[0-9]+}}, {{w[0-9]+}}, #4096
+  ; CHECK: add   {{w[0-9]+}}, {{w[0-9]+}}, 4096
   ret i32 %1
 }
 
@@ -91,13 +91,13 @@ define i32 @constraint_J(i32 %i, i32 %j, i64 %k) nounwind {
 entry:
   ; CHECK-LABEL: constraint_J:
   %0 = tail call i32 asm sideeffect "sub ${0:w}, ${1:w}, $2", "=r,r,J"(i32 %i, i32 -16773120) nounwind
-  ; CHECK: sub   {{w[0-9]+}}, {{w[0-9]+}}, #-16773120
+  ; CHECK: sub   {{w[0-9]+}}, {{w[0-9]+}}, -16773120
   %1 = tail call i32 asm sideeffect "sub ${0:w}, ${1:w}, $2", "=r,r,J"(i32 %i, i32 -1) nounwind
-  ; CHECK: sub   {{w[0-9]+}}, {{w[0-9]+}}, #-1
+  ; CHECK: sub   {{w[0-9]+}}, {{w[0-9]+}}, -1
   %2 = tail call i64 asm sideeffect "sub ${0:x}, ${1:x}, $2", "=r,r,J"(i64 %k, i32 -1) nounwind
-  ; CHECK: sub   {{x[0-9]+}}, {{x[0-9]+}}, #-1
+  ; CHECK: sub   {{x[0-9]+}}, {{x[0-9]+}}, -1
   %3 = tail call i64 asm sideeffect "sub ${0:x}, ${1:x}, $2", "=r,r,J"(i64 %k, i64 -1) nounwind
-  ; CHECK: sub   {{x[0-9]+}}, {{x[0-9]+}}, #-1
+  ; CHECK: sub   {{x[0-9]+}}, {{x[0-9]+}}, -1
   ret i32 %1
 }
 
@@ -105,9 +105,9 @@ define i32 @constraint_KL(i32 %i, i32 %j) nounwind {
 entry:
   ; CHECK-LABEL: constraint_KL:
   %0 = tail call i32 asm sideeffect "eor ${0:w}, ${1:w}, $2", "=r,r,K"(i32 %i, i32 255) nounwind
-  ; CHECK: eor {{w[0-9]+}}, {{w[0-9]+}}, #255
+  ; CHECK: eor {{w[0-9]+}}, {{w[0-9]+}}, 255
   %1 = tail call i32 asm sideeffect "eor ${0:w}, ${1:w}, $2", "=r,r,L"(i32 %i, i64 16711680) nounwind
-  ; CHECK: eor {{w[0-9]+}}, {{w[0-9]+}}, #16711680
+  ; CHECK: eor {{w[0-9]+}}, {{w[0-9]+}}, 16711680
   ret i32 %1
 }
 
@@ -115,9 +115,9 @@ define i32 @constraint_MN(i32 %i, i32 %j) nounwind {
 entry:
   ; CHECK-LABEL: constraint_MN:
   %0 = tail call i32 asm sideeffect "movk ${0:w}, $1", "=r,M"(i32 65535) nounwind
-  ; CHECK: movk  {{w[0-9]+}}, #65535
+  ; CHECK: movk  {{w[0-9]+}}, 65535
   %1 = tail call i32 asm sideeffect "movz ${0:w}, $1", "=r,N"(i64 0) nounwind
-  ; CHECK: movz  {{w[0-9]+}}, #0
+  ; CHECK: movz  {{w[0-9]+}}, 0
   ret i32 %1
 }
 
@@ -138,6 +138,8 @@ entry:
   %a = alloca [2 x float], align 4
   %arraydecay = getelementptr inbounds [2 x float], [2 x float]* %a, i32 0, i32 0
   %0 = load <2 x float>, <2 x float>* %data, align 8
+  call void asm sideeffect "ldr ${1:z}, [$0]\0A", "r,w"(float* %arraydecay, <2 x float> %0) nounwind
+  ; CHECK: ldr {{z[0-9]+}}, [{{x[0-9]+}}]
   call void asm sideeffect "ldr ${1:q}, [$0]\0A", "r,w"(float* %arraydecay, <2 x float> %0) nounwind
   ; CHECK: ldr {{q[0-9]+}}, [{{x[0-9]+}}]
   call void asm sideeffect "ldr ${1:d}, [$0]\0A", "r,w"(float* %arraydecay, <2 x float> %0) nounwind
@@ -178,13 +180,13 @@ define void @t13() nounwind {
 entry:
   ; CHECK-LABEL: t13:
   tail call void asm sideeffect "mov x4, $0\0A", "N"(i64 1311673391471656960) nounwind
-  ; CHECK: mov x4, #1311673391471656960
+  ; CHECK: mov x4, 1311673391471656960
   tail call void asm sideeffect "mov x4, $0\0A", "N"(i64 -4662) nounwind
-  ; CHECK: mov x4, #-4662
+  ; CHECK: mov x4, -4662
   tail call void asm sideeffect "mov x4, $0\0A", "N"(i64 4660) nounwind
-  ; CHECK: mov x4, #4660
+  ; CHECK: mov x4, 4660
   call void asm sideeffect "mov x4, $0\0A", "N"(i64 -71777214294589696) nounwind
-  ; CHECK: mov x4, #-71777214294589696
+  ; CHECK: mov x4, -71777214294589696
   ret void
 }
 
@@ -192,13 +194,13 @@ define void @t14() nounwind {
 entry:
   ; CHECK-LABEL: t14:
   tail call void asm sideeffect "mov w4, $0\0A", "M"(i32 305397760) nounwind
-  ; CHECK: mov w4, #305397760
+  ; CHECK: mov w4, 305397760
   tail call void asm sideeffect "mov w4, $0\0A", "M"(i32 -4662) nounwind
-  ; CHECK: mov w4, #4294962634
+  ; CHECK: mov w4, 4294962634
   tail call void asm sideeffect "mov w4, $0\0A", "M"(i32 4660) nounwind
-  ; CHECK: mov w4, #4660
+  ; CHECK: mov w4, 4660
   call void asm sideeffect "mov w4, $0\0A", "M"(i32 -16711936) nounwind
-  ; CHECK: mov w4, #4278255360
+  ; CHECK: mov w4, 4278255360
   ret void
 }
 
@@ -271,3 +273,12 @@ entry:
   tail call i32 asm sideeffect "ldr $0, $1 \0A", "=r,*Q"(i32* null)
   ret void
 }
+
+; No '#' in lane specifier
+define void @test_no_hash_in_lane_specifier() {
+; CHECK-LABEL: test_no_hash_in_lane_specifier
+; CHECK: fmla v2.4s, v0.4s, v1.s[1]
+; CHECK: ret
+  tail call void asm sideeffect "fmla v2.4s, v0.4s, v1.s[$0]", "I"(i32 1) #1
+  ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/arm64-large-frame.ll b/llvm/test/CodeGen/AArch64/arm64-large-frame.ll
index cfda00ceb..897d6e586 100644
--- a/llvm/test/CodeGen/AArch64/arm64-large-frame.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-large-frame.ll
@@ -1,4 +1,4 @@
-; RUN: llc -verify-machineinstrs -mtriple=arm64-none-linux-gnu -frame-pointer=all -disable-post-ra < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -mtriple=arm64-none-linux-gnu -frame-pointer=non-leaf -disable-post-ra < %s | FileCheck %s
 declare void @use_addr(i8*)
 
 @addr = global i8* null
diff --git a/llvm/test/CodeGen/AArch64/arm64-ldp-cluster.ll b/llvm/test/CodeGen/AArch64/arm64-ldp-cluster.ll
index 80b677706..a45373a1d 100644
--- a/llvm/test/CodeGen/AArch64/arm64-ldp-cluster.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-ldp-cluster.ll
@@ -1,6 +1,5 @@
 ; REQUIRES: asserts
 ; RUN: llc < %s -mtriple=arm64-linux-gnu -mcpu=cortex-a57 -verify-misched -debug-only=machine-scheduler -o - 2>&1 > /dev/null | FileCheck %s
-; RUN: llc < %s -mtriple=arm64-linux-gnu -mcpu=exynos-m1 -verify-misched -debug-only=machine-scheduler -o - 2>&1 > /dev/null | FileCheck --check-prefix=EXYNOSM1 %s
 ; RUN: llc < %s -mtriple=arm64-linux-gnu -mcpu=exynos-m3 -verify-misched -debug-only=machine-scheduler -o - 2>&1 > /dev/null | FileCheck %s
 
 ; Test ldr clustering.
@@ -9,11 +8,6 @@
 ; CHECK: Cluster ld/st SU(1) - SU(2)
 ; CHECK: SU(1):   %{{[0-9]+}}:gpr32 = LDRWui
 ; CHECK: SU(2):   %{{[0-9]+}}:gpr32 = LDRWui
-; EXYNOSM1: ********** MI Scheduling **********
-; EXYNOSM1-LABEL: ldr_int:%bb.0
-; EXYNOSM1: Cluster ld/st SU(1) - SU(2)
-; EXYNOSM1: SU(1):   %{{[0-9]+}}:gpr32 = LDRWui
-; EXYNOSM1: SU(2):   %{{[0-9]+}}:gpr32 = LDRWui
 define i32 @ldr_int(i32* %a) nounwind {
   %p1 = getelementptr inbounds i32, i32* %a, i32 1
   %tmp1 = load i32, i32* %p1, align 2
@@ -29,11 +23,6 @@ define i32 @ldr_int(i32* %a) nounwind {
 ; CHECK: Cluster ld/st SU(1) - SU(2)
 ; CHECK: SU(1):   %{{[0-9]+}}:gpr64 = LDRSWui
 ; CHECK: SU(2):   %{{[0-9]+}}:gpr64 = LDRSWui
-; EXYNOSM1: ********** MI Scheduling **********
-; EXYNOSM1-LABEL: ldp_sext_int:%bb.0
-; EXYNOSM1: Cluster ld/st SU(1) - SU(2)
-; EXYNOSM1: SU(1):   %{{[0-9]+}}:gpr64 = LDRSWui
-; EXYNOSM1: SU(2):   %{{[0-9]+}}:gpr64 = LDRSWui
 define i64 @ldp_sext_int(i32* %p) nounwind {
   %tmp = load i32, i32* %p, align 4
   %add.ptr = getelementptr inbounds i32, i32* %p, i64 1
@@ -47,14 +36,9 @@ define i64 @ldp_sext_int(i32* %p) nounwind {
 ; Test ldur clustering.
 ; CHECK: ********** MI Scheduling **********
 ; CHECK-LABEL: ldur_int:%bb.0
-; CHECK: Cluster ld/st SU(2) - SU(1)
+; CHECK: Cluster ld/st SU(1) - SU(2)
 ; CHECK: SU(1):   %{{[0-9]+}}:gpr32 = LDURWi
 ; CHECK: SU(2):   %{{[0-9]+}}:gpr32 = LDURWi
-; EXYNOSM1: ********** MI Scheduling **********
-; EXYNOSM1-LABEL: ldur_int:%bb.0
-; EXYNOSM1: Cluster ld/st SU(2) - SU(1)
-; EXYNOSM1: SU(1):   %{{[0-9]+}}:gpr32 = LDURWi
-; EXYNOSM1: SU(2):   %{{[0-9]+}}:gpr32 = LDURWi
 define i32 @ldur_int(i32* %a) nounwind {
   %p1 = getelementptr inbounds i32, i32* %a, i32 -1
   %tmp1 = load i32, i32* %p1, align 2
@@ -70,11 +54,6 @@ define i32 @ldur_int(i32* %a) nounwind {
 ; CHECK: Cluster ld/st SU(3) - SU(4)
 ; CHECK: SU(3):   %{{[0-9]+}}:gpr64 = LDRSWui
 ; CHECK: SU(4):   undef %{{[0-9]+}}.sub_32:gpr64 = LDRWui
-; EXYNOSM1: ********** MI Scheduling **********
-; EXYNOSM1-LABEL: ldp_half_sext_zext_int:%bb.0
-; EXYNOSM1: Cluster ld/st SU(3) - SU(4)
-; EXYNOSM1: SU(3):   %{{[0-9]+}}:gpr64 = LDRSWui
-; EXYNOSM1: SU(4):   undef %{{[0-9]+}}.sub_32:gpr64 = LDRWui
 define i64 @ldp_half_sext_zext_int(i64* %q, i32* %p) nounwind {
   %tmp0 = load i64, i64* %q, align 4
   %tmp = load i32, i32* %p, align 4
@@ -93,11 +72,6 @@ define i64 @ldp_half_sext_zext_int(i64* %q, i32* %p) nounwind {
 ; CHECK: Cluster ld/st SU(3) - SU(4)
 ; CHECK: SU(3):   undef %{{[0-9]+}}.sub_32:gpr64 = LDRWui
 ; CHECK: SU(4):   %{{[0-9]+}}:gpr64 = LDRSWui
-; EXYNOSM1: ********** MI Scheduling **********
-; EXYNOSM1-LABEL: ldp_half_zext_sext_int:%bb.0
-; EXYNOSM1: Cluster ld/st SU(3) - SU(4)
-; EXYNOSM1: SU(3):   undef %{{[0-9]+}}.sub_32:gpr64 = LDRWui
-; EXYNOSM1: SU(4):   %{{[0-9]+}}:gpr64 = LDRSWui
 define i64 @ldp_half_zext_sext_int(i64* %q, i32* %p) nounwind {
   %tmp0 = load i64, i64* %q, align 4
   %tmp = load i32, i32* %p, align 4
@@ -116,11 +90,6 @@ define i64 @ldp_half_zext_sext_int(i64* %q, i32* %p) nounwind {
 ; CHECK-NOT: Cluster ld/st
 ; CHECK: SU(1):   %{{[0-9]+}}:gpr32 = LDRWui
 ; CHECK: SU(2):   %{{[0-9]+}}:gpr32 = LDRWui
-; EXYNOSM1: ********** MI Scheduling **********
-; EXYNOSM1-LABEL: ldr_int_volatile:%bb.0
-; EXYNOSM1-NOT: Cluster ld/st
-; EXYNOSM1: SU(1):   %{{[0-9]+}}:gpr32 = LDRWui
-; EXYNOSM1: SU(2):   %{{[0-9]+}}:gpr32 = LDRWui
 define i32 @ldr_int_volatile(i32* %a) nounwind {
   %p1 = getelementptr inbounds i32, i32* %a, i32 1
   %tmp1 = load volatile i32, i32* %p1, align 2
@@ -136,9 +105,6 @@ define i32 @ldr_int_volatile(i32* %a) nounwind {
 ; CHECK: Cluster ld/st SU(1) - SU(3)
 ; CHECK: SU(1):   %{{[0-9]+}}:fpr128 = LDRQui
 ; CHECK: SU(3):   %{{[0-9]+}}:fpr128 = LDRQui
-; EXYNOSM1: ********** MI Scheduling **********
-; EXYNOSM1-LABEL: ldq_cluster:%bb.0
-; EXYNOSM1-NOT: Cluster ld/st
 define <2 x i64> @ldq_cluster(i64* %p) {
   %a1 = bitcast i64* %p to <2 x i64>*
   %tmp1 = load <2 x i64>, < 2 x i64>* %a1, align 8
diff --git a/llvm/test/CodeGen/AArch64/arm64-ldxr-stxr.ll b/llvm/test/CodeGen/AArch64/arm64-ldxr-stxr.ll
index 4f87121c6..aa9c5dfff 100644
--- a/llvm/test/CodeGen/AArch64/arm64-ldxr-stxr.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-ldxr-stxr.ll
@@ -33,6 +33,7 @@ declare i32 @llvm.aarch64.stxp(i64, i64, i8*) nounwind
 
 @var = global i64 0, align 8
 
+; FALLBACK-NOT: remark:{{.*}}test_load_i8
 define void @test_load_i8(i8* %addr) {
 ; CHECK-LABEL: test_load_i8:
 ; CHECK: ldxrb w[[LOADVAL:[0-9]+]], [x0]
@@ -40,6 +41,12 @@ define void @test_load_i8(i8* %addr) {
 ; CHECK-NOT: and
 ; CHECK: str x[[LOADVAL]], [{{x[0-9]+}}, :lo12:var]
 
+; FIXME: GlobalISel doesn't fold ands/adds into load/store addressing modes
+; right now/ So, we won't get the :lo12:var.
+; GISEL-LABEL: test_load_i8:
+; GISEL: ldxrb w[[LOADVAL:[0-9]+]], [x0]
+; GISEL-NOT: uxtb
+; GISEL: str x[[LOADVAL]], [{{x[0-9]+}}]
   %val = call i64 @llvm.aarch64.ldxr.p0i8(i8* %addr)
   %shortval = trunc i64 %val to i8
   %extval = zext i8 %shortval to i64
@@ -47,6 +54,7 @@ define void @test_load_i8(i8* %addr) {
   ret void
 }
 
+; FALLBACK-NOT: remark:{{.*}}test_load_i16
 define void @test_load_i16(i16* %addr) {
 ; CHECK-LABEL: test_load_i16:
 ; CHECK: ldxrh w[[LOADVAL:[0-9]+]], [x0]
@@ -54,6 +62,10 @@ define void @test_load_i16(i16* %addr) {
 ; CHECK-NOT: and
 ; CHECK: str x[[LOADVAL]], [{{x[0-9]+}}, :lo12:var]
 
+; GISEL-LABEL: test_load_i16:
+; GISEL: ldxrh w[[LOADVAL:[0-9]+]], [x0]
+; GISEL-NOT: uxtb
+; GISEL: str x[[LOADVAL]], [{{x[0-9]+}}]
   %val = call i64 @llvm.aarch64.ldxr.p0i16(i16* %addr)
   %shortval = trunc i64 %val to i16
   %extval = zext i16 %shortval to i64
@@ -61,6 +73,7 @@ define void @test_load_i16(i16* %addr) {
   ret void
 }
 
+; FALLBACK-NOT: remark:{{.*}}test_load_i32
 define void @test_load_i32(i32* %addr) {
 ; CHECK-LABEL: test_load_i32:
 ; CHECK: ldxr w[[LOADVAL:[0-9]+]], [x0]
@@ -68,6 +81,10 @@ define void @test_load_i32(i32* %addr) {
 ; CHECK-NOT: and
 ; CHECK: str x[[LOADVAL]], [{{x[0-9]+}}, :lo12:var]
 
+; GISEL-LABEL: test_load_i32:
+; GISEL: ldxr w[[LOADVAL:[0-9]+]], [x0]
+; GISEL-NOT: uxtb
+; GISEL: str x[[LOADVAL]], [{{x[0-9]+}}]
   %val = call i64 @llvm.aarch64.ldxr.p0i32(i32* %addr)
   %shortval = trunc i64 %val to i32
   %extval = zext i32 %shortval to i64
@@ -75,11 +92,16 @@ define void @test_load_i32(i32* %addr) {
   ret void
 }
 
+; FALLBACK-NOT: remark:{{.*}}test_load_i64
 define void @test_load_i64(i64* %addr) {
 ; CHECK-LABEL: test_load_i64:
 ; CHECK: ldxr x[[LOADVAL:[0-9]+]], [x0]
 ; CHECK: str x[[LOADVAL]], [{{x[0-9]+}}, :lo12:var]
 
+; GISEL-LABEL: test_load_i64:
+; GISEL: ldxr x[[LOADVAL:[0-9]+]], [x0]
+; GISEL-NOT: uxtb
+; GISEL: str x[[LOADVAL]], [{{x[0-9]+}}]
   %val = call i64 @llvm.aarch64.ldxr.p0i64(i64* %addr)
   store i64 %val, i64* @var, align 8
   ret void
@@ -91,39 +113,57 @@ declare i64 @llvm.aarch64.ldxr.p0i16(i16*) nounwind
 declare i64 @llvm.aarch64.ldxr.p0i32(i32*) nounwind
 declare i64 @llvm.aarch64.ldxr.p0i64(i64*) nounwind
 
+; FALLBACK-NOT: remark:{{.*}}test_store_i8
 define i32 @test_store_i8(i32, i8 %val, i8* %addr) {
 ; CHECK-LABEL: test_store_i8:
 ; CHECK-NOT: uxtb
 ; CHECK-NOT: and
 ; CHECK: stxrb w0, w1, [x2]
+; GISEL-LABEL: test_store_i8:
+; GISEL-NOT: uxtb
+; GISEL-NOT: and
+; GISEL: stxrb w0, w1, [x2]
   %extval = zext i8 %val to i64
   %res = call i32 @llvm.aarch64.stxr.p0i8(i64 %extval, i8* %addr)
   ret i32 %res
 }
 
+; FALLBACK-NOT: remark:{{.*}}test_store_i16
 define i32 @test_store_i16(i32, i16 %val, i16* %addr) {
 ; CHECK-LABEL: test_store_i16:
 ; CHECK-NOT: uxth
 ; CHECK-NOT: and
 ; CHECK: stxrh w0, w1, [x2]
+; GISEL-LABEL: test_store_i16:
+; GISEL-NOT: uxth
+; GISEL-NOT: and
+; GISEL: stxrh w0, w1, [x2]
   %extval = zext i16 %val to i64
   %res = call i32 @llvm.aarch64.stxr.p0i16(i64 %extval, i16* %addr)
   ret i32 %res
 }
 
+; FALLBACK-NOT: remark:{{.*}}test_store_i32
 define i32 @test_store_i32(i32, i32 %val, i32* %addr) {
 ; CHECK-LABEL: test_store_i32:
 ; CHECK-NOT: uxtw
 ; CHECK-NOT: and
 ; CHECK: stxr w0, w1, [x2]
+; GISEL-LABEL: test_store_i32:
+; GISEL-NOT: uxtw
+; GISEL-NOT: and
+; GISEL: stxr w0, w1, [x2]
   %extval = zext i32 %val to i64
   %res = call i32 @llvm.aarch64.stxr.p0i32(i64 %extval, i32* %addr)
   ret i32 %res
 }
 
+; FALLBACK-NOT: remark:{{.*}}test_store_i64
 define i32 @test_store_i64(i32, i64 %val, i64* %addr) {
 ; CHECK-LABEL: test_store_i64:
 ; CHECK: stxr w0, x1, [x2]
+; GISEL-LABEL: test_store_i64:
+; GISEL: stxr w0, x1, [x2]
   %res = call i32 @llvm.aarch64.stxr.p0i64(i64 %val, i64* %addr)
   ret i32 %res
 }
@@ -170,6 +210,7 @@ entry:
 declare %0 @llvm.aarch64.ldaxp(i8*) nounwind
 declare i32 @llvm.aarch64.stlxp(i64, i64, i8*) nounwind
 
+; FALLBACK-NOT: remark:{{.*}}test_load_acquire_i8
 define void @test_load_acquire_i8(i8* %addr) {
 ; CHECK-LABEL: test_load_acquire_i8:
 ; CHECK: ldaxrb w[[LOADVAL:[0-9]+]], [x0]
@@ -177,6 +218,11 @@ define void @test_load_acquire_i8(i8* %addr) {
 ; CHECK-NOT: and
 ; CHECK: str x[[LOADVAL]], [{{x[0-9]+}}, :lo12:var]
 
+; FIXME: GlobalISel doesn't fold ands/adds into load/store addressing modes
+; right now/ So, we won't get the :lo12:var.
+; GISEL-LABEL: test_load_acquire_i8:
+; GISEL: ldaxrb w[[LOADVAL:[0-9]+]], [x0]
+; GISEL-DAG: str x[[LOADVAL]], [{{x[0-9]+}}]
   %val = call i64 @llvm.aarch64.ldaxr.p0i8(i8* %addr)
   %shortval = trunc i64 %val to i8
   %extval = zext i8 %shortval to i64
@@ -184,6 +230,7 @@ define void @test_load_acquire_i8(i8* %addr) {
   ret void
 }
 
+; FALLBACK-NOT: remark:{{.*}}test_load_acquire_i16
 define void @test_load_acquire_i16(i16* %addr) {
 ; CHECK-LABEL: test_load_acquire_i16:
 ; CHECK: ldaxrh w[[LOADVAL:[0-9]+]], [x0]
@@ -191,6 +238,9 @@ define void @test_load_acquire_i16(i16* %addr) {
 ; CHECK-NOT: and
 ; CHECK: str x[[LOADVAL]], [{{x[0-9]+}}, :lo12:var]
 
+; GISEL-LABEL: test_load_acquire_i16:
+; GISEL: ldaxrh w[[LOADVAL:[0-9]+]], [x0]
+; GISEL: str x[[LOADVAL]], [{{x[0-9]+}}]
   %val = call i64 @llvm.aarch64.ldaxr.p0i16(i16* %addr)
   %shortval = trunc i64 %val to i16
   %extval = zext i16 %shortval to i64
@@ -198,6 +248,7 @@ define void @test_load_acquire_i16(i16* %addr) {
   ret void
 }
 
+; FALLBACK-NOT: remark:{{.*}}test_load_acquire_i32
 define void @test_load_acquire_i32(i32* %addr) {
 ; CHECK-LABEL: test_load_acquire_i32:
 ; CHECK: ldaxr w[[LOADVAL:[0-9]+]], [x0]
@@ -205,6 +256,9 @@ define void @test_load_acquire_i32(i32* %addr) {
 ; CHECK-NOT: and
 ; CHECK: str x[[LOADVAL]], [{{x[0-9]+}}, :lo12:var]
 
+; GISEL-LABEL: test_load_acquire_i32:
+; GISEL: ldaxr w[[LOADVAL:[0-9]+]], [x0]
+; GISEL: str x[[LOADVAL]], [{{x[0-9]+}}]
   %val = call i64 @llvm.aarch64.ldaxr.p0i32(i32* %addr)
   %shortval = trunc i64 %val to i32
   %extval = zext i32 %shortval to i64
@@ -212,11 +266,15 @@ define void @test_load_acquire_i32(i32* %addr) {
   ret void
 }
 
+; FALLBACK-NOT: remark:{{.*}}test_load_acquire_i64
 define void @test_load_acquire_i64(i64* %addr) {
 ; CHECK-LABEL: test_load_acquire_i64:
 ; CHECK: ldaxr x[[LOADVAL:[0-9]+]], [x0]
 ; CHECK: str x[[LOADVAL]], [{{x[0-9]+}}, :lo12:var]
 
+; GISEL-LABEL: test_load_acquire_i64:
+; GISEL: ldaxr x[[LOADVAL:[0-9]+]], [x0]
+; GISEL: str x[[LOADVAL]], [{{x[0-9]+}}]
   %val = call i64 @llvm.aarch64.ldaxr.p0i64(i64* %addr)
   store i64 %val, i64* @var, align 8
   ret void
@@ -228,31 +286,46 @@ declare i64 @llvm.aarch64.ldaxr.p0i16(i16*) nounwind
 declare i64 @llvm.aarch64.ldaxr.p0i32(i32*) nounwind
 declare i64 @llvm.aarch64.ldaxr.p0i64(i64*) nounwind
 
+; FALLBACK-NOT: remark:{{.*}}test_store_release_i8
 define i32 @test_store_release_i8(i32, i8 %val, i8* %addr) {
 ; CHECK-LABEL: test_store_release_i8:
 ; CHECK-NOT: uxtb
 ; CHECK-NOT: and
 ; CHECK: stlxrb w0, w1, [x2]
+; GISEL-LABEL: test_store_release_i8:
+; GISEL-NOT: uxtb
+; GISEL-NOT: and
+; GISEL: stlxrb w0, w1, [x2]
   %extval = zext i8 %val to i64
   %res = call i32 @llvm.aarch64.stlxr.p0i8(i64 %extval, i8* %addr)
   ret i32 %res
 }
 
+; FALLBACK-NOT: remark:{{.*}}test_store_release_i16
 define i32 @test_store_release_i16(i32, i16 %val, i16* %addr) {
 ; CHECK-LABEL: test_store_release_i16:
 ; CHECK-NOT: uxth
 ; CHECK-NOT: and
 ; CHECK: stlxrh w0, w1, [x2]
+; GISEL-LABEL: test_store_release_i16:
+; GISEL-NOT: uxth
+; GISEL-NOT: and
+; GISEL: stlxrh w0, w1, [x2]
   %extval = zext i16 %val to i64
   %res = call i32 @llvm.aarch64.stlxr.p0i16(i64 %extval, i16* %addr)
   ret i32 %res
 }
 
+; FALLBACK-NOT: remark:{{.*}}test_store_release_i32
 define i32 @test_store_release_i32(i32, i32 %val, i32* %addr) {
 ; CHECK-LABEL: test_store_release_i32:
 ; CHECK-NOT: uxtw
 ; CHECK-NOT: and
 ; CHECK: stlxr w0, w1, [x2]
+; GISEL-LABEL: test_store_release_i32:
+; GISEL-NOT: uxtw
+; GISEL-NOT: and
+; GISEL: stlxr w0, w1, [x2]
   %extval = zext i32 %val to i64
   %res = call i32 @llvm.aarch64.stlxr.p0i32(i64 %extval, i32* %addr)
   ret i32 %res
diff --git a/llvm/test/CodeGen/AArch64/arm64-memset-inline.ll b/llvm/test/CodeGen/AArch64/arm64-memset-inline.ll
index 460ccc25a..7a9f3b2fa 100644
--- a/llvm/test/CodeGen/AArch64/arm64-memset-inline.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-memset-inline.ll
@@ -113,9 +113,9 @@ define void @bzero_20_stack() {
 
 define void @bzero_26_stack() {
 ; CHECK-LABEL: bzero_26_stack:
-; CHECK:       stp xzr, xzr, [sp, #8]
-; CHECK-NEXT:  str xzr, [sp]
+; CHECK:       stp xzr, xzr, [sp]
 ; CHECK-NEXT:  strh wzr, [sp, #24]
+; CHECK-NEXT:  str xzr, [sp, #16]
 ; CHECK-NEXT:  bl something
   %buf = alloca [26 x i8], align 1
   %cast = bitcast [26 x i8]* %buf to i8*
diff --git a/llvm/test/CodeGen/AArch64/arm64-memset-to-bzero-pgso.ll b/llvm/test/CodeGen/AArch64/arm64-memset-to-bzero-pgso.ll
new file mode 100644
index 000000000..086592bf1
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/arm64-memset-to-bzero-pgso.ll
@@ -0,0 +1,128 @@
+; RUN: llc %s -enable-machine-outliner=never -mtriple=arm64-linux-gnu -o - | \
+; RUN:   FileCheck --check-prefixes=CHECK,CHECK-LINUX %s
+; <rdar://problem/14199482> ARM64: Calls to bzero() replaced with calls to memset()
+
+; CHECK-LABEL: fct1:
+; For small size (<= 256), we do not change memset to bzero.
+; CHECK-DARWIN: {{b|bl}} _memset
+; CHECK-LINUX: {{b|bl}} memset
+define void @fct1(i8* nocapture %ptr) !prof !14 {
+entry:
+  tail call void @llvm.memset.p0i8.i64(i8* %ptr, i8 0, i64 256, i1 false)
+  ret void
+}
+
+declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i1)
+
+; CHECK-LABEL: fct2:
+; When the size is bigger than 256, change into bzero.
+; CHECK-DARWIN: {{b|bl}} _bzero
+; CHECK-LINUX: {{b|bl}} memset
+define void @fct2(i8* nocapture %ptr) !prof !14 {
+entry:
+  tail call void @llvm.memset.p0i8.i64(i8* %ptr, i8 0, i64 257, i1 false)
+  ret void
+}
+
+; CHECK-LABEL: fct3:
+; For unknown size, change to bzero.
+; CHECK-DARWIN: {{b|bl}} _bzero
+; CHECK-LINUX: {{b|bl}} memset
+define void @fct3(i8* nocapture %ptr, i32 %unknown) !prof !14 {
+entry:
+  %conv = sext i32 %unknown to i64
+  tail call void @llvm.memset.p0i8.i64(i8* %ptr, i8 0, i64 %conv, i1 false)
+  ret void
+}
+
+; CHECK-LABEL: fct4:
+; Size <= 256, no change.
+; CHECK-DARWIN: {{b|bl}} _memset
+; CHECK-LINUX: {{b|bl}} memset
+define void @fct4(i8* %ptr) !prof !14 {
+entry:
+  %tmp = tail call i64 @llvm.objectsize.i64(i8* %ptr, i1 false)
+  %call = tail call i8* @__memset_chk(i8* %ptr, i32 0, i64 256, i64 %tmp)
+  ret void
+}
+
+declare i8* @__memset_chk(i8*, i32, i64, i64)
+
+declare i64 @llvm.objectsize.i64(i8*, i1)
+
+; CHECK-LABEL: fct5:
+; Size > 256, change.
+; CHECK-DARWIN: {{b|bl}} _bzero
+; CHECK-LINUX: {{b|bl}} memset
+define void @fct5(i8* %ptr) !prof !14 {
+entry:
+  %tmp = tail call i64 @llvm.objectsize.i64(i8* %ptr, i1 false)
+  %call = tail call i8* @__memset_chk(i8* %ptr, i32 0, i64 257, i64 %tmp)
+  ret void
+}
+
+; CHECK-LABEL: fct6:
+; Size = unknown, change.
+; CHECK-DARWIN: {{b|bl}} _bzero
+; CHECK-LINUX: {{b|bl}} memset
+define void @fct6(i8* %ptr, i32 %unknown) !prof !14 {
+entry:
+  %conv = sext i32 %unknown to i64
+  %tmp = tail call i64 @llvm.objectsize.i64(i8* %ptr, i1 false)
+  %call = tail call i8* @__memset_chk(i8* %ptr, i32 0, i64 %conv, i64 %tmp)
+  ret void
+}
+
+; Next functions check that memset is not turned into bzero
+; when the set constant is non-zero, whatever the given size.
+
+; CHECK-LABEL: fct7:
+; memset with something that is not a zero, no change.
+; CHECK-DARWIN: {{b|bl}} _memset
+; CHECK-LINUX: {{b|bl}} memset
+define void @fct7(i8* %ptr) !prof !14 {
+entry:
+  %tmp = tail call i64 @llvm.objectsize.i64(i8* %ptr, i1 false)
+  %call = tail call i8* @__memset_chk(i8* %ptr, i32 1, i64 256, i64 %tmp)
+  ret void
+}
+
+; CHECK-LABEL: fct8:
+; memset with something that is not a zero, no change.
+; CHECK-DARWIN: {{b|bl}} _memset
+; CHECK-LINUX: {{b|bl}} memset
+define void @fct8(i8* %ptr) !prof !14 {
+entry:
+  %tmp = tail call i64 @llvm.objectsize.i64(i8* %ptr, i1 false)
+  %call = tail call i8* @__memset_chk(i8* %ptr, i32 1, i64 257, i64 %tmp)
+  ret void
+}
+
+; CHECK-LABEL: fct9:
+; memset with something that is not a zero, no change.
+; CHECK-DARWIN: {{b|bl}} _memset
+; CHECK-LINUX: {{b|bl}} memset
+define void @fct9(i8* %ptr, i32 %unknown) !prof !14 {
+entry:
+  %conv = sext i32 %unknown to i64
+  %tmp = tail call i64 @llvm.objectsize.i64(i8* %ptr, i1 false)
+  %call = tail call i8* @__memset_chk(i8* %ptr, i32 1, i64 %conv, i64 %tmp)
+  ret void
+}
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 1, !"ProfileSummary", !1}
+!1 = !{!2, !3, !4, !5, !6, !7, !8, !9}
+!2 = !{!"ProfileFormat", !"InstrProf"}
+!3 = !{!"TotalCount", i64 10000}
+!4 = !{!"MaxCount", i64 10}
+!5 = !{!"MaxInternalCount", i64 1}
+!6 = !{!"MaxFunctionCount", i64 1000}
+!7 = !{!"NumCounts", i64 3}
+!8 = !{!"NumFunctions", i64 3}
+!9 = !{!"DetailedSummary", !10}
+!10 = !{!11, !12, !13}
+!11 = !{i32 10000, i64 100, i32 1}
+!12 = !{i32 999000, i64 100, i32 1}
+!13 = !{i32 999999, i64 1, i32 2}
+!14 = !{!"function_entry_count", i64 0}
diff --git a/llvm/test/CodeGen/AArch64/arm64-misched-basic-A53.ll b/llvm/test/CodeGen/AArch64/arm64-misched-basic-A53.ll
index f0b9ccc8b..a4efc4d36 100644
--- a/llvm/test/CodeGen/AArch64/arm64-misched-basic-A53.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-misched-basic-A53.ll
@@ -107,7 +107,7 @@ define <4 x float> @neon4xfloat(<4 x float> %A, <4 x float> %B) {
 ; Function Attrs: nounwind
 declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture readonly, i64, i1) #1
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind }
 
 
diff --git a/llvm/test/CodeGen/AArch64/arm64-misched-basic-A57.ll b/llvm/test/CodeGen/AArch64/arm64-misched-basic-A57.ll
index c2f53e88a..fc4972338 100644
--- a/llvm/test/CodeGen/AArch64/arm64-misched-basic-A57.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-misched-basic-A57.ll
@@ -108,5 +108,5 @@ for.end:                                          ; preds = %for.cond
 ; Function Attrs: nounwind
 declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture readonly, i64, i1) #1
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind }
diff --git a/llvm/test/CodeGen/AArch64/arm64-neon-2velem.ll b/llvm/test/CodeGen/AArch64/arm64-neon-2velem.ll
index d11ed170d..268267891 100644
--- a/llvm/test/CodeGen/AArch64/arm64-neon-2velem.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-neon-2velem.ll
@@ -1,7 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -verify-machineinstrs -mtriple=arm64-none-linux-gnu -mattr=+neon -fp-contract=fast                 | FileCheck %s --check-prefixes=CHECK,GENERIC
-; The instruction latencies of Exynos-M1 trigger the transform we see under the Exynos check.
-; RUN: llc < %s -verify-machineinstrs -mtriple=arm64-none-linux-gnu -mattr=+neon -fp-contract=fast -mcpu=exynos-m1 | FileCheck %s --check-prefixes=CHECK,EXYNOSM1
 ; RUN: llc < %s -verify-machineinstrs -mtriple=arm64-none-linux-gnu -mattr=+neon -fp-contract=fast -mcpu=exynos-m3 | FileCheck %s --check-prefixes=CHECK,EXYNOSM3
 
 declare <2 x double> @llvm.aarch64.neon.fmulx.v2f64(<2 x double>, <2 x double>)
@@ -431,24 +429,11 @@ entry:
 }
 
 define <2 x float> @test_vfma_lane_f32(<2 x float> %a, <2 x float> %b, <2 x float> %v) {
-; GENERIC-LABEL: test_vfma_lane_f32:
-; GENERIC:       // %bb.0: // %entry
-; GENERIC-NEXT:    // kill: def $d2 killed $d2 def $q2
-; GENERIC-NEXT:    fmla v0.2s, v1.2s, v2.s[1]
-; GENERIC-NEXT:    ret
-;
-; EXYNOSM1-LABEL: test_vfma_lane_f32:
-; EXYNOSM1:       // %bb.0: // %entry
-; EXYNOSM1-NEXT:    // kill: def $d2 killed $d2 def $q2
-; EXYNOSM1-NEXT:    dup v2.2s, v2.s[1]
-; EXYNOSM1-NEXT:    fmla v0.2s, v1.2s, v2.2s
-; EXYNOSM1-NEXT:    ret
-;
-; EXYNOSM3-LABEL: test_vfma_lane_f32:
-; EXYNOSM3:       // %bb.0: // %entry
-; EXYNOSM3-NEXT:    // kill: def $d2 killed $d2 def $q2
-; EXYNOSM3-NEXT:    fmla v0.2s, v1.2s, v2.s[1]
-; EXYNOSM3-NEXT:    ret
+; CHECK-LABEL: test_vfma_lane_f32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-NEXT:    fmla v0.2s, v1.2s, v2.s[1]
+; CHECK-NEXT:    ret
 entry:
   %lane = shufflevector <2 x float> %v, <2 x float> undef, <2 x i32> <i32 1, i32 1>
   %0 = tail call <2 x float> @llvm.fma.v2f32(<2 x float> %lane, <2 x float> %b, <2 x float> %a)
@@ -458,24 +443,11 @@ entry:
 declare <2 x float> @llvm.fma.v2f32(<2 x float>, <2 x float>, <2 x float>)
 
 define <4 x float> @test_vfmaq_lane_f32(<4 x float> %a, <4 x float> %b, <2 x float> %v) {
-; GENERIC-LABEL: test_vfmaq_lane_f32:
-; GENERIC:       // %bb.0: // %entry
-; GENERIC-NEXT:    // kill: def $d2 killed $d2 def $q2
-; GENERIC-NEXT:    fmla v0.4s, v1.4s, v2.s[1]
-; GENERIC-NEXT:    ret
-;
-; EXYNOSM1-LABEL: test_vfmaq_lane_f32:
-; EXYNOSM1:       // %bb.0: // %entry
-; EXYNOSM1-NEXT:    // kill: def $d2 killed $d2 def $q2
-; EXYNOSM1-NEXT:    dup v2.4s, v2.s[1]
-; EXYNOSM1-NEXT:    fmla v0.4s, v1.4s, v2.4s
-; EXYNOSM1-NEXT:    ret
-;
-; EXYNOSM3-LABEL: test_vfmaq_lane_f32:
-; EXYNOSM3:       // %bb.0: // %entry
-; EXYNOSM3-NEXT:    // kill: def $d2 killed $d2 def $q2
-; EXYNOSM3-NEXT:    fmla v0.4s, v1.4s, v2.s[1]
-; EXYNOSM3-NEXT:    ret
+; CHECK-LABEL: test_vfmaq_lane_f32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-NEXT:    fmla v0.4s, v1.4s, v2.s[1]
+; CHECK-NEXT:    ret
 entry:
   %lane = shufflevector <2 x float> %v, <2 x float> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
   %0 = tail call <4 x float> @llvm.fma.v4f32(<4 x float> %lane, <4 x float> %b, <4 x float> %a)
@@ -485,21 +457,10 @@ entry:
 declare <4 x float> @llvm.fma.v4f32(<4 x float>, <4 x float>, <4 x float>)
 
 define <2 x float> @test_vfma_laneq_f32(<2 x float> %a, <2 x float> %b, <4 x float> %v) {
-; GENERIC-LABEL: test_vfma_laneq_f32:
-; GENERIC:       // %bb.0: // %entry
-; GENERIC-NEXT:    fmla v0.2s, v1.2s, v2.s[3]
-; GENERIC-NEXT:    ret
-;
-; EXYNOSM1-LABEL: test_vfma_laneq_f32:
-; EXYNOSM1:       // %bb.0: // %entry
-; EXYNOSM1-NEXT:    dup v2.2s, v2.s[3]
-; EXYNOSM1-NEXT:    fmla v0.2s, v1.2s, v2.2s
-; EXYNOSM1-NEXT:    ret
-;
-; EXYNOSM3-LABEL: test_vfma_laneq_f32:
-; EXYNOSM3:       // %bb.0: // %entry
-; EXYNOSM3-NEXT:    fmla v0.2s, v1.2s, v2.s[3]
-; EXYNOSM3-NEXT:    ret
+; CHECK-LABEL: test_vfma_laneq_f32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmla v0.2s, v1.2s, v2.s[3]
+; CHECK-NEXT:    ret
 entry:
   %lane = shufflevector <4 x float> %v, <4 x float> undef, <2 x i32> <i32 3, i32 3>
   %0 = tail call <2 x float> @llvm.fma.v2f32(<2 x float> %lane, <2 x float> %b, <2 x float> %a)
@@ -507,21 +468,10 @@ entry:
 }
 
 define <4 x float> @test_vfmaq_laneq_f32(<4 x float> %a, <4 x float> %b, <4 x float> %v) {
-; GENERIC-LABEL: test_vfmaq_laneq_f32:
-; GENERIC:       // %bb.0: // %entry
-; GENERIC-NEXT:    fmla v0.4s, v1.4s, v2.s[3]
-; GENERIC-NEXT:    ret
-;
-; EXYNOSM1-LABEL: test_vfmaq_laneq_f32:
-; EXYNOSM1:       // %bb.0: // %entry
-; EXYNOSM1-NEXT:    dup v2.4s, v2.s[3]
-; EXYNOSM1-NEXT:    fmla v0.4s, v1.4s, v2.4s
-; EXYNOSM1-NEXT:    ret
-;
-; EXYNOSM3-LABEL: test_vfmaq_laneq_f32:
-; EXYNOSM3:       // %bb.0: // %entry
-; EXYNOSM3-NEXT:    fmla v0.4s, v1.4s, v2.s[3]
-; EXYNOSM3-NEXT:    ret
+; CHECK-LABEL: test_vfmaq_laneq_f32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmla v0.4s, v1.4s, v2.s[3]
+; CHECK-NEXT:    ret
 entry:
   %lane = shufflevector <4 x float> %v, <4 x float> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
   %0 = tail call <4 x float> @llvm.fma.v4f32(<4 x float> %lane, <4 x float> %b, <4 x float> %a)
@@ -529,24 +479,11 @@ entry:
 }
 
 define <2 x float> @test_vfms_lane_f32(<2 x float> %a, <2 x float> %b, <2 x float> %v) {
-; GENERIC-LABEL: test_vfms_lane_f32:
-; GENERIC:       // %bb.0: // %entry
-; GENERIC-NEXT:    // kill: def $d2 killed $d2 def $q2
-; GENERIC-NEXT:    fmls v0.2s, v1.2s, v2.s[1]
-; GENERIC-NEXT:    ret
-;
-; EXYNOSM1-LABEL: test_vfms_lane_f32:
-; EXYNOSM1:       // %bb.0: // %entry
-; EXYNOSM1-NEXT:    // kill: def $d2 killed $d2 def $q2
-; EXYNOSM1-NEXT:    dup v2.2s, v2.s[1]
-; EXYNOSM1-NEXT:    fmls v0.2s, v1.2s, v2.2s
-; EXYNOSM1-NEXT:    ret
-;
-; EXYNOSM3-LABEL: test_vfms_lane_f32:
-; EXYNOSM3:       // %bb.0: // %entry
-; EXYNOSM3-NEXT:    // kill: def $d2 killed $d2 def $q2
-; EXYNOSM3-NEXT:    fmls v0.2s, v1.2s, v2.s[1]
-; EXYNOSM3-NEXT:    ret
+; CHECK-LABEL: test_vfms_lane_f32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-NEXT:    fmls v0.2s, v1.2s, v2.s[1]
+; CHECK-NEXT:    ret
 entry:
   %sub = fsub <2 x float> <float -0.000000e+00, float -0.000000e+00>, %v
   %lane = shufflevector <2 x float> %sub, <2 x float> undef, <2 x i32> <i32 1, i32 1>
@@ -555,24 +492,11 @@ entry:
 }
 
 define <4 x float> @test_vfmsq_lane_f32(<4 x float> %a, <4 x float> %b, <2 x float> %v) {
-; GENERIC-LABEL: test_vfmsq_lane_f32:
-; GENERIC:       // %bb.0: // %entry
-; GENERIC-NEXT:    // kill: def $d2 killed $d2 def $q2
-; GENERIC-NEXT:    fmls v0.4s, v1.4s, v2.s[1]
-; GENERIC-NEXT:    ret
-;
-; EXYNOSM1-LABEL: test_vfmsq_lane_f32:
-; EXYNOSM1:       // %bb.0: // %entry
-; EXYNOSM1-NEXT:    // kill: def $d2 killed $d2 def $q2
-; EXYNOSM1-NEXT:    dup v2.4s, v2.s[1]
-; EXYNOSM1-NEXT:    fmls v0.4s, v1.4s, v2.4s
-; EXYNOSM1-NEXT:    ret
-;
-; EXYNOSM3-LABEL: test_vfmsq_lane_f32:
-; EXYNOSM3:       // %bb.0: // %entry
-; EXYNOSM3-NEXT:    // kill: def $d2 killed $d2 def $q2
-; EXYNOSM3-NEXT:    fmls v0.4s, v1.4s, v2.s[1]
-; EXYNOSM3-NEXT:    ret
+; CHECK-LABEL: test_vfmsq_lane_f32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-NEXT:    fmls v0.4s, v1.4s, v2.s[1]
+; CHECK-NEXT:    ret
 entry:
   %sub = fsub <2 x float> <float -0.000000e+00, float -0.000000e+00>, %v
   %lane = shufflevector <2 x float> %sub, <2 x float> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
@@ -581,21 +505,10 @@ entry:
 }
 
 define <2 x float> @test_vfms_laneq_f32(<2 x float> %a, <2 x float> %b, <4 x float> %v) {
-; GENERIC-LABEL: test_vfms_laneq_f32:
-; GENERIC:       // %bb.0: // %entry
-; GENERIC-NEXT:    fmls v0.2s, v1.2s, v2.s[3]
-; GENERIC-NEXT:    ret
-;
-; EXYNOSM1-LABEL: test_vfms_laneq_f32:
-; EXYNOSM1:       // %bb.0: // %entry
-; EXYNOSM1-NEXT:    dup v2.2s, v2.s[3]
-; EXYNOSM1-NEXT:    fmls v0.2s, v1.2s, v2.2s
-; EXYNOSM1-NEXT:    ret
-;
-; EXYNOSM3-LABEL: test_vfms_laneq_f32:
-; EXYNOSM3:       // %bb.0: // %entry
-; EXYNOSM3-NEXT:    fmls v0.2s, v1.2s, v2.s[3]
-; EXYNOSM3-NEXT:    ret
+; CHECK-LABEL: test_vfms_laneq_f32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmls v0.2s, v1.2s, v2.s[3]
+; CHECK-NEXT:    ret
 entry:
   %sub = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %v
   %lane = shufflevector <4 x float> %sub, <4 x float> undef, <2 x i32> <i32 3, i32 3>
@@ -604,21 +517,10 @@ entry:
 }
 
 define <4 x float> @test_vfmsq_laneq_f32(<4 x float> %a, <4 x float> %b, <4 x float> %v) {
-; GENERIC-LABEL: test_vfmsq_laneq_f32:
-; GENERIC:       // %bb.0: // %entry
-; GENERIC-NEXT:    fmls v0.4s, v1.4s, v2.s[3]
-; GENERIC-NEXT:    ret
-;
-; EXYNOSM1-LABEL: test_vfmsq_laneq_f32:
-; EXYNOSM1:       // %bb.0: // %entry
-; EXYNOSM1-NEXT:    dup v2.4s, v2.s[3]
-; EXYNOSM1-NEXT:    fmls v0.4s, v1.4s, v2.4s
-; EXYNOSM1-NEXT:    ret
-;
-; EXYNOSM3-LABEL: test_vfmsq_laneq_f32:
-; EXYNOSM3:       // %bb.0: // %entry
-; EXYNOSM3-NEXT:    fmls v0.4s, v1.4s, v2.s[3]
-; EXYNOSM3-NEXT:    ret
+; CHECK-LABEL: test_vfmsq_laneq_f32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmls v0.4s, v1.4s, v2.s[3]
+; CHECK-NEXT:    ret
 entry:
   %sub = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %v
   %lane = shufflevector <4 x float> %sub, <4 x float> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
@@ -627,24 +529,11 @@ entry:
 }
 
 define <2 x double> @test_vfmaq_lane_f64(<2 x double> %a, <2 x double> %b, <1 x double> %v) {
-; GENERIC-LABEL: test_vfmaq_lane_f64:
-; GENERIC:       // %bb.0: // %entry
-; GENERIC-NEXT:    // kill: def $d2 killed $d2 def $q2
-; GENERIC-NEXT:    fmla v0.2d, v1.2d, v2.d[0]
-; GENERIC-NEXT:    ret
-;
-; EXYNOSM1-LABEL: test_vfmaq_lane_f64:
-; EXYNOSM1:       // %bb.0: // %entry
-; EXYNOSM1-NEXT:    // kill: def $d2 killed $d2 def $q2
-; EXYNOSM1-NEXT:    dup v2.2d, v2.d[0]
-; EXYNOSM1-NEXT:    fmla v0.2d, v1.2d, v2.2d
-; EXYNOSM1-NEXT:    ret
-;
-; EXYNOSM3-LABEL: test_vfmaq_lane_f64:
-; EXYNOSM3:       // %bb.0: // %entry
-; EXYNOSM3-NEXT:    // kill: def $d2 killed $d2 def $q2
-; EXYNOSM3-NEXT:    fmla v0.2d, v1.2d, v2.d[0]
-; EXYNOSM3-NEXT:    ret
+; CHECK-LABEL: test_vfmaq_lane_f64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-NEXT:    fmla v0.2d, v1.2d, v2.d[0]
+; CHECK-NEXT:    ret
 entry:
   %lane = shufflevector <1 x double> %v, <1 x double> undef, <2 x i32> zeroinitializer
   %0 = tail call <2 x double> @llvm.fma.v2f64(<2 x double> %lane, <2 x double> %b, <2 x double> %a)
@@ -654,21 +543,10 @@ entry:
 declare <2 x double> @llvm.fma.v2f64(<2 x double>, <2 x double>, <2 x double>)
 
 define <2 x double> @test_vfmaq_laneq_f64(<2 x double> %a, <2 x double> %b, <2 x double> %v) {
-; GENERIC-LABEL: test_vfmaq_laneq_f64:
-; GENERIC:       // %bb.0: // %entry
-; GENERIC-NEXT:    fmla v0.2d, v1.2d, v2.d[1]
-; GENERIC-NEXT:    ret
-;
-; EXYNOSM1-LABEL: test_vfmaq_laneq_f64:
-; EXYNOSM1:       // %bb.0: // %entry
-; EXYNOSM1-NEXT:    dup v2.2d, v2.d[1]
-; EXYNOSM1-NEXT:    fmla v0.2d, v1.2d, v2.2d
-; EXYNOSM1-NEXT:    ret
-;
-; EXYNOSM3-LABEL: test_vfmaq_laneq_f64:
-; EXYNOSM3:       // %bb.0: // %entry
-; EXYNOSM3-NEXT:    fmla v0.2d, v1.2d, v2.d[1]
-; EXYNOSM3-NEXT:    ret
+; CHECK-LABEL: test_vfmaq_laneq_f64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmla v0.2d, v1.2d, v2.d[1]
+; CHECK-NEXT:    ret
 entry:
   %lane = shufflevector <2 x double> %v, <2 x double> undef, <2 x i32> <i32 1, i32 1>
   %0 = tail call <2 x double> @llvm.fma.v2f64(<2 x double> %lane, <2 x double> %b, <2 x double> %a)
@@ -676,24 +554,11 @@ entry:
 }
 
 define <2 x double> @test_vfmsq_lane_f64(<2 x double> %a, <2 x double> %b, <1 x double> %v) {
-; GENERIC-LABEL: test_vfmsq_lane_f64:
-; GENERIC:       // %bb.0: // %entry
-; GENERIC-NEXT:    // kill: def $d2 killed $d2 def $q2
-; GENERIC-NEXT:    fmls v0.2d, v1.2d, v2.d[0]
-; GENERIC-NEXT:    ret
-;
-; EXYNOSM1-LABEL: test_vfmsq_lane_f64:
-; EXYNOSM1:       // %bb.0: // %entry
-; EXYNOSM1-NEXT:    // kill: def $d2 killed $d2 def $q2
-; EXYNOSM1-NEXT:    dup v2.2d, v2.d[0]
-; EXYNOSM1-NEXT:    fmls v0.2d, v1.2d, v2.2d
-; EXYNOSM1-NEXT:    ret
-;
-; EXYNOSM3-LABEL: test_vfmsq_lane_f64:
-; EXYNOSM3:       // %bb.0: // %entry
-; EXYNOSM3-NEXT:    // kill: def $d2 killed $d2 def $q2
-; EXYNOSM3-NEXT:    fmls v0.2d, v1.2d, v2.d[0]
-; EXYNOSM3-NEXT:    ret
+; CHECK-LABEL: test_vfmsq_lane_f64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-NEXT:    fmls v0.2d, v1.2d, v2.d[0]
+; CHECK-NEXT:    ret
 entry:
   %sub = fsub <1 x double> <double -0.000000e+00>, %v
   %lane = shufflevector <1 x double> %sub, <1 x double> undef, <2 x i32> zeroinitializer
@@ -702,21 +567,10 @@ entry:
 }
 
 define <2 x double> @test_vfmsq_laneq_f64(<2 x double> %a, <2 x double> %b, <2 x double> %v) {
-; GENERIC-LABEL: test_vfmsq_laneq_f64:
-; GENERIC:       // %bb.0: // %entry
-; GENERIC-NEXT:    fmls v0.2d, v1.2d, v2.d[1]
-; GENERIC-NEXT:    ret
-;
-; EXYNOSM1-LABEL: test_vfmsq_laneq_f64:
-; EXYNOSM1:       // %bb.0: // %entry
-; EXYNOSM1-NEXT:    dup v2.2d, v2.d[1]
-; EXYNOSM1-NEXT:    fmls v0.2d, v1.2d, v2.2d
-; EXYNOSM1-NEXT:    ret
-;
-; EXYNOSM3-LABEL: test_vfmsq_laneq_f64:
-; EXYNOSM3:       // %bb.0: // %entry
-; EXYNOSM3-NEXT:    fmls v0.2d, v1.2d, v2.d[1]
-; EXYNOSM3-NEXT:    ret
+; CHECK-LABEL: test_vfmsq_laneq_f64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmls v0.2d, v1.2d, v2.d[1]
+; CHECK-NEXT:    ret
 entry:
   %sub = fsub <2 x double> <double -0.000000e+00, double -0.000000e+00>, %v
   %lane = shufflevector <2 x double> %sub, <2 x double> undef, <2 x i32> <i32 1, i32 1>
@@ -1746,24 +1600,11 @@ entry:
 }
 
 define <2 x float> @test_vmul_lane_f32(<2 x float> %a, <2 x float> %v) {
-; GENERIC-LABEL: test_vmul_lane_f32:
-; GENERIC:       // %bb.0: // %entry
-; GENERIC-NEXT:    // kill: def $d1 killed $d1 def $q1
-; GENERIC-NEXT:    fmul v0.2s, v0.2s, v1.s[1]
-; GENERIC-NEXT:    ret
-;
-; EXYNOSM1-LABEL: test_vmul_lane_f32:
-; EXYNOSM1:       // %bb.0: // %entry
-; EXYNOSM1-NEXT:    // kill: def $d1 killed $d1 def $q1
-; EXYNOSM1-NEXT:    dup v1.2s, v1.s[1]
-; EXYNOSM1-NEXT:    fmul v0.2s, v0.2s, v1.2s
-; EXYNOSM1-NEXT:    ret
-;
-; EXYNOSM3-LABEL: test_vmul_lane_f32:
-; EXYNOSM3:       // %bb.0: // %entry
-; EXYNOSM3-NEXT:    // kill: def $d1 killed $d1 def $q1
-; EXYNOSM3-NEXT:    fmul v0.2s, v0.2s, v1.s[1]
-; EXYNOSM3-NEXT:    ret
+; CHECK-LABEL: test_vmul_lane_f32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT:    fmul v0.2s, v0.2s, v1.s[1]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <2 x float> %v, <2 x float> undef, <2 x i32> <i32 1, i32 1>
   %mul = fmul <2 x float> %shuffle, %a
@@ -1785,24 +1626,11 @@ entry:
 }
 
 define <4 x float> @test_vmulq_lane_f32(<4 x float> %a, <2 x float> %v) {
-; GENERIC-LABEL: test_vmulq_lane_f32:
-; GENERIC:       // %bb.0: // %entry
-; GENERIC-NEXT:    // kill: def $d1 killed $d1 def $q1
-; GENERIC-NEXT:    fmul v0.4s, v0.4s, v1.s[1]
-; GENERIC-NEXT:    ret
-;
-; EXYNOSM1-LABEL: test_vmulq_lane_f32:
-; EXYNOSM1:       // %bb.0: // %entry
-; EXYNOSM1-NEXT:    // kill: def $d1 killed $d1 def $q1
-; EXYNOSM1-NEXT:    dup v1.4s, v1.s[1]
-; EXYNOSM1-NEXT:    fmul v0.4s, v0.4s, v1.4s
-; EXYNOSM1-NEXT:    ret
-;
-; EXYNOSM3-LABEL: test_vmulq_lane_f32:
-; EXYNOSM3:       // %bb.0: // %entry
-; EXYNOSM3-NEXT:    // kill: def $d1 killed $d1 def $q1
-; EXYNOSM3-NEXT:    fmul v0.4s, v0.4s, v1.s[1]
-; EXYNOSM3-NEXT:    ret
+; CHECK-LABEL: test_vmulq_lane_f32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT:    fmul v0.4s, v0.4s, v1.s[1]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <2 x float> %v, <2 x float> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
   %mul = fmul <4 x float> %shuffle, %a
@@ -1810,24 +1638,11 @@ entry:
 }
 
 define <2 x double> @test_vmulq_lane_f64(<2 x double> %a, <1 x double> %v) {
-; GENERIC-LABEL: test_vmulq_lane_f64:
-; GENERIC:       // %bb.0: // %entry
-; GENERIC-NEXT:    // kill: def $d1 killed $d1 def $q1
-; GENERIC-NEXT:    fmul v0.2d, v0.2d, v1.d[0]
-; GENERIC-NEXT:    ret
-;
-; EXYNOSM1-LABEL: test_vmulq_lane_f64:
-; EXYNOSM1:       // %bb.0: // %entry
-; EXYNOSM1-NEXT:    // kill: def $d1 killed $d1 def $q1
-; EXYNOSM1-NEXT:    dup v1.2d, v1.d[0]
-; EXYNOSM1-NEXT:    fmul v0.2d, v0.2d, v1.2d
-; EXYNOSM1-NEXT:    ret
-;
-; EXYNOSM3-LABEL: test_vmulq_lane_f64:
-; EXYNOSM3:       // %bb.0: // %entry
-; EXYNOSM3-NEXT:    // kill: def $d1 killed $d1 def $q1
-; EXYNOSM3-NEXT:    fmul v0.2d, v0.2d, v1.d[0]
-; EXYNOSM3-NEXT:    ret
+; CHECK-LABEL: test_vmulq_lane_f64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT:    fmul v0.2d, v0.2d, v1.d[0]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <1 x double> %v, <1 x double> undef, <2 x i32> zeroinitializer
   %mul = fmul <2 x double> %shuffle, %a
@@ -1835,27 +1650,98 @@ entry:
 }
 
 define <2 x float> @test_vmul_laneq_f32(<2 x float> %a, <4 x float> %v) {
-; GENERIC-LABEL: test_vmul_laneq_f32:
-; GENERIC:       // %bb.0: // %entry
-; GENERIC-NEXT:    fmul v0.2s, v0.2s, v1.s[3]
-; GENERIC-NEXT:    ret
-;
-; EXYNOSM1-LABEL: test_vmul_laneq_f32:
-; EXYNOSM1:       // %bb.0: // %entry
-; EXYNOSM1-NEXT:    dup v1.2s, v1.s[3]
-; EXYNOSM1-NEXT:    fmul v0.2s, v0.2s, v1.2s
-; EXYNOSM1-NEXT:    ret
-;
-; EXYNOSM3-LABEL: test_vmul_laneq_f32:
-; EXYNOSM3:       // %bb.0: // %entry
-; EXYNOSM3-NEXT:    fmul v0.2s, v0.2s, v1.s[3]
-; EXYNOSM3-NEXT:    ret
+; CHECK-LABEL: test_vmul_laneq_f32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmul v0.2s, v0.2s, v1.s[3]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <4 x float> %v, <4 x float> undef, <2 x i32> <i32 3, i32 3>
   %mul = fmul <2 x float> %shuffle, %a
   ret <2 x float> %mul
 }
 
+define <2 x float> @test_vmul_laneq3_f32_bitcast(<2 x float> %a, <2 x double> %v) {
+; CHECK-LABEL: test_vmul_laneq3_f32_bitcast:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmul v0.2s, v0.2s, v1.s[3]
+; CHECK-NEXT:    ret
+  %extract = shufflevector <2 x double> %v, <2 x double> undef, <1 x i32> <i32 1>
+  %bc = bitcast <1 x double> %extract to <2 x float>
+  %splat = shufflevector <2 x float> %bc, <2 x float> undef, <2 x i32> <i32 1, i32 1>
+  %mul = fmul <2 x float> %splat, %a
+  ret <2 x float> %mul
+}
+
+define <2 x float> @test_vmul_laneq2_f32_bitcast(<2 x float> %a, <2 x double> %v) {
+; CHECK-LABEL: test_vmul_laneq2_f32_bitcast:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmul v0.2s, v0.2s, v1.s[2]
+; CHECK-NEXT:    ret
+  %extract = shufflevector <2 x double> %v, <2 x double> undef, <1 x i32> <i32 1>
+  %bc = bitcast <1 x double> %extract to <2 x float>
+  %splat = shufflevector <2 x float> %bc, <2 x float> undef, <2 x i32> <i32 0, i32 0>
+  %mul = fmul <2 x float> %splat, %a
+  ret <2 x float> %mul
+}
+
+define <4 x i16> @test_vadd_laneq5_i16_bitcast(<4 x i16> %a, <2 x double> %v) {
+; CHECK-LABEL: test_vadd_laneq5_i16_bitcast:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    dup v1.4h, v1.h[5]
+; CHECK-NEXT:    add v0.4h, v1.4h, v0.4h
+; CHECK-NEXT:    ret
+  %extract = shufflevector <2 x double> %v, <2 x double> undef, <1 x i32> <i32 1>
+  %bc = bitcast <1 x double> %extract to <4 x i16>
+  %splat = shufflevector <4 x i16> %bc, <4 x i16> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  %r = add <4 x i16> %splat, %a
+  ret <4 x i16> %r
+}
+
+; TODO: The pattern in LowerVECTOR_SHUFFLE does not match what we are looking for.
+
+define <4 x i16> @test_vadd_lane2_i16_bitcast_bigger_aligned(<4 x i16> %a, <16 x i8> %v) {
+; CHECK-LABEL: test_vadd_lane2_i16_bitcast_bigger_aligned:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ext v1.8b, v1.8b, v0.8b, #2
+; CHECK-NEXT:    dup v1.4h, v1.h[1]
+; CHECK-NEXT:    add v0.4h, v1.4h, v0.4h
+; CHECK-NEXT:    ret
+  %extract = shufflevector <16 x i8> %v, <16 x i8> undef, <8 x i32> <i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9>
+  %bc = bitcast <8 x i8> %extract to <4 x i16>
+  %splat = shufflevector <4 x i16> %bc, <4 x i16> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  %r = add <4 x i16> %splat, %a
+  ret <4 x i16> %r
+}
+
+define <4 x i16> @test_vadd_lane5_i16_bitcast_bigger_aligned(<4 x i16> %a, <16 x i8> %v) {
+; CHECK-LABEL: test_vadd_lane5_i16_bitcast_bigger_aligned:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    dup v1.4h, v1.h[5]
+; CHECK-NEXT:    add v0.4h, v1.4h, v0.4h
+; CHECK-NEXT:    ret
+  %extract = shufflevector <16 x i8> %v, <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %bc = bitcast <8 x i8> %extract to <4 x i16>
+  %splat = shufflevector <4 x i16> %bc, <4 x i16> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  %r = add <4 x i16> %splat, %a
+  ret <4 x i16> %r
+}
+
+; Negative test - can't dup bytes {3,4} of v8i16.
+
+define <4 x i16> @test_vadd_lane_i16_bitcast_bigger_unaligned(<4 x i16> %a, <16 x i8> %v) {
+; CHECK-LABEL: test_vadd_lane_i16_bitcast_bigger_unaligned:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ext v1.8b, v1.8b, v0.8b, #1
+; CHECK-NEXT:    dup v1.4h, v1.h[1]
+; CHECK-NEXT:    add v0.4h, v1.4h, v0.4h
+; CHECK-NEXT:    ret
+  %extract = shufflevector <16 x i8> %v, <16 x i8> undef, <8 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
+  %bc = bitcast <8 x i8> %extract to <4 x i16>
+  %splat = shufflevector <4 x i16> %bc, <4 x i16> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  %r = add <4 x i16> %splat, %a
+  ret <4 x i16> %r
+}
+
 define <1 x double> @test_vmul_laneq_f64(<1 x double> %a, <2 x double> %v) {
 ; CHECK-LABEL: test_vmul_laneq_f64:
 ; CHECK:       // %bb.0: // %entry
@@ -1871,21 +1757,10 @@ entry:
 }
 
 define <4 x float> @test_vmulq_laneq_f32(<4 x float> %a, <4 x float> %v) {
-; GENERIC-LABEL: test_vmulq_laneq_f32:
-; GENERIC:       // %bb.0: // %entry
-; GENERIC-NEXT:    fmul v0.4s, v0.4s, v1.s[3]
-; GENERIC-NEXT:    ret
-;
-; EXYNOSM1-LABEL: test_vmulq_laneq_f32:
-; EXYNOSM1:       // %bb.0: // %entry
-; EXYNOSM1-NEXT:    dup v1.4s, v1.s[3]
-; EXYNOSM1-NEXT:    fmul v0.4s, v0.4s, v1.4s
-; EXYNOSM1-NEXT:    ret
-;
-; EXYNOSM3-LABEL: test_vmulq_laneq_f32:
-; EXYNOSM3:       // %bb.0: // %entry
-; EXYNOSM3-NEXT:    fmul v0.4s, v0.4s, v1.s[3]
-; EXYNOSM3-NEXT:    ret
+; CHECK-LABEL: test_vmulq_laneq_f32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmul v0.4s, v0.4s, v1.s[3]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <4 x float> %v, <4 x float> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
   %mul = fmul <4 x float> %shuffle, %a
@@ -1893,21 +1768,10 @@ entry:
 }
 
 define <2 x double> @test_vmulq_laneq_f64(<2 x double> %a, <2 x double> %v) {
-; GENERIC-LABEL: test_vmulq_laneq_f64:
-; GENERIC:       // %bb.0: // %entry
-; GENERIC-NEXT:    fmul v0.2d, v0.2d, v1.d[1]
-; GENERIC-NEXT:    ret
-;
-; EXYNOSM1-LABEL: test_vmulq_laneq_f64:
-; EXYNOSM1:       // %bb.0: // %entry
-; EXYNOSM1-NEXT:    dup v1.2d, v1.d[1]
-; EXYNOSM1-NEXT:    fmul v0.2d, v0.2d, v1.2d
-; EXYNOSM1-NEXT:    ret
-;
-; EXYNOSM3-LABEL: test_vmulq_laneq_f64:
-; EXYNOSM3:       // %bb.0: // %entry
-; EXYNOSM3-NEXT:    fmul v0.2d, v0.2d, v1.d[1]
-; EXYNOSM3-NEXT:    ret
+; CHECK-LABEL: test_vmulq_laneq_f64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmul v0.2d, v0.2d, v1.d[1]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <2 x double> %v, <2 x double> undef, <2 x i32> <i32 1, i32 1>
   %mul = fmul <2 x double> %shuffle, %a
@@ -1915,24 +1779,11 @@ entry:
 }
 
 define <2 x float> @test_vmulx_lane_f32(<2 x float> %a, <2 x float> %v) {
-; GENERIC-LABEL: test_vmulx_lane_f32:
-; GENERIC:       // %bb.0: // %entry
-; GENERIC-NEXT:    // kill: def $d1 killed $d1 def $q1
-; GENERIC-NEXT:    fmulx v0.2s, v0.2s, v1.s[1]
-; GENERIC-NEXT:    ret
-;
-; EXYNOSM1-LABEL: test_vmulx_lane_f32:
-; EXYNOSM1:       // %bb.0: // %entry
-; EXYNOSM1-NEXT:    // kill: def $d1 killed $d1 def $q1
-; EXYNOSM1-NEXT:    dup v1.2s, v1.s[1]
-; EXYNOSM1-NEXT:    fmulx v0.2s, v0.2s, v1.2s
-; EXYNOSM1-NEXT:    ret
-;
-; EXYNOSM3-LABEL: test_vmulx_lane_f32:
-; EXYNOSM3:       // %bb.0: // %entry
-; EXYNOSM3-NEXT:    // kill: def $d1 killed $d1 def $q1
-; EXYNOSM3-NEXT:    fmulx v0.2s, v0.2s, v1.s[1]
-; EXYNOSM3-NEXT:    ret
+; CHECK-LABEL: test_vmulx_lane_f32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT:    fmulx v0.2s, v0.2s, v1.s[1]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <2 x float> %v, <2 x float> undef, <2 x i32> <i32 1, i32 1>
   %vmulx2.i = tail call <2 x float> @llvm.aarch64.neon.fmulx.v2f32(<2 x float> %a, <2 x float> %shuffle)
@@ -1940,24 +1791,11 @@ entry:
 }
 
 define <4 x float> @test_vmulxq_lane_f32(<4 x float> %a, <2 x float> %v) {
-; GENERIC-LABEL: test_vmulxq_lane_f32:
-; GENERIC:       // %bb.0: // %entry
-; GENERIC-NEXT:    // kill: def $d1 killed $d1 def $q1
-; GENERIC-NEXT:    fmulx v0.4s, v0.4s, v1.s[1]
-; GENERIC-NEXT:    ret
-;
-; EXYNOSM1-LABEL: test_vmulxq_lane_f32:
-; EXYNOSM1:       // %bb.0: // %entry
-; EXYNOSM1-NEXT:    // kill: def $d1 killed $d1 def $q1
-; EXYNOSM1-NEXT:    dup v1.4s, v1.s[1]
-; EXYNOSM1-NEXT:    fmulx v0.4s, v0.4s, v1.4s
-; EXYNOSM1-NEXT:    ret
-;
-; EXYNOSM3-LABEL: test_vmulxq_lane_f32:
-; EXYNOSM3:       // %bb.0: // %entry
-; EXYNOSM3-NEXT:    // kill: def $d1 killed $d1 def $q1
-; EXYNOSM3-NEXT:    fmulx v0.4s, v0.4s, v1.s[1]
-; EXYNOSM3-NEXT:    ret
+; CHECK-LABEL: test_vmulxq_lane_f32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT:    fmulx v0.4s, v0.4s, v1.s[1]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <2 x float> %v, <2 x float> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
   %vmulx2.i = tail call <4 x float> @llvm.aarch64.neon.fmulx.v4f32(<4 x float> %a, <4 x float> %shuffle)
@@ -1965,24 +1803,11 @@ entry:
 }
 
 define <2 x double> @test_vmulxq_lane_f64(<2 x double> %a, <1 x double> %v) {
-; GENERIC-LABEL: test_vmulxq_lane_f64:
-; GENERIC:       // %bb.0: // %entry
-; GENERIC-NEXT:    // kill: def $d1 killed $d1 def $q1
-; GENERIC-NEXT:    fmulx v0.2d, v0.2d, v1.d[0]
-; GENERIC-NEXT:    ret
-;
-; EXYNOSM1-LABEL: test_vmulxq_lane_f64:
-; EXYNOSM1:       // %bb.0: // %entry
-; EXYNOSM1-NEXT:    // kill: def $d1 killed $d1 def $q1
-; EXYNOSM1-NEXT:    dup v1.2d, v1.d[0]
-; EXYNOSM1-NEXT:    fmulx v0.2d, v0.2d, v1.2d
-; EXYNOSM1-NEXT:    ret
-;
-; EXYNOSM3-LABEL: test_vmulxq_lane_f64:
-; EXYNOSM3:       // %bb.0: // %entry
-; EXYNOSM3-NEXT:    // kill: def $d1 killed $d1 def $q1
-; EXYNOSM3-NEXT:    fmulx v0.2d, v0.2d, v1.d[0]
-; EXYNOSM3-NEXT:    ret
+; CHECK-LABEL: test_vmulxq_lane_f64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT:    fmulx v0.2d, v0.2d, v1.d[0]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <1 x double> %v, <1 x double> undef, <2 x i32> zeroinitializer
   %vmulx2.i = tail call <2 x double> @llvm.aarch64.neon.fmulx.v2f64(<2 x double> %a, <2 x double> %shuffle)
@@ -1990,21 +1815,10 @@ entry:
 }
 
 define <2 x float> @test_vmulx_laneq_f32(<2 x float> %a, <4 x float> %v) {
-; GENERIC-LABEL: test_vmulx_laneq_f32:
-; GENERIC:       // %bb.0: // %entry
-; GENERIC-NEXT:    fmulx v0.2s, v0.2s, v1.s[3]
-; GENERIC-NEXT:    ret
-;
-; EXYNOSM1-LABEL: test_vmulx_laneq_f32:
-; EXYNOSM1:       // %bb.0: // %entry
-; EXYNOSM1-NEXT:    dup v1.2s, v1.s[3]
-; EXYNOSM1-NEXT:    fmulx v0.2s, v0.2s, v1.2s
-; EXYNOSM1-NEXT:    ret
-;
-; EXYNOSM3-LABEL: test_vmulx_laneq_f32:
-; EXYNOSM3:       // %bb.0: // %entry
-; EXYNOSM3-NEXT:    fmulx v0.2s, v0.2s, v1.s[3]
-; EXYNOSM3-NEXT:    ret
+; CHECK-LABEL: test_vmulx_laneq_f32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmulx v0.2s, v0.2s, v1.s[3]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <4 x float> %v, <4 x float> undef, <2 x i32> <i32 3, i32 3>
   %vmulx2.i = tail call <2 x float> @llvm.aarch64.neon.fmulx.v2f32(<2 x float> %a, <2 x float> %shuffle)
@@ -2012,21 +1826,10 @@ entry:
 }
 
 define <4 x float> @test_vmulxq_laneq_f32(<4 x float> %a, <4 x float> %v) {
-; GENERIC-LABEL: test_vmulxq_laneq_f32:
-; GENERIC:       // %bb.0: // %entry
-; GENERIC-NEXT:    fmulx v0.4s, v0.4s, v1.s[3]
-; GENERIC-NEXT:    ret
-;
-; EXYNOSM1-LABEL: test_vmulxq_laneq_f32:
-; EXYNOSM1:       // %bb.0: // %entry
-; EXYNOSM1-NEXT:    dup v1.4s, v1.s[3]
-; EXYNOSM1-NEXT:    fmulx v0.4s, v0.4s, v1.4s
-; EXYNOSM1-NEXT:    ret
-;
-; EXYNOSM3-LABEL: test_vmulxq_laneq_f32:
-; EXYNOSM3:       // %bb.0: // %entry
-; EXYNOSM3-NEXT:    fmulx v0.4s, v0.4s, v1.s[3]
-; EXYNOSM3-NEXT:    ret
+; CHECK-LABEL: test_vmulxq_laneq_f32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmulx v0.4s, v0.4s, v1.s[3]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <4 x float> %v, <4 x float> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
   %vmulx2.i = tail call <4 x float> @llvm.aarch64.neon.fmulx.v4f32(<4 x float> %a, <4 x float> %shuffle)
@@ -2034,21 +1837,10 @@ entry:
 }
 
 define <2 x double> @test_vmulxq_laneq_f64(<2 x double> %a, <2 x double> %v) {
-; GENERIC-LABEL: test_vmulxq_laneq_f64:
-; GENERIC:       // %bb.0: // %entry
-; GENERIC-NEXT:    fmulx v0.2d, v0.2d, v1.d[1]
-; GENERIC-NEXT:    ret
-;
-; EXYNOSM1-LABEL: test_vmulxq_laneq_f64:
-; EXYNOSM1:       // %bb.0: // %entry
-; EXYNOSM1-NEXT:    dup v1.2d, v1.d[1]
-; EXYNOSM1-NEXT:    fmulx v0.2d, v0.2d, v1.2d
-; EXYNOSM1-NEXT:    ret
-;
-; EXYNOSM3-LABEL: test_vmulxq_laneq_f64:
-; EXYNOSM3:       // %bb.0: // %entry
-; EXYNOSM3-NEXT:    fmulx v0.2d, v0.2d, v1.d[1]
-; EXYNOSM3-NEXT:    ret
+; CHECK-LABEL: test_vmulxq_laneq_f64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmulx v0.2d, v0.2d, v1.d[1]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <2 x double> %v, <2 x double> undef, <2 x i32> <i32 1, i32 1>
   %vmulx2.i = tail call <2 x double> @llvm.aarch64.neon.fmulx.v2f64(<2 x double> %a, <2 x double> %shuffle)
@@ -2440,24 +2232,11 @@ entry:
 }
 
 define <2 x float> @test_vfma_lane_f32_0(<2 x float> %a, <2 x float> %b, <2 x float> %v) {
-; GENERIC-LABEL: test_vfma_lane_f32_0:
-; GENERIC:       // %bb.0: // %entry
-; GENERIC-NEXT:    // kill: def $d2 killed $d2 def $q2
-; GENERIC-NEXT:    fmla v0.2s, v1.2s, v2.s[0]
-; GENERIC-NEXT:    ret
-;
-; EXYNOSM1-LABEL: test_vfma_lane_f32_0:
-; EXYNOSM1:       // %bb.0: // %entry
-; EXYNOSM1-NEXT:    // kill: def $d2 killed $d2 def $q2
-; EXYNOSM1-NEXT:    dup v2.2s, v2.s[0]
-; EXYNOSM1-NEXT:    fmla v0.2s, v1.2s, v2.2s
-; EXYNOSM1-NEXT:    ret
-;
-; EXYNOSM3-LABEL: test_vfma_lane_f32_0:
-; EXYNOSM3:       // %bb.0: // %entry
-; EXYNOSM3-NEXT:    // kill: def $d2 killed $d2 def $q2
-; EXYNOSM3-NEXT:    fmla v0.2s, v1.2s, v2.s[0]
-; EXYNOSM3-NEXT:    ret
+; CHECK-LABEL: test_vfma_lane_f32_0:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-NEXT:    fmla v0.2s, v1.2s, v2.s[0]
+; CHECK-NEXT:    ret
 entry:
   %lane = shufflevector <2 x float> %v, <2 x float> undef, <2 x i32> zeroinitializer
   %0 = tail call <2 x float> @llvm.fma.v2f32(<2 x float> %lane, <2 x float> %b, <2 x float> %a)
@@ -2465,24 +2244,11 @@ entry:
 }
 
 define <4 x float> @test_vfmaq_lane_f32_0(<4 x float> %a, <4 x float> %b, <2 x float> %v) {
-; GENERIC-LABEL: test_vfmaq_lane_f32_0:
-; GENERIC:       // %bb.0: // %entry
-; GENERIC-NEXT:    // kill: def $d2 killed $d2 def $q2
-; GENERIC-NEXT:    fmla v0.4s, v1.4s, v2.s[0]
-; GENERIC-NEXT:    ret
-;
-; EXYNOSM1-LABEL: test_vfmaq_lane_f32_0:
-; EXYNOSM1:       // %bb.0: // %entry
-; EXYNOSM1-NEXT:    // kill: def $d2 killed $d2 def $q2
-; EXYNOSM1-NEXT:    dup v2.4s, v2.s[0]
-; EXYNOSM1-NEXT:    fmla v0.4s, v1.4s, v2.4s
-; EXYNOSM1-NEXT:    ret
-;
-; EXYNOSM3-LABEL: test_vfmaq_lane_f32_0:
-; EXYNOSM3:       // %bb.0: // %entry
-; EXYNOSM3-NEXT:    // kill: def $d2 killed $d2 def $q2
-; EXYNOSM3-NEXT:    fmla v0.4s, v1.4s, v2.s[0]
-; EXYNOSM3-NEXT:    ret
+; CHECK-LABEL: test_vfmaq_lane_f32_0:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-NEXT:    fmla v0.4s, v1.4s, v2.s[0]
+; CHECK-NEXT:    ret
 entry:
   %lane = shufflevector <2 x float> %v, <2 x float> undef, <4 x i32> zeroinitializer
   %0 = tail call <4 x float> @llvm.fma.v4f32(<4 x float> %lane, <4 x float> %b, <4 x float> %a)
@@ -2490,21 +2256,10 @@ entry:
 }
 
 define <2 x float> @test_vfma_laneq_f32_0(<2 x float> %a, <2 x float> %b, <4 x float> %v) {
-; GENERIC-LABEL: test_vfma_laneq_f32_0:
-; GENERIC:       // %bb.0: // %entry
-; GENERIC-NEXT:    fmla v0.2s, v1.2s, v2.s[0]
-; GENERIC-NEXT:    ret
-;
-; EXYNOSM1-LABEL: test_vfma_laneq_f32_0:
-; EXYNOSM1:       // %bb.0: // %entry
-; EXYNOSM1-NEXT:    dup v2.2s, v2.s[0]
-; EXYNOSM1-NEXT:    fmla v0.2s, v1.2s, v2.2s
-; EXYNOSM1-NEXT:    ret
-;
-; EXYNOSM3-LABEL: test_vfma_laneq_f32_0:
-; EXYNOSM3:       // %bb.0: // %entry
-; EXYNOSM3-NEXT:    fmla v0.2s, v1.2s, v2.s[0]
-; EXYNOSM3-NEXT:    ret
+; CHECK-LABEL: test_vfma_laneq_f32_0:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmla v0.2s, v1.2s, v2.s[0]
+; CHECK-NEXT:    ret
 entry:
   %lane = shufflevector <4 x float> %v, <4 x float> undef, <2 x i32> zeroinitializer
   %0 = tail call <2 x float> @llvm.fma.v2f32(<2 x float> %lane, <2 x float> %b, <2 x float> %a)
@@ -2512,21 +2267,10 @@ entry:
 }
 
 define <4 x float> @test_vfmaq_laneq_f32_0(<4 x float> %a, <4 x float> %b, <4 x float> %v) {
-; GENERIC-LABEL: test_vfmaq_laneq_f32_0:
-; GENERIC:       // %bb.0: // %entry
-; GENERIC-NEXT:    fmla v0.4s, v1.4s, v2.s[0]
-; GENERIC-NEXT:    ret
-;
-; EXYNOSM1-LABEL: test_vfmaq_laneq_f32_0:
-; EXYNOSM1:       // %bb.0: // %entry
-; EXYNOSM1-NEXT:    dup v2.4s, v2.s[0]
-; EXYNOSM1-NEXT:    fmla v0.4s, v1.4s, v2.4s
-; EXYNOSM1-NEXT:    ret
-;
-; EXYNOSM3-LABEL: test_vfmaq_laneq_f32_0:
-; EXYNOSM3:       // %bb.0: // %entry
-; EXYNOSM3-NEXT:    fmla v0.4s, v1.4s, v2.s[0]
-; EXYNOSM3-NEXT:    ret
+; CHECK-LABEL: test_vfmaq_laneq_f32_0:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmla v0.4s, v1.4s, v2.s[0]
+; CHECK-NEXT:    ret
 entry:
   %lane = shufflevector <4 x float> %v, <4 x float> undef, <4 x i32> zeroinitializer
   %0 = tail call <4 x float> @llvm.fma.v4f32(<4 x float> %lane, <4 x float> %b, <4 x float> %a)
@@ -2534,24 +2278,11 @@ entry:
 }
 
 define <2 x float> @test_vfms_lane_f32_0(<2 x float> %a, <2 x float> %b, <2 x float> %v) {
-; GENERIC-LABEL: test_vfms_lane_f32_0:
-; GENERIC:       // %bb.0: // %entry
-; GENERIC-NEXT:    // kill: def $d2 killed $d2 def $q2
-; GENERIC-NEXT:    fmls v0.2s, v1.2s, v2.s[0]
-; GENERIC-NEXT:    ret
-;
-; EXYNOSM1-LABEL: test_vfms_lane_f32_0:
-; EXYNOSM1:       // %bb.0: // %entry
-; EXYNOSM1-NEXT:    // kill: def $d2 killed $d2 def $q2
-; EXYNOSM1-NEXT:    dup v2.2s, v2.s[0]
-; EXYNOSM1-NEXT:    fmls v0.2s, v1.2s, v2.2s
-; EXYNOSM1-NEXT:    ret
-;
-; EXYNOSM3-LABEL: test_vfms_lane_f32_0:
-; EXYNOSM3:       // %bb.0: // %entry
-; EXYNOSM3-NEXT:    // kill: def $d2 killed $d2 def $q2
-; EXYNOSM3-NEXT:    fmls v0.2s, v1.2s, v2.s[0]
-; EXYNOSM3-NEXT:    ret
+; CHECK-LABEL: test_vfms_lane_f32_0:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-NEXT:    fmls v0.2s, v1.2s, v2.s[0]
+; CHECK-NEXT:    ret
 entry:
   %sub = fsub <2 x float> <float -0.000000e+00, float -0.000000e+00>, %v
   %lane = shufflevector <2 x float> %sub, <2 x float> undef, <2 x i32> zeroinitializer
@@ -2560,24 +2291,11 @@ entry:
 }
 
 define <4 x float> @test_vfmsq_lane_f32_0(<4 x float> %a, <4 x float> %b, <2 x float> %v) {
-; GENERIC-LABEL: test_vfmsq_lane_f32_0:
-; GENERIC:       // %bb.0: // %entry
-; GENERIC-NEXT:    // kill: def $d2 killed $d2 def $q2
-; GENERIC-NEXT:    fmls v0.4s, v1.4s, v2.s[0]
-; GENERIC-NEXT:    ret
-;
-; EXYNOSM1-LABEL: test_vfmsq_lane_f32_0:
-; EXYNOSM1:       // %bb.0: // %entry
-; EXYNOSM1-NEXT:    // kill: def $d2 killed $d2 def $q2
-; EXYNOSM1-NEXT:    dup v2.4s, v2.s[0]
-; EXYNOSM1-NEXT:    fmls v0.4s, v1.4s, v2.4s
-; EXYNOSM1-NEXT:    ret
-;
-; EXYNOSM3-LABEL: test_vfmsq_lane_f32_0:
-; EXYNOSM3:       // %bb.0: // %entry
-; EXYNOSM3-NEXT:    // kill: def $d2 killed $d2 def $q2
-; EXYNOSM3-NEXT:    fmls v0.4s, v1.4s, v2.s[0]
-; EXYNOSM3-NEXT:    ret
+; CHECK-LABEL: test_vfmsq_lane_f32_0:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-NEXT:    fmls v0.4s, v1.4s, v2.s[0]
+; CHECK-NEXT:    ret
 entry:
   %sub = fsub <2 x float> <float -0.000000e+00, float -0.000000e+00>, %v
   %lane = shufflevector <2 x float> %sub, <2 x float> undef, <4 x i32> zeroinitializer
@@ -2586,21 +2304,10 @@ entry:
 }
 
 define <2 x float> @test_vfms_laneq_f32_0(<2 x float> %a, <2 x float> %b, <4 x float> %v) {
-; GENERIC-LABEL: test_vfms_laneq_f32_0:
-; GENERIC:       // %bb.0: // %entry
-; GENERIC-NEXT:    fmls v0.2s, v1.2s, v2.s[0]
-; GENERIC-NEXT:    ret
-;
-; EXYNOSM1-LABEL: test_vfms_laneq_f32_0:
-; EXYNOSM1:       // %bb.0: // %entry
-; EXYNOSM1-NEXT:    dup v2.2s, v2.s[0]
-; EXYNOSM1-NEXT:    fmls v0.2s, v1.2s, v2.2s
-; EXYNOSM1-NEXT:    ret
-;
-; EXYNOSM3-LABEL: test_vfms_laneq_f32_0:
-; EXYNOSM3:       // %bb.0: // %entry
-; EXYNOSM3-NEXT:    fmls v0.2s, v1.2s, v2.s[0]
-; EXYNOSM3-NEXT:    ret
+; CHECK-LABEL: test_vfms_laneq_f32_0:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmls v0.2s, v1.2s, v2.s[0]
+; CHECK-NEXT:    ret
 entry:
   %sub = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %v
   %lane = shufflevector <4 x float> %sub, <4 x float> undef, <2 x i32> zeroinitializer
@@ -2609,21 +2316,10 @@ entry:
 }
 
 define <4 x float> @test_vfmsq_laneq_f32_0(<4 x float> %a, <4 x float> %b, <4 x float> %v) {
-; GENERIC-LABEL: test_vfmsq_laneq_f32_0:
-; GENERIC:       // %bb.0: // %entry
-; GENERIC-NEXT:    fmls v0.4s, v1.4s, v2.s[0]
-; GENERIC-NEXT:    ret
-;
-; EXYNOSM1-LABEL: test_vfmsq_laneq_f32_0:
-; EXYNOSM1:       // %bb.0: // %entry
-; EXYNOSM1-NEXT:    dup v2.4s, v2.s[0]
-; EXYNOSM1-NEXT:    fmls v0.4s, v1.4s, v2.4s
-; EXYNOSM1-NEXT:    ret
-;
-; EXYNOSM3-LABEL: test_vfmsq_laneq_f32_0:
-; EXYNOSM3:       // %bb.0: // %entry
-; EXYNOSM3-NEXT:    fmls v0.4s, v1.4s, v2.s[0]
-; EXYNOSM3-NEXT:    ret
+; CHECK-LABEL: test_vfmsq_laneq_f32_0:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmls v0.4s, v1.4s, v2.s[0]
+; CHECK-NEXT:    ret
 entry:
   %sub = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %v
   %lane = shufflevector <4 x float> %sub, <4 x float> undef, <4 x i32> zeroinitializer
@@ -2632,21 +2328,10 @@ entry:
 }
 
 define <2 x double> @test_vfmaq_laneq_f64_0(<2 x double> %a, <2 x double> %b, <2 x double> %v) {
-; GENERIC-LABEL: test_vfmaq_laneq_f64_0:
-; GENERIC:       // %bb.0: // %entry
-; GENERIC-NEXT:    fmla v0.2d, v1.2d, v2.d[0]
-; GENERIC-NEXT:    ret
-;
-; EXYNOSM1-LABEL: test_vfmaq_laneq_f64_0:
-; EXYNOSM1:       // %bb.0: // %entry
-; EXYNOSM1-NEXT:    dup v2.2d, v2.d[0]
-; EXYNOSM1-NEXT:    fmla v0.2d, v1.2d, v2.2d
-; EXYNOSM1-NEXT:    ret
-;
-; EXYNOSM3-LABEL: test_vfmaq_laneq_f64_0:
-; EXYNOSM3:       // %bb.0: // %entry
-; EXYNOSM3-NEXT:    fmla v0.2d, v1.2d, v2.d[0]
-; EXYNOSM3-NEXT:    ret
+; CHECK-LABEL: test_vfmaq_laneq_f64_0:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmla v0.2d, v1.2d, v2.d[0]
+; CHECK-NEXT:    ret
 entry:
   %lane = shufflevector <2 x double> %v, <2 x double> undef, <2 x i32> zeroinitializer
   %0 = tail call <2 x double> @llvm.fma.v2f64(<2 x double> %lane, <2 x double> %b, <2 x double> %a)
@@ -2654,21 +2339,10 @@ entry:
 }
 
 define <2 x double> @test_vfmsq_laneq_f64_0(<2 x double> %a, <2 x double> %b, <2 x double> %v) {
-; GENERIC-LABEL: test_vfmsq_laneq_f64_0:
-; GENERIC:       // %bb.0: // %entry
-; GENERIC-NEXT:    fmls v0.2d, v1.2d, v2.d[0]
-; GENERIC-NEXT:    ret
-;
-; EXYNOSM1-LABEL: test_vfmsq_laneq_f64_0:
-; EXYNOSM1:       // %bb.0: // %entry
-; EXYNOSM1-NEXT:    dup v2.2d, v2.d[0]
-; EXYNOSM1-NEXT:    fmls v0.2d, v1.2d, v2.2d
-; EXYNOSM1-NEXT:    ret
-;
-; EXYNOSM3-LABEL: test_vfmsq_laneq_f64_0:
-; EXYNOSM3:       // %bb.0: // %entry
-; EXYNOSM3-NEXT:    fmls v0.2d, v1.2d, v2.d[0]
-; EXYNOSM3-NEXT:    ret
+; CHECK-LABEL: test_vfmsq_laneq_f64_0:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmls v0.2d, v1.2d, v2.d[0]
+; CHECK-NEXT:    ret
 entry:
   %sub = fsub <2 x double> <double -0.000000e+00, double -0.000000e+00>, %v
   %lane = shufflevector <2 x double> %sub, <2 x double> undef, <2 x i32> zeroinitializer
@@ -3585,24 +3259,11 @@ entry:
 }
 
 define <2 x float> @test_vmul_lane_f32_0(<2 x float> %a, <2 x float> %v) {
-; GENERIC-LABEL: test_vmul_lane_f32_0:
-; GENERIC:       // %bb.0: // %entry
-; GENERIC-NEXT:    // kill: def $d1 killed $d1 def $q1
-; GENERIC-NEXT:    fmul v0.2s, v0.2s, v1.s[0]
-; GENERIC-NEXT:    ret
-;
-; EXYNOSM1-LABEL: test_vmul_lane_f32_0:
-; EXYNOSM1:       // %bb.0: // %entry
-; EXYNOSM1-NEXT:    // kill: def $d1 killed $d1 def $q1
-; EXYNOSM1-NEXT:    dup v1.2s, v1.s[0]
-; EXYNOSM1-NEXT:    fmul v0.2s, v0.2s, v1.2s
-; EXYNOSM1-NEXT:    ret
-;
-; EXYNOSM3-LABEL: test_vmul_lane_f32_0:
-; EXYNOSM3:       // %bb.0: // %entry
-; EXYNOSM3-NEXT:    // kill: def $d1 killed $d1 def $q1
-; EXYNOSM3-NEXT:    fmul v0.2s, v0.2s, v1.s[0]
-; EXYNOSM3-NEXT:    ret
+; CHECK-LABEL: test_vmul_lane_f32_0:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT:    fmul v0.2s, v0.2s, v1.s[0]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <2 x float> %v, <2 x float> undef, <2 x i32> zeroinitializer
   %mul = fmul <2 x float> %shuffle, %a
@@ -3610,24 +3271,11 @@ entry:
 }
 
 define <4 x float> @test_vmulq_lane_f32_0(<4 x float> %a, <2 x float> %v) {
-; GENERIC-LABEL: test_vmulq_lane_f32_0:
-; GENERIC:       // %bb.0: // %entry
-; GENERIC-NEXT:    // kill: def $d1 killed $d1 def $q1
-; GENERIC-NEXT:    fmul v0.4s, v0.4s, v1.s[0]
-; GENERIC-NEXT:    ret
-;
-; EXYNOSM1-LABEL: test_vmulq_lane_f32_0:
-; EXYNOSM1:       // %bb.0: // %entry
-; EXYNOSM1-NEXT:    // kill: def $d1 killed $d1 def $q1
-; EXYNOSM1-NEXT:    dup v1.4s, v1.s[0]
-; EXYNOSM1-NEXT:    fmul v0.4s, v0.4s, v1.4s
-; EXYNOSM1-NEXT:    ret
-;
-; EXYNOSM3-LABEL: test_vmulq_lane_f32_0:
-; EXYNOSM3:       // %bb.0: // %entry
-; EXYNOSM3-NEXT:    // kill: def $d1 killed $d1 def $q1
-; EXYNOSM3-NEXT:    fmul v0.4s, v0.4s, v1.s[0]
-; EXYNOSM3-NEXT:    ret
+; CHECK-LABEL: test_vmulq_lane_f32_0:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT:    fmul v0.4s, v0.4s, v1.s[0]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <2 x float> %v, <2 x float> undef, <4 x i32> zeroinitializer
   %mul = fmul <4 x float> %shuffle, %a
@@ -3635,21 +3283,10 @@ entry:
 }
 
 define <2 x float> @test_vmul_laneq_f32_0(<2 x float> %a, <4 x float> %v) {
-; GENERIC-LABEL: test_vmul_laneq_f32_0:
-; GENERIC:       // %bb.0: // %entry
-; GENERIC-NEXT:    fmul v0.2s, v0.2s, v1.s[0]
-; GENERIC-NEXT:    ret
-;
-; EXYNOSM1-LABEL: test_vmul_laneq_f32_0:
-; EXYNOSM1:       // %bb.0: // %entry
-; EXYNOSM1-NEXT:    dup v1.2s, v1.s[0]
-; EXYNOSM1-NEXT:    fmul v0.2s, v0.2s, v1.2s
-; EXYNOSM1-NEXT:    ret
-;
-; EXYNOSM3-LABEL: test_vmul_laneq_f32_0:
-; EXYNOSM3:       // %bb.0: // %entry
-; EXYNOSM3-NEXT:    fmul v0.2s, v0.2s, v1.s[0]
-; EXYNOSM3-NEXT:    ret
+; CHECK-LABEL: test_vmul_laneq_f32_0:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmul v0.2s, v0.2s, v1.s[0]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <4 x float> %v, <4 x float> undef, <2 x i32> zeroinitializer
   %mul = fmul <2 x float> %shuffle, %a
@@ -3671,21 +3308,10 @@ entry:
 }
 
 define <4 x float> @test_vmulq_laneq_f32_0(<4 x float> %a, <4 x float> %v) {
-; GENERIC-LABEL: test_vmulq_laneq_f32_0:
-; GENERIC:       // %bb.0: // %entry
-; GENERIC-NEXT:    fmul v0.4s, v0.4s, v1.s[0]
-; GENERIC-NEXT:    ret
-;
-; EXYNOSM1-LABEL: test_vmulq_laneq_f32_0:
-; EXYNOSM1:       // %bb.0: // %entry
-; EXYNOSM1-NEXT:    dup v1.4s, v1.s[0]
-; EXYNOSM1-NEXT:    fmul v0.4s, v0.4s, v1.4s
-; EXYNOSM1-NEXT:    ret
-;
-; EXYNOSM3-LABEL: test_vmulq_laneq_f32_0:
-; EXYNOSM3:       // %bb.0: // %entry
-; EXYNOSM3-NEXT:    fmul v0.4s, v0.4s, v1.s[0]
-; EXYNOSM3-NEXT:    ret
+; CHECK-LABEL: test_vmulq_laneq_f32_0:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmul v0.4s, v0.4s, v1.s[0]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <4 x float> %v, <4 x float> undef, <4 x i32> zeroinitializer
   %mul = fmul <4 x float> %shuffle, %a
@@ -3693,21 +3319,10 @@ entry:
 }
 
 define <2 x double> @test_vmulq_laneq_f64_0(<2 x double> %a, <2 x double> %v) {
-; GENERIC-LABEL: test_vmulq_laneq_f64_0:
-; GENERIC:       // %bb.0: // %entry
-; GENERIC-NEXT:    fmul v0.2d, v0.2d, v1.d[0]
-; GENERIC-NEXT:    ret
-;
-; EXYNOSM1-LABEL: test_vmulq_laneq_f64_0:
-; EXYNOSM1:       // %bb.0: // %entry
-; EXYNOSM1-NEXT:    dup v1.2d, v1.d[0]
-; EXYNOSM1-NEXT:    fmul v0.2d, v0.2d, v1.2d
-; EXYNOSM1-NEXT:    ret
-;
-; EXYNOSM3-LABEL: test_vmulq_laneq_f64_0:
-; EXYNOSM3:       // %bb.0: // %entry
-; EXYNOSM3-NEXT:    fmul v0.2d, v0.2d, v1.d[0]
-; EXYNOSM3-NEXT:    ret
+; CHECK-LABEL: test_vmulq_laneq_f64_0:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmul v0.2d, v0.2d, v1.d[0]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <2 x double> %v, <2 x double> undef, <2 x i32> zeroinitializer
   %mul = fmul <2 x double> %shuffle, %a
@@ -3715,24 +3330,11 @@ entry:
 }
 
 define <2 x float> @test_vmulx_lane_f32_0(<2 x float> %a, <2 x float> %v) {
-; GENERIC-LABEL: test_vmulx_lane_f32_0:
-; GENERIC:       // %bb.0: // %entry
-; GENERIC-NEXT:    // kill: def $d1 killed $d1 def $q1
-; GENERIC-NEXT:    fmulx v0.2s, v0.2s, v1.s[0]
-; GENERIC-NEXT:    ret
-;
-; EXYNOSM1-LABEL: test_vmulx_lane_f32_0:
-; EXYNOSM1:       // %bb.0: // %entry
-; EXYNOSM1-NEXT:    // kill: def $d1 killed $d1 def $q1
-; EXYNOSM1-NEXT:    dup v1.2s, v1.s[0]
-; EXYNOSM1-NEXT:    fmulx v0.2s, v0.2s, v1.2s
-; EXYNOSM1-NEXT:    ret
-;
-; EXYNOSM3-LABEL: test_vmulx_lane_f32_0:
-; EXYNOSM3:       // %bb.0: // %entry
-; EXYNOSM3-NEXT:    // kill: def $d1 killed $d1 def $q1
-; EXYNOSM3-NEXT:    fmulx v0.2s, v0.2s, v1.s[0]
-; EXYNOSM3-NEXT:    ret
+; CHECK-LABEL: test_vmulx_lane_f32_0:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT:    fmulx v0.2s, v0.2s, v1.s[0]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <2 x float> %v, <2 x float> undef, <2 x i32> zeroinitializer
   %vmulx2.i = tail call <2 x float> @llvm.aarch64.neon.fmulx.v2f32(<2 x float> %a, <2 x float> %shuffle)
@@ -3740,24 +3342,11 @@ entry:
 }
 
 define <4 x float> @test_vmulxq_lane_f32_0(<4 x float> %a, <2 x float> %v) {
-; GENERIC-LABEL: test_vmulxq_lane_f32_0:
-; GENERIC:       // %bb.0: // %entry
-; GENERIC-NEXT:    // kill: def $d1 killed $d1 def $q1
-; GENERIC-NEXT:    fmulx v0.4s, v0.4s, v1.s[0]
-; GENERIC-NEXT:    ret
-;
-; EXYNOSM1-LABEL: test_vmulxq_lane_f32_0:
-; EXYNOSM1:       // %bb.0: // %entry
-; EXYNOSM1-NEXT:    // kill: def $d1 killed $d1 def $q1
-; EXYNOSM1-NEXT:    dup v1.4s, v1.s[0]
-; EXYNOSM1-NEXT:    fmulx v0.4s, v0.4s, v1.4s
-; EXYNOSM1-NEXT:    ret
-;
-; EXYNOSM3-LABEL: test_vmulxq_lane_f32_0:
-; EXYNOSM3:       // %bb.0: // %entry
-; EXYNOSM3-NEXT:    // kill: def $d1 killed $d1 def $q1
-; EXYNOSM3-NEXT:    fmulx v0.4s, v0.4s, v1.s[0]
-; EXYNOSM3-NEXT:    ret
+; CHECK-LABEL: test_vmulxq_lane_f32_0:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT:    fmulx v0.4s, v0.4s, v1.s[0]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <2 x float> %v, <2 x float> undef, <4 x i32> zeroinitializer
   %vmulx2.i = tail call <4 x float> @llvm.aarch64.neon.fmulx.v4f32(<4 x float> %a, <4 x float> %shuffle)
@@ -3765,24 +3354,11 @@ entry:
 }
 
 define <2 x double> @test_vmulxq_lane_f64_0(<2 x double> %a, <1 x double> %v) {
-; GENERIC-LABEL: test_vmulxq_lane_f64_0:
-; GENERIC:       // %bb.0: // %entry
-; GENERIC-NEXT:    // kill: def $d1 killed $d1 def $q1
-; GENERIC-NEXT:    fmulx v0.2d, v0.2d, v1.d[0]
-; GENERIC-NEXT:    ret
-;
-; EXYNOSM1-LABEL: test_vmulxq_lane_f64_0:
-; EXYNOSM1:       // %bb.0: // %entry
-; EXYNOSM1-NEXT:    // kill: def $d1 killed $d1 def $q1
-; EXYNOSM1-NEXT:    dup v1.2d, v1.d[0]
-; EXYNOSM1-NEXT:    fmulx v0.2d, v0.2d, v1.2d
-; EXYNOSM1-NEXT:    ret
-;
-; EXYNOSM3-LABEL: test_vmulxq_lane_f64_0:
-; EXYNOSM3:       // %bb.0: // %entry
-; EXYNOSM3-NEXT:    // kill: def $d1 killed $d1 def $q1
-; EXYNOSM3-NEXT:    fmulx v0.2d, v0.2d, v1.d[0]
-; EXYNOSM3-NEXT:    ret
+; CHECK-LABEL: test_vmulxq_lane_f64_0:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT:    fmulx v0.2d, v0.2d, v1.d[0]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <1 x double> %v, <1 x double> undef, <2 x i32> zeroinitializer
   %vmulx2.i = tail call <2 x double> @llvm.aarch64.neon.fmulx.v2f64(<2 x double> %a, <2 x double> %shuffle)
@@ -3790,21 +3366,10 @@ entry:
 }
 
 define <2 x float> @test_vmulx_laneq_f32_0(<2 x float> %a, <4 x float> %v) {
-; GENERIC-LABEL: test_vmulx_laneq_f32_0:
-; GENERIC:       // %bb.0: // %entry
-; GENERIC-NEXT:    fmulx v0.2s, v0.2s, v1.s[0]
-; GENERIC-NEXT:    ret
-;
-; EXYNOSM1-LABEL: test_vmulx_laneq_f32_0:
-; EXYNOSM1:       // %bb.0: // %entry
-; EXYNOSM1-NEXT:    dup v1.2s, v1.s[0]
-; EXYNOSM1-NEXT:    fmulx v0.2s, v0.2s, v1.2s
-; EXYNOSM1-NEXT:    ret
-;
-; EXYNOSM3-LABEL: test_vmulx_laneq_f32_0:
-; EXYNOSM3:       // %bb.0: // %entry
-; EXYNOSM3-NEXT:    fmulx v0.2s, v0.2s, v1.s[0]
-; EXYNOSM3-NEXT:    ret
+; CHECK-LABEL: test_vmulx_laneq_f32_0:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmulx v0.2s, v0.2s, v1.s[0]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <4 x float> %v, <4 x float> undef, <2 x i32> zeroinitializer
   %vmulx2.i = tail call <2 x float> @llvm.aarch64.neon.fmulx.v2f32(<2 x float> %a, <2 x float> %shuffle)
@@ -3812,21 +3377,10 @@ entry:
 }
 
 define <4 x float> @test_vmulxq_laneq_f32_0(<4 x float> %a, <4 x float> %v) {
-; GENERIC-LABEL: test_vmulxq_laneq_f32_0:
-; GENERIC:       // %bb.0: // %entry
-; GENERIC-NEXT:    fmulx v0.4s, v0.4s, v1.s[0]
-; GENERIC-NEXT:    ret
-;
-; EXYNOSM1-LABEL: test_vmulxq_laneq_f32_0:
-; EXYNOSM1:       // %bb.0: // %entry
-; EXYNOSM1-NEXT:    dup v1.4s, v1.s[0]
-; EXYNOSM1-NEXT:    fmulx v0.4s, v0.4s, v1.4s
-; EXYNOSM1-NEXT:    ret
-;
-; EXYNOSM3-LABEL: test_vmulxq_laneq_f32_0:
-; EXYNOSM3:       // %bb.0: // %entry
-; EXYNOSM3-NEXT:    fmulx v0.4s, v0.4s, v1.s[0]
-; EXYNOSM3-NEXT:    ret
+; CHECK-LABEL: test_vmulxq_laneq_f32_0:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmulx v0.4s, v0.4s, v1.s[0]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <4 x float> %v, <4 x float> undef, <4 x i32> zeroinitializer
   %vmulx2.i = tail call <4 x float> @llvm.aarch64.neon.fmulx.v4f32(<4 x float> %a, <4 x float> %shuffle)
@@ -3834,21 +3388,10 @@ entry:
 }
 
 define <2 x double> @test_vmulxq_laneq_f64_0(<2 x double> %a, <2 x double> %v) {
-; GENERIC-LABEL: test_vmulxq_laneq_f64_0:
-; GENERIC:       // %bb.0: // %entry
-; GENERIC-NEXT:    fmulx v0.2d, v0.2d, v1.d[0]
-; GENERIC-NEXT:    ret
-;
-; EXYNOSM1-LABEL: test_vmulxq_laneq_f64_0:
-; EXYNOSM1:       // %bb.0: // %entry
-; EXYNOSM1-NEXT:    dup v1.2d, v1.d[0]
-; EXYNOSM1-NEXT:    fmulx v0.2d, v0.2d, v1.2d
-; EXYNOSM1-NEXT:    ret
-;
-; EXYNOSM3-LABEL: test_vmulxq_laneq_f64_0:
-; EXYNOSM3:       // %bb.0: // %entry
-; EXYNOSM3-NEXT:    fmulx v0.2d, v0.2d, v1.d[0]
-; EXYNOSM3-NEXT:    ret
+; CHECK-LABEL: test_vmulxq_laneq_f64_0:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmulx v0.2d, v0.2d, v1.d[0]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <2 x double> %v, <2 x double> undef, <2 x i32> zeroinitializer
   %vmulx2.i = tail call <2 x double> @llvm.aarch64.neon.fmulx.v2f64(<2 x double> %a, <2 x double> %shuffle)
@@ -3856,24 +3399,11 @@ entry:
 }
 
 define <4 x float> @optimize_dup(<4 x float> %a, <4 x float> %b, <4 x float> %c, <4 x float> %v) {
-; GENERIC-LABEL: optimize_dup:
-; GENERIC:       // %bb.0: // %entry
-; GENERIC-NEXT:    fmla v0.4s, v1.4s, v3.s[3]
-; GENERIC-NEXT:    fmls v0.4s, v2.4s, v3.s[3]
-; GENERIC-NEXT:    ret
-;
-; EXYNOSM1-LABEL: optimize_dup:
-; EXYNOSM1:       // %bb.0: // %entry
-; EXYNOSM1-NEXT:    dup v3.4s, v3.s[3]
-; EXYNOSM1-NEXT:    fmla v0.4s, v1.4s, v3.4s
-; EXYNOSM1-NEXT:    fmls v0.4s, v2.4s, v3.4s
-; EXYNOSM1-NEXT:    ret
-;
-; EXYNOSM3-LABEL: optimize_dup:
-; EXYNOSM3:       // %bb.0: // %entry
-; EXYNOSM3-NEXT:    fmla v0.4s, v1.4s, v3.s[3]
-; EXYNOSM3-NEXT:    fmls v0.4s, v2.4s, v3.s[3]
-; EXYNOSM3-NEXT:    ret
+; CHECK-LABEL: optimize_dup:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmla v0.4s, v1.4s, v3.s[3]
+; CHECK-NEXT:    fmls v0.4s, v2.4s, v3.s[3]
+; CHECK-NEXT:    ret
 entry:
   %lane1 = shufflevector <4 x float> %v, <4 x float> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
   %0 = tail call <4 x float> @llvm.fma.v4f32(<4 x float> %lane1, <4 x float> %b, <4 x float> %a)
@@ -3884,25 +3414,11 @@ entry:
 }
 
 define <4 x float> @no_optimize_dup(<4 x float> %a, <4 x float> %b, <4 x float> %c, <4 x float> %v) {
-; GENERIC-LABEL: no_optimize_dup:
-; GENERIC:       // %bb.0: // %entry
-; GENERIC-NEXT:    fmla v0.4s, v1.4s, v3.s[3]
-; GENERIC-NEXT:    fmls v0.4s, v2.4s, v3.s[1]
-; GENERIC-NEXT:    ret
-;
-; EXYNOSM1-LABEL: no_optimize_dup:
-; EXYNOSM1:       // %bb.0: // %entry
-; EXYNOSM1-NEXT:    dup v4.4s, v3.s[3]
-; EXYNOSM1-NEXT:    fmla v0.4s, v1.4s, v4.4s
-; EXYNOSM1-NEXT:    dup v1.4s, v3.s[1]
-; EXYNOSM1-NEXT:    fmls v0.4s, v2.4s, v1.4s
-; EXYNOSM1-NEXT:    ret
-;
-; EXYNOSM3-LABEL: no_optimize_dup:
-; EXYNOSM3:       // %bb.0: // %entry
-; EXYNOSM3-NEXT:    fmla v0.4s, v1.4s, v3.s[3]
-; EXYNOSM3-NEXT:    fmls v0.4s, v2.4s, v3.s[1]
-; EXYNOSM3-NEXT:    ret
+; CHECK-LABEL: no_optimize_dup:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmla v0.4s, v1.4s, v3.s[3]
+; CHECK-NEXT:    fmls v0.4s, v2.4s, v3.s[1]
+; CHECK-NEXT:    ret
 entry:
   %lane1 = shufflevector <4 x float> %v, <4 x float> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
   %0 = tail call <4 x float> @llvm.fma.v4f32(<4 x float> %lane1, <4 x float> %b, <4 x float> %a)
@@ -3913,50 +3429,11 @@ entry:
 }
 
 define <2 x float> @test_vfma_lane_simdinstr_opt_pass_caching_a57(<2 x float> %a, <2 x float> %b, <2 x float> %v) "target-cpu"="cortex-a57" {
-; GENERIC-LABEL: test_vfma_lane_simdinstr_opt_pass_caching_a57:
-; GENERIC:       // %bb.0: // %entry
-; GENERIC-NEXT:    // kill: def $d2 killed $d2 def $q2
-; GENERIC-NEXT:    fmla v0.2s, v1.2s, v2.s[1]
-; GENERIC-NEXT:    ret
-;
-; EXYNOSM1-LABEL: test_vfma_lane_simdinstr_opt_pass_caching_a57:
-; EXYNOSM1:       // %bb.0: // %entry
-; EXYNOSM1-NEXT:    // kill: def $d2 killed $d2 def $q2
-; EXYNOSM1-NEXT:    dup v2.2s, v2.s[1]
-; EXYNOSM1-NEXT:    fmla v0.2s, v1.2s, v2.2s
-; EXYNOSM1-NEXT:    ret
-;
-; EXYNOSM3-LABEL: test_vfma_lane_simdinstr_opt_pass_caching_a57:
-; EXYNOSM3:       // %bb.0: // %entry
-; EXYNOSM3-NEXT:    // kill: def $d2 killed $d2 def $q2
-; EXYNOSM3-NEXT:    fmla v0.2s, v1.2s, v2.s[1]
-; EXYNOSM3-NEXT:    ret
-entry:
-  %lane = shufflevector <2 x float> %v, <2 x float> undef, <2 x i32> <i32 1, i32 1>
-  %0 = tail call <2 x float> @llvm.fma.v2f32(<2 x float> %lane, <2 x float> %b, <2 x float> %a)
-  ret <2 x float> %0
-}
-
-define <2 x float> @test_vfma_lane_simdinstr_opt_pass_caching_m1(<2 x float> %a, <2 x float> %b, <2 x float> %v) "target-cpu"="exynos-m1" {
-; GENERIC-LABEL: test_vfma_lane_simdinstr_opt_pass_caching_m1:
-; GENERIC:       // %bb.0: // %entry
-; GENERIC-NEXT:    // kill: def $d2 killed $d2 def $q2
-; GENERIC-NEXT:    dup v2.2s, v2.s[1]
-; GENERIC-NEXT:    fmla v0.2s, v1.2s, v2.2s
-; GENERIC-NEXT:    ret
-;
-; EXYNOSM1-LABEL: test_vfma_lane_simdinstr_opt_pass_caching_m1:
-; EXYNOSM1:       // %bb.0: // %entry
-; EXYNOSM1-NEXT:    // kill: def $d2 killed $d2 def $q2
-; EXYNOSM1-NEXT:    dup v2.2s, v2.s[1]
-; EXYNOSM1-NEXT:    fmla v0.2s, v1.2s, v2.2s
-; EXYNOSM1-NEXT:    ret
-;
-; EXYNOSM3-LABEL: test_vfma_lane_simdinstr_opt_pass_caching_m1:
-; EXYNOSM3:       // %bb.0: // %entry
-; EXYNOSM3-NEXT:    // kill: def $d2 killed $d2 def $q2
-; EXYNOSM3-NEXT:    fmla v0.2s, v1.2s, v2.s[1]
-; EXYNOSM3-NEXT:    ret
+; CHECK-LABEL: test_vfma_lane_simdinstr_opt_pass_caching_a57:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-NEXT:    fmla v0.2s, v1.2s, v2.s[1]
+; CHECK-NEXT:    ret
 entry:
   %lane = shufflevector <2 x float> %v, <2 x float> undef, <2 x i32> <i32 1, i32 1>
   %0 = tail call <2 x float> @llvm.fma.v2f32(<2 x float> %lane, <2 x float> %b, <2 x float> %a)
@@ -3964,24 +3441,11 @@ entry:
 }
 
 define <2 x float> @test_vfma_lane_simdinstr_opt_pass_caching_m3(<2 x float> %a, <2 x float> %b, <2 x float> %v) "target-cpu"="exynos-m3" {
-; GENERIC-LABEL: test_vfma_lane_simdinstr_opt_pass_caching_m3:
-; GENERIC:       // %bb.0: // %entry
-; GENERIC-NEXT:    // kill: def $d2 killed $d2 def $q2
-; GENERIC-NEXT:    fmla v0.2s, v1.2s, v2.s[1]
-; GENERIC-NEXT:    ret
-;
-; EXYNOSM1-LABEL: test_vfma_lane_simdinstr_opt_pass_caching_m3:
-; EXYNOSM1:       // %bb.0: // %entry
-; EXYNOSM1-NEXT:    // kill: def $d2 killed $d2 def $q2
-; EXYNOSM1-NEXT:    dup v2.2s, v2.s[1]
-; EXYNOSM1-NEXT:    fmla v0.2s, v1.2s, v2.2s
-; EXYNOSM1-NEXT:    ret
-;
-; EXYNOSM3-LABEL: test_vfma_lane_simdinstr_opt_pass_caching_m3:
-; EXYNOSM3:       // %bb.0: // %entry
-; EXYNOSM3-NEXT:    // kill: def $d2 killed $d2 def $q2
-; EXYNOSM3-NEXT:    fmla v0.2s, v1.2s, v2.s[1]
-; EXYNOSM3-NEXT:    ret
+; CHECK-LABEL: test_vfma_lane_simdinstr_opt_pass_caching_m3:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-NEXT:    fmla v0.2s, v1.2s, v2.s[1]
+; CHECK-NEXT:    ret
 entry:
   %lane = shufflevector <2 x float> %v, <2 x float> undef, <2 x i32> <i32 1, i32 1>
   %0 = tail call <2 x float> @llvm.fma.v2f32(<2 x float> %lane, <2 x float> %b, <2 x float> %a)
diff --git a/llvm/test/CodeGen/AArch64/arm64-neon-vector-shuffle-extract.ll b/llvm/test/CodeGen/AArch64/arm64-neon-vector-shuffle-extract.ll
index 7ed0e59e2..2be8b014e 100644
--- a/llvm/test/CodeGen/AArch64/arm64-neon-vector-shuffle-extract.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-neon-vector-shuffle-extract.ll
@@ -31,3 +31,29 @@ define <4 x i32> @widen_shuffles_reduced(<3 x i32> %x, <3 x i32> %y) {
   %s3 = shufflevector <3 x i32> %y, <3 x i32> %x, <4 x i32> <i32 1, i32 4, i32 3, i32 0>
   ret <4 x i32> %s3
 }
+
+define void @zip_mask_check(<3 x float>* %p1, <3 x float>* %p2, i32* %p3) {
+; CHECK-LABEL: zip_mask_check:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    trn2 v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    fmla v0.4s, v0.4s, v0.4s
+; CHECK-NEXT:    fmla v0.4s, v0.4s, v0.4s
+; CHECK-NEXT:    str s0, [x2]
+; CHECK-NEXT:    ret
+  %tmp3 = load <3 x float>, <3 x float>* %p1, align 16
+  %tmp4 = load <3 x float>, <3 x float>* %p2, align 4
+  %tmp5 = shufflevector <3 x float> %tmp3, <3 x float> %tmp4, <4 x i32> <i32 1, i32 4, i32 undef, i32 undef>
+  %tmp6 = shufflevector <4 x float> %tmp5, <4 x float> undef, <4 x i32> <i32 0, i32 1, i32 5, i32 undef>
+  %tmp7 = shufflevector <4 x float> %tmp6, <4 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 5>
+  %tmp8 = call <4 x float> @llvm.fma.v4f32(<4 x float> %tmp7, <4 x float> undef, <4 x float> undef)
+  %tmp9 = call <4 x float> @llvm.fma.v4f32(<4 x float> undef, <4 x float> undef, <4 x float> %tmp8)
+  %tmp10 = shufflevector <4 x float> %tmp9, <4 x float> undef, <16 x i32> <i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %tmp11 = bitcast <16 x float> %tmp10 to <16 x i32>
+  %tmp12 = extractelement <16 x i32> %tmp11, i32 0
+  store i32 %tmp12, i32* %p3, align 4
+  ret void
+}
+
+declare <4 x float> @llvm.fma.v4f32(<4 x float>, <4 x float>, <4 x float>) #1
diff --git a/llvm/test/CodeGen/AArch64/arm64-preserve-most.ll b/llvm/test/CodeGen/AArch64/arm64-preserve-most.ll
new file mode 100644
index 000000000..7085bf396
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/arm64-preserve-most.ll
@@ -0,0 +1,38 @@
+; RUN: llc -O0 --march=aarch64 -verify-machineinstrs --filetype=asm %s -o - 2>&1 | FileCheck %s
+; RUN: llc -O1 --march=aarch64 -verify-machineinstrs --filetype=asm %s -o - 2>&1 | FileCheck %s
+; RUN: llc -O2 --march=aarch64 -verify-machineinstrs --filetype=asm %s -o - 2>&1 | FileCheck %s
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64-unknown-linux-gnu"
+
+declare void @normal_cc()
+
+; Caller: preserve_mostcc; callee: normalcc. Normally callee saved registers
+; x9~x15 need to be spilled. Since most of them will be spilled in pairs in
+; reverse order, we only check the odd number ones due to FileCheck not
+; matching the same line of assembly twice.
+; CHECK-LABEL: preserve_most
+; CHECK-DAG: {{st[rp]}} {{(x[0-9]+, )?x9(, x[0-9]+)?}}, [sp, #{{[-0-9]+}}]
+; CHECK-DAG: {{st[rp]}} {{(x[0-9]+, )?x11(, x[0-9]+)?}}, [sp, #{{[-0-9]+}}]
+; CHECK-DAG: {{st[rp]}} {{(x[0-9]+, )?x13(, x[0-9]+)?}}, [sp, #{{[-0-9]+}}]
+; CHECK-DAG: {{st[rp]}} {{(x[0-9]+, )?x15(, x[0-9]+)?}}, [sp, #{{[-0-9]+}}]
+define preserve_mostcc void @preserve_most() {
+  call void @normal_cc()
+  ret void
+}
+
+; Caller: normalcc; callee: preserve_mostcc. x9 does not need to be spilled.
+; The same holds for x10 through x15, but we only check x9.
+; CHECK-LABEL: normal_cc_caller
+; CHECK-NOT: stp {{x[0-9]+}}, x9, [sp, #{{[-0-9]+}}]
+; CHECK-NOT: stp x9, {{x[0-9]+}}, [sp, #{{[-0-9]+}}]
+; CHECK-NOT: str x9, [sp, {{#[-0-9]+}}]
+define dso_local void @normal_cc_caller() {
+entry:
+  %v = alloca i32, align 4
+  call void asm sideeffect "mov x9, $0", "N,~{x9}"(i32 48879) #2
+  call preserve_mostcc void @preserve_most()
+  %0 = load i32, i32* %v, align 4
+  %1 = call i32 asm sideeffect "mov ${0:w}, w9", "=r,r"(i32 %0) #2
+  store i32 %1, i32* %v, align 4
+  ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/arm64-rev.ll b/llvm/test/CodeGen/AArch64/arm64-rev.ll
index a04fe0513..6b063687d 100644
--- a/llvm/test/CodeGen/AArch64/arm64-rev.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-rev.ll
@@ -8,10 +8,11 @@ define i32 @test_rev_w(i32 %a) nounwind {
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    rev w0, w0
 ; CHECK-NEXT:    ret
-; GISEL-LABEL: test_rev_w:
-; GISEL:       // %bb.0: // %entry
-; GISEL-NEXT:    rev w0, w0
-; GISEL-NEXT:    ret
+;
+; FALLBACK-LABEL: test_rev_w:
+; FALLBACK:       // %bb.0: // %entry
+; FALLBACK-NEXT:    rev w0, w0
+; FALLBACK-NEXT:    ret
 entry:
   %0 = tail call i32 @llvm.bswap.i32(i32 %a)
   ret i32 %0
@@ -23,10 +24,11 @@ define i64 @test_rev_x(i64 %a) nounwind {
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    rev x0, x0
 ; CHECK-NEXT:    ret
-; GISEL-LABEL: test_rev_x:
-; GISEL:       // %bb.0: // %entry
-; GISEL-NEXT:    rev x0, x0
-; GISEL-NEXT:    ret
+;
+; FALLBACK-LABEL: test_rev_x:
+; FALLBACK:       // %bb.0: // %entry
+; FALLBACK-NEXT:    rev x0, x0
+; FALLBACK-NEXT:    ret
 entry:
   %0 = tail call i64 @llvm.bswap.i64(i64 %a)
   ret i64 %0
@@ -37,9 +39,16 @@ entry:
 define i32 @test_rev_w_srl16(i16 %a) {
 ; CHECK-LABEL: test_rev_w_srl16:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    and w8, w0, #0xffff
-; CHECK-NEXT:    rev16 w0, w8
+; CHECK-NEXT:    rev w8, w0
+; CHECK-NEXT:    lsr w0, w8, #16
 ; CHECK-NEXT:    ret
+;
+; FALLBACK-LABEL: test_rev_w_srl16:
+; FALLBACK:       // %bb.0: // %entry
+; FALLBACK-NEXT:    and w8, w0, #0xffff
+; FALLBACK-NEXT:    rev w8, w8
+; FALLBACK-NEXT:    lsr w0, w8, #16
+; FALLBACK-NEXT:    ret
 entry:
   %0 = zext i16 %a to i32
   %1 = tail call i32 @llvm.bswap.i32(i32 %0)
@@ -51,8 +60,16 @@ define i32 @test_rev_w_srl16_load(i16 *%a) {
 ; CHECK-LABEL: test_rev_w_srl16_load:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    ldrh w8, [x0]
-; CHECK-NEXT:    rev16 w0, w8
+; CHECK-NEXT:    rev w8, w8
+; CHECK-NEXT:    lsr w0, w8, #16
 ; CHECK-NEXT:    ret
+;
+; FALLBACK-LABEL: test_rev_w_srl16_load:
+; FALLBACK:       // %bb.0: // %entry
+; FALLBACK-NEXT:    ldrh w8, [x0]
+; FALLBACK-NEXT:    rev w8, w8
+; FALLBACK-NEXT:    lsr w0, w8, #16
+; FALLBACK-NEXT:    ret
 entry:
   %0 = load i16, i16 *%a
   %1 = zext i16 %0 to i32
@@ -68,6 +85,14 @@ define i32 @test_rev_w_srl16_add(i8 %a, i8 %b) {
 ; CHECK-NEXT:    add w8, w8, w1, uxtb
 ; CHECK-NEXT:    rev16 w0, w8
 ; CHECK-NEXT:    ret
+;
+; FALLBACK-LABEL: test_rev_w_srl16_add:
+; FALLBACK:       // %bb.0: // %entry
+; FALLBACK-NEXT:    and w8, w1, #0xff
+; FALLBACK-NEXT:    add w8, w8, w0, uxtb
+; FALLBACK-NEXT:    rev w8, w8
+; FALLBACK-NEXT:    lsr w0, w8, #16
+; FALLBACK-NEXT:    ret
 entry:
   %0 = zext i8 %a to i32
   %1 = zext i8 %b to i32
@@ -82,9 +107,18 @@ entry:
 define i64 @test_rev_x_srl32(i32 %a) {
 ; CHECK-LABEL: test_rev_x_srl32:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    mov w8, w0
-; CHECK-NEXT:    rev32 x0, x8
+; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT:    rev x8, x0
+; CHECK-NEXT:    lsr x0, x8, #32
 ; CHECK-NEXT:    ret
+;
+; FALLBACK-LABEL: test_rev_x_srl32:
+; FALLBACK:       // %bb.0: // %entry
+; FALLBACK-NEXT:    // kill: def $w0 killed $w0 def $x0
+; FALLBACK-NEXT:    ubfx x8, x0, #0, #32
+; FALLBACK-NEXT:    rev x8, x8
+; FALLBACK-NEXT:    lsr x0, x8, #32
+; FALLBACK-NEXT:    ret
 entry:
   %0 = zext i32 %a to i64
   %1 = tail call i64 @llvm.bswap.i64(i64 %0)
@@ -96,8 +130,16 @@ define i64 @test_rev_x_srl32_load(i32 *%a) {
 ; CHECK-LABEL: test_rev_x_srl32_load:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    ldr w8, [x0]
-; CHECK-NEXT:    rev32 x0, x8
+; CHECK-NEXT:    rev x8, x8
+; CHECK-NEXT:    lsr x0, x8, #32
 ; CHECK-NEXT:    ret
+;
+; FALLBACK-LABEL: test_rev_x_srl32_load:
+; FALLBACK:       // %bb.0: // %entry
+; FALLBACK-NEXT:    ldr w8, [x0]
+; FALLBACK-NEXT:    rev x8, x8
+; FALLBACK-NEXT:    lsr x0, x8, #32
+; FALLBACK-NEXT:    ret
 entry:
   %0 = load i32, i32 *%a
   %1 = zext i32 %0 to i64
@@ -112,6 +154,14 @@ define i64 @test_rev_x_srl32_shift(i64 %a) {
 ; CHECK-NEXT:    ubfx x8, x0, #2, #29
 ; CHECK-NEXT:    rev32 x0, x8
 ; CHECK-NEXT:    ret
+;
+; FALLBACK-LABEL: test_rev_x_srl32_shift:
+; FALLBACK:       // %bb.0: // %entry
+; FALLBACK-NEXT:    lsl x8, x0, #33
+; FALLBACK-NEXT:    lsr x8, x8, #35
+; FALLBACK-NEXT:    rev x8, x8
+; FALLBACK-NEXT:    lsr x0, x8, #32
+; FALLBACK-NEXT:    ret
 entry:
   %0 = shl i64 %a, 33
   %1 = lshr i64 %0, 35
@@ -128,6 +178,19 @@ define i32 @test_rev16_w(i32 %X) nounwind {
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    rev16 w0, w0
 ; CHECK-NEXT:    ret
+;
+; FALLBACK-LABEL: test_rev16_w:
+; FALLBACK:       // %bb.0: // %entry
+; FALLBACK-NEXT:    lsr w8, w0, #8
+; FALLBACK-NEXT:    lsl w9, w0, #8
+; FALLBACK-NEXT:    and w10, w8, #0xff0000
+; FALLBACK-NEXT:    and w11, w9, #0xff000000
+; FALLBACK-NEXT:    and w9, w9, #0xff00
+; FALLBACK-NEXT:    orr w10, w11, w10
+; FALLBACK-NEXT:    and w8, w8, #0xff
+; FALLBACK-NEXT:    orr w9, w10, w9
+; FALLBACK-NEXT:    orr w0, w9, w8
+; FALLBACK-NEXT:    ret
 entry:
   %tmp1 = lshr i32 %X, 8
   %X15 = bitcast i32 %X to i32
@@ -151,6 +214,13 @@ define i64 @test_rev16_x(i64 %a) nounwind {
 ; CHECK-NEXT:    rev x8, x0
 ; CHECK-NEXT:    ror x0, x8, #16
 ; CHECK-NEXT:    ret
+;
+; FALLBACK-LABEL: test_rev16_x:
+; FALLBACK:       // %bb.0: // %entry
+; FALLBACK-NEXT:    rev x8, x0
+; FALLBACK-NEXT:    lsl x9, x8, #48
+; FALLBACK-NEXT:    orr x0, x9, x8, lsr #16
+; FALLBACK-NEXT:    ret
 entry:
   %0 = tail call i64 @llvm.bswap.i64(i64 %a)
   %1 = lshr i64 %0, 16
@@ -164,6 +234,13 @@ define i64 @test_rev32_x(i64 %a) nounwind {
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    rev32 x0, x0
 ; CHECK-NEXT:    ret
+;
+; FALLBACK-LABEL: test_rev32_x:
+; FALLBACK:       // %bb.0: // %entry
+; FALLBACK-NEXT:    rev x8, x0
+; FALLBACK-NEXT:    lsl x9, x8, #32
+; FALLBACK-NEXT:    orr x0, x9, x8, lsr #32
+; FALLBACK-NEXT:    ret
 entry:
   %0 = tail call i64 @llvm.bswap.i64(i64 %a)
   %1 = lshr i64 %0, 32
@@ -178,6 +255,12 @@ define <8 x i8> @test_vrev64D8(<8 x i8>* %A) nounwind {
 ; CHECK-NEXT:    ldr d0, [x0]
 ; CHECK-NEXT:    rev64.8b v0, v0
 ; CHECK-NEXT:    ret
+;
+; FALLBACK-LABEL: test_vrev64D8:
+; FALLBACK:       // %bb.0:
+; FALLBACK-NEXT:    ldr d0, [x0]
+; FALLBACK-NEXT:    rev64.8b v0, v0
+; FALLBACK-NEXT:    ret
 	%tmp1 = load <8 x i8>, <8 x i8>* %A
 	%tmp2 = shufflevector <8 x i8> %tmp1, <8 x i8> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 	ret <8 x i8> %tmp2
@@ -189,6 +272,12 @@ define <4 x i16> @test_vrev64D16(<4 x i16>* %A) nounwind {
 ; CHECK-NEXT:    ldr d0, [x0]
 ; CHECK-NEXT:    rev64.4h v0, v0
 ; CHECK-NEXT:    ret
+;
+; FALLBACK-LABEL: test_vrev64D16:
+; FALLBACK:       // %bb.0:
+; FALLBACK-NEXT:    ldr d0, [x0]
+; FALLBACK-NEXT:    rev64.4h v0, v0
+; FALLBACK-NEXT:    ret
 	%tmp1 = load <4 x i16>, <4 x i16>* %A
 	%tmp2 = shufflevector <4 x i16> %tmp1, <4 x i16> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
 	ret <4 x i16> %tmp2
@@ -200,6 +289,16 @@ define <2 x i32> @test_vrev64D32(<2 x i32>* %A) nounwind {
 ; CHECK-NEXT:    ldr d0, [x0]
 ; CHECK-NEXT:    rev64.2s v0, v0
 ; CHECK-NEXT:    ret
+;
+; FALLBACK-LABEL: test_vrev64D32:
+; FALLBACK:       // %bb.0:
+; FALLBACK-NEXT:    ldr d0, [x0]
+; FALLBACK-NEXT:    adrp x8, .LCPI13_0
+; FALLBACK-NEXT:    ldr d1, [x8, :lo12:.LCPI13_0]
+; FALLBACK-NEXT:    mov.d v0[1], v0[0]
+; FALLBACK-NEXT:    tbl.16b v0, { v0 }, v1
+; FALLBACK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; FALLBACK-NEXT:    ret
 	%tmp1 = load <2 x i32>, <2 x i32>* %A
 	%tmp2 = shufflevector <2 x i32> %tmp1, <2 x i32> undef, <2 x i32> <i32 1, i32 0>
 	ret <2 x i32> %tmp2
@@ -211,6 +310,16 @@ define <2 x float> @test_vrev64Df(<2 x float>* %A) nounwind {
 ; CHECK-NEXT:    ldr d0, [x0]
 ; CHECK-NEXT:    rev64.2s v0, v0
 ; CHECK-NEXT:    ret
+;
+; FALLBACK-LABEL: test_vrev64Df:
+; FALLBACK:       // %bb.0:
+; FALLBACK-NEXT:    ldr d0, [x0]
+; FALLBACK-NEXT:    adrp x8, .LCPI14_0
+; FALLBACK-NEXT:    ldr d1, [x8, :lo12:.LCPI14_0]
+; FALLBACK-NEXT:    mov.d v0[1], v0[0]
+; FALLBACK-NEXT:    tbl.16b v0, { v0 }, v1
+; FALLBACK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; FALLBACK-NEXT:    ret
 	%tmp1 = load <2 x float>, <2 x float>* %A
 	%tmp2 = shufflevector <2 x float> %tmp1, <2 x float> undef, <2 x i32> <i32 1, i32 0>
 	ret <2 x float> %tmp2
@@ -222,6 +331,12 @@ define <16 x i8> @test_vrev64Q8(<16 x i8>* %A) nounwind {
 ; CHECK-NEXT:    ldr q0, [x0]
 ; CHECK-NEXT:    rev64.16b v0, v0
 ; CHECK-NEXT:    ret
+;
+; FALLBACK-LABEL: test_vrev64Q8:
+; FALLBACK:       // %bb.0:
+; FALLBACK-NEXT:    ldr q0, [x0]
+; FALLBACK-NEXT:    rev64.16b v0, v0
+; FALLBACK-NEXT:    ret
 	%tmp1 = load <16 x i8>, <16 x i8>* %A
 	%tmp2 = shufflevector <16 x i8> %tmp1, <16 x i8> undef, <16 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8>
 	ret <16 x i8> %tmp2
@@ -233,6 +348,12 @@ define <8 x i16> @test_vrev64Q16(<8 x i16>* %A) nounwind {
 ; CHECK-NEXT:    ldr q0, [x0]
 ; CHECK-NEXT:    rev64.8h v0, v0
 ; CHECK-NEXT:    ret
+;
+; FALLBACK-LABEL: test_vrev64Q16:
+; FALLBACK:       // %bb.0:
+; FALLBACK-NEXT:    ldr q0, [x0]
+; FALLBACK-NEXT:    rev64.8h v0, v0
+; FALLBACK-NEXT:    ret
 	%tmp1 = load <8 x i16>, <8 x i16>* %A
 	%tmp2 = shufflevector <8 x i16> %tmp1, <8 x i16> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
 	ret <8 x i16> %tmp2
@@ -244,6 +365,14 @@ define <4 x i32> @test_vrev64Q32(<4 x i32>* %A) nounwind {
 ; CHECK-NEXT:    ldr q0, [x0]
 ; CHECK-NEXT:    rev64.4s v0, v0
 ; CHECK-NEXT:    ret
+;
+; FALLBACK-LABEL: test_vrev64Q32:
+; FALLBACK:       // %bb.0:
+; FALLBACK-NEXT:    adrp x8, .LCPI17_0
+; FALLBACK-NEXT:    ldr q0, [x0]
+; FALLBACK-NEXT:    ldr q2, [x8, :lo12:.LCPI17_0]
+; FALLBACK-NEXT:    tbl.16b v0, { v0, v1 }, v2
+; FALLBACK-NEXT:    ret
 	%tmp1 = load <4 x i32>, <4 x i32>* %A
 	%tmp2 = shufflevector <4 x i32> %tmp1, <4 x i32> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
 	ret <4 x i32> %tmp2
@@ -255,6 +384,14 @@ define <4 x float> @test_vrev64Qf(<4 x float>* %A) nounwind {
 ; CHECK-NEXT:    ldr q0, [x0]
 ; CHECK-NEXT:    rev64.4s v0, v0
 ; CHECK-NEXT:    ret
+;
+; FALLBACK-LABEL: test_vrev64Qf:
+; FALLBACK:       // %bb.0:
+; FALLBACK-NEXT:    adrp x8, .LCPI18_0
+; FALLBACK-NEXT:    ldr q0, [x0]
+; FALLBACK-NEXT:    ldr q2, [x8, :lo12:.LCPI18_0]
+; FALLBACK-NEXT:    tbl.16b v0, { v0, v1 }, v2
+; FALLBACK-NEXT:    ret
 	%tmp1 = load <4 x float>, <4 x float>* %A
 	%tmp2 = shufflevector <4 x float> %tmp1, <4 x float> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
 	ret <4 x float> %tmp2
@@ -266,6 +403,12 @@ define <8 x i8> @test_vrev32D8(<8 x i8>* %A) nounwind {
 ; CHECK-NEXT:    ldr d0, [x0]
 ; CHECK-NEXT:    rev32.8b v0, v0
 ; CHECK-NEXT:    ret
+;
+; FALLBACK-LABEL: test_vrev32D8:
+; FALLBACK:       // %bb.0:
+; FALLBACK-NEXT:    ldr d0, [x0]
+; FALLBACK-NEXT:    rev32.8b v0, v0
+; FALLBACK-NEXT:    ret
 	%tmp1 = load <8 x i8>, <8 x i8>* %A
 	%tmp2 = shufflevector <8 x i8> %tmp1, <8 x i8> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
 	ret <8 x i8> %tmp2
@@ -277,6 +420,12 @@ define <4 x i16> @test_vrev32D16(<4 x i16>* %A) nounwind {
 ; CHECK-NEXT:    ldr d0, [x0]
 ; CHECK-NEXT:    rev32.4h v0, v0
 ; CHECK-NEXT:    ret
+;
+; FALLBACK-LABEL: test_vrev32D16:
+; FALLBACK:       // %bb.0:
+; FALLBACK-NEXT:    ldr d0, [x0]
+; FALLBACK-NEXT:    rev32.4h v0, v0
+; FALLBACK-NEXT:    ret
 	%tmp1 = load <4 x i16>, <4 x i16>* %A
 	%tmp2 = shufflevector <4 x i16> %tmp1, <4 x i16> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
 	ret <4 x i16> %tmp2
@@ -288,6 +437,12 @@ define <16 x i8> @test_vrev32Q8(<16 x i8>* %A) nounwind {
 ; CHECK-NEXT:    ldr q0, [x0]
 ; CHECK-NEXT:    rev32.16b v0, v0
 ; CHECK-NEXT:    ret
+;
+; FALLBACK-LABEL: test_vrev32Q8:
+; FALLBACK:       // %bb.0:
+; FALLBACK-NEXT:    ldr q0, [x0]
+; FALLBACK-NEXT:    rev32.16b v0, v0
+; FALLBACK-NEXT:    ret
 	%tmp1 = load <16 x i8>, <16 x i8>* %A
 	%tmp2 = shufflevector <16 x i8> %tmp1, <16 x i8> undef, <16 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 11, i32 10, i32 9, i32 8, i32 15, i32 14, i32 13, i32 12>
 	ret <16 x i8> %tmp2
@@ -299,6 +454,12 @@ define <8 x i16> @test_vrev32Q16(<8 x i16>* %A) nounwind {
 ; CHECK-NEXT:    ldr q0, [x0]
 ; CHECK-NEXT:    rev32.8h v0, v0
 ; CHECK-NEXT:    ret
+;
+; FALLBACK-LABEL: test_vrev32Q16:
+; FALLBACK:       // %bb.0:
+; FALLBACK-NEXT:    ldr q0, [x0]
+; FALLBACK-NEXT:    rev32.8h v0, v0
+; FALLBACK-NEXT:    ret
 	%tmp1 = load <8 x i16>, <8 x i16>* %A
 	%tmp2 = shufflevector <8 x i16> %tmp1, <8 x i16> undef, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
 	ret <8 x i16> %tmp2
@@ -310,6 +471,12 @@ define <8 x i8> @test_vrev16D8(<8 x i8>* %A) nounwind {
 ; CHECK-NEXT:    ldr d0, [x0]
 ; CHECK-NEXT:    rev16.8b v0, v0
 ; CHECK-NEXT:    ret
+;
+; FALLBACK-LABEL: test_vrev16D8:
+; FALLBACK:       // %bb.0:
+; FALLBACK-NEXT:    ldr d0, [x0]
+; FALLBACK-NEXT:    rev16.8b v0, v0
+; FALLBACK-NEXT:    ret
 	%tmp1 = load <8 x i8>, <8 x i8>* %A
 	%tmp2 = shufflevector <8 x i8> %tmp1, <8 x i8> undef, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
 	ret <8 x i8> %tmp2
@@ -321,6 +488,12 @@ define <16 x i8> @test_vrev16Q8(<16 x i8>* %A) nounwind {
 ; CHECK-NEXT:    ldr q0, [x0]
 ; CHECK-NEXT:    rev16.16b v0, v0
 ; CHECK-NEXT:    ret
+;
+; FALLBACK-LABEL: test_vrev16Q8:
+; FALLBACK:       // %bb.0:
+; FALLBACK-NEXT:    ldr q0, [x0]
+; FALLBACK-NEXT:    rev16.16b v0, v0
+; FALLBACK-NEXT:    ret
 	%tmp1 = load <16 x i8>, <16 x i8>* %A
 	%tmp2 = shufflevector <16 x i8> %tmp1, <16 x i8> undef, <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14>
 	ret <16 x i8> %tmp2
@@ -334,6 +507,12 @@ define <8 x i8> @test_vrev64D8_undef(<8 x i8>* %A) nounwind {
 ; CHECK-NEXT:    ldr d0, [x0]
 ; CHECK-NEXT:    rev64.8b v0, v0
 ; CHECK-NEXT:    ret
+;
+; FALLBACK-LABEL: test_vrev64D8_undef:
+; FALLBACK:       // %bb.0:
+; FALLBACK-NEXT:    ldr d0, [x0]
+; FALLBACK-NEXT:    rev64.8b v0, v0
+; FALLBACK-NEXT:    ret
 	%tmp1 = load <8 x i8>, <8 x i8>* %A
 	%tmp2 = shufflevector <8 x i8> %tmp1, <8 x i8> undef, <8 x i32> <i32 7, i32 undef, i32 undef, i32 4, i32 3, i32 2, i32 1, i32 0>
 	ret <8 x i8> %tmp2
@@ -345,6 +524,12 @@ define <8 x i16> @test_vrev32Q16_undef(<8 x i16>* %A) nounwind {
 ; CHECK-NEXT:    ldr q0, [x0]
 ; CHECK-NEXT:    rev32.8h v0, v0
 ; CHECK-NEXT:    ret
+;
+; FALLBACK-LABEL: test_vrev32Q16_undef:
+; FALLBACK:       // %bb.0:
+; FALLBACK-NEXT:    ldr q0, [x0]
+; FALLBACK-NEXT:    rev32.8h v0, v0
+; FALLBACK-NEXT:    ret
 	%tmp1 = load <8 x i16>, <8 x i16>* %A
 	%tmp2 = shufflevector <8 x i16> %tmp1, <8 x i16> undef, <8 x i32> <i32 undef, i32 0, i32 undef, i32 2, i32 5, i32 4, i32 7, i32 undef>
 	ret <8 x i16> %tmp2
@@ -359,6 +544,14 @@ define void @test_vrev64(<4 x i16>* nocapture %source, <2 x i16>* nocapture %dst
 ; CHECK-NEXT:    st1.h { v0 }[5], [x8]
 ; CHECK-NEXT:    st1.h { v0 }[6], [x1]
 ; CHECK-NEXT:    ret
+;
+; FALLBACK-LABEL: test_vrev64:
+; FALLBACK:       // %bb.0: // %entry
+; FALLBACK-NEXT:    ldr q0, [x0]
+; FALLBACK-NEXT:    add x8, x1, #2 // =2
+; FALLBACK-NEXT:    st1.h { v0 }[5], [x8]
+; FALLBACK-NEXT:    st1.h { v0 }[6], [x1]
+; FALLBACK-NEXT:    ret
 entry:
   %0 = bitcast <4 x i16>* %source to <8 x i16>*
   %tmp2 = load <8 x i16>, <8 x i16>* %0, align 4
@@ -381,6 +574,19 @@ define void @float_vrev64(float* nocapture %source, <4 x float>* nocapture %dest
 ; CHECK-NEXT:    rev64.4s v0, v0
 ; CHECK-NEXT:    str q0, [x1, #176]
 ; CHECK-NEXT:    ret
+;
+; FALLBACK-LABEL: float_vrev64:
+; FALLBACK:       // %bb.0: // %entry
+; FALLBACK-NEXT:    fmov s0, wzr
+; FALLBACK-NEXT:    mov.s v0[1], v0[0]
+; FALLBACK-NEXT:    mov.s v0[2], v0[0]
+; FALLBACK-NEXT:    adrp x8, .LCPI28_0
+; FALLBACK-NEXT:    mov.s v0[3], v0[0]
+; FALLBACK-NEXT:    ldr q1, [x0]
+; FALLBACK-NEXT:    ldr q2, [x8, :lo12:.LCPI28_0]
+; FALLBACK-NEXT:    tbl.16b v0, { v0, v1 }, v2
+; FALLBACK-NEXT:    str q0, [x1, #176]
+; FALLBACK-NEXT:    ret
 entry:
   %0 = bitcast float* %source to <4 x float>*
   %tmp2 = load <4 x float>, <4 x float>* %0, align 4
@@ -396,10 +602,11 @@ define <4 x i32> @test_vrev32_bswap(<4 x i32> %source) nounwind {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    rev32.16b v0, v0
 ; CHECK-NEXT:    ret
-; GISEL-LABEL: test_vrev32_bswap:
-; GISEL:       // %bb.0:
-; GISEL-NEXT:    rev32.16b v0, v0
-; GISEL-NEXT:    ret
+;
+; FALLBACK-LABEL: test_vrev32_bswap:
+; FALLBACK:       // %bb.0:
+; FALLBACK-NEXT:    rev32.16b v0, v0
+; FALLBACK-NEXT:    ret
   %bswap = call <4 x i32> @llvm.bswap.v4i32(<4 x i32> %source)
   ret <4 x i32> %bswap
 }
diff --git a/llvm/test/CodeGen/AArch64/arm64-shrink-wrapping.ll b/llvm/test/CodeGen/AArch64/arm64-shrink-wrapping.ll
index e511ea155..d1089b304 100644
--- a/llvm/test/CodeGen/AArch64/arm64-shrink-wrapping.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-shrink-wrapping.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc %s -o - -mtriple=arm64-apple-ios -enable-shrink-wrap=true -disable-post-ra -frame-pointer=all | FileCheck %s --check-prefix=ENABLE
-; RUN: llc %s -o - -enable-shrink-wrap=false -disable-post-ra -frame-pointer=all | FileCheck %s --check-prefix=DISABLE
+; RUN: llc %s -o - -mtriple=arm64-apple-ios -enable-shrink-wrap=true -disable-post-ra -frame-pointer=non-leaf | FileCheck %s --check-prefix=ENABLE
+; RUN: llc %s -o - -enable-shrink-wrap=false -disable-post-ra -frame-pointer=non-leaf | FileCheck %s --check-prefix=DISABLE
 target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
 target triple = "arm64-apple-ios"
 
diff --git a/llvm/test/CodeGen/AArch64/arm64-st1.ll b/llvm/test/CodeGen/AArch64/arm64-st1.ll
index af234a9ac..28ddd94a7 100644
--- a/llvm/test/CodeGen/AArch64/arm64-st1.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-st1.ll
@@ -1,6 +1,6 @@
 ; RUN: llc < %s -mtriple=arm64-eabi -aarch64-neon-syntax=apple -verify-machineinstrs | FileCheck %s
-; RUN: llc < %s -mtriple=arm64-eabi -aarch64-neon-syntax=apple -verify-machineinstrs -mcpu=exynos-m1 | FileCheck --check-prefix=EXYNOS %s
-; The instruction latencies of Exynos-M1 trigger the transform we see under the Exynos check.
+; The instruction latencies of Exynos-M3 trigger the transform we see under the Exynos check.
+; RUN: llc < %s -mtriple=arm64-eabi -aarch64-neon-syntax=apple -verify-machineinstrs -mcpu=exynos-m3 | FileCheck --check-prefix=EXYNOS %s
 
 define void @st1lane_16b(<16 x i8> %A, i8* %D) {
 ; CHECK-LABEL: st1lane_16b
diff --git a/llvm/test/CodeGen/AArch64/arm64-stacksave.ll b/llvm/test/CodeGen/AArch64/arm64-stacksave.ll
index a79e99ba3..13d4ae23d 100644
--- a/llvm/test/CodeGen/AArch64/arm64-stacksave.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-stacksave.ll
@@ -1,6 +1,6 @@
-; RUN: llc < %s -verify-coalescing
+; RUN: llc -mtriple=arm64-apple-macosx10.8.0 < %s -verify-coalescing
+; RUN: llc -mtriple=arm64_32-apple-ios9.0 < %s -verify-coalescing
 ; <rdar://problem/11522048>
-target triple = "arm64-apple-macosx10.8.0"
 
 ; Verify that we can handle spilling the stack pointer without attempting
 ; spilling it directly.
diff --git a/llvm/test/CodeGen/AArch64/arm64-storebytesmerge.ll b/llvm/test/CodeGen/AArch64/arm64-storebytesmerge.ll
index fb0613124..b3efd9ece 100644
--- a/llvm/test/CodeGen/AArch64/arm64-storebytesmerge.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-storebytesmerge.ll
@@ -41,6 +41,6 @@ for.end705.i:                                     ; preds = %for.body453.i
 
 declare void @f() local_unnamed_addr #1
 
-attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="true" "no-jump-tables"="false" "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="true" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="cortex-a57" "target-features"="+crc,+crypto,+fp-armv8,+neon" "unsafe-fp-math"="true" "use-soft-float"="false" }
-attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="true" "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="true" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="cortex-a57" "target-features"="+crc,+crypto,+fp-armv8,+neon" "unsafe-fp-math"="true" "use-soft-float"="false" }
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="true" "no-jump-tables"="false" "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="true" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="cortex-a57" "target-features"="+crc,+crypto,+fp-armv8,+neon" "unsafe-fp-math"="true" "use-soft-float"="false" }
+attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="true" "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="true" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="cortex-a57" "target-features"="+crc,+crypto,+fp-armv8,+neon" "unsafe-fp-math"="true" "use-soft-float"="false" }
 attributes #2 = { nounwind }
diff --git a/llvm/test/CodeGen/AArch64/arm64-tls-darwin.ll b/llvm/test/CodeGen/AArch64/arm64-tls-darwin.ll
index fa4e833d4..b90848bc9 100644
--- a/llvm/test/CodeGen/AArch64/arm64-tls-darwin.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-tls-darwin.ll
@@ -1,4 +1,5 @@
 ; RUN: llc -mtriple=arm64-apple-ios7.0 %s -o - | FileCheck %s
+; RUN: llc -mtriple=arm64-apple-ios7.0 -global-isel -global-isel-abort=1 -verify-machineinstrs %s -o - | FileCheck %s
 
 @var = thread_local global i8 0
 
diff --git a/llvm/test/CodeGen/AArch64/arm64-tls-initial-exec.ll b/llvm/test/CodeGen/AArch64/arm64-tls-initial-exec.ll
new file mode 100644
index 000000000..4f1696789
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/arm64-tls-initial-exec.ll
@@ -0,0 +1,51 @@
+; RUN: llc -mtriple=arm64-none-linux-gnu -verify-machineinstrs -show-mc-encoding < %s | FileCheck %s
+; RUN: llc -mtriple=arm64-none-linux-gnu -filetype=obj < %s | llvm-objdump -r - | FileCheck --check-prefix=CHECK-RELOC %s
+; RUN: llc -mtriple=arm64-none-linux-gnu -verify-machineinstrs -show-mc-encoding -code-model=tiny < %s | FileCheck %s --check-prefix=CHECK-TINY
+; RUN: llc -mtriple=arm64-none-linux-gnu -filetype=obj < %s -code-model=tiny | llvm-objdump -r - | FileCheck --check-prefix=CHECK-TINY-RELOC %s
+; FIXME: We currently error for the large code model
+; RUN: not llc -mtriple=arm64-none-linux-gnu -verify-machineinstrs -show-mc-encoding -code-model=large < %s 2>&1 | FileCheck %s --check-prefix=CHECK-LARGE
+
+; CHECK-LARGE: ELF TLS only supported in small memory model
+
+@initial_exec_var = external thread_local(initialexec) global i32
+
+define i32 @test_initial_exec() {
+; CHECK-LABEL: test_initial_exec:
+  %val = load i32, i32* @initial_exec_var
+
+; CHECK: adrp x[[GOTADDR:[0-9]+]], :gottprel:initial_exec_var
+; CHECK: ldr x[[TP_OFFSET:[0-9]+]], [x[[GOTADDR]], :gottprel_lo12:initial_exec_var]
+; CHECK: mrs x[[TP:[0-9]+]], TPIDR_EL0
+; CHECK: ldr w0, [x[[TP]], x[[TP_OFFSET]]]
+
+; CHECK-RELOC: R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21
+; CHECK-RELOC: R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC
+
+; CHECK-TINY: ldr x[[TP_OFFSET:[0-9]+]], :gottprel:initial_exec_var
+; CHECK-TINY: mrs x[[TP:[0-9]+]], TPIDR_EL0
+; CHECK-TINY: ldr w0, [x[[TP]], x[[TP_OFFSET]]]
+
+; CHECK-TINY-RELOC: R_AARCH64_TLSIE_LD_GOTTPREL_PREL19
+
+  ret i32 %val
+}
+
+define i32* @test_initial_exec_addr() {
+; CHECK-LABEL: test_initial_exec_addr:
+  ret i32* @initial_exec_var
+
+; CHECK: adrp x[[GOTADDR:[0-9]+]], :gottprel:initial_exec_var
+; CHECK: ldr [[TP_OFFSET:x[0-9]+]], [x[[GOTADDR]], :gottprel_lo12:initial_exec_var]
+; CHECK: mrs [[TP:x[0-9]+]], TPIDR_EL0
+; CHECK: add x0, [[TP]], [[TP_OFFSET]]
+
+; CHECK-RELOC: R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21
+; CHECK-RELOC: R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC
+
+; CHECK-TINY: ldr x[[TP_OFFSET:[0-9]+]], :gottprel:initial_exec_var
+; CHECK-TINY: mrs [[TP:x[0-9]+]], TPIDR_EL0
+; CHECK-TINY: add x0, [[TP]], x[[TP_OFFSET]]
+
+; CHECK-TINY-RELOC: R_AARCH64_TLSIE_LD_GOTTPREL_PREL19
+
+}
diff --git a/llvm/test/CodeGen/AArch64/arm64-tls-local-exec.ll b/llvm/test/CodeGen/AArch64/arm64-tls-local-exec.ll
new file mode 100644
index 000000000..101edf53a
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/arm64-tls-local-exec.ll
@@ -0,0 +1,106 @@
+; Test each TLS size option
+; RUN: llc -mtriple=arm64-none-linux-gnu -verify-machineinstrs -show-mc-encoding -tls-size=12 < %s | FileCheck %s --check-prefix=CHECK-12
+; RUN: llc -mtriple=arm64-none-linux-gnu -filetype=obj < %s -tls-size=12 | llvm-objdump -r - | FileCheck --check-prefix=CHECK-12-RELOC %s
+; RUN: llc -mtriple=arm64-none-linux-gnu -verify-machineinstrs -show-mc-encoding -code-model=tiny -tls-size=24 < %s | FileCheck %s --check-prefix=CHECK-24
+; RUN: llc -mtriple=arm64-none-linux-gnu -filetype=obj < %s -code-model=tiny -tls-size=24 | llvm-objdump -r - | FileCheck --check-prefix=CHECK-24-RELOC %s
+; RUN: llc -mtriple=arm64-none-linux-gnu -verify-machineinstrs -show-mc-encoding -code-model=small -tls-size=32 < %s | FileCheck %s --check-prefix=CHECK-32
+; RUN: llc -mtriple=arm64-none-linux-gnu -filetype=obj < %s -code-model=small -tls-size=32 | llvm-objdump -r - | FileCheck --check-prefix=CHECK-32-RELOC %s
+; RUN: llc -mtriple=arm64-none-linux-gnu -verify-machineinstrs -show-mc-encoding -code-model=large -tls-size=48 < %s | FileCheck %s --check-prefix=CHECK-48
+; RUN: llc -mtriple=arm64-none-linux-gnu -filetype=obj < %s -code-model=large -tls-size=48 | llvm-objdump -r - | FileCheck --check-prefix=CHECK-48-RELOC %s
+;
+; Test the maximum TLS size for each code model (fallback to a smaller size from the specified size)
+; RUN: llc -mtriple=arm64-none-linux-gnu -verify-machineinstrs -show-mc-encoding -tls-size=32 < %s | FileCheck %s --check-prefix=CHECK-32
+; RUN: llc -mtriple=arm64-none-linux-gnu -filetype=obj < %s -tls-size=32 | llvm-objdump -r - | FileCheck --check-prefix=CHECK-32-RELOC %s
+; RUN: llc -mtriple=arm64-none-linux-gnu -verify-machineinstrs -show-mc-encoding -code-model=tiny -tls-size=32 < %s | FileCheck %s --check-prefix=CHECK-24
+; RUN: llc -mtriple=arm64-none-linux-gnu -filetype=obj < %s -code-model=tiny -tls-size=32 | llvm-objdump -r - | FileCheck --check-prefix=CHECK-24-RELOC %s
+; RUN: llc -mtriple=arm64-none-linux-gnu -verify-machineinstrs -show-mc-encoding -code-model=small -tls-size=48 < %s | FileCheck %s --check-prefix=CHECK-32
+; RUN: llc -mtriple=arm64-none-linux-gnu -filetype=obj < %s -code-model=small -tls-size=48 | llvm-objdump -r - | FileCheck --check-prefix=CHECK-32-RELOC %s
+;
+; Test the default TLS size for each code model
+; RUN: llc -mtriple=arm64-none-linux-gnu -verify-machineinstrs -show-mc-encoding < %s | FileCheck --check-prefix=CHECK-24 %s
+; RUN: llc -mtriple=arm64-none-linux-gnu -filetype=obj < %s | llvm-objdump -r - | FileCheck --check-prefix=CHECK-24-RELOC %s
+; RUN: llc -mtriple=arm64-none-linux-gnu -verify-machineinstrs -show-mc-encoding -code-model=tiny < %s | FileCheck %s --check-prefix=CHECK-24
+; RUN: llc -mtriple=arm64-none-linux-gnu -filetype=obj < %s -code-model=tiny | llvm-objdump -r - | FileCheck --check-prefix=CHECK-24-RELOC %s
+; RUN: llc -mtriple=arm64-none-linux-gnu -verify-machineinstrs -show-mc-encoding -code-model=small < %s | FileCheck %s --check-prefix=CHECK-24
+; RUN: llc -mtriple=arm64-none-linux-gnu -filetype=obj < %s -code-model=small | llvm-objdump -r - | FileCheck --check-prefix=CHECK-24-RELOC %s
+; RUN: llc -mtriple=arm64-none-linux-gnu -verify-machineinstrs -show-mc-encoding -code-model=large < %s | FileCheck %s --check-prefix=CHECK-24
+; RUN: llc -mtriple=arm64-none-linux-gnu -filetype=obj < %s -code-model=large | llvm-objdump -r - | FileCheck --check-prefix=CHECK-24-RELOC %s
+
+@local_exec_var = thread_local(localexec) global i32 0
+
+define i32 @test_local_exec() {
+; CHECK-LABEL: test_local_exec:
+  %val = load i32, i32* @local_exec_var
+
+; CHECK-12: mrs x[[R1:[0-9]+]], TPIDR_EL0
+; CHECK-12: add x[[R2:[0-9]+]], x[[R1]], :tprel_lo12:local_exec_var
+; CHECK-12: ldr w0, [x[[R2]]]
+
+; CHECK-12-RELOC: R_AARCH64_TLSLE_ADD_TPREL_LO12
+
+; CHECK-24: mrs x[[R1:[0-9]+]], TPIDR_EL0
+; CHECK-24: add x[[R2:[0-9]+]], x[[R1]], :tprel_hi12:local_exec_var
+; CHECK-24: add x[[R3:[0-9]+]], x[[R2]], :tprel_lo12_nc:local_exec_var
+; CHECK-24: ldr w0, [x[[R3]]]
+
+; CHECK-24-RELOC: R_AARCH64_TLSLE_ADD_TPREL_HI12
+; CHECK-24-RELOC: R_AARCH64_TLSLE_ADD_TPREL_LO12_NC
+
+; CHECK-32: movz x[[R2:[0-9]+]], #:tprel_g1:local_exec_var
+; CHECK-32: movk x[[R2]], #:tprel_g0_nc:local_exec_var
+; CHECK-32: mrs x[[R1:[0-9]+]], TPIDR_EL0
+; CHECK-32: ldr w0, [x[[R1]], x[[R2]]]
+
+; CHECK-32-RELOC: R_AARCH64_TLSLE_MOVW_TPREL_G1
+; CHECK-32-RELOC: R_AARCH64_TLSLE_MOVW_TPREL_G0_NC
+
+; CHECK-48: movz x[[R2:[0-9]+]], #:tprel_g2:local_exec_var
+; CHECK-48: movk x[[R2]], #:tprel_g1_nc:local_exec_var
+; CHECK-48: movk x[[R2]], #:tprel_g0_nc:local_exec_var
+; CHECK-48: mrs x[[R1:[0-9]+]], TPIDR_EL0
+; CHECK-48: ldr w0, [x[[R1]], x[[R2]]]
+
+; CHECK-48-RELOC: R_AARCH64_TLSLE_MOVW_TPREL_G2
+; CHECK-48-RELOC: R_AARCH64_TLSLE_MOVW_TPREL_G1_NC
+; CHECK-48-RELOC: R_AARCH64_TLSLE_MOVW_TPREL_G0_NC
+  ret i32 %val
+}
+
+define i32* @test_local_exec_addr() {
+; CHECK-LABEL: test_local_exec_addr:
+  ret i32* @local_exec_var
+
+; CHECK-12: mrs x[[R1:[0-9]+]], TPIDR_EL0
+; CHECK-12: add x0, x[[R1]], :tprel_lo12:local_exec_var
+; CHECK-12: ret
+
+; CHECK-12-RELOC: R_AARCH64_TLSLE_ADD_TPREL_LO12
+
+; CHECK-24: mrs x[[R1:[0-9]+]], TPIDR_EL0
+; CHECK-24: add x[[R2:[0-9]+]], x[[R1]], :tprel_hi12:local_exec_var
+; CHECK-24: add x0, x[[R2]], :tprel_lo12_nc:local_exec_var
+; CHECK-24: ret
+
+; CHECK-24-RELOC: R_AARCH64_TLSLE_ADD_TPREL_HI12
+; CHECK-24-RELOC: R_AARCH64_TLSLE_ADD_TPREL_LO12_NC
+
+; CHECK-32: movz x[[R2:[0-9]+]], #:tprel_g1:local_exec_var
+; CHECK-32: movk x[[R2]], #:tprel_g0_nc:local_exec_var
+; CHECK-32: mrs x[[R1:[0-9]+]], TPIDR_EL0
+; CHECK-32: add x0, x[[R1]], x[[R2]]
+; CHECK-32: ret
+
+; CHECK-32-RELOC: R_AARCH64_TLSLE_MOVW_TPREL_G1
+; CHECK-32-RELOC: R_AARCH64_TLSLE_MOVW_TPREL_G0_NC
+
+; CHECK-48: movz x[[R2:[0-9]+]], #:tprel_g2:local_exec_var
+; CHECK-48: movk x[[R2]], #:tprel_g1_nc:local_exec_var
+; CHECK-48: movk x[[R2]], #:tprel_g0_nc:local_exec_var
+; CHECK-48: mrs x[[R1:[0-9]+]], TPIDR_EL0
+; CHECK-48: add x0, x[[R1]], x[[R2]]
+; CHECK-48: ret
+
+; CHECK-48-RELOC: R_AARCH64_TLSLE_MOVW_TPREL_G2
+; CHECK-48-RELOC: R_AARCH64_TLSLE_MOVW_TPREL_G1_NC
+; CHECK-48-RELOC: R_AARCH64_TLSLE_MOVW_TPREL_G0_NC
+}
diff --git a/llvm/test/CodeGen/AArch64/arm64-triv-disjoint-mem-access.ll b/llvm/test/CodeGen/AArch64/arm64-triv-disjoint-mem-access.ll
index 1b1681dc4..f74156408 100644
--- a/llvm/test/CodeGen/AArch64/arm64-triv-disjoint-mem-access.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-triv-disjoint-mem-access.ll
@@ -18,7 +18,7 @@ entry:
   ret i32 %1
 }
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="true" "no-nans-fp-math"="true" "stack-protector-buffer-size"="8" "unsafe-fp-math"="true" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="true" "no-nans-fp-math"="true" "stack-protector-buffer-size"="8" "unsafe-fp-math"="true" "use-soft-float"="false" }
 
 !llvm.ident = !{!0}
 
diff --git a/llvm/test/CodeGen/AArch64/arm64-variadic-aapcs.ll b/llvm/test/CodeGen/AArch64/arm64-variadic-aapcs.ll
index db87d7fae..94c6d69e3 100644
--- a/llvm/test/CodeGen/AArch64/arm64-variadic-aapcs.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-variadic-aapcs.ll
@@ -1,4 +1,4 @@
-; RUN: llc -verify-machineinstrs -mtriple=arm64-linux-gnu -pre-RA-sched=linearize -enable-misched=false -disable-post-ra < %s | FileCheck %s
+; RUN: llc -aarch64-load-store-renaming=true -verify-machineinstrs -mtriple=arm64-linux-gnu -pre-RA-sched=linearize -enable-misched=false -disable-post-ra < %s | FileCheck %s
 
 %va_list = type {i8*, i8*, i8*, i32, i32}
 
@@ -26,11 +26,11 @@ define void @test_simple(i32 %n, ...) {
 
 ; CHECK: add [[GR_TOPTMP:x[0-9]+]], sp, #[[GR_BASE]]
 ; CHECK: add [[GR_TOP:x[0-9]+]], [[GR_TOPTMP]], #56
-; CHECK: str [[GR_TOP]], [x[[VA_LIST]], #8]
+
 
 ; CHECK: mov [[VR_TOPTMP:x[0-9]+]], sp
 ; CHECK: add [[VR_TOP:x[0-9]+]], [[VR_TOPTMP]], #128
-; CHECK: str [[VR_TOP]], [x[[VA_LIST]], #16]
+; CHECK: stp [[GR_TOP]], [[VR_TOP]], [x[[VA_LIST]], #8]
 
 ; CHECK: mov     [[GRVR:x[0-9]+]], #-56
 ; CHECK: movk    [[GRVR]], #65408, lsl #32
@@ -62,11 +62,10 @@ define void @test_fewargs(i32 %n, i32 %n1, i32 %n2, float %m, ...) {
 
 ; CHECK: add [[GR_TOPTMP:x[0-9]+]], sp, #[[GR_BASE]]
 ; CHECK: add [[GR_TOP:x[0-9]+]], [[GR_TOPTMP]], #40
-; CHECK: str [[GR_TOP]], [x[[VA_LIST]], #8]
 
 ; CHECK: mov [[VR_TOPTMP:x[0-9]+]], sp
 ; CHECK: add [[VR_TOP:x[0-9]+]], [[VR_TOPTMP]], #112
-; CHECK: str [[VR_TOP]], [x[[VA_LIST]], #16]
+; CHECK: stp [[GR_TOP]], [[VR_TOP]], [x[[VA_LIST]], #8]
 
 ; CHECK: mov  [[GRVR_OFFS:x[0-9]+]], #-40
 ; CHECK: movk [[GRVR_OFFS]], #65424, lsl #32
diff --git a/llvm/test/CodeGen/AArch64/arm64-vcvt_f.ll b/llvm/test/CodeGen/AArch64/arm64-vcvt_f.ll
index bc1c57144..6b3e8d747 100644
--- a/llvm/test/CodeGen/AArch64/arm64-vcvt_f.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-vcvt_f.ll
@@ -38,6 +38,151 @@ define <2 x double> @test_vcvt_high_f64_f32(<4 x float> %x) nounwind readnone ss
   ret <2 x double> %vcvt1.i
 }
 
+define <2 x double> @test_vcvt_high_v1f64_f32_bitcast(<4 x float> %x) nounwind readnone ssp {
+; CHECK-LABEL: test_vcvt_high_v1f64_f32_bitcast:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fcvtl2 v0.2d, v0.4s
+; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: test_vcvt_high_v1f64_f32_bitcast:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcvtl2 v0.2d, v0.4s
+; GISEL-NEXT:    ret
+  %bc1 = bitcast <4 x float> %x to <2 x double>
+  %ext = shufflevector <2 x double> %bc1, <2 x double> undef, <1 x i32> <i32 1>
+  %bc2 = bitcast <1 x double> %ext to <2 x float>
+  %r = fpext <2 x float> %bc2 to <2 x double>
+  ret <2 x double> %r
+}
+
+define <2 x double> @test_vcvt_high_v1i64_f32_bitcast(<2 x i64> %x) nounwind readnone ssp {
+; CHECK-LABEL: test_vcvt_high_v1i64_f32_bitcast:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fcvtl2 v0.2d, v0.4s
+; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: test_vcvt_high_v1i64_f32_bitcast:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcvtl2 v0.2d, v0.4s
+; GISEL-NEXT:    ret
+  %ext = shufflevector <2 x i64> %x, <2 x i64> undef, <1 x i32> <i32 1>
+  %bc2 = bitcast <1 x i64> %ext to <2 x float>
+  %r = fpext <2 x float> %bc2 to <2 x double>
+  ret <2 x double> %r
+}
+
+define <2 x double> @test_vcvt_high_v2i32_f32_bitcast(<4 x i32> %x) nounwind readnone ssp {
+; CHECK-LABEL: test_vcvt_high_v2i32_f32_bitcast:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fcvtl2 v0.2d, v0.4s
+; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: test_vcvt_high_v2i32_f32_bitcast:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcvtl2 v0.2d, v0.4s
+; GISEL-NEXT:    ret
+  %ext = shufflevector <4 x i32> %x, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
+  %bc2 = bitcast <2 x i32> %ext to <2 x float>
+  %r = fpext <2 x float> %bc2 to <2 x double>
+  ret <2 x double> %r
+}
+
+define <2 x double> @test_vcvt_high_v4i16_f32_bitcast(<8 x i16> %x) nounwind readnone ssp {
+; CHECK-LABEL: test_vcvt_high_v4i16_f32_bitcast:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fcvtl2 v0.2d, v0.4s
+; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: test_vcvt_high_v4i16_f32_bitcast:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcvtl2 v0.2d, v0.4s
+; GISEL-NEXT:    ret
+  %ext = shufflevector <8 x i16> %x, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %bc2 = bitcast <4 x i16> %ext to <2 x float>
+  %r = fpext <2 x float> %bc2 to <2 x double>
+  ret <2 x double> %r
+}
+
+define <2 x double> @test_vcvt_high_v8i8_f32_bitcast(<16 x i8> %x) nounwind readnone ssp {
+; CHECK-LABEL: test_vcvt_high_v8i8_f32_bitcast:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fcvtl2 v0.2d, v0.4s
+; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: test_vcvt_high_v8i8_f32_bitcast:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcvtl2 v0.2d, v0.4s
+; GISEL-NEXT:    ret
+  %ext = shufflevector <16 x i8> %x, <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %bc2 = bitcast <8 x i8> %ext to <2 x float>
+  %r = fpext <2 x float> %bc2 to <2 x double>
+  ret <2 x double> %r
+}
+
+define <4 x float> @test_vcvt_high_v1i64_f16_bitcast(<2 x i64> %x) nounwind readnone ssp {
+; CHECK-LABEL: test_vcvt_high_v1i64_f16_bitcast:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: test_vcvt_high_v1i64_f16_bitcast:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcvtl2 v0.4s, v0.8h
+; GISEL-NEXT:    ret
+  %ext = shufflevector <2 x i64> %x, <2 x i64> undef, <1 x i32> <i32 1>
+  %bc2 = bitcast <1 x i64> %ext to <4 x half>
+  %r = fpext <4 x half> %bc2 to <4 x float>
+  ret <4 x float> %r
+}
+
+define <4 x float> @test_vcvt_high_v2i32_f16_bitcast(<4 x i32> %x) nounwind readnone ssp {
+; CHECK-LABEL: test_vcvt_high_v2i32_f16_bitcast:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: test_vcvt_high_v2i32_f16_bitcast:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcvtl2 v0.4s, v0.8h
+; GISEL-NEXT:    ret
+  %ext = shufflevector <4 x i32> %x, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
+  %bc2 = bitcast <2 x i32> %ext to <4 x half>
+  %r = fpext <4 x half> %bc2 to <4 x float>
+  ret <4 x float> %r
+}
+
+define <4 x float> @test_vcvt_high_v4i16_f16_bitcast(<8 x i16> %x) nounwind readnone ssp {
+; CHECK-LABEL: test_vcvt_high_v4i16_f16_bitcast:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: test_vcvt_high_v4i16_f16_bitcast:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcvtl2 v0.4s, v0.8h
+; GISEL-NEXT:    ret
+  %ext = shufflevector <8 x i16> %x, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %bc2 = bitcast <4 x i16> %ext to <4 x half>
+  %r = fpext <4 x half> %bc2 to <4 x float>
+  ret <4 x float> %r
+}
+
+define <4 x float> @test_vcvt_high_v8i8_f16_bitcast(<16 x i8> %x) nounwind readnone ssp {
+; CHECK-LABEL: test_vcvt_high_v8i8_f16_bitcast:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: test_vcvt_high_v8i8_f16_bitcast:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcvtl2 v0.4s, v0.8h
+; GISEL-NEXT:    ret
+  %ext = shufflevector <16 x i8> %x, <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %bc2 = bitcast <8 x i8> %ext to <4 x half>
+  %r = fpext <4 x half> %bc2 to <4 x float>
+  ret <4 x float> %r
+}
+
 ; FALLBACK-NOT: remark{{.*}}G_FPEXT{{.*}}(in function: test_vcvt_f32_f64)
 ; FALLBACK-NOT: remark{{.*}}fpext{{.*}}(in function: test_vcvt_f32_f64)
 define <2 x float> @test_vcvt_f32_f64(<2 x double> %v) nounwind readnone ssp {
diff --git a/llvm/test/CodeGen/AArch64/arm64-vmul.ll b/llvm/test/CodeGen/AArch64/arm64-vmul.ll
index 6d795dbcf..bfb4183f4 100644
--- a/llvm/test/CodeGen/AArch64/arm64-vmul.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-vmul.ll
@@ -1,9 +1,13 @@
-; RUN: llc < %s -asm-verbose=false -mtriple=arm64-eabi -aarch64-neon-syntax=apple | FileCheck %s
-
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=arm64-eabi -aarch64-neon-syntax=apple | FileCheck %s
 
 define <8 x i16> @smull8h(<8 x i8>* %A, <8 x i8>* %B) nounwind {
-;CHECK-LABEL: smull8h:
-;CHECK: smull.8h
+; CHECK-LABEL: smull8h:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    smull.8h v0, v0, v1
+; CHECK-NEXT:    ret
   %tmp1 = load <8 x i8>, <8 x i8>* %A
   %tmp2 = load <8 x i8>, <8 x i8>* %B
   %tmp3 = call <8 x i16> @llvm.aarch64.neon.smull.v8i16(<8 x i8> %tmp1, <8 x i8> %tmp2)
@@ -11,8 +15,12 @@ define <8 x i16> @smull8h(<8 x i8>* %A, <8 x i8>* %B) nounwind {
 }
 
 define <4 x i32> @smull4s(<4 x i16>* %A, <4 x i16>* %B) nounwind {
-;CHECK-LABEL: smull4s:
-;CHECK: smull.4s
+; CHECK-LABEL: smull4s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    smull.4s v0, v0, v1
+; CHECK-NEXT:    ret
   %tmp1 = load <4 x i16>, <4 x i16>* %A
   %tmp2 = load <4 x i16>, <4 x i16>* %B
   %tmp3 = call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %tmp1, <4 x i16> %tmp2)
@@ -20,8 +28,12 @@ define <4 x i32> @smull4s(<4 x i16>* %A, <4 x i16>* %B) nounwind {
 }
 
 define <2 x i64> @smull2d(<2 x i32>* %A, <2 x i32>* %B) nounwind {
-;CHECK-LABEL: smull2d:
-;CHECK: smull.2d
+; CHECK-LABEL: smull2d:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    smull.2d v0, v0, v1
+; CHECK-NEXT:    ret
   %tmp1 = load <2 x i32>, <2 x i32>* %A
   %tmp2 = load <2 x i32>, <2 x i32>* %B
   %tmp3 = call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %tmp1, <2 x i32> %tmp2)
@@ -33,8 +45,12 @@ declare <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16>, <4 x i16>) nounwind
 declare <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32>, <2 x i32>) nounwind readnone
 
 define <8 x i16> @umull8h(<8 x i8>* %A, <8 x i8>* %B) nounwind {
-;CHECK-LABEL: umull8h:
-;CHECK: umull.8h
+; CHECK-LABEL: umull8h:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    umull.8h v0, v0, v1
+; CHECK-NEXT:    ret
   %tmp1 = load <8 x i8>, <8 x i8>* %A
   %tmp2 = load <8 x i8>, <8 x i8>* %B
   %tmp3 = call <8 x i16> @llvm.aarch64.neon.umull.v8i16(<8 x i8> %tmp1, <8 x i8> %tmp2)
@@ -42,8 +58,12 @@ define <8 x i16> @umull8h(<8 x i8>* %A, <8 x i8>* %B) nounwind {
 }
 
 define <4 x i32> @umull4s(<4 x i16>* %A, <4 x i16>* %B) nounwind {
-;CHECK-LABEL: umull4s:
-;CHECK: umull.4s
+; CHECK-LABEL: umull4s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    umull.4s v0, v0, v1
+; CHECK-NEXT:    ret
   %tmp1 = load <4 x i16>, <4 x i16>* %A
   %tmp2 = load <4 x i16>, <4 x i16>* %B
   %tmp3 = call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %tmp1, <4 x i16> %tmp2)
@@ -51,8 +71,12 @@ define <4 x i32> @umull4s(<4 x i16>* %A, <4 x i16>* %B) nounwind {
 }
 
 define <2 x i64> @umull2d(<2 x i32>* %A, <2 x i32>* %B) nounwind {
-;CHECK-LABEL: umull2d:
-;CHECK: umull.2d
+; CHECK-LABEL: umull2d:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    umull.2d v0, v0, v1
+; CHECK-NEXT:    ret
   %tmp1 = load <2 x i32>, <2 x i32>* %A
   %tmp2 = load <2 x i32>, <2 x i32>* %B
   %tmp3 = call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %tmp1, <2 x i32> %tmp2)
@@ -64,8 +88,12 @@ declare <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16>, <4 x i16>) nounwind
 declare <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32>, <2 x i32>) nounwind readnone
 
 define <4 x i32> @sqdmull4s(<4 x i16>* %A, <4 x i16>* %B) nounwind {
-;CHECK-LABEL: sqdmull4s:
-;CHECK: sqdmull.4s
+; CHECK-LABEL: sqdmull4s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    sqdmull.4s v0, v0, v1
+; CHECK-NEXT:    ret
   %tmp1 = load <4 x i16>, <4 x i16>* %A
   %tmp2 = load <4 x i16>, <4 x i16>* %B
   %tmp3 = call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %tmp1, <4 x i16> %tmp2)
@@ -73,8 +101,12 @@ define <4 x i32> @sqdmull4s(<4 x i16>* %A, <4 x i16>* %B) nounwind {
 }
 
 define <2 x i64> @sqdmull2d(<2 x i32>* %A, <2 x i32>* %B) nounwind {
-;CHECK-LABEL: sqdmull2d:
-;CHECK: sqdmull.2d
+; CHECK-LABEL: sqdmull2d:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    sqdmull.2d v0, v0, v1
+; CHECK-NEXT:    ret
   %tmp1 = load <2 x i32>, <2 x i32>* %A
   %tmp2 = load <2 x i32>, <2 x i32>* %B
   %tmp3 = call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %tmp1, <2 x i32> %tmp2)
@@ -82,8 +114,12 @@ define <2 x i64> @sqdmull2d(<2 x i32>* %A, <2 x i32>* %B) nounwind {
 }
 
 define <4 x i32> @sqdmull2_4s(<8 x i16>* %A, <8 x i16>* %B) nounwind {
-;CHECK-LABEL: sqdmull2_4s:
-;CHECK: sqdmull.4s
+; CHECK-LABEL: sqdmull2_4s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0, #8]
+; CHECK-NEXT:    ldr d1, [x1, #8]
+; CHECK-NEXT:    sqdmull.4s v0, v0, v1
+; CHECK-NEXT:    ret
   %load1 = load <8 x i16>, <8 x i16>* %A
   %load2 = load <8 x i16>, <8 x i16>* %B
   %tmp1 = shufflevector <8 x i16> %load1, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
@@ -93,8 +129,12 @@ define <4 x i32> @sqdmull2_4s(<8 x i16>* %A, <8 x i16>* %B) nounwind {
 }
 
 define <2 x i64> @sqdmull2_2d(<4 x i32>* %A, <4 x i32>* %B) nounwind {
-;CHECK-LABEL: sqdmull2_2d:
-;CHECK: sqdmull.2d
+; CHECK-LABEL: sqdmull2_2d:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0, #8]
+; CHECK-NEXT:    ldr d1, [x1, #8]
+; CHECK-NEXT:    sqdmull.2d v0, v0, v1
+; CHECK-NEXT:    ret
   %load1 = load <4 x i32>, <4 x i32>* %A
   %load2 = load <4 x i32>, <4 x i32>* %B
   %tmp1 = shufflevector <4 x i32> %load1, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
@@ -108,8 +148,12 @@ declare <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16>, <4 x i16>) nounwin
 declare <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32>, <2 x i32>) nounwind readnone
 
 define <8 x i16> @pmull8h(<8 x i8>* %A, <8 x i8>* %B) nounwind {
-;CHECK-LABEL: pmull8h:
-;CHECK: pmull.8h
+; CHECK-LABEL: pmull8h:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    pmull.8h v0, v0, v1
+; CHECK-NEXT:    ret
   %tmp1 = load <8 x i8>, <8 x i8>* %A
   %tmp2 = load <8 x i8>, <8 x i8>* %B
   %tmp3 = call <8 x i16> @llvm.aarch64.neon.pmull.v8i16(<8 x i8> %tmp1, <8 x i8> %tmp2)
@@ -119,8 +163,12 @@ define <8 x i16> @pmull8h(<8 x i8>* %A, <8 x i8>* %B) nounwind {
 declare <8 x i16> @llvm.aarch64.neon.pmull.v8i16(<8 x i8>, <8 x i8>) nounwind readnone
 
 define <4 x i16> @sqdmulh_4h(<4 x i16>* %A, <4 x i16>* %B) nounwind {
-;CHECK-LABEL: sqdmulh_4h:
-;CHECK: sqdmulh.4h
+; CHECK-LABEL: sqdmulh_4h:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    sqdmulh.4h v0, v0, v1
+; CHECK-NEXT:    ret
   %tmp1 = load <4 x i16>, <4 x i16>* %A
   %tmp2 = load <4 x i16>, <4 x i16>* %B
   %tmp3 = call <4 x i16> @llvm.aarch64.neon.sqdmulh.v4i16(<4 x i16> %tmp1, <4 x i16> %tmp2)
@@ -128,8 +176,12 @@ define <4 x i16> @sqdmulh_4h(<4 x i16>* %A, <4 x i16>* %B) nounwind {
 }
 
 define <8 x i16> @sqdmulh_8h(<8 x i16>* %A, <8 x i16>* %B) nounwind {
-;CHECK-LABEL: sqdmulh_8h:
-;CHECK: sqdmulh.8h
+; CHECK-LABEL: sqdmulh_8h:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    ldr q1, [x1]
+; CHECK-NEXT:    sqdmulh.8h v0, v0, v1
+; CHECK-NEXT:    ret
   %tmp1 = load <8 x i16>, <8 x i16>* %A
   %tmp2 = load <8 x i16>, <8 x i16>* %B
   %tmp3 = call <8 x i16> @llvm.aarch64.neon.sqdmulh.v8i16(<8 x i16> %tmp1, <8 x i16> %tmp2)
@@ -137,8 +189,12 @@ define <8 x i16> @sqdmulh_8h(<8 x i16>* %A, <8 x i16>* %B) nounwind {
 }
 
 define <2 x i32> @sqdmulh_2s(<2 x i32>* %A, <2 x i32>* %B) nounwind {
-;CHECK-LABEL: sqdmulh_2s:
-;CHECK: sqdmulh.2s
+; CHECK-LABEL: sqdmulh_2s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    sqdmulh.2s v0, v0, v1
+; CHECK-NEXT:    ret
   %tmp1 = load <2 x i32>, <2 x i32>* %A
   %tmp2 = load <2 x i32>, <2 x i32>* %B
   %tmp3 = call <2 x i32> @llvm.aarch64.neon.sqdmulh.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2)
@@ -146,8 +202,12 @@ define <2 x i32> @sqdmulh_2s(<2 x i32>* %A, <2 x i32>* %B) nounwind {
 }
 
 define <4 x i32> @sqdmulh_4s(<4 x i32>* %A, <4 x i32>* %B) nounwind {
-;CHECK-LABEL: sqdmulh_4s:
-;CHECK: sqdmulh.4s
+; CHECK-LABEL: sqdmulh_4s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    ldr q1, [x1]
+; CHECK-NEXT:    sqdmulh.4s v0, v0, v1
+; CHECK-NEXT:    ret
   %tmp1 = load <4 x i32>, <4 x i32>* %A
   %tmp2 = load <4 x i32>, <4 x i32>* %B
   %tmp3 = call <4 x i32> @llvm.aarch64.neon.sqdmulh.v4i32(<4 x i32> %tmp1, <4 x i32> %tmp2)
@@ -155,8 +215,15 @@ define <4 x i32> @sqdmulh_4s(<4 x i32>* %A, <4 x i32>* %B) nounwind {
 }
 
 define i32 @sqdmulh_1s(i32* %A, i32* %B) nounwind {
-;CHECK-LABEL: sqdmulh_1s:
-;CHECK: sqdmulh s0, {{s[0-9]+}}, {{s[0-9]+}}
+; CHECK-LABEL: sqdmulh_1s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr w8, [x0]
+; CHECK-NEXT:    ldr w9, [x1]
+; CHECK-NEXT:    fmov s0, w8
+; CHECK-NEXT:    fmov s1, w9
+; CHECK-NEXT:    sqdmulh s0, s0, s1
+; CHECK-NEXT:    fmov w0, s0
+; CHECK-NEXT:    ret
   %tmp1 = load i32, i32* %A
   %tmp2 = load i32, i32* %B
   %tmp3 = call i32 @llvm.aarch64.neon.sqdmulh.i32(i32 %tmp1, i32 %tmp2)
@@ -170,8 +237,12 @@ declare <4 x i32> @llvm.aarch64.neon.sqdmulh.v4i32(<4 x i32>, <4 x i32>) nounwin
 declare i32 @llvm.aarch64.neon.sqdmulh.i32(i32, i32) nounwind readnone
 
 define <4 x i16> @sqrdmulh_4h(<4 x i16>* %A, <4 x i16>* %B) nounwind {
-;CHECK-LABEL: sqrdmulh_4h:
-;CHECK: sqrdmulh.4h
+; CHECK-LABEL: sqrdmulh_4h:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    sqrdmulh.4h v0, v0, v1
+; CHECK-NEXT:    ret
   %tmp1 = load <4 x i16>, <4 x i16>* %A
   %tmp2 = load <4 x i16>, <4 x i16>* %B
   %tmp3 = call <4 x i16> @llvm.aarch64.neon.sqrdmulh.v4i16(<4 x i16> %tmp1, <4 x i16> %tmp2)
@@ -179,8 +250,12 @@ define <4 x i16> @sqrdmulh_4h(<4 x i16>* %A, <4 x i16>* %B) nounwind {
 }
 
 define <8 x i16> @sqrdmulh_8h(<8 x i16>* %A, <8 x i16>* %B) nounwind {
-;CHECK-LABEL: sqrdmulh_8h:
-;CHECK: sqrdmulh.8h
+; CHECK-LABEL: sqrdmulh_8h:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    ldr q1, [x1]
+; CHECK-NEXT:    sqrdmulh.8h v0, v0, v1
+; CHECK-NEXT:    ret
   %tmp1 = load <8 x i16>, <8 x i16>* %A
   %tmp2 = load <8 x i16>, <8 x i16>* %B
   %tmp3 = call <8 x i16> @llvm.aarch64.neon.sqrdmulh.v8i16(<8 x i16> %tmp1, <8 x i16> %tmp2)
@@ -188,8 +263,12 @@ define <8 x i16> @sqrdmulh_8h(<8 x i16>* %A, <8 x i16>* %B) nounwind {
 }
 
 define <2 x i32> @sqrdmulh_2s(<2 x i32>* %A, <2 x i32>* %B) nounwind {
-;CHECK-LABEL: sqrdmulh_2s:
-;CHECK: sqrdmulh.2s
+; CHECK-LABEL: sqrdmulh_2s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    sqrdmulh.2s v0, v0, v1
+; CHECK-NEXT:    ret
   %tmp1 = load <2 x i32>, <2 x i32>* %A
   %tmp2 = load <2 x i32>, <2 x i32>* %B
   %tmp3 = call <2 x i32> @llvm.aarch64.neon.sqrdmulh.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2)
@@ -197,8 +276,12 @@ define <2 x i32> @sqrdmulh_2s(<2 x i32>* %A, <2 x i32>* %B) nounwind {
 }
 
 define <4 x i32> @sqrdmulh_4s(<4 x i32>* %A, <4 x i32>* %B) nounwind {
-;CHECK-LABEL: sqrdmulh_4s:
-;CHECK: sqrdmulh.4s
+; CHECK-LABEL: sqrdmulh_4s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    ldr q1, [x1]
+; CHECK-NEXT:    sqrdmulh.4s v0, v0, v1
+; CHECK-NEXT:    ret
   %tmp1 = load <4 x i32>, <4 x i32>* %A
   %tmp2 = load <4 x i32>, <4 x i32>* %B
   %tmp3 = call <4 x i32> @llvm.aarch64.neon.sqrdmulh.v4i32(<4 x i32> %tmp1, <4 x i32> %tmp2)
@@ -206,8 +289,15 @@ define <4 x i32> @sqrdmulh_4s(<4 x i32>* %A, <4 x i32>* %B) nounwind {
 }
 
 define i32 @sqrdmulh_1s(i32* %A, i32* %B) nounwind {
-;CHECK-LABEL: sqrdmulh_1s:
-;CHECK: sqrdmulh s0, {{s[0-9]+}}, {{s[0-9]+}}
+; CHECK-LABEL: sqrdmulh_1s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr w8, [x0]
+; CHECK-NEXT:    ldr w9, [x1]
+; CHECK-NEXT:    fmov s0, w8
+; CHECK-NEXT:    fmov s1, w9
+; CHECK-NEXT:    sqrdmulh s0, s0, s1
+; CHECK-NEXT:    fmov w0, s0
+; CHECK-NEXT:    ret
   %tmp1 = load i32, i32* %A
   %tmp2 = load i32, i32* %B
   %tmp3 = call i32 @llvm.aarch64.neon.sqrdmulh.i32(i32 %tmp1, i32 %tmp2)
@@ -221,8 +311,12 @@ declare <4 x i32> @llvm.aarch64.neon.sqrdmulh.v4i32(<4 x i32>, <4 x i32>) nounwi
 declare i32 @llvm.aarch64.neon.sqrdmulh.i32(i32, i32) nounwind readnone
 
 define <2 x float> @fmulx_2s(<2 x float>* %A, <2 x float>* %B) nounwind {
-;CHECK-LABEL: fmulx_2s:
-;CHECK: fmulx.2s
+; CHECK-LABEL: fmulx_2s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    fmulx.2s v0, v0, v1
+; CHECK-NEXT:    ret
   %tmp1 = load <2 x float>, <2 x float>* %A
   %tmp2 = load <2 x float>, <2 x float>* %B
   %tmp3 = call <2 x float> @llvm.aarch64.neon.fmulx.v2f32(<2 x float> %tmp1, <2 x float> %tmp2)
@@ -230,8 +324,12 @@ define <2 x float> @fmulx_2s(<2 x float>* %A, <2 x float>* %B) nounwind {
 }
 
 define <4 x float> @fmulx_4s(<4 x float>* %A, <4 x float>* %B) nounwind {
-;CHECK-LABEL: fmulx_4s:
-;CHECK: fmulx.4s
+; CHECK-LABEL: fmulx_4s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    ldr q1, [x1]
+; CHECK-NEXT:    fmulx.4s v0, v0, v1
+; CHECK-NEXT:    ret
   %tmp1 = load <4 x float>, <4 x float>* %A
   %tmp2 = load <4 x float>, <4 x float>* %B
   %tmp3 = call <4 x float> @llvm.aarch64.neon.fmulx.v4f32(<4 x float> %tmp1, <4 x float> %tmp2)
@@ -239,8 +337,12 @@ define <4 x float> @fmulx_4s(<4 x float>* %A, <4 x float>* %B) nounwind {
 }
 
 define <2 x double> @fmulx_2d(<2 x double>* %A, <2 x double>* %B) nounwind {
-;CHECK-LABEL: fmulx_2d:
-;CHECK: fmulx.2d
+; CHECK-LABEL: fmulx_2d:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    ldr q1, [x1]
+; CHECK-NEXT:    fmulx.2d v0, v0, v1
+; CHECK-NEXT:    ret
   %tmp1 = load <2 x double>, <2 x double>* %A
   %tmp2 = load <2 x double>, <2 x double>* %B
   %tmp3 = call <2 x double> @llvm.aarch64.neon.fmulx.v2f64(<2 x double> %tmp1, <2 x double> %tmp2)
@@ -252,8 +354,13 @@ declare <4 x float> @llvm.aarch64.neon.fmulx.v4f32(<4 x float>, <4 x float>) nou
 declare <2 x double> @llvm.aarch64.neon.fmulx.v2f64(<2 x double>, <2 x double>) nounwind readnone
 
 define <4 x i32> @smlal4s(<4 x i16>* %A, <4 x i16>* %B, <4 x i32>* %C) nounwind {
-;CHECK-LABEL: smlal4s:
-;CHECK: smlal.4s
+; CHECK-LABEL: smlal4s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d1, [x0]
+; CHECK-NEXT:    ldr d2, [x1]
+; CHECK-NEXT:    ldr q0, [x2]
+; CHECK-NEXT:    smlal.4s v0, v1, v2
+; CHECK-NEXT:    ret
   %tmp1 = load <4 x i16>, <4 x i16>* %A
   %tmp2 = load <4 x i16>, <4 x i16>* %B
   %tmp3 = load <4 x i32>, <4 x i32>* %C
@@ -263,8 +370,13 @@ define <4 x i32> @smlal4s(<4 x i16>* %A, <4 x i16>* %B, <4 x i32>* %C) nounwind
 }
 
 define <2 x i64> @smlal2d(<2 x i32>* %A, <2 x i32>* %B, <2 x i64>* %C) nounwind {
-;CHECK-LABEL: smlal2d:
-;CHECK: smlal.2d
+; CHECK-LABEL: smlal2d:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d1, [x0]
+; CHECK-NEXT:    ldr d2, [x1]
+; CHECK-NEXT:    ldr q0, [x2]
+; CHECK-NEXT:    smlal.2d v0, v1, v2
+; CHECK-NEXT:    ret
   %tmp1 = load <2 x i32>, <2 x i32>* %A
   %tmp2 = load <2 x i32>, <2 x i32>* %B
   %tmp3 = load <2 x i64>, <2 x i64>* %C
@@ -274,8 +386,13 @@ define <2 x i64> @smlal2d(<2 x i32>* %A, <2 x i32>* %B, <2 x i64>* %C) nounwind
 }
 
 define <4 x i32> @smlsl4s(<4 x i16>* %A, <4 x i16>* %B, <4 x i32>* %C) nounwind {
-;CHECK-LABEL: smlsl4s:
-;CHECK: smlsl.4s
+; CHECK-LABEL: smlsl4s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d1, [x0]
+; CHECK-NEXT:    ldr d2, [x1]
+; CHECK-NEXT:    ldr q0, [x2]
+; CHECK-NEXT:    smlsl.4s v0, v1, v2
+; CHECK-NEXT:    ret
   %tmp1 = load <4 x i16>, <4 x i16>* %A
   %tmp2 = load <4 x i16>, <4 x i16>* %B
   %tmp3 = load <4 x i32>, <4 x i32>* %C
@@ -285,8 +402,13 @@ define <4 x i32> @smlsl4s(<4 x i16>* %A, <4 x i16>* %B, <4 x i32>* %C) nounwind
 }
 
 define <2 x i64> @smlsl2d(<2 x i32>* %A, <2 x i32>* %B, <2 x i64>* %C) nounwind {
-;CHECK-LABEL: smlsl2d:
-;CHECK: smlsl.2d
+; CHECK-LABEL: smlsl2d:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d1, [x0]
+; CHECK-NEXT:    ldr d2, [x1]
+; CHECK-NEXT:    ldr q0, [x2]
+; CHECK-NEXT:    smlsl.2d v0, v1, v2
+; CHECK-NEXT:    ret
   %tmp1 = load <2 x i32>, <2 x i32>* %A
   %tmp2 = load <2 x i32>, <2 x i32>* %B
   %tmp3 = load <2 x i64>, <2 x i64>* %C
@@ -301,8 +423,13 @@ declare <4 x i32> @llvm.aarch64.neon.sqsub.v4i32(<4 x i32>, <4 x i32>)
 declare <2 x i64> @llvm.aarch64.neon.sqsub.v2i64(<2 x i64>, <2 x i64>)
 
 define <4 x i32> @sqdmlal4s(<4 x i16>* %A, <4 x i16>* %B, <4 x i32>* %C) nounwind {
-;CHECK-LABEL: sqdmlal4s:
-;CHECK: sqdmlal.4s
+; CHECK-LABEL: sqdmlal4s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d1, [x0]
+; CHECK-NEXT:    ldr d2, [x1]
+; CHECK-NEXT:    ldr q0, [x2]
+; CHECK-NEXT:    sqdmlal.4s v0, v1, v2
+; CHECK-NEXT:    ret
   %tmp1 = load <4 x i16>, <4 x i16>* %A
   %tmp2 = load <4 x i16>, <4 x i16>* %B
   %tmp3 = load <4 x i32>, <4 x i32>* %C
@@ -312,8 +439,13 @@ define <4 x i32> @sqdmlal4s(<4 x i16>* %A, <4 x i16>* %B, <4 x i32>* %C) nounwin
 }
 
 define <2 x i64> @sqdmlal2d(<2 x i32>* %A, <2 x i32>* %B, <2 x i64>* %C) nounwind {
-;CHECK-LABEL: sqdmlal2d:
-;CHECK: sqdmlal.2d
+; CHECK-LABEL: sqdmlal2d:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d1, [x0]
+; CHECK-NEXT:    ldr d2, [x1]
+; CHECK-NEXT:    ldr q0, [x2]
+; CHECK-NEXT:    sqdmlal.2d v0, v1, v2
+; CHECK-NEXT:    ret
   %tmp1 = load <2 x i32>, <2 x i32>* %A
   %tmp2 = load <2 x i32>, <2 x i32>* %B
   %tmp3 = load <2 x i64>, <2 x i64>* %C
@@ -323,8 +455,13 @@ define <2 x i64> @sqdmlal2d(<2 x i32>* %A, <2 x i32>* %B, <2 x i64>* %C) nounwin
 }
 
 define <4 x i32> @sqdmlal2_4s(<8 x i16>* %A, <8 x i16>* %B, <4 x i32>* %C) nounwind {
-;CHECK-LABEL: sqdmlal2_4s:
-;CHECK: sqdmlal.4s
+; CHECK-LABEL: sqdmlal2_4s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x2]
+; CHECK-NEXT:    ldr d1, [x0, #8]
+; CHECK-NEXT:    ldr d2, [x1, #8]
+; CHECK-NEXT:    sqdmlal.4s v0, v1, v2
+; CHECK-NEXT:    ret
   %load1 = load <8 x i16>, <8 x i16>* %A
   %load2 = load <8 x i16>, <8 x i16>* %B
   %tmp3 = load <4 x i32>, <4 x i32>* %C
@@ -336,8 +473,13 @@ define <4 x i32> @sqdmlal2_4s(<8 x i16>* %A, <8 x i16>* %B, <4 x i32>* %C) nounw
 }
 
 define <2 x i64> @sqdmlal2_2d(<4 x i32>* %A, <4 x i32>* %B, <2 x i64>* %C) nounwind {
-;CHECK-LABEL: sqdmlal2_2d:
-;CHECK: sqdmlal.2d
+; CHECK-LABEL: sqdmlal2_2d:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x2]
+; CHECK-NEXT:    ldr d1, [x0, #8]
+; CHECK-NEXT:    ldr d2, [x1, #8]
+; CHECK-NEXT:    sqdmlal.2d v0, v1, v2
+; CHECK-NEXT:    ret
   %load1 = load <4 x i32>, <4 x i32>* %A
   %load2 = load <4 x i32>, <4 x i32>* %B
   %tmp3 = load <2 x i64>, <2 x i64>* %C
@@ -349,8 +491,13 @@ define <2 x i64> @sqdmlal2_2d(<4 x i32>* %A, <4 x i32>* %B, <2 x i64>* %C) nounw
 }
 
 define <4 x i32> @sqdmlsl4s(<4 x i16>* %A, <4 x i16>* %B, <4 x i32>* %C) nounwind {
-;CHECK-LABEL: sqdmlsl4s:
-;CHECK: sqdmlsl.4s
+; CHECK-LABEL: sqdmlsl4s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d1, [x0]
+; CHECK-NEXT:    ldr d2, [x1]
+; CHECK-NEXT:    ldr q0, [x2]
+; CHECK-NEXT:    sqdmlsl.4s v0, v1, v2
+; CHECK-NEXT:    ret
   %tmp1 = load <4 x i16>, <4 x i16>* %A
   %tmp2 = load <4 x i16>, <4 x i16>* %B
   %tmp3 = load <4 x i32>, <4 x i32>* %C
@@ -360,8 +507,13 @@ define <4 x i32> @sqdmlsl4s(<4 x i16>* %A, <4 x i16>* %B, <4 x i32>* %C) nounwin
 }
 
 define <2 x i64> @sqdmlsl2d(<2 x i32>* %A, <2 x i32>* %B, <2 x i64>* %C) nounwind {
-;CHECK-LABEL: sqdmlsl2d:
-;CHECK: sqdmlsl.2d
+; CHECK-LABEL: sqdmlsl2d:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d1, [x0]
+; CHECK-NEXT:    ldr d2, [x1]
+; CHECK-NEXT:    ldr q0, [x2]
+; CHECK-NEXT:    sqdmlsl.2d v0, v1, v2
+; CHECK-NEXT:    ret
   %tmp1 = load <2 x i32>, <2 x i32>* %A
   %tmp2 = load <2 x i32>, <2 x i32>* %B
   %tmp3 = load <2 x i64>, <2 x i64>* %C
@@ -371,8 +523,13 @@ define <2 x i64> @sqdmlsl2d(<2 x i32>* %A, <2 x i32>* %B, <2 x i64>* %C) nounwin
 }
 
 define <4 x i32> @sqdmlsl2_4s(<8 x i16>* %A, <8 x i16>* %B, <4 x i32>* %C) nounwind {
-;CHECK-LABEL: sqdmlsl2_4s:
-;CHECK: sqdmlsl.4s
+; CHECK-LABEL: sqdmlsl2_4s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x2]
+; CHECK-NEXT:    ldr d1, [x0, #8]
+; CHECK-NEXT:    ldr d2, [x1, #8]
+; CHECK-NEXT:    sqdmlsl.4s v0, v1, v2
+; CHECK-NEXT:    ret
   %load1 = load <8 x i16>, <8 x i16>* %A
   %load2 = load <8 x i16>, <8 x i16>* %B
   %tmp3 = load <4 x i32>, <4 x i32>* %C
@@ -384,8 +541,13 @@ define <4 x i32> @sqdmlsl2_4s(<8 x i16>* %A, <8 x i16>* %B, <4 x i32>* %C) nounw
 }
 
 define <2 x i64> @sqdmlsl2_2d(<4 x i32>* %A, <4 x i32>* %B, <2 x i64>* %C) nounwind {
-;CHECK-LABEL: sqdmlsl2_2d:
-;CHECK: sqdmlsl.2d
+; CHECK-LABEL: sqdmlsl2_2d:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x2]
+; CHECK-NEXT:    ldr d1, [x0, #8]
+; CHECK-NEXT:    ldr d2, [x1, #8]
+; CHECK-NEXT:    sqdmlsl.2d v0, v1, v2
+; CHECK-NEXT:    ret
   %load1 = load <4 x i32>, <4 x i32>* %A
   %load2 = load <4 x i32>, <4 x i32>* %B
   %tmp3 = load <2 x i64>, <2 x i64>* %C
@@ -397,8 +559,13 @@ define <2 x i64> @sqdmlsl2_2d(<4 x i32>* %A, <4 x i32>* %B, <2 x i64>* %C) nounw
 }
 
 define <4 x i32> @umlal4s(<4 x i16>* %A, <4 x i16>* %B, <4 x i32>* %C) nounwind {
-;CHECK-LABEL: umlal4s:
-;CHECK: umlal.4s
+; CHECK-LABEL: umlal4s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d1, [x0]
+; CHECK-NEXT:    ldr d2, [x1]
+; CHECK-NEXT:    ldr q0, [x2]
+; CHECK-NEXT:    umlal.4s v0, v1, v2
+; CHECK-NEXT:    ret
   %tmp1 = load <4 x i16>, <4 x i16>* %A
   %tmp2 = load <4 x i16>, <4 x i16>* %B
   %tmp3 = load <4 x i32>, <4 x i32>* %C
@@ -408,8 +575,13 @@ define <4 x i32> @umlal4s(<4 x i16>* %A, <4 x i16>* %B, <4 x i32>* %C) nounwind
 }
 
 define <2 x i64> @umlal2d(<2 x i32>* %A, <2 x i32>* %B, <2 x i64>* %C) nounwind {
-;CHECK-LABEL: umlal2d:
-;CHECK: umlal.2d
+; CHECK-LABEL: umlal2d:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d1, [x0]
+; CHECK-NEXT:    ldr d2, [x1]
+; CHECK-NEXT:    ldr q0, [x2]
+; CHECK-NEXT:    umlal.2d v0, v1, v2
+; CHECK-NEXT:    ret
   %tmp1 = load <2 x i32>, <2 x i32>* %A
   %tmp2 = load <2 x i32>, <2 x i32>* %B
   %tmp3 = load <2 x i64>, <2 x i64>* %C
@@ -419,8 +591,13 @@ define <2 x i64> @umlal2d(<2 x i32>* %A, <2 x i32>* %B, <2 x i64>* %C) nounwind
 }
 
 define <4 x i32> @umlsl4s(<4 x i16>* %A, <4 x i16>* %B, <4 x i32>* %C) nounwind {
-;CHECK-LABEL: umlsl4s:
-;CHECK: umlsl.4s
+; CHECK-LABEL: umlsl4s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d1, [x0]
+; CHECK-NEXT:    ldr d2, [x1]
+; CHECK-NEXT:    ldr q0, [x2]
+; CHECK-NEXT:    umlsl.4s v0, v1, v2
+; CHECK-NEXT:    ret
   %tmp1 = load <4 x i16>, <4 x i16>* %A
   %tmp2 = load <4 x i16>, <4 x i16>* %B
   %tmp3 = load <4 x i32>, <4 x i32>* %C
@@ -430,8 +607,13 @@ define <4 x i32> @umlsl4s(<4 x i16>* %A, <4 x i16>* %B, <4 x i32>* %C) nounwind
 }
 
 define <2 x i64> @umlsl2d(<2 x i32>* %A, <2 x i32>* %B, <2 x i64>* %C) nounwind {
-;CHECK-LABEL: umlsl2d:
-;CHECK: umlsl.2d
+; CHECK-LABEL: umlsl2d:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d1, [x0]
+; CHECK-NEXT:    ldr d2, [x1]
+; CHECK-NEXT:    ldr q0, [x2]
+; CHECK-NEXT:    umlsl.2d v0, v1, v2
+; CHECK-NEXT:    ret
   %tmp1 = load <2 x i32>, <2 x i32>* %A
   %tmp2 = load <2 x i32>, <2 x i32>* %B
   %tmp3 = load <2 x i64>, <2 x i64>* %C
@@ -441,8 +623,13 @@ define <2 x i64> @umlsl2d(<2 x i32>* %A, <2 x i32>* %B, <2 x i64>* %C) nounwind
 }
 
 define <2 x float> @fmla_2s(<2 x float>* %A, <2 x float>* %B, <2 x float>* %C) nounwind {
-;CHECK-LABEL: fmla_2s:
-;CHECK: fmla.2s
+; CHECK-LABEL: fmla_2s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d1, [x0]
+; CHECK-NEXT:    ldr d2, [x1]
+; CHECK-NEXT:    ldr d0, [x2]
+; CHECK-NEXT:    fmla.2s v0, v2, v1
+; CHECK-NEXT:    ret
   %tmp1 = load <2 x float>, <2 x float>* %A
   %tmp2 = load <2 x float>, <2 x float>* %B
   %tmp3 = load <2 x float>, <2 x float>* %C
@@ -451,8 +638,13 @@ define <2 x float> @fmla_2s(<2 x float>* %A, <2 x float>* %B, <2 x float>* %C) n
 }
 
 define <4 x float> @fmla_4s(<4 x float>* %A, <4 x float>* %B, <4 x float>* %C) nounwind {
-;CHECK-LABEL: fmla_4s:
-;CHECK: fmla.4s
+; CHECK-LABEL: fmla_4s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q1, [x0]
+; CHECK-NEXT:    ldr q2, [x1]
+; CHECK-NEXT:    ldr q0, [x2]
+; CHECK-NEXT:    fmla.4s v0, v2, v1
+; CHECK-NEXT:    ret
   %tmp1 = load <4 x float>, <4 x float>* %A
   %tmp2 = load <4 x float>, <4 x float>* %B
   %tmp3 = load <4 x float>, <4 x float>* %C
@@ -461,8 +653,13 @@ define <4 x float> @fmla_4s(<4 x float>* %A, <4 x float>* %B, <4 x float>* %C) n
 }
 
 define <2 x double> @fmla_2d(<2 x double>* %A, <2 x double>* %B, <2 x double>* %C) nounwind {
-;CHECK-LABEL: fmla_2d:
-;CHECK: fmla.2d
+; CHECK-LABEL: fmla_2d:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q1, [x0]
+; CHECK-NEXT:    ldr q2, [x1]
+; CHECK-NEXT:    ldr q0, [x2]
+; CHECK-NEXT:    fmla.2d v0, v2, v1
+; CHECK-NEXT:    ret
   %tmp1 = load <2 x double>, <2 x double>* %A
   %tmp2 = load <2 x double>, <2 x double>* %B
   %tmp3 = load <2 x double>, <2 x double>* %C
@@ -475,8 +672,13 @@ declare <4 x float> @llvm.fma.v4f32(<4 x float>, <4 x float>, <4 x float>) nounw
 declare <2 x double> @llvm.fma.v2f64(<2 x double>, <2 x double>, <2 x double>) nounwind readnone
 
 define <2 x float> @fmls_2s(<2 x float>* %A, <2 x float>* %B, <2 x float>* %C) nounwind {
-;CHECK-LABEL: fmls_2s:
-;CHECK: fmls.2s
+; CHECK-LABEL: fmls_2s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d1, [x0]
+; CHECK-NEXT:    ldr d2, [x1]
+; CHECK-NEXT:    ldr d0, [x2]
+; CHECK-NEXT:    fmls.2s v0, v1, v2
+; CHECK-NEXT:    ret
   %tmp1 = load <2 x float>, <2 x float>* %A
   %tmp2 = load <2 x float>, <2 x float>* %B
   %tmp3 = load <2 x float>, <2 x float>* %C
@@ -486,8 +688,13 @@ define <2 x float> @fmls_2s(<2 x float>* %A, <2 x float>* %B, <2 x float>* %C) n
 }
 
 define <4 x float> @fmls_4s(<4 x float>* %A, <4 x float>* %B, <4 x float>* %C) nounwind {
-;CHECK-LABEL: fmls_4s:
-;CHECK: fmls.4s
+; CHECK-LABEL: fmls_4s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q1, [x0]
+; CHECK-NEXT:    ldr q2, [x1]
+; CHECK-NEXT:    ldr q0, [x2]
+; CHECK-NEXT:    fmls.4s v0, v1, v2
+; CHECK-NEXT:    ret
   %tmp1 = load <4 x float>, <4 x float>* %A
   %tmp2 = load <4 x float>, <4 x float>* %B
   %tmp3 = load <4 x float>, <4 x float>* %C
@@ -497,8 +704,13 @@ define <4 x float> @fmls_4s(<4 x float>* %A, <4 x float>* %B, <4 x float>* %C) n
 }
 
 define <2 x double> @fmls_2d(<2 x double>* %A, <2 x double>* %B, <2 x double>* %C) nounwind {
-;CHECK-LABEL: fmls_2d:
-;CHECK: fmls.2d
+; CHECK-LABEL: fmls_2d:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q1, [x0]
+; CHECK-NEXT:    ldr q2, [x1]
+; CHECK-NEXT:    ldr q0, [x2]
+; CHECK-NEXT:    fmls.2d v0, v1, v2
+; CHECK-NEXT:    ret
   %tmp1 = load <2 x double>, <2 x double>* %A
   %tmp2 = load <2 x double>, <2 x double>* %B
   %tmp3 = load <2 x double>, <2 x double>* %C
@@ -508,8 +720,13 @@ define <2 x double> @fmls_2d(<2 x double>* %A, <2 x double>* %B, <2 x double>* %
 }
 
 define <2 x float> @fmls_commuted_neg_2s(<2 x float>* %A, <2 x float>* %B, <2 x float>* %C) nounwind {
-;CHECK-LABEL: fmls_commuted_neg_2s:
-;CHECK: fmls.2s
+; CHECK-LABEL: fmls_commuted_neg_2s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d1, [x0]
+; CHECK-NEXT:    ldr d2, [x1]
+; CHECK-NEXT:    ldr d0, [x2]
+; CHECK-NEXT:    fmls.2s v0, v2, v1
+; CHECK-NEXT:    ret
   %tmp1 = load <2 x float>, <2 x float>* %A
   %tmp2 = load <2 x float>, <2 x float>* %B
   %tmp3 = load <2 x float>, <2 x float>* %C
@@ -519,8 +736,13 @@ define <2 x float> @fmls_commuted_neg_2s(<2 x float>* %A, <2 x float>* %B, <2 x
 }
 
 define <4 x float> @fmls_commuted_neg_4s(<4 x float>* %A, <4 x float>* %B, <4 x float>* %C) nounwind {
-;CHECK-LABEL: fmls_commuted_neg_4s:
-;CHECK: fmls.4s
+; CHECK-LABEL: fmls_commuted_neg_4s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q1, [x0]
+; CHECK-NEXT:    ldr q2, [x1]
+; CHECK-NEXT:    ldr q0, [x2]
+; CHECK-NEXT:    fmls.4s v0, v2, v1
+; CHECK-NEXT:    ret
   %tmp1 = load <4 x float>, <4 x float>* %A
   %tmp2 = load <4 x float>, <4 x float>* %B
   %tmp3 = load <4 x float>, <4 x float>* %C
@@ -530,8 +752,13 @@ define <4 x float> @fmls_commuted_neg_4s(<4 x float>* %A, <4 x float>* %B, <4 x
 }
 
 define <2 x double> @fmls_commuted_neg_2d(<2 x double>* %A, <2 x double>* %B, <2 x double>* %C) nounwind {
-;CHECK-LABEL: fmls_commuted_neg_2d:
-;CHECK: fmls.2d
+; CHECK-LABEL: fmls_commuted_neg_2d:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q1, [x0]
+; CHECK-NEXT:    ldr q2, [x1]
+; CHECK-NEXT:    ldr q0, [x2]
+; CHECK-NEXT:    fmls.2d v0, v2, v1
+; CHECK-NEXT:    ret
   %tmp1 = load <2 x double>, <2 x double>* %A
   %tmp2 = load <2 x double>, <2 x double>* %B
   %tmp3 = load <2 x double>, <2 x double>* %C
@@ -541,8 +768,11 @@ define <2 x double> @fmls_commuted_neg_2d(<2 x double>* %A, <2 x double>* %B, <2
 }
 
 define <2 x float> @fmls_indexed_2s(<2 x float> %a, <2 x float> %b, <2 x float> %c) nounwind readnone ssp {
-;CHECK-LABEL: fmls_indexed_2s:
-;CHECK: fmls.2s
+; CHECK-LABEL: fmls_indexed_2s:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT:    fmls.2s v0, v2, v1[0]
+; CHECK-NEXT:    ret
 entry:
   %0 = fsub <2 x float> <float -0.000000e+00, float -0.000000e+00>, %c
   %lane = shufflevector <2 x float> %b, <2 x float> undef, <2 x i32> zeroinitializer
@@ -551,8 +781,10 @@ entry:
 }
 
 define <4 x float> @fmls_indexed_4s(<4 x float> %a, <4 x float> %b, <4 x float> %c) nounwind readnone ssp {
-;CHECK-LABEL: fmls_indexed_4s:
-;CHECK: fmls.4s
+; CHECK-LABEL: fmls_indexed_4s:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmls.4s v0, v2, v1[0]
+; CHECK-NEXT:    ret
 entry:
   %0 = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %c
   %lane = shufflevector <4 x float> %b, <4 x float> undef, <4 x i32> zeroinitializer
@@ -561,8 +793,10 @@ entry:
 }
 
 define <2 x double> @fmls_indexed_2d(<2 x double> %a, <2 x double> %b, <2 x double> %c) nounwind readnone ssp {
-;CHECK-LABEL: fmls_indexed_2d:
-;CHECK: fmls.2d
+; CHECK-LABEL: fmls_indexed_2d:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmls.2d v0, v2, v1[0]
+; CHECK-NEXT:    ret
 entry:
   %0 = fsub <2 x double> <double -0.000000e+00, double -0.000000e+00>, %c
   %lane = shufflevector <2 x double> %b, <2 x double> undef, <2 x i32> zeroinitializer
@@ -571,10 +805,12 @@ entry:
 }
 
 define <2 x float> @fmla_indexed_scalar_2s(<2 x float> %a, <2 x float> %b, float %c) nounwind readnone ssp {
-entry:
 ; CHECK-LABEL: fmla_indexed_scalar_2s:
-; CHECK-NEXT: fmla.2s
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $s2 killed $s2 def $d2
+; CHECK-NEXT:    fmla.2s v0, v1, v2
+; CHECK-NEXT:    ret
+entry:
   %v1 = insertelement <2 x float> undef, float %c, i32 0
   %v2 = insertelement <2 x float> %v1, float %c, i32 1
   %fmla1 = tail call <2 x float> @llvm.fma.v2f32(<2 x float> %v1, <2 x float> %b, <2 x float> %a) nounwind
@@ -582,10 +818,12 @@ entry:
 }
 
 define <4 x float> @fmla_indexed_scalar_4s(<4 x float> %a, <4 x float> %b, float %c) nounwind readnone ssp {
-entry:
 ; CHECK-LABEL: fmla_indexed_scalar_4s:
-; CHECK-NEXT: fmla.4s
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $s2 killed $s2 def $q2
+; CHECK-NEXT:    fmla.4s v0, v1, v2[0]
+; CHECK-NEXT:    ret
+entry:
   %v1 = insertelement <4 x float> undef, float %c, i32 0
   %v2 = insertelement <4 x float> %v1, float %c, i32 1
   %v3 = insertelement <4 x float> %v2, float %c, i32 2
@@ -596,8 +834,10 @@ entry:
 
 define <2 x double> @fmla_indexed_scalar_2d(<2 x double> %a, <2 x double> %b, double %c) nounwind readnone ssp {
 ; CHECK-LABEL: fmla_indexed_scalar_2d:
-; CHECK-NEXT: fmla.2d
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-NEXT:    fmla.2d v0, v1, v2[0]
+; CHECK-NEXT:    ret
 entry:
   %v1 = insertelement <2 x double> undef, double %c, i32 0
   %v2 = insertelement <2 x double> %v1, double %c, i32 1
@@ -606,9 +846,12 @@ entry:
 }
 
 define <4 x i16> @mul_4h(<4 x i16>* %A, <4 x i16>* %B) nounwind {
-;CHECK-LABEL: mul_4h:
-;CHECK-NOT: dup
-;CHECK: mul.4h
+; CHECK-LABEL: mul_4h:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    mul.4h v0, v0, v1[1]
+; CHECK-NEXT:    ret
   %tmp1 = load <4 x i16>, <4 x i16>* %A
   %tmp2 = load <4 x i16>, <4 x i16>* %B
   %tmp3 = shufflevector <4 x i16> %tmp2, <4 x i16> %tmp2, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
@@ -617,9 +860,12 @@ define <4 x i16> @mul_4h(<4 x i16>* %A, <4 x i16>* %B) nounwind {
 }
 
 define <8 x i16> @mul_8h(<8 x i16>* %A, <8 x i16>* %B) nounwind {
-;CHECK-LABEL: mul_8h:
-;CHECK-NOT: dup
-;CHECK: mul.8h
+; CHECK-LABEL: mul_8h:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    ldr q1, [x1]
+; CHECK-NEXT:    mul.8h v0, v0, v1[1]
+; CHECK-NEXT:    ret
   %tmp1 = load <8 x i16>, <8 x i16>* %A
   %tmp2 = load <8 x i16>, <8 x i16>* %B
   %tmp3 = shufflevector <8 x i16> %tmp2, <8 x i16> %tmp2, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
@@ -628,9 +874,12 @@ define <8 x i16> @mul_8h(<8 x i16>* %A, <8 x i16>* %B) nounwind {
 }
 
 define <2 x i32> @mul_2s(<2 x i32>* %A, <2 x i32>* %B) nounwind {
-;CHECK-LABEL: mul_2s:
-;CHECK-NOT: dup
-;CHECK: mul.2s
+; CHECK-LABEL: mul_2s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    mul.2s v0, v0, v1[1]
+; CHECK-NEXT:    ret
   %tmp1 = load <2 x i32>, <2 x i32>* %A
   %tmp2 = load <2 x i32>, <2 x i32>* %B
   %tmp3 = shufflevector <2 x i32> %tmp2, <2 x i32> %tmp2, <2 x i32> <i32 1, i32 1>
@@ -639,9 +888,12 @@ define <2 x i32> @mul_2s(<2 x i32>* %A, <2 x i32>* %B) nounwind {
 }
 
 define <4 x i32> @mul_4s(<4 x i32>* %A, <4 x i32>* %B) nounwind {
-;CHECK-LABEL: mul_4s:
-;CHECK-NOT: dup
-;CHECK: mul.4s
+; CHECK-LABEL: mul_4s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    ldr q1, [x1]
+; CHECK-NEXT:    mul.4s v0, v0, v1[1]
+; CHECK-NEXT:    ret
   %tmp1 = load <4 x i32>, <4 x i32>* %A
   %tmp2 = load <4 x i32>, <4 x i32>* %B
   %tmp3 = shufflevector <4 x i32> %tmp2, <4 x i32> %tmp2, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
@@ -651,16 +903,27 @@ define <4 x i32> @mul_4s(<4 x i32>* %A, <4 x i32>* %B) nounwind {
 
 define <2 x i64> @mul_2d(<2 x i64> %A, <2 x i64> %B) nounwind {
 ; CHECK-LABEL: mul_2d:
-; CHECK: mul
-; CHECK: mul
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmov x10, d1
+; CHECK-NEXT:    fmov x11, d0
+; CHECK-NEXT:    mov.d x8, v1[1]
+; CHECK-NEXT:    mov.d x9, v0[1]
+; CHECK-NEXT:    mul x10, x11, x10
+; CHECK-NEXT:    mul x8, x9, x8
+; CHECK-NEXT:    fmov d0, x10
+; CHECK-NEXT:    mov.d v0[1], x8
+; CHECK-NEXT:    ret
   %tmp1 = mul <2 x i64> %A, %B
   ret <2 x i64> %tmp1
 }
 
 define <2 x float> @fmul_lane_2s(<2 x float>* %A, <2 x float>* %B) nounwind {
-;CHECK-LABEL: fmul_lane_2s:
-;CHECK-NOT: dup
-;CHECK: fmul.2s
+; CHECK-LABEL: fmul_lane_2s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    fmul.2s v0, v0, v1[1]
+; CHECK-NEXT:    ret
   %tmp1 = load <2 x float>, <2 x float>* %A
   %tmp2 = load <2 x float>, <2 x float>* %B
   %tmp3 = shufflevector <2 x float> %tmp2, <2 x float> %tmp2, <2 x i32> <i32 1, i32 1>
@@ -669,9 +932,12 @@ define <2 x float> @fmul_lane_2s(<2 x float>* %A, <2 x float>* %B) nounwind {
 }
 
 define <4 x float> @fmul_lane_4s(<4 x float>* %A, <4 x float>* %B) nounwind {
-;CHECK-LABEL: fmul_lane_4s:
-;CHECK-NOT: dup
-;CHECK: fmul.4s
+; CHECK-LABEL: fmul_lane_4s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    ldr q1, [x1]
+; CHECK-NEXT:    fmul.4s v0, v0, v1[1]
+; CHECK-NEXT:    ret
   %tmp1 = load <4 x float>, <4 x float>* %A
   %tmp2 = load <4 x float>, <4 x float>* %B
   %tmp3 = shufflevector <4 x float> %tmp2, <4 x float> %tmp2, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
@@ -680,9 +946,12 @@ define <4 x float> @fmul_lane_4s(<4 x float>* %A, <4 x float>* %B) nounwind {
 }
 
 define <2 x double> @fmul_lane_2d(<2 x double>* %A, <2 x double>* %B) nounwind {
-;CHECK-LABEL: fmul_lane_2d:
-;CHECK-NOT: dup
-;CHECK: fmul.2d
+; CHECK-LABEL: fmul_lane_2d:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    ldr q1, [x1]
+; CHECK-NEXT:    fmul.2d v0, v0, v1[1]
+; CHECK-NEXT:    ret
   %tmp1 = load <2 x double>, <2 x double>* %A
   %tmp2 = load <2 x double>, <2 x double>* %B
   %tmp3 = shufflevector <2 x double> %tmp2, <2 x double> %tmp2, <2 x i32> <i32 1, i32 1>
@@ -691,18 +960,20 @@ define <2 x double> @fmul_lane_2d(<2 x double>* %A, <2 x double>* %B) nounwind {
 }
 
 define float @fmul_lane_s(float %A, <4 x float> %vec) nounwind {
-;CHECK-LABEL: fmul_lane_s:
-;CHECK-NOT: dup
-;CHECK: fmul.s s0, s0, v1[3]
+; CHECK-LABEL: fmul_lane_s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmul.s s0, s0, v1[3]
+; CHECK-NEXT:    ret
   %B = extractelement <4 x float> %vec, i32 3
   %res = fmul float %A, %B
   ret float %res
 }
 
 define double @fmul_lane_d(double %A, <2 x double> %vec) nounwind {
-;CHECK-LABEL: fmul_lane_d:
-;CHECK-NOT: dup
-;CHECK: fmul.d d0, d0, v1[1]
+; CHECK-LABEL: fmul_lane_d:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmul.d d0, d0, v1[1]
+; CHECK-NEXT:    ret
   %B = extractelement <2 x double> %vec, i32 1
   %res = fmul double %A, %B
   ret double %res
@@ -711,9 +982,12 @@ define double @fmul_lane_d(double %A, <2 x double> %vec) nounwind {
 
 
 define <2 x float> @fmulx_lane_2s(<2 x float>* %A, <2 x float>* %B) nounwind {
-;CHECK-LABEL: fmulx_lane_2s:
-;CHECK-NOT: dup
-;CHECK: fmulx.2s
+; CHECK-LABEL: fmulx_lane_2s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    fmulx.2s v0, v0, v1[1]
+; CHECK-NEXT:    ret
   %tmp1 = load <2 x float>, <2 x float>* %A
   %tmp2 = load <2 x float>, <2 x float>* %B
   %tmp3 = shufflevector <2 x float> %tmp2, <2 x float> %tmp2, <2 x i32> <i32 1, i32 1>
@@ -722,9 +996,12 @@ define <2 x float> @fmulx_lane_2s(<2 x float>* %A, <2 x float>* %B) nounwind {
 }
 
 define <4 x float> @fmulx_lane_4s(<4 x float>* %A, <4 x float>* %B) nounwind {
-;CHECK-LABEL: fmulx_lane_4s:
-;CHECK-NOT: dup
-;CHECK: fmulx.4s
+; CHECK-LABEL: fmulx_lane_4s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    ldr q1, [x1]
+; CHECK-NEXT:    fmulx.4s v0, v0, v1[1]
+; CHECK-NEXT:    ret
   %tmp1 = load <4 x float>, <4 x float>* %A
   %tmp2 = load <4 x float>, <4 x float>* %B
   %tmp3 = shufflevector <4 x float> %tmp2, <4 x float> %tmp2, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
@@ -733,9 +1010,12 @@ define <4 x float> @fmulx_lane_4s(<4 x float>* %A, <4 x float>* %B) nounwind {
 }
 
 define <2 x double> @fmulx_lane_2d(<2 x double>* %A, <2 x double>* %B) nounwind {
-;CHECK-LABEL: fmulx_lane_2d:
-;CHECK-NOT: dup
-;CHECK: fmulx.2d
+; CHECK-LABEL: fmulx_lane_2d:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    ldr q1, [x1]
+; CHECK-NEXT:    fmulx.2d v0, v0, v1[1]
+; CHECK-NEXT:    ret
   %tmp1 = load <2 x double>, <2 x double>* %A
   %tmp2 = load <2 x double>, <2 x double>* %B
   %tmp3 = shufflevector <2 x double> %tmp2, <2 x double> %tmp2, <2 x i32> <i32 1, i32 1>
@@ -744,9 +1024,12 @@ define <2 x double> @fmulx_lane_2d(<2 x double>* %A, <2 x double>* %B) nounwind
 }
 
 define <4 x i16> @sqdmulh_lane_4h(<4 x i16>* %A, <4 x i16>* %B) nounwind {
-;CHECK-LABEL: sqdmulh_lane_4h:
-;CHECK-NOT: dup
-;CHECK: sqdmulh.4h
+; CHECK-LABEL: sqdmulh_lane_4h:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    sqdmulh.4h v0, v0, v1[1]
+; CHECK-NEXT:    ret
   %tmp1 = load <4 x i16>, <4 x i16>* %A
   %tmp2 = load <4 x i16>, <4 x i16>* %B
   %tmp3 = shufflevector <4 x i16> %tmp2, <4 x i16> %tmp2, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
@@ -755,9 +1038,12 @@ define <4 x i16> @sqdmulh_lane_4h(<4 x i16>* %A, <4 x i16>* %B) nounwind {
 }
 
 define <8 x i16> @sqdmulh_lane_8h(<8 x i16>* %A, <8 x i16>* %B) nounwind {
-;CHECK-LABEL: sqdmulh_lane_8h:
-;CHECK-NOT: dup
-;CHECK: sqdmulh.8h
+; CHECK-LABEL: sqdmulh_lane_8h:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    ldr q1, [x1]
+; CHECK-NEXT:    sqdmulh.8h v0, v0, v1[1]
+; CHECK-NEXT:    ret
   %tmp1 = load <8 x i16>, <8 x i16>* %A
   %tmp2 = load <8 x i16>, <8 x i16>* %B
   %tmp3 = shufflevector <8 x i16> %tmp2, <8 x i16> %tmp2, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
@@ -766,9 +1052,12 @@ define <8 x i16> @sqdmulh_lane_8h(<8 x i16>* %A, <8 x i16>* %B) nounwind {
 }
 
 define <2 x i32> @sqdmulh_lane_2s(<2 x i32>* %A, <2 x i32>* %B) nounwind {
-;CHECK-LABEL: sqdmulh_lane_2s:
-;CHECK-NOT: dup
-;CHECK: sqdmulh.2s
+; CHECK-LABEL: sqdmulh_lane_2s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    sqdmulh.2s v0, v0, v1[1]
+; CHECK-NEXT:    ret
   %tmp1 = load <2 x i32>, <2 x i32>* %A
   %tmp2 = load <2 x i32>, <2 x i32>* %B
   %tmp3 = shufflevector <2 x i32> %tmp2, <2 x i32> %tmp2, <2 x i32> <i32 1, i32 1>
@@ -777,9 +1066,12 @@ define <2 x i32> @sqdmulh_lane_2s(<2 x i32>* %A, <2 x i32>* %B) nounwind {
 }
 
 define <4 x i32> @sqdmulh_lane_4s(<4 x i32>* %A, <4 x i32>* %B) nounwind {
-;CHECK-LABEL: sqdmulh_lane_4s:
-;CHECK-NOT: dup
-;CHECK: sqdmulh.4s
+; CHECK-LABEL: sqdmulh_lane_4s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    ldr q1, [x1]
+; CHECK-NEXT:    sqdmulh.4s v0, v0, v1[1]
+; CHECK-NEXT:    ret
   %tmp1 = load <4 x i32>, <4 x i32>* %A
   %tmp2 = load <4 x i32>, <4 x i32>* %B
   %tmp3 = shufflevector <4 x i32> %tmp2, <4 x i32> %tmp2, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
@@ -788,18 +1080,24 @@ define <4 x i32> @sqdmulh_lane_4s(<4 x i32>* %A, <4 x i32>* %B) nounwind {
 }
 
 define i32 @sqdmulh_lane_1s(i32 %A, <4 x i32> %B) nounwind {
-;CHECK-LABEL: sqdmulh_lane_1s:
-;CHECK-NOT: dup
-;CHECK: sqdmulh.s s0, {{s[0-9]+}}, {{v[0-9]+}}[1]
+; CHECK-LABEL: sqdmulh_lane_1s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmov s1, w0
+; CHECK-NEXT:    sqdmulh.s s0, s1, v0[1]
+; CHECK-NEXT:    fmov w0, s0
+; CHECK-NEXT:    ret
   %tmp1 = extractelement <4 x i32> %B, i32 1
   %tmp2 = call i32 @llvm.aarch64.neon.sqdmulh.i32(i32 %A, i32 %tmp1)
   ret i32 %tmp2
 }
 
 define <4 x i16> @sqrdmulh_lane_4h(<4 x i16>* %A, <4 x i16>* %B) nounwind {
-;CHECK-LABEL: sqrdmulh_lane_4h:
-;CHECK-NOT: dup
-;CHECK: sqrdmulh.4h
+; CHECK-LABEL: sqrdmulh_lane_4h:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    sqrdmulh.4h v0, v0, v1[1]
+; CHECK-NEXT:    ret
   %tmp1 = load <4 x i16>, <4 x i16>* %A
   %tmp2 = load <4 x i16>, <4 x i16>* %B
   %tmp3 = shufflevector <4 x i16> %tmp2, <4 x i16> %tmp2, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
@@ -808,9 +1106,12 @@ define <4 x i16> @sqrdmulh_lane_4h(<4 x i16>* %A, <4 x i16>* %B) nounwind {
 }
 
 define <8 x i16> @sqrdmulh_lane_8h(<8 x i16>* %A, <8 x i16>* %B) nounwind {
-;CHECK-LABEL: sqrdmulh_lane_8h:
-;CHECK-NOT: dup
-;CHECK: sqrdmulh.8h
+; CHECK-LABEL: sqrdmulh_lane_8h:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    ldr q1, [x1]
+; CHECK-NEXT:    sqrdmulh.8h v0, v0, v1[1]
+; CHECK-NEXT:    ret
   %tmp1 = load <8 x i16>, <8 x i16>* %A
   %tmp2 = load <8 x i16>, <8 x i16>* %B
   %tmp3 = shufflevector <8 x i16> %tmp2, <8 x i16> %tmp2, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
@@ -819,9 +1120,12 @@ define <8 x i16> @sqrdmulh_lane_8h(<8 x i16>* %A, <8 x i16>* %B) nounwind {
 }
 
 define <2 x i32> @sqrdmulh_lane_2s(<2 x i32>* %A, <2 x i32>* %B) nounwind {
-;CHECK-LABEL: sqrdmulh_lane_2s:
-;CHECK-NOT: dup
-;CHECK: sqrdmulh.2s
+; CHECK-LABEL: sqrdmulh_lane_2s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    sqrdmulh.2s v0, v0, v1[1]
+; CHECK-NEXT:    ret
   %tmp1 = load <2 x i32>, <2 x i32>* %A
   %tmp2 = load <2 x i32>, <2 x i32>* %B
   %tmp3 = shufflevector <2 x i32> %tmp2, <2 x i32> %tmp2, <2 x i32> <i32 1, i32 1>
@@ -830,9 +1134,12 @@ define <2 x i32> @sqrdmulh_lane_2s(<2 x i32>* %A, <2 x i32>* %B) nounwind {
 }
 
 define <4 x i32> @sqrdmulh_lane_4s(<4 x i32>* %A, <4 x i32>* %B) nounwind {
-;CHECK-LABEL: sqrdmulh_lane_4s:
-;CHECK-NOT: dup
-;CHECK: sqrdmulh.4s
+; CHECK-LABEL: sqrdmulh_lane_4s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    ldr q1, [x1]
+; CHECK-NEXT:    sqrdmulh.4s v0, v0, v1[1]
+; CHECK-NEXT:    ret
   %tmp1 = load <4 x i32>, <4 x i32>* %A
   %tmp2 = load <4 x i32>, <4 x i32>* %B
   %tmp3 = shufflevector <4 x i32> %tmp2, <4 x i32> %tmp2, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
@@ -841,18 +1148,24 @@ define <4 x i32> @sqrdmulh_lane_4s(<4 x i32>* %A, <4 x i32>* %B) nounwind {
 }
 
 define i32 @sqrdmulh_lane_1s(i32 %A, <4 x i32> %B) nounwind {
-;CHECK-LABEL: sqrdmulh_lane_1s:
-;CHECK-NOT: dup
-;CHECK: sqrdmulh.s s0, {{s[0-9]+}}, {{v[0-9]+}}[1]
+; CHECK-LABEL: sqrdmulh_lane_1s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmov s1, w0
+; CHECK-NEXT:    sqrdmulh.s s0, s1, v0[1]
+; CHECK-NEXT:    fmov w0, s0
+; CHECK-NEXT:    ret
   %tmp1 = extractelement <4 x i32> %B, i32 1
   %tmp2 = call i32 @llvm.aarch64.neon.sqrdmulh.i32(i32 %A, i32 %tmp1)
   ret i32 %tmp2
 }
 
 define <4 x i32> @sqdmull_lane_4s(<4 x i16>* %A, <4 x i16>* %B) nounwind {
-;CHECK-LABEL: sqdmull_lane_4s:
-;CHECK-NOT: dup
-;CHECK: sqdmull.4s
+; CHECK-LABEL: sqdmull_lane_4s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    sqdmull.4s v0, v0, v1[1]
+; CHECK-NEXT:    ret
   %tmp1 = load <4 x i16>, <4 x i16>* %A
   %tmp2 = load <4 x i16>, <4 x i16>* %B
   %tmp3 = shufflevector <4 x i16> %tmp2, <4 x i16> %tmp2, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
@@ -861,9 +1174,12 @@ define <4 x i32> @sqdmull_lane_4s(<4 x i16>* %A, <4 x i16>* %B) nounwind {
 }
 
 define <2 x i64> @sqdmull_lane_2d(<2 x i32>* %A, <2 x i32>* %B) nounwind {
-;CHECK-LABEL: sqdmull_lane_2d:
-;CHECK-NOT: dup
-;CHECK: sqdmull.2d
+; CHECK-LABEL: sqdmull_lane_2d:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    sqdmull.2d v0, v0, v1[1]
+; CHECK-NEXT:    ret
   %tmp1 = load <2 x i32>, <2 x i32>* %A
   %tmp2 = load <2 x i32>, <2 x i32>* %B
   %tmp3 = shufflevector <2 x i32> %tmp2, <2 x i32> %tmp2, <2 x i32> <i32 1, i32 1>
@@ -872,9 +1188,12 @@ define <2 x i64> @sqdmull_lane_2d(<2 x i32>* %A, <2 x i32>* %B) nounwind {
 }
 
 define <4 x i32> @sqdmull2_lane_4s(<8 x i16>* %A, <8 x i16>* %B) nounwind {
-;CHECK-LABEL: sqdmull2_lane_4s:
-;CHECK-NOT: dup
-;CHECK: sqdmull.4s
+; CHECK-LABEL: sqdmull2_lane_4s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0, #8]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    sqdmull.4s v0, v0, v1[1]
+; CHECK-NEXT:    ret
   %load1 = load <8 x i16>, <8 x i16>* %A
   %load2 = load <8 x i16>, <8 x i16>* %B
   %tmp1 = shufflevector <8 x i16> %load1, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
@@ -884,9 +1203,12 @@ define <4 x i32> @sqdmull2_lane_4s(<8 x i16>* %A, <8 x i16>* %B) nounwind {
 }
 
 define <2 x i64> @sqdmull2_lane_2d(<4 x i32>* %A, <4 x i32>* %B) nounwind {
-;CHECK-LABEL: sqdmull2_lane_2d:
-;CHECK-NOT: dup
-;CHECK: sqdmull.2d
+; CHECK-LABEL: sqdmull2_lane_2d:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0, #8]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    sqdmull.2d v0, v0, v1[1]
+; CHECK-NEXT:    ret
   %load1 = load <4 x i32>, <4 x i32>* %A
   %load2 = load <4 x i32>, <4 x i32>* %B
   %tmp1 = shufflevector <4 x i32> %load1, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
@@ -896,9 +1218,12 @@ define <2 x i64> @sqdmull2_lane_2d(<4 x i32>* %A, <4 x i32>* %B) nounwind {
 }
 
 define <4 x i32> @umull_lane_4s(<4 x i16>* %A, <4 x i16>* %B) nounwind {
-;CHECK-LABEL: umull_lane_4s:
-;CHECK-NOT: dup
-;CHECK: umull.4s
+; CHECK-LABEL: umull_lane_4s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    umull.4s v0, v0, v1[1]
+; CHECK-NEXT:    ret
   %tmp1 = load <4 x i16>, <4 x i16>* %A
   %tmp2 = load <4 x i16>, <4 x i16>* %B
   %tmp3 = shufflevector <4 x i16> %tmp2, <4 x i16> %tmp2, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
@@ -907,9 +1232,12 @@ define <4 x i32> @umull_lane_4s(<4 x i16>* %A, <4 x i16>* %B) nounwind {
 }
 
 define <2 x i64> @umull_lane_2d(<2 x i32>* %A, <2 x i32>* %B) nounwind {
-;CHECK-LABEL: umull_lane_2d:
-;CHECK-NOT: dup
-;CHECK: umull.2d
+; CHECK-LABEL: umull_lane_2d:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    umull.2d v0, v0, v1[1]
+; CHECK-NEXT:    ret
   %tmp1 = load <2 x i32>, <2 x i32>* %A
   %tmp2 = load <2 x i32>, <2 x i32>* %B
   %tmp3 = shufflevector <2 x i32> %tmp2, <2 x i32> %tmp2, <2 x i32> <i32 1, i32 1>
@@ -918,9 +1246,12 @@ define <2 x i64> @umull_lane_2d(<2 x i32>* %A, <2 x i32>* %B) nounwind {
 }
 
 define <4 x i32> @smull_lane_4s(<4 x i16>* %A, <4 x i16>* %B) nounwind {
-;CHECK-LABEL: smull_lane_4s:
-;CHECK-NOT: dup
-;CHECK: smull.4s
+; CHECK-LABEL: smull_lane_4s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    smull.4s v0, v0, v1[1]
+; CHECK-NEXT:    ret
   %tmp1 = load <4 x i16>, <4 x i16>* %A
   %tmp2 = load <4 x i16>, <4 x i16>* %B
   %tmp3 = shufflevector <4 x i16> %tmp2, <4 x i16> %tmp2, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
@@ -929,9 +1260,12 @@ define <4 x i32> @smull_lane_4s(<4 x i16>* %A, <4 x i16>* %B) nounwind {
 }
 
 define <2 x i64> @smull_lane_2d(<2 x i32>* %A, <2 x i32>* %B) nounwind {
-;CHECK-LABEL: smull_lane_2d:
-;CHECK-NOT: dup
-;CHECK: smull.2d
+; CHECK-LABEL: smull_lane_2d:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    smull.2d v0, v0, v1[1]
+; CHECK-NEXT:    ret
   %tmp1 = load <2 x i32>, <2 x i32>* %A
   %tmp2 = load <2 x i32>, <2 x i32>* %B
   %tmp3 = shufflevector <2 x i32> %tmp2, <2 x i32> %tmp2, <2 x i32> <i32 1, i32 1>
@@ -940,9 +1274,13 @@ define <2 x i64> @smull_lane_2d(<2 x i32>* %A, <2 x i32>* %B) nounwind {
 }
 
 define <4 x i32> @smlal_lane_4s(<4 x i16>* %A, <4 x i16>* %B, <4 x i32>* %C) nounwind {
-;CHECK-LABEL: smlal_lane_4s:
-;CHECK-NOT: dup
-;CHECK: smlal.4s
+; CHECK-LABEL: smlal_lane_4s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d1, [x0]
+; CHECK-NEXT:    ldr d2, [x1]
+; CHECK-NEXT:    ldr q0, [x2]
+; CHECK-NEXT:    smlal.4s v0, v1, v2[1]
+; CHECK-NEXT:    ret
   %tmp1 = load <4 x i16>, <4 x i16>* %A
   %tmp2 = load <4 x i16>, <4 x i16>* %B
   %tmp3 = load <4 x i32>, <4 x i32>* %C
@@ -953,9 +1291,13 @@ define <4 x i32> @smlal_lane_4s(<4 x i16>* %A, <4 x i16>* %B, <4 x i32>* %C) nou
 }
 
 define <2 x i64> @smlal_lane_2d(<2 x i32>* %A, <2 x i32>* %B, <2 x i64>* %C) nounwind {
-;CHECK-LABEL: smlal_lane_2d:
-;CHECK-NOT: dup
-;CHECK: smlal.2d
+; CHECK-LABEL: smlal_lane_2d:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d1, [x0]
+; CHECK-NEXT:    ldr d2, [x1]
+; CHECK-NEXT:    ldr q0, [x2]
+; CHECK-NEXT:    smlal.2d v0, v1, v2[1]
+; CHECK-NEXT:    ret
   %tmp1 = load <2 x i32>, <2 x i32>* %A
   %tmp2 = load <2 x i32>, <2 x i32>* %B
   %tmp3 = load <2 x i64>, <2 x i64>* %C
@@ -966,9 +1308,13 @@ define <2 x i64> @smlal_lane_2d(<2 x i32>* %A, <2 x i32>* %B, <2 x i64>* %C) nou
 }
 
 define <4 x i32> @sqdmlal_lane_4s(<4 x i16>* %A, <4 x i16>* %B, <4 x i32>* %C) nounwind {
-;CHECK-LABEL: sqdmlal_lane_4s:
-;CHECK-NOT: dup
-;CHECK: sqdmlal.4s
+; CHECK-LABEL: sqdmlal_lane_4s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d1, [x0]
+; CHECK-NEXT:    ldr d2, [x1]
+; CHECK-NEXT:    ldr q0, [x2]
+; CHECK-NEXT:    sqdmlal.4s v0, v1, v2[1]
+; CHECK-NEXT:    ret
   %tmp1 = load <4 x i16>, <4 x i16>* %A
   %tmp2 = load <4 x i16>, <4 x i16>* %B
   %tmp3 = load <4 x i32>, <4 x i32>* %C
@@ -979,9 +1325,13 @@ define <4 x i32> @sqdmlal_lane_4s(<4 x i16>* %A, <4 x i16>* %B, <4 x i32>* %C) n
 }
 
 define <2 x i64> @sqdmlal_lane_2d(<2 x i32>* %A, <2 x i32>* %B, <2 x i64>* %C) nounwind {
-;CHECK-LABEL: sqdmlal_lane_2d:
-;CHECK-NOT: dup
-;CHECK: sqdmlal.2d
+; CHECK-LABEL: sqdmlal_lane_2d:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d1, [x0]
+; CHECK-NEXT:    ldr d2, [x1]
+; CHECK-NEXT:    ldr q0, [x2]
+; CHECK-NEXT:    sqdmlal.2d v0, v1, v2[1]
+; CHECK-NEXT:    ret
   %tmp1 = load <2 x i32>, <2 x i32>* %A
   %tmp2 = load <2 x i32>, <2 x i32>* %B
   %tmp3 = load <2 x i64>, <2 x i64>* %C
@@ -992,9 +1342,13 @@ define <2 x i64> @sqdmlal_lane_2d(<2 x i32>* %A, <2 x i32>* %B, <2 x i64>* %C) n
 }
 
 define <4 x i32> @sqdmlal2_lane_4s(<8 x i16>* %A, <8 x i16>* %B, <4 x i32>* %C) nounwind {
-;CHECK-LABEL: sqdmlal2_lane_4s:
-;CHECK-NOT: dup
-;CHECK: sqdmlal.4s
+; CHECK-LABEL: sqdmlal2_lane_4s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x2]
+; CHECK-NEXT:    ldr d1, [x0, #8]
+; CHECK-NEXT:    ldr d2, [x1]
+; CHECK-NEXT:    sqdmlal.4s v0, v1, v2[1]
+; CHECK-NEXT:    ret
   %load1 = load <8 x i16>, <8 x i16>* %A
   %load2 = load <8 x i16>, <8 x i16>* %B
   %tmp3 = load <4 x i32>, <4 x i32>* %C
@@ -1006,9 +1360,13 @@ define <4 x i32> @sqdmlal2_lane_4s(<8 x i16>* %A, <8 x i16>* %B, <4 x i32>* %C)
 }
 
 define <2 x i64> @sqdmlal2_lane_2d(<4 x i32>* %A, <4 x i32>* %B, <2 x i64>* %C) nounwind {
-;CHECK-LABEL: sqdmlal2_lane_2d:
-;CHECK-NOT: dup
-;CHECK: sqdmlal.2d
+; CHECK-LABEL: sqdmlal2_lane_2d:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x2]
+; CHECK-NEXT:    ldr d1, [x0, #8]
+; CHECK-NEXT:    ldr d2, [x1]
+; CHECK-NEXT:    sqdmlal.2d v0, v1, v2[1]
+; CHECK-NEXT:    ret
   %load1 = load <4 x i32>, <4 x i32>* %A
   %load2 = load <4 x i32>, <4 x i32>* %B
   %tmp3 = load <2 x i64>, <2 x i64>* %C
@@ -1020,8 +1378,14 @@ define <2 x i64> @sqdmlal2_lane_2d(<4 x i32>* %A, <4 x i32>* %B, <2 x i64>* %C)
 }
 
 define i32 @sqdmlal_lane_1s(i32 %A, i16 %B, <4 x i16> %C) nounwind {
-;CHECK-LABEL: sqdmlal_lane_1s:
-;CHECK: sqdmlal.4s
+; CHECK-LABEL: sqdmlal_lane_1s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmov s1, w1
+; CHECK-NEXT:    fmov s2, w0
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    sqdmlal.4s v2, v1, v0[1]
+; CHECK-NEXT:    fmov w0, s2
+; CHECK-NEXT:    ret
   %lhs = insertelement <4 x i16> undef, i16 %B, i32 0
   %rhs = shufflevector <4 x i16> %C, <4 x i16> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
   %prod.vec = call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %lhs, <4 x i16> %rhs)
@@ -1032,8 +1396,14 @@ define i32 @sqdmlal_lane_1s(i32 %A, i16 %B, <4 x i16> %C) nounwind {
 declare i32 @llvm.aarch64.neon.sqadd.i32(i32, i32)
 
 define i32 @sqdmlsl_lane_1s(i32 %A, i16 %B, <4 x i16> %C) nounwind {
-;CHECK-LABEL: sqdmlsl_lane_1s:
-;CHECK: sqdmlsl.4s
+; CHECK-LABEL: sqdmlsl_lane_1s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmov s1, w1
+; CHECK-NEXT:    fmov s2, w0
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    sqdmlsl.4s v2, v1, v0[1]
+; CHECK-NEXT:    fmov w0, s2
+; CHECK-NEXT:    ret
   %lhs = insertelement <4 x i16> undef, i16 %B, i32 0
   %rhs = shufflevector <4 x i16> %C, <4 x i16> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
   %prod.vec = call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %lhs, <4 x i16> %rhs)
@@ -1044,8 +1414,14 @@ define i32 @sqdmlsl_lane_1s(i32 %A, i16 %B, <4 x i16> %C) nounwind {
 declare i32 @llvm.aarch64.neon.sqsub.i32(i32, i32)
 
 define i64 @sqdmlal_lane_1d(i64 %A, i32 %B, <2 x i32> %C) nounwind {
-;CHECK-LABEL: sqdmlal_lane_1d:
-;CHECK: sqdmlal.s
+; CHECK-LABEL: sqdmlal_lane_1d:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmov d1, x0
+; CHECK-NEXT:    fmov s2, w1
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    sqdmlal.s d1, s2, v0[1]
+; CHECK-NEXT:    fmov x0, d1
+; CHECK-NEXT:    ret
   %rhs = extractelement <2 x i32> %C, i32 1
   %prod = call i64 @llvm.aarch64.neon.sqdmulls.scalar(i32 %B, i32 %rhs)
   %res = call i64 @llvm.aarch64.neon.sqadd.i64(i64 %A, i64 %prod)
@@ -1055,8 +1431,14 @@ declare i64 @llvm.aarch64.neon.sqdmulls.scalar(i32, i32)
 declare i64 @llvm.aarch64.neon.sqadd.i64(i64, i64)
 
 define i64 @sqdmlsl_lane_1d(i64 %A, i32 %B, <2 x i32> %C) nounwind {
-;CHECK-LABEL: sqdmlsl_lane_1d:
-;CHECK: sqdmlsl.s
+; CHECK-LABEL: sqdmlsl_lane_1d:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmov d1, x0
+; CHECK-NEXT:    fmov s2, w1
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    sqdmlsl.s d1, s2, v0[1]
+; CHECK-NEXT:    fmov x0, d1
+; CHECK-NEXT:    ret
   %rhs = extractelement <2 x i32> %C, i32 1
   %prod = call i64 @llvm.aarch64.neon.sqdmulls.scalar(i32 %B, i32 %rhs)
   %res = call i64 @llvm.aarch64.neon.sqsub.i64(i64 %A, i64 %prod)
@@ -1066,9 +1448,13 @@ declare i64 @llvm.aarch64.neon.sqsub.i64(i64, i64)
 
 
 define <4 x i32> @umlal_lane_4s(<4 x i16>* %A, <4 x i16>* %B, <4 x i32>* %C) nounwind {
-;CHECK-LABEL: umlal_lane_4s:
-;CHECK-NOT: dup
-;CHECK: umlal.4s
+; CHECK-LABEL: umlal_lane_4s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d1, [x0]
+; CHECK-NEXT:    ldr d2, [x1]
+; CHECK-NEXT:    ldr q0, [x2]
+; CHECK-NEXT:    umlal.4s v0, v1, v2[1]
+; CHECK-NEXT:    ret
   %tmp1 = load <4 x i16>, <4 x i16>* %A
   %tmp2 = load <4 x i16>, <4 x i16>* %B
   %tmp3 = load <4 x i32>, <4 x i32>* %C
@@ -1079,9 +1465,13 @@ define <4 x i32> @umlal_lane_4s(<4 x i16>* %A, <4 x i16>* %B, <4 x i32>* %C) nou
 }
 
 define <2 x i64> @umlal_lane_2d(<2 x i32>* %A, <2 x i32>* %B, <2 x i64>* %C) nounwind {
-;CHECK-LABEL: umlal_lane_2d:
-;CHECK-NOT: dup
-;CHECK: umlal.2d
+; CHECK-LABEL: umlal_lane_2d:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d1, [x0]
+; CHECK-NEXT:    ldr d2, [x1]
+; CHECK-NEXT:    ldr q0, [x2]
+; CHECK-NEXT:    umlal.2d v0, v1, v2[1]
+; CHECK-NEXT:    ret
   %tmp1 = load <2 x i32>, <2 x i32>* %A
   %tmp2 = load <2 x i32>, <2 x i32>* %B
   %tmp3 = load <2 x i64>, <2 x i64>* %C
@@ -1093,9 +1483,13 @@ define <2 x i64> @umlal_lane_2d(<2 x i32>* %A, <2 x i32>* %B, <2 x i64>* %C) nou
 
 
 define <4 x i32> @smlsl_lane_4s(<4 x i16>* %A, <4 x i16>* %B, <4 x i32>* %C) nounwind {
-;CHECK-LABEL: smlsl_lane_4s:
-;CHECK-NOT: dup
-;CHECK: smlsl.4s
+; CHECK-LABEL: smlsl_lane_4s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d1, [x0]
+; CHECK-NEXT:    ldr d2, [x1]
+; CHECK-NEXT:    ldr q0, [x2]
+; CHECK-NEXT:    smlsl.4s v0, v1, v2[1]
+; CHECK-NEXT:    ret
   %tmp1 = load <4 x i16>, <4 x i16>* %A
   %tmp2 = load <4 x i16>, <4 x i16>* %B
   %tmp3 = load <4 x i32>, <4 x i32>* %C
@@ -1106,9 +1500,13 @@ define <4 x i32> @smlsl_lane_4s(<4 x i16>* %A, <4 x i16>* %B, <4 x i32>* %C) nou
 }
 
 define <2 x i64> @smlsl_lane_2d(<2 x i32>* %A, <2 x i32>* %B, <2 x i64>* %C) nounwind {
-;CHECK-LABEL: smlsl_lane_2d:
-;CHECK-NOT: dup
-;CHECK: smlsl.2d
+; CHECK-LABEL: smlsl_lane_2d:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d1, [x0]
+; CHECK-NEXT:    ldr d2, [x1]
+; CHECK-NEXT:    ldr q0, [x2]
+; CHECK-NEXT:    smlsl.2d v0, v1, v2[1]
+; CHECK-NEXT:    ret
   %tmp1 = load <2 x i32>, <2 x i32>* %A
   %tmp2 = load <2 x i32>, <2 x i32>* %B
   %tmp3 = load <2 x i64>, <2 x i64>* %C
@@ -1119,9 +1517,13 @@ define <2 x i64> @smlsl_lane_2d(<2 x i32>* %A, <2 x i32>* %B, <2 x i64>* %C) nou
 }
 
 define <4 x i32> @sqdmlsl_lane_4s(<4 x i16>* %A, <4 x i16>* %B, <4 x i32>* %C) nounwind {
-;CHECK-LABEL: sqdmlsl_lane_4s:
-;CHECK-NOT: dup
-;CHECK: sqdmlsl.4s
+; CHECK-LABEL: sqdmlsl_lane_4s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d1, [x0]
+; CHECK-NEXT:    ldr d2, [x1]
+; CHECK-NEXT:    ldr q0, [x2]
+; CHECK-NEXT:    sqdmlsl.4s v0, v1, v2[1]
+; CHECK-NEXT:    ret
   %tmp1 = load <4 x i16>, <4 x i16>* %A
   %tmp2 = load <4 x i16>, <4 x i16>* %B
   %tmp3 = load <4 x i32>, <4 x i32>* %C
@@ -1132,9 +1534,13 @@ define <4 x i32> @sqdmlsl_lane_4s(<4 x i16>* %A, <4 x i16>* %B, <4 x i32>* %C) n
 }
 
 define <2 x i64> @sqdmlsl_lane_2d(<2 x i32>* %A, <2 x i32>* %B, <2 x i64>* %C) nounwind {
-;CHECK-LABEL: sqdmlsl_lane_2d:
-;CHECK-NOT: dup
-;CHECK: sqdmlsl.2d
+; CHECK-LABEL: sqdmlsl_lane_2d:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d1, [x0]
+; CHECK-NEXT:    ldr d2, [x1]
+; CHECK-NEXT:    ldr q0, [x2]
+; CHECK-NEXT:    sqdmlsl.2d v0, v1, v2[1]
+; CHECK-NEXT:    ret
   %tmp1 = load <2 x i32>, <2 x i32>* %A
   %tmp2 = load <2 x i32>, <2 x i32>* %B
   %tmp3 = load <2 x i64>, <2 x i64>* %C
@@ -1145,9 +1551,13 @@ define <2 x i64> @sqdmlsl_lane_2d(<2 x i32>* %A, <2 x i32>* %B, <2 x i64>* %C) n
 }
 
 define <4 x i32> @sqdmlsl2_lane_4s(<8 x i16>* %A, <8 x i16>* %B, <4 x i32>* %C) nounwind {
-;CHECK-LABEL: sqdmlsl2_lane_4s:
-;CHECK-NOT: dup
-;CHECK: sqdmlsl.4s
+; CHECK-LABEL: sqdmlsl2_lane_4s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x2]
+; CHECK-NEXT:    ldr d1, [x0, #8]
+; CHECK-NEXT:    ldr d2, [x1]
+; CHECK-NEXT:    sqdmlsl.4s v0, v1, v2[1]
+; CHECK-NEXT:    ret
   %load1 = load <8 x i16>, <8 x i16>* %A
   %load2 = load <8 x i16>, <8 x i16>* %B
   %tmp3 = load <4 x i32>, <4 x i32>* %C
@@ -1159,9 +1569,13 @@ define <4 x i32> @sqdmlsl2_lane_4s(<8 x i16>* %A, <8 x i16>* %B, <4 x i32>* %C)
 }
 
 define <2 x i64> @sqdmlsl2_lane_2d(<4 x i32>* %A, <4 x i32>* %B, <2 x i64>* %C) nounwind {
-;CHECK-LABEL: sqdmlsl2_lane_2d:
-;CHECK-NOT: dup
-;CHECK: sqdmlsl.2d
+; CHECK-LABEL: sqdmlsl2_lane_2d:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x2]
+; CHECK-NEXT:    ldr d1, [x0, #8]
+; CHECK-NEXT:    ldr d2, [x1]
+; CHECK-NEXT:    sqdmlsl.2d v0, v1, v2[1]
+; CHECK-NEXT:    ret
   %load1 = load <4 x i32>, <4 x i32>* %A
   %load2 = load <4 x i32>, <4 x i32>* %B
   %tmp3 = load <2 x i64>, <2 x i64>* %C
@@ -1173,9 +1587,13 @@ define <2 x i64> @sqdmlsl2_lane_2d(<4 x i32>* %A, <4 x i32>* %B, <2 x i64>* %C)
 }
 
 define <4 x i32> @umlsl_lane_4s(<4 x i16>* %A, <4 x i16>* %B, <4 x i32>* %C) nounwind {
-;CHECK-LABEL: umlsl_lane_4s:
-;CHECK-NOT: dup
-;CHECK: umlsl.4s
+; CHECK-LABEL: umlsl_lane_4s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d1, [x0]
+; CHECK-NEXT:    ldr d2, [x1]
+; CHECK-NEXT:    ldr q0, [x2]
+; CHECK-NEXT:    umlsl.4s v0, v1, v2[1]
+; CHECK-NEXT:    ret
   %tmp1 = load <4 x i16>, <4 x i16>* %A
   %tmp2 = load <4 x i16>, <4 x i16>* %B
   %tmp3 = load <4 x i32>, <4 x i32>* %C
@@ -1186,9 +1604,13 @@ define <4 x i32> @umlsl_lane_4s(<4 x i16>* %A, <4 x i16>* %B, <4 x i32>* %C) nou
 }
 
 define <2 x i64> @umlsl_lane_2d(<2 x i32>* %A, <2 x i32>* %B, <2 x i64>* %C) nounwind {
-;CHECK-LABEL: umlsl_lane_2d:
-;CHECK-NOT: dup
-;CHECK: umlsl.2d
+; CHECK-LABEL: umlsl_lane_2d:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d1, [x0]
+; CHECK-NEXT:    ldr d2, [x1]
+; CHECK-NEXT:    ldr q0, [x2]
+; CHECK-NEXT:    umlsl.2d v0, v1, v2[1]
+; CHECK-NEXT:    ret
   %tmp1 = load <2 x i32>, <2 x i32>* %A
   %tmp2 = load <2 x i32>, <2 x i32>* %B
   %tmp3 = load <2 x i64>, <2 x i64>* %C
@@ -1201,35 +1623,39 @@ define <2 x i64> @umlsl_lane_2d(<2 x i32>* %A, <2 x i32>* %B, <2 x i64>* %C) nou
 ; Scalar FMULX
 define float @fmulxs(float %a, float %b) nounwind {
 ; CHECK-LABEL: fmulxs:
-; CHECK-NEXT: fmulx s0, s0, s1
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmulx s0, s0, s1
+; CHECK-NEXT:    ret
   %fmulx.i = tail call float @llvm.aarch64.neon.fmulx.f32(float %a, float %b) nounwind
-; CHECK-NEXT: ret
   ret float %fmulx.i
 }
 
 define double @fmulxd(double %a, double %b) nounwind {
 ; CHECK-LABEL: fmulxd:
-; CHECK-NEXT: fmulx d0, d0, d1
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmulx d0, d0, d1
+; CHECK-NEXT:    ret
   %fmulx.i = tail call double @llvm.aarch64.neon.fmulx.f64(double %a, double %b) nounwind
-; CHECK-NEXT: ret
   ret double %fmulx.i
 }
 
 define float @fmulxs_lane(float %a, <4 x float> %vec) nounwind {
 ; CHECK-LABEL: fmulxs_lane:
-; CHECK-NEXT: fmulx.s s0, s0, v1[3]
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmulx.s s0, s0, v1[3]
+; CHECK-NEXT:    ret
   %b = extractelement <4 x float> %vec, i32 3
   %fmulx.i = tail call float @llvm.aarch64.neon.fmulx.f32(float %a, float %b) nounwind
-; CHECK-NEXT: ret
   ret float %fmulx.i
 }
 
 define double @fmulxd_lane(double %a, <2 x double> %vec) nounwind {
 ; CHECK-LABEL: fmulxd_lane:
-; CHECK-NEXT: fmulx.d d0, d0, v1[1]
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmulx.d d0, d0, v1[1]
+; CHECK-NEXT:    ret
   %b = extractelement <2 x double> %vec, i32 1
   %fmulx.i = tail call double @llvm.aarch64.neon.fmulx.f64(double %a, double %b) nounwind
-; CHECK-NEXT: ret
   ret double %fmulx.i
 }
 
@@ -1239,8 +1665,9 @@ declare float @llvm.aarch64.neon.fmulx.f32(float, float) nounwind readnone
 
 define <8 x i16> @smull2_8h_simple(<16 x i8> %a, <16 x i8> %b) nounwind {
 ; CHECK-LABEL: smull2_8h_simple:
-; CHECK-NEXT: smull2.8h v0, v0, v1
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    smull2.8h v0, v0, v1
+; CHECK-NEXT:    ret
   %1 = shufflevector <16 x i8> %a, <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
   %2 = shufflevector <16 x i8> %b, <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
   %3 = tail call <8 x i16> @llvm.aarch64.neon.smull.v8i16(<8 x i8> %1, <8 x i8> %2) #2
@@ -1249,7 +1676,9 @@ define <8 x i16> @smull2_8h_simple(<16 x i8> %a, <16 x i8> %b) nounwind {
 
 define <8 x i16> @foo0(<16 x i8> %a, <16 x i8> %b) nounwind {
 ; CHECK-LABEL: foo0:
-; CHECK: smull2.8h v0, v0, v1
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    smull2.8h v0, v0, v1
+; CHECK-NEXT:    ret
   %tmp = bitcast <16 x i8> %a to <2 x i64>
   %shuffle.i.i = shufflevector <2 x i64> %tmp, <2 x i64> undef, <1 x i32> <i32 1>
   %tmp1 = bitcast <1 x i64> %shuffle.i.i to <8 x i8>
@@ -1262,7 +1691,9 @@ define <8 x i16> @foo0(<16 x i8> %a, <16 x i8> %b) nounwind {
 
 define <4 x i32> @foo1(<8 x i16> %a, <8 x i16> %b) nounwind {
 ; CHECK-LABEL: foo1:
-; CHECK: smull2.4s v0, v0, v1
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    smull2.4s v0, v0, v1
+; CHECK-NEXT:    ret
   %tmp = bitcast <8 x i16> %a to <2 x i64>
   %shuffle.i.i = shufflevector <2 x i64> %tmp, <2 x i64> undef, <1 x i32> <i32 1>
   %tmp1 = bitcast <1 x i64> %shuffle.i.i to <4 x i16>
@@ -1275,7 +1706,9 @@ define <4 x i32> @foo1(<8 x i16> %a, <8 x i16> %b) nounwind {
 
 define <2 x i64> @foo2(<4 x i32> %a, <4 x i32> %b) nounwind {
 ; CHECK-LABEL: foo2:
-; CHECK: smull2.2d v0, v0, v1
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    smull2.2d v0, v0, v1
+; CHECK-NEXT:    ret
   %tmp = bitcast <4 x i32> %a to <2 x i64>
   %shuffle.i.i = shufflevector <2 x i64> %tmp, <2 x i64> undef, <1 x i32> <i32 1>
   %tmp1 = bitcast <1 x i64> %shuffle.i.i to <2 x i32>
@@ -1288,7 +1721,9 @@ define <2 x i64> @foo2(<4 x i32> %a, <4 x i32> %b) nounwind {
 
 define <8 x i16> @foo3(<16 x i8> %a, <16 x i8> %b) nounwind {
 ; CHECK-LABEL: foo3:
-; CHECK: umull2.8h v0, v0, v1
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    umull2.8h v0, v0, v1
+; CHECK-NEXT:    ret
   %tmp = bitcast <16 x i8> %a to <2 x i64>
   %shuffle.i.i = shufflevector <2 x i64> %tmp, <2 x i64> undef, <1 x i32> <i32 1>
   %tmp1 = bitcast <1 x i64> %shuffle.i.i to <8 x i8>
@@ -1301,7 +1736,9 @@ define <8 x i16> @foo3(<16 x i8> %a, <16 x i8> %b) nounwind {
 
 define <4 x i32> @foo4(<8 x i16> %a, <8 x i16> %b) nounwind {
 ; CHECK-LABEL: foo4:
-; CHECK: umull2.4s v0, v0, v1
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    umull2.4s v0, v0, v1
+; CHECK-NEXT:    ret
   %tmp = bitcast <8 x i16> %a to <2 x i64>
   %shuffle.i.i = shufflevector <2 x i64> %tmp, <2 x i64> undef, <1 x i32> <i32 1>
   %tmp1 = bitcast <1 x i64> %shuffle.i.i to <4 x i16>
@@ -1314,7 +1751,9 @@ define <4 x i32> @foo4(<8 x i16> %a, <8 x i16> %b) nounwind {
 
 define <2 x i64> @foo5(<4 x i32> %a, <4 x i32> %b) nounwind {
 ; CHECK-LABEL: foo5:
-; CHECK: umull2.2d v0, v0, v1
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    umull2.2d v0, v0, v1
+; CHECK-NEXT:    ret
   %tmp = bitcast <4 x i32> %a to <2 x i64>
   %shuffle.i.i = shufflevector <2 x i64> %tmp, <2 x i64> undef, <1 x i32> <i32 1>
   %tmp1 = bitcast <1 x i64> %shuffle.i.i to <2 x i32>
@@ -1327,8 +1766,10 @@ define <2 x i64> @foo5(<4 x i32> %a, <4 x i32> %b) nounwind {
 
 define <4 x i32> @foo6(<4 x i32> %a, <8 x i16> %b, <4 x i16> %c) nounwind readnone optsize ssp {
 ; CHECK-LABEL: foo6:
-; CHECK-NEXT: smull2.4s v0, v1, v2[1]
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-NEXT:    smull2.4s v0, v1, v2[1]
+; CHECK-NEXT:    ret
 entry:
   %0 = bitcast <8 x i16> %b to <2 x i64>
   %shuffle.i = shufflevector <2 x i64> %0, <2 x i64> undef, <1 x i32> <i32 1>
@@ -1340,8 +1781,10 @@ entry:
 
 define <4 x i32> @foo6a(<4 x i32> %a, <8 x i16> %b, <4 x i16> %c) nounwind readnone optsize ssp {
 ; CHECK-LABEL: foo6a:
-; CHECK-NEXT: smull.4s v0, v1, v2[1]
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-NEXT:    smull.4s v0, v1, v2[1]
+; CHECK-NEXT:    ret
 entry:
   %0 = bitcast <8 x i16> %b to <2 x i64>
   %shuffle.i = shufflevector <2 x i64> %0, <2 x i64> undef, <1 x i32> <i32 0>
@@ -1353,8 +1796,10 @@ entry:
 
 define <2 x i64> @foo7(<2 x i64> %a, <4 x i32> %b, <2 x i32> %c) nounwind readnone optsize ssp {
 ; CHECK-LABEL: foo7:
-; CHECK-NEXT: smull2.2d v0, v1, v2[1]
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-NEXT:    smull2.2d v0, v1, v2[1]
+; CHECK-NEXT:    ret
 entry:
   %0 = bitcast <4 x i32> %b to <2 x i64>
   %shuffle.i = shufflevector <2 x i64> %0, <2 x i64> undef, <1 x i32> <i32 1>
@@ -1366,8 +1811,10 @@ entry:
 
 define <2 x i64> @foo7a(<2 x i64> %a, <4 x i32> %b, <2 x i32> %c) nounwind readnone optsize ssp {
 ; CHECK-LABEL: foo7a:
-; CHECK-NEXT: smull.2d v0, v1, v2[1]
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-NEXT:    smull.2d v0, v1, v2[1]
+; CHECK-NEXT:    ret
 entry:
   %0 = bitcast <4 x i32> %b to <2 x i64>
   %shuffle.i = shufflevector <2 x i64> %0, <2 x i64> undef, <1 x i32> <i32 0>
@@ -1380,8 +1827,10 @@ entry:
 
 define <4 x i32> @foo8(<4 x i32> %a, <8 x i16> %b, <4 x i16> %c) nounwind readnone optsize ssp {
 ; CHECK-LABEL: foo8:
-; CHECK-NEXT: umull2.4s v0, v1, v2[1]
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-NEXT:    umull2.4s v0, v1, v2[1]
+; CHECK-NEXT:    ret
 entry:
   %0 = bitcast <8 x i16> %b to <2 x i64>
   %shuffle.i = shufflevector <2 x i64> %0, <2 x i64> undef, <1 x i32> <i32 1>
@@ -1393,8 +1842,10 @@ entry:
 
 define <4 x i32> @foo8a(<4 x i32> %a, <8 x i16> %b, <4 x i16> %c) nounwind readnone optsize ssp {
 ; CHECK-LABEL: foo8a:
-; CHECK-NEXT: umull.4s v0, v1, v2[1]
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-NEXT:    umull.4s v0, v1, v2[1]
+; CHECK-NEXT:    ret
 entry:
   %0 = bitcast <8 x i16> %b to <2 x i64>
   %shuffle.i = shufflevector <2 x i64> %0, <2 x i64> undef, <1 x i32> <i32 0>
@@ -1406,8 +1857,10 @@ entry:
 
 define <2 x i64> @foo9(<2 x i64> %a, <4 x i32> %b, <2 x i32> %c) nounwind readnone optsize ssp {
 ; CHECK-LABEL: foo9:
-; CHECK-NEXT: umull2.2d v0, v1, v2[1]
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-NEXT:    umull2.2d v0, v1, v2[1]
+; CHECK-NEXT:    ret
 entry:
   %0 = bitcast <4 x i32> %b to <2 x i64>
   %shuffle.i = shufflevector <2 x i64> %0, <2 x i64> undef, <1 x i32> <i32 1>
@@ -1419,8 +1872,10 @@ entry:
 
 define <2 x i64> @foo9a(<2 x i64> %a, <4 x i32> %b, <2 x i32> %c) nounwind readnone optsize ssp {
 ; CHECK-LABEL: foo9a:
-; CHECK-NEXT: umull.2d v0, v1, v2[1]
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-NEXT:    umull.2d v0, v1, v2[1]
+; CHECK-NEXT:    ret
 entry:
   %0 = bitcast <4 x i32> %b to <2 x i64>
   %shuffle.i = shufflevector <2 x i64> %0, <2 x i64> undef, <1 x i32> <i32 0>
@@ -1432,9 +1887,9 @@ entry:
 
 define <8 x i16> @bar0(<8 x i16> %a, <16 x i8> %b, <16 x i8> %c) nounwind {
 ; CHECK-LABEL: bar0:
-; CHECK: smlal2.8h v0, v1, v2
-; CHECK-NEXT: ret
-
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    smlal2.8h v0, v1, v2
+; CHECK-NEXT:    ret
   %tmp = bitcast <16 x i8> %b to <2 x i64>
   %shuffle.i.i.i = shufflevector <2 x i64> %tmp, <2 x i64> undef, <1 x i32> <i32 1>
   %tmp1 = bitcast <1 x i64> %shuffle.i.i.i to <8 x i8>
@@ -1448,9 +1903,9 @@ define <8 x i16> @bar0(<8 x i16> %a, <16 x i8> %b, <16 x i8> %c) nounwind {
 
 define <4 x i32> @bar1(<4 x i32> %a, <8 x i16> %b, <8 x i16> %c) nounwind {
 ; CHECK-LABEL: bar1:
-; CHECK: smlal2.4s v0, v1, v2
-; CHECK-NEXT: ret
-
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    smlal2.4s v0, v1, v2
+; CHECK-NEXT:    ret
   %tmp = bitcast <8 x i16> %b to <2 x i64>
   %shuffle.i.i.i = shufflevector <2 x i64> %tmp, <2 x i64> undef, <1 x i32> <i32 1>
   %tmp1 = bitcast <1 x i64> %shuffle.i.i.i to <4 x i16>
@@ -1464,9 +1919,9 @@ define <4 x i32> @bar1(<4 x i32> %a, <8 x i16> %b, <8 x i16> %c) nounwind {
 
 define <2 x i64> @bar2(<2 x i64> %a, <4 x i32> %b, <4 x i32> %c) nounwind {
 ; CHECK-LABEL: bar2:
-; CHECK: smlal2.2d v0, v1, v2
-; CHECK-NEXT: ret
-
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    smlal2.2d v0, v1, v2
+; CHECK-NEXT:    ret
   %tmp = bitcast <4 x i32> %b to <2 x i64>
   %shuffle.i.i.i = shufflevector <2 x i64> %tmp, <2 x i64> undef, <1 x i32> <i32 1>
   %tmp1 = bitcast <1 x i64> %shuffle.i.i.i to <2 x i32>
@@ -1480,9 +1935,9 @@ define <2 x i64> @bar2(<2 x i64> %a, <4 x i32> %b, <4 x i32> %c) nounwind {
 
 define <8 x i16> @bar3(<8 x i16> %a, <16 x i8> %b, <16 x i8> %c) nounwind {
 ; CHECK-LABEL: bar3:
-; CHECK: umlal2.8h v0, v1, v2
-; CHECK-NEXT: ret
-
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    umlal2.8h v0, v1, v2
+; CHECK-NEXT:    ret
   %tmp = bitcast <16 x i8> %b to <2 x i64>
   %shuffle.i.i.i = shufflevector <2 x i64> %tmp, <2 x i64> undef, <1 x i32> <i32 1>
   %tmp1 = bitcast <1 x i64> %shuffle.i.i.i to <8 x i8>
@@ -1496,9 +1951,9 @@ define <8 x i16> @bar3(<8 x i16> %a, <16 x i8> %b, <16 x i8> %c) nounwind {
 
 define <4 x i32> @bar4(<4 x i32> %a, <8 x i16> %b, <8 x i16> %c) nounwind {
 ; CHECK-LABEL: bar4:
-; CHECK: umlal2.4s v0, v1, v2
-; CHECK-NEXT: ret
-
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    umlal2.4s v0, v1, v2
+; CHECK-NEXT:    ret
   %tmp = bitcast <8 x i16> %b to <2 x i64>
   %shuffle.i.i.i = shufflevector <2 x i64> %tmp, <2 x i64> undef, <1 x i32> <i32 1>
   %tmp1 = bitcast <1 x i64> %shuffle.i.i.i to <4 x i16>
@@ -1512,9 +1967,9 @@ define <4 x i32> @bar4(<4 x i32> %a, <8 x i16> %b, <8 x i16> %c) nounwind {
 
 define <2 x i64> @bar5(<2 x i64> %a, <4 x i32> %b, <4 x i32> %c) nounwind {
 ; CHECK-LABEL: bar5:
-; CHECK: umlal2.2d v0, v1, v2
-; CHECK-NEXT: ret
-
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    umlal2.2d v0, v1, v2
+; CHECK-NEXT:    ret
   %tmp = bitcast <4 x i32> %b to <2 x i64>
   %shuffle.i.i.i = shufflevector <2 x i64> %tmp, <2 x i64> undef, <1 x i32> <i32 1>
   %tmp1 = bitcast <1 x i64> %shuffle.i.i.i to <2 x i32>
@@ -1528,8 +1983,10 @@ define <2 x i64> @bar5(<2 x i64> %a, <4 x i32> %b, <4 x i32> %c) nounwind {
 
 define <4 x i32> @mlal2_1(<4 x i32> %a, <8 x i16> %b, <4 x i16> %c) nounwind {
 ; CHECK-LABEL: mlal2_1:
-; CHECK: smlal2.4s v0, v1, v2[3]
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-NEXT:    smlal2.4s v0, v1, v2[3]
+; CHECK-NEXT:    ret
   %shuffle = shufflevector <4 x i16> %c, <4 x i16> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
   %tmp = bitcast <8 x i16> %b to <2 x i64>
   %shuffle.i.i = shufflevector <2 x i64> %tmp, <2 x i64> undef, <1 x i32> <i32 1>
@@ -1544,8 +2001,10 @@ define <4 x i32> @mlal2_1(<4 x i32> %a, <8 x i16> %b, <4 x i16> %c) nounwind {
 
 define <2 x i64> @mlal2_2(<2 x i64> %a, <4 x i32> %b, <2 x i32> %c) nounwind {
 ; CHECK-LABEL: mlal2_2:
-; CHECK: smlal2.2d v0, v1, v2[1]
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-NEXT:    smlal2.2d v0, v1, v2[1]
+; CHECK-NEXT:    ret
   %shuffle = shufflevector <2 x i32> %c, <2 x i32> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
   %tmp = bitcast <4 x i32> %b to <2 x i64>
   %shuffle.i.i = shufflevector <2 x i64> %tmp, <2 x i64> undef, <1 x i32> <i32 1>
@@ -1560,9 +2019,10 @@ define <2 x i64> @mlal2_2(<2 x i64> %a, <4 x i32> %b, <2 x i32> %c) nounwind {
 
 define <4 x i32> @mlal2_4(<4 x i32> %a, <8 x i16> %b, <4 x i16> %c) nounwind {
 ; CHECK-LABEL: mlal2_4:
-; CHECK: umlal2.4s v0, v1, v2[2]
-; CHECK-NEXT: ret
-
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-NEXT:    umlal2.4s v0, v1, v2[2]
+; CHECK-NEXT:    ret
   %shuffle = shufflevector <4 x i16> %c, <4 x i16> undef, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
   %tmp = bitcast <8 x i16> %b to <2 x i64>
   %shuffle.i.i = shufflevector <2 x i64> %tmp, <2 x i64> undef, <1 x i32> <i32 1>
@@ -1577,8 +2037,10 @@ define <4 x i32> @mlal2_4(<4 x i32> %a, <8 x i16> %b, <4 x i16> %c) nounwind {
 
 define <2 x i64> @mlal2_5(<2 x i64> %a, <4 x i32> %b, <2 x i32> %c) nounwind {
 ; CHECK-LABEL: mlal2_5:
-; CHECK: umlal2.2d v0, v1, v2[0]
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-NEXT:    umlal2.2d v0, v1, v2[0]
+; CHECK-NEXT:    ret
   %shuffle = shufflevector <2 x i32> %c, <2 x i32> undef, <4 x i32> zeroinitializer
   %tmp = bitcast <4 x i32> %b to <2 x i64>
   %shuffle.i.i = shufflevector <2 x i64> %tmp, <2 x i64> undef, <1 x i32> <i32 1>
@@ -1593,10 +2055,12 @@ define <2 x i64> @mlal2_5(<2 x i64> %a, <4 x i32> %b, <2 x i32> %c) nounwind {
 
 ; rdar://12328502
 define <2 x double> @vmulq_n_f64(<2 x double> %x, double %y) nounwind readnone ssp {
-entry:
 ; CHECK-LABEL: vmulq_n_f64:
-; CHECK-NOT: dup.2d
-; CHECK: fmul.2d v0, v0, v1[0]
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT:    fmul.2d v0, v0, v1[0]
+; CHECK-NEXT:    ret
+entry:
   %vecinit.i = insertelement <2 x double> undef, double %y, i32 0
   %vecinit1.i = insertelement <2 x double> %vecinit.i, double %y, i32 1
   %mul.i = fmul <2 x double> %vecinit1.i, %x
@@ -1604,10 +2068,12 @@ entry:
 }
 
 define <4 x float> @vmulq_n_f32(<4 x float> %x, float %y) nounwind readnone ssp {
-entry:
 ; CHECK-LABEL: vmulq_n_f32:
-; CHECK-NOT: dup.4s
-; CHECK: fmul.4s v0, v0, v1[0]
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $s1 killed $s1 def $q1
+; CHECK-NEXT:    fmul.4s v0, v0, v1[0]
+; CHECK-NEXT:    ret
+entry:
   %vecinit.i = insertelement <4 x float> undef, float %y, i32 0
   %vecinit1.i = insertelement <4 x float> %vecinit.i, float %y, i32 1
   %vecinit2.i = insertelement <4 x float> %vecinit1.i, float %y, i32 2
@@ -1617,10 +2083,12 @@ entry:
 }
 
 define <2 x float> @vmul_n_f32(<2 x float> %x, float %y) nounwind readnone ssp {
-entry:
 ; CHECK-LABEL: vmul_n_f32:
-; CHECK-NOT: dup.2s
-; CHECK: fmul.2s v0, v0, v1[0]
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $s1 killed $s1 def $q1
+; CHECK-NEXT:    fmul.2s v0, v0, v1[0]
+; CHECK-NEXT:    ret
+entry:
   %vecinit.i = insertelement <2 x float> undef, float %y, i32 0
   %vecinit1.i = insertelement <2 x float> %vecinit.i, float %y, i32 1
   %mul.i = fmul <2 x float> %vecinit1.i, %x
@@ -1628,11 +2096,11 @@ entry:
 }
 
 define <4 x i16> @vmla_laneq_s16_test(<4 x i16> %a, <4 x i16> %b, <8 x i16> %c) nounwind readnone ssp {
+; CHECK-LABEL: vmla_laneq_s16_test:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mla.4h v0, v1, v2[6]
+; CHECK-NEXT:    ret
 entry:
-; CHECK: vmla_laneq_s16_test
-; CHECK-NOT: ext
-; CHECK: mla.4h v0, v1, v2[6]
-; CHECK-NEXT: ret
   %shuffle = shufflevector <8 x i16> %c, <8 x i16> undef, <4 x i32> <i32 6, i32 6, i32 6, i32 6>
   %mul = mul <4 x i16> %shuffle, %b
   %add = add <4 x i16> %mul, %a
@@ -1640,11 +2108,11 @@ entry:
 }
 
 define <2 x i32> @vmla_laneq_s32_test(<2 x i32> %a, <2 x i32> %b, <4 x i32> %c) nounwind readnone ssp {
+; CHECK-LABEL: vmla_laneq_s32_test:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mla.2s v0, v1, v2[3]
+; CHECK-NEXT:    ret
 entry:
-; CHECK: vmla_laneq_s32_test
-; CHECK-NOT: ext
-; CHECK: mla.2s v0, v1, v2[3]
-; CHECK-NEXT: ret
   %shuffle = shufflevector <4 x i32> %c, <4 x i32> undef, <2 x i32> <i32 3, i32 3>
   %mul = mul <2 x i32> %shuffle, %b
   %add = add <2 x i32> %mul, %a
@@ -1652,11 +2120,11 @@ entry:
 }
 
 define <8 x i16> @not_really_vmlaq_laneq_s16_test(<8 x i16> %a, <8 x i16> %b, <8 x i16> %c) nounwind readnone ssp {
+; CHECK-LABEL: not_really_vmlaq_laneq_s16_test:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mla.8h v0, v1, v2[5]
+; CHECK-NEXT:    ret
 entry:
-; CHECK: not_really_vmlaq_laneq_s16_test
-; CHECK-NOT: ext
-; CHECK: mla.8h v0, v1, v2[5]
-; CHECK-NEXT: ret
   %shuffle1 = shufflevector <8 x i16> %c, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
   %shuffle2 = shufflevector <4 x i16> %shuffle1, <4 x i16> undef, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
   %mul = mul <8 x i16> %shuffle2, %b
@@ -1665,11 +2133,11 @@ entry:
 }
 
 define <4 x i32> @not_really_vmlaq_laneq_s32_test(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) nounwind readnone ssp {
+; CHECK-LABEL: not_really_vmlaq_laneq_s32_test:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mla.4s v0, v1, v2[3]
+; CHECK-NEXT:    ret
 entry:
-; CHECK: not_really_vmlaq_laneq_s32_test
-; CHECK-NOT: ext
-; CHECK: mla.4s v0, v1, v2[3]
-; CHECK-NEXT: ret
   %shuffle1 = shufflevector <4 x i32> %c, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
   %shuffle2 = shufflevector <2 x i32> %shuffle1, <2 x i32> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
   %mul = mul <4 x i32> %shuffle2, %b
@@ -1678,54 +2146,55 @@ entry:
 }
 
 define <4 x i32> @vmull_laneq_s16_test(<4 x i16> %a, <8 x i16> %b) nounwind readnone ssp {
+; CHECK-LABEL: vmull_laneq_s16_test:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    smull.4s v0, v0, v1[6]
+; CHECK-NEXT:    ret
 entry:
-; CHECK: vmull_laneq_s16_test
-; CHECK-NOT: ext
-; CHECK: smull.4s v0, v0, v1[6]
-; CHECK-NEXT: ret
   %shuffle = shufflevector <8 x i16> %b, <8 x i16> undef, <4 x i32> <i32 6, i32 6, i32 6, i32 6>
   %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %a, <4 x i16> %shuffle) #2
   ret <4 x i32> %vmull2.i
 }
 
 define <2 x i64> @vmull_laneq_s32_test(<2 x i32> %a, <4 x i32> %b) nounwind readnone ssp {
+; CHECK-LABEL: vmull_laneq_s32_test:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    smull.2d v0, v0, v1[2]
+; CHECK-NEXT:    ret
 entry:
-; CHECK: vmull_laneq_s32_test
-; CHECK-NOT: ext
-; CHECK: smull.2d v0, v0, v1[2]
-; CHECK-NEXT: ret
   %shuffle = shufflevector <4 x i32> %b, <4 x i32> undef, <2 x i32> <i32 2, i32 2>
   %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %a, <2 x i32> %shuffle) #2
   ret <2 x i64> %vmull2.i
 }
 define <4 x i32> @vmull_laneq_u16_test(<4 x i16> %a, <8 x i16> %b) nounwind readnone ssp {
+; CHECK-LABEL: vmull_laneq_u16_test:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    umull.4s v0, v0, v1[6]
+; CHECK-NEXT:    ret
 entry:
-; CHECK: vmull_laneq_u16_test
-; CHECK-NOT: ext
-; CHECK: umull.4s v0, v0, v1[6]
-; CHECK-NEXT: ret
   %shuffle = shufflevector <8 x i16> %b, <8 x i16> undef, <4 x i32> <i32 6, i32 6, i32 6, i32 6>
   %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %a, <4 x i16> %shuffle) #2
   ret <4 x i32> %vmull2.i
 }
 
 define <2 x i64> @vmull_laneq_u32_test(<2 x i32> %a, <4 x i32> %b) nounwind readnone ssp {
+; CHECK-LABEL: vmull_laneq_u32_test:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    umull.2d v0, v0, v1[2]
+; CHECK-NEXT:    ret
 entry:
-; CHECK: vmull_laneq_u32_test
-; CHECK-NOT: ext
-; CHECK: umull.2d v0, v0, v1[2]
-; CHECK-NEXT: ret
   %shuffle = shufflevector <4 x i32> %b, <4 x i32> undef, <2 x i32> <i32 2, i32 2>
   %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %a, <2 x i32> %shuffle) #2
   ret <2 x i64> %vmull2.i
 }
 
 define <4 x i32> @vmull_low_n_s16_test(<4 x i32> %a, <8 x i16> %b, <4 x i16> %c, i32 %d) nounwind readnone optsize ssp {
+; CHECK-LABEL: vmull_low_n_s16_test:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    dup.4h v0, w0
+; CHECK-NEXT:    smull.4s v0, v1, v0
+; CHECK-NEXT:    ret
 entry:
-; CHECK: vmull_low_n_s16_test
-; CHECK-NOT: ext
-; CHECK: smull.4s
-; CHECK-NEXT: ret
   %conv = trunc i32 %d to i16
   %0 = bitcast <8 x i16> %b to <2 x i64>
   %shuffle.i.i = shufflevector <2 x i64> %0, <2 x i64> undef, <1 x i32> <i32 0>
@@ -1739,11 +2208,12 @@ entry:
 }
 
 define <4 x i32> @vmull_high_n_s16_test(<4 x i32> %a, <8 x i16> %b, <4 x i16> %c, i32 %d) nounwind readnone optsize ssp {
+; CHECK-LABEL: vmull_high_n_s16_test:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    dup.8h v0, w0
+; CHECK-NEXT:    smull2.4s v0, v1, v0
+; CHECK-NEXT:    ret
 entry:
-; CHECK: vmull_high_n_s16_test
-; CHECK-NOT: ext
-; CHECK: smull2.4s
-; CHECK-NEXT: ret
   %conv = trunc i32 %d to i16
   %0 = bitcast <8 x i16> %b to <2 x i64>
   %shuffle.i.i = shufflevector <2 x i64> %0, <2 x i64> undef, <1 x i32> <i32 1>
@@ -1757,11 +2227,12 @@ entry:
 }
 
 define <2 x i64> @vmull_high_n_s32_test(<2 x i64> %a, <4 x i32> %b, <2 x i32> %c, i32 %d) nounwind readnone optsize ssp {
+; CHECK-LABEL: vmull_high_n_s32_test:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    dup.4s v0, w0
+; CHECK-NEXT:    smull2.2d v0, v1, v0
+; CHECK-NEXT:    ret
 entry:
-; CHECK: vmull_high_n_s32_test
-; CHECK-NOT: ext
-; CHECK: smull2.2d
-; CHECK-NEXT: ret
   %0 = bitcast <4 x i32> %b to <2 x i64>
   %shuffle.i.i = shufflevector <2 x i64> %0, <2 x i64> undef, <1 x i32> <i32 1>
   %1 = bitcast <1 x i64> %shuffle.i.i to <2 x i32>
@@ -1772,11 +2243,12 @@ entry:
 }
 
 define <4 x i32> @vmull_high_n_u16_test(<4 x i32> %a, <8 x i16> %b, <4 x i16> %c, i32 %d) nounwind readnone optsize ssp {
+; CHECK-LABEL: vmull_high_n_u16_test:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    dup.8h v0, w0
+; CHECK-NEXT:    umull2.4s v0, v1, v0
+; CHECK-NEXT:    ret
 entry:
-; CHECK: vmull_high_n_u16_test
-; CHECK-NOT: ext
-; CHECK: umull2.4s
-; CHECK-NEXT: ret
   %conv = trunc i32 %d to i16
   %0 = bitcast <8 x i16> %b to <2 x i64>
   %shuffle.i.i = shufflevector <2 x i64> %0, <2 x i64> undef, <1 x i32> <i32 1>
@@ -1790,11 +2262,12 @@ entry:
 }
 
 define <2 x i64> @vmull_high_n_u32_test(<2 x i64> %a, <4 x i32> %b, <2 x i32> %c, i32 %d) nounwind readnone optsize ssp {
+; CHECK-LABEL: vmull_high_n_u32_test:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    dup.4s v0, w0
+; CHECK-NEXT:    umull2.2d v0, v1, v0
+; CHECK-NEXT:    ret
 entry:
-; CHECK: vmull_high_n_u32_test
-; CHECK-NOT: ext
-; CHECK: umull2.2d
-; CHECK-NEXT: ret
   %0 = bitcast <4 x i32> %b to <2 x i64>
   %shuffle.i.i = shufflevector <2 x i64> %0, <2 x i64> undef, <1 x i32> <i32 1>
   %1 = bitcast <1 x i64> %shuffle.i.i to <2 x i32>
@@ -1806,9 +2279,9 @@ entry:
 
 define <4 x i32> @vmul_built_dup_test(<4 x i32> %a, <4 x i32> %b) {
 ; CHECK-LABEL: vmul_built_dup_test:
-; CHECK-NOT: ins
-; CHECK-NOT: dup
-; CHECK: mul.4s {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}[1]
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mul.4s v0, v0, v1[1]
+; CHECK-NEXT:    ret
   %vget_lane = extractelement <4 x i32> %b, i32 1
   %vecinit.i = insertelement <4 x i32> undef, i32 %vget_lane, i32 0
   %vecinit1.i = insertelement <4 x i32> %vecinit.i, i32 %vget_lane, i32 1
@@ -1820,9 +2293,10 @@ define <4 x i32> @vmul_built_dup_test(<4 x i32> %a, <4 x i32> %b) {
 
 define <4 x i16> @vmul_built_dup_fromsmall_test(<4 x i16> %a, <4 x i16> %b) {
 ; CHECK-LABEL: vmul_built_dup_fromsmall_test:
-; CHECK-NOT: ins
-; CHECK-NOT: dup
-; CHECK: mul.4h {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}[3]
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT:    mul.4h v0, v0, v1[3]
+; CHECK-NEXT:    ret
   %vget_lane = extractelement <4 x i16> %b, i32 3
   %vecinit.i = insertelement <4 x i16> undef, i16 %vget_lane, i32 0
   %vecinit1.i = insertelement <4 x i16> %vecinit.i, i16 %vget_lane, i32 1
@@ -1834,9 +2308,10 @@ define <4 x i16> @vmul_built_dup_fromsmall_test(<4 x i16> %a, <4 x i16> %b) {
 
 define <8 x i16> @vmulq_built_dup_fromsmall_test(<8 x i16> %a, <4 x i16> %b) {
 ; CHECK-LABEL: vmulq_built_dup_fromsmall_test:
-; CHECK-NOT: ins
-; CHECK-NOT: dup
-; CHECK: mul.8h {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}[0]
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT:    mul.8h v0, v0, v1[0]
+; CHECK-NEXT:    ret
   %vget_lane = extractelement <4 x i16> %b, i32 0
   %vecinit.i = insertelement <8 x i16> undef, i16 %vget_lane, i32 0
   %vecinit1.i = insertelement <8 x i16> %vecinit.i, i16 %vget_lane, i32 1
@@ -1852,9 +2327,9 @@ define <8 x i16> @vmulq_built_dup_fromsmall_test(<8 x i16> %a, <4 x i16> %b) {
 
 define <2 x i64> @mull_from_two_extracts(<4 x i32> %lhs, <4 x i32> %rhs) {
 ; CHECK-LABEL: mull_from_two_extracts:
-; CHECK-NOT: ext
-; CHECK: sqdmull2.2d
-
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sqdmull2.2d v0, v0, v1
+; CHECK-NEXT:    ret
   %lhs.high = shufflevector <4 x i32> %lhs, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
   %rhs.high = shufflevector <4 x i32> %rhs, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
 
@@ -1864,9 +2339,9 @@ define <2 x i64> @mull_from_two_extracts(<4 x i32> %lhs, <4 x i32> %rhs) {
 
 define <2 x i64> @mlal_from_two_extracts(<2 x i64> %accum, <4 x i32> %lhs, <4 x i32> %rhs) {
 ; CHECK-LABEL: mlal_from_two_extracts:
-; CHECK-NOT: ext
-; CHECK: sqdmlal2.2d
-
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sqdmlal2.2d v0, v1, v2
+; CHECK-NEXT:    ret
   %lhs.high = shufflevector <4 x i32> %lhs, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
   %rhs.high = shufflevector <4 x i32> %rhs, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
 
@@ -1877,8 +2352,10 @@ define <2 x i64> @mlal_from_two_extracts(<2 x i64> %accum, <4 x i32> %lhs, <4 x
 
 define <2 x i64> @mull_from_extract_dup_low(<4 x i32> %lhs, i32 %rhs) {
 ; CHECK-LABEL: mull_from_extract_dup_low:
-; CHECK-NOT: ext
-; CHECK: sqdmull.2d
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    dup.2s v1, w0
+; CHECK-NEXT:    sqdmull.2d v0, v0, v1
+; CHECK-NEXT:    ret
   %rhsvec.tmp = insertelement <2 x i32> undef, i32 %rhs, i32 0
   %rhsvec = insertelement <2 x i32> %rhsvec.tmp, i32 %rhs, i32 1
 
@@ -1890,8 +2367,10 @@ define <2 x i64> @mull_from_extract_dup_low(<4 x i32> %lhs, i32 %rhs) {
 
 define <2 x i64> @mull_from_extract_dup_high(<4 x i32> %lhs, i32 %rhs) {
 ; CHECK-LABEL: mull_from_extract_dup_high:
-; CHECK-NOT: ext
-; CHECK: sqdmull2.2d
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    dup.4s v1, w0
+; CHECK-NEXT:    sqdmull2.2d v0, v0, v1
+; CHECK-NEXT:    ret
   %rhsvec.tmp = insertelement <2 x i32> undef, i32 %rhs, i32 0
   %rhsvec = insertelement <2 x i32> %rhsvec.tmp, i32 %rhs, i32 1
 
@@ -1903,8 +2382,10 @@ define <2 x i64> @mull_from_extract_dup_high(<4 x i32> %lhs, i32 %rhs) {
 
 define <8 x i16> @pmull_from_extract_dup_low(<16 x i8> %lhs, i8 %rhs) {
 ; CHECK-LABEL: pmull_from_extract_dup_low:
-; CHECK-NOT: ext
-; CHECK: pmull.8h
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    dup.8b v1, w0
+; CHECK-NEXT:    pmull.8h v0, v0, v1
+; CHECK-NEXT:    ret
   %rhsvec.0 = insertelement <8 x i8> undef, i8 %rhs, i32 0
   %rhsvec = shufflevector <8 x i8> %rhsvec.0, <8 x i8> undef, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
 
@@ -1916,8 +2397,10 @@ define <8 x i16> @pmull_from_extract_dup_low(<16 x i8> %lhs, i8 %rhs) {
 
 define <8 x i16> @pmull_from_extract_dup_high(<16 x i8> %lhs, i8 %rhs) {
 ; CHECK-LABEL: pmull_from_extract_dup_high:
-; CHECK-NOT: ext
-; CHECK: pmull2.8h
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    dup.16b v1, w0
+; CHECK-NEXT:    pmull2.8h v0, v0, v1
+; CHECK-NEXT:    ret
   %rhsvec.0 = insertelement <8 x i8> undef, i8 %rhs, i32 0
   %rhsvec = shufflevector <8 x i8> %rhsvec.0, <8 x i8> undef, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
 
@@ -1929,9 +2412,11 @@ define <8 x i16> @pmull_from_extract_dup_high(<16 x i8> %lhs, i8 %rhs) {
 
 define <8 x i16> @pmull_from_extract_duplane_low(<16 x i8> %lhs, <8 x i8> %rhs) {
 ; CHECK-LABEL: pmull_from_extract_duplane_low:
-; CHECK-NOT: ext
-; CHECK: pmull.8h
-
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT:    dup.8b v1, v1[0]
+; CHECK-NEXT:    pmull.8h v0, v0, v1
+; CHECK-NEXT:    ret
   %lhs.high = shufflevector <16 x i8> %lhs, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   %rhs.high = shufflevector <8 x i8> %rhs, <8 x i8> undef, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
 
@@ -1941,9 +2426,11 @@ define <8 x i16> @pmull_from_extract_duplane_low(<16 x i8> %lhs, <8 x i8> %rhs)
 
 define <8 x i16> @pmull_from_extract_duplane_high(<16 x i8> %lhs, <8 x i8> %rhs) {
 ; CHECK-LABEL: pmull_from_extract_duplane_high:
-; CHECK-NOT: ext
-; CHECK: pmull2.8h
-
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT:    dup.16b v1, v1[0]
+; CHECK-NEXT:    pmull2.8h v0, v0, v1
+; CHECK-NEXT:    ret
   %lhs.high = shufflevector <16 x i8> %lhs, <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
   %rhs.high = shufflevector <8 x i8> %rhs, <8 x i8> undef, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
 
@@ -1953,9 +2440,9 @@ define <8 x i16> @pmull_from_extract_duplane_high(<16 x i8> %lhs, <8 x i8> %rhs)
 
 define <2 x i64> @sqdmull_from_extract_duplane_low(<4 x i32> %lhs, <4 x i32> %rhs) {
 ; CHECK-LABEL: sqdmull_from_extract_duplane_low:
-; CHECK-NOT: ext
-; CHECK: sqdmull.2d
-
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sqdmull.2d v0, v0, v1[0]
+; CHECK-NEXT:    ret
   %lhs.high = shufflevector <4 x i32> %lhs, <4 x i32> undef, <2 x i32> <i32 0, i32 1>
   %rhs.high = shufflevector <4 x i32> %rhs, <4 x i32> undef, <2 x i32> <i32 0, i32 0>
 
@@ -1965,9 +2452,9 @@ define <2 x i64> @sqdmull_from_extract_duplane_low(<4 x i32> %lhs, <4 x i32> %rh
 
 define <2 x i64> @sqdmull_from_extract_duplane_high(<4 x i32> %lhs, <4 x i32> %rhs) {
 ; CHECK-LABEL: sqdmull_from_extract_duplane_high:
-; CHECK-NOT: ext
-; CHECK: sqdmull2.2d
-
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sqdmull2.2d v0, v0, v1[0]
+; CHECK-NEXT:    ret
   %lhs.high = shufflevector <4 x i32> %lhs, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
   %rhs.high = shufflevector <4 x i32> %rhs, <4 x i32> undef, <2 x i32> <i32 0, i32 0>
 
@@ -1977,9 +2464,9 @@ define <2 x i64> @sqdmull_from_extract_duplane_high(<4 x i32> %lhs, <4 x i32> %r
 
 define <2 x i64> @sqdmlal_from_extract_duplane_low(<2 x i64> %accum, <4 x i32> %lhs, <4 x i32> %rhs) {
 ; CHECK-LABEL: sqdmlal_from_extract_duplane_low:
-; CHECK-NOT: ext
-; CHECK: sqdmlal.2d
-
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sqdmlal.2d v0, v1, v2[0]
+; CHECK-NEXT:    ret
   %lhs.high = shufflevector <4 x i32> %lhs, <4 x i32> undef, <2 x i32> <i32 0, i32 1>
   %rhs.high = shufflevector <4 x i32> %rhs, <4 x i32> undef, <2 x i32> <i32 0, i32 0>
 
@@ -1990,9 +2477,9 @@ define <2 x i64> @sqdmlal_from_extract_duplane_low(<2 x i64> %accum, <4 x i32> %
 
 define <2 x i64> @sqdmlal_from_extract_duplane_high(<2 x i64> %accum, <4 x i32> %lhs, <4 x i32> %rhs) {
 ; CHECK-LABEL: sqdmlal_from_extract_duplane_high:
-; CHECK-NOT: ext
-; CHECK: sqdmlal2.2d
-
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sqdmlal2.2d v0, v1, v2[0]
+; CHECK-NEXT:    ret
   %lhs.high = shufflevector <4 x i32> %lhs, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
   %rhs.high = shufflevector <4 x i32> %rhs, <4 x i32> undef, <2 x i32> <i32 0, i32 0>
 
@@ -2003,9 +2490,9 @@ define <2 x i64> @sqdmlal_from_extract_duplane_high(<2 x i64> %accum, <4 x i32>
 
 define <2 x i64> @umlal_from_extract_duplane_low(<2 x i64> %accum, <4 x i32> %lhs, <4 x i32> %rhs) {
 ; CHECK-LABEL: umlal_from_extract_duplane_low:
-; CHECK-NOT: ext
-; CHECK: umlal.2d
-
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    umlal.2d v0, v1, v2[0]
+; CHECK-NEXT:    ret
   %lhs.high = shufflevector <4 x i32> %lhs, <4 x i32> undef, <2 x i32> <i32 0, i32 1>
   %rhs.high = shufflevector <4 x i32> %rhs, <4 x i32> undef, <2 x i32> <i32 0, i32 0>
 
@@ -2016,9 +2503,9 @@ define <2 x i64> @umlal_from_extract_duplane_low(<2 x i64> %accum, <4 x i32> %lh
 
 define <2 x i64> @umlal_from_extract_duplane_high(<2 x i64> %accum, <4 x i32> %lhs, <4 x i32> %rhs) {
 ; CHECK-LABEL: umlal_from_extract_duplane_high:
-; CHECK-NOT: ext
-; CHECK: umlal2.2d
-
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    umlal2.2d v0, v1, v2[0]
+; CHECK-NEXT:    ret
   %lhs.high = shufflevector <4 x i32> %lhs, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
   %rhs.high = shufflevector <4 x i32> %rhs, <4 x i32> undef, <2 x i32> <i32 0, i32 0>
 
@@ -2029,7 +2516,9 @@ define <2 x i64> @umlal_from_extract_duplane_high(<2 x i64> %accum, <4 x i32> %l
 
 define float @scalar_fmla_from_extract_v4f32(float %accum, float %lhs, <4 x float> %rvec) {
 ; CHECK-LABEL: scalar_fmla_from_extract_v4f32:
-; CHECK: fmla.s s0, s1, v2[3]
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmla.s s0, s1, v2[3]
+; CHECK-NEXT:    ret
   %rhs = extractelement <4 x float> %rvec, i32 3
   %res = call float @llvm.fma.f32(float %lhs, float %rhs, float %accum)
   ret float %res
@@ -2037,7 +2526,10 @@ define float @scalar_fmla_from_extract_v4f32(float %accum, float %lhs, <4 x floa
 
 define float @scalar_fmla_from_extract_v2f32(float %accum, float %lhs, <2 x float> %rvec) {
 ; CHECK-LABEL: scalar_fmla_from_extract_v2f32:
-; CHECK: fmla.s s0, s1, v2[1]
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-NEXT:    fmla.s s0, s1, v2[1]
+; CHECK-NEXT:    ret
   %rhs = extractelement <2 x float> %rvec, i32 1
   %res = call float @llvm.fma.f32(float %lhs, float %rhs, float %accum)
   ret float %res
@@ -2045,7 +2537,9 @@ define float @scalar_fmla_from_extract_v2f32(float %accum, float %lhs, <2 x floa
 
 define float @scalar_fmls_from_extract_v4f32(float %accum, float %lhs, <4 x float> %rvec) {
 ; CHECK-LABEL: scalar_fmls_from_extract_v4f32:
-; CHECK: fmls.s s0, s1, v2[3]
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmls.s s0, s1, v2[3]
+; CHECK-NEXT:    ret
   %rhs.scal = extractelement <4 x float> %rvec, i32 3
   %rhs = fsub float -0.0, %rhs.scal
   %res = call float @llvm.fma.f32(float %lhs, float %rhs, float %accum)
@@ -2054,7 +2548,10 @@ define float @scalar_fmls_from_extract_v4f32(float %accum, float %lhs, <4 x floa
 
 define float @scalar_fmls_from_extract_v2f32(float %accum, float %lhs, <2 x float> %rvec) {
 ; CHECK-LABEL: scalar_fmls_from_extract_v2f32:
-; CHECK: fmls.s s0, s1, v2[1]
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-NEXT:    fmls.s s0, s1, v2[1]
+; CHECK-NEXT:    ret
   %rhs.scal = extractelement <2 x float> %rvec, i32 1
   %rhs = fsub float -0.0, %rhs.scal
   %res = call float @llvm.fma.f32(float %lhs, float %rhs, float %accum)
@@ -2065,7 +2562,9 @@ declare float @llvm.fma.f32(float, float, float)
 
 define double @scalar_fmla_from_extract_v2f64(double %accum, double %lhs, <2 x double> %rvec) {
 ; CHECK-LABEL: scalar_fmla_from_extract_v2f64:
-; CHECK: fmla.d d0, d1, v2[1]
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmla.d d0, d1, v2[1]
+; CHECK-NEXT:    ret
   %rhs = extractelement <2 x double> %rvec, i32 1
   %res = call double @llvm.fma.f64(double %lhs, double %rhs, double %accum)
   ret double %res
@@ -2073,7 +2572,9 @@ define double @scalar_fmla_from_extract_v2f64(double %accum, double %lhs, <2 x d
 
 define double @scalar_fmls_from_extract_v2f64(double %accum, double %lhs, <2 x double> %rvec) {
 ; CHECK-LABEL: scalar_fmls_from_extract_v2f64:
-; CHECK: fmls.d d0, d1, v2[1]
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmls.d d0, d1, v2[1]
+; CHECK-NEXT:    ret
   %rhs.scal = extractelement <2 x double> %rvec, i32 1
   %rhs = fsub double -0.0, %rhs.scal
   %res = call double @llvm.fma.f64(double %lhs, double %rhs, double %accum)
@@ -2084,7 +2585,9 @@ declare double @llvm.fma.f64(double, double, double)
 
 define <2 x float> @fmls_with_fneg_before_extract_v2f32(<2 x float> %accum, <2 x float> %lhs, <4 x float> %rhs) {
 ; CHECK-LABEL: fmls_with_fneg_before_extract_v2f32:
-; CHECK: fmls.2s v0, v1, v2[3]
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmls.2s v0, v1, v2[3]
+; CHECK-NEXT:    ret
   %rhs_neg = fsub <4 x float> <float -0.0, float -0.0, float -0.0, float -0.0>, %rhs
   %splat = shufflevector <4 x float> %rhs_neg, <4 x float> undef, <2 x i32> <i32 3, i32 3>
   %res = call <2 x float> @llvm.fma.v2f32(<2 x float> %lhs, <2 x float> %splat, <2 x float> %accum)
@@ -2093,7 +2596,10 @@ define <2 x float> @fmls_with_fneg_before_extract_v2f32(<2 x float> %accum, <2 x
 
 define <2 x float> @fmls_with_fneg_before_extract_v2f32_1(<2 x float> %accum, <2 x float> %lhs, <2 x float> %rhs) {
 ; CHECK-LABEL: fmls_with_fneg_before_extract_v2f32_1:
-; CHECK: fmls.2s v0, v1, v2[1]
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-NEXT:    fmls.2s v0, v1, v2[1]
+; CHECK-NEXT:    ret
   %rhs_neg = fsub <2 x float> <float -0.0, float -0.0>, %rhs
   %splat = shufflevector <2 x float> %rhs_neg, <2 x float> undef, <2 x i32> <i32 1, i32 1>
   %res = call <2 x float> @llvm.fma.v2f32(<2 x float> %lhs, <2 x float> %splat, <2 x float> %accum)
@@ -2102,7 +2608,9 @@ define <2 x float> @fmls_with_fneg_before_extract_v2f32_1(<2 x float> %accum, <2
 
 define <4 x float> @fmls_with_fneg_before_extract_v4f32(<4 x float> %accum, <4 x float> %lhs, <4 x float> %rhs) {
 ; CHECK-LABEL: fmls_with_fneg_before_extract_v4f32:
-; CHECK: fmls.4s v0, v1, v2[3]
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmls.4s v0, v1, v2[3]
+; CHECK-NEXT:    ret
   %rhs_neg = fsub <4 x float> <float -0.0, float -0.0, float -0.0, float -0.0>, %rhs
   %splat = shufflevector <4 x float> %rhs_neg, <4 x float> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
   %res = call <4 x float> @llvm.fma.v4f32(<4 x float> %lhs, <4 x float> %splat, <4 x float> %accum)
@@ -2111,7 +2619,10 @@ define <4 x float> @fmls_with_fneg_before_extract_v4f32(<4 x float> %accum, <4 x
 
 define <4 x float> @fmls_with_fneg_before_extract_v4f32_1(<4 x float> %accum, <4 x float> %lhs, <2 x float> %rhs) {
 ; CHECK-LABEL: fmls_with_fneg_before_extract_v4f32_1:
-; CHECK: fmls.4s v0, v1, v2[1]
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-NEXT:    fmls.4s v0, v1, v2[1]
+; CHECK-NEXT:    ret
   %rhs_neg = fsub <2 x float> <float -0.0, float -0.0>, %rhs
   %splat = shufflevector <2 x float> %rhs_neg, <2 x float> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
   %res = call <4 x float> @llvm.fma.v4f32(<4 x float> %lhs, <4 x float> %splat, <4 x float> %accum)
@@ -2120,7 +2631,9 @@ define <4 x float> @fmls_with_fneg_before_extract_v4f32_1(<4 x float> %accum, <4
 
 define <2 x double> @fmls_with_fneg_before_extract_v2f64(<2 x double> %accum, <2 x double> %lhs, <2 x double> %rhs) {
 ; CHECK-LABEL: fmls_with_fneg_before_extract_v2f64:
-; CHECK: fmls.2d v0, v1, v2[1]
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmls.2d v0, v1, v2[1]
+; CHECK-NEXT:    ret
   %rhs_neg = fsub <2 x double> <double -0.0, double -0.0>, %rhs
   %splat = shufflevector <2 x double> %rhs_neg, <2 x double> undef, <2 x i32> <i32 1, i32 1>
   %res = call <2 x double> @llvm.fma.v2f64(<2 x double> %lhs, <2 x double> %splat, <2 x double> %accum)
@@ -2129,29 +2642,45 @@ define <2 x double> @fmls_with_fneg_before_extract_v2f64(<2 x double> %accum, <2
 
 define <1 x double> @test_fmul_v1f64(<1 x double> %L, <1 x double> %R) nounwind {
 ; CHECK-LABEL: test_fmul_v1f64:
-; CHECK: fmul
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmul d0, d0, d1
+; CHECK-NEXT:    ret
   %prod = fmul <1 x double> %L, %R
   ret <1 x double> %prod
 }
 
 define <1 x double> @test_fdiv_v1f64(<1 x double> %L, <1 x double> %R) nounwind {
 ; CHECK-LABEL: test_fdiv_v1f64:
-; CHECK-LABEL: fdiv
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fdiv d0, d0, d1
+; CHECK-NEXT:    ret
   %prod = fdiv <1 x double> %L, %R
   ret <1 x double> %prod
 }
 
 define i64 @sqdmlal_d(i32 %A, i32 %B, i64 %C) nounwind {
-;CHECK-LABEL: sqdmlal_d:
-;CHECK: sqdmlal
+; CHECK-LABEL: sqdmlal_d:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmov d0, x2
+; CHECK-NEXT:    fmov s1, w0
+; CHECK-NEXT:    fmov s2, w1
+; CHECK-NEXT:    sqdmlal d0, s1, s2
+; CHECK-NEXT:    fmov x0, d0
+; CHECK-NEXT:    ret
   %tmp4 = call i64 @llvm.aarch64.neon.sqdmulls.scalar(i32 %A, i32 %B)
   %tmp5 = call i64 @llvm.aarch64.neon.sqadd.i64(i64 %C, i64 %tmp4)
   ret i64 %tmp5
 }
 
 define i64 @sqdmlsl_d(i32 %A, i32 %B, i64 %C) nounwind {
-;CHECK-LABEL: sqdmlsl_d:
-;CHECK: sqdmlsl
+; CHECK-LABEL: sqdmlsl_d:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmov d0, x2
+; CHECK-NEXT:    fmov s1, w0
+; CHECK-NEXT:    fmov s2, w1
+; CHECK-NEXT:    sqdmlsl d0, s1, s2
+; CHECK-NEXT:    fmov x0, d0
+; CHECK-NEXT:    ret
   %tmp4 = call i64 @llvm.aarch64.neon.sqdmulls.scalar(i32 %A, i32 %B)
   %tmp5 = call i64 @llvm.aarch64.neon.sqsub.i64(i64 %C, i64 %tmp4)
   ret i64 %tmp5
@@ -2159,14 +2688,20 @@ define i64 @sqdmlsl_d(i32 %A, i32 %B, i64 %C) nounwind {
 
 define <16 x i8> @test_pmull_64(i64 %l, i64 %r) nounwind {
 ; CHECK-LABEL: test_pmull_64:
-; CHECK: pmull.1q
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmov d0, x0
+; CHECK-NEXT:    fmov d1, x1
+; CHECK-NEXT:    pmull.1q v0, v0, v1
+; CHECK-NEXT:    ret
   %val = call <16 x i8> @llvm.aarch64.neon.pmull64(i64 %l, i64 %r)
   ret <16 x i8> %val
 }
 
 define <16 x i8> @test_pmull_high_64(<2 x i64> %l, <2 x i64> %r) nounwind {
 ; CHECK-LABEL: test_pmull_high_64:
-; CHECK: pmull2.1q
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    pmull2.1q v0, v0, v1
+; CHECK-NEXT:    ret
   %l_hi = extractelement <2 x i64> %l, i32 1
   %r_hi = extractelement <2 x i64> %r, i32 1
   %val = call <16 x i8> @llvm.aarch64.neon.pmull64(i64 %l_hi, i64 %r_hi)
@@ -2177,7 +2712,14 @@ declare <16 x i8> @llvm.aarch64.neon.pmull64(i64, i64)
 
 define <1 x i64> @test_mul_v1i64(<1 x i64> %lhs, <1 x i64> %rhs) nounwind {
 ; CHECK-LABEL: test_mul_v1i64:
-; CHECK: mul
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    fmov x8, d1
+; CHECK-NEXT:    fmov x9, d0
+; CHECK-NEXT:    mul x8, x9, x8
+; CHECK-NEXT:    fmov d0, x8
+; CHECK-NEXT:    ret
   %prod = mul <1 x i64> %lhs, %rhs
   ret <1 x i64> %prod
 }
diff --git a/llvm/test/CodeGen/AArch64/arm64-vshift.ll b/llvm/test/CodeGen/AArch64/arm64-vshift.ll
index 6b0fe40b5..8c78bf673 100644
--- a/llvm/test/CodeGen/AArch64/arm64-vshift.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-vshift.ll
@@ -1192,6 +1192,96 @@ define <2 x i64> @ushll2_2d(<4 x i32>* %A) nounwind {
         ret <2 x i64> %tmp3
 }
 
+declare <16 x i8> @llvm.aarch64.neon.ushl.v16i8(<16 x i8>, <16 x i8>)
+declare <8 x i16> @llvm.aarch64.neon.ushl.v8i16(<8 x i16>, <8 x i16>)
+declare <4 x i32> @llvm.aarch64.neon.ushl.v4i32(<4 x i32>, <4 x i32>)
+declare <2 x i64> @llvm.aarch64.neon.ushl.v2i64(<2 x i64>, <2 x i64>)
+
+define <8 x i16> @neon.ushll8h_constant_shift(<8 x i8>* %A) nounwind {
+;CHECK-LABEL: neon.ushll8h_constant_shift
+;CHECK: ushll.8h v0, {{v[0-9]+}}, #1
+  %tmp1 = load <8 x i8>, <8 x i8>* %A
+  %tmp2 = zext <8 x i8> %tmp1 to <8 x i16>
+  %tmp3 = call <8 x i16> @llvm.aarch64.neon.ushl.v8i16(<8 x i16> %tmp2, <8 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>)
+  ret <8 x i16> %tmp3
+}
+
+define <8 x i16> @neon.ushl8h_no_constant_shift(<8 x i8>* %A) nounwind {
+;CHECK-LABEL: neon.ushl8h_no_constant_shift
+;CHECK: ushl.8h v0, v0, v0
+  %tmp1 = load <8 x i8>, <8 x i8>* %A
+  %tmp2 = zext <8 x i8> %tmp1 to <8 x i16>
+  %tmp3 = call <8 x i16> @llvm.aarch64.neon.ushl.v8i16(<8 x i16> %tmp2, <8 x i16> %tmp2)
+  ret <8 x i16> %tmp3
+}
+
+define <4 x i32> @neon.ushl8h_constant_shift_extend_not_2x(<4 x i8>* %A) nounwind {
+;CHECK-LABEL: @neon.ushl8h_constant_shift_extend_not_2x
+;CHECK-NOT: ushll.8h v0,
+;CHECK: ldrb    w8, [x0]
+;CHECK: fmov    s0, w8
+;CHECK: ldrb    w8, [x0, #1]
+;CHECK: mov.s   v0[1], w8
+;CHECK: ldrb    w8, [x0, #2]
+;CHECK: mov.s   v0[2], w8
+;CHECK: ldrb    w8, [x0, #3]
+;CHECK: mov.s   v0[3], w8
+;CHECK: shl.4s v0, v0, #1
+  %tmp1 = load <4 x i8>, <4 x i8>* %A
+  %tmp2 = zext <4 x i8> %tmp1 to <4 x i32>
+  %tmp3 = call <4 x i32> @llvm.aarch64.neon.ushl.v4i32(<4 x i32> %tmp2, <4 x i32> <i32 1, i32 1, i32 1, i32 1>)
+  ret <4 x i32> %tmp3
+}
+
+define <8 x i16> @neon.ushl8_noext_constant_shift(<8 x i16>* %A) nounwind {
+; CHECK-LABEL: neon.ushl8_noext_constant_shift
+; CHECK:      ldr       q0, [x0]
+; CHECK-NEXT: shl.8h   v0, v0, #1
+; CHECK-NEXT: ret
+  %tmp1 = load <8 x i16>, <8 x i16>* %A
+  %tmp3 = call <8 x i16> @llvm.aarch64.neon.ushl.v8i16(<8 x i16> %tmp1, <8 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>)
+  ret <8 x i16> %tmp3
+}
+
+define <4 x i32> @neon.ushll4s_constant_shift(<4 x i16>* %A) nounwind {
+;CHECK-LABEL: neon.ushll4s_constant_shift
+;CHECK: ushll.4s v0, {{v[0-9]+}}, #1
+  %tmp1 = load <4 x i16>, <4 x i16>* %A
+  %tmp2 = zext <4 x i16> %tmp1 to <4 x i32>
+  %tmp3 = call <4 x i32> @llvm.aarch64.neon.ushl.v4i32(<4 x i32> %tmp2, <4 x i32> <i32 1, i32 1, i32 1, i32 1>)
+  ret <4 x i32> %tmp3
+}
+
+; FIXME: unnecessary ushll.4s v0, v0, #0?
+define <4 x i32> @neon.ushll4s_neg_constant_shift(<4 x i16>* %A) nounwind {
+; CHECK-LABEL: neon.ushll4s_neg_constant_shift
+; CHECK: movi.2d v1, #0xffffffffffffffff
+; CHECK: ushll.4s v0, v0, #0
+; CHECK: ushl.4s v0, v0, v1
+  %tmp1 = load <4 x i16>, <4 x i16>* %A
+  %tmp2 = zext <4 x i16> %tmp1 to <4 x i32>
+  %tmp3 = call <4 x i32> @llvm.aarch64.neon.ushl.v4i32(<4 x i32> %tmp2, <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>)
+  ret <4 x i32> %tmp3
+}
+
+; FIXME: should be constant folded.
+define <4 x i32> @neon.ushll4s_constant_fold() nounwind {
+; CHECK-LABEL: neon.ushll4s_constant_fold
+; CHECK: shl.4s v0, v0, #1
+;
+  %tmp3 = call <4 x i32> @llvm.aarch64.neon.ushl.v4i32(<4 x i32> <i32 0, i32 1, i32 2, i32 3>, <4 x i32> <i32 1, i32 1, i32 1, i32 1>)
+  ret <4 x i32> %tmp3
+}
+
+define <2 x i64> @neon.ushll2d_constant_shift(<2 x i32>* %A) nounwind {
+;CHECK-LABEL: neon.ushll2d_constant_shift
+;CHECK: ushll.2d v0, {{v[0-9]+}}, #1
+  %tmp1 = load <2 x i32>, <2 x i32>* %A
+  %tmp2 = zext <2 x i32> %tmp1 to <2 x i64>
+  %tmp3 = call <2 x i64> @llvm.aarch64.neon.ushl.v2i64(<2 x i64> %tmp2, <2 x i64> <i64 1, i64 1>)
+  ret <2 x i64> %tmp3
+}
+
 define <8 x i16> @sshll8h(<8 x i8>* %A) nounwind {
 ;CHECK-LABEL: sshll8h:
 ;CHECK: sshll.8h v0, {{v[0-9]+}}, #1
@@ -1201,21 +1291,128 @@ define <8 x i16> @sshll8h(<8 x i8>* %A) nounwind {
         ret <8 x i16> %tmp3
 }
 
-define <4 x i32> @sshll4s(<4 x i16>* %A) nounwind {
-;CHECK-LABEL: sshll4s:
+define <2 x i64> @sshll2d(<2 x i32>* %A) nounwind {
+;CHECK-LABEL: sshll2d:
+;CHECK: sshll.2d v0, {{v[0-9]+}}, #1
+        %tmp1 = load <2 x i32>, <2 x i32>* %A
+        %tmp2 = sext <2 x i32> %tmp1 to <2 x i64>
+        %tmp3 = shl <2 x i64> %tmp2, <i64 1, i64 1>
+        ret <2 x i64> %tmp3
+}
+
+declare <16 x i8> @llvm.aarch64.neon.sshl.v16i8(<16 x i8>, <16 x i8>)
+declare <8 x i16> @llvm.aarch64.neon.sshl.v8i16(<8 x i16>, <8 x i16>)
+declare <4 x i32> @llvm.aarch64.neon.sshl.v4i32(<4 x i32>, <4 x i32>)
+declare <2 x i64> @llvm.aarch64.neon.sshl.v2i64(<2 x i64>, <2 x i64>)
+
+define <16 x i8> @neon.sshl16b_constant_shift(<16 x i8>* %A) nounwind {
+;CHECK-LABEL: neon.sshl16b_constant_shift
+;CHECK: shl.16b {{v[0-9]+}}, {{v[0-9]+}}, #1
+        %tmp1 = load <16 x i8>, <16 x i8>* %A
+        %tmp2 = call <16 x i8> @llvm.aarch64.neon.sshl.v16i8(<16 x i8> %tmp1, <16 x i8> <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>)
+        ret <16 x i8> %tmp2
+}
+
+define <16 x i8> @neon.sshl16b_non_splat_constant_shift(<16 x i8>* %A) nounwind {
+;CHECK-LABEL: neon.sshl16b_non_splat_constant_shift
+;CHECK: sshl.16b {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
+        %tmp1 = load <16 x i8>, <16 x i8>* %A
+        %tmp2 = call <16 x i8> @llvm.aarch64.neon.sshl.v16i8(<16 x i8> %tmp1, <16 x i8> <i8 6, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>)
+        ret <16 x i8> %tmp2
+}
+
+define <16 x i8> @neon.sshl16b_neg_constant_shift(<16 x i8>* %A) nounwind {
+;CHECK-LABEL: neon.sshl16b_neg_constant_shift
+;CHECK: sshl.16b {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
+        %tmp1 = load <16 x i8>, <16 x i8>* %A
+        %tmp2 = call <16 x i8> @llvm.aarch64.neon.sshl.v16i8(<16 x i8> %tmp1, <16 x i8> <i8 -2, i8 -2, i8 -2, i8 -2, i8 -2, i8 -2, i8 -2, i8 -2, i8 -2, i8 -2, i8 -2, i8 -2, i8 -2, i8 -2, i8 -2, i8 -2>)
+        ret <16 x i8> %tmp2
+}
+
+define <8 x i16> @neon.sshll8h_constant_shift(<8 x i8>* %A) nounwind {
+;CHECK-LABEL: neon.sshll8h_constant_shift
+;CHECK: sshll.8h v0, {{v[0-9]+}}, #1
+        %tmp1 = load <8 x i8>, <8 x i8>* %A
+        %tmp2 = sext <8 x i8> %tmp1 to <8 x i16>
+        %tmp3 = call <8 x i16> @llvm.aarch64.neon.sshl.v8i16(<8 x i16> %tmp2, <8 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>)
+        ret <8 x i16> %tmp3
+}
+
+define <4 x i32> @neon.sshl4s_wrong_ext_constant_shift(<4 x i8>* %A) nounwind {
+;CHECK-LABEL: neon.sshl4s_wrong_ext_constant_shift
+;CHECK:       ldrsb   w8, [x0]
+;CHECK-NEXT:  fmov    s0, w8
+;CHECK-NEXT:  ldrsb   w8, [x0, #1]
+;CHECK-NEXT:  mov.s   v0[1], w8
+;CHECK-NEXT:  ldrsb   w8, [x0, #2]
+;CHECK-NEXT:  mov.s   v0[2], w8
+;CHECK-NEXT:  ldrsb   w8, [x0, #3]
+;CHECK-NEXT:  mov.s   v0[3], w8
+;CHECK-NEXT:  shl.4s  v0, v0, #1
+        %tmp1 = load <4 x i8>, <4 x i8>* %A
+        %tmp2 = sext <4 x i8> %tmp1 to <4 x i32>
+        %tmp3 = call <4 x i32> @llvm.aarch64.neon.sshl.v4i32(<4 x i32> %tmp2, <4 x i32> <i32 1, i32 1, i32 1, i32 1>)
+        ret <4 x i32> %tmp3
+}
+
+define <4 x i32> @neon.sshll4s_constant_shift(<4 x i16>* %A) nounwind {
+;CHECK-LABEL: neon.sshll4s_constant_shift
 ;CHECK: sshll.4s v0, {{v[0-9]+}}, #1
         %tmp1 = load <4 x i16>, <4 x i16>* %A
         %tmp2 = sext <4 x i16> %tmp1 to <4 x i32>
-        %tmp3 = shl <4 x i32> %tmp2, <i32 1, i32 1, i32 1, i32 1>
+        %tmp3 = call <4 x i32> @llvm.aarch64.neon.sshl.v4i32(<4 x i32> %tmp2, <4 x i32> <i32 1, i32 1, i32 1, i32 1>)
         ret <4 x i32> %tmp3
 }
 
-define <2 x i64> @sshll2d(<2 x i32>* %A) nounwind {
-;CHECK-LABEL: sshll2d:
+define <4 x i32> @neon.sshll4s_neg_constant_shift(<4 x i16>* %A) nounwind {
+;CHECK-LABEL: neon.sshll4s_neg_constant_shift
+;CHECK: movi.2d v1, #0xffffffffffffffff
+;CHECK: sshll.4s v0, v0, #0
+;CHECK: sshl.4s v0, v0, v1
+        %tmp1 = load <4 x i16>, <4 x i16>* %A
+        %tmp2 = sext <4 x i16> %tmp1 to <4 x i32>
+        %tmp3 = call <4 x i32> @llvm.aarch64.neon.sshl.v4i32(<4 x i32> %tmp2, <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>)
+        ret <4 x i32> %tmp3
+}
+
+; FIXME: should be constant folded.
+define <4 x i32> @neon.sshl4s_constant_fold() nounwind {
+;CHECK-LABEL: neon.sshl4s_constant_fold
+;CHECK: shl.4s {{v[0-9]+}}, {{v[0-9]+}}, #2
+        %tmp3 = call <4 x i32> @llvm.aarch64.neon.sshl.v4i32(<4 x i32> <i32 0, i32 1, i32 2, i32 3>, <4 x i32> <i32 2, i32 2, i32 2, i32 2>)
+        ret <4 x i32> %tmp3
+}
+
+define <4 x i32> @neon.sshl4s_no_fold(<4 x i32>* %A) nounwind {
+;CHECK-LABEL: neon.sshl4s_no_fold
+;CHECK: shl.4s {{v[0-9]+}}, {{v[0-9]+}}, #1
+        %tmp1 = load <4 x i32>, <4 x i32>* %A
+        %tmp3 = call <4 x i32> @llvm.aarch64.neon.sshl.v4i32(<4 x i32> %tmp1, <4 x i32> <i32 1, i32 1, i32 1, i32 1>)
+        ret <4 x i32> %tmp3
+}
+
+define <2 x i64> @neon.sshll2d_constant_shift(<2 x i32>* %A) nounwind {
+;CHECK-LABEL: neon.sshll2d_constant_shift
 ;CHECK: sshll.2d v0, {{v[0-9]+}}, #1
         %tmp1 = load <2 x i32>, <2 x i32>* %A
         %tmp2 = sext <2 x i32> %tmp1 to <2 x i64>
-        %tmp3 = shl <2 x i64> %tmp2, <i64 1, i64 1>
+        %tmp3 = call <2 x i64> @llvm.aarch64.neon.sshl.v2i64(<2 x i64> %tmp2, <2 x i64> <i64 1, i64 1>)
+        ret <2 x i64> %tmp3
+}
+
+; FIXME: should be constant folded.
+define <2 x i64> @neon.sshl2d_constant_fold() nounwind {
+;CHECK-LABEL: neon.sshl2d_constant_fold
+;CHECK: shl.2d {{v[0-9]+}}, {{v[0-9]+}}, #1
+        %tmp3 = call <2 x i64> @llvm.aarch64.neon.sshl.v2i64(<2 x i64> <i64 99, i64 1000>, <2 x i64> <i64 1, i64 1>)
+        ret <2 x i64> %tmp3
+}
+
+define <2 x i64> @neon.sshl2d_no_fold(<2 x i64>* %A) nounwind {
+;CHECK-LABEL: neon.sshl2d_no_fold
+;CHECK: shl.2d {{v[0-9]+}}, {{v[0-9]+}}, #2
+        %tmp2 = load <2 x i64>, <2 x i64>* %A
+        %tmp3 = call <2 x i64> @llvm.aarch64.neon.sshl.v2i64(<2 x i64> %tmp2, <2 x i64> <i64 2, i64 2>)
         ret <2 x i64> %tmp3
 }
 
diff --git a/llvm/test/CodeGen/AArch64/arm64-zero-cycle-zeroing.ll b/llvm/test/CodeGen/AArch64/arm64-zero-cycle-zeroing.ll
index 784b4c486..82fec7489 100644
--- a/llvm/test/CodeGen/AArch64/arm64-zero-cycle-zeroing.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-zero-cycle-zeroing.ll
@@ -4,8 +4,8 @@
 ; RUN: llc < %s -mtriple=aarch64-linux-gnu -mattr=+zcz-gp                 | FileCheck %s -check-prefixes=ALL,ZEROGP,NONEFP
 ; RUN: llc < %s -mtriple=aarch64-linux-gnu -mattr=+zcz-fp                 | FileCheck %s -check-prefixes=ALL,NONEGP,ZEROFP
 ; RUN: llc < %s -mtriple=arm64-apple-ios   -mcpu=cyclone                  | FileCheck %s -check-prefixes=ALL,ZEROGP,NONEFP
+; RUN: llc < %s -mtriple=arm64-linux-gnu   -mcpu=apple-a10                | FileCheck %s -check-prefixes=ALL,ZEROGP,ZEROFP
 ; RUN: llc < %s -mtriple=arm64-apple-ios   -mcpu=cyclone -mattr=+fullfp16 | FileCheck %s -check-prefixes=ALL,ZEROGP,NONE16
-; RUN: llc < %s -mtriple=aarch64-linux-gnu -mcpu=exynos-m1                | FileCheck %s -check-prefixes=ALL,NONEGP,ZEROFP
 ; RUN: llc < %s -mtriple=aarch64-linux-gnu -mcpu=exynos-m3                | FileCheck %s -check-prefixes=ALL,NONEGP,ZEROFP
 ; RUN: llc < %s -mtriple=aarch64-linux-gnu -mcpu=kryo                     | FileCheck %s -check-prefixes=ALL,ZEROGP,ZEROFP
 ; RUN: llc < %s -mtriple=aarch64-linux-gnu -mcpu=falkor                   | FileCheck %s -check-prefixes=ALL,ZEROGP,ZEROFP
diff --git a/llvm/test/CodeGen/AArch64/arm64_32-addrs.ll b/llvm/test/CodeGen/AArch64/arm64_32-addrs.ll
new file mode 100644
index 000000000..5995de294
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/arm64_32-addrs.ll
@@ -0,0 +1,44 @@
+; RUN: llc -mtriple=arm64_32-apple-ios %s -o - | FileCheck %s
+
+; If %base < 96 then the sum will not wrap (in an unsigned sense), but "ldr w0,
+; [x0, #-96]" would.
+define i32 @test_valid_wrap(i32 %base) {
+; CHECK-LABEL: test_valid_wrap:
+; CHECK: sub w[[ADDR:[0-9]+]], w0, #96
+; CHECK: ldr w0, [x[[ADDR]]]
+
+  %newaddr = add nuw i32 %base, -96
+  %ptr = inttoptr i32 %newaddr to i32*
+  %val = load i32, i32* %ptr
+  ret i32 %val
+}
+
+define i8 @test_valid_wrap_optimizable(i8* %base) {
+; CHECK-LABEL: test_valid_wrap_optimizable:
+; CHECK: ldurb w0, [x0, #-96]
+
+  %newaddr = getelementptr inbounds i8, i8* %base, i32 -96
+  %val = load i8, i8* %newaddr
+  ret i8 %val
+}
+
+define i8 @test_valid_wrap_optimizable1(i8* %base, i32 %offset) {
+; CHECK-LABEL: test_valid_wrap_optimizable1:
+; CHECK: ldrb w0, [x0, w1, sxtw]
+
+  %newaddr = getelementptr inbounds i8, i8* %base, i32 %offset
+  %val = load i8, i8* %newaddr
+  ret i8 %val
+}
+
+;
+define i8 @test_valid_wrap_optimizable2(i8* %base, i32 %offset) {
+; CHECK-LABEL: test_valid_wrap_optimizable2:
+; CHECK: sxtw x[[OFFSET:[0-9]+]], w1
+; CHECK: mov w[[BASE:[0-9]+]], #-100
+; CHECK: ldrb w0, [x[[OFFSET]], x[[BASE]]]
+
+  %newaddr = getelementptr inbounds i8, i8* inttoptr(i32 -100 to i8*), i32 %offset
+  %val = load i8, i8* %newaddr
+  ret i8 %val
+}
diff --git a/llvm/test/CodeGen/AArch64/arm64_32-atomics.ll b/llvm/test/CodeGen/AArch64/arm64_32-atomics.ll
new file mode 100644
index 000000000..34682e82f
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/arm64_32-atomics.ll
@@ -0,0 +1,261 @@
+; RUN: llc -mtriple=arm64_32-apple-ios7.0 -o - %s | FileCheck %s
+
+define i8 @test_load_8(i8* %addr) {
+; CHECK-LABAL: test_load_8:
+; CHECK: ldarb w0, [x0]
+  %val = load atomic i8, i8* %addr seq_cst, align 1
+  ret i8 %val
+}
+
+define i16 @test_load_16(i16* %addr) {
+; CHECK-LABAL: test_load_16:
+; CHECK: ldarh w0, [x0]
+  %val = load atomic i16, i16* %addr acquire, align 2
+  ret i16 %val
+}
+
+define i32 @test_load_32(i32* %addr) {
+; CHECK-LABAL: test_load_32:
+; CHECK: ldar w0, [x0]
+  %val = load atomic i32, i32* %addr seq_cst, align 4
+  ret i32 %val
+}
+
+define i64 @test_load_64(i64* %addr) {
+; CHECK-LABAL: test_load_64:
+; CHECK: ldar x0, [x0]
+  %val = load atomic i64, i64* %addr seq_cst, align 8
+  ret i64 %val
+}
+
+define i8* @test_load_ptr(i8** %addr) {
+; CHECK-LABAL: test_load_ptr:
+; CHECK: ldar w0, [x0]
+  %val = load atomic i8*, i8** %addr seq_cst, align 8
+  ret i8* %val
+}
+
+define void @test_store_8(i8* %addr) {
+; CHECK-LABAL: test_store_8:
+; CHECK: stlrb wzr, [x0]
+  store atomic i8 0, i8* %addr seq_cst, align 1
+  ret void
+}
+
+define void @test_store_16(i16* %addr) {
+; CHECK-LABAL: test_store_16:
+; CHECK: stlrh wzr, [x0]
+  store atomic i16 0, i16* %addr seq_cst, align 2
+  ret void
+}
+
+define void @test_store_32(i32* %addr) {
+; CHECK-LABAL: test_store_32:
+; CHECK: stlr wzr, [x0]
+  store atomic i32 0, i32* %addr seq_cst, align 4
+  ret void
+}
+
+define void @test_store_64(i64* %addr) {
+; CHECK-LABAL: test_store_64:
+; CHECK: stlr xzr, [x0]
+  store atomic i64 0, i64* %addr seq_cst, align 8
+  ret void
+}
+
+define void @test_store_ptr(i8** %addr) {
+; CHECK-LABAL: test_store_ptr:
+; CHECK: stlr wzr, [x0]
+  store atomic i8* null, i8** %addr seq_cst, align 8
+  ret void
+}
+
+declare i64 @llvm.aarch64.ldxr.p0i8(i8* %addr)
+declare i64 @llvm.aarch64.ldxr.p0i16(i16* %addr)
+declare i64 @llvm.aarch64.ldxr.p0i32(i32* %addr)
+declare i64 @llvm.aarch64.ldxr.p0i64(i64* %addr)
+
+define i8 @test_ldxr_8(i8* %addr) {
+; CHECK-LABEL: test_ldxr_8:
+; CHECK: ldxrb w0, [x0]
+
+  %val = call i64 @llvm.aarch64.ldxr.p0i8(i8* %addr)
+  %val8 = trunc i64 %val to i8
+  ret i8 %val8
+}
+
+define i16 @test_ldxr_16(i16* %addr) {
+; CHECK-LABEL: test_ldxr_16:
+; CHECK: ldxrh w0, [x0]
+
+  %val = call i64 @llvm.aarch64.ldxr.p0i16(i16* %addr)
+  %val16 = trunc i64 %val to i16
+  ret i16 %val16
+}
+
+define i32 @test_ldxr_32(i32* %addr) {
+; CHECK-LABEL: test_ldxr_32:
+; CHECK: ldxr w0, [x0]
+
+  %val = call i64 @llvm.aarch64.ldxr.p0i32(i32* %addr)
+  %val32 = trunc i64 %val to i32
+  ret i32 %val32
+}
+
+define i64 @test_ldxr_64(i64* %addr) {
+; CHECK-LABEL: test_ldxr_64:
+; CHECK: ldxr x0, [x0]
+
+  %val = call i64 @llvm.aarch64.ldxr.p0i64(i64* %addr)
+  ret i64 %val
+}
+
+declare i64 @llvm.aarch64.ldaxr.p0i8(i8* %addr)
+declare i64 @llvm.aarch64.ldaxr.p0i16(i16* %addr)
+declare i64 @llvm.aarch64.ldaxr.p0i32(i32* %addr)
+declare i64 @llvm.aarch64.ldaxr.p0i64(i64* %addr)
+
+define i8 @test_ldaxr_8(i8* %addr) {
+; CHECK-LABEL: test_ldaxr_8:
+; CHECK: ldaxrb w0, [x0]
+
+  %val = call i64 @llvm.aarch64.ldaxr.p0i8(i8* %addr)
+  %val8 = trunc i64 %val to i8
+  ret i8 %val8
+}
+
+define i16 @test_ldaxr_16(i16* %addr) {
+; CHECK-LABEL: test_ldaxr_16:
+; CHECK: ldaxrh w0, [x0]
+
+  %val = call i64 @llvm.aarch64.ldaxr.p0i16(i16* %addr)
+  %val16 = trunc i64 %val to i16
+  ret i16 %val16
+}
+
+define i32 @test_ldaxr_32(i32* %addr) {
+; CHECK-LABEL: test_ldaxr_32:
+; CHECK: ldaxr w0, [x0]
+
+  %val = call i64 @llvm.aarch64.ldaxr.p0i32(i32* %addr)
+  %val32 = trunc i64 %val to i32
+  ret i32 %val32
+}
+
+define i64 @test_ldaxr_64(i64* %addr) {
+; CHECK-LABEL: test_ldaxr_64:
+; CHECK: ldaxr x0, [x0]
+
+  %val = call i64 @llvm.aarch64.ldaxr.p0i64(i64* %addr)
+  ret i64 %val
+}
+
+declare i32 @llvm.aarch64.stxr.p0i8(i64, i8*)
+declare i32 @llvm.aarch64.stxr.p0i16(i64, i16*)
+declare i32 @llvm.aarch64.stxr.p0i32(i64, i32*)
+declare i32 @llvm.aarch64.stxr.p0i64(i64, i64*)
+
+define i32 @test_stxr_8(i8* %addr, i8 %val) {
+; CHECK-LABEL: test_stxr_8:
+; CHECK: stxrb [[TMP:w[0-9]+]], w1, [x0]
+; CHECK: mov w0, [[TMP]]
+
+  %extval = zext i8 %val to i64
+  %success = call i32 @llvm.aarch64.stxr.p0i8(i64 %extval, i8* %addr)
+  ret i32 %success
+}
+
+define i32 @test_stxr_16(i16* %addr, i16 %val) {
+; CHECK-LABEL: test_stxr_16:
+; CHECK: stxrh [[TMP:w[0-9]+]], w1, [x0]
+; CHECK: mov w0, [[TMP]]
+
+  %extval = zext i16 %val to i64
+  %success = call i32 @llvm.aarch64.stxr.p0i16(i64 %extval, i16* %addr)
+  ret i32 %success
+}
+
+define i32 @test_stxr_32(i32* %addr, i32 %val) {
+; CHECK-LABEL: test_stxr_32:
+; CHECK: stxr [[TMP:w[0-9]+]], w1, [x0]
+; CHECK: mov w0, [[TMP]]
+
+  %extval = zext i32 %val to i64
+  %success = call i32 @llvm.aarch64.stxr.p0i32(i64 %extval, i32* %addr)
+  ret i32 %success
+}
+
+define i32 @test_stxr_64(i64* %addr, i64 %val) {
+; CHECK-LABEL: test_stxr_64:
+; CHECK: stxr [[TMP:w[0-9]+]], x1, [x0]
+; CHECK: mov w0, [[TMP]]
+
+  %success = call i32 @llvm.aarch64.stxr.p0i64(i64 %val, i64* %addr)
+  ret i32 %success
+}
+
+declare i32 @llvm.aarch64.stlxr.p0i8(i64, i8*)
+declare i32 @llvm.aarch64.stlxr.p0i16(i64, i16*)
+declare i32 @llvm.aarch64.stlxr.p0i32(i64, i32*)
+declare i32 @llvm.aarch64.stlxr.p0i64(i64, i64*)
+
+define i32 @test_stlxr_8(i8* %addr, i8 %val) {
+; CHECK-LABEL: test_stlxr_8:
+; CHECK: stlxrb [[TMP:w[0-9]+]], w1, [x0]
+; CHECK: mov w0, [[TMP]]
+
+  %extval = zext i8 %val to i64
+  %success = call i32 @llvm.aarch64.stlxr.p0i8(i64 %extval, i8* %addr)
+  ret i32 %success
+}
+
+define i32 @test_stlxr_16(i16* %addr, i16 %val) {
+; CHECK-LABEL: test_stlxr_16:
+; CHECK: stlxrh [[TMP:w[0-9]+]], w1, [x0]
+; CHECK: mov w0, [[TMP]]
+
+  %extval = zext i16 %val to i64
+  %success = call i32 @llvm.aarch64.stlxr.p0i16(i64 %extval, i16* %addr)
+  ret i32 %success
+}
+
+define i32 @test_stlxr_32(i32* %addr, i32 %val) {
+; CHECK-LABEL: test_stlxr_32:
+; CHECK: stlxr [[TMP:w[0-9]+]], w1, [x0]
+; CHECK: mov w0, [[TMP]]
+
+  %extval = zext i32 %val to i64
+  %success = call i32 @llvm.aarch64.stlxr.p0i32(i64 %extval, i32* %addr)
+  ret i32 %success
+}
+
+define i32 @test_stlxr_64(i64* %addr, i64 %val) {
+; CHECK-LABEL: test_stlxr_64:
+; CHECK: stlxr [[TMP:w[0-9]+]], x1, [x0]
+; CHECK: mov w0, [[TMP]]
+
+  %success = call i32 @llvm.aarch64.stlxr.p0i64(i64 %val, i64* %addr)
+  ret i32 %success
+}
+
+define {i8*, i1} @test_cmpxchg_ptr(i8** %addr, i8* %cmp, i8* %new) {
+; CHECK-LABEL: test_cmpxchg_ptr:
+; CHECK: [[LOOP:LBB[0-9]+_[0-9]+]]:
+; CHECK:     ldaxr [[OLD:w[0-9]+]], [x0]
+; CHECK:     cmp [[OLD]], w1
+; CHECK:     b.ne [[DONE:LBB[0-9]+_[0-9]+]]
+; CHECK:     stlxr [[SUCCESS:w[0-9]+]], w2, [x0]
+; CHECK:     cbnz [[SUCCESS]], [[LOOP]]
+
+; CHECK:     mov w1, #1
+; CHECK:     mov w0, [[OLD]]
+; CHECK:     ret
+
+; CHECK: [[DONE]]:
+; CHECK:     clrex
+; CHECK:     mov w1, wzr
+; CHECK:     mov w0, [[OLD]]
+; CHECK:     ret
+  %res = cmpxchg i8** %addr, i8* %cmp, i8* %new acq_rel acquire
+  ret {i8*, i1} %res
+}
diff --git a/llvm/test/CodeGen/AArch64/arm64_32-fastisel.ll b/llvm/test/CodeGen/AArch64/arm64_32-fastisel.ll
new file mode 100644
index 000000000..0467a2cba
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/arm64_32-fastisel.ll
@@ -0,0 +1,49 @@
+; RUN: llc -mtriple=arm64_32-apple-ios -O0 -fast-isel %s -o - | FileCheck %s
+@var = global i8* null
+
+define void @test_store_release_ptr() {
+; CHECK-LABEL: test_store_release_ptr
+; CHECK: mov [[ZERO:w[0-9]+]], wzr
+; CHECK: stlr [[ZERO]]
+  store atomic i8* null, i8** @var release, align 4
+  br label %next
+
+next:
+  ret void
+}
+
+declare [2 x i32] @callee()
+
+define void @test_struct_return(i32* %addr) {
+; CHECK-LABEL: test_struct_return:
+; CHECK: bl _callee
+; CHECK-DAG: lsr [[HI:x[0-9]+]], x0, #32
+; CHECK-DAG: str w0
+  %res = call [2 x i32] @callee()
+  %res.0 = extractvalue [2 x i32] %res, 0
+  store i32 %res.0, i32* %addr
+  %res.1 = extractvalue [2 x i32] %res, 1
+  store i32 %res.1, i32* %addr
+  ret void
+}
+
+define i8* @test_ret_ptr(i64 %in) {
+; CHECK-LABEL: test_ret_ptr:
+; CHECK: add [[TMP:x[0-9]]], x0, #1
+; CHECK: and x0, [[TMP]], #0xffffffff
+
+  %sum = add i64 %in, 1
+  %res = inttoptr i64 %sum to i8*
+  ret i8* %res
+}
+
+; Handled by SDAG because the struct confuses FastISel, which is fine.
+define {i8*} @test_ret_ptr_struct(i64 %in) {
+; CHECK-LABEL: test_ret_ptr_struct:
+; CHECK: add {{w[0-9]+}}, {{w[0-9]+}}, #1
+
+  %sum = add i64 %in, 1
+  %res.ptr = inttoptr i64 %sum to i8*
+  %res = insertvalue {i8*} undef, i8* %res.ptr, 0
+  ret {i8*} %res
+}
diff --git a/llvm/test/CodeGen/AArch64/arm64_32-frame-pointers.ll b/llvm/test/CodeGen/AArch64/arm64_32-frame-pointers.ll
new file mode 100644
index 000000000..34f5d9b31
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/arm64_32-frame-pointers.ll
@@ -0,0 +1,26 @@
+; RUN: llc -mtriple=arm64_32-apple-ios8.0 %s -o - | FileCheck %s
+
+; We're provoking LocalStackSlotAllocation to create some shared frame bases
+; here: it wants multiple <fi#N> using instructions that can be satisfied by a
+; single base, but not within the addressing-mode.
+;
+; When that happens it's important that we don't mix our pointer sizes
+; (e.g. try to create an ldr from a w-register base).
+define i8 @test_register_wrangling() {
+; CHECK-LABEL: test_register_wrangling:
+; CHECK: add [[TMP:x[0-9]+]], sp,
+; CHECK: add x[[BASE:[0-9]+]], [[TMP]],
+; CHECK: ldrb {{w[0-9]+}}, [x[[BASE]], #1]
+; CHECK: ldrb {{w[0-9]+}}, [x[[BASE]]]
+
+  %var1 = alloca i8, i32 4100
+  %var3 = alloca i8
+  %dummy = alloca i8, i32 4100
+
+  %var1p1 = getelementptr i8, i8* %var1, i32 1
+  %val1 = load i8, i8* %var1
+  %val2 = load i8, i8* %var3
+
+  %sum = add i8 %val1, %val2
+  ret i8 %sum
+}
diff --git a/llvm/test/CodeGen/AArch64/arm64_32-gep-sink.ll b/llvm/test/CodeGen/AArch64/arm64_32-gep-sink.ll
new file mode 100644
index 000000000..21c49d388
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/arm64_32-gep-sink.ll
@@ -0,0 +1,61 @@
+; RUN: opt -codegenprepare -mtriple=arm64_32-apple-ios %s -S -o - | FileCheck %s
+
+define void @test_simple_sink(i1* %base, i64 %offset) {
+; CHECK-LABEL: @test_simple_sink
+; CHECK: next:
+; CHECK:   [[BASE8:%.*]] = bitcast i1* %base to i8*
+; CHECK:   [[ADDR8:%.*]] = getelementptr i8, i8* [[BASE8]], i64 %offset
+; CHECK:   [[ADDR:%.*]] = bitcast i8* [[ADDR8]] to i1*
+; CHECK:   load volatile i1, i1* [[ADDR]]
+  %addr = getelementptr i1, i1* %base, i64 %offset
+  %tst = load i1, i1* %addr
+  br i1 %tst, label %next, label %end
+
+next:
+  load volatile i1, i1* %addr
+  ret void
+
+end:
+  ret void
+}
+
+define void @test_inbounds_sink(i1* %base, i64 %offset) {
+; CHECK-LABEL: @test_inbounds_sink
+; CHECK: next:
+; CHECK:   [[BASE8:%.*]] = bitcast i1* %base to i8*
+; CHECK:   [[ADDR8:%.*]] = getelementptr inbounds i8, i8* [[BASE8]], i64 %offset
+; CHECK:   [[ADDR:%.*]] = bitcast i8* [[ADDR8]] to i1*
+; CHECK:   load volatile i1, i1* [[ADDR]]
+  %addr = getelementptr inbounds i1, i1* %base, i64 %offset
+  %tst = load i1, i1* %addr
+  br i1 %tst, label %next, label %end
+
+next:
+  load volatile i1, i1* %addr
+  ret void
+
+end:
+  ret void
+}
+
+; No address derived via an add can be guaranteed inbounds
+define void @test_add_sink(i1* %base, i64 %offset) {
+; CHECK-LABEL: @test_add_sink
+; CHECK: next:
+; CHECK:   [[BASE8:%.*]] = bitcast i1* %base to i8*
+; CHECK:   [[ADDR8:%.*]] = getelementptr i8, i8* [[BASE8]], i64 %offset
+; CHECK:   [[ADDR:%.*]] = bitcast i8* [[ADDR8]] to i1*
+; CHECK:   load volatile i1, i1* [[ADDR]]
+  %base64 = ptrtoint i1* %base to i64
+  %addr64 = add nsw nuw i64 %base64, %offset
+  %addr = inttoptr i64 %addr64 to i1*
+  %tst = load i1, i1* %addr
+  br i1 %tst, label %next, label %end
+
+next:
+  load volatile i1, i1* %addr
+  ret void
+
+end:
+  ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/arm64_32-memcpy.ll b/llvm/test/CodeGen/AArch64/arm64_32-memcpy.ll
new file mode 100644
index 000000000..f484a2fe6
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/arm64_32-memcpy.ll
@@ -0,0 +1,66 @@
+; RUN: llc -mtriple=arm64_32-apple-ios9.0 -o - %s | FileCheck %s
+
+define i64 @test_memcpy(i64* %addr, i8* %src, i1 %tst) minsize {
+; CHECK-LABEL: test_memcpy:
+; CHECK: ldr [[VAL64:x[0-9]+]], [x0]
+; [...]
+; CHECK: and x0, [[VAL64]], #0xffffffff
+; CHECK: bl _memcpy
+
+  %val64 = load i64, i64* %addr
+  br i1 %tst, label %true, label %false
+
+true:
+  ret i64 %val64
+
+false:
+  %val32 = trunc i64 %val64 to i32
+  %val.ptr = inttoptr i32 %val32 to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %val.ptr, i8* %src, i32 128, i32 0, i1 1)
+  ret i64 undef
+}
+
+define i64 @test_memmove(i64* %addr, i8* %src, i1 %tst) minsize {
+; CHECK-LABEL: test_memmove:
+; CHECK: ldr [[VAL64:x[0-9]+]], [x0]
+; [...]
+; CHECK: and x0, [[VAL64]], #0xffffffff
+; CHECK: bl _memmove
+
+  %val64 = load i64, i64* %addr
+  br i1 %tst, label %true, label %false
+
+true:
+  ret i64 %val64
+
+false:
+  %val32 = trunc i64 %val64 to i32
+  %val.ptr = inttoptr i32 %val32 to i8*
+  call void @llvm.memmove.p0i8.p0i8.i32(i8* %val.ptr, i8* %src, i32 128, i32 0, i1 1)
+  ret i64 undef
+}
+
+define i64 @test_memset(i64* %addr, i8* %src, i1 %tst) minsize {
+; CHECK-LABEL: test_memset:
+; CHECK: ldr [[VAL64:x[0-9]+]], [x0]
+; [...]
+; CHECK: and x0, [[VAL64]], #0xffffffff
+; CHECK: bl _memset
+
+  %val64 = load i64, i64* %addr
+  br i1 %tst, label %true, label %false
+
+true:
+  ret i64 %val64
+
+false:
+  %val32 = trunc i64 %val64 to i32
+  %val.ptr = inttoptr i32 %val32 to i8*
+  call void @llvm.memset.p0i8.i32(i8* %val.ptr, i8 42, i32 256, i32 0, i1 1)
+  ret i64 undef
+}
+
+declare void @llvm.memcpy.p0i8.p0i8.i32(i8*, i8*, i32, i32, i1)
+declare void @llvm.memmove.p0i8.p0i8.i32(i8*, i8*, i32, i32, i1)
+declare void @llvm.memset.p0i8.i32(i8*, i8, i32, i32, i1)
+
diff --git a/llvm/test/CodeGen/AArch64/arm64_32-neon.ll b/llvm/test/CodeGen/AArch64/arm64_32-neon.ll
new file mode 100644
index 000000000..9a1ecb2bc
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/arm64_32-neon.ll
@@ -0,0 +1,198 @@
+; RUN: llc -mtriple=arm64_32-apple-ios7.0 -mcpu=cyclone %s -o - | FileCheck %s
+
+define <2 x double> @test_insert_elt(<2 x double> %vec, double %val) {
+; CHECK-LABEL: test_insert_elt:
+; CHECK: mov.d v0[0], v1[0]
+  %res = insertelement <2 x double> %vec, double %val, i32 0
+  ret <2 x double> %res
+}
+
+define void @test_split_16B(<4 x float> %val, <4 x float>* %addr) {
+; CHECK-LABEL: test_split_16B:
+; CHECK: str q0, [x0]
+  store <4 x float> %val, <4 x float>* %addr, align 8
+  ret void
+}
+
+define void @test_split_16B_splat(<4 x i32>, <4 x i32>* %addr) {
+; CHECK-LABEL: test_split_16B_splat:
+; CHECK: str {{q[0-9]+}}
+
+  %vec.tmp0 = insertelement <4 x i32> undef, i32 42, i32 0
+  %vec.tmp1 = insertelement <4 x i32> %vec.tmp0, i32 42, i32 1
+  %vec.tmp2 = insertelement <4 x i32> %vec.tmp1, i32 42, i32 2
+  %vec = insertelement <4 x i32> %vec.tmp2, i32 42, i32 3
+
+  store <4 x i32> %vec, <4 x i32>* %addr, align 8
+  ret void
+}
+
+
+%vec = type <2 x double>
+
+declare {%vec, %vec} @llvm.aarch64.neon.ld2r.v2f64.p0i8(i8*)
+define {%vec, %vec} @test_neon_load(i8* %addr) {
+; CHECK-LABEL: test_neon_load:
+; CHECK: ld2r.2d { v0, v1 }, [x0]
+  %res = call {%vec, %vec} @llvm.aarch64.neon.ld2r.v2f64.p0i8(i8* %addr)
+  ret {%vec, %vec} %res
+}
+
+declare {%vec, %vec} @llvm.aarch64.neon.ld2lane.v2f64.p0i8(%vec, %vec, i64, i8*)
+define {%vec, %vec} @test_neon_load_lane(i8* %addr, %vec %in1, %vec %in2) {
+; CHECK-LABEL: test_neon_load_lane:
+; CHECK: ld2.d { v0, v1 }[0], [x0]
+  %res = call {%vec, %vec} @llvm.aarch64.neon.ld2lane.v2f64.p0i8(%vec %in1, %vec %in2, i64 0, i8* %addr)
+  ret {%vec, %vec} %res
+}
+
+declare void @llvm.aarch64.neon.st2.v2f64.p0i8(%vec, %vec, i8*)
+define void @test_neon_store(i8* %addr, %vec %in1, %vec %in2) {
+; CHECK-LABEL: test_neon_store:
+; CHECK: st2.2d { v0, v1 }, [x0]
+  call void @llvm.aarch64.neon.st2.v2f64.p0i8(%vec %in1, %vec %in2, i8* %addr)
+  ret void
+}
+
+declare void @llvm.aarch64.neon.st2lane.v2f64.p0i8(%vec, %vec, i64, i8*)
+define void @test_neon_store_lane(i8* %addr, %vec %in1, %vec %in2) {
+; CHECK-LABEL: test_neon_store_lane:
+; CHECK: st2.d { v0, v1 }[1], [x0]
+  call void @llvm.aarch64.neon.st2lane.v2f64.p0i8(%vec %in1, %vec %in2, i64 1, i8* %addr)
+  ret void
+}
+
+declare {%vec, %vec} @llvm.aarch64.neon.ld2.v2f64.p0i8(i8*)
+define {{%vec, %vec}, i8*} @test_neon_load_post(i8* %addr, i32 %offset) {
+; CHECK-LABEL: test_neon_load_post:
+; CHECK-DAG: sxtw [[OFFSET:x[0-9]+]], w1
+; CHECK: ld2.2d { v0, v1 }, [x0], [[OFFSET]]
+
+  %vecs = call {%vec, %vec} @llvm.aarch64.neon.ld2.v2f64.p0i8(i8* %addr)
+
+  %addr.new = getelementptr inbounds i8, i8* %addr, i32 %offset
+
+  %res.tmp = insertvalue {{%vec, %vec}, i8*} undef, {%vec, %vec} %vecs, 0
+  %res = insertvalue {{%vec, %vec}, i8*} %res.tmp, i8* %addr.new, 1
+  ret {{%vec, %vec}, i8*} %res
+}
+
+define {{%vec, %vec}, i8*} @test_neon_load_post_lane(i8* %addr, i32 %offset, %vec %in1, %vec %in2) {
+; CHECK-LABEL: test_neon_load_post_lane:
+; CHECK-DAG: sxtw [[OFFSET:x[0-9]+]], w1
+; CHECK: ld2.d { v0, v1 }[1], [x0], [[OFFSET]]
+
+  %vecs = call {%vec, %vec} @llvm.aarch64.neon.ld2lane.v2f64.p0i8(%vec %in1, %vec %in2, i64 1, i8* %addr)
+
+  %addr.new = getelementptr inbounds i8, i8* %addr, i32 %offset
+
+  %res.tmp = insertvalue {{%vec, %vec}, i8*} undef, {%vec, %vec} %vecs, 0
+  %res = insertvalue {{%vec, %vec}, i8*} %res.tmp, i8* %addr.new, 1
+  ret {{%vec, %vec}, i8*} %res
+}
+
+define i8* @test_neon_store_post(i8* %addr, i32 %offset, %vec %in1, %vec %in2) {
+; CHECK-LABEL: test_neon_store_post:
+; CHECK-DAG: sxtw [[OFFSET:x[0-9]+]], w1
+; CHECK: st2.2d { v0, v1 }, [x0], [[OFFSET]]
+
+  call void @llvm.aarch64.neon.st2.v2f64.p0i8(%vec %in1, %vec %in2, i8* %addr)
+
+  %addr.new = getelementptr inbounds i8, i8* %addr, i32 %offset
+
+  ret i8* %addr.new
+}
+
+define i8* @test_neon_store_post_lane(i8* %addr, i32 %offset, %vec %in1, %vec %in2) {
+; CHECK-LABEL: test_neon_store_post_lane:
+; CHECK: sxtw [[OFFSET:x[0-9]+]], w1
+; CHECK: st2.d { v0, v1 }[0], [x0], [[OFFSET]]
+
+  call void @llvm.aarch64.neon.st2lane.v2f64.p0i8(%vec %in1, %vec %in2, i64 0, i8* %addr)
+
+  %addr.new = getelementptr inbounds i8, i8* %addr, i32 %offset
+
+  ret i8* %addr.new
+}
+
+; ld1 is slightly different because it goes via ISelLowering of normal IR ops
+; rather than an intrinsic.
+define {%vec, double*} @test_neon_ld1_post_lane(double* %addr, i32 %offset, %vec %in) {
+; CHECK-LABEL: test_neon_ld1_post_lane:
+; CHECK: sbfiz [[OFFSET:x[0-9]+]], x1, #3, #32
+; CHECK: ld1.d { v0 }[0], [x0], [[OFFSET]]
+
+  %loaded = load double, double* %addr, align 8
+  %newvec = insertelement %vec %in, double %loaded, i32 0
+
+  %addr.new = getelementptr inbounds double, double* %addr, i32 %offset
+
+  %res.tmp = insertvalue {%vec, double*} undef, %vec %newvec, 0
+  %res = insertvalue {%vec, double*} %res.tmp, double* %addr.new, 1
+
+  ret {%vec, double*} %res
+}
+
+define {{%vec, %vec}, i8*} @test_neon_load_post_exact(i8* %addr) {
+; CHECK-LABEL: test_neon_load_post_exact:
+; CHECK: ld2.2d { v0, v1 }, [x0], #32
+
+  %vecs = call {%vec, %vec} @llvm.aarch64.neon.ld2.v2f64.p0i8(i8* %addr)
+
+  %addr.new = getelementptr inbounds i8, i8* %addr, i32 32
+
+  %res.tmp = insertvalue {{%vec, %vec}, i8*} undef, {%vec, %vec} %vecs, 0
+  %res = insertvalue {{%vec, %vec}, i8*} %res.tmp, i8* %addr.new, 1
+  ret {{%vec, %vec}, i8*} %res
+}
+
+define {%vec, double*} @test_neon_ld1_post_lane_exact(double* %addr, %vec %in) {
+; CHECK-LABEL: test_neon_ld1_post_lane_exact:
+; CHECK: ld1.d { v0 }[0], [x0], #8
+
+  %loaded = load double, double* %addr, align 8
+  %newvec = insertelement %vec %in, double %loaded, i32 0
+
+  %addr.new = getelementptr inbounds double, double* %addr, i32 1
+
+  %res.tmp = insertvalue {%vec, double*} undef, %vec %newvec, 0
+  %res = insertvalue {%vec, double*} %res.tmp, double* %addr.new, 1
+
+  ret {%vec, double*} %res
+}
+
+; As in the general load/store case, this GEP has defined semantics when the
+; address wraps. We cannot use post-indexed addressing.
+define {%vec, double*} @test_neon_ld1_notpost_lane_exact(double* %addr, %vec %in) {
+; CHECK-LABEL: test_neon_ld1_notpost_lane_exact:
+; CHECK-NOT: ld1.d { {{v[0-9]+}} }[0], [{{x[0-9]+|sp}}], #8
+; CHECK: add w0, w0, #8
+; CHECK: ret
+
+  %loaded = load double, double* %addr, align 8
+  %newvec = insertelement %vec %in, double %loaded, i32 0
+
+  %addr.new = getelementptr double, double* %addr, i32 1
+
+  %res.tmp = insertvalue {%vec, double*} undef, %vec %newvec, 0
+  %res = insertvalue {%vec, double*} %res.tmp, double* %addr.new, 1
+
+  ret {%vec, double*} %res
+}
+
+define {%vec, double*} @test_neon_ld1_notpost_lane(double* %addr, i32 %offset, %vec %in) {
+; CHECK-LABEL: test_neon_ld1_notpost_lane:
+; CHECK-NOT: ld1.d { {{v[0-9]+}} }[0], [{{x[0-9]+|sp}}], {{x[0-9]+|sp}}
+; CHECK: add w0, w0, w1, lsl #3
+; CHECK: ret
+
+  %loaded = load double, double* %addr, align 8
+  %newvec = insertelement %vec %in, double %loaded, i32 0
+
+  %addr.new = getelementptr double, double* %addr, i32 %offset
+
+  %res.tmp = insertvalue {%vec, double*} undef, %vec %newvec, 0
+  %res = insertvalue {%vec, double*} %res.tmp, double* %addr.new, 1
+
+  ret {%vec, double*} %res
+}
diff --git a/llvm/test/CodeGen/AArch64/arm64_32-null.ll b/llvm/test/CodeGen/AArch64/arm64_32-null.ll
new file mode 100644
index 000000000..9d62c5624
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/arm64_32-null.ll
@@ -0,0 +1,28 @@
+; RUN: llc -fast-isel=true  -global-isel=false -O0 -mtriple=arm64_32-apple-ios %s -o - | FileCheck %s
+; RUN: llc -fast-isel=false -global-isel=false -O0 -mtriple=arm64_32-apple-ios %s -o - | FileCheck %s
+
+define void @test_store(i8** %p) {
+; CHECK-LABEL: test_store:
+; CHECK: mov [[R1:w[0-9]+]], wzr
+; CHECK: str [[R1]], [x0]
+
+  store i8* null, i8** %p
+  ret void
+}
+
+define void @test_phi(i8** %p) {
+; CHECK-LABEL: test_phi:
+; CHECK: mov [[R1:x[0-9]+]], xzr
+; CHECK: str [[R1]], [sp]
+; CHECK: b [[BB:LBB[0-9_]+]]
+; CHECK: [[BB]]:
+; CHECK: ldr x0, [sp]
+; CHECK: str w0, [x{{.*}}]
+
+bb0:
+  br label %bb1
+bb1:
+  %tmp0 = phi i8* [ null, %bb0 ]
+  store i8* %tmp0, i8** %p
+  ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/arm64_32-pointer-extend.ll b/llvm/test/CodeGen/AArch64/arm64_32-pointer-extend.ll
new file mode 100644
index 000000000..74b88305b
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/arm64_32-pointer-extend.ll
@@ -0,0 +1,49 @@
+; RUN: llc -mtriple=arm64_32-apple-ios7.0 %s -o - | FileCheck %s
+
+define void @pass_pointer(i64 %in) {
+; CHECK-LABEL: pass_pointer:
+; CHECK: and x0, x0, #0xffffffff
+; CHECK: bl _take_pointer
+
+  %in32 = trunc i64 %in to i32
+  %ptr = inttoptr i32 %in32 to i8*
+  call i64 @take_pointer(i8* %ptr)
+  ret void
+}
+
+define i64 @take_pointer(i8* %ptr) nounwind {
+; CHECK-LABEL: take_pointer:
+; CHECK-NEXT: %bb.0
+; CHECK-NEXT: ret
+
+  %val = ptrtoint i8* %ptr to i32
+  %res = zext i32 %val to i64
+  ret i64 %res
+}
+
+define i32 @callee_ptr_stack_slot([8 x i64], i8*, i32 %val) {
+; CHECK-LABEL: callee_ptr_stack_slot:
+; CHECK: ldr w0, [sp, #4]
+
+  ret i32 %val
+}
+
+define void @caller_ptr_stack_slot(i8* %ptr) {
+; CHECK-LABEL: caller_ptr_stack_slot:
+; CHECK-DAG: mov [[VAL:w[0-9]]], #42
+; CHECK: stp w0, [[VAL]], [sp]
+
+  call i32 @callee_ptr_stack_slot([8 x i64] undef, i8* %ptr, i32 42)
+  ret void
+}
+
+define i8* @return_ptr(i64 %in, i64 %r) {
+; CHECK-LABEL: return_ptr:
+; CHECK: sdiv [[VAL64:x[0-9]+]], x0, x1
+; CHECK: and x0, [[VAL64]], #0xffffffff
+
+  %sum = sdiv i64 %in, %r
+  %sum32 = trunc i64 %sum to i32
+  %res = inttoptr i32 %sum32 to i8*
+  ret i8* %res
+}
diff --git a/llvm/test/CodeGen/AArch64/arm64_32-stack-pointers.ll b/llvm/test/CodeGen/AArch64/arm64_32-stack-pointers.ll
new file mode 100644
index 000000000..a233e3416
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/arm64_32-stack-pointers.ll
@@ -0,0 +1,13 @@
+; RUN: llc -mtriple=arm64_32-apple-ios9.0 -o - %s | FileCheck %s
+
+declare void @callee([8 x i64], i8*, i8*)
+
+; Make sure we don't accidentally store X0 or XZR, which might well
+; clobber other arguments or data.
+define void @test_stack_ptr_32bits(i8* %in) {
+; CHECK-LABEL: test_stack_ptr_32bits:
+; CHECK-DAG: stp wzr, w0, [sp]
+
+  call void @callee([8 x i64] undef, i8* null, i8* %in)
+  ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/arm64_32-tls.ll b/llvm/test/CodeGen/AArch64/arm64_32-tls.ll
new file mode 100644
index 000000000..fada71530
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/arm64_32-tls.ll
@@ -0,0 +1,22 @@
+; RUN: llc -mtriple=arm64_32-apple-ios %s -o - | FileCheck %s
+
+define i32 @test_thread_local() {
+; CHECK-LABEL: test_thread_local:
+; CHECK: adrp x[[TMP:[0-9]+]], _var@TLVPPAGE
+; CHECK: ldr w0, [x[[TMP]], _var@TLVPPAGEOFF]
+; CHECK: ldr w[[DEST:[0-9]+]], [x0]
+; CHECK: blr x[[DEST]]
+
+  %val = load i32, i32* @var
+  ret i32 %val
+}
+
+@var = thread_local global i32 zeroinitializer
+
+; CHECK: .tbss _var$tlv$init, 4, 2
+
+; CHECK-LABEL: __DATA,__thread_vars
+; CHECK: _var:
+; CHECK:    .long __tlv_bootstrap
+; CHECK:    .long 0
+; CHECK:    .long _var$tlv$init
diff --git a/llvm/test/CodeGen/AArch64/arm64_32-va.ll b/llvm/test/CodeGen/AArch64/arm64_32-va.ll
new file mode 100644
index 000000000..94ff47161
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/arm64_32-va.ll
@@ -0,0 +1,56 @@
+; RUN: llc -mtriple=arm64_32-apple-ios %s -o - | FileCheck %s
+
+define void @test_va_copy(i8* %dst, i8* %src) {
+; CHECK-LABEL: test_va_copy:
+; CHECK: ldr [[PTR:w[0-9]+]], [x1]
+; CHECK: str [[PTR]], [x0]
+
+  call void @llvm.va_copy(i8* %dst, i8* %src)
+  ret void
+}
+
+define void @test_va_start(i32, ...)  {
+; CHECK-LABEL: test_va_start
+; CHECK: add x[[LIST:[0-9]+]], sp, #16
+; CHECK: str w[[LIST]],
+  %slot = alloca i8*, align 4
+  %list = bitcast i8** %slot to i8*
+  call void @llvm.va_start(i8* %list)
+  ret void
+}
+
+define void @test_va_start_odd([8 x i64], i32, ...) {
+; CHECK-LABEL: test_va_start_odd:
+; CHECK: add x[[LIST:[0-9]+]], sp, #20
+; CHECK: str w[[LIST]],
+  %slot = alloca i8*, align 4
+  %list = bitcast i8** %slot to i8*
+  call void @llvm.va_start(i8* %list)
+  ret void
+}
+
+define i8* @test_va_arg(i8** %list) {
+; CHECK-LABEL: test_va_arg:
+; CHECK: ldr w[[LOC:[0-9]+]], [x0]
+; CHECK: add [[NEXTLOC:w[0-9]+]], w[[LOC]], #4
+; CHECK: str [[NEXTLOC]], [x0]
+; CHECK: ldr w0, [x[[LOC]]]
+  %res = va_arg i8** %list, i8*
+  ret i8* %res
+}
+
+define i8* @really_test_va_arg(i8** %list, i1 %tst) {
+; CHECK-LABEL: really_test_va_arg:
+; CHECK: ldr w[[LOC:[0-9]+]], [x0]
+; CHECK: add [[NEXTLOC:w[0-9]+]], w[[LOC]], #4
+; CHECK: str [[NEXTLOC]], [x0]
+; CHECK: ldr w[[VAARG:[0-9]+]], [x[[LOC]]]
+; CHECK: csel x0, x[[VAARG]], xzr
+  %tmp = va_arg i8** %list, i8*
+  %res = select i1 %tst, i8* %tmp, i8* null
+  ret i8* %res
+}
+
+declare void @llvm.va_start(i8*) 
+
+declare void @llvm.va_copy(i8*, i8*)
diff --git a/llvm/test/CodeGen/AArch64/arm64_32.ll b/llvm/test/CodeGen/AArch64/arm64_32.ll
new file mode 100644
index 000000000..c30f651f9
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/arm64_32.ll
@@ -0,0 +1,730 @@
+; RUN: llc -mtriple=arm64_32-apple-ios7.0 %s -filetype=obj -o - -disable-post-ra -frame-pointer=non-leaf | \
+; RUN:     llvm-objdump -private-headers - | \
+; RUN:     FileCheck %s --check-prefix=CHECK-MACHO
+; RUN: llc -mtriple=arm64_32-apple-ios7.0 %s -o - -aarch64-enable-atomic-cfg-tidy=0 -disable-post-ra -frame-pointer=non-leaf | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-OPT
+; RUN: llc -mtriple=arm64_32-apple-ios7.0 %s -o - -fast-isel -aarch64-enable-atomic-cfg-tidy=0 -disable-post-ra -frame-pointer=non-leaf | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-FAST
+
+; CHECK-MACHO: Mach header
+; CHECK-MACHO: MH_MAGIC ARM64_32 V8
+
+@var64 = global i64 zeroinitializer, align 8
+@var32 = global i32 zeroinitializer, align 4
+
+@var_got = external global i8
+
+define i32* @test_global_addr() {
+; CHECK-LABEL: test_global_addr:
+; CHECK: adrp [[PAGE:x[0-9]+]], _var32@PAGE
+; CHECK-OPT: add x0, [[PAGE]], _var32@PAGEOFF
+; CHECK-FAST: add [[TMP:x[0-9]+]], [[PAGE]], _var32@PAGEOFF
+; CHECK-FAST: and x0, [[TMP]], #0xffffffff
+  ret i32* @var32
+}
+
+; ADRP is necessarily 64-bit. The important point to check is that, however that
+; gets truncated to 32-bits, it's free. No need to zero out higher bits of that
+; register.
+define i64 @test_global_addr_extension() {
+; CHECK-LABEL: test_global_addr_extension:
+; CHECK: adrp [[PAGE:x[0-9]+]], _var32@PAGE
+; CHECK: add x0, [[PAGE]], _var32@PAGEOFF
+; CHECK-NOT: and
+; CHECK: ret
+
+  ret i64 ptrtoint(i32* @var32 to i64)
+}
+
+define i32 @test_global_value() {
+; CHECK-LABEL: test_global_value:
+; CHECK: adrp x[[PAGE:[0-9]+]], _var32@PAGE
+; CHECK: ldr w0, [x[[PAGE]], _var32@PAGEOFF]
+  %val = load i32, i32* @var32, align 4
+  ret i32 %val
+}
+
+; Because the addition may wrap, it is not safe to use "ldr w0, [xN, #32]" here.
+define i32 @test_unsafe_indexed_add() {
+; CHECK-LABEL: test_unsafe_indexed_add:
+; CHECK: add x[[VAR32:[0-9]+]], {{x[0-9]+}}, _var32@PAGEOFF
+; CHECK: add w[[ADDR:[0-9]+]], w[[VAR32]], #32
+; CHECK: ldr w0, [x[[ADDR]]]
+  %addr_int = ptrtoint i32* @var32 to i32
+  %addr_plus_32 = add i32 %addr_int, 32
+  %addr = inttoptr i32 %addr_plus_32 to i32*
+  %val = load i32, i32* %addr, align 4
+  ret i32 %val
+}
+
+; Since we've promised there is no unsigned overflow, @var32 must be at least
+; 32-bytes below 2^32, and we can use the load this time.
+define i32 @test_safe_indexed_add() {
+; CHECK-LABEL: test_safe_indexed_add:
+; CHECK: add x[[VAR32:[0-9]+]], {{x[0-9]+}}, _var32@PAGEOFF
+; CHECK: add w[[ADDR:[0-9]+]], w[[VAR32]], #32
+; CHECK: ldr w0, [x[[ADDR]]]
+  %addr_int = ptrtoint i32* @var32 to i64
+  %addr_plus_32 = add nuw i64 %addr_int, 32
+  %addr = inttoptr i64 %addr_plus_32 to i32*
+  %val = load i32, i32* %addr, align 4
+  ret i32 %val
+}
+
+define i32 @test_safe_indexed_or(i32 %in) {
+; CHECK-LABEL: test_safe_indexed_or:
+; CHECK: and [[TMP:w[0-9]+]], {{w[0-9]+}}, #0xfffffff0
+; CHECK: orr w[[ADDR:[0-9]+]], [[TMP]], #0x4
+; CHECK: ldr w0, [x[[ADDR]]]
+  %addr_int = and i32 %in, -16
+  %addr_plus_4 = or i32 %addr_int, 4
+  %addr = inttoptr i32 %addr_plus_4 to i32*
+  %val = load i32, i32* %addr, align 4
+  ret i32 %val
+}
+
+
+; Promising nsw is not sufficient because the addressing mode basically
+; calculates "zext(base) + zext(offset)" and nsw only guarantees
+; "sext(base) + sext(offset) == base + offset".
+define i32 @test_unsafe_nsw_indexed_add() {
+; CHECK-LABEL: test_unsafe_nsw_indexed_add:
+; CHECK: add x[[VAR32:[0-9]+]], {{x[0-9]+}}, _var32@PAGEOFF
+; CHECK: add w[[ADDR:[0-9]+]], w[[VAR32]], #32
+; CHECK-NOT: ubfx
+; CHECK: ldr w0, [x[[ADDR]]]
+  %addr_int = ptrtoint i32* @var32 to i32
+  %addr_plus_32 = add nsw i32 %addr_int, 32
+  %addr = inttoptr i32 %addr_plus_32 to i32*
+  %val = load i32, i32* %addr, align 4
+  ret i32 %val
+}
+
+; Because the addition may wrap, it is not safe to use "ldr w0, [xN, #32]" here.
+define i32 @test_unsafe_unscaled_add() {
+; CHECK-LABEL: test_unsafe_unscaled_add:
+; CHECK: add x[[VAR32:[0-9]+]], {{x[0-9]+}}, _var32@PAGEOFF
+; CHECK: add w[[ADDR:[0-9]+]], w[[VAR32]], #3
+; CHECK: ldr w0, [x[[ADDR]]]
+  %addr_int = ptrtoint i32* @var32 to i32
+  %addr_plus_3 = add i32 %addr_int, 3
+  %addr = inttoptr i32 %addr_plus_3 to i32*
+  %val = load i32, i32* %addr, align 1
+  ret i32 %val
+}
+
+; Since we've promised there is no unsigned overflow, @var32 must be at least
+; 32-bytes below 2^32, and we can use the load this time.
+define i32 @test_safe_unscaled_add() {
+; CHECK-LABEL: test_safe_unscaled_add:
+; CHECK: add x[[VAR32:[0-9]+]], {{x[0-9]+}}, _var32@PAGEOFF
+; CHECK: add w[[ADDR:[0-9]+]], w[[VAR32]], #3
+; CHECK: ldr w0, [x[[ADDR]]]
+  %addr_int = ptrtoint i32* @var32 to i32
+  %addr_plus_3 = add nuw i32 %addr_int, 3
+  %addr = inttoptr i32 %addr_plus_3 to i32*
+  %val = load i32, i32* %addr, align 1
+  ret i32 %val
+}
+
+; Promising nsw is not sufficient because the addressing mode basically
+; calculates "zext(base) + zext(offset)" and nsw only guarantees
+; "sext(base) + sext(offset) == base + offset".
+define i32 @test_unsafe_nsw_unscaled_add() {
+; CHECK-LABEL: test_unsafe_nsw_unscaled_add:
+; CHECK: add x[[VAR32:[0-9]+]], {{x[0-9]+}}, _var32@PAGEOFF
+; CHECK: add w[[ADDR:[0-9]+]], w[[VAR32]], #3
+; CHECK-NOT: ubfx
+; CHECK: ldr w0, [x[[ADDR]]]
+  %addr_int = ptrtoint i32* @var32 to i32
+  %addr_plus_3 = add nsw i32 %addr_int, 3
+  %addr = inttoptr i32 %addr_plus_3 to i32*
+  %val = load i32, i32* %addr, align 1
+  ret i32 %val
+}
+
+; Because the addition may wrap, it is not safe to use "ldur w0, [xN, #-3]"
+; here.
+define i32 @test_unsafe_negative_unscaled_add() {
+; CHECK-LABEL: test_unsafe_negative_unscaled_add:
+; CHECK: add x[[VAR32:[0-9]+]], {{x[0-9]+}}, _var32@PAGEOFF
+; CHECK: sub w[[ADDR:[0-9]+]], w[[VAR32]], #3
+; CHECK: ldr w0, [x[[ADDR]]]
+  %addr_int = ptrtoint i32* @var32 to i32
+  %addr_minus_3 = add i32 %addr_int, -3
+  %addr = inttoptr i32 %addr_minus_3 to i32*
+  %val = load i32, i32* %addr, align 1
+  ret i32 %val
+}
+
+define i8* @test_got_addr() {
+; CHECK-LABEL: test_got_addr:
+; CHECK: adrp x[[PAGE:[0-9]+]], _var_got@GOTPAGE
+; CHECK-OPT: ldr w0, [x[[PAGE]], _var_got@GOTPAGEOFF]
+; CHECK-FAST: ldr w[[TMP:[0-9]+]], [x[[PAGE]], _var_got@GOTPAGEOFF]
+; CHECK-FAST: and x0, x[[TMP]], #0xffffffff
+  ret i8* @var_got
+}
+
+define float @test_va_arg_f32(i8** %list) {
+; CHECK-LABEL: test_va_arg_f32:
+
+; CHECK: ldr w[[START:[0-9]+]], [x0]
+; CHECK: add [[AFTER:w[0-9]+]], w[[START]], #8
+; CHECK: str [[AFTER]], [x0]
+
+  ; Floating point arguments get promoted to double as per C99.
+; CHECK: ldr [[DBL:d[0-9]+]], [x[[START]]]
+; CHECK: fcvt s0, [[DBL]]
+  %res = va_arg i8** %list, float
+  ret float %res
+}
+
+; Interesting point is that the slot is 4 bytes.
+define i8 @test_va_arg_i8(i8** %list) {
+; CHECK-LABEL: test_va_arg_i8:
+
+; CHECK: ldr w[[START:[0-9]+]], [x0]
+; CHECK: add [[AFTER:w[0-9]+]], w[[START]], #4
+; CHECK: str [[AFTER]], [x0]
+
+  ; i8 gets promoted to int (again, as per C99).
+; CHECK: ldr w0, [x[[START]]]
+
+  %res = va_arg i8** %list, i8
+  ret i8 %res
+}
+
+; Interesting point is that the slot needs aligning (again, min size is 4
+; bytes).
+define i64 @test_va_arg_i64(i64** %list) {
+; CHECK-LABEL: test_va_arg_i64:
+
+  ; Update the list for the next user (minimum slot size is 4, but the actual
+  ; argument is 8 which had better be reflected!)
+; CHECK: ldr w[[UNALIGNED_START:[0-9]+]], [x0]
+; CHECK: add [[ALIGN_TMP:x[0-9]+]], x[[UNALIGNED_START]], #7
+; CHECK: and x[[START:[0-9]+]], [[ALIGN_TMP]], #0x1fffffff8
+; CHECK: add w[[AFTER:[0-9]+]], w[[START]], #8
+; CHECK: str w[[AFTER]], [x0]
+
+; CHECK: ldr x0, [x[[START]]]
+
+  %res = va_arg i64** %list, i64
+  ret i64 %res
+}
+
+declare void @bar(...)
+define void @test_va_call(i8 %l, i8 %r, float %in, i8* %ptr) {
+; CHECK-LABEL: test_va_call:
+; CHECK: add [[SUM:w[0-9]+]], {{w[0-9]+}}, w1
+
+; CHECK-DAG: str w2, [sp, #32]
+; CHECK-DAG: str xzr, [sp, #24]
+; CHECK-DAG: str s0, [sp, #16]
+; CHECK-DAG: str xzr, [sp, #8]
+; CHECK-DAG: str [[SUM]], [sp]
+
+  ; Add them to ensure real promotion occurs.
+  %sum = add i8 %l, %r
+  call void(...) @bar(i8 %sum, i64 0, float %in, double 0.0, i8* %ptr)
+  ret void
+}
+
+declare i8* @llvm.frameaddress(i32)
+
+define i8* @test_frameaddr() {
+; CHECK-LABEL: test_frameaddr:
+; CHECK-OPT: ldr x0, [x29]
+; CHECK-FAST: ldr [[TMP:x[0-9]+]], [x29]
+; CHECK-FAST: and x0, [[TMP]], #0xffffffff
+  %val = call i8* @llvm.frameaddress(i32 1)
+  ret i8* %val
+}
+
+declare i8* @llvm.returnaddress(i32)
+
+define i8* @test_toplevel_returnaddr() {
+; CHECK-LABEL: test_toplevel_returnaddr:
+; CHECK-OPT: mov x0, x30
+; CHECK-FAST: and x0, x30, #0xffffffff
+  %val = call i8* @llvm.returnaddress(i32 0)
+  ret i8* %val
+}
+
+define i8* @test_deep_returnaddr() {
+; CHECK-LABEL: test_deep_returnaddr:
+; CHECK: ldr x[[FRAME_REC:[0-9]+]], [x29]
+; CHECK-OPT: ldr x0, [x[[FRAME_REC]], #8]
+; CHECK-FAST: ldr [[TMP:x[0-9]+]], [x[[FRAME_REC]], #8]
+; CHECK-FAST: and x0, [[TMP]], #0xffffffff
+  %val = call i8* @llvm.returnaddress(i32 1)
+  ret i8* %val
+}
+
+define void @test_indirect_call(void()* %func) {
+; CHECK-LABEL: test_indirect_call:
+; CHECK: blr x0
+  call void() %func()
+  ret void
+}
+
+; Safe to use the unextended address here
+define void @test_indirect_safe_call(i32* %weird_funcs) {
+; CHECK-LABEL: test_indirect_safe_call:
+; CHECK: add w[[ADDR32:[0-9]+]], w0, #4
+; CHECK-OPT-NOT: ubfx
+; CHECK: blr x[[ADDR32]]
+  %addr = getelementptr i32, i32* %weird_funcs, i32 1
+  %func = bitcast i32* %addr to void()*
+  call void() %func()
+  ret void
+}
+
+declare void @simple()
+define void @test_simple_tail_call() {
+; CHECK-LABEL: test_simple_tail_call:
+; CHECK: b _simple
+  tail call void @simple()
+  ret void
+}
+
+define void @test_indirect_tail_call(void()* %func) {
+; CHECK-LABEL: test_indirect_tail_call:
+; CHECK: br x0
+  tail call void() %func()
+  ret void
+}
+
+; Safe to use the unextended address here
+define void @test_indirect_safe_tail_call(i32* %weird_funcs) {
+; CHECK-LABEL: test_indirect_safe_tail_call:
+; CHECK: add w[[ADDR32:[0-9]+]], w0, #4
+; CHECK-OPT-NOT: ubfx
+; CHECK-OPT: br x[[ADDR32]]
+  %addr = getelementptr i32, i32* %weird_funcs, i32 1
+  %func = bitcast i32* %addr to void()*
+  tail call void() %func()
+  ret void
+}
+
+; For the "armv7k" slice, Clang will be emitting some small structs as [N x
+; i32]. For ABI compatibility with arm64_32 these need to be passed in *X*
+; registers (e.g. [2 x i32] would be packed into a single register).
+
+define i32 @test_in_smallstruct_low([3 x i32] %in) {
+; CHECK-LABEL: test_in_smallstruct_low:
+; CHECK: mov x0, x1
+  %val = extractvalue [3 x i32] %in, 2
+  ret i32 %val
+}
+
+define i32 @test_in_smallstruct_high([3 x i32] %in) {
+; CHECK-LABEL: test_in_smallstruct_high:
+; CHECK: lsr x0, x0, #32
+  %val = extractvalue [3 x i32] %in, 1
+  ret i32 %val
+}
+
+; The 64-bit DarwinPCS ABI has the quirk that structs on the stack are always
+; 64-bit aligned. This must not happen for arm64_32 since othwerwise va_arg will
+; be incompatible with the armv7k ABI.
+define i32 @test_in_smallstruct_stack([8 x i64], i32, [3 x i32] %in) {
+; CHECK-LABEL: test_in_smallstruct_stack:
+; CHECK: ldr w0, [sp, #4]
+  %val = extractvalue [3 x i32] %in, 0
+  ret i32 %val
+}
+
+define [2 x i32] @test_ret_smallstruct([3 x i32] %in) {
+; CHECK-LABEL: test_ret_smallstruct:
+; CHECK: mov x0, #1
+; CHECK: movk x0, #2, lsl #32
+
+  ret [2 x i32] [i32 1, i32 2]
+}
+
+declare void @smallstruct_callee([4 x i32])
+define void @test_call_smallstruct() {
+; CHECK-LABEL: test_call_smallstruct:
+; CHECK: mov x0, #1
+; CHECK: movk x0, #2, lsl #32
+; CHECK: mov x1, #3
+; CHECK: movk x1, #4, lsl #32
+; CHECK: bl _smallstruct_callee
+
+  call void @smallstruct_callee([4 x i32] [i32 1, i32 2, i32 3, i32 4])
+  ret void
+}
+
+declare void @smallstruct_callee_stack([8 x i64], i32, [2 x i32])
+define void @test_call_smallstruct_stack() {
+; CHECK-LABEL: test_call_smallstruct_stack:
+; CHECK: mov [[VAL:x[0-9]+]], #1
+; CHECK: movk [[VAL]], #2, lsl #32
+; CHECK: stur [[VAL]], [sp, #4]
+
+  call void @smallstruct_callee_stack([8 x i64] undef, i32 undef, [2 x i32] [i32 1, i32 2])
+  ret void
+}
+
+declare [3 x i32] @returns_smallstruct()
+define i32 @test_use_smallstruct_low() {
+; CHECK-LABEL: test_use_smallstruct_low:
+; CHECK: bl _returns_smallstruct
+; CHECK: mov x0, x1
+
+  %struct = call [3 x i32] @returns_smallstruct()
+  %val = extractvalue [3 x i32] %struct, 2
+  ret i32 %val
+}
+
+define i32 @test_use_smallstruct_high() {
+; CHECK-LABEL: test_use_smallstruct_high:
+; CHECK: bl _returns_smallstruct
+; CHECK: lsr x0, x0, #32
+
+  %struct = call [3 x i32] @returns_smallstruct()
+  %val = extractvalue [3 x i32] %struct, 1
+  ret i32 %val
+}
+
+; If a small struct can't be allocated to x0-x7, the remaining registers should
+; be marked as unavailable and subsequent GPR arguments should also be on the
+; stack. Obviously the struct itself should be passed entirely on the stack.
+define i32 @test_smallstruct_padding([7 x i64], [4 x i32] %struct, i32 %in) {
+; CHECK-LABEL: test_smallstruct_padding:
+; CHECK-DAG: ldr [[IN:w[0-9]+]], [sp, #16]
+; CHECK-DAG: ldr [[LHS:w[0-9]+]], [sp]
+; CHECK: add w0, [[LHS]], [[IN]]
+  %lhs = extractvalue [4 x i32] %struct, 0
+  %sum = add i32 %lhs, %in
+  ret i32 %sum
+}
+
+declare void @take_small_smallstruct(i64, [1 x i32])
+define void @test_small_smallstruct() {
+; CHECK-LABEL: test_small_smallstruct:
+; CHECK-DAG: mov w0, #1
+; CHECK-DAG: mov w1, #2
+; CHECK: bl _take_small_smallstruct
+  call void @take_small_smallstruct(i64 1, [1 x i32] [i32 2])
+  ret void
+}
+
+define void @test_bare_frameaddr(i8** %addr) {
+; CHECK-LABEL: test_bare_frameaddr:
+; CHECK: add x[[LOCAL:[0-9]+]], sp, #{{[0-9]+}}
+; CHECK: str w[[LOCAL]],
+
+  %ptr = alloca i8
+  store i8* %ptr, i8** %addr, align 4
+  ret void
+}
+
+define void @test_sret_use([8 x i64]* sret %out) {
+; CHECK-LABEL: test_sret_use:
+; CHECK: str xzr, [x8]
+  %addr = getelementptr [8 x i64], [8 x i64]* %out, i32 0, i32 0
+  store i64 0, i64* %addr
+  ret void
+}
+
+define i64 @test_sret_call() {
+; CHECK-LABEL: test_sret_call:
+; CHECK: mov x8, sp
+; CHECK: bl _test_sret_use
+  %arr = alloca [8 x i64]
+  call void @test_sret_use([8 x i64]* sret %arr)
+
+  %addr = getelementptr [8 x i64], [8 x i64]* %arr, i32 0, i32 0
+  %val = load i64, i64* %addr
+  ret i64 %val
+}
+
+define double @test_constpool() {
+; CHECK-LABEL: test_constpool:
+; CHECK: adrp x[[PAGE:[0-9]+]], [[POOL:lCPI[0-9]+_[0-9]+]]@PAGE
+; CHECK: ldr d0, [x[[PAGE]], [[POOL]]@PAGEOFF]
+  ret double 1.0e-6
+}
+
+define i8* @test_blockaddress() {
+; CHECK-LABEL: test_blockaddress:
+; CHECK: [[BLOCK:Ltmp[0-9]+]]:
+; CHECK: adrp [[PAGE:x[0-9]+]], [[BLOCK]]@PAGE
+; CHECK: add x0, [[PAGE]], [[BLOCK]]@PAGEOFF
+  br label %dest
+dest:
+  ret i8* blockaddress(@test_blockaddress, %dest)
+}
+
+define i8* @test_indirectbr(i8* %dest) {
+; CHECK-LABEL: test_indirectbr:
+; CHECK: br x0
+  indirectbr i8* %dest, [label %true, label %false]
+
+true:
+  ret i8* blockaddress(@test_indirectbr, %true)
+false:
+  ret i8* blockaddress(@test_indirectbr, %false)
+}
+
+; ISelDAGToDAG tries to fold an offset FI load (in this case var+4) into the
+; actual load instruction. This needs to be done slightly carefully since we
+; claim the FI in the process -- it doesn't need extending.
+define float @test_frameindex_offset_load() {
+; CHECK-LABEL: test_frameindex_offset_load:
+; CHECK: ldr s0, [sp, #4]
+  %arr = alloca float, i32 4, align 8
+  %addr = getelementptr inbounds float, float* %arr, i32 1
+
+  %val = load float, float* %addr, align 4
+  ret float %val
+}
+
+define void @test_unaligned_frameindex_offset_store() {
+; CHECK-LABEL: test_unaligned_frameindex_offset_store:
+; CHECK: mov x[[TMP:[0-9]+]], sp
+; CHECK: orr w[[ADDR:[0-9]+]], w[[TMP]], #0x2
+; CHECK: mov [[VAL:w[0-9]+]], #42
+; CHECK: str [[VAL]], [x[[ADDR]]]
+  %arr = alloca [4 x i32]
+
+  %addr.int = ptrtoint [4 x i32]* %arr to i32
+  %addr.nextint = add nuw i32 %addr.int, 2
+  %addr.next = inttoptr i32 %addr.nextint to i32*
+  store i32 42, i32* %addr.next
+  ret void
+}
+
+
+define {i64, i64*} @test_pre_idx(i64* %addr) {
+; CHECK-LABEL: test_pre_idx:
+
+; CHECK: add w[[ADDR:[0-9]+]], w0, #8
+; CHECK: ldr x0, [x[[ADDR]]]
+  %addr.int = ptrtoint i64* %addr to i32
+  %addr.next.int = add nuw i32 %addr.int, 8
+  %addr.next = inttoptr i32 %addr.next.int to i64*
+  %val = load i64, i64* %addr.next
+
+  %tmp = insertvalue {i64, i64*} undef, i64 %val, 0
+  %res = insertvalue {i64, i64*} %tmp, i64* %addr.next, 1
+
+  ret {i64, i64*} %res
+}
+
+; Forming a post-indexed load is invalid here since the GEP needs to work when
+; %addr wraps round to 0.
+define {i64, i64*} @test_invalid_pre_idx(i64* %addr) {
+; CHECK-LABEL: test_invalid_pre_idx:
+; CHECK: add w1, w0, #8
+; CHECK: ldr x0, [x1]
+  %addr.next = getelementptr i64, i64* %addr, i32 1
+  %val = load i64, i64* %addr.next
+
+  %tmp = insertvalue {i64, i64*} undef, i64 %val, 0
+  %res = insertvalue {i64, i64*} %tmp, i64* %addr.next, 1
+
+  ret {i64, i64*} %res
+}
+
+declare void @callee([8 x i32]*)
+define void @test_stack_guard() ssp {
+; CHECK-LABEL: test_stack_guard:
+; CHECK: adrp x[[GUARD_GOTPAGE:[0-9]+]], ___stack_chk_guard@GOTPAGE
+; CHECK: ldr w[[GUARD_ADDR:[0-9]+]], [x[[GUARD_GOTPAGE]], ___stack_chk_guard@GOTPAGEOFF]
+; CHECK: ldr [[GUARD_VAL:w[0-9]+]], [x[[GUARD_ADDR]]]
+; CHECK: stur [[GUARD_VAL]], [x29, #[[GUARD_OFFSET:-[0-9]+]]]
+
+; CHECK: add x0, sp, #{{[0-9]+}}
+; CHECK: bl _callee
+
+; CHECK-OPT: adrp x[[GUARD_GOTPAGE:[0-9]+]], ___stack_chk_guard@GOTPAGE
+; CHECK-OPT: ldr w[[GUARD_ADDR:[0-9]+]], [x[[GUARD_GOTPAGE]], ___stack_chk_guard@GOTPAGEOFF]
+; CHECK-OPT: ldr [[GUARD_VAL:w[0-9]+]], [x[[GUARD_ADDR]]]
+; CHECK-OPT: ldur [[NEW_VAL:w[0-9]+]], [x29, #[[GUARD_OFFSET]]]
+; CHECK-OPT: cmp [[GUARD_VAL]], [[NEW_VAL]]
+; CHECK-OPT: b.ne [[FAIL:LBB[0-9]+_[0-9]+]]
+
+; CHECK-OPT: [[FAIL]]:
+; CHECK-OPT-NEXT: bl ___stack_chk_fail
+  %arr = alloca [8 x i32]
+  call void @callee([8 x i32]* %arr)
+  ret void
+}
+
+declare i32 @__gxx_personality_v0(...)
+declare void @eat_landingpad_args(i32, i8*, i32)
+@_ZTI8Whatever = external global i8
+define void @test_landingpad_marshalling() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+; CHECK-LABEL: test_landingpad_marshalling:
+; CHECK-OPT: mov x2, x1
+; CHECK-OPT: mov x1, x0
+; CHECK: bl _eat_landingpad_args
+  invoke void @callee([8 x i32]* undef) to label %done unwind label %lpad
+
+lpad:                                             ; preds = %entry
+  %exc = landingpad { i8*, i32 }
+          catch i8* @_ZTI8Whatever
+  %pointer = extractvalue { i8*, i32 } %exc, 0
+  %selector = extractvalue { i8*, i32 } %exc, 1
+  call void @eat_landingpad_args(i32 undef, i8* %pointer, i32 %selector)
+  ret void
+
+done:
+  ret void
+}
+
+define void @test_dynamic_stackalloc() {
+; CHECK-LABEL: test_dynamic_stackalloc:
+; CHECK: sub [[REG:x[0-9]+]], sp, #32
+; CHECK: mov sp, [[REG]]
+; CHECK-OPT-NOT: ubfx
+; CHECK: bl _callee
+  br label %next
+
+next:
+  %val = alloca [8 x i32]
+  call void @callee([8 x i32]* %val)
+  ret void
+}
+
+define void @test_asm_memory(i32* %base.addr) {
+; CHECK-LABEL: test_asm_memory:
+; CHECK: add w[[ADDR:[0-9]+]], w0, #4
+; CHECK: str wzr, [x[[ADDR]]
+  %addr = getelementptr i32, i32* %base.addr, i32 1
+  call void asm sideeffect "str wzr, $0", "*m"(i32* %addr)
+  ret void
+}
+
+define void @test_unsafe_asm_memory(i64 %val) {
+; CHECK-LABEL: test_unsafe_asm_memory:
+; CHECK: and x[[ADDR:[0-9]+]], x0, #0xffffffff
+; CHECK: str wzr, [x[[ADDR]]]
+  %addr_int = trunc i64 %val to i32
+  %addr = inttoptr i32 %addr_int to i32*
+  call void asm sideeffect "str wzr, $0", "*m"(i32* %addr)
+  ret void
+}
+
+define [9 x i8*] @test_demoted_return(i8* %in) {
+; CHECK-LABEL: test_demoted_return:
+; CHECK: str w0, [x8, #32]
+  %res = insertvalue [9 x i8*] undef, i8* %in, 8
+  ret [9 x i8*] %res
+}
+
+define i8* @test_inttoptr(i64 %in) {
+; CHECK-LABEL: test_inttoptr:
+; CHECK: and x0, x0, #0xffffffff
+  %res = inttoptr i64 %in to i8*
+  ret i8* %res
+}
+
+declare i32 @llvm.get.dynamic.area.offset.i32()
+define i32 @test_dynamic_area() {
+; CHECK-LABEL: test_dynamic_area:
+; CHECK: mov w0, wzr
+  %res = call i32 @llvm.get.dynamic.area.offset.i32()
+  ret i32 %res
+}
+
+define void @test_pointer_vec_store(<2 x i8*>* %addr) {
+; CHECK-LABEL: test_pointer_vec_store:
+; CHECK: str xzr, [x0]
+; CHECK-NOT: str
+; CHECK-NOT: stp
+
+  store <2 x i8*> zeroinitializer, <2 x i8*>* %addr, align 16
+  ret void
+}
+
+define <2 x i8*> @test_pointer_vec_load(<2 x i8*>* %addr) {
+; CHECK-LABEL: test_pointer_vec_load:
+; CHECK: ldr d[[TMP:[0-9]+]], [x0]
+; CHECK: ushll.2d v0, v[[TMP]], #0
+  %val = load <2 x i8*>, <2 x i8*>* %addr, align 16
+  ret <2 x i8*> %val
+}
+
+define void @test_inline_asm_mem_pointer(i32* %in) {
+; CHECK-LABEL: test_inline_asm_mem_pointer:
+; CHECK: str w0,
+  tail call void asm sideeffect "ldr x0, $0", "rm"(i32* %in)
+  ret void
+}
+
+
+define void @test_struct_hi(i32 %hi) nounwind {
+; CHECK-LABEL: test_struct_hi:
+; CHECK: mov w[[IN:[0-9]+]], w0
+; CHECK: bl _get_int
+; CHECK-FAST-NEXT: mov w0, w0
+; CHECK-NEXT: bfi x0, x[[IN]], #32, #32
+; CHECK-NEXT: bl _take_pair
+  %val.64 = call i64 @get_int()
+  %val.32 = trunc i64 %val.64 to i32
+
+  %pair.0 = insertvalue [2 x i32] undef, i32 %val.32, 0
+  %pair.1 = insertvalue [2 x i32] %pair.0, i32 %hi, 1
+  call void @take_pair([2 x i32] %pair.1)
+
+  ret void
+}
+declare void @take_pair([2 x i32])
+declare i64 @get_int()
+
+define i1 @test_icmp_ptr(i8* %in) {
+; CHECK-LABEL: test_icmp_ptr
+; CHECK: ubfx x0, x0, #31, #1
+  %res = icmp slt i8* %in, null
+  ret i1 %res
+}
+
+define void @test_multiple_icmp_ptr(i8* %l, i8* %r) {
+; CHECK-LABEL: test_multiple_icmp_ptr:
+; CHECK: tbnz w0, #31, [[FALSEBB:LBB[0-9]+_[0-9]+]]
+; CHECK: tbnz w1, #31, [[FALSEBB]]
+  %tst1 = icmp sgt i8* %l, inttoptr (i32 -1 to i8*)
+  %tst2 = icmp sgt i8* %r, inttoptr (i32 -1 to i8*)
+  %tst = and i1 %tst1, %tst2
+  br i1 %tst, label %true, label %false
+
+true:
+  call void(...) @bar()
+  ret void
+
+false:
+  ret void
+}
+
+define { [18 x i8] }* @test_gep_nonpow2({ [18 x i8] }* %a0, i32 %a1) {
+; CHECK-LABEL: test_gep_nonpow2:
+; CHECK-OPT:      mov w[[SIZE:[0-9]+]], #18
+; CHECK-OPT-NEXT: smaddl x0, w1, w[[SIZE]], x0
+; CHECK-OPT-NEXT: ret
+
+; CHECK-FAST:      mov w[[SIZE:[0-9]+]], #18
+; CHECK-FAST-NEXT: smaddl [[TMP:x[0-9]+]], w1, w[[SIZE]], x0
+; CHECK-FAST-NEXT: and x0, [[TMP]], #0xffffffff
+; CHECK-FAST-NEXT: ret
+  %tmp0 = getelementptr inbounds { [18 x i8] }, { [18 x i8] }* %a0, i32 %a1
+  ret { [18 x i8] }* %tmp0
+}
+
+define void @test_bzero(i64 %in)  {
+; CHECK-LABEL: test_bzero:
+; CHECK-DAG: lsr x1, x0, #32
+; CHECK-DAG: and x0, x0, #0xffffffff
+; CHECK: bl _bzero
+
+  %ptr.i32 = trunc i64 %in to i32
+  %size.64 = lshr i64 %in, 32
+  %size = trunc i64 %size.64 to i32
+  %ptr = inttoptr i32 %ptr.i32 to i8*
+  tail call void @llvm.memset.p0i8.i32(i8* align 4 %ptr, i8 0, i32 %size, i1 false)
+  ret void
+}
+
+declare void @llvm.memset.p0i8.i32(i8* nocapture writeonly, i8, i32, i1)
diff --git a/llvm/test/CodeGen/AArch64/basic-pic.ll b/llvm/test/CodeGen/AArch64/basic-pic.ll
index 8765a6d12..e5faf5218 100644
--- a/llvm/test/CodeGen/AArch64/basic-pic.ll
+++ b/llvm/test/CodeGen/AArch64/basic-pic.ll
@@ -7,7 +7,7 @@ define i32 @get_globalvar() {
 
   %val = load i32, i32* @var
 ; CHECK: adrp x[[GOTHI:[0-9]+]], :got:var
-; CHECK: ldr x[[GOTLOC:[0-9]+]], [x[[GOTHI]], {{#?}}:got_lo12:var]
+; CHECK: ldr x[[GOTLOC:[0-9]+]], [x[[GOTHI]], :got_lo12:var]
 ; CHECK: ldr w0, [x[[GOTLOC]]]
 
   ret i32 %val
@@ -18,7 +18,7 @@ define i32* @get_globalvaraddr() {
 
   %val = load i32, i32* @var
 ; CHECK: adrp x[[GOTHI:[0-9]+]], :got:var
-; CHECK: ldr x0, [x[[GOTHI]], {{#?}}:got_lo12:var]
+; CHECK: ldr x0, [x[[GOTHI]], :got_lo12:var]
 
   ret i32* @var
 }
@@ -30,7 +30,7 @@ define i32 @get_hiddenvar() {
 
   %val = load i32, i32* @hiddenvar
 ; CHECK: adrp x[[HI:[0-9]+]], hiddenvar
-; CHECK: ldr w0, [x[[HI]], {{#?}}:lo12:hiddenvar]
+; CHECK: ldr w0, [x[[HI]], :lo12:hiddenvar]
 
   ret i32 %val
 }
@@ -40,7 +40,7 @@ define i32* @get_hiddenvaraddr() {
 
   %val = load i32, i32* @hiddenvar
 ; CHECK: adrp [[HI:x[0-9]+]], hiddenvar
-; CHECK: add x0, [[HI]], {{#?}}:lo12:hiddenvar
+; CHECK: add x0, [[HI]], :lo12:hiddenvar
 
   ret i32* @hiddenvar
 }
@@ -50,5 +50,5 @@ define void()* @get_func() {
 
   ret void()* bitcast(void()*()* @get_func to void()*)
 ; CHECK: adrp x[[GOTHI:[0-9]+]], :got:get_func
-; CHECK: ldr x0, [x[[GOTHI]], {{#?}}:got_lo12:get_func]
+; CHECK: ldr x0, [x[[GOTHI]], :got_lo12:get_func]
 }
diff --git a/llvm/test/CodeGen/AArch64/bitfield-insert.ll b/llvm/test/CodeGen/AArch64/bitfield-insert.ll
index 0753113fc..3015844c9 100644
--- a/llvm/test/CodeGen/AArch64/bitfield-insert.ll
+++ b/llvm/test/CodeGen/AArch64/bitfield-insert.ll
@@ -265,12 +265,12 @@ define void @test_32bit_opnd1_better(i32* %existing, i32* %new) {
 define i32 @test_nouseful_bits(i8 %a, i32 %b) {
 ; CHECK-LABEL: test_nouseful_bits:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, w0
-; CHECK-NEXT:    bfi w8, w8, #8, #24
-; CHECK-NEXT:    mov w9, w0
-; CHECK-NEXT:    bfi w9, w8, #8, #24
-; CHECK-NEXT:    bfi w0, w9, #8, #24
-; CHECK-NEXT:    lsl w0, w0, #8
+; CHECK-NEXT:    and w8, w0, #0xff
+; CHECK-NEXT:    lsl w8, w8, #8
+; CHECK-NEXT:    mov w9, w8
+; CHECK-NEXT:    bfxil w9, w0, #0, #8
+; CHECK-NEXT:    bfi w8, w9, #16, #16
+; CHECK-NEXT:    mov w0, w8
 ; CHECK-NEXT:    ret
   %conv = zext i8 %a to i32     ;   0  0  0  A
   %shl = shl i32 %b, 8          ;   B2 B1 B0 0
diff --git a/llvm/test/CodeGen/AArch64/branch-target-enforcement-indirect-calls.ll b/llvm/test/CodeGen/AArch64/branch-target-enforcement-indirect-calls.ll
index d7e3748d2..3fb9e320f 100644
--- a/llvm/test/CodeGen/AArch64/branch-target-enforcement-indirect-calls.ll
+++ b/llvm/test/CodeGen/AArch64/branch-target-enforcement-indirect-calls.ll
@@ -1,4 +1,7 @@
 ; RUN: llc -mtriple aarch64--none-eabi -mattr=+bti < %s | FileCheck %s
+; RUN: llc -mtriple aarch64--none-eabi -global-isel -global-isel-abort=2 -pass-remarks-missed=gisel* -mattr=+bti %s -verify-machineinstrs -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,FALLBACK
+
+; FALLBACK: remark: <unknown>:0:0: unable to translate instruction: call: '  tail call void %p()' (in function: bti_enabled)
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64-arm-none-eabi"
diff --git a/llvm/test/CodeGen/AArch64/bti-branch-relaxation.ll b/llvm/test/CodeGen/AArch64/bti-branch-relaxation.ll
new file mode 100644
index 000000000..93cbc3b85
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/bti-branch-relaxation.ll
@@ -0,0 +1,64 @@
+; RUN: llc %s -o - | FileCheck %s
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64-unknown-unknown-eabi"
+
+; Function Attrs: nounwind
+define dso_local void @f(i64 %v) local_unnamed_addr #0 {
+entry:
+  %call = tail call i32 bitcast (i32 (...)* @test to i32 ()*)() #0
+  %and = and i32 %call, 2
+  %cmp = icmp eq i32 %and, 0
+  br i1 %cmp, label %if.then, label %if.else
+; CHECK: tbz
+; CHECK-NEXT: b
+if.then:                                          ; preds = %entry
+  switch i64 %v, label %sw.epilog [
+    i64 0, label %sw.bb
+    i64 1, label %sw.bb1
+    i64 2, label %sw.bb2
+    i64 3, label %sw.bb3
+  ]
+
+sw.bb:                                            ; preds = %if.then
+  tail call void bitcast (void (...)* @g0 to void ()*)() #0
+  br label %sw.bb1
+
+sw.bb1:                                           ; preds = %if.then, %sw.bb
+  tail call void bitcast (void (...)* @g1 to void ()*)() #0
+  br label %sw.bb2
+
+sw.bb2:                                           ; preds = %if.then, %sw.bb1
+  tail call void bitcast (void (...)* @g2 to void ()*)() #0
+  br label %sw.bb3
+
+sw.bb3:                                           ; preds = %if.then, %sw.bb2
+  tail call void bitcast (void (...)* @g3 to void ()*)() #0
+  br label %sw.epilog
+
+sw.epilog:                                        ; preds = %sw.bb3, %if.then
+  %dummy = tail call i64 @llvm.aarch64.space(i32 32700, i64 %v)
+  br label %if.end
+
+if.else:                                          ; preds = %entry
+  tail call void bitcast (void (...)* @e to void ()*)() #0
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %sw.epilog
+  ret void
+}
+
+declare dso_local i32 @test(...) local_unnamed_addr #0
+
+declare dso_local void @g0(...) local_unnamed_addr #0
+
+declare dso_local void @g1(...) local_unnamed_addr #0
+
+declare dso_local void @g2(...) local_unnamed_addr #0
+
+declare dso_local void @g3(...) local_unnamed_addr #0
+
+declare dso_local void @e(...) local_unnamed_addr #0
+
+declare dso_local i64 @llvm.aarch64.space(i32, i64) local_unnamed_addr #0
+
+attributes #0 = { nounwind "branch-target-enforcement" "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="generic" "target-features"="+neon,+v8.5a" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/AArch64/callbr-asm-obj-file.ll b/llvm/test/CodeGen/AArch64/callbr-asm-obj-file.ll
index 579158568..345c3f9c4 100644
--- a/llvm/test/CodeGen/AArch64/callbr-asm-obj-file.ll
+++ b/llvm/test/CodeGen/AArch64/callbr-asm-obj-file.ll
@@ -6,11 +6,11 @@
 
 @l = common hidden local_unnamed_addr global i32 0, align 4
 
-; CHECK-LABEL: 0000000000000000 test1:
-; CHECK-LABEL: 0000000000000018 $d.1:
-; CHECK-LABEL: 0000000000000020 $x.2:
+; CHECK-LABEL: test1:
+; CHECK-LABEL: $d.1:
+; CHECK-LABEL: $x.2:
 ; CHECK-NEXT:    b #16 <$x.4+0x4>
-; CHECK-LABEL: 000000000000002c $x.4:
+; CHECK-LABEL: $x.4:
 ; CHECK-NEXT:    b #4 <$x.4+0x4>
 ; CHECK-NEXT:    mov w0, wzr
 ; CHECK-NEXT:    ldr x30, [sp], #16
@@ -40,11 +40,11 @@ declare dso_local i32 @g(...) local_unnamed_addr
 
 declare dso_local i32 @i(...) local_unnamed_addr
 
-; CHECK-LABEL: 000000000000003c test2:
-; CHECK:         bl #0 <test2+0x18>
-; CHECK-LABEL: 0000000000000064 $d.5:
-; CHECK-LABEL: 000000000000006c $x.6:
-; CHECK-NEXT:    b #-24 <test2+0x18>
+; CHECK-LABEL: test2:
+; CHECK:         bl #0 <test2+0x10>
+; CHECK-LABEL: $d.5:
+; CHECK-LABEL: $x.6:
+; CHECK-NEXT:    b #16 <$x.8+0x4>
 define hidden i32 @test2() local_unnamed_addr {
   %1 = load i32, i32* @l, align 4
   %2 = icmp eq i32 %1, 0
@@ -57,7 +57,7 @@ define hidden i32 @test2() local_unnamed_addr {
 
 6:                                                ; preds = %3
   callbr void asm sideeffect "1: nop\0A\09.quad b\0A\09b ${1:l}\0A\09.quad ${0:c}", "i,X"(i32* null, i8* blockaddress(@test2, %7))
-          to label %10 [label %9]
+          to label %10 [label %7]
 
 7:                                                ; preds = %3
   %8 = tail call i32 bitcast (i32 (...)* @i to i32 ()*)()
@@ -70,11 +70,11 @@ define hidden i32 @test2() local_unnamed_addr {
   ret i32 undef
 }
 
-; CHECK-LABEL: 0000000000000084 test3:
-; CHECK-LABEL: 00000000000000a8 $d.9:
-; CHECK-LABEL: 00000000000000b0 $x.10:
-; CHECK-NEXT:    b #20 <$x.12+0x8>
-; CHECK-LABEL: 00000000000000bc $x.12:
+; CHECK-LABEL: test3:
+; CHECK-LABEL: $d.9:
+; CHECK-LABEL: $x.10:
+; CHECK-NEXT:    b #-20 <test3+0x18>
+; CHECK-LABEL: $x.12:
 ; CHECK-NEXT:    b #4 <$x.12+0x4>
 ; CHECK-NEXT:    mov w0, wzr
 ; CHECK-NEXT:    ldr x30, [sp], #16
diff --git a/llvm/test/CodeGen/AArch64/cfguard-checks.ll b/llvm/test/CodeGen/AArch64/cfguard-checks.ll
new file mode 100644
index 000000000..5ebe1dd13
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/cfguard-checks.ll
@@ -0,0 +1,147 @@
+; RUN: llc < %s -mtriple=aarch64-pc-windows-msvc | FileCheck %s
+; Control Flow Guard is currently only available on Windows
+
+; Test that Control Flow Guard checks are correctly added when required.
+
+
+declare i32 @target_func()
+
+
+; Test that Control Flow Guard checks are not added on calls with the "guard_nocf" attribute.
+define i32 @func_guard_nocf() {
+entry:
+  %func_ptr = alloca i32 ()*, align 8
+  store i32 ()* @target_func, i32 ()** %func_ptr, align 8
+  %0 = load i32 ()*, i32 ()** %func_ptr, align 8
+  %1 = call i32 %0() #0
+  ret i32 %1
+
+  ; CHECK-LABEL: func_guard_nocf
+  ; CHECK:       adrp x8, target_func
+	; CHECK:       add x8, x8, target_func
+  ; CHECK-NOT:   __guard_check_icall_fptr
+	; CHECK:       blr x8
+}
+attributes #0 = { "guard_nocf" }
+
+
+; Test that Control Flow Guard checks are added even at -O0.
+define i32 @func_optnone_cf() #1 {
+entry:
+  %func_ptr = alloca i32 ()*, align 8
+  store i32 ()* @target_func, i32 ()** %func_ptr, align 8
+  %0 = load i32 ()*, i32 ()** %func_ptr, align 8
+  %1 = call i32 %0()
+  ret i32 %1
+
+  ; The call to __guard_check_icall_fptr should come immediately before the call to the target function.
+  ; CHECK-LABEL: func_optnone_cf
+	; CHECK:        adrp x8, __guard_check_icall_fptr
+	; CHECK:        add x9, x8, __guard_check_icall_fptr
+	; CHECK:        adrp x8, target_func
+	; CHECK:        add x8, x8, target_func
+	; CHECK:        ldr x9, [x9]
+	; CHECK:        mov x15, x8
+	; CHECK:        blr x9
+	; CHECK-NEXT:   blr x8
+}
+attributes #1 = { noinline optnone }
+
+
+; Test that Control Flow Guard checks are correctly added in optimized code (common case).
+define i32 @func_cf() {
+entry:
+  %func_ptr = alloca i32 ()*, align 8
+  store i32 ()* @target_func, i32 ()** %func_ptr, align 8
+  %0 = load i32 ()*, i32 ()** %func_ptr, align 8
+  %1 = call i32 %0()
+  ret i32 %1
+
+  ; The call to __guard_check_icall_fptr should come immediately before the call to the target function.
+  ; CHECK-LABEL: func_cf
+  ; CHECK:        adrp x8, __guard_check_icall_fptr
+	; CHECK:        ldr x9, [x8, __guard_check_icall_fptr]
+	; CHECK:        adrp x8, target_func
+	; CHECK:        add x8, x8, target_func
+	; CHECK:        mov x15, x8
+	; CHECK: 	     	blr x9
+	; CHECK-NEXT:   blr x8
+}
+
+
+; Test that Control Flow Guard checks are correctly added on invoke instructions.
+define i32 @func_cf_invoke() personality i8* bitcast (void ()* @h to i8*) {
+entry:
+  %0 = alloca i32, align 4
+  %func_ptr = alloca i32 ()*, align 8
+  store i32 ()* @target_func, i32 ()** %func_ptr, align 8
+  %1 = load i32 ()*, i32 ()** %func_ptr, align 8
+  %2 = invoke i32 %1()
+          to label %invoke.cont unwind label %lpad
+invoke.cont:                                      ; preds = %entry
+  ret i32 %2
+
+lpad:                                             ; preds = %entry
+  %tmp = landingpad { i8*, i32 }
+          catch i8* null
+  ret i32 -1
+
+  ; The call to __guard_check_icall_fptr should come immediately before the call to the target function.
+  ; CHECK-LABEL: func_cf_invoke
+  ; CHECK:        adrp x8, __guard_check_icall_fptr
+	; CHECK:        ldr x9, [x8, __guard_check_icall_fptr]
+	; CHECK:        adrp x8, target_func
+	; CHECK:        add x8, x8, target_func
+	; CHECK:        mov x15, x8
+	; CHECK:        blr x9
+  ; CHECK-NEXT:   .Ltmp0:
+	; CHECK-NEXT:   blr x8
+  ; CHECK:       ; %invoke.cont
+  ; CHECK:       ; %lpad
+}
+
+declare void @h()
+
+
+; Test that longjmp targets have public labels and are included in the .gljmp section.
+%struct._SETJMP_FLOAT128 = type { [2 x i64] }
+@buf1 = internal global [16 x %struct._SETJMP_FLOAT128] zeroinitializer, align 16
+
+define i32 @func_cf_setjmp() {
+  %1 = alloca i32, align 4
+  %2 = alloca i32, align 4
+  store i32 0, i32* %1, align 4
+  store i32 -1, i32* %2, align 4
+  %3 = call i8* @llvm.frameaddress(i32 0)
+  %4 = call i32 @_setjmp(i8* bitcast ([16 x %struct._SETJMP_FLOAT128]* @buf1 to i8*), i8* %3) #2
+
+  ; CHECK-LABEL: func_cf_setjmp
+  ; CHECK:       bl _setjmp
+  ; CHECK-NEXT:  $cfgsj_func_cf_setjmp0:
+
+  %5 = call i8* @llvm.frameaddress(i32 0)
+  %6 = call i32 @_setjmp(i8* bitcast ([16 x %struct._SETJMP_FLOAT128]* @buf1 to i8*), i8* %5) #3
+
+  ; CHECK:       bl _setjmp
+  ; CHECK-NEXT:  $cfgsj_func_cf_setjmp1:
+
+  store i32 1, i32* %2, align 4
+  %7 = load i32, i32* %2, align 4
+  ret i32 %7
+
+  ; CHECK:       .section .gljmp$y,"dr"
+  ; CHECK-NEXT:  .symidx $cfgsj_func_cf_setjmp0
+  ; CHECK-NEXT:  .symidx $cfgsj_func_cf_setjmp1
+}
+
+declare i8* @llvm.frameaddress(i32)
+
+; Function Attrs: returns_twice
+declare dso_local i32 @_setjmp(i8*, i8*) #2
+
+attributes #2 = { returns_twice }
+attributes #3 = { returns_twice }
+
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 2, !"cfguard", i32 2}
diff --git a/llvm/test/CodeGen/AArch64/cfguard-module-flag.ll b/llvm/test/CodeGen/AArch64/cfguard-module-flag.ll
new file mode 100644
index 000000000..8eb77b6ad
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/cfguard-module-flag.ll
@@ -0,0 +1,25 @@
+
+; RUN: llc < %s -mtriple=aarch64-pc-windows-msvc | FileCheck %s
+; Control Flow Guard is currently only available on Windows
+
+; Test that Control Flow Guard checks are not added in modules with the
+; cfguard=1 flag (emit tables but no checks).
+
+
+declare void @target_func()
+
+define void @func_in_module_without_cfguard() #0 {
+entry:
+  %func_ptr = alloca void ()*, align 8
+  store void ()* @target_func, void ()** %func_ptr, align 8
+  %0 = load void ()*, void ()** %func_ptr, align 8
+
+  call void %0()
+  ret void
+
+  ; CHECK-NOT: __guard_check_icall_fptr
+  ; CHECK-NOT: __guard_dispatch_icall_fptr
+}
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 2, !"cfguard", i32 1}
diff --git a/llvm/test/CodeGen/AArch64/cgp-usubo.ll b/llvm/test/CodeGen/AArch64/cgp-usubo.ll
index 58821c395..4dfd427d6 100644
--- a/llvm/test/CodeGen/AArch64/cgp-usubo.ll
+++ b/llvm/test/CodeGen/AArch64/cgp-usubo.ll
@@ -151,10 +151,10 @@ end:
 define i1 @usubo_ult_cmp_dominates_i64(i64 %x, i64 %y, i64* %p, i1 %cond) nounwind {
 ; CHECK-LABEL: usubo_ult_cmp_dominates_i64:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    str x22, [sp, #-48]! // 8-byte Folded Spill
-; CHECK-NEXT:    stp x21, x20, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    str x30, [sp, #-48]! // 8-byte Folded Spill
+; CHECK-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    mov w20, w3
-; CHECK-NEXT:    stp x19, x30, [sp, #32] // 16-byte Folded Spill
+; CHECK-NEXT:    stp x22, x21, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    tbz w3, #0, .LBB8_3
 ; CHECK-NEXT:  // %bb.1: // %t
 ; CHECK-NEXT:    cmp x0, x1
@@ -172,9 +172,9 @@ define i1 @usubo_ult_cmp_dominates_i64(i64 %x, i64 %y, i64* %p, i1 %cond) nounwi
 ; CHECK-NEXT:  .LBB8_3: // %f
 ; CHECK-NEXT:    and w0, w20, #0x1
 ; CHECK-NEXT:  .LBB8_4: // %f
-; CHECK-NEXT:    ldp x19, x30, [sp, #32] // 16-byte Folded Reload
-; CHECK-NEXT:    ldp x21, x20, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x22, [sp], #48 // 8-byte Folded Reload
+; CHECK-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-NEXT:    ldp x22, x21, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x30, [sp], #48 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
 entry:
   br i1 %cond, label %t, label %f
diff --git a/llvm/test/CodeGen/AArch64/cls.ll b/llvm/test/CodeGen/AArch64/cls.ll
new file mode 100644
index 000000000..f17ccf7d6
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/cls.ll
@@ -0,0 +1,20 @@
+; RUN: llc -mtriple=aarch64 %s -o - | FileCheck %s
+
+; @llvm.aarch64.cls must be directly translated into the 'cls' instruction
+
+; CHECK-LABEL: cls
+; CHECK: cls [[REG:w[0-9]+]], [[REG]]
+define i32 @cls(i32 %t) {
+  %cls.i = call i32 @llvm.aarch64.cls(i32 %t)
+  ret i32 %cls.i
+}
+
+; CHECK-LABEL: cls64
+; CHECK: cls [[REG:x[0-9]+]], [[REG]]
+define i32 @cls64(i64 %t) {
+  %cls.i = call i32 @llvm.aarch64.cls64(i64 %t)
+  ret i32 %cls.i
+}
+
+declare i32 @llvm.aarch64.cls(i32) nounwind
+declare i32 @llvm.aarch64.cls64(i64) nounwind
diff --git a/llvm/test/CodeGen/AArch64/cmpxchg-O0.ll b/llvm/test/CodeGen/AArch64/cmpxchg-O0.ll
index bd3d328ec..bfb7b5809 100644
--- a/llvm/test/CodeGen/AArch64/cmpxchg-O0.ll
+++ b/llvm/test/CodeGen/AArch64/cmpxchg-O0.ll
@@ -87,10 +87,8 @@ define { i128, i1 } @test_cmpxchg_128(i128* %addr, i128 %desired, i128 %new) nou
 define {i128, i1} @test_cmpxchg_128_unsplit(i128* %addr) {
 ; CHECK-LABEL: test_cmpxchg_128_unsplit:
 ; CHECK:     add x[[VAR128:[0-9]+]], {{x[0-9]+}}, :lo12:var128
-; CHECK:     ldr [[DESIRED_HI:x[0-9]+]], [x[[VAR128]], #8]
-; CHECK:     ldr [[DESIRED_LO:x[0-9]+]], [x[[VAR128]]]
-; CHECK:     ldr [[NEW_HI:x[0-9]+]], [x[[VAR128]], #8]
-; CHECK:     ldr [[NEW_LO:x[0-9]+]], [x[[VAR128]]]
+; CHECK:     ldp [[DESIRED_LO:x[0-9]+]], [[DESIRED_HI:x[0-9]+]], [x[[VAR128]]]
+; CHECK:     ldp [[NEW_LO:x[0-9]+]], [[NEW_HI:x[0-9]+]], [x[[VAR128]]]
 ; CHECK: [[RETRY:.LBB[0-9]+_[0-9]+]]:
 ; CHECK:     ldaxp [[OLD_LO:x[0-9]+]], [[OLD_HI:x[0-9]+]], [x0]
 ; CHECK:     cmp [[OLD_LO]], [[DESIRED_LO]]
diff --git a/llvm/test/CodeGen/AArch64/code-model-large-abs.ll b/llvm/test/CodeGen/AArch64/code-model-large-abs.ll
index 82169acc3..e7c2db1ac 100644
--- a/llvm/test/CodeGen/AArch64/code-model-large-abs.ll
+++ b/llvm/test/CodeGen/AArch64/code-model-large-abs.ll
@@ -59,3 +59,14 @@ define i64 @global_i64() {
 ; CHECK: movk x[[ADDR_REG]], #:abs_g3:var64
 ; CHECK: ldr x0, [x[[ADDR_REG]]]
 }
+
+define <2 x i64> @constpool() {
+; CHECK-LABEL: constpool:
+  ret <2 x i64> <i64 123456789, i64 987654321100>
+
+; CHECK: movz x[[ADDR_REG:[0-9]+]], #:abs_g0_nc:[[CPADDR:.LCPI[0-9]+_[0-9]+]]
+; CHECK: movk x[[ADDR_REG]], #:abs_g1_nc:[[CPADDR]]
+; CHECK: movk x[[ADDR_REG]], #:abs_g2_nc:[[CPADDR]]
+; CHECK: movk x[[ADDR_REG]], #:abs_g3:[[CPADDR]]
+; CHECK: ldr q0, [x[[ADDR_REG]]]
+}
diff --git a/llvm/test/CodeGen/AArch64/consthoist-gep.ll b/llvm/test/CodeGen/AArch64/consthoist-gep.ll
index 6b68fd57a..507f949b6 100644
--- a/llvm/test/CodeGen/AArch64/consthoist-gep.ll
+++ b/llvm/test/CodeGen/AArch64/consthoist-gep.ll
@@ -65,7 +65,7 @@ bb19:                                             ; preds = %bb3, %bb
   ret void
 }
 
-attributes #0 = { norecurse nounwind optsize ssp "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { norecurse nounwind optsize ssp "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.module.flags = !{!0, !1}
 !llvm.ident = !{!2}
diff --git a/llvm/test/CodeGen/AArch64/cpus.ll b/llvm/test/CodeGen/AArch64/cpus.ll
index d0ca70694..2971cb64a 100644
--- a/llvm/test/CodeGen/AArch64/cpus.ll
+++ b/llvm/test/CodeGen/AArch64/cpus.ll
@@ -6,13 +6,15 @@
 ; RUN: llc < %s -mtriple=arm64-unknown-unknown -mcpu=cortex-a53 2>&1 | FileCheck %s
 ; RUN: llc < %s -mtriple=arm64-unknown-unknown -mcpu=cortex-a55 2>&1 | FileCheck %s
 ; RUN: llc < %s -mtriple=arm64-unknown-unknown -mcpu=cortex-a57 2>&1 | FileCheck %s
+; RUN: llc < %s -mtriple=arm64-unknown-unknown -mcpu=cortex-a65 2>&1 | FileCheck %s
+; RUN: llc < %s -mtriple=arm64-unknown-unknown -mcpu=cortex-a65ae 2>&1 | FileCheck %s
 ; RUN: llc < %s -mtriple=arm64-unknown-unknown -mcpu=cortex-a72 2>&1 | FileCheck %s
 ; RUN: llc < %s -mtriple=arm64-unknown-unknown -mcpu=cortex-a73 2>&1 | FileCheck %s
 ; RUN: llc < %s -mtriple=arm64-unknown-unknown -mcpu=cortex-a75 2>&1 | FileCheck %s
 ; RUN: llc < %s -mtriple=arm64-unknown-unknown -mcpu=cortex-a76ae 2>&1 | FileCheck %s
 ; RUN: llc < %s -mtriple=arm64-unknown-unknown -mcpu=cortex-a76 2>&1 | FileCheck %s
-; RUN: llc < %s -mtriple=arm64-unknown-unknown -mcpu=exynos-m1 2>&1 | FileCheck %s
-; RUN: llc < %s -mtriple=arm64-unknown-unknown -mcpu=exynos-m2 2>&1 | FileCheck %s
+; RUN: llc < %s -mtriple=arm64-unknown-unknown -mcpu=neoverse-e1 2>&1 | FileCheck %s
+; RUN: llc < %s -mtriple=arm64-unknown-unknown -mcpu=neoverse-n1 2>&1 | FileCheck %s
 ; RUN: llc < %s -mtriple=arm64-unknown-unknown -mcpu=exynos-m3 2>&1 | FileCheck %s
 ; RUN: llc < %s -mtriple=arm64-unknown-unknown -mcpu=exynos-m4 2>&1 | FileCheck %s
 ; RUN: llc < %s -mtriple=arm64-unknown-unknown -mcpu=exynos-m5 2>&1 | FileCheck %s
diff --git a/llvm/test/CodeGen/AArch64/csr-split.ll b/llvm/test/CodeGen/AArch64/csr-split.ll
new file mode 100644
index 000000000..1bee7f05a
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/csr-split.ll
@@ -0,0 +1,224 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -verify-machineinstrs -mtriple=aarch64-unknown-linux < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -mtriple=arm64-apple-ios < %s | FileCheck %s --check-prefix=CHECK-APPLE
+
+; Check CSR split can work properly for tests below.
+
+@a = common dso_local local_unnamed_addr global i32 0, align 4
+
+define dso_local signext i32 @test1(i32* %b) local_unnamed_addr  {
+; CHECK-LABEL: test1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    stp x30, x19, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset w19, -8
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    adrp x8, a
+; CHECK-NEXT:    ldrsw x8, [x8, :lo12:a]
+; CHECK-NEXT:    cmp x8, x0
+; CHECK-NEXT:    b.eq .LBB0_2
+; CHECK-NEXT:  // %bb.1: // %if.end
+; CHECK-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB0_2: // %if.then
+; CHECK-NEXT:    mov x19, x0
+; CHECK-NEXT:    bl callVoid
+; CHECK-NEXT:    mov x0, x19
+; CHECK-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    b callNonVoid
+;
+; CHECK-APPLE-LABEL: test1:
+; CHECK-APPLE:       ; %bb.0: ; %entry
+; CHECK-APPLE-NEXT:    stp x20, x19, [sp, #-32]! ; 16-byte Folded Spill
+; CHECK-APPLE-NEXT:    stp x29, x30, [sp, #16] ; 16-byte Folded Spill
+; CHECK-APPLE-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-APPLE-NEXT:    .cfi_offset w30, -8
+; CHECK-APPLE-NEXT:    .cfi_offset w29, -16
+; CHECK-APPLE-NEXT:    .cfi_offset w19, -24
+; CHECK-APPLE-NEXT:    .cfi_offset w20, -32
+; CHECK-APPLE-NEXT:  Lloh0:
+; CHECK-APPLE-NEXT:    adrp x8, _a@PAGE
+; CHECK-APPLE-NEXT:  Lloh1:
+; CHECK-APPLE-NEXT:    ldrsw x8, [x8, _a@PAGEOFF]
+; CHECK-APPLE-NEXT:    cmp x8, x0
+; CHECK-APPLE-NEXT:    b.eq LBB0_2
+; CHECK-APPLE-NEXT:  ; %bb.1: ; %if.end
+; CHECK-APPLE-NEXT:    ldp x29, x30, [sp, #16] ; 16-byte Folded Reload
+; CHECK-APPLE-NEXT:    ldp x20, x19, [sp], #32 ; 16-byte Folded Reload
+; CHECK-APPLE-NEXT:    ret
+; CHECK-APPLE-NEXT:  LBB0_2: ; %if.then
+; CHECK-APPLE-NEXT:    mov x19, x0
+; CHECK-APPLE-NEXT:    bl _callVoid
+; CHECK-APPLE-NEXT:    ldp x29, x30, [sp, #16] ; 16-byte Folded Reload
+; CHECK-APPLE-NEXT:    mov x0, x19
+; CHECK-APPLE-NEXT:    ldp x20, x19, [sp], #32 ; 16-byte Folded Reload
+; CHECK-APPLE-NEXT:    b _callNonVoid
+; CHECK-APPLE-NEXT:    .loh AdrpLdr Lloh0, Lloh1
+entry:
+  %0 = load i32, i32* @a, align 4, !tbaa !2
+  %conv = sext i32 %0 to i64
+  %1 = inttoptr i64 %conv to i32*
+  %cmp = icmp eq i32* %1, %b
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %call = tail call signext i32 bitcast (i32 (...)* @callVoid to i32 ()*)()
+  %call2 = tail call signext i32 @callNonVoid(i32* %b)
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %entry
+  %retval.0 = phi i32 [ %call2, %if.then ], [ undef, %entry ]
+  ret i32 %retval.0
+}
+
+declare signext i32 @callVoid(...) local_unnamed_addr
+
+declare signext i32 @callNonVoid(i32*) local_unnamed_addr
+
+define dso_local signext i32 @test2(i32* %p1) local_unnamed_addr  {
+; CHECK-LABEL: test2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    stp x30, x19, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset w19, -8
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    cbz x0, .LBB1_2
+; CHECK-NEXT:  // %bb.1: // %if.end
+; CHECK-NEXT:    adrp x8, a
+; CHECK-NEXT:    ldrsw x8, [x8, :lo12:a]
+; CHECK-NEXT:    mov x19, x0
+; CHECK-NEXT:    cmp x8, x0
+; CHECK-NEXT:    b.eq .LBB1_3
+; CHECK-NEXT:  .LBB1_2: // %return
+; CHECK-NEXT:    mov w0, wzr
+; CHECK-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB1_3: // %if.then2
+; CHECK-NEXT:    bl callVoid
+; CHECK-NEXT:    mov x0, x19
+; CHECK-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    b callNonVoid
+;
+; CHECK-APPLE-LABEL: test2:
+; CHECK-APPLE:       ; %bb.0: ; %entry
+; CHECK-APPLE-NEXT:    stp x20, x19, [sp, #-32]! ; 16-byte Folded Spill
+; CHECK-APPLE-NEXT:    stp x29, x30, [sp, #16] ; 16-byte Folded Spill
+; CHECK-APPLE-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-APPLE-NEXT:    .cfi_offset w30, -8
+; CHECK-APPLE-NEXT:    .cfi_offset w29, -16
+; CHECK-APPLE-NEXT:    .cfi_offset w19, -24
+; CHECK-APPLE-NEXT:    .cfi_offset w20, -32
+; CHECK-APPLE-NEXT:    cbz x0, LBB1_2
+; CHECK-APPLE-NEXT:  ; %bb.1: ; %if.end
+; CHECK-APPLE-NEXT:  Lloh2:
+; CHECK-APPLE-NEXT:    adrp x8, _a@PAGE
+; CHECK-APPLE-NEXT:  Lloh3:
+; CHECK-APPLE-NEXT:    ldrsw x8, [x8, _a@PAGEOFF]
+; CHECK-APPLE-NEXT:    mov x19, x0
+; CHECK-APPLE-NEXT:    cmp x8, x0
+; CHECK-APPLE-NEXT:    b.eq LBB1_3
+; CHECK-APPLE-NEXT:  LBB1_2: ; %return
+; CHECK-APPLE-NEXT:    ldp x29, x30, [sp, #16] ; 16-byte Folded Reload
+; CHECK-APPLE-NEXT:    mov w0, wzr
+; CHECK-APPLE-NEXT:    ldp x20, x19, [sp], #32 ; 16-byte Folded Reload
+; CHECK-APPLE-NEXT:    ret
+; CHECK-APPLE-NEXT:  LBB1_3: ; %if.then2
+; CHECK-APPLE-NEXT:    bl _callVoid
+; CHECK-APPLE-NEXT:    ldp x29, x30, [sp, #16] ; 16-byte Folded Reload
+; CHECK-APPLE-NEXT:    mov x0, x19
+; CHECK-APPLE-NEXT:    ldp x20, x19, [sp], #32 ; 16-byte Folded Reload
+; CHECK-APPLE-NEXT:    b _callNonVoid
+; CHECK-APPLE-NEXT:    .loh AdrpLdr Lloh2, Lloh3
+entry:
+  %tobool = icmp eq i32* %p1, null
+  br i1 %tobool, label %return, label %if.end
+
+if.end:                                           ; preds = %entry
+  %0 = load i32, i32* @a, align 4, !tbaa !2
+  %conv = sext i32 %0 to i64
+  %1 = inttoptr i64 %conv to i32*
+  %cmp = icmp eq i32* %1, %p1
+  br i1 %cmp, label %if.then2, label %return
+
+if.then2:                                         ; preds = %if.end
+  %call = tail call signext i32 bitcast (i32 (...)* @callVoid to i32 ()*)()
+  %call3 = tail call signext i32 @callNonVoid(i32* nonnull %p1)
+  br label %return
+
+return:                                           ; preds = %if.end, %entry, %if.then2
+  %retval.0 = phi i32 [ %call3, %if.then2 ], [ 0, %entry ], [ 0, %if.end ]
+  ret i32 %retval.0
+}
+
+
+define dso_local i8* @test3(i8** nocapture %p1, i8 zeroext %p2) local_unnamed_addr  {
+; CHECK-LABEL: test3:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    str x30, [sp, #-32]! // 8-byte Folded Spill
+; CHECK-NEXT:    stp x20, x19, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    .cfi_offset w19, -8
+; CHECK-NEXT:    .cfi_offset w20, -16
+; CHECK-NEXT:    .cfi_offset w30, -32
+; CHECK-NEXT:    ldr x19, [x0]
+; CHECK-NEXT:    cbz x19, .LBB2_2
+; CHECK-NEXT:  // %bb.1: // %land.rhs
+; CHECK-NEXT:    mov x20, x0
+; CHECK-NEXT:    mov x0, x19
+; CHECK-NEXT:    bl bar
+; CHECK-NEXT:    str x0, [x20]
+; CHECK-NEXT:  .LBB2_2: // %land.end
+; CHECK-NEXT:    mov x0, x19
+; CHECK-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x30, [sp], #32 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+;
+; CHECK-APPLE-LABEL: test3:
+; CHECK-APPLE:       ; %bb.0: ; %entry
+; CHECK-APPLE-NEXT:    stp x20, x19, [sp, #-32]! ; 16-byte Folded Spill
+; CHECK-APPLE-NEXT:    stp x29, x30, [sp, #16] ; 16-byte Folded Spill
+; CHECK-APPLE-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-APPLE-NEXT:    .cfi_offset w30, -8
+; CHECK-APPLE-NEXT:    .cfi_offset w29, -16
+; CHECK-APPLE-NEXT:    .cfi_offset w19, -24
+; CHECK-APPLE-NEXT:    .cfi_offset w20, -32
+; CHECK-APPLE-NEXT:    ldr x19, [x0]
+; CHECK-APPLE-NEXT:    cbz x19, LBB2_2
+; CHECK-APPLE-NEXT:  ; %bb.1: ; %land.rhs
+; CHECK-APPLE-NEXT:    mov x20, x0
+; CHECK-APPLE-NEXT:    mov x0, x19
+; CHECK-APPLE-NEXT:    bl _bar
+; CHECK-APPLE-NEXT:    str x0, [x20]
+; CHECK-APPLE-NEXT:  LBB2_2: ; %land.end
+; CHECK-APPLE-NEXT:    ldp x29, x30, [sp, #16] ; 16-byte Folded Reload
+; CHECK-APPLE-NEXT:    mov x0, x19
+; CHECK-APPLE-NEXT:    ldp x20, x19, [sp], #32 ; 16-byte Folded Reload
+; CHECK-APPLE-NEXT:    ret
+entry:
+  %0 = load i8*, i8** %p1, align 8, !tbaa !6
+  %tobool = icmp eq i8* %0, null
+  br i1 %tobool, label %land.end, label %land.rhs
+
+land.rhs:                                         ; preds = %entry
+  %call = tail call i8* @bar(i8* nonnull %0, i8 zeroext %p2)
+  store i8* %call, i8** %p1, align 8, !tbaa !6
+  br label %land.end
+
+land.end:                                         ; preds = %entry, %land.rhs
+  ret i8* %0
+}
+
+declare i8* @bar(i8*, i8 zeroext) local_unnamed_addr
+
+
+!llvm.module.flags = !{!0}
+!llvm.ident = !{!1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{!"clang version 10.0.0 (trunk 367381) (llvm/trunk 367388)"}
+!2 = !{!3, !3, i64 0}
+!3 = !{!"int", !4, i64 0}
+!4 = !{!"omnipotent char", !5, i64 0}
+!5 = !{!"Simple C/C++ TBAA"}
+!6 = !{!7, !7, i64 0}
+!7 = !{!"any pointer", !4, i64 0}
diff --git a/llvm/test/CodeGen/AArch64/cxx-tlscc.ll b/llvm/test/CodeGen/AArch64/cxx-tlscc.ll
index d179eab7e..6de6fe082 100644
--- a/llvm/test/CodeGen/AArch64/cxx-tlscc.ll
+++ b/llvm/test/CodeGen/AArch64/cxx-tlscc.ll
@@ -220,5 +220,5 @@ entry:
   ret void
 }
 
-attributes #0 = { nounwind "no-frame-pointer-elim"="true" }
+attributes #0 = { nounwind "frame-pointer"="all" }
 attributes #1 = { nounwind }
diff --git a/llvm/test/CodeGen/AArch64/dag-combine-invaraints.ll b/llvm/test/CodeGen/AArch64/dag-combine-invaraints.ll
index 107f1eb0c..cdbc6d77a 100644
--- a/llvm/test/CodeGen/AArch64/dag-combine-invaraints.ll
+++ b/llvm/test/CodeGen/AArch64/dag-combine-invaraints.ll
@@ -32,5 +32,5 @@ main_:
 
 declare i32 @printf(i8*, ...) #1
 
-attributes #0 = { nounwind ssp "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind ssp "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/AArch64/dbg-declare-tag-offset.ll b/llvm/test/CodeGen/AArch64/dbg-declare-tag-offset.ll
index 626db0589..1db53a63b 100644
--- a/llvm/test/CodeGen/AArch64/dbg-declare-tag-offset.ll
+++ b/llvm/test/CodeGen/AArch64/dbg-declare-tag-offset.ll
@@ -1,17 +1,16 @@
-; RUN: llc -o - %s | FileCheck %s
+; RUN: llc -filetype=obj -o - %s | llvm-dwarfdump - | FileCheck %s
 
 target triple="aarch64--"
 
-; CHECK: .Linfo_string4:
-; CHECK-NEXT: .asciz "a"
-; CHECK: .Linfo_string6:
-; CHECK-NEXT: .asciz "b"
+; CHECK:      DW_TAG_variable
+; CHECK-NOT:  DW_TAG
+; CHECK:        DW_AT_LLVM_tag_offset (0x01)
+; CHECK-NEXT:   DW_AT_name    ("a")
 
-; CHECK: .byte 1 // DW_AT_LLVM_tag_offset
-; CHECK: .word .Linfo_string4 // DW_AT_name
-
-; CHECK: .byte 2 // DW_AT_LLVM_tag_offset
-; CHECK: .word .Linfo_string6 // DW_AT_name
+; CHECK:      DW_TAG_variable
+; CHECK-NOT:  DW_TAG
+; CHECK:        DW_AT_LLVM_tag_offset (0x02)
+; CHECK-NEXT:   DW_AT_name    ("b")
 
 define void @f() !dbg !6 {
 entry:
diff --git a/llvm/test/CodeGen/AArch64/dbg-value-tag-offset.ll b/llvm/test/CodeGen/AArch64/dbg-value-tag-offset.ll
new file mode 100644
index 000000000..45c491869
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/dbg-value-tag-offset.ll
@@ -0,0 +1,68 @@
+; RUN: llc -filetype=obj -o - %s | llvm-dwarfdump - | FileCheck %s
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64-unknown-linux-android24"
+
+; CHECK:      DW_TAG_variable
+; CHECK-NOT:  DW_TAG
+; CHECK:        DW_AT_LLVM_tag_offset (0x80)
+; CHECK-NEXT:   DW_AT_name    ("y")
+
+; CHECK:      DW_TAG_variable
+; CHECK-NOT:  DW_TAG
+; CHECK:        DW_AT_LLVM_tag_offset (0x00)
+; CHECK-NEXT:   DW_AT_name    ("x")
+
+define dso_local void @f() !dbg !14 {
+  %1 = alloca i32, align 4
+  %2 = alloca i32, align 4
+  %3 = bitcast i32* %1 to i8*, !dbg !21
+  %4 = bitcast i32* %2 to i8*, !dbg !21
+  call void @llvm.dbg.value(metadata i32 1, metadata !20, metadata !DIExpression()), !dbg !22
+  store i32 1, i32* %2, align 4, !dbg !23, !tbaa !24
+  call void @llvm.dbg.value(metadata i32* %1, metadata !18, metadata !DIExpression(DW_OP_LLVM_tag_offset, 0, DW_OP_deref)), !dbg !22
+  call void @use(i8* nonnull %3), !dbg !28
+  call void @llvm.dbg.value(metadata i32* %2, metadata !20, metadata !DIExpression(DW_OP_LLVM_tag_offset, 128, DW_OP_deref)), !dbg !22
+  call void @use(i8* nonnull %4), !dbg !29
+  ret void, !dbg !30
+}
+
+declare !dbg !5 void @use(i8*)
+
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!8, !9, !10, !11, !12}
+!llvm.ident = !{!13}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 10.0.0 (git@github.com:llvm/llvm-project.git 5560dd08b99a0e8b0c55116376624e4f967caec5)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !3, nameTableKind: None)
+!1 = !DIFile(filename: "dbg.cc", directory: "/tmp")
+!2 = !{}
+!3 = !{!4, !5}
+!4 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: null, size: 64)
+!5 = !DISubprogram(name: "use", scope: !1, file: !1, line: 2, type: !6, flags: DIFlagPrototyped, spFlags: DISPFlagOptimized, retainedNodes: !2)
+!6 = !DISubroutineType(types: !7)
+!7 = !{null, !4}
+!8 = !{i32 7, !"Dwarf Version", i32 4}
+!9 = !{i32 2, !"Debug Info Version", i32 3}
+!10 = !{i32 1, !"wchar_size", i32 4}
+!11 = !{i32 7, !"PIC Level", i32 2}
+!12 = !{i32 7, !"PIE Level", i32 2}
+!13 = !{!"clang version 10.0.0 (git@github.com:llvm/llvm-project.git 5560dd08b99a0e8b0c55116376624e4f967caec5)"}
+!14 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 4, type: !15, scopeLine: 4, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !17)
+!15 = !DISubroutineType(types: !16)
+!16 = !{null}
+!17 = !{!18, !20}
+!18 = !DILocalVariable(name: "x", scope: !14, file: !1, line: 5, type: !19)
+!19 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!20 = !DILocalVariable(name: "y", scope: !14, file: !1, line: 5, type: !19)
+!21 = !DILocation(line: 5, column: 3, scope: !14)
+!22 = !DILocation(line: 0, scope: !14)
+!23 = !DILocation(line: 5, column: 10, scope: !14)
+!24 = !{!25, !25, i64 0}
+!25 = !{!"int", !26, i64 0}
+!26 = !{!"omnipotent char", !27, i64 0}
+!27 = !{!"Simple C++ TBAA"}
+!28 = !DILocation(line: 6, column: 3, scope: !14)
+!29 = !DILocation(line: 7, column: 3, scope: !14)
+!30 = !DILocation(line: 8, column: 1, scope: !14)
diff --git a/llvm/test/CodeGen/AArch64/div-rem-pair-recomposition-signed.ll b/llvm/test/CodeGen/AArch64/div-rem-pair-recomposition-signed.ll
new file mode 100644
index 000000000..22bdf92db
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/div-rem-pair-recomposition-signed.ll
@@ -0,0 +1,319 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=aarch64-unknown-unknown | FileCheck %s --check-prefixes=ALL,X64
+
+; If the target does not have a single div/rem operation,
+; -div-rem-pairs pass will decompose the remainder calculation as:
+;   X % Y --> X - ((X / Y) * Y)
+; But if the target does have a single div/rem operation,
+; the opposite transform is likely beneficial.
+
+define i8 @scalar_i8(i8 %x, i8 %y, i8* %divdst) nounwind {
+; ALL-LABEL: scalar_i8:
+; ALL:       // %bb.0:
+; ALL-NEXT:    sxtb w8, w1
+; ALL-NEXT:    sxtb w9, w0
+; ALL-NEXT:    sdiv w8, w9, w8
+; ALL-NEXT:    msub w0, w8, w1, w0
+; ALL-NEXT:    strb w8, [x2]
+; ALL-NEXT:    ret
+  %div = sdiv i8 %x, %y
+  store i8 %div, i8* %divdst, align 4
+  %t1 = mul i8 %div, %y
+  %t2 = sub i8 %x, %t1
+  ret i8 %t2
+}
+
+define i16 @scalar_i16(i16 %x, i16 %y, i16* %divdst) nounwind {
+; ALL-LABEL: scalar_i16:
+; ALL:       // %bb.0:
+; ALL-NEXT:    sxth w8, w1
+; ALL-NEXT:    sxth w9, w0
+; ALL-NEXT:    sdiv w8, w9, w8
+; ALL-NEXT:    msub w0, w8, w1, w0
+; ALL-NEXT:    strh w8, [x2]
+; ALL-NEXT:    ret
+  %div = sdiv i16 %x, %y
+  store i16 %div, i16* %divdst, align 4
+  %t1 = mul i16 %div, %y
+  %t2 = sub i16 %x, %t1
+  ret i16 %t2
+}
+
+define i32 @scalar_i32(i32 %x, i32 %y, i32* %divdst) nounwind {
+; ALL-LABEL: scalar_i32:
+; ALL:       // %bb.0:
+; ALL-NEXT:    sdiv w8, w0, w1
+; ALL-NEXT:    msub w0, w8, w1, w0
+; ALL-NEXT:    str w8, [x2]
+; ALL-NEXT:    ret
+  %div = sdiv i32 %x, %y
+  store i32 %div, i32* %divdst, align 4
+  %t1 = mul i32 %div, %y
+  %t2 = sub i32 %x, %t1
+  ret i32 %t2
+}
+
+define i64 @scalar_i64(i64 %x, i64 %y, i64* %divdst) nounwind {
+; ALL-LABEL: scalar_i64:
+; ALL:       // %bb.0:
+; ALL-NEXT:    sdiv x8, x0, x1
+; ALL-NEXT:    msub x0, x8, x1, x0
+; ALL-NEXT:    str x8, [x2]
+; ALL-NEXT:    ret
+  %div = sdiv i64 %x, %y
+  store i64 %div, i64* %divdst, align 4
+  %t1 = mul i64 %div, %y
+  %t2 = sub i64 %x, %t1
+  ret i64 %t2
+}
+
+define <16 x i8> @vector_i128_i8(<16 x i8> %x, <16 x i8> %y, <16 x i8>* %divdst) nounwind {
+; ALL-LABEL: vector_i128_i8:
+; ALL:       // %bb.0:
+; ALL-NEXT:    smov w10, v1.b[0]
+; ALL-NEXT:    smov w11, v0.b[0]
+; ALL-NEXT:    smov w8, v1.b[1]
+; ALL-NEXT:    smov w9, v0.b[1]
+; ALL-NEXT:    sdiv w10, w11, w10
+; ALL-NEXT:    smov w12, v1.b[2]
+; ALL-NEXT:    smov w13, v0.b[2]
+; ALL-NEXT:    sdiv w8, w9, w8
+; ALL-NEXT:    fmov s2, w10
+; ALL-NEXT:    smov w14, v1.b[3]
+; ALL-NEXT:    smov w15, v0.b[3]
+; ALL-NEXT:    sdiv w12, w13, w12
+; ALL-NEXT:    mov v2.b[1], w8
+; ALL-NEXT:    smov w16, v1.b[4]
+; ALL-NEXT:    smov w17, v0.b[4]
+; ALL-NEXT:    sdiv w14, w15, w14
+; ALL-NEXT:    mov v2.b[2], w12
+; ALL-NEXT:    smov w18, v1.b[5]
+; ALL-NEXT:    smov w1, v0.b[5]
+; ALL-NEXT:    sdiv w16, w17, w16
+; ALL-NEXT:    mov v2.b[3], w14
+; ALL-NEXT:    smov w2, v1.b[6]
+; ALL-NEXT:    smov w3, v0.b[6]
+; ALL-NEXT:    sdiv w18, w1, w18
+; ALL-NEXT:    mov v2.b[4], w16
+; ALL-NEXT:    smov w4, v1.b[7]
+; ALL-NEXT:    smov w5, v0.b[7]
+; ALL-NEXT:    sdiv w2, w3, w2
+; ALL-NEXT:    mov v2.b[5], w18
+; ALL-NEXT:    smov w9, v1.b[8]
+; ALL-NEXT:    smov w11, v0.b[8]
+; ALL-NEXT:    sdiv w4, w5, w4
+; ALL-NEXT:    mov v2.b[6], w2
+; ALL-NEXT:    smov w13, v1.b[9]
+; ALL-NEXT:    smov w15, v0.b[9]
+; ALL-NEXT:    sdiv w9, w11, w9
+; ALL-NEXT:    mov v2.b[7], w4
+; ALL-NEXT:    smov w17, v1.b[10]
+; ALL-NEXT:    smov w1, v0.b[10]
+; ALL-NEXT:    sdiv w13, w15, w13
+; ALL-NEXT:    mov v2.b[8], w9
+; ALL-NEXT:    smov w3, v1.b[11]
+; ALL-NEXT:    smov w5, v0.b[11]
+; ALL-NEXT:    sdiv w17, w1, w17
+; ALL-NEXT:    mov v2.b[9], w13
+; ALL-NEXT:    smov w11, v1.b[12]
+; ALL-NEXT:    smov w15, v0.b[12]
+; ALL-NEXT:    sdiv w3, w5, w3
+; ALL-NEXT:    mov v2.b[10], w17
+; ALL-NEXT:    smov w1, v1.b[13]
+; ALL-NEXT:    smov w5, v0.b[13]
+; ALL-NEXT:    sdiv w11, w15, w11
+; ALL-NEXT:    mov v2.b[11], w3
+; ALL-NEXT:    smov w15, v1.b[14]
+; ALL-NEXT:    sdiv w1, w5, w1
+; ALL-NEXT:    smov w5, v0.b[14]
+; ALL-NEXT:    mov v2.b[12], w11
+; ALL-NEXT:    sdiv w15, w5, w15
+; ALL-NEXT:    smov w8, v1.b[15]
+; ALL-NEXT:    mov v2.b[13], w1
+; ALL-NEXT:    smov w9, v0.b[15]
+; ALL-NEXT:    mov v2.b[14], w15
+; ALL-NEXT:    sdiv w8, w9, w8
+; ALL-NEXT:    mov v2.b[15], w8
+; ALL-NEXT:    mls v0.16b, v2.16b, v1.16b
+; ALL-NEXT:    str q2, [x0]
+; ALL-NEXT:    ret
+  %div = sdiv <16 x i8> %x, %y
+  store <16 x i8> %div, <16 x i8>* %divdst, align 16
+  %t1 = mul <16 x i8> %div, %y
+  %t2 = sub <16 x i8> %x, %t1
+  ret <16 x i8> %t2
+}
+
+define <8 x i16> @vector_i128_i16(<8 x i16> %x, <8 x i16> %y, <8 x i16>* %divdst) nounwind {
+; ALL-LABEL: vector_i128_i16:
+; ALL:       // %bb.0:
+; ALL-NEXT:    smov w10, v1.h[0]
+; ALL-NEXT:    smov w11, v0.h[0]
+; ALL-NEXT:    smov w8, v1.h[1]
+; ALL-NEXT:    smov w9, v0.h[1]
+; ALL-NEXT:    sdiv w10, w11, w10
+; ALL-NEXT:    smov w12, v1.h[2]
+; ALL-NEXT:    smov w13, v0.h[2]
+; ALL-NEXT:    sdiv w8, w9, w8
+; ALL-NEXT:    fmov s2, w10
+; ALL-NEXT:    smov w14, v1.h[3]
+; ALL-NEXT:    smov w15, v0.h[3]
+; ALL-NEXT:    sdiv w12, w13, w12
+; ALL-NEXT:    mov v2.h[1], w8
+; ALL-NEXT:    smov w9, v1.h[4]
+; ALL-NEXT:    smov w11, v0.h[4]
+; ALL-NEXT:    sdiv w14, w15, w14
+; ALL-NEXT:    mov v2.h[2], w12
+; ALL-NEXT:    smov w13, v1.h[5]
+; ALL-NEXT:    smov w15, v0.h[5]
+; ALL-NEXT:    sdiv w9, w11, w9
+; ALL-NEXT:    mov v2.h[3], w14
+; ALL-NEXT:    smov w11, v1.h[6]
+; ALL-NEXT:    sdiv w13, w15, w13
+; ALL-NEXT:    smov w15, v0.h[6]
+; ALL-NEXT:    mov v2.h[4], w9
+; ALL-NEXT:    sdiv w11, w15, w11
+; ALL-NEXT:    smov w8, v1.h[7]
+; ALL-NEXT:    mov v2.h[5], w13
+; ALL-NEXT:    smov w9, v0.h[7]
+; ALL-NEXT:    mov v2.h[6], w11
+; ALL-NEXT:    sdiv w8, w9, w8
+; ALL-NEXT:    mov v2.h[7], w8
+; ALL-NEXT:    mls v0.8h, v2.8h, v1.8h
+; ALL-NEXT:    str q2, [x0]
+; ALL-NEXT:    ret
+  %div = sdiv <8 x i16> %x, %y
+  store <8 x i16> %div, <8 x i16>* %divdst, align 16
+  %t1 = mul <8 x i16> %div, %y
+  %t2 = sub <8 x i16> %x, %t1
+  ret <8 x i16> %t2
+}
+
+define <4 x i32> @vector_i128_i32(<4 x i32> %x, <4 x i32> %y, <4 x i32>* %divdst) nounwind {
+; ALL-LABEL: vector_i128_i32:
+; ALL:       // %bb.0:
+; ALL-NEXT:    fmov w9, s1
+; ALL-NEXT:    fmov w10, s0
+; ALL-NEXT:    mov w8, v1.s[1]
+; ALL-NEXT:    sdiv w9, w10, w9
+; ALL-NEXT:    mov w10, v0.s[1]
+; ALL-NEXT:    sdiv w8, w10, w8
+; ALL-NEXT:    mov w10, v1.s[2]
+; ALL-NEXT:    fmov s2, w9
+; ALL-NEXT:    mov w9, v0.s[2]
+; ALL-NEXT:    sdiv w9, w9, w10
+; ALL-NEXT:    mov w10, v1.s[3]
+; ALL-NEXT:    mov v2.s[1], w8
+; ALL-NEXT:    mov w8, v0.s[3]
+; ALL-NEXT:    mov v2.s[2], w9
+; ALL-NEXT:    sdiv w8, w8, w10
+; ALL-NEXT:    mov v2.s[3], w8
+; ALL-NEXT:    mls v0.4s, v2.4s, v1.4s
+; ALL-NEXT:    str q2, [x0]
+; ALL-NEXT:    ret
+  %div = sdiv <4 x i32> %x, %y
+  store <4 x i32> %div, <4 x i32>* %divdst, align 16
+  %t1 = mul <4 x i32> %div, %y
+  %t2 = sub <4 x i32> %x, %t1
+  ret <4 x i32> %t2
+}
+
+define <2 x i64> @vector_i128_i64(<2 x i64> %x, <2 x i64> %y, <2 x i64>* %divdst) nounwind {
+; ALL-LABEL: vector_i128_i64:
+; ALL:       // %bb.0:
+; ALL-NEXT:    fmov x10, d1
+; ALL-NEXT:    fmov x11, d0
+; ALL-NEXT:    mov x8, v1.d[1]
+; ALL-NEXT:    mov x9, v0.d[1]
+; ALL-NEXT:    sdiv x11, x11, x10
+; ALL-NEXT:    sdiv x9, x9, x8
+; ALL-NEXT:    mul x10, x11, x10
+; ALL-NEXT:    mul x8, x9, x8
+; ALL-NEXT:    fmov d1, x10
+; ALL-NEXT:    mov v1.d[1], x8
+; ALL-NEXT:    sub v0.2d, v0.2d, v1.2d
+; ALL-NEXT:    fmov d1, x11
+; ALL-NEXT:    mov v1.d[1], x9
+; ALL-NEXT:    str q1, [x0]
+; ALL-NEXT:    ret
+  %div = sdiv <2 x i64> %x, %y
+  store <2 x i64> %div, <2 x i64>* %divdst, align 16
+  %t1 = mul <2 x i64> %div, %y
+  %t2 = sub <2 x i64> %x, %t1
+  ret <2 x i64> %t2
+}
+
+; Special tests.
+
+define i32 @scalar_i32_commutative(i32 %x, i32* %ysrc, i32* %divdst) nounwind {
+; ALL-LABEL: scalar_i32_commutative:
+; ALL:       // %bb.0:
+; ALL-NEXT:    ldr w8, [x1]
+; ALL-NEXT:    sdiv w9, w0, w8
+; ALL-NEXT:    msub w0, w8, w9, w0
+; ALL-NEXT:    str w9, [x2]
+; ALL-NEXT:    ret
+  %y = load i32, i32* %ysrc, align 4
+  %div = sdiv i32 %x, %y
+  store i32 %div, i32* %divdst, align 4
+  %t1 = mul i32 %y, %div ; commutative
+  %t2 = sub i32 %x, %t1
+  ret i32 %t2
+}
+
+; We do not care about extra uses.
+define i32 @extrause(i32 %x, i32 %y, i32* %divdst, i32* %t1dst) nounwind {
+; ALL-LABEL: extrause:
+; ALL:       // %bb.0:
+; ALL-NEXT:    sdiv w8, w0, w1
+; ALL-NEXT:    str w8, [x2]
+; ALL-NEXT:    mul w8, w8, w1
+; ALL-NEXT:    sub w0, w0, w8
+; ALL-NEXT:    str w8, [x3]
+; ALL-NEXT:    ret
+  %div = sdiv i32 %x, %y
+  store i32 %div, i32* %divdst, align 4
+  %t1 = mul i32 %div, %y
+  store i32 %t1, i32* %t1dst, align 4
+  %t2 = sub i32 %x, %t1
+  ret i32 %t2
+}
+
+; 'rem' should appear next to 'div'.
+define i32 @multiple_bb(i32 %x, i32 %y, i32* %divdst, i1 zeroext %store_srem, i32* %sremdst) nounwind {
+; ALL-LABEL: multiple_bb:
+; ALL:       // %bb.0:
+; ALL-NEXT:    mov w8, w0
+; ALL-NEXT:    sdiv w0, w0, w1
+; ALL-NEXT:    str w0, [x2]
+; ALL-NEXT:    cbz w3, .LBB10_2
+; ALL-NEXT:  // %bb.1: // %do_srem
+; ALL-NEXT:    msub w8, w0, w1, w8
+; ALL-NEXT:    str w8, [x4]
+; ALL-NEXT:  .LBB10_2: // %end
+; ALL-NEXT:    ret
+  %div = sdiv i32 %x, %y
+  store i32 %div, i32* %divdst, align 4
+  br i1 %store_srem, label %do_srem, label %end
+do_srem:
+  %t1 = mul i32 %div, %y
+  %t2 = sub i32 %x, %t1
+  store i32 %t2, i32* %sremdst, align 4
+  br label %end
+end:
+  ret i32 %div
+}
+
+define i32 @negative_different_x(i32 %x0, i32 %x1, i32 %y, i32* %divdst) nounwind {
+; ALL-LABEL: negative_different_x:
+; ALL:       // %bb.0:
+; ALL-NEXT:    sdiv w8, w0, w2
+; ALL-NEXT:    msub w0, w8, w2, w1
+; ALL-NEXT:    str w8, [x3]
+; ALL-NEXT:    ret
+  %div = sdiv i32 %x0, %y ; not %x1
+  store i32 %div, i32* %divdst, align 4
+  %t1 = mul i32 %div, %y
+  %t2 = sub i32 %x1, %t1 ; not %x0
+  ret i32 %t2
+}
diff --git a/llvm/test/CodeGen/AArch64/div-rem-pair-recomposition-unsigned.ll b/llvm/test/CodeGen/AArch64/div-rem-pair-recomposition-unsigned.ll
new file mode 100644
index 000000000..523770481
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/div-rem-pair-recomposition-unsigned.ll
@@ -0,0 +1,319 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=aarch64-unknown-unknown | FileCheck %s --check-prefixes=ALL,X64
+
+; If the target does not have a single div/rem operation,
+; -div-rem-pairs pass will decompose the remainder calculation as:
+;   X % Y --> X - ((X / Y) * Y)
+; But if the target does have a single div/rem operation,
+; the opposite transform is likely beneficial.
+
+define i8 @scalar_i8(i8 %x, i8 %y, i8* %divdst) nounwind {
+; ALL-LABEL: scalar_i8:
+; ALL:       // %bb.0:
+; ALL-NEXT:    and w8, w1, #0xff
+; ALL-NEXT:    and w9, w0, #0xff
+; ALL-NEXT:    udiv w8, w9, w8
+; ALL-NEXT:    msub w0, w8, w1, w0
+; ALL-NEXT:    strb w8, [x2]
+; ALL-NEXT:    ret
+  %div = udiv i8 %x, %y
+  store i8 %div, i8* %divdst, align 4
+  %t1 = mul i8 %div, %y
+  %t2 = sub i8 %x, %t1
+  ret i8 %t2
+}
+
+define i16 @scalar_i16(i16 %x, i16 %y, i16* %divdst) nounwind {
+; ALL-LABEL: scalar_i16:
+; ALL:       // %bb.0:
+; ALL-NEXT:    and w8, w1, #0xffff
+; ALL-NEXT:    and w9, w0, #0xffff
+; ALL-NEXT:    udiv w8, w9, w8
+; ALL-NEXT:    msub w0, w8, w1, w0
+; ALL-NEXT:    strh w8, [x2]
+; ALL-NEXT:    ret
+  %div = udiv i16 %x, %y
+  store i16 %div, i16* %divdst, align 4
+  %t1 = mul i16 %div, %y
+  %t2 = sub i16 %x, %t1
+  ret i16 %t2
+}
+
+define i32 @scalar_i32(i32 %x, i32 %y, i32* %divdst) nounwind {
+; ALL-LABEL: scalar_i32:
+; ALL:       // %bb.0:
+; ALL-NEXT:    udiv w8, w0, w1
+; ALL-NEXT:    msub w0, w8, w1, w0
+; ALL-NEXT:    str w8, [x2]
+; ALL-NEXT:    ret
+  %div = udiv i32 %x, %y
+  store i32 %div, i32* %divdst, align 4
+  %t1 = mul i32 %div, %y
+  %t2 = sub i32 %x, %t1
+  ret i32 %t2
+}
+
+define i64 @scalar_i64(i64 %x, i64 %y, i64* %divdst) nounwind {
+; ALL-LABEL: scalar_i64:
+; ALL:       // %bb.0:
+; ALL-NEXT:    udiv x8, x0, x1
+; ALL-NEXT:    msub x0, x8, x1, x0
+; ALL-NEXT:    str x8, [x2]
+; ALL-NEXT:    ret
+  %div = udiv i64 %x, %y
+  store i64 %div, i64* %divdst, align 4
+  %t1 = mul i64 %div, %y
+  %t2 = sub i64 %x, %t1
+  ret i64 %t2
+}
+
+define <16 x i8> @vector_i128_i8(<16 x i8> %x, <16 x i8> %y, <16 x i8>* %divdst) nounwind {
+; ALL-LABEL: vector_i128_i8:
+; ALL:       // %bb.0:
+; ALL-NEXT:    umov w10, v1.b[0]
+; ALL-NEXT:    umov w11, v0.b[0]
+; ALL-NEXT:    umov w8, v1.b[1]
+; ALL-NEXT:    umov w9, v0.b[1]
+; ALL-NEXT:    udiv w10, w11, w10
+; ALL-NEXT:    umov w12, v1.b[2]
+; ALL-NEXT:    umov w13, v0.b[2]
+; ALL-NEXT:    udiv w8, w9, w8
+; ALL-NEXT:    fmov s2, w10
+; ALL-NEXT:    umov w14, v1.b[3]
+; ALL-NEXT:    umov w15, v0.b[3]
+; ALL-NEXT:    udiv w12, w13, w12
+; ALL-NEXT:    mov v2.b[1], w8
+; ALL-NEXT:    umov w16, v1.b[4]
+; ALL-NEXT:    umov w17, v0.b[4]
+; ALL-NEXT:    udiv w14, w15, w14
+; ALL-NEXT:    mov v2.b[2], w12
+; ALL-NEXT:    umov w18, v1.b[5]
+; ALL-NEXT:    umov w1, v0.b[5]
+; ALL-NEXT:    udiv w16, w17, w16
+; ALL-NEXT:    mov v2.b[3], w14
+; ALL-NEXT:    umov w2, v1.b[6]
+; ALL-NEXT:    umov w3, v0.b[6]
+; ALL-NEXT:    udiv w18, w1, w18
+; ALL-NEXT:    mov v2.b[4], w16
+; ALL-NEXT:    umov w4, v1.b[7]
+; ALL-NEXT:    umov w5, v0.b[7]
+; ALL-NEXT:    udiv w2, w3, w2
+; ALL-NEXT:    mov v2.b[5], w18
+; ALL-NEXT:    umov w9, v1.b[8]
+; ALL-NEXT:    umov w11, v0.b[8]
+; ALL-NEXT:    udiv w4, w5, w4
+; ALL-NEXT:    mov v2.b[6], w2
+; ALL-NEXT:    umov w13, v1.b[9]
+; ALL-NEXT:    umov w15, v0.b[9]
+; ALL-NEXT:    udiv w9, w11, w9
+; ALL-NEXT:    mov v2.b[7], w4
+; ALL-NEXT:    umov w17, v1.b[10]
+; ALL-NEXT:    umov w1, v0.b[10]
+; ALL-NEXT:    udiv w13, w15, w13
+; ALL-NEXT:    mov v2.b[8], w9
+; ALL-NEXT:    umov w3, v1.b[11]
+; ALL-NEXT:    umov w5, v0.b[11]
+; ALL-NEXT:    udiv w17, w1, w17
+; ALL-NEXT:    mov v2.b[9], w13
+; ALL-NEXT:    umov w11, v1.b[12]
+; ALL-NEXT:    umov w15, v0.b[12]
+; ALL-NEXT:    udiv w3, w5, w3
+; ALL-NEXT:    mov v2.b[10], w17
+; ALL-NEXT:    umov w1, v1.b[13]
+; ALL-NEXT:    umov w5, v0.b[13]
+; ALL-NEXT:    udiv w11, w15, w11
+; ALL-NEXT:    mov v2.b[11], w3
+; ALL-NEXT:    umov w15, v1.b[14]
+; ALL-NEXT:    udiv w1, w5, w1
+; ALL-NEXT:    umov w5, v0.b[14]
+; ALL-NEXT:    mov v2.b[12], w11
+; ALL-NEXT:    udiv w15, w5, w15
+; ALL-NEXT:    umov w8, v1.b[15]
+; ALL-NEXT:    mov v2.b[13], w1
+; ALL-NEXT:    umov w9, v0.b[15]
+; ALL-NEXT:    mov v2.b[14], w15
+; ALL-NEXT:    udiv w8, w9, w8
+; ALL-NEXT:    mov v2.b[15], w8
+; ALL-NEXT:    mls v0.16b, v2.16b, v1.16b
+; ALL-NEXT:    str q2, [x0]
+; ALL-NEXT:    ret
+  %div = udiv <16 x i8> %x, %y
+  store <16 x i8> %div, <16 x i8>* %divdst, align 16
+  %t1 = mul <16 x i8> %div, %y
+  %t2 = sub <16 x i8> %x, %t1
+  ret <16 x i8> %t2
+}
+
+define <8 x i16> @vector_i128_i16(<8 x i16> %x, <8 x i16> %y, <8 x i16>* %divdst) nounwind {
+; ALL-LABEL: vector_i128_i16:
+; ALL:       // %bb.0:
+; ALL-NEXT:    umov w10, v1.h[0]
+; ALL-NEXT:    umov w11, v0.h[0]
+; ALL-NEXT:    umov w8, v1.h[1]
+; ALL-NEXT:    umov w9, v0.h[1]
+; ALL-NEXT:    udiv w10, w11, w10
+; ALL-NEXT:    umov w12, v1.h[2]
+; ALL-NEXT:    umov w13, v0.h[2]
+; ALL-NEXT:    udiv w8, w9, w8
+; ALL-NEXT:    fmov s2, w10
+; ALL-NEXT:    umov w14, v1.h[3]
+; ALL-NEXT:    umov w15, v0.h[3]
+; ALL-NEXT:    udiv w12, w13, w12
+; ALL-NEXT:    mov v2.h[1], w8
+; ALL-NEXT:    umov w9, v1.h[4]
+; ALL-NEXT:    umov w11, v0.h[4]
+; ALL-NEXT:    udiv w14, w15, w14
+; ALL-NEXT:    mov v2.h[2], w12
+; ALL-NEXT:    umov w13, v1.h[5]
+; ALL-NEXT:    umov w15, v0.h[5]
+; ALL-NEXT:    udiv w9, w11, w9
+; ALL-NEXT:    mov v2.h[3], w14
+; ALL-NEXT:    umov w11, v1.h[6]
+; ALL-NEXT:    udiv w13, w15, w13
+; ALL-NEXT:    umov w15, v0.h[6]
+; ALL-NEXT:    mov v2.h[4], w9
+; ALL-NEXT:    udiv w11, w15, w11
+; ALL-NEXT:    umov w8, v1.h[7]
+; ALL-NEXT:    mov v2.h[5], w13
+; ALL-NEXT:    umov w9, v0.h[7]
+; ALL-NEXT:    mov v2.h[6], w11
+; ALL-NEXT:    udiv w8, w9, w8
+; ALL-NEXT:    mov v2.h[7], w8
+; ALL-NEXT:    mls v0.8h, v2.8h, v1.8h
+; ALL-NEXT:    str q2, [x0]
+; ALL-NEXT:    ret
+  %div = udiv <8 x i16> %x, %y
+  store <8 x i16> %div, <8 x i16>* %divdst, align 16
+  %t1 = mul <8 x i16> %div, %y
+  %t2 = sub <8 x i16> %x, %t1
+  ret <8 x i16> %t2
+}
+
+define <4 x i32> @vector_i128_i32(<4 x i32> %x, <4 x i32> %y, <4 x i32>* %divdst) nounwind {
+; ALL-LABEL: vector_i128_i32:
+; ALL:       // %bb.0:
+; ALL-NEXT:    fmov w9, s1
+; ALL-NEXT:    fmov w10, s0
+; ALL-NEXT:    mov w8, v1.s[1]
+; ALL-NEXT:    udiv w9, w10, w9
+; ALL-NEXT:    mov w10, v0.s[1]
+; ALL-NEXT:    udiv w8, w10, w8
+; ALL-NEXT:    mov w10, v1.s[2]
+; ALL-NEXT:    fmov s2, w9
+; ALL-NEXT:    mov w9, v0.s[2]
+; ALL-NEXT:    udiv w9, w9, w10
+; ALL-NEXT:    mov w10, v1.s[3]
+; ALL-NEXT:    mov v2.s[1], w8
+; ALL-NEXT:    mov w8, v0.s[3]
+; ALL-NEXT:    mov v2.s[2], w9
+; ALL-NEXT:    udiv w8, w8, w10
+; ALL-NEXT:    mov v2.s[3], w8
+; ALL-NEXT:    mls v0.4s, v2.4s, v1.4s
+; ALL-NEXT:    str q2, [x0]
+; ALL-NEXT:    ret
+  %div = udiv <4 x i32> %x, %y
+  store <4 x i32> %div, <4 x i32>* %divdst, align 16
+  %t1 = mul <4 x i32> %div, %y
+  %t2 = sub <4 x i32> %x, %t1
+  ret <4 x i32> %t2
+}
+
+define <2 x i64> @vector_i128_i64(<2 x i64> %x, <2 x i64> %y, <2 x i64>* %divdst) nounwind {
+; ALL-LABEL: vector_i128_i64:
+; ALL:       // %bb.0:
+; ALL-NEXT:    fmov x10, d1
+; ALL-NEXT:    fmov x11, d0
+; ALL-NEXT:    mov x8, v1.d[1]
+; ALL-NEXT:    mov x9, v0.d[1]
+; ALL-NEXT:    udiv x11, x11, x10
+; ALL-NEXT:    udiv x9, x9, x8
+; ALL-NEXT:    mul x10, x11, x10
+; ALL-NEXT:    mul x8, x9, x8
+; ALL-NEXT:    fmov d1, x10
+; ALL-NEXT:    mov v1.d[1], x8
+; ALL-NEXT:    sub v0.2d, v0.2d, v1.2d
+; ALL-NEXT:    fmov d1, x11
+; ALL-NEXT:    mov v1.d[1], x9
+; ALL-NEXT:    str q1, [x0]
+; ALL-NEXT:    ret
+  %div = udiv <2 x i64> %x, %y
+  store <2 x i64> %div, <2 x i64>* %divdst, align 16
+  %t1 = mul <2 x i64> %div, %y
+  %t2 = sub <2 x i64> %x, %t1
+  ret <2 x i64> %t2
+}
+
+; Special tests.
+
+define i32 @scalar_i32_commutative(i32 %x, i32* %ysrc, i32* %divdst) nounwind {
+; ALL-LABEL: scalar_i32_commutative:
+; ALL:       // %bb.0:
+; ALL-NEXT:    ldr w8, [x1]
+; ALL-NEXT:    udiv w9, w0, w8
+; ALL-NEXT:    msub w0, w8, w9, w0
+; ALL-NEXT:    str w9, [x2]
+; ALL-NEXT:    ret
+  %y = load i32, i32* %ysrc, align 4
+  %div = udiv i32 %x, %y
+  store i32 %div, i32* %divdst, align 4
+  %t1 = mul i32 %y, %div ; commutative
+  %t2 = sub i32 %x, %t1
+  ret i32 %t2
+}
+
+; We do not care about extra uses.
+define i32 @extrause(i32 %x, i32 %y, i32* %divdst, i32* %t1dst) nounwind {
+; ALL-LABEL: extrause:
+; ALL:       // %bb.0:
+; ALL-NEXT:    udiv w8, w0, w1
+; ALL-NEXT:    str w8, [x2]
+; ALL-NEXT:    mul w8, w8, w1
+; ALL-NEXT:    sub w0, w0, w8
+; ALL-NEXT:    str w8, [x3]
+; ALL-NEXT:    ret
+  %div = udiv i32 %x, %y
+  store i32 %div, i32* %divdst, align 4
+  %t1 = mul i32 %div, %y
+  store i32 %t1, i32* %t1dst, align 4
+  %t2 = sub i32 %x, %t1
+  ret i32 %t2
+}
+
+; 'rem' should appear next to 'div'.
+define i32 @multiple_bb(i32 %x, i32 %y, i32* %divdst, i1 zeroext %store_urem, i32* %uremdst) nounwind {
+; ALL-LABEL: multiple_bb:
+; ALL:       // %bb.0:
+; ALL-NEXT:    mov w8, w0
+; ALL-NEXT:    udiv w0, w0, w1
+; ALL-NEXT:    str w0, [x2]
+; ALL-NEXT:    cbz w3, .LBB10_2
+; ALL-NEXT:  // %bb.1: // %do_urem
+; ALL-NEXT:    msub w8, w0, w1, w8
+; ALL-NEXT:    str w8, [x4]
+; ALL-NEXT:  .LBB10_2: // %end
+; ALL-NEXT:    ret
+  %div = udiv i32 %x, %y
+  store i32 %div, i32* %divdst, align 4
+  br i1 %store_urem, label %do_urem, label %end
+do_urem:
+  %t1 = mul i32 %div, %y
+  %t2 = sub i32 %x, %t1
+  store i32 %t2, i32* %uremdst, align 4
+  br label %end
+end:
+  ret i32 %div
+}
+
+define i32 @negative_different_x(i32 %x0, i32 %x1, i32 %y, i32* %divdst) nounwind {
+; ALL-LABEL: negative_different_x:
+; ALL:       // %bb.0:
+; ALL-NEXT:    udiv w8, w0, w2
+; ALL-NEXT:    msub w0, w8, w2, w1
+; ALL-NEXT:    str w8, [x3]
+; ALL-NEXT:    ret
+  %div = udiv i32 %x0, %y ; not %x1
+  store i32 %div, i32* %divdst, align 4
+  %t1 = mul i32 %div, %y
+  %t2 = sub i32 %x1, %t1 ; not %x0
+  ret i32 %t2
+}
diff --git a/llvm/test/CodeGen/AArch64/dllimport.ll b/llvm/test/CodeGen/AArch64/dllimport.ll
index 281c847a3..61db9ea73 100644
--- a/llvm/test/CodeGen/AArch64/dllimport.ll
+++ b/llvm/test/CodeGen/AArch64/dllimport.ll
@@ -1,6 +1,6 @@
 ; RUN: llc -mtriple aarch64-unknown-windows-msvc -filetype asm -o - %s | FileCheck %s -check-prefixes=CHECK,DAG-ISEL
 ; RUN: llc -mtriple aarch64-unknown-windows-msvc -fast-isel -filetype asm -o - %s | FileCheck %s -check-prefixes=CHECK,FAST-ISEL
-; RUN: llc -mtriple aarch64-unknown-windows-msvc -O0 -filetype asm -o - %s | FileCheck %s -check-prefixes=CHECK,GLOBAL-ISEL,GLOBAL-ISEL-FALLBACK
+; RUN: llc -mtriple aarch64-unknown-windows-msvc -verify-machineinstrs -O0 -filetype asm -o - %s | FileCheck %s -check-prefixes=CHECK,GLOBAL-ISEL,GLOBAL-ISEL-FALLBACK
 
 @var = external dllimport global i32
 @ext = external global i32
@@ -59,4 +59,4 @@ define i32 @call_internal() {
 ; CHECK-LABEL: call_internal
 ; DAG-ISEL: b internal
 ; FAST-ISEL: b internal
-; GLOBAL-ISEL: bl internal
+; GLOBAL-ISEL: b internal
diff --git a/llvm/test/CodeGen/AArch64/eon.ll b/llvm/test/CodeGen/AArch64/eon.ll
index ea61ce34c..29c4c8ffd 100644
--- a/llvm/test/CodeGen/AArch64/eon.ll
+++ b/llvm/test/CodeGen/AArch64/eon.ll
@@ -1,4 +1,7 @@
 ; RUN: llc -mtriple=aarch64-none-linux-gnu < %s | FileCheck %s
+; RUN: llc %s -pass-remarks-missed=gisel* -mtriple=aarch64-none-linux-gnu -global-isel -o - 2>&1 | FileCheck %s
+
+; CHECK-NOT: remark
 
 ; Check that the eon instruction is generated instead of eor,movn
 define i64 @test1(i64 %a, i64 %b, i64 %c) {
diff --git a/llvm/test/CodeGen/AArch64/expand-select.ll b/llvm/test/CodeGen/AArch64/expand-select.ll
index da1e8dcba..82cf7a876 100644
--- a/llvm/test/CodeGen/AArch64/expand-select.ll
+++ b/llvm/test/CodeGen/AArch64/expand-select.ll
@@ -4,20 +4,20 @@
 define void @foo(i32 %In1, <2 x i128> %In2, <2 x i128> %In3, <2 x i128> *%Out) {
 ; CHECK-LABEL: foo:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    and w9, w0, #0x1
+; CHECK-NEXT:    and w8, w0, #0x1
 ; CHECK-NEXT:    fmov s0, wzr
-; CHECK-NEXT:    ldp x10, x8, [sp, #8]
-; CHECK-NEXT:    fmov s1, w9
-; CHECK-NEXT:    ldr x9, [sp]
+; CHECK-NEXT:    fmov s1, w8
+; CHECK-NEXT:    ldp x8, x9, [sp, #8]
+; CHECK-NEXT:    ldr x10, [sp]
 ; CHECK-NEXT:    cmeq v0.4s, v1.4s, v0.4s
 ; CHECK-NEXT:    fmov w11, s0
 ; CHECK-NEXT:    tst w11, #0x1
 ; CHECK-NEXT:    csel x11, x2, x6, ne
 ; CHECK-NEXT:    csel x12, x3, x7, ne
-; CHECK-NEXT:    csel x9, x4, x9, ne
-; CHECK-NEXT:    csel x10, x5, x10, ne
-; CHECK-NEXT:    stp x9, x10, [x8, #16]
-; CHECK-NEXT:    stp x11, x12, [x8]
+; CHECK-NEXT:    csel x10, x4, x10, ne
+; CHECK-NEXT:    csel x8, x5, x8, ne
+; CHECK-NEXT:    stp x10, x8, [x9, #16]
+; CHECK-NEXT:    stp x11, x12, [x9]
 ; CHECK-NEXT:    ret
   %cond = and i32 %In1, 1
   %cbool = icmp eq i32 %cond, 0
@@ -31,25 +31,25 @@ define void @foo(i32 %In1, <2 x i128> %In2, <2 x i128> %In3, <2 x i128> *%Out) {
 define void @bar(i32 %In1, <2 x i96> %In2, <2 x i96> %In3, <2 x i96> *%Out) {
 ; CHECK-LABEL: bar:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    and w10, w0, #0x1
+; CHECK-NEXT:    and w9, w0, #0x1
 ; CHECK-NEXT:    fmov s0, wzr
-; CHECK-NEXT:    fmov s1, w10
+; CHECK-NEXT:    fmov s1, w9
 ; CHECK-NEXT:    cmeq v0.4s, v1.4s, v0.4s
 ; CHECK-NEXT:    ldp x11, x8, [sp, #8]
-; CHECK-NEXT:    ldr x9, [sp]
+; CHECK-NEXT:    ldr x10, [sp]
 ; CHECK-NEXT:    dup v1.4s, v0.s[0]
-; CHECK-NEXT:    mov x10, v1.d[1]
-; CHECK-NEXT:    lsr x10, x10, #32
-; CHECK-NEXT:    tst w10, #0x1
-; CHECK-NEXT:    fmov w10, s0
+; CHECK-NEXT:    mov x9, v1.d[1]
+; CHECK-NEXT:    lsr x9, x9, #32
+; CHECK-NEXT:    tst w9, #0x1
+; CHECK-NEXT:    fmov w9, s0
 ; CHECK-NEXT:    csel x11, x5, x11, ne
-; CHECK-NEXT:    csel x9, x4, x9, ne
-; CHECK-NEXT:    tst w10, #0x1
-; CHECK-NEXT:    csel x10, x3, x7, ne
+; CHECK-NEXT:    csel x10, x4, x10, ne
+; CHECK-NEXT:    tst w9, #0x1
+; CHECK-NEXT:    csel x9, x3, x7, ne
 ; CHECK-NEXT:    csel x12, x2, x6, ne
-; CHECK-NEXT:    stur x9, [x8, #12]
+; CHECK-NEXT:    stur x10, [x8, #12]
 ; CHECK-NEXT:    str x12, [x8]
-; CHECK-NEXT:    str w10, [x8, #8]
+; CHECK-NEXT:    str w9, [x8, #8]
 ; CHECK-NEXT:    str w11, [x8, #20]
 ; CHECK-NEXT:    ret
   %cond = and i32 %In1, 1
diff --git a/llvm/test/CodeGen/AArch64/f16-instructions.ll b/llvm/test/CodeGen/AArch64/f16-instructions.ll
index 1ed4bdec9..6c1ea868a 100644
--- a/llvm/test/CodeGen/AArch64/f16-instructions.ll
+++ b/llvm/test/CodeGen/AArch64/f16-instructions.ll
@@ -1,14 +1,14 @@
-; RUN: llc < %s -mtriple aarch64-unknown-unknown -aarch64-neon-syntax=apple -asm-verbose=false -disable-post-ra -frame-pointer=all | FileCheck %s --check-prefix=CHECK-CVT --check-prefix=CHECK-COMMON
-; RUN: llc < %s -mtriple aarch64-unknown-unknown -mattr=+fullfp16 -aarch64-neon-syntax=apple -asm-verbose=false -disable-post-ra -frame-pointer=all | FileCheck %s --check-prefix=CHECK-COMMON --check-prefix=CHECK-FP16
+; RUN: llc < %s -mtriple aarch64-unknown-unknown -aarch64-neon-syntax=apple -asm-verbose=false -disable-post-ra -frame-pointer=non-leaf | FileCheck %s --check-prefix=CHECK-CVT --check-prefix=CHECK-COMMON
+; RUN: llc < %s -mtriple aarch64-unknown-unknown -mattr=+fullfp16 -aarch64-neon-syntax=apple -asm-verbose=false -disable-post-ra -frame-pointer=non-leaf | FileCheck %s --check-prefix=CHECK-COMMON --check-prefix=CHECK-FP16
 
 ; RUN: llc < %s -mtriple aarch64-unknown-unknown -aarch64-neon-syntax=apple \
-; RUN: -asm-verbose=false -disable-post-ra -frame-pointer=all -global-isel \
+; RUN: -asm-verbose=false -disable-post-ra -frame-pointer=non-leaf -global-isel \
 ; RUN: -global-isel-abort=2 -pass-remarks-missed=gisel-* 2>&1 | FileCheck %s \
 ; RUN: --check-prefixes=FALLBACK,GISEL-CVT,GISEL
 
 ; RUN: llc < %s -mtriple aarch64-unknown-unknown -mattr=+fullfp16 \
 ; RUN: -aarch64-neon-syntax=apple -asm-verbose=false -disable-post-ra \
-; RUN: -frame-pointer=all -global-isel -global-isel-abort=2 \
+; RUN: -frame-pointer=non-leaf -global-isel -global-isel-abort=2 \
 ; RUN: -pass-remarks-missed=gisel-* 2>&1 | FileCheck %s \
 ; RUN: --check-prefixes=FALLBACK-FP16,GISEL-FP16,GISEL
 
diff --git a/llvm/test/CodeGen/AArch64/fadd-combines.ll b/llvm/test/CodeGen/AArch64/fadd-combines.ll
index a204684db..640c1a100 100644
--- a/llvm/test/CodeGen/AArch64/fadd-combines.ll
+++ b/llvm/test/CodeGen/AArch64/fadd-combines.ll
@@ -146,28 +146,51 @@ define float @fadd_const_multiuse_fmf(float %x) {
   ret float %a3
 }
 
-; DAGCombiner transforms this into: (x + 59.0) + (x + 17.0).
-; The machine combiner transforms this into a chain of 3 dependent adds:
-; ((x + 59.0) + 17.0) + x
-
-define float @fadd_const_multiuse_attr(float %x) #0 {
+; DAGCombiner transforms this into: (x + 17.0) + (x + 59.0).
+define float @fadd_const_multiuse_attr(float %x) {
 ; CHECK-LABEL: fadd_const_multiuse_attr:
 ; CHECK:       // %bb.0:
-; CHECK-DAG:     mov  [[W59:w[0-9]+]], #1114374144
 ; CHECK-DAG:     mov  [[W17:w[0-9]+]], #1109917696
-; CHECK-NEXT:    fmov [[FP59:s[0-9]+]], [[W59]]
+; CHECK-DAG:     mov  [[W59:w[0-9]+]], #1114374144
 ; CHECK-NEXT:    fmov [[FP17:s[0-9]+]], [[W17]]
-; CHECK-NEXT:    fadd [[TMP1:s[0-9]+]], s0, [[FP59]]
-; CHECK-NEXT:    fadd [[TMP2:s[0-9]+]], [[FP17]], [[TMP1]]
-; CHECK-NEXT:    fadd s0, s0, [[TMP2]]
+; CHECK-NEXT:    fmov [[FP59:s[0-9]+]], [[W59]]
+; CHECK-NEXT:    fadd [[TMP1:s[0-9]+]], s0, [[FP17]]
+; CHECK-NEXT:    fadd [[TMP2:s[0-9]+]], s0, [[FP59]]
+; CHECK-NEXT:    fadd s0, [[TMP1]], [[TMP2]]
 ; CHECK-NEXT:    ret
-  %a1 = fadd float %x, 42.0
-  %a2 = fadd float %a1, 17.0
-  %a3 = fadd float %a1, %a2
+  %a1 = fadd fast float %x, 42.0
+  %a2 = fadd fast float %a1, 17.0
+  %a3 = fadd fast float %a1, %a2
   ret float %a3
 }
 
-attributes #0 = { "unsafe-fp-math"="true" }
+; PR32939 - https://bugs.llvm.org/show_bug.cgi?id=32939
+
+define double @fmul2_negated(double %a, double %b, double %c) {
+; CHECK-LABEL: fmul2_negated:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fadd d1, d1, d1
+; CHECK-NEXT:    fmul d1, d1, d2
+; CHECK-NEXT:    fsub d0, d0, d1
+; CHECK-NEXT:    ret
+  %mul = fmul double %b, 2.0
+  %mul1 = fmul double %mul, %c
+  %sub = fsub double %a, %mul1
+  ret double %sub
+}
+
+define <2 x double> @fmul2_negated_vec(<2 x double> %a, <2 x double> %b, <2 x double> %c) {
+; CHECK-LABEL: fmul2_negated_vec:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fadd v1.2d, v1.2d, v1.2d
+; CHECK-NEXT:    fmul v1.2d, v1.2d, v2.2d
+; CHECK-NEXT:    fsub v0.2d, v0.2d, v1.2d
+; CHECK-NEXT:    ret
+  %mul = fmul <2 x double> %b, <double 2.0, double 2.0>
+  %mul1 = fmul <2 x double> %mul, %c
+  %sub = fsub <2 x double> %a, %mul1
+  ret <2 x double> %sub
+}
 
 declare void @use(double)
 
diff --git a/llvm/test/CodeGen/AArch64/fast-isel-branch-uncond-debug.ll b/llvm/test/CodeGen/AArch64/fast-isel-branch-uncond-debug.ll
new file mode 100644
index 000000000..902ec7a9c
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/fast-isel-branch-uncond-debug.ll
@@ -0,0 +1,44 @@
+; RUN: llc -mtriple=aarch64-arm-none-eabi -O1 -opt-bisect-limit=2 -o - %s  2> /dev/null | FileCheck %s
+
+define dso_local i32 @a() #0 !dbg !7 {
+entry:
+; CHECK:    b       .LBB0_1
+; CHECK:  .LBB0_1:
+  call void @llvm.dbg.value(metadata i32 0, metadata !12, metadata !DIExpression()), !dbg !13
+  br label %for.cond, !dbg !14
+
+; CHECK:    b       .LBB0_1
+; CHECK:  .Lfunc_end0:
+for.cond:
+  br label %for.cond, !dbg !15, !llvm.loop !18
+}
+declare void @llvm.lifetime.start.p0i8(i64 immarg, i8* nocapture) #1
+
+declare void @llvm.dbg.value(metadata, metadata, metadata) #2
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "fast-isel-branch-uncond-debug.ll", directory: "/tmp")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{!""}
+!7 = distinct !DISubprogram(name: "a", scope: !1, file: !1, line: 1, type: !8, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !11)
+!8 = !DISubroutineType(types: !9)
+!9 = !{!10}
+!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!11 = !{!12}
+!12 = !DILocalVariable(name: "b", scope: !7, file: !1, line: 2, type: !10)
+!13 = !DILocation(line: 0, scope: !7)
+!14 = !DILocation(line: 3, column: 3, scope: !7)
+!15 = !DILocation(line: 3, column: 3, scope: !16)
+!16 = distinct !DILexicalBlock(scope: !17, file: !1, line: 3, column: 3)
+!17 = distinct !DILexicalBlock(scope: !7, file: !1, line: 3, column: 3)
+!18 = distinct !{!18, !19, !20}
+!19 = !DILocation(line: 3, column: 3, scope: !17)
+!20 = !DILocation(line: 4, column: 5, scope: !17)
+ 
\ No newline at end of file
diff --git a/llvm/test/CodeGen/AArch64/fast-isel-sp-adjust.ll b/llvm/test/CodeGen/AArch64/fast-isel-sp-adjust.ll
index a17a2564b..62815daa6 100644
--- a/llvm/test/CodeGen/AArch64/fast-isel-sp-adjust.ll
+++ b/llvm/test/CodeGen/AArch64/fast-isel-sp-adjust.ll
@@ -13,6 +13,7 @@
 ; CHECK-ERRORS: LLVM ERROR: FastISel missed call
 
 ; CHECK-LABEL: foo:
+; CHECK: sub
 ; CHECK-DAG: mov x[[SP:[0-9]+]], sp
 ; CHECK-DAG: mov [[TMP:w[0-9]+]], #4104
 ; CHECK: mov w[[OFFSET:[0-9]+]], [[TMP]]
diff --git a/llvm/test/CodeGen/AArch64/fastcc-reserved.ll b/llvm/test/CodeGen/AArch64/fastcc-reserved.ll
index b5e03f082..a527354ea 100644
--- a/llvm/test/CodeGen/AArch64/fastcc-reserved.ll
+++ b/llvm/test/CodeGen/AArch64/fastcc-reserved.ll
@@ -1,10 +1,11 @@
 ; RUN: llc -verify-machineinstrs < %s -mtriple=aarch64-none-linux-gnu -tailcallopt | FileCheck %s
+; RUN: llc -global-isel -global-isel-abort=1 -verify-machineinstrs < %s -mtriple=aarch64-none-linux-gnu -tailcallopt | FileCheck %s
 
 ; This test is designed to be run in the situation where the
 ; call-frame is not reserved (hence disable-fp-elim), but where
 ; callee-pop can occur (hence tailcallopt).
 
-declare fastcc void @will_pop([8 x i32], i32 %val)
+declare fastcc void @will_pop([8 x i64], i32 %val)
 
 define fastcc void @foo(i32 %in) {
 ; CHECK-LABEL: foo:
@@ -18,7 +19,7 @@ define fastcc void @foo(i32 %in) {
 ; Reserve space for call-frame:
 ; CHECK: str w{{[0-9]+}}, [sp, #-16]!
 
-  call fastcc void @will_pop([8 x i32] undef, i32 42)
+  call fastcc void @will_pop([8 x i64] undef, i32 42)
 ; CHECK: bl will_pop
 
 ; Since @will_pop is fastcc with tailcallopt, it will put the stack
@@ -31,7 +32,7 @@ define fastcc void @foo(i32 %in) {
   ret void
 }
 
-declare void @wont_pop([8 x i32], i32 %val)
+declare void @wont_pop([8 x i64], i32 %val)
 
 define void @foo1(i32 %in) {
 ; CHECK-LABEL: foo1:
@@ -44,7 +45,7 @@ define void @foo1(i32 %in) {
 ; Reserve space for call-frame
 ; CHECK: str w{{[0-9]+}}, [sp, #-16]!
 
-  call void @wont_pop([8 x i32] undef, i32 42)
+  call void @wont_pop([8 x i64] undef, i32 42)
 ; CHECK: bl wont_pop
 
 ; This time we *do* need to unreserve the call-frame
diff --git a/llvm/test/CodeGen/AArch64/fastcc.ll b/llvm/test/CodeGen/AArch64/fastcc.ll
index d4e116134..56a35b9eb 100644
--- a/llvm/test/CodeGen/AArch64/fastcc.ll
+++ b/llvm/test/CodeGen/AArch64/fastcc.ll
@@ -1,6 +1,6 @@
-; RUN: llc -verify-machineinstrs < %s -mtriple=aarch64-none-linux-gnu -frame-pointer=all -tailcallopt | FileCheck %s -check-prefix CHECK-TAIL
-; RUN: llc -verify-machineinstrs < %s -mtriple=aarch64-none-linux-gnu -frame-pointer=all | FileCheck %s
-; RUN: llc -verify-machineinstrs < %s -mtriple=aarch64-none-linux-gnu -frame-pointer=all -tailcallopt -aarch64-redzone | FileCheck %s -check-prefix CHECK-TAIL-RZ
+; RUN: llc -verify-machineinstrs < %s -mtriple=aarch64-none-linux-gnu -frame-pointer=non-leaf -tailcallopt | FileCheck %s -check-prefix CHECK-TAIL
+; RUN: llc -verify-machineinstrs < %s -mtriple=aarch64-none-linux-gnu -frame-pointer=non-leaf | FileCheck %s
+; RUN: llc -verify-machineinstrs < %s -mtriple=aarch64-none-linux-gnu -frame-pointer=non-leaf -tailcallopt -aarch64-redzone | FileCheck %s -check-prefix CHECK-TAIL-RZ
 
 ; Without tailcallopt fastcc still means the caller cleans up the
 ; stack, so try to make sure this is respected.
@@ -18,7 +18,7 @@ define fastcc void @func_stack0() {
 ; CHECK-TAIL: str w{{[0-9]+}}, [sp]
 
 
-  call fastcc void @func_stack8([8 x i32] undef, i32 42)
+  call fastcc void @func_stack8([8 x i64] undef, i32 42)
 ; CHECK:  bl func_stack8
 ; CHECK-NOT: sub sp, sp,
 ; CHECK-NOT: [sp, #{{[-0-9]+}}]!
@@ -28,7 +28,7 @@ define fastcc void @func_stack0() {
 ; CHECK-TAIL: stp xzr, xzr, [sp, #-16]!
 
 
-  call fastcc void @func_stack32([8 x i32] undef, i128 0, i128 9)
+  call fastcc void @func_stack32([8 x i64] undef, i128 0, i128 9)
 ; CHECK: bl func_stack32
 ; CHECK-NOT: sub sp, sp,
 
@@ -56,7 +56,7 @@ define fastcc void @func_stack0() {
 ; CHECK-TAIL-NEXT: ret
 }
 
-define fastcc void @func_stack8([8 x i32], i32 %stacked) {
+define fastcc void @func_stack8([8 x i64], i32 %stacked) {
 ; CHECK-LABEL: func_stack8:
 ; CHECK: sub sp, sp, #48
 ; CHECK: stp x29, x30, [sp, #32]
@@ -71,7 +71,7 @@ define fastcc void @func_stack8([8 x i32], i32 %stacked) {
 ; CHECK-TAIL: str w{{[0-9]+}}, [sp]
 
 
-  call fastcc void @func_stack8([8 x i32] undef, i32 42)
+  call fastcc void @func_stack8([8 x i64] undef, i32 42)
 ; CHECK:  bl func_stack8
 ; CHECK-NOT: sub sp, sp,
 ; CHECK-NOT: [sp, #{{[-0-9]+}}]!
@@ -82,7 +82,7 @@ define fastcc void @func_stack8([8 x i32], i32 %stacked) {
 ; CHECK-TAIL: stp xzr, xzr, [sp, #-16]!
 
 
-  call fastcc void @func_stack32([8 x i32] undef, i128 0, i128 9)
+  call fastcc void @func_stack32([8 x i64] undef, i128 0, i128 9)
 ; CHECK: bl func_stack32
 ; CHECK-NOT: sub sp, sp,
 
@@ -109,7 +109,7 @@ define fastcc void @func_stack8([8 x i32], i32 %stacked) {
 ; CHECK-TAIL-NEXT: ret
 }
 
-define fastcc void @func_stack32([8 x i32], i128 %stacked0, i128 %stacked1) {
+define fastcc void @func_stack32([8 x i64], i128 %stacked0, i128 %stacked1) {
 ; CHECK-LABEL: func_stack32:
 ; CHECK: add x29, sp, #32
 
@@ -117,7 +117,7 @@ define fastcc void @func_stack32([8 x i32], i128 %stacked0, i128 %stacked1) {
 ; CHECK-TAIL: add x29, sp, #32
 
 
-  call fastcc void @func_stack8([8 x i32] undef, i32 42)
+  call fastcc void @func_stack8([8 x i64] undef, i32 42)
 ; CHECK:  bl func_stack8
 ; CHECK-NOT: sub sp, sp,
 ; CHECK-NOT: [sp, #{{[-0-9]+}}]!
@@ -127,7 +127,7 @@ define fastcc void @func_stack32([8 x i32], i128 %stacked0, i128 %stacked1) {
 ; CHECK-TAIL: stp xzr, xzr, [sp, #-16]!
 
 
-  call fastcc void @func_stack32([8 x i32] undef, i128 0, i128 9)
+  call fastcc void @func_stack32([8 x i64] undef, i128 0, i128 9)
 ; CHECK: bl func_stack32
 ; CHECK-NOT: sub sp, sp,
 
@@ -155,7 +155,7 @@ define fastcc void @func_stack32([8 x i32], i128 %stacked0, i128 %stacked1) {
 }
 
 ; Check that arg stack pop is done after callee-save restore when no frame pointer is used.
-define fastcc void @func_stack32_leaf([8 x i32], i128 %stacked0, i128 %stacked1) {
+define fastcc void @func_stack32_leaf([8 x i64], i128 %stacked0, i128 %stacked1) {
 ; CHECK-LABEL: func_stack32_leaf:
 ; CHECK: str     x20, [sp, #-16]!
 ; CHECK: nop
@@ -186,7 +186,7 @@ define fastcc void @func_stack32_leaf([8 x i32], i128 %stacked0, i128 %stacked1)
 }
 
 ; Check that arg stack pop is done after callee-save restore when no frame pointer is used.
-define fastcc void @func_stack32_leaf_local([8 x i32], i128 %stacked0, i128 %stacked1) {
+define fastcc void @func_stack32_leaf_local([8 x i64], i128 %stacked0, i128 %stacked1) {
 ; CHECK-LABEL: func_stack32_leaf_local:
 ; CHECK: sub     sp, sp, #32
 ; CHECK-NEXT: str     x20, [sp, #16]
@@ -222,7 +222,7 @@ define fastcc void @func_stack32_leaf_local([8 x i32], i128 %stacked0, i128 %sta
 }
 
 ; Check that arg stack pop is done after callee-save restore when no frame pointer is used.
-define fastcc void @func_stack32_leaf_local_nocs([8 x i32], i128 %stacked0, i128 %stacked1) {
+define fastcc void @func_stack32_leaf_local_nocs([8 x i64], i128 %stacked0, i128 %stacked1) {
 ; CHECK-LABEL: func_stack32_leaf_local_nocs:
 ; CHECK: sub     sp, sp, #16
 ; CHECK: add     sp, sp, #16
diff --git a/llvm/test/CodeGen/AArch64/fp-intrinsics.ll b/llvm/test/CodeGen/AArch64/fp-intrinsics.ll
new file mode 100644
index 000000000..3c412a5f7
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/fp-intrinsics.ll
@@ -0,0 +1,1568 @@
+; RUN: llc -mtriple=aarch64-none-eabi %s -o - | FileCheck %s
+
+; Check that constrained fp intrinsics are correctly lowered.
+
+
+; Single-precision intrinsics
+
+; CHECK-LABEL: add_f32:
+; CHECK: fadd s0, s0, s1
+define float @add_f32(float %x, float %y) #0 {
+  %val = call float @llvm.experimental.constrained.fadd.f32(float %x, float %y, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret float %val
+}
+
+; CHECK-LABEL: sub_f32:
+; CHECK: fsub s0, s0, s1
+define float @sub_f32(float %x, float %y) #0 {
+  %val = call float @llvm.experimental.constrained.fsub.f32(float %x, float %y, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret float %val
+}
+
+; CHECK-LABEL: mul_f32:
+; CHECK: fmul s0, s0, s1
+define float @mul_f32(float %x, float %y) #0 {
+  %val = call float @llvm.experimental.constrained.fmul.f32(float %x, float %y, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret float %val
+}
+
+; CHECK-LABEL: div_f32:
+; CHECK: fdiv s0, s0, s1
+define float @div_f32(float %x, float %y) #0 {
+  %val = call float @llvm.experimental.constrained.fdiv.f32(float %x, float %y, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret float %val
+}
+
+; CHECK-LABEL: frem_f32:
+; CHECK: bl fmodf
+define float @frem_f32(float %x, float %y) #0 {
+  %val = call float @llvm.experimental.constrained.frem.f32(float %x, float %y, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret float %val
+}
+
+; CHECK-LABEL: fma_f32:
+; CHECK: fmadd s0, s0, s1, s2
+define float @fma_f32(float %x, float %y, float %z) #0 {
+  %val = call float @llvm.experimental.constrained.fma.f32(float %x, float %y, float %z, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret float %val
+}
+
+; CHECK-LABEL: fptosi_i32_f32:
+; CHECK: fcvtzs w0, s0
+define i32 @fptosi_i32_f32(float %x) #0 {
+  %val = call i32 @llvm.experimental.constrained.fptosi.i32.f32(float %x, metadata !"fpexcept.strict") #0
+  ret i32 %val
+}
+
+; CHECK-LABEL: fptoui_i32_f32:
+; CHECK: fcvtzu w0, s0
+define i32 @fptoui_i32_f32(float %x) #0 {
+  %val = call i32 @llvm.experimental.constrained.fptoui.i32.f32(float %x, metadata !"fpexcept.strict") #0
+  ret i32 %val
+}
+
+; CHECK-LABEL: fptosi_i64_f32:
+; CHECK: fcvtzs x0, s0
+define i64 @fptosi_i64_f32(float %x) #0 {
+  %val = call i64 @llvm.experimental.constrained.fptosi.i64.f32(float %x, metadata !"fpexcept.strict") #0
+  ret i64 %val
+}
+
+; CHECK-LABEL: fptoui_i64_f32:
+; CHECK: fcvtzu x0, s0
+define i64 @fptoui_i64_f32(float %x) #0 {
+  %val = call i64 @llvm.experimental.constrained.fptoui.i64.f32(float %x, metadata !"fpexcept.strict") #0
+  ret i64 %val
+}
+
+; CHECK-LABEL: sitofp_f32_i32:
+; CHECK: scvtf s0, w0
+define float @sitofp_f32_i32(i32 %x) #0 {
+  %val = call float @llvm.experimental.constrained.sitofp.f32.i32(i32 %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret float %val
+}
+
+; CHECK-LABEL: uitofp_f32_i32:
+; CHECK: ucvtf s0, w0
+define float @uitofp_f32_i32(i32 %x) #0 {
+  %val = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret float %val
+}
+
+; CHECK-LABEL: sitofp_f32_i64:
+; CHECK: scvtf s0, x0
+define float @sitofp_f32_i64(i64 %x) #0 {
+  %val = call float @llvm.experimental.constrained.sitofp.f32.i64(i64 %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret float %val
+}
+
+; CHECK-LABEL: uitofp_f32_i64:
+; CHECK: ucvtf s0, x0
+define float @uitofp_f32_i64(i64 %x) #0 {
+  %val = call float @llvm.experimental.constrained.uitofp.f32.i64(i64 %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret float %val
+}
+
+; CHECK-LABEL: sitofp_f32_i128:
+; CHECK: bl __floattisf
+define float @sitofp_f32_i128(i128 %x) #0 {
+  %val = call float @llvm.experimental.constrained.sitofp.f32.i128(i128 %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret float %val
+}
+
+; CHECK-LABEL: uitofp_f32_i128:
+; CHECK: bl __floatuntisf
+define float @uitofp_f32_i128(i128 %x) #0 {
+  %val = call float @llvm.experimental.constrained.uitofp.f32.i128(i128 %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret float %val
+}
+
+; CHECK-LABEL: sqrt_f32:
+; CHECK: fsqrt s0, s0
+define float @sqrt_f32(float %x) #0 {
+  %val = call float @llvm.experimental.constrained.sqrt.f32(float %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret float %val
+}
+
+; CHECK-LABEL: powi_f32:
+; CHECK: bl __powisf2
+define float @powi_f32(float %x, i32 %y) #0 {
+  %val = call float @llvm.experimental.constrained.powi.f32(float %x, i32 %y, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret float %val
+}
+
+; CHECK-LABEL: sin_f32:
+; CHECK: bl sinf
+define float @sin_f32(float %x) #0 {
+  %val = call float @llvm.experimental.constrained.sin.f32(float %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret float %val
+}
+
+; CHECK-LABEL: cos_f32:
+; CHECK: bl cosf
+define float @cos_f32(float %x) #0 {
+  %val = call float @llvm.experimental.constrained.cos.f32(float %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret float %val
+}
+
+; CHECK-LABEL: pow_f32:
+; CHECK: bl powf
+define float @pow_f32(float %x, float %y) #0 {
+  %val = call float @llvm.experimental.constrained.pow.f32(float %x, float %y, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret float %val
+}
+
+; CHECK-LABEL: log_f32:
+; CHECK: bl logf
+define float @log_f32(float %x) #0 {
+  %val = call float @llvm.experimental.constrained.log.f32(float %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret float %val
+}
+
+; CHECK-LABEL: log10_f32:
+; CHECK: bl log10f
+define float @log10_f32(float %x) #0 {
+  %val = call float @llvm.experimental.constrained.log10.f32(float %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret float %val
+}
+
+; CHECK-LABEL: log2_f32:
+; CHECK: bl log2f
+define float @log2_f32(float %x) #0 {
+  %val = call float @llvm.experimental.constrained.log2.f32(float %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret float %val
+}
+
+; CHECK-LABEL: exp_f32:
+; CHECK: bl expf
+define float @exp_f32(float %x) #0 {
+  %val = call float @llvm.experimental.constrained.exp.f32(float %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret float %val
+}
+
+; CHECK-LABEL: exp2_f32:
+; CHECK: bl exp2f
+define float @exp2_f32(float %x) #0 {
+  %val = call float @llvm.experimental.constrained.exp2.f32(float %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret float %val
+}
+
+; CHECK-LABEL: rint_f32:
+; CHECK: frintx s0, s0
+define float @rint_f32(float %x) #0 {
+  %val = call float @llvm.experimental.constrained.rint.f32(float %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret float %val
+}
+
+; CHECK-LABEL: nearbyint_f32:
+; CHECK: frinti s0, s0
+define float @nearbyint_f32(float %x) #0 {
+  %val = call float @llvm.experimental.constrained.nearbyint.f32(float %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret float %val
+}
+
+; CHECK-LABEL: lrint_f32:
+; CHECK: frintx [[REG:s[0-9]+]], s0
+; CHECK: fcvtzs w0, [[REG]]
+define i32 @lrint_f32(float %x) #0 {
+  %val = call i32 @llvm.experimental.constrained.lrint.f32(float %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret i32 %val
+}
+
+; CHECK-LABEL: llrint_f32:
+; CHECK: frintx [[REG:s[0-9]+]], s0
+; CHECK: fcvtzs x0, [[REG]]
+define i64 @llrint_f32(float %x) #0 {
+  %val = call i64 @llvm.experimental.constrained.llrint.f32(float %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret i64 %val
+}
+
+; CHECK-LABEL: maxnum_f32:
+; CHECK: fmaxnm s0, s0, s1
+define float @maxnum_f32(float %x, float %y) #0 {
+  %val = call float @llvm.experimental.constrained.maxnum.f32(float %x, float %y, metadata !"fpexcept.strict") #0
+  ret float %val
+}
+
+; CHECK-LABEL: minnum_f32:
+; CHECK: fminnm s0, s0, s1
+define float @minnum_f32(float %x, float %y) #0 {
+  %val = call float @llvm.experimental.constrained.minnum.f32(float %x, float %y, metadata !"fpexcept.strict") #0
+  ret float %val
+}
+
+; CHECK-LABEL: ceil_f32:
+; CHECK: frintp s0, s0
+define float @ceil_f32(float %x) #0 {
+  %val = call float @llvm.experimental.constrained.ceil.f32(float %x, metadata !"fpexcept.strict") #0
+  ret float %val
+}
+
+; CHECK-LABEL: floor_f32:
+; CHECK: frintm s0, s0
+define float @floor_f32(float %x) #0 {
+  %val = call float @llvm.experimental.constrained.floor.f32(float %x, metadata !"fpexcept.strict") #0
+  ret float %val
+}
+
+; CHECK-LABEL: lround_f32:
+; CHECK: fcvtas w0, s0
+define i32 @lround_f32(float %x) #0 {
+  %val = call i32 @llvm.experimental.constrained.lround.f32(float %x, metadata !"fpexcept.strict") #0
+  ret i32 %val
+}
+
+; CHECK-LABEL: llround_f32:
+; CHECK: fcvtas x0, s0
+define i64 @llround_f32(float %x) #0 {
+  %val = call i64 @llvm.experimental.constrained.llround.f32(float %x, metadata !"fpexcept.strict") #0
+  ret i64 %val
+}
+
+; CHECK-LABEL: round_f32:
+; CHECK: frinta s0, s0
+define float @round_f32(float %x) #0 {
+  %val = call float @llvm.experimental.constrained.round.f32(float %x, metadata !"fpexcept.strict") #0
+  ret float %val
+}
+
+; CHECK-LABEL: trunc_f32:
+; CHECK: frintz s0, s0
+define float @trunc_f32(float %x) #0 {
+  %val = call float @llvm.experimental.constrained.trunc.f32(float %x, metadata !"fpexcept.strict") #0
+  ret float %val
+}
+
+; CHECK-LABEL: fcmp_olt_f32:
+; CHECK: fcmp s0, s1
+define i32 @fcmp_olt_f32(float %a, float %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f32(float %a, float %b, metadata !"olt", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmp_ole_f32:
+; CHECK: fcmp s0, s1
+define i32 @fcmp_ole_f32(float %a, float %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f32(float %a, float %b, metadata !"ole", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmp_ogt_f32:
+; CHECK: fcmp s0, s1
+define i32 @fcmp_ogt_f32(float %a, float %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f32(float %a, float %b, metadata !"ogt", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmp_oge_f32:
+; CHECK: fcmp s0, s1
+define i32 @fcmp_oge_f32(float %a, float %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f32(float %a, float %b, metadata !"oge", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmp_oeq_f32:
+; CHECK: fcmp s0, s1
+define i32 @fcmp_oeq_f32(float %a, float %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f32(float %a, float %b, metadata !"oeq", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmp_one_f32:
+; CHECK: fcmp s0, s1
+define i32 @fcmp_one_f32(float %a, float %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f32(float %a, float %b, metadata !"one", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmp_ult_f32:
+; CHECK: fcmp s0, s1
+define i32 @fcmp_ult_f32(float %a, float %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f32(float %a, float %b, metadata !"ult", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmp_ule_f32:
+; CHECK: fcmp s0, s1
+define i32 @fcmp_ule_f32(float %a, float %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f32(float %a, float %b, metadata !"ule", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmp_ugt_f32:
+; CHECK: fcmp s0, s1
+define i32 @fcmp_ugt_f32(float %a, float %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f32(float %a, float %b, metadata !"ugt", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmp_uge_f32:
+; CHECK: fcmp s0, s1
+define i32 @fcmp_uge_f32(float %a, float %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f32(float %a, float %b, metadata !"uge", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmp_ueq_f32:
+; CHECK: fcmp s0, s1
+define i32 @fcmp_ueq_f32(float %a, float %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f32(float %a, float %b, metadata !"ueq", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmp_une_f32:
+; CHECK: fcmp s0, s1
+define i32 @fcmp_une_f32(float %a, float %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f32(float %a, float %b, metadata !"une", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmps_olt_f32:
+; CHECK: fcmpe s0, s1
+define i32 @fcmps_olt_f32(float %a, float %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmps.f32(float %a, float %b, metadata !"olt", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmps_ole_f32:
+; CHECK: fcmpe s0, s1
+define i32 @fcmps_ole_f32(float %a, float %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmps.f32(float %a, float %b, metadata !"ole", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmps_ogt_f32:
+; CHECK: fcmpe s0, s1
+define i32 @fcmps_ogt_f32(float %a, float %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmps.f32(float %a, float %b, metadata !"ogt", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmps_oge_f32:
+; CHECK: fcmpe s0, s1
+define i32 @fcmps_oge_f32(float %a, float %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmps.f32(float %a, float %b, metadata !"oge", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmps_oeq_f32:
+; CHECK: fcmpe s0, s1
+define i32 @fcmps_oeq_f32(float %a, float %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmps.f32(float %a, float %b, metadata !"oeq", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmps_one_f32:
+; CHECK: fcmpe s0, s1
+define i32 @fcmps_one_f32(float %a, float %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmps.f32(float %a, float %b, metadata !"one", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmps_ult_f32:
+; CHECK: fcmpe s0, s1
+define i32 @fcmps_ult_f32(float %a, float %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmps.f32(float %a, float %b, metadata !"ult", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmps_ule_f32:
+; CHECK: fcmpe s0, s1
+define i32 @fcmps_ule_f32(float %a, float %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmps.f32(float %a, float %b, metadata !"ule", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmps_ugt_f32:
+; CHECK: fcmpe s0, s1
+define i32 @fcmps_ugt_f32(float %a, float %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmps.f32(float %a, float %b, metadata !"ugt", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmps_uge_f32:
+; CHECK: fcmpe s0, s1
+define i32 @fcmps_uge_f32(float %a, float %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmps.f32(float %a, float %b, metadata !"uge", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmps_ueq_f32:
+; CHECK: fcmpe s0, s1
+define i32 @fcmps_ueq_f32(float %a, float %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmps.f32(float %a, float %b, metadata !"ueq", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmps_une_f32:
+; CHECK: fcmpe s0, s1
+define i32 @fcmps_une_f32(float %a, float %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmps.f32(float %a, float %b, metadata !"une", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+
+; Double-precision intrinsics
+
+; CHECK-LABEL: add_f64:
+; CHECK: fadd d0, d0, d1
+define double @add_f64(double %x, double %y) #0 {
+  %val = call double @llvm.experimental.constrained.fadd.f64(double %x, double %y, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret double %val
+}
+
+; CHECK-LABEL: sub_f64:
+; CHECK: fsub d0, d0, d1
+define double @sub_f64(double %x, double %y) #0 {
+  %val = call double @llvm.experimental.constrained.fsub.f64(double %x, double %y, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret double %val
+}
+
+; CHECK-LABEL: mul_f64:
+; CHECK: fmul d0, d0, d1
+define double @mul_f64(double %x, double %y) #0 {
+  %val = call double @llvm.experimental.constrained.fmul.f64(double %x, double %y, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret double %val
+}
+
+; CHECK-LABEL: div_f64:
+; CHECK: fdiv d0, d0, d1
+define double @div_f64(double %x, double %y) #0 {
+  %val = call double @llvm.experimental.constrained.fdiv.f64(double %x, double %y, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret double %val
+}
+
+; CHECK-LABEL: frem_f64:
+; CHECK: bl fmod
+define double @frem_f64(double %x, double %y) #0 {
+  %val = call double @llvm.experimental.constrained.frem.f64(double %x, double %y, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret double %val
+}
+
+; CHECK-LABEL: fma_f64:
+; CHECK: fmadd d0, d0, d1, d2
+define double @fma_f64(double %x, double %y, double %z) #0 {
+  %val = call double @llvm.experimental.constrained.fma.f64(double %x, double %y, double %z, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret double %val
+}
+
+; CHECK-LABEL: fptosi_i32_f64:
+; CHECK: fcvtzs w0, d0
+define i32 @fptosi_i32_f64(double %x) #0 {
+  %val = call i32 @llvm.experimental.constrained.fptosi.i32.f64(double %x, metadata !"fpexcept.strict") #0
+  ret i32 %val
+}
+
+; CHECK-LABEL: fptoui_i32_f64:
+; CHECK: fcvtzu w0, d0
+define i32 @fptoui_i32_f64(double %x) #0 {
+  %val = call i32 @llvm.experimental.constrained.fptoui.i32.f64(double %x, metadata !"fpexcept.strict") #0
+  ret i32 %val
+}
+
+; CHECK-LABEL: fptosi_i64_f64:
+; CHECK: fcvtzs x0, d0
+define i64 @fptosi_i64_f64(double %x) #0 {
+  %val = call i64 @llvm.experimental.constrained.fptosi.i64.f64(double %x, metadata !"fpexcept.strict") #0
+  ret i64 %val
+}
+
+; CHECK-LABEL: fptoui_i64_f64:
+; CHECK: fcvtzu x0, d0
+define i64 @fptoui_i64_f64(double %x) #0 {
+  %val = call i64 @llvm.experimental.constrained.fptoui.i64.f64(double %x, metadata !"fpexcept.strict") #0
+  ret i64 %val
+}
+
+; CHECK-LABEL: sitofp_f64_i32:
+; CHECK: scvtf d0, w0
+define double @sitofp_f64_i32(i32 %x) #0 {
+  %val = call double @llvm.experimental.constrained.sitofp.f64.i32(i32 %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret double %val
+}
+
+; CHECK-LABEL: uitofp_f64_i32:
+; CHECK: ucvtf d0, w0
+define double @uitofp_f64_i32(i32 %x) #0 {
+  %val = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret double %val
+}
+
+; CHECK-LABEL: sitofp_f64_i64:
+; CHECK: scvtf d0, x0
+define double @sitofp_f64_i64(i64 %x) #0 {
+  %val = call double @llvm.experimental.constrained.sitofp.f64.i64(i64 %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret double %val
+}
+
+; CHECK-LABEL: uitofp_f64_i64:
+; CHECK: ucvtf d0, x0
+define double @uitofp_f64_i64(i64 %x) #0 {
+  %val = call double @llvm.experimental.constrained.uitofp.f64.i64(i64 %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret double %val
+}
+
+; CHECK-LABEL: sitofp_f64_i128:
+; CHECK: bl __floattidf
+define double @sitofp_f64_i128(i128 %x) #0 {
+  %val = call double @llvm.experimental.constrained.sitofp.f64.i128(i128 %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret double %val
+}
+
+; CHECK-LABEL: uitofp_f64_i128:
+; CHECK: bl __floatuntidf
+define double @uitofp_f64_i128(i128 %x) #0 {
+  %val = call double @llvm.experimental.constrained.uitofp.f64.i128(i128 %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret double %val
+}
+
+; CHECK-LABEL: sqrt_f64:
+; CHECK: fsqrt d0, d0
+define double @sqrt_f64(double %x) #0 {
+  %val = call double @llvm.experimental.constrained.sqrt.f64(double %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret double %val
+}
+
+; CHECK-LABEL: powi_f64:
+; CHECK: bl __powidf2
+define double @powi_f64(double %x, i32 %y) #0 {
+  %val = call double @llvm.experimental.constrained.powi.f64(double %x, i32 %y, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret double %val
+}
+
+; CHECK-LABEL: sin_f64:
+; CHECK: bl sin
+define double @sin_f64(double %x) #0 {
+  %val = call double @llvm.experimental.constrained.sin.f64(double %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret double %val
+}
+
+; CHECK-LABEL: cos_f64:
+; CHECK: bl cos
+define double @cos_f64(double %x) #0 {
+  %val = call double @llvm.experimental.constrained.cos.f64(double %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret double %val
+}
+
+; CHECK-LABEL: pow_f64:
+; CHECK: bl pow
+define double @pow_f64(double %x, double %y) #0 {
+  %val = call double @llvm.experimental.constrained.pow.f64(double %x, double %y, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret double %val
+}
+
+; CHECK-LABEL: log_f64:
+; CHECK: bl log
+define double @log_f64(double %x) #0 {
+  %val = call double @llvm.experimental.constrained.log.f64(double %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret double %val
+}
+
+; CHECK-LABEL: log10_f64:
+; CHECK: bl log10
+define double @log10_f64(double %x) #0 {
+  %val = call double @llvm.experimental.constrained.log10.f64(double %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret double %val
+}
+
+; CHECK-LABEL: log2_f64:
+; CHECK: bl log2
+define double @log2_f64(double %x) #0 {
+  %val = call double @llvm.experimental.constrained.log2.f64(double %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret double %val
+}
+
+; CHECK-LABEL: exp_f64:
+; CHECK: bl exp
+define double @exp_f64(double %x) #0 {
+  %val = call double @llvm.experimental.constrained.exp.f64(double %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret double %val
+}
+
+; CHECK-LABEL: exp2_f64:
+; CHECK: bl exp2
+define double @exp2_f64(double %x) #0 {
+  %val = call double @llvm.experimental.constrained.exp2.f64(double %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret double %val
+}
+
+; CHECK-LABEL: rint_f64:
+; CHECK: frintx d0, d0
+define double @rint_f64(double %x) #0 {
+  %val = call double @llvm.experimental.constrained.rint.f64(double %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret double %val
+}
+
+; CHECK-LABEL: nearbyint_f64:
+; CHECK: frinti d0, d0
+define double @nearbyint_f64(double %x) #0 {
+  %val = call double @llvm.experimental.constrained.nearbyint.f64(double %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret double %val
+}
+
+; CHECK-LABEL: lrint_f64:
+; CHECK: frintx [[REG:d[0-9]+]], d0
+; CHECK: fcvtzs w0, [[REG]]
+define i32 @lrint_f64(double %x) #0 {
+  %val = call i32 @llvm.experimental.constrained.lrint.f64(double %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret i32 %val
+}
+
+; CHECK-LABEL: llrint_f64:
+; CHECK: frintx [[REG:d[0-9]+]], d0
+; CHECK: fcvtzs x0, [[REG]]
+define i64 @llrint_f64(double %x) #0 {
+  %val = call i64 @llvm.experimental.constrained.llrint.f64(double %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret i64 %val
+}
+
+; CHECK-LABEL: maxnum_f64:
+; CHECK: fmaxnm d0, d0, d1
+define double @maxnum_f64(double %x, double %y) #0 {
+  %val = call double @llvm.experimental.constrained.maxnum.f64(double %x, double %y, metadata !"fpexcept.strict") #0
+  ret double %val
+}
+
+; CHECK-LABEL: minnum_f64:
+; CHECK: fminnm d0, d0, d1
+define double @minnum_f64(double %x, double %y) #0 {
+  %val = call double @llvm.experimental.constrained.minnum.f64(double %x, double %y, metadata !"fpexcept.strict") #0
+  ret double %val
+}
+
+; CHECK-LABEL: ceil_f64:
+; CHECK: frintp d0, d0
+define double @ceil_f64(double %x) #0 {
+  %val = call double @llvm.experimental.constrained.ceil.f64(double %x, metadata !"fpexcept.strict") #0
+  ret double %val
+}
+
+; CHECK-LABEL: floor_f64:
+; CHECK: frintm d0, d0
+define double @floor_f64(double %x) #0 {
+  %val = call double @llvm.experimental.constrained.floor.f64(double %x, metadata !"fpexcept.strict") #0
+  ret double %val
+}
+
+; CHECK-LABEL: lround_f64:
+; CHECK: fcvtas w0, d0
+define i32 @lround_f64(double %x) #0 {
+  %val = call i32 @llvm.experimental.constrained.lround.f64(double %x, metadata !"fpexcept.strict") #0
+  ret i32 %val
+}
+
+; CHECK-LABEL: llround_f64:
+; CHECK: fcvtas x0, d0
+define i64 @llround_f64(double %x) #0 {
+  %val = call i64 @llvm.experimental.constrained.llround.f64(double %x, metadata !"fpexcept.strict") #0
+  ret i64 %val
+}
+
+; CHECK-LABEL: round_f64:
+; CHECK: frinta d0, d0
+define double @round_f64(double %x) #0 {
+  %val = call double @llvm.experimental.constrained.round.f64(double %x, metadata !"fpexcept.strict") #0
+  ret double %val
+}
+
+; CHECK-LABEL: trunc_f64:
+; CHECK: frintz d0, d0
+define double @trunc_f64(double %x) #0 {
+  %val = call double @llvm.experimental.constrained.trunc.f64(double %x, metadata !"fpexcept.strict") #0
+  ret double %val
+}
+
+; CHECK-LABEL: fcmp_olt_f64:
+; CHECK: fcmp d0, d1
+define i32 @fcmp_olt_f64(double %a, double %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f64(double %a, double %b, metadata !"olt", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmp_ole_f64:
+; CHECK: fcmp d0, d1
+define i32 @fcmp_ole_f64(double %a, double %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f64(double %a, double %b, metadata !"ole", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmp_ogt_f64:
+; CHECK: fcmp d0, d1
+define i32 @fcmp_ogt_f64(double %a, double %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f64(double %a, double %b, metadata !"ogt", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmp_oge_f64:
+; CHECK: fcmp d0, d1
+define i32 @fcmp_oge_f64(double %a, double %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f64(double %a, double %b, metadata !"oge", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmp_oeq_f64:
+; CHECK: fcmp d0, d1
+define i32 @fcmp_oeq_f64(double %a, double %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f64(double %a, double %b, metadata !"oeq", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmp_one_f64:
+; CHECK: fcmp d0, d1
+define i32 @fcmp_one_f64(double %a, double %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f64(double %a, double %b, metadata !"one", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmp_ult_f64:
+; CHECK: fcmp d0, d1
+define i32 @fcmp_ult_f64(double %a, double %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f64(double %a, double %b, metadata !"ult", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmp_ule_f64:
+; CHECK: fcmp d0, d1
+define i32 @fcmp_ule_f64(double %a, double %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f64(double %a, double %b, metadata !"ule", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmp_ugt_f64:
+; CHECK: fcmp d0, d1
+define i32 @fcmp_ugt_f64(double %a, double %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f64(double %a, double %b, metadata !"ugt", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmp_uge_f64:
+; CHECK: fcmp d0, d1
+define i32 @fcmp_uge_f64(double %a, double %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f64(double %a, double %b, metadata !"uge", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmp_ueq_f64:
+; CHECK: fcmp d0, d1
+define i32 @fcmp_ueq_f64(double %a, double %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f64(double %a, double %b, metadata !"ueq", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmp_une_f64:
+; CHECK: fcmp d0, d1
+define i32 @fcmp_une_f64(double %a, double %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f64(double %a, double %b, metadata !"une", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmps_olt_f64:
+; CHECK: fcmpe d0, d1
+define i32 @fcmps_olt_f64(double %a, double %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmps.f64(double %a, double %b, metadata !"olt", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmps_ole_f64:
+; CHECK: fcmpe d0, d1
+define i32 @fcmps_ole_f64(double %a, double %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmps.f64(double %a, double %b, metadata !"ole", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmps_ogt_f64:
+; CHECK: fcmpe d0, d1
+define i32 @fcmps_ogt_f64(double %a, double %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmps.f64(double %a, double %b, metadata !"ogt", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmps_oge_f64:
+; CHECK: fcmpe d0, d1
+define i32 @fcmps_oge_f64(double %a, double %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmps.f64(double %a, double %b, metadata !"oge", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmps_oeq_f64:
+; CHECK: fcmpe d0, d1
+define i32 @fcmps_oeq_f64(double %a, double %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmps.f64(double %a, double %b, metadata !"oeq", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmps_one_f64:
+; CHECK: fcmpe d0, d1
+define i32 @fcmps_one_f64(double %a, double %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmps.f64(double %a, double %b, metadata !"one", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmps_ult_f64:
+; CHECK: fcmpe d0, d1
+define i32 @fcmps_ult_f64(double %a, double %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmps.f64(double %a, double %b, metadata !"ult", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmps_ule_f64:
+; CHECK: fcmpe d0, d1
+define i32 @fcmps_ule_f64(double %a, double %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmps.f64(double %a, double %b, metadata !"ule", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmps_ugt_f64:
+; CHECK: fcmpe d0, d1
+define i32 @fcmps_ugt_f64(double %a, double %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmps.f64(double %a, double %b, metadata !"ugt", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmps_uge_f64:
+; CHECK: fcmpe d0, d1
+define i32 @fcmps_uge_f64(double %a, double %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmps.f64(double %a, double %b, metadata !"uge", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmps_ueq_f64:
+; CHECK: fcmpe d0, d1
+define i32 @fcmps_ueq_f64(double %a, double %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmps.f64(double %a, double %b, metadata !"ueq", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmps_une_f64:
+; CHECK: fcmpe d0, d1
+define i32 @fcmps_une_f64(double %a, double %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmps.f64(double %a, double %b, metadata !"une", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+
+; Long-double-precision intrinsics
+
+; CHECK-LABEL: add_f128:
+; CHECK: bl __addtf3
+define fp128 @add_f128(fp128 %x, fp128 %y) #0 {
+  %val = call fp128 @llvm.experimental.constrained.fadd.f128(fp128 %x, fp128 %y, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret fp128 %val
+}
+
+; CHECK-LABEL: sub_f128:
+; CHECK: bl __subtf3
+define fp128 @sub_f128(fp128 %x, fp128 %y) #0 {
+  %val = call fp128 @llvm.experimental.constrained.fsub.f128(fp128 %x, fp128 %y, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret fp128 %val
+}
+
+; CHECK-LABEL: mul_f128:
+; CHECK: bl __multf3
+define fp128 @mul_f128(fp128 %x, fp128 %y) #0 {
+  %val = call fp128 @llvm.experimental.constrained.fmul.f128(fp128 %x, fp128 %y, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret fp128 %val
+}
+
+; CHECK-LABEL: div_f128:
+; CHECK: bl __divtf3
+define fp128 @div_f128(fp128 %x, fp128 %y) #0 {
+  %val = call fp128 @llvm.experimental.constrained.fdiv.f128(fp128 %x, fp128 %y, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret fp128 %val
+}
+
+; CHECK-LABEL: frem_f128:
+; CHECK: bl fmodl
+define fp128 @frem_f128(fp128 %x, fp128 %y) #0 {
+  %val = call fp128 @llvm.experimental.constrained.frem.f128(fp128 %x, fp128 %y, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret fp128 %val
+}
+
+; CHECK-LABEL: fma_f128:
+; CHECK: fmal
+define fp128 @fma_f128(fp128 %x, fp128 %y, fp128 %z) #0 {
+  %val = call fp128 @llvm.experimental.constrained.fma.f128(fp128 %x, fp128 %y, fp128 %z, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret fp128 %val
+}
+
+; CHECK-LABEL: fptosi_i32_f128:
+; CHECK: bl __fixtfsi
+define i32 @fptosi_i32_f128(fp128 %x) #0 {
+  %val = call i32 @llvm.experimental.constrained.fptosi.i32.f128(fp128 %x, metadata !"fpexcept.strict") #0
+  ret i32 %val
+}
+
+; CHECK-LABEL: fptoui_i32_f128:
+; CHECK: bl __fixunstfsi
+define i32 @fptoui_i32_f128(fp128 %x) #0 {
+  %val = call i32 @llvm.experimental.constrained.fptoui.i32.f128(fp128 %x, metadata !"fpexcept.strict") #0
+  ret i32 %val
+}
+
+; CHECK-LABEL: fptosi_i64_f128:
+; CHECK: bl __fixtfdi
+define i64 @fptosi_i64_f128(fp128 %x) #0 {
+  %val = call i64 @llvm.experimental.constrained.fptosi.i64.f128(fp128 %x, metadata !"fpexcept.strict") #0
+  ret i64 %val
+}
+
+; CHECK-LABEL: fptoui_i64_f128:
+; CHECK: bl __fixunstfdi
+define i64 @fptoui_i64_f128(fp128 %x) #0 {
+  %val = call i64 @llvm.experimental.constrained.fptoui.i64.f128(fp128 %x, metadata !"fpexcept.strict") #0
+  ret i64 %val
+}
+
+; CHECK-LABEL: sitofp_f128_i32:
+; CHECK: bl __floatsitf
+define fp128 @sitofp_f128_i32(i32 %x) #0 {
+  %val = call fp128 @llvm.experimental.constrained.sitofp.f128.i32(i32 %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret fp128 %val
+}
+
+; CHECK-LABEL: uitofp_f128_i32:
+; CHECK: bl __floatunsitf
+define fp128 @uitofp_f128_i32(i32 %x) #0 {
+  %val = call fp128 @llvm.experimental.constrained.uitofp.f128.i32(i32 %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret fp128 %val
+}
+
+; CHECK-LABEL: sitofp_f128_i64:
+; CHECK: bl __floatditf
+define fp128 @sitofp_f128_i64(i64 %x) #0 {
+  %val = call fp128 @llvm.experimental.constrained.sitofp.f128.i64(i64 %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret fp128 %val
+}
+
+; CHECK-LABEL: uitofp_f128_i64:
+; CHECK: bl __floatunditf
+define fp128 @uitofp_f128_i64(i64 %x) #0 {
+  %val = call fp128 @llvm.experimental.constrained.uitofp.f128.i64(i64 %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret fp128 %val
+}
+
+; CHECK-LABEL: sitofp_f128_i128:
+; CHECK: bl __floattitf
+define fp128 @sitofp_f128_i128(i128 %x) #0 {
+  %val = call fp128 @llvm.experimental.constrained.sitofp.f128.i128(i128 %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret fp128 %val
+}
+
+; CHECK-LABEL: uitofp_f128_i128:
+; CHECK: bl __floatuntitf
+define fp128 @uitofp_f128_i128(i128 %x) #0 {
+  %val = call fp128 @llvm.experimental.constrained.uitofp.f128.i128(i128 %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret fp128 %val
+}
+
+; CHECK-LABEL: sqrt_f128:
+; CHECK: bl sqrtl
+define fp128 @sqrt_f128(fp128 %x) #0 {
+  %val = call fp128 @llvm.experimental.constrained.sqrt.f128(fp128 %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret fp128 %val
+}
+
+; CHECK-LABEL: powi_f128:
+; CHECK: bl __powitf2
+define fp128 @powi_f128(fp128 %x, i32 %y) #0 {
+  %val = call fp128 @llvm.experimental.constrained.powi.f128(fp128 %x, i32 %y, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret fp128 %val
+}
+
+; CHECK-LABEL: sin_f128:
+; CHECK: bl sinl
+define fp128 @sin_f128(fp128 %x) #0 {
+  %val = call fp128 @llvm.experimental.constrained.sin.f128(fp128 %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret fp128 %val
+}
+
+; CHECK-LABEL: cos_f128:
+; CHECK: bl cosl
+define fp128 @cos_f128(fp128 %x) #0 {
+  %val = call fp128 @llvm.experimental.constrained.cos.f128(fp128 %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret fp128 %val
+}
+
+; CHECK-LABEL: pow_f128:
+; CHECK: bl powl
+define fp128 @pow_f128(fp128 %x, fp128 %y) #0 {
+  %val = call fp128 @llvm.experimental.constrained.pow.f128(fp128 %x, fp128 %y, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret fp128 %val
+}
+
+; CHECK-LABEL: log_f128:
+; CHECK: bl logl
+define fp128 @log_f128(fp128 %x) #0 {
+  %val = call fp128 @llvm.experimental.constrained.log.f128(fp128 %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret fp128 %val
+}
+
+; CHECK-LABEL: log10_f128:
+; CHECK: bl log10l
+define fp128 @log10_f128(fp128 %x) #0 {
+  %val = call fp128 @llvm.experimental.constrained.log10.f128(fp128 %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret fp128 %val
+}
+
+; CHECK-LABEL: log2_f128:
+; CHECK: bl log2l
+define fp128 @log2_f128(fp128 %x) #0 {
+  %val = call fp128 @llvm.experimental.constrained.log2.f128(fp128 %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret fp128 %val
+}
+
+; CHECK-LABEL: exp_f128:
+; CHECK: bl expl
+define fp128 @exp_f128(fp128 %x) #0 {
+  %val = call fp128 @llvm.experimental.constrained.exp.f128(fp128 %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret fp128 %val
+}
+
+; CHECK-LABEL: exp2_f128:
+; CHECK: bl exp2l
+define fp128 @exp2_f128(fp128 %x) #0 {
+  %val = call fp128 @llvm.experimental.constrained.exp2.f128(fp128 %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret fp128 %val
+}
+
+; CHECK-LABEL: rint_f128:
+; CHECK: bl rintl
+define fp128 @rint_f128(fp128 %x) #0 {
+  %val = call fp128 @llvm.experimental.constrained.rint.f128(fp128 %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret fp128 %val
+}
+
+; CHECK-LABEL: nearbyint_f128:
+; CHECK: bl nearbyintl
+define fp128 @nearbyint_f128(fp128 %x) #0 {
+  %val = call fp128 @llvm.experimental.constrained.nearbyint.f128(fp128 %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret fp128 %val
+}
+
+; CHECK-LABEL: lrint_f128:
+; CHECK: bl lrintl
+define i32 @lrint_f128(fp128 %x) #0 {
+  %val = call i32 @llvm.experimental.constrained.lrint.f128(fp128 %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret i32 %val
+}
+
+; CHECK-LABEL: llrint_f128:
+; CHECK: bl llrintl
+define i64 @llrint_f128(fp128 %x) #0 {
+  %val = call i64 @llvm.experimental.constrained.llrint.f128(fp128 %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret i64 %val
+}
+
+; CHECK-LABEL: maxnum_f128:
+; CHECK: bl fmaxl
+define fp128 @maxnum_f128(fp128 %x, fp128 %y) #0 {
+  %val = call fp128 @llvm.experimental.constrained.maxnum.f128(fp128 %x, fp128 %y, metadata !"fpexcept.strict") #0
+  ret fp128 %val
+}
+
+; CHECK-LABEL: minnum_f128:
+; CHECK: bl fminl
+define fp128 @minnum_f128(fp128 %x, fp128 %y) #0 {
+  %val = call fp128 @llvm.experimental.constrained.minnum.f128(fp128 %x, fp128 %y, metadata !"fpexcept.strict") #0
+  ret fp128 %val
+}
+
+; CHECK-LABEL: ceil_f128:
+; CHECK: bl ceill
+define fp128 @ceil_f128(fp128 %x) #0 {
+  %val = call fp128 @llvm.experimental.constrained.ceil.f128(fp128 %x, metadata !"fpexcept.strict") #0
+  ret fp128 %val
+}
+
+; CHECK-LABEL: floor_f128:
+; CHECK: bl floorl
+define fp128 @floor_f128(fp128 %x) #0 {
+  %val = call fp128 @llvm.experimental.constrained.floor.f128(fp128 %x, metadata !"fpexcept.strict") #0
+  ret fp128 %val
+}
+
+; CHECK-LABEL: lround_f128:
+; CHECK: bl lroundl
+define i32 @lround_f128(fp128 %x) #0 {
+  %val = call i32 @llvm.experimental.constrained.lround.f128(fp128 %x, metadata !"fpexcept.strict") #0
+  ret i32 %val
+}
+
+; CHECK-LABEL: llround_f128:
+; CHECK: bl llroundl
+define i64 @llround_f128(fp128 %x) #0 {
+  %val = call i64 @llvm.experimental.constrained.llround.f128(fp128 %x, metadata !"fpexcept.strict") #0
+  ret i64 %val
+}
+
+; CHECK-LABEL: round_f128:
+; CHECK: bl roundl
+define fp128 @round_f128(fp128 %x) #0 {
+  %val = call fp128 @llvm.experimental.constrained.round.f128(fp128 %x, metadata !"fpexcept.strict") #0
+  ret fp128 %val
+}
+
+; CHECK-LABEL: trunc_f128:
+; CHECK: bl truncl
+define fp128 @trunc_f128(fp128 %x) #0 {
+  %val = call fp128 @llvm.experimental.constrained.trunc.f128(fp128 %x, metadata !"fpexcept.strict") #0
+  ret fp128 %val
+}
+
+; CHECK-LABEL: fcmp_olt_f128:
+; CHECK: bl __lttf2
+define i32 @fcmp_olt_f128(fp128 %a, fp128 %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f128(fp128 %a, fp128 %b, metadata !"olt", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmp_ole_f128:
+; CHECK: bl __letf2
+define i32 @fcmp_ole_f128(fp128 %a, fp128 %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f128(fp128 %a, fp128 %b, metadata !"ole", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmp_ogt_f128:
+; CHECK: bl __gttf2
+define i32 @fcmp_ogt_f128(fp128 %a, fp128 %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f128(fp128 %a, fp128 %b, metadata !"ogt", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmp_oge_f128:
+; CHECK: bl __getf2
+define i32 @fcmp_oge_f128(fp128 %a, fp128 %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f128(fp128 %a, fp128 %b, metadata !"oge", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmp_oeq_f128:
+; CHECK: bl __eqtf2
+define i32 @fcmp_oeq_f128(fp128 %a, fp128 %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f128(fp128 %a, fp128 %b, metadata !"oeq", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmp_one_f128:
+; CHECK: bl __eqtf2
+define i32 @fcmp_one_f128(fp128 %a, fp128 %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f128(fp128 %a, fp128 %b, metadata !"one", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmp_ult_f128:
+; CHECK: bl __getf2
+define i32 @fcmp_ult_f128(fp128 %a, fp128 %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f128(fp128 %a, fp128 %b, metadata !"ult", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmp_ule_f128:
+; CHECK: bl __gttf2
+define i32 @fcmp_ule_f128(fp128 %a, fp128 %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f128(fp128 %a, fp128 %b, metadata !"ule", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmp_ugt_f128:
+; CHECK: bl __letf2
+define i32 @fcmp_ugt_f128(fp128 %a, fp128 %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f128(fp128 %a, fp128 %b, metadata !"ugt", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmp_uge_f128:
+; CHECK: bl __lttf2
+define i32 @fcmp_uge_f128(fp128 %a, fp128 %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f128(fp128 %a, fp128 %b, metadata !"uge", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmp_ueq_f128:
+; CHECK: bl __eqtf2
+define i32 @fcmp_ueq_f128(fp128 %a, fp128 %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f128(fp128 %a, fp128 %b, metadata !"ueq", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmp_une_f128:
+; CHECK: bl __netf2
+define i32 @fcmp_une_f128(fp128 %a, fp128 %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f128(fp128 %a, fp128 %b, metadata !"une", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmps_olt_f128:
+; CHECK: bl __lttf2
+define i32 @fcmps_olt_f128(fp128 %a, fp128 %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmps.f128(fp128 %a, fp128 %b, metadata !"olt", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmps_ole_f128:
+; CHECK: bl __letf2
+define i32 @fcmps_ole_f128(fp128 %a, fp128 %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmps.f128(fp128 %a, fp128 %b, metadata !"ole", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmps_ogt_f128:
+; CHECK: bl __gttf2
+define i32 @fcmps_ogt_f128(fp128 %a, fp128 %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmps.f128(fp128 %a, fp128 %b, metadata !"ogt", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmps_oge_f128:
+; CHECK: bl __getf2
+define i32 @fcmps_oge_f128(fp128 %a, fp128 %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmps.f128(fp128 %a, fp128 %b, metadata !"oge", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmps_oeq_f128:
+; CHECK: bl __eqtf2
+define i32 @fcmps_oeq_f128(fp128 %a, fp128 %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmps.f128(fp128 %a, fp128 %b, metadata !"oeq", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmps_one_f128:
+; CHECK: bl __eqtf2
+define i32 @fcmps_one_f128(fp128 %a, fp128 %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmps.f128(fp128 %a, fp128 %b, metadata !"one", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmps_ult_f128:
+; CHECK: bl __getf2
+define i32 @fcmps_ult_f128(fp128 %a, fp128 %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmps.f128(fp128 %a, fp128 %b, metadata !"ult", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmps_ule_f128:
+; CHECK: bl __gttf2
+define i32 @fcmps_ule_f128(fp128 %a, fp128 %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmps.f128(fp128 %a, fp128 %b, metadata !"ule", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmps_ugt_f128:
+; CHECK: bl __letf2
+define i32 @fcmps_ugt_f128(fp128 %a, fp128 %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmps.f128(fp128 %a, fp128 %b, metadata !"ugt", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmps_uge_f128:
+; CHECK: bl __lttf2
+define i32 @fcmps_uge_f128(fp128 %a, fp128 %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmps.f128(fp128 %a, fp128 %b, metadata !"uge", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmps_ueq_f128:
+; CHECK: bl __eqtf2
+define i32 @fcmps_ueq_f128(fp128 %a, fp128 %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmps.f128(fp128 %a, fp128 %b, metadata !"ueq", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmps_une_f128:
+; CHECK: bl __netf2
+define i32 @fcmps_une_f128(fp128 %a, fp128 %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmps.f128(fp128 %a, fp128 %b, metadata !"une", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+
+; Intrinsics to convert between floating-point types
+
+; CHECK-LABEL: fptrunc_f32_f64:
+; CHECK: fcvt s0, d0
+define float @fptrunc_f32_f64(double %x) #0 {
+  %val = call float @llvm.experimental.constrained.fptrunc.f32.f64(double %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret float %val
+}
+
+; CHECK-LABEL: fptrunc_f32_f128:
+; CHECK: bl __trunctfsf2
+define float @fptrunc_f32_f128(fp128 %x) #0 {
+  %val = call float @llvm.experimental.constrained.fptrunc.f32.f128(fp128 %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret float %val
+}
+
+; CHECK-LABEL: fptrunc_f64_f128:
+; CHECK: bl __trunctfdf2
+define double @fptrunc_f64_f128(fp128 %x) #0 {
+  %val = call double @llvm.experimental.constrained.fptrunc.f64.f128(fp128 %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret double %val
+}
+
+; CHECK-LABEL: fpext_f64_f32:
+; CHECK: fcvt d0, s0
+define double @fpext_f64_f32(float %x) #0 {
+  %val = call double @llvm.experimental.constrained.fpext.f64.f32(float %x, metadata !"fpexcept.strict") #0
+  ret double %val
+}
+
+; CHECK-LABEL: fpext_f128_f32:
+; CHECK: bl __extendsftf2
+define fp128 @fpext_f128_f32(float %x) #0 {
+  %val = call fp128 @llvm.experimental.constrained.fpext.f128.f32(float %x, metadata !"fpexcept.strict") #0
+  ret fp128 %val
+}
+
+; CHECK-LABEL: fpext_f128_f64:
+; CHECK: bl __extenddftf2
+define fp128 @fpext_f128_f64(double %x) #0 {
+  %val = call fp128 @llvm.experimental.constrained.fpext.f128.f64(double %x, metadata !"fpexcept.strict") #0
+  ret fp128 %val
+}
+
+
+attributes #0 = { strictfp }
+
+declare float @llvm.experimental.constrained.fadd.f32(float, float, metadata, metadata)
+declare float @llvm.experimental.constrained.fsub.f32(float, float, metadata, metadata)
+declare float @llvm.experimental.constrained.fmul.f32(float, float, metadata, metadata)
+declare float @llvm.experimental.constrained.fdiv.f32(float, float, metadata, metadata)
+declare float @llvm.experimental.constrained.frem.f32(float, float, metadata, metadata)
+declare float @llvm.experimental.constrained.fma.f32(float, float, float, metadata, metadata)
+declare i32 @llvm.experimental.constrained.fptosi.i32.f32(float, metadata)
+declare i32 @llvm.experimental.constrained.fptoui.i32.f32(float, metadata)
+declare i64 @llvm.experimental.constrained.fptosi.i64.f32(float, metadata)
+declare i64 @llvm.experimental.constrained.fptoui.i64.f32(float, metadata)
+declare float @llvm.experimental.constrained.sitofp.f32.i32(i32, metadata, metadata)
+declare float @llvm.experimental.constrained.uitofp.f32.i32(i32, metadata, metadata)
+declare float @llvm.experimental.constrained.sitofp.f32.i64(i64, metadata, metadata)
+declare float @llvm.experimental.constrained.uitofp.f32.i64(i64, metadata, metadata)
+declare float @llvm.experimental.constrained.sitofp.f32.i128(i128, metadata, metadata)
+declare float @llvm.experimental.constrained.uitofp.f32.i128(i128, metadata, metadata)
+declare float @llvm.experimental.constrained.sqrt.f32(float, metadata, metadata)
+declare float @llvm.experimental.constrained.powi.f32(float, i32, metadata, metadata)
+declare float @llvm.experimental.constrained.sin.f32(float, metadata, metadata)
+declare float @llvm.experimental.constrained.cos.f32(float, metadata, metadata)
+declare float @llvm.experimental.constrained.pow.f32(float, float, metadata, metadata)
+declare float @llvm.experimental.constrained.log.f32(float, metadata, metadata)
+declare float @llvm.experimental.constrained.log10.f32(float, metadata, metadata)
+declare float @llvm.experimental.constrained.log2.f32(float, metadata, metadata)
+declare float @llvm.experimental.constrained.exp.f32(float, metadata, metadata)
+declare float @llvm.experimental.constrained.exp2.f32(float, metadata, metadata)
+declare float @llvm.experimental.constrained.rint.f32(float, metadata, metadata)
+declare float @llvm.experimental.constrained.nearbyint.f32(float, metadata, metadata)
+declare i32 @llvm.experimental.constrained.lrint.f32(float, metadata, metadata)
+declare i64 @llvm.experimental.constrained.llrint.f32(float, metadata, metadata)
+declare float @llvm.experimental.constrained.maxnum.f32(float, float, metadata)
+declare float @llvm.experimental.constrained.minnum.f32(float, float, metadata)
+declare float @llvm.experimental.constrained.ceil.f32(float, metadata)
+declare float @llvm.experimental.constrained.floor.f32(float, metadata)
+declare i32 @llvm.experimental.constrained.lround.f32(float, metadata)
+declare i64 @llvm.experimental.constrained.llround.f32(float, metadata)
+declare float @llvm.experimental.constrained.round.f32(float, metadata)
+declare float @llvm.experimental.constrained.trunc.f32(float, metadata)
+declare i1 @llvm.experimental.constrained.fcmps.f32(float, float, metadata, metadata)
+declare i1 @llvm.experimental.constrained.fcmp.f32(float, float, metadata, metadata)
+
+declare double @llvm.experimental.constrained.fadd.f64(double, double, metadata, metadata)
+declare double @llvm.experimental.constrained.fsub.f64(double, double, metadata, metadata)
+declare double @llvm.experimental.constrained.fmul.f64(double, double, metadata, metadata)
+declare double @llvm.experimental.constrained.fdiv.f64(double, double, metadata, metadata)
+declare double @llvm.experimental.constrained.frem.f64(double, double, metadata, metadata)
+declare double @llvm.experimental.constrained.fma.f64(double, double, double, metadata, metadata)
+declare i32 @llvm.experimental.constrained.fptosi.i32.f64(double, metadata)
+declare i32 @llvm.experimental.constrained.fptoui.i32.f64(double, metadata)
+declare i64 @llvm.experimental.constrained.fptosi.i64.f64(double, metadata)
+declare i64 @llvm.experimental.constrained.fptoui.i64.f64(double, metadata)
+declare double @llvm.experimental.constrained.sitofp.f64.i32(i32, metadata, metadata)
+declare double @llvm.experimental.constrained.uitofp.f64.i32(i32, metadata, metadata)
+declare double @llvm.experimental.constrained.sitofp.f64.i64(i64, metadata, metadata)
+declare double @llvm.experimental.constrained.uitofp.f64.i64(i64, metadata, metadata)
+declare double @llvm.experimental.constrained.sitofp.f64.i128(i128, metadata, metadata)
+declare double @llvm.experimental.constrained.uitofp.f64.i128(i128, metadata, metadata)
+declare double @llvm.experimental.constrained.sqrt.f64(double, metadata, metadata)
+declare double @llvm.experimental.constrained.powi.f64(double, i32, metadata, metadata)
+declare double @llvm.experimental.constrained.sin.f64(double, metadata, metadata)
+declare double @llvm.experimental.constrained.cos.f64(double, metadata, metadata)
+declare double @llvm.experimental.constrained.pow.f64(double, double, metadata, metadata)
+declare double @llvm.experimental.constrained.log.f64(double, metadata, metadata)
+declare double @llvm.experimental.constrained.log10.f64(double, metadata, metadata)
+declare double @llvm.experimental.constrained.log2.f64(double, metadata, metadata)
+declare double @llvm.experimental.constrained.exp.f64(double, metadata, metadata)
+declare double @llvm.experimental.constrained.exp2.f64(double, metadata, metadata)
+declare double @llvm.experimental.constrained.rint.f64(double, metadata, metadata)
+declare double @llvm.experimental.constrained.nearbyint.f64(double, metadata, metadata)
+declare i32 @llvm.experimental.constrained.lrint.f64(double, metadata, metadata)
+declare i64 @llvm.experimental.constrained.llrint.f64(double, metadata, metadata)
+declare double @llvm.experimental.constrained.maxnum.f64(double, double, metadata)
+declare double @llvm.experimental.constrained.minnum.f64(double, double, metadata)
+declare double @llvm.experimental.constrained.ceil.f64(double, metadata)
+declare double @llvm.experimental.constrained.floor.f64(double, metadata)
+declare i32 @llvm.experimental.constrained.lround.f64(double, metadata)
+declare i64 @llvm.experimental.constrained.llround.f64(double, metadata)
+declare double @llvm.experimental.constrained.round.f64(double, metadata)
+declare double @llvm.experimental.constrained.trunc.f64(double, metadata)
+declare i1 @llvm.experimental.constrained.fcmps.f64(double, double, metadata, metadata)
+declare i1 @llvm.experimental.constrained.fcmp.f64(double, double, metadata, metadata)
+
+declare fp128 @llvm.experimental.constrained.fadd.f128(fp128, fp128, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.fsub.f128(fp128, fp128, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.fmul.f128(fp128, fp128, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.fdiv.f128(fp128, fp128, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.frem.f128(fp128, fp128, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.fma.f128(fp128, fp128, fp128, metadata, metadata)
+declare i32 @llvm.experimental.constrained.fptosi.i32.f128(fp128, metadata)
+declare i32 @llvm.experimental.constrained.fptoui.i32.f128(fp128, metadata)
+declare i64 @llvm.experimental.constrained.fptosi.i64.f128(fp128, metadata)
+declare i64 @llvm.experimental.constrained.fptoui.i64.f128(fp128, metadata)
+declare fp128 @llvm.experimental.constrained.sitofp.f128.i32(i32, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.uitofp.f128.i32(i32, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.sitofp.f128.i64(i64, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.uitofp.f128.i64(i64, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.sitofp.f128.i128(i128, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.uitofp.f128.i128(i128, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.sqrt.f128(fp128, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.powi.f128(fp128, i32, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.sin.f128(fp128, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.cos.f128(fp128, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.pow.f128(fp128, fp128, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.log.f128(fp128, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.log10.f128(fp128, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.log2.f128(fp128, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.exp.f128(fp128, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.exp2.f128(fp128, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.rint.f128(fp128, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.nearbyint.f128(fp128, metadata, metadata)
+declare i32 @llvm.experimental.constrained.lrint.f128(fp128, metadata, metadata)
+declare i64 @llvm.experimental.constrained.llrint.f128(fp128, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.maxnum.f128(fp128, fp128, metadata)
+declare fp128 @llvm.experimental.constrained.minnum.f128(fp128, fp128, metadata)
+declare fp128 @llvm.experimental.constrained.ceil.f128(fp128, metadata)
+declare fp128 @llvm.experimental.constrained.floor.f128(fp128, metadata)
+declare i32 @llvm.experimental.constrained.lround.f128(fp128, metadata)
+declare i64 @llvm.experimental.constrained.llround.f128(fp128, metadata)
+declare fp128 @llvm.experimental.constrained.round.f128(fp128, metadata)
+declare fp128 @llvm.experimental.constrained.trunc.f128(fp128, metadata)
+declare i1 @llvm.experimental.constrained.fcmps.f128(fp128, fp128, metadata, metadata)
+declare i1 @llvm.experimental.constrained.fcmp.f128(fp128, fp128, metadata, metadata)
+
+declare float @llvm.experimental.constrained.fptrunc.f32.f64(double, metadata, metadata)
+declare float @llvm.experimental.constrained.fptrunc.f32.f128(fp128, metadata, metadata)
+declare double @llvm.experimental.constrained.fptrunc.f64.f128(fp128, metadata, metadata)
+declare double @llvm.experimental.constrained.fpext.f64.f32(float, metadata)
+declare fp128 @llvm.experimental.constrained.fpext.f128.f32(float, metadata)
+declare fp128 @llvm.experimental.constrained.fpext.f128.f64(double, metadata)
diff --git a/llvm/test/CodeGen/AArch64/fp16-fmla.ll b/llvm/test/CodeGen/AArch64/fp16-fmla.ll
new file mode 100644
index 000000000..a81721afb
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/fp16-fmla.ll
@@ -0,0 +1,220 @@
+; RUN: llc < %s -mtriple=aarch64-none-linux-gnu -mattr=+v8.2a,+fullfp16 -fp-contract=fast  | FileCheck %s
+
+define half @test_FMULADDH_OP1(half %a, half %b, half %c) {
+; CHECK-LABEL: test_FMULADDH_OP1:
+; CHECK: fmadd    {{h[0-9]+}}, {{h[0-9]+}}, {{h[0-9]+}}
+entry:
+  %mul = fmul fast half %c, %b
+  %add = fadd fast half %mul, %a
+  ret half %add
+}
+
+define half @test_FMULADDH_OP2(half %a, half %b, half %c) {
+; CHECK-LABEL: test_FMULADDH_OP2:
+; CHECK: fmadd    {{h[0-9]+}}, {{h[0-9]+}}, {{h[0-9]+}}
+entry:
+  %mul = fmul fast half %c, %b
+  %add = fadd fast half %a, %mul
+  ret half %add
+}
+
+define half @test_FMULSUBH_OP1(half %a, half %b, half %c) {
+; CHECK-LABEL: test_FMULSUBH_OP1:
+; CHECK: fnmsub    {{h[0-9]+}}, {{h[0-9]+}}, {{h[0-9]+}}
+entry:
+  %mul = fmul fast half %c, %b
+  %sub = fsub fast half %mul, %a
+  ret half %sub
+}
+
+define half @test_FMULSUBH_OP2(half %a, half %b, half %c) {
+; CHECK-LABEL: test_FMULSUBH_OP2:
+; CHECK: fmsub    {{h[0-9]+}}, {{h[0-9]+}}, {{h[0-9]+}}
+entry:
+  %mul = fmul fast half %c, %b
+  %add = fsub fast half %a, %mul
+  ret half %add
+}
+
+define half @test_FNMULSUBH_OP1(half %a, half %b, half %c) {
+; CHECK-LABEL: test_FNMULSUBH_OP1:
+; CHECK: fnmadd    {{h[0-9]+}}, {{h[0-9]+}}, {{h[0-9]+}}
+entry:
+  %mul = fmul fast half %c, %b
+  %neg = fsub fast half -0.0, %mul
+  %add = fsub fast half %neg, %a
+  ret half %add
+}
+
+define <4 x half> @test_FMLAv4f16_OP1(<4 x half> %a, <4 x half> %b, <4 x half> %c) {
+; CHECK-LABEL: test_FMLAv4f16_OP1:
+; CHECK: fmla    {{v[0-9]+}}.4h, {{v[0-9]+}}.4h, {{v[0-9]+}}.4h
+entry:
+  %mul = fmul fast <4 x half> %c, %b
+  %add = fadd fast <4 x half> %mul, %a
+  ret <4 x half> %add
+}
+
+define <4 x half> @test_FMLAv4f16_OP2(<4 x half> %a, <4 x half> %b, <4 x half> %c) {
+; CHECK-LABEL: test_FMLAv4f16_OP2:
+; CHECK: fmla    {{v[0-9]+}}.4h, {{v[0-9]+}}.4h, {{v[0-9]+}}.4h
+entry:
+  %mul = fmul fast <4 x half> %c, %b
+  %add = fadd fast <4 x half> %a, %mul
+  ret <4 x half> %add
+}
+
+define <8 x half> @test_FMLAv8f16_OP1(<8 x half> %a, <8 x half> %b, <8 x half> %c) {
+; CHECK-LABEL: test_FMLAv8f16_OP1:
+; CHECK: fmla    {{v[0-9]+}}.8h, {{v[0-9]+}}.8h, {{v[0-9]+}}.8h
+entry:
+  %mul = fmul fast <8 x half> %c, %b
+  %add = fadd fast <8 x half> %mul, %a
+  ret <8 x half> %add
+}
+
+define <8 x half> @test_FMLAv8f16_OP2(<8 x half> %a, <8 x half> %b, <8 x half> %c) {
+; CHECK-LABEL: test_FMLAv8f16_OP2:
+; CHECK: fmla    {{v[0-9]+}}.8h, {{v[0-9]+}}.8h, {{v[0-9]+}}.8h
+entry:
+  %mul = fmul fast <8 x half> %c, %b
+  %add = fadd fast <8 x half> %a, %mul
+  ret <8 x half> %add
+}
+
+define <4 x half> @test_FMLAv4i16_indexed_OP1(<4 x half> %a, <4 x i16> %b, <4 x i16> %c) {
+; CHECK-LABEL: test_FMLAv4i16_indexed_OP1:
+; CHECK-FIXME: Currently LLVM produces inefficient code:
+; CHECK: mul
+; CHECK: fadd
+; CHECK-FIXME: It should instead produce the following instruction:
+; CHECK-FIXME: fmla    {{v[0-9]+}}.4h, {{v[0-9]+}}.4h, {{v[0-9]+}}.4h
+entry:
+  %mul = mul <4 x i16> %c, %b
+  %m = bitcast <4 x i16> %mul to <4 x half>
+  %add = fadd fast <4 x half> %m, %a
+  ret <4 x half> %add
+}
+
+define <4 x half> @test_FMLAv4i16_indexed_OP2(<4 x half> %a, <4 x i16> %b, <4 x i16> %c) {
+; CHECK-LABEL: test_FMLAv4i16_indexed_OP2:
+; CHECK-FIXME: Currently LLVM produces inefficient code:
+; CHECK: mul
+; CHECK: fadd
+; CHECK-FIXME: It should instead produce the following instruction:
+; CHECK-FIXME: fmla    {{v[0-9]+}}.4h, {{v[0-9]+}}.4h, {{v[0-9]+}}.4h
+entry:
+  %mul = mul <4 x i16> %c, %b
+  %m = bitcast <4 x i16> %mul to <4 x half>
+  %add = fadd fast <4 x half> %a, %m
+  ret <4 x half> %add
+}
+
+define <8 x half> @test_FMLAv8i16_indexed_OP1(<8 x half> %a, <8 x i16> %b, <8 x i16> %c) {
+; CHECK-LABEL: test_FMLAv8i16_indexed_OP1:
+; CHECK-FIXME: Currently LLVM produces inefficient code:
+; CHECK: mul
+; CHECK: fadd
+; CHECK-FIXME: It should instead produce the following instruction:
+; CHECK-FIXME: fmla    {{v[0-9]+}}.8h, {{v[0-9]+}}.8h, {{v[0-9]+}}.8h
+entry:
+  %mul = mul <8 x i16> %c, %b
+  %m = bitcast <8 x i16> %mul to <8 x half>
+  %add = fadd fast <8 x half> %m, %a
+  ret <8 x half> %add
+}
+
+define <8 x half> @test_FMLAv8i16_indexed_OP2(<8 x half> %a, <8 x i16> %b, <8 x i16> %c) {
+; CHECK-LABEL: test_FMLAv8i16_indexed_OP2:
+; CHECK-FIXME: Currently LLVM produces inefficient code:
+; CHECK: mul
+; CHECK: fadd
+; CHECK-FIXME: It should instead produce the following instruction:
+; CHECK-FIXME: fmla    {{v[0-9]+}}.8h, {{v[0-9]+}}.8h, {{v[0-9]+}}.8h
+entry:
+  %mul = mul <8 x i16> %c, %b
+  %m = bitcast <8 x i16> %mul to <8 x half>
+  %add = fadd fast <8 x half> %a, %m
+  ret <8 x half> %add
+}
+
+define <4 x half> @test_FMLSv4f16_OP1(<4 x half> %a, <4 x half> %b, <4 x half> %c) {
+; CHECK-LABEL: test_FMLSv4f16_OP1:
+; CHECK: fneg    {{v[0-9]+}}.4h, {{v[0-9]+}}.4h
+; CHECK: fmla    {{v[0-9]+}}.4h, {{v[0-9]+}}.4h, {{v[0-9]+}}.4h
+entry:
+  %mul = fmul fast <4 x half> %c, %b
+  %sub = fsub fast <4 x half> %mul, %a
+  ret <4 x half> %sub
+}
+
+define <4 x half> @test_FMLSv4f16_OP2(<4 x half> %a, <4 x half> %b, <4 x half> %c) {
+; CHECK-LABEL: test_FMLSv4f16_OP2:
+; CHECK: fmls    {{v[0-9]+}}.4h, {{v[0-9]+}}.4h, {{v[0-9]+}}.4h
+entry:
+  %mul = fmul fast <4 x half> %c, %b
+  %sub = fsub fast <4 x half> %a, %mul
+  ret <4 x half> %sub
+}
+
+define <8 x half> @test_FMLSv8f16_OP1(<8 x half> %a, <8 x half> %b, <8 x half> %c) {
+; CHECK-LABEL: test_FMLSv8f16_OP1:
+; CHECK: fneg    {{v[0-9]+}}.8h, {{v[0-9]+}}.8h
+; CHECK: fmla    {{v[0-9]+}}.8h, {{v[0-9]+}}.8h, {{v[0-9]+}}.8h
+entry:
+  %mul = fmul fast <8 x half> %c, %b
+  %sub = fsub fast <8 x half> %mul, %a
+  ret <8 x half> %sub
+}
+
+define <8 x half> @test_FMLSv8f16_OP2(<8 x half> %a, <8 x half> %b, <8 x half> %c) {
+; CHECK-LABEL: test_FMLSv8f16_OP2:
+; CHECK: fmls    {{v[0-9]+}}.8h, {{v[0-9]+}}.8h, {{v[0-9]+}}.8h
+entry:
+  %mul = fmul fast <8 x half> %c, %b
+  %sub = fsub fast <8 x half> %a, %mul
+  ret <8 x half> %sub
+}
+
+define <4 x half> @test_FMLSv4i16_indexed_OP2(<4 x half> %a, <4 x i16> %b, <4 x i16> %c) {
+; CHECK-LABEL: test_FMLSv4i16_indexed_OP2:
+; CHECK-FIXME: Currently LLVM produces inefficient code:
+; CHECK: mul
+; CHECK: fsub
+; CHECK-FIXME: It should instead produce the following instruction:
+; CHECK-FIXME: fmls    {{v[0-9]+}}.4h, {{v[0-9]+}}.4h, {{v[0-9]+}}.4h
+entry:
+  %mul = mul <4 x i16> %c, %b
+  %m = bitcast <4 x i16> %mul to <4 x half>
+  %sub = fsub fast <4 x half> %a, %m
+  ret <4 x half> %sub
+}
+
+define <8 x half> @test_FMLSv8i16_indexed_OP1(<8 x half> %a, <8 x i16> %b, <8 x i16> %c) {
+; CHECK-LABEL: test_FMLSv8i16_indexed_OP1:
+; CHECK-FIXME: Currently LLVM produces inefficient code:
+; CHECK: mul
+; CHECK: fsub
+; CHECK-FIXME: It should instead produce the following instruction:
+; CHECK-FIXME: fneg    {{v[0-9]+}}.8h, {{v[0-9]+}}.8h
+; CHECK-FIXME: fmla    {{v[0-9]+}}.8h, {{v[0-9]+}}.8h, {{v[0-9]+}}.8h
+entry:
+  %mul = mul <8 x i16> %c, %b
+  %m = bitcast <8 x i16> %mul to <8 x half>
+  %sub = fsub fast <8 x half> %m, %a
+  ret <8 x half> %sub
+}
+
+define <8 x half> @test_FMLSv8i16_indexed_OP2(<8 x half> %a, <8 x i16> %b, <8 x i16> %c) {
+; CHECK-LABEL: test_FMLSv8i16_indexed_OP2:
+; CHECK-FIXME: Currently LLVM produces inefficient code:
+; CHECK: mul
+; CHECK: fsub
+; CHECK-FIXME: It should instead produce the following instruction:
+; CHECK-FIXME: fmls    {{v[0-9]+}}.8h, {{v[0-9]+}}.8h, {{v[0-9]+}}.8h
+entry:
+  %mul = mul <8 x i16> %c, %b
+  %m = bitcast <8 x i16> %mul to <8 x half>
+  %sub = fsub fast <8 x half> %a, %m
+  ret <8 x half> %sub
+}
diff --git a/llvm/test/CodeGen/AArch64/fp16_intrinsic_lane.ll b/llvm/test/CodeGen/AArch64/fp16_intrinsic_lane.ll
index 9e276cdfb..856b00a84 100644
--- a/llvm/test/CodeGen/AArch64/fp16_intrinsic_lane.ll
+++ b/llvm/test/CodeGen/AArch64/fp16_intrinsic_lane.ll
@@ -175,8 +175,7 @@ entry:
 
 define dso_local half @t_vfmsh_lane_f16(half %a, half %b, <4 x half> %c, i32 %lane) {
 ; CHECK-LABEL: t_vfmsh_lane_f16:
-; CHECK:         fneg h1, h1
-; CHECK:         fmadd h0, h1, h2, h0
+; CHECK:         fmsub h0, h1, h2, h0
 ; CHECK-NEXT:    ret
 entry:
   %0 = fsub half 0xH8000, %b
@@ -187,9 +186,8 @@ entry:
 
 define dso_local half @t_vfmsh_laneq_f16(half %a, half %b, <8 x half> %c, i32 %lane) {
 ; CHECK-LABEL: t_vfmsh_laneq_f16:
-; CHECK:         fneg h1, h1
-; CHECK-NEXT:    fmadd h0, h1, h2, h0
-; CHECK-NEXT:    ret
+; CHECK:       fmsub h0, h1, h2, h0
+; CHECK-NEXT:  ret
 entry:
   %0 = fsub half 0xH8000, %b
   %extract = extractelement <8 x half> %c, i32 0
diff --git a/llvm/test/CodeGen/AArch64/fp16_intrinsic_scalar_3op.ll b/llvm/test/CodeGen/AArch64/fp16_intrinsic_scalar_3op.ll
index d43834d05..f3ad71b9c 100644
--- a/llvm/test/CodeGen/AArch64/fp16_intrinsic_scalar_3op.ll
+++ b/llvm/test/CodeGen/AArch64/fp16_intrinsic_scalar_3op.ll
@@ -1,6 +1,4 @@
-; RUN: llc < %s -mtriple=aarch64-eabi -mattr=+v8.2a,+fullfp16  | FileCheck %s
-
-declare half @llvm.fma.f16(half, half, half)
+; RUN: llc < %s -mtriple=aarch64-eabi -mattr=+v8.2a,+neon,+fullfp16  | FileCheck %s
 
 define dso_local half @t_vfmah_f16(half %a, half %b, half %c) {
 ; CHECK-LABEL: t_vfmah_f16:
@@ -11,3 +9,84 @@ entry:
   ret half %0
 }
 
+define half @fnma16(half %a, half %b, half %c) nounwind readnone ssp {
+; CHECK-LABEL: fnma16:
+; CHECK: fnmadd h0, h0, h1, h2
+entry:
+  %0 = tail call half @llvm.fma.f16(half %a, half %b, half %c)
+  %mul = fmul half %0, -1.000000e+00
+  ret half %mul
+}
+
+define half @fms16(half %a, half %b, half %c) nounwind readnone ssp {
+; CHECK-LABEL: fms16:
+; CHECK: fmsub h0, h0, h1, h2
+entry:
+  %mul = fmul half %b, -1.000000e+00
+  %0 = tail call half @llvm.fma.f16(half %a, half %mul, half %c)
+  ret half %0
+}
+
+define half @fms16_com(half %a, half %b, half %c) nounwind readnone ssp {
+; CHECK-LABEL: fms16_com:
+; CHECK:       fmsub h0, h1, h0, h2
+; CHECK-NEXT:  ret
+entry:
+  %mul = fmul half %b, -1.000000e+00
+  %0 = tail call half @llvm.fma.f16(half %mul, half %a, half %c)
+  ret half %0
+}
+
+define half @fnms16(half %a, half %b, half %c) nounwind readnone ssp {
+; CHECK-LABEL: fnms16:
+; CHECK:       fnmsub h0, h0, h1, h2
+; CHECK-NEXT:  ret
+entry:
+  %mul = fmul half %c, -1.000000e+00
+  %0 = tail call half @llvm.fma.f16(half %a, half %b, half %mul)
+  ret half %0
+}
+
+define half @test_fmsub(half %a, half %b, half %c) {
+; CHECK-LABEL: test_fmsub:
+; CHECK:       fmsub h0, h0, h1, h2
+; CHECK-NEXT:  ret
+entry:
+  %nega = fsub half -0.0, %a
+  %val = call half @llvm.fma.f16(half %nega, half %b, half %c)
+  ret half %val
+}
+
+define half @test_fnmadd(half %a, half %b, half %c) {
+; CHECK-LABEL: test_fnmadd:
+; CHECK:       fnmadd h0, h0, h1, h2
+; CHECK-NEXT:  ret
+entry:
+  %nega = fsub half -0.0, %a
+  %negc = fsub half -0.0, %c
+  %val = call half @llvm.fma.f16(half %nega, half %b, half %negc)
+  ret half %val
+}
+
+define half @test_fmadd(half %a, half %b, half %c) {
+; CHECK-LABEL: test_fmadd:
+; CHECK:       fmadd h0, h0, h1, h2
+; CHECK-NEXT:  ret
+entry:
+  %nega = fsub half -0.0, %a
+  %negb = fsub half -0.0, %b
+  %val = call half @llvm.fma.f16(half %nega, half %negb, half %c)
+  ret half %val
+}
+
+define half @test_fnmsub(half %a, half %b, half %c) {
+; CHECK-LABEL: test_fnmsub:
+; CHECK:       fnmsub h0, h0, h1, h2
+; CHECK-NEXT:  ret
+entry:
+  %negc = fsub half -0.0, %c
+  %val = call half @llvm.fma.f16(half %a, half %b, half %negc)
+  ret half %val
+}
+
+declare half @llvm.fma.f16(half, half, half)
diff --git a/llvm/test/CodeGen/AArch64/fpconv-vector-op-scalarize-strict.ll b/llvm/test/CodeGen/AArch64/fpconv-vector-op-scalarize-strict.ll
new file mode 100644
index 000000000..eba7fa88d
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/fpconv-vector-op-scalarize-strict.ll
@@ -0,0 +1,33 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=aarch64-apple-darwin | FileCheck %s
+
+; Check that the legalizer doesn't crash when scalarizing FP conversion
+; instructions' operands.  The operands are all illegal on AArch64,
+; ensuring they are legalized.  The results are all legal.
+
+define <1 x double> @test_sitofp(<1 x i1> %in) #0 {
+; CHECK-LABEL: test_sitofp:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    sbfx w8, w0, #0, #1
+; CHECK-NEXT:    scvtf d0, w8
+; CHECK-NEXT:    ret
+entry:
+  %0 = call <1 x double> @llvm.experimental.constrained.sitofp.v1f64.v1i1(<1 x i1> %in, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret <1 x double> %0
+}
+
+define <1 x double> @test_uitofp(<1 x i1> %in) #0 {
+; CHECK-LABEL: test_uitofp:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    and w8, w0, #0x1
+; CHECK-NEXT:    ucvtf d0, w8
+; CHECK-NEXT:    ret
+entry:
+  %0 = call <1 x double> @llvm.experimental.constrained.uitofp.v1f64.v1i1(<1 x i1> %in, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret <1 x double> %0
+}
+
+attributes #0 = { strictfp }
+
+declare <1 x double> @llvm.experimental.constrained.sitofp.v1f64.v1i1(<1 x i1>, metadata, metadata)
+declare <1 x double> @llvm.experimental.constrained.uitofp.v1f64.v1i1(<1 x i1>, metadata, metadata)
diff --git a/llvm/test/CodeGen/AArch64/global-merge-group-by-use.ll b/llvm/test/CodeGen/AArch64/global-merge-group-by-use.ll
index 99866c84b..cea9d2ac0 100644
--- a/llvm/test/CodeGen/AArch64/global-merge-group-by-use.ll
+++ b/llvm/test/CodeGen/AArch64/global-merge-group-by-use.ll
@@ -65,8 +65,8 @@ define void @f3(i32 %a1, i32 %a2) #0 {
 define void @f4(i32 %a1, i32 %a2, i32 %a3) #0 {
 ; CHECK-NEXT: adrp x8, [[SET3]]@PAGE
 ; CHECK-NEXT: add x8, x8, [[SET3]]@PAGEOFF
-; CHECK-NEXT: stp w2, w0, [x8]
-; CHECK-NEXT: str w1, [x8, #8]
+; CHECK-NEXT: stp w0, w1, [x8, #4]
+; CHECK-NEXT: str w2, [x8]
 ; CHECK-NEXT: ret
   store i32 %a1, i32* @m4, align 4
   store i32 %a2, i32* @n4, align 4
diff --git a/llvm/test/CodeGen/AArch64/global-merge-hidden-minsize.ll b/llvm/test/CodeGen/AArch64/global-merge-hidden-minsize.ll
new file mode 100644
index 000000000..aed6a40dc
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/global-merge-hidden-minsize.ll
@@ -0,0 +1,25 @@
+; RUN: llc %s -mtriple=aarch64-none-linux-gnu -o - | FileCheck %s
+
+@x = hidden global i32 0, align 4
+@y = hidden global i32 0, align 4
+
+define hidden void @f() #0 {
+  store i32 0, i32* @x, align 4
+  store i32 0, i32* @y, align 4
+  ret void
+}
+
+attributes #0 = { minsize optsize }
+
+; CHECK: .local .L_MergedGlobals
+; CHECK: .comm .L_MergedGlobals,8,4
+
+; CHECK: .globl x
+; CHECK: .hidden x
+; CHECK: .set x, .L_MergedGlobals
+; CHECK: .size x, 4
+
+; CHECK: .globl y
+; CHECK: .hidden y
+; CHECK: .set y, .L_MergedGlobals+4
+; CHECK: .size y, 4
diff --git a/llvm/test/CodeGen/AArch64/hoist-and-by-const-from-lshr-in-eqcmp-zero.ll b/llvm/test/CodeGen/AArch64/hoist-and-by-const-from-lshr-in-eqcmp-zero.ll
index 433904a6f..11f552437 100644
--- a/llvm/test/CodeGen/AArch64/hoist-and-by-const-from-lshr-in-eqcmp-zero.ll
+++ b/llvm/test/CodeGen/AArch64/hoist-and-by-const-from-lshr-in-eqcmp-zero.ll
@@ -15,11 +15,9 @@
 define i1 @scalar_i8_signbit_eq(i8 %x, i8 %y) nounwind {
 ; CHECK-LABEL: scalar_i8_signbit_eq:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #128
 ; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-NEXT:    lsr w8, w8, w1
-; CHECK-NEXT:    and w8, w8, w0
-; CHECK-NEXT:    tst w8, #0xff
+; CHECK-NEXT:    lsl w8, w0, w1
+; CHECK-NEXT:    tst w8, #0x80
 ; CHECK-NEXT:    cset w0, eq
 ; CHECK-NEXT:    ret
   %t0 = lshr i8 128, %y
@@ -31,11 +29,9 @@ define i1 @scalar_i8_signbit_eq(i8 %x, i8 %y) nounwind {
 define i1 @scalar_i8_lowestbit_eq(i8 %x, i8 %y) nounwind {
 ; CHECK-LABEL: scalar_i8_lowestbit_eq:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #1
 ; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-NEXT:    lsr w8, w8, w1
-; CHECK-NEXT:    and w8, w8, w0
-; CHECK-NEXT:    tst w8, #0xff
+; CHECK-NEXT:    lsl w8, w0, w1
+; CHECK-NEXT:    tst w8, #0x1
 ; CHECK-NEXT:    cset w0, eq
 ; CHECK-NEXT:    ret
   %t0 = lshr i8 1, %y
@@ -47,11 +43,9 @@ define i1 @scalar_i8_lowestbit_eq(i8 %x, i8 %y) nounwind {
 define i1 @scalar_i8_bitsinmiddle_eq(i8 %x, i8 %y) nounwind {
 ; CHECK-LABEL: scalar_i8_bitsinmiddle_eq:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #24
 ; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-NEXT:    lsr w8, w8, w1
-; CHECK-NEXT:    and w8, w8, w0
-; CHECK-NEXT:    tst w8, #0xff
+; CHECK-NEXT:    lsl w8, w0, w1
+; CHECK-NEXT:    tst w8, #0x18
 ; CHECK-NEXT:    cset w0, eq
 ; CHECK-NEXT:    ret
   %t0 = lshr i8 24, %y
@@ -65,11 +59,9 @@ define i1 @scalar_i8_bitsinmiddle_eq(i8 %x, i8 %y) nounwind {
 define i1 @scalar_i16_signbit_eq(i16 %x, i16 %y) nounwind {
 ; CHECK-LABEL: scalar_i16_signbit_eq:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #32768
 ; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-NEXT:    lsr w8, w8, w1
-; CHECK-NEXT:    and w8, w8, w0
-; CHECK-NEXT:    tst w8, #0xffff
+; CHECK-NEXT:    lsl w8, w0, w1
+; CHECK-NEXT:    tst w8, #0x8000
 ; CHECK-NEXT:    cset w0, eq
 ; CHECK-NEXT:    ret
   %t0 = lshr i16 32768, %y
@@ -81,11 +73,9 @@ define i1 @scalar_i16_signbit_eq(i16 %x, i16 %y) nounwind {
 define i1 @scalar_i16_lowestbit_eq(i16 %x, i16 %y) nounwind {
 ; CHECK-LABEL: scalar_i16_lowestbit_eq:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #1
 ; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-NEXT:    lsr w8, w8, w1
-; CHECK-NEXT:    and w8, w8, w0
-; CHECK-NEXT:    tst w8, #0xffff
+; CHECK-NEXT:    lsl w8, w0, w1
+; CHECK-NEXT:    tst w8, #0x1
 ; CHECK-NEXT:    cset w0, eq
 ; CHECK-NEXT:    ret
   %t0 = lshr i16 1, %y
@@ -97,11 +87,9 @@ define i1 @scalar_i16_lowestbit_eq(i16 %x, i16 %y) nounwind {
 define i1 @scalar_i16_bitsinmiddle_eq(i16 %x, i16 %y) nounwind {
 ; CHECK-LABEL: scalar_i16_bitsinmiddle_eq:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #4080
 ; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-NEXT:    lsr w8, w8, w1
-; CHECK-NEXT:    and w8, w8, w0
-; CHECK-NEXT:    tst w8, #0xffff
+; CHECK-NEXT:    lsl w8, w0, w1
+; CHECK-NEXT:    tst w8, #0xff0
 ; CHECK-NEXT:    cset w0, eq
 ; CHECK-NEXT:    ret
   %t0 = lshr i16 4080, %y
@@ -115,9 +103,8 @@ define i1 @scalar_i16_bitsinmiddle_eq(i16 %x, i16 %y) nounwind {
 define i1 @scalar_i32_signbit_eq(i32 %x, i32 %y) nounwind {
 ; CHECK-LABEL: scalar_i32_signbit_eq:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #-2147483648
-; CHECK-NEXT:    lsr w8, w8, w1
-; CHECK-NEXT:    tst w8, w0
+; CHECK-NEXT:    lsl w8, w0, w1
+; CHECK-NEXT:    tst w8, #0x80000000
 ; CHECK-NEXT:    cset w0, eq
 ; CHECK-NEXT:    ret
   %t0 = lshr i32 2147483648, %y
@@ -129,9 +116,8 @@ define i1 @scalar_i32_signbit_eq(i32 %x, i32 %y) nounwind {
 define i1 @scalar_i32_lowestbit_eq(i32 %x, i32 %y) nounwind {
 ; CHECK-LABEL: scalar_i32_lowestbit_eq:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #1
-; CHECK-NEXT:    lsr w8, w8, w1
-; CHECK-NEXT:    tst w8, w0
+; CHECK-NEXT:    lsl w8, w0, w1
+; CHECK-NEXT:    tst w8, #0x1
 ; CHECK-NEXT:    cset w0, eq
 ; CHECK-NEXT:    ret
   %t0 = lshr i32 1, %y
@@ -143,9 +129,8 @@ define i1 @scalar_i32_lowestbit_eq(i32 %x, i32 %y) nounwind {
 define i1 @scalar_i32_bitsinmiddle_eq(i32 %x, i32 %y) nounwind {
 ; CHECK-LABEL: scalar_i32_bitsinmiddle_eq:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #16776960
-; CHECK-NEXT:    lsr w8, w8, w1
-; CHECK-NEXT:    tst w8, w0
+; CHECK-NEXT:    lsl w8, w0, w1
+; CHECK-NEXT:    tst w8, #0xffff00
 ; CHECK-NEXT:    cset w0, eq
 ; CHECK-NEXT:    ret
   %t0 = lshr i32 16776960, %y
@@ -159,9 +144,8 @@ define i1 @scalar_i32_bitsinmiddle_eq(i32 %x, i32 %y) nounwind {
 define i1 @scalar_i64_signbit_eq(i64 %x, i64 %y) nounwind {
 ; CHECK-LABEL: scalar_i64_signbit_eq:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov x8, #-9223372036854775808
-; CHECK-NEXT:    lsr x8, x8, x1
-; CHECK-NEXT:    tst x8, x0
+; CHECK-NEXT:    lsl x8, x0, x1
+; CHECK-NEXT:    tst x8, #0x8000000000000000
 ; CHECK-NEXT:    cset w0, eq
 ; CHECK-NEXT:    ret
   %t0 = lshr i64 9223372036854775808, %y
@@ -173,9 +157,8 @@ define i1 @scalar_i64_signbit_eq(i64 %x, i64 %y) nounwind {
 define i1 @scalar_i64_lowestbit_eq(i64 %x, i64 %y) nounwind {
 ; CHECK-LABEL: scalar_i64_lowestbit_eq:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #1
-; CHECK-NEXT:    lsr x8, x8, x1
-; CHECK-NEXT:    tst x8, x0
+; CHECK-NEXT:    lsl x8, x0, x1
+; CHECK-NEXT:    tst x8, #0x1
 ; CHECK-NEXT:    cset w0, eq
 ; CHECK-NEXT:    ret
   %t0 = lshr i64 1, %y
@@ -187,9 +170,8 @@ define i1 @scalar_i64_lowestbit_eq(i64 %x, i64 %y) nounwind {
 define i1 @scalar_i64_bitsinmiddle_eq(i64 %x, i64 %y) nounwind {
 ; CHECK-LABEL: scalar_i64_bitsinmiddle_eq:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov x8, #281474976645120
-; CHECK-NEXT:    lsr x8, x8, x1
-; CHECK-NEXT:    tst x8, x0
+; CHECK-NEXT:    lsl x8, x0, x1
+; CHECK-NEXT:    tst x8, #0xffffffff0000
 ; CHECK-NEXT:    cset w0, eq
 ; CHECK-NEXT:    ret
   %t0 = lshr i64 281474976645120, %y
@@ -205,10 +187,9 @@ define i1 @scalar_i64_bitsinmiddle_eq(i64 %x, i64 %y) nounwind {
 define <4 x i1> @vec_4xi32_splat_eq(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; CHECK-LABEL: vec_4xi32_splat_eq:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    neg v1.4s, v1.4s
 ; CHECK-NEXT:    movi v2.4s, #1
-; CHECK-NEXT:    ushl v1.4s, v2.4s, v1.4s
-; CHECK-NEXT:    and v0.16b, v1.16b, v0.16b
+; CHECK-NEXT:    ushl v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    and v0.16b, v0.16b, v2.16b
 ; CHECK-NEXT:    cmeq v0.4s, v0.4s, #0
 ; CHECK-NEXT:    xtn v0.4h, v0.4s
 ; CHECK-NEXT:    ret
@@ -238,10 +219,9 @@ define <4 x i1> @vec_4xi32_nonsplat_eq(<4 x i32> %x, <4 x i32> %y) nounwind {
 define <4 x i1> @vec_4xi32_nonsplat_undef0_eq(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; CHECK-LABEL: vec_4xi32_nonsplat_undef0_eq:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    neg v1.4s, v1.4s
 ; CHECK-NEXT:    movi v2.4s, #1
-; CHECK-NEXT:    ushl v1.4s, v2.4s, v1.4s
-; CHECK-NEXT:    and v0.16b, v1.16b, v0.16b
+; CHECK-NEXT:    ushl v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    and v0.16b, v0.16b, v2.16b
 ; CHECK-NEXT:    cmeq v0.4s, v0.4s, #0
 ; CHECK-NEXT:    xtn v0.4h, v0.4s
 ; CHECK-NEXT:    ret
@@ -288,12 +268,9 @@ define <4 x i1> @vec_4xi32_nonsplat_undef2_eq(<4 x i32> %x, <4 x i32> %y) nounwi
 define i1 @scalar_i8_signbit_ne(i8 %x, i8 %y) nounwind {
 ; CHECK-LABEL: scalar_i8_signbit_ne:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #128
 ; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-NEXT:    lsr w8, w8, w1
-; CHECK-NEXT:    and w8, w8, w0
-; CHECK-NEXT:    tst w8, #0xff
-; CHECK-NEXT:    cset w0, ne
+; CHECK-NEXT:    lsl w8, w0, w1
+; CHECK-NEXT:    ubfx w0, w8, #7, #1
 ; CHECK-NEXT:    ret
   %t0 = lshr i8 128, %y
   %t1 = and i8 %t0, %x
@@ -323,10 +300,8 @@ define i1 @scalar_i32_x_is_const2_eq(i32 %y) nounwind {
 ; CHECK-LABEL: scalar_i32_x_is_const2_eq:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov w8, #1
-; CHECK-NEXT:    mov w9, #43605
 ; CHECK-NEXT:    lsr w8, w8, w0
-; CHECK-NEXT:    movk w9, #43605, lsl #16
-; CHECK-NEXT:    tst w8, w9
+; CHECK-NEXT:    cmp w8, #0 // =0
 ; CHECK-NEXT:    cset w0, eq
 ; CHECK-NEXT:    ret
   %t0 = lshr i32 1, %y
@@ -345,9 +320,7 @@ define i1 @negative_scalar_i8_bitsinmiddle_slt(i8 %x, i8 %y) nounwind {
 ; CHECK-NEXT:    mov w8, #24
 ; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
 ; CHECK-NEXT:    lsr w8, w8, w1
-; CHECK-NEXT:    and w8, w8, w0
-; CHECK-NEXT:    sxtb w8, w8
-; CHECK-NEXT:    cmp w8, #0 // =0
+; CHECK-NEXT:    tst w8, w0
 ; CHECK-NEXT:    cset w0, lt
 ; CHECK-NEXT:    ret
   %t0 = lshr i8 24, %y
@@ -363,7 +336,6 @@ define i1 @scalar_i8_signbit_eq_with_nonzero(i8 %x, i8 %y) nounwind {
 ; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
 ; CHECK-NEXT:    lsr w8, w8, w1
 ; CHECK-NEXT:    and w8, w8, w0
-; CHECK-NEXT:    and w8, w8, #0xff
 ; CHECK-NEXT:    cmp w8, #1 // =1
 ; CHECK-NEXT:    cset w0, eq
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/AArch64/hoist-and-by-const-from-shl-in-eqcmp-zero.ll b/llvm/test/CodeGen/AArch64/hoist-and-by-const-from-shl-in-eqcmp-zero.ll
index 856f29467..2a5bfeb30 100644
--- a/llvm/test/CodeGen/AArch64/hoist-and-by-const-from-shl-in-eqcmp-zero.ll
+++ b/llvm/test/CodeGen/AArch64/hoist-and-by-const-from-shl-in-eqcmp-zero.ll
@@ -15,11 +15,10 @@
 define i1 @scalar_i8_signbit_eq(i8 %x, i8 %y) nounwind {
 ; CHECK-LABEL: scalar_i8_signbit_eq:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #-128
+; CHECK-NEXT:    and w8, w0, #0xff
 ; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-NEXT:    lsl w8, w8, w1
-; CHECK-NEXT:    and w8, w8, w0
-; CHECK-NEXT:    tst w8, #0xff
+; CHECK-NEXT:    lsr w8, w8, w1
+; CHECK-NEXT:    tst w8, #0x80
 ; CHECK-NEXT:    cset w0, eq
 ; CHECK-NEXT:    ret
   %t0 = shl i8 128, %y
@@ -31,11 +30,10 @@ define i1 @scalar_i8_signbit_eq(i8 %x, i8 %y) nounwind {
 define i1 @scalar_i8_lowestbit_eq(i8 %x, i8 %y) nounwind {
 ; CHECK-LABEL: scalar_i8_lowestbit_eq:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #1
+; CHECK-NEXT:    and w8, w0, #0xff
 ; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-NEXT:    lsl w8, w8, w1
-; CHECK-NEXT:    and w8, w8, w0
-; CHECK-NEXT:    tst w8, #0xff
+; CHECK-NEXT:    lsr w8, w8, w1
+; CHECK-NEXT:    tst w8, #0x1
 ; CHECK-NEXT:    cset w0, eq
 ; CHECK-NEXT:    ret
   %t0 = shl i8 1, %y
@@ -47,11 +45,10 @@ define i1 @scalar_i8_lowestbit_eq(i8 %x, i8 %y) nounwind {
 define i1 @scalar_i8_bitsinmiddle_eq(i8 %x, i8 %y) nounwind {
 ; CHECK-LABEL: scalar_i8_bitsinmiddle_eq:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #24
+; CHECK-NEXT:    and w8, w0, #0xff
 ; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-NEXT:    lsl w8, w8, w1
-; CHECK-NEXT:    and w8, w8, w0
-; CHECK-NEXT:    tst w8, #0xff
+; CHECK-NEXT:    lsr w8, w8, w1
+; CHECK-NEXT:    tst w8, #0x18
 ; CHECK-NEXT:    cset w0, eq
 ; CHECK-NEXT:    ret
   %t0 = shl i8 24, %y
@@ -65,11 +62,10 @@ define i1 @scalar_i8_bitsinmiddle_eq(i8 %x, i8 %y) nounwind {
 define i1 @scalar_i16_signbit_eq(i16 %x, i16 %y) nounwind {
 ; CHECK-LABEL: scalar_i16_signbit_eq:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #-32768
+; CHECK-NEXT:    and w8, w0, #0xffff
 ; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-NEXT:    lsl w8, w8, w1
-; CHECK-NEXT:    and w8, w8, w0
-; CHECK-NEXT:    tst w8, #0xffff
+; CHECK-NEXT:    lsr w8, w8, w1
+; CHECK-NEXT:    tst w8, #0x8000
 ; CHECK-NEXT:    cset w0, eq
 ; CHECK-NEXT:    ret
   %t0 = shl i16 32768, %y
@@ -81,11 +77,10 @@ define i1 @scalar_i16_signbit_eq(i16 %x, i16 %y) nounwind {
 define i1 @scalar_i16_lowestbit_eq(i16 %x, i16 %y) nounwind {
 ; CHECK-LABEL: scalar_i16_lowestbit_eq:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #1
+; CHECK-NEXT:    and w8, w0, #0xffff
 ; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-NEXT:    lsl w8, w8, w1
-; CHECK-NEXT:    and w8, w8, w0
-; CHECK-NEXT:    tst w8, #0xffff
+; CHECK-NEXT:    lsr w8, w8, w1
+; CHECK-NEXT:    tst w8, #0x1
 ; CHECK-NEXT:    cset w0, eq
 ; CHECK-NEXT:    ret
   %t0 = shl i16 1, %y
@@ -97,11 +92,10 @@ define i1 @scalar_i16_lowestbit_eq(i16 %x, i16 %y) nounwind {
 define i1 @scalar_i16_bitsinmiddle_eq(i16 %x, i16 %y) nounwind {
 ; CHECK-LABEL: scalar_i16_bitsinmiddle_eq:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #4080
+; CHECK-NEXT:    and w8, w0, #0xffff
 ; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-NEXT:    lsl w8, w8, w1
-; CHECK-NEXT:    and w8, w8, w0
-; CHECK-NEXT:    tst w8, #0xffff
+; CHECK-NEXT:    lsr w8, w8, w1
+; CHECK-NEXT:    tst w8, #0xff0
 ; CHECK-NEXT:    cset w0, eq
 ; CHECK-NEXT:    ret
   %t0 = shl i16 4080, %y
@@ -115,9 +109,8 @@ define i1 @scalar_i16_bitsinmiddle_eq(i16 %x, i16 %y) nounwind {
 define i1 @scalar_i32_signbit_eq(i32 %x, i32 %y) nounwind {
 ; CHECK-LABEL: scalar_i32_signbit_eq:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #-2147483648
-; CHECK-NEXT:    lsl w8, w8, w1
-; CHECK-NEXT:    tst w8, w0
+; CHECK-NEXT:    lsr w8, w0, w1
+; CHECK-NEXT:    tst w8, #0x80000000
 ; CHECK-NEXT:    cset w0, eq
 ; CHECK-NEXT:    ret
   %t0 = shl i32 2147483648, %y
@@ -129,9 +122,8 @@ define i1 @scalar_i32_signbit_eq(i32 %x, i32 %y) nounwind {
 define i1 @scalar_i32_lowestbit_eq(i32 %x, i32 %y) nounwind {
 ; CHECK-LABEL: scalar_i32_lowestbit_eq:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #1
-; CHECK-NEXT:    lsl w8, w8, w1
-; CHECK-NEXT:    tst w8, w0
+; CHECK-NEXT:    lsr w8, w0, w1
+; CHECK-NEXT:    tst w8, #0x1
 ; CHECK-NEXT:    cset w0, eq
 ; CHECK-NEXT:    ret
   %t0 = shl i32 1, %y
@@ -143,9 +135,8 @@ define i1 @scalar_i32_lowestbit_eq(i32 %x, i32 %y) nounwind {
 define i1 @scalar_i32_bitsinmiddle_eq(i32 %x, i32 %y) nounwind {
 ; CHECK-LABEL: scalar_i32_bitsinmiddle_eq:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #16776960
-; CHECK-NEXT:    lsl w8, w8, w1
-; CHECK-NEXT:    tst w8, w0
+; CHECK-NEXT:    lsr w8, w0, w1
+; CHECK-NEXT:    tst w8, #0xffff00
 ; CHECK-NEXT:    cset w0, eq
 ; CHECK-NEXT:    ret
   %t0 = shl i32 16776960, %y
@@ -159,9 +150,8 @@ define i1 @scalar_i32_bitsinmiddle_eq(i32 %x, i32 %y) nounwind {
 define i1 @scalar_i64_signbit_eq(i64 %x, i64 %y) nounwind {
 ; CHECK-LABEL: scalar_i64_signbit_eq:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov x8, #-9223372036854775808
-; CHECK-NEXT:    lsl x8, x8, x1
-; CHECK-NEXT:    tst x8, x0
+; CHECK-NEXT:    lsr x8, x0, x1
+; CHECK-NEXT:    tst x8, #0x8000000000000000
 ; CHECK-NEXT:    cset w0, eq
 ; CHECK-NEXT:    ret
   %t0 = shl i64 9223372036854775808, %y
@@ -173,9 +163,8 @@ define i1 @scalar_i64_signbit_eq(i64 %x, i64 %y) nounwind {
 define i1 @scalar_i64_lowestbit_eq(i64 %x, i64 %y) nounwind {
 ; CHECK-LABEL: scalar_i64_lowestbit_eq:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #1
-; CHECK-NEXT:    lsl x8, x8, x1
-; CHECK-NEXT:    tst x8, x0
+; CHECK-NEXT:    lsr x8, x0, x1
+; CHECK-NEXT:    tst x8, #0x1
 ; CHECK-NEXT:    cset w0, eq
 ; CHECK-NEXT:    ret
   %t0 = shl i64 1, %y
@@ -187,9 +176,8 @@ define i1 @scalar_i64_lowestbit_eq(i64 %x, i64 %y) nounwind {
 define i1 @scalar_i64_bitsinmiddle_eq(i64 %x, i64 %y) nounwind {
 ; CHECK-LABEL: scalar_i64_bitsinmiddle_eq:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov x8, #281474976645120
-; CHECK-NEXT:    lsl x8, x8, x1
-; CHECK-NEXT:    tst x8, x0
+; CHECK-NEXT:    lsr x8, x0, x1
+; CHECK-NEXT:    tst x8, #0xffffffff0000
 ; CHECK-NEXT:    cset w0, eq
 ; CHECK-NEXT:    ret
   %t0 = shl i64 281474976645120, %y
@@ -283,12 +271,10 @@ define <4 x i1> @vec_4xi32_nonsplat_undef2_eq(<4 x i32> %x, <4 x i32> %y) nounwi
 define i1 @scalar_i8_signbit_ne(i8 %x, i8 %y) nounwind {
 ; CHECK-LABEL: scalar_i8_signbit_ne:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #-128
+; CHECK-NEXT:    and w8, w0, #0xff
 ; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-NEXT:    lsl w8, w8, w1
-; CHECK-NEXT:    and w8, w8, w0
-; CHECK-NEXT:    tst w8, #0xff
-; CHECK-NEXT:    cset w0, ne
+; CHECK-NEXT:    lsr w8, w8, w1
+; CHECK-NEXT:    lsr w0, w8, #7
 ; CHECK-NEXT:    ret
   %t0 = shl i8 128, %y
   %t1 = and i8 %t0, %x
@@ -358,7 +344,7 @@ define i1 @scalar_i8_signbit_eq_with_nonzero(i8 %x, i8 %y) nounwind {
 ; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
 ; CHECK-NEXT:    lsl w8, w8, w1
 ; CHECK-NEXT:    and w8, w8, w0
-; CHECK-NEXT:    and w8, w8, #0xff
+; CHECK-NEXT:    and w8, w8, #0x80
 ; CHECK-NEXT:    cmp w8, #1 // =1
 ; CHECK-NEXT:    cset w0, eq
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/AArch64/hwasan-check-memaccess.ll b/llvm/test/CodeGen/AArch64/hwasan-check-memaccess.ll
index d853b148d..48c5bf68e 100644
--- a/llvm/test/CodeGen/AArch64/hwasan-check-memaccess.ll
+++ b/llvm/test/CodeGen/AArch64/hwasan-check-memaccess.ll
@@ -8,11 +8,11 @@ define i8* @f1(i8* %x0, i8* %x1) {
   ; CHECK-NEXT: .cfi_def_cfa_offset 16
   ; CHECK-NEXT: .cfi_offset w30, -16
   ; CHECK-NEXT: mov x9, x0
-  ; CHECK-NEXT: bl __hwasan_check_x1_123
+  ; CHECK-NEXT: bl __hwasan_check_x1_1
   ; CHECK-NEXT: mov x0, x1
   ; CHECK-NEXT: ldr x30, [sp], #16
   ; CHECK-NEXT: ret
-  call void @llvm.hwasan.check.memaccess(i8* %x0, i8* %x1, i32 123)
+  call void @llvm.hwasan.check.memaccess(i8* %x0, i8* %x1, i32 1)
   ret i8* %x1
 }
 
@@ -22,20 +22,21 @@ define i8* @f2(i8* %x0, i8* %x1) {
   ; CHECK-NEXT: .cfi_def_cfa_offset 16
   ; CHECK-NEXT: .cfi_offset w30, -16
   ; CHECK-NEXT: mov x9, x1
-  ; CHECK-NEXT: bl __hwasan_check_x0_456
+  ; CHECK-NEXT: bl __hwasan_check_x0_2_short
   ; CHECK-NEXT: ldr x30, [sp], #16
   ; CHECK-NEXT: ret
-  call void @llvm.hwasan.check.memaccess(i8* %x1, i8* %x0, i32 456)
+  call void @llvm.hwasan.check.memaccess.shortgranules(i8* %x1, i8* %x0, i32 2)
   ret i8* %x0
 }
 
 declare void @llvm.hwasan.check.memaccess(i8*, i8*, i32)
+declare void @llvm.hwasan.check.memaccess.shortgranules(i8*, i8*, i32)
 
-; CHECK:      .section .text.hot,"axG",@progbits,__hwasan_check_x0_456,comdat
-; CHECK-NEXT: .type __hwasan_check_x0_456,@function
-; CHECK-NEXT: .weak __hwasan_check_x0_456
-; CHECK-NEXT: .hidden __hwasan_check_x0_456
-; CHECK-NEXT: __hwasan_check_x0_456:
+; CHECK:      .section .text.hot,"axG",@progbits,__hwasan_check_x0_2_short,comdat
+; CHECK-NEXT: .type __hwasan_check_x0_2_short,@function
+; CHECK-NEXT: .weak __hwasan_check_x0_2_short
+; CHECK-NEXT: .hidden __hwasan_check_x0_2_short
+; CHECK-NEXT: __hwasan_check_x0_2_short:
 ; CHECK-NEXT: ubfx x16, x0, #4, #52
 ; CHECK-NEXT: ldrb w16, [x9, x16]
 ; CHECK-NEXT: cmp x16, x0, lsr #56
@@ -46,7 +47,7 @@ declare void @llvm.hwasan.check.memaccess(i8*, i8*, i32)
 ; CHECK-NEXT: cmp w16, #15
 ; CHECK-NEXT: b.hi .Ltmp2
 ; CHECK-NEXT: and x17, x0, #0xf
-; CHECK-NEXT: add x17, x17, #255
+; CHECK-NEXT: add x17, x17, #3
 ; CHECK-NEXT: cmp w16, w17
 ; CHECK-NEXT: b.ls .Ltmp2
 ; CHECK-NEXT: orr x16, x0, #0xf
@@ -56,17 +57,17 @@ declare void @llvm.hwasan.check.memaccess(i8*, i8*, i32)
 ; CHECK-NEXT: .Ltmp2:
 ; CHECK-NEXT: stp x0, x1, [sp, #-256]!
 ; CHECK-NEXT: stp x29, x30, [sp, #232]
-; CHECK-NEXT: mov x1, #456
-; CHECK-NEXT: adrp  x16, :got:__hwasan_tag_mismatch
-; CHECK-NEXT: ldr x16, [x16, :got_lo12:__hwasan_tag_mismatch]
+; CHECK-NEXT: mov x1, #2
+; CHECK-NEXT: adrp  x16, :got:__hwasan_tag_mismatch_v2
+; CHECK-NEXT: ldr x16, [x16, :got_lo12:__hwasan_tag_mismatch_v2]
 ; CHECK-NEXT: br  x16
 
 
-; CHECK:      .section .text.hot,"axG",@progbits,__hwasan_check_x1_123,comdat
-; CHECK-NEXT: .type __hwasan_check_x1_123,@function
-; CHECK-NEXT: .weak __hwasan_check_x1_123
-; CHECK-NEXT: .hidden __hwasan_check_x1_123
-; CHECK-NEXT: __hwasan_check_x1_123:
+; CHECK:      .section .text.hot,"axG",@progbits,__hwasan_check_x1_1,comdat
+; CHECK-NEXT: .type __hwasan_check_x1_1,@function
+; CHECK-NEXT: .weak __hwasan_check_x1_1
+; CHECK-NEXT: .hidden __hwasan_check_x1_1
+; CHECK-NEXT: __hwasan_check_x1_1:
 ; CHECK-NEXT: ubfx x16, x1, #4, #52
 ; CHECK-NEXT: ldrb w16, [x9, x16]
 ; CHECK-NEXT: cmp x16, x1, lsr #56
@@ -74,21 +75,10 @@ declare void @llvm.hwasan.check.memaccess(i8*, i8*, i32)
 ; CHECK-NEXT: .Ltmp4:
 ; CHECK-NEXT: ret
 ; CHECK-NEXT: .Ltmp3:
-; CHECK-NEXT: cmp w16, #15
-; CHECK-NEXT: b.hi .Ltmp5
-; CHECK-NEXT: and x17, x1, #0xf
-; CHECK-NEXT: add x17, x17, #2047
-; CHECK-NEXT: cmp w16, w17
-; CHECK-NEXT: b.ls .Ltmp5
-; CHECK-NEXT: orr x16, x1, #0xf
-; CHECK-NEXT: ldrb w16, [x16]
-; CHECK-NEXT: cmp x16, x1, lsr #56
-; CHECK-NEXT: b.eq .Ltmp4
-; CHECK-NEXT: .Ltmp5:
 ; CHECK-NEXT: stp x0, x1, [sp, #-256]!
 ; CHECK-NEXT: stp x29, x30, [sp, #232]
 ; CHECK-NEXT: mov x0, x1
-; CHECK-NEXT: mov x1, #123
+; CHECK-NEXT: mov x1, #1
 ; CHECK-NEXT: adrp  x16, :got:__hwasan_tag_mismatch
 ; CHECK-NEXT: ldr x16, [x16, :got_lo12:__hwasan_tag_mismatch]
 ; CHECK-NEXT: br  x16
diff --git a/llvm/test/CodeGen/AArch64/i128_volatile_load_store.ll b/llvm/test/CodeGen/AArch64/i128_volatile_load_store.ll
new file mode 100644
index 000000000..3a2bf3645
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/i128_volatile_load_store.ll
@@ -0,0 +1,117 @@
+; RUN: llc -mtriple=aarch64 %s -o - | FileCheck %s
+
+@x = common dso_local global i128 0
+@y = common dso_local global i128 0
+
+define void @test1() {
+; CHECK-LABEL: test1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, x
+; CHECK-NEXT:    add x8, x8, :lo12:x
+; CHECK-NEXT:    ldp x8, x9, [x8]
+; CHECK-NEXT:    adrp x10, y
+; CHECK-NEXT:    add x10, x10, :lo12:y
+; CHECK-NEXT:    stp x8, x9, [x10]
+; CHECK-NEXT:    ret
+  %tmp = load volatile i128, i128* @x
+  store volatile i128 %tmp, i128* @y
+  ret void
+}
+
+define void @test2() {
+; CHECK-LABEL: test2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, x
+; CHECK-NEXT:    add x8, x8, :lo12:x
+; CHECK-NEXT:    ldp x8, x9, [x8, #504]
+; CHECK-NEXT:    adrp x10, y
+; CHECK-NEXT:    add x10, x10, :lo12:y
+; CHECK-NEXT:    stp x8, x9, [x10, #504]
+; CHECK-NEXT:    ret
+  %tmp = load volatile i128, i128* bitcast (i8* getelementptr (i8, i8* bitcast (i128* @x to i8*), i64 504) to i128*)
+  store volatile i128 %tmp, i128* bitcast (i8* getelementptr (i8, i8* bitcast (i128* @y to i8*), i64 504) to i128*)
+  ret void
+}
+
+define void @test3() {
+; CHECK-LABEL: test3:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, x
+; CHECK-NEXT:    add x8, x8, :lo12:x
+; CHECK-NEXT:    add x8, x8, #512 // =512
+; CHECK-NEXT:    ldp x8, x9, [x8]
+; CHECK-NEXT:    adrp x10, y
+; CHECK-NEXT:    add x10, x10, :lo12:y
+; CHECK-NEXT:    add x10, x10, #512 // =512
+; CHECK-NEXT:    stp x8, x9, [x10]
+; CHECK-NEXT:    ret
+  %tmp = load volatile i128, i128* bitcast (i8* getelementptr (i8, i8* bitcast (i128* @x to i8*), i64 512) to i128*)
+  store volatile i128 %tmp, i128* bitcast (i8* getelementptr (i8, i8* bitcast (i128* @y to i8*), i64 512) to i128*)
+  ret void
+}
+
+define void @test4() {
+; CHECK-LABEL: test4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, x
+; CHECK-NEXT:    add x8, x8, :lo12:x
+; CHECK-NEXT:    ldp x8, x9, [x8, #-512]
+; CHECK-NEXT:    adrp x10, y
+; CHECK-NEXT:    add x10, x10, :lo12:y
+; CHECK-NEXT:    stp x8, x9, [x10, #-512]
+; CHECK-NEXT:    ret
+  %tmp = load volatile i128, i128* bitcast (i8* getelementptr (i8, i8* bitcast (i128* @x to i8*), i64 -512) to i128*)
+  store volatile i128 %tmp, i128* bitcast (i8* getelementptr (i8, i8* bitcast (i128* @y to i8*), i64 -512) to i128*)
+  ret void
+}
+
+define void @test5() {
+; CHECK-LABEL: test5:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, x
+; CHECK-NEXT:    add x8, x8, :lo12:x
+; CHECK-NEXT:    sub x8, x8, #520 // =520
+; CHECK-NEXT:    ldp x8, x9, [x8]
+; CHECK-NEXT:    adrp x10, y
+; CHECK-NEXT:    add x10, x10, :lo12:y
+; CHECK-NEXT:    sub x10, x10, #520 // =520
+; CHECK-NEXT:    stp x8, x9, [x10]
+; CHECK-NEXT:    ret
+  %tmp = load volatile i128, i128* bitcast (i8* getelementptr (i8, i8* bitcast (i128* @x to i8*), i64 -520) to i128*)
+  store volatile i128 %tmp, i128* bitcast (i8* getelementptr (i8, i8* bitcast (i128* @y to i8*), i64 -520) to i128*)
+  ret void
+}
+
+define void @test6() {
+; CHECK-LABEL: test6:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, x
+; CHECK-NEXT:    add x8, x8, :lo12:x
+; CHECK-NEXT:    sub x8, x8, #520 // =520
+; CHECK-NEXT:    ldp x8, x9, [x8]
+; CHECK-NEXT:    adrp x10, y
+; CHECK-NEXT:    add x10, x10, :lo12:y
+; CHECK-NEXT:    sub x10, x10, #520 // =520
+; CHECK-NEXT:    stp x8, x9, [x10]
+; CHECK-NEXT:    ret
+  %tmp = load volatile i128, i128* bitcast (i8* getelementptr (i8, i8* bitcast (i128* @x to i8*), i64 -520) to i128*)
+  store volatile i128 %tmp, i128* bitcast (i8* getelementptr (i8, i8* bitcast (i128* @y to i8*), i64 -520) to i128*)
+  ret void
+}
+
+define void @test7() {
+; CHECK-LABEL: test7:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, x
+; CHECK-NEXT:    add x8, x8, :lo12:x
+; CHECK-NEXT:    add x8, x8, #503 // =503
+; CHECK-NEXT:    ldp x8, x9, [x8]
+; CHECK-NEXT:    adrp x10, y
+; CHECK-NEXT:    add x10, x10, :lo12:y
+; CHECK-NEXT:    add x10, x10, #503 // =503
+; CHECK-NEXT:    stp x8, x9, [x10]
+; CHECK-NEXT:    ret
+  %tmp = load volatile i128, i128* bitcast (i8* getelementptr (i8, i8* bitcast (i128* @x to i8*), i64 503) to i128*)
+  store volatile i128 %tmp, i128* bitcast (i8* getelementptr (i8, i8* bitcast (i128* @y to i8*), i64 503) to i128*)
+  ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/illegal-float-ops.ll b/llvm/test/CodeGen/AArch64/illegal-float-ops.ll
index 8bee4437f..5f24cd4bf 100644
--- a/llvm/test/CodeGen/AArch64/illegal-float-ops.ll
+++ b/llvm/test/CodeGen/AArch64/illegal-float-ops.ll
@@ -1,5 +1,5 @@
 ; RUN: llc -mtriple=aarch64-none-linux-gnu -verify-machineinstrs -o - %s | FileCheck %s
-; RUN: llc -mtriple=aarch64-linux-android -verify-machineinstrs -o - %s | FileCheck --check-prefix=ANDROID-AARCH64 %s
+; RUN: llc -mtriple=aarch64-linux-android -verify-machineinstrs -o - %s | FileCheck %s
 
 @varfloat = global float 0.0
 @vardouble = global double 0.0
@@ -247,11 +247,59 @@ define void @test_fmuladd(fp128 %fp128) {
   ret void
 }
 
+define i32 @test_fptosi32(fp128 %a) {
+; CHECK-LABEL: test_fptosi32:
+; CHECK: bl __fixtfsi
+  %conv.i = fptosi fp128 %a to i32
+  %b = add nsw i32 %conv.i, 48
+  ret i32 %b
+}
+
+define i64 @test_fptosi64(fp128 %a) {
+; CHECK-LABEL: test_fptosi64:
+; CHECK: bl __fixtfdi
+  %conv.i = fptosi fp128 %a to i64
+  %b = add nsw i64 %conv.i, 48
+  ret i64 %b
+}
+
+define i128 @test_fptosi128(fp128 %a) {
+; CHECK-LABEL: test_fptosi128:
+; CHECK: bl __fixtfti
+  %conv.i = fptosi fp128 %a to i128
+  %b = add nsw i128 %conv.i, 48
+  ret i128 %b
+}
+
+define i32 @test_fptoui32(fp128 %a) {
+; CHECK-LABEL: test_fptoui32:
+; CHECK: bl __fixunstfsi
+  %conv.i = fptoui fp128 %a to i32
+  %b = add nsw i32 %conv.i, 48
+  ret i32 %b
+}
+
+define i64 @test_fptoui64(fp128 %a) {
+; CHECK-LABEL: test_fptoui64:
+; CHECK: bl __fixunstfdi
+  %conv.i = fptoui fp128 %a to i64
+  %b = add nsw i64 %conv.i, 48
+  ret i64 %b
+}
+
+define i128 @test_fptoui128(fp128 %a) {
+; CHECK-LABEL: test_fptoui128:
+; CHECK: bl __fixunstfti
+  %conv.i = fptoui fp128 %a to i128
+  %b = add nsw i128 %conv.i, 48
+  ret i128 %b
+}
+
 define void @test_exp_finite(double %double) #0 {
   %expdouble = call double @llvm.exp.f64(double %double)
   store double %expdouble, double* @vardouble
   ; ANDROID-AARCH64-NOT: bl __exp_finite
-  ; CHECK: bl __exp_finite
+  ; CHECK: bl exp
 
   ret void
 }
@@ -259,8 +307,8 @@ define void @test_exp_finite(double %double) #0 {
 define void @test_exp2_finite(double %double) #0 {
   %expdouble = call double @llvm.exp2.f64(double %double)
   store double %expdouble, double* @vardouble
-  ; ANDROID-AARCH64-NOT: bl __exp2_finite
-  ; CHECK: bl __exp2_finite
+  ; CHECK-NOT: bl __exp2_finite
+  ; CHECK: bl exp2
 
   ret void
 }
@@ -268,32 +316,32 @@ define void @test_exp2_finite(double %double) #0 {
 define void @test_log_finite(double %double) #0 {
   %logdouble = call double @llvm.log.f64(double %double)
   store double %logdouble, double* @vardouble
-  ; ANDROID-AARCH64-NOT: bl __log_finite
-  ; CHECK: bl __log_finite
+  ; CHECK-NOT: bl __log_finite
+  ; CHECK: bl log
   ret void
 }
 
 define void @test_log2_finite(double %double) #0 {
   %log2double = call double @llvm.log2.f64(double %double)
   store double %log2double, double* @vardouble
-  ; ANDROID-AARCH64-NOT: bl __log2_finite
-  ; CHECK: bl __log2_finite
+  ; CHECK-NOT: bl __log2_finite
+  ; CHECK: bl log2
   ret void
 }
 
 define void @test_log10_finite(double %double) #0 {
   %log10double = call double @llvm.log10.f64(double %double)
   store double %log10double, double* @vardouble
-  ; ANDROID-AARCH64-NOT: bl __log10_finite
-  ; CHECK: bl __log10_finite
+  ; CHECK-NOT: bl __log10_finite
+  ; CHECK: bl log10
   ret void
 }
 
 define void @test_pow_finite(double %double) #0 {
   %powdouble = call double @llvm.pow.f64(double %double, double %double)
   store double %powdouble, double* @vardouble
-  ; ANDROID-AARCH64-NOT: bl __pow_finite
-  ; CHECK: bl __pow_finite
+  ; CHECK-NOT: bl __pow_finite
+  ; CHECK: bl pow
   ret void
 }
 
diff --git a/llvm/test/CodeGen/AArch64/irg_sp_tagp.ll b/llvm/test/CodeGen/AArch64/irg_sp_tagp.ll
index 1232090fc..4af79acf2 100644
--- a/llvm/test/CodeGen/AArch64/irg_sp_tagp.ll
+++ b/llvm/test/CodeGen/AArch64/irg_sp_tagp.ll
@@ -35,7 +35,7 @@ entry:
 define void @realign() {
 entry:
 ; CHECK-LABEL: realign:
-; CHECK:      add  x29, sp, #16
+; CHECK:      mov  x29, sp
 ; CHECK:      and  sp, x{{[0-9]*}}, #0xffffffffffffffc0
 ; CHECK:      irg  [[R:x[0-9]+]], sp{{$}}
 ; CHECK:      addg x0, [[R]], #0, #1
diff --git a/llvm/test/CodeGen/AArch64/jump-table-32.ll b/llvm/test/CodeGen/AArch64/jump-table-32.ll
new file mode 100644
index 000000000..339a44fc9
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/jump-table-32.ll
@@ -0,0 +1,42 @@
+; RUN: llc -verify-machineinstrs -o - %s -mtriple=arm64_32-apple-ios7.0 -aarch64-enable-atomic-cfg-tidy=0 | FileCheck %s
+
+define i32 @test_jumptable(i32 %in) {
+; CHECK: test_jumptable
+
+  switch i32 %in, label %def [
+    i32 0, label %lbl1
+    i32 1, label %lbl2
+    i32 2, label %lbl3
+    i32 4, label %lbl4
+  ]
+; CHECK: adrp    [[JTPAGE:x[0-9]+]], LJTI0_0@PAGE
+; CHECK: mov     w[[INDEX:[0-9]+]], w0
+; CHECK: add     x[[JT:[0-9]+]], [[JTPAGE]], LJTI0_0@PAGEOFF
+; CHECK: adr     [[BASE_BLOCK:x[0-9]+]], LBB0_2
+; CHECK: ldrb    w[[OFFSET:[0-9]+]], [x[[JT]], x[[INDEX]]]
+; CHECK: add     [[DEST:x[0-9]+]], [[BASE_BLOCK]], x[[OFFSET]], lsl #2
+; CHECK: br      [[DEST]]
+
+def:
+  ret i32 0
+
+lbl1:
+  ret i32 1
+
+lbl2:
+  ret i32 2
+
+lbl3:
+  ret i32 4
+
+lbl4:
+  ret i32 8
+
+}
+
+; CHECK: LJTI0_0:
+; CHECK-NEXT: .byte
+; CHECK-NEXT: .byte
+; CHECK-NEXT: .byte
+; CHECK-NEXT: .byte
+; CHECK-NEXT: .byte
diff --git a/llvm/test/CodeGen/AArch64/jump-table-exynos.ll b/llvm/test/CodeGen/AArch64/jump-table-exynos.ll
index e01841079..8dca2de6e 100644
--- a/llvm/test/CodeGen/AArch64/jump-table-exynos.ll
+++ b/llvm/test/CodeGen/AArch64/jump-table-exynos.ll
@@ -1,6 +1,4 @@
 ; RUN: llc -o - %s -mtriple=aarch64-none-linux-gnu -mattr=+force-32bit-jump-tables -aarch64-enable-atomic-cfg-tidy=0 | FileCheck %s
-; RUN: llc -o - %s -mtriple=aarch64-none-linux-gnu -mcpu=exynos-m1 -aarch64-enable-atomic-cfg-tidy=0 | FileCheck %s
-; RUN: llc -o - %s -mtriple=aarch64-none-linux-gnu -mcpu=exynos-m2 -aarch64-enable-atomic-cfg-tidy=0 | FileCheck %s
 ; RUN: llc -o - %s -mtriple=aarch64-none-linux-gnu -mcpu=exynos-m3 -aarch64-enable-atomic-cfg-tidy=0 | FileCheck %s
 
 ; Exynos doesn't want jump tables to be compressed for now.
diff --git a/llvm/test/CodeGen/AArch64/large-stack.ll b/llvm/test/CodeGen/AArch64/large-stack.ll
new file mode 100644
index 000000000..11027652d
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/large-stack.ll
@@ -0,0 +1,49 @@
+; RUN: llc < %s -mtriple=aarch64-none-linux-gnu | FileCheck %s
+
+@.str = private unnamed_addr constant [11 x i8] c"val = %ld\0A\00", align 1
+
+; Function Attrs: noinline optnone
+define dso_local void @set_large(i64 %val) #0 {
+entry:
+  %val.addr = alloca i64, align 8
+  %large = alloca [268435456 x i64], align 8
+  %i = alloca i32, align 4
+  store i64 %val, i64* %val.addr, align 8
+  %0 = load i64, i64* %val.addr, align 8
+  %arrayidx = getelementptr inbounds [268435456 x i64], [268435456 x i64]* %large, i64 0, i64 %0
+  store i64 1, i64* %arrayidx, align 8
+  %1 = load i64, i64* %val.addr, align 8
+  %arrayidx1 = getelementptr inbounds [268435456 x i64], [268435456 x i64]* %large, i64 0, i64 %1
+  %2 = load i64, i64* %arrayidx1, align 8
+  %call = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([11 x i8], [11 x i8]* @.str, i64 0, i64 0), i64 %2)
+  ret void
+}
+
+declare dso_local i32 @printf(i8*, ...)
+
+attributes #0 = { noinline optnone "frame-pointer"="all" }
+
+; CHECK:                  stp	x[[SPILL_REG1:[0-9]+]], x[[SPILL_REG2:[0-9]+]], [sp, #-[[SPILL_OFFSET1:[0-9]+]]]
+; CHECK-NEXT:             str	x[[SPILL_REG3:[0-9]+]], [sp, #[[SPILL_OFFSET2:[0-9]+]]]
+; CHECK-NEXT:             mov	x[[FRAME:[0-9]+]], sp
+; CHECK-COUNT-128:        sub	sp, sp, #[[STACK1:[0-9]+]], lsl #12
+; CHECK-NEXT:             sub	sp, sp, #[[STACK2:[0-9]+]], lsl #12
+; CHECK-NEXT:             sub	sp, sp, #[[STACK3:[0-9]+]]
+; CHECK:                  sub	x[[INDEX:[0-9]+]], x[[FRAME]], #8
+; CHECK-NEXT:             str	x0, [x[[INDEX]]]
+; CHECK-NEXT:             ldr	x[[VAL1:[0-9]+]], [x[[INDEX]]]
+; CHECK-NEXT:             mov	x[[VAL2:[0-9]+]], #8
+; CHECK-NEXT:             add	x[[VAL3:[0-9]+]], sp, #8
+; CHECK-NEXT:             madd	x[[VAL1]], x[[VAL1]], x[[VAL2]], x[[VAL3]]
+; CHECK-NEXT:             mov	x[[TMP1:[0-9]+]], #1
+; CHECK-NEXT:             str	x[[TMP1]], [x[[VAL1]]]
+; CHECK-NEXT:             ldr	x[[INDEX]], [x[[INDEX]]]
+; CHECK-NEXT:             mov	x[[VAL4:[0-9]+]], #8
+; CHECK-NEXT:             madd	x[[INDEX]], x[[INDEX]], x[[VAL4]], x[[VAL3]]
+; CHECK-NEXT:             ldr	x1, [x[[INDEX]]
+; CHECK:                  bl	printf
+; CHECK-COUNT-128:        add	sp, sp, #[[STACK1]], lsl #12
+; CHECK-NEXT:             add	sp, sp, #[[STACK2]], lsl #12
+; CHECK-NEXT:             add	sp, sp, #[[STACK3]]
+; CHECK-NEXT:             ldr	x[[SPILL_REG3]], [sp, #[[SPILL_OFFSET2]]]
+; CHECK-NEXT:             ldp	x[[SPILL_REG1]], x[[SPILL_REG2]], [sp], #[[SPILL_OFFSET1]]
diff --git a/llvm/test/CodeGen/AArch64/ldst-paired-aliasing.ll b/llvm/test/CodeGen/AArch64/ldst-paired-aliasing.ll
index f36131223..697ba4615 100644
--- a/llvm/test/CodeGen/AArch64/ldst-paired-aliasing.ll
+++ b/llvm/test/CodeGen/AArch64/ldst-paired-aliasing.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mcpu cortex-a53 < %s | FileCheck %s
 target datalayout = "e-m:e-i64:64-i128:128-n8:16:32:64-S128"
 target triple = "aarch64--linux-gnu"
@@ -10,11 +11,33 @@ declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i1) #3
 define i32 @main() local_unnamed_addr #1 {
 ; Make sure the stores happen in the correct order (the exact instructions could change).
 ; CHECK-LABEL: main:
+; CHECK:       // %bb.0: // %for.body.lr.ph.i.i.i.i.i.i63
+; CHECK-NEXT:    sub sp, sp, #112 // =112
+; CHECK-NEXT:    str x30, [sp, #96] // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 112
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    bl _Z5setupv
+; CHECK-NEXT:    movi v0.4s, #1
+; CHECK-NEXT:    mov w9, #1
+; CHECK-NEXT:    add x0, sp, #48 // =48
+; CHECK-NEXT:    mov x1, sp
+; CHECK-NEXT:    str xzr, [sp, #80]
+; CHECK-NEXT:    str w9, [sp, #80]
+; CHECK-NEXT:    stp q0, q0, [sp, #48]
+; CHECK-NEXT:    ldr w8, [sp, #48]
+; CHECK-NEXT:    cmp w8, #1 // =1
+; CHECK-NEXT:    b.ne .LBB0_2
+; CHECK-NEXT:  // %bb.1: // %for.inc
+; CHECK-NEXT:    bl f
+; CHECK-NEXT:    b .LBB0_3
+; CHECK-NEXT:  .LBB0_2: // %if.then
+; CHECK-NEXT:    bl f2
+; CHECK-NEXT:  .LBB0_3: // %for.inc
+; CHECK-NEXT:    ldr x30, [sp, #96] // 8-byte Folded Reload
+; CHECK-NEXT:    mov w0, wzr
+; CHECK-NEXT:    add sp, sp, #112 // =112
+; CHECK-NEXT:    ret
 
-; CHECK: mov w9, #1
-; CHECK: str x9, [sp, #80]
-; CHECK: stp q0, q0, [sp, #48]
-; CHECK: ldr w8, [sp, #48]
 
 for.body.lr.ph.i.i.i.i.i.i63:
   %b1 = alloca [10 x i32], align 16
diff --git a/llvm/test/CodeGen/AArch64/load-combine-big-endian.ll b/llvm/test/CodeGen/AArch64/load-combine-big-endian.ll
index 918ceaeb1..19de95198 100644
--- a/llvm/test/CodeGen/AArch64/load-combine-big-endian.ll
+++ b/llvm/test/CodeGen/AArch64/load-combine-big-endian.ll
@@ -1,11 +1,13 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=arm64eb-unknown | FileCheck %s
 
 ; i8* p; // p is 4 byte aligned
 ; ((i32) p[0] << 24) | ((i32) p[1] << 16) | ((i32) p[2] << 8) | (i32) p[3]
 define i32 @load_i32_by_i8_big_endian(i32* %arg) {
 ; CHECK-LABEL: load_i32_by_i8_big_endian:
-; CHECK: ldr		w0, [x0]
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr w0, [x0]
+; CHECK-NEXT:    ret
   %tmp = bitcast i32* %arg to i8*
   %tmp1 = load i8, i8* %tmp, align 4
   %tmp2 = zext i8 %tmp1 to i32
@@ -31,8 +33,9 @@ define i32 @load_i32_by_i8_big_endian(i32* %arg) {
 ; ((i32) (((i16) p[0] << 8) | (i16) p[1]) << 16) | (i32) (((i16) p[3] << 8) | (i16) p[4])
 define i32 @load_i32_by_i16_by_i8_big_endian(i32* %arg) {
 ; CHECK-LABEL: load_i32_by_i16_by_i8_big_endian:
-; CHECK: ldr		w0, [x0]
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr w0, [x0]
+; CHECK-NEXT:    ret
   %tmp = bitcast i32* %arg to i8*
   %tmp1 = load i8, i8* %tmp, align 4
   %tmp2 = zext i8 %tmp1 to i16
@@ -60,8 +63,9 @@ define i32 @load_i32_by_i16_by_i8_big_endian(i32* %arg) {
 ; ((i32) p[0] << 16) | (i32) p[1]
 define i32 @load_i32_by_i16(i32* %arg) {
 ; CHECK-LABEL: load_i32_by_i16:
-; CHECK: ldr		w0, [x0]
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr w0, [x0]
+; CHECK-NEXT:    ret
   %tmp = bitcast i32* %arg to i16*
   %tmp1 = load i16, i16* %tmp, align 4
   %tmp2 = zext i16 %tmp1 to i32
@@ -78,8 +82,9 @@ define i32 @load_i32_by_i16(i32* %arg) {
 ; (i32) (p_16[0] << 16) | ((i32) p[2] << 8) | (i32) p[3]
 define i32 @load_i32_by_i16_i8(i32* %arg) {
 ; CHECK-LABEL: load_i32_by_i16_i8:
-; CHECK: ldr		w0, [x0]
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr w0, [x0]
+; CHECK-NEXT:    ret
   %tmp = bitcast i32* %arg to i16*
   %tmp1 = bitcast i32* %arg to i8*
   %tmp2 = load i16, i16* %tmp, align 4
@@ -101,9 +106,10 @@ define i32 @load_i32_by_i16_i8(i32* %arg) {
 ; (i64) p[0] | ((i64) p[1] << 8) | ((i64) p[2] << 16) | ((i64) p[3] << 24) | ((i64) p[4] << 32) | ((i64) p[5] << 40) | ((i64) p[6] << 48) | ((i64) p[7] << 56)
 define i64 @load_i64_by_i8_bswap(i64* %arg) {
 ; CHECK-LABEL: load_i64_by_i8_bswap:
-; CHECK: ldr		x8, [x0]
-; CHECK-NEXT: rev	x0, x8
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr x8, [x0]
+; CHECK-NEXT:    rev x0, x8
+; CHECK-NEXT:    ret
   %tmp = bitcast i64* %arg to i8*
   %tmp1 = load i8, i8* %tmp, align 8
   %tmp2 = zext i8 %tmp1 to i64
@@ -149,8 +155,9 @@ define i64 @load_i64_by_i8_bswap(i64* %arg) {
 ; ((i64) p[0] << 56) | ((i64) p[1] << 48) | ((i64) p[2] << 40) | ((i64) p[3] << 32) | ((i64) p[4] << 24) | ((i64) p[5] << 16) | ((i64) p[6] << 8) | (i64) p[7]
 define i64 @load_i64_by_i8(i64* %arg) {
 ; CHECK-LABEL: load_i64_by_i8:
-; CHECK: ldr		x0, [x0]
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr x0, [x0]
+; CHECK-NEXT:    ret
   %tmp = bitcast i64* %arg to i8*
   %tmp1 = load i8, i8* %tmp, align 8
   %tmp2 = zext i8 %tmp1 to i64
@@ -196,9 +203,10 @@ define i64 @load_i64_by_i8(i64* %arg) {
 ; (i32) p[1] | ((i32) p[2] << 8) | ((i32) p[3] << 16) | ((i32) p[4] << 24)
 define i32 @load_i32_by_i8_nonzero_offset(i32* %arg) {
 ; CHECK-LABEL: load_i32_by_i8_nonzero_offset:
-; CHECK: ldur  w8, [x0, #1]
-; CHECK-NEXT: rev w0, w8
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldur w8, [x0, #1]
+; CHECK-NEXT:    rev w0, w8
+; CHECK-NEXT:    ret
 
   %tmp = bitcast i32* %arg to i8*
   %tmp1 = getelementptr inbounds i8, i8* %tmp, i32 1
@@ -226,9 +234,10 @@ define i32 @load_i32_by_i8_nonzero_offset(i32* %arg) {
 ; (i32) p[-4] | ((i32) p[-3] << 8) | ((i32) p[-2] << 16) | ((i32) p[-1] << 24)
 define i32 @load_i32_by_i8_neg_offset(i32* %arg) {
 ; CHECK-LABEL: load_i32_by_i8_neg_offset:
-; CHECK: ldur  w8, [x0, #-4]
-; CHECK-NEXT: rev w0, w8
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldur w8, [x0, #-4]
+; CHECK-NEXT:    rev w0, w8
+; CHECK-NEXT:    ret
 
   %tmp = bitcast i32* %arg to i8*
   %tmp1 = getelementptr inbounds i8, i8* %tmp, i32 -4
@@ -256,8 +265,9 @@ define i32 @load_i32_by_i8_neg_offset(i32* %arg) {
 ; (i32) p[4] | ((i32) p[3] << 8) | ((i32) p[2] << 16) | ((i32) p[1] << 24)
 define i32 @load_i32_by_i8_nonzero_offset_bswap(i32* %arg) {
 ; CHECK-LABEL: load_i32_by_i8_nonzero_offset_bswap:
-; CHECK: ldur w0, [x0, #1]
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldur w0, [x0, #1]
+; CHECK-NEXT:    ret
 
   %tmp = bitcast i32* %arg to i8*
   %tmp1 = getelementptr inbounds i8, i8* %tmp, i32 4
@@ -285,8 +295,9 @@ define i32 @load_i32_by_i8_nonzero_offset_bswap(i32* %arg) {
 ; (i32) p[-1] | ((i32) p[-2] << 8) | ((i32) p[-3] << 16) | ((i32) p[-4] << 24)
 define i32 @load_i32_by_i8_neg_offset_bswap(i32* %arg) {
 ; CHECK-LABEL: load_i32_by_i8_neg_offset_bswap:
-; CHECK: ldur w0, [x0, #-4]
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldur w0, [x0, #-4]
+; CHECK-NEXT:    ret
 
   %tmp = bitcast i32* %arg to i8*
   %tmp1 = getelementptr inbounds i8, i8* %tmp, i32 -1
@@ -316,9 +327,10 @@ declare i16 @llvm.bswap.i16(i16)
 ; (i32) bswap(p[0]) | (i32) bswap(p[1] << 16)
 define i32 @load_i32_by_bswap_i16(i32* %arg) {
 ; CHECK-LABEL: load_i32_by_bswap_i16:
-; CHECK: ldr   w8, [x0]
-; CHECK-NEXT: rev w0, w8
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr w8, [x0]
+; CHECK-NEXT:    rev w0, w8
+; CHECK-NEXT:    ret
   %tmp = bitcast i32* %arg to i16*
   %tmp1 = load i16, i16* %tmp, align 4
   %tmp11 = call i16 @llvm.bswap.i16(i16 %tmp1)
@@ -336,8 +348,9 @@ define i32 @load_i32_by_bswap_i16(i32* %arg) {
 ; (i32) p[1] | (sext(p[0] << 16) to i32)
 define i32 @load_i32_by_sext_i16(i32* %arg) {
 ; CHECK-LABEL: load_i32_by_sext_i16:
-; CHECK: ldr   w0, [x0]
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr w0, [x0]
+; CHECK-NEXT:    ret
   %tmp = bitcast i32* %arg to i16*
   %tmp1 = load i16, i16* %tmp, align 4
   %tmp2 = sext i16 %tmp1 to i32
@@ -354,10 +367,11 @@ define i32 @load_i32_by_sext_i16(i32* %arg) {
 ; (i32) p[i] | ((i32) p[i + 1] << 8) | ((i32) p[i + 2] << 16) | ((i32) p[i + 3] << 24)
 define i32 @load_i32_by_i8_base_offset_index(i8* %arg, i32 %i) {
 ; CHECK-LABEL: load_i32_by_i8_base_offset_index:
-; CHECK: add  x8, x0, w1, uxtw
-; CHECK-NEXT: ldr w8, [x8, #12]
-; CHECK-NEXT: rev w0, w8
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    add x8, x0, w1, uxtw
+; CHECK-NEXT:    ldr w8, [x8, #12]
+; CHECK-NEXT:    rev w0, w8
+; CHECK-NEXT:    ret
   %tmp = add nuw nsw i32 %i, 3
   %tmp2 = add nuw nsw i32 %i, 2
   %tmp3 = add nuw nsw i32 %i, 1
@@ -392,10 +406,11 @@ define i32 @load_i32_by_i8_base_offset_index(i8* %arg, i32 %i) {
 ; (i32) p[i + 1] | ((i32) p[i + 2] << 8) | ((i32) p[i + 3] << 16) | ((i32) p[i + 4] << 24)
 define i32 @load_i32_by_i8_base_offset_index_2(i8* %arg, i32 %i) {
 ; CHECK-LABEL: load_i32_by_i8_base_offset_index_2:
-; CHECK: add x8, x0, w1, uxtw
-; CHECK-NEXT: ldur  w8, [x8, #13]
-; CHECK-NEXT: rev w0, w8
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    add x8, x0, w1, uxtw
+; CHECK-NEXT:    ldur w8, [x8, #13]
+; CHECK-NEXT:    rev w0, w8
+; CHECK-NEXT:    ret
   %tmp = add nuw nsw i32 %i, 4
   %tmp2 = add nuw nsw i32 %i, 3
   %tmp3 = add nuw nsw i32 %i, 2
@@ -429,11 +444,11 @@ define i32 @load_i32_by_i8_base_offset_index_2(i8* %arg, i32 %i) {
 ; (i32) p[0] | ((i32) p[1] << 8)
 define i32 @zext_load_i32_by_i8(i32* %arg) {
 ; CHECK-LABEL: zext_load_i32_by_i8:
-; CHECK: ldrb  w8, [x0]
-; CHECK-NEXT: ldrb  w9, [x0, #1]
-; CHECK-NEXT: bfi w8, w9, #8, #8
-; CHECK-NEXT: mov  w0, w8
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldrh w8, [x0]
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    rev w0, w8
+; CHECK-NEXT:    ret
 
   %tmp = bitcast i32* %arg to i8*
   %tmp1 = getelementptr inbounds i8, i8* %tmp, i32 0
@@ -451,11 +466,12 @@ define i32 @zext_load_i32_by_i8(i32* %arg) {
 ; ((i32) p[0] << 8) | ((i32) p[1] << 16)
 define i32 @zext_load_i32_by_i8_shl_8(i32* %arg) {
 ; CHECK-LABEL: zext_load_i32_by_i8_shl_8:
-; CHECK: ldrb  w8, [x0]
-; CHECK-NEXT: ldrb  w9, [x0, #1]
-; CHECK-NEXT: lsl w0, w8, #8
-; CHECK-NEXT: bfi w0, w9, #16, #8
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldrb w8, [x0]
+; CHECK-NEXT:    ldrb w9, [x0, #1]
+; CHECK-NEXT:    lsl w0, w8, #8
+; CHECK-NEXT:    bfi w0, w9, #16, #8
+; CHECK-NEXT:    ret
 
   %tmp = bitcast i32* %arg to i8*
   %tmp1 = getelementptr inbounds i8, i8* %tmp, i32 0
@@ -474,11 +490,12 @@ define i32 @zext_load_i32_by_i8_shl_8(i32* %arg) {
 ; ((i32) p[0] << 16) | ((i32) p[1] << 24)
 define i32 @zext_load_i32_by_i8_shl_16(i32* %arg) {
 ; CHECK-LABEL: zext_load_i32_by_i8_shl_16:
-; CHECK: ldrb  w8, [x0]
-; CHECK-NEXT: ldrb  w9, [x0, #1]
-; CHECK-NEXT: lsl w0, w8, #16
-; CHECK-NEXT: bfi w0, w9, #24, #8
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldrb w8, [x0]
+; CHECK-NEXT:    ldrb w9, [x0, #1]
+; CHECK-NEXT:    lsl w0, w8, #16
+; CHECK-NEXT:    bfi w0, w9, #24, #8
+; CHECK-NEXT:    ret
 
   %tmp = bitcast i32* %arg to i8*
   %tmp1 = getelementptr inbounds i8, i8* %tmp, i32 0
@@ -496,11 +513,9 @@ define i32 @zext_load_i32_by_i8_shl_16(i32* %arg) {
 ; (i32) p[1] | ((i32) p[0] << 8)
 define i32 @zext_load_i32_by_i8_bswap(i32* %arg) {
 ; CHECK-LABEL: zext_load_i32_by_i8_bswap:
-; CHECK: ldrb  w8, [x0, #1]
-; CHECK-NEXT: ldrb    w9, [x0]
-; CHECK-NEXT: bfi w8, w9, #8, #8
-; CHECK-NEXT: mov  w0, w8
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldrh w0, [x0]
+; CHECK-NEXT:    ret
 
   %tmp = bitcast i32* %arg to i8*
   %tmp1 = getelementptr inbounds i8, i8* %tmp, i32 1
@@ -518,11 +533,12 @@ define i32 @zext_load_i32_by_i8_bswap(i32* %arg) {
 ; ((i32) p[1] << 8) | ((i32) p[0] << 16)
 define i32 @zext_load_i32_by_i8_bswap_shl_8(i32* %arg) {
 ; CHECK-LABEL: zext_load_i32_by_i8_bswap_shl_8:
-; CHECK: ldrb  w8, [x0, #1]
-; CHECK-NEXT: ldrb    w9, [x0]
-; CHECK-NEXT: lsl w0, w8, #8
-; CHECK-NEXT: bfi w0, w9, #16, #8
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldrb w8, [x0, #1]
+; CHECK-NEXT:    ldrb w9, [x0]
+; CHECK-NEXT:    lsl w0, w8, #8
+; CHECK-NEXT:    bfi w0, w9, #16, #8
+; CHECK-NEXT:    ret
 
   %tmp = bitcast i32* %arg to i8*
   %tmp1 = getelementptr inbounds i8, i8* %tmp, i32 1
@@ -541,11 +557,12 @@ define i32 @zext_load_i32_by_i8_bswap_shl_8(i32* %arg) {
 ; ((i32) p[1] << 16) | ((i32) p[0] << 24)
 define i32 @zext_load_i32_by_i8_bswap_shl_16(i32* %arg) {
 ; CHECK-LABEL: zext_load_i32_by_i8_bswap_shl_16:
-; CHECK: ldrb  w8, [x0, #1]
-; CHECK-NEXT: ldrb    w9, [x0]
-; CHECK-NEXT: lsl w0, w8, #16
-; CHECK-NEXT: bfi w0, w9, #24, #8
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldrb w8, [x0, #1]
+; CHECK-NEXT:    ldrb w9, [x0]
+; CHECK-NEXT:    lsl w0, w8, #16
+; CHECK-NEXT:    bfi w0, w9, #24, #8
+; CHECK-NEXT:    ret
 
   %tmp = bitcast i32* %arg to i8*
   %tmp1 = getelementptr inbounds i8, i8* %tmp, i32 1
@@ -568,10 +585,11 @@ define i32 @zext_load_i32_by_i8_bswap_shl_16(i32* %arg) {
 ; because in the original DAG we don't have p[1] address available
 define i16 @load_i16_from_nonzero_offset(i8* %p) {
 ; CHECK-LABEL: load_i16_from_nonzero_offset:
-; CHECK:  ldrh    w8, [x0]
-; CHECK-NEXT: ldrb  w0, [x0, #2]
-; CHECK-NEXT: bfi w0, w8, #8, #24
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldrh w8, [x0]
+; CHECK-NEXT:    ldrb w0, [x0, #2]
+; CHECK-NEXT:    bfi w0, w8, #8, #24
+; CHECK-NEXT:    ret
 
   %p1.i16 = bitcast i8* %p to i16*
   %p2.i8 = getelementptr i8, i8* %p, i64 2
diff --git a/llvm/test/CodeGen/AArch64/load-combine.ll b/llvm/test/CodeGen/AArch64/load-combine.ll
index b9879b555..066ecb21d 100644
--- a/llvm/test/CodeGen/AArch64/load-combine.ll
+++ b/llvm/test/CodeGen/AArch64/load-combine.ll
@@ -1,11 +1,13 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=arm64-unknown | FileCheck %s
 
 ; i8* p; // p is 1 byte aligned
 ; (i32) p[0] | ((i32) p[1] << 8) | ((i32) p[2] << 16) | ((i32) p[3] << 24)
 define i32 @load_i32_by_i8_unaligned(i32* %arg) {
 ; CHECK-LABEL: load_i32_by_i8_unaligned:
-; CHECK: ldr		w0, [x0]
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr w0, [x0]
+; CHECK-NEXT:    ret
   %tmp = bitcast i32* %arg to i8*
   %tmp1 = getelementptr inbounds i8, i8* %tmp, i32 0
   %tmp2 = load i8, i8* %tmp1, align 1
@@ -32,8 +34,9 @@ define i32 @load_i32_by_i8_unaligned(i32* %arg) {
 ; (i32) p[0] | ((i32) p[1] << 8) | ((i32) p[2] << 16) | ((i32) p[3] << 24)
 define i32 @load_i32_by_i8_aligned(i32* %arg) {
 ; CHECK-LABEL: load_i32_by_i8_aligned:
-; CHECK: ldr    w0, [x0]
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr w0, [x0]
+; CHECK-NEXT:    ret
   %tmp = bitcast i32* %arg to i8*
   %tmp1 = getelementptr inbounds i8, i8* %tmp, i32 0
   %tmp2 = load i8, i8* %tmp1, align 4
@@ -60,9 +63,10 @@ define i32 @load_i32_by_i8_aligned(i32* %arg) {
 ; ((i32) p[0] << 24) | ((i32) p[1] << 16) | ((i32) p[2] << 8) | (i32) p[3]
 define i32 @load_i32_by_i8_bswap(i32* %arg) {
 ; CHECK-LABEL: load_i32_by_i8_bswap:
-; CHECK: ldr		w8, [x0]
-; CHECK-NEXT: rev	w0, w8
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr w8, [x0]
+; CHECK-NEXT:    rev w0, w8
+; CHECK-NEXT:    ret
   %tmp = bitcast i32* %arg to i8*
   %tmp1 = load i8, i8* %tmp, align 4
   %tmp2 = zext i8 %tmp1 to i32
@@ -88,8 +92,9 @@ define i32 @load_i32_by_i8_bswap(i32* %arg) {
 ; (i64) p[0] | ((i64) p[1] << 8) | ((i64) p[2] << 16) | ((i64) p[3] << 24) | ((i64) p[4] << 32) | ((i64) p[5] << 40) | ((i64) p[6] << 48) | ((i64) p[7] << 56)
 define i64 @load_i64_by_i8(i64* %arg) {
 ; CHECK-LABEL: load_i64_by_i8:
-; CHECK: ldr		x0, [x0]
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr x0, [x0]
+; CHECK-NEXT:    ret
   %tmp = bitcast i64* %arg to i8*
   %tmp1 = load i8, i8* %tmp, align 8
   %tmp2 = zext i8 %tmp1 to i64
@@ -135,9 +140,10 @@ define i64 @load_i64_by_i8(i64* %arg) {
 ; ((i64) p[0] << 56) | ((i64) p[1] << 48) | ((i64) p[2] << 40) | ((i64) p[3] << 32) | ((i64) p[4] << 24) | ((i64) p[5] << 16) | ((i64) p[6] << 8) | (i64) p[7]
 define i64 @load_i64_by_i8_bswap(i64* %arg) {
 ; CHECK-LABEL: load_i64_by_i8_bswap:
-; CHECK: ldr		x8, [x0]
-; CHECK-NEXT: rev	x0, x8
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr x8, [x0]
+; CHECK-NEXT:    rev x0, x8
+; CHECK-NEXT:    ret
   %tmp = bitcast i64* %arg to i8*
   %tmp1 = load i8, i8* %tmp, align 8
   %tmp2 = zext i8 %tmp1 to i64
@@ -183,8 +189,9 @@ define i64 @load_i64_by_i8_bswap(i64* %arg) {
 ; (i32) p[1] | ((i32) p[2] << 8) | ((i32) p[3] << 16) | ((i32) p[4] << 24)
 define i32 @load_i32_by_i8_nonzero_offset(i32* %arg) {
 ; CHECK-LABEL: load_i32_by_i8_nonzero_offset:
-; CHECK: ldur w0, [x0, #1]
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldur w0, [x0, #1]
+; CHECK-NEXT:    ret
 
   %tmp = bitcast i32* %arg to i8*
   %tmp1 = getelementptr inbounds i8, i8* %tmp, i32 1
@@ -212,8 +219,9 @@ define i32 @load_i32_by_i8_nonzero_offset(i32* %arg) {
 ; (i32) p[-4] | ((i32) p[-3] << 8) | ((i32) p[-2] << 16) | ((i32) p[-1] << 24)
 define i32 @load_i32_by_i8_neg_offset(i32* %arg) {
 ; CHECK-LABEL: load_i32_by_i8_neg_offset:
-; CHECK: ldur w0, [x0, #-4]
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldur w0, [x0, #-4]
+; CHECK-NEXT:    ret
 
   %tmp = bitcast i32* %arg to i8*
   %tmp1 = getelementptr inbounds i8, i8* %tmp, i32 -4
@@ -241,9 +249,10 @@ define i32 @load_i32_by_i8_neg_offset(i32* %arg) {
 ; (i32) p[4] | ((i32) p[3] << 8) | ((i32) p[2] << 16) | ((i32) p[1] << 24)
 define i32 @load_i32_by_i8_nonzero_offset_bswap(i32* %arg) {
 ; CHECK-LABEL: load_i32_by_i8_nonzero_offset_bswap:
-; CHECK: ldur  w8, [x0, #1]
-; CHECK-NEXT: rev w0, w8
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldur w8, [x0, #1]
+; CHECK-NEXT:    rev w0, w8
+; CHECK-NEXT:    ret
 
   %tmp = bitcast i32* %arg to i8*
   %tmp1 = getelementptr inbounds i8, i8* %tmp, i32 4
@@ -271,9 +280,10 @@ define i32 @load_i32_by_i8_nonzero_offset_bswap(i32* %arg) {
 ; (i32) p[-1] | ((i32) p[-2] << 8) | ((i32) p[-3] << 16) | ((i32) p[-4] << 24)
 define i32 @load_i32_by_i8_neg_offset_bswap(i32* %arg) {
 ; CHECK-LABEL: load_i32_by_i8_neg_offset_bswap:
-; CHECK: ldur  w8, [x0, #-4]
-; CHECK-NEXT: rev w0, w8
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldur w8, [x0, #-4]
+; CHECK-NEXT:    rev w0, w8
+; CHECK-NEXT:    ret
 
   %tmp = bitcast i32* %arg to i8*
   %tmp1 = getelementptr inbounds i8, i8* %tmp, i32 -1
@@ -303,9 +313,10 @@ declare i16 @llvm.bswap.i16(i16)
 ; (i32) bswap(p[1]) | (i32) bswap(p[0] << 16)
 define i32 @load_i32_by_bswap_i16(i32* %arg) {
 ; CHECK-LABEL: load_i32_by_bswap_i16:
-; CHECK: ldr    w8, [x0]
-; CHECK-NEXT: rev w0, w8
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr w8, [x0]
+; CHECK-NEXT:    rev w0, w8
+; CHECK-NEXT:    ret
 
   %tmp = bitcast i32* %arg to i16*
   %tmp1 = load i16, i16* %tmp, align 4
@@ -324,8 +335,9 @@ define i32 @load_i32_by_bswap_i16(i32* %arg) {
 ; (i32) p[0] | (sext(p[1] << 16) to i32)
 define i32 @load_i32_by_sext_i16(i32* %arg) {
 ; CHECK-LABEL: load_i32_by_sext_i16:
-; CHECK: ldr   w0, [x0]
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr w0, [x0]
+; CHECK-NEXT:    ret
   %tmp = bitcast i32* %arg to i16*
   %tmp1 = load i16, i16* %tmp, align 4
   %tmp2 = zext i16 %tmp1 to i32
@@ -342,9 +354,10 @@ define i32 @load_i32_by_sext_i16(i32* %arg) {
 ; (i32) p[i] | ((i32) p[i + 1] << 8) | ((i32) p[i + 2] << 16) | ((i32) p[i + 3] << 24)
 define i32 @load_i32_by_i8_base_offset_index(i8* %arg, i32 %i) {
 ; CHECK-LABEL: load_i32_by_i8_base_offset_index:
-; CHECK: add x8, x0, w1, uxtw
-; CHECK-NEXT: ldr w0, [x8, #12]
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    add x8, x0, w1, uxtw
+; CHECK-NEXT:    ldr w0, [x8, #12]
+; CHECK-NEXT:    ret
   %tmp = add nuw nsw i32 %i, 3
   %tmp2 = add nuw nsw i32 %i, 2
   %tmp3 = add nuw nsw i32 %i, 1
@@ -379,9 +392,10 @@ define i32 @load_i32_by_i8_base_offset_index(i8* %arg, i32 %i) {
 ; (i32) p[i + 1] | ((i32) p[i + 2] << 8) | ((i32) p[i + 3] << 16) | ((i32) p[i + 4] << 24)
 define i32 @load_i32_by_i8_base_offset_index_2(i8* %arg, i32 %i) {
 ; CHECK-LABEL: load_i32_by_i8_base_offset_index_2:
-; CHECK: add x8, x0, w1, uxtw
-; CHECK-NEXT: ldur  w0, [x8, #13]
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    add x8, x0, w1, uxtw
+; CHECK-NEXT:    ldur w0, [x8, #13]
+; CHECK-NEXT:    ret
   %tmp = add nuw nsw i32 %i, 4
   %tmp2 = add nuw nsw i32 %i, 3
   %tmp3 = add nuw nsw i32 %i, 2
@@ -416,11 +430,9 @@ define i32 @load_i32_by_i8_base_offset_index_2(i8* %arg, i32 %i) {
 ; (i32) p[0] | ((i32) p[1] << 8)
 define i32 @zext_load_i32_by_i8(i32* %arg) {
 ; CHECK-LABEL: zext_load_i32_by_i8:
-; CHECK: ldrb  w8, [x0]
-; CHECK-NEXT: ldrb  w9, [x0, #1]
-; CHECK-NEXT: bfi w8, w9, #8, #8
-; CHECK-NEXT: mov  w0, w8
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldrh w0, [x0]
+; CHECK-NEXT:    ret
 
   %tmp = bitcast i32* %arg to i8*
   %tmp1 = getelementptr inbounds i8, i8* %tmp, i32 0
@@ -438,11 +450,12 @@ define i32 @zext_load_i32_by_i8(i32* %arg) {
 ; ((i32) p[0] << 8) | ((i32) p[1] << 16)
 define i32 @zext_load_i32_by_i8_shl_8(i32* %arg) {
 ; CHECK-LABEL: zext_load_i32_by_i8_shl_8:
-; CHECK: ldrb  w8, [x0]
-; CHECK-NEXT: ldrb  w9, [x0, #1]
-; CHECK-NEXT: lsl w0, w8, #8
-; CHECK-NEXT: bfi w0, w9, #16, #8
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldrb w8, [x0]
+; CHECK-NEXT:    ldrb w9, [x0, #1]
+; CHECK-NEXT:    lsl w0, w8, #8
+; CHECK-NEXT:    bfi w0, w9, #16, #8
+; CHECK-NEXT:    ret
 
   %tmp = bitcast i32* %arg to i8*
   %tmp1 = getelementptr inbounds i8, i8* %tmp, i32 0
@@ -461,11 +474,12 @@ define i32 @zext_load_i32_by_i8_shl_8(i32* %arg) {
 ; ((i32) p[0] << 16) | ((i32) p[1] << 24)
 define i32 @zext_load_i32_by_i8_shl_16(i32* %arg) {
 ; CHECK-LABEL: zext_load_i32_by_i8_shl_16:
-; CHECK: ldrb  w8, [x0]
-; CHECK-NEXT: ldrb  w9, [x0, #1]
-; CHECK-NEXT: lsl w0, w8, #16
-; CHECK-NEXT: bfi w0, w9, #24, #8
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldrb w8, [x0]
+; CHECK-NEXT:    ldrb w9, [x0, #1]
+; CHECK-NEXT:    lsl w0, w8, #16
+; CHECK-NEXT:    bfi w0, w9, #24, #8
+; CHECK-NEXT:    ret
 
   %tmp = bitcast i32* %arg to i8*
   %tmp1 = getelementptr inbounds i8, i8* %tmp, i32 0
@@ -483,11 +497,11 @@ define i32 @zext_load_i32_by_i8_shl_16(i32* %arg) {
 ; (i32) p[1] | ((i32) p[0] << 8)
 define i32 @zext_load_i32_by_i8_bswap(i32* %arg) {
 ; CHECK-LABEL: zext_load_i32_by_i8_bswap:
-; CHECK: ldrb  w8, [x0, #1]
-; CHECK-NEXT: ldrb    w9, [x0]
-; CHECK-NEXT: bfi w8, w9, #8, #8
-; CHECK-NEXT: mov  w0, w8
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldrh w8, [x0]
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    rev w0, w8
+; CHECK-NEXT:    ret
 
   %tmp = bitcast i32* %arg to i8*
   %tmp1 = getelementptr inbounds i8, i8* %tmp, i32 1
@@ -505,11 +519,12 @@ define i32 @zext_load_i32_by_i8_bswap(i32* %arg) {
 ; ((i32) p[1] << 8) | ((i32) p[0] << 16)
 define i32 @zext_load_i32_by_i8_bswap_shl_8(i32* %arg) {
 ; CHECK-LABEL: zext_load_i32_by_i8_bswap_shl_8:
-; CHECK: ldrb  w8, [x0, #1]
-; CHECK-NEXT: ldrb    w9, [x0]
-; CHECK-NEXT: lsl w0, w8, #8
-; CHECK-NEXT: bfi w0, w9, #16, #8
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldrb w8, [x0, #1]
+; CHECK-NEXT:    ldrb w9, [x0]
+; CHECK-NEXT:    lsl w0, w8, #8
+; CHECK-NEXT:    bfi w0, w9, #16, #8
+; CHECK-NEXT:    ret
 
   %tmp = bitcast i32* %arg to i8*
   %tmp1 = getelementptr inbounds i8, i8* %tmp, i32 1
@@ -528,11 +543,12 @@ define i32 @zext_load_i32_by_i8_bswap_shl_8(i32* %arg) {
 ; ((i32) p[1] << 16) | ((i32) p[0] << 24)
 define i32 @zext_load_i32_by_i8_bswap_shl_16(i32* %arg) {
 ; CHECK-LABEL: zext_load_i32_by_i8_bswap_shl_16:
-; CHECK: ldrb  w8, [x0, #1]
-; CHECK-NEXT: ldrb    w9, [x0]
-; CHECK-NEXT: lsl w0, w8, #16
-; CHECK-NEXT: bfi w0, w9, #24, #8
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldrb w8, [x0, #1]
+; CHECK-NEXT:    ldrb w9, [x0]
+; CHECK-NEXT:    lsl w0, w8, #16
+; CHECK-NEXT:    bfi w0, w9, #24, #8
+; CHECK-NEXT:    ret
 
   %tmp = bitcast i32* %arg to i8*
   %tmp1 = getelementptr inbounds i8, i8* %tmp, i32 1
diff --git a/llvm/test/CodeGen/AArch64/loopvectorize_pr33804_double.ll b/llvm/test/CodeGen/AArch64/loopvectorize_pr33804_double.ll
index c18fc0cb8..85ebd042c 100644
--- a/llvm/test/CodeGen/AArch64/loopvectorize_pr33804_double.ll
+++ b/llvm/test/CodeGen/AArch64/loopvectorize_pr33804_double.ll
@@ -110,5 +110,5 @@ for.end22.i.i:                                    ; preds = %for.body14.i.i
 
 declare i32 @__gxx_personality_v0(...)
 
-attributes #0 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="generic" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="generic" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
diff --git a/llvm/test/CodeGen/AArch64/lower-ptrmask.ll b/llvm/test/CodeGen/AArch64/lower-ptrmask.ll
new file mode 100644
index 000000000..12cce0677
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/lower-ptrmask.ll
@@ -0,0 +1,29 @@
+; RUN: llc -mtriple=arm64-apple-iphoneos -stop-after=finalize-isel %s -o - | FileCheck %s
+
+declare i8* @llvm.ptrmask.p0i8.i64(i8* , i64)
+
+; CHECK-LABEL: name: test1
+; CHECK:         %0:gpr64 = COPY $x0
+; CHECK-NEXT:    %1:gpr64sp = ANDXri %0, 8052
+; CHECK-NEXT:    $x0 = COPY %1
+; CHECK-NEXT:    RET_ReallyLR implicit $x0
+
+define i8* @test1(i8* %src) {
+  %ptr = call i8* @llvm.ptrmask.p0i8.i64(i8* %src, i64 72057594037927928)
+  ret i8* %ptr
+}
+
+declare i8* @llvm.ptrmask.p0i8.i32(i8*, i32)
+
+; CHECK-LABEL: name: test2
+; CHECK:         %0:gpr64 = COPY $x0
+; CHECK-NEXT:    %1:gpr32 = MOVi32imm 10000
+; CHECK-NEXT:    %2:gpr64 = SUBREG_TO_REG 0, killed %1, %subreg.sub_32
+; CHECK-NEXT:    %3:gpr64 = ANDXrr %0, killed %2
+; CHECK-NEXT:    $x0 = COPY %3
+; CHECK-NEXT:    RET_ReallyLR implicit $x0
+
+define i8* @test2(i8* %src) {
+  %ptr = call i8* @llvm.ptrmask.p0i8.i32(i8* %src, i32 10000)
+  ret i8* %ptr
+}
diff --git a/llvm/test/CodeGen/AArch64/machine-combiner-madd.ll b/llvm/test/CodeGen/AArch64/machine-combiner-madd.ll
index eeeafbbfa..dfd5b187e 100644
--- a/llvm/test/CodeGen/AArch64/machine-combiner-madd.ll
+++ b/llvm/test/CodeGen/AArch64/machine-combiner-madd.ll
@@ -3,8 +3,6 @@
 ; RUN: llc -mtriple=aarch64-linux-gnu -mcpu=cortex-a72 < %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64-linux-gnu -mcpu=cortex-a73 < %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64-linux-gnu -mcpu=cyclone    < %s | FileCheck %s
-; RUN: llc -mtriple=aarch64-linux-gnu -mcpu=exynos-m1  < %s | FileCheck %s
-; RUN: llc -mtriple=aarch64-linux-gnu -mcpu=exynos-m2  < %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64-linux-gnu -mcpu=exynos-m3  < %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64-linux-gnu -mcpu=kryo       < %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64-linux-gnu -mcpu=thunderx2t99 < %s | FileCheck %s
diff --git a/llvm/test/CodeGen/AArch64/machine-outliner-remarks.ll b/llvm/test/CodeGen/AArch64/machine-outliner-remarks.ll
index 19351262b..50b4ed3af 100644
--- a/llvm/test/CodeGen/AArch64/machine-outliner-remarks.ll
+++ b/llvm/test/CodeGen/AArch64/machine-outliner-remarks.ll
@@ -102,7 +102,7 @@ define void @bar() #0 !dbg !27 {
   ret void
 }
 
-attributes #0 = { noredzone nounwind ssp uwtable "no-frame-pointer-elim"="false" "target-cpu"="cyclone" }
+attributes #0 = { noredzone nounwind ssp uwtable "frame-pointer"="none" "target-cpu"="cyclone" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!3, !4, !5, !6}
diff --git a/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-cfi.ll b/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-cfi.ll
new file mode 100644
index 000000000..49e538bce
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-cfi.ll
@@ -0,0 +1,91 @@
+; RUN: llc -verify-machineinstrs -enable-machine-outliner -mtriple \
+; RUN: aarch64-arm-none-eabi %s -o - | FileCheck %s --check-prefixes CHECK,V8A
+; RUN-V83A: llc -verify-machineinstrs -enable-machine-outliner -mtriple \
+; RUN-V83A: aarch64-arm-none-eabi -mattr=+v8.3a %s -o - > %t
+; RUN-V83A: FileCheck --check-prefixes CHECK,V83A < %t %s
+
+; Function a's outlining candidate contains a sp modifying add without a
+; corresponsing sub, so we shouldn't outline it.
+define void @a() "sign-return-address"="all" "sign-return-address-key"="b_key" {
+; CHECK-LABEL:         a:                     // @a
+; CHECK:               // %bb.0:
+; CHECK-NEXT:          .cfi_b_key_frame
+; V8A-NEXT:            hint #27
+; V83A-NEXT:           pacibsp
+; V8A-NEXT, V83A-NEXT: .cfi_negate_ra_state
+  %1 = alloca i32, align 4
+  %2 = alloca i32, align 4
+  %3 = alloca i32, align 4
+  %4 = alloca i32, align 4
+  %5 = alloca i32, align 4
+  %6 = alloca i32, align 4
+  store i32 1, i32* %1, align 4
+  store i32 2, i32* %2, align 4
+  store i32 3, i32* %3, align 4
+  store i32 4, i32* %4, align 4
+  store i32 5, i32* %5, align 4
+  store i32 6, i32* %6, align 4
+; CHECK-NOT:          bl OUTLINED_FUNCTION_{{[0-9]+}}
+; V8A:                hint #31
+; V83A:               autibsp
+; CHECK-NEXT:         ret
+  ret void
+}
+
+define void @b() "sign-return-address"="all" "sign-return-address-key"="b_key" nounwind {
+; CHECK-LABEL:      b:                                     // @b
+; CHECK-NEXT:       // %bb.0:
+; V8A-NEXT:         hint #27
+; V83A-NEXT:        pacibsp
+  %1 = alloca i32, align 4
+  %2 = alloca i32, align 4
+  %3 = alloca i32, align 4
+  %4 = alloca i32, align 4
+  %5 = alloca i32, align 4
+  %6 = alloca i32, align 4
+  store i32 1, i32* %1, align 4
+  store i32 2, i32* %2, align 4
+  store i32 3, i32* %3, align 4
+  store i32 4, i32* %4, align 4
+  store i32 5, i32* %5, align 4
+  store i32 6, i32* %6, align 4
+; CHECK:                bl [[OUTLINED_FUNC:OUTLINED_FUNCTION_[0-9]+]]
+; V8A:                  hint #31
+; V83A:                 autibsp
+; V8A-NEXT, V83A-NEXT:  ret
+  ret void
+}
+
+define void @c() "sign-return-address"="all" "sign-return-address-key"="b_key" nounwind {
+; CHECK-LABEL:      c:                                     // @c
+; CHECK-NEXT:       // %bb.0:
+; V8A-NEXT:         hint #27
+; V83A-NEXT:        pacibsp
+  %1 = alloca i32, align 4
+  %2 = alloca i32, align 4
+  %3 = alloca i32, align 4
+  %4 = alloca i32, align 4
+  %5 = alloca i32, align 4
+  %6 = alloca i32, align 4
+  store i32 1, i32* %1, align 4
+  store i32 2, i32* %2, align 4
+  store i32 3, i32* %3, align 4
+  store i32 4, i32* %4, align 4
+  store i32 5, i32* %5, align 4
+  store i32 6, i32* %6, align 4
+; CHECK:                bl [[OUTLINED_FUNC]]
+; V8A:                  hint #31
+; V83A:                 autibsp
+; V8A-NEXT, V83A-NEXT:  ret
+  ret void
+}
+
+; CHECK:            [[OUTLINED_FUNC]]
+; CHECK:            // %bb.0:
+; CHECK-NEXT:           .cfi_b_key_frame
+; V8A-NEXT:             hint #27
+; V83A-NEXT:            pacibsp
+; CHECK-NEXT:           .cfi_negate_ra_state
+; V8A:                  hint #31
+; V83A:                 autibsp
+; V8A-NEXT, V83A-NEXT:  ret
diff --git a/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-diff-scope-same-key.ll b/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-diff-scope-same-key.ll
new file mode 100644
index 000000000..4efbc3af5
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-diff-scope-same-key.ll
@@ -0,0 +1,77 @@
+; RUN: llc -verify-machineinstrs -enable-machine-outliner -mtriple \
+; RUN: aarch64-arm-none-eabi %s -o - | FileCheck %s --check-prefixes CHECK,V8A
+; RUN-V83A: llc -verify-machineinstrs -enable-machine-outliner -mtriple \
+; RUN-V83A: aarch64-arm-none-eabi -mattr=+v8.3a %s -o - > %t
+; RUN-V83A: FileCheck --check-prefixes CHECK,V83A < %t %s
+
+define void @a() "sign-return-address"="all" {
+; CHECK-LABEL:      a:                                     // @a
+; V8A:              hint #25
+; V83A:             paciasp
+; CHECK-NEXT:       .cfi_negate_ra_state
+  %1 = alloca i32, align 4
+  %2 = alloca i32, align 4
+  %3 = alloca i32, align 4
+  %4 = alloca i32, align 4
+  %5 = alloca i32, align 4
+  %6 = alloca i32, align 4
+  store i32 1, i32* %1, align 4
+  store i32 2, i32* %2, align 4
+  store i32 3, i32* %3, align 4
+  store i32 4, i32* %4, align 4
+  store i32 5, i32* %5, align 4
+  store i32 6, i32* %6, align 4
+; V8A:            hint #29
+; V83A:           autiasp
+  ret void
+; CHECK:          .cfi_endproc
+}
+
+define void @b() "sign-return-address"="non-leaf" {
+; CHECK-LABE:      b:                                     // @b
+; V8A-NOT:         hint #25
+; V83A-NOT:        paciasp
+; CHECK-NOT:       .cfi_negate_ra_state
+  %1 = alloca i32, align 4
+  %2 = alloca i32, align 4
+  %3 = alloca i32, align 4
+  %4 = alloca i32, align 4
+  %5 = alloca i32, align 4
+  %6 = alloca i32, align 4
+  store i32 1, i32* %1, align 4
+  store i32 2, i32* %2, align 4
+  store i32 3, i32* %3, align 4
+  store i32 4, i32* %4, align 4
+  store i32 5, i32* %5, align 4
+  store i32 6, i32* %6, align 4
+; V8A-NOT:          hint #29
+; V83A-NOT:         autiasp
+  ret void
+; CHECK:            .cfi_endproc
+}
+
+define void @c() "sign-return-address"="all" {
+; CHECK-LABEL:         c:              // @c
+; V8A:                 hint #25
+; V83A:                paciasp
+; V8A-NEXT, V83A-NEXT: .cfi_negate_ra_state
+  %1 = alloca i32, align 4
+  %2 = alloca i32, align 4
+  %3 = alloca i32, align 4
+  %4 = alloca i32, align 4
+  %5 = alloca i32, align 4
+  %6 = alloca i32, align 4
+  store i32 1, i32* %1, align 4
+  store i32 2, i32* %2, align 4
+  store i32 3, i32* %3, align 4
+  store i32 4, i32* %4, align 4
+  store i32 5, i32* %5, align 4
+  store i32 6, i32* %6, align 4
+; V8A:            hint #29
+; V83A:           autiasp
+  ret void
+; CHECK:          .cfi_endproc
+}
+
+; CHECK-NOT:      OUTLINED_FUNCTION_{{[0-9]+}}:
+; CHECK-NOT:      // -- Begin function
diff --git a/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-non-leaf.ll b/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-non-leaf.ll
new file mode 100644
index 000000000..9b281d19d
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-non-leaf.ll
@@ -0,0 +1,79 @@
+; RUN: llc -verify-machineinstrs -enable-machine-outliner -mtriple \
+; RUN: aarch64-arm-none-eabi %s -o - | FileCheck %s --check-prefixes CHECK,V8A
+; RUN-V83A: llc -verify-machineinstrs -enable-machine-outliner -mtriple \
+; RUN-V83A: aarch64-arm-none-eabi -mattr=+v8.3a %s -o - > %t
+; RUN-V83A: FileCheck --check-prefixes CHECK,V83A < %t %s
+
+define i64 @a(i64 %x) "sign-return-address"="non-leaf" "sign-return-address-key"="b_key" {
+; CHECK-LABEL:      a:                                     // @a
+; CHECK:                .cfi_b_key_frame
+; V8A-NEXT:             hint #27
+; V83A-NEXT:            pacibsp
+; CHECK-NEXT:           .cfi_negate_ra_state
+  %1 = alloca i32, align 4
+  %2 = alloca i32, align 4
+  %3 = alloca i32, align 4
+  %4 = alloca i32, align 4
+  %5 = alloca i32, align 4
+  %6 = alloca i32, align 4
+  store i32 1, i32* %1, align 4
+  store i32 2, i32* %2, align 4
+  store i32 3, i32* %3, align 4
+  store i32 4, i32* %4, align 4
+  store i32 5, i32* %5, align 4
+  store i32 6, i32* %6, align 4
+  call void asm sideeffect "mov x30, $0", "r,~{lr}"(i64 %x) #1
+  ret i64 %x
+}
+
+define i64 @b(i64 %x) "sign-return-address"="non-leaf" "sign-return-address-key"="b_key" {
+; CHECK-LABEL:      b:                                     // @b
+; CHECK:                .cfi_b_key_frame
+; V8A-NEXT:             hint #27
+; V83A-NEXT:            pacibsp
+; CHECK-NEXT:           .cfi_negate_ra_state
+  %1 = alloca i32, align 4
+  %2 = alloca i32, align 4
+  %3 = alloca i32, align 4
+  %4 = alloca i32, align 4
+  %5 = alloca i32, align 4
+  %6 = alloca i32, align 4
+  store i32 1, i32* %1, align 4
+  store i32 2, i32* %2, align 4
+  store i32 3, i32* %3, align 4
+  store i32 4, i32* %4, align 4
+  store i32 5, i32* %5, align 4
+  store i32 6, i32* %6, align 4
+  call void asm sideeffect "mov x30, $0", "r,~{lr}"(i64 %x) #1
+  ret i64 %x
+}
+
+define i64 @c(i64 %x) "sign-return-address"="non-leaf" "sign-return-address-key"="b_key" {
+; CHECK-LABEL:      c:                                     // @c
+; CHECK:                .cfi_b_key_frame
+; V8A-NEXT:             hint #27
+; V83A-NEXT:            pacibsp
+; CHECK-NEXT:           .cfi_negate_ra_state
+  %1 = alloca i32, align 4
+  %2 = alloca i32, align 4
+  %3 = alloca i32, align 4
+  %4 = alloca i32, align 4
+  %5 = alloca i32, align 4
+  %6 = alloca i32, align 4
+  store i32 1, i32* %1, align 4
+  store i32 2, i32* %2, align 4
+  store i32 3, i32* %3, align 4
+  store i32 4, i32* %4, align 4
+  store i32 5, i32* %5, align 4
+  store i32 6, i32* %6, align 4
+  call void asm sideeffect "mov x30, $0", "r,~{lr}"(i64 %x) #1
+  ret i64 %x
+}
+
+; Outlined function is leaf-function => don't sign it
+; CHECK-LABEL:      OUTLINED_FUNCTION_0:
+; CHECK-NOT:            .cfi_b_key_frame
+; CHECK-NOT:            paci{{[a,b]}}sp
+; CHECK-NOT:            hint #2{{[5,7]}}
+; CHECK-NOT:            .cfi_negate_ra_state
+; CHECK-NOT:            auti{{[a,b]}}sp
diff --git a/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-same-scope-diff-key.ll b/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-same-scope-diff-key.ll
new file mode 100644
index 000000000..513843033
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-same-scope-diff-key.ll
@@ -0,0 +1,78 @@
+; RUN: llc -verify-machineinstrs -enable-machine-outliner -mtriple \
+; RUN: aarch64-arm-none-eabi %s -o - | FileCheck %s --check-prefixes CHECK,V8A
+; RUN-V83A: llc -verify-machineinstrs -enable-machine-outliner -mtriple \
+; RUN-V83A: aarch64-arm-none-eabi -mattr=+v8.3a %s -o - > %t
+; RUN-V83A: FileCheck --check-prefixes CHECK,V83A < %t %s
+
+define void @a() "sign-return-address"="all" {
+; CHECK-LABEL:      a:                                     // @a
+; V8A:              hint #25
+; V83A:             paciasp
+; CHECK-NEXT:       .cfi_negate_ra_state
+  %1 = alloca i32, align 4
+  %2 = alloca i32, align 4
+  %3 = alloca i32, align 4
+  %4 = alloca i32, align 4
+  %5 = alloca i32, align 4
+  %6 = alloca i32, align 4
+  store i32 1, i32* %1, align 4
+  store i32 2, i32* %2, align 4
+  store i32 3, i32* %3, align 4
+  store i32 4, i32* %4, align 4
+  store i32 5, i32* %5, align 4
+  store i32 6, i32* %6, align 4
+; V8A:            hint #29
+; V83A:           autiasp
+  ret void
+; CHECK:          .cfi_endproc
+}
+
+define void @b() "sign-return-address"="all" "sign-return-address-key"="b_key" {
+; CHECK-LABEL:      b:                                     // @b
+; CHECK:            .cfi_b_key_frame
+; V8A-NEXT:         hint #27
+; V83A-NEXT:        pacibsp
+; CHECK-NEXT:       .cfi_negate_ra_state
+  %1 = alloca i32, align 4
+  %2 = alloca i32, align 4
+  %3 = alloca i32, align 4
+  %4 = alloca i32, align 4
+  %5 = alloca i32, align 4
+  %6 = alloca i32, align 4
+  store i32 1, i32* %1, align 4
+  store i32 2, i32* %2, align 4
+  store i32 3, i32* %3, align 4
+  store i32 4, i32* %4, align 4
+  store i32 5, i32* %5, align 4
+  store i32 6, i32* %6, align 4
+; V8A-NOT:          hint #29
+; V83A-NOT:         autiasp
+  ret void
+; CHECK:            .cfi_endproc
+}
+
+define void @c() "sign-return-address"="all" {
+; CHECK-LABEL:      c:                                     // @c
+; V8A:              hint #25
+; V83A:             paciasp
+; CHECK-NEXT:       .cfi_negate_ra_state
+  %1 = alloca i32, align 4
+  %2 = alloca i32, align 4
+  %3 = alloca i32, align 4
+  %4 = alloca i32, align 4
+  %5 = alloca i32, align 4
+  %6 = alloca i32, align 4
+  store i32 1, i32* %1, align 4
+  store i32 2, i32* %2, align 4
+  store i32 3, i32* %3, align 4
+  store i32 4, i32* %4, align 4
+  store i32 5, i32* %5, align 4
+  store i32 6, i32* %6, align 4
+; V8A:            hint #29
+; V83A:           autiasp
+  ret void
+; CHECK:          .cfi_endproc
+}
+
+; CHECK-NOT:      OUTLINED_FUNCTION_0:
+; CHECK-NOT:      // -- Begin function
diff --git a/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-same-scope-same-key-a.ll b/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-same-scope-same-key-a.ll
new file mode 100644
index 000000000..e0b42bf54
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-same-scope-same-key-a.ll
@@ -0,0 +1,75 @@
+; RUN: llc -verify-machineinstrs -enable-machine-outliner -mtriple \
+; RUN: aarch64-arm-none-eabi %s -o - | FileCheck %s --check-prefixes CHECK,V8A
+; RUN-V83A: llc -verify-machineinstrs -enable-machine-outliner -mtriple \
+; RUN-V83A: aarch64-arm-none-eabi -mattr=+v8.3a %s -o - > %t
+; RUN-V83A: FileCheck --check-prefixes CHECK,V83A < %t %s
+
+define void @a() "sign-return-address"="all" "sign-return-address-key"="a_key" nounwind {
+; CHECK-LABEL:      a:                                     // @a
+; V8A:              hint #25
+; V83A:             paciasp
+  %1 = alloca i32, align 4
+  %2 = alloca i32, align 4
+  %3 = alloca i32, align 4
+  %4 = alloca i32, align 4
+  %5 = alloca i32, align 4
+  %6 = alloca i32, align 4
+  store i32 1, i32* %1, align 4
+  store i32 2, i32* %2, align 4
+  store i32 3, i32* %3, align 4
+  store i32 4, i32* %4, align 4
+  store i32 5, i32* %5, align 4
+  store i32 6, i32* %6, align 4
+; V8A:              hint #29
+; V83A:             autiasp
+  ret void
+}
+
+define void @b() "sign-return-address"="all" nounwind {
+; CHECK-LABEL:      b:                                     // @b
+; V8A:              hint #25
+; V83A:             paciasp
+  %1 = alloca i32, align 4
+  %2 = alloca i32, align 4
+  %3 = alloca i32, align 4
+  %4 = alloca i32, align 4
+  %5 = alloca i32, align 4
+  %6 = alloca i32, align 4
+  store i32 1, i32* %1, align 4
+  store i32 2, i32* %2, align 4
+  store i32 3, i32* %3, align 4
+  store i32 4, i32* %4, align 4
+  store i32 5, i32* %5, align 4
+  store i32 6, i32* %6, align 4
+; V8A:              hint #29
+; V83A:             autiasp
+  ret void
+}
+
+define void @c() "sign-return-address"="all" nounwind {
+; CHECK-LABEL:      c:                                     // @c
+; V8A:              hint #25
+; V83A:             paciasp
+  %1 = alloca i32, align 4
+  %2 = alloca i32, align 4
+  %3 = alloca i32, align 4
+  %4 = alloca i32, align 4
+  %5 = alloca i32, align 4
+  %6 = alloca i32, align 4
+  store i32 1, i32* %1, align 4
+  store i32 2, i32* %2, align 4
+  store i32 3, i32* %3, align 4
+  store i32 4, i32* %4, align 4
+  store i32 5, i32* %5, align 4
+  store i32 6, i32* %6, align 4
+; V8A:              hint #29
+; V83A:             autiasp
+  ret void
+}
+
+; CHECK-LABEL:      OUTLINED_FUNCTION_0:
+; V8A:                hint #25
+; V83A:               paciasp
+; V8A:                hint #29
+; V83A:               autiasp
+; CHECK-NEXT:         ret
diff --git a/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-same-scope-same-key-b.ll b/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-same-scope-same-key-b.ll
new file mode 100644
index 000000000..de32f059b
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-same-scope-same-key-b.ll
@@ -0,0 +1,81 @@
+; RUN: llc -verify-machineinstrs -enable-machine-outliner -mtriple \
+; RUN: aarch64-arm-none-eabi %s -o - | FileCheck %s --check-prefixes CHECK,V8A
+; RUN-V83A: llc -verify-machineinstrs -enable-machine-outliner -mtriple \
+; RUN-V83A: aarch64-arm-none-eabi -mattr=+v8.3a %s -o - > %t
+; RUN-V83A: FileCheck --check-prefixes CHECK,V83A < %t %s
+
+define void @a() "sign-return-address"="all" "sign-return-address-key"="b_key" nounwind {
+; CHECK-LABEL:      a:                                     // @a
+; CHECK-NEXT:       // %bb.0:
+; V8A:              hint #27
+; V83A:             pacibsp
+  %1 = alloca i32, align 4
+  %2 = alloca i32, align 4
+  %3 = alloca i32, align 4
+  %4 = alloca i32, align 4
+  %5 = alloca i32, align 4
+  %6 = alloca i32, align 4
+  store i32 1, i32* %1, align 4
+  store i32 2, i32* %2, align 4
+  store i32 3, i32* %3, align 4
+  store i32 4, i32* %4, align 4
+  store i32 5, i32* %5, align 4
+  store i32 6, i32* %6, align 4
+; V8A:              hint #31
+; V83A:             autibsp
+  ret void
+}
+
+define void @b() "sign-return-address"="all" "sign-return-address-key"="b_key" nounwind {
+; CHECK-LABEL:      b:                                     // @b
+; CHECK-NEXT:       // %bb.0:
+; V8A:              hint #27
+; V83A:             pacibsp
+  %1 = alloca i32, align 4
+  %2 = alloca i32, align 4
+  %3 = alloca i32, align 4
+  %4 = alloca i32, align 4
+  %5 = alloca i32, align 4
+  %6 = alloca i32, align 4
+  store i32 1, i32* %1, align 4
+  store i32 2, i32* %2, align 4
+  store i32 3, i32* %3, align 4
+  store i32 4, i32* %4, align 4
+  store i32 5, i32* %5, align 4
+  store i32 6, i32* %6, align 4
+; V8A:              hint #31
+; V83A:             autibsp
+  ret void
+}
+
+define void @c() "sign-return-address"="all" "sign-return-address-key"="b_key" nounwind {
+; CHECK-LABEL:      c:                                     // @c
+; CHECK-NEXT:       // %bb.0:
+; V8A:              hint #27
+; V83A:             pacibsp
+  %1 = alloca i32, align 4
+  %2 = alloca i32, align 4
+  %3 = alloca i32, align 4
+  %4 = alloca i32, align 4
+  %5 = alloca i32, align 4
+  %6 = alloca i32, align 4
+  store i32 1, i32* %1, align 4
+  store i32 2, i32* %2, align 4
+  store i32 3, i32* %3, align 4
+  store i32 4, i32* %4, align 4
+  store i32 5, i32* %5, align 4
+  store i32 6, i32* %6, align 4
+; V8A:              hint #31
+; V83A:             autibsp
+  ret void
+}
+
+; CHECK-LABEL:      OUTLINED_FUNCTION_0:
+; CHECK:            // %bb.0:
+; CHECK-NEXT:           .cfi_b_key_frame
+; V8A-NEXT:             hint #27
+; V83A-NEXT:            pacibsp
+; CHECK-NEXT:           .cfi_negate_ra_state
+; V8A:                  hint #31
+; V83A:                 autibsp
+; CHECK-NEXT:           ret
diff --git a/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-sp-mod.ll b/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-sp-mod.ll
new file mode 100644
index 000000000..8fd152869
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-sp-mod.ll
@@ -0,0 +1,44 @@
+; RUN: llc -mtriple aarch64-arm-none-eabi -enable-machine-outliner \
+; RUN:  -verify-machineinstrs %s -o - | FileCheck %s
+
+@v = common dso_local global i32* null, align 8
+
+; CHECK-LABEL:  foo:                                    // @foo
+; CHECK-NEXT:   // %bb.0:                               // %entry
+; CHECK-NEXT:       paciasp
+; CHECK-NOT:        OUTLINED_FUNCTION_
+; CHECK:            retaa
+define dso_local void @foo(i32 %x) #0 {
+entry:
+  %0 = zext i32 %x to i64
+  %vla = alloca i32, i64 %0, align 4
+  store volatile i32* %vla, i32** @v, align 8
+  store volatile i32* %vla, i32** @v, align 8
+  store volatile i32* %vla, i32** @v, align 8
+  store volatile i32* %vla, i32** @v, align 8
+  store volatile i32* %vla, i32** @v, align 8
+  store volatile i32* %vla, i32** @v, align 8
+  ret void
+}
+
+; CHECK-LABEL:  bar:                                    // @bar
+; CHECK-NEXT:   // %bb.0:                               // %entry
+; CHECK-NEXT:       paciasp
+; CHECK-NOT:        OUTLINED_FUNCTION_
+; CHECK:            retaa
+define dso_local void @bar(i32 %x) #0 {
+entry:
+  %0 = zext i32 %x to i64
+  %vla = alloca i32, i64 %0, align 4
+  store volatile i32* null, i32** @v, align 8
+  store volatile i32* %vla, i32** @v, align 8
+  store volatile i32* %vla, i32** @v, align 8
+  store volatile i32* %vla, i32** @v, align 8
+  store volatile i32* %vla, i32** @v, align 8
+  store volatile i32* %vla, i32** @v, align 8
+  ret void
+}
+
+attributes #0 = { nounwind "target-features"="+v8.3a" "frame-pointer"="all" "sign-return-address"="all" "sign-return-address-key"="a_key" }
+
+; CHECK-NOT:  OUTLINED_FUNCTION_
diff --git a/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-subtarget.ll b/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-subtarget.ll
new file mode 100644
index 000000000..1459b62ee
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-subtarget.ll
@@ -0,0 +1,87 @@
+; RUN: llc -verify-machineinstrs -enable-machine-outliner -mtriple \
+; RUN: aarch64-arm-linux-gnu %s -o - | FileCheck %s
+
+; Check that functions that should sign their return addresses don't get
+; outlined if not all of the function either support v8.3a features or all of
+; the functions don't!!
+
+define void @a() #0 {
+; CHECK-LABEL:      a:                                     // @a
+; CHECK:            // %bb.0:
+; CHECK-NEXT:               .cfi_b_key_frame
+; CHECK-NEXT:               pacibsp
+; CHECK-NEXT:               .cfi_negate_ra_state
+; CHECK-NOT:                OUTLINED_FUNCTION_
+  %1 = alloca i32, align 4
+  %2 = alloca i32, align 4
+  %3 = alloca i32, align 4
+  %4 = alloca i32, align 4
+  %5 = alloca i32, align 4
+  %6 = alloca i32, align 4
+  store i32 1, i32* %1, align 4
+  store i32 2, i32* %2, align 4
+  store i32 3, i32* %3, align 4
+  store i32 4, i32* %4, align 4
+  store i32 5, i32* %5, align 4
+  store i32 6, i32* %6, align 4
+; CHECK:                  retab
+; CHECK-NOT:              auti[a,b]sp
+  ret void
+}
+
+define void @b() #0 {
+; CHECK-LABEL:      b:                                     // @b
+; CHECK:            // %bb.0:
+; CHECK-NEXT:               .cfi_b_key_frame
+; CHECK-NEXT:               pacibsp
+; CHECK-NEXT:               .cfi_negate_ra_state
+; CHECK-NOT:                OUTLINED_FUNCTION_
+  %1 = alloca i32, align 4
+  %2 = alloca i32, align 4
+  %3 = alloca i32, align 4
+  %4 = alloca i32, align 4
+  %5 = alloca i32, align 4
+  %6 = alloca i32, align 4
+  store i32 1, i32* %1, align 4
+  store i32 2, i32* %2, align 4
+  store i32 3, i32* %3, align 4
+  store i32 4, i32* %4, align 4
+  store i32 5, i32* %5, align 4
+  store i32 6, i32* %6, align 4
+; CHECK:                  retab
+; CHECK-NOT:              auti[a,b]sp
+  ret void
+}
+
+define void @c() #1 {
+; CHECK-LABEL:      c:                                     // @c
+; CHECK:            // %bb.0:
+; CHECK-NEXT:               .cfi_b_key_frame
+; CHECK-NEXT:               hint #27
+; CHECK-NEXT:               .cfi_negate_ra_state
+; CHECK-NOT:                OUTLINED_FUNCTION_
+  %1 = alloca i32, align 4
+  %2 = alloca i32, align 4
+  %3 = alloca i32, align 4
+  %4 = alloca i32, align 4
+  %5 = alloca i32, align 4
+  %6 = alloca i32, align 4
+  store i32 1, i32* %1, align 4
+  store i32 2, i32* %2, align 4
+  store i32 3, i32* %3, align 4
+  store i32 4, i32* %4, align 4
+  store i32 5, i32* %5, align 4
+  store i32 6, i32* %6, align 4
+; CHECK:                  hint #31
+; CHECK-NOT:              ret{{[a,b]}}
+  ret void
+}
+
+attributes #0 = { "sign-return-address"="all"
+                  "sign-return-address-key"="b_key"
+                  "target-features"="+v8.3a" }
+
+attributes #1 = { "sign-return-address"="all"
+                  "sign-return-address-key"="b_key" }
+
+; CHECK-NOT:                OUTLINED_FUNCTION_
diff --git a/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-thunk.ll b/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-thunk.ll
new file mode 100644
index 000000000..aea36d969
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-thunk.ll
@@ -0,0 +1,76 @@
+; RUN: llc -mtriple aarch64-arm-linux-gnu --enable-machine-outliner \
+; RUN: -verify-machineinstrs %s -o - | FileCheck --check-prefixes CHECK,V8A %s
+; RUN-V83A: llc -mtriple aarch64-arm-none-eabi -enable-machine-outliner \
+; RUN-V83A: -verify-machineinstrs -mattr=+v8.3a %s -o - > %t
+; RUN-V83A: FileCheck --check-prefixes CHECK,V83A < %t %s
+
+declare i32 @thunk_called_fn(i32, i32, i32, i32)
+
+define i32 @a() #0 {
+; CHECK-LABEL:  a:                                      // @a
+; CHECK:        // %bb.0:                               // %entry
+; V8A-NEXT:         hint #25
+; V83A-NEXT:        paciasp
+; V8A:              hint #29
+; V83A:             autiasp
+; CHECK-NEXT:       ret
+entry:
+  %call = tail call i32 @thunk_called_fn(i32 1, i32 2, i32 3, i32 4)
+  %cx = add i32 %call, 8
+  ret i32 %cx
+}
+
+define i32 @b() #0 {
+; CHECK-LABEL:  b:                                      // @b
+; CHECK:        // %bb.0:                               // %entry
+; V8A-NEXT:         hint #25
+; V83A-NEXT:        paciasp
+; CHECK-NEXT:       .cfi_negate_ra_state
+; V8A:              hint #29
+; V83A:             autiasp
+; CHECK-NEXT:       ret
+entry:
+  %call = tail call i32 @thunk_called_fn(i32 1, i32 2, i32 3, i32 4)
+  %cx = add i32 %call, 88
+  ret i32 %cx
+}
+
+define hidden i32 @c(i32 (i32, i32, i32, i32)* %fptr) #0 {
+; CHECK-LABEL:  c:                                      // @c
+; CHECK:        // %bb.0:                               // %entry
+; V8A-NEXT:         hint #25
+; V83A-NEXT:        paciasp
+; CHECK-NEXT:       .cfi_negate_ra_state
+; V8A:              hint #29
+; V83A:             autiasp
+; CHECK-NEXT:       ret
+entry:
+  %call = tail call i32 %fptr(i32 1, i32 2, i32 3, i32 4)
+  %add = add nsw i32 %call, 8
+  ret i32 %add
+}
+
+define hidden i32 @d(i32 (i32, i32, i32, i32)* %fptr) #0 {
+; CHECK-LABEL:  d:                                      // @d
+; CHECK:        // %bb.0:                               // %entry
+; V8A-NEXT:         hint #25
+; V83A-NEXT:        paciasp
+; CHECK-NEXT:       .cfi_negate_ra_state
+; V8A:              hint #29
+; V83A:             autiasp
+; CHECK-NEXT:       ret
+entry:
+  %call = tail call i32 %fptr(i32 1, i32 2, i32 3, i32 4)
+  %add = add nsw i32 %call, 88
+  ret i32 %add
+}
+
+attributes #0 = { "sign-return-address"="non-leaf" }
+
+; CHECK-NOT:        [[OUTLINED_FUNCTION_{{.*}}]]
+; CHECK-NOT:         .cfi_b_key_frame
+; CHECK-NOT:         paci{{[a,b]}}sp
+; CHECK-NOT:         hint #2{{[5,7]}}
+; CHECK-NOT:         .cfi_negate_ra_state
+; CHECK-NOT:         auti{{[a,b]}}sp
+; CHECK-NOT:         hint #{{[29,31]}}
diff --git a/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-v8-3.ll b/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-v8-3.ll
new file mode 100644
index 000000000..5d728aa37
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-v8-3.ll
@@ -0,0 +1,83 @@
+; RUN: llc -verify-machineinstrs -enable-machine-outliner -mtriple \
+; RUN: aarch64-arm-linux-gnu %s -o - | FileCheck %s
+
+; Check that outlined functions use the dedicated RETAA/RETAB instructions
+; to sign their return address if available.
+
+define void @a() #0 {
+; CHECK-LABEL:      a:                                     // @a
+; CHECK:            // %bb.0:
+; CHECK-NEXT:               pacibsp
+; CHECK:                    bl [[OUTLINED_FUNC:OUTLINED_FUNCTION_[0-9]+]]
+  %1 = alloca i32, align 4
+  %2 = alloca i32, align 4
+  %3 = alloca i32, align 4
+  %4 = alloca i32, align 4
+  %5 = alloca i32, align 4
+  %6 = alloca i32, align 4
+  store i32 1, i32* %1, align 4
+  store i32 2, i32* %2, align 4
+  store i32 3, i32* %3, align 4
+  store i32 4, i32* %4, align 4
+  store i32 5, i32* %5, align 4
+  store i32 6, i32* %6, align 4
+; CHECK:                  retab
+; CHECK-NOT:              auti[a,b]sp
+  ret void
+}
+
+define void @b() #0 {
+; CHECK-LABEL:      b:                                     // @b
+; CHECK:            // %bb.0:
+; CHECK-NEXT:               pacibsp
+; CHECK:                    bl OUTLINED_FUNC
+  %1 = alloca i32, align 4
+  %2 = alloca i32, align 4
+  %3 = alloca i32, align 4
+  %4 = alloca i32, align 4
+  %5 = alloca i32, align 4
+  %6 = alloca i32, align 4
+  store i32 1, i32* %1, align 4
+  store i32 2, i32* %2, align 4
+  store i32 3, i32* %3, align 4
+  store i32 4, i32* %4, align 4
+  store i32 5, i32* %5, align 4
+  store i32 6, i32* %6, align 4
+; CHECK:                  retab
+; CHECK-NOT:              auti[a,b]sp
+  ret void
+}
+
+define void @c() #0 {
+; CHECK-LABEL:      c:                                     // @c
+; CHECK:            // %bb.0:
+; CHECK-NEXT:               pacibsp
+; CHECK:                    bl OUTLINED_FUNC
+  %1 = alloca i32, align 4
+  %2 = alloca i32, align 4
+  %3 = alloca i32, align 4
+  %4 = alloca i32, align 4
+  %5 = alloca i32, align 4
+  %6 = alloca i32, align 4
+  store i32 1, i32* %1, align 4
+  store i32 2, i32* %2, align 4
+  store i32 3, i32* %3, align 4
+  store i32 4, i32* %4, align 4
+  store i32 5, i32* %5, align 4
+  store i32 6, i32* %6, align 4
+; CHECK:                  retab
+; CHECK-NOT:              auti[a,b]sp
+  ret void
+}
+
+attributes #0 = { "sign-return-address"="all"
+                  "sign-return-address-key"="b_key"
+                  "target-features"="+v8.3a"
+                  nounwind }
+
+; CHECK:            OUTLINED_FUNC
+; CHECK:            // %bb.0:
+; CHECK-NEXT:               .cfi_b_key_frame
+; CHECK-NEXT:               pacibsp
+; CHECK:                    retab
+; CHECK-NOT:                auti[a,b]sp
diff --git a/llvm/test/CodeGen/AArch64/machine-outliner.ll b/llvm/test/CodeGen/AArch64/machine-outliner.ll
index 15afdd43d..6c76f894c 100644
--- a/llvm/test/CodeGen/AArch64/machine-outliner.ll
+++ b/llvm/test/CodeGen/AArch64/machine-outliner.ll
@@ -1,5 +1,5 @@
-; RUN: llc -verify-machineinstrs -enable-machine-outliner -mtriple=aarch64-apple-darwin < %s | FileCheck %s
-; RUN: llc -verify-machineinstrs -enable-machine-outliner -mtriple=aarch64-apple-darwin -mcpu=cortex-a53 -enable-misched=false < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -enable-machine-outliner -aarch64-load-store-renaming=true -mtriple=aarch64-apple-darwin < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -enable-machine-outliner -aarch64-load-store-renaming=true -mtriple=aarch64-apple-darwin -mcpu=cortex-a53 -enable-misched=false < %s | FileCheck %s
 ; RUN: llc -verify-machineinstrs -enable-machine-outliner -enable-linkonceodr-outlining -mtriple=aarch64-apple-darwin < %s | FileCheck %s -check-prefix=ODR
 ; RUN: llc -verify-machineinstrs -enable-machine-outliner -mtriple=aarch64-apple-darwin -stop-after=machine-outliner < %s | FileCheck %s -check-prefix=TARGET_FEATURES
 
@@ -91,19 +91,16 @@ define void @dog() #0 {
 ; ODR: [[OUTLINED]]:
 ; CHECK: .p2align 2
 ; CHECK-NEXT: [[OUTLINED]]:
-; CHECK: mov     w8, #1
-; CHECK-NEXT: str     w8, [sp, #28]
-; CHECK-NEXT: mov     w8, #2
-; CHECK-NEXT: str     w8, [sp, #24]
-; CHECK-NEXT: mov     w8, #3
-; CHECK-NEXT: str     w8, [sp, #20]
-; CHECK-NEXT: mov     w8, #4
-; CHECK-NEXT: str     w8, [sp, #16]
-; CHECK-NEXT: mov     w8, #5
-; CHECK-NEXT: str     w8, [sp, #12]
-; CHECK-NEXT: mov     w8, #6
-; CHECK-NEXT: str     w8, [sp, #8]
-; CHECK-NEXT: add     sp, sp, #32
-; CHECK-NEXT: ret
+; CHECK:      mov     w9, #1
+; CHECK-DAG: mov     w8, #2
+; CHECK-DAG: stp     w8, w9, [sp, #24]
+; CHECK-DAG: mov     w9, #3
+; CHECK-DAG: mov     w8, #4
+; CHECK-DAG: stp     w8, w9, [sp, #16]
+; CHECK-DAG: mov     w9, #5
+; CHECK-DAG: mov     w8, #6
+; CHECK-DAG: stp     w8, w9, [sp, #8]
+; CHECK-DAG: add     sp, sp, #32
+; CHECK-DAG: ret
 
 attributes #0 = { noredzone "target-cpu"="cyclone" "target-features"="+sse" }
diff --git a/llvm/test/CodeGen/AArch64/macro-fusion.ll b/llvm/test/CodeGen/AArch64/macro-fusion.ll
new file mode 100644
index 000000000..b9a263fe2
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/macro-fusion.ll
@@ -0,0 +1,20 @@
+; REQUIRES: asserts
+; RUN: llc < %s -mtriple=aarch64-linux-gnu -mattr=+fuse-arith-logic -verify-misched -debug-only=machine-scheduler 2>&1 > /dev/null | FileCheck %s
+
+; Verify that, the macro-fusion creates the necessary dependencies between SUs and
+; only 2 SU's are fused at most.
+define signext i32 @test(i32 signext %a, i32 signext %b, i32 signext %c, i32 signext %d) {
+entry:
+; CHECK: ********** MI Scheduling **********
+; CHECK-LABEL: %bb.0 entry 
+; CHECK: Macro fuse: SU([[SU4:[0-9]+]]) - SU([[SU5:[0-9]+]])
+; CHECK: Bind SU([[SU1:[0-9]+]]) - SU([[SU4]])
+; CHECK-NOT: Macro fuse:
+; CHECK: SU([[SU1]]):   %{{[0-9]+}}:gpr32 = COPY $w2
+; CHECK: SU([[SU4]]):   %{{[0-9]+}}:gpr32 = nsw ADDWrr
+; CHECK: SU([[SU5]]):   %{{[0-9]+}}:gpr32 = nsw ADDWrr
+  %add = add nsw i32 %b, %a
+  %add1 = add nsw i32 %add, %c
+  %sub = sub nsw i32 %add1, %d
+  ret i32 %sub
+}
diff --git a/llvm/test/CodeGen/AArch64/max-jump-table.ll b/llvm/test/CodeGen/AArch64/max-jump-table.ll
index 431db2743..2d0dbbf24 100644
--- a/llvm/test/CodeGen/AArch64/max-jump-table.ll
+++ b/llvm/test/CodeGen/AArch64/max-jump-table.ll
@@ -2,7 +2,6 @@
 ; RUN: llc %s -O2 -print-machineinstrs -mtriple=aarch64-linux-gnu -jump-table-density=40 -max-jump-table-size=4  -o /dev/null 2> %t; FileCheck %s --check-prefixes=CHECK,CHECK4  < %t
 ; RUN: llc %s -O2 -print-machineinstrs -mtriple=aarch64-linux-gnu -jump-table-density=40 -max-jump-table-size=8  -o /dev/null 2> %t; FileCheck %s --check-prefixes=CHECK,CHECK8  < %t
 ; RUN: llc %s -O2 -print-machineinstrs -mtriple=aarch64-linux-gnu -jump-table-density=40 -max-jump-table-size=16 -o /dev/null 2> %t; FileCheck %s --check-prefixes=CHECK,CHECK16 < %t
-; RUN: llc %s -O2 -print-machineinstrs -mtriple=aarch64-linux-gnu -jump-table-density=40 -mcpu=exynos-m1         -o /dev/null 2> %t; FileCheck %s --check-prefixes=CHECK,CHECKM1 < %t
 ; RUN: llc %s -O2 -print-machineinstrs -mtriple=aarch64-linux-gnu -jump-table-density=40 -mcpu=exynos-m3         -o /dev/null 2> %t; FileCheck %s --check-prefixes=CHECK,CHECKM3 < %t
 
 declare void @ext(i32, i32)
@@ -42,9 +41,6 @@ entry:
 ; CHECK8-NOT:   %jump-table.2:
 ; CHECK16-NEXT: %jump-table.0: %bb.2 %bb.3 %bb.4 %bb.5 %bb.6 %bb.7 %bb.8 %bb.9 %bb.10 %bb.11 %bb.12 %bb.13 %bb.14 %bb.15 %bb.16 %bb.17
 ; CHECK16-NOT:  %jump-table.1:
-; CHECKM1-NEXT: %jump-table.0: %bb.2 %bb.3 %bb.4 %bb.5 %bb.6 %bb.7 %bb.8 %bb.9
-; CHECKM1-NEXT: %jump-table.1: %bb.10 %bb.11 %bb.12 %bb.13 %bb.14 %bb.15 %bb.16 %bb.17
-; CHECKM1-NOT:  %jump-table.2:
 ; CHECKM3-NEXT: %jump-table.0: %bb.1 %bb.2 %bb.3 %bb.4 %bb.5 %bb.6 %bb.7 %bb.8 %bb.9 %bb.10 %bb.11 %bb.12 %bb.13 %bb.14 %bb.15 %bb.16 %bb.17
 ; CHECKM3-NOT:  %jump-table.1:
 
@@ -90,8 +86,6 @@ entry:
 ; CHECK8-NOT:   %jump-table.1:
 ; CHECK16-NEXT: %jump-table.0: %bb.1 %bb.2 %bb.3 %bb.4 %bb.7 %bb.7 %bb.7 %bb.7 %bb.7 %bb.7 %bb.7 %bb.7 %bb.7 %bb.5 %bb.6{{$}}
 ; CHECK16-NOT:  %jump-table.1:
-; CHECKM1-NEXT: %jump-table.0: %bb.1 %bb.2 %bb.3 %bb.4{{$}}
-; CHECKM1-NOT:  %jump-table.1:
 ; CHECKM3-NEXT: %jump-table.0: %bb.1 %bb.2 %bb.3 %bb.4 %bb.7 %bb.7 %bb.7 %bb.7 %bb.7 %bb.7 %bb.7 %bb.7 %bb.7 %bb.5 %bb.6{{$}}
 ; CHECKM3-NOT:  %jump-table.1:
 ; CHECK-DAG: End machine code for function jt2.
@@ -137,9 +131,6 @@ entry:
 ; CHECK16-NEXT: %jump-table.0: %bb.1 %bb.2 %bb.3 %bb.4 %bb.13 %bb.13 %bb.13 %bb.13 %bb.13 %bb.13 %bb.13 %bb.13 %bb.13 %bb.5 %bb.6 %bb.7
 ; CHECK16-NEXT: %jump-table.1: %bb.8 %bb.13 %bb.9 %bb.10 %bb.13 %bb.11 %bb.12
 ; CHECK16-NOT:  %jump-table.2:
-; CHECKM1-NEXT: %jump-table.0: %bb.1 %bb.2 %bb.3 %bb.4
-; CHECKM1-NEXT: %jump-table.1: %bb.5 %bb.6 %bb.7 %bb.8 %bb.13 %bb.9 %bb.10
-; CHECKM1-NOT:  %jump-table.2:
 ; CHECKM3-NEXT: %jump-table.0: %bb.1 %bb.2 %bb.3 %bb.4 %bb.13 %bb.13 %bb.13 %bb.13 %bb.13 %bb.13 %bb.13 %bb.13 %bb.13 %bb.5 %bb.6 %bb.7 %bb.8 %bb.13 %bb.9 %bb.10
 ; CHECKM3-NOT:  %jump-table.1:
 ; CHECK-DAG: End machine code for function jt3.
@@ -192,9 +183,6 @@ entry:
 ; CHECK16-NEXT: %jump-table.0: %bb.1 %bb.2 %bb.3 %bb.4 %bb.13 %bb.13 %bb.13 %bb.13 %bb.13 %bb.13 %bb.13 %bb.13 %bb.13 %bb.5 %bb.6 %bb.7
 ; CHECK16-NEXT: %jump-table.1: %bb.8 %bb.13 %bb.9 %bb.10 %bb.13 %bb.11 %bb.12
 ; CHECK16-NOT:  %jump-table.2:
-; CHECKM1-NEXT: %jump-table.0: %bb.1 %bb.2 %bb.3 %bb.4
-; CHECKM1-NEXT: %jump-table.1: %bb.5 %bb.6 %bb.7 %bb.8 %bb.13 %bb.9 %bb.10
-; CHECKM1-NOT:  %jump-table.2:
 ; CHECKM3-NEXT: %jump-table.0: %bb.1 %bb.2 %bb.3 %bb.4 %bb.13 %bb.13 %bb.13 %bb.13 %bb.13 %bb.13 %bb.13 %bb.13 %bb.13 %bb.5 %bb.6 %bb.7 %bb.8 %bb.13 %bb.9 %bb.10
 ; CHECKM3-NOT:  %jump-table.1:
 ; CHECK-DAG: End machine code for function jt4.
@@ -215,3 +203,136 @@ default: unreachable
 
 return: ret void
 }
+
+define i32 @jt1_optsize(i32 %a, i32 %b) optsize {
+entry:
+  switch i32 %a, label %return [
+    i32 1,  label %bb1
+    i32 2,  label %bb2
+    i32 3,  label %bb3
+    i32 4,  label %bb4
+    i32 5,  label %bb5
+    i32 6,  label %bb6
+    i32 7,  label %bb7
+    i32 8,  label %bb8
+    i32 9,  label %bb9
+    i32 10, label %bb10
+    i32 11, label %bb11
+    i32 12, label %bb12
+    i32 13, label %bb13
+    i32 14, label %bb14
+    i32 15, label %bb15
+    i32 16, label %bb16
+    i32 17, label %bb17
+  ]
+; CHECK-LABEL: function jt1_optsize:
+; CHECK-NEXT: Jump Tables:
+; CHECK0-NEXT:  %jump-table.0: %bb.1 %bb.2 %bb.3 %bb.4 %bb.5 %bb.6 %bb.7 %bb.8 %bb.9 %bb.10 %bb.11 %bb.12 %bb.13 %bb.14 %bb.15 %bb.16 %bb.17
+; CHECK0-NOT:   %jump-table.1:
+; CHECK4-NEXT:  %jump-table.0: %bb.1 %bb.2 %bb.3 %bb.4 %bb.5 %bb.6 %bb.7 %bb.8 %bb.9 %bb.10 %bb.11 %bb.12 %bb.13 %bb.14 %bb.15 %bb.16 %bb.17
+; CHECK4-NOT:   %jump-table.1:
+; CHECK8-NEXT:  %jump-table.0: %bb.1 %bb.2 %bb.3 %bb.4 %bb.5 %bb.6 %bb.7 %bb.8 %bb.9 %bb.10 %bb.11 %bb.12 %bb.13 %bb.14 %bb.15 %bb.16 %bb.17
+; CHECK8-NOT:   %jump-table.1:
+; CHECK16-NEXT: %jump-table.0: %bb.1 %bb.2 %bb.3 %bb.4 %bb.5 %bb.6 %bb.7 %bb.8 %bb.9 %bb.10 %bb.11 %bb.12 %bb.13 %bb.14 %bb.15 %bb.16 %bb.17
+; CHECK16-NOT:  %jump-table.1:
+; CHECKM1-NEXT: %jump-table.0: %bb.1 %bb.2 %bb.3 %bb.4 %bb.5 %bb.6 %bb.7 %bb.8 %bb.9 %bb.10 %bb.11 %bb.12 %bb.13 %bb.14 %bb.15 %bb.16 %bb.17
+; CHECKM1-NOT:  %jump-table.1:
+; CHECKM3-NEXT: %jump-table.0: %bb.1 %bb.2 %bb.3 %bb.4 %bb.5 %bb.6 %bb.7 %bb.8 %bb.9 %bb.10 %bb.11 %bb.12 %bb.13 %bb.14 %bb.15 %bb.16 %bb.17
+; CHECKM3-NOT:  %jump-table.1:
+; CHECK-DAG: End machine code for function jt1_optsize.
+
+bb1:  tail call void @ext(i32 1, i32 0)  br label %return
+bb2:  tail call void @ext(i32 2, i32 2)  br label %return
+bb3:  tail call void @ext(i32 3, i32 4)  br label %return
+bb4:  tail call void @ext(i32 4, i32 6)  br label %return
+bb5:  tail call void @ext(i32 5, i32 8)  br label %return
+bb6:  tail call void @ext(i32 6, i32 10) br label %return
+bb7:  tail call void @ext(i32 7, i32 12) br label %return
+bb8:  tail call void @ext(i32 8, i32 14) br label %return
+bb9:  tail call void @ext(i32 9, i32 16) br label %return
+bb10: tail call void @ext(i32 1, i32 18) br label %return
+bb11: tail call void @ext(i32 2, i32 20) br label %return
+bb12: tail call void @ext(i32 3, i32 22) br label %return
+bb13: tail call void @ext(i32 4, i32 24) br label %return
+bb14: tail call void @ext(i32 5, i32 26) br label %return
+bb15: tail call void @ext(i32 6, i32 28) br label %return
+bb16: tail call void @ext(i32 7, i32 30) br label %return
+bb17: tail call void @ext(i32 8, i32 32) br label %return
+
+return: ret i32 %b
+}
+
+define i32 @jt1_pgso(i32 %a, i32 %b) !prof !14 {
+entry:
+  switch i32 %a, label %return [
+    i32 1,  label %bb1
+    i32 2,  label %bb2
+    i32 3,  label %bb3
+    i32 4,  label %bb4
+    i32 5,  label %bb5
+    i32 6,  label %bb6
+    i32 7,  label %bb7
+    i32 8,  label %bb8
+    i32 9,  label %bb9
+    i32 10, label %bb10
+    i32 11, label %bb11
+    i32 12, label %bb12
+    i32 13, label %bb13
+    i32 14, label %bb14
+    i32 15, label %bb15
+    i32 16, label %bb16
+    i32 17, label %bb17
+  ]
+; CHECK-LABEL: function jt1_pgso:
+; CHECK-NEXT: Jump Tables:
+; CHECK0-NEXT:  %jump-table.0: %bb.1 %bb.2 %bb.3 %bb.4 %bb.5 %bb.6 %bb.7 %bb.8 %bb.9 %bb.10 %bb.11 %bb.12 %bb.13 %bb.14 %bb.15 %bb.16 %bb.17
+; CHECK0-NOT:   %jump-table.1:
+; CHECK4-NEXT:  %jump-table.0: %bb.1 %bb.2 %bb.3 %bb.4 %bb.5 %bb.6 %bb.7 %bb.8 %bb.9 %bb.10 %bb.11 %bb.12 %bb.13 %bb.14 %bb.15 %bb.16 %bb.17
+; CHECK4-NOT:   %jump-table.1:
+; CHECK8-NEXT:  %jump-table.0: %bb.1 %bb.2 %bb.3 %bb.4 %bb.5 %bb.6 %bb.7 %bb.8 %bb.9 %bb.10 %bb.11 %bb.12 %bb.13 %bb.14 %bb.15 %bb.16 %bb.17
+; CHECK8-NOT:   %jump-table.1:
+; CHECK16-NEXT: %jump-table.0: %bb.1 %bb.2 %bb.3 %bb.4 %bb.5 %bb.6 %bb.7 %bb.8 %bb.9 %bb.10 %bb.11 %bb.12 %bb.13 %bb.14 %bb.15 %bb.16 %bb.17
+; CHECK16-NOT:  %jump-table.1:
+; CHECKM1-NEXT: %jump-table.0: %bb.1 %bb.2 %bb.3 %bb.4 %bb.5 %bb.6 %bb.7 %bb.8 %bb.9 %bb.10 %bb.11 %bb.12 %bb.13 %bb.14 %bb.15 %bb.16 %bb.17
+; CHECKM1-NOT:  %jump-table.1:
+; CHECKM3-NEXT: %jump-table.0: %bb.1 %bb.2 %bb.3 %bb.4 %bb.5 %bb.6 %bb.7 %bb.8 %bb.9 %bb.10 %bb.11 %bb.12 %bb.13 %bb.14 %bb.15 %bb.16 %bb.17
+; CHECKM3-NOT:  %jump-table.1:
+; CHECK-DAG: End machine code for function jt1_pgso.
+
+bb1:  tail call void @ext(i32 1, i32 0)  br label %return
+bb2:  tail call void @ext(i32 2, i32 2)  br label %return
+bb3:  tail call void @ext(i32 3, i32 4)  br label %return
+bb4:  tail call void @ext(i32 4, i32 6)  br label %return
+bb5:  tail call void @ext(i32 5, i32 8)  br label %return
+bb6:  tail call void @ext(i32 6, i32 10) br label %return
+bb7:  tail call void @ext(i32 7, i32 12) br label %return
+bb8:  tail call void @ext(i32 8, i32 14) br label %return
+bb9:  tail call void @ext(i32 9, i32 16) br label %return
+bb10: tail call void @ext(i32 1, i32 18) br label %return
+bb11: tail call void @ext(i32 2, i32 20) br label %return
+bb12: tail call void @ext(i32 3, i32 22) br label %return
+bb13: tail call void @ext(i32 4, i32 24) br label %return
+bb14: tail call void @ext(i32 5, i32 26) br label %return
+bb15: tail call void @ext(i32 6, i32 28) br label %return
+bb16: tail call void @ext(i32 7, i32 30) br label %return
+bb17: tail call void @ext(i32 8, i32 32) br label %return
+
+return: ret i32 %b
+}
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 1, !"ProfileSummary", !1}
+!1 = !{!2, !3, !4, !5, !6, !7, !8, !9}
+!2 = !{!"ProfileFormat", !"InstrProf"}
+!3 = !{!"TotalCount", i64 10000}
+!4 = !{!"MaxCount", i64 10}
+!5 = !{!"MaxInternalCount", i64 1}
+!6 = !{!"MaxFunctionCount", i64 1000}
+!7 = !{!"NumCounts", i64 3}
+!8 = !{!"NumFunctions", i64 3}
+!9 = !{!"DetailedSummary", !10}
+!10 = !{!11, !12, !13}
+!11 = !{i32 10000, i64 100, i32 1}
+!12 = !{i32 999000, i64 100, i32 1}
+!13 = !{i32 999999, i64 1, i32 2}
+!14 = !{!"function_entry_count", i64 0}
diff --git a/llvm/test/CodeGen/AArch64/merge-store-dependency.ll b/llvm/test/CodeGen/AArch64/merge-store-dependency.ll
index 3b68cbb8c..5613db1e5 100644
--- a/llvm/test/CodeGen/AArch64/merge-store-dependency.ll
+++ b/llvm/test/CodeGen/AArch64/merge-store-dependency.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mcpu cortex-a53 -mtriple=aarch64-eabi | FileCheck %s --check-prefix=A53
 
 ; PR26827 - Merge stores causes wrong dependency.
@@ -7,11 +8,48 @@
 
 define void @test(%struct1* %fde, i32 %fd, void (i32, i32, i8*)* %func, i8* %arg)  {
 ;CHECK-LABEL: test
+; A53-LABEL: test:
+; A53:       // %bb.0: // %entry
+; A53-NEXT:    stp x30, x19, [sp, #-16]! // 16-byte Folded Spill
+; A53-NEXT:    .cfi_def_cfa_offset 16
+; A53-NEXT:    .cfi_offset w19, -8
+; A53-NEXT:    .cfi_offset w30, -16
+; A53-NEXT:    movi v0.2d, #0000000000000000
+; A53-NEXT:    mov x8, x0
+; A53-NEXT:    mov x19, x8
+; A53-NEXT:    mov w0, w1
+; A53-NEXT:    mov w9, #256
+; A53-NEXT:    str q0, [x19, #16]!
+; A53-NEXT:    str w1, [x19]
+; A53-NEXT:    mov w1, #4
+; A53-NEXT:    stp x2, x3, [x8, #32]
+; A53-NEXT:    mov x2, x8
+; A53-NEXT:    str q0, [x8]
+; A53-NEXT:    strh w9, [x8, #24]
+; A53-NEXT:    str wzr, [x8, #20]
+; A53-NEXT:    bl fcntl
+; A53-NEXT:    adrp x9, gv0
+; A53-NEXT:    add x9, x9, :lo12:gv0
+; A53-NEXT:    cmp x19, x9
+; A53-NEXT:    b.eq .LBB0_4
+; A53-NEXT:  // %bb.1:
+; A53-NEXT:    ldr w8, [x19]
+; A53-NEXT:    ldr w9, [x9]
+; A53-NEXT:  .LBB0_2: // %while.body.i.split.ver.us
+; A53-NEXT:    // =>This Inner Loop Header: Depth=1
+; A53-NEXT:    lsl w9, w9, #1
+; A53-NEXT:    cmp w9, w8
+; A53-NEXT:    b.le .LBB0_2
+; A53-NEXT:  // %bb.3: // %while.end.i
+; A53-NEXT:    bl foo
+; A53-NEXT:    adrp x8, gv1
+; A53-NEXT:    str x0, [x8, :lo12:gv1]
+; A53-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
+; A53-NEXT:    ret
+; A53-NEXT:  .LBB0_4: // %while.body.i.split
+; A53-NEXT:    // =>This Inner Loop Header: Depth=1
+; A53-NEXT:    b .LBB0_4
 entry:
-; A53: mov [[DATA:w[0-9]+]], w1
-; A53: str q{{[0-9]+}}, {{.*}}
-; A53: str q{{[0-9]+}}, {{.*}}
-; A53: str w1, {{.*}}
 
   %0 = bitcast %struct1* %fde to i8*
   tail call void @llvm.memset.p0i8.i64(i8* align 8 %0, i8 0, i64 40, i1 false)
diff --git a/llvm/test/CodeGen/AArch64/merge-store.ll b/llvm/test/CodeGen/AArch64/merge-store.ll
index cd9564210..f0a53384c 100644
--- a/llvm/test/CodeGen/AArch64/merge-store.ll
+++ b/llvm/test/CodeGen/AArch64/merge-store.ll
@@ -42,17 +42,10 @@ define void @blam() {
 ; the fastness of unaligned accesses was not specified correctly.
 
 define void @merge_vec_extract_stores(<4 x float> %v1, <2 x float>* %ptr) {
-; SPLITTING-LABEL: merge_vec_extract_stores:
-; SPLITTING:       // %bb.0:
-; SPLITTING-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
-; SPLITTING-NEXT:    str d0, [x0, #24]
-; SPLITTING-NEXT:    str d1, [x0, #32]
-; SPLITTING-NEXT:    ret
-;
-; MISALIGNED-LABEL: merge_vec_extract_stores:
-; MISALIGNED:       // %bb.0:
-; MISALIGNED-NEXT:    stur q0, [x0, #24]
-; MISALIGNED-NEXT:    ret
+; CHECK-LABEL: merge_vec_extract_stores:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    stur q0, [x0, #24]
+; CHECK-NEXT:    ret
   %idx0 = getelementptr inbounds <2 x float>, <2 x float>* %ptr, i64 3
   %idx1 = getelementptr inbounds <2 x float>, <2 x float>* %ptr, i64 4
 
@@ -62,9 +55,4 @@ define void @merge_vec_extract_stores(<4 x float> %v1, <2 x float>* %ptr) {
   store <2 x float> %shuffle0, <2 x float>* %idx0, align 8
   store <2 x float> %shuffle1, <2 x float>* %idx1, align 8
   ret void
-
-
-; FIXME: Ideally we would like to use a generic target for this test, but this relies
-; on suppressing store pairs.
-
 }
diff --git a/llvm/test/CodeGen/AArch64/misched-fusion-aes.ll b/llvm/test/CodeGen/AArch64/misched-fusion-aes.ll
index ac2c5045d..70038e934 100644
--- a/llvm/test/CodeGen/AArch64/misched-fusion-aes.ll
+++ b/llvm/test/CodeGen/AArch64/misched-fusion-aes.ll
@@ -4,8 +4,6 @@
 ; RUN: llc %s -o - -mtriple=aarch64-unknown -mcpu=cortex-a57 | FileCheck %s
 ; RUN: llc %s -o - -mtriple=aarch64-unknown -mcpu=cortex-a72 | FileCheck %s
 ; RUN: llc %s -o - -mtriple=aarch64-unknown -mcpu=cortex-a73 | FileCheck %s
-; RUN: llc %s -o - -mtriple=aarch64-unknown -mcpu=exynos-m1  | FileCheck %s
-; RUN: llc %s -o - -mtriple=aarch64-unknown -mcpu=exynos-m2  | FileCheck %s
 ; RUN: llc %s -o - -mtriple=aarch64-unknown -mcpu=exynos-m3  | FileCheck %s
 ; RUN: llc %s -o - -mtriple=aarch64-unknown -mcpu=exynos-m4  | FileCheck %s
 ; RUN: llc %s -o - -mtriple=aarch64-unknown -mcpu=exynos-m5  | FileCheck %s
diff --git a/llvm/test/CodeGen/AArch64/neon-dot-product.ll b/llvm/test/CodeGen/AArch64/neon-dot-product.ll
index 5ad75a7d2..b6131e1d0 100644
--- a/llvm/test/CodeGen/AArch64/neon-dot-product.ll
+++ b/llvm/test/CodeGen/AArch64/neon-dot-product.ll
@@ -1,4 +1,8 @@
-; RUN: llc -mtriple aarch64-none-linux-gnu -mattr=+dotprod < %s | FileCheck %s
+; RUN: llc -mtriple aarch64-none-linux-gnu -mattr=+dotprod    < %s | FileCheck %s
+; RUN: llc -mtriple aarch64-none-linux-gnu -mcpu=cortex-a65   < %s | FileCheck %s
+; RUN: llc -mtriple aarch64-none-linux-gnu -mcpu=cortex-a65ae < %s | FileCheck %s
+; RUN: llc -mtriple aarch64-none-linux-gnu -mcpu=neoverse-e1  < %s | FileCheck %s
+; RUN: llc -mtriple aarch64-none-linux-gnu -mcpu=neoverse-n1  < %s | FileCheck %s
 
 declare <2 x i32> @llvm.aarch64.neon.udot.v2i32.v8i8(<2 x i32>, <8 x i8>, <8 x i8>)
 declare <4 x i32> @llvm.aarch64.neon.udot.v4i32.v16i8(<4 x i32>, <16 x i8>, <16 x i8>)
@@ -124,3 +128,145 @@ entry:
   %vdot1.i = call <4 x i32> @llvm.aarch64.neon.sdot.v4i32.v16i8(<4 x i32> %a, <16 x i8> %b, <16 x i8> %.cast3) #2
   ret <4 x i32> %vdot1.i
 }
+
+define fastcc void @test_sdot_v4i8(i8* noalias nocapture %0, i8* noalias nocapture readonly %1, i8* noalias nocapture readonly %2) {
+entry:
+; CHECK-LABEL: test_sdot_v4i8:
+; CHECK:  sdot {{v[0-9]+}}.2s, {{v[0-9]+}}.8b, {{v[0-9]+}}.8b
+  %3 = bitcast i8* %0 to i32*
+  %4 = load i8, i8* %1, align 1
+  %5 = sext i8 %4 to i32
+  %6 = load i8, i8* %2, align 1
+  %7 = sext i8 %6 to i32
+  %8 = mul nsw i32 %7, %5
+  %9 = getelementptr inbounds i8, i8* %1, i64 1
+  %10 = load i8, i8* %9, align 1
+  %11 = sext i8 %10 to i32
+  %12 = getelementptr inbounds i8, i8* %2, i64 1
+  %13 = load i8, i8* %12, align 1
+  %14 = sext i8 %13 to i32
+  %15 = mul nsw i32 %14, %11
+  %16 = add nsw i32 %15, %8
+  %17 = getelementptr inbounds i8, i8* %1, i64 2
+  %18 = load i8, i8* %17, align 1
+  %19 = sext i8 %18 to i32
+  %20 = getelementptr inbounds i8, i8* %2, i64 2
+  %21 = load i8, i8* %20, align 1
+  %22 = sext i8 %21 to i32
+  %23 = mul nsw i32 %22, %19
+  %24 = add nsw i32 %23, %16
+  %25 = getelementptr inbounds i8, i8* %1, i64 3
+  %26 = load i8, i8* %25, align 1
+  %27 = sext i8 %26 to i32
+  %28 = getelementptr inbounds i8, i8* %2, i64 3
+  %29 = load i8, i8* %28, align 1
+  %30 = sext i8 %29 to i32
+  %31 = mul nsw i32 %30, %27
+  %32 = add nsw i32 %31, %24
+  store i32 %32, i32* %3, align 64
+  ret void
+}
+
+define fastcc void @test_udot_v4i8(i8* noalias nocapture %0, i8* noalias nocapture readonly %1, i8* noalias nocapture readonly %2) {
+entry:
+; CHECK-LABEL: test_udot_v4i8:
+; CHECK:  udot {{v[0-9]+}}.2s, {{v[0-9]+}}.8b, {{v[0-9]+}}.8b
+  %3 = bitcast i8* %0 to i32*
+  %4 = load i8, i8* %1, align 1
+  %5 = zext i8 %4 to i32
+  %6 = load i8, i8* %2, align 1
+  %7 = zext i8 %6 to i32
+  %8 = mul nsw i32 %7, %5
+  %9 = getelementptr inbounds i8, i8* %1, i64 1
+  %10 = load i8, i8* %9, align 1
+  %11 = zext i8 %10 to i32
+  %12 = getelementptr inbounds i8, i8* %2, i64 1
+  %13 = load i8, i8* %12, align 1
+  %14 = zext i8 %13 to i32
+  %15 = mul nsw i32 %14, %11
+  %16 = add nsw i32 %15, %8
+  %17 = getelementptr inbounds i8, i8* %1, i64 2
+  %18 = load i8, i8* %17, align 1
+  %19 = zext i8 %18 to i32
+  %20 = getelementptr inbounds i8, i8* %2, i64 2
+  %21 = load i8, i8* %20, align 1
+  %22 = zext i8 %21 to i32
+  %23 = mul nsw i32 %22, %19
+  %24 = add nsw i32 %23, %16
+  %25 = getelementptr inbounds i8, i8* %1, i64 3
+  %26 = load i8, i8* %25, align 1
+  %27 = zext i8 %26 to i32
+  %28 = getelementptr inbounds i8, i8* %2, i64 3
+  %29 = load i8, i8* %28, align 1
+  %30 = zext i8 %29 to i32
+  %31 = mul nsw i32 %30, %27
+  %32 = add nsw i32 %31, %24
+  store i32 %32, i32* %3, align 64
+  ret void
+}
+
+declare i32 @llvm.experimental.vector.reduce.add.v8i32(<8 x i32>)
+
+define i32 @test_udot_v8i8(i8* nocapture readonly %a, i8* nocapture readonly %b) {
+entry:
+; CHECK-LABEL: test_udot_v8i8:
+; CHECK:  udot {{v[0-9]+}}.2s, {{v[0-9]+}}.8b, {{v[0-9]+}}.8b
+  %0 = bitcast i8* %a to <8 x i8>*
+  %1 = load <8 x i8>, <8 x i8>* %0
+  %2 = zext <8 x i8> %1 to <8 x i32>
+  %3 = bitcast i8* %b to <8 x i8>*
+  %4 = load <8 x i8>, <8 x i8>* %3
+  %5 = zext <8 x i8> %4 to <8 x i32>
+  %6 = mul nuw nsw <8 x i32> %5, %2
+  %7 = call i32 @llvm.experimental.vector.reduce.add.v8i32(<8 x i32> %6)
+  ret i32 %7
+}
+
+define i32 @test_sdot_v8i8(i8* nocapture readonly %a, i8* nocapture readonly %b) {
+entry:
+; CHECK-LABEL: test_sdot_v8i8:
+; CHECK:  sdot {{v[0-9]+}}.2s, {{v[0-9]+}}.8b, {{v[0-9]+}}.8b
+  %0 = bitcast i8* %a to <8 x i8>*
+  %1 = load <8 x i8>, <8 x i8>* %0
+  %2 = sext <8 x i8> %1 to <8 x i32>
+  %3 = bitcast i8* %b to <8 x i8>*
+  %4 = load <8 x i8>, <8 x i8>* %3
+  %5 = sext <8 x i8> %4 to <8 x i32>
+  %6 = mul nsw <8 x i32> %5, %2
+  %7 = call i32 @llvm.experimental.vector.reduce.add.v8i32(<8 x i32> %6)
+  ret i32 %7
+}
+
+declare i32 @llvm.experimental.vector.reduce.add.v16i32(<16 x i32>)
+
+define i32 @test_udot_v16i8(i8* nocapture readonly %a, i8* nocapture readonly %b, i32 %sum) {
+entry:
+; CHECK-LABEL: test_udot_v16i8:
+; CHECK:  udot {{v[0-9]+}}.4s, {{v[0-9]+}}.16b, {{v[0-9]+}}.16b
+  %0 = bitcast i8* %a to <16 x i8>*
+  %1 = load <16 x i8>, <16 x i8>* %0
+  %2 = zext <16 x i8> %1 to <16 x i32>
+  %3 = bitcast i8* %b to <16 x i8>*
+  %4 = load <16 x i8>, <16 x i8>* %3
+  %5 = zext <16 x i8> %4 to <16 x i32>
+  %6 = mul nuw nsw <16 x i32> %5, %2
+  %7 = call i32 @llvm.experimental.vector.reduce.add.v16i32(<16 x i32> %6)
+  %op.extra = add i32 %7, %sum
+  ret i32 %op.extra
+}
+
+define i32 @test_sdot_v16i8(i8* nocapture readonly %a, i8* nocapture readonly %b, i32 %sum) {
+entry:
+; CHECK-LABEL: test_sdot_v16i8:
+; CHECK:  sdot {{v[0-9]+}}.4s, {{v[0-9]+}}.16b, {{v[0-9]+}}.16b
+  %0 = bitcast i8* %a to <16 x i8>*
+  %1 = load <16 x i8>, <16 x i8>* %0
+  %2 = sext <16 x i8> %1 to <16 x i32>
+  %3 = bitcast i8* %b to <16 x i8>*
+  %4 = load <16 x i8>, <16 x i8>* %3
+  %5 = sext <16 x i8> %4 to <16 x i32>
+  %6 = mul nsw <16 x i32> %5, %2
+  %7 = call i32 @llvm.experimental.vector.reduce.add.v16i32(<16 x i32> %6)
+  %op.extra = add nsw i32 %7, %sum
+  ret i32 %op.extra
+}
diff --git a/llvm/test/CodeGen/AArch64/neon-mla-mls.ll b/llvm/test/CodeGen/AArch64/neon-mla-mls.ll
index 71bb0e70a..08fb8a563 100644
--- a/llvm/test/CodeGen/AArch64/neon-mla-mls.ll
+++ b/llvm/test/CodeGen/AArch64/neon-mla-mls.ll
@@ -1,88 +1,209 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -verify-machineinstrs -mtriple=aarch64-none-linux-gnu -mattr=+neon | FileCheck %s
 
 
 define <8 x i8> @mla8xi8(<8 x i8> %A, <8 x i8> %B, <8 x i8> %C) {
-;CHECK: mla {{v[0-9]+}}.8b, {{v[0-9]+}}.8b, {{v[0-9]+}}.8b
+; CHECK-LABEL: mla8xi8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mla v2.8b, v0.8b, v1.8b
+; CHECK-NEXT:    mov v0.16b, v2.16b
+; CHECK-NEXT:    ret
 	%tmp1 = mul <8 x i8> %A, %B;
 	%tmp2 = add <8 x i8> %C, %tmp1;
 	ret <8 x i8> %tmp2
 }
 
 define <16 x i8> @mla16xi8(<16 x i8> %A, <16 x i8> %B, <16 x i8> %C) {
-;CHECK: mla {{v[0-9]+}}.16b, {{v[0-9]+}}.16b, {{v[0-9]+}}.16b
+; CHECK-LABEL: mla16xi8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mla v2.16b, v0.16b, v1.16b
+; CHECK-NEXT:    mov v0.16b, v2.16b
+; CHECK-NEXT:    ret
 	%tmp1 = mul <16 x i8> %A, %B;
 	%tmp2 = add <16 x i8> %C, %tmp1;
 	ret <16 x i8> %tmp2
 }
 
 define <4 x i16> @mla4xi16(<4 x i16> %A, <4 x i16> %B, <4 x i16> %C) {
-;CHECK: mla {{v[0-9]+}}.4h, {{v[0-9]+}}.4h, {{v[0-9]+}}.4h
+; CHECK-LABEL: mla4xi16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mla v2.4h, v0.4h, v1.4h
+; CHECK-NEXT:    mov v0.16b, v2.16b
+; CHECK-NEXT:    ret
 	%tmp1 = mul <4 x i16> %A, %B;
 	%tmp2 = add <4 x i16> %C, %tmp1;
 	ret <4 x i16> %tmp2
 }
 
 define <8 x i16> @mla8xi16(<8 x i16> %A, <8 x i16> %B, <8 x i16> %C) {
-;CHECK: mla {{v[0-9]+}}.8h, {{v[0-9]+}}.8h, {{v[0-9]+}}.8h
+; CHECK-LABEL: mla8xi16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mla v2.8h, v0.8h, v1.8h
+; CHECK-NEXT:    mov v0.16b, v2.16b
+; CHECK-NEXT:    ret
 	%tmp1 = mul <8 x i16> %A, %B;
 	%tmp2 = add <8 x i16> %C, %tmp1;
 	ret <8 x i16> %tmp2
 }
 
 define <2 x i32> @mla2xi32(<2 x i32> %A, <2 x i32> %B, <2 x i32> %C) {
-;CHECK: mla {{v[0-9]+}}.2s, {{v[0-9]+}}.2s, {{v[0-9]+}}.2s
+; CHECK-LABEL: mla2xi32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mla v2.2s, v0.2s, v1.2s
+; CHECK-NEXT:    mov v0.16b, v2.16b
+; CHECK-NEXT:    ret
 	%tmp1 = mul <2 x i32> %A, %B;
 	%tmp2 = add <2 x i32> %C, %tmp1;
 	ret <2 x i32> %tmp2
 }
 
 define <4 x i32> @mla4xi32(<4 x i32> %A, <4 x i32> %B, <4 x i32> %C) {
-;CHECK: mla {{v[0-9]+}}.4s, {{v[0-9]+}}.4s, {{v[0-9]+}}.4s
+; CHECK-LABEL: mla4xi32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mla v2.4s, v0.4s, v1.4s
+; CHECK-NEXT:    mov v0.16b, v2.16b
+; CHECK-NEXT:    ret
 	%tmp1 = mul <4 x i32> %A, %B;
 	%tmp2 = add <4 x i32> %C, %tmp1;
 	ret <4 x i32> %tmp2
 }
 
 define <8 x i8> @mls8xi8(<8 x i8> %A, <8 x i8> %B, <8 x i8> %C) {
-;CHECK: mls {{v[0-9]+}}.8b, {{v[0-9]+}}.8b, {{v[0-9]+}}.8b
+; CHECK-LABEL: mls8xi8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mls v2.8b, v0.8b, v1.8b
+; CHECK-NEXT:    mov v0.16b, v2.16b
+; CHECK-NEXT:    ret
 	%tmp1 = mul <8 x i8> %A, %B;
 	%tmp2 = sub <8 x i8> %C, %tmp1;
 	ret <8 x i8> %tmp2
 }
 
 define <16 x i8> @mls16xi8(<16 x i8> %A, <16 x i8> %B, <16 x i8> %C) {
-;CHECK: mls {{v[0-9]+}}.16b, {{v[0-9]+}}.16b, {{v[0-9]+}}.16b
+; CHECK-LABEL: mls16xi8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mls v2.16b, v0.16b, v1.16b
+; CHECK-NEXT:    mov v0.16b, v2.16b
+; CHECK-NEXT:    ret
 	%tmp1 = mul <16 x i8> %A, %B;
 	%tmp2 = sub <16 x i8> %C, %tmp1;
 	ret <16 x i8> %tmp2
 }
 
 define <4 x i16> @mls4xi16(<4 x i16> %A, <4 x i16> %B, <4 x i16> %C) {
-;CHECK: mls {{v[0-9]+}}.4h, {{v[0-9]+}}.4h, {{v[0-9]+}}.4h
+; CHECK-LABEL: mls4xi16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mls v2.4h, v0.4h, v1.4h
+; CHECK-NEXT:    mov v0.16b, v2.16b
+; CHECK-NEXT:    ret
 	%tmp1 = mul <4 x i16> %A, %B;
 	%tmp2 = sub <4 x i16> %C, %tmp1;
 	ret <4 x i16> %tmp2
 }
 
 define <8 x i16> @mls8xi16(<8 x i16> %A, <8 x i16> %B, <8 x i16> %C) {
-;CHECK: mls {{v[0-9]+}}.8h, {{v[0-9]+}}.8h, {{v[0-9]+}}.8h
+; CHECK-LABEL: mls8xi16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mls v2.8h, v0.8h, v1.8h
+; CHECK-NEXT:    mov v0.16b, v2.16b
+; CHECK-NEXT:    ret
 	%tmp1 = mul <8 x i16> %A, %B;
 	%tmp2 = sub <8 x i16> %C, %tmp1;
 	ret <8 x i16> %tmp2
 }
 
 define <2 x i32> @mls2xi32(<2 x i32> %A, <2 x i32> %B, <2 x i32> %C) {
-;CHECK: mls {{v[0-9]+}}.2s, {{v[0-9]+}}.2s, {{v[0-9]+}}.2s
+; CHECK-LABEL: mls2xi32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mls v2.2s, v0.2s, v1.2s
+; CHECK-NEXT:    mov v0.16b, v2.16b
+; CHECK-NEXT:    ret
 	%tmp1 = mul <2 x i32> %A, %B;
 	%tmp2 = sub <2 x i32> %C, %tmp1;
 	ret <2 x i32> %tmp2
 }
 
 define <4 x i32> @mls4xi32(<4 x i32> %A, <4 x i32> %B, <4 x i32> %C) {
-;CHECK: mls {{v[0-9]+}}.4s, {{v[0-9]+}}.4s, {{v[0-9]+}}.4s
+; CHECK-LABEL: mls4xi32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mls v2.4s, v0.4s, v1.4s
+; CHECK-NEXT:    mov v0.16b, v2.16b
+; CHECK-NEXT:    ret
 	%tmp1 = mul <4 x i32> %A, %B;
 	%tmp2 = sub <4 x i32> %C, %tmp1;
 	ret <4 x i32> %tmp2
 }
 
 
+define <8 x i8> @mls2v8xi8(<8 x i8> %A, <8 x i8> %B, <8 x i8> %C) {
+; CHECK-LABEL: mls2v8xi8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    neg v2.8b, v2.8b
+; CHECK-NEXT:    mla v2.8b, v0.8b, v1.8b
+; CHECK-NEXT:    mov v0.16b, v2.16b
+; CHECK-NEXT:    ret
+	%tmp1 = mul <8 x i8> %A, %B;
+	%tmp2 = sub <8 x i8> %tmp1, %C;
+	ret <8 x i8> %tmp2
+}
+
+define <16 x i8> @mls2v16xi8(<16 x i8> %A, <16 x i8> %B, <16 x i8> %C) {
+; CHECK-LABEL: mls2v16xi8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    neg v2.16b, v2.16b
+; CHECK-NEXT:    mla v2.16b, v0.16b, v1.16b
+; CHECK-NEXT:    mov v0.16b, v2.16b
+; CHECK-NEXT:    ret
+	%tmp1 = mul <16 x i8> %A, %B;
+	%tmp2 = sub <16 x i8> %tmp1, %C;
+	ret <16 x i8> %tmp2
+}
+
+define <4 x i16> @mls2v4xi16(<4 x i16> %A, <4 x i16> %B, <4 x i16> %C) {
+; CHECK-LABEL: mls2v4xi16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    neg v2.4h, v2.4h
+; CHECK-NEXT:    mla v2.4h, v0.4h, v1.4h
+; CHECK-NEXT:    mov v0.16b, v2.16b
+; CHECK-NEXT:    ret
+	%tmp1 = mul <4 x i16> %A, %B;
+	%tmp2 = sub <4 x i16> %tmp1, %C;
+	ret <4 x i16> %tmp2
+}
+
+define <8 x i16> @mls2v8xi16(<8 x i16> %A, <8 x i16> %B, <8 x i16> %C) {
+; CHECK-LABEL: mls2v8xi16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    neg v2.8h, v2.8h
+; CHECK-NEXT:    mla v2.8h, v0.8h, v1.8h
+; CHECK-NEXT:    mov v0.16b, v2.16b
+; CHECK-NEXT:    ret
+	%tmp1 = mul <8 x i16> %A, %B;
+	%tmp2 = sub <8 x i16> %tmp1, %C;
+	ret <8 x i16> %tmp2
+}
+
+define <2 x i32> @mls2v2xi32(<2 x i32> %A, <2 x i32> %B, <2 x i32> %C) {
+; CHECK-LABEL: mls2v2xi32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    neg v2.2s, v2.2s
+; CHECK-NEXT:    mla v2.2s, v0.2s, v1.2s
+; CHECK-NEXT:    mov v0.16b, v2.16b
+; CHECK-NEXT:    ret
+	%tmp1 = mul <2 x i32> %A, %B;
+	%tmp2 = sub <2 x i32> %tmp1, %C;
+	ret <2 x i32> %tmp2
+}
+
+define <4 x i32> @mls2v4xi32(<4 x i32> %A, <4 x i32> %B, <4 x i32> %C) {
+; CHECK-LABEL: mls2v4xi32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    neg v2.4s, v2.4s
+; CHECK-NEXT:    mla v2.4s, v0.4s, v1.4s
+; CHECK-NEXT:    mov v0.16b, v2.16b
+; CHECK-NEXT:    ret
+	%tmp1 = mul <4 x i32> %A, %B;
+	%tmp2 = sub <4 x i32> %tmp1, %C;
+	ret <4 x i32> %tmp2
+}
+
diff --git a/llvm/test/CodeGen/AArch64/neon-vcadd.ll b/llvm/test/CodeGen/AArch64/neon-vcadd.ll
new file mode 100644
index 000000000..11605267c
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/neon-vcadd.ll
@@ -0,0 +1,67 @@
+; RUN: llc %s -mtriple=aarch64 -mattr=+v8.3a,+fullfp16 -o - | FileCheck %s
+
+define <4 x half> @foo16x4_rot(<4 x half> %a, <4 x half> %b) {
+entry:
+; CHECK-LABEL: foo16x4_rot
+; CHECK-DAG: fcadd v{{[0-9]+}}.4h, v{{[0-9]+}}.4h, v{{[0-9]+}}.4h, #90
+; CHECK-DAG: fcadd v{{[0-9]+}}.4h, v{{[0-9]+}}.4h, v{{[0-9]+}}.4h, #270
+  %vcadd_rot90_v2.i = tail call <4 x half> @llvm.aarch64.neon.vcadd.rot90.v4f16(<4 x half> %a, <4 x half> %b)
+  %vcadd_rot270_v2.i = tail call <4 x half> @llvm.aarch64.neon.vcadd.rot270.v4f16(<4 x half> %a, <4 x half> %b)
+  %add = fadd <4 x half> %vcadd_rot90_v2.i, %vcadd_rot270_v2.i
+  ret <4 x half> %add
+}
+
+define <2 x float> @foo32x2_rot(<2 x float> %a, <2 x float> %b) {
+entry:
+; CHECK-LABEL: foo32x2_rot
+; CHECK-DAG: fcadd v{{[0-9]+}}.2s, v{{[0-9]+}}.2s, v{{[0-9]+}}.2s, #90
+; CHECK-DAG: fcadd v{{[0-9]+}}.2s, v{{[0-9]+}}.2s, v{{[0-9]+}}.2s, #270
+  %vcadd_rot90_v2.i = tail call <2 x float> @llvm.aarch64.neon.vcadd.rot90.v2f32(<2 x float> %a, <2 x float> %b)
+  %vcadd_rot270_v2.i = tail call <2 x float> @llvm.aarch64.neon.vcadd.rot270.v2f32(<2 x float> %a, <2 x float> %b)
+  %add = fadd <2 x float> %vcadd_rot90_v2.i, %vcadd_rot270_v2.i
+  ret <2 x float> %add
+}
+
+define <8 x half> @foo16x8_rot(<8 x half> %a, <8 x half> %b) {
+entry:
+; CHECK-LABEL: foo16x8_rot
+; CHECK-DAG: fcadd v{{[0-9]+}}.8h, v{{[0-9]+}}.8h, v{{[0-9]+}}.8h, #90
+; CHECK-DAG: fcadd v{{[0-9]+}}.8h, v{{[0-9]+}}.8h, v{{[0-9]+}}.8h, #270
+  %vcaddq_rot90_v2.i = tail call <8 x half> @llvm.aarch64.neon.vcadd.rot90.v8f16(<8 x half> %a, <8 x half> %b)
+  %vcaddq_rot270_v2.i = tail call <8 x half> @llvm.aarch64.neon.vcadd.rot270.v8f16(<8 x half> %a, <8 x half> %b)
+  %add = fadd <8 x half> %vcaddq_rot90_v2.i, %vcaddq_rot270_v2.i
+  ret <8 x half> %add
+}
+
+define <4 x float> @foo32x4_rot(<4 x float> %a, <4 x float> %b) {
+entry:
+; CHECK-LABEL: foo32x4_rot
+; CHECK-DAG: fcadd v{{[0-9]+}}.4s, v{{[0-9]+}}.4s, v{{[0-9]+}}.4s, #90
+; CHECK-DAG: fcadd v{{[0-9]+}}.4s, v{{[0-9]+}}.4s, v{{[0-9]+}}.4s, #270
+  %vcaddq_rot90_v2.i = tail call <4 x float> @llvm.aarch64.neon.vcadd.rot90.v4f32(<4 x float> %a, <4 x float> %b)
+  %vcaddq_rot270_v2.i = tail call <4 x float> @llvm.aarch64.neon.vcadd.rot270.v4f32(<4 x float> %a, <4 x float> %b)
+  %add = fadd <4 x float> %vcaddq_rot90_v2.i, %vcaddq_rot270_v2.i
+  ret <4 x float> %add
+}
+
+define <2 x double> @foo64x2_rot(<2 x double> %a, <2 x double> %b) {
+entry:
+; CHECK-LABEL: foo64x2_rot
+; CHECK-DAG: fcadd v{{[0-9]+}}.2d, v{{[0-9]+}}.2d, v{{[0-9]+}}.2d, #90
+; CHECK-DAG: fcadd v{{[0-9]+}}.2d, v{{[0-9]+}}.2d, v{{[0-9]+}}.2d, #270
+  %vcaddq_rot90_v2.i = tail call <2 x double> @llvm.aarch64.neon.vcadd.rot90.v2f64(<2 x double> %a, <2 x double> %b)
+  %vcaddq_rot270_v2.i = tail call <2 x double> @llvm.aarch64.neon.vcadd.rot270.v2f64(<2 x double> %a, <2 x double> %b)
+  %add = fadd <2 x double> %vcaddq_rot90_v2.i, %vcaddq_rot270_v2.i
+  ret <2 x double> %add
+}
+
+declare <4 x half> @llvm.aarch64.neon.vcadd.rot90.v4f16(<4 x half>, <4 x half>)
+declare <4 x half> @llvm.aarch64.neon.vcadd.rot270.v4f16(<4 x half>, <4 x half>)
+declare <2 x float> @llvm.aarch64.neon.vcadd.rot90.v2f32(<2 x float>, <2 x float>)
+declare <2 x float> @llvm.aarch64.neon.vcadd.rot270.v2f32(<2 x float>, <2 x float>)
+declare <8 x half> @llvm.aarch64.neon.vcadd.rot90.v8f16(<8 x half>, <8 x half>)
+declare <8 x half> @llvm.aarch64.neon.vcadd.rot270.v8f16(<8 x half>, <8 x half>)
+declare <4 x float> @llvm.aarch64.neon.vcadd.rot90.v4f32(<4 x float>, <4 x float>)
+declare <4 x float> @llvm.aarch64.neon.vcadd.rot270.v4f32(<4 x float>, <4 x float>)
+declare <2 x double> @llvm.aarch64.neon.vcadd.rot90.v2f64(<2 x double>, <2 x double>)
+declare <2 x double> @llvm.aarch64.neon.vcadd.rot270.v2f64(<2 x double>, <2 x double>)
diff --git a/llvm/test/CodeGen/AArch64/no-quad-ldp-stp.ll b/llvm/test/CodeGen/AArch64/no-quad-ldp-stp.ll
index 32f57cd21..6840e844e 100644
--- a/llvm/test/CodeGen/AArch64/no-quad-ldp-stp.ll
+++ b/llvm/test/CodeGen/AArch64/no-quad-ldp-stp.ll
@@ -1,5 +1,4 @@
 ; RUN: llc < %s -mtriple=aarch64-eabi -mattr=+slow-paired-128 -verify-machineinstrs -asm-verbose=false | FileCheck %s --check-prefixes=CHECK,SLOW
-; RUN: llc < %s -mtriple=aarch64-eabi -mcpu=exynos-m1         -verify-machineinstrs -asm-verbose=false | FileCheck %s --check-prefixes=CHECK,SLOW
 ; RUN: llc < %s -mtriple=aarch64-eabi -mcpu=exynos-m3         -verify-machineinstrs -asm-verbose=false | FileCheck %s --check-prefixes=CHECK,FAST
 
 ; CHECK-LABEL: test_nopair_st
diff --git a/llvm/test/CodeGen/AArch64/no_cfi.ll b/llvm/test/CodeGen/AArch64/no_cfi.ll
new file mode 100644
index 000000000..49e34b3c5
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/no_cfi.ll
@@ -0,0 +1,13 @@
+; RUN: llc -mtriple aarch64-arm-linux-gnu -o - %s | FileCheck %s
+
+; CHECK:        a:                                      // @a
+; CHECK-NEXT:   // %bb.0:
+; CHECK-NEXT:           sub     sp, sp, #16
+; CHECK-NOT:            .cfi{{.*}}
+; CHECK:                ret
+define void @a() nounwind {
+  %1 = alloca i32, align 4
+  store i32 1, i32* %1, align 4
+  ret void
+}
+
diff --git a/llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-0.ll b/llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-0.ll
new file mode 100644
index 000000000..cd5b99765
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-0.ll
@@ -0,0 +1,14 @@
+; RUN: llc -mtriple=aarch64-linux %s               -o - | \
+; RUN:   FileCheck %s --check-prefix=ASM
+; RUN: llc -mtriple=aarch64-linux %s -filetype=obj -o - |  \
+; RUN:   llvm-readelf --notes | FileCheck %s --check-prefix=OBJ
+@x = common dso_local global i32 0, align 4
+
+attributes #0 = { "branch-target-enforcement" }
+
+; Both attributes present in a file with no functions.
+; ASM:	    .word	3221225472
+; ASM-NEXT:	.word	4
+; ASM-NEXT	.word	3
+
+; OBJ: Properties: aarch64 feature: BTI, PAC
diff --git a/llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-1.ll b/llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-1.ll
new file mode 100644
index 000000000..3be749761
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-1.ll
@@ -0,0 +1,18 @@
+; RUN: llc -mtriple=aarch64-linux %s               -o - | \
+; RUN:   FileCheck %s --check-prefix=ASM
+; RUN: llc -mtriple=aarch64-linux %s -filetype=obj -o - |  \
+; RUN:   llvm-readelf --notes | FileCheck %s --check-prefix=OBJ
+
+define dso_local i32 @f() #0 {
+entry:
+  ret i32 0
+}
+
+attributes #0 = { "branch-target-enforcement" }
+
+; BTI attribute present
+; ASM:	    .word	3221225472
+; ASM-NEXT:	.word	4
+; ASM-NEXT	.word	1
+
+; OBJ: Properties: aarch64 feature: BTI
diff --git a/llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-2.ll b/llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-2.ll
new file mode 100644
index 000000000..bb63a8629
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-2.ll
@@ -0,0 +1,18 @@
+; RUN: llc -mtriple=aarch64-linux %s               -o - | \
+; RUN:   FileCheck %s --check-prefix=ASM
+; RUN: llc -mtriple=aarch64-linux %s -filetype=obj -o - |  \
+; RUN:   llvm-readelf --notes | FileCheck %s --check-prefix=OBJ
+
+define dso_local i32 @f() #0 {
+entry:
+  ret i32 0
+}
+
+attributes #0 = { "sign-return-address"="all" }
+
+; PAC attribute present
+; ASM:	    .word	3221225472
+; ASM-NEXT:	.word	4
+; ASM-NEXT	.word	2
+
+; OBJ: Properties: aarch64 feature: PAC
diff --git a/llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-3.ll b/llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-3.ll
new file mode 100644
index 000000000..98a8e5e75
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-3.ll
@@ -0,0 +1,18 @@
+; RUN: llc -mtriple=aarch64-linux %s               -o - | \
+; RUN:   FileCheck %s --check-prefix=ASM
+; RUN: llc -mtriple=aarch64-linux %s -filetype=obj -o - |  \
+; RUN:   llvm-readelf --notes | FileCheck %s --check-prefix=OBJ
+
+define dso_local i32 @f() #0 {
+entry:
+  ret i32 0
+}
+
+attributes #0 = { "branch-target-enforcement" "sign-return-address"="non-leaf" }
+
+; Both attribute present
+; ASM:	    .word	3221225472
+; ASM-NEXT:	.word	4
+; ASM-NEXT	.word	3
+
+; OBJ: Properties: aarch64 feature: BTI, PAC
diff --git a/llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-4.ll b/llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-4.ll
new file mode 100644
index 000000000..b9179fd26
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-4.ll
@@ -0,0 +1,25 @@
+; RUN: llc -mtriple=aarch64-linux %s               -o - | \
+; RUN:   FileCheck %s --check-prefix=ASM
+; RUN: llc -mtriple=aarch64-linux %s -filetype=obj -o - |  \
+; RUN:   llvm-readelf --notes | FileCheck %s --check-prefix=OBJ
+
+define dso_local i32 @f() #0 {
+entry:
+  ret i32 0
+}
+
+define dso_local i32 @g() #1 {
+entry:
+  ret i32 0
+}
+
+attributes #0 = { "branch-target-enforcement" "sign-return-address"="non-leaf" }
+
+attributes #1 = { "branch-target-enforcement" }
+
+; Only the common atttribute (BTI)
+; ASM:	    .word	3221225472
+; ASM-NEXT:	.word	4
+; ASM-NEXT	.word	1
+
+; OBJ: Properties: aarch64 feature: BTI
diff --git a/llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-5.ll b/llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-5.ll
new file mode 100644
index 000000000..8959c16a1
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-5.ll
@@ -0,0 +1,26 @@
+; RUN: llc -mtriple=aarch64-linux %s               -o - 2>&1 | \
+; RUN:   FileCheck %s --check-prefix=ASM
+; RUN: llc -mtriple=aarch64-linux %s -filetype=obj -o -      |  \
+; RUN:   llvm-readelf --notes | FileCheck %s --check-prefix=OBJ
+
+define dso_local i32 @f() #0 {
+entry:
+  ret i32 0
+}
+
+define dso_local i32 @g() #1 {
+entry:
+  ret i32 0
+}
+
+attributes #0 = { "branch-target-enforcement" "sign-return-address"="non-leaf" }
+
+attributes #1 = { "sign-return-address"="all" }
+
+; Only the common atttribute (PAC)
+; ASM: warning: not setting BTI in feature flags
+; ASM:	    .word	3221225472
+; ASM-NEXT:	.word	4
+; ASM-NEXT	.word	2
+
+; OBJ: Properties: aarch64 feature: PAC
diff --git a/llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-6.ll b/llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-6.ll
new file mode 100644
index 000000000..b57a23b97
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-6.ll
@@ -0,0 +1,22 @@
+; RUN: llc -mtriple=aarch64-linux %s               -o - | \
+; RUN:   FileCheck %s --check-prefix=ASM
+; RUN: llc -mtriple=aarch64-linux %s -filetype=obj -o - |  \
+; RUN:   llvm-readelf -S | FileCheck %s --check-prefix=OBJ
+
+define dso_local i32 @f() #0 {
+entry:
+  ret i32 0
+}
+
+define dso_local i32 @g() #1 {
+entry:
+  ret i32 0
+}
+
+attributes #0 = { "sign-return-address"="non-leaf" }
+
+attributes #1 = { "sign-return-address"="none" }
+
+; No common attribute, no note section
+; ASM-NOT: .note.gnu.property
+; OBJ-NOT: .note.gnu.property
diff --git a/llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-7.ll b/llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-7.ll
new file mode 100644
index 000000000..b663dafe4
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-7.ll
@@ -0,0 +1,23 @@
+; RUN: llc -mtriple=aarch64-linux %s               -o - 2>&1 | \
+; RUN:   FileCheck %s --check-prefix=ASM
+; RUN: llc -mtriple=aarch64-linux %s -filetype=obj -o -      |  \
+; RUN:   llvm-readelf -S | FileCheck %s --check-prefix=OBJ
+
+define dso_local i32 @f() #0 {
+entry:
+  ret i32 0
+}
+
+define dso_local i32 @g() #1 {
+entry:
+  ret i32 0
+}
+
+attributes #0 = { "sign-return-address"="non-leaf" }
+
+attributes #1 = { "branch-target-enforcement" }
+
+; No common attribute, no note section
+; ASM: warning: not setting BTI in feature flags
+; ASM-NOT: .note.gnu.property
+; OBJ-NOT: .note.gnu.property
diff --git a/llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-8.ll b/llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-8.ll
new file mode 100644
index 000000000..1e9497bd8
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-8.ll
@@ -0,0 +1,21 @@
+; RUN: llc -mtriple=aarch64-linux %s               -o - | \
+; RUN:   FileCheck %s --check-prefix=ASM
+; RUN: llc -mtriple=aarch64-linux %s -filetype=obj -o - | \
+; RUN:   llvm-readelf --notes | FileCheck %s --check-prefix=OBJ
+
+define dso_local i32 @f() #0 {
+entry:
+  %r = tail call i32 @g()
+  ret i32 %r
+}
+
+declare dso_local i32 @g()
+
+attributes #0 = { "branch-target-enforcement" }
+
+; Declarations don't prevent setting BTI
+; ASM:	    .word	3221225472
+; ASM-NEXT:	.word	4
+; ASM-NEXT	.word	1
+
+; OBJ: Properties: aarch64 feature: BTI
diff --git a/llvm/test/CodeGen/AArch64/overeager_mla_fusing.ll b/llvm/test/CodeGen/AArch64/overeager_mla_fusing.ll
new file mode 100644
index 000000000..67b54e46e
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/overeager_mla_fusing.ll
@@ -0,0 +1,59 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc %s --mtriple aarch64 -verify-machineinstrs -o - | FileCheck %s
+
+define dso_local void @jsimd_idct_ifast_neon_intrinsic(i8* nocapture readonly %dct_table, i16* nocapture readonly %coef_block, i8** nocapture readonly %output_buf, i32 %output_col) local_unnamed_addr #0 {
+; CHECK-LABEL: jsimd_idct_ifast_neon_intrinsic:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr q0, [x1, #32]
+; CHECK-NEXT:    ldr q1, [x1, #96]
+; CHECK-NEXT:    ldr q2, [x0, #32]
+; CHECK-NEXT:    ldr q3, [x0, #96]
+; CHECK-NEXT:    ldr x8, [x2, #48]
+; CHECK-NEXT:    mov w9, w3
+; CHECK-NEXT:    mul v0.8h, v2.8h, v0.8h
+; CHECK-NEXT:    mul v1.8h, v3.8h, v1.8h
+; CHECK-NEXT:    add v2.8h, v0.8h, v1.8h
+; CHECK-NEXT:    str q2, [x8, x9]
+; CHECK-NEXT:    ldr x8, [x2, #56]
+; CHECK-NEXT:    sub v0.8h, v0.8h, v1.8h
+; CHECK-NEXT:    str q0, [x8, x9]
+; CHECK-NEXT:    ret
+entry:
+  %add.ptr5 = getelementptr inbounds i16, i16* %coef_block, i64 16
+  %0 = bitcast i16* %add.ptr5 to <8 x i16>*
+  %1 = load <8 x i16>, <8 x i16>* %0, align 16
+
+  %add.ptr17 = getelementptr inbounds i16, i16* %coef_block, i64 48
+  %2 = bitcast i16* %add.ptr17 to <8 x i16>*
+  %3 = load <8 x i16>, <8 x i16>* %2, align 16
+
+  %add.ptr29 = getelementptr inbounds i8, i8* %dct_table, i64 32
+  %4 = bitcast i8* %add.ptr29 to <8 x i16>*
+  %5 = load <8 x i16>, <8 x i16>* %4, align 16
+
+  %add.ptr41 = getelementptr inbounds i8, i8* %dct_table, i64 96
+  %6 = bitcast i8* %add.ptr41 to <8 x i16>*
+  %7 = load <8 x i16>, <8 x i16>* %6, align 16
+
+  %mul.i966 = mul <8 x i16> %5, %1
+  %mul.i964 = mul <8 x i16> %7, %3
+
+  %add.i961 = add <8 x i16> %mul.i966, %mul.i964
+  %sub.i960 = sub <8 x i16> %mul.i966, %mul.i964
+
+  %idx.ext = zext i32 %output_col to i64
+
+  %arrayidx404 = getelementptr inbounds i8*, i8** %output_buf, i64 6
+  %8 = load i8*, i8** %arrayidx404, align 8
+  %add.ptr406 = getelementptr inbounds i8, i8* %8, i64 %idx.ext
+  %9 = bitcast i8* %add.ptr406 to <8 x i16>*
+  store <8 x i16> %add.i961, <8 x i16>* %9, align 8
+
+  %arrayidx408 = getelementptr inbounds i8*, i8** %output_buf, i64 7
+  %10 = load i8*, i8** %arrayidx408, align 8
+  %add.ptr410 = getelementptr inbounds i8, i8* %10, i64 %idx.ext
+  %11 = bitcast i8* %add.ptr410 to <8 x i16>*
+  store <8 x i16> %sub.i960, <8 x i16>* %11, align 8
+
+  ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/patchable-function-entry-bti.ll b/llvm/test/CodeGen/AArch64/patchable-function-entry-bti.ll
new file mode 100644
index 000000000..19386e943
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/patchable-function-entry-bti.ll
@@ -0,0 +1,86 @@
+; RUN: llc -mtriple=aarch64 %s -o - | FileCheck --check-prefixes=CHECK %s
+
+define void @f0() "patchable-function-entry"="0" "branch-target-enforcement" {
+; CHECK-LABEL: f0:
+; CHECK-NEXT: .Lfunc_begin0:
+; CHECK:      // %bb.0:
+; CHECK-NEXT:  hint #34
+; CHECK-NEXT:  ret
+; CHECK-NOT:  .section __patchable_function_entries
+  ret void
+}
+
+;; -fpatchable-function-entry=1 -mbranch-protection=bti
+;; For M=0, place the label .Lpatch0 after the initial BTI.
+define void @f1() "patchable-function-entry"="1" "branch-target-enforcement" {
+; CHECK-LABEL: f1:
+; CHECK-NEXT: .Lfunc_begin1:
+; CHECK-NEXT: .cfi_startproc
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT:  hint #34
+; CHECK-NEXT: .Lpatch0:
+; CHECK-NEXT:  nop
+; CHECK-NEXT:  ret
+; CHECK:      .section __patchable_function_entries,"awo",@progbits,f1,unique,0
+; CHECK-NEXT: .p2align 3
+; CHECK-NEXT: .xword .Lpatch0
+  ret void
+}
+
+;; -fpatchable-function-entry=2,1 -mbranch-protection=bti
+define void @f2_1() "patchable-function-entry"="1" "patchable-function-prefix"="1" "branch-target-enforcement" {
+; CHECK-LABEL: .type f2_1,@function
+; CHECK-NEXT: .Ltmp0:
+; CHECK-NEXT:  nop
+; CHECK-NEXT: f2_1:
+; CHECK-NEXT: .Lfunc_begin2:
+; CHECK-NEXT: .cfi_startproc
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT:  hint #34
+; CHECK-NEXT:  nop
+; CHECK-NEXT:  ret
+; CHECK:      .Lfunc_end2:
+; CHECK-NEXT: .size f2_1, .Lfunc_end2-f2_1
+; CHECK:      .section __patchable_function_entries,"awo",@progbits,f1,unique,0
+; CHECK-NEXT: .p2align 3
+; CHECK-NEXT: .xword .Ltmp0
+  ret void
+}
+
+;; -fpatchable-function-entry=1 -mbranch-protection=bti
+;; For M=0, don't create .Lpatch0 if the initial instruction is not BTI,
+;; even if other basic blocks may have BTI.
+define internal void @f1i(i64 %v) "patchable-function-entry"="1" "branch-target-enforcement" {
+; CHECK-LABEL: f1i:
+; CHECK-NEXT: .Lfunc_begin3:
+; CHECK:      // %bb.0:
+; CHECK-NEXT:  nop
+;; Other basic blocks have BTI, but they don't affect our decision to not create .Lpatch0
+; CHECK:      .LBB{{.+}} // %sw.bb1
+; CHECK-NEXT:  hint #36
+; CHECK:      .section __patchable_function_entries,"awo",@progbits,f1,unique,0
+; CHECK-NEXT: .p2align 3
+; CHECK-NEXT: .xword .Lfunc_begin3
+entry:
+  switch i64 %v, label %sw.bb0 [
+    i64 1, label %sw.bb1
+    i64 2, label %sw.bb2
+    i64 3, label %sw.bb3
+    i64 4, label %sw.bb4
+  ]
+sw.bb0:
+  call void asm sideeffect "", ""()
+  ret void
+sw.bb1:
+  call void asm sideeffect "", ""()
+  ret void
+sw.bb2:
+  call void asm sideeffect "", ""()
+  ret void
+sw.bb3:
+  call void asm sideeffect "", ""()
+  ret void
+sw.bb4:
+  call void asm sideeffect "", ""()
+  ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/patchable-function-entry.ll b/llvm/test/CodeGen/AArch64/patchable-function-entry.ll
new file mode 100644
index 000000000..5ae9d88a2
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/patchable-function-entry.ll
@@ -0,0 +1,105 @@
+; RUN: llc -mtriple=aarch64 %s -o - | FileCheck --check-prefixes=CHECK,NOFSECT %s
+; RUN: llc -mtriple=aarch64 -function-sections %s -o - | FileCheck --check-prefixes=CHECK,FSECT %s
+; RUN: llc -mtriple=aarch64 -no-integrated-as %s -o - | FileCheck --check-prefix=NOLINK %s
+
+; NOLINK-NOT: "awo"
+; NOLINK-NOT: ,unique,0
+
+define i32 @f0() "patchable-function-entry"="0" {
+; CHECK-LABEL: f0:
+; CHECK-NEXT: .Lfunc_begin0:
+; CHECK-NOT:   nop
+; CHECK:       mov w0, wzr
+; CHECK-NOT:   .section __patchable_function_entries
+  ret i32 0
+}
+
+define i32 @f1() "patchable-function-entry"="1" {
+; CHECK-LABEL: f1:
+; CHECK-NEXT: .Lfunc_begin1:
+; CHECK:       nop
+; CHECK-NEXT:  mov w0, wzr
+; CHECK:       .section __patchable_function_entries,"awo",@progbits,f1,unique,0
+; CHECK-NEXT:  .p2align 3
+; CHECK-NEXT:  .xword .Lfunc_begin1
+  ret i32 0
+}
+
+;; Without -function-sections, f2 is in the same text section as f1.
+;; They share the __patchable_function_entries section.
+;; With -function-sections, f1 and f2 are in different text sections.
+;; Use separate __patchable_function_entries.
+define void @f2() "patchable-function-entry"="2" {
+; CHECK-LABEL: f2:
+; CHECK-NEXT: .Lfunc_begin2:
+; CHECK-COUNT-2: nop
+; CHECK-NEXT:  ret
+; NOFSECT:     .section __patchable_function_entries,"awo",@progbits,f1,unique,0
+; FSECT:       .section __patchable_function_entries,"awo",@progbits,f2,unique,1
+; CHECK-NEXT:  .p2align 3
+; CHECK-NEXT:  .xword .Lfunc_begin2
+  ret void
+}
+
+$f3 = comdat any
+define void @f3() "patchable-function-entry"="3" comdat {
+; CHECK-LABEL: f3:
+; CHECK-NEXT: .Lfunc_begin3:
+; CHECK-COUNT-3: nop
+; CHECK-NEXT:  ret
+; NOFSECT:     .section __patchable_function_entries,"aGwo",@progbits,f3,comdat,f3,unique,1
+; FSECT:       .section __patchable_function_entries,"aGwo",@progbits,f3,comdat,f3,unique,2
+; CHECK-NEXT:  .p2align 3
+; CHECK-NEXT:  .xword .Lfunc_begin3
+  ret void
+}
+
+$f5 = comdat any
+define void @f5() "patchable-function-entry"="5" comdat {
+; CHECK-LABEL: f5:
+; CHECK-NEXT: .Lfunc_begin4:
+; CHECK-COUNT-5: nop
+; CHECK-NEXT:  sub sp, sp, #16
+; NOFSECT      .section __patchable_function_entries,"aGwo",@progbits,f5,comdat,f5,unique,2
+; FSECT:       .section __patchable_function_entries,"aGwo",@progbits,f5,comdat,f5,unique,3
+; CHECK:       .p2align 3
+; CHECK-NEXT:  .xword .Lfunc_begin4
+  %frame = alloca i8, i32 16
+  ret void
+}
+
+;; -fpatchable-function-entry=3,2
+;; "patchable-function-prefix" emits data before the function entry label.
+define void @f3_2() "patchable-function-entry"="1" "patchable-function-prefix"="2" {
+; CHECK-LABEL: .type f3_2,@function
+; CHECK-NEXT: .Ltmp1: // @f3_2
+; CHECK-NEXT:  nop
+; CHECK-NEXT:  nop
+; CHECK-NEXT: f3_2:
+; CHECK:      // %bb.0:
+; CHECK-NEXT:  nop
+; CHECK-NEXT:  ret
+;; .size does not include the prefix.
+; CHECK:      .Lfunc_end5:
+; CHECK-NEXT: .size f3_2, .Lfunc_end5-f3_2
+; NOFSECT     .section __patchable_function_entries,"awo",@progbits,f1,unique,0
+; FSECT:      .section __patchable_function_entries,"awo",@progbits,f3_2,unique,4
+; CHECK:      .p2align 3
+; CHECK-NEXT: .xword .Ltmp1
+  ret void
+}
+
+;; When prefix data is used, arbitrarily place NOPs after prefix data.
+define void @prefix() "patchable-function-entry"="0" "patchable-function-prefix"="1" prefix i32 1 {
+; CHECK-LABEL: .type prefix,@function
+; CHECK-NEXT: .word 1 // @prefix
+; CHECK:      .Ltmp2:
+; CHECK:       nop
+; CHECK-NEXT: prefix:
+;; Emit a __patchable_function_entries entry even if "patchable-function-entry" is 0.
+; NOFSECT     .section __patchable_function_entries,"awo",@progbits,prefix,unique,0
+; FSECT:      .section __patchable_function_entries,"awo",@progbits,prefix,unique,5
+; CHECK:      .p2align 3
+; CHECK-NEXT: .xword .Ltmp2
+  ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/powi-windows.ll b/llvm/test/CodeGen/AArch64/powi-windows.ll
new file mode 100644
index 000000000..859d772b4
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/powi-windows.ll
@@ -0,0 +1,46 @@
+; RUN: llc -mtriple aarch64-windows < %s | FileCheck %s
+
+declare double @llvm.powi.f64(double, i32)
+declare float @llvm.powi.f32(float, i32)
+
+define double @d(double %d, i32 %i) {
+entry:
+  %0 = tail call double @llvm.powi.f64(double %d, i32 %i)
+  ret double %0
+}
+
+; CHECK-LABEL: d:
+; CHECK: scvtf d1, w0
+; CHECK-NEXT: b pow
+
+define float @f(float %f, i32 %i) {
+entry:
+  %0 = tail call float @llvm.powi.f32(float %f, i32 %i)
+  ret float %0
+}
+
+; CHECK-LABEL: f:
+; CHECK: scvtf s1, w0
+; CHECK-NEXT: b powf
+
+define float @g(double %d, i32 %i) {
+entry:
+  %0 = tail call double @llvm.powi.f64(double %d, i32 %i)
+  %conv = fptrunc double %0 to float
+  ret float %conv
+}
+
+; CHECK-LABEL: g:
+; CHECK: scvtf d1, w0
+; CHECK-NEXT: bl pow
+
+define double @h(float %f, i32 %i) {
+entry:
+  %0 = tail call float @llvm.powi.f32(float %f, i32 %i)
+  %conv = fpext float %0 to double
+  ret double %conv
+}
+
+; CHECK-LABEL: h:
+; CHECK: scvtf s1, w0
+; CHECK-NEXT: bl powf
diff --git a/llvm/test/CodeGen/AArch64/preferred-alignment.ll b/llvm/test/CodeGen/AArch64/preferred-alignment.ll
index b39a5e870..ffff7e1d0 100644
--- a/llvm/test/CodeGen/AArch64/preferred-alignment.ll
+++ b/llvm/test/CodeGen/AArch64/preferred-alignment.ll
@@ -23,6 +23,6 @@ entry:
 
 declare i32 @bar(i8*, i16*, i32*) #1
 
-attributes #0 = { nounwind "no-frame-pointer-elim"="false" }
-attributes #1 = { "no-frame-pointer-elim"="false" }
+attributes #0 = { nounwind "frame-pointer"="none" }
+attributes #1 = { "frame-pointer"="none" }
 
diff --git a/llvm/test/CodeGen/AArch64/preferred-function-alignment.ll b/llvm/test/CodeGen/AArch64/preferred-function-alignment.ll
index 048813b83..012fe4fb1 100644
--- a/llvm/test/CodeGen/AArch64/preferred-function-alignment.ll
+++ b/llvm/test/CodeGen/AArch64/preferred-function-alignment.ll
@@ -1,19 +1,23 @@
 ; RUN: llc -mtriple=aarch64-unknown-linux -mcpu=generic < %s | FileCheck --check-prefixes=ALIGN2,CHECK %s
 ; RUN: llc -mtriple=aarch64-unknown-linux -mcpu=cortex-a35 < %s | FileCheck --check-prefixes=ALIGN2,CHECK %s
+; RUN: llc -mtriple=aarch64-unknown-linux -mcpu=cortex-a53 < %s | FileCheck --check-prefixes=ALIGN3,CHECK %s
+; RUN: llc -mtriple=aarch64-unknown-linux -mcpu=cortex-a57 < %s | FileCheck --check-prefixes=ALIGN4,CHECK %s
+; RUN: llc -mtriple=aarch64-unknown-linux -mcpu=cortex-a65 < %s | FileCheck --check-prefixes=ALIGN3,CHECK %s
+; RUN: llc -mtriple=aarch64-unknown-linux -mcpu=cortex-a65ae < %s | FileCheck --check-prefixes=ALIGN3,CHECK %s
+; RUN: llc -mtriple=aarch64-unknown-linux -mcpu=cortex-a72 < %s | FileCheck --check-prefixes=ALIGN4,CHECK %s
+; RUN: llc -mtriple=aarch64-unknown-linux -mcpu=cortex-a73 < %s | FileCheck --check-prefixes=ALIGN4,CHECK %s
+; RUN: llc -mtriple=aarch64-unknown-linux -mcpu=cortex-a75 < %s | FileCheck --check-prefixes=ALIGN4,CHECK %s
+; RUN: llc -mtriple=aarch64-unknown-linux -mcpu=cortex-a76 < %s | FileCheck --check-prefixes=ALIGN4,CHECK %s
 ; RUN: llc -mtriple=aarch64-unknown-linux -mcpu=cyclone < %s | FileCheck --check-prefixes=ALIGN2,CHECK %s
 ; RUN: llc -mtriple=aarch64-unknown-linux -mcpu=falkor < %s | FileCheck --check-prefixes=ALIGN2,CHECK %s
 ; RUN: llc -mtriple=aarch64-unknown-linux -mcpu=kryo < %s | FileCheck --check-prefixes=ALIGN2,CHECK %s
-; RUN: llc -mtriple=aarch64-unknown-linux -mcpu=cortex-a53 < %s | FileCheck --check-prefixes=ALIGN3,CHECK %s
+; RUN: llc -mtriple=aarch64-unknown-linux -mcpu=neoverse-e1 < %s | FileCheck --check-prefixes=ALIGN3,CHECK %s
+; RUN: llc -mtriple=aarch64-unknown-linux -mcpu=neoverse-n1 < %s | FileCheck --check-prefixes=ALIGN4,CHECK %s
 ; RUN: llc -mtriple=aarch64-unknown-linux -mcpu=thunderx < %s | FileCheck --check-prefixes=ALIGN3,CHECK %s
 ; RUN: llc -mtriple=aarch64-unknown-linux -mcpu=thunderxt81 < %s | FileCheck --check-prefixes=ALIGN3,CHECK %s
 ; RUN: llc -mtriple=aarch64-unknown-linux -mcpu=thunderxt83 < %s | FileCheck --check-prefixes=ALIGN3,CHECK %s
 ; RUN: llc -mtriple=aarch64-unknown-linux -mcpu=thunderxt88 < %s | FileCheck --check-prefixes=ALIGN3,CHECK %s
 ; RUN: llc -mtriple=aarch64-unknown-linux -mcpu=thunderx2t99 < %s | FileCheck --check-prefixes=ALIGN3,CHECK %s
-; RUN: llc -mtriple=aarch64-unknown-linux -mcpu=cortex-a57 < %s | FileCheck --check-prefixes=ALIGN4,CHECK %s
-; RUN: llc -mtriple=aarch64-unknown-linux -mcpu=cortex-a72 < %s | FileCheck --check-prefixes=ALIGN4,CHECK %s
-; RUN: llc -mtriple=aarch64-unknown-linux -mcpu=cortex-a73 < %s | FileCheck --check-prefixes=ALIGN4,CHECK %s
-; RUN: llc -mtriple=aarch64-unknown-linux -mcpu=exynos-m1 < %s | FileCheck --check-prefixes=ALIGN4,CHECK %s
-; RUN: llc -mtriple=aarch64-unknown-linux -mcpu=exynos-m2 < %s | FileCheck --check-prefixes=ALIGN4,CHECK %s
 ; RUN: llc -mtriple=aarch64-unknown-linux -mcpu=exynos-m3 < %s | FileCheck --check-prefixes=ALIGN5,CHECK %s
 
 define void @test() {
diff --git a/llvm/test/CodeGen/AArch64/ragreedy-local-interval-cost.ll b/llvm/test/CodeGen/AArch64/ragreedy-local-interval-cost.ll
new file mode 100644
index 000000000..f3bd66cea
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/ragreedy-local-interval-cost.ll
@@ -0,0 +1,339 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64-arm-none-eabi < %s | FileCheck %s
+
+@A = external dso_local local_unnamed_addr global [8 x [8 x i64]], align 8
+@B = external dso_local local_unnamed_addr global [8 x [8 x i64]], align 8
+@C = external dso_local local_unnamed_addr global [8 x [8 x i64]], align 8
+
+define dso_local void @run_test() local_unnamed_addr #0 {
+; CHECK-LABEL: run_test:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sub sp, sp, #80 // =80
+; CHECK-NEXT:    stp d15, d14, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    stp d13, d12, [sp, #32] // 16-byte Folded Spill
+; CHECK-NEXT:    stp d11, d10, [sp, #48] // 16-byte Folded Spill
+; CHECK-NEXT:    stp d9, d8, [sp, #64] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 80
+; CHECK-NEXT:    .cfi_offset b8, -8
+; CHECK-NEXT:    .cfi_offset b9, -16
+; CHECK-NEXT:    .cfi_offset b10, -24
+; CHECK-NEXT:    .cfi_offset b11, -32
+; CHECK-NEXT:    .cfi_offset b12, -40
+; CHECK-NEXT:    .cfi_offset b13, -48
+; CHECK-NEXT:    .cfi_offset b14, -56
+; CHECK-NEXT:    .cfi_offset b15, -64
+; CHECK-NEXT:    adrp x10, B+48
+; CHECK-NEXT:    adrp x11, A
+; CHECK-NEXT:    mov x8, xzr
+; CHECK-NEXT:    mov x9, xzr
+; CHECK-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-NEXT:    add x10, x10, :lo12:B+48
+; CHECK-NEXT:    add x11, x11, :lo12:A
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    // implicit-def: $q1
+; CHECK-NEXT:    // implicit-def: $q2
+; CHECK-NEXT:    // implicit-def: $q3
+; CHECK-NEXT:    // implicit-def: $q4
+; CHECK-NEXT:    // implicit-def: $q5
+; CHECK-NEXT:    // implicit-def: $q6
+; CHECK-NEXT:    // implicit-def: $q7
+; CHECK-NEXT:    // implicit-def: $q16
+; CHECK-NEXT:    // implicit-def: $q17
+; CHECK-NEXT:    // implicit-def: $q18
+; CHECK-NEXT:    // implicit-def: $q19
+; CHECK-NEXT:    // implicit-def: $q20
+; CHECK-NEXT:    // implicit-def: $q21
+; CHECK-NEXT:    // implicit-def: $q22
+; CHECK-NEXT:    // implicit-def: $q23
+; CHECK-NEXT:    // implicit-def: $q24
+; CHECK-NEXT:    // implicit-def: $q25
+; CHECK-NEXT:    // implicit-def: $q26
+; CHECK-NEXT:    // implicit-def: $q27
+; CHECK-NEXT:    // implicit-def: $q28
+; CHECK-NEXT:    // implicit-def: $q29
+; CHECK-NEXT:    // implicit-def: $q30
+; CHECK-NEXT:    // implicit-def: $q31
+; CHECK-NEXT:    // implicit-def: $q8
+; CHECK-NEXT:    // implicit-def: $q9
+; CHECK-NEXT:    // implicit-def: $q10
+; CHECK-NEXT:    // implicit-def: $q11
+; CHECK-NEXT:    // implicit-def: $q12
+; CHECK-NEXT:    // implicit-def: $q13
+; CHECK-NEXT:  .LBB0_1: // %for.cond1.preheader
+; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    mov x12, xzr
+; CHECK-NEXT:    ldr q15, [x8]
+; CHECK-NEXT:    ldr q14, [x12]
+; CHECK-NEXT:    ldr q0, [x10], #64
+; CHECK-NEXT:    ldr x18, [x12]
+; CHECK-NEXT:    fmov x15, d15
+; CHECK-NEXT:    mov x14, v15.d[1]
+; CHECK-NEXT:    fmov x13, d14
+; CHECK-NEXT:    mul x1, x15, x18
+; CHECK-NEXT:    mov x16, v0.d[1]
+; CHECK-NEXT:    fmov x17, d0
+; CHECK-NEXT:    fmov d0, x1
+; CHECK-NEXT:    mul x1, x14, x18
+; CHECK-NEXT:    mov x12, v14.d[1]
+; CHECK-NEXT:    ldr x0, [x8]
+; CHECK-NEXT:    mov v0.d[1], x1
+; CHECK-NEXT:    mul x1, x13, x18
+; CHECK-NEXT:    add v12.2d, v12.2d, v0.2d
+; CHECK-NEXT:    fmov d0, x1
+; CHECK-NEXT:    mul x1, x12, x18
+; CHECK-NEXT:    mov v0.d[1], x1
+; CHECK-NEXT:    mul x1, x17, x18
+; CHECK-NEXT:    add v13.2d, v13.2d, v0.2d
+; CHECK-NEXT:    add v11.2d, v11.2d, v0.2d
+; CHECK-NEXT:    fmov d0, x1
+; CHECK-NEXT:    mul x18, x16, x18
+; CHECK-NEXT:    ldr q14, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    mov v0.d[1], x18
+; CHECK-NEXT:    mul x18, x15, x0
+; CHECK-NEXT:    add x1, x11, x8
+; CHECK-NEXT:    add v10.2d, v10.2d, v0.2d
+; CHECK-NEXT:    fmov d0, x18
+; CHECK-NEXT:    mul x18, x14, x0
+; CHECK-NEXT:    ldr x1, [x1, #128]
+; CHECK-NEXT:    mov v0.d[1], x18
+; CHECK-NEXT:    mul x18, x13, x0
+; CHECK-NEXT:    add v8.2d, v8.2d, v0.2d
+; CHECK-NEXT:    add v25.2d, v25.2d, v0.2d
+; CHECK-NEXT:    add v22.2d, v22.2d, v0.2d
+; CHECK-NEXT:    add v18.2d, v18.2d, v0.2d
+; CHECK-NEXT:    add v6.2d, v6.2d, v0.2d
+; CHECK-NEXT:    add v14.2d, v14.2d, v0.2d
+; CHECK-NEXT:    fmov d0, x18
+; CHECK-NEXT:    mul x18, x12, x0
+; CHECK-NEXT:    mov v0.d[1], x18
+; CHECK-NEXT:    mul x18, x17, x0
+; CHECK-NEXT:    mul x0, x16, x0
+; CHECK-NEXT:    add v9.2d, v9.2d, v0.2d
+; CHECK-NEXT:    add v31.2d, v31.2d, v0.2d
+; CHECK-NEXT:    add v26.2d, v26.2d, v0.2d
+; CHECK-NEXT:    add v23.2d, v23.2d, v0.2d
+; CHECK-NEXT:    add v21.2d, v21.2d, v0.2d
+; CHECK-NEXT:    add v19.2d, v19.2d, v0.2d
+; CHECK-NEXT:    add v17.2d, v17.2d, v0.2d
+; CHECK-NEXT:    add v7.2d, v7.2d, v0.2d
+; CHECK-NEXT:    add v5.2d, v5.2d, v0.2d
+; CHECK-NEXT:    add v3.2d, v3.2d, v0.2d
+; CHECK-NEXT:    add v2.2d, v2.2d, v0.2d
+; CHECK-NEXT:    fmov d0, x18
+; CHECK-NEXT:    mul x15, x15, x1
+; CHECK-NEXT:    mov v0.d[1], x0
+; CHECK-NEXT:    mul x14, x14, x1
+; CHECK-NEXT:    add v30.2d, v30.2d, v0.2d
+; CHECK-NEXT:    add v24.2d, v24.2d, v0.2d
+; CHECK-NEXT:    add v20.2d, v20.2d, v0.2d
+; CHECK-NEXT:    add v16.2d, v16.2d, v0.2d
+; CHECK-NEXT:    add v4.2d, v4.2d, v0.2d
+; CHECK-NEXT:    add v1.2d, v1.2d, v0.2d
+; CHECK-NEXT:    fmov d0, x15
+; CHECK-NEXT:    mul x13, x13, x1
+; CHECK-NEXT:    mov v0.d[1], x14
+; CHECK-NEXT:    mul x12, x12, x1
+; CHECK-NEXT:    add v29.2d, v29.2d, v0.2d
+; CHECK-NEXT:    fmov d0, x13
+; CHECK-NEXT:    mul x17, x17, x1
+; CHECK-NEXT:    mov v0.d[1], x12
+; CHECK-NEXT:    mul x16, x16, x1
+; CHECK-NEXT:    add v28.2d, v28.2d, v0.2d
+; CHECK-NEXT:    fmov d0, x17
+; CHECK-NEXT:    mov v0.d[1], x16
+; CHECK-NEXT:    add x8, x8, #8 // =8
+; CHECK-NEXT:    add v27.2d, v27.2d, v0.2d
+; CHECK-NEXT:    cmp x8, #64 // =64
+; CHECK-NEXT:    add x9, x9, #1 // =1
+; CHECK-NEXT:    str q14, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    b.ne .LBB0_1
+; CHECK-NEXT:  // %bb.2: // %for.cond.cleanup
+; CHECK-NEXT:    adrp x8, C
+; CHECK-NEXT:    add x8, x8, :lo12:C
+; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    stp q13, q12, [x8]
+; CHECK-NEXT:    stp q11, q10, [x8, #32]
+; CHECK-NEXT:    stp q9, q8, [x8, #64]
+; CHECK-NEXT:    ldp d9, d8, [sp, #64] // 16-byte Folded Reload
+; CHECK-NEXT:    ldp d11, d10, [sp, #48] // 16-byte Folded Reload
+; CHECK-NEXT:    ldp d13, d12, [sp, #32] // 16-byte Folded Reload
+; CHECK-NEXT:    ldp d15, d14, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    stp q31, q30, [x8, #96]
+; CHECK-NEXT:    stp q29, q28, [x8, #144]
+; CHECK-NEXT:    stp q27, q26, [x8, #176]
+; CHECK-NEXT:    str q25, [x8, #208]
+; CHECK-NEXT:    stp q24, q23, [x8, #240]
+; CHECK-NEXT:    stp q22, q21, [x8, #272]
+; CHECK-NEXT:    stp q20, q19, [x8, #304]
+; CHECK-NEXT:    stp q18, q17, [x8, #336]
+; CHECK-NEXT:    stp q16, q7, [x8, #368]
+; CHECK-NEXT:    stp q6, q5, [x8, #400]
+; CHECK-NEXT:    stp q4, q3, [x8, #432]
+; CHECK-NEXT:    stp q0, q2, [x8, #464]
+; CHECK-NEXT:    str q1, [x8, #496]
+; CHECK-NEXT:    add sp, sp, #80 // =80
+; CHECK-NEXT:    ret
+entry:
+  br label %for.cond1.preheader
+
+for.cond1.preheader:                              ; preds = %for.cond1.preheader, %entry
+  %0 = phi <2 x i64> [ undef, %entry ], [ %118, %for.cond1.preheader ]
+  %1 = phi <2 x i64> [ undef, %entry ], [ %116, %for.cond1.preheader ]
+  %2 = phi <2 x i64> [ zeroinitializer, %entry ], [ %114, %for.cond1.preheader ]
+  %3 = phi <2 x i64> [ undef, %entry ], [ %112, %for.cond1.preheader ]
+  %4 = phi <2 x i64> [ undef, %entry ], [ %107, %for.cond1.preheader ]
+  %5 = phi <2 x i64> [ undef, %entry ], [ %105, %for.cond1.preheader ]
+  %6 = phi <2 x i64> [ undef, %entry ], [ %103, %for.cond1.preheader ]
+  %7 = phi <2 x i64> [ undef, %entry ], [ %101, %for.cond1.preheader ]
+  %8 = phi <2 x i64> [ undef, %entry ], [ %96, %for.cond1.preheader ]
+  %9 = phi <2 x i64> [ undef, %entry ], [ %94, %for.cond1.preheader ]
+  %10 = phi <2 x i64> [ undef, %entry ], [ %92, %for.cond1.preheader ]
+  %11 = phi <2 x i64> [ undef, %entry ], [ %90, %for.cond1.preheader ]
+  %12 = phi <2 x i64> [ undef, %entry ], [ %85, %for.cond1.preheader ]
+  %13 = phi <2 x i64> [ undef, %entry ], [ %83, %for.cond1.preheader ]
+  %14 = phi <2 x i64> [ undef, %entry ], [ %81, %for.cond1.preheader ]
+  %15 = phi <2 x i64> [ undef, %entry ], [ %79, %for.cond1.preheader ]
+  %16 = phi <2 x i64> [ undef, %entry ], [ %74, %for.cond1.preheader ]
+  %17 = phi <2 x i64> [ undef, %entry ], [ %72, %for.cond1.preheader ]
+  %18 = phi <2 x i64> [ undef, %entry ], [ %70, %for.cond1.preheader ]
+  %19 = phi <2 x i64> [ undef, %entry ], [ %65, %for.cond1.preheader ]
+  %20 = phi <2 x i64> [ undef, %entry ], [ %63, %for.cond1.preheader ]
+  %21 = phi <2 x i64> [ undef, %entry ], [ %61, %for.cond1.preheader ]
+  %22 = phi <2 x i64> [ undef, %entry ], [ %56, %for.cond1.preheader ]
+  %23 = phi <2 x i64> [ undef, %entry ], [ %54, %for.cond1.preheader ]
+  %24 = phi <2 x i64> [ undef, %entry ], [ %52, %for.cond1.preheader ]
+  %25 = phi <2 x i64> [ undef, %entry ], [ %50, %for.cond1.preheader ]
+  %26 = phi <2 x i64> [ undef, %entry ], [ %45, %for.cond1.preheader ]
+  %27 = phi <2 x i64> [ undef, %entry ], [ %43, %for.cond1.preheader ]
+  %28 = phi <2 x i64> [ undef, %entry ], [ %41, %for.cond1.preheader ]
+  %29 = phi <2 x i64> [ undef, %entry ], [ %39, %for.cond1.preheader ]
+  %indvars.iv40 = phi i64 [ 0, %entry ], [ %indvars.iv.next41, %for.cond1.preheader ]
+  %30 = load <2 x i64>, <2 x i64>* null, align 8
+  %31 = load <2 x i64>, <2 x i64>* undef, align 8
+  %arrayidx14.4.phi.trans.insert = getelementptr inbounds [8 x [8 x i64]], [8 x [8 x i64]]* @B, i64 0, i64 %indvars.iv40, i64 4
+  %32 = load <2 x i64>, <2 x i64>* null, align 8
+  %arrayidx14.6.phi.trans.insert = getelementptr inbounds [8 x [8 x i64]], [8 x [8 x i64]]* @B, i64 0, i64 %indvars.iv40, i64 6
+  %33 = bitcast i64* %arrayidx14.6.phi.trans.insert to <2 x i64>*
+  %34 = load <2 x i64>, <2 x i64>* %33, align 8
+  %35 = load i64, i64* null, align 8
+  %36 = insertelement <2 x i64> undef, i64 %35, i32 0
+  %37 = shufflevector <2 x i64> %36, <2 x i64> undef, <2 x i32> zeroinitializer
+  %38 = mul nsw <2 x i64> %30, %37
+  %39 = add nsw <2 x i64> %29, %38
+  %40 = mul nsw <2 x i64> %31, %37
+  %41 = add nsw <2 x i64> %28, %40
+  %42 = mul nsw <2 x i64> %32, %37
+  %43 = add nsw <2 x i64> %27, %42
+  %44 = mul nsw <2 x i64> %34, %37
+  %45 = add nsw <2 x i64> %26, %44
+  %46 = load i64, i64* undef, align 8
+  %47 = insertelement <2 x i64> undef, i64 %46, i32 0
+  %48 = shufflevector <2 x i64> %47, <2 x i64> undef, <2 x i32> zeroinitializer
+  %49 = mul nsw <2 x i64> %30, %48
+  %50 = add nsw <2 x i64> %25, %49
+  %51 = mul nsw <2 x i64> %31, %48
+  %52 = add nsw <2 x i64> %24, %51
+  %53 = mul nsw <2 x i64> %32, %48
+  %54 = add nsw <2 x i64> %23, %53
+  %55 = mul nsw <2 x i64> %34, %48
+  %56 = add nsw <2 x i64> %22, %55
+  %arrayidx10.2 = getelementptr inbounds [8 x [8 x i64]], [8 x [8 x i64]]* @A, i64 0, i64 2, i64 %indvars.iv40
+  %57 = load i64, i64* %arrayidx10.2, align 8
+  %58 = insertelement <2 x i64> undef, i64 %57, i32 0
+  %59 = shufflevector <2 x i64> %58, <2 x i64> undef, <2 x i32> zeroinitializer
+  %60 = mul nsw <2 x i64> %31, %59
+  %61 = add nsw <2 x i64> %21, %60
+  %62 = mul nsw <2 x i64> %32, %59
+  %63 = add nsw <2 x i64> %20, %62
+  %64 = mul nsw <2 x i64> %34, %59
+  %65 = add nsw <2 x i64> %19, %64
+  %66 = load i64, i64* undef, align 8
+  %67 = insertelement <2 x i64> undef, i64 %66, i32 0
+  %68 = shufflevector <2 x i64> %67, <2 x i64> undef, <2 x i32> zeroinitializer
+  %69 = mul nsw <2 x i64> %30, %68
+  %70 = add nsw <2 x i64> %18, %69
+  %71 = mul nsw <2 x i64> %31, %68
+  %72 = add nsw <2 x i64> %17, %71
+  %73 = mul nsw <2 x i64> %34, %68
+  %74 = add nsw <2 x i64> %16, %73
+  %75 = load i64, i64* undef, align 8
+  %76 = insertelement <2 x i64> undef, i64 %75, i32 0
+  %77 = shufflevector <2 x i64> %76, <2 x i64> undef, <2 x i32> zeroinitializer
+  %78 = mul nsw <2 x i64> %30, %77
+  %79 = add nsw <2 x i64> %15, %78
+  %80 = mul nsw <2 x i64> %31, %77
+  %81 = add nsw <2 x i64> %14, %80
+  %82 = mul nsw <2 x i64> %32, %77
+  %83 = add nsw <2 x i64> %13, %82
+  %84 = mul nsw <2 x i64> %34, %77
+  %85 = add nsw <2 x i64> %12, %84
+  %86 = load i64, i64* undef, align 8
+  %87 = insertelement <2 x i64> undef, i64 %86, i32 0
+  %88 = shufflevector <2 x i64> %87, <2 x i64> undef, <2 x i32> zeroinitializer
+  %89 = mul nsw <2 x i64> %30, %88
+  %90 = add nsw <2 x i64> %11, %89
+  %91 = mul nsw <2 x i64> %31, %88
+  %92 = add nsw <2 x i64> %10, %91
+  %93 = mul nsw <2 x i64> %32, %88
+  %94 = add nsw <2 x i64> %9, %93
+  %95 = mul nsw <2 x i64> %34, %88
+  %96 = add nsw <2 x i64> %8, %95
+  %97 = load i64, i64* undef, align 8
+  %98 = insertelement <2 x i64> undef, i64 %97, i32 0
+  %99 = shufflevector <2 x i64> %98, <2 x i64> undef, <2 x i32> zeroinitializer
+  %100 = mul nsw <2 x i64> %30, %99
+  %101 = add nsw <2 x i64> %7, %100
+  %102 = mul nsw <2 x i64> %31, %99
+  %103 = add nsw <2 x i64> %6, %102
+  %104 = mul nsw <2 x i64> %32, %99
+  %105 = add nsw <2 x i64> %5, %104
+  %106 = mul nsw <2 x i64> %34, %99
+  %107 = add nsw <2 x i64> %4, %106
+  %108 = load i64, i64* undef, align 8
+  %109 = insertelement <2 x i64> undef, i64 %108, i32 0
+  %110 = shufflevector <2 x i64> %109, <2 x i64> undef, <2 x i32> zeroinitializer
+  %111 = mul nsw <2 x i64> %30, %110
+  %112 = add nsw <2 x i64> %3, %111
+  %113 = mul nsw <2 x i64> %31, %110
+  %114 = add nsw <2 x i64> %2, %113
+  %115 = mul nsw <2 x i64> %32, %110
+  %116 = add nsw <2 x i64> %1, %115
+  %117 = mul nsw <2 x i64> %34, %110
+  %118 = add nsw <2 x i64> %0, %117
+  %indvars.iv.next41 = add nuw nsw i64 %indvars.iv40, 1
+  %exitcond42 = icmp eq i64 %indvars.iv.next41, 8
+  br i1 %exitcond42, label %for.cond.cleanup, label %for.cond1.preheader
+
+for.cond.cleanup:                                 ; preds = %for.cond1.preheader
+  store <2 x i64> %39, <2 x i64>* bitcast ([8 x [8 x i64]]* @C to <2 x i64>*), align 8
+  store <2 x i64> %41, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x [8 x i64]], [8 x [8 x i64]]* @C, i64 0, i64 0, i64 2) to <2 x i64>*), align 8
+  store <2 x i64> %43, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x [8 x i64]], [8 x [8 x i64]]* @C, i64 0, i64 0, i64 4) to <2 x i64>*), align 8
+  store <2 x i64> %45, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x [8 x i64]], [8 x [8 x i64]]* @C, i64 0, i64 0, i64 6) to <2 x i64>*), align 8
+  store <2 x i64> %50, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x [8 x i64]], [8 x [8 x i64]]* @C, i64 0, i64 1, i64 0) to <2 x i64>*), align 8
+  store <2 x i64> %52, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x [8 x i64]], [8 x [8 x i64]]* @C, i64 0, i64 1, i64 2) to <2 x i64>*), align 8
+  store <2 x i64> %54, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x [8 x i64]], [8 x [8 x i64]]* @C, i64 0, i64 1, i64 4) to <2 x i64>*), align 8
+  store <2 x i64> %56, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x [8 x i64]], [8 x [8 x i64]]* @C, i64 0, i64 1, i64 6) to <2 x i64>*), align 8
+  store <2 x i64> %61, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x [8 x i64]], [8 x [8 x i64]]* @C, i64 0, i64 2, i64 2) to <2 x i64>*), align 8
+  store <2 x i64> %63, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x [8 x i64]], [8 x [8 x i64]]* @C, i64 0, i64 2, i64 4) to <2 x i64>*), align 8
+  store <2 x i64> %65, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x [8 x i64]], [8 x [8 x i64]]* @C, i64 0, i64 2, i64 6) to <2 x i64>*), align 8
+  store <2 x i64> %70, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x [8 x i64]], [8 x [8 x i64]]* @C, i64 0, i64 3, i64 0) to <2 x i64>*), align 8
+  store <2 x i64> %72, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x [8 x i64]], [8 x [8 x i64]]* @C, i64 0, i64 3, i64 2) to <2 x i64>*), align 8
+  store <2 x i64> %74, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x [8 x i64]], [8 x [8 x i64]]* @C, i64 0, i64 3, i64 6) to <2 x i64>*), align 8
+  store <2 x i64> %79, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x [8 x i64]], [8 x [8 x i64]]* @C, i64 0, i64 4, i64 0) to <2 x i64>*), align 8
+  store <2 x i64> %81, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x [8 x i64]], [8 x [8 x i64]]* @C, i64 0, i64 4, i64 2) to <2 x i64>*), align 8
+  store <2 x i64> %83, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x [8 x i64]], [8 x [8 x i64]]* @C, i64 0, i64 4, i64 4) to <2 x i64>*), align 8
+  store <2 x i64> %85, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x [8 x i64]], [8 x [8 x i64]]* @C, i64 0, i64 4, i64 6) to <2 x i64>*), align 8
+  store <2 x i64> %90, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x [8 x i64]], [8 x [8 x i64]]* @C, i64 0, i64 5, i64 0) to <2 x i64>*), align 8
+  store <2 x i64> %92, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x [8 x i64]], [8 x [8 x i64]]* @C, i64 0, i64 5, i64 2) to <2 x i64>*), align 8
+  store <2 x i64> %94, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x [8 x i64]], [8 x [8 x i64]]* @C, i64 0, i64 5, i64 4) to <2 x i64>*), align 8
+  store <2 x i64> %96, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x [8 x i64]], [8 x [8 x i64]]* @C, i64 0, i64 5, i64 6) to <2 x i64>*), align 8
+  store <2 x i64> %101, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x [8 x i64]], [8 x [8 x i64]]* @C, i64 0, i64 6, i64 0) to <2 x i64>*), align 8
+  store <2 x i64> %103, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x [8 x i64]], [8 x [8 x i64]]* @C, i64 0, i64 6, i64 2) to <2 x i64>*), align 8
+  store <2 x i64> %105, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x [8 x i64]], [8 x [8 x i64]]* @C, i64 0, i64 6, i64 4) to <2 x i64>*), align 8
+  store <2 x i64> %107, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x [8 x i64]], [8 x [8 x i64]]* @C, i64 0, i64 6, i64 6) to <2 x i64>*), align 8
+  store <2 x i64> %112, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x [8 x i64]], [8 x [8 x i64]]* @C, i64 0, i64 7, i64 0) to <2 x i64>*), align 8
+  store <2 x i64> %114, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x [8 x i64]], [8 x [8 x i64]]* @C, i64 0, i64 7, i64 2) to <2 x i64>*), align 8
+  store <2 x i64> %116, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x [8 x i64]], [8 x [8 x i64]]* @C, i64 0, i64 7, i64 4) to <2 x i64>*), align 8
+  store <2 x i64> %118, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x [8 x i64]], [8 x [8 x i64]]* @C, i64 0, i64 7, i64 6) to <2 x i64>*), align 8
+  ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/regress-w29-reserved-with-fp.ll b/llvm/test/CodeGen/AArch64/regress-w29-reserved-with-fp.ll
index 2cbcad6c7..0d1ce4169 100644
--- a/llvm/test/CodeGen/AArch64/regress-w29-reserved-with-fp.ll
+++ b/llvm/test/CodeGen/AArch64/regress-w29-reserved-with-fp.ll
@@ -5,7 +5,7 @@ declare void @bar()
 
 define void @test_w29_reserved() {
 ; CHECK-LABEL: test_w29_reserved:
-; CHECK: add x29, sp, #{{[0-9]+}}
+; CHECK: mov x29, sp
 
   %val1 = load volatile i32, i32* @var
   %val2 = load volatile i32, i32* @var
diff --git a/llvm/test/CodeGen/AArch64/remat.ll b/llvm/test/CodeGen/AArch64/remat.ll
index f110c38f9..bf2b68e56 100644
--- a/llvm/test/CodeGen/AArch64/remat.ll
+++ b/llvm/test/CodeGen/AArch64/remat.ll
@@ -2,11 +2,13 @@
 ; RUN: llc -mtriple=aarch64-linux-gnuabi -mcpu=cortex-a53 -o - %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64-linux-gnuabi -mcpu=cortex-a55 -o - %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64-linux-gnuabi -mcpu=cortex-a57 -o - %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnuabi -mcpu=cortex-a65 -o - %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnuabi -mcpu=cortex-a65ae -o - %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64-linux-gnuabi -mcpu=cortex-a72 -o - %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64-linux-gnuabi -mcpu=cortex-a73 -o - %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64-linux-gnuabi -mcpu=cortex-a75 -o - %s | FileCheck %s
-; RUN: llc -mtriple=aarch64-linux-gnuabi -mcpu=exynos-m1 -o - %s | FileCheck %s
-; RUN: llc -mtriple=aarch64-linux-gnuabi -mcpu=exynos-m2 -o - %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnuabi -mcpu=neoverse-e1 -o - %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnuabi -mcpu=neoverse-n1 -o - %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64-linux-gnuabi -mcpu=exynos-m3 -o - %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64-linux-gnuabi -mcpu=exynos-m4 -o - %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64-linux-gnuabi -mcpu=exynos-m5 -o - %s | FileCheck %s
@@ -26,9 +28,9 @@ entry:
 ; CHECK: add x0, sp, #8
 ; CHECK-NOT: mov
 ; CHECK-NEXT: bl f
-  call void @f(%X* %tmp)               
+  call void @f(%X* %tmp)
 ; CHECK: add x0, sp, #8
 ; CHECK-NOT: mov
 ; CHECK-NEXT: bl f
-  ret void 
+  ret void
 }
diff --git a/llvm/test/CodeGen/AArch64/sadd_sat.ll b/llvm/test/CodeGen/AArch64/sadd_sat.ll
index 943458496..997116606 100644
--- a/llvm/test/CodeGen/AArch64/sadd_sat.ll
+++ b/llvm/test/CodeGen/AArch64/sadd_sat.ll
@@ -1,10 +1,12 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=aarch64-- | FileCheck %s
 
-declare  i4  @llvm.sadd.sat.i4   (i4,  i4)
-declare  i32 @llvm.sadd.sat.i32  (i32, i32)
-declare  i64 @llvm.sadd.sat.i64  (i64, i64)
-declare  <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32>, <4 x i32>)
+declare i4 @llvm.sadd.sat.i4(i4, i4)
+declare i8 @llvm.sadd.sat.i8(i8, i8)
+declare i16 @llvm.sadd.sat.i16(i16, i16)
+declare i32 @llvm.sadd.sat.i32(i32, i32)
+declare i64 @llvm.sadd.sat.i64(i64, i64)
+declare <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32>, <4 x i32>)
 
 define i32 @func(i32 %x, i32 %y) nounwind {
 ; CHECK-LABEL: func:
@@ -34,17 +36,50 @@ define i64 @func2(i64 %x, i64 %y) nounwind {
   ret i64 %tmp;
 }
 
+define i16 @func16(i16 %x, i16 %y) nounwind {
+; CHECK-LABEL: func16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sxth w8, w0
+; CHECK-NEXT:    mov w9, #32767
+; CHECK-NEXT:    add w8, w8, w1, sxth
+; CHECK-NEXT:    cmp w8, w9
+; CHECK-NEXT:    csel w8, w8, w9, lt
+; CHECK-NEXT:    cmn w8, #8, lsl #12 // =32768
+; CHECK-NEXT:    mov w9, #-32768
+; CHECK-NEXT:    csel w0, w8, w9, gt
+; CHECK-NEXT:    ret
+  %tmp = call i16 @llvm.sadd.sat.i16(i16 %x, i16 %y);
+  ret i16 %tmp;
+}
+
+define i8 @func8(i8 %x, i8 %y) nounwind {
+; CHECK-LABEL: func8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sxtb w8, w0
+; CHECK-NEXT:    add w8, w8, w1, sxtb
+; CHECK-NEXT:    mov w9, #127
+; CHECK-NEXT:    cmp w8, #127 // =127
+; CHECK-NEXT:    csel w8, w8, w9, lt
+; CHECK-NEXT:    cmn w8, #128 // =128
+; CHECK-NEXT:    mov w9, #-128
+; CHECK-NEXT:    csel w0, w8, w9, gt
+; CHECK-NEXT:    ret
+  %tmp = call i8 @llvm.sadd.sat.i8(i8 %x, i8 %y);
+  ret i8 %tmp;
+}
+
 define i4 @func3(i4 %x, i4 %y) nounwind {
 ; CHECK-LABEL: func3:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    lsl w8, w0, #28
-; CHECK-NEXT:    adds w10, w8, w1, lsl #28
-; CHECK-NEXT:    mov w9, #2147483647
-; CHECK-NEXT:    cmp w10, #0 // =0
-; CHECK-NEXT:    cinv w9, w9, ge
-; CHECK-NEXT:    adds w8, w8, w1, lsl #28
-; CHECK-NEXT:    csel w8, w9, w8, vs
-; CHECK-NEXT:    asr w0, w8, #28
+; CHECK-NEXT:    lsl w8, w1, #28
+; CHECK-NEXT:    sbfx w9, w0, #0, #4
+; CHECK-NEXT:    add w8, w9, w8, asr #28
+; CHECK-NEXT:    mov w10, #7
+; CHECK-NEXT:    cmp w8, #7 // =7
+; CHECK-NEXT:    csel w8, w8, w10, lt
+; CHECK-NEXT:    cmn w8, #8 // =8
+; CHECK-NEXT:    mov w9, #-8
+; CHECK-NEXT:    csel w0, w8, w9, gt
 ; CHECK-NEXT:    ret
   %tmp = call i4 @llvm.sadd.sat.i4(i4 %x, i4 %y);
   ret i4 %tmp;
@@ -53,19 +88,7 @@ define i4 @func3(i4 %x, i4 %y) nounwind {
 define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; CHECK-LABEL: vec:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    add v2.4s, v0.4s, v1.4s
-; CHECK-NEXT:    cmge v1.4s, v1.4s, #0
-; CHECK-NEXT:    cmge v0.4s, v0.4s, #0
-; CHECK-NEXT:    cmge v5.4s, v2.4s, #0
-; CHECK-NEXT:    cmlt v4.4s, v2.4s, #0
-; CHECK-NEXT:    cmeq v1.4s, v0.4s, v1.4s
-; CHECK-NEXT:    cmeq v0.4s, v0.4s, v5.4s
-; CHECK-NEXT:    mvni v3.4s, #128, lsl #24
-; CHECK-NEXT:    mvn v5.16b, v4.16b
-; CHECK-NEXT:    mvn v0.16b, v0.16b
-; CHECK-NEXT:    bsl v3.16b, v4.16b, v5.16b
-; CHECK-NEXT:    and v0.16b, v1.16b, v0.16b
-; CHECK-NEXT:    bsl v0.16b, v3.16b, v2.16b
+; CHECK-NEXT:    sqadd v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    ret
   %tmp = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> %x, <4 x i32> %y);
   ret <4 x i32> %tmp;
diff --git a/llvm/test/CodeGen/AArch64/sadd_sat_plus.ll b/llvm/test/CodeGen/AArch64/sadd_sat_plus.ll
new file mode 100644
index 000000000..4acda701b
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sadd_sat_plus.ll
@@ -0,0 +1,94 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=aarch64-- | FileCheck %s
+
+declare i4 @llvm.sadd.sat.i4(i4, i4)
+declare i8 @llvm.sadd.sat.i8(i8, i8)
+declare i16 @llvm.sadd.sat.i16(i16, i16)
+declare i32 @llvm.sadd.sat.i32(i32, i32)
+declare i64 @llvm.sadd.sat.i64(i64, i64)
+
+define i32 @func32(i32 %x, i32 %y, i32 %z) nounwind {
+; CHECK-LABEL: func32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mul w8, w1, w2
+; CHECK-NEXT:    adds w10, w0, w8
+; CHECK-NEXT:    mov w9, #2147483647
+; CHECK-NEXT:    cmp w10, #0 // =0
+; CHECK-NEXT:    cinv w9, w9, ge
+; CHECK-NEXT:    adds w8, w0, w8
+; CHECK-NEXT:    csel w0, w9, w8, vs
+; CHECK-NEXT:    ret
+  %a = mul i32 %y, %z
+  %tmp = call i32 @llvm.sadd.sat.i32(i32 %x, i32 %a)
+  ret i32 %tmp
+}
+
+define i64 @func64(i64 %x, i64 %y, i64 %z) nounwind {
+; CHECK-LABEL: func64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adds x8, x0, x2
+; CHECK-NEXT:    mov x9, #9223372036854775807
+; CHECK-NEXT:    cmp x8, #0 // =0
+; CHECK-NEXT:    cinv x8, x9, ge
+; CHECK-NEXT:    adds x9, x0, x2
+; CHECK-NEXT:    csel x0, x8, x9, vs
+; CHECK-NEXT:    ret
+  %a = mul i64 %y, %z
+  %tmp = call i64 @llvm.sadd.sat.i64(i64 %x, i64 %z)
+  ret i64 %tmp
+}
+
+define i16 @func16(i16 %x, i16 %y, i16 %z) nounwind {
+; CHECK-LABEL: func16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sxth w8, w0
+; CHECK-NEXT:    mul w9, w1, w2
+; CHECK-NEXT:    mov w10, #32767
+; CHECK-NEXT:    add w8, w8, w9, sxth
+; CHECK-NEXT:    cmp w8, w10
+; CHECK-NEXT:    csel w8, w8, w10, lt
+; CHECK-NEXT:    cmn w8, #8, lsl #12 // =32768
+; CHECK-NEXT:    mov w9, #-32768
+; CHECK-NEXT:    csel w0, w8, w9, gt
+; CHECK-NEXT:    ret
+  %a = mul i16 %y, %z
+  %tmp = call i16 @llvm.sadd.sat.i16(i16 %x, i16 %a)
+  ret i16 %tmp
+}
+
+define i8 @func8(i8 %x, i8 %y, i8 %z) nounwind {
+; CHECK-LABEL: func8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sxtb w8, w0
+; CHECK-NEXT:    mul w9, w1, w2
+; CHECK-NEXT:    add w8, w8, w9, sxtb
+; CHECK-NEXT:    mov w10, #127
+; CHECK-NEXT:    cmp w8, #127 // =127
+; CHECK-NEXT:    csel w8, w8, w10, lt
+; CHECK-NEXT:    cmn w8, #128 // =128
+; CHECK-NEXT:    mov w9, #-128
+; CHECK-NEXT:    csel w0, w8, w9, gt
+; CHECK-NEXT:    ret
+  %a = mul i8 %y, %z
+  %tmp = call i8 @llvm.sadd.sat.i8(i8 %x, i8 %a)
+  ret i8 %tmp
+}
+
+define i4 @func4(i4 %x, i4 %y, i4 %z) nounwind {
+; CHECK-LABEL: func4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mul w9, w1, w2
+; CHECK-NEXT:    sbfx w8, w0, #0, #4
+; CHECK-NEXT:    lsl w9, w9, #28
+; CHECK-NEXT:    add w8, w8, w9, asr #28
+; CHECK-NEXT:    mov w10, #7
+; CHECK-NEXT:    cmp w8, #7 // =7
+; CHECK-NEXT:    csel w8, w8, w10, lt
+; CHECK-NEXT:    cmn w8, #8 // =8
+; CHECK-NEXT:    mov w9, #-8
+; CHECK-NEXT:    csel w0, w8, w9, gt
+; CHECK-NEXT:    ret
+  %a = mul i4 %y, %z
+  %tmp = call i4 @llvm.sadd.sat.i4(i4 %x, i4 %a)
+  ret i4 %tmp
+}
diff --git a/llvm/test/CodeGen/AArch64/sadd_sat_vec.ll b/llvm/test/CodeGen/AArch64/sadd_sat_vec.ll
index 40b945885..893ed6445 100644
--- a/llvm/test/CodeGen/AArch64/sadd_sat_vec.ll
+++ b/llvm/test/CodeGen/AArch64/sadd_sat_vec.ll
@@ -35,19 +35,7 @@ declare <2 x i128> @llvm.sadd.sat.v2i128(<2 x i128>, <2 x i128>)
 define <16 x i8> @v16i8(<16 x i8> %x, <16 x i8> %y) nounwind {
 ; CHECK-LABEL: v16i8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    add v2.16b, v0.16b, v1.16b
-; CHECK-NEXT:    cmge v1.16b, v1.16b, #0
-; CHECK-NEXT:    cmge v0.16b, v0.16b, #0
-; CHECK-NEXT:    cmge v5.16b, v2.16b, #0
-; CHECK-NEXT:    cmlt v4.16b, v2.16b, #0
-; CHECK-NEXT:    cmeq v1.16b, v0.16b, v1.16b
-; CHECK-NEXT:    cmeq v0.16b, v0.16b, v5.16b
-; CHECK-NEXT:    movi v3.16b, #127
-; CHECK-NEXT:    mvn v5.16b, v4.16b
-; CHECK-NEXT:    mvn v0.16b, v0.16b
-; CHECK-NEXT:    bsl v3.16b, v4.16b, v5.16b
-; CHECK-NEXT:    and v0.16b, v1.16b, v0.16b
-; CHECK-NEXT:    bsl v0.16b, v3.16b, v2.16b
+; CHECK-NEXT:    sqadd v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
   %z = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> %x, <16 x i8> %y)
   ret <16 x i8> %z
@@ -56,32 +44,8 @@ define <16 x i8> @v16i8(<16 x i8> %x, <16 x i8> %y) nounwind {
 define <32 x i8> @v32i8(<32 x i8> %x, <32 x i8> %y) nounwind {
 ; CHECK-LABEL: v32i8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    add v4.16b, v0.16b, v2.16b
-; CHECK-NEXT:    cmlt v16.16b, v4.16b, #0
-; CHECK-NEXT:    movi v6.16b, #127
-; CHECK-NEXT:    add v7.16b, v1.16b, v3.16b
-; CHECK-NEXT:    mvn v17.16b, v16.16b
-; CHECK-NEXT:    bsl v6.16b, v16.16b, v17.16b
-; CHECK-NEXT:    cmlt v16.16b, v7.16b, #0
-; CHECK-NEXT:    movi v5.16b, #127
-; CHECK-NEXT:    mvn v17.16b, v16.16b
-; CHECK-NEXT:    bsl v5.16b, v16.16b, v17.16b
-; CHECK-NEXT:    cmge v2.16b, v2.16b, #0
-; CHECK-NEXT:    cmge v0.16b, v0.16b, #0
-; CHECK-NEXT:    cmge v16.16b, v4.16b, #0
-; CHECK-NEXT:    cmge v3.16b, v3.16b, #0
-; CHECK-NEXT:    cmge v1.16b, v1.16b, #0
-; CHECK-NEXT:    cmeq v2.16b, v0.16b, v2.16b
-; CHECK-NEXT:    cmeq v0.16b, v0.16b, v16.16b
-; CHECK-NEXT:    cmge v16.16b, v7.16b, #0
-; CHECK-NEXT:    cmeq v3.16b, v1.16b, v3.16b
-; CHECK-NEXT:    cmeq v1.16b, v1.16b, v16.16b
-; CHECK-NEXT:    mvn v0.16b, v0.16b
-; CHECK-NEXT:    mvn v1.16b, v1.16b
-; CHECK-NEXT:    and v0.16b, v2.16b, v0.16b
-; CHECK-NEXT:    and v1.16b, v3.16b, v1.16b
-; CHECK-NEXT:    bsl v0.16b, v6.16b, v4.16b
-; CHECK-NEXT:    bsl v1.16b, v5.16b, v7.16b
+; CHECK-NEXT:    sqadd v0.16b, v0.16b, v2.16b
+; CHECK-NEXT:    sqadd v1.16b, v1.16b, v3.16b
 ; CHECK-NEXT:    ret
   %z = call <32 x i8> @llvm.sadd.sat.v32i8(<32 x i8> %x, <32 x i8> %y)
   ret <32 x i8> %z
@@ -90,58 +54,10 @@ define <32 x i8> @v32i8(<32 x i8> %x, <32 x i8> %y) nounwind {
 define <64 x i8> @v64i8(<64 x i8> %x, <64 x i8> %y) nounwind {
 ; CHECK-LABEL: v64i8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    add v16.16b, v0.16b, v4.16b
-; CHECK-NEXT:    cmlt v24.16b, v16.16b, #0
-; CHECK-NEXT:    movi v18.16b, #127
-; CHECK-NEXT:    add v19.16b, v1.16b, v5.16b
-; CHECK-NEXT:    mvn v25.16b, v24.16b
-; CHECK-NEXT:    bsl v18.16b, v24.16b, v25.16b
-; CHECK-NEXT:    cmlt v24.16b, v19.16b, #0
-; CHECK-NEXT:    movi v20.16b, #127
-; CHECK-NEXT:    add v21.16b, v2.16b, v6.16b
-; CHECK-NEXT:    mvn v25.16b, v24.16b
-; CHECK-NEXT:    bsl v20.16b, v24.16b, v25.16b
-; CHECK-NEXT:    cmlt v24.16b, v21.16b, #0
-; CHECK-NEXT:    movi v22.16b, #127
-; CHECK-NEXT:    add v23.16b, v3.16b, v7.16b
-; CHECK-NEXT:    mvn v25.16b, v24.16b
-; CHECK-NEXT:    bsl v22.16b, v24.16b, v25.16b
-; CHECK-NEXT:    cmlt v24.16b, v23.16b, #0
-; CHECK-NEXT:    movi v17.16b, #127
-; CHECK-NEXT:    mvn v25.16b, v24.16b
-; CHECK-NEXT:    bsl v17.16b, v24.16b, v25.16b
-; CHECK-NEXT:    cmge v4.16b, v4.16b, #0
-; CHECK-NEXT:    cmge v0.16b, v0.16b, #0
-; CHECK-NEXT:    cmge v24.16b, v16.16b, #0
-; CHECK-NEXT:    cmge v5.16b, v5.16b, #0
-; CHECK-NEXT:    cmge v1.16b, v1.16b, #0
-; CHECK-NEXT:    cmeq v4.16b, v0.16b, v4.16b
-; CHECK-NEXT:    cmeq v0.16b, v0.16b, v24.16b
-; CHECK-NEXT:    cmge v24.16b, v19.16b, #0
-; CHECK-NEXT:    cmge v6.16b, v6.16b, #0
-; CHECK-NEXT:    cmge v2.16b, v2.16b, #0
-; CHECK-NEXT:    cmeq v5.16b, v1.16b, v5.16b
-; CHECK-NEXT:    cmeq v1.16b, v1.16b, v24.16b
-; CHECK-NEXT:    cmge v24.16b, v21.16b, #0
-; CHECK-NEXT:    cmge v7.16b, v7.16b, #0
-; CHECK-NEXT:    cmge v3.16b, v3.16b, #0
-; CHECK-NEXT:    cmeq v6.16b, v2.16b, v6.16b
-; CHECK-NEXT:    cmeq v2.16b, v2.16b, v24.16b
-; CHECK-NEXT:    cmge v24.16b, v23.16b, #0
-; CHECK-NEXT:    cmeq v7.16b, v3.16b, v7.16b
-; CHECK-NEXT:    cmeq v3.16b, v3.16b, v24.16b
-; CHECK-NEXT:    mvn v0.16b, v0.16b
-; CHECK-NEXT:    mvn v1.16b, v1.16b
-; CHECK-NEXT:    mvn v2.16b, v2.16b
-; CHECK-NEXT:    mvn v3.16b, v3.16b
-; CHECK-NEXT:    and v0.16b, v4.16b, v0.16b
-; CHECK-NEXT:    and v1.16b, v5.16b, v1.16b
-; CHECK-NEXT:    and v2.16b, v6.16b, v2.16b
-; CHECK-NEXT:    and v3.16b, v7.16b, v3.16b
-; CHECK-NEXT:    bsl v0.16b, v18.16b, v16.16b
-; CHECK-NEXT:    bsl v1.16b, v20.16b, v19.16b
-; CHECK-NEXT:    bsl v2.16b, v22.16b, v21.16b
-; CHECK-NEXT:    bsl v3.16b, v17.16b, v23.16b
+; CHECK-NEXT:    sqadd v0.16b, v0.16b, v4.16b
+; CHECK-NEXT:    sqadd v1.16b, v1.16b, v5.16b
+; CHECK-NEXT:    sqadd v2.16b, v2.16b, v6.16b
+; CHECK-NEXT:    sqadd v3.16b, v3.16b, v7.16b
 ; CHECK-NEXT:    ret
   %z = call <64 x i8> @llvm.sadd.sat.v64i8(<64 x i8> %x, <64 x i8> %y)
   ret <64 x i8> %z
@@ -150,19 +66,7 @@ define <64 x i8> @v64i8(<64 x i8> %x, <64 x i8> %y) nounwind {
 define <8 x i16> @v8i16(<8 x i16> %x, <8 x i16> %y) nounwind {
 ; CHECK-LABEL: v8i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    add v2.8h, v0.8h, v1.8h
-; CHECK-NEXT:    cmge v1.8h, v1.8h, #0
-; CHECK-NEXT:    cmge v0.8h, v0.8h, #0
-; CHECK-NEXT:    cmge v5.8h, v2.8h, #0
-; CHECK-NEXT:    cmlt v4.8h, v2.8h, #0
-; CHECK-NEXT:    cmeq v1.8h, v0.8h, v1.8h
-; CHECK-NEXT:    cmeq v0.8h, v0.8h, v5.8h
-; CHECK-NEXT:    mvni v3.8h, #128, lsl #8
-; CHECK-NEXT:    mvn v5.16b, v4.16b
-; CHECK-NEXT:    mvn v0.16b, v0.16b
-; CHECK-NEXT:    bsl v3.16b, v4.16b, v5.16b
-; CHECK-NEXT:    and v0.16b, v1.16b, v0.16b
-; CHECK-NEXT:    bsl v0.16b, v3.16b, v2.16b
+; CHECK-NEXT:    sqadd v0.8h, v0.8h, v1.8h
 ; CHECK-NEXT:    ret
   %z = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> %x, <8 x i16> %y)
   ret <8 x i16> %z
@@ -171,32 +75,8 @@ define <8 x i16> @v8i16(<8 x i16> %x, <8 x i16> %y) nounwind {
 define <16 x i16> @v16i16(<16 x i16> %x, <16 x i16> %y) nounwind {
 ; CHECK-LABEL: v16i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    add v4.8h, v0.8h, v2.8h
-; CHECK-NEXT:    cmlt v16.8h, v4.8h, #0
-; CHECK-NEXT:    mvni v6.8h, #128, lsl #8
-; CHECK-NEXT:    add v7.8h, v1.8h, v3.8h
-; CHECK-NEXT:    mvn v17.16b, v16.16b
-; CHECK-NEXT:    bsl v6.16b, v16.16b, v17.16b
-; CHECK-NEXT:    cmlt v16.8h, v7.8h, #0
-; CHECK-NEXT:    mvni v5.8h, #128, lsl #8
-; CHECK-NEXT:    mvn v17.16b, v16.16b
-; CHECK-NEXT:    bsl v5.16b, v16.16b, v17.16b
-; CHECK-NEXT:    cmge v2.8h, v2.8h, #0
-; CHECK-NEXT:    cmge v0.8h, v0.8h, #0
-; CHECK-NEXT:    cmge v16.8h, v4.8h, #0
-; CHECK-NEXT:    cmge v3.8h, v3.8h, #0
-; CHECK-NEXT:    cmge v1.8h, v1.8h, #0
-; CHECK-NEXT:    cmeq v2.8h, v0.8h, v2.8h
-; CHECK-NEXT:    cmeq v0.8h, v0.8h, v16.8h
-; CHECK-NEXT:    cmge v16.8h, v7.8h, #0
-; CHECK-NEXT:    cmeq v3.8h, v1.8h, v3.8h
-; CHECK-NEXT:    cmeq v1.8h, v1.8h, v16.8h
-; CHECK-NEXT:    mvn v0.16b, v0.16b
-; CHECK-NEXT:    mvn v1.16b, v1.16b
-; CHECK-NEXT:    and v0.16b, v2.16b, v0.16b
-; CHECK-NEXT:    and v1.16b, v3.16b, v1.16b
-; CHECK-NEXT:    bsl v0.16b, v6.16b, v4.16b
-; CHECK-NEXT:    bsl v1.16b, v5.16b, v7.16b
+; CHECK-NEXT:    sqadd v0.8h, v0.8h, v2.8h
+; CHECK-NEXT:    sqadd v1.8h, v1.8h, v3.8h
 ; CHECK-NEXT:    ret
   %z = call <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16> %x, <16 x i16> %y)
   ret <16 x i16> %z
@@ -205,58 +85,10 @@ define <16 x i16> @v16i16(<16 x i16> %x, <16 x i16> %y) nounwind {
 define <32 x i16> @v32i16(<32 x i16> %x, <32 x i16> %y) nounwind {
 ; CHECK-LABEL: v32i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    add v16.8h, v0.8h, v4.8h
-; CHECK-NEXT:    cmlt v24.8h, v16.8h, #0
-; CHECK-NEXT:    mvni v18.8h, #128, lsl #8
-; CHECK-NEXT:    add v19.8h, v1.8h, v5.8h
-; CHECK-NEXT:    mvn v25.16b, v24.16b
-; CHECK-NEXT:    bsl v18.16b, v24.16b, v25.16b
-; CHECK-NEXT:    cmlt v24.8h, v19.8h, #0
-; CHECK-NEXT:    mvni v20.8h, #128, lsl #8
-; CHECK-NEXT:    add v21.8h, v2.8h, v6.8h
-; CHECK-NEXT:    mvn v25.16b, v24.16b
-; CHECK-NEXT:    bsl v20.16b, v24.16b, v25.16b
-; CHECK-NEXT:    cmlt v24.8h, v21.8h, #0
-; CHECK-NEXT:    mvni v22.8h, #128, lsl #8
-; CHECK-NEXT:    add v23.8h, v3.8h, v7.8h
-; CHECK-NEXT:    mvn v25.16b, v24.16b
-; CHECK-NEXT:    bsl v22.16b, v24.16b, v25.16b
-; CHECK-NEXT:    cmlt v24.8h, v23.8h, #0
-; CHECK-NEXT:    mvni v17.8h, #128, lsl #8
-; CHECK-NEXT:    mvn v25.16b, v24.16b
-; CHECK-NEXT:    bsl v17.16b, v24.16b, v25.16b
-; CHECK-NEXT:    cmge v4.8h, v4.8h, #0
-; CHECK-NEXT:    cmge v0.8h, v0.8h, #0
-; CHECK-NEXT:    cmge v24.8h, v16.8h, #0
-; CHECK-NEXT:    cmge v5.8h, v5.8h, #0
-; CHECK-NEXT:    cmge v1.8h, v1.8h, #0
-; CHECK-NEXT:    cmeq v4.8h, v0.8h, v4.8h
-; CHECK-NEXT:    cmeq v0.8h, v0.8h, v24.8h
-; CHECK-NEXT:    cmge v24.8h, v19.8h, #0
-; CHECK-NEXT:    cmge v6.8h, v6.8h, #0
-; CHECK-NEXT:    cmge v2.8h, v2.8h, #0
-; CHECK-NEXT:    cmeq v5.8h, v1.8h, v5.8h
-; CHECK-NEXT:    cmeq v1.8h, v1.8h, v24.8h
-; CHECK-NEXT:    cmge v24.8h, v21.8h, #0
-; CHECK-NEXT:    cmge v7.8h, v7.8h, #0
-; CHECK-NEXT:    cmge v3.8h, v3.8h, #0
-; CHECK-NEXT:    cmeq v6.8h, v2.8h, v6.8h
-; CHECK-NEXT:    cmeq v2.8h, v2.8h, v24.8h
-; CHECK-NEXT:    cmge v24.8h, v23.8h, #0
-; CHECK-NEXT:    cmeq v7.8h, v3.8h, v7.8h
-; CHECK-NEXT:    cmeq v3.8h, v3.8h, v24.8h
-; CHECK-NEXT:    mvn v0.16b, v0.16b
-; CHECK-NEXT:    mvn v1.16b, v1.16b
-; CHECK-NEXT:    mvn v2.16b, v2.16b
-; CHECK-NEXT:    mvn v3.16b, v3.16b
-; CHECK-NEXT:    and v0.16b, v4.16b, v0.16b
-; CHECK-NEXT:    and v1.16b, v5.16b, v1.16b
-; CHECK-NEXT:    and v2.16b, v6.16b, v2.16b
-; CHECK-NEXT:    and v3.16b, v7.16b, v3.16b
-; CHECK-NEXT:    bsl v0.16b, v18.16b, v16.16b
-; CHECK-NEXT:    bsl v1.16b, v20.16b, v19.16b
-; CHECK-NEXT:    bsl v2.16b, v22.16b, v21.16b
-; CHECK-NEXT:    bsl v3.16b, v17.16b, v23.16b
+; CHECK-NEXT:    sqadd v0.8h, v0.8h, v4.8h
+; CHECK-NEXT:    sqadd v1.8h, v1.8h, v5.8h
+; CHECK-NEXT:    sqadd v2.8h, v2.8h, v6.8h
+; CHECK-NEXT:    sqadd v3.8h, v3.8h, v7.8h
 ; CHECK-NEXT:    ret
   %z = call <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16> %x, <32 x i16> %y)
   ret <32 x i16> %z
@@ -267,19 +99,7 @@ define void @v8i8(<8 x i8>* %px, <8 x i8>* %py, <8 x i8>* %pz) nounwind {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldr d0, [x0]
 ; CHECK-NEXT:    ldr d1, [x1]
-; CHECK-NEXT:    movi v2.8b, #127
-; CHECK-NEXT:    add v3.8b, v0.8b, v1.8b
-; CHECK-NEXT:    cmge v1.8b, v1.8b, #0
-; CHECK-NEXT:    cmge v0.8b, v0.8b, #0
-; CHECK-NEXT:    cmge v5.8b, v3.8b, #0
-; CHECK-NEXT:    cmlt v4.8b, v3.8b, #0
-; CHECK-NEXT:    cmeq v1.8b, v0.8b, v1.8b
-; CHECK-NEXT:    cmeq v0.8b, v0.8b, v5.8b
-; CHECK-NEXT:    mvn v5.8b, v4.8b
-; CHECK-NEXT:    mvn v0.8b, v0.8b
-; CHECK-NEXT:    bsl v2.8b, v4.8b, v5.8b
-; CHECK-NEXT:    and v0.8b, v1.8b, v0.8b
-; CHECK-NEXT:    bsl v0.8b, v2.8b, v3.8b
+; CHECK-NEXT:    sqadd v0.8b, v0.8b, v1.8b
 ; CHECK-NEXT:    str d0, [x2]
 ; CHECK-NEXT:    ret
   %x = load <8 x i8>, <8 x i8>* %px
@@ -292,37 +112,25 @@ define void @v8i8(<8 x i8>* %px, <8 x i8>* %py, <8 x i8>* %pz) nounwind {
 define void @v4i8(<4 x i8>* %px, <4 x i8>* %py, <4 x i8>* %pz) nounwind {
 ; CHECK-LABEL: v4i8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldrb w8, [x0]
-; CHECK-NEXT:    ldrb w9, [x1]
-; CHECK-NEXT:    ldrb w10, [x0, #1]
-; CHECK-NEXT:    ldrb w11, [x1, #1]
-; CHECK-NEXT:    ldrb w12, [x0, #2]
+; CHECK-NEXT:    ldrsb w8, [x0]
+; CHECK-NEXT:    ldrsb w9, [x1]
+; CHECK-NEXT:    ldrsb w10, [x0, #1]
+; CHECK-NEXT:    ldrsb w11, [x1, #1]
 ; CHECK-NEXT:    fmov s0, w8
-; CHECK-NEXT:    ldrb w8, [x1, #2]
 ; CHECK-NEXT:    fmov s1, w9
+; CHECK-NEXT:    ldrsb w8, [x0, #2]
+; CHECK-NEXT:    ldrsb w9, [x1, #2]
 ; CHECK-NEXT:    mov v0.h[1], w10
-; CHECK-NEXT:    ldrb w9, [x0, #3]
-; CHECK-NEXT:    ldrb w10, [x1, #3]
 ; CHECK-NEXT:    mov v1.h[1], w11
-; CHECK-NEXT:    mov v0.h[2], w12
-; CHECK-NEXT:    mov v1.h[2], w8
-; CHECK-NEXT:    mov v0.h[3], w9
-; CHECK-NEXT:    mov v1.h[3], w10
+; CHECK-NEXT:    ldrsb w10, [x0, #3]
+; CHECK-NEXT:    ldrsb w11, [x1, #3]
+; CHECK-NEXT:    mov v0.h[2], w8
+; CHECK-NEXT:    mov v1.h[2], w9
+; CHECK-NEXT:    mov v0.h[3], w10
+; CHECK-NEXT:    mov v1.h[3], w11
 ; CHECK-NEXT:    shl v1.4h, v1.4h, #8
 ; CHECK-NEXT:    shl v0.4h, v0.4h, #8
-; CHECK-NEXT:    add v3.4h, v0.4h, v1.4h
-; CHECK-NEXT:    cmge v1.4h, v1.4h, #0
-; CHECK-NEXT:    cmge v0.4h, v0.4h, #0
-; CHECK-NEXT:    cmge v5.4h, v3.4h, #0
-; CHECK-NEXT:    cmlt v4.4h, v3.4h, #0
-; CHECK-NEXT:    cmeq v1.4h, v0.4h, v1.4h
-; CHECK-NEXT:    cmeq v0.4h, v0.4h, v5.4h
-; CHECK-NEXT:    mvni v2.4h, #128, lsl #8
-; CHECK-NEXT:    mvn v5.8b, v4.8b
-; CHECK-NEXT:    mvn v0.8b, v0.8b
-; CHECK-NEXT:    bsl v2.8b, v4.8b, v5.8b
-; CHECK-NEXT:    and v0.8b, v1.8b, v0.8b
-; CHECK-NEXT:    bsl v0.8b, v2.8b, v3.8b
+; CHECK-NEXT:    sqadd v0.4h, v0.4h, v1.4h
 ; CHECK-NEXT:    sshr v0.4h, v0.4h, #8
 ; CHECK-NEXT:    xtn v0.8b, v0.8h
 ; CHECK-NEXT:    str s0, [x2]
@@ -342,24 +150,12 @@ define void @v2i8(<2 x i8>* %px, <2 x i8>* %py, <2 x i8>* %pz) nounwind {
 ; CHECK-NEXT:    ldrb w10, [x0, #1]
 ; CHECK-NEXT:    ldrb w11, [x1, #1]
 ; CHECK-NEXT:    fmov s0, w8
-; CHECK-NEXT:    fmov s2, w9
+; CHECK-NEXT:    fmov s1, w9
 ; CHECK-NEXT:    mov v0.s[1], w10
-; CHECK-NEXT:    mov v2.s[1], w11
-; CHECK-NEXT:    shl v2.2s, v2.2s, #24
+; CHECK-NEXT:    mov v1.s[1], w11
+; CHECK-NEXT:    shl v1.2s, v1.2s, #24
 ; CHECK-NEXT:    shl v0.2s, v0.2s, #24
-; CHECK-NEXT:    add v3.2s, v0.2s, v2.2s
-; CHECK-NEXT:    cmge v2.2s, v2.2s, #0
-; CHECK-NEXT:    cmge v0.2s, v0.2s, #0
-; CHECK-NEXT:    cmge v5.2s, v3.2s, #0
-; CHECK-NEXT:    cmlt v4.2s, v3.2s, #0
-; CHECK-NEXT:    cmeq v2.2s, v0.2s, v2.2s
-; CHECK-NEXT:    cmeq v0.2s, v0.2s, v5.2s
-; CHECK-NEXT:    mvni v1.2s, #128, lsl #24
-; CHECK-NEXT:    mvn v5.8b, v4.8b
-; CHECK-NEXT:    mvn v0.8b, v0.8b
-; CHECK-NEXT:    bsl v1.8b, v4.8b, v5.8b
-; CHECK-NEXT:    and v0.8b, v2.8b, v0.8b
-; CHECK-NEXT:    bsl v0.8b, v1.8b, v3.8b
+; CHECK-NEXT:    sqadd v0.2s, v0.2s, v1.2s
 ; CHECK-NEXT:    ushr v0.2s, v0.2s, #24
 ; CHECK-NEXT:    mov w8, v0.s[1]
 ; CHECK-NEXT:    fmov w9, s0
@@ -378,19 +174,7 @@ define void @v4i16(<4 x i16>* %px, <4 x i16>* %py, <4 x i16>* %pz) nounwind {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldr d0, [x0]
 ; CHECK-NEXT:    ldr d1, [x1]
-; CHECK-NEXT:    mvni v2.4h, #128, lsl #8
-; CHECK-NEXT:    add v3.4h, v0.4h, v1.4h
-; CHECK-NEXT:    cmge v1.4h, v1.4h, #0
-; CHECK-NEXT:    cmge v0.4h, v0.4h, #0
-; CHECK-NEXT:    cmge v5.4h, v3.4h, #0
-; CHECK-NEXT:    cmlt v4.4h, v3.4h, #0
-; CHECK-NEXT:    cmeq v1.4h, v0.4h, v1.4h
-; CHECK-NEXT:    cmeq v0.4h, v0.4h, v5.4h
-; CHECK-NEXT:    mvn v5.8b, v4.8b
-; CHECK-NEXT:    mvn v0.8b, v0.8b
-; CHECK-NEXT:    bsl v2.8b, v4.8b, v5.8b
-; CHECK-NEXT:    and v0.8b, v1.8b, v0.8b
-; CHECK-NEXT:    bsl v0.8b, v2.8b, v3.8b
+; CHECK-NEXT:    sqadd v0.4h, v0.4h, v1.4h
 ; CHECK-NEXT:    str d0, [x2]
 ; CHECK-NEXT:    ret
   %x = load <4 x i16>, <4 x i16>* %px
@@ -408,24 +192,12 @@ define void @v2i16(<2 x i16>* %px, <2 x i16>* %py, <2 x i16>* %pz) nounwind {
 ; CHECK-NEXT:    ldrh w10, [x0, #2]
 ; CHECK-NEXT:    ldrh w11, [x1, #2]
 ; CHECK-NEXT:    fmov s0, w8
-; CHECK-NEXT:    fmov s2, w9
+; CHECK-NEXT:    fmov s1, w9
 ; CHECK-NEXT:    mov v0.s[1], w10
-; CHECK-NEXT:    mov v2.s[1], w11
-; CHECK-NEXT:    shl v2.2s, v2.2s, #16
+; CHECK-NEXT:    mov v1.s[1], w11
+; CHECK-NEXT:    shl v1.2s, v1.2s, #16
 ; CHECK-NEXT:    shl v0.2s, v0.2s, #16
-; CHECK-NEXT:    add v3.2s, v0.2s, v2.2s
-; CHECK-NEXT:    cmge v2.2s, v2.2s, #0
-; CHECK-NEXT:    cmge v0.2s, v0.2s, #0
-; CHECK-NEXT:    cmge v5.2s, v3.2s, #0
-; CHECK-NEXT:    cmlt v4.2s, v3.2s, #0
-; CHECK-NEXT:    cmeq v2.2s, v0.2s, v2.2s
-; CHECK-NEXT:    cmeq v0.2s, v0.2s, v5.2s
-; CHECK-NEXT:    mvni v1.2s, #128, lsl #24
-; CHECK-NEXT:    mvn v5.8b, v4.8b
-; CHECK-NEXT:    mvn v0.8b, v0.8b
-; CHECK-NEXT:    bsl v1.8b, v4.8b, v5.8b
-; CHECK-NEXT:    and v0.8b, v2.8b, v0.8b
-; CHECK-NEXT:    bsl v0.8b, v1.8b, v3.8b
+; CHECK-NEXT:    sqadd v0.2s, v0.2s, v1.2s
 ; CHECK-NEXT:    ushr v0.2s, v0.2s, #16
 ; CHECK-NEXT:    mov w8, v0.s[1]
 ; CHECK-NEXT:    fmov w9, s0
@@ -442,19 +214,7 @@ define void @v2i16(<2 x i16>* %px, <2 x i16>* %py, <2 x i16>* %pz) nounwind {
 define <12 x i8> @v12i8(<12 x i8> %x, <12 x i8> %y) nounwind {
 ; CHECK-LABEL: v12i8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    add v2.16b, v0.16b, v1.16b
-; CHECK-NEXT:    cmge v1.16b, v1.16b, #0
-; CHECK-NEXT:    cmge v0.16b, v0.16b, #0
-; CHECK-NEXT:    cmge v5.16b, v2.16b, #0
-; CHECK-NEXT:    cmlt v4.16b, v2.16b, #0
-; CHECK-NEXT:    cmeq v1.16b, v0.16b, v1.16b
-; CHECK-NEXT:    cmeq v0.16b, v0.16b, v5.16b
-; CHECK-NEXT:    movi v3.16b, #127
-; CHECK-NEXT:    mvn v5.16b, v4.16b
-; CHECK-NEXT:    mvn v0.16b, v0.16b
-; CHECK-NEXT:    bsl v3.16b, v4.16b, v5.16b
-; CHECK-NEXT:    and v0.16b, v1.16b, v0.16b
-; CHECK-NEXT:    bsl v0.16b, v3.16b, v2.16b
+; CHECK-NEXT:    sqadd v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
   %z = call <12 x i8> @llvm.sadd.sat.v12i8(<12 x i8> %x, <12 x i8> %y)
   ret <12 x i8> %z
@@ -465,32 +225,8 @@ define void @v12i16(<12 x i16>* %px, <12 x i16>* %py, <12 x i16>* %pz) nounwind
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldp q0, q1, [x0]
 ; CHECK-NEXT:    ldp q3, q2, [x1]
-; CHECK-NEXT:    mvni v5.8h, #128, lsl #8
-; CHECK-NEXT:    mvni v4.8h, #128, lsl #8
-; CHECK-NEXT:    add v6.8h, v1.8h, v2.8h
-; CHECK-NEXT:    cmlt v16.8h, v6.8h, #0
-; CHECK-NEXT:    add v7.8h, v0.8h, v3.8h
-; CHECK-NEXT:    mvn v17.16b, v16.16b
-; CHECK-NEXT:    bsl v5.16b, v16.16b, v17.16b
-; CHECK-NEXT:    cmlt v16.8h, v7.8h, #0
-; CHECK-NEXT:    mvn v17.16b, v16.16b
-; CHECK-NEXT:    bsl v4.16b, v16.16b, v17.16b
-; CHECK-NEXT:    cmge v2.8h, v2.8h, #0
-; CHECK-NEXT:    cmge v1.8h, v1.8h, #0
-; CHECK-NEXT:    cmge v16.8h, v6.8h, #0
-; CHECK-NEXT:    cmge v3.8h, v3.8h, #0
-; CHECK-NEXT:    cmge v0.8h, v0.8h, #0
-; CHECK-NEXT:    cmeq v2.8h, v1.8h, v2.8h
-; CHECK-NEXT:    cmeq v1.8h, v1.8h, v16.8h
-; CHECK-NEXT:    cmge v16.8h, v7.8h, #0
-; CHECK-NEXT:    cmeq v3.8h, v0.8h, v3.8h
-; CHECK-NEXT:    cmeq v0.8h, v0.8h, v16.8h
-; CHECK-NEXT:    mvn v1.16b, v1.16b
-; CHECK-NEXT:    mvn v0.16b, v0.16b
-; CHECK-NEXT:    and v1.16b, v2.16b, v1.16b
-; CHECK-NEXT:    and v0.16b, v3.16b, v0.16b
-; CHECK-NEXT:    bsl v1.16b, v5.16b, v6.16b
-; CHECK-NEXT:    bsl v0.16b, v4.16b, v7.16b
+; CHECK-NEXT:    sqadd v1.8h, v1.8h, v2.8h
+; CHECK-NEXT:    sqadd v0.8h, v0.8h, v3.8h
 ; CHECK-NEXT:    str q0, [x2]
 ; CHECK-NEXT:    str d1, [x2, #16]
 ; CHECK-NEXT:    ret
@@ -506,19 +242,7 @@ define void @v1i8(<1 x i8>* %px, <1 x i8>* %py, <1 x i8>* %pz) nounwind {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldr b0, [x0]
 ; CHECK-NEXT:    ldr b1, [x1]
-; CHECK-NEXT:    movi v2.8b, #127
-; CHECK-NEXT:    add v3.8b, v0.8b, v1.8b
-; CHECK-NEXT:    cmge v1.8b, v1.8b, #0
-; CHECK-NEXT:    cmge v0.8b, v0.8b, #0
-; CHECK-NEXT:    cmge v5.8b, v3.8b, #0
-; CHECK-NEXT:    cmlt v4.8b, v3.8b, #0
-; CHECK-NEXT:    cmeq v1.8b, v0.8b, v1.8b
-; CHECK-NEXT:    cmeq v0.8b, v0.8b, v5.8b
-; CHECK-NEXT:    mvn v5.8b, v4.8b
-; CHECK-NEXT:    mvn v0.8b, v0.8b
-; CHECK-NEXT:    bsl v2.8b, v4.8b, v5.8b
-; CHECK-NEXT:    and v0.8b, v1.8b, v0.8b
-; CHECK-NEXT:    bsl v0.8b, v2.8b, v3.8b
+; CHECK-NEXT:    sqadd v0.8b, v0.8b, v1.8b
 ; CHECK-NEXT:    st1 { v0.b }[0], [x2]
 ; CHECK-NEXT:    ret
   %x = load <1 x i8>, <1 x i8>* %px
@@ -533,19 +257,7 @@ define void @v1i16(<1 x i16>* %px, <1 x i16>* %py, <1 x i16>* %pz) nounwind {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldr h0, [x0]
 ; CHECK-NEXT:    ldr h1, [x1]
-; CHECK-NEXT:    mvni v2.4h, #128, lsl #8
-; CHECK-NEXT:    add v3.4h, v0.4h, v1.4h
-; CHECK-NEXT:    cmge v1.4h, v1.4h, #0
-; CHECK-NEXT:    cmge v0.4h, v0.4h, #0
-; CHECK-NEXT:    cmge v5.4h, v3.4h, #0
-; CHECK-NEXT:    cmlt v4.4h, v3.4h, #0
-; CHECK-NEXT:    cmeq v1.4h, v0.4h, v1.4h
-; CHECK-NEXT:    cmeq v0.4h, v0.4h, v5.4h
-; CHECK-NEXT:    mvn v5.8b, v4.8b
-; CHECK-NEXT:    mvn v0.8b, v0.8b
-; CHECK-NEXT:    bsl v2.8b, v4.8b, v5.8b
-; CHECK-NEXT:    and v0.8b, v1.8b, v0.8b
-; CHECK-NEXT:    bsl v0.8b, v2.8b, v3.8b
+; CHECK-NEXT:    sqadd v0.4h, v0.4h, v1.4h
 ; CHECK-NEXT:    str h0, [x2]
 ; CHECK-NEXT:    ret
   %x = load <1 x i16>, <1 x i16>* %px
@@ -558,21 +270,13 @@ define void @v1i16(<1 x i16>* %px, <1 x i16>* %py, <1 x i16>* %pz) nounwind {
 define <16 x i4> @v16i4(<16 x i4> %x, <16 x i4> %y) nounwind {
 ; CHECK-LABEL: v16i4:
 ; CHECK:       // %bb.0:
+; CHECK-NEXT:    shl v0.16b, v0.16b, #4
+; CHECK-NEXT:    shl v1.16b, v1.16b, #4
+; CHECK-NEXT:    sshr v0.16b, v0.16b, #4
+; CHECK-NEXT:    sshr v1.16b, v1.16b, #4
 ; CHECK-NEXT:    shl v1.16b, v1.16b, #4
 ; CHECK-NEXT:    shl v0.16b, v0.16b, #4
-; CHECK-NEXT:    add v3.16b, v0.16b, v1.16b
-; CHECK-NEXT:    cmge v1.16b, v1.16b, #0
-; CHECK-NEXT:    cmge v0.16b, v0.16b, #0
-; CHECK-NEXT:    cmge v5.16b, v3.16b, #0
-; CHECK-NEXT:    cmlt v4.16b, v3.16b, #0
-; CHECK-NEXT:    cmeq v1.16b, v0.16b, v1.16b
-; CHECK-NEXT:    cmeq v0.16b, v0.16b, v5.16b
-; CHECK-NEXT:    movi v2.16b, #127
-; CHECK-NEXT:    mvn v5.16b, v4.16b
-; CHECK-NEXT:    mvn v0.16b, v0.16b
-; CHECK-NEXT:    bsl v2.16b, v4.16b, v5.16b
-; CHECK-NEXT:    and v0.16b, v1.16b, v0.16b
-; CHECK-NEXT:    bsl v0.16b, v2.16b, v3.16b
+; CHECK-NEXT:    sqadd v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    sshr v0.16b, v0.16b, #4
 ; CHECK-NEXT:    ret
   %z = call <16 x i4> @llvm.sadd.sat.v16i4(<16 x i4> %x, <16 x i4> %y)
@@ -582,21 +286,13 @@ define <16 x i4> @v16i4(<16 x i4> %x, <16 x i4> %y) nounwind {
 define <16 x i1> @v16i1(<16 x i1> %x, <16 x i1> %y) nounwind {
 ; CHECK-LABEL: v16i1:
 ; CHECK:       // %bb.0:
+; CHECK-NEXT:    shl v0.16b, v0.16b, #7
+; CHECK-NEXT:    shl v1.16b, v1.16b, #7
+; CHECK-NEXT:    sshr v0.16b, v0.16b, #7
+; CHECK-NEXT:    sshr v1.16b, v1.16b, #7
 ; CHECK-NEXT:    shl v1.16b, v1.16b, #7
 ; CHECK-NEXT:    shl v0.16b, v0.16b, #7
-; CHECK-NEXT:    add v3.16b, v0.16b, v1.16b
-; CHECK-NEXT:    cmge v1.16b, v1.16b, #0
-; CHECK-NEXT:    cmge v0.16b, v0.16b, #0
-; CHECK-NEXT:    cmge v5.16b, v3.16b, #0
-; CHECK-NEXT:    cmlt v4.16b, v3.16b, #0
-; CHECK-NEXT:    cmeq v1.16b, v0.16b, v1.16b
-; CHECK-NEXT:    cmeq v0.16b, v0.16b, v5.16b
-; CHECK-NEXT:    movi v2.16b, #127
-; CHECK-NEXT:    mvn v5.16b, v4.16b
-; CHECK-NEXT:    mvn v0.16b, v0.16b
-; CHECK-NEXT:    bsl v2.16b, v4.16b, v5.16b
-; CHECK-NEXT:    and v0.16b, v1.16b, v0.16b
-; CHECK-NEXT:    bsl v0.16b, v2.16b, v3.16b
+; CHECK-NEXT:    sqadd v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    sshr v0.16b, v0.16b, #7
 ; CHECK-NEXT:    ret
   %z = call <16 x i1> @llvm.sadd.sat.v16i1(<16 x i1> %x, <16 x i1> %y)
@@ -606,19 +302,7 @@ define <16 x i1> @v16i1(<16 x i1> %x, <16 x i1> %y) nounwind {
 define <2 x i32> @v2i32(<2 x i32> %x, <2 x i32> %y) nounwind {
 ; CHECK-LABEL: v2i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    add v2.2s, v0.2s, v1.2s
-; CHECK-NEXT:    cmge v1.2s, v1.2s, #0
-; CHECK-NEXT:    cmge v0.2s, v0.2s, #0
-; CHECK-NEXT:    cmge v5.2s, v2.2s, #0
-; CHECK-NEXT:    cmlt v4.2s, v2.2s, #0
-; CHECK-NEXT:    cmeq v1.2s, v0.2s, v1.2s
-; CHECK-NEXT:    cmeq v0.2s, v0.2s, v5.2s
-; CHECK-NEXT:    mvni v3.2s, #128, lsl #24
-; CHECK-NEXT:    mvn v5.8b, v4.8b
-; CHECK-NEXT:    mvn v0.8b, v0.8b
-; CHECK-NEXT:    bsl v3.8b, v4.8b, v5.8b
-; CHECK-NEXT:    and v0.8b, v1.8b, v0.8b
-; CHECK-NEXT:    bsl v0.8b, v3.8b, v2.8b
+; CHECK-NEXT:    sqadd v0.2s, v0.2s, v1.2s
 ; CHECK-NEXT:    ret
   %z = call <2 x i32> @llvm.sadd.sat.v2i32(<2 x i32> %x, <2 x i32> %y)
   ret <2 x i32> %z
@@ -627,19 +311,7 @@ define <2 x i32> @v2i32(<2 x i32> %x, <2 x i32> %y) nounwind {
 define <4 x i32> @v4i32(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; CHECK-LABEL: v4i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    add v2.4s, v0.4s, v1.4s
-; CHECK-NEXT:    cmge v1.4s, v1.4s, #0
-; CHECK-NEXT:    cmge v0.4s, v0.4s, #0
-; CHECK-NEXT:    cmge v5.4s, v2.4s, #0
-; CHECK-NEXT:    cmlt v4.4s, v2.4s, #0
-; CHECK-NEXT:    cmeq v1.4s, v0.4s, v1.4s
-; CHECK-NEXT:    cmeq v0.4s, v0.4s, v5.4s
-; CHECK-NEXT:    mvni v3.4s, #128, lsl #24
-; CHECK-NEXT:    mvn v5.16b, v4.16b
-; CHECK-NEXT:    mvn v0.16b, v0.16b
-; CHECK-NEXT:    bsl v3.16b, v4.16b, v5.16b
-; CHECK-NEXT:    and v0.16b, v1.16b, v0.16b
-; CHECK-NEXT:    bsl v0.16b, v3.16b, v2.16b
+; CHECK-NEXT:    sqadd v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    ret
   %z = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> %x, <4 x i32> %y)
   ret <4 x i32> %z
@@ -648,32 +320,8 @@ define <4 x i32> @v4i32(<4 x i32> %x, <4 x i32> %y) nounwind {
 define <8 x i32> @v8i32(<8 x i32> %x, <8 x i32> %y) nounwind {
 ; CHECK-LABEL: v8i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    add v4.4s, v0.4s, v2.4s
-; CHECK-NEXT:    cmlt v16.4s, v4.4s, #0
-; CHECK-NEXT:    mvni v6.4s, #128, lsl #24
-; CHECK-NEXT:    add v7.4s, v1.4s, v3.4s
-; CHECK-NEXT:    mvn v17.16b, v16.16b
-; CHECK-NEXT:    bsl v6.16b, v16.16b, v17.16b
-; CHECK-NEXT:    cmlt v16.4s, v7.4s, #0
-; CHECK-NEXT:    mvni v5.4s, #128, lsl #24
-; CHECK-NEXT:    mvn v17.16b, v16.16b
-; CHECK-NEXT:    bsl v5.16b, v16.16b, v17.16b
-; CHECK-NEXT:    cmge v2.4s, v2.4s, #0
-; CHECK-NEXT:    cmge v0.4s, v0.4s, #0
-; CHECK-NEXT:    cmge v16.4s, v4.4s, #0
-; CHECK-NEXT:    cmge v3.4s, v3.4s, #0
-; CHECK-NEXT:    cmge v1.4s, v1.4s, #0
-; CHECK-NEXT:    cmeq v2.4s, v0.4s, v2.4s
-; CHECK-NEXT:    cmeq v0.4s, v0.4s, v16.4s
-; CHECK-NEXT:    cmge v16.4s, v7.4s, #0
-; CHECK-NEXT:    cmeq v3.4s, v1.4s, v3.4s
-; CHECK-NEXT:    cmeq v1.4s, v1.4s, v16.4s
-; CHECK-NEXT:    mvn v0.16b, v0.16b
-; CHECK-NEXT:    mvn v1.16b, v1.16b
-; CHECK-NEXT:    and v0.16b, v2.16b, v0.16b
-; CHECK-NEXT:    and v1.16b, v3.16b, v1.16b
-; CHECK-NEXT:    bsl v0.16b, v6.16b, v4.16b
-; CHECK-NEXT:    bsl v1.16b, v5.16b, v7.16b
+; CHECK-NEXT:    sqadd v0.4s, v0.4s, v2.4s
+; CHECK-NEXT:    sqadd v1.4s, v1.4s, v3.4s
 ; CHECK-NEXT:    ret
   %z = call <8 x i32> @llvm.sadd.sat.v8i32(<8 x i32> %x, <8 x i32> %y)
   ret <8 x i32> %z
@@ -682,58 +330,10 @@ define <8 x i32> @v8i32(<8 x i32> %x, <8 x i32> %y) nounwind {
 define <16 x i32> @v16i32(<16 x i32> %x, <16 x i32> %y) nounwind {
 ; CHECK-LABEL: v16i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    add v16.4s, v0.4s, v4.4s
-; CHECK-NEXT:    cmlt v24.4s, v16.4s, #0
-; CHECK-NEXT:    mvni v18.4s, #128, lsl #24
-; CHECK-NEXT:    add v19.4s, v1.4s, v5.4s
-; CHECK-NEXT:    mvn v25.16b, v24.16b
-; CHECK-NEXT:    bsl v18.16b, v24.16b, v25.16b
-; CHECK-NEXT:    cmlt v24.4s, v19.4s, #0
-; CHECK-NEXT:    mvni v20.4s, #128, lsl #24
-; CHECK-NEXT:    add v21.4s, v2.4s, v6.4s
-; CHECK-NEXT:    mvn v25.16b, v24.16b
-; CHECK-NEXT:    bsl v20.16b, v24.16b, v25.16b
-; CHECK-NEXT:    cmlt v24.4s, v21.4s, #0
-; CHECK-NEXT:    mvni v22.4s, #128, lsl #24
-; CHECK-NEXT:    add v23.4s, v3.4s, v7.4s
-; CHECK-NEXT:    mvn v25.16b, v24.16b
-; CHECK-NEXT:    bsl v22.16b, v24.16b, v25.16b
-; CHECK-NEXT:    cmlt v24.4s, v23.4s, #0
-; CHECK-NEXT:    mvni v17.4s, #128, lsl #24
-; CHECK-NEXT:    mvn v25.16b, v24.16b
-; CHECK-NEXT:    bsl v17.16b, v24.16b, v25.16b
-; CHECK-NEXT:    cmge v4.4s, v4.4s, #0
-; CHECK-NEXT:    cmge v0.4s, v0.4s, #0
-; CHECK-NEXT:    cmge v24.4s, v16.4s, #0
-; CHECK-NEXT:    cmge v5.4s, v5.4s, #0
-; CHECK-NEXT:    cmge v1.4s, v1.4s, #0
-; CHECK-NEXT:    cmeq v4.4s, v0.4s, v4.4s
-; CHECK-NEXT:    cmeq v0.4s, v0.4s, v24.4s
-; CHECK-NEXT:    cmge v24.4s, v19.4s, #0
-; CHECK-NEXT:    cmge v6.4s, v6.4s, #0
-; CHECK-NEXT:    cmge v2.4s, v2.4s, #0
-; CHECK-NEXT:    cmeq v5.4s, v1.4s, v5.4s
-; CHECK-NEXT:    cmeq v1.4s, v1.4s, v24.4s
-; CHECK-NEXT:    cmge v24.4s, v21.4s, #0
-; CHECK-NEXT:    cmge v7.4s, v7.4s, #0
-; CHECK-NEXT:    cmge v3.4s, v3.4s, #0
-; CHECK-NEXT:    cmeq v6.4s, v2.4s, v6.4s
-; CHECK-NEXT:    cmeq v2.4s, v2.4s, v24.4s
-; CHECK-NEXT:    cmge v24.4s, v23.4s, #0
-; CHECK-NEXT:    cmeq v7.4s, v3.4s, v7.4s
-; CHECK-NEXT:    cmeq v3.4s, v3.4s, v24.4s
-; CHECK-NEXT:    mvn v0.16b, v0.16b
-; CHECK-NEXT:    mvn v1.16b, v1.16b
-; CHECK-NEXT:    mvn v2.16b, v2.16b
-; CHECK-NEXT:    mvn v3.16b, v3.16b
-; CHECK-NEXT:    and v0.16b, v4.16b, v0.16b
-; CHECK-NEXT:    and v1.16b, v5.16b, v1.16b
-; CHECK-NEXT:    and v2.16b, v6.16b, v2.16b
-; CHECK-NEXT:    and v3.16b, v7.16b, v3.16b
-; CHECK-NEXT:    bsl v0.16b, v18.16b, v16.16b
-; CHECK-NEXT:    bsl v1.16b, v20.16b, v19.16b
-; CHECK-NEXT:    bsl v2.16b, v22.16b, v21.16b
-; CHECK-NEXT:    bsl v3.16b, v17.16b, v23.16b
+; CHECK-NEXT:    sqadd v0.4s, v0.4s, v4.4s
+; CHECK-NEXT:    sqadd v1.4s, v1.4s, v5.4s
+; CHECK-NEXT:    sqadd v2.4s, v2.4s, v6.4s
+; CHECK-NEXT:    sqadd v3.4s, v3.4s, v7.4s
 ; CHECK-NEXT:    ret
   %z = call <16 x i32> @llvm.sadd.sat.v16i32(<16 x i32> %x, <16 x i32> %y)
   ret <16 x i32> %z
@@ -742,20 +342,7 @@ define <16 x i32> @v16i32(<16 x i32> %x, <16 x i32> %y) nounwind {
 define <2 x i64> @v2i64(<2 x i64> %x, <2 x i64> %y) nounwind {
 ; CHECK-LABEL: v2i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    add v2.2d, v0.2d, v1.2d
-; CHECK-NEXT:    cmge v1.2d, v1.2d, #0
-; CHECK-NEXT:    cmge v0.2d, v0.2d, #0
-; CHECK-NEXT:    cmge v5.2d, v2.2d, #0
-; CHECK-NEXT:    mov x8, #9223372036854775807
-; CHECK-NEXT:    cmlt v3.2d, v2.2d, #0
-; CHECK-NEXT:    cmeq v1.2d, v0.2d, v1.2d
-; CHECK-NEXT:    cmeq v0.2d, v0.2d, v5.2d
-; CHECK-NEXT:    dup v4.2d, x8
-; CHECK-NEXT:    mvn v5.16b, v3.16b
-; CHECK-NEXT:    mvn v0.16b, v0.16b
-; CHECK-NEXT:    bsl v4.16b, v3.16b, v5.16b
-; CHECK-NEXT:    and v0.16b, v1.16b, v0.16b
-; CHECK-NEXT:    bsl v0.16b, v4.16b, v2.16b
+; CHECK-NEXT:    sqadd v0.2d, v0.2d, v1.2d
 ; CHECK-NEXT:    ret
   %z = call <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64> %x, <2 x i64> %y)
   ret <2 x i64> %z
@@ -764,33 +351,8 @@ define <2 x i64> @v2i64(<2 x i64> %x, <2 x i64> %y) nounwind {
 define <4 x i64> @v4i64(<4 x i64> %x, <4 x i64> %y) nounwind {
 ; CHECK-LABEL: v4i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    add v4.2d, v0.2d, v2.2d
-; CHECK-NEXT:    mov x8, #9223372036854775807
-; CHECK-NEXT:    cmlt v6.2d, v4.2d, #0
-; CHECK-NEXT:    dup v7.2d, x8
-; CHECK-NEXT:    add v5.2d, v1.2d, v3.2d
-; CHECK-NEXT:    mvn v16.16b, v6.16b
-; CHECK-NEXT:    mov v17.16b, v7.16b
-; CHECK-NEXT:    bsl v17.16b, v6.16b, v16.16b
-; CHECK-NEXT:    cmlt v6.2d, v5.2d, #0
-; CHECK-NEXT:    mvn v16.16b, v6.16b
-; CHECK-NEXT:    bsl v7.16b, v6.16b, v16.16b
-; CHECK-NEXT:    cmge v2.2d, v2.2d, #0
-; CHECK-NEXT:    cmge v0.2d, v0.2d, #0
-; CHECK-NEXT:    cmge v6.2d, v4.2d, #0
-; CHECK-NEXT:    cmge v3.2d, v3.2d, #0
-; CHECK-NEXT:    cmge v1.2d, v1.2d, #0
-; CHECK-NEXT:    cmeq v2.2d, v0.2d, v2.2d
-; CHECK-NEXT:    cmeq v0.2d, v0.2d, v6.2d
-; CHECK-NEXT:    cmge v6.2d, v5.2d, #0
-; CHECK-NEXT:    cmeq v3.2d, v1.2d, v3.2d
-; CHECK-NEXT:    cmeq v1.2d, v1.2d, v6.2d
-; CHECK-NEXT:    mvn v0.16b, v0.16b
-; CHECK-NEXT:    mvn v1.16b, v1.16b
-; CHECK-NEXT:    and v0.16b, v2.16b, v0.16b
-; CHECK-NEXT:    and v1.16b, v3.16b, v1.16b
-; CHECK-NEXT:    bsl v0.16b, v17.16b, v4.16b
-; CHECK-NEXT:    bsl v1.16b, v7.16b, v5.16b
+; CHECK-NEXT:    sqadd v0.2d, v0.2d, v2.2d
+; CHECK-NEXT:    sqadd v1.2d, v1.2d, v3.2d
 ; CHECK-NEXT:    ret
   %z = call <4 x i64> @llvm.sadd.sat.v4i64(<4 x i64> %x, <4 x i64> %y)
   ret <4 x i64> %z
@@ -799,59 +361,10 @@ define <4 x i64> @v4i64(<4 x i64> %x, <4 x i64> %y) nounwind {
 define <8 x i64> @v8i64(<8 x i64> %x, <8 x i64> %y) nounwind {
 ; CHECK-LABEL: v8i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    add v16.2d, v0.2d, v4.2d
-; CHECK-NEXT:    mov x8, #9223372036854775807
-; CHECK-NEXT:    add v17.2d, v1.2d, v5.2d
-; CHECK-NEXT:    cmlt v20.2d, v16.2d, #0
-; CHECK-NEXT:    dup v21.2d, x8
-; CHECK-NEXT:    add v18.2d, v2.2d, v6.2d
-; CHECK-NEXT:    cmlt v22.2d, v17.2d, #0
-; CHECK-NEXT:    mvn v24.16b, v20.16b
-; CHECK-NEXT:    mov v25.16b, v21.16b
-; CHECK-NEXT:    cmlt v23.2d, v18.2d, #0
-; CHECK-NEXT:    bsl v25.16b, v20.16b, v24.16b
-; CHECK-NEXT:    mvn v20.16b, v22.16b
-; CHECK-NEXT:    mov v24.16b, v21.16b
-; CHECK-NEXT:    add v19.2d, v3.2d, v7.2d
-; CHECK-NEXT:    bsl v24.16b, v22.16b, v20.16b
-; CHECK-NEXT:    mvn v20.16b, v23.16b
-; CHECK-NEXT:    mov v22.16b, v21.16b
-; CHECK-NEXT:    bsl v22.16b, v23.16b, v20.16b
-; CHECK-NEXT:    cmlt v20.2d, v19.2d, #0
-; CHECK-NEXT:    mvn v23.16b, v20.16b
-; CHECK-NEXT:    bsl v21.16b, v20.16b, v23.16b
-; CHECK-NEXT:    cmge v4.2d, v4.2d, #0
-; CHECK-NEXT:    cmge v0.2d, v0.2d, #0
-; CHECK-NEXT:    cmge v20.2d, v16.2d, #0
-; CHECK-NEXT:    cmge v5.2d, v5.2d, #0
-; CHECK-NEXT:    cmge v1.2d, v1.2d, #0
-; CHECK-NEXT:    cmeq v4.2d, v0.2d, v4.2d
-; CHECK-NEXT:    cmeq v0.2d, v0.2d, v20.2d
-; CHECK-NEXT:    cmge v20.2d, v17.2d, #0
-; CHECK-NEXT:    cmge v6.2d, v6.2d, #0
-; CHECK-NEXT:    cmge v2.2d, v2.2d, #0
-; CHECK-NEXT:    cmeq v5.2d, v1.2d, v5.2d
-; CHECK-NEXT:    cmeq v1.2d, v1.2d, v20.2d
-; CHECK-NEXT:    cmge v20.2d, v18.2d, #0
-; CHECK-NEXT:    cmge v7.2d, v7.2d, #0
-; CHECK-NEXT:    cmge v3.2d, v3.2d, #0
-; CHECK-NEXT:    cmeq v6.2d, v2.2d, v6.2d
-; CHECK-NEXT:    cmeq v2.2d, v2.2d, v20.2d
-; CHECK-NEXT:    cmge v20.2d, v19.2d, #0
-; CHECK-NEXT:    cmeq v7.2d, v3.2d, v7.2d
-; CHECK-NEXT:    cmeq v3.2d, v3.2d, v20.2d
-; CHECK-NEXT:    mvn v0.16b, v0.16b
-; CHECK-NEXT:    mvn v1.16b, v1.16b
-; CHECK-NEXT:    mvn v2.16b, v2.16b
-; CHECK-NEXT:    mvn v3.16b, v3.16b
-; CHECK-NEXT:    and v0.16b, v4.16b, v0.16b
-; CHECK-NEXT:    and v1.16b, v5.16b, v1.16b
-; CHECK-NEXT:    and v2.16b, v6.16b, v2.16b
-; CHECK-NEXT:    and v3.16b, v7.16b, v3.16b
-; CHECK-NEXT:    bsl v0.16b, v25.16b, v16.16b
-; CHECK-NEXT:    bsl v1.16b, v24.16b, v17.16b
-; CHECK-NEXT:    bsl v2.16b, v22.16b, v18.16b
-; CHECK-NEXT:    bsl v3.16b, v21.16b, v19.16b
+; CHECK-NEXT:    sqadd v0.2d, v0.2d, v4.2d
+; CHECK-NEXT:    sqadd v1.2d, v1.2d, v5.2d
+; CHECK-NEXT:    sqadd v2.2d, v2.2d, v6.2d
+; CHECK-NEXT:    sqadd v3.2d, v3.2d, v7.2d
 ; CHECK-NEXT:    ret
   %z = call <8 x i64> @llvm.sadd.sat.v8i64(<8 x i64> %x, <8 x i64> %y)
   ret <8 x i64> %z
diff --git a/llvm/test/CodeGen/AArch64/sched-past-vector-ldst.ll b/llvm/test/CodeGen/AArch64/sched-past-vector-ldst.ll
index 52333463c..0619e7483 100644
--- a/llvm/test/CodeGen/AArch64/sched-past-vector-ldst.ll
+++ b/llvm/test/CodeGen/AArch64/sched-past-vector-ldst.ll
@@ -7,6 +7,8 @@
 
 
 ; CHECK: fmla
+; CHECK-NEXT: mov
+; CHECK-NEXT: mov
 ; CHECK-NEXT: fmla
 ; CHECK-NEXT: fmla
 ; CHECK-NEXT: fmla
@@ -16,21 +18,17 @@ target triple = "aarch64--linux-gnu"
 %Struct = type { i64*, [9 x double], [16 x {float, float}], [16 x {float, float}], i32, i32 }
 
 ; Function Attrs: nounwind
-define linkonce_odr void @func(%Struct* nocapture %this) unnamed_addr #0 align 2 {
+define linkonce_odr void @func(%Struct* nocapture %this, <4 x float> %f) unnamed_addr #0 align 2 {
 entry:
-  %0 = insertelement <4 x float> undef, float undef, i32 0
-  %1 = insertelement <4 x float> %0, float undef, i32 1
-  %2 = insertelement <4 x float> %1, float undef, i32 2
-  %3 = insertelement <4 x float> %2, float undef, i32 3
   %scevgep = getelementptr %Struct, %Struct* %this, i64 0, i32 2, i64 8, i32 0
   %struct_ptr = bitcast float* %scevgep to i8*
   %vec1 = tail call { <4 x float>, <4 x float> } @llvm.aarch64.neon.ld2.v4f32.p0i8(i8* %struct_ptr)
   %ev1 = extractvalue { <4 x float>, <4 x float> } %vec1, 1
-  %fm1 = fmul <4 x float> %0, %ev1
-  %av1 = fadd <4 x float> %1, %fm1
+  %fm1 = fmul <4 x float> %f, %ev1
+  %av1 = fadd <4 x float> %f, %fm1
   %ev2 = extractvalue { <4 x float>, <4 x float> } %vec1, 0
-  %fm2 = fmul <4 x float> %2, %ev2
-  %av2 = fadd <4 x float> %3, %fm2
+  %fm2 = fmul <4 x float> %f, %ev2
+  %av2 = fadd <4 x float> %f, %fm2
   %scevgep2 = getelementptr %Struct, %Struct* %this, i64 0, i32 3, i64 8, i32 0
   %struct_ptr2 = bitcast float* %scevgep2 to i8*
   tail call void @llvm.aarch64.neon.st2.v4f32.p0i8(<4 x float> %av2, <4 x float> %av1, i8* %struct_ptr2)
@@ -38,11 +36,11 @@ entry:
   %struct_ptr3 = bitcast float* %scevgep3 to i8*
   %vec2 = tail call { <4 x float>, <4 x float> } @llvm.aarch64.neon.ld2.v4f32.p0i8(i8* %struct_ptr3)
   %ev3 = extractvalue { <4 x float>, <4 x float> } %vec2, 1
-  %fm3 = fmul <4 x float> %0, %ev3
-  %av3 = fadd <4 x float> %1, %fm3
+  %fm3 = fmul <4 x float> %f, %ev3
+  %av3 = fadd <4 x float> %f, %fm3
   %ev4 = extractvalue { <4 x float>, <4 x float> } %vec2, 0
-  %fm4 = fmul <4 x float> %2, %ev4
-  %av4 = fadd <4 x float> %3, %fm4
+  %fm4 = fmul <4 x float> %f, %ev4
+  %av4 = fadd <4 x float> %f, %fm4
   %scevgep4 = getelementptr %Struct, %Struct* %this, i64 0, i32 3, i64 12, i32 0
   %struct_ptr4 = bitcast float* %scevgep4 to i8*
   tail call void @llvm.aarch64.neon.st2.v4f32.p0i8(<4 x float> %av4, <4 x float> %av3, i8* %struct_ptr4)
@@ -55,6 +53,6 @@ declare { <4 x float>, <4 x float> } @llvm.aarch64.neon.ld2.v4f32.p0i8(i8*) #2
 ; Function Attrs: nounwind
 declare void @llvm.aarch64.neon.st2.v4f32.p0i8(<4 x float>, <4 x float>, i8* nocapture) #1
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="true" "no-nans-fp-math"="true" "stack-protector-buffer-size"="8" "unsafe-fp-math"="true" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="true" "no-nans-fp-math"="true" "stack-protector-buffer-size"="8" "unsafe-fp-math"="true" "use-soft-float"="false" }
 attributes #1 = { nounwind }
 attributes #2 = { nounwind readonly }
diff --git a/llvm/test/CodeGen/AArch64/sdivpow2.ll b/llvm/test/CodeGen/AArch64/sdivpow2.ll
index 158a778f0..25e89176a 100644
--- a/llvm/test/CodeGen/AArch64/sdivpow2.ll
+++ b/llvm/test/CodeGen/AArch64/sdivpow2.ll
@@ -87,3 +87,22 @@ define i64 @test7(i64 %x) {
   ret i64 %div
 }
 
+define i64 @test8(i64 %x) {
+; ISEL-LABEL: test8:
+; ISEL:       // %bb.0:
+; ISEL-NEXT:    cmp x0, #0 // =0
+; ISEL-NEXT:    cinc x8, x0, lt
+; ISEL-NEXT:    asr x0, x8, #1
+; ISEL-NEXT:    ret
+;
+; FAST-LABEL: test8:
+; FAST:       // %bb.0:
+; FAST-NEXT:    add x8, x0, #1 // =1
+; FAST-NEXT:    cmp x0, #0 // =0
+; FAST-NEXT:    csel x8, x8, x0, lt
+; FAST-NEXT:    asr x0, x8, #1
+; FAST-NEXT:    ret
+  %div = sdiv i64 %x, 2
+  ret i64 %div
+}
+
diff --git a/llvm/test/CodeGen/AArch64/seh-finally.ll b/llvm/test/CodeGen/AArch64/seh-finally.ll
index b7027fb43..66558c90a 100644
--- a/llvm/test/CodeGen/AArch64/seh-finally.ll
+++ b/llvm/test/CodeGen/AArch64/seh-finally.ll
@@ -86,7 +86,7 @@ entry:
 define void @stack_realign() #0 personality i8* bitcast (i32 (...)* @__C_specific_handler to i8*) {
 entry:
 ; CHECK-LABEL: stack_realign
-; CHECK: add     x29, sp, #16
+; CHECK: mov     x29, sp
 ; CHECK: sub     x9, sp, #64
 ; CHECK: and     sp, x9, #0xffffffffffffffe0
 ; CHECK: mov     x19, sp
@@ -205,15 +205,15 @@ entry:
 define void @vla_and_realign(i32 %n) #0 personality i8* bitcast (i32 (...)* @__C_specific_handler to i8*) {
 entry:
 ; CHECK-LABEL: vla_and_realign
-; CHECK: add     x29, sp, #16
+; CHECK: mov     x29, sp
 ; CHECK: sub     x9, sp, #64
 ; CHECK: and     sp, x9, #0xffffffffffffffe0
 ; CHECK: mov     x19, sp
 ; CHECK: mov     x1, #-2
 ; CHECK: stur    x1, [x19]
 ; CHECK: .set .Lvla_and_realign$frame_escape_0, 32
-; CHECK: stur    w0, [x29, #-4]
-; CHECK: ldur    w8, [x29, #-4]
+; CHECK: str     w0, [x29, #28]
+; CHECK: ldr     w8, [x29, #28]
 ; CHECK: mov     x9, sp
 ; CHECK: str     x9, [x19, #24]
 ; CHECK: str     x8, [x19, #16]
diff --git a/llvm/test/CodeGen/AArch64/select_const.ll b/llvm/test/CodeGen/AArch64/select_const.ll
new file mode 100644
index 000000000..affb8150f
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/select_const.ll
@@ -0,0 +1,625 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=aarch64-- | FileCheck %s
+
+; Select of constants: control flow / conditional moves can always be replaced by logic+math (but may not be worth it?).
+; Test the zeroext/signext variants of each pattern to see if that makes a difference.
+
+; select Cond, 0, 1 --> zext (!Cond)
+
+define i32 @select_0_or_1(i1 %cond) {
+; CHECK-LABEL: select_0_or_1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mvn w8, w0
+; CHECK-NEXT:    and w0, w8, #0x1
+; CHECK-NEXT:    ret
+  %sel = select i1 %cond, i32 0, i32 1
+  ret i32 %sel
+}
+
+define i32 @select_0_or_1_zeroext(i1 zeroext %cond) {
+; CHECK-LABEL: select_0_or_1_zeroext:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    eor w0, w0, #0x1
+; CHECK-NEXT:    ret
+  %sel = select i1 %cond, i32 0, i32 1
+  ret i32 %sel
+}
+
+define i32 @select_0_or_1_signext(i1 signext %cond) {
+; CHECK-LABEL: select_0_or_1_signext:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mvn w8, w0
+; CHECK-NEXT:    and w0, w8, #0x1
+; CHECK-NEXT:    ret
+  %sel = select i1 %cond, i32 0, i32 1
+  ret i32 %sel
+}
+
+; select Cond, 1, 0 --> zext (Cond)
+
+define i32 @select_1_or_0(i1 %cond) {
+; CHECK-LABEL: select_1_or_0:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    and w0, w0, #0x1
+; CHECK-NEXT:    ret
+  %sel = select i1 %cond, i32 1, i32 0
+  ret i32 %sel
+}
+
+define i32 @select_1_or_0_zeroext(i1 zeroext %cond) {
+; CHECK-LABEL: select_1_or_0_zeroext:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ret
+  %sel = select i1 %cond, i32 1, i32 0
+  ret i32 %sel
+}
+
+define i32 @select_1_or_0_signext(i1 signext %cond) {
+; CHECK-LABEL: select_1_or_0_signext:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    and w0, w0, #0x1
+; CHECK-NEXT:    ret
+  %sel = select i1 %cond, i32 1, i32 0
+  ret i32 %sel
+}
+
+; select Cond, 0, -1 --> sext (!Cond)
+
+define i32 @select_0_or_neg1(i1 %cond) {
+; CHECK-LABEL: select_0_or_neg1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mvn w8, w0
+; CHECK-NEXT:    sbfx w0, w8, #0, #1
+; CHECK-NEXT:    ret
+  %sel = select i1 %cond, i32 0, i32 -1
+  ret i32 %sel
+}
+
+define i32 @select_0_or_neg1_zeroext(i1 zeroext %cond) {
+; CHECK-LABEL: select_0_or_neg1_zeroext:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mvn w8, w0
+; CHECK-NEXT:    sbfx w0, w8, #0, #1
+; CHECK-NEXT:    ret
+  %sel = select i1 %cond, i32 0, i32 -1
+  ret i32 %sel
+}
+
+define i32 @select_0_or_neg1_signext(i1 signext %cond) {
+; CHECK-LABEL: select_0_or_neg1_signext:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mvn w0, w0
+; CHECK-NEXT:    ret
+  %sel = select i1 %cond, i32 0, i32 -1
+  ret i32 %sel
+}
+
+; select Cond, -1, 0 --> sext (Cond)
+
+define i32 @select_neg1_or_0(i1 %cond) {
+; CHECK-LABEL: select_neg1_or_0:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sbfx w0, w0, #0, #1
+; CHECK-NEXT:    ret
+  %sel = select i1 %cond, i32 -1, i32 0
+  ret i32 %sel
+}
+
+define i32 @select_neg1_or_0_zeroext(i1 zeroext %cond) {
+; CHECK-LABEL: select_neg1_or_0_zeroext:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sbfx w0, w0, #0, #1
+; CHECK-NEXT:    ret
+  %sel = select i1 %cond, i32 -1, i32 0
+  ret i32 %sel
+}
+
+define i32 @select_neg1_or_0_signext(i1 signext %cond) {
+; CHECK-LABEL: select_neg1_or_0_signext:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ret
+  %sel = select i1 %cond, i32 -1, i32 0
+  ret i32 %sel
+}
+
+; select Cond, C+1, C --> add (zext Cond), C
+
+define i32 @select_Cplus1_C(i1 %cond) {
+; CHECK-LABEL: select_Cplus1_C:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    tst w0, #0x1
+; CHECK-NEXT:    mov w8, #41
+; CHECK-NEXT:    cinc w0, w8, ne
+; CHECK-NEXT:    ret
+  %sel = select i1 %cond, i32 42, i32 41
+  ret i32 %sel
+}
+
+define i32 @select_Cplus1_C_zeroext(i1 zeroext %cond) {
+; CHECK-LABEL: select_Cplus1_C_zeroext:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmp w0, #0 // =0
+; CHECK-NEXT:    mov w8, #41
+; CHECK-NEXT:    cinc w0, w8, ne
+; CHECK-NEXT:    ret
+  %sel = select i1 %cond, i32 42, i32 41
+  ret i32 %sel
+}
+
+define i32 @select_Cplus1_C_signext(i1 signext %cond) {
+; CHECK-LABEL: select_Cplus1_C_signext:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    tst w0, #0x1
+; CHECK-NEXT:    mov w8, #41
+; CHECK-NEXT:    cinc w0, w8, ne
+; CHECK-NEXT:    ret
+  %sel = select i1 %cond, i32 42, i32 41
+  ret i32 %sel
+}
+
+; select Cond, C, C+1 --> add (sext Cond), C
+
+define i32 @select_C_Cplus1(i1 %cond) {
+; CHECK-LABEL: select_C_Cplus1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    tst w0, #0x1
+; CHECK-NEXT:    mov w8, #41
+; CHECK-NEXT:    cinc w0, w8, eq
+; CHECK-NEXT:    ret
+  %sel = select i1 %cond, i32 41, i32 42
+  ret i32 %sel
+}
+
+define i32 @select_C_Cplus1_zeroext(i1 zeroext %cond) {
+; CHECK-LABEL: select_C_Cplus1_zeroext:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmp w0, #0 // =0
+; CHECK-NEXT:    mov w8, #41
+; CHECK-NEXT:    cinc w0, w8, eq
+; CHECK-NEXT:    ret
+  %sel = select i1 %cond, i32 41, i32 42
+  ret i32 %sel
+}
+
+define i32 @select_C_Cplus1_signext(i1 signext %cond) {
+; CHECK-LABEL: select_C_Cplus1_signext:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    tst w0, #0x1
+; CHECK-NEXT:    mov w8, #41
+; CHECK-NEXT:    cinc w0, w8, eq
+; CHECK-NEXT:    ret
+  %sel = select i1 %cond, i32 41, i32 42
+  ret i32 %sel
+}
+
+; In general, select of 2 constants could be:
+; select Cond, C1, C2 --> add (mul (zext Cond), C1-C2), C2 --> add (and (sext Cond), C1-C2), C2
+
+define i32 @select_C1_C2(i1 %cond) {
+; CHECK-LABEL: select_C1_C2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    tst w0, #0x1
+; CHECK-NEXT:    mov w8, #42
+; CHECK-NEXT:    mov w9, #421
+; CHECK-NEXT:    csel w0, w9, w8, ne
+; CHECK-NEXT:    ret
+  %sel = select i1 %cond, i32 421, i32 42
+  ret i32 %sel
+}
+
+define i32 @select_C1_C2_zeroext(i1 zeroext %cond) {
+; CHECK-LABEL: select_C1_C2_zeroext:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmp w0, #0 // =0
+; CHECK-NEXT:    mov w8, #42
+; CHECK-NEXT:    mov w9, #421
+; CHECK-NEXT:    csel w0, w9, w8, ne
+; CHECK-NEXT:    ret
+  %sel = select i1 %cond, i32 421, i32 42
+  ret i32 %sel
+}
+
+define i32 @select_C1_C2_signext(i1 signext %cond) {
+; CHECK-LABEL: select_C1_C2_signext:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    tst w0, #0x1
+; CHECK-NEXT:    mov w8, #42
+; CHECK-NEXT:    mov w9, #421
+; CHECK-NEXT:    csel w0, w9, w8, ne
+; CHECK-NEXT:    ret
+  %sel = select i1 %cond, i32 421, i32 42
+  ret i32 %sel
+}
+
+; A binary operator with constant after the select should always get folded into the select.
+
+define i8 @sel_constants_add_constant(i1 %cond) {
+; CHECK-LABEL: sel_constants_add_constant:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    tst w0, #0x1
+; CHECK-NEXT:    mov w8, #28
+; CHECK-NEXT:    csinc w0, w8, wzr, eq
+; CHECK-NEXT:    ret
+  %sel = select i1 %cond, i8 -4, i8 23
+  %bo = add i8 %sel, 5
+  ret i8 %bo
+}
+
+define i8 @sel_constants_sub_constant(i1 %cond) {
+; CHECK-LABEL: sel_constants_sub_constant:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    tst w0, #0x1
+; CHECK-NEXT:    mov w8, #18
+; CHECK-NEXT:    mov w9, #-9
+; CHECK-NEXT:    csel w0, w9, w8, ne
+; CHECK-NEXT:    ret
+  %sel = select i1 %cond, i8 -4, i8 23
+  %bo = sub i8 %sel, 5
+  ret i8 %bo
+}
+
+define i8 @sel_constants_sub_constant_sel_constants(i1 %cond) {
+; CHECK-LABEL: sel_constants_sub_constant_sel_constants:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    tst w0, #0x1
+; CHECK-NEXT:    mov w8, #2
+; CHECK-NEXT:    mov w9, #9
+; CHECK-NEXT:    csel w0, w9, w8, ne
+; CHECK-NEXT:    ret
+  %sel = select i1 %cond, i8 -4, i8 3
+  %bo = sub i8 5, %sel
+  ret i8 %bo
+}
+
+define i8 @sel_constants_mul_constant(i1 %cond) {
+; CHECK-LABEL: sel_constants_mul_constant:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    tst w0, #0x1
+; CHECK-NEXT:    mov w8, #115
+; CHECK-NEXT:    mov w9, #-20
+; CHECK-NEXT:    csel w0, w9, w8, ne
+; CHECK-NEXT:    ret
+  %sel = select i1 %cond, i8 -4, i8 23
+  %bo = mul i8 %sel, 5
+  ret i8 %bo
+}
+
+define i8 @sel_constants_sdiv_constant(i1 %cond) {
+; CHECK-LABEL: sel_constants_sdiv_constant:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    tst w0, #0x1
+; CHECK-NEXT:    mov w8, #4
+; CHECK-NEXT:    csel w0, wzr, w8, ne
+; CHECK-NEXT:    ret
+  %sel = select i1 %cond, i8 -4, i8 23
+  %bo = sdiv i8 %sel, 5
+  ret i8 %bo
+}
+
+define i8 @sdiv_constant_sel_constants(i1 %cond) {
+; CHECK-LABEL: sdiv_constant_sel_constants:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    tst w0, #0x1
+; CHECK-NEXT:    mov w8, #5
+; CHECK-NEXT:    csel w0, wzr, w8, ne
+; CHECK-NEXT:    ret
+  %sel = select i1 %cond, i8 121, i8 23
+  %bo = sdiv i8 120, %sel
+  ret i8 %bo
+}
+
+define i8 @sel_constants_udiv_constant(i1 %cond) {
+; CHECK-LABEL: sel_constants_udiv_constant:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    tst w0, #0x1
+; CHECK-NEXT:    mov w8, #4
+; CHECK-NEXT:    mov w9, #50
+; CHECK-NEXT:    csel w0, w9, w8, ne
+; CHECK-NEXT:    ret
+  %sel = select i1 %cond, i8 -4, i8 23
+  %bo = udiv i8 %sel, 5
+  ret i8 %bo
+}
+
+define i8 @udiv_constant_sel_constants(i1 %cond) {
+; CHECK-LABEL: udiv_constant_sel_constants:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    tst w0, #0x1
+; CHECK-NEXT:    mov w8, #5
+; CHECK-NEXT:    csel w0, wzr, w8, ne
+; CHECK-NEXT:    ret
+  %sel = select i1 %cond, i8 -4, i8 23
+  %bo = udiv i8 120, %sel
+  ret i8 %bo
+}
+
+define i8 @sel_constants_srem_constant(i1 %cond) {
+; CHECK-LABEL: sel_constants_srem_constant:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    tst w0, #0x1
+; CHECK-NEXT:    mov w8, #-4
+; CHECK-NEXT:    cinv w0, w8, eq
+; CHECK-NEXT:    ret
+  %sel = select i1 %cond, i8 -4, i8 23
+  %bo = srem i8 %sel, 5
+  ret i8 %bo
+}
+
+define i8 @srem_constant_sel_constants(i1 %cond) {
+; CHECK-LABEL: srem_constant_sel_constants:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    tst w0, #0x1
+; CHECK-NEXT:    mov w8, #5
+; CHECK-NEXT:    mov w9, #120
+; CHECK-NEXT:    csel w0, w9, w8, ne
+; CHECK-NEXT:    ret
+  %sel = select i1 %cond, i8 121, i8 23
+  %bo = srem i8 120, %sel
+  ret i8 %bo
+}
+
+define i8 @sel_constants_urem_constant(i1 %cond) {
+; CHECK-LABEL: sel_constants_urem_constant:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    tst w0, #0x1
+; CHECK-NEXT:    mov w8, #2
+; CHECK-NEXT:    cinc w0, w8, eq
+; CHECK-NEXT:    ret
+  %sel = select i1 %cond, i8 -4, i8 23
+  %bo = urem i8 %sel, 5
+  ret i8 %bo
+}
+
+define i8 @urem_constant_sel_constants(i1 %cond) {
+; CHECK-LABEL: urem_constant_sel_constants:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    tst w0, #0x1
+; CHECK-NEXT:    mov w8, #5
+; CHECK-NEXT:    mov w9, #120
+; CHECK-NEXT:    csel w0, w9, w8, ne
+; CHECK-NEXT:    ret
+  %sel = select i1 %cond, i8 -4, i8 23
+  %bo = urem i8 120, %sel
+  ret i8 %bo
+}
+
+define i8 @sel_constants_and_constant(i1 %cond) {
+; CHECK-LABEL: sel_constants_and_constant:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    tst w0, #0x1
+; CHECK-NEXT:    mov w8, #4
+; CHECK-NEXT:    cinc w0, w8, eq
+; CHECK-NEXT:    ret
+  %sel = select i1 %cond, i8 -4, i8 23
+  %bo = and i8 %sel, 5
+  ret i8 %bo
+}
+
+define i8 @sel_constants_or_constant(i1 %cond) {
+; CHECK-LABEL: sel_constants_or_constant:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    tst w0, #0x1
+; CHECK-NEXT:    mov w8, #23
+; CHECK-NEXT:    mov w9, #-3
+; CHECK-NEXT:    csel w0, w9, w8, ne
+; CHECK-NEXT:    ret
+  %sel = select i1 %cond, i8 -4, i8 23
+  %bo = or i8 %sel, 5
+  ret i8 %bo
+}
+
+define i8 @sel_constants_xor_constant(i1 %cond) {
+; CHECK-LABEL: sel_constants_xor_constant:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    tst w0, #0x1
+; CHECK-NEXT:    mov w8, #18
+; CHECK-NEXT:    mov w9, #-7
+; CHECK-NEXT:    csel w0, w9, w8, ne
+; CHECK-NEXT:    ret
+  %sel = select i1 %cond, i8 -4, i8 23
+  %bo = xor i8 %sel, 5
+  ret i8 %bo
+}
+
+define i8 @sel_constants_shl_constant(i1 %cond) {
+; CHECK-LABEL: sel_constants_shl_constant:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    tst w0, #0x1
+; CHECK-NEXT:    mov w8, #-32
+; CHECK-NEXT:    mov w9, #-128
+; CHECK-NEXT:    csel w0, w9, w8, ne
+; CHECK-NEXT:    ret
+  %sel = select i1 %cond, i8 -4, i8 23
+  %bo = shl i8 %sel, 5
+  ret i8 %bo
+}
+
+define i8 @shl_constant_sel_constants(i1 %cond) {
+; CHECK-LABEL: shl_constant_sel_constants:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    tst w0, #0x1
+; CHECK-NEXT:    mov w8, #2
+; CHECK-NEXT:    cinc x8, x8, eq
+; CHECK-NEXT:    mov w9, #1
+; CHECK-NEXT:    lsl w0, w9, w8
+; CHECK-NEXT:    ret
+  %sel = select i1 %cond, i8 2, i8 3
+  %bo = shl i8 1, %sel
+  ret i8 %bo
+}
+
+define i8 @sel_constants_lshr_constant(i1 %cond) {
+; CHECK-LABEL: sel_constants_lshr_constant:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    tst w0, #0x1
+; CHECK-NEXT:    mov w8, #7
+; CHECK-NEXT:    csel w0, w8, wzr, ne
+; CHECK-NEXT:    ret
+  %sel = select i1 %cond, i8 -4, i8 23
+  %bo = lshr i8 %sel, 5
+  ret i8 %bo
+}
+
+define i8 @lshr_constant_sel_constants(i1 %cond) {
+; CHECK-LABEL: lshr_constant_sel_constants:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    tst w0, #0x1
+; CHECK-NEXT:    mov w8, #2
+; CHECK-NEXT:    cinc x8, x8, eq
+; CHECK-NEXT:    mov w9, #64
+; CHECK-NEXT:    lsr w0, w9, w8
+; CHECK-NEXT:    ret
+  %sel = select i1 %cond, i8 2, i8 3
+  %bo = lshr i8 64, %sel
+  ret i8 %bo
+}
+
+
+define i8 @sel_constants_ashr_constant(i1 %cond) {
+; CHECK-LABEL: sel_constants_ashr_constant:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sbfx w0, w0, #0, #1
+; CHECK-NEXT:    ret
+  %sel = select i1 %cond, i8 -4, i8 23
+  %bo = ashr i8 %sel, 5
+  ret i8 %bo
+}
+
+define i8 @ashr_constant_sel_constants(i1 %cond) {
+; CHECK-LABEL: ashr_constant_sel_constants:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    tst w0, #0x1
+; CHECK-NEXT:    mov w8, #2
+; CHECK-NEXT:    cinc x8, x8, eq
+; CHECK-NEXT:    mov w9, #-128
+; CHECK-NEXT:    asr w0, w9, w8
+; CHECK-NEXT:    ret
+  %sel = select i1 %cond, i8 2, i8 3
+  %bo = ashr i8 128, %sel
+  ret i8 %bo
+}
+
+define double @sel_constants_fadd_constant(i1 %cond) {
+; CHECK-LABEL: sel_constants_fadd_constant:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI42_0
+; CHECK-NEXT:    ldr d0, [x8, :lo12:.LCPI42_0]
+; CHECK-NEXT:    mov x8, #7378697629483820646
+; CHECK-NEXT:    movk x8, #16444, lsl #48
+; CHECK-NEXT:    tst w0, #0x1
+; CHECK-NEXT:    fmov d1, x8
+; CHECK-NEXT:    fcsel d0, d0, d1, ne
+; CHECK-NEXT:    ret
+  %sel = select i1 %cond, double -4.0, double 23.3
+  %bo = fadd double %sel, 5.1
+  ret double %bo
+}
+
+define double @sel_constants_fsub_constant(i1 %cond) {
+; CHECK-LABEL: sel_constants_fsub_constant:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI43_0
+; CHECK-NEXT:    ldr d0, [x8, :lo12:.LCPI43_0]
+; CHECK-NEXT:    mov x8, #3689348814741910323
+; CHECK-NEXT:    movk x8, #49186, lsl #48
+; CHECK-NEXT:    tst w0, #0x1
+; CHECK-NEXT:    fmov d1, x8
+; CHECK-NEXT:    fcsel d0, d1, d0, ne
+; CHECK-NEXT:    ret
+  %sel = select i1 %cond, double -4.0, double 23.3
+  %bo = fsub double %sel, 5.1
+  ret double %bo
+}
+
+define double @fsub_constant_sel_constants(i1 %cond) {
+; CHECK-LABEL: fsub_constant_sel_constants:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI44_0
+; CHECK-NEXT:    ldr d0, [x8, :lo12:.LCPI44_0]
+; CHECK-NEXT:    mov x8, #3689348814741910323
+; CHECK-NEXT:    movk x8, #16418, lsl #48
+; CHECK-NEXT:    tst w0, #0x1
+; CHECK-NEXT:    fmov d1, x8
+; CHECK-NEXT:    fcsel d0, d1, d0, ne
+; CHECK-NEXT:    ret
+  %sel = select i1 %cond, double -4.0, double 23.3
+  %bo = fsub double 5.1, %sel
+  ret double %bo
+}
+
+define double @sel_constants_fmul_constant(i1 %cond) {
+; CHECK-LABEL: sel_constants_fmul_constant:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI45_0
+; CHECK-NEXT:    ldr d0, [x8, :lo12:.LCPI45_0]
+; CHECK-NEXT:    mov x8, #7378697629483820646
+; CHECK-NEXT:    movk x8, #49204, lsl #48
+; CHECK-NEXT:    tst w0, #0x1
+; CHECK-NEXT:    fmov d1, x8
+; CHECK-NEXT:    fcsel d0, d1, d0, ne
+; CHECK-NEXT:    ret
+  %sel = select i1 %cond, double -4.0, double 23.3
+  %bo = fmul double %sel, 5.1
+  ret double %bo
+}
+
+define double @sel_constants_fdiv_constant(i1 %cond) {
+; CHECK-LABEL: sel_constants_fdiv_constant:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI46_0
+; CHECK-NEXT:    adrp x9, .LCPI46_1
+; CHECK-NEXT:    ldr d0, [x8, :lo12:.LCPI46_0]
+; CHECK-NEXT:    ldr d1, [x9, :lo12:.LCPI46_1]
+; CHECK-NEXT:    tst w0, #0x1
+; CHECK-NEXT:    fcsel d0, d1, d0, ne
+; CHECK-NEXT:    ret
+  %sel = select i1 %cond, double -4.0, double 23.3
+  %bo = fdiv double %sel, 5.1
+  ret double %bo
+}
+
+define double @fdiv_constant_sel_constants(i1 %cond) {
+; CHECK-LABEL: fdiv_constant_sel_constants:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI47_0
+; CHECK-NEXT:    ldr d0, [x8, :lo12:.LCPI47_0]
+; CHECK-NEXT:    mov x8, #7378697629483820646
+; CHECK-NEXT:    movk x8, #49140, lsl #48
+; CHECK-NEXT:    tst w0, #0x1
+; CHECK-NEXT:    fmov d1, x8
+; CHECK-NEXT:    fcsel d0, d1, d0, ne
+; CHECK-NEXT:    ret
+  %sel = select i1 %cond, double -4.0, double 23.3
+  %bo = fdiv double 5.1, %sel
+  ret double %bo
+}
+
+define double @sel_constants_frem_constant(i1 %cond) {
+; CHECK-LABEL: sel_constants_frem_constant:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI48_0
+; CHECK-NEXT:    ldr d0, [x8, :lo12:.LCPI48_0]
+; CHECK-NEXT:    tst w0, #0x1
+; CHECK-NEXT:    fmov d1, #-4.00000000
+; CHECK-NEXT:    fcsel d0, d1, d0, ne
+; CHECK-NEXT:    ret
+  %sel = select i1 %cond, double -4.0, double 23.3
+  %bo = frem double %sel, 5.1
+  ret double %bo
+}
+
+define double @frem_constant_sel_constants(i1 %cond) {
+; CHECK-LABEL: frem_constant_sel_constants:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI49_0
+; CHECK-NEXT:    ldr d0, [x8, :lo12:.LCPI49_0]
+; CHECK-NEXT:    mov x8, #7378697629483820646
+; CHECK-NEXT:    movk x8, #16404, lsl #48
+; CHECK-NEXT:    tst w0, #0x1
+; CHECK-NEXT:    fmov d1, x8
+; CHECK-NEXT:    fcsel d0, d0, d1, ne
+; CHECK-NEXT:    ret
+  %sel = select i1 %cond, double -4.0, double 23.3
+  %bo = frem double 5.1, %sel
+  ret double %bo
+}
diff --git a/llvm/test/CodeGen/AArch64/selectcc-to-shiftand.ll b/llvm/test/CodeGen/AArch64/selectcc-to-shiftand.ll
index bd1635a1e..292a934d2 100644
--- a/llvm/test/CodeGen/AArch64/selectcc-to-shiftand.ll
+++ b/llvm/test/CodeGen/AArch64/selectcc-to-shiftand.ll
@@ -118,3 +118,98 @@ define i32 @PR31175(i32 %x, i32 %y) {
   ret i32 %sel
 }
 
+define i8 @sel_shift_bool_i8(i1 %t) {
+; CHECK-LABEL: sel_shift_bool_i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    tst w0, #0x1
+; CHECK-NEXT:    mov w8, #-128
+; CHECK-NEXT:    csel w0, w8, wzr, ne
+; CHECK-NEXT:    ret
+  %shl = select i1 %t, i8 128, i8 0
+  ret i8 %shl
+}
+
+define i16 @sel_shift_bool_i16(i1 %t) {
+; CHECK-LABEL: sel_shift_bool_i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    tst w0, #0x1
+; CHECK-NEXT:    mov w8, #128
+; CHECK-NEXT:    csel w0, w8, wzr, ne
+; CHECK-NEXT:    ret
+  %shl = select i1 %t, i16 128, i16 0
+  ret i16 %shl
+}
+
+define i32 @sel_shift_bool_i32(i1 %t) {
+; CHECK-LABEL: sel_shift_bool_i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    tst w0, #0x1
+; CHECK-NEXT:    mov w8, #64
+; CHECK-NEXT:    csel w0, w8, wzr, ne
+; CHECK-NEXT:    ret
+  %shl = select i1 %t, i32 64, i32 0
+  ret i32 %shl
+}
+
+define i64 @sel_shift_bool_i64(i1 %t) {
+; CHECK-LABEL: sel_shift_bool_i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    tst w0, #0x1
+; CHECK-NEXT:    mov w8, #65536
+; CHECK-NEXT:    csel x0, x8, xzr, ne
+; CHECK-NEXT:    ret
+  %shl = select i1 %t, i64 65536, i64 0
+  ret i64 %shl
+}
+
+define <16 x i8> @sel_shift_bool_v16i8(<16 x i1> %t) {
+; CHECK-LABEL: sel_shift_bool_v16i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    shl v0.16b, v0.16b, #7
+; CHECK-NEXT:    sshr v0.16b, v0.16b, #7
+; CHECK-NEXT:    movi v1.16b, #128
+; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    ret
+  %shl = select <16 x i1> %t, <16 x i8> <i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128>, <16 x i8> zeroinitializer
+  ret <16 x i8> %shl
+}
+
+define <8 x i16> @sel_shift_bool_v8i16(<8 x i1> %t) {
+; CHECK-LABEL: sel_shift_bool_v8i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-NEXT:    shl v0.8h, v0.8h, #15
+; CHECK-NEXT:    sshr v0.8h, v0.8h, #15
+; CHECK-NEXT:    movi v1.8h, #128
+; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    ret
+  %shl= select <8 x i1> %t, <8 x i16> <i16 128, i16 128, i16 128, i16 128, i16 128, i16 128, i16 128, i16 128>, <8 x i16> zeroinitializer
+  ret <8 x i16> %shl
+}
+
+define <4 x i32> @sel_shift_bool_v4i32(<4 x i1> %t) {
+; CHECK-LABEL: sel_shift_bool_v4i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-NEXT:    shl v0.4s, v0.4s, #31
+; CHECK-NEXT:    sshr v0.4s, v0.4s, #31
+; CHECK-NEXT:    movi v1.4s, #64
+; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    ret
+  %shl = select <4 x i1> %t, <4 x i32> <i32 64, i32 64, i32 64, i32 64>, <4 x i32> zeroinitializer
+  ret <4 x i32> %shl
+}
+
+define <2 x i64> @sel_shift_bool_v2i64(<2 x i1> %t) {
+; CHECK-LABEL: sel_shift_bool_v2i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-NEXT:    mov w8, #65536
+; CHECK-NEXT:    shl v0.2d, v0.2d, #63
+; CHECK-NEXT:    sshr v0.2d, v0.2d, #63
+; CHECK-NEXT:    dup v1.2d, x8
+; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    ret
+  %shl = select <2 x i1> %t, <2 x i64> <i64 65536, i64 65536>, <2 x i64> zeroinitializer
+  ret <2 x i64> %shl
+}
diff --git a/llvm/test/CodeGen/AArch64/shadow-call-stack.ll b/llvm/test/CodeGen/AArch64/shadow-call-stack.ll
index 060d81b7f..9c6c36f57 100644
--- a/llvm/test/CodeGen/AArch64/shadow-call-stack.ll
+++ b/llvm/test/CodeGen/AArch64/shadow-call-stack.ll
@@ -41,7 +41,7 @@ define i32 @f4() shadowcallstack {
   %res12 = add i32 %res1, %res2
   %res34 = add i32 %res3, %res4
   %res1234 = add i32 %res12, %res34
-  ; CHECK: ldp {{.*}}x30, [sp
+  ; CHECK: ldp x30,{{.*}}, [sp
   ; CHECK: ldr x30, [x18, #-8]!
   ; CHECK: ret
   ret i32 %res1234
diff --git a/llvm/test/CodeGen/AArch64/shift-amount-mod.ll b/llvm/test/CodeGen/AArch64/shift-amount-mod.ll
index 6b8d19c83..403839044 100644
--- a/llvm/test/CodeGen/AArch64/shift-amount-mod.ll
+++ b/llvm/test/CodeGen/AArch64/shift-amount-mod.ll
@@ -666,3 +666,15 @@ define i64 @reg64_lshr_by_masked_negated_unfolded_add_b(i64 %val, i64 %a, i64 %b
   %shifted = lshr i64 %val, %negaaddbitwidthaddb
   ret i64 %shifted
 }
+
+define i32 @t(i64 %x) {
+; CHECK-LABEL: t:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ubfx x0, x0, #17, #28
+; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
+; CHECK-NEXT:    ret
+  %s = lshr i64 %x, 13
+  %t = trunc i64 %s to i32
+  %r = lshr i32 %t, 4
+  ret i32 %r
+}
diff --git a/llvm/test/CodeGen/AArch64/shift-by-signext.ll b/llvm/test/CodeGen/AArch64/shift-by-signext.ll
new file mode 100644
index 000000000..8e6a4d250
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/shift-by-signext.ll
@@ -0,0 +1,122 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64-unknown-linux-gnu < %s | FileCheck %s
+
+; If we have a shift by sign-extended value, we can replace sign-extension
+; with zero-extension.
+
+define i32 @t0_shl(i32 %x, i8 %shamt) nounwind {
+; CHECK-LABEL: t0_shl:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sxtb w8, w1
+; CHECK-NEXT:    lsl w0, w0, w8
+; CHECK-NEXT:    ret
+  %shamt_wide = sext i8 %shamt to i32
+  %r = shl i32 %x, %shamt_wide
+  ret i32 %r
+}
+define i32 @t1_lshr(i32 %x, i8 %shamt) nounwind {
+; CHECK-LABEL: t1_lshr:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sxtb w8, w1
+; CHECK-NEXT:    lsr w0, w0, w8
+; CHECK-NEXT:    ret
+  %shamt_wide = sext i8 %shamt to i32
+  %r = lshr i32 %x, %shamt_wide
+  ret i32 %r
+}
+define i32 @t2_ashr(i32 %x, i8 %shamt) nounwind {
+; CHECK-LABEL: t2_ashr:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sxtb w8, w1
+; CHECK-NEXT:    asr w0, w0, w8
+; CHECK-NEXT:    ret
+  %shamt_wide = sext i8 %shamt to i32
+  %r = ashr i32 %x, %shamt_wide
+  ret i32 %r
+}
+
+define <4 x i32> @t3_vec_shl(<4 x i32> %x, <4 x i8> %shamt) nounwind {
+; CHECK-LABEL: t3_vec_shl:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ushll v1.4s, v1.4h, #0
+; CHECK-NEXT:    shl v1.4s, v1.4s, #24
+; CHECK-NEXT:    sshr v1.4s, v1.4s, #24
+; CHECK-NEXT:    ushl v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    ret
+  %shamt_wide = sext <4 x i8> %shamt to <4 x i32>
+  %r = shl <4 x i32> %x, %shamt_wide
+  ret <4 x i32> %r
+}
+define <4 x i32> @t4_vec_lshr(<4 x i32> %x, <4 x i8> %shamt) nounwind {
+; CHECK-LABEL: t4_vec_lshr:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ushll v1.4s, v1.4h, #0
+; CHECK-NEXT:    shl v1.4s, v1.4s, #24
+; CHECK-NEXT:    sshr v1.4s, v1.4s, #24
+; CHECK-NEXT:    neg v1.4s, v1.4s
+; CHECK-NEXT:    ushl v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    ret
+  %shamt_wide = sext <4 x i8> %shamt to <4 x i32>
+  %r = lshr <4 x i32> %x, %shamt_wide
+  ret <4 x i32> %r
+}
+define <4 x i32> @t5_vec_ashr(<4 x i32> %x, <4 x i8> %shamt) nounwind {
+; CHECK-LABEL: t5_vec_ashr:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ushll v1.4s, v1.4h, #0
+; CHECK-NEXT:    shl v1.4s, v1.4s, #24
+; CHECK-NEXT:    sshr v1.4s, v1.4s, #24
+; CHECK-NEXT:    neg v1.4s, v1.4s
+; CHECK-NEXT:    sshl v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    ret
+  %shamt_wide = sext <4 x i8> %shamt to <4 x i32>
+  %r = ashr <4 x i32> %x, %shamt_wide
+  ret <4 x i32> %r
+}
+
+; This is not valid for funnel shifts
+declare i32 @llvm.fshl.i32(i32 %a, i32 %b, i32 %c)
+declare i32 @llvm.fshr.i32(i32 %a, i32 %b, i32 %c)
+define i32 @n6_fshl(i32 %x, i32 %y, i8 %shamt) nounwind {
+; CHECK-LABEL: n6_fshl:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    and w9, w2, #0x1f
+; CHECK-NEXT:    neg w9, w9
+; CHECK-NEXT:    lsl w8, w0, w2
+; CHECK-NEXT:    lsr w9, w1, w9
+; CHECK-NEXT:    orr w8, w8, w9
+; CHECK-NEXT:    tst w2, #0x1f
+; CHECK-NEXT:    csel w0, w0, w8, eq
+; CHECK-NEXT:    ret
+  %shamt_wide = sext i8 %shamt to i32
+  %r = call i32 @llvm.fshl.i32(i32 %x, i32 %y, i32 %shamt_wide)
+  ret i32 %r
+}
+define i32 @n7_fshr(i32 %x, i32 %y, i8 %shamt) nounwind {
+; CHECK-LABEL: n7_fshr:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    and w9, w2, #0x1f
+; CHECK-NEXT:    neg w9, w9
+; CHECK-NEXT:    lsr w8, w1, w2
+; CHECK-NEXT:    lsl w9, w0, w9
+; CHECK-NEXT:    orr w8, w9, w8
+; CHECK-NEXT:    tst w2, #0x1f
+; CHECK-NEXT:    csel w0, w1, w8, eq
+; CHECK-NEXT:    ret
+  %shamt_wide = sext i8 %shamt to i32
+  %r = call i32 @llvm.fshr.i32(i32 %x, i32 %y, i32 %shamt_wide)
+  ret i32 %r
+}
+
+define i32 @n8_extrause(i32 %x, i8 %shamt, i32* %shamt_wide_store) nounwind {
+; CHECK-LABEL: n8_extrause:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sxtb w8, w1
+; CHECK-NEXT:    lsl w0, w0, w8
+; CHECK-NEXT:    str w8, [x2]
+; CHECK-NEXT:    ret
+  %shamt_wide = sext i8 %shamt to i32
+  store i32 %shamt_wide, i32* %shamt_wide_store, align 4
+  %r = shl i32 %x, %shamt_wide
+  ret i32 %r
+}
diff --git a/llvm/test/CodeGen/AArch64/shift-logic.ll b/llvm/test/CodeGen/AArch64/shift-logic.ll
new file mode 100644
index 000000000..af684bbb8
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/shift-logic.ll
@@ -0,0 +1,153 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=aarch64-- | FileCheck %s
+
+define i8 @shl_and(i8 %x, i8 %y) nounwind {
+; CHECK-LABEL: shl_and:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    lsl w8, w0, #5
+; CHECK-NEXT:    and w0, w8, w1, lsl #2
+; CHECK-NEXT:    ret
+  %sh0 = shl i8 %x, 3
+  %r = and i8 %sh0, %y
+  %sh1 = shl i8 %r, 2
+  ret i8 %sh1
+}
+
+define i16 @shl_or(i16 %x, i16 %y) nounwind {
+; CHECK-LABEL: shl_or:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    lsl w8, w0, #12
+; CHECK-NEXT:    orr w0, w8, w1, lsl #7
+; CHECK-NEXT:    ret
+  %sh0 = shl i16 %x, 5
+  %r = or i16 %y, %sh0
+  %sh1 = shl i16 %r, 7
+  ret i16 %sh1
+}
+
+define i32 @shl_xor(i32 %x, i32 %y) nounwind {
+; CHECK-LABEL: shl_xor:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    lsl w8, w0, #12
+; CHECK-NEXT:    eor w0, w8, w1, lsl #7
+; CHECK-NEXT:    ret
+  %sh0 = shl i32 %x, 5
+  %r = xor i32 %sh0, %y
+  %sh1 = shl i32 %r, 7
+  ret i32 %sh1
+}
+
+define i64 @lshr_and(i64 %x, i64 %y) nounwind {
+; CHECK-LABEL: lshr_and:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    lsr x8, x0, #12
+; CHECK-NEXT:    and x0, x8, x1, lsr #7
+; CHECK-NEXT:    ret
+  %sh0 = lshr i64 %x, 5
+  %r = and i64 %y, %sh0
+  %sh1 = lshr i64 %r, 7
+  ret i64 %sh1
+}
+
+define <4 x i32> @lshr_or(<4 x i32> %x, <4 x i32> %y) nounwind {
+; CHECK-LABEL: lshr_or:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ushr v1.4s, v1.4s, #7
+; CHECK-NEXT:    ushr v0.4s, v0.4s, #12
+; CHECK-NEXT:    orr v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    ret
+  %sh0 = lshr <4 x i32> %x, <i32 5, i32 5, i32 5, i32 5>
+  %r = or <4 x i32> %sh0, %y
+  %sh1 = lshr <4 x i32> %r, <i32 7, i32 7, i32 7, i32 7>
+  ret <4 x i32> %sh1
+}
+
+define <8 x i16> @lshr_xor(<8 x i16> %x, <8 x i16> %y) nounwind {
+; CHECK-LABEL: lshr_xor:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ushr v1.8h, v1.8h, #7
+; CHECK-NEXT:    ushr v0.8h, v0.8h, #12
+; CHECK-NEXT:    eor v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    ret
+  %sh0 = lshr <8 x i16> %x, <i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5>
+  %r = xor <8 x i16> %y, %sh0
+  %sh1 = lshr <8 x i16> %r, <i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7>
+  ret <8 x i16> %sh1
+}
+
+
+define <16 x i8> @ashr_and(<16 x i8> %x, <16 x i8> %y) nounwind {
+; CHECK-LABEL: ashr_and:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sshr v1.16b, v1.16b, #2
+; CHECK-NEXT:    sshr v0.16b, v0.16b, #5
+; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    ret
+  %sh0 = ashr <16 x i8> %x, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
+  %r = and <16 x i8> %y, %sh0
+  %sh1 = ashr <16 x i8> %r, <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>
+  ret <16 x i8> %sh1
+}
+
+define <2 x i64> @ashr_or(<2 x i64> %x, <2 x i64> %y) nounwind {
+; CHECK-LABEL: ashr_or:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sshr v1.2d, v1.2d, #7
+; CHECK-NEXT:    sshr v0.2d, v0.2d, #12
+; CHECK-NEXT:    orr v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    ret
+  %sh0 = ashr <2 x i64> %x, <i64 5, i64 5>
+  %r = or <2 x i64> %sh0, %y
+  %sh1 = ashr <2 x i64> %r, <i64 7, i64 7>
+  ret <2 x i64> %sh1
+}
+
+define i32 @ashr_xor(i32 %x, i32 %y) nounwind {
+; CHECK-LABEL: ashr_xor:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    asr w8, w0, #12
+; CHECK-NEXT:    eor w0, w8, w1, asr #7
+; CHECK-NEXT:    ret
+  %sh0 = ashr i32 %x, 5
+  %r = xor i32 %y, %sh0
+  %sh1 = ashr i32 %r, 7
+  ret i32 %sh1
+}
+
+define i32 @shr_mismatch_xor(i32 %x, i32 %y) nounwind {
+; CHECK-LABEL: shr_mismatch_xor:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    eor w8, w1, w0, asr #5
+; CHECK-NEXT:    lsr w0, w8, #7
+; CHECK-NEXT:    ret
+  %sh0 = ashr i32 %x, 5
+  %r = xor i32 %y, %sh0
+  %sh1 = lshr i32 %r, 7
+  ret i32 %sh1
+}
+
+define i32 @ashr_overshift_xor(i32 %x, i32 %y) nounwind {
+; CHECK-LABEL: ashr_overshift_xor:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    eor w8, w1, w0, asr #15
+; CHECK-NEXT:    asr w0, w8, #17
+; CHECK-NEXT:    ret
+  %sh0 = ashr i32 %x, 15
+  %r = xor i32 %y, %sh0
+  %sh1 = ashr i32 %r, 17
+  ret i32 %sh1
+}
+
+define i32 @lshr_or_extra_use(i32 %x, i32 %y, i32* %p) nounwind {
+; CHECK-LABEL: lshr_or_extra_use:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    orr w8, w1, w0, lsr #5
+; CHECK-NEXT:    lsr w0, w8, #7
+; CHECK-NEXT:    str w8, [x2]
+; CHECK-NEXT:    ret
+  %sh0 = lshr i32 %x, 5
+  %r = or i32 %sh0, %y
+  store i32 %r, i32* %p
+  %sh1 = lshr i32 %r, 7
+  ret i32 %sh1
+}
diff --git a/llvm/test/CodeGen/AArch64/shift-mod.ll b/llvm/test/CodeGen/AArch64/shift-mod.ll
index 8eba4ab1a..6c3521d26 100644
--- a/llvm/test/CodeGen/AArch64/shift-mod.ll
+++ b/llvm/test/CodeGen/AArch64/shift-mod.ll
@@ -72,3 +72,80 @@ entry:
   %shr = shl i64 %y, %sh_prom
   ret i64 %shr
 }
+
+; PR42644 - https://bugs.llvm.org/show_bug.cgi?id=42644
+
+define i64 @ashr_add_shl_i32(i64 %r) {
+; CHECK-LABEL: ashr_add_shl_i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    add w8, w0, #1 // =1
+; CHECK-NEXT:    sxtw x0, w8
+; CHECK-NEXT:    ret
+  %conv = shl i64 %r, 32
+  %sext = add i64 %conv, 4294967296
+  %conv1 = ashr i64 %sext, 32
+  ret i64 %conv1
+}
+
+define i64 @ashr_add_shl_i8(i64 %r) {
+; CHECK-LABEL: ashr_add_shl_i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    add w8, w0, #1 // =1
+; CHECK-NEXT:    sxtb x0, w8
+; CHECK-NEXT:    ret
+  %conv = shl i64 %r, 56
+  %sext = add i64 %conv, 72057594037927936
+  %conv1 = ashr i64 %sext, 56
+  ret i64 %conv1
+}
+
+define <4 x i32> @ashr_add_shl_v4i8(<4 x i32> %r) {
+; CHECK-LABEL: ashr_add_shl_v4i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    shl v0.4s, v0.4s, #24
+; CHECK-NEXT:    movi v1.4s, #1, lsl #24
+; CHECK-NEXT:    add v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    sshr v0.4s, v0.4s, #24
+; CHECK-NEXT:    ret
+  %conv = shl <4 x i32> %r, <i32 24, i32 24, i32 24, i32 24>
+  %sext = add <4 x i32> %conv, <i32 16777216, i32 16777216, i32 16777216, i32 16777216>
+  %conv1 = ashr <4 x i32> %sext, <i32 24, i32 24, i32 24, i32 24>
+  ret <4 x i32> %conv1
+}
+
+define i64 @ashr_add_shl_i36(i64 %r) {
+; CHECK-LABEL: ashr_add_shl_i36:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sbfx x0, x0, #0, #28
+; CHECK-NEXT:    ret
+  %conv = shl i64 %r, 36
+  %sext = add i64 %conv, 4294967296
+  %conv1 = ashr i64 %sext, 36
+  ret i64 %conv1
+}
+
+define i64 @ashr_add_shl_mismatch_shifts1(i64 %r) {
+; CHECK-LABEL: ashr_add_shl_mismatch_shifts1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov x8, #4294967296
+; CHECK-NEXT:    add x8, x8, x0, lsl #8
+; CHECK-NEXT:    asr x0, x8, #32
+; CHECK-NEXT:    ret
+  %conv = shl i64 %r, 8
+  %sext = add i64 %conv, 4294967296
+  %conv1 = ashr i64 %sext, 32
+  ret i64 %conv1
+}
+
+define i64 @ashr_add_shl_mismatch_shifts2(i64 %r) {
+; CHECK-LABEL: ashr_add_shl_mismatch_shifts2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov x8, #4294967296
+; CHECK-NEXT:    add x8, x8, x0, lsr #8
+; CHECK-NEXT:    lsr x0, x8, #8
+; CHECK-NEXT:    ret
+  %conv = lshr i64 %r, 8
+  %sext = add i64 %conv, 4294967296
+  %conv1 = ashr i64 %sext, 8
+  ret i64 %conv1
+}
diff --git a/llvm/test/CodeGen/AArch64/shift_minsize.ll b/llvm/test/CodeGen/AArch64/shift_minsize.ll
index 2728b3f4c..ac48975f1 100644
--- a/llvm/test/CodeGen/AArch64/shift_minsize.ll
+++ b/llvm/test/CodeGen/AArch64/shift_minsize.ll
@@ -1,11 +1,16 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=aarch64-unknown-unknown | FileCheck %s
 ; RUN: llc < %s -mtriple=aarch64-windows         | FileCheck %s -check-prefix=CHECK-WIN
+; RUN: llc < %s -mtriple=aarch64-apple-darwin    | FileCheck %s -check-prefix=CHECK-DARWIN
 
 ; The Windows runtime doesn't have these.
 ; CHECK-WIN-NOT: __ashlti3
 ; CHECK-WIN-NOT: __ashrti3
 
+; Darwin compiler-rt excludes these.
+; CHECK-DARWIN-NOT: __ashlti3
+; CHECK-DARWIN-NOT: __ashrti3
+
 define i64 @f0(i64 %val, i64 %amt) minsize optsize {
 ; CHECK-LABEL: f0:
 ; CHECK:       // %bb.0:
diff --git a/llvm/test/CodeGen/AArch64/sibling-call.ll b/llvm/test/CodeGen/AArch64/sibling-call.ll
index be59f27fa..a9e022518 100644
--- a/llvm/test/CodeGen/AArch64/sibling-call.ll
+++ b/llvm/test/CodeGen/AArch64/sibling-call.ll
@@ -1,8 +1,8 @@
 ; RUN: llc -verify-machineinstrs < %s -mtriple=aarch64-none-linux-gnu -aarch64-enable-ldst-opt=0 | FileCheck %s
 
 declare void @callee_stack0()
-declare void @callee_stack8([8 x i32], i64)
-declare void @callee_stack16([8 x i32], i64, i64)
+declare void @callee_stack8([8 x i64], i64)
+declare void @callee_stack16([8 x i64], i64, i64)
 
 define void @caller_to0_from0() nounwind {
 ; CHECK-LABEL: caller_to0_from0:
@@ -12,7 +12,7 @@ define void @caller_to0_from0() nounwind {
 ; CHECK-NEXT: b callee_stack0
 }
 
-define void @caller_to0_from8([8 x i32], i64) nounwind{
+define void @caller_to0_from8([8 x i64], i64) nounwind{
 ; CHECK-LABEL: caller_to0_from8:
 ; CHECK-NEXT: // %bb.
 
@@ -26,51 +26,51 @@ define void @caller_to8_from0() {
 
 ; Caller isn't going to clean up any extra stack we allocate, so it
 ; can't be a tail call.
-  tail call void @callee_stack8([8 x i32] undef, i64 42)
+  tail call void @callee_stack8([8 x i64] undef, i64 42)
   ret void
 ; CHECK: bl callee_stack8
 }
 
-define void @caller_to8_from8([8 x i32], i64 %a) {
+define void @caller_to8_from8([8 x i64], i64 %a) {
 ; CHECK-LABEL: caller_to8_from8:
 ; CHECK-NOT: sub sp, sp,
 
 ; This should reuse our stack area for the 42
-  tail call void @callee_stack8([8 x i32] undef, i64 42)
+  tail call void @callee_stack8([8 x i64] undef, i64 42)
   ret void
 ; CHECK: str {{x[0-9]+}}, [sp]
 ; CHECK-NEXT: b callee_stack8
 }
 
-define void @caller_to16_from8([8 x i32], i64 %a) {
+define void @caller_to16_from8([8 x i64], i64 %a) {
 ; CHECK-LABEL: caller_to16_from8:
 
 ; Shouldn't be a tail call: we can't use SP+8 because our caller might
 ; have something there. This may sound obvious but implementation does
 ; some funky aligning.
-  tail call void @callee_stack16([8 x i32] undef, i64 undef, i64 undef)
+  tail call void @callee_stack16([8 x i64] undef, i64 undef, i64 undef)
 ; CHECK: bl callee_stack16
   ret void
 }
 
-define void @caller_to8_from24([8 x i32], i64 %a, i64 %b, i64 %c) {
+define void @caller_to8_from24([8 x i64], i64 %a, i64 %b, i64 %c) {
 ; CHECK-LABEL: caller_to8_from24:
 ; CHECK-NOT: sub sp, sp
 
 ; Reuse our area, putting "42" at incoming sp
-  tail call void @callee_stack8([8 x i32] undef, i64 42)
+  tail call void @callee_stack8([8 x i64] undef, i64 42)
   ret void
 ; CHECK: str {{x[0-9]+}}, [sp]
 ; CHECK-NEXT: b callee_stack8
 }
 
-define void @caller_to16_from16([8 x i32], i64 %a, i64 %b) {
+define void @caller_to16_from16([8 x i64], i64 %a, i64 %b) {
 ; CHECK-LABEL: caller_to16_from16:
 ; CHECK-NOT: sub sp, sp,
 
 ; Here we want to make sure that both loads happen before the stores:
 ; otherwise either %a or %b will be wrongly clobbered.
-  tail call void @callee_stack16([8 x i32] undef, i64 %b, i64 %a)
+  tail call void @callee_stack16([8 x i64] undef, i64 %b, i64 %a)
   ret void
 
 ; CHECK: ldr [[VAL0:x[0-9]+]],
diff --git a/llvm/test/CodeGen/AArch64/sign-return-address.ll b/llvm/test/CodeGen/AArch64/sign-return-address.ll
index dfd52f87f..51df38aa5 100644
--- a/llvm/test/CodeGen/AArch64/sign-return-address.ll
+++ b/llvm/test/CodeGen/AArch64/sign-return-address.ll
@@ -1,4 +1,5 @@
-; RUN: llc -mtriple=aarch64-none-eabi < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-none-eabi              < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-none-eabi -mattr=v8.3a < %s | FileCheck --check-prefix CHECK-V83A %s
 
 ; CHECK-LABEL: @leaf
 ; CHECK-NOT: paci{{[a,b]}}sp
@@ -22,19 +23,23 @@ define i32 @leaf_sign_non_leaf(i32 %x) "sign-return-address"="non-leaf"  {
 }
 
 ; CHECK-LABEL: @leaf_sign_all
-; CHECK: paciasp
-; CHECK: autiasp
-; CHECK: ret
+; CHECK:       hint #25
+; CHECK:       hint #29
+; CHECK:       ret
+; CHECK-V83A:  paciasp
+; CHECK-V83A:  retaa
 define i32 @leaf_sign_all(i32 %x) "sign-return-address"="all" {
   ret i32 %x
 }
 
-; CHECK: @leaf_clobbers_lr
-; CHECK: paciasp
-; CHECK: str x30, [sp, #-16]!
-; CHECK: ldr  x30, [sp], #16
-; CHECK-NEXT: autiasp
-; CHECK: ret
+; CHECK:             @leaf_clobbers_lr
+; CHECK:             hint #25
+; CHECK-V83A:        paciasp
+; CHECK, CHECK-V83A: str x30, [sp, #-16]!
+; CHECK, CHECK-V83A: ldr x30, [sp], #16
+; CHECK:             hint #29
+; CHECK:             ret
+; CHECK-V32A-NEXT:   retaa
 define i64 @leaf_clobbers_lr(i64 %x) "sign-return-address"="non-leaf"  {
   call void asm sideeffect "mov x30, $0", "r,~{lr}"(i64 %x) #1
   ret i64 %x
@@ -42,21 +47,25 @@ define i64 @leaf_clobbers_lr(i64 %x) "sign-return-address"="non-leaf"  {
 
 declare i32 @foo(i32)
 
-; CHECK: @non_leaf_sign_all
-; CHECK: paciasp
-; CHECK: autiasp
-; CHECK: ret
+; CHECK:      @non_leaf_sign_all
+; CHECK:      hint #25
+; CHECK:      hint #29
+; CHECK:      ret
+; CHECK-V83A: paciasp
+; CHECK-V83A: retaa
 define i32 @non_leaf_sign_all(i32 %x) "sign-return-address"="all" {
   %call = call i32 @foo(i32 %x)
   ret i32 %call
 }
 
-; CHECK: @non_leaf_sign_non_leaf
-; CHECK: paciasp
-; CHECK: str x30, [sp, #-16]!
-; CHECK: ldr  x30, [sp], #16
-; CHECK: autiasp
-; CHECK: ret
+; CHECK:             @non_leaf_sign_non_leaf
+; CHECK:             hint #25
+; CHECK-V83A:        paciasp
+; CHECK, CHECK-V83A: str x30, [sp, #-16]!
+; CHECK, CHECK-V83A: ldr x30, [sp], #16
+; CHECK:             hint #29
+; CHECK:             ret
+; CHECK-V83A:        retaa
 define i32 @non_leaf_sign_non_leaf(i32 %x) "sign-return-address"="non-leaf"  {
   %call = call i32 @foo(i32 %x)
   ret i32 %call
@@ -73,12 +82,14 @@ define i32 @leaf_sign_all_v83(i32 %x) "sign-return-address"="all" "target-featur
 
 declare fastcc i64 @bar(i64)
 
-; CHECK-LABEL: @spill_lr_and_tail_call
-; CHECK: paciasp
-; CHECK: str x30, [sp, #-16]!
-; CHECK: ldr  x30, [sp], #16
-; CHECK: autiasp
-; CHECK: b  bar
+; CHECK-LABEL:       @spill_lr_and_tail_call
+; CHECK:             hint #25
+; CHECK-V83A:        paciasp
+; CHECK, CHECK-V83A: str x30, [sp, #-16]!
+; CHECK, CHECK-V83A: ldr x30, [sp], #16
+; CHECK-V83A:        autiasp
+; CHECK:             hint #29
+; CHECK:             b bar
 define fastcc void @spill_lr_and_tail_call(i64 %x) "sign-return-address"="all" {
   call void asm sideeffect "mov x30, $0", "r,~{lr}"(i64 %x) #1
   tail call fastcc i64 @bar(i64 %x)
@@ -86,15 +97,19 @@ define fastcc void @spill_lr_and_tail_call(i64 %x) "sign-return-address"="all" {
 }
 
 ; CHECK-LABEL: @leaf_sign_all_a_key
-; CHECK: paciasp
-; CHECK: autiasp
+; CHECK:       hint #25
+; CHECK:       hint #29
+; CHECK-V83A:  paciasp
+; CHECK-V83A:  retaa
 define i32 @leaf_sign_all_a_key(i32 %x) "sign-return-address"="all" "sign-return-address-key"="a_key" {
   ret i32 %x
 }
 
 ; CHECK-LABEL: @leaf_sign_all_b_key
-; CHECK: pacibsp
-; CHECK: autibsp
+; CHECK:       hint #27
+; CHECK:       hint #31
+; CHECK-V83A:  pacibsp
+; CHECK-V83A:  retab
 define i32 @leaf_sign_all_b_key(i32 %x) "sign-return-address"="all" "sign-return-address-key"="b_key" {
   ret i32 %x
 }
diff --git a/llvm/test/CodeGen/AArch64/sink-copy-for-shrink-wrap.ll b/llvm/test/CodeGen/AArch64/sink-copy-for-shrink-wrap.ll
index 7c4a3238c..23c7f0b70 100644
--- a/llvm/test/CodeGen/AArch64/sink-copy-for-shrink-wrap.ll
+++ b/llvm/test/CodeGen/AArch64/sink-copy-for-shrink-wrap.ll
@@ -4,7 +4,7 @@
 ; CHECK-NOT: stp
 ; CHECK-NOT: mov w{{[0-9]+}}, w0
 ; CHECK-LABEL: %bb.1:
-; CHECK: stp x19
+; CHECK: stp {{.*}}x19
 ; CHECK: mov w{{[0-9]+}}, w0
 
 define i32 @shrinkwrapme(i32 %paramAcrossCall, i32 %paramNotAcrossCall) {
diff --git a/llvm/test/CodeGen/AArch64/space.ll b/llvm/test/CodeGen/AArch64/space.ll
new file mode 100644
index 000000000..746d03772
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/space.ll
@@ -0,0 +1,16 @@
+; RUN: llc -mtriple aarch64               %s -o - | FileCheck %s
+; RUN: llc -mtriple aarch64 -filetype=obj %s -o - | llvm-objdump --arch=aarch64  -d - | FileCheck %s --check-prefix=DUMP
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+
+define dso_local void @f(i64 %v) {
+entry:
+  %dummy = tail call i64 @llvm.aarch64.space(i32 32684, i64 %v)
+  ret void
+}
+; CHECK: // SPACE
+; CHECK-NEXT: ret
+; DUMP-LABEL: f:
+; DUMP-NEXT: ret
+
+declare dso_local i64 @llvm.aarch64.space(i32, i64) local_unnamed_addr #0
diff --git a/llvm/test/CodeGen/AArch64/speculation-hardening-dagisel.ll b/llvm/test/CodeGen/AArch64/speculation-hardening-dagisel.ll
index 4d13d9844..72f3170fb 100644
--- a/llvm/test/CodeGen/AArch64/speculation-hardening-dagisel.ll
+++ b/llvm/test/CodeGen/AArch64/speculation-hardening-dagisel.ll
@@ -18,7 +18,7 @@ entry:
 ; NOSLH-NOT: dsb sy
 ; NOSLH-NOT: isb
 if.then:
-  %0 = tail call i64 asm "autia1716", "={x17},{x16},0"(i64 %b, i64 %a)
+  %0 = tail call i64 asm "hint #12", "={x17},{x16},0"(i64 %b, i64 %a)
 ; CHECK: bl g
 ; SLH: dsb sy
 ; SLH: isb
diff --git a/llvm/test/CodeGen/AArch64/speculation-hardening-loads.ll b/llvm/test/CodeGen/AArch64/speculation-hardening-loads.ll
index e90fb19a5..c5aae0514 100644
--- a/llvm/test/CodeGen/AArch64/speculation-hardening-loads.ll
+++ b/llvm/test/CodeGen/AArch64/speculation-hardening-loads.ll
@@ -90,7 +90,7 @@ define i64 @no_masking_with_full_control_flow_barriers(i64 %a, i64 %b, i64* %p)
 ; CHECK: dsb sy
 ; CHECK: isb
 entry:
-  %0 = tail call i64 asm "autia1716", "={x17},{x16},0"(i64 %b, i64 %a)
+  %0 = tail call i64 asm "hint #12", "={x17},{x16},0"(i64 %b, i64 %a)
   %X = load i64, i64* %p, align 8
   %ret = add i64 %X, %0
 ; CHECK-NOT: csdb
diff --git a/llvm/test/CodeGen/AArch64/speculation-hardening.ll b/llvm/test/CodeGen/AArch64/speculation-hardening.ll
index a6c3ed683..81aea75e3 100644
--- a/llvm/test/CodeGen/AArch64/speculation-hardening.ll
+++ b/llvm/test/CodeGen/AArch64/speculation-hardening.ll
@@ -1,9 +1,9 @@
-; RUN: sed -e 's/SLHATTR/speculative_load_hardening/' %s | llc -verify-machineinstrs -mtriple=aarch64-none-linux-gnu | FileCheck %s --check-prefixes=CHECK,SLH,NOGISELSLH --dump-input-on-failure
-; RUN: sed -e 's/SLHATTR//' %s | llc -verify-machineinstrs -mtriple=aarch64-none-linux-gnu | FileCheck %s --check-prefixes=CHECK,NOSLH,NOGISELNOSLH --dump-input-on-failure
-; RUN: sed -e 's/SLHATTR/speculative_load_hardening/' %s | llc -verify-machineinstrs -mtriple=aarch64-none-linux-gnu -global-isel | FileCheck %s --check-prefixes=CHECK,SLH,GISELSLH --dump-input-on-failure
-; RUN sed -e 's/SLHATTR//' %s | llc -verify-machineinstrs -mtriple=aarch64-none-linux-gnu -global-isel | FileCheck %s --check-prefixes=CHECK,NOSLH,GISELNOSLH --dump-input-on-failure
-; RUN: sed -e 's/SLHATTR/speculative_load_hardening/' %s | llc -verify-machineinstrs -mtriple=aarch64-none-linux-gnu -fast-isel | FileCheck %s --check-prefixes=CHECK,SLH,NOGISELSLH --dump-input-on-failure
-; RUN: sed -e 's/SLHATTR//' %s | llc -verify-machineinstrs -mtriple=aarch64-none-linux-gnu -fast-isel | FileCheck %s --check-prefixes=CHECK,NOSLH,NOGISELNOSLH --dump-input-on-failure
+; RUN: sed -e 's/SLHATTR/speculative_load_hardening/' %s | llc -verify-machineinstrs -mtriple=aarch64-none-linux-gnu | FileCheck %s --check-prefixes=CHECK,SLH --dump-input-on-failure
+; RUN: sed -e 's/SLHATTR//' %s | llc -verify-machineinstrs -mtriple=aarch64-none-linux-gnu | FileCheck %s --check-prefixes=CHECK,NOSLH --dump-input-on-failure
+; RUN: sed -e 's/SLHATTR/speculative_load_hardening/' %s | llc -verify-machineinstrs -mtriple=aarch64-none-linux-gnu -global-isel | FileCheck %s --check-prefixes=CHECK,SLH --dump-input-on-failure
+; RUN sed -e 's/SLHATTR//' %s | llc -verify-machineinstrs -mtriple=aarch64-none-linux-gnu -global-isel | FileCheck %s --check-prefixes=CHECK,NOSLH --dump-input-on-failure
+; RUN: sed -e 's/SLHATTR/speculative_load_hardening/' %s | llc -verify-machineinstrs -mtriple=aarch64-none-linux-gnu -fast-isel | FileCheck %s --check-prefixes=CHECK,SLH --dump-input-on-failure
+; RUN: sed -e 's/SLHATTR//' %s | llc -verify-machineinstrs -mtriple=aarch64-none-linux-gnu -fast-isel | FileCheck %s --check-prefixes=CHECK,NOSLH --dump-input-on-failure
 
 define i32 @f(i8* nocapture readonly %p, i32 %i, i32 %N) local_unnamed_addr SLHATTR {
 ; CHECK-LABEL: f
@@ -54,24 +54,13 @@ return:                                           ; preds = %entry, %if.then
 ; Make sure that for a tail call, taint doesn't get put into SP twice.
 define i32 @tail_caller(i32 %a) local_unnamed_addr SLHATTR {
 ; CHECK-LABEL: tail_caller:
-; NOGISELSLH:     mov [[TMPREG:x[0-9]+]], sp
-; NOGISELSLH:     and [[TMPREG]], [[TMPREG]], x16
-; NOGISELSLH:     mov sp, [[TMPREG]]
-; NOGISELNOSLH-NOT:     mov [[TMPREG:x[0-9]+]], sp
-; NOGISELNOSLH-NOT:     and [[TMPREG]], [[TMPREG]], x16
-; NOGISELNOSLH-NOT:     mov sp, [[TMPREG]]
-; GISELSLH:     mov [[TMPREG:x[0-9]+]], sp
-; GISELSLH:     and [[TMPREG]], [[TMPREG]], x16
-; GISELSLH:     mov sp, [[TMPREG]]
-; GISELNOSLH-NOT:     mov [[TMPREG:x[0-9]+]], sp
-; GISELNOSLH-NOT:     and [[TMPREG]], [[TMPREG]], x16
-; GISELNOSLH-NOT:     mov sp, [[TMPREG]]
-;  GlobalISel doesn't optimize tail calls (yet?), so only check that
-;  cross-call taint register setup code is missing if a tail call was
-;  actually produced.
-; NOGISELSLH:     b tail_callee
-; GISELSLH:       bl tail_callee
-; GISELSLH:       cmp sp, #0
+; SLH:     mov [[TMPREG:x[0-9]+]], sp
+; SLH:     and [[TMPREG]], [[TMPREG]], x16
+; SLH:     mov sp, [[TMPREG]]
+; NOSLH-NOT:     mov [[TMPREG:x[0-9]+]], sp
+; NOSLH-NOT:     and [[TMPREG]], [[TMPREG]], x16
+; NOSLH-NOT:     mov sp, [[TMPREG]]
+; SLH:     b tail_callee
 ; SLH-NOT:        cmp sp, #0
   %call = tail call i32 @tail_callee(i32 %a)
   ret i32 %call
diff --git a/llvm/test/CodeGen/AArch64/sponentry.ll b/llvm/test/CodeGen/AArch64/sponentry.ll
index 7e14cf4ac..c308eb408 100644
--- a/llvm/test/CodeGen/AArch64/sponentry.ll
+++ b/llvm/test/CodeGen/AArch64/sponentry.ll
@@ -38,8 +38,8 @@ define dso_local void @foo([24 x i64]*) {
 
 ; CHECK: foo:
 ; CHECK: sub     sp, sp, #448
-; CHECK: add     x29, sp, #432
-; CHECK: add     x1, x29, #16
+; CHECK: add     x29, sp, #416
+; CHECK: add     x1, x29, #32
 ; CHECK: bl      _setjmpex
 
 ; NOFP: sub     sp, sp, #432
diff --git a/llvm/test/CodeGen/AArch64/srem-lkk.ll b/llvm/test/CodeGen/AArch64/srem-lkk.ll
new file mode 100644
index 000000000..321791e92
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/srem-lkk.ll
@@ -0,0 +1,149 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64-unknown-linux-gnu < %s | FileCheck %s
+
+define i32 @fold_srem_positive_odd(i32 %x) {
+; CHECK-LABEL: fold_srem_positive_odd:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #37253
+; CHECK-NEXT:    movk w8, #44150, lsl #16
+; CHECK-NEXT:    smull x8, w0, w8
+; CHECK-NEXT:    lsr x8, x8, #32
+; CHECK-NEXT:    add w8, w8, w0
+; CHECK-NEXT:    asr w9, w8, #6
+; CHECK-NEXT:    add w8, w9, w8, lsr #31
+; CHECK-NEXT:    mov w9, #95
+; CHECK-NEXT:    msub w0, w8, w9, w0
+; CHECK-NEXT:    ret
+  %1 = srem i32 %x, 95
+  ret i32 %1
+}
+
+
+define i32 @fold_srem_positive_even(i32 %x) {
+; CHECK-LABEL: fold_srem_positive_even:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #36849
+; CHECK-NEXT:    movk w8, #15827, lsl #16
+; CHECK-NEXT:    smull x8, w0, w8
+; CHECK-NEXT:    lsr x9, x8, #63
+; CHECK-NEXT:    asr x8, x8, #40
+; CHECK-NEXT:    add w8, w8, w9
+; CHECK-NEXT:    mov w9, #1060
+; CHECK-NEXT:    msub w0, w8, w9, w0
+; CHECK-NEXT:    ret
+  %1 = srem i32 %x, 1060
+  ret i32 %1
+}
+
+
+define i32 @fold_srem_negative_odd(i32 %x) {
+; CHECK-LABEL: fold_srem_negative_odd:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #65445
+; CHECK-NEXT:    movk w8, #42330, lsl #16
+; CHECK-NEXT:    smull x8, w0, w8
+; CHECK-NEXT:    lsr x9, x8, #63
+; CHECK-NEXT:    asr x8, x8, #40
+; CHECK-NEXT:    add w8, w8, w9
+; CHECK-NEXT:    mov w9, #-723
+; CHECK-NEXT:    msub w0, w8, w9, w0
+; CHECK-NEXT:    ret
+  %1 = srem i32 %x, -723
+  ret i32 %1
+}
+
+
+define i32 @fold_srem_negative_even(i32 %x) {
+; CHECK-LABEL: fold_srem_negative_even:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #62439
+; CHECK-NEXT:    movk w8, #64805, lsl #16
+; CHECK-NEXT:    smull x8, w0, w8
+; CHECK-NEXT:    lsr x9, x8, #63
+; CHECK-NEXT:    asr x8, x8, #40
+; CHECK-NEXT:    add w8, w8, w9
+; CHECK-NEXT:    mov w9, #-22981
+; CHECK-NEXT:    msub w0, w8, w9, w0
+; CHECK-NEXT:    ret
+  %1 = srem i32 %x, -22981
+  ret i32 %1
+}
+
+
+; Don't fold if we can combine srem with sdiv.
+define i32 @combine_srem_sdiv(i32 %x) {
+; CHECK-LABEL: combine_srem_sdiv:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #37253
+; CHECK-NEXT:    movk w8, #44150, lsl #16
+; CHECK-NEXT:    smull x8, w0, w8
+; CHECK-NEXT:    lsr x8, x8, #32
+; CHECK-NEXT:    add w8, w8, w0
+; CHECK-NEXT:    asr w9, w8, #6
+; CHECK-NEXT:    add w8, w9, w8, lsr #31
+; CHECK-NEXT:    mov w9, #95
+; CHECK-NEXT:    msub w9, w8, w9, w0
+; CHECK-NEXT:    add w0, w9, w8
+; CHECK-NEXT:    ret
+  %1 = srem i32 %x, 95
+  %2 = sdiv i32 %x, 95
+  %3 = add i32 %1, %2
+  ret i32 %3
+}
+
+; Don't fold for divisors that are a power of two.
+define i32 @dont_fold_srem_power_of_two(i32 %x) {
+; CHECK-LABEL: dont_fold_srem_power_of_two:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    add w8, w0, #63 // =63
+; CHECK-NEXT:    cmp w0, #0 // =0
+; CHECK-NEXT:    csel w8, w8, w0, lt
+; CHECK-NEXT:    and w8, w8, #0xffffffc0
+; CHECK-NEXT:    sub w0, w0, w8
+; CHECK-NEXT:    ret
+  %1 = srem i32 %x, 64
+  ret i32 %1
+}
+
+; Don't fold if the divisor is one.
+define i32 @dont_fold_srem_one(i32 %x) {
+; CHECK-LABEL: dont_fold_srem_one:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w0, wzr
+; CHECK-NEXT:    ret
+  %1 = srem i32 %x, 1
+  ret i32 %1
+}
+
+; Don't fold if the divisor is 2^31.
+define i32 @dont_fold_srem_i32_smax(i32 %x) {
+; CHECK-LABEL: dont_fold_srem_i32_smax:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #2147483647
+; CHECK-NEXT:    add w8, w0, w8
+; CHECK-NEXT:    cmp w0, #0 // =0
+; CHECK-NEXT:    csel w8, w8, w0, lt
+; CHECK-NEXT:    and w8, w8, #0x80000000
+; CHECK-NEXT:    add w0, w0, w8
+; CHECK-NEXT:    ret
+  %1 = srem i32 %x, 2147483648
+  ret i32 %1
+}
+
+; Don't fold i64 srem
+define i64 @dont_fold_srem_i64(i64 %x) {
+; CHECK-LABEL: dont_fold_srem_i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov x8, #58849
+; CHECK-NEXT:    movk x8, #48148, lsl #16
+; CHECK-NEXT:    movk x8, #33436, lsl #32
+; CHECK-NEXT:    movk x8, #21399, lsl #48
+; CHECK-NEXT:    smulh x8, x0, x8
+; CHECK-NEXT:    asr x9, x8, #5
+; CHECK-NEXT:    add x8, x9, x8, lsr #63
+; CHECK-NEXT:    mov w9, #98
+; CHECK-NEXT:    msub x0, x8, x9, x0
+; CHECK-NEXT:    ret
+  %1 = srem i64 %x, 98
+  ret i64 %1
+}
diff --git a/llvm/test/CodeGen/AArch64/srem-seteq-optsize.ll b/llvm/test/CodeGen/AArch64/srem-seteq-optsize.ll
new file mode 100644
index 000000000..e6037f6b3
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/srem-seteq-optsize.ll
@@ -0,0 +1,39 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64-unknown-linux-gnu < %s | FileCheck %s
+
+define i32 @test_minsize(i32 %X) optsize minsize nounwind readnone {
+; CHECK-LABEL: test_minsize:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #5
+; CHECK-NEXT:    sdiv w8, w0, w8
+; CHECK-NEXT:    add w8, w8, w8, lsl #2
+; CHECK-NEXT:    mov w9, #-10
+; CHECK-NEXT:    cmp w0, w8
+; CHECK-NEXT:    mov w8, #42
+; CHECK-NEXT:    csel w0, w8, w9, eq
+; CHECK-NEXT:    ret
+  %rem = srem i32 %X, 5
+  %cmp = icmp eq i32 %rem, 0
+  %ret = select i1 %cmp, i32 42, i32 -10
+  ret i32 %ret
+}
+
+define i32 @test_optsize(i32 %X) optsize nounwind readnone {
+; CHECK-LABEL: test_optsize:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #52429
+; CHECK-NEXT:    mov w9, #39321
+; CHECK-NEXT:    movk w8, #52428, lsl #16
+; CHECK-NEXT:    movk w9, #6553, lsl #16
+; CHECK-NEXT:    mov w10, #858993459
+; CHECK-NEXT:    madd w8, w0, w8, w9
+; CHECK-NEXT:    mov w11, #-10
+; CHECK-NEXT:    cmp w8, w10
+; CHECK-NEXT:    mov w8, #42
+; CHECK-NEXT:    csel w0, w8, w11, lo
+; CHECK-NEXT:    ret
+  %rem = srem i32 %X, 5
+  %cmp = icmp eq i32 %rem, 0
+  %ret = select i1 %cmp, i32 42, i32 -10
+  ret i32 %ret
+}
diff --git a/llvm/test/CodeGen/AArch64/srem-seteq-vec-nonsplat.ll b/llvm/test/CodeGen/AArch64/srem-seteq-vec-nonsplat.ll
new file mode 100644
index 000000000..7e78d11f6
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/srem-seteq-vec-nonsplat.ll
@@ -0,0 +1,847 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64-unknown-linux-gnu < %s | FileCheck %s
+
+; Odd+Even divisors
+define <4 x i32> @test_srem_odd_even(<4 x i32> %X) nounwind {
+; CHECK-LABEL: test_srem_odd_even:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI0_0
+; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI0_0]
+; CHECK-NEXT:    adrp x8, .LCPI0_1
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI0_1]
+; CHECK-NEXT:    adrp x8, .LCPI0_2
+; CHECK-NEXT:    smull2 v3.2d, v0.4s, v1.4s
+; CHECK-NEXT:    smull v1.2d, v0.2s, v1.2s
+; CHECK-NEXT:    uzp2 v1.4s, v1.4s, v3.4s
+; CHECK-NEXT:    ldr q3, [x8, :lo12:.LCPI0_2]
+; CHECK-NEXT:    adrp x8, .LCPI0_3
+; CHECK-NEXT:    mla v1.4s, v0.4s, v2.4s
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI0_3]
+; CHECK-NEXT:    neg v3.4s, v3.4s
+; CHECK-NEXT:    sshl v3.4s, v1.4s, v3.4s
+; CHECK-NEXT:    usra v3.4s, v1.4s, #31
+; CHECK-NEXT:    mls v0.4s, v3.4s, v2.4s
+; CHECK-NEXT:    cmeq v0.4s, v0.4s, #0
+; CHECK-NEXT:    movi v1.4s, #1
+; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    ret
+  %srem = srem <4 x i32> %X, <i32 5, i32 14, i32 25, i32 100>
+  %cmp = icmp eq <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+;==============================================================================;
+
+; One all-ones divisor in odd divisor
+define <4 x i32> @test_srem_odd_allones_eq(<4 x i32> %X) nounwind {
+; CHECK-LABEL: test_srem_odd_allones_eq:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x10, .LCPI1_0
+; CHECK-NEXT:    mov w8, #52429
+; CHECK-NEXT:    mov w9, #39321
+; CHECK-NEXT:    ldr q1, [x10, :lo12:.LCPI1_0]
+; CHECK-NEXT:    movk w8, #52428, lsl #16
+; CHECK-NEXT:    movk w9, #6553, lsl #16
+; CHECK-NEXT:    dup v2.4s, w8
+; CHECK-NEXT:    dup v3.4s, w9
+; CHECK-NEXT:    mla v3.4s, v0.4s, v2.4s
+; CHECK-NEXT:    cmhs v0.4s, v1.4s, v3.4s
+; CHECK-NEXT:    movi v1.4s, #1
+; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    ret
+  %srem = srem <4 x i32> %X, <i32 5, i32 5, i32 4294967295, i32 5>
+  %cmp = icmp eq <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+define <4 x i32> @test_srem_odd_allones_ne(<4 x i32> %X) nounwind {
+; CHECK-LABEL: test_srem_odd_allones_ne:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x10, .LCPI2_0
+; CHECK-NEXT:    mov w8, #52429
+; CHECK-NEXT:    mov w9, #39321
+; CHECK-NEXT:    ldr q1, [x10, :lo12:.LCPI2_0]
+; CHECK-NEXT:    movk w8, #52428, lsl #16
+; CHECK-NEXT:    movk w9, #6553, lsl #16
+; CHECK-NEXT:    dup v2.4s, w8
+; CHECK-NEXT:    dup v3.4s, w9
+; CHECK-NEXT:    mla v3.4s, v0.4s, v2.4s
+; CHECK-NEXT:    cmhi v0.4s, v3.4s, v1.4s
+; CHECK-NEXT:    movi v1.4s, #1
+; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    ret
+  %srem = srem <4 x i32> %X, <i32 5, i32 5, i32 4294967295, i32 5>
+  %cmp = icmp ne <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; One all-ones divisor in even divisor
+define <4 x i32> @test_srem_even_allones_eq(<4 x i32> %X) nounwind {
+; CHECK-LABEL: test_srem_even_allones_eq:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI3_0
+; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI3_0]
+; CHECK-NEXT:    adrp x8, .LCPI3_1
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI3_1]
+; CHECK-NEXT:    adrp x8, .LCPI3_2
+; CHECK-NEXT:    ldr q3, [x8, :lo12:.LCPI3_2]
+; CHECK-NEXT:    adrp x8, .LCPI3_3
+; CHECK-NEXT:    smull2 v4.2d, v0.4s, v1.4s
+; CHECK-NEXT:    smull v1.2d, v0.2s, v1.2s
+; CHECK-NEXT:    uzp2 v1.4s, v1.4s, v4.4s
+; CHECK-NEXT:    ldr q4, [x8, :lo12:.LCPI3_3]
+; CHECK-NEXT:    adrp x8, .LCPI3_4
+; CHECK-NEXT:    mla v1.4s, v0.4s, v2.4s
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI3_4]
+; CHECK-NEXT:    neg v3.4s, v3.4s
+; CHECK-NEXT:    sshl v3.4s, v1.4s, v3.4s
+; CHECK-NEXT:    ushr v1.4s, v1.4s, #31
+; CHECK-NEXT:    and v1.16b, v1.16b, v4.16b
+; CHECK-NEXT:    add v1.4s, v3.4s, v1.4s
+; CHECK-NEXT:    mls v0.4s, v1.4s, v2.4s
+; CHECK-NEXT:    cmeq v0.4s, v0.4s, #0
+; CHECK-NEXT:    movi v1.4s, #1
+; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    ret
+  %srem = srem <4 x i32> %X, <i32 14, i32 14, i32 4294967295, i32 14>
+  %cmp = icmp eq <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+define <4 x i32> @test_srem_even_allones_ne(<4 x i32> %X) nounwind {
+; CHECK-LABEL: test_srem_even_allones_ne:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI4_0
+; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI4_0]
+; CHECK-NEXT:    adrp x8, .LCPI4_1
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI4_1]
+; CHECK-NEXT:    adrp x8, .LCPI4_2
+; CHECK-NEXT:    ldr q3, [x8, :lo12:.LCPI4_2]
+; CHECK-NEXT:    adrp x8, .LCPI4_3
+; CHECK-NEXT:    smull2 v4.2d, v0.4s, v1.4s
+; CHECK-NEXT:    smull v1.2d, v0.2s, v1.2s
+; CHECK-NEXT:    uzp2 v1.4s, v1.4s, v4.4s
+; CHECK-NEXT:    ldr q4, [x8, :lo12:.LCPI4_3]
+; CHECK-NEXT:    adrp x8, .LCPI4_4
+; CHECK-NEXT:    mla v1.4s, v0.4s, v2.4s
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI4_4]
+; CHECK-NEXT:    neg v3.4s, v3.4s
+; CHECK-NEXT:    sshl v3.4s, v1.4s, v3.4s
+; CHECK-NEXT:    ushr v1.4s, v1.4s, #31
+; CHECK-NEXT:    and v1.16b, v1.16b, v4.16b
+; CHECK-NEXT:    add v1.4s, v3.4s, v1.4s
+; CHECK-NEXT:    mls v0.4s, v1.4s, v2.4s
+; CHECK-NEXT:    cmeq v0.4s, v0.4s, #0
+; CHECK-NEXT:    mvn v0.16b, v0.16b
+; CHECK-NEXT:    movi v1.4s, #1
+; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    ret
+  %srem = srem <4 x i32> %X, <i32 14, i32 14, i32 4294967295, i32 14>
+  %cmp = icmp ne <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; One all-ones divisor in odd+even divisor
+define <4 x i32> @test_srem_odd_even_allones_eq(<4 x i32> %X) nounwind {
+; CHECK-LABEL: test_srem_odd_even_allones_eq:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI5_0
+; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI5_0]
+; CHECK-NEXT:    adrp x8, .LCPI5_1
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI5_1]
+; CHECK-NEXT:    adrp x8, .LCPI5_2
+; CHECK-NEXT:    ldr q3, [x8, :lo12:.LCPI5_2]
+; CHECK-NEXT:    adrp x8, .LCPI5_3
+; CHECK-NEXT:    smull2 v4.2d, v0.4s, v1.4s
+; CHECK-NEXT:    smull v1.2d, v0.2s, v1.2s
+; CHECK-NEXT:    uzp2 v1.4s, v1.4s, v4.4s
+; CHECK-NEXT:    ldr q4, [x8, :lo12:.LCPI5_3]
+; CHECK-NEXT:    adrp x8, .LCPI5_4
+; CHECK-NEXT:    mla v1.4s, v0.4s, v2.4s
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI5_4]
+; CHECK-NEXT:    neg v3.4s, v3.4s
+; CHECK-NEXT:    sshl v3.4s, v1.4s, v3.4s
+; CHECK-NEXT:    ushr v1.4s, v1.4s, #31
+; CHECK-NEXT:    and v1.16b, v1.16b, v4.16b
+; CHECK-NEXT:    add v1.4s, v3.4s, v1.4s
+; CHECK-NEXT:    mls v0.4s, v1.4s, v2.4s
+; CHECK-NEXT:    cmeq v0.4s, v0.4s, #0
+; CHECK-NEXT:    movi v1.4s, #1
+; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    ret
+  %srem = srem <4 x i32> %X, <i32 5, i32 14, i32 4294967295, i32 100>
+  %cmp = icmp eq <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+define <4 x i32> @test_srem_odd_even_allones_ne(<4 x i32> %X) nounwind {
+; CHECK-LABEL: test_srem_odd_even_allones_ne:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI6_0
+; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI6_0]
+; CHECK-NEXT:    adrp x8, .LCPI6_1
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI6_1]
+; CHECK-NEXT:    adrp x8, .LCPI6_2
+; CHECK-NEXT:    ldr q3, [x8, :lo12:.LCPI6_2]
+; CHECK-NEXT:    adrp x8, .LCPI6_3
+; CHECK-NEXT:    smull2 v4.2d, v0.4s, v1.4s
+; CHECK-NEXT:    smull v1.2d, v0.2s, v1.2s
+; CHECK-NEXT:    uzp2 v1.4s, v1.4s, v4.4s
+; CHECK-NEXT:    ldr q4, [x8, :lo12:.LCPI6_3]
+; CHECK-NEXT:    adrp x8, .LCPI6_4
+; CHECK-NEXT:    mla v1.4s, v0.4s, v2.4s
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI6_4]
+; CHECK-NEXT:    neg v3.4s, v3.4s
+; CHECK-NEXT:    sshl v3.4s, v1.4s, v3.4s
+; CHECK-NEXT:    ushr v1.4s, v1.4s, #31
+; CHECK-NEXT:    and v1.16b, v1.16b, v4.16b
+; CHECK-NEXT:    add v1.4s, v3.4s, v1.4s
+; CHECK-NEXT:    mls v0.4s, v1.4s, v2.4s
+; CHECK-NEXT:    cmeq v0.4s, v0.4s, #0
+; CHECK-NEXT:    mvn v0.16b, v0.16b
+; CHECK-NEXT:    movi v1.4s, #1
+; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    ret
+  %srem = srem <4 x i32> %X, <i32 5, i32 14, i32 4294967295, i32 100>
+  %cmp = icmp ne <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+;------------------------------------------------------------------------------;
+
+; One power-of-two divisor in odd divisor
+define <4 x i32> @test_srem_odd_poweroftwo(<4 x i32> %X) nounwind {
+; CHECK-LABEL: test_srem_odd_poweroftwo:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI7_0
+; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI7_0]
+; CHECK-NEXT:    adrp x8, .LCPI7_1
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI7_1]
+; CHECK-NEXT:    adrp x8, .LCPI7_2
+; CHECK-NEXT:    smull2 v3.2d, v0.4s, v1.4s
+; CHECK-NEXT:    smull v1.2d, v0.2s, v1.2s
+; CHECK-NEXT:    uzp2 v1.4s, v1.4s, v3.4s
+; CHECK-NEXT:    ldr q3, [x8, :lo12:.LCPI7_2]
+; CHECK-NEXT:    adrp x8, .LCPI7_3
+; CHECK-NEXT:    mla v1.4s, v0.4s, v2.4s
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI7_3]
+; CHECK-NEXT:    neg v3.4s, v3.4s
+; CHECK-NEXT:    sshl v3.4s, v1.4s, v3.4s
+; CHECK-NEXT:    usra v3.4s, v1.4s, #31
+; CHECK-NEXT:    mls v0.4s, v3.4s, v2.4s
+; CHECK-NEXT:    cmeq v0.4s, v0.4s, #0
+; CHECK-NEXT:    movi v1.4s, #1
+; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    ret
+  %srem = srem <4 x i32> %X, <i32 5, i32 5, i32 16, i32 5>
+  %cmp = icmp eq <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; One power-of-two divisor in even divisor
+define <4 x i32> @test_srem_even_poweroftwo(<4 x i32> %X) nounwind {
+; CHECK-LABEL: test_srem_even_poweroftwo:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI8_0
+; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI8_0]
+; CHECK-NEXT:    adrp x8, .LCPI8_1
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI8_1]
+; CHECK-NEXT:    smull2 v3.2d, v0.4s, v1.4s
+; CHECK-NEXT:    smull v1.2d, v0.2s, v1.2s
+; CHECK-NEXT:    uzp2 v1.4s, v1.4s, v3.4s
+; CHECK-NEXT:    add v1.4s, v1.4s, v0.4s
+; CHECK-NEXT:    sshr v3.4s, v1.4s, #3
+; CHECK-NEXT:    usra v3.4s, v1.4s, #31
+; CHECK-NEXT:    mls v0.4s, v3.4s, v2.4s
+; CHECK-NEXT:    cmeq v0.4s, v0.4s, #0
+; CHECK-NEXT:    movi v1.4s, #1
+; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    ret
+  %srem = srem <4 x i32> %X, <i32 14, i32 14, i32 16, i32 14>
+  %cmp = icmp eq <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; One power-of-two divisor in odd+even divisor
+define <4 x i32> @test_srem_odd_even_poweroftwo(<4 x i32> %X) nounwind {
+; CHECK-LABEL: test_srem_odd_even_poweroftwo:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI9_0
+; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI9_0]
+; CHECK-NEXT:    adrp x8, .LCPI9_1
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI9_1]
+; CHECK-NEXT:    adrp x8, .LCPI9_2
+; CHECK-NEXT:    smull2 v3.2d, v0.4s, v1.4s
+; CHECK-NEXT:    smull v1.2d, v0.2s, v1.2s
+; CHECK-NEXT:    uzp2 v1.4s, v1.4s, v3.4s
+; CHECK-NEXT:    ldr q3, [x8, :lo12:.LCPI9_2]
+; CHECK-NEXT:    adrp x8, .LCPI9_3
+; CHECK-NEXT:    mla v1.4s, v0.4s, v2.4s
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI9_3]
+; CHECK-NEXT:    neg v3.4s, v3.4s
+; CHECK-NEXT:    sshl v3.4s, v1.4s, v3.4s
+; CHECK-NEXT:    usra v3.4s, v1.4s, #31
+; CHECK-NEXT:    mls v0.4s, v3.4s, v2.4s
+; CHECK-NEXT:    cmeq v0.4s, v0.4s, #0
+; CHECK-NEXT:    movi v1.4s, #1
+; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    ret
+  %srem = srem <4 x i32> %X, <i32 5, i32 14, i32 16, i32 100>
+  %cmp = icmp eq <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+;------------------------------------------------------------------------------;
+
+; One one divisor in odd divisor
+define <4 x i32> @test_srem_odd_one(<4 x i32> %X) nounwind {
+; CHECK-LABEL: test_srem_odd_one:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x10, .LCPI10_0
+; CHECK-NEXT:    mov w8, #52429
+; CHECK-NEXT:    mov w9, #39321
+; CHECK-NEXT:    ldr q1, [x10, :lo12:.LCPI10_0]
+; CHECK-NEXT:    movk w8, #52428, lsl #16
+; CHECK-NEXT:    movk w9, #6553, lsl #16
+; CHECK-NEXT:    dup v2.4s, w8
+; CHECK-NEXT:    dup v3.4s, w9
+; CHECK-NEXT:    mla v3.4s, v0.4s, v2.4s
+; CHECK-NEXT:    cmhs v0.4s, v1.4s, v3.4s
+; CHECK-NEXT:    movi v1.4s, #1
+; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    ret
+  %srem = srem <4 x i32> %X, <i32 5, i32 5, i32 1, i32 5>
+  %cmp = icmp eq <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; One one divisor in even divisor
+define <4 x i32> @test_srem_even_one(<4 x i32> %X) nounwind {
+; CHECK-LABEL: test_srem_even_one:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI11_0
+; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI11_0]
+; CHECK-NEXT:    adrp x8, .LCPI11_1
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI11_1]
+; CHECK-NEXT:    adrp x8, .LCPI11_2
+; CHECK-NEXT:    ldr q3, [x8, :lo12:.LCPI11_2]
+; CHECK-NEXT:    smull2 v4.2d, v0.4s, v1.4s
+; CHECK-NEXT:    smull v1.2d, v0.2s, v1.2s
+; CHECK-NEXT:    adrp x8, .LCPI11_3
+; CHECK-NEXT:    uzp2 v1.4s, v1.4s, v4.4s
+; CHECK-NEXT:    ldr q4, [x8, :lo12:.LCPI11_3]
+; CHECK-NEXT:    neg v2.4s, v2.4s
+; CHECK-NEXT:    add v1.4s, v1.4s, v0.4s
+; CHECK-NEXT:    sshl v2.4s, v1.4s, v2.4s
+; CHECK-NEXT:    ushr v1.4s, v1.4s, #31
+; CHECK-NEXT:    and v1.16b, v1.16b, v3.16b
+; CHECK-NEXT:    add v1.4s, v2.4s, v1.4s
+; CHECK-NEXT:    mls v0.4s, v1.4s, v4.4s
+; CHECK-NEXT:    cmeq v0.4s, v0.4s, #0
+; CHECK-NEXT:    movi v1.4s, #1
+; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    ret
+  %srem = srem <4 x i32> %X, <i32 14, i32 14, i32 1, i32 14>
+  %cmp = icmp eq <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; One one divisor in odd+even divisor
+define <4 x i32> @test_srem_odd_even_one(<4 x i32> %X) nounwind {
+; CHECK-LABEL: test_srem_odd_even_one:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI12_0
+; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI12_0]
+; CHECK-NEXT:    adrp x8, .LCPI12_1
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI12_1]
+; CHECK-NEXT:    adrp x8, .LCPI12_2
+; CHECK-NEXT:    ldr q3, [x8, :lo12:.LCPI12_2]
+; CHECK-NEXT:    adrp x8, .LCPI12_3
+; CHECK-NEXT:    smull2 v4.2d, v0.4s, v1.4s
+; CHECK-NEXT:    smull v1.2d, v0.2s, v1.2s
+; CHECK-NEXT:    uzp2 v1.4s, v1.4s, v4.4s
+; CHECK-NEXT:    ldr q4, [x8, :lo12:.LCPI12_3]
+; CHECK-NEXT:    adrp x8, .LCPI12_4
+; CHECK-NEXT:    mla v1.4s, v0.4s, v2.4s
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI12_4]
+; CHECK-NEXT:    neg v3.4s, v3.4s
+; CHECK-NEXT:    sshl v3.4s, v1.4s, v3.4s
+; CHECK-NEXT:    ushr v1.4s, v1.4s, #31
+; CHECK-NEXT:    and v1.16b, v1.16b, v4.16b
+; CHECK-NEXT:    add v1.4s, v3.4s, v1.4s
+; CHECK-NEXT:    mls v0.4s, v1.4s, v2.4s
+; CHECK-NEXT:    cmeq v0.4s, v0.4s, #0
+; CHECK-NEXT:    movi v1.4s, #1
+; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    ret
+  %srem = srem <4 x i32> %X, <i32 5, i32 14, i32 1, i32 100>
+  %cmp = icmp eq <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+;------------------------------------------------------------------------------;
+
+; One INT_MIN divisor in odd divisor
+define <4 x i32> @test_srem_odd_INT_MIN(<4 x i32> %X) nounwind {
+; CHECK-LABEL: test_srem_odd_INT_MIN:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI13_0
+; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI13_0]
+; CHECK-NEXT:    adrp x8, .LCPI13_1
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI13_1]
+; CHECK-NEXT:    adrp x8, .LCPI13_2
+; CHECK-NEXT:    smull2 v3.2d, v0.4s, v1.4s
+; CHECK-NEXT:    smull v1.2d, v0.2s, v1.2s
+; CHECK-NEXT:    uzp2 v1.4s, v1.4s, v3.4s
+; CHECK-NEXT:    ldr q3, [x8, :lo12:.LCPI13_2]
+; CHECK-NEXT:    adrp x8, .LCPI13_3
+; CHECK-NEXT:    mla v1.4s, v0.4s, v2.4s
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI13_3]
+; CHECK-NEXT:    neg v3.4s, v3.4s
+; CHECK-NEXT:    sshl v3.4s, v1.4s, v3.4s
+; CHECK-NEXT:    usra v3.4s, v1.4s, #31
+; CHECK-NEXT:    mls v0.4s, v3.4s, v2.4s
+; CHECK-NEXT:    cmeq v0.4s, v0.4s, #0
+; CHECK-NEXT:    movi v1.4s, #1
+; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    ret
+  %srem = srem <4 x i32> %X, <i32 5, i32 5, i32 2147483648, i32 5>
+  %cmp = icmp eq <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; One INT_MIN divisor in even divisor
+define <4 x i32> @test_srem_even_INT_MIN(<4 x i32> %X) nounwind {
+; CHECK-LABEL: test_srem_even_INT_MIN:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI14_0
+; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI14_0]
+; CHECK-NEXT:    adrp x8, .LCPI14_1
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI14_1]
+; CHECK-NEXT:    adrp x8, .LCPI14_2
+; CHECK-NEXT:    smull2 v3.2d, v0.4s, v1.4s
+; CHECK-NEXT:    smull v1.2d, v0.2s, v1.2s
+; CHECK-NEXT:    uzp2 v1.4s, v1.4s, v3.4s
+; CHECK-NEXT:    ldr q3, [x8, :lo12:.LCPI14_2]
+; CHECK-NEXT:    adrp x8, .LCPI14_3
+; CHECK-NEXT:    mla v1.4s, v0.4s, v2.4s
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI14_3]
+; CHECK-NEXT:    neg v3.4s, v3.4s
+; CHECK-NEXT:    sshl v3.4s, v1.4s, v3.4s
+; CHECK-NEXT:    usra v3.4s, v1.4s, #31
+; CHECK-NEXT:    mls v0.4s, v3.4s, v2.4s
+; CHECK-NEXT:    cmeq v0.4s, v0.4s, #0
+; CHECK-NEXT:    movi v1.4s, #1
+; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    ret
+  %srem = srem <4 x i32> %X, <i32 14, i32 14, i32 2147483648, i32 14>
+  %cmp = icmp eq <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; One INT_MIN divisor in odd+even divisor
+define <4 x i32> @test_srem_odd_even_INT_MIN(<4 x i32> %X) nounwind {
+; CHECK-LABEL: test_srem_odd_even_INT_MIN:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI15_0
+; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI15_0]
+; CHECK-NEXT:    adrp x8, .LCPI15_1
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI15_1]
+; CHECK-NEXT:    adrp x8, .LCPI15_2
+; CHECK-NEXT:    smull2 v3.2d, v0.4s, v1.4s
+; CHECK-NEXT:    smull v1.2d, v0.2s, v1.2s
+; CHECK-NEXT:    uzp2 v1.4s, v1.4s, v3.4s
+; CHECK-NEXT:    ldr q3, [x8, :lo12:.LCPI15_2]
+; CHECK-NEXT:    adrp x8, .LCPI15_3
+; CHECK-NEXT:    mla v1.4s, v0.4s, v2.4s
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI15_3]
+; CHECK-NEXT:    neg v3.4s, v3.4s
+; CHECK-NEXT:    sshl v3.4s, v1.4s, v3.4s
+; CHECK-NEXT:    usra v3.4s, v1.4s, #31
+; CHECK-NEXT:    mls v0.4s, v3.4s, v2.4s
+; CHECK-NEXT:    cmeq v0.4s, v0.4s, #0
+; CHECK-NEXT:    movi v1.4s, #1
+; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    ret
+  %srem = srem <4 x i32> %X, <i32 5, i32 14, i32 2147483648, i32 100>
+  %cmp = icmp eq <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+;==============================================================================;
+
+; One all-ones divisor and power-of-two divisor divisor in odd divisor
+define <4 x i32> @test_srem_odd_allones_and_poweroftwo(<4 x i32> %X) nounwind {
+; CHECK-LABEL: test_srem_odd_allones_and_poweroftwo:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI16_0
+; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI16_0]
+; CHECK-NEXT:    adrp x8, .LCPI16_1
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI16_1]
+; CHECK-NEXT:    adrp x8, .LCPI16_2
+; CHECK-NEXT:    ldr q3, [x8, :lo12:.LCPI16_2]
+; CHECK-NEXT:    adrp x8, .LCPI16_3
+; CHECK-NEXT:    smull2 v4.2d, v0.4s, v1.4s
+; CHECK-NEXT:    smull v1.2d, v0.2s, v1.2s
+; CHECK-NEXT:    uzp2 v1.4s, v1.4s, v4.4s
+; CHECK-NEXT:    ldr q4, [x8, :lo12:.LCPI16_3]
+; CHECK-NEXT:    adrp x8, .LCPI16_4
+; CHECK-NEXT:    mla v1.4s, v0.4s, v2.4s
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI16_4]
+; CHECK-NEXT:    neg v3.4s, v3.4s
+; CHECK-NEXT:    sshl v3.4s, v1.4s, v3.4s
+; CHECK-NEXT:    ushr v1.4s, v1.4s, #31
+; CHECK-NEXT:    and v1.16b, v1.16b, v4.16b
+; CHECK-NEXT:    add v1.4s, v3.4s, v1.4s
+; CHECK-NEXT:    mls v0.4s, v1.4s, v2.4s
+; CHECK-NEXT:    cmeq v0.4s, v0.4s, #0
+; CHECK-NEXT:    movi v1.4s, #1
+; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    ret
+  %srem = srem <4 x i32> %X, <i32 5, i32 4294967295, i32 16, i32 5>
+  %cmp = icmp eq <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; One all-ones divisor and power-of-two divisor divisor in even divisor
+define <4 x i32> @test_srem_even_allones_and_poweroftwo(<4 x i32> %X) nounwind {
+; CHECK-LABEL: test_srem_even_allones_and_poweroftwo:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI17_0
+; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI17_0]
+; CHECK-NEXT:    adrp x8, .LCPI17_1
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI17_1]
+; CHECK-NEXT:    adrp x8, .LCPI17_2
+; CHECK-NEXT:    ldr q3, [x8, :lo12:.LCPI17_2]
+; CHECK-NEXT:    adrp x8, .LCPI17_3
+; CHECK-NEXT:    smull2 v4.2d, v0.4s, v1.4s
+; CHECK-NEXT:    smull v1.2d, v0.2s, v1.2s
+; CHECK-NEXT:    uzp2 v1.4s, v1.4s, v4.4s
+; CHECK-NEXT:    ldr q4, [x8, :lo12:.LCPI17_3]
+; CHECK-NEXT:    adrp x8, .LCPI17_4
+; CHECK-NEXT:    mla v1.4s, v0.4s, v2.4s
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI17_4]
+; CHECK-NEXT:    neg v3.4s, v3.4s
+; CHECK-NEXT:    sshl v3.4s, v1.4s, v3.4s
+; CHECK-NEXT:    ushr v1.4s, v1.4s, #31
+; CHECK-NEXT:    and v1.16b, v1.16b, v4.16b
+; CHECK-NEXT:    add v1.4s, v3.4s, v1.4s
+; CHECK-NEXT:    mls v0.4s, v1.4s, v2.4s
+; CHECK-NEXT:    cmeq v0.4s, v0.4s, #0
+; CHECK-NEXT:    movi v1.4s, #1
+; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    ret
+  %srem = srem <4 x i32> %X, <i32 14, i32 4294967295, i32 16, i32 14>
+  %cmp = icmp eq <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; One all-ones divisor and power-of-two divisor divisor in odd+even divisor
+define <4 x i32> @test_srem_odd_even_allones_and_poweroftwo(<4 x i32> %X) nounwind {
+; CHECK-LABEL: test_srem_odd_even_allones_and_poweroftwo:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI18_0
+; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI18_0]
+; CHECK-NEXT:    adrp x8, .LCPI18_1
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI18_1]
+; CHECK-NEXT:    adrp x8, .LCPI18_2
+; CHECK-NEXT:    ldr q3, [x8, :lo12:.LCPI18_2]
+; CHECK-NEXT:    adrp x8, .LCPI18_3
+; CHECK-NEXT:    smull2 v4.2d, v0.4s, v1.4s
+; CHECK-NEXT:    smull v1.2d, v0.2s, v1.2s
+; CHECK-NEXT:    uzp2 v1.4s, v1.4s, v4.4s
+; CHECK-NEXT:    ldr q4, [x8, :lo12:.LCPI18_3]
+; CHECK-NEXT:    adrp x8, .LCPI18_4
+; CHECK-NEXT:    mla v1.4s, v0.4s, v2.4s
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI18_4]
+; CHECK-NEXT:    neg v3.4s, v3.4s
+; CHECK-NEXT:    sshl v3.4s, v1.4s, v3.4s
+; CHECK-NEXT:    ushr v1.4s, v1.4s, #31
+; CHECK-NEXT:    and v1.16b, v1.16b, v4.16b
+; CHECK-NEXT:    add v1.4s, v3.4s, v1.4s
+; CHECK-NEXT:    mls v0.4s, v1.4s, v2.4s
+; CHECK-NEXT:    cmeq v0.4s, v0.4s, #0
+; CHECK-NEXT:    movi v1.4s, #1
+; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    ret
+  %srem = srem <4 x i32> %X, <i32 5, i32 4294967295, i32 16, i32 100>
+  %cmp = icmp eq <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+;------------------------------------------------------------------------------;
+
+; One all-ones divisor and one one divisor in odd divisor
+define <4 x i32> @test_srem_odd_allones_and_one(<4 x i32> %X) nounwind {
+; CHECK-LABEL: test_srem_odd_allones_and_one:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x10, .LCPI19_0
+; CHECK-NEXT:    mov w8, #52429
+; CHECK-NEXT:    mov w9, #39321
+; CHECK-NEXT:    ldr q1, [x10, :lo12:.LCPI19_0]
+; CHECK-NEXT:    movk w8, #52428, lsl #16
+; CHECK-NEXT:    movk w9, #6553, lsl #16
+; CHECK-NEXT:    dup v2.4s, w8
+; CHECK-NEXT:    dup v3.4s, w9
+; CHECK-NEXT:    mla v3.4s, v0.4s, v2.4s
+; CHECK-NEXT:    cmhs v0.4s, v1.4s, v3.4s
+; CHECK-NEXT:    movi v1.4s, #1
+; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    ret
+  %srem = srem <4 x i32> %X, <i32 5, i32 4294967295, i32 1, i32 5>
+  %cmp = icmp eq <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; One all-ones divisor and one one divisor in even divisor
+define <4 x i32> @test_srem_even_allones_and_one(<4 x i32> %X) nounwind {
+; CHECK-LABEL: test_srem_even_allones_and_one:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI20_0
+; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI20_0]
+; CHECK-NEXT:    adrp x8, .LCPI20_1
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI20_1]
+; CHECK-NEXT:    adrp x8, .LCPI20_2
+; CHECK-NEXT:    ldr q3, [x8, :lo12:.LCPI20_2]
+; CHECK-NEXT:    adrp x8, .LCPI20_3
+; CHECK-NEXT:    smull2 v4.2d, v0.4s, v1.4s
+; CHECK-NEXT:    smull v1.2d, v0.2s, v1.2s
+; CHECK-NEXT:    uzp2 v1.4s, v1.4s, v4.4s
+; CHECK-NEXT:    ldr q4, [x8, :lo12:.LCPI20_3]
+; CHECK-NEXT:    adrp x8, .LCPI20_4
+; CHECK-NEXT:    mla v1.4s, v0.4s, v2.4s
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI20_4]
+; CHECK-NEXT:    neg v3.4s, v3.4s
+; CHECK-NEXT:    sshl v3.4s, v1.4s, v3.4s
+; CHECK-NEXT:    ushr v1.4s, v1.4s, #31
+; CHECK-NEXT:    and v1.16b, v1.16b, v4.16b
+; CHECK-NEXT:    add v1.4s, v3.4s, v1.4s
+; CHECK-NEXT:    mls v0.4s, v1.4s, v2.4s
+; CHECK-NEXT:    cmeq v0.4s, v0.4s, #0
+; CHECK-NEXT:    movi v1.4s, #1
+; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    ret
+  %srem = srem <4 x i32> %X, <i32 14, i32 4294967295, i32 1, i32 14>
+  %cmp = icmp eq <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; One all-ones divisor and one one divisor in odd+even divisor
+define <4 x i32> @test_srem_odd_even_allones_and_one(<4 x i32> %X) nounwind {
+; CHECK-LABEL: test_srem_odd_even_allones_and_one:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI21_0
+; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI21_0]
+; CHECK-NEXT:    adrp x8, .LCPI21_1
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI21_1]
+; CHECK-NEXT:    adrp x8, .LCPI21_2
+; CHECK-NEXT:    ldr q3, [x8, :lo12:.LCPI21_2]
+; CHECK-NEXT:    adrp x8, .LCPI21_3
+; CHECK-NEXT:    smull2 v4.2d, v0.4s, v1.4s
+; CHECK-NEXT:    smull v1.2d, v0.2s, v1.2s
+; CHECK-NEXT:    uzp2 v1.4s, v1.4s, v4.4s
+; CHECK-NEXT:    ldr q4, [x8, :lo12:.LCPI21_3]
+; CHECK-NEXT:    adrp x8, .LCPI21_4
+; CHECK-NEXT:    mla v1.4s, v0.4s, v2.4s
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI21_4]
+; CHECK-NEXT:    neg v3.4s, v3.4s
+; CHECK-NEXT:    sshl v3.4s, v1.4s, v3.4s
+; CHECK-NEXT:    ushr v1.4s, v1.4s, #31
+; CHECK-NEXT:    and v1.16b, v1.16b, v4.16b
+; CHECK-NEXT:    add v1.4s, v3.4s, v1.4s
+; CHECK-NEXT:    mls v0.4s, v1.4s, v2.4s
+; CHECK-NEXT:    cmeq v0.4s, v0.4s, #0
+; CHECK-NEXT:    movi v1.4s, #1
+; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    ret
+  %srem = srem <4 x i32> %X, <i32 5, i32 4294967295, i32 1, i32 100>
+  %cmp = icmp eq <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+;------------------------------------------------------------------------------;
+
+; One power-of-two divisor divisor and one divisor in odd divisor
+define <4 x i32> @test_srem_odd_poweroftwo_and_one(<4 x i32> %X) nounwind {
+; CHECK-LABEL: test_srem_odd_poweroftwo_and_one:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI22_0
+; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI22_0]
+; CHECK-NEXT:    adrp x8, .LCPI22_1
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI22_1]
+; CHECK-NEXT:    adrp x8, .LCPI22_2
+; CHECK-NEXT:    ldr q3, [x8, :lo12:.LCPI22_2]
+; CHECK-NEXT:    adrp x8, .LCPI22_3
+; CHECK-NEXT:    smull2 v4.2d, v0.4s, v1.4s
+; CHECK-NEXT:    smull v1.2d, v0.2s, v1.2s
+; CHECK-NEXT:    uzp2 v1.4s, v1.4s, v4.4s
+; CHECK-NEXT:    ldr q4, [x8, :lo12:.LCPI22_3]
+; CHECK-NEXT:    adrp x8, .LCPI22_4
+; CHECK-NEXT:    mla v1.4s, v0.4s, v2.4s
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI22_4]
+; CHECK-NEXT:    neg v3.4s, v3.4s
+; CHECK-NEXT:    sshl v3.4s, v1.4s, v3.4s
+; CHECK-NEXT:    ushr v1.4s, v1.4s, #31
+; CHECK-NEXT:    and v1.16b, v1.16b, v4.16b
+; CHECK-NEXT:    add v1.4s, v3.4s, v1.4s
+; CHECK-NEXT:    mls v0.4s, v1.4s, v2.4s
+; CHECK-NEXT:    cmeq v0.4s, v0.4s, #0
+; CHECK-NEXT:    movi v1.4s, #1
+; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    ret
+  %srem = srem <4 x i32> %X, <i32 5, i32 16, i32 1, i32 5>
+  %cmp = icmp eq <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; One power-of-two divisor divisor and one divisor in even divisor
+define <4 x i32> @test_srem_even_poweroftwo_and_one(<4 x i32> %X) nounwind {
+; CHECK-LABEL: test_srem_even_poweroftwo_and_one:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI23_0
+; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI23_0]
+; CHECK-NEXT:    adrp x8, .LCPI23_1
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI23_1]
+; CHECK-NEXT:    adrp x8, .LCPI23_2
+; CHECK-NEXT:    ldr q3, [x8, :lo12:.LCPI23_2]
+; CHECK-NEXT:    smull2 v4.2d, v0.4s, v1.4s
+; CHECK-NEXT:    smull v1.2d, v0.2s, v1.2s
+; CHECK-NEXT:    adrp x8, .LCPI23_3
+; CHECK-NEXT:    uzp2 v1.4s, v1.4s, v4.4s
+; CHECK-NEXT:    ldr q4, [x8, :lo12:.LCPI23_3]
+; CHECK-NEXT:    neg v2.4s, v2.4s
+; CHECK-NEXT:    add v1.4s, v1.4s, v0.4s
+; CHECK-NEXT:    sshl v2.4s, v1.4s, v2.4s
+; CHECK-NEXT:    ushr v1.4s, v1.4s, #31
+; CHECK-NEXT:    and v1.16b, v1.16b, v3.16b
+; CHECK-NEXT:    add v1.4s, v2.4s, v1.4s
+; CHECK-NEXT:    mls v0.4s, v1.4s, v4.4s
+; CHECK-NEXT:    cmeq v0.4s, v0.4s, #0
+; CHECK-NEXT:    movi v1.4s, #1
+; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    ret
+  %srem = srem <4 x i32> %X, <i32 14, i32 16, i32 1, i32 14>
+  %cmp = icmp eq <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; One power-of-two divisor divisor and one divisor in odd+even divisor
+define <4 x i32> @test_srem_odd_even_poweroftwo_and_one(<4 x i32> %X) nounwind {
+; CHECK-LABEL: test_srem_odd_even_poweroftwo_and_one:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI24_0
+; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI24_0]
+; CHECK-NEXT:    adrp x8, .LCPI24_1
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI24_1]
+; CHECK-NEXT:    adrp x8, .LCPI24_2
+; CHECK-NEXT:    ldr q3, [x8, :lo12:.LCPI24_2]
+; CHECK-NEXT:    adrp x8, .LCPI24_3
+; CHECK-NEXT:    smull2 v4.2d, v0.4s, v1.4s
+; CHECK-NEXT:    smull v1.2d, v0.2s, v1.2s
+; CHECK-NEXT:    uzp2 v1.4s, v1.4s, v4.4s
+; CHECK-NEXT:    ldr q4, [x8, :lo12:.LCPI24_3]
+; CHECK-NEXT:    adrp x8, .LCPI24_4
+; CHECK-NEXT:    mla v1.4s, v0.4s, v2.4s
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI24_4]
+; CHECK-NEXT:    neg v3.4s, v3.4s
+; CHECK-NEXT:    sshl v3.4s, v1.4s, v3.4s
+; CHECK-NEXT:    ushr v1.4s, v1.4s, #31
+; CHECK-NEXT:    and v1.16b, v1.16b, v4.16b
+; CHECK-NEXT:    add v1.4s, v3.4s, v1.4s
+; CHECK-NEXT:    mls v0.4s, v1.4s, v2.4s
+; CHECK-NEXT:    cmeq v0.4s, v0.4s, #0
+; CHECK-NEXT:    movi v1.4s, #1
+; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    ret
+  %srem = srem <4 x i32> %X, <i32 5, i32 16, i32 1, i32 100>
+  %cmp = icmp eq <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+;------------------------------------------------------------------------------;
+
+define <4 x i32> @test_srem_odd_allones_and_poweroftwo_and_one(<4 x i32> %X) nounwind {
+; CHECK-LABEL: test_srem_odd_allones_and_poweroftwo_and_one:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI25_0
+; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI25_0]
+; CHECK-NEXT:    adrp x8, .LCPI25_1
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI25_1]
+; CHECK-NEXT:    adrp x8, .LCPI25_2
+; CHECK-NEXT:    smull2 v4.2d, v0.4s, v1.4s
+; CHECK-NEXT:    smull v1.2d, v0.2s, v1.2s
+; CHECK-NEXT:    uzp2 v1.4s, v1.4s, v4.4s
+; CHECK-NEXT:    ldr q4, [x8, :lo12:.LCPI25_2]
+; CHECK-NEXT:    adrp x8, .LCPI25_3
+; CHECK-NEXT:    mla v1.4s, v0.4s, v2.4s
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI25_3]
+; CHECK-NEXT:    neg v4.4s, v4.4s
+; CHECK-NEXT:    movi v3.2d, #0x000000ffffffff
+; CHECK-NEXT:    sshl v4.4s, v1.4s, v4.4s
+; CHECK-NEXT:    ushr v1.4s, v1.4s, #31
+; CHECK-NEXT:    and v1.16b, v1.16b, v3.16b
+; CHECK-NEXT:    add v1.4s, v4.4s, v1.4s
+; CHECK-NEXT:    mls v0.4s, v1.4s, v2.4s
+; CHECK-NEXT:    cmeq v0.4s, v0.4s, #0
+; CHECK-NEXT:    movi v1.4s, #1
+; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    ret
+  %srem = srem <4 x i32> %X, <i32 5, i32 4294967295, i32 16, i32 1>
+  %cmp = icmp eq <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+define <4 x i32> @test_srem_even_allones_and_poweroftwo_and_one(<4 x i32> %X) nounwind {
+; CHECK-LABEL: test_srem_even_allones_and_poweroftwo_and_one:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI26_0
+; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI26_0]
+; CHECK-NEXT:    adrp x8, .LCPI26_1
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI26_1]
+; CHECK-NEXT:    adrp x8, .LCPI26_2
+; CHECK-NEXT:    smull2 v4.2d, v0.4s, v1.4s
+; CHECK-NEXT:    smull v1.2d, v0.2s, v1.2s
+; CHECK-NEXT:    uzp2 v1.4s, v1.4s, v4.4s
+; CHECK-NEXT:    ldr q4, [x8, :lo12:.LCPI26_2]
+; CHECK-NEXT:    adrp x8, .LCPI26_3
+; CHECK-NEXT:    mla v1.4s, v0.4s, v2.4s
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI26_3]
+; CHECK-NEXT:    neg v4.4s, v4.4s
+; CHECK-NEXT:    movi v3.2d, #0x000000ffffffff
+; CHECK-NEXT:    sshl v4.4s, v1.4s, v4.4s
+; CHECK-NEXT:    ushr v1.4s, v1.4s, #31
+; CHECK-NEXT:    and v1.16b, v1.16b, v3.16b
+; CHECK-NEXT:    add v1.4s, v4.4s, v1.4s
+; CHECK-NEXT:    mls v0.4s, v1.4s, v2.4s
+; CHECK-NEXT:    cmeq v0.4s, v0.4s, #0
+; CHECK-NEXT:    movi v1.4s, #1
+; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    ret
+  %srem = srem <4 x i32> %X, <i32 14, i32 4294967295, i32 16, i32 1>
+  %cmp = icmp eq <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
diff --git a/llvm/test/CodeGen/AArch64/srem-seteq-vec-splat.ll b/llvm/test/CodeGen/AArch64/srem-seteq-vec-splat.ll
new file mode 100644
index 000000000..328815b60
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/srem-seteq-vec-splat.ll
@@ -0,0 +1,226 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64-unknown-linux-gnu < %s | FileCheck %s
+
+; Odd divisor
+define <4 x i32> @test_srem_odd_25(<4 x i32> %X) nounwind {
+; CHECK-LABEL: test_srem_odd_25:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #23593
+; CHECK-NEXT:    mov w9, #47185
+; CHECK-NEXT:    movk w8, #49807, lsl #16
+; CHECK-NEXT:    movk w9, #1310, lsl #16
+; CHECK-NEXT:    mov w10, #28834
+; CHECK-NEXT:    movk w10, #2621, lsl #16
+; CHECK-NEXT:    dup v1.4s, w8
+; CHECK-NEXT:    dup v2.4s, w9
+; CHECK-NEXT:    dup v3.4s, w10
+; CHECK-NEXT:    mla v2.4s, v0.4s, v1.4s
+; CHECK-NEXT:    cmhs v0.4s, v3.4s, v2.4s
+; CHECK-NEXT:    movi v1.4s, #1
+; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    ret
+  %srem = srem <4 x i32> %X, <i32 25, i32 25, i32 25, i32 25>
+  %cmp = icmp eq <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Even divisors
+define <4 x i32> @test_srem_even_100(<4 x i32> %X) nounwind {
+; CHECK-LABEL: test_srem_even_100:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #34079
+; CHECK-NEXT:    movk w8, #20971, lsl #16
+; CHECK-NEXT:    dup v2.4s, w8
+; CHECK-NEXT:    smull2 v3.2d, v0.4s, v2.4s
+; CHECK-NEXT:    smull v2.2d, v0.2s, v2.2s
+; CHECK-NEXT:    uzp2 v2.4s, v2.4s, v3.4s
+; CHECK-NEXT:    sshr v3.4s, v2.4s, #5
+; CHECK-NEXT:    movi v1.4s, #100
+; CHECK-NEXT:    usra v3.4s, v2.4s, #31
+; CHECK-NEXT:    mls v0.4s, v3.4s, v1.4s
+; CHECK-NEXT:    cmeq v0.4s, v0.4s, #0
+; CHECK-NEXT:    movi v1.4s, #1
+; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    ret
+  %srem = srem <4 x i32> %X, <i32 100, i32 100, i32 100, i32 100>
+  %cmp = icmp eq <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Negative divisors should be negated, and thus this is still splat vectors.
+
+; Odd divisor
+define <4 x i32> @test_srem_odd_neg25(<4 x i32> %X) nounwind {
+; CHECK-LABEL: test_srem_odd_neg25:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #23593
+; CHECK-NEXT:    mov w9, #47185
+; CHECK-NEXT:    movk w8, #49807, lsl #16
+; CHECK-NEXT:    movk w9, #1310, lsl #16
+; CHECK-NEXT:    mov w10, #28834
+; CHECK-NEXT:    movk w10, #2621, lsl #16
+; CHECK-NEXT:    dup v1.4s, w8
+; CHECK-NEXT:    dup v2.4s, w9
+; CHECK-NEXT:    dup v3.4s, w10
+; CHECK-NEXT:    mla v2.4s, v0.4s, v1.4s
+; CHECK-NEXT:    cmhs v0.4s, v3.4s, v2.4s
+; CHECK-NEXT:    movi v1.4s, #1
+; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    ret
+  %srem = srem <4 x i32> %X, <i32 25, i32 -25, i32 -25, i32 25>
+  %cmp = icmp eq <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Even divisors
+define <4 x i32> @test_srem_even_neg100(<4 x i32> %X) nounwind {
+; CHECK-LABEL: test_srem_even_neg100:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI3_0
+; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI3_0]
+; CHECK-NEXT:    adrp x8, .LCPI3_1
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI3_1]
+; CHECK-NEXT:    smull2 v3.2d, v0.4s, v1.4s
+; CHECK-NEXT:    smull v1.2d, v0.2s, v1.2s
+; CHECK-NEXT:    uzp2 v1.4s, v1.4s, v3.4s
+; CHECK-NEXT:    sshr v3.4s, v1.4s, #5
+; CHECK-NEXT:    usra v3.4s, v1.4s, #31
+; CHECK-NEXT:    mls v0.4s, v3.4s, v2.4s
+; CHECK-NEXT:    cmeq v0.4s, v0.4s, #0
+; CHECK-NEXT:    movi v1.4s, #1
+; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    ret
+  %srem = srem <4 x i32> %X, <i32 -100, i32 100, i32 -100, i32 100>
+  %cmp = icmp eq <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+;------------------------------------------------------------------------------;
+; Comparison constant has undef elements.
+;------------------------------------------------------------------------------;
+
+define <4 x i32> @test_srem_odd_undef1(<4 x i32> %X) nounwind {
+; CHECK-LABEL: test_srem_odd_undef1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #34079
+; CHECK-NEXT:    movk w8, #20971, lsl #16
+; CHECK-NEXT:    dup v2.4s, w8
+; CHECK-NEXT:    smull2 v3.2d, v0.4s, v2.4s
+; CHECK-NEXT:    smull v2.2d, v0.2s, v2.2s
+; CHECK-NEXT:    uzp2 v2.4s, v2.4s, v3.4s
+; CHECK-NEXT:    sshr v3.4s, v2.4s, #3
+; CHECK-NEXT:    movi v1.4s, #25
+; CHECK-NEXT:    usra v3.4s, v2.4s, #31
+; CHECK-NEXT:    mls v0.4s, v3.4s, v1.4s
+; CHECK-NEXT:    cmeq v0.4s, v0.4s, #0
+; CHECK-NEXT:    movi v1.4s, #1
+; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    ret
+  %srem = srem <4 x i32> %X, <i32 25, i32 25, i32 25, i32 25>
+  %cmp = icmp eq <4 x i32> %srem, <i32 0, i32 0, i32 undef, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+define <4 x i32> @test_srem_even_undef1(<4 x i32> %X) nounwind {
+; CHECK-LABEL: test_srem_even_undef1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #34079
+; CHECK-NEXT:    movk w8, #20971, lsl #16
+; CHECK-NEXT:    dup v2.4s, w8
+; CHECK-NEXT:    smull2 v3.2d, v0.4s, v2.4s
+; CHECK-NEXT:    smull v2.2d, v0.2s, v2.2s
+; CHECK-NEXT:    uzp2 v2.4s, v2.4s, v3.4s
+; CHECK-NEXT:    sshr v3.4s, v2.4s, #5
+; CHECK-NEXT:    movi v1.4s, #100
+; CHECK-NEXT:    usra v3.4s, v2.4s, #31
+; CHECK-NEXT:    mls v0.4s, v3.4s, v1.4s
+; CHECK-NEXT:    cmeq v0.4s, v0.4s, #0
+; CHECK-NEXT:    movi v1.4s, #1
+; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    ret
+  %srem = srem <4 x i32> %X, <i32 100, i32 100, i32 100, i32 100>
+  %cmp = icmp eq <4 x i32> %srem, <i32 0, i32 0, i32 undef, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+;------------------------------------------------------------------------------;
+; Negative tests
+;------------------------------------------------------------------------------;
+
+define <4 x i32> @test_srem_one_eq(<4 x i32> %X) nounwind {
+; CHECK-LABEL: test_srem_one_eq:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    movi v0.4s, #1
+; CHECK-NEXT:    ret
+  %srem = srem <4 x i32> %X, <i32 1, i32 1, i32 1, i32 1>
+  %cmp = icmp eq <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+define <4 x i32> @test_srem_one_ne(<4 x i32> %X) nounwind {
+; CHECK-LABEL: test_srem_one_ne:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-NEXT:    ret
+  %srem = srem <4 x i32> %X, <i32 1, i32 1, i32 1, i32 1>
+  %cmp = icmp ne <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; We can lower remainder of division by powers of two much better elsewhere.
+define <4 x i32> @test_srem_pow2(<4 x i32> %X) nounwind {
+; CHECK-LABEL: test_srem_pow2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sshr v1.4s, v0.4s, #31
+; CHECK-NEXT:    mov v2.16b, v0.16b
+; CHECK-NEXT:    usra v2.4s, v1.4s, #28
+; CHECK-NEXT:    bic v2.4s, #15
+; CHECK-NEXT:    sub v0.4s, v0.4s, v2.4s
+; CHECK-NEXT:    cmeq v0.4s, v0.4s, #0
+; CHECK-NEXT:    movi v1.4s, #1
+; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    ret
+  %srem = srem <4 x i32> %X, <i32 16, i32 16, i32 16, i32 16>
+  %cmp = icmp eq <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; We could lower remainder of division by INT_MIN much better elsewhere.
+define <4 x i32> @test_srem_int_min(<4 x i32> %X) nounwind {
+; CHECK-LABEL: test_srem_int_min:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sshr v1.4s, v0.4s, #31
+; CHECK-NEXT:    mov v2.16b, v0.16b
+; CHECK-NEXT:    movi v3.4s, #128, lsl #24
+; CHECK-NEXT:    usra v2.4s, v1.4s, #1
+; CHECK-NEXT:    and v1.16b, v2.16b, v3.16b
+; CHECK-NEXT:    sub v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    cmeq v0.4s, v0.4s, #0
+; CHECK-NEXT:    movi v1.4s, #1
+; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    ret
+  %srem = srem <4 x i32> %X, <i32 2147483648, i32 2147483648, i32 2147483648, i32 2147483648>
+  %cmp = icmp eq <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; We could lower remainder of division by all-ones much better elsewhere.
+define <4 x i32> @test_srem_allones(<4 x i32> %X) nounwind {
+; CHECK-LABEL: test_srem_allones:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    movi v0.4s, #1
+; CHECK-NEXT:    ret
+  %srem = srem <4 x i32> %X, <i32 4294967295, i32 4294967295, i32 4294967295, i32 4294967295>
+  %cmp = icmp eq <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
diff --git a/llvm/test/CodeGen/AArch64/srem-seteq.ll b/llvm/test/CodeGen/AArch64/srem-seteq.ll
new file mode 100644
index 000000000..e7557d0ea
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/srem-seteq.ll
@@ -0,0 +1,286 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64-unknown-linux-gnu < %s | FileCheck %s
+
+;------------------------------------------------------------------------------;
+; Odd divisors
+;------------------------------------------------------------------------------;
+
+define i32 @test_srem_odd(i32 %X) nounwind {
+; CHECK-LABEL: test_srem_odd:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #52429
+; CHECK-NEXT:    mov w9, #39321
+; CHECK-NEXT:    movk w8, #52428, lsl #16
+; CHECK-NEXT:    movk w9, #6553, lsl #16
+; CHECK-NEXT:    madd w8, w0, w8, w9
+; CHECK-NEXT:    mov w9, #858993459
+; CHECK-NEXT:    cmp w8, w9
+; CHECK-NEXT:    cset w0, lo
+; CHECK-NEXT:    ret
+  %srem = srem i32 %X, 5
+  %cmp = icmp eq i32 %srem, 0
+  %ret = zext i1 %cmp to i32
+  ret i32 %ret
+}
+
+define i32 @test_srem_odd_25(i32 %X) nounwind {
+; CHECK-LABEL: test_srem_odd_25:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #23593
+; CHECK-NEXT:    mov w9, #47185
+; CHECK-NEXT:    movk w8, #49807, lsl #16
+; CHECK-NEXT:    movk w9, #1310, lsl #16
+; CHECK-NEXT:    madd w8, w0, w8, w9
+; CHECK-NEXT:    mov w9, #28835
+; CHECK-NEXT:    movk w9, #2621, lsl #16
+; CHECK-NEXT:    cmp w8, w9
+; CHECK-NEXT:    cset w0, lo
+; CHECK-NEXT:    ret
+  %srem = srem i32 %X, 25
+  %cmp = icmp eq i32 %srem, 0
+  %ret = zext i1 %cmp to i32
+  ret i32 %ret
+}
+
+; This is like test_srem_odd, except the divisor has bit 30 set.
+define i32 @test_srem_odd_bit30(i32 %X) nounwind {
+; CHECK-LABEL: test_srem_odd_bit30:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #43691
+; CHECK-NEXT:    movk w8, #27306, lsl #16
+; CHECK-NEXT:    orr w9, wzr, #0x1
+; CHECK-NEXT:    madd w8, w0, w8, w9
+; CHECK-NEXT:    cmp w8, #3 // =3
+; CHECK-NEXT:    cset w0, lo
+; CHECK-NEXT:    ret
+  %srem = srem i32 %X, 1073741827
+  %cmp = icmp eq i32 %srem, 0
+  %ret = zext i1 %cmp to i32
+  ret i32 %ret
+}
+
+; This is like test_srem_odd, except the divisor has bit 31 set.
+define i32 @test_srem_odd_bit31(i32 %X) nounwind {
+; CHECK-LABEL: test_srem_odd_bit31:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #21845
+; CHECK-NEXT:    movk w8, #54613, lsl #16
+; CHECK-NEXT:    orr w9, wzr, #0x1
+; CHECK-NEXT:    madd w8, w0, w8, w9
+; CHECK-NEXT:    cmp w8, #3 // =3
+; CHECK-NEXT:    cset w0, lo
+; CHECK-NEXT:    ret
+  %srem = srem i32 %X, 2147483651
+  %cmp = icmp eq i32 %srem, 0
+  %ret = zext i1 %cmp to i32
+  ret i32 %ret
+}
+
+;------------------------------------------------------------------------------;
+; Even divisors
+;------------------------------------------------------------------------------;
+
+define i16 @test_srem_even(i16 %X) nounwind {
+; CHECK-LABEL: test_srem_even:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w9, #9363
+; CHECK-NEXT:    sxth w8, w0
+; CHECK-NEXT:    movk w9, #37449, lsl #16
+; CHECK-NEXT:    smull x9, w8, w9
+; CHECK-NEXT:    lsr x9, x9, #32
+; CHECK-NEXT:    add w8, w9, w8
+; CHECK-NEXT:    asr w9, w8, #3
+; CHECK-NEXT:    add w8, w9, w8, lsr #31
+; CHECK-NEXT:    mov w9, #14
+; CHECK-NEXT:    msub w8, w8, w9, w0
+; CHECK-NEXT:    tst w8, #0xffff
+; CHECK-NEXT:    cset w0, ne
+; CHECK-NEXT:    ret
+  %srem = srem i16 %X, 14
+  %cmp = icmp ne i16 %srem, 0
+  %ret = zext i1 %cmp to i16
+  ret i16 %ret
+}
+
+define i32 @test_srem_even_100(i32 %X) nounwind {
+; CHECK-LABEL: test_srem_even_100:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #23593
+; CHECK-NEXT:    mov w9, #47184
+; CHECK-NEXT:    movk w8, #49807, lsl #16
+; CHECK-NEXT:    movk w9, #1310, lsl #16
+; CHECK-NEXT:    madd w8, w0, w8, w9
+; CHECK-NEXT:    mov w9, #23593
+; CHECK-NEXT:    ror w8, w8, #2
+; CHECK-NEXT:    movk w9, #655, lsl #16
+; CHECK-NEXT:    cmp w8, w9
+; CHECK-NEXT:    cset w0, lo
+; CHECK-NEXT:    ret
+  %srem = srem i32 %X, 100
+  %cmp = icmp eq i32 %srem, 0
+  %ret = zext i1 %cmp to i32
+  ret i32 %ret
+}
+
+; This is like test_srem_even, except the divisor has bit 30 set.
+define i32 @test_srem_even_bit30(i32 %X) nounwind {
+; CHECK-LABEL: test_srem_even_bit30:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #20165
+; CHECK-NEXT:    movk w8, #64748, lsl #16
+; CHECK-NEXT:    orr w9, wzr, #0x8
+; CHECK-NEXT:    madd w8, w0, w8, w9
+; CHECK-NEXT:    ror w8, w8, #3
+; CHECK-NEXT:    cmp w8, #3 // =3
+; CHECK-NEXT:    cset w0, lo
+; CHECK-NEXT:    ret
+  %srem = srem i32 %X, 1073741928
+  %cmp = icmp eq i32 %srem, 0
+  %ret = zext i1 %cmp to i32
+  ret i32 %ret
+}
+
+; This is like test_srem_odd, except the divisor has bit 31 set.
+define i32 @test_srem_even_bit31(i32 %X) nounwind {
+; CHECK-LABEL: test_srem_even_bit31:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #1285
+; CHECK-NEXT:    movk w8, #50437, lsl #16
+; CHECK-NEXT:    orr w9, wzr, #0x2
+; CHECK-NEXT:    madd w8, w0, w8, w9
+; CHECK-NEXT:    ror w8, w8, #1
+; CHECK-NEXT:    cmp w8, #3 // =3
+; CHECK-NEXT:    cset w0, lo
+; CHECK-NEXT:    ret
+  %srem = srem i32 %X, 2147483750
+  %cmp = icmp eq i32 %srem, 0
+  %ret = zext i1 %cmp to i32
+  ret i32 %ret
+}
+
+;------------------------------------------------------------------------------;
+; Special case
+;------------------------------------------------------------------------------;
+
+; 'NE' predicate is fine too.
+define i32 @test_srem_odd_setne(i32 %X) nounwind {
+; CHECK-LABEL: test_srem_odd_setne:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #52429
+; CHECK-NEXT:    mov w9, #39321
+; CHECK-NEXT:    movk w8, #52428, lsl #16
+; CHECK-NEXT:    movk w9, #6553, lsl #16
+; CHECK-NEXT:    madd w8, w0, w8, w9
+; CHECK-NEXT:    mov w9, #13106
+; CHECK-NEXT:    movk w9, #13107, lsl #16
+; CHECK-NEXT:    cmp w8, w9
+; CHECK-NEXT:    cset w0, hi
+; CHECK-NEXT:    ret
+  %srem = srem i32 %X, 5
+  %cmp = icmp ne i32 %srem, 0
+  %ret = zext i1 %cmp to i32
+  ret i32 %ret
+}
+
+; The fold is only valid for positive divisors, negative-ones should be negated.
+define i32 @test_srem_negative_odd(i32 %X) nounwind {
+; CHECK-LABEL: test_srem_negative_odd:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #52429
+; CHECK-NEXT:    mov w9, #39321
+; CHECK-NEXT:    movk w8, #52428, lsl #16
+; CHECK-NEXT:    movk w9, #6553, lsl #16
+; CHECK-NEXT:    madd w8, w0, w8, w9
+; CHECK-NEXT:    mov w9, #13106
+; CHECK-NEXT:    movk w9, #13107, lsl #16
+; CHECK-NEXT:    cmp w8, w9
+; CHECK-NEXT:    cset w0, hi
+; CHECK-NEXT:    ret
+  %srem = srem i32 %X, -5
+  %cmp = icmp ne i32 %srem, 0
+  %ret = zext i1 %cmp to i32
+  ret i32 %ret
+}
+define i32 @test_srem_negative_even(i32 %X) nounwind {
+; CHECK-LABEL: test_srem_negative_even:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #28087
+; CHECK-NEXT:    mov w9, #9362
+; CHECK-NEXT:    movk w8, #46811, lsl #16
+; CHECK-NEXT:    movk w9, #4681, lsl #16
+; CHECK-NEXT:    madd w8, w0, w8, w9
+; CHECK-NEXT:    ror w8, w8, #1
+; CHECK-NEXT:    cmp w8, w9
+; CHECK-NEXT:    cset w0, hi
+; CHECK-NEXT:    ret
+  %srem = srem i32 %X, -14
+  %cmp = icmp ne i32 %srem, 0
+  %ret = zext i1 %cmp to i32
+  ret i32 %ret
+}
+
+;------------------------------------------------------------------------------;
+; Negative tests
+;------------------------------------------------------------------------------;
+
+; We can lower remainder of division by one much better elsewhere.
+define i32 @test_srem_one(i32 %X) nounwind {
+; CHECK-LABEL: test_srem_one:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w0, #1
+; CHECK-NEXT:    ret
+  %srem = srem i32 %X, 1
+  %cmp = icmp eq i32 %srem, 0
+  %ret = zext i1 %cmp to i32
+  ret i32 %ret
+}
+
+; We can lower remainder of division by powers of two much better elsewhere.
+define i32 @test_srem_pow2(i32 %X) nounwind {
+; CHECK-LABEL: test_srem_pow2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    add w8, w0, #15 // =15
+; CHECK-NEXT:    cmp w0, #0 // =0
+; CHECK-NEXT:    csel w8, w8, w0, lt
+; CHECK-NEXT:    and w8, w8, #0xfffffff0
+; CHECK-NEXT:    cmp w0, w8
+; CHECK-NEXT:    cset w0, eq
+; CHECK-NEXT:    ret
+  %srem = srem i32 %X, 16
+  %cmp = icmp eq i32 %srem, 0
+  %ret = zext i1 %cmp to i32
+  ret i32 %ret
+}
+
+; The fold is only valid for positive divisors, and we can't negate INT_MIN.
+define i32 @test_srem_int_min(i32 %X) nounwind {
+; CHECK-LABEL: test_srem_int_min:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #2147483647
+; CHECK-NEXT:    add w8, w0, w8
+; CHECK-NEXT:    cmp w0, #0 // =0
+; CHECK-NEXT:    csel w8, w8, w0, lt
+; CHECK-NEXT:    and w8, w8, #0x80000000
+; CHECK-NEXT:    cmn w0, w8
+; CHECK-NEXT:    cset w0, eq
+; CHECK-NEXT:    ret
+  %srem = srem i32 %X, 2147483648
+  %cmp = icmp eq i32 %srem, 0
+  %ret = zext i1 %cmp to i32
+  ret i32 %ret
+}
+
+; We can lower remainder of division by all-ones much better elsewhere.
+define i32 @test_srem_allones(i32 %X) nounwind {
+; CHECK-LABEL: test_srem_allones:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmp w0, #0 // =0
+; CHECK-NEXT:    csel w8, w0, w0, lt
+; CHECK-NEXT:    cmp w0, w8
+; CHECK-NEXT:    cset w0, eq
+; CHECK-NEXT:    ret
+  %srem = srem i32 %X, 4294967295
+  %cmp = icmp eq i32 %srem, 0
+  %ret = zext i1 %cmp to i32
+  ret i32 %ret
+}
diff --git a/llvm/test/CodeGen/AArch64/srem-vector-lkk.ll b/llvm/test/CodeGen/AArch64/srem-vector-lkk.ll
new file mode 100644
index 000000000..5597e1657
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/srem-vector-lkk.ll
@@ -0,0 +1,324 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64-unknown-linux-gnu < %s | FileCheck %s
+
+define <4 x i16> @fold_srem_vec_1(<4 x i16> %x) {
+; CHECK-LABEL: fold_srem_vec_1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w9, #63421
+; CHECK-NEXT:    mov w12, #33437
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    smov w8, v0.h[1]
+; CHECK-NEXT:    movk w9, #31710, lsl #16
+; CHECK-NEXT:    smov w11, v0.h[2]
+; CHECK-NEXT:    movk w12, #21399, lsl #16
+; CHECK-NEXT:    smull x12, w11, w12
+; CHECK-NEXT:    smull x9, w8, w9
+; CHECK-NEXT:    lsr x13, x12, #63
+; CHECK-NEXT:    asr x12, x12, #37
+; CHECK-NEXT:    lsr x9, x9, #32
+; CHECK-NEXT:    add w12, w12, w13
+; CHECK-NEXT:    mov w13, #98
+; CHECK-NEXT:    sub w9, w9, w8
+; CHECK-NEXT:    msub w11, w12, w13, w11
+; CHECK-NEXT:    asr w13, w9, #6
+; CHECK-NEXT:    add w9, w13, w9, lsr #31
+; CHECK-NEXT:    mov w13, #37253
+; CHECK-NEXT:    mov w10, #-124
+; CHECK-NEXT:    smov w12, v0.h[0]
+; CHECK-NEXT:    movk w13, #44150, lsl #16
+; CHECK-NEXT:    msub w8, w9, w10, w8
+; CHECK-NEXT:    smull x10, w12, w13
+; CHECK-NEXT:    lsr x10, x10, #32
+; CHECK-NEXT:    add w10, w10, w12
+; CHECK-NEXT:    asr w13, w10, #6
+; CHECK-NEXT:    mov w9, #95
+; CHECK-NEXT:    add w10, w13, w10, lsr #31
+; CHECK-NEXT:    msub w9, w10, w9, w12
+; CHECK-NEXT:    mov w10, #63249
+; CHECK-NEXT:    smov w13, v0.h[3]
+; CHECK-NEXT:    movk w10, #48808, lsl #16
+; CHECK-NEXT:    smull x10, w13, w10
+; CHECK-NEXT:    lsr x12, x10, #63
+; CHECK-NEXT:    asr x10, x10, #40
+; CHECK-NEXT:    fmov s0, w9
+; CHECK-NEXT:    add w10, w10, w12
+; CHECK-NEXT:    mov v0.h[1], w8
+; CHECK-NEXT:    mov w8, #-1003
+; CHECK-NEXT:    mov v0.h[2], w11
+; CHECK-NEXT:    msub w8, w10, w8, w13
+; CHECK-NEXT:    mov v0.h[3], w8
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    ret
+  %1 = srem <4 x i16> %x, <i16 95, i16 -124, i16 98, i16 -1003>
+  ret <4 x i16> %1
+}
+
+define <4 x i16> @fold_srem_vec_2(<4 x i16> %x) {
+; CHECK-LABEL: fold_srem_vec_2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w9, #37253
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    smov w8, v0.h[1]
+; CHECK-NEXT:    movk w9, #44150, lsl #16
+; CHECK-NEXT:    smov w10, v0.h[0]
+; CHECK-NEXT:    smull x13, w8, w9
+; CHECK-NEXT:    smov w11, v0.h[2]
+; CHECK-NEXT:    smull x14, w10, w9
+; CHECK-NEXT:    lsr x13, x13, #32
+; CHECK-NEXT:    smov w12, v0.h[3]
+; CHECK-NEXT:    smull x15, w11, w9
+; CHECK-NEXT:    lsr x14, x14, #32
+; CHECK-NEXT:    add w13, w13, w8
+; CHECK-NEXT:    smull x9, w12, w9
+; CHECK-NEXT:    lsr x15, x15, #32
+; CHECK-NEXT:    add w14, w14, w10
+; CHECK-NEXT:    asr w16, w13, #6
+; CHECK-NEXT:    lsr x9, x9, #32
+; CHECK-NEXT:    add w15, w15, w11
+; CHECK-NEXT:    add w13, w16, w13, lsr #31
+; CHECK-NEXT:    asr w16, w14, #6
+; CHECK-NEXT:    add w9, w9, w12
+; CHECK-NEXT:    add w14, w16, w14, lsr #31
+; CHECK-NEXT:    asr w16, w15, #6
+; CHECK-NEXT:    add w15, w16, w15, lsr #31
+; CHECK-NEXT:    asr w16, w9, #6
+; CHECK-NEXT:    add w9, w16, w9, lsr #31
+; CHECK-NEXT:    mov w16, #95
+; CHECK-NEXT:    msub w10, w14, w16, w10
+; CHECK-NEXT:    msub w8, w13, w16, w8
+; CHECK-NEXT:    fmov s0, w10
+; CHECK-NEXT:    msub w11, w15, w16, w11
+; CHECK-NEXT:    mov v0.h[1], w8
+; CHECK-NEXT:    mov v0.h[2], w11
+; CHECK-NEXT:    msub w8, w9, w16, w12
+; CHECK-NEXT:    mov v0.h[3], w8
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    ret
+  %1 = srem <4 x i16> %x, <i16 95, i16 95, i16 95, i16 95>
+  ret <4 x i16> %1
+}
+
+
+; Don't fold if we can combine srem with sdiv.
+define <4 x i16> @combine_srem_sdiv(<4 x i16> %x) {
+; CHECK-LABEL: combine_srem_sdiv:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #37253
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    movk w8, #44150, lsl #16
+; CHECK-NEXT:    smov w9, v0.h[1]
+; CHECK-NEXT:    smov w10, v0.h[0]
+; CHECK-NEXT:    smull x13, w9, w8
+; CHECK-NEXT:    smov w11, v0.h[2]
+; CHECK-NEXT:    smull x14, w10, w8
+; CHECK-NEXT:    lsr x13, x13, #32
+; CHECK-NEXT:    smov w12, v0.h[3]
+; CHECK-NEXT:    smull x15, w11, w8
+; CHECK-NEXT:    lsr x14, x14, #32
+; CHECK-NEXT:    add w13, w13, w9
+; CHECK-NEXT:    smull x8, w12, w8
+; CHECK-NEXT:    lsr x15, x15, #32
+; CHECK-NEXT:    add w14, w14, w10
+; CHECK-NEXT:    asr w16, w13, #6
+; CHECK-NEXT:    lsr x8, x8, #32
+; CHECK-NEXT:    add w15, w15, w11
+; CHECK-NEXT:    add w13, w16, w13, lsr #31
+; CHECK-NEXT:    asr w16, w14, #6
+; CHECK-NEXT:    add w8, w8, w12
+; CHECK-NEXT:    add w14, w16, w14, lsr #31
+; CHECK-NEXT:    asr w16, w15, #6
+; CHECK-NEXT:    add w15, w16, w15, lsr #31
+; CHECK-NEXT:    asr w16, w8, #6
+; CHECK-NEXT:    add w8, w16, w8, lsr #31
+; CHECK-NEXT:    mov w16, #95
+; CHECK-NEXT:    msub w10, w14, w16, w10
+; CHECK-NEXT:    msub w9, w13, w16, w9
+; CHECK-NEXT:    fmov s0, w14
+; CHECK-NEXT:    fmov s1, w10
+; CHECK-NEXT:    msub w11, w15, w16, w11
+; CHECK-NEXT:    mov v0.h[1], w13
+; CHECK-NEXT:    mov v1.h[1], w9
+; CHECK-NEXT:    msub w12, w8, w16, w12
+; CHECK-NEXT:    mov v0.h[2], w15
+; CHECK-NEXT:    mov v1.h[2], w11
+; CHECK-NEXT:    mov v1.h[3], w12
+; CHECK-NEXT:    mov v0.h[3], w8
+; CHECK-NEXT:    add v0.4h, v1.4h, v0.4h
+; CHECK-NEXT:    ret
+  %1 = srem <4 x i16> %x, <i16 95, i16 95, i16 95, i16 95>
+  %2 = sdiv <4 x i16> %x, <i16 95, i16 95, i16 95, i16 95>
+  %3 = add <4 x i16> %1, %2
+  ret <4 x i16> %3
+}
+
+; Don't fold for divisors that are a power of two.
+define <4 x i16> @dont_fold_srem_power_of_two(<4 x i16> %x) {
+; CHECK-LABEL: dont_fold_srem_power_of_two:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    smov w8, v0.h[1]
+; CHECK-NEXT:    add w12, w8, #31 // =31
+; CHECK-NEXT:    cmp w8, #0 // =0
+; CHECK-NEXT:    mov w11, #37253
+; CHECK-NEXT:    csel w12, w12, w8, lt
+; CHECK-NEXT:    smov w9, v0.h[0]
+; CHECK-NEXT:    smov w10, v0.h[3]
+; CHECK-NEXT:    movk w11, #44150, lsl #16
+; CHECK-NEXT:    and w12, w12, #0xffffffe0
+; CHECK-NEXT:    sub w8, w8, w12
+; CHECK-NEXT:    add w12, w9, #63 // =63
+; CHECK-NEXT:    smull x11, w10, w11
+; CHECK-NEXT:    cmp w9, #0 // =0
+; CHECK-NEXT:    lsr x11, x11, #32
+; CHECK-NEXT:    csel w12, w12, w9, lt
+; CHECK-NEXT:    add w11, w11, w10
+; CHECK-NEXT:    and w12, w12, #0xffffffc0
+; CHECK-NEXT:    sub w9, w9, w12
+; CHECK-NEXT:    asr w12, w11, #6
+; CHECK-NEXT:    add w11, w12, w11, lsr #31
+; CHECK-NEXT:    smov w12, v0.h[2]
+; CHECK-NEXT:    fmov s0, w9
+; CHECK-NEXT:    add w9, w12, #7 // =7
+; CHECK-NEXT:    cmp w12, #0 // =0
+; CHECK-NEXT:    csel w9, w9, w12, lt
+; CHECK-NEXT:    and w9, w9, #0xfffffff8
+; CHECK-NEXT:    sub w9, w12, w9
+; CHECK-NEXT:    mov v0.h[1], w8
+; CHECK-NEXT:    mov w8, #95
+; CHECK-NEXT:    mov v0.h[2], w9
+; CHECK-NEXT:    msub w8, w11, w8, w10
+; CHECK-NEXT:    mov v0.h[3], w8
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    ret
+  %1 = srem <4 x i16> %x, <i16 64, i16 32, i16 8, i16 95>
+  ret <4 x i16> %1
+}
+
+; Don't fold if the divisor is one.
+define <4 x i16> @dont_fold_srem_one(<4 x i16> %x) {
+; CHECK-LABEL: dont_fold_srem_one:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w9, #17097
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    smov w8, v0.h[2]
+; CHECK-NEXT:    movk w9, #45590, lsl #16
+; CHECK-NEXT:    smull x9, w8, w9
+; CHECK-NEXT:    lsr x9, x9, #32
+; CHECK-NEXT:    add w9, w9, w8
+; CHECK-NEXT:    asr w12, w9, #4
+; CHECK-NEXT:    add w9, w12, w9, lsr #31
+; CHECK-NEXT:    mov w12, #30865
+; CHECK-NEXT:    mov w10, #23
+; CHECK-NEXT:    smov w11, v0.h[1]
+; CHECK-NEXT:    movk w12, #51306, lsl #16
+; CHECK-NEXT:    msub w8, w9, w10, w8
+; CHECK-NEXT:    smull x10, w11, w12
+; CHECK-NEXT:    lsr x10, x10, #32
+; CHECK-NEXT:    add w10, w10, w11
+; CHECK-NEXT:    asr w12, w10, #9
+; CHECK-NEXT:    mov w9, #654
+; CHECK-NEXT:    add w10, w12, w10, lsr #31
+; CHECK-NEXT:    msub w9, w10, w9, w11
+; CHECK-NEXT:    mov w10, #47143
+; CHECK-NEXT:    smov w12, v0.h[3]
+; CHECK-NEXT:    movk w10, #24749, lsl #16
+; CHECK-NEXT:    smull x10, w12, w10
+; CHECK-NEXT:    lsr x11, x10, #63
+; CHECK-NEXT:    asr x10, x10, #43
+; CHECK-NEXT:    movi d0, #0000000000000000
+; CHECK-NEXT:    add w10, w10, w11
+; CHECK-NEXT:    mov v0.h[1], w9
+; CHECK-NEXT:    mov w9, #5423
+; CHECK-NEXT:    mov v0.h[2], w8
+; CHECK-NEXT:    msub w8, w10, w9, w12
+; CHECK-NEXT:    mov v0.h[3], w8
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    ret
+  %1 = srem <4 x i16> %x, <i16 1, i16 654, i16 23, i16 5423>
+  ret <4 x i16> %1
+}
+
+; Don't fold if the divisor is 2^15.
+define <4 x i16> @dont_fold_srem_i16_smax(<4 x i16> %x) {
+; CHECK-LABEL: dont_fold_srem_i16_smax:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w10, #17097
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    smov w9, v0.h[2]
+; CHECK-NEXT:    movk w10, #45590, lsl #16
+; CHECK-NEXT:    smull x10, w9, w10
+; CHECK-NEXT:    lsr x10, x10, #32
+; CHECK-NEXT:    add w10, w10, w9
+; CHECK-NEXT:    asr w12, w10, #4
+; CHECK-NEXT:    mov w11, #23
+; CHECK-NEXT:    add w10, w12, w10, lsr #31
+; CHECK-NEXT:    msub w9, w10, w11, w9
+; CHECK-NEXT:    mov w10, #47143
+; CHECK-NEXT:    smov w12, v0.h[3]
+; CHECK-NEXT:    movk w10, #24749, lsl #16
+; CHECK-NEXT:    smull x10, w12, w10
+; CHECK-NEXT:    lsr x11, x10, #63
+; CHECK-NEXT:    asr x10, x10, #43
+; CHECK-NEXT:    smov w8, v0.h[1]
+; CHECK-NEXT:    add w10, w10, w11
+; CHECK-NEXT:    mov w11, #32767
+; CHECK-NEXT:    add w11, w8, w11
+; CHECK-NEXT:    cmp w8, #0 // =0
+; CHECK-NEXT:    csel w11, w11, w8, lt
+; CHECK-NEXT:    and w11, w11, #0xffff8000
+; CHECK-NEXT:    sub w8, w8, w11
+; CHECK-NEXT:    movi d0, #0000000000000000
+; CHECK-NEXT:    mov v0.h[1], w8
+; CHECK-NEXT:    mov w8, #5423
+; CHECK-NEXT:    mov v0.h[2], w9
+; CHECK-NEXT:    msub w8, w10, w8, w12
+; CHECK-NEXT:    mov v0.h[3], w8
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    ret
+  %1 = srem <4 x i16> %x, <i16 1, i16 32768, i16 23, i16 5423>
+  ret <4 x i16> %1
+}
+
+; Don't fold i64 srem.
+define <4 x i64> @dont_fold_srem_i64(<4 x i64> %x) {
+; CHECK-LABEL: dont_fold_srem_i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov x9, #6055
+; CHECK-NEXT:    movk x9, #58853, lsl #16
+; CHECK-NEXT:    movk x9, #47142, lsl #32
+; CHECK-NEXT:    mov x8, v1.d[1]
+; CHECK-NEXT:    movk x9, #24749, lsl #48
+; CHECK-NEXT:    smulh x9, x8, x9
+; CHECK-NEXT:    asr x12, x9, #11
+; CHECK-NEXT:    mov w10, #5423
+; CHECK-NEXT:    add x9, x12, x9, lsr #63
+; CHECK-NEXT:    msub x8, x9, x10, x8
+; CHECK-NEXT:    mov x9, #21445
+; CHECK-NEXT:    movk x9, #1603, lsl #16
+; CHECK-NEXT:    movk x9, #15432, lsl #32
+; CHECK-NEXT:    mov x12, v0.d[1]
+; CHECK-NEXT:    movk x9, #25653, lsl #48
+; CHECK-NEXT:    smulh x9, x12, x9
+; CHECK-NEXT:    asr x10, x9, #8
+; CHECK-NEXT:    add x9, x10, x9, lsr #63
+; CHECK-NEXT:    mov w10, #654
+; CHECK-NEXT:    msub x9, x9, x10, x12
+; CHECK-NEXT:    mov x10, #8549
+; CHECK-NEXT:    movk x10, #22795, lsl #16
+; CHECK-NEXT:    movk x10, #17096, lsl #32
+; CHECK-NEXT:    fmov x11, d1
+; CHECK-NEXT:    movk x10, #45590, lsl #48
+; CHECK-NEXT:    smulh x10, x11, x10
+; CHECK-NEXT:    add x10, x10, x11
+; CHECK-NEXT:    asr x12, x10, #4
+; CHECK-NEXT:    add x10, x12, x10, lsr #63
+; CHECK-NEXT:    mov w12, #23
+; CHECK-NEXT:    msub x10, x10, x12, x11
+; CHECK-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-NEXT:    fmov d1, x10
+; CHECK-NEXT:    mov v1.d[1], x8
+; CHECK-NEXT:    mov v0.d[1], x9
+; CHECK-NEXT:    ret
+  %1 = srem <4 x i64> %x, <i64 1, i64 654, i64 23, i64 5423>
+  ret <4 x i64> %1
+}
diff --git a/llvm/test/CodeGen/AArch64/ssub_sat.ll b/llvm/test/CodeGen/AArch64/ssub_sat.ll
index 9dc200523..4fab86346 100644
--- a/llvm/test/CodeGen/AArch64/ssub_sat.ll
+++ b/llvm/test/CodeGen/AArch64/ssub_sat.ll
@@ -1,10 +1,12 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=aarch64-- | FileCheck %s
 
-declare  i4  @llvm.ssub.sat.i4   (i4,  i4)
-declare  i32 @llvm.ssub.sat.i32  (i32, i32)
-declare  i64 @llvm.ssub.sat.i64  (i64, i64)
-declare  <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32>, <4 x i32>)
+declare i4 @llvm.ssub.sat.i4(i4, i4)
+declare i8 @llvm.ssub.sat.i8(i8, i8)
+declare i16 @llvm.ssub.sat.i16(i16, i16)
+declare i32 @llvm.ssub.sat.i32(i32, i32)
+declare i64 @llvm.ssub.sat.i64(i64, i64)
+declare <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32>, <4 x i32>)
 
 define i32 @func(i32 %x, i32 %y) nounwind {
 ; CHECK-LABEL: func:
@@ -34,17 +36,50 @@ define i64 @func2(i64 %x, i64 %y) nounwind {
   ret i64 %tmp;
 }
 
+define i16 @func16(i16 %x, i16 %y) nounwind {
+; CHECK-LABEL: func16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sxth w8, w0
+; CHECK-NEXT:    mov w9, #32767
+; CHECK-NEXT:    sub w8, w8, w1, sxth
+; CHECK-NEXT:    cmp w8, w9
+; CHECK-NEXT:    csel w8, w8, w9, lt
+; CHECK-NEXT:    cmn w8, #8, lsl #12 // =32768
+; CHECK-NEXT:    mov w9, #-32768
+; CHECK-NEXT:    csel w0, w8, w9, gt
+; CHECK-NEXT:    ret
+  %tmp = call i16 @llvm.ssub.sat.i16(i16 %x, i16 %y);
+  ret i16 %tmp;
+}
+
+define i8 @func8(i8 %x, i8 %y) nounwind {
+; CHECK-LABEL: func8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sxtb w8, w0
+; CHECK-NEXT:    sub w8, w8, w1, sxtb
+; CHECK-NEXT:    mov w9, #127
+; CHECK-NEXT:    cmp w8, #127 // =127
+; CHECK-NEXT:    csel w8, w8, w9, lt
+; CHECK-NEXT:    cmn w8, #128 // =128
+; CHECK-NEXT:    mov w9, #-128
+; CHECK-NEXT:    csel w0, w8, w9, gt
+; CHECK-NEXT:    ret
+  %tmp = call i8 @llvm.ssub.sat.i8(i8 %x, i8 %y);
+  ret i8 %tmp;
+}
+
 define i4 @func3(i4 %x, i4 %y) nounwind {
 ; CHECK-LABEL: func3:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    lsl w8, w0, #28
-; CHECK-NEXT:    subs w10, w8, w1, lsl #28
-; CHECK-NEXT:    mov w9, #2147483647
-; CHECK-NEXT:    cmp w10, #0 // =0
-; CHECK-NEXT:    cinv w9, w9, ge
-; CHECK-NEXT:    subs w8, w8, w1, lsl #28
-; CHECK-NEXT:    csel w8, w9, w8, vs
-; CHECK-NEXT:    asr w0, w8, #28
+; CHECK-NEXT:    lsl w8, w1, #28
+; CHECK-NEXT:    sbfx w9, w0, #0, #4
+; CHECK-NEXT:    sub w8, w9, w8, asr #28
+; CHECK-NEXT:    mov w10, #7
+; CHECK-NEXT:    cmp w8, #7 // =7
+; CHECK-NEXT:    csel w8, w8, w10, lt
+; CHECK-NEXT:    cmn w8, #8 // =8
+; CHECK-NEXT:    mov w9, #-8
+; CHECK-NEXT:    csel w0, w8, w9, gt
 ; CHECK-NEXT:    ret
   %tmp = call i4 @llvm.ssub.sat.i4(i4 %x, i4 %y);
   ret i4 %tmp;
@@ -53,20 +88,7 @@ define i4 @func3(i4 %x, i4 %y) nounwind {
 define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; CHECK-LABEL: vec:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    sub v2.4s, v0.4s, v1.4s
-; CHECK-NEXT:    cmge v1.4s, v1.4s, #0
-; CHECK-NEXT:    cmge v0.4s, v0.4s, #0
-; CHECK-NEXT:    cmge v5.4s, v2.4s, #0
-; CHECK-NEXT:    cmlt v4.4s, v2.4s, #0
-; CHECK-NEXT:    cmeq v1.4s, v0.4s, v1.4s
-; CHECK-NEXT:    cmeq v0.4s, v0.4s, v5.4s
-; CHECK-NEXT:    mvni v3.4s, #128, lsl #24
-; CHECK-NEXT:    mvn v5.16b, v4.16b
-; CHECK-NEXT:    mvn v1.16b, v1.16b
-; CHECK-NEXT:    mvn v0.16b, v0.16b
-; CHECK-NEXT:    bsl v3.16b, v4.16b, v5.16b
-; CHECK-NEXT:    and v0.16b, v1.16b, v0.16b
-; CHECK-NEXT:    bsl v0.16b, v3.16b, v2.16b
+; CHECK-NEXT:    sqsub v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    ret
   %tmp = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> %x, <4 x i32> %y);
   ret <4 x i32> %tmp;
diff --git a/llvm/test/CodeGen/AArch64/ssub_sat_plus.ll b/llvm/test/CodeGen/AArch64/ssub_sat_plus.ll
new file mode 100644
index 000000000..1528b752c
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/ssub_sat_plus.ll
@@ -0,0 +1,94 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=aarch64-- | FileCheck %s
+
+declare i4 @llvm.ssub.sat.i4(i4, i4)
+declare i8 @llvm.ssub.sat.i8(i8, i8)
+declare i16 @llvm.ssub.sat.i16(i16, i16)
+declare i32 @llvm.ssub.sat.i32(i32, i32)
+declare i64 @llvm.ssub.sat.i64(i64, i64)
+
+define i32 @func32(i32 %x, i32 %y, i32 %z) nounwind {
+; CHECK-LABEL: func32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mul w8, w1, w2
+; CHECK-NEXT:    subs w10, w0, w8
+; CHECK-NEXT:    mov w9, #2147483647
+; CHECK-NEXT:    cmp w10, #0 // =0
+; CHECK-NEXT:    cinv w9, w9, ge
+; CHECK-NEXT:    subs w8, w0, w8
+; CHECK-NEXT:    csel w0, w9, w8, vs
+; CHECK-NEXT:    ret
+  %a = mul i32 %y, %z
+  %tmp = call i32 @llvm.ssub.sat.i32(i32 %x, i32 %a)
+  ret i32 %tmp
+}
+
+define i64 @func64(i64 %x, i64 %y, i64 %z) nounwind {
+; CHECK-LABEL: func64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    subs x8, x0, x2
+; CHECK-NEXT:    mov x9, #9223372036854775807
+; CHECK-NEXT:    cmp x8, #0 // =0
+; CHECK-NEXT:    cinv x8, x9, ge
+; CHECK-NEXT:    subs x9, x0, x2
+; CHECK-NEXT:    csel x0, x8, x9, vs
+; CHECK-NEXT:    ret
+  %a = mul i64 %y, %z
+  %tmp = call i64 @llvm.ssub.sat.i64(i64 %x, i64 %z)
+  ret i64 %tmp
+}
+
+define i16 @func16(i16 %x, i16 %y, i16 %z) nounwind {
+; CHECK-LABEL: func16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sxth w8, w0
+; CHECK-NEXT:    mul w9, w1, w2
+; CHECK-NEXT:    mov w10, #32767
+; CHECK-NEXT:    sub w8, w8, w9, sxth
+; CHECK-NEXT:    cmp w8, w10
+; CHECK-NEXT:    csel w8, w8, w10, lt
+; CHECK-NEXT:    cmn w8, #8, lsl #12 // =32768
+; CHECK-NEXT:    mov w9, #-32768
+; CHECK-NEXT:    csel w0, w8, w9, gt
+; CHECK-NEXT:    ret
+  %a = mul i16 %y, %z
+  %tmp = call i16 @llvm.ssub.sat.i16(i16 %x, i16 %a)
+  ret i16 %tmp
+}
+
+define i8 @func8(i8 %x, i8 %y, i8 %z) nounwind {
+; CHECK-LABEL: func8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sxtb w8, w0
+; CHECK-NEXT:    mul w9, w1, w2
+; CHECK-NEXT:    sub w8, w8, w9, sxtb
+; CHECK-NEXT:    mov w10, #127
+; CHECK-NEXT:    cmp w8, #127 // =127
+; CHECK-NEXT:    csel w8, w8, w10, lt
+; CHECK-NEXT:    cmn w8, #128 // =128
+; CHECK-NEXT:    mov w9, #-128
+; CHECK-NEXT:    csel w0, w8, w9, gt
+; CHECK-NEXT:    ret
+  %a = mul i8 %y, %z
+  %tmp = call i8 @llvm.ssub.sat.i8(i8 %x, i8 %a)
+  ret i8 %tmp
+}
+
+define i4 @func4(i4 %x, i4 %y, i4 %z) nounwind {
+; CHECK-LABEL: func4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mul w9, w1, w2
+; CHECK-NEXT:    sbfx w8, w0, #0, #4
+; CHECK-NEXT:    lsl w9, w9, #28
+; CHECK-NEXT:    sub w8, w8, w9, asr #28
+; CHECK-NEXT:    mov w10, #7
+; CHECK-NEXT:    cmp w8, #7 // =7
+; CHECK-NEXT:    csel w8, w8, w10, lt
+; CHECK-NEXT:    cmn w8, #8 // =8
+; CHECK-NEXT:    mov w9, #-8
+; CHECK-NEXT:    csel w0, w8, w9, gt
+; CHECK-NEXT:    ret
+  %a = mul i4 %y, %z
+  %tmp = call i4 @llvm.ssub.sat.i4(i4 %x, i4 %a)
+  ret i4 %tmp
+}
diff --git a/llvm/test/CodeGen/AArch64/ssub_sat_vec.ll b/llvm/test/CodeGen/AArch64/ssub_sat_vec.ll
index 3718b6bbc..2cf6e896b 100644
--- a/llvm/test/CodeGen/AArch64/ssub_sat_vec.ll
+++ b/llvm/test/CodeGen/AArch64/ssub_sat_vec.ll
@@ -36,20 +36,7 @@ declare <2 x i128> @llvm.ssub.sat.v2i128(<2 x i128>, <2 x i128>)
 define <16 x i8> @v16i8(<16 x i8> %x, <16 x i8> %y) nounwind {
 ; CHECK-LABEL: v16i8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    sub v2.16b, v0.16b, v1.16b
-; CHECK-NEXT:    cmge v1.16b, v1.16b, #0
-; CHECK-NEXT:    cmge v0.16b, v0.16b, #0
-; CHECK-NEXT:    cmge v5.16b, v2.16b, #0
-; CHECK-NEXT:    cmlt v4.16b, v2.16b, #0
-; CHECK-NEXT:    cmeq v1.16b, v0.16b, v1.16b
-; CHECK-NEXT:    cmeq v0.16b, v0.16b, v5.16b
-; CHECK-NEXT:    movi v3.16b, #127
-; CHECK-NEXT:    mvn v5.16b, v4.16b
-; CHECK-NEXT:    mvn v1.16b, v1.16b
-; CHECK-NEXT:    mvn v0.16b, v0.16b
-; CHECK-NEXT:    bsl v3.16b, v4.16b, v5.16b
-; CHECK-NEXT:    and v0.16b, v1.16b, v0.16b
-; CHECK-NEXT:    bsl v0.16b, v3.16b, v2.16b
+; CHECK-NEXT:    sqsub v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
   %z = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> %x, <16 x i8> %y)
   ret <16 x i8> %z
@@ -58,34 +45,8 @@ define <16 x i8> @v16i8(<16 x i8> %x, <16 x i8> %y) nounwind {
 define <32 x i8> @v32i8(<32 x i8> %x, <32 x i8> %y) nounwind {
 ; CHECK-LABEL: v32i8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    sub v4.16b, v0.16b, v2.16b
-; CHECK-NEXT:    cmlt v16.16b, v4.16b, #0
-; CHECK-NEXT:    movi v6.16b, #127
-; CHECK-NEXT:    sub v7.16b, v1.16b, v3.16b
-; CHECK-NEXT:    mvn v17.16b, v16.16b
-; CHECK-NEXT:    bsl v6.16b, v16.16b, v17.16b
-; CHECK-NEXT:    cmlt v16.16b, v7.16b, #0
-; CHECK-NEXT:    movi v5.16b, #127
-; CHECK-NEXT:    mvn v17.16b, v16.16b
-; CHECK-NEXT:    bsl v5.16b, v16.16b, v17.16b
-; CHECK-NEXT:    cmge v2.16b, v2.16b, #0
-; CHECK-NEXT:    cmge v0.16b, v0.16b, #0
-; CHECK-NEXT:    cmge v16.16b, v4.16b, #0
-; CHECK-NEXT:    cmge v3.16b, v3.16b, #0
-; CHECK-NEXT:    cmge v1.16b, v1.16b, #0
-; CHECK-NEXT:    cmeq v2.16b, v0.16b, v2.16b
-; CHECK-NEXT:    cmeq v0.16b, v0.16b, v16.16b
-; CHECK-NEXT:    cmge v16.16b, v7.16b, #0
-; CHECK-NEXT:    cmeq v3.16b, v1.16b, v3.16b
-; CHECK-NEXT:    cmeq v1.16b, v1.16b, v16.16b
-; CHECK-NEXT:    mvn v2.16b, v2.16b
-; CHECK-NEXT:    mvn v3.16b, v3.16b
-; CHECK-NEXT:    mvn v0.16b, v0.16b
-; CHECK-NEXT:    mvn v1.16b, v1.16b
-; CHECK-NEXT:    and v0.16b, v2.16b, v0.16b
-; CHECK-NEXT:    and v1.16b, v3.16b, v1.16b
-; CHECK-NEXT:    bsl v0.16b, v6.16b, v4.16b
-; CHECK-NEXT:    bsl v1.16b, v5.16b, v7.16b
+; CHECK-NEXT:    sqsub v0.16b, v0.16b, v2.16b
+; CHECK-NEXT:    sqsub v1.16b, v1.16b, v3.16b
 ; CHECK-NEXT:    ret
   %z = call <32 x i8> @llvm.ssub.sat.v32i8(<32 x i8> %x, <32 x i8> %y)
   ret <32 x i8> %z
@@ -94,62 +55,10 @@ define <32 x i8> @v32i8(<32 x i8> %x, <32 x i8> %y) nounwind {
 define <64 x i8> @v64i8(<64 x i8> %x, <64 x i8> %y) nounwind {
 ; CHECK-LABEL: v64i8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    sub v16.16b, v0.16b, v4.16b
-; CHECK-NEXT:    cmlt v24.16b, v16.16b, #0
-; CHECK-NEXT:    movi v18.16b, #127
-; CHECK-NEXT:    sub v19.16b, v1.16b, v5.16b
-; CHECK-NEXT:    mvn v25.16b, v24.16b
-; CHECK-NEXT:    bsl v18.16b, v24.16b, v25.16b
-; CHECK-NEXT:    cmlt v24.16b, v19.16b, #0
-; CHECK-NEXT:    movi v20.16b, #127
-; CHECK-NEXT:    sub v21.16b, v2.16b, v6.16b
-; CHECK-NEXT:    mvn v25.16b, v24.16b
-; CHECK-NEXT:    bsl v20.16b, v24.16b, v25.16b
-; CHECK-NEXT:    cmlt v24.16b, v21.16b, #0
-; CHECK-NEXT:    movi v22.16b, #127
-; CHECK-NEXT:    sub v23.16b, v3.16b, v7.16b
-; CHECK-NEXT:    mvn v25.16b, v24.16b
-; CHECK-NEXT:    bsl v22.16b, v24.16b, v25.16b
-; CHECK-NEXT:    cmlt v24.16b, v23.16b, #0
-; CHECK-NEXT:    movi v17.16b, #127
-; CHECK-NEXT:    mvn v25.16b, v24.16b
-; CHECK-NEXT:    bsl v17.16b, v24.16b, v25.16b
-; CHECK-NEXT:    cmge v4.16b, v4.16b, #0
-; CHECK-NEXT:    cmge v0.16b, v0.16b, #0
-; CHECK-NEXT:    cmge v24.16b, v16.16b, #0
-; CHECK-NEXT:    cmge v5.16b, v5.16b, #0
-; CHECK-NEXT:    cmge v1.16b, v1.16b, #0
-; CHECK-NEXT:    cmeq v4.16b, v0.16b, v4.16b
-; CHECK-NEXT:    cmeq v0.16b, v0.16b, v24.16b
-; CHECK-NEXT:    cmge v24.16b, v19.16b, #0
-; CHECK-NEXT:    cmge v6.16b, v6.16b, #0
-; CHECK-NEXT:    cmge v2.16b, v2.16b, #0
-; CHECK-NEXT:    cmeq v5.16b, v1.16b, v5.16b
-; CHECK-NEXT:    cmeq v1.16b, v1.16b, v24.16b
-; CHECK-NEXT:    cmge v24.16b, v21.16b, #0
-; CHECK-NEXT:    mvn v4.16b, v4.16b
-; CHECK-NEXT:    mvn v0.16b, v0.16b
-; CHECK-NEXT:    cmge v7.16b, v7.16b, #0
-; CHECK-NEXT:    cmge v3.16b, v3.16b, #0
-; CHECK-NEXT:    cmeq v6.16b, v2.16b, v6.16b
-; CHECK-NEXT:    cmeq v2.16b, v2.16b, v24.16b
-; CHECK-NEXT:    cmge v24.16b, v23.16b, #0
-; CHECK-NEXT:    and v0.16b, v4.16b, v0.16b
-; CHECK-NEXT:    mvn v4.16b, v5.16b
-; CHECK-NEXT:    mvn v1.16b, v1.16b
-; CHECK-NEXT:    cmeq v7.16b, v3.16b, v7.16b
-; CHECK-NEXT:    cmeq v3.16b, v3.16b, v24.16b
-; CHECK-NEXT:    and v1.16b, v4.16b, v1.16b
-; CHECK-NEXT:    mvn v4.16b, v6.16b
-; CHECK-NEXT:    mvn v2.16b, v2.16b
-; CHECK-NEXT:    and v2.16b, v4.16b, v2.16b
-; CHECK-NEXT:    mvn v4.16b, v7.16b
-; CHECK-NEXT:    mvn v3.16b, v3.16b
-; CHECK-NEXT:    and v3.16b, v4.16b, v3.16b
-; CHECK-NEXT:    bsl v0.16b, v18.16b, v16.16b
-; CHECK-NEXT:    bsl v1.16b, v20.16b, v19.16b
-; CHECK-NEXT:    bsl v2.16b, v22.16b, v21.16b
-; CHECK-NEXT:    bsl v3.16b, v17.16b, v23.16b
+; CHECK-NEXT:    sqsub v0.16b, v0.16b, v4.16b
+; CHECK-NEXT:    sqsub v1.16b, v1.16b, v5.16b
+; CHECK-NEXT:    sqsub v2.16b, v2.16b, v6.16b
+; CHECK-NEXT:    sqsub v3.16b, v3.16b, v7.16b
 ; CHECK-NEXT:    ret
   %z = call <64 x i8> @llvm.ssub.sat.v64i8(<64 x i8> %x, <64 x i8> %y)
   ret <64 x i8> %z
@@ -158,20 +67,7 @@ define <64 x i8> @v64i8(<64 x i8> %x, <64 x i8> %y) nounwind {
 define <8 x i16> @v8i16(<8 x i16> %x, <8 x i16> %y) nounwind {
 ; CHECK-LABEL: v8i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    sub v2.8h, v0.8h, v1.8h
-; CHECK-NEXT:    cmge v1.8h, v1.8h, #0
-; CHECK-NEXT:    cmge v0.8h, v0.8h, #0
-; CHECK-NEXT:    cmge v5.8h, v2.8h, #0
-; CHECK-NEXT:    cmlt v4.8h, v2.8h, #0
-; CHECK-NEXT:    cmeq v1.8h, v0.8h, v1.8h
-; CHECK-NEXT:    cmeq v0.8h, v0.8h, v5.8h
-; CHECK-NEXT:    mvni v3.8h, #128, lsl #8
-; CHECK-NEXT:    mvn v5.16b, v4.16b
-; CHECK-NEXT:    mvn v1.16b, v1.16b
-; CHECK-NEXT:    mvn v0.16b, v0.16b
-; CHECK-NEXT:    bsl v3.16b, v4.16b, v5.16b
-; CHECK-NEXT:    and v0.16b, v1.16b, v0.16b
-; CHECK-NEXT:    bsl v0.16b, v3.16b, v2.16b
+; CHECK-NEXT:    sqsub v0.8h, v0.8h, v1.8h
 ; CHECK-NEXT:    ret
   %z = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> %x, <8 x i16> %y)
   ret <8 x i16> %z
@@ -180,34 +76,8 @@ define <8 x i16> @v8i16(<8 x i16> %x, <8 x i16> %y) nounwind {
 define <16 x i16> @v16i16(<16 x i16> %x, <16 x i16> %y) nounwind {
 ; CHECK-LABEL: v16i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    sub v4.8h, v0.8h, v2.8h
-; CHECK-NEXT:    cmlt v16.8h, v4.8h, #0
-; CHECK-NEXT:    mvni v6.8h, #128, lsl #8
-; CHECK-NEXT:    sub v7.8h, v1.8h, v3.8h
-; CHECK-NEXT:    mvn v17.16b, v16.16b
-; CHECK-NEXT:    bsl v6.16b, v16.16b, v17.16b
-; CHECK-NEXT:    cmlt v16.8h, v7.8h, #0
-; CHECK-NEXT:    mvni v5.8h, #128, lsl #8
-; CHECK-NEXT:    mvn v17.16b, v16.16b
-; CHECK-NEXT:    bsl v5.16b, v16.16b, v17.16b
-; CHECK-NEXT:    cmge v2.8h, v2.8h, #0
-; CHECK-NEXT:    cmge v0.8h, v0.8h, #0
-; CHECK-NEXT:    cmge v16.8h, v4.8h, #0
-; CHECK-NEXT:    cmge v3.8h, v3.8h, #0
-; CHECK-NEXT:    cmge v1.8h, v1.8h, #0
-; CHECK-NEXT:    cmeq v2.8h, v0.8h, v2.8h
-; CHECK-NEXT:    cmeq v0.8h, v0.8h, v16.8h
-; CHECK-NEXT:    cmge v16.8h, v7.8h, #0
-; CHECK-NEXT:    cmeq v3.8h, v1.8h, v3.8h
-; CHECK-NEXT:    cmeq v1.8h, v1.8h, v16.8h
-; CHECK-NEXT:    mvn v2.16b, v2.16b
-; CHECK-NEXT:    mvn v3.16b, v3.16b
-; CHECK-NEXT:    mvn v0.16b, v0.16b
-; CHECK-NEXT:    mvn v1.16b, v1.16b
-; CHECK-NEXT:    and v0.16b, v2.16b, v0.16b
-; CHECK-NEXT:    and v1.16b, v3.16b, v1.16b
-; CHECK-NEXT:    bsl v0.16b, v6.16b, v4.16b
-; CHECK-NEXT:    bsl v1.16b, v5.16b, v7.16b
+; CHECK-NEXT:    sqsub v0.8h, v0.8h, v2.8h
+; CHECK-NEXT:    sqsub v1.8h, v1.8h, v3.8h
 ; CHECK-NEXT:    ret
   %z = call <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16> %x, <16 x i16> %y)
   ret <16 x i16> %z
@@ -216,62 +86,10 @@ define <16 x i16> @v16i16(<16 x i16> %x, <16 x i16> %y) nounwind {
 define <32 x i16> @v32i16(<32 x i16> %x, <32 x i16> %y) nounwind {
 ; CHECK-LABEL: v32i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    sub v16.8h, v0.8h, v4.8h
-; CHECK-NEXT:    cmlt v24.8h, v16.8h, #0
-; CHECK-NEXT:    mvni v18.8h, #128, lsl #8
-; CHECK-NEXT:    sub v19.8h, v1.8h, v5.8h
-; CHECK-NEXT:    mvn v25.16b, v24.16b
-; CHECK-NEXT:    bsl v18.16b, v24.16b, v25.16b
-; CHECK-NEXT:    cmlt v24.8h, v19.8h, #0
-; CHECK-NEXT:    mvni v20.8h, #128, lsl #8
-; CHECK-NEXT:    sub v21.8h, v2.8h, v6.8h
-; CHECK-NEXT:    mvn v25.16b, v24.16b
-; CHECK-NEXT:    bsl v20.16b, v24.16b, v25.16b
-; CHECK-NEXT:    cmlt v24.8h, v21.8h, #0
-; CHECK-NEXT:    mvni v22.8h, #128, lsl #8
-; CHECK-NEXT:    sub v23.8h, v3.8h, v7.8h
-; CHECK-NEXT:    mvn v25.16b, v24.16b
-; CHECK-NEXT:    bsl v22.16b, v24.16b, v25.16b
-; CHECK-NEXT:    cmlt v24.8h, v23.8h, #0
-; CHECK-NEXT:    mvni v17.8h, #128, lsl #8
-; CHECK-NEXT:    mvn v25.16b, v24.16b
-; CHECK-NEXT:    bsl v17.16b, v24.16b, v25.16b
-; CHECK-NEXT:    cmge v4.8h, v4.8h, #0
-; CHECK-NEXT:    cmge v0.8h, v0.8h, #0
-; CHECK-NEXT:    cmge v24.8h, v16.8h, #0
-; CHECK-NEXT:    cmge v5.8h, v5.8h, #0
-; CHECK-NEXT:    cmge v1.8h, v1.8h, #0
-; CHECK-NEXT:    cmeq v4.8h, v0.8h, v4.8h
-; CHECK-NEXT:    cmeq v0.8h, v0.8h, v24.8h
-; CHECK-NEXT:    cmge v24.8h, v19.8h, #0
-; CHECK-NEXT:    cmge v6.8h, v6.8h, #0
-; CHECK-NEXT:    cmge v2.8h, v2.8h, #0
-; CHECK-NEXT:    cmeq v5.8h, v1.8h, v5.8h
-; CHECK-NEXT:    cmeq v1.8h, v1.8h, v24.8h
-; CHECK-NEXT:    cmge v24.8h, v21.8h, #0
-; CHECK-NEXT:    mvn v4.16b, v4.16b
-; CHECK-NEXT:    mvn v0.16b, v0.16b
-; CHECK-NEXT:    cmge v7.8h, v7.8h, #0
-; CHECK-NEXT:    cmge v3.8h, v3.8h, #0
-; CHECK-NEXT:    cmeq v6.8h, v2.8h, v6.8h
-; CHECK-NEXT:    cmeq v2.8h, v2.8h, v24.8h
-; CHECK-NEXT:    cmge v24.8h, v23.8h, #0
-; CHECK-NEXT:    and v0.16b, v4.16b, v0.16b
-; CHECK-NEXT:    mvn v4.16b, v5.16b
-; CHECK-NEXT:    mvn v1.16b, v1.16b
-; CHECK-NEXT:    cmeq v7.8h, v3.8h, v7.8h
-; CHECK-NEXT:    cmeq v3.8h, v3.8h, v24.8h
-; CHECK-NEXT:    and v1.16b, v4.16b, v1.16b
-; CHECK-NEXT:    mvn v4.16b, v6.16b
-; CHECK-NEXT:    mvn v2.16b, v2.16b
-; CHECK-NEXT:    and v2.16b, v4.16b, v2.16b
-; CHECK-NEXT:    mvn v4.16b, v7.16b
-; CHECK-NEXT:    mvn v3.16b, v3.16b
-; CHECK-NEXT:    and v3.16b, v4.16b, v3.16b
-; CHECK-NEXT:    bsl v0.16b, v18.16b, v16.16b
-; CHECK-NEXT:    bsl v1.16b, v20.16b, v19.16b
-; CHECK-NEXT:    bsl v2.16b, v22.16b, v21.16b
-; CHECK-NEXT:    bsl v3.16b, v17.16b, v23.16b
+; CHECK-NEXT:    sqsub v0.8h, v0.8h, v4.8h
+; CHECK-NEXT:    sqsub v1.8h, v1.8h, v5.8h
+; CHECK-NEXT:    sqsub v2.8h, v2.8h, v6.8h
+; CHECK-NEXT:    sqsub v3.8h, v3.8h, v7.8h
 ; CHECK-NEXT:    ret
   %z = call <32 x i16> @llvm.ssub.sat.v32i16(<32 x i16> %x, <32 x i16> %y)
   ret <32 x i16> %z
@@ -282,20 +100,7 @@ define void @v8i8(<8 x i8>* %px, <8 x i8>* %py, <8 x i8>* %pz) nounwind {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldr d0, [x0]
 ; CHECK-NEXT:    ldr d1, [x1]
-; CHECK-NEXT:    movi v2.8b, #127
-; CHECK-NEXT:    sub v3.8b, v0.8b, v1.8b
-; CHECK-NEXT:    cmge v1.8b, v1.8b, #0
-; CHECK-NEXT:    cmge v0.8b, v0.8b, #0
-; CHECK-NEXT:    cmge v5.8b, v3.8b, #0
-; CHECK-NEXT:    cmlt v4.8b, v3.8b, #0
-; CHECK-NEXT:    cmeq v1.8b, v0.8b, v1.8b
-; CHECK-NEXT:    cmeq v0.8b, v0.8b, v5.8b
-; CHECK-NEXT:    mvn v5.8b, v4.8b
-; CHECK-NEXT:    mvn v1.8b, v1.8b
-; CHECK-NEXT:    mvn v0.8b, v0.8b
-; CHECK-NEXT:    bsl v2.8b, v4.8b, v5.8b
-; CHECK-NEXT:    and v0.8b, v1.8b, v0.8b
-; CHECK-NEXT:    bsl v0.8b, v2.8b, v3.8b
+; CHECK-NEXT:    sqsub v0.8b, v0.8b, v1.8b
 ; CHECK-NEXT:    str d0, [x2]
 ; CHECK-NEXT:    ret
   %x = load <8 x i8>, <8 x i8>* %px
@@ -308,38 +113,25 @@ define void @v8i8(<8 x i8>* %px, <8 x i8>* %py, <8 x i8>* %pz) nounwind {
 define void @v4i8(<4 x i8>* %px, <4 x i8>* %py, <4 x i8>* %pz) nounwind {
 ; CHECK-LABEL: v4i8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldrb w8, [x0]
-; CHECK-NEXT:    ldrb w9, [x1]
-; CHECK-NEXT:    ldrb w10, [x0, #1]
-; CHECK-NEXT:    ldrb w11, [x1, #1]
-; CHECK-NEXT:    ldrb w12, [x0, #2]
+; CHECK-NEXT:    ldrsb w8, [x0]
+; CHECK-NEXT:    ldrsb w9, [x1]
+; CHECK-NEXT:    ldrsb w10, [x0, #1]
+; CHECK-NEXT:    ldrsb w11, [x1, #1]
 ; CHECK-NEXT:    fmov s0, w8
-; CHECK-NEXT:    ldrb w8, [x1, #2]
 ; CHECK-NEXT:    fmov s1, w9
+; CHECK-NEXT:    ldrsb w8, [x0, #2]
+; CHECK-NEXT:    ldrsb w9, [x1, #2]
 ; CHECK-NEXT:    mov v0.h[1], w10
-; CHECK-NEXT:    ldrb w9, [x0, #3]
-; CHECK-NEXT:    ldrb w10, [x1, #3]
 ; CHECK-NEXT:    mov v1.h[1], w11
-; CHECK-NEXT:    mov v0.h[2], w12
-; CHECK-NEXT:    mov v1.h[2], w8
-; CHECK-NEXT:    mov v0.h[3], w9
-; CHECK-NEXT:    mov v1.h[3], w10
+; CHECK-NEXT:    ldrsb w10, [x0, #3]
+; CHECK-NEXT:    ldrsb w11, [x1, #3]
+; CHECK-NEXT:    mov v0.h[2], w8
+; CHECK-NEXT:    mov v1.h[2], w9
+; CHECK-NEXT:    mov v0.h[3], w10
+; CHECK-NEXT:    mov v1.h[3], w11
 ; CHECK-NEXT:    shl v1.4h, v1.4h, #8
 ; CHECK-NEXT:    shl v0.4h, v0.4h, #8
-; CHECK-NEXT:    sub v3.4h, v0.4h, v1.4h
-; CHECK-NEXT:    cmge v1.4h, v1.4h, #0
-; CHECK-NEXT:    cmge v0.4h, v0.4h, #0
-; CHECK-NEXT:    cmge v5.4h, v3.4h, #0
-; CHECK-NEXT:    cmlt v4.4h, v3.4h, #0
-; CHECK-NEXT:    cmeq v1.4h, v0.4h, v1.4h
-; CHECK-NEXT:    cmeq v0.4h, v0.4h, v5.4h
-; CHECK-NEXT:    mvni v2.4h, #128, lsl #8
-; CHECK-NEXT:    mvn v5.8b, v4.8b
-; CHECK-NEXT:    mvn v1.8b, v1.8b
-; CHECK-NEXT:    mvn v0.8b, v0.8b
-; CHECK-NEXT:    bsl v2.8b, v4.8b, v5.8b
-; CHECK-NEXT:    and v0.8b, v1.8b, v0.8b
-; CHECK-NEXT:    bsl v0.8b, v2.8b, v3.8b
+; CHECK-NEXT:    sqsub v0.4h, v0.4h, v1.4h
 ; CHECK-NEXT:    sshr v0.4h, v0.4h, #8
 ; CHECK-NEXT:    xtn v0.8b, v0.8h
 ; CHECK-NEXT:    str s0, [x2]
@@ -359,25 +151,12 @@ define void @v2i8(<2 x i8>* %px, <2 x i8>* %py, <2 x i8>* %pz) nounwind {
 ; CHECK-NEXT:    ldrb w10, [x0, #1]
 ; CHECK-NEXT:    ldrb w11, [x1, #1]
 ; CHECK-NEXT:    fmov s0, w8
-; CHECK-NEXT:    fmov s2, w9
+; CHECK-NEXT:    fmov s1, w9
 ; CHECK-NEXT:    mov v0.s[1], w10
-; CHECK-NEXT:    mov v2.s[1], w11
-; CHECK-NEXT:    shl v2.2s, v2.2s, #24
+; CHECK-NEXT:    mov v1.s[1], w11
+; CHECK-NEXT:    shl v1.2s, v1.2s, #24
 ; CHECK-NEXT:    shl v0.2s, v0.2s, #24
-; CHECK-NEXT:    sub v3.2s, v0.2s, v2.2s
-; CHECK-NEXT:    cmge v2.2s, v2.2s, #0
-; CHECK-NEXT:    cmge v0.2s, v0.2s, #0
-; CHECK-NEXT:    cmge v5.2s, v3.2s, #0
-; CHECK-NEXT:    cmlt v4.2s, v3.2s, #0
-; CHECK-NEXT:    cmeq v2.2s, v0.2s, v2.2s
-; CHECK-NEXT:    cmeq v0.2s, v0.2s, v5.2s
-; CHECK-NEXT:    mvni v1.2s, #128, lsl #24
-; CHECK-NEXT:    mvn v5.8b, v4.8b
-; CHECK-NEXT:    mvn v2.8b, v2.8b
-; CHECK-NEXT:    mvn v0.8b, v0.8b
-; CHECK-NEXT:    bsl v1.8b, v4.8b, v5.8b
-; CHECK-NEXT:    and v0.8b, v2.8b, v0.8b
-; CHECK-NEXT:    bsl v0.8b, v1.8b, v3.8b
+; CHECK-NEXT:    sqsub v0.2s, v0.2s, v1.2s
 ; CHECK-NEXT:    ushr v0.2s, v0.2s, #24
 ; CHECK-NEXT:    mov w8, v0.s[1]
 ; CHECK-NEXT:    fmov w9, s0
@@ -396,20 +175,7 @@ define void @v4i16(<4 x i16>* %px, <4 x i16>* %py, <4 x i16>* %pz) nounwind {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldr d0, [x0]
 ; CHECK-NEXT:    ldr d1, [x1]
-; CHECK-NEXT:    mvni v2.4h, #128, lsl #8
-; CHECK-NEXT:    sub v3.4h, v0.4h, v1.4h
-; CHECK-NEXT:    cmge v1.4h, v1.4h, #0
-; CHECK-NEXT:    cmge v0.4h, v0.4h, #0
-; CHECK-NEXT:    cmge v5.4h, v3.4h, #0
-; CHECK-NEXT:    cmlt v4.4h, v3.4h, #0
-; CHECK-NEXT:    cmeq v1.4h, v0.4h, v1.4h
-; CHECK-NEXT:    cmeq v0.4h, v0.4h, v5.4h
-; CHECK-NEXT:    mvn v5.8b, v4.8b
-; CHECK-NEXT:    mvn v1.8b, v1.8b
-; CHECK-NEXT:    mvn v0.8b, v0.8b
-; CHECK-NEXT:    bsl v2.8b, v4.8b, v5.8b
-; CHECK-NEXT:    and v0.8b, v1.8b, v0.8b
-; CHECK-NEXT:    bsl v0.8b, v2.8b, v3.8b
+; CHECK-NEXT:    sqsub v0.4h, v0.4h, v1.4h
 ; CHECK-NEXT:    str d0, [x2]
 ; CHECK-NEXT:    ret
   %x = load <4 x i16>, <4 x i16>* %px
@@ -427,25 +193,12 @@ define void @v2i16(<2 x i16>* %px, <2 x i16>* %py, <2 x i16>* %pz) nounwind {
 ; CHECK-NEXT:    ldrh w10, [x0, #2]
 ; CHECK-NEXT:    ldrh w11, [x1, #2]
 ; CHECK-NEXT:    fmov s0, w8
-; CHECK-NEXT:    fmov s2, w9
+; CHECK-NEXT:    fmov s1, w9
 ; CHECK-NEXT:    mov v0.s[1], w10
-; CHECK-NEXT:    mov v2.s[1], w11
-; CHECK-NEXT:    shl v2.2s, v2.2s, #16
+; CHECK-NEXT:    mov v1.s[1], w11
+; CHECK-NEXT:    shl v1.2s, v1.2s, #16
 ; CHECK-NEXT:    shl v0.2s, v0.2s, #16
-; CHECK-NEXT:    sub v3.2s, v0.2s, v2.2s
-; CHECK-NEXT:    cmge v2.2s, v2.2s, #0
-; CHECK-NEXT:    cmge v0.2s, v0.2s, #0
-; CHECK-NEXT:    cmge v5.2s, v3.2s, #0
-; CHECK-NEXT:    cmlt v4.2s, v3.2s, #0
-; CHECK-NEXT:    cmeq v2.2s, v0.2s, v2.2s
-; CHECK-NEXT:    cmeq v0.2s, v0.2s, v5.2s
-; CHECK-NEXT:    mvni v1.2s, #128, lsl #24
-; CHECK-NEXT:    mvn v5.8b, v4.8b
-; CHECK-NEXT:    mvn v2.8b, v2.8b
-; CHECK-NEXT:    mvn v0.8b, v0.8b
-; CHECK-NEXT:    bsl v1.8b, v4.8b, v5.8b
-; CHECK-NEXT:    and v0.8b, v2.8b, v0.8b
-; CHECK-NEXT:    bsl v0.8b, v1.8b, v3.8b
+; CHECK-NEXT:    sqsub v0.2s, v0.2s, v1.2s
 ; CHECK-NEXT:    ushr v0.2s, v0.2s, #16
 ; CHECK-NEXT:    mov w8, v0.s[1]
 ; CHECK-NEXT:    fmov w9, s0
@@ -462,20 +215,7 @@ define void @v2i16(<2 x i16>* %px, <2 x i16>* %py, <2 x i16>* %pz) nounwind {
 define <12 x i8> @v12i8(<12 x i8> %x, <12 x i8> %y) nounwind {
 ; CHECK-LABEL: v12i8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    sub v2.16b, v0.16b, v1.16b
-; CHECK-NEXT:    cmge v1.16b, v1.16b, #0
-; CHECK-NEXT:    cmge v0.16b, v0.16b, #0
-; CHECK-NEXT:    cmge v5.16b, v2.16b, #0
-; CHECK-NEXT:    cmlt v4.16b, v2.16b, #0
-; CHECK-NEXT:    cmeq v1.16b, v0.16b, v1.16b
-; CHECK-NEXT:    cmeq v0.16b, v0.16b, v5.16b
-; CHECK-NEXT:    movi v3.16b, #127
-; CHECK-NEXT:    mvn v5.16b, v4.16b
-; CHECK-NEXT:    mvn v1.16b, v1.16b
-; CHECK-NEXT:    mvn v0.16b, v0.16b
-; CHECK-NEXT:    bsl v3.16b, v4.16b, v5.16b
-; CHECK-NEXT:    and v0.16b, v1.16b, v0.16b
-; CHECK-NEXT:    bsl v0.16b, v3.16b, v2.16b
+; CHECK-NEXT:    sqsub v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
   %z = call <12 x i8> @llvm.ssub.sat.v12i8(<12 x i8> %x, <12 x i8> %y)
   ret <12 x i8> %z
@@ -486,34 +226,8 @@ define void @v12i16(<12 x i16>* %px, <12 x i16>* %py, <12 x i16>* %pz) nounwind
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldp q0, q1, [x0]
 ; CHECK-NEXT:    ldp q3, q2, [x1]
-; CHECK-NEXT:    mvni v5.8h, #128, lsl #8
-; CHECK-NEXT:    mvni v4.8h, #128, lsl #8
-; CHECK-NEXT:    sub v6.8h, v1.8h, v2.8h
-; CHECK-NEXT:    cmlt v16.8h, v6.8h, #0
-; CHECK-NEXT:    sub v7.8h, v0.8h, v3.8h
-; CHECK-NEXT:    mvn v17.16b, v16.16b
-; CHECK-NEXT:    bsl v5.16b, v16.16b, v17.16b
-; CHECK-NEXT:    cmlt v16.8h, v7.8h, #0
-; CHECK-NEXT:    mvn v17.16b, v16.16b
-; CHECK-NEXT:    bsl v4.16b, v16.16b, v17.16b
-; CHECK-NEXT:    cmge v2.8h, v2.8h, #0
-; CHECK-NEXT:    cmge v1.8h, v1.8h, #0
-; CHECK-NEXT:    cmge v16.8h, v6.8h, #0
-; CHECK-NEXT:    cmge v3.8h, v3.8h, #0
-; CHECK-NEXT:    cmge v0.8h, v0.8h, #0
-; CHECK-NEXT:    cmeq v2.8h, v1.8h, v2.8h
-; CHECK-NEXT:    cmeq v1.8h, v1.8h, v16.8h
-; CHECK-NEXT:    cmge v16.8h, v7.8h, #0
-; CHECK-NEXT:    cmeq v3.8h, v0.8h, v3.8h
-; CHECK-NEXT:    cmeq v0.8h, v0.8h, v16.8h
-; CHECK-NEXT:    mvn v2.16b, v2.16b
-; CHECK-NEXT:    mvn v3.16b, v3.16b
-; CHECK-NEXT:    mvn v1.16b, v1.16b
-; CHECK-NEXT:    mvn v0.16b, v0.16b
-; CHECK-NEXT:    and v1.16b, v2.16b, v1.16b
-; CHECK-NEXT:    and v0.16b, v3.16b, v0.16b
-; CHECK-NEXT:    bsl v1.16b, v5.16b, v6.16b
-; CHECK-NEXT:    bsl v0.16b, v4.16b, v7.16b
+; CHECK-NEXT:    sqsub v1.8h, v1.8h, v2.8h
+; CHECK-NEXT:    sqsub v0.8h, v0.8h, v3.8h
 ; CHECK-NEXT:    str q0, [x2]
 ; CHECK-NEXT:    str d1, [x2, #16]
 ; CHECK-NEXT:    ret
@@ -529,20 +243,7 @@ define void @v1i8(<1 x i8>* %px, <1 x i8>* %py, <1 x i8>* %pz) nounwind {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldr b0, [x0]
 ; CHECK-NEXT:    ldr b1, [x1]
-; CHECK-NEXT:    movi v2.8b, #127
-; CHECK-NEXT:    sub v3.8b, v0.8b, v1.8b
-; CHECK-NEXT:    cmge v1.8b, v1.8b, #0
-; CHECK-NEXT:    cmge v0.8b, v0.8b, #0
-; CHECK-NEXT:    cmge v5.8b, v3.8b, #0
-; CHECK-NEXT:    cmlt v4.8b, v3.8b, #0
-; CHECK-NEXT:    cmeq v1.8b, v0.8b, v1.8b
-; CHECK-NEXT:    cmeq v0.8b, v0.8b, v5.8b
-; CHECK-NEXT:    mvn v5.8b, v4.8b
-; CHECK-NEXT:    mvn v1.8b, v1.8b
-; CHECK-NEXT:    mvn v0.8b, v0.8b
-; CHECK-NEXT:    bsl v2.8b, v4.8b, v5.8b
-; CHECK-NEXT:    and v0.8b, v1.8b, v0.8b
-; CHECK-NEXT:    bsl v0.8b, v2.8b, v3.8b
+; CHECK-NEXT:    sqsub v0.8b, v0.8b, v1.8b
 ; CHECK-NEXT:    st1 { v0.b }[0], [x2]
 ; CHECK-NEXT:    ret
   %x = load <1 x i8>, <1 x i8>* %px
@@ -557,20 +258,7 @@ define void @v1i16(<1 x i16>* %px, <1 x i16>* %py, <1 x i16>* %pz) nounwind {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldr h0, [x0]
 ; CHECK-NEXT:    ldr h1, [x1]
-; CHECK-NEXT:    mvni v2.4h, #128, lsl #8
-; CHECK-NEXT:    sub v3.4h, v0.4h, v1.4h
-; CHECK-NEXT:    cmge v1.4h, v1.4h, #0
-; CHECK-NEXT:    cmge v0.4h, v0.4h, #0
-; CHECK-NEXT:    cmge v5.4h, v3.4h, #0
-; CHECK-NEXT:    cmlt v4.4h, v3.4h, #0
-; CHECK-NEXT:    cmeq v1.4h, v0.4h, v1.4h
-; CHECK-NEXT:    cmeq v0.4h, v0.4h, v5.4h
-; CHECK-NEXT:    mvn v5.8b, v4.8b
-; CHECK-NEXT:    mvn v1.8b, v1.8b
-; CHECK-NEXT:    mvn v0.8b, v0.8b
-; CHECK-NEXT:    bsl v2.8b, v4.8b, v5.8b
-; CHECK-NEXT:    and v0.8b, v1.8b, v0.8b
-; CHECK-NEXT:    bsl v0.8b, v2.8b, v3.8b
+; CHECK-NEXT:    sqsub v0.4h, v0.4h, v1.4h
 ; CHECK-NEXT:    str h0, [x2]
 ; CHECK-NEXT:    ret
   %x = load <1 x i16>, <1 x i16>* %px
@@ -583,22 +271,13 @@ define void @v1i16(<1 x i16>* %px, <1 x i16>* %py, <1 x i16>* %pz) nounwind {
 define <16 x i4> @v16i4(<16 x i4> %x, <16 x i4> %y) nounwind {
 ; CHECK-LABEL: v16i4:
 ; CHECK:       // %bb.0:
+; CHECK-NEXT:    shl v0.16b, v0.16b, #4
+; CHECK-NEXT:    shl v1.16b, v1.16b, #4
+; CHECK-NEXT:    sshr v0.16b, v0.16b, #4
+; CHECK-NEXT:    sshr v1.16b, v1.16b, #4
 ; CHECK-NEXT:    shl v1.16b, v1.16b, #4
 ; CHECK-NEXT:    shl v0.16b, v0.16b, #4
-; CHECK-NEXT:    sub v3.16b, v0.16b, v1.16b
-; CHECK-NEXT:    cmge v1.16b, v1.16b, #0
-; CHECK-NEXT:    cmge v0.16b, v0.16b, #0
-; CHECK-NEXT:    cmge v5.16b, v3.16b, #0
-; CHECK-NEXT:    cmlt v4.16b, v3.16b, #0
-; CHECK-NEXT:    cmeq v1.16b, v0.16b, v1.16b
-; CHECK-NEXT:    cmeq v0.16b, v0.16b, v5.16b
-; CHECK-NEXT:    movi v2.16b, #127
-; CHECK-NEXT:    mvn v5.16b, v4.16b
-; CHECK-NEXT:    mvn v1.16b, v1.16b
-; CHECK-NEXT:    mvn v0.16b, v0.16b
-; CHECK-NEXT:    bsl v2.16b, v4.16b, v5.16b
-; CHECK-NEXT:    and v0.16b, v1.16b, v0.16b
-; CHECK-NEXT:    bsl v0.16b, v2.16b, v3.16b
+; CHECK-NEXT:    sqsub v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    sshr v0.16b, v0.16b, #4
 ; CHECK-NEXT:    ret
   %z = call <16 x i4> @llvm.ssub.sat.v16i4(<16 x i4> %x, <16 x i4> %y)
@@ -608,22 +287,13 @@ define <16 x i4> @v16i4(<16 x i4> %x, <16 x i4> %y) nounwind {
 define <16 x i1> @v16i1(<16 x i1> %x, <16 x i1> %y) nounwind {
 ; CHECK-LABEL: v16i1:
 ; CHECK:       // %bb.0:
+; CHECK-NEXT:    shl v0.16b, v0.16b, #7
+; CHECK-NEXT:    shl v1.16b, v1.16b, #7
+; CHECK-NEXT:    sshr v0.16b, v0.16b, #7
+; CHECK-NEXT:    sshr v1.16b, v1.16b, #7
 ; CHECK-NEXT:    shl v1.16b, v1.16b, #7
 ; CHECK-NEXT:    shl v0.16b, v0.16b, #7
-; CHECK-NEXT:    sub v3.16b, v0.16b, v1.16b
-; CHECK-NEXT:    cmge v1.16b, v1.16b, #0
-; CHECK-NEXT:    cmge v0.16b, v0.16b, #0
-; CHECK-NEXT:    cmge v5.16b, v3.16b, #0
-; CHECK-NEXT:    cmlt v4.16b, v3.16b, #0
-; CHECK-NEXT:    cmeq v1.16b, v0.16b, v1.16b
-; CHECK-NEXT:    cmeq v0.16b, v0.16b, v5.16b
-; CHECK-NEXT:    movi v2.16b, #127
-; CHECK-NEXT:    mvn v5.16b, v4.16b
-; CHECK-NEXT:    mvn v1.16b, v1.16b
-; CHECK-NEXT:    mvn v0.16b, v0.16b
-; CHECK-NEXT:    bsl v2.16b, v4.16b, v5.16b
-; CHECK-NEXT:    and v0.16b, v1.16b, v0.16b
-; CHECK-NEXT:    bsl v0.16b, v2.16b, v3.16b
+; CHECK-NEXT:    sqsub v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    sshr v0.16b, v0.16b, #7
 ; CHECK-NEXT:    ret
   %z = call <16 x i1> @llvm.ssub.sat.v16i1(<16 x i1> %x, <16 x i1> %y)
@@ -633,20 +303,7 @@ define <16 x i1> @v16i1(<16 x i1> %x, <16 x i1> %y) nounwind {
 define <2 x i32> @v2i32(<2 x i32> %x, <2 x i32> %y) nounwind {
 ; CHECK-LABEL: v2i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    sub v2.2s, v0.2s, v1.2s
-; CHECK-NEXT:    cmge v1.2s, v1.2s, #0
-; CHECK-NEXT:    cmge v0.2s, v0.2s, #0
-; CHECK-NEXT:    cmge v5.2s, v2.2s, #0
-; CHECK-NEXT:    cmlt v4.2s, v2.2s, #0
-; CHECK-NEXT:    cmeq v1.2s, v0.2s, v1.2s
-; CHECK-NEXT:    cmeq v0.2s, v0.2s, v5.2s
-; CHECK-NEXT:    mvni v3.2s, #128, lsl #24
-; CHECK-NEXT:    mvn v5.8b, v4.8b
-; CHECK-NEXT:    mvn v1.8b, v1.8b
-; CHECK-NEXT:    mvn v0.8b, v0.8b
-; CHECK-NEXT:    bsl v3.8b, v4.8b, v5.8b
-; CHECK-NEXT:    and v0.8b, v1.8b, v0.8b
-; CHECK-NEXT:    bsl v0.8b, v3.8b, v2.8b
+; CHECK-NEXT:    sqsub v0.2s, v0.2s, v1.2s
 ; CHECK-NEXT:    ret
   %z = call <2 x i32> @llvm.ssub.sat.v2i32(<2 x i32> %x, <2 x i32> %y)
   ret <2 x i32> %z
@@ -655,20 +312,7 @@ define <2 x i32> @v2i32(<2 x i32> %x, <2 x i32> %y) nounwind {
 define <4 x i32> @v4i32(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; CHECK-LABEL: v4i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    sub v2.4s, v0.4s, v1.4s
-; CHECK-NEXT:    cmge v1.4s, v1.4s, #0
-; CHECK-NEXT:    cmge v0.4s, v0.4s, #0
-; CHECK-NEXT:    cmge v5.4s, v2.4s, #0
-; CHECK-NEXT:    cmlt v4.4s, v2.4s, #0
-; CHECK-NEXT:    cmeq v1.4s, v0.4s, v1.4s
-; CHECK-NEXT:    cmeq v0.4s, v0.4s, v5.4s
-; CHECK-NEXT:    mvni v3.4s, #128, lsl #24
-; CHECK-NEXT:    mvn v5.16b, v4.16b
-; CHECK-NEXT:    mvn v1.16b, v1.16b
-; CHECK-NEXT:    mvn v0.16b, v0.16b
-; CHECK-NEXT:    bsl v3.16b, v4.16b, v5.16b
-; CHECK-NEXT:    and v0.16b, v1.16b, v0.16b
-; CHECK-NEXT:    bsl v0.16b, v3.16b, v2.16b
+; CHECK-NEXT:    sqsub v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    ret
   %z = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> %x, <4 x i32> %y)
   ret <4 x i32> %z
@@ -677,34 +321,8 @@ define <4 x i32> @v4i32(<4 x i32> %x, <4 x i32> %y) nounwind {
 define <8 x i32> @v8i32(<8 x i32> %x, <8 x i32> %y) nounwind {
 ; CHECK-LABEL: v8i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    sub v4.4s, v0.4s, v2.4s
-; CHECK-NEXT:    cmlt v16.4s, v4.4s, #0
-; CHECK-NEXT:    mvni v6.4s, #128, lsl #24
-; CHECK-NEXT:    sub v7.4s, v1.4s, v3.4s
-; CHECK-NEXT:    mvn v17.16b, v16.16b
-; CHECK-NEXT:    bsl v6.16b, v16.16b, v17.16b
-; CHECK-NEXT:    cmlt v16.4s, v7.4s, #0
-; CHECK-NEXT:    mvni v5.4s, #128, lsl #24
-; CHECK-NEXT:    mvn v17.16b, v16.16b
-; CHECK-NEXT:    bsl v5.16b, v16.16b, v17.16b
-; CHECK-NEXT:    cmge v2.4s, v2.4s, #0
-; CHECK-NEXT:    cmge v0.4s, v0.4s, #0
-; CHECK-NEXT:    cmge v16.4s, v4.4s, #0
-; CHECK-NEXT:    cmge v3.4s, v3.4s, #0
-; CHECK-NEXT:    cmge v1.4s, v1.4s, #0
-; CHECK-NEXT:    cmeq v2.4s, v0.4s, v2.4s
-; CHECK-NEXT:    cmeq v0.4s, v0.4s, v16.4s
-; CHECK-NEXT:    cmge v16.4s, v7.4s, #0
-; CHECK-NEXT:    cmeq v3.4s, v1.4s, v3.4s
-; CHECK-NEXT:    cmeq v1.4s, v1.4s, v16.4s
-; CHECK-NEXT:    mvn v2.16b, v2.16b
-; CHECK-NEXT:    mvn v3.16b, v3.16b
-; CHECK-NEXT:    mvn v0.16b, v0.16b
-; CHECK-NEXT:    mvn v1.16b, v1.16b
-; CHECK-NEXT:    and v0.16b, v2.16b, v0.16b
-; CHECK-NEXT:    and v1.16b, v3.16b, v1.16b
-; CHECK-NEXT:    bsl v0.16b, v6.16b, v4.16b
-; CHECK-NEXT:    bsl v1.16b, v5.16b, v7.16b
+; CHECK-NEXT:    sqsub v0.4s, v0.4s, v2.4s
+; CHECK-NEXT:    sqsub v1.4s, v1.4s, v3.4s
 ; CHECK-NEXT:    ret
   %z = call <8 x i32> @llvm.ssub.sat.v8i32(<8 x i32> %x, <8 x i32> %y)
   ret <8 x i32> %z
@@ -713,62 +331,10 @@ define <8 x i32> @v8i32(<8 x i32> %x, <8 x i32> %y) nounwind {
 define <16 x i32> @v16i32(<16 x i32> %x, <16 x i32> %y) nounwind {
 ; CHECK-LABEL: v16i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    sub v16.4s, v0.4s, v4.4s
-; CHECK-NEXT:    cmlt v24.4s, v16.4s, #0
-; CHECK-NEXT:    mvni v18.4s, #128, lsl #24
-; CHECK-NEXT:    sub v19.4s, v1.4s, v5.4s
-; CHECK-NEXT:    mvn v25.16b, v24.16b
-; CHECK-NEXT:    bsl v18.16b, v24.16b, v25.16b
-; CHECK-NEXT:    cmlt v24.4s, v19.4s, #0
-; CHECK-NEXT:    mvni v20.4s, #128, lsl #24
-; CHECK-NEXT:    sub v21.4s, v2.4s, v6.4s
-; CHECK-NEXT:    mvn v25.16b, v24.16b
-; CHECK-NEXT:    bsl v20.16b, v24.16b, v25.16b
-; CHECK-NEXT:    cmlt v24.4s, v21.4s, #0
-; CHECK-NEXT:    mvni v22.4s, #128, lsl #24
-; CHECK-NEXT:    sub v23.4s, v3.4s, v7.4s
-; CHECK-NEXT:    mvn v25.16b, v24.16b
-; CHECK-NEXT:    bsl v22.16b, v24.16b, v25.16b
-; CHECK-NEXT:    cmlt v24.4s, v23.4s, #0
-; CHECK-NEXT:    mvni v17.4s, #128, lsl #24
-; CHECK-NEXT:    mvn v25.16b, v24.16b
-; CHECK-NEXT:    bsl v17.16b, v24.16b, v25.16b
-; CHECK-NEXT:    cmge v4.4s, v4.4s, #0
-; CHECK-NEXT:    cmge v0.4s, v0.4s, #0
-; CHECK-NEXT:    cmge v24.4s, v16.4s, #0
-; CHECK-NEXT:    cmge v5.4s, v5.4s, #0
-; CHECK-NEXT:    cmge v1.4s, v1.4s, #0
-; CHECK-NEXT:    cmeq v4.4s, v0.4s, v4.4s
-; CHECK-NEXT:    cmeq v0.4s, v0.4s, v24.4s
-; CHECK-NEXT:    cmge v24.4s, v19.4s, #0
-; CHECK-NEXT:    cmge v6.4s, v6.4s, #0
-; CHECK-NEXT:    cmge v2.4s, v2.4s, #0
-; CHECK-NEXT:    cmeq v5.4s, v1.4s, v5.4s
-; CHECK-NEXT:    cmeq v1.4s, v1.4s, v24.4s
-; CHECK-NEXT:    cmge v24.4s, v21.4s, #0
-; CHECK-NEXT:    mvn v4.16b, v4.16b
-; CHECK-NEXT:    mvn v0.16b, v0.16b
-; CHECK-NEXT:    cmge v7.4s, v7.4s, #0
-; CHECK-NEXT:    cmge v3.4s, v3.4s, #0
-; CHECK-NEXT:    cmeq v6.4s, v2.4s, v6.4s
-; CHECK-NEXT:    cmeq v2.4s, v2.4s, v24.4s
-; CHECK-NEXT:    cmge v24.4s, v23.4s, #0
-; CHECK-NEXT:    and v0.16b, v4.16b, v0.16b
-; CHECK-NEXT:    mvn v4.16b, v5.16b
-; CHECK-NEXT:    mvn v1.16b, v1.16b
-; CHECK-NEXT:    cmeq v7.4s, v3.4s, v7.4s
-; CHECK-NEXT:    cmeq v3.4s, v3.4s, v24.4s
-; CHECK-NEXT:    and v1.16b, v4.16b, v1.16b
-; CHECK-NEXT:    mvn v4.16b, v6.16b
-; CHECK-NEXT:    mvn v2.16b, v2.16b
-; CHECK-NEXT:    and v2.16b, v4.16b, v2.16b
-; CHECK-NEXT:    mvn v4.16b, v7.16b
-; CHECK-NEXT:    mvn v3.16b, v3.16b
-; CHECK-NEXT:    and v3.16b, v4.16b, v3.16b
-; CHECK-NEXT:    bsl v0.16b, v18.16b, v16.16b
-; CHECK-NEXT:    bsl v1.16b, v20.16b, v19.16b
-; CHECK-NEXT:    bsl v2.16b, v22.16b, v21.16b
-; CHECK-NEXT:    bsl v3.16b, v17.16b, v23.16b
+; CHECK-NEXT:    sqsub v0.4s, v0.4s, v4.4s
+; CHECK-NEXT:    sqsub v1.4s, v1.4s, v5.4s
+; CHECK-NEXT:    sqsub v2.4s, v2.4s, v6.4s
+; CHECK-NEXT:    sqsub v3.4s, v3.4s, v7.4s
 ; CHECK-NEXT:    ret
   %z = call <16 x i32> @llvm.ssub.sat.v16i32(<16 x i32> %x, <16 x i32> %y)
   ret <16 x i32> %z
@@ -777,21 +343,7 @@ define <16 x i32> @v16i32(<16 x i32> %x, <16 x i32> %y) nounwind {
 define <2 x i64> @v2i64(<2 x i64> %x, <2 x i64> %y) nounwind {
 ; CHECK-LABEL: v2i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    sub v2.2d, v0.2d, v1.2d
-; CHECK-NEXT:    cmge v1.2d, v1.2d, #0
-; CHECK-NEXT:    cmge v0.2d, v0.2d, #0
-; CHECK-NEXT:    cmge v5.2d, v2.2d, #0
-; CHECK-NEXT:    mov x8, #9223372036854775807
-; CHECK-NEXT:    cmlt v3.2d, v2.2d, #0
-; CHECK-NEXT:    cmeq v1.2d, v0.2d, v1.2d
-; CHECK-NEXT:    cmeq v0.2d, v0.2d, v5.2d
-; CHECK-NEXT:    dup v4.2d, x8
-; CHECK-NEXT:    mvn v5.16b, v3.16b
-; CHECK-NEXT:    mvn v1.16b, v1.16b
-; CHECK-NEXT:    mvn v0.16b, v0.16b
-; CHECK-NEXT:    bsl v4.16b, v3.16b, v5.16b
-; CHECK-NEXT:    and v0.16b, v1.16b, v0.16b
-; CHECK-NEXT:    bsl v0.16b, v4.16b, v2.16b
+; CHECK-NEXT:    sqsub v0.2d, v0.2d, v1.2d
 ; CHECK-NEXT:    ret
   %z = call <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64> %x, <2 x i64> %y)
   ret <2 x i64> %z
@@ -800,35 +352,8 @@ define <2 x i64> @v2i64(<2 x i64> %x, <2 x i64> %y) nounwind {
 define <4 x i64> @v4i64(<4 x i64> %x, <4 x i64> %y) nounwind {
 ; CHECK-LABEL: v4i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    sub v4.2d, v0.2d, v2.2d
-; CHECK-NEXT:    mov x8, #9223372036854775807
-; CHECK-NEXT:    cmlt v6.2d, v4.2d, #0
-; CHECK-NEXT:    dup v7.2d, x8
-; CHECK-NEXT:    sub v5.2d, v1.2d, v3.2d
-; CHECK-NEXT:    mvn v16.16b, v6.16b
-; CHECK-NEXT:    mov v17.16b, v7.16b
-; CHECK-NEXT:    bsl v17.16b, v6.16b, v16.16b
-; CHECK-NEXT:    cmlt v6.2d, v5.2d, #0
-; CHECK-NEXT:    mvn v16.16b, v6.16b
-; CHECK-NEXT:    bsl v7.16b, v6.16b, v16.16b
-; CHECK-NEXT:    cmge v2.2d, v2.2d, #0
-; CHECK-NEXT:    cmge v0.2d, v0.2d, #0
-; CHECK-NEXT:    cmge v6.2d, v4.2d, #0
-; CHECK-NEXT:    cmge v3.2d, v3.2d, #0
-; CHECK-NEXT:    cmge v1.2d, v1.2d, #0
-; CHECK-NEXT:    cmeq v2.2d, v0.2d, v2.2d
-; CHECK-NEXT:    cmeq v0.2d, v0.2d, v6.2d
-; CHECK-NEXT:    cmge v6.2d, v5.2d, #0
-; CHECK-NEXT:    cmeq v3.2d, v1.2d, v3.2d
-; CHECK-NEXT:    cmeq v1.2d, v1.2d, v6.2d
-; CHECK-NEXT:    mvn v2.16b, v2.16b
-; CHECK-NEXT:    mvn v3.16b, v3.16b
-; CHECK-NEXT:    mvn v0.16b, v0.16b
-; CHECK-NEXT:    mvn v1.16b, v1.16b
-; CHECK-NEXT:    and v0.16b, v2.16b, v0.16b
-; CHECK-NEXT:    and v1.16b, v3.16b, v1.16b
-; CHECK-NEXT:    bsl v0.16b, v17.16b, v4.16b
-; CHECK-NEXT:    bsl v1.16b, v7.16b, v5.16b
+; CHECK-NEXT:    sqsub v0.2d, v0.2d, v2.2d
+; CHECK-NEXT:    sqsub v1.2d, v1.2d, v3.2d
 ; CHECK-NEXT:    ret
   %z = call <4 x i64> @llvm.ssub.sat.v4i64(<4 x i64> %x, <4 x i64> %y)
   ret <4 x i64> %z
@@ -837,63 +362,10 @@ define <4 x i64> @v4i64(<4 x i64> %x, <4 x i64> %y) nounwind {
 define <8 x i64> @v8i64(<8 x i64> %x, <8 x i64> %y) nounwind {
 ; CHECK-LABEL: v8i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    sub v16.2d, v0.2d, v4.2d
-; CHECK-NEXT:    mov x8, #9223372036854775807
-; CHECK-NEXT:    sub v17.2d, v1.2d, v5.2d
-; CHECK-NEXT:    cmlt v20.2d, v16.2d, #0
-; CHECK-NEXT:    dup v21.2d, x8
-; CHECK-NEXT:    sub v18.2d, v2.2d, v6.2d
-; CHECK-NEXT:    cmlt v22.2d, v17.2d, #0
-; CHECK-NEXT:    mvn v24.16b, v20.16b
-; CHECK-NEXT:    mov v25.16b, v21.16b
-; CHECK-NEXT:    cmlt v23.2d, v18.2d, #0
-; CHECK-NEXT:    bsl v25.16b, v20.16b, v24.16b
-; CHECK-NEXT:    mvn v20.16b, v22.16b
-; CHECK-NEXT:    mov v24.16b, v21.16b
-; CHECK-NEXT:    sub v19.2d, v3.2d, v7.2d
-; CHECK-NEXT:    bsl v24.16b, v22.16b, v20.16b
-; CHECK-NEXT:    mvn v20.16b, v23.16b
-; CHECK-NEXT:    mov v22.16b, v21.16b
-; CHECK-NEXT:    bsl v22.16b, v23.16b, v20.16b
-; CHECK-NEXT:    cmlt v20.2d, v19.2d, #0
-; CHECK-NEXT:    mvn v23.16b, v20.16b
-; CHECK-NEXT:    bsl v21.16b, v20.16b, v23.16b
-; CHECK-NEXT:    cmge v4.2d, v4.2d, #0
-; CHECK-NEXT:    cmge v0.2d, v0.2d, #0
-; CHECK-NEXT:    cmge v20.2d, v16.2d, #0
-; CHECK-NEXT:    cmge v5.2d, v5.2d, #0
-; CHECK-NEXT:    cmge v1.2d, v1.2d, #0
-; CHECK-NEXT:    cmeq v4.2d, v0.2d, v4.2d
-; CHECK-NEXT:    cmeq v0.2d, v0.2d, v20.2d
-; CHECK-NEXT:    cmge v20.2d, v17.2d, #0
-; CHECK-NEXT:    cmge v6.2d, v6.2d, #0
-; CHECK-NEXT:    cmge v2.2d, v2.2d, #0
-; CHECK-NEXT:    cmeq v5.2d, v1.2d, v5.2d
-; CHECK-NEXT:    cmeq v1.2d, v1.2d, v20.2d
-; CHECK-NEXT:    cmge v20.2d, v18.2d, #0
-; CHECK-NEXT:    mvn v4.16b, v4.16b
-; CHECK-NEXT:    mvn v0.16b, v0.16b
-; CHECK-NEXT:    cmge v7.2d, v7.2d, #0
-; CHECK-NEXT:    cmge v3.2d, v3.2d, #0
-; CHECK-NEXT:    cmeq v6.2d, v2.2d, v6.2d
-; CHECK-NEXT:    cmeq v2.2d, v2.2d, v20.2d
-; CHECK-NEXT:    cmge v20.2d, v19.2d, #0
-; CHECK-NEXT:    and v0.16b, v4.16b, v0.16b
-; CHECK-NEXT:    mvn v4.16b, v5.16b
-; CHECK-NEXT:    mvn v1.16b, v1.16b
-; CHECK-NEXT:    cmeq v7.2d, v3.2d, v7.2d
-; CHECK-NEXT:    cmeq v3.2d, v3.2d, v20.2d
-; CHECK-NEXT:    and v1.16b, v4.16b, v1.16b
-; CHECK-NEXT:    mvn v4.16b, v6.16b
-; CHECK-NEXT:    mvn v2.16b, v2.16b
-; CHECK-NEXT:    and v2.16b, v4.16b, v2.16b
-; CHECK-NEXT:    mvn v4.16b, v7.16b
-; CHECK-NEXT:    mvn v3.16b, v3.16b
-; CHECK-NEXT:    and v3.16b, v4.16b, v3.16b
-; CHECK-NEXT:    bsl v0.16b, v25.16b, v16.16b
-; CHECK-NEXT:    bsl v1.16b, v24.16b, v17.16b
-; CHECK-NEXT:    bsl v2.16b, v22.16b, v18.16b
-; CHECK-NEXT:    bsl v3.16b, v21.16b, v19.16b
+; CHECK-NEXT:    sqsub v0.2d, v0.2d, v4.2d
+; CHECK-NEXT:    sqsub v1.2d, v1.2d, v5.2d
+; CHECK-NEXT:    sqsub v2.2d, v2.2d, v6.2d
+; CHECK-NEXT:    sqsub v3.2d, v3.2d, v7.2d
 ; CHECK-NEXT:    ret
   %z = call <8 x i64> @llvm.ssub.sat.v8i64(<8 x i64> %x, <8 x i64> %y)
   ret <8 x i64> %z
diff --git a/llvm/test/CodeGen/AArch64/stack-guard-reassign.ll b/llvm/test/CodeGen/AArch64/stack-guard-reassign.ll
index 632774c97..6b00eda62 100644
--- a/llvm/test/CodeGen/AArch64/stack-guard-reassign.ll
+++ b/llvm/test/CodeGen/AArch64/stack-guard-reassign.ll
@@ -3,7 +3,6 @@
 ; Verify that the offset assigned to the stack protector is at the top of the
 ; frame, covering the locals.
 ; CHECK-LABEL: fn:
-; CHECK:      sub x8, x29, #24
-; CHECK-NEXT: adrp x9, __stack_chk_guard
-; CHECK-NEXT: ldr x9, [x9, :lo12:__stack_chk_guard]
-; CHECK-NEXT: str x9, [x8]
+; CHECK:      adrp x8, __stack_chk_guard
+; CHECK-NEXT: ldr x8, [x8, :lo12:__stack_chk_guard]
+; CHECK-NEXT: stur x8, [x29, #-8]
diff --git a/llvm/test/CodeGen/AArch64/stack-guard-vaarg.ll b/llvm/test/CodeGen/AArch64/stack-guard-vaarg.ll
index 29e66d957..bc039b857 100644
--- a/llvm/test/CodeGen/AArch64/stack-guard-vaarg.ll
+++ b/llvm/test/CodeGen/AArch64/stack-guard-vaarg.ll
@@ -9,7 +9,7 @@
 ; CHECK: ldr [[GUARD:x[0-9]+]]{{.*}}:lo12:__stack_chk_guard]
 ; Make sure the canary is placed relative to the frame pointer, not
 ; the stack pointer.
-; CHECK: stur [[GUARD]], [x29, #-24]
+; CHECK: stur [[GUARD]], [x29, #-8]
 define void @test(i8* %i, ...) #0 {
 entry:
   %buf = alloca [10 x i8], align 1
diff --git a/llvm/test/CodeGen/AArch64/stack-tagging-ex-1.ll b/llvm/test/CodeGen/AArch64/stack-tagging-ex-1.ll
new file mode 100644
index 000000000..258586824
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/stack-tagging-ex-1.ll
@@ -0,0 +1,69 @@
+; RUN: opt -S -stack-tagging %s -o - | FileCheck %s
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64-arm-unknown-eabi"
+
+define  void @f() local_unnamed_addr #0 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+start:
+; CHECK-LABEL: start:
+  %a = alloca i8, i32 48, align 8
+  call void @llvm.lifetime.start.p0i8(i64 48, i8* nonnull %a) #2
+; CHECK: call void @llvm.aarch64.settag(i8* %a.tag, i64 48)
+  %b = alloca i8, i32 48, align 8
+  call void @llvm.lifetime.start.p0i8(i64 48, i8* nonnull %b) #2
+; CHECK: call void @llvm.aarch64.settag(i8* %b.tag, i64 48)
+  invoke void @g (i8 * nonnull %a, i8 * nonnull %b) to label %next0 unwind label %lpad0
+; CHECK-NOT: settag
+
+next0:
+; CHECK-LABEL: next0:
+  call void @llvm.lifetime.end.p0i8(i64 40, i8* nonnull %a)
+  call void @llvm.lifetime.end.p0i8(i64 40, i8* nonnull %b)
+  br label %exit
+; CHECK-NOT: settag
+
+lpad0:
+; CHECK-LABEL: lpad0:
+  %pad0v = landingpad { i8*, i32 } catch i8* null
+  %v = extractvalue { i8*, i32 } %pad0v, 0
+  %x = call i8* @__cxa_begin_catch(i8* %v) #2
+  invoke void @__cxa_end_catch() to label %next1 unwind label %lpad1
+; CHECK-NOT: settag
+
+next1:
+; CHECK-LABEL: next1:
+  br label %exit
+; CHECK-NOT: settag
+
+lpad1:
+; CHECK-LABEL: lpad1:
+; CHECK-DAG: call void @llvm.aarch64.settag(i8* %a, i64 48)
+; CHECK-DAG: call void @llvm.aarch64.settag(i8* %b, i64 48)
+  %pad1v = landingpad { i8*, i32 } cleanup
+  resume { i8*, i32 } %pad1v
+
+exit:
+; CHECK-LABEL: exit:
+; CHECK-DAG: call void @llvm.aarch64.settag(i8* %a, i64 48)
+; CHECK-DAG: call void @llvm.aarch64.settag(i8* %b, i64 48)
+  ret void
+; CHECK: ret void
+}
+
+declare void @g(i8 *, i8 *) #0
+
+declare dso_local i32 @__gxx_personality_v0(...)
+
+declare dso_local i8* @__cxa_begin_catch(i8*) local_unnamed_addr
+
+declare dso_local void @__cxa_end_catch() local_unnamed_addr
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.lifetime.start.p0i8(i64 immarg, i8* nocapture) #1
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.lifetime.end.p0i8(i64 immarg, i8* nocapture) #1
+
+attributes #0 = { sanitize_memtag "correctly-rounded-divide-sqrt-fp-math"="false" "denormal-fp-math"="preserve-sign" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="true" "no-jump-tables"="false" "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="true" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="generic" "target-features"="+mte,+neon,+v8.5a" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { argmemonly nounwind willreturn }
+attributes #2 = { nounwind }
diff --git a/llvm/test/CodeGen/AArch64/stack-tagging-ex-2.ll b/llvm/test/CodeGen/AArch64/stack-tagging-ex-2.ll
new file mode 100644
index 000000000..820041e06
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/stack-tagging-ex-2.ll
@@ -0,0 +1,183 @@
+; clang -target aarch64-eabi -O2 -march=armv8.5-a+memtag -fsanitize=memtag -S -emit-llvm test.cc
+; void bar() {
+;   throw 42;
+; }
+
+; void foo() {
+;   int A0;
+;   __asm volatile("" : : "r"(&A0));
+
+;   try {
+;     bar();
+;   } catch (int exc) {
+;   }
+
+;   throw 15532;
+; }
+
+; int main() {
+;   try {
+;     foo();
+;   } catch (int exc) {
+;   }
+
+;   return 0;
+; }
+
+; RUN: opt -S -stack-tagging %s -o - | FileCheck %s
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64-unknown-unknown-eabi"
+
+@_ZTIi = external dso_local constant i8*
+
+; Function Attrs: noreturn sanitize_memtag
+define dso_local void @_Z3barv() local_unnamed_addr #0 {
+entry:
+  %exception = tail call i8* @__cxa_allocate_exception(i64 4) #4
+  %0 = bitcast i8* %exception to i32*
+  store i32 42, i32* %0, align 16, !tbaa !2
+  tail call void @__cxa_throw(i8* %exception, i8* bitcast (i8** @_ZTIi to i8*), i8* null) #5
+  unreachable
+}
+
+declare dso_local i8* @__cxa_allocate_exception(i64) local_unnamed_addr
+
+declare dso_local void @__cxa_throw(i8*, i8*, i8*) local_unnamed_addr
+
+; Function Attrs: noreturn sanitize_memtag
+define dso_local void @_Z3foov() local_unnamed_addr #0 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+entry:
+  %A0 = alloca i32, align 4
+  %0 = bitcast i32* %A0 to i8*
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* nonnull %0) #4
+  call void asm sideeffect "", "r"(i32* nonnull %A0) #4, !srcloc !6
+  invoke void @_Z3barv()
+          to label %try.cont unwind label %lpad
+
+lpad:                                             ; preds = %entry
+  %1 = landingpad { i8*, i32 }
+          cleanup
+          catch i8* bitcast (i8** @_ZTIi to i8*)
+  %2 = extractvalue { i8*, i32 } %1, 1
+  %3 = call i32 @llvm.eh.typeid.for(i8* bitcast (i8** @_ZTIi to i8*)) #4
+  %matches = icmp eq i32 %2, %3
+  br i1 %matches, label %catch, label %ehcleanup
+
+catch:                                            ; preds = %lpad
+  %4 = extractvalue { i8*, i32 } %1, 0
+  %5 = call i8* @__cxa_begin_catch(i8* %4) #4
+  call void @__cxa_end_catch() #4
+  br label %try.cont
+
+try.cont:                                         ; preds = %entry, %catch
+  %exception = call i8* @__cxa_allocate_exception(i64 4) #4
+  %6 = bitcast i8* %exception to i32*
+  store i32 15532, i32* %6, align 16, !tbaa !2
+  call void @__cxa_throw(i8* %exception, i8* bitcast (i8** @_ZTIi to i8*), i8* null) #5
+  unreachable
+
+ehcleanup:                                        ; preds = %lpad
+  call void @llvm.lifetime.end.p0i8(i64 4, i8* nonnull %0) #4
+  resume { i8*, i32 } %1
+}
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.lifetime.start.p0i8(i64 immarg, i8* nocapture) #1
+
+declare dso_local i32 @__gxx_personality_v0(...)
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.eh.typeid.for(i8*) #2
+
+declare dso_local i8* @__cxa_begin_catch(i8*) local_unnamed_addr
+
+declare dso_local void @__cxa_end_catch() local_unnamed_addr
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.lifetime.end.p0i8(i64 immarg, i8* nocapture) #1
+
+; Function Attrs: norecurse sanitize_memtag
+define dso_local i32 @main() local_unnamed_addr #3 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+entry:
+; CHECK-LABEL: entry:
+  %A0.i = alloca i32, align 4
+  %0 = bitcast i32* %A0.i to i8*
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* nonnull %0) #4
+  call void asm sideeffect "", "r"(i32* nonnull %A0.i) #4, !srcloc !6
+; CHECK: call void @llvm.aarch64.settag(i8* %1, i64 16)
+; CHECK-NEXT: call void asm sideeffect
+  %exception.i6 = call i8* @__cxa_allocate_exception(i64 4) #4
+  %1 = bitcast i8* %exception.i6 to i32*
+  store i32 42, i32* %1, align 16, !tbaa !2
+  invoke void @__cxa_throw(i8* %exception.i6, i8* bitcast (i8** @_ZTIi to i8*), i8* null) #5
+          to label %.noexc7 unwind label %lpad.i
+
+.noexc7:                                          ; preds = %entry
+  unreachable
+
+lpad.i:                                           ; preds = %entry
+  %2 = landingpad { i8*, i32 }
+          cleanup
+          catch i8* bitcast (i8** @_ZTIi to i8*)
+  %3 = extractvalue { i8*, i32 } %2, 1
+  %4 = call i32 @llvm.eh.typeid.for(i8* bitcast (i8** @_ZTIi to i8*)) #4
+  %matches.i = icmp eq i32 %3, %4
+  br i1 %matches.i, label %catch.i, label %ehcleanup.i
+
+catch.i:                                          ; preds = %lpad.i
+  %5 = extractvalue { i8*, i32 } %2, 0
+  %6 = call i8* @__cxa_begin_catch(i8* %5) #4
+  call void @__cxa_end_catch() #4
+  %exception.i = call i8* @__cxa_allocate_exception(i64 4) #4
+  %7 = bitcast i8* %exception.i to i32*
+  store i32 15532, i32* %7, align 16, !tbaa !2
+  invoke void @__cxa_throw(i8* %exception.i, i8* bitcast (i8** @_ZTIi to i8*), i8* null) #5
+          to label %.noexc unwind label %lpad
+
+.noexc:                                           ; preds = %catch.i
+  unreachable
+
+ehcleanup.i:                                      ; preds = %lpad.i
+  call void @llvm.lifetime.end.p0i8(i64 4, i8* nonnull %0) #4
+  br label %lpad.body
+
+lpad:                                             ; preds = %catch.i
+  %8 = landingpad { i8*, i32 }
+          catch i8* bitcast (i8** @_ZTIi to i8*)
+  %.pre = extractvalue { i8*, i32 } %8, 1
+  br label %lpad.body
+
+lpad.body:                                        ; preds = %ehcleanup.i, %lpad
+  %.pre-phi = phi i32 [ %3, %ehcleanup.i ], [ %.pre, %lpad ]
+  %eh.lpad-body = phi { i8*, i32 } [ %2, %ehcleanup.i ], [ %8, %lpad ]
+  %matches = icmp eq i32 %.pre-phi, %4
+  br i1 %matches, label %catch, label %eh.resume
+
+catch:                                            ; preds = %lpad.body
+  %9 = extractvalue { i8*, i32 } %eh.lpad-body, 0
+  %10 = call i8* @__cxa_begin_catch(i8* %9) #4
+  call void @__cxa_end_catch() #4
+  ret i32 0
+
+eh.resume:                                        ; preds = %lpad.body
+  resume { i8*, i32 } %eh.lpad-body
+}
+
+attributes #0 = { noreturn sanitize_memtag "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="generic" "target-features"="+mte,+neon,+v8.5a" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { argmemonly nounwind willreturn }
+attributes #2 = { nounwind readnone }
+attributes #3 = { norecurse sanitize_memtag "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="generic" "target-features"="+mte,+neon,+v8.5a" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #4 = { nounwind }
+attributes #5 = { noreturn }
+
+!llvm.module.flags = !{!0}
+!llvm.ident = !{!1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{!"clang version 10.0.0 (https://github.com/llvm/llvm-project.git c38188c5fe41751fda095edde1a878b2a051ae58)"}
+!2 = !{!3, !3, i64 0}
+!3 = !{!"int", !4, i64 0}
+!4 = !{!"omnipotent char", !5, i64 0}
+!5 = !{!"Simple C++ TBAA"}
+!6 = !{i32 70}
diff --git a/llvm/test/CodeGen/AArch64/stack-tagging-initializer-merge.ll b/llvm/test/CodeGen/AArch64/stack-tagging-initializer-merge.ll
new file mode 100644
index 000000000..5c4be4354
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/stack-tagging-initializer-merge.ll
@@ -0,0 +1,308 @@
+; RUN: opt < %s -stack-tagging -S -o - | FileCheck %s
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64--linux-android"
+
+declare void @use(i8*)
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture)
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture)
+declare void @llvm.memset.p0i8.i64(i8* nocapture writeonly, i8, i64, i1 immarg)
+
+define void @OneVarNoInit() sanitize_memtag {
+entry:
+  %x = alloca i32, align 4
+  %0 = bitcast i32* %x to i8*
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* nonnull %0)
+  call void @use(i8* nonnull %0)
+  call void @llvm.lifetime.end.p0i8(i64 4, i8* nonnull %0)
+  ret void
+}
+
+; CHECK-LABEL: define void @OneVarNoInit(
+; CHECK-DAG:  [[X:%.*]] = alloca { i32, [12 x i8] }, align 16
+; CHECK-DAG:  [[TX:%.*]] = call { i32, [12 x i8] }* @llvm.aarch64.tagp.{{.*}}({ i32, [12 x i8] }* [[X]], {{.*}}, i64 0)
+; CHECK-DAG:  [[TX32:%.*]] = bitcast { i32, [12 x i8] }* [[TX]] to i32*
+; CHECK-DAG:  [[TX8:%.*]] = bitcast i32* [[TX32]] to i8*
+; CHECK-DAG:  call void @llvm.lifetime.start.p0i8(i64 4, i8* nonnull [[TX8]])
+; CHECK-DAG:  call void @llvm.aarch64.settag(i8* [[TX8]], i64 16)
+; CHECK-DAG:  call void @use(i8* nonnull [[TX8]])
+; CHECK-DAG:  call void @llvm.lifetime.end.p0i8(i64 4, i8* nonnull [[TX8]])
+
+define void @OneVarInitConst() sanitize_memtag {
+entry:
+  %x = alloca i32, align 4
+  %0 = bitcast i32* %x to i8*
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* nonnull %0)
+  store i32 42, i32* %x, align 4
+  call void @use(i8* nonnull %0)
+  call void @llvm.lifetime.end.p0i8(i64 4, i8* nonnull %0)
+  ret void
+}
+
+; CHECK-LABEL: define void @OneVarInitConst(
+; CHECK:  [[TX:%.*]] = call { i32, [12 x i8] }* @llvm.aarch64.tagp
+; CHECK:  [[TX32:%.*]] = bitcast { i32, [12 x i8] }* [[TX]] to i32*
+; CHECK:  [[TX8:%.*]] = bitcast i32* [[TX32]] to i8*
+; CHECK-NOT: aarch64.settag
+; CHECK:  call void @llvm.aarch64.stgp(i8* [[TX8]], i64 42, i64 0)
+; Untagging before lifetime.end:
+; CHECK:  call void @llvm.aarch64.settag(
+; CHECK-NOT: aarch64.settag
+; CHECK:  ret void
+
+define void @ArrayInitConst() sanitize_memtag {
+entry:
+  %x = alloca i32, i32 16, align 4
+  %0 = bitcast i32* %x to i8*
+  call void @llvm.lifetime.start.p0i8(i64 64, i8* nonnull %0)
+  store i32 42, i32* %x, align 4
+  call void @use(i8* nonnull %0)
+  call void @llvm.lifetime.end.p0i8(i64 64, i8* nonnull %0)
+  ret void
+}
+
+; CHECK-LABEL: define void @ArrayInitConst(
+; CHECK:  [[TX:%.*]] = call i32* @llvm.aarch64.tagp.
+; CHECK:  [[TX8:%.*]] = bitcast i32* [[TX]] to i8*
+; CHECK:  call void @llvm.aarch64.stgp(i8* [[TX8]], i64 42, i64 0)
+; CHECK:  [[TX8_16:%.*]] = getelementptr i8, i8* [[TX8]], i32 16
+; CHECK:  call void @llvm.aarch64.settag.zero(i8* [[TX8_16]], i64 48)
+; CHECK:  ret void
+
+define void @ArrayInitConst2() sanitize_memtag {
+entry:
+  %x = alloca i32, i32 16, align 4
+  %0 = bitcast i32* %x to i8*
+  call void @llvm.lifetime.start.p0i8(i64 64, i8* nonnull %0)
+  store i32 42, i32* %x, align 4
+  %1 = getelementptr i32, i32* %x, i32 1
+  store i32 43, i32* %1, align 4
+  %2 = getelementptr i32, i32* %x, i32 2
+  %3 = bitcast i32* %2 to i64*
+  store i64 -1, i64* %3, align 4
+  call void @use(i8* nonnull %0)
+  call void @llvm.lifetime.end.p0i8(i64 64, i8* nonnull %0)
+  ret void
+}
+
+; CHECK-LABEL: define void @ArrayInitConst2(
+; CHECK:  [[TX:%.*]] = call i32* @llvm.aarch64.tagp.
+; CHECK:  [[TX8:%.*]] = bitcast i32* [[TX]] to i8*
+; CHECK:  call void @llvm.aarch64.stgp(i8* [[TX8]], i64 184683593770, i64 -1)
+; CHECK:  [[TX8_16:%.*]] = getelementptr i8, i8* [[TX8]], i32 16
+; CHECK:  call void @llvm.aarch64.settag.zero(i8* [[TX8_16]], i64 48)
+; CHECK:  ret void
+
+define void @ArrayInitConstSplit() sanitize_memtag {
+entry:
+  %x = alloca i32, i32 16, align 4
+  %0 = bitcast i32* %x to i8*
+  call void @llvm.lifetime.start.p0i8(i64 64, i8* nonnull %0)
+  %1 = getelementptr i32, i32* %x, i32 1
+  %2 = bitcast i32* %1 to i64*
+  store i64 -1, i64* %2, align 4
+  call void @use(i8* nonnull %0)
+  call void @llvm.lifetime.end.p0i8(i64 64, i8* nonnull %0)
+  ret void
+}
+
+; CHECK-LABEL: define void @ArrayInitConstSplit(
+; CHECK:  [[TX:%.*]] = call i32* @llvm.aarch64.tagp.
+; CHECK:  [[TX8:%.*]] = bitcast i32* [[TX]] to i8*
+; CHECK:  call void @llvm.aarch64.stgp(i8* [[TX8]], i64 -4294967296, i64 4294967295)
+; CHECK:  ret void
+
+define void @ArrayInitConstWithHoles() sanitize_memtag {
+entry:
+  %x = alloca i32, i32 32, align 4
+  %0 = bitcast i32* %x to i8*
+  call void @llvm.lifetime.start.p0i8(i64 128, i8* nonnull %0)
+  %1 = getelementptr i32, i32* %x, i32 5
+  store i32 42, i32* %1, align 4
+  %2 = getelementptr i32, i32* %x, i32 14
+  store i32 43, i32* %2, align 4
+  call void @use(i8* nonnull %0)
+  call void @llvm.lifetime.end.p0i8(i64 128, i8* nonnull %0)
+  ret void
+}
+
+; CHECK-LABEL: define void @ArrayInitConstWithHoles(
+; CHECK:  [[TX:%.*]] = call i32* @llvm.aarch64.tagp.
+; CHECK:  [[TX8:%.*]] = bitcast i32* [[TX]] to i8*
+; CHECK:  call void @llvm.aarch64.settag.zero(i8* [[TX8]], i64 16)
+; CHECK:  [[TX8_16:%.*]] = getelementptr i8, i8* %0, i32 16
+; CHECK:  call void @llvm.aarch64.stgp(i8* [[TX8_16]], i64 180388626432, i64 0)
+; CHECK:  [[TX8_32:%.*]] = getelementptr i8, i8* %0, i32 32
+; CHECK:  call void @llvm.aarch64.settag.zero(i8* [[TX8_32]], i64 16)
+; CHECK:  [[TX8_48:%.*]] = getelementptr i8, i8* %0, i32 48
+; CHECK:  call void @llvm.aarch64.stgp(i8* [[TX8_48]], i64 0, i64 43)
+; CHECK:  [[TX8_64:%.*]] = getelementptr i8, i8* %0, i32 64
+; CHECK:  call void @llvm.aarch64.settag.zero(i8* [[TX8_64]], i64 64)
+; CHECK:  ret void
+
+define void @InitNonConst(i32 %v) sanitize_memtag {
+entry:
+  %x = alloca i32, align 4
+  %0 = bitcast i32* %x to i8*
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* nonnull %0)
+  store i32 %v, i32* %x, align 4
+  call void @use(i8* nonnull %0)
+  call void @llvm.lifetime.end.p0i8(i64 4, i8* nonnull %0)
+  ret void
+}
+
+; CHECK-LABEL: define void @InitNonConst(
+; CHECK:  [[TX:%.*]] = call { i32, [12 x i8] }* @llvm.aarch64.tagp
+; CHECK:  [[TX32:%.*]] = bitcast { i32, [12 x i8] }* [[TX]] to i32*
+; CHECK:  [[TX8:%.*]] = bitcast i32* [[TX32]] to i8*
+; CHECK:  [[V:%.*]] = zext i32 %v to i64
+; CHECK:  call void @llvm.aarch64.stgp(i8* [[TX8]], i64 [[V]], i64 0)
+; CHECK:  ret void
+
+define void @InitNonConst2(i32 %v, i32 %w) sanitize_memtag {
+entry:
+  %x = alloca i32, i32 4, align 4
+  %0 = bitcast i32* %x to i8*
+  call void @llvm.lifetime.start.p0i8(i64 16, i8* nonnull %0)
+  store i32 %v, i32* %x, align 4
+  %1 = getelementptr i32, i32* %x, i32 1
+  store i32 %w, i32* %1, align 4
+  call void @use(i8* nonnull %0)
+  call void @llvm.lifetime.end.p0i8(i64 16, i8* nonnull %0)
+  ret void
+}
+
+; CHECK-LABEL: define void @InitNonConst2(
+; CHECK:  [[TX:%.*]] = call i32* @llvm.aarch64.tagp
+; CHECK:  [[TX8:%.*]] = bitcast i32* [[TX]] to i8*
+; CHECK:  [[V:%.*]] = zext i32 %v to i64
+; CHECK:  [[W:%.*]] = zext i32 %w to i64
+; CHECK:  [[WS:%.*]] = shl i64 [[W]], 32
+; CHECK:  [[VW:%.*]] = or i64 [[V]], [[WS]]
+; CHECK:  call void @llvm.aarch64.stgp(i8* [[TX8]], i64 [[VW]], i64 0)
+; CHECK:  ret void
+
+define void @InitVector() sanitize_memtag {
+entry:
+  %x = alloca i32, i32 4, align 4
+  %0 = bitcast i32* %x to i8*
+  call void @llvm.lifetime.start.p0i8(i64 16, i8* nonnull %0)
+  %1 = bitcast i32* %x to <2 x i32>*
+  store <2 x i32> <i32 1, i32 2>, <2 x i32>* %1, align 4
+  call void @use(i8* nonnull %0)
+  call void @llvm.lifetime.end.p0i8(i64 16, i8* nonnull %0)
+  ret void
+}
+
+; CHECK-LABEL: define void @InitVector(
+; CHECK:  [[TX:%.*]] = call i32* @llvm.aarch64.tagp
+; CHECK:  [[TX8:%.*]] = bitcast i32* [[TX]] to i8*
+; CHECK:  call void @llvm.aarch64.stgp(i8* [[TX8]], i64 bitcast (<2 x i32> <i32 1, i32 2> to i64), i64 0)
+; CHECK:  ret void
+
+define void @InitVectorPtr(i32* %p) sanitize_memtag {
+entry:
+  %s = alloca <4 x i32*>, align 8
+  %v0 = insertelement <4 x i32*> undef, i32* %p, i32 0
+  %v1 = shufflevector <4 x i32*> %v0, <4 x i32*> undef, <4 x i32> zeroinitializer
+  store <4 x i32*> %v1, <4 x i32*>* %s
+  %0 = bitcast <4 x i32*>* %s to i8*
+  call void @use(i8* nonnull %0)
+  ret void
+}
+
+; CHECK-LABEL: define void @InitVectorPtr(
+; CHECK:  call <4 x i32*>* @llvm.aarch64.tagp
+; CHECK:  [[V1:%.*]] = shufflevector
+; CHECK:  [[V2:%.*]] = ptrtoint <4 x i32*> [[V1]] to <4 x i64>
+; CHECK:  [[V3:%.*]] = bitcast <4 x i64> [[V2]] to i256
+; CHECK:  [[A1:%.*]] = trunc i256 [[V3]] to i64
+; CHECK:  [[A2_:%.*]] = lshr i256 [[V3]], 64
+; CHECK:  [[A2:%.*]] = trunc i256 [[A2_]] to i64
+; CHECK:  [[A3_:%.*]] = lshr i256 [[V3]], 128
+; CHECK:  [[A3:%.*]] = trunc i256 [[A3_]] to i64
+; CHECK:  [[A4_:%.*]] = lshr i256 [[V3]], 192
+; CHECK:  [[A4:%.*]] = trunc i256 [[A4_]] to i64
+; CHECK:  call void @llvm.aarch64.stgp({{.*}}, i64 [[A1]], i64 [[A2]])
+; CHECK:  call void @llvm.aarch64.stgp({{.*}}, i64 [[A3]], i64 [[A4]])
+; CHECK:  ret void
+
+define void @InitVectorSplit() sanitize_memtag {
+entry:
+  %x = alloca i32, i32 4, align 4
+  %0 = bitcast i32* %x to i8*
+  call void @llvm.lifetime.start.p0i8(i64 16, i8* nonnull %0)
+  %1 = getelementptr i32, i32* %x, i32 1
+  %2 = bitcast i32* %1 to <2 x i32>*
+  store <2 x i32> <i32 1, i32 2>, <2 x i32>* %2, align 4
+  call void @use(i8* nonnull %0)
+  call void @llvm.lifetime.end.p0i8(i64 16, i8* nonnull %0)
+  ret void
+}
+
+; CHECK-LABEL: define void @InitVectorSplit(
+; CHECK:  [[TX:%.*]] = call i32* @llvm.aarch64.tagp
+; CHECK:  [[TX8:%.*]] = bitcast i32* [[TX]] to i8*
+; CHECK:  call void @llvm.aarch64.stgp(i8* [[TX8]], i64 shl (i64 bitcast (<2 x i32> <i32 1, i32 2> to i64), i64 32), i64 lshr (i64 bitcast (<2 x i32> <i32 1, i32 2> to i64), i64 32))
+; CHECK:  ret void
+
+define void @MemSetZero() sanitize_memtag {
+entry:
+  %x = alloca i32, i32 8, align 16
+  %0 = bitcast i32* %x to i8*
+  call void @llvm.memset.p0i8.i64(i8* nonnull align 16 %0, i8 0, i64 32, i1 false)
+  call void @use(i8* nonnull %0)
+  ret void
+}
+
+; CHECK-LABEL: define void @MemSetZero(
+; CHECK:  [[TX:%.*]] = call i32* @llvm.aarch64.tagp
+; CHECK:  [[TX8:%.*]] = bitcast i32* [[TX]] to i8*
+; CHECK:  call void @llvm.aarch64.settag.zero(i8* [[TX8]], i64 32)
+; CHECK:  ret void
+
+
+define void @MemSetNonZero() sanitize_memtag {
+entry:
+  %x = alloca i32, i32 8, align 16
+  %0 = bitcast i32* %x to i8*
+  call void @llvm.memset.p0i8.i64(i8* nonnull align 16 %0, i8 42, i64 32, i1 false)
+  call void @use(i8* nonnull %0)
+  ret void
+}
+
+; CHECK-LABEL: define void @MemSetNonZero(
+; CHECK:  call void @llvm.aarch64.stgp(i8* {{.*}}, i64 3038287259199220266, i64 3038287259199220266)
+; CHECK:  call void @llvm.aarch64.stgp(i8* {{.*}}, i64 3038287259199220266, i64 3038287259199220266)
+; CHECK:  ret void
+
+
+define void @MemSetNonZero2() sanitize_memtag {
+entry:
+  %x = alloca [32 x i8], align 16
+  %0 = getelementptr inbounds [32 x i8], [32 x i8]* %x, i64 0, i64 2
+  call void @llvm.memset.p0i8.i64(i8* nonnull %0, i8 42, i64 28, i1 false)
+  call void @use(i8* nonnull %0)
+  ret void
+}
+
+; CHECK-LABEL: define void @MemSetNonZero2(
+; CHECK:  call void @llvm.aarch64.stgp(i8* {{.*}}, i64 3038287259199209472, i64 3038287259199220266)
+; CHECK:  call void @llvm.aarch64.stgp(i8* {{.*}}, i64 3038287259199220266, i64 46360584399402)
+; CHECK:  ret void
+
+define void @MemSetNonZero3() sanitize_memtag {
+entry:
+  %x = alloca [32 x i8], align 16
+  %0 = getelementptr inbounds [32 x i8], [32 x i8]* %x, i64 0, i64 2
+  call void @llvm.memset.p0i8.i64(i8* nonnull %0, i8 42, i64 4, i1 false)
+  %1 = getelementptr inbounds [32 x i8], [32 x i8]* %x, i64 0, i64 24
+  call void @llvm.memset.p0i8.i64(i8* nonnull %1, i8 42, i64 8, i1 false)
+  call void @use(i8* nonnull %0)
+  ret void
+}
+
+; CHECK-LABEL: define void @MemSetNonZero3(
+; CHECK:  call void @llvm.aarch64.stgp(i8* {{.*}}, i64 46360584388608, i64 0)
+; CHECK:  call void @llvm.aarch64.stgp(i8* {{.*}}, i64 0, i64 3038287259199220266)
+; CHECK:  ret void
diff --git a/llvm/test/CodeGen/AArch64/stack-tagging-unchecked-ld-st.ll b/llvm/test/CodeGen/AArch64/stack-tagging-unchecked-ld-st.ll
new file mode 100644
index 000000000..200837dab
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/stack-tagging-unchecked-ld-st.ll
@@ -0,0 +1,260 @@
+; RUN: llc < %s -mtriple=aarch64 -mattr=+mte | FileCheck %s --check-prefixes=DEFAULT,COMMON
+; RUN: llc < %s -mtriple=aarch64 -mattr=+mte -stack-tagging-unchecked-ld-st=never | FileCheck %s --check-prefixes=NEVER,COMMON
+; RUN: llc < %s -mtriple=aarch64 -mattr=+mte -stack-tagging-unchecked-ld-st=always | FileCheck %s --check-prefixes=ALWAYS,COMMON
+
+declare void @use8(i8*)
+declare void @use16(i16*)
+declare void @use32(i32*)
+declare void @use64(i64*)
+declare void @use2x64([2 x i64]*)
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture)
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture)
+
+define i64 @CallLd64() sanitize_memtag {
+entry:
+  %x = alloca i64, align 4
+  call void @use64(i64* %x)
+  %a = load i64, i64* %x
+  ret i64 %a
+}
+
+; COMMON:  CallLd64:
+; COMMON:  bl  use64
+
+; ALWAYS:  ldr x0, [sp]
+; DEFAULT: ldr x0, [sp]
+; NEVER:   ldr x0, [x{{.*}}]
+
+; COMMON:  ret
+
+
+define i32 @CallLd32() sanitize_memtag {
+entry:
+  %x = alloca i32, align 4
+  call void @use32(i32* %x)
+  %a = load i32, i32* %x
+  ret i32 %a
+}
+
+; COMMON:  CallLd32:
+; COMMON:  bl  use32
+
+; ALWAYS:  ldr w0, [sp]
+; DEFAULT: ldr w0, [sp]
+; NEVER:   ldr w0, [x{{.*}}]
+
+; COMMON:  ret
+
+
+define i16 @CallLd16() sanitize_memtag {
+entry:
+  %x = alloca i16, align 4
+  call void @use16(i16* %x)
+  %a = load i16, i16* %x
+  ret i16 %a
+}
+
+; COMMON:  CallLd16:
+; COMMON:  bl  use16
+
+; ALWAYS:  ldrh w0, [sp]
+; DEFAULT: ldrh w0, [sp]
+; NEVER:   ldrh w0, [x{{.*}}]
+
+; COMMON:  ret
+
+
+define i8 @CallLd8() sanitize_memtag {
+entry:
+  %x = alloca i8, align 4
+  call void @use8(i8* %x)
+  %a = load i8, i8* %x
+  ret i8 %a
+}
+
+; COMMON:  CallLd8:
+; COMMON:  bl  use8
+
+; ALWAYS:  ldrb w0, [sp]
+; DEFAULT: ldrb w0, [sp]
+; NEVER:   ldrb w0, [x{{.*}}]
+
+; COMMON:  ret
+
+
+define void @CallSt64Call() sanitize_memtag {
+entry:
+  %x = alloca i64, align 4
+  call void @use64(i64* %x)
+  store i64 42, i64* %x
+  call void @use64(i64* %x)
+  ret void
+}
+
+; COMMON:  CallSt64Call:
+; COMMON:  bl  use64
+
+; ALWAYS:  str x{{.*}}, [sp]
+; DEFAULT: str x{{.*}}, [sp]
+; NEVER:   str x{{.*}}, [x{{.*}}]
+
+; COMMON:  bl  use64
+; COMMON:  ret
+
+
+define void @CallSt32Call() sanitize_memtag {
+entry:
+  %x = alloca i32, align 4
+  call void @use32(i32* %x)
+  store i32 42, i32* %x
+  call void @use32(i32* %x)
+  ret void
+}
+
+; COMMON:  CallSt32Call:
+; COMMON:  bl  use32
+
+; ALWAYS:  str w{{.*}}, [sp]
+; DEFAULT: str w{{.*}}, [sp]
+; NEVER:   str w{{.*}}, [x{{.*}}]
+
+; COMMON:  bl  use32
+; COMMON:  ret
+
+
+define void @CallSt16Call() sanitize_memtag {
+entry:
+  %x = alloca i16, align 4
+  call void @use16(i16* %x)
+  store i16 42, i16* %x
+  call void @use16(i16* %x)
+  ret void
+}
+
+
+; COMMON:  CallSt16Call:
+; COMMON:  bl  use16
+
+; ALWAYS:  strh w{{.*}}, [sp]
+; DEFAULT: strh w{{.*}}, [sp]
+; NEVER:   strh w{{.*}}, [x{{.*}}]
+
+; COMMON:  bl  use16
+; COMMON:  ret
+
+
+define void @CallSt8Call() sanitize_memtag {
+entry:
+  %x = alloca i8, align 4
+  call void @use8(i8* %x)
+  store i8 42, i8* %x
+  call void @use8(i8* %x)
+  ret void
+}
+
+; COMMON:  CallSt8Call:
+; COMMON:  bl  use8
+
+; ALWAYS:  strb w{{.*}}, [sp]
+; DEFAULT: strb w{{.*}}, [sp]
+; NEVER:   strb w{{.*}}, [x{{.*}}]
+
+; COMMON:  bl  use8
+; COMMON:  ret
+
+
+define void @CallStPair(i64 %z) sanitize_memtag {
+entry:
+  %x = alloca [2 x i64], align 8
+  call void @use2x64([2 x i64]* %x)
+  %x0 = getelementptr inbounds [2 x i64], [2 x i64]* %x, i64 0, i64 0
+  store i64 %z, i64* %x0, align 8
+  %x1 = getelementptr inbounds [2 x i64], [2 x i64]* %x, i64 0, i64 1
+  store i64 %z, i64* %x1, align 8
+  call void @use2x64([2 x i64]* %x)
+  ret void
+}
+
+; COMMON:  CallStPair:
+; COMMON:  bl  use2x64
+
+; ALWAYS:  stp {{.*}}, [sp]
+; DEFAULT: stp {{.*}}, [sp]
+; NEVER:   stp {{.*}}, [x{{.*}}]
+
+; COMMON:  bl  use2x64
+; COMMON:  ret
+
+; One of the two allocas will end up out of range of ldrb [sp].
+define dso_local i8 @LargeFrame() sanitize_memtag {
+entry:
+  %x = alloca [4096 x i8], align 4
+  %y = alloca [4096 x i8], align 4
+  %0 = getelementptr inbounds [4096 x i8], [4096 x i8]* %x, i64 0, i64 0
+  %1 = getelementptr inbounds [4096 x i8], [4096 x i8]* %y, i64 0, i64 0
+  call void @use8(i8* %0)
+  call void @use8(i8* %1)
+  %2 = load i8, i8* %0, align 4
+  %3 = load i8, i8* %1, align 4
+  %add = add i8 %3, %2
+  ret i8 %add
+}
+
+; COMMON: LargeFrame:
+; COMMON: bl use8
+; COMMON: bl use8
+
+; NEVER:  ldrb [[A:w.*]], [x{{.*}}]
+; NEVER:  ldrb [[B:w.*]], [x{{.*}}]
+
+; DEFAULT:  ldrb [[A:w.*]], [x{{.*}}]
+; DEFAULT:  ldrb [[B:w.*]], [x{{.*}}]
+
+; ALWAYS: ldg [[PA:x.*]], [x{{.*}}]
+; ALWAYS: ldrb [[B:w.*]], [sp]
+; ALWAYS: ldrb [[A:w.*]], {{\[}}[[PA]]{{\]}}
+
+; COMMON: add w0, [[B]], [[A]]
+; COMMON: ret
+
+; One of these allocas is closer to FP than to SP, and within 256 bytes
+; of the former (see hardcoded limit in resolveFrameOffsetReference).
+; It could be lowered to an FP-relative load, but not when doing an
+; unchecked access to tagged memory!
+define i8 @FPOffset() "frame-pointer"="all" sanitize_memtag {
+  %x = alloca [200 x i8], align 4
+  %y = alloca [200 x i8], align 4
+  %z = alloca [200 x i8], align 4
+  %x0 = getelementptr inbounds [200 x i8], [200 x i8]* %x, i64 0, i64 0
+  %y0 = getelementptr inbounds [200 x i8], [200 x i8]* %y, i64 0, i64 0
+  %z0 = getelementptr inbounds [200 x i8], [200 x i8]* %z, i64 0, i64 0
+  call void @use8(i8* %x0)
+  call void @use8(i8* %y0)
+  call void @use8(i8* %z0)
+  %x1 = load i8, i8* %x0, align 4
+  %y1 = load i8, i8* %y0, align 4
+  %z1 = load i8, i8* %z0, align 4
+  %a = add i8 %x1, %y1
+  %b = add i8 %a, %z1
+  ret i8 %b
+}
+
+; COMMON: FPOffset:
+; COMMON: bl use8
+; COMMON: bl use8
+; COMMON: bl use8
+
+; All three loads are SP-based.
+; ALWAYS-DAG: ldrb  w{{.*}}, [sp, #416]
+; ALWAYS-DAG: ldrb  w{{.*}}, [sp, #208]
+; ALWAYS-DAG: ldrb  w{{.*}}, [sp]
+
+; DEFAULT-DAG: ldrb  w{{.*}}, [sp, #416]
+; DEFAULT-DAG: ldrb  w{{.*}}, [sp, #208]
+; DEFAULT-DAG: ldrb  w{{.*}}, [sp]
+
+; NEVER-DAG: ldrb  w{{.*}}, [x{{.*}}]
+; NEVER-DAG: ldrb  w{{.*}}, [x{{.*}}]
+; NEVER-DAG: ldrb  w{{.*}}, [x{{.*}}]
+
+; COMMON: ret
diff --git a/llvm/test/CodeGen/AArch64/stack-tagging-untag-placement.ll b/llvm/test/CodeGen/AArch64/stack-tagging-untag-placement.ll
new file mode 100644
index 000000000..3c0dc3df9
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/stack-tagging-untag-placement.ll
@@ -0,0 +1,82 @@
+;; RUN:  opt -S -stack-tagging %s -o - | FileCheck %s
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64-arm-unknown-eabi"
+
+define void @f() local_unnamed_addr #0  {
+S0:
+; CHECK-LABEL: S0:
+; CHECK: %basetag = call i8* @llvm.aarch64.irg.sp(i64 0)
+  %v = alloca i8, i32 48, align 8
+; CHECK: %v.tag = call i8* @llvm.aarch64.tagp.p0i8(i8* %v, i8* %basetag, i64 0)
+  %w = alloca i8, i32 48, align 16
+; CHECK: %w.tag = call i8* @llvm.aarch64.tagp.p0i8(i8* %w, i8* %basetag, i64 1)
+
+  %t0 = call i32 @g0() #1
+  %b0 = icmp eq i32 %t0, 0
+  br i1 %b0, label %S1, label %exit3
+
+S1:
+; CHECK-LABEL: S1:
+  call void @llvm.lifetime.start.p0i8(i64 48, i8 * nonnull %v) #1
+; CHECK: call void @llvm.aarch64.settag(i8* %v.tag, i64 48)
+  call void @llvm.lifetime.start.p0i8(i64 48, i8 * nonnull %w) #1
+; CHECK: call void @llvm.aarch64.settag(i8* %w.tag, i64 48)
+  %t1 = call i32 @g1(i8 * nonnull %v, i8 * nonnull %w) #1
+; CHECK: call i32 @g1
+; CHECK-NOT: settag{{.*}}%v
+; CHECK: call void @llvm.aarch64.settag(i8* %w, i64 48)
+; CHECK-NOT: settag{{.*}}%v
+  call void @llvm.lifetime.end.p0i8(i64 48, i8 * nonnull %w) #1
+; CHECK: call void @llvm.lifetime.end.p0i8(i64 48, i8* nonnull %w.tag)
+  %b1 = icmp eq i32 %t1, 0
+  br i1 %b1, label %S2, label %S3
+; CHECK-NOT: settag
+
+S2:
+; CHECK-LABEL: S2:
+  call void @z0() #1
+  br label %exit1
+; CHECK-NOT: settag
+
+S3:
+; CHECK-LABEL: S3:
+  call void @llvm.lifetime.end.p0i8(i64 48, i8 * nonnull %v) #1
+  tail call void @z1() #1
+  br label %exit2
+; CHECK-NOT: settag
+
+exit1:
+; CHECK-LABEL: exit1:
+; CHECK: call void @llvm.aarch64.settag(i8* %v, i64 48)
+  ret void
+
+exit2:
+; CHECK-LABEL: exit2:
+; CHECK: call void @llvm.aarch64.settag(i8* %v, i64 48)
+  ret void
+
+exit3:
+; CHECK-LABEL: exit3:
+  call void @z2() #1
+; CHECK-NOT: settag
+  ret void
+; CHECK:  ret void
+}
+
+declare i32 @g0() #0
+
+declare i32 @g1(i8 *, i8 *) #0
+
+declare void @z0() #0
+
+declare void @z1() #0
+
+declare void @z2() #0
+
+declare void @llvm.lifetime.start.p0i8(i64 immarg, i8 * nocapture) #1
+
+declare void @llvm.lifetime.end.p0i8(i64 immarg, i8 * nocapture) #1
+
+attributes #0 = { sanitize_memtag "correctly-rounded-divide-sqrt-fp-math"="false" "denormal-fp-math"="preserve-sign" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="true" "no-jump-tables"="false" "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="true" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="generic" "target-features"="+mte,+neon,+v8.5a" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind }
+
diff --git a/llvm/test/CodeGen/AArch64/stack_guard_remat.ll b/llvm/test/CodeGen/AArch64/stack_guard_remat.ll
index 3a87a3731..8a6ac9ba3 100644
--- a/llvm/test/CodeGen/AArch64/stack_guard_remat.ll
+++ b/llvm/test/CodeGen/AArch64/stack_guard_remat.ll
@@ -53,4 +53,4 @@ declare void @foo3(i32*)
 ; Function Attrs: nounwind
 declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture)
 
-attributes #0 = { nounwind sspstrong "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind sspstrong "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/AArch64/stackmap.ll b/llvm/test/CodeGen/AArch64/stackmap.ll
new file mode 100644
index 000000000..8a58f9e6b
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/stackmap.ll
@@ -0,0 +1,492 @@
+; RUN: llc < %s -mtriple=arm64-linux-gnu | FileCheck %s
+;
+; Note: Print verbose stackmaps using -debug-only=stackmaps.
+
+; CHECK-LABEL:  .section  .llvm_stackmaps
+; CHECK-NEXT:  __LLVM_StackMaps:
+; Header
+; CHECK-NEXT:   .byte 3
+; CHECK-NEXT:   .byte 0
+; CHECK-NEXT:   .hword 0
+; Num Functions
+; CHECK-NEXT:   .word 14
+; Num LargeConstants
+; CHECK-NEXT:   .word 3
+; Num Callsites
+; CHECK-NEXT:   .word 18
+
+; Functions and stack size
+; CHECK-NEXT:   .xword constantargs
+; CHECK-NEXT:   .xword 16
+; CHECK-NEXT:   .xword 1
+; CHECK-NEXT:   .xword osrinline
+; CHECK-NEXT:   .xword 32
+; CHECK-NEXT:   .xword 1
+; CHECK-NEXT:   .xword osrcold
+; CHECK-NEXT:   .xword 16
+; CHECK-NEXT:   .xword 1
+; CHECK-NEXT:   .xword propertyRead
+; CHECK-NEXT:   .xword 16
+; CHECK-NEXT:   .xword 1
+; CHECK-NEXT:   .xword propertyWrite
+; CHECK-NEXT:   .xword 16
+; CHECK-NEXT:   .xword 1
+; CHECK-NEXT:   .xword jsVoidCall
+; CHECK-NEXT:   .xword 16
+; CHECK-NEXT:   .xword 1
+; CHECK-NEXT:   .xword jsIntCall
+; CHECK-NEXT:   .xword 16
+; CHECK-NEXT:   .xword 1
+; CHECK-NEXT:   .xword spilledValue
+; CHECK-NEXT:   .xword 144
+; CHECK-NEXT:   .xword 1
+; CHECK-NEXT:   .xword spilledStackMapValue
+; CHECK-NEXT:   .xword 128
+; CHECK-NEXT:   .xword 1
+; CHECK-NEXT:   .xword liveConstant
+; CHECK-NEXT:   .xword 16
+; CHECK-NEXT:   .xword 1
+; CHECK-NEXT:   .xword directFrameIdx
+; CHECK-NEXT:   .xword 64
+; CHECK-NEXT:   .xword 2
+; CHECK-NEXT:   .xword longid
+; CHECK-NEXT:   .xword 16
+; CHECK-NEXT:   .xword 4
+; CHECK-NEXT:   .xword clobberLR
+; CHECK-NEXT:   .xword 112
+; CHECK-NEXT:   .xword 1
+; CHECK-NEXT:   .xword needsStackRealignment
+; CHECK-NEXT:   .xword -1
+; CHECK-NEXT:   .xword 1
+
+; Large Constants
+; CHECK-NEXT:   .xword   2147483648
+; CHECK-NEXT:   .xword   4294967295
+; CHECK-NEXT:   .xword   4294967296
+
+; Callsites
+; Constant arguments
+;
+; CHECK-NEXT:   .xword   1
+; CHECK-NEXT:   .word   .L{{.*}}-constantargs
+; CHECK-NEXT:   .hword  0
+; CHECK-NEXT:   .hword  12
+; SmallConstant
+; CHECK-NEXT:   .byte   4
+; CHECK-NEXT:   .byte   0
+; CHECK-NEXT:   .hword  8
+; CHECK-NEXT:   .hword  0
+; CHECK-NEXT:   .hword  0
+; CHECK-NEXT:   .word   -1
+; SmallConstant
+; CHECK-NEXT:   .byte   4
+; CHECK-NEXT:   .byte   0
+; CHECK-NEXT:   .hword  8
+; CHECK-NEXT:   .hword  0
+; CHECK-NEXT:   .hword  0
+; CHECK-NEXT:   .word   -1
+; SmallConstant
+; CHECK-NEXT:   .byte   4
+; CHECK-NEXT:   .byte   0
+; CHECK-NEXT:   .hword  8
+; CHECK-NEXT:   .hword  0
+; CHECK-NEXT:   .hword  0
+; CHECK-NEXT:   .word   65536
+; SmallConstant
+; CHECK-NEXT:   .byte   4
+; CHECK-NEXT:   .byte   0
+; CHECK-NEXT:   .hword  8
+; CHECK-NEXT:   .hword  0
+; CHECK-NEXT:   .hword  0
+; CHECK-NEXT:   .word   2000000000
+; SmallConstant
+; CHECK-NEXT:   .byte   4
+; CHECK-NEXT:   .byte   0
+; CHECK-NEXT:   .hword  8
+; CHECK-NEXT:   .hword  0
+; CHECK-NEXT:   .hword  0
+; CHECK-NEXT:   .word   2147483647
+; SmallConstant
+; CHECK-NEXT:   .byte   4
+; CHECK-NEXT:   .byte   0
+; CHECK-NEXT:   .hword  8
+; CHECK-NEXT:   .hword  0
+; CHECK-NEXT:   .hword  0
+; CHECK-NEXT:   .word   -1
+; SmallConstant
+; CHECK-NEXT:   .byte   4
+; CHECK-NEXT:   .byte   0
+; CHECK-NEXT:   .hword  8
+; CHECK-NEXT:   .hword  0
+; CHECK-NEXT:   .hword  0
+; CHECK-NEXT:   .word   -1
+; SmallConstant
+; CHECK-NEXT:   .byte   4
+; CHECK-NEXT:   .byte   0
+; CHECK-NEXT:   .hword  8
+; CHECK-NEXT:   .hword  0
+; CHECK-NEXT:   .hword  0
+; CHECK-NEXT:   .word   0
+; LargeConstant at index 0
+; CHECK-NEXT:   .byte   5
+; CHECK-NEXT:   .byte   0
+; CHECK-NEXT:   .hword  8
+; CHECK-NEXT:   .hword  0
+; CHECK-NEXT:   .hword  0
+; CHECK-NEXT:   .word   0
+; LargeConstant at index 1
+; CHECK-NEXT:   .byte   5
+; CHECK-NEXT:   .byte   0
+; CHECK-NEXT:   .hword  8
+; CHECK-NEXT:   .hword  0
+; CHECK-NEXT:   .hword  0
+; CHECK-NEXT:   .word   1
+; LargeConstant at index 2
+; CHECK-NEXT:   .byte   5
+; CHECK-NEXT:   .byte   0
+; CHECK-NEXT:   .hword  8
+; CHECK-NEXT:   .hword  0
+; CHECK-NEXT:   .hword  0
+; CHECK-NEXT:   .word   2
+; SmallConstant
+; CHECK-NEXT:   .byte   4
+; CHECK-NEXT:   .byte   0
+; CHECK-NEXT:   .hword  8
+; CHECK-NEXT:   .hword  0
+; CHECK-NEXT:   .hword  0
+; CHECK-NEXT:   .word   -1
+
+define void @constantargs() {
+entry:
+  %0 = inttoptr i64 12345 to i8*
+  tail call void (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.void(i64 1, i32 16, i8* %0, i32 0, i16 65535, i16 -1, i32 65536, i32 2000000000, i32 2147483647, i32 -1, i32 4294967295, i32 4294967296, i64 2147483648, i64 4294967295, i64 4294967296, i64 -1)
+  ret void
+}
+
+; Inline OSR Exit
+;
+; CHECK-LABEL:  .word   .L{{.*}}-osrinline
+; CHECK-NEXT:   .hword  0
+; CHECK-NEXT:   .hword  2
+; CHECK-NEXT:   .byte   1
+; CHECK-NEXT:   .byte   0
+; CHECK-NEXT:   .hword  8
+; CHECK-NEXT:   .hword  {{[0-9]+}}
+; CHECK-NEXT:   .hword  0
+; CHECK-NEXT:   .word   0
+; CHECK-NEXT:   .byte   1
+; CHECK-NEXT:   .byte   0
+; CHECK-NEXT:   .hword  8
+; CHECK-NEXT:   .hword  {{[0-9]+}}
+; CHECK-NEXT:   .hword  0
+; CHECK-NEXT:   .word  0
+define void @osrinline(i64 %a, i64 %b) {
+entry:
+  ; Runtime void->void call.
+  call void inttoptr (i64 -559038737 to void ()*)()
+  ; Followed by inline OSR patchpoint with 12-byte shadow and 2 live vars.
+  call void (i64, i32, ...) @llvm.experimental.stackmap(i64 3, i32 12, i64 %a, i64 %b)
+  ret void
+}
+
+; Cold OSR Exit
+;
+; 2 live variables in register.
+;
+; CHECK-LABEL:  .word   .L{{.*}}-osrcold
+; CHECK-NEXT:   .hword  0
+; CHECK-NEXT:   .hword  2
+; CHECK-NEXT:   .byte   1
+; CHECK-NEXT:   .byte   0
+; CHECK-NEXT:   .hword  8
+; CHECK-NEXT:   .hword  {{[0-9]+}}
+; CHECK-NEXT:   .hword  0
+; CHECK-NEXT:   .word   0
+; CHECK-NEXT:   .byte   1
+; CHECK-NEXT:   .byte   0
+; CHECK-NEXT:   .hword  8
+; CHECK-NEXT:   .hword  {{[0-9]+}}
+; CHECK-NEXT:   .hword  0
+; CHECK-NEXT:   .word   0
+define void @osrcold(i64 %a, i64 %b) {
+entry:
+  %test = icmp slt i64 %a, %b
+  br i1 %test, label %ret, label %cold
+cold:
+  ; OSR patchpoint with 12-byte nop-slide and 2 live vars.
+  %thunk = inttoptr i64 3735928559 to i8*
+  call void (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.void(i64 4, i32 16, i8* %thunk, i32 0, i64 %a, i64 %b)
+  unreachable
+ret:
+  ret void
+}
+
+; Property Read
+; CHECK-LABEL:  .word   .L{{.*}}-propertyRead
+; CHECK-NEXT:   .hword  0
+; CHECK-NEXT:   .hword  2
+; CHECK-NEXT:   .byte   1
+; CHECK-NEXT:   .byte   0
+; CHECK-NEXT:   .hword  8
+; CHECK-NEXT:   .hword  {{[0-9]+}}
+; CHECK-NEXT:   .hword  0
+; CHECK-NEXT:   .word   0
+; CHECK-NEXT:   .byte   1
+; CHECK-NEXT:   .byte   0
+; CHECK-NEXT:   .hword  8
+; CHECK-NEXT:   .hword  {{[0-9]+}}
+; CHECK-NEXT:   .hword  0
+; CHECK-NEXT:   .word   0
+define i64 @propertyRead(i64* %obj) {
+entry:
+  %resolveRead = inttoptr i64 3735928559 to i8*
+  %result = call anyregcc i64 (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.i64(i64 5, i32 16, i8* %resolveRead, i32 1, i64* %obj)
+  %add = add i64 %result, 3
+  ret i64 %add
+}
+
+; Property Write
+; CHECK-LABEL:  .word   .L{{.*}}-propertyWrite
+; CHECK-NEXT:   .hword  0
+; CHECK-NEXT:   .hword  2
+; CHECK-NEXT:   .byte   1
+; CHECK-NEXT:   .byte   0
+; CHECK-NEXT:   .hword  8
+; CHECK-NEXT:   .hword  {{[0-9]+}}
+; CHECK-NEXT:   .hword  0
+; CHECK-NEXT:   .word   0
+; CHECK-NEXT:   .byte   1
+; CHECK-NEXT:   .byte   0
+; CHECK-NEXT:   .hword  8
+; CHECK-NEXT:   .hword  {{[0-9]+}}
+; CHECK-NEXT:   .hword  0
+; CHECK-NEXT:   .word   0
+define void @propertyWrite(i64 %dummy1, i64* %obj, i64 %dummy2, i64 %a) {
+entry:
+  %resolveWrite = inttoptr i64 3735928559 to i8*
+  call anyregcc void (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.void(i64 6, i32 16, i8* %resolveWrite, i32 2, i64* %obj, i64 %a)
+  ret void
+}
+
+; Void JS Call
+;
+; 2 live variables in registers.
+;
+; CHECK-LABEL:  .word   .L{{.*}}-jsVoidCall
+; CHECK-NEXT:   .hword  0
+; CHECK-NEXT:   .hword  2
+; CHECK-NEXT:   .byte   1
+; CHECK-NEXT:   .byte   0
+; CHECK-NEXT:   .hword  8
+; CHECK-NEXT:   .hword  {{[0-9]+}}
+; CHECK-NEXT:   .hword  0
+; CHECK-NEXT:   .word   0
+; CHECK-NEXT:   .byte   1
+; CHECK-NEXT:   .byte   0
+; CHECK-NEXT:   .hword  8
+; CHECK-NEXT:   .hword  {{[0-9]+}}
+; CHECK-NEXT:   .hword  0
+; CHECK-NEXT:   .word   0
+define void @jsVoidCall(i64 %dummy1, i64* %obj, i64 %arg, i64 %l1, i64 %l2) {
+entry:
+  %resolveCall = inttoptr i64 3735928559 to i8*
+  call void (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.void(i64 7, i32 16, i8* %resolveCall, i32 2, i64* %obj, i64 %arg, i64 %l1, i64 %l2)
+  ret void
+}
+
+; i64 JS Call
+;
+; 2 live variables in registers.
+;
+; CHECK-LABEL:  .word   .L{{.*}}-jsIntCall
+; CHECK-NEXT:   .hword  0
+; CHECK-NEXT:   .hword  2
+; CHECK-NEXT:   .byte   1
+; CHECK-NEXT:   .byte   0
+; CHECK-NEXT:   .hword  8
+; CHECK-NEXT:   .hword  {{[0-9]+}}
+; CHECK-NEXT:   .hword  0
+; CHECK-NEXT:   .word   0
+; CHECK-NEXT:   .byte   1
+; CHECK-NEXT:   .byte   0
+; CHECK-NEXT:   .hword  8
+; CHECK-NEXT:   .hword  {{[0-9]+}}
+; CHECK-NEXT:   .hword  0
+; CHECK-NEXT:   .word   0
+define i64 @jsIntCall(i64 %dummy1, i64* %obj, i64 %arg, i64 %l1, i64 %l2) {
+entry:
+  %resolveCall = inttoptr i64 3735928559 to i8*
+  %result = call i64 (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.i64(i64 8, i32 16, i8* %resolveCall, i32 2, i64* %obj, i64 %arg, i64 %l1, i64 %l2)
+  %add = add i64 %result, 3
+  ret i64 %add
+}
+
+; Spilled stack map values.
+;
+; Verify 28 stack map entries.
+;
+; CHECK-LABEL:  .word .L{{.*}}-spilledValue
+; CHECK-NEXT:   .hword 0
+; CHECK-NEXT:   .hword 28
+;
+; Check that at least one is a spilled entry from RBP.
+; Location: Indirect RBP + ...
+; CHECK:        .byte 3
+; CHECK-NEXT:   .byte 0
+; CHECK-NEXT:   .hword 8
+; CHECK-NEXT:   .hword 29
+; CHECK-NEXT:   .hword 0
+; CHECK-NEXT:   .word
+define void @spilledValue(i64 %arg0, i64 %arg1, i64 %arg2, i64 %arg3, i64 %arg4, i64 %l0, i64 %l1, i64 %l2, i64 %l3, i64 %l4, i64 %l5, i64 %l6, i64 %l7, i64 %l8, i64 %l9, i64 %l10, i64 %l11, i64 %l12, i64 %l13, i64 %l14, i64 %l15, i64 %l16, i64 %l17, i64 %l18, i64 %l19, i64 %l20, i64 %l21, i64 %l22, i64 %l23, i64 %l24, i64 %l25, i64 %l26, i64 %l27) {
+entry:
+  call void (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.void(i64 11, i32 20, i8* null, i32 5, i64 %arg0, i64 %arg1, i64 %arg2, i64 %arg3, i64 %arg4, i64 %l0, i64 %l1, i64 %l2, i64 %l3, i64 %l4, i64 %l5, i64 %l6, i64 %l7, i64 %l8, i64 %l9, i64 %l10, i64 %l11, i64 %l12, i64 %l13, i64 %l14, i64 %l15, i64 %l16, i64 %l17, i64 %l18, i64 %l19, i64 %l20, i64 %l21, i64 %l22, i64 %l23, i64 %l24, i64 %l25, i64 %l26, i64 %l27)
+  ret void
+}
+
+; Spilled stack map values.
+;
+; Verify 30 stack map entries.
+;
+; CHECK-LABEL:  .word .L{{.*}}-spilledStackMapValue
+; CHECK-NEXT:   .hword 0
+; CHECK-NEXT:   .hword 30
+;
+; Check that at least one is a spilled entry from RBP.
+; Location: Indirect RBP + ...
+; CHECK:        .byte 3
+; CHECK-NEXT:   .byte   0
+; CHECK-NEXT:   .hword 8
+; CHECK-NEXT:   .hword 29
+; CHECK-NEXT:   .hword  0
+; CHECK-NEXT:   .word
+define webkit_jscc void @spilledStackMapValue(i64 %l0, i64 %l1, i64 %l2, i64 %l3, i64 %l4, i64 %l5, i64 %l6, i64 %l7, i64 %l8, i64 %l9, i64 %l10, i64 %l11, i64 %l12, i64 %l13, i64 %l14, i64 %l15, i64 %l16, i64 %l17, i64 %l18, i64 %l19, i64 %l20, i64 %l21, i64 %l22, i64 %l23, i64 %l24, i64 %l25, i64 %l26, i64 %l27, i64 %l28, i64 %l29) {
+entry:
+  call void (i64, i32, ...) @llvm.experimental.stackmap(i64 12, i32 16, i64 %l0, i64 %l1, i64 %l2, i64 %l3, i64 %l4, i64 %l5, i64 %l6, i64 %l7, i64 %l8, i64 %l9, i64 %l10, i64 %l11, i64 %l12, i64 %l13, i64 %l14, i64 %l15, i64 %l16, i64 %l17, i64 %l18, i64 %l19, i64 %l20, i64 %l21, i64 %l22, i64 %l23, i64 %l24, i64 %l25, i64 %l26, i64 %l27, i64 %l28, i64 %l29)
+  ret void
+}
+
+; Map a constant value.
+;
+; CHECK-LABEL:  .word .L{{.*}}-liveConstant
+; CHECK-NEXT:   .hword 0
+; 1 location
+; CHECK-NEXT:   .hword 1
+; Loc 0: SmallConstant
+; CHECK-NEXT:   .byte   4
+; CHECK-NEXT:   .byte   0
+; CHECK-NEXT:   .hword  8
+; CHECK-NEXT:   .hword  0
+; CHECK-NEXT:   .hword  0
+; CHECK-NEXT:   .word   33
+
+define void @liveConstant() {
+  tail call void (i64, i32, ...) @llvm.experimental.stackmap(i64 15, i32 4, i32 33)
+  ret void
+}
+
+; Directly map an alloca's address.
+;
+; Callsite 16
+; CHECK-LABEL:  .word .L{{.*}}-directFrameIdx
+; CHECK-NEXT:   .hword 0
+; 1 location
+; CHECK-NEXT:   .hword  1
+; Loc 0: Direct RBP - ofs
+; CHECK-NEXT:   .byte   2
+; CHECK-NEXT:   .byte   0
+; CHECK-NEXT:   .hword  8
+; CHECK-NEXT:   .hword  29
+; CHECK-NEXT:   .hword  0
+; CHECK-NEXT:   .word
+
+; Callsite 17
+; CHECK-LABEL:  .word   .L{{.*}}-directFrameIdx
+; CHECK-NEXT:   .hword  0
+; 2 locations
+; CHECK-NEXT:   .hword  2
+; Loc 0: Direct RBP - ofs
+; CHECK-NEXT:   .byte   2
+; CHECK-NEXT:   .byte   0
+; CHECK-NEXT:   .hword  8
+; CHECK-NEXT:   .hword  29
+; CHECK-NEXT:   .hword  0
+; CHECK-NEXT:   .word
+; Loc 1: Direct RBP - ofs
+; CHECK-NEXT:   .byte   2
+; CHECK-NEXT:   .byte   0
+; CHECK-NEXT:   .hword  8
+; CHECK-NEXT:   .hword  29
+; CHECK-NEXT:   .hword  0
+; CHECK-NEXT:   .word
+define void @directFrameIdx() {
+entry:
+  %metadata1 = alloca i64, i32 3, align 8
+  store i64 11, i64* %metadata1
+  store i64 12, i64* %metadata1
+  store i64 13, i64* %metadata1
+  call void (i64, i32, ...) @llvm.experimental.stackmap(i64 16, i32 0, i64* %metadata1)
+  %metadata2 = alloca i8, i32 4, align 8
+  %metadata3 = alloca i16, i32 4, align 8
+  call void (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.void(i64 17, i32 4, i8* null, i32 0, i8* %metadata2, i16* %metadata3)
+  ret void
+}
+
+; Test a 64-bit ID.
+;
+; CHECK:        .xword 4294967295
+; CHECK-LABEL:  .word .L{{.*}}-longid
+; CHECK:        .xword 4294967296
+; CHECK-LABEL:  .word .L{{.*}}-longid
+; CHECK:        .xword 9223372036854775807
+; CHECK-LABEL:  .word .L{{.*}}-longid
+; CHECK:        .xword -1
+; CHECK-LABEL:  .word .L{{.*}}-longid
+define void @longid() {
+entry:
+  tail call void (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.void(i64 4294967295, i32 0, i8* null, i32 0)
+  tail call void (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.void(i64 4294967296, i32 0, i8* null, i32 0)
+  tail call void (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.void(i64 9223372036854775807, i32 0, i8* null, i32 0)
+  tail call void (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.void(i64 -1, i32 0, i8* null, i32 0)
+  ret void
+}
+
+; Map a value when R11 is the only free register.
+; The scratch register should not be used for a live stackmap value.
+;
+; CHECK-LABEL:  .word .L{{.*}}-clobberLR
+; CHECK-NEXT:   .hword 0
+; 1 location
+; CHECK-NEXT:   .hword 1
+; Loc 0: Indirect fp - offset
+; CHECK-NEXT:   .byte   3
+; CHECK-NEXT:   .byte   0
+; CHECK-NEXT:   .hword  4
+; CHECK-NEXT:   .hword  29
+; CHECK-NEXT:   .hword  0
+; CHECK-NEXT:   .word   -{{[0-9]+}}
+define void @clobberLR(i32 %a) {
+  tail call void asm sideeffect "nop", "~{x0},~{x1},~{x2},~{x3},~{x4},~{x5},~{x6},~{x7},~{x8},~{x9},~{x10},~{x11},~{x12},~{x13},~{x14},~{x15},~{x16},~{x17},~{x18},~{x19},~{x20},~{x21},~{x22},~{x23},~{x24},~{x25},~{x26},~{x27},~{x28},~{x29},~{x31}"() nounwind
+  tail call void (i64, i32, ...) @llvm.experimental.stackmap(i64 16, i32 8, i32 %a)
+  ret void
+}
+
+; A stack frame which needs to be realigned at runtime (to meet alignment 
+; criteria for values on the stack) does not have a fixed frame size. 
+; CHECK-LABEL:  .word .L{{.*}}-needsStackRealignment
+; CHECK-NEXT:   .hword 0
+; 0 locations
+; CHECK-NEXT:   .hword 0
+define void @needsStackRealignment() {
+  %val = alloca i64, i32 3, align 128
+  tail call void (...) @escape_values(i64* %val)
+; Note: Adding any non-constant to the stackmap would fail because we
+; expected to be able to address off the frame pointer.  In a realigned
+; frame, we must use the stack pointer instead.  This is a separate bug.
+  tail call void (i64, i32, ...) @llvm.experimental.stackmap(i64 0, i32 0)
+  ret void
+}
+declare void @escape_values(...)
+
+declare void @llvm.experimental.stackmap(i64, i32, ...)
+declare void @llvm.experimental.patchpoint.void(i64, i32, i8*, i32, ...)
+declare i64 @llvm.experimental.patchpoint.i64(i64, i32, i8*, i32, ...)
diff --git a/llvm/test/CodeGen/AArch64/stgp.ll b/llvm/test/CodeGen/AArch64/stgp.ll
index b4af16fd9..d82b45134 100644
--- a/llvm/test/CodeGen/AArch64/stgp.ll
+++ b/llvm/test/CodeGen/AArch64/stgp.ll
@@ -65,7 +65,7 @@ entry:
 define void @stgp_alloca(i64 %a, i64 %b) {
 entry:
 ; CHECK-LABEL: stgp_alloca:
-; CHECK: stgp x0, x1, [sp]
+; CHECK: stgp x0, x1, [sp, #-32]!
 ; CHECK: stgp x1, x0, [sp, #16]
 ; CHECK: ret
   %x = alloca i8, i32 32, align 16
diff --git a/llvm/test/CodeGen/AArch64/strict-fp-int-promote.ll b/llvm/test/CodeGen/AArch64/strict-fp-int-promote.ll
new file mode 100644
index 000000000..296f058d1
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/strict-fp-int-promote.ll
@@ -0,0 +1,67 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -O0 -o - %s | FileCheck %s
+; RUN: llc -O3 -o - %s | FileCheck %s --check-prefix=SUBOPTIMAL
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64-arm-none-eabi"
+
+declare float @llvm.experimental.constrained.sitofp.f32.i32(i32, metadata, metadata)
+declare float @llvm.experimental.constrained.sitofp.f32.i16(i16, metadata, metadata)
+declare i1 @llvm.experimental.constrained.fcmp.f32(float, float, metadata, metadata)
+declare float @llvm.experimental.constrained.uitofp.f32.i16(i16, metadata, metadata)
+
+define i32 @test() #0 {
+; CHECK-LABEL: test:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov w8, #1
+; CHECK-NEXT:    scvtf s0, w8
+; CHECK-NEXT:    fcmp s0, s0
+; CHECK-NEXT:    cset w0, eq
+; CHECK-NEXT:    ret
+;
+; SUBOPTIMAL-LABEL: test:
+; SUBOPTIMAL:       // %bb.0: // %entry
+; SUBOPTIMAL-NEXT:    mov w8, #1
+; SUBOPTIMAL-NEXT:    scvtf s0, w8
+; SUBOPTIMAL-NEXT:    mov w8, #1
+; SUBOPTIMAL-NEXT:    scvtf s1, w8
+; SUBOPTIMAL-NEXT:    fcmp s0, s1
+; SUBOPTIMAL-NEXT:    cset w8, eq
+; SUBOPTIMAL-NEXT:    and w0, w8, #0x1
+; SUBOPTIMAL-NEXT:    ret
+entry:
+  %conv = call float @llvm.experimental.constrained.sitofp.f32.i32(i32 1, metadata !"round.tonearest", metadata !"fpexcept.strict")
+  %conv1 = call float @llvm.experimental.constrained.sitofp.f32.i16(i16 1, metadata !"round.tonearest", metadata !"fpexcept.strict")
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f32(float %conv, float %conv1, metadata !"oeq", metadata !"fpexcept.strict")
+  %conv2 = zext i1 %cmp to i32
+  ret i32 %conv2
+}
+
+define i32 @test2() #0 {
+; CHECK-LABEL: test2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov w8, #1
+; CHECK-NEXT:    scvtf s0, w8
+; CHECK-NEXT:    ucvtf s1, w8
+; CHECK-NEXT:    fcmp s0, s1
+; CHECK-NEXT:    cset w0, eq
+; CHECK-NEXT:    ret
+;
+; SUBOPTIMAL-LABEL: test2:
+; SUBOPTIMAL:       // %bb.0: // %entry
+; SUBOPTIMAL-NEXT:    mov w8, #1
+; SUBOPTIMAL-NEXT:    scvtf s0, w8
+; SUBOPTIMAL-NEXT:    mov w8, #1
+; SUBOPTIMAL-NEXT:    ucvtf s1, w8
+; SUBOPTIMAL-NEXT:    fcmp s0, s1
+; SUBOPTIMAL-NEXT:    cset w8, eq
+; SUBOPTIMAL-NEXT:    and w0, w8, #0x1
+; SUBOPTIMAL-NEXT:    ret
+entry:
+  %conv = call float @llvm.experimental.constrained.sitofp.f32.i32(i32 1, metadata !"round.tonearest", metadata !"fpexcept.strict")
+  %conv1 = call float @llvm.experimental.constrained.uitofp.f32.i16(i16 1, metadata !"round.tonearest", metadata !"fpexcept.strict")
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f32(float %conv, float %conv1, metadata !"oeq", metadata !"fpexcept.strict")
+  %conv2 = zext i1 %cmp to i32
+  ret i32 %conv2
+}
+
+attributes #0 = { noinline optnone }
diff --git a/llvm/test/CodeGen/AArch64/strqu.ll b/llvm/test/CodeGen/AArch64/strqu.ll
index f20a30e3a..f34624973 100644
--- a/llvm/test/CodeGen/AArch64/strqu.ll
+++ b/llvm/test/CodeGen/AArch64/strqu.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -verify-machineinstrs -mtriple=aarch64-linux-gnu    | FileCheck --check-prefixes=CHECK,NOSPLIT %s
 ; RUN: llc < %s -verify-machineinstrs -mtriple=aarch64_be-linux-gnu | FileCheck --check-prefixes=CHECK,NOSPLIT %s
-; RUN: llc < %s -verify-machineinstrs -mtriple=aarch64-linux-gnu    -mcpu=exynos-m1 | FileCheck --check-prefixes=CHECK,NOSPLIT %s
-; RUN: llc < %s -verify-machineinstrs -mtriple=aarch64_be-linux-gnu -mcpu=exynos-m1 | FileCheck --check-prefixes=CHECK,SPLIT %s
+; RUN: llc < %s -verify-machineinstrs -mtriple=aarch64-linux-gnu    -mcpu=exynos-m3 | FileCheck --check-prefixes=CHECK,NOSPLIT %s
+; RUN: llc < %s -verify-machineinstrs -mtriple=aarch64_be-linux-gnu -mcpu=exynos-m3 | FileCheck --check-prefixes=CHECK,NOSPLIT %s
 
 define void @test_split_f(<4 x float> %val, <4 x float>* %addr) {
 ; NOSPLIT-LABEL: test_split_f:
diff --git a/llvm/test/CodeGen/AArch64/sub-of-bias.ll b/llvm/test/CodeGen/AArch64/sub-of-bias.ll
new file mode 100644
index 000000000..17b548cbe
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sub-of-bias.ll
@@ -0,0 +1,103 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64-unknown-linux-gnu < %s | FileCheck %s
+
+; Fold
+;   ptr - (ptr & mask)
+; To
+;   ptr & (~mask)
+;
+; This needs to be a backend-level fold because only by now pointers
+; are just registers; in middle-end IR this can only be done via @llvm.ptrmask()
+; intrinsic which is not sufficiently widely-spread yet.
+;
+; https://bugs.llvm.org/show_bug.cgi?id=44448
+
+; The basic positive tests
+
+define i32 @t0_32(i32 %ptr, i32 %mask) nounwind {
+; CHECK-LABEL: t0_32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    bic w0, w0, w1
+; CHECK-NEXT:    ret
+  %bias = and i32 %ptr, %mask
+  %r = sub i32 %ptr, %bias
+  ret i32 %r
+}
+define i64 @t1_64(i64 %ptr, i64 %mask) nounwind {
+; CHECK-LABEL: t1_64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    bic x0, x0, x1
+; CHECK-NEXT:    ret
+  %bias = and i64 %ptr, %mask
+  %r = sub i64 %ptr, %bias
+  ret i64 %r
+}
+
+define i32 @t2_commutative(i32 %ptr, i32 %mask) nounwind {
+; CHECK-LABEL: t2_commutative:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    bic w0, w0, w1
+; CHECK-NEXT:    ret
+  %bias = and i32 %mask, %ptr ; swapped
+  %r = sub i32 %ptr, %bias
+  ret i32 %r
+}
+
+; Extra use tests
+
+define i32 @n3_extrause1(i32 %ptr, i32 %mask, i32* %bias_storage) nounwind {
+; CHECK-LABEL: n3_extrause1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    and w8, w0, w1
+; CHECK-NEXT:    sub w0, w0, w8
+; CHECK-NEXT:    str w8, [x2]
+; CHECK-NEXT:    ret
+  %bias = and i32 %ptr, %mask ; has extra uses, can't fold
+  store i32 %bias, i32* %bias_storage
+  %r = sub i32 %ptr, %bias
+  ret i32 %r
+}
+
+; Negative tests
+
+define i32 @n4_different_ptrs(i32 %ptr0, i32 %ptr1, i32 %mask) nounwind {
+; CHECK-LABEL: n4_different_ptrs:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    and w8, w1, w2
+; CHECK-NEXT:    sub w0, w0, w8
+; CHECK-NEXT:    ret
+  %bias = and i32 %ptr1, %mask ; not %ptr0
+  %r = sub i32 %ptr0, %bias ; not %ptr1
+  ret i32 %r
+}
+define i32 @n5_different_ptrs_commutative(i32 %ptr0, i32 %ptr1, i32 %mask) nounwind {
+; CHECK-LABEL: n5_different_ptrs_commutative:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    and w8, w2, w1
+; CHECK-NEXT:    sub w0, w0, w8
+; CHECK-NEXT:    ret
+  %bias = and i32 %mask, %ptr1 ; swapped, not %ptr0
+  %r = sub i32 %ptr0, %bias ; not %ptr1
+  ret i32 %r
+}
+
+define i32 @n6_not_lowbit_mask(i32 %ptr, i32 %mask) nounwind {
+; CHECK-LABEL: n6_not_lowbit_mask:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    bic w0, w0, w1
+; CHECK-NEXT:    ret
+  %bias = and i32 %ptr, %mask
+  %r = sub i32 %ptr, %bias
+  ret i32 %r
+}
+
+define i32 @n7_sub_is_not_commutative(i32 %ptr, i32 %mask) nounwind {
+; CHECK-LABEL: n7_sub_is_not_commutative:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    and w8, w0, w1
+; CHECK-NEXT:    sub w0, w8, w0
+; CHECK-NEXT:    ret
+  %bias = and i32 %ptr, %mask
+  %r = sub i32 %bias, %ptr ; wrong order
+  ret i32 %r
+}
diff --git a/llvm/test/CodeGen/AArch64/sve-alloca-stackid.ll b/llvm/test/CodeGen/AArch64/sve-alloca-stackid.ll
new file mode 100644
index 000000000..4aae5bef2
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-alloca-stackid.ll
@@ -0,0 +1,17 @@
+; RUN: llc -mtriple=aarch64 -mattr=+sve < %s | FileCheck %s --check-prefix=CHECKCG
+; RUN: llc -mtriple=aarch64 -mattr=+sve -stop-after=finalize-isel < %s | FileCheck %s --check-prefix=CHECKISEL
+
+; CHECKCG-LABEL: foo:
+; CHECKCG: addvl   sp, sp, #-1
+
+; CHECKISEL-LABEL: name: foo
+; CHECKISEL:       stack:
+; CHECKISEL:       id: 0, name: ptr, type: default, offset: 0, size: 16, alignment: 16,
+; CHECKISEL-NEXT:  stack-id: sve-vec
+define i32 @foo(<vscale x 16 x i8> %val) {
+  %ptr = alloca <vscale x 16 x i8>
+  %res = call i32 @bar(<vscale x 16 x i8>* %ptr)
+  ret i32 %res
+}
+
+declare i32 @bar(<vscale x 16 x i8>* %ptr);
diff --git a/llvm/test/CodeGen/AArch64/sve-calling-convention.ll b/llvm/test/CodeGen/AArch64/sve-calling-convention.ll
new file mode 100644
index 000000000..b1dabddee
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-calling-convention.ll
@@ -0,0 +1,121 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve -stop-after=finalize-isel < %s | FileCheck %s
+
+; CHECK-LABEL: name: nosve_signature
+define i32 @nosve_signature() nounwind {
+  ret i32 42
+}
+
+; CHECK-LABEL: name: sve_signature_ret_vec
+define <vscale x 4 x i32> @sve_signature_ret_vec() nounwind {
+  ret <vscale x 4 x i32> undef
+}
+
+; CHECK-LABEL: name: sve_signature_ret_pred
+define <vscale x 4 x i1> @sve_signature_ret_pred() nounwind {
+  ret <vscale x 4 x i1> undef
+}
+
+; CHECK-LABEL: name: sve_signature_arg_vec
+define void @sve_signature_arg_vec(<vscale x 4 x i32> %arg) nounwind {
+  ret void
+}
+
+; CHECK-LABEL: name: sve_signature_arg_pred
+define void @sve_signature_arg_pred(<vscale x 4 x i1> %arg) nounwind {
+  ret void
+}
+
+; CHECK-LABEL: name: caller_nosve_signature
+; CHECK: BL @nosve_signature, csr_aarch64_aapcs
+define i32 @caller_nosve_signature() nounwind {
+  %res = call i32 @nosve_signature()
+  ret i32 %res
+}
+
+; CHECK-LABEL: name: sve_signature_ret_vec_caller
+; CHECK: BL @sve_signature_ret_vec, csr_aarch64_sve_aapcs
+define <vscale x 4 x i32>  @sve_signature_ret_vec_caller() nounwind {
+  %res = call <vscale x 4 x i32> @sve_signature_ret_vec()
+  ret <vscale x 4 x i32> %res
+}
+
+; CHECK-LABEL: name: sve_signature_ret_pred_caller
+; CHECK: BL @sve_signature_ret_pred, csr_aarch64_sve_aapcs
+define <vscale x 4 x i1>  @sve_signature_ret_pred_caller() nounwind {
+  %res = call <vscale x 4 x i1> @sve_signature_ret_pred()
+  ret <vscale x 4 x i1> %res
+}
+
+; CHECK-LABEL: name: sve_signature_arg_vec_caller
+; CHECK: BL @sve_signature_arg_vec, csr_aarch64_sve_aapcs
+define void @sve_signature_arg_vec_caller(<vscale x 4 x i32> %arg) nounwind {
+  call void @sve_signature_arg_vec(<vscale x 4 x i32> %arg)
+  ret void
+}
+
+; CHECK-LABEL: name: sve_signature_arg_pred_caller
+; CHECK: BL @sve_signature_arg_pred, csr_aarch64_sve_aapcs
+define void @sve_signature_arg_pred_caller(<vscale x 4 x i1> %arg) nounwind {
+  call void @sve_signature_arg_pred(<vscale x 4 x i1> %arg)
+  ret void
+}
+
+; CHECK-LABEL: name: sve_signature_many_arg_vec
+; CHECK: [[RES:%[0-9]+]]:zpr = COPY $z7
+; CHECK: $z0 = COPY [[RES]]
+; CHECK: RET_ReallyLR implicit $z0
+define <vscale x 4 x i32> @sve_signature_many_arg_vec(<vscale x 4 x i32> %arg1, <vscale x 4 x i32> %arg2, <vscale x 4 x i32> %arg3, <vscale x 4 x i32> %arg4, <vscale x 4 x i32> %arg5, <vscale x 4 x i32> %arg6, <vscale x 4 x i32> %arg7, <vscale x 4 x i32> %arg8) nounwind {
+  ret <vscale x 4 x i32> %arg8
+}
+
+; CHECK-LABEL: name: sve_signature_many_arg_pred
+; CHECK: [[RES:%[0-9]+]]:ppr = COPY $p3
+; CHECK: $p0 = COPY [[RES]]
+; CHECK: RET_ReallyLR implicit $p0
+define <vscale x 4 x i1> @sve_signature_many_arg_pred(<vscale x 4 x i1> %arg1, <vscale x 4 x i1> %arg2, <vscale x 4 x i1> %arg3, <vscale x 4 x i1> %arg4) nounwind {
+  ret <vscale x 4 x i1> %arg4
+}
+
+; CHECK-LABEL: name: sve_signature_vec
+; CHECK: [[RES:%[0-9]+]]:zpr = COPY $z1
+; CHECK: $z0 = COPY [[RES]]
+; CHECK: RET_ReallyLR implicit $z0
+define <vscale x 4 x i32> @sve_signature_vec(<vscale x 4 x i32> %arg1, <vscale x 4 x i32> %arg2) nounwind {
+ ret <vscale x 4 x i32> %arg2
+}
+
+; CHECK-LABEL: name: sve_signature_pred
+; CHECK: [[RES:%[0-9]+]]:ppr = COPY $p1
+; CHECK: $p0 = COPY [[RES]]
+; CHECK: RET_ReallyLR implicit $p0
+define <vscale x 4 x i1> @sve_signature_pred(<vscale x 4 x i1> %arg1, <vscale x 4 x i1> %arg2) nounwind {
+  ret <vscale x 4 x i1> %arg2
+}
+
+; CHECK-LABEL: name: sve_signature_vec_caller
+; CHECK-DAG: [[ARG2:%[0-9]+]]:zpr = COPY $z1
+; CHECK-DAG: [[ARG1:%[0-9]+]]:zpr = COPY $z0
+; CHECK-DAG: $z0 = COPY [[ARG2]]
+; CHECK-DAG: $z1 = COPY [[ARG1]]
+; CHECK-NEXT: BL @sve_signature_vec, csr_aarch64_sve_aapcs
+; CHECK: [[RES:%[0-9]+]]:zpr = COPY $z0
+; CHECK: $z0 = COPY [[RES]]
+; CHECK: RET_ReallyLR implicit $z0
+define <vscale x 4 x i32> @sve_signature_vec_caller(<vscale x 4 x i32> %arg1, <vscale x 4 x i32> %arg2) nounwind {
+  %res = call <vscale x 4 x i32> @sve_signature_vec(<vscale x 4 x i32> %arg2, <vscale x 4 x i32> %arg1)
+  ret <vscale x 4 x i32> %res
+}
+
+; CHECK-LABEL: name: sve_signature_pred_caller
+; CHECK-DAG: [[ARG2:%[0-9]+]]:ppr = COPY $p1
+; CHECK-DAG: [[ARG1:%[0-9]+]]:ppr = COPY $p0
+; CHECK-DAG: $p0 = COPY [[ARG2]]
+; CHECK-DAG: $p1 = COPY [[ARG1]]
+; CHECK-NEXT: BL @sve_signature_pred, csr_aarch64_sve_aapcs
+; CHECK: [[RES:%[0-9]+]]:ppr = COPY $p0
+; CHECK: $p0 = COPY [[RES]]
+; CHECK: RET_ReallyLR implicit $p0
+define <vscale x 4 x i1> @sve_signature_pred_caller(<vscale x 4 x i1> %arg1, <vscale x 4 x i1> %arg2) nounwind {
+  %res = call <vscale x 4 x i1> @sve_signature_pred(<vscale x 4 x i1> %arg2, <vscale x 4 x i1> %arg1)
+  ret <vscale x 4 x i1> %res
+}
diff --git a/llvm/test/CodeGen/AArch64/sve-fp.ll b/llvm/test/CodeGen/AArch64/sve-fp.ll
new file mode 100644
index 000000000..4af45c198
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-fp.ll
@@ -0,0 +1,129 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+define <vscale x 8 x half> @fadd_h(<vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: fadd_h:
+; CHECK: fadd z0.h, z0.h, z1.h
+; CHECK-NEXT: ret
+  %res = fadd <vscale x 8 x half> %a, %b
+  ret <vscale x 8 x half> %res
+}
+
+define <vscale x 4 x float> @fadd_s(<vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: fadd_s:
+; CHECK: fadd z0.s, z0.s, z1.s
+; CHECK-NEXT: ret
+  %res = fadd <vscale x 4 x float> %a, %b
+  ret <vscale x 4 x float> %res
+}
+
+define <vscale x 2 x double> @fadd_d(<vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: fadd_d:
+; CHECK: fadd z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+  %res = fadd <vscale x 2 x double> %a, %b
+  ret <vscale x 2 x double> %res
+}
+
+define <vscale x 8 x half> @fsub_h(<vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: fsub_h:
+; CHECK: fsub z0.h, z0.h, z1.h
+; CHECK-NEXT: ret
+  %res = fsub <vscale x 8 x half> %a, %b
+  ret <vscale x 8 x half> %res
+}
+
+define <vscale x 4 x float> @fsub_s(<vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: fsub_s:
+; CHECK: fsub z0.s, z0.s, z1.s
+; CHECK-NEXT: ret
+  %res = fsub <vscale x 4 x float> %a, %b
+  ret <vscale x 4 x float> %res
+}
+
+define <vscale x 2 x double> @fsub_d(<vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: fsub_d:
+; CHECK: fsub z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+  %res = fsub <vscale x 2 x double> %a, %b
+  ret <vscale x 2 x double> %res
+}
+
+define <vscale x 8 x half> @fmul_h(<vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: fmul_h:
+; CHECK: fmul z0.h, z0.h, z1.h
+; CHECK-NEXT: ret
+  %res = fmul <vscale x 8 x half> %a, %b
+  ret <vscale x 8 x half> %res
+}
+
+define <vscale x 4 x float> @fmul_s(<vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: fmul_s:
+; CHECK: fmul z0.s, z0.s, z1.s
+; CHECK-NEXT: ret
+  %res = fmul <vscale x 4 x float> %a, %b
+  ret <vscale x 4 x float> %res
+}
+
+define <vscale x 2 x double> @fmul_d(<vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: fmul_d:
+; CHECK: fmul z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+  %res = fmul <vscale x 2 x double> %a, %b
+  ret <vscale x 2 x double> %res
+}
+
+define <vscale x 8 x half> @frecps_h(<vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: frecps_h:
+; CHECK: frecps z0.h, z0.h, z1.h
+; CHECK-NEXT: ret
+  %res = call <vscale x 8 x half> @llvm.aarch64.sve.frecps.x.nxv8f16(<vscale x 8 x half> %a, <vscale x 8 x half> %b)
+  ret <vscale x 8 x half> %res
+}
+
+define <vscale x 4 x float> @frecps_s(<vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: frecps_s:
+; CHECK: frecps z0.s, z0.s, z1.s
+; CHECK-NEXT: ret
+  %res = call <vscale x 4 x float> @llvm.aarch64.sve.frecps.x.nxv4f32(<vscale x 4 x float> %a, <vscale x 4 x float> %b)
+  ret <vscale x 4 x float> %res
+}
+
+define <vscale x 2 x double> @frecps_d(<vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: frecps_d:
+; CHECK: frecps z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+  %res = call <vscale x 2 x double> @llvm.aarch64.sve.frecps.x.nxv2f64(<vscale x 2 x double> %a, <vscale x 2 x double> %b)
+  ret <vscale x 2 x double> %res
+}
+
+define <vscale x 8 x half> @frsqrts_h(<vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: frsqrts_h:
+; CHECK: frsqrts z0.h, z0.h, z1.h
+; CHECK-NEXT: ret
+  %res = call <vscale x 8 x half> @llvm.aarch64.sve.frsqrts.x.nxv8f16(<vscale x 8 x half> %a, <vscale x 8 x half> %b)
+  ret <vscale x 8 x half> %res
+}
+
+define <vscale x 4 x float> @frsqrts_s(<vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: frsqrts_s:
+; CHECK: frsqrts z0.s, z0.s, z1.s
+; CHECK-NEXT: ret
+  %res = call <vscale x 4 x float> @llvm.aarch64.sve.frsqrts.x.nxv4f32(<vscale x 4 x float> %a, <vscale x 4 x float> %b)
+  ret <vscale x 4 x float> %res
+}
+
+define <vscale x 2 x double> @frsqrts_d(<vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: frsqrts_d:
+; CHECK: frsqrts z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+  %res = call <vscale x 2 x double> @llvm.aarch64.sve.frsqrts.x.nxv2f64(<vscale x 2 x double> %a, <vscale x 2 x double> %b)
+  ret <vscale x 2 x double> %res
+}
+
+declare <vscale x 8 x half> @llvm.aarch64.sve.frecps.x.nxv8f16(<vscale x 8 x half>, <vscale x 8 x half>)
+declare <vscale x 4 x float>  @llvm.aarch64.sve.frecps.x.nxv4f32(<vscale x 4 x float> , <vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.frecps.x.nxv2f64(<vscale x 2 x double>, <vscale x 2 x double>)
+
+declare <vscale x 8 x half> @llvm.aarch64.sve.frsqrts.x.nxv8f16(<vscale x 8 x half>, <vscale x 8 x half>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.frsqrts.x.nxv4f32(<vscale x 4 x float>, <vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.frsqrts.x.nxv2f64(<vscale x 2 x double>, <vscale x 2 x double>)
diff --git a/llvm/test/CodeGen/AArch64/sve-gather-scatter-dag-combine.ll b/llvm/test/CodeGen/AArch64/sve-gather-scatter-dag-combine.ll
new file mode 100644
index 000000000..e887aabf2
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-gather-scatter-dag-combine.ll
@@ -0,0 +1,72 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+; Verify that DAG combine rules for LD1 + sext/zext don't apply when the
+; result of LD1 has multiple uses
+
+define <vscale x 2 x i64> @no_dag_combine_zext_sext(<vscale x 2 x i1> %pg,
+                                                    <vscale x 2 x i64> %base,
+                                                    <vscale x 2 x i8>* %res_out,
+                                                    <vscale x 2 x i1> %pred) {
+; CHECK-LABEL: no_dag_combine_zext_sext
+; CHECK:  	ld1b	{ z0.d }, p0/z, [z0.d, #16]
+; CHECK-NEXT:	st1b	{ z0.d }, p1, [x0]
+; CHECK-NEXT:	and	z0.d, z0.d, #0xff
+; CHECK-NEXT: ret
+  %load = call <vscale x 2 x i8> @llvm.aarch64.sve.ld1.gather.imm.nxv2i8.nxv2i64(<vscale x 2 x i1> %pg,
+                                                                                 <vscale x 2 x i64> %base,
+                                                                                 i64 16)
+  %res1 = zext <vscale x 2 x i8> %load to <vscale x 2 x i64>
+  %res2 = sext <vscale x 2 x i8> %load to <vscale x 2 x i64>
+  call void @llvm.masked.store.nxv2i8(<vscale x 2 x i8> %load,
+                                      <vscale x 2 x i8> *%res_out,
+                                      i32 8,
+                                      <vscale x 2 x i1> %pred)
+
+  ret <vscale x 2 x i64> %res1
+}
+
+define <vscale x 2 x i64> @no_dag_combine_sext(<vscale x 2 x i1> %pg,
+                                               <vscale x 2 x i64> %base,
+                                               <vscale x 2 x i8>* %res_out,
+                                               <vscale x 2 x i1> %pred) {
+; CHECK-LABEL: no_dag_combine_sext
+; CHECK:  	ld1b	{ z1.d }, p0/z, [z0.d, #16]
+; CHECK-NEXT:	ptrue	p0.d
+; CHECK-NEXT:	sxtb	z0.d, p0/m, z1.d
+; CHECK-NEXT:	st1b	{ z1.d }, p1, [x0]
+; CHECK-NEXT:	ret
+  %load = call <vscale x 2 x i8> @llvm.aarch64.sve.ld1.gather.imm.nxv2i8.nxv2i64(<vscale x 2 x i1> %pg,
+                                                                                 <vscale x 2 x i64> %base,
+                                                                                 i64 16)
+  %res = sext <vscale x 2 x i8> %load to <vscale x 2 x i64>
+  call void @llvm.masked.store.nxv2i8(<vscale x 2 x i8> %load,
+                                      <vscale x 2 x i8> *%res_out,
+                                      i32 8,
+                                      <vscale x 2 x i1> %pred)
+
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @no_dag_combine_zext(<vscale x 2 x i1> %pg,
+                                               <vscale x 2 x i64> %base,
+                                               <vscale x 2 x i8>* %res_out,
+                                               <vscale x 2 x i1> %pred) {
+; CHECK-LABEL: no_dag_combine_zext
+; CHECK:  	ld1b	{ z0.d }, p0/z, [z0.d, #16]
+; CHECK-NEXT:	st1b	{ z0.d }, p1, [x0]
+; CHECK-NEXT:	and	z0.d, z0.d, #0xff
+; CHECK-NEXT:	ret
+  %load = call <vscale x 2 x i8> @llvm.aarch64.sve.ld1.gather.imm.nxv2i8.nxv2i64(<vscale x 2 x i1> %pg,
+                                                                                 <vscale x 2 x i64> %base,
+                                                                                 i64 16)
+  %res = zext <vscale x 2 x i8> %load to <vscale x 2 x i64>
+  call void @llvm.masked.store.nxv2i8(<vscale x 2 x i8> %load,
+                                      <vscale x 2 x i8> *%res_out,
+                                      i32 8,
+                                      <vscale x 2 x i1> %pred)
+
+  ret <vscale x 2 x i64> %res
+}
+
+declare <vscale x 2 x i8> @llvm.aarch64.sve.ld1.gather.imm.nxv2i8.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>, i64)
+declare void @llvm.masked.store.nxv2i8(<vscale x 2 x i8>, <vscale x 2 x i8>*, i32, <vscale x 2 x i1>)
diff --git a/llvm/test/CodeGen/AArch64/sve-int-arith-imm.ll b/llvm/test/CodeGen/AArch64/sve-int-arith-imm.ll
new file mode 100644
index 000000000..451ddbbd8
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-int-arith-imm.ll
@@ -0,0 +1,448 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+;
+; SMAX
+;
+define <vscale x 16 x i8> @smax_i8_pos(<vscale x 16 x i8> %a) {
+; CHECK-LABEL: smax_i8_pos
+; CHECK: smax z0.b, z0.b, #27
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 16 x i8> undef, i8 27, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %cmp = icmp sgt <vscale x 16 x i8> %a, %splat
+  %res = select <vscale x 16 x i1> %cmp, <vscale x 16 x i8> %a, <vscale x 16 x i8> %splat
+  ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 16 x i8> @smax_i8_neg(<vscale x 16 x i8> %a) {
+; CHECK-LABEL: smax_i8_neg
+; CHECK: smax z0.b, z0.b, #-58
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 16 x i8> undef, i8 -58, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %cmp = icmp sgt <vscale x 16 x i8> %a, %splat
+  %res = select <vscale x 16 x i1> %cmp, <vscale x 16 x i8> %a, <vscale x 16 x i8> %splat
+  ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 8 x i16> @smax_i16_pos(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: smax_i16_pos
+; CHECK: smax z0.h, z0.h, #27
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 8 x i16> undef, i16 27, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %cmp = icmp sgt <vscale x 8 x i16> %a, %splat
+  %res = select <vscale x 8 x i1> %cmp, <vscale x 8 x i16> %a, <vscale x 8 x i16> %splat
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 8 x i16> @smax_i16_neg(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: smax_i16_neg
+; CHECK: smax z0.h, z0.h, #-58
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 8 x i16> undef, i16 -58, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %cmp = icmp sgt <vscale x 8 x i16> %a, %splat
+  %res = select <vscale x 8 x i1> %cmp, <vscale x 8 x i16> %a, <vscale x 8 x i16> %splat
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 4 x i32> @smax_i32_pos(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: smax_i32_pos
+; CHECK: smax z0.s, z0.s, #27
+; CHECK: ret
+  %elt = insertelement <vscale x 4 x i32> undef, i32 27, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %cmp = icmp sgt <vscale x 4 x i32> %a, %splat
+  %res = select <vscale x 4 x i1> %cmp, <vscale x 4 x i32> %a, <vscale x 4 x i32> %splat
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 4 x i32> @smax_i32_neg(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: smax_i32_neg
+; CHECK: smax z0.s, z0.s, #-58
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 4 x i32> undef, i32 -58, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %cmp = icmp sgt <vscale x 4 x i32> %a, %splat
+  %res = select <vscale x 4 x i1> %cmp, <vscale x 4 x i32> %a, <vscale x 4 x i32> %splat
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @smax_i64_pos(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: smax_i64_pos
+; CHECK: smax z0.d, z0.d, #27
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 2 x i64> undef, i64 27, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %cmp = icmp sgt <vscale x 2 x i64> %a, %splat
+  %res = select <vscale x 2 x i1> %cmp, <vscale x 2 x i64> %a, <vscale x 2 x i64> %splat
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @smax_i64_neg(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: smax_i64_neg
+; CHECK: smax z0.d, z0.d, #-58
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 2 x i64> undef, i64 -58, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %cmp = icmp sgt <vscale x 2 x i64> %a, %splat
+  %res = select <vscale x 2 x i1> %cmp, <vscale x 2 x i64> %a, <vscale x 2 x i64> %splat
+  ret <vscale x 2 x i64> %res
+}
+
+;
+; SMIN
+;
+define <vscale x 16 x i8> @smin_i8_pos(<vscale x 16 x i8> %a) {
+; CHECK-LABEL: smin_i8_pos
+; CHECK: smin z0.b, z0.b, #27
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 16 x i8> undef, i8 27, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %cmp = icmp slt <vscale x 16 x i8> %a, %splat
+  %res = select <vscale x 16 x i1> %cmp, <vscale x 16 x i8> %a, <vscale x 16 x i8> %splat
+  ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 16 x i8> @smin_i8_neg(<vscale x 16 x i8> %a) {
+; CHECK-LABEL: smin_i8_neg
+; CHECK: smin z0.b, z0.b, #-58
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 16 x i8> undef, i8 -58, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %cmp = icmp slt <vscale x 16 x i8> %a, %splat
+  %res = select <vscale x 16 x i1> %cmp, <vscale x 16 x i8> %a, <vscale x 16 x i8> %splat
+  ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 8 x i16> @smin_i16_pos(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: smin_i16_pos
+; CHECK: smin z0.h, z0.h, #27
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 8 x i16> undef, i16 27, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %cmp = icmp slt <vscale x 8 x i16> %a, %splat
+  %res = select <vscale x 8 x i1> %cmp, <vscale x 8 x i16> %a, <vscale x 8 x i16> %splat
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 8 x i16> @smin_i16_neg(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: smin_i16_neg
+; CHECK: smin z0.h, z0.h, #-58
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 8 x i16> undef, i16 -58, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %cmp = icmp slt <vscale x 8 x i16> %a, %splat
+  %res = select <vscale x 8 x i1> %cmp, <vscale x 8 x i16> %a, <vscale x 8 x i16> %splat
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 4 x i32> @smin_i32_pos(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: smin_i32_pos
+; CHECK: smin z0.s, z0.s, #27
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 4 x i32> undef, i32 27, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %cmp = icmp slt <vscale x 4 x i32> %a, %splat
+  %res = select <vscale x 4 x i1> %cmp, <vscale x 4 x i32> %a, <vscale x 4 x i32> %splat
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 4 x i32> @smin_i32_neg(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: smin_i32_neg
+; CHECK: smin z0.s, z0.s, #-58
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 4 x i32> undef, i32 -58, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %cmp = icmp slt <vscale x 4 x i32> %a, %splat
+  %res = select <vscale x 4 x i1> %cmp, <vscale x 4 x i32> %a, <vscale x 4 x i32> %splat
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @smin_i64_pos(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: smin_i64_pos
+; CHECK: smin z0.d, z0.d, #27
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 2 x i64> undef, i64 27, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %cmp = icmp slt <vscale x 2 x i64> %a, %splat
+  %res = select <vscale x 2 x i1> %cmp, <vscale x 2 x i64> %a, <vscale x 2 x i64> %splat
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @smin_i64_neg(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: smin_i64_neg
+; CHECK: smin z0.d, z0.d, #-58
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 2 x i64> undef, i64 -58, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %cmp = icmp slt <vscale x 2 x i64> %a, %splat
+  %res = select <vscale x 2 x i1> %cmp, <vscale x 2 x i64> %a, <vscale x 2 x i64> %splat
+  ret <vscale x 2 x i64> %res
+}
+
+;
+; UMAX
+;
+define <vscale x 16 x i8> @umax_i8_pos(<vscale x 16 x i8> %a) {
+; CHECK-LABEL: umax_i8_pos
+; CHECK: umax z0.b, z0.b, #27
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 16 x i8> undef, i8 27, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %cmp = icmp ugt <vscale x 16 x i8> %a, %splat
+  %res = select <vscale x 16 x i1> %cmp, <vscale x 16 x i8> %a, <vscale x 16 x i8> %splat
+  ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 16 x i8> @umax_i8_large(<vscale x 16 x i8> %a) {
+; CHECK-LABEL: umax_i8_large
+; CHECK: umax z0.b, z0.b, #129
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 16 x i8> undef, i8 129, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %cmp = icmp ugt <vscale x 16 x i8> %a, %splat
+  %res = select <vscale x 16 x i1> %cmp, <vscale x 16 x i8> %a, <vscale x 16 x i8> %splat
+  ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 8 x i16> @umax_i16_pos(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: umax_i16_pos
+; CHECK: umax z0.h, z0.h, #27
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 8 x i16> undef, i16 27, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %cmp = icmp ugt <vscale x 8 x i16> %a, %splat
+  %res = select <vscale x 8 x i1> %cmp, <vscale x 8 x i16> %a, <vscale x 8 x i16> %splat
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 8 x i16> @umax_i16_large(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: umax_i16_large
+; CHECK: umax z0.h, z0.h, #129
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 8 x i16> undef, i16 129, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %cmp = icmp ugt <vscale x 8 x i16> %a, %splat
+  %res = select <vscale x 8 x i1> %cmp, <vscale x 8 x i16> %a, <vscale x 8 x i16> %splat
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 4 x i32> @umax_i32_pos(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: umax_i32_pos
+; CHECK: umax z0.s, z0.s, #27
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 4 x i32> undef, i32 27, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %cmp = icmp ugt <vscale x 4 x i32> %a, %splat
+  %res = select <vscale x 4 x i1> %cmp, <vscale x 4 x i32> %a, <vscale x 4 x i32> %splat
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 4 x i32> @umax_i32_large(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: umax_i32_large
+; CHECK: umax z0.s, z0.s, #129
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 4 x i32> undef, i32 129, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %cmp = icmp ugt <vscale x 4 x i32> %a, %splat
+  %res = select <vscale x 4 x i1> %cmp, <vscale x 4 x i32> %a, <vscale x 4 x i32> %splat
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @umax_i64_pos(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: umax_i64_pos
+; CHECK: umax z0.d, z0.d, #27
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 2 x i64> undef, i64 27, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %cmp = icmp ugt <vscale x 2 x i64> %a, %splat
+  %res = select <vscale x 2 x i1> %cmp, <vscale x 2 x i64> %a, <vscale x 2 x i64> %splat
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @umax_i64_large(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: umax_i64_large
+; CHECK: umax z0.d, z0.d, #129
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 2 x i64> undef, i64 129, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %cmp = icmp ugt <vscale x 2 x i64> %a, %splat
+  %res = select <vscale x 2 x i1> %cmp, <vscale x 2 x i64> %a, <vscale x 2 x i64> %splat
+  ret <vscale x 2 x i64> %res
+}
+
+;
+; UMIN
+;
+define <vscale x 16 x i8> @umin_i8_pos(<vscale x 16 x i8> %a) {
+; CHECK-LABEL: umin_i8_pos
+; CHECK: umin z0.b, z0.b, #27
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 16 x i8> undef, i8 27, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %cmp = icmp ult <vscale x 16 x i8> %a, %splat
+  %res = select <vscale x 16 x i1> %cmp, <vscale x 16 x i8> %a, <vscale x 16 x i8> %splat
+  ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 16 x i8> @umin_i8_large(<vscale x 16 x i8> %a) {
+; CHECK-LABEL: umin_i8_large
+; CHECK: umin z0.b, z0.b, #129
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 16 x i8> undef, i8 129, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %cmp = icmp ult <vscale x 16 x i8> %a, %splat
+  %res = select <vscale x 16 x i1> %cmp, <vscale x 16 x i8> %a, <vscale x 16 x i8> %splat
+  ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 8 x i16> @umin_i16_pos(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: umin_i16_pos
+; CHECK: umin z0.h, z0.h, #27
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 8 x i16> undef, i16 27, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %cmp = icmp ult <vscale x 8 x i16> %a, %splat
+  %res = select <vscale x 8 x i1> %cmp, <vscale x 8 x i16> %a, <vscale x 8 x i16> %splat
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 8 x i16> @umin_i16_large(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: umin_i16_large
+; CHECK: umin z0.h, z0.h, #129
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 8 x i16> undef, i16 129, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %cmp = icmp ult <vscale x 8 x i16> %a, %splat
+  %res = select <vscale x 8 x i1> %cmp, <vscale x 8 x i16> %a, <vscale x 8 x i16> %splat
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 4 x i32> @umin_i32_pos(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: umin_i32_pos
+; CHECK: umin z0.s, z0.s, #27
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 4 x i32> undef, i32 27, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %cmp = icmp ult <vscale x 4 x i32> %a, %splat
+  %res = select <vscale x 4 x i1> %cmp, <vscale x 4 x i32> %a, <vscale x 4 x i32> %splat
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 4 x i32> @umin_i32_large(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: umin_i32_large
+; CHECK: umin z0.s, z0.s, #129
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 4 x i32> undef, i32 129, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %cmp = icmp ult <vscale x 4 x i32> %a, %splat
+  %res = select <vscale x 4 x i1> %cmp, <vscale x 4 x i32> %a, <vscale x 4 x i32> %splat
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @umin_i64_pos(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: umin_i64_pos
+; CHECK: umin z0.d, z0.d, #27
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 2 x i64> undef, i64 27, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %cmp = icmp ult <vscale x 2 x i64> %a, %splat
+  %res = select <vscale x 2 x i1> %cmp, <vscale x 2 x i64> %a, <vscale x 2 x i64> %splat
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @umin_i64_large(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: umin_i64_large
+; CHECK: umin z0.d, z0.d, #129
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 2 x i64> undef, i64 129, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %cmp = icmp ult <vscale x 2 x i64> %a, %splat
+  %res = select <vscale x 2 x i1> %cmp, <vscale x 2 x i64> %a, <vscale x 2 x i64> %splat
+  ret <vscale x 2 x i64> %res
+}
+
+;
+; MUL
+;
+define <vscale x 16 x i8> @mul_i8_neg(<vscale x 16 x i8> %a) {
+; CHECK-LABEL: mul_i8_neg
+; CHECK: mul z0.b, z0.b, #-17
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 16 x i8> undef, i8 -17, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %res = mul <vscale x 16 x i8> %a, %splat
+  ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 16 x i8> @mul_i8_pos(<vscale x 16 x i8> %a) {
+; CHECK-LABEL: mul_i8_pos
+; CHECK: mul z0.b, z0.b, #105
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 16 x i8> undef, i8 105, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %res = mul <vscale x 16 x i8> %a, %splat
+  ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 8 x i16> @mul_i16_neg(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: mul_i16_neg
+; CHECK: mul z0.h, z0.h, #-17
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 8 x i16> undef, i16 -17, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %res = mul <vscale x 8 x i16> %a, %splat
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 8 x i16> @mul_i16_pos(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: mul_i16_pos
+; CHECK: mul z0.h, z0.h, #105
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 8 x i16> undef, i16 105, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %res = mul <vscale x 8 x i16> %a, %splat
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 4 x i32> @mul_i32_neg(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: mul_i32_neg
+; CHECK: mul z0.s, z0.s, #-17
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 4 x i32> undef, i32 -17, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %res = mul <vscale x 4 x i32> %a, %splat
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 4 x i32> @mul_i32_pos(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: mul_i32_pos
+; CHECK: mul z0.s, z0.s, #105
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 4 x i32> undef, i32 105, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %res = mul <vscale x 4 x i32> %a, %splat
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @mul_i64_neg(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: mul_i64_neg
+; CHECK: mul z0.d, z0.d, #-17
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 2 x i64> undef, i64 -17, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %res = mul <vscale x 2 x i64> %a, %splat
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @mul_i64_pos(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: mul_i64_pos
+; CHECK: mul z0.d, z0.d, #105
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 2 x i64> undef, i64 105, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %res = mul <vscale x 2 x i64> %a, %splat
+  ret <vscale x 2 x i64> %res
+}
diff --git a/llvm/test/CodeGen/AArch64/sve-int-arith-pred.ll b/llvm/test/CodeGen/AArch64/sve-int-arith-pred.ll
new file mode 100644
index 000000000..d0660e733
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-int-arith-pred.ll
@@ -0,0 +1,406 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+define <vscale x 16 x i8> @add_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: add_i8:
+; CHECK: add z0.b, p0/m, z0.b, z1.b
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.add.nxv16i8(<vscale x 16 x i1> %pg,
+                                                               <vscale x 16 x i8> %a,
+                                                               <vscale x 16 x i8> %b)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @add_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: add_i16:
+; CHECK: add z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.add.nxv8i16(<vscale x 8 x i1> %pg,
+                                                               <vscale x 8 x i16> %a,
+                                                               <vscale x 8 x i16> %b)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @add_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: add_i32:
+; CHECK: add z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.add.nxv4i32(<vscale x 4 x i1> %pg,
+                                                               <vscale x 4 x i32> %a,
+                                                               <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @add_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: add_i64:
+; CHECK: add z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.add.nxv2i64(<vscale x 2 x i1> %pg,
+                                                               <vscale x 2 x i64> %a,
+                                                               <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i64> %out
+}
+
+define <vscale x 16 x i8> @sub_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: sub_i8:
+; CHECK: sub z0.b, p0/m, z0.b, z1.b
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.sub.nxv16i8(<vscale x 16 x i1> %pg,
+                                                               <vscale x 16 x i8> %a,
+                                                               <vscale x 16 x i8> %b)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @sub_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: sub_i16:
+; CHECK: sub z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.sub.nxv8i16(<vscale x 8 x i1> %pg,
+                                                               <vscale x 8 x i16> %a,
+                                                               <vscale x 8 x i16> %b)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @sub_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: sub_i32:
+; CHECK: sub z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.sub.nxv4i32(<vscale x 4 x i1> %pg,
+                                                               <vscale x 4 x i32> %a,
+                                                               <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @sub_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: sub_i64:
+; CHECK: sub z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.sub.nxv2i64(<vscale x 2 x i1> %pg,
+                                                               <vscale x 2 x i64> %a,
+                                                               <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i64> %out
+}
+
+define <vscale x 16 x i8> @subr_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: subr_i8:
+; CHECK: subr z0.b, p0/m, z0.b, z1.b
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.subr.nxv16i8(<vscale x 16 x i1> %pg,
+                                                               <vscale x 16 x i8> %a,
+                                                               <vscale x 16 x i8> %b)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @subr_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: subr_i16:
+; CHECK: subr z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.subr.nxv8i16(<vscale x 8 x i1> %pg,
+                                                               <vscale x 8 x i16> %a,
+                                                               <vscale x 8 x i16> %b)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @subr_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: subr_i32:
+; CHECK: subr z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.subr.nxv4i32(<vscale x 4 x i1> %pg,
+                                                               <vscale x 4 x i32> %a,
+                                                               <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @subr_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: subr_i64:
+; CHECK: subr z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.subr.nxv2i64(<vscale x 2 x i1> %pg,
+                                                               <vscale x 2 x i64> %a,
+                                                               <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i64> %out
+}
+
+define <vscale x 16 x i8> @smax_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: smax_i8:
+; CHECK: smax z0.b, p0/m, z0.b, z1.b
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.smax.nxv16i8(<vscale x 16 x i1> %pg,
+                                                               <vscale x 16 x i8> %a,
+                                                               <vscale x 16 x i8> %b)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @smax_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: smax_i16:
+; CHECK: smax z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.smax.nxv8i16(<vscale x 8 x i1> %pg,
+                                                               <vscale x 8 x i16> %a,
+                                                               <vscale x 8 x i16> %b)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @smax_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: smax_i32:
+; CHECK: smax z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.smax.nxv4i32(<vscale x 4 x i1> %pg,
+                                                               <vscale x 4 x i32> %a,
+                                                               <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @smax_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: smax_i64:
+; CHECK: smax z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.smax.nxv2i64(<vscale x 2 x i1> %pg,
+                                                               <vscale x 2 x i64> %a,
+                                                               <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i64> %out
+}
+
+define <vscale x 16 x i8> @umax_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: umax_i8:
+; CHECK: umax z0.b, p0/m, z0.b, z1.b
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.umax.nxv16i8(<vscale x 16 x i1> %pg,
+                                                               <vscale x 16 x i8> %a,
+                                                               <vscale x 16 x i8> %b)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @umax_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: umax_i16:
+; CHECK: umax z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.umax.nxv8i16(<vscale x 8 x i1> %pg,
+                                                               <vscale x 8 x i16> %a,
+                                                               <vscale x 8 x i16> %b)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @umax_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: umax_i32:
+; CHECK: umax z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.umax.nxv4i32(<vscale x 4 x i1> %pg,
+                                                               <vscale x 4 x i32> %a,
+                                                               <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @umax_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: umax_i64:
+; CHECK: umax z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.umax.nxv2i64(<vscale x 2 x i1> %pg,
+                                                               <vscale x 2 x i64> %a,
+                                                               <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i64> %out
+}
+
+define <vscale x 16 x i8> @smin_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: smin_i8:
+; CHECK: smin z0.b, p0/m, z0.b, z1.b
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.smin.nxv16i8(<vscale x 16 x i1> %pg,
+                                                               <vscale x 16 x i8> %a,
+                                                               <vscale x 16 x i8> %b)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @smin_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: smin_i16:
+; CHECK: smin z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.smin.nxv8i16(<vscale x 8 x i1> %pg,
+                                                               <vscale x 8 x i16> %a,
+                                                               <vscale x 8 x i16> %b)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @smin_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: smin_i32:
+; CHECK: smin z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.smin.nxv4i32(<vscale x 4 x i1> %pg,
+                                                               <vscale x 4 x i32> %a,
+                                                               <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @smin_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: smin_i64:
+; CHECK: smin z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.smin.nxv2i64(<vscale x 2 x i1> %pg,
+                                                               <vscale x 2 x i64> %a,
+                                                               <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i64> %out
+}
+
+define <vscale x 16 x i8> @umin_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: umin_i8:
+; CHECK: umin z0.b, p0/m, z0.b, z1.b
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.umin.nxv16i8(<vscale x 16 x i1> %pg,
+                                                               <vscale x 16 x i8> %a,
+                                                               <vscale x 16 x i8> %b)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @umin_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: umin_i16:
+; CHECK: umin z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.umin.nxv8i16(<vscale x 8 x i1> %pg,
+                                                               <vscale x 8 x i16> %a,
+                                                               <vscale x 8 x i16> %b)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @umin_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: umin_i32:
+; CHECK: umin z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.umin.nxv4i32(<vscale x 4 x i1> %pg,
+                                                               <vscale x 4 x i32> %a,
+                                                               <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @umin_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: umin_i64:
+; CHECK: umin z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.umin.nxv2i64(<vscale x 2 x i1> %pg,
+                                                               <vscale x 2 x i64> %a,
+                                                               <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i64> %out
+}
+
+define <vscale x 16 x i8> @sabd_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: sabd_i8:
+; CHECK: sabd z0.b, p0/m, z0.b, z1.b
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.sabd.nxv16i8(<vscale x 16 x i1> %pg,
+                                                               <vscale x 16 x i8> %a,
+                                                               <vscale x 16 x i8> %b)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @sabd_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: sabd_i16:
+; CHECK: sabd z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.sabd.nxv8i16(<vscale x 8 x i1> %pg,
+                                                               <vscale x 8 x i16> %a,
+                                                               <vscale x 8 x i16> %b)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @sabd_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: sabd_i32:
+; CHECK: sabd z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.sabd.nxv4i32(<vscale x 4 x i1> %pg,
+                                                               <vscale x 4 x i32> %a,
+                                                               <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @sabd_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: sabd_i64:
+; CHECK: sabd z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.sabd.nxv2i64(<vscale x 2 x i1> %pg,
+                                                               <vscale x 2 x i64> %a,
+                                                               <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i64> %out
+}
+
+define <vscale x 16 x i8> @uabd_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: uabd_i8:
+; CHECK: uabd z0.b, p0/m, z0.b, z1.b
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.uabd.nxv16i8(<vscale x 16 x i1> %pg,
+                                                               <vscale x 16 x i8> %a,
+                                                               <vscale x 16 x i8> %b)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @uabd_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: uabd_i16:
+; CHECK: uabd z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.uabd.nxv8i16(<vscale x 8 x i1> %pg,
+                                                               <vscale x 8 x i16> %a,
+                                                               <vscale x 8 x i16> %b)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @uabd_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: uabd_i32:
+; CHECK: uabd z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.uabd.nxv4i32(<vscale x 4 x i1> %pg,
+                                                               <vscale x 4 x i32> %a,
+                                                               <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @uabd_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: uabd_i64:
+; CHECK: uabd z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.uabd.nxv2i64(<vscale x 2 x i1> %pg,
+                                                               <vscale x 2 x i64> %a,
+                                                               <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i64> %out
+}
+
+declare <vscale x 16 x  i8> @llvm.aarch64.sve.add.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x  i8>, <vscale x 16 x  i8>)
+declare <vscale x  8 x i16> @llvm.aarch64.sve.add.nxv8i16(<vscale x  8 x i1>, <vscale x  8 x i16>, <vscale x  8 x i16>)
+declare <vscale x  4 x i32> @llvm.aarch64.sve.add.nxv4i32(<vscale x  4 x i1>, <vscale x  4 x i32>, <vscale x  4 x i32>)
+declare <vscale x  2 x i64> @llvm.aarch64.sve.add.nxv2i64(<vscale x  2 x i1>, <vscale x  2 x i64>, <vscale x  2 x i64>)
+
+declare <vscale x 16 x  i8> @llvm.aarch64.sve.sub.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x  i8>, <vscale x 16 x  i8>)
+declare <vscale x  8 x i16> @llvm.aarch64.sve.sub.nxv8i16(<vscale x  8 x i1>, <vscale x  8 x i16>, <vscale x  8 x i16>)
+declare <vscale x  4 x i32> @llvm.aarch64.sve.sub.nxv4i32(<vscale x  4 x i1>, <vscale x  4 x i32>, <vscale x  4 x i32>)
+declare <vscale x  2 x i64> @llvm.aarch64.sve.sub.nxv2i64(<vscale x  2 x i1>, <vscale x  2 x i64>, <vscale x  2 x i64>)
+
+declare <vscale x 16 x  i8> @llvm.aarch64.sve.subr.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x  i8>, <vscale x 16 x  i8>)
+declare <vscale x  8 x i16> @llvm.aarch64.sve.subr.nxv8i16(<vscale x  8 x i1>, <vscale x  8 x i16>, <vscale x  8 x i16>)
+declare <vscale x  4 x i32> @llvm.aarch64.sve.subr.nxv4i32(<vscale x  4 x i1>, <vscale x  4 x i32>, <vscale x  4 x i32>)
+declare <vscale x  2 x i64> @llvm.aarch64.sve.subr.nxv2i64(<vscale x  2 x i1>, <vscale x  2 x i64>, <vscale x  2 x i64>)
+
+declare <vscale x 16 x  i8> @llvm.aarch64.sve.smax.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x  i8>, <vscale x 16 x  i8>)
+declare <vscale x  8 x i16> @llvm.aarch64.sve.smax.nxv8i16(<vscale x  8 x i1>, <vscale x  8 x i16>, <vscale x  8 x i16>)
+declare <vscale x  4 x i32> @llvm.aarch64.sve.smax.nxv4i32(<vscale x  4 x i1>, <vscale x  4 x i32>, <vscale x  4 x i32>)
+declare <vscale x  2 x i64> @llvm.aarch64.sve.smax.nxv2i64(<vscale x  2 x i1>, <vscale x  2 x i64>, <vscale x  2 x i64>)
+
+declare <vscale x 16 x  i8> @llvm.aarch64.sve.umax.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x  i8>, <vscale x 16 x  i8>)
+declare <vscale x  8 x i16> @llvm.aarch64.sve.umax.nxv8i16(<vscale x  8 x i1>, <vscale x  8 x i16>, <vscale x  8 x i16>)
+declare <vscale x  4 x i32> @llvm.aarch64.sve.umax.nxv4i32(<vscale x  4 x i1>, <vscale x  4 x i32>, <vscale x  4 x i32>)
+declare <vscale x  2 x i64> @llvm.aarch64.sve.umax.nxv2i64(<vscale x  2 x i1>, <vscale x  2 x i64>, <vscale x  2 x i64>)
+
+declare <vscale x 16 x  i8> @llvm.aarch64.sve.smin.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x  i8>, <vscale x 16 x  i8>)
+declare <vscale x  8 x i16> @llvm.aarch64.sve.smin.nxv8i16(<vscale x  8 x i1>, <vscale x  8 x i16>, <vscale x  8 x i16>)
+declare <vscale x  4 x i32> @llvm.aarch64.sve.smin.nxv4i32(<vscale x  4 x i1>, <vscale x  4 x i32>, <vscale x  4 x i32>)
+declare <vscale x  2 x i64> @llvm.aarch64.sve.smin.nxv2i64(<vscale x  2 x i1>, <vscale x  2 x i64>, <vscale x  2 x i64>)
+
+declare <vscale x 16 x  i8> @llvm.aarch64.sve.umin.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x  i8>, <vscale x 16 x  i8>)
+declare <vscale x  8 x i16> @llvm.aarch64.sve.umin.nxv8i16(<vscale x  8 x i1>, <vscale x  8 x i16>, <vscale x  8 x i16>)
+declare <vscale x  4 x i32> @llvm.aarch64.sve.umin.nxv4i32(<vscale x  4 x i1>, <vscale x  4 x i32>, <vscale x  4 x i32>)
+declare <vscale x  2 x i64> @llvm.aarch64.sve.umin.nxv2i64(<vscale x  2 x i1>, <vscale x  2 x i64>, <vscale x  2 x i64>)
+
+declare <vscale x 16 x  i8> @llvm.aarch64.sve.sabd.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x  i8>, <vscale x 16 x  i8>)
+declare <vscale x  8 x i16> @llvm.aarch64.sve.sabd.nxv8i16(<vscale x  8 x i1>, <vscale x  8 x i16>, <vscale x  8 x i16>)
+declare <vscale x  4 x i32> @llvm.aarch64.sve.sabd.nxv4i32(<vscale x  4 x i1>, <vscale x  4 x i32>, <vscale x  4 x i32>)
+declare <vscale x  2 x i64> @llvm.aarch64.sve.sabd.nxv2i64(<vscale x  2 x i1>, <vscale x  2 x i64>, <vscale x  2 x i64>)
+
+declare <vscale x 16 x  i8> @llvm.aarch64.sve.uabd.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x  i8>, <vscale x 16 x  i8>)
+declare <vscale x  8 x i16> @llvm.aarch64.sve.uabd.nxv8i16(<vscale x  8 x i1>, <vscale x  8 x i16>, <vscale x  8 x i16>)
+declare <vscale x  4 x i32> @llvm.aarch64.sve.uabd.nxv4i32(<vscale x  4 x i1>, <vscale x  4 x i32>, <vscale x  4 x i32>)
+declare <vscale x  2 x i64> @llvm.aarch64.sve.uabd.nxv2i64(<vscale x  2 x i1>, <vscale x  2 x i64>, <vscale x  2 x i64>)
diff --git a/llvm/test/CodeGen/AArch64/sve-int-arith.ll b/llvm/test/CodeGen/AArch64/sve-int-arith.ll
new file mode 100644
index 000000000..def788842
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-int-arith.ll
@@ -0,0 +1,216 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+define <vscale x 2 x i64> @add_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: add_i64
+; CHECK: add z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+  %res = add <vscale x 2 x i64> %a, %b
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 4 x i32> @add_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: add_i32
+; CHECK: add z0.s, z0.s, z1.s
+; CHECK-NEXT: ret
+  %res = add <vscale x 4 x i32> %a, %b
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 8 x i16> @add_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: add_i16
+; CHECK: add z0.h, z0.h, z1.h
+; CHECK-NEXT: ret
+  %res = add <vscale x 8 x i16> %a, %b
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 16 x i8> @add_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: add_i8
+; CHECK: add z0.b, z0.b, z1.b
+; CHECK-NEXT: ret
+  %res = add <vscale x 16 x i8> %a, %b
+  ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 2 x i64> @sub_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: sub_i64
+; CHECK: sub z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+  %res = sub <vscale x 2 x i64> %a, %b
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 4 x i32> @sub_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: sub_i32
+; CHECK: sub z0.s, z0.s, z1.s
+; CHECK-NEXT: ret
+  %res = sub <vscale x 4 x i32> %a, %b
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 8 x i16> @sub_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: sub_i16
+; CHECK: sub z0.h, z0.h, z1.h
+; CHECK-NEXT: ret
+  %res = sub <vscale x 8 x i16> %a, %b
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 16 x i8> @sub_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: sub_i8
+; CHECK: sub z0.b, z0.b, z1.b
+; CHECK-NEXT: ret
+  %res = sub <vscale x 16 x i8> %a, %b
+  ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 2 x i64> @sqadd_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: sqadd_i64
+; CHECK: sqadd  z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+  %res = call <vscale x 2 x i64> @llvm.sadd.sat.nxv2i64(<vscale x 2 x i64>  %a, <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 4 x i32> @sqadd_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: sqadd_i32
+; CHECK: sqadd  z0.s, z0.s, z1.s
+; CHECK-NEXT: ret
+  %res = call <vscale x 4 x i32> @llvm.sadd.sat.nxv4i32(<vscale x 4 x i32>  %a, <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 8 x i16> @sqadd_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: sqadd_i16
+; CHECK: sqadd  z0.h, z0.h, z1.h
+; CHECK-NEXT: ret
+  %res = call <vscale x 8 x i16> @llvm.sadd.sat.nxv8i16(<vscale x 8 x i16>  %a, <vscale x 8 x i16> %b)
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 16 x i8> @sqadd_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: sqadd_i8
+; CHECK: sqadd  z0.b, z0.b, z1.b
+; CHECK-NEXT: ret
+  %res = call <vscale x 16 x i8> @llvm.sadd.sat.nxv16i8(<vscale x 16 x i8>  %a, <vscale x 16 x i8> %b)
+  ret <vscale x 16 x i8> %res
+}
+
+
+define <vscale x 2 x i64> @sqsub_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: sqsub_i64
+; CHECK: sqsub  z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+  %res = call <vscale x 2 x i64> @llvm.ssub.sat.nxv2i64(<vscale x 2 x i64>  %a, <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 4 x i32> @sqsub_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: sqsub_i32
+; CHECK: sqsub  z0.s, z0.s, z1.s
+; CHECK-NEXT: ret
+  %res = call <vscale x 4 x i32> @llvm.ssub.sat.nxv4i32(<vscale x 4 x i32>  %a, <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 8 x i16> @sqsub_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: sqsub_i16
+; CHECK: sqsub  z0.h, z0.h, z1.h
+; CHECK-NEXT: ret
+  %res = call <vscale x 8 x i16> @llvm.ssub.sat.nxv8i16(<vscale x 8 x i16>  %a, <vscale x 8 x i16> %b)
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 16 x i8> @sqsub_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: sqsub_i8
+; CHECK: sqsub  z0.b, z0.b, z1.b
+; CHECK-NEXT: ret
+  %res = call <vscale x 16 x i8> @llvm.ssub.sat.nxv16i8(<vscale x 16 x i8>  %a, <vscale x 16 x i8> %b)
+  ret <vscale x 16 x i8> %res
+}
+
+
+define <vscale x 2 x i64> @uqadd_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: uqadd_i64
+; CHECK: uqadd  z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+  %res = call <vscale x 2 x i64> @llvm.uadd.sat.nxv2i64(<vscale x 2 x i64>  %a, <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 4 x i32> @uqadd_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: uqadd_i32
+; CHECK: uqadd  z0.s, z0.s, z1.s
+; CHECK-NEXT: ret
+  %res = call <vscale x 4 x i32> @llvm.uadd.sat.nxv4i32(<vscale x 4 x i32>  %a, <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 8 x i16> @uqadd_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: uqadd_i16
+; CHECK: uqadd  z0.h, z0.h, z1.h
+; CHECK-NEXT: ret
+  %res = call <vscale x 8 x i16> @llvm.uadd.sat.nxv8i16(<vscale x 8 x i16>  %a, <vscale x 8 x i16> %b)
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 16 x i8> @uqadd_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: uqadd_i8
+; CHECK: uqadd  z0.b, z0.b, z1.b
+; CHECK-NEXT: ret
+  %res = call <vscale x 16 x i8> @llvm.uadd.sat.nxv16i8(<vscale x 16 x i8>  %a, <vscale x 16 x i8> %b)
+  ret <vscale x 16 x i8> %res
+}
+
+
+define <vscale x 2 x i64> @uqsub_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: uqsub_i64
+; CHECK: uqsub  z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+  %res = call <vscale x 2 x i64> @llvm.usub.sat.nxv2i64(<vscale x 2 x i64>  %a, <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 4 x i32> @uqsub_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: uqsub_i32
+; CHECK: uqsub  z0.s, z0.s, z1.s
+; CHECK-NEXT: ret
+  %res = call <vscale x 4 x i32> @llvm.usub.sat.nxv4i32(<vscale x 4 x i32>  %a, <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 8 x i16> @uqsub_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: uqsub_i16
+; CHECK: uqsub  z0.h, z0.h, z1.h
+; CHECK-NEXT: ret
+  %res = call <vscale x 8 x i16> @llvm.usub.sat.nxv8i16(<vscale x 8 x i16>  %a, <vscale x 8 x i16> %b)
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 16 x i8> @uqsub_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: uqsub_i8
+; CHECK: uqsub  z0.b, z0.b, z1.b
+; CHECK-NEXT: ret
+  %res = call <vscale x 16 x i8> @llvm.usub.sat.nxv16i8(<vscale x 16 x i8>  %a, <vscale x 16 x i8> %b)
+  ret <vscale x 16 x i8> %res
+}
+
+declare <vscale x 16 x i8> @llvm.sadd.sat.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i8>)
+declare <vscale x 8 x i16> @llvm.sadd.sat.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i16>)
+declare <vscale x 4 x i32> @llvm.sadd.sat.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>)
+declare <vscale x 2 x i64> @llvm.sadd.sat.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>)
+
+declare <vscale x 16 x i8> @llvm.ssub.sat.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i8>)
+declare <vscale x 8 x i16> @llvm.ssub.sat.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i16>)
+declare <vscale x 4 x i32> @llvm.ssub.sat.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>)
+declare <vscale x 2 x i64> @llvm.ssub.sat.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>)
+
+declare <vscale x 16 x i8> @llvm.uadd.sat.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i8>)
+declare <vscale x 8 x i16> @llvm.uadd.sat.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i16>)
+declare <vscale x 4 x i32> @llvm.uadd.sat.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>)
+declare <vscale x 2 x i64> @llvm.uadd.sat.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>)
+
+declare <vscale x 16 x i8> @llvm.usub.sat.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i8>)
+declare <vscale x 8 x i16> @llvm.usub.sat.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i16>)
+declare <vscale x 4 x i32> @llvm.usub.sat.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>)
+declare <vscale x 2 x i64> @llvm.usub.sat.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>)
diff --git a/llvm/test/CodeGen/AArch64/sve-int-div-pred.ll b/llvm/test/CodeGen/AArch64/sve-int-div-pred.ll
new file mode 100644
index 000000000..dd25f27ab
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-int-div-pred.ll
@@ -0,0 +1,91 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+define <vscale x 4 x i32> @sdiv_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: sdiv_i32:
+; CHECK: sdiv z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.sdiv.nxv4i32(<vscale x 4 x i1> %pg,
+                                                               <vscale x 4 x i32> %a,
+                                                               <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @sdiv_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: sdiv_i64:
+; CHECK: sdiv z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.sdiv.nxv2i64(<vscale x 2 x i1> %pg,
+                                                               <vscale x 2 x i64> %a,
+                                                               <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i64> %out
+}
+
+define <vscale x 4 x i32> @udiv_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: udiv_i32:
+; CHECK: udiv z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.udiv.nxv4i32(<vscale x 4 x i1> %pg,
+                                                               <vscale x 4 x i32> %a,
+                                                               <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @udiv_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: udiv_i64:
+; CHECK: udiv z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.udiv.nxv2i64(<vscale x 2 x i1> %pg,
+                                                               <vscale x 2 x i64> %a,
+                                                               <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i64> %out
+}
+
+define <vscale x 4 x i32> @sdivr_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: sdivr_i32:
+; CHECK: sdivr z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.sdivr.nxv4i32(<vscale x 4 x i1> %pg,
+                                                               <vscale x 4 x i32> %a,
+                                                               <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @sdivr_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: sdivr_i64:
+; CHECK: sdivr z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.sdivr.nxv2i64(<vscale x 2 x i1> %pg,
+                                                               <vscale x 2 x i64> %a,
+                                                               <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i64> %out
+}
+
+define <vscale x 4 x i32> @udivr_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: udivr_i32:
+; CHECK: udivr z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.udivr.nxv4i32(<vscale x 4 x i1> %pg,
+                                                               <vscale x 4 x i32> %a,
+                                                               <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @udivr_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: udivr_i64:
+; CHECK: udivr z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.udivr.nxv2i64(<vscale x 2 x i1> %pg,
+                                                               <vscale x 2 x i64> %a,
+                                                               <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i64> %out
+}
+
+declare <vscale x  4 x i32> @llvm.aarch64.sve.sdiv.nxv4i32(<vscale x  4 x i1>, <vscale x  4 x i32>, <vscale x  4 x i32>)
+declare <vscale x  2 x i64> @llvm.aarch64.sve.sdiv.nxv2i64(<vscale x  2 x i1>, <vscale x  2 x i64>, <vscale x  2 x i64>)
+declare <vscale x  4 x i32> @llvm.aarch64.sve.udiv.nxv4i32(<vscale x  4 x i1>, <vscale x  4 x i32>, <vscale x  4 x i32>)
+declare <vscale x  2 x i64> @llvm.aarch64.sve.udiv.nxv2i64(<vscale x  2 x i1>, <vscale x  2 x i64>, <vscale x  2 x i64>)
+declare <vscale x  4 x i32> @llvm.aarch64.sve.sdivr.nxv4i32(<vscale x  4 x i1>, <vscale x  4 x i32>, <vscale x  4 x i32>)
+declare <vscale x  2 x i64> @llvm.aarch64.sve.sdivr.nxv2i64(<vscale x  2 x i1>, <vscale x  2 x i64>, <vscale x  2 x i64>)
+declare <vscale x  4 x i32> @llvm.aarch64.sve.udivr.nxv4i32(<vscale x  4 x i1>, <vscale x  4 x i32>, <vscale x  4 x i32>)
+declare <vscale x  2 x i64> @llvm.aarch64.sve.udivr.nxv2i64(<vscale x  2 x i1>, <vscale x  2 x i64>, <vscale x  2 x i64>)
+
diff --git a/llvm/test/CodeGen/AArch64/sve-int-imm.ll b/llvm/test/CodeGen/AArch64/sve-int-imm.ll
new file mode 100644
index 000000000..57d9540c2
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-int-imm.ll
@@ -0,0 +1,519 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+;
+; SVE Arith Vector Immediate Unpredicated CodeGen
+;
+
+; ADD
+define <vscale x 16 x i8> @add_i8_low(<vscale x 16 x i8> %a) {
+; CHECK-LABEL: add_i8_low
+; CHECK: add  z0.b, z0.b, #30
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 16 x i8> undef, i8 30, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %res =  add <vscale x 16 x i8> %a, %splat
+  ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 8 x i16> @add_i16_low(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: add_i16_low
+; CHECK: add  z0.h, z0.h, #30
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 8 x i16> undef, i16 30, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %res =  add <vscale x 8 x i16> %a, %splat
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 8 x i16> @add_i16_high(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: add_i16_high
+; CHECK: add  z0.h, z0.h, #1024
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 8 x i16> undef, i16 1024, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %res =  add <vscale x 8 x i16> %a, %splat
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 4 x i32> @add_i32_low(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: add_i32_low
+; CHECK: add  z0.s, z0.s, #30
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 4 x i32> undef, i32 30, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %res = add <vscale x 4 x i32> %a, %splat
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 4 x i32> @add_i32_high(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: add_i32_high
+; CHECK: add  z0.s, z0.s, #1024
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 4 x i32> undef, i32 1024, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %res =  add <vscale x 4 x i32> %a, %splat
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @add_i64_low(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: add_i64_low
+; CHECK: add  z0.d, z0.d, #30
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 2 x i64> undef, i64 30, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %res =  add <vscale x 2 x i64> %a, %splat
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @add_i64_high(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: add_i64_high
+; CHECK: add  z0.d, z0.d, #1024
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 2 x i64> undef, i64 1024, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %res = add <vscale x 2 x i64> %a, %splat
+  ret <vscale x 2 x i64> %res
+}
+
+; SUBR
+define <vscale x 16 x i8> @subr_i8_low(<vscale x 16 x i8> %a) {
+; CHECK-LABEL: subr_i8_low
+; CHECK: subr  z0.b, z0.b, #30
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 16 x i8> undef, i8 30, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %res =  sub <vscale x 16 x i8> %splat, %a
+  ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 8 x i16> @subr_i16_low(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: subr_i16_low
+; CHECK: subr  z0.h, z0.h, #30
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 8 x i16> undef, i16 30, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %res =  sub <vscale x 8 x i16> %splat, %a
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 8 x i16> @subr_i16_high(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: subr_i16_high
+; CHECK: subr  z0.h, z0.h, #1024
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 8 x i16> undef, i16 1024, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %res =  sub <vscale x 8 x i16> %splat, %a
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 4 x i32> @subr_i32_low(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: subr_i32_low
+; CHECK: subr  z0.s, z0.s, #30
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 4 x i32> undef, i32 30, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %res =  sub <vscale x 4 x i32> %splat, %a
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 4 x i32> @subr_i32_high(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: subr_i32_high
+; CHECK: subr  z0.s, z0.s, #1024
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 4 x i32> undef, i32 1024, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %res =  sub <vscale x 4 x i32> %splat, %a
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @subr_i64_low(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: subr_i64_low
+; CHECK: subr  z0.d, z0.d, #30
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 2 x i64> undef, i64 30, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %res =  sub <vscale x 2 x i64> %splat, %a
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @subr_i64_high(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: subr_i64_high
+; CHECK: subr  z0.d, z0.d, #1024
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 2 x i64> undef, i64 1024, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %res =  sub <vscale x 2 x i64> %splat, %a
+  ret <vscale x 2 x i64> %res
+}
+
+; SUB
+define <vscale x 16 x i8> @sub_i8_low(<vscale x 16 x i8> %a) {
+; CHECK-LABEL: sub_i8_low
+; CHECK: sub  z0.b, z0.b, #30
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 16 x i8> undef, i8 30, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %res =  sub <vscale x 16 x i8> %a, %splat
+  ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 8 x i16> @sub_i16_low(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: sub_i16_low
+; CHECK: sub  z0.h, z0.h, #30
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 8 x i16> undef, i16 30, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %res =  sub <vscale x 8 x i16> %a, %splat
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 8 x i16> @sub_i16_high(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: sub_i16_high
+; CHECK: sub  z0.h, z0.h, #1024
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 8 x i16> undef, i16 1024, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %res =  sub <vscale x 8 x i16> %a, %splat
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 4 x i32> @sub_i32_low(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: sub_i32_low
+; CHECK: sub  z0.s, z0.s, #30
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 4 x i32> undef, i32 30, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %res = sub <vscale x 4 x i32> %a, %splat
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 4 x i32> @sub_i32_high(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: sub_i32_high
+; CHECK: sub  z0.s, z0.s, #1024
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 4 x i32> undef, i32 1024, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %res =  sub <vscale x 4 x i32> %a, %splat
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @sub_i64_low(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: sub_i64_low
+; CHECK: sub  z0.d, z0.d, #30
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 2 x i64> undef, i64 30, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %res =  sub <vscale x 2 x i64> %a, %splat
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @sub_i64_high(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: sub_i64_high
+; CHECK: sub  z0.d, z0.d, #1024
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 2 x i64> undef, i64 1024, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %res = sub <vscale x 2 x i64> %a, %splat
+  ret <vscale x 2 x i64> %res
+}
+
+; SQADD
+define <vscale x 16 x i8> @sqadd_i8_low(<vscale x 16 x i8> %a) {
+; CHECK-LABEL: sqadd_i8_low
+; CHECK: sqadd  z0.b, z0.b, #30
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 16 x i8> undef, i8 30, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %res =  call <vscale x 16 x i8> @llvm.sadd.sat.nxv16i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %splat)
+  ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 8 x i16> @sqadd_i16_low(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: sqadd_i16_low
+; CHECK: sqadd  z0.h, z0.h, #30
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 8 x i16> undef, i16 30, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %res =  call <vscale x 8 x i16> @llvm.sadd.sat.nxv8i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %splat)
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 8 x i16> @sqadd_i16_high(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: sqadd_i16_high
+; CHECK: sqadd  z0.h, z0.h, #1024
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 8 x i16> undef, i16 1024, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %res =  call <vscale x 8 x i16> @llvm.sadd.sat.nxv8i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %splat)
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 4 x i32> @sqadd_i32_low(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: sqadd_i32_low
+; CHECK: sqadd  z0.s, z0.s, #30
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 4 x i32> undef, i32 30, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %res =  call <vscale x 4 x i32> @llvm.sadd.sat.nxv4i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %splat)
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 4 x i32> @sqadd_i32_high(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: sqadd_i32_high
+; CHECK: sqadd  z0.s, z0.s, #1024
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 4 x i32> undef, i32 1024, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %res =  call <vscale x 4 x i32> @llvm.sadd.sat.nxv4i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %splat)
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @sqadd_i64_low(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: sqadd_i64_low
+; CHECK: sqadd  z0.d, z0.d, #30
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 2 x i64> undef, i64 30, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %res =  call <vscale x 2 x i64> @llvm.sadd.sat.nxv2i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %splat)
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @sqadd_i64_high(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: sqadd_i64_high
+; CHECK: sqadd  z0.d, z0.d, #1024
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 2 x i64> undef, i64 1024, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %res =  call <vscale x 2 x i64> @llvm.sadd.sat.nxv2i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %splat)
+  ret <vscale x 2 x i64> %res
+}
+
+; UQADD
+define <vscale x 16 x i8> @uqadd_i8_low(<vscale x 16 x i8> %a) {
+; CHECK-LABEL: uqadd_i8_low
+; CHECK: uqadd  z0.b, z0.b, #30
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 16 x i8> undef, i8 30, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %res =  call <vscale x 16 x i8> @llvm.uadd.sat.nxv16i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %splat)
+  ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 8 x i16> @uqadd_i16_low(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: uqadd_i16_low
+; CHECK: uqadd  z0.h, z0.h, #30
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 8 x i16> undef, i16 30, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %res =  call <vscale x 8 x i16> @llvm.uadd.sat.nxv8i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %splat)
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 8 x i16> @uqadd_i16_high(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: uqadd_i16_high
+; CHECK: uqadd  z0.h, z0.h, #1024
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 8 x i16> undef, i16 1024, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %res =  call <vscale x 8 x i16> @llvm.uadd.sat.nxv8i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %splat)
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 4 x i32> @uqadd_i32_low(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: uqadd_i32_low
+; CHECK: uqadd  z0.s, z0.s, #30
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 4 x i32> undef, i32 30, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %res =  call <vscale x 4 x i32> @llvm.uadd.sat.nxv4i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %splat)
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 4 x i32> @uqadd_i32_high(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: uqadd_i32_high
+; CHECK: uqadd  z0.s, z0.s, #1024
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 4 x i32> undef, i32 1024, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %res =  call <vscale x 4 x i32> @llvm.uadd.sat.nxv4i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %splat)
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @uqadd_i64_low(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: uqadd_i64_low
+; CHECK: uqadd  z0.d, z0.d, #30
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 2 x i64> undef, i64 30, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %res =  call <vscale x 2 x i64> @llvm.uadd.sat.nxv2i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %splat)
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @uqadd_i64_high(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: uqadd_i64_high
+; CHECK: uqadd  z0.d, z0.d, #1024
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 2 x i64> undef, i64 1024, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %res =  call <vscale x 2 x i64> @llvm.uadd.sat.nxv2i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %splat)
+  ret <vscale x 2 x i64> %res
+}
+
+; SQSUB
+define <vscale x 16 x i8> @sqsub_i8_low(<vscale x 16 x i8> %a) {
+; CHECK-LABEL: sqsub_i8_low
+; CHECK: sqsub  z0.b, z0.b, #30
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 16 x i8> undef, i8 30, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %res =  call <vscale x 16 x i8> @llvm.ssub.sat.nxv16i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %splat)
+  ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 8 x i16> @sqsub_i16_low(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: sqsub_i16_low
+; CHECK: sqsub  z0.h, z0.h, #30
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 8 x i16> undef, i16 30, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %res =  call <vscale x 8 x i16> @llvm.ssub.sat.nxv8i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %splat)
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 8 x i16> @sqsub_i16_high(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: sqsub_i16_high
+; CHECK: sqsub  z0.h, z0.h, #1024
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 8 x i16> undef, i16 1024, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %res =  call <vscale x 8 x i16> @llvm.ssub.sat.nxv8i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %splat)
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 4 x i32> @sqsub_i32_low(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: sqsub_i32_low
+; CHECK: sqsub  z0.s, z0.s, #30
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 4 x i32> undef, i32 30, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %res =  call <vscale x 4 x i32> @llvm.ssub.sat.nxv4i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %splat)
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 4 x i32> @sqsub_i32_high(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: sqsub_i32_high
+; CHECK: sqsub  z0.s, z0.s, #1024
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 4 x i32> undef, i32 1024, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %res =  call <vscale x 4 x i32> @llvm.ssub.sat.nxv4i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %splat)
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @sqsub_i64_low(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: sqsub_i64_low
+; CHECK: sqsub  z0.d, z0.d, #30
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 2 x i64> undef, i64 30, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %res =  call <vscale x 2 x i64> @llvm.ssub.sat.nxv2i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %splat)
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @sqsub_i64_high(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: sqsub_i64_high
+; CHECK: sqsub  z0.d, z0.d, #1024
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 2 x i64> undef, i64 1024, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %res =  call <vscale x 2 x i64> @llvm.ssub.sat.nxv2i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %splat)
+  ret <vscale x 2 x i64> %res
+}
+
+; UQSUB
+define <vscale x 16 x i8> @uqsub_i8_low(<vscale x 16 x i8> %a) {
+; CHECK-LABEL: uqsub_i8_low
+; CHECK: uqsub  z0.b, z0.b, #30
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 16 x i8> undef, i8 30, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %res =  call <vscale x 16 x i8> @llvm.usub.sat.nxv16i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %splat)
+  ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 8 x i16> @uqsub_i16_low(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: uqsub_i16_low
+; CHECK: uqsub  z0.h, z0.h, #30
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 8 x i16> undef, i16 30, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %res =  call <vscale x 8 x i16> @llvm.usub.sat.nxv8i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %splat)
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 8 x i16> @uqsub_i16_high(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: uqsub_i16_high
+; CHECK: uqsub  z0.h, z0.h, #1024
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 8 x i16> undef, i16 1024, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %res =  call <vscale x 8 x i16> @llvm.usub.sat.nxv8i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %splat)
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 4 x i32> @uqsub_i32_low(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: uqsub_i32_low
+; CHECK: uqsub  z0.s, z0.s, #30
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 4 x i32> undef, i32 30, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %res =  call <vscale x 4 x i32> @llvm.usub.sat.nxv4i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %splat)
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 4 x i32> @uqsub_i32_high(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: uqsub_i32_high
+; CHECK: uqsub  z0.s, z0.s, #1024
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 4 x i32> undef, i32 1024, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %res =  call <vscale x 4 x i32> @llvm.usub.sat.nxv4i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %splat)
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @uqsub_i64_low(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: uqsub_i64_low
+; CHECK: uqsub  z0.d, z0.d, #30
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 2 x i64> undef, i64 30, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %res =  call <vscale x 2 x i64> @llvm.usub.sat.nxv2i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %splat)
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @uqsub_i64_high(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: uqsub_i64_high
+; CHECK: uqsub  z0.d, z0.d, #1024
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 2 x i64> undef, i64 1024, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %res =  call <vscale x 2 x i64> @llvm.usub.sat.nxv2i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %splat)
+  ret <vscale x 2 x i64> %res
+}
+
+declare <vscale x 16 x i8> @llvm.sadd.sat.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i8>)
+declare <vscale x 8 x i16> @llvm.sadd.sat.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i16>)
+declare <vscale x 4 x i32> @llvm.sadd.sat.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>)
+declare <vscale x 2 x i64> @llvm.sadd.sat.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>)
+declare <vscale x 16 x i8> @llvm.uadd.sat.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i8>)
+declare <vscale x 8 x i16> @llvm.uadd.sat.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i16>)
+declare <vscale x 4 x i32> @llvm.uadd.sat.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>)
+declare <vscale x 2 x i64> @llvm.uadd.sat.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>)
+declare <vscale x 16 x i8> @llvm.ssub.sat.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i8>)
+declare <vscale x 8 x i16> @llvm.ssub.sat.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i16>)
+declare <vscale x 4 x i32> @llvm.ssub.sat.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>)
+declare <vscale x 2 x i64> @llvm.ssub.sat.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>)
+declare <vscale x 16 x i8> @llvm.usub.sat.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i8>)
+declare <vscale x 8 x i16> @llvm.usub.sat.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i16>)
+declare <vscale x 4 x i32> @llvm.usub.sat.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>)
+declare <vscale x 2 x i64> @llvm.usub.sat.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>)
diff --git a/llvm/test/CodeGen/AArch64/sve-int-log-imm.ll b/llvm/test/CodeGen/AArch64/sve-int-log-imm.ll
new file mode 100644
index 000000000..52b56d5ad
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-int-log-imm.ll
@@ -0,0 +1,128 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+;
+; SVE Logical Vector Immediate Unpredicated CodeGen
+;
+
+; ORR
+define <vscale x 16 x i8> @orr_i8(<vscale x 16 x i8> %a) {
+; CHECK-LABEL: orr_i8:
+; CHECK: orr z0.b, z0.b, #0xf
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 16 x i8> undef, i8 15, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %res = or <vscale x 16 x i8> %a, %splat
+  ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 8 x i16> @orr_i16(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: orr_i16:
+; CHECK: orr z0.h, z0.h, #0xfc07
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 8 x i16> undef, i16 64519, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %res = or <vscale x 8 x i16> %a, %splat
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 4 x i32> @orr_i32(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: orr_i32:
+; CHECK: orr z0.s, z0.s, #0xffff00
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 4 x i32> undef, i32 16776960, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %res = or <vscale x 4 x i32> %a, %splat
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @orr_i64(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: orr_i64:
+; CHECK: orr z0.d, z0.d, #0xfffc000000000000
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 2 x i64> undef, i64 18445618173802708992, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %res = or <vscale x 2 x i64> %a, %splat
+  ret <vscale x 2 x i64> %res
+}
+
+; EOR
+define <vscale x 16 x i8> @eor_i8(<vscale x 16 x i8> %a) {
+; CHECK-LABEL: eor_i8:
+; CHECK: eor z0.b, z0.b, #0xf
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 16 x i8> undef, i8 15, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %res = xor <vscale x 16 x i8> %a, %splat
+  ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 8 x i16> @eor_i16(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: eor_i16:
+; CHECK: eor z0.h, z0.h, #0xfc07
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 8 x i16> undef, i16 64519, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %res = xor <vscale x 8 x i16> %a, %splat
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 4 x i32> @eor_i32(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: eor_i32:
+; CHECK: eor z0.s, z0.s, #0xffff00
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 4 x i32> undef, i32 16776960, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %res = xor <vscale x 4 x i32> %a, %splat
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @eor_i64(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: eor_i64:
+; CHECK: eor z0.d, z0.d, #0xfffc000000000000
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 2 x i64> undef, i64 18445618173802708992, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %res = xor <vscale x 2 x i64> %a, %splat
+  ret <vscale x 2 x i64> %res
+}
+
+; AND
+define <vscale x 16 x i8> @and_i8(<vscale x 16 x i8> %a) {
+; CHECK-LABEL: and_i8:
+; CHECK: and z0.b, z0.b, #0xf
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 16 x i8> undef, i8 15, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %res = and <vscale x 16 x i8> %a, %splat
+  ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 8 x i16> @and_i16(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: and_i16:
+; CHECK: and z0.h, z0.h, #0xfc07
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 8 x i16> undef, i16 64519, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %res = and <vscale x 8 x i16> %a, %splat
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 4 x i32> @and_i32(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: and_i32:
+; CHECK: and z0.s, z0.s, #0xffff00
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 4 x i32> undef, i32 16776960, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %res = and <vscale x 4 x i32> %a, %splat
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @and_i64(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: and_i64:
+; CHECK: and z0.d, z0.d, #0xfffc000000000000
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 2 x i64> undef, i64 18445618173802708992, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %res = and <vscale x 2 x i64> %a, %splat
+  ret <vscale x 2 x i64> %res
+}
diff --git a/llvm/test/CodeGen/AArch64/sve-int-log-pred.ll b/llvm/test/CodeGen/AArch64/sve-int-log-pred.ll
new file mode 100644
index 000000000..fc158d755
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-int-log-pred.ll
@@ -0,0 +1,179 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+define <vscale x 16 x i8> @and_pred_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: and_pred_i8:
+; CHECK: and z0.b, p0/m, z0.b, z1.b
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.and.nxv2i8(<vscale x 16 x i1> %pg,
+                                                              <vscale x 16 x i8> %a,
+                                                              <vscale x 16 x i8> %b)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @and_pred_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: and_pred_i16:
+; CHECK: and z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.and.nxv2i16(<vscale x 8 x i1> %pg,
+                                                               <vscale x 8 x i16> %a,
+                                                               <vscale x 8 x i16> %b)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @and_pred_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: and_pred_i32:
+; CHECK: and z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.and.nxv2i32(<vscale x 4 x i1> %pg,
+                                                               <vscale x 4 x i32> %a,
+                                                               <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @and_pred_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: and_pred_i64:
+; CHECK: and z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.and.nxv2i64(<vscale x 2 x i1> %pg,
+                                                               <vscale x 2 x i64> %a,
+                                                               <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i64> %out
+}
+
+define <vscale x 16 x i8> @or_pred_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: or_pred_i8:
+; CHECK: orr z0.b, p0/m, z0.b, z1.b
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.orr.nxv2i8(<vscale x 16 x i1> %pg,
+                                                              <vscale x 16 x i8> %a,
+                                                              <vscale x 16 x i8> %b)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @or_pred_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: or_pred_i16:
+; CHECK: orr z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.orr.nxv2i16(<vscale x 8 x i1> %pg,
+                                                               <vscale x 8 x i16> %a,
+                                                               <vscale x 8 x i16> %b)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @or_pred_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: or_pred_i32:
+; CHECK: orr z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.orr.nxv2i32(<vscale x 4 x i1> %pg,
+                                                               <vscale x 4 x i32> %a,
+                                                               <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @or_pred_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: or_pred_i64:
+; CHECK: orr z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.orr.nxv2i64(<vscale x 2 x i1> %pg,
+                                                               <vscale x 2 x i64> %a,
+                                                               <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i64> %out
+}
+
+define <vscale x 16 x i8> @xor_pred_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: xor_pred_i8:
+; CHECK: eor z0.b, p0/m, z0.b, z1.b
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.eor.nxv2i8(<vscale x 16 x i1> %pg,
+                                                              <vscale x 16 x i8> %a,
+                                                              <vscale x 16 x i8> %b)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @xor_pred_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: xor_pred_i16:
+; CHECK: eor z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.eor.nxv2i16(<vscale x 8 x i1> %pg,
+                                                               <vscale x 8 x i16> %a,
+                                                               <vscale x 8 x i16> %b)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @xor_pred_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: xor_pred_i32:
+; CHECK: eor z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.eor.nxv2i32(<vscale x 4 x i1> %pg,
+                                                               <vscale x 4 x i32> %a,
+                                                               <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @xor_pred_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: xor_pred_i64:
+; CHECK: eor z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.eor.nxv2i64(<vscale x 2 x i1> %pg,
+                                                               <vscale x 2 x i64> %a,
+                                                               <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i64> %out
+}
+
+define <vscale x 16 x i8> @bic_pred_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: bic_pred_i8:
+; CHECK: bic z0.b, p0/m, z0.b, z1.b
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.bic.nxv2i8(<vscale x 16 x i1> %pg,
+                                                              <vscale x 16 x i8> %a,
+                                                              <vscale x 16 x i8> %b)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @bic_pred_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: bic_pred_i16:
+; CHECK: bic z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.bic.nxv2i16(<vscale x 8 x i1> %pg,
+                                                               <vscale x 8 x i16> %a,
+                                                               <vscale x 8 x i16> %b)
+  ret <vscale x 8 x i16> %out
+}
+
+
+define <vscale x 4 x i32> @bic_pred_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: bic_pred_i32:
+; CHECK: bic z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.bic.nxv2i32(<vscale x 4 x i1> %pg,
+                                                               <vscale x 4 x i32> %a,
+                                                               <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @bic_pred_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: bic_pred_i64:
+; CHECK: bic z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.bic.nxv2i64(<vscale x 2 x i1> %pg,
+                                                               <vscale x 2 x i64> %a,
+                                                               <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i64> %out
+}
+
+declare <vscale x 16 x i8> @llvm.aarch64.sve.and.nxv2i8(<vscale x 16 x i1>,<vscale x 16 x i8>,<vscale x 16 x i8>)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.and.nxv2i16(<vscale x 8 x i1>,<vscale x 8 x i16>,<vscale x 8 x i16>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.and.nxv2i32(<vscale x 4 x i1>,<vscale x 4 x i32>,<vscale x 4 x i32>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.and.nxv2i64(<vscale x 2 x i1>,<vscale x 2 x i64>,<vscale x 2 x i64>)
+declare <vscale x 16 x i8> @llvm.aarch64.sve.orr.nxv2i8(<vscale x 16 x i1>,<vscale x 16 x i8>,<vscale x 16 x i8>)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.orr.nxv2i16(<vscale x 8 x i1>,<vscale x 8 x i16>,<vscale x 8 x i16>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.orr.nxv2i32(<vscale x 4 x i1>,<vscale x 4 x i32>,<vscale x 4 x i32>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.orr.nxv2i64(<vscale x 2 x i1>,<vscale x 2 x i64>,<vscale x 2 x i64>)
+declare <vscale x 16 x i8> @llvm.aarch64.sve.eor.nxv2i8(<vscale x 16 x i1>,<vscale x 16 x i8>,<vscale x 16 x i8>)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.eor.nxv2i16(<vscale x 8 x i1>,<vscale x 8 x i16>,<vscale x 8 x i16>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.eor.nxv2i32(<vscale x 4 x i1>,<vscale x 4 x i32>,<vscale x 4 x i32>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.eor.nxv2i64(<vscale x 2 x i1>,<vscale x 2 x i64>,<vscale x 2 x i64>)
+declare <vscale x 16 x i8> @llvm.aarch64.sve.bic.nxv2i8(<vscale x 16 x i1>,<vscale x 16 x i8>,<vscale x 16 x i8>)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.bic.nxv2i16(<vscale x 8 x i1>,<vscale x 8 x i16>,<vscale x 8 x i16>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.bic.nxv2i32(<vscale x 4 x i1>,<vscale x 4 x i32>,<vscale x 4 x i32>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.bic.nxv2i64(<vscale x 2 x i1>,<vscale x 2 x i64>,<vscale x 2 x i64>)
diff --git a/llvm/test/CodeGen/AArch64/sve-int-log.ll b/llvm/test/CodeGen/AArch64/sve-int-log.ll
new file mode 100644
index 000000000..2e958e792
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-int-log.ll
@@ -0,0 +1,96 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+define <vscale x 2 x i64> @and_d(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: and_d
+; CHECK: and z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+  %res = and <vscale x 2 x i64> %a, %b
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 4 x i32> @and_s(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: and_s
+; CHECK: and z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+  %res = and <vscale x 4 x i32> %a, %b
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 8 x i16> @and_h(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: and_h
+; CHECK: and z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+  %res = and <vscale x 8 x i16> %a, %b
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 16 x i8> @and_b(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: and_b
+; CHECK: and z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+  %res = and <vscale x 16 x i8> %a, %b
+  ret <vscale x 16 x i8> %res
+}                                                                                          
+define <vscale x 2 x i64> @or_d(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: or_d
+; CHECK: orr z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+  %res = or <vscale x 2 x i64> %a, %b
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 4 x i32> @or_s(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: or_s
+; CHECK: orr z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+  %res = or <vscale x 4 x i32> %a, %b
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 8 x i16> @or_h(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: or_h
+; CHECK: orr z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+  %res = or <vscale x 8 x i16> %a, %b
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 16 x i8> @or_b(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: or_b
+; CHECK: orr z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+  %res = or <vscale x 16 x i8> %a, %b
+  ret <vscale x 16 x i8> %res
+}                                                                                          
+
+define <vscale x 2 x i64> @xor_d(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: xor_d
+; CHECK: eor z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+  %res = xor <vscale x 2 x i64> %a, %b
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 4 x i32> @xor_s(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: xor_s
+; CHECK: eor z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+  %res = xor <vscale x 4 x i32> %a, %b
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 8 x i16> @xor_h(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: xor_h
+; CHECK: eor z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+  %res = xor <vscale x 8 x i16> %a, %b
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 16 x i8> @xor_b(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: xor_b
+; CHECK: eor z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+  %res = xor <vscale x 16 x i8> %a, %b
+  ret <vscale x 16 x i8> %res
+}
diff --git a/llvm/test/CodeGen/AArch64/sve-int-mad-pred.ll b/llvm/test/CodeGen/AArch64/sve-int-mad-pred.ll
new file mode 100644
index 000000000..30dc76daa
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-int-mad-pred.ll
@@ -0,0 +1,199 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+define <vscale x 16 x i8> @mad_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b, <vscale x 16 x i8> %c) {
+; CHECK-LABEL: mad_i8:
+; CHECK: mad z0.b, p0/m, z1.b, z2.b 
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.mad.nxv16i8(<vscale x 16 x i1> %pg,
+                                                               <vscale x 16 x i8> %a,
+                                                               <vscale x 16 x i8> %b,
+                                                               <vscale x 16 x i8> %c)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @mad_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b, <vscale x 8 x i16> %c) {
+; CHECK-LABEL: mad_i16:
+; CHECK: mad z0.h, p0/m, z1.h, z2.h 
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.mad.nxv8i16(<vscale x 8 x i1> %pg,
+                                                               <vscale x 8 x i16> %a,
+                                                               <vscale x 8 x i16> %b,
+                                                               <vscale x 8 x i16> %c)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @mad_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %c) {
+; CHECK-LABEL: mad_i32:
+; CHECK: mad z0.s, p0/m, z1.s, z2.s 
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.mad.nxv4i32(<vscale x 4 x i1> %pg,
+                                                               <vscale x 4 x i32> %a,
+                                                               <vscale x 4 x i32> %b,
+                                                               <vscale x 4 x i32> %c)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @mad_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> %b, <vscale x 2 x i64> %c) {
+; CHECK-LABEL: mad_i64:
+; CHECK: mad z0.d, p0/m, z1.d, z2.d 
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.mad.nxv2i64(<vscale x 2 x i1> %pg,
+                                                               <vscale x 2 x i64> %a,
+                                                               <vscale x 2 x i64> %b,
+                                                               <vscale x 2 x i64> %c)
+  ret <vscale x 2 x i64> %out
+}
+
+define <vscale x 16 x i8> @msb_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b, <vscale x 16 x i8> %c) {
+; CHECK-LABEL: msb_i8:
+; CHECK: msb z0.b, p0/m, z1.b, z2.b 
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.msb.nxv16i8(<vscale x 16 x i1> %pg,
+                                                               <vscale x 16 x i8> %a,
+                                                               <vscale x 16 x i8> %b,
+                                                               <vscale x 16 x i8> %c)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @msb_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b, <vscale x 8 x i16> %c) {
+; CHECK-LABEL: msb_i16:
+; CHECK: msb z0.h, p0/m, z1.h, z2.h 
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.msb.nxv8i16(<vscale x 8 x i1> %pg,
+                                                               <vscale x 8 x i16> %a,
+                                                               <vscale x 8 x i16> %b,
+                                                               <vscale x 8 x i16> %c)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @msb_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %c) {
+; CHECK-LABEL: msb_i32:
+; CHECK: msb z0.s, p0/m, z1.s, z2.s 
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.msb.nxv4i32(<vscale x 4 x i1> %pg,
+                                                               <vscale x 4 x i32> %a,
+                                                               <vscale x 4 x i32> %b,
+                                                               <vscale x 4 x i32> %c)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @msb_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> %b, <vscale x 2 x i64> %c) {
+; CHECK-LABEL: msb_i64:
+; CHECK: msb z0.d, p0/m, z1.d, z2.d 
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.msb.nxv2i64(<vscale x 2 x i1> %pg,
+                                                               <vscale x 2 x i64> %a,
+                                                               <vscale x 2 x i64> %b,
+                                                               <vscale x 2 x i64> %c)
+  ret <vscale x 2 x i64> %out
+}
+
+
+define <vscale x 16 x i8> @mla_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b, <vscale x 16 x i8> %c) {
+; CHECK-LABEL: mla_i8:
+; CHECK: mla z0.b, p0/m, z1.b, z2.b 
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.mla.nxv16i8(<vscale x 16 x i1> %pg,
+                                                               <vscale x 16 x i8> %a,
+                                                               <vscale x 16 x i8> %b,
+                                                               <vscale x 16 x i8> %c)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @mla_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b, <vscale x 8 x i16> %c) {
+; CHECK-LABEL: mla_i16:
+; CHECK: mla z0.h, p0/m, z1.h, z2.h 
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.mla.nxv8i16(<vscale x 8 x i1> %pg,
+                                                               <vscale x 8 x i16> %a,
+                                                               <vscale x 8 x i16> %b,
+                                                               <vscale x 8 x i16> %c)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @mla_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %c) {
+; CHECK-LABEL: mla_i32:
+; CHECK: mla z0.s, p0/m, z1.s, z2.s 
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.mla.nxv4i32(<vscale x 4 x i1> %pg,
+                                                               <vscale x 4 x i32> %a,
+                                                               <vscale x 4 x i32> %b,
+                                                               <vscale x 4 x i32> %c)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @mla_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> %b, <vscale x 2 x i64> %c) {
+; CHECK-LABEL: mla_i64:
+; CHECK: mla z0.d, p0/m, z1.d, z2.d 
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.mla.nxv2i64(<vscale x 2 x i1> %pg,
+                                                               <vscale x 2 x i64> %a,
+                                                               <vscale x 2 x i64> %b,
+                                                               <vscale x 2 x i64> %c)
+  ret <vscale x 2 x i64> %out
+}
+
+
+define <vscale x 16 x i8> @mls_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b, <vscale x 16 x i8> %c) {
+; CHECK-LABEL: mls_i8:
+; CHECK: mls z0.b, p0/m, z1.b, z2.b 
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.mls.nxv16i8(<vscale x 16 x i1> %pg,
+                                                               <vscale x 16 x i8> %a,
+                                                               <vscale x 16 x i8> %b,
+                                                               <vscale x 16 x i8> %c)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @mls_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b, <vscale x 8 x i16> %c) {
+; CHECK-LABEL: mls_i16:
+; CHECK: mls z0.h, p0/m, z1.h, z2.h 
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.mls.nxv8i16(<vscale x 8 x i1> %pg,
+                                                               <vscale x 8 x i16> %a,
+                                                               <vscale x 8 x i16> %b,
+                                                               <vscale x 8 x i16> %c)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @mls_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %c) {
+; CHECK-LABEL: mls_i32:
+; CHECK: mls z0.s, p0/m, z1.s, z2.s 
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.mls.nxv4i32(<vscale x 4 x i1> %pg,
+                                                               <vscale x 4 x i32> %a,
+                                                               <vscale x 4 x i32> %b,
+                                                               <vscale x 4 x i32> %c)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @mls_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> %b, <vscale x 2 x i64> %c) {
+; CHECK-LABEL: mls_i64:
+; CHECK: mls z0.d, p0/m, z1.d, z2.d 
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.mls.nxv2i64(<vscale x 2 x i1> %pg,
+                                                               <vscale x 2 x i64> %a,
+                                                               <vscale x 2 x i64> %b,
+                                                               <vscale x 2 x i64> %c)
+  ret <vscale x 2 x i64> %out
+}
+
+declare <vscale x 16 x  i8> @llvm.aarch64.sve.mad.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x  i8>,<vscale x 16 x  i8>,<vscale x 16 x  i8>)
+declare <vscale x 8 x  i16> @llvm.aarch64.sve.mad.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x  i16>,<vscale x 8 x  i16>,<vscale x 8 x  i16>)
+declare <vscale x 4 x  i32> @llvm.aarch64.sve.mad.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x  i32>,<vscale x 4 x  i32>,<vscale x 4 x  i32>)
+declare <vscale x 2 x  i64> @llvm.aarch64.sve.mad.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x  i64>,<vscale x 2 x  i64>,<vscale x 2 x  i64>)
+
+declare <vscale x 16 x  i8> @llvm.aarch64.sve.msb.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x  i8>,<vscale x 16 x  i8>,<vscale x 16 x  i8>)
+declare <vscale x 8 x  i16> @llvm.aarch64.sve.msb.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x  i16>,<vscale x 8 x  i16>,<vscale x 8 x  i16>)
+declare <vscale x 4 x  i32> @llvm.aarch64.sve.msb.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x  i32>,<vscale x 4 x  i32>,<vscale x 4 x  i32>)
+declare <vscale x 2 x  i64> @llvm.aarch64.sve.msb.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x  i64>,<vscale x 2 x  i64>,<vscale x 2 x  i64>)
+
+declare <vscale x 16 x  i8> @llvm.aarch64.sve.mla.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x  i8>,<vscale x 16 x  i8>,<vscale x 16 x  i8>)
+declare <vscale x 8 x  i16> @llvm.aarch64.sve.mla.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x  i16>,<vscale x 8 x  i16>,<vscale x 8 x  i16>)
+declare <vscale x 4 x  i32> @llvm.aarch64.sve.mla.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x  i32>,<vscale x 4 x  i32>,<vscale x 4 x  i32>)
+declare <vscale x 2 x  i64> @llvm.aarch64.sve.mla.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x  i64>,<vscale x 2 x  i64>,<vscale x 2 x  i64>)
+
+declare <vscale x 16 x  i8> @llvm.aarch64.sve.mls.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x  i8>,<vscale x 16 x  i8>,<vscale x 16 x  i8>)
+declare <vscale x 8 x  i16> @llvm.aarch64.sve.mls.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x  i16>,<vscale x 8 x  i16>,<vscale x 8 x  i16>)
+declare <vscale x 4 x  i32> @llvm.aarch64.sve.mls.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x  i32>,<vscale x 4 x  i32>,<vscale x 4 x  i32>)
+declare <vscale x 2 x  i64> @llvm.aarch64.sve.mls.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x  i64>,<vscale x 2 x  i64>,<vscale x 2 x  i64>)
diff --git a/llvm/test/CodeGen/AArch64/sve-int-mul-pred.ll b/llvm/test/CodeGen/AArch64/sve-int-mul-pred.ll
new file mode 100644
index 000000000..287a33729
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-int-mul-pred.ll
@@ -0,0 +1,134 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+define <vscale x 16 x i8> @mul_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: mul_i8:
+; CHECK: mul z0.b, p0/m, z0.b, z1.b
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.mul.nxv16i8(<vscale x 16 x i1> %pg,
+                                                               <vscale x 16 x i8> %a,
+                                                               <vscale x 16 x i8> %b)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @mul_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: mul_i16:
+; CHECK: mul z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.mul.nxv8i16(<vscale x 8 x i1> %pg,
+                                                               <vscale x 8 x i16> %a,
+                                                               <vscale x 8 x i16> %b)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @mul_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: mul_i32:
+; CHECK: mul z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.mul.nxv4i32(<vscale x 4 x i1> %pg,
+                                                               <vscale x 4 x i32> %a,
+                                                               <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @mul_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: mul_i64:
+; CHECK: mul z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.mul.nxv2i64(<vscale x 2 x i1> %pg,
+                                                               <vscale x 2 x i64> %a,
+                                                               <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i64> %out
+}
+
+define <vscale x 16 x i8> @smulh_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: smulh_i8:
+; CHECK: smulh z0.b, p0/m, z0.b, z1.b
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.smulh.nxv16i8(<vscale x 16 x i1> %pg,
+                                                               <vscale x 16 x i8> %a,
+                                                               <vscale x 16 x i8> %b)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @smulh_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: smulh_i16:
+; CHECK: smulh z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.smulh.nxv8i16(<vscale x 8 x i1> %pg,
+                                                               <vscale x 8 x i16> %a,
+                                                               <vscale x 8 x i16> %b)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @smulh_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: smulh_i32:
+; CHECK: smulh z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.smulh.nxv4i32(<vscale x 4 x i1> %pg,
+                                                               <vscale x 4 x i32> %a,
+                                                               <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @smulh_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: smulh_i64:
+; CHECK: smulh z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.smulh.nxv2i64(<vscale x 2 x i1> %pg,
+                                                               <vscale x 2 x i64> %a,
+                                                               <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i64> %out
+}
+
+define <vscale x 16 x i8> @umulh_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: umulh_i8:
+; CHECK: umulh z0.b, p0/m, z0.b, z1.b
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.umulh.nxv16i8(<vscale x 16 x i1> %pg,
+                                                               <vscale x 16 x i8> %a,
+                                                               <vscale x 16 x i8> %b)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @umulh_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: umulh_i16:
+; CHECK: umulh z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.umulh.nxv8i16(<vscale x 8 x i1> %pg,
+                                                               <vscale x 8 x i16> %a,
+                                                               <vscale x 8 x i16> %b)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @umulh_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: umulh_i32:
+; CHECK: umulh z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.umulh.nxv4i32(<vscale x 4 x i1> %pg,
+                                                               <vscale x 4 x i32> %a,
+                                                               <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @umulh_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: umulh_i64:
+; CHECK: umulh z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.umulh.nxv2i64(<vscale x 2 x i1> %pg,
+                                                               <vscale x 2 x i64> %a,
+                                                               <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i64> %out
+}
+
+declare <vscale x 16 x  i8> @llvm.aarch64.sve.mul.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x  i8>, <vscale x 16 x  i8>)
+declare <vscale x  8 x i16> @llvm.aarch64.sve.mul.nxv8i16(<vscale x  8 x i1>, <vscale x  8 x i16>, <vscale x  8 x i16>)
+declare <vscale x  4 x i32> @llvm.aarch64.sve.mul.nxv4i32(<vscale x  4 x i1>, <vscale x  4 x i32>, <vscale x  4 x i32>)
+declare <vscale x  2 x i64> @llvm.aarch64.sve.mul.nxv2i64(<vscale x  2 x i1>, <vscale x  2 x i64>, <vscale x  2 x i64>)
+declare <vscale x 16 x  i8> @llvm.aarch64.sve.smulh.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x  i8>, <vscale x 16 x  i8>)
+declare <vscale x  8 x i16> @llvm.aarch64.sve.smulh.nxv8i16(<vscale x  8 x i1>, <vscale x  8 x i16>, <vscale x  8 x i16>)
+declare <vscale x  4 x i32> @llvm.aarch64.sve.smulh.nxv4i32(<vscale x  4 x i1>, <vscale x  4 x i32>, <vscale x  4 x i32>)
+declare <vscale x  2 x i64> @llvm.aarch64.sve.smulh.nxv2i64(<vscale x  2 x i1>, <vscale x  2 x i64>, <vscale x  2 x i64>)
+declare <vscale x 16 x  i8> @llvm.aarch64.sve.umulh.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x  i8>, <vscale x 16 x  i8>)
+declare <vscale x  8 x i16> @llvm.aarch64.sve.umulh.nxv8i16(<vscale x  8 x i1>, <vscale x  8 x i16>, <vscale x  8 x i16>)
+declare <vscale x  4 x i32> @llvm.aarch64.sve.umulh.nxv4i32(<vscale x  4 x i1>, <vscale x  4 x i32>, <vscale x  4 x i32>)
+declare <vscale x  2 x i64> @llvm.aarch64.sve.umulh.nxv2i64(<vscale x  2 x i1>, <vscale x  2 x i64>, <vscale x  2 x i64>)
diff --git a/llvm/test/CodeGen/AArch64/sve-int-reduce-pred.ll b/llvm/test/CodeGen/AArch64/sve-int-reduce-pred.ll
new file mode 100644
index 000000000..204fb6d6d
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-int-reduce-pred.ll
@@ -0,0 +1,400 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+define i64 @saddv_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a) {
+; CHECK-LABEL: saddv_i8:
+; CHECK: saddv d[[REDUCE:[0-9]+]], p0, z0.b
+; CHECK: fmov x0, d[[REDUCE]]
+; CHECK-NEXT: ret
+  %out = call i64 @llvm.aarch64.sve.saddv.nxv16i8(<vscale x 16 x i1> %pg,
+                                                  <vscale x 16 x i8> %a)
+  ret i64 %out
+}
+
+define i64 @saddv_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a) {
+; CHECK-LABEL: saddv_i16:
+; CHECK: saddv d[[REDUCE:[0-9]+]], p0, z0.h
+; CHECK: fmov x0, d[[REDUCE]]
+; CHECK-NEXT: ret
+  %out = call i64 @llvm.aarch64.sve.saddv.nxv8i16(<vscale x 8 x i1> %pg,
+                                                  <vscale x 8 x i16> %a)
+  ret i64 %out
+}
+
+
+define i64 @saddv_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a) {
+; CHECK-LABEL: saddv_i32:
+; CHECK: saddv d[[REDUCE:[0-9]+]], p0, z0.s
+; CHECK: fmov x0, d[[REDUCE]]
+; CHECK-NEXT: ret
+  %out = call i64 @llvm.aarch64.sve.saddv.nxv4i32(<vscale x 4 x i1> %pg,
+                                                  <vscale x 4 x i32> %a)
+  ret i64 %out
+}
+
+define i64 @saddv_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a) {
+; CHECK-LABEL: saddv_i64
+; CHECK: uaddv d[[REDUCE:[0-9]+]], p0, z0.d
+; CHECK: fmov x0, d[[REDUCE]]
+; CHECK-NEXT: ret
+  %out = call i64 @llvm.aarch64.sve.saddv.nxv2i64(<vscale x 2 x i1> %pg,
+                                                  <vscale x 2 x i64> %a)
+  ret i64 %out
+}
+
+define i64 @uaddv_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a) {
+; CHECK-LABEL: uaddv_i8:
+; CHECK: uaddv d[[REDUCE:[0-9]+]], p0, z0.b
+; CHECK: fmov x0, d[[REDUCE]]
+; CHECK-NEXT: ret
+  %out = call i64 @llvm.aarch64.sve.uaddv.nxv16i8(<vscale x 16 x i1> %pg,
+                                                  <vscale x 16 x i8> %a)
+  ret i64 %out
+}
+
+define i64 @uaddv_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a) {
+; CHECK-LABEL: uaddv_i16:
+; CHECK: uaddv d[[REDUCE:[0-9]+]], p0, z0.h
+; CHECK: fmov x0, d[[REDUCE]]
+; CHECK-NEXT: ret
+  %out = call i64 @llvm.aarch64.sve.uaddv.nxv8i16(<vscale x 8 x i1> %pg,
+                                                  <vscale x 8 x i16> %a)
+  ret i64 %out
+}
+
+
+define i64 @uaddv_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a) {
+; CHECK-LABEL: uaddv_i32:
+; CHECK: uaddv d[[REDUCE:[0-9]+]], p0, z0.s
+; CHECK: fmov x0, d[[REDUCE]]
+; CHECK-NEXT: ret
+  %out = call i64 @llvm.aarch64.sve.uaddv.nxv4i32(<vscale x 4 x i1> %pg,
+                                                  <vscale x 4 x i32> %a)
+  ret i64 %out
+}
+
+define i64 @uaddv_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a) {
+; CHECK-LABEL: uaddv_i64:
+; CHECK: uaddv d[[REDUCE:[0-9]+]], p0, z0.d
+; CHECK: fmov x0, d[[REDUCE]]
+; CHECK-NEXT: ret
+  %out = call i64 @llvm.aarch64.sve.uaddv.nxv2i64(<vscale x 2 x i1> %pg,
+                                                  <vscale x 2 x i64> %a)
+  ret i64 %out
+}
+
+define i8 @smaxv_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a) {
+; CHECK-LABEL: smaxv_i8:
+; CHECK: smaxv b[[REDUCE:[0-9]+]], p0, z0.b
+; CHECK: umov w0, v[[REDUCE]].b[0]
+; CHECK-NEXT: ret
+  %out = call i8 @llvm.aarch64.sve.smaxv.nxv16i8(<vscale x 16 x i1> %pg,
+                                                 <vscale x 16 x i8> %a)
+  ret i8 %out
+}
+
+define i16 @smaxv_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a) {
+; CHECK-LABEL: smaxv_i16:
+; CHECK: smaxv h[[REDUCE:[0-9]+]], p0, z0.h
+; CHECK: umov w0, v[[REDUCE]].h[0]
+; CHECK-NEXT: ret
+  %out = call i16 @llvm.aarch64.sve.smaxv.nxv8i16(<vscale x 8 x i1> %pg,
+                                                  <vscale x 8 x i16> %a)
+  ret i16 %out
+}
+
+define i32 @smaxv_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a) {
+; CHECK-LABEL: smaxv_i32:
+; CHECK: smaxv s[[REDUCE:[0-9]+]], p0, z0.s
+; CHECK: fmov w0, s[[REDUCE]]
+; CHECK-NEXT: ret
+  %out = call i32 @llvm.aarch64.sve.smaxv.nxv4i32(<vscale x 4 x i1> %pg,
+                                                  <vscale x 4 x i32> %a)
+  ret i32 %out
+}
+
+define i64 @smaxv_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a) {
+; CHECK-LABEL: smaxv_i64:
+; CHECK: smaxv d[[REDUCE:[0-9]+]], p0, z0.d
+; CHECK: fmov x0, d[[REDUCE]]
+; CHECK-NEXT: ret
+  %out = call i64 @llvm.aarch64.sve.smaxv.nxv2i64(<vscale x 2 x i1> %pg,
+                                                  <vscale x 2 x i64> %a)
+  ret i64 %out
+}
+
+define i8 @umaxv_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a) {
+; CHECK-LABEL: umaxv_i8:
+; CHECK: umaxv b[[REDUCE:[0-9]+]], p0, z0.b
+; CHECK: umov w0, v[[REDUCE]].b[0]
+; CHECK-NEXT: ret
+  %out = call i8 @llvm.aarch64.sve.umaxv.nxv16i8(<vscale x 16 x i1> %pg,
+                                                 <vscale x 16 x i8> %a)
+  ret i8 %out
+}
+
+define i16 @umaxv_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a) {
+; CHECK-LABEL: umaxv_i16:
+; CHECK: umaxv h[[REDUCE:[0-9]+]], p0, z0.h
+; CHECK: umov w0, v[[REDUCE]].h[0]
+; CHECK-NEXT: ret
+  %out = call i16 @llvm.aarch64.sve.umaxv.nxv8i16(<vscale x 8 x i1> %pg,
+                                                  <vscale x 8 x i16> %a)
+  ret i16 %out
+}
+
+define i32 @umaxv_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a) {
+; CHECK-LABEL: umaxv_i32:
+; CHECK: umaxv s[[REDUCE:[0-9]+]], p0, z0.s
+; CHECK: fmov w0, s[[REDUCE]]
+; CHECK-NEXT: ret
+  %out = call i32 @llvm.aarch64.sve.umaxv.nxv4i32(<vscale x 4 x i1> %pg,
+                                                  <vscale x 4 x i32> %a)
+  ret i32 %out
+}
+
+define i64 @umaxv_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a) {
+; CHECK-LABEL: umaxv_i64:
+; CHECK: umaxv d[[REDUCE:[0-9]+]], p0, z0.d
+; CHECK: fmov x0, d[[REDUCE]]
+; CHECK-NEXT: ret
+  %out = call i64 @llvm.aarch64.sve.umaxv.nxv2i64(<vscale x 2 x i1> %pg,
+                                                  <vscale x 2 x i64> %a)
+  ret i64 %out
+}
+
+define i8 @sminv_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a) {
+; CHECK-LABEL: sminv_i8:
+; CHECK: sminv b[[REDUCE:[0-9]+]], p0, z0.b
+; CHECK: umov w0, v[[REDUCE]].b[0]
+; CHECK-NEXT: ret
+  %out = call i8 @llvm.aarch64.sve.sminv.nxv16i8(<vscale x 16 x i1> %pg,
+                                                 <vscale x 16 x i8> %a)
+  ret i8 %out
+}
+
+define i16 @sminv_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a) {
+; CHECK-LABEL: sminv_i16:
+; CHECK: sminv h[[REDUCE:[0-9]+]], p0, z0.h
+; CHECK: umov w0, v[[REDUCE]].h[0]
+; CHECK-NEXT: ret
+  %out = call i16 @llvm.aarch64.sve.sminv.nxv8i16(<vscale x 8 x i1> %pg,
+                                                  <vscale x 8 x i16> %a)
+  ret i16 %out
+}
+
+define i32 @sminv_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a) {
+; CHECK-LABEL: sminv_i32:
+; CHECK: sminv s[[REDUCE:[0-9]+]], p0, z0.s
+; CHECK: fmov w0, s[[REDUCE]]
+; CHECK-NEXT: ret
+  %out = call i32 @llvm.aarch64.sve.sminv.nxv4i32(<vscale x 4 x i1> %pg,
+                                                  <vscale x 4 x i32> %a)
+  ret i32 %out
+}
+
+define i64 @sminv_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a) {
+; CHECK-LABEL: sminv_i64:
+; CHECK: sminv d[[REDUCE:[0-9]+]], p0, z0.d
+; CHECK: fmov x0, d[[REDUCE]]
+; CHECK-NEXT: ret
+  %out = call i64 @llvm.aarch64.sve.sminv.nxv2i64(<vscale x 2 x i1> %pg,
+                                                  <vscale x 2 x i64> %a)
+  ret i64 %out
+}
+
+define i8 @uminv_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a) {
+; CHECK-LABEL: uminv_i8:
+; CHECK: uminv b[[REDUCE:[0-9]+]], p0, z0.b
+; CHECK: umov w0, v[[REDUCE]].b[0]
+; CHECK-NEXT: ret
+  %out = call i8 @llvm.aarch64.sve.uminv.nxv16i8(<vscale x 16 x i1> %pg,
+                                                 <vscale x 16 x i8> %a)
+  ret i8 %out
+}
+
+define i16 @uminv_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a) {
+; CHECK-LABEL: uminv_i16:
+; CHECK: uminv h[[REDUCE:[0-9]+]], p0, z0.h
+; CHECK: umov w0, v[[REDUCE]].h[0]
+; CHECK-NEXT: ret
+  %out = call i16 @llvm.aarch64.sve.uminv.nxv8i16(<vscale x 8 x i1> %pg,
+                                                  <vscale x 8 x i16> %a)
+  ret i16 %out
+}
+
+define i32 @uminv_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a) {
+; CHECK-LABEL: uminv_i32:
+; CHECK: uminv s[[REDUCE:[0-9]+]], p0, z0.s
+; CHECK: fmov w0, s[[REDUCE]]
+; CHECK-NEXT: ret
+  %out = call i32 @llvm.aarch64.sve.uminv.nxv4i32(<vscale x 4 x i1> %pg,
+                                                  <vscale x 4 x i32> %a)
+  ret i32 %out
+}
+
+define i64 @uminv_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a) {
+; CHECK-LABEL: uminv_i64:
+; CHECK: uminv d[[REDUCE:[0-9]+]], p0, z0.d
+; CHECK: fmov x0, d[[REDUCE]]
+; CHECK-NEXT: ret
+  %out = call i64 @llvm.aarch64.sve.uminv.nxv2i64(<vscale x 2 x i1> %pg,
+                                                  <vscale x 2 x i64> %a)
+  ret i64 %out
+}
+
+define i8 @orv_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a) {
+; CHECK-LABEL: orv_i8:
+; CHECK: orv b[[REDUCE:[0-9]+]], p0, z0.b
+; CHECK: umov w0, v[[REDUCE]].b[0]
+; CHECK-NEXT: ret
+  %out = call i8 @llvm.aarch64.sve.orv.nxv16i8(<vscale x 16 x i1> %pg,
+                                               <vscale x 16 x i8> %a)
+  ret i8 %out
+}
+
+define i16 @orv_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a) {
+; CHECK-LABEL: orv_i16:
+; CHECK: orv h[[REDUCE:[0-9]+]], p0, z0.h
+; CHECK: umov w0, v[[REDUCE]].h[0]
+; CHECK-NEXT: ret
+  %out = call i16 @llvm.aarch64.sve.orv.nxv8i16(<vscale x 8 x i1> %pg,
+                                                <vscale x 8 x i16> %a)
+  ret i16 %out
+}
+
+define i32 @orv_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a) {
+; CHECK-LABEL: orv_i32:
+; CHECK: orv s[[REDUCE:[0-9]+]], p0, z0.s
+; CHECK: fmov w0, s[[REDUCE]]
+; CHECK-NEXT: ret
+  %out = call i32 @llvm.aarch64.sve.orv.nxv4i32(<vscale x 4 x i1> %pg,
+                                                <vscale x 4 x i32> %a)
+  ret i32 %out
+}
+
+define i64 @orv_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a) {
+; CHECK-LABEL: orv_i64:
+; CHECK: orv d[[REDUCE:[0-9]+]], p0, z0.d
+; CHECK: fmov x0, d[[REDUCE]]
+; CHECK-NEXT: ret
+  %out = call i64 @llvm.aarch64.sve.orv.nxv2i64(<vscale x 2 x i1> %pg,
+                                                <vscale x 2 x i64> %a)
+  ret i64 %out
+}
+
+define i8 @eorv_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a) {
+; CHECK-LABEL: eorv_i8:
+; CHECK: eorv b[[REDUCE:[0-9]+]], p0, z0.b
+; CHECK: umov w0, v[[REDUCE]].b[0]
+; CHECK-NEXT: ret
+  %out = call i8 @llvm.aarch64.sve.eorv.nxv16i8(<vscale x 16 x i1> %pg,
+                                                <vscale x 16 x i8> %a)
+  ret i8 %out
+}
+
+define i16 @eorv_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a) {
+; CHECK-LABEL: eorv_i16:
+; CHECK: eorv h[[REDUCE:[0-9]+]], p0, z0.h
+; CHECK: umov w0, v[[REDUCE]].h[0]
+; CHECK-NEXT: ret
+  %out = call i16 @llvm.aarch64.sve.eorv.nxv8i16(<vscale x 8 x i1> %pg,
+                                                 <vscale x 8 x i16> %a)
+  ret i16 %out
+}
+
+define i32 @eorv_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a) {
+; CHECK-LABEL: eorv_i32:
+; CHECK: eorv s[[REDUCE:[0-9]+]], p0, z0.s
+; CHECK: fmov w0, s[[REDUCE]]
+; CHECK-NEXT: ret
+  %out = call i32 @llvm.aarch64.sve.eorv.nxv4i32(<vscale x 4 x i1> %pg,
+                                                 <vscale x 4 x i32> %a)
+  ret i32 %out
+}
+
+define i64 @eorv_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a) {
+; CHECK-LABEL: eorv_i64:
+; CHECK: eorv d[[REDUCE:[0-9]+]], p0, z0.d
+; CHECK: fmov x0, d[[REDUCE]]
+; CHECK-NEXT: ret
+  %out = call i64 @llvm.aarch64.sve.eorv.nxv2i64(<vscale x 2 x i1> %pg,
+                                                 <vscale x 2 x i64> %a)
+  ret i64 %out
+}
+
+define i8 @andv_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a) {
+; CHECK-LABEL: andv_i8:
+; CHECK: andv b[[REDUCE:[0-9]+]], p0, z0.b
+; CHECK: umov w0, v[[REDUCE]].b[0]
+; CHECK-NEXT: ret
+  %out = call i8 @llvm.aarch64.sve.andv.nxv16i8(<vscale x 16 x i1> %pg,
+                                                <vscale x 16 x i8> %a)
+  ret i8 %out
+}
+
+define i16 @andv_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a) {
+; CHECK-LABEL: andv_i16:
+; CHECK: andv h[[REDUCE:[0-9]+]], p0, z0.h
+; CHECK: umov w0, v[[REDUCE]].h[0]
+; CHECK-NEXT: ret
+  %out = call i16 @llvm.aarch64.sve.andv.nxv8i16(<vscale x 8 x i1> %pg,
+                                                 <vscale x 8 x i16> %a)
+  ret i16 %out
+}
+
+define i32 @andv_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a) {
+; CHECK-LABEL: andv_i32:
+; CHECK: andv s[[REDUCE:[0-9]+]], p0, z0.s
+; CHECK: fmov w0, s[[REDUCE]]
+; CHECK-NEXT: ret
+  %out = call i32 @llvm.aarch64.sve.andv.nxv4i32(<vscale x 4 x i1> %pg,
+                                                 <vscale x 4 x i32> %a)
+  ret i32 %out
+}
+
+define i64 @andv_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a) {
+; CHECK-LABEL: andv_i64:
+; CHECK: andv d[[REDUCE:[0-9]+]], p0, z0.d
+; CHECK: fmov x0, d[[REDUCE]]
+; CHECK-NEXT: ret
+  %out = call i64 @llvm.aarch64.sve.andv.nxv2i64(<vscale x 2 x i1> %pg,
+                                                 <vscale x 2 x i64> %a)
+  ret i64 %out
+}
+
+declare i64 @llvm.aarch64.sve.saddv.nxv16i8(<vscale x 16 x i1>, <vscale x  16 x i8>)
+declare i64 @llvm.aarch64.sve.saddv.nxv8i16(<vscale x 8 x  i1>, <vscale x  8 x  i16>)
+declare i64 @llvm.aarch64.sve.saddv.nxv4i32(<vscale x 4 x  i1>, <vscale x  4 x  i32>)
+declare i64 @llvm.aarch64.sve.saddv.nxv2i64(<vscale x 2 x  i1>, <vscale x  2 x  i64>)
+declare i64 @llvm.aarch64.sve.uaddv.nxv16i8(<vscale x 16 x i1>, <vscale x  16 x i8>)
+declare i64 @llvm.aarch64.sve.uaddv.nxv8i16(<vscale x 8 x  i1>, <vscale x  8 x  i16>)
+declare i64 @llvm.aarch64.sve.uaddv.nxv4i32(<vscale x 4 x  i1>, <vscale x  4 x  i32>)
+declare i64 @llvm.aarch64.sve.uaddv.nxv2i64(<vscale x 2 x  i1>, <vscale x  2 x  i64>)
+declare i8 @llvm.aarch64.sve.smaxv.nxv16i8(<vscale x 16 x  i1>, <vscale x  16 x  i8>)
+declare i16 @llvm.aarch64.sve.smaxv.nxv8i16(<vscale x 8 x  i1>, <vscale x  8 x  i16>)
+declare i32 @llvm.aarch64.sve.smaxv.nxv4i32(<vscale x 4 x  i1>, <vscale x  4 x  i32>)
+declare i64 @llvm.aarch64.sve.smaxv.nxv2i64(<vscale x 2 x  i1>, <vscale x  2 x  i64>)
+declare i8 @llvm.aarch64.sve.umaxv.nxv16i8(<vscale x 16 x  i1>, <vscale x  16 x  i8>)
+declare i16 @llvm.aarch64.sve.umaxv.nxv8i16(<vscale x 8 x  i1>, <vscale x  8 x  i16>)
+declare i32 @llvm.aarch64.sve.umaxv.nxv4i32(<vscale x 4 x  i1>, <vscale x  4 x  i32>)
+declare i64 @llvm.aarch64.sve.umaxv.nxv2i64(<vscale x 2 x  i1>, <vscale x  2 x  i64>)
+declare i8 @llvm.aarch64.sve.sminv.nxv16i8(<vscale x 16 x  i1>, <vscale x  16 x  i8>)
+declare i16 @llvm.aarch64.sve.sminv.nxv8i16(<vscale x 8 x  i1>, <vscale x  8 x  i16>)
+declare i32 @llvm.aarch64.sve.sminv.nxv4i32(<vscale x 4 x  i1>, <vscale x  4 x  i32>)
+declare i64 @llvm.aarch64.sve.sminv.nxv2i64(<vscale x 2 x  i1>, <vscale x  2 x  i64>)
+declare i8 @llvm.aarch64.sve.uminv.nxv16i8(<vscale x 16 x  i1>, <vscale x  16 x  i8>)
+declare i16 @llvm.aarch64.sve.uminv.nxv8i16(<vscale x 8 x  i1>, <vscale x  8 x  i16>)
+declare i32 @llvm.aarch64.sve.uminv.nxv4i32(<vscale x 4 x  i1>, <vscale x  4 x  i32>)
+declare i64 @llvm.aarch64.sve.uminv.nxv2i64(<vscale x 2 x  i1>, <vscale x  2 x  i64>)
+declare i8 @llvm.aarch64.sve.orv.nxv16i8(<vscale x 16 x  i1>, <vscale x  16 x  i8>)
+declare i16 @llvm.aarch64.sve.orv.nxv8i16(<vscale x 8 x  i1>, <vscale x  8 x  i16>)
+declare i32 @llvm.aarch64.sve.orv.nxv4i32  (<vscale x 4 x  i1>, <vscale x  4 x  i32>)
+declare i64 @llvm.aarch64.sve.orv.nxv2i64  (<vscale x 2 x  i1>, <vscale x  2 x  i64>)
+declare i8 @llvm.aarch64.sve.eorv.nxv16i8(<vscale x 16 x  i1>, <vscale x  16 x  i8>)
+declare i16 @llvm.aarch64.sve.eorv.nxv8i16(<vscale x 8 x  i1>, <vscale x  8 x  i16>)
+declare i32 @llvm.aarch64.sve.eorv.nxv4i32 (<vscale x 4 x  i1>, <vscale x  4 x  i32>)
+declare i64 @llvm.aarch64.sve.eorv.nxv2i64 (<vscale x 2 x  i1>, <vscale x  2 x  i64>)
+declare i8 @llvm.aarch64.sve.andv.nxv16i8(<vscale x 16 x  i1>, <vscale x  16 x  i8>)
+declare i16 @llvm.aarch64.sve.andv.nxv8i16(<vscale x 8 x  i1>, <vscale x  8 x  i16>)
+declare i32 @llvm.aarch64.sve.andv.nxv4i32 (<vscale x 4 x  i1>, <vscale x  4 x  i32>)
+declare i64 @llvm.aarch64.sve.andv.nxv2i64 (<vscale x 2 x  i1>, <vscale x  2 x  i64>)
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-conversion.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-conversion.ll
new file mode 100644
index 000000000..ac2b63d0b
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-conversion.ll
@@ -0,0 +1,159 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+;
+; SXTB
+;
+
+define <vscale x 8 x i16> @sxtb_i16(<vscale x 8 x i16> %a, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: sxtb_i16:
+; CHECK: sxtb z0.h, p0/m, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.sxtb.nxv8i16(<vscale x 8 x i16> %a,
+                                                                <vscale x 8 x i1> %pg,
+                                                                <vscale x 8 x i16> %b)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @sxtb_i32(<vscale x 4 x i32> %a, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: sxtb_i32:
+; CHECK: sxtb z0.s, p0/m, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.sxtb.nxv4i32(<vscale x 4 x i32> %a,
+                                                                <vscale x 4 x i1> %pg,
+                                                                <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @sxtb_i64(<vscale x 2 x i64> %a, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: sxtb_i64:
+; CHECK: sxtb z0.d, p0/m, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.sxtb.nxv2i64(<vscale x 2 x i64> %a,
+                                                                <vscale x 2 x i1> %pg,
+                                                                <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i64> %out
+}
+
+;
+; SXTH
+;
+
+define <vscale x 4 x i32> @sxth_i32(<vscale x 4 x i32> %a, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: sxth_i32:
+; CHECK: sxth z0.s, p0/m, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.sxth.nxv4i32(<vscale x 4 x i32> %a,
+                                                                <vscale x 4 x i1> %pg,
+                                                                <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @sxth_i64(<vscale x 2 x i64> %a, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: sxth_i64:
+; CHECK: sxth z0.d, p0/m, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.sxth.nxv2i64(<vscale x 2 x i64> %a,
+                                                                <vscale x 2 x i1> %pg,
+                                                                <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i64> %out
+}
+
+;
+; SXTW
+;
+
+define <vscale x 2 x i64> @sxtw_i64(<vscale x 2 x i64> %a, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: sxtw_i64:
+; CHECK: sxtw z0.d, p0/m, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.sxtw.nxv2i64(<vscale x 2 x i64> %a,
+                                                                <vscale x 2 x i1> %pg,
+                                                                <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i64> %out
+}
+
+;
+; UXTB
+;
+
+define <vscale x 8 x i16> @uxtb_i16(<vscale x 8 x i16> %a, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: uxtb_i16:
+; CHECK: uxtb z0.h, p0/m, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.uxtb.nxv8i16(<vscale x 8 x i16> %a,
+                                                                <vscale x 8 x i1> %pg,
+                                                                <vscale x 8 x i16> %b)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @uxtb_i32(<vscale x 4 x i32> %a, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: uxtb_i32:
+; CHECK: uxtb z0.s, p0/m, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.uxtb.nxv4i32(<vscale x 4 x i32> %a,
+                                                                <vscale x 4 x i1> %pg,
+                                                                <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @uxtb_i64(<vscale x 2 x i64> %a, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: uxtb_i64:
+; CHECK: uxtb z0.d, p0/m, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.uxtb.nxv2i64(<vscale x 2 x i64> %a,
+                                                                <vscale x 2 x i1> %pg,
+                                                                <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i64> %out
+}
+
+;
+; UXTH
+;
+
+define <vscale x 4 x i32> @uxth_i32(<vscale x 4 x i32> %a, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: uxth_i32:
+; CHECK: uxth z0.s, p0/m, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.uxth.nxv4i32(<vscale x 4 x i32> %a,
+                                                                <vscale x 4 x i1> %pg,
+                                                                <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @uxth_i64(<vscale x 2 x i64> %a, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: uxth_i64:
+; CHECK: uxth z0.d, p0/m, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.uxth.nxv2i64(<vscale x 2 x i64> %a,
+                                                                <vscale x 2 x i1> %pg,
+                                                                <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i64> %out
+}
+
+;
+; UXTW
+;
+
+define <vscale x 2 x i64> @uxtw_i64(<vscale x 2 x i64> %a, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: uxtw_i64:
+; CHECK: uxtw z0.d, p0/m, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.uxtw.nxv2i64(<vscale x 2 x i64> %a,
+                                                                <vscale x 2 x i1> %pg,
+                                                                <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i64> %out
+}
+
+declare <vscale x 8 x i16> @llvm.aarch64.sve.sxtb.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i1>, <vscale x 8 x i16>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.sxtb.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i1>, <vscale x 4 x i32>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.sxtb.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i1>, <vscale x 2 x i64>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.sxth.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i1>, <vscale x 4 x i32>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.sxth.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i1>, <vscale x 2 x i64>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.sxtw.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i1>, <vscale x 2 x i64>)
+
+declare <vscale x 8 x i16> @llvm.aarch64.sve.uxtb.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i1>, <vscale x 8 x i16>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.uxtb.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i1>, <vscale x 4 x i32>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.uxtb.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i1>, <vscale x 2 x i64>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.uxth.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i1>, <vscale x 4 x i32>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.uxth.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i1>, <vscale x 2 x i64>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.uxtw.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i1>, <vscale x 2 x i64>)
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-counting-bits.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-counting-bits.ll
new file mode 100644
index 000000000..197e723e1
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-counting-bits.ll
@@ -0,0 +1,181 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+;
+; CLS
+;
+
+define <vscale x 16 x i8> @cls_i8(<vscale x 16 x i8> %a, <vscale x 16 x i1> %pg, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: cls_i8:
+; CHECK: cls z0.b, p0/m, z1.b
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.cls.nxv16i8(<vscale x 16 x i8> %a,
+                                                               <vscale x 16 x i1> %pg,
+                                                               <vscale x 16 x i8> %b)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @cls_i16(<vscale x 8 x i16> %a, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: cls_i16:
+; CHECK: cls z0.h, p0/m, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.cls.nxv8i16(<vscale x 8 x i16> %a,
+                                                               <vscale x 8 x i1> %pg,
+                                                               <vscale x 8 x i16> %b)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @cls_i32(<vscale x 4 x i32> %a, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: cls_i32:
+; CHECK: cls z0.s, p0/m, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.cls.nxv4i32(<vscale x 4 x i32> %a,
+                                                               <vscale x 4 x i1> %pg,
+                                                               <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @cls_i64(<vscale x 2 x i64> %a, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cls_i64:
+; CHECK: cls z0.d, p0/m, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.cls.nxv2i64(<vscale x 2 x i64> %a,
+                                                               <vscale x 2 x i1> %pg,
+                                                               <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i64> %out
+}
+
+;
+; CLZ
+;
+
+define <vscale x 16 x i8> @clz_i8(<vscale x 16 x i8> %a, <vscale x 16 x i1> %pg, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: clz_i8:
+; CHECK: clz z0.b, p0/m, z1.b
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.clz.nxv16i8(<vscale x 16 x i8> %a,
+                                                               <vscale x 16 x i1> %pg,
+                                                               <vscale x 16 x i8> %b)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @clz_i16(<vscale x 8 x i16> %a, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: clz_i16:
+; CHECK: clz z0.h, p0/m, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.clz.nxv8i16(<vscale x 8 x i16> %a,
+                                                               <vscale x 8 x i1> %pg,
+                                                               <vscale x 8 x i16> %b)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @clz_i32(<vscale x 4 x i32> %a, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: clz_i32:
+; CHECK: clz z0.s, p0/m, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.clz.nxv4i32(<vscale x 4 x i32> %a,
+                                                               <vscale x 4 x i1> %pg,
+                                                               <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @clz_i64(<vscale x 2 x i64> %a, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: clz_i64:
+; CHECK: clz z0.d, p0/m, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.clz.nxv2i64(<vscale x 2 x i64> %a,
+                                                               <vscale x 2 x i1> %pg,
+                                                               <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i64> %out
+}
+
+;
+; CNT
+;
+
+define <vscale x 16 x i8> @cnt_i8(<vscale x 16 x i8> %a, <vscale x 16 x i1> %pg, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: cnt_i8:
+; CHECK: cnt z0.b, p0/m, z1.b
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.cnt.nxv16i8(<vscale x 16 x i8> %a,
+                                                               <vscale x 16 x i1> %pg,
+                                                               <vscale x 16 x i8> %b)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @cnt_i16(<vscale x 8 x i16> %a, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: cnt_i16:
+; CHECK: cnt z0.h, p0/m, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.cnt.nxv8i16(<vscale x 8 x i16> %a,
+                                                               <vscale x 8 x i1> %pg,
+                                                               <vscale x 8 x i16> %b)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @cnt_i32(<vscale x 4 x i32> %a, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: cnt_i32:
+; CHECK: cnt z0.s, p0/m, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.cnt.nxv4i32(<vscale x 4 x i32> %a,
+                                                               <vscale x 4 x i1> %pg,
+                                                               <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @cnt_i64(<vscale x 2 x i64> %a, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cnt_i64:
+; CHECK: cnt z0.d, p0/m, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.cnt.nxv2i64(<vscale x 2 x i64> %a,
+                                                               <vscale x 2 x i1> %pg,
+                                                               <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i64> %out
+}
+
+define <vscale x 8 x i16> @cnt_f16(<vscale x 8 x i16> %a, <vscale x 8 x i1> %pg, <vscale x 8 x half> %b) {
+; CHECK-LABEL: cnt_f16:
+; CHECK: cnt z0.h, p0/m, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.cnt.nxv8f16(<vscale x 8 x i16> %a,
+                                                               <vscale x 8 x i1> %pg,
+                                                               <vscale x 8 x half> %b)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @cnt_f32(<vscale x 4 x i32> %a, <vscale x 4 x i1> %pg, <vscale x 4 x float> %b) {
+; CHECK-LABEL: cnt_f32:
+; CHECK: cnt z0.s, p0/m, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.cnt.nxv4f32(<vscale x 4 x i32> %a,
+                                                               <vscale x 4 x i1> %pg,
+                                                               <vscale x 4 x float> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @cnt_f64(<vscale x 2 x i64> %a, <vscale x 2 x i1> %pg, <vscale x 2 x double> %b) {
+; CHECK-LABEL: cnt_f64:
+; CHECK: cnt z0.d, p0/m, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.cnt.nxv2f64(<vscale x 2 x i64> %a,
+                                                               <vscale x 2 x i1> %pg,
+                                                               <vscale x 2 x double> %b)
+  ret <vscale x 2 x i64> %out
+}
+
+declare <vscale x 16 x i8> @llvm.aarch64.sve.cls.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i1>, <vscale x 16 x i8>)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.cls.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i1>, <vscale x 8 x i16>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.cls.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i1>, <vscale x 4 x i32>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.cls.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i1>, <vscale x 2 x i64>)
+
+declare <vscale x 16 x i8> @llvm.aarch64.sve.clz.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i1>, <vscale x 16 x i8>)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.clz.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i1>, <vscale x 8 x i16>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.clz.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i1>, <vscale x 4 x i32>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.clz.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i1>, <vscale x 2 x i64>)
+
+declare <vscale x 16 x i8> @llvm.aarch64.sve.cnt.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i1>, <vscale x 16 x i8>)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.cnt.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i1>, <vscale x 8 x i16>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.cnt.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i1>, <vscale x 4 x i32>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.cnt.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i1>, <vscale x 2 x i64>)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.cnt.nxv8f16(<vscale x 8 x i16>, <vscale x 8 x i1>, <vscale x 8 x half>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.cnt.nxv4f32(<vscale x 4 x i32>, <vscale x 4 x i1>, <vscale x 4 x float>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.cnt.nxv2f64(<vscale x 2 x i64>, <vscale x 2 x i1>, <vscale x 2 x double>)
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-counting-elems.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-counting-elems.ll
new file mode 100644
index 000000000..b37e3d8b8
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-counting-elems.ll
@@ -0,0 +1,171 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+;
+; CNTB
+;
+
+define i64 @cntb() {
+; CHECK-LABEL: cntb:
+; CHECK: cntb x0, vl2
+; CHECK-NEXT: ret
+  %out = call i64 @llvm.aarch64.sve.cntb(i32 2)
+  ret i64 %out
+}
+
+define i64 @cntb_mul3() {
+; CHECK-LABEL: cntb_mul3:
+; CHECK: cntb x0, vl6, mul #3
+; CHECK-NEXT: ret
+  %cnt = call i64 @llvm.aarch64.sve.cntb(i32 6)
+  %out = mul i64 %cnt, 3
+  ret i64 %out
+}
+
+define i64 @cntb_mul4() {
+; CHECK-LABEL: cntb_mul4:
+; CHECK: cntb x0, vl8, mul #4
+; CHECK-NEXT: ret
+  %cnt = call i64 @llvm.aarch64.sve.cntb(i32 8)
+  %out = mul i64 %cnt, 4
+  ret i64 %out
+}
+
+;
+; CNTH
+;
+
+define i64 @cnth() {
+; CHECK-LABEL: cnth:
+; CHECK: cnth x0, vl3
+; CHECK-NEXT: ret
+  %out = call i64 @llvm.aarch64.sve.cnth(i32 3)
+  ret i64 %out
+}
+
+define i64 @cnth_mul5() {
+; CHECK-LABEL: cnth_mul5:
+; CHECK: cnth x0, vl7, mul #5
+; CHECK-NEXT: ret
+  %cnt = call i64 @llvm.aarch64.sve.cnth(i32 7)
+  %out = mul i64 %cnt, 5
+  ret i64 %out
+}
+
+define i64 @cnth_mul8() {
+; CHECK-LABEL: cnth_mul8:
+; CHECK: cnth x0, vl5, mul #8
+; CHECK-NEXT: ret
+  %cnt = call i64 @llvm.aarch64.sve.cnth(i32 5)
+  %out = mul i64 %cnt, 8
+  ret i64 %out
+}
+
+;
+; CNTW
+;
+
+define i64 @cntw() {
+; CHECK-LABEL: cntw:
+; CHECK: cntw x0, vl4
+; CHECK-NEXT: ret
+  %out = call i64 @llvm.aarch64.sve.cntw(i32 4)
+  ret i64 %out
+}
+
+define i64 @cntw_mul11() {
+; CHECK-LABEL: cntw_mul11:
+; CHECK: cntw x0, vl8, mul #11
+; CHECK-NEXT: ret
+  %cnt = call i64 @llvm.aarch64.sve.cntw(i32 8)
+  %out = mul i64 %cnt, 11
+  ret i64 %out
+}
+
+define i64 @cntw_mul2() {
+; CHECK-LABEL: cntw_mul2:
+; CHECK: cntw x0, vl6, mul #2
+; CHECK-NEXT: ret
+  %cnt = call i64 @llvm.aarch64.sve.cntw(i32 6)
+  %out = mul i64 %cnt, 2
+  ret i64 %out
+}
+
+;
+; CNTD
+;
+
+define i64 @cntd() {
+; CHECK-LABEL: cntd:
+; CHECK: cntd x0, vl5
+; CHECK-NEXT: ret
+  %out = call i64 @llvm.aarch64.sve.cntd(i32 5)
+  ret i64 %out
+}
+
+define i64 @cntd_mul15() {
+; CHECK-LABEL: cntd_mul15:
+; CHECK: cntd x0, vl16, mul #15
+; CHECK-NEXT: ret
+  %cnt = call i64 @llvm.aarch64.sve.cntd(i32 9)
+  %out = mul i64 %cnt, 15
+  ret i64 %out
+}
+
+define i64 @cntd_mul16() {
+; CHECK-LABEL: cntd_mul16:
+; CHECK: cntd x0, vl32, mul #16
+; CHECK-NEXT: ret
+  %cnt = call i64 @llvm.aarch64.sve.cntd(i32 10)
+  %out = mul i64 %cnt, 16
+  ret i64 %out
+}
+
+;
+; CNTP
+;
+
+define i64 @cntp_b8(<vscale x 16 x i1> %pg, <vscale x 16 x i1> %a) {
+; CHECK-LABEL: cntp_b8:
+; CHECK: cntp x0, p0, p1.b
+; CHECK-NEXT: ret
+  %out = call i64 @llvm.aarch64.sve.cntp.nxv16i1(<vscale x 16 x i1> %pg,
+                                                 <vscale x 16 x i1> %a)
+  ret i64 %out
+}
+
+define i64 @cntp_b16(<vscale x 8 x i1> %pg, <vscale x 8 x i1> %a) {
+; CHECK-LABEL: cntp_b16:
+; CHECK: cntp x0, p0, p1.h
+; CHECK-NEXT: ret
+  %out = call i64 @llvm.aarch64.sve.cntp.nxv8i1(<vscale x 8 x i1> %pg,
+                                                <vscale x 8 x i1> %a)
+  ret i64 %out
+}
+
+define i64 @cntp_b32(<vscale x 4 x i1> %pg, <vscale x 4 x i1> %a) {
+; CHECK-LABEL: cntp_b32:
+; CHECK: cntp x0, p0, p1.s
+; CHECK-NEXT: ret
+  %out = call i64 @llvm.aarch64.sve.cntp.nxv4i1(<vscale x 4 x i1> %pg,
+                                                <vscale x 4 x i1> %a)
+  ret i64 %out
+}
+
+define i64 @cntp_b64(<vscale x 2 x i1> %pg, <vscale x 2 x i1> %a) {
+; CHECK-LABEL: cntp_b64:
+; CHECK: cntp x0, p0, p1.d
+; CHECK-NEXT: ret
+  %out = call i64 @llvm.aarch64.sve.cntp.nxv2i1(<vscale x 2 x i1> %pg,
+                                                <vscale x 2 x i1> %a)
+  ret i64 %out
+}
+
+declare i64 @llvm.aarch64.sve.cntb(i32 %pattern)
+declare i64 @llvm.aarch64.sve.cnth(i32 %pattern)
+declare i64 @llvm.aarch64.sve.cntw(i32 %pattern)
+declare i64 @llvm.aarch64.sve.cntd(i32 %pattern)
+
+declare i64 @llvm.aarch64.sve.cntp.nxv16i1(<vscale x 16 x i1>, <vscale x 16 x i1>)
+declare i64 @llvm.aarch64.sve.cntp.nxv8i1(<vscale x 8 x i1>, <vscale x 8 x i1>)
+declare i64 @llvm.aarch64.sve.cntp.nxv4i1(<vscale x 4 x i1>, <vscale x 4 x i1>)
+declare i64 @llvm.aarch64.sve.cntp.nxv2i1(<vscale x 2 x i1>, <vscale x 2 x i1>)
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-fp-arith.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-fp-arith.ll
new file mode 100644
index 000000000..16c8529a8
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-fp-arith.ll
@@ -0,0 +1,1681 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+;
+; FABD
+;
+
+define <vscale x 8 x half> @fabd_h(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: fabd_h:
+; CHECK: fabd z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.fabd.nxv8f16(<vscale x 8 x i1> %pg,
+                                                                 <vscale x 8 x half> %a,
+                                                                 <vscale x 8 x half> %b)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x float> @fabd_s(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: fabd_s:
+; CHECK: fabd z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.fabd.nxv4f32(<vscale x 4 x i1> %pg,
+                                                                  <vscale x 4 x float> %a,
+                                                                  <vscale x 4 x float> %b)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @fabd_d(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: fabd_d:
+; CHECK: fabd z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.fabd.nxv2f64(<vscale x 2 x i1> %pg,
+                                                                   <vscale x 2 x double> %a,
+                                                                   <vscale x 2 x double> %b)
+  ret <vscale x 2 x double> %out
+}
+
+;
+; FABS
+;
+
+define <vscale x 8 x half> @fabs_h(<vscale x 8 x half> %a, <vscale x 8 x i1> %pg, <vscale x 8 x half> %b) {
+; CHECK-LABEL: fabs_h:
+; CHECK: fabs z0.h, p0/m, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.fabs.nxv8f16(<vscale x 8 x half> %a,
+                                                                 <vscale x 8 x i1> %pg,
+                                                                 <vscale x 8 x half> %b)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x float> @fabs_s(<vscale x 4 x float> %a, <vscale x 4 x i1> %pg, <vscale x 4 x float> %b) {
+; CHECK-LABEL: fabs_s:
+; CHECK: fabs z0.s, p0/m, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.fabs.nxv4f32(<vscale x 4 x float> %a,
+                                                                  <vscale x 4 x i1> %pg,
+                                                                  <vscale x 4 x float> %b)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @fabs_d(<vscale x 2 x double> %a, <vscale x 2 x i1> %pg, <vscale x 2 x double> %b) {
+; CHECK-LABEL: fabs_d:
+; CHECK: fabs z0.d, p0/m, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.fabs.nxv2f64(<vscale x 2 x double> %a,
+                                                                   <vscale x 2 x i1> %pg,
+                                                                   <vscale x 2 x double> %b)
+  ret <vscale x 2 x double> %out
+}
+
+;
+; FADD
+;
+
+define <vscale x 8 x half> @fadd_h(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: fadd_h:
+; CHECK: fadd z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.fadd.nxv8f16(<vscale x 8 x i1> %pg,
+                                                                 <vscale x 8 x half> %a,
+                                                                 <vscale x 8 x half> %b)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x float> @fadd_s(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: fadd_s:
+; CHECK: fadd z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.fadd.nxv4f32(<vscale x 4 x i1> %pg,
+                                                                  <vscale x 4 x float> %a,
+                                                                  <vscale x 4 x float> %b)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @fadd_d(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: fadd_d:
+; CHECK: fadd z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.fadd.nxv2f64(<vscale x 2 x i1> %pg,
+                                                                   <vscale x 2 x double> %a,
+                                                                   <vscale x 2 x double> %b)
+  ret <vscale x 2 x double> %out
+}
+
+;
+; FCADD
+;
+
+define <vscale x 8 x half> @fcadd_h(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: fcadd_h:
+; CHECK: fcadd z0.h, p0/m, z0.h, z1.h, #90
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.fcadd.nxv8f16(<vscale x 8 x i1> %pg,
+                                                                  <vscale x 8 x half> %a,
+                                                                  <vscale x 8 x half> %b,
+                                                                  i32 90)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x float> @fcadd_s(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: fcadd_s:
+; CHECK: fcadd z0.s, p0/m, z0.s, z1.s, #270
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.fcadd.nxv4f32(<vscale x 4 x i1> %pg,
+                                                                   <vscale x 4 x float> %a,
+                                                                   <vscale x 4 x float> %b,
+                                                                   i32 270)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @fcadd_d(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: fcadd_d:
+; CHECK: fcadd z0.d, p0/m, z0.d, z1.d, #90
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.fcadd.nxv2f64(<vscale x 2 x i1> %pg,
+                                                                    <vscale x 2 x double> %a,
+                                                                    <vscale x 2 x double> %b,
+                                                                    i32 90)
+  ret <vscale x 2 x double> %out
+}
+
+;
+; FCMLA
+;
+
+define <vscale x 8 x half> @fcmla_h(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> %b, <vscale x 8 x half> %c) {
+; CHECK-LABEL: fcmla_h:
+; CHECK: fcmla z0.h, p0/m, z1.h, z2.h, #90
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.fcmla.nxv8f16(<vscale x 8 x i1> %pg,
+                                                                  <vscale x 8 x half> %a,
+                                                                  <vscale x 8 x half> %b,
+                                                                  <vscale x 8 x half> %c,
+                                                                  i32 90)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x float> @fcmla_s(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> %b, <vscale x 4 x float> %c) {
+; CHECK-LABEL: fcmla_s:
+; CHECK: fcmla z0.s, p0/m, z1.s, z2.s, #180
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.fcmla.nxv4f32(<vscale x 4 x i1> %pg,
+                                                                   <vscale x 4 x float> %a,
+                                                                   <vscale x 4 x float> %b,
+                                                                   <vscale x 4 x float> %c,
+                                                                   i32 180)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @fcmla_d(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> %b, <vscale x 2 x double> %c) {
+; CHECK-LABEL: fcmla_d:
+; CHECK: fcmla z0.d, p0/m, z1.d, z2.d, #270
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.fcmla.nxv2f64(<vscale x 2 x i1> %pg,
+                                                                    <vscale x 2 x double> %a,
+                                                                    <vscale x 2 x double> %b,
+                                                                    <vscale x 2 x double> %c,
+                                                                    i32 270)
+  ret <vscale x 2 x double> %out
+}
+
+;
+; FCMLA (Indexed)
+;
+
+define <vscale x 8 x half> @fcmla_lane_h(<vscale x 8 x half> %a, <vscale x 8 x half> %b, <vscale x 8 x half> %c) {
+; CHECK-LABEL: fcmla_lane_h:
+; CHECK: fcmla z0.h, z1.h, z2.h[3], #0
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.fcmla.lane.nxv8f16(<vscale x 8 x half> %a,
+                                                                       <vscale x 8 x half> %b,
+                                                                       <vscale x 8 x half> %c,
+                                                                       i32 3,
+                                                                       i32 0)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x float> @fcmla_lane_s(<vscale x 4 x float> %a, <vscale x 4 x float> %b, <vscale x 4 x float> %c) {
+; CHECK-LABEL: fcmla_lane_s:
+; CHECK: fcmla z0.s, z1.s, z2.s[1], #90
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.fcmla.lane.nxv4f32(<vscale x 4 x float> %a,
+                                                                        <vscale x 4 x float> %b,
+                                                                        <vscale x 4 x float> %c,
+                                                                        i32 1,
+                                                                        i32 90)
+  ret <vscale x 4 x float> %out
+}
+
+;
+; FDIV
+;
+
+define <vscale x 8 x half> @fdiv_h(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: fdiv_h:
+; CHECK: fdiv z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.fdiv.nxv8f16(<vscale x 8 x i1> %pg,
+                                                                 <vscale x 8 x half> %a,
+                                                                 <vscale x 8 x half> %b)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x float> @fdiv_s(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: fdiv_s:
+; CHECK: fdiv z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.fdiv.nxv4f32(<vscale x 4 x i1> %pg,
+                                                                  <vscale x 4 x float> %a,
+                                                                  <vscale x 4 x float> %b)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @fdiv_d(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: fdiv_d:
+; CHECK: fdiv z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.fdiv.nxv2f64(<vscale x 2 x i1> %pg,
+                                                                   <vscale x 2 x double> %a,
+                                                                   <vscale x 2 x double> %b)
+  ret <vscale x 2 x double> %out
+}
+
+;
+; FDIVR
+;
+
+define <vscale x 8 x half> @fdivr_h(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: fdivr_h:
+; CHECK: fdivr z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.fdivr.nxv8f16(<vscale x 8 x i1> %pg,
+                                                                  <vscale x 8 x half> %a,
+                                                                  <vscale x 8 x half> %b)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x float> @fdivr_s(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: fdivr_s:
+; CHECK: fdivr z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.fdivr.nxv4f32(<vscale x 4 x i1> %pg,
+                                                                   <vscale x 4 x float> %a,
+                                                                   <vscale x 4 x float> %b)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @fdivr_d(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: fdivr_d:
+; CHECK: fdivr z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.fdivr.nxv2f64(<vscale x 2 x i1> %pg,
+                                                                    <vscale x 2 x double> %a,
+                                                                    <vscale x 2 x double> %b)
+  ret <vscale x 2 x double> %out
+}
+
+;
+; FEXPA
+;
+
+define <vscale x 8 x half> @fexpa_h(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: fexpa_h:
+; CHECK: fexpa z0.h, z0.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.fexpa.x.nxv8f16(<vscale x 8 x i16> %a)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x float> @fexpa_s(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: fexpa_s:
+; CHECK: fexpa z0.s, z0.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.fexpa.x.nxv4f32(<vscale x 4 x i32> %a)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @fexpa_d(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a) {
+; CHECK-LABEL: fexpa_d:
+; CHECK: fexpa z0.d, z0.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.fexpa.x.nxv2f64(<vscale x 2 x i64> %a)
+  ret <vscale x 2 x double> %out
+}
+
+;
+; FMAD
+;
+
+define <vscale x 8 x half> @fmad_h(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> %b, <vscale x 8 x half> %c) {
+; CHECK-LABEL: fmad_h:
+; CHECK: fmad z0.h, p0/m, z1.h, z2.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.fmad.nxv8f16(<vscale x 8 x i1> %pg,
+                                                                 <vscale x 8 x half> %a,
+                                                                 <vscale x 8 x half> %b,
+                                                                 <vscale x 8 x half> %c)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x float> @fmad_s(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> %b, <vscale x 4 x float> %c) {
+; CHECK-LABEL: fmad_s:
+; CHECK: fmad z0.s, p0/m, z1.s, z2.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.fmad.nxv4f32(<vscale x 4 x i1> %pg,
+                                                                  <vscale x 4 x float> %a,
+                                                                  <vscale x 4 x float> %b,
+                                                                  <vscale x 4 x float> %c)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @fmad_d(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> %b, <vscale x 2 x double> %c) {
+; CHECK-LABEL: fmad_d:
+; CHECK: fmad z0.d, p0/m, z1.d, z2.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.fmad.nxv2f64(<vscale x 2 x i1> %pg,
+                                                                   <vscale x 2 x double> %a,
+                                                                   <vscale x 2 x double> %b,
+                                                                   <vscale x 2 x double> %c)
+  ret <vscale x 2 x double> %out
+}
+
+;
+; FMAX
+;
+
+define <vscale x 8 x half> @fmax_h(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: fmax_h:
+; CHECK: fmax z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.fmax.nxv8f16(<vscale x 8 x i1> %pg,
+                                                                 <vscale x 8 x half> %a,
+                                                                 <vscale x 8 x half> %b)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x float> @fmax_s(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: fmax_s:
+; CHECK: fmax z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.fmax.nxv4f32(<vscale x 4 x i1> %pg,
+                                                                  <vscale x 4 x float> %a,
+                                                                  <vscale x 4 x float> %b)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @fmax_d(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: fmax_d:
+; CHECK: fmax z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.fmax.nxv2f64(<vscale x 2 x i1> %pg,
+                                                                   <vscale x 2 x double> %a,
+                                                                   <vscale x 2 x double> %b)
+  ret <vscale x 2 x double> %out
+}
+
+;
+; FMAXNM
+;
+
+define <vscale x 8 x half> @fmaxnm_h(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: fmaxnm_h:
+; CHECK: fmaxnm z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.fmaxnm.nxv8f16(<vscale x 8 x i1> %pg,
+                                                                   <vscale x 8 x half> %a,
+                                                                   <vscale x 8 x half> %b)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x float> @fmaxnm_s(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: fmaxnm_s:
+; CHECK: fmaxnm z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.fmaxnm.nxv4f32(<vscale x 4 x i1> %pg,
+                                                                    <vscale x 4 x float> %a,
+                                                                    <vscale x 4 x float> %b)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @fmaxnm_d(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: fmaxnm_d:
+; CHECK: fmaxnm z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.fmaxnm.nxv2f64(<vscale x 2 x i1> %pg,
+                                                                     <vscale x 2 x double> %a,
+                                                                     <vscale x 2 x double> %b)
+  ret <vscale x 2 x double> %out
+}
+
+;
+; FMIN
+;
+
+define <vscale x 8 x half> @fmin_h(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: fmin_h:
+; CHECK: fmin z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.fmin.nxv8f16(<vscale x 8 x i1> %pg,
+                                                                 <vscale x 8 x half> %a,
+                                                                 <vscale x 8 x half> %b)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x float> @fmin_s(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: fmin_s:
+; CHECK: fmin z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.fmin.nxv4f32(<vscale x 4 x i1> %pg,
+                                                                  <vscale x 4 x float> %a,
+                                                                  <vscale x 4 x float> %b)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @fmin_d(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: fmin_d:
+; CHECK: fmin z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.fmin.nxv2f64(<vscale x 2 x i1> %pg,
+                                                                   <vscale x 2 x double> %a,
+                                                                   <vscale x 2 x double> %b)
+  ret <vscale x 2 x double> %out
+}
+
+;
+; FMINNM
+;
+
+define <vscale x 8 x half> @fminnm_h(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: fminnm_h:
+; CHECK: fminnm z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.fminnm.nxv8f16(<vscale x 8 x i1> %pg,
+                                                                   <vscale x 8 x half> %a,
+                                                                   <vscale x 8 x half> %b)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x float> @fminnm_s(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: fminnm_s:
+; CHECK: fminnm z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.fminnm.nxv4f32(<vscale x 4 x i1> %pg,
+                                                                    <vscale x 4 x float> %a,
+                                                                    <vscale x 4 x float> %b)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @fminnm_d(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: fminnm_d:
+; CHECK: fminnm z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.fminnm.nxv2f64(<vscale x 2 x i1> %pg,
+                                                                     <vscale x 2 x double> %a,
+                                                                     <vscale x 2 x double> %b)
+  ret <vscale x 2 x double> %out
+}
+
+;
+; FMLA
+;
+
+define <vscale x 8 x half> @fmla_h(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> %b, <vscale x 8 x half> %c) {
+; CHECK-LABEL: fmla_h:
+; CHECK: fmla z0.h, p0/m, z1.h, z2.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.fmla.nxv8f16(<vscale x 8 x i1> %pg,
+                                                                 <vscale x 8 x half> %a,
+                                                                 <vscale x 8 x half> %b,
+                                                                 <vscale x 8 x half> %c)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x float> @fmla_s(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> %b, <vscale x 4 x float> %c) {
+; CHECK-LABEL: fmla_s:
+; CHECK: fmla z0.s, p0/m, z1.s, z2.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.fmla.nxv4f32(<vscale x 4 x i1> %pg,
+                                                                  <vscale x 4 x float> %a,
+                                                                  <vscale x 4 x float> %b,
+                                                                  <vscale x 4 x float> %c)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @fmla_d(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> %b, <vscale x 2 x double> %c) {
+; CHECK-LABEL: fmla_d:
+; CHECK: fmla z0.d, p0/m, z1.d, z2.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.fmla.nxv2f64(<vscale x 2 x i1> %pg,
+                                                                   <vscale x 2 x double> %a,
+                                                                   <vscale x 2 x double> %b,
+                                                                   <vscale x 2 x double> %c)
+  ret <vscale x 2 x double> %out
+}
+
+;
+; FMLA (Indexed)
+;
+
+define <vscale x 8 x half> @fmla_lane_h(<vscale x 8 x half> %a, <vscale x 8 x half> %b, <vscale x 8 x half> %c) {
+; CHECK-LABEL: fmla_lane_h:
+; CHECK: fmla z0.h, z1.h, z2.h[3]
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.fmla.lane.nxv8f16(<vscale x 8 x half> %a,
+                                                                      <vscale x 8 x half> %b,
+                                                                      <vscale x 8 x half> %c,
+                                                                      i32 3)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x float> @fmla_lane_s(<vscale x 4 x float> %a, <vscale x 4 x float> %b, <vscale x 4 x float> %c) {
+; CHECK-LABEL: fmla_lane_s:
+; CHECK: fmla z0.s, z1.s, z2.s[2]
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.fmla.lane.nxv4f32(<vscale x 4 x float> %a,
+                                                                       <vscale x 4 x float> %b,
+                                                                       <vscale x 4 x float> %c,
+                                                                       i32 2)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @fmla_lane_d(<vscale x 2 x double> %a, <vscale x 2 x double> %b, <vscale x 2 x double> %c) {
+; CHECK-LABEL: fmla_lane_d:
+; CHECK: fmla z0.d, z1.d, z2.d[1]
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.fmla.lane.nxv2f64(<vscale x 2 x double> %a,
+                                                                        <vscale x 2 x double> %b,
+                                                                        <vscale x 2 x double> %c,
+                                                                        i32 1)
+  ret <vscale x 2 x double> %out
+}
+
+;
+; FMLS
+;
+
+define <vscale x 8 x half> @fmls_h(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> %b, <vscale x 8 x half> %c) {
+; CHECK-LABEL: fmls_h:
+; CHECK: fmls z0.h, p0/m, z1.h, z2.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.fmls.nxv8f16(<vscale x 8 x i1> %pg,
+                                                                 <vscale x 8 x half> %a,
+                                                                 <vscale x 8 x half> %b,
+                                                                 <vscale x 8 x half> %c)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x float> @fmls_s(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> %b, <vscale x 4 x float> %c) {
+; CHECK-LABEL: fmls_s:
+; CHECK: fmls z0.s, p0/m, z1.s, z2.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.fmls.nxv4f32(<vscale x 4 x i1> %pg,
+                                                                  <vscale x 4 x float> %a,
+                                                                  <vscale x 4 x float> %b,
+                                                                  <vscale x 4 x float> %c)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @fmls_d(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> %b, <vscale x 2 x double> %c) {
+; CHECK-LABEL: fmls_d:
+; CHECK: fmls z0.d, p0/m, z1.d, z2.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.fmls.nxv2f64(<vscale x 2 x i1> %pg,
+                                                                   <vscale x 2 x double> %a,
+                                                                   <vscale x 2 x double> %b,
+                                                                   <vscale x 2 x double> %c)
+  ret <vscale x 2 x double> %out
+}
+
+;
+; FMLS (Indexed)
+;
+
+define <vscale x 8 x half> @fmls_lane_h(<vscale x 8 x half> %a, <vscale x 8 x half> %b, <vscale x 8 x half> %c) {
+; CHECK-LABEL: fmls_lane_h:
+; CHECK: fmls z0.h, z1.h, z2.h[3]
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.fmls.lane.nxv8f16(<vscale x 8 x half> %a,
+                                                                 <vscale x 8 x half> %b,
+                                                                 <vscale x 8 x half> %c,
+                                                                 i32 3)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x float> @fmls_lane_s(<vscale x 4 x float> %a, <vscale x 4 x float> %b, <vscale x 4 x float> %c) {
+; CHECK-LABEL: fmls_lane_s:
+; CHECK: fmls z0.s, z1.s, z2.s[2]
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.fmls.lane.nxv4f32(<vscale x 4 x float> %a,
+                                                                  <vscale x 4 x float> %b,
+                                                                  <vscale x 4 x float> %c,
+                                                                  i32 2)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @fmls_lane_d(<vscale x 2 x double> %a, <vscale x 2 x double> %b, <vscale x 2 x double> %c) {
+; CHECK-LABEL: fmls_lane_d:
+; CHECK: fmls z0.d, z1.d, z2.d[1]
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.fmls.lane.nxv2f64(<vscale x 2 x double> %a,
+                                                                   <vscale x 2 x double> %b,
+                                                                   <vscale x 2 x double> %c,
+                                                                   i32 1)
+  ret <vscale x 2 x double> %out
+}
+
+;
+; FMSB
+;
+
+define <vscale x 8 x half> @fmsb_h(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> %b, <vscale x 8 x half> %c) {
+; CHECK-LABEL: fmsb_h:
+; CHECK: fmsb z0.h, p0/m, z1.h, z2.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.fmsb.nxv8f16(<vscale x 8 x i1> %pg,
+                                                                 <vscale x 8 x half> %a,
+                                                                 <vscale x 8 x half> %b,
+                                                                 <vscale x 8 x half> %c)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x float> @fmsb_s(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> %b, <vscale x 4 x float> %c) {
+; CHECK-LABEL: fmsb_s:
+; CHECK: fmsb z0.s, p0/m, z1.s, z2.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.fmsb.nxv4f32(<vscale x 4 x i1> %pg,
+                                                                  <vscale x 4 x float> %a,
+                                                                  <vscale x 4 x float> %b,
+                                                                  <vscale x 4 x float> %c)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @fmsb_d(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> %b, <vscale x 2 x double> %c) {
+; CHECK-LABEL: fmsb_d:
+; CHECK: fmsb z0.d, p0/m, z1.d, z2.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.fmsb.nxv2f64(<vscale x 2 x i1> %pg,
+                                                                   <vscale x 2 x double> %a,
+                                                                   <vscale x 2 x double> %b,
+                                                                   <vscale x 2 x double> %c)
+  ret <vscale x 2 x double> %out
+}
+
+;
+; FMUL
+;
+
+define <vscale x 8 x half> @fmul_h(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: fmul_h:
+; CHECK: fmul z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.fmul.nxv8f16(<vscale x 8 x i1> %pg,
+                                                                 <vscale x 8 x half> %a,
+                                                                 <vscale x 8 x half> %b)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x float> @fmul_s(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: fmul_s:
+; CHECK: fmul z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.fmul.nxv4f32(<vscale x 4 x i1> %pg,
+                                                                  <vscale x 4 x float> %a,
+                                                                  <vscale x 4 x float> %b)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @fmul_d(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: fmul_d:
+; CHECK: fmul z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.fmul.nxv2f64(<vscale x 2 x i1> %pg,
+                                                                   <vscale x 2 x double> %a,
+                                                                   <vscale x 2 x double> %b)
+  ret <vscale x 2 x double> %out
+}
+
+;
+; FMUL (Indexed)
+;
+
+define <vscale x 8 x half> @fmul_lane_h(<vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: fmul_lane_h:
+; CHECK: fmul z0.h, z0.h, z1.h[3]
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.fmul.lane.nxv8f16(<vscale x 8 x half> %a,
+                                                                      <vscale x 8 x half> %b,
+                                                                      i32 3)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x float> @fmul_lane_s(<vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: fmul_lane_s:
+; CHECK: fmul z0.s, z0.s, z1.s[2]
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.fmul.lane.nxv4f32(<vscale x 4 x float> %a,
+                                                                       <vscale x 4 x float> %b,
+                                                                       i32 2)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @fmul_lane_d(<vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: fmul_lane_d:
+; CHECK: fmul z0.d, z0.d, z1.d[1]
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.fmul.lane.nxv2f64(<vscale x 2 x double> %a,
+                                                                        <vscale x 2 x double> %b,
+                                                                        i32 1)
+  ret <vscale x 2 x double> %out
+}
+
+;
+; FMULX
+;
+
+define <vscale x 8 x half> @fmulx_h(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: fmulx_h:
+; CHECK: fmulx z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.fmulx.nxv8f16(<vscale x 8 x i1> %pg,
+                                                                  <vscale x 8 x half> %a,
+                                                                  <vscale x 8 x half> %b)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x float> @fmulx_s(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: fmulx_s:
+; CHECK: fmulx z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.fmulx.nxv4f32(<vscale x 4 x i1> %pg,
+                                                                   <vscale x 4 x float> %a,
+                                                                   <vscale x 4 x float> %b)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @fmulx_d(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: fmulx_d:
+; CHECK: fmulx z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.fmulx.nxv2f64(<vscale x 2 x i1> %pg,
+                                                                    <vscale x 2 x double> %a,
+                                                                    <vscale x 2 x double> %b)
+  ret <vscale x 2 x double> %out
+}
+
+;
+; FNEG
+;
+
+define <vscale x 8 x half> @fneg_h(<vscale x 8 x half> %a, <vscale x 8 x i1> %pg, <vscale x 8 x half> %b) {
+; CHECK-LABEL: fneg_h:
+; CHECK: fneg z0.h, p0/m, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.fneg.nxv8f16(<vscale x 8 x half> %a,
+                                                                 <vscale x 8 x i1> %pg,
+                                                                 <vscale x 8 x half> %b)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x float> @fneg_s(<vscale x 4 x float> %a, <vscale x 4 x i1> %pg, <vscale x 4 x float> %b) {
+; CHECK-LABEL: fneg_s:
+; CHECK: fneg z0.s, p0/m, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.fneg.nxv4f32(<vscale x 4 x float> %a,
+                                                                  <vscale x 4 x i1> %pg,
+                                                                  <vscale x 4 x float> %b)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @fneg_d(<vscale x 2 x double> %a, <vscale x 2 x i1> %pg, <vscale x 2 x double> %b) {
+; CHECK-LABEL: fneg_d:
+; CHECK: fneg z0.d, p0/m, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.fneg.nxv2f64(<vscale x 2 x double> %a,
+                                                                   <vscale x 2 x i1> %pg,
+                                                                   <vscale x 2 x double> %b)
+  ret <vscale x 2 x double> %out
+}
+
+;
+; FNMAD
+;
+
+define <vscale x 8 x half> @fnmad_h(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> %b, <vscale x 8 x half> %c) {
+; CHECK-LABEL: fnmad_h:
+; CHECK: fnmad z0.h, p0/m, z1.h, z2.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.fnmad.nxv8f16(<vscale x 8 x i1> %pg,
+                                                                  <vscale x 8 x half> %a,
+                                                                  <vscale x 8 x half> %b,
+                                                                  <vscale x 8 x half> %c)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x float> @fnmad_s(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> %b, <vscale x 4 x float> %c) {
+; CHECK-LABEL: fnmad_s:
+; CHECK: fnmad z0.s, p0/m, z1.s, z2.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.fnmad.nxv4f32(<vscale x 4 x i1> %pg,
+                                                                   <vscale x 4 x float> %a,
+                                                                   <vscale x 4 x float> %b,
+                                                                   <vscale x 4 x float> %c)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @fnmad_d(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> %b, <vscale x 2 x double> %c) {
+; CHECK-LABEL: fnmad_d:
+; CHECK: fnmad z0.d, p0/m, z1.d, z2.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.fnmad.nxv2f64(<vscale x 2 x i1> %pg,
+                                                                    <vscale x 2 x double> %a,
+                                                                    <vscale x 2 x double> %b,
+                                                                    <vscale x 2 x double> %c)
+  ret <vscale x 2 x double> %out
+}
+
+;
+; FNMLA
+;
+
+define <vscale x 8 x half> @fnmla_h(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> %b, <vscale x 8 x half> %c) {
+; CHECK-LABEL: fnmla_h:
+; CHECK: fnmla z0.h, p0/m, z1.h, z2.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.fnmla.nxv8f16(<vscale x 8 x i1> %pg,
+                                                                  <vscale x 8 x half> %a,
+                                                                  <vscale x 8 x half> %b,
+                                                                  <vscale x 8 x half> %c)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x float> @fnmla_s(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> %b, <vscale x 4 x float> %c) {
+; CHECK-LABEL: fnmla_s:
+; CHECK: fnmla z0.s, p0/m, z1.s, z2.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.fnmla.nxv4f32(<vscale x 4 x i1> %pg,
+                                                                   <vscale x 4 x float> %a,
+                                                                   <vscale x 4 x float> %b,
+                                                                   <vscale x 4 x float> %c)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @fnmla_d(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> %b, <vscale x 2 x double> %c) {
+; CHECK-LABEL: fnmla_d:
+; CHECK: fnmla z0.d, p0/m, z1.d, z2.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.fnmla.nxv2f64(<vscale x 2 x i1> %pg,
+                                                                    <vscale x 2 x double> %a,
+                                                                    <vscale x 2 x double> %b,
+                                                                    <vscale x 2 x double> %c)
+  ret <vscale x 2 x double> %out
+}
+
+;
+; FNMLS
+;
+
+define <vscale x 8 x half> @fnmls_h(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> %b, <vscale x 8 x half> %c) {
+; CHECK-LABEL: fnmls_h:
+; CHECK: fnmls z0.h, p0/m, z1.h, z2.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.fnmls.nxv8f16(<vscale x 8 x i1> %pg,
+                                                                  <vscale x 8 x half> %a,
+                                                                  <vscale x 8 x half> %b,
+                                                                  <vscale x 8 x half> %c)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x float> @fnmls_s(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> %b, <vscale x 4 x float> %c) {
+; CHECK-LABEL: fnmls_s:
+; CHECK: fnmls z0.s, p0/m, z1.s, z2.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.fnmls.nxv4f32(<vscale x 4 x i1> %pg,
+                                                                   <vscale x 4 x float> %a,
+                                                                   <vscale x 4 x float> %b,
+                                                                   <vscale x 4 x float> %c)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @fnmls_d(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> %b, <vscale x 2 x double> %c) {
+; CHECK-LABEL: fnmls_d:
+; CHECK: fnmls z0.d, p0/m, z1.d, z2.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.fnmls.nxv2f64(<vscale x 2 x i1> %pg,
+                                                                    <vscale x 2 x double> %a,
+                                                                    <vscale x 2 x double> %b,
+                                                                    <vscale x 2 x double> %c)
+  ret <vscale x 2 x double> %out
+}
+
+;
+; FNMSB
+;
+
+define <vscale x 8 x half> @fnmsb_h(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> %b, <vscale x 8 x half> %c) {
+; CHECK-LABEL: fnmsb_h:
+; CHECK: fnmsb z0.h, p0/m, z1.h, z2.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.fnmsb.nxv8f16(<vscale x 8 x i1> %pg,
+                                                                  <vscale x 8 x half> %a,
+                                                                  <vscale x 8 x half> %b,
+                                                                  <vscale x 8 x half> %c)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x float> @fnmsb_s(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> %b, <vscale x 4 x float> %c) {
+; CHECK-LABEL: fnmsb_s:
+; CHECK: fnmsb z0.s, p0/m, z1.s, z2.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.fnmsb.nxv4f32(<vscale x 4 x i1> %pg,
+                                                                   <vscale x 4 x float> %a,
+                                                                   <vscale x 4 x float> %b,
+                                                                   <vscale x 4 x float> %c)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @fnmsb_d(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> %b, <vscale x 2 x double> %c) {
+; CHECK-LABEL: fnmsb_d:
+; CHECK: fnmsb z0.d, p0/m, z1.d, z2.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.fnmsb.nxv2f64(<vscale x 2 x i1> %pg,
+                                                                    <vscale x 2 x double> %a,
+                                                                    <vscale x 2 x double> %b,
+                                                                    <vscale x 2 x double> %c)
+  ret <vscale x 2 x double> %out
+}
+
+;
+; FRECPE
+;
+
+define <vscale x 8 x half> @frecpe_h(<vscale x 8 x half> %a) {
+; CHECK-LABEL: frecpe_h:
+; CHECK: frecpe z0.h, z0.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.frecpe.x.nxv8f16(<vscale x 8 x half> %a)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x float> @frecpe_s(<vscale x 4 x float> %a) {
+; CHECK-LABEL: frecpe_s:
+; CHECK: frecpe z0.s, z0.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.frecpe.x.nxv4f32(<vscale x 4 x float> %a)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @frecpe_d(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a) {
+; CHECK-LABEL: frecpe_d:
+; CHECK: frecpe z0.d, z0.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.frecpe.x.nxv2f64(<vscale x 2 x double> %a)
+  ret <vscale x 2 x double> %out
+}
+
+;
+; FRECPX
+;
+
+define <vscale x 8 x half> @frecpx_h(<vscale x 8 x half> %a, <vscale x 8 x i1> %pg, <vscale x 8 x half> %b) {
+; CHECK-LABEL: frecpx_h:
+; CHECK: frecpx z0.h, p0/m, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.frecpx.nxv8f16(<vscale x 8 x half> %a,
+                                                                  <vscale x 8 x i1> %pg,
+                                                                  <vscale x 8 x half> %b)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x float> @frecpx_s(<vscale x 4 x float> %a, <vscale x 4 x i1> %pg, <vscale x 4 x float> %b) {
+; CHECK-LABEL: frecpx_s:
+; CHECK: frecpx z0.s, p0/m, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.frecpx.nxv4f32(<vscale x 4 x float> %a,
+                                                                   <vscale x 4 x i1> %pg,
+                                                                   <vscale x 4 x float> %b)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @frecpx_d(<vscale x 2 x double> %a, <vscale x 2 x i1> %pg, <vscale x 2 x double> %b) {
+; CHECK-LABEL: frecpx_d:
+; CHECK: frecpx z0.d, p0/m, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.frecpx.nxv2f64(<vscale x 2 x double> %a,
+                                                                    <vscale x 2 x i1> %pg,
+                                                                    <vscale x 2 x double> %b)
+  ret <vscale x 2 x double> %out
+}
+
+;
+; FRINTA
+;
+
+define <vscale x 8 x half> @frinta_h(<vscale x 8 x half> %a, <vscale x 8 x i1> %pg, <vscale x 8 x half> %b) {
+; CHECK-LABEL: frinta_h:
+; CHECK: frinta z0.h, p0/m, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.frinta.nxv8f16(<vscale x 8 x half> %a,
+                                                                   <vscale x 8 x i1> %pg,
+                                                                   <vscale x 8 x half> %b)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x float> @frinta_s(<vscale x 4 x float> %a, <vscale x 4 x i1> %pg, <vscale x 4 x float> %b) {
+; CHECK-LABEL: frinta_s:
+; CHECK: frinta z0.s, p0/m, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.frinta.nxv4f32(<vscale x 4 x float> %a,
+                                                                    <vscale x 4 x i1> %pg,
+                                                                    <vscale x 4 x float> %b)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @frinta_d(<vscale x 2 x double> %a, <vscale x 2 x i1> %pg, <vscale x 2 x double> %b) {
+; CHECK-LABEL: frinta_d:
+; CHECK: frinta z0.d, p0/m, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.frinta.nxv2f64(<vscale x 2 x double> %a,
+                                                                     <vscale x 2 x i1> %pg,
+                                                                     <vscale x 2 x double> %b)
+  ret <vscale x 2 x double> %out
+}
+
+;
+; FRINTI
+;
+
+define <vscale x 8 x half> @frinti_h(<vscale x 8 x half> %a, <vscale x 8 x i1> %pg, <vscale x 8 x half> %b) {
+; CHECK-LABEL: frinti_h:
+; CHECK: frinti z0.h, p0/m, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.frinti.nxv8f16(<vscale x 8 x half> %a,
+                                                                   <vscale x 8 x i1> %pg,
+                                                                   <vscale x 8 x half> %b)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x float> @frinti_s(<vscale x 4 x float> %a, <vscale x 4 x i1> %pg, <vscale x 4 x float> %b) {
+; CHECK-LABEL: frinti_s:
+; CHECK: frinti z0.s, p0/m, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.frinti.nxv4f32(<vscale x 4 x float> %a,
+                                                                    <vscale x 4 x i1> %pg,
+                                                                    <vscale x 4 x float> %b)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @frinti_d(<vscale x 2 x double> %a, <vscale x 2 x i1> %pg, <vscale x 2 x double> %b) {
+; CHECK-LABEL: frinti_d:
+; CHECK: frinti z0.d, p0/m, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.frinti.nxv2f64(<vscale x 2 x double> %a,
+                                                                     <vscale x 2 x i1> %pg,
+                                                                     <vscale x 2 x double> %b)
+  ret <vscale x 2 x double> %out
+}
+
+;
+; FRINTM
+;
+
+define <vscale x 8 x half> @frintm_h(<vscale x 8 x half> %a, <vscale x 8 x i1> %pg, <vscale x 8 x half> %b) {
+; CHECK-LABEL: frintm_h:
+; CHECK: frintm z0.h, p0/m, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.frintm.nxv8f16(<vscale x 8 x half> %a,
+                                                                   <vscale x 8 x i1> %pg,
+                                                                   <vscale x 8 x half> %b)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x float> @frintm_s(<vscale x 4 x float> %a, <vscale x 4 x i1> %pg, <vscale x 4 x float> %b) {
+; CHECK-LABEL: frintm_s:
+; CHECK: frintm z0.s, p0/m, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.frintm.nxv4f32(<vscale x 4 x float> %a,
+                                                                    <vscale x 4 x i1> %pg,
+                                                                    <vscale x 4 x float> %b)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @frintm_d(<vscale x 2 x double> %a, <vscale x 2 x i1> %pg, <vscale x 2 x double> %b) {
+; CHECK-LABEL: frintm_d:
+; CHECK: frintm z0.d, p0/m, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.frintm.nxv2f64(<vscale x 2 x double> %a,
+                                                                     <vscale x 2 x i1> %pg,
+                                                                     <vscale x 2 x double> %b)
+  ret <vscale x 2 x double> %out
+}
+
+;
+; FRINTN
+;
+
+define <vscale x 8 x half> @frintn_h(<vscale x 8 x half> %a, <vscale x 8 x i1> %pg, <vscale x 8 x half> %b) {
+; CHECK-LABEL: frintn_h:
+; CHECK: frintn z0.h, p0/m, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.frintn.nxv8f16(<vscale x 8 x half> %a,
+                                                                   <vscale x 8 x i1> %pg,
+                                                                   <vscale x 8 x half> %b)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x float> @frintn_s(<vscale x 4 x float> %a, <vscale x 4 x i1> %pg, <vscale x 4 x float> %b) {
+; CHECK-LABEL: frintn_s:
+; CHECK: frintn z0.s, p0/m, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.frintn.nxv4f32(<vscale x 4 x float> %a,
+                                                                    <vscale x 4 x i1> %pg,
+                                                                    <vscale x 4 x float> %b)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @frintn_d(<vscale x 2 x double> %a, <vscale x 2 x i1> %pg, <vscale x 2 x double> %b) {
+; CHECK-LABEL: frintn_d:
+; CHECK: frintn z0.d, p0/m, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.frintn.nxv2f64(<vscale x 2 x double> %a,
+                                                                     <vscale x 2 x i1> %pg,
+                                                                     <vscale x 2 x double> %b)
+  ret <vscale x 2 x double> %out
+}
+
+;
+; FRINTP
+;
+
+define <vscale x 8 x half> @frintp_h(<vscale x 8 x half> %a, <vscale x 8 x i1> %pg, <vscale x 8 x half> %b) {
+; CHECK-LABEL: frintp_h:
+; CHECK: frintp z0.h, p0/m, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.frintp.nxv8f16(<vscale x 8 x half> %a,
+                                                                   <vscale x 8 x i1> %pg,
+                                                                   <vscale x 8 x half> %b)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x float> @frintp_s(<vscale x 4 x float> %a, <vscale x 4 x i1> %pg, <vscale x 4 x float> %b) {
+; CHECK-LABEL: frintp_s:
+; CHECK: frintp z0.s, p0/m, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.frintp.nxv4f32(<vscale x 4 x float> %a,
+                                                                    <vscale x 4 x i1> %pg,
+                                                                    <vscale x 4 x float> %b)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @frintp_d(<vscale x 2 x double> %a, <vscale x 2 x i1> %pg, <vscale x 2 x double> %b) {
+; CHECK-LABEL: frintp_d:
+; CHECK: frintp z0.d, p0/m, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.frintp.nxv2f64(<vscale x 2 x double> %a,
+                                                                     <vscale x 2 x i1> %pg,
+                                                                     <vscale x 2 x double> %b)
+  ret <vscale x 2 x double> %out
+}
+
+;
+; FRINTX
+;
+
+define <vscale x 8 x half> @frintx_h(<vscale x 8 x half> %a, <vscale x 8 x i1> %pg, <vscale x 8 x half> %b) {
+; CHECK-LABEL: frintx_h:
+; CHECK: frintx z0.h, p0/m, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.frintx.nxv8f16(<vscale x 8 x half> %a,
+                                                                   <vscale x 8 x i1> %pg,
+                                                                   <vscale x 8 x half> %b)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x float> @frintx_s(<vscale x 4 x float> %a, <vscale x 4 x i1> %pg, <vscale x 4 x float> %b) {
+; CHECK-LABEL: frintx_s:
+; CHECK: frintx z0.s, p0/m, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.frintx.nxv4f32(<vscale x 4 x float> %a,
+                                                                    <vscale x 4 x i1> %pg,
+                                                                    <vscale x 4 x float> %b)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @frintx_d(<vscale x 2 x double> %a, <vscale x 2 x i1> %pg, <vscale x 2 x double> %b) {
+; CHECK-LABEL: frintx_d:
+; CHECK: frintx z0.d, p0/m, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.frintx.nxv2f64(<vscale x 2 x double> %a,
+                                                                     <vscale x 2 x i1> %pg,
+                                                                     <vscale x 2 x double> %b)
+  ret <vscale x 2 x double> %out
+}
+
+;
+; FRINTZ
+;
+
+define <vscale x 8 x half> @frintz_h(<vscale x 8 x half> %a, <vscale x 8 x i1> %pg, <vscale x 8 x half> %b) {
+; CHECK-LABEL: frintz_h:
+; CHECK: frintz z0.h, p0/m, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.frintz.nxv8f16(<vscale x 8 x half> %a,
+                                                                   <vscale x 8 x i1> %pg,
+                                                                   <vscale x 8 x half> %b)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x float> @frintz_s(<vscale x 4 x float> %a, <vscale x 4 x i1> %pg, <vscale x 4 x float> %b) {
+; CHECK-LABEL: frintz_s:
+; CHECK: frintz z0.s, p0/m, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.frintz.nxv4f32(<vscale x 4 x float> %a,
+                                                                    <vscale x 4 x i1> %pg,
+                                                                    <vscale x 4 x float> %b)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @frintz_d(<vscale x 2 x double> %a, <vscale x 2 x i1> %pg, <vscale x 2 x double> %b) {
+; CHECK-LABEL: frintz_d:
+; CHECK: frintz z0.d, p0/m, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.frintz.nxv2f64(<vscale x 2 x double> %a,
+                                                                     <vscale x 2 x i1> %pg,
+                                                                     <vscale x 2 x double> %b)
+  ret <vscale x 2 x double> %out
+}
+
+;
+; FRSQRTE
+;
+
+define <vscale x 8 x half> @frsqrte_h(<vscale x 8 x half> %a) {
+; CHECK-LABEL: frsqrte_h:
+; CHECK: frsqrte z0.h, z0.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.frsqrte.x.nxv8f16(<vscale x 8 x half> %a)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x float> @frsqrte_s(<vscale x 4 x float> %a) {
+; CHECK-LABEL: frsqrte_s:
+; CHECK: frsqrte z0.s, z0.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.frsqrte.x.nxv4f32(<vscale x 4 x float> %a)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @frsqrte_d(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a) {
+; CHECK-LABEL: frsqrte_d:
+; CHECK: frsqrte z0.d, z0.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.frsqrte.x.nxv2f64(<vscale x 2 x double> %a)
+  ret <vscale x 2 x double> %out
+}
+
+;
+; FSCALE
+;
+
+define <vscale x 8 x half> @fscale_h(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: fscale_h:
+; CHECK: fscale z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.fscale.nxv8f16(<vscale x 8 x i1> %pg,
+                                                                   <vscale x 8 x half> %a,
+                                                                   <vscale x 8 x i16> %b)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x float> @fscale_s(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: fscale_s:
+; CHECK: fscale z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.fscale.nxv4f32(<vscale x 4 x i1> %pg,
+                                                                    <vscale x 4 x float> %a,
+                                                                    <vscale x 4 x i32> %b)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @fscale_d(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: fscale_d:
+; CHECK: fscale z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.fscale.nxv2f64(<vscale x 2 x i1> %pg,
+                                                                     <vscale x 2 x double> %a,
+                                                                     <vscale x 2 x i64> %b)
+  ret <vscale x 2 x double> %out
+}
+
+;
+; FSQRT
+;
+
+define <vscale x 8 x half> @fsqrt_h(<vscale x 8 x half> %a, <vscale x 8 x i1> %pg, <vscale x 8 x half> %b) {
+; CHECK-LABEL: fsqrt_h:
+; CHECK: fsqrt z0.h, p0/m, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.fsqrt.nxv8f16(<vscale x 8 x half> %a,
+                                                                  <vscale x 8 x i1> %pg,
+                                                                  <vscale x 8 x half> %b)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x float> @fsqrt_s(<vscale x 4 x float> %a, <vscale x 4 x i1> %pg, <vscale x 4 x float> %b) {
+; CHECK-LABEL: fsqrt_s:
+; CHECK: fsqrt z0.s, p0/m, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.fsqrt.nxv4f32(<vscale x 4 x float> %a,
+                                                                   <vscale x 4 x i1> %pg,
+                                                                   <vscale x 4 x float> %b)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @fsqrt_d(<vscale x 2 x double> %a, <vscale x 2 x i1> %pg, <vscale x 2 x double> %b) {
+; CHECK-LABEL: fsqrt_d:
+; CHECK: fsqrt z0.d, p0/m, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.fsqrt.nxv2f64(<vscale x 2 x double> %a,
+                                                                    <vscale x 2 x i1> %pg,
+                                                                    <vscale x 2 x double> %b)
+  ret <vscale x 2 x double> %out
+}
+
+;
+; FSUB
+;
+
+define <vscale x 8 x half> @fsub_h(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: fsub_h:
+; CHECK: fsub z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.fsub.nxv8f16(<vscale x 8 x i1> %pg,
+                                                                 <vscale x 8 x half> %a,
+                                                                 <vscale x 8 x half> %b)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x float> @fsub_s(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: fsub_s:
+; CHECK: fsub z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.fsub.nxv4f32(<vscale x 4 x i1> %pg,
+                                                                  <vscale x 4 x float> %a,
+                                                                  <vscale x 4 x float> %b)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @fsub_d(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: fsub_d:
+; CHECK: fsub z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.fsub.nxv2f64(<vscale x 2 x i1> %pg,
+                                                                   <vscale x 2 x double> %a,
+                                                                   <vscale x 2 x double> %b)
+  ret <vscale x 2 x double> %out
+}
+
+;
+; FSUBR
+;
+
+define <vscale x 8 x half> @fsubr_h(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: fsubr_h:
+; CHECK: fsubr z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.fsubr.nxv8f16(<vscale x 8 x i1> %pg,
+                                                                  <vscale x 8 x half> %a,
+                                                                  <vscale x 8 x half> %b)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x float> @fsubr_s(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: fsubr_s:
+; CHECK: fsubr z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.fsubr.nxv4f32(<vscale x 4 x i1> %pg,
+                                                                   <vscale x 4 x float> %a,
+                                                                   <vscale x 4 x float> %b)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @fsubr_d(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: fsubr_d:
+; CHECK: fsubr z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.fsubr.nxv2f64(<vscale x 2 x i1> %pg,
+                                                                    <vscale x 2 x double> %a,
+                                                                    <vscale x 2 x double> %b)
+  ret <vscale x 2 x double> %out
+}
+
+;
+; FTMAD
+;
+
+define <vscale x 8 x half> @ftmad_h(<vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: ftmad_h:
+; CHECK: ftmad z0.h, z0.h, z1.h, #0
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.ftmad.x.nxv8f16(<vscale x 8 x half> %a,
+                                                                    <vscale x 8 x half> %b,
+                                                                    i32 0)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x float> @ftmad_s(<vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: ftmad_s:
+; CHECK: ftmad z0.s, z0.s, z1.s, #0
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.ftmad.x.nxv4f32(<vscale x 4 x float> %a,
+                                                                     <vscale x 4 x float> %b,
+                                                                     i32 0)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @ftmad_d(<vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: ftmad_d:
+; CHECK: ftmad z0.d, z0.d, z1.d, #7
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.ftmad.x.nxv2f64(<vscale x 2 x double> %a,
+                                                                      <vscale x 2 x double> %b,
+                                                                      i32 7)
+  ret <vscale x 2 x double> %out
+}
+
+;
+; FTSMUL
+;
+
+define <vscale x 8 x half> @ftsmul_h(<vscale x 8 x half> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: ftsmul_h:
+; CHECK: ftsmul z0.h, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.ftsmul.x.nxv8f16(<vscale x 8 x half> %a,
+                                                                     <vscale x 8 x i16> %b)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x float> @ftsmul_s(<vscale x 4 x float> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: ftsmul_s:
+; CHECK: ftsmul z0.s, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.ftsmul.x.nxv4f32(<vscale x 4 x float> %a,
+                                                                      <vscale x 4 x i32> %b)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @ftsmul_d(<vscale x 2 x double> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: ftsmul_d:
+; CHECK: ftsmul z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.ftsmul.x.nxv2f64(<vscale x 2 x double> %a,
+                                                                       <vscale x 2 x i64> %b)
+  ret <vscale x 2 x double> %out
+}
+
+;
+; FTSSEL
+;
+
+define <vscale x 8 x half> @ftssel_h(<vscale x 8 x half> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: ftssel_h:
+; CHECK: ftssel z0.h, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.ftssel.x.nxv8f16(<vscale x 8 x half> %a,
+                                                                     <vscale x 8 x i16> %b)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x float> @ftssel_s(<vscale x 4 x float> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: ftssel_s:
+; CHECK: ftssel z0.s, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.ftssel.x.nxv4f32(<vscale x 4 x float> %a,
+                                                                      <vscale x 4 x i32> %b)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @ftssel_d(<vscale x 2 x double> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: ftssel_d:
+; CHECK: ftssel z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.ftssel.x.nxv2f64(<vscale x 2 x double> %a,
+                                                                       <vscale x 2 x i64> %b)
+  ret <vscale x 2 x double> %out
+}
+
+declare <vscale x 8 x half> @llvm.aarch64.sve.fabd.nxv8f16(<vscale x 8 x i1>, <vscale x 8 x half>, <vscale x 8 x half>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.fabd.nxv4f32(<vscale x 4 x i1>, <vscale x 4 x float>, <vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.fabd.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>, <vscale x 2 x double>)
+
+declare <vscale x 8 x half> @llvm.aarch64.sve.fabs.nxv8f16(<vscale x 8 x half>, <vscale x 8 x i1>, <vscale x 8 x half>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.fabs.nxv4f32(<vscale x 4 x float>, <vscale x 4 x i1>, <vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.fabs.nxv2f64(<vscale x 2 x double>, <vscale x 2 x i1>, <vscale x 2 x double>)
+
+declare <vscale x 8 x half> @llvm.aarch64.sve.fadd.nxv8f16(<vscale x 8 x i1>, <vscale x 8 x half>, <vscale x 8 x half>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.fadd.nxv4f32(<vscale x 4 x i1>, <vscale x 4 x float>, <vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.fadd.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>, <vscale x 2 x double>)
+
+declare <vscale x 8 x half> @llvm.aarch64.sve.fcadd.nxv8f16(<vscale x 8 x i1>, <vscale x 8 x half>, <vscale x 8 x half>, i32)
+declare <vscale x 4 x float> @llvm.aarch64.sve.fcadd.nxv4f32(<vscale x 4 x i1>, <vscale x 4 x float>, <vscale x 4 x float>, i32)
+declare <vscale x 2 x double> @llvm.aarch64.sve.fcadd.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>, <vscale x 2 x double>, i32)
+
+declare <vscale x 8 x half> @llvm.aarch64.sve.fcmla.nxv8f16(<vscale x 8 x i1>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, i32)
+declare <vscale x 4 x float> @llvm.aarch64.sve.fcmla.nxv4f32(<vscale x 4 x i1>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, i32)
+declare <vscale x 2 x double> @llvm.aarch64.sve.fcmla.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, i32)
+
+declare <vscale x 8 x half> @llvm.aarch64.sve.fcmla.lane.nxv8f16(<vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, i32, i32)
+declare <vscale x 4 x float> @llvm.aarch64.sve.fcmla.lane.nxv4f32(<vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, i32, i32)
+
+declare <vscale x 8 x half> @llvm.aarch64.sve.fdiv.nxv8f16(<vscale x 8 x i1>, <vscale x 8 x half>, <vscale x 8 x half>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.fdiv.nxv4f32(<vscale x 4 x i1>, <vscale x 4 x float>, <vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.fdiv.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>, <vscale x 2 x double>)
+
+declare <vscale x 8 x half> @llvm.aarch64.sve.fdivr.nxv8f16(<vscale x 8 x i1>, <vscale x 8 x half>, <vscale x 8 x half>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.fdivr.nxv4f32(<vscale x 4 x i1>, <vscale x 4 x float>, <vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.fdivr.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>, <vscale x 2 x double>)
+
+declare <vscale x 8 x half> @llvm.aarch64.sve.fexpa.x.nxv8f16(<vscale x 8 x i16>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.fexpa.x.nxv4f32(<vscale x 4 x i32>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.fexpa.x.nxv2f64(<vscale x 2 x i64>)
+
+declare <vscale x 8 x half> @llvm.aarch64.sve.fmad.nxv8f16(<vscale x 8 x i1>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.fmad.nxv4f32(<vscale x 4 x i1>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.fmad.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>)
+
+declare <vscale x 8 x half> @llvm.aarch64.sve.fmax.nxv8f16(<vscale x 8 x i1>, <vscale x 8 x half>, <vscale x 8 x half>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.fmax.nxv4f32(<vscale x 4 x i1>, <vscale x 4 x float>, <vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.fmax.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>, <vscale x 2 x double>)
+
+declare <vscale x 8 x half> @llvm.aarch64.sve.fmaxnm.nxv8f16(<vscale x 8 x i1>, <vscale x 8 x half>, <vscale x 8 x half>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.fmaxnm.nxv4f32(<vscale x 4 x i1>, <vscale x 4 x float>, <vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.fmaxnm.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>, <vscale x 2 x double>)
+
+declare <vscale x 8 x half> @llvm.aarch64.sve.fmin.nxv8f16(<vscale x 8 x i1>, <vscale x 8 x half>, <vscale x 8 x half>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.fmin.nxv4f32(<vscale x 4 x i1>, <vscale x 4 x float>, <vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.fmin.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>, <vscale x 2 x double>)
+
+declare <vscale x 8 x half> @llvm.aarch64.sve.fminnm.nxv8f16(<vscale x 8 x i1>, <vscale x 8 x half>, <vscale x 8 x half>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.fminnm.nxv4f32(<vscale x 4 x i1>, <vscale x 4 x float>, <vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.fminnm.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>, <vscale x 2 x double>)
+
+declare <vscale x 8 x half> @llvm.aarch64.sve.fmla.nxv8f16(<vscale x 8 x i1>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.fmla.nxv4f32(<vscale x 4 x i1>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.fmla.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>)
+
+declare <vscale x 8 x half> @llvm.aarch64.sve.fmla.lane.nxv8f16(<vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, i32)
+declare <vscale x 4 x float> @llvm.aarch64.sve.fmla.lane.nxv4f32(<vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, i32)
+declare <vscale x 2 x double> @llvm.aarch64.sve.fmla.lane.nxv2f64(<vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, i32)
+
+declare <vscale x 8 x half> @llvm.aarch64.sve.fmls.nxv8f16(<vscale x 8 x i1>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.fmls.nxv4f32(<vscale x 4 x i1>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.fmls.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>)
+
+declare <vscale x 8 x half> @llvm.aarch64.sve.fmls.lane.nxv8f16(<vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, i32)
+declare <vscale x 4 x float> @llvm.aarch64.sve.fmls.lane.nxv4f32(<vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, i32)
+declare <vscale x 2 x double> @llvm.aarch64.sve.fmls.lane.nxv2f64(<vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, i32)
+
+declare <vscale x 8 x half> @llvm.aarch64.sve.fmsb.nxv8f16(<vscale x 8 x i1>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.fmsb.nxv4f32(<vscale x 4 x i1>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.fmsb.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>)
+
+declare <vscale x 8 x half> @llvm.aarch64.sve.fmul.nxv8f16(<vscale x 8 x i1>, <vscale x 8 x half>, <vscale x 8 x half>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.fmul.nxv4f32(<vscale x 4 x i1>, <vscale x 4 x float>, <vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.fmul.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>, <vscale x 2 x double>)
+
+declare <vscale x 8 x half> @llvm.aarch64.sve.fmul.lane.nxv8f16(<vscale x 8 x half>, <vscale x 8 x half>, i32)
+declare <vscale x 4 x float> @llvm.aarch64.sve.fmul.lane.nxv4f32(<vscale x 4 x float>, <vscale x 4 x float>, i32)
+declare <vscale x 2 x double> @llvm.aarch64.sve.fmul.lane.nxv2f64(<vscale x 2 x double>, <vscale x 2 x double>, i32)
+
+declare <vscale x 8 x half> @llvm.aarch64.sve.fmulx.nxv8f16(<vscale x 8 x i1>, <vscale x 8 x half>, <vscale x 8 x half>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.fmulx.nxv4f32(<vscale x 4 x i1>, <vscale x 4 x float>, <vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.fmulx.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>, <vscale x 2 x double>)
+
+declare <vscale x 8 x half> @llvm.aarch64.sve.fneg.nxv8f16(<vscale x 8 x half>, <vscale x 8 x i1>, <vscale x 8 x half>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.fneg.nxv4f32(<vscale x 4 x float>, <vscale x 4 x i1>, <vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.fneg.nxv2f64(<vscale x 2 x double>, <vscale x 2 x i1>, <vscale x 2 x double>)
+
+declare <vscale x 8 x half> @llvm.aarch64.sve.fnmad.nxv8f16(<vscale x 8 x i1>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.fnmad.nxv4f32(<vscale x 4 x i1>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.fnmad.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>)
+
+declare <vscale x 8 x half> @llvm.aarch64.sve.fnmla.nxv8f16(<vscale x 8 x i1>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.fnmla.nxv4f32(<vscale x 4 x i1>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.fnmla.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>)
+
+declare <vscale x 8 x half> @llvm.aarch64.sve.fnmls.nxv8f16(<vscale x 8 x i1>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.fnmls.nxv4f32(<vscale x 4 x i1>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.fnmls.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>)
+
+declare <vscale x 8 x half> @llvm.aarch64.sve.fnmsb.nxv8f16(<vscale x 8 x i1>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.fnmsb.nxv4f32(<vscale x 4 x i1>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.fnmsb.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>)
+
+declare <vscale x 8 x half> @llvm.aarch64.sve.frecpe.x.nxv8f16(<vscale x 8 x half>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.frecpe.x.nxv4f32(<vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.frecpe.x.nxv2f64(<vscale x 2 x double>)
+
+declare <vscale x 8 x half> @llvm.aarch64.sve.frecpx.nxv8f16(<vscale x 8 x half>, <vscale x 8 x i1>, <vscale x 8 x half>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.frecpx.nxv4f32(<vscale x 4 x float>, <vscale x 4 x i1>, <vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.frecpx.nxv2f64(<vscale x 2 x double>, <vscale x 2 x i1>, <vscale x 2 x double>)
+
+declare <vscale x 8 x half> @llvm.aarch64.sve.frinta.nxv8f16(<vscale x 8 x half>, <vscale x 8 x i1>, <vscale x 8 x half>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.frinta.nxv4f32(<vscale x 4 x float>, <vscale x 4 x i1>, <vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.frinta.nxv2f64(<vscale x 2 x double>, <vscale x 2 x i1>, <vscale x 2 x double>)
+
+declare <vscale x 8 x half> @llvm.aarch64.sve.frinti.nxv8f16(<vscale x 8 x half>, <vscale x 8 x i1>, <vscale x 8 x half>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.frinti.nxv4f32(<vscale x 4 x float>, <vscale x 4 x i1>, <vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.frinti.nxv2f64(<vscale x 2 x double>, <vscale x 2 x i1>, <vscale x 2 x double>)
+
+declare <vscale x 8 x half> @llvm.aarch64.sve.frintm.nxv8f16(<vscale x 8 x half>, <vscale x 8 x i1>, <vscale x 8 x half>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.frintm.nxv4f32(<vscale x 4 x float>, <vscale x 4 x i1>, <vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.frintm.nxv2f64(<vscale x 2 x double>, <vscale x 2 x i1>, <vscale x 2 x double>)
+
+declare <vscale x 8 x half> @llvm.aarch64.sve.frintn.nxv8f16(<vscale x 8 x half>, <vscale x 8 x i1>, <vscale x 8 x half>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.frintn.nxv4f32(<vscale x 4 x float>, <vscale x 4 x i1>, <vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.frintn.nxv2f64(<vscale x 2 x double>, <vscale x 2 x i1>, <vscale x 2 x double>)
+
+declare <vscale x 8 x half> @llvm.aarch64.sve.frintp.nxv8f16(<vscale x 8 x half>, <vscale x 8 x i1>, <vscale x 8 x half>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.frintp.nxv4f32(<vscale x 4 x float>, <vscale x 4 x i1>, <vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.frintp.nxv2f64(<vscale x 2 x double>, <vscale x 2 x i1>, <vscale x 2 x double>)
+
+declare <vscale x 8 x half> @llvm.aarch64.sve.frintx.nxv8f16(<vscale x 8 x half>, <vscale x 8 x i1>, <vscale x 8 x half>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.frintx.nxv4f32(<vscale x 4 x float>, <vscale x 4 x i1>, <vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.frintx.nxv2f64(<vscale x 2 x double>, <vscale x 2 x i1>, <vscale x 2 x double>)
+
+declare <vscale x 8 x half> @llvm.aarch64.sve.frintz.nxv8f16(<vscale x 8 x half>, <vscale x 8 x i1>, <vscale x 8 x half>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.frintz.nxv4f32(<vscale x 4 x float>, <vscale x 4 x i1>, <vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.frintz.nxv2f64(<vscale x 2 x double>, <vscale x 2 x i1>, <vscale x 2 x double>)
+
+declare <vscale x 8 x half> @llvm.aarch64.sve.frsqrte.x.nxv8f16(<vscale x 8 x half>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.frsqrte.x.nxv4f32(<vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.frsqrte.x.nxv2f64(<vscale x 2 x double>)
+
+declare <vscale x 8 x half> @llvm.aarch64.sve.fscale.nxv8f16(<vscale x 8 x i1>, <vscale x 8 x half>, <vscale x 8 x i16>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.fscale.nxv4f32(<vscale x 4 x i1>, <vscale x 4 x float>, <vscale x 4 x i32>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.fscale.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>, <vscale x 2 x i64>)
+
+declare <vscale x 8 x half> @llvm.aarch64.sve.fsqrt.nxv8f16(<vscale x 8 x half>, <vscale x 8 x i1>, <vscale x 8 x half>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.fsqrt.nxv4f32(<vscale x 4 x float>, <vscale x 4 x i1>, <vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.fsqrt.nxv2f64(<vscale x 2 x double>, <vscale x 2 x i1>, <vscale x 2 x double>)
+
+declare <vscale x 8 x half> @llvm.aarch64.sve.fsub.nxv8f16(<vscale x 8 x i1>, <vscale x 8 x half>, <vscale x 8 x half>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.fsub.nxv4f32(<vscale x 4 x i1>, <vscale x 4 x float>, <vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.fsub.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>, <vscale x 2 x double>)
+
+declare <vscale x 8 x half> @llvm.aarch64.sve.fsubr.nxv8f16(<vscale x 8 x i1>, <vscale x 8 x half>, <vscale x 8 x half>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.fsubr.nxv4f32(<vscale x 4 x i1>, <vscale x 4 x float>, <vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.fsubr.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>, <vscale x 2 x double>)
+
+declare <vscale x 8 x half> @llvm.aarch64.sve.ftmad.x.nxv8f16(<vscale x 8 x half>, <vscale x 8 x half>, i32)
+declare <vscale x 4 x float> @llvm.aarch64.sve.ftmad.x.nxv4f32(<vscale x 4 x float>, <vscale x 4 x float>, i32)
+declare <vscale x 2 x double> @llvm.aarch64.sve.ftmad.x.nxv2f64(<vscale x 2 x double>, <vscale x 2 x double>, i32)
+
+declare <vscale x 8 x half> @llvm.aarch64.sve.ftsmul.x.nxv8f16(<vscale x 8 x half>, <vscale x 8 x i16>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.ftsmul.x.nxv4f32(<vscale x 4 x float>, <vscale x 4 x i32>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.ftsmul.x.nxv2f64(<vscale x 2 x double>, <vscale x 2 x i64>)
+
+declare <vscale x 8 x half> @llvm.aarch64.sve.ftssel.x.nxv8f16(<vscale x 8 x half>, <vscale x 8 x i16>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.ftssel.x.nxv4f32(<vscale x 4 x float>, <vscale x 4 x i32>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.ftssel.x.nxv2f64(<vscale x 2 x double>, <vscale x 2 x i64>)
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-fp-compares.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-fp-compares.ll
new file mode 100644
index 000000000..3200f1468
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-fp-compares.ll
@@ -0,0 +1,267 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+;
+; FACGE
+;
+
+define <vscale x 8 x i1> @facge_h(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: facge_h:
+; CHECK: facge p0.h, p0/z, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.facge.nxv8f16(<vscale x 8 x i1> %pg,
+                                                                <vscale x 8 x half> %a,
+                                                                <vscale x 8 x half> %b)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @facge_s(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: facge_s:
+; CHECK: facge p0.s, p0/z, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.facge.nxv4f32(<vscale x 4 x i1> %pg,
+                                                                <vscale x 4 x float> %a,
+                                                                <vscale x 4 x float> %b)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 2 x i1> @facge_d(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: facge_d:
+; CHECK: facge p0.d, p0/z, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.facge.nxv2f64(<vscale x 2 x i1> %pg,
+                                                                <vscale x 2 x double> %a,
+                                                                <vscale x 2 x double> %b)
+  ret <vscale x 2 x i1> %out
+}
+
+;
+; FACGT
+;
+
+define <vscale x 8 x i1> @facgt_h(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: facgt_h:
+; CHECK: facgt p0.h, p0/z, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.facgt.nxv8f16(<vscale x 8 x i1> %pg,
+                                                                <vscale x 8 x half> %a,
+                                                                <vscale x 8 x half> %b)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @facgt_s(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: facgt_s:
+; CHECK: facgt p0.s, p0/z, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.facgt.nxv4f32(<vscale x 4 x i1> %pg,
+                                                                <vscale x 4 x float> %a,
+                                                                <vscale x 4 x float> %b)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 2 x i1> @facgt_d(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: facgt_d:
+; CHECK: facgt p0.d, p0/z, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.facgt.nxv2f64(<vscale x 2 x i1> %pg,
+                                                                <vscale x 2 x double> %a,
+                                                                <vscale x 2 x double> %b)
+  ret <vscale x 2 x i1> %out
+}
+
+;
+; FCMEQ
+;
+
+define <vscale x 8 x i1> @fcmeq_h(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: fcmeq_h:
+; CHECK: fcmeq p0.h, p0/z, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.fcmpeq.nxv8f16(<vscale x 8 x i1> %pg,
+                                                                 <vscale x 8 x half> %a,
+                                                                 <vscale x 8 x half> %b)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @fcmeq_s(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: fcmeq_s:
+; CHECK: fcmeq p0.s, p0/z, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.fcmpeq.nxv4f32(<vscale x 4 x i1> %pg,
+                                                                 <vscale x 4 x float> %a,
+                                                                 <vscale x 4 x float> %b)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 2 x i1> @fcmeq_d(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: fcmeq_d:
+; CHECK: fcmeq p0.d, p0/z, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.fcmpeq.nxv2f64(<vscale x 2 x i1> %pg,
+                                                                 <vscale x 2 x double> %a,
+                                                                 <vscale x 2 x double> %b)
+  ret <vscale x 2 x i1> %out
+}
+
+;
+; FCMGE
+;
+
+define <vscale x 8 x i1> @fcmge_h(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: fcmge_h:
+; CHECK: fcmge p0.h, p0/z, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.fcmpge.nxv8f16(<vscale x 8 x i1> %pg,
+                                                                 <vscale x 8 x half> %a,
+                                                                 <vscale x 8 x half> %b)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @fcmge_s(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: fcmge_s:
+; CHECK: fcmge p0.s, p0/z, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.fcmpge.nxv4f32(<vscale x 4 x i1> %pg,
+                                                                 <vscale x 4 x float> %a,
+                                                                 <vscale x 4 x float> %b)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 2 x i1> @fcmge_d(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: fcmge_d:
+; CHECK: fcmge p0.d, p0/z, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.fcmpge.nxv2f64(<vscale x 2 x i1> %pg,
+                                                                 <vscale x 2 x double> %a,
+                                                                 <vscale x 2 x double> %b)
+  ret <vscale x 2 x i1> %out
+}
+
+;
+; FCMGT
+;
+
+define <vscale x 8 x i1> @fcmgt_h(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: fcmgt_h:
+; CHECK: fcmgt p0.h, p0/z, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.fcmpgt.nxv8f16(<vscale x 8 x i1> %pg,
+                                                                 <vscale x 8 x half> %a,
+                                                                 <vscale x 8 x half> %b)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @fcmgt_s(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: fcmgt_s:
+; CHECK: fcmgt p0.s, p0/z, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.fcmpgt.nxv4f32(<vscale x 4 x i1> %pg,
+                                                                 <vscale x 4 x float> %a,
+                                                                 <vscale x 4 x float> %b)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 2 x i1> @fcmgt_d(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: fcmgt_d:
+; CHECK: fcmgt p0.d, p0/z, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.fcmpgt.nxv2f64(<vscale x 2 x i1> %pg,
+                                                                 <vscale x 2 x double> %a,
+                                                                 <vscale x 2 x double> %b)
+  ret <vscale x 2 x i1> %out
+}
+
+;
+; FCMNE
+;
+
+define <vscale x 8 x i1> @fcmne_h(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: fcmne_h:
+; CHECK: fcmne p0.h, p0/z, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.fcmpne.nxv8f16(<vscale x 8 x i1> %pg,
+                                                                 <vscale x 8 x half> %a,
+                                                                 <vscale x 8 x half> %b)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @fcmne_s(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: fcmne_s:
+; CHECK: fcmne p0.s, p0/z, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.fcmpne.nxv4f32(<vscale x 4 x i1> %pg,
+                                                                 <vscale x 4 x float> %a,
+                                                                 <vscale x 4 x float> %b)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 2 x i1> @fcmne_d(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: fcmne_d:
+; CHECK: fcmne p0.d, p0/z, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.fcmpne.nxv2f64(<vscale x 2 x i1> %pg,
+                                                                 <vscale x 2 x double> %a,
+                                                                 <vscale x 2 x double> %b)
+  ret <vscale x 2 x i1> %out
+}
+
+;
+; FCMPUO
+;
+
+define <vscale x 8 x i1> @fcmuo_h(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: fcmuo_h:
+; CHECK: fcmuo p0.h, p0/z, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.fcmpuo.nxv8f16(<vscale x 8 x i1> %pg,
+                                                                 <vscale x 8 x half> %a,
+                                                                 <vscale x 8 x half> %b)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @fcmuo_s(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: fcmuo_s:
+; CHECK: fcmuo p0.s, p0/z, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.fcmpuo.nxv4f32(<vscale x 4 x i1> %pg,
+                                                                 <vscale x 4 x float> %a,
+                                                                 <vscale x 4 x float> %b)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 2 x i1> @fcmuo_d(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: fcmuo_d:
+; CHECK: fcmuo p0.d, p0/z, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.fcmpuo.nxv2f64(<vscale x 2 x i1> %pg,
+                                                                 <vscale x 2 x double> %a,
+                                                                 <vscale x 2 x double> %b)
+  ret <vscale x 2 x i1> %out
+}
+
+declare <vscale x 8 x i1> @llvm.aarch64.sve.facge.nxv8f16(<vscale x 8 x i1>, <vscale x 8 x half>, <vscale x 8 x half>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.facge.nxv4f32(<vscale x 4 x i1>, <vscale x 4 x float>, <vscale x 4 x float>)
+declare <vscale x 2 x i1> @llvm.aarch64.sve.facge.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>, <vscale x 2 x double>)
+
+declare <vscale x 8 x i1> @llvm.aarch64.sve.facgt.nxv8f16(<vscale x 8 x i1>, <vscale x 8 x half>, <vscale x 8 x half>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.facgt.nxv4f32(<vscale x 4 x i1>, <vscale x 4 x float>, <vscale x 4 x float>)
+declare <vscale x 2 x i1> @llvm.aarch64.sve.facgt.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>, <vscale x 2 x double>)
+
+declare <vscale x 8 x i1> @llvm.aarch64.sve.fcmpeq.nxv8f16(<vscale x 8 x i1>, <vscale x 8 x half>, <vscale x 8 x half>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.fcmpeq.nxv4f32(<vscale x 4 x i1>, <vscale x 4 x float>, <vscale x 4 x float>)
+declare <vscale x 2 x i1> @llvm.aarch64.sve.fcmpeq.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>, <vscale x 2 x double>)
+
+declare <vscale x 8 x i1> @llvm.aarch64.sve.fcmpge.nxv8f16(<vscale x 8 x i1>, <vscale x 8 x half>, <vscale x 8 x half>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.fcmpge.nxv4f32(<vscale x 4 x i1>, <vscale x 4 x float>, <vscale x 4 x float>)
+declare <vscale x 2 x i1> @llvm.aarch64.sve.fcmpge.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>, <vscale x 2 x double>)
+
+declare <vscale x 8 x i1> @llvm.aarch64.sve.fcmpgt.nxv8f16(<vscale x 8 x i1>, <vscale x 8 x half>, <vscale x 8 x half>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.fcmpgt.nxv4f32(<vscale x 4 x i1>, <vscale x 4 x float>, <vscale x 4 x float>)
+declare <vscale x 2 x i1> @llvm.aarch64.sve.fcmpgt.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>, <vscale x 2 x double>)
+
+declare <vscale x 8 x i1> @llvm.aarch64.sve.fcmpne.nxv8f16(<vscale x 8 x i1>, <vscale x 8 x half>, <vscale x 8 x half>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.fcmpne.nxv4f32(<vscale x 4 x i1>, <vscale x 4 x float>, <vscale x 4 x float>)
+declare <vscale x 2 x i1> @llvm.aarch64.sve.fcmpne.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>, <vscale x 2 x double>)
+
+declare <vscale x 8 x i1> @llvm.aarch64.sve.fcmpuo.nxv8f16(<vscale x 8 x i1>, <vscale x 8 x half>, <vscale x 8 x half>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.fcmpuo.nxv4f32(<vscale x 4 x i1>, <vscale x 4 x float>, <vscale x 4 x float>)
+declare <vscale x 2 x i1> @llvm.aarch64.sve.fcmpuo.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>, <vscale x 2 x double>)
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-fp-converts.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-fp-converts.ll
new file mode 100644
index 000000000..e777a2f3b
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-fp-converts.ll
@@ -0,0 +1,400 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+;
+; FCVT
+;
+
+define <vscale x 8 x half> @fcvt_f16_f32(<vscale x 8 x half> %a, <vscale x 16 x i1> %pg, <vscale x 4 x float> %b) {
+; CHECK-LABEL: fcvt_f16_f32:
+; CHECK: fcvt z0.h, p0/m, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.fcvt.f16f32(<vscale x 8 x half> %a,
+                                                                <vscale x 16 x i1> %pg,
+                                                                <vscale x 4 x float> %b)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 8 x half> @fcvt_f16_f64(<vscale x 8 x half> %a, <vscale x 16 x i1> %pg, <vscale x 2 x double> %b) {
+; CHECK-LABEL: fcvt_f16_f64:
+; CHECK: fcvt z0.h, p0/m, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.fcvt.f16f64(<vscale x 8 x half> %a,
+                                                                <vscale x 16 x i1> %pg,
+                                                                <vscale x 2 x double> %b)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x float> @fcvt_f32_f16(<vscale x 4 x float> %a, <vscale x 16 x i1> %pg, <vscale x 8 x half> %b) {
+; CHECK-LABEL: fcvt_f32_f16:
+; CHECK: fcvt z0.s, p0/m, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.fcvt.f32f16(<vscale x 4 x float> %a,
+                                                                 <vscale x 16 x i1> %pg,
+                                                                 <vscale x 8 x half> %b)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 4 x float> @fcvt_f32_f64(<vscale x 4 x float> %a, <vscale x 16 x i1> %pg, <vscale x 2 x double> %b) {
+; CHECK-LABEL: fcvt_f32_f64:
+; CHECK: fcvt z0.s, p0/m, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.fcvt.f32f64(<vscale x 4 x float> %a,
+                                                                 <vscale x 16 x i1> %pg,
+                                                                 <vscale x 2 x double> %b)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @fcvt_f64_f16(<vscale x 2 x double> %a, <vscale x 16 x i1> %pg, <vscale x 8 x half> %b) {
+; CHECK-LABEL: fcvt_f64_f16:
+; CHECK: fcvt z0.d, p0/m, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.fcvt.f64f16(<vscale x 2 x double> %a,
+                                                                  <vscale x 16 x i1> %pg,
+                                                                  <vscale x 8 x half> %b)
+  ret <vscale x 2 x double> %out
+}
+
+define <vscale x 2 x double> @fcvt_f64_f32(<vscale x 2 x double> %a, <vscale x 16 x i1> %pg, <vscale x 4 x float> %b) {
+; CHECK-LABEL: fcvt_f64_f32:
+; CHECK: fcvt z0.d, p0/m, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.fcvt.f64f32(<vscale x 2 x double> %a,
+                                                                  <vscale x 16 x i1> %pg,
+                                                                  <vscale x 4 x float> %b)
+  ret <vscale x 2 x double> %out
+}
+
+;
+; FCVTZS
+;
+
+define <vscale x 8 x i16> @fcvtzs_i16_f16(<vscale x 8 x i16> %a, <vscale x 8 x i1> %pg, <vscale x 8 x half> %b) {
+; CHECK-LABEL: fcvtzs_i16_f16:
+; CHECK: fcvtzs z0.h, p0/m, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.fcvtzs.nxv8i16.nxv8f16(<vscale x 8 x i16> %a,
+                                                                          <vscale x 8 x i1> %pg,
+                                                                          <vscale x 8 x half> %b)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @fcvtzs_i32_f32(<vscale x 4 x i32> %a, <vscale x 4 x i1> %pg, <vscale x 4 x float> %b) {
+; CHECK-LABEL: fcvtzs_i32_f32:
+; CHECK: fcvtzs z0.s, p0/m, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.fcvtzs.nxv4i32.nxv4f32(<vscale x 4 x i32> %a,
+                                                                          <vscale x 4 x i1> %pg,
+                                                                          <vscale x 4 x float> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @fcvtzs_i64_f64(<vscale x 2 x i64> %a, <vscale x 2 x i1> %pg, <vscale x 2 x double> %b) {
+; CHECK-LABEL: fcvtzs_i64_f64:
+; CHECK: fcvtzs z0.d, p0/m, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.fcvtzs.nxv2i64.nxv2f64(<vscale x 2 x i64> %a,
+                                                                          <vscale x 2 x i1> %pg,
+                                                                          <vscale x 2 x double> %b)
+  ret <vscale x 2 x i64> %out
+}
+
+define <vscale x 4 x i32> @fcvtzs_i32_f16(<vscale x 4 x i32> %a, <vscale x 16 x i1> %pg, <vscale x 8 x half> %b) {
+; CHECK-LABEL: fcvtzs_i32_f16:
+; CHECK: fcvtzs z0.s, p0/m, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.fcvtzs.i32f16(<vscale x 4 x i32> %a,
+                                                                 <vscale x 16 x i1> %pg,
+                                                                 <vscale x 8 x half> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 4 x i32> @fcvtzs_i32_f64(<vscale x 4 x i32> %a, <vscale x 16 x i1> %pg, <vscale x 2 x double> %b) {
+; CHECK-LABEL: fcvtzs_i32_f64:
+; CHECK: fcvtzs z0.s, p0/m, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.fcvtzs.i32f64(<vscale x 4 x i32> %a,
+                                                                 <vscale x 16 x i1> %pg,
+                                                                 <vscale x 2 x double> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @fcvtzs_i64_f16(<vscale x 2 x i64> %a, <vscale x 16 x i1> %pg, <vscale x 8 x half> %b) {
+; CHECK-LABEL: fcvtzs_i64_f16:
+; CHECK: fcvtzs z0.d, p0/m, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.fcvtzs.i64f16(<vscale x 2 x i64> %a,
+                                                                 <vscale x 16 x i1> %pg,
+                                                                 <vscale x 8 x half> %b)
+  ret <vscale x 2 x i64> %out
+}
+
+define <vscale x 2 x i64> @fcvtzs_i64_f32(<vscale x 2 x i64> %a, <vscale x 16 x i1> %pg, <vscale x 4 x float> %b) {
+; CHECK-LABEL: fcvtzs_i64_f32:
+; CHECK: fcvtzs z0.d, p0/m, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.fcvtzs.i64f32(<vscale x 2 x i64> %a,
+                                                                 <vscale x 16 x i1> %pg,
+                                                                 <vscale x 4 x float> %b)
+  ret <vscale x 2 x i64> %out
+}
+
+;
+; FCVTZU
+;
+
+define <vscale x 8 x i16> @fcvtzu_i16_f16(<vscale x 8 x i16> %a, <vscale x 8 x i1> %pg, <vscale x 8 x half> %b) {
+; CHECK-LABEL: fcvtzu_i16_f16:
+; CHECK: fcvtzu z0.h, p0/m, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.fcvtzu.nxv8i16.nxv8f16(<vscale x 8 x i16> %a,
+                                                                          <vscale x 8 x i1> %pg,
+                                                                          <vscale x 8 x half> %b)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @fcvtzu_i32_f32(<vscale x 4 x i32> %a, <vscale x 4 x i1> %pg, <vscale x 4 x float> %b) {
+; CHECK-LABEL: fcvtzu_i32_f32:
+; CHECK: fcvtzu z0.s, p0/m, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.fcvtzu.nxv4i32.nxv4f32(<vscale x 4 x i32> %a,
+                                                                          <vscale x 4 x i1> %pg,
+                                                                          <vscale x 4 x float> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @fcvtzu_i64_f64(<vscale x 2 x i64> %a, <vscale x 2 x i1> %pg, <vscale x 2 x double> %b) {
+; CHECK-LABEL: fcvtzu_i64_f64:
+; CHECK: fcvtzu z0.d, p0/m, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.fcvtzu.nxv2i64.nxv2f64(<vscale x 2 x i64> %a,
+                                                                          <vscale x 2 x i1> %pg,
+                                                                          <vscale x 2 x double> %b)
+  ret <vscale x 2 x i64> %out
+}
+
+define <vscale x 4 x i32> @fcvtzu_i32_f16(<vscale x 4 x i32> %a, <vscale x 16 x i1> %pg, <vscale x 8 x half> %b) {
+; CHECK-LABEL: fcvtzu_i32_f16:
+; CHECK: fcvtzu z0.s, p0/m, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.fcvtzu.i32f16(<vscale x 4 x i32> %a,
+                                                                 <vscale x 16 x i1> %pg,
+                                                                 <vscale x 8 x half> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 4 x i32> @fcvtzu_i32_f64(<vscale x 4 x i32> %a, <vscale x 16 x i1> %pg, <vscale x 2 x double> %b) {
+; CHECK-LABEL: fcvtzu_i32_f64:
+; CHECK: fcvtzu z0.s, p0/m, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.fcvtzu.i32f64(<vscale x 4 x i32> %a,
+                                                                 <vscale x 16 x i1> %pg,
+                                                                 <vscale x 2 x double> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @fcvtzu_i64_f16(<vscale x 2 x i64> %a, <vscale x 16 x i1> %pg, <vscale x 8 x half> %b) {
+; CHECK-LABEL: fcvtzu_i64_f16:
+; CHECK: fcvtzu z0.d, p0/m, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.fcvtzu.i64f16(<vscale x 2 x i64> %a,
+                                                                 <vscale x 16 x i1> %pg,
+                                                                 <vscale x 8 x half> %b)
+  ret <vscale x 2 x i64> %out
+}
+
+define <vscale x 2 x i64> @fcvtzu_i64_f32(<vscale x 2 x i64> %a, <vscale x 16 x i1> %pg, <vscale x 4 x float> %b) {
+; CHECK-LABEL: fcvtzu_i64_f32:
+; CHECK: fcvtzu z0.d, p0/m, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.fcvtzu.i64f32(<vscale x 2 x i64> %a,
+                                                                 <vscale x 16 x i1> %pg,
+                                                                 <vscale x 4 x float> %b)
+  ret <vscale x 2 x i64> %out
+}
+
+;
+; SCVTF
+;
+
+define <vscale x 8 x half> @scvtf_f16_i16(<vscale x 8 x half> %a, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: scvtf_f16_i16:
+; CHECK: scvtf z0.h, p0/m, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.scvtf.nxv8f16.nxv8i16(<vscale x 8 x half> %a,
+                                                                          <vscale x 8 x i1> %pg,
+                                                                          <vscale x 8 x i16> %b)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x float> @scvtf_f32_i32(<vscale x 4 x float> %a, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: scvtf_f32_i32:
+; CHECK: scvtf z0.s, p0/m, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.scvtf.nxv4f32.nxv4i32(<vscale x 4 x float> %a,
+                                                                           <vscale x 4 x i1> %pg,
+                                                                           <vscale x 4 x i32> %b)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @scvtf_f64_i64(<vscale x 2 x double> %a, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: scvtf_f64_i64:
+; CHECK: scvtf z0.d, p0/m, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.scvtf.nxv2f64.nxv2i64(<vscale x 2 x double> %a,
+                                                                            <vscale x 2 x i1> %pg,
+                                                                            <vscale x 2 x i64> %b)
+  ret <vscale x 2 x double> %out
+}
+
+define <vscale x 8 x half> @scvtf_f16_i32(<vscale x 8 x half> %a, <vscale x 16 x i1> %pg, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: scvtf_f16_i32:
+; CHECK: scvtf z0.h, p0/m, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.scvtf.f16i32(<vscale x 8 x half> %a,
+                                                                 <vscale x 16 x i1> %pg,
+                                                                 <vscale x 4 x i32> %b)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 8 x half> @scvtf_f16_i64(<vscale x 8 x half> %a, <vscale x 16 x i1> %pg, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: scvtf_f16_i64:
+; CHECK: scvtf z0.h, p0/m, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.scvtf.f16i64(<vscale x 8 x half> %a,
+                                                                 <vscale x 16 x i1> %pg,
+                                                                 <vscale x 2 x i64> %b)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x float> @scvtf_f32_i64(<vscale x 4 x float> %a, <vscale x 16 x i1> %pg, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: scvtf_f32_i64:
+; CHECK: scvtf z0.s, p0/m, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.scvtf.f32i64(<vscale x 4 x float> %a,
+                                                                  <vscale x 16 x i1> %pg,
+                                                                  <vscale x 2 x i64> %b)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @scvtf_f64_i32(<vscale x 2 x double> %a, <vscale x 16 x i1> %pg, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: scvtf_f64_i32:
+; CHECK: scvtf z0.d, p0/m, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.scvtf.f64i32(<vscale x 2 x double> %a,
+                                                                   <vscale x 16 x i1> %pg,
+                                                                   <vscale x 4 x i32> %b)
+  ret <vscale x 2 x double> %out
+}
+
+;
+; UCVTF
+;
+
+define <vscale x 8 x half> @ucvtf_f16_i16(<vscale x 8 x half> %a, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: ucvtf_f16_i16:
+; CHECK: ucvtf z0.h, p0/m, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.ucvtf.nxv8f16.nxv8i16(<vscale x 8 x half> %a,
+                                                                          <vscale x 8 x i1> %pg,
+                                                                          <vscale x 8 x i16> %b)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x float> @ucvtf_f32_i32(<vscale x 4 x float> %a, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: ucvtf_f32_i32:
+; CHECK: ucvtf z0.s, p0/m, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.ucvtf.nxv4f32.nxv4i32(<vscale x 4 x float> %a,
+                                                                           <vscale x 4 x i1> %pg,
+                                                                           <vscale x 4 x i32> %b)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @ucvtf_f64_i64(<vscale x 2 x double> %a, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: ucvtf_f64_i64:
+; CHECK: ucvtf z0.d, p0/m, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.ucvtf.nxv2f64.nxv2i64(<vscale x 2 x double> %a,
+                                                                            <vscale x 2 x i1> %pg,
+                                                                            <vscale x 2 x i64> %b)
+  ret <vscale x 2 x double> %out
+}
+
+define <vscale x 8 x half> @ucvtf_f16_i32(<vscale x 8 x half> %a, <vscale x 16 x i1> %pg, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: ucvtf_f16_i32:
+; CHECK: ucvtf z0.h, p0/m, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.ucvtf.f16i32(<vscale x 8 x half> %a,
+                                                                 <vscale x 16 x i1> %pg,
+                                                                 <vscale x 4 x i32> %b)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 8 x half> @ucvtf_f16_i64(<vscale x 8 x half> %a, <vscale x 16 x i1> %pg, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: ucvtf_f16_i64:
+; CHECK: ucvtf z0.h, p0/m, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.ucvtf.f16i64(<vscale x 8 x half> %a,
+                                                                 <vscale x 16 x i1> %pg,
+                                                                 <vscale x 2 x i64> %b)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x float> @ucvtf_f32_i64(<vscale x 4 x float> %a, <vscale x 16 x i1> %pg, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: ucvtf_f32_i64:
+; CHECK: ucvtf z0.s, p0/m, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.ucvtf.f32i64(<vscale x 4 x float> %a,
+                                                                  <vscale x 16 x i1> %pg,
+                                                                  <vscale x 2 x i64> %b)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @ucvtf_f64_i32(<vscale x 2 x double> %a, <vscale x 16 x i1> %pg, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: ucvtf_f64_i32:
+; CHECK: ucvtf z0.d, p0/m, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.ucvtf.f64i32(<vscale x 2 x double> %a,
+                                                                   <vscale x 16 x i1> %pg,
+                                                                   <vscale x 4 x i32> %b)
+  ret <vscale x 2 x double> %out
+}
+
+declare <vscale x 8 x half> @llvm.aarch64.sve.fcvt.f16f32(<vscale x 8 x half>, <vscale x 16 x i1>, <vscale x 4 x float>)
+declare <vscale x 8 x half> @llvm.aarch64.sve.fcvt.f16f64(<vscale x 8 x half>, <vscale x 16 x i1>, <vscale x 2 x double>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.fcvt.f32f16(<vscale x 4 x float>, <vscale x 16 x i1>, <vscale x 8 x half>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.fcvt.f32f64(<vscale x 4 x float>, <vscale x 16 x i1>, <vscale x 2 x double>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.fcvt.f64f16(<vscale x 2 x double>, <vscale x 16 x i1>, <vscale x 8 x half>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.fcvt.f64f32(<vscale x 2 x double>, <vscale x 16 x i1>, <vscale x 4 x float>)
+
+declare <vscale x 8 x i16> @llvm.aarch64.sve.fcvtzs.nxv8i16.nxv8f16(<vscale x 8 x i16>, <vscale x 8 x i1>, <vscale x 8 x half>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.fcvtzs.nxv4i32.nxv4f32(<vscale x 4 x i32>, <vscale x 4 x i1>, <vscale x 4 x float>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.fcvtzs.nxv2i64.nxv2f64(<vscale x 2 x i64>, <vscale x 2 x i1>, <vscale x 2 x double>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.fcvtzs.i32f16(<vscale x 4 x i32>, <vscale x 16 x i1>, <vscale x 8 x half>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.fcvtzs.i32f64(<vscale x 4 x i32>, <vscale x 16 x i1>, <vscale x 2 x double>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.fcvtzs.i64f16(<vscale x 2 x i64>, <vscale x 16 x i1>, <vscale x 8 x half>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.fcvtzs.i64f32(<vscale x 2 x i64>, <vscale x 16 x i1>, <vscale x 4 x float>)
+
+declare <vscale x 8 x i16> @llvm.aarch64.sve.fcvtzu.nxv8i16.nxv8f16(<vscale x 8 x i16>, <vscale x 8 x i1>, <vscale x 8 x half>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.fcvtzu.nxv4i32.nxv4f32(<vscale x 4 x i32>, <vscale x 4 x i1>, <vscale x 4 x float>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.fcvtzu.nxv2i64.nxv2f64(<vscale x 2 x i64>, <vscale x 2 x i1>, <vscale x 2 x double>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.fcvtzu.i32f16(<vscale x 4 x i32>, <vscale x 16 x i1>, <vscale x 8 x half>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.fcvtzu.i32f64(<vscale x 4 x i32>, <vscale x 16 x i1>, <vscale x 2 x double>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.fcvtzu.i64f16(<vscale x 2 x i64>, <vscale x 16 x i1>, <vscale x 8 x half>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.fcvtzu.i64f32(<vscale x 2 x i64>, <vscale x 16 x i1>, <vscale x 4 x float>)
+
+declare <vscale x 8 x half> @llvm.aarch64.sve.scvtf.nxv8f16.nxv8i16(<vscale x 8 x half>, <vscale x 8 x i1>, <vscale x 8 x i16>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.scvtf.nxv4f32.nxv4i32(<vscale x 4 x float>, <vscale x 4 x i1>, <vscale x 4 x i32>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.scvtf.nxv2f64.nxv2i64(<vscale x 2 x double>, <vscale x 2 x i1>, <vscale x 2 x i64>)
+declare <vscale x 8 x half> @llvm.aarch64.sve.scvtf.f16i32(<vscale x 8 x half>, <vscale x 16 x i1>, <vscale x 4 x i32>)
+declare <vscale x 8 x half> @llvm.aarch64.sve.scvtf.f16i64(<vscale x 8 x half>, <vscale x 16 x i1>, <vscale x 2 x i64>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.scvtf.f32i64(<vscale x 4 x float>, <vscale x 16 x i1>, <vscale x 2 x i64>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.scvtf.f64i32(<vscale x 2 x double>, <vscale x 16 x i1>, <vscale x 4 x i32>)
+
+declare <vscale x 8 x half> @llvm.aarch64.sve.ucvtf.nxv8f16.nxv8i16(<vscale x 8 x half>, <vscale x 8 x i1>, <vscale x 8 x i16>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.ucvtf.nxv4f32.nxv4i32(<vscale x 4 x float>, <vscale x 4 x i1>, <vscale x 4 x i32>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.ucvtf.nxv2f64.nxv2i64(<vscale x 2 x double>, <vscale x 2 x i1>, <vscale x 2 x i64>)
+declare <vscale x 8 x half> @llvm.aarch64.sve.ucvtf.f16i32(<vscale x 8 x half>, <vscale x 16 x i1>, <vscale x 4 x i32>)
+declare <vscale x 8 x half> @llvm.aarch64.sve.ucvtf.f16i64(<vscale x 8 x half>, <vscale x 16 x i1>, <vscale x 2 x i64>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.ucvtf.f32i64(<vscale x 4 x float>, <vscale x 16 x i1>, <vscale x 2 x i64>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.ucvtf.f64i32(<vscale x 2 x double>, <vscale x 16 x i1>, <vscale x 4 x i32>)
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-fp-reduce.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-fp-reduce.ll
new file mode 100644
index 000000000..083a7d354
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-fp-reduce.ll
@@ -0,0 +1,214 @@
+; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve < %s | FileCheck %s
+
+;
+; FADDA
+;
+
+define half @fadda_f16(<vscale x 8 x i1> %pg, half %init, <vscale x 8 x half> %a) {
+; CHECK-LABEL: fadda_f16:
+; CHECK: fadda h0, p0, h0, z1.h
+; CHECK-NEXT: ret
+  %res = call half @llvm.aarch64.sve.fadda.nxv8f16(<vscale x 8 x i1> %pg,
+                                                   half %init,
+                                                   <vscale x 8 x half> %a)
+  ret half %res
+}
+
+define float @fadda_f32(<vscale x 4 x i1> %pg, float %init, <vscale x 4 x float> %a) {
+; CHECK-LABEL: fadda_f32:
+; CHECK: fadda s0, p0, s0, z1.s
+; CHECK-NEXT: ret
+  %res = call float @llvm.aarch64.sve.fadda.nxv4f32(<vscale x 4 x i1> %pg,
+                                                    float %init,
+                                                    <vscale x 4 x float> %a)
+  ret float %res
+}
+
+define double @fadda_f64(<vscale x 2 x i1> %pg, double %init, <vscale x 2 x double> %a) {
+; CHECK-LABEL: fadda_f64:
+; CHECK: fadda d0, p0, d0, z1.d
+; CHECK-NEXT: ret
+  %res = call double @llvm.aarch64.sve.fadda.nxv2f64(<vscale x 2 x i1> %pg,
+                                                     double %init,
+                                                     <vscale x 2 x double> %a)
+  ret double %res
+}
+
+;
+; FADDV
+;
+
+define half @faddv_f16(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a) {
+; CHECK-LABEL: faddv_f16:
+; CHECK: faddv h0, p0, z0.h
+; CHECK-NEXT: ret
+  %res = call half @llvm.aarch64.sve.faddv.nxv8f16(<vscale x 8 x i1> %pg,
+                                                   <vscale x 8 x half> %a)
+  ret half %res
+}
+
+define float @faddv_f32(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a) {
+; CHECK-LABEL: faddv_f32:
+; CHECK: faddv s0, p0, z0.s
+; CHECK-NEXT: ret
+  %res = call float @llvm.aarch64.sve.faddv.nxv4f32(<vscale x 4 x i1> %pg,
+                                                    <vscale x 4 x float> %a)
+  ret float %res
+}
+
+define double @faddv_f64(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a) {
+; CHECK-LABEL: faddv_f64:
+; CHECK: faddv d0, p0, z0.d
+; CHECK-NEXT: ret
+  %res = call double @llvm.aarch64.sve.faddv.nxv2f64(<vscale x 2 x i1> %pg,
+                                                     <vscale x 2 x double> %a)
+  ret double %res
+}
+
+;
+; FMAXNMV
+;
+
+define half @fmaxnmv_f16(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a) {
+; CHECK-LABEL: fmaxnmv_f16:
+; CHECK: fmaxnmv h0, p0, z0.h
+; CHECK-NEXT: ret
+  %res = call half @llvm.aarch64.sve.fmaxnmv.nxv8f16(<vscale x 8 x i1> %pg,
+                                                     <vscale x 8 x half> %a)
+  ret half %res
+}
+
+define float @fmaxnmv_f32(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a) {
+; CHECK-LABEL: fmaxnmv_f32:
+; CHECK: fmaxnmv s0, p0, z0.s
+; CHECK-NEXT: ret
+  %res = call float @llvm.aarch64.sve.fmaxnmv.nxv4f32(<vscale x 4 x i1> %pg,
+                                                      <vscale x 4 x float> %a)
+  ret float %res
+}
+
+define double @fmaxnmv_f64(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a) {
+; CHECK-LABEL: fmaxnmv_f64:
+; CHECK: fmaxnmv d0, p0, z0.d
+; CHECK-NEXT: ret
+  %res = call double @llvm.aarch64.sve.fmaxnmv.nxv2f64(<vscale x 2 x i1> %pg,
+                                                       <vscale x 2 x double> %a)
+  ret double %res
+}
+
+;
+; FMAXV
+;
+
+define half @fmaxv_f16(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a) {
+; CHECK-LABEL: fmaxv_f16:
+; CHECK: fmaxv h0, p0, z0.h
+; CHECK-NEXT: ret
+  %res = call half @llvm.aarch64.sve.fmaxv.nxv8f16(<vscale x 8 x i1> %pg,
+                                                   <vscale x 8 x half> %a)
+  ret half %res
+}
+
+define float @fmaxv_f32(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a) {
+; CHECK-LABEL: fmaxv_f32:
+; CHECK: fmaxv s0, p0, z0.s
+; CHECK-NEXT: ret
+  %res = call float @llvm.aarch64.sve.fmaxv.nxv4f32(<vscale x 4 x i1> %pg,
+                                                    <vscale x 4 x float> %a)
+  ret float %res
+}
+
+define double @fmaxv_f64(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a) {
+; CHECK-LABEL: fmaxv_f64:
+; CHECK: fmaxv d0, p0, z0.d
+; CHECK-NEXT: ret
+  %res = call double @llvm.aarch64.sve.fmaxv.nxv2f64(<vscale x 2 x i1> %pg,
+                                                     <vscale x 2 x double> %a)
+  ret double %res
+}
+
+;
+; FMINNMV
+;
+
+define half @fminnmv_f16(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a) {
+; CHECK-LABEL: fminnmv_f16:
+; CHECK: fminnmv h0, p0, z0.h
+; CHECK-NEXT: ret
+  %res = call half @llvm.aarch64.sve.fminnmv.nxv8f16(<vscale x 8 x i1> %pg,
+                                                     <vscale x 8 x half> %a)
+  ret half %res
+}
+
+define float @fminnmv_f32(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a) {
+; CHECK-LABEL: fminnmv_f32:
+; CHECK: fminnmv s0, p0, z0.s
+; CHECK-NEXT: ret
+  %res = call float @llvm.aarch64.sve.fminnmv.nxv4f32(<vscale x 4 x i1> %pg,
+                                                      <vscale x 4 x float> %a)
+  ret float %res
+}
+
+define double @fminnmv_f64(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a) {
+; CHECK-LABEL: fminnmv_f64:
+; CHECK: fminnmv d0, p0, z0.d
+; CHECK-NEXT: ret
+  %res = call double @llvm.aarch64.sve.fminnmv.nxv2f64(<vscale x 2 x i1> %pg,
+                                                       <vscale x 2 x double> %a)
+  ret double %res
+}
+
+;
+; FMINV
+;
+
+define half @fminv_f16(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a) {
+; CHECK-LABEL: fminv_f16:
+; CHECK: fminv h0, p0, z0.h
+; CHECK-NEXT: ret
+  %res = call half @llvm.aarch64.sve.fminv.nxv8f16(<vscale x 8 x i1> %pg,
+                                                   <vscale x 8 x half> %a)
+  ret half %res
+}
+
+define float @fminv_f32(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a) {
+; CHECK-LABEL: fminv_f32:
+; CHECK: fminv s0, p0, z0.s
+; CHECK-NEXT: ret
+  %res = call float @llvm.aarch64.sve.fminv.nxv4f32(<vscale x 4 x i1> %pg,
+                                                    <vscale x 4 x float> %a)
+  ret float %res
+}
+
+define double @fminv_f64(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a) {
+; CHECK-LABEL: fminv_f64:
+; CHECK: fminv d0, p0, z0.d
+; CHECK-NEXT: ret
+  %res = call double @llvm.aarch64.sve.fminv.nxv2f64(<vscale x 2 x i1> %pg,
+                                                     <vscale x 2 x double> %a)
+  ret double %res
+}
+
+declare half @llvm.aarch64.sve.fadda.nxv8f16(<vscale x 8 x i1>, half, <vscale x 8 x half>)
+declare float @llvm.aarch64.sve.fadda.nxv4f32(<vscale x 4 x i1>, float, <vscale x 4 x float>)
+declare double @llvm.aarch64.sve.fadda.nxv2f64(<vscale x 2 x i1>, double, <vscale x 2 x double>)
+
+declare half @llvm.aarch64.sve.faddv.nxv8f16(<vscale x 8 x i1>, <vscale x 8 x half>)
+declare float @llvm.aarch64.sve.faddv.nxv4f32(<vscale x 4 x i1>, <vscale x 4 x float>)
+declare double @llvm.aarch64.sve.faddv.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>)
+
+declare half @llvm.aarch64.sve.fmaxnmv.nxv8f16(<vscale x 8 x i1>, <vscale x 8 x half>)
+declare float @llvm.aarch64.sve.fmaxnmv.nxv4f32(<vscale x 4 x i1>, <vscale x 4 x float>)
+declare double @llvm.aarch64.sve.fmaxnmv.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>)
+
+declare half @llvm.aarch64.sve.fmaxv.nxv8f16(<vscale x 8 x i1>, <vscale x 8 x half>)
+declare float @llvm.aarch64.sve.fmaxv.nxv4f32(<vscale x 4 x i1>, <vscale x 4 x float>)
+declare double @llvm.aarch64.sve.fmaxv.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>)
+
+declare half @llvm.aarch64.sve.fminnmv.nxv8f16(<vscale x 8 x i1>, <vscale x 8 x half>)
+declare float @llvm.aarch64.sve.fminnmv.nxv4f32(<vscale x 4 x i1>, <vscale x 4 x float>)
+declare double @llvm.aarch64.sve.fminnmv.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>)
+
+declare half @llvm.aarch64.sve.fminv.nxv8f16(<vscale x 8 x i1>, <vscale x 8 x half>)
+declare float @llvm.aarch64.sve.fminv.nxv4f32(<vscale x 4 x i1>, <vscale x 4 x float>)
+declare double @llvm.aarch64.sve.fminv.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>)
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-gather-loads-32bit-scaled-offsets.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-gather-loads-32bit-scaled-offsets.ll
new file mode 100644
index 000000000..db593413f
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-gather-loads-32bit-scaled-offsets.ll
@@ -0,0 +1,255 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+;
+; LD1H, LD1W, LD1D: base + 32-bit scaled offset, sign (sxtw) or zero (uxtw)
+; extended to 64 bits
+;   e.g. ld1h z0.d, p0/z, [x0, z0.d, uxtw #1]
+;
+
+; LD1H
+define <vscale x 4 x i32> @gld1h_s_uxtw_index(<vscale x 4 x i1> %pg, i16* %base, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: gld1h_s_uxtw_index:
+; CHECK: ld1h { z0.s }, p0/z, [x0, z0.s, uxtw #1]
+; CHECK-NEXT:	ret
+  %load = call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.uxtw.index.nxv4i16(<vscale x 4 x i1> %pg,
+                                                                                  i16* %base,
+                                                                                  <vscale x 4 x i32> %b)
+  %res = zext <vscale x 4 x i16> %load to <vscale x 4 x i32>
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 4 x i32> @gld1h_s_sxtw_index(<vscale x 4 x i1> %pg, i16* %base, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: gld1h_s_sxtw_index:
+; CHECK: ld1h { z0.s }, p0/z, [x0, z0.s, sxtw #1]
+; CHECK-NEXT: ret
+  %load = call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.sxtw.index.nxv4i16(<vscale x 4 x i1> %pg,
+                                                                                  i16* %base,
+                                                                                  <vscale x 4 x i32> %b)
+  %res = zext <vscale x 4 x i16> %load to <vscale x 4 x i32>
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @gld1h_d_uxtw_index(<vscale x 2 x i1> %pg, i16* %base, <vscale x 2 x i32> %b) {
+; CHECK-LABEL: gld1h_d_uxtw_index:
+; CHECK: ld1h { z0.d }, p0/z, [x0, z0.d, uxtw #1]
+; CHECK-NEXT: ret
+  %load = call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.uxtw.index.nxv2i16(<vscale x 2 x i1> %pg,
+                                                                                  i16* %base,
+                                                                                  <vscale x 2 x i32> %b)
+  %res = zext <vscale x 2 x i16> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @gld1h_d_sxtw_index(<vscale x 2 x i1> %pg, i16* %base, <vscale x 2 x i32> %b) {
+; CHECK-LABEL: gld1h_d_sxtw_index:
+; CHECK: ld1h { z0.d }, p0/z, [x0, z0.d, sxtw #1]
+; CHECK-NEXT: ret
+  %load = call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.sxtw.index.nxv2i16(<vscale x 2 x i1> %pg,
+                                                                                  i16* %base,
+                                                                                  <vscale x 2 x i32> %b)
+  %res = zext <vscale x 2 x i16> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %res
+}
+
+; LD1W
+define <vscale x 4 x i32> @gld1w_s_uxtw_index(<vscale x 4 x i1> %pg, i32* %base, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: gld1w_s_uxtw_index:
+; CHECK: ld1w { z0.s }, p0/z, [x0, z0.s, uxtw #2]
+; CHECK-NEXT:	ret
+  %load = call <vscale x 4 x i32> @llvm.aarch64.sve.ld1.gather.uxtw.index.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                                  i32* %base,
+                                                                                  <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i32> %load
+}
+
+define <vscale x 4 x i32> @gld1w_s_sxtw_index(<vscale x 4 x i1> %pg, i32* %base, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: gld1w_s_sxtw_index:
+; CHECK: ld1w { z0.s }, p0/z, [x0, z0.s, sxtw #2]
+; CHECK-NEXT: ret
+  %load = call <vscale x 4 x i32> @llvm.aarch64.sve.ld1.gather.sxtw.index.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                                  i32* %base,
+                                                                                  <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i32> %load
+}
+
+define <vscale x 2 x i64> @gld1w_d_uxtw_index(<vscale x 2 x i1> %pg, i32* %base, <vscale x 2 x i32> %b) {
+; CHECK-LABEL: gld1w_d_uxtw_index:
+; CHECK: ld1w { z0.d }, p0/z, [x0, z0.d, uxtw #2]
+; CHECK-NEXT: ret
+  %load = call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.uxtw.index.nxv2i32(<vscale x 2 x i1> %pg,
+                                                                                  i32* %base,
+                                                                                  <vscale x 2 x i32> %b)
+  %res = zext <vscale x 2 x i32> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @gld1w_d_sxtw_index(<vscale x 2 x i1> %pg, i32* %base, <vscale x 2 x i32> %b) {
+; CHECK-LABEL: gld1w_d_sxtw_index:
+; CHECK: ld1w { z0.d }, p0/z, [x0, z0.d, sxtw #2]
+; CHECK-NEXT: ret
+  %load = call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.sxtw.index.nxv2i32(<vscale x 2 x i1> %pg,
+                                                                                  i32* %base,
+                                                                                  <vscale x 2 x i32> %b)
+  %res = zext <vscale x 2 x i32> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 4 x float> @gld1w_s_uxtw_index_float(<vscale x 4 x i1> %pg, float* %base, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: gld1w_s_uxtw_index_float:
+; CHECK: ld1w { z0.s }, p0/z, [x0, z0.s, uxtw #2]
+; CHECK-NEXT:	ret
+  %load = call <vscale x 4 x float> @llvm.aarch64.sve.ld1.gather.uxtw.index.nxv4f32(<vscale x 4 x i1> %pg,
+                                                                                    float* %base,
+                                                                                    <vscale x 4 x i32> %b)
+  ret <vscale x 4 x float> %load
+}
+
+define <vscale x 4 x float> @gld1w_s_sxtw_index_float(<vscale x 4 x i1> %pg, float* %base, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: gld1w_s_sxtw_index_float:
+; CHECK: ld1w { z0.s }, p0/z, [x0, z0.s, sxtw #2]
+; CHECK-NEXT:	ret
+  %load = call <vscale x 4 x float> @llvm.aarch64.sve.ld1.gather.sxtw.index.nxv4f32(<vscale x 4 x i1> %pg,
+                                                                                    float* %base,
+                                                                                    <vscale x 4 x i32> %b)
+  ret <vscale x 4 x float> %load
+}
+
+; LD1D
+define <vscale x 2 x i64> @gld1d_s_uxtw_index(<vscale x 2 x i1> %pg, i64* %base, <vscale x 2 x i32> %b) {
+; CHECK-LABEL: gld1d_s_uxtw_index:
+; CHECK: ld1d { z0.d }, p0/z, [x0, z0.d, uxtw #3]
+; CHECK-NEXT:	ret
+  %load = call <vscale x 2 x i64> @llvm.aarch64.sve.ld1.gather.uxtw.index.nxv2i64(<vscale x 2 x i1> %pg,
+                                                                                  i64* %base,
+                                                                                  <vscale x 2 x i32> %b)
+  ret <vscale x 2 x i64> %load
+}
+
+define <vscale x 2 x i64> @gld1d_sxtw_index(<vscale x 2 x i1> %pg, i64* %base, <vscale x 2 x i32> %b) {
+; CHECK-LABEL: gld1d_sxtw_index:
+; CHECK: ld1d { z0.d }, p0/z, [x0, z0.d, sxtw #3]
+; CHECK-NEXT:	ret
+  %load = call <vscale x 2 x i64> @llvm.aarch64.sve.ld1.gather.sxtw.index.nxv2i64(<vscale x 2 x i1> %pg,
+                                                                                  i64* %base,
+                                                                                  <vscale x 2 x i32> %b)
+  ret <vscale x 2 x i64> %load
+}
+
+define <vscale x 2 x double> @gld1d_uxtw_index_double(<vscale x 2 x i1> %pg, double* %base, <vscale x 2 x i32> %b) {
+; CHECK-LABEL: gld1d_uxtw_index_double:
+; CHECK: ld1d { z0.d }, p0/z, [x0, z0.d, uxtw #3]
+; CHECK-NEXT:	ret
+  %load = call <vscale x 2 x double> @llvm.aarch64.sve.ld1.gather.uxtw.index.nxv2f64(<vscale x 2 x i1> %pg,
+                                                                                     double* %base,
+                                                                                     <vscale x 2 x i32> %b)
+  ret <vscale x 2 x double> %load
+}
+
+define <vscale x 2 x double> @gld1d_sxtw_index_double(<vscale x 2 x i1> %pg, double* %base, <vscale x 2 x i32> %b) {
+; CHECK-LABEL: gld1d_sxtw_index_double:
+; CHECK: ld1d { z0.d }, p0/z, [x0, z0.d, sxtw #3]
+; CHECK-NEXT:	ret
+  %load = call <vscale x 2 x double> @llvm.aarch64.sve.ld1.gather.sxtw.index.nxv2f64(<vscale x 2 x i1> %pg,
+                                                                                     double* %base,
+                                                                                     <vscale x 2 x i32> %b)
+  ret <vscale x 2 x double> %load
+}
+
+;
+; LD1SH, LD1SW, LD1SD: base + 32-bit scaled offset, sign (sxtw) or zero (uxtw)
+; extended to 64 bits
+;   e.g. ld1sh z0.d, p0/z, [x0, z0.d, uxtw #1]
+;
+
+; LD1SH
+define <vscale x 4 x i32> @gld1sh_s_uxtw_index(<vscale x 4 x i1> %pg, i16* %base, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: gld1sh_s_uxtw_index:
+; CHECK: ld1sh { z0.s }, p0/z, [x0, z0.s, uxtw #1]
+; CHECK-NEXT:	ret
+  %load = call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.uxtw.index.nxv4i16(<vscale x 4 x i1> %pg,
+                                                                                  i16* %base,
+                                                                                  <vscale x 4 x i32> %b)
+  %res = sext <vscale x 4 x i16> %load to <vscale x 4 x i32>
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 4 x i32> @gld1sh_s_sxtw_index(<vscale x 4 x i1> %pg, i16* %base, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: gld1sh_s_sxtw_index:
+; CHECK: ld1sh { z0.s }, p0/z, [x0, z0.s, sxtw #1]
+; CHECK-NEXT: ret
+  %load = call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.sxtw.index.nxv4i16(<vscale x 4 x i1> %pg,
+                                                                                  i16* %base,
+                                                                                  <vscale x 4 x i32> %b)
+  %res = sext <vscale x 4 x i16> %load to <vscale x 4 x i32>
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @gld1sh_d_uxtw_index(<vscale x 2 x i1> %pg, i16* %base, <vscale x 2 x i32> %b) {
+; CHECK-LABEL: gld1sh_d_uxtw_index:
+; CHECK: ld1sh { z0.d }, p0/z, [x0, z0.d, uxtw #1]
+; CHECK-NEXT: ret
+  %load = call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.uxtw.index.nxv2i16(<vscale x 2 x i1> %pg,
+                                                                                  i16* %base,
+                                                                                  <vscale x 2 x i32> %b)
+  %res = sext <vscale x 2 x i16> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @gld1sh_d_sxtw_index(<vscale x 2 x i1> %pg, i16* %base, <vscale x 2 x i32> %b) {
+; CHECK-LABEL: gld1sh_d_sxtw_index:
+; CHECK: ld1sh { z0.d }, p0/z, [x0, z0.d, sxtw #1]
+; CHECK-NEXT: ret
+  %load = call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.sxtw.index.nxv2i16(<vscale x 2 x i1> %pg,
+                                                                                  i16* %base,
+                                                                                  <vscale x 2 x i32> %b)
+  %res = sext <vscale x 2 x i16> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %res
+}
+
+; LD1SW
+define <vscale x 2 x i64> @gld1sw_d_uxtw_index(<vscale x 2 x i1> %pg, i32* %base, <vscale x 2 x i32> %b) {
+; CHECK-LABEL: gld1sw_d_uxtw_index:
+; CHECK: ld1sw { z0.d }, p0/z, [x0, z0.d, uxtw #2]
+; CHECK-NEXT: ret
+  %load = call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.uxtw.index.nxv2i32(<vscale x 2 x i1> %pg,
+                                                                                  i32* %base,
+                                                                                  <vscale x 2 x i32> %b)
+  %res = sext <vscale x 2 x i32> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @gld1sw_d_sxtw_index(<vscale x 2 x i1> %pg, i32* %base, <vscale x 2 x i32> %b) {
+; CHECK-LABEL: gld1sw_d_sxtw_index:
+; CHECK: ld1sw { z0.d }, p0/z, [x0, z0.d, sxtw #2]
+; CHECK-NEXT: ret
+  %load = call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.sxtw.index.nxv2i32(<vscale x 2 x i1> %pg,
+                                                                                  i32* %base,
+                                                                                  <vscale x 2 x i32> %b)
+  %res = sext <vscale x 2 x i32> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %res
+}
+
+
+; LD1H/LD1SH
+declare <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.uxtw.index.nxv4i16(<vscale x 4 x i1>, i16*, <vscale x 4 x i32>)
+declare <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.sxtw.index.nxv4i16(<vscale x 4 x i1>, i16*, <vscale x 4 x i32>)
+
+declare <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.uxtw.index.nxv2i16(<vscale x 2 x i1>, i16*, <vscale x 2 x i32>)
+declare <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.sxtw.index.nxv2i16(<vscale x 2 x i1>, i16*, <vscale x 2 x i32>)
+
+; LD1W/LD1SW
+declare <vscale x 4 x i32> @llvm.aarch64.sve.ld1.gather.uxtw.index.nxv4i32(<vscale x 4 x i1>, i32*, <vscale x 4 x i32>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.ld1.gather.sxtw.index.nxv4i32(<vscale x 4 x i1>, i32*, <vscale x 4 x i32>)
+
+declare <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.uxtw.index.nxv2i32(<vscale x 2 x i1>, i32*, <vscale x 2 x i32>)
+declare <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.sxtw.index.nxv2i32(<vscale x 2 x i1>, i32*, <vscale x 2 x i32>)
+
+declare <vscale x 4 x float> @llvm.aarch64.sve.ld1.gather.uxtw.index.nxv4f32(<vscale x 4 x i1>, float*, <vscale x 4 x i32>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.ld1.gather.sxtw.index.nxv4f32(<vscale x 4 x i1>, float*, <vscale x 4 x i32>)
+
+; LD1D
+declare <vscale x 2 x i64> @llvm.aarch64.sve.ld1.gather.uxtw.index.nxv2i64(<vscale x 2 x i1>, i64*, <vscale x 2 x i32>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.ld1.gather.sxtw.index.nxv2i64(<vscale x 2 x i1>, i64*, <vscale x 2 x i32>)
+
+declare <vscale x 2 x double> @llvm.aarch64.sve.ld1.gather.uxtw.index.nxv2f64(<vscale x 2 x i1>, double*, <vscale x 2 x i32>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.ld1.gather.sxtw.index.nxv2f64(<vscale x 2 x i1>, double*, <vscale x 2 x i32>)
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-gather-loads-32bit-unscaled-offsets.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-gather-loads-32bit-unscaled-offsets.ll
new file mode 100644
index 000000000..ba8806986
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-gather-loads-32bit-unscaled-offsets.ll
@@ -0,0 +1,348 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+;
+; LD1B, LD1W, LD1H, LD1D: base + 32-bit unscaled offset, sign (sxtw) or zero
+; (uxtw) extended to 64 bits.
+;   e.g. ld1h { z0.d }, p0/z, [x0, z0.d, uxtw]
+;
+
+; LD1B
+define <vscale x 4 x i32> @gld1b_s_uxtw(<vscale x 4 x i1> %pg, i8* %base, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: gld1b_s_uxtw:
+; CHECK: ld1b { z0.s }, p0/z, [x0, z0.s, uxtw]
+; CHECK-NEXT: ret
+  %load = call <vscale x 4 x i8> @llvm.aarch64.sve.ld1.gather.uxtw.nxv4i8(<vscale x 4 x i1> %pg,
+                                                                          i8* %base,
+                                                                          <vscale x 4 x i32> %b)
+  %res = zext <vscale x 4 x i8> %load to <vscale x 4 x i32>
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 4 x i32> @gld1b_s_sxtw(<vscale x 4 x i1> %pg, i8* %base, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: gld1b_s_sxtw:
+; CHECK: ld1b { z0.s }, p0/z, [x0, z0.s, sxtw]
+; CHECK-NEXT: ret
+  %load = call <vscale x 4 x i8> @llvm.aarch64.sve.ld1.gather.sxtw.nxv4i8(<vscale x 4 x i1> %pg,
+                                                                          i8* %base,
+                                                                          <vscale x 4 x i32> %b)
+  %res = zext <vscale x 4 x i8> %load to <vscale x 4 x i32>
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @gld1b_d_uxtw(<vscale x 2 x i1> %pg, i8* %base, <vscale x 2 x i32> %b) {
+; CHECK-LABEL: gld1b_d_uxtw:
+; CHECK: ld1b { z0.d }, p0/z, [x0, z0.d, uxtw]
+; CHECK-NEXT: ret
+  %load = call <vscale x 2 x i8> @llvm.aarch64.sve.ld1.gather.uxtw.nxv2i8(<vscale x 2 x i1> %pg,
+                                                                          i8* %base,
+                                                                          <vscale x 2 x i32> %b)
+  %res = zext <vscale x 2 x i8> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @gld1b_d_sxtw(<vscale x 2 x i1> %pg, i8* %base, <vscale x 2 x i32> %b) {
+; CHECK-LABEL: gld1b_d_sxtw:
+; CHECK: ld1b { z0.d }, p0/z, [x0, z0.d, sxtw]
+; CHECK-NEXT: ret
+  %load = call <vscale x 2 x i8> @llvm.aarch64.sve.ld1.gather.sxtw.nxv2i8(<vscale x 2 x i1> %pg,
+                                                                          i8* %base,
+                                                                          <vscale x 2 x i32> %b)
+  %res = zext <vscale x 2 x i8> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %res
+}
+
+; LD1H
+define <vscale x 4 x i32> @gld1h_s_uxtw(<vscale x 4 x i1> %pg, i16* %base, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: gld1h_s_uxtw:
+; CHECK: ld1h { z0.s }, p0/z, [x0, z0.s, uxtw]
+; CHECK-NEXT:	ret
+  %load = call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.uxtw.nxv4i16(<vscale x 4 x i1> %pg,
+                                                                            i16* %base,
+                                                                            <vscale x 4 x i32> %b)
+  %res = zext <vscale x 4 x i16> %load to <vscale x 4 x i32>
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 4 x i32> @gld1h_s_sxtw(<vscale x 4 x i1> %pg, i16* %base, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: gld1h_s_sxtw:
+; CHECK: ld1h { z0.s }, p0/z, [x0, z0.s, sxtw]
+; CHECK-NEXT: ret
+  %load = call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.sxtw.nxv4i16(<vscale x 4 x i1> %pg,
+                                                                            i16* %base,
+                                                                            <vscale x 4 x i32> %b)
+  %res = zext <vscale x 4 x i16> %load to <vscale x 4 x i32>
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @gld1h_d_uxtw(<vscale x 2 x i1> %pg, i16* %base, <vscale x 2 x i32> %b) {
+; CHECK-LABEL: gld1h_d_uxtw:
+; CHECK: ld1h { z0.d }, p0/z, [x0, z0.d, uxtw]
+; CHECK-NEXT: ret
+  %load = call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.uxtw.nxv2i16(<vscale x 2 x i1> %pg,
+                                                                            i16* %base,
+                                                                            <vscale x 2 x i32> %b)
+  %res = zext <vscale x 2 x i16> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @gld1h_d_sxtw(<vscale x 2 x i1> %pg, i16* %base, <vscale x 2 x i32> %b) {
+; CHECK-LABEL: gld1h_d_sxtw:
+; CHECK: ld1h { z0.d }, p0/z, [x0, z0.d, sxtw]
+; CHECK-NEXT: ret
+  %load = call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.sxtw.nxv2i16(<vscale x 2 x i1> %pg,
+                                                                            i16* %base,
+                                                                            <vscale x 2 x i32> %b)
+  %res = zext <vscale x 2 x i16> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %res
+}
+
+; LD1W
+define <vscale x 4 x i32> @gld1w_s_uxtw(<vscale x 4 x i1> %pg, i32* %base, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: gld1w_s_uxtw:
+; CHECK: ld1w { z0.s }, p0/z, [x0, z0.s, uxtw]
+; CHECK-NEXT:	ret
+  %load = call <vscale x 4 x i32> @llvm.aarch64.sve.ld1.gather.uxtw.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                            i32* %base,
+                                                                            <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i32> %load
+}
+
+define <vscale x 4 x i32> @gld1w_s_sxtw(<vscale x 4 x i1> %pg, i32* %base, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: gld1w_s_sxtw:
+; CHECK: ld1w { z0.s }, p0/z, [x0, z0.s, sxtw]
+; CHECK-NEXT: ret
+  %load = call <vscale x 4 x i32> @llvm.aarch64.sve.ld1.gather.sxtw.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                            i32* %base,
+                                                                            <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i32> %load
+}
+
+define <vscale x 2 x i64> @gld1w_d_uxtw(<vscale x 2 x i1> %pg, i32* %base, <vscale x 2 x i32> %b) {
+; CHECK-LABEL: gld1w_d_uxtw:
+; CHECK: ld1w { z0.d }, p0/z, [x0, z0.d, uxtw]
+; CHECK-NEXT: ret
+  %load = call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.uxtw.nxv2i32(<vscale x 2 x i1> %pg,
+                                                                            i32* %base,
+                                                                            <vscale x 2 x i32> %b)
+  %res = zext <vscale x 2 x i32> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @gld1w_d_sxtw(<vscale x 2 x i1> %pg, i32* %base, <vscale x 2 x i32> %b) {
+; CHECK-LABEL: gld1w_d_sxtw:
+; CHECK: ld1w { z0.d }, p0/z, [x0, z0.d, sxtw]
+; CHECK-NEXT: ret
+  %load = call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.sxtw.nxv2i32(<vscale x 2 x i1> %pg,
+                                                                            i32* %base,
+                                                                            <vscale x 2 x i32> %b)
+  %res = zext <vscale x 2 x i32> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 4 x float> @gld1w_s_uxtw_float(<vscale x 4 x i1> %pg, float* %base, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: gld1w_s_uxtw_float:
+; CHECK: ld1w { z0.s }, p0/z, [x0, z0.s, uxtw]
+; CHECK-NEXT:	ret
+  %load = call <vscale x 4 x float> @llvm.aarch64.sve.ld1.gather.uxtw.nxv4f32(<vscale x 4 x i1> %pg,
+                                                                              float* %base,
+                                                                              <vscale x 4 x i32> %b)
+  ret <vscale x 4 x float> %load
+}
+
+define <vscale x 4 x float> @gld1w_s_sxtw_float(<vscale x 4 x i1> %pg, float* %base, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: gld1w_s_sxtw_float:
+; CHECK: ld1w { z0.s }, p0/z, [x0, z0.s, sxtw]
+; CHECK-NEXT: ret
+  %load = call <vscale x 4 x float> @llvm.aarch64.sve.ld1.gather.sxtw.nxv4f32(<vscale x 4 x i1> %pg,
+                                                                              float* %base,
+                                                                              <vscale x 4 x i32> %b)
+  ret <vscale x 4 x float> %load
+}
+
+; LD1D
+define <vscale x 2 x i64> @gld1d_d_uxtw(<vscale x 2 x i1> %pg, i64* %base, <vscale x 2 x i32> %b) {
+; CHECK-LABEL: gld1d_d_uxtw:
+; CHECK: ld1d { z0.d }, p0/z, [x0, z0.d, uxtw]
+; CHECK-NEXT: ret
+  %load = call <vscale x 2 x i64> @llvm.aarch64.sve.ld1.gather.uxtw.nxv2i64(<vscale x 2 x i1> %pg,
+                                                                            i64* %base,
+                                                                            <vscale x 2 x i32> %b)
+  ret <vscale x 2 x i64> %load
+}
+
+define <vscale x 2 x i64> @gld1d_d_sxtw(<vscale x 2 x i1> %pg, i64* %base, <vscale x 2 x i32> %b) {
+; CHECK-LABEL: gld1d_d_sxtw:
+; CHECK: ld1d { z0.d }, p0/z, [x0, z0.d, sxtw]
+; CHECK-NEXT: ret
+  %load = call <vscale x 2 x i64> @llvm.aarch64.sve.ld1.gather.sxtw.nxv2i64(<vscale x 2 x i1> %pg,
+                                                                            i64* %base,
+                                                                            <vscale x 2 x i32> %b)
+  ret <vscale x 2 x i64> %load
+}
+
+define <vscale x 2 x double> @gld1d_d_uxtw_double(<vscale x 2 x i1> %pg, double* %base, <vscale x 2 x i32> %b) {
+; CHECK-LABEL: gld1d_d_uxtw_double:
+; CHECK: ld1d { z0.d }, p0/z, [x0, z0.d, uxtw]
+; CHECK-NEXT: ret
+  %load = call <vscale x 2 x double> @llvm.aarch64.sve.ld1.gather.uxtw.nxv2f64(<vscale x 2 x i1> %pg,
+                                                                               double* %base,
+                                                                               <vscale x 2 x i32> %b)
+  ret <vscale x 2 x double> %load
+}
+
+define <vscale x 2 x double> @gld1d_d_sxtw_double(<vscale x 2 x i1> %pg, double* %base, <vscale x 2 x i32> %b) {
+; CHECK-LABEL: gld1d_d_sxtw_double:
+; CHECK: ld1d { z0.d }, p0/z, [x0, z0.d, sxtw]
+; CHECK-NEXT: ret
+  %load = call <vscale x 2 x double> @llvm.aarch64.sve.ld1.gather.sxtw.nxv2f64(<vscale x 2 x i1> %pg,
+                                                                               double* %base,
+                                                                               <vscale x 2 x i32> %b)
+  ret <vscale x 2 x double> %load
+}
+
+;
+; LD1SB, LD1SW, LD1SH: base + 32-bit unscaled offset, sign (sxtw) or zero
+; (uxtw) extended to 64 bits.
+;   e.g. ld1sh { z0.d }, p0/z, [x0, z0.d, uxtw]
+;
+
+; LD1SB
+define <vscale x 4 x i32> @gld1sb_s_uxtw(<vscale x 4 x i1> %pg, i8* %base, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: gld1sb_s_uxtw:
+; CHECK: ld1sb { z0.s }, p0/z, [x0, z0.s, uxtw]
+; CHECK-NEXT: ret
+  %load = call <vscale x 4 x i8> @llvm.aarch64.sve.ld1.gather.uxtw.nxv4i8(<vscale x 4 x i1> %pg,
+                                                                          i8* %base,
+                                                                          <vscale x 4 x i32> %b)
+  %res = sext <vscale x 4 x i8> %load to <vscale x 4 x i32>
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 4 x i32> @gld1sb_s_sxtw(<vscale x 4 x i1> %pg, i8* %base, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: gld1sb_s_sxtw:
+; CHECK: ld1sb { z0.s }, p0/z, [x0, z0.s, sxtw]
+; CHECK-NEXT: ret
+  %load = call <vscale x 4 x i8> @llvm.aarch64.sve.ld1.gather.sxtw.nxv4i8(<vscale x 4 x i1> %pg,
+                                                                          i8* %base,
+                                                                          <vscale x 4 x i32> %b)
+  %res = sext <vscale x 4 x i8> %load to <vscale x 4 x i32>
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @gld1sb_d_uxtw(<vscale x 2 x i1> %pg, i8* %base, <vscale x 2 x i32> %b) {
+; CHECK-LABEL: gld1sb_d_uxtw:
+; CHECK: ld1sb { z0.d }, p0/z, [x0, z0.d, uxtw]
+; CHECK-NEXT: ret
+  %load = call <vscale x 2 x i8> @llvm.aarch64.sve.ld1.gather.uxtw.nxv2i8(<vscale x 2 x i1> %pg,
+                                                                          i8* %base,
+                                                                          <vscale x 2 x i32> %b)
+  %res = sext <vscale x 2 x i8> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @gld1sb_d_sxtw(<vscale x 2 x i1> %pg, i8* %base, <vscale x 2 x i32> %b) {
+; CHECK-LABEL: gld1sb_d_sxtw:
+; CHECK: ld1sb { z0.d }, p0/z, [x0, z0.d, sxtw]
+; CHECK-NEXT: ret
+  %load = call <vscale x 2 x i8> @llvm.aarch64.sve.ld1.gather.sxtw.nxv2i8(<vscale x 2 x i1> %pg,
+                                                                          i8* %base,
+                                                                          <vscale x 2 x i32> %b)
+  %res = sext <vscale x 2 x i8> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %res
+}
+
+; LD1SH
+define <vscale x 4 x i32> @gld1sh_s_uxtw(<vscale x 4 x i1> %pg, i16* %base, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: gld1sh_s_uxtw:
+; CHECK: ld1sh { z0.s }, p0/z, [x0, z0.s, uxtw]
+; CHECK-NEXT:	ret
+  %load = call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.uxtw.nxv4i16(<vscale x 4 x i1> %pg,
+                                                                            i16* %base,
+                                                                            <vscale x 4 x i32> %b)
+  %res = sext <vscale x 4 x i16> %load to <vscale x 4 x i32>
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 4 x i32> @gld1sh_s_sxtw(<vscale x 4 x i1> %pg, i16* %base, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: gld1sh_s_sxtw:
+; CHECK: ld1sh { z0.s }, p0/z, [x0, z0.s, sxtw]
+; CHECK-NEXT: ret
+  %load = call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.sxtw.nxv4i16(<vscale x 4 x i1> %pg,
+                                                                            i16* %base,
+                                                                            <vscale x 4 x i32> %b)
+  %res = sext <vscale x 4 x i16> %load to <vscale x 4 x i32>
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @gld1sh_d_uxtw(<vscale x 2 x i1> %pg, i16* %base, <vscale x 2 x i32> %b) {
+; CHECK-LABEL: gld1sh_d_uxtw:
+; CHECK: ld1sh { z0.d }, p0/z, [x0, z0.d, uxtw]
+; CHECK-NEXT: ret
+  %load = call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.uxtw.nxv2i16(<vscale x 2 x i1> %pg,
+                                                                            i16* %base,
+                                                                            <vscale x 2 x i32> %b)
+  %res = sext <vscale x 2 x i16> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @gld1sh_d_sxtw(<vscale x 2 x i1> %pg, i16* %base, <vscale x 2 x i32> %b) {
+; CHECK-LABEL: gld1sh_d_sxtw:
+; CHECK: ld1sh { z0.d }, p0/z, [x0, z0.d, sxtw]
+; CHECK-NEXT: ret
+  %load = call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.sxtw.nxv2i16(<vscale x 2 x i1> %pg,
+                                                                            i16* %base,
+                                                                            <vscale x 2 x i32> %b)
+  %res = sext <vscale x 2 x i16> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %res
+}
+
+; LD1SW
+define <vscale x 2 x i64> @gld1sw_d_uxtw(<vscale x 2 x i1> %pg, i32* %base, <vscale x 2 x i32> %b) {
+; CHECK-LABEL: gld1sw_d_uxtw:
+; CHECK: ld1sw { z0.d }, p0/z, [x0, z0.d, uxtw]
+; CHECK-NEXT: ret
+  %load = call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.uxtw.nxv2i32(<vscale x 2 x i1> %pg,
+                                                                            i32* %base,
+                                                                            <vscale x 2 x i32> %b)
+  %res = sext <vscale x 2 x i32> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @gld1sw_d_sxtw(<vscale x 2 x i1> %pg, i32* %base, <vscale x 2 x i32> %b) {
+; CHECK-LABEL: gld1sw_d_sxtw:
+; CHECK: ld1sw { z0.d }, p0/z, [x0, z0.d, sxtw]
+; CHECK-NEXT: ret
+  %load = call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.sxtw.nxv2i32(<vscale x 2 x i1> %pg,
+                                                                            i32* %base,
+                                                                            <vscale x 2 x i32> %b)
+  %res = sext <vscale x 2 x i32> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %res
+}
+
+; LD1B/LD1SB
+declare <vscale x 4 x i8> @llvm.aarch64.sve.ld1.gather.uxtw.nxv4i8(<vscale x 4 x i1>, i8*, <vscale x 4 x i32>)
+declare <vscale x 2 x i8> @llvm.aarch64.sve.ld1.gather.uxtw.nxv2i8(<vscale x 2 x i1>, i8*, <vscale x 2 x i32>)
+declare <vscale x 4 x i8> @llvm.aarch64.sve.ld1.gather.sxtw.nxv4i8(<vscale x 4 x i1>, i8*, <vscale x 4 x i32>)
+declare <vscale x 2 x i8> @llvm.aarch64.sve.ld1.gather.sxtw.nxv2i8(<vscale x 2 x i1>, i8*, <vscale x 2 x i32>)
+
+; LD1H/LD1SH
+declare <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.sxtw.nxv4i16(<vscale x 4 x i1>, i16*, <vscale x 4 x i32>)
+declare <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.sxtw.nxv2i16(<vscale x 2 x i1>, i16*, <vscale x 2 x i32>)
+declare <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.uxtw.nxv4i16(<vscale x 4 x i1>, i16*, <vscale x 4 x i32>)
+declare <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.uxtw.nxv2i16(<vscale x 2 x i1>, i16*, <vscale x 2 x i32>)
+
+; LD1W/LD1SW
+declare <vscale x 4 x i32> @llvm.aarch64.sve.ld1.gather.sxtw.nxv4i32(<vscale x 4 x i1>, i32*, <vscale x 4 x i32>)
+declare <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.sxtw.nxv2i32(<vscale x 2 x i1>, i32*, <vscale x 2 x i32>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.ld1.gather.uxtw.nxv4i32(<vscale x 4 x i1>, i32*, <vscale x 4 x i32>)
+declare <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.uxtw.nxv2i32(<vscale x 2 x i1>, i32*, <vscale x 2 x i32>)
+
+declare <vscale x 4 x float> @llvm.aarch64.sve.ld1.gather.sxtw.nxv4f32(<vscale x 4 x i1>, float*, <vscale x 4 x i32>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.ld1.gather.uxtw.nxv4f32(<vscale x 4 x i1>, float*, <vscale x 4 x i32>)
+
+; LD1D
+declare <vscale x 2 x i64> @llvm.aarch64.sve.ld1.gather.sxtw.nxv2i64(<vscale x 2 x i1>, i64*, <vscale x 2 x i32>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.ld1.gather.uxtw.nxv2i64(<vscale x 2 x i1>, i64*, <vscale x 2 x i32>)
+
+declare <vscale x 2 x double> @llvm.aarch64.sve.ld1.gather.sxtw.nxv2f64(<vscale x 2 x i1>, double*, <vscale x 2 x i32>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.ld1.gather.uxtw.nxv2f64(<vscale x 2 x i1>, double*, <vscale x 2 x i32>)
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-gather-loads-64bit-scaled-offset.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-gather-loads-64bit-scaled-offset.ll
new file mode 100644
index 000000000..10de34975
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-gather-loads-64bit-scaled-offset.ll
@@ -0,0 +1,80 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+;
+; LD1H, LD1W, LD1D: base + 64-bit scaled offset
+;   e.g. ld1h z0.d, p0/z, [x0, z0.d, lsl #1]
+;
+
+define <vscale x 2 x i64> @gld1h_index(<vscale x 2 x i1> %pg, i16* %base, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: gld1h_index
+; CHECK:	    ld1h	{ z0.d }, p0/z, [x0, z0.d, lsl #1]
+; CHECK-NEXT:	ret
+  %load = call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.index.nxv2i16(<vscale x 2 x i1> %pg,
+                                                                             i16* %base,
+                                                                             <vscale x 2 x i64> %b)
+  %res = zext <vscale x 2 x i16> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @gld1w_index(<vscale x 2 x i1> %pg, i32* %base, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: gld1w_index
+; CHECK:	    ld1w	{ z0.d }, p0/z, [x0, z0.d, lsl #2]
+; CHECK-NEXT:	ret
+  %load = call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.index.nxv2i32(<vscale x 2 x i1> %pg,
+                                                                             i32* %base,
+                                                                             <vscale x 2 x i64> %b)
+  %res = zext <vscale x 2 x i32> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @gld1d_index(<vscale x 2 x i1> %pg, i64* %base, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: gld1d_index
+; CHECK:	    ld1d	{ z0.d }, p0/z, [x0, z0.d, lsl #3]
+; CHECK-NEXT:	ret
+  %load = call <vscale x 2 x i64> @llvm.aarch64.sve.ld1.gather.index.nxv2i64(<vscale x 2 x i1> %pg,
+                                                                             i64* %base,
+                                                                             <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i64> %load
+}
+
+define <vscale x 2 x double> @gld1d_index_double(<vscale x 2 x i1> %pg, double* %base, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: gld1d_index_double
+; CHECK:	    ld1d	{ z0.d }, p0/z, [x0, z0.d, lsl #3]
+; CHECK-NEXT:	ret
+  %load = call <vscale x 2 x double> @llvm.aarch64.sve.ld1.gather.index.nxv2f64(<vscale x 2 x i1> %pg,
+                                                                                double* %base,
+                                                                                <vscale x 2 x i64> %b)
+  ret <vscale x 2 x double> %load
+}
+
+;
+; LD1SH, LD1SW: base + 64-bit scaled offset
+;   e.g. ld1sh z0.d, p0/z, [x0, z0.d, lsl #1]
+;
+
+define <vscale x 2 x i64> @gld1sh_index(<vscale x 2 x i1> %pg, i16* %base, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: gld1sh_index
+; CHECK:	    ld1sh	{ z0.d }, p0/z, [x0, z0.d, lsl #1]
+; CHECK-NEXT:	ret
+  %load = call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.index.nxv2i16(<vscale x 2 x i1> %pg,
+                                                                             i16* %base,
+                                                                             <vscale x 2 x i64> %b)
+  %res = sext <vscale x 2 x i16> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @gld1sw_index(<vscale x 2 x i1> %pg, i32* %base, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: gld1sw_index
+; CHECK:	    ld1sw	{ z0.d }, p0/z, [x0, z0.d, lsl #2]
+; CHECK-NEXT:	ret
+  %load = call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.index.nxv2i32(<vscale x 2 x i1> %pg,
+                                                                             i32* %base,
+                                                                             <vscale x 2 x i64> %b)
+  %res = sext <vscale x 2 x i32> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %res
+}
+
+declare <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.index.nxv2i16(<vscale x 2 x i1>, i16*, <vscale x 2 x i64>)
+declare <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.index.nxv2i32(<vscale x 2 x i1>, i32*, <vscale x 2 x i64>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.ld1.gather.index.nxv2i64(<vscale x 2 x i1>, i64*, <vscale x 2 x i64>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.ld1.gather.index.nxv2f64(<vscale x 2 x i1>, double*, <vscale x 2 x i64>)
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-gather-loads-64bit-unscaled-offset.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-gather-loads-64bit-unscaled-offset.ll
new file mode 100644
index 000000000..fddbc24e9
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-gather-loads-64bit-unscaled-offset.ll
@@ -0,0 +1,103 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+;
+; LD1B, LD1W, LD1H, LD1D: base + 64-bit unscaled offset
+;   e.g. ld1h { z0.d }, p0/z, [x0, z0.d]
+;
+
+define <vscale x 2 x i64> @gld1b_d(<vscale x 2 x i1> %pg, i8* %base, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: gld1b_d:
+; CHECK: ld1b { z0.d }, p0/z, [x0, z0.d]
+; CHECK-NEXT: ret
+  %load = call <vscale x 2 x i8> @llvm.aarch64.sve.ld1.gather.nxv2i8(<vscale x 2 x i1> %pg,
+                                                                     i8* %base,
+                                                                     <vscale x 2 x i64> %b)
+  %res = zext <vscale x 2 x i8> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @gld1h_d(<vscale x 2 x i1> %pg, i16* %base, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: gld1h_d:
+; CHECK: ld1h { z0.d }, p0/z, [x0, z0.d]
+; CHECK-NEXT: ret
+  %load = call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.nxv2i16(<vscale x 2 x i1> %pg,
+                                                                       i16* %base,
+                                                                       <vscale x 2 x i64> %b)
+  %res = zext <vscale x 2 x i16> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @gld1w_d(<vscale x 2 x i1> %pg, i32* %base, <vscale x 2 x i64> %offsets) {
+; CHECK-LABEL: gld1w_d:
+; CHECK: ld1w { z0.d }, p0/z, [x0, z0.d]
+; CHECK-NEXT: ret
+  %load = call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.nxv2i32(<vscale x 2 x i1> %pg,
+                                                                       i32* %base,
+                                                                       <vscale x 2 x i64> %offsets)
+  %res = zext <vscale x 2 x i32> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @gld1d_d(<vscale x 2 x i1> %pg, i64* %base, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: gld1d_d:
+; CHECK: ld1d { z0.d }, p0/z, [x0, z0.d]
+; CHECK-NEXT: ret
+  %load = call <vscale x 2 x i64> @llvm.aarch64.sve.ld1.gather.nxv2i64(<vscale x 2 x i1> %pg,
+                                                                       i64* %base,
+                                                                       <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i64> %load
+}
+
+define <vscale x 2 x double> @gld1d_d_double(<vscale x 2 x i1> %pg, double* %base, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: gld1d_d_double:
+; CHECK: ld1d { z0.d }, p0/z, [x0, z0.d]
+; CHECK-NEXT: ret
+  %load = call <vscale x 2 x double> @llvm.aarch64.sve.ld1.gather.nxv2f64(<vscale x 2 x i1> %pg,
+                                                                       double* %base,
+                                                                       <vscale x 2 x i64> %b)
+  ret <vscale x 2 x double> %load
+}
+
+;
+; LD1SB, LD1SW, LD1SH: base + 64-bit unscaled offset
+;   e.g. ld1sh { z0.d }, p0/z, [x0, z0.d]
+;
+
+define <vscale x 2 x i64> @gld1sb_d(<vscale x 2 x i1> %pg, i8* %base, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: gld1sb_d:
+; CHECK: ld1sb { z0.d }, p0/z, [x0, z0.d]
+; CHECK-NEXT: ret
+  %load = call <vscale x 2 x i8> @llvm.aarch64.sve.ld1.gather.nxv2i8(<vscale x 2 x i1> %pg,
+                                                                     i8* %base,
+                                                                     <vscale x 2 x i64> %b)
+  %res = sext <vscale x 2 x i8> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @gld1sh_d(<vscale x 2 x i1> %pg, i16* %base, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: gld1sh_d:
+; CHECK: ld1sh { z0.d }, p0/z, [x0, z0.d]
+; CHECK-NEXT: ret
+  %load = call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.nxv2i16(<vscale x 2 x i1> %pg,
+                                                                       i16* %base,
+                                                                       <vscale x 2 x i64> %b)
+  %res = sext <vscale x 2 x i16> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @gld1sw_d(<vscale x 2 x i1> %pg, i32* %base, <vscale x 2 x i64> %offsets) {
+; CHECK-LABEL: gld1sw_d:
+; CHECK: ld1sw { z0.d }, p0/z, [x0, z0.d]
+; CHECK-NEXT: ret
+  %load = call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.nxv2i32(<vscale x 2 x i1> %pg,
+                                                                       i32* %base,
+                                                                       <vscale x 2 x i64> %offsets)
+  %res = sext <vscale x 2 x i32> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %res
+}
+
+declare <vscale x 2 x i8> @llvm.aarch64.sve.ld1.gather.nxv2i8(<vscale x 2 x i1>, i8*, <vscale x 2 x i64>)
+declare <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.nxv2i16(<vscale x 2 x i1>, i16*, <vscale x 2 x i64>)
+declare <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.nxv2i32(<vscale x 2 x i1>, i32*, <vscale x 2 x i64>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.ld1.gather.nxv2i64(<vscale x 2 x i1>, i64*, <vscale x 2 x i64>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.ld1.gather.nxv2f64(<vscale x 2 x i1>, double*, <vscale x 2 x i64>)
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-gather-loads-vector-base.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-gather-loads-vector-base.ll
new file mode 100644
index 000000000..847c0d11d
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-gather-loads-vector-base.ll
@@ -0,0 +1,186 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+;
+; LD1B, LD1W, LD1H, LD1D: vector + immediate (index)
+;   e.g. ld1h { z0.s }, p0/z, [z0.s, #16]
+;
+
+; LD1B
+define <vscale x 4 x i32> @gld1b_s_imm(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %base) {
+; CHECK-LABEL: gld1b_s_imm:
+; CHECK: ld1b { z0.s }, p0/z, [z0.s, #16]
+; CHECK-NEXT: ret
+  %load = call <vscale x 4 x i8> @llvm.aarch64.sve.ld1.gather.imm.nxv4i8.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                                 <vscale x 4 x i32> %base,
+                                                                                 i64 16)
+  %res = zext <vscale x 4 x i8> %load to <vscale x 4 x i32>
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @gld1b_d_imm(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %base) {
+; CHECK-LABEL: gld1b_d_imm:
+; CHECK: ld1b { z0.d }, p0/z, [z0.d, #16]
+; CHECK-NEXT: ret
+  %load = call <vscale x 2 x i8> @llvm.aarch64.sve.ld1.gather.imm.nxv2i8.nxv2i64(<vscale x 2 x i1> %pg,
+                                                                           <vscale x 2 x i64> %base,
+                                                                           i64 16)
+  %res = zext <vscale x 2 x i8> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %res
+}
+
+; LD1H
+define <vscale x 4 x i32> @gld1h_s_imm(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %base) {
+; CHECK-LABEL: gld1h_s_imm:
+; CHECK: ld1h { z0.s }, p0/z, [z0.s, #16]
+; CHECK-NEXT: ret
+  %load = call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.imm.nxv4i16.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                            <vscale x 4 x i32> %base,
+                                                                            i64 16)
+  %res = zext <vscale x 4 x i16> %load to <vscale x 4 x i32>
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @gld1h_d_imm(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %base) {
+; CHECK-LABEL: gld1h_d_imm:
+; CHECK: ld1h { z0.d }, p0/z, [z0.d, #16]
+; CHECK-NEXT: ret
+  %load = call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.imm.nxv2i16.nxv2i64(<vscale x 2 x i1> %pg,
+                                                                           <vscale x 2 x i64> %base,
+                                                                           i64 16)
+  %res = zext <vscale x 2 x i16> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %res
+}
+
+; LD1W
+define <vscale x 4 x i32> @gld1w_s_imm(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %base) {
+; CHECK-LABEL: gld1w_s_imm:
+; CHECK: ld1w { z0.s }, p0/z, [z0.s, #16]
+; CHECK-NEXT: ret
+  %load = call <vscale x 4 x i32> @llvm.aarch64.sve.ld1.gather.imm.nxv4i32.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                            <vscale x 4 x i32> %base,
+                                                                            i64 16)
+  ret <vscale x 4 x i32> %load
+}
+
+define <vscale x 2 x i64> @gld1w_d_imm(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %base) {
+; CHECK-LABEL: gld1w_d_imm:
+; CHECK: ld1w { z0.d }, p0/z, [z0.d, #16]
+; CHECK-NEXT: ret
+  %load = call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.imm.nxv2i32.nxv2i64(<vscale x 2 x i1> %pg,
+                                                                                   <vscale x 2 x i64> %base,
+                                                                                   i64 16)
+  %res = zext <vscale x 2 x i32> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 4 x float> @gld1w_s_imm_float(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %base) {
+; CHECK-LABEL: gld1w_s_imm_float:
+; CHECK: ld1w { z0.s }, p0/z, [z0.s, #16]
+; CHECK-NEXT: ret
+  %load = call <vscale x 4 x float> @llvm.aarch64.sve.ld1.gather.imm.nxv4f32.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                                     <vscale x 4 x i32> %base,
+                                                                                     i64 16)
+  ret <vscale x 4 x float> %load
+}
+
+; LD1D
+define <vscale x 2 x i64> @gld1d_d_imm(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %base) {
+; CHECK-LABEL: gld1d_d_imm:
+; CHECK: ld1d { z0.d }, p0/z, [z0.d, #16]
+; CHECK-NEXT: ret
+  %load = call <vscale x 2 x i64> @llvm.aarch64.sve.ld1.gather.imm.nxv2i64.nxv2i64(<vscale x 2 x i1> %pg,
+                                                                                   <vscale x 2 x i64> %base,
+                                                                                   i64 16)
+  ret <vscale x 2 x i64> %load
+}
+
+define <vscale x 2 x double> @gld1d_d_imm_double(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %base) {
+; CHECK-LABEL: gld1d_d_imm_double:
+; CHECK: ld1d { z0.d }, p0/z, [z0.d, #16]
+; CHECK-NEXT: ret
+  %load = call <vscale x 2 x double> @llvm.aarch64.sve.ld1.gather.imm.nxv2f64.nxv2i64(<vscale x 2 x i1> %pg,
+                                                                                      <vscale x 2 x i64> %base,
+                                                                                      i64 16)
+  ret <vscale x 2 x double> %load
+}
+
+; LD1SB, LD1SW, LD1SH: vector + immediate (index)
+;   e.g. ld1sh { z0.s }, p0/z, [z0.s, #16]
+;
+
+; LD1SB
+define <vscale x 4 x i32> @gld1sb_s_imm(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %base) {
+; CHECK-LABEL: gld1sb_s_imm:
+; CHECK: ld1sb { z0.s }, p0/z, [z0.s, #16]
+; CHECK-NEXT: ret
+  %load = call <vscale x 4 x i8> @llvm.aarch64.sve.ld1.gather.imm.nxv4i8.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                                 <vscale x 4 x i32> %base,
+                                                                                 i64 16)
+  %res = sext <vscale x 4 x i8> %load to <vscale x 4 x i32>
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @gld1sb_d_imm(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %base) {
+; CHECK-LABEL: gld1sb_d_imm:
+; CHECK: ld1sb { z0.d }, p0/z, [z0.d, #16]
+; CHECK-NEXT: ret
+  %load = call <vscale x 2 x i8> @llvm.aarch64.sve.ld1.gather.imm.nxv2i8.nxv2i64(<vscale x 2 x i1> %pg,
+                                                                           <vscale x 2 x i64> %base,
+                                                                           i64 16)
+  %res = sext <vscale x 2 x i8> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %res
+}
+
+; LD1SH
+define <vscale x 4 x i32> @gld1sh_s_imm(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %base) {
+; CHECK-LABEL: gld1sh_s_imm:
+; CHECK: ld1sh { z0.s }, p0/z, [z0.s, #16]
+; CHECK-NEXT: ret
+  %load = call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.imm.nxv4i16.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                                   <vscale x 4 x i32> %base,
+                                                                                   i64 16)
+  %res = sext <vscale x 4 x i16> %load to <vscale x 4 x i32>
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @gld1sh_d_imm(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %base) {
+; CHECK-LABEL: gld1sh_d_imm:
+; CHECK: ld1sh { z0.d }, p0/z, [z0.d, #16]
+; CHECK-NEXT: ret
+  %load = call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.imm.nxv2i16.nxv2i64(<vscale x 2 x i1> %pg,
+                                                                                   <vscale x 2 x i64> %base,
+                                                                                   i64 16)
+  %res = sext <vscale x 2 x i16> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %res
+}
+
+; LD1SW
+define <vscale x 2 x i64> @gld1sw_d_imm(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %base) {
+; CHECK-LABEL: gld1sw_d_imm:
+; CHECK: ld1sw { z0.d }, p0/z, [z0.d, #16]
+; CHECK-NEXT: ret
+  %load = call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.imm.nxv2i32.nxv2i64(<vscale x 2 x i1> %pg,
+                                                                                   <vscale x 2 x i64> %base,
+                                                                                   i64 16)
+  %res = sext <vscale x 2 x i32> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %res
+}
+
+; LD1B/LD1SB
+declare <vscale x 4 x i8> @llvm.aarch64.sve.ld1.gather.imm.nxv4i8.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, i64)
+declare <vscale x 2 x i8> @llvm.aarch64.sve.ld1.gather.imm.nxv2i8.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>, i64)
+
+; LD1H/LD1SH
+declare <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.imm.nxv4i16.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, i64)
+declare <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.imm.nxv2i16.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>, i64)
+
+; LD1W/LD1SW
+declare <vscale x 4 x i32> @llvm.aarch64.sve.ld1.gather.imm.nxv4i32.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, i64)
+declare <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.imm.nxv2i32.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>, i64)
+
+declare <vscale x 4 x float> @llvm.aarch64.sve.ld1.gather.imm.nxv4f32.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, i64)
+
+; LD1D
+declare <vscale x 2 x i64> @llvm.aarch64.sve.ld1.gather.imm.nxv2i64.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>, i64)
+
+declare <vscale x 2 x double> @llvm.aarch64.sve.ld1.gather.imm.nxv2f64.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>, i64)
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-int-arith.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-int-arith.ll
new file mode 100644
index 000000000..6ddd42a58
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-int-arith.ll
@@ -0,0 +1,192 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+;
+; ABS
+;
+
+define <vscale x 16 x i8> @abs_i8(<vscale x 16 x i8> %a, <vscale x 16 x i1> %pg, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: abs_i8:
+; CHECK: abs z0.b, p0/m, z1.b
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.abs.nxv16i8(<vscale x 16 x i8> %a,
+                                                               <vscale x 16 x i1> %pg,
+                                                               <vscale x 16 x i8> %b)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @abs_i16(<vscale x 8 x i16> %a, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: abs_i16:
+; CHECK: abs z0.h, p0/m, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.abs.nxv8i16(<vscale x 8 x i16> %a,
+                                                               <vscale x 8 x i1> %pg,
+                                                               <vscale x 8 x i16> %b)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @abs_i32(<vscale x 4 x i32> %a, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: abs_i32:
+; CHECK: abs z0.s, p0/m, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.abs.nxv4i32(<vscale x 4 x i32> %a,
+                                                               <vscale x 4 x i1> %pg,
+                                                               <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @abs_i64(<vscale x 2 x i64> %a, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: abs_i64:
+; CHECK: abs z0.d, p0/m, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.abs.nxv2i64(<vscale x 2 x i64> %a,
+                                                               <vscale x 2 x i1> %pg,
+                                                               <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i64> %out
+}
+
+;
+; NEG
+;
+
+define <vscale x 16 x i8> @neg_i8(<vscale x 16 x i8> %a, <vscale x 16 x i1> %pg, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: neg_i8:
+; CHECK: neg z0.b, p0/m, z1.b
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.neg.nxv16i8(<vscale x 16 x i8> %a,
+                                                               <vscale x 16 x i1> %pg,
+                                                               <vscale x 16 x i8> %b)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @neg_i16(<vscale x 8 x i16> %a, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: neg_i16:
+; CHECK: neg z0.h, p0/m, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.neg.nxv8i16(<vscale x 8 x i16> %a,
+                                                               <vscale x 8 x i1> %pg,
+                                                               <vscale x 8 x i16> %b)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @neg_i32(<vscale x 4 x i32> %a, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: neg_i32:
+; CHECK: neg z0.s, p0/m, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.neg.nxv4i32(<vscale x 4 x i32> %a,
+                                                               <vscale x 4 x i1> %pg,
+                                                               <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @neg_i64(<vscale x 2 x i64> %a, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: neg_i64:
+; CHECK: neg z0.d, p0/m, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.neg.nxv2i64(<vscale x 2 x i64> %a,
+                                                               <vscale x 2 x i1> %pg,
+                                                               <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i64> %out
+}
+
+; SDOT
+
+define <vscale x 4 x i32> @sdot_i32(<vscale x 4 x i32> %a, <vscale x 16 x i8> %b, <vscale x 16 x i8> %c) {
+; CHECK-LABEL: sdot_i32:
+; CHECK: sdot z0.s, z1.b, z2.b
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.sdot.nxv4i32(<vscale x 4 x i32> %a,
+                                                                <vscale x 16 x i8> %b,
+                                                                <vscale x 16 x i8> %c)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @sdot_i64(<vscale x 2 x i64> %a, <vscale x 8 x i16> %b, <vscale x 8 x i16> %c) {
+; CHECK-LABEL: sdot_i64:
+; CHECK: sdot z0.d, z1.h, z2.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.sdot.nxv2i64(<vscale x 2 x i64> %a,
+                                                                <vscale x 8 x i16> %b,
+                                                                <vscale x 8 x i16> %c)
+  ret <vscale x 2 x i64> %out
+}
+
+; SDOT (Indexed)
+
+define <vscale x 4 x i32> @sdot_lane_i32(<vscale x 4 x i32> %a, <vscale x 16 x i8> %b, <vscale x 16 x i8> %c) {
+; CHECK-LABEL: sdot_lane_i32:
+; CHECK: sdot z0.s, z1.b, z2.b[2]
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.sdot.lane.nxv4i32(<vscale x 4 x i32> %a,
+                                                                     <vscale x 16 x i8> %b,
+                                                                     <vscale x 16 x i8> %c,
+                                                                     i32 2)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @sdot_lane_i64(<vscale x 2 x i64> %a, <vscale x 8 x i16> %b, <vscale x 8 x i16> %c) {
+; CHECK-LABEL: sdot_lane_i64:
+; CHECK: sdot z0.d, z1.h, z2.h[1]
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.sdot.lane.nxv2i64(<vscale x 2 x i64> %a,
+                                                                     <vscale x 8 x i16> %b,
+                                                                     <vscale x 8 x i16> %c,
+                                                                     i32 1)
+  ret <vscale x 2 x i64> %out
+}
+
+; UDOT
+
+define <vscale x 4 x i32> @udot_i32(<vscale x 4 x i32> %a, <vscale x 16 x i8> %b, <vscale x 16 x i8> %c) {
+; CHECK-LABEL: udot_i32:
+; CHECK: udot z0.s, z1.b, z2.b
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.udot.nxv4i32(<vscale x 4 x i32> %a,
+                                                                <vscale x 16 x i8> %b,
+                                                                <vscale x 16 x i8> %c)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @udot_i64(<vscale x 2 x i64> %a, <vscale x 8 x i16> %b, <vscale x 8 x i16> %c) {
+; CHECK-LABEL: udot_i64:
+; CHECK: udot z0.d, z1.h, z2.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.udot.nxv2i64(<vscale x 2 x i64> %a,
+                                                                <vscale x 8 x i16> %b,
+                                                                <vscale x 8 x i16> %c)
+  ret <vscale x 2 x i64> %out
+}
+
+; UDOT (Indexed)
+
+define <vscale x 4 x i32> @udot_lane_i32(<vscale x 4 x i32> %a, <vscale x 16 x i8> %b, <vscale x 16 x i8> %c) {
+; CHECK-LABEL: udot_lane_i32:
+; CHECK: udot z0.s, z1.b, z2.b[2]
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.udot.lane.nxv4i32(<vscale x 4 x i32> %a,
+                                                                     <vscale x 16 x i8> %b,
+                                                                     <vscale x 16 x i8> %c,
+                                                                     i32 2)
+  ret <vscale x 4 x i32> %out
+}
+
+declare <vscale x 16 x i8> @llvm.aarch64.sve.abs.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i1>, <vscale x 16 x i8>)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.abs.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i1>, <vscale x 8 x i16>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.abs.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i1>, <vscale x 4 x i32>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.abs.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i1>, <vscale x 2 x i64>)
+
+declare <vscale x 16 x i8> @llvm.aarch64.sve.neg.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i1>, <vscale x 16 x i8>)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.neg.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i1>, <vscale x 8 x i16>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.neg.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i1>, <vscale x 4 x i32>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.neg.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i1>, <vscale x 2 x i64>)
+
+declare <vscale x 4 x i32> @llvm.aarch64.sve.sdot.nxv4i32(<vscale x 4 x i32>, <vscale x 16 x i8>, <vscale x 16 x i8>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.sdot.nxv2i64(<vscale x 2 x i64>, <vscale x 8 x i16>, <vscale x 8 x i16>)
+
+declare <vscale x 4 x i32> @llvm.aarch64.sve.sdot.lane.nxv4i32(<vscale x 4 x i32>, <vscale x 16 x i8>, <vscale x 16 x i8>, i32)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.sdot.lane.nxv2i64(<vscale x 2 x i64>, <vscale x 8 x i16>, <vscale x 8 x i16>, i32)
+
+declare <vscale x 4 x i32> @llvm.aarch64.sve.udot.nxv4i32(<vscale x 4 x i32>, <vscale x 16 x i8>, <vscale x 16 x i8>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.udot.nxv2i64(<vscale x 2 x i64>, <vscale x 8 x i16>, <vscale x 8 x i16>)
+
+declare <vscale x 4 x i32> @llvm.aarch64.sve.udot.lane.nxv4i32(<vscale x 4 x i32>, <vscale x 16 x i8>, <vscale x 16 x i8>, i32)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.udot.lane.nxv2i64(<vscale x 2 x i64>, <vscale x 8 x i16>, <vscale x 8 x i16>, i32)
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-int-compares-with-imm.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-int-compares-with-imm.ll
new file mode 100644
index 000000000..9e3190aaa
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-int-compares-with-imm.ll
@@ -0,0 +1,1353 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;;                             Signed Comparisons                             ;;
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;
+; CMPEQ
+;
+
+define <vscale x 16 x i1> @ir_cmpeq_b(<vscale x 16 x i8> %a) {
+; CHECK-LABEL: ir_cmpeq_b
+; CHECK: cmpeq p0.b, p0/z, z0.b, #4
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 16 x i8> undef, i8 4, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %out = icmp eq <vscale x 16 x i8> %a, %splat
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 16 x i1> @int_cmpeq_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a) {
+; CHECK-LABEL: int_cmpeq_b
+; CHECK: cmpeq p0.b, p0/z, z0.b, #4
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 16 x i8> undef, i8 4, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmpeq.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                 <vscale x 16 x i8> %a,
+                                                                 <vscale x 16 x i8> %splat)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 16 x i1> @wide_cmpeq_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a) {
+; CHECK-LABEL: wide_cmpeq_b
+; CHECK: cmpeq p0.b, p0/z, z0.b, #4
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 4, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmpeq.wide.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                      <vscale x 16 x i8> %a,
+                                                                      <vscale x 2 x i64> %splat)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @ir_cmpeq_h(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: ir_cmpeq_h
+; CHECK: cmpeq p0.h, p0/z, z0.h, #-16
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 8 x i16> undef, i16 -16, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %out = icmp eq <vscale x 8 x i16> %a, %splat
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 8 x i1> @int_cmpeq_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a) {
+; CHECK-LABEL: int_cmpeq_h
+; CHECK: cmpeq p0.h, p0/z, z0.h, #-16
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 8 x i16> undef, i16 -16, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmpeq.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                <vscale x 8 x i16> %a,
+                                                                <vscale x 8 x i16> %splat)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 8 x i1> @wide_cmpeq_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a) {
+; CHECK-LABEL: wide_cmpeq_h
+; CHECK: cmpeq p0.h, p0/z, z0.h, #-16
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 -16, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmpeq.wide.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                     <vscale x 8 x i16> %a,
+                                                                     <vscale x 2 x i64> %splat)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @ir_cmpeq_s(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: ir_cmpeq_s
+; CHECK: cmpeq p0.s, p0/z, z0.s, #15
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 4 x i32> undef, i32 15, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %out = icmp eq <vscale x 4 x i32> %a, %splat
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 4 x i1> @int_cmpeq_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a) {
+; CHECK-LABEL: int_cmpeq_s
+; CHECK: cmpeq p0.s, p0/z, z0.s, #15
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 4 x i32> undef, i32 15, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmpeq.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                <vscale x 4 x i32> %a,
+                                                                <vscale x 4 x i32> %splat)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 4 x i1> @wide_cmpeq_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a) {
+; CHECK-LABEL: wide_cmpeq_s
+; CHECK: cmpeq p0.s, p0/z, z0.s, #15
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 15, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmpeq.wide.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                     <vscale x 4 x i32> %a,
+                                                                     <vscale x 2 x i64> %splat)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 2 x i1> @ir_cmpeq_d(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: ir_cmpeq_d
+; CHECK: cmpeq p0.d, p0/z, z0.d, #0
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 0, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = icmp eq <vscale x 2 x i64> %a, %splat
+  ret <vscale x 2 x i1> %out
+}
+
+define <vscale x 2 x i1> @int_cmpeq_d(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a) {
+; CHECK-LABEL: int_cmpeq_d
+; CHECK: cmpeq p0.d, p0/z, z0.d, #0
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 0, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.cmpeq.nxv2i64(<vscale x 2 x i1> %pg,
+                                                                <vscale x 2 x i64> %a,
+                                                                <vscale x 2 x i64> %splat)
+  ret <vscale x 2 x i1> %out
+}
+
+;
+; CMPGE
+;
+
+define <vscale x 16 x i1> @ir_cmpge_b(<vscale x 16 x i8> %a) {
+; CHECK-LABEL: ir_cmpge_b
+; CHECK: cmpge p0.b, p0/z, z0.b, #4
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 16 x i8> undef, i8 4, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %out = icmp sge <vscale x 16 x i8> %a, %splat
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 16 x i1> @int_cmpge_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a) {
+; CHECK-LABEL: int_cmpge_b
+; CHECK: cmpge p0.b, p0/z, z0.b, #4
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 16 x i8> undef, i8 4, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmpge.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                 <vscale x 16 x i8> %a,
+                                                                 <vscale x 16 x i8> %splat)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 16 x i1> @wide_cmpge_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a) {
+; CHECK-LABEL: wide_cmpge_b
+; CHECK: cmpge p0.b, p0/z, z0.b, #4
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 4, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmpge.wide.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                      <vscale x 16 x i8> %a,
+                                                                      <vscale x 2 x i64> %splat)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @ir_cmpge_h(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: ir_cmpge_h
+; CHECK: cmpge p0.h, p0/z, z0.h, #-16
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 8 x i16> undef, i16 -16, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %out = icmp sge <vscale x 8 x i16> %a, %splat
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 8 x i1> @int_cmpge_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a) {
+; CHECK-LABEL: int_cmpge_h
+; CHECK: cmpge p0.h, p0/z, z0.h, #-16
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 8 x i16> undef, i16 -16, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmpge.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                <vscale x 8 x i16> %a,
+                                                                <vscale x 8 x i16> %splat)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 8 x i1> @wide_cmpge_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a) {
+; CHECK-LABEL: wide_cmpge_h
+; CHECK: cmpge p0.h, p0/z, z0.h, #-16
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 -16, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmpge.wide.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                     <vscale x 8 x i16> %a,
+                                                                     <vscale x 2 x i64> %splat)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @ir_cmpge_s(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: ir_cmpge_s
+; CHECK: cmpge p0.s, p0/z, z0.s, #15
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 4 x i32> undef, i32 15, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %out = icmp sge <vscale x 4 x i32> %a, %splat
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 4 x i1> @int_cmpge_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a) {
+; CHECK-LABEL: int_cmpge_s
+; CHECK: cmpge p0.s, p0/z, z0.s, #15
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 4 x i32> undef, i32 15, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmpge.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                <vscale x 4 x i32> %a,
+                                                                <vscale x 4 x i32> %splat)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 4 x i1> @wide_cmpge_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a) {
+; CHECK-LABEL: wide_cmpge_s
+; CHECK: cmpge p0.s, p0/z, z0.s, #15
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 15, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmpge.wide.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                     <vscale x 4 x i32> %a,
+                                                                     <vscale x 2 x i64> %splat)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 2 x i1> @ir_cmpge_d(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: ir_cmpge_d
+; CHECK: cmpge p0.d, p0/z, z0.d, #0
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 0, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = icmp sge <vscale x 2 x i64> %a, %splat
+  ret <vscale x 2 x i1> %out
+}
+
+define <vscale x 2 x i1> @int_cmpge_d(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a) {
+; CHECK-LABEL: int_cmpge_d
+; CHECK: cmpge p0.d, p0/z, z0.d, #0
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 0, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.cmpge.nxv2i64(<vscale x 2 x i1> %pg,
+                                                                <vscale x 2 x i64> %a,
+                                                                <vscale x 2 x i64> %splat)
+  ret <vscale x 2 x i1> %out
+}
+
+;
+; CMPGT
+;
+
+define <vscale x 16 x i1> @ir_cmpgt_b(<vscale x 16 x i8> %a) {
+; CHECK-LABEL: ir_cmpgt_b
+; CHECK: cmpgt p0.b, p0/z, z0.b, #4
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 16 x i8> undef, i8 4, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %out = icmp sgt <vscale x 16 x i8> %a, %splat
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 16 x i1> @int_cmpgt_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a) {
+; CHECK-LABEL: int_cmpgt_b
+; CHECK: cmpgt p0.b, p0/z, z0.b, #4
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 16 x i8> undef, i8 4, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmpgt.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                 <vscale x 16 x i8> %a,
+                                                                 <vscale x 16 x i8> %splat)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 16 x i1> @wide_cmpgt_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a) {
+; CHECK-LABEL: wide_cmpgt_b
+; CHECK: cmpgt p0.b, p0/z, z0.b, #4
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 4, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmpgt.wide.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                      <vscale x 16 x i8> %a,
+                                                                      <vscale x 2 x i64> %splat)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @ir_cmpgt_h(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: ir_cmpgt_h
+; CHECK: cmpgt p0.h, p0/z, z0.h, #-16
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 8 x i16> undef, i16 -16, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %out = icmp sgt <vscale x 8 x i16> %a, %splat
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 8 x i1> @int_cmpgt_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a) {
+; CHECK-LABEL: int_cmpgt_h
+; CHECK: cmpgt p0.h, p0/z, z0.h, #-16
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 8 x i16> undef, i16 -16, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmpgt.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                <vscale x 8 x i16> %a,
+                                                                <vscale x 8 x i16> %splat)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 8 x i1> @wide_cmpgt_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a) {
+; CHECK-LABEL: wide_cmpgt_h
+; CHECK: cmpgt p0.h, p0/z, z0.h, #-16
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 -16, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmpgt.wide.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                     <vscale x 8 x i16> %a,
+                                                                     <vscale x 2 x i64> %splat)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @ir_cmpgt_s(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: ir_cmpgt_s
+; CHECK: cmpgt p0.s, p0/z, z0.s, #15
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 4 x i32> undef, i32 15, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %out = icmp sgt <vscale x 4 x i32> %a, %splat
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 4 x i1> @int_cmpgt_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a) {
+; CHECK-LABEL: int_cmpgt_s
+; CHECK: cmpgt p0.s, p0/z, z0.s, #15
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 4 x i32> undef, i32 15, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmpgt.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                <vscale x 4 x i32> %a,
+                                                                <vscale x 4 x i32> %splat)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 4 x i1> @wide_cmpgt_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a) {
+; CHECK-LABEL: wide_cmpgt_s
+; CHECK: cmpgt p0.s, p0/z, z0.s, #15
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 15, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmpgt.wide.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                     <vscale x 4 x i32> %a,
+                                                                     <vscale x 2 x i64> %splat)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 2 x i1> @ir_cmpgt_d(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: ir_cmpgt_d
+; CHECK: cmpgt p0.d, p0/z, z0.d, #0
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 0, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = icmp sgt <vscale x 2 x i64> %a, %splat
+  ret <vscale x 2 x i1> %out
+}
+
+define <vscale x 2 x i1> @int_cmpgt_d(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a) {
+; CHECK-LABEL: int_cmpgt_d
+; CHECK: cmpgt p0.d, p0/z, z0.d, #0
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 0, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.cmpgt.nxv2i64(<vscale x 2 x i1> %pg,
+                                                                <vscale x 2 x i64> %a,
+                                                                <vscale x 2 x i64> %splat)
+  ret <vscale x 2 x i1> %out
+}
+
+;
+; CMPLE
+;
+
+define <vscale x 16 x i1> @ir_cmple_b(<vscale x 16 x i8> %a) {
+; CHECK-LABEL: ir_cmple_b
+; CHECK: cmple p0.b, p0/z, z0.b, #4
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 16 x i8> undef, i8 4, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %out = icmp sle <vscale x 16 x i8> %a, %splat
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 16 x i1> @int_cmple_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a) {
+; CHECK-LABEL: int_cmple_b
+; CHECK: cmple p0.b, p0/z, z0.b, #4
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 16 x i8> undef, i8 4, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmpge.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                 <vscale x 16 x i8> %splat,
+                                                                 <vscale x 16 x i8> %a)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 16 x i1> @wide_cmple_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a) {
+; CHECK-LABEL: wide_cmple_b
+; CHECK: cmple p0.b, p0/z, z0.b, #4
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 4, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmple.wide.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                      <vscale x 16 x i8> %a,
+                                                                      <vscale x 2 x i64> %splat)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @ir_cmple_h(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: ir_cmple_h
+; CHECK: cmple p0.h, p0/z, z0.h, #-16
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 8 x i16> undef, i16 -16, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %out = icmp sle <vscale x 8 x i16> %a, %splat
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 8 x i1> @int_cmple_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a) {
+; CHECK-LABEL: int_cmple_h
+; CHECK: cmple p0.h, p0/z, z0.h, #-16
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 8 x i16> undef, i16 -16, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmpge.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                <vscale x 8 x i16> %splat,
+                                                                <vscale x 8 x i16> %a)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 8 x i1> @wide_cmple_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a) {
+; CHECK-LABEL: wide_cmple_h
+; CHECK: cmple p0.h, p0/z, z0.h, #-16
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 -16, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmple.wide.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                     <vscale x 8 x i16> %a,
+                                                                     <vscale x 2 x i64> %splat)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @ir_cmple_s(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: ir_cmple_s
+; CHECK: cmple p0.s, p0/z, z0.s, #15
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 4 x i32> undef, i32 15, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %out = icmp sle <vscale x 4 x i32> %a, %splat
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 4 x i1> @int_cmple_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a) {
+; CHECK-LABEL: int_cmple_s
+; CHECK: cmple p0.s, p0/z, z0.s, #15
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 4 x i32> undef, i32 15, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmpge.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                <vscale x 4 x i32> %splat,
+                                                                <vscale x 4 x i32> %a)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 4 x i1> @wide_cmple_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a) {
+; CHECK-LABEL: wide_cmple_s
+; CHECK: cmple p0.s, p0/z, z0.s, #15
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 15, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmple.wide.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                     <vscale x 4 x i32> %a,
+                                                                     <vscale x 2 x i64> %splat)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 2 x i1> @ir_cmple_d(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: ir_cmple_d
+; CHECK: cmple p0.d, p0/z, z0.d, #0
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 0, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = icmp sle <vscale x 2 x i64> %a, %splat
+  ret <vscale x 2 x i1> %out
+}
+
+define <vscale x 2 x i1> @int_cmple_d(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a) {
+; CHECK-LABEL: int_cmple_d
+; CHECK: cmple p0.d, p0/z, z0.d, #0
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 0, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.cmpge.nxv2i64(<vscale x 2 x i1> %pg,
+                                                                <vscale x 2 x i64> %splat,
+                                                                <vscale x 2 x i64> %a)
+  ret <vscale x 2 x i1> %out
+}
+
+;
+; CMPLT
+;
+
+define <vscale x 16 x i1> @ir_cmplt_b(<vscale x 16 x i8> %a) {
+; CHECK-LABEL: ir_cmplt_b
+; CHECK: cmplt p0.b, p0/z, z0.b, #4
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 16 x i8> undef, i8 4, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %out = icmp slt <vscale x 16 x i8> %a, %splat
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 16 x i1> @int_cmplt_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a) {
+; CHECK-LABEL: int_cmplt_b
+; CHECK: cmplt p0.b, p0/z, z0.b, #4
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 16 x i8> undef, i8 4, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmpgt.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                 <vscale x 16 x i8> %splat,
+                                                                 <vscale x 16 x i8> %a)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 16 x i1> @wide_cmplt_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a) {
+; CHECK-LABEL: wide_cmplt_b
+; CHECK: cmplt p0.b, p0/z, z0.b, #4
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 4, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmplt.wide.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                      <vscale x 16 x i8> %a,
+                                                                      <vscale x 2 x i64> %splat)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @ir_cmplt_h(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: ir_cmplt_h
+; CHECK: cmplt p0.h, p0/z, z0.h, #-16
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 8 x i16> undef, i16 -16, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %out = icmp slt <vscale x 8 x i16> %a, %splat
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 8 x i1> @int_cmplt_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a) {
+; CHECK-LABEL: int_cmplt_h
+; CHECK: cmplt p0.h, p0/z, z0.h, #-16
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 8 x i16> undef, i16 -16, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmpgt.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                <vscale x 8 x i16> %splat,
+                                                                <vscale x 8 x i16> %a)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 8 x i1> @wide_cmplt_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a) {
+; CHECK-LABEL: wide_cmplt_h
+; CHECK: cmplt p0.h, p0/z, z0.h, #-16
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 -16, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmplt.wide.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                     <vscale x 8 x i16> %a,
+                                                                     <vscale x 2 x i64> %splat)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @ir_cmplt_s(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: ir_cmplt_s
+; CHECK: cmplt p0.s, p0/z, z0.s, #15
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 4 x i32> undef, i32 15, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %out = icmp slt <vscale x 4 x i32> %a, %splat
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 4 x i1> @int_cmplt_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a) {
+; CHECK-LABEL: int_cmplt_s
+; CHECK: cmplt p0.s, p0/z, z0.s, #15
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 4 x i32> undef, i32 15, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmpgt.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                <vscale x 4 x i32> %splat,
+                                                                <vscale x 4 x i32> %a)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 4 x i1> @wide_cmplt_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a) {
+; CHECK-LABEL: wide_cmplt_s
+; CHECK: cmplt p0.s, p0/z, z0.s, #15
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 15, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmplt.wide.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                     <vscale x 4 x i32> %a,
+                                                                     <vscale x 2 x i64> %splat)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 2 x i1> @ir_cmplt_d(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: ir_cmplt_d
+; CHECK: cmplt p0.d, p0/z, z0.d, #0
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 0, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = icmp slt <vscale x 2 x i64> %a, %splat
+  ret <vscale x 2 x i1> %out
+}
+
+define <vscale x 2 x i1> @int_cmplt_d(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a) {
+; CHECK-LABEL: int_cmplt_d
+; CHECK: cmplt p0.d, p0/z, z0.d, #0
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 0, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.cmpgt.nxv2i64(<vscale x 2 x i1> %pg,
+                                                                <vscale x 2 x i64> %splat,
+                                                                <vscale x 2 x i64> %a)
+  ret <vscale x 2 x i1> %out
+}
+
+;
+; CMPNE
+;
+
+define <vscale x 16 x i1> @ir_cmpne_b(<vscale x 16 x i8> %a) {
+; CHECK-LABEL: ir_cmpne_b
+; CHECK: cmpne p0.b, p0/z, z0.b, #4
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 16 x i8> undef, i8 4, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %out = icmp ne <vscale x 16 x i8> %a, %splat
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 16 x i1> @int_cmpne_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a) {
+; CHECK-LABEL: int_cmpne_b
+; CHECK: cmpne p0.b, p0/z, z0.b, #4
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 16 x i8> undef, i8 4, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmpne.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                 <vscale x 16 x i8> %a,
+                                                                 <vscale x 16 x i8> %splat)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 16 x i1> @wide_cmpne_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a) {
+; CHECK-LABEL: wide_cmpne_b
+; CHECK: cmpne p0.b, p0/z, z0.b, #4
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 4, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmpne.wide.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                      <vscale x 16 x i8> %a,
+                                                                      <vscale x 2 x i64> %splat)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @ir_cmpne_h(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: ir_cmpne_h
+; CHECK: cmpne p0.h, p0/z, z0.h, #-16
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 8 x i16> undef, i16 -16, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %out = icmp ne <vscale x 8 x i16> %a, %splat
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 8 x i1> @int_cmpne_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a) {
+; CHECK-LABEL: int_cmpne_h
+; CHECK: cmpne p0.h, p0/z, z0.h, #-16
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 8 x i16> undef, i16 -16, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmpne.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                <vscale x 8 x i16> %a,
+                                                                <vscale x 8 x i16> %splat)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 8 x i1> @wide_cmpne_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a) {
+; CHECK-LABEL: wide_cmpne_h
+; CHECK: cmpne p0.h, p0/z, z0.h, #-16
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 -16, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmpne.wide.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                     <vscale x 8 x i16> %a,
+                                                                     <vscale x 2 x i64> %splat)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @ir_cmpne_s(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: ir_cmpne_s
+; CHECK: cmpne p0.s, p0/z, z0.s, #15
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 4 x i32> undef, i32 15, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %out = icmp ne <vscale x 4 x i32> %a, %splat
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 4 x i1> @int_cmpne_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a) {
+; CHECK-LABEL: int_cmpne_s
+; CHECK: cmpne p0.s, p0/z, z0.s, #15
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 4 x i32> undef, i32 15, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmpne.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                <vscale x 4 x i32> %a,
+                                                                <vscale x 4 x i32> %splat)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 4 x i1> @wide_cmpne_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a) {
+; CHECK-LABEL: wide_cmpne_s
+; CHECK: cmpne p0.s, p0/z, z0.s, #15
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 15, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmpne.wide.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                     <vscale x 4 x i32> %a,
+                                                                     <vscale x 2 x i64> %splat)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 2 x i1> @ir_cmpne_d(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: ir_cmpne_d
+; CHECK: cmpne p0.d, p0/z, z0.d, #0
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 0, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = icmp ne <vscale x 2 x i64> %a, %splat
+  ret <vscale x 2 x i1> %out
+}
+
+define <vscale x 2 x i1> @int_cmpne_d(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a) {
+; CHECK-LABEL: int_cmpne_d
+; CHECK: cmpne p0.d, p0/z, z0.d, #0
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 0, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.cmpne.nxv2i64(<vscale x 2 x i1> %pg,
+                                                                <vscale x 2 x i64> %a,
+                                                                <vscale x 2 x i64> %splat)
+  ret <vscale x 2 x i1> %out
+}
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;;                            Unsigned Comparisons                            ;;
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;
+; CMPHI
+;
+
+define <vscale x 16 x i1> @ir_cmphi_b(<vscale x 16 x i8> %a) {
+; CHECK-LABEL: ir_cmphi_b
+; CHECK: cmphi p0.b, p0/z, z0.b, #4
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 16 x i8> undef, i8 4, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %out = icmp ugt <vscale x 16 x i8> %a, %splat
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 16 x i1> @int_cmphi_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a) {
+; CHECK-LABEL: int_cmphi_b
+; CHECK: cmphi p0.b, p0/z, z0.b, #4
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 16 x i8> undef, i8 4, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmphi.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                 <vscale x 16 x i8> %a,
+                                                                 <vscale x 16 x i8> %splat)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 16 x i1> @wide_cmphi_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a) {
+; CHECK-LABEL: wide_cmphi_b
+; CHECK: cmphi p0.b, p0/z, z0.b, #4
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 4, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmphi.wide.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                      <vscale x 16 x i8> %a,
+                                                                      <vscale x 2 x i64> %splat)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @ir_cmphi_h(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: ir_cmphi_h
+; CHECK: cmphi p0.h, p0/z, z0.h, #0
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 8 x i16> undef, i16 0, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %out = icmp ugt <vscale x 8 x i16> %a, %splat
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 8 x i1> @int_cmphi_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a) {
+; CHECK-LABEL: int_cmphi_h
+; CHECK: cmphi p0.h, p0/z, z0.h, #0
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 8 x i16> undef, i16 0, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmphi.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                <vscale x 8 x i16> %a,
+                                                                <vscale x 8 x i16> %splat)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 8 x i1> @wide_cmphi_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a) {
+; CHECK-LABEL: wide_cmphi_h
+; CHECK: cmphi p0.h, p0/z, z0.h, #0
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 0, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmphi.wide.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                     <vscale x 8 x i16> %a,
+                                                                     <vscale x 2 x i64> %splat)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @ir_cmphi_s(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: ir_cmphi_s
+; CHECK: cmphi p0.s, p0/z, z0.s, #68
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 4 x i32> undef, i32 68, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %out = icmp ugt <vscale x 4 x i32> %a, %splat
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 4 x i1> @int_cmphi_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a) {
+; CHECK-LABEL: int_cmphi_s
+; CHECK: cmphi p0.s, p0/z, z0.s, #68
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 4 x i32> undef, i32 68, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmphi.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                <vscale x 4 x i32> %a,
+                                                                <vscale x 4 x i32> %splat)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 4 x i1> @wide_cmphi_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a) {
+; CHECK-LABEL: wide_cmphi_s
+; CHECK: cmphi p0.s, p0/z, z0.s, #68
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 68, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmphi.wide.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                     <vscale x 4 x i32> %a,
+                                                                     <vscale x 2 x i64> %splat)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 2 x i1> @ir_cmphi_d(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: ir_cmphi_d
+; CHECK: cmphi p0.d, p0/z, z0.d, #127
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 127, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = icmp ugt <vscale x 2 x i64> %a, %splat
+  ret <vscale x 2 x i1> %out
+}
+
+define <vscale x 2 x i1> @int_cmphi_d(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a) {
+; CHECK-LABEL: int_cmphi_d
+; CHECK: cmphi p0.d, p0/z, z0.d, #127
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 127, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.cmphi.nxv2i64(<vscale x 2 x i1> %pg,
+                                                                <vscale x 2 x i64> %a,
+                                                                <vscale x 2 x i64> %splat)
+  ret <vscale x 2 x i1> %out
+}
+
+;
+; CMPHS
+;
+
+define <vscale x 16 x i1> @ir_cmphs_b(<vscale x 16 x i8> %a) {
+; CHECK-LABEL: ir_cmphs_b
+; CHECK: cmphs p0.b, p0/z, z0.b, #4
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 16 x i8> undef, i8 4, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %out = icmp uge <vscale x 16 x i8> %a, %splat
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 16 x i1> @int_cmphs_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a) {
+; CHECK-LABEL: int_cmphs_b
+; CHECK: cmphs p0.b, p0/z, z0.b, #4
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 16 x i8> undef, i8 4, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmphs.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                 <vscale x 16 x i8> %a,
+                                                                 <vscale x 16 x i8> %splat)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 16 x i1> @wide_cmphs_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a) {
+; CHECK-LABEL: wide_cmphs_b
+; CHECK: cmphs p0.b, p0/z, z0.b, #4
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 4, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmphs.wide.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                      <vscale x 16 x i8> %a,
+                                                                      <vscale x 2 x i64> %splat)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @ir_cmphs_h(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: ir_cmphs_h
+; CHECK: cmphs p0.h, p0/z, z0.h, #0
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 8 x i16> undef, i16 0, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %out = icmp uge <vscale x 8 x i16> %a, %splat
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 8 x i1> @int_cmphs_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a) {
+; CHECK-LABEL: int_cmphs_h
+; CHECK: cmphs p0.h, p0/z, z0.h, #0
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 8 x i16> undef, i16 0, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmphs.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                <vscale x 8 x i16> %a,
+                                                                <vscale x 8 x i16> %splat)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 8 x i1> @wide_cmphs_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a) {
+; CHECK-LABEL: wide_cmphs_h
+; CHECK: cmphs p0.h, p0/z, z0.h, #0
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 0, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmphs.wide.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                     <vscale x 8 x i16> %a,
+                                                                     <vscale x 2 x i64> %splat)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @ir_cmphs_s(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: ir_cmphs_s
+; CHECK: cmphs p0.s, p0/z, z0.s, #68
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 4 x i32> undef, i32 68, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %out = icmp uge <vscale x 4 x i32> %a, %splat
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 4 x i1> @int_cmphs_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a) {
+; CHECK-LABEL: int_cmphs_s
+; CHECK: cmphs p0.s, p0/z, z0.s, #68
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 4 x i32> undef, i32 68, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmphs.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                <vscale x 4 x i32> %a,
+                                                                <vscale x 4 x i32> %splat)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 4 x i1> @wide_cmphs_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a) {
+; CHECK-LABEL: wide_cmphs_s
+; CHECK: cmphs p0.s, p0/z, z0.s, #68
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 68, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmphs.wide.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                     <vscale x 4 x i32> %a,
+                                                                     <vscale x 2 x i64> %splat)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 2 x i1> @ir_cmphs_d(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: ir_cmphs_d
+; CHECK: cmphs p0.d, p0/z, z0.d, #127
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 127, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = icmp uge <vscale x 2 x i64> %a, %splat
+  ret <vscale x 2 x i1> %out
+}
+
+define <vscale x 2 x i1> @int_cmphs_d(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a) {
+; CHECK-LABEL: int_cmphs_d
+; CHECK: cmphs p0.d, p0/z, z0.d, #127
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 127, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.cmphs.nxv2i64(<vscale x 2 x i1> %pg,
+                                                                <vscale x 2 x i64> %a,
+                                                                <vscale x 2 x i64> %splat)
+  ret <vscale x 2 x i1> %out
+}
+
+;
+; CMPLO
+;
+
+define <vscale x 16 x i1> @ir_cmplo_b(<vscale x 16 x i8> %a) {
+; CHECK-LABEL: ir_cmplo_b
+; CHECK: cmplo p0.b, p0/z, z0.b, #4
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 16 x i8> undef, i8 4, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %out = icmp ult <vscale x 16 x i8> %a, %splat
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 16 x i1> @int_cmplo_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a) {
+; CHECK-LABEL: int_cmplo_b
+; CHECK: cmplo p0.b, p0/z, z0.b, #4
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 16 x i8> undef, i8 4, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmphi.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                 <vscale x 16 x i8> %splat,
+                                                                 <vscale x 16 x i8> %a)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 16 x i1> @wide_cmplo_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a) {
+; CHECK-LABEL: wide_cmplo_b
+; CHECK: cmplo p0.b, p0/z, z0.b, #4
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 4, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmplo.wide.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                      <vscale x 16 x i8> %a,
+                                                                      <vscale x 2 x i64> %splat)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @ir_cmplo_h(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: ir_cmplo_h
+; CHECK: cmplo p0.h, p0/z, z0.h, #0
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 8 x i16> undef, i16 0, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %out = icmp ult <vscale x 8 x i16> %a, %splat
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 8 x i1> @int_cmplo_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a) {
+; CHECK-LABEL: int_cmplo_h
+; CHECK: cmplo p0.h, p0/z, z0.h, #0
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 8 x i16> undef, i16 0, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmphi.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                <vscale x 8 x i16> %splat,
+                                                                <vscale x 8 x i16> %a)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 8 x i1> @wide_cmplo_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a) {
+; CHECK-LABEL: wide_cmplo_h
+; CHECK: cmplo p0.h, p0/z, z0.h, #0
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 0, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmplo.wide.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                     <vscale x 8 x i16> %a,
+                                                                     <vscale x 2 x i64> %splat)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @ir_cmplo_s(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: ir_cmplo_s
+; CHECK: cmplo p0.s, p0/z, z0.s, #68
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 4 x i32> undef, i32 68, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %out = icmp ult <vscale x 4 x i32> %a, %splat
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 4 x i1> @int_cmplo_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a) {
+; CHECK-LABEL: int_cmplo_s
+; CHECK: cmplo p0.s, p0/z, z0.s, #68
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 4 x i32> undef, i32 68, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmphi.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                <vscale x 4 x i32> %splat,
+                                                                <vscale x 4 x i32> %a)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 4 x i1> @wide_cmplo_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a) {
+; CHECK-LABEL: wide_cmplo_s
+; CHECK: cmplo p0.s, p0/z, z0.s, #68
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 68, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmplo.wide.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                     <vscale x 4 x i32> %a,
+                                                                     <vscale x 2 x i64> %splat)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 2 x i1> @ir_cmplo_d(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: ir_cmplo_d
+; CHECK: cmplo p0.d, p0/z, z0.d, #127
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 127, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = icmp ult <vscale x 2 x i64> %a, %splat
+  ret <vscale x 2 x i1> %out
+}
+
+define <vscale x 2 x i1> @int_cmplo_d(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a) {
+; CHECK-LABEL: int_cmplo_d
+; CHECK: cmplo p0.d, p0/z, z0.d, #127
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 127, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.cmphi.nxv2i64(<vscale x 2 x i1> %pg,
+                                                                <vscale x 2 x i64> %splat,
+                                                                <vscale x 2 x i64> %a)
+  ret <vscale x 2 x i1> %out
+}
+
+;
+; CMPLS
+;
+
+define <vscale x 16 x i1> @ir_cmpls_b(<vscale x 16 x i8> %a) {
+; CHECK-LABEL: ir_cmpls_b
+; CHECK: cmpls p0.b, p0/z, z0.b, #4
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 16 x i8> undef, i8 4, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %out = icmp ule <vscale x 16 x i8> %a, %splat
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 16 x i1> @int_cmpls_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a) {
+; CHECK-LABEL: int_cmpls_b
+; CHECK: cmpls p0.b, p0/z, z0.b, #4
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 16 x i8> undef, i8 4, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmphs.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                 <vscale x 16 x i8> %splat,
+                                                                 <vscale x 16 x i8> %a)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 16 x i1> @wide_cmpls_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a) {
+; CHECK-LABEL: wide_cmpls_b
+; CHECK: cmpls p0.b, p0/z, z0.b, #4
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 4, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmpls.wide.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                      <vscale x 16 x i8> %a,
+                                                                      <vscale x 2 x i64> %splat)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @ir_cmpls_h(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: ir_cmpls_h
+; CHECK: cmpls p0.h, p0/z, z0.h, #0
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 8 x i16> undef, i16 0, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %out = icmp ule <vscale x 8 x i16> %a, %splat
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 8 x i1> @int_cmpls_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a) {
+; CHECK-LABEL: int_cmpls_h
+; CHECK: cmpls p0.h, p0/z, z0.h, #0
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 8 x i16> undef, i16 0, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmphs.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                <vscale x 8 x i16> %splat,
+                                                                <vscale x 8 x i16> %a)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 8 x i1> @wide_cmpls_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a) {
+; CHECK-LABEL: wide_cmpls_h
+; CHECK: cmpls p0.h, p0/z, z0.h, #0
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 0, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmpls.wide.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                     <vscale x 8 x i16> %a,
+                                                                     <vscale x 2 x i64> %splat)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @ir_cmpls_s(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: ir_cmpls_s
+; CHECK: cmpls p0.s, p0/z, z0.s, #68
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 4 x i32> undef, i32 68, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %out = icmp ule <vscale x 4 x i32> %a, %splat
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 4 x i1> @int_cmpls_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a) {
+; CHECK-LABEL: int_cmpls_s
+; CHECK: cmpls p0.s, p0/z, z0.s, #68
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 4 x i32> undef, i32 68, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmphs.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                <vscale x 4 x i32> %splat,
+                                                                <vscale x 4 x i32> %a)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 4 x i1> @wide_cmpls_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a) {
+; CHECK-LABEL: wide_cmpls_s
+; CHECK: cmpls p0.s, p0/z, z0.s, #68
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 68, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmpls.wide.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                     <vscale x 4 x i32> %a,
+                                                                     <vscale x 2 x i64> %splat)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 2 x i1> @ir_cmpls_d(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: ir_cmpls_d
+; CHECK: cmpls p0.d, p0/z, z0.d, #127
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 127, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = icmp ule <vscale x 2 x i64> %a, %splat
+  ret <vscale x 2 x i1> %out
+}
+
+define <vscale x 2 x i1> @int_cmpls_d(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a) {
+; CHECK-LABEL: int_cmpls_d
+; CHECK: cmpls p0.d, p0/z, z0.d, #127
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 127, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.cmphs.nxv2i64(<vscale x 2 x i1> %pg,
+                                                                <vscale x 2 x i64> %splat,
+                                                                <vscale x 2 x i64> %a)
+  ret <vscale x 2 x i1> %out
+}
+
+declare <vscale x 16 x i1> @llvm.aarch64.sve.cmpeq.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 16 x i8>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.cmpeq.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 8 x i16>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.cmpeq.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 4 x i32>)
+declare <vscale x 2 x i1> @llvm.aarch64.sve.cmpeq.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>, <vscale x 2 x i64>)
+declare <vscale x 16 x i1> @llvm.aarch64.sve.cmpeq.wide.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 2 x i64>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.cmpeq.wide.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 2 x i64>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.cmpeq.wide.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 2 x i64>)
+
+declare <vscale x 16 x i1> @llvm.aarch64.sve.cmpge.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 16 x i8>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.cmpge.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 8 x i16>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.cmpge.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 4 x i32>)
+declare <vscale x 2 x i1> @llvm.aarch64.sve.cmpge.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>, <vscale x 2 x i64>)
+declare <vscale x 16 x i1> @llvm.aarch64.sve.cmpge.wide.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 2 x i64>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.cmpge.wide.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 2 x i64>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.cmpge.wide.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 2 x i64>)
+
+declare <vscale x 16 x i1> @llvm.aarch64.sve.cmpgt.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 16 x i8>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.cmpgt.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 8 x i16>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.cmpgt.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 4 x i32>)
+declare <vscale x 2 x i1> @llvm.aarch64.sve.cmpgt.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>, <vscale x 2 x i64>)
+declare <vscale x 16 x i1> @llvm.aarch64.sve.cmpgt.wide.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 2 x i64>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.cmpgt.wide.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 2 x i64>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.cmpgt.wide.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 2 x i64>)
+
+declare <vscale x 16 x i1> @llvm.aarch64.sve.cmphi.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 16 x i8>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.cmphi.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 8 x i16>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.cmphi.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 4 x i32>)
+declare <vscale x 2 x i1> @llvm.aarch64.sve.cmphi.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>, <vscale x 2 x i64>)
+declare <vscale x 16 x i1> @llvm.aarch64.sve.cmphi.wide.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 2 x i64>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.cmphi.wide.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 2 x i64>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.cmphi.wide.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 2 x i64>)
+
+declare <vscale x 16 x i1> @llvm.aarch64.sve.cmphs.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 16 x i8>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.cmphs.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 8 x i16>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.cmphs.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 4 x i32>)
+declare <vscale x 2 x i1> @llvm.aarch64.sve.cmphs.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>, <vscale x 2 x i64>)
+declare <vscale x 16 x i1> @llvm.aarch64.sve.cmphs.wide.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 2 x i64>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.cmphs.wide.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 2 x i64>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.cmphs.wide.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 2 x i64>)
+
+declare <vscale x 16 x i1> @llvm.aarch64.sve.cmple.wide.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 2 x i64>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.cmple.wide.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 2 x i64>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.cmple.wide.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 2 x i64>)
+
+declare <vscale x 16 x i1> @llvm.aarch64.sve.cmplo.wide.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 2 x i64>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.cmplo.wide.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 2 x i64>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.cmplo.wide.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 2 x i64>)
+
+declare <vscale x 16 x i1> @llvm.aarch64.sve.cmpls.wide.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 2 x i64>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.cmpls.wide.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 2 x i64>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.cmpls.wide.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 2 x i64>)
+
+declare <vscale x 16 x i1> @llvm.aarch64.sve.cmplt.wide.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 2 x i64>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.cmplt.wide.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 2 x i64>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.cmplt.wide.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 2 x i64>)
+
+declare <vscale x 16 x i1> @llvm.aarch64.sve.cmpne.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 16 x i8>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.cmpne.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 8 x i16>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.cmpne.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 4 x i32>)
+declare <vscale x 2 x i1> @llvm.aarch64.sve.cmpne.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>, <vscale x 2 x i64>)
+declare <vscale x 16 x i1> @llvm.aarch64.sve.cmpne.wide.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 2 x i64>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.cmpne.wide.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 2 x i64>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.cmpne.wide.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 2 x i64>)
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-int-compares.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-int-compares.ll
new file mode 100644
index 000000000..2a4984f6e
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-int-compares.ll
@@ -0,0 +1,645 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+;
+; CMPEQ
+;
+
+define <vscale x 16 x i1> @cmpeq_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: cmpeq_b:
+; CHECK: cmpeq p0.b, p0/z, z0.b, z1.b
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmpeq.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                 <vscale x 16 x i8> %a,
+                                                                 <vscale x 16 x i8> %b)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @cmpeq_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: cmpeq_h:
+; CHECK: cmpeq p0.h, p0/z, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmpeq.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                <vscale x 8 x i16> %a,
+                                                                <vscale x 8 x i16> %b)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @cmpeq_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: cmpeq_s:
+; CHECK: cmpeq p0.s, p0/z, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmpeq.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                <vscale x 4 x i32> %a,
+                                                                <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 2 x i1> @cmpeq_d(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmpeq_d:
+; CHECK: cmpeq p0.d, p0/z, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.cmpeq.nxv2i64(<vscale x 2 x i1> %pg,
+                                                                <vscale x 2 x i64> %a,
+                                                                <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i1> %out
+}
+
+define <vscale x 16 x i1> @cmpeq_wide_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmpeq_wide_b:
+; CHECK: cmpeq p0.b, p0/z, z0.b, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmpeq.wide.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                      <vscale x 16 x i8> %a,
+                                                                      <vscale x 2 x i64> %b)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @cmpeq_wide_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmpeq_wide_h:
+; CHECK: cmpeq p0.h, p0/z, z0.h, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmpeq.wide.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                     <vscale x 8 x i16> %a,
+                                                                     <vscale x 2 x i64> %b)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @cmpeq_wide_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmpeq_wide_s:
+; CHECK: cmpeq p0.s, p0/z, z0.s, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmpeq.wide.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                     <vscale x 4 x i32> %a,
+                                                                     <vscale x 2 x i64> %b)
+  ret <vscale x 4 x i1> %out
+}
+
+;
+; CMPGE
+;
+
+define <vscale x 16 x i1> @cmpge_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: cmpge_b:
+; CHECK: cmpge p0.b, p0/z, z0.b, z1.b
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmpge.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                 <vscale x 16 x i8> %a,
+                                                                 <vscale x 16 x i8> %b)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @cmpge_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: cmpge_h:
+; CHECK: cmpge p0.h, p0/z, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmpge.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                <vscale x 8 x i16> %a,
+                                                                <vscale x 8 x i16> %b)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @cmpge_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: cmpge_s:
+; CHECK: cmpge p0.s, p0/z, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmpge.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                <vscale x 4 x i32> %a,
+                                                                <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 2 x i1> @cmpge_d(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmpge_d:
+; CHECK: cmpge p0.d, p0/z, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.cmpge.nxv2i64(<vscale x 2 x i1> %pg,
+                                                                <vscale x 2 x i64> %a,
+                                                                <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i1> %out
+}
+
+define <vscale x 16 x i1> @cmpge_wide_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmpge_wide_b:
+; CHECK: cmpge p0.b, p0/z, z0.b, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmpge.wide.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                      <vscale x 16 x i8> %a,
+                                                                      <vscale x 2 x i64> %b)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @cmpge_wide_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmpge_wide_h:
+; CHECK: cmpge p0.h, p0/z, z0.h, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmpge.wide.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                     <vscale x 8 x i16> %a,
+                                                                     <vscale x 2 x i64> %b)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @cmpge_wide_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmpge_wide_s:
+; CHECK: cmpge p0.s, p0/z, z0.s, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmpge.wide.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                     <vscale x 4 x i32> %a,
+                                                                     <vscale x 2 x i64> %b)
+  ret <vscale x 4 x i1> %out
+}
+
+;
+; CMPGT
+;
+
+define <vscale x 16 x i1> @cmpgt_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: cmpgt_b:
+; CHECK: cmpgt p0.b, p0/z, z0.b, z1.b
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmpgt.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                 <vscale x 16 x i8> %a,
+                                                                 <vscale x 16 x i8> %b)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @cmpgt_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: cmpgt_h:
+; CHECK: cmpgt p0.h, p0/z, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmpgt.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                <vscale x 8 x i16> %a,
+                                                                <vscale x 8 x i16> %b)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @cmpgt_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: cmpgt_s:
+; CHECK: cmpgt p0.s, p0/z, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmpgt.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                <vscale x 4 x i32> %a,
+                                                                <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 2 x i1> @cmpgt_d(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmpgt_d:
+; CHECK: cmpgt p0.d, p0/z, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.cmpgt.nxv2i64(<vscale x 2 x i1> %pg,
+                                                                <vscale x 2 x i64> %a,
+                                                                <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i1> %out
+}
+
+define <vscale x 16 x i1> @cmpgt_wide_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmpgt_wide_b:
+; CHECK: cmpgt p0.b, p0/z, z0.b, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmpgt.wide.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                      <vscale x 16 x i8> %a,
+                                                                      <vscale x 2 x i64> %b)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @cmpgt_wide_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmpgt_wide_h:
+; CHECK: cmpgt p0.h, p0/z, z0.h, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmpgt.wide.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                     <vscale x 8 x i16> %a,
+                                                                     <vscale x 2 x i64> %b)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @cmpgt_wide_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmpgt_wide_s:
+; CHECK: cmpgt p0.s, p0/z, z0.s, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmpgt.wide.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                     <vscale x 4 x i32> %a,
+                                                                     <vscale x 2 x i64> %b)
+  ret <vscale x 4 x i1> %out
+}
+
+;
+; CMPHI
+;
+
+define <vscale x 16 x i1> @cmphi_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: cmphi_b:
+; CHECK: cmphi p0.b, p0/z, z0.b, z1.b
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmphi.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                 <vscale x 16 x i8> %a,
+                                                                 <vscale x 16 x i8> %b)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @cmphi_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: cmphi_h:
+; CHECK: cmphi p0.h, p0/z, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmphi.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                <vscale x 8 x i16> %a,
+                                                                <vscale x 8 x i16> %b)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @cmphi_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: cmphi_s:
+; CHECK: cmphi p0.s, p0/z, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmphi.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                <vscale x 4 x i32> %a,
+                                                                <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 2 x i1> @cmphi_d(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmphi_d:
+; CHECK: cmphi p0.d, p0/z, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.cmphi.nxv2i64(<vscale x 2 x i1> %pg,
+                                                                <vscale x 2 x i64> %a,
+                                                                <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i1> %out
+}
+
+define <vscale x 16 x i1> @cmphi_wide_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmphi_wide_b:
+; CHECK: cmphi p0.b, p0/z, z0.b, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmphi.wide.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                      <vscale x 16 x i8> %a,
+                                                                      <vscale x 2 x i64> %b)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @cmphi_wide_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmphi_wide_h:
+; CHECK: cmphi p0.h, p0/z, z0.h, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmphi.wide.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                     <vscale x 8 x i16> %a,
+                                                                     <vscale x 2 x i64> %b)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @cmphi_wide_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmphi_wide_s:
+; CHECK: cmphi p0.s, p0/z, z0.s, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmphi.wide.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                     <vscale x 4 x i32> %a,
+                                                                     <vscale x 2 x i64> %b)
+  ret <vscale x 4 x i1> %out
+}
+
+;
+; CMPHS
+;
+
+define <vscale x 16 x i1> @cmphs_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: cmphs_b:
+; CHECK: cmphs p0.b, p0/z, z0.b, z1.b
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmphs.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                 <vscale x 16 x i8> %a,
+                                                                 <vscale x 16 x i8> %b)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @cmphs_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: cmphs_h:
+; CHECK: cmphs p0.h, p0/z, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmphs.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                <vscale x 8 x i16> %a,
+                                                                <vscale x 8 x i16> %b)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @cmphs_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: cmphs_s:
+; CHECK: cmphs p0.s, p0/z, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmphs.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                <vscale x 4 x i32> %a,
+                                                                <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 2 x i1> @cmphs_d(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmphs_d:
+; CHECK: cmphs p0.d, p0/z, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.cmphs.nxv2i64(<vscale x 2 x i1> %pg,
+                                                                <vscale x 2 x i64> %a,
+                                                                <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i1> %out
+}
+
+define <vscale x 16 x i1> @cmphs_wide_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmphs_wide_b:
+; CHECK: cmphs p0.b, p0/z, z0.b, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmphs.wide.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                      <vscale x 16 x i8> %a,
+                                                                      <vscale x 2 x i64> %b)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @cmphs_wide_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmphs_wide_h:
+; CHECK: cmphs p0.h, p0/z, z0.h, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmphs.wide.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                     <vscale x 8 x i16> %a,
+                                                                     <vscale x 2 x i64> %b)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @cmphs_wide_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmphs_wide_s:
+; CHECK: cmphs p0.s, p0/z, z0.s, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmphs.wide.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                     <vscale x 4 x i32> %a,
+                                                                     <vscale x 2 x i64> %b)
+  ret <vscale x 4 x i1> %out
+}
+
+;
+; CMPLE
+;
+
+define <vscale x 16 x i1> @cmple_wide_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmple_wide_b:
+; CHECK: cmple p0.b, p0/z, z0.b, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmple.wide.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                      <vscale x 16 x i8> %a,
+                                                                      <vscale x 2 x i64> %b)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @cmple_wide_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmple_wide_h:
+; CHECK: cmple p0.h, p0/z, z0.h, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmple.wide.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                     <vscale x 8 x i16> %a,
+                                                                     <vscale x 2 x i64> %b)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @cmple_wide_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmple_wide_s:
+; CHECK: cmple p0.s, p0/z, z0.s, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmple.wide.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                     <vscale x 4 x i32> %a,
+                                                                     <vscale x 2 x i64> %b)
+  ret <vscale x 4 x i1> %out
+}
+
+;
+; CMPLO
+;
+
+define <vscale x 16 x i1> @cmplo_wide_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmplo_wide_b:
+; CHECK: cmplo p0.b, p0/z, z0.b, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmplo.wide.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                      <vscale x 16 x i8> %a,
+                                                                      <vscale x 2 x i64> %b)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @cmplo_wide_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmplo_wide_h:
+; CHECK: cmplo p0.h, p0/z, z0.h, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmplo.wide.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                     <vscale x 8 x i16> %a,
+                                                                     <vscale x 2 x i64> %b)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @cmplo_wide_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmplo_wide_s:
+; CHECK: cmplo p0.s, p0/z, z0.s, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmplo.wide.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                     <vscale x 4 x i32> %a,
+                                                                     <vscale x 2 x i64> %b)
+  ret <vscale x 4 x i1> %out
+}
+
+;
+; CMPLS
+;
+
+define <vscale x 16 x i1> @cmpls_wide_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmpls_wide_b:
+; CHECK: cmpls p0.b, p0/z, z0.b, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmpls.wide.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                      <vscale x 16 x i8> %a,
+                                                                      <vscale x 2 x i64> %b)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @cmpls_wide_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmpls_wide_h:
+; CHECK: cmpls p0.h, p0/z, z0.h, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmpls.wide.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                     <vscale x 8 x i16> %a,
+                                                                     <vscale x 2 x i64> %b)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @cmpls_wide_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmpls_wide_s:
+; CHECK: cmpls p0.s, p0/z, z0.s, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmpls.wide.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                     <vscale x 4 x i32> %a,
+                                                                     <vscale x 2 x i64> %b)
+  ret <vscale x 4 x i1> %out
+}
+
+;
+; CMPLT
+;
+
+define <vscale x 16 x i1> @cmplt_wide_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmplt_wide_b:
+; CHECK: cmplt p0.b, p0/z, z0.b, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmplt.wide.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                      <vscale x 16 x i8> %a,
+                                                                      <vscale x 2 x i64> %b)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @cmplt_wide_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmplt_wide_h:
+; CHECK: cmplt p0.h, p0/z, z0.h, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmplt.wide.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                     <vscale x 8 x i16> %a,
+                                                                     <vscale x 2 x i64> %b)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @cmplt_wide_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmplt_wide_s:
+; CHECK: cmplt p0.s, p0/z, z0.s, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmplt.wide.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                     <vscale x 4 x i32> %a,
+                                                                     <vscale x 2 x i64> %b)
+  ret <vscale x 4 x i1> %out
+}
+
+;
+; CMPNE
+;
+
+define <vscale x 16 x i1> @cmpne_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: cmpne_b:
+; CHECK: cmpne p0.b, p0/z, z0.b, z1.b
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmpne.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                 <vscale x 16 x i8> %a,
+                                                                 <vscale x 16 x i8> %b)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @cmpne_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: cmpne_h:
+; CHECK: cmpne p0.h, p0/z, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmpne.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                <vscale x 8 x i16> %a,
+                                                                <vscale x 8 x i16> %b)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @cmpne_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: cmpne_s:
+; CHECK: cmpne p0.s, p0/z, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmpne.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                <vscale x 4 x i32> %a,
+                                                                <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 2 x i1> @cmpne_d(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmpne_d:
+; CHECK: cmpne p0.d, p0/z, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.cmpne.nxv2i64(<vscale x 2 x i1> %pg,
+                                                                <vscale x 2 x i64> %a,
+                                                                <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i1> %out
+}
+
+define <vscale x 16 x i1> @cmpne_wide_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmpne_wide_b:
+; CHECK: cmpne p0.b, p0/z, z0.b, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmpne.wide.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                      <vscale x 16 x i8> %a,
+                                                                      <vscale x 2 x i64> %b)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @cmpne_wide_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmpne_wide_h:
+; CHECK: cmpne p0.h, p0/z, z0.h, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmpne.wide.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                     <vscale x 8 x i16> %a,
+                                                                     <vscale x 2 x i64> %b)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @cmpne_wide_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmpne_wide_s:
+; CHECK: cmpne p0.s, p0/z, z0.s, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmpne.wide.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                     <vscale x 4 x i32> %a,
+                                                                     <vscale x 2 x i64> %b)
+  ret <vscale x 4 x i1> %out
+}
+
+declare <vscale x 16 x i1> @llvm.aarch64.sve.cmpeq.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 16 x i8>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.cmpeq.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 8 x i16>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.cmpeq.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 4 x i32>)
+declare <vscale x 2 x i1> @llvm.aarch64.sve.cmpeq.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>, <vscale x 2 x i64>)
+declare <vscale x 16 x i1> @llvm.aarch64.sve.cmpeq.wide.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 2 x i64>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.cmpeq.wide.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 2 x i64>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.cmpeq.wide.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 2 x i64>)
+
+declare <vscale x 16 x i1> @llvm.aarch64.sve.cmpge.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 16 x i8>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.cmpge.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 8 x i16>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.cmpge.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 4 x i32>)
+declare <vscale x 2 x i1> @llvm.aarch64.sve.cmpge.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>, <vscale x 2 x i64>)
+declare <vscale x 16 x i1> @llvm.aarch64.sve.cmpge.wide.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 2 x i64>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.cmpge.wide.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 2 x i64>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.cmpge.wide.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 2 x i64>)
+
+declare <vscale x 16 x i1> @llvm.aarch64.sve.cmpgt.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 16 x i8>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.cmpgt.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 8 x i16>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.cmpgt.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 4 x i32>)
+declare <vscale x 2 x i1> @llvm.aarch64.sve.cmpgt.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>, <vscale x 2 x i64>)
+declare <vscale x 16 x i1> @llvm.aarch64.sve.cmpgt.wide.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 2 x i64>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.cmpgt.wide.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 2 x i64>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.cmpgt.wide.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 2 x i64>)
+
+declare <vscale x 16 x i1> @llvm.aarch64.sve.cmphi.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 16 x i8>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.cmphi.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 8 x i16>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.cmphi.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 4 x i32>)
+declare <vscale x 2 x i1> @llvm.aarch64.sve.cmphi.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>, <vscale x 2 x i64>)
+declare <vscale x 16 x i1> @llvm.aarch64.sve.cmphi.wide.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 2 x i64>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.cmphi.wide.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 2 x i64>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.cmphi.wide.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 2 x i64>)
+
+declare <vscale x 16 x i1> @llvm.aarch64.sve.cmphs.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 16 x i8>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.cmphs.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 8 x i16>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.cmphs.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 4 x i32>)
+declare <vscale x 2 x i1> @llvm.aarch64.sve.cmphs.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>, <vscale x 2 x i64>)
+declare <vscale x 16 x i1> @llvm.aarch64.sve.cmphs.wide.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 2 x i64>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.cmphs.wide.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 2 x i64>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.cmphs.wide.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 2 x i64>)
+
+declare <vscale x 16 x i1> @llvm.aarch64.sve.cmple.wide.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 2 x i64>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.cmple.wide.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 2 x i64>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.cmple.wide.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 2 x i64>)
+
+declare <vscale x 16 x i1> @llvm.aarch64.sve.cmplo.wide.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 2 x i64>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.cmplo.wide.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 2 x i64>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.cmplo.wide.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 2 x i64>)
+
+declare <vscale x 16 x i1> @llvm.aarch64.sve.cmpls.wide.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 2 x i64>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.cmpls.wide.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 2 x i64>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.cmpls.wide.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 2 x i64>)
+
+declare <vscale x 16 x i1> @llvm.aarch64.sve.cmplt.wide.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 2 x i64>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.cmplt.wide.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 2 x i64>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.cmplt.wide.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 2 x i64>)
+
+declare <vscale x 16 x i1> @llvm.aarch64.sve.cmpne.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 16 x i8>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.cmpne.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 8 x i16>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.cmpne.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 4 x i32>)
+declare <vscale x 2 x i1> @llvm.aarch64.sve.cmpne.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>, <vscale x 2 x i64>)
+declare <vscale x 16 x i1> @llvm.aarch64.sve.cmpne.wide.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 2 x i64>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.cmpne.wide.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 2 x i64>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.cmpne.wide.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 2 x i64>)
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-loads.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-loads.ll
new file mode 100644
index 000000000..b82ae30ba
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-loads.ll
@@ -0,0 +1,88 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+;
+; LDNT1B
+;
+
+define <vscale x 16 x i8> @ldnt1b_i8(<vscale x 16 x i1> %pred, <vscale x 16 x i8>* %addr) {
+; CHECK-LABEL: ldnt1b_i8:
+; CHECK: ldnt1b { z0.b }, p0/z, [x0]
+; CHECK-NEXT: ret
+  %res = call <vscale x 16 x i8> @llvm.aarch64.sve.ldnt1.nxv16i8(<vscale x 16 x i1> %pred,
+                                                                 <vscale x 16 x i8>* %addr)
+  ret <vscale x 16 x i8> %res
+}
+
+;
+; LDNT1H
+;
+
+define <vscale x 8 x i16> @ldnt1h_i16(<vscale x 8 x i1> %pred, <vscale x 8 x i16>* %addr) {
+; CHECK-LABEL: ldnt1h_i16:
+; CHECK: ldnt1h { z0.h }, p0/z, [x0]
+; CHECK-NEXT: ret
+  %res = call <vscale x 8 x i16> @llvm.aarch64.sve.ldnt1.nxv8i16(<vscale x 8 x i1> %pred,
+                                                                 <vscale x 8 x i16>* %addr)
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 8 x half> @ldnt1h_f16(<vscale x 8 x i1> %pred, <vscale x 8 x half>* %addr) {
+; CHECK-LABEL: ldnt1h_f16:
+; CHECK: ldnt1h { z0.h }, p0/z, [x0]
+; CHECK-NEXT: ret
+  %res = call <vscale x 8 x half> @llvm.aarch64.sve.ldnt1.nxv8f16(<vscale x 8 x i1> %pred,
+                                                                  <vscale x 8 x half>* %addr)
+  ret <vscale x 8 x half> %res
+}
+
+;
+; LDNT1W
+;
+
+define <vscale x 4 x i32> @ldnt1w_i32(<vscale x 4 x i1> %pred, <vscale x 4 x i32>* %addr) {
+; CHECK-LABEL: ldnt1w_i32:
+; CHECK: ldnt1w { z0.s }, p0/z, [x0]
+; CHECK-NEXT: ret
+  %res = call <vscale x 4 x i32> @llvm.aarch64.sve.ldnt1.nxv4i32(<vscale x 4 x i1> %pred,
+                                                                 <vscale x 4 x i32>* %addr)
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 4 x float> @ldnt1w_f32(<vscale x 4 x i1> %pred, <vscale x 4 x float>* %addr) {
+; CHECK-LABEL: ldnt1w_f32:
+; CHECK: ldnt1w { z0.s }, p0/z, [x0]
+; CHECK-NEXT: ret
+  %res = call <vscale x 4 x float> @llvm.aarch64.sve.ldnt1.nxv4f32(<vscale x 4 x i1> %pred,
+                                                                   <vscale x 4 x float>* %addr)
+  ret <vscale x 4 x float> %res
+}
+
+;
+; LDNT1D
+;
+
+define <vscale x 2 x i64> @ldnt1d_i64(<vscale x 2 x i1> %pred, <vscale x 2 x i64>* %addr) {
+; CHECK-LABEL: ldnt1d_i64:
+; CHECK: ldnt1d { z0.d }, p0/z, [x0]
+; CHECK-NEXT: ret
+  %res = call <vscale x 2 x i64> @llvm.aarch64.sve.ldnt1.nxv2i64(<vscale x 2 x i1> %pred,
+                                                                 <vscale x 2 x i64>* %addr)
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x double> @ldnt1d_f64(<vscale x 2 x i1> %pred, <vscale x 2 x double>* %addr) {
+; CHECK-LABEL: ldnt1d_f64:
+; CHECK: ldnt1d { z0.d }, p0/z, [x0]
+; CHECK-NEXT: ret
+  %res = call <vscale x 2 x double> @llvm.aarch64.sve.ldnt1.nxv2f64(<vscale x 2 x i1> %pred,
+                                                                    <vscale x 2 x double>* %addr)
+  ret <vscale x 2 x double> %res
+}
+
+declare <vscale x 16 x i8> @llvm.aarch64.sve.ldnt1.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>*)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.ldnt1.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>*)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.ldnt1.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>*)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.ldnt1.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>*)
+declare <vscale x 8 x half> @llvm.aarch64.sve.ldnt1.nxv8f16(<vscale x 8 x i1>, <vscale x 8 x half>*)
+declare <vscale x 4 x float> @llvm.aarch64.sve.ldnt1.nxv4f32(<vscale x 4 x i1>, <vscale x 4 x float>*)
+declare <vscale x 2 x double> @llvm.aarch64.sve.ldnt1.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>*)
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-logical.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-logical.ll
new file mode 100644
index 000000000..05c98fbb2
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-logical.ll
@@ -0,0 +1,99 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+;
+; CNOT
+;
+
+define <vscale x 16 x i8> @cnot_i8(<vscale x 16 x i8> %a, <vscale x 16 x i1> %pg, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: cnot_i8:
+; CHECK: cnot z0.b, p0/m, z1.b
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.cnot.nxv16i8(<vscale x 16 x i8> %a,
+                                                                <vscale x 16 x i1> %pg,
+                                                                <vscale x 16 x i8> %b)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @cnot_i16(<vscale x 8 x i16> %a, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: cnot_i16:
+; CHECK: cnot z0.h, p0/m, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.cnot.nxv8i16(<vscale x 8 x i16> %a,
+                                                                <vscale x 8 x i1> %pg,
+                                                                <vscale x 8 x i16> %b)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @cnot_i32(<vscale x 4 x i32> %a, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: cnot_i32:
+; CHECK: cnot z0.s, p0/m, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.cnot.nxv4i32(<vscale x 4 x i32> %a,
+                                                                <vscale x 4 x i1> %pg,
+                                                                <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @cnot_i64(<vscale x 2 x i64> %a, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cnot_i64:
+; CHECK: cnot z0.d, p0/m, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.cnot.nxv2i64(<vscale x 2 x i64> %a,
+                                                                <vscale x 2 x i1> %pg,
+                                                                <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i64> %out
+}
+
+;
+; NOT
+;
+
+define <vscale x 16 x i8> @not_i8(<vscale x 16 x i8> %a, <vscale x 16 x i1> %pg, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: not_i8:
+; CHECK: not z0.b, p0/m, z1.b
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.not.nxv16i8(<vscale x 16 x i8> %a,
+                                                               <vscale x 16 x i1> %pg,
+                                                               <vscale x 16 x i8> %b)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @not_i16(<vscale x 8 x i16> %a, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: not_i16:
+; CHECK: not z0.h, p0/m, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.not.nxv8i16(<vscale x 8 x i16> %a,
+                                                               <vscale x 8 x i1> %pg,
+                                                               <vscale x 8 x i16> %b)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @not_i32(<vscale x 4 x i32> %a, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: not_i32:
+; CHECK: not z0.s, p0/m, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.not.nxv4i32(<vscale x 4 x i32> %a,
+                                                               <vscale x 4 x i1> %pg,
+                                                               <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @not_i64(<vscale x 2 x i64> %a, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: not_i64:
+; CHECK: not z0.d, p0/m, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.not.nxv2i64(<vscale x 2 x i64> %a,
+                                                               <vscale x 2 x i1> %pg,
+                                                               <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i64> %out
+}
+
+declare <vscale x 16 x i8> @llvm.aarch64.sve.cnot.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i1>, <vscale x 16 x i8>)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.cnot.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i1>, <vscale x 8 x i16>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.cnot.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i1>, <vscale x 4 x i32>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.cnot.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i1>, <vscale x 2 x i64>)
+
+declare <vscale x 16 x i8> @llvm.aarch64.sve.not.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i1>, <vscale x 16 x i8>)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.not.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i1>, <vscale x 8 x i16>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.not.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i1>, <vscale x 4 x i32>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.not.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i1>, <vscale x 2 x i64>)
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-perm-select.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-perm-select.ll
new file mode 100644
index 000000000..dae5adbd4
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-perm-select.ll
@@ -0,0 +1,1765 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+;
+; CLASTA (Vectors)
+;
+
+define <vscale x 16 x i8> @clasta_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: clasta_i8:
+; CHECK: clasta z0.b, p0, z0.b, z1.b
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.clasta.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                  <vscale x 16 x i8> %a,
+                                                                  <vscale x 16 x i8> %b)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @clasta_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: clasta_i16:
+; CHECK: clasta z0.h, p0, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.clasta.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                  <vscale x 8 x i16> %a,
+                                                                  <vscale x 8 x i16> %b)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @clasta_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: clasta_i32:
+; CHECK: clasta z0.s, p0, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.clasta.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                  <vscale x 4 x i32> %a,
+                                                                  <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @clasta_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: clasta_i64:
+; CHECK: clasta z0.d, p0, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.clasta.nxv2i64(<vscale x 2 x i1> %pg,
+                                                                  <vscale x 2 x i64> %a,
+                                                                  <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i64> %out
+}
+
+define <vscale x 8 x half> @clasta_f16(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: clasta_f16:
+; CHECK: clasta z0.h, p0, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.clasta.nxv8f16(<vscale x 8 x i1> %pg,
+                                                                   <vscale x 8 x half> %a,
+                                                                   <vscale x 8 x half> %b)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x float> @clasta_f32(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: clasta_f32:
+; CHECK: clasta z0.s, p0, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.clasta.nxv4f32(<vscale x 4 x i1> %pg,
+                                                                    <vscale x 4 x float> %a,
+                                                                    <vscale x 4 x float> %b)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @clasta_f64(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: clasta_f64:
+; CHECK: clasta z0.d, p0, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.clasta.nxv2f64(<vscale x 2 x i1> %pg,
+                                                                     <vscale x 2 x double> %a,
+                                                                     <vscale x 2 x double> %b)
+  ret <vscale x 2 x double> %out
+}
+
+;
+; CLASTA (Scalar)
+;
+
+define i8 @clasta_n_i8(<vscale x 16 x i1> %pg, i8 %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: clasta_n_i8:
+; CHECK: clasta w0, p0, w0, z0.b
+; CHECK-NEXT: ret
+  %out = call i8 @llvm.aarch64.sve.clasta.n.nxv16i8(<vscale x 16 x i1> %pg,
+                                                    i8 %a,
+                                                    <vscale x 16 x i8> %b)
+  ret i8 %out
+}
+
+define i16 @clasta_n_i16(<vscale x 8 x i1> %pg, i16 %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: clasta_n_i16:
+; CHECK: clasta w0, p0, w0, z0.h
+; CHECK-NEXT: ret
+  %out = call i16 @llvm.aarch64.sve.clasta.n.nxv8i16(<vscale x 8 x i1> %pg,
+                                                     i16 %a,
+                                                     <vscale x 8 x i16> %b)
+  ret i16 %out
+}
+
+define i32 @clasta_n_i32(<vscale x 4 x i1> %pg, i32 %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: clasta_n_i32:
+; CHECK: clasta w0, p0, w0, z0.s
+; CHECK-NEXT: ret
+  %out = call i32 @llvm.aarch64.sve.clasta.n.nxv4i32(<vscale x 4 x i1> %pg,
+                                                     i32 %a,
+                                                     <vscale x 4 x i32> %b)
+  ret i32 %out
+}
+
+define i64 @clasta_n_i64(<vscale x 2 x i1> %pg, i64 %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: clasta_n_i64:
+; CHECK: clasta x0, p0, x0, z0.d
+; CHECK-NEXT: ret
+  %out = call i64 @llvm.aarch64.sve.clasta.n.nxv2i64(<vscale x 2 x i1> %pg,
+                                                     i64 %a,
+                                                     <vscale x 2 x i64> %b)
+  ret i64 %out
+}
+
+define half @clasta_n_f16(<vscale x 8 x i1> %pg, half %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: clasta_n_f16:
+; CHECK: clasta h0, p0, h0, z1.h
+; CHECK-NEXT: ret
+  %out = call half @llvm.aarch64.sve.clasta.n.nxv8f16(<vscale x 8 x i1> %pg,
+                                                      half %a,
+                                                      <vscale x 8 x half> %b)
+  ret half %out
+}
+
+define float @clasta_n_f32(<vscale x 4 x i1> %pg, float %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: clasta_n_f32:
+; CHECK: clasta s0, p0, s0, z1.s
+; CHECK-NEXT: ret
+  %out = call float @llvm.aarch64.sve.clasta.n.nxv4f32(<vscale x 4 x i1> %pg,
+                                                       float %a,
+                                                       <vscale x 4 x float> %b)
+  ret float %out
+}
+
+define double @clasta_n_f64(<vscale x 2 x i1> %pg, double %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: clasta_n_f64:
+; CHECK: clasta d0, p0, d0, z1.d
+; CHECK-NEXT: ret
+  %out = call double @llvm.aarch64.sve.clasta.n.nxv2f64(<vscale x 2 x i1> %pg,
+                                                        double %a,
+                                                        <vscale x 2 x double> %b)
+  ret double %out
+}
+
+;
+; CLASTB (Vectors)
+;
+
+define <vscale x 16 x i8> @clastb_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: clastb_i8:
+; CHECK: clastb z0.b, p0, z0.b, z1.b
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.clastb.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                  <vscale x 16 x i8> %a,
+                                                                  <vscale x 16 x i8> %b)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @clastb_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: clastb_i16:
+; CHECK: clastb z0.h, p0, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.clastb.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                  <vscale x 8 x i16> %a,
+                                                                  <vscale x 8 x i16> %b)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @clastb_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: clastb_i32:
+; CHECK: clastb z0.s, p0, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.clastb.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                  <vscale x 4 x i32> %a,
+                                                                  <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @clastb_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: clastb_i64:
+; CHECK: clastb z0.d, p0, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.clastb.nxv2i64(<vscale x 2 x i1> %pg,
+                                                                  <vscale x 2 x i64> %a,
+                                                                  <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i64> %out
+}
+
+define <vscale x 8 x half> @clastb_f16(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: clastb_f16:
+; CHECK: clastb z0.h, p0, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.clastb.nxv8f16(<vscale x 8 x i1> %pg,
+                                                                   <vscale x 8 x half> %a,
+                                                                   <vscale x 8 x half> %b)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x float> @clastb_f32(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: clastb_f32:
+; CHECK: clastb z0.s, p0, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.clastb.nxv4f32(<vscale x 4 x i1> %pg,
+                                                                    <vscale x 4 x float> %a,
+                                                                    <vscale x 4 x float> %b)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @clastb_f64(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: clastb_f64:
+; CHECK: clastb z0.d, p0, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.clastb.nxv2f64(<vscale x 2 x i1> %pg,
+                                                                     <vscale x 2 x double> %a,
+                                                                     <vscale x 2 x double> %b)
+  ret <vscale x 2 x double> %out
+}
+
+;
+; CLASTB (Scalar)
+;
+
+define i8 @clastb_n_i8(<vscale x 16 x i1> %pg, i8 %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: clastb_n_i8:
+; CHECK: clastb w0, p0, w0, z0.b
+; CHECK-NEXT: ret
+  %out = call i8 @llvm.aarch64.sve.clastb.n.nxv16i8(<vscale x 16 x i1> %pg,
+                                                    i8 %a,
+                                                    <vscale x 16 x i8> %b)
+  ret i8 %out
+}
+
+define i16 @clastb_n_i16(<vscale x 8 x i1> %pg, i16 %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: clastb_n_i16:
+; CHECK: clastb w0, p0, w0, z0.h
+; CHECK-NEXT: ret
+  %out = call i16 @llvm.aarch64.sve.clastb.n.nxv8i16(<vscale x 8 x i1> %pg,
+                                                     i16 %a,
+                                                     <vscale x 8 x i16> %b)
+  ret i16 %out
+}
+
+define i32 @clastb_n_i32(<vscale x 4 x i1> %pg, i32 %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: clastb_n_i32:
+; CHECK: clastb w0, p0, w0, z0.s
+; CHECK-NEXT: ret
+  %out = call i32 @llvm.aarch64.sve.clastb.n.nxv4i32(<vscale x 4 x i1> %pg,
+                                                     i32 %a,
+                                                     <vscale x 4 x i32> %b)
+  ret i32 %out
+}
+
+define i64 @clastb_n_i64(<vscale x 2 x i1> %pg, i64 %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: clastb_n_i64:
+; CHECK: clastb x0, p0, x0, z0.d
+; CHECK-NEXT: ret
+  %out = call i64 @llvm.aarch64.sve.clastb.n.nxv2i64(<vscale x 2 x i1> %pg,
+                                                     i64 %a,
+                                                     <vscale x 2 x i64> %b)
+  ret i64 %out
+}
+
+define half @clastb_n_f16(<vscale x 8 x i1> %pg, half %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: clastb_n_f16:
+; CHECK: clastb h0, p0, h0, z1.h
+; CHECK-NEXT: ret
+  %out = call half @llvm.aarch64.sve.clastb.n.nxv8f16(<vscale x 8 x i1> %pg,
+                                                      half %a,
+                                                      <vscale x 8 x half> %b)
+  ret half %out
+}
+
+define float @clastb_n_f32(<vscale x 4 x i1> %pg, float %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: clastb_n_f32:
+; CHECK: clastb s0, p0, s0, z1.s
+; CHECK-NEXT: ret
+  %out = call float @llvm.aarch64.sve.clastb.n.nxv4f32(<vscale x 4 x i1> %pg,
+                                                       float %a,
+                                                       <vscale x 4 x float> %b)
+  ret float %out
+}
+
+define double @clastb_n_f64(<vscale x 2 x i1> %pg, double %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: clastb_n_f64:
+; CHECK: clastb d0, p0, d0, z1.d
+; CHECK-NEXT: ret
+  %out = call double @llvm.aarch64.sve.clastb.n.nxv2f64(<vscale x 2 x i1> %pg,
+                                                        double %a,
+                                                        <vscale x 2 x double> %b)
+  ret double %out
+}
+
+;
+; EXT
+;
+
+define <vscale x 16 x i8> @ext_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: ext_i8:
+; CHECK: ext z0.b, z0.b, z1.b, #255
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.ext.nxv16i8(<vscale x 16 x i8> %a,
+                                                               <vscale x 16 x i8> %b,
+                                                               i32 255)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @ext_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: ext_i16:
+; CHECK: ext z0.b, z0.b, z1.b, #0
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.ext.nxv8i16(<vscale x 8 x i16> %a,
+                                                               <vscale x 8 x i16> %b,
+                                                               i32 0)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @ext_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: ext_i32:
+; CHECK: ext z0.b, z0.b, z1.b, #4
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.ext.nxv4i32(<vscale x 4 x i32> %a,
+                                                               <vscale x 4 x i32> %b,
+                                                               i32 1)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @ext_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: ext_i64:
+; CHECK: ext z0.b, z0.b, z1.b, #16
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.ext.nxv2i64(<vscale x 2 x i64> %a,
+                                                               <vscale x 2 x i64> %b,
+                                                               i32 2)
+  ret <vscale x 2 x i64> %out
+}
+
+define <vscale x 8 x half> @ext_f16(<vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: ext_f16:
+; CHECK: ext z0.b, z0.b, z1.b, #6
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.ext.nxv8f16(<vscale x 8 x half> %a,
+                                                                <vscale x 8 x half> %b,
+                                                                i32 3)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x float> @ext_f32(<vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: ext_f32:
+; CHECK: ext z0.b, z0.b, z1.b, #16
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.ext.nxv4f32(<vscale x 4 x float> %a,
+                                                                 <vscale x 4 x float> %b,
+                                                                 i32 4)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @ext_f64(<vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: ext_f64:
+; CHECK: ext z0.b, z0.b, z1.b, #40
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.ext.nxv2f64(<vscale x 2 x double> %a,
+                                                                  <vscale x 2 x double> %b,
+                                                                  i32 5)
+  ret <vscale x 2 x double> %out
+}
+
+;
+; LASTA
+;
+
+define i8 @lasta_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a) {
+; CHECK-LABEL: lasta_i8
+; CHECK: lasta w0, p0, z0.b
+; CHECK-NEXT: ret
+  %res = call i8 @llvm.aarch64.sve.lasta.nxv16i8(<vscale x 16 x i1> %pg,
+                                                 <vscale x 16 x i8> %a)
+  ret i8 %res
+}
+
+define i16 @lasta_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a) {
+; CHECK-LABEL: lasta_i16
+; CHECK: lasta w0, p0, z0.h
+; CHECK-NEXT: ret
+  %res = call i16 @llvm.aarch64.sve.lasta.nxv8i16(<vscale x 8 x i1> %pg,
+                                                  <vscale x 8 x i16> %a)
+  ret i16 %res
+}
+
+define i32 @lasta_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a) {
+; CHECK-LABEL: lasta_i32
+; CHECK: lasta w0, p0, z0.s
+; CHECK-NEXT: ret
+  %res = call i32 @llvm.aarch64.sve.lasta.nxv4i32(<vscale x 4 x i1> %pg,
+                                                  <vscale x 4 x i32> %a)
+  ret i32 %res
+}
+
+define i64 @lasta_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a) {
+; CHECK-LABEL:  lasta_i64
+; CHECK: lasta x0, p0, z0.d
+; CHECK-NEXT: ret
+  %res = call i64 @llvm.aarch64.sve.lasta.nxv2i64(<vscale x 2 x i1> %pg,
+                                                  <vscale x 2 x i64> %a)
+  ret i64 %res
+}
+
+define half @lasta_f16(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a) {
+; CHECK-LABEL: lasta_f16
+; CHECK: lasta h0, p0, z0.h
+; CHECK-NEXT: ret
+  %res = call half @llvm.aarch64.sve.lasta.nxv8f16(<vscale x 8 x i1> %pg,
+                                                   <vscale x 8 x half> %a)
+  ret half %res
+}
+
+define float @lasta_f32(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a) {
+; CHECK-LABEL: lasta_f32
+; CHECK: lasta s0, p0, z0.s
+; CHECK-NEXT: ret
+  %res = call float @llvm.aarch64.sve.lasta.nxv4f32(<vscale x 4 x i1> %pg,
+                                                    <vscale x 4 x float> %a)
+  ret float %res
+}
+
+define float @lasta_f32_v2(<vscale x 2 x i1> %pg, <vscale x 2 x float> %a) {
+; CHECK-LABEL: lasta_f32_v2
+; CHECK: lasta s0, p0, z0.s
+; CHECK-NEXT: ret
+  %res = call float @llvm.aarch64.sve.lasta.nxv2f32(<vscale x 2 x i1> %pg,
+                                                    <vscale x 2 x float> %a)
+  ret float %res
+}
+
+define double @lasta_f64(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a) {
+; CHECK-LABEL:  lasta_f64
+; CHECK: lasta d0, p0, z0.d
+; CHECK-NEXT: ret
+  %res = call double @llvm.aarch64.sve.lasta.nxv2f64(<vscale x 2 x i1> %pg,
+                                                     <vscale x 2 x double> %a)
+  ret double %res
+}
+
+;
+; LASTB
+;
+
+define i8 @lastb_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a) {
+; CHECK-LABEL: lastb_i8
+; CHECK: lastb w0, p0, z0.b
+; CHECK-NEXT: ret
+  %res = call i8 @llvm.aarch64.sve.lastb.nxv16i8(<vscale x 16 x i1> %pg,
+                                                 <vscale x 16 x i8> %a)
+  ret i8 %res
+}
+
+define i16 @lastb_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a) {
+; CHECK-LABEL: lastb_i16
+; CHECK: lastb w0, p0, z0.h
+; CHECK-NEXT: ret
+  %res = call i16 @llvm.aarch64.sve.lastb.nxv8i16(<vscale x 8 x i1> %pg,
+                                                  <vscale x 8 x i16> %a)
+  ret i16 %res
+}
+
+define i32 @lastb_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a) {
+; CHECK-LABEL: lastb_i32
+; CHECK: lastb w0, p0, z0.s
+; CHECK-NEXT: ret
+  %res = call i32 @llvm.aarch64.sve.lastb.nxv4i32(<vscale x 4 x i1> %pg,
+                                                  <vscale x 4 x i32> %a)
+  ret i32 %res
+}
+
+define i64 @lastb_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a) {
+; CHECK-LABEL:  lastb_i64
+; CHECK: lastb x0, p0, z0.d
+; CHECK-NEXT: ret
+  %res = call i64 @llvm.aarch64.sve.lastb.nxv2i64(<vscale x 2 x i1> %pg,
+                                                  <vscale x 2 x i64> %a)
+  ret i64 %res
+}
+
+define half @lastb_f16(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a) {
+; CHECK-LABEL: lastb_f16
+; CHECK: lastb h0, p0, z0.h
+; CHECK-NEXT: ret
+  %res = call half @llvm.aarch64.sve.lastb.nxv8f16(<vscale x 8 x i1> %pg,
+                                                   <vscale x 8 x half> %a)
+  ret half %res
+}
+
+define float @lastb_f32(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a) {
+; CHECK-LABEL: lastb_f32
+; CHECK: lastb s0, p0, z0.s
+; CHECK-NEXT: ret
+  %res = call float @llvm.aarch64.sve.lastb.nxv4f32(<vscale x 4 x i1> %pg,
+                                                    <vscale x 4 x float> %a)
+  ret float %res
+}
+
+define float @lastb_f32_v2(<vscale x 2 x i1> %pg, <vscale x 2 x float> %a) {
+; CHECK-LABEL: lastb_f32_v2
+; CHECK: lastb s0, p0, z0.s
+; CHECK-NEXT: ret
+  %res = call float @llvm.aarch64.sve.lastb.nxv2f32(<vscale x 2 x i1> %pg,
+                                                    <vscale x 2 x float> %a)
+  ret float %res
+}
+
+define double @lastb_f64(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a) {
+; CHECK-LABEL:  lastb_f64
+; CHECK: lastb d0, p0, z0.d
+; CHECK-NEXT: ret
+  %res = call double @llvm.aarch64.sve.lastb.nxv2f64(<vscale x 2 x i1> %pg,
+                                                     <vscale x 2 x double> %a)
+  ret double %res
+}
+
+;
+; COMPACT
+;
+
+define <vscale x 4 x i32> @compact_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a) {
+; CHECK-LABEL: compact_i32:
+; CHECK: compact z0.s, p0, z0.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.compact.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                   <vscale x 4 x i32> %a)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @compact_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a) {
+; CHECK-LABEL: compact_i64:
+; CHECK: compact z0.d, p0, z0.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.compact.nxv2i64(<vscale x 2 x i1> %pg,
+                                                                   <vscale x 2 x i64> %a)
+  ret <vscale x 2 x i64> %out
+}
+
+define <vscale x 4 x float> @compact_f32(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a) {
+; CHECK-LABEL: compact_f32:
+; CHECK: compact z0.s, p0, z0.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.compact.nxv4f32(<vscale x 4 x i1> %pg,
+                                                                     <vscale x 4 x float> %a)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @compact_f64(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a) {
+; CHECK-LABEL: compact_f64:
+; CHECK: compact z0.d, p0, z0.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.compact.nxv2f64(<vscale x 2 x i1> %pg,
+                                                                      <vscale x 2 x double> %a)
+  ret <vscale x 2 x double> %out
+}
+
+;
+; REV
+;
+
+define <vscale x 16 x i1> @rev_b8( <vscale x 16 x i1> %a) {
+; CHECK-LABEL: rev_b8
+; CHECK: rev p0.b, p0.b
+; CHECK-NEXT: ret
+  %res = call <vscale x 16 x i1> @llvm.aarch64.sve.rev.nxv16i1(<vscale x 16 x i1> %a)
+  ret <vscale x 16 x i1> %res
+}
+
+define <vscale x 8 x i1> @rev_b16(<vscale x 8 x i1> %a) {
+; CHECK-LABEL: rev_b16
+; CHECK: rev p0.h, p0.h
+; CHECK-NEXT: ret
+  %res = call <vscale x 8 x i1> @llvm.aarch64.sve.rev.nxv8i1(<vscale x 8 x i1> %a)
+  ret <vscale x 8 x i1> %res
+}
+
+define <vscale x 4 x i1> @rev_b32(<vscale x 4 x i1> %a) {
+; CHECK-LABEL: rev_b32
+; CHECK: rev p0.s, p0.s
+; CHECK-NEXT: ret
+  %res = call <vscale x 4 x i1> @llvm.aarch64.sve.rev.nxv4i1(<vscale x 4 x i1> %a)
+  ret <vscale x 4 x i1> %res
+}
+
+define <vscale x 2 x i1> @rev_b64(<vscale x 2 x i1> %a) {
+; CHECK-LABEL:  rev_b64
+; CHECK: rev p0.d, p0.d
+; CHECK-NEXT: ret
+  %res = call <vscale x 2 x i1> @llvm.aarch64.sve.rev.nxv2i1(<vscale x 2 x i1> %a)
+  ret <vscale x 2 x i1> %res
+}
+
+define <vscale x 16 x i8> @rev_i8( <vscale x 16 x i8> %a) {
+; CHECK-LABEL: rev_i8
+; CHECK: rev z0.b, z0.b
+; CHECK-NEXT: ret
+  %res = call <vscale x 16 x i8> @llvm.aarch64.sve.rev.nxv16i8(<vscale x 16 x i8> %a)
+  ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 8 x i16> @rev_i16(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: rev_i16
+; CHECK: rev z0.h, z0.h
+; CHECK-NEXT: ret
+  %res = call <vscale x 8 x i16> @llvm.aarch64.sve.rev.nxv8i16(<vscale x 8 x i16> %a)
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 4 x i32> @rev_i32(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: rev_i32
+; CHECK: rev z0.s, z0.s
+; CHECK-NEXT: ret
+  %res = call <vscale x 4 x i32> @llvm.aarch64.sve.rev.nxv4i32(<vscale x 4 x i32> %a)
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @rev_i64(<vscale x 2 x i64> %a) {
+; CHECK-LABEL:  rev_i64
+; CHECK: rev z0.d, z0.d
+; CHECK-NEXT: ret
+  %res = call <vscale x 2 x i64> @llvm.aarch64.sve.rev.nxv2i64(<vscale x 2 x i64> %a)
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 8 x half> @rev_f16(<vscale x 8 x half> %a) {
+; CHECK-LABEL: rev_f16
+; CHECK: rev z0.h, z0.h
+; CHECK-NEXT: ret
+  %res = call <vscale x 8 x half> @llvm.aarch64.sve.rev.nxv8f16(<vscale x 8 x half> %a)
+  ret <vscale x 8 x half> %res
+}
+
+define <vscale x 4 x float> @rev_f32(<vscale x 4 x float> %a) {
+; CHECK-LABEL: rev_f32
+; CHECK: rev z0.s, z0.s
+; CHECK-NEXT: ret
+  %res = call <vscale x 4 x float> @llvm.aarch64.sve.rev.nxv4f32(<vscale x 4 x float> %a)
+  ret <vscale x 4 x float> %res
+}
+
+define <vscale x 2 x double> @rev_f64(<vscale x 2 x double> %a) {
+; CHECK-LABEL:  rev_f64
+; CHECK: rev z0.d, z0.d
+; CHECK-NEXT: ret
+  %res = call <vscale x 2 x double> @llvm.aarch64.sve.rev.nxv2f64(<vscale x 2 x double> %a)
+  ret <vscale x 2 x double> %res
+}
+
+;
+; SPLICE
+;
+
+define <vscale x 16 x i8> @splice_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: splice_i8:
+; CHECK: splice z0.b, p0, z0.b, z1.b
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.splice.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                  <vscale x 16 x i8> %a,
+                                                                  <vscale x 16 x i8> %b)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @splice_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: splice_i16:
+; CHECK: splice z0.h, p0, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.splice.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                  <vscale x 8 x i16> %a,
+                                                                  <vscale x 8 x i16> %b)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @splice_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: splice_i32:
+; CHECK: splice z0.s, p0, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.splice.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                  <vscale x 4 x i32> %a,
+                                                                  <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @splice_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: splice_i64:
+; CHECK: splice z0.d, p0, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.splice.nxv2i64(<vscale x 2 x i1> %pg,
+                                                                  <vscale x 2 x i64> %a,
+                                                                  <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i64> %out
+}
+
+define <vscale x 8 x half> @splice_f16(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: splice_f16:
+; CHECK: splice z0.h, p0, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.splice.nxv8f16(<vscale x 8 x i1> %pg,
+                                                                   <vscale x 8 x half> %a,
+                                                                   <vscale x 8 x half> %b)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x float> @splice_f32(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: splice_f32:
+; CHECK: splice z0.s, p0, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.splice.nxv4f32(<vscale x 4 x i1> %pg,
+                                                                    <vscale x 4 x float> %a,
+                                                                    <vscale x 4 x float> %b)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @splice_f64(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: splice_f64:
+; CHECK: splice z0.d, p0, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.splice.nxv2f64(<vscale x 2 x i1> %pg,
+                                                                     <vscale x 2 x double> %a,
+                                                                     <vscale x 2 x double> %b)
+  ret <vscale x 2 x double> %out
+}
+
+;
+; SUNPKHI
+;
+
+define <vscale x 8 x i16> @sunpkhi_i16(<vscale x 16 x i8> %a) {
+; CHECK-LABEL: sunpkhi_i16
+; CHECK: sunpkhi z0.h, z0.b
+; CHECK-NEXT: ret
+  %res = call <vscale x 8 x i16> @llvm.aarch64.sve.sunpkhi.nxv8i16(<vscale x 16 x i8> %a)
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 4 x i32> @sunpkhi_i32(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: sunpkhi_i32
+; CHECK: sunpkhi z0.s, z0.h
+; CHECK-NEXT: ret
+  %res = call <vscale x 4 x i32> @llvm.aarch64.sve.sunpkhi.nxv4i32(<vscale x 8 x i16> %a)
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @sunpkhi_i64(<vscale x 4 x i32> %a) {
+; CHECK-LABEL:  sunpkhi_i64
+; CHECK: sunpkhi z0.d, z0.s
+; CHECK-NEXT: ret
+  %res = call <vscale x 2 x i64> @llvm.aarch64.sve.sunpkhi.nxv2i64(<vscale x 4 x i32> %a)
+  ret <vscale x 2 x i64> %res
+}
+
+;
+; SUNPKLO
+;
+
+define <vscale x 8 x i16> @sunpklo_i16(<vscale x 16 x i8> %a) {
+; CHECK-LABEL: sunpklo_i16
+; CHECK: sunpklo z0.h, z0.b
+; CHECK-NEXT: ret
+  %res = call <vscale x 8 x i16> @llvm.aarch64.sve.sunpklo.nxv8i16(<vscale x 16 x i8> %a)
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 4 x i32> @sunpklo_i32(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: sunpklo_i32
+; CHECK: sunpklo z0.s, z0.h
+; CHECK-NEXT: ret
+  %res = call <vscale x 4 x i32> @llvm.aarch64.sve.sunpklo.nxv4i32(<vscale x 8 x i16> %a)
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @sunpklo_i64(<vscale x 4 x i32> %a) {
+; CHECK-LABEL:  sunpklo_i64
+; CHECK: sunpklo z0.d, z0.s
+; CHECK-NEXT: ret
+  %res = call <vscale x 2 x i64> @llvm.aarch64.sve.sunpklo.nxv2i64(<vscale x 4 x i32> %a)
+  ret <vscale x 2 x i64> %res
+}
+
+;
+; TBL
+;
+
+define <vscale x 16 x i8> @tbl_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: tbl_i8:
+; CHECK: tbl z0.b, { z0.b }, z1.b
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.tbl.nxv16i8(<vscale x 16 x i8> %a,
+                                                               <vscale x 16 x i8> %b)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @tbl_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: tbl_i16:
+; CHECK: tbl z0.h, { z0.h }, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.tbl.nxv8i16(<vscale x 8 x i16> %a,
+                                                               <vscale x 8 x i16> %b)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @tbl_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: tbl_i32:
+; CHECK: tbl z0.s, { z0.s }, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.tbl.nxv4i32(<vscale x 4 x i32> %a,
+                                                               <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @tbl_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: tbl_i64:
+; CHECK: tbl z0.d, { z0.d }, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.tbl.nxv2i64(<vscale x 2 x i64> %a,
+                                                               <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i64> %out
+}
+
+define <vscale x 8 x half> @tbl_f16(<vscale x 8 x half> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: tbl_f16:
+; CHECK: tbl z0.h, { z0.h }, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.tbl.nxv8f16(<vscale x 8 x half> %a,
+                                                                <vscale x 8 x i16> %b)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x float> @tbl_f32(<vscale x 4 x float> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: tbl_f32:
+; CHECK: tbl z0.s, { z0.s }, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.tbl.nxv4f32(<vscale x 4 x float> %a,
+                                                                 <vscale x 4 x i32> %b)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @tbl_f64(<vscale x 2 x double> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: tbl_f64:
+; CHECK: tbl z0.d, { z0.d }, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.tbl.nxv2f64(<vscale x 2 x double> %a,
+                                                                  <vscale x 2 x i64> %b)
+  ret <vscale x 2 x double> %out
+}
+
+;
+; UUNPKHI
+;
+
+define <vscale x 8 x i16> @uunpkhi_i16(<vscale x 16 x i8> %a) {
+; CHECK-LABEL: uunpkhi_i16
+; CHECK: uunpkhi z0.h, z0.b
+; CHECK-NEXT: ret
+  %res = call <vscale x 8 x i16> @llvm.aarch64.sve.uunpkhi.nxv8i16(<vscale x 16 x i8> %a)
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 4 x i32> @uunpkhi_i32(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: uunpkhi_i32
+; CHECK: uunpkhi z0.s, z0.h
+; CHECK-NEXT: ret
+  %res = call <vscale x 4 x i32> @llvm.aarch64.sve.uunpkhi.nxv4i32(<vscale x 8 x i16> %a)
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @uunpkhi_i64(<vscale x 4 x i32> %a) {
+; CHECK-LABEL:  uunpkhi_i64
+; CHECK: uunpkhi z0.d, z0.s
+; CHECK-NEXT: ret
+  %res = call <vscale x 2 x i64> @llvm.aarch64.sve.uunpkhi.nxv2i64(<vscale x 4 x i32> %a)
+  ret <vscale x 2 x i64> %res
+}
+
+;
+; UUNPKLO
+;
+
+define <vscale x 8 x i16> @uunpklo_i16(<vscale x 16 x i8> %a) {
+; CHECK-LABEL: uunpklo_i16
+; CHECK: uunpklo z0.h, z0.b
+; CHECK-NEXT: ret
+  %res = call <vscale x 8 x i16> @llvm.aarch64.sve.uunpklo.nxv8i16(<vscale x 16 x i8> %a)
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 4 x i32> @uunpklo_i32(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: uunpklo_i32
+; CHECK: uunpklo z0.s, z0.h
+; CHECK-NEXT: ret
+  %res = call <vscale x 4 x i32> @llvm.aarch64.sve.uunpklo.nxv4i32(<vscale x 8 x i16> %a)
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @uunpklo_i64(<vscale x 4 x i32> %a) {
+; CHECK-LABEL:  uunpklo_i64
+; CHECK: uunpklo z0.d, z0.s
+; CHECK-NEXT: ret
+  %res = call <vscale x 2 x i64> @llvm.aarch64.sve.uunpklo.nxv2i64(<vscale x 4 x i32> %a)
+  ret <vscale x 2 x i64> %res
+}
+
+;
+; TRN1
+;
+
+define <vscale x 16 x i1> @trn1_b8(<vscale x 16 x i1> %a, <vscale x 16 x i1> %b) {
+; CHECK-LABEL: trn1_b8:
+; CHECK: trn1 p0.b, p0.b, p1.b
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.trn1.nxv16i1(<vscale x 16 x i1> %a,
+                                                                <vscale x 16 x i1> %b)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @trn1_b16(<vscale x 8 x i1> %a, <vscale x 8 x i1> %b) {
+; CHECK-LABEL: trn1_b16:
+; CHECK: trn1 p0.h, p0.h, p1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.trn1.nxv8i1(<vscale x 8 x i1> %a,
+                                                              <vscale x 8 x i1> %b)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @trn1_b32(<vscale x 4 x i1> %a, <vscale x 4 x i1> %b) {
+; CHECK-LABEL: trn1_b32:
+; CHECK: trn1 p0.s, p0.s, p1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.trn1.nxv4i1(<vscale x 4 x i1> %a,
+                                                              <vscale x 4 x i1> %b)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 2 x i1> @trn1_b64(<vscale x 2 x i1> %a, <vscale x 2 x i1> %b) {
+; CHECK-LABEL: trn1_b64:
+; CHECK: trn1 p0.d, p0.d, p1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.trn1.nxv2i1(<vscale x 2 x i1> %a,
+                                                              <vscale x 2 x i1> %b)
+  ret <vscale x 2 x i1> %out
+}
+
+define <vscale x 16 x i8> @trn1_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: trn1_i8:
+; CHECK: trn1 z0.b, z0.b, z1.b
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.trn1.nxv16i8(<vscale x 16 x i8> %a,
+                                                                <vscale x 16 x i8> %b)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @trn1_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: trn1_i16:
+; CHECK: trn1 z0.h, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.trn1.nxv8i16(<vscale x 8 x i16> %a,
+                                                                <vscale x 8 x i16> %b)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @trn1_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: trn1_i32:
+; CHECK: trn1 z0.s, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.trn1.nxv4i32(<vscale x 4 x i32> %a,
+                                                                <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @trn1_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: trn1_i64:
+; CHECK: trn1 z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.trn1.nxv2i64(<vscale x 2 x i64> %a,
+                                                                <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i64> %out
+}
+
+define <vscale x 4 x half> @trn1_f16_v4(<vscale x 4 x half> %a, <vscale x 4 x half> %b) {
+; CHECK-LABEL: trn1_f16_v4:
+; CHECK: trn1 z0.s, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x half> @llvm.aarch64.sve.trn1.nxv4f16(<vscale x 4 x half> %a,
+                                                                 <vscale x 4 x half> %b)
+  ret <vscale x 4 x half> %out
+}
+
+define <vscale x 8 x half> @trn1_f16(<vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: trn1_f16:
+; CHECK: trn1 z0.h, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.trn1.nxv8f16(<vscale x 8 x half> %a,
+                                                                 <vscale x 8 x half> %b)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x float> @trn1_f32(<vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: trn1_f32:
+; CHECK: trn1 z0.s, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.trn1.nxv4f32(<vscale x 4 x float> %a,
+                                                                  <vscale x 4 x float> %b)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @trn1_f64(<vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: trn1_f64:
+; CHECK: trn1 z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.trn1.nxv2f64(<vscale x 2 x double> %a,
+                                                                   <vscale x 2 x double> %b)
+  ret <vscale x 2 x double> %out
+}
+
+;
+; TRN2
+;
+
+define <vscale x 16 x i1> @trn2_b8(<vscale x 16 x i1> %a, <vscale x 16 x i1> %b) {
+; CHECK-LABEL: trn2_b8:
+; CHECK: trn2 p0.b, p0.b, p1.b
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.trn2.nxv16i1(<vscale x 16 x i1> %a,
+                                                                <vscale x 16 x i1> %b)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @trn2_b16(<vscale x 8 x i1> %a, <vscale x 8 x i1> %b) {
+; CHECK-LABEL: trn2_b16:
+; CHECK: trn2 p0.h, p0.h, p1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.trn2.nxv8i1(<vscale x 8 x i1> %a,
+                                                              <vscale x 8 x i1> %b)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @trn2_b32(<vscale x 4 x i1> %a, <vscale x 4 x i1> %b) {
+; CHECK-LABEL: trn2_b32:
+; CHECK: trn2 p0.s, p0.s, p1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.trn2.nxv4i1(<vscale x 4 x i1> %a,
+                                                              <vscale x 4 x i1> %b)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 2 x i1> @trn2_b64(<vscale x 2 x i1> %a, <vscale x 2 x i1> %b) {
+; CHECK-LABEL: trn2_b64:
+; CHECK: trn2 p0.d, p0.d, p1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.trn2.nxv2i1(<vscale x 2 x i1> %a,
+                                                              <vscale x 2 x i1> %b)
+  ret <vscale x 2 x i1> %out
+}
+
+define <vscale x 16 x i8> @trn2_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: trn2_i8:
+; CHECK: trn2 z0.b, z0.b, z1.b
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.trn2.nxv16i8(<vscale x 16 x i8> %a,
+                                                                <vscale x 16 x i8> %b)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @trn2_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: trn2_i16:
+; CHECK: trn2 z0.h, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.trn2.nxv8i16(<vscale x 8 x i16> %a,
+                                                                <vscale x 8 x i16> %b)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @trn2_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: trn2_i32:
+; CHECK: trn2 z0.s, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.trn2.nxv4i32(<vscale x 4 x i32> %a,
+                                                                <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @trn2_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: trn2_i64:
+; CHECK: trn2 z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.trn2.nxv2i64(<vscale x 2 x i64> %a,
+                                                                <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i64> %out
+}
+
+define <vscale x 4 x half> @trn2_f16_v4(<vscale x 4 x half> %a, <vscale x 4 x half> %b) {
+; CHECK-LABEL: trn2_f16_v4:
+; CHECK: trn2 z0.s, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x half> @llvm.aarch64.sve.trn2.nxv4f16(<vscale x 4 x half> %a,
+                                                                 <vscale x 4 x half> %b)
+  ret <vscale x 4 x half> %out
+}
+
+define <vscale x 8 x half> @trn2_f16(<vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: trn2_f16:
+; CHECK: trn2 z0.h, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.trn2.nxv8f16(<vscale x 8 x half> %a,
+                                                                 <vscale x 8 x half> %b)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x float> @trn2_f32(<vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: trn2_f32:
+; CHECK: trn2 z0.s, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.trn2.nxv4f32(<vscale x 4 x float> %a,
+                                                                  <vscale x 4 x float> %b)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @trn2_f64(<vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: trn2_f64:
+; CHECK: trn2 z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.trn2.nxv2f64(<vscale x 2 x double> %a,
+                                                                   <vscale x 2 x double> %b)
+  ret <vscale x 2 x double> %out
+}
+
+;
+; UZP1
+;
+
+define <vscale x 16 x i1> @uzp1_b8(<vscale x 16 x i1> %a, <vscale x 16 x i1> %b) {
+; CHECK-LABEL: uzp1_b8:
+; CHECK: uzp1 p0.b, p0.b, p1.b
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.uzp1.nxv16i1(<vscale x 16 x i1> %a,
+                                                                <vscale x 16 x i1> %b)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @uzp1_b16(<vscale x 8 x i1> %a, <vscale x 8 x i1> %b) {
+; CHECK-LABEL: uzp1_b16:
+; CHECK: uzp1 p0.h, p0.h, p1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.uzp1.nxv8i1(<vscale x 8 x i1> %a,
+                                                              <vscale x 8 x i1> %b)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @uzp1_b32(<vscale x 4 x i1> %a, <vscale x 4 x i1> %b) {
+; CHECK-LABEL: uzp1_b32:
+; CHECK: uzp1 p0.s, p0.s, p1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.uzp1.nxv4i1(<vscale x 4 x i1> %a,
+                                                              <vscale x 4 x i1> %b)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 2 x i1> @uzp1_b64(<vscale x 2 x i1> %a, <vscale x 2 x i1> %b) {
+; CHECK-LABEL: uzp1_b64:
+; CHECK: uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.uzp1.nxv2i1(<vscale x 2 x i1> %a,
+                                                              <vscale x 2 x i1> %b)
+  ret <vscale x 2 x i1> %out
+}
+
+define <vscale x 16 x i8> @uzp1_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: uzp1_i8:
+; CHECK: uzp1 z0.b, z0.b, z1.b
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.uzp1.nxv16i8(<vscale x 16 x i8> %a,
+                                                                <vscale x 16 x i8> %b)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @uzp1_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: uzp1_i16:
+; CHECK: uzp1 z0.h, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.uzp1.nxv8i16(<vscale x 8 x i16> %a,
+                                                                <vscale x 8 x i16> %b)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @uzp1_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: uzp1_i32:
+; CHECK: uzp1 z0.s, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.uzp1.nxv4i32(<vscale x 4 x i32> %a,
+                                                                <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @uzp1_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: uzp1_i64:
+; CHECK: uzp1 z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.uzp1.nxv2i64(<vscale x 2 x i64> %a,
+                                                                <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i64> %out
+}
+
+define <vscale x 4 x half> @uzp1_f16_v4(<vscale x 4 x half> %a, <vscale x 4 x half> %b) {
+; CHECK-LABEL: uzp1_f16_v4:
+; CHECK: uzp1 z0.s, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x half> @llvm.aarch64.sve.uzp1.nxv4f16(<vscale x 4 x half> %a,
+                                                                 <vscale x 4 x half> %b)
+  ret <vscale x 4 x half> %out
+}
+
+define <vscale x 8 x half> @uzp1_f16(<vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: uzp1_f16:
+; CHECK: uzp1 z0.h, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.uzp1.nxv8f16(<vscale x 8 x half> %a,
+                                                                 <vscale x 8 x half> %b)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x float> @uzp1_f32(<vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: uzp1_f32:
+; CHECK: uzp1 z0.s, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.uzp1.nxv4f32(<vscale x 4 x float> %a,
+                                                                  <vscale x 4 x float> %b)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @uzp1_f64(<vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: uzp1_f64:
+; CHECK: uzp1 z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.uzp1.nxv2f64(<vscale x 2 x double> %a,
+                                                                   <vscale x 2 x double> %b)
+  ret <vscale x 2 x double> %out
+}
+
+;
+; UZP2
+;
+
+define <vscale x 16 x i1> @uzp2_b8(<vscale x 16 x i1> %a, <vscale x 16 x i1> %b) {
+; CHECK-LABEL: uzp2_b8:
+; CHECK: uzp2 p0.b, p0.b, p1.b
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.uzp2.nxv16i1(<vscale x 16 x i1> %a,
+                                                                <vscale x 16 x i1> %b)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @uzp2_b16(<vscale x 8 x i1> %a, <vscale x 8 x i1> %b) {
+; CHECK-LABEL: uzp2_b16:
+; CHECK: uzp2 p0.h, p0.h, p1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.uzp2.nxv8i1(<vscale x 8 x i1> %a,
+                                                              <vscale x 8 x i1> %b)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @uzp2_b32(<vscale x 4 x i1> %a, <vscale x 4 x i1> %b) {
+; CHECK-LABEL: uzp2_b32:
+; CHECK: uzp2 p0.s, p0.s, p1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.uzp2.nxv4i1(<vscale x 4 x i1> %a,
+                                                              <vscale x 4 x i1> %b)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 2 x i1> @uzp2_b64(<vscale x 2 x i1> %a, <vscale x 2 x i1> %b) {
+; CHECK-LABEL: uzp2_b64:
+; CHECK: uzp2 p0.d, p0.d, p1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.uzp2.nxv2i1(<vscale x 2 x i1> %a,
+                                                              <vscale x 2 x i1> %b)
+  ret <vscale x 2 x i1> %out
+}
+
+define <vscale x 16 x i8> @uzp2_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: uzp2_i8:
+; CHECK: uzp2 z0.b, z0.b, z1.b
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.uzp2.nxv16i8(<vscale x 16 x i8> %a,
+                                                                <vscale x 16 x i8> %b)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @uzp2_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: uzp2_i16:
+; CHECK: uzp2 z0.h, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.uzp2.nxv8i16(<vscale x 8 x i16> %a,
+                                                                <vscale x 8 x i16> %b)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @uzp2_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: uzp2_i32:
+; CHECK: uzp2 z0.s, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.uzp2.nxv4i32(<vscale x 4 x i32> %a,
+                                                                <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @uzp2_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: uzp2_i64:
+; CHECK: uzp2 z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.uzp2.nxv2i64(<vscale x 2 x i64> %a,
+                                                                <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i64> %out
+}
+
+define <vscale x 4 x half> @uzp2_f16_v4(<vscale x 4 x half> %a, <vscale x 4 x half> %b) {
+; CHECK-LABEL: uzp2_f16_v4:
+; CHECK: uzp2 z0.s, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x half> @llvm.aarch64.sve.uzp2.nxv4f16(<vscale x 4 x half> %a,
+                                                                 <vscale x 4 x half> %b)
+  ret <vscale x 4 x half> %out
+}
+
+define <vscale x 8 x half> @uzp2_f16(<vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: uzp2_f16:
+; CHECK: uzp2 z0.h, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.uzp2.nxv8f16(<vscale x 8 x half> %a,
+                                                                 <vscale x 8 x half> %b)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x float> @uzp2_f32(<vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: uzp2_f32:
+; CHECK: uzp2 z0.s, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.uzp2.nxv4f32(<vscale x 4 x float> %a,
+                                                                  <vscale x 4 x float> %b)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @uzp2_f64(<vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: uzp2_f64:
+; CHECK: uzp2 z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.uzp2.nxv2f64(<vscale x 2 x double> %a,
+                                                                   <vscale x 2 x double> %b)
+  ret <vscale x 2 x double> %out
+}
+
+;
+; ZIP1
+;
+
+define <vscale x 16 x i1> @zip1_b8(<vscale x 16 x i1> %a, <vscale x 16 x i1> %b) {
+; CHECK-LABEL: zip1_b8:
+; CHECK: zip1 p0.b, p0.b, p1.b
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.zip1.nxv16i1(<vscale x 16 x i1> %a,
+                                                                <vscale x 16 x i1> %b)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @zip1_b16(<vscale x 8 x i1> %a, <vscale x 8 x i1> %b) {
+; CHECK-LABEL: zip1_b16:
+; CHECK: zip1 p0.h, p0.h, p1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.zip1.nxv8i1(<vscale x 8 x i1> %a,
+                                                              <vscale x 8 x i1> %b)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @zip1_b32(<vscale x 4 x i1> %a, <vscale x 4 x i1> %b) {
+; CHECK-LABEL: zip1_b32:
+; CHECK: zip1 p0.s, p0.s, p1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.zip1.nxv4i1(<vscale x 4 x i1> %a,
+                                                              <vscale x 4 x i1> %b)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 2 x i1> @zip1_b64(<vscale x 2 x i1> %a, <vscale x 2 x i1> %b) {
+; CHECK-LABEL: zip1_b64:
+; CHECK: zip1 p0.d, p0.d, p1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.zip1.nxv2i1(<vscale x 2 x i1> %a,
+                                                              <vscale x 2 x i1> %b)
+  ret <vscale x 2 x i1> %out
+}
+
+define <vscale x 16 x i8> @zip1_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: zip1_i8:
+; CHECK: zip1 z0.b, z0.b, z1.b
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.zip1.nxv16i8(<vscale x 16 x i8> %a,
+                                                                <vscale x 16 x i8> %b)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @zip1_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: zip1_i16:
+; CHECK: zip1 z0.h, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.zip1.nxv8i16(<vscale x 8 x i16> %a,
+                                                                <vscale x 8 x i16> %b)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @zip1_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: zip1_i32:
+; CHECK: zip1 z0.s, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.zip1.nxv4i32(<vscale x 4 x i32> %a,
+                                                                <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @zip1_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: zip1_i64:
+; CHECK: zip1 z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.zip1.nxv2i64(<vscale x 2 x i64> %a,
+                                                                <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i64> %out
+}
+
+define <vscale x 4 x half> @zip1_f16_v4(<vscale x 4 x half> %a, <vscale x 4 x half> %b) {
+; CHECK-LABEL: zip1_f16_v4:
+; CHECK: zip1 z0.s, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x half> @llvm.aarch64.sve.zip1.nxv4f16(<vscale x 4 x half> %a,
+                                                                 <vscale x 4 x half> %b)
+  ret <vscale x 4 x half> %out
+}
+
+define <vscale x 8 x half> @zip1_f16(<vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: zip1_f16:
+; CHECK: zip1 z0.h, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.zip1.nxv8f16(<vscale x 8 x half> %a,
+                                                                 <vscale x 8 x half> %b)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x float> @zip1_f32(<vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: zip1_f32:
+; CHECK: zip1 z0.s, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.zip1.nxv4f32(<vscale x 4 x float> %a,
+                                                                  <vscale x 4 x float> %b)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @zip1_f64(<vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: zip1_f64:
+; CHECK: zip1 z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.zip1.nxv2f64(<vscale x 2 x double> %a,
+                                                                   <vscale x 2 x double> %b)
+  ret <vscale x 2 x double> %out
+}
+
+;
+; ZIP2
+;
+
+define <vscale x 16 x i1> @zip2_b8(<vscale x 16 x i1> %a, <vscale x 16 x i1> %b) {
+; CHECK-LABEL: zip2_b8:
+; CHECK: zip2 p0.b, p0.b, p1.b
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.zip2.nxv16i1(<vscale x 16 x i1> %a,
+                                                                <vscale x 16 x i1> %b)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @zip2_b16(<vscale x 8 x i1> %a, <vscale x 8 x i1> %b) {
+; CHECK-LABEL: zip2_b16:
+; CHECK: zip2 p0.h, p0.h, p1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.zip2.nxv8i1(<vscale x 8 x i1> %a,
+                                                              <vscale x 8 x i1> %b)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @zip2_b32(<vscale x 4 x i1> %a, <vscale x 4 x i1> %b) {
+; CHECK-LABEL: zip2_b32:
+; CHECK: zip2 p0.s, p0.s, p1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.zip2.nxv4i1(<vscale x 4 x i1> %a,
+                                                              <vscale x 4 x i1> %b)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 2 x i1> @zip2_b64(<vscale x 2 x i1> %a, <vscale x 2 x i1> %b) {
+; CHECK-LABEL: zip2_b64:
+; CHECK: zip2 p0.d, p0.d, p1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.zip2.nxv2i1(<vscale x 2 x i1> %a,
+                                                              <vscale x 2 x i1> %b)
+  ret <vscale x 2 x i1> %out
+}
+
+define <vscale x 16 x i8> @zip2_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: zip2_i8:
+; CHECK: zip2 z0.b, z0.b, z1.b
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.zip2.nxv16i8(<vscale x 16 x i8> %a,
+                                                                <vscale x 16 x i8> %b)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @zip2_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: zip2_i16:
+; CHECK: zip2 z0.h, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.zip2.nxv8i16(<vscale x 8 x i16> %a,
+                                                                <vscale x 8 x i16> %b)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @zip2_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: zip2_i32:
+; CHECK: zip2 z0.s, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.zip2.nxv4i32(<vscale x 4 x i32> %a,
+                                                                <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @zip2_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: zip2_i64:
+; CHECK: zip2 z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.zip2.nxv2i64(<vscale x 2 x i64> %a,
+                                                                <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i64> %out
+}
+
+define <vscale x 4 x half> @zip2_f16_v4(<vscale x 4 x half> %a, <vscale x 4 x half> %b) {
+; CHECK-LABEL: zip2_f16_v4:
+; CHECK: zip2 z0.s, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x half> @llvm.aarch64.sve.zip2.nxv4f16(<vscale x 4 x half> %a,
+                                                                 <vscale x 4 x half> %b)
+  ret <vscale x 4 x half> %out
+}
+
+define <vscale x 8 x half> @zip2_f16(<vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: zip2_f16:
+; CHECK: zip2 z0.h, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.zip2.nxv8f16(<vscale x 8 x half> %a,
+                                                                 <vscale x 8 x half> %b)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x float> @zip2_f32(<vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: zip2_f32:
+; CHECK: zip2 z0.s, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.zip2.nxv4f32(<vscale x 4 x float> %a,
+                                                                  <vscale x 4 x float> %b)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @zip2_f64(<vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: zip2_f64:
+; CHECK: zip2 z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.zip2.nxv2f64(<vscale x 2 x double> %a,
+                                                                   <vscale x 2 x double> %b)
+  ret <vscale x 2 x double> %out
+}
+
+declare <vscale x 16 x i8> @llvm.aarch64.sve.clasta.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 16 x i8>)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.clasta.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 8 x i16>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.clasta.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 4 x i32>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.clasta.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>, <vscale x 2 x i64>)
+declare <vscale x 8 x half> @llvm.aarch64.sve.clasta.nxv8f16(<vscale x 8 x i1>, <vscale x 8 x half>, <vscale x 8 x half>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.clasta.nxv4f32(<vscale x 4 x i1>, <vscale x 4 x float>, <vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.clasta.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>, <vscale x 2 x double>)
+
+declare i8 @llvm.aarch64.sve.clasta.n.nxv16i8(<vscale x 16 x i1>, i8, <vscale x 16 x i8>)
+declare i16 @llvm.aarch64.sve.clasta.n.nxv8i16(<vscale x 8 x i1>, i16, <vscale x 8 x i16>)
+declare i32 @llvm.aarch64.sve.clasta.n.nxv4i32(<vscale x 4 x i1>, i32, <vscale x 4 x i32>)
+declare i64 @llvm.aarch64.sve.clasta.n.nxv2i64(<vscale x 2 x i1>, i64, <vscale x 2 x i64>)
+declare half @llvm.aarch64.sve.clasta.n.nxv8f16(<vscale x 8 x i1>, half, <vscale x 8 x half>)
+declare float @llvm.aarch64.sve.clasta.n.nxv4f32(<vscale x 4 x i1>, float, <vscale x 4 x float>)
+declare double @llvm.aarch64.sve.clasta.n.nxv2f64(<vscale x 2 x i1>, double, <vscale x 2 x double>)
+
+declare <vscale x 16 x i8> @llvm.aarch64.sve.clastb.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 16 x i8>)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.clastb.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 8 x i16>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.clastb.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 4 x i32>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.clastb.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>, <vscale x 2 x i64>)
+declare <vscale x 8 x half> @llvm.aarch64.sve.clastb.nxv8f16(<vscale x 8 x i1>, <vscale x 8 x half>, <vscale x 8 x half>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.clastb.nxv4f32(<vscale x 4 x i1>, <vscale x 4 x float>, <vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.clastb.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>, <vscale x 2 x double>)
+
+declare i8 @llvm.aarch64.sve.clastb.n.nxv16i8(<vscale x 16 x i1>, i8, <vscale x 16 x i8>)
+declare i16 @llvm.aarch64.sve.clastb.n.nxv8i16(<vscale x 8 x i1>, i16, <vscale x 8 x i16>)
+declare i32 @llvm.aarch64.sve.clastb.n.nxv4i32(<vscale x 4 x i1>, i32, <vscale x 4 x i32>)
+declare i64 @llvm.aarch64.sve.clastb.n.nxv2i64(<vscale x 2 x i1>, i64, <vscale x 2 x i64>)
+declare half @llvm.aarch64.sve.clastb.n.nxv8f16(<vscale x 8 x i1>, half, <vscale x 8 x half>)
+declare float @llvm.aarch64.sve.clastb.n.nxv4f32(<vscale x 4 x i1>, float, <vscale x 4 x float>)
+declare double @llvm.aarch64.sve.clastb.n.nxv2f64(<vscale x 2 x i1>, double, <vscale x 2 x double>)
+
+declare <vscale x 4 x i32> @llvm.aarch64.sve.compact.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.compact.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.compact.nxv4f32(<vscale x 4 x i1>, <vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.compact.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>)
+
+declare <vscale x 16 x i8> @llvm.aarch64.sve.ext.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i8>, i32)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.ext.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i16>, i32)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.ext.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>, i32)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.ext.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>, i32)
+declare <vscale x 8 x half> @llvm.aarch64.sve.ext.nxv8f16(<vscale x 8 x half>, <vscale x 8 x half>, i32)
+declare <vscale x 4 x float> @llvm.aarch64.sve.ext.nxv4f32(<vscale x 4 x float>, <vscale x 4 x float>, i32)
+declare <vscale x 2 x double> @llvm.aarch64.sve.ext.nxv2f64(<vscale x 2 x double>, <vscale x 2 x double>, i32)
+
+declare i8 @llvm.aarch64.sve.lasta.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>)
+declare i16 @llvm.aarch64.sve.lasta.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>)
+declare i32 @llvm.aarch64.sve.lasta.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>)
+declare i64 @llvm.aarch64.sve.lasta.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>)
+declare half @llvm.aarch64.sve.lasta.nxv8f16(<vscale x 8 x i1>, <vscale x 8 x half>)
+declare float @llvm.aarch64.sve.lasta.nxv2f32(<vscale x 2 x i1>, <vscale x 2 x float>)
+declare float @llvm.aarch64.sve.lasta.nxv4f32(<vscale x 4 x i1>, <vscale x 4 x float>)
+declare double @llvm.aarch64.sve.lasta.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>)
+
+declare i8 @llvm.aarch64.sve.lastb.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>)
+declare i16 @llvm.aarch64.sve.lastb.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>)
+declare i32 @llvm.aarch64.sve.lastb.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>)
+declare i64 @llvm.aarch64.sve.lastb.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>)
+declare half @llvm.aarch64.sve.lastb.nxv8f16(<vscale x 8 x i1>, <vscale x 8 x half>)
+declare float @llvm.aarch64.sve.lastb.nxv2f32(<vscale x 2 x i1>, <vscale x 2 x float>)
+declare float @llvm.aarch64.sve.lastb.nxv4f32(<vscale x 4 x i1>, <vscale x 4 x float>)
+declare double @llvm.aarch64.sve.lastb.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>)
+
+declare <vscale x 16 x i1> @llvm.aarch64.sve.rev.nxv16i1(<vscale x 16 x i1>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.rev.nxv8i1(<vscale x 8 x i1>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.rev.nxv4i1(<vscale x 4 x i1>)
+declare <vscale x 2 x i1> @llvm.aarch64.sve.rev.nxv2i1(<vscale x 2 x i1>)
+declare <vscale x 16 x i8> @llvm.aarch64.sve.rev.nxv16i8(<vscale x 16 x i8>)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.rev.nxv8i16(<vscale x 8 x i16>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.rev.nxv4i32(<vscale x 4 x i32>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.rev.nxv2i64(<vscale x 2 x i64>)
+declare <vscale x 8 x half> @llvm.aarch64.sve.rev.nxv8f16(<vscale x 8 x half>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.rev.nxv4f32(<vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.rev.nxv2f64(<vscale x 2 x double>)
+
+declare <vscale x 16 x i8> @llvm.aarch64.sve.splice.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 16 x i8>)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.splice.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 8 x i16>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.splice.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 4 x i32>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.splice.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>, <vscale x 2 x i64>)
+declare <vscale x 8 x half> @llvm.aarch64.sve.splice.nxv8f16(<vscale x 8 x i1>, <vscale x 8 x half>, <vscale x 8 x half>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.splice.nxv4f32(<vscale x 4 x i1>, <vscale x 4 x float>, <vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.splice.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>, <vscale x 2 x double>)
+
+declare <vscale x 8 x i16> @llvm.aarch64.sve.sunpkhi.nxv8i16(<vscale x 16 x i8>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.sunpkhi.nxv4i32(<vscale x 8 x i16>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.sunpkhi.nxv2i64(<vscale x 4 x i32>)
+
+declare <vscale x 8 x i16> @llvm.aarch64.sve.sunpklo.nxv8i16(<vscale x 16 x i8>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.sunpklo.nxv4i32(<vscale x 8 x i16>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.sunpklo.nxv2i64(<vscale x 4 x i32>)
+
+declare <vscale x 16 x i8> @llvm.aarch64.sve.tbl.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i8>)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.tbl.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i16>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.tbl.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.tbl.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>)
+declare <vscale x 8 x half> @llvm.aarch64.sve.tbl.nxv8f16(<vscale x 8 x half>, <vscale x 8 x i16>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.tbl.nxv4f32(<vscale x 4 x float>, <vscale x 4 x i32>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.tbl.nxv2f64(<vscale x 2 x double>, <vscale x 2 x i64>)
+
+declare <vscale x 8 x i16> @llvm.aarch64.sve.uunpkhi.nxv8i16(<vscale x 16 x i8>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.uunpkhi.nxv4i32(<vscale x 8 x i16>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.uunpkhi.nxv2i64(<vscale x 4 x i32>)
+
+declare <vscale x 8 x i16> @llvm.aarch64.sve.uunpklo.nxv8i16(<vscale x 16 x i8>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.uunpklo.nxv4i32(<vscale x 8 x i16>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.uunpklo.nxv2i64(<vscale x 4 x i32>)
+
+declare <vscale x 16 x i1> @llvm.aarch64.sve.trn1.nxv16i1(<vscale x 16 x i1>, <vscale x 16 x i1>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.trn1.nxv8i1(<vscale x 8 x i1>, <vscale x 8 x i1>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.trn1.nxv4i1(<vscale x 4 x i1>, <vscale x 4 x i1>)
+declare <vscale x 2 x i1> @llvm.aarch64.sve.trn1.nxv2i1(<vscale x 2 x i1>, <vscale x 2 x i1>)
+declare <vscale x 16 x i8> @llvm.aarch64.sve.trn1.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i8>)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.trn1.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i16>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.trn1.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.trn1.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>)
+declare <vscale x 4 x half> @llvm.aarch64.sve.trn1.nxv4f16(<vscale x 4 x half>, <vscale x 4 x half>)
+declare <vscale x 8 x half> @llvm.aarch64.sve.trn1.nxv8f16(<vscale x 8 x half>, <vscale x 8 x half>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.trn1.nxv4f32(<vscale x 4 x float>, <vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.trn1.nxv2f64(<vscale x 2 x double>, <vscale x 2 x double>)
+
+declare <vscale x 16 x i1> @llvm.aarch64.sve.trn2.nxv16i1(<vscale x 16 x i1>, <vscale x 16 x i1>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.trn2.nxv8i1(<vscale x 8 x i1>, <vscale x 8 x i1>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.trn2.nxv4i1(<vscale x 4 x i1>, <vscale x 4 x i1>)
+declare <vscale x 2 x i1> @llvm.aarch64.sve.trn2.nxv2i1(<vscale x 2 x i1>, <vscale x 2 x i1>)
+declare <vscale x 16 x i8> @llvm.aarch64.sve.trn2.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i8>)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.trn2.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i16>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.trn2.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.trn2.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>)
+declare <vscale x 4 x half> @llvm.aarch64.sve.trn2.nxv4f16(<vscale x 4 x half>, <vscale x 4 x half>)
+declare <vscale x 8 x half> @llvm.aarch64.sve.trn2.nxv8f16(<vscale x 8 x half>, <vscale x 8 x half>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.trn2.nxv4f32(<vscale x 4 x float>, <vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.trn2.nxv2f64(<vscale x 2 x double>, <vscale x 2 x double>)
+
+declare <vscale x 16 x i1> @llvm.aarch64.sve.uzp1.nxv16i1(<vscale x 16 x i1>, <vscale x 16 x i1>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.uzp1.nxv8i1(<vscale x 8 x i1>, <vscale x 8 x i1>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.uzp1.nxv4i1(<vscale x 4 x i1>, <vscale x 4 x i1>)
+declare <vscale x 2 x i1> @llvm.aarch64.sve.uzp1.nxv2i1(<vscale x 2 x i1>, <vscale x 2 x i1>)
+declare <vscale x 16 x i8> @llvm.aarch64.sve.uzp1.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i8>)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.uzp1.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i16>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.uzp1.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.uzp1.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>)
+declare <vscale x 4 x half> @llvm.aarch64.sve.uzp1.nxv4f16(<vscale x 4 x half>, <vscale x 4 x half>)
+declare <vscale x 8 x half> @llvm.aarch64.sve.uzp1.nxv8f16(<vscale x 8 x half>, <vscale x 8 x half>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.uzp1.nxv4f32(<vscale x 4 x float>, <vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.uzp1.nxv2f64(<vscale x 2 x double>, <vscale x 2 x double>)
+
+declare <vscale x 16 x i1> @llvm.aarch64.sve.uzp2.nxv16i1(<vscale x 16 x i1>, <vscale x 16 x i1>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.uzp2.nxv8i1(<vscale x 8 x i1>, <vscale x 8 x i1>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.uzp2.nxv4i1(<vscale x 4 x i1>, <vscale x 4 x i1>)
+declare <vscale x 2 x i1> @llvm.aarch64.sve.uzp2.nxv2i1(<vscale x 2 x i1>, <vscale x 2 x i1>)
+declare <vscale x 16 x i8> @llvm.aarch64.sve.uzp2.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i8>)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.uzp2.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i16>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.uzp2.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.uzp2.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>)
+declare <vscale x 4 x half> @llvm.aarch64.sve.uzp2.nxv4f16(<vscale x 4 x half>, <vscale x 4 x half>)
+declare <vscale x 8 x half> @llvm.aarch64.sve.uzp2.nxv8f16(<vscale x 8 x half>, <vscale x 8 x half>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.uzp2.nxv4f32(<vscale x 4 x float>, <vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.uzp2.nxv2f64(<vscale x 2 x double>, <vscale x 2 x double>)
+
+declare <vscale x 16 x i1> @llvm.aarch64.sve.zip1.nxv16i1(<vscale x 16 x i1>, <vscale x 16 x i1>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.zip1.nxv8i1(<vscale x 8 x i1>, <vscale x 8 x i1>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.zip1.nxv4i1(<vscale x 4 x i1>, <vscale x 4 x i1>)
+declare <vscale x 2 x i1> @llvm.aarch64.sve.zip1.nxv2i1(<vscale x 2 x i1>, <vscale x 2 x i1>)
+declare <vscale x 16 x i8> @llvm.aarch64.sve.zip1.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i8>)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.zip1.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i16>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.zip1.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.zip1.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>)
+declare <vscale x 4 x half> @llvm.aarch64.sve.zip1.nxv4f16(<vscale x 4 x half>, <vscale x 4 x half>)
+declare <vscale x 8 x half> @llvm.aarch64.sve.zip1.nxv8f16(<vscale x 8 x half>, <vscale x 8 x half>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.zip1.nxv4f32(<vscale x 4 x float>, <vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.zip1.nxv2f64(<vscale x 2 x double>, <vscale x 2 x double>)
+
+declare <vscale x 16 x i1> @llvm.aarch64.sve.zip2.nxv16i1(<vscale x 16 x i1>, <vscale x 16 x i1>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.zip2.nxv8i1(<vscale x 8 x i1>, <vscale x 8 x i1>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.zip2.nxv4i1(<vscale x 4 x i1>, <vscale x 4 x i1>)
+declare <vscale x 2 x i1> @llvm.aarch64.sve.zip2.nxv2i1(<vscale x 2 x i1>, <vscale x 2 x i1>)
+declare <vscale x 16 x i8> @llvm.aarch64.sve.zip2.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i8>)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.zip2.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i16>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.zip2.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.zip2.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>)
+declare <vscale x 4 x half> @llvm.aarch64.sve.zip2.nxv4f16(<vscale x 4 x half>, <vscale x 4 x half>)
+declare <vscale x 8 x half> @llvm.aarch64.sve.zip2.nxv8f16(<vscale x 8 x half>, <vscale x 8 x half>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.zip2.nxv4f32(<vscale x 4 x float>, <vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.zip2.nxv2f64(<vscale x 2 x double>, <vscale x 2 x double>)
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-pred-creation.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-pred-creation.ll
new file mode 100644
index 000000000..b7131d319
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-pred-creation.ll
@@ -0,0 +1,42 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+;
+; PTRUE
+;
+
+define <vscale x 16 x i1> @ptrue_b8() {
+; CHECK-LABEL: ptrue_b8:
+; CHECK: ptrue p0.b, pow2
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 0)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @ptrue_b16() {
+; CHECK-LABEL: ptrue_b16:
+; CHECK: ptrue p0.h, vl1
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 1)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @ptrue_b32() {
+; CHECK-LABEL: ptrue_b32:
+; CHECK: ptrue p0.s, mul3
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 30)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 2 x i1> @ptrue_b64() {
+; CHECK-LABEL: ptrue_b64:
+; CHECK: ptrue p0.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  ret <vscale x 2 x i1> %out
+}
+
+declare <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 %pattern)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 %pattern)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 %pattern)
+declare <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 %pattern)
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-pred-operations.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-pred-operations.ll
new file mode 100644
index 000000000..cf7a8dc79
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-pred-operations.ll
@@ -0,0 +1,130 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+;
+; PFIRST
+;
+
+define <vscale x 16 x i1> @pfirst_b8(<vscale x 16 x i1> %pg, <vscale x 16 x i1> %a) {
+; CHECK-LABEL: pfirst_b8:
+; CHECK: pfirst p1.b, p0, p1.b
+; CHECK-NEXT: mov p0.b, p1.b
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.pfirst.nxv16i1(<vscale x 16 x i1> %pg,
+                                                                  <vscale x 16 x i1> %a)
+  ret <vscale x 16 x i1> %out
+}
+
+;
+; PNEXT
+;
+
+define <vscale x 16 x i1> @pnext_b8(<vscale x 16 x i1> %pg, <vscale x 16 x i1> %a) {
+; CHECK-LABEL: pnext_b8:
+; CHECK: pnext p1.b, p0, p1.b
+; CHECK-NEXT: mov p0.b, p1.b
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.pnext.nxv16i1(<vscale x 16 x i1> %pg,
+                                                                 <vscale x 16 x i1> %a)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @pnext_b16(<vscale x 8 x i1> %pg, <vscale x 8 x i1> %a) {
+; CHECK-LABEL: pnext_b16:
+; CHECK: pnext p1.h, p0, p1.h
+; CHECK-NEXT: mov p0.b, p1.b
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.pnext.nxv8i1(<vscale x 8 x i1> %pg,
+                                                               <vscale x 8 x i1> %a)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @pnext_b32(<vscale x 4 x i1> %pg, <vscale x 4 x i1> %a) {
+; CHECK-LABEL: pnext_b32:
+; CHECK: pnext p1.s, p0, p1.s
+; CHECK-NEXT: mov p0.b, p1.b
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.pnext.nxv4i1(<vscale x 4 x i1> %pg,
+                                                               <vscale x 4 x i1> %a)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 2 x i1> @pnext_b64(<vscale x 2 x i1> %pg, <vscale x 2 x i1> %a) {
+; CHECK-LABEL: pnext_b64:
+; CHECK: pnext p1.d, p0, p1.d
+; CHECK-NEXT: mov p0.b, p1.b
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.pnext.nxv2i1(<vscale x 2 x i1> %pg,
+                                                               <vscale x 2 x i1> %a)
+  ret <vscale x 2 x i1> %out
+}
+
+;
+; PUNPKHI
+;
+
+define <vscale x 8 x i1> @punpkhi_b16(<vscale x 16 x i1> %a) {
+; CHECK-LABEL: punpkhi_b16
+; CHECK: punpkhi p0.h, p0.b
+; CHECK-NEXT: ret
+  %res = call <vscale x 8 x i1> @llvm.aarch64.sve.punpkhi.nxv8i1(<vscale x 16 x i1> %a)
+  ret <vscale x 8 x i1> %res
+}
+
+define <vscale x 4 x i1> @punpkhi_b8(<vscale x 8 x i1> %a) {
+; CHECK-LABEL: punpkhi_b8
+; CHECK: punpkhi p0.h, p0.b
+; CHECK-NEXT: ret
+  %res = call <vscale x 4 x i1> @llvm.aarch64.sve.punpkhi.nxv4i1(<vscale x 8 x i1> %a)
+  ret <vscale x 4 x i1> %res
+}
+
+define <vscale x 2 x i1> @punpkhi_b4(<vscale x 4 x i1> %a) {
+; CHECK-LABEL: punpkhi_b4
+; CHECK: punpkhi p0.h, p0.b
+; CHECK-NEXT: ret
+  %res = call <vscale x 2 x i1> @llvm.aarch64.sve.punpkhi.nxv2i1(<vscale x 4 x i1> %a)
+  ret <vscale x 2 x i1> %res
+}
+
+;
+; PUNPKLO
+;
+
+define <vscale x 8 x i1> @punpklo_b16(<vscale x 16 x i1> %a) {
+; CHECK-LABEL: punpklo_b16
+; CHECK: punpklo p0.h, p0.b
+; CHECK-NEXT: ret
+  %res = call <vscale x 8 x i1> @llvm.aarch64.sve.punpklo.nxv8i1(<vscale x 16 x i1> %a)
+  ret <vscale x 8 x i1> %res
+}
+
+define <vscale x 4 x i1> @punpklo_b8(<vscale x 8 x i1> %a) {
+; CHECK-LABEL: punpklo_b8
+; CHECK: punpklo p0.h, p0.b
+; CHECK-NEXT: ret
+  %res = call <vscale x 4 x i1> @llvm.aarch64.sve.punpklo.nxv4i1(<vscale x 8 x i1> %a)
+  ret <vscale x 4 x i1> %res
+}
+
+define <vscale x 2 x i1> @punpklo_b4(<vscale x 4 x i1> %a) {
+; CHECK-LABEL: punpklo_b4
+; CHECK: punpklo p0.h, p0.b
+; CHECK-NEXT: ret
+  %res = call <vscale x 2 x i1> @llvm.aarch64.sve.punpklo.nxv2i1(<vscale x 4 x i1> %a)
+  ret <vscale x 2 x i1> %res
+}
+
+declare <vscale x 16 x i1> @llvm.aarch64.sve.pfirst.nxv16i1(<vscale x 16 x i1>, <vscale x 16 x i1>)
+
+declare <vscale x 16 x i1> @llvm.aarch64.sve.pnext.nxv16i1(<vscale x 16 x i1>, <vscale x 16 x i1>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.pnext.nxv8i1(<vscale x 8 x i1>, <vscale x 8 x i1>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.pnext.nxv4i1(<vscale x 4 x i1>, <vscale x 4 x i1>)
+declare <vscale x 2 x i1> @llvm.aarch64.sve.pnext.nxv2i1(<vscale x 2 x i1>, <vscale x 2 x i1>)
+
+declare <vscale x 8 x i1> @llvm.aarch64.sve.punpkhi.nxv8i1(<vscale x 16 x i1>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.punpkhi.nxv4i1(<vscale x 8 x i1>)
+declare <vscale x 2 x i1> @llvm.aarch64.sve.punpkhi.nxv2i1(<vscale x 4 x i1>)
+
+declare <vscale x 8 x i1> @llvm.aarch64.sve.punpklo.nxv8i1(<vscale x 16 x i1>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.punpklo.nxv4i1(<vscale x 8 x i1>)
+declare <vscale x 2 x i1> @llvm.aarch64.sve.punpklo.nxv2i1(<vscale x 4 x i1>)
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-pred-testing.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-pred-testing.ll
new file mode 100644
index 000000000..03093851c
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-pred-testing.ll
@@ -0,0 +1,36 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+;
+; PTEST
+;
+
+define i1 @ptest_any(<vscale x 16 x i1> %pg, <vscale x 16 x i1> %a) {
+; CHECK-LABEL: ptest_any:
+; CHECK: ptest p0, p1.b
+; CHECK-NEXT: cset w0, ne
+; CHECK-NEXT: ret
+  %out = call i1 @llvm.aarch64.sve.ptest.any(<vscale x 16 x i1> %pg, <vscale x 16 x i1> %a)
+  ret i1 %out
+}
+
+define i1 @ptest_first(<vscale x 16 x i1> %pg, <vscale x 16 x i1> %a) {
+; CHECK-LABEL: ptest_first:
+; CHECK: ptest p0, p1.b
+; CHECK-NEXT: cset w0, mi
+; CHECK-NEXT: ret
+  %out = call i1 @llvm.aarch64.sve.ptest.first(<vscale x 16 x i1> %pg, <vscale x 16 x i1> %a)
+  ret i1 %out
+}
+
+define i1 @ptest_last(<vscale x 16 x i1> %pg, <vscale x 16 x i1> %a) {
+; CHECK-LABEL: ptest_last:
+; CHECK: ptest p0, p1.b
+; CHECK-NEXT: cset w0, lo
+; CHECK-NEXT: ret
+  %out = call i1 @llvm.aarch64.sve.ptest.last(<vscale x 16 x i1> %pg, <vscale x 16 x i1> %a)
+  ret i1 %out
+}
+
+declare i1 @llvm.aarch64.sve.ptest.any(<vscale x 16 x i1> %pg, <vscale x 16 x i1> %a)
+declare i1 @llvm.aarch64.sve.ptest.first(<vscale x 16 x i1> %pg, <vscale x 16 x i1> %a)
+declare i1 @llvm.aarch64.sve.ptest.last(<vscale x 16 x i1> %pg, <vscale x 16 x i1> %a)
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-reversal.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-reversal.ll
new file mode 100644
index 000000000..69adf7fc6
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-reversal.ll
@@ -0,0 +1,166 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+;
+; RBIT
+;
+
+define <vscale x 16 x i8> @rbit_i8(<vscale x 16 x i8> %a, <vscale x 16 x i1> %pg, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: rbit_i8:
+; CHECK: rbit z0.b, p0/m, z1.b
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.rbit.nxv16i8(<vscale x 16 x i8> %a,
+                                                                <vscale x 16 x i1> %pg,
+                                                                <vscale x 16 x i8> %b)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @rbit_i16(<vscale x 8 x i16> %a, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: rbit_i16:
+; CHECK: rbit z0.h, p0/m, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.rbit.nxv8i16(<vscale x 8 x i16> %a,
+                                                                <vscale x 8 x i1> %pg,
+                                                                <vscale x 8 x i16> %b)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @rbit_i32(<vscale x 4 x i32> %a, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: rbit_i32:
+; CHECK: rbit z0.s, p0/m, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.rbit.nxv4i32(<vscale x 4 x i32> %a,
+                                                                <vscale x 4 x i1> %pg,
+                                                                <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @rbit_i64(<vscale x 2 x i64> %a, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: rbit_i64:
+; CHECK: rbit z0.d, p0/m, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.rbit.nxv2i64(<vscale x 2 x i64> %a,
+                                                                <vscale x 2 x i1> %pg,
+                                                                <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i64> %out
+}
+
+;
+; REVB
+;
+
+define <vscale x 8 x i16> @revb_i16(<vscale x 8 x i16> %a, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: revb_i16:
+; CHECK: revb z0.h, p0/m, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.revb.nxv8i16(<vscale x 8 x i16> %a,
+                                                                <vscale x 8 x i1> %pg,
+                                                                <vscale x 8 x i16> %b)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @revb_i32(<vscale x 4 x i32> %a, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: revb_i32:
+; CHECK: revb z0.s, p0/m, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.revb.nxv4i32(<vscale x 4 x i32> %a,
+                                                                <vscale x 4 x i1> %pg,
+                                                                <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @revb_i64(<vscale x 2 x i64> %a, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: revb_i64:
+; CHECK: revb z0.d, p0/m, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.revb.nxv2i64(<vscale x 2 x i64> %a,
+                                                                <vscale x 2 x i1> %pg,
+                                                                <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i64> %out
+}
+
+;
+; REVB (bswap)
+;
+
+define <vscale x 8 x i16> @revb_i16_bswap(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: revb_i16_bswap:
+; CHECK: ptrue [[PG:p[0-9]+]].h
+; CHECK-NEXT: revb z0.h, [[PG]]/m, z0.h
+; CHECK-NEXT: ret
+  %res = call <vscale x 8 x i16> @llvm.bswap.nxv8i16(<vscale x 8 x i16> %a)
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 4 x i32> @revb_i32_bswap(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: revb_i32_bswap:
+; CHECK: ptrue [[PG:p[0-9]+]].s
+; CHECK-NEXT: revb z0.s, [[PG]]/m, z0.s
+; CHECK-NEXT: ret
+  %res = call <vscale x 4 x i32> @llvm.bswap.nxv4i32(<vscale x 4 x i32> %a)
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @revb_i64_bswap(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: revb_i64_bswap:
+; CHECK: ptrue [[PG:p[0-9]+]].d
+; CHECK-NEXT: revb z0.d, [[PG]]/m, z0.d
+; CHECK-NEXT: ret
+  %res = call <vscale x 2 x i64> @llvm.bswap.nxv2i64(<vscale x 2 x i64> %a)
+  ret <vscale x 2 x i64> %res
+}
+
+;
+; REVH
+;
+
+define <vscale x 4 x i32> @revh_i32(<vscale x 4 x i32> %a, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: revh_i32:
+; CHECK: revh z0.s, p0/m, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.revh.nxv4i32(<vscale x 4 x i32> %a,
+                                                                <vscale x 4 x i1> %pg,
+                                                                <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @revh_i64(<vscale x 2 x i64> %a, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: revh_i64:
+; CHECK: revh z0.d, p0/m, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.revh.nxv2i64(<vscale x 2 x i64> %a,
+                                                                <vscale x 2 x i1> %pg,
+                                                                <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i64> %out
+}
+
+;
+; REVW
+;
+
+define <vscale x 2 x i64> @revw_i64(<vscale x 2 x i64> %a, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: revw_i64:
+; CHECK: revw z0.d, p0/m, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.revw.nxv2i64(<vscale x 2 x i64> %a,
+                                                                <vscale x 2 x i1> %pg,
+                                                                <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i64> %out
+}
+
+declare <vscale x 16 x i8> @llvm.aarch64.sve.rbit.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i1>, <vscale x 16 x i8>)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.rbit.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i1>, <vscale x 8 x i16>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.rbit.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i1>, <vscale x 4 x i32>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.rbit.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i1>, <vscale x 2 x i64>)
+
+declare <vscale x 8 x i16> @llvm.aarch64.sve.revb.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i1>, <vscale x 8 x i16>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.revb.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i1>, <vscale x 4 x i32>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.revb.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i1>, <vscale x 2 x i64>)
+
+declare <vscale x 8 x i16> @llvm.bswap.nxv8i16(<vscale x 8 x i16>)
+declare <vscale x 4 x i32> @llvm.bswap.nxv4i32(<vscale x 4 x i32>)
+declare <vscale x 2 x i64> @llvm.bswap.nxv2i64(<vscale x 2 x i64>)
+
+declare <vscale x 4 x i32> @llvm.aarch64.sve.revh.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i1>, <vscale x 4 x i32>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.revh.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i1>, <vscale x 2 x i64>)
+
+declare <vscale x 2 x i64> @llvm.aarch64.sve.revw.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i1>, <vscale x 2 x i64>)
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-scatter-stores-32bit-scaled-offsets.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-scatter-stores-32bit-scaled-offsets.ll
new file mode 100644
index 000000000..380a158b7
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-scatter-stores-32bit-scaled-offsets.ll
@@ -0,0 +1,193 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+;
+; ST1H, ST1W, ST1D: base + 32-bit scaled offset, sign (sxtw) or zero
+; (uxtw) extended to 64 bits.
+;   e.g. st1h { z0.d }, p0, [x0, z1.d, uxtw #1]
+;
+
+; ST1H
+define void @sst1h_s_uxtw(<vscale x 4 x i32> %data, <vscale x 4 x i1> %pg, i16* %base, <vscale x 4 x i32> %indices) {
+; CHECK-LABEL: sst1h_s_uxtw:
+; CHECK: st1h { z0.s }, p0, [x0, z1.s, uxtw #1]
+; CHECK-NEXT:	ret
+  %data_trunc = trunc <vscale x 4 x i32> %data to <vscale x 4 x i16>
+  call void @llvm.aarch64.sve.st1.scatter.uxtw.index.nxv4i16(<vscale x 4 x i16> %data_trunc,
+                                                             <vscale x 4 x i1> %pg,
+                                                             i16* %base,
+                                                             <vscale x 4 x i32> %indices)
+  ret void
+}
+
+define void @sst1h_s_sxtw(<vscale x 4 x i32> %data, <vscale x 4 x i1> %pg, i16* %base, <vscale x 4 x i32> %indices) {
+; CHECK-LABEL: sst1h_s_sxtw:
+; CHECK: st1h { z0.s }, p0, [x0, z1.s, sxtw #1]
+; CHECK-NEXT: ret
+  %data_trunc = trunc <vscale x 4 x i32> %data to <vscale x 4 x i16>
+  call void @llvm.aarch64.sve.st1.scatter.sxtw.index.nxv4i16(<vscale x 4 x i16> %data_trunc,
+                                                             <vscale x 4 x i1> %pg,
+                                                             i16* %base,
+                                                             <vscale x 4 x i32> %indices)
+  ret void
+}
+
+define void @sst1h_d_uxtw(<vscale x 2 x i64> %data, <vscale x 2 x i1> %pg, i16* %base, <vscale x 2 x i32> %indices) {
+; CHECK-LABEL: sst1h_d_uxtw:
+; CHECK: st1h { z0.d }, p0, [x0, z1.d, uxtw #1]
+; CHECK-NEXT: ret
+  %data_trunc = trunc <vscale x 2 x i64> %data to <vscale x 2 x i16>
+  call void @llvm.aarch64.sve.st1.scatter.uxtw.index.nxv2i16(<vscale x 2 x i16> %data_trunc,
+                                                             <vscale x 2 x i1> %pg,
+                                                             i16* %base,
+                                                             <vscale x 2 x i32> %indices)
+  ret void
+}
+
+define void @sst1h_d_sxtw(<vscale x 2 x i64> %data, <vscale x 2 x i1> %pg, i16* %base, <vscale x 2 x i32> %indices) {
+; CHECK-LABEL: sst1h_d_sxtw:
+; CHECK: st1h { z0.d }, p0, [x0, z1.d, sxtw #1]
+; CHECK-NEXT: ret
+  %data_trunc = trunc <vscale x 2 x i64> %data to <vscale x 2 x i16>
+  call void @llvm.aarch64.sve.st1.scatter.sxtw.index.nxv2i16(<vscale x 2 x i16> %data_trunc,
+                                                             <vscale x 2 x i1> %pg,
+                                                             i16* %base,
+                                                             <vscale x 2 x i32> %indices)
+  ret void
+}
+
+; ST1W
+define void @sst1w_s_uxtw(<vscale x 4 x i32> %data, <vscale x 4 x i1> %pg, i32* %base, <vscale x 4 x i32> %indices) {
+; CHECK-LABEL: sst1w_s_uxtw:
+; CHECK: st1w { z0.s }, p0, [x0, z1.s, uxtw #2]
+; CHECK-NEXT:	ret
+  call void @llvm.aarch64.sve.st1.scatter.uxtw.index.nxv4i32(<vscale x 4 x i32> %data,
+                                                             <vscale x 4 x i1> %pg,
+                                                             i32* %base,
+                                                             <vscale x 4 x i32> %indices)
+  ret void
+}
+
+define void @sst1w_s_sxtw(<vscale x 4 x i32> %data, <vscale x 4 x i1> %pg, i32* %base, <vscale x 4 x i32> %indices) {
+; CHECK-LABEL: sst1w_s_sxtw:
+; CHECK: st1w { z0.s }, p0, [x0, z1.s, sxtw #2]
+; CHECK-NEXT: ret
+  call void @llvm.aarch64.sve.st1.scatter.sxtw.index.nxv4i32(<vscale x 4 x i32> %data,
+                                                             <vscale x 4 x i1> %pg,
+                                                             i32* %base,
+                                                             <vscale x 4 x i32> %indices)
+  ret void
+}
+
+define void @sst1w_d_uxtw(<vscale x 2 x i64> %data, <vscale x 2 x i1> %pg, i32* %base, <vscale x 2 x i32> %indices) {
+; CHECK-LABEL: sst1w_d_uxtw:
+; CHECK: st1w { z0.d }, p0, [x0, z1.d, uxtw #2]
+; CHECK-NEXT: ret
+  %data_trunc = trunc <vscale x 2 x i64> %data to <vscale x 2 x i32>
+  call void @llvm.aarch64.sve.st1.scatter.uxtw.index.nxv2i32(<vscale x 2 x i32> %data_trunc,
+                                                             <vscale x 2 x i1> %pg,
+                                                             i32* %base,
+                                                             <vscale x 2 x i32> %indices)
+  ret void
+}
+
+define void @sst1w_d_sxtw(<vscale x 2 x i64> %data, <vscale x 2 x i1> %pg, i32* %base, <vscale x 2 x i32> %indices) {
+; CHECK-LABEL: sst1w_d_sxtw:
+; CHECK: st1w { z0.d }, p0, [x0, z1.d, sxtw #2]
+; CHECK-NEXT: ret
+  %data_trunc = trunc <vscale x 2 x i64> %data to <vscale x 2 x i32>
+  call void @llvm.aarch64.sve.st1.scatter.sxtw.index.nxv2i32(<vscale x 2 x i32> %data_trunc,
+                                                             <vscale x 2 x i1> %pg,
+                                                             i32* %base,
+                                                             <vscale x 2 x i32> %indices)
+  ret void
+}
+
+define void @sst1w_s_uxtw_float(<vscale x 4 x float> %data, <vscale x 4 x i1> %pg, float* %base, <vscale x 4 x i32> %indices) {
+; CHECK-LABEL: sst1w_s_uxtw_float:
+; CHECK: st1w { z0.s }, p0, [x0, z1.s, uxtw #2]
+; CHECK-NEXT:	ret
+  call void @llvm.aarch64.sve.st1.scatter.uxtw.index.nxv4f32(<vscale x 4 x float> %data,
+                                                             <vscale x 4 x i1> %pg,
+                                                             float* %base,
+                                                             <vscale x 4 x i32> %indices)
+  ret void
+}
+
+define void @sst1w_s_sxtw_float(<vscale x 4 x float> %data, <vscale x 4 x i1> %pg, float* %base, <vscale x 4 x i32> %indices) {
+; CHECK-LABEL: sst1w_s_sxtw_float:
+; CHECK: st1w { z0.s }, p0, [x0, z1.s, sxtw #2]
+; CHECK-NEXT: ret
+  call void @llvm.aarch64.sve.st1.scatter.sxtw.index.nxv4f32(<vscale x 4 x float> %data,
+                                                             <vscale x 4 x i1> %pg,
+                                                             float* %base,
+                                                             <vscale x 4 x i32> %indices)
+  ret void
+}
+
+; ST1D
+define void @sst1d_d_uxtw(<vscale x 2 x i64> %data, <vscale x 2 x i1> %pg, i64* %base, <vscale x 2 x i32> %indices) {
+; CHECK-LABEL: sst1d_d_uxtw:
+; CHECK: st1d { z0.d }, p0, [x0, z1.d, uxtw #3]
+; CHECK-NEXT: ret
+  call void @llvm.aarch64.sve.st1.scatter.uxtw.index.nxv2i64(<vscale x 2 x i64> %data,
+                                                             <vscale x 2 x i1> %pg,
+                                                             i64* %base,
+                                                             <vscale x 2 x i32> %indices)
+  ret void
+}
+
+define void @sst1d_d_sxtw(<vscale x 2 x i64> %data, <vscale x 2 x i1> %pg, i64* %base, <vscale x 2 x i32> %indices) {
+; CHECK-LABEL: sst1d_d_sxtw:
+; CHECK: st1d { z0.d }, p0, [x0, z1.d, sxtw #3]
+; CHECK-NEXT: ret
+  call void @llvm.aarch64.sve.st1.scatter.sxtw.index.nxv2i64(<vscale x 2 x i64> %data,
+                                                             <vscale x 2 x i1> %pg,
+                                                             i64* %base,
+                                                             <vscale x 2 x i32> %indices)
+  ret void
+}
+
+define void @sst1d_d_uxtw_double(<vscale x 2 x double> %data, <vscale x 2 x i1> %pg, double* %base, <vscale x 2 x i32> %indices) {
+; CHECK-LABEL: sst1d_d_uxtw_double:
+; CHECK: st1d { z0.d }, p0, [x0, z1.d, uxtw #3]
+; CHECK-NEXT: ret
+  call void @llvm.aarch64.sve.st1.scatter.uxtw.index.nxv2f64(<vscale x 2 x double> %data,
+                                                             <vscale x 2 x i1> %pg,
+                                                             double* %base,
+                                                             <vscale x 2 x i32> %indices)
+  ret void
+}
+
+define void @sst1d_d_sxtw_double(<vscale x 2 x double> %data, <vscale x 2 x i1> %pg, double* %base, <vscale x 2 x i32> %indices) {
+; CHECK-LABEL: sst1d_d_sxtw_double:
+; CHECK: st1d { z0.d }, p0, [x0, z1.d, sxtw #3]
+; CHECK-NEXT: ret
+  call void @llvm.aarch64.sve.st1.scatter.sxtw.index.nxv2f64(<vscale x 2 x double> %data,
+                                                             <vscale x 2 x i1> %pg,
+                                                             double* %base,
+                                                             <vscale x 2 x i32> %indices)
+  ret void
+}
+
+
+; ST1H
+declare void @llvm.aarch64.sve.st1.scatter.sxtw.index.nxv4i16(<vscale x 4 x i16>, <vscale x 4 x i1>, i16*, <vscale x 4 x i32>)
+declare void @llvm.aarch64.sve.st1.scatter.sxtw.index.nxv2i16(<vscale x 2 x i16>, <vscale x 2 x i1>, i16*, <vscale x 2 x i32>)
+declare void @llvm.aarch64.sve.st1.scatter.uxtw.index.nxv4i16(<vscale x 4 x i16>, <vscale x 4 x i1>, i16*, <vscale x 4 x i32>)
+declare void @llvm.aarch64.sve.st1.scatter.uxtw.index.nxv2i16(<vscale x 2 x i16>, <vscale x 2 x i1>, i16*, <vscale x 2 x i32>)
+
+; ST1W
+declare void @llvm.aarch64.sve.st1.scatter.sxtw.index.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i1>, i32*, <vscale x 4 x i32>)
+declare void @llvm.aarch64.sve.st1.scatter.sxtw.index.nxv2i32(<vscale x 2 x i32>, <vscale x 2 x i1>, i32*, <vscale x 2 x i32>)
+declare void @llvm.aarch64.sve.st1.scatter.uxtw.index.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i1>, i32*, <vscale x 4 x i32>)
+declare void @llvm.aarch64.sve.st1.scatter.uxtw.index.nxv2i32(<vscale x 2 x i32>, <vscale x 2 x i1>, i32*, <vscale x 2 x i32>)
+
+declare void @llvm.aarch64.sve.st1.scatter.sxtw.index.nxv4f32(<vscale x 4 x float>, <vscale x 4 x i1>, float*, <vscale x 4 x i32>)
+declare void @llvm.aarch64.sve.st1.scatter.uxtw.index.nxv4f32(<vscale x 4 x float>, <vscale x 4 x i1>, float*, <vscale x 4 x i32>)
+
+; ST1D
+declare void @llvm.aarch64.sve.st1.scatter.sxtw.index.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i1>, i64*, <vscale x 2 x i32>)
+declare void @llvm.aarch64.sve.st1.scatter.uxtw.index.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i1>, i64*, <vscale x 2 x i32>)
+
+declare void @llvm.aarch64.sve.st1.scatter.sxtw.index.nxv2f64(<vscale x 2 x double>, <vscale x 2 x i1>, double*, <vscale x 2 x i32>)
+declare void @llvm.aarch64.sve.st1.scatter.uxtw.index.nxv2f64(<vscale x 2 x double>, <vscale x 2 x i1>, double*, <vscale x 2 x i32>)
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-scatter-stores-32bit-unscaled-offsets.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-scatter-stores-32bit-unscaled-offsets.ll
new file mode 100644
index 000000000..52f988e8a
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-scatter-stores-32bit-unscaled-offsets.ll
@@ -0,0 +1,248 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+;
+; ST1B, ST1W, ST1H, ST1D: base + 32-bit unscaled offset, sign (sxtw) or zero
+; (uxtw) extended to 64 bits.
+;   e.g. st1h { z0.d }, p0, [x0, z1.d, uxtw]
+;
+
+; ST1B
+define void @sst1b_s_uxtw(<vscale x 4 x i32> %data, <vscale x 4 x i1> %pg, i8* %base, <vscale x 4 x i32> %offsets) {
+; CHECK-LABEL: sst1b_s_uxtw:
+; CHECK: st1b { z0.s }, p0, [x0, z1.s, uxtw]
+; CHECK-NEXT: ret
+  %data_trunc = trunc <vscale x 4 x i32> %data to <vscale x 4 x i8>
+  call void  @llvm.aarch64.sve.st1.scatter.uxtw.nxv4i8(<vscale x 4 x i8> %data_trunc,
+                                                       <vscale x 4 x i1> %pg,
+                                                       i8* %base,
+                                                       <vscale x 4 x i32> %offsets)
+  ret void
+}
+
+define void @sst1b_s_sxtw(<vscale x 4 x i32> %data, <vscale x 4 x i1> %pg, i8* %base, <vscale x 4 x i32> %offsets) {
+; CHECK-LABEL: sst1b_s_sxtw:
+; CHECK: st1b { z0.s }, p0, [x0, z1.s, sxtw]
+; CHECK-NEXT: ret
+  %data_trunc = trunc <vscale x 4 x i32> %data to <vscale x 4 x i8>
+  call void @llvm.aarch64.sve.st1.scatter.sxtw.nxv4i8(<vscale x 4 x i8> %data_trunc,
+                                                      <vscale x 4 x i1> %pg,
+                                                      i8* %base,
+                                                      <vscale x 4 x i32> %offsets)
+  ret void
+}
+
+define void @sst1b_d_uxtw(<vscale x 2 x i64> %data, <vscale x 2 x i1> %pg, i8* %base, <vscale x 2 x i32> %offsets) {
+; CHECK-LABEL: sst1b_d_uxtw:
+; CHECK: st1b { z0.d }, p0, [x0, z1.d, uxtw]
+; CHECK-NEXT: ret
+  %data_trunc = trunc <vscale x 2 x i64> %data to <vscale x 2 x i8>
+  call void @llvm.aarch64.sve.st1.scatter.uxtw.nxv2i8(<vscale x 2 x i8> %data_trunc,
+                                                      <vscale x 2 x i1> %pg,
+                                                      i8* %base,
+                                                      <vscale x 2 x i32> %offsets)
+  ret void
+}
+
+define void @sst1b_d_sxtw(<vscale x 2 x i64> %data, <vscale x 2 x i1> %pg, i8* %base, <vscale x 2 x i32> %offsets) {
+; CHECK-LABEL: sst1b_d_sxtw:
+; CHECK: st1b { z0.d }, p0, [x0, z1.d, sxtw]
+; CHECK-NEXT: ret
+  %data_trunc = trunc <vscale x 2 x i64> %data to <vscale x 2 x i8>
+  call void  @llvm.aarch64.sve.st1.scatter.sxtw.nxv2i8(<vscale x 2 x i8> %data_trunc,
+                                                       <vscale x 2 x i1> %pg,
+                                                       i8* %base,
+                                                       <vscale x 2 x i32> %offsets)
+  ret void
+}
+
+; ST1H
+define void @sst1h_s_uxtw(<vscale x 4 x i32> %data, <vscale x 4 x i1> %pg, i16* %base, <vscale x 4 x i32> %offsets) {
+; CHECK-LABEL: sst1h_s_uxtw:
+; CHECK: st1h { z0.s }, p0, [x0, z1.s, uxtw]
+; CHECK-NEXT:	ret
+  %data_trunc = trunc <vscale x 4 x i32> %data to <vscale x 4 x i16>
+  call void @llvm.aarch64.sve.st1.scatter.uxtw.nxv4i16(<vscale x 4 x i16> %data_trunc,
+                                                       <vscale x 4 x i1> %pg,
+                                                       i16* %base,
+                                                       <vscale x 4 x i32> %offsets)
+  ret void
+}
+
+define void @sst1h_s_sxtw(<vscale x 4 x i32> %data, <vscale x 4 x i1> %pg, i16* %base, <vscale x 4 x i32> %offsets) {
+; CHECK-LABEL: sst1h_s_sxtw:
+; CHECK: st1h { z0.s }, p0, [x0, z1.s, sxtw]
+; CHECK-NEXT: ret
+  %data_trunc = trunc <vscale x 4 x i32> %data to <vscale x 4 x i16>
+  call void @llvm.aarch64.sve.st1.scatter.sxtw.nxv4i16(<vscale x 4 x i16> %data_trunc,
+                                                       <vscale x 4 x i1> %pg,
+                                                       i16* %base,
+                                                       <vscale x 4 x i32> %offsets)
+  ret void
+}
+
+define void @sst1h_d_uxtw(<vscale x 2 x i64> %data, <vscale x 2 x i1> %pg, i16* %base, <vscale x 2 x i32> %offsets) {
+; CHECK-LABEL: sst1h_d_uxtw:
+; CHECK: st1h { z0.d }, p0, [x0, z1.d, uxtw]
+; CHECK-NEXT: ret
+  %data_trunc = trunc <vscale x 2 x i64> %data to <vscale x 2 x i16>
+  call void @llvm.aarch64.sve.st1.scatter.uxtw.nxv2i16(<vscale x 2 x i16> %data_trunc,
+                                                       <vscale x 2 x i1> %pg,
+                                                       i16* %base,
+                                                       <vscale x 2 x i32> %offsets)
+  ret void
+}
+
+define void @sst1h_d_sxtw(<vscale x 2 x i64> %data, <vscale x 2 x i1> %pg, i16* %base, <vscale x 2 x i32> %offsets) {
+; CHECK-LABEL: sst1h_d_sxtw:
+; CHECK: st1h { z0.d }, p0, [x0, z1.d, sxtw]
+; CHECK-NEXT: ret
+  %data_trunc = trunc <vscale x 2 x i64> %data to <vscale x 2 x i16>
+  call void @llvm.aarch64.sve.st1.scatter.sxtw.nxv2i16(<vscale x 2 x i16> %data_trunc,
+                                                       <vscale x 2 x i1> %pg,
+                                                       i16* %base,
+                                                       <vscale x 2 x i32> %offsets)
+  ret void
+}
+
+; ST1W
+define void @sst1w_s_uxtw(<vscale x 4 x i32> %data, <vscale x 4 x i1> %pg, i32* %base, <vscale x 4 x i32> %offsets) {
+; CHECK-LABEL: sst1w_s_uxtw:
+; CHECK: st1w { z0.s }, p0, [x0, z1.s, uxtw]
+; CHECK-NEXT:	ret
+  call void @llvm.aarch64.sve.st1.scatter.uxtw.nxv4i32(<vscale x 4 x i32> %data,
+                                                       <vscale x 4 x i1> %pg,
+                                                       i32* %base,
+                                                       <vscale x 4 x i32> %offsets)
+  ret void
+}
+
+define void @sst1w_s_sxtw(<vscale x 4 x i32> %data, <vscale x 4 x i1> %pg, i32* %base, <vscale x 4 x i32> %offsets) {
+; CHECK-LABEL: sst1w_s_sxtw:
+; CHECK: st1w { z0.s }, p0, [x0, z1.s, sxtw]
+; CHECK-NEXT: ret
+  call void @llvm.aarch64.sve.st1.scatter.sxtw.nxv4i32(<vscale x 4 x i32> %data,
+                                                       <vscale x 4 x i1> %pg,
+                                                       i32* %base,
+                                                       <vscale x 4 x i32> %offsets)
+  ret void
+}
+
+define void @sst1w_d_uxtw(<vscale x 2 x i64> %data, <vscale x 2 x i1> %pg, i32* %base, <vscale x 2 x i32> %offsets) {
+; CHECK-LABEL: sst1w_d_uxtw:
+; CHECK: st1w { z0.d }, p0, [x0, z1.d, uxtw]
+; CHECK-NEXT: ret
+  %data_trunc = trunc <vscale x 2 x i64> %data to <vscale x 2 x i32>
+  call void @llvm.aarch64.sve.st1.scatter.uxtw.nxv2i32(<vscale x 2 x i32> %data_trunc,
+                                                       <vscale x 2 x i1> %pg,
+                                                       i32* %base,
+                                                       <vscale x 2 x i32> %offsets)
+  ret void
+}
+
+define void @sst1w_d_sxtw(<vscale x 2 x i64> %data, <vscale x 2 x i1> %pg, i32* %base, <vscale x 2 x i32> %offsets) {
+; CHECK-LABEL: sst1w_d_sxtw:
+; CHECK: st1w { z0.d }, p0, [x0, z1.d, sxtw]
+; CHECK-NEXT: ret
+  %data_trunc = trunc <vscale x 2 x i64> %data to <vscale x 2 x i32>
+  call void @llvm.aarch64.sve.st1.scatter.sxtw.nxv2i32(<vscale x 2 x i32> %data_trunc,
+                                                       <vscale x 2 x i1> %pg,
+                                                       i32* %base,
+                                                       <vscale x 2 x i32> %offsets)
+  ret void
+}
+
+define void @sst1w_s_uxtw_float(<vscale x 4 x float> %data, <vscale x 4 x i1> %pg, float* %base, <vscale x 4 x i32> %offsets) {
+; CHECK-LABEL: sst1w_s_uxtw_float:
+; CHECK: st1w { z0.s }, p0, [x0, z1.s, uxtw]
+; CHECK-NEXT:	ret
+  call void @llvm.aarch64.sve.st1.scatter.uxtw.nxv4f32(<vscale x 4 x float> %data,
+                                                       <vscale x 4 x i1> %pg,
+                                                       float* %base,
+                                                       <vscale x 4 x i32> %offsets)
+  ret void
+}
+
+define void @sst1w_s_sxtw_float(<vscale x 4 x float> %data, <vscale x 4 x i1> %pg, float* %base, <vscale x 4 x i32> %offsets) {
+; CHECK-LABEL: sst1w_s_sxtw_float:
+; CHECK: st1w { z0.s }, p0, [x0, z1.s, sxtw]
+; CHECK-NEXT: ret
+  call void @llvm.aarch64.sve.st1.scatter.sxtw.nxv4f32(<vscale x 4 x float> %data,
+                                                       <vscale x 4 x i1> %pg,
+                                                       float* %base,
+                                                       <vscale x 4 x i32> %offsets)
+  ret void
+}
+
+; ST1D
+define void @sst1d_d_uxtw(<vscale x 2 x i64> %data, <vscale x 2 x i1> %pg, i64* %base, <vscale x 2 x i32> %offsets) {
+; CHECK-LABEL: sst1d_d_uxtw:
+; CHECK: st1d { z0.d }, p0, [x0, z1.d, uxtw]
+; CHECK-NEXT: ret
+  call void @llvm.aarch64.sve.st1.scatter.uxtw.nxv2i64(<vscale x 2 x i64> %data,
+                                                       <vscale x 2 x i1> %pg,
+                                                       i64* %base,
+                                                       <vscale x 2 x i32> %offsets)
+  ret void
+}
+
+define void @sst1d_d_sxtw(<vscale x 2 x i64> %data, <vscale x 2 x i1> %pg, i64* %base, <vscale x 2 x i32> %offsets) {
+; CHECK-LABEL: sst1d_d_sxtw:
+; CHECK: st1d { z0.d }, p0, [x0, z1.d, sxtw]
+; CHECK-NEXT: ret
+  call void @llvm.aarch64.sve.st1.scatter.sxtw.nxv2i64(<vscale x 2 x i64> %data,
+                                                       <vscale x 2 x i1> %pg,
+                                                       i64* %base,
+                                                       <vscale x 2 x i32> %offsets)
+  ret void
+}
+
+define void @sst1d_d_uxtw_double(<vscale x 2 x double> %data, <vscale x 2 x i1> %pg, double* %base, <vscale x 2 x i32> %offsets) {
+; CHECK-LABEL: sst1d_d_uxtw_double:
+; CHECK: st1d { z0.d }, p0, [x0, z1.d, uxtw]
+; CHECK-NEXT: ret
+  call void @llvm.aarch64.sve.st1.scatter.uxtw.nxv2f64(<vscale x 2 x double> %data,
+                                                       <vscale x 2 x i1> %pg,
+                                                       double* %base,
+                                                       <vscale x 2 x i32> %offsets)
+  ret void
+}
+
+define void @sst1d_d_sxtw_double(<vscale x 2 x double> %data, <vscale x 2 x i1> %pg, double* %base, <vscale x 2 x i32> %offsets) {
+; CHECK-LABEL: sst1d_d_sxtw_double:
+; CHECK: st1d { z0.d }, p0, [x0, z1.d, sxtw]
+; CHECK-NEXT: ret
+  call void @llvm.aarch64.sve.st1.scatter.sxtw.nxv2f64(<vscale x 2 x double> %data,
+                                                       <vscale x 2 x i1> %pg,
+                                                       double* %base,
+                                                       <vscale x 2 x i32> %offsets)
+  ret void
+}
+
+
+; ST1B
+declare void @llvm.aarch64.sve.st1.scatter.uxtw.nxv4i8(<vscale x 4 x i8>, <vscale x 4 x i1>, i8*, <vscale x 4 x i32>)
+declare void @llvm.aarch64.sve.st1.scatter.uxtw.nxv2i8(<vscale x 2 x i8>, <vscale x 2 x i1>, i8*, <vscale x 2 x i32>)
+declare void @llvm.aarch64.sve.st1.scatter.sxtw.nxv4i8(<vscale x 4 x i8>, <vscale x 4 x i1>, i8*, <vscale x 4 x i32>)
+declare void @llvm.aarch64.sve.st1.scatter.sxtw.nxv2i8(<vscale x 2 x i8>, <vscale x 2 x i1>, i8*, <vscale x 2 x i32>)
+
+; ST1H
+declare void @llvm.aarch64.sve.st1.scatter.sxtw.nxv4i16(<vscale x 4 x i16>, <vscale x 4 x i1>, i16*, <vscale x 4 x i32>)
+declare void @llvm.aarch64.sve.st1.scatter.sxtw.nxv2i16(<vscale x 2 x i16>, <vscale x 2 x i1>, i16*, <vscale x 2 x i32>)
+declare void @llvm.aarch64.sve.st1.scatter.uxtw.nxv4i16(<vscale x 4 x i16>, <vscale x 4 x i1>, i16*, <vscale x 4 x i32>)
+declare void @llvm.aarch64.sve.st1.scatter.uxtw.nxv2i16(<vscale x 2 x i16>, <vscale x 2 x i1>, i16*, <vscale x 2 x i32>)
+
+; ST1W
+declare void @llvm.aarch64.sve.st1.scatter.sxtw.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i1>, i32*, <vscale x 4 x i32>)
+declare void @llvm.aarch64.sve.st1.scatter.sxtw.nxv2i32(<vscale x 2 x i32>, <vscale x 2 x i1>, i32*, <vscale x 2 x i32>)
+declare void @llvm.aarch64.sve.st1.scatter.uxtw.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i1>, i32*, <vscale x 4 x i32>)
+declare void @llvm.aarch64.sve.st1.scatter.uxtw.nxv2i32(<vscale x 2 x i32>, <vscale x 2 x i1>, i32*, <vscale x 2 x i32>)
+
+declare void @llvm.aarch64.sve.st1.scatter.sxtw.nxv4f32(<vscale x 4 x float>, <vscale x 4 x i1>, float*, <vscale x 4 x i32>)
+declare void @llvm.aarch64.sve.st1.scatter.uxtw.nxv4f32(<vscale x 4 x float>, <vscale x 4 x i1>, float*, <vscale x 4 x i32>)
+
+; ST1D
+declare void @llvm.aarch64.sve.st1.scatter.sxtw.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i1>, i64*, <vscale x 2 x i32>)
+declare void @llvm.aarch64.sve.st1.scatter.uxtw.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i1>, i64*, <vscale x 2 x i32>)
+
+declare void @llvm.aarch64.sve.st1.scatter.sxtw.nxv2f64(<vscale x 2 x double>, <vscale x 2 x i1>, double*, <vscale x 2 x i32>)
+declare void @llvm.aarch64.sve.st1.scatter.uxtw.nxv2f64(<vscale x 2 x double>, <vscale x 2 x i1>, double*, <vscale x 2 x i32>)
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-scatter-stores-64bit-scaled-offset.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-scatter-stores-64bit-scaled-offset.ll
new file mode 100644
index 000000000..ca81fe14e
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-scatter-stores-64bit-scaled-offset.ll
@@ -0,0 +1,58 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+;
+; ST1H, ST1W, ST1D: base + 64-bit scaled offset
+;   e.g. st1h { z0.d }, p0, [x0, z0.d, lsl #1]
+;
+
+define void @sst1h_index(<vscale x 2 x i64> %data, <vscale x 2 x i1> %pg, i16* %base, <vscale x 2 x i64> %offsets) {
+; CHECK-LABEL: sst1h_index
+; CHECK:	    st1h	{ z0.d }, p0, [x0, z1.d, lsl #1]
+; CHECK-NEXT:	ret
+  %data_trunc = trunc <vscale x 2 x i64> %data to <vscale x 2 x i16>
+  call void @llvm.aarch64.sve.st1.scatter.index.nxv2i16(<vscale x 2 x i16> %data_trunc,
+                                                        <vscale x 2 x i1> %pg,
+                                                        i16* %base,
+                                                        <vscale x 2 x i64> %offsets)
+  ret void
+}
+
+define void @sst1w_index(<vscale x 2 x i64> %data, <vscale x 2 x i1> %pg, i32* %base, <vscale x 2 x i64> %offsets) {
+; CHECK-LABEL: sst1w_index
+; CHECK:	    st1w	{ z0.d }, p0, [x0, z1.d, lsl #2]
+; CHECK-NEXT:	ret
+  %data_trunc = trunc <vscale x 2 x i64> %data to <vscale x 2 x i32>
+  call void @llvm.aarch64.sve.st1.scatter.index.nxv2i32(<vscale x 2 x i32> %data_trunc,
+                                                        <vscale x 2 x i1> %pg,
+                                                        i32* %base,
+                                                        <vscale x 2 x i64> %offsets)
+  ret void
+}
+
+define void  @sst1d_index(<vscale x 2 x i64> %data, <vscale x 2 x i1> %pg, i64* %base, <vscale x 2 x i64> %offsets) {
+; CHECK-LABEL: sst1d_index
+; CHECK:	    st1d	{ z0.d }, p0, [x0, z1.d, lsl #3]
+; CHECK-NEXT:	ret
+  call void @llvm.aarch64.sve.st1.scatter.index.nxv2i64(<vscale x 2 x i64> %data,
+                                                        <vscale x 2 x i1> %pg,
+                                                        i64* %base,
+                                                        <vscale x 2 x i64> %offsets)
+  ret void
+}
+
+define void  @sst1d_index_double(<vscale x 2 x double> %data, <vscale x 2 x i1> %pg, double* %base, <vscale x 2 x i64> %offsets) {
+; CHECK-LABEL: sst1d_index_double
+; CHECK:	    st1d	{ z0.d }, p0, [x0, z1.d, lsl #3]
+; CHECK-NEXT:	ret
+  call void @llvm.aarch64.sve.st1.scatter.index.nxv2f64(<vscale x 2 x double> %data,
+                                                        <vscale x 2 x i1> %pg,
+                                                        double* %base,
+                                                        <vscale x 2 x i64> %offsets)
+  ret void
+}
+
+
+declare void @llvm.aarch64.sve.st1.scatter.index.nxv2i16(<vscale x 2 x i16>, <vscale x 2 x i1>, i16*, <vscale x 2 x i64>)
+declare void @llvm.aarch64.sve.st1.scatter.index.nxv2i32(<vscale x 2 x i32>, <vscale x 2 x i1>, i32*, <vscale x 2 x i64>)
+declare void @llvm.aarch64.sve.st1.scatter.index.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i1>, i64*, <vscale x 2 x i64>)
+declare void @llvm.aarch64.sve.st1.scatter.index.nxv2f64(<vscale x 2 x double>, <vscale x 2 x i1>, double*, <vscale x 2 x i64>)
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-scatter-stores-64bit-unscaled-offset.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-scatter-stores-64bit-unscaled-offset.ll
new file mode 100644
index 000000000..00e72a5d4
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-scatter-stores-64bit-unscaled-offset.ll
@@ -0,0 +1,70 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+;
+; ST1B, ST1W, ST1H, ST1D: base + 64-bit unscaled offset
+;   e.g. st1h { z0.d }, p0, [x0, z1.d]
+;
+
+define void @sst1b_d(<vscale x 2 x i64> %data, <vscale x 2 x i1> %pg, i8* %base, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: sst1b_d:
+; CHECK: st1b { z0.d }, p0, [x0, z1.d]
+; CHECK-NEXT: ret
+  %data_trunc = trunc <vscale x 2 x i64> %data to <vscale x 2 x i8>
+  call void @llvm.aarch64.sve.st1.scatter.nxv2i8(<vscale x 2 x i8> %data_trunc,
+                                                 <vscale x 2 x i1> %pg,
+                                                 i8* %base,
+                                                 <vscale x 2 x i64> %b)
+  ret void
+}
+
+define void @sst1h_d(<vscale x 2 x i64> %data, <vscale x 2 x i1> %pg, i16* %base, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: sst1h_d:
+; CHECK: st1h { z0.d }, p0, [x0, z1.d]
+; CHECK-NEXT: ret
+  %data_trunc = trunc <vscale x 2 x i64> %data to <vscale x 2 x i16>
+  call void @llvm.aarch64.sve.st1.scatter.nxv2i16(<vscale x 2 x i16> %data_trunc,
+                                                 <vscale x 2 x i1> %pg,
+                                                 i16* %base,
+                                                 <vscale x 2 x i64> %b)
+  ret void
+}
+
+define void @sst1w_d(<vscale x 2 x i64> %data, <vscale x 2 x i1> %pg, i32* %base, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: sst1w_d:
+; CHECK: st1w { z0.d }, p0, [x0, z1.d]
+; CHECK-NEXT: ret
+  %data_trunc = trunc <vscale x 2 x i64> %data to <vscale x 2 x i32>
+  call void @llvm.aarch64.sve.st1.scatter.nxv2i32(<vscale x 2 x i32> %data_trunc,
+                                                 <vscale x 2 x i1> %pg,
+                                                 i32* %base,
+                                                 <vscale x 2 x i64> %b)
+  ret void
+}
+
+define void @sst1d_d(<vscale x 2 x i64> %data, <vscale x 2 x i1> %pg, i64* %base, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: sst1d_d:
+; CHECK: st1d { z0.d }, p0, [x0, z1.d]
+; CHECK-NEXT: ret
+  call void @llvm.aarch64.sve.st1.scatter.nxv2i64(<vscale x 2 x i64> %data,
+                                                 <vscale x 2 x i1> %pg,
+                                                 i64* %base,
+                                                 <vscale x 2 x i64> %b)
+  ret void
+}
+
+define void @sst1d_d_double(<vscale x 2 x double> %data, <vscale x 2 x i1> %pg, double* %base, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: sst1d_d_double:
+; CHECK: st1d { z0.d }, p0, [x0, z1.d]
+; CHECK-NEXT: ret
+  call void @llvm.aarch64.sve.st1.scatter.nxv2f64(<vscale x 2 x double> %data,
+                                                 <vscale x 2 x i1> %pg,
+                                                 double* %base,
+                                                 <vscale x 2 x i64> %b)
+  ret void
+}
+
+declare void @llvm.aarch64.sve.st1.scatter.nxv2i8(<vscale x 2 x i8>, <vscale x 2 x i1>, i8*, <vscale x 2 x i64>)
+declare void @llvm.aarch64.sve.st1.scatter.nxv2i16(<vscale x 2 x i16>, <vscale x 2 x i1>, i16*, <vscale x 2 x i64>)
+declare void @llvm.aarch64.sve.st1.scatter.nxv2i32(<vscale x 2 x i32>, <vscale x 2 x i1>, i32*, <vscale x 2 x i64>)
+declare void @llvm.aarch64.sve.st1.scatter.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i1>, i64*, <vscale x 2 x i64>)
+declare void @llvm.aarch64.sve.st1.scatter.nxv2f64(<vscale x 2 x double>, <vscale x 2 x i1>, double*, <vscale x 2 x i64>)
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-scatter-stores-vector-base.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-scatter-stores-vector-base.ll
new file mode 100644
index 000000000..66a0489cb
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-scatter-stores-vector-base.ll
@@ -0,0 +1,133 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+;
+; ST1B, ST1W, ST1H, ST1D: vector + immediate (index)
+;   e.g. st1h { z0.s }, p0, [z1.s, #16]
+;
+
+; ST1B
+define void @sst1b_s_imm(<vscale x 4 x i32> %data, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %base) {
+; CHECK-LABEL: sst1b_s_imm:
+; CHECK: st1b { z0.s }, p0, [z1.s, #16]
+; CHECK-NEXT: ret
+  %data_trunc = trunc <vscale x 4 x i32> %data to <vscale x 4 x i8>
+  call void @llvm.aarch64.sve.st1.scatter.imm.nxv4i8.nxv4i32(<vscale x 4 x i8> %data_trunc,
+                                                             <vscale x 4 x i1> %pg,
+                                                             <vscale x 4 x i32> %base,
+                                                             i64 16)
+  ret void
+}
+
+define void @sst1b_d_imm(<vscale x 2 x i64> %data, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %base) {
+; CHECK-LABEL: sst1b_d_imm:
+; CHECK: st1b { z0.d }, p0, [z1.d, #16]
+; CHECK-NEXT: ret
+  %data_trunc = trunc <vscale x 2 x i64> %data to <vscale x 2 x i8>
+  call void @llvm.aarch64.sve.st1.scatter.imm.nxv2i8.nxv2i64(<vscale x 2 x i8> %data_trunc,
+                                                             <vscale x 2 x i1> %pg,
+                                                             <vscale x 2 x i64> %base,
+                                                             i64 16)
+  ret void
+}
+
+; ST1H
+define void @sst1h_s_imm(<vscale x 4 x i32> %data, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %base) {
+; CHECK-LABEL: sst1h_s_imm:
+; CHECK: st1h { z0.s }, p0, [z1.s, #16]
+; CHECK-NEXT: ret
+  %data_trunc = trunc <vscale x 4 x i32> %data to <vscale x 4 x i16>
+  call void @llvm.aarch64.sve.st1.scatter.imm.nxv4i16.nxv4i32(<vscale x 4 x i16> %data_trunc,
+                                                              <vscale x 4 x i1> %pg,
+                                                              <vscale x 4 x i32> %base,
+                                                              i64 16)
+  ret void
+}
+
+define void @sst1h_d_imm(<vscale x 2 x i64> %data, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %base) {
+; CHECK-LABEL: sst1h_d_imm:
+; CHECK: st1h { z0.d }, p0, [z1.d, #16]
+; CHECK-NEXT: ret
+  %data_trunc = trunc <vscale x 2 x i64> %data to <vscale x 2 x i16>
+  call void @llvm.aarch64.sve.st1.scatter.imm.nxv2i16.nxv2i64(<vscale x 2 x i16> %data_trunc,
+                                                              <vscale x 2 x i1> %pg,
+                                                              <vscale x 2 x i64> %base,
+                                                              i64 16)
+  ret void
+}
+
+; ST1W
+define void @sst1w_s_imm(<vscale x 4 x i32> %data, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %base) {
+; CHECK-LABEL: sst1w_s_imm:
+; CHECK: st1w { z0.s }, p0, [z1.s, #16]
+; CHECK-NEXT: ret
+  call void @llvm.aarch64.sve.st1.scatter.imm.nxv4i32.nxv4i32(<vscale x 4 x i32> %data,
+                                                              <vscale x 4 x i1> %pg,
+                                                              <vscale x 4 x i32> %base,
+                                                              i64 16)
+  ret void
+}
+
+define void @sst1w_d_imm(<vscale x 2 x i64> %data, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %base) {
+; CHECK-LABEL: sst1w_d_imm:
+; CHECK: st1w { z0.d }, p0, [z1.d, #16]
+; CHECK-NEXT: ret
+  %data_trunc = trunc <vscale x 2 x i64> %data to <vscale x 2 x i32>
+  call void @llvm.aarch64.sve.st1.scatter.imm.nxv2i32.nxv2i64(<vscale x 2 x i32> %data_trunc,
+                                                              <vscale x 2 x i1> %pg,
+                                                              <vscale x 2 x i64> %base,
+                                                              i64 16)
+  ret void
+}
+
+define void @sst1w_s_imm_float(<vscale x 4 x float> %data, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %base) {
+; CHECK-LABEL: sst1w_s_imm_float:
+; CHECK: st1w { z0.s }, p0, [z1.s, #16]
+; CHECK-NEXT: ret
+  call void @llvm.aarch64.sve.st1.scatter.imm.nxv4f32.nxv4i32(<vscale x 4 x float> %data,
+                                                              <vscale x 4 x i1> %pg,
+                                                              <vscale x 4 x i32> %base,
+                                                              i64 16)
+  ret void
+}
+
+; ST1D
+define void @sst1d_d_imm(<vscale x 2 x i64> %data, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %base) {
+; CHECK-LABEL: sst1d_d_imm:
+; CHECK: st1d { z0.d }, p0, [z1.d, #16]
+; CHECK-NEXT: ret
+  call void @llvm.aarch64.sve.st1.scatter.imm.nxv2i64.nxv2i64(<vscale x 2 x i64> %data,
+                                                              <vscale x 2 x i1> %pg,
+                                                              <vscale x 2 x i64> %base,
+                                                              i64 16)
+  ret void
+}
+
+define void @sst1d_d_imm_double(<vscale x 2 x double> %data, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %base) {
+; CHECK-LABEL: sst1d_d_imm_double:
+; CHECK: st1d { z0.d }, p0, [z1.d, #16]
+; CHECK-NEXT: ret
+  call void @llvm.aarch64.sve.st1.scatter.imm.nxv2f64.nxv2i64(<vscale x 2 x double> %data,
+                                                              <vscale x 2 x i1> %pg,
+                                                              <vscale x 2 x i64> %base,
+                                                              i64 16)
+  ret void
+}
+
+; ST1B
+declare void @llvm.aarch64.sve.st1.scatter.imm.nxv4i8.nxv4i32(<vscale x 4 x i8>, <vscale x 4 x i1>, <vscale x 4 x i32>, i64)
+declare void @llvm.aarch64.sve.st1.scatter.imm.nxv2i8.nxv2i64(<vscale x 2 x i8>, <vscale x 2 x i1>, <vscale x 2 x i64>, i64)
+
+; ST1H
+declare void @llvm.aarch64.sve.st1.scatter.imm.nxv4i16.nxv4i32(<vscale x 4 x i16>, <vscale x 4 x i1>, <vscale x 4 x i32>, i64)
+declare void @llvm.aarch64.sve.st1.scatter.imm.nxv2i16.nxv2i64(<vscale x 2 x i16>, <vscale x 2 x i1>, <vscale x 2 x i64>, i64)
+
+; ST1W
+declare void @llvm.aarch64.sve.st1.scatter.imm.nxv4i32.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i1>, <vscale x 4 x i32>, i64)
+declare void @llvm.aarch64.sve.st1.scatter.imm.nxv2i32.nxv2i64(<vscale x 2 x i32>, <vscale x 2 x i1>, <vscale x 2 x i64>, i64)
+
+declare void @llvm.aarch64.sve.st1.scatter.imm.nxv4f32.nxv4i32(<vscale x 4 x float>, <vscale x 4 x i1>, <vscale x 4 x i32>, i64)
+
+; ST1D
+declare void @llvm.aarch64.sve.st1.scatter.imm.nxv2i64.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i1>, <vscale x 2 x i64>, i64)
+
+declare void @llvm.aarch64.sve.st1.scatter.imm.nxv2f64.nxv2i64(<vscale x 2 x double>, <vscale x 2 x i1>, <vscale x 2 x i64>, i64)
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-shifts.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-shifts.ll
new file mode 100644
index 000000000..b1b3dc615
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-shifts.ll
@@ -0,0 +1,367 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+;
+; ASR
+;
+
+define <vscale x 16 x i8> @asr_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: asr_i8:
+; CHECK: asr z0.b, p0/m, z0.b, z1.b
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.asr.nxv16i8(<vscale x 16 x i1> %pg,
+                                                               <vscale x 16 x i8> %a,
+                                                               <vscale x 16 x i8> %b)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @asr_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: asr_i16:
+; CHECK: asr z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.asr.nxv8i16(<vscale x 8 x i1> %pg,
+                                                               <vscale x 8 x i16> %a,
+                                                               <vscale x 8 x i16> %b)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @asr_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: asr_i32:
+; CHECK: asr z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.asr.nxv4i32(<vscale x 4 x i1> %pg,
+                                                               <vscale x 4 x i32> %a,
+                                                               <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @asr_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: asr_i64:
+; CHECK: asr z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.asr.nxv2i64(<vscale x 2 x i1> %pg,
+                                                               <vscale x 2 x i64> %a,
+                                                               <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i64> %out
+}
+
+define <vscale x 16 x i8> @asr_wide_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: asr_wide_i8:
+; CHECK: asr z0.b, p0/m, z0.b, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.asr.wide.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                    <vscale x 16 x i8> %a,
+                                                                    <vscale x 2 x i64> %b)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @asr_wide_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: asr_wide_i16:
+; CHECK: asr z0.h, p0/m, z0.h, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.asr.wide.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                    <vscale x 8 x i16> %a,
+                                                                    <vscale x 2 x i64> %b)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @asr_wide_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: asr_wide_i32:
+; CHECK: asr z0.s, p0/m, z0.s, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.asr.wide.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                    <vscale x 4 x i32> %a,
+                                                                    <vscale x 2 x i64> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+;
+; ASRD
+;
+
+define <vscale x 16 x i8> @asrd_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a) {
+; CHECK-LABEL: asrd_i8:
+; CHECK: asrd z0.b, p0/m, z0.b, #1
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.asrd.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                <vscale x 16 x i8> %a,
+                                                                i32 1)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @asrd_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a) {
+; CHECK-LABEL: asrd_i16:
+; CHECK: asrd z0.h, p0/m, z0.h, #2
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.asrd.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                <vscale x 8 x i16> %a,
+                                                                i32 2)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @asrd_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a) {
+; CHECK-LABEL: asrd_i32:
+; CHECK: asrd z0.s, p0/m, z0.s, #31
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.asrd.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                <vscale x 4 x i32> %a,
+                                                                i32 31)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @asrd_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a) {
+; CHECK-LABEL: asrd_i64:
+; CHECK: asrd z0.d, p0/m, z0.d, #64
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.asrd.nxv2i64(<vscale x 2 x i1> %pg,
+                                                                <vscale x 2 x i64> %a,
+                                                                i32 64)
+  ret <vscale x 2 x i64> %out
+}
+
+;
+; INSR
+;
+
+define <vscale x 16 x i8> @insr_i8(<vscale x 16 x i8> %a, i8 %b) {
+; CHECK-LABEL: insr_i8:
+; CHECK: insr z0.b, w0
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.insr.nxv16i8(<vscale x 16 x i8> %a, i8 %b)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @insr_i16(<vscale x 8 x i16> %a, i16 %b) {
+; CHECK-LABEL: insr_i16:
+; CHECK: insr z0.h, w0
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.insr.nxv8i16(<vscale x 8 x i16> %a, i16 %b)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @insr_i32(<vscale x 4 x i32> %a, i32 %b) {
+; CHECK-LABEL: insr_i32:
+; CHECK: insr z0.s, w0
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.insr.nxv4i32(<vscale x 4 x i32> %a, i32 %b)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @insr_i64(<vscale x 2 x i64> %a, i64 %b) {
+; CHECK-LABEL: insr_i64:
+; CHECK: insr z0.d, x0
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.insr.nxv2i64(<vscale x 2 x i64> %a, i64 %b)
+  ret <vscale x 2 x i64> %out
+}
+
+define <vscale x 8 x half> @insr_f16(<vscale x 8 x half> %a, half %b) {
+; CHECK-LABEL: insr_f16:
+; CHECK: insr z0.h, h1
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.insr.nxv8f16(<vscale x 8 x half> %a, half %b)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x float> @insr_f32(<vscale x 4 x float> %a, float %b) {
+; CHECK-LABEL: insr_f32:
+; CHECK: insr z0.s, s1
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.insr.nxv4f32(<vscale x 4 x float> %a, float %b)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @insr_f64(<vscale x 2 x double> %a, double %b) {
+; CHECK-LABEL: insr_f64:
+; CHECK: insr z0.d, d1
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.insr.nxv2f64(<vscale x 2 x double> %a, double %b)
+  ret <vscale x 2 x double> %out
+}
+
+;
+; LSL
+;
+
+define <vscale x 16 x i8> @lsl_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: lsl_i8:
+; CHECK: lsl z0.b, p0/m, z0.b, z1.b
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.lsl.nxv16i8(<vscale x 16 x i1> %pg,
+                                                               <vscale x 16 x i8> %a,
+                                                               <vscale x 16 x i8> %b)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @lsl_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: lsl_i16:
+; CHECK: lsl z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.lsl.nxv8i16(<vscale x 8 x i1> %pg,
+                                                               <vscale x 8 x i16> %a,
+                                                               <vscale x 8 x i16> %b)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @lsl_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: lsl_i32:
+; CHECK: lsl z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.lsl.nxv4i32(<vscale x 4 x i1> %pg,
+                                                               <vscale x 4 x i32> %a,
+                                                               <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @lsl_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: lsl_i64:
+; CHECK: lsl z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.lsl.nxv2i64(<vscale x 2 x i1> %pg,
+                                                               <vscale x 2 x i64> %a,
+                                                               <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i64> %out
+}
+
+define <vscale x 16 x i8> @lsl_wide_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: lsl_wide_i8:
+; CHECK: lsl z0.b, p0/m, z0.b, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.lsl.wide.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                    <vscale x 16 x i8> %a,
+                                                                    <vscale x 2 x i64> %b)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @lsl_wide_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: lsl_wide_i16:
+; CHECK: lsl z0.h, p0/m, z0.h, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.lsl.wide.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                    <vscale x 8 x i16> %a,
+                                                                    <vscale x 2 x i64> %b)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @lsl_wide_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: lsl_wide_i32:
+; CHECK: lsl z0.s, p0/m, z0.s, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.lsl.wide.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                    <vscale x 4 x i32> %a,
+                                                                    <vscale x 2 x i64> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+;
+; LSR
+;
+
+define <vscale x 16 x i8> @lsr_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: lsr_i8:
+; CHECK: lsr z0.b, p0/m, z0.b, z1.b
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.lsr.nxv16i8(<vscale x 16 x i1> %pg,
+                                                               <vscale x 16 x i8> %a,
+                                                               <vscale x 16 x i8> %b)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @lsr_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: lsr_i16:
+; CHECK: lsr z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.lsr.nxv8i16(<vscale x 8 x i1> %pg,
+                                                               <vscale x 8 x i16> %a,
+                                                               <vscale x 8 x i16> %b)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @lsr_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: lsr_i32:
+; CHECK: lsr z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.lsr.nxv4i32(<vscale x 4 x i1> %pg,
+                                                               <vscale x 4 x i32> %a,
+                                                               <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @lsr_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: lsr_i64:
+; CHECK: lsr z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.lsr.nxv2i64(<vscale x 2 x i1> %pg,
+                                                               <vscale x 2 x i64> %a,
+                                                               <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i64> %out
+}
+
+define <vscale x 16 x i8> @lsr_wide_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: lsr_wide_i8:
+; CHECK: lsr z0.b, p0/m, z0.b, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.lsr.wide.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                    <vscale x 16 x i8> %a,
+                                                                    <vscale x 2 x i64> %b)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @lsr_wide_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: lsr_wide_i16:
+; CHECK: lsr z0.h, p0/m, z0.h, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.lsr.wide.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                    <vscale x 8 x i16> %a,
+                                                                    <vscale x 2 x i64> %b)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @lsr_wide_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: lsr_wide_i32:
+; CHECK: lsr z0.s, p0/m, z0.s, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.lsr.wide.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                    <vscale x 4 x i32> %a,
+                                                                    <vscale x 2 x i64> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+declare <vscale x 16 x i8> @llvm.aarch64.sve.asr.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 16 x i8>)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.asr.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 8 x i16>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.asr.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 4 x i32>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.asr.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>, <vscale x 2 x i64>)
+
+declare <vscale x 16 x i8> @llvm.aarch64.sve.asr.wide.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 2 x i64>)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.asr.wide.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 2 x i64>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.asr.wide.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 2 x i64>)
+
+declare <vscale x 16 x i8> @llvm.aarch64.sve.asrd.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, i32)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.asrd.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, i32)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.asrd.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, i32)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.asrd.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>, i32)
+
+declare <vscale x 16 x i8> @llvm.aarch64.sve.insr.nxv16i8(<vscale x 16 x i8>, i8)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.insr.nxv8i16(<vscale x 8 x i16>, i16)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.insr.nxv4i32(<vscale x 4 x i32>, i32)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.insr.nxv2i64(<vscale x 2 x i64>, i64)
+declare <vscale x 8 x half> @llvm.aarch64.sve.insr.nxv8f16(<vscale x 8 x half>, half)
+declare <vscale x 4 x float> @llvm.aarch64.sve.insr.nxv4f32(<vscale x 4 x float>, float)
+declare <vscale x 2 x double> @llvm.aarch64.sve.insr.nxv2f64(<vscale x 2 x double>, double)
+
+declare <vscale x 16 x i8> @llvm.aarch64.sve.lsl.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 16 x i8>)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.lsl.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 8 x i16>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.lsl.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 4 x i32>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.lsl.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>, <vscale x 2 x i64>)
+
+declare <vscale x 16 x i8> @llvm.aarch64.sve.lsl.wide.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 2 x i64>)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.lsl.wide.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 2 x i64>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.lsl.wide.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 2 x i64>)
+
+declare <vscale x 16 x i8> @llvm.aarch64.sve.lsr.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 16 x i8>)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.lsr.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 8 x i16>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.lsr.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 4 x i32>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.lsr.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>, <vscale x 2 x i64>)
+
+declare <vscale x 16 x i8> @llvm.aarch64.sve.lsr.wide.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 2 x i64>)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.lsr.wide.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 2 x i64>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.lsr.wide.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 2 x i64>)
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-sqdec.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-sqdec.ll
new file mode 100644
index 000000000..01f4577d3
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-sqdec.ll
@@ -0,0 +1,337 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve -asm-verbose=0 < %s | FileCheck %s
+
+; Since SQDEC{B|H|W|D|P} and SQINC{B|H|W|D|P} have identical semantics, the tests for
+;   * @llvm.aarch64.sve.sqinc{b|h|w|d|p}, and
+;   * @llvm.aarch64.sve.sqdec{b|h|w|d|p}
+; should also be identical (with the instruction name being adjusted). When
+; updating this file remember to make similar changes in the file testing the
+; other intrinsic.
+
+;
+; SQDECH (vector)
+;
+
+define <vscale x 8 x i16> @sqdech(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: sqdech:
+; CHECK: sqdech z0.h, pow2
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.sqdech.nxv8i16(<vscale x 8 x i16> %a,
+                                                                  i32 0, i32 1)
+  ret <vscale x 8 x i16> %out
+}
+
+;
+; SQDECW (vector)
+;
+
+define <vscale x 4 x i32> @sqdecw(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: sqdecw:
+; CHECK: sqdecw z0.s, vl1, mul #2
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.sqdecw.nxv4i32(<vscale x 4 x i32> %a,
+                                                                  i32 1, i32 2)
+  ret <vscale x 4 x i32> %out
+}
+
+;
+; SQDECD (vector)
+;
+
+define <vscale x 2 x i64> @sqdecd(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: sqdecd:
+; CHECK: sqdecd z0.d, vl2, mul #3
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.sqdecd.nxv2i64(<vscale x 2 x i64> %a,
+                                                                  i32 2, i32 3)
+  ret <vscale x 2 x i64> %out
+}
+
+;
+; SQDECP (vector)
+;
+
+define <vscale x 8 x i16> @sqdecp_b16(<vscale x 8 x i16> %a, <vscale x 8 x i1> %b) {
+; CHECK-LABEL: sqdecp_b16:
+; CHECK: sqdecp z0.h, p0
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.sqdecp.nxv8i16(<vscale x 8 x i16> %a,
+                                                                  <vscale x 8 x i1> %b)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @sqdecp_b32(<vscale x 4 x i32> %a, <vscale x 4 x i1> %b) {
+; CHECK-LABEL: sqdecp_b32:
+; CHECK: sqdecp z0.s, p0
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.sqdecp.nxv4i32(<vscale x 4 x i32> %a,
+                                                                  <vscale x 4 x i1> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @sqdecp_b64(<vscale x 2 x i64> %a, <vscale x 2 x i1> %b) {
+; CHECK-LABEL: sqdecp_b64:
+; CHECK: sqdecp z0.d, p0
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.sqdecp.nxv2i64(<vscale x 2 x i64> %a,
+                                                                  <vscale x 2 x i1> %b)
+  ret <vscale x 2 x i64> %out
+}
+
+;
+; SQDECB (scalar)
+;
+
+define i32 @sqdecb_n32_i32(i32 %a) {
+; CHECK-LABEL: sqdecb_n32_i32:
+; CHECK: sqdecb x0, w0, vl3, mul #4
+; CHECK-NEXT: ret
+  %out = call i32 @llvm.aarch64.sve.sqdecb.n32(i32 %a, i32 3, i32 4)
+  ret i32 %out
+}
+
+define i64 @sqdecb_n32_i64(i32 %a) {
+; CHECK-LABEL: sqdecb_n32_i64:
+; CHECK: sqdecb x0, w0, vl3, mul #4
+; CHECK-NEXT: ret
+  %out = call i32 @llvm.aarch64.sve.sqdecb.n32(i32 %a, i32 3, i32 4)
+  %out_sext = sext i32 %out to i64
+
+  ret i64 %out_sext
+}
+
+define i64 @sqdecb_n64(i64 %a) {
+; CHECK-LABEL: sqdecb_n64:
+; CHECK: sqdecb x0, vl4, mul #5
+; CHECK-NEXT: ret
+  %out = call i64 @llvm.aarch64.sve.sqdecb.n64(i64 %a, i32 4, i32 5)
+  ret i64 %out
+}
+
+;
+; SQDECH (scalar)
+;
+
+define i32 @sqdech_n32_i32(i32 %a) {
+; CHECK-LABEL: sqdech_n32_i32:
+; CHECK: sqdech x0, w0, vl5, mul #6
+; CHECK-NEXT: ret
+  %out = call i32 @llvm.aarch64.sve.sqdech.n32(i32 %a, i32 5, i32 6)
+  ret i32 %out
+}
+
+define i64 @sqdech_n32_i64(i32 %a) {
+; CHECK-LABEL: sqdech_n32_i64:
+; CHECK: sqdech x0, w0, vl3, mul #4
+; CHECK-NEXT: ret
+  %out = call i32 @llvm.aarch64.sve.sqdech.n32(i32 %a, i32 3, i32 4)
+  %out_sext = sext i32 %out to i64
+
+  ret i64 %out_sext
+}
+
+define i64 @sqdech_n64(i64 %a) {
+; CHECK-LABEL: sqdech_n64:
+; CHECK: sqdech x0, vl6, mul #7
+; CHECK-NEXT: ret
+  %out = call i64 @llvm.aarch64.sve.sqdech.n64(i64 %a, i32 6, i32 7)
+  ret i64 %out
+}
+
+;
+; SQDECW (scalar)
+;
+
+define i32 @sqdecw_n32_i32(i32 %a) {
+; CHECK-LABEL: sqdecw_n32_i32:
+; CHECK: sqdecw x0, w0, vl7, mul #8
+; CHECK-NEXT: ret
+  %out = call i32 @llvm.aarch64.sve.sqdecw.n32(i32 %a, i32 7, i32 8)
+  ret i32 %out
+}
+
+define i64 @sqdecw_n32_i64(i32 %a) {
+; CHECK-LABEL: sqdecw_n32_i64:
+; CHECK: sqdecw x0, w0, vl3, mul #4
+; CHECK-NEXT: ret
+  %out = call i32 @llvm.aarch64.sve.sqdecw.n32(i32 %a, i32 3, i32 4)
+  %out_sext = sext i32 %out to i64
+
+  ret i64 %out_sext
+}
+
+define i64 @sqdecw_n64(i64 %a) {
+; CHECK-LABEL: sqdecw_n64:
+; CHECK: sqdecw x0, vl8, mul #9
+; CHECK-NEXT: ret
+  %out = call i64 @llvm.aarch64.sve.sqdecw.n64(i64 %a, i32 8, i32 9)
+  ret i64 %out
+}
+
+;
+; SQDECD (scalar)
+;
+
+define i32 @sqdecd_n32_i32(i32 %a) {
+; CHECK-LABEL: sqdecd_n32_i32:
+; CHECK: sqdecd x0, w0, vl16, mul #10
+; CHECK-NEXT: ret
+  %out = call i32 @llvm.aarch64.sve.sqdecd.n32(i32 %a, i32 9, i32 10)
+  ret i32 %out
+}
+
+define i64 @sqdecd_n32_i64(i32 %a) {
+; CHECK-LABEL: sqdecd_n32_i64:
+; CHECK: sqdecd x0, w0, vl3, mul #4
+; CHECK-NEXT: ret
+  %out = call i32 @llvm.aarch64.sve.sqdecd.n32(i32 %a, i32 3, i32 4)
+  %out_sext = sext i32 %out to i64
+
+  ret i64 %out_sext
+}
+
+define i64 @sqdecd_n64(i64 %a) {
+; CHECK-LABEL: sqdecd_n64:
+; CHECK: sqdecd x0, vl32, mul #11
+; CHECK-NEXT: ret
+  %out = call i64 @llvm.aarch64.sve.sqdecd.n64(i64 %a, i32 10, i32 11)
+  ret i64 %out
+}
+
+;
+; SQDECP (scalar)
+;
+
+define i32 @sqdecp_n32_b8_i32(i32 %a, <vscale x 16 x i1> %b) {
+; CHECK-LABEL: sqdecp_n32_b8_i32:
+; CHECK: sqdecp x0, p0.b, w0
+; CHECK-NEXT: ret
+  %out = call i32 @llvm.aarch64.sve.sqdecp.n32.nxv16i1(i32 %a, <vscale x 16 x i1> %b)
+  ret i32 %out
+}
+
+define i64 @sqdecp_n32_b8_i64(i32 %a, <vscale x 16 x i1> %b) {
+; CHECK-LABEL: sqdecp_n32_b8_i64:
+; CHECK: sqdecp x0, p0.b, w0
+; CHECK-NEXT: ret
+  %out = call i32 @llvm.aarch64.sve.sqdecp.n32.nxv16i1(i32 %a, <vscale x 16 x i1> %b)
+  %out_sext = sext i32 %out to i64
+
+  ret i64 %out_sext
+}
+
+define i32 @sqdecp_n32_b16_i32(i32 %a, <vscale x 8 x i1> %b) {
+; CHECK-LABEL: sqdecp_n32_b16_i32:
+; CHECK: sqdecp x0, p0.h, w0
+; CHECK-NEXT: ret
+  %out = call i32 @llvm.aarch64.sve.sqdecp.n32.nxv8i1(i32 %a, <vscale x 8 x i1> %b)
+  ret i32 %out
+}
+
+define i64 @sqdecp_n32_b16_i64(i32 %a, <vscale x 8 x i1> %b) {
+; CHECK-LABEL: sqdecp_n32_b16_i64:
+; CHECK: sqdecp x0, p0.h, w0
+; CHECK-NEXT: ret
+  %out = call i32 @llvm.aarch64.sve.sqdecp.n32.nxv8i1(i32 %a, <vscale x 8 x i1> %b)
+  %out_sext = sext i32 %out to i64
+
+  ret i64 %out_sext
+}
+
+define i32 @sqdecp_n32_b32_i32(i32 %a, <vscale x 4 x i1> %b) {
+; CHECK-LABEL: sqdecp_n32_b32_i32:
+; CHECK: sqdecp x0, p0.s, w0
+; CHECK-NEXT: ret
+  %out = call i32 @llvm.aarch64.sve.sqdecp.n32.nxv4i1(i32 %a, <vscale x 4 x i1> %b)
+  ret i32 %out
+}
+
+define i64 @sqdecp_n32_b32_i64(i32 %a, <vscale x 4 x i1> %b) {
+; CHECK-LABEL: sqdecp_n32_b32_i64:
+; CHECK: sqdecp x0, p0.s, w0
+; CHECK-NEXT: ret
+  %out = call i32 @llvm.aarch64.sve.sqdecp.n32.nxv4i1(i32 %a, <vscale x 4 x i1> %b)
+  %out_sext = sext i32 %out to i64
+
+  ret i64 %out_sext
+}
+
+define i32 @sqdecp_n32_b64_i32(i32 %a, <vscale x 2 x i1> %b) {
+; CHECK-LABEL: sqdecp_n32_b64_i32:
+; CHECK: sqdecp x0, p0.d, w0
+; CHECK-NEXT: ret
+  %out = call i32 @llvm.aarch64.sve.sqdecp.n32.nxv2i1(i32 %a, <vscale x 2 x i1> %b)
+  ret i32 %out
+}
+
+define i64 @sqdecp_n32_b64_i64(i32 %a, <vscale x 2 x i1> %b) {
+; CHECK-LABEL: sqdecp_n32_b64_i64:
+; CHECK: sqdecp x0, p0.d, w0
+; CHECK-NEXT: ret
+  %out = call i32 @llvm.aarch64.sve.sqdecp.n32.nxv2i1(i32 %a, <vscale x 2 x i1> %b)
+  %out_sext = sext i32 %out to i64
+
+  ret i64 %out_sext
+}
+
+define i64 @sqdecp_n64_b8(i64 %a, <vscale x 16 x i1> %b) {
+; CHECK-LABEL: sqdecp_n64_b8:
+; CHECK: sqdecp x0, p0.b
+; CHECK-NEXT: ret
+  %out = call i64 @llvm.aarch64.sve.sqdecp.n64.nxv16i1(i64 %a, <vscale x 16 x i1> %b)
+  ret i64 %out
+}
+
+define i64 @sqdecp_n64_b16(i64 %a, <vscale x 8 x i1> %b) {
+; CHECK-LABEL: sqdecp_n64_b16:
+; CHECK: sqdecp x0, p0.h
+; CHECK-NEXT: ret
+  %out = call i64 @llvm.aarch64.sve.sqdecp.n64.nxv8i1(i64 %a, <vscale x 8 x i1> %b)
+  ret i64 %out
+}
+
+define i64 @sqdecp_n64_b32(i64 %a, <vscale x 4 x i1> %b) {
+; CHECK-LABEL: sqdecp_n64_b32:
+; CHECK: sqdecp x0, p0.s
+; CHECK-NEXT: ret
+  %out = call i64 @llvm.aarch64.sve.sqdecp.n64.nxv4i1(i64 %a, <vscale x 4 x i1> %b)
+  ret i64 %out
+}
+
+define i64 @sqdecp_n64_b64(i64 %a, <vscale x 2 x i1> %b) {
+; CHECK-LABEL: sqdecp_n64_b64:
+; CHECK: sqdecp x0, p0.d
+; CHECK-NEXT: ret
+  %out = call i64 @llvm.aarch64.sve.sqdecp.n64.nxv2i1(i64 %a, <vscale x 2 x i1> %b)
+  ret i64 %out
+}
+
+; sqdec{h|w|d}(vector, pattern, multiplier)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.sqdech.nxv8i16(<vscale x 8 x i16>, i32, i32)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.sqdecw.nxv4i32(<vscale x 4 x i32>, i32, i32)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.sqdecd.nxv2i64(<vscale x 2 x i64>, i32, i32)
+
+; sqdec{b|h|w|d}(scalar, pattern, multiplier)
+declare i32 @llvm.aarch64.sve.sqdecb.n32(i32, i32, i32)
+declare i64 @llvm.aarch64.sve.sqdecb.n64(i64, i32, i32)
+declare i32 @llvm.aarch64.sve.sqdech.n32(i32, i32, i32)
+declare i64 @llvm.aarch64.sve.sqdech.n64(i64, i32, i32)
+declare i32 @llvm.aarch64.sve.sqdecw.n32(i32, i32, i32)
+declare i64 @llvm.aarch64.sve.sqdecw.n64(i64, i32, i32)
+declare i32 @llvm.aarch64.sve.sqdecd.n32(i32, i32, i32)
+declare i64 @llvm.aarch64.sve.sqdecd.n64(i64, i32, i32)
+
+; sqdecp(scalar, predicate)
+declare i32 @llvm.aarch64.sve.sqdecp.n32.nxv16i1(i32, <vscale x 16 x i1>)
+declare i32 @llvm.aarch64.sve.sqdecp.n32.nxv8i1(i32, <vscale x 8 x i1>)
+declare i32 @llvm.aarch64.sve.sqdecp.n32.nxv4i1(i32, <vscale x 4 x i1>)
+declare i32 @llvm.aarch64.sve.sqdecp.n32.nxv2i1(i32, <vscale x 2 x i1>)
+
+declare i64 @llvm.aarch64.sve.sqdecp.n64.nxv16i1(i64, <vscale x 16 x i1>)
+declare i64 @llvm.aarch64.sve.sqdecp.n64.nxv8i1(i64, <vscale x 8 x i1>)
+declare i64 @llvm.aarch64.sve.sqdecp.n64.nxv4i1(i64, <vscale x 4 x i1>)
+declare i64 @llvm.aarch64.sve.sqdecp.n64.nxv2i1(i64, <vscale x 2 x i1>)
+
+; sqdecp(vector, predicate)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.sqdecp.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i1>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.sqdecp.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i1>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.sqdecp.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i1>)
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-sqinc.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-sqinc.ll
new file mode 100644
index 000000000..d6462b148
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-sqinc.ll
@@ -0,0 +1,337 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve -asm-verbose=0 < %s | FileCheck %s
+
+; Since SQDEC{B|H|W|D|P} and SQINC{B|H|W|D|P} have identical semantics, the tests for
+;   * @llvm.aarch64.sve.sqinc{b|h|w|d|p}, and
+;   * @llvm.aarch64.sve.sqdec{b|h|w|d|p}
+; should also be identical (with the instruction name being adjusted). When
+; updating this file remember to make similar changes in the file testing the
+; other intrinsic.
+
+;
+; SQINCH (vector)
+;
+
+define <vscale x 8 x i16> @sqinch(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: sqinch:
+; CHECK: sqinch z0.h, pow2
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.sqinch.nxv8i16(<vscale x 8 x i16> %a,
+                                                                  i32 0, i32 1)
+  ret <vscale x 8 x i16> %out
+}
+
+;
+; SQINCW (vector)
+;
+
+define <vscale x 4 x i32> @sqincw(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: sqincw:
+; CHECK: sqincw z0.s, vl1, mul #2
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.sqincw.nxv4i32(<vscale x 4 x i32> %a,
+                                                                  i32 1, i32 2)
+  ret <vscale x 4 x i32> %out
+}
+
+;
+; SQINCD (vector)
+;
+
+define <vscale x 2 x i64> @sqincd(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: sqincd:
+; CHECK: sqincd z0.d, vl2, mul #3
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.sqincd.nxv2i64(<vscale x 2 x i64> %a,
+                                                                  i32 2, i32 3)
+  ret <vscale x 2 x i64> %out
+}
+
+;
+; SQINCP (vector)
+;
+
+define <vscale x 8 x i16> @sqincp_b16(<vscale x 8 x i16> %a, <vscale x 8 x i1> %b) {
+; CHECK-LABEL: sqincp_b16:
+; CHECK: sqincp z0.h, p0
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.sqincp.nxv8i16(<vscale x 8 x i16> %a,
+                                                                  <vscale x 8 x i1> %b)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @sqincp_b32(<vscale x 4 x i32> %a, <vscale x 4 x i1> %b) {
+; CHECK-LABEL: sqincp_b32:
+; CHECK: sqincp z0.s, p0
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.sqincp.nxv4i32(<vscale x 4 x i32> %a,
+                                                                  <vscale x 4 x i1> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @sqincp_b64(<vscale x 2 x i64> %a, <vscale x 2 x i1> %b) {
+; CHECK-LABEL: sqincp_b64:
+; CHECK: sqincp z0.d, p0
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.sqincp.nxv2i64(<vscale x 2 x i64> %a,
+                                                                  <vscale x 2 x i1> %b)
+  ret <vscale x 2 x i64> %out
+}
+
+;
+; SQINCB (scalar)
+;
+
+define i32 @sqincb_n32_i32(i32 %a) {
+; CHECK-LABEL: sqincb_n32_i32:
+; CHECK: sqincb x0, w0, vl3, mul #4
+; CHECK-NEXT: ret
+  %out = call i32 @llvm.aarch64.sve.sqincb.n32(i32 %a, i32 3, i32 4)
+  ret i32 %out
+}
+
+define i64 @sqincb_n32_i64(i32 %a) {
+; CHECK-LABEL: sqincb_n32_i64:
+; CHECK: sqincb x0, w0, vl3, mul #4
+; CHECK-NEXT: ret
+  %out = call i32 @llvm.aarch64.sve.sqincb.n32(i32 %a, i32 3, i32 4)
+  %out_sext = sext i32 %out to i64
+
+  ret i64 %out_sext
+}
+
+define i64 @sqincb_n64(i64 %a) {
+; CHECK-LABEL: sqincb_n64:
+; CHECK: sqincb x0, vl4, mul #5
+; CHECK-NEXT: ret
+  %out = call i64 @llvm.aarch64.sve.sqincb.n64(i64 %a, i32 4, i32 5)
+  ret i64 %out
+}
+
+;
+; SQINCH (scalar)
+;
+
+define i32 @sqinch_n32_i32(i32 %a) {
+; CHECK-LABEL: sqinch_n32_i32:
+; CHECK: sqinch x0, w0, vl5, mul #6
+; CHECK-NEXT: ret
+  %out = call i32 @llvm.aarch64.sve.sqinch.n32(i32 %a, i32 5, i32 6)
+  ret i32 %out
+}
+
+define i64 @sqinch_n32_i64(i32 %a) {
+; CHECK-LABEL: sqinch_n32_i64:
+; CHECK: sqinch x0, w0, vl3, mul #4
+; CHECK-NEXT: ret
+  %out = call i32 @llvm.aarch64.sve.sqinch.n32(i32 %a, i32 3, i32 4)
+  %out_sext = sext i32 %out to i64
+
+  ret i64 %out_sext
+}
+
+define i64 @sqinch_n64(i64 %a) {
+; CHECK-LABEL: sqinch_n64:
+; CHECK: sqinch x0, vl6, mul #7
+; CHECK-NEXT: ret
+  %out = call i64 @llvm.aarch64.sve.sqinch.n64(i64 %a, i32 6, i32 7)
+  ret i64 %out
+}
+
+;
+; SQINCW (scalar)
+;
+
+define i32 @sqincw_n32_i32(i32 %a) {
+; CHECK-LABEL: sqincw_n32_i32:
+; CHECK: sqincw x0, w0, vl7, mul #8
+; CHECK-NEXT: ret
+  %out = call i32 @llvm.aarch64.sve.sqincw.n32(i32 %a, i32 7, i32 8)
+  ret i32 %out
+}
+
+define i64 @sqincw_n32_i64(i32 %a) {
+; CHECK-LABEL: sqincw_n32_i64:
+; CHECK: sqincw x0, w0, vl3, mul #4
+; CHECK-NEXT: ret
+  %out = call i32 @llvm.aarch64.sve.sqincw.n32(i32 %a, i32 3, i32 4)
+  %out_sext = sext i32 %out to i64
+
+  ret i64 %out_sext
+}
+
+define i64 @sqincw_n64(i64 %a) {
+; CHECK-LABEL: sqincw_n64:
+; CHECK: sqincw x0, vl8, mul #9
+; CHECK-NEXT: ret
+  %out = call i64 @llvm.aarch64.sve.sqincw.n64(i64 %a, i32 8, i32 9)
+  ret i64 %out
+}
+
+;
+; SQINCD (scalar)
+;
+
+define i32 @sqincd_n32_i32(i32 %a) {
+; CHECK-LABEL: sqincd_n32_i32:
+; CHECK: sqincd x0, w0, vl16, mul #10
+; CHECK-NEXT: ret
+  %out = call i32 @llvm.aarch64.sve.sqincd.n32(i32 %a, i32 9, i32 10)
+  ret i32 %out
+}
+
+define i64 @sqincd_n32_i64(i32 %a) {
+; CHECK-LABEL: sqincd_n32_i64:
+; CHECK: sqincd x0, w0, vl3, mul #4
+; CHECK-NEXT: ret
+  %out = call i32 @llvm.aarch64.sve.sqincd.n32(i32 %a, i32 3, i32 4)
+  %out_sext = sext i32 %out to i64
+
+  ret i64 %out_sext
+}
+
+define i64 @sqincd_n64(i64 %a) {
+; CHECK-LABEL: sqincd_n64:
+; CHECK: sqincd x0, vl32, mul #11
+; CHECK-NEXT: ret
+  %out = call i64 @llvm.aarch64.sve.sqincd.n64(i64 %a, i32 10, i32 11)
+  ret i64 %out
+}
+
+;
+; SQINCP (scalar)
+;
+
+define i32 @sqincp_n32_b8_i32(i32 %a, <vscale x 16 x i1> %b) {
+; CHECK-LABEL: sqincp_n32_b8_i32:
+; CHECK: sqincp x0, p0.b, w0
+; CHECK-NEXT: ret
+  %out = call i32 @llvm.aarch64.sve.sqincp.n32.nxv16i1(i32 %a, <vscale x 16 x i1> %b)
+  ret i32 %out
+}
+
+define i64 @sqincp_n32_b8_i64(i32 %a, <vscale x 16 x i1> %b) {
+; CHECK-LABEL: sqincp_n32_b8_i64:
+; CHECK: sqincp x0, p0.b, w0
+; CHECK-NEXT: ret
+  %out = call i32 @llvm.aarch64.sve.sqincp.n32.nxv16i1(i32 %a, <vscale x 16 x i1> %b)
+  %out_sext = sext i32 %out to i64
+
+  ret i64 %out_sext
+}
+
+define i32 @sqincp_n32_b16_i32(i32 %a, <vscale x 8 x i1> %b) {
+; CHECK-LABEL: sqincp_n32_b16_i32:
+; CHECK: sqincp x0, p0.h, w0
+; CHECK-NEXT: ret
+  %out = call i32 @llvm.aarch64.sve.sqincp.n32.nxv8i1(i32 %a, <vscale x 8 x i1> %b)
+  ret i32 %out
+}
+
+define i64 @sqincp_n32_b16_i64(i32 %a, <vscale x 8 x i1> %b) {
+; CHECK-LABEL: sqincp_n32_b16_i64:
+; CHECK: sqincp x0, p0.h, w0
+; CHECK-NEXT: ret
+  %out = call i32 @llvm.aarch64.sve.sqincp.n32.nxv8i1(i32 %a, <vscale x 8 x i1> %b)
+  %out_sext = sext i32 %out to i64
+
+  ret i64 %out_sext
+}
+
+define i32 @sqincp_n32_b32_i32(i32 %a, <vscale x 4 x i1> %b) {
+; CHECK-LABEL: sqincp_n32_b32_i32:
+; CHECK: sqincp x0, p0.s, w0
+; CHECK-NEXT: ret
+  %out = call i32 @llvm.aarch64.sve.sqincp.n32.nxv4i1(i32 %a, <vscale x 4 x i1> %b)
+  ret i32 %out
+}
+
+define i64 @sqincp_n32_b32_i64(i32 %a, <vscale x 4 x i1> %b) {
+; CHECK-LABEL: sqincp_n32_b32_i64:
+; CHECK: sqincp x0, p0.s, w0
+; CHECK-NEXT: ret
+  %out = call i32 @llvm.aarch64.sve.sqincp.n32.nxv4i1(i32 %a, <vscale x 4 x i1> %b)
+  %out_sext = sext i32 %out to i64
+
+  ret i64 %out_sext
+}
+
+define i32 @sqincp_n32_b64_i32(i32 %a, <vscale x 2 x i1> %b) {
+; CHECK-LABEL: sqincp_n32_b64_i32:
+; CHECK: sqincp x0, p0.d, w0
+; CHECK-NEXT: ret
+  %out = call i32 @llvm.aarch64.sve.sqincp.n32.nxv2i1(i32 %a, <vscale x 2 x i1> %b)
+  ret i32 %out
+}
+
+define i64 @sqincp_n32_b64_i64(i32 %a, <vscale x 2 x i1> %b) {
+; CHECK-LABEL: sqincp_n32_b64_i64:
+; CHECK: sqincp x0, p0.d, w0
+; CHECK-NEXT: ret
+  %out = call i32 @llvm.aarch64.sve.sqincp.n32.nxv2i1(i32 %a, <vscale x 2 x i1> %b)
+  %out_sext = sext i32 %out to i64
+
+  ret i64 %out_sext
+}
+
+define i64 @sqincp_n64_b8(i64 %a, <vscale x 16 x i1> %b) {
+; CHECK-LABEL: sqincp_n64_b8:
+; CHECK: sqincp x0, p0.b
+; CHECK-NEXT: ret
+  %out = call i64 @llvm.aarch64.sve.sqincp.n64.nxv16i1(i64 %a, <vscale x 16 x i1> %b)
+  ret i64 %out
+}
+
+define i64 @sqincp_n64_b16(i64 %a, <vscale x 8 x i1> %b) {
+; CHECK-LABEL: sqincp_n64_b16:
+; CHECK: sqincp x0, p0.h
+; CHECK-NEXT: ret
+  %out = call i64 @llvm.aarch64.sve.sqincp.n64.nxv8i1(i64 %a, <vscale x 8 x i1> %b)
+  ret i64 %out
+}
+
+define i64 @sqincp_n64_b32(i64 %a, <vscale x 4 x i1> %b) {
+; CHECK-LABEL: sqincp_n64_b32:
+; CHECK: sqincp x0, p0.s
+; CHECK-NEXT: ret
+  %out = call i64 @llvm.aarch64.sve.sqincp.n64.nxv4i1(i64 %a, <vscale x 4 x i1> %b)
+  ret i64 %out
+}
+
+define i64 @sqincp_n64_b64(i64 %a, <vscale x 2 x i1> %b) {
+; CHECK-LABEL: sqincp_n64_b64:
+; CHECK: sqincp x0, p0.d
+; CHECK-NEXT: ret
+  %out = call i64 @llvm.aarch64.sve.sqincp.n64.nxv2i1(i64 %a, <vscale x 2 x i1> %b)
+  ret i64 %out
+}
+
+; sqinc{h|w|d}(vector, pattern, multiplier)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.sqinch.nxv8i16(<vscale x 8 x i16>, i32, i32)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.sqincw.nxv4i32(<vscale x 4 x i32>, i32, i32)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.sqincd.nxv2i64(<vscale x 2 x i64>, i32, i32)
+
+; sqinc{b|h|w|d}(scalar, pattern, multiplier)
+declare i32 @llvm.aarch64.sve.sqincb.n32(i32, i32, i32)
+declare i64 @llvm.aarch64.sve.sqincb.n64(i64, i32, i32)
+declare i32 @llvm.aarch64.sve.sqinch.n32(i32, i32, i32)
+declare i64 @llvm.aarch64.sve.sqinch.n64(i64, i32, i32)
+declare i32 @llvm.aarch64.sve.sqincw.n32(i32, i32, i32)
+declare i64 @llvm.aarch64.sve.sqincw.n64(i64, i32, i32)
+declare i32 @llvm.aarch64.sve.sqincd.n32(i32, i32, i32)
+declare i64 @llvm.aarch64.sve.sqincd.n64(i64, i32, i32)
+
+; sqincp(scalar, predicate)
+declare i32 @llvm.aarch64.sve.sqincp.n32.nxv16i1(i32, <vscale x 16 x i1>)
+declare i32 @llvm.aarch64.sve.sqincp.n32.nxv8i1(i32, <vscale x 8 x i1>)
+declare i32 @llvm.aarch64.sve.sqincp.n32.nxv4i1(i32, <vscale x 4 x i1>)
+declare i32 @llvm.aarch64.sve.sqincp.n32.nxv2i1(i32, <vscale x 2 x i1>)
+
+declare i64 @llvm.aarch64.sve.sqincp.n64.nxv16i1(i64, <vscale x 16 x i1>)
+declare i64 @llvm.aarch64.sve.sqincp.n64.nxv8i1(i64, <vscale x 8 x i1>)
+declare i64 @llvm.aarch64.sve.sqincp.n64.nxv4i1(i64, <vscale x 4 x i1>)
+declare i64 @llvm.aarch64.sve.sqincp.n64.nxv2i1(i64, <vscale x 2 x i1>)
+
+; sqincp(vector, predicate)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.sqincp.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i1>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.sqincp.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i1>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.sqincp.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i1>)
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-stores.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-stores.ll
new file mode 100644
index 000000000..29d369ba4
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-stores.ll
@@ -0,0 +1,95 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+;
+; STNT1B
+;
+
+define void @stnt1b_i8(<vscale x 16 x i8> %data, <vscale x 16 x i1> %pred, <vscale x 16 x i8>* %addr) {
+; CHECK-LABEL: stnt1b_i8:
+; CHECK: stnt1b { z0.b }, p0, [x0]
+; CHECK-NEXT: ret
+  call void @llvm.aarch64.sve.stnt1.nxv16i8(<vscale x 16 x i8> %data,
+                                            <vscale x 16 x i1> %pred,
+                                            <vscale x 16 x i8>* %addr)
+  ret void
+}
+
+;
+; STNT1H
+;
+
+define void @stnt1h_i16(<vscale x 8 x i16> %data, <vscale x 8 x i1> %pred, <vscale x 8 x i16>* %addr) {
+; CHECK-LABEL: stnt1h_i16:
+; CHECK: stnt1h { z0.h }, p0, [x0]
+; CHECK-NEXT: ret
+  call void @llvm.aarch64.sve.stnt1.nxv8i16(<vscale x 8 x i16> %data,
+                                            <vscale x 8 x i1> %pred,
+                                            <vscale x 8 x i16>* %addr)
+  ret void
+}
+
+define void @stnt1h_f16(<vscale x 8 x half> %data, <vscale x 8 x i1> %pred, <vscale x 8 x half>* %addr) {
+; CHECK-LABEL: stnt1h_f16:
+; CHECK: stnt1h { z0.h }, p0, [x0]
+; CHECK-NEXT: ret
+  call void @llvm.aarch64.sve.stnt1.nxv8f16(<vscale x 8 x half> %data,
+                                            <vscale x 8 x i1> %pred,
+                                            <vscale x 8 x half>* %addr)
+  ret void
+}
+
+;
+; STNT1W
+;
+
+define void @stnt1w_i32(<vscale x 4 x i32> %data, <vscale x 4 x i1> %pred, <vscale x 4 x i32>* %addr) {
+; CHECK-LABEL: stnt1w_i32:
+; CHECK: stnt1w { z0.s }, p0, [x0]
+; CHECK-NEXT: ret
+  call void @llvm.aarch64.sve.stnt1.nxv4i32(<vscale x 4 x i32> %data,
+                                            <vscale x 4 x i1> %pred,
+                                            <vscale x 4 x i32>* %addr)
+  ret void
+}
+
+define void @stnt1w_f32(<vscale x 4 x float> %data, <vscale x 4 x i1> %pred, <vscale x 4 x float>* %addr) {
+; CHECK-LABEL: stnt1w_f32:
+; CHECK: stnt1w { z0.s }, p0, [x0]
+; CHECK-NEXT: ret
+  call void @llvm.aarch64.sve.stnt1.nxv4f32(<vscale x 4 x float> %data,
+                                            <vscale x 4 x i1> %pred,
+                                            <vscale x 4 x float>* %addr)
+  ret void
+}
+
+;
+; STNT1D
+;
+
+define void @stnt1d_i64(<vscale x 2 x i64> %data, <vscale x 2 x i1> %pred, <vscale x 2 x i64>* %addr) {
+; CHECK-LABEL: stnt1d_i64:
+; CHECK: stnt1d { z0.d }, p0, [x0]
+; CHECK-NEXT: ret
+  call void @llvm.aarch64.sve.stnt1.nxv2i64(<vscale x 2 x i64> %data,
+                                            <vscale x 2 x i1> %pred,
+                                            <vscale x 2 x i64>* %addr)
+  ret void
+}
+
+define void @stnt1d_f64(<vscale x 2 x double> %data, <vscale x 2 x i1> %pred, <vscale x 2 x double>* %addr) {
+; CHECK-LABEL: stnt1d_f64:
+; CHECK: stnt1d { z0.d }, p0, [x0]
+; CHECK-NEXT: ret
+  call void @llvm.aarch64.sve.stnt1.nxv2f64(<vscale x 2 x double> %data,
+                                            <vscale x 2 x i1> %pred,
+                                            <vscale x 2 x double>* %addr)
+  ret void
+}
+
+declare void @llvm.aarch64.sve.stnt1.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i1>, <vscale x 16 x i8>*)
+declare void @llvm.aarch64.sve.stnt1.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i1>, <vscale x 8 x i16>*)
+declare void @llvm.aarch64.sve.stnt1.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i1>, <vscale x 4 x i32>*)
+declare void @llvm.aarch64.sve.stnt1.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i1>, <vscale x 2 x i64>*)
+declare void @llvm.aarch64.sve.stnt1.nxv8f16(<vscale x 8 x half>, <vscale x 8 x i1>, <vscale x 8 x half>*)
+declare void @llvm.aarch64.sve.stnt1.nxv4f32(<vscale x 4 x float>, <vscale x 4 x i1>, <vscale x 4 x float>*)
+declare void @llvm.aarch64.sve.stnt1.nxv2f64(<vscale x 2 x double>, <vscale x 2 x i1>, <vscale x 2 x double>*)
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-uqdec.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-uqdec.ll
new file mode 100644
index 000000000..8329ad97e
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-uqdec.ll
@@ -0,0 +1,257 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve -asm-verbose=0 < %s | FileCheck %s
+
+; Since UQDEC{B|H|W|D|P} and UQINC{B|H|W|D|P} have identical semantics, the tests for
+;   * @llvm.aarch64.sve.uqinc{b|h|w|d|p}, and
+;   * @llvm.aarch64.sve.uqdec{b|h|w|d|p}
+; should also be identical (with the instruction name being adjusted). When
+; updating this file remember to make similar changes in the file testing the
+; other intrinsic.
+
+;
+; UQDECH (vector)
+;
+
+define <vscale x 8 x i16> @uqdech(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: uqdech:
+; CHECK: uqdech z0.h, pow2
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.uqdech.nxv8i16(<vscale x 8 x i16> %a,
+                                                                  i32 0, i32 1)
+  ret <vscale x 8 x i16> %out
+}
+
+;
+; UQDECW (vector)
+;
+
+define <vscale x 4 x i32> @uqdecw(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: uqdecw:
+; CHECK: uqdecw z0.s, vl1, mul #2
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.uqdecw.nxv4i32(<vscale x 4 x i32> %a,
+                                                                  i32 1, i32 2)
+  ret <vscale x 4 x i32> %out
+}
+
+;
+; UQDECD (vector)
+;
+
+define <vscale x 2 x i64> @uqdecd(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: uqdecd:
+; CHECK: uqdecd z0.d, vl2, mul #3
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.uqdecd.nxv2i64(<vscale x 2 x i64> %a,
+                                                                  i32 2, i32 3)
+  ret <vscale x 2 x i64> %out
+}
+
+;
+; UQDECP (vector)
+;
+
+define <vscale x 8 x i16> @uqdecp_b16(<vscale x 8 x i16> %a, <vscale x 8 x i1> %b) {
+; CHECK-LABEL: uqdecp_b16:
+; CHECK: uqdecp z0.h, p0
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.uqdecp.nxv8i16(<vscale x 8 x i16> %a,
+                                                                  <vscale x 8 x i1> %b)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @uqdecp_b32(<vscale x 4 x i32> %a, <vscale x 4 x i1> %b) {
+; CHECK-LABEL: uqdecp_b32:
+; CHECK: uqdecp z0.s, p0
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.uqdecp.nxv4i32(<vscale x 4 x i32> %a,
+                                                                  <vscale x 4 x i1> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @uqdecp_b64(<vscale x 2 x i64> %a, <vscale x 2 x i1> %b) {
+; CHECK-LABEL: uqdecp_b64:
+; CHECK: uqdecp z0.d, p0
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.uqdecp.nxv2i64(<vscale x 2 x i64> %a,
+                                                                  <vscale x 2 x i1> %b)
+  ret <vscale x 2 x i64> %out
+}
+
+;
+; UQDECB (scalar)
+;
+
+define i32 @uqdecb_n32(i32 %a) {
+; CHECK-LABEL: uqdecb_n32:
+; CHECK: uqdecb w0, vl3, mul #4
+; CHECK-NEXT: ret
+  %out = call i32 @llvm.aarch64.sve.uqdecb.n32(i32 %a, i32 3, i32 4)
+  ret i32 %out
+}
+
+define i64 @uqdecb_n64(i64 %a) {
+; CHECK-LABEL: uqdecb_n64:
+; CHECK: uqdecb x0, vl4, mul #5
+; CHECK-NEXT: ret
+  %out = call i64 @llvm.aarch64.sve.uqdecb.n64(i64 %a, i32 4, i32 5)
+  ret i64 %out
+}
+
+;
+; UQDECH (scalar)
+;
+
+define i32 @uqdech_n32(i32 %a) {
+; CHECK-LABEL: uqdech_n32:
+; CHECK: uqdech w0, vl5, mul #6
+; CHECK-NEXT: ret
+  %out = call i32 @llvm.aarch64.sve.uqdech.n32(i32 %a, i32 5, i32 6)
+  ret i32 %out
+}
+
+define i64 @uqdech_n64(i64 %a) {
+; CHECK-LABEL: uqdech_n64:
+; CHECK: uqdech x0, vl6, mul #7
+; CHECK-NEXT: ret
+  %out = call i64 @llvm.aarch64.sve.uqdech.n64(i64 %a, i32 6, i32 7)
+  ret i64 %out
+}
+
+;
+; UQDECW (scalar)
+;
+
+define i32 @uqdecw_n32(i32 %a) {
+; CHECK-LABEL: uqdecw_n32:
+; CHECK: uqdecw w0, vl7, mul #8
+; CHECK-NEXT: ret
+  %out = call i32 @llvm.aarch64.sve.uqdecw.n32(i32 %a, i32 7, i32 8)
+  ret i32 %out
+}
+
+define i64 @uqdecw_n64(i64 %a) {
+; CHECK-LABEL: uqdecw_n64:
+; CHECK: uqdecw x0, vl8, mul #9
+; CHECK-NEXT: ret
+  %out = call i64 @llvm.aarch64.sve.uqdecw.n64(i64 %a, i32 8, i32 9)
+  ret i64 %out
+}
+
+;
+; UQDECD (scalar)
+;
+
+define i32 @uqdecd_n32(i32 %a) {
+; CHECK-LABEL: uqdecd_n32:
+; CHECK: uqdecd w0, vl16, mul #10
+; CHECK-NEXT: ret
+  %out = call i32 @llvm.aarch64.sve.uqdecd.n32(i32 %a, i32 9, i32 10)
+  ret i32 %out
+}
+
+define i64 @uqdecd_n64(i64 %a) {
+; CHECK-LABEL: uqdecd_n64:
+; CHECK: uqdecd x0, vl32, mul #11
+; CHECK-NEXT: ret
+  %out = call i64 @llvm.aarch64.sve.uqdecd.n64(i64 %a, i32 10, i32 11)
+  ret i64 %out
+}
+
+;
+; UQDECP (scalar)
+;
+
+define i32 @uqdecp_n32_b8(i32 %a, <vscale x 16 x i1> %b) {
+; CHECK-LABEL: uqdecp_n32_b8:
+; CHECK: uqdecp w0, p0.b
+; CHECK-NEXT: ret
+  %out = call i32 @llvm.aarch64.sve.uqdecp.n32.nxv16i1(i32 %a, <vscale x 16 x i1> %b)
+  ret i32 %out
+}
+
+define i32 @uqdecp_n32_b16(i32 %a, <vscale x 8 x i1> %b) {
+; CHECK-LABEL: uqdecp_n32_b16:
+; CHECK: uqdecp w0, p0.h
+; CHECK-NEXT: ret
+  %out = call i32 @llvm.aarch64.sve.uqdecp.n32.nxv8i1(i32 %a, <vscale x 8 x i1> %b)
+  ret i32 %out
+}
+
+define i32 @uqdecp_n32_b32(i32 %a, <vscale x 4 x i1> %b) {
+; CHECK-LABEL: uqdecp_n32_b32:
+; CHECK: uqdecp w0, p0.s
+; CHECK-NEXT: ret
+  %out = call i32 @llvm.aarch64.sve.uqdecp.n32.nxv4i1(i32 %a, <vscale x 4 x i1> %b)
+  ret i32 %out
+}
+
+define i32 @uqdecp_n32_b64(i32 %a, <vscale x 2 x i1> %b) {
+; CHECK-LABEL: uqdecp_n32_b64:
+; CHECK: uqdecp w0, p0.d
+; CHECK-NEXT: ret
+  %out = call i32 @llvm.aarch64.sve.uqdecp.n32.nxv2i1(i32 %a, <vscale x 2 x i1> %b)
+  ret i32 %out
+}
+
+define i64 @uqdecp_n64_b8(i64 %a, <vscale x 16 x i1> %b) {
+; CHECK-LABEL: uqdecp_n64_b8:
+; CHECK: uqdecp x0, p0.b
+; CHECK-NEXT: ret
+  %out = call i64 @llvm.aarch64.sve.uqdecp.n64.nxv16i1(i64 %a, <vscale x 16 x i1> %b)
+  ret i64 %out
+}
+
+define i64 @uqdecp_n64_b16(i64 %a, <vscale x 8 x i1> %b) {
+; CHECK-LABEL: uqdecp_n64_b16:
+; CHECK: uqdecp x0, p0.h
+; CHECK-NEXT: ret
+  %out = call i64 @llvm.aarch64.sve.uqdecp.n64.nxv8i1(i64 %a, <vscale x 8 x i1> %b)
+  ret i64 %out
+}
+
+define i64 @uqdecp_n64_b32(i64 %a, <vscale x 4 x i1> %b) {
+; CHECK-LABEL: uqdecp_n64_b32:
+; CHECK: uqdecp x0, p0.s
+; CHECK-NEXT: ret
+  %out = call i64 @llvm.aarch64.sve.uqdecp.n64.nxv4i1(i64 %a, <vscale x 4 x i1> %b)
+  ret i64 %out
+}
+
+define i64 @uqdecp_n64_b64(i64 %a, <vscale x 2 x i1> %b) {
+; CHECK-LABEL: uqdecp_n64_b64:
+; CHECK: uqdecp x0, p0.d
+; CHECK-NEXT: ret
+  %out = call i64 @llvm.aarch64.sve.uqdecp.n64.nxv2i1(i64 %a, <vscale x 2 x i1> %b)
+  ret i64 %out
+}
+
+; uqdec{h|w|d}(vector, pattern, multiplier)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.uqdech.nxv8i16(<vscale x 8 x i16>, i32, i32)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.uqdecw.nxv4i32(<vscale x 4 x i32>, i32, i32)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.uqdecd.nxv2i64(<vscale x 2 x i64>, i32, i32)
+
+; uqdec{b|h|w|d}(scalar, pattern, multiplier)
+declare i32 @llvm.aarch64.sve.uqdecb.n32(i32, i32, i32)
+declare i64 @llvm.aarch64.sve.uqdecb.n64(i64, i32, i32)
+declare i32 @llvm.aarch64.sve.uqdech.n32(i32, i32, i32)
+declare i64 @llvm.aarch64.sve.uqdech.n64(i64, i32, i32)
+declare i32 @llvm.aarch64.sve.uqdecw.n32(i32, i32, i32)
+declare i64 @llvm.aarch64.sve.uqdecw.n64(i64, i32, i32)
+declare i32 @llvm.aarch64.sve.uqdecd.n32(i32, i32, i32)
+declare i64 @llvm.aarch64.sve.uqdecd.n64(i64, i32, i32)
+
+; uqdecp(scalar, predicate)
+declare i32 @llvm.aarch64.sve.uqdecp.n32.nxv16i1(i32, <vscale x 16 x i1>)
+declare i32 @llvm.aarch64.sve.uqdecp.n32.nxv8i1(i32, <vscale x 8 x i1>)
+declare i32 @llvm.aarch64.sve.uqdecp.n32.nxv4i1(i32, <vscale x 4 x i1>)
+declare i32 @llvm.aarch64.sve.uqdecp.n32.nxv2i1(i32, <vscale x 2 x i1>)
+
+declare i64 @llvm.aarch64.sve.uqdecp.n64.nxv16i1(i64, <vscale x 16 x i1>)
+declare i64 @llvm.aarch64.sve.uqdecp.n64.nxv8i1(i64, <vscale x 8 x i1>)
+declare i64 @llvm.aarch64.sve.uqdecp.n64.nxv4i1(i64, <vscale x 4 x i1>)
+declare i64 @llvm.aarch64.sve.uqdecp.n64.nxv2i1(i64, <vscale x 2 x i1>)
+
+; uqdecp(vector, predicate)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.uqdecp.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i1>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.uqdecp.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i1>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.uqdecp.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i1>)
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-uqinc.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-uqinc.ll
new file mode 100644
index 000000000..844403921
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-uqinc.ll
@@ -0,0 +1,257 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve -asm-verbose=0 < %s | FileCheck %s
+
+; Since UQDEC{B|H|W|D|P} and UQINC{B|H|W|D|P} have identical semantics, the tests for
+;   * @llvm.aarch64.sve.uqinc{b|h|w|d|p}, and
+;   * @llvm.aarch64.sve.uqdec{b|h|w|d|p}
+; should also be identical (with the instruction name being adjusted). When
+; updating this file remember to make similar changes in the file testing the
+; other intrinsic.
+
+;
+; UQINCH (vector)
+;
+
+define <vscale x 8 x i16> @uqinch(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: uqinch:
+; CHECK: uqinch z0.h, pow2
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.uqinch.nxv8i16(<vscale x 8 x i16> %a,
+                                                                  i32 0, i32 1)
+  ret <vscale x 8 x i16> %out
+}
+
+;
+; UQINCW (vector)
+;
+
+define <vscale x 4 x i32> @uqincw(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: uqincw:
+; CHECK: uqincw z0.s, vl1, mul #2
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.uqincw.nxv4i32(<vscale x 4 x i32> %a,
+                                                                  i32 1, i32 2)
+  ret <vscale x 4 x i32> %out
+}
+
+;
+; UQINCD (vector)
+;
+
+define <vscale x 2 x i64> @uqincd(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: uqincd:
+; CHECK: uqincd z0.d, vl2, mul #3
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.uqincd.nxv2i64(<vscale x 2 x i64> %a,
+                                                                  i32 2, i32 3)
+  ret <vscale x 2 x i64> %out
+}
+
+;
+; UQINCP (vector)
+;
+
+define <vscale x 8 x i16> @uqincp_b16(<vscale x 8 x i16> %a, <vscale x 8 x i1> %b) {
+; CHECK-LABEL: uqincp_b16:
+; CHECK: uqincp z0.h, p0
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.uqincp.nxv8i16(<vscale x 8 x i16> %a,
+                                                                  <vscale x 8 x i1> %b)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @uqincp_b32(<vscale x 4 x i32> %a, <vscale x 4 x i1> %b) {
+; CHECK-LABEL: uqincp_b32:
+; CHECK: uqincp z0.s, p0
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.uqincp.nxv4i32(<vscale x 4 x i32> %a,
+                                                                  <vscale x 4 x i1> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @uqincp_b64(<vscale x 2 x i64> %a, <vscale x 2 x i1> %b) {
+; CHECK-LABEL: uqincp_b64:
+; CHECK: uqincp z0.d, p0
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.uqincp.nxv2i64(<vscale x 2 x i64> %a,
+                                                                  <vscale x 2 x i1> %b)
+  ret <vscale x 2 x i64> %out
+}
+
+;
+; UQINCB (scalar)
+;
+
+define i32 @uqincb_n32(i32 %a) {
+; CHECK-LABEL: uqincb_n32:
+; CHECK: uqincb w0, vl3, mul #4
+; CHECK-NEXT: ret
+  %out = call i32 @llvm.aarch64.sve.uqincb.n32(i32 %a, i32 3, i32 4)
+  ret i32 %out
+}
+
+define i64 @uqincb_n64(i64 %a) {
+; CHECK-LABEL: uqincb_n64:
+; CHECK: uqincb x0, vl4, mul #5
+; CHECK-NEXT: ret
+  %out = call i64 @llvm.aarch64.sve.uqincb.n64(i64 %a, i32 4, i32 5)
+  ret i64 %out
+}
+
+;
+; UQINCH (scalar)
+;
+
+define i32 @uqinch_n32(i32 %a) {
+; CHECK-LABEL: uqinch_n32:
+; CHECK: uqinch w0, vl5, mul #6
+; CHECK-NEXT: ret
+  %out = call i32 @llvm.aarch64.sve.uqinch.n32(i32 %a, i32 5, i32 6)
+  ret i32 %out
+}
+
+define i64 @uqinch_n64(i64 %a) {
+; CHECK-LABEL: uqinch_n64:
+; CHECK: uqinch x0, vl6, mul #7
+; CHECK-NEXT: ret
+  %out = call i64 @llvm.aarch64.sve.uqinch.n64(i64 %a, i32 6, i32 7)
+  ret i64 %out
+}
+
+;
+; UQINCW (scalar)
+;
+
+define i32 @uqincw_n32(i32 %a) {
+; CHECK-LABEL: uqincw_n32:
+; CHECK: uqincw w0, vl7, mul #8
+; CHECK-NEXT: ret
+  %out = call i32 @llvm.aarch64.sve.uqincw.n32(i32 %a, i32 7, i32 8)
+  ret i32 %out
+}
+
+define i64 @uqincw_n64(i64 %a) {
+; CHECK-LABEL: uqincw_n64:
+; CHECK: uqincw x0, vl8, mul #9
+; CHECK-NEXT: ret
+  %out = call i64 @llvm.aarch64.sve.uqincw.n64(i64 %a, i32 8, i32 9)
+  ret i64 %out
+}
+
+;
+; UQINCD (scalar)
+;
+
+define i32 @uqincd_n32(i32 %a) {
+; CHECK-LABEL: uqincd_n32:
+; CHECK: uqincd w0, vl16, mul #10
+; CHECK-NEXT: ret
+  %out = call i32 @llvm.aarch64.sve.uqincd.n32(i32 %a, i32 9, i32 10)
+  ret i32 %out
+}
+
+define i64 @uqincd_n64(i64 %a) {
+; CHECK-LABEL: uqincd_n64:
+; CHECK: uqincd x0, vl32, mul #11
+; CHECK-NEXT: ret
+  %out = call i64 @llvm.aarch64.sve.uqincd.n64(i64 %a, i32 10, i32 11)
+  ret i64 %out
+}
+
+;
+; UQINCP (scalar)
+;
+
+define i32 @uqincp_n32_b8(i32 %a, <vscale x 16 x i1> %b) {
+; CHECK-LABEL: uqincp_n32_b8:
+; CHECK: uqincp w0, p0.b
+; CHECK-NEXT: ret
+  %out = call i32 @llvm.aarch64.sve.uqincp.n32.nxv16i1(i32 %a, <vscale x 16 x i1> %b)
+  ret i32 %out
+}
+
+define i32 @uqincp_n32_b16(i32 %a, <vscale x 8 x i1> %b) {
+; CHECK-LABEL: uqincp_n32_b16:
+; CHECK: uqincp w0, p0.h
+; CHECK-NEXT: ret
+  %out = call i32 @llvm.aarch64.sve.uqincp.n32.nxv8i1(i32 %a, <vscale x 8 x i1> %b)
+  ret i32 %out
+}
+
+define i32 @uqincp_n32_b32(i32 %a, <vscale x 4 x i1> %b) {
+; CHECK-LABEL: uqincp_n32_b32:
+; CHECK: uqincp w0, p0.s
+; CHECK-NEXT: ret
+  %out = call i32 @llvm.aarch64.sve.uqincp.n32.nxv4i1(i32 %a, <vscale x 4 x i1> %b)
+  ret i32 %out
+}
+
+define i32 @uqincp_n32_b64(i32 %a, <vscale x 2 x i1> %b) {
+; CHECK-LABEL: uqincp_n32_b64:
+; CHECK: uqincp w0, p0.d
+; CHECK-NEXT: ret
+  %out = call i32 @llvm.aarch64.sve.uqincp.n32.nxv2i1(i32 %a, <vscale x 2 x i1> %b)
+  ret i32 %out
+}
+
+define i64 @uqincp_n64_b8(i64 %a, <vscale x 16 x i1> %b) {
+; CHECK-LABEL: uqincp_n64_b8:
+; CHECK: uqincp x0, p0.b
+; CHECK-NEXT: ret
+  %out = call i64 @llvm.aarch64.sve.uqincp.n64.nxv16i1(i64 %a, <vscale x 16 x i1> %b)
+  ret i64 %out
+}
+
+define i64 @uqincp_n64_b16(i64 %a, <vscale x 8 x i1> %b) {
+; CHECK-LABEL: uqincp_n64_b16:
+; CHECK: uqincp x0, p0.h
+; CHECK-NEXT: ret
+  %out = call i64 @llvm.aarch64.sve.uqincp.n64.nxv8i1(i64 %a, <vscale x 8 x i1> %b)
+  ret i64 %out
+}
+
+define i64 @uqincp_n64_b32(i64 %a, <vscale x 4 x i1> %b) {
+; CHECK-LABEL: uqincp_n64_b32:
+; CHECK: uqincp x0, p0.s
+; CHECK-NEXT: ret
+  %out = call i64 @llvm.aarch64.sve.uqincp.n64.nxv4i1(i64 %a, <vscale x 4 x i1> %b)
+  ret i64 %out
+}
+
+define i64 @uqincp_n64_b64(i64 %a, <vscale x 2 x i1> %b) {
+; CHECK-LABEL: uqincp_n64_b64:
+; CHECK: uqincp x0, p0.d
+; CHECK-NEXT: ret
+  %out = call i64 @llvm.aarch64.sve.uqincp.n64.nxv2i1(i64 %a, <vscale x 2 x i1> %b)
+  ret i64 %out
+}
+
+; uqinc{h|w|d}(vector, pattern, multiplier)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.uqinch.nxv8i16(<vscale x 8 x i16>, i32, i32)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.uqincw.nxv4i32(<vscale x 4 x i32>, i32, i32)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.uqincd.nxv2i64(<vscale x 2 x i64>, i32, i32)
+
+; uqinc{b|h|w|d}(scalar, pattern, multiplier)
+declare i32 @llvm.aarch64.sve.uqincb.n32(i32, i32, i32)
+declare i64 @llvm.aarch64.sve.uqincb.n64(i64, i32, i32)
+declare i32 @llvm.aarch64.sve.uqinch.n32(i32, i32, i32)
+declare i64 @llvm.aarch64.sve.uqinch.n64(i64, i32, i32)
+declare i32 @llvm.aarch64.sve.uqincw.n32(i32, i32, i32)
+declare i64 @llvm.aarch64.sve.uqincw.n64(i64, i32, i32)
+declare i32 @llvm.aarch64.sve.uqincd.n32(i32, i32, i32)
+declare i64 @llvm.aarch64.sve.uqincd.n64(i64, i32, i32)
+
+; uqincp(scalar, predicate)
+declare i32 @llvm.aarch64.sve.uqincp.n32.nxv16i1(i32, <vscale x 16 x i1>)
+declare i32 @llvm.aarch64.sve.uqincp.n32.nxv8i1(i32, <vscale x 8 x i1>)
+declare i32 @llvm.aarch64.sve.uqincp.n32.nxv4i1(i32, <vscale x 4 x i1>)
+declare i32 @llvm.aarch64.sve.uqincp.n32.nxv2i1(i32, <vscale x 2 x i1>)
+
+declare i64 @llvm.aarch64.sve.uqincp.n64.nxv16i1(i64, <vscale x 16 x i1>)
+declare i64 @llvm.aarch64.sve.uqincp.n64.nxv8i1(i64, <vscale x 8 x i1>)
+declare i64 @llvm.aarch64.sve.uqincp.n64.nxv4i1(i64, <vscale x 4 x i1>)
+declare i64 @llvm.aarch64.sve.uqincp.n64.nxv2i1(i64, <vscale x 2 x i1>)
+
+; uqincp(vector, predicate)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.uqincp.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i1>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.uqincp.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i1>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.uqincp.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i1>)
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-while.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-while.ll
new file mode 100644
index 000000000..0590c74d2
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-while.ll
@@ -0,0 +1,309 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+;
+; WHILELE
+;
+
+define <vscale x 16 x i1> @whilele_b_ww(i32 %a, i32 %b) {
+; CHECK-LABEL: whilele_b_ww:
+; CHECK: whilele p0.b, w0, w1
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.whilele.nxv16i1.i32(i32 %a, i32 %b)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 16 x i1> @whilele_b_xx(i64 %a, i64 %b) {
+; CHECK-LABEL: whilele_b_xx:
+; CHECK: whilele p0.b, x0, x1
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.whilele.nxv16i1.i64(i64 %a, i64 %b)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @whilele_h_ww(i32 %a, i32 %b) {
+; CHECK-LABEL: whilele_h_ww:
+; CHECK: whilele p0.h, w0, w1
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.whilele.nxv8i1.i32(i32 %a, i32 %b)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 8 x i1> @whilele_h_xx(i64 %a, i64 %b) {
+; CHECK-LABEL: whilele_h_xx:
+; CHECK: whilele p0.h, x0, x1
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.whilele.nxv8i1.i64(i64 %a, i64 %b)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @whilele_s_ww(i32 %a, i32 %b) {
+; CHECK-LABEL: whilele_s_ww:
+; CHECK: whilele p0.s, w0, w1
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.whilele.nxv4i1.i32(i32 %a, i32 %b)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 4 x i1> @whilele_s_xx(i64 %a, i64 %b) {
+; CHECK-LABEL: whilele_s_xx:
+; CHECK: whilele p0.s, x0, x1
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.whilele.nxv4i1.i64(i64 %a, i64 %b)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 2 x i1> @whilele_d_ww(i32 %a, i32 %b) {
+; CHECK-LABEL: whilele_d_ww:
+; CHECK: whilele p0.d, w0, w1
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.whilele.nxv2i1.i32(i32 %a, i32 %b)
+  ret <vscale x 2 x i1> %out
+}
+
+define <vscale x 2 x i1> @whilele_d_xx(i64 %a, i64 %b) {
+; CHECK-LABEL: whilele_d_xx:
+; CHECK: whilele p0.d, x0, x1
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.whilele.nxv2i1.i64(i64 %a, i64 %b)
+  ret <vscale x 2 x i1> %out
+}
+
+;
+; WHILELO
+;
+
+define <vscale x 16 x i1> @whilelo_b_ww(i32 %a, i32 %b) {
+; CHECK-LABEL: whilelo_b_ww:
+; CHECK: whilelo p0.b, w0, w1
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.whilelo.nxv16i1.i32(i32 %a, i32 %b)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 16 x i1> @whilelo_b_xx(i64 %a, i64 %b) {
+; CHECK-LABEL: whilelo_b_xx:
+; CHECK: whilelo p0.b, x0, x1
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.whilelo.nxv16i1.i64(i64 %a, i64 %b)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @whilelo_h_ww(i32 %a, i32 %b) {
+; CHECK-LABEL: whilelo_h_ww:
+; CHECK: whilelo p0.h, w0, w1
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.whilelo.nxv8i1.i32(i32 %a, i32 %b)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 8 x i1> @whilelo_h_xx(i64 %a, i64 %b) {
+; CHECK-LABEL: whilelo_h_xx:
+; CHECK: whilelo p0.h, x0, x1
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.whilelo.nxv8i1.i64(i64 %a, i64 %b)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @whilelo_s_ww(i32 %a, i32 %b) {
+; CHECK-LABEL: whilelo_s_ww:
+; CHECK: whilelo p0.s, w0, w1
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.whilelo.nxv4i1.i32(i32 %a, i32 %b)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 4 x i1> @whilelo_s_xx(i64 %a, i64 %b) {
+; CHECK-LABEL: whilelo_s_xx:
+; CHECK: whilelo p0.s, x0, x1
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.whilelo.nxv4i1.i64(i64 %a, i64 %b)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 2 x i1> @whilelo_d_ww(i32 %a, i32 %b) {
+; CHECK-LABEL: whilelo_d_ww:
+; CHECK: whilelo p0.d, w0, w1
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.whilelo.nxv2i1.i32(i32 %a, i32 %b)
+  ret <vscale x 2 x i1> %out
+}
+
+define <vscale x 2 x i1> @whilelo_d_xx(i64 %a, i64 %b) {
+; CHECK-LABEL: whilelo_d_xx:
+; CHECK: whilelo p0.d, x0, x1
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.whilelo.nxv2i1.i64(i64 %a, i64 %b)
+  ret <vscale x 2 x i1> %out
+}
+
+;
+; WHILELS
+;
+
+define <vscale x 16 x i1> @whilels_b_ww(i32 %a, i32 %b) {
+; CHECK-LABEL: whilels_b_ww:
+; CHECK: whilels p0.b, w0, w1
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.whilels.nxv16i1.i32(i32 %a, i32 %b)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 16 x i1> @whilels_b_xx(i64 %a, i64 %b) {
+; CHECK-LABEL: whilels_b_xx:
+; CHECK: whilels p0.b, x0, x1
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.whilels.nxv16i1.i64(i64 %a, i64 %b)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @whilels_h_ww(i32 %a, i32 %b) {
+; CHECK-LABEL: whilels_h_ww:
+; CHECK: whilels p0.h, w0, w1
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.whilels.nxv8i1.i32(i32 %a, i32 %b)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 8 x i1> @whilels_h_xx(i64 %a, i64 %b) {
+; CHECK-LABEL: whilels_h_xx:
+; CHECK: whilels p0.h, x0, x1
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.whilels.nxv8i1.i64(i64 %a, i64 %b)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @whilels_s_ww(i32 %a, i32 %b) {
+; CHECK-LABEL: whilels_s_ww:
+; CHECK: whilels p0.s, w0, w1
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.whilels.nxv4i1.i32(i32 %a, i32 %b)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 4 x i1> @whilels_s_xx(i64 %a, i64 %b) {
+; CHECK-LABEL: whilels_s_xx:
+; CHECK: whilels p0.s, x0, x1
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.whilels.nxv4i1.i64(i64 %a, i64 %b)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 2 x i1> @whilels_d_ww(i32 %a, i32 %b) {
+; CHECK-LABEL: whilels_d_ww:
+; CHECK: whilels p0.d, w0, w1
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.whilels.nxv2i1.i32(i32 %a, i32 %b)
+  ret <vscale x 2 x i1> %out
+}
+
+define <vscale x 2 x i1> @whilels_d_xx(i64 %a, i64 %b) {
+; CHECK-LABEL: whilels_d_xx:
+; CHECK: whilels p0.d, x0, x1
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.whilels.nxv2i1.i64(i64 %a, i64 %b)
+  ret <vscale x 2 x i1> %out
+}
+
+;
+; WHILELT
+;
+
+define <vscale x 16 x i1> @whilelt_b_ww(i32 %a, i32 %b) {
+; CHECK-LABEL: whilelt_b_ww:
+; CHECK: whilelt p0.b, w0, w1
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.whilelt.nxv16i1.i32(i32 %a, i32 %b)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 16 x i1> @whilelt_b_xx(i64 %a, i64 %b) {
+; CHECK-LABEL: whilelt_b_xx:
+; CHECK: whilelt p0.b, x0, x1
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.whilelt.nxv16i1.i64(i64 %a, i64 %b)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @whilelt_h_ww(i32 %a, i32 %b) {
+; CHECK-LABEL: whilelt_h_ww:
+; CHECK: whilelt p0.h, w0, w1
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.whilelt.nxv8i1.i32(i32 %a, i32 %b)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 8 x i1> @whilelt_h_xx(i64 %a, i64 %b) {
+; CHECK-LABEL: whilelt_h_xx:
+; CHECK: whilelt p0.h, x0, x1
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.whilelt.nxv8i1.i64(i64 %a, i64 %b)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @whilelt_s_ww(i32 %a, i32 %b) {
+; CHECK-LABEL: whilelt_s_ww:
+; CHECK: whilelt p0.s, w0, w1
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.whilelt.nxv4i1.i32(i32 %a, i32 %b)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 4 x i1> @whilelt_s_xx(i64 %a, i64 %b) {
+; CHECK-LABEL: whilelt_s_xx:
+; CHECK: whilelt p0.s, x0, x1
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.whilelt.nxv4i1.i64(i64 %a, i64 %b)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 2 x i1> @whilelt_d_ww(i32 %a, i32 %b) {
+; CHECK-LABEL: whilelt_d_ww:
+; CHECK: whilelt p0.d, w0, w1
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.whilelt.nxv2i1.i32(i32 %a, i32 %b)
+  ret <vscale x 2 x i1> %out
+}
+
+define <vscale x 2 x i1> @whilelt_d_xx(i64 %a, i64 %b) {
+; CHECK-LABEL: whilelt_d_xx:
+; CHECK: whilelt p0.d, x0, x1
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.whilelt.nxv2i1.i64(i64 %a, i64 %b)
+  ret <vscale x 2 x i1> %out
+}
+
+declare <vscale x 16 x i1> @llvm.aarch64.sve.whilele.nxv16i1.i32(i32, i32)
+declare <vscale x 16 x i1> @llvm.aarch64.sve.whilele.nxv16i1.i64(i64, i64)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.whilele.nxv8i1.i32(i32, i32)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.whilele.nxv8i1.i64(i64, i64)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.whilele.nxv4i1.i32(i32, i32)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.whilele.nxv4i1.i64(i64, i64)
+declare <vscale x 2 x i1> @llvm.aarch64.sve.whilele.nxv2i1.i32(i32, i32)
+declare <vscale x 2 x i1> @llvm.aarch64.sve.whilele.nxv2i1.i64(i64, i64)
+
+declare <vscale x 16 x i1> @llvm.aarch64.sve.whilelo.nxv16i1.i32(i32, i32)
+declare <vscale x 16 x i1> @llvm.aarch64.sve.whilelo.nxv16i1.i64(i64, i64)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.whilelo.nxv8i1.i32(i32, i32)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.whilelo.nxv8i1.i64(i64, i64)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.whilelo.nxv4i1.i32(i32, i32)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.whilelo.nxv4i1.i64(i64, i64)
+declare <vscale x 2 x i1> @llvm.aarch64.sve.whilelo.nxv2i1.i32(i32, i32)
+declare <vscale x 2 x i1> @llvm.aarch64.sve.whilelo.nxv2i1.i64(i64, i64)
+
+declare <vscale x 16 x i1> @llvm.aarch64.sve.whilels.nxv16i1.i32(i32, i32)
+declare <vscale x 16 x i1> @llvm.aarch64.sve.whilels.nxv16i1.i64(i64, i64)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.whilels.nxv8i1.i32(i32, i32)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.whilels.nxv8i1.i64(i64, i64)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.whilels.nxv4i1.i32(i32, i32)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.whilels.nxv4i1.i64(i64, i64)
+declare <vscale x 2 x i1> @llvm.aarch64.sve.whilels.nxv2i1.i32(i32, i32)
+declare <vscale x 2 x i1> @llvm.aarch64.sve.whilels.nxv2i1.i64(i64, i64)
+
+declare <vscale x 16 x i1> @llvm.aarch64.sve.whilelt.nxv16i1.i32(i32, i32)
+declare <vscale x 16 x i1> @llvm.aarch64.sve.whilelt.nxv16i1.i64(i64, i64)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.whilelt.nxv8i1.i32(i32, i32)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.whilelt.nxv8i1.i64(i64, i64)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.whilelt.nxv4i1.i32(i32, i32)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.whilelt.nxv4i1.i64(i64, i64)
+declare <vscale x 2 x i1> @llvm.aarch64.sve.whilelt.nxv2i1.i32(i32, i32)
+declare <vscale x 2 x i1> @llvm.aarch64.sve.whilelt.nxv2i1.i64(i64, i64)
diff --git a/llvm/test/CodeGen/AArch64/sve-masked-ldst-nonext.ll b/llvm/test/CodeGen/AArch64/sve-masked-ldst-nonext.ll
new file mode 100644
index 000000000..b7542668c
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-masked-ldst-nonext.ll
@@ -0,0 +1,193 @@
+; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve -asm-verbose=0 < %s | FileCheck %s
+
+;
+; Masked Loads
+;
+
+define <vscale x 2 x i64> @masked_load_nxv2i64(<vscale x 2 x i64> *%a, <vscale x 2 x i1> %mask) nounwind {
+; CHECK-LABEL: masked_load_nxv2i64:
+; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]
+; CHECK-NEXT: ret
+  %load = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64(<vscale x 2 x i64> *%a, i32 8, <vscale x 2 x i1> %mask, <vscale x 2 x i64> undef)
+  ret <vscale x 2 x i64> %load
+}
+
+define <vscale x 4 x i32> @masked_load_nxv4i32(<vscale x 4 x i32> *%a, <vscale x 4 x i1> %mask) nounwind {
+; CHECK-LABEL: masked_load_nxv4i32:
+; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]
+; CHECK-NEXT: ret
+  %load = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32(<vscale x 4 x i32> *%a, i32 4, <vscale x 4 x i1> %mask, <vscale x 4 x i32> undef)
+  ret <vscale x 4 x i32> %load
+}
+
+define <vscale x 8 x i16> @masked_load_nxv8i16(<vscale x 8 x i16> *%a, <vscale x 8 x i1> %mask) nounwind {
+; CHECK-LABEL: masked_load_nxv8i16:
+; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]
+; CHECK-NEXT: ret
+  %load = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16(<vscale x 8 x i16> *%a, i32 2, <vscale x 8 x i1> %mask, <vscale x 8 x i16> undef)
+  ret <vscale x 8 x i16> %load
+}
+
+define <vscale x 16 x i8> @masked_load_nxv16i8(<vscale x 16 x i8> *%a, <vscale x 16 x i1> %mask) nounwind {
+; CHECK-LABEL: masked_load_nxv16i8:
+; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]
+; CHECK-NEXT: ret
+  %load = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8(<vscale x 16 x i8> *%a, i32 1, <vscale x 16 x i1> %mask, <vscale x 16 x i8> undef)
+  ret <vscale x 16 x i8> %load
+}
+
+define <vscale x 2 x double> @masked_load_nxv2f64(<vscale x 2 x double> *%a, <vscale x 2 x i1> %mask) nounwind {
+; CHECK-LABEL: masked_load_nxv2f64:
+; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]
+; CHECK-NEXT: ret
+  %load = call <vscale x 2 x double> @llvm.masked.load.nxv2f64(<vscale x 2 x double> *%a, i32 8, <vscale x 2 x i1> %mask, <vscale x 2 x double> undef)
+  ret <vscale x 2 x double> %load
+}
+
+define <vscale x 2 x float> @masked_load_nxv2f32(<vscale x 2 x float> *%a, <vscale x 2 x i1> %mask) nounwind {
+; CHECK-LABEL: masked_load_nxv2f32:
+; CHECK-NEXT: ld1w { z0.d }, p0/z, [x0]
+; CHECK-NEXT: ret
+  %load = call <vscale x 2 x float> @llvm.masked.load.nxv2f32(<vscale x 2 x float> *%a, i32 4, <vscale x 2 x i1> %mask, <vscale x 2 x float> undef)
+  ret <vscale x 2 x float> %load
+}
+
+define <vscale x 2 x half> @masked_load_nxv2f16(<vscale x 2 x half> *%a, <vscale x 2 x i1> %mask) nounwind {
+; CHECK-LABEL: masked_load_nxv2f16:
+; CHECK-NEXT: ld1h { z0.d }, p0/z, [x0]
+; CHECK-NEXT: ret
+  %load = call <vscale x 2 x half> @llvm.masked.load.nxv2f16(<vscale x 2 x half> *%a, i32 2, <vscale x 2 x i1> %mask, <vscale x 2 x half> undef)
+  ret <vscale x 2 x half> %load
+}
+
+define <vscale x 4 x float> @masked_load_nxv4f32(<vscale x 4 x float> *%a, <vscale x 4 x i1> %mask) nounwind {
+; CHECK-LABEL: masked_load_nxv4f32:
+; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]
+; CHECK-NEXT: ret
+  %load = call <vscale x 4 x float> @llvm.masked.load.nxv4f32(<vscale x 4 x float> *%a, i32 4, <vscale x 4 x i1> %mask, <vscale x 4 x float> undef)
+  ret <vscale x 4 x float> %load
+}
+
+define <vscale x 4 x half> @masked_load_nxv4f16(<vscale x 4 x half> *%a, <vscale x 4 x i1> %mask) nounwind {
+; CHECK-LABEL: masked_load_nxv4f16:
+; CHECK-NEXT: ld1h { z0.s }, p0/z, [x0]
+; CHECK-NEXT: ret
+  %load = call <vscale x 4 x half> @llvm.masked.load.nxv4f16(<vscale x 4 x half> *%a, i32 2, <vscale x 4 x i1> %mask, <vscale x 4 x half> undef)
+  ret <vscale x 4 x half> %load
+}
+
+define <vscale x 8 x half> @masked_load_nxv8f16(<vscale x 8 x half> *%a, <vscale x 8 x i1> %mask) nounwind {
+; CHECK-LABEL: masked_load_nxv8f16:
+; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]
+; CHECK-NEXT: ret
+  %load = call <vscale x 8 x half> @llvm.masked.load.nxv8f16(<vscale x 8 x half> *%a, i32 2, <vscale x 8 x i1> %mask, <vscale x 8 x half> undef)
+  ret <vscale x 8 x half> %load
+}
+
+;
+; Masked Stores
+;
+
+define void @masked_store_nxv2i64(<vscale x 2 x i64> *%a, <vscale x 2 x i64> %val, <vscale x 2 x i1> %mask) nounwind {
+; CHECK-LABEL: masked_store_nxv2i64:
+; CHECK-NEXT: st1d { z0.d }, p0, [x0]
+; CHECK-NEXT: ret
+  call void @llvm.masked.store.nxv2i64(<vscale x 2 x i64> %val, <vscale x 2 x i64> *%a, i32 8, <vscale x 2 x i1> %mask)
+  ret void
+}
+
+define void @masked_store_nxv4i32(<vscale x 4 x i32> *%a, <vscale x 4 x i32> %val, <vscale x 4 x i1> %mask) nounwind {
+; CHECK-LABEL: masked_store_nxv4i32:
+; CHECK-NEXT: st1w { z0.s }, p0, [x0]
+; CHECK-NEXT: ret
+  call void @llvm.masked.store.nxv4i32(<vscale x 4 x i32> %val, <vscale x 4 x i32> *%a, i32 4, <vscale x 4 x i1> %mask)
+  ret void
+}
+
+define void @masked_store_nxv8i16(<vscale x 8 x i16> *%a, <vscale x 8 x i16> %val, <vscale x 8 x i1> %mask) nounwind {
+; CHECK-LABEL: masked_store_nxv8i16:
+; CHECK-NEXT: st1h { z0.h }, p0, [x0]
+; CHECK-NEXT: ret
+  call void @llvm.masked.store.nxv8i16(<vscale x 8 x i16> %val, <vscale x 8 x i16> *%a, i32 2, <vscale x 8 x i1> %mask)
+  ret void
+}
+
+define void @masked_store_nxv16i8(<vscale x 16 x i8> *%a, <vscale x 16 x i8> %val, <vscale x 16 x i1> %mask) nounwind {
+; CHECK-LABEL: masked_store_nxv16i8:
+; CHECK-NEXT: st1b { z0.b }, p0, [x0]
+; CHECK-NEXT: ret
+  call void @llvm.masked.store.nxv16i8(<vscale x 16 x i8> %val, <vscale x 16 x i8> *%a, i32 1, <vscale x 16 x i1> %mask)
+  ret void
+}
+
+define void @masked_store_nxv2f64(<vscale x 2 x double> *%a, <vscale x 2 x double> %val, <vscale x 2 x i1> %mask) nounwind {
+; CHECK-LABEL: masked_store_nxv2f64:
+; CHECK-NEXT: st1d { z0.d }, p0, [x0]
+; CHECK-NEXT: ret
+  call void @llvm.masked.store.nxv2f64(<vscale x 2 x double> %val, <vscale x 2 x double> *%a, i32 8, <vscale x 2 x i1> %mask)
+  ret void
+}
+
+define void @masked_store_nxv2f32(<vscale x 2 x float> *%a, <vscale x 2 x float> %val, <vscale x 2 x i1> %mask) nounwind {
+; CHECK-LABEL: masked_store_nxv2f32:
+; CHECK-NEXT: st1w { z0.d }, p0, [x0]
+; CHECK-NEXT: ret
+  call void @llvm.masked.store.nxv2f32(<vscale x 2 x float> %val, <vscale x 2 x float> *%a, i32 4, <vscale x 2 x i1> %mask)
+  ret void
+}
+
+define void @masked_store_nxv2f16(<vscale x 2 x half> *%a, <vscale x 2 x half> %val, <vscale x 2 x i1> %mask) nounwind {
+; CHECK-LABEL: masked_store_nxv2f16:
+; CHECK-NEXT: st1h { z0.d }, p0, [x0]
+; CHECK-NEXT: ret
+  call void @llvm.masked.store.nxv2f16(<vscale x 2 x half> %val, <vscale x 2 x half> *%a, i32 4, <vscale x 2 x i1> %mask)
+  ret void
+}
+
+define void @masked_store_nxv4f32(<vscale x 4 x float> *%a, <vscale x 4 x float> %val, <vscale x 4 x i1> %mask) nounwind {
+; CHECK-LABEL: masked_store_nxv4f32:
+; CHECK-NEXT: st1w { z0.s }, p0, [x0]
+; CHECK-NEXT: ret
+  call void @llvm.masked.store.nxv4f32(<vscale x 4 x float> %val, <vscale x 4 x float> *%a, i32 4, <vscale x 4 x i1> %mask)
+  ret void
+}
+
+define void @masked_store_nxv4f16(<vscale x 4 x half> *%a, <vscale x 4 x half> %val, <vscale x 4 x i1> %mask) nounwind {
+; CHECK-LABEL: masked_store_nxv4f16:
+; CHECK-NEXT: st1h { z0.s }, p0, [x0]
+; CHECK-NEXT: ret
+  call void @llvm.masked.store.nxv4f16(<vscale x 4 x half> %val, <vscale x 4 x half> *%a, i32 2, <vscale x 4 x i1> %mask)
+  ret void
+}
+
+define void @masked_store_nxv8f16(<vscale x 8 x half> *%a, <vscale x 8 x half> %val, <vscale x 8 x i1> %mask) nounwind {
+; CHECK-LABEL: masked_store_nxv8f16:
+; CHECK-NEXT: st1h { z0.h }, p0, [x0]
+; CHECK-NEXT: ret
+  call void @llvm.masked.store.nxv8f16(<vscale x 8 x half> %val, <vscale x 8 x half> *%a, i32 2, <vscale x 8 x i1> %mask)
+  ret void
+}
+
+declare <vscale x 2 x i64> @llvm.masked.load.nxv2i64(<vscale x 2 x i64>*, i32, <vscale x 2 x i1>, <vscale x 2 x i64>)
+declare <vscale x 4 x i32> @llvm.masked.load.nxv4i32(<vscale x 4 x i32>*, i32, <vscale x 4 x i1>, <vscale x 4 x i32>)
+declare <vscale x 8 x i16> @llvm.masked.load.nxv8i16(<vscale x 8 x i16>*, i32, <vscale x 8 x i1>, <vscale x 8 x i16>)
+declare <vscale x 16 x i8> @llvm.masked.load.nxv16i8(<vscale x 16 x i8>*, i32, <vscale x 16 x i1>, <vscale x 16 x i8>)
+
+declare <vscale x 2 x double> @llvm.masked.load.nxv2f64(<vscale x 2 x double>*, i32, <vscale x 2 x i1>, <vscale x 2 x double>)
+declare <vscale x 2 x float> @llvm.masked.load.nxv2f32(<vscale x 2 x float>*, i32, <vscale x 2 x i1>, <vscale x 2 x float>)
+declare <vscale x 2 x half> @llvm.masked.load.nxv2f16(<vscale x 2 x half>*, i32, <vscale x 2 x i1>, <vscale x 2 x half>)
+declare <vscale x 4 x float> @llvm.masked.load.nxv4f32(<vscale x 4 x float>*, i32, <vscale x 4 x i1>, <vscale x 4 x float>)
+declare <vscale x 4 x half> @llvm.masked.load.nxv4f16(<vscale x 4 x half>*, i32, <vscale x 4 x i1>, <vscale x 4 x half>)
+declare <vscale x 8 x half> @llvm.masked.load.nxv8f16(<vscale x 8 x half>*, i32, <vscale x 8 x i1>, <vscale x 8 x half>)
+
+declare void @llvm.masked.store.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>*, i32, <vscale x 2 x i1>)
+declare void @llvm.masked.store.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>*, i32, <vscale x 4 x i1>)
+declare void @llvm.masked.store.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i16>*, i32, <vscale x 8 x i1>)
+declare void @llvm.masked.store.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i8>*, i32, <vscale x 16 x i1>)
+
+declare void @llvm.masked.store.nxv2f64(<vscale x 2 x double>, <vscale x 2 x double>*, i32, <vscale x 2 x i1>)
+declare void @llvm.masked.store.nxv2f32(<vscale x 2 x float>, <vscale x 2 x float>*, i32, <vscale x 2 x i1>)
+declare void @llvm.masked.store.nxv2f16(<vscale x 2 x half>, <vscale x 2 x half>*, i32, <vscale x 2 x i1>)
+declare void @llvm.masked.store.nxv4f32(<vscale x 4 x float>, <vscale x 4 x float>*, i32, <vscale x 4 x i1>)
+declare void @llvm.masked.store.nxv4f16(<vscale x 4 x half>, <vscale x 4 x half>*, i32, <vscale x 4 x i1>)
+declare void @llvm.masked.store.nxv8f16(<vscale x 8 x half>, <vscale x 8 x half>*, i32, <vscale x 8 x i1>)
diff --git a/llvm/test/CodeGen/AArch64/sve-masked-ldst-sext.ll b/llvm/test/CodeGen/AArch64/sve-masked-ldst-sext.ll
new file mode 100644
index 000000000..8edd35435
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-masked-ldst-sext.ll
@@ -0,0 +1,66 @@
+; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve < %s | FileCheck %s
+
+;
+; Masked Loads
+;
+
+define <vscale x 2 x i64> @masked_sload_nxv2i8(<vscale x 2 x i8> *%a, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: masked_sload_nxv2i8:
+; CHECK: ld1sb { [[IN:z[0-9]+]].d }, [[PG:p[0-9]+]]/z, [x0]
+; CHECK-NEXT: ret
+  %load = call <vscale x 2 x i8> @llvm.masked.load.nxv2i8(<vscale x 2 x i8> *%a, i32 1, <vscale x 2 x i1> %mask, <vscale x 2 x i8> undef)
+  %ext = sext <vscale x 2 x i8> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %ext
+}
+
+define <vscale x 2 x i64> @masked_sload_nxv2i16(<vscale x 2 x i16> *%a, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: masked_sload_nxv2i16:
+; CHECK: ld1sh { [[IN:z[0-9]+]].d }, [[PG:p[0-9]+]]/z, [x0]
+; CHECK-NEXT: ret
+  %load = call <vscale x 2 x i16> @llvm.masked.load.nxv2i16(<vscale x 2 x i16> *%a, i32 1, <vscale x 2 x i1> %mask, <vscale x 2 x i16> undef)
+  %ext = sext <vscale x 2 x i16> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %ext
+}
+
+define <vscale x 2 x i64> @masked_sload_nxv2i32(<vscale x 2 x i32> *%a, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: masked_sload_nxv2i32:
+; CHECK: ld1sw { [[IN:z[0-9]+]].d }, [[PG:p[0-9]+]]/z, [x0]
+; CHECK-NEXT: ret
+  %load = call <vscale x 2 x i32> @llvm.masked.load.nxv2i32(<vscale x 2 x i32> *%a, i32 1, <vscale x 2 x i1> %mask, <vscale x 2 x i32> undef)
+  %ext = sext <vscale x 2 x i32> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %ext
+}
+
+define <vscale x 4 x i32> @masked_sload_nxv4i8(<vscale x 4 x i8> *%a, <vscale x 4 x i1> %mask) {
+; CHECK-LABEL: masked_sload_nxv4i8:
+; CHECK: ld1sb { [[IN:z[0-9]+]].s }, [[PG:p[0-9]+]]/z, [x0]
+; CHECK-NEXT: ret
+  %load = call <vscale x 4 x i8> @llvm.masked.load.nxv4i8(<vscale x 4 x i8> *%a, i32 1, <vscale x 4 x i1> %mask, <vscale x 4 x i8> undef)
+  %ext = sext <vscale x 4 x i8> %load to <vscale x 4 x i32>
+  ret <vscale x 4 x i32> %ext
+}
+
+define <vscale x 4 x i32> @masked_sload_nxv4i16(<vscale x 4 x i16> *%a, <vscale x 4 x i1> %mask) {
+; CHECK-LABEL: masked_sload_nxv4i16:
+; CHECK: ld1sh { [[IN:z[0-9]+]].s }, [[PG:p[0-9]+]]/z, [x0]
+; CHECK-NEXT: ret
+  %load = call <vscale x 4 x i16> @llvm.masked.load.nxv4i16(<vscale x 4 x i16> *%a, i32 1, <vscale x 4 x i1> %mask, <vscale x 4 x i16> undef)
+  %ext = sext <vscale x 4 x i16> %load to <vscale x 4 x i32>
+  ret <vscale x 4 x i32> %ext
+}
+
+define <vscale x 8 x i16> @masked_sload_nxv8i8(<vscale x 8 x i8> *%a, <vscale x 8 x i1> %mask) {
+; CHECK-LABEL: masked_sload_nxv8i8:
+; CHECK: ld1sb { [[IN:z[0-9]+]].h }, [[PG:p[0-9]+]]/z, [x0]
+; CHECK-NEXT: ret
+  %load = call <vscale x 8 x i8> @llvm.masked.load.nxv8i8(<vscale x 8 x i8> *%a, i32 1, <vscale x 8 x i1> %mask, <vscale x 8 x i8> undef)
+  %ext = sext <vscale x 8 x i8> %load to <vscale x 8 x i16>
+  ret <vscale x 8 x i16> %ext
+}
+
+declare <vscale x 2 x i8> @llvm.masked.load.nxv2i8(<vscale x 2 x i8>*, i32, <vscale x 2 x i1>, <vscale x 2 x i8>)
+declare <vscale x 2 x i16> @llvm.masked.load.nxv2i16(<vscale x 2 x i16>*, i32, <vscale x 2 x i1>, <vscale x 2 x i16>)
+declare <vscale x 2 x i32> @llvm.masked.load.nxv2i32(<vscale x 2 x i32>*, i32, <vscale x 2 x i1>, <vscale x 2 x i32>)
+declare <vscale x 4 x i8> @llvm.masked.load.nxv4i8(<vscale x 4 x i8>*, i32, <vscale x 4 x i1>, <vscale x 4 x i8>)
+declare <vscale x 4 x i16> @llvm.masked.load.nxv4i16(<vscale x 4 x i16>*, i32, <vscale x 4 x i1>, <vscale x 4 x i16>)
+declare <vscale x 8 x i8> @llvm.masked.load.nxv8i8(<vscale x 8 x i8>*, i32, <vscale x 8 x i1>, <vscale x 8 x i8>)
diff --git a/llvm/test/CodeGen/AArch64/sve-masked-ldst-trunc.ll b/llvm/test/CodeGen/AArch64/sve-masked-ldst-trunc.ll
new file mode 100644
index 000000000..d2069da00
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-masked-ldst-trunc.ll
@@ -0,0 +1,66 @@
+; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve -asm-verbose=0 < %s | FileCheck %s
+
+;
+; Masked Stores
+;
+
+define void @masked_trunc_store_nxv2i8(<vscale x 2 x i64> *%a, <vscale x 2 x i64> %val, <vscale x 2 x i8> *%b, <vscale x 2 x i1> %mask) nounwind {
+; CHECK-LABEL: masked_trunc_store_nxv2i8:
+; CHECK-NEXT: st1b { z0.d }, p0, [x1]
+; CHECK-NEXT: ret
+  %trunc = trunc <vscale x 2 x i64> %val to <vscale x 2 x i8>
+  call void @llvm.masked.store.nxv2i8(<vscale x 2 x i8> %trunc, <vscale x 2 x i8> *%b, i32 8, <vscale x 2 x i1> %mask)
+  ret void
+}
+
+define void @masked_trunc_store_nxv2i16(<vscale x 2 x i64> *%a, <vscale x 2 x i64> %val, <vscale x 2 x i16> *%b, <vscale x 2 x i1> %mask) nounwind {
+; CHECK-LABEL: masked_trunc_store_nxv2i16:
+; CHECK-NEXT: st1h { z0.d }, p0, [x1]
+; CHECK-NEXT: ret
+  %trunc = trunc <vscale x 2 x i64> %val to <vscale x 2 x i16>
+  call void @llvm.masked.store.nxv2i16(<vscale x 2 x i16> %trunc, <vscale x 2 x i16> *%b, i32 8, <vscale x 2 x i1> %mask)
+  ret void
+}
+
+define void @masked_trunc_store_nxv2i32(<vscale x 2 x i64> *%a, <vscale x 2 x i64> %val, <vscale x 2 x i32> *%b, <vscale x 2 x i1> %mask) nounwind {
+; CHECK-LABEL: masked_trunc_store_nxv2i32:
+; CHECK-NEXT: st1w { z0.d }, p0, [x1]
+; CHECK-NEXT: ret
+  %trunc = trunc <vscale x 2 x i64> %val to <vscale x 2 x i32>
+  call void @llvm.masked.store.nxv2i32(<vscale x 2 x i32> %trunc, <vscale x 2 x i32> *%b, i32 8, <vscale x 2 x i1> %mask)
+  ret void
+}
+
+define void @masked_trunc_store_nxv4i8(<vscale x 4 x i32> *%a, <vscale x 4 x i32> %val, <vscale x 4 x i8> *%b, <vscale x 4 x i1> %mask) nounwind {
+; CHECK-LABEL: masked_trunc_store_nxv4i8:
+; CHECK-NEXT: st1b { z0.s }, p0, [x1]
+; CHECK-NEXT: ret
+  %trunc = trunc <vscale x 4 x i32> %val to <vscale x 4 x i8>
+  call void @llvm.masked.store.nxv4i8(<vscale x 4 x i8> %trunc, <vscale x 4 x i8> *%b, i32 4, <vscale x 4 x i1> %mask)
+  ret void
+}
+
+define void @masked_trunc_store_nxv4i16(<vscale x 4 x i32> *%a, <vscale x 4 x i32> %val, <vscale x 4 x i16> *%b, <vscale x 4 x i1> %mask) nounwind {
+; CHECK-LABEL: masked_trunc_store_nxv4i16:
+; CHECK-NEXT: st1h { z0.s }, p0, [x1]
+; CHECK-NEXT: ret
+  %trunc = trunc <vscale x 4 x i32> %val to <vscale x 4 x i16>
+  call void @llvm.masked.store.nxv4i16(<vscale x 4 x i16> %trunc, <vscale x 4 x i16> *%b, i32 4, <vscale x 4 x i1> %mask)
+  ret void
+}
+
+define void @masked_trunc_store_nxv8i8(<vscale x 8 x i16> *%a, <vscale x 8 x i16> %val, <vscale x 8 x i8> *%b, <vscale x 8 x i1> %mask) nounwind {
+; CHECK-LABEL: masked_trunc_store_nxv8i8:
+; CHECK-NEXT: st1b { z0.h }, p0, [x1]
+; CHECK-NEXT: ret
+  %trunc = trunc <vscale x 8 x i16> %val to <vscale x 8 x i8>
+  call void @llvm.masked.store.nxv8i8(<vscale x 8 x i8> %trunc, <vscale x 8 x i8> *%b, i32 2, <vscale x 8 x i1> %mask)
+  ret void
+}
+
+declare void @llvm.masked.store.nxv2i8(<vscale x 2 x i8>, <vscale x 2 x i8>*, i32, <vscale x 2 x i1>)
+declare void @llvm.masked.store.nxv2i16(<vscale x 2 x i16>, <vscale x 2 x i16>*, i32, <vscale x 2 x i1>)
+declare void @llvm.masked.store.nxv2i32(<vscale x 2 x i32>, <vscale x 2 x i32>*, i32, <vscale x 2 x i1>)
+declare void @llvm.masked.store.nxv4i8(<vscale x 4 x i8>, <vscale x 4 x i8>*, i32, <vscale x 4 x i1>)
+declare void @llvm.masked.store.nxv4i16(<vscale x 4 x i16>, <vscale x 4 x i16>*, i32, <vscale x 4 x i1>)
+declare void @llvm.masked.store.nxv8i8(<vscale x 8 x i8>, <vscale x 8 x i8>*, i32, <vscale x 8 x i1>)
diff --git a/llvm/test/CodeGen/AArch64/sve-masked-ldst-zext.ll b/llvm/test/CodeGen/AArch64/sve-masked-ldst-zext.ll
new file mode 100644
index 000000000..1747616d6
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-masked-ldst-zext.ll
@@ -0,0 +1,72 @@
+; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve < %s | FileCheck %s
+
+;
+; Masked Loads
+;
+
+define <vscale x 2 x i64> @masked_zload_nxv2i8(<vscale x 2 x i8>* %src, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: masked_zload_nxv2i8:
+; CHECK-NOT: ld1sb
+; CHECK: ld1b { [[IN:z[0-9]+]].d }, [[PG:p[0-9]+]]/z, [x0]
+; CHECK-NEXT: ret
+  %load = call <vscale x 2 x i8> @llvm.masked.load.nxv2i8(<vscale x 2 x i8>* %src, i32 1, <vscale x 2 x i1> %mask, <vscale x 2 x i8> undef)
+  %ext = zext <vscale x 2 x i8> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %ext
+}
+
+define <vscale x 2 x i64> @masked_zload_nxv2i16(<vscale x 2 x i16>* %src, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: masked_zload_nxv2i16:
+; CHECK-NOT: ld1sh
+; CHECK: ld1h { [[IN:z[0-9]+]].d }, [[PG:p[0-9]+]]/z, [x0]
+; CHECK-NEXT: ret
+  %load = call <vscale x 2 x i16> @llvm.masked.load.nxv2i16(<vscale x 2 x i16>* %src, i32 1, <vscale x 2 x i1> %mask, <vscale x 2 x i16> undef)
+  %ext = zext <vscale x 2 x i16> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %ext
+}
+
+define <vscale x 2 x i64> @masked_zload_nxv2i32(<vscale x 2 x i32>* %src, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: masked_zload_nxv2i32:
+; CHECK-NOT: ld1sw
+; CHECK: ld1w { [[IN:z[0-9]+]].d }, [[PG:p[0-9]+]]/z, [x0]
+; CHECK-NEXT: ret
+  %load = call <vscale x 2 x i32> @llvm.masked.load.nxv2i32(<vscale x 2 x i32>* %src, i32 1, <vscale x 2 x i1> %mask, <vscale x 2 x i32> undef)
+  %ext = zext <vscale x 2 x i32> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %ext
+}
+
+define <vscale x 4 x i32> @masked_zload_nxv4i8(<vscale x 4 x i8>* %src, <vscale x 4 x i1> %mask) {
+; CHECK-LABEL: masked_zload_nxv4i8:
+; CHECK-NOT: ld1sb
+; CHECK: ld1b { [[IN:z[0-9]+]].s }, [[PG:p[0-9]+]]/z, [x0]
+; CHECK-NEXT: ret
+  %load = call <vscale x 4 x i8> @llvm.masked.load.nxv4i8(<vscale x 4 x i8>* %src, i32 1, <vscale x 4 x i1> %mask, <vscale x 4 x i8> undef)
+  %ext = zext <vscale x 4 x i8> %load to <vscale x 4 x i32>
+  ret <vscale x 4 x i32> %ext
+}
+
+define <vscale x 4 x i32> @masked_zload_nxv4i16(<vscale x 4 x i16>* %src, <vscale x 4 x i1> %mask) {
+; CHECK-LABEL: masked_zload_nxv4i16:
+; CHECK-NOT: ld1sh
+; CHECK: ld1h { [[IN:z[0-9]+]].s }, [[PG:p[0-9]+]]/z, [x0]
+; CHECK-NEXT: ret
+  %load = call <vscale x 4 x i16> @llvm.masked.load.nxv4i16(<vscale x 4 x i16>* %src, i32 1, <vscale x 4 x i1> %mask, <vscale x 4 x i16> undef)
+  %ext = zext <vscale x 4 x i16> %load to <vscale x 4 x i32>
+  ret <vscale x 4 x i32> %ext
+}
+
+define <vscale x 8 x i16> @masked_zload_nxv8i8(<vscale x 8 x i8>* %src, <vscale x 8 x i1> %mask) {
+; CHECK-LABEL: masked_zload_nxv8i8:
+; CHECK-NOT: ld1sb
+; CHECK: ld1b { [[IN:z[0-9]+]].h }, [[PG:p[0-9]+]]/z, [x0]
+; CHECK-NEXT: ret
+  %load = call <vscale x 8 x i8> @llvm.masked.load.nxv8i8(<vscale x 8 x i8>* %src, i32 1, <vscale x 8 x i1> %mask, <vscale x 8 x i8> undef)
+  %ext = zext <vscale x 8 x i8> %load to <vscale x 8 x i16>
+  ret <vscale x 8 x i16> %ext
+}
+
+declare <vscale x 2 x i8> @llvm.masked.load.nxv2i8(<vscale x 2 x i8>*, i32, <vscale x 2 x i1>, <vscale x 2 x i8>)
+declare <vscale x 2 x i16> @llvm.masked.load.nxv2i16(<vscale x 2 x i16>*, i32, <vscale x 2 x i1>, <vscale x 2 x i16>)
+declare <vscale x 2 x i32> @llvm.masked.load.nxv2i32(<vscale x 2 x i32>*, i32, <vscale x 2 x i1>, <vscale x 2 x i32>)
+declare <vscale x 4 x i8> @llvm.masked.load.nxv4i8(<vscale x 4 x i8>*, i32, <vscale x 4 x i1>, <vscale x 4 x i8>)
+declare <vscale x 4 x i16> @llvm.masked.load.nxv4i16(<vscale x 4 x i16>*, i32, <vscale x 4 x i1>, <vscale x 4 x i16>)
+declare <vscale x 8 x i8> @llvm.masked.load.nxv8i8(<vscale x 8 x i8>*, i32, <vscale x 8 x i1>, <vscale x 8 x i8>)
diff --git a/llvm/test/CodeGen/AArch64/sve-neg-int-arith-imm-2.ll b/llvm/test/CodeGen/AArch64/sve-neg-int-arith-imm-2.ll
new file mode 100644
index 000000000..56d65425d
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-neg-int-arith-imm-2.ll
@@ -0,0 +1,12 @@
+; RUN: not llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s
+
+; Numbers smaller than -127 and greater than or equal to 127 are not allowed.
+; This should get lowered to a regular vector multiply and these tests should
+; be updated when those patterns are added.
+
+define <vscale x 2 x i64> @mul_i64_neg_1(<vscale x 2 x i64> %a) {
+  %elt = insertelement <vscale x 2 x i64> undef, i64 255, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %res = mul <vscale x 2 x i64> %a, %splat
+  ret <vscale x 2 x i64> %res
+}
diff --git a/llvm/test/CodeGen/AArch64/sve-neg-int-arith-imm.ll b/llvm/test/CodeGen/AArch64/sve-neg-int-arith-imm.ll
new file mode 100644
index 000000000..992b15815
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-neg-int-arith-imm.ll
@@ -0,0 +1,11 @@
+; RUN: not llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s
+
+; Numbers smaller than -127 and greater than or equal to 127 allowed for imm mul.
+; This should get lowered to a regular vector multiply and these tests should
+; be updated when those patterns are added.
+define <vscale x 2 x i64> @mul_i64_neg_1(<vscale x 2 x i64> %a) {
+  %elt = insertelement <vscale x 2 x i64> undef, i64 -130, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %res = mul <vscale x 2 x i64> %a, %splat
+  ret <vscale x 2 x i64> %res
+}
diff --git a/llvm/test/CodeGen/AArch64/sve-pred-log.ll b/llvm/test/CodeGen/AArch64/sve-pred-log.ll
new file mode 100644
index 000000000..7116ccf8a
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-pred-log.ll
@@ -0,0 +1,286 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+define <vscale x 16 x i1> @vselect_16(<vscale x 16 x i1> %Pg, <vscale x 16 x i1> %Pn, <vscale x 16 x i1> %Pd) {
+; CHECK-LABEL: vselect_16:
+; CHECK: sel p0.b, p0, p1.b, p2.b
+; CHECK-NEXT: ret
+  %res = select <vscale x 16 x i1> %Pg, <vscale x 16 x i1> %Pn, <vscale x 16 x i1> %Pd
+  ret <vscale x 16 x i1> %res;
+}
+
+define <vscale x 8 x i1> @vselect_8(<vscale x 8 x i1> %Pg, <vscale x 8 x i1> %Pn, <vscale x 8 x i1> %Pd) {
+; CHECK-LABEL: vselect_8:
+; CHECK: sel p0.b, p0, p1.b, p2.b
+; CHECK-NEXT: ret
+  %res = select <vscale x 8 x i1> %Pg, <vscale x 8 x i1> %Pn, <vscale x 8 x i1> %Pd
+  ret <vscale x 8 x i1> %res;
+}
+
+define <vscale x 4 x i1> @vselect_4(<vscale x 4 x i1> %Pg, <vscale x 4 x i1> %Pn, <vscale x 4 x i1> %Pd) {
+; CHECK-LABEL: vselect_4:
+; CHECK: sel p0.b, p0, p1.b, p2.b
+; CHECK-NEXT: ret
+  %res = select <vscale x 4 x i1> %Pg, <vscale x 4 x i1> %Pn, <vscale x 4 x i1> %Pd
+  ret <vscale x 4 x i1> %res;
+}
+
+define <vscale x 2 x i1> @vselect_2(<vscale x 2 x i1> %Pg, <vscale x 2 x i1> %Pn, <vscale x 2 x i1> %Pd) {
+; CHECK-LABEL: vselect_2:
+; CHECK: sel p0.b, p0, p1.b, p2.b
+; CHECK-NEXT: ret
+  %res = select <vscale x 2 x i1> %Pg, <vscale x 2 x i1> %Pn, <vscale x 2 x i1> %Pd
+  ret <vscale x 2 x i1> %res;
+}
+
+define <vscale x 16 x i1> @and_16(<vscale x 16 x i1> %Pg, <vscale x 16 x i1> %Pn, <vscale x 16 x i1> %Pd) {
+; CHECK-LABEL: and_16:
+; CHECK: and p0.b, p0/z, p1.b, p2.b
+; CHECK-NEXT: ret
+  %res = call <vscale x 16 x i1> @llvm.aarch64.sve.and.z.nxv16i1(<vscale x 16 x i1> %Pg, <vscale x 16 x i1> %Pn, <vscale x 16 x i1> %Pd)
+  ret <vscale x 16 x i1> %res;
+}
+
+define <vscale x 8 x i1> @and_8(<vscale x 8 x i1> %Pg, <vscale x 8 x i1> %Pn, <vscale x 8 x i1> %Pd) {
+; CHECK-LABEL: and_8:
+; CHECK: and p0.b, p0/z, p1.b, p2.b
+; CHECK-NEXT: ret
+  %res = call <vscale x 8 x i1> @llvm.aarch64.sve.and.z.nxv8i1(<vscale x 8 x i1> %Pg, <vscale x 8 x i1> %Pn, <vscale x 8 x i1> %Pd)
+  ret <vscale x 8 x i1> %res;
+}
+
+define <vscale x 4 x i1> @and_4(<vscale x 4 x i1> %Pg, <vscale x 4 x i1> %Pn, <vscale x 4 x i1> %Pd) {
+; CHECK-LABEL: and_4:
+; CHECK: and p0.b, p0/z, p1.b, p2.b
+; CHECK-NEXT: ret
+  %res = call <vscale x 4 x i1> @llvm.aarch64.sve.and.z.nxv4i1(<vscale x 4 x i1> %Pg, <vscale x 4 x i1> %Pn, <vscale x 4 x i1> %Pd)
+  ret <vscale x 4 x i1> %res;
+}
+
+define <vscale x 2 x i1> @and_2(<vscale x 2 x i1> %Pg, <vscale x 2 x i1> %Pn, <vscale x 2 x i1> %Pd) {
+; CHECK-LABEL: and_2:
+; CHECK: and p0.b, p0/z, p1.b, p2.b
+; CHECK-NEXT: ret
+  %res = call <vscale x 2 x i1> @llvm.aarch64.sve.and.z.nxv2i1(<vscale x 2 x i1> %Pg, <vscale x 2 x i1> %Pn, <vscale x 2 x i1> %Pd)
+  ret <vscale x 2 x i1> %res;
+}
+
+define <vscale x 16 x i1> @bic_16(<vscale x 16 x i1> %Pg, <vscale x 16 x i1> %Pn, <vscale x 16 x i1> %Pd) {
+; CHECK-LABEL: bic_16:
+; CHECK: bic p0.b, p0/z, p1.b, p2.b
+; CHECK-NEXT: ret
+  %res = call <vscale x 16 x i1> @llvm.aarch64.sve.bic.z.nxv16i1(<vscale x 16 x i1> %Pg, <vscale x 16 x i1> %Pn, <vscale x 16 x i1> %Pd)
+  ret <vscale x 16 x i1> %res;
+}
+
+define <vscale x 8 x i1> @bic_8(<vscale x 8 x i1> %Pg, <vscale x 8 x i1> %Pn, <vscale x 8 x i1> %Pd) {
+; CHECK-LABEL: bic_8:
+; CHECK: bic p0.b, p0/z, p1.b, p2.b
+; CHECK-NEXT: ret
+  %res = call <vscale x 8 x i1> @llvm.aarch64.sve.bic.z.nxv8i1(<vscale x 8 x i1> %Pg, <vscale x 8 x i1> %Pn, <vscale x 8 x i1> %Pd)
+  ret <vscale x 8 x i1> %res;
+}
+
+define <vscale x 4 x i1> @bic_4(<vscale x 4 x i1> %Pg, <vscale x 4 x i1> %Pn, <vscale x 4 x i1> %Pd) {
+; CHECK-LABEL: bic_4:
+; CHECK: bic p0.b, p0/z, p1.b, p2.b
+; CHECK-NEXT: ret
+  %res = call <vscale x 4 x i1> @llvm.aarch64.sve.bic.z.nxv4i1(<vscale x 4 x i1> %Pg, <vscale x 4 x i1> %Pn, <vscale x 4 x i1> %Pd)
+  ret <vscale x 4 x i1> %res;
+}
+
+define <vscale x 2 x i1> @bic_2(<vscale x 2 x i1> %Pg, <vscale x 2 x i1> %Pn, <vscale x 2 x i1> %Pd) {
+; CHECK-LABEL: bic_2:
+; CHECK: bic p0.b, p0/z, p1.b, p2.b
+; CHECK-NEXT: ret
+  %res = call <vscale x 2 x i1> @llvm.aarch64.sve.bic.z.nxv2i1(<vscale x 2 x i1> %Pg, <vscale x 2 x i1> %Pn, <vscale x 2 x i1> %Pd)
+  ret <vscale x 2 x i1> %res;
+}
+
+define <vscale x 16 x i1> @eor_16(<vscale x 16 x i1> %Pg, <vscale x 16 x i1> %Pn, <vscale x 16 x i1> %Pd) {
+; CHECK-LABEL: eor_16:
+; CHECK: eor p0.b, p0/z, p1.b, p2.b
+; CHECK-NEXT: ret
+  %res = call <vscale x 16 x i1> @llvm.aarch64.sve.eor.z.nxv16i1(<vscale x 16 x i1> %Pg, <vscale x 16 x i1> %Pn, <vscale x 16 x i1> %Pd)
+  ret <vscale x 16 x i1> %res;
+}
+
+define <vscale x 8 x i1> @eor_8(<vscale x 8 x i1> %Pg, <vscale x 8 x i1> %Pn, <vscale x 8 x i1> %Pd) {
+; CHECK-LABEL: eor_8:
+; CHECK: eor p0.b, p0/z, p1.b, p2.b
+; CHECK-NEXT: ret
+  %res = call <vscale x 8 x i1> @llvm.aarch64.sve.eor.z.nxv8i1(<vscale x 8 x i1> %Pg, <vscale x 8 x i1> %Pn, <vscale x 8 x i1> %Pd)
+  ret <vscale x 8 x i1> %res;
+}
+
+define <vscale x 4 x i1> @eor_4(<vscale x 4 x i1> %Pg, <vscale x 4 x i1> %Pn, <vscale x 4 x i1> %Pd) {
+; CHECK-LABEL: eor_4:
+; CHECK: eor p0.b, p0/z, p1.b, p2.b
+; CHECK-NEXT: ret
+  %res = call <vscale x 4 x i1> @llvm.aarch64.sve.eor.z.nxv4i1(<vscale x 4 x i1> %Pg, <vscale x 4 x i1> %Pn, <vscale x 4 x i1> %Pd)
+  ret <vscale x 4 x i1> %res;
+}
+
+define <vscale x 2 x i1> @eor_2(<vscale x 2 x i1> %Pg, <vscale x 2 x i1> %Pn, <vscale x 2 x i1> %Pd) {
+; CHECK-LABEL: eor_2:
+; CHECK: eor p0.b, p0/z, p1.b, p2.b
+; CHECK-NEXT: ret
+  %res = call <vscale x 2 x i1> @llvm.aarch64.sve.eor.z.nxv2i1(<vscale x 2 x i1> %Pg, <vscale x 2 x i1> %Pn, <vscale x 2 x i1> %Pd)
+  ret <vscale x 2 x i1> %res;
+}
+
+define <vscale x 16 x i1> @orr_16(<vscale x 16 x i1> %Pg, <vscale x 16 x i1> %Pn, <vscale x 16 x i1> %Pd) {
+; CHECK-LABEL: orr_16:
+; CHECK: orr p0.b, p0/z, p1.b, p2.b
+; CHECK-NEXT: ret
+  %res = call <vscale x 16 x i1> @llvm.aarch64.sve.orr.z.nxv16i1(<vscale x 16 x i1> %Pg, <vscale x 16 x i1> %Pn, <vscale x 16 x i1> %Pd)
+  ret <vscale x 16 x i1> %res;
+}
+
+define <vscale x 8 x i1> @orr_8(<vscale x 8 x i1> %Pg, <vscale x 8 x i1> %Pn, <vscale x 8 x i1> %Pd) {
+; CHECK-LABEL: orr_8:
+; CHECK: orr p0.b, p0/z, p1.b, p2.b
+; CHECK-NEXT: ret
+  %res = call <vscale x 8 x i1> @llvm.aarch64.sve.orr.z.nxv8i1(<vscale x 8 x i1> %Pg, <vscale x 8 x i1> %Pn, <vscale x 8 x i1> %Pd)
+  ret <vscale x 8 x i1> %res;
+}
+
+define <vscale x 4 x i1> @orr_4(<vscale x 4 x i1> %Pg, <vscale x 4 x i1> %Pn, <vscale x 4 x i1> %Pd) {
+; CHECK-LABEL: orr_4:
+; CHECK: orr p0.b, p0/z, p1.b, p2.b
+; CHECK-NEXT: ret
+  %res = call <vscale x 4 x i1> @llvm.aarch64.sve.orr.z.nxv4i1(<vscale x 4 x i1> %Pg, <vscale x 4 x i1> %Pn, <vscale x 4 x i1> %Pd)
+  ret <vscale x 4 x i1> %res;
+}
+
+define <vscale x 2 x i1> @orr_2(<vscale x 2 x i1> %Pg, <vscale x 2 x i1> %Pn, <vscale x 2 x i1> %Pd) {
+; CHECK-LABEL: orr_2:
+; CHECK: orr p0.b, p0/z, p1.b, p2.b
+; CHECK-NEXT: ret
+  %res = call <vscale x 2 x i1> @llvm.aarch64.sve.orr.z.nxv2i1(<vscale x 2 x i1> %Pg, <vscale x 2 x i1> %Pn, <vscale x 2 x i1> %Pd)
+  ret <vscale x 2 x i1> %res;
+}
+
+define <vscale x 16 x i1> @orn_16(<vscale x 16 x i1> %Pg, <vscale x 16 x i1> %Pn, <vscale x 16 x i1> %Pd) {
+; CHECK-LABEL: orn_16:
+; CHECK: orn p0.b, p0/z, p1.b, p2.b
+; CHECK-NEXT: ret
+  %res = call <vscale x 16 x i1> @llvm.aarch64.sve.orn.z.nxv16i1(<vscale x 16 x i1> %Pg, <vscale x 16 x i1> %Pn, <vscale x 16 x i1> %Pd)
+  ret <vscale x 16 x i1> %res;
+}
+
+define <vscale x 8 x i1> @orn_8(<vscale x 8 x i1> %Pg, <vscale x 8 x i1> %Pn, <vscale x 8 x i1> %Pd) {
+; CHECK-LABEL: orn_8:
+; CHECK: orn p0.b, p0/z, p1.b, p2.b
+; CHECK-NEXT: ret
+  %res = call <vscale x 8 x i1> @llvm.aarch64.sve.orn.z.nxv8i1(<vscale x 8 x i1> %Pg, <vscale x 8 x i1> %Pn, <vscale x 8 x i1> %Pd)
+  ret <vscale x 8 x i1> %res;
+}
+
+define <vscale x 4 x i1> @orn_4(<vscale x 4 x i1> %Pg, <vscale x 4 x i1> %Pn, <vscale x 4 x i1> %Pd) {
+; CHECK-LABEL: orn_4:
+; CHECK: orn p0.b, p0/z, p1.b, p2.b
+; CHECK-NEXT: ret
+  %res = call <vscale x 4 x i1> @llvm.aarch64.sve.orn.z.nxv4i1(<vscale x 4 x i1> %Pg, <vscale x 4 x i1> %Pn, <vscale x 4 x i1> %Pd)
+  ret <vscale x 4 x i1> %res;
+}
+
+define <vscale x 2 x i1> @orn_2(<vscale x 2 x i1> %Pg, <vscale x 2 x i1> %Pn, <vscale x 2 x i1> %Pd) {
+; CHECK-LABEL: orn_2:
+; CHECK: orn p0.b, p0/z, p1.b, p2.b
+; CHECK-NEXT: ret
+  %res = call <vscale x 2 x i1> @llvm.aarch64.sve.orn.z.nxv2i1(<vscale x 2 x i1> %Pg, <vscale x 2 x i1> %Pn, <vscale x 2 x i1> %Pd)
+  ret <vscale x 2 x i1> %res;
+}
+
+define <vscale x 16 x i1> @nor_16(<vscale x 16 x i1> %Pg, <vscale x 16 x i1> %Pn, <vscale x 16 x i1> %Pd) {
+; CHECK-LABEL: nor_16:
+; CHECK: nor p0.b, p0/z, p1.b, p2.b
+; CHECK-NEXT: ret
+  %res = call <vscale x 16 x i1> @llvm.aarch64.sve.nor.z.nxv16i1(<vscale x 16 x i1> %Pg, <vscale x 16 x i1> %Pn, <vscale x 16 x i1> %Pd)
+  ret <vscale x 16 x i1> %res;
+}
+
+define <vscale x 8 x i1> @nor_8(<vscale x 8 x i1> %Pg, <vscale x 8 x i1> %Pn, <vscale x 8 x i1> %Pd) {
+; CHECK-LABEL: nor_8:
+; CHECK: nor p0.b, p0/z, p1.b, p2.b
+; CHECK-NEXT: ret
+  %res = call <vscale x 8 x i1> @llvm.aarch64.sve.nor.z.nxv8i1(<vscale x 8 x i1> %Pg, <vscale x 8 x i1> %Pn, <vscale x 8 x i1> %Pd)
+  ret <vscale x 8 x i1> %res;
+}
+
+define <vscale x 4 x i1> @nor_4(<vscale x 4 x i1> %Pg, <vscale x 4 x i1> %Pn, <vscale x 4 x i1> %Pd) {
+; CHECK-LABEL: nor_4:
+; CHECK: nor p0.b, p0/z, p1.b, p2.b
+; CHECK-NEXT: ret
+  %res = call <vscale x 4 x i1> @llvm.aarch64.sve.nor.z.nxv4i1(<vscale x 4 x i1> %Pg, <vscale x 4 x i1> %Pn, <vscale x 4 x i1> %Pd)
+  ret <vscale x 4 x i1> %res;
+}
+
+define <vscale x 2 x i1> @nor_2(<vscale x 2 x i1> %Pg, <vscale x 2 x i1> %Pn, <vscale x 2 x i1> %Pd) {
+; CHECK-LABEL: nor_2:
+; CHECK: nor p0.b, p0/z, p1.b, p2.b
+; CHECK-NEXT: ret
+  %res = call <vscale x 2 x i1> @llvm.aarch64.sve.nor.z.nxv2i1(<vscale x 2 x i1> %Pg, <vscale x 2 x i1> %Pn, <vscale x 2 x i1> %Pd)
+  ret <vscale x 2 x i1> %res;
+}
+
+define <vscale x 16 x i1> @nand_16(<vscale x 16 x i1> %Pg, <vscale x 16 x i1> %Pn, <vscale x 16 x i1> %Pd) {
+; CHECK-LABEL: nand_16:
+; CHECK: nand p0.b, p0/z, p1.b, p2.b
+; CHECK-NEXT: ret
+  %res = call <vscale x 16 x i1> @llvm.aarch64.sve.nand.z.nxv16i1(<vscale x 16 x i1> %Pg, <vscale x 16 x i1> %Pn,  <vscale x 16 x i1> %Pd)
+  ret <vscale x 16 x i1> %res;
+}
+
+define <vscale x 8 x i1> @nand_8(<vscale x 8 x i1> %Pg, <vscale x 8 x i1> %Pn, <vscale x 8 x i1> %Pd) {
+; CHECK-LABEL: nand_8:
+; CHECK: nand p0.b, p0/z, p1.b, p2.b
+; CHECK-NEXT: ret
+  %res = call <vscale x 8 x i1> @llvm.aarch64.sve.nand.z.nxv8i1(<vscale x 8 x i1> %Pg, <vscale x 8 x i1> %Pn, <vscale x 8 x i1> %Pd)
+  ret <vscale x 8 x i1> %res;
+}
+
+define <vscale x 4 x i1> @nand_4(<vscale x 4 x i1> %Pg, <vscale x 4 x i1> %Pn, <vscale x 4 x i1> %Pd) {
+; CHECK-LABEL: nand_4:
+; CHECK: nand p0.b, p0/z, p1.b, p2.b
+; CHECK-NEXT: ret
+  %res = call <vscale x 4 x i1> @llvm.aarch64.sve.nand.z.nxv4i1(<vscale x 4 x i1> %Pg, <vscale x 4 x i1> %Pn, <vscale x 4 x i1> %Pd)
+  ret <vscale x 4 x i1> %res;
+}
+
+define <vscale x 2 x i1> @nand_2(<vscale x 2 x i1> %Pg, <vscale x 2 x i1> %Pn, <vscale x 2 x i1> %Pd) {
+; CHECK-LABEL: nand_2:
+; CHECK: nand p0.b, p0/z, p1.b, p2.b
+; CHECK-NEXT: ret
+  %res = call <vscale x 2 x i1> @llvm.aarch64.sve.nand.z.nxv2i1(<vscale x 2 x i1> %Pg, <vscale x 2 x i1> %Pn, <vscale x 2 x i1> %Pd)
+  ret <vscale x 2 x i1> %res;
+}
+
+declare <vscale x 16 x i1> @llvm.aarch64.sve.and.z.nxv16i1(<vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.and.z.nxv8i1(<vscale x 8 x i1>, <vscale x 8 x i1>, <vscale x 8 x i1>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.and.z.nxv4i1(<vscale x 4 x i1>, <vscale x 4 x i1>, <vscale x 4 x i1>)
+declare <vscale x 2 x i1> @llvm.aarch64.sve.and.z.nxv2i1(<vscale x 2 x i1>, <vscale x 2 x i1>, <vscale x 2 x i1>)
+declare <vscale x 16 x i1> @llvm.aarch64.sve.bic.z.nxv16i1(<vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.bic.z.nxv8i1(<vscale x 8 x i1>, <vscale x 8 x i1>, <vscale x 8 x i1>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.bic.z.nxv4i1(<vscale x 4 x i1>, <vscale x 4 x i1>, <vscale x 4 x i1>)
+declare <vscale x 2 x i1> @llvm.aarch64.sve.bic.z.nxv2i1(<vscale x 2 x i1>, <vscale x 2 x i1>, <vscale x 2 x i1>)
+declare <vscale x 16 x i1> @llvm.aarch64.sve.eor.z.nxv16i1(<vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.eor.z.nxv8i1(<vscale x 8 x i1>, <vscale x 8 x i1>, <vscale x 8 x i1>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.eor.z.nxv4i1(<vscale x 4 x i1>, <vscale x 4 x i1>, <vscale x 4 x i1>)
+declare <vscale x 2 x i1> @llvm.aarch64.sve.eor.z.nxv2i1(<vscale x 2 x i1>, <vscale x 2 x i1>, <vscale x 2 x i1>)
+declare <vscale x 16 x i1> @llvm.aarch64.sve.orr.z.nxv16i1(<vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.orr.z.nxv8i1(<vscale x 8 x i1>, <vscale x 8 x i1>, <vscale x 8 x i1>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.orr.z.nxv4i1(<vscale x 4 x i1>, <vscale x 4 x i1>, <vscale x 4 x i1>)
+declare <vscale x 2 x i1> @llvm.aarch64.sve.orr.z.nxv2i1(<vscale x 2 x i1>, <vscale x 2 x i1>, <vscale x 2 x i1>)
+declare <vscale x 16 x i1> @llvm.aarch64.sve.orn.z.nxv16i1(<vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.orn.z.nxv8i1(<vscale x 8 x i1>, <vscale x 8 x i1>, <vscale x 8 x i1>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.orn.z.nxv4i1(<vscale x 4 x i1>, <vscale x 4 x i1>, <vscale x 4 x i1>)
+declare <vscale x 2 x i1> @llvm.aarch64.sve.orn.z.nxv2i1(<vscale x 2 x i1>, <vscale x 2 x i1>, <vscale x 2 x i1>)
+declare <vscale x 16 x i1> @llvm.aarch64.sve.nor.z.nxv16i1(<vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.nor.z.nxv8i1(<vscale x 8 x i1>, <vscale x 8 x i1>, <vscale x 8 x i1>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.nor.z.nxv4i1(<vscale x 4 x i1>, <vscale x 4 x i1>, <vscale x 4 x i1>)
+declare <vscale x 2 x i1> @llvm.aarch64.sve.nor.z.nxv2i1(<vscale x 2 x i1>, <vscale x 2 x i1>, <vscale x 2 x i1>)
+declare <vscale x 16 x i1> @llvm.aarch64.sve.nand.z.nxv16i1(<vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.nand.z.nxv8i1(<vscale x 8 x i1>, <vscale x 8 x i1>, <vscale x 8 x i1>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.nand.z.nxv4i1(<vscale x 4 x i1>, <vscale x 4 x i1>, <vscale x 4 x i1>)
+declare <vscale x 2 x i1> @llvm.aarch64.sve.nand.z.nxv2i1(<vscale x 2 x i1>, <vscale x 2 x i1>, <vscale x 2 x i1>)
diff --git a/llvm/test/CodeGen/AArch64/sve-select.ll b/llvm/test/CodeGen/AArch64/sve-select.ll
new file mode 100644
index 000000000..2d2ea47ae
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-select.ll
@@ -0,0 +1,85 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+; Integer vector select
+
+define <vscale x 16 x i8> @sel_nxv16i8(<vscale x 16 x i1> %p,
+                                       <vscale x 16 x i8> %dst,
+                                       <vscale x 16 x i8> %a) {
+; CHECK-LABEL: sel_nxv16i8:
+; CHECK:         mov z0.b, p0/m, z1.b
+; CHECK-NEXT:    ret
+  %sel = select <vscale x 16 x i1> %p, <vscale x 16 x i8> %a, <vscale x 16 x i8> %dst
+  ret <vscale x 16 x i8> %sel
+}
+
+define <vscale x 8 x i16> @sel_nxv8i16(<vscale x 8 x i1> %p,
+                                       <vscale x 8 x i16> %dst,
+                                       <vscale x 8 x i16> %a) {
+; CHECK-LABEL: sel_nxv8i16:
+; CHECK:         mov z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %sel = select <vscale x 8 x i1> %p, <vscale x 8 x i16> %a, <vscale x 8 x i16> %dst
+  ret <vscale x 8 x i16> %sel
+}
+
+define <vscale x 4 x i32> @sel_nxv4i32(<vscale x 4 x i1> %p,
+                                       <vscale x 4 x i32> %dst,
+                                       <vscale x 4 x i32> %a) {
+; CHECK-LABEL: sel_nxv4i32:
+; CHECK:         mov z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %sel = select <vscale x 4 x i1> %p, <vscale x 4 x i32> %a, <vscale x 4 x i32> %dst
+  ret <vscale x 4 x i32> %sel
+}
+
+define <vscale x 2 x i64> @sel_nxv2i64(<vscale x 2 x i1> %p,
+                                       <vscale x 2 x i64> %dst,
+                                       <vscale x 2 x i64> %a) {
+; CHECK-LABEL: sel_nxv2i64:
+; CHECK:         mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %sel = select <vscale x 2 x i1> %p, <vscale x 2 x i64> %a, <vscale x 2 x i64> %dst
+  ret <vscale x 2 x i64> %sel
+}
+
+; Floating point vector select
+
+define <vscale x 8 x half> @sel_nxv8f16(<vscale x 8 x i1> %p,
+                                        <vscale x 8 x half> %dst,
+                                        <vscale x 8 x half> %a) {
+; CHECK-LABEL: sel_nxv8f16:
+; CHECK:         mov z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %sel = select <vscale x 8 x i1> %p, <vscale x 8 x half> %a, <vscale x 8 x half> %dst
+  ret <vscale x 8 x half> %sel
+}
+
+define <vscale x 4 x float> @sel_nxv4f32(<vscale x 4 x i1> %p,
+                                         <vscale x 4 x float> %dst,
+                                         <vscale x 4 x float> %a) {
+; CHECK-LABEL: sel_nxv4f32:
+; CHECK:         mov z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %sel = select <vscale x 4 x i1> %p, <vscale x 4 x float> %a, <vscale x 4 x float> %dst
+  ret <vscale x 4 x float> %sel
+}
+
+define <vscale x 2 x float> @sel_nxv2f32(<vscale x 2 x i1> %p,
+                                         <vscale x 2 x float> %dst,
+                                         <vscale x 2 x float> %a) {
+; CHECK-LABEL: sel_nxv2f32:
+; CHECK:         mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %sel = select <vscale x 2 x i1> %p, <vscale x 2 x float> %a, <vscale x 2 x float> %dst
+  ret <vscale x 2 x float> %sel
+}
+
+define <vscale x 2 x double> @sel_nxv8f64(<vscale x 2 x i1> %p,
+                                          <vscale x 2 x double> %dst,
+                                          <vscale x 2 x double> %a) {
+; CHECK-LABEL: sel_nxv8f64:
+; CHECK:         mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %sel = select <vscale x 2 x i1> %p, <vscale x 2 x double> %a, <vscale x 2 x double> %dst
+  ret <vscale x 2 x double> %sel
+}
diff --git a/llvm/test/CodeGen/AArch64/sve-setcc.ll b/llvm/test/CodeGen/AArch64/sve-setcc.ll
new file mode 100644
index 000000000..7c44501d1
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-setcc.ll
@@ -0,0 +1,26 @@
+; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve < %s | FileCheck %s
+
+; Ensure we use the inverted CC result of SVE compare instructions when branching.
+define void @sve_cmplt_setcc_inverted(<vscale x 8 x i16>* %out, <vscale x 8 x i16> %in, <vscale x 8 x i1> %pg) {
+; CHECK-LABEL: @sve_cmplt_setcc_inverted
+; CHECK: cmplt p1.h, p0/z, z0.h, #0
+; CHECK-NEXT: ptest p0, p1.b
+; CHECK-NEXT: b.ne
+entry:
+  %0 = tail call <vscale x 8 x i1> @llvm.aarch64.sve.cmplt.wide.nxv8i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %in, <vscale x 2 x i64> zeroinitializer)
+  %1 = tail call i1 @llvm.aarch64.sve.ptest.any.nxv8i1(<vscale x 8 x i1> %pg, <vscale x 8 x i1> %0)
+  br i1 %1, label %if.end, label %if.then
+
+if.then:
+  tail call void @llvm.masked.store.nxv8i16.p0nxv8i16(<vscale x 8 x i16> %in, <vscale x 8 x i16>* %out, i32 2, <vscale x 8 x i1> %pg)
+  br label %if.end
+
+if.end:
+  ret void
+}
+
+declare i1 @llvm.aarch64.sve.ptest.any.nxv8i1(<vscale x 8 x i1>, <vscale x 8 x i1>)
+
+declare <vscale x 8 x i1> @llvm.aarch64.sve.cmplt.wide.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 2 x i64>)
+
+declare void @llvm.masked.store.nxv8i16.p0nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i16>*, i32, <vscale x 8 x i1>)
diff --git a/llvm/test/CodeGen/AArch64/sve-vector-splat.ll b/llvm/test/CodeGen/AArch64/sve-vector-splat.ll
new file mode 100644
index 000000000..086241c4e
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-vector-splat.ll
@@ -0,0 +1,135 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+;; Splats of legal integer vector types
+
+define <vscale x 16 x i8> @sve_splat_16xi8(i8 %val) {
+; CHECK-LABEL: @sve_splat_16xi8
+; CHECK: mov z0.b, w0
+; CHECK-NEXT: ret
+  %ins = insertelement <vscale x 16 x i8> undef, i8 %val, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %ins, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  ret <vscale x 16 x i8> %splat
+}
+
+define <vscale x 8 x i16> @sve_splat_8xi16(i16 %val) {
+; CHECK-LABEL: @sve_splat_8xi16
+; CHECK: mov z0.h, w0
+; CHECK-NEXT: ret
+  %ins = insertelement <vscale x 8 x i16> undef, i16 %val, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %ins, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  ret <vscale x 8 x i16> %splat
+}
+
+define <vscale x 4 x i32> @sve_splat_4xi32(i32 %val) {
+; CHECK-LABEL: @sve_splat_4xi32
+; CHECK: mov z0.s, w0
+; CHECK-NEXT: ret
+  %ins = insertelement <vscale x 4 x i32> undef, i32 %val, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %ins, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  ret <vscale x 4 x i32> %splat
+}
+
+define <vscale x 2 x i64> @sve_splat_2xi64(i64 %val) {
+; CHECK-LABEL: @sve_splat_2xi64
+; CHECK: mov z0.d, x0
+; CHECK-NEXT: ret
+  %ins = insertelement <vscale x 2 x i64> undef, i64 %val, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %ins, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  ret <vscale x 2 x i64> %splat
+}
+
+;; Promote splats of smaller illegal integer vector types
+
+define <vscale x 2 x i8> @sve_splat_2xi8(i8 %val) {
+; CHECK-LABEL: @sve_splat_2xi8
+; CHECK: mov z0.d, x0
+; CHECK-NEXT: ret
+  %ins = insertelement <vscale x 2 x i8> undef, i8 %val, i32 0
+  %splat = shufflevector <vscale x 2 x i8> %ins, <vscale x 2 x i8> undef, <vscale x 2 x i32> zeroinitializer
+  ret <vscale x 2 x i8> %splat
+}
+
+define <vscale x 4 x i8> @sve_splat_4xi8(i8 %val) {
+; CHECK-LABEL: @sve_splat_4xi8
+; CHECK: mov z0.s, w0
+; CHECK-NEXT: ret
+  %ins = insertelement <vscale x 4 x i8> undef, i8 %val, i32 0
+  %splat = shufflevector <vscale x 4 x i8> %ins, <vscale x 4 x i8> undef, <vscale x 4 x i32> zeroinitializer
+  ret <vscale x 4 x i8> %splat
+}
+
+define <vscale x 8 x i8> @sve_splat_8xi8(i8 %val) {
+; CHECK-LABEL: @sve_splat_8xi8
+; CHECK: mov z0.h, w0
+; CHECK-NEXT: ret
+  %ins = insertelement <vscale x 8 x i8> undef, i8 %val, i32 0
+  %splat = shufflevector <vscale x 8 x i8> %ins, <vscale x 8 x i8> undef, <vscale x 8 x i32> zeroinitializer
+  ret <vscale x 8 x i8> %splat
+}
+
+define <vscale x 2 x i16> @sve_splat_2xi16(i16 %val) {
+; CHECK-LABEL: @sve_splat_2xi16
+; CHECK: mov z0.d, x0
+; CHECK-NEXT: ret
+  %ins = insertelement <vscale x 2 x i16> undef, i16 %val, i32 0
+  %splat = shufflevector <vscale x 2 x i16> %ins, <vscale x 2 x i16> undef, <vscale x 2 x i32> zeroinitializer
+  ret <vscale x 2 x i16> %splat
+}
+
+define <vscale x 4 x i16> @sve_splat_4xi16(i16 %val) {
+; CHECK-LABEL: @sve_splat_4xi16
+; CHECK: mov z0.s, w0
+; CHECK-NEXT: ret
+  %ins = insertelement <vscale x 4 x i16> undef, i16 %val, i32 0
+  %splat = shufflevector <vscale x 4 x i16> %ins, <vscale x 4 x i16> undef, <vscale x 4 x i32> zeroinitializer
+  ret <vscale x 4 x i16> %splat
+}
+
+define <vscale x 2 x i32> @sve_splat_2xi32(i32 %val) {
+; CHECK-LABEL: @sve_splat_2xi32
+; CHECK: mov z0.d, x0
+; CHECK-NEXT: ret
+  %ins = insertelement <vscale x 2 x i32> undef, i32 %val, i32 0
+  %splat = shufflevector <vscale x 2 x i32> %ins, <vscale x 2 x i32> undef, <vscale x 2 x i32> zeroinitializer
+  ret <vscale x 2 x i32> %splat
+}
+
+define <vscale x 2 x i1> @sve_splat_2xi1(i1 %val) {
+; CHECK-LABEL: @sve_splat_2xi1
+; CHECK: sbfx x8, x0, #0, #1
+; CHECK-NEXT: whilelo p0.d, xzr, x8
+; CHECK-NEXT: ret
+  %ins = insertelement <vscale x 2 x i1> undef, i1 %val, i32 0
+  %splat = shufflevector <vscale x 2 x i1> %ins, <vscale x 2 x i1> undef, <vscale x 2 x i32> zeroinitializer
+  ret <vscale x 2 x i1> %splat
+}
+
+define <vscale x 4 x i1> @sve_splat_4xi1(i1 %val) {
+; CHECK-LABEL: @sve_splat_4xi1
+; CHECK: sbfx x8, x0, #0, #1
+; CHECK-NEXT: whilelo p0.s, xzr, x8
+; CHECK-NEXT: ret
+  %ins = insertelement <vscale x 4 x i1> undef, i1 %val, i32 0
+  %splat = shufflevector <vscale x 4 x i1> %ins, <vscale x 4 x i1> undef, <vscale x 4 x i32> zeroinitializer
+  ret <vscale x 4 x i1> %splat
+}
+
+define <vscale x 8 x i1> @sve_splat_8xi1(i1 %val) {
+; CHECK-LABEL: @sve_splat_8xi1
+; CHECK: sbfx x8, x0, #0, #1
+; CHECK-NEXT: whilelo p0.h, xzr, x8
+; CHECK-NEXT: ret
+  %ins = insertelement <vscale x 8 x i1> undef, i1 %val, i32 0
+  %splat = shufflevector <vscale x 8 x i1> %ins, <vscale x 8 x i1> undef, <vscale x 8 x i32> zeroinitializer
+  ret <vscale x 8 x i1> %splat
+}
+
+define <vscale x 16 x i1> @sve_splat_16xi1(i1 %val) {
+; CHECK-LABEL: @sve_splat_16xi1
+; CHECK: sbfx x8, x0, #0, #1
+; CHECK-NEXT: whilelo p0.b, xzr, x8
+; CHECK-NEXT: ret
+  %ins = insertelement <vscale x 16 x i1> undef, i1 %val, i32 0
+  %splat = shufflevector <vscale x 16 x i1> %ins, <vscale x 16 x i1> undef, <vscale x 16 x i32> zeroinitializer
+  ret <vscale x 16 x i1> %splat
+}
diff --git a/llvm/test/CodeGen/AArch64/sve2-intrinsics-binary-narrowing-add-sub.ll b/llvm/test/CodeGen/AArch64/sve2-intrinsics-binary-narrowing-add-sub.ll
new file mode 100644
index 000000000..d6b34fc87
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve2-intrinsics-binary-narrowing-add-sub.ll
@@ -0,0 +1,278 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2 < %s | FileCheck %s
+
+; ADDHNB
+
+define <vscale x 16 x i8> @addhnb_h(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: addhnb_h:
+; CHECK: addhnb z0.b, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.addhnb.nxv8i16(<vscale x 8 x i16> %a,
+                                                                  <vscale x 8 x i16> %b)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @addhnb_s(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: addhnb_s:
+; CHECK: addhnb z0.h, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.addhnb.nxv4i32(<vscale x 4 x i32> %a,
+                                                                  <vscale x 4 x i32> %b)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @addhnb_d(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: addhnb_d:
+; CHECK: addhnb z0.s, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.addhnb.nxv2i64(<vscale x 2 x i64> %a,
+                                                                  <vscale x 2 x i64> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+; ADDHNT
+
+define <vscale x 16 x i8> @addhnt_h(<vscale x 16 x i8> %a, <vscale x 8 x i16> %b, <vscale x 8 x i16> %c) {
+; CHECK-LABEL: addhnt_h:
+; CHECK: addhnt z0.b, z1.h, z2.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.addhnt.nxv8i16(<vscale x 16 x i8> %a,
+                                                                  <vscale x 8 x i16> %b,
+                                                                  <vscale x 8 x i16> %c)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @addhnt_s(<vscale x 8 x i16> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %c) {
+; CHECK-LABEL: addhnt_s:
+; CHECK: addhnt z0.h, z1.s, z2.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.addhnt.nxv4i32(<vscale x 8 x i16> %a,
+                                                                  <vscale x 4 x i32> %b,
+                                                                  <vscale x 4 x i32> %c)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @addhnt_d(<vscale x 4 x i32> %a, <vscale x 2 x i64> %b, <vscale x 2 x i64> %c) {
+; CHECK-LABEL: addhnt_d:
+; CHECK: addhnt z0.s, z1.d, z2.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.addhnt.nxv2i64(<vscale x 4 x i32> %a,
+                                                                  <vscale x 2 x i64> %b,
+                                                                  <vscale x 2 x i64> %c)
+  ret <vscale x 4 x i32> %out
+}
+
+; RADDHNB
+
+define <vscale x 16 x i8> @raddhnb_h(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: raddhnb_h:
+; CHECK: raddhnb z0.b, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.raddhnb.nxv8i16(<vscale x 8 x i16> %a,
+                                                                   <vscale x 8 x i16> %b)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @raddhnb_s(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: raddhnb_s:
+; CHECK: raddhnb z0.h, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.raddhnb.nxv4i32(<vscale x 4 x i32> %a,
+                                                                   <vscale x 4 x i32> %b)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @raddhnb_d(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: raddhnb_d:
+; CHECK: raddhnb z0.s, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.raddhnb.nxv2i64(<vscale x 2 x i64> %a,
+                                                                   <vscale x 2 x i64> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+; RADDHNT
+
+define <vscale x 16 x i8> @raddhnt_h(<vscale x 16 x i8> %a, <vscale x 8 x i16> %b, <vscale x 8 x i16> %c) {
+; CHECK-LABEL: raddhnt_h:
+; CHECK: raddhnt z0.b, z1.h, z2.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.raddhnt.nxv8i16(<vscale x 16 x i8> %a,
+                                                                   <vscale x 8 x i16> %b,
+                                                                   <vscale x 8 x i16> %c)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @raddhnt_s(<vscale x 8 x i16> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %c) {
+; CHECK-LABEL: raddhnt_s:
+; CHECK: raddhnt z0.h, z1.s, z2.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.raddhnt.nxv4i32(<vscale x 8 x i16> %a,
+                                                                   <vscale x 4 x i32> %b,
+                                                                   <vscale x 4 x i32> %c)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @raddhnt_d(<vscale x 4 x i32> %a, <vscale x 2 x i64> %b, <vscale x 2 x i64> %c) {
+; CHECK-LABEL: raddhnt_d:
+; CHECK: raddhnt z0.s, z1.d, z2.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.raddhnt.nxv2i64(<vscale x 4 x i32> %a,
+                                                                   <vscale x 2 x i64> %b,
+                                                                   <vscale x 2 x i64> %c)
+  ret <vscale x 4 x i32> %out
+}
+
+; RSUBHNB
+
+define <vscale x 16 x i8> @rsubhnb_h(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: rsubhnb_h:
+; CHECK: rsubhnb z0.b, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.rsubhnb.nxv8i16(<vscale x 8 x i16> %a,
+                                                                   <vscale x 8 x i16> %b)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @rsubhnb_s(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: rsubhnb_s:
+; CHECK: rsubhnb z0.h, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.rsubhnb.nxv4i32(<vscale x 4 x i32> %a,
+                                                                   <vscale x 4 x i32> %b)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @rsubhnb_d(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: rsubhnb_d:
+; CHECK: rsubhnb z0.s, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.rsubhnb.nxv2i64(<vscale x 2 x i64> %a,
+                                                                   <vscale x 2 x i64> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+; RSUBHNT
+
+define <vscale x 16 x i8> @rsubhnt_h(<vscale x 16 x i8> %a, <vscale x 8 x i16> %b, <vscale x 8 x i16> %c) {
+; CHECK-LABEL: rsubhnt_h:
+; CHECK: rsubhnt z0.b, z1.h, z2.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.rsubhnt.nxv8i16(<vscale x 16 x i8> %a,
+                                                                   <vscale x 8 x i16> %b,
+                                                                   <vscale x 8 x i16> %c)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @rsubhnt_s(<vscale x 8 x i16> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %c) {
+; CHECK-LABEL: rsubhnt_s:
+; CHECK: rsubhnt z0.h, z1.s, z2.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.rsubhnt.nxv4i32(<vscale x 8 x i16> %a,
+                                                                   <vscale x 4 x i32> %b,
+                                                                   <vscale x 4 x i32> %c)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @rsubhnt_d(<vscale x 4 x i32> %a, <vscale x 2 x i64> %b, <vscale x 2 x i64> %c) {
+; CHECK-LABEL: rsubhnt_d:
+; CHECK: rsubhnt z0.s, z1.d, z2.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.rsubhnt.nxv2i64(<vscale x 4 x i32> %a,
+                                                                   <vscale x 2 x i64> %b,
+                                                                   <vscale x 2 x i64> %c)
+  ret <vscale x 4 x i32> %out
+}
+
+; SUBHNB
+
+define <vscale x 16 x i8> @subhnb_h(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: subhnb_h:
+; CHECK: subhnb z0.b, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.subhnb.nxv8i16(<vscale x 8 x i16> %a,
+                                                                  <vscale x 8 x i16> %b)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @subhnb_s(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: subhnb_s:
+; CHECK: subhnb z0.h, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.subhnb.nxv4i32(<vscale x 4 x i32> %a,
+                                                                  <vscale x 4 x i32> %b)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @subhnb_d(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: subhnb_d:
+; CHECK: subhnb z0.s, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.subhnb.nxv2i64(<vscale x 2 x i64> %a,
+                                                                  <vscale x 2 x i64> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+; SUBHNT
+
+define <vscale x 16 x i8> @subhnt_h(<vscale x 16 x i8> %a, <vscale x 8 x i16> %b, <vscale x 8 x i16> %c) {
+; CHECK-LABEL: subhnt_h:
+; CHECK: subhnt z0.b, z1.h, z2.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.subhnt.nxv8i16(<vscale x 16 x i8> %a,
+                                                                  <vscale x 8 x i16> %b,
+                                                                  <vscale x 8 x i16> %c)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @subhnt_s(<vscale x 8 x i16> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %c) {
+; CHECK-LABEL: subhnt_s:
+; CHECK: subhnt z0.h, z1.s, z2.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.subhnt.nxv4i32(<vscale x 8 x i16> %a,
+                                                                  <vscale x 4 x i32> %b,
+                                                                  <vscale x 4 x i32> %c)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @subhnt_d(<vscale x 4 x i32> %a, <vscale x 2 x i64> %b, <vscale x 2 x i64> %c) {
+; CHECK-LABEL: subhnt_d:
+; CHECK: subhnt z0.s, z1.d, z2.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.subhnt.nxv2i64(<vscale x 4 x i32> %a,
+                                                                  <vscale x 2 x i64> %b,
+                                                                  <vscale x 2 x i64> %c)
+  ret <vscale x 4 x i32> %out
+}
+
+
+declare <vscale x 16 x i8> @llvm.aarch64.sve.addhnb.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i16>)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.addhnb.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.addhnb.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>)
+
+declare <vscale x 16 x i8> @llvm.aarch64.sve.addhnt.nxv8i16(<vscale x 16 x i8>, <vscale x 8 x i16>, <vscale x 8 x i16>)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.addhnt.nxv4i32(<vscale x 8 x i16>, <vscale x 4 x i32>, <vscale x 4 x i32>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.addhnt.nxv2i64(<vscale x 4 x i32>, <vscale x 2 x i64>, <vscale x 2 x i64>)
+
+declare <vscale x 16 x i8> @llvm.aarch64.sve.raddhnb.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i16>)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.raddhnb.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.raddhnb.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>)
+
+declare <vscale x 16 x i8> @llvm.aarch64.sve.raddhnt.nxv8i16(<vscale x 16 x i8>, <vscale x 8 x i16>, <vscale x 8 x i16>)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.raddhnt.nxv4i32(<vscale x 8 x i16>, <vscale x 4 x i32>, <vscale x 4 x i32>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.raddhnt.nxv2i64(<vscale x 4 x i32>, <vscale x 2 x i64>, <vscale x 2 x i64>)
+
+declare <vscale x 16 x i8> @llvm.aarch64.sve.subhnb.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i16>)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.subhnb.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.subhnb.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>)
+
+declare <vscale x 16 x i8> @llvm.aarch64.sve.subhnt.nxv8i16(<vscale x 16 x i8>, <vscale x 8 x i16>, <vscale x 8 x i16>)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.subhnt.nxv4i32(<vscale x 8 x i16>, <vscale x 4 x i32>, <vscale x 4 x i32>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.subhnt.nxv2i64(<vscale x 4 x i32>, <vscale x 2 x i64>, <vscale x 2 x i64>)
+
+declare <vscale x 16 x i8> @llvm.aarch64.sve.rsubhnb.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i16>)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.rsubhnb.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.rsubhnb.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>)
+
+declare <vscale x 16 x i8> @llvm.aarch64.sve.rsubhnt.nxv8i16(<vscale x 16 x i8>, <vscale x 8 x i16>, <vscale x 8 x i16>)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.rsubhnt.nxv4i32(<vscale x 8 x i16>, <vscale x 4 x i32>, <vscale x 4 x i32>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.rsubhnt.nxv2i64(<vscale x 4 x i32>, <vscale x 2 x i64>, <vscale x 2 x i64>)
diff --git a/llvm/test/CodeGen/AArch64/sve2-intrinsics-binary-narrowing-shr.ll b/llvm/test/CodeGen/AArch64/sve2-intrinsics-binary-narrowing-shr.ll
new file mode 100644
index 000000000..27f7d71c5
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve2-intrinsics-binary-narrowing-shr.ll
@@ -0,0 +1,512 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2 < %s | FileCheck %s
+
+;
+; SHRNB
+;
+
+define <vscale x 16 x i8> @shrnb_h(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: shrnb_h:
+; CHECK: shrnb z0.b, z0.h, #8
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.shrnb.nxv8i16(<vscale x 8 x i16> %a,
+                                                                 i32 8)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @shrnb_s(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: shrnb_s:
+; CHECK: shrnb z0.h, z0.s, #16
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.shrnb.nxv4i32(<vscale x 4 x i32> %a,
+                                                                 i32 16)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @shrnb_d(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: shrnb_d:
+; CHECK: shrnb z0.s, z0.d, #32
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.shrnb.nxv2i64(<vscale x 2 x i64> %a,
+                                                                 i32 32)
+  ret <vscale x 4 x i32> %out
+}
+
+;
+; UQSHRNB
+;
+
+define <vscale x 16 x i8> @uqshrnb_h(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: uqshrnb_h:
+; CHECK: uqshrnb z0.b, z0.h, #1
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.uqshrnb.nxv8i16(<vscale x 8 x i16> %a,
+                                                                   i32 1)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @uqshrnb_s(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: uqshrnb_s:
+; CHECK: uqshrnb z0.h, z0.s, #1
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.uqshrnb.nxv4i32(<vscale x 4 x i32> %a,
+                                                                   i32 1)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @uqshrnb_d(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: uqshrnb_d:
+; CHECK: uqshrnb z0.s, z0.d, #1
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.uqshrnb.nxv2i64(<vscale x 2 x i64> %a,
+                                                                   i32 1)
+  ret <vscale x 4 x i32> %out
+}
+
+;
+; SQSHRNB
+;
+
+define <vscale x 16 x i8> @sqshrnb_h(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: sqshrnb_h:
+; CHECK: sqshrnb z0.b, z0.h, #1
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.sqshrnb.nxv8i16(<vscale x 8 x i16> %a,
+                                                                   i32 1)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @sqshrnb_s(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: sqshrnb_s:
+; CHECK: sqshrnb z0.h, z0.s, #1
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.sqshrnb.nxv4i32(<vscale x 4 x i32> %a,
+                                                                   i32 1)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @sqshrnb_d(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: sqshrnb_d:
+; CHECK: sqshrnb z0.s, z0.d, #1
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.sqshrnb.nxv2i64(<vscale x 2 x i64> %a,
+                                                                   i32 1)
+  ret <vscale x 4 x i32> %out
+}
+
+;
+; SQSHRUNB
+;
+
+define <vscale x 16 x i8> @sqshrunb_h(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: qshrunb_h:
+; CHECK: sqshrunb z0.b, z0.h, #7
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.sqshrunb.nxv8i16(<vscale x 8 x i16> %a,
+                                                                    i32 7)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @sqshrunb_s(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: sqshrunb_s:
+; CHECK: sqshrunb z0.h, z0.s, #15
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.sqshrunb.nxv4i32(<vscale x 4 x i32> %a,
+                                                                    i32 15)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @sqshrunb_d(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: sqshrunb_d:
+; CHECK: sqshrunb z0.s, z0.d, #31
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.sqshrunb.nxv2i64(<vscale x 2 x i64> %a,
+                                                                    i32 31)
+  ret <vscale x 4 x i32> %out
+}
+
+;
+; UQRSHRNB
+;
+
+define <vscale x 16 x i8> @uqrshrnb_h(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: uqrshrnb_h:
+; CHECK: uqrshrnb z0.b, z0.h, #2
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.uqrshrnb.nxv8i16(<vscale x 8 x i16> %a,
+                                                                    i32 2)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @uqrshrnb_s(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: uqrshrnb_s:
+; CHECK: uqrshrnb z0.h, z0.s, #2
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.uqrshrnb.nxv4i32(<vscale x 4 x i32> %a,
+                                                                    i32 2)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @uqrshrnb_d(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: uqrshrnb_d:
+; CHECK: uqrshrnb z0.s, z0.d, #2
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.uqrshrnb.nxv2i64(<vscale x 2 x i64> %a,
+                                                                    i32 2)
+  ret <vscale x 4 x i32> %out
+}
+
+;
+; SQRSHRNB
+;
+
+define <vscale x 16 x i8> @sqrshrnb_h(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: sqrshrnb_h:
+; CHECK: sqrshrnb z0.b, z0.h, #2
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.sqrshrnb.nxv8i16(<vscale x 8 x i16> %a,
+                                                                    i32 2)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @sqrshrnb_s(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: sqrshrnb_s:
+; CHECK: sqrshrnb z0.h, z0.s, #2
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.sqrshrnb.nxv4i32(<vscale x 4 x i32> %a,
+                                                                    i32 2)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @sqrshrnb_d(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: sqrshrnb_d:
+; CHECK: sqrshrnb z0.s, z0.d, #2
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.sqrshrnb.nxv2i64(<vscale x 2 x i64> %a,
+                                                                    i32 2)
+  ret <vscale x 4 x i32> %out
+}
+
+;
+; SQRSHRUNB
+;
+
+define <vscale x 16 x i8> @sqrshrunb_h(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: sqrshrunb_h:
+; CHECK: sqrshrunb z0.b, z0.h, #6
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.sqrshrunb.nxv8i16(<vscale x 8 x i16> %a,
+                                                                     i32 6)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @sqrshrunb_s(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: sqrshrunb_s:
+; CHECK: sqrshrunb z0.h, z0.s, #14
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.sqrshrunb.nxv4i32(<vscale x 4 x i32> %a,
+                                                                     i32 14)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @sqrshrunb_d(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: sqrshrunb_d:
+; CHECK: sqrshrunb z0.s, z0.d, #30
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.sqrshrunb.nxv2i64(<vscale x 2 x i64> %a,
+                                                                     i32 30)
+  ret <vscale x 4 x i32> %out
+}
+
+;
+; SHRNT
+;
+
+define <vscale x 16 x i8> @shrnt_h(<vscale x 16 x i8> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: shrnt_h:
+; CHECK: shrnt z0.b, z1.h, #3
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.shrnt.nxv8i16(<vscale x 16 x i8> %a,
+                                                                 <vscale x 8 x i16> %b,
+                                                                 i32 3)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @shrnt_s(<vscale x 8 x i16> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: shrnt_s:
+; CHECK: shrnt z0.h, z1.s, #3
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.shrnt.nxv4i32(<vscale x 8 x i16> %a,
+                                                                 <vscale x 4 x i32> %b,
+                                                                 i32 3)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @shrnt_d(<vscale x 4 x i32> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: shrnt_d:
+; CHECK: shrnt z0.s, z1.d, #3
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.shrnt.nxv2i64(<vscale x 4 x i32> %a,
+                                                                 <vscale x 2 x i64> %b,
+                                                                 i32 3)
+  ret <vscale x 4 x i32> %out
+}
+
+;
+; UQSHRNT
+;
+
+define <vscale x 16 x i8> @uqshrnt_h(<vscale x 16 x i8> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: uqshrnt_h:
+; CHECK: uqshrnt z0.b, z1.h, #5
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.uqshrnt.nxv8i16(<vscale x 16 x i8> %a,
+                                                                   <vscale x 8 x i16> %b,
+                                                                   i32 5)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @uqshrnt_s(<vscale x 8 x i16> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: uqshrnt_s:
+; CHECK: uqshrnt z0.h, z1.s, #13
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.uqshrnt.nxv4i32(<vscale x 8 x i16> %a,
+                                                                   <vscale x 4 x i32> %b,
+                                                                   i32 13)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @uqshrnt_d(<vscale x 4 x i32> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: uqshrnt_d:
+; CHECK: uqshrnt z0.s, z1.d, #29
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.uqshrnt.nxv2i64(<vscale x 4 x i32> %a,
+                                                                   <vscale x 2 x i64> %b,
+                                                                   i32 29)
+  ret <vscale x 4 x i32> %out
+}
+
+;
+; SQSHRNT
+;
+
+define <vscale x 16 x i8> @sqshrnt_h(<vscale x 16 x i8> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: sqshrnt_h:
+; CHECK: sqshrnt z0.b, z1.h, #5
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.sqshrnt.nxv8i16(<vscale x 16 x i8> %a,
+                                                                   <vscale x 8 x i16> %b,
+                                                                   i32 5)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @sqshrnt_s(<vscale x 8 x i16> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: sqshrnt_s:
+; CHECK: sqshrnt z0.h, z1.s, #13
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.sqshrnt.nxv4i32(<vscale x 8 x i16> %a,
+                                                                   <vscale x 4 x i32> %b,
+                                                                   i32 13)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @sqshrnt_d(<vscale x 4 x i32> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: sqshrnt_d:
+; CHECK: sqshrnt z0.s, z1.d, #29
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.sqshrnt.nxv2i64(<vscale x 4 x i32> %a,
+                                                                   <vscale x 2 x i64> %b,
+                                                                   i32 29)
+  ret <vscale x 4 x i32> %out
+}
+
+;
+; SQSHRUNT
+;
+
+define <vscale x 16 x i8> @sqshrunt_h(<vscale x 16 x i8> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: sqshrunt_h:
+; CHECK: sqshrunt z0.b, z1.h, #4
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.sqshrunt.nxv8i16(<vscale x 16 x i8> %a,
+                                                                    <vscale x 8 x i16> %b,
+                                                                    i32 4)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @sqshrunt_s(<vscale x 8 x i16> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: sqshrunt_s:
+; CHECK: sqshrunt z0.h, z1.s, #4
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.sqshrunt.nxv4i32(<vscale x 8 x i16> %a,
+                                                                    <vscale x 4 x i32> %b,
+                                                                    i32 4)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @sqshrunt_d(<vscale x 4 x i32> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: sqshrunt_d:
+; CHECK: sqshrunt z0.s, z1.d, #4
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.sqshrunt.nxv2i64(<vscale x 4 x i32> %a,
+                                                                    <vscale x 2 x i64> %b,
+                                                                    i32 4)
+  ret <vscale x 4 x i32> %out
+}
+
+;
+; UQRSHRNT
+;
+
+define <vscale x 16 x i8> @uqrshrnt_h(<vscale x 16 x i8> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: uqrshrnt_h:
+; CHECK: uqrshrnt z0.b, z1.h, #8
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.uqrshrnt.nxv8i16(<vscale x 16 x i8> %a,
+                                                                    <vscale x 8 x i16> %b,
+                                                                    i32 8)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @uqrshrnt_s(<vscale x 8 x i16> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: uqrshrnt_s:
+; CHECK: uqrshrnt z0.h, z1.s, #12
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.uqrshrnt.nxv4i32(<vscale x 8 x i16> %a,
+                                                                    <vscale x 4 x i32> %b,
+                                                                    i32 12)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @uqrshrnt_d(<vscale x 4 x i32> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: uqrshrnt_d:
+; CHECK: uqrshrnt z0.s, z1.d, #28
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.uqrshrnt.nxv2i64(<vscale x 4 x i32> %a,
+                                                                    <vscale x 2 x i64> %b,
+                                                                    i32 28)
+  ret <vscale x 4 x i32> %out
+}
+
+;
+; SQRSHRNT
+;
+
+define <vscale x 16 x i8> @sqrshrnt_h(<vscale x 16 x i8> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: sqrshrnt_h:
+; CHECK: sqrshrnt z0.b, z1.h, #8
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.sqrshrnt.nxv8i16(<vscale x 16 x i8> %a,
+                                                                    <vscale x 8 x i16> %b,
+                                                                    i32 8)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @sqrshrnt_s(<vscale x 8 x i16> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: sqrshrnt_s:
+; CHECK: sqrshrnt z0.h, z1.s, #12
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.sqrshrnt.nxv4i32(<vscale x 8 x i16> %a,
+                                                                    <vscale x 4 x i32> %b,
+                                                                    i32 12)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @sqrshrnt_d(<vscale x 4 x i32> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: sqrshrnt_d:
+; CHECK: sqrshrnt z0.s, z1.d, #28
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.sqrshrnt.nxv2i64(<vscale x 4 x i32> %a,
+                                                                    <vscale x 2 x i64> %b,
+                                                                    i32 28)
+  ret <vscale x 4 x i32> %out
+}
+
+;
+; SQRSHRUNT
+;
+
+define <vscale x 16 x i8> @sqrshrunt_h(<vscale x 16 x i8> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: sqrshrunt_h:
+; CHECK: sqrshrunt z0.b, z1.h, #1
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.sqrshrunt.nxv8i16(<vscale x 16 x i8> %a,
+                                                                     <vscale x 8 x i16> %b,
+                                                                     i32 1)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @sqrshrunt_s(<vscale x 8 x i16> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: sqrshrunt_s:
+; CHECK: sqrshrunt z0.h, z1.s, #5
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.sqrshrunt.nxv4i32(<vscale x 8 x i16> %a,
+                                                                     <vscale x 4 x i32> %b,
+                                                                     i32 5)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @sqrshrunt_d(<vscale x 4 x i32> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: sqrshrunt_d:
+; CHECK: sqrshrunt z0.s, z1.d, #5
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.sqrshrunt.nxv2i64(<vscale x 4 x i32> %a,
+                                                                     <vscale x 2 x i64> %b,
+                                                                     i32 5)
+  ret <vscale x 4 x i32> %out
+}
+
+declare <vscale x 16 x i8> @llvm.aarch64.sve.shrnb.nxv8i16(<vscale x 8 x i16>, i32)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.shrnb.nxv4i32(<vscale x 4 x i32>, i32)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.shrnb.nxv2i64(<vscale x 2 x i64>, i32)
+
+declare <vscale x 16 x i8> @llvm.aarch64.sve.uqshrnb.nxv8i16(<vscale x 8 x i16>, i32)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.uqshrnb.nxv4i32(<vscale x 4 x i32>, i32)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.uqshrnb.nxv2i64(<vscale x 2 x i64>, i32)
+
+declare <vscale x 16 x i8> @llvm.aarch64.sve.sqshrnb.nxv8i16(<vscale x 8 x i16>, i32)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.sqshrnb.nxv4i32(<vscale x 4 x i32>, i32)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.sqshrnb.nxv2i64(<vscale x 2 x i64>, i32)
+
+declare <vscale x 16 x i8> @llvm.aarch64.sve.uqrshrnb.nxv8i16(<vscale x 8 x i16>, i32)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.uqrshrnb.nxv4i32(<vscale x 4 x i32>, i32)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.uqrshrnb.nxv2i64(<vscale x 2 x i64>, i32)
+
+declare <vscale x 16 x i8> @llvm.aarch64.sve.sqrshrnb.nxv8i16(<vscale x 8 x i16>, i32)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.sqrshrnb.nxv4i32(<vscale x 4 x i32>, i32)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.sqrshrnb.nxv2i64(<vscale x 2 x i64>, i32)
+
+declare <vscale x 16 x i8> @llvm.aarch64.sve.sqshrunb.nxv8i16(<vscale x 8 x i16>, i32)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.sqshrunb.nxv4i32(<vscale x 4 x i32>, i32)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.sqshrunb.nxv2i64(<vscale x 2 x i64>, i32)
+
+declare <vscale x 16 x i8> @llvm.aarch64.sve.sqrshrunb.nxv8i16(<vscale x 8 x i16>, i32)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.sqrshrunb.nxv4i32(<vscale x 4 x i32>, i32)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.sqrshrunb.nxv2i64(<vscale x 2 x i64>, i32)
+
+declare <vscale x 16 x i8> @llvm.aarch64.sve.shrnt.nxv8i16(<vscale x 16 x i8>, <vscale x 8 x i16>, i32)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.shrnt.nxv4i32(<vscale x 8 x i16>, <vscale x 4 x i32>, i32)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.shrnt.nxv2i64(<vscale x 4 x i32>, <vscale x 2 x i64>, i32)
+
+declare <vscale x 16 x i8> @llvm.aarch64.sve.uqshrnt.nxv8i16(<vscale x 16 x i8>, <vscale x 8 x i16>, i32)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.uqshrnt.nxv4i32(<vscale x 8 x i16>, <vscale x 4 x i32>, i32)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.uqshrnt.nxv2i64(<vscale x 4 x i32>, <vscale x 2 x i64>, i32)
+
+declare <vscale x 16 x i8> @llvm.aarch64.sve.sqshrnt.nxv8i16(<vscale x 16 x i8>, <vscale x 8 x i16>, i32)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.sqshrnt.nxv4i32(<vscale x 8 x i16>, <vscale x 4 x i32>, i32)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.sqshrnt.nxv2i64(<vscale x 4 x i32>, <vscale x 2 x i64>, i32)
+
+declare <vscale x 16 x i8> @llvm.aarch64.sve.sqshrunt.nxv8i16(<vscale x 16 x i8>, <vscale x 8 x i16>, i32)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.sqshrunt.nxv4i32(<vscale x 8 x i16>, <vscale x 4 x i32>, i32)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.sqshrunt.nxv2i64(<vscale x 4 x i32>, <vscale x 2 x i64>, i32)
+
+declare <vscale x 16 x i8> @llvm.aarch64.sve.uqrshrnt.nxv8i16(<vscale x 16 x i8>, <vscale x 8 x i16>, i32)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.uqrshrnt.nxv4i32(<vscale x 8 x i16>, <vscale x 4 x i32>, i32)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.uqrshrnt.nxv2i64(<vscale x 4 x i32>, <vscale x 2 x i64>, i32)
+
+declare <vscale x 16 x i8> @llvm.aarch64.sve.sqrshrnt.nxv8i16(<vscale x 16 x i8>, <vscale x 8 x i16>, i32)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.sqrshrnt.nxv4i32(<vscale x 8 x i16>, <vscale x 4 x i32>, i32)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.sqrshrnt.nxv2i64(<vscale x 4 x i32>, <vscale x 2 x i64>, i32)
+
+declare <vscale x 16 x i8> @llvm.aarch64.sve.sqrshrunt.nxv8i16(<vscale x 16 x i8>, <vscale x 8 x i16>, i32)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.sqrshrunt.nxv4i32(<vscale x 8 x i16>, <vscale x 4 x i32>, i32)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.sqrshrunt.nxv2i64(<vscale x 4 x i32>, <vscale x 2 x i64>, i32)
diff --git a/llvm/test/CodeGen/AArch64/sve2-intrinsics-fp-converts.ll b/llvm/test/CodeGen/AArch64/sve2-intrinsics-fp-converts.ll
new file mode 100644
index 000000000..4d110fee4
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve2-intrinsics-fp-converts.ll
@@ -0,0 +1,84 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2 < %s | FileCheck %s
+
+;
+; FCVTLT
+;
+
+define <vscale x 4 x float> @fcvtlt_f32_f16(<vscale x 4 x float> %a, <vscale x 16 x i1> %pg, <vscale x 8 x half> %b) {
+; CHECK-LABEL: fcvtlt_f32_f16:
+; CHECK: fcvtlt z0.s, p0/m, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.fcvtlt.f32f16(<vscale x 4 x float> %a,
+                                                                   <vscale x 16 x i1> %pg,
+                                                                   <vscale x 8 x half> %b)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @fcvtlt_f64_f32(<vscale x 2 x double> %a, <vscale x 16 x i1> %pg, <vscale x 4 x float> %b) {
+; CHECK-LABEL: fcvtlt_f64_f32:
+; CHECK: fcvtlt	z0.d, p0/m, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.fcvtlt.f64f32(<vscale x 2 x double> %a,
+                                                                    <vscale x 16 x i1> %pg,
+                                                                    <vscale x 4 x float> %b)
+  ret <vscale x 2 x double> %out
+}
+
+;
+; FCVTNT
+;
+
+define <vscale x 8 x half> @fcvtnt_f16_f32(<vscale x 8 x half> %a, <vscale x 16 x i1> %pg, <vscale x 4 x float> %b) {
+; CHECK-LABEL: fcvtnt_f16_f32:
+; CHECK: fcvtnt z0.h, p0/m, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.fcvtnt.f16f32(<vscale x 8 x half> %a,
+                                                             <vscale x 16 x i1> %pg,
+                                                             <vscale x 4 x float> %b)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x float> @fcvtnt_f32_f64(<vscale x 4 x float> %a, <vscale x 16 x i1> %pg, <vscale x 2 x double> %b) {
+; CHECK-LABEL: fcvtnt_f32_f64:
+; CHECK: fcvtnt	z0.s, p0/m, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.fcvtnt.f32f64(<vscale x 4 x float> %a,
+                                                                   <vscale x 16 x i1> %pg,
+                                                                   <vscale x 2 x double> %b)
+  ret <vscale x 4 x float> %out
+}
+
+;
+; FCVTX
+;
+
+define <vscale x 4 x float> @fcvtx_f32_f64(<vscale x 4 x float> %a, <vscale x 16 x i1> %pg, <vscale x 2 x double> %b) {
+; CHECK-LABEL: fcvtx_f32_f64:
+; CHECK: fcvtx z0.s, p0/m, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.fcvtx.f32f64(<vscale x 4 x float> %a,
+                                                                  <vscale x 16 x i1> %pg,
+                                                                  <vscale x 2 x double> %b)
+  ret <vscale x 4 x float> %out
+}
+
+;
+; FCVTXNT
+;
+
+define <vscale x 4 x float> @fcvtxnt_f32_f64(<vscale x 4 x float> %a, <vscale x 16 x i1> %pg, <vscale x 2 x double> %b) {
+; CHECK-LABEL: fcvtxnt_f32_f64:
+; CHECK: fcvtxnt z0.s, p0/m, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.fcvtxnt.f32f64(<vscale x 4 x float> %a,
+                                                                    <vscale x 16 x i1> %pg,
+                                                                    <vscale x 2 x double> %b)
+  ret <vscale x 4 x float> %out
+}
+
+declare <vscale x 4 x float> @llvm.aarch64.sve.fcvtlt.f32f16(<vscale x 4 x float>, <vscale x 16 x i1>, <vscale x 8 x half>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.fcvtlt.f64f32(<vscale x 2 x double>, <vscale x 16 x i1>, <vscale x 4 x float>)
+declare <vscale x 8 x half> @llvm.aarch64.sve.fcvtnt.f16f32(<vscale x 8 x half>, <vscale x 16 x i1>, <vscale x 4 x float>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.fcvtnt.f32f64(<vscale x 4 x float>, <vscale x 16 x i1>, <vscale x 2 x double>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.fcvtx.f32f64(<vscale x 4 x float>, <vscale x 16 x i1>, <vscale x 2 x double>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.fcvtxnt.f32f64(<vscale x 4 x float>, <vscale x 16 x i1>, <vscale x 2 x double>)
diff --git a/llvm/test/CodeGen/AArch64/sve2-intrinsics-fp-int-binary-logarithm.ll b/llvm/test/CodeGen/AArch64/sve2-intrinsics-fp-int-binary-logarithm.ll
new file mode 100644
index 000000000..fe12324a4
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve2-intrinsics-fp-int-binary-logarithm.ll
@@ -0,0 +1,39 @@
+;RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2 -asm-verbose=0 < %s | FileCheck %s
+
+;
+; FLOGB
+;
+
+define <vscale x 8 x i16> @flogb_f16(<vscale x 8 x i16> %a, <vscale x 8 x i1> %pg, <vscale x 8 x half> %b) {
+; CHECK-LABEL: flogb_f16:
+; CHECK: flogb z0.h, p0/m, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.flogb.nxv8f16(<vscale x 8 x i16> %a,
+                                                                 <vscale x 8 x i1> %pg,
+                                                                 <vscale x 8 x half> %b)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @flogb_f32(<vscale x 4 x i32> %a, <vscale x 4 x i1> %pg, <vscale x 4 x float> %b) {
+; CHECK-LABEL: flogb_f32:
+; CHECK: flogb z0.s, p0/m, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.flogb.nxv4f32(<vscale x 4 x i32> %a,
+                                                                 <vscale x 4 x i1> %pg,
+                                                                 <vscale x 4 x float> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @flogb_f64(<vscale x 2 x i64> %a, <vscale x 2 x i1> %pg, <vscale x 2 x double> %b) {
+; CHECK-LABEL: flogb_f64:
+; CHECK: flogb z0.d, p0/m, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.flogb.nxv2f64(<vscale x 2 x i64> %a,
+                                                                 <vscale x 2 x i1> %pg,
+                                                                 <vscale x 2 x double> %b)
+  ret <vscale x 2 x i64> %out
+}
+
+declare <vscale x 8 x i16> @llvm.aarch64.sve.flogb.nxv8f16(<vscale x 8 x i16>, <vscale x 8 x i1>, <vscale x 8 x half>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.flogb.nxv4f32(<vscale x 4 x i32>, <vscale x 4 x i1>, <vscale x 4 x float>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.flogb.nxv2f64(<vscale x 2 x i64>, <vscale x 2 x i1>, <vscale x 2 x double>)
diff --git a/llvm/test/CodeGen/AArch64/sve2-intrinsics-fp-widening-mul-acc.ll b/llvm/test/CodeGen/AArch64/sve2-intrinsics-fp-widening-mul-acc.ll
new file mode 100644
index 000000000..12cc12cca
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve2-intrinsics-fp-widening-mul-acc.ll
@@ -0,0 +1,127 @@
+;RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2 < %s | FileCheck %s
+
+;
+; FMLALB (Vectors)
+;
+
+define <vscale x 4 x float> @fmlalb_h(<vscale x 4 x float> %a, <vscale x 8 x half> %b, <vscale x 8 x half> %c) {
+; CHECK-LABEL: fmlalb_h:
+; CHECK: fmlalb z0.s, z1.h, z2.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.fmlalb.nxv4f32(<vscale x 4 x float> %a,
+                                                                    <vscale x 8 x half> %b,
+                                                                    <vscale x 8 x half> %c)
+  ret <vscale x 4 x float> %out
+}
+
+;
+; FMLALB (Indexed)
+;
+
+define <vscale x 4 x float> @fmlalb_lane_h(<vscale x 4 x float> %a, <vscale x 8 x half> %b, <vscale x 8 x half> %c) {
+; CHECK-LABEL: fmlalb_lane_h:
+; CHECK: fmlalb z0.s, z1.h, z2.h[0]
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.fmlalb.lane.nxv4f32(<vscale x 4 x float> %a,
+                                                                         <vscale x 8 x half> %b,
+                                                                         <vscale x 8 x half> %c,
+                                                                         i32 0)
+  ret <vscale x 4 x float> %out
+}
+
+;
+; FMLALT (Vectors)
+;
+
+define <vscale x 4 x float> @fmlalt_h(<vscale x 4 x float> %a, <vscale x 8 x half> %b, <vscale x 8 x half> %c) {
+; CHECK-LABEL: fmlalt_h:
+; CHECK: fmlalt z0.s, z1.h, z2.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.fmlalt.nxv4f32(<vscale x 4 x float> %a,
+                                                                    <vscale x 8 x half> %b,
+                                                                    <vscale x 8 x half> %c)
+  ret <vscale x 4 x float> %out
+}
+
+;
+; FMLALT (Indexed)
+;
+
+define <vscale x 4 x float> @fmlalt_lane_h(<vscale x 4 x float> %a, <vscale x 8 x half> %b, <vscale x 8 x half> %c) {
+; CHECK-LABEL: fmlalt_lane_h:
+; CHECK: fmlalt z0.s, z1.h, z2.h[1]
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.fmlalt.lane.nxv4f32(<vscale x 4 x float> %a,
+                                                                         <vscale x 8 x half> %b,
+                                                                         <vscale x 8 x half> %c,
+                                                                         i32 1)
+  ret <vscale x 4 x float> %out
+}
+
+;
+; FMLSLB (Vectors)
+;
+
+define <vscale x 4 x float> @fmlslb_h(<vscale x 4 x float> %a, <vscale x 8 x half> %b, <vscale x 8 x half> %c) {
+; CHECK-LABEL: fmlslb_h:
+; CHECK: fmlslb z0.s, z1.h, z2.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.fmlslb.nxv4f32(<vscale x 4 x float> %a,
+                                                                    <vscale x 8 x half> %b,
+                                                                    <vscale x 8 x half> %c)
+  ret <vscale x 4 x float> %out
+}
+
+;
+; FMLSLB (Indexed)
+;
+
+define <vscale x 4 x float> @fmlslb_lane_h(<vscale x 4 x float> %a, <vscale x 8 x half> %b, <vscale x 8 x half> %c) {
+; CHECK-LABEL: fmlslb_lane_h:
+; CHECK: fmlslb z0.s, z1.h, z2.h[2]
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.fmlslb.lane.nxv4f32(<vscale x 4 x float> %a,
+                                                                         <vscale x 8 x half> %b,
+                                                                         <vscale x 8 x half> %c,
+                                                                         i32 2)
+  ret <vscale x 4 x float> %out
+}
+
+;
+; FMLSLT (Vectors)
+;
+
+define <vscale x 4 x float> @fmlslt_h(<vscale x 4 x float> %a, <vscale x 8 x half> %b, <vscale x 8 x half> %c) {
+; CHECK-LABEL: fmlslt_h:
+; CHECK: fmlslt z0.s, z1.h, z2.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.fmlslt.nxv4f32(<vscale x 4 x float> %a,
+                                                                    <vscale x 8 x half> %b,
+                                                                    <vscale x 8 x half> %c)
+ ret <vscale x 4 x float> %out
+}
+
+;
+; FMLSLT (Indexed)
+;
+
+define <vscale x 4 x float> @fmlslt_lane_h(<vscale x 4 x float> %a, <vscale x 8 x half> %b, <vscale x 8 x half> %c) {
+; CHECK-LABEL: fmlslt_lane_h:
+; CHECK: fmlslt z0.s, z1.h, z2.h[3]
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.fmlslt.lane.nxv4f32(<vscale x 4 x float> %a,
+                                                                         <vscale x 8 x half> %b,
+                                                                         <vscale x 8 x half> %c,
+                                                                         i32 3)
+  ret <vscale x 4 x float> %out
+}
+
+declare <vscale x 4 x float> @llvm.aarch64.sve.fmlalb.nxv4f32(<vscale x 4 x float>, <vscale x 8 x half>, <vscale x 8 x half>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.fmlalb.lane.nxv4f32(<vscale x 4 x float>, <vscale x 8 x half>, <vscale x 8 x half>, i32)
+declare <vscale x 4 x float> @llvm.aarch64.sve.fmlalt.nxv4f32(<vscale x 4 x float>, <vscale x 8 x half>, <vscale x 8 x half>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.fmlalt.lane.nxv4f32(<vscale x 4 x float>, <vscale x 8 x half>, <vscale x 8 x half>, i32)
+
+declare <vscale x 4 x float> @llvm.aarch64.sve.fmlslb.nxv4f32(<vscale x 4 x float>, <vscale x 8 x half>, <vscale x 8 x half>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.fmlslb.lane.nxv4f32(<vscale x 4 x float>, <vscale x 8 x half>, <vscale x 8 x half>, i32)
+declare <vscale x 4 x float> @llvm.aarch64.sve.fmlslt.nxv4f32(<vscale x 4 x float>, <vscale x 8 x half>, <vscale x 8 x half>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.fmlslt.lane.nxv4f32(<vscale x 4 x float>, <vscale x 8 x half>, <vscale x 8 x half>, i32)
diff --git a/llvm/test/CodeGen/AArch64/sve2-intrinsics-non-widening-pairwise-arith.ll b/llvm/test/CodeGen/AArch64/sve2-intrinsics-non-widening-pairwise-arith.ll
new file mode 100644
index 000000000..055c24b93
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve2-intrinsics-non-widening-pairwise-arith.ll
@@ -0,0 +1,191 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2 < %s | FileCheck %s
+
+;
+; FADDP
+;
+
+define <vscale x 8 x half> @faddp_f16(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: faddp_f16:
+; CHECK: faddp z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.faddp.nxv8f16(<vscale x 8 x i1> %pg,
+                                                                  <vscale x 8 x half> %a,
+                                                                  <vscale x 8 x half> %b)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x float> @faddp_f32(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: faddp_f32:
+; CHECK: faddp z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.faddp.nxv4f32(<vscale x 4 x i1> %pg,
+                                                                   <vscale x 4 x float> %a,
+                                                                   <vscale x 4 x float> %b)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @faddp_f64(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: faddp_f64:
+; CHECK: faddp z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.faddp.nxv2f64(<vscale x 2 x i1> %pg,
+                                                                    <vscale x 2 x double> %a,
+                                                                    <vscale x 2 x double> %b)
+  ret <vscale x 2 x double> %out
+}
+
+;
+; FMAXP
+;
+
+define <vscale x 8 x half> @fmaxp_f16(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: fmaxp_f16:
+; CHECK: fmaxp z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.fmaxp.nxv8f16(<vscale x 8 x i1> %pg,
+                                                                  <vscale x 8 x half> %a,
+                                                                  <vscale x 8 x half> %b)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x float> @fmaxp_f32(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: fmaxp_f32:
+; CHECK: fmaxp z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.fmaxp.nxv4f32(<vscale x 4 x i1> %pg,
+                                                                   <vscale x 4 x float> %a,
+                                                                   <vscale x 4 x float> %b)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @fmaxp_f64(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: fmaxp_f64:
+; CHECK: fmaxp z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.fmaxp.nxv2f64(<vscale x 2 x i1> %pg,
+                                                                    <vscale x 2 x double> %a,
+                                                                    <vscale x 2 x double> %b)
+  ret <vscale x 2 x double> %out
+}
+
+;
+; FMAXNMP
+;
+
+define <vscale x 8 x half> @fmaxnmp_f16(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: fmaxnmp_f16:
+; CHECK: fmaxnmp z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.fmaxnmp.nxv8f16(<vscale x 8 x i1> %pg,
+                                                                    <vscale x 8 x half> %a,
+                                                                    <vscale x 8 x half> %b)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x float> @fmaxnmp_f32(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: fmaxnmp_f32:
+; CHECK: fmaxnmp z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.fmaxnmp.nxv4f32(<vscale x 4 x i1> %pg,
+                                                                     <vscale x 4 x float> %a,
+                                                                     <vscale x 4 x float> %b)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @fmaxnmp_f64(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: fmaxnmp_f64:
+; CHECK: fmaxnmp z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.fmaxnmp.nxv2f64(<vscale x 2 x i1> %pg,
+                                                                      <vscale x 2 x double> %a,
+                                                                      <vscale x 2 x double> %b)
+  ret <vscale x 2 x double> %out
+}
+
+;
+; FMINP
+;
+
+define <vscale x 8 x half> @fminp_f16(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: fminp_f16:
+; CHECK: fminp z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.fminp.nxv8f16(<vscale x 8 x i1> %pg,
+                                                                  <vscale x 8 x half> %a,
+                                                                  <vscale x 8 x half> %b)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x float> @fminp_f32(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: fminp_f32:
+; CHECK: fminp z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.fminp.nxv4f32(<vscale x 4 x i1> %pg,
+                                                                   <vscale x 4 x float> %a,
+                                                                   <vscale x 4 x float> %b)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @fminp_f64(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: fminp_f64:
+; CHECK: fminp z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.fminp.nxv2f64(<vscale x 2 x i1> %pg,
+                                                                    <vscale x 2 x double> %a,
+                                                                    <vscale x 2 x double> %b)
+  ret <vscale x 2 x double> %out
+}
+
+;
+; FMINNMP
+;
+
+define <vscale x 8 x half> @fminnmp_f16(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: fminnmp_f16:
+; CHECK: fminnmp z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.fminnmp.nxv8f16(<vscale x 8 x i1> %pg,
+                                                                    <vscale x 8 x half> %a,
+                                                                    <vscale x 8 x half> %b)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x float> @fminnmp_f32(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: fminnmp_f32:
+; CHECK: fminnmp z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.fminnmp.nxv4f32(<vscale x 4 x i1> %pg,
+                                                                     <vscale x 4 x float> %a,
+                                                                     <vscale x 4 x float> %b)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @fminnmp_f64(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: fminnmp_f64:
+; CHECK: fminnmp z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.fminnmp.nxv2f64(<vscale x 2 x i1> %pg,
+                                                                      <vscale x 2 x double> %a,
+                                                                      <vscale x 2 x double> %b)
+  ret <vscale x 2 x double> %out
+}
+
+declare <vscale x 8 x half> @llvm.aarch64.sve.faddp.nxv8f16(<vscale x 8 x i1>, <vscale x 8 x half>, <vscale x 8 x half>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.faddp.nxv4f32(<vscale x 4 x i1>, <vscale x 4 x float>, <vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.faddp.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>, <vscale x 2 x double>)
+
+declare <vscale x 8 x half> @llvm.aarch64.sve.fmaxp.nxv8f16(<vscale x 8 x i1>, <vscale x 8 x half>, <vscale x 8 x half>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.fmaxp.nxv4f32(<vscale x 4 x i1>, <vscale x 4 x float>, <vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.fmaxp.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>, <vscale x 2 x double>)
+
+declare <vscale x 8 x half> @llvm.aarch64.sve.fmaxnmp.nxv8f16(<vscale x 8 x i1>, <vscale x 8 x half>, <vscale x 8 x half>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.fmaxnmp.nxv4f32(<vscale x 4 x i1>, <vscale x 4 x float>, <vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.fmaxnmp.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>, <vscale x 2 x double>)
+
+declare <vscale x 8 x half> @llvm.aarch64.sve.fminp.nxv8f16(<vscale x 8 x i1>, <vscale x 8 x half>, <vscale x 8 x half>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.fminp.nxv4f32(<vscale x 4 x i1>, <vscale x 4 x float>, <vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.fminp.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>, <vscale x 2 x double>)
+
+declare <vscale x 8 x half> @llvm.aarch64.sve.fminnmp.nxv8f16(<vscale x 8 x i1>, <vscale x 8 x half>, <vscale x 8 x half>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.fminnmp.nxv4f32(<vscale x 4 x i1>, <vscale x 4 x float>, <vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.fminnmp.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>, <vscale x 2 x double>)
diff --git a/llvm/test/CodeGen/AArch64/sve2-intrinsics-unary-narrowing.ll b/llvm/test/CodeGen/AArch64/sve2-intrinsics-unary-narrowing.ll
new file mode 100644
index 000000000..3f8373991
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve2-intrinsics-unary-narrowing.ll
@@ -0,0 +1,202 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2 < %s | FileCheck %s
+
+;
+; SQXTNB
+;
+
+define <vscale x 16 x i8> @sqxtnb_h(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: sqxtnb_h:
+; CHECK: sqxtnb z0.b, z0.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.sqxtnb.nxv8i16(<vscale x 8 x i16> %a)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @sqxtnb_s(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: sqxtnb_s:
+; CHECK: sqxtnb z0.h, z0.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.sqxtnb.nxv4i32(<vscale x 4 x i32> %a)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @sqxtnb_d(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: sqxtnb_d:
+; CHECK: sqxtnb z0.s, z0.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.sqxtnb.nxv2i64(<vscale x 2 x i64> %a)
+  ret <vscale x 4 x i32> %out
+}
+
+;
+; UQXTNB
+;
+
+define <vscale x 16 x i8> @uqxtnb_h(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: uqxtnb_h:
+; CHECK: uqxtnb z0.b, z0.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.uqxtnb.nxv8i16(<vscale x 8 x i16> %a)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @uqxtnb_s(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: uqxtnb_s:
+; CHECK: uqxtnb z0.h, z0.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.uqxtnb.nxv4i32(<vscale x 4 x i32> %a)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @uqxtnb_d(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: uqxtnb_d:
+; CHECK: uqxtnb z0.s, z0.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.uqxtnb.nxv2i64(<vscale x 2 x i64> %a)
+  ret <vscale x 4 x i32> %out
+}
+
+;
+; SQXTUNB
+;
+
+define <vscale x 16 x i8> @sqxtunb_h(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: sqxtunb_h:
+; CHECK: sqxtunb z0.b, z0.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.sqxtunb.nxv8i16(<vscale x 8 x i16> %a)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @sqxtunb_s(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: sqxtunb_s:
+; CHECK: sqxtunb z0.h, z0.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.sqxtunb.nxv4i32(<vscale x 4 x i32> %a)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @sqxtunb_d(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: sqxtunb_d:
+; CHECK: sqxtunb z0.s, z0.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.sqxtunb.nxv2i64(<vscale x 2 x i64> %a)
+  ret <vscale x 4 x i32> %out
+}
+
+;
+; SQXTNT
+;
+
+define <vscale x 16 x i8> @sqxtnt_h(<vscale x 16 x i8> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: sqxtnt_h:
+; CHECK: sqxtnt z0.b, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.sqxtnt.nxv8i16(<vscale x 16 x i8> %a,
+                                                             <vscale x 8 x i16> %b)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @sqxtnt_s(<vscale x 8 x i16> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: sqxtnt_s:
+; CHECK: sqxtnt z0.h, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.sqxtnt.nxv4i32(<vscale x 8 x i16> %a,
+                                                             <vscale x 4 x i32> %b)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @sqxtnt_d(<vscale x 4 x i32> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: sqxtnt_d:
+; CHECK: sqxtnt z0.s, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.sqxtnt.nxv2i64(<vscale x 4 x i32> %a,
+                                                             <vscale x 2 x i64> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+;
+; UQXTNT
+;
+
+define <vscale x 16 x i8> @uqxtnt_h(<vscale x 16 x i8> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: uqxtnt_h:
+; CHECK: uqxtnt z0.b, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.uqxtnt.nxv8i16(<vscale x 16 x i8> %a,
+                                                             <vscale x 8 x i16> %b)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @uqxtnt_s(<vscale x 8 x i16> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: uqxtnt_s:
+; CHECK: uqxtnt z0.h, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.uqxtnt.nxv4i32(<vscale x 8 x i16> %a,
+                                                             <vscale x 4 x i32> %b)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @uqxtnt_d(<vscale x 4 x i32> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: uqxtnt_d:
+; CHECK: uqxtnt z0.s, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.uqxtnt.nxv2i64(<vscale x 4 x i32> %a,
+                                                             <vscale x 2 x i64> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+;
+; SQXTUNT
+;
+
+define <vscale x 16 x i8> @sqxtunt_h(<vscale x 16 x i8> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: sqxtunt_h:
+; CHECK: sqxtunt z0.b, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.sqxtunt.nxv8i16(<vscale x 16 x i8> %a,
+                                                              <vscale x 8 x i16> %b)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @sqxtunt_s(<vscale x 8 x i16> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: sqxtunt_s:
+; CHECK: sqxtunt z0.h, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.sqxtunt.nxv4i32(<vscale x 8 x i16> %a,
+                                                              <vscale x 4 x i32> %b)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @sqxtunt_d(<vscale x 4 x i32> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: sqxtunt_d:
+; CHECK: sqxtunt z0.s, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.sqxtunt.nxv2i64(<vscale x 4 x i32> %a,
+                                                              <vscale x 2 x i64> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+declare <vscale x 16 x i8> @llvm.aarch64.sve.sqxtnb.nxv8i16(<vscale x 8 x i16>)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.sqxtnb.nxv4i32(<vscale x 4 x i32>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.sqxtnb.nxv2i64(<vscale x 2 x i64>)
+
+declare <vscale x 16 x i8> @llvm.aarch64.sve.uqxtnb.nxv8i16(<vscale x 8 x i16>)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.uqxtnb.nxv4i32(<vscale x 4 x i32>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.uqxtnb.nxv2i64(<vscale x 2 x i64>)
+
+declare <vscale x 16 x i8> @llvm.aarch64.sve.sqxtunb.nxv8i16(<vscale x 8 x i16>)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.sqxtunb.nxv4i32(<vscale x 4 x i32>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.sqxtunb.nxv2i64(<vscale x 2 x i64>)
+
+declare <vscale x 16 x i8> @llvm.aarch64.sve.sqxtnt.nxv8i16(<vscale x 16 x i8>, <vscale x 8 x i16>)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.sqxtnt.nxv4i32(<vscale x 8 x i16>, <vscale x 4 x i32>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.sqxtnt.nxv2i64(<vscale x 4 x i32>, <vscale x 2 x i64>)
+
+declare <vscale x 16 x i8> @llvm.aarch64.sve.uqxtnt.nxv8i16(<vscale x 16 x i8>, <vscale x 8 x i16>)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.uqxtnt.nxv4i32(<vscale x 8 x i16>, <vscale x 4 x i32>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.uqxtnt.nxv2i64(<vscale x 4 x i32>, <vscale x 2 x i64>)
+
+declare <vscale x 16 x i8> @llvm.aarch64.sve.sqxtunt.nxv8i16(<vscale x 16 x i8>, <vscale x 8 x i16>)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.sqxtunt.nxv4i32(<vscale x 8 x i16>, <vscale x 4 x i32>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.sqxtunt.nxv2i64(<vscale x 4 x i32>, <vscale x 2 x i64>)
diff --git a/llvm/test/CodeGen/AArch64/sve2-intrinsics-while.ll b/llvm/test/CodeGen/AArch64/sve2-intrinsics-while.ll
new file mode 100644
index 000000000..e53c981b6
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve2-intrinsics-while.ll
@@ -0,0 +1,309 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2 < %s | FileCheck %s
+
+;
+; WHILEGE
+;
+
+define <vscale x 16 x i1> @whilege_b_ww(i32 %a, i32 %b) {
+; CHECK-LABEL: whilege_b_ww:
+; CHECK: whilege p0.b, w0, w1
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.whilege.nxv16i1.i32(i32 %a, i32 %b)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 16 x i1> @whilege_b_xx(i64 %a, i64 %b) {
+; CHECK-LABEL: whilege_b_xx:
+; CHECK: whilege p0.b, x0, x1
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.whilege.nxv16i1.i64(i64 %a, i64 %b)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @whilege_h_ww(i32 %a, i32 %b) {
+; CHECK-LABEL: whilege_h_ww:
+; CHECK: whilege p0.h, w0, w1
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.whilege.nxv8i1.i32(i32 %a, i32 %b)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 8 x i1> @whilege_h_xx(i64 %a, i64 %b) {
+; CHECK-LABEL: whilege_h_xx:
+; CHECK: whilege p0.h, x0, x1
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.whilege.nxv8i1.i64(i64 %a, i64 %b)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @whilege_s_ww(i32 %a, i32 %b) {
+; CHECK-LABEL: whilege_s_ww:
+; CHECK: whilege p0.s, w0, w1
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.whilege.nxv4i1.i32(i32 %a, i32 %b)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 4 x i1> @whilege_s_xx(i64 %a, i64 %b) {
+; CHECK-LABEL: whilege_s_xx:
+; CHECK: whilege p0.s, x0, x1
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.whilege.nxv4i1.i64(i64 %a, i64 %b)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 2 x i1> @whilege_d_ww(i32 %a, i32 %b) {
+; CHECK-LABEL: whilege_d_ww:
+; CHECK: whilege p0.d, w0, w1
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.whilege.nxv2i1.i32(i32 %a, i32 %b)
+  ret <vscale x 2 x i1> %out
+}
+
+define <vscale x 2 x i1> @whilege_d_xx(i64 %a, i64 %b) {
+; CHECK-LABEL: whilege_d_xx:
+; CHECK: whilege p0.d, x0, x1
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.whilege.nxv2i1.i64(i64 %a, i64 %b)
+  ret <vscale x 2 x i1> %out
+}
+
+;
+; WHILEHS
+;
+
+define <vscale x 16 x i1> @whilehs_b_ww(i32 %a, i32 %b) {
+; CHECK-LABEL: whilehs_b_ww:
+; CHECK: whilehs p0.b, w0, w1
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.whilehs.nxv16i1.i32(i32 %a, i32 %b)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 16 x i1> @whilehs_b_xx(i64 %a, i64 %b) {
+; CHECK-LABEL: whilehs_b_xx:
+; CHECK: whilehs p0.b, x0, x1
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.whilehs.nxv16i1.i64(i64 %a, i64 %b)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @whilehs_h_ww(i32 %a, i32 %b) {
+; CHECK-LABEL: whilehs_h_ww:
+; CHECK: whilehs p0.h, w0, w1
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.whilehs.nxv8i1.i32(i32 %a, i32 %b)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 8 x i1> @whilehs_h_xx(i64 %a, i64 %b) {
+; CHECK-LABEL: whilehs_h_xx:
+; CHECK: whilehs p0.h, x0, x1
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.whilehs.nxv8i1.i64(i64 %a, i64 %b)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @whilehs_s_ww(i32 %a, i32 %b) {
+; CHECK-LABEL: whilehs_s_ww:
+; CHECK: whilehs p0.s, w0, w1
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.whilehs.nxv4i1.i32(i32 %a, i32 %b)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 4 x i1> @whilehs_s_xx(i64 %a, i64 %b) {
+; CHECK-LABEL: whilehs_s_xx:
+; CHECK: whilehs p0.s, x0, x1
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.whilehs.nxv4i1.i64(i64 %a, i64 %b)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 2 x i1> @whilehs_d_ww(i32 %a, i32 %b) {
+; CHECK-LABEL: whilehs_d_ww:
+; CHECK: whilehs p0.d, w0, w1
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.whilehs.nxv2i1.i32(i32 %a, i32 %b)
+  ret <vscale x 2 x i1> %out
+}
+
+define <vscale x 2 x i1> @whilehs_d_xx(i64 %a, i64 %b) {
+; CHECK-LABEL: whilehs_d_xx:
+; CHECK: whilehs p0.d, x0, x1
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.whilehs.nxv2i1.i64(i64 %a, i64 %b)
+  ret <vscale x 2 x i1> %out
+}
+
+;
+; WHILEGT
+;
+
+define <vscale x 16 x i1> @whilegt_b_ww(i32 %a, i32 %b) {
+; CHECK-LABEL: whilegt_b_ww:
+; CHECK: whilegt p0.b, w0, w1
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.whilegt.nxv16i1.i32(i32 %a, i32 %b)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 16 x i1> @whilegt_b_xx(i64 %a, i64 %b) {
+; CHECK-LABEL: whilegt_b_xx:
+; CHECK: whilegt p0.b, x0, x1
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.whilegt.nxv16i1.i64(i64 %a, i64 %b)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @whilegt_h_ww(i32 %a, i32 %b) {
+; CHECK-LABEL: whilegt_h_ww:
+; CHECK: whilegt p0.h, w0, w1
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.whilegt.nxv8i1.i32(i32 %a, i32 %b)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 8 x i1> @whilegt_h_xx(i64 %a, i64 %b) {
+; CHECK-LABEL: whilegt_h_xx:
+; CHECK: whilegt p0.h, x0, x1
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.whilegt.nxv8i1.i64(i64 %a, i64 %b)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @whilegt_s_ww(i32 %a, i32 %b) {
+; CHECK-LABEL: whilegt_s_ww:
+; CHECK: whilegt p0.s, w0, w1
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.whilegt.nxv4i1.i32(i32 %a, i32 %b)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 4 x i1> @whilegt_s_xx(i64 %a, i64 %b) {
+; CHECK-LABEL: whilegt_s_xx:
+; CHECK: whilegt p0.s, x0, x1
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.whilegt.nxv4i1.i64(i64 %a, i64 %b)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 2 x i1> @whilegt_d_ww(i32 %a, i32 %b) {
+; CHECK-LABEL: whilegt_d_ww:
+; CHECK: whilegt p0.d, w0, w1
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.whilegt.nxv2i1.i32(i32 %a, i32 %b)
+  ret <vscale x 2 x i1> %out
+}
+
+define <vscale x 2 x i1> @whilegt_d_xx(i64 %a, i64 %b) {
+; CHECK-LABEL: whilegt_d_xx:
+; CHECK: whilegt p0.d, x0, x1
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.whilegt.nxv2i1.i64(i64 %a, i64 %b)
+  ret <vscale x 2 x i1> %out
+}
+
+;
+; WHILEHI
+;
+
+define <vscale x 16 x i1> @whilehi_b_ww(i32 %a, i32 %b) {
+; CHECK-LABEL: whilehi_b_ww:
+; CHECK: whilehi p0.b, w0, w1
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.whilehi.nxv16i1.i32(i32 %a, i32 %b)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 16 x i1> @whilehi_b_xx(i64 %a, i64 %b) {
+; CHECK-LABEL: whilehi_b_xx:
+; CHECK: whilehi p0.b, x0, x1
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.whilehi.nxv16i1.i64(i64 %a, i64 %b)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @whilehi_h_ww(i32 %a, i32 %b) {
+; CHECK-LABEL: whilehi_h_ww:
+; CHECK: whilehi p0.h, w0, w1
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.whilehi.nxv8i1.i32(i32 %a, i32 %b)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 8 x i1> @whilehi_h_xx(i64 %a, i64 %b) {
+; CHECK-LABEL: whilehi_h_xx:
+; CHECK: whilehi p0.h, x0, x1
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.whilehi.nxv8i1.i64(i64 %a, i64 %b)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @whilehi_s_ww(i32 %a, i32 %b) {
+; CHECK-LABEL: whilehi_s_ww:
+; CHECK: whilehi p0.s, w0, w1
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.whilehi.nxv4i1.i32(i32 %a, i32 %b)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 4 x i1> @whilehi_s_xx(i64 %a, i64 %b) {
+; CHECK-LABEL: whilehi_s_xx:
+; CHECK: whilehi p0.s, x0, x1
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.whilehi.nxv4i1.i64(i64 %a, i64 %b)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 2 x i1> @whilehi_d_ww(i32 %a, i32 %b) {
+; CHECK-LABEL: whilehi_d_ww:
+; CHECK: whilehi p0.d, w0, w1
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.whilehi.nxv2i1.i32(i32 %a, i32 %b)
+  ret <vscale x 2 x i1> %out
+}
+
+define <vscale x 2 x i1> @whilehi_d_xx(i64 %a, i64 %b) {
+; CHECK-LABEL: whilehi_d_xx:
+; CHECK: whilehi p0.d, x0, x1
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.whilehi.nxv2i1.i64(i64 %a, i64 %b)
+  ret <vscale x 2 x i1> %out
+}
+
+declare <vscale x 16 x i1> @llvm.aarch64.sve.whilege.nxv16i1.i32(i32, i32)
+declare <vscale x 16 x i1> @llvm.aarch64.sve.whilege.nxv16i1.i64(i64, i64)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.whilege.nxv8i1.i32(i32, i32)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.whilege.nxv8i1.i64(i64, i64)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.whilege.nxv4i1.i32(i32, i32)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.whilege.nxv4i1.i64(i64, i64)
+declare <vscale x 2 x i1> @llvm.aarch64.sve.whilege.nxv2i1.i32(i32, i32)
+declare <vscale x 2 x i1> @llvm.aarch64.sve.whilege.nxv2i1.i64(i64, i64)
+
+declare <vscale x 16 x i1> @llvm.aarch64.sve.whilehs.nxv16i1.i32(i32, i32)
+declare <vscale x 16 x i1> @llvm.aarch64.sve.whilehs.nxv16i1.i64(i64, i64)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.whilehs.nxv8i1.i32(i32, i32)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.whilehs.nxv8i1.i64(i64, i64)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.whilehs.nxv4i1.i32(i32, i32)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.whilehs.nxv4i1.i64(i64, i64)
+declare <vscale x 2 x i1> @llvm.aarch64.sve.whilehs.nxv2i1.i32(i32, i32)
+declare <vscale x 2 x i1> @llvm.aarch64.sve.whilehs.nxv2i1.i64(i64, i64)
+
+declare <vscale x 16 x i1> @llvm.aarch64.sve.whilegt.nxv16i1.i32(i32, i32)
+declare <vscale x 16 x i1> @llvm.aarch64.sve.whilegt.nxv16i1.i64(i64, i64)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.whilegt.nxv8i1.i32(i32, i32)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.whilegt.nxv8i1.i64(i64, i64)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.whilegt.nxv4i1.i32(i32, i32)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.whilegt.nxv4i1.i64(i64, i64)
+declare <vscale x 2 x i1> @llvm.aarch64.sve.whilegt.nxv2i1.i32(i32, i32)
+declare <vscale x 2 x i1> @llvm.aarch64.sve.whilegt.nxv2i1.i64(i64, i64)
+
+declare <vscale x 16 x i1> @llvm.aarch64.sve.whilehi.nxv16i1.i32(i32, i32)
+declare <vscale x 16 x i1> @llvm.aarch64.sve.whilehi.nxv16i1.i64(i64, i64)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.whilehi.nxv8i1.i32(i32, i32)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.whilehi.nxv8i1.i64(i64, i64)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.whilehi.nxv4i1.i32(i32, i32)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.whilehi.nxv4i1.i64(i64, i64)
+declare <vscale x 2 x i1> @llvm.aarch64.sve.whilehi.nxv2i1.i32(i32, i32)
+declare <vscale x 2 x i1> @llvm.aarch64.sve.whilehi.nxv2i1.i64(i64, i64)
diff --git a/llvm/test/CodeGen/AArch64/swift-return.ll b/llvm/test/CodeGen/AArch64/swift-return.ll
index 3d0bed4c9..2bf5e379b 100644
--- a/llvm/test/CodeGen/AArch64/swift-return.ll
+++ b/llvm/test/CodeGen/AArch64/swift-return.ll
@@ -1,5 +1,7 @@
 ; RUN: llc -verify-machineinstrs -mtriple=aarch64-apple-ios -o - %s | FileCheck %s
 ; RUN: llc -O0 -fast-isel -verify-machineinstrs -mtriple=aarch64-apple-ios -o - %s | FileCheck %s --check-prefix=CHECK-O0
+; RUN: llc -verify-machineinstrs -mtriple=arm64_32-apple-ios -o - %s | FileCheck %s
+; RUN: llc -O0 -fast-isel -verify-machineinstrs -mtriple=arm64_32-apple-ios -o - %s | FileCheck %s --check-prefix=CHECK-O0
 
 ; CHECK-LABEL: test1
 ; CHECK: bl      _gen
@@ -8,7 +10,7 @@
 ; CHECK-O0-LABEL: test1
 ; CHECK-O0: bl      _gen
 ; CHECK-O0: sxth    [[TMP:w.*]], w0
-; CHECK-O0: add     w8, [[TMP]], w1, sxtb
+; CHECK-O0: add     {{w[0-9]+}}, [[TMP]], w1, sxtb
 define i16 @test1(i32) {
 entry:
   %call = call swiftcc { i16, i8 } @gen(i32 %0)
diff --git a/llvm/test/CodeGen/AArch64/swiftcc.ll b/llvm/test/CodeGen/AArch64/swiftcc.ll
index 432495427..fb74fe4a6 100644
--- a/llvm/test/CodeGen/AArch64/swiftcc.ll
+++ b/llvm/test/CodeGen/AArch64/swiftcc.ll
@@ -1,5 +1,7 @@
 ; RUN: llc -verify-machineinstrs -mtriple=aarch64-apple-ios -o - %s | FileCheck %s
 ; RUN: llc -O0 -verify-machineinstrs -mtriple=aarch64-apple-ios -o - %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -mtriple=arm64_32-apple-ios -o - %s | FileCheck %s
+; RUN: llc -O0 -verify-machineinstrs -mtriple=arm64_32-apple-ios -o - %s | FileCheck %s
 
 ; CHECK: t1
 ; CHECK: fadd s0, s0, s1
diff --git a/llvm/test/CodeGen/AArch64/swifterror.ll b/llvm/test/CodeGen/AArch64/swifterror.ll
index 823599d5d..1eedb7620 100644
--- a/llvm/test/CodeGen/AArch64/swifterror.ll
+++ b/llvm/test/CodeGen/AArch64/swifterror.ll
@@ -1,5 +1,7 @@
-; RUN: llc -fast-isel-sink-local-values -verify-machineinstrs -frame-pointer=all -enable-shrink-wrap=false < %s -mtriple=aarch64-apple-ios -disable-post-ra | FileCheck -allow-deprecated-dag-overlap --check-prefix=CHECK-APPLE %s
-; RUN: llc -fast-isel-sink-local-values -verify-machineinstrs -frame-pointer=all -O0 -fast-isel < %s -mtriple=aarch64-apple-ios -disable-post-ra | FileCheck -allow-deprecated-dag-overlap --check-prefix=CHECK-O0 %s
+; RUN: llc -fast-isel-sink-local-values -verify-machineinstrs -frame-pointer=all -enable-shrink-wrap=false < %s -mtriple=aarch64-apple-ios -disable-post-ra | FileCheck -allow-deprecated-dag-overlap --check-prefix=CHECK-APPLE --check-prefix=CHECK-APPLE-AARCH64 %s
+; RUN: llc -fast-isel-sink-local-values -verify-machineinstrs -frame-pointer=all -O0 -fast-isel < %s -mtriple=aarch64-apple-ios -disable-post-ra | FileCheck -allow-deprecated-dag-overlap --check-prefix=CHECK-O0 --check-prefix=CHECK-O0-AARCH64 %s
+; RUN: llc -fast-isel-sink-local-values -verify-machineinstrs -frame-pointer=all -enable-shrink-wrap=false < %s -mtriple=arm64_32-apple-ios -disable-post-ra | FileCheck -allow-deprecated-dag-overlap --check-prefix=CHECK-APPLE --check-prefix=CHECK-APPLE-ARM64_32 %s
+; RUN: llc -fast-isel-sink-local-values -verify-machineinstrs -O0 -fast-isel < %s -mtriple=arm64_32-apple-ios -disable-post-ra | FileCheck -allow-deprecated-dag-overlap --check-prefix=CHECK-O0 --check-prefix=CHECK-O0-ARM64_32 %s
 
 declare i8* @malloc(i64)
 declare void @free(i8*)
@@ -40,7 +42,8 @@ define float @caller(i8* %error_ref) {
 ; CHECK-APPLE: mov x21, xzr
 ; CHECK-APPLE: bl {{.*}}foo
 ; CHECK-APPLE: mov x0, x21
-; CHECK-APPLE: cbnz x21
+; CHECK-APPLE-AARCH64: cbnz x21
+; CHECK-APPLE-ARM64_32: cbnz w0
 ; Access part of the error object and save it to error_ref
 ; CHECK-APPLE: ldrb [[CODE:w[0-9]+]], [x0, #8]
 ; CHECK-APPLE: strb [[CODE]], [{{.*}}[[ID]]]
@@ -50,7 +53,8 @@ define float @caller(i8* %error_ref) {
 ; CHECK-O0: mov x21
 ; CHECK-O0: bl {{.*}}foo
 ; CHECK-O0: mov [[ID:x[0-9]+]], x21
-; CHECK-O0: cbnz x21
+; CHECK-O0-AARCH64: cbnz x21
+; CHECK-O0-ARM64_32: cmp x21, #0
 entry:
   %error_ptr_ref = alloca swifterror %swift_error*
   store %swift_error* null, %swift_error** %error_ptr_ref
@@ -76,7 +80,8 @@ define float @caller2(i8* %error_ref) {
 ; CHECK-APPLE: fmov [[CMP:s[0-9]+]], #1.0
 ; CHECK-APPLE: mov x21, xzr
 ; CHECK-APPLE: bl {{.*}}foo
-; CHECK-APPLE: cbnz x21
+; CHECK-APPLE-AARCH64: cbnz x21
+; CHECK-APPLE-ARM64_32: cbnz w21
 ; CHECK-APPLE: fcmp s0, [[CMP]]
 ; CHECK-APPLE: b.le
 ; Access part of the error object and save it to error_ref
@@ -89,7 +94,8 @@ define float @caller2(i8* %error_ref) {
 ; CHECK-O0: mov x21
 ; CHECK-O0: bl {{.*}}foo
 ; CHECK-O0: mov [[ID:x[0-9]+]], x21
-; CHECK-O0: cbnz x21
+; CHECK-O0-AARCH64: cbnz x21
+; CHECK-O0-ARM64_32: cmp x21, #0
 entry:
   %error_ptr_ref = alloca swifterror %swift_error*
   br label %bb_loop
@@ -162,38 +168,61 @@ normal:
 define float @foo_loop(%swift_error** swifterror %error_ptr_ref, i32 %cc, float %cc2) {
 ; CHECK-APPLE-LABEL: foo_loop:
 ; CHECK-APPLE: mov x0, x21
+; CHECK-APPLE: fcmp
+; CHECK-APPLE: b.gt
 ; CHECK-APPLE: cbz
 ; CHECK-APPLE: mov w0, #16
 ; CHECK-APPLE: malloc
 ; CHECK-APPLE: strb w{{.*}}, [x0, #8]
-; CHECK-APPLE: fcmp
-; CHECK-APPLE: b.le
 ; CHECK-APPLE: mov x21, x0
 ; CHECK-APPLE: ret
 
-; CHECK-O0-LABEL: foo_loop:
+; CHECK-O0-AARCH64-LABEL: foo_loop:
 ; spill x21
-; CHECK-O0: str x21, [sp, [[SLOT:#[0-9]+]]]
-; CHECK-O0: b [[BB1:[A-Za-z0-9_]*]]
-; CHECK-O0: [[BB1]]:
-; CHECK-O0: ldr     x0, [sp, [[SLOT]]]
-; CHECK-O0: str     x0, [sp, [[SLOT2:#[0-9]+]]]
-; CHECK-O0: cbz {{.*}}, [[BB2:[A-Za-z0-9_]*]]
-; CHECK-O0: mov w{{.*}}, #16
-; CHECK-O0: malloc
-; CHECK-O0: mov [[ID:x[0-9]+]], x0
-; CHECK-O0: strb w{{.*}}, [{{.*}}[[ID]], #8]
+; CHECK-O0-AARCH64: str x21, [sp, [[SLOT:#[0-9]+]]]
+; CHECK-O0-AARCH64: b [[BB1:[A-Za-z0-9_]*]]
+; CHECK-O0-AARCH64: [[BB1]]:
+; CHECK-O0-AARCH64: ldr     x0, [sp, [[SLOT]]]
+; CHECK-O0-AARCH64: str     x0, [sp, [[SLOT2:#[0-9]+]]]
+; CHECK-O0-AARCH64: cbz {{.*}}, [[BB2:[A-Za-z0-9_]*]]
+; CHECK-O0-AARCH64: mov w{{.*}}, #16
+; CHECK-O0-AARCH64: malloc
+; CHECK-O0-AARCH64: mov [[ID:x[0-9]+]], x0
+; CHECK-O0-AARCH64: strb w{{.*}}, [{{.*}}[[ID]], #8]
 ; spill x0
-; CHECK-O0: str x0, [sp, [[SLOT2]]]
-; CHECK-O0:[[BB2]]:
-; CHECK-O0: ldr     x0, [sp, [[SLOT2]]]
-; CHECK-O0: fcmp
-; CHECK-O0: str     x0, [sp]
-; CHECK-O0: b.le [[BB1]]
+; CHECK-O0-AARCH64: str x0, [sp, [[SLOT2]]]
+; CHECK-O0-AARCH64:[[BB2]]:
+; CHECK-O0-AARCH64: ldr     x0, [sp, [[SLOT2]]]
+; CHECK-O0-AARCH64: fcmp
+; CHECK-O0-AARCH64: str     x0, [sp]
+; CHECK-O0-AARCH64: b.le [[BB1]]
 ; reload from stack
-; CHECK-O0: ldr [[ID3:x[0-9]+]], [sp]
-; CHECK-O0: mov x21, [[ID3]]
-; CHECK-O0: ret
+; CHECK-O0-AARCH64: ldr [[ID3:x[0-9]+]], [sp]
+; CHECK-O0-AARCH64: mov x21, [[ID3]]
+; CHECK-O0-AARCH64: ret
+
+; CHECK-O0-ARM64_32-LABEL: foo_loop:
+; spill x21
+; CHECK-O0-ARM64_32: str x21, [sp, [[SLOT:#[0-9]+]]]
+; CHECK-O0-ARM64_32: b [[BB1:[A-Za-z0-9_]*]]
+; CHECK-O0-ARM64_32: [[BB1]]:
+; CHECK-O0-ARM64_32: ldr     x0, [sp, [[SLOT]]]
+; CHECK-O0-ARM64_32: str     x0, [sp, [[SLOT2:#[0-9]+]]]
+; CHECK-O0-ARM64_32: cbz {{.*}}, [[BB2:[A-Za-z0-9_]*]]
+; CHECK-O0-ARM64_32: mov w{{.*}}, #16
+; CHECK-O0-ARM64_32: malloc
+; CHECK-O0-ARM64_32: mov {{.*}}, x0
+; CHECK-O0-ARM64_32: strb w{{.*}},
+; CHECK-O0-ARM64_32:[[BB2]]:
+; CHECK-O0-ARM64_32: ldr     x0, [sp, [[SLOT2]]]
+; CHECK-O0-ARM64_32: fcmp
+; CHECK-O0-ARM64_32: str     x0, [sp[[OFFSET:.*]]]
+; CHECK-O0-ARM64_32: b.le [[BB1]]
+; reload from stack
+; CHECK-O0-ARM64_32: ldr [[ID3:x[0-9]+]], [sp[[OFFSET]]]
+; CHECK-O0-ARM64_32: mov x21, [[ID3]]
+; CHECK-O0-ARM64_32: ret
+
 entry:
   br label %bb_loop
 
@@ -261,7 +290,8 @@ define float @caller3(i8* %error_ref) {
 ; CHECK-APPLE: mov x21, xzr
 ; CHECK-APPLE: bl {{.*}}foo_sret
 ; CHECK-APPLE: mov x0, x21
-; CHECK-APPLE: cbnz x21
+; CHECK-APPLE-AARCH64: cbnz x21
+; CHECK-APPLE-ARM64_32: cbnz w0
 ; Access part of the error object and save it to error_ref
 ; CHECK-APPLE: ldrb [[CODE:w[0-9]+]], [x0, #8]
 ; CHECK-APPLE: strb [[CODE]], [{{.*}}[[ID]]]
@@ -273,7 +303,8 @@ define float @caller3(i8* %error_ref) {
 ; CHECK-O0: mov x21
 ; CHECK-O0: bl {{.*}}foo_sret
 ; CHECK-O0: mov [[ID2:x[0-9]+]], x21
-; CHECK-O0: cbnz x21
+; CHECK-O0-AARCH64: cbnz x21
+; CHECK-O0-ARM64_32: cmp x21, #0
 ; Access part of the error object and save it to error_ref
 ; reload from stack
 ; CHECK-O0: ldrb [[CODE:w[0-9]+]]
@@ -306,20 +337,22 @@ define float @foo_vararg(%swift_error** swifterror %error_ptr_ref, ...) {
 ; CHECK-APPLE-LABEL: foo_vararg:
 ; CHECK-APPLE: mov w0, #16
 ; CHECK-APPLE: malloc
-; CHECK-APPLE-DAG: mov [[ID:w[0-9]+]], #1
-; CHECK-APPLE-DAG: add [[ARGS:x[0-9]+]], [[TMP:x[0-9]+]], #16
-; CHECK-APPLE-DAG: strb [[ID]], [x0, #8]
 
 ; First vararg
-; CHECK-APPLE-DAG: ldr {{w[0-9]+}}, [{{.*}}[[TMP]], #16]
+; CHECK-APPLE-AARCH64: ldr {{w[0-9]+}}, [{{.*}}[[TMP:x[0-9]+]], #16]
+; CHECK-APPLE-AARCH64: mov [[ID:w[0-9]+]], #1
+; CHECK-APPLE-AARCH64: add [[ARGS:x[0-9]+]], [[TMP]], #16
+; CHECK-APPLE-AARCH64: strb [[ID]], [x0, #8]
 ; Second vararg
-; CHECK-APPLE-DAG: ldr {{w[0-9]+}}, [{{.*}}[[TMP]], #24]
-; CHECK-APPLE-DAG: add {{x[0-9]+}}, {{x[0-9]+}}, #16
+; CHECK-APPLE-AARCH64: ldr {{w[0-9]+}}, [{{.*}}[[TMP]], #24]
 ; Third vararg
-; CHECK-APPLE-DAG: ldr {{w[0-9]+}}, [{{.*}}[[TMP]], #32]
+; CHECK-APPLE-AARCH64: ldr {{w[0-9]+}}, [{{.*}}[[TMP]], #32]
+
+; CHECK-APPLE-ARM64_32: mov [[ID:w[0-9]+]], #1
+; CHECK-APPLE-ARM64_32: add [[ARGS:x[0-9]+]], [[TMP:x[0-9]+]], #16
+; CHECK-APPLE-ARM64_32: strb [[ID]], [x0, #8]
+
 
-; CHECK-APPLE: mov x21, x0
-; CHECK-APPLE-NOT: x21
 entry:
   %call = call i8* @malloc(i64 16)
   %call.0 = bitcast i8* %call to %swift_error*
@@ -347,18 +380,18 @@ entry:
 define float @caller4(i8* %error_ref) {
 ; CHECK-APPLE-LABEL: caller4:
 
-; CHECK-APPLE: mov [[ID:x[0-9]+]], x0
-; CHECK-APPLE: stp {{x[0-9]+}}, {{x[0-9]+}}, [sp, #8]
-; CHECK-APPLE: str {{x[0-9]+}}, [sp]
+; CHECK-APPLE-AARCH64: mov [[ID:x[0-9]+]], x0
+; CHECK-APPLE-AARCH64: stp {{x[0-9]+}}, {{x[0-9]+}}, [sp, #8]
+; CHECK-APPLE-AARCH64: str {{x[0-9]+}}, [sp]
 
-; CHECK-APPLE: mov x21, xzr
-; CHECK-APPLE: bl {{.*}}foo_vararg
-; CHECK-APPLE: mov x0, x21
-; CHECK-APPLE: cbnz x21
+; CHECK-APPLE-AARCH64: mov x21, xzr
+; CHECK-APPLE-AARCH64: bl {{.*}}foo_vararg
+; CHECK-APPLE-AARCH64: mov x0, x21
+; CHECK-APPLE-AARCH64: cbnz x21
 ; Access part of the error object and save it to error_ref
-; CHECK-APPLE: ldrb [[CODE:w[0-9]+]], [x0, #8]
-; CHECK-APPLE: strb [[CODE]], [{{.*}}[[ID]]]
-; CHECK-APPLE: bl {{.*}}free
+; CHECK-APPLE-AARCH64: ldrb [[CODE:w[0-9]+]], [x0, #8]
+; CHECK-APPLE-AARCH64: strb [[CODE]], [{{.*}}[[ID]]]
+; CHECK-APPLE-AARCH64: bl {{.*}}free
 entry:
   %error_ptr_ref = alloca swifterror %swift_error*
   store %swift_error* null, %swift_error** %error_ptr_ref
diff --git a/llvm/test/CodeGen/AArch64/swiftself.ll b/llvm/test/CodeGen/AArch64/swiftself.ll
index 063085636..a13fbb8d5 100644
--- a/llvm/test/CodeGen/AArch64/swiftself.ll
+++ b/llvm/test/CodeGen/AArch64/swiftself.ll
@@ -1,6 +1,7 @@
-; RUN: llc -verify-machineinstrs -mtriple=aarch64-apple-ios -o - %s | FileCheck --check-prefix=CHECK --check-prefix=OPT %s
+; RUN: llc -verify-machineinstrs -mtriple=aarch64-apple-ios -o - %s | FileCheck --check-prefix=CHECK --check-prefix=OPT --check-prefix=OPTAARCH64 %s
 ; RUN: llc -O0 -fast-isel -verify-machineinstrs -mtriple=aarch64-apple-ios -o - %s | FileCheck %s
-; RUN: llc -verify-machineinstrs -mtriple=aarch64-unknown-linux-gnu -o - %s | FileCheck --check-prefix=CHECK --check-prefix=OPT %s
+; RUN: llc -verify-machineinstrs -mtriple=aarch64-unknown-linux-gnu -o - %s | FileCheck --check-prefix=CHECK --check-prefix=OPT --check-prefix=OPTAARCH64 %s
+; RUN: llc -verify-machineinstrs -mtriple=arm64_32-apple-ios -o - %s | FileCheck --check-prefix=CHECK --check-prefix=OPT --check-prefix=OPTARM64_32 %s
 
 ; Parameter with swiftself should be allocated to x20.
 ; CHECK-LABEL: swiftself_param:
@@ -48,8 +49,9 @@ define void @swiftself_passthrough(i8* swiftself %addr0) {
 ; We can use a tail call if the callee swiftself is the same as the caller one.
 ; This should also work with fast-isel.
 ; CHECK-LABEL: swiftself_tail:
-; CHECK: b {{_?}}swiftself_param
-; CHECK-NOT: ret
+; OPTAARCH64: b {{_?}}swiftself_param
+; OPTAARCH64-NOT: ret
+; OPTARM64_32: bl {{_?}}swiftself_param
 define i8* @swiftself_tail(i8* swiftself %addr0) {
   call void asm sideeffect "", "~{x20}"()
   %res = tail call i8* @swiftself_param(i8* swiftself %addr0)
@@ -71,12 +73,19 @@ define i8* @swiftself_notail(i8* swiftself %addr0, i8* %addr1) nounwind {
 ; we normally would. We marked the first parameter with swiftself which means it
 ; will no longer be passed in x0.
 declare swiftcc i8* @thisreturn_attribute(i8* returned swiftself)
-; OPT-LABEL: swiftself_nothisreturn:
-; OPT-DAG: ldr  x20, [x20]
-; OPT-DAG: mov [[CSREG:x[1-9].*]], x8
-; OPT: bl {{_?}}thisreturn_attribute
-; OPT: str x0, {{\[}}[[CSREG]]
-; OPT: ret
+; OPTAARCH64-LABEL: swiftself_nothisreturn:
+; OPTAARCH64-DAG: ldr  x20, [x20]
+; OPTAARCH64-DAG: mov [[CSREG:x[1-9].*]], x8
+; OPTAARCH64: bl {{_?}}thisreturn_attribute
+; OPTAARCH64: str x0, {{\[}}[[CSREG]]
+; OPTAARCH64: ret
+
+; OPTARM64_32-LABEL: swiftself_nothisreturn:
+; OPTARM64_32-DAG: ldr  w20, [x20]
+; OPTARM64_32-DAG: mov [[CSREG:x[1-9].*]], x8
+; OPTARM64_32: bl {{_?}}thisreturn_attribute
+; OPTARM64_32: str w0, {{\[}}[[CSREG]]
+; OPTARM64_32: ret
 define hidden swiftcc void @swiftself_nothisreturn(i8** noalias nocapture sret, i8** noalias nocapture readonly swiftself) {
 entry:
   %2 = load i8*, i8** %1, align 8
diff --git a/llvm/test/CodeGen/AArch64/tagged-globals.ll b/llvm/test/CodeGen/AArch64/tagged-globals.ll
new file mode 100644
index 000000000..b0cf88258
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/tagged-globals.ll
@@ -0,0 +1,32 @@
+; RUN: llc < %s | FileCheck %s
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64-unknown-linux-android"
+
+@global = external hidden global i32
+declare void @func()
+
+define i32* @global_addr() #0 {
+  ; CHECK: global_addr:
+  ; CHECK: adrp x0, :pg_hi21_nc:global
+  ; CHECK: movk x0, #:prel_g3:global+4294967296
+  ; CHECK: add x0, x0, :lo12:global
+  ret i32* @global
+}
+
+define i32 @global_load() #0 {
+  ; CHECK: global_load:
+  ; CHECK: adrp x8, :pg_hi21_nc:global
+  ; CHECK: ldr w0, [x8, :lo12:global]
+  %load = load i32, i32* @global
+  ret i32 %load
+}
+
+define void ()* @func_addr() #0 {
+  ; CHECK: func_addr:
+  ; CHECK: adrp x0, func
+  ; CHECK: add x0, x0, :lo12:func
+  ret void ()* @func
+}
+
+attributes #0 = { "target-features"="+tagged-globals" }
diff --git a/llvm/test/CodeGen/AArch64/tail-call.ll b/llvm/test/CodeGen/AArch64/tail-call.ll
index ab63413bd..39a9a7118 100644
--- a/llvm/test/CodeGen/AArch64/tail-call.ll
+++ b/llvm/test/CodeGen/AArch64/tail-call.ll
@@ -1,103 +1,114 @@
-; RUN: llc -verify-machineinstrs < %s -mtriple=aarch64-none-linux-gnu -tailcallopt | FileCheck %s
+; RUN: llc -verify-machineinstrs < %s -mtriple=aarch64-none-linux-gnu -tailcallopt | FileCheck %s --check-prefixes=SDAG,COMMON
+; RUN: llc -global-isel -global-isel-abort=1 -verify-machineinstrs < %s -mtriple=aarch64-none-linux-gnu -tailcallopt | FileCheck %s --check-prefixes=GISEL,COMMON
 
 declare fastcc void @callee_stack0()
-declare fastcc void @callee_stack8([8 x i32], i64)
-declare fastcc void @callee_stack16([8 x i32], i64, i64)
+declare fastcc void @callee_stack8([8 x i64], i64)
+declare fastcc void @callee_stack16([8 x i64], i64, i64)
 declare extern_weak fastcc void @callee_weak()
 
 define fastcc void @caller_to0_from0() nounwind {
-; CHECK-LABEL: caller_to0_from0:
-; CHECK-NEXT: // %bb.
+; COMMON-LABEL: caller_to0_from0:
+; COMMON-NEXT: // %bb.
 
   tail call fastcc void @callee_stack0()
   ret void
 
-; CHECK-NEXT: b callee_stack0
+; COMMON-NEXT: b callee_stack0
 }
 
-define fastcc void @caller_to0_from8([8 x i32], i64) {
-; CHECK-LABEL: caller_to0_from8:
+define fastcc void @caller_to0_from8([8 x i64], i64) {
+; COMMON-LABEL: caller_to0_from8:
 
   tail call fastcc void @callee_stack0()
   ret void
 
-; CHECK: add sp, sp, #16
-; CHECK-NEXT: b callee_stack0
+; COMMON: add sp, sp, #16
+; COMMON-NEXT: b callee_stack0
 }
 
 define fastcc void @caller_to8_from0() {
-; CHECK-LABEL: caller_to8_from0:
-; CHECK: sub sp, sp, #32
+; COMMON-LABEL: caller_to8_from0:
+; COMMON: sub sp, sp, #32
 
 ; Key point is that the "42" should go #16 below incoming stack
 ; pointer (we didn't have arg space to reuse).
-  tail call fastcc void @callee_stack8([8 x i32] undef, i64 42)
+  tail call fastcc void @callee_stack8([8 x i64] undef, i64 42)
   ret void
 
-; CHECK: str {{x[0-9]+}}, [sp, #16]!
-; CHECK-NEXT: b callee_stack8
+; COMMON: str {{x[0-9]+}}, [sp, #16]!
+; COMMON-NEXT: b callee_stack8
 }
 
-define fastcc void @caller_to8_from8([8 x i32], i64 %a) {
-; CHECK-LABEL: caller_to8_from8:
-; CHECK: sub sp, sp, #16
+define fastcc void @caller_to8_from8([8 x i64], i64 %a) {
+; COMMON-LABEL: caller_to8_from8:
+; COMMON: sub sp, sp, #16
 
 ; Key point is that the "%a" should go where at SP on entry.
-  tail call fastcc void @callee_stack8([8 x i32] undef, i64 42)
+  tail call fastcc void @callee_stack8([8 x i64] undef, i64 42)
   ret void
 
-; CHECK: str {{x[0-9]+}}, [sp, #16]!
-; CHECK-NEXT: b callee_stack8
+; COMMON: str {{x[0-9]+}}, [sp, #16]!
+; COMMON-NEXT: b callee_stack8
 }
 
-define fastcc void @caller_to16_from8([8 x i32], i64 %a) {
-; CHECK-LABEL: caller_to16_from8:
-; CHECK: sub sp, sp, #16
+define fastcc void @caller_to16_from8([8 x i64], i64 %a) {
+; COMMON-LABEL: caller_to16_from8:
+; COMMON: sub sp, sp, #16
 
 ; Important point is that the call reuses the "dead" argument space
 ; above %a on the stack. If it tries to go below incoming-SP then the
 ; callee will not deallocate the space, even in fastcc.
-  tail call fastcc void @callee_stack16([8 x i32] undef, i64 42, i64 2)
+  tail call fastcc void @callee_stack16([8 x i64] undef, i64 42, i64 2)
 
-; CHECK: stp {{x[0-9]+}}, {{x[0-9]+}}, [sp, #16]!
-; CHECK-NEXT: b callee_stack16
+; COMMON: stp {{x[0-9]+}}, {{x[0-9]+}}, [sp, #16]!
+; COMMON-NEXT: b callee_stack16
   ret void
 }
 
 
-define fastcc void @caller_to8_from24([8 x i32], i64 %a, i64 %b, i64 %c) {
-; CHECK-LABEL: caller_to8_from24:
-; CHECK: sub sp, sp, #16
+define fastcc void @caller_to8_from24([8 x i64], i64 %a, i64 %b, i64 %c) {
+; COMMON-LABEL: caller_to8_from24:
+; COMMON: sub sp, sp, #16
 
 ; Key point is that the "%a" should go where at #16 above SP on entry.
-  tail call fastcc void @callee_stack8([8 x i32] undef, i64 42)
+  tail call fastcc void @callee_stack8([8 x i64] undef, i64 42)
   ret void
 
-; CHECK: str {{x[0-9]+}}, [sp, #32]!
-; CHECK-NEXT: b callee_stack8
+; COMMON: str {{x[0-9]+}}, [sp, #32]!
+; COMMON-NEXT: b callee_stack8
 }
 
 
-define fastcc void @caller_to16_from16([8 x i32], i64 %a, i64 %b) {
-; CHECK-LABEL: caller_to16_from16:
-; CHECK: sub sp, sp, #16
+define fastcc void @caller_to16_from16([8 x i64], i64 %a, i64 %b) {
+; COMMON-LABEL: caller_to16_from16:
+; COMMON: sub sp, sp, #16
 
 ; Here we want to make sure that both loads happen before the stores:
 ; otherwise either %a or %b will be wrongly clobbered.
-  tail call fastcc void @callee_stack16([8 x i32] undef, i64 %b, i64 %a)
+  tail call fastcc void @callee_stack16([8 x i64] undef, i64 %b, i64 %a)
   ret void
 
-; CHECK: ldp {{x[0-9]+}}, {{x[0-9]+}}, [sp, #16]
-; CHECK: stp {{x[0-9]+}}, {{x[0-9]+}}, [sp, #16]!
-; CHECK-NEXT: b callee_stack16
+; COMMON: ldp {{x[0-9]+}}, {{x[0-9]+}}, [sp, #16]
+; COMMON: stp {{x[0-9]+}}, {{x[0-9]+}}, [sp, #16]!
+; COMMON-NEXT: b callee_stack16
 }
 
+define fastcc void @disable_tail_calls() nounwind "disable-tail-calls"="true" {
+; COMMON-LABEL: disable_tail_calls:
+; COMMON-NEXT: // %bb.
+
+  tail call fastcc void @callee_stack0()
+  ret void
+
+; COMMON: bl callee_stack0
+; COMMON: ret
+}
 
 ; Weakly-referenced extern functions cannot be tail-called, as AAELF does
 ; not define the behaviour of branch instructions to undefined weak symbols.
 define fastcc void @caller_weak() {
-; CHECK-LABEL: caller_weak:
-; CHECK: bl callee_weak
+; COMMON-LABEL: caller_weak:
+; COMMON: bl callee_weak
   tail call void @callee_weak()
   ret void
 }
@@ -105,10 +116,17 @@ define fastcc void @caller_weak() {
 declare { [2 x float] } @get_vec2()
 
 define { [3 x float] } @test_add_elem() {
-; CHECK-LABEL: test_add_elem:
-; CHECK: bl get_vec2
-; CHECK: fmov s2, #1.0
-; CHECK: ret
+; SDAG-LABEL: test_add_elem:
+; SDAG: bl get_vec2
+; SDAG: fmov s2, #1.0
+; SDAG: ret
+; GISEL-LABEL: test_add_elem:
+; GISEL: fmov	s8, #1.00000000
+; GISEL: bl get_vec2
+; GISEL: ldr	x30, [sp, #8]
+; GISEL: mov	v2.16b, v8.16b
+; GISEL: ldr	d8, [sp], #16
+; GISEL: ret
 
   %call = tail call { [2 x float] } @get_vec2()
   %arr = extractvalue { [2 x float] } %call, 0
@@ -123,11 +141,11 @@ define { [3 x float] } @test_add_elem() {
 
 declare double @get_double()
 define { double, [2 x double] } @test_mismatched_insert() {
-; CHECK-LABEL: test_mismatched_insert:
-; CHECK: bl get_double
-; CHECK: bl get_double
-; CHECK: bl get_double
-; CHECK: ret
+; COMMON-LABEL: test_mismatched_insert:
+; COMMON: bl get_double
+; COMMON: bl get_double
+; COMMON: bl get_double
+; COMMON: ret
 
   %val0 = call double @get_double()
   %val1 = call double @get_double()
diff --git a/llvm/test/CodeGen/AArch64/tailcall-bitcast-memcpy.ll b/llvm/test/CodeGen/AArch64/tailcall-bitcast-memcpy.ll
new file mode 100644
index 000000000..88a07498a
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/tailcall-bitcast-memcpy.ll
@@ -0,0 +1,18 @@
+;RUN: llc %s -o - -verify-machineinstrs | FileCheck %s
+target triple = "aarch64-arm-none-eabi"
+
+;CHECK-LABEL: @wmemcpy
+;CHECK: lsl
+;CHECK-NOT: bl
+;CHECK-NOT: mov
+;CHECK-NOT: ldp
+;CHECK-NEXT: b memcpy
+define dso_local i32* @wmemcpy(i32* returned, i32* nocapture readonly, i64) local_unnamed_addr {
+  %4 = bitcast i32* %0 to i8*
+  %5 = bitcast i32* %1 to i8*
+  %6 = shl i64 %2, 2
+  tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %4, i8* align 4 %5, i64 %6, i1 false)
+  ret i32* %0
+}
+
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture writeonly, i8* nocapture readonly, i64, i1)
diff --git a/llvm/test/CodeGen/AArch64/tailcall-ccmismatch.ll b/llvm/test/CodeGen/AArch64/tailcall-ccmismatch.ll
index ab96e609d..64a5fad59 100644
--- a/llvm/test/CodeGen/AArch64/tailcall-ccmismatch.ll
+++ b/llvm/test/CodeGen/AArch64/tailcall-ccmismatch.ll
@@ -1,4 +1,5 @@
 ; RUN: llc -o - %s | FileCheck %s
+; RUN: llc -global-isel -verify-machineinstrs -o - %s | FileCheck %s
 target triple="aarch64--"
 
 declare void @somefunc()
diff --git a/llvm/test/CodeGen/AArch64/tailcall-mem-intrinsics.ll b/llvm/test/CodeGen/AArch64/tailcall-mem-intrinsics.ll
index c780d15b5..8fd302fe5 100644
--- a/llvm/test/CodeGen/AArch64/tailcall-mem-intrinsics.ll
+++ b/llvm/test/CodeGen/AArch64/tailcall-mem-intrinsics.ll
@@ -1,4 +1,5 @@
 ; RUN: llc -mtriple=aarch64-unknown-unknown < %s | FileCheck %s
+; RUN: llc -global-isel-abort=1 -verify-machineinstrs -mtriple=aarch64-unknown-unknown -global-isel < %s | FileCheck %s
 
 ; CHECK-LABEL: tail_memcpy:
 ; CHECK: b memcpy
diff --git a/llvm/test/CodeGen/AArch64/tailcall-string-rvo.ll b/llvm/test/CodeGen/AArch64/tailcall-string-rvo.ll
index d9d2180b5..ac9ce4d10 100644
--- a/llvm/test/CodeGen/AArch64/tailcall-string-rvo.ll
+++ b/llvm/test/CodeGen/AArch64/tailcall-string-rvo.ll
@@ -1,4 +1,5 @@
 ; RUN: llc -relocation-model=static -verify-machineinstrs -O2 < %s | FileCheck %s
+; RUN: llc -relocation-model=static -verify-machineinstrs -global-isel -O2 < %s | FileCheck %s
 
 ; The call to function TestBar should be a tail call, when in C++ the string
 ; `ret` is RVO returned.
diff --git a/llvm/test/CodeGen/AArch64/tailcall_misched_graph.ll b/llvm/test/CodeGen/AArch64/tailcall_misched_graph.ll
index 8b7f9796e..776ec3977 100644
--- a/llvm/test/CodeGen/AArch64/tailcall_misched_graph.ll
+++ b/llvm/test/CodeGen/AArch64/tailcall_misched_graph.ll
@@ -1,4 +1,5 @@
-; RUN: llc -mcpu=cyclone -debug-only=machine-scheduler < %s 2>&1 | FileCheck %s
+; RUN: llc -mcpu=cyclone -debug-only=machine-scheduler < %s 2>&1 | FileCheck %s --check-prefixes=COMMON,SDAG
+; RUN: llc -mcpu=cyclone -global-isel -debug-only=machine-scheduler < %s 2>&1 | FileCheck %s --check-prefixes=COMMON,GISEL
 
 ; REQUIRES: asserts
 
@@ -20,25 +21,34 @@ declare void @callee2(i8*, i8*, i8*, i8*, i8*,
 ; PR23459 has a test case that we where miscompiling because of this at the
 ; time.
 
-; CHECK: Frame Objects
-; CHECK:  fi#-4: {{.*}} fixed, at location [SP+8]
-; CHECK:  fi#-3: {{.*}} fixed, at location [SP]
-; CHECK:  fi#-2: {{.*}} fixed, at location [SP+8]
-; CHECK:  fi#-1: {{.*}} fixed, at location [SP]
-
-; CHECK:  [[VRA:%.*]]:gpr64 = LDRXui %fixed-stack.3
-; CHECK:  [[VRB:%.*]]:gpr64 = LDRXui %fixed-stack.2
-; CHECK:  STRXui %{{.*}}, %fixed-stack.0
-; CHECK:  STRXui [[VRB]]{{[^,]*}}, %fixed-stack.1
+; COMMON: Frame Objects
+; COMMON:  fi#-4: {{.*}} fixed, at location [SP+8]
+; COMMON:  fi#-3: {{.*}} fixed, at location [SP]
+; COMMON:  fi#-2: {{.*}} fixed, at location [SP+8]
+; COMMON:  fi#-1: {{.*}} fixed, at location [SP]
+
+; The order that these appear in differes in GISel than SDAG, but the
+; dependency relationship still holds.
+; COMMON:  [[VRA:%.*]]:gpr64 = LDRXui %fixed-stack.3
+; COMMON:  [[VRB:%.*]]:gpr64 = LDRXui %fixed-stack.2
+; SDAG:  STRXui %{{.*}}, %fixed-stack.0
+; SDAG:  STRXui [[VRB]]{{[^,]*}}, %fixed-stack.1
+; GISEL:  STRXui [[VRB]]{{[^,]*}}, %fixed-stack.1
+; GISEL:  STRXui %{{.*}}, %fixed-stack.0
 
 ; Make sure that there is an dependence edge between fi#-2 and fi#-4.
 ; Without this edge the scheduler would be free to move the store accross the load.
 
-; CHECK: SU({{.*}}):   [[VRB]]:gpr64 = LDRXui %fixed-stack.2
-; CHECK-NOT: SU
-; CHECK:  Successors:
-; CHECK:   SU([[DEPSTOREB:.*]]): Ord  Latency=0
-; CHECK:   SU([[DEPSTOREA:.*]]): Ord  Latency=0
+; COMMON: {{^SU(.*)}}:   [[VRB]]:gpr64 = LDRXui %fixed-stack.2
+; COMMON-NOT: {{^SU(.*)}}:
+; COMMON:  Successors:
+; COMMON:   SU([[DEPSTOREB:.*]]): Ord  Latency=0
+; COMMON:   SU([[DEPSTOREA:.*]]): Ord  Latency=0
+
+; GlobalISel outputs DEPSTOREB before DEPSTOREA, but the dependency relationship
+; still holds.
+; SDAG: SU([[DEPSTOREA]]):   STRXui %{{.*}}, %fixed-stack.0
+; SDAG: SU([[DEPSTOREB]]):   STRXui %{{.*}}, %fixed-stack.1
 
-; CHECK: SU([[DEPSTOREA]]):   STRXui %{{.*}}, %fixed-stack.0
-; CHECK: SU([[DEPSTOREB]]):   STRXui %{{.*}}, %fixed-stack.1
+; GISEL: SU([[DEPSTOREB]]):   STRXui %{{.*}}, %fixed-stack.0
+; GISEL: SU([[DEPSTOREA]]):   STRXui %{{.*}}, %fixed-stack.1
diff --git a/llvm/test/CodeGen/AArch64/tbz-tbnz.ll b/llvm/test/CodeGen/AArch64/tbz-tbnz.ll
index 5d6b3903f..5018ce0b3 100644
--- a/llvm/test/CodeGen/AArch64/tbz-tbnz.ll
+++ b/llvm/test/CodeGen/AArch64/tbz-tbnz.ll
@@ -153,7 +153,7 @@ if.then2:
   br i1 %tst3, label %if.then3, label %if.end
 
 ; CHECK: tst x0, x1, lsl #63
-; CHECK: b.ge
+; CHECK: b.lt
 
 if.then3:
   %shifted_op2 = shl i64 %val2, 62
diff --git a/llvm/test/CodeGen/AArch64/tme.ll b/llvm/test/CodeGen/AArch64/tme.ll
new file mode 100644
index 000000000..887728029
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/tme.ll
@@ -0,0 +1,44 @@
+; RUN: llc %s -verify-machineinstrs -o - | FileCheck %s
+
+target triple = "aarch64-unknown-unknown-eabi"
+
+define i64 @test_tstart() #0 {
+  %r = tail call i64 @llvm.aarch64.tstart()
+  ret i64 %r
+}
+declare i64 @llvm.aarch64.tstart() #1
+; CHECK-LABEL: test_tstart
+; CHECK: tstart x
+
+define i64 @test_ttest() #0 {
+  %r = tail call i64 @llvm.aarch64.ttest()
+  ret i64 %r
+}
+declare i64 @llvm.aarch64.ttest() #1
+; CHECK-LABEL: test_ttest
+; CHECK: ttest x
+
+define void @test_tcommit() #0 {
+  tail call void @llvm.aarch64.tcommit()
+  ret void
+}
+declare void @llvm.aarch64.tcommit() #1
+; CHECK-LABEL: test_tcommit
+; CHECK: tcommit
+
+define void @test_tcancel() #0 {
+  tail call void @llvm.aarch64.tcancel(i64 0) #1
+  tail call void @llvm.aarch64.tcancel(i64 1) #1
+  tail call void @llvm.aarch64.tcancel(i64 65534) #1
+  tail call void @llvm.aarch64.tcancel(i64 65535) #1
+  ret void
+}
+declare void @llvm.aarch64.tcancel(i64 immarg) #1
+; CHECK-LABEL: test_tcancel
+; CHECK: tcancel #0
+; CHECK: tcancel #0x1
+; CHECK: tcancel #0xfffe
+; CHECK: tcancel #0xffff
+
+attributes #0 = { "target-features"="+tme" }
+attributes #1 = { nounwind }
diff --git a/llvm/test/CodeGen/AArch64/uadd_sat.ll b/llvm/test/CodeGen/AArch64/uadd_sat.ll
index da3eda405..120f101d1 100644
--- a/llvm/test/CodeGen/AArch64/uadd_sat.ll
+++ b/llvm/test/CodeGen/AArch64/uadd_sat.ll
@@ -1,9 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=aarch64-- | FileCheck %s
 
-declare  i4  @llvm.uadd.sat.i4   (i4,  i4)
-declare  i32 @llvm.uadd.sat.i32  (i32, i32)
-declare  i64 @llvm.uadd.sat.i64  (i64, i64)
+declare i4 @llvm.uadd.sat.i4(i4, i4)
+declare i8 @llvm.uadd.sat.i8(i8, i8)
+declare i16 @llvm.uadd.sat.i16(i16, i16)
+declare i32 @llvm.uadd.sat.i32(i32, i32)
+declare i64 @llvm.uadd.sat.i64(i64, i64)
 
 define i32 @func(i32 %x, i32 %y) nounwind {
 ; CHECK-LABEL: func:
@@ -25,13 +27,41 @@ define i64 @func2(i64 %x, i64 %y) nounwind {
   ret i64 %tmp;
 }
 
+define i16 @func16(i16 %x, i16 %y) nounwind {
+; CHECK-LABEL: func16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    and w8, w0, #0xffff
+; CHECK-NEXT:    add w8, w8, w1, uxth
+; CHECK-NEXT:    mov w9, #65535
+; CHECK-NEXT:    cmp w8, w9
+; CHECK-NEXT:    csel w0, w8, w9, lo
+; CHECK-NEXT:    ret
+  %tmp = call i16 @llvm.uadd.sat.i16(i16 %x, i16 %y);
+  ret i16 %tmp;
+}
+
+define i8 @func8(i8 %x, i8 %y) nounwind {
+; CHECK-LABEL: func8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    and w8, w0, #0xff
+; CHECK-NEXT:    add w8, w8, w1, uxtb
+; CHECK-NEXT:    cmp w8, #255 // =255
+; CHECK-NEXT:    mov w9, #255
+; CHECK-NEXT:    csel w0, w8, w9, lo
+; CHECK-NEXT:    ret
+  %tmp = call i8 @llvm.uadd.sat.i8(i8 %x, i8 %y);
+  ret i8 %tmp;
+}
+
 define i4 @func3(i4 %x, i4 %y) nounwind {
 ; CHECK-LABEL: func3:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    lsl w8, w0, #28
-; CHECK-NEXT:    adds w8, w8, w1, lsl #28
-; CHECK-NEXT:    csinv w8, w8, wzr, lo
-; CHECK-NEXT:    lsr w0, w8, #28
+; CHECK-NEXT:    and w8, w1, #0xf
+; CHECK-NEXT:    and w9, w0, #0xf
+; CHECK-NEXT:    add w8, w9, w8
+; CHECK-NEXT:    cmp w8, #15 // =15
+; CHECK-NEXT:    mov w9, #15
+; CHECK-NEXT:    csel w0, w8, w9, lo
 ; CHECK-NEXT:    ret
   %tmp = call i4 @llvm.uadd.sat.i4(i4 %x, i4 %y);
   ret i4 %tmp;
diff --git a/llvm/test/CodeGen/AArch64/uadd_sat_plus.ll b/llvm/test/CodeGen/AArch64/uadd_sat_plus.ll
new file mode 100644
index 000000000..277f40d37
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/uadd_sat_plus.ll
@@ -0,0 +1,77 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=aarch64-- | FileCheck %s
+
+declare i4 @llvm.uadd.sat.i4(i4, i4)
+declare i8 @llvm.uadd.sat.i8(i8, i8)
+declare i16 @llvm.uadd.sat.i16(i16, i16)
+declare i32 @llvm.uadd.sat.i32(i32, i32)
+declare i64 @llvm.uadd.sat.i64(i64, i64)
+
+define i32 @func32(i32 %x, i32 %y, i32 %z) nounwind {
+; CHECK-LABEL: func32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mul w8, w1, w2
+; CHECK-NEXT:    adds w8, w0, w8
+; CHECK-NEXT:    csinv w0, w8, wzr, lo
+; CHECK-NEXT:    ret
+  %a = mul i32 %y, %z
+  %tmp = call i32 @llvm.uadd.sat.i32(i32 %x, i32 %a)
+  ret i32 %tmp
+}
+
+define i64 @func64(i64 %x, i64 %y, i64 %z) nounwind {
+; CHECK-LABEL: func64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adds x8, x0, x2
+; CHECK-NEXT:    csinv x0, x8, xzr, lo
+; CHECK-NEXT:    ret
+  %a = mul i64 %y, %z
+  %tmp = call i64 @llvm.uadd.sat.i64(i64 %x, i64 %z)
+  ret i64 %tmp
+}
+
+define i16 @func16(i16 %x, i16 %y, i16 %z) nounwind {
+; CHECK-LABEL: func16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    and w8, w0, #0xffff
+; CHECK-NEXT:    mul w9, w1, w2
+; CHECK-NEXT:    add w8, w8, w9, uxth
+; CHECK-NEXT:    mov w9, #65535
+; CHECK-NEXT:    cmp w8, w9
+; CHECK-NEXT:    csel w0, w8, w9, lo
+; CHECK-NEXT:    ret
+  %a = mul i16 %y, %z
+  %tmp = call i16 @llvm.uadd.sat.i16(i16 %x, i16 %a)
+  ret i16 %tmp
+}
+
+define i8 @func8(i8 %x, i8 %y, i8 %z) nounwind {
+; CHECK-LABEL: func8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    and w8, w0, #0xff
+; CHECK-NEXT:    mul w9, w1, w2
+; CHECK-NEXT:    add w8, w8, w9, uxtb
+; CHECK-NEXT:    cmp w8, #255 // =255
+; CHECK-NEXT:    mov w9, #255
+; CHECK-NEXT:    csel w0, w8, w9, lo
+; CHECK-NEXT:    ret
+  %a = mul i8 %y, %z
+  %tmp = call i8 @llvm.uadd.sat.i8(i8 %x, i8 %a)
+  ret i8 %tmp
+}
+
+define i4 @func4(i4 %x, i4 %y, i4 %z) nounwind {
+; CHECK-LABEL: func4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mul w9, w1, w2
+; CHECK-NEXT:    and w8, w0, #0xf
+; CHECK-NEXT:    and w9, w9, #0xf
+; CHECK-NEXT:    add w8, w8, w9
+; CHECK-NEXT:    cmp w8, #15 // =15
+; CHECK-NEXT:    mov w9, #15
+; CHECK-NEXT:    csel w0, w8, w9, lo
+; CHECK-NEXT:    ret
+  %a = mul i4 %y, %z
+  %tmp = call i4 @llvm.uadd.sat.i4(i4 %x, i4 %a)
+  ret i4 %tmp
+}
diff --git a/llvm/test/CodeGen/AArch64/uadd_sat_vec.ll b/llvm/test/CodeGen/AArch64/uadd_sat_vec.ll
index 55b42e790..40bbac2c0 100644
--- a/llvm/test/CodeGen/AArch64/uadd_sat_vec.ll
+++ b/llvm/test/CodeGen/AArch64/uadd_sat_vec.ll
@@ -35,9 +35,7 @@ declare <2 x i128> @llvm.uadd.sat.v2i128(<2 x i128>, <2 x i128>)
 define <16 x i8> @v16i8(<16 x i8> %x, <16 x i8> %y) nounwind {
 ; CHECK-LABEL: v16i8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mvn v2.16b, v1.16b
-; CHECK-NEXT:    umin v0.16b, v0.16b, v2.16b
-; CHECK-NEXT:    add v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    uqadd v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
   %z = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> %x, <16 x i8> %y)
   ret <16 x i8> %z
@@ -46,12 +44,8 @@ define <16 x i8> @v16i8(<16 x i8> %x, <16 x i8> %y) nounwind {
 define <32 x i8> @v32i8(<32 x i8> %x, <32 x i8> %y) nounwind {
 ; CHECK-LABEL: v32i8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mvn v4.16b, v2.16b
-; CHECK-NEXT:    mvn v5.16b, v3.16b
-; CHECK-NEXT:    umin v0.16b, v0.16b, v4.16b
-; CHECK-NEXT:    umin v1.16b, v1.16b, v5.16b
-; CHECK-NEXT:    add v0.16b, v0.16b, v2.16b
-; CHECK-NEXT:    add v1.16b, v1.16b, v3.16b
+; CHECK-NEXT:    uqadd v0.16b, v0.16b, v2.16b
+; CHECK-NEXT:    uqadd v1.16b, v1.16b, v3.16b
 ; CHECK-NEXT:    ret
   %z = call <32 x i8> @llvm.uadd.sat.v32i8(<32 x i8> %x, <32 x i8> %y)
   ret <32 x i8> %z
@@ -60,18 +54,10 @@ define <32 x i8> @v32i8(<32 x i8> %x, <32 x i8> %y) nounwind {
 define <64 x i8> @v64i8(<64 x i8> %x, <64 x i8> %y) nounwind {
 ; CHECK-LABEL: v64i8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mvn v16.16b, v4.16b
-; CHECK-NEXT:    umin v0.16b, v0.16b, v16.16b
-; CHECK-NEXT:    mvn v16.16b, v5.16b
-; CHECK-NEXT:    umin v1.16b, v1.16b, v16.16b
-; CHECK-NEXT:    mvn v16.16b, v6.16b
-; CHECK-NEXT:    umin v2.16b, v2.16b, v16.16b
-; CHECK-NEXT:    mvn v16.16b, v7.16b
-; CHECK-NEXT:    umin v3.16b, v3.16b, v16.16b
-; CHECK-NEXT:    add v0.16b, v0.16b, v4.16b
-; CHECK-NEXT:    add v1.16b, v1.16b, v5.16b
-; CHECK-NEXT:    add v2.16b, v2.16b, v6.16b
-; CHECK-NEXT:    add v3.16b, v3.16b, v7.16b
+; CHECK-NEXT:    uqadd v0.16b, v0.16b, v4.16b
+; CHECK-NEXT:    uqadd v1.16b, v1.16b, v5.16b
+; CHECK-NEXT:    uqadd v2.16b, v2.16b, v6.16b
+; CHECK-NEXT:    uqadd v3.16b, v3.16b, v7.16b
 ; CHECK-NEXT:    ret
   %z = call <64 x i8> @llvm.uadd.sat.v64i8(<64 x i8> %x, <64 x i8> %y)
   ret <64 x i8> %z
@@ -80,9 +66,7 @@ define <64 x i8> @v64i8(<64 x i8> %x, <64 x i8> %y) nounwind {
 define <8 x i16> @v8i16(<8 x i16> %x, <8 x i16> %y) nounwind {
 ; CHECK-LABEL: v8i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mvn v2.16b, v1.16b
-; CHECK-NEXT:    umin v0.8h, v0.8h, v2.8h
-; CHECK-NEXT:    add v0.8h, v0.8h, v1.8h
+; CHECK-NEXT:    uqadd v0.8h, v0.8h, v1.8h
 ; CHECK-NEXT:    ret
   %z = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> %x, <8 x i16> %y)
   ret <8 x i16> %z
@@ -91,12 +75,8 @@ define <8 x i16> @v8i16(<8 x i16> %x, <8 x i16> %y) nounwind {
 define <16 x i16> @v16i16(<16 x i16> %x, <16 x i16> %y) nounwind {
 ; CHECK-LABEL: v16i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mvn v4.16b, v2.16b
-; CHECK-NEXT:    mvn v5.16b, v3.16b
-; CHECK-NEXT:    umin v0.8h, v0.8h, v4.8h
-; CHECK-NEXT:    umin v1.8h, v1.8h, v5.8h
-; CHECK-NEXT:    add v0.8h, v0.8h, v2.8h
-; CHECK-NEXT:    add v1.8h, v1.8h, v3.8h
+; CHECK-NEXT:    uqadd v0.8h, v0.8h, v2.8h
+; CHECK-NEXT:    uqadd v1.8h, v1.8h, v3.8h
 ; CHECK-NEXT:    ret
   %z = call <16 x i16> @llvm.uadd.sat.v16i16(<16 x i16> %x, <16 x i16> %y)
   ret <16 x i16> %z
@@ -105,18 +85,10 @@ define <16 x i16> @v16i16(<16 x i16> %x, <16 x i16> %y) nounwind {
 define <32 x i16> @v32i16(<32 x i16> %x, <32 x i16> %y) nounwind {
 ; CHECK-LABEL: v32i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mvn v16.16b, v4.16b
-; CHECK-NEXT:    umin v0.8h, v0.8h, v16.8h
-; CHECK-NEXT:    mvn v16.16b, v5.16b
-; CHECK-NEXT:    umin v1.8h, v1.8h, v16.8h
-; CHECK-NEXT:    mvn v16.16b, v6.16b
-; CHECK-NEXT:    umin v2.8h, v2.8h, v16.8h
-; CHECK-NEXT:    mvn v16.16b, v7.16b
-; CHECK-NEXT:    umin v3.8h, v3.8h, v16.8h
-; CHECK-NEXT:    add v0.8h, v0.8h, v4.8h
-; CHECK-NEXT:    add v1.8h, v1.8h, v5.8h
-; CHECK-NEXT:    add v2.8h, v2.8h, v6.8h
-; CHECK-NEXT:    add v3.8h, v3.8h, v7.8h
+; CHECK-NEXT:    uqadd v0.8h, v0.8h, v4.8h
+; CHECK-NEXT:    uqadd v1.8h, v1.8h, v5.8h
+; CHECK-NEXT:    uqadd v2.8h, v2.8h, v6.8h
+; CHECK-NEXT:    uqadd v3.8h, v3.8h, v7.8h
 ; CHECK-NEXT:    ret
   %z = call <32 x i16> @llvm.uadd.sat.v32i16(<32 x i16> %x, <32 x i16> %y)
   ret <32 x i16> %z
@@ -125,11 +97,9 @@ define <32 x i16> @v32i16(<32 x i16> %x, <32 x i16> %y) nounwind {
 define void @v8i8(<8 x i8>* %px, <8 x i8>* %py, <8 x i8>* %pz) nounwind {
 ; CHECK-LABEL: v8i8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldr d0, [x1]
-; CHECK-NEXT:    ldr d1, [x0]
-; CHECK-NEXT:    mvn v2.8b, v0.8b
-; CHECK-NEXT:    umin v1.8b, v1.8b, v2.8b
-; CHECK-NEXT:    add v0.8b, v1.8b, v0.8b
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    uqadd v0.8b, v0.8b, v1.8b
 ; CHECK-NEXT:    str d0, [x2]
 ; CHECK-NEXT:    ret
   %x = load <8 x i8>, <8 x i8>* %px
@@ -142,27 +112,25 @@ define void @v8i8(<8 x i8>* %px, <8 x i8>* %py, <8 x i8>* %pz) nounwind {
 define void @v4i8(<4 x i8>* %px, <4 x i8>* %py, <4 x i8>* %pz) nounwind {
 ; CHECK-LABEL: v4i8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldrb w9, [x1]
 ; CHECK-NEXT:    ldrb w8, [x0]
-; CHECK-NEXT:    ldrb w11, [x1, #1]
+; CHECK-NEXT:    ldrb w9, [x1]
 ; CHECK-NEXT:    ldrb w10, [x0, #1]
-; CHECK-NEXT:    fmov s1, w9
-; CHECK-NEXT:    ldrb w9, [x1, #2]
+; CHECK-NEXT:    ldrb w11, [x1, #1]
 ; CHECK-NEXT:    fmov s0, w8
+; CHECK-NEXT:    fmov s1, w9
 ; CHECK-NEXT:    ldrb w8, [x0, #2]
-; CHECK-NEXT:    mov v1.h[1], w11
-; CHECK-NEXT:    ldrb w11, [x1, #3]
+; CHECK-NEXT:    ldrb w9, [x1, #2]
 ; CHECK-NEXT:    mov v0.h[1], w10
+; CHECK-NEXT:    mov v1.h[1], w11
 ; CHECK-NEXT:    ldrb w10, [x0, #3]
-; CHECK-NEXT:    mov v1.h[2], w9
+; CHECK-NEXT:    ldrb w11, [x1, #3]
 ; CHECK-NEXT:    mov v0.h[2], w8
-; CHECK-NEXT:    mov v1.h[3], w11
+; CHECK-NEXT:    mov v1.h[2], w9
 ; CHECK-NEXT:    mov v0.h[3], w10
+; CHECK-NEXT:    mov v1.h[3], w11
 ; CHECK-NEXT:    shl v1.4h, v1.4h, #8
 ; CHECK-NEXT:    shl v0.4h, v0.4h, #8
-; CHECK-NEXT:    mvn v2.8b, v1.8b
-; CHECK-NEXT:    umin v0.4h, v0.4h, v2.4h
-; CHECK-NEXT:    add v0.4h, v0.4h, v1.4h
+; CHECK-NEXT:    uqadd v0.4h, v0.4h, v1.4h
 ; CHECK-NEXT:    ushr v0.4h, v0.4h, #8
 ; CHECK-NEXT:    xtn v0.8b, v0.8h
 ; CHECK-NEXT:    str s0, [x2]
@@ -177,19 +145,17 @@ define void @v4i8(<4 x i8>* %px, <4 x i8>* %py, <4 x i8>* %pz) nounwind {
 define void @v2i8(<2 x i8>* %px, <2 x i8>* %py, <2 x i8>* %pz) nounwind {
 ; CHECK-LABEL: v2i8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldrb w9, [x1]
 ; CHECK-NEXT:    ldrb w8, [x0]
-; CHECK-NEXT:    ldrb w11, [x1, #1]
+; CHECK-NEXT:    ldrb w9, [x1]
 ; CHECK-NEXT:    ldrb w10, [x0, #1]
-; CHECK-NEXT:    fmov s1, w9
+; CHECK-NEXT:    ldrb w11, [x1, #1]
 ; CHECK-NEXT:    fmov s0, w8
-; CHECK-NEXT:    mov v1.s[1], w11
+; CHECK-NEXT:    fmov s1, w9
 ; CHECK-NEXT:    mov v0.s[1], w10
+; CHECK-NEXT:    mov v1.s[1], w11
 ; CHECK-NEXT:    shl v1.2s, v1.2s, #24
 ; CHECK-NEXT:    shl v0.2s, v0.2s, #24
-; CHECK-NEXT:    mvn v2.8b, v1.8b
-; CHECK-NEXT:    umin v0.2s, v0.2s, v2.2s
-; CHECK-NEXT:    add v0.2s, v0.2s, v1.2s
+; CHECK-NEXT:    uqadd v0.2s, v0.2s, v1.2s
 ; CHECK-NEXT:    ushr v0.2s, v0.2s, #24
 ; CHECK-NEXT:    mov w8, v0.s[1]
 ; CHECK-NEXT:    fmov w9, s0
@@ -206,11 +172,9 @@ define void @v2i8(<2 x i8>* %px, <2 x i8>* %py, <2 x i8>* %pz) nounwind {
 define void @v4i16(<4 x i16>* %px, <4 x i16>* %py, <4 x i16>* %pz) nounwind {
 ; CHECK-LABEL: v4i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldr d0, [x1]
-; CHECK-NEXT:    ldr d1, [x0]
-; CHECK-NEXT:    mvn v2.8b, v0.8b
-; CHECK-NEXT:    umin v1.4h, v1.4h, v2.4h
-; CHECK-NEXT:    add v0.4h, v1.4h, v0.4h
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    uqadd v0.4h, v0.4h, v1.4h
 ; CHECK-NEXT:    str d0, [x2]
 ; CHECK-NEXT:    ret
   %x = load <4 x i16>, <4 x i16>* %px
@@ -223,19 +187,17 @@ define void @v4i16(<4 x i16>* %px, <4 x i16>* %py, <4 x i16>* %pz) nounwind {
 define void @v2i16(<2 x i16>* %px, <2 x i16>* %py, <2 x i16>* %pz) nounwind {
 ; CHECK-LABEL: v2i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldrh w9, [x1]
 ; CHECK-NEXT:    ldrh w8, [x0]
-; CHECK-NEXT:    ldrh w11, [x1, #2]
+; CHECK-NEXT:    ldrh w9, [x1]
 ; CHECK-NEXT:    ldrh w10, [x0, #2]
-; CHECK-NEXT:    fmov s1, w9
+; CHECK-NEXT:    ldrh w11, [x1, #2]
 ; CHECK-NEXT:    fmov s0, w8
-; CHECK-NEXT:    mov v1.s[1], w11
+; CHECK-NEXT:    fmov s1, w9
 ; CHECK-NEXT:    mov v0.s[1], w10
+; CHECK-NEXT:    mov v1.s[1], w11
 ; CHECK-NEXT:    shl v1.2s, v1.2s, #16
 ; CHECK-NEXT:    shl v0.2s, v0.2s, #16
-; CHECK-NEXT:    mvn v2.8b, v1.8b
-; CHECK-NEXT:    umin v0.2s, v0.2s, v2.2s
-; CHECK-NEXT:    add v0.2s, v0.2s, v1.2s
+; CHECK-NEXT:    uqadd v0.2s, v0.2s, v1.2s
 ; CHECK-NEXT:    ushr v0.2s, v0.2s, #16
 ; CHECK-NEXT:    mov w8, v0.s[1]
 ; CHECK-NEXT:    fmov w9, s0
@@ -252,9 +214,7 @@ define void @v2i16(<2 x i16>* %px, <2 x i16>* %py, <2 x i16>* %pz) nounwind {
 define <12 x i8> @v12i8(<12 x i8> %x, <12 x i8> %y) nounwind {
 ; CHECK-LABEL: v12i8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mvn v2.16b, v1.16b
-; CHECK-NEXT:    umin v0.16b, v0.16b, v2.16b
-; CHECK-NEXT:    add v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    uqadd v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
   %z = call <12 x i8> @llvm.uadd.sat.v12i8(<12 x i8> %x, <12 x i8> %y)
   ret <12 x i8> %z
@@ -263,16 +223,12 @@ define <12 x i8> @v12i8(<12 x i8> %x, <12 x i8> %y) nounwind {
 define void @v12i16(<12 x i16>* %px, <12 x i16>* %py, <12 x i16>* %pz) nounwind {
 ; CHECK-LABEL: v12i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldp q1, q0, [x1]
-; CHECK-NEXT:    ldp q3, q2, [x0]
-; CHECK-NEXT:    mvn v4.16b, v0.16b
-; CHECK-NEXT:    mvn v5.16b, v1.16b
-; CHECK-NEXT:    umin v2.8h, v2.8h, v4.8h
-; CHECK-NEXT:    umin v3.8h, v3.8h, v5.8h
-; CHECK-NEXT:    add v0.8h, v2.8h, v0.8h
-; CHECK-NEXT:    add v1.8h, v3.8h, v1.8h
-; CHECK-NEXT:    str q1, [x2]
-; CHECK-NEXT:    str d0, [x2, #16]
+; CHECK-NEXT:    ldp q0, q1, [x0]
+; CHECK-NEXT:    ldp q3, q2, [x1]
+; CHECK-NEXT:    uqadd v1.8h, v1.8h, v2.8h
+; CHECK-NEXT:    uqadd v0.8h, v0.8h, v3.8h
+; CHECK-NEXT:    str q0, [x2]
+; CHECK-NEXT:    str d1, [x2, #16]
 ; CHECK-NEXT:    ret
   %x = load <12 x i16>, <12 x i16>* %px
   %y = load <12 x i16>, <12 x i16>* %py
@@ -284,11 +240,9 @@ define void @v12i16(<12 x i16>* %px, <12 x i16>* %py, <12 x i16>* %pz) nounwind
 define void @v1i8(<1 x i8>* %px, <1 x i8>* %py, <1 x i8>* %pz) nounwind {
 ; CHECK-LABEL: v1i8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldr b0, [x1]
-; CHECK-NEXT:    ldr b1, [x0]
-; CHECK-NEXT:    mvn v2.8b, v0.8b
-; CHECK-NEXT:    umin v1.8b, v1.8b, v2.8b
-; CHECK-NEXT:    add v0.8b, v1.8b, v0.8b
+; CHECK-NEXT:    ldr b0, [x0]
+; CHECK-NEXT:    ldr b1, [x1]
+; CHECK-NEXT:    uqadd v0.8b, v0.8b, v1.8b
 ; CHECK-NEXT:    st1 { v0.b }[0], [x2]
 ; CHECK-NEXT:    ret
   %x = load <1 x i8>, <1 x i8>* %px
@@ -301,11 +255,9 @@ define void @v1i8(<1 x i8>* %px, <1 x i8>* %py, <1 x i8>* %pz) nounwind {
 define void @v1i16(<1 x i16>* %px, <1 x i16>* %py, <1 x i16>* %pz) nounwind {
 ; CHECK-LABEL: v1i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldr h0, [x1]
-; CHECK-NEXT:    ldr h1, [x0]
-; CHECK-NEXT:    mvn v2.8b, v0.8b
-; CHECK-NEXT:    umin v1.4h, v1.4h, v2.4h
-; CHECK-NEXT:    add v0.4h, v1.4h, v0.4h
+; CHECK-NEXT:    ldr h0, [x0]
+; CHECK-NEXT:    ldr h1, [x1]
+; CHECK-NEXT:    uqadd v0.4h, v0.4h, v1.4h
 ; CHECK-NEXT:    str h0, [x2]
 ; CHECK-NEXT:    ret
   %x = load <1 x i16>, <1 x i16>* %px
@@ -318,11 +270,12 @@ define void @v1i16(<1 x i16>* %px, <1 x i16>* %py, <1 x i16>* %pz) nounwind {
 define <16 x i4> @v16i4(<16 x i4> %x, <16 x i4> %y) nounwind {
 ; CHECK-LABEL: v16i4:
 ; CHECK:       // %bb.0:
+; CHECK-NEXT:    movi v2.16b, #15
+; CHECK-NEXT:    and v0.16b, v0.16b, v2.16b
+; CHECK-NEXT:    and v1.16b, v1.16b, v2.16b
 ; CHECK-NEXT:    shl v1.16b, v1.16b, #4
 ; CHECK-NEXT:    shl v0.16b, v0.16b, #4
-; CHECK-NEXT:    mvn v2.16b, v1.16b
-; CHECK-NEXT:    umin v0.16b, v0.16b, v2.16b
-; CHECK-NEXT:    add v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    uqadd v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ushr v0.16b, v0.16b, #4
 ; CHECK-NEXT:    ret
   %z = call <16 x i4> @llvm.uadd.sat.v16i4(<16 x i4> %x, <16 x i4> %y)
@@ -332,11 +285,12 @@ define <16 x i4> @v16i4(<16 x i4> %x, <16 x i4> %y) nounwind {
 define <16 x i1> @v16i1(<16 x i1> %x, <16 x i1> %y) nounwind {
 ; CHECK-LABEL: v16i1:
 ; CHECK:       // %bb.0:
+; CHECK-NEXT:    movi v2.16b, #1
+; CHECK-NEXT:    and v0.16b, v0.16b, v2.16b
+; CHECK-NEXT:    and v1.16b, v1.16b, v2.16b
 ; CHECK-NEXT:    shl v1.16b, v1.16b, #7
 ; CHECK-NEXT:    shl v0.16b, v0.16b, #7
-; CHECK-NEXT:    mvn v2.16b, v1.16b
-; CHECK-NEXT:    umin v0.16b, v0.16b, v2.16b
-; CHECK-NEXT:    add v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    uqadd v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ushr v0.16b, v0.16b, #7
 ; CHECK-NEXT:    ret
   %z = call <16 x i1> @llvm.uadd.sat.v16i1(<16 x i1> %x, <16 x i1> %y)
@@ -346,9 +300,7 @@ define <16 x i1> @v16i1(<16 x i1> %x, <16 x i1> %y) nounwind {
 define <2 x i32> @v2i32(<2 x i32> %x, <2 x i32> %y) nounwind {
 ; CHECK-LABEL: v2i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mvn v2.8b, v1.8b
-; CHECK-NEXT:    umin v0.2s, v0.2s, v2.2s
-; CHECK-NEXT:    add v0.2s, v0.2s, v1.2s
+; CHECK-NEXT:    uqadd v0.2s, v0.2s, v1.2s
 ; CHECK-NEXT:    ret
   %z = call <2 x i32> @llvm.uadd.sat.v2i32(<2 x i32> %x, <2 x i32> %y)
   ret <2 x i32> %z
@@ -357,9 +309,7 @@ define <2 x i32> @v2i32(<2 x i32> %x, <2 x i32> %y) nounwind {
 define <4 x i32> @v4i32(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; CHECK-LABEL: v4i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mvn v2.16b, v1.16b
-; CHECK-NEXT:    umin v0.4s, v0.4s, v2.4s
-; CHECK-NEXT:    add v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    uqadd v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    ret
   %z = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> %x, <4 x i32> %y)
   ret <4 x i32> %z
@@ -368,12 +318,8 @@ define <4 x i32> @v4i32(<4 x i32> %x, <4 x i32> %y) nounwind {
 define <8 x i32> @v8i32(<8 x i32> %x, <8 x i32> %y) nounwind {
 ; CHECK-LABEL: v8i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mvn v4.16b, v2.16b
-; CHECK-NEXT:    mvn v5.16b, v3.16b
-; CHECK-NEXT:    umin v0.4s, v0.4s, v4.4s
-; CHECK-NEXT:    umin v1.4s, v1.4s, v5.4s
-; CHECK-NEXT:    add v0.4s, v0.4s, v2.4s
-; CHECK-NEXT:    add v1.4s, v1.4s, v3.4s
+; CHECK-NEXT:    uqadd v0.4s, v0.4s, v2.4s
+; CHECK-NEXT:    uqadd v1.4s, v1.4s, v3.4s
 ; CHECK-NEXT:    ret
   %z = call <8 x i32> @llvm.uadd.sat.v8i32(<8 x i32> %x, <8 x i32> %y)
   ret <8 x i32> %z
@@ -382,18 +328,10 @@ define <8 x i32> @v8i32(<8 x i32> %x, <8 x i32> %y) nounwind {
 define <16 x i32> @v16i32(<16 x i32> %x, <16 x i32> %y) nounwind {
 ; CHECK-LABEL: v16i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mvn v16.16b, v4.16b
-; CHECK-NEXT:    umin v0.4s, v0.4s, v16.4s
-; CHECK-NEXT:    mvn v16.16b, v5.16b
-; CHECK-NEXT:    umin v1.4s, v1.4s, v16.4s
-; CHECK-NEXT:    mvn v16.16b, v6.16b
-; CHECK-NEXT:    umin v2.4s, v2.4s, v16.4s
-; CHECK-NEXT:    mvn v16.16b, v7.16b
-; CHECK-NEXT:    umin v3.4s, v3.4s, v16.4s
-; CHECK-NEXT:    add v0.4s, v0.4s, v4.4s
-; CHECK-NEXT:    add v1.4s, v1.4s, v5.4s
-; CHECK-NEXT:    add v2.4s, v2.4s, v6.4s
-; CHECK-NEXT:    add v3.4s, v3.4s, v7.4s
+; CHECK-NEXT:    uqadd v0.4s, v0.4s, v4.4s
+; CHECK-NEXT:    uqadd v1.4s, v1.4s, v5.4s
+; CHECK-NEXT:    uqadd v2.4s, v2.4s, v6.4s
+; CHECK-NEXT:    uqadd v3.4s, v3.4s, v7.4s
 ; CHECK-NEXT:    ret
   %z = call <16 x i32> @llvm.uadd.sat.v16i32(<16 x i32> %x, <16 x i32> %y)
   ret <16 x i32> %z
@@ -402,9 +340,7 @@ define <16 x i32> @v16i32(<16 x i32> %x, <16 x i32> %y) nounwind {
 define <2 x i64> @v2i64(<2 x i64> %x, <2 x i64> %y) nounwind {
 ; CHECK-LABEL: v2i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    add v1.2d, v0.2d, v1.2d
-; CHECK-NEXT:    cmhi v0.2d, v0.2d, v1.2d
-; CHECK-NEXT:    orr v0.16b, v1.16b, v0.16b
+; CHECK-NEXT:    uqadd v0.2d, v0.2d, v1.2d
 ; CHECK-NEXT:    ret
   %z = call <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64> %x, <2 x i64> %y)
   ret <2 x i64> %z
@@ -413,12 +349,8 @@ define <2 x i64> @v2i64(<2 x i64> %x, <2 x i64> %y) nounwind {
 define <4 x i64> @v4i64(<4 x i64> %x, <4 x i64> %y) nounwind {
 ; CHECK-LABEL: v4i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    add v2.2d, v0.2d, v2.2d
-; CHECK-NEXT:    add v3.2d, v1.2d, v3.2d
-; CHECK-NEXT:    cmhi v0.2d, v0.2d, v2.2d
-; CHECK-NEXT:    cmhi v1.2d, v1.2d, v3.2d
-; CHECK-NEXT:    orr v0.16b, v2.16b, v0.16b
-; CHECK-NEXT:    orr v1.16b, v3.16b, v1.16b
+; CHECK-NEXT:    uqadd v0.2d, v0.2d, v2.2d
+; CHECK-NEXT:    uqadd v1.2d, v1.2d, v3.2d
 ; CHECK-NEXT:    ret
   %z = call <4 x i64> @llvm.uadd.sat.v4i64(<4 x i64> %x, <4 x i64> %y)
   ret <4 x i64> %z
@@ -427,18 +359,10 @@ define <4 x i64> @v4i64(<4 x i64> %x, <4 x i64> %y) nounwind {
 define <8 x i64> @v8i64(<8 x i64> %x, <8 x i64> %y) nounwind {
 ; CHECK-LABEL: v8i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    add v4.2d, v0.2d, v4.2d
-; CHECK-NEXT:    add v5.2d, v1.2d, v5.2d
-; CHECK-NEXT:    add v6.2d, v2.2d, v6.2d
-; CHECK-NEXT:    add v7.2d, v3.2d, v7.2d
-; CHECK-NEXT:    cmhi v0.2d, v0.2d, v4.2d
-; CHECK-NEXT:    cmhi v1.2d, v1.2d, v5.2d
-; CHECK-NEXT:    cmhi v2.2d, v2.2d, v6.2d
-; CHECK-NEXT:    cmhi v3.2d, v3.2d, v7.2d
-; CHECK-NEXT:    orr v0.16b, v4.16b, v0.16b
-; CHECK-NEXT:    orr v1.16b, v5.16b, v1.16b
-; CHECK-NEXT:    orr v2.16b, v6.16b, v2.16b
-; CHECK-NEXT:    orr v3.16b, v7.16b, v3.16b
+; CHECK-NEXT:    uqadd v0.2d, v0.2d, v4.2d
+; CHECK-NEXT:    uqadd v1.2d, v1.2d, v5.2d
+; CHECK-NEXT:    uqadd v2.2d, v2.2d, v6.2d
+; CHECK-NEXT:    uqadd v3.2d, v3.2d, v7.2d
 ; CHECK-NEXT:    ret
   %z = call <8 x i64> @llvm.uadd.sat.v8i64(<8 x i64> %x, <8 x i64> %y)
   ret <8 x i64> %z
diff --git a/llvm/test/CodeGen/AArch64/umulo-128-legalisation-lowering.ll b/llvm/test/CodeGen/AArch64/umulo-128-legalisation-lowering.ll
index 51522e1d1..8edd867ff 100644
--- a/llvm/test/CodeGen/AArch64/umulo-128-legalisation-lowering.ll
+++ b/llvm/test/CodeGen/AArch64/umulo-128-legalisation-lowering.ll
@@ -27,8 +27,8 @@ define { i128, i8 } @muloti_test(i128 %l, i128 %r) unnamed_addr #0 {
 ; AARCH-NEXT:    orr w10, w10, w11
 ; AARCH-NEXT:    orr w9, w10, w9
 ; AARCH-NEXT:    mul x0, x0, x2
-; AARCH-NEXT:    mov x1, x8
-; AARCH-NEXT:    mov w2, w9
+; AARCH-DAG:    mov x1, x8
+; AARCH-DAG:    mov w2, w9
 ; AARCH-NEXT:    ret
 start:
   %0 = tail call { i128, i1 } @llvm.umul.with.overflow.i128(i128 %l, i128 %r) #2
diff --git a/llvm/test/CodeGen/AArch64/unfold-masked-merge-scalar-constmask-innerouter.ll b/llvm/test/CodeGen/AArch64/unfold-masked-merge-scalar-constmask-innerouter.ll
index 188cb5b18..2e20ef67b 100644
--- a/llvm/test/CodeGen/AArch64/unfold-masked-merge-scalar-constmask-innerouter.ll
+++ b/llvm/test/CodeGen/AArch64/unfold-masked-merge-scalar-constmask-innerouter.ll
@@ -202,16 +202,16 @@ declare void @use32(i32) nounwind
 define i32 @in_multiuse_A_constmask(i32 %x, i32 %y, i32 %z) nounwind {
 ; CHECK-LABEL: in_multiuse_A_constmask:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    str x20, [sp, #-32]! // 8-byte Folded Spill
+; CHECK-NEXT:    str x30, [sp, #-32]! // 8-byte Folded Spill
 ; CHECK-NEXT:    eor w8, w0, w1
+; CHECK-NEXT:    stp x20, x19, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    and w20, w8, #0xffff00
 ; CHECK-NEXT:    mov w0, w20
-; CHECK-NEXT:    stp x19, x30, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    mov w19, w1
 ; CHECK-NEXT:    bl use32
 ; CHECK-NEXT:    eor w0, w20, w19
-; CHECK-NEXT:    ldp x19, x30, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x20, [sp], #32 // 8-byte Folded Reload
+; CHECK-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x30, [sp], #32 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
   %n0 = xor i32 %x, %y
   %n1 = and i32 %n0, 16776960
@@ -223,15 +223,15 @@ define i32 @in_multiuse_A_constmask(i32 %x, i32 %y, i32 %z) nounwind {
 define i32 @in_multiuse_B_constmask(i32 %x, i32 %y, i32 %z) nounwind {
 ; CHECK-LABEL: in_multiuse_B_constmask:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    str x20, [sp, #-32]! // 8-byte Folded Spill
+; CHECK-NEXT:    str x30, [sp, #-32]! // 8-byte Folded Spill
 ; CHECK-NEXT:    eor w0, w0, w1
-; CHECK-NEXT:    stp x19, x30, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    stp x20, x19, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    mov w19, w1
 ; CHECK-NEXT:    and w20, w0, #0xffff00
 ; CHECK-NEXT:    bl use32
 ; CHECK-NEXT:    eor w0, w20, w19
-; CHECK-NEXT:    ldp x19, x30, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x20, [sp], #32 // 8-byte Folded Reload
+; CHECK-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x30, [sp], #32 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
   %n0 = xor i32 %x, %y
   %n1 = and i32 %n0, 16776960
diff --git a/llvm/test/CodeGen/AArch64/unfold-masked-merge-scalar-constmask-interleavedbits.ll b/llvm/test/CodeGen/AArch64/unfold-masked-merge-scalar-constmask-interleavedbits.ll
index 2eeffe081..488d2cf93 100644
--- a/llvm/test/CodeGen/AArch64/unfold-masked-merge-scalar-constmask-interleavedbits.ll
+++ b/llvm/test/CodeGen/AArch64/unfold-masked-merge-scalar-constmask-interleavedbits.ll
@@ -208,16 +208,16 @@ declare void @use32(i32) nounwind
 define i32 @in_multiuse_A_constmask(i32 %x, i32 %y, i32 %z) nounwind {
 ; CHECK-LABEL: in_multiuse_A_constmask:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    str x20, [sp, #-32]! // 8-byte Folded Spill
+; CHECK-NEXT:    str x30, [sp, #-32]! // 8-byte Folded Spill
 ; CHECK-NEXT:    eor w8, w0, w1
+; CHECK-NEXT:    stp x20, x19, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    and w20, w8, #0x55555555
 ; CHECK-NEXT:    mov w0, w20
-; CHECK-NEXT:    stp x19, x30, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    mov w19, w1
 ; CHECK-NEXT:    bl use32
 ; CHECK-NEXT:    eor w0, w20, w19
-; CHECK-NEXT:    ldp x19, x30, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x20, [sp], #32 // 8-byte Folded Reload
+; CHECK-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x30, [sp], #32 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
   %n0 = xor i32 %x, %y
   %n1 = and i32 %n0, 1431655765
@@ -229,15 +229,15 @@ define i32 @in_multiuse_A_constmask(i32 %x, i32 %y, i32 %z) nounwind {
 define i32 @in_multiuse_B_constmask(i32 %x, i32 %y, i32 %z) nounwind {
 ; CHECK-LABEL: in_multiuse_B_constmask:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    str x20, [sp, #-32]! // 8-byte Folded Spill
+; CHECK-NEXT:    str x30, [sp, #-32]! // 8-byte Folded Spill
 ; CHECK-NEXT:    eor w0, w0, w1
-; CHECK-NEXT:    stp x19, x30, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    stp x20, x19, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    mov w19, w1
 ; CHECK-NEXT:    and w20, w0, #0x55555555
 ; CHECK-NEXT:    bl use32
 ; CHECK-NEXT:    eor w0, w20, w19
-; CHECK-NEXT:    ldp x19, x30, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x20, [sp], #32 // 8-byte Folded Reload
+; CHECK-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x30, [sp], #32 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
   %n0 = xor i32 %x, %y
   %n1 = and i32 %n0, 1431655765
diff --git a/llvm/test/CodeGen/AArch64/unfold-masked-merge-scalar-constmask-interleavedbytehalves.ll b/llvm/test/CodeGen/AArch64/unfold-masked-merge-scalar-constmask-interleavedbytehalves.ll
index a6ebeb1fb..d67ecfa70 100644
--- a/llvm/test/CodeGen/AArch64/unfold-masked-merge-scalar-constmask-interleavedbytehalves.ll
+++ b/llvm/test/CodeGen/AArch64/unfold-masked-merge-scalar-constmask-interleavedbytehalves.ll
@@ -204,16 +204,16 @@ declare void @use32(i32) nounwind
 define i32 @in_multiuse_A_constmask(i32 %x, i32 %y, i32 %z) nounwind {
 ; CHECK-LABEL: in_multiuse_A_constmask:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    str x20, [sp, #-32]! // 8-byte Folded Spill
+; CHECK-NEXT:    str x30, [sp, #-32]! // 8-byte Folded Spill
 ; CHECK-NEXT:    eor w8, w0, w1
+; CHECK-NEXT:    stp x20, x19, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    and w20, w8, #0xf0f0f0f
 ; CHECK-NEXT:    mov w0, w20
-; CHECK-NEXT:    stp x19, x30, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    mov w19, w1
 ; CHECK-NEXT:    bl use32
 ; CHECK-NEXT:    eor w0, w20, w19
-; CHECK-NEXT:    ldp x19, x30, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x20, [sp], #32 // 8-byte Folded Reload
+; CHECK-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x30, [sp], #32 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
   %n0 = xor i32 %x, %y
   %n1 = and i32 %n0, 252645135
@@ -225,15 +225,15 @@ define i32 @in_multiuse_A_constmask(i32 %x, i32 %y, i32 %z) nounwind {
 define i32 @in_multiuse_B_constmask(i32 %x, i32 %y, i32 %z) nounwind {
 ; CHECK-LABEL: in_multiuse_B_constmask:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    str x20, [sp, #-32]! // 8-byte Folded Spill
+; CHECK-NEXT:    str x30, [sp, #-32]! // 8-byte Folded Spill
 ; CHECK-NEXT:    eor w0, w0, w1
-; CHECK-NEXT:    stp x19, x30, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    stp x20, x19, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    mov w19, w1
 ; CHECK-NEXT:    and w20, w0, #0xf0f0f0f
 ; CHECK-NEXT:    bl use32
 ; CHECK-NEXT:    eor w0, w20, w19
-; CHECK-NEXT:    ldp x19, x30, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x20, [sp], #32 // 8-byte Folded Reload
+; CHECK-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x30, [sp], #32 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
   %n0 = xor i32 %x, %y
   %n1 = and i32 %n0, 252645135
diff --git a/llvm/test/CodeGen/AArch64/unfold-masked-merge-scalar-constmask-lowhigh.ll b/llvm/test/CodeGen/AArch64/unfold-masked-merge-scalar-constmask-lowhigh.ll
index fc4f7ebb2..23f2206c7 100644
--- a/llvm/test/CodeGen/AArch64/unfold-masked-merge-scalar-constmask-lowhigh.ll
+++ b/llvm/test/CodeGen/AArch64/unfold-masked-merge-scalar-constmask-lowhigh.ll
@@ -197,16 +197,16 @@ declare void @use32(i32) nounwind
 define i32 @in_multiuse_A_constmask(i32 %x, i32 %y, i32 %z) nounwind {
 ; CHECK-LABEL: in_multiuse_A_constmask:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    str x20, [sp, #-32]! // 8-byte Folded Spill
+; CHECK-NEXT:    str x30, [sp, #-32]! // 8-byte Folded Spill
 ; CHECK-NEXT:    eor w8, w0, w1
+; CHECK-NEXT:    stp x20, x19, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    and w20, w8, #0xffff
 ; CHECK-NEXT:    mov w0, w20
-; CHECK-NEXT:    stp x19, x30, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    mov w19, w1
 ; CHECK-NEXT:    bl use32
 ; CHECK-NEXT:    eor w0, w20, w19
-; CHECK-NEXT:    ldp x19, x30, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x20, [sp], #32 // 8-byte Folded Reload
+; CHECK-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x30, [sp], #32 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
   %n0 = xor i32 %x, %y
   %n1 = and i32 %n0, 65535
@@ -218,15 +218,15 @@ define i32 @in_multiuse_A_constmask(i32 %x, i32 %y, i32 %z) nounwind {
 define i32 @in_multiuse_B_constmask(i32 %x, i32 %y, i32 %z) nounwind {
 ; CHECK-LABEL: in_multiuse_B_constmask:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    str x20, [sp, #-32]! // 8-byte Folded Spill
+; CHECK-NEXT:    str x30, [sp, #-32]! // 8-byte Folded Spill
 ; CHECK-NEXT:    eor w0, w0, w1
-; CHECK-NEXT:    stp x19, x30, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    stp x20, x19, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    mov w19, w1
 ; CHECK-NEXT:    and w20, w0, #0xffff
 ; CHECK-NEXT:    bl use32
 ; CHECK-NEXT:    eor w0, w20, w19
-; CHECK-NEXT:    ldp x19, x30, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x20, [sp], #32 // 8-byte Folded Reload
+; CHECK-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x30, [sp], #32 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
   %n0 = xor i32 %x, %y
   %n1 = and i32 %n0, 65535
diff --git a/llvm/test/CodeGen/AArch64/unfold-masked-merge-scalar-variablemask.ll b/llvm/test/CodeGen/AArch64/unfold-masked-merge-scalar-variablemask.ll
index fdb5965db..186068643 100644
--- a/llvm/test/CodeGen/AArch64/unfold-masked-merge-scalar-variablemask.ll
+++ b/llvm/test/CodeGen/AArch64/unfold-masked-merge-scalar-variablemask.ll
@@ -552,16 +552,16 @@ declare void @use32(i32) nounwind
 define i32 @in_multiuse_A(i32 %x, i32 %y, i32 %z, i32 %mask) nounwind {
 ; CHECK-LABEL: in_multiuse_A:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    str x20, [sp, #-32]! // 8-byte Folded Spill
+; CHECK-NEXT:    str x30, [sp, #-32]! // 8-byte Folded Spill
 ; CHECK-NEXT:    eor w8, w0, w1
+; CHECK-NEXT:    stp x20, x19, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    and w20, w8, w3
 ; CHECK-NEXT:    mov w0, w20
-; CHECK-NEXT:    stp x19, x30, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    mov w19, w1
 ; CHECK-NEXT:    bl use32
 ; CHECK-NEXT:    eor w0, w20, w19
-; CHECK-NEXT:    ldp x19, x30, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x20, [sp], #32 // 8-byte Folded Reload
+; CHECK-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x30, [sp], #32 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
   %n0 = xor i32 %x, %y
   %n1 = and i32 %n0, %mask
@@ -572,15 +572,15 @@ define i32 @in_multiuse_A(i32 %x, i32 %y, i32 %z, i32 %mask) nounwind {
 define i32 @in_multiuse_B(i32 %x, i32 %y, i32 %z, i32 %mask) nounwind {
 ; CHECK-LABEL: in_multiuse_B:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    str x20, [sp, #-32]! // 8-byte Folded Spill
+; CHECK-NEXT:    str x30, [sp, #-32]! // 8-byte Folded Spill
 ; CHECK-NEXT:    eor w0, w0, w1
-; CHECK-NEXT:    stp x19, x30, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    stp x20, x19, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    mov w19, w1
 ; CHECK-NEXT:    and w20, w0, w3
 ; CHECK-NEXT:    bl use32
 ; CHECK-NEXT:    eor w0, w20, w19
-; CHECK-NEXT:    ldp x19, x30, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x20, [sp], #32 // 8-byte Folded Reload
+; CHECK-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x30, [sp], #32 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
   %n0 = xor i32 %x, %y
   %n1 = and i32 %n0, %mask
diff --git a/llvm/test/CodeGen/AArch64/urem-lkk.ll b/llvm/test/CodeGen/AArch64/urem-lkk.ll
new file mode 100644
index 000000000..3d7f309dd
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/urem-lkk.ll
@@ -0,0 +1,103 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64-unknown-linux-gnu < %s | FileCheck %s
+
+define i32 @fold_urem_positive_odd(i32 %x) {
+; CHECK-LABEL: fold_urem_positive_odd:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #8969
+; CHECK-NEXT:    movk w8, #22765, lsl #16
+; CHECK-NEXT:    umull x8, w0, w8
+; CHECK-NEXT:    lsr x8, x8, #32
+; CHECK-NEXT:    sub w9, w0, w8
+; CHECK-NEXT:    add w8, w8, w9, lsr #1
+; CHECK-NEXT:    lsr w8, w8, #6
+; CHECK-NEXT:    mov w9, #95
+; CHECK-NEXT:    msub w0, w8, w9, w0
+; CHECK-NEXT:    ret
+  %1 = urem i32 %x, 95
+  ret i32 %1
+}
+
+
+define i32 @fold_urem_positive_even(i32 %x) {
+; CHECK-LABEL: fold_urem_positive_even:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #16323
+; CHECK-NEXT:    movk w8, #63310, lsl #16
+; CHECK-NEXT:    umull x8, w0, w8
+; CHECK-NEXT:    lsr x8, x8, #42
+; CHECK-NEXT:    mov w9, #1060
+; CHECK-NEXT:    msub w0, w8, w9, w0
+; CHECK-NEXT:    ret
+  %1 = urem i32 %x, 1060
+  ret i32 %1
+}
+
+
+; Don't fold if we can combine urem with udiv.
+define i32 @combine_urem_udiv(i32 %x) {
+; CHECK-LABEL: combine_urem_udiv:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #8969
+; CHECK-NEXT:    movk w8, #22765, lsl #16
+; CHECK-NEXT:    umull x8, w0, w8
+; CHECK-NEXT:    lsr x8, x8, #32
+; CHECK-NEXT:    sub w9, w0, w8
+; CHECK-NEXT:    add w8, w8, w9, lsr #1
+; CHECK-NEXT:    lsr w8, w8, #6
+; CHECK-NEXT:    mov w9, #95
+; CHECK-NEXT:    msub w9, w8, w9, w0
+; CHECK-NEXT:    add w0, w9, w8
+; CHECK-NEXT:    ret
+  %1 = urem i32 %x, 95
+  %2 = udiv i32 %x, 95
+  %3 = add i32 %1, %2
+  ret i32 %3
+}
+
+; Don't fold for divisors that are a power of two.
+define i32 @dont_fold_urem_power_of_two(i32 %x) {
+; CHECK-LABEL: dont_fold_urem_power_of_two:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    and w0, w0, #0x3f
+; CHECK-NEXT:    ret
+  %1 = urem i32 %x, 64
+  ret i32 %1
+}
+
+; Don't fold if the divisor is one.
+define i32 @dont_fold_urem_one(i32 %x) {
+; CHECK-LABEL: dont_fold_urem_one:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w0, wzr
+; CHECK-NEXT:    ret
+  %1 = urem i32 %x, 1
+  ret i32 %1
+}
+
+; Don't fold if the divisor is 2^32.
+define i32 @dont_fold_urem_i32_umax(i32 %x) {
+; CHECK-LABEL: dont_fold_urem_i32_umax:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ret
+  %1 = urem i32 %x, 4294967296
+  ret i32 %1
+}
+
+; Don't fold i64 urem
+define i64 @dont_fold_urem_i64(i64 %x) {
+; CHECK-LABEL: dont_fold_urem_i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov x9, #58849
+; CHECK-NEXT:    movk x9, #48148, lsl #16
+; CHECK-NEXT:    movk x9, #33436, lsl #32
+; CHECK-NEXT:    lsr x8, x0, #1
+; CHECK-NEXT:    movk x9, #21399, lsl #48
+; CHECK-NEXT:    umulh x8, x8, x9
+; CHECK-NEXT:    lsr x8, x8, #4
+; CHECK-NEXT:    mov w9, #98
+; CHECK-NEXT:    msub x0, x8, x9, x0
+; CHECK-NEXT:    ret
+  %1 = urem i64 %x, 98
+  ret i64 %1
+}
diff --git a/llvm/test/CodeGen/AArch64/urem-seteq-nonzero.ll b/llvm/test/CodeGen/AArch64/urem-seteq-nonzero.ll
new file mode 100644
index 000000000..f29add835
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/urem-seteq-nonzero.ll
@@ -0,0 +1,243 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64-unknown-linux-gnu < %s | FileCheck %s
+
+define i1 @t32_3_1(i32 %X) nounwind {
+; CHECK-LABEL: t32_3_1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #43691
+; CHECK-NEXT:    movk w8, #43690, lsl #16
+; CHECK-NEXT:    mov w9, #1431655765
+; CHECK-NEXT:    madd w8, w0, w8, w9
+; CHECK-NEXT:    cmp w8, w9
+; CHECK-NEXT:    cset w0, lo
+; CHECK-NEXT:    ret
+  %urem = urem i32 %X, 3
+  %cmp = icmp eq i32 %urem, 1
+  ret i1 %cmp
+}
+
+define i1 @t32_3_2(i32 %X) nounwind {
+; CHECK-LABEL: t32_3_2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #43691
+; CHECK-NEXT:    movk w8, #43690, lsl #16
+; CHECK-NEXT:    mov w9, #-1431655766
+; CHECK-NEXT:    madd w8, w0, w8, w9
+; CHECK-NEXT:    mov w9, #1431655765
+; CHECK-NEXT:    cmp w8, w9
+; CHECK-NEXT:    cset w0, lo
+; CHECK-NEXT:    ret
+  %urem = urem i32 %X, 3
+  %cmp = icmp eq i32 %urem, 2
+  ret i1 %cmp
+}
+
+
+define i1 @t32_5_1(i32 %X) nounwind {
+; CHECK-LABEL: t32_5_1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #52429
+; CHECK-NEXT:    movk w8, #52428, lsl #16
+; CHECK-NEXT:    mov w9, #858993459
+; CHECK-NEXT:    madd w8, w0, w8, w9
+; CHECK-NEXT:    cmp w8, w9
+; CHECK-NEXT:    cset w0, lo
+; CHECK-NEXT:    ret
+  %urem = urem i32 %X, 5
+  %cmp = icmp eq i32 %urem, 1
+  ret i1 %cmp
+}
+
+define i1 @t32_5_2(i32 %X) nounwind {
+; CHECK-LABEL: t32_5_2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #52429
+; CHECK-NEXT:    movk w8, #52428, lsl #16
+; CHECK-NEXT:    mov w9, #1717986918
+; CHECK-NEXT:    madd w8, w0, w8, w9
+; CHECK-NEXT:    mov w9, #858993459
+; CHECK-NEXT:    cmp w8, w9
+; CHECK-NEXT:    cset w0, lo
+; CHECK-NEXT:    ret
+  %urem = urem i32 %X, 5
+  %cmp = icmp eq i32 %urem, 2
+  ret i1 %cmp
+}
+
+define i1 @t32_5_3(i32 %X) nounwind {
+; CHECK-LABEL: t32_5_3:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #52429
+; CHECK-NEXT:    movk w8, #52428, lsl #16
+; CHECK-NEXT:    mov w9, #-1717986919
+; CHECK-NEXT:    madd w8, w0, w8, w9
+; CHECK-NEXT:    mov w9, #858993459
+; CHECK-NEXT:    cmp w8, w9
+; CHECK-NEXT:    cset w0, lo
+; CHECK-NEXT:    ret
+  %urem = urem i32 %X, 5
+  %cmp = icmp eq i32 %urem, 3
+  ret i1 %cmp
+}
+
+define i1 @t32_5_4(i32 %X) nounwind {
+; CHECK-LABEL: t32_5_4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #52429
+; CHECK-NEXT:    movk w8, #52428, lsl #16
+; CHECK-NEXT:    mov w9, #-858993460
+; CHECK-NEXT:    madd w8, w0, w8, w9
+; CHECK-NEXT:    mov w9, #858993459
+; CHECK-NEXT:    cmp w8, w9
+; CHECK-NEXT:    cset w0, lo
+; CHECK-NEXT:    ret
+  %urem = urem i32 %X, 5
+  %cmp = icmp eq i32 %urem, 4
+  ret i1 %cmp
+}
+
+
+define i1 @t32_6_1(i32 %X) nounwind {
+; CHECK-LABEL: t32_6_1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #43691
+; CHECK-NEXT:    movk w8, #43690, lsl #16
+; CHECK-NEXT:    mov w9, #1431655765
+; CHECK-NEXT:    madd w8, w0, w8, w9
+; CHECK-NEXT:    mov w9, #43691
+; CHECK-NEXT:    ror w8, w8, #1
+; CHECK-NEXT:    movk w9, #10922, lsl #16
+; CHECK-NEXT:    cmp w8, w9
+; CHECK-NEXT:    cset w0, lo
+; CHECK-NEXT:    ret
+  %urem = urem i32 %X, 6
+  %cmp = icmp eq i32 %urem, 1
+  ret i1 %cmp
+}
+
+define i1 @t32_6_2(i32 %X) nounwind {
+; CHECK-LABEL: t32_6_2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #43691
+; CHECK-NEXT:    movk w8, #43690, lsl #16
+; CHECK-NEXT:    mov w9, #-1431655766
+; CHECK-NEXT:    madd w8, w0, w8, w9
+; CHECK-NEXT:    mov w9, #43691
+; CHECK-NEXT:    ror w8, w8, #1
+; CHECK-NEXT:    movk w9, #10922, lsl #16
+; CHECK-NEXT:    cmp w8, w9
+; CHECK-NEXT:    cset w0, lo
+; CHECK-NEXT:    ret
+  %urem = urem i32 %X, 6
+  %cmp = icmp eq i32 %urem, 2
+  ret i1 %cmp
+}
+
+define i1 @t32_6_3(i32 %X) nounwind {
+; CHECK-LABEL: t32_6_3:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #43691
+; CHECK-NEXT:    movk w8, #43690, lsl #16
+; CHECK-NEXT:    mul w8, w0, w8
+; CHECK-NEXT:    sub w8, w8, #1 // =1
+; CHECK-NEXT:    mov w9, #43691
+; CHECK-NEXT:    ror w8, w8, #1
+; CHECK-NEXT:    movk w9, #10922, lsl #16
+; CHECK-NEXT:    cmp w8, w9
+; CHECK-NEXT:    cset w0, lo
+; CHECK-NEXT:    ret
+  %urem = urem i32 %X, 6
+  %cmp = icmp eq i32 %urem, 3
+  ret i1 %cmp
+}
+
+define i1 @t32_6_4(i32 %X) nounwind {
+; CHECK-LABEL: t32_6_4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #43691
+; CHECK-NEXT:    mov w9, #21844
+; CHECK-NEXT:    movk w8, #43690, lsl #16
+; CHECK-NEXT:    movk w9, #21845, lsl #16
+; CHECK-NEXT:    madd w8, w0, w8, w9
+; CHECK-NEXT:    mov w9, #43690
+; CHECK-NEXT:    ror w8, w8, #1
+; CHECK-NEXT:    movk w9, #10922, lsl #16
+; CHECK-NEXT:    cmp w8, w9
+; CHECK-NEXT:    cset w0, lo
+; CHECK-NEXT:    ret
+  %urem = urem i32 %X, 6
+  %cmp = icmp eq i32 %urem, 4
+  ret i1 %cmp
+}
+
+define i1 @t32_6_5(i32 %X) nounwind {
+; CHECK-LABEL: t32_6_5:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #43691
+; CHECK-NEXT:    mov w9, #43689
+; CHECK-NEXT:    movk w8, #43690, lsl #16
+; CHECK-NEXT:    movk w9, #43690, lsl #16
+; CHECK-NEXT:    madd w8, w0, w8, w9
+; CHECK-NEXT:    mov w9, #43690
+; CHECK-NEXT:    ror w8, w8, #1
+; CHECK-NEXT:    movk w9, #10922, lsl #16
+; CHECK-NEXT:    cmp w8, w9
+; CHECK-NEXT:    cset w0, lo
+; CHECK-NEXT:    ret
+  %urem = urem i32 %X, 6
+  %cmp = icmp eq i32 %urem, 5
+  ret i1 %cmp
+}
+
+;-------------------------------------------------------------------------------
+; Other widths.
+
+define i1 @t16_3_2(i16 %X) nounwind {
+; CHECK-LABEL: t16_3_2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w9, #43691
+; CHECK-NEXT:    and w8, w0, #0xffff
+; CHECK-NEXT:    movk w9, #43690, lsl #16
+; CHECK-NEXT:    mov w10, #-1431655766
+; CHECK-NEXT:    madd w8, w8, w9, w10
+; CHECK-NEXT:    mov w9, #1431655765
+; CHECK-NEXT:    cmp w8, w9
+; CHECK-NEXT:    cset w0, lo
+; CHECK-NEXT:    ret
+  %urem = urem i16 %X, 3
+  %cmp = icmp eq i16 %urem, 2
+  ret i1 %cmp
+}
+
+define i1 @t8_3_2(i8 %X) nounwind {
+; CHECK-LABEL: t8_3_2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w9, #43691
+; CHECK-NEXT:    and w8, w0, #0xff
+; CHECK-NEXT:    movk w9, #43690, lsl #16
+; CHECK-NEXT:    mov w10, #-1431655766
+; CHECK-NEXT:    madd w8, w8, w9, w10
+; CHECK-NEXT:    mov w9, #1431655765
+; CHECK-NEXT:    cmp w8, w9
+; CHECK-NEXT:    cset w0, lo
+; CHECK-NEXT:    ret
+  %urem = urem i8 %X, 3
+  %cmp = icmp eq i8 %urem, 2
+  ret i1 %cmp
+}
+
+define i1 @t64_3_2(i64 %X) nounwind {
+; CHECK-LABEL: t64_3_2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov x8, #-6148914691236517206
+; CHECK-NEXT:    movk x8, #43691
+; CHECK-NEXT:    mov x9, #-6148914691236517206
+; CHECK-NEXT:    madd x8, x0, x8, x9
+; CHECK-NEXT:    mov x9, #6148914691236517205
+; CHECK-NEXT:    cmp x8, x9
+; CHECK-NEXT:    cset w0, lo
+; CHECK-NEXT:    ret
+  %urem = urem i64 %X, 3
+  %cmp = icmp eq i64 %urem, 2
+  ret i1 %cmp
+}
diff --git a/llvm/test/CodeGen/AArch64/urem-seteq-optsize.ll b/llvm/test/CodeGen/AArch64/urem-seteq-optsize.ll
index ea098ec45..344a07779 100644
--- a/llvm/test/CodeGen/AArch64/urem-seteq-optsize.ll
+++ b/llvm/test/CodeGen/AArch64/urem-seteq-optsize.ll
@@ -1,9 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64-unknown-linux-gnu < %s | FileCheck %s
 
-; On AArch64, division in expensive. BuildRemEqFold should therefore run even
-; when optimizing for size. Only optimizing for minimum size retains a plain div.
-
 define i32 @test_minsize(i32 %X) optsize minsize nounwind readnone {
 ; CHECK-LABEL: test_minsize:
 ; CHECK:       // %bb.0:
diff --git a/llvm/test/CodeGen/AArch64/urem-seteq-vec-nonsplat.ll b/llvm/test/CodeGen/AArch64/urem-seteq-vec-nonsplat.ll
index 57f6941d5..de3c1fafb 100644
--- a/llvm/test/CodeGen/AArch64/urem-seteq-vec-nonsplat.ll
+++ b/llvm/test/CodeGen/AArch64/urem-seteq-vec-nonsplat.ll
@@ -1,8 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64-unknown-linux-gnu < %s | FileCheck %s
 
-; At the moment, BuildUREMEqFold does not handle nonsplat vectors.
-
 ; Odd+Even divisors
 define <4 x i32> @test_urem_odd_even(<4 x i32> %X) nounwind {
 ; CHECK-LABEL: test_urem_odd_even:
@@ -40,18 +38,11 @@ define <4 x i32> @test_urem_odd_allones_eq(<4 x i32> %X) nounwind {
 ; CHECK-LABEL: test_urem_odd_allones_eq:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    adrp x8, .LCPI1_0
+; CHECK-NEXT:    adrp x9, .LCPI1_1
 ; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI1_0]
-; CHECK-NEXT:    adrp x8, .LCPI1_1
-; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI1_1]
-; CHECK-NEXT:    adrp x8, .LCPI1_2
-; CHECK-NEXT:    ldr q3, [x8, :lo12:.LCPI1_2]
-; CHECK-NEXT:    umull2 v4.2d, v0.4s, v1.4s
-; CHECK-NEXT:    umull v1.2d, v0.2s, v1.2s
-; CHECK-NEXT:    uzp2 v1.4s, v1.4s, v4.4s
-; CHECK-NEXT:    neg v2.4s, v2.4s
-; CHECK-NEXT:    ushl v1.4s, v1.4s, v2.4s
-; CHECK-NEXT:    mls v0.4s, v1.4s, v3.4s
-; CHECK-NEXT:    cmeq v0.4s, v0.4s, #0
+; CHECK-NEXT:    ldr q2, [x9, :lo12:.LCPI1_1]
+; CHECK-NEXT:    mul v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    cmhs v0.4s, v2.4s, v0.4s
 ; CHECK-NEXT:    movi v1.4s, #1
 ; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
@@ -64,19 +55,11 @@ define <4 x i32> @test_urem_odd_allones_ne(<4 x i32> %X) nounwind {
 ; CHECK-LABEL: test_urem_odd_allones_ne:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    adrp x8, .LCPI2_0
+; CHECK-NEXT:    adrp x9, .LCPI2_1
 ; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI2_0]
-; CHECK-NEXT:    adrp x8, .LCPI2_1
-; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI2_1]
-; CHECK-NEXT:    adrp x8, .LCPI2_2
-; CHECK-NEXT:    ldr q3, [x8, :lo12:.LCPI2_2]
-; CHECK-NEXT:    umull2 v4.2d, v0.4s, v1.4s
-; CHECK-NEXT:    umull v1.2d, v0.2s, v1.2s
-; CHECK-NEXT:    uzp2 v1.4s, v1.4s, v4.4s
-; CHECK-NEXT:    neg v2.4s, v2.4s
-; CHECK-NEXT:    ushl v1.4s, v1.4s, v2.4s
-; CHECK-NEXT:    mls v0.4s, v1.4s, v3.4s
-; CHECK-NEXT:    cmeq v0.4s, v0.4s, #0
-; CHECK-NEXT:    mvn v0.16b, v0.16b
+; CHECK-NEXT:    ldr q2, [x9, :lo12:.LCPI2_1]
+; CHECK-NEXT:    mul v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    cmhi v0.4s, v0.4s, v2.4s
 ; CHECK-NEXT:    movi v1.4s, #1
 ; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
@@ -300,20 +283,11 @@ define <4 x i32> @test_urem_odd_one(<4 x i32> %X) nounwind {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    adrp x8, .LCPI10_0
 ; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI10_0]
-; CHECK-NEXT:    adrp x8, .LCPI10_1
-; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI10_1]
-; CHECK-NEXT:    adrp x8, .LCPI10_2
-; CHECK-NEXT:    ldr q3, [x8, :lo12:.LCPI10_2]
-; CHECK-NEXT:    adrp x8, .LCPI10_3
-; CHECK-NEXT:    umull2 v4.2d, v0.4s, v1.4s
-; CHECK-NEXT:    umull v1.2d, v0.2s, v1.2s
-; CHECK-NEXT:    uzp2 v1.4s, v1.4s, v4.4s
-; CHECK-NEXT:    ldr q4, [x8, :lo12:.LCPI10_3]
-; CHECK-NEXT:    neg v2.4s, v2.4s
-; CHECK-NEXT:    ushl v1.4s, v1.4s, v2.4s
-; CHECK-NEXT:    bsl v3.16b, v0.16b, v1.16b
-; CHECK-NEXT:    mls v0.4s, v3.4s, v4.4s
-; CHECK-NEXT:    cmeq v0.4s, v0.4s, #0
+; CHECK-NEXT:    mov w8, #52429
+; CHECK-NEXT:    movk w8, #52428, lsl #16
+; CHECK-NEXT:    dup v2.4s, w8
+; CHECK-NEXT:    mul v0.4s, v0.4s, v2.4s
+; CHECK-NEXT:    cmhs v0.4s, v1.4s, v0.4s
 ; CHECK-NEXT:    movi v1.4s, #1
 ; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
@@ -389,11 +363,11 @@ define <4 x i32> @test_urem_odd_even_one(<4 x i32> %X) nounwind {
   ret <4 x i32> %ret
 }
 
-;==============================================================================;
+;------------------------------------------------------------------------------;
 
-; One all-ones divisor and power-of-two divisor divisor in odd divisor
-define <4 x i32> @test_urem_odd_allones_and_poweroftwo(<4 x i32> %X) nounwind {
-; CHECK-LABEL: test_urem_odd_allones_and_poweroftwo:
+; One INT_MIN divisor in odd divisor
+define <4 x i32> @test_urem_odd_INT_MIN(<4 x i32> %X) nounwind {
+; CHECK-LABEL: test_urem_odd_INT_MIN:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    adrp x8, .LCPI13_0
 ; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI13_0]
@@ -411,15 +385,15 @@ define <4 x i32> @test_urem_odd_allones_and_poweroftwo(<4 x i32> %X) nounwind {
 ; CHECK-NEXT:    movi v1.4s, #1
 ; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
-  %urem = urem <4 x i32> %X, <i32 5, i32 4294967295, i32 16, i32 5>
+  %urem = urem <4 x i32> %X, <i32 5, i32 5, i32 2147483648, i32 5>
   %cmp = icmp eq <4 x i32> %urem, <i32 0, i32 0, i32 0, i32 0>
   %ret = zext <4 x i1> %cmp to <4 x i32>
   ret <4 x i32> %ret
 }
 
-; One all-ones divisor and power-of-two divisor divisor in even divisor
-define <4 x i32> @test_urem_even_allones_and_poweroftwo(<4 x i32> %X) nounwind {
-; CHECK-LABEL: test_urem_even_allones_and_poweroftwo:
+; One INT_MIN divisor in even divisor
+define <4 x i32> @test_urem_even_INT_MIN(<4 x i32> %X) nounwind {
+; CHECK-LABEL: test_urem_even_INT_MIN:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    adrp x8, .LCPI14_0
 ; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI14_0]
@@ -441,15 +415,15 @@ define <4 x i32> @test_urem_even_allones_and_poweroftwo(<4 x i32> %X) nounwind {
 ; CHECK-NEXT:    movi v1.4s, #1
 ; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
-  %urem = urem <4 x i32> %X, <i32 14, i32 4294967295, i32 16, i32 14>
+  %urem = urem <4 x i32> %X, <i32 14, i32 14, i32 2147483648, i32 14>
   %cmp = icmp eq <4 x i32> %urem, <i32 0, i32 0, i32 0, i32 0>
   %ret = zext <4 x i1> %cmp to <4 x i32>
   ret <4 x i32> %ret
 }
 
-; One all-ones divisor and power-of-two divisor divisor in odd+even divisor
-define <4 x i32> @test_urem_odd_even_allones_and_poweroftwo(<4 x i32> %X) nounwind {
-; CHECK-LABEL: test_urem_odd_even_allones_and_poweroftwo:
+; One INT_MIN divisor in odd+even divisor
+define <4 x i32> @test_urem_odd_even_INT_MIN(<4 x i32> %X) nounwind {
+; CHECK-LABEL: test_urem_odd_even_INT_MIN:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    adrp x8, .LCPI15_0
 ; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI15_0]
@@ -457,27 +431,31 @@ define <4 x i32> @test_urem_odd_even_allones_and_poweroftwo(<4 x i32> %X) nounwi
 ; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI15_1]
 ; CHECK-NEXT:    adrp x8, .LCPI15_2
 ; CHECK-NEXT:    ldr q3, [x8, :lo12:.LCPI15_2]
-; CHECK-NEXT:    umull2 v4.2d, v0.4s, v1.4s
-; CHECK-NEXT:    umull v1.2d, v0.2s, v1.2s
+; CHECK-NEXT:    neg v1.4s, v1.4s
+; CHECK-NEXT:    adrp x8, .LCPI15_3
+; CHECK-NEXT:    ushl v1.4s, v0.4s, v1.4s
+; CHECK-NEXT:    umull2 v4.2d, v1.4s, v2.4s
+; CHECK-NEXT:    umull v1.2d, v1.2s, v2.2s
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI15_3]
 ; CHECK-NEXT:    uzp2 v1.4s, v1.4s, v4.4s
-; CHECK-NEXT:    neg v2.4s, v2.4s
-; CHECK-NEXT:    ushl v1.4s, v1.4s, v2.4s
-; CHECK-NEXT:    mls v0.4s, v1.4s, v3.4s
+; CHECK-NEXT:    neg v3.4s, v3.4s
+; CHECK-NEXT:    ushl v1.4s, v1.4s, v3.4s
+; CHECK-NEXT:    mls v0.4s, v1.4s, v2.4s
 ; CHECK-NEXT:    cmeq v0.4s, v0.4s, #0
 ; CHECK-NEXT:    movi v1.4s, #1
 ; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
-  %urem = urem <4 x i32> %X, <i32 5, i32 4294967295, i32 16, i32 100>
+  %urem = urem <4 x i32> %X, <i32 5, i32 14, i32 2147483648, i32 100>
   %cmp = icmp eq <4 x i32> %urem, <i32 0, i32 0, i32 0, i32 0>
   %ret = zext <4 x i1> %cmp to <4 x i32>
   ret <4 x i32> %ret
 }
 
-;------------------------------------------------------------------------------;
+;==============================================================================;
 
-; One all-ones divisor and one one divisor in odd divisor
-define <4 x i32> @test_urem_odd_allones_and_one(<4 x i32> %X) nounwind {
-; CHECK-LABEL: test_urem_odd_allones_and_one:
+; One all-ones divisor and power-of-two divisor divisor in odd divisor
+define <4 x i32> @test_urem_odd_allones_and_poweroftwo(<4 x i32> %X) nounwind {
+; CHECK-LABEL: test_urem_odd_allones_and_poweroftwo:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    adrp x8, .LCPI16_0
 ; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI16_0]
@@ -485,28 +463,25 @@ define <4 x i32> @test_urem_odd_allones_and_one(<4 x i32> %X) nounwind {
 ; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI16_1]
 ; CHECK-NEXT:    adrp x8, .LCPI16_2
 ; CHECK-NEXT:    ldr q3, [x8, :lo12:.LCPI16_2]
-; CHECK-NEXT:    adrp x8, .LCPI16_3
 ; CHECK-NEXT:    umull2 v4.2d, v0.4s, v1.4s
 ; CHECK-NEXT:    umull v1.2d, v0.2s, v1.2s
 ; CHECK-NEXT:    uzp2 v1.4s, v1.4s, v4.4s
-; CHECK-NEXT:    ldr q4, [x8, :lo12:.LCPI16_3]
 ; CHECK-NEXT:    neg v2.4s, v2.4s
 ; CHECK-NEXT:    ushl v1.4s, v1.4s, v2.4s
-; CHECK-NEXT:    bsl v3.16b, v0.16b, v1.16b
-; CHECK-NEXT:    mls v0.4s, v3.4s, v4.4s
+; CHECK-NEXT:    mls v0.4s, v1.4s, v3.4s
 ; CHECK-NEXT:    cmeq v0.4s, v0.4s, #0
 ; CHECK-NEXT:    movi v1.4s, #1
 ; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
-  %urem = urem <4 x i32> %X, <i32 5, i32 4294967295, i32 1, i32 5>
+  %urem = urem <4 x i32> %X, <i32 5, i32 4294967295, i32 16, i32 5>
   %cmp = icmp eq <4 x i32> %urem, <i32 0, i32 0, i32 0, i32 0>
   %ret = zext <4 x i1> %cmp to <4 x i32>
   ret <4 x i32> %ret
 }
 
-; One all-ones divisor and one one divisor in even divisor
-define <4 x i32> @test_urem_even_allones_and_one(<4 x i32> %X) nounwind {
-; CHECK-LABEL: test_urem_even_allones_and_one:
+; One all-ones divisor and power-of-two divisor divisor in even divisor
+define <4 x i32> @test_urem_even_allones_and_poweroftwo(<4 x i32> %X) nounwind {
+; CHECK-LABEL: test_urem_even_allones_and_poweroftwo:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    adrp x8, .LCPI17_0
 ; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI17_0]
@@ -520,26 +495,23 @@ define <4 x i32> @test_urem_even_allones_and_one(<4 x i32> %X) nounwind {
 ; CHECK-NEXT:    umull2 v4.2d, v1.4s, v2.4s
 ; CHECK-NEXT:    umull v1.2d, v1.2s, v2.2s
 ; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI17_3]
-; CHECK-NEXT:    adrp x8, .LCPI17_4
 ; CHECK-NEXT:    uzp2 v1.4s, v1.4s, v4.4s
-; CHECK-NEXT:    ldr q4, [x8, :lo12:.LCPI17_4]
 ; CHECK-NEXT:    neg v3.4s, v3.4s
 ; CHECK-NEXT:    ushl v1.4s, v1.4s, v3.4s
-; CHECK-NEXT:    bsl v2.16b, v0.16b, v1.16b
-; CHECK-NEXT:    mls v0.4s, v2.4s, v4.4s
+; CHECK-NEXT:    mls v0.4s, v1.4s, v2.4s
 ; CHECK-NEXT:    cmeq v0.4s, v0.4s, #0
 ; CHECK-NEXT:    movi v1.4s, #1
 ; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
-  %urem = urem <4 x i32> %X, <i32 14, i32 4294967295, i32 1, i32 14>
+  %urem = urem <4 x i32> %X, <i32 14, i32 4294967295, i32 16, i32 14>
   %cmp = icmp eq <4 x i32> %urem, <i32 0, i32 0, i32 0, i32 0>
   %ret = zext <4 x i1> %cmp to <4 x i32>
   ret <4 x i32> %ret
 }
 
-; One all-ones divisor and one one divisor in odd+even divisor
-define <4 x i32> @test_urem_odd_even_allones_and_one(<4 x i32> %X) nounwind {
-; CHECK-LABEL: test_urem_odd_even_allones_and_one:
+; One all-ones divisor and power-of-two divisor divisor in odd+even divisor
+define <4 x i32> @test_urem_odd_even_allones_and_poweroftwo(<4 x i32> %X) nounwind {
+; CHECK-LABEL: test_urem_odd_even_allones_and_poweroftwo:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    adrp x8, .LCPI18_0
 ; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI18_0]
@@ -547,20 +519,17 @@ define <4 x i32> @test_urem_odd_even_allones_and_one(<4 x i32> %X) nounwind {
 ; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI18_1]
 ; CHECK-NEXT:    adrp x8, .LCPI18_2
 ; CHECK-NEXT:    ldr q3, [x8, :lo12:.LCPI18_2]
-; CHECK-NEXT:    adrp x8, .LCPI18_3
 ; CHECK-NEXT:    umull2 v4.2d, v0.4s, v1.4s
 ; CHECK-NEXT:    umull v1.2d, v0.2s, v1.2s
 ; CHECK-NEXT:    uzp2 v1.4s, v1.4s, v4.4s
-; CHECK-NEXT:    ldr q4, [x8, :lo12:.LCPI18_3]
 ; CHECK-NEXT:    neg v2.4s, v2.4s
 ; CHECK-NEXT:    ushl v1.4s, v1.4s, v2.4s
-; CHECK-NEXT:    bsl v3.16b, v0.16b, v1.16b
-; CHECK-NEXT:    mls v0.4s, v3.4s, v4.4s
+; CHECK-NEXT:    mls v0.4s, v1.4s, v3.4s
 ; CHECK-NEXT:    cmeq v0.4s, v0.4s, #0
 ; CHECK-NEXT:    movi v1.4s, #1
 ; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
-  %urem = urem <4 x i32> %X, <i32 5, i32 4294967295, i32 1, i32 100>
+  %urem = urem <4 x i32> %X, <i32 5, i32 4294967295, i32 16, i32 100>
   %cmp = icmp eq <4 x i32> %urem, <i32 0, i32 0, i32 0, i32 0>
   %ret = zext <4 x i1> %cmp to <4 x i32>
   ret <4 x i32> %ret
@@ -568,38 +537,28 @@ define <4 x i32> @test_urem_odd_even_allones_and_one(<4 x i32> %X) nounwind {
 
 ;------------------------------------------------------------------------------;
 
-; One power-of-two divisor divisor and one divisor in odd divisor
-define <4 x i32> @test_urem_odd_poweroftwo_and_one(<4 x i32> %X) nounwind {
-; CHECK-LABEL: test_urem_odd_poweroftwo_and_one:
+; One all-ones divisor and one one divisor in odd divisor
+define <4 x i32> @test_urem_odd_allones_and_one(<4 x i32> %X) nounwind {
+; CHECK-LABEL: test_urem_odd_allones_and_one:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    adrp x8, .LCPI19_0
+; CHECK-NEXT:    adrp x9, .LCPI19_1
 ; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI19_0]
-; CHECK-NEXT:    adrp x8, .LCPI19_1
-; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI19_1]
-; CHECK-NEXT:    adrp x8, .LCPI19_2
-; CHECK-NEXT:    ldr q3, [x8, :lo12:.LCPI19_2]
-; CHECK-NEXT:    adrp x8, .LCPI19_3
-; CHECK-NEXT:    umull2 v4.2d, v0.4s, v1.4s
-; CHECK-NEXT:    umull v1.2d, v0.2s, v1.2s
-; CHECK-NEXT:    uzp2 v1.4s, v1.4s, v4.4s
-; CHECK-NEXT:    ldr q4, [x8, :lo12:.LCPI19_3]
-; CHECK-NEXT:    neg v2.4s, v2.4s
-; CHECK-NEXT:    ushl v1.4s, v1.4s, v2.4s
-; CHECK-NEXT:    bsl v3.16b, v0.16b, v1.16b
-; CHECK-NEXT:    mls v0.4s, v3.4s, v4.4s
-; CHECK-NEXT:    cmeq v0.4s, v0.4s, #0
+; CHECK-NEXT:    ldr q2, [x9, :lo12:.LCPI19_1]
+; CHECK-NEXT:    mul v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    cmhs v0.4s, v2.4s, v0.4s
 ; CHECK-NEXT:    movi v1.4s, #1
 ; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
-  %urem = urem <4 x i32> %X, <i32 5, i32 16, i32 1, i32 5>
+  %urem = urem <4 x i32> %X, <i32 5, i32 4294967295, i32 1, i32 5>
   %cmp = icmp eq <4 x i32> %urem, <i32 0, i32 0, i32 0, i32 0>
   %ret = zext <4 x i1> %cmp to <4 x i32>
   ret <4 x i32> %ret
 }
 
-; One power-of-two divisor divisor and one divisor in even divisor
-define <4 x i32> @test_urem_even_poweroftwo_and_one(<4 x i32> %X) nounwind {
-; CHECK-LABEL: test_urem_even_poweroftwo_and_one:
+; One all-ones divisor and one one divisor in even divisor
+define <4 x i32> @test_urem_even_allones_and_one(<4 x i32> %X) nounwind {
+; CHECK-LABEL: test_urem_even_allones_and_one:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    adrp x8, .LCPI20_0
 ; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI20_0]
@@ -624,15 +583,15 @@ define <4 x i32> @test_urem_even_poweroftwo_and_one(<4 x i32> %X) nounwind {
 ; CHECK-NEXT:    movi v1.4s, #1
 ; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
-  %urem = urem <4 x i32> %X, <i32 14, i32 16, i32 1, i32 14>
+  %urem = urem <4 x i32> %X, <i32 14, i32 4294967295, i32 1, i32 14>
   %cmp = icmp eq <4 x i32> %urem, <i32 0, i32 0, i32 0, i32 0>
   %ret = zext <4 x i1> %cmp to <4 x i32>
   ret <4 x i32> %ret
 }
 
-; One power-of-two divisor divisor and one divisor in odd+even divisor
-define <4 x i32> @test_urem_odd_even_poweroftwo_and_one(<4 x i32> %X) nounwind {
-; CHECK-LABEL: test_urem_odd_even_poweroftwo_and_one:
+; One all-ones divisor and one one divisor in odd+even divisor
+define <4 x i32> @test_urem_odd_even_allones_and_one(<4 x i32> %X) nounwind {
+; CHECK-LABEL: test_urem_odd_even_allones_and_one:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    adrp x8, .LCPI21_0
 ; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI21_0]
@@ -653,7 +612,7 @@ define <4 x i32> @test_urem_odd_even_poweroftwo_and_one(<4 x i32> %X) nounwind {
 ; CHECK-NEXT:    movi v1.4s, #1
 ; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
-  %urem = urem <4 x i32> %X, <i32 5, i32 16, i32 1, i32 100>
+  %urem = urem <4 x i32> %X, <i32 5, i32 4294967295, i32 1, i32 100>
   %cmp = icmp eq <4 x i32> %urem, <i32 0, i32 0, i32 0, i32 0>
   %ret = zext <4 x i1> %cmp to <4 x i32>
   ret <4 x i32> %ret
@@ -661,8 +620,9 @@ define <4 x i32> @test_urem_odd_even_poweroftwo_and_one(<4 x i32> %X) nounwind {
 
 ;------------------------------------------------------------------------------;
 
-define <4 x i32> @test_urem_odd_allones_and_poweroftwo_and_one(<4 x i32> %X) nounwind {
-; CHECK-LABEL: test_urem_odd_allones_and_poweroftwo_and_one:
+; One power-of-two divisor divisor and one divisor in odd divisor
+define <4 x i32> @test_urem_odd_poweroftwo_and_one(<4 x i32> %X) nounwind {
+; CHECK-LABEL: test_urem_odd_poweroftwo_and_one:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    adrp x8, .LCPI22_0
 ; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI22_0]
@@ -683,14 +643,15 @@ define <4 x i32> @test_urem_odd_allones_and_poweroftwo_and_one(<4 x i32> %X) nou
 ; CHECK-NEXT:    movi v1.4s, #1
 ; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
-  %urem = urem <4 x i32> %X, <i32 5, i32 4294967295, i32 16, i32 1>
+  %urem = urem <4 x i32> %X, <i32 5, i32 16, i32 1, i32 5>
   %cmp = icmp eq <4 x i32> %urem, <i32 0, i32 0, i32 0, i32 0>
   %ret = zext <4 x i1> %cmp to <4 x i32>
   ret <4 x i32> %ret
 }
 
-define <4 x i32> @test_urem_even_allones_and_poweroftwo_and_one(<4 x i32> %X) nounwind {
-; CHECK-LABEL: test_urem_even_allones_and_poweroftwo_and_one:
+; One power-of-two divisor divisor and one divisor in even divisor
+define <4 x i32> @test_urem_even_poweroftwo_and_one(<4 x i32> %X) nounwind {
+; CHECK-LABEL: test_urem_even_poweroftwo_and_one:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    adrp x8, .LCPI23_0
 ; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI23_0]
@@ -714,6 +675,97 @@ define <4 x i32> @test_urem_even_allones_and_poweroftwo_and_one(<4 x i32> %X) no
 ; CHECK-NEXT:    cmeq v0.4s, v0.4s, #0
 ; CHECK-NEXT:    movi v1.4s, #1
 ; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    ret
+  %urem = urem <4 x i32> %X, <i32 14, i32 16, i32 1, i32 14>
+  %cmp = icmp eq <4 x i32> %urem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; One power-of-two divisor divisor and one divisor in odd+even divisor
+define <4 x i32> @test_urem_odd_even_poweroftwo_and_one(<4 x i32> %X) nounwind {
+; CHECK-LABEL: test_urem_odd_even_poweroftwo_and_one:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI24_0
+; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI24_0]
+; CHECK-NEXT:    adrp x8, .LCPI24_1
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI24_1]
+; CHECK-NEXT:    adrp x8, .LCPI24_2
+; CHECK-NEXT:    ldr q3, [x8, :lo12:.LCPI24_2]
+; CHECK-NEXT:    adrp x8, .LCPI24_3
+; CHECK-NEXT:    umull2 v4.2d, v0.4s, v1.4s
+; CHECK-NEXT:    umull v1.2d, v0.2s, v1.2s
+; CHECK-NEXT:    uzp2 v1.4s, v1.4s, v4.4s
+; CHECK-NEXT:    ldr q4, [x8, :lo12:.LCPI24_3]
+; CHECK-NEXT:    neg v2.4s, v2.4s
+; CHECK-NEXT:    ushl v1.4s, v1.4s, v2.4s
+; CHECK-NEXT:    bsl v3.16b, v0.16b, v1.16b
+; CHECK-NEXT:    mls v0.4s, v3.4s, v4.4s
+; CHECK-NEXT:    cmeq v0.4s, v0.4s, #0
+; CHECK-NEXT:    movi v1.4s, #1
+; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    ret
+  %urem = urem <4 x i32> %X, <i32 5, i32 16, i32 1, i32 100>
+  %cmp = icmp eq <4 x i32> %urem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+;------------------------------------------------------------------------------;
+
+define <4 x i32> @test_urem_odd_allones_and_poweroftwo_and_one(<4 x i32> %X) nounwind {
+; CHECK-LABEL: test_urem_odd_allones_and_poweroftwo_and_one:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI25_0
+; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI25_0]
+; CHECK-NEXT:    adrp x8, .LCPI25_1
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI25_1]
+; CHECK-NEXT:    adrp x8, .LCPI25_2
+; CHECK-NEXT:    ldr q3, [x8, :lo12:.LCPI25_2]
+; CHECK-NEXT:    adrp x8, .LCPI25_3
+; CHECK-NEXT:    umull2 v4.2d, v0.4s, v1.4s
+; CHECK-NEXT:    umull v1.2d, v0.2s, v1.2s
+; CHECK-NEXT:    uzp2 v1.4s, v1.4s, v4.4s
+; CHECK-NEXT:    ldr q4, [x8, :lo12:.LCPI25_3]
+; CHECK-NEXT:    neg v2.4s, v2.4s
+; CHECK-NEXT:    ushl v1.4s, v1.4s, v2.4s
+; CHECK-NEXT:    bsl v3.16b, v0.16b, v1.16b
+; CHECK-NEXT:    mls v0.4s, v3.4s, v4.4s
+; CHECK-NEXT:    cmeq v0.4s, v0.4s, #0
+; CHECK-NEXT:    movi v1.4s, #1
+; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    ret
+  %urem = urem <4 x i32> %X, <i32 5, i32 4294967295, i32 16, i32 1>
+  %cmp = icmp eq <4 x i32> %urem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+define <4 x i32> @test_urem_even_allones_and_poweroftwo_and_one(<4 x i32> %X) nounwind {
+; CHECK-LABEL: test_urem_even_allones_and_poweroftwo_and_one:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI26_0
+; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI26_0]
+; CHECK-NEXT:    adrp x8, .LCPI26_1
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI26_1]
+; CHECK-NEXT:    adrp x8, .LCPI26_2
+; CHECK-NEXT:    ldr q3, [x8, :lo12:.LCPI26_2]
+; CHECK-NEXT:    neg v1.4s, v1.4s
+; CHECK-NEXT:    adrp x8, .LCPI26_3
+; CHECK-NEXT:    ushl v1.4s, v0.4s, v1.4s
+; CHECK-NEXT:    umull2 v4.2d, v1.4s, v2.4s
+; CHECK-NEXT:    umull v1.2d, v1.2s, v2.2s
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI26_3]
+; CHECK-NEXT:    adrp x8, .LCPI26_4
+; CHECK-NEXT:    uzp2 v1.4s, v1.4s, v4.4s
+; CHECK-NEXT:    ldr q4, [x8, :lo12:.LCPI26_4]
+; CHECK-NEXT:    neg v3.4s, v3.4s
+; CHECK-NEXT:    ushl v1.4s, v1.4s, v3.4s
+; CHECK-NEXT:    bsl v2.16b, v0.16b, v1.16b
+; CHECK-NEXT:    mls v0.4s, v2.4s, v4.4s
+; CHECK-NEXT:    cmeq v0.4s, v0.4s, #0
+; CHECK-NEXT:    movi v1.4s, #1
+; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
   %urem = urem <4 x i32> %X, <i32 14, i32 4294967295, i32 16, i32 1>
   %cmp = icmp eq <4 x i32> %urem, <i32 0, i32 0, i32 0, i32 0>
diff --git a/llvm/test/CodeGen/AArch64/urem-seteq-vec-nonzero.ll b/llvm/test/CodeGen/AArch64/urem-seteq-vec-nonzero.ll
new file mode 100644
index 000000000..5ee7c2a9a
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/urem-seteq-vec-nonzero.ll
@@ -0,0 +1,115 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64-unknown-linux-gnu < %s | FileCheck %s
+
+define <4 x i1> @t32_3(<4 x i32> %X) nounwind {
+; CHECK-LABEL: t32_3:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI0_0
+; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI0_0]
+; CHECK-NEXT:    adrp x9, .LCPI0_1
+; CHECK-NEXT:    mov w8, #43691
+; CHECK-NEXT:    ldr q2, [x9, :lo12:.LCPI0_1]
+; CHECK-NEXT:    movk w8, #43690, lsl #16
+; CHECK-NEXT:    sub v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    dup v1.4s, w8
+; CHECK-NEXT:    mul v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    cmhs v0.4s, v2.4s, v0.4s
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    ret
+  %urem = urem <4 x i32> %X, <i32 3, i32 3, i32 3, i32 3>
+  %cmp = icmp eq <4 x i32> %urem, <i32 0, i32 1, i32 2, i32 2>
+  ret <4 x i1> %cmp
+}
+
+define <4 x i1> @t32_5(<4 x i32> %X) nounwind {
+; CHECK-LABEL: t32_5:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI1_0
+; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI1_0]
+; CHECK-NEXT:    mov w8, #52429
+; CHECK-NEXT:    movk w8, #52428, lsl #16
+; CHECK-NEXT:    mov w9, #13106
+; CHECK-NEXT:    movk w9, #13107, lsl #16
+; CHECK-NEXT:    dup v2.4s, w8
+; CHECK-NEXT:    sub v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    mul v0.4s, v0.4s, v2.4s
+; CHECK-NEXT:    dup v1.4s, w9
+; CHECK-NEXT:    cmhs v0.4s, v1.4s, v0.4s
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    ret
+  %urem = urem <4 x i32> %X, <i32 5, i32 5, i32 5, i32 5>
+  %cmp = icmp eq <4 x i32> %urem, <i32 1, i32 2, i32 3, i32 4>
+  ret <4 x i1> %cmp
+}
+
+define <4 x i1> @t32_6_part0(<4 x i32> %X) nounwind {
+; CHECK-LABEL: t32_6_part0:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #43691
+; CHECK-NEXT:    movk w8, #43690, lsl #16
+; CHECK-NEXT:    adrp x9, .LCPI2_0
+; CHECK-NEXT:    dup v1.4s, w8
+; CHECK-NEXT:    ldr q2, [x9, :lo12:.LCPI2_0]
+; CHECK-NEXT:    umull2 v3.2d, v0.4s, v1.4s
+; CHECK-NEXT:    umull v1.2d, v0.2s, v1.2s
+; CHECK-NEXT:    uzp2 v1.4s, v1.4s, v3.4s
+; CHECK-NEXT:    ushr v1.4s, v1.4s, #2
+; CHECK-NEXT:    movi v3.4s, #6
+; CHECK-NEXT:    mls v0.4s, v1.4s, v3.4s
+; CHECK-NEXT:    cmeq v0.4s, v0.4s, v2.4s
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    ret
+  %urem = urem <4 x i32> %X, <i32 6, i32 6, i32 6, i32 6>
+  %cmp = icmp eq <4 x i32> %urem, <i32 0, i32 1, i32 2, i32 3>
+  ret <4 x i1> %cmp
+}
+
+define <4 x i1> @t32_6_part1(<4 x i32> %X) nounwind {
+; CHECK-LABEL: t32_6_part1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #43691
+; CHECK-NEXT:    movk w8, #43690, lsl #16
+; CHECK-NEXT:    adrp x9, .LCPI3_0
+; CHECK-NEXT:    dup v1.4s, w8
+; CHECK-NEXT:    ldr q2, [x9, :lo12:.LCPI3_0]
+; CHECK-NEXT:    umull2 v3.2d, v0.4s, v1.4s
+; CHECK-NEXT:    umull v1.2d, v0.2s, v1.2s
+; CHECK-NEXT:    uzp2 v1.4s, v1.4s, v3.4s
+; CHECK-NEXT:    ushr v1.4s, v1.4s, #2
+; CHECK-NEXT:    movi v3.4s, #6
+; CHECK-NEXT:    mls v0.4s, v1.4s, v3.4s
+; CHECK-NEXT:    cmeq v0.4s, v0.4s, v2.4s
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    ret
+  %urem = urem <4 x i32> %X, <i32 6, i32 6, i32 6, i32 6>
+  %cmp = icmp eq <4 x i32> %urem, <i32 4, i32 5, i32 0, i32 0>
+  ret <4 x i1> %cmp
+}
+
+define <4 x i1> @t32_tautological(<4 x i32> %X) nounwind {
+; CHECK-LABEL: t32_tautological:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI4_0
+; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI4_0]
+; CHECK-NEXT:    adrp x8, .LCPI4_1
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI4_1]
+; CHECK-NEXT:    adrp x8, .LCPI4_2
+; CHECK-NEXT:    ldr q3, [x8, :lo12:.LCPI4_2]
+; CHECK-NEXT:    adrp x8, .LCPI4_3
+; CHECK-NEXT:    ldr q4, [x8, :lo12:.LCPI4_3]
+; CHECK-NEXT:    adrp x8, .LCPI4_4
+; CHECK-NEXT:    umull2 v5.2d, v0.4s, v1.4s
+; CHECK-NEXT:    umull v1.2d, v0.2s, v1.2s
+; CHECK-NEXT:    neg v2.4s, v2.4s
+; CHECK-NEXT:    uzp2 v1.4s, v1.4s, v5.4s
+; CHECK-NEXT:    ldr q5, [x8, :lo12:.LCPI4_4]
+; CHECK-NEXT:    ushl v1.4s, v1.4s, v2.4s
+; CHECK-NEXT:    bsl v3.16b, v0.16b, v1.16b
+; CHECK-NEXT:    mls v0.4s, v3.4s, v4.4s
+; CHECK-NEXT:    cmeq v0.4s, v0.4s, v5.4s
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    ret
+  %urem = urem <4 x i32> %X, <i32 1, i32 1, i32 2, i32 3>
+  %cmp = icmp eq <4 x i32> %urem, <i32 0, i32 1, i32 2, i32 2>
+  ret <4 x i1> %cmp
+}
diff --git a/llvm/test/CodeGen/AArch64/urem-seteq-vec-splat.ll b/llvm/test/CodeGen/AArch64/urem-seteq-vec-splat.ll
index d544e5f9c..ae51708e0 100644
--- a/llvm/test/CodeGen/AArch64/urem-seteq-vec-splat.ll
+++ b/llvm/test/CodeGen/AArch64/urem-seteq-vec-splat.ll
@@ -45,6 +45,57 @@ define <4 x i32> @test_urem_even_100(<4 x i32> %X) nounwind {
   ret <4 x i32> %ret
 }
 
+; Negative divisors should be negated, and thus this is still splat vectors.
+
+; Odd divisor
+define <4 x i32> @test_urem_odd_neg25(<4 x i32> %X) nounwind {
+; CHECK-LABEL: test_urem_odd_neg25:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI2_0
+; CHECK-NEXT:    adrp x9, .LCPI2_1
+; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI2_0]
+; CHECK-NEXT:    ldr q2, [x9, :lo12:.LCPI2_1]
+; CHECK-NEXT:    mul v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    cmhs v0.4s, v2.4s, v0.4s
+; CHECK-NEXT:    movi v1.4s, #1
+; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    ret
+  %urem = urem <4 x i32> %X, <i32 25, i32 -25, i32 -25, i32 25>
+  %cmp = icmp eq <4 x i32> %urem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Even divisors
+define <4 x i32> @test_urem_even_neg100(<4 x i32> %X) nounwind {
+; CHECK-LABEL: test_urem_even_neg100:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI3_0
+; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI3_0]
+; CHECK-NEXT:    adrp x8, .LCPI3_1
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI3_1]
+; CHECK-NEXT:    adrp x8, .LCPI3_2
+; CHECK-NEXT:    ldr q3, [x8, :lo12:.LCPI3_2]
+; CHECK-NEXT:    neg v1.4s, v1.4s
+; CHECK-NEXT:    adrp x8, .LCPI3_3
+; CHECK-NEXT:    ushl v1.4s, v0.4s, v1.4s
+; CHECK-NEXT:    umull2 v4.2d, v1.4s, v2.4s
+; CHECK-NEXT:    umull v1.2d, v1.2s, v2.2s
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI3_3]
+; CHECK-NEXT:    uzp2 v1.4s, v1.4s, v4.4s
+; CHECK-NEXT:    neg v3.4s, v3.4s
+; CHECK-NEXT:    ushl v1.4s, v1.4s, v3.4s
+; CHECK-NEXT:    mls v0.4s, v1.4s, v2.4s
+; CHECK-NEXT:    cmeq v0.4s, v0.4s, #0
+; CHECK-NEXT:    movi v1.4s, #1
+; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    ret
+  %urem = urem <4 x i32> %X, <i32 -100, i32 100, i32 -100, i32 100>
+  %cmp = icmp eq <4 x i32> %urem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
 ;------------------------------------------------------------------------------;
 ; Comparison constant has undef elements.
 ;------------------------------------------------------------------------------;
@@ -97,6 +148,27 @@ define <4 x i32> @test_urem_even_undef1(<4 x i32> %X) nounwind {
 ; Negative tests
 ;------------------------------------------------------------------------------;
 
+define <4 x i32> @test_urem_one_eq(<4 x i32> %X) nounwind {
+; CHECK-LABEL: test_urem_one_eq:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    movi v0.4s, #1
+; CHECK-NEXT:    ret
+  %urem = urem <4 x i32> %X, <i32 1, i32 1, i32 1, i32 1>
+  %cmp = icmp eq <4 x i32> %urem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+define <4 x i32> @test_urem_one_ne(<4 x i32> %X) nounwind {
+; CHECK-LABEL: test_urem_one_ne:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-NEXT:    ret
+  %urem = urem <4 x i32> %X, <i32 1, i32 1, i32 1, i32 1>
+  %cmp = icmp ne <4 x i32> %urem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
 ; We can lower remainder of division by powers of two much better elsewhere.
 define <4 x i32> @test_urem_pow2(<4 x i32> %X) nounwind {
 ; CHECK-LABEL: test_urem_pow2:
@@ -113,39 +185,32 @@ define <4 x i32> @test_urem_pow2(<4 x i32> %X) nounwind {
   ret <4 x i32> %ret
 }
 
-; We could lower remainder of division by all-ones much better elsewhere.
-define <4 x i32> @test_urem_allones(<4 x i32> %X) nounwind {
-; CHECK-LABEL: test_urem_allones:
+; We could lower remainder of division by INT_MIN much better elsewhere.
+define <4 x i32> @test_urem_int_min(<4 x i32> %X) nounwind {
+; CHECK-LABEL: test_urem_int_min:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    neg v0.4s, v0.4s
+; CHECK-NEXT:    bic v0.4s, #128, lsl #24
 ; CHECK-NEXT:    movi v1.4s, #1
-; CHECK-NEXT:    cmhs v0.4s, v1.4s, v0.4s
+; CHECK-NEXT:    cmeq v0.4s, v0.4s, #0
 ; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
-  %urem = urem <4 x i32> %X, <i32 4294967295, i32 4294967295, i32 4294967295, i32 4294967295>
+  %urem = urem <4 x i32> %X, <i32 2147483648, i32 2147483648, i32 2147483648, i32 2147483648>
   %cmp = icmp eq <4 x i32> %urem, <i32 0, i32 0, i32 0, i32 0>
   %ret = zext <4 x i1> %cmp to <4 x i32>
   ret <4 x i32> %ret
 }
 
-; If all divisors are ones, this is constant-folded.
-define <4 x i32> @test_urem_one_eq(<4 x i32> %X) nounwind {
-; CHECK-LABEL: test_urem_one_eq:
+; We could lower remainder of division by all-ones much better elsewhere.
+define <4 x i32> @test_urem_allones(<4 x i32> %X) nounwind {
+; CHECK-LABEL: test_urem_allones:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    movi v0.4s, #1
+; CHECK-NEXT:    neg v0.4s, v0.4s
+; CHECK-NEXT:    movi v1.4s, #1
+; CHECK-NEXT:    cmhs v0.4s, v1.4s, v0.4s
+; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
-  %urem = urem <4 x i32> %X, <i32 1, i32 1, i32 1, i32 1>
+  %urem = urem <4 x i32> %X, <i32 4294967295, i32 4294967295, i32 4294967295, i32 4294967295>
   %cmp = icmp eq <4 x i32> %urem, <i32 0, i32 0, i32 0, i32 0>
   %ret = zext <4 x i1> %cmp to <4 x i32>
   ret <4 x i32> %ret
 }
-define <4 x i32> @test_urem_one_ne(<4 x i32> %X) nounwind {
-; CHECK-LABEL: test_urem_one_ne:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    movi v0.2d, #0000000000000000
-; CHECK-NEXT:    ret
-  %urem = urem <4 x i32> %X, <i32 1, i32 1, i32 1, i32 1>
-  %cmp = icmp ne <4 x i32> %urem, <i32 0, i32 0, i32 0, i32 0>
-  %ret = zext <4 x i1> %cmp to <4 x i32>
-  ret <4 x i32> %ret
-}
diff --git a/llvm/test/CodeGen/AArch64/urem-seteq-vec-tautological.ll b/llvm/test/CodeGen/AArch64/urem-seteq-vec-tautological.ll
new file mode 100644
index 000000000..cb66f0e23
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/urem-seteq-vec-tautological.ll
@@ -0,0 +1,97 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64-unknown-linux-gnu < %s | FileCheck %s
+
+define <4 x i1> @t0_all_tautological(<4 x i32> %X) nounwind {
+; CHECK-LABEL: t0_all_tautological:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI0_0
+; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI0_0]
+; CHECK-NEXT:    adrp x8, .LCPI0_1
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI0_1]
+; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    cmeq v0.4s, v0.4s, v2.4s
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    ret
+  %urem = urem <4 x i32> %X, <i32 1, i32 1, i32 2, i32 2>
+  %cmp = icmp eq <4 x i32> %urem, <i32 0, i32 1, i32 2, i32 3>
+  ret <4 x i1> %cmp
+}
+
+define <4 x i1> @t1_all_odd_eq(<4 x i32> %X) nounwind {
+; CHECK-LABEL: t1_all_odd_eq:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI1_0
+; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI1_0]
+; CHECK-NEXT:    mov w8, #43691
+; CHECK-NEXT:    movk w8, #43690, lsl #16
+; CHECK-NEXT:    dup v2.4s, w8
+; CHECK-NEXT:    mul v0.4s, v0.4s, v2.4s
+; CHECK-NEXT:    cmhs v0.4s, v1.4s, v0.4s
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    movi d1, #0xffff0000ffff0000
+; CHECK-NEXT:    eor v0.8b, v0.8b, v1.8b
+; CHECK-NEXT:    ret
+  %urem = urem <4 x i32> %X, <i32 3, i32 1, i32 1, i32 9>
+  %cmp = icmp eq <4 x i32> %urem, <i32 0, i32 42, i32 0, i32 42>
+  ret <4 x i1> %cmp
+}
+
+define <4 x i1> @t1_all_odd_ne(<4 x i32> %X) nounwind {
+; CHECK-LABEL: t1_all_odd_ne:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI2_0
+; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI2_0]
+; CHECK-NEXT:    mov w8, #43691
+; CHECK-NEXT:    movk w8, #43690, lsl #16
+; CHECK-NEXT:    dup v2.4s, w8
+; CHECK-NEXT:    mul v0.4s, v0.4s, v2.4s
+; CHECK-NEXT:    cmhi v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    movi d1, #0xffff0000ffff0000
+; CHECK-NEXT:    eor v0.8b, v0.8b, v1.8b
+; CHECK-NEXT:    ret
+  %urem = urem <4 x i32> %X, <i32 3, i32 1, i32 1, i32 9>
+  %cmp = icmp ne <4 x i32> %urem, <i32 0, i32 42, i32 0, i32 42>
+  ret <4 x i1> %cmp
+}
+
+define <8 x i1> @t2_narrow(<8 x i16> %X) nounwind {
+; CHECK-LABEL: t2_narrow:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI3_0
+; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI3_0]
+; CHECK-NEXT:    mov w8, #43691
+; CHECK-NEXT:    dup v2.8h, w8
+; CHECK-NEXT:    mul v0.8h, v0.8h, v2.8h
+; CHECK-NEXT:    cmhs v0.8h, v1.8h, v0.8h
+; CHECK-NEXT:    xtn v0.8b, v0.8h
+; CHECK-NEXT:    movi d1, #0xffff0000ffff0000
+; CHECK-NEXT:    eor v0.8b, v0.8b, v1.8b
+; CHECK-NEXT:    ret
+  %urem = urem <8 x i16> %X, <i16 3, i16 1, i16 1, i16 9, i16 3, i16 1, i16 1, i16 9>
+  %cmp = icmp eq <8 x i16> %urem, <i16 0, i16 0, i16 42, i16 42, i16 0, i16 0, i16 42, i16 42>
+  ret <8 x i1> %cmp
+}
+
+define <2 x i1> @t3_wide(<2 x i64> %X) nounwind {
+; CHECK-LABEL: t3_wide:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov x9, #-6148914691236517206
+; CHECK-NEXT:    adrp x11, .LCPI4_0
+; CHECK-NEXT:    mov x8, v0.d[1]
+; CHECK-NEXT:    movk x9, #43691
+; CHECK-NEXT:    fmov x10, d0
+; CHECK-NEXT:    ldr q0, [x11, :lo12:.LCPI4_0]
+; CHECK-NEXT:    mul x10, x10, x9
+; CHECK-NEXT:    mul x8, x8, x9
+; CHECK-NEXT:    fmov d1, x10
+; CHECK-NEXT:    mov v1.d[1], x8
+; CHECK-NEXT:    cmhs v0.2d, v0.2d, v1.2d
+; CHECK-NEXT:    xtn v0.2s, v0.2d
+; CHECK-NEXT:    movi d1, #0xffffffff00000000
+; CHECK-NEXT:    eor v0.8b, v0.8b, v1.8b
+; CHECK-NEXT:    ret
+  %urem = urem <2 x i64> %X, <i64 3, i64 1>
+  %cmp = icmp eq <2 x i64> %urem, <i64 0, i64 42>
+  ret <2 x i1> %cmp
+}
diff --git a/llvm/test/CodeGen/AArch64/urem-seteq.ll b/llvm/test/CodeGen/AArch64/urem-seteq.ll
index 164c0a5f1..69e8e8257 100644
--- a/llvm/test/CodeGen/AArch64/urem-seteq.ll
+++ b/llvm/test/CodeGen/AArch64/urem-seteq.ll
@@ -5,10 +5,6 @@
 ; Odd divisors
 ;------------------------------------------------------------------------------;
 
-; This tests the BuildREMEqFold optimization with UREM, i32, odd divisor, SETEQ.
-; The corresponding pseudocode is:
-; Q <- [N * multInv(5, 2^32)] <=> [N * 0xCCCCCCCD] <=> [N * (-858993459)]
-; res <- [Q <= (2^32 - 1) / 5] <=> [Q <= 858993459] <=> [Q < 858993460]
 define i32 @test_urem_odd(i32 %X) nounwind {
 ; CHECK-LABEL: test_urem_odd:
 ; CHECK:       // %bb.0:
@@ -79,12 +75,6 @@ define i32 @test_urem_odd_bit31(i32 %X) nounwind {
 ; Even divisors
 ;------------------------------------------------------------------------------;
 
-; This tests the BuildREMEqFold optimization with UREM, i16, even divisor, SETNE.
-; In this case, D <=> 14 <=> 7 * 2^1, so D0 = 7 and K = 1.
-; The corresponding pseudocode is:
-; Q <- [N * multInv(D0, 2^16)] <=> [N * multInv(7, 2^16)] <=> [N * 28087]
-; Q <- [Q >>rot K] <=> [Q >>rot 1]
-; res <- ![Q <= (2^16 - 1) / 7] <=> ![Q <= 9362] <=> [Q > 9362]
 define i16 @test_urem_even(i16 %X) nounwind {
 ; CHECK-LABEL: test_urem_even:
 ; CHECK:       // %bb.0:
@@ -177,11 +167,41 @@ define i32 @test_urem_odd_setne(i32 %X) nounwind {
   ret i32 %ret
 }
 
+; The fold is only valid for positive divisors, negative-ones should be negated.
+define i32 @test_urem_negative_odd(i32 %X) nounwind {
+; CHECK-LABEL: test_urem_negative_odd:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #858993459
+; CHECK-NEXT:    mul w8, w0, w8
+; CHECK-NEXT:    cmp w8, #1 // =1
+; CHECK-NEXT:    cset w0, hi
+; CHECK-NEXT:    ret
+  %urem = urem i32 %X, -5
+  %cmp = icmp ne i32 %urem, 0
+  %ret = zext i1 %cmp to i32
+  ret i32 %ret
+}
+define i32 @test_urem_negative_even(i32 %X) nounwind {
+; CHECK-LABEL: test_urem_negative_even:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #37449
+; CHECK-NEXT:    movk w8, #51492, lsl #16
+; CHECK-NEXT:    mul w8, w0, w8
+; CHECK-NEXT:    ror w8, w8, #1
+; CHECK-NEXT:    cmp w8, #1 // =1
+; CHECK-NEXT:    cset w0, hi
+; CHECK-NEXT:    ret
+  %urem = urem i32 %X, -14
+  %cmp = icmp ne i32 %urem, 0
+  %ret = zext i1 %cmp to i32
+  ret i32 %ret
+}
+
 ;------------------------------------------------------------------------------;
 ; Negative tests
 ;------------------------------------------------------------------------------;
 
-; The fold is invalid if divisor is 1.
+; We can lower remainder of division by one much better elsewhere.
 define i32 @test_urem_one(i32 %X) nounwind {
 ; CHECK-LABEL: test_urem_one:
 ; CHECK:       // %bb.0:
@@ -193,28 +213,41 @@ define i32 @test_urem_one(i32 %X) nounwind {
   ret i32 %ret
 }
 
-; We can lower remainder of division by all-ones much better elsewhere.
-define i32 @test_urem_allones(i32 %X) nounwind {
-; CHECK-LABEL: test_urem_allones:
+; We can lower remainder of division by powers of two much better elsewhere.
+define i32 @test_urem_pow2(i32 %X) nounwind {
+; CHECK-LABEL: test_urem_pow2:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    neg w8, w0
-; CHECK-NEXT:    cmp w8, #2 // =2
-; CHECK-NEXT:    cset w0, lo
+; CHECK-NEXT:    tst w0, #0xf
+; CHECK-NEXT:    cset w0, eq
 ; CHECK-NEXT:    ret
-  %urem = urem i32 %X, 4294967295
+  %urem = urem i32 %X, 16
   %cmp = icmp eq i32 %urem, 0
   %ret = zext i1 %cmp to i32
   ret i32 %ret
 }
 
-; We can lower remainder of division by powers of two much better elsewhere.
-define i32 @test_urem_pow2(i32 %X) nounwind {
-; CHECK-LABEL: test_urem_pow2:
+; The fold is only valid for positive divisors, and we can't negate INT_MIN.
+define i32 @test_urem_int_min(i32 %X) nounwind {
+; CHECK-LABEL: test_urem_int_min:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    tst w0, #0xf
+; CHECK-NEXT:    tst w0, #0x7fffffff
 ; CHECK-NEXT:    cset w0, eq
 ; CHECK-NEXT:    ret
-  %urem = urem i32 %X, 16
+  %urem = urem i32 %X, 2147483648
+  %cmp = icmp eq i32 %urem, 0
+  %ret = zext i1 %cmp to i32
+  ret i32 %ret
+}
+
+; We can lower remainder of division by all-ones much better elsewhere.
+define i32 @test_urem_allones(i32 %X) nounwind {
+; CHECK-LABEL: test_urem_allones:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    neg w8, w0
+; CHECK-NEXT:    cmp w8, #2 // =2
+; CHECK-NEXT:    cset w0, lo
+; CHECK-NEXT:    ret
+  %urem = urem i32 %X, 4294967295
   %cmp = icmp eq i32 %urem, 0
   %ret = zext i1 %cmp to i32
   ret i32 %ret
diff --git a/llvm/test/CodeGen/AArch64/urem-vector-lkk.ll b/llvm/test/CodeGen/AArch64/urem-vector-lkk.ll
new file mode 100644
index 000000000..c5951a499
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/urem-vector-lkk.ll
@@ -0,0 +1,267 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64-unknown-linux-gnu < %s | FileCheck %s
+
+define <4 x i16> @fold_urem_vec_1(<4 x i16> %x) {
+; CHECK-LABEL: fold_urem_vec_1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w11, #33437
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    umov w10, v0.h[2]
+; CHECK-NEXT:    movk w11, #21399, lsl #16
+; CHECK-NEXT:    umull x11, w10, w11
+; CHECK-NEXT:    umov w8, v0.h[1]
+; CHECK-NEXT:    mov w9, #16913
+; CHECK-NEXT:    mov w12, #98
+; CHECK-NEXT:    lsr x11, x11, #37
+; CHECK-NEXT:    movk w9, #8456, lsl #16
+; CHECK-NEXT:    msub w10, w11, w12, w10
+; CHECK-NEXT:    ubfx w12, w8, #2, #14
+; CHECK-NEXT:    umull x9, w12, w9
+; CHECK-NEXT:    mov w11, #124
+; CHECK-NEXT:    lsr x9, x9, #34
+; CHECK-NEXT:    msub w8, w9, w11, w8
+; CHECK-NEXT:    mov w9, #8969
+; CHECK-NEXT:    umov w12, v0.h[0]
+; CHECK-NEXT:    movk w9, #22765, lsl #16
+; CHECK-NEXT:    umull x9, w12, w9
+; CHECK-NEXT:    lsr x9, x9, #32
+; CHECK-NEXT:    sub w11, w12, w9
+; CHECK-NEXT:    add w9, w9, w11, lsr #1
+; CHECK-NEXT:    mov w11, #95
+; CHECK-NEXT:    lsr w9, w9, #6
+; CHECK-NEXT:    msub w9, w9, w11, w12
+; CHECK-NEXT:    umov w11, v0.h[3]
+; CHECK-NEXT:    fmov s0, w9
+; CHECK-NEXT:    mov w9, #2287
+; CHECK-NEXT:    movk w9, #16727, lsl #16
+; CHECK-NEXT:    umull x9, w11, w9
+; CHECK-NEXT:    mov v0.h[1], w8
+; CHECK-NEXT:    mov w8, #1003
+; CHECK-NEXT:    lsr x9, x9, #40
+; CHECK-NEXT:    mov v0.h[2], w10
+; CHECK-NEXT:    msub w8, w9, w8, w11
+; CHECK-NEXT:    mov v0.h[3], w8
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    ret
+  %1 = urem <4 x i16> %x, <i16 95, i16 124, i16 98, i16 1003>
+  ret <4 x i16> %1
+}
+
+define <4 x i16> @fold_urem_vec_2(<4 x i16> %x) {
+; CHECK-LABEL: fold_urem_vec_2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w9, #8969
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    umov w8, v0.h[1]
+; CHECK-NEXT:    movk w9, #22765, lsl #16
+; CHECK-NEXT:    umov w10, v0.h[0]
+; CHECK-NEXT:    umull x13, w8, w9
+; CHECK-NEXT:    umov w11, v0.h[2]
+; CHECK-NEXT:    umull x14, w10, w9
+; CHECK-NEXT:    lsr x13, x13, #32
+; CHECK-NEXT:    umov w12, v0.h[3]
+; CHECK-NEXT:    umull x15, w11, w9
+; CHECK-NEXT:    lsr x14, x14, #32
+; CHECK-NEXT:    sub w16, w8, w13
+; CHECK-NEXT:    umull x9, w12, w9
+; CHECK-NEXT:    lsr x15, x15, #32
+; CHECK-NEXT:    add w13, w13, w16, lsr #1
+; CHECK-NEXT:    sub w16, w10, w14
+; CHECK-NEXT:    lsr x9, x9, #32
+; CHECK-NEXT:    add w14, w14, w16, lsr #1
+; CHECK-NEXT:    sub w16, w11, w15
+; CHECK-NEXT:    add w15, w15, w16, lsr #1
+; CHECK-NEXT:    sub w16, w12, w9
+; CHECK-NEXT:    add w9, w9, w16, lsr #1
+; CHECK-NEXT:    mov w16, #95
+; CHECK-NEXT:    lsr w13, w13, #6
+; CHECK-NEXT:    msub w8, w13, w16, w8
+; CHECK-NEXT:    lsr w13, w14, #6
+; CHECK-NEXT:    msub w10, w13, w16, w10
+; CHECK-NEXT:    lsr w13, w15, #6
+; CHECK-NEXT:    fmov s0, w10
+; CHECK-NEXT:    msub w11, w13, w16, w11
+; CHECK-NEXT:    lsr w9, w9, #6
+; CHECK-NEXT:    mov v0.h[1], w8
+; CHECK-NEXT:    mov v0.h[2], w11
+; CHECK-NEXT:    msub w8, w9, w16, w12
+; CHECK-NEXT:    mov v0.h[3], w8
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    ret
+  %1 = urem <4 x i16> %x, <i16 95, i16 95, i16 95, i16 95>
+  ret <4 x i16> %1
+}
+
+
+; Don't fold if we can combine urem with udiv.
+define <4 x i16> @combine_urem_udiv(<4 x i16> %x) {
+; CHECK-LABEL: combine_urem_udiv:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #8969
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    movk w8, #22765, lsl #16
+; CHECK-NEXT:    umov w9, v0.h[1]
+; CHECK-NEXT:    umov w10, v0.h[0]
+; CHECK-NEXT:    umull x13, w9, w8
+; CHECK-NEXT:    umov w11, v0.h[2]
+; CHECK-NEXT:    umull x14, w10, w8
+; CHECK-NEXT:    lsr x13, x13, #32
+; CHECK-NEXT:    umov w12, v0.h[3]
+; CHECK-NEXT:    umull x15, w11, w8
+; CHECK-NEXT:    lsr x14, x14, #32
+; CHECK-NEXT:    sub w16, w9, w13
+; CHECK-NEXT:    umull x8, w12, w8
+; CHECK-NEXT:    lsr x15, x15, #32
+; CHECK-NEXT:    add w13, w13, w16, lsr #1
+; CHECK-NEXT:    sub w16, w10, w14
+; CHECK-NEXT:    lsr x8, x8, #32
+; CHECK-NEXT:    add w14, w14, w16, lsr #1
+; CHECK-NEXT:    sub w16, w11, w15
+; CHECK-NEXT:    add w15, w15, w16, lsr #1
+; CHECK-NEXT:    sub w16, w12, w8
+; CHECK-NEXT:    add w8, w8, w16, lsr #1
+; CHECK-NEXT:    mov w16, #95
+; CHECK-NEXT:    lsr w14, w14, #6
+; CHECK-NEXT:    lsr w13, w13, #6
+; CHECK-NEXT:    msub w10, w14, w16, w10
+; CHECK-NEXT:    lsr w15, w15, #6
+; CHECK-NEXT:    msub w9, w13, w16, w9
+; CHECK-NEXT:    fmov s0, w14
+; CHECK-NEXT:    fmov s1, w10
+; CHECK-NEXT:    lsr w8, w8, #6
+; CHECK-NEXT:    msub w11, w15, w16, w11
+; CHECK-NEXT:    mov v0.h[1], w13
+; CHECK-NEXT:    mov v1.h[1], w9
+; CHECK-NEXT:    msub w12, w8, w16, w12
+; CHECK-NEXT:    mov v0.h[2], w15
+; CHECK-NEXT:    mov v1.h[2], w11
+; CHECK-NEXT:    mov v1.h[3], w12
+; CHECK-NEXT:    mov v0.h[3], w8
+; CHECK-NEXT:    add v0.4h, v1.4h, v0.4h
+; CHECK-NEXT:    ret
+  %1 = urem <4 x i16> %x, <i16 95, i16 95, i16 95, i16 95>
+  %2 = udiv <4 x i16> %x, <i16 95, i16 95, i16 95, i16 95>
+  %3 = add <4 x i16> %1, %2
+  ret <4 x i16> %3
+}
+
+
+; Don't fold for divisors that are a power of two.
+define <4 x i16> @dont_fold_urem_power_of_two(<4 x i16> %x) {
+; CHECK-LABEL: dont_fold_urem_power_of_two:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w9, #8969
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    umov w8, v0.h[3]
+; CHECK-NEXT:    movk w9, #22765, lsl #16
+; CHECK-NEXT:    umull x9, w8, w9
+; CHECK-NEXT:    lsr x9, x9, #32
+; CHECK-NEXT:    sub w10, w8, w9
+; CHECK-NEXT:    add w9, w9, w10, lsr #1
+; CHECK-NEXT:    mov w10, #95
+; CHECK-NEXT:    lsr w9, w9, #6
+; CHECK-NEXT:    msub w8, w9, w10, w8
+; CHECK-NEXT:    umov w9, v0.h[0]
+; CHECK-NEXT:    and w9, w9, #0x3f
+; CHECK-NEXT:    umov w10, v0.h[1]
+; CHECK-NEXT:    fmov s1, w9
+; CHECK-NEXT:    umov w9, v0.h[2]
+; CHECK-NEXT:    and w10, w10, #0x1f
+; CHECK-NEXT:    and w9, w9, #0x7
+; CHECK-NEXT:    mov v1.h[1], w10
+; CHECK-NEXT:    mov v1.h[2], w9
+; CHECK-NEXT:    mov v1.h[3], w8
+; CHECK-NEXT:    mov v0.16b, v1.16b
+; CHECK-NEXT:    ret
+  %1 = urem <4 x i16> %x, <i16 64, i16 32, i16 8, i16 95>
+  ret <4 x i16> %1
+}
+
+; Don't fold if the divisor is one.
+define <4 x i16> @dont_fold_srem_one(<4 x i16> %x) {
+; CHECK-LABEL: dont_fold_srem_one:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w9, #17097
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    umov w8, v0.h[2]
+; CHECK-NEXT:    movk w9, #45590, lsl #16
+; CHECK-NEXT:    umull x9, w8, w9
+; CHECK-NEXT:    mov w10, #23
+; CHECK-NEXT:    lsr x9, x9, #36
+; CHECK-NEXT:    umov w11, v0.h[1]
+; CHECK-NEXT:    msub w8, w9, w10, w8
+; CHECK-NEXT:    mov w9, #30865
+; CHECK-NEXT:    movk w9, #51306, lsl #16
+; CHECK-NEXT:    ubfx w10, w11, #1, #15
+; CHECK-NEXT:    umull x9, w10, w9
+; CHECK-NEXT:    mov w10, #654
+; CHECK-NEXT:    lsr x9, x9, #40
+; CHECK-NEXT:    msub w9, w9, w10, w11
+; CHECK-NEXT:    mov w11, #47143
+; CHECK-NEXT:    umov w10, v0.h[3]
+; CHECK-NEXT:    movk w11, #24749, lsl #16
+; CHECK-NEXT:    movi d1, #0000000000000000
+; CHECK-NEXT:    umull x11, w10, w11
+; CHECK-NEXT:    mov v1.h[1], w9
+; CHECK-NEXT:    mov w9, #5423
+; CHECK-NEXT:    lsr x11, x11, #43
+; CHECK-NEXT:    mov v1.h[2], w8
+; CHECK-NEXT:    msub w8, w11, w9, w10
+; CHECK-NEXT:    mov v1.h[3], w8
+; CHECK-NEXT:    mov v0.16b, v1.16b
+; CHECK-NEXT:    ret
+  %1 = urem <4 x i16> %x, <i16 1, i16 654, i16 23, i16 5423>
+  ret <4 x i16> %1
+}
+
+; Don't fold if the divisor is 2^16.
+define <4 x i16> @dont_fold_urem_i16_smax(<4 x i16> %x) {
+; CHECK-LABEL: dont_fold_urem_i16_smax:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ret
+  %1 = urem <4 x i16> %x, <i16 1, i16 65536, i16 23, i16 5423>
+  ret <4 x i16> %1
+}
+
+; Don't fold i64 urem.
+define <4 x i64> @dont_fold_urem_i64(<4 x i64> %x) {
+; CHECK-LABEL: dont_fold_urem_i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov x10, #12109
+; CHECK-NEXT:    movk x10, #52170, lsl #16
+; CHECK-NEXT:    movk x10, #28749, lsl #32
+; CHECK-NEXT:    mov x8, v1.d[1]
+; CHECK-NEXT:    movk x10, #49499, lsl #48
+; CHECK-NEXT:    umulh x10, x8, x10
+; CHECK-NEXT:    mov w11, #5423
+; CHECK-NEXT:    lsr x10, x10, #12
+; CHECK-NEXT:    msub x8, x10, x11, x8
+; CHECK-NEXT:    mov x10, #21445
+; CHECK-NEXT:    movk x10, #1603, lsl #16
+; CHECK-NEXT:    mov x12, v0.d[1]
+; CHECK-NEXT:    movk x10, #15432, lsl #32
+; CHECK-NEXT:    movk x10, #25653, lsl #48
+; CHECK-NEXT:    lsr x11, x12, #1
+; CHECK-NEXT:    umulh x10, x11, x10
+; CHECK-NEXT:    mov w11, #654
+; CHECK-NEXT:    lsr x10, x10, #7
+; CHECK-NEXT:    msub x10, x10, x11, x12
+; CHECK-NEXT:    mov x11, #17097
+; CHECK-NEXT:    movk x11, #45590, lsl #16
+; CHECK-NEXT:    movk x11, #34192, lsl #32
+; CHECK-NEXT:    fmov x9, d1
+; CHECK-NEXT:    movk x11, #25644, lsl #48
+; CHECK-NEXT:    umulh x11, x9, x11
+; CHECK-NEXT:    sub x12, x9, x11
+; CHECK-NEXT:    add x11, x11, x12, lsr #1
+; CHECK-NEXT:    mov w12, #23
+; CHECK-NEXT:    lsr x11, x11, #4
+; CHECK-NEXT:    msub x9, x11, x12, x9
+; CHECK-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-NEXT:    fmov d1, x9
+; CHECK-NEXT:    mov v1.d[1], x8
+; CHECK-NEXT:    mov v0.d[1], x10
+; CHECK-NEXT:    ret
+  %1 = urem <4 x i64> %x, <i64 1, i64 654, i64 23, i64 5423>
+  ret <4 x i64> %1
+}
diff --git a/llvm/test/CodeGen/AArch64/use-cr-result-of-dom-icmp-st.ll b/llvm/test/CodeGen/AArch64/use-cr-result-of-dom-icmp-st.ll
new file mode 100644
index 000000000..2983e2409
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/use-cr-result-of-dom-icmp-st.ll
@@ -0,0 +1,547 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64-unknown-unknown -O3 -cgp-icmp-eq2icmp-st -verify-machineinstrs < %s | FileCheck %s
+
+; Test cases are generated from:
+; long long NAME(PARAM a, PARAM b) {
+;   if (LHS > RHS)
+;     return b;
+;   if (LHS < RHS)
+;     return a;\
+;   return a * b;
+; }
+; Please note funtion name is defined as <PARAM>_<LHS>_<RHS>. Take ll_a_op_b__1
+; for example. ll is PARAM, a_op_b (i.e., a << b) is LHS, _1 (i.e., -1) is RHS.
+
+target datalayout = "e-m:e-i64:64-n32:64"
+
+define i64 @ll_a_op_b__2(i64 %a, i64 %b) {
+; CHECK-LABEL: ll_a_op_b__2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    lsl x8, x0, x1
+; CHECK-NEXT:    cmn x8, #2 // =2
+; CHECK-NEXT:    b.le .LBB0_2
+; CHECK-NEXT:  // %bb.1: // %return
+; CHECK-NEXT:    mov x0, x1
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB0_2: // %if.end
+; CHECK-NEXT:    csinc x8, x1, xzr, ge
+; CHECK-NEXT:    mul x0, x8, x0
+; CHECK-NEXT:    ret
+entry:
+  %shl = shl i64 %a, %b
+  %cmp = icmp sgt i64 %shl, -2
+  br i1 %cmp, label %return, label %if.end
+
+if.end:                                           ; preds = %entry
+  %cmp2 = icmp eq i64 %shl, -2
+  %mul = select i1 %cmp2, i64 %b, i64 1
+  %spec.select = mul nsw i64 %mul, %a
+  ret i64 %spec.select
+
+return:                                           ; preds = %entry
+  ret i64 %b
+}
+
+define i64 @ll_a_op_b__1(i64 %a, i64 %b) {
+; CHECK-LABEL: ll_a_op_b__1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    lsl x8, x0, x1
+; CHECK-NEXT:    tbnz x8, #63, .LBB1_2
+; CHECK-NEXT:  // %bb.1: // %return
+; CHECK-NEXT:    mov x0, x1
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB1_2: // %if.end
+; CHECK-NEXT:    cmn x8, #1 // =1
+; CHECK-NEXT:    csinc x8, x1, xzr, ge
+; CHECK-NEXT:    mul x0, x8, x0
+; CHECK-NEXT:    ret
+entry:
+  %shl = shl i64 %a, %b
+  %cmp = icmp sgt i64 %shl, -1
+  br i1 %cmp, label %return, label %if.end
+
+if.end:                                           ; preds = %entry
+  %cmp2 = icmp eq i64 %shl, -1
+  %mul = select i1 %cmp2, i64 %b, i64 1
+  %spec.select = mul nsw i64 %mul, %a
+  ret i64 %spec.select
+
+return:                                           ; preds = %entry
+  ret i64 %b
+}
+
+define i64 @ll_a_op_b_0(i64 %a, i64 %b) {
+; CHECK-LABEL: ll_a_op_b_0:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    lsl x8, x0, x1
+; CHECK-NEXT:    cmp x8, #0 // =0
+; CHECK-NEXT:    b.le .LBB2_2
+; CHECK-NEXT:  // %bb.1: // %return
+; CHECK-NEXT:    mov x0, x1
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB2_2: // %if.end
+; CHECK-NEXT:    csinc x8, x1, xzr, ge
+; CHECK-NEXT:    mul x0, x8, x0
+; CHECK-NEXT:    ret
+entry:
+  %shl = shl i64 %a, %b
+  %cmp = icmp sgt i64 %shl, 0
+  br i1 %cmp, label %return, label %if.end
+
+if.end:                                           ; preds = %entry
+  %cmp2 = icmp eq i64 %shl, 0
+  %mul = select i1 %cmp2, i64 %b, i64 1
+  %spec.select = mul nsw i64 %mul, %a
+  ret i64 %spec.select
+
+return:                                           ; preds = %entry
+  ret i64 %b
+}
+
+define i64 @ll_a_op_b_1(i64 %a, i64 %b) {
+; CHECK-LABEL: ll_a_op_b_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    lsl x8, x0, x1
+; CHECK-NEXT:    cmp x8, #1 // =1
+; CHECK-NEXT:    b.le .LBB3_2
+; CHECK-NEXT:  // %bb.1: // %return
+; CHECK-NEXT:    mov x0, x1
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB3_2: // %if.end
+; CHECK-NEXT:    csinc x8, x1, xzr, ge
+; CHECK-NEXT:    mul x0, x8, x0
+; CHECK-NEXT:    ret
+entry:
+  %shl = shl i64 %a, %b
+  %cmp = icmp sgt i64 %shl, 1
+  br i1 %cmp, label %return, label %if.end
+
+if.end:                                           ; preds = %entry
+  %cmp2 = icmp eq i64 %shl, 1
+  %mul = select i1 %cmp2, i64 %b, i64 1
+  %spec.select = mul nsw i64 %mul, %a
+  ret i64 %spec.select
+
+return:                                           ; preds = %entry
+  ret i64 %b
+}
+
+define i64 @ll_a_op_b_2(i64 %a, i64 %b) {
+; CHECK-LABEL: ll_a_op_b_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    lsl x8, x0, x1
+; CHECK-NEXT:    cmp x8, #2 // =2
+; CHECK-NEXT:    b.le .LBB4_2
+; CHECK-NEXT:  // %bb.1: // %return
+; CHECK-NEXT:    mov x0, x1
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB4_2: // %if.end
+; CHECK-NEXT:    csinc x8, x1, xzr, ge
+; CHECK-NEXT:    mul x0, x8, x0
+; CHECK-NEXT:    ret
+entry:
+  %shl = shl i64 %a, %b
+  %cmp = icmp sgt i64 %shl, 2
+  br i1 %cmp, label %return, label %if.end
+
+if.end:                                           ; preds = %entry
+  %cmp2 = icmp eq i64 %shl, 2
+  %mul = select i1 %cmp2, i64 %b, i64 1
+  %spec.select = mul nsw i64 %mul, %a
+  ret i64 %spec.select
+
+return:                                           ; preds = %entry
+  ret i64 %b
+}
+
+define i64 @ll_a__2(i64 %a, i64 %b) {
+; CHECK-LABEL: ll_a__2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    cmn x0, #2 // =2
+; CHECK-NEXT:    b.le .LBB5_2
+; CHECK-NEXT:  // %bb.1: // %return
+; CHECK-NEXT:    mov x0, x1
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB5_2: // %if.end
+; CHECK-NEXT:    csinc x8, x1, xzr, ge
+; CHECK-NEXT:    mul x0, x8, x0
+; CHECK-NEXT:    ret
+entry:
+  %cmp = icmp sgt i64 %a, -2
+  br i1 %cmp, label %return, label %if.end
+
+if.end:                                           ; preds = %entry
+  %cmp1 = icmp eq i64 %a, -2
+  %mul = select i1 %cmp1, i64 %b, i64 1
+  %spec.select = mul nsw i64 %mul, %a
+  ret i64 %spec.select
+
+return:                                           ; preds = %entry
+  ret i64 %b
+}
+
+define i64 @ll_a__1(i64 %a, i64 %b) {
+; CHECK-LABEL: ll_a__1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    tbnz x0, #63, .LBB6_2
+; CHECK-NEXT:  // %bb.1: // %return
+; CHECK-NEXT:    mov x0, x1
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB6_2: // %if.end
+; CHECK-NEXT:    cmn x0, #1 // =1
+; CHECK-NEXT:    csinc x8, x1, xzr, ge
+; CHECK-NEXT:    mul x0, x8, x0
+; CHECK-NEXT:    ret
+entry:
+  %cmp = icmp sgt i64 %a, -1
+  br i1 %cmp, label %return, label %if.end
+
+if.end:                                           ; preds = %entry
+  %cmp1 = icmp eq i64 %a, -1
+  %mul = select i1 %cmp1, i64 %b, i64 1
+  %spec.select = mul nsw i64 %mul, %a
+  ret i64 %spec.select
+
+return:                                           ; preds = %entry
+  ret i64 %b
+}
+
+define i64 @ll_a_0(i64 %a, i64 %b) {
+; CHECK-LABEL: ll_a_0:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    cmp x0, #0 // =0
+; CHECK-NEXT:    b.le .LBB7_2
+; CHECK-NEXT:  // %bb.1: // %return
+; CHECK-NEXT:    mov x0, x1
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB7_2: // %if.end
+; CHECK-NEXT:    csinc x8, x1, xzr, ge
+; CHECK-NEXT:    mul x0, x8, x0
+; CHECK-NEXT:    ret
+entry:
+  %cmp = icmp sgt i64 %a, 0
+  br i1 %cmp, label %return, label %if.end
+
+if.end:                                           ; preds = %entry
+  %cmp1 = icmp eq i64 %a, 0
+  %mul = select i1 %cmp1, i64 %b, i64 1
+  %spec.select = mul nsw i64 %mul, %a
+  ret i64 %spec.select
+
+return:                                           ; preds = %entry
+  ret i64 %b
+}
+
+define i64 @ll_a_1(i64 %a, i64 %b) {
+; CHECK-LABEL: ll_a_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    cmp x0, #1 // =1
+; CHECK-NEXT:    b.le .LBB8_2
+; CHECK-NEXT:  // %bb.1: // %return
+; CHECK-NEXT:    mov x0, x1
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB8_2: // %if.end
+; CHECK-NEXT:    csinc x8, x1, xzr, ge
+; CHECK-NEXT:    mul x0, x8, x0
+; CHECK-NEXT:    ret
+entry:
+  %cmp = icmp sgt i64 %a, 1
+  br i1 %cmp, label %return, label %if.end
+
+if.end:                                           ; preds = %entry
+  %cmp1 = icmp eq i64 %a, 1
+  %mul = select i1 %cmp1, i64 %b, i64 1
+  %spec.select = mul nsw i64 %mul, %a
+  ret i64 %spec.select
+
+return:                                           ; preds = %entry
+  ret i64 %b
+}
+
+define i64 @ll_a_2(i64 %a, i64 %b) {
+; CHECK-LABEL: ll_a_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    cmp x0, #2 // =2
+; CHECK-NEXT:    b.le .LBB9_2
+; CHECK-NEXT:  // %bb.1: // %return
+; CHECK-NEXT:    mov x0, x1
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB9_2: // %if.end
+; CHECK-NEXT:    csinc x8, x1, xzr, ge
+; CHECK-NEXT:    mul x0, x8, x0
+; CHECK-NEXT:    ret
+entry:
+  %cmp = icmp sgt i64 %a, 2
+  br i1 %cmp, label %return, label %if.end
+
+if.end:                                           ; preds = %entry
+  %cmp1 = icmp eq i64 %a, 2
+  %mul = select i1 %cmp1, i64 %b, i64 1
+  %spec.select = mul nsw i64 %mul, %a
+  ret i64 %spec.select
+
+return:                                           ; preds = %entry
+  ret i64 %b
+}
+
+define i64 @i_a_op_b__2(i32 signext %a, i32 signext %b) {
+; CHECK-LABEL: i_a_op_b__2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    lsl w8, w0, w1
+; CHECK-NEXT:    cmn w8, #2 // =2
+; CHECK-NEXT:    csinc w8, w1, wzr, eq
+; CHECK-NEXT:    mul w8, w8, w0
+; CHECK-NEXT:    csel w8, w1, w8, gt
+; CHECK-NEXT:    sxtw x0, w8
+; CHECK-NEXT:    ret
+entry:
+  %shl = shl i32 %a, %b
+  %cmp = icmp sgt i32 %shl, -2
+  br i1 %cmp, label %return, label %if.end
+
+if.end:                                           ; preds = %entry
+  %cmp2 = icmp eq i32 %shl, -2
+  %mul = select i1 %cmp2, i32 %b, i32 1
+  %spec.select = mul nsw i32 %mul, %a
+  br label %return
+
+return:                                           ; preds = %if.end, %entry
+  %retval.0.in = phi i32 [ %b, %entry ], [ %spec.select, %if.end ]
+  %retval.0 = sext i32 %retval.0.in to i64
+  ret i64 %retval.0
+}
+
+define i64 @i_a_op_b__1(i32 signext %a, i32 signext %b) {
+; CHECK-LABEL: i_a_op_b__1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    lsl w8, w0, w1
+; CHECK-NEXT:    cmn w8, #1 // =1
+; CHECK-NEXT:    csinc w9, w1, wzr, eq
+; CHECK-NEXT:    mul w9, w9, w0
+; CHECK-NEXT:    cmp w8, #0 // =0
+; CHECK-NEXT:    csel w8, w1, w9, ge
+; CHECK-NEXT:    sxtw x0, w8
+; CHECK-NEXT:    ret
+entry:
+  %shl = shl i32 %a, %b
+  %cmp = icmp sgt i32 %shl, -1
+  br i1 %cmp, label %return, label %if.end
+
+if.end:                                           ; preds = %entry
+  %cmp2 = icmp eq i32 %shl, -1
+  %mul = select i1 %cmp2, i32 %b, i32 1
+  %spec.select = mul nsw i32 %mul, %a
+  br label %return
+
+return:                                           ; preds = %if.end, %entry
+  %retval.0.in = phi i32 [ %b, %entry ], [ %spec.select, %if.end ]
+  %retval.0 = sext i32 %retval.0.in to i64
+  ret i64 %retval.0
+}
+
+define i64 @i_a_op_b_0(i32 signext %a, i32 signext %b) {
+; CHECK-LABEL: i_a_op_b_0:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    lsl w8, w0, w1
+; CHECK-NEXT:    cmp w8, #0 // =0
+; CHECK-NEXT:    csinc w8, w1, wzr, eq
+; CHECK-NEXT:    mul w8, w8, w0
+; CHECK-NEXT:    csel w8, w1, w8, gt
+; CHECK-NEXT:    sxtw x0, w8
+; CHECK-NEXT:    ret
+entry:
+  %shl = shl i32 %a, %b
+  %cmp = icmp sgt i32 %shl, 0
+  br i1 %cmp, label %return, label %if.end
+
+if.end:                                           ; preds = %entry
+  %cmp2 = icmp eq i32 %shl, 0
+  %mul = select i1 %cmp2, i32 %b, i32 1
+  %spec.select = mul nsw i32 %mul, %a
+  br label %return
+
+return:                                           ; preds = %if.end, %entry
+  %retval.0.in = phi i32 [ %b, %entry ], [ %spec.select, %if.end ]
+  %retval.0 = sext i32 %retval.0.in to i64
+  ret i64 %retval.0
+}
+
+define i64 @i_a_op_b_1(i32 signext %a, i32 signext %b) {
+; CHECK-LABEL: i_a_op_b_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    lsl w8, w0, w1
+; CHECK-NEXT:    cmp w8, #1 // =1
+; CHECK-NEXT:    csinc w8, w1, wzr, eq
+; CHECK-NEXT:    mul w8, w8, w0
+; CHECK-NEXT:    csel w8, w1, w8, gt
+; CHECK-NEXT:    sxtw x0, w8
+; CHECK-NEXT:    ret
+entry:
+  %shl = shl i32 %a, %b
+  %cmp = icmp sgt i32 %shl, 1
+  br i1 %cmp, label %return, label %if.end
+
+if.end:                                           ; preds = %entry
+  %cmp2 = icmp eq i32 %shl, 1
+  %mul = select i1 %cmp2, i32 %b, i32 1
+  %spec.select = mul nsw i32 %mul, %a
+  br label %return
+
+return:                                           ; preds = %if.end, %entry
+  %retval.0.in = phi i32 [ %b, %entry ], [ %spec.select, %if.end ]
+  %retval.0 = sext i32 %retval.0.in to i64
+  ret i64 %retval.0
+}
+
+define i64 @i_a_op_b_2(i32 signext %a, i32 signext %b) {
+; CHECK-LABEL: i_a_op_b_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    lsl w8, w0, w1
+; CHECK-NEXT:    cmp w8, #2 // =2
+; CHECK-NEXT:    csinc w8, w1, wzr, eq
+; CHECK-NEXT:    mul w8, w8, w0
+; CHECK-NEXT:    csel w8, w1, w8, gt
+; CHECK-NEXT:    sxtw x0, w8
+; CHECK-NEXT:    ret
+entry:
+  %shl = shl i32 %a, %b
+  %cmp = icmp sgt i32 %shl, 2
+  br i1 %cmp, label %return, label %if.end
+
+if.end:                                           ; preds = %entry
+  %cmp2 = icmp eq i32 %shl, 2
+  %mul = select i1 %cmp2, i32 %b, i32 1
+  %spec.select = mul nsw i32 %mul, %a
+  br label %return
+
+return:                                           ; preds = %if.end, %entry
+  %retval.0.in = phi i32 [ %b, %entry ], [ %spec.select, %if.end ]
+  %retval.0 = sext i32 %retval.0.in to i64
+  ret i64 %retval.0
+}
+
+define i64 @i_a__2(i32 signext %a, i32 signext %b) {
+; CHECK-LABEL: i_a__2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    cmn w0, #2 // =2
+; CHECK-NEXT:    csinc w8, w1, wzr, eq
+; CHECK-NEXT:    mul w8, w8, w0
+; CHECK-NEXT:    csel w8, w1, w8, gt
+; CHECK-NEXT:    sxtw x0, w8
+; CHECK-NEXT:    ret
+entry:
+  %cmp = icmp sgt i32 %a, -2
+  br i1 %cmp, label %return, label %if.end
+
+if.end:                                           ; preds = %entry
+  %cmp1 = icmp eq i32 %a, -2
+  %mul = select i1 %cmp1, i32 %b, i32 1
+  %spec.select = mul nsw i32 %mul, %a
+  br label %return
+
+return:                                           ; preds = %if.end, %entry
+  %retval.0.in = phi i32 [ %b, %entry ], [ %spec.select, %if.end ]
+  %retval.0 = sext i32 %retval.0.in to i64
+  ret i64 %retval.0
+}
+
+define i64 @i_a__1(i32 signext %a, i32 signext %b) {
+; CHECK-LABEL: i_a__1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    cmn w0, #1 // =1
+; CHECK-NEXT:    csinc w8, w1, wzr, eq
+; CHECK-NEXT:    mul w8, w8, w0
+; CHECK-NEXT:    cmp w0, #0 // =0
+; CHECK-NEXT:    csel w8, w1, w8, ge
+; CHECK-NEXT:    sxtw x0, w8
+; CHECK-NEXT:    ret
+entry:
+  %cmp = icmp sgt i32 %a, -1
+  br i1 %cmp, label %return, label %if.end
+
+if.end:                                           ; preds = %entry
+  %cmp1 = icmp eq i32 %a, -1
+  %mul = select i1 %cmp1, i32 %b, i32 1
+  %spec.select = mul nsw i32 %mul, %a
+  br label %return
+
+return:                                           ; preds = %if.end, %entry
+  %retval.0.in = phi i32 [ %b, %entry ], [ %spec.select, %if.end ]
+  %retval.0 = sext i32 %retval.0.in to i64
+  ret i64 %retval.0
+}
+
+define i64 @i_a_0(i32 signext %a, i32 signext %b) {
+; CHECK-LABEL: i_a_0:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    cmp w0, #0 // =0
+; CHECK-NEXT:    csinc w8, w1, wzr, eq
+; CHECK-NEXT:    mul w8, w8, w0
+; CHECK-NEXT:    csel w8, w1, w8, gt
+; CHECK-NEXT:    sxtw x0, w8
+; CHECK-NEXT:    ret
+entry:
+  %cmp = icmp sgt i32 %a, 0
+  br i1 %cmp, label %return, label %if.end
+
+if.end:                                           ; preds = %entry
+  %cmp1 = icmp eq i32 %a, 0
+  %mul = select i1 %cmp1, i32 %b, i32 1
+  %spec.select = mul nsw i32 %mul, %a
+  br label %return
+
+return:                                           ; preds = %if.end, %entry
+  %retval.0.in = phi i32 [ %b, %entry ], [ %spec.select, %if.end ]
+  %retval.0 = sext i32 %retval.0.in to i64
+  ret i64 %retval.0
+}
+
+define i64 @i_a_1(i32 signext %a, i32 signext %b) {
+; CHECK-LABEL: i_a_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    cmp w0, #1 // =1
+; CHECK-NEXT:    csinc w8, w1, wzr, eq
+; CHECK-NEXT:    mul w8, w8, w0
+; CHECK-NEXT:    csel w8, w1, w8, gt
+; CHECK-NEXT:    sxtw x0, w8
+; CHECK-NEXT:    ret
+entry:
+  %cmp = icmp sgt i32 %a, 1
+  br i1 %cmp, label %return, label %if.end
+
+if.end:                                           ; preds = %entry
+  %cmp1 = icmp eq i32 %a, 1
+  %mul = select i1 %cmp1, i32 %b, i32 1
+  %spec.select = mul nsw i32 %mul, %a
+  br label %return
+
+return:                                           ; preds = %if.end, %entry
+  %retval.0.in = phi i32 [ %b, %entry ], [ %spec.select, %if.end ]
+  %retval.0 = sext i32 %retval.0.in to i64
+  ret i64 %retval.0
+}
+
+define i64 @i_a_2(i32 signext %a, i32 signext %b) {
+; CHECK-LABEL: i_a_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    cmp w0, #2 // =2
+; CHECK-NEXT:    csinc w8, w1, wzr, eq
+; CHECK-NEXT:    mul w8, w8, w0
+; CHECK-NEXT:    csel w8, w1, w8, gt
+; CHECK-NEXT:    sxtw x0, w8
+; CHECK-NEXT:    ret
+entry:
+  %cmp = icmp sgt i32 %a, 2
+  br i1 %cmp, label %return, label %if.end
+
+if.end:                                           ; preds = %entry
+  %cmp1 = icmp eq i32 %a, 2
+  %mul = select i1 %cmp1, i32 %b, i32 1
+  %spec.select = mul nsw i32 %mul, %a
+  br label %return
+
+return:                                           ; preds = %if.end, %entry
+  %retval.0.in = phi i32 [ %b, %entry ], [ %spec.select, %if.end ]
+  %retval.0 = sext i32 %retval.0.in to i64
+  ret i64 %retval.0
+}
diff --git a/llvm/test/CodeGen/AArch64/usub_sat.ll b/llvm/test/CodeGen/AArch64/usub_sat.ll
index c03137dc0..aa240f800 100644
--- a/llvm/test/CodeGen/AArch64/usub_sat.ll
+++ b/llvm/test/CodeGen/AArch64/usub_sat.ll
@@ -1,9 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=aarch64-- | FileCheck %s
 
-declare  i4  @llvm.usub.sat.i4   (i4,  i4)
-declare  i32 @llvm.usub.sat.i32  (i32, i32)
-declare  i64 @llvm.usub.sat.i64  (i64, i64)
+declare i4 @llvm.usub.sat.i4(i4, i4)
+declare i8 @llvm.usub.sat.i8(i8, i8)
+declare i16 @llvm.usub.sat.i16(i16, i16)
+declare i32 @llvm.usub.sat.i32(i32, i32)
+declare i64 @llvm.usub.sat.i64(i64, i64)
 
 define i32 @func(i32 %x, i32 %y) nounwind {
 ; CHECK-LABEL: func:
@@ -25,13 +27,40 @@ define i64 @func2(i64 %x, i64 %y) nounwind {
   ret i64 %tmp;
 }
 
+define i16 @func16(i16 %x, i16 %y) nounwind {
+; CHECK-LABEL: func16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    and w8, w1, #0xffff
+; CHECK-NEXT:    and w9, w0, #0xffff
+; CHECK-NEXT:    cmp w9, w8
+; CHECK-NEXT:    csel w9, w9, w8, hi
+; CHECK-NEXT:    sub w0, w9, w8
+; CHECK-NEXT:    ret
+  %tmp = call i16 @llvm.usub.sat.i16(i16 %x, i16 %y);
+  ret i16 %tmp;
+}
+
+define i8 @func8(i8 %x, i8 %y) nounwind {
+; CHECK-LABEL: func8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    and w8, w1, #0xff
+; CHECK-NEXT:    and w9, w0, #0xff
+; CHECK-NEXT:    cmp w9, w8
+; CHECK-NEXT:    csel w9, w9, w8, hi
+; CHECK-NEXT:    sub w0, w9, w8
+; CHECK-NEXT:    ret
+  %tmp = call i8 @llvm.usub.sat.i8(i8 %x, i8 %y);
+  ret i8 %tmp;
+}
+
 define i4 @func3(i4 %x, i4 %y) nounwind {
 ; CHECK-LABEL: func3:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    lsl w8, w0, #28
-; CHECK-NEXT:    subs w8, w8, w1, lsl #28
-; CHECK-NEXT:    csel w8, wzr, w8, lo
-; CHECK-NEXT:    lsr w0, w8, #28
+; CHECK-NEXT:    and w8, w1, #0xf
+; CHECK-NEXT:    and w9, w0, #0xf
+; CHECK-NEXT:    cmp w9, w8
+; CHECK-NEXT:    csel w9, w9, w8, hi
+; CHECK-NEXT:    sub w0, w9, w8
 ; CHECK-NEXT:    ret
   %tmp = call i4 @llvm.usub.sat.i4(i4 %x, i4 %y);
   ret i4 %tmp;
diff --git a/llvm/test/CodeGen/AArch64/usub_sat_plus.ll b/llvm/test/CodeGen/AArch64/usub_sat_plus.ll
new file mode 100644
index 000000000..d5cab2ea5
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/usub_sat_plus.ll
@@ -0,0 +1,76 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=aarch64-- | FileCheck %s
+
+declare i4 @llvm.usub.sat.i4(i4, i4)
+declare i8 @llvm.usub.sat.i8(i8, i8)
+declare i16 @llvm.usub.sat.i16(i16, i16)
+declare i32 @llvm.usub.sat.i32(i32, i32)
+declare i64 @llvm.usub.sat.i64(i64, i64)
+
+define i32 @func32(i32 %x, i32 %y, i32 %z) nounwind {
+; CHECK-LABEL: func32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mul w8, w1, w2
+; CHECK-NEXT:    subs w8, w0, w8
+; CHECK-NEXT:    csel w0, wzr, w8, lo
+; CHECK-NEXT:    ret
+  %a = mul i32 %y, %z
+  %tmp = call i32 @llvm.usub.sat.i32(i32 %x, i32 %a)
+  ret i32 %tmp
+}
+
+define i64 @func64(i64 %x, i64 %y, i64 %z) nounwind {
+; CHECK-LABEL: func64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    subs x8, x0, x2
+; CHECK-NEXT:    csel x0, xzr, x8, lo
+; CHECK-NEXT:    ret
+  %a = mul i64 %y, %z
+  %tmp = call i64 @llvm.usub.sat.i64(i64 %x, i64 %z)
+  ret i64 %tmp
+}
+
+define i16 @func16(i16 %x, i16 %y, i16 %z) nounwind {
+; CHECK-LABEL: func16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mul w9, w1, w2
+; CHECK-NEXT:    and w8, w0, #0xffff
+; CHECK-NEXT:    and w9, w9, #0xffff
+; CHECK-NEXT:    cmp w8, w9
+; CHECK-NEXT:    csel w8, w8, w9, hi
+; CHECK-NEXT:    sub w0, w8, w9
+; CHECK-NEXT:    ret
+  %a = mul i16 %y, %z
+  %tmp = call i16 @llvm.usub.sat.i16(i16 %x, i16 %a)
+  ret i16 %tmp
+}
+
+define i8 @func8(i8 %x, i8 %y, i8 %z) nounwind {
+; CHECK-LABEL: func8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mul w9, w1, w2
+; CHECK-NEXT:    and w8, w0, #0xff
+; CHECK-NEXT:    and w9, w9, #0xff
+; CHECK-NEXT:    cmp w8, w9
+; CHECK-NEXT:    csel w8, w8, w9, hi
+; CHECK-NEXT:    sub w0, w8, w9
+; CHECK-NEXT:    ret
+  %a = mul i8 %y, %z
+  %tmp = call i8 @llvm.usub.sat.i8(i8 %x, i8 %a)
+  ret i8 %tmp
+}
+
+define i4 @func4(i4 %x, i4 %y, i4 %z) nounwind {
+; CHECK-LABEL: func4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mul w9, w1, w2
+; CHECK-NEXT:    and w8, w0, #0xf
+; CHECK-NEXT:    and w9, w9, #0xf
+; CHECK-NEXT:    cmp w8, w9
+; CHECK-NEXT:    csel w8, w8, w9, hi
+; CHECK-NEXT:    sub w0, w8, w9
+; CHECK-NEXT:    ret
+  %a = mul i4 %y, %z
+  %tmp = call i4 @llvm.usub.sat.i4(i4 %x, i4 %a)
+  ret i4 %tmp
+}
diff --git a/llvm/test/CodeGen/AArch64/usub_sat_vec.ll b/llvm/test/CodeGen/AArch64/usub_sat_vec.ll
index b7a8be433..3eacf03dc 100644
--- a/llvm/test/CodeGen/AArch64/usub_sat_vec.ll
+++ b/llvm/test/CodeGen/AArch64/usub_sat_vec.ll
@@ -36,8 +36,7 @@ declare <2 x i128> @llvm.usub.sat.v2i128(<2 x i128>, <2 x i128>)
 define <16 x i8> @v16i8(<16 x i8> %x, <16 x i8> %y) nounwind {
 ; CHECK-LABEL: v16i8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    umax v0.16b, v0.16b, v1.16b
-; CHECK-NEXT:    sub v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    uqsub v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
   %z = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> %x, <16 x i8> %y)
   ret <16 x i8> %z
@@ -46,10 +45,8 @@ define <16 x i8> @v16i8(<16 x i8> %x, <16 x i8> %y) nounwind {
 define <32 x i8> @v32i8(<32 x i8> %x, <32 x i8> %y) nounwind {
 ; CHECK-LABEL: v32i8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    umax v0.16b, v0.16b, v2.16b
-; CHECK-NEXT:    umax v1.16b, v1.16b, v3.16b
-; CHECK-NEXT:    sub v0.16b, v0.16b, v2.16b
-; CHECK-NEXT:    sub v1.16b, v1.16b, v3.16b
+; CHECK-NEXT:    uqsub v0.16b, v0.16b, v2.16b
+; CHECK-NEXT:    uqsub v1.16b, v1.16b, v3.16b
 ; CHECK-NEXT:    ret
   %z = call <32 x i8> @llvm.usub.sat.v32i8(<32 x i8> %x, <32 x i8> %y)
   ret <32 x i8> %z
@@ -58,14 +55,10 @@ define <32 x i8> @v32i8(<32 x i8> %x, <32 x i8> %y) nounwind {
 define <64 x i8> @v64i8(<64 x i8> %x, <64 x i8> %y) nounwind {
 ; CHECK-LABEL: v64i8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    umax v0.16b, v0.16b, v4.16b
-; CHECK-NEXT:    umax v1.16b, v1.16b, v5.16b
-; CHECK-NEXT:    umax v2.16b, v2.16b, v6.16b
-; CHECK-NEXT:    umax v3.16b, v3.16b, v7.16b
-; CHECK-NEXT:    sub v0.16b, v0.16b, v4.16b
-; CHECK-NEXT:    sub v1.16b, v1.16b, v5.16b
-; CHECK-NEXT:    sub v2.16b, v2.16b, v6.16b
-; CHECK-NEXT:    sub v3.16b, v3.16b, v7.16b
+; CHECK-NEXT:    uqsub v0.16b, v0.16b, v4.16b
+; CHECK-NEXT:    uqsub v1.16b, v1.16b, v5.16b
+; CHECK-NEXT:    uqsub v2.16b, v2.16b, v6.16b
+; CHECK-NEXT:    uqsub v3.16b, v3.16b, v7.16b
 ; CHECK-NEXT:    ret
   %z = call <64 x i8> @llvm.usub.sat.v64i8(<64 x i8> %x, <64 x i8> %y)
   ret <64 x i8> %z
@@ -74,8 +67,7 @@ define <64 x i8> @v64i8(<64 x i8> %x, <64 x i8> %y) nounwind {
 define <8 x i16> @v8i16(<8 x i16> %x, <8 x i16> %y) nounwind {
 ; CHECK-LABEL: v8i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    umax v0.8h, v0.8h, v1.8h
-; CHECK-NEXT:    sub v0.8h, v0.8h, v1.8h
+; CHECK-NEXT:    uqsub v0.8h, v0.8h, v1.8h
 ; CHECK-NEXT:    ret
   %z = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> %x, <8 x i16> %y)
   ret <8 x i16> %z
@@ -84,10 +76,8 @@ define <8 x i16> @v8i16(<8 x i16> %x, <8 x i16> %y) nounwind {
 define <16 x i16> @v16i16(<16 x i16> %x, <16 x i16> %y) nounwind {
 ; CHECK-LABEL: v16i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    umax v0.8h, v0.8h, v2.8h
-; CHECK-NEXT:    umax v1.8h, v1.8h, v3.8h
-; CHECK-NEXT:    sub v0.8h, v0.8h, v2.8h
-; CHECK-NEXT:    sub v1.8h, v1.8h, v3.8h
+; CHECK-NEXT:    uqsub v0.8h, v0.8h, v2.8h
+; CHECK-NEXT:    uqsub v1.8h, v1.8h, v3.8h
 ; CHECK-NEXT:    ret
   %z = call <16 x i16> @llvm.usub.sat.v16i16(<16 x i16> %x, <16 x i16> %y)
   ret <16 x i16> %z
@@ -96,14 +86,10 @@ define <16 x i16> @v16i16(<16 x i16> %x, <16 x i16> %y) nounwind {
 define <32 x i16> @v32i16(<32 x i16> %x, <32 x i16> %y) nounwind {
 ; CHECK-LABEL: v32i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    umax v0.8h, v0.8h, v4.8h
-; CHECK-NEXT:    umax v1.8h, v1.8h, v5.8h
-; CHECK-NEXT:    umax v2.8h, v2.8h, v6.8h
-; CHECK-NEXT:    umax v3.8h, v3.8h, v7.8h
-; CHECK-NEXT:    sub v0.8h, v0.8h, v4.8h
-; CHECK-NEXT:    sub v1.8h, v1.8h, v5.8h
-; CHECK-NEXT:    sub v2.8h, v2.8h, v6.8h
-; CHECK-NEXT:    sub v3.8h, v3.8h, v7.8h
+; CHECK-NEXT:    uqsub v0.8h, v0.8h, v4.8h
+; CHECK-NEXT:    uqsub v1.8h, v1.8h, v5.8h
+; CHECK-NEXT:    uqsub v2.8h, v2.8h, v6.8h
+; CHECK-NEXT:    uqsub v3.8h, v3.8h, v7.8h
 ; CHECK-NEXT:    ret
   %z = call <32 x i16> @llvm.usub.sat.v32i16(<32 x i16> %x, <32 x i16> %y)
   ret <32 x i16> %z
@@ -114,8 +100,7 @@ define void @v8i8(<8 x i8>* %px, <8 x i8>* %py, <8 x i8>* %pz) nounwind {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldr d0, [x0]
 ; CHECK-NEXT:    ldr d1, [x1]
-; CHECK-NEXT:    umax v0.8b, v0.8b, v1.8b
-; CHECK-NEXT:    sub v0.8b, v0.8b, v1.8b
+; CHECK-NEXT:    uqsub v0.8b, v0.8b, v1.8b
 ; CHECK-NEXT:    str d0, [x2]
 ; CHECK-NEXT:    ret
   %x = load <8 x i8>, <8 x i8>* %px
@@ -146,8 +131,7 @@ define void @v4i8(<4 x i8>* %px, <4 x i8>* %py, <4 x i8>* %pz) nounwind {
 ; CHECK-NEXT:    mov v1.h[3], w11
 ; CHECK-NEXT:    shl v1.4h, v1.4h, #8
 ; CHECK-NEXT:    shl v0.4h, v0.4h, #8
-; CHECK-NEXT:    umax v0.4h, v0.4h, v1.4h
-; CHECK-NEXT:    sub v0.4h, v0.4h, v1.4h
+; CHECK-NEXT:    uqsub v0.4h, v0.4h, v1.4h
 ; CHECK-NEXT:    ushr v0.4h, v0.4h, #8
 ; CHECK-NEXT:    xtn v0.8b, v0.8h
 ; CHECK-NEXT:    str s0, [x2]
@@ -172,8 +156,7 @@ define void @v2i8(<2 x i8>* %px, <2 x i8>* %py, <2 x i8>* %pz) nounwind {
 ; CHECK-NEXT:    mov v1.s[1], w11
 ; CHECK-NEXT:    shl v1.2s, v1.2s, #24
 ; CHECK-NEXT:    shl v0.2s, v0.2s, #24
-; CHECK-NEXT:    umax v0.2s, v0.2s, v1.2s
-; CHECK-NEXT:    sub v0.2s, v0.2s, v1.2s
+; CHECK-NEXT:    uqsub v0.2s, v0.2s, v1.2s
 ; CHECK-NEXT:    ushr v0.2s, v0.2s, #24
 ; CHECK-NEXT:    mov w8, v0.s[1]
 ; CHECK-NEXT:    fmov w9, s0
@@ -192,8 +175,7 @@ define void @v4i16(<4 x i16>* %px, <4 x i16>* %py, <4 x i16>* %pz) nounwind {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldr d0, [x0]
 ; CHECK-NEXT:    ldr d1, [x1]
-; CHECK-NEXT:    umax v0.4h, v0.4h, v1.4h
-; CHECK-NEXT:    sub v0.4h, v0.4h, v1.4h
+; CHECK-NEXT:    uqsub v0.4h, v0.4h, v1.4h
 ; CHECK-NEXT:    str d0, [x2]
 ; CHECK-NEXT:    ret
   %x = load <4 x i16>, <4 x i16>* %px
@@ -216,8 +198,7 @@ define void @v2i16(<2 x i16>* %px, <2 x i16>* %py, <2 x i16>* %pz) nounwind {
 ; CHECK-NEXT:    mov v1.s[1], w11
 ; CHECK-NEXT:    shl v1.2s, v1.2s, #16
 ; CHECK-NEXT:    shl v0.2s, v0.2s, #16
-; CHECK-NEXT:    umax v0.2s, v0.2s, v1.2s
-; CHECK-NEXT:    sub v0.2s, v0.2s, v1.2s
+; CHECK-NEXT:    uqsub v0.2s, v0.2s, v1.2s
 ; CHECK-NEXT:    ushr v0.2s, v0.2s, #16
 ; CHECK-NEXT:    mov w8, v0.s[1]
 ; CHECK-NEXT:    fmov w9, s0
@@ -234,8 +215,7 @@ define void @v2i16(<2 x i16>* %px, <2 x i16>* %py, <2 x i16>* %pz) nounwind {
 define <12 x i8> @v12i8(<12 x i8> %x, <12 x i8> %y) nounwind {
 ; CHECK-LABEL: v12i8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    umax v0.16b, v0.16b, v1.16b
-; CHECK-NEXT:    sub v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    uqsub v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
   %z = call <12 x i8> @llvm.usub.sat.v12i8(<12 x i8> %x, <12 x i8> %y)
   ret <12 x i8> %z
@@ -246,10 +226,8 @@ define void @v12i16(<12 x i16>* %px, <12 x i16>* %py, <12 x i16>* %pz) nounwind
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldp q0, q1, [x0]
 ; CHECK-NEXT:    ldp q3, q2, [x1]
-; CHECK-NEXT:    umax v1.8h, v1.8h, v2.8h
-; CHECK-NEXT:    umax v0.8h, v0.8h, v3.8h
-; CHECK-NEXT:    sub v1.8h, v1.8h, v2.8h
-; CHECK-NEXT:    sub v0.8h, v0.8h, v3.8h
+; CHECK-NEXT:    uqsub v1.8h, v1.8h, v2.8h
+; CHECK-NEXT:    uqsub v0.8h, v0.8h, v3.8h
 ; CHECK-NEXT:    str q0, [x2]
 ; CHECK-NEXT:    str d1, [x2, #16]
 ; CHECK-NEXT:    ret
@@ -265,8 +243,7 @@ define void @v1i8(<1 x i8>* %px, <1 x i8>* %py, <1 x i8>* %pz) nounwind {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldr b0, [x0]
 ; CHECK-NEXT:    ldr b1, [x1]
-; CHECK-NEXT:    umax v0.8b, v0.8b, v1.8b
-; CHECK-NEXT:    sub v0.8b, v0.8b, v1.8b
+; CHECK-NEXT:    uqsub v0.8b, v0.8b, v1.8b
 ; CHECK-NEXT:    st1 { v0.b }[0], [x2]
 ; CHECK-NEXT:    ret
   %x = load <1 x i8>, <1 x i8>* %px
@@ -281,8 +258,7 @@ define void @v1i16(<1 x i16>* %px, <1 x i16>* %py, <1 x i16>* %pz) nounwind {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldr h0, [x0]
 ; CHECK-NEXT:    ldr h1, [x1]
-; CHECK-NEXT:    umax v0.4h, v0.4h, v1.4h
-; CHECK-NEXT:    sub v0.4h, v0.4h, v1.4h
+; CHECK-NEXT:    uqsub v0.4h, v0.4h, v1.4h
 ; CHECK-NEXT:    str h0, [x2]
 ; CHECK-NEXT:    ret
   %x = load <1 x i16>, <1 x i16>* %px
@@ -295,10 +271,12 @@ define void @v1i16(<1 x i16>* %px, <1 x i16>* %py, <1 x i16>* %pz) nounwind {
 define <16 x i4> @v16i4(<16 x i4> %x, <16 x i4> %y) nounwind {
 ; CHECK-LABEL: v16i4:
 ; CHECK:       // %bb.0:
+; CHECK-NEXT:    movi v2.16b, #15
+; CHECK-NEXT:    and v0.16b, v0.16b, v2.16b
+; CHECK-NEXT:    and v1.16b, v1.16b, v2.16b
 ; CHECK-NEXT:    shl v1.16b, v1.16b, #4
 ; CHECK-NEXT:    shl v0.16b, v0.16b, #4
-; CHECK-NEXT:    umax v0.16b, v0.16b, v1.16b
-; CHECK-NEXT:    sub v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    uqsub v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ushr v0.16b, v0.16b, #4
 ; CHECK-NEXT:    ret
   %z = call <16 x i4> @llvm.usub.sat.v16i4(<16 x i4> %x, <16 x i4> %y)
@@ -308,10 +286,12 @@ define <16 x i4> @v16i4(<16 x i4> %x, <16 x i4> %y) nounwind {
 define <16 x i1> @v16i1(<16 x i1> %x, <16 x i1> %y) nounwind {
 ; CHECK-LABEL: v16i1:
 ; CHECK:       // %bb.0:
+; CHECK-NEXT:    movi v2.16b, #1
+; CHECK-NEXT:    and v0.16b, v0.16b, v2.16b
+; CHECK-NEXT:    and v1.16b, v1.16b, v2.16b
 ; CHECK-NEXT:    shl v1.16b, v1.16b, #7
 ; CHECK-NEXT:    shl v0.16b, v0.16b, #7
-; CHECK-NEXT:    umax v0.16b, v0.16b, v1.16b
-; CHECK-NEXT:    sub v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    uqsub v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ushr v0.16b, v0.16b, #7
 ; CHECK-NEXT:    ret
   %z = call <16 x i1> @llvm.usub.sat.v16i1(<16 x i1> %x, <16 x i1> %y)
@@ -321,8 +301,7 @@ define <16 x i1> @v16i1(<16 x i1> %x, <16 x i1> %y) nounwind {
 define <2 x i32> @v2i32(<2 x i32> %x, <2 x i32> %y) nounwind {
 ; CHECK-LABEL: v2i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    umax v0.2s, v0.2s, v1.2s
-; CHECK-NEXT:    sub v0.2s, v0.2s, v1.2s
+; CHECK-NEXT:    uqsub v0.2s, v0.2s, v1.2s
 ; CHECK-NEXT:    ret
   %z = call <2 x i32> @llvm.usub.sat.v2i32(<2 x i32> %x, <2 x i32> %y)
   ret <2 x i32> %z
@@ -331,8 +310,7 @@ define <2 x i32> @v2i32(<2 x i32> %x, <2 x i32> %y) nounwind {
 define <4 x i32> @v4i32(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; CHECK-LABEL: v4i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    umax v0.4s, v0.4s, v1.4s
-; CHECK-NEXT:    sub v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    uqsub v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    ret
   %z = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> %x, <4 x i32> %y)
   ret <4 x i32> %z
@@ -341,10 +319,8 @@ define <4 x i32> @v4i32(<4 x i32> %x, <4 x i32> %y) nounwind {
 define <8 x i32> @v8i32(<8 x i32> %x, <8 x i32> %y) nounwind {
 ; CHECK-LABEL: v8i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    umax v0.4s, v0.4s, v2.4s
-; CHECK-NEXT:    umax v1.4s, v1.4s, v3.4s
-; CHECK-NEXT:    sub v0.4s, v0.4s, v2.4s
-; CHECK-NEXT:    sub v1.4s, v1.4s, v3.4s
+; CHECK-NEXT:    uqsub v0.4s, v0.4s, v2.4s
+; CHECK-NEXT:    uqsub v1.4s, v1.4s, v3.4s
 ; CHECK-NEXT:    ret
   %z = call <8 x i32> @llvm.usub.sat.v8i32(<8 x i32> %x, <8 x i32> %y)
   ret <8 x i32> %z
@@ -353,14 +329,10 @@ define <8 x i32> @v8i32(<8 x i32> %x, <8 x i32> %y) nounwind {
 define <16 x i32> @v16i32(<16 x i32> %x, <16 x i32> %y) nounwind {
 ; CHECK-LABEL: v16i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    umax v0.4s, v0.4s, v4.4s
-; CHECK-NEXT:    umax v1.4s, v1.4s, v5.4s
-; CHECK-NEXT:    umax v2.4s, v2.4s, v6.4s
-; CHECK-NEXT:    umax v3.4s, v3.4s, v7.4s
-; CHECK-NEXT:    sub v0.4s, v0.4s, v4.4s
-; CHECK-NEXT:    sub v1.4s, v1.4s, v5.4s
-; CHECK-NEXT:    sub v2.4s, v2.4s, v6.4s
-; CHECK-NEXT:    sub v3.4s, v3.4s, v7.4s
+; CHECK-NEXT:    uqsub v0.4s, v0.4s, v4.4s
+; CHECK-NEXT:    uqsub v1.4s, v1.4s, v5.4s
+; CHECK-NEXT:    uqsub v2.4s, v2.4s, v6.4s
+; CHECK-NEXT:    uqsub v3.4s, v3.4s, v7.4s
 ; CHECK-NEXT:    ret
   %z = call <16 x i32> @llvm.usub.sat.v16i32(<16 x i32> %x, <16 x i32> %y)
   ret <16 x i32> %z
@@ -369,9 +341,7 @@ define <16 x i32> @v16i32(<16 x i32> %x, <16 x i32> %y) nounwind {
 define <2 x i64> @v2i64(<2 x i64> %x, <2 x i64> %y) nounwind {
 ; CHECK-LABEL: v2i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    sub v1.2d, v0.2d, v1.2d
-; CHECK-NEXT:    cmhi v0.2d, v1.2d, v0.2d
-; CHECK-NEXT:    bic v0.16b, v1.16b, v0.16b
+; CHECK-NEXT:    uqsub v0.2d, v0.2d, v1.2d
 ; CHECK-NEXT:    ret
   %z = call <2 x i64> @llvm.usub.sat.v2i64(<2 x i64> %x, <2 x i64> %y)
   ret <2 x i64> %z
@@ -380,12 +350,8 @@ define <2 x i64> @v2i64(<2 x i64> %x, <2 x i64> %y) nounwind {
 define <4 x i64> @v4i64(<4 x i64> %x, <4 x i64> %y) nounwind {
 ; CHECK-LABEL: v4i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    sub v2.2d, v0.2d, v2.2d
-; CHECK-NEXT:    sub v3.2d, v1.2d, v3.2d
-; CHECK-NEXT:    cmhi v0.2d, v2.2d, v0.2d
-; CHECK-NEXT:    cmhi v1.2d, v3.2d, v1.2d
-; CHECK-NEXT:    bic v0.16b, v2.16b, v0.16b
-; CHECK-NEXT:    bic v1.16b, v3.16b, v1.16b
+; CHECK-NEXT:    uqsub v0.2d, v0.2d, v2.2d
+; CHECK-NEXT:    uqsub v1.2d, v1.2d, v3.2d
 ; CHECK-NEXT:    ret
   %z = call <4 x i64> @llvm.usub.sat.v4i64(<4 x i64> %x, <4 x i64> %y)
   ret <4 x i64> %z
@@ -394,18 +360,10 @@ define <4 x i64> @v4i64(<4 x i64> %x, <4 x i64> %y) nounwind {
 define <8 x i64> @v8i64(<8 x i64> %x, <8 x i64> %y) nounwind {
 ; CHECK-LABEL: v8i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    sub v4.2d, v0.2d, v4.2d
-; CHECK-NEXT:    sub v5.2d, v1.2d, v5.2d
-; CHECK-NEXT:    sub v6.2d, v2.2d, v6.2d
-; CHECK-NEXT:    sub v7.2d, v3.2d, v7.2d
-; CHECK-NEXT:    cmhi v0.2d, v4.2d, v0.2d
-; CHECK-NEXT:    cmhi v1.2d, v5.2d, v1.2d
-; CHECK-NEXT:    cmhi v2.2d, v6.2d, v2.2d
-; CHECK-NEXT:    cmhi v3.2d, v7.2d, v3.2d
-; CHECK-NEXT:    bic v0.16b, v4.16b, v0.16b
-; CHECK-NEXT:    bic v1.16b, v5.16b, v1.16b
-; CHECK-NEXT:    bic v2.16b, v6.16b, v2.16b
-; CHECK-NEXT:    bic v3.16b, v7.16b, v3.16b
+; CHECK-NEXT:    uqsub v0.2d, v0.2d, v4.2d
+; CHECK-NEXT:    uqsub v1.2d, v1.2d, v5.2d
+; CHECK-NEXT:    uqsub v2.2d, v2.2d, v6.2d
+; CHECK-NEXT:    uqsub v3.2d, v3.2d, v7.2d
 ; CHECK-NEXT:    ret
   %z = call <8 x i64> @llvm.usub.sat.v8i64(<8 x i64> %x, <8 x i64> %y)
   ret <8 x i64> %z
diff --git a/llvm/test/CodeGen/AArch64/vararg-tallcall.ll b/llvm/test/CodeGen/AArch64/vararg-tallcall.ll
index 56c56213a..aa239a8d5 100644
--- a/llvm/test/CodeGen/AArch64/vararg-tallcall.ll
+++ b/llvm/test/CodeGen/AArch64/vararg-tallcall.ll
@@ -1,5 +1,7 @@
 ; RUN: llc -mtriple=aarch64-windows-msvc %s -o - | FileCheck %s
 ; RUN: llc -mtriple=aarch64-linux-gnu %s -o - | FileCheck %s
+; RUN: llc -global-isel -global-isel-abort=2 -verify-machineinstrs -mtriple=aarch64-windows-msvc %s -o - | FileCheck %s
+; RUN: llc -global-isel -global-isel-abort=2 -verify-machineinstrs -mtriple=aarch64-linux-gnu %s -o - | FileCheck %s
 
 target datalayout = "e-m:w-p:64:64-i32:32-i64:64-i128:128-n32:64-S128"
 
diff --git a/llvm/test/CodeGen/AArch64/vecreduce-and-legalization.ll b/llvm/test/CodeGen/AArch64/vecreduce-and-legalization.ll
index bf186263b..375f220fe 100644
--- a/llvm/test/CodeGen/AArch64/vecreduce-and-legalization.ll
+++ b/llvm/test/CodeGen/AArch64/vecreduce-and-legalization.ll
@@ -105,17 +105,17 @@ define i8 @test_v9i8(<9 x i8> %a) nounwind {
 ; CHECK-NEXT:    mov v0.b[14], w8
 ; CHECK-NEXT:    mov v0.b[15], w8
 ; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT:    and v0.8b, v0.8b, v1.8b
-; CHECK-NEXT:    umov w8, v0.b[1]
-; CHECK-NEXT:    umov w9, v0.b[0]
+; CHECK-NEXT:    and v1.8b, v0.8b, v1.8b
+; CHECK-NEXT:    umov w8, v1.b[1]
+; CHECK-NEXT:    umov w9, v1.b[0]
 ; CHECK-NEXT:    and w8, w9, w8
-; CHECK-NEXT:    umov w9, v0.b[2]
+; CHECK-NEXT:    umov w9, v1.b[2]
 ; CHECK-NEXT:    and w8, w8, w9
-; CHECK-NEXT:    umov w9, v0.b[3]
+; CHECK-NEXT:    umov w9, v1.b[3]
 ; CHECK-NEXT:    and w8, w8, w9
-; CHECK-NEXT:    umov w9, v0.b[4]
+; CHECK-NEXT:    umov w9, v1.b[4]
 ; CHECK-NEXT:    and w8, w8, w9
-; CHECK-NEXT:    umov w9, v0.b[5]
+; CHECK-NEXT:    umov w9, v1.b[5]
 ; CHECK-NEXT:    and w8, w8, w9
 ; CHECK-NEXT:    umov w9, v0.b[6]
 ; CHECK-NEXT:    and w8, w8, w9
@@ -132,9 +132,9 @@ define i32 @test_v3i32(<3 x i32> %a) nounwind {
 ; CHECK-NEXT:    mov w8, #-1
 ; CHECK-NEXT:    mov v0.s[3], w8
 ; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-NEXT:    and v1.8b, v0.8b, v1.8b
 ; CHECK-NEXT:    mov w8, v0.s[1]
-; CHECK-NEXT:    fmov w9, s0
+; CHECK-NEXT:    fmov w9, s1
 ; CHECK-NEXT:    and w0, w9, w8
 ; CHECK-NEXT:    ret
   %b = call i32 @llvm.experimental.vector.reduce.and.v3i32(<3 x i32> %a)
diff --git a/llvm/test/CodeGen/AArch64/vecreduce-fadd-legalization-strict.ll b/llvm/test/CodeGen/AArch64/vecreduce-fadd-legalization-strict.ll
new file mode 100644
index 000000000..5d6f2e40d
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/vecreduce-fadd-legalization-strict.ll
@@ -0,0 +1,128 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=aarch64-none-linux-gnu -mattr=+neon | FileCheck %s --check-prefix=CHECK
+
+; Same as vecreduce-fadd-legalization.ll, but without fmf.
+
+declare half @llvm.experimental.vector.reduce.v2.fadd.f16.v1f16(half, <1 x half>)
+declare float @llvm.experimental.vector.reduce.v2.fadd.f32.v1f32(float, <1 x float>)
+declare double @llvm.experimental.vector.reduce.v2.fadd.f64.v1f64(double, <1 x double>)
+declare fp128 @llvm.experimental.vector.reduce.v2.fadd.f128.v1f128(fp128, <1 x fp128>)
+
+declare float @llvm.experimental.vector.reduce.v2.fadd.f32.v3f32(float, <3 x float>)
+declare fp128 @llvm.experimental.vector.reduce.v2.fadd.f128.v2f128(fp128, <2 x fp128>)
+declare float @llvm.experimental.vector.reduce.v2.fadd.f32.v16f32(float, <16 x float>)
+
+define half @test_v1f16(<1 x half> %a) nounwind {
+; CHECK-LABEL: test_v1f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fcvt s0, h0
+; CHECK-NEXT:    fmov s1, wzr
+; CHECK-NEXT:    fadd s0, s0, s1
+; CHECK-NEXT:    fcvt h0, s0
+; CHECK-NEXT:    ret
+  %b = call half @llvm.experimental.vector.reduce.v2.fadd.f16.v1f16(half 0.0, <1 x half> %a)
+  ret half %b
+}
+
+define float @test_v1f32(<1 x float> %a) nounwind {
+; CHECK-LABEL: test_v1f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    fmov s1, wzr
+; CHECK-NEXT:    fadd s0, s0, s1
+; CHECK-NEXT:    ret
+  %b = call float @llvm.experimental.vector.reduce.v2.fadd.f32.v1f32(float 0.0, <1 x float> %a)
+  ret float %b
+}
+
+define double @test_v1f64(<1 x double> %a) nounwind {
+; CHECK-LABEL: test_v1f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmov d1, xzr
+; CHECK-NEXT:    fadd d0, d0, d1
+; CHECK-NEXT:    ret
+  %b = call double @llvm.experimental.vector.reduce.v2.fadd.f64.v1f64(double 0.0, <1 x double> %a)
+  ret double %b
+}
+
+define fp128 @test_v1f128(<1 x fp128> %a) nounwind {
+; CHECK-LABEL: test_v1f128:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    adrp x8, .LCPI3_0
+; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI3_0]
+; CHECK-NEXT:    bl __addtf3
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %b = call fp128 @llvm.experimental.vector.reduce.v2.fadd.f128.v1f128(fp128 zeroinitializer, <1 x fp128> %a)
+  ret fp128 %b
+}
+
+define float @test_v3f32(<3 x float> %a) nounwind {
+; CHECK-LABEL: test_v3f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmov s1, wzr
+; CHECK-NEXT:    mov s2, v0.s[1]
+; CHECK-NEXT:    fadd s1, s0, s1
+; CHECK-NEXT:    fadd s1, s1, s2
+; CHECK-NEXT:    mov s0, v0.s[2]
+; CHECK-NEXT:    fadd s0, s1, s0
+; CHECK-NEXT:    ret
+  %b = call float @llvm.experimental.vector.reduce.v2.fadd.f32.v3f32(float 0.0, <3 x float> %a)
+  ret float %b
+}
+
+define fp128 @test_v2f128(<2 x fp128> %a) nounwind {
+; CHECK-LABEL: test_v2f128:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #32 // =32
+; CHECK-NEXT:    adrp x8, .LCPI5_0
+; CHECK-NEXT:    str q1, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI5_0]
+; CHECK-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-NEXT:    bl __addtf3
+; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    bl __addtf3
+; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-NEXT:    add sp, sp, #32 // =32
+; CHECK-NEXT:    ret
+  %b = call fp128 @llvm.experimental.vector.reduce.v2.fadd.f128.v2f128(fp128 zeroinitializer, <2 x fp128> %a)
+  ret fp128 %b
+}
+
+define float @test_v16f32(<16 x float> %a) nounwind {
+; CHECK-LABEL: test_v16f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmov s4, wzr
+; CHECK-NEXT:    mov s5, v0.s[1]
+; CHECK-NEXT:    fadd s4, s0, s4
+; CHECK-NEXT:    fadd s4, s4, s5
+; CHECK-NEXT:    mov s5, v0.s[2]
+; CHECK-NEXT:    mov s0, v0.s[3]
+; CHECK-NEXT:    fadd s4, s4, s5
+; CHECK-NEXT:    fadd s0, s4, s0
+; CHECK-NEXT:    mov s5, v1.s[1]
+; CHECK-NEXT:    fadd s0, s0, s1
+; CHECK-NEXT:    mov s4, v1.s[2]
+; CHECK-NEXT:    fadd s0, s0, s5
+; CHECK-NEXT:    mov s1, v1.s[3]
+; CHECK-NEXT:    fadd s0, s0, s4
+; CHECK-NEXT:    fadd s0, s0, s1
+; CHECK-NEXT:    mov s5, v2.s[1]
+; CHECK-NEXT:    fadd s0, s0, s2
+; CHECK-NEXT:    mov s4, v2.s[2]
+; CHECK-NEXT:    fadd s0, s0, s5
+; CHECK-NEXT:    mov s1, v2.s[3]
+; CHECK-NEXT:    fadd s0, s0, s4
+; CHECK-NEXT:    fadd s0, s0, s1
+; CHECK-NEXT:    mov s2, v3.s[1]
+; CHECK-NEXT:    fadd s0, s0, s3
+; CHECK-NEXT:    mov s5, v3.s[2]
+; CHECK-NEXT:    fadd s0, s0, s2
+; CHECK-NEXT:    fadd s0, s0, s5
+; CHECK-NEXT:    mov s1, v3.s[3]
+; CHECK-NEXT:    fadd s0, s0, s1
+; CHECK-NEXT:    ret
+  %b = call float @llvm.experimental.vector.reduce.v2.fadd.f32.v16f32(float 0.0, <16 x float> %a)
+  ret float %b
+}
diff --git a/llvm/test/CodeGen/AArch64/vecreduce-fmax-legalization-nan.ll b/llvm/test/CodeGen/AArch64/vecreduce-fmax-legalization-nan.ll
new file mode 100644
index 000000000..c669a5551
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/vecreduce-fmax-legalization-nan.ll
@@ -0,0 +1,88 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=aarch64-none-linux-gnu -mattr=+neon | FileCheck %s --check-prefix=CHECK
+
+declare half @llvm.experimental.vector.reduce.fmax.v1f16(<1 x half> %a)
+declare float @llvm.experimental.vector.reduce.fmax.v1f32(<1 x float> %a)
+declare double @llvm.experimental.vector.reduce.fmax.v1f64(<1 x double> %a)
+declare fp128 @llvm.experimental.vector.reduce.fmax.v1f128(<1 x fp128> %a)
+
+declare float @llvm.experimental.vector.reduce.fmax.v3f32(<3 x float> %a)
+declare fp128 @llvm.experimental.vector.reduce.fmax.v2f128(<2 x fp128> %a)
+declare float @llvm.experimental.vector.reduce.fmax.v16f32(<16 x float> %a)
+
+define half @test_v1f16(<1 x half> %a) nounwind {
+; CHECK-LABEL: test_v1f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ret
+  %b = call half @llvm.experimental.vector.reduce.fmax.v1f16(<1 x half> %a)
+  ret half %b
+}
+
+define float @test_v1f32(<1 x float> %a) nounwind {
+; CHECK-LABEL: test_v1f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-NEXT:    ret
+  %b = call float @llvm.experimental.vector.reduce.fmax.v1f32(<1 x float> %a)
+  ret float %b
+}
+
+define double @test_v1f64(<1 x double> %a) nounwind {
+; CHECK-LABEL: test_v1f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ret
+  %b = call double @llvm.experimental.vector.reduce.fmax.v1f64(<1 x double> %a)
+  ret double %b
+}
+
+define fp128 @test_v1f128(<1 x fp128> %a) nounwind {
+; CHECK-LABEL: test_v1f128:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ret
+  %b = call fp128 @llvm.experimental.vector.reduce.fmax.v1f128(<1 x fp128> %a)
+  ret fp128 %b
+}
+
+; TODO: This doesn't work, because ExpandReductions only supports power of two
+; unordered reductions.
+;define float @test_v3f32(<3 x float> %a) nounwind {
+;  %b = call float @llvm.experimental.vector.reduce.fmax.v3f32(<3 x float> %a)
+;  ret float %b
+;}
+
+define fp128 @test_v2f128(<2 x fp128> %a) nounwind {
+; CHECK-LABEL: test_v2f128:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #48 // =48
+; CHECK-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
+; CHECK-NEXT:    stp q0, q1, [sp] // 32-byte Folded Spill
+; CHECK-NEXT:    bl __gttf2
+; CHECK-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    cmp w0, #0 // =0
+; CHECK-NEXT:    b.le .LBB4_2
+; CHECK-NEXT:  // %bb.1:
+; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:  .LBB4_2:
+; CHECK-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
+; CHECK-NEXT:    add sp, sp, #48 // =48
+; CHECK-NEXT:    ret
+  %b = call fp128 @llvm.experimental.vector.reduce.fmax.v2f128(<2 x fp128> %a)
+  ret fp128 %b
+}
+
+define float @test_v16f32(<16 x float> %a) nounwind {
+; CHECK-LABEL: test_v16f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmaxnm v1.4s, v1.4s, v3.4s
+; CHECK-NEXT:    fmaxnm v0.4s, v0.4s, v2.4s
+; CHECK-NEXT:    fmaxnm v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    fmaxnm v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    dup v1.4s, v0.s[1]
+; CHECK-NEXT:    fmaxnm v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-NEXT:    ret
+  %b = call float @llvm.experimental.vector.reduce.fmax.v16f32(<16 x float> %a)
+  ret float %b
+}
diff --git a/llvm/test/CodeGen/AArch64/vecreduce-fmax-legalization.ll b/llvm/test/CodeGen/AArch64/vecreduce-fmax-legalization.ll
index 72c94aaf2..975ba2687 100644
--- a/llvm/test/CodeGen/AArch64/vecreduce-fmax-legalization.ll
+++ b/llvm/test/CodeGen/AArch64/vecreduce-fmax-legalization.ll
@@ -47,7 +47,7 @@ define fp128 @test_v1f128(<1 x fp128> %a) nounwind {
 define float @test_v3f32(<3 x float> %a) nounwind {
 ; CHECK-LABEL: test_v3f32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #2139095040
+; CHECK-NEXT:    mov w8, #-8388608
 ; CHECK-NEXT:    fmov s1, w8
 ; CHECK-NEXT:    mov v0.s[3], v1.s[0]
 ; CHECK-NEXT:    fmaxnmv s0, v0.4s
diff --git a/llvm/test/CodeGen/AArch64/vecreduce-fmul-legalization-strict.ll b/llvm/test/CodeGen/AArch64/vecreduce-fmul-legalization-strict.ll
new file mode 100644
index 000000000..41914ca73
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/vecreduce-fmul-legalization-strict.ll
@@ -0,0 +1,114 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=aarch64-none-linux-gnu -mattr=+neon | FileCheck %s --check-prefix=CHECK
+
+; Same as vecreduce-fmul-legalization.ll, but without fmf.
+
+declare half @llvm.experimental.vector.reduce.v2.fmul.f16.v1f16(half, <1 x half>)
+declare float @llvm.experimental.vector.reduce.v2.fmul.f32.v1f32(float, <1 x float>)
+declare double @llvm.experimental.vector.reduce.v2.fmul.f64.v1f64(double, <1 x double>)
+declare fp128 @llvm.experimental.vector.reduce.v2.fmul.f128.v1f128(fp128, <1 x fp128>)
+
+declare float @llvm.experimental.vector.reduce.v2.fmul.f32.v3f32(float, <3 x float>)
+declare fp128 @llvm.experimental.vector.reduce.v2.fmul.f128.v2f128(fp128, <2 x fp128>)
+declare float @llvm.experimental.vector.reduce.v2.fmul.f32.v16f32(float, <16 x float>)
+
+define half @test_v1f16(<1 x half> %a) nounwind {
+; CHECK-LABEL: test_v1f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fcvt s0, h0
+; CHECK-NEXT:    fmov s1, wzr
+; CHECK-NEXT:    fmul s0, s0, s1
+; CHECK-NEXT:    fcvt h0, s0
+; CHECK-NEXT:    ret
+  %b = call half @llvm.experimental.vector.reduce.v2.fmul.f16.v1f16(half 0.0, <1 x half> %a)
+  ret half %b
+}
+
+define float @test_v1f32(<1 x float> %a) nounwind {
+; CHECK-LABEL: test_v1f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    fmov s1, wzr
+; CHECK-NEXT:    fmul s0, s1, v0.s[0]
+; CHECK-NEXT:    ret
+  %b = call float @llvm.experimental.vector.reduce.v2.fmul.f32.v1f32(float 0.0, <1 x float> %a)
+  ret float %b
+}
+
+define double @test_v1f64(<1 x double> %a) nounwind {
+; CHECK-LABEL: test_v1f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmov d1, xzr
+; CHECK-NEXT:    fmul d0, d0, d1
+; CHECK-NEXT:    ret
+  %b = call double @llvm.experimental.vector.reduce.v2.fmul.f64.v1f64(double 0.0, <1 x double> %a)
+  ret double %b
+}
+
+define fp128 @test_v1f128(<1 x fp128> %a) nounwind {
+; CHECK-LABEL: test_v1f128:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    adrp x8, .LCPI3_0
+; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI3_0]
+; CHECK-NEXT:    bl __multf3
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %b = call fp128 @llvm.experimental.vector.reduce.v2.fmul.f128.v1f128(fp128 zeroinitializer, <1 x fp128> %a)
+  ret fp128 %b
+}
+
+define float @test_v3f32(<3 x float> %a) nounwind {
+; CHECK-LABEL: test_v3f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmov s1, wzr
+; CHECK-NEXT:    fmul s1, s1, v0.s[0]
+; CHECK-NEXT:    fmul s1, s1, v0.s[1]
+; CHECK-NEXT:    fmul s0, s1, v0.s[2]
+; CHECK-NEXT:    ret
+  %b = call float @llvm.experimental.vector.reduce.v2.fmul.f32.v3f32(float 0.0, <3 x float> %a)
+  ret float %b
+}
+
+define fp128 @test_v2f128(<2 x fp128> %a) nounwind {
+; CHECK-LABEL: test_v2f128:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #32 // =32
+; CHECK-NEXT:    adrp x8, .LCPI5_0
+; CHECK-NEXT:    str q1, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI5_0]
+; CHECK-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-NEXT:    bl __multf3
+; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    bl __multf3
+; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-NEXT:    add sp, sp, #32 // =32
+; CHECK-NEXT:    ret
+  %b = call fp128 @llvm.experimental.vector.reduce.v2.fmul.f128.v2f128(fp128 zeroinitializer, <2 x fp128> %a)
+  ret fp128 %b
+}
+
+define float @test_v16f32(<16 x float> %a) nounwind {
+; CHECK-LABEL: test_v16f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmov s4, wzr
+; CHECK-NEXT:    fmul s4, s4, v0.s[0]
+; CHECK-NEXT:    fmul s4, s4, v0.s[1]
+; CHECK-NEXT:    fmul s4, s4, v0.s[2]
+; CHECK-NEXT:    fmul s0, s4, v0.s[3]
+; CHECK-NEXT:    fmul s0, s0, v1.s[0]
+; CHECK-NEXT:    fmul s0, s0, v1.s[1]
+; CHECK-NEXT:    fmul s0, s0, v1.s[2]
+; CHECK-NEXT:    fmul s0, s0, v1.s[3]
+; CHECK-NEXT:    fmul s0, s0, v2.s[0]
+; CHECK-NEXT:    fmul s0, s0, v2.s[1]
+; CHECK-NEXT:    fmul s0, s0, v2.s[2]
+; CHECK-NEXT:    fmul s0, s0, v2.s[3]
+; CHECK-NEXT:    fmul s0, s0, v3.s[0]
+; CHECK-NEXT:    fmul s0, s0, v3.s[1]
+; CHECK-NEXT:    fmul s0, s0, v3.s[2]
+; CHECK-NEXT:    fmul s0, s0, v3.s[3]
+; CHECK-NEXT:    ret
+  %b = call float @llvm.experimental.vector.reduce.v2.fmul.f32.v16f32(float 0.0, <16 x float> %a)
+  ret float %b
+}
diff --git a/llvm/test/CodeGen/AArch64/vector_merge_dep_check.ll b/llvm/test/CodeGen/AArch64/vector_merge_dep_check.ll
index e4e64ef8c..a5e419616 100644
--- a/llvm/test/CodeGen/AArch64/vector_merge_dep_check.ll
+++ b/llvm/test/CodeGen/AArch64/vector_merge_dep_check.ll
@@ -24,7 +24,7 @@ define void @fn(<2 x i64>* %argA, <2 x i64>* %argB, i64* %a) #0 align 2 {
 
 ; CHECK: ret
 
-attributes #0 = { noinline norecurse nounwind ssp uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "polly-optimized" "stack-protector-buffer-size"="8" "target-features"="+crc,+crypto,+neon" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noinline norecurse nounwind ssp uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "polly-optimized" "stack-protector-buffer-size"="8" "target-features"="+crc,+crypto,+neon" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.ident = !{!0}
 
diff --git a/llvm/test/CodeGen/AArch64/vselect-constants.ll b/llvm/test/CodeGen/AArch64/vselect-constants.ll
new file mode 100644
index 000000000..0bc328331
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/vselect-constants.ll
@@ -0,0 +1,195 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=aarch64-- | FileCheck %s
+
+; First, check the generic pattern for any 2 vector constants. Then, check special cases where
+; the constants are all off-by-one. Finally, check the extra special cases where the constants
+; include 0 or -1.
+; Each minimal select test is repeated with a more typical pattern that includes a compare to
+; generate the condition value.
+
+define <4 x i32> @sel_C1_or_C2_vec(<4 x i1> %cond) {
+; CHECK-LABEL: sel_C1_or_C2_vec:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI0_0
+; CHECK-NEXT:    adrp x9, .LCPI0_1
+; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI0_0]
+; CHECK-NEXT:    ldr q2, [x9, :lo12:.LCPI0_1]
+; CHECK-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-NEXT:    shl v0.4s, v0.4s, #31
+; CHECK-NEXT:    sshr v0.4s, v0.4s, #31
+; CHECK-NEXT:    bsl v0.16b, v2.16b, v1.16b
+; CHECK-NEXT:    ret
+  %add = select <4 x i1> %cond, <4 x i32> <i32 3000, i32 1, i32 -1, i32 0>, <4 x i32> <i32 42, i32 0, i32 -2, i32 -1>
+  ret <4 x i32> %add
+}
+
+define <4 x i32> @cmp_sel_C1_or_C2_vec(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: cmp_sel_C1_or_C2_vec:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI1_0
+; CHECK-NEXT:    adrp x9, .LCPI1_1
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI1_0]
+; CHECK-NEXT:    ldr q3, [x9, :lo12:.LCPI1_1]
+; CHECK-NEXT:    cmeq v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    bsl v0.16b, v3.16b, v2.16b
+; CHECK-NEXT:    ret
+  %cond = icmp eq <4 x i32> %x, %y
+  %add = select <4 x i1> %cond, <4 x i32> <i32 3000, i32 1, i32 -1, i32 0>, <4 x i32> <i32 42, i32 0, i32 -2, i32 -1>
+  ret <4 x i32> %add
+}
+
+define <4 x i32> @sel_Cplus1_or_C_vec(<4 x i1> %cond) {
+; CHECK-LABEL: sel_Cplus1_or_C_vec:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI2_0
+; CHECK-NEXT:    adrp x9, .LCPI2_1
+; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI2_0]
+; CHECK-NEXT:    ldr q2, [x9, :lo12:.LCPI2_1]
+; CHECK-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-NEXT:    shl v0.4s, v0.4s, #31
+; CHECK-NEXT:    sshr v0.4s, v0.4s, #31
+; CHECK-NEXT:    bsl v0.16b, v2.16b, v1.16b
+; CHECK-NEXT:    ret
+  %add = select <4 x i1> %cond, <4 x i32> <i32 43, i32 1, i32 -1, i32 0>, <4 x i32> <i32 42, i32 0, i32 -2, i32 -1>
+  ret <4 x i32> %add
+}
+
+define <4 x i32> @cmp_sel_Cplus1_or_C_vec(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: cmp_sel_Cplus1_or_C_vec:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI3_0
+; CHECK-NEXT:    adrp x9, .LCPI3_1
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI3_0]
+; CHECK-NEXT:    ldr q3, [x9, :lo12:.LCPI3_1]
+; CHECK-NEXT:    cmeq v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    bsl v0.16b, v3.16b, v2.16b
+; CHECK-NEXT:    ret
+  %cond = icmp eq <4 x i32> %x, %y
+  %add = select <4 x i1> %cond, <4 x i32> <i32 43, i32 1, i32 -1, i32 0>, <4 x i32> <i32 42, i32 0, i32 -2, i32 -1>
+  ret <4 x i32> %add
+}
+
+define <4 x i32> @sel_Cminus1_or_C_vec(<4 x i1> %cond) {
+; CHECK-LABEL: sel_Cminus1_or_C_vec:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI4_0
+; CHECK-NEXT:    adrp x9, .LCPI4_1
+; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI4_0]
+; CHECK-NEXT:    ldr q2, [x9, :lo12:.LCPI4_1]
+; CHECK-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-NEXT:    shl v0.4s, v0.4s, #31
+; CHECK-NEXT:    sshr v0.4s, v0.4s, #31
+; CHECK-NEXT:    bsl v0.16b, v2.16b, v1.16b
+; CHECK-NEXT:    ret
+  %add = select <4 x i1> %cond, <4 x i32> <i32 43, i32 1, i32 -1, i32 0>, <4 x i32> <i32 44, i32 2, i32 0, i32 1>
+  ret <4 x i32> %add
+}
+
+define <4 x i32> @cmp_sel_Cminus1_or_C_vec(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: cmp_sel_Cminus1_or_C_vec:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI5_0
+; CHECK-NEXT:    adrp x9, .LCPI5_1
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI5_0]
+; CHECK-NEXT:    ldr q3, [x9, :lo12:.LCPI5_1]
+; CHECK-NEXT:    cmeq v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    bsl v0.16b, v3.16b, v2.16b
+; CHECK-NEXT:    ret
+  %cond = icmp eq <4 x i32> %x, %y
+  %add = select <4 x i1> %cond, <4 x i32> <i32 43, i32 1, i32 -1, i32 0>, <4 x i32> <i32 44, i32 2, i32 0, i32 1>
+  ret <4 x i32> %add
+}
+
+define <4 x i32> @sel_minus1_or_0_vec(<4 x i1> %cond) {
+; CHECK-LABEL: sel_minus1_or_0_vec:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-NEXT:    shl v0.4s, v0.4s, #31
+; CHECK-NEXT:    sshr v0.4s, v0.4s, #31
+; CHECK-NEXT:    ret
+  %add = select <4 x i1> %cond, <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>, <4 x i32> <i32 0, i32 0, i32 0, i32 0>
+  ret <4 x i32> %add
+}
+
+define <4 x i32> @cmp_sel_minus1_or_0_vec(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: cmp_sel_minus1_or_0_vec:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmeq v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    ret
+  %cond = icmp eq <4 x i32> %x, %y
+  %add = select <4 x i1> %cond, <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>, <4 x i32> <i32 0, i32 0, i32 0, i32 0>
+  ret <4 x i32> %add
+}
+
+define <4 x i32> @sel_0_or_minus1_vec(<4 x i1> %cond) {
+; CHECK-LABEL: sel_0_or_minus1_vec:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-NEXT:    shl v0.4s, v0.4s, #31
+; CHECK-NEXT:    cmge v0.4s, v0.4s, #0
+; CHECK-NEXT:    ret
+  %add = select <4 x i1> %cond, <4 x i32> <i32 0, i32 0, i32 0, i32 0>, <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>
+  ret <4 x i32> %add
+}
+
+define <4 x i32> @cmp_sel_0_or_minus1_vec(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: cmp_sel_0_or_minus1_vec:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmeq v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    mvn v0.16b, v0.16b
+; CHECK-NEXT:    ret
+  %cond = icmp eq <4 x i32> %x, %y
+  %add = select <4 x i1> %cond, <4 x i32> <i32 0, i32 0, i32 0, i32 0>, <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>
+  ret <4 x i32> %add
+}
+
+define <4 x i32> @sel_1_or_0_vec(<4 x i1> %cond) {
+; CHECK-LABEL: sel_1_or_0_vec:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-NEXT:    shl v0.4s, v0.4s, #31
+; CHECK-NEXT:    sshr v0.4s, v0.4s, #31
+; CHECK-NEXT:    movi v1.4s, #1
+; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    ret
+  %add = select <4 x i1> %cond, <4 x i32> <i32 1, i32 1, i32 1, i32 1>, <4 x i32> <i32 0, i32 0, i32 0, i32 0>
+  ret <4 x i32> %add
+}
+
+define <4 x i32> @cmp_sel_1_or_0_vec(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: cmp_sel_1_or_0_vec:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmeq v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    movi v1.4s, #1
+; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    ret
+  %cond = icmp eq <4 x i32> %x, %y
+  %add = select <4 x i1> %cond, <4 x i32> <i32 1, i32 1, i32 1, i32 1>, <4 x i32> <i32 0, i32 0, i32 0, i32 0>
+  ret <4 x i32> %add
+}
+
+define <4 x i32> @sel_0_or_1_vec(<4 x i1> %cond) {
+; CHECK-LABEL: sel_0_or_1_vec:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-NEXT:    shl v0.4s, v0.4s, #31
+; CHECK-NEXT:    cmge v0.4s, v0.4s, #0
+; CHECK-NEXT:    movi v1.4s, #1
+; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    ret
+  %add = select <4 x i1> %cond, <4 x i32> <i32 0, i32 0, i32 0, i32 0>, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  ret <4 x i32> %add
+}
+
+define <4 x i32> @cmp_sel_0_or_1_vec(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: cmp_sel_0_or_1_vec:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmeq v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    movi v1.4s, #1
+; CHECK-NEXT:    bic v0.16b, v1.16b, v0.16b
+; CHECK-NEXT:    ret
+  %cond = icmp eq <4 x i32> %x, %y
+  %add = select <4 x i1> %cond, <4 x i32> <i32 0, i32 0, i32 0, i32 0>, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  ret <4 x i32> %add
+}
+
diff --git a/llvm/test/CodeGen/AArch64/win64-no-uwtable.ll b/llvm/test/CodeGen/AArch64/win64-no-uwtable.ll
new file mode 100644
index 000000000..f04a47cd1
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/win64-no-uwtable.ll
@@ -0,0 +1,35 @@
+; RUN: llc -mtriple=arm64-windows -o - %s | FileCheck %s
+
+declare void @f()
+declare void @g()
+
+; Function Attrs: nounwind
+define dso_local void @SEHfilter() nounwind "frame-pointer"="all" {
+; CHECK-LABEL: @SEHfilter
+; CHECK:       %bb.0:
+; CHECK-NEXT:  stp     x30, x29, [sp, #-32]!
+; CHECK-NEXT:  str     x19, [sp, #16]
+; CHECK-NEXT:  ldr     w19, [x8]
+; CHECK-NEXT:  mov     x29, sp
+; CHECK-NEXT:  bl      g
+; CHECK-NEXT:  cbz     w19, .LBB0_2
+; CHECK-NEXT:  ; %bb.1:
+; CHECK-NEXT:  ldr     x19, [sp, #16]
+; CHECK-NEXT:  ldp     x30, x29, [sp], #32
+; CHECK-NEXT:  ret
+; CHECK-NEXT:  .LBB0_2:                                ; %if.end.i
+; CHECK-NEXT:  bl      f
+; CHECK-NEXT:  brk     #0x1
+  %1 = load i32, i32* undef, align 4
+  tail call void @g()
+  %tobool.i = icmp eq i32 %1, 0
+  br i1 %tobool.i, label %if.end.i, label %exit
+
+if.end.i:
+  call void @f()
+  unreachable
+
+exit:
+  ret void
+}
+
diff --git a/llvm/test/CodeGen/AArch64/win64_vararg.ll b/llvm/test/CodeGen/AArch64/win64_vararg.ll
index d9bb2ff6b..23e259104 100644
--- a/llvm/test/CodeGen/AArch64/win64_vararg.ll
+++ b/llvm/test/CodeGen/AArch64/win64_vararg.ll
@@ -103,21 +103,21 @@ declare i32 @__stdio_common_vsprintf(i64, i8*, i64, i8*, i8*, i8*) local_unnamed
 declare i64* @__local_stdio_printf_options() local_unnamed_addr #4
 
 ; CHECK-LABEL: fp
-; CHECK: str     x21, [sp, #-96]!
-; CHECK: stp     x19, x20, [sp, #16]
-; CHECK: stp     x29, x30, [sp, #32]
-; CHECK: add     x29, sp, #32
-; CHECK: add     x8, x29, #24
+; CHECK: stp     x29, x30, [sp, #-96]
+; CHECK: str     x21,      [sp, #16]
+; CHECK: stp     x19, x20, [sp, #32]
+; CHECK: mov     x29, sp
+; CHECK: add     x8, x29, #56
 ; CHECK: mov     x19, x2
 ; CHECK: mov     x20, x1
 ; CHECK: mov     x21, x0
-; CHECK: stp     x3, x4, [x29, #24]
-; CHECK: stp     x5, x6, [x29, #40]
-; CHECK: str     x7, [x29, #56]
-; CHECK: str     x8, [sp, #8]
+; CHECK: stp     x3, x4, [x29, #56]
+; CHECK: stp     x5, x6, [x29, #72]
+; CHECK: str     x7, [x29, #88]
+; CHECK: str     x8, [x29, #24]
 ; CHECK: bl      __local_stdio_printf_options
 ; CHECK: ldr     x8, [x0]
-; CHECK: add     x5, x29, #24
+; CHECK: add     x5, x29, #56
 ; CHECK: mov     x1, x21
 ; CHECK: mov     x2, x20
 ; CHECK: orr     x0, x8, #0x2
@@ -126,9 +126,9 @@ declare i64* @__local_stdio_printf_options() local_unnamed_addr #4
 ; CHECK: bl      __stdio_common_vsprintf
 ; CHECK: cmp     w0, #0
 ; CHECK: csinv   w0, w0, wzr, ge
-; CHECK: ldp     x29, x30, [sp, #32]
-; CHECK: ldp     x19, x20, [sp, #16]
-; CHECK: ldr     x21, [sp], #96
+; CHECK: ldp     x19, x20, [sp, #32]
+; CHECK: ldr     x21,      [sp, #16]
+; CHECK: ldp     x29, x30, [sp], #96
 ; CHECK: ret
 define i32 @fp(i8*, i64, i8*, ...) local_unnamed_addr #6 {
   %4 = alloca i8*, align 8
@@ -147,29 +147,29 @@ define i32 @fp(i8*, i64, i8*, ...) local_unnamed_addr #6 {
   ret i32 %12
 }
 
-attributes #6 = { "no-frame-pointer-elim"="true" }
+attributes #6 = { "frame-pointer"="all" }
 
 ; CHECK-LABEL: vla
-; CHECK: str     x23, [sp, #-112]!
-; CHECK: stp     x21, x22, [sp, #16]
-; CHECK: stp     x19, x20, [sp, #32]
-; CHECK: stp     x29, x30, [sp, #48]
-; CHECK: add     x29, sp, #48
-; CHECK: add     x8, x29, #16
-; CHECK: stur    x8, [x29, #-40]
+; CHECK: stp     x29, x30, [sp, #-112]!
+; CHECK: str     x23, [sp, #16]
+; CHECK: stp     x21, x22, [sp, #32]
+; CHECK: stp     x19, x20, [sp, #48]
+; CHECK: mov     x29, sp
+; CHECK: add     x8, x29, #64
+; CHECK: str     x8, [x29, #24]
 ; CHECK: mov     w8, w0
 ; CHECK: add     x8, x8, #15
 ; CHECK: lsr     x15, x8, #4
 ; CHECK: mov     x19, x1
 ; CHECK: mov     [[REG2:x[0-9]+]], sp
-; CHECK: stp     x2, x3, [x29, #16]
-; CHECK: stp     x4, x5, [x29, #32]
-; CHECK: stp     x6, x7, [x29, #48]
+; CHECK: stp     x2, x3, [x29, #64]
+; CHECK: stp     x4, x5, [x29, #80]
+; CHECK: stp     x6, x7, [x29, #96]
 ; CHECK: bl      __chkstk
 ; CHECK: mov     x8, sp
 ; CHECK: sub     [[REG:x[0-9]+]], x8, x15, lsl #4
 ; CHECK: mov     sp, [[REG]]
-; CHECK: ldur    [[REG3:x[0-9]+]], [x29, #-40]
+; CHECK: ldr     [[REG3:x[0-9]+]], [x29, #24]
 ; CHECK: sxtw    [[REG4:x[0-9]+]], w0
 ; CHECK: bl      __local_stdio_printf_options
 ; CHECK: ldr     x8, [x0]
@@ -181,11 +181,11 @@ attributes #6 = { "no-frame-pointer-elim"="true" }
 ; CHECK: mov     x5, [[REG3]]
 ; CHECK: bl      __stdio_common_vsprintf
 ; CHECK: mov     sp, [[REG2]]
-; CHECK: sub     sp, x29, #48
-; CHECK: ldp     x29, x30, [sp, #48]
-; CHECK: ldp     x19, x20, [sp, #32]
-; CHECK: ldp     x21, x22, [sp, #16]
-; CHECK: ldr     x23, [sp], #112
+; CHECK: mov     sp, x29
+; CHECK: ldp     x19, x20, [sp, #48]
+; CHECK: ldp     x21, x22, [sp, #32]
+; CHECK: ldr     x23, [sp, #16]
+; CHECK: ldp     x29, x30, [sp], #112
 ; CHECK: ret
 define void @vla(i32, i8*, ...) local_unnamed_addr {
   %3 = alloca i8*, align 8
@@ -212,9 +212,9 @@ declare void @llvm.stackrestore(i8*)
 
 ; CHECK-LABEL: snprintf
 ; CHECK-DAG: sub     sp,  sp, #96
-; CHECK-DAG: str     x21, [sp, #16]
-; CHECK-DAG: stp     x19, x20, [sp, #24]
-; CHECK-DAG: str     x30, [sp, #40]
+; CHECK-DAG: str     x30, [sp, #16]
+; CHECK-DAG: str     x21, [sp, #24]
+; CHECK-DAG: stp     x19, x20, [sp, #32]
 ; CHECK-DAG: add     x8, sp, #56
 ; CHECK-DAG: mov     x19, x2
 ; CHECK-DAG: mov     x20, x1
@@ -232,9 +232,9 @@ declare void @llvm.stackrestore(i8*)
 ; CHECK-DAG: mov     x3, x19
 ; CHECK-DAG: mov     x4, xzr
 ; CHECK-DAG: bl      __stdio_common_vsprintf
-; CHECK-DAG: ldr     x30, [sp, #40]
-; CHECK-DAG: ldp     x19, x20, [sp, #24]
-; CHECK-DAG: ldr     x21, [sp, #16]
+; CHECK-DAG: ldr     x30, [sp, #16]
+; CHECK-DAG: ldr     x21, [sp, #24]
+; CHECK-DAG: ldp     x19, x20, [sp, #32]
 ; CHECK-DAG: cmp     w0, #0
 ; CHECK-DAG: csinv   w0, w0, wzr, ge
 ; CHECK-DAG: add     sp, sp, #96
@@ -261,11 +261,11 @@ define i32 @snprintf(i8*, i64, i8*, ...) local_unnamed_addr #5 {
 ; CHECK-DAG: mov     w6,  w3
 ; CHECK-DAG: mov     [[REG1:w[0-9]+]],  w2
 ; CHECK: mov     w2, w1
-; CHECK: str     w4,  [sp]
 ; CHECK: fmov    x1,  d0
 ; CHECK: fmov    x3,  d1
 ; CHECK: fmov    x5,  d2
 ; CHECK: fmov    x7,  d3
+; CHECK: str     w4,  [sp]
 ; CHECK: mov     w4,  [[REG1]]
 ; CHECK: str     x30, [sp, #16]
 ; CHECK: str     d4,  [sp, #8]
diff --git a/llvm/test/CodeGen/AArch64/windows-extern-weak.ll b/llvm/test/CodeGen/AArch64/windows-extern-weak.ll
new file mode 100644
index 000000000..e1e6de496
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/windows-extern-weak.ll
@@ -0,0 +1,27 @@
+; RUN: llc -mtriple aarch64-windows -filetype asm -o - < %s | FileCheck %s
+; RUN: llc -mtriple aarch64-windows -filetype asm -o - -fast-isel %s | FileCheck %s
+; RUN: llc -mtriple aarch64-windows -filetype asm -o - -global-isel -global-isel-abort=0 %s | FileCheck %s
+
+define void @func() {
+; CHECK-LABEL: func:
+; CHECK:      str x30, [sp, #-16]!
+; CHECK-NEXT: adrp x8, .refptr.weakfunc
+; CHECK-NEXT: ldr x8, [x8, .refptr.weakfunc]
+; CHECK-NEXT: cbz     x8, .LBB0_2
+; CHECK-NEXT: ; %bb.1:
+; CHECK-NEXT: blr     x8
+; CHECK-NEXT: .LBB0_2:
+; CHECK-NEXT: ldr x30, [sp], #16
+; CHECK-NEXT: ret
+
+  br i1 icmp ne (void ()* @weakfunc, void ()* null), label %1, label %2
+
+1:
+  call void @weakfunc()
+  br label %2
+
+2:
+  ret void
+}
+
+declare extern_weak void @weakfunc()
diff --git a/llvm/test/CodeGen/AArch64/wineh-try-catch-realign.ll b/llvm/test/CodeGen/AArch64/wineh-try-catch-realign.ll
index 78255fbb1..b10a0f303 100644
--- a/llvm/test/CodeGen/AArch64/wineh-try-catch-realign.ll
+++ b/llvm/test/CodeGen/AArch64/wineh-try-catch-realign.ll
@@ -9,17 +9,17 @@
 ; it shouldn't access the parent's frame via sp, and the prologue and
 ; epilogue should be symmetrical.
 ; CHECK-LABEL: "?catch$2@?0??a@@YAXXZ@4HA":
-; CHECK:      str     x28, [sp, #-32]!
-; CHECK-NEXT: str     x19, [sp, #8]
-; CHECK-NEXT: stp     x29, x30, [sp, #16]
+; CHECK:      stp     x29, x30, [sp, #-32]!
+; CHECK-NEXT: str     x28, [sp, #16]
+; CHECK-NEXT: str     x19, [sp, #24]
 ; CHECK-NEXT: add     x0, x19, #64
 ; CHECK-NEXT: mov     w1, wzr
 ; CHECK-NEXT: bl      "?bb@@YAXPEAHH@Z"
 ; CHECK-NEXT: adrp    x0, .LBB0_1
 ; CHECK-NEXT: add     x0, x0, .LBB0_1
-; CHECK-NEXT: ldp     x29, x30, [sp, #16]
-; CHECK-NEXT: ldr     x19, [sp, #8]
-; CHECK-NEXT: ldr     x28, [sp], #32
+; CHECK-NEXT: ldr     x19, [sp, #24]
+; CHECK-NEXT: ldr     x28, [sp, #16]
+; CHECK-NEXT: ldp     x29, x30, [sp], #32
 ; CHECK-NEXT: ret
 
 
diff --git a/llvm/test/CodeGen/AArch64/wineh-try-catch.ll b/llvm/test/CodeGen/AArch64/wineh-try-catch.ll
index c631d77fd..3ae2df37e 100644
--- a/llvm/test/CodeGen/AArch64/wineh-try-catch.ll
+++ b/llvm/test/CodeGen/AArch64/wineh-try-catch.ll
@@ -15,11 +15,11 @@
 ; on-entry sp - 672.  We check this offset in the table later on.
 
 ; CHECK-LABEL: "?func@@YAHXZ":
-; CHECK:       str     x28, [sp, #-48]!
-; CHECK:       str     x21, [sp, #8]
-; CHECK:       stp     x19, x20, [sp, #16]
-; CHECK:       stp     x29, x30, [sp, #32]
-; CHECK:       add     x29, sp, #32
+; CHECK:       stp     x29, x30, [sp, #-48]!
+; CHECK:       str     x28, [sp, #16]
+; CHECK:       str     x21, [sp, #24]
+; CHECK:       stp     x19, x20, [sp, #32]
+; CHECK:       mov     x29, sp
 ; CHECK:       sub     sp, sp, #624
 ; CHECK:       mov     x19, sp
 ; CHECK:       mov     x0, #-2
@@ -47,10 +47,10 @@
 ; CHECK-LABEL: "?catch$2@?0??func@@YAHXZ@4HA":
 
 ; Check that the stack space is allocated only for the callee saved registers.
-; CHECK:       str     x28, [sp, #-48]!
-; CHECK:       str     x21, [sp, #8]
-; CHECK:       stp     x19, x20, [sp, #16]
-; CHECK:       stp     x29, x30, [sp, #32]
+; CHECK:       stp     x29, x30, [sp, #-48]!
+; CHECK:       str     x28, [sp, #16]
+; CHECK:       str     x21, [sp, #24]
+; CHECK:       stp     x19, x20, [sp, #32]
 ; CHECK:       add     x20, x19, #12
 
 ; Check that there are no further stack updates.
@@ -87,18 +87,18 @@
 ; UNWIND: Prologue [
 ; UNWIND-NEXT: ; nop
 ; UNWIND-NEXT: ; sub sp, #624
-; UNWIND-NEXT: ; add fp, sp, #32
-; UNWIND-NEXT: ; stp x29, x30, [sp, #32]
-; UNWIND-NEXT: ; stp x19, x20, [sp, #16]
-; UNWIND-NEXT: ; str x21, [sp, #8]
-; UNWIND-NEXT: ; str x28, [sp, #48]!
+; UNWIND-NEXT: ; mov fp, sp
+; UNWIND-NEXT: ; stp x19, x20, [sp, #32]
+; UNWIND-NEXT: ; str x21, [sp, #24]
+; UNWIND-NEXT: ; str x28, [sp, #16]
+; UNWIND-NEXT: ; stp x29, x30, [sp, #-48]!
 ; UNWIND-NEXT: ; end
 ; UNWIND: Function: ?catch$2@?0??func@@YAHXZ@4HA
 ; UNWIND: Prologue [
-; UNWIND-NEXT: ; stp x29, x30, [sp, #32]
-; UNWIND-NEXT: ; stp x19, x20, [sp, #16]
-; UNWIND-NEXT: ; str x21, [sp, #8]
-; UNWIND-NEXT: ; str x28, [sp, #48]!
+; UNWIND-NEXT: ; stp x19, x20, [sp, #32]
+; UNWIND-NEXT: ; str x21, [sp, #24]
+; UNWIND-NEXT: ; str x28, [sp, #16]
+; UNWIND-NEXT: ; stp x29, x30, [sp, #-48]!
 ; UNWIND-NEXT: ; end
 
 target datalayout = "e-m:w-p:64:64-i32:32-i64:64-i128:128-n32:64-S128"
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/bool-legalization.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/bool-legalization.ll
new file mode 100644
index 000000000..bf4a3a254
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/bool-legalization.ll
@@ -0,0 +1,105 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -global-isel -march=amdgcn -mcpu=hawaii -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+
+; End to end tests for scalar vs. vector boolean legalization strategies.
+
+define amdgpu_ps float @select_vgpr_sgpr_trunc_cond(i32 inreg %a, i32 %b, i32 %c) {
+; GCN-LABEL: select_vgpr_sgpr_trunc_cond:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_and_b32 s0, 1, s0
+; GCN-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GCN-NEXT:    ; return to shader part epilog
+  %cc = trunc i32 %a to i1
+  %r = select i1 %cc, i32 %b, i32 %c
+  %r.f = bitcast i32 %r to float
+  ret float %r.f
+}
+
+define amdgpu_ps float @select_vgpr_sgpr_trunc_and_cond(i32 inreg %a.0, i32 inreg %a.1, i32 %b, i32 %c) {
+; GCN-LABEL: select_vgpr_sgpr_trunc_and_cond:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_and_b32 s0, s0, s1
+; GCN-NEXT:    s_and_b32 s0, 1, s0
+; GCN-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GCN-NEXT:    ; return to shader part epilog
+  %cc.0 = trunc i32 %a.0 to i1
+  %cc.1 = trunc i32 %a.1 to i1
+  %and = and i1 %cc.0, %cc.1
+  %r = select i1 %and, i32 %b, i32 %c
+  %r.f = bitcast i32 %r to float
+  ret float %r.f
+}
+
+define amdgpu_ps i32 @select_sgpr_trunc_and_cond(i32 inreg %a.0, i32 inreg %a.1, i32 inreg %b, i32 inreg %c) {
+; GCN-LABEL: select_sgpr_trunc_and_cond:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_and_b32 s0, s0, s1
+; GCN-NEXT:    s_and_b32 s0, s0, 1
+; GCN-NEXT:    s_cmp_lg_u32 s0, 0
+; GCN-NEXT:    s_cselect_b32 s0, s2, s3
+; GCN-NEXT:    ; return to shader part epilog
+  %cc.0 = trunc i32 %a.0 to i1
+  %cc.1 = trunc i32 %a.1 to i1
+  %and = and i1 %cc.0, %cc.1
+  %r = select i1 %and, i32 %b, i32 %c
+  ret i32 %r
+}
+
+define amdgpu_kernel void @sgpr_trunc_brcond(i32 %cond) {
+; GCN-LABEL: sgpr_trunc_brcond:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_load_dword s0, s[0:1], 0x9
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_and_b32 s0, s0, 1
+; GCN-NEXT:    s_cmp_lg_u32 s0, 0
+; GCN-NEXT:    s_cbranch_scc0 BB3_2
+; GCN-NEXT:  ; %bb.1: ; %bb0
+; GCN-NEXT:    v_mov_b32_e32 v0, 0
+; GCN-NEXT:    flat_store_dword v[0:1], v0
+; GCN-NEXT:  BB3_2: ; %bb1
+; GCN-NEXT:    v_mov_b32_e32 v0, 1
+; GCN-NEXT:    flat_store_dword v[0:1], v0
+entry:
+  %trunc = trunc i32 %cond to i1
+  br i1 %trunc, label %bb0, label %bb1
+
+bb0:
+  store volatile i32 0, i32 addrspace(1)* undef
+  unreachable
+
+bb1:
+  store volatile i32 1, i32 addrspace(1)* undef
+  unreachable
+}
+
+define amdgpu_kernel void @brcond_sgpr_trunc_and(i32 %cond0, i32 %cond1) {
+; GCN-LABEL: brcond_sgpr_trunc_and:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_and_b32 s0, s0, s1
+; GCN-NEXT:    s_and_b32 s0, s0, 1
+; GCN-NEXT:    s_cmp_lg_u32 s0, 0
+; GCN-NEXT:    s_cbranch_scc0 BB4_2
+; GCN-NEXT:  ; %bb.1: ; %bb0
+; GCN-NEXT:    v_mov_b32_e32 v0, 0
+; GCN-NEXT:    flat_store_dword v[0:1], v0
+; GCN-NEXT:  BB4_2: ; %bb1
+; GCN-NEXT:    v_mov_b32_e32 v0, 1
+; GCN-NEXT:    flat_store_dword v[0:1], v0
+entry:
+  %trunc0 = trunc i32 %cond0 to i1
+  %trunc1 = trunc i32 %cond1 to i1
+  %and = and i1 %trunc0, %trunc1
+  br i1 %and, label %bb0, label %bb1
+
+bb0:
+  store volatile i32 0, i32 addrspace(1)* undef
+  unreachable
+
+bb1:
+  store volatile i32 1, i32 addrspace(1)* undef
+  unreachable
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergent-control-flow.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergent-control-flow.ll
new file mode 100644
index 000000000..40e182067
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergent-control-flow.ll
@@ -0,0 +1,61 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 < %s | FileCheck %s
+
+; Make sure the branch targets are correct after lowering llvm.amdgcn.if
+
+define i32 @divergent_if_swap_brtarget_order0(i32 %value) {
+; CHECK-LABEL: divergent_if_swap_brtarget_order0:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; CHECK-NEXT:    ; implicit-def: $vgpr0
+; CHECK-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; CHECK-NEXT:    ; mask branch BB0_2
+; CHECK-NEXT:    s_cbranch_execz BB0_2
+; CHECK-NEXT:  BB0_1: ; %if.true
+; CHECK-NEXT:    global_load_dword v0, v[0:1], off
+; CHECK-NEXT:  BB0_2: ; %endif
+; CHECK-NEXT:    s_or_b64 exec, exec, s[4:5]
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+entry:
+  %c = icmp ne i32 %value, 0
+  br i1 %c, label %if.true, label %endif
+
+if.true:
+  %val = load volatile i32, i32 addrspace(1)* undef
+  br label %endif
+
+endif:
+  %v = phi i32 [ %val, %if.true ], [ undef, %entry ]
+  ret i32 %v
+}
+
+define i32 @divergent_if_swap_brtarget_order1(i32 %value) {
+; CHECK-LABEL: divergent_if_swap_brtarget_order1:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; CHECK-NEXT:    ; implicit-def: $vgpr0
+; CHECK-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; CHECK-NEXT:    ; mask branch BB1_2
+; CHECK-NEXT:  BB1_1: ; %endif
+; CHECK-NEXT:    s_or_b64 exec, exec, s[4:5]
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+; CHECK-NEXT:  BB1_2: ; %if.true
+; CHECK-NEXT:    global_load_dword v0, v[0:1], off
+; CHECK-NEXT:    s_or_b64 exec, exec, s[4:5]
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+entry:
+  %c = icmp ne i32 %value, 0
+  br i1 %c, label %if.true, label %endif
+
+endif:
+  %v = phi i32 [ %val, %if.true ], [ undef, %entry ]
+  ret i32 %v
+
+if.true:
+  %val = load volatile i32, i32 addrspace(1)* undef
+  br label %endif
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.ll
new file mode 100644
index 000000000..4f9d35dd9
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.ll
@@ -0,0 +1,1289 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefix=GPRIDX %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=MOVREL %s
+
+define float @dyn_extract_v8f32_const_s_v(i32 %sel) {
+; GPRIDX-LABEL: dyn_extract_v8f32_const_s_v:
+; GPRIDX:       ; %bb.0: ; %entry
+; GPRIDX-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GPRIDX-NEXT:    s_mov_b32 s4, 1.0
+; GPRIDX-NEXT:    s_mov_b32 s5, 2.0
+; GPRIDX-NEXT:    s_mov_b32 s6, 0x40400000
+; GPRIDX-NEXT:    s_mov_b32 s7, 4.0
+; GPRIDX-NEXT:    s_mov_b32 s8, 0x40a00000
+; GPRIDX-NEXT:    s_mov_b32 s9, 0x40c00000
+; GPRIDX-NEXT:    s_mov_b32 s10, 0x40e00000
+; GPRIDX-NEXT:    s_mov_b32 s11, 0x41000000
+; GPRIDX-NEXT:    s_mov_b64 s[12:13], exec
+; GPRIDX-NEXT:  BB0_1: ; =>This Inner Loop Header: Depth=1
+; GPRIDX-NEXT:    v_readfirstlane_b32 s14, v0
+; GPRIDX-NEXT:    s_mov_b32 m0, s14
+; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, s14, v0
+; GPRIDX-NEXT:    s_movrels_b32 s14, s4
+; GPRIDX-NEXT:    v_mov_b32_e32 v1, s14
+; GPRIDX-NEXT:    s_and_saveexec_b64 vcc, vcc
+; GPRIDX-NEXT:    s_xor_b64 exec, exec, vcc
+; GPRIDX-NEXT:    s_cbranch_execnz BB0_1
+; GPRIDX-NEXT:  ; %bb.2:
+; GPRIDX-NEXT:    s_mov_b64 exec, s[12:13]
+; GPRIDX-NEXT:    v_mov_b32_e32 v0, v1
+; GPRIDX-NEXT:    s_setpc_b64 s[30:31]
+;
+; MOVREL-LABEL: dyn_extract_v8f32_const_s_v:
+; MOVREL:       ; %bb.0: ; %entry
+; MOVREL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; MOVREL-NEXT:    s_mov_b32 s4, 1.0
+; MOVREL-NEXT:    s_mov_b32 s5, 2.0
+; MOVREL-NEXT:    s_mov_b32 s6, 0x40400000
+; MOVREL-NEXT:    s_mov_b32 s7, 4.0
+; MOVREL-NEXT:    s_mov_b32 s8, 0x40a00000
+; MOVREL-NEXT:    s_mov_b32 s9, 0x40c00000
+; MOVREL-NEXT:    s_mov_b32 s10, 0x40e00000
+; MOVREL-NEXT:    s_mov_b32 s11, 0x41000000
+; MOVREL-NEXT:    s_mov_b64 s[12:13], exec
+; MOVREL-NEXT:  BB0_1: ; =>This Inner Loop Header: Depth=1
+; MOVREL-NEXT:    v_readfirstlane_b32 s14, v0
+; MOVREL-NEXT:    s_mov_b32 m0, s14
+; MOVREL-NEXT:    v_cmp_eq_u32_e32 vcc, s14, v0
+; MOVREL-NEXT:    s_movrels_b32 s14, s4
+; MOVREL-NEXT:    v_mov_b32_e32 v1, s14
+; MOVREL-NEXT:    s_and_saveexec_b64 vcc, vcc
+; MOVREL-NEXT:    s_xor_b64 exec, exec, vcc
+; MOVREL-NEXT:    s_cbranch_execnz BB0_1
+; MOVREL-NEXT:  ; %bb.2:
+; MOVREL-NEXT:    s_mov_b64 exec, s[12:13]
+; MOVREL-NEXT:    v_mov_b32_e32 v0, v1
+; MOVREL-NEXT:    s_setpc_b64 s[30:31]
+entry:
+  %ext = extractelement <8 x float> <float 1.0, float 2.0, float 3.0, float 4.0, float 5.0, float 6.0, float 7.0, float 8.0>, i32 %sel
+  ret float %ext
+}
+
+define amdgpu_ps float @dyn_extract_v8f32_const_s_s(i32 inreg %sel) {
+; GPRIDX-LABEL: dyn_extract_v8f32_const_s_s:
+; GPRIDX:       ; %bb.0: ; %entry
+; GPRIDX-NEXT:    s_mov_b32 s4, 1.0
+; GPRIDX-NEXT:    s_mov_b32 m0, s2
+; GPRIDX-NEXT:    s_mov_b32 s5, 2.0
+; GPRIDX-NEXT:    s_mov_b32 s6, 0x40400000
+; GPRIDX-NEXT:    s_mov_b32 s7, 4.0
+; GPRIDX-NEXT:    s_mov_b32 s8, 0x40a00000
+; GPRIDX-NEXT:    s_mov_b32 s9, 0x40c00000
+; GPRIDX-NEXT:    s_mov_b32 s10, 0x40e00000
+; GPRIDX-NEXT:    s_mov_b32 s11, 0x41000000
+; GPRIDX-NEXT:    s_movrels_b32 s0, s4
+; GPRIDX-NEXT:    v_mov_b32_e32 v0, s0
+; GPRIDX-NEXT:    ; return to shader part epilog
+;
+; MOVREL-LABEL: dyn_extract_v8f32_const_s_s:
+; MOVREL:       ; %bb.0: ; %entry
+; MOVREL-NEXT:    s_mov_b32 s4, 1.0
+; MOVREL-NEXT:    s_mov_b32 m0, s2
+; MOVREL-NEXT:    s_mov_b32 s5, 2.0
+; MOVREL-NEXT:    s_mov_b32 s6, 0x40400000
+; MOVREL-NEXT:    s_mov_b32 s7, 4.0
+; MOVREL-NEXT:    s_mov_b32 s8, 0x40a00000
+; MOVREL-NEXT:    s_mov_b32 s9, 0x40c00000
+; MOVREL-NEXT:    s_mov_b32 s10, 0x40e00000
+; MOVREL-NEXT:    s_mov_b32 s11, 0x41000000
+; MOVREL-NEXT:    s_movrels_b32 s0, s4
+; MOVREL-NEXT:    v_mov_b32_e32 v0, s0
+; MOVREL-NEXT:    ; return to shader part epilog
+entry:
+  %ext = extractelement <8 x float> <float 1.0, float 2.0, float 3.0, float 4.0, float 5.0, float 6.0, float 7.0, float 8.0>, i32 %sel
+  ret float %ext
+}
+
+define amdgpu_ps float @dyn_extract_v8f32_s_v(<8 x float> inreg %vec, i32 %sel) {
+; GPRIDX-LABEL: dyn_extract_v8f32_s_v:
+; GPRIDX:       ; %bb.0: ; %entry
+; GPRIDX-NEXT:    s_mov_b32 s0, s2
+; GPRIDX-NEXT:    s_mov_b32 s1, s3
+; GPRIDX-NEXT:    s_mov_b32 s2, s4
+; GPRIDX-NEXT:    s_mov_b32 s3, s5
+; GPRIDX-NEXT:    s_mov_b32 s4, s6
+; GPRIDX-NEXT:    s_mov_b32 s5, s7
+; GPRIDX-NEXT:    s_mov_b32 s6, s8
+; GPRIDX-NEXT:    s_mov_b32 s7, s9
+; GPRIDX-NEXT:    s_mov_b64 s[8:9], exec
+; GPRIDX-NEXT:  BB2_1: ; =>This Inner Loop Header: Depth=1
+; GPRIDX-NEXT:    v_readfirstlane_b32 s10, v0
+; GPRIDX-NEXT:    s_mov_b32 m0, s10
+; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, s10, v0
+; GPRIDX-NEXT:    s_movrels_b32 s10, s0
+; GPRIDX-NEXT:    v_mov_b32_e32 v1, s10
+; GPRIDX-NEXT:    s_and_saveexec_b64 vcc, vcc
+; GPRIDX-NEXT:    s_xor_b64 exec, exec, vcc
+; GPRIDX-NEXT:    s_cbranch_execnz BB2_1
+; GPRIDX-NEXT:  ; %bb.2:
+; GPRIDX-NEXT:    s_mov_b64 exec, s[8:9]
+; GPRIDX-NEXT:    v_mov_b32_e32 v0, v1
+; GPRIDX-NEXT:    ; return to shader part epilog
+;
+; MOVREL-LABEL: dyn_extract_v8f32_s_v:
+; MOVREL:       ; %bb.0: ; %entry
+; MOVREL-NEXT:    s_mov_b32 s0, s2
+; MOVREL-NEXT:    s_mov_b32 s1, s3
+; MOVREL-NEXT:    s_mov_b32 s2, s4
+; MOVREL-NEXT:    s_mov_b32 s3, s5
+; MOVREL-NEXT:    s_mov_b32 s4, s6
+; MOVREL-NEXT:    s_mov_b32 s5, s7
+; MOVREL-NEXT:    s_mov_b32 s6, s8
+; MOVREL-NEXT:    s_mov_b32 s7, s9
+; MOVREL-NEXT:    s_mov_b64 s[8:9], exec
+; MOVREL-NEXT:  BB2_1: ; =>This Inner Loop Header: Depth=1
+; MOVREL-NEXT:    v_readfirstlane_b32 s10, v0
+; MOVREL-NEXT:    s_mov_b32 m0, s10
+; MOVREL-NEXT:    v_cmp_eq_u32_e32 vcc, s10, v0
+; MOVREL-NEXT:    s_movrels_b32 s10, s0
+; MOVREL-NEXT:    v_mov_b32_e32 v1, s10
+; MOVREL-NEXT:    s_and_saveexec_b64 vcc, vcc
+; MOVREL-NEXT:    s_xor_b64 exec, exec, vcc
+; MOVREL-NEXT:    s_cbranch_execnz BB2_1
+; MOVREL-NEXT:  ; %bb.2:
+; MOVREL-NEXT:    s_mov_b64 exec, s[8:9]
+; MOVREL-NEXT:    v_mov_b32_e32 v0, v1
+; MOVREL-NEXT:    ; return to shader part epilog
+entry:
+  %ext = extractelement <8 x float> %vec, i32 %sel
+  ret float %ext
+}
+
+define float @dyn_extract_v8f32_v_v(<8 x float> %vec, i32 %sel) {
+; GPRIDX-LABEL: dyn_extract_v8f32_v_v:
+; GPRIDX:       ; %bb.0: ; %entry
+; GPRIDX-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GPRIDX-NEXT:    s_mov_b64 s[4:5], exec
+; GPRIDX-NEXT:  BB3_1: ; =>This Inner Loop Header: Depth=1
+; GPRIDX-NEXT:    v_readfirstlane_b32 s6, v8
+; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, s6, v8
+; GPRIDX-NEXT:    s_set_gpr_idx_on s6, gpr_idx(SRC0)
+; GPRIDX-NEXT:    v_mov_b32_e32 v9, v0
+; GPRIDX-NEXT:    s_set_gpr_idx_off
+; GPRIDX-NEXT:    s_and_saveexec_b64 vcc, vcc
+; GPRIDX-NEXT:    s_xor_b64 exec, exec, vcc
+; GPRIDX-NEXT:    s_cbranch_execnz BB3_1
+; GPRIDX-NEXT:  ; %bb.2:
+; GPRIDX-NEXT:    s_mov_b64 exec, s[4:5]
+; GPRIDX-NEXT:    v_mov_b32_e32 v0, v9
+; GPRIDX-NEXT:    s_setpc_b64 s[30:31]
+;
+; MOVREL-LABEL: dyn_extract_v8f32_v_v:
+; MOVREL:       ; %bb.0: ; %entry
+; MOVREL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; MOVREL-NEXT:    s_mov_b64 s[4:5], exec
+; MOVREL-NEXT:  BB3_1: ; =>This Inner Loop Header: Depth=1
+; MOVREL-NEXT:    v_readfirstlane_b32 s6, v8
+; MOVREL-NEXT:    s_mov_b32 m0, s6
+; MOVREL-NEXT:    v_cmp_eq_u32_e32 vcc, s6, v8
+; MOVREL-NEXT:    v_movrels_b32_e32 v9, v0
+; MOVREL-NEXT:    s_and_saveexec_b64 vcc, vcc
+; MOVREL-NEXT:    s_xor_b64 exec, exec, vcc
+; MOVREL-NEXT:    s_cbranch_execnz BB3_1
+; MOVREL-NEXT:  ; %bb.2:
+; MOVREL-NEXT:    s_mov_b64 exec, s[4:5]
+; MOVREL-NEXT:    v_mov_b32_e32 v0, v9
+; MOVREL-NEXT:    s_setpc_b64 s[30:31]
+entry:
+  %ext = extractelement <8 x float> %vec, i32 %sel
+  ret float %ext
+}
+
+define amdgpu_ps float @dyn_extract_v8f32_v_s(<8 x float> %vec, i32 inreg %sel) {
+; GPRIDX-LABEL: dyn_extract_v8f32_v_s:
+; GPRIDX:       ; %bb.0: ; %entry
+; GPRIDX-NEXT:    s_set_gpr_idx_on s2, gpr_idx(SRC0)
+; GPRIDX-NEXT:    v_mov_b32_e32 v0, v0
+; GPRIDX-NEXT:    s_set_gpr_idx_off
+; GPRIDX-NEXT:    ; return to shader part epilog
+;
+; MOVREL-LABEL: dyn_extract_v8f32_v_s:
+; MOVREL:       ; %bb.0: ; %entry
+; MOVREL-NEXT:    s_mov_b32 m0, s2
+; MOVREL-NEXT:    v_movrels_b32_e32 v0, v0
+; MOVREL-NEXT:    ; return to shader part epilog
+entry:
+  %ext = extractelement <8 x float> %vec, i32 %sel
+  ret float %ext
+}
+
+define amdgpu_ps float @dyn_extract_v8f32_s_s(<8 x float> inreg %vec, i32 inreg %sel) {
+; GPRIDX-LABEL: dyn_extract_v8f32_s_s:
+; GPRIDX:       ; %bb.0: ; %entry
+; GPRIDX-NEXT:    s_mov_b32 s0, s2
+; GPRIDX-NEXT:    s_mov_b32 m0, s10
+; GPRIDX-NEXT:    s_mov_b32 s1, s3
+; GPRIDX-NEXT:    s_mov_b32 s2, s4
+; GPRIDX-NEXT:    s_mov_b32 s3, s5
+; GPRIDX-NEXT:    s_mov_b32 s4, s6
+; GPRIDX-NEXT:    s_mov_b32 s5, s7
+; GPRIDX-NEXT:    s_mov_b32 s6, s8
+; GPRIDX-NEXT:    s_mov_b32 s7, s9
+; GPRIDX-NEXT:    s_movrels_b32 s0, s0
+; GPRIDX-NEXT:    v_mov_b32_e32 v0, s0
+; GPRIDX-NEXT:    ; return to shader part epilog
+;
+; MOVREL-LABEL: dyn_extract_v8f32_s_s:
+; MOVREL:       ; %bb.0: ; %entry
+; MOVREL-NEXT:    s_mov_b32 s0, s2
+; MOVREL-NEXT:    s_mov_b32 m0, s10
+; MOVREL-NEXT:    s_mov_b32 s1, s3
+; MOVREL-NEXT:    s_mov_b32 s2, s4
+; MOVREL-NEXT:    s_mov_b32 s3, s5
+; MOVREL-NEXT:    s_mov_b32 s4, s6
+; MOVREL-NEXT:    s_mov_b32 s5, s7
+; MOVREL-NEXT:    s_mov_b32 s6, s8
+; MOVREL-NEXT:    s_mov_b32 s7, s9
+; MOVREL-NEXT:    s_movrels_b32 s0, s0
+; MOVREL-NEXT:    v_mov_b32_e32 v0, s0
+; MOVREL-NEXT:    ; return to shader part epilog
+entry:
+  %ext = extractelement <8 x float> %vec, i32 %sel
+  ret float %ext
+}
+
+define i64 @dyn_extract_v8i64_const_s_v(i32 %sel) {
+; GPRIDX-LABEL: dyn_extract_v8i64_const_s_v:
+; GPRIDX:       ; %bb.0: ; %entry
+; GPRIDX-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GPRIDX-NEXT:    s_mov_b64 s[4:5], 1
+; GPRIDX-NEXT:    s_mov_b64 s[6:7], 2
+; GPRIDX-NEXT:    s_mov_b64 s[8:9], 3
+; GPRIDX-NEXT:    s_mov_b64 s[10:11], 4
+; GPRIDX-NEXT:    s_mov_b64 s[12:13], 5
+; GPRIDX-NEXT:    s_mov_b64 s[14:15], 6
+; GPRIDX-NEXT:    s_mov_b64 s[16:17], 7
+; GPRIDX-NEXT:    s_mov_b64 s[18:19], 8
+; GPRIDX-NEXT:    s_mov_b64 s[20:21], exec
+; GPRIDX-NEXT:  BB6_1: ; =>This Inner Loop Header: Depth=1
+; GPRIDX-NEXT:    v_readfirstlane_b32 s22, v0
+; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, s22, v0
+; GPRIDX-NEXT:    s_lshl_b32 s22, s22, 1
+; GPRIDX-NEXT:    s_add_u32 s23, s22, 1
+; GPRIDX-NEXT:    s_mov_b32 m0, s22
+; GPRIDX-NEXT:    s_nop 0
+; GPRIDX-NEXT:    s_movrels_b32 s22, s4
+; GPRIDX-NEXT:    s_mov_b32 m0, s23
+; GPRIDX-NEXT:    s_nop 0
+; GPRIDX-NEXT:    s_movrels_b32 s23, s4
+; GPRIDX-NEXT:    s_and_saveexec_b64 vcc, vcc
+; GPRIDX-NEXT:    s_xor_b64 exec, exec, vcc
+; GPRIDX-NEXT:    s_cbranch_execnz BB6_1
+; GPRIDX-NEXT:  ; %bb.2:
+; GPRIDX-NEXT:    s_mov_b64 exec, s[20:21]
+; GPRIDX-NEXT:    v_mov_b32_e32 v0, s22
+; GPRIDX-NEXT:    v_mov_b32_e32 v1, s23
+; GPRIDX-NEXT:    s_setpc_b64 s[30:31]
+;
+; MOVREL-LABEL: dyn_extract_v8i64_const_s_v:
+; MOVREL:       ; %bb.0: ; %entry
+; MOVREL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; MOVREL-NEXT:    s_mov_b64 s[4:5], 1
+; MOVREL-NEXT:    s_mov_b64 s[6:7], 2
+; MOVREL-NEXT:    s_mov_b64 s[8:9], 3
+; MOVREL-NEXT:    s_mov_b64 s[10:11], 4
+; MOVREL-NEXT:    s_mov_b64 s[12:13], 5
+; MOVREL-NEXT:    s_mov_b64 s[14:15], 6
+; MOVREL-NEXT:    s_mov_b64 s[16:17], 7
+; MOVREL-NEXT:    s_mov_b64 s[18:19], 8
+; MOVREL-NEXT:    s_mov_b64 s[20:21], exec
+; MOVREL-NEXT:  BB6_1: ; =>This Inner Loop Header: Depth=1
+; MOVREL-NEXT:    v_readfirstlane_b32 s22, v0
+; MOVREL-NEXT:    v_cmp_eq_u32_e32 vcc, s22, v0
+; MOVREL-NEXT:    s_lshl_b32 s22, s22, 1
+; MOVREL-NEXT:    s_add_u32 s23, s22, 1
+; MOVREL-NEXT:    s_mov_b32 m0, s22
+; MOVREL-NEXT:    s_movrels_b32 s22, s4
+; MOVREL-NEXT:    s_mov_b32 m0, s23
+; MOVREL-NEXT:    s_movrels_b32 s23, s4
+; MOVREL-NEXT:    s_and_saveexec_b64 vcc, vcc
+; MOVREL-NEXT:    s_xor_b64 exec, exec, vcc
+; MOVREL-NEXT:    s_cbranch_execnz BB6_1
+; MOVREL-NEXT:  ; %bb.2:
+; MOVREL-NEXT:    s_mov_b64 exec, s[20:21]
+; MOVREL-NEXT:    v_mov_b32_e32 v0, s22
+; MOVREL-NEXT:    v_mov_b32_e32 v1, s23
+; MOVREL-NEXT:    s_setpc_b64 s[30:31]
+entry:
+  %ext = extractelement <8 x i64> <i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8>, i32 %sel
+  ret i64 %ext
+}
+
+define amdgpu_ps void @dyn_extract_v8i64_const_s_s(i32 inreg %sel) {
+; GPRIDX-LABEL: dyn_extract_v8i64_const_s_s:
+; GPRIDX:       ; %bb.0: ; %entry
+; GPRIDX-NEXT:    s_mov_b64 s[4:5], 1
+; GPRIDX-NEXT:    s_mov_b32 m0, s2
+; GPRIDX-NEXT:    s_mov_b64 s[6:7], 2
+; GPRIDX-NEXT:    s_mov_b64 s[8:9], 3
+; GPRIDX-NEXT:    s_mov_b64 s[10:11], 4
+; GPRIDX-NEXT:    s_mov_b64 s[12:13], 5
+; GPRIDX-NEXT:    s_mov_b64 s[14:15], 6
+; GPRIDX-NEXT:    s_mov_b64 s[16:17], 7
+; GPRIDX-NEXT:    s_mov_b64 s[18:19], 8
+; GPRIDX-NEXT:    s_movrels_b64 s[0:1], s[4:5]
+; GPRIDX-NEXT:    v_mov_b32_e32 v0, s0
+; GPRIDX-NEXT:    v_mov_b32_e32 v1, s1
+; GPRIDX-NEXT:    global_store_dwordx2 v[0:1], v[0:1], off
+; GPRIDX-NEXT:    s_endpgm
+;
+; MOVREL-LABEL: dyn_extract_v8i64_const_s_s:
+; MOVREL:       ; %bb.0: ; %entry
+; MOVREL-NEXT:    s_mov_b64 s[4:5], 1
+; MOVREL-NEXT:    s_mov_b32 m0, s2
+; MOVREL-NEXT:    s_mov_b64 s[6:7], 2
+; MOVREL-NEXT:    s_mov_b64 s[8:9], 3
+; MOVREL-NEXT:    s_mov_b64 s[10:11], 4
+; MOVREL-NEXT:    s_mov_b64 s[12:13], 5
+; MOVREL-NEXT:    s_mov_b64 s[14:15], 6
+; MOVREL-NEXT:    s_mov_b64 s[16:17], 7
+; MOVREL-NEXT:    s_mov_b64 s[18:19], 8
+; MOVREL-NEXT:    s_movrels_b64 s[0:1], s[4:5]
+; MOVREL-NEXT:    v_mov_b32_e32 v0, s0
+; MOVREL-NEXT:    v_mov_b32_e32 v1, s1
+; MOVREL-NEXT:    flat_store_dwordx2 v[0:1], v[0:1]
+; MOVREL-NEXT:    s_endpgm
+entry:
+  %ext = extractelement <8 x i64> <i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8>, i32 %sel
+  store i64 %ext, i64 addrspace(1)* undef
+  ret void
+}
+
+define amdgpu_ps void @dyn_extract_v8i64_s_v(<8 x i64> inreg %vec, i32 %sel) {
+; GPRIDX-LABEL: dyn_extract_v8i64_s_v:
+; GPRIDX:       ; %bb.0: ; %entry
+; GPRIDX-NEXT:    s_mov_b32 s0, s2
+; GPRIDX-NEXT:    s_mov_b32 s1, s3
+; GPRIDX-NEXT:    s_mov_b32 s2, s4
+; GPRIDX-NEXT:    s_mov_b32 s3, s5
+; GPRIDX-NEXT:    s_mov_b32 s4, s6
+; GPRIDX-NEXT:    s_mov_b32 s5, s7
+; GPRIDX-NEXT:    s_mov_b32 s6, s8
+; GPRIDX-NEXT:    s_mov_b32 s7, s9
+; GPRIDX-NEXT:    s_mov_b32 s8, s10
+; GPRIDX-NEXT:    s_mov_b32 s9, s11
+; GPRIDX-NEXT:    s_mov_b32 s10, s12
+; GPRIDX-NEXT:    s_mov_b32 s11, s13
+; GPRIDX-NEXT:    s_mov_b32 s12, s14
+; GPRIDX-NEXT:    s_mov_b32 s13, s15
+; GPRIDX-NEXT:    s_mov_b32 s14, s16
+; GPRIDX-NEXT:    s_mov_b32 s15, s17
+; GPRIDX-NEXT:    s_mov_b64 s[16:17], exec
+; GPRIDX-NEXT:  BB8_1: ; =>This Inner Loop Header: Depth=1
+; GPRIDX-NEXT:    v_readfirstlane_b32 s18, v0
+; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, s18, v0
+; GPRIDX-NEXT:    s_lshl_b32 s18, s18, 1
+; GPRIDX-NEXT:    s_add_u32 s19, s18, 1
+; GPRIDX-NEXT:    s_mov_b32 m0, s18
+; GPRIDX-NEXT:    s_nop 0
+; GPRIDX-NEXT:    s_movrels_b32 s18, s0
+; GPRIDX-NEXT:    s_mov_b32 m0, s19
+; GPRIDX-NEXT:    v_mov_b32_e32 v1, s18
+; GPRIDX-NEXT:    s_movrels_b32 s19, s0
+; GPRIDX-NEXT:    v_mov_b32_e32 v2, s19
+; GPRIDX-NEXT:    s_and_saveexec_b64 vcc, vcc
+; GPRIDX-NEXT:    s_xor_b64 exec, exec, vcc
+; GPRIDX-NEXT:    s_cbranch_execnz BB8_1
+; GPRIDX-NEXT:  ; %bb.2:
+; GPRIDX-NEXT:    s_mov_b64 exec, s[16:17]
+; GPRIDX-NEXT:    global_store_dwordx2 v[0:1], v[1:2], off
+; GPRIDX-NEXT:    s_endpgm
+;
+; MOVREL-LABEL: dyn_extract_v8i64_s_v:
+; MOVREL:       ; %bb.0: ; %entry
+; MOVREL-NEXT:    s_mov_b32 s0, s2
+; MOVREL-NEXT:    s_mov_b32 s1, s3
+; MOVREL-NEXT:    s_mov_b32 s2, s4
+; MOVREL-NEXT:    s_mov_b32 s3, s5
+; MOVREL-NEXT:    s_mov_b32 s4, s6
+; MOVREL-NEXT:    s_mov_b32 s5, s7
+; MOVREL-NEXT:    s_mov_b32 s6, s8
+; MOVREL-NEXT:    s_mov_b32 s7, s9
+; MOVREL-NEXT:    s_mov_b32 s8, s10
+; MOVREL-NEXT:    s_mov_b32 s9, s11
+; MOVREL-NEXT:    s_mov_b32 s10, s12
+; MOVREL-NEXT:    s_mov_b32 s11, s13
+; MOVREL-NEXT:    s_mov_b32 s12, s14
+; MOVREL-NEXT:    s_mov_b32 s13, s15
+; MOVREL-NEXT:    s_mov_b32 s14, s16
+; MOVREL-NEXT:    s_mov_b32 s15, s17
+; MOVREL-NEXT:    s_mov_b64 s[16:17], exec
+; MOVREL-NEXT:  BB8_1: ; =>This Inner Loop Header: Depth=1
+; MOVREL-NEXT:    v_readfirstlane_b32 s18, v0
+; MOVREL-NEXT:    v_cmp_eq_u32_e32 vcc, s18, v0
+; MOVREL-NEXT:    s_lshl_b32 s18, s18, 1
+; MOVREL-NEXT:    s_add_u32 s19, s18, 1
+; MOVREL-NEXT:    s_mov_b32 m0, s18
+; MOVREL-NEXT:    s_movrels_b32 s18, s0
+; MOVREL-NEXT:    s_mov_b32 m0, s19
+; MOVREL-NEXT:    s_movrels_b32 s19, s0
+; MOVREL-NEXT:    v_mov_b32_e32 v1, s18
+; MOVREL-NEXT:    v_mov_b32_e32 v2, s19
+; MOVREL-NEXT:    s_and_saveexec_b64 vcc, vcc
+; MOVREL-NEXT:    s_xor_b64 exec, exec, vcc
+; MOVREL-NEXT:    s_cbranch_execnz BB8_1
+; MOVREL-NEXT:  ; %bb.2:
+; MOVREL-NEXT:    s_mov_b64 exec, s[16:17]
+; MOVREL-NEXT:    flat_store_dwordx2 v[0:1], v[1:2]
+; MOVREL-NEXT:    s_endpgm
+entry:
+  %ext = extractelement <8 x i64> %vec, i32 %sel
+  store i64 %ext, i64 addrspace(1)* undef
+  ret void
+}
+
+define i64 @dyn_extract_v8i64_v_v(<8 x i64> %vec, i32 %sel) {
+; GPRIDX-LABEL: dyn_extract_v8i64_v_v:
+; GPRIDX:       ; %bb.0: ; %entry
+; GPRIDX-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GPRIDX-NEXT:    s_mov_b64 s[4:5], exec
+; GPRIDX-NEXT:  BB9_1: ; =>This Inner Loop Header: Depth=1
+; GPRIDX-NEXT:    v_readfirstlane_b32 s6, v16
+; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, s6, v16
+; GPRIDX-NEXT:    s_lshl_b32 s6, s6, 1
+; GPRIDX-NEXT:    s_add_u32 s7, s6, 1
+; GPRIDX-NEXT:    s_set_gpr_idx_on s6, gpr_idx(SRC0)
+; GPRIDX-NEXT:    v_mov_b32_e32 v17, v0
+; GPRIDX-NEXT:    s_set_gpr_idx_off
+; GPRIDX-NEXT:    s_set_gpr_idx_on s7, gpr_idx(SRC0)
+; GPRIDX-NEXT:    v_mov_b32_e32 v18, v0
+; GPRIDX-NEXT:    s_set_gpr_idx_off
+; GPRIDX-NEXT:    s_and_saveexec_b64 vcc, vcc
+; GPRIDX-NEXT:    s_xor_b64 exec, exec, vcc
+; GPRIDX-NEXT:    s_cbranch_execnz BB9_1
+; GPRIDX-NEXT:  ; %bb.2:
+; GPRIDX-NEXT:    s_mov_b64 exec, s[4:5]
+; GPRIDX-NEXT:    v_mov_b32_e32 v0, v17
+; GPRIDX-NEXT:    v_mov_b32_e32 v1, v18
+; GPRIDX-NEXT:    s_setpc_b64 s[30:31]
+;
+; MOVREL-LABEL: dyn_extract_v8i64_v_v:
+; MOVREL:       ; %bb.0: ; %entry
+; MOVREL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; MOVREL-NEXT:    s_mov_b64 s[4:5], exec
+; MOVREL-NEXT:  BB9_1: ; =>This Inner Loop Header: Depth=1
+; MOVREL-NEXT:    v_readfirstlane_b32 s6, v16
+; MOVREL-NEXT:    v_cmp_eq_u32_e32 vcc, s6, v16
+; MOVREL-NEXT:    s_lshl_b32 s6, s6, 1
+; MOVREL-NEXT:    s_mov_b32 m0, s6
+; MOVREL-NEXT:    s_add_u32 s7, s6, 1
+; MOVREL-NEXT:    v_movrels_b32_e32 v17, v0
+; MOVREL-NEXT:    s_mov_b32 m0, s7
+; MOVREL-NEXT:    v_movrels_b32_e32 v18, v0
+; MOVREL-NEXT:    s_and_saveexec_b64 vcc, vcc
+; MOVREL-NEXT:    s_xor_b64 exec, exec, vcc
+; MOVREL-NEXT:    s_cbranch_execnz BB9_1
+; MOVREL-NEXT:  ; %bb.2:
+; MOVREL-NEXT:    s_mov_b64 exec, s[4:5]
+; MOVREL-NEXT:    v_mov_b32_e32 v0, v17
+; MOVREL-NEXT:    v_mov_b32_e32 v1, v18
+; MOVREL-NEXT:    s_setpc_b64 s[30:31]
+entry:
+  %ext = extractelement <8 x i64> %vec, i32 %sel
+  ret i64 %ext
+}
+
+define amdgpu_ps void @dyn_extract_v8i64_v_s(<8 x i64> %vec, i32 inreg %sel) {
+; GPRIDX-LABEL: dyn_extract_v8i64_v_s:
+; GPRIDX:       ; %bb.0: ; %entry
+; GPRIDX-NEXT:    s_lshl_b32 s0, s2, 1
+; GPRIDX-NEXT:    s_add_u32 s1, s0, 1
+; GPRIDX-NEXT:    s_set_gpr_idx_on s0, gpr_idx(SRC0)
+; GPRIDX-NEXT:    v_mov_b32_e32 v16, v0
+; GPRIDX-NEXT:    s_set_gpr_idx_off
+; GPRIDX-NEXT:    s_set_gpr_idx_on s1, gpr_idx(SRC0)
+; GPRIDX-NEXT:    v_mov_b32_e32 v17, v0
+; GPRIDX-NEXT:    s_set_gpr_idx_off
+; GPRIDX-NEXT:    global_store_dwordx2 v[0:1], v[16:17], off
+; GPRIDX-NEXT:    s_endpgm
+;
+; MOVREL-LABEL: dyn_extract_v8i64_v_s:
+; MOVREL:       ; %bb.0: ; %entry
+; MOVREL-NEXT:    s_lshl_b32 s0, s2, 1
+; MOVREL-NEXT:    s_mov_b32 m0, s0
+; MOVREL-NEXT:    s_add_u32 s0, s0, 1
+; MOVREL-NEXT:    v_movrels_b32_e32 v16, v0
+; MOVREL-NEXT:    s_mov_b32 m0, s0
+; MOVREL-NEXT:    v_movrels_b32_e32 v17, v0
+; MOVREL-NEXT:    flat_store_dwordx2 v[0:1], v[16:17]
+; MOVREL-NEXT:    s_endpgm
+entry:
+  %ext = extractelement <8 x i64> %vec, i32 %sel
+  store i64 %ext, i64 addrspace(1)* undef
+  ret void
+}
+
+define amdgpu_ps void @dyn_extract_v8i64_s_s(<8 x i64> inreg %vec, i32 inreg %sel) {
+; GPRIDX-LABEL: dyn_extract_v8i64_s_s:
+; GPRIDX:       ; %bb.0: ; %entry
+; GPRIDX-NEXT:    s_mov_b32 s0, s2
+; GPRIDX-NEXT:    s_mov_b32 s1, s3
+; GPRIDX-NEXT:    s_mov_b32 m0, s18
+; GPRIDX-NEXT:    s_mov_b32 s2, s4
+; GPRIDX-NEXT:    s_mov_b32 s3, s5
+; GPRIDX-NEXT:    s_mov_b32 s4, s6
+; GPRIDX-NEXT:    s_mov_b32 s5, s7
+; GPRIDX-NEXT:    s_mov_b32 s6, s8
+; GPRIDX-NEXT:    s_mov_b32 s7, s9
+; GPRIDX-NEXT:    s_mov_b32 s8, s10
+; GPRIDX-NEXT:    s_mov_b32 s9, s11
+; GPRIDX-NEXT:    s_mov_b32 s10, s12
+; GPRIDX-NEXT:    s_mov_b32 s11, s13
+; GPRIDX-NEXT:    s_mov_b32 s12, s14
+; GPRIDX-NEXT:    s_mov_b32 s13, s15
+; GPRIDX-NEXT:    s_mov_b32 s14, s16
+; GPRIDX-NEXT:    s_mov_b32 s15, s17
+; GPRIDX-NEXT:    s_movrels_b64 s[0:1], s[0:1]
+; GPRIDX-NEXT:    v_mov_b32_e32 v0, s0
+; GPRIDX-NEXT:    v_mov_b32_e32 v1, s1
+; GPRIDX-NEXT:    global_store_dwordx2 v[0:1], v[0:1], off
+; GPRIDX-NEXT:    s_endpgm
+;
+; MOVREL-LABEL: dyn_extract_v8i64_s_s:
+; MOVREL:       ; %bb.0: ; %entry
+; MOVREL-NEXT:    s_mov_b32 s0, s2
+; MOVREL-NEXT:    s_mov_b32 s1, s3
+; MOVREL-NEXT:    s_mov_b32 m0, s18
+; MOVREL-NEXT:    s_mov_b32 s2, s4
+; MOVREL-NEXT:    s_mov_b32 s3, s5
+; MOVREL-NEXT:    s_mov_b32 s4, s6
+; MOVREL-NEXT:    s_mov_b32 s5, s7
+; MOVREL-NEXT:    s_mov_b32 s6, s8
+; MOVREL-NEXT:    s_mov_b32 s7, s9
+; MOVREL-NEXT:    s_mov_b32 s8, s10
+; MOVREL-NEXT:    s_mov_b32 s9, s11
+; MOVREL-NEXT:    s_mov_b32 s10, s12
+; MOVREL-NEXT:    s_mov_b32 s11, s13
+; MOVREL-NEXT:    s_mov_b32 s12, s14
+; MOVREL-NEXT:    s_mov_b32 s13, s15
+; MOVREL-NEXT:    s_mov_b32 s14, s16
+; MOVREL-NEXT:    s_mov_b32 s15, s17
+; MOVREL-NEXT:    s_movrels_b64 s[0:1], s[0:1]
+; MOVREL-NEXT:    v_mov_b32_e32 v0, s0
+; MOVREL-NEXT:    v_mov_b32_e32 v1, s1
+; MOVREL-NEXT:    flat_store_dwordx2 v[0:1], v[0:1]
+; MOVREL-NEXT:    s_endpgm
+entry:
+  %ext = extractelement <8 x i64> %vec, i32 %sel
+  store i64 %ext, i64 addrspace(1)* undef
+  ret void
+}
+
+define amdgpu_ps float @dyn_extract_v8f32_s_s_offset3(<8 x float> inreg %vec, i32 inreg %sel) {
+; GPRIDX-LABEL: dyn_extract_v8f32_s_s_offset3:
+; GPRIDX:       ; %bb.0: ; %entry
+; GPRIDX-NEXT:    s_mov_b32 s0, s2
+; GPRIDX-NEXT:    s_add_u32 m0, s10, 3
+; GPRIDX-NEXT:    s_mov_b32 s1, s3
+; GPRIDX-NEXT:    s_mov_b32 s2, s4
+; GPRIDX-NEXT:    s_mov_b32 s3, s5
+; GPRIDX-NEXT:    s_mov_b32 s4, s6
+; GPRIDX-NEXT:    s_mov_b32 s5, s7
+; GPRIDX-NEXT:    s_mov_b32 s6, s8
+; GPRIDX-NEXT:    s_mov_b32 s7, s9
+; GPRIDX-NEXT:    s_movrels_b32 s0, s0
+; GPRIDX-NEXT:    v_mov_b32_e32 v0, s0
+; GPRIDX-NEXT:    ; return to shader part epilog
+;
+; MOVREL-LABEL: dyn_extract_v8f32_s_s_offset3:
+; MOVREL:       ; %bb.0: ; %entry
+; MOVREL-NEXT:    s_mov_b32 s0, s2
+; MOVREL-NEXT:    s_add_u32 m0, s10, 3
+; MOVREL-NEXT:    s_mov_b32 s1, s3
+; MOVREL-NEXT:    s_mov_b32 s2, s4
+; MOVREL-NEXT:    s_mov_b32 s3, s5
+; MOVREL-NEXT:    s_mov_b32 s4, s6
+; MOVREL-NEXT:    s_mov_b32 s5, s7
+; MOVREL-NEXT:    s_mov_b32 s6, s8
+; MOVREL-NEXT:    s_mov_b32 s7, s9
+; MOVREL-NEXT:    s_movrels_b32 s0, s0
+; MOVREL-NEXT:    v_mov_b32_e32 v0, s0
+; MOVREL-NEXT:    ; return to shader part epilog
+entry:
+  %add = add i32 %sel, 3
+  %ext = extractelement <8 x float> %vec, i32 %add
+  ret float %ext
+}
+
+define float @dyn_extract_v8f32_v_v_offset3(<8 x float> %vec, i32 %sel) {
+; GPRIDX-LABEL: dyn_extract_v8f32_v_v_offset3:
+; GPRIDX:       ; %bb.0: ; %entry
+; GPRIDX-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GPRIDX-NEXT:    v_add_u32_e32 v9, 3, v8
+; GPRIDX-NEXT:    s_mov_b64 s[4:5], exec
+; GPRIDX-NEXT:  BB13_1: ; =>This Inner Loop Header: Depth=1
+; GPRIDX-NEXT:    v_readfirstlane_b32 s6, v9
+; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, s6, v9
+; GPRIDX-NEXT:    s_set_gpr_idx_on s6, gpr_idx(SRC0)
+; GPRIDX-NEXT:    v_mov_b32_e32 v8, v0
+; GPRIDX-NEXT:    s_set_gpr_idx_off
+; GPRIDX-NEXT:    s_and_saveexec_b64 vcc, vcc
+; GPRIDX-NEXT:    s_xor_b64 exec, exec, vcc
+; GPRIDX-NEXT:    s_cbranch_execnz BB13_1
+; GPRIDX-NEXT:  ; %bb.2:
+; GPRIDX-NEXT:    s_mov_b64 exec, s[4:5]
+; GPRIDX-NEXT:    v_mov_b32_e32 v0, v8
+; GPRIDX-NEXT:    s_setpc_b64 s[30:31]
+;
+; MOVREL-LABEL: dyn_extract_v8f32_v_v_offset3:
+; MOVREL:       ; %bb.0: ; %entry
+; MOVREL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; MOVREL-NEXT:    v_add_u32_e32 v9, vcc, 3, v8
+; MOVREL-NEXT:    s_mov_b64 s[4:5], exec
+; MOVREL-NEXT:  BB13_1: ; =>This Inner Loop Header: Depth=1
+; MOVREL-NEXT:    v_readfirstlane_b32 s6, v9
+; MOVREL-NEXT:    s_mov_b32 m0, s6
+; MOVREL-NEXT:    v_cmp_eq_u32_e32 vcc, s6, v9
+; MOVREL-NEXT:    v_movrels_b32_e32 v8, v0
+; MOVREL-NEXT:    s_and_saveexec_b64 vcc, vcc
+; MOVREL-NEXT:    s_xor_b64 exec, exec, vcc
+; MOVREL-NEXT:    s_cbranch_execnz BB13_1
+; MOVREL-NEXT:  ; %bb.2:
+; MOVREL-NEXT:    s_mov_b64 exec, s[4:5]
+; MOVREL-NEXT:    v_mov_b32_e32 v0, v8
+; MOVREL-NEXT:    s_setpc_b64 s[30:31]
+entry:
+  %add = add i32 %sel, 3
+  %ext = extractelement <8 x float> %vec, i32 %add
+  ret float %ext
+}
+
+define amdgpu_ps double @dyn_extract_v8f64_s_s_offset1(<8 x double> inreg %vec, i32 inreg %sel) {
+; GPRIDX-LABEL: dyn_extract_v8f64_s_s_offset1:
+; GPRIDX:       ; %bb.0: ; %entry
+; GPRIDX-NEXT:    s_mov_b32 s0, s2
+; GPRIDX-NEXT:    s_mov_b32 s1, s3
+; GPRIDX-NEXT:    s_add_u32 m0, s18, 1
+; GPRIDX-NEXT:    s_mov_b32 s2, s4
+; GPRIDX-NEXT:    s_mov_b32 s3, s5
+; GPRIDX-NEXT:    s_mov_b32 s4, s6
+; GPRIDX-NEXT:    s_mov_b32 s5, s7
+; GPRIDX-NEXT:    s_mov_b32 s6, s8
+; GPRIDX-NEXT:    s_mov_b32 s7, s9
+; GPRIDX-NEXT:    s_mov_b32 s8, s10
+; GPRIDX-NEXT:    s_mov_b32 s9, s11
+; GPRIDX-NEXT:    s_mov_b32 s10, s12
+; GPRIDX-NEXT:    s_mov_b32 s11, s13
+; GPRIDX-NEXT:    s_mov_b32 s12, s14
+; GPRIDX-NEXT:    s_mov_b32 s13, s15
+; GPRIDX-NEXT:    s_mov_b32 s14, s16
+; GPRIDX-NEXT:    s_mov_b32 s15, s17
+; GPRIDX-NEXT:    s_movrels_b64 s[0:1], s[0:1]
+; GPRIDX-NEXT:    ; return to shader part epilog
+;
+; MOVREL-LABEL: dyn_extract_v8f64_s_s_offset1:
+; MOVREL:       ; %bb.0: ; %entry
+; MOVREL-NEXT:    s_mov_b32 s0, s2
+; MOVREL-NEXT:    s_mov_b32 s1, s3
+; MOVREL-NEXT:    s_add_u32 m0, s18, 1
+; MOVREL-NEXT:    s_mov_b32 s2, s4
+; MOVREL-NEXT:    s_mov_b32 s3, s5
+; MOVREL-NEXT:    s_mov_b32 s4, s6
+; MOVREL-NEXT:    s_mov_b32 s5, s7
+; MOVREL-NEXT:    s_mov_b32 s6, s8
+; MOVREL-NEXT:    s_mov_b32 s7, s9
+; MOVREL-NEXT:    s_mov_b32 s8, s10
+; MOVREL-NEXT:    s_mov_b32 s9, s11
+; MOVREL-NEXT:    s_mov_b32 s10, s12
+; MOVREL-NEXT:    s_mov_b32 s11, s13
+; MOVREL-NEXT:    s_mov_b32 s12, s14
+; MOVREL-NEXT:    s_mov_b32 s13, s15
+; MOVREL-NEXT:    s_mov_b32 s14, s16
+; MOVREL-NEXT:    s_mov_b32 s15, s17
+; MOVREL-NEXT:    s_movrels_b64 s[0:1], s[0:1]
+; MOVREL-NEXT:    ; return to shader part epilog
+entry:
+  %add = add i32 %sel, 1
+  %ext = extractelement <8 x double> %vec, i32 %add
+  ret double %ext
+}
+
+define amdgpu_ps double @dyn_extract_v8f64_s_s_offset2(<8 x double> inreg %vec, i32 inreg %sel) {
+; GPRIDX-LABEL: dyn_extract_v8f64_s_s_offset2:
+; GPRIDX:       ; %bb.0: ; %entry
+; GPRIDX-NEXT:    s_mov_b32 s0, s2
+; GPRIDX-NEXT:    s_mov_b32 s1, s3
+; GPRIDX-NEXT:    s_add_u32 m0, s18, 2
+; GPRIDX-NEXT:    s_mov_b32 s2, s4
+; GPRIDX-NEXT:    s_mov_b32 s3, s5
+; GPRIDX-NEXT:    s_mov_b32 s4, s6
+; GPRIDX-NEXT:    s_mov_b32 s5, s7
+; GPRIDX-NEXT:    s_mov_b32 s6, s8
+; GPRIDX-NEXT:    s_mov_b32 s7, s9
+; GPRIDX-NEXT:    s_mov_b32 s8, s10
+; GPRIDX-NEXT:    s_mov_b32 s9, s11
+; GPRIDX-NEXT:    s_mov_b32 s10, s12
+; GPRIDX-NEXT:    s_mov_b32 s11, s13
+; GPRIDX-NEXT:    s_mov_b32 s12, s14
+; GPRIDX-NEXT:    s_mov_b32 s13, s15
+; GPRIDX-NEXT:    s_mov_b32 s14, s16
+; GPRIDX-NEXT:    s_mov_b32 s15, s17
+; GPRIDX-NEXT:    s_movrels_b64 s[0:1], s[0:1]
+; GPRIDX-NEXT:    ; return to shader part epilog
+;
+; MOVREL-LABEL: dyn_extract_v8f64_s_s_offset2:
+; MOVREL:       ; %bb.0: ; %entry
+; MOVREL-NEXT:    s_mov_b32 s0, s2
+; MOVREL-NEXT:    s_mov_b32 s1, s3
+; MOVREL-NEXT:    s_add_u32 m0, s18, 2
+; MOVREL-NEXT:    s_mov_b32 s2, s4
+; MOVREL-NEXT:    s_mov_b32 s3, s5
+; MOVREL-NEXT:    s_mov_b32 s4, s6
+; MOVREL-NEXT:    s_mov_b32 s5, s7
+; MOVREL-NEXT:    s_mov_b32 s6, s8
+; MOVREL-NEXT:    s_mov_b32 s7, s9
+; MOVREL-NEXT:    s_mov_b32 s8, s10
+; MOVREL-NEXT:    s_mov_b32 s9, s11
+; MOVREL-NEXT:    s_mov_b32 s10, s12
+; MOVREL-NEXT:    s_mov_b32 s11, s13
+; MOVREL-NEXT:    s_mov_b32 s12, s14
+; MOVREL-NEXT:    s_mov_b32 s13, s15
+; MOVREL-NEXT:    s_mov_b32 s14, s16
+; MOVREL-NEXT:    s_mov_b32 s15, s17
+; MOVREL-NEXT:    s_movrels_b64 s[0:1], s[0:1]
+; MOVREL-NEXT:    ; return to shader part epilog
+entry:
+  %add = add i32 %sel, 2
+  %ext = extractelement <8 x double> %vec, i32 %add
+  ret double %ext
+}
+
+define amdgpu_ps double @dyn_extract_v8f64_s_s_offset3(<8 x double> inreg %vec, i32 inreg %sel) {
+; GPRIDX-LABEL: dyn_extract_v8f64_s_s_offset3:
+; GPRIDX:       ; %bb.0: ; %entry
+; GPRIDX-NEXT:    s_mov_b32 s0, s2
+; GPRIDX-NEXT:    s_mov_b32 s1, s3
+; GPRIDX-NEXT:    s_add_u32 m0, s18, 3
+; GPRIDX-NEXT:    s_mov_b32 s2, s4
+; GPRIDX-NEXT:    s_mov_b32 s3, s5
+; GPRIDX-NEXT:    s_mov_b32 s4, s6
+; GPRIDX-NEXT:    s_mov_b32 s5, s7
+; GPRIDX-NEXT:    s_mov_b32 s6, s8
+; GPRIDX-NEXT:    s_mov_b32 s7, s9
+; GPRIDX-NEXT:    s_mov_b32 s8, s10
+; GPRIDX-NEXT:    s_mov_b32 s9, s11
+; GPRIDX-NEXT:    s_mov_b32 s10, s12
+; GPRIDX-NEXT:    s_mov_b32 s11, s13
+; GPRIDX-NEXT:    s_mov_b32 s12, s14
+; GPRIDX-NEXT:    s_mov_b32 s13, s15
+; GPRIDX-NEXT:    s_mov_b32 s14, s16
+; GPRIDX-NEXT:    s_mov_b32 s15, s17
+; GPRIDX-NEXT:    s_movrels_b64 s[0:1], s[0:1]
+; GPRIDX-NEXT:    ; return to shader part epilog
+;
+; MOVREL-LABEL: dyn_extract_v8f64_s_s_offset3:
+; MOVREL:       ; %bb.0: ; %entry
+; MOVREL-NEXT:    s_mov_b32 s0, s2
+; MOVREL-NEXT:    s_mov_b32 s1, s3
+; MOVREL-NEXT:    s_add_u32 m0, s18, 3
+; MOVREL-NEXT:    s_mov_b32 s2, s4
+; MOVREL-NEXT:    s_mov_b32 s3, s5
+; MOVREL-NEXT:    s_mov_b32 s4, s6
+; MOVREL-NEXT:    s_mov_b32 s5, s7
+; MOVREL-NEXT:    s_mov_b32 s6, s8
+; MOVREL-NEXT:    s_mov_b32 s7, s9
+; MOVREL-NEXT:    s_mov_b32 s8, s10
+; MOVREL-NEXT:    s_mov_b32 s9, s11
+; MOVREL-NEXT:    s_mov_b32 s10, s12
+; MOVREL-NEXT:    s_mov_b32 s11, s13
+; MOVREL-NEXT:    s_mov_b32 s12, s14
+; MOVREL-NEXT:    s_mov_b32 s13, s15
+; MOVREL-NEXT:    s_mov_b32 s14, s16
+; MOVREL-NEXT:    s_mov_b32 s15, s17
+; MOVREL-NEXT:    s_movrels_b64 s[0:1], s[0:1]
+; MOVREL-NEXT:    ; return to shader part epilog
+entry:
+  %add = add i32 %sel, 3
+  %ext = extractelement <8 x double> %vec, i32 %add
+  ret double %ext
+}
+
+define amdgpu_ps double @dyn_extract_v8f64_s_s_offset4(<8 x double> inreg %vec, i32 inreg %sel) {
+; GPRIDX-LABEL: dyn_extract_v8f64_s_s_offset4:
+; GPRIDX:       ; %bb.0: ; %entry
+; GPRIDX-NEXT:    s_mov_b32 s0, s2
+; GPRIDX-NEXT:    s_mov_b32 s1, s3
+; GPRIDX-NEXT:    s_add_u32 m0, s18, 4
+; GPRIDX-NEXT:    s_mov_b32 s2, s4
+; GPRIDX-NEXT:    s_mov_b32 s3, s5
+; GPRIDX-NEXT:    s_mov_b32 s4, s6
+; GPRIDX-NEXT:    s_mov_b32 s5, s7
+; GPRIDX-NEXT:    s_mov_b32 s6, s8
+; GPRIDX-NEXT:    s_mov_b32 s7, s9
+; GPRIDX-NEXT:    s_mov_b32 s8, s10
+; GPRIDX-NEXT:    s_mov_b32 s9, s11
+; GPRIDX-NEXT:    s_mov_b32 s10, s12
+; GPRIDX-NEXT:    s_mov_b32 s11, s13
+; GPRIDX-NEXT:    s_mov_b32 s12, s14
+; GPRIDX-NEXT:    s_mov_b32 s13, s15
+; GPRIDX-NEXT:    s_mov_b32 s14, s16
+; GPRIDX-NEXT:    s_mov_b32 s15, s17
+; GPRIDX-NEXT:    s_movrels_b64 s[0:1], s[0:1]
+; GPRIDX-NEXT:    ; return to shader part epilog
+;
+; MOVREL-LABEL: dyn_extract_v8f64_s_s_offset4:
+; MOVREL:       ; %bb.0: ; %entry
+; MOVREL-NEXT:    s_mov_b32 s0, s2
+; MOVREL-NEXT:    s_mov_b32 s1, s3
+; MOVREL-NEXT:    s_add_u32 m0, s18, 4
+; MOVREL-NEXT:    s_mov_b32 s2, s4
+; MOVREL-NEXT:    s_mov_b32 s3, s5
+; MOVREL-NEXT:    s_mov_b32 s4, s6
+; MOVREL-NEXT:    s_mov_b32 s5, s7
+; MOVREL-NEXT:    s_mov_b32 s6, s8
+; MOVREL-NEXT:    s_mov_b32 s7, s9
+; MOVREL-NEXT:    s_mov_b32 s8, s10
+; MOVREL-NEXT:    s_mov_b32 s9, s11
+; MOVREL-NEXT:    s_mov_b32 s10, s12
+; MOVREL-NEXT:    s_mov_b32 s11, s13
+; MOVREL-NEXT:    s_mov_b32 s12, s14
+; MOVREL-NEXT:    s_mov_b32 s13, s15
+; MOVREL-NEXT:    s_mov_b32 s14, s16
+; MOVREL-NEXT:    s_mov_b32 s15, s17
+; MOVREL-NEXT:    s_movrels_b64 s[0:1], s[0:1]
+; MOVREL-NEXT:    ; return to shader part epilog
+entry:
+  %add = add i32 %sel, 4
+  %ext = extractelement <8 x double> %vec, i32 %add
+  ret double %ext
+}
+
+define amdgpu_ps double @dyn_extract_v8f64_s_s_offset5(<8 x double> inreg %vec, i32 inreg %sel) {
+; GPRIDX-LABEL: dyn_extract_v8f64_s_s_offset5:
+; GPRIDX:       ; %bb.0: ; %entry
+; GPRIDX-NEXT:    s_mov_b32 s0, s2
+; GPRIDX-NEXT:    s_mov_b32 s1, s3
+; GPRIDX-NEXT:    s_add_u32 m0, s18, 5
+; GPRIDX-NEXT:    s_mov_b32 s2, s4
+; GPRIDX-NEXT:    s_mov_b32 s3, s5
+; GPRIDX-NEXT:    s_mov_b32 s4, s6
+; GPRIDX-NEXT:    s_mov_b32 s5, s7
+; GPRIDX-NEXT:    s_mov_b32 s6, s8
+; GPRIDX-NEXT:    s_mov_b32 s7, s9
+; GPRIDX-NEXT:    s_mov_b32 s8, s10
+; GPRIDX-NEXT:    s_mov_b32 s9, s11
+; GPRIDX-NEXT:    s_mov_b32 s10, s12
+; GPRIDX-NEXT:    s_mov_b32 s11, s13
+; GPRIDX-NEXT:    s_mov_b32 s12, s14
+; GPRIDX-NEXT:    s_mov_b32 s13, s15
+; GPRIDX-NEXT:    s_mov_b32 s14, s16
+; GPRIDX-NEXT:    s_mov_b32 s15, s17
+; GPRIDX-NEXT:    s_movrels_b64 s[0:1], s[0:1]
+; GPRIDX-NEXT:    ; return to shader part epilog
+;
+; MOVREL-LABEL: dyn_extract_v8f64_s_s_offset5:
+; MOVREL:       ; %bb.0: ; %entry
+; MOVREL-NEXT:    s_mov_b32 s0, s2
+; MOVREL-NEXT:    s_mov_b32 s1, s3
+; MOVREL-NEXT:    s_add_u32 m0, s18, 5
+; MOVREL-NEXT:    s_mov_b32 s2, s4
+; MOVREL-NEXT:    s_mov_b32 s3, s5
+; MOVREL-NEXT:    s_mov_b32 s4, s6
+; MOVREL-NEXT:    s_mov_b32 s5, s7
+; MOVREL-NEXT:    s_mov_b32 s6, s8
+; MOVREL-NEXT:    s_mov_b32 s7, s9
+; MOVREL-NEXT:    s_mov_b32 s8, s10
+; MOVREL-NEXT:    s_mov_b32 s9, s11
+; MOVREL-NEXT:    s_mov_b32 s10, s12
+; MOVREL-NEXT:    s_mov_b32 s11, s13
+; MOVREL-NEXT:    s_mov_b32 s12, s14
+; MOVREL-NEXT:    s_mov_b32 s13, s15
+; MOVREL-NEXT:    s_mov_b32 s14, s16
+; MOVREL-NEXT:    s_mov_b32 s15, s17
+; MOVREL-NEXT:    s_movrels_b64 s[0:1], s[0:1]
+; MOVREL-NEXT:    ; return to shader part epilog
+entry:
+  %add = add i32 %sel, 5
+  %ext = extractelement <8 x double> %vec, i32 %add
+  ret double %ext
+}
+
+define amdgpu_ps double @dyn_extract_v8f64_s_s_offset6(<8 x double> inreg %vec, i32 inreg %sel) {
+; GPRIDX-LABEL: dyn_extract_v8f64_s_s_offset6:
+; GPRIDX:       ; %bb.0: ; %entry
+; GPRIDX-NEXT:    s_mov_b32 s0, s2
+; GPRIDX-NEXT:    s_mov_b32 s1, s3
+; GPRIDX-NEXT:    s_add_u32 m0, s18, 6
+; GPRIDX-NEXT:    s_mov_b32 s2, s4
+; GPRIDX-NEXT:    s_mov_b32 s3, s5
+; GPRIDX-NEXT:    s_mov_b32 s4, s6
+; GPRIDX-NEXT:    s_mov_b32 s5, s7
+; GPRIDX-NEXT:    s_mov_b32 s6, s8
+; GPRIDX-NEXT:    s_mov_b32 s7, s9
+; GPRIDX-NEXT:    s_mov_b32 s8, s10
+; GPRIDX-NEXT:    s_mov_b32 s9, s11
+; GPRIDX-NEXT:    s_mov_b32 s10, s12
+; GPRIDX-NEXT:    s_mov_b32 s11, s13
+; GPRIDX-NEXT:    s_mov_b32 s12, s14
+; GPRIDX-NEXT:    s_mov_b32 s13, s15
+; GPRIDX-NEXT:    s_mov_b32 s14, s16
+; GPRIDX-NEXT:    s_mov_b32 s15, s17
+; GPRIDX-NEXT:    s_movrels_b64 s[0:1], s[0:1]
+; GPRIDX-NEXT:    ; return to shader part epilog
+;
+; MOVREL-LABEL: dyn_extract_v8f64_s_s_offset6:
+; MOVREL:       ; %bb.0: ; %entry
+; MOVREL-NEXT:    s_mov_b32 s0, s2
+; MOVREL-NEXT:    s_mov_b32 s1, s3
+; MOVREL-NEXT:    s_add_u32 m0, s18, 6
+; MOVREL-NEXT:    s_mov_b32 s2, s4
+; MOVREL-NEXT:    s_mov_b32 s3, s5
+; MOVREL-NEXT:    s_mov_b32 s4, s6
+; MOVREL-NEXT:    s_mov_b32 s5, s7
+; MOVREL-NEXT:    s_mov_b32 s6, s8
+; MOVREL-NEXT:    s_mov_b32 s7, s9
+; MOVREL-NEXT:    s_mov_b32 s8, s10
+; MOVREL-NEXT:    s_mov_b32 s9, s11
+; MOVREL-NEXT:    s_mov_b32 s10, s12
+; MOVREL-NEXT:    s_mov_b32 s11, s13
+; MOVREL-NEXT:    s_mov_b32 s12, s14
+; MOVREL-NEXT:    s_mov_b32 s13, s15
+; MOVREL-NEXT:    s_mov_b32 s14, s16
+; MOVREL-NEXT:    s_mov_b32 s15, s17
+; MOVREL-NEXT:    s_movrels_b64 s[0:1], s[0:1]
+; MOVREL-NEXT:    ; return to shader part epilog
+entry:
+  %add = add i32 %sel, 6
+  %ext = extractelement <8 x double> %vec, i32 %add
+  ret double %ext
+}
+
+define amdgpu_ps double @dyn_extract_v8f64_s_s_offset7(<8 x double> inreg %vec, i32 inreg %sel) {
+; GPRIDX-LABEL: dyn_extract_v8f64_s_s_offset7:
+; GPRIDX:       ; %bb.0: ; %entry
+; GPRIDX-NEXT:    s_mov_b32 s0, s2
+; GPRIDX-NEXT:    s_mov_b32 s1, s3
+; GPRIDX-NEXT:    s_add_u32 m0, s18, 7
+; GPRIDX-NEXT:    s_mov_b32 s2, s4
+; GPRIDX-NEXT:    s_mov_b32 s3, s5
+; GPRIDX-NEXT:    s_mov_b32 s4, s6
+; GPRIDX-NEXT:    s_mov_b32 s5, s7
+; GPRIDX-NEXT:    s_mov_b32 s6, s8
+; GPRIDX-NEXT:    s_mov_b32 s7, s9
+; GPRIDX-NEXT:    s_mov_b32 s8, s10
+; GPRIDX-NEXT:    s_mov_b32 s9, s11
+; GPRIDX-NEXT:    s_mov_b32 s10, s12
+; GPRIDX-NEXT:    s_mov_b32 s11, s13
+; GPRIDX-NEXT:    s_mov_b32 s12, s14
+; GPRIDX-NEXT:    s_mov_b32 s13, s15
+; GPRIDX-NEXT:    s_mov_b32 s14, s16
+; GPRIDX-NEXT:    s_mov_b32 s15, s17
+; GPRIDX-NEXT:    s_movrels_b64 s[0:1], s[0:1]
+; GPRIDX-NEXT:    ; return to shader part epilog
+;
+; MOVREL-LABEL: dyn_extract_v8f64_s_s_offset7:
+; MOVREL:       ; %bb.0: ; %entry
+; MOVREL-NEXT:    s_mov_b32 s0, s2
+; MOVREL-NEXT:    s_mov_b32 s1, s3
+; MOVREL-NEXT:    s_add_u32 m0, s18, 7
+; MOVREL-NEXT:    s_mov_b32 s2, s4
+; MOVREL-NEXT:    s_mov_b32 s3, s5
+; MOVREL-NEXT:    s_mov_b32 s4, s6
+; MOVREL-NEXT:    s_mov_b32 s5, s7
+; MOVREL-NEXT:    s_mov_b32 s6, s8
+; MOVREL-NEXT:    s_mov_b32 s7, s9
+; MOVREL-NEXT:    s_mov_b32 s8, s10
+; MOVREL-NEXT:    s_mov_b32 s9, s11
+; MOVREL-NEXT:    s_mov_b32 s10, s12
+; MOVREL-NEXT:    s_mov_b32 s11, s13
+; MOVREL-NEXT:    s_mov_b32 s12, s14
+; MOVREL-NEXT:    s_mov_b32 s13, s15
+; MOVREL-NEXT:    s_mov_b32 s14, s16
+; MOVREL-NEXT:    s_mov_b32 s15, s17
+; MOVREL-NEXT:    s_movrels_b64 s[0:1], s[0:1]
+; MOVREL-NEXT:    ; return to shader part epilog
+entry:
+  %add = add i32 %sel, 7
+  %ext = extractelement <8 x double> %vec, i32 %add
+  ret double %ext
+}
+
+define amdgpu_ps double @dyn_extract_v8f64_s_s_offsetm1(<8 x double> inreg %vec, i32 inreg %sel) {
+; GPRIDX-LABEL: dyn_extract_v8f64_s_s_offsetm1:
+; GPRIDX:       ; %bb.0: ; %entry
+; GPRIDX-NEXT:    s_mov_b32 s0, s2
+; GPRIDX-NEXT:    s_mov_b32 s1, s3
+; GPRIDX-NEXT:    s_add_u32 m0, s18, -1
+; GPRIDX-NEXT:    s_mov_b32 s2, s4
+; GPRIDX-NEXT:    s_mov_b32 s3, s5
+; GPRIDX-NEXT:    s_mov_b32 s4, s6
+; GPRIDX-NEXT:    s_mov_b32 s5, s7
+; GPRIDX-NEXT:    s_mov_b32 s6, s8
+; GPRIDX-NEXT:    s_mov_b32 s7, s9
+; GPRIDX-NEXT:    s_mov_b32 s8, s10
+; GPRIDX-NEXT:    s_mov_b32 s9, s11
+; GPRIDX-NEXT:    s_mov_b32 s10, s12
+; GPRIDX-NEXT:    s_mov_b32 s11, s13
+; GPRIDX-NEXT:    s_mov_b32 s12, s14
+; GPRIDX-NEXT:    s_mov_b32 s13, s15
+; GPRIDX-NEXT:    s_mov_b32 s14, s16
+; GPRIDX-NEXT:    s_mov_b32 s15, s17
+; GPRIDX-NEXT:    s_movrels_b64 s[0:1], s[0:1]
+; GPRIDX-NEXT:    ; return to shader part epilog
+;
+; MOVREL-LABEL: dyn_extract_v8f64_s_s_offsetm1:
+; MOVREL:       ; %bb.0: ; %entry
+; MOVREL-NEXT:    s_mov_b32 s0, s2
+; MOVREL-NEXT:    s_mov_b32 s1, s3
+; MOVREL-NEXT:    s_add_u32 m0, s18, -1
+; MOVREL-NEXT:    s_mov_b32 s2, s4
+; MOVREL-NEXT:    s_mov_b32 s3, s5
+; MOVREL-NEXT:    s_mov_b32 s4, s6
+; MOVREL-NEXT:    s_mov_b32 s5, s7
+; MOVREL-NEXT:    s_mov_b32 s6, s8
+; MOVREL-NEXT:    s_mov_b32 s7, s9
+; MOVREL-NEXT:    s_mov_b32 s8, s10
+; MOVREL-NEXT:    s_mov_b32 s9, s11
+; MOVREL-NEXT:    s_mov_b32 s10, s12
+; MOVREL-NEXT:    s_mov_b32 s11, s13
+; MOVREL-NEXT:    s_mov_b32 s12, s14
+; MOVREL-NEXT:    s_mov_b32 s13, s15
+; MOVREL-NEXT:    s_mov_b32 s14, s16
+; MOVREL-NEXT:    s_mov_b32 s15, s17
+; MOVREL-NEXT:    s_movrels_b64 s[0:1], s[0:1]
+; MOVREL-NEXT:    ; return to shader part epilog
+entry:
+  %add = add i32 %sel, -1
+  %ext = extractelement <8 x double> %vec, i32 %add
+  ret double %ext
+}
+
+define double @dyn_extract_v8f64_v_v_offset3(<8 x double> %vec, i32 %sel) {
+; GPRIDX-LABEL: dyn_extract_v8f64_v_v_offset3:
+; GPRIDX:       ; %bb.0: ; %entry
+; GPRIDX-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GPRIDX-NEXT:    v_add_u32_e32 v18, 3, v16
+; GPRIDX-NEXT:    s_mov_b64 s[4:5], exec
+; GPRIDX-NEXT:  BB22_1: ; =>This Inner Loop Header: Depth=1
+; GPRIDX-NEXT:    v_readfirstlane_b32 s6, v18
+; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, s6, v18
+; GPRIDX-NEXT:    s_lshl_b32 s6, s6, 1
+; GPRIDX-NEXT:    s_add_u32 s7, s6, 1
+; GPRIDX-NEXT:    s_set_gpr_idx_on s6, gpr_idx(SRC0)
+; GPRIDX-NEXT:    v_mov_b32_e32 v16, v0
+; GPRIDX-NEXT:    s_set_gpr_idx_off
+; GPRIDX-NEXT:    s_set_gpr_idx_on s7, gpr_idx(SRC0)
+; GPRIDX-NEXT:    v_mov_b32_e32 v17, v0
+; GPRIDX-NEXT:    s_set_gpr_idx_off
+; GPRIDX-NEXT:    s_and_saveexec_b64 vcc, vcc
+; GPRIDX-NEXT:    s_xor_b64 exec, exec, vcc
+; GPRIDX-NEXT:    s_cbranch_execnz BB22_1
+; GPRIDX-NEXT:  ; %bb.2:
+; GPRIDX-NEXT:    s_mov_b64 exec, s[4:5]
+; GPRIDX-NEXT:    v_mov_b32_e32 v0, v16
+; GPRIDX-NEXT:    v_mov_b32_e32 v1, v17
+; GPRIDX-NEXT:    s_setpc_b64 s[30:31]
+;
+; MOVREL-LABEL: dyn_extract_v8f64_v_v_offset3:
+; MOVREL:       ; %bb.0: ; %entry
+; MOVREL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; MOVREL-NEXT:    v_add_u32_e32 v18, vcc, 3, v16
+; MOVREL-NEXT:    s_mov_b64 s[4:5], exec
+; MOVREL-NEXT:  BB22_1: ; =>This Inner Loop Header: Depth=1
+; MOVREL-NEXT:    v_readfirstlane_b32 s6, v18
+; MOVREL-NEXT:    v_cmp_eq_u32_e32 vcc, s6, v18
+; MOVREL-NEXT:    s_lshl_b32 s6, s6, 1
+; MOVREL-NEXT:    s_mov_b32 m0, s6
+; MOVREL-NEXT:    s_add_u32 s7, s6, 1
+; MOVREL-NEXT:    v_movrels_b32_e32 v16, v0
+; MOVREL-NEXT:    s_mov_b32 m0, s7
+; MOVREL-NEXT:    v_movrels_b32_e32 v17, v0
+; MOVREL-NEXT:    s_and_saveexec_b64 vcc, vcc
+; MOVREL-NEXT:    s_xor_b64 exec, exec, vcc
+; MOVREL-NEXT:    s_cbranch_execnz BB22_1
+; MOVREL-NEXT:  ; %bb.2:
+; MOVREL-NEXT:    s_mov_b64 exec, s[4:5]
+; MOVREL-NEXT:    v_mov_b32_e32 v0, v16
+; MOVREL-NEXT:    v_mov_b32_e32 v1, v17
+; MOVREL-NEXT:    s_setpc_b64 s[30:31]
+entry:
+  %add = add i32 %sel, 3
+  %ext = extractelement <8 x double> %vec, i32 %add
+  ret double %ext
+}
+
+define i8 addrspace(3)* @dyn_extract_v8p3_v_v(<8 x i8 addrspace(3)*> %vec, i32 %idx) {
+; GPRIDX-LABEL: dyn_extract_v8p3_v_v:
+; GPRIDX:       ; %bb.0: ; %entry
+; GPRIDX-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GPRIDX-NEXT:    s_mov_b64 s[4:5], exec
+; GPRIDX-NEXT:  BB23_1: ; =>This Inner Loop Header: Depth=1
+; GPRIDX-NEXT:    v_readfirstlane_b32 s6, v8
+; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, s6, v8
+; GPRIDX-NEXT:    s_set_gpr_idx_on s6, gpr_idx(SRC0)
+; GPRIDX-NEXT:    v_mov_b32_e32 v9, v0
+; GPRIDX-NEXT:    s_set_gpr_idx_off
+; GPRIDX-NEXT:    s_and_saveexec_b64 vcc, vcc
+; GPRIDX-NEXT:    s_xor_b64 exec, exec, vcc
+; GPRIDX-NEXT:    s_cbranch_execnz BB23_1
+; GPRIDX-NEXT:  ; %bb.2:
+; GPRIDX-NEXT:    s_mov_b64 exec, s[4:5]
+; GPRIDX-NEXT:    v_mov_b32_e32 v0, v9
+; GPRIDX-NEXT:    s_setpc_b64 s[30:31]
+;
+; MOVREL-LABEL: dyn_extract_v8p3_v_v:
+; MOVREL:       ; %bb.0: ; %entry
+; MOVREL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; MOVREL-NEXT:    s_mov_b64 s[4:5], exec
+; MOVREL-NEXT:  BB23_1: ; =>This Inner Loop Header: Depth=1
+; MOVREL-NEXT:    v_readfirstlane_b32 s6, v8
+; MOVREL-NEXT:    s_mov_b32 m0, s6
+; MOVREL-NEXT:    v_cmp_eq_u32_e32 vcc, s6, v8
+; MOVREL-NEXT:    v_movrels_b32_e32 v9, v0
+; MOVREL-NEXT:    s_and_saveexec_b64 vcc, vcc
+; MOVREL-NEXT:    s_xor_b64 exec, exec, vcc
+; MOVREL-NEXT:    s_cbranch_execnz BB23_1
+; MOVREL-NEXT:  ; %bb.2:
+; MOVREL-NEXT:    s_mov_b64 exec, s[4:5]
+; MOVREL-NEXT:    v_mov_b32_e32 v0, v9
+; MOVREL-NEXT:    s_setpc_b64 s[30:31]
+entry:
+  %ext = extractelement <8 x i8 addrspace(3)*> %vec, i32 %idx
+  ret i8 addrspace(3)* %ext
+}
+
+define amdgpu_ps void @dyn_extract_v8p3_s_s(<8 x i8 addrspace(3)*> inreg %vec, i32 inreg %idx) {
+; GPRIDX-LABEL: dyn_extract_v8p3_s_s:
+; GPRIDX:       ; %bb.0: ; %entry
+; GPRIDX-NEXT:    s_mov_b32 s0, s2
+; GPRIDX-NEXT:    s_mov_b32 m0, s10
+; GPRIDX-NEXT:    s_mov_b32 s1, s3
+; GPRIDX-NEXT:    s_mov_b32 s2, s4
+; GPRIDX-NEXT:    s_mov_b32 s3, s5
+; GPRIDX-NEXT:    s_mov_b32 s4, s6
+; GPRIDX-NEXT:    s_mov_b32 s5, s7
+; GPRIDX-NEXT:    s_mov_b32 s6, s8
+; GPRIDX-NEXT:    s_mov_b32 s7, s9
+; GPRIDX-NEXT:    s_movrels_b32 s0, s0
+; GPRIDX-NEXT:    v_mov_b32_e32 v0, s0
+; GPRIDX-NEXT:    ds_write_b32 v0, v0
+; GPRIDX-NEXT:    s_endpgm
+;
+; MOVREL-LABEL: dyn_extract_v8p3_s_s:
+; MOVREL:       ; %bb.0: ; %entry
+; MOVREL-NEXT:    s_mov_b32 s0, s2
+; MOVREL-NEXT:    s_mov_b32 m0, s10
+; MOVREL-NEXT:    s_mov_b32 s1, s3
+; MOVREL-NEXT:    s_mov_b32 s2, s4
+; MOVREL-NEXT:    s_mov_b32 s3, s5
+; MOVREL-NEXT:    s_mov_b32 s4, s6
+; MOVREL-NEXT:    s_mov_b32 s5, s7
+; MOVREL-NEXT:    s_mov_b32 s6, s8
+; MOVREL-NEXT:    s_mov_b32 s7, s9
+; MOVREL-NEXT:    s_movrels_b32 s0, s0
+; MOVREL-NEXT:    v_mov_b32_e32 v0, s0
+; MOVREL-NEXT:    s_mov_b32 m0, -1
+; MOVREL-NEXT:    ds_write_b32 v0, v0
+; MOVREL-NEXT:    s_endpgm
+entry:
+  %ext = extractelement <8 x i8 addrspace(3)*> %vec, i32 %idx
+  store i8 addrspace(3)* %ext, i8 addrspace(3)* addrspace(3)* undef
+  ret void
+}
+
+define i8 addrspace(1)* @dyn_extract_v8p1_v_v(<8 x i8 addrspace(1)*> %vec, i32 %idx) {
+; GPRIDX-LABEL: dyn_extract_v8p1_v_v:
+; GPRIDX:       ; %bb.0: ; %entry
+; GPRIDX-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GPRIDX-NEXT:    s_mov_b64 s[4:5], exec
+; GPRIDX-NEXT:  BB25_1: ; =>This Inner Loop Header: Depth=1
+; GPRIDX-NEXT:    v_readfirstlane_b32 s6, v16
+; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, s6, v16
+; GPRIDX-NEXT:    s_lshl_b32 s6, s6, 1
+; GPRIDX-NEXT:    s_add_u32 s7, s6, 1
+; GPRIDX-NEXT:    s_set_gpr_idx_on s6, gpr_idx(SRC0)
+; GPRIDX-NEXT:    v_mov_b32_e32 v17, v0
+; GPRIDX-NEXT:    s_set_gpr_idx_off
+; GPRIDX-NEXT:    s_set_gpr_idx_on s7, gpr_idx(SRC0)
+; GPRIDX-NEXT:    v_mov_b32_e32 v18, v0
+; GPRIDX-NEXT:    s_set_gpr_idx_off
+; GPRIDX-NEXT:    s_and_saveexec_b64 vcc, vcc
+; GPRIDX-NEXT:    s_xor_b64 exec, exec, vcc
+; GPRIDX-NEXT:    s_cbranch_execnz BB25_1
+; GPRIDX-NEXT:  ; %bb.2:
+; GPRIDX-NEXT:    s_mov_b64 exec, s[4:5]
+; GPRIDX-NEXT:    v_mov_b32_e32 v0, v17
+; GPRIDX-NEXT:    v_mov_b32_e32 v1, v18
+; GPRIDX-NEXT:    s_setpc_b64 s[30:31]
+;
+; MOVREL-LABEL: dyn_extract_v8p1_v_v:
+; MOVREL:       ; %bb.0: ; %entry
+; MOVREL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; MOVREL-NEXT:    s_mov_b64 s[4:5], exec
+; MOVREL-NEXT:  BB25_1: ; =>This Inner Loop Header: Depth=1
+; MOVREL-NEXT:    v_readfirstlane_b32 s6, v16
+; MOVREL-NEXT:    v_cmp_eq_u32_e32 vcc, s6, v16
+; MOVREL-NEXT:    s_lshl_b32 s6, s6, 1
+; MOVREL-NEXT:    s_mov_b32 m0, s6
+; MOVREL-NEXT:    s_add_u32 s7, s6, 1
+; MOVREL-NEXT:    v_movrels_b32_e32 v17, v0
+; MOVREL-NEXT:    s_mov_b32 m0, s7
+; MOVREL-NEXT:    v_movrels_b32_e32 v18, v0
+; MOVREL-NEXT:    s_and_saveexec_b64 vcc, vcc
+; MOVREL-NEXT:    s_xor_b64 exec, exec, vcc
+; MOVREL-NEXT:    s_cbranch_execnz BB25_1
+; MOVREL-NEXT:  ; %bb.2:
+; MOVREL-NEXT:    s_mov_b64 exec, s[4:5]
+; MOVREL-NEXT:    v_mov_b32_e32 v0, v17
+; MOVREL-NEXT:    v_mov_b32_e32 v1, v18
+; MOVREL-NEXT:    s_setpc_b64 s[30:31]
+entry:
+  %ext = extractelement <8 x i8 addrspace(1)*> %vec, i32 %idx
+  ret i8 addrspace(1)* %ext
+}
+
+define amdgpu_ps void @dyn_extract_v8p1_s_s(<8 x i8 addrspace(1)*> inreg %vec, i32 inreg %idx) {
+; GPRIDX-LABEL: dyn_extract_v8p1_s_s:
+; GPRIDX:       ; %bb.0: ; %entry
+; GPRIDX-NEXT:    s_mov_b32 s0, s2
+; GPRIDX-NEXT:    s_mov_b32 s1, s3
+; GPRIDX-NEXT:    s_mov_b32 m0, s18
+; GPRIDX-NEXT:    s_mov_b32 s2, s4
+; GPRIDX-NEXT:    s_mov_b32 s3, s5
+; GPRIDX-NEXT:    s_mov_b32 s4, s6
+; GPRIDX-NEXT:    s_mov_b32 s5, s7
+; GPRIDX-NEXT:    s_mov_b32 s6, s8
+; GPRIDX-NEXT:    s_mov_b32 s7, s9
+; GPRIDX-NEXT:    s_mov_b32 s8, s10
+; GPRIDX-NEXT:    s_mov_b32 s9, s11
+; GPRIDX-NEXT:    s_mov_b32 s10, s12
+; GPRIDX-NEXT:    s_mov_b32 s11, s13
+; GPRIDX-NEXT:    s_mov_b32 s12, s14
+; GPRIDX-NEXT:    s_mov_b32 s13, s15
+; GPRIDX-NEXT:    s_mov_b32 s14, s16
+; GPRIDX-NEXT:    s_mov_b32 s15, s17
+; GPRIDX-NEXT:    s_movrels_b64 s[0:1], s[0:1]
+; GPRIDX-NEXT:    v_mov_b32_e32 v0, s0
+; GPRIDX-NEXT:    v_mov_b32_e32 v1, s1
+; GPRIDX-NEXT:    global_store_dwordx2 v[0:1], v[0:1], off
+; GPRIDX-NEXT:    s_endpgm
+;
+; MOVREL-LABEL: dyn_extract_v8p1_s_s:
+; MOVREL:       ; %bb.0: ; %entry
+; MOVREL-NEXT:    s_mov_b32 s0, s2
+; MOVREL-NEXT:    s_mov_b32 s1, s3
+; MOVREL-NEXT:    s_mov_b32 m0, s18
+; MOVREL-NEXT:    s_mov_b32 s2, s4
+; MOVREL-NEXT:    s_mov_b32 s3, s5
+; MOVREL-NEXT:    s_mov_b32 s4, s6
+; MOVREL-NEXT:    s_mov_b32 s5, s7
+; MOVREL-NEXT:    s_mov_b32 s6, s8
+; MOVREL-NEXT:    s_mov_b32 s7, s9
+; MOVREL-NEXT:    s_mov_b32 s8, s10
+; MOVREL-NEXT:    s_mov_b32 s9, s11
+; MOVREL-NEXT:    s_mov_b32 s10, s12
+; MOVREL-NEXT:    s_mov_b32 s11, s13
+; MOVREL-NEXT:    s_mov_b32 s12, s14
+; MOVREL-NEXT:    s_mov_b32 s13, s15
+; MOVREL-NEXT:    s_mov_b32 s14, s16
+; MOVREL-NEXT:    s_mov_b32 s15, s17
+; MOVREL-NEXT:    s_movrels_b64 s[0:1], s[0:1]
+; MOVREL-NEXT:    v_mov_b32_e32 v0, s0
+; MOVREL-NEXT:    v_mov_b32_e32 v1, s1
+; MOVREL-NEXT:    flat_store_dwordx2 v[0:1], v[0:1]
+; MOVREL-NEXT:    s_endpgm
+entry:
+  %ext = extractelement <8 x i8 addrspace(1)*> %vec, i32 %idx
+  store i8 addrspace(1)* %ext, i8 addrspace(1)* addrspace(1)* undef
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/function-returns.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/function-returns.ll
new file mode 100644
index 000000000..7d217afe9
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/function-returns.ll
@@ -0,0 +1,1063 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; RUN: llc -global-isel -global-isel-abort=0 -stop-after=irtranslator -mtriple=amdgcn-mesa-mesa3d -verify-machineinstrs -o - %s | FileCheck -enable-var-scope %s
+
+define i1 @i1_func_void() #0 {
+  ; CHECK-LABEL: name: i1_func_void
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(s1) = G_LOAD [[DEF]](p1) :: (load 1 from `i1 addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[LOAD]](s1)
+  ; CHECK:   $vgpr0 = COPY [[ANYEXT]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+  ; CHECK:   S_SETPC_B64_return [[COPY1]], implicit $vgpr0
+  %val = load i1, i1 addrspace(1)* undef
+  ret i1 %val
+}
+
+define zeroext i1 @i1_zeroext_func_void() #0 {
+  ; CHECK-LABEL: name: i1_zeroext_func_void
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(s1) = G_LOAD [[DEF]](p1) :: (load 1 from `i1 addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[LOAD]](s1)
+  ; CHECK:   $vgpr0 = COPY [[ZEXT]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+  ; CHECK:   S_SETPC_B64_return [[COPY1]], implicit $vgpr0
+  %val = load i1, i1 addrspace(1)* undef
+  ret i1 %val
+}
+
+define signext i1 @i1_signext_func_void() #0 {
+  ; CHECK-LABEL: name: i1_signext_func_void
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(s1) = G_LOAD [[DEF]](p1) :: (load 1 from `i1 addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[LOAD]](s1)
+  ; CHECK:   $vgpr0 = COPY [[SEXT]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+  ; CHECK:   S_SETPC_B64_return [[COPY1]], implicit $vgpr0
+  %val = load i1, i1 addrspace(1)* undef
+  ret i1 %val
+}
+
+define i8 @i8_func_void() #0 {
+  ; CHECK-LABEL: name: i8_func_void
+  ; CHECK: bb.0:
+  ; CHECK:   successors: %bb.1(0x80000000)
+  ; CHECK:   liveins: $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(s8) = G_LOAD [[DEF]](p1) :: (load 1 from `i8 addrspace(1)* undef`, addrspace 1)
+  %val = load i8, i8 addrspace(1)* undef
+  ret i8 %val
+}
+
+define zeroext i8 @i8_zeroext_func_void() #0 {
+  ; CHECK-LABEL: name: i8_zeroext_func_void
+  ; CHECK: bb.0:
+  ; CHECK:   successors: %bb.1(0x80000000)
+  ; CHECK:   liveins: $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(s8) = G_LOAD [[DEF]](p1) :: (load 1 from `i8 addrspace(1)* undef`, addrspace 1)
+  %val = load i8, i8 addrspace(1)* undef
+  ret i8 %val
+}
+
+define signext i8 @i8_signext_func_void() #0 {
+  ; CHECK-LABEL: name: i8_signext_func_void
+  ; CHECK: bb.0:
+  ; CHECK:   successors: %bb.1(0x80000000)
+  ; CHECK:   liveins: $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(s8) = G_LOAD [[DEF]](p1) :: (load 1 from `i8 addrspace(1)* undef`, addrspace 1)
+  %val = load i8, i8 addrspace(1)* undef
+  ret i8 %val
+}
+
+define i16 @i16_func_void() #0 {
+  ; CHECK-LABEL: name: i16_func_void
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(s16) = G_LOAD [[DEF]](p1) :: (load 2 from `i16 addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[LOAD]](s16)
+  ; CHECK:   $vgpr0 = COPY [[ANYEXT]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+  ; CHECK:   S_SETPC_B64_return [[COPY1]], implicit $vgpr0
+  %val = load i16, i16 addrspace(1)* undef
+  ret i16 %val
+}
+
+define zeroext i16 @i16_zeroext_func_void() #0 {
+  ; CHECK-LABEL: name: i16_zeroext_func_void
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(s16) = G_LOAD [[DEF]](p1) :: (load 2 from `i16 addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[LOAD]](s16)
+  ; CHECK:   $vgpr0 = COPY [[ZEXT]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+  ; CHECK:   S_SETPC_B64_return [[COPY1]], implicit $vgpr0
+  %val = load i16, i16 addrspace(1)* undef
+  ret i16 %val
+}
+
+define signext i16 @i16_signext_func_void() #0 {
+  ; CHECK-LABEL: name: i16_signext_func_void
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(s16) = G_LOAD [[DEF]](p1) :: (load 2 from `i16 addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[LOAD]](s16)
+  ; CHECK:   $vgpr0 = COPY [[SEXT]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+  ; CHECK:   S_SETPC_B64_return [[COPY1]], implicit $vgpr0
+  %val = load i16, i16 addrspace(1)* undef
+  ret i16 %val
+}
+
+define i32 @i32_func_void() #0 {
+  ; CHECK-LABEL: name: i32_func_void
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[DEF]](p1) :: (load 4 from `i32 addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   $vgpr0 = COPY [[LOAD]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+  ; CHECK:   S_SETPC_B64_return [[COPY1]], implicit $vgpr0
+  %val = load i32, i32 addrspace(1)* undef
+  ret i32 %val
+}
+
+define i48 @i48_func_void() #0 {
+  ; CHECK-LABEL: name: i48_func_void
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(s48) = G_LOAD [[DEF]](p1) :: (load 6 from `i48 addrspace(1)* undef`, align 8, addrspace 1)
+  ; CHECK:   [[DEF1:%[0-9]+]]:_(s64) = G_IMPLICIT_DEF
+  ; CHECK:   [[INSERT:%[0-9]+]]:_(s64) = G_INSERT [[DEF1]], [[LOAD]](s48), 0
+  ; CHECK:   [[EXTRACT:%[0-9]+]]:_(s32) = G_EXTRACT [[INSERT]](s64), 0
+  ; CHECK:   [[EXTRACT1:%[0-9]+]]:_(s32) = G_EXTRACT [[INSERT]](s64), 32
+  ; CHECK:   $vgpr0 = COPY [[EXTRACT]](s32)
+  ; CHECK:   $vgpr1 = COPY [[EXTRACT1]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+  ; CHECK:   S_SETPC_B64_return [[COPY1]], implicit $vgpr0, implicit $vgpr1
+  %val = load i48, i48 addrspace(1)* undef, align 8
+  ret i48 %val
+}
+
+define i64 @i64_func_void() #0 {
+  ; CHECK-LABEL: name: i64_func_void
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[DEF]](p1) :: (load 8 from `i64 addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[LOAD]](s64)
+  ; CHECK:   $vgpr0 = COPY [[UV]](s32)
+  ; CHECK:   $vgpr1 = COPY [[UV1]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+  ; CHECK:   S_SETPC_B64_return [[COPY1]], implicit $vgpr0, implicit $vgpr1
+  %val = load i64, i64 addrspace(1)* undef
+  ret i64 %val
+}
+
+define i65 @i65_func_void() #0 {
+  ; CHECK-LABEL: name: i65_func_void
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(s65) = G_LOAD [[DEF]](p1) :: (load 9 from `i65 addrspace(1)* undef`, align 8, addrspace 1)
+  ; CHECK:   [[DEF1:%[0-9]+]]:_(s96) = G_IMPLICIT_DEF
+  ; CHECK:   [[INSERT:%[0-9]+]]:_(s96) = G_INSERT [[DEF1]], [[LOAD]](s65), 0
+  ; CHECK:   [[EXTRACT:%[0-9]+]]:_(s32) = G_EXTRACT [[INSERT]](s96), 0
+  ; CHECK:   [[EXTRACT1:%[0-9]+]]:_(s32) = G_EXTRACT [[INSERT]](s96), 32
+  ; CHECK:   [[EXTRACT2:%[0-9]+]]:_(s32) = G_EXTRACT [[INSERT]](s96), 64
+  ; CHECK:   $vgpr0 = COPY [[EXTRACT]](s32)
+  ; CHECK:   $vgpr1 = COPY [[EXTRACT1]](s32)
+  ; CHECK:   $vgpr2 = COPY [[EXTRACT2]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+  ; CHECK:   S_SETPC_B64_return [[COPY1]], implicit $vgpr0, implicit $vgpr1, implicit $vgpr2
+  %val = load i65, i65 addrspace(1)* undef
+  ret i65 %val
+}
+
+define float @f32_func_void() #0 {
+  ; CHECK-LABEL: name: f32_func_void
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[DEF]](p1) :: (load 4 from `float addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   $vgpr0 = COPY [[LOAD]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+  ; CHECK:   S_SETPC_B64_return [[COPY1]], implicit $vgpr0
+  %val = load float, float addrspace(1)* undef
+  ret float %val
+}
+
+define double @f64_func_void() #0 {
+  ; CHECK-LABEL: name: f64_func_void
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[DEF]](p1) :: (load 8 from `double addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[LOAD]](s64)
+  ; CHECK:   $vgpr0 = COPY [[UV]](s32)
+  ; CHECK:   $vgpr1 = COPY [[UV1]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+  ; CHECK:   S_SETPC_B64_return [[COPY1]], implicit $vgpr0, implicit $vgpr1
+  %val = load double, double addrspace(1)* undef
+  ret double %val
+}
+
+define <2 x double> @v2f64_func_void() #0 {
+  ; CHECK-LABEL: name: v2f64_func_void
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(<2 x s64>) = G_LOAD [[DEF]](p1) :: (load 16 from `<2 x double> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[LOAD]](<2 x s64>)
+  ; CHECK:   $vgpr0 = COPY [[UV]](s32)
+  ; CHECK:   $vgpr1 = COPY [[UV1]](s32)
+  ; CHECK:   $vgpr2 = COPY [[UV2]](s32)
+  ; CHECK:   $vgpr3 = COPY [[UV3]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+  ; CHECK:   S_SETPC_B64_return [[COPY1]], implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+  %val = load <2 x double>, <2 x double> addrspace(1)* undef
+  ret <2 x double> %val
+}
+
+define <2 x i32> @v2i32_func_void() #0 {
+  ; CHECK-LABEL: name: v2i32_func_void
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[DEF]](p1) :: (load 8 from `<2 x i32> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[LOAD]](<2 x s32>)
+  ; CHECK:   $vgpr0 = COPY [[UV]](s32)
+  ; CHECK:   $vgpr1 = COPY [[UV1]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+  ; CHECK:   S_SETPC_B64_return [[COPY1]], implicit $vgpr0, implicit $vgpr1
+  %val = load <2 x i32>, <2 x i32> addrspace(1)* undef
+  ret <2 x i32> %val
+}
+
+define <3 x i32> @v3i32_func_void() #0 {
+  ; CHECK-LABEL: name: v3i32_func_void
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(<3 x s32>) = G_LOAD [[DEF]](p1) :: (load 12 from `<3 x i32> addrspace(1)* undef`, align 16, addrspace 1)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[LOAD]](<3 x s32>)
+  ; CHECK:   $vgpr0 = COPY [[UV]](s32)
+  ; CHECK:   $vgpr1 = COPY [[UV1]](s32)
+  ; CHECK:   $vgpr2 = COPY [[UV2]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+  ; CHECK:   S_SETPC_B64_return [[COPY1]], implicit $vgpr0, implicit $vgpr1, implicit $vgpr2
+  %val = load <3 x i32>, <3 x i32> addrspace(1)* undef
+  ret <3 x i32> %val
+}
+
+define <4 x i32> @v4i32_func_void() #0 {
+  ; CHECK-LABEL: name: v4i32_func_void
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(<4 x s32>) = G_LOAD [[DEF]](p1) :: (load 16 from `<4 x i32> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[LOAD]](<4 x s32>)
+  ; CHECK:   $vgpr0 = COPY [[UV]](s32)
+  ; CHECK:   $vgpr1 = COPY [[UV1]](s32)
+  ; CHECK:   $vgpr2 = COPY [[UV2]](s32)
+  ; CHECK:   $vgpr3 = COPY [[UV3]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+  ; CHECK:   S_SETPC_B64_return [[COPY1]], implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+  %val = load <4 x i32>, <4 x i32> addrspace(1)* undef
+  ret <4 x i32> %val
+}
+
+define <5 x i32> @v5i32_func_void() #0 {
+  ; CHECK-LABEL: name: v5i32_func_void
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(<5 x s32>) = G_LOAD [[DEF]](p1) :: (volatile load 20 from `<5 x i32> addrspace(1)* undef`, align 32, addrspace 1)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[LOAD]](<5 x s32>)
+  ; CHECK:   $vgpr0 = COPY [[UV]](s32)
+  ; CHECK:   $vgpr1 = COPY [[UV1]](s32)
+  ; CHECK:   $vgpr2 = COPY [[UV2]](s32)
+  ; CHECK:   $vgpr3 = COPY [[UV3]](s32)
+  ; CHECK:   $vgpr4 = COPY [[UV4]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+  ; CHECK:   S_SETPC_B64_return [[COPY1]], implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4
+  %val = load volatile <5 x i32>, <5 x i32> addrspace(1)* undef
+  ret <5 x i32> %val
+}
+
+define <8 x i32> @v8i32_func_void() #0 {
+  ; CHECK-LABEL: name: v8i32_func_void
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p4) = G_IMPLICIT_DEF
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[DEF]](p4) :: (volatile load 8 from `<8 x i32> addrspace(1)* addrspace(4)* undef`, addrspace 4)
+  ; CHECK:   [[LOAD1:%[0-9]+]]:_(<8 x s32>) = G_LOAD [[LOAD]](p1) :: (load 32 from %ir.ptr, addrspace 1)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32), [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[LOAD1]](<8 x s32>)
+  ; CHECK:   $vgpr0 = COPY [[UV]](s32)
+  ; CHECK:   $vgpr1 = COPY [[UV1]](s32)
+  ; CHECK:   $vgpr2 = COPY [[UV2]](s32)
+  ; CHECK:   $vgpr3 = COPY [[UV3]](s32)
+  ; CHECK:   $vgpr4 = COPY [[UV4]](s32)
+  ; CHECK:   $vgpr5 = COPY [[UV5]](s32)
+  ; CHECK:   $vgpr6 = COPY [[UV6]](s32)
+  ; CHECK:   $vgpr7 = COPY [[UV7]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+  ; CHECK:   S_SETPC_B64_return [[COPY1]], implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5, implicit $vgpr6, implicit $vgpr7
+  %ptr = load volatile <8 x i32> addrspace(1)*, <8 x i32> addrspace(1)* addrspace(4)* undef
+  %val = load <8 x i32>, <8 x i32> addrspace(1)* %ptr
+  ret <8 x i32> %val
+}
+
+define <16 x i32> @v16i32_func_void() #0 {
+  ; CHECK-LABEL: name: v16i32_func_void
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p4) = G_IMPLICIT_DEF
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[DEF]](p4) :: (volatile load 8 from `<16 x i32> addrspace(1)* addrspace(4)* undef`, addrspace 4)
+  ; CHECK:   [[LOAD1:%[0-9]+]]:_(<16 x s32>) = G_LOAD [[LOAD]](p1) :: (load 64 from %ir.ptr, addrspace 1)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32), [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32), [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32), [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32), [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32), [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[LOAD1]](<16 x s32>)
+  ; CHECK:   $vgpr0 = COPY [[UV]](s32)
+  ; CHECK:   $vgpr1 = COPY [[UV1]](s32)
+  ; CHECK:   $vgpr2 = COPY [[UV2]](s32)
+  ; CHECK:   $vgpr3 = COPY [[UV3]](s32)
+  ; CHECK:   $vgpr4 = COPY [[UV4]](s32)
+  ; CHECK:   $vgpr5 = COPY [[UV5]](s32)
+  ; CHECK:   $vgpr6 = COPY [[UV6]](s32)
+  ; CHECK:   $vgpr7 = COPY [[UV7]](s32)
+  ; CHECK:   $vgpr8 = COPY [[UV8]](s32)
+  ; CHECK:   $vgpr9 = COPY [[UV9]](s32)
+  ; CHECK:   $vgpr10 = COPY [[UV10]](s32)
+  ; CHECK:   $vgpr11 = COPY [[UV11]](s32)
+  ; CHECK:   $vgpr12 = COPY [[UV12]](s32)
+  ; CHECK:   $vgpr13 = COPY [[UV13]](s32)
+  ; CHECK:   $vgpr14 = COPY [[UV14]](s32)
+  ; CHECK:   $vgpr15 = COPY [[UV15]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+  ; CHECK:   S_SETPC_B64_return [[COPY1]], implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5, implicit $vgpr6, implicit $vgpr7, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11, implicit $vgpr12, implicit $vgpr13, implicit $vgpr14, implicit $vgpr15
+  %ptr = load volatile <16 x i32> addrspace(1)*, <16 x i32> addrspace(1)* addrspace(4)* undef
+  %val = load <16 x i32>, <16 x i32> addrspace(1)* %ptr
+  ret <16 x i32> %val
+}
+
+define <32 x i32> @v32i32_func_void() #0 {
+  ; CHECK-LABEL: name: v32i32_func_void
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p4) = G_IMPLICIT_DEF
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[DEF]](p4) :: (volatile load 8 from `<32 x i32> addrspace(1)* addrspace(4)* undef`, addrspace 4)
+  ; CHECK:   [[LOAD1:%[0-9]+]]:_(<32 x s32>) = G_LOAD [[LOAD]](p1) :: (load 128 from %ir.ptr, addrspace 1)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32), [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32), [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32), [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32), [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32), [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32), [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32), [[UV18:%[0-9]+]]:_(s32), [[UV19:%[0-9]+]]:_(s32), [[UV20:%[0-9]+]]:_(s32), [[UV21:%[0-9]+]]:_(s32), [[UV22:%[0-9]+]]:_(s32), [[UV23:%[0-9]+]]:_(s32), [[UV24:%[0-9]+]]:_(s32), [[UV25:%[0-9]+]]:_(s32), [[UV26:%[0-9]+]]:_(s32), [[UV27:%[0-9]+]]:_(s32), [[UV28:%[0-9]+]]:_(s32), [[UV29:%[0-9]+]]:_(s32), [[UV30:%[0-9]+]]:_(s32), [[UV31:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[LOAD1]](<32 x s32>)
+  ; CHECK:   $vgpr0 = COPY [[UV]](s32)
+  ; CHECK:   $vgpr1 = COPY [[UV1]](s32)
+  ; CHECK:   $vgpr2 = COPY [[UV2]](s32)
+  ; CHECK:   $vgpr3 = COPY [[UV3]](s32)
+  ; CHECK:   $vgpr4 = COPY [[UV4]](s32)
+  ; CHECK:   $vgpr5 = COPY [[UV5]](s32)
+  ; CHECK:   $vgpr6 = COPY [[UV6]](s32)
+  ; CHECK:   $vgpr7 = COPY [[UV7]](s32)
+  ; CHECK:   $vgpr8 = COPY [[UV8]](s32)
+  ; CHECK:   $vgpr9 = COPY [[UV9]](s32)
+  ; CHECK:   $vgpr10 = COPY [[UV10]](s32)
+  ; CHECK:   $vgpr11 = COPY [[UV11]](s32)
+  ; CHECK:   $vgpr12 = COPY [[UV12]](s32)
+  ; CHECK:   $vgpr13 = COPY [[UV13]](s32)
+  ; CHECK:   $vgpr14 = COPY [[UV14]](s32)
+  ; CHECK:   $vgpr15 = COPY [[UV15]](s32)
+  ; CHECK:   $vgpr16 = COPY [[UV16]](s32)
+  ; CHECK:   $vgpr17 = COPY [[UV17]](s32)
+  ; CHECK:   $vgpr18 = COPY [[UV18]](s32)
+  ; CHECK:   $vgpr19 = COPY [[UV19]](s32)
+  ; CHECK:   $vgpr20 = COPY [[UV20]](s32)
+  ; CHECK:   $vgpr21 = COPY [[UV21]](s32)
+  ; CHECK:   $vgpr22 = COPY [[UV22]](s32)
+  ; CHECK:   $vgpr23 = COPY [[UV23]](s32)
+  ; CHECK:   $vgpr24 = COPY [[UV24]](s32)
+  ; CHECK:   $vgpr25 = COPY [[UV25]](s32)
+  ; CHECK:   $vgpr26 = COPY [[UV26]](s32)
+  ; CHECK:   $vgpr27 = COPY [[UV27]](s32)
+  ; CHECK:   $vgpr28 = COPY [[UV28]](s32)
+  ; CHECK:   $vgpr29 = COPY [[UV29]](s32)
+  ; CHECK:   $vgpr30 = COPY [[UV30]](s32)
+  ; CHECK:   $vgpr31 = COPY [[UV31]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+  ; CHECK:   S_SETPC_B64_return [[COPY1]], implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5, implicit $vgpr6, implicit $vgpr7, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11, implicit $vgpr12, implicit $vgpr13, implicit $vgpr14, implicit $vgpr15, implicit $vgpr16, implicit $vgpr17, implicit $vgpr18, implicit $vgpr19, implicit $vgpr20, implicit $vgpr21, implicit $vgpr22, implicit $vgpr23, implicit $vgpr24, implicit $vgpr25, implicit $vgpr26, implicit $vgpr27, implicit $vgpr28, implicit $vgpr29, implicit $vgpr30, implicit $vgpr31
+  %ptr = load volatile <32 x i32> addrspace(1)*, <32 x i32> addrspace(1)* addrspace(4)* undef
+  %val = load <32 x i32>, <32 x i32> addrspace(1)* %ptr
+  ret <32 x i32> %val
+}
+
+define <2 x i64> @v2i64_func_void() #0 {
+  ; CHECK-LABEL: name: v2i64_func_void
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(<2 x s64>) = G_LOAD [[DEF]](p1) :: (load 16 from `<2 x i64> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[LOAD]](<2 x s64>)
+  ; CHECK:   $vgpr0 = COPY [[UV]](s32)
+  ; CHECK:   $vgpr1 = COPY [[UV1]](s32)
+  ; CHECK:   $vgpr2 = COPY [[UV2]](s32)
+  ; CHECK:   $vgpr3 = COPY [[UV3]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+  ; CHECK:   S_SETPC_B64_return [[COPY1]], implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+  %val = load <2 x i64>, <2 x i64> addrspace(1)* undef
+  ret <2 x i64> %val
+}
+
+define <3 x i64> @v3i64_func_void() #0 {
+  ; CHECK-LABEL: name: v3i64_func_void
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p4) = G_IMPLICIT_DEF
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[DEF]](p4) :: (volatile load 8 from `<3 x i64> addrspace(1)* addrspace(4)* undef`, addrspace 4)
+  ; CHECK:   [[LOAD1:%[0-9]+]]:_(<3 x s64>) = G_LOAD [[LOAD]](p1) :: (load 24 from %ir.ptr, align 32, addrspace 1)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[LOAD1]](<3 x s64>)
+  ; CHECK:   $vgpr0 = COPY [[UV]](s32)
+  ; CHECK:   $vgpr1 = COPY [[UV1]](s32)
+  ; CHECK:   $vgpr2 = COPY [[UV2]](s32)
+  ; CHECK:   $vgpr3 = COPY [[UV3]](s32)
+  ; CHECK:   $vgpr4 = COPY [[UV4]](s32)
+  ; CHECK:   $vgpr5 = COPY [[UV5]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+  ; CHECK:   S_SETPC_B64_return [[COPY1]], implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5
+  %ptr = load volatile <3 x i64> addrspace(1)*, <3 x i64> addrspace(1)* addrspace(4)* undef
+  %val = load <3 x i64>, <3 x i64> addrspace(1)* %ptr
+  ret <3 x i64> %val
+}
+
+define <4 x i64> @v4i64_func_void() #0 {
+  ; CHECK-LABEL: name: v4i64_func_void
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p4) = G_IMPLICIT_DEF
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[DEF]](p4) :: (volatile load 8 from `<4 x i64> addrspace(1)* addrspace(4)* undef`, addrspace 4)
+  ; CHECK:   [[LOAD1:%[0-9]+]]:_(<4 x s64>) = G_LOAD [[LOAD]](p1) :: (load 32 from %ir.ptr, addrspace 1)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32), [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[LOAD1]](<4 x s64>)
+  ; CHECK:   $vgpr0 = COPY [[UV]](s32)
+  ; CHECK:   $vgpr1 = COPY [[UV1]](s32)
+  ; CHECK:   $vgpr2 = COPY [[UV2]](s32)
+  ; CHECK:   $vgpr3 = COPY [[UV3]](s32)
+  ; CHECK:   $vgpr4 = COPY [[UV4]](s32)
+  ; CHECK:   $vgpr5 = COPY [[UV5]](s32)
+  ; CHECK:   $vgpr6 = COPY [[UV6]](s32)
+  ; CHECK:   $vgpr7 = COPY [[UV7]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+  ; CHECK:   S_SETPC_B64_return [[COPY1]], implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5, implicit $vgpr6, implicit $vgpr7
+  %ptr = load volatile <4 x i64> addrspace(1)*, <4 x i64> addrspace(1)* addrspace(4)* undef
+  %val = load <4 x i64>, <4 x i64> addrspace(1)* %ptr
+  ret <4 x i64> %val
+}
+
+define <5 x i64> @v5i64_func_void() #0 {
+  ; CHECK-LABEL: name: v5i64_func_void
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p4) = G_IMPLICIT_DEF
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[DEF]](p4) :: (volatile load 8 from `<5 x i64> addrspace(1)* addrspace(4)* undef`, addrspace 4)
+  ; CHECK:   [[LOAD1:%[0-9]+]]:_(<5 x s64>) = G_LOAD [[LOAD]](p1) :: (load 40 from %ir.ptr, align 64, addrspace 1)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32), [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32), [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[LOAD1]](<5 x s64>)
+  ; CHECK:   $vgpr0 = COPY [[UV]](s32)
+  ; CHECK:   $vgpr1 = COPY [[UV1]](s32)
+  ; CHECK:   $vgpr2 = COPY [[UV2]](s32)
+  ; CHECK:   $vgpr3 = COPY [[UV3]](s32)
+  ; CHECK:   $vgpr4 = COPY [[UV4]](s32)
+  ; CHECK:   $vgpr5 = COPY [[UV5]](s32)
+  ; CHECK:   $vgpr6 = COPY [[UV6]](s32)
+  ; CHECK:   $vgpr7 = COPY [[UV7]](s32)
+  ; CHECK:   $vgpr8 = COPY [[UV8]](s32)
+  ; CHECK:   $vgpr9 = COPY [[UV9]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+  ; CHECK:   S_SETPC_B64_return [[COPY1]], implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5, implicit $vgpr6, implicit $vgpr7, implicit $vgpr8, implicit $vgpr9
+  %ptr = load volatile <5 x i64> addrspace(1)*, <5 x i64> addrspace(1)* addrspace(4)* undef
+  %val = load <5 x i64>, <5 x i64> addrspace(1)* %ptr
+  ret <5 x i64> %val
+}
+
+define <8 x i64> @v8i64_func_void() #0 {
+  ; CHECK-LABEL: name: v8i64_func_void
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p4) = G_IMPLICIT_DEF
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[DEF]](p4) :: (volatile load 8 from `<8 x i64> addrspace(1)* addrspace(4)* undef`, addrspace 4)
+  ; CHECK:   [[LOAD1:%[0-9]+]]:_(<8 x s64>) = G_LOAD [[LOAD]](p1) :: (load 64 from %ir.ptr, addrspace 1)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32), [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32), [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32), [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32), [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32), [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[LOAD1]](<8 x s64>)
+  ; CHECK:   $vgpr0 = COPY [[UV]](s32)
+  ; CHECK:   $vgpr1 = COPY [[UV1]](s32)
+  ; CHECK:   $vgpr2 = COPY [[UV2]](s32)
+  ; CHECK:   $vgpr3 = COPY [[UV3]](s32)
+  ; CHECK:   $vgpr4 = COPY [[UV4]](s32)
+  ; CHECK:   $vgpr5 = COPY [[UV5]](s32)
+  ; CHECK:   $vgpr6 = COPY [[UV6]](s32)
+  ; CHECK:   $vgpr7 = COPY [[UV7]](s32)
+  ; CHECK:   $vgpr8 = COPY [[UV8]](s32)
+  ; CHECK:   $vgpr9 = COPY [[UV9]](s32)
+  ; CHECK:   $vgpr10 = COPY [[UV10]](s32)
+  ; CHECK:   $vgpr11 = COPY [[UV11]](s32)
+  ; CHECK:   $vgpr12 = COPY [[UV12]](s32)
+  ; CHECK:   $vgpr13 = COPY [[UV13]](s32)
+  ; CHECK:   $vgpr14 = COPY [[UV14]](s32)
+  ; CHECK:   $vgpr15 = COPY [[UV15]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+  ; CHECK:   S_SETPC_B64_return [[COPY1]], implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5, implicit $vgpr6, implicit $vgpr7, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11, implicit $vgpr12, implicit $vgpr13, implicit $vgpr14, implicit $vgpr15
+  %ptr = load volatile <8 x i64> addrspace(1)*, <8 x i64> addrspace(1)* addrspace(4)* undef
+  %val = load <8 x i64>, <8 x i64> addrspace(1)* %ptr
+  ret <8 x i64> %val
+}
+
+define <16 x i64> @v16i64_func_void() #0 {
+  ; CHECK-LABEL: name: v16i64_func_void
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p4) = G_IMPLICIT_DEF
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[DEF]](p4) :: (volatile load 8 from `<16 x i64> addrspace(1)* addrspace(4)* undef`, addrspace 4)
+  ; CHECK:   [[LOAD1:%[0-9]+]]:_(<16 x s64>) = G_LOAD [[LOAD]](p1) :: (load 128 from %ir.ptr, addrspace 1)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32), [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32), [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32), [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32), [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32), [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32), [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32), [[UV18:%[0-9]+]]:_(s32), [[UV19:%[0-9]+]]:_(s32), [[UV20:%[0-9]+]]:_(s32), [[UV21:%[0-9]+]]:_(s32), [[UV22:%[0-9]+]]:_(s32), [[UV23:%[0-9]+]]:_(s32), [[UV24:%[0-9]+]]:_(s32), [[UV25:%[0-9]+]]:_(s32), [[UV26:%[0-9]+]]:_(s32), [[UV27:%[0-9]+]]:_(s32), [[UV28:%[0-9]+]]:_(s32), [[UV29:%[0-9]+]]:_(s32), [[UV30:%[0-9]+]]:_(s32), [[UV31:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[LOAD1]](<16 x s64>)
+  ; CHECK:   $vgpr0 = COPY [[UV]](s32)
+  ; CHECK:   $vgpr1 = COPY [[UV1]](s32)
+  ; CHECK:   $vgpr2 = COPY [[UV2]](s32)
+  ; CHECK:   $vgpr3 = COPY [[UV3]](s32)
+  ; CHECK:   $vgpr4 = COPY [[UV4]](s32)
+  ; CHECK:   $vgpr5 = COPY [[UV5]](s32)
+  ; CHECK:   $vgpr6 = COPY [[UV6]](s32)
+  ; CHECK:   $vgpr7 = COPY [[UV7]](s32)
+  ; CHECK:   $vgpr8 = COPY [[UV8]](s32)
+  ; CHECK:   $vgpr9 = COPY [[UV9]](s32)
+  ; CHECK:   $vgpr10 = COPY [[UV10]](s32)
+  ; CHECK:   $vgpr11 = COPY [[UV11]](s32)
+  ; CHECK:   $vgpr12 = COPY [[UV12]](s32)
+  ; CHECK:   $vgpr13 = COPY [[UV13]](s32)
+  ; CHECK:   $vgpr14 = COPY [[UV14]](s32)
+  ; CHECK:   $vgpr15 = COPY [[UV15]](s32)
+  ; CHECK:   $vgpr16 = COPY [[UV16]](s32)
+  ; CHECK:   $vgpr17 = COPY [[UV17]](s32)
+  ; CHECK:   $vgpr18 = COPY [[UV18]](s32)
+  ; CHECK:   $vgpr19 = COPY [[UV19]](s32)
+  ; CHECK:   $vgpr20 = COPY [[UV20]](s32)
+  ; CHECK:   $vgpr21 = COPY [[UV21]](s32)
+  ; CHECK:   $vgpr22 = COPY [[UV22]](s32)
+  ; CHECK:   $vgpr23 = COPY [[UV23]](s32)
+  ; CHECK:   $vgpr24 = COPY [[UV24]](s32)
+  ; CHECK:   $vgpr25 = COPY [[UV25]](s32)
+  ; CHECK:   $vgpr26 = COPY [[UV26]](s32)
+  ; CHECK:   $vgpr27 = COPY [[UV27]](s32)
+  ; CHECK:   $vgpr28 = COPY [[UV28]](s32)
+  ; CHECK:   $vgpr29 = COPY [[UV29]](s32)
+  ; CHECK:   $vgpr30 = COPY [[UV30]](s32)
+  ; CHECK:   $vgpr31 = COPY [[UV31]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+  ; CHECK:   S_SETPC_B64_return [[COPY1]], implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5, implicit $vgpr6, implicit $vgpr7, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11, implicit $vgpr12, implicit $vgpr13, implicit $vgpr14, implicit $vgpr15, implicit $vgpr16, implicit $vgpr17, implicit $vgpr18, implicit $vgpr19, implicit $vgpr20, implicit $vgpr21, implicit $vgpr22, implicit $vgpr23, implicit $vgpr24, implicit $vgpr25, implicit $vgpr26, implicit $vgpr27, implicit $vgpr28, implicit $vgpr29, implicit $vgpr30, implicit $vgpr31
+  %ptr = load volatile <16 x i64> addrspace(1)*, <16 x i64> addrspace(1)* addrspace(4)* undef
+  %val = load <16 x i64>, <16 x i64> addrspace(1)* %ptr
+  ret <16 x i64> %val
+}
+
+define <2 x i16> @v2i16_func_void() #0 {
+  ; CHECK-LABEL: name: v2i16_func_void
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(<2 x s16>) = G_LOAD [[DEF]](p1) :: (load 4 from `<2 x i16> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[LOAD]](<2 x s16>)
+  ; CHECK:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[UV]](s16)
+  ; CHECK:   [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[UV1]](s16)
+  ; CHECK:   $vgpr0 = COPY [[ANYEXT]](s32)
+  ; CHECK:   $vgpr1 = COPY [[ANYEXT1]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+  ; CHECK:   S_SETPC_B64_return [[COPY1]], implicit $vgpr0, implicit $vgpr1
+  %val = load <2 x i16>, <2 x i16> addrspace(1)* undef
+  ret <2 x i16> %val
+}
+
+define <3 x i16> @v3i16_func_void() #0 {
+  ; CHECK-LABEL: name: v3i16_func_void
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(<3 x s16>) = G_LOAD [[DEF]](p1) :: (load 6 from `<3 x i16> addrspace(1)* undef`, align 8, addrspace 1)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[LOAD]](<3 x s16>)
+  ; CHECK:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[UV]](s16)
+  ; CHECK:   [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[UV1]](s16)
+  ; CHECK:   [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[UV2]](s16)
+  ; CHECK:   $vgpr0 = COPY [[ANYEXT]](s32)
+  ; CHECK:   $vgpr1 = COPY [[ANYEXT1]](s32)
+  ; CHECK:   $vgpr2 = COPY [[ANYEXT2]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+  ; CHECK:   S_SETPC_B64_return [[COPY1]], implicit $vgpr0, implicit $vgpr1, implicit $vgpr2
+  %val = load <3 x i16>, <3 x i16> addrspace(1)* undef
+  ret <3 x i16> %val
+}
+
+define <4 x i16> @v4i16_func_void() #0 {
+  ; CHECK-LABEL: name: v4i16_func_void
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(<4 x s16>) = G_LOAD [[DEF]](p1) :: (load 8 from `<4 x i16> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[LOAD]](<4 x s16>)
+  ; CHECK:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[UV]](s16)
+  ; CHECK:   [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[UV1]](s16)
+  ; CHECK:   [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[UV2]](s16)
+  ; CHECK:   [[ANYEXT3:%[0-9]+]]:_(s32) = G_ANYEXT [[UV3]](s16)
+  ; CHECK:   $vgpr0 = COPY [[ANYEXT]](s32)
+  ; CHECK:   $vgpr1 = COPY [[ANYEXT1]](s32)
+  ; CHECK:   $vgpr2 = COPY [[ANYEXT2]](s32)
+  ; CHECK:   $vgpr3 = COPY [[ANYEXT3]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+  ; CHECK:   S_SETPC_B64_return [[COPY1]], implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+  %val = load <4 x i16>, <4 x i16> addrspace(1)* undef
+  ret <4 x i16> %val
+}
+
+define <4 x half> @v4f16_func_void() #0 {
+  ; CHECK-LABEL: name: v4f16_func_void
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(<4 x s16>) = G_LOAD [[DEF]](p1) :: (load 8 from `<4 x half> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[LOAD]](<4 x s16>)
+  ; CHECK:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[UV]](s16)
+  ; CHECK:   [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[UV1]](s16)
+  ; CHECK:   [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[UV2]](s16)
+  ; CHECK:   [[ANYEXT3:%[0-9]+]]:_(s32) = G_ANYEXT [[UV3]](s16)
+  ; CHECK:   $vgpr0 = COPY [[ANYEXT]](s32)
+  ; CHECK:   $vgpr1 = COPY [[ANYEXT1]](s32)
+  ; CHECK:   $vgpr2 = COPY [[ANYEXT2]](s32)
+  ; CHECK:   $vgpr3 = COPY [[ANYEXT3]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+  ; CHECK:   S_SETPC_B64_return [[COPY1]], implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+  %val = load <4 x half>, <4 x half> addrspace(1)* undef
+  ret <4 x half> %val
+}
+
+define <5 x i16> @v5i16_func_void() #0 {
+  ; CHECK-LABEL: name: v5i16_func_void
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p4) = G_IMPLICIT_DEF
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[DEF]](p4) :: (volatile load 8 from `<5 x i16> addrspace(1)* addrspace(4)* undef`, addrspace 4)
+  ; CHECK:   [[LOAD1:%[0-9]+]]:_(<5 x s16>) = G_LOAD [[LOAD]](p1) :: (load 10 from %ir.ptr, align 16, addrspace 1)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16), [[UV4:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[LOAD1]](<5 x s16>)
+  ; CHECK:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[UV]](s16)
+  ; CHECK:   [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[UV1]](s16)
+  ; CHECK:   [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[UV2]](s16)
+  ; CHECK:   [[ANYEXT3:%[0-9]+]]:_(s32) = G_ANYEXT [[UV3]](s16)
+  ; CHECK:   [[ANYEXT4:%[0-9]+]]:_(s32) = G_ANYEXT [[UV4]](s16)
+  ; CHECK:   $vgpr0 = COPY [[ANYEXT]](s32)
+  ; CHECK:   $vgpr1 = COPY [[ANYEXT1]](s32)
+  ; CHECK:   $vgpr2 = COPY [[ANYEXT2]](s32)
+  ; CHECK:   $vgpr3 = COPY [[ANYEXT3]](s32)
+  ; CHECK:   $vgpr4 = COPY [[ANYEXT4]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+  ; CHECK:   S_SETPC_B64_return [[COPY1]], implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4
+  %ptr = load volatile <5 x i16> addrspace(1)*, <5 x i16> addrspace(1)* addrspace(4)* undef
+  %val = load <5 x i16>, <5 x i16> addrspace(1)* %ptr
+  ret <5 x i16> %val
+}
+
+define <8 x i16> @v8i16_func_void() #0 {
+  ; CHECK-LABEL: name: v8i16_func_void
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p4) = G_IMPLICIT_DEF
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[DEF]](p4) :: (volatile load 8 from `<8 x i16> addrspace(1)* addrspace(4)* undef`, addrspace 4)
+  ; CHECK:   [[LOAD1:%[0-9]+]]:_(<8 x s16>) = G_LOAD [[LOAD]](p1) :: (load 16 from %ir.ptr, addrspace 1)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16), [[UV4:%[0-9]+]]:_(s16), [[UV5:%[0-9]+]]:_(s16), [[UV6:%[0-9]+]]:_(s16), [[UV7:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[LOAD1]](<8 x s16>)
+  ; CHECK:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[UV]](s16)
+  ; CHECK:   [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[UV1]](s16)
+  ; CHECK:   [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[UV2]](s16)
+  ; CHECK:   [[ANYEXT3:%[0-9]+]]:_(s32) = G_ANYEXT [[UV3]](s16)
+  ; CHECK:   [[ANYEXT4:%[0-9]+]]:_(s32) = G_ANYEXT [[UV4]](s16)
+  ; CHECK:   [[ANYEXT5:%[0-9]+]]:_(s32) = G_ANYEXT [[UV5]](s16)
+  ; CHECK:   [[ANYEXT6:%[0-9]+]]:_(s32) = G_ANYEXT [[UV6]](s16)
+  ; CHECK:   [[ANYEXT7:%[0-9]+]]:_(s32) = G_ANYEXT [[UV7]](s16)
+  ; CHECK:   $vgpr0 = COPY [[ANYEXT]](s32)
+  ; CHECK:   $vgpr1 = COPY [[ANYEXT1]](s32)
+  ; CHECK:   $vgpr2 = COPY [[ANYEXT2]](s32)
+  ; CHECK:   $vgpr3 = COPY [[ANYEXT3]](s32)
+  ; CHECK:   $vgpr4 = COPY [[ANYEXT4]](s32)
+  ; CHECK:   $vgpr5 = COPY [[ANYEXT5]](s32)
+  ; CHECK:   $vgpr6 = COPY [[ANYEXT6]](s32)
+  ; CHECK:   $vgpr7 = COPY [[ANYEXT7]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+  ; CHECK:   S_SETPC_B64_return [[COPY1]], implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5, implicit $vgpr6, implicit $vgpr7
+  %ptr = load volatile <8 x i16> addrspace(1)*, <8 x i16> addrspace(1)* addrspace(4)* undef
+  %val = load <8 x i16>, <8 x i16> addrspace(1)* %ptr
+  ret <8 x i16> %val
+}
+
+define <16 x i16> @v16i16_func_void() #0 {
+  ; CHECK-LABEL: name: v16i16_func_void
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p4) = G_IMPLICIT_DEF
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[DEF]](p4) :: (volatile load 8 from `<16 x i16> addrspace(1)* addrspace(4)* undef`, addrspace 4)
+  ; CHECK:   [[LOAD1:%[0-9]+]]:_(<16 x s16>) = G_LOAD [[LOAD]](p1) :: (load 32 from %ir.ptr, addrspace 1)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16), [[UV4:%[0-9]+]]:_(s16), [[UV5:%[0-9]+]]:_(s16), [[UV6:%[0-9]+]]:_(s16), [[UV7:%[0-9]+]]:_(s16), [[UV8:%[0-9]+]]:_(s16), [[UV9:%[0-9]+]]:_(s16), [[UV10:%[0-9]+]]:_(s16), [[UV11:%[0-9]+]]:_(s16), [[UV12:%[0-9]+]]:_(s16), [[UV13:%[0-9]+]]:_(s16), [[UV14:%[0-9]+]]:_(s16), [[UV15:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[LOAD1]](<16 x s16>)
+  ; CHECK:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[UV]](s16)
+  ; CHECK:   [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[UV1]](s16)
+  ; CHECK:   [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[UV2]](s16)
+  ; CHECK:   [[ANYEXT3:%[0-9]+]]:_(s32) = G_ANYEXT [[UV3]](s16)
+  ; CHECK:   [[ANYEXT4:%[0-9]+]]:_(s32) = G_ANYEXT [[UV4]](s16)
+  ; CHECK:   [[ANYEXT5:%[0-9]+]]:_(s32) = G_ANYEXT [[UV5]](s16)
+  ; CHECK:   [[ANYEXT6:%[0-9]+]]:_(s32) = G_ANYEXT [[UV6]](s16)
+  ; CHECK:   [[ANYEXT7:%[0-9]+]]:_(s32) = G_ANYEXT [[UV7]](s16)
+  ; CHECK:   [[ANYEXT8:%[0-9]+]]:_(s32) = G_ANYEXT [[UV8]](s16)
+  ; CHECK:   [[ANYEXT9:%[0-9]+]]:_(s32) = G_ANYEXT [[UV9]](s16)
+  ; CHECK:   [[ANYEXT10:%[0-9]+]]:_(s32) = G_ANYEXT [[UV10]](s16)
+  ; CHECK:   [[ANYEXT11:%[0-9]+]]:_(s32) = G_ANYEXT [[UV11]](s16)
+  ; CHECK:   [[ANYEXT12:%[0-9]+]]:_(s32) = G_ANYEXT [[UV12]](s16)
+  ; CHECK:   [[ANYEXT13:%[0-9]+]]:_(s32) = G_ANYEXT [[UV13]](s16)
+  ; CHECK:   [[ANYEXT14:%[0-9]+]]:_(s32) = G_ANYEXT [[UV14]](s16)
+  ; CHECK:   [[ANYEXT15:%[0-9]+]]:_(s32) = G_ANYEXT [[UV15]](s16)
+  ; CHECK:   $vgpr0 = COPY [[ANYEXT]](s32)
+  ; CHECK:   $vgpr1 = COPY [[ANYEXT1]](s32)
+  ; CHECK:   $vgpr2 = COPY [[ANYEXT2]](s32)
+  ; CHECK:   $vgpr3 = COPY [[ANYEXT3]](s32)
+  ; CHECK:   $vgpr4 = COPY [[ANYEXT4]](s32)
+  ; CHECK:   $vgpr5 = COPY [[ANYEXT5]](s32)
+  ; CHECK:   $vgpr6 = COPY [[ANYEXT6]](s32)
+  ; CHECK:   $vgpr7 = COPY [[ANYEXT7]](s32)
+  ; CHECK:   $vgpr8 = COPY [[ANYEXT8]](s32)
+  ; CHECK:   $vgpr9 = COPY [[ANYEXT9]](s32)
+  ; CHECK:   $vgpr10 = COPY [[ANYEXT10]](s32)
+  ; CHECK:   $vgpr11 = COPY [[ANYEXT11]](s32)
+  ; CHECK:   $vgpr12 = COPY [[ANYEXT12]](s32)
+  ; CHECK:   $vgpr13 = COPY [[ANYEXT13]](s32)
+  ; CHECK:   $vgpr14 = COPY [[ANYEXT14]](s32)
+  ; CHECK:   $vgpr15 = COPY [[ANYEXT15]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+  ; CHECK:   S_SETPC_B64_return [[COPY1]], implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5, implicit $vgpr6, implicit $vgpr7, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11, implicit $vgpr12, implicit $vgpr13, implicit $vgpr14, implicit $vgpr15
+  %ptr = load volatile <16 x i16> addrspace(1)*, <16 x i16> addrspace(1)* addrspace(4)* undef
+  %val = load <16 x i16>, <16 x i16> addrspace(1)* %ptr
+  ret <16 x i16> %val
+}
+
+define <16 x i8> @v16i8_func_void() #0 {
+  ; CHECK-LABEL: name: v16i8_func_void
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p4) = G_IMPLICIT_DEF
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[DEF]](p4) :: (volatile load 8 from `<16 x i8> addrspace(1)* addrspace(4)* undef`, addrspace 4)
+  ; CHECK:   [[LOAD1:%[0-9]+]]:_(<16 x s8>) = G_LOAD [[LOAD]](p1) :: (load 16 from %ir.ptr, addrspace 1)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8), [[UV2:%[0-9]+]]:_(s8), [[UV3:%[0-9]+]]:_(s8), [[UV4:%[0-9]+]]:_(s8), [[UV5:%[0-9]+]]:_(s8), [[UV6:%[0-9]+]]:_(s8), [[UV7:%[0-9]+]]:_(s8), [[UV8:%[0-9]+]]:_(s8), [[UV9:%[0-9]+]]:_(s8), [[UV10:%[0-9]+]]:_(s8), [[UV11:%[0-9]+]]:_(s8), [[UV12:%[0-9]+]]:_(s8), [[UV13:%[0-9]+]]:_(s8), [[UV14:%[0-9]+]]:_(s8), [[UV15:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[LOAD1]](<16 x s8>)
+  ; CHECK:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[UV]](s8)
+  ; CHECK:   [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[UV1]](s8)
+  ; CHECK:   [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[UV2]](s8)
+  ; CHECK:   [[ANYEXT3:%[0-9]+]]:_(s32) = G_ANYEXT [[UV3]](s8)
+  ; CHECK:   [[ANYEXT4:%[0-9]+]]:_(s32) = G_ANYEXT [[UV4]](s8)
+  ; CHECK:   [[ANYEXT5:%[0-9]+]]:_(s32) = G_ANYEXT [[UV5]](s8)
+  ; CHECK:   [[ANYEXT6:%[0-9]+]]:_(s32) = G_ANYEXT [[UV6]](s8)
+  ; CHECK:   [[ANYEXT7:%[0-9]+]]:_(s32) = G_ANYEXT [[UV7]](s8)
+  ; CHECK:   [[ANYEXT8:%[0-9]+]]:_(s32) = G_ANYEXT [[UV8]](s8)
+  ; CHECK:   [[ANYEXT9:%[0-9]+]]:_(s32) = G_ANYEXT [[UV9]](s8)
+  ; CHECK:   [[ANYEXT10:%[0-9]+]]:_(s32) = G_ANYEXT [[UV10]](s8)
+  ; CHECK:   [[ANYEXT11:%[0-9]+]]:_(s32) = G_ANYEXT [[UV11]](s8)
+  ; CHECK:   [[ANYEXT12:%[0-9]+]]:_(s32) = G_ANYEXT [[UV12]](s8)
+  ; CHECK:   [[ANYEXT13:%[0-9]+]]:_(s32) = G_ANYEXT [[UV13]](s8)
+  ; CHECK:   [[ANYEXT14:%[0-9]+]]:_(s32) = G_ANYEXT [[UV14]](s8)
+  ; CHECK:   [[ANYEXT15:%[0-9]+]]:_(s32) = G_ANYEXT [[UV15]](s8)
+  ; CHECK:   $vgpr0 = COPY [[ANYEXT]](s32)
+  ; CHECK:   $vgpr1 = COPY [[ANYEXT1]](s32)
+  ; CHECK:   $vgpr2 = COPY [[ANYEXT2]](s32)
+  ; CHECK:   $vgpr3 = COPY [[ANYEXT3]](s32)
+  ; CHECK:   $vgpr4 = COPY [[ANYEXT4]](s32)
+  ; CHECK:   $vgpr5 = COPY [[ANYEXT5]](s32)
+  ; CHECK:   $vgpr6 = COPY [[ANYEXT6]](s32)
+  ; CHECK:   $vgpr7 = COPY [[ANYEXT7]](s32)
+  ; CHECK:   $vgpr8 = COPY [[ANYEXT8]](s32)
+  ; CHECK:   $vgpr9 = COPY [[ANYEXT9]](s32)
+  ; CHECK:   $vgpr10 = COPY [[ANYEXT10]](s32)
+  ; CHECK:   $vgpr11 = COPY [[ANYEXT11]](s32)
+  ; CHECK:   $vgpr12 = COPY [[ANYEXT12]](s32)
+  ; CHECK:   $vgpr13 = COPY [[ANYEXT13]](s32)
+  ; CHECK:   $vgpr14 = COPY [[ANYEXT14]](s32)
+  ; CHECK:   $vgpr15 = COPY [[ANYEXT15]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+  ; CHECK:   S_SETPC_B64_return [[COPY1]], implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5, implicit $vgpr6, implicit $vgpr7, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11, implicit $vgpr12, implicit $vgpr13, implicit $vgpr14, implicit $vgpr15
+  %ptr = load volatile <16 x i8> addrspace(1)*, <16 x i8> addrspace(1)* addrspace(4)* undef
+  %val = load <16 x i8>, <16 x i8> addrspace(1)* %ptr
+  ret <16 x i8> %val
+}
+
+define <4  x i8> @v4i8_func_void() #0 {
+  ; CHECK-LABEL: name: v4i8_func_void
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p4) = G_IMPLICIT_DEF
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[DEF]](p4) :: (volatile load 8 from `<4 x i8> addrspace(1)* addrspace(4)* undef`, addrspace 4)
+  ; CHECK:   [[LOAD1:%[0-9]+]]:_(<4 x s8>) = G_LOAD [[LOAD]](p1) :: (load 4 from %ir.ptr, addrspace 1)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8), [[UV2:%[0-9]+]]:_(s8), [[UV3:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[LOAD1]](<4 x s8>)
+  ; CHECK:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[UV]](s8)
+  ; CHECK:   [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[UV1]](s8)
+  ; CHECK:   [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[UV2]](s8)
+  ; CHECK:   [[ANYEXT3:%[0-9]+]]:_(s32) = G_ANYEXT [[UV3]](s8)
+  ; CHECK:   $vgpr0 = COPY [[ANYEXT]](s32)
+  ; CHECK:   $vgpr1 = COPY [[ANYEXT1]](s32)
+  ; CHECK:   $vgpr2 = COPY [[ANYEXT2]](s32)
+  ; CHECK:   $vgpr3 = COPY [[ANYEXT3]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+  ; CHECK:   S_SETPC_B64_return [[COPY1]], implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+  %ptr = load volatile <4  x i8> addrspace(1)*, <4  x i8> addrspace(1)* addrspace(4)* undef
+  %val = load <4  x i8>, <4  x i8> addrspace(1)* %ptr
+  ret <4  x i8> %val
+}
+
+define {i8, i32} @struct_i8_i32_func_void() #0 {
+  ; CHECK-LABEL: name: struct_i8_i32_func_void
+  ; CHECK: bb.0:
+  ; CHECK:   successors: %bb.1(0x80000000)
+  ; CHECK:   liveins: $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(s8) = G_LOAD [[DEF]](p1) :: (load 1 from `{ i8, i32 } addrspace(1)* undef`, align 4, addrspace 1)
+  ; CHECK:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
+  ; CHECK:   [[GEP:%[0-9]+]]:_(p1) = G_PTR_ADD [[DEF]], [[C]](s64)
+  ; CHECK:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[GEP]](p1) :: (load 4 from `{ i8, i32 } addrspace(1)* undef` + 4, addrspace 1)
+  %val = load { i8, i32 }, { i8, i32 } addrspace(1)* undef
+  ret { i8, i32 } %val
+}
+
+define void @void_func_sret_struct_i8_i32({ i8, i32 } addrspace(5)* sret %arg0) #0 {
+  ; CHECK-LABEL: name: void_func_sret_struct_i8_i32
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[DEF1:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(s8) = G_LOAD [[DEF]](p1) :: (volatile load 1 from `i8 addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[DEF1]](p1) :: (volatile load 4 from `i32 addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+  ; CHECK:   [[GEP:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
+  ; CHECK:   G_STORE [[LOAD]](s8), [[COPY]](p5) :: (store 1 into %ir.gep01, addrspace 5)
+  ; CHECK:   G_STORE [[LOAD1]](s32), [[GEP]](p5) :: (store 4 into %ir.gep1, addrspace 5)
+  ; CHECK:   [[COPY2:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY1]]
+  ; CHECK:   S_SETPC_B64_return [[COPY2]]
+  %val0 = load volatile i8, i8 addrspace(1)* undef
+  %val1 = load volatile i32, i32 addrspace(1)* undef
+  %gep0 = getelementptr inbounds { i8, i32 }, { i8, i32 } addrspace(5)* %arg0, i32 0, i32 0
+  %gep1 = getelementptr inbounds { i8, i32 }, { i8, i32 } addrspace(5)* %arg0, i32 0, i32 1
+  store i8 %val0, i8 addrspace(5)* %gep0
+  store i32 %val1, i32 addrspace(5)* %gep1
+  ret void
+}
+
+; FIXME: Should be able to fold offsets in all of these pre-gfx9. Call
+; lowering introduces an extra CopyToReg/CopyFromReg obscuring the
+; AssertZext inserted. Not using it introduces the spills.
+
+define <33 x i32> @v33i32_func_void() #0 {
+  ; CHECK-LABEL: name: v33i32_func_void
+  ; CHECK: bb.0:
+  ; CHECK:   successors: %bb.1(0x80000000)
+  ; CHECK:   liveins: $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p4) = G_IMPLICIT_DEF
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[DEF]](p4) :: (volatile load 8 from `<33 x i32> addrspace(1)* addrspace(4)* undef`, addrspace 4)
+  ; CHECK:   [[LOAD1:%[0-9]+]]:_(<33 x s32>) = G_LOAD [[LOAD]](p1) :: (load 132 from %ir.ptr, align 256, addrspace 1)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32), [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32), [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32), [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32), [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32), [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32), [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32), [[UV18:%[0-9]+]]:_(s32), [[UV19:%[0-9]+]]:_(s32), [[UV20:%[0-9]+]]:_(s32), [[UV21:%[0-9]+]]:_(s32), [[UV22:%[0-9]+]]:_(s32), [[UV23:%[0-9]+]]:_(s32), [[UV24:%[0-9]+]]:_(s32), [[UV25:%[0-9]+]]:_(s32), [[UV26:%[0-9]+]]:_(s32), [[UV27:%[0-9]+]]:_(s32), [[UV28:%[0-9]+]]:_(s32), [[UV29:%[0-9]+]]:_(s32), [[UV30:%[0-9]+]]:_(s32), [[UV31:%[0-9]+]]:_(s32), [[UV32:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[LOAD1]](<33 x s32>)
+  %ptr = load volatile <33 x i32> addrspace(1)*, <33 x i32> addrspace(1)* addrspace(4)* undef
+  %val = load <33 x i32>, <33 x i32> addrspace(1)* %ptr
+  ret <33 x i32> %val
+}
+
+define { <32 x i32>, i32 } @struct_v32i32_i32_func_void() #0 {
+  ; CHECK-LABEL: name: struct_v32i32_i32_func_void
+  ; CHECK: bb.0:
+  ; CHECK:   successors: %bb.1(0x80000000)
+  ; CHECK:   liveins: $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p4) = G_IMPLICIT_DEF
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[DEF]](p4) :: (volatile load 8 from `{ <32 x i32>, i32 } addrspace(1)* addrspace(4)* undef`, addrspace 4)
+  ; CHECK:   [[LOAD1:%[0-9]+]]:_(<32 x s32>) = G_LOAD [[LOAD]](p1) :: (load 128 from %ir.ptr, addrspace 1)
+  ; CHECK:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 128
+  ; CHECK:   [[GEP:%[0-9]+]]:_(p1) = G_PTR_ADD [[LOAD]], [[C]](s64)
+  ; CHECK:   [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[GEP]](p1) :: (load 4 from %ir.ptr + 128, align 128, addrspace 1)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32), [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32), [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32), [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32), [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32), [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32), [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32), [[UV18:%[0-9]+]]:_(s32), [[UV19:%[0-9]+]]:_(s32), [[UV20:%[0-9]+]]:_(s32), [[UV21:%[0-9]+]]:_(s32), [[UV22:%[0-9]+]]:_(s32), [[UV23:%[0-9]+]]:_(s32), [[UV24:%[0-9]+]]:_(s32), [[UV25:%[0-9]+]]:_(s32), [[UV26:%[0-9]+]]:_(s32), [[UV27:%[0-9]+]]:_(s32), [[UV28:%[0-9]+]]:_(s32), [[UV29:%[0-9]+]]:_(s32), [[UV30:%[0-9]+]]:_(s32), [[UV31:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[LOAD1]](<32 x s32>)
+  %ptr = load volatile { <32 x i32>, i32 } addrspace(1)*, { <32 x i32>, i32 } addrspace(1)* addrspace(4)* undef
+  %val = load { <32 x i32>, i32 }, { <32 x i32>, i32 } addrspace(1)* %ptr
+  ret { <32 x i32>, i32 }%val
+}
+
+define { i32, <32 x i32> } @struct_i32_v32i32_func_void() #0 {
+  ; CHECK-LABEL: name: struct_i32_v32i32_func_void
+  ; CHECK: bb.0:
+  ; CHECK:   successors: %bb.1(0x80000000)
+  ; CHECK:   liveins: $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p4) = G_IMPLICIT_DEF
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[DEF]](p4) :: (volatile load 8 from `{ i32, <32 x i32> } addrspace(1)* addrspace(4)* undef`, addrspace 4)
+  ; CHECK:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[LOAD]](p1) :: (load 4 from %ir.ptr, align 128, addrspace 1)
+  ; CHECK:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 128
+  ; CHECK:   [[GEP:%[0-9]+]]:_(p1) = G_PTR_ADD [[LOAD]], [[C]](s64)
+  ; CHECK:   [[LOAD2:%[0-9]+]]:_(<32 x s32>) = G_LOAD [[GEP]](p1) :: (load 128 from %ir.ptr + 128, addrspace 1)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32), [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32), [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32), [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32), [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32), [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32), [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32), [[UV18:%[0-9]+]]:_(s32), [[UV19:%[0-9]+]]:_(s32), [[UV20:%[0-9]+]]:_(s32), [[UV21:%[0-9]+]]:_(s32), [[UV22:%[0-9]+]]:_(s32), [[UV23:%[0-9]+]]:_(s32), [[UV24:%[0-9]+]]:_(s32), [[UV25:%[0-9]+]]:_(s32), [[UV26:%[0-9]+]]:_(s32), [[UV27:%[0-9]+]]:_(s32), [[UV28:%[0-9]+]]:_(s32), [[UV29:%[0-9]+]]:_(s32), [[UV30:%[0-9]+]]:_(s32), [[UV31:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[LOAD2]](<32 x s32>)
+  %ptr = load volatile { i32, <32 x i32> } addrspace(1)*, { i32, <32 x i32> } addrspace(1)* addrspace(4)* undef
+  %val = load { i32, <32 x i32> }, { i32, <32 x i32> } addrspace(1)* %ptr
+  ret { i32, <32 x i32> }%val
+}
+
+; Make sure the last struct component is returned in v3, not v4.
+define { <3 x i32>, i32 } @v3i32_struct_func_void_wasted_reg() #0 {
+  ; CHECK-LABEL: name: v3i32_struct_func_void_wasted_reg
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p3) = G_IMPLICIT_DEF
+  ; CHECK:   [[DEF1:%[0-9]+]]:_(<3 x s32>) = G_IMPLICIT_DEF
+  ; CHECK:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; CHECK:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; CHECK:   [[DEF2:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[DEF]](p3) :: (volatile load 4 from `i32 addrspace(3)* undef`, addrspace 3)
+  ; CHECK:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[DEF]](p3) :: (volatile load 4 from `i32 addrspace(3)* undef`, addrspace 3)
+  ; CHECK:   [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[DEF]](p3) :: (volatile load 4 from `i32 addrspace(3)* undef`, addrspace 3)
+  ; CHECK:   [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[DEF]](p3) :: (volatile load 4 from `i32 addrspace(3)* undef`, addrspace 3)
+  ; CHECK:   [[IVEC:%[0-9]+]]:_(<3 x s32>) = G_INSERT_VECTOR_ELT [[DEF1]], [[LOAD]](s32), [[C]](s32)
+  ; CHECK:   [[IVEC1:%[0-9]+]]:_(<3 x s32>) = G_INSERT_VECTOR_ELT [[IVEC]], [[LOAD1]](s32), [[C1]](s32)
+  ; CHECK:   [[IVEC2:%[0-9]+]]:_(<3 x s32>) = G_INSERT_VECTOR_ELT [[IVEC1]], [[LOAD2]](s32), [[C2]](s32)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[IVEC2]](<3 x s32>)
+  ; CHECK:   $vgpr0 = COPY [[UV]](s32)
+  ; CHECK:   $vgpr1 = COPY [[UV1]](s32)
+  ; CHECK:   $vgpr2 = COPY [[UV2]](s32)
+  ; CHECK:   $vgpr3 = COPY [[LOAD3]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+  ; CHECK:   S_SETPC_B64_return [[COPY1]], implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+  %load0 = load volatile i32, i32 addrspace(3)* undef
+  %load1 = load volatile i32, i32 addrspace(3)* undef
+  %load2 = load volatile i32, i32 addrspace(3)* undef
+  %load3 = load volatile i32, i32 addrspace(3)* undef
+
+  %insert.0 = insertelement <3 x i32> undef, i32 %load0, i32 0
+  %insert.1 = insertelement <3 x i32> %insert.0, i32 %load1, i32 1
+  %insert.2 = insertelement <3 x i32> %insert.1, i32 %load2, i32 2
+  %insert.3 = insertvalue { <3 x i32>, i32 } undef, <3 x i32> %insert.2, 0
+  %insert.4 = insertvalue { <3 x i32>, i32 } %insert.3, i32 %load3, 1
+  ret { <3 x i32>, i32 } %insert.4
+}
+
+define { <3 x float>, i32 } @v3f32_struct_func_void_wasted_reg() #0 {
+  ; CHECK-LABEL: name: v3f32_struct_func_void_wasted_reg
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p3) = G_IMPLICIT_DEF
+  ; CHECK:   [[DEF1:%[0-9]+]]:_(p3) = G_IMPLICIT_DEF
+  ; CHECK:   [[DEF2:%[0-9]+]]:_(<3 x s32>) = G_IMPLICIT_DEF
+  ; CHECK:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; CHECK:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; CHECK:   [[DEF3:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[DEF]](p3) :: (volatile load 4 from `float addrspace(3)* undef`, addrspace 3)
+  ; CHECK:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[DEF]](p3) :: (volatile load 4 from `float addrspace(3)* undef`, addrspace 3)
+  ; CHECK:   [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[DEF]](p3) :: (volatile load 4 from `float addrspace(3)* undef`, addrspace 3)
+  ; CHECK:   [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[DEF1]](p3) :: (volatile load 4 from `i32 addrspace(3)* undef`, addrspace 3)
+  ; CHECK:   [[IVEC:%[0-9]+]]:_(<3 x s32>) = G_INSERT_VECTOR_ELT [[DEF2]], [[LOAD]](s32), [[C]](s32)
+  ; CHECK:   [[IVEC1:%[0-9]+]]:_(<3 x s32>) = G_INSERT_VECTOR_ELT [[IVEC]], [[LOAD1]](s32), [[C1]](s32)
+  ; CHECK:   [[IVEC2:%[0-9]+]]:_(<3 x s32>) = G_INSERT_VECTOR_ELT [[IVEC1]], [[LOAD2]](s32), [[C2]](s32)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[IVEC2]](<3 x s32>)
+  ; CHECK:   $vgpr0 = COPY [[UV]](s32)
+  ; CHECK:   $vgpr1 = COPY [[UV1]](s32)
+  ; CHECK:   $vgpr2 = COPY [[UV2]](s32)
+  ; CHECK:   $vgpr3 = COPY [[LOAD3]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+  ; CHECK:   S_SETPC_B64_return [[COPY1]], implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+  %load0 = load volatile float, float addrspace(3)* undef
+  %load1 = load volatile float, float addrspace(3)* undef
+  %load2 = load volatile float, float addrspace(3)* undef
+  %load3 = load volatile i32, i32 addrspace(3)* undef
+
+  %insert.0 = insertelement <3 x float> undef, float %load0, i32 0
+  %insert.1 = insertelement <3 x float> %insert.0, float %load1, i32 1
+  %insert.2 = insertelement <3 x float> %insert.1, float %load2, i32 2
+  %insert.3 = insertvalue { <3 x float>, i32 } undef, <3 x float> %insert.2, 0
+  %insert.4 = insertvalue { <3 x float>, i32 } %insert.3, i32 %load3, 1
+  ret { <3 x float>, i32 } %insert.4
+}
+
+define void @void_func_sret_max_known_zero_bits(i8 addrspace(5)* sret %arg0) #0 {
+  ; CHECK-LABEL: name: void_func_sret_max_known_zero_bits
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; CHECK:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 17
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 18
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p3) = G_IMPLICIT_DEF
+  ; CHECK:   [[PTRTOINT:%[0-9]+]]:_(s32) = G_PTRTOINT [[COPY]](p5)
+  ; CHECK:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[PTRTOINT]], [[C]](s32)
+  ; CHECK:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[PTRTOINT]], [[C1]](s32)
+  ; CHECK:   [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[PTRTOINT]], [[C2]](s32)
+  ; CHECK:   G_STORE [[LSHR]](s32), [[DEF]](p3) :: (volatile store 4 into `i32 addrspace(3)* undef`, addrspace 3)
+  ; CHECK:   G_STORE [[LSHR1]](s32), [[DEF]](p3) :: (volatile store 4 into `i32 addrspace(3)* undef`, addrspace 3)
+  ; CHECK:   G_STORE [[LSHR2]](s32), [[DEF]](p3) :: (volatile store 4 into `i32 addrspace(3)* undef`, addrspace 3)
+  ; CHECK:   [[COPY2:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY1]]
+  ; CHECK:   S_SETPC_B64_return [[COPY2]]
+  %arg0.int = ptrtoint i8 addrspace(5)* %arg0 to i32
+
+  %lshr0 = lshr i32 %arg0.int, 16
+  %lshr1 = lshr i32 %arg0.int, 17
+  %lshr2 = lshr i32 %arg0.int, 18
+
+  store volatile i32 %lshr0, i32 addrspace(3)* undef
+  store volatile i32 %lshr1, i32 addrspace(3)* undef
+  store volatile i32 %lshr2, i32 addrspace(3)* undef
+  ret void
+}
+
+attributes #0 = { nounwind }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/global-value.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/global-value.ll
new file mode 100644
index 000000000..fac8511c0
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/global-value.ll
@@ -0,0 +1,156 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=hawaii -stop-after=legalizer < %s | FileCheck -check-prefix=HSA %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=hawaii -stop-after=legalizer < %s | FileCheck -check-prefix=PAL %s
+
+@external_constant = external addrspace(4) constant i32, align 4
+@external_constant32 = external addrspace(6) constant i32, align 4
+@external_global = external addrspace(1) global i32, align 4
+
+@internal_constant = internal addrspace(4) constant i32 9, align 4
+@internal_constant32 = internal addrspace(6) constant i32 9, align 4
+@internal_global = internal addrspace(1) global i32 9, align 4
+
+
+define i32 addrspace(4)* @external_constant_got() {
+  ; HSA-LABEL: name: external_constant_got
+  ; HSA: bb.1 (%ir-block.0):
+  ; HSA:   liveins: $sgpr30_sgpr31
+  ; HSA:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; HSA:   [[SI_PC_ADD_REL_OFFSET:%[0-9]+]]:sreg_64(p4) = SI_PC_ADD_REL_OFFSET target-flags(amdgpu-gotprel32-lo) @external_constant + 4, target-flags(amdgpu-gotprel32-hi) @external_constant + 4, implicit-def $scc
+  ; HSA:   [[LOAD:%[0-9]+]]:_(p4) = G_LOAD [[SI_PC_ADD_REL_OFFSET]](p4) :: (dereferenceable invariant load 8 from got, addrspace 4)
+  ; HSA:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[LOAD]](p4)
+  ; HSA:   $vgpr0 = COPY [[UV]](s32)
+  ; HSA:   $vgpr1 = COPY [[UV1]](s32)
+  ; HSA:   [[COPY1:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+  ; HSA:   S_SETPC_B64_return [[COPY1]], implicit $vgpr0, implicit $vgpr1
+  ; PAL-LABEL: name: external_constant_got
+  ; PAL: bb.1 (%ir-block.0):
+  ; PAL:   liveins: $sgpr30_sgpr31
+  ; PAL:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; PAL:   [[SI_PC_ADD_REL_OFFSET:%[0-9]+]]:sreg_64(p4) = SI_PC_ADD_REL_OFFSET @external_constant + 4, 0, implicit-def $scc
+  ; PAL:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[SI_PC_ADD_REL_OFFSET]](p4)
+  ; PAL:   $vgpr0 = COPY [[UV]](s32)
+  ; PAL:   $vgpr1 = COPY [[UV1]](s32)
+  ; PAL:   [[COPY1:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+  ; PAL:   S_SETPC_B64_return [[COPY1]], implicit $vgpr0, implicit $vgpr1
+  ret i32 addrspace(4)* @external_constant
+}
+
+define i32 addrspace(1)* @external_global_got() {
+  ; HSA-LABEL: name: external_global_got
+  ; HSA: bb.1 (%ir-block.0):
+  ; HSA:   liveins: $sgpr30_sgpr31
+  ; HSA:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; HSA:   [[SI_PC_ADD_REL_OFFSET:%[0-9]+]]:sreg_64(p4) = SI_PC_ADD_REL_OFFSET target-flags(amdgpu-gotprel32-lo) @external_global + 4, target-flags(amdgpu-gotprel32-hi) @external_global + 4, implicit-def $scc
+  ; HSA:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[SI_PC_ADD_REL_OFFSET]](p4) :: (dereferenceable invariant load 8 from got, addrspace 4)
+  ; HSA:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[LOAD]](p1)
+  ; HSA:   $vgpr0 = COPY [[UV]](s32)
+  ; HSA:   $vgpr1 = COPY [[UV1]](s32)
+  ; HSA:   [[COPY1:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+  ; HSA:   S_SETPC_B64_return [[COPY1]], implicit $vgpr0, implicit $vgpr1
+  ; PAL-LABEL: name: external_global_got
+  ; PAL: bb.1 (%ir-block.0):
+  ; PAL:   liveins: $sgpr30_sgpr31
+  ; PAL:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; PAL:   [[SI_PC_ADD_REL_OFFSET:%[0-9]+]]:sreg_64(p4) = SI_PC_ADD_REL_OFFSET target-flags(amdgpu-gotprel32-lo) @external_global + 4, target-flags(amdgpu-gotprel32-hi) @external_global + 4, implicit-def $scc
+  ; PAL:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[SI_PC_ADD_REL_OFFSET]](p4) :: (dereferenceable invariant load 8 from got, addrspace 4)
+  ; PAL:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[LOAD]](p1)
+  ; PAL:   $vgpr0 = COPY [[UV]](s32)
+  ; PAL:   $vgpr1 = COPY [[UV1]](s32)
+  ; PAL:   [[COPY1:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+  ; PAL:   S_SETPC_B64_return [[COPY1]], implicit $vgpr0, implicit $vgpr1
+  ret i32 addrspace(1)* @external_global
+}
+
+define i32 addrspace(4)* @internal_constant_pcrel() {
+  ; HSA-LABEL: name: internal_constant_pcrel
+  ; HSA: bb.1 (%ir-block.0):
+  ; HSA:   liveins: $sgpr30_sgpr31
+  ; HSA:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; HSA:   [[SI_PC_ADD_REL_OFFSET:%[0-9]+]]:sreg_64(p4) = SI_PC_ADD_REL_OFFSET target-flags(amdgpu-rel32-lo) @internal_constant + 4, target-flags(amdgpu-rel32-hi) @internal_constant + 4, implicit-def $scc
+  ; HSA:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[SI_PC_ADD_REL_OFFSET]](p4)
+  ; HSA:   $vgpr0 = COPY [[UV]](s32)
+  ; HSA:   $vgpr1 = COPY [[UV1]](s32)
+  ; HSA:   [[COPY1:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+  ; HSA:   S_SETPC_B64_return [[COPY1]], implicit $vgpr0, implicit $vgpr1
+  ; PAL-LABEL: name: internal_constant_pcrel
+  ; PAL: bb.1 (%ir-block.0):
+  ; PAL:   liveins: $sgpr30_sgpr31
+  ; PAL:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; PAL:   [[SI_PC_ADD_REL_OFFSET:%[0-9]+]]:sreg_64(p4) = SI_PC_ADD_REL_OFFSET @internal_constant + 4, 0, implicit-def $scc
+  ; PAL:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[SI_PC_ADD_REL_OFFSET]](p4)
+  ; PAL:   $vgpr0 = COPY [[UV]](s32)
+  ; PAL:   $vgpr1 = COPY [[UV1]](s32)
+  ; PAL:   [[COPY1:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+  ; PAL:   S_SETPC_B64_return [[COPY1]], implicit $vgpr0, implicit $vgpr1
+  ret i32 addrspace(4)* @internal_constant
+}
+
+define i32 addrspace(1)* @internal_global_pcrel() {
+  ; HSA-LABEL: name: internal_global_pcrel
+  ; HSA: bb.1 (%ir-block.0):
+  ; HSA:   liveins: $sgpr30_sgpr31
+  ; HSA:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; HSA:   [[SI_PC_ADD_REL_OFFSET:%[0-9]+]]:sreg_64(p1) = SI_PC_ADD_REL_OFFSET target-flags(amdgpu-rel32-lo) @internal_global + 4, target-flags(amdgpu-rel32-hi) @internal_global + 4, implicit-def $scc
+  ; HSA:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[SI_PC_ADD_REL_OFFSET]](p1)
+  ; HSA:   $vgpr0 = COPY [[UV]](s32)
+  ; HSA:   $vgpr1 = COPY [[UV1]](s32)
+  ; HSA:   [[COPY1:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+  ; HSA:   S_SETPC_B64_return [[COPY1]], implicit $vgpr0, implicit $vgpr1
+  ; PAL-LABEL: name: internal_global_pcrel
+  ; PAL: bb.1 (%ir-block.0):
+  ; PAL:   liveins: $sgpr30_sgpr31
+  ; PAL:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; PAL:   [[SI_PC_ADD_REL_OFFSET:%[0-9]+]]:sreg_64(p1) = SI_PC_ADD_REL_OFFSET target-flags(amdgpu-rel32-lo) @internal_global + 4, target-flags(amdgpu-rel32-hi) @internal_global + 4, implicit-def $scc
+  ; PAL:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[SI_PC_ADD_REL_OFFSET]](p1)
+  ; PAL:   $vgpr0 = COPY [[UV]](s32)
+  ; PAL:   $vgpr1 = COPY [[UV1]](s32)
+  ; PAL:   [[COPY1:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+  ; PAL:   S_SETPC_B64_return [[COPY1]], implicit $vgpr0, implicit $vgpr1
+  ret i32 addrspace(1)* @internal_global
+}
+
+define i32 addrspace(6)* @external_constant32_got() {
+  ; HSA-LABEL: name: external_constant32_got
+  ; HSA: bb.1 (%ir-block.0):
+  ; HSA:   liveins: $sgpr30_sgpr31
+  ; HSA:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; HSA:   [[SI_PC_ADD_REL_OFFSET:%[0-9]+]]:sreg_64(p4) = SI_PC_ADD_REL_OFFSET target-flags(amdgpu-gotprel32-lo) @external_constant32 + 4, target-flags(amdgpu-gotprel32-hi) @external_constant32 + 4, implicit-def $scc
+  ; HSA:   [[LOAD:%[0-9]+]]:_(p4) = G_LOAD [[SI_PC_ADD_REL_OFFSET]](p4) :: (dereferenceable invariant load 8 from got, addrspace 4)
+  ; HSA:   [[EXTRACT:%[0-9]+]]:_(p6) = G_EXTRACT [[LOAD]](p4), 0
+  ; HSA:   $vgpr0 = COPY [[EXTRACT]](p6)
+  ; HSA:   [[COPY1:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+  ; HSA:   S_SETPC_B64_return [[COPY1]], implicit $vgpr0
+  ; PAL-LABEL: name: external_constant32_got
+  ; PAL: bb.1 (%ir-block.0):
+  ; PAL:   liveins: $sgpr30_sgpr31
+  ; PAL:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; PAL:   [[SI_PC_ADD_REL_OFFSET:%[0-9]+]]:sreg_64(p4) = SI_PC_ADD_REL_OFFSET @external_constant32 + 4, 0, implicit-def $scc
+  ; PAL:   [[EXTRACT:%[0-9]+]]:_(p6) = G_EXTRACT [[SI_PC_ADD_REL_OFFSET]](p4), 0
+  ; PAL:   $vgpr0 = COPY [[EXTRACT]](p6)
+  ; PAL:   [[COPY1:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+  ; PAL:   S_SETPC_B64_return [[COPY1]], implicit $vgpr0
+  ret i32 addrspace(6)* @external_constant32
+}
+
+define i32 addrspace(6)* @internal_constant32_pcrel() {
+  ; HSA-LABEL: name: internal_constant32_pcrel
+  ; HSA: bb.1 (%ir-block.0):
+  ; HSA:   liveins: $sgpr30_sgpr31
+  ; HSA:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; HSA:   [[SI_PC_ADD_REL_OFFSET:%[0-9]+]]:sreg_64(p4) = SI_PC_ADD_REL_OFFSET target-flags(amdgpu-rel32-lo) @internal_constant32 + 4, target-flags(amdgpu-rel32-hi) @internal_constant32 + 4, implicit-def $scc
+  ; HSA:   [[EXTRACT:%[0-9]+]]:_(p6) = G_EXTRACT [[SI_PC_ADD_REL_OFFSET]](p4), 0
+  ; HSA:   $vgpr0 = COPY [[EXTRACT]](p6)
+  ; HSA:   [[COPY1:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+  ; HSA:   S_SETPC_B64_return [[COPY1]], implicit $vgpr0
+  ; PAL-LABEL: name: internal_constant32_pcrel
+  ; PAL: bb.1 (%ir-block.0):
+  ; PAL:   liveins: $sgpr30_sgpr31
+  ; PAL:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; PAL:   [[SI_PC_ADD_REL_OFFSET:%[0-9]+]]:sreg_64(p4) = SI_PC_ADD_REL_OFFSET @internal_constant32 + 4, 0, implicit-def $scc
+  ; PAL:   [[EXTRACT:%[0-9]+]]:_(p6) = G_EXTRACT [[SI_PC_ADD_REL_OFFSET]](p4), 0
+  ; PAL:   $vgpr0 = COPY [[EXTRACT]](p6)
+  ; PAL:   [[COPY1:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+  ; PAL:   S_SETPC_B64_return [[COPY1]], implicit $vgpr0
+  ret i32 addrspace(6)* @internal_constant32
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgcn-sendmsg.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgcn-sendmsg.ll
new file mode 100644
index 000000000..0934375dc
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgcn-sendmsg.ll
@@ -0,0 +1,15 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; RUN: llc -march=amdgcn -O0 -stop-after=irtranslator -global-isel -verify-machineinstrs %s -o - | FileCheck %s
+
+declare void @llvm.amdgcn.s.sendmsg(i32 immarg, i32)
+
+define amdgpu_ps void @test_sendmsg(i32 inreg %m0) {
+  ; CHECK-LABEL: name: test_sendmsg
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr0
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr0
+  ; CHECK:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.s.sendmsg), 12, [[COPY]](s32)
+  ; CHECK:   S_ENDPGM
+  call void @llvm.amdgcn.s.sendmsg(i32 12, i32 %m0)
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgpu_kernel.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgpu_kernel.ll
index 85102491a..4d405c682 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgpu_kernel.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgpu_kernel.ll
@@ -7,14 +7,14 @@ define amdgpu_kernel void @i8_arg(i32 addrspace(1)* nocapture %out, i8 %in) noun
   ; HSA-VI:   liveins: $sgpr4_sgpr5
   ; HSA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr4_sgpr5
   ; HSA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C]](s64)
-  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (non-temporal invariant load 8 from `i32 addrspace(1)* addrspace(4)* undef`, align 16, addrspace 4)
+  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (dereferenceable invariant load 8, align 16, addrspace 4)
   ; HSA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C1]](s64)
-  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(s8) = G_LOAD [[GEP1]](p4) :: (non-temporal invariant load 1 from `i8 addrspace(4)* undef`, align 8, addrspace 4)
+  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(s8) = G_LOAD [[GEP1]](p4) :: (dereferenceable invariant load 1, align 8, addrspace 4)
   ; HSA-VI:   [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[LOAD1]](s8)
   ; HSA-VI:   G_STORE [[ZEXT]](s32), [[LOAD]](p1) :: (store 4 into %ir.out, addrspace 1)
-  ; HSA-VI:   S_ENDPGM
+  ; HSA-VI:   S_ENDPGM 0
   %ext = zext i8 %in to i32
   store i32 %ext, i32 addrspace(1)* %out, align 4
   ret void
@@ -26,14 +26,14 @@ define amdgpu_kernel void @i8_zext_arg(i32 addrspace(1)* nocapture %out, i8 zero
   ; HSA-VI:   liveins: $sgpr4_sgpr5
   ; HSA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr4_sgpr5
   ; HSA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C]](s64)
-  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (non-temporal invariant load 8 from `i32 addrspace(1)* addrspace(4)* undef`, align 16, addrspace 4)
+  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (dereferenceable invariant load 8, align 16, addrspace 4)
   ; HSA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C1]](s64)
-  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(s8) = G_LOAD [[GEP1]](p4) :: (non-temporal invariant load 1 from `i8 addrspace(4)* undef`, align 8, addrspace 4)
+  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(s8) = G_LOAD [[GEP1]](p4) :: (dereferenceable invariant load 1, align 8, addrspace 4)
   ; HSA-VI:   [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[LOAD1]](s8)
   ; HSA-VI:   G_STORE [[ZEXT]](s32), [[LOAD]](p1) :: (store 4 into %ir.out, addrspace 1)
-  ; HSA-VI:   S_ENDPGM
+  ; HSA-VI:   S_ENDPGM 0
   %ext = zext i8 %in to i32
   store i32 %ext, i32 addrspace(1)* %out, align 4
   ret void
@@ -45,14 +45,14 @@ define amdgpu_kernel void @i8_sext_arg(i32 addrspace(1)* nocapture %out, i8 sign
   ; HSA-VI:   liveins: $sgpr4_sgpr5
   ; HSA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr4_sgpr5
   ; HSA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C]](s64)
-  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (non-temporal invariant load 8 from `i32 addrspace(1)* addrspace(4)* undef`, align 16, addrspace 4)
+  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (dereferenceable invariant load 8, align 16, addrspace 4)
   ; HSA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C1]](s64)
-  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(s8) = G_LOAD [[GEP1]](p4) :: (non-temporal invariant load 1 from `i8 addrspace(4)* undef`, align 8, addrspace 4)
+  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(s8) = G_LOAD [[GEP1]](p4) :: (dereferenceable invariant load 1, align 8, addrspace 4)
   ; HSA-VI:   [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[LOAD1]](s8)
   ; HSA-VI:   G_STORE [[SEXT]](s32), [[LOAD]](p1) :: (store 4 into %ir.out, addrspace 1)
-  ; HSA-VI:   S_ENDPGM
+  ; HSA-VI:   S_ENDPGM 0
   %ext = sext i8 %in to i32
   store i32 %ext, i32 addrspace(1)* %out, align 4
   ret void
@@ -64,14 +64,14 @@ define amdgpu_kernel void @i16_arg(i32 addrspace(1)* nocapture %out, i16 %in) no
   ; HSA-VI:   liveins: $sgpr4_sgpr5
   ; HSA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr4_sgpr5
   ; HSA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C]](s64)
-  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (non-temporal invariant load 8 from `i32 addrspace(1)* addrspace(4)* undef`, align 16, addrspace 4)
+  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (dereferenceable invariant load 8, align 16, addrspace 4)
   ; HSA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C1]](s64)
-  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(s16) = G_LOAD [[GEP1]](p4) :: (non-temporal invariant load 2 from `i16 addrspace(4)* undef`, align 8, addrspace 4)
+  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(s16) = G_LOAD [[GEP1]](p4) :: (dereferenceable invariant load 2, align 8, addrspace 4)
   ; HSA-VI:   [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[LOAD1]](s16)
   ; HSA-VI:   G_STORE [[ZEXT]](s32), [[LOAD]](p1) :: (store 4 into %ir.out, addrspace 1)
-  ; HSA-VI:   S_ENDPGM
+  ; HSA-VI:   S_ENDPGM 0
   %ext = zext i16 %in to i32
   store i32 %ext, i32 addrspace(1)* %out, align 4
   ret void
@@ -83,14 +83,14 @@ define amdgpu_kernel void @i16_zext_arg(i32 addrspace(1)* nocapture %out, i16 ze
   ; HSA-VI:   liveins: $sgpr4_sgpr5
   ; HSA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr4_sgpr5
   ; HSA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C]](s64)
-  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (non-temporal invariant load 8 from `i32 addrspace(1)* addrspace(4)* undef`, align 16, addrspace 4)
+  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (dereferenceable invariant load 8, align 16, addrspace 4)
   ; HSA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C1]](s64)
-  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(s16) = G_LOAD [[GEP1]](p4) :: (non-temporal invariant load 2 from `i16 addrspace(4)* undef`, align 8, addrspace 4)
+  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(s16) = G_LOAD [[GEP1]](p4) :: (dereferenceable invariant load 2, align 8, addrspace 4)
   ; HSA-VI:   [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[LOAD1]](s16)
   ; HSA-VI:   G_STORE [[ZEXT]](s32), [[LOAD]](p1) :: (store 4 into %ir.out, addrspace 1)
-  ; HSA-VI:   S_ENDPGM
+  ; HSA-VI:   S_ENDPGM 0
   %ext = zext i16 %in to i32
   store i32 %ext, i32 addrspace(1)* %out, align 4
   ret void
@@ -102,14 +102,14 @@ define amdgpu_kernel void @i16_sext_arg(i32 addrspace(1)* nocapture %out, i16 si
   ; HSA-VI:   liveins: $sgpr4_sgpr5
   ; HSA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr4_sgpr5
   ; HSA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C]](s64)
-  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (non-temporal invariant load 8 from `i32 addrspace(1)* addrspace(4)* undef`, align 16, addrspace 4)
+  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (dereferenceable invariant load 8, align 16, addrspace 4)
   ; HSA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C1]](s64)
-  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(s16) = G_LOAD [[GEP1]](p4) :: (non-temporal invariant load 2 from `i16 addrspace(4)* undef`, align 8, addrspace 4)
+  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(s16) = G_LOAD [[GEP1]](p4) :: (dereferenceable invariant load 2, align 8, addrspace 4)
   ; HSA-VI:   [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[LOAD1]](s16)
   ; HSA-VI:   G_STORE [[SEXT]](s32), [[LOAD]](p1) :: (store 4 into %ir.out, addrspace 1)
-  ; HSA-VI:   S_ENDPGM
+  ; HSA-VI:   S_ENDPGM 0
   %ext = sext i16 %in to i32
   store i32 %ext, i32 addrspace(1)* %out, align 4
   ret void
@@ -121,13 +121,13 @@ define amdgpu_kernel void @i32_arg(i32 addrspace(1)* nocapture %out, i32 %in) no
   ; HSA-VI:   liveins: $sgpr4_sgpr5
   ; HSA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr4_sgpr5
   ; HSA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C]](s64)
-  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (non-temporal invariant load 8 from `i32 addrspace(1)* addrspace(4)* undef`, align 16, addrspace 4)
+  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (dereferenceable invariant load 8, align 16, addrspace 4)
   ; HSA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C1]](s64)
-  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[GEP1]](p4) :: (non-temporal invariant load 4 from `i32 addrspace(4)* undef`, align 8, addrspace 4)
+  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[GEP1]](p4) :: (dereferenceable invariant load 4, align 8, addrspace 4)
   ; HSA-VI:   G_STORE [[LOAD1]](s32), [[LOAD]](p1) :: (store 4 into %ir.out, addrspace 1)
-  ; HSA-VI:   S_ENDPGM
+  ; HSA-VI:   S_ENDPGM 0
 entry:
   store i32 %in, i32 addrspace(1)* %out, align 4
   ret void
@@ -139,13 +139,13 @@ define amdgpu_kernel void @f32_arg(float addrspace(1)* nocapture %out, float %in
   ; HSA-VI:   liveins: $sgpr4_sgpr5
   ; HSA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr4_sgpr5
   ; HSA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C]](s64)
-  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (non-temporal invariant load 8 from `float addrspace(1)* addrspace(4)* undef`, align 16, addrspace 4)
+  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (dereferenceable invariant load 8, align 16, addrspace 4)
   ; HSA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C1]](s64)
-  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[GEP1]](p4) :: (non-temporal invariant load 4 from `float addrspace(4)* undef`, align 8, addrspace 4)
+  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[GEP1]](p4) :: (dereferenceable invariant load 4, align 8, addrspace 4)
   ; HSA-VI:   G_STORE [[LOAD1]](s32), [[LOAD]](p1) :: (store 4 into %ir.out, addrspace 1)
-  ; HSA-VI:   S_ENDPGM
+  ; HSA-VI:   S_ENDPGM 0
 entry:
   store float %in, float addrspace(1)* %out, align 4
   ret void
@@ -157,13 +157,13 @@ define amdgpu_kernel void @v2i8_arg(<2 x i8> addrspace(1)* %out, <2 x i8> %in) {
   ; HSA-VI:   liveins: $sgpr4_sgpr5
   ; HSA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr4_sgpr5
   ; HSA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C]](s64)
-  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (non-temporal invariant load 8 from `<2 x i8> addrspace(1)* addrspace(4)* undef`, align 16, addrspace 4)
+  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (dereferenceable invariant load 8, align 16, addrspace 4)
   ; HSA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C1]](s64)
-  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(<2 x s8>) = G_LOAD [[GEP1]](p4) :: (non-temporal invariant load 2 from `<2 x i8> addrspace(4)* undef`, align 8, addrspace 4)
+  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(<2 x s8>) = G_LOAD [[GEP1]](p4) :: (dereferenceable invariant load 2, align 8, addrspace 4)
   ; HSA-VI:   G_STORE [[LOAD1]](<2 x s8>), [[LOAD]](p1) :: (store 2 into %ir.out, addrspace 1)
-  ; HSA-VI:   S_ENDPGM
+  ; HSA-VI:   S_ENDPGM 0
 entry:
   store <2 x i8> %in, <2 x i8> addrspace(1)* %out
   ret void
@@ -175,13 +175,13 @@ define amdgpu_kernel void @v2i16_arg(<2 x i16> addrspace(1)* %out, <2 x i16> %in
   ; HSA-VI:   liveins: $sgpr4_sgpr5
   ; HSA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr4_sgpr5
   ; HSA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C]](s64)
-  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (non-temporal invariant load 8 from `<2 x i16> addrspace(1)* addrspace(4)* undef`, align 16, addrspace 4)
+  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (dereferenceable invariant load 8, align 16, addrspace 4)
   ; HSA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C1]](s64)
-  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(<2 x s16>) = G_LOAD [[GEP1]](p4) :: (non-temporal invariant load 4 from `<2 x i16> addrspace(4)* undef`, align 8, addrspace 4)
+  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(<2 x s16>) = G_LOAD [[GEP1]](p4) :: (dereferenceable invariant load 4, align 8, addrspace 4)
   ; HSA-VI:   G_STORE [[LOAD1]](<2 x s16>), [[LOAD]](p1) :: (store 4 into %ir.out, addrspace 1)
-  ; HSA-VI:   S_ENDPGM
+  ; HSA-VI:   S_ENDPGM 0
 entry:
   store <2 x i16> %in, <2 x i16> addrspace(1)* %out
   ret void
@@ -193,13 +193,13 @@ define amdgpu_kernel void @v2i32_arg(<2 x i32> addrspace(1)* nocapture %out, <2
   ; HSA-VI:   liveins: $sgpr4_sgpr5
   ; HSA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr4_sgpr5
   ; HSA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C]](s64)
-  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (non-temporal invariant load 8 from `<2 x i32> addrspace(1)* addrspace(4)* undef`, align 16, addrspace 4)
+  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (dereferenceable invariant load 8, align 16, addrspace 4)
   ; HSA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C1]](s64)
-  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[GEP1]](p4) :: (non-temporal invariant load 8 from `<2 x i32> addrspace(4)* undef`, addrspace 4)
+  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[GEP1]](p4) :: (dereferenceable invariant load 8, addrspace 4)
   ; HSA-VI:   G_STORE [[LOAD1]](<2 x s32>), [[LOAD]](p1) :: (store 8 into %ir.out, align 4, addrspace 1)
-  ; HSA-VI:   S_ENDPGM
+  ; HSA-VI:   S_ENDPGM 0
 entry:
   store <2 x i32> %in, <2 x i32> addrspace(1)* %out, align 4
   ret void
@@ -211,13 +211,13 @@ define amdgpu_kernel void @v2f32_arg(<2 x float> addrspace(1)* nocapture %out, <
   ; HSA-VI:   liveins: $sgpr4_sgpr5
   ; HSA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr4_sgpr5
   ; HSA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C]](s64)
-  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (non-temporal invariant load 8 from `<2 x float> addrspace(1)* addrspace(4)* undef`, align 16, addrspace 4)
+  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (dereferenceable invariant load 8, align 16, addrspace 4)
   ; HSA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C1]](s64)
-  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[GEP1]](p4) :: (non-temporal invariant load 8 from `<2 x float> addrspace(4)* undef`, addrspace 4)
+  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[GEP1]](p4) :: (dereferenceable invariant load 8, addrspace 4)
   ; HSA-VI:   G_STORE [[LOAD1]](<2 x s32>), [[LOAD]](p1) :: (store 8 into %ir.out, align 4, addrspace 1)
-  ; HSA-VI:   S_ENDPGM
+  ; HSA-VI:   S_ENDPGM 0
 entry:
   store <2 x float> %in, <2 x float> addrspace(1)* %out, align 4
   ret void
@@ -229,13 +229,13 @@ define amdgpu_kernel void @v3i8_arg(<3 x i8> addrspace(1)* nocapture %out, <3 x
   ; HSA-VI:   liveins: $sgpr4_sgpr5
   ; HSA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr4_sgpr5
   ; HSA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C]](s64)
-  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (non-temporal invariant load 8 from `<3 x i8> addrspace(1)* addrspace(4)* undef`, align 16, addrspace 4)
+  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (dereferenceable invariant load 8, align 16, addrspace 4)
   ; HSA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C1]](s64)
-  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(<3 x s8>) = G_LOAD [[GEP1]](p4) :: (non-temporal invariant load 3 from `<3 x i8> addrspace(4)* undef`, align 8, addrspace 4)
+  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(<3 x s8>) = G_LOAD [[GEP1]](p4) :: (dereferenceable invariant load 3, align 8, addrspace 4)
   ; HSA-VI:   G_STORE [[LOAD1]](<3 x s8>), [[LOAD]](p1) :: (store 3 into %ir.out, align 4, addrspace 1)
-  ; HSA-VI:   S_ENDPGM
+  ; HSA-VI:   S_ENDPGM 0
 entry:
   store <3 x i8> %in, <3 x i8> addrspace(1)* %out, align 4
   ret void
@@ -247,13 +247,13 @@ define amdgpu_kernel void @v3i16_arg(<3 x i16> addrspace(1)* nocapture %out, <3
   ; HSA-VI:   liveins: $sgpr4_sgpr5
   ; HSA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr4_sgpr5
   ; HSA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C]](s64)
-  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (non-temporal invariant load 8 from `<3 x i16> addrspace(1)* addrspace(4)* undef`, align 16, addrspace 4)
+  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (dereferenceable invariant load 8, align 16, addrspace 4)
   ; HSA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C1]](s64)
-  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(<3 x s16>) = G_LOAD [[GEP1]](p4) :: (non-temporal invariant load 6 from `<3 x i16> addrspace(4)* undef`, align 8, addrspace 4)
+  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(<3 x s16>) = G_LOAD [[GEP1]](p4) :: (dereferenceable invariant load 6, align 8, addrspace 4)
   ; HSA-VI:   G_STORE [[LOAD1]](<3 x s16>), [[LOAD]](p1) :: (store 6 into %ir.out, align 4, addrspace 1)
-  ; HSA-VI:   S_ENDPGM
+  ; HSA-VI:   S_ENDPGM 0
 entry:
   store <3 x i16> %in, <3 x i16> addrspace(1)* %out, align 4
   ret void
@@ -265,13 +265,13 @@ define amdgpu_kernel void @v3i32_arg(<3 x i32> addrspace(1)* nocapture %out, <3
   ; HSA-VI:   liveins: $sgpr4_sgpr5
   ; HSA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr4_sgpr5
   ; HSA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C]](s64)
-  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (non-temporal invariant load 8 from `<3 x i32> addrspace(1)* addrspace(4)* undef`, align 16, addrspace 4)
+  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (dereferenceable invariant load 8, align 16, addrspace 4)
   ; HSA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
-  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C1]](s64)
-  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(<3 x s32>) = G_LOAD [[GEP1]](p4) :: (non-temporal invariant load 12 from `<3 x i32> addrspace(4)* undef`, align 16, addrspace 4)
+  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(<3 x s32>) = G_LOAD [[GEP1]](p4) :: (dereferenceable invariant load 12, align 16, addrspace 4)
   ; HSA-VI:   G_STORE [[LOAD1]](<3 x s32>), [[LOAD]](p1) :: (store 12 into %ir.out, align 4, addrspace 1)
-  ; HSA-VI:   S_ENDPGM
+  ; HSA-VI:   S_ENDPGM 0
 entry:
   store <3 x i32> %in, <3 x i32> addrspace(1)* %out, align 4
   ret void
@@ -283,13 +283,13 @@ define amdgpu_kernel void @v3f32_arg(<3 x float> addrspace(1)* nocapture %out, <
   ; HSA-VI:   liveins: $sgpr4_sgpr5
   ; HSA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr4_sgpr5
   ; HSA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C]](s64)
-  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (non-temporal invariant load 8 from `<3 x float> addrspace(1)* addrspace(4)* undef`, align 16, addrspace 4)
+  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (dereferenceable invariant load 8, align 16, addrspace 4)
   ; HSA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
-  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C1]](s64)
-  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(<3 x s32>) = G_LOAD [[GEP1]](p4) :: (non-temporal invariant load 12 from `<3 x float> addrspace(4)* undef`, align 16, addrspace 4)
+  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(<3 x s32>) = G_LOAD [[GEP1]](p4) :: (dereferenceable invariant load 12, align 16, addrspace 4)
   ; HSA-VI:   G_STORE [[LOAD1]](<3 x s32>), [[LOAD]](p1) :: (store 12 into %ir.out, align 4, addrspace 1)
-  ; HSA-VI:   S_ENDPGM
+  ; HSA-VI:   S_ENDPGM 0
 entry:
   store <3 x float> %in, <3 x float> addrspace(1)* %out, align 4
   ret void
@@ -301,13 +301,13 @@ define amdgpu_kernel void @v4i8_arg(<4 x i8> addrspace(1)* %out, <4 x i8> %in) {
   ; HSA-VI:   liveins: $sgpr4_sgpr5
   ; HSA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr4_sgpr5
   ; HSA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C]](s64)
-  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (non-temporal invariant load 8 from `<4 x i8> addrspace(1)* addrspace(4)* undef`, align 16, addrspace 4)
+  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (dereferenceable invariant load 8, align 16, addrspace 4)
   ; HSA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C1]](s64)
-  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(<4 x s8>) = G_LOAD [[GEP1]](p4) :: (non-temporal invariant load 4 from `<4 x i8> addrspace(4)* undef`, align 8, addrspace 4)
+  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(<4 x s8>) = G_LOAD [[GEP1]](p4) :: (dereferenceable invariant load 4, align 8, addrspace 4)
   ; HSA-VI:   G_STORE [[LOAD1]](<4 x s8>), [[LOAD]](p1) :: (store 4 into %ir.out, addrspace 1)
-  ; HSA-VI:   S_ENDPGM
+  ; HSA-VI:   S_ENDPGM 0
 entry:
   store <4 x i8> %in, <4 x i8> addrspace(1)* %out
   ret void
@@ -319,13 +319,13 @@ define amdgpu_kernel void @v4i16_arg(<4 x i16> addrspace(1)* %out, <4 x i16> %in
   ; HSA-VI:   liveins: $sgpr4_sgpr5
   ; HSA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr4_sgpr5
   ; HSA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C]](s64)
-  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (non-temporal invariant load 8 from `<4 x i16> addrspace(1)* addrspace(4)* undef`, align 16, addrspace 4)
+  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (dereferenceable invariant load 8, align 16, addrspace 4)
   ; HSA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C1]](s64)
-  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(<4 x s16>) = G_LOAD [[GEP1]](p4) :: (non-temporal invariant load 8 from `<4 x i16> addrspace(4)* undef`, addrspace 4)
+  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(<4 x s16>) = G_LOAD [[GEP1]](p4) :: (dereferenceable invariant load 8, addrspace 4)
   ; HSA-VI:   G_STORE [[LOAD1]](<4 x s16>), [[LOAD]](p1) :: (store 8 into %ir.out, addrspace 1)
-  ; HSA-VI:   S_ENDPGM
+  ; HSA-VI:   S_ENDPGM 0
 entry:
   store <4 x i16> %in, <4 x i16> addrspace(1)* %out
   ret void
@@ -337,13 +337,13 @@ define amdgpu_kernel void @v4i32_arg(<4 x i32> addrspace(1)* nocapture %out, <4
   ; HSA-VI:   liveins: $sgpr4_sgpr5
   ; HSA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr4_sgpr5
   ; HSA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C]](s64)
-  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (non-temporal invariant load 8 from `<4 x i32> addrspace(1)* addrspace(4)* undef`, align 16, addrspace 4)
+  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (dereferenceable invariant load 8, align 16, addrspace 4)
   ; HSA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
-  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C1]](s64)
-  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(<4 x s32>) = G_LOAD [[GEP1]](p4) :: (non-temporal invariant load 16 from `<4 x i32> addrspace(4)* undef`, addrspace 4)
+  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(<4 x s32>) = G_LOAD [[GEP1]](p4) :: (dereferenceable invariant load 16, addrspace 4)
   ; HSA-VI:   G_STORE [[LOAD1]](<4 x s32>), [[LOAD]](p1) :: (store 16 into %ir.out, align 4, addrspace 1)
-  ; HSA-VI:   S_ENDPGM
+  ; HSA-VI:   S_ENDPGM 0
 entry:
   store <4 x i32> %in, <4 x i32> addrspace(1)* %out, align 4
   ret void
@@ -355,13 +355,13 @@ define amdgpu_kernel void @v4f32_arg(<4 x float> addrspace(1)* nocapture %out, <
   ; HSA-VI:   liveins: $sgpr4_sgpr5
   ; HSA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr4_sgpr5
   ; HSA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C]](s64)
-  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (non-temporal invariant load 8 from `<4 x float> addrspace(1)* addrspace(4)* undef`, align 16, addrspace 4)
+  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (dereferenceable invariant load 8, align 16, addrspace 4)
   ; HSA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
-  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C1]](s64)
-  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(<4 x s32>) = G_LOAD [[GEP1]](p4) :: (non-temporal invariant load 16 from `<4 x float> addrspace(4)* undef`, addrspace 4)
+  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(<4 x s32>) = G_LOAD [[GEP1]](p4) :: (dereferenceable invariant load 16, addrspace 4)
   ; HSA-VI:   G_STORE [[LOAD1]](<4 x s32>), [[LOAD]](p1) :: (store 16 into %ir.out, align 4, addrspace 1)
-  ; HSA-VI:   S_ENDPGM
+  ; HSA-VI:   S_ENDPGM 0
 entry:
   store <4 x float> %in, <4 x float> addrspace(1)* %out, align 4
   ret void
@@ -373,13 +373,13 @@ define amdgpu_kernel void @v8i8_arg(<8 x i8> addrspace(1)* %out, <8 x i8> %in) {
   ; HSA-VI:   liveins: $sgpr4_sgpr5
   ; HSA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr4_sgpr5
   ; HSA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C]](s64)
-  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (non-temporal invariant load 8 from `<8 x i8> addrspace(1)* addrspace(4)* undef`, align 16, addrspace 4)
+  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (dereferenceable invariant load 8, align 16, addrspace 4)
   ; HSA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C1]](s64)
-  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(<8 x s8>) = G_LOAD [[GEP1]](p4) :: (non-temporal invariant load 8 from `<8 x i8> addrspace(4)* undef`, addrspace 4)
+  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(<8 x s8>) = G_LOAD [[GEP1]](p4) :: (dereferenceable invariant load 8, addrspace 4)
   ; HSA-VI:   G_STORE [[LOAD1]](<8 x s8>), [[LOAD]](p1) :: (store 8 into %ir.out, addrspace 1)
-  ; HSA-VI:   S_ENDPGM
+  ; HSA-VI:   S_ENDPGM 0
 entry:
   store <8 x i8> %in, <8 x i8> addrspace(1)* %out
   ret void
@@ -391,13 +391,13 @@ define amdgpu_kernel void @v8i16_arg(<8 x i16> addrspace(1)* %out, <8 x i16> %in
   ; HSA-VI:   liveins: $sgpr4_sgpr5
   ; HSA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr4_sgpr5
   ; HSA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C]](s64)
-  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (non-temporal invariant load 8 from `<8 x i16> addrspace(1)* addrspace(4)* undef`, align 16, addrspace 4)
+  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (dereferenceable invariant load 8, align 16, addrspace 4)
   ; HSA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
-  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C1]](s64)
-  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(<8 x s16>) = G_LOAD [[GEP1]](p4) :: (non-temporal invariant load 16 from `<8 x i16> addrspace(4)* undef`, addrspace 4)
+  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(<8 x s16>) = G_LOAD [[GEP1]](p4) :: (dereferenceable invariant load 16, addrspace 4)
   ; HSA-VI:   G_STORE [[LOAD1]](<8 x s16>), [[LOAD]](p1) :: (store 16 into %ir.out, addrspace 1)
-  ; HSA-VI:   S_ENDPGM
+  ; HSA-VI:   S_ENDPGM 0
 entry:
   store <8 x i16> %in, <8 x i16> addrspace(1)* %out
   ret void
@@ -409,13 +409,13 @@ define amdgpu_kernel void @v8i32_arg(<8 x i32> addrspace(1)* nocapture %out, <8
   ; HSA-VI:   liveins: $sgpr4_sgpr5
   ; HSA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr4_sgpr5
   ; HSA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C]](s64)
-  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (non-temporal invariant load 8 from `<8 x i32> addrspace(1)* addrspace(4)* undef`, align 16, addrspace 4)
+  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (dereferenceable invariant load 8, align 16, addrspace 4)
   ; HSA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 32
-  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C1]](s64)
-  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(<8 x s32>) = G_LOAD [[GEP1]](p4) :: (non-temporal invariant load 32 from `<8 x i32> addrspace(4)* undef`, align 16, addrspace 4)
+  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(<8 x s32>) = G_LOAD [[GEP1]](p4) :: (dereferenceable invariant load 32, align 16, addrspace 4)
   ; HSA-VI:   G_STORE [[LOAD1]](<8 x s32>), [[LOAD]](p1) :: (store 32 into %ir.out, align 4, addrspace 1)
-  ; HSA-VI:   S_ENDPGM
+  ; HSA-VI:   S_ENDPGM 0
 entry:
   store <8 x i32> %in, <8 x i32> addrspace(1)* %out, align 4
   ret void
@@ -427,13 +427,13 @@ define amdgpu_kernel void @v8f32_arg(<8 x float> addrspace(1)* nocapture %out, <
   ; HSA-VI:   liveins: $sgpr4_sgpr5
   ; HSA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr4_sgpr5
   ; HSA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C]](s64)
-  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (non-temporal invariant load 8 from `<8 x float> addrspace(1)* addrspace(4)* undef`, align 16, addrspace 4)
+  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (dereferenceable invariant load 8, align 16, addrspace 4)
   ; HSA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 32
-  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C1]](s64)
-  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(<8 x s32>) = G_LOAD [[GEP1]](p4) :: (non-temporal invariant load 32 from `<8 x float> addrspace(4)* undef`, align 16, addrspace 4)
+  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(<8 x s32>) = G_LOAD [[GEP1]](p4) :: (dereferenceable invariant load 32, align 16, addrspace 4)
   ; HSA-VI:   G_STORE [[LOAD1]](<8 x s32>), [[LOAD]](p1) :: (store 32 into %ir.out, align 4, addrspace 1)
-  ; HSA-VI:   S_ENDPGM
+  ; HSA-VI:   S_ENDPGM 0
 entry:
   store <8 x float> %in, <8 x float> addrspace(1)* %out, align 4
   ret void
@@ -445,13 +445,13 @@ define amdgpu_kernel void @v16i8_arg(<16 x i8> addrspace(1)* %out, <16 x i8> %in
   ; HSA-VI:   liveins: $sgpr4_sgpr5
   ; HSA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr4_sgpr5
   ; HSA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C]](s64)
-  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (non-temporal invariant load 8 from `<16 x i8> addrspace(1)* addrspace(4)* undef`, align 16, addrspace 4)
+  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (dereferenceable invariant load 8, align 16, addrspace 4)
   ; HSA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
-  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C1]](s64)
-  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(<16 x s8>) = G_LOAD [[GEP1]](p4) :: (non-temporal invariant load 16 from `<16 x i8> addrspace(4)* undef`, addrspace 4)
+  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(<16 x s8>) = G_LOAD [[GEP1]](p4) :: (dereferenceable invariant load 16, addrspace 4)
   ; HSA-VI:   G_STORE [[LOAD1]](<16 x s8>), [[LOAD]](p1) :: (store 16 into %ir.out, addrspace 1)
-  ; HSA-VI:   S_ENDPGM
+  ; HSA-VI:   S_ENDPGM 0
 entry:
   store <16 x i8> %in, <16 x i8> addrspace(1)* %out
   ret void
@@ -463,13 +463,13 @@ define amdgpu_kernel void @v16i16_arg(<16 x i16> addrspace(1)* %out, <16 x i16>
   ; HSA-VI:   liveins: $sgpr4_sgpr5
   ; HSA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr4_sgpr5
   ; HSA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C]](s64)
-  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (non-temporal invariant load 8 from `<16 x i16> addrspace(1)* addrspace(4)* undef`, align 16, addrspace 4)
+  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (dereferenceable invariant load 8, align 16, addrspace 4)
   ; HSA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 32
-  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C1]](s64)
-  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(<16 x s16>) = G_LOAD [[GEP1]](p4) :: (non-temporal invariant load 32 from `<16 x i16> addrspace(4)* undef`, align 16, addrspace 4)
+  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(<16 x s16>) = G_LOAD [[GEP1]](p4) :: (dereferenceable invariant load 32, align 16, addrspace 4)
   ; HSA-VI:   G_STORE [[LOAD1]](<16 x s16>), [[LOAD]](p1) :: (store 32 into %ir.out, addrspace 1)
-  ; HSA-VI:   S_ENDPGM
+  ; HSA-VI:   S_ENDPGM 0
 entry:
   store <16 x i16> %in, <16 x i16> addrspace(1)* %out
   ret void
@@ -481,13 +481,13 @@ define amdgpu_kernel void @v16i32_arg(<16 x i32> addrspace(1)* nocapture %out, <
   ; HSA-VI:   liveins: $sgpr4_sgpr5
   ; HSA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr4_sgpr5
   ; HSA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C]](s64)
-  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (non-temporal invariant load 8 from `<16 x i32> addrspace(1)* addrspace(4)* undef`, align 16, addrspace 4)
+  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (dereferenceable invariant load 8, align 16, addrspace 4)
   ; HSA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 64
-  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C1]](s64)
-  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(<16 x s32>) = G_LOAD [[GEP1]](p4) :: (non-temporal invariant load 64 from `<16 x i32> addrspace(4)* undef`, align 16, addrspace 4)
+  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(<16 x s32>) = G_LOAD [[GEP1]](p4) :: (dereferenceable invariant load 64, align 16, addrspace 4)
   ; HSA-VI:   G_STORE [[LOAD1]](<16 x s32>), [[LOAD]](p1) :: (store 64 into %ir.out, align 4, addrspace 1)
-  ; HSA-VI:   S_ENDPGM
+  ; HSA-VI:   S_ENDPGM 0
 entry:
   store <16 x i32> %in, <16 x i32> addrspace(1)* %out, align 4
   ret void
@@ -499,13 +499,13 @@ define amdgpu_kernel void @v16f32_arg(<16 x float> addrspace(1)* nocapture %out,
   ; HSA-VI:   liveins: $sgpr4_sgpr5
   ; HSA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr4_sgpr5
   ; HSA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C]](s64)
-  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (non-temporal invariant load 8 from `<16 x float> addrspace(1)* addrspace(4)* undef`, align 16, addrspace 4)
+  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (dereferenceable invariant load 8, align 16, addrspace 4)
   ; HSA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 64
-  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C1]](s64)
-  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(<16 x s32>) = G_LOAD [[GEP1]](p4) :: (non-temporal invariant load 64 from `<16 x float> addrspace(4)* undef`, align 16, addrspace 4)
+  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(<16 x s32>) = G_LOAD [[GEP1]](p4) :: (dereferenceable invariant load 64, align 16, addrspace 4)
   ; HSA-VI:   G_STORE [[LOAD1]](<16 x s32>), [[LOAD]](p1) :: (store 64 into %ir.out, align 4, addrspace 1)
-  ; HSA-VI:   S_ENDPGM
+  ; HSA-VI:   S_ENDPGM 0
 entry:
   store <16 x float> %in, <16 x float> addrspace(1)* %out, align 4
   ret void
@@ -517,13 +517,13 @@ define amdgpu_kernel void @kernel_arg_i64(i64 addrspace(1)* %out, i64 %a) nounwi
   ; HSA-VI:   liveins: $sgpr4_sgpr5
   ; HSA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr4_sgpr5
   ; HSA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C]](s64)
-  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (non-temporal invariant load 8 from `i64 addrspace(1)* addrspace(4)* undef`, align 16, addrspace 4)
+  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (dereferenceable invariant load 8, align 16, addrspace 4)
   ; HSA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C1]](s64)
-  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(s64) = G_LOAD [[GEP1]](p4) :: (non-temporal invariant load 8 from `i64 addrspace(4)* undef`, addrspace 4)
+  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(s64) = G_LOAD [[GEP1]](p4) :: (dereferenceable invariant load 8, addrspace 4)
   ; HSA-VI:   G_STORE [[LOAD1]](s64), [[LOAD]](p1) :: (store 8 into %ir.out, addrspace 1)
-  ; HSA-VI:   S_ENDPGM
+  ; HSA-VI:   S_ENDPGM 0
   store i64 %a, i64 addrspace(1)* %out, align 8
   ret void
 }
@@ -534,13 +534,13 @@ define amdgpu_kernel void @f64_kernel_arg(double addrspace(1)* %out, double  %in
   ; HSA-VI:   liveins: $sgpr4_sgpr5
   ; HSA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr4_sgpr5
   ; HSA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C]](s64)
-  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (non-temporal invariant load 8 from `double addrspace(1)* addrspace(4)* undef`, align 16, addrspace 4)
+  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (dereferenceable invariant load 8, align 16, addrspace 4)
   ; HSA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C1]](s64)
-  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(s64) = G_LOAD [[GEP1]](p4) :: (non-temporal invariant load 8 from `double addrspace(4)* undef`, addrspace 4)
+  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(s64) = G_LOAD [[GEP1]](p4) :: (dereferenceable invariant load 8, addrspace 4)
   ; HSA-VI:   G_STORE [[LOAD1]](s64), [[LOAD]](p1) :: (store 8 into %ir.out, addrspace 1)
-  ; HSA-VI:   S_ENDPGM
+  ; HSA-VI:   S_ENDPGM 0
 entry:
   store double %in, double addrspace(1)* %out
   ret void
@@ -552,13 +552,13 @@ define amdgpu_kernel void @i1_arg(i1 addrspace(1)* %out, i1 %x) nounwind {
   ; HSA-VI:   liveins: $sgpr4_sgpr5
   ; HSA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr4_sgpr5
   ; HSA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C]](s64)
-  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (non-temporal invariant load 8 from `i1 addrspace(1)* addrspace(4)* undef`, align 16, addrspace 4)
+  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (dereferenceable invariant load 8, align 16, addrspace 4)
   ; HSA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C1]](s64)
-  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(s1) = G_LOAD [[GEP1]](p4) :: (non-temporal invariant load 1 from `i1 addrspace(4)* undef`, align 8, addrspace 4)
+  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(s1) = G_LOAD [[GEP1]](p4) :: (dereferenceable invariant load 1, align 8, addrspace 4)
   ; HSA-VI:   G_STORE [[LOAD1]](s1), [[LOAD]](p1) :: (store 1 into %ir.out, addrspace 1)
-  ; HSA-VI:   S_ENDPGM
+  ; HSA-VI:   S_ENDPGM 0
   store i1 %x, i1 addrspace(1)* %out, align 1
   ret void
 }
@@ -569,14 +569,14 @@ define amdgpu_kernel void @i1_arg_zext_i32(i32 addrspace(1)* %out, i1 %x) nounwi
   ; HSA-VI:   liveins: $sgpr4_sgpr5
   ; HSA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr4_sgpr5
   ; HSA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C]](s64)
-  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (non-temporal invariant load 8 from `i32 addrspace(1)* addrspace(4)* undef`, align 16, addrspace 4)
+  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (dereferenceable invariant load 8, align 16, addrspace 4)
   ; HSA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C1]](s64)
-  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(s1) = G_LOAD [[GEP1]](p4) :: (non-temporal invariant load 1 from `i1 addrspace(4)* undef`, align 8, addrspace 4)
+  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(s1) = G_LOAD [[GEP1]](p4) :: (dereferenceable invariant load 1, align 8, addrspace 4)
   ; HSA-VI:   [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[LOAD1]](s1)
   ; HSA-VI:   G_STORE [[ZEXT]](s32), [[LOAD]](p1) :: (store 4 into %ir.out, addrspace 1)
-  ; HSA-VI:   S_ENDPGM
+  ; HSA-VI:   S_ENDPGM 0
   %ext = zext i1 %x to i32
   store i32 %ext, i32 addrspace(1)* %out, align 4
   ret void
@@ -588,14 +588,14 @@ define amdgpu_kernel void @i1_arg_zext_i64(i64 addrspace(1)* %out, i1 %x) nounwi
   ; HSA-VI:   liveins: $sgpr4_sgpr5
   ; HSA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr4_sgpr5
   ; HSA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C]](s64)
-  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (non-temporal invariant load 8 from `i64 addrspace(1)* addrspace(4)* undef`, align 16, addrspace 4)
+  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (dereferenceable invariant load 8, align 16, addrspace 4)
   ; HSA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C1]](s64)
-  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(s1) = G_LOAD [[GEP1]](p4) :: (non-temporal invariant load 1 from `i1 addrspace(4)* undef`, align 8, addrspace 4)
+  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(s1) = G_LOAD [[GEP1]](p4) :: (dereferenceable invariant load 1, align 8, addrspace 4)
   ; HSA-VI:   [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[LOAD1]](s1)
   ; HSA-VI:   G_STORE [[ZEXT]](s64), [[LOAD]](p1) :: (store 8 into %ir.out, addrspace 1)
-  ; HSA-VI:   S_ENDPGM
+  ; HSA-VI:   S_ENDPGM 0
   %ext = zext i1 %x to i64
   store i64 %ext, i64 addrspace(1)* %out, align 8
   ret void
@@ -607,14 +607,14 @@ define amdgpu_kernel void @i1_arg_sext_i32(i32 addrspace(1)* %out, i1 %x) nounwi
   ; HSA-VI:   liveins: $sgpr4_sgpr5
   ; HSA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr4_sgpr5
   ; HSA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C]](s64)
-  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (non-temporal invariant load 8 from `i32 addrspace(1)* addrspace(4)* undef`, align 16, addrspace 4)
+  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (dereferenceable invariant load 8, align 16, addrspace 4)
   ; HSA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C1]](s64)
-  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(s1) = G_LOAD [[GEP1]](p4) :: (non-temporal invariant load 1 from `i1 addrspace(4)* undef`, align 8, addrspace 4)
+  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(s1) = G_LOAD [[GEP1]](p4) :: (dereferenceable invariant load 1, align 8, addrspace 4)
   ; HSA-VI:   [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[LOAD1]](s1)
   ; HSA-VI:   G_STORE [[SEXT]](s32), [[LOAD]](p1) :: (store 4 into %ir.out, addrspace 1)
-  ; HSA-VI:   S_ENDPGM
+  ; HSA-VI:   S_ENDPGM 0
   %ext = sext i1 %x to i32
   store i32 %ext, i32addrspace(1)* %out, align 4
   ret void
@@ -626,14 +626,14 @@ define amdgpu_kernel void @i1_arg_sext_i64(i64 addrspace(1)* %out, i1 %x) nounwi
   ; HSA-VI:   liveins: $sgpr4_sgpr5
   ; HSA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr4_sgpr5
   ; HSA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C]](s64)
-  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (non-temporal invariant load 8 from `i64 addrspace(1)* addrspace(4)* undef`, align 16, addrspace 4)
+  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[GEP]](p4) :: (dereferenceable invariant load 8, align 16, addrspace 4)
   ; HSA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C1]](s64)
-  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(s1) = G_LOAD [[GEP1]](p4) :: (non-temporal invariant load 1 from `i1 addrspace(4)* undef`, align 8, addrspace 4)
+  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(s1) = G_LOAD [[GEP1]](p4) :: (dereferenceable invariant load 1, align 8, addrspace 4)
   ; HSA-VI:   [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[LOAD1]](s1)
   ; HSA-VI:   G_STORE [[SEXT]](s64), [[LOAD]](p1) :: (store 8 into %ir.out, addrspace 1)
-  ; HSA-VI:   S_ENDPGM
+  ; HSA-VI:   S_ENDPGM 0
   %ext = sext i1 %x to i64
   store i64 %ext, i64 addrspace(1)* %out, align 8
   ret void
@@ -644,7 +644,7 @@ define amdgpu_kernel void @empty_struct_arg({} %in) nounwind {
   ; HSA-VI: bb.1 (%ir-block.0):
   ; HSA-VI:   liveins: $sgpr4_sgpr5
   ; HSA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr4_sgpr5
-  ; HSA-VI:   S_ENDPGM
+  ; HSA-VI:   S_ENDPGM 0
   ret void
 }
 
@@ -673,19 +673,19 @@ define amdgpu_kernel void @struct_argument_alignment({i32, i64} %arg0, i8, {i32,
   ; HSA-VI:   liveins: $sgpr4_sgpr5
   ; HSA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr4_sgpr5
   ; HSA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C]](s64)
-  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(s128) = G_LOAD [[GEP]](p4) :: (non-temporal invariant load 16 from `{ i32, i64 } addrspace(4)* undef`, addrspace 4)
+  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(s128) = G_LOAD [[GEP]](p4) :: (dereferenceable invariant load 16, addrspace 4)
   ; HSA-VI:   [[EXTRACT:%[0-9]+]]:_(s32) = G_EXTRACT [[LOAD]](s128), 0
   ; HSA-VI:   [[EXTRACT1:%[0-9]+]]:_(s64) = G_EXTRACT [[LOAD]](s128), 64
   ; HSA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
-  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C1]](s64)
-  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(s8) = G_LOAD [[GEP1]](p4) :: (non-temporal invariant load 1 from `i8 addrspace(4)* undef`, align 16, addrspace 4)
+  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(s8) = G_LOAD [[GEP1]](p4) :: (dereferenceable invariant load 1, align 16, addrspace 4)
   ; HSA-VI:   [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 24
-  ; HSA-VI:   [[GEP2:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C2]](s64)
-  ; HSA-VI:   [[LOAD2:%[0-9]+]]:_(s128) = G_LOAD [[GEP2]](p4) :: (non-temporal invariant load 16 from `{ i32, i64 } addrspace(4)* undef`, align 8, addrspace 4)
+  ; HSA-VI:   [[GEP2:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C2]](s64)
+  ; HSA-VI:   [[LOAD2:%[0-9]+]]:_(s128) = G_LOAD [[GEP2]](p4) :: (dereferenceable invariant load 16, align 8, addrspace 4)
   ; HSA-VI:   [[EXTRACT2:%[0-9]+]]:_(s32) = G_EXTRACT [[LOAD2]](s128), 0
   ; HSA-VI:   [[EXTRACT3:%[0-9]+]]:_(s64) = G_EXTRACT [[LOAD2]](s128), 64
-  ; HSA-VI:   S_ENDPGM
+  ; HSA-VI:   S_ENDPGM 0
   ret void
 }
 
@@ -705,18 +705,18 @@ define amdgpu_kernel void @packed_struct_argument_alignment(<{i32, i64}> %arg0,
   ; HSA-VI:   liveins: $sgpr4_sgpr5
   ; HSA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr4_sgpr5
   ; HSA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C]](s64)
-  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(s96) = G_LOAD [[GEP]](p4) :: (non-temporal invariant load 12 from `<{ i32, i64 }> addrspace(4)* undef`, align 16, addrspace 4)
+  ; HSA-VI:   [[GEP:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(s96) = G_LOAD [[GEP]](p4) :: (dereferenceable invariant load 12, align 16, addrspace 4)
   ; HSA-VI:   [[EXTRACT:%[0-9]+]]:_(s32) = G_EXTRACT [[LOAD]](s96), 0
   ; HSA-VI:   [[EXTRACT1:%[0-9]+]]:_(s64) = G_EXTRACT [[LOAD]](s96), 32
   ; HSA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 12
-  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C1]](s64)
-  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(s8) = G_LOAD [[GEP1]](p4) :: (non-temporal invariant load 1 from `i8 addrspace(4)* undef`, align 4, addrspace 4)
+  ; HSA-VI:   [[GEP1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(s8) = G_LOAD [[GEP1]](p4) :: (dereferenceable invariant load 1, align 4, addrspace 4)
   ; HSA-VI:   [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 13
-  ; HSA-VI:   [[GEP2:%[0-9]+]]:_(p4) = G_GEP [[COPY]], [[C2]](s64)
-  ; HSA-VI:   [[LOAD2:%[0-9]+]]:_(s96) = G_LOAD [[GEP2]](p4) :: (non-temporal invariant load 12 from `<{ i32, i64 }> addrspace(4)* undef`, align 1, addrspace 4)
+  ; HSA-VI:   [[GEP2:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C2]](s64)
+  ; HSA-VI:   [[LOAD2:%[0-9]+]]:_(s96) = G_LOAD [[GEP2]](p4) :: (dereferenceable invariant load 12, align 1, addrspace 4)
   ; HSA-VI:   [[EXTRACT2:%[0-9]+]]:_(s32) = G_EXTRACT [[LOAD2]](s96), 0
   ; HSA-VI:   [[EXTRACT3:%[0-9]+]]:_(s64) = G_EXTRACT [[LOAD2]](s96), 32
-  ; HSA-VI:   S_ENDPGM
+  ; HSA-VI:   S_ENDPGM 0
   ret void
 }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgpu_ps.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgpu_ps.ll
index 0d11aa993..2957b1833 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgpu_ps.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgpu_ps.ll
@@ -1,16 +1,37 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 ; RUN: llc -mtriple=amdgcn-mesa-mesa3d -stop-after=irtranslator -global-isel %s -o - | FileCheck %s
 
 ; Check that we correctly skip over disabled inputs
-; CHECK: [[S0:%[0-9]+]]:_(s32) = COPY $sgpr2
-; CHECK: [[V0:%[0-9]+]]:_(s32) = COPY $vgpr0
-; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.exp), %{{[0-9]+}}(s32), %{{[0-9]+}}(s32), [[S0]](s32), [[S0]](s32), [[S0]](s32), [[V0]](s32)
-define amdgpu_ps void @ps0(float inreg %arg0, float %psinput0, float %psinput1) #1 {
+define amdgpu_ps void @disabled_input(float inreg %arg0, float %psinput0, float %psinput1) #1 {
+  ; CHECK-LABEL: name: disabled_input
+  ; CHECK: bb.1.main_body:
+  ; CHECK:   liveins: $sgpr2, $vgpr0
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+  ; CHECK:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.exp), 0, 15, [[COPY]](s32), [[COPY]](s32), [[COPY]](s32), [[COPY1]](s32), 0, 0
+  ; CHECK:   S_ENDPGM 0
 main_body:
   call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %arg0, float %arg0, float %arg0, float %psinput1, i1 false, i1 false) #0
   ret void
 }
 
-declare void @llvm.amdgcn.exp.f32(i32, i32, float, float, float, float, i1, i1)  #0
+define amdgpu_ps void @disabled_input_struct(float inreg %arg0, { float, float } %psinput0, float %psinput1) #1 {
+  ; CHECK-LABEL: name: disabled_input_struct
+  ; CHECK: bb.1.main_body:
+  ; CHECK:   liveins: $sgpr2, $vgpr0
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+  ; CHECK:   [[DEF1:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+  ; CHECK:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.exp), 0, 15, [[COPY]](s32), [[COPY]](s32), [[COPY]](s32), [[COPY1]](s32), 0, 0
+  ; CHECK:   S_ENDPGM 0
+main_body:
+  call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %arg0, float %arg0, float %arg0, float %psinput1, i1 false, i1 false) #0
+  ret void
+}
+
+declare void @llvm.amdgcn.exp.f32(i32 immarg, i32 immarg, float, float, float, float, i1 immarg, i1 immarg)  #0
 
 attributes #0 = { nounwind }
 attributes #1 = { "InitialPSInputAddr"="0x00002" }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgpu_vs.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgpu_vs.ll
index 44b06b223..e93d39289 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgpu_vs.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgpu_vs.ll
@@ -1,9 +1,8 @@
 ; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -stop-after=irtranslator -global-isel %s -o - | FileCheck %s
 
-
 ; CHECK-LABEL: name: test_f32_inreg
 ; CHECK: [[S0:%[0-9]+]]:_(s32) = COPY $sgpr2
-; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.exp), %{{[0-9]+}}(s32), %{{[0-9]+}}(s32), [[S0]]
+; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.exp), 32, 15, [[S0]]
 define amdgpu_vs void @test_f32_inreg(float inreg %arg0) {
   call void @llvm.amdgcn.exp.f32(i32 32, i32 15, float %arg0, float undef, float undef, float undef, i1 false, i1 false) #0
   ret void
@@ -11,25 +10,29 @@ define amdgpu_vs void @test_f32_inreg(float inreg %arg0) {
 
 ; CHECK-LABEL: name: test_f32
 ; CHECK: [[V0:%[0-9]+]]:_(s32) = COPY $vgpr0
-; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.exp), %{{[0-9]+}}(s32), %{{[0-9]+}}(s32), [[V0]]
+; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.exp), 32, 15, [[V0]]
 define amdgpu_vs void @test_f32(float %arg0) {
   call void @llvm.amdgcn.exp.f32(i32 32, i32 15, float %arg0, float undef, float undef, float undef, i1 false, i1 false) #0
   ret void
 }
 
 ; CHECK-LABEL: name: test_ptr2_inreg
-; CHECK: [[S01:%[0-9]+]]:_(p4) = COPY $sgpr2_sgpr3
-; CHECK: G_LOAD [[S01]]
+; CHECK: [[S2:%[0-9]+]]:_(s32) = COPY $sgpr2
+; CHECK: [[S3:%[0-9]+]]:_(s32) = COPY $sgpr3
+; CHECK: [[PTR:%[0-9]+]]:_(p4) = G_MERGE_VALUES [[S2]](s32), [[S3]](s32)
+; CHECK: G_LOAD [[PTR]]
 define amdgpu_vs void @test_ptr2_inreg(i32 addrspace(4)* inreg %arg0) {
   %tmp0 = load volatile i32, i32 addrspace(4)* %arg0
   ret void
 }
 
 ; CHECK-LABEL: name: test_sgpr_alignment0
-; CHECK: [[S0:%[0-9]+]]:_(s32) = COPY $sgpr2
-; CHECK: [[S23:%[0-9]+]]:_(p4) = COPY $sgpr4_sgpr5
-; CHECK: G_LOAD [[S23]]
-; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.exp), %{{[0-9]+}}(s32), %{{[0-9]+}}(s32), [[S0]]
+; CHECK: [[S2:%[0-9]+]]:_(s32) = COPY $sgpr2
+; CHECK: [[S3:%[0-9]+]]:_(s32) = COPY $sgpr3
+; CHECK: [[S4:%[0-9]+]]:_(s32) = COPY $sgpr4
+; CHECK: [[S34:%[0-9]+]]:_(p4) = G_MERGE_VALUES [[S3]](s32), [[S4]](s32)
+; CHECK: G_LOAD [[S34]]
+; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.exp), 32, 15, [[S2]](s32)
 define amdgpu_vs void @test_sgpr_alignment0(float inreg %arg0, i32 addrspace(4)* inreg %arg1) {
   %tmp0 = load volatile i32, i32 addrspace(4)* %arg1
   call void @llvm.amdgcn.exp.f32(i32 32, i32 15, float %arg0, float undef, float undef, float undef, i1 false, i1 false) #0
@@ -41,7 +44,7 @@ define amdgpu_vs void @test_sgpr_alignment0(float inreg %arg0, i32 addrspace(4)*
 ; CHECK: [[S1:%[0-9]+]]:_(s32) = COPY $sgpr3
 ; CHECK: [[V0:%[0-9]+]]:_(s32) = COPY $vgpr0
 ; CHECK: [[V1:%[0-9]+]]:_(s32) = COPY $vgpr1
-; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.exp), %{{[0-9]+}}(s32), %{{[0-9]+}}(s32), [[V0]](s32), [[S0]](s32), [[V1]](s32), [[S1]](s32)
+; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.exp), 32, 15, [[V0]](s32), [[S0]](s32), [[V1]](s32), [[S1]](s32)
 define amdgpu_vs void @test_order(float inreg %arg0, float inreg %arg1, float %arg2, float %arg3) {
   call void @llvm.amdgcn.exp.f32(i32 32, i32 15, float %arg2, float %arg0, float %arg3, float %arg1, i1 false, i1 false) #0
   ret void
@@ -52,7 +55,7 @@ define amdgpu_vs void @test_order(float inreg %arg0, float inreg %arg1, float %a
 ; CHECK: [[S1:%[0-9]+]]:_(s32) = COPY $sgpr3
 ; CHECK: $sgpr0 = COPY [[S0]]
 ; CHECK: $sgpr1 = COPY [[S1]]
-; CHECK: SI_RETURN_TO_EPILOG $sgpr0, $sgpr1
+; CHECK: SI_RETURN_TO_EPILOG implicit $sgpr0, implicit $sgpr1
 define amdgpu_vs <{ i32, i32 }> @ret_struct(i32 inreg %arg0, i32 inreg %arg1) {
 main_body:
   %tmp0 = insertvalue <{ i32, i32 }> undef, i32 %arg0, 0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-atomicrmw.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-atomicrmw.ll
new file mode 100644
index 000000000..34c167878
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-atomicrmw.ll
@@ -0,0 +1,48 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; RUN: llc -global-isel -march=amdgcn -mcpu=fiji -O0 -stop-after=irtranslator -o - %s | FileCheck %s
+
+define float @test_atomicrmw_fadd(float addrspace(3)* %addr) {
+  ; CHECK-LABEL: name: test_atomicrmw_fadd
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 1.000000e+00
+  ; CHECK:   [[ATOMICRMW_FADD:%[0-9]+]]:_(s32) = G_ATOMICRMW_FADD [[COPY]](p3), [[C]] :: (load store seq_cst 4 on %ir.addr, addrspace 3)
+  ; CHECK:   $vgpr0 = COPY [[ATOMICRMW_FADD]](s32)
+  ; CHECK:   [[COPY2:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY1]]
+  ; CHECK:   S_SETPC_B64_return [[COPY2]], implicit $vgpr0
+  %oldval = atomicrmw fadd float addrspace(3)* %addr, float 1.0 seq_cst
+  ret float %oldval
+}
+
+define float @test_atomicrmw_fsub(float addrspace(3)* %addr) {
+  ; CHECK-LABEL: name: test_atomicrmw_fsub
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   liveins: $vgpr0, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 1.000000e+00
+  ; CHECK:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p3) :: (load 4 from %ir.addr, addrspace 3)
+  ; CHECK: bb.2.atomicrmw.start:
+  ; CHECK:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   [[PHI:%[0-9]+]]:_(s64) = G_PHI %9(s64), %bb.2, [[C1]](s64), %bb.1
+  ; CHECK:   [[PHI1:%[0-9]+]]:_(s32) = G_PHI [[LOAD]](s32), %bb.1, %7(s32), %bb.2
+  ; CHECK:   [[FSUB:%[0-9]+]]:_(s32) = G_FSUB [[PHI1]], [[C]]
+  ; CHECK:   [[ATOMIC_CMPXCHG_WITH_SUCCESS:%[0-9]+]]:_(s32), [[ATOMIC_CMPXCHG_WITH_SUCCESS1:%[0-9]+]]:_(s1) = G_ATOMIC_CMPXCHG_WITH_SUCCESS [[COPY]](p3), [[PHI1]], [[FSUB]] :: (load store seq_cst seq_cst 4 on %ir.2, addrspace 3)
+  ; CHECK:   [[INT:%[0-9]+]]:_(s64) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), [[ATOMIC_CMPXCHG_WITH_SUCCESS1]](s1), [[PHI]](s64)
+  ; CHECK:   [[INT1:%[0-9]+]]:_(s1) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.loop), [[INT]](s64)
+  ; CHECK:   G_BRCOND [[INT1]](s1), %bb.3
+  ; CHECK:   G_BR %bb.2
+  ; CHECK: bb.3.atomicrmw.end:
+  ; CHECK:   [[PHI2:%[0-9]+]]:_(s32) = G_PHI [[ATOMIC_CMPXCHG_WITH_SUCCESS]](s32), %bb.2
+  ; CHECK:   [[PHI3:%[0-9]+]]:_(s64) = G_PHI [[INT]](s64), %bb.2
+  ; CHECK:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[PHI3]](s64)
+  ; CHECK:   $vgpr0 = COPY [[PHI2]](s32)
+  ; CHECK:   [[COPY2:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY1]]
+  ; CHECK:   S_SETPC_B64_return [[COPY2]], implicit $vgpr0
+  %oldval = atomicrmw fsub float addrspace(3)* %addr, float 1.0 seq_cst
+  ret float %oldval
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-function-args.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-function-args.ll
new file mode 100644
index 000000000..c07e9ad4d
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-function-args.ll
@@ -0,0 +1,2227 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; RUN: llc -march=amdgcn -mcpu=tahiti -O0 -stop-after=irtranslator -global-isel -verify-machineinstrs -o - %s | FileCheck %s
+
+define void @void_func_i1(i1 %arg0) #0 {
+  ; CHECK-LABEL: name: void_func_i1
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[TRUNC:%[0-9]+]]:_(s1) = G_TRUNC [[COPY]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   G_STORE [[TRUNC]](s1), [[DEF]](p1) :: (store 1 into `i1 addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[COPY2:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY1]]
+  ; CHECK:   S_SETPC_B64_return [[COPY2]]
+  store i1 %arg0, i1 addrspace(1)* undef
+  ret void
+}
+
+define void @void_func_i1_zeroext(i1 zeroext %arg0) #0 {
+  ; CHECK-LABEL: name: void_func_i1_zeroext
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[TRUNC:%[0-9]+]]:_(s1) = G_TRUNC [[COPY]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[TRUNC]](s1)
+  ; CHECK:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[ZEXT]], [[C]]
+  ; CHECK:   G_STORE [[ADD]](s32), [[DEF]](p1) :: (store 4 into `i32 addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[COPY2:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY1]]
+  ; CHECK:   S_SETPC_B64_return [[COPY2]]
+  %ext = zext i1 %arg0 to i32
+  %add = add i32 %ext, 12
+  store i32 %add, i32 addrspace(1)* undef
+  ret void
+}
+
+define void @void_func_i1_signext(i1 signext %arg0) #0 {
+  ; CHECK-LABEL: name: void_func_i1_signext
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[TRUNC:%[0-9]+]]:_(s1) = G_TRUNC [[COPY]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[TRUNC]](s1)
+  ; CHECK:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[SEXT]], [[C]]
+  ; CHECK:   G_STORE [[ADD]](s32), [[DEF]](p1) :: (store 4 into `i32 addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[COPY2:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY1]]
+  ; CHECK:   S_SETPC_B64_return [[COPY2]]
+  %ext = sext i1 %arg0 to i32
+  %add = add i32 %ext, 12
+  store i32 %add, i32 addrspace(1)* undef
+  ret void
+}
+
+define void @i1_arg_i1_use(i1 %arg) #0 {
+  ; CHECK-LABEL: name: i1_arg_i1_use
+  ; CHECK: bb.1.bb:
+  ; CHECK:   successors: %bb.2(0x40000000), %bb.3(0x40000000)
+  ; CHECK:   liveins: $vgpr0, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[TRUNC:%[0-9]+]]:_(s1) = G_TRUNC [[COPY]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[C:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
+  ; CHECK:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[XOR:%[0-9]+]]:_(s1) = G_XOR [[TRUNC]], [[C]]
+  ; CHECK:   [[INT:%[0-9]+]]:_(s1), [[INT1:%[0-9]+]]:_(s64) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.if), [[XOR]](s1)
+  ; CHECK:   G_BRCOND [[INT]](s1), %bb.2
+  ; CHECK:   G_BR %bb.3
+  ; CHECK: bb.2.bb1:
+  ; CHECK:   successors: %bb.3(0x80000000)
+  ; CHECK:   G_STORE [[C1]](s32), [[DEF]](p1) :: (volatile store 4 into `i32 addrspace(1)* undef`, addrspace 1)
+  ; CHECK: bb.3.bb2:
+  ; CHECK:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[INT1]](s64)
+  ; CHECK:   [[COPY2:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY1]]
+  ; CHECK:   S_SETPC_B64_return [[COPY2]]
+bb:
+  br i1 %arg, label %bb2, label %bb1
+
+bb1:
+  store volatile i32 0, i32 addrspace(1)* undef
+  br label %bb2
+
+bb2:
+  ret void
+}
+
+define void @void_func_i8(i8 %arg0) #0 {
+  ; CHECK-LABEL: name: void_func_i8
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[COPY]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   G_STORE [[TRUNC]](s8), [[DEF]](p1) :: (store 1 into `i8 addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[COPY2:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY1]]
+  ; CHECK:   S_SETPC_B64_return [[COPY2]]
+  store i8 %arg0, i8 addrspace(1)* undef
+  ret void
+}
+
+define void @void_func_i8_zeroext(i8 zeroext %arg0) #0 {
+  ; CHECK-LABEL: name: void_func_i8_zeroext
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[COPY]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[TRUNC]](s8)
+  ; CHECK:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[ZEXT]], [[C]]
+  ; CHECK:   G_STORE [[ADD]](s32), [[DEF]](p1) :: (store 4 into `i32 addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[COPY2:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY1]]
+  ; CHECK:   S_SETPC_B64_return [[COPY2]]
+  %ext = zext i8 %arg0 to i32
+  %add = add i32 %ext, 12
+  store i32 %add, i32 addrspace(1)* undef
+  ret void
+}
+
+define void @void_func_i8_signext(i8 signext %arg0) #0 {
+  ; CHECK-LABEL: name: void_func_i8_signext
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[COPY]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[TRUNC]](s8)
+  ; CHECK:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[SEXT]], [[C]]
+  ; CHECK:   G_STORE [[ADD]](s32), [[DEF]](p1) :: (store 4 into `i32 addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[COPY2:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY1]]
+  ; CHECK:   S_SETPC_B64_return [[COPY2]]
+  %ext = sext i8 %arg0 to i32
+  %add = add i32 %ext, 12
+  store i32 %add, i32 addrspace(1)* undef
+  ret void
+}
+
+define void @void_func_i16(i16 %arg0) #0 {
+  ; CHECK-LABEL: name: void_func_i16
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   G_STORE [[TRUNC]](s16), [[DEF]](p1) :: (store 2 into `i16 addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[COPY2:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY1]]
+  ; CHECK:   S_SETPC_B64_return [[COPY2]]
+  store i16 %arg0, i16 addrspace(1)* undef
+  ret void
+}
+
+define void @void_func_i16_zeroext(i16 zeroext %arg0) #0 {
+  ; CHECK-LABEL: name: void_func_i16_zeroext
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[TRUNC]](s16)
+  ; CHECK:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[ZEXT]], [[C]]
+  ; CHECK:   G_STORE [[ADD]](s32), [[DEF]](p1) :: (store 4 into `i32 addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[COPY2:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY1]]
+  ; CHECK:   S_SETPC_B64_return [[COPY2]]
+  %ext = zext i16 %arg0 to i32
+  %add = add i32 %ext, 12
+  store i32 %add, i32 addrspace(1)* undef
+  ret void
+}
+
+define void @void_func_i16_signext(i16 signext %arg0) #0 {
+  ; CHECK-LABEL: name: void_func_i16_signext
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[TRUNC]](s16)
+  ; CHECK:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[SEXT]], [[C]]
+  ; CHECK:   G_STORE [[ADD]](s32), [[DEF]](p1) :: (store 4 into `i32 addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[COPY2:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY1]]
+  ; CHECK:   S_SETPC_B64_return [[COPY2]]
+  %ext = sext i16 %arg0 to i32
+  %add = add i32 %ext, 12
+  store i32 %add, i32 addrspace(1)* undef
+  ret void
+}
+
+define void @void_func_i32(i32 %arg0) #0 {
+  ; CHECK-LABEL: name: void_func_i32
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   G_STORE [[COPY]](s32), [[DEF]](p1) :: (store 4 into `i32 addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[COPY2:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY1]]
+  ; CHECK:   S_SETPC_B64_return [[COPY2]]
+  store i32 %arg0, i32 addrspace(1)* undef
+  ret void
+}
+
+define void @void_func_p3i8(i8 addrspace(3)* %arg0) #0 {
+  ; CHECK-LABEL: name: void_func_p3i8
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   G_STORE [[COPY]](p3), [[DEF]](p1) :: (store 4 into `i8 addrspace(3)* addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[COPY2:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY1]]
+  ; CHECK:   S_SETPC_B64_return [[COPY2]]
+  store i8 addrspace(3)* %arg0, i8 addrspace(3)* addrspace(1)* undef
+  ret void
+}
+
+define void @void_func_i64(i64 %arg0) #0 {
+  ; CHECK-LABEL: name: void_func_i64
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY]](s32), [[COPY1]](s32)
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   G_STORE [[MV]](s64), [[DEF]](p1) :: (store 8 into `i64 addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[COPY3:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY2]]
+  ; CHECK:   S_SETPC_B64_return [[COPY3]]
+  store i64 %arg0, i64 addrspace(1)* undef
+  ret void
+}
+
+define void @void_func_p0i8(i8* %arg0) #0 {
+  ; CHECK-LABEL: name: void_func_p0i8
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[MV:%[0-9]+]]:_(p0) = G_MERGE_VALUES [[COPY]](s32), [[COPY1]](s32)
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   G_STORE [[MV]](p0), [[DEF]](p1) :: (store 8 into `i8* addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[COPY3:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY2]]
+  ; CHECK:   S_SETPC_B64_return [[COPY3]]
+  store i8* %arg0, i8* addrspace(1)* undef
+  ret void
+}
+
+define void @void_func_p1i8(i8 addrspace(1)* %arg0) #0 {
+  ; CHECK-LABEL: name: void_func_p1i8
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY]](s32), [[COPY1]](s32)
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   G_STORE [[MV]](p1), [[DEF]](p1) :: (store 8 into `i8 addrspace(1)* addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[COPY3:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY2]]
+  ; CHECK:   S_SETPC_B64_return [[COPY3]]
+  store i8 addrspace(1)* %arg0, i8 addrspace(1)* addrspace(1)* undef
+  ret void
+}
+
+define void @void_func_f16(half %arg0) #0 {
+  ; CHECK-LABEL: name: void_func_f16
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   G_STORE [[TRUNC]](s16), [[DEF]](p1) :: (store 2 into `half addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[COPY2:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY1]]
+  ; CHECK:   S_SETPC_B64_return [[COPY2]]
+  store half %arg0, half addrspace(1)* undef
+  ret void
+}
+
+define void @void_func_f32(float %arg0) #0 {
+  ; CHECK-LABEL: name: void_func_f32
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   G_STORE [[COPY]](s32), [[DEF]](p1) :: (store 4 into `float addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[COPY2:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY1]]
+  ; CHECK:   S_SETPC_B64_return [[COPY2]]
+  store float %arg0, float addrspace(1)* undef
+  ret void
+}
+
+define void @void_func_f64(double %arg0) #0 {
+  ; CHECK-LABEL: name: void_func_f64
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY]](s32), [[COPY1]](s32)
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   G_STORE [[MV]](s64), [[DEF]](p1) :: (store 8 into `double addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[COPY3:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY2]]
+  ; CHECK:   S_SETPC_B64_return [[COPY3]]
+  store double %arg0, double addrspace(1)* undef
+  ret void
+}
+
+define void @void_func_v2i32(<2 x i32> %arg0) #0 {
+  ; CHECK-LABEL: name: void_func_v2i32
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32)
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   G_STORE [[BUILD_VECTOR]](<2 x s32>), [[DEF]](p1) :: (store 8 into `<2 x i32> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[COPY3:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY2]]
+  ; CHECK:   S_SETPC_B64_return [[COPY3]]
+  store <2 x i32> %arg0, <2 x i32> addrspace(1)* undef
+  ret void
+}
+
+define void @void_func_v2p3i8(<2 x i8 addrspace(3)*> %arg0) #0 {
+  ; CHECK-LABEL: name: void_func_v2p3i8
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(p3) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<2 x p3>) = G_BUILD_VECTOR [[COPY]](p3), [[COPY1]](p3)
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   G_STORE [[BUILD_VECTOR]](<2 x p3>), [[DEF]](p1) :: (store 8 into `<2 x i8 addrspace(3)*> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[COPY3:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY2]]
+  ; CHECK:   S_SETPC_B64_return [[COPY3]]
+  store <2 x i8 addrspace(3)*> %arg0, <2 x i8 addrspace(3)*> addrspace(1)* undef
+  ret void
+}
+
+define void @void_func_v3i32(<3 x i32> %arg0) #0 {
+  ; CHECK-LABEL: name: void_func_v3i32
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $vgpr2, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32)
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   G_STORE [[BUILD_VECTOR]](<3 x s32>), [[DEF]](p1) :: (store 12 into `<3 x i32> addrspace(1)* undef`, align 16, addrspace 1)
+  ; CHECK:   [[COPY4:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY3]]
+  ; CHECK:   S_SETPC_B64_return [[COPY4]]
+  store <3 x i32> %arg0, <3 x i32> addrspace(1)* undef
+  ret void
+}
+
+define void @void_func_v4i32(<4 x i32> %arg0) #0 {
+  ; CHECK-LABEL: name: void_func_v4i32
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32)
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   G_STORE [[BUILD_VECTOR]](<4 x s32>), [[DEF]](p1) :: (store 16 into `<4 x i32> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[COPY5:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY4]]
+  ; CHECK:   S_SETPC_B64_return [[COPY5]]
+  store <4 x i32> %arg0, <4 x i32> addrspace(1)* undef
+  ret void
+}
+
+define void @void_func_v5i32(<5 x i32> %arg0) #0 {
+  ; CHECK-LABEL: name: void_func_v5i32
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<5 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32)
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   G_STORE [[BUILD_VECTOR]](<5 x s32>), [[DEF]](p1) :: (store 20 into `<5 x i32> addrspace(1)* undef`, align 32, addrspace 1)
+  ; CHECK:   [[COPY6:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY5]]
+  ; CHECK:   S_SETPC_B64_return [[COPY6]]
+  store <5 x i32> %arg0, <5 x i32> addrspace(1)* undef
+  ret void
+}
+
+define void @void_func_v8i32(<8 x i32> %arg0) #0 {
+  ; CHECK-LABEL: name: void_func_v8i32
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
+  ; CHECK:   [[COPY6:%[0-9]+]]:_(s32) = COPY $vgpr6
+  ; CHECK:   [[COPY7:%[0-9]+]]:_(s32) = COPY $vgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   G_STORE [[BUILD_VECTOR]](<8 x s32>), [[DEF]](p1) :: (store 32 into `<8 x i32> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[COPY9:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY8]]
+  ; CHECK:   S_SETPC_B64_return [[COPY9]]
+  store <8 x i32> %arg0, <8 x i32> addrspace(1)* undef
+  ret void
+}
+
+define void @void_func_v16i32(<16 x i32> %arg0) #0 {
+  ; CHECK-LABEL: name: void_func_v16i32
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10, $vgpr11, $vgpr12, $vgpr13, $vgpr14, $vgpr15, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
+  ; CHECK:   [[COPY6:%[0-9]+]]:_(s32) = COPY $vgpr6
+  ; CHECK:   [[COPY7:%[0-9]+]]:_(s32) = COPY $vgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr8
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr9
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr10
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(s32) = COPY $vgpr11
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr12
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr13
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr14
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr15
+  ; CHECK:   [[COPY16:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<16 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32), [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32), [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[COPY15]](s32)
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   G_STORE [[BUILD_VECTOR]](<16 x s32>), [[DEF]](p1) :: (store 64 into `<16 x i32> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[COPY17:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY16]]
+  ; CHECK:   S_SETPC_B64_return [[COPY17]]
+  store <16 x i32> %arg0, <16 x i32> addrspace(1)* undef
+  ret void
+}
+
+define void @void_func_v32i32(<32 x i32> %arg0) #0 {
+  ; CHECK-LABEL: name: void_func_v32i32
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10, $vgpr11, $vgpr12, $vgpr13, $vgpr14, $vgpr15, $vgpr16, $vgpr17, $vgpr18, $vgpr19, $vgpr20, $vgpr21, $vgpr22, $vgpr23, $vgpr24, $vgpr25, $vgpr26, $vgpr27, $vgpr28, $vgpr29, $vgpr30, $vgpr31, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
+  ; CHECK:   [[COPY6:%[0-9]+]]:_(s32) = COPY $vgpr6
+  ; CHECK:   [[COPY7:%[0-9]+]]:_(s32) = COPY $vgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr8
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr9
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr10
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(s32) = COPY $vgpr11
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr12
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr13
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr14
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr15
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY $vgpr16
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY $vgpr17
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY $vgpr18
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY $vgpr19
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(s32) = COPY $vgpr20
+  ; CHECK:   [[COPY21:%[0-9]+]]:_(s32) = COPY $vgpr21
+  ; CHECK:   [[COPY22:%[0-9]+]]:_(s32) = COPY $vgpr22
+  ; CHECK:   [[COPY23:%[0-9]+]]:_(s32) = COPY $vgpr23
+  ; CHECK:   [[COPY24:%[0-9]+]]:_(s32) = COPY $vgpr24
+  ; CHECK:   [[COPY25:%[0-9]+]]:_(s32) = COPY $vgpr25
+  ; CHECK:   [[COPY26:%[0-9]+]]:_(s32) = COPY $vgpr26
+  ; CHECK:   [[COPY27:%[0-9]+]]:_(s32) = COPY $vgpr27
+  ; CHECK:   [[COPY28:%[0-9]+]]:_(s32) = COPY $vgpr28
+  ; CHECK:   [[COPY29:%[0-9]+]]:_(s32) = COPY $vgpr29
+  ; CHECK:   [[COPY30:%[0-9]+]]:_(s32) = COPY $vgpr30
+  ; CHECK:   [[COPY31:%[0-9]+]]:_(s32) = COPY $vgpr31
+  ; CHECK:   [[COPY32:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<32 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32), [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32), [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[COPY15]](s32), [[COPY16]](s32), [[COPY17]](s32), [[COPY18]](s32), [[COPY19]](s32), [[COPY20]](s32), [[COPY21]](s32), [[COPY22]](s32), [[COPY23]](s32), [[COPY24]](s32), [[COPY25]](s32), [[COPY26]](s32), [[COPY27]](s32), [[COPY28]](s32), [[COPY29]](s32), [[COPY30]](s32), [[COPY31]](s32)
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   G_STORE [[BUILD_VECTOR]](<32 x s32>), [[DEF]](p1) :: (store 128 into `<32 x i32> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[COPY33:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY32]]
+  ; CHECK:   S_SETPC_B64_return [[COPY33]]
+  store <32 x i32> %arg0, <32 x i32> addrspace(1)* undef
+  ret void
+}
+
+; 1 over register limit
+define void @void_func_v33i32(<33 x i32> %arg0) #0 {
+  ; CHECK-LABEL: name: void_func_v33i32
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10, $vgpr11, $vgpr12, $vgpr13, $vgpr14, $vgpr15, $vgpr16, $vgpr17, $vgpr18, $vgpr19, $vgpr20, $vgpr21, $vgpr22, $vgpr23, $vgpr24, $vgpr25, $vgpr26, $vgpr27, $vgpr28, $vgpr29, $vgpr30, $vgpr31, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
+  ; CHECK:   [[COPY6:%[0-9]+]]:_(s32) = COPY $vgpr6
+  ; CHECK:   [[COPY7:%[0-9]+]]:_(s32) = COPY $vgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr8
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr9
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr10
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(s32) = COPY $vgpr11
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr12
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr13
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr14
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr15
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY $vgpr16
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY $vgpr17
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY $vgpr18
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY $vgpr19
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(s32) = COPY $vgpr20
+  ; CHECK:   [[COPY21:%[0-9]+]]:_(s32) = COPY $vgpr21
+  ; CHECK:   [[COPY22:%[0-9]+]]:_(s32) = COPY $vgpr22
+  ; CHECK:   [[COPY23:%[0-9]+]]:_(s32) = COPY $vgpr23
+  ; CHECK:   [[COPY24:%[0-9]+]]:_(s32) = COPY $vgpr24
+  ; CHECK:   [[COPY25:%[0-9]+]]:_(s32) = COPY $vgpr25
+  ; CHECK:   [[COPY26:%[0-9]+]]:_(s32) = COPY $vgpr26
+  ; CHECK:   [[COPY27:%[0-9]+]]:_(s32) = COPY $vgpr27
+  ; CHECK:   [[COPY28:%[0-9]+]]:_(s32) = COPY $vgpr28
+  ; CHECK:   [[COPY29:%[0-9]+]]:_(s32) = COPY $vgpr29
+  ; CHECK:   [[COPY30:%[0-9]+]]:_(s32) = COPY $vgpr30
+  ; CHECK:   [[COPY31:%[0-9]+]]:_(s32) = COPY $vgpr31
+  ; CHECK:   [[FRAME_INDEX:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.0
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX]](p5) :: (invariant load 4 from %fixed-stack.0, align 1, addrspace 5)
+  ; CHECK:   [[COPY32:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<33 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32), [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32), [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[COPY15]](s32), [[COPY16]](s32), [[COPY17]](s32), [[COPY18]](s32), [[COPY19]](s32), [[COPY20]](s32), [[COPY21]](s32), [[COPY22]](s32), [[COPY23]](s32), [[COPY24]](s32), [[COPY25]](s32), [[COPY26]](s32), [[COPY27]](s32), [[COPY28]](s32), [[COPY29]](s32), [[COPY30]](s32), [[COPY31]](s32), [[LOAD]](s32)
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   G_STORE [[BUILD_VECTOR]](<33 x s32>), [[DEF]](p1) :: (store 132 into `<33 x i32> addrspace(1)* undef`, align 256, addrspace 1)
+  ; CHECK:   [[COPY33:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY32]]
+  ; CHECK:   S_SETPC_B64_return [[COPY33]]
+  store <33 x i32> %arg0, <33 x i32> addrspace(1)* undef
+  ret void
+}
+
+define void @void_func_v2i64(<2 x i64> %arg0) #0 {
+  ; CHECK-LABEL: name: void_func_v2i64
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY]](s32), [[COPY1]](s32)
+  ; CHECK:   [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY2]](s32), [[COPY3]](s32)
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64)
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   G_STORE [[BUILD_VECTOR]](<2 x s64>), [[DEF]](p1) :: (store 16 into `<2 x i64> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[COPY5:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY4]]
+  ; CHECK:   S_SETPC_B64_return [[COPY5]]
+  store <2 x i64> %arg0, <2 x i64> addrspace(1)* undef
+  ret void
+}
+
+define void @void_func_v2p0i8(<2 x i8*> %arg0) #0 {
+  ; CHECK-LABEL: name: void_func_v2p0i8
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[MV:%[0-9]+]]:_(p0) = G_MERGE_VALUES [[COPY]](s32), [[COPY1]](s32)
+  ; CHECK:   [[MV1:%[0-9]+]]:_(p0) = G_MERGE_VALUES [[COPY2]](s32), [[COPY3]](s32)
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<2 x p0>) = G_BUILD_VECTOR [[MV]](p0), [[MV1]](p0)
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   G_STORE [[BUILD_VECTOR]](<2 x p0>), [[DEF]](p1) :: (store 16 into `<2 x i8*> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[COPY5:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY4]]
+  ; CHECK:   S_SETPC_B64_return [[COPY5]]
+  store <2 x i8*> %arg0, <2 x i8*> addrspace(1)* undef
+  ret void
+}
+
+define void @void_func_v2p1i8(<2 x i8 addrspace(1)*> %arg0) #0 {
+  ; CHECK-LABEL: name: void_func_v2p1i8
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY]](s32), [[COPY1]](s32)
+  ; CHECK:   [[MV1:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY2]](s32), [[COPY3]](s32)
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<2 x p1>) = G_BUILD_VECTOR [[MV]](p1), [[MV1]](p1)
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   G_STORE [[BUILD_VECTOR]](<2 x p1>), [[DEF]](p1) :: (store 16 into `<2 x i8 addrspace(1)*> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[COPY5:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY4]]
+  ; CHECK:   S_SETPC_B64_return [[COPY5]]
+  store <2 x i8 addrspace(1)*> %arg0, <2 x i8 addrspace(1)*> addrspace(1)* undef
+  ret void
+}
+
+define void @void_func_v3i64(<3 x i64> %arg0) #0 {
+  ; CHECK-LABEL: name: void_func_v3i64
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY]](s32), [[COPY1]](s32)
+  ; CHECK:   [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY2]](s32), [[COPY3]](s32)
+  ; CHECK:   [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY4]](s32), [[COPY5]](s32)
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64), [[MV2]](s64)
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   G_STORE [[BUILD_VECTOR]](<3 x s64>), [[DEF]](p1) :: (store 24 into `<3 x i64> addrspace(1)* undef`, align 32, addrspace 1)
+  ; CHECK:   [[COPY7:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY6]]
+  ; CHECK:   S_SETPC_B64_return [[COPY7]]
+  store <3 x i64> %arg0, <3 x i64> addrspace(1)* undef
+  ret void
+}
+
+define void @void_func_v4i64(<4 x i64> %arg0) #0 {
+  ; CHECK-LABEL: name: void_func_v4i64
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
+  ; CHECK:   [[COPY6:%[0-9]+]]:_(s32) = COPY $vgpr6
+  ; CHECK:   [[COPY7:%[0-9]+]]:_(s32) = COPY $vgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY]](s32), [[COPY1]](s32)
+  ; CHECK:   [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY2]](s32), [[COPY3]](s32)
+  ; CHECK:   [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY4]](s32), [[COPY5]](s32)
+  ; CHECK:   [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY6]](s32), [[COPY7]](s32)
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64), [[MV2]](s64), [[MV3]](s64)
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   G_STORE [[BUILD_VECTOR]](<4 x s64>), [[DEF]](p1) :: (store 32 into `<4 x i64> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[COPY9:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY8]]
+  ; CHECK:   S_SETPC_B64_return [[COPY9]]
+  store <4 x i64> %arg0, <4 x i64> addrspace(1)* undef
+  ret void
+}
+
+define void @void_func_v5i64(<5 x i64> %arg0) #0 {
+  ; CHECK-LABEL: name: void_func_v5i64
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
+  ; CHECK:   [[COPY6:%[0-9]+]]:_(s32) = COPY $vgpr6
+  ; CHECK:   [[COPY7:%[0-9]+]]:_(s32) = COPY $vgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr8
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr9
+  ; CHECK:   [[COPY10:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY]](s32), [[COPY1]](s32)
+  ; CHECK:   [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY2]](s32), [[COPY3]](s32)
+  ; CHECK:   [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY4]](s32), [[COPY5]](s32)
+  ; CHECK:   [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY6]](s32), [[COPY7]](s32)
+  ; CHECK:   [[MV4:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY8]](s32), [[COPY9]](s32)
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<5 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64), [[MV2]](s64), [[MV3]](s64), [[MV4]](s64)
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   G_STORE [[BUILD_VECTOR]](<5 x s64>), [[DEF]](p1) :: (store 40 into `<5 x i64> addrspace(1)* undef`, align 64, addrspace 1)
+  ; CHECK:   [[COPY11:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY10]]
+  ; CHECK:   S_SETPC_B64_return [[COPY11]]
+  store <5 x i64> %arg0, <5 x i64> addrspace(1)* undef
+  ret void
+}
+
+define void @void_func_v8i64(<8 x i64> %arg0) #0 {
+  ; CHECK-LABEL: name: void_func_v8i64
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10, $vgpr11, $vgpr12, $vgpr13, $vgpr14, $vgpr15, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
+  ; CHECK:   [[COPY6:%[0-9]+]]:_(s32) = COPY $vgpr6
+  ; CHECK:   [[COPY7:%[0-9]+]]:_(s32) = COPY $vgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr8
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr9
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr10
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(s32) = COPY $vgpr11
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr12
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr13
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr14
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr15
+  ; CHECK:   [[COPY16:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY]](s32), [[COPY1]](s32)
+  ; CHECK:   [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY2]](s32), [[COPY3]](s32)
+  ; CHECK:   [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY4]](s32), [[COPY5]](s32)
+  ; CHECK:   [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY6]](s32), [[COPY7]](s32)
+  ; CHECK:   [[MV4:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY8]](s32), [[COPY9]](s32)
+  ; CHECK:   [[MV5:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY10]](s32), [[COPY11]](s32)
+  ; CHECK:   [[MV6:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY12]](s32), [[COPY13]](s32)
+  ; CHECK:   [[MV7:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY14]](s32), [[COPY15]](s32)
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64), [[MV2]](s64), [[MV3]](s64), [[MV4]](s64), [[MV5]](s64), [[MV6]](s64), [[MV7]](s64)
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   G_STORE [[BUILD_VECTOR]](<8 x s64>), [[DEF]](p1) :: (store 64 into `<8 x i64> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[COPY17:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY16]]
+  ; CHECK:   S_SETPC_B64_return [[COPY17]]
+  store <8 x i64> %arg0, <8 x i64> addrspace(1)* undef
+  ret void
+}
+
+define void @void_func_v16i64(<16 x i64> %arg0) #0 {
+  ; CHECK-LABEL: name: void_func_v16i64
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10, $vgpr11, $vgpr12, $vgpr13, $vgpr14, $vgpr15, $vgpr16, $vgpr17, $vgpr18, $vgpr19, $vgpr20, $vgpr21, $vgpr22, $vgpr23, $vgpr24, $vgpr25, $vgpr26, $vgpr27, $vgpr28, $vgpr29, $vgpr30, $vgpr31, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
+  ; CHECK:   [[COPY6:%[0-9]+]]:_(s32) = COPY $vgpr6
+  ; CHECK:   [[COPY7:%[0-9]+]]:_(s32) = COPY $vgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr8
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr9
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr10
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(s32) = COPY $vgpr11
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr12
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr13
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr14
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr15
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY $vgpr16
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY $vgpr17
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY $vgpr18
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY $vgpr19
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(s32) = COPY $vgpr20
+  ; CHECK:   [[COPY21:%[0-9]+]]:_(s32) = COPY $vgpr21
+  ; CHECK:   [[COPY22:%[0-9]+]]:_(s32) = COPY $vgpr22
+  ; CHECK:   [[COPY23:%[0-9]+]]:_(s32) = COPY $vgpr23
+  ; CHECK:   [[COPY24:%[0-9]+]]:_(s32) = COPY $vgpr24
+  ; CHECK:   [[COPY25:%[0-9]+]]:_(s32) = COPY $vgpr25
+  ; CHECK:   [[COPY26:%[0-9]+]]:_(s32) = COPY $vgpr26
+  ; CHECK:   [[COPY27:%[0-9]+]]:_(s32) = COPY $vgpr27
+  ; CHECK:   [[COPY28:%[0-9]+]]:_(s32) = COPY $vgpr28
+  ; CHECK:   [[COPY29:%[0-9]+]]:_(s32) = COPY $vgpr29
+  ; CHECK:   [[COPY30:%[0-9]+]]:_(s32) = COPY $vgpr30
+  ; CHECK:   [[COPY31:%[0-9]+]]:_(s32) = COPY $vgpr31
+  ; CHECK:   [[COPY32:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY]](s32), [[COPY1]](s32)
+  ; CHECK:   [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY2]](s32), [[COPY3]](s32)
+  ; CHECK:   [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY4]](s32), [[COPY5]](s32)
+  ; CHECK:   [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY6]](s32), [[COPY7]](s32)
+  ; CHECK:   [[MV4:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY8]](s32), [[COPY9]](s32)
+  ; CHECK:   [[MV5:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY10]](s32), [[COPY11]](s32)
+  ; CHECK:   [[MV6:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY12]](s32), [[COPY13]](s32)
+  ; CHECK:   [[MV7:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY14]](s32), [[COPY15]](s32)
+  ; CHECK:   [[MV8:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY16]](s32), [[COPY17]](s32)
+  ; CHECK:   [[MV9:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY18]](s32), [[COPY19]](s32)
+  ; CHECK:   [[MV10:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY20]](s32), [[COPY21]](s32)
+  ; CHECK:   [[MV11:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY22]](s32), [[COPY23]](s32)
+  ; CHECK:   [[MV12:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY24]](s32), [[COPY25]](s32)
+  ; CHECK:   [[MV13:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY26]](s32), [[COPY27]](s32)
+  ; CHECK:   [[MV14:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY28]](s32), [[COPY29]](s32)
+  ; CHECK:   [[MV15:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY30]](s32), [[COPY31]](s32)
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<16 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64), [[MV2]](s64), [[MV3]](s64), [[MV4]](s64), [[MV5]](s64), [[MV6]](s64), [[MV7]](s64), [[MV8]](s64), [[MV9]](s64), [[MV10]](s64), [[MV11]](s64), [[MV12]](s64), [[MV13]](s64), [[MV14]](s64), [[MV15]](s64)
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   G_STORE [[BUILD_VECTOR]](<16 x s64>), [[DEF]](p1) :: (store 128 into `<16 x i64> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[COPY33:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY32]]
+  ; CHECK:   S_SETPC_B64_return [[COPY33]]
+  store <16 x i64> %arg0, <16 x i64> addrspace(1)* undef
+  ret void
+}
+
+define void @void_func_v2i16(<2 x i16> %arg0) #0 {
+  ; CHECK-LABEL: name: void_func_v2i16
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32)
+  ; CHECK:   [[TRUNC:%[0-9]+]]:_(<2 x s16>) = G_TRUNC [[BUILD_VECTOR]](<2 x s32>)
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   G_STORE [[TRUNC]](<2 x s16>), [[DEF]](p1) :: (store 4 into `<2 x i16> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[COPY3:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY2]]
+  ; CHECK:   S_SETPC_B64_return [[COPY3]]
+  store <2 x i16> %arg0, <2 x i16> addrspace(1)* undef
+  ret void
+}
+
+define void @void_func_v3i16(<3 x i16> %arg0) #0 {
+  ; CHECK-LABEL: name: void_func_v3i16
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $vgpr2, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32)
+  ; CHECK:   [[TRUNC:%[0-9]+]]:_(<3 x s16>) = G_TRUNC [[BUILD_VECTOR]](<3 x s32>)
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   G_STORE [[TRUNC]](<3 x s16>), [[DEF]](p1) :: (store 6 into `<3 x i16> addrspace(1)* undef`, align 8, addrspace 1)
+  ; CHECK:   [[COPY4:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY3]]
+  ; CHECK:   S_SETPC_B64_return [[COPY4]]
+  store <3 x i16> %arg0, <3 x i16> addrspace(1)* undef
+  ret void
+}
+
+define void @void_func_v4i16(<4 x i16> %arg0) #0 {
+  ; CHECK-LABEL: name: void_func_v4i16
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32)
+  ; CHECK:   [[TRUNC:%[0-9]+]]:_(<4 x s16>) = G_TRUNC [[BUILD_VECTOR]](<4 x s32>)
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   G_STORE [[TRUNC]](<4 x s16>), [[DEF]](p1) :: (store 8 into `<4 x i16> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[COPY5:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY4]]
+  ; CHECK:   S_SETPC_B64_return [[COPY5]]
+  store <4 x i16> %arg0, <4 x i16> addrspace(1)* undef
+  ret void
+}
+
+define void @void_func_v5i16(<5 x i16> %arg0) #0 {
+  ; CHECK-LABEL: name: void_func_v5i16
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<5 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32)
+  ; CHECK:   [[TRUNC:%[0-9]+]]:_(<5 x s16>) = G_TRUNC [[BUILD_VECTOR]](<5 x s32>)
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   G_STORE [[TRUNC]](<5 x s16>), [[DEF]](p1) :: (store 10 into `<5 x i16> addrspace(1)* undef`, align 16, addrspace 1)
+  ; CHECK:   [[COPY6:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY5]]
+  ; CHECK:   S_SETPC_B64_return [[COPY6]]
+  store <5 x i16> %arg0, <5 x i16> addrspace(1)* undef
+  ret void
+}
+
+define void @void_func_v8i16(<8 x i16> %arg0) #0 {
+  ; CHECK-LABEL: name: void_func_v8i16
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
+  ; CHECK:   [[COPY6:%[0-9]+]]:_(s32) = COPY $vgpr6
+  ; CHECK:   [[COPY7:%[0-9]+]]:_(s32) = COPY $vgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; CHECK:   [[TRUNC:%[0-9]+]]:_(<8 x s16>) = G_TRUNC [[BUILD_VECTOR]](<8 x s32>)
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   G_STORE [[TRUNC]](<8 x s16>), [[DEF]](p1) :: (store 16 into `<8 x i16> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[COPY9:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY8]]
+  ; CHECK:   S_SETPC_B64_return [[COPY9]]
+  store <8 x i16> %arg0, <8 x i16> addrspace(1)* undef
+  ret void
+}
+
+define void @void_func_v16i16(<16 x i16> %arg0) #0 {
+  ; CHECK-LABEL: name: void_func_v16i16
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10, $vgpr11, $vgpr12, $vgpr13, $vgpr14, $vgpr15, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
+  ; CHECK:   [[COPY6:%[0-9]+]]:_(s32) = COPY $vgpr6
+  ; CHECK:   [[COPY7:%[0-9]+]]:_(s32) = COPY $vgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr8
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr9
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr10
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(s32) = COPY $vgpr11
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr12
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr13
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr14
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr15
+  ; CHECK:   [[COPY16:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<16 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32), [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32), [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[COPY15]](s32)
+  ; CHECK:   [[TRUNC:%[0-9]+]]:_(<16 x s16>) = G_TRUNC [[BUILD_VECTOR]](<16 x s32>)
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   G_STORE [[TRUNC]](<16 x s16>), [[DEF]](p1) :: (store 32 into `<16 x i16> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[COPY17:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY16]]
+  ; CHECK:   S_SETPC_B64_return [[COPY17]]
+  store <16 x i16> %arg0, <16 x i16> addrspace(1)* undef
+  ret void
+}
+
+define void @void_func_v2f32(<2 x float> %arg0) #0 {
+  ; CHECK-LABEL: name: void_func_v2f32
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32)
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   G_STORE [[BUILD_VECTOR]](<2 x s32>), [[DEF]](p1) :: (store 8 into `<2 x float> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[COPY3:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY2]]
+  ; CHECK:   S_SETPC_B64_return [[COPY3]]
+  store <2 x float> %arg0, <2 x float> addrspace(1)* undef
+  ret void
+}
+
+define void @void_func_v3f32(<3 x float> %arg0) #0 {
+  ; CHECK-LABEL: name: void_func_v3f32
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $vgpr2, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32)
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   G_STORE [[BUILD_VECTOR]](<3 x s32>), [[DEF]](p1) :: (store 12 into `<3 x float> addrspace(1)* undef`, align 16, addrspace 1)
+  ; CHECK:   [[COPY4:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY3]]
+  ; CHECK:   S_SETPC_B64_return [[COPY4]]
+  store <3 x float> %arg0, <3 x float> addrspace(1)* undef
+  ret void
+}
+
+define void @void_func_v4f32(<4 x float> %arg0) #0 {
+  ; CHECK-LABEL: name: void_func_v4f32
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32)
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   G_STORE [[BUILD_VECTOR]](<4 x s32>), [[DEF]](p1) :: (store 16 into `<4 x float> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[COPY5:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY4]]
+  ; CHECK:   S_SETPC_B64_return [[COPY5]]
+  store <4 x float> %arg0, <4 x float> addrspace(1)* undef
+  ret void
+}
+
+define void @void_func_v8f32(<8 x float> %arg0) #0 {
+  ; CHECK-LABEL: name: void_func_v8f32
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
+  ; CHECK:   [[COPY6:%[0-9]+]]:_(s32) = COPY $vgpr6
+  ; CHECK:   [[COPY7:%[0-9]+]]:_(s32) = COPY $vgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   G_STORE [[BUILD_VECTOR]](<8 x s32>), [[DEF]](p1) :: (store 32 into `<8 x float> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[COPY9:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY8]]
+  ; CHECK:   S_SETPC_B64_return [[COPY9]]
+  store <8 x float> %arg0, <8 x float> addrspace(1)* undef
+  ret void
+}
+
+define void @void_func_v16f32(<16 x float> %arg0) #0 {
+  ; CHECK-LABEL: name: void_func_v16f32
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10, $vgpr11, $vgpr12, $vgpr13, $vgpr14, $vgpr15, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
+  ; CHECK:   [[COPY6:%[0-9]+]]:_(s32) = COPY $vgpr6
+  ; CHECK:   [[COPY7:%[0-9]+]]:_(s32) = COPY $vgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr8
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr9
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr10
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(s32) = COPY $vgpr11
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr12
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr13
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr14
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr15
+  ; CHECK:   [[COPY16:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<16 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32), [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32), [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[COPY15]](s32)
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   G_STORE [[BUILD_VECTOR]](<16 x s32>), [[DEF]](p1) :: (store 64 into `<16 x float> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[COPY17:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY16]]
+  ; CHECK:   S_SETPC_B64_return [[COPY17]]
+  store <16 x float> %arg0, <16 x float> addrspace(1)* undef
+  ret void
+}
+
+define void @void_func_v2f64(<2 x double> %arg0) #0 {
+  ; CHECK-LABEL: name: void_func_v2f64
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY]](s32), [[COPY1]](s32)
+  ; CHECK:   [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY2]](s32), [[COPY3]](s32)
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64)
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   G_STORE [[BUILD_VECTOR]](<2 x s64>), [[DEF]](p1) :: (store 16 into `<2 x double> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[COPY5:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY4]]
+  ; CHECK:   S_SETPC_B64_return [[COPY5]]
+  store <2 x double> %arg0, <2 x double> addrspace(1)* undef
+  ret void
+}
+
+define void @void_func_v3f64(<3 x double> %arg0) #0 {
+  ; CHECK-LABEL: name: void_func_v3f64
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY]](s32), [[COPY1]](s32)
+  ; CHECK:   [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY2]](s32), [[COPY3]](s32)
+  ; CHECK:   [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY4]](s32), [[COPY5]](s32)
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64), [[MV2]](s64)
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   G_STORE [[BUILD_VECTOR]](<3 x s64>), [[DEF]](p1) :: (store 24 into `<3 x double> addrspace(1)* undef`, align 32, addrspace 1)
+  ; CHECK:   [[COPY7:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY6]]
+  ; CHECK:   S_SETPC_B64_return [[COPY7]]
+  store <3 x double> %arg0, <3 x double> addrspace(1)* undef
+  ret void
+}
+
+define void @void_func_v4f64(<4 x double> %arg0) #0 {
+  ; CHECK-LABEL: name: void_func_v4f64
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
+  ; CHECK:   [[COPY6:%[0-9]+]]:_(s32) = COPY $vgpr6
+  ; CHECK:   [[COPY7:%[0-9]+]]:_(s32) = COPY $vgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY]](s32), [[COPY1]](s32)
+  ; CHECK:   [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY2]](s32), [[COPY3]](s32)
+  ; CHECK:   [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY4]](s32), [[COPY5]](s32)
+  ; CHECK:   [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY6]](s32), [[COPY7]](s32)
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64), [[MV2]](s64), [[MV3]](s64)
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   G_STORE [[BUILD_VECTOR]](<4 x s64>), [[DEF]](p1) :: (store 32 into `<4 x double> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[COPY9:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY8]]
+  ; CHECK:   S_SETPC_B64_return [[COPY9]]
+  store <4 x double> %arg0, <4 x double> addrspace(1)* undef
+  ret void
+}
+
+define void @void_func_v8f64(<8 x double> %arg0) #0 {
+  ; CHECK-LABEL: name: void_func_v8f64
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10, $vgpr11, $vgpr12, $vgpr13, $vgpr14, $vgpr15, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
+  ; CHECK:   [[COPY6:%[0-9]+]]:_(s32) = COPY $vgpr6
+  ; CHECK:   [[COPY7:%[0-9]+]]:_(s32) = COPY $vgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr8
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr9
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr10
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(s32) = COPY $vgpr11
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr12
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr13
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr14
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr15
+  ; CHECK:   [[COPY16:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY]](s32), [[COPY1]](s32)
+  ; CHECK:   [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY2]](s32), [[COPY3]](s32)
+  ; CHECK:   [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY4]](s32), [[COPY5]](s32)
+  ; CHECK:   [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY6]](s32), [[COPY7]](s32)
+  ; CHECK:   [[MV4:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY8]](s32), [[COPY9]](s32)
+  ; CHECK:   [[MV5:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY10]](s32), [[COPY11]](s32)
+  ; CHECK:   [[MV6:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY12]](s32), [[COPY13]](s32)
+  ; CHECK:   [[MV7:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY14]](s32), [[COPY15]](s32)
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64), [[MV2]](s64), [[MV3]](s64), [[MV4]](s64), [[MV5]](s64), [[MV6]](s64), [[MV7]](s64)
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   G_STORE [[BUILD_VECTOR]](<8 x s64>), [[DEF]](p1) :: (store 64 into `<8 x double> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[COPY17:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY16]]
+  ; CHECK:   S_SETPC_B64_return [[COPY17]]
+  store <8 x double> %arg0, <8 x double> addrspace(1)* undef
+  ret void
+}
+
+define void @void_func_v16f64(<16 x double> %arg0) #0 {
+  ; CHECK-LABEL: name: void_func_v16f64
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10, $vgpr11, $vgpr12, $vgpr13, $vgpr14, $vgpr15, $vgpr16, $vgpr17, $vgpr18, $vgpr19, $vgpr20, $vgpr21, $vgpr22, $vgpr23, $vgpr24, $vgpr25, $vgpr26, $vgpr27, $vgpr28, $vgpr29, $vgpr30, $vgpr31, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
+  ; CHECK:   [[COPY6:%[0-9]+]]:_(s32) = COPY $vgpr6
+  ; CHECK:   [[COPY7:%[0-9]+]]:_(s32) = COPY $vgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr8
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr9
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr10
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(s32) = COPY $vgpr11
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr12
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr13
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr14
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr15
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY $vgpr16
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY $vgpr17
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY $vgpr18
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY $vgpr19
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(s32) = COPY $vgpr20
+  ; CHECK:   [[COPY21:%[0-9]+]]:_(s32) = COPY $vgpr21
+  ; CHECK:   [[COPY22:%[0-9]+]]:_(s32) = COPY $vgpr22
+  ; CHECK:   [[COPY23:%[0-9]+]]:_(s32) = COPY $vgpr23
+  ; CHECK:   [[COPY24:%[0-9]+]]:_(s32) = COPY $vgpr24
+  ; CHECK:   [[COPY25:%[0-9]+]]:_(s32) = COPY $vgpr25
+  ; CHECK:   [[COPY26:%[0-9]+]]:_(s32) = COPY $vgpr26
+  ; CHECK:   [[COPY27:%[0-9]+]]:_(s32) = COPY $vgpr27
+  ; CHECK:   [[COPY28:%[0-9]+]]:_(s32) = COPY $vgpr28
+  ; CHECK:   [[COPY29:%[0-9]+]]:_(s32) = COPY $vgpr29
+  ; CHECK:   [[COPY30:%[0-9]+]]:_(s32) = COPY $vgpr30
+  ; CHECK:   [[COPY31:%[0-9]+]]:_(s32) = COPY $vgpr31
+  ; CHECK:   [[COPY32:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY]](s32), [[COPY1]](s32)
+  ; CHECK:   [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY2]](s32), [[COPY3]](s32)
+  ; CHECK:   [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY4]](s32), [[COPY5]](s32)
+  ; CHECK:   [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY6]](s32), [[COPY7]](s32)
+  ; CHECK:   [[MV4:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY8]](s32), [[COPY9]](s32)
+  ; CHECK:   [[MV5:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY10]](s32), [[COPY11]](s32)
+  ; CHECK:   [[MV6:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY12]](s32), [[COPY13]](s32)
+  ; CHECK:   [[MV7:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY14]](s32), [[COPY15]](s32)
+  ; CHECK:   [[MV8:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY16]](s32), [[COPY17]](s32)
+  ; CHECK:   [[MV9:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY18]](s32), [[COPY19]](s32)
+  ; CHECK:   [[MV10:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY20]](s32), [[COPY21]](s32)
+  ; CHECK:   [[MV11:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY22]](s32), [[COPY23]](s32)
+  ; CHECK:   [[MV12:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY24]](s32), [[COPY25]](s32)
+  ; CHECK:   [[MV13:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY26]](s32), [[COPY27]](s32)
+  ; CHECK:   [[MV14:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY28]](s32), [[COPY29]](s32)
+  ; CHECK:   [[MV15:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY30]](s32), [[COPY31]](s32)
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<16 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64), [[MV2]](s64), [[MV3]](s64), [[MV4]](s64), [[MV5]](s64), [[MV6]](s64), [[MV7]](s64), [[MV8]](s64), [[MV9]](s64), [[MV10]](s64), [[MV11]](s64), [[MV12]](s64), [[MV13]](s64), [[MV14]](s64), [[MV15]](s64)
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   G_STORE [[BUILD_VECTOR]](<16 x s64>), [[DEF]](p1) :: (store 128 into `<16 x double> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[COPY33:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY32]]
+  ; CHECK:   S_SETPC_B64_return [[COPY33]]
+  store <16 x double> %arg0, <16 x double> addrspace(1)* undef
+  ret void
+}
+
+define void @void_func_v2f16(<2 x half> %arg0) #0 {
+  ; CHECK-LABEL: name: void_func_v2f16
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32)
+  ; CHECK:   [[TRUNC:%[0-9]+]]:_(<2 x s16>) = G_TRUNC [[BUILD_VECTOR]](<2 x s32>)
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   G_STORE [[TRUNC]](<2 x s16>), [[DEF]](p1) :: (store 4 into `<2 x half> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[COPY3:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY2]]
+  ; CHECK:   S_SETPC_B64_return [[COPY3]]
+  store <2 x half> %arg0, <2 x half> addrspace(1)* undef
+  ret void
+}
+
+define void @void_func_v3f16(<3 x half> %arg0) #0 {
+  ; CHECK-LABEL: name: void_func_v3f16
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $vgpr2, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32)
+  ; CHECK:   [[TRUNC:%[0-9]+]]:_(<3 x s16>) = G_TRUNC [[BUILD_VECTOR]](<3 x s32>)
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   G_STORE [[TRUNC]](<3 x s16>), [[DEF]](p1) :: (store 6 into `<3 x half> addrspace(1)* undef`, align 8, addrspace 1)
+  ; CHECK:   [[COPY4:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY3]]
+  ; CHECK:   S_SETPC_B64_return [[COPY4]]
+  store <3 x half> %arg0, <3 x half> addrspace(1)* undef
+  ret void
+}
+
+define void @void_func_v4f16(<4 x half> %arg0) #0 {
+  ; CHECK-LABEL: name: void_func_v4f16
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32)
+  ; CHECK:   [[TRUNC:%[0-9]+]]:_(<4 x s16>) = G_TRUNC [[BUILD_VECTOR]](<4 x s32>)
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   G_STORE [[TRUNC]](<4 x s16>), [[DEF]](p1) :: (store 8 into `<4 x half> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[COPY5:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY4]]
+  ; CHECK:   S_SETPC_B64_return [[COPY5]]
+  store <4 x half> %arg0, <4 x half> addrspace(1)* undef
+  ret void
+}
+
+define void @void_func_v8f16(<8 x half> %arg0) #0 {
+  ; CHECK-LABEL: name: void_func_v8f16
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
+  ; CHECK:   [[COPY6:%[0-9]+]]:_(s32) = COPY $vgpr6
+  ; CHECK:   [[COPY7:%[0-9]+]]:_(s32) = COPY $vgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; CHECK:   [[TRUNC:%[0-9]+]]:_(<8 x s16>) = G_TRUNC [[BUILD_VECTOR]](<8 x s32>)
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   G_STORE [[TRUNC]](<8 x s16>), [[DEF]](p1) :: (store 16 into `<8 x half> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[COPY9:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY8]]
+  ; CHECK:   S_SETPC_B64_return [[COPY9]]
+  store <8 x half> %arg0, <8 x half> addrspace(1)* undef
+  ret void
+}
+
+define void @void_func_v16f16(<16 x half> %arg0) #0 {
+  ; CHECK-LABEL: name: void_func_v16f16
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10, $vgpr11, $vgpr12, $vgpr13, $vgpr14, $vgpr15, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
+  ; CHECK:   [[COPY6:%[0-9]+]]:_(s32) = COPY $vgpr6
+  ; CHECK:   [[COPY7:%[0-9]+]]:_(s32) = COPY $vgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr8
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr9
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr10
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(s32) = COPY $vgpr11
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr12
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr13
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr14
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr15
+  ; CHECK:   [[COPY16:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<16 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32), [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32), [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[COPY15]](s32)
+  ; CHECK:   [[TRUNC:%[0-9]+]]:_(<16 x s16>) = G_TRUNC [[BUILD_VECTOR]](<16 x s32>)
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   G_STORE [[TRUNC]](<16 x s16>), [[DEF]](p1) :: (store 32 into `<16 x half> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[COPY17:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY16]]
+  ; CHECK:   S_SETPC_B64_return [[COPY17]]
+  store <16 x half> %arg0, <16 x half> addrspace(1)* undef
+  ret void
+}
+
+; Make sure there is no alignment requirement for passed vgprs.
+define void @void_func_i32_i64_i32(i32 %arg0, i64 %arg1, i32 %arg2) #0 {
+  ; CHECK-LABEL: name: void_func_i32_i64_i32
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY1]](s32), [[COPY2]](s32)
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[DEF1:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   G_STORE [[COPY]](s32), [[DEF]](p1) :: (volatile store 4 into `i32 addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   G_STORE [[MV]](s64), [[DEF1]](p1) :: (volatile store 8 into `i64 addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   G_STORE [[COPY3]](s32), [[DEF]](p1) :: (volatile store 4 into `i32 addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[COPY5:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY4]]
+  ; CHECK:   S_SETPC_B64_return [[COPY5]]
+  store volatile i32 %arg0, i32 addrspace(1)* undef
+  store volatile i64 %arg1, i64 addrspace(1)* undef
+  store volatile i32 %arg2, i32 addrspace(1)* undef
+  ret void
+}
+
+define void @void_func_struct_i32({ i32 } %arg0) #0 {
+  ; CHECK-LABEL: name: void_func_struct_i32
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   G_STORE [[COPY]](s32), [[DEF]](p1) :: (store 4 into `{ i32 } addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[COPY2:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY1]]
+  ; CHECK:   S_SETPC_B64_return [[COPY2]]
+  store { i32 } %arg0, { i32 } addrspace(1)* undef
+  ret void
+}
+
+define void @void_func_struct_i8_i32({ i8, i32 } %arg0) #0 {
+  ; CHECK-LABEL: name: void_func_struct_i8_i32
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[COPY]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   G_STORE [[TRUNC]](s8), [[DEF]](p1) :: (store 1 into `{ i8, i32 } addrspace(1)* undef`, align 4, addrspace 1)
+  ; CHECK:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[DEF]], [[C]](s64)
+  ; CHECK:   G_STORE [[COPY1]](s32), [[PTR_ADD]](p1) :: (store 4 into `{ i8, i32 } addrspace(1)* undef` + 4, addrspace 1)
+  ; CHECK:   [[COPY3:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY2]]
+  ; CHECK:   S_SETPC_B64_return [[COPY3]]
+  store { i8, i32 } %arg0, { i8, i32 } addrspace(1)* undef
+  ret void
+}
+
+define void @void_func_byval_struct_i8_i32({ i8, i32 } addrspace(5)* byval %arg0) #0 {
+  ; CHECK-LABEL: name: void_func_byval_struct_i8_i32
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr30_sgpr31
+  ; CHECK:   [[FRAME_INDEX:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.0
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(p5) = G_LOAD [[FRAME_INDEX]](p5) :: (invariant load 4 from %fixed-stack.0, align 1, addrspace 5)
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[LOAD1:%[0-9]+]]:_(s8) = G_LOAD [[LOAD]](p5) :: (load 1 from %ir.arg0, align 4, addrspace 5)
+  ; CHECK:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[LOAD]], [[C]](s32)
+  ; CHECK:   [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load 4 from %ir.arg0 + 4, addrspace 5)
+  ; CHECK:   G_STORE [[LOAD1]](s8), [[DEF]](p1) :: (store 1 into `{ i8, i32 } addrspace(1)* undef`, align 4, addrspace 1)
+  ; CHECK:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
+  ; CHECK:   [[PTR_ADD1:%[0-9]+]]:_(p1) = G_PTR_ADD [[DEF]], [[C1]](s64)
+  ; CHECK:   G_STORE [[LOAD2]](s32), [[PTR_ADD1]](p1) :: (store 4 into `{ i8, i32 } addrspace(1)* undef` + 4, addrspace 1)
+  ; CHECK:   [[COPY1:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+  ; CHECK:   S_SETPC_B64_return [[COPY1]]
+  %arg0.load = load { i8, i32 }, { i8, i32 } addrspace(5)* %arg0
+  store { i8, i32 } %arg0.load, { i8, i32 } addrspace(1)* undef
+  ret void
+}
+
+define void @void_func_byval_struct_i8_i32_x2({ i8, i32 } addrspace(5)* byval %arg0, { i8, i32 } addrspace(5)* byval %arg1, i32 %arg2) #0 {
+  ; CHECK-LABEL: name: void_func_byval_struct_i8_i32_x2
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $sgpr30_sgpr31
+  ; CHECK:   [[FRAME_INDEX:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.1
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(p5) = G_LOAD [[FRAME_INDEX]](p5) :: (invariant load 4 from %fixed-stack.1, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX1:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.0
+  ; CHECK:   [[LOAD1:%[0-9]+]]:_(p5) = G_LOAD [[FRAME_INDEX1]](p5) :: (invariant load 4 from %fixed-stack.0, align 1, addrspace 5)
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[DEF1:%[0-9]+]]:_(p3) = G_IMPLICIT_DEF
+  ; CHECK:   [[LOAD2:%[0-9]+]]:_(s8) = G_LOAD [[LOAD]](p5) :: (volatile load 1 from %ir.arg0, align 4, addrspace 5)
+  ; CHECK:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[LOAD]], [[C]](s32)
+  ; CHECK:   [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (volatile load 4 from %ir.arg0 + 4, addrspace 5)
+  ; CHECK:   [[LOAD4:%[0-9]+]]:_(s8) = G_LOAD [[LOAD1]](p5) :: (volatile load 1 from %ir.arg1, align 4, addrspace 5)
+  ; CHECK:   [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[LOAD1]], [[C]](s32)
+  ; CHECK:   [[LOAD5:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (volatile load 4 from %ir.arg1 + 4, addrspace 5)
+  ; CHECK:   G_STORE [[LOAD2]](s8), [[DEF]](p1) :: (volatile store 1 into `{ i8, i32 } addrspace(1)* undef`, align 4, addrspace 1)
+  ; CHECK:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
+  ; CHECK:   [[PTR_ADD2:%[0-9]+]]:_(p1) = G_PTR_ADD [[DEF]], [[C1]](s64)
+  ; CHECK:   G_STORE [[LOAD3]](s32), [[PTR_ADD2]](p1) :: (volatile store 4 into `{ i8, i32 } addrspace(1)* undef` + 4, addrspace 1)
+  ; CHECK:   G_STORE [[LOAD4]](s8), [[DEF]](p1) :: (volatile store 1 into `{ i8, i32 } addrspace(1)* undef`, align 4, addrspace 1)
+  ; CHECK:   [[PTR_ADD3:%[0-9]+]]:_(p1) = G_PTR_ADD [[DEF]], [[C1]](s64)
+  ; CHECK:   G_STORE [[LOAD5]](s32), [[PTR_ADD3]](p1) :: (volatile store 4 into `{ i8, i32 } addrspace(1)* undef` + 4, addrspace 1)
+  ; CHECK:   G_STORE [[COPY]](s32), [[DEF1]](p3) :: (volatile store 4 into `i32 addrspace(3)* undef`, addrspace 3)
+  ; CHECK:   [[COPY2:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY1]]
+  ; CHECK:   S_SETPC_B64_return [[COPY2]]
+  %arg0.load = load volatile { i8, i32 }, { i8, i32 } addrspace(5)* %arg0
+  %arg1.load = load volatile { i8, i32 }, { i8, i32 } addrspace(5)* %arg1
+  store volatile { i8, i32 } %arg0.load, { i8, i32 } addrspace(1)* undef
+  store volatile { i8, i32 } %arg1.load, { i8, i32 } addrspace(1)* undef
+  store volatile i32 %arg2, i32 addrspace(3)* undef
+  ret void
+}
+
+define void @void_func_byval_i32_byval_i64(i32 addrspace(5)* byval %arg0, i64 addrspace(5)* byval %arg1) #0 {
+  ; CHECK-LABEL: name: void_func_byval_i32_byval_i64
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr30_sgpr31
+  ; CHECK:   [[FRAME_INDEX:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.1
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(p5) = G_LOAD [[FRAME_INDEX]](p5) :: (invariant load 4 from %fixed-stack.1, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX1:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.0
+  ; CHECK:   [[LOAD1:%[0-9]+]]:_(p5) = G_LOAD [[FRAME_INDEX1]](p5) :: (invariant load 4 from %fixed-stack.0, align 1, addrspace 5)
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[DEF1:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[LOAD]](p5) :: (load 4 from %ir.arg0, addrspace 5)
+  ; CHECK:   [[LOAD3:%[0-9]+]]:_(s64) = G_LOAD [[LOAD1]](p5) :: (load 8 from %ir.arg1, addrspace 5)
+  ; CHECK:   G_STORE [[LOAD2]](s32), [[DEF]](p1) :: (store 4 into `i32 addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   G_STORE [[LOAD3]](s64), [[DEF1]](p1) :: (store 8 into `i64 addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[COPY1:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+  ; CHECK:   S_SETPC_B64_return [[COPY1]]
+  %arg0.load = load i32, i32 addrspace(5)* %arg0
+  %arg1.load = load i64, i64 addrspace(5)* %arg1
+  store i32 %arg0.load, i32 addrspace(1)* undef
+  store i64 %arg1.load, i64 addrspace(1)* undef
+  ret void
+}
+
+define void @void_func_v32i32_i32_i64(<32 x i32> %arg0, i32 %arg1, i64 %arg2) #0 {
+  ; CHECK-LABEL: name: void_func_v32i32_i32_i64
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10, $vgpr11, $vgpr12, $vgpr13, $vgpr14, $vgpr15, $vgpr16, $vgpr17, $vgpr18, $vgpr19, $vgpr20, $vgpr21, $vgpr22, $vgpr23, $vgpr24, $vgpr25, $vgpr26, $vgpr27, $vgpr28, $vgpr29, $vgpr30, $vgpr31, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
+  ; CHECK:   [[COPY6:%[0-9]+]]:_(s32) = COPY $vgpr6
+  ; CHECK:   [[COPY7:%[0-9]+]]:_(s32) = COPY $vgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr8
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr9
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr10
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(s32) = COPY $vgpr11
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr12
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr13
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr14
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr15
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY $vgpr16
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY $vgpr17
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY $vgpr18
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY $vgpr19
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(s32) = COPY $vgpr20
+  ; CHECK:   [[COPY21:%[0-9]+]]:_(s32) = COPY $vgpr21
+  ; CHECK:   [[COPY22:%[0-9]+]]:_(s32) = COPY $vgpr22
+  ; CHECK:   [[COPY23:%[0-9]+]]:_(s32) = COPY $vgpr23
+  ; CHECK:   [[COPY24:%[0-9]+]]:_(s32) = COPY $vgpr24
+  ; CHECK:   [[COPY25:%[0-9]+]]:_(s32) = COPY $vgpr25
+  ; CHECK:   [[COPY26:%[0-9]+]]:_(s32) = COPY $vgpr26
+  ; CHECK:   [[COPY27:%[0-9]+]]:_(s32) = COPY $vgpr27
+  ; CHECK:   [[COPY28:%[0-9]+]]:_(s32) = COPY $vgpr28
+  ; CHECK:   [[COPY29:%[0-9]+]]:_(s32) = COPY $vgpr29
+  ; CHECK:   [[COPY30:%[0-9]+]]:_(s32) = COPY $vgpr30
+  ; CHECK:   [[COPY31:%[0-9]+]]:_(s32) = COPY $vgpr31
+  ; CHECK:   [[FRAME_INDEX:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.2
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX]](p5) :: (invariant load 4 from %fixed-stack.2, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX1:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.1
+  ; CHECK:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX1]](p5) :: (invariant load 4 from %fixed-stack.1, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX2:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.0
+  ; CHECK:   [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX2]](p5) :: (invariant load 4 from %fixed-stack.0, align 1, addrspace 5)
+  ; CHECK:   [[COPY32:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<32 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32), [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32), [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[COPY15]](s32), [[COPY16]](s32), [[COPY17]](s32), [[COPY18]](s32), [[COPY19]](s32), [[COPY20]](s32), [[COPY21]](s32), [[COPY22]](s32), [[COPY23]](s32), [[COPY24]](s32), [[COPY25]](s32), [[COPY26]](s32), [[COPY27]](s32), [[COPY28]](s32), [[COPY29]](s32), [[COPY30]](s32), [[COPY31]](s32)
+  ; CHECK:   [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD1]](s32), [[LOAD2]](s32)
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[DEF1:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[DEF2:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   G_STORE [[BUILD_VECTOR]](<32 x s32>), [[DEF]](p1) :: (volatile store 128 into `<32 x i32> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   G_STORE [[LOAD]](s32), [[DEF1]](p1) :: (volatile store 4 into `i32 addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   G_STORE [[MV]](s64), [[DEF2]](p1) :: (volatile store 8 into `i64 addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[COPY33:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY32]]
+  ; CHECK:   S_SETPC_B64_return [[COPY33]]
+  store volatile <32 x i32> %arg0, <32 x i32> addrspace(1)* undef
+  store volatile i32 %arg1, i32 addrspace(1)* undef
+  store volatile i64 %arg2, i64 addrspace(1)* undef
+  ret void
+}
+
+; FIXME: Different ext load types on CI vs. VI
+define void @void_func_v32i32_i1_i8_i16(<32 x i32> %arg0, i1 %arg1, i8 %arg2, i16 %arg3, half %arg4) #0 {
+  ; CHECK-LABEL: name: void_func_v32i32_i1_i8_i16
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10, $vgpr11, $vgpr12, $vgpr13, $vgpr14, $vgpr15, $vgpr16, $vgpr17, $vgpr18, $vgpr19, $vgpr20, $vgpr21, $vgpr22, $vgpr23, $vgpr24, $vgpr25, $vgpr26, $vgpr27, $vgpr28, $vgpr29, $vgpr30, $vgpr31, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
+  ; CHECK:   [[COPY6:%[0-9]+]]:_(s32) = COPY $vgpr6
+  ; CHECK:   [[COPY7:%[0-9]+]]:_(s32) = COPY $vgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr8
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr9
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr10
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(s32) = COPY $vgpr11
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr12
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr13
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr14
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr15
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY $vgpr16
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY $vgpr17
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY $vgpr18
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY $vgpr19
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(s32) = COPY $vgpr20
+  ; CHECK:   [[COPY21:%[0-9]+]]:_(s32) = COPY $vgpr21
+  ; CHECK:   [[COPY22:%[0-9]+]]:_(s32) = COPY $vgpr22
+  ; CHECK:   [[COPY23:%[0-9]+]]:_(s32) = COPY $vgpr23
+  ; CHECK:   [[COPY24:%[0-9]+]]:_(s32) = COPY $vgpr24
+  ; CHECK:   [[COPY25:%[0-9]+]]:_(s32) = COPY $vgpr25
+  ; CHECK:   [[COPY26:%[0-9]+]]:_(s32) = COPY $vgpr26
+  ; CHECK:   [[COPY27:%[0-9]+]]:_(s32) = COPY $vgpr27
+  ; CHECK:   [[COPY28:%[0-9]+]]:_(s32) = COPY $vgpr28
+  ; CHECK:   [[COPY29:%[0-9]+]]:_(s32) = COPY $vgpr29
+  ; CHECK:   [[COPY30:%[0-9]+]]:_(s32) = COPY $vgpr30
+  ; CHECK:   [[COPY31:%[0-9]+]]:_(s32) = COPY $vgpr31
+  ; CHECK:   [[FRAME_INDEX:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.3
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(s1) = G_LOAD [[FRAME_INDEX]](p5) :: (invariant load 1 from %fixed-stack.3, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX1:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.2
+  ; CHECK:   [[LOAD1:%[0-9]+]]:_(s8) = G_LOAD [[FRAME_INDEX1]](p5) :: (invariant load 1 from %fixed-stack.2, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX2:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.1
+  ; CHECK:   [[LOAD2:%[0-9]+]]:_(s16) = G_LOAD [[FRAME_INDEX2]](p5) :: (invariant load 2 from %fixed-stack.1, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX3:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.0
+  ; CHECK:   [[LOAD3:%[0-9]+]]:_(s16) = G_LOAD [[FRAME_INDEX3]](p5) :: (invariant load 2 from %fixed-stack.0, align 1, addrspace 5)
+  ; CHECK:   [[COPY32:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<32 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32), [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32), [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[COPY15]](s32), [[COPY16]](s32), [[COPY17]](s32), [[COPY18]](s32), [[COPY19]](s32), [[COPY20]](s32), [[COPY21]](s32), [[COPY22]](s32), [[COPY23]](s32), [[COPY24]](s32), [[COPY25]](s32), [[COPY26]](s32), [[COPY27]](s32), [[COPY28]](s32), [[COPY29]](s32), [[COPY30]](s32), [[COPY31]](s32)
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[DEF1:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[DEF2:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[DEF3:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[DEF4:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   G_STORE [[BUILD_VECTOR]](<32 x s32>), [[DEF]](p1) :: (volatile store 128 into `<32 x i32> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   G_STORE [[LOAD]](s1), [[DEF1]](p1) :: (volatile store 1 into `i1 addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   G_STORE [[LOAD1]](s8), [[DEF2]](p1) :: (volatile store 1 into `i8 addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   G_STORE [[LOAD2]](s16), [[DEF3]](p1) :: (volatile store 2 into `i16 addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   G_STORE [[LOAD3]](s16), [[DEF4]](p1) :: (volatile store 2 into `half addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[COPY33:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY32]]
+  ; CHECK:   S_SETPC_B64_return [[COPY33]]
+  store volatile <32 x i32> %arg0, <32 x i32> addrspace(1)* undef
+  store volatile i1 %arg1, i1 addrspace(1)* undef
+  store volatile i8 %arg2, i8 addrspace(1)* undef
+  store volatile i16 %arg3, i16 addrspace(1)* undef
+  store volatile half %arg4, half addrspace(1)* undef
+  ret void
+}
+
+define void @void_func_v32i32_v2i32_v2f32(<32 x i32> %arg0, <2 x i32> %arg1, <2 x float> %arg2) #0 {
+  ; CHECK-LABEL: name: void_func_v32i32_v2i32_v2f32
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10, $vgpr11, $vgpr12, $vgpr13, $vgpr14, $vgpr15, $vgpr16, $vgpr17, $vgpr18, $vgpr19, $vgpr20, $vgpr21, $vgpr22, $vgpr23, $vgpr24, $vgpr25, $vgpr26, $vgpr27, $vgpr28, $vgpr29, $vgpr30, $vgpr31, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
+  ; CHECK:   [[COPY6:%[0-9]+]]:_(s32) = COPY $vgpr6
+  ; CHECK:   [[COPY7:%[0-9]+]]:_(s32) = COPY $vgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr8
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr9
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr10
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(s32) = COPY $vgpr11
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr12
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr13
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr14
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr15
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY $vgpr16
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY $vgpr17
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY $vgpr18
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY $vgpr19
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(s32) = COPY $vgpr20
+  ; CHECK:   [[COPY21:%[0-9]+]]:_(s32) = COPY $vgpr21
+  ; CHECK:   [[COPY22:%[0-9]+]]:_(s32) = COPY $vgpr22
+  ; CHECK:   [[COPY23:%[0-9]+]]:_(s32) = COPY $vgpr23
+  ; CHECK:   [[COPY24:%[0-9]+]]:_(s32) = COPY $vgpr24
+  ; CHECK:   [[COPY25:%[0-9]+]]:_(s32) = COPY $vgpr25
+  ; CHECK:   [[COPY26:%[0-9]+]]:_(s32) = COPY $vgpr26
+  ; CHECK:   [[COPY27:%[0-9]+]]:_(s32) = COPY $vgpr27
+  ; CHECK:   [[COPY28:%[0-9]+]]:_(s32) = COPY $vgpr28
+  ; CHECK:   [[COPY29:%[0-9]+]]:_(s32) = COPY $vgpr29
+  ; CHECK:   [[COPY30:%[0-9]+]]:_(s32) = COPY $vgpr30
+  ; CHECK:   [[COPY31:%[0-9]+]]:_(s32) = COPY $vgpr31
+  ; CHECK:   [[FRAME_INDEX:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.3
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX]](p5) :: (invariant load 4 from %fixed-stack.3, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX1:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.2
+  ; CHECK:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX1]](p5) :: (invariant load 4 from %fixed-stack.2, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX2:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.1
+  ; CHECK:   [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX2]](p5) :: (invariant load 4 from %fixed-stack.1, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX3:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.0
+  ; CHECK:   [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX3]](p5) :: (invariant load 4 from %fixed-stack.0, align 1, addrspace 5)
+  ; CHECK:   [[COPY32:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<32 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32), [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32), [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[COPY15]](s32), [[COPY16]](s32), [[COPY17]](s32), [[COPY18]](s32), [[COPY19]](s32), [[COPY20]](s32), [[COPY21]](s32), [[COPY22]](s32), [[COPY23]](s32), [[COPY24]](s32), [[COPY25]](s32), [[COPY26]](s32), [[COPY27]](s32), [[COPY28]](s32), [[COPY29]](s32), [[COPY30]](s32), [[COPY31]](s32)
+  ; CHECK:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[LOAD]](s32), [[LOAD1]](s32)
+  ; CHECK:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[LOAD2]](s32), [[LOAD3]](s32)
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[DEF1:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[DEF2:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   G_STORE [[BUILD_VECTOR]](<32 x s32>), [[DEF]](p1) :: (volatile store 128 into `<32 x i32> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   G_STORE [[BUILD_VECTOR1]](<2 x s32>), [[DEF1]](p1) :: (volatile store 8 into `<2 x i32> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   G_STORE [[BUILD_VECTOR2]](<2 x s32>), [[DEF2]](p1) :: (volatile store 8 into `<2 x float> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[COPY33:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY32]]
+  ; CHECK:   S_SETPC_B64_return [[COPY33]]
+  store volatile <32 x i32> %arg0, <32 x i32> addrspace(1)* undef
+  store volatile <2 x i32> %arg1, <2 x i32> addrspace(1)* undef
+  store volatile <2 x float> %arg2, <2 x float> addrspace(1)* undef
+  ret void
+}
+
+define void @void_func_v32i32_v2i16_v2f16(<32 x i32> %arg0, <2 x i16> %arg1, <2 x half> %arg2) #0 {
+  ; CHECK-LABEL: name: void_func_v32i32_v2i16_v2f16
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10, $vgpr11, $vgpr12, $vgpr13, $vgpr14, $vgpr15, $vgpr16, $vgpr17, $vgpr18, $vgpr19, $vgpr20, $vgpr21, $vgpr22, $vgpr23, $vgpr24, $vgpr25, $vgpr26, $vgpr27, $vgpr28, $vgpr29, $vgpr30, $vgpr31, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
+  ; CHECK:   [[COPY6:%[0-9]+]]:_(s32) = COPY $vgpr6
+  ; CHECK:   [[COPY7:%[0-9]+]]:_(s32) = COPY $vgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr8
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr9
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr10
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(s32) = COPY $vgpr11
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr12
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr13
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr14
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr15
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY $vgpr16
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY $vgpr17
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY $vgpr18
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY $vgpr19
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(s32) = COPY $vgpr20
+  ; CHECK:   [[COPY21:%[0-9]+]]:_(s32) = COPY $vgpr21
+  ; CHECK:   [[COPY22:%[0-9]+]]:_(s32) = COPY $vgpr22
+  ; CHECK:   [[COPY23:%[0-9]+]]:_(s32) = COPY $vgpr23
+  ; CHECK:   [[COPY24:%[0-9]+]]:_(s32) = COPY $vgpr24
+  ; CHECK:   [[COPY25:%[0-9]+]]:_(s32) = COPY $vgpr25
+  ; CHECK:   [[COPY26:%[0-9]+]]:_(s32) = COPY $vgpr26
+  ; CHECK:   [[COPY27:%[0-9]+]]:_(s32) = COPY $vgpr27
+  ; CHECK:   [[COPY28:%[0-9]+]]:_(s32) = COPY $vgpr28
+  ; CHECK:   [[COPY29:%[0-9]+]]:_(s32) = COPY $vgpr29
+  ; CHECK:   [[COPY30:%[0-9]+]]:_(s32) = COPY $vgpr30
+  ; CHECK:   [[COPY31:%[0-9]+]]:_(s32) = COPY $vgpr31
+  ; CHECK:   [[FRAME_INDEX:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.3
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX]](p5) :: (invariant load 4 from %fixed-stack.3, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX1:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.2
+  ; CHECK:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX1]](p5) :: (invariant load 4 from %fixed-stack.2, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX2:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.1
+  ; CHECK:   [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX2]](p5) :: (invariant load 4 from %fixed-stack.1, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX3:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.0
+  ; CHECK:   [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX3]](p5) :: (invariant load 4 from %fixed-stack.0, align 1, addrspace 5)
+  ; CHECK:   [[COPY32:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<32 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32), [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32), [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[COPY15]](s32), [[COPY16]](s32), [[COPY17]](s32), [[COPY18]](s32), [[COPY19]](s32), [[COPY20]](s32), [[COPY21]](s32), [[COPY22]](s32), [[COPY23]](s32), [[COPY24]](s32), [[COPY25]](s32), [[COPY26]](s32), [[COPY27]](s32), [[COPY28]](s32), [[COPY29]](s32), [[COPY30]](s32), [[COPY31]](s32)
+  ; CHECK:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[LOAD]](s32), [[LOAD1]](s32)
+  ; CHECK:   [[TRUNC:%[0-9]+]]:_(<2 x s16>) = G_TRUNC [[BUILD_VECTOR1]](<2 x s32>)
+  ; CHECK:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[LOAD2]](s32), [[LOAD3]](s32)
+  ; CHECK:   [[TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_TRUNC [[BUILD_VECTOR2]](<2 x s32>)
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[DEF1:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[DEF2:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   G_STORE [[BUILD_VECTOR]](<32 x s32>), [[DEF]](p1) :: (volatile store 128 into `<32 x i32> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   G_STORE [[TRUNC]](<2 x s16>), [[DEF1]](p1) :: (volatile store 4 into `<2 x i16> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   G_STORE [[TRUNC1]](<2 x s16>), [[DEF2]](p1) :: (volatile store 4 into `<2 x half> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[COPY33:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY32]]
+  ; CHECK:   S_SETPC_B64_return [[COPY33]]
+  store volatile <32 x i32> %arg0, <32 x i32> addrspace(1)* undef
+  store volatile <2 x i16> %arg1, <2 x i16> addrspace(1)* undef
+  store volatile <2 x half> %arg2, <2 x half> addrspace(1)* undef
+  ret void
+}
+
+define void @void_func_v32i32_v2i64_v2f64(<32 x i32> %arg0, <2 x i64> %arg1, <2 x double> %arg2) #0 {
+  ; CHECK-LABEL: name: void_func_v32i32_v2i64_v2f64
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10, $vgpr11, $vgpr12, $vgpr13, $vgpr14, $vgpr15, $vgpr16, $vgpr17, $vgpr18, $vgpr19, $vgpr20, $vgpr21, $vgpr22, $vgpr23, $vgpr24, $vgpr25, $vgpr26, $vgpr27, $vgpr28, $vgpr29, $vgpr30, $vgpr31, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
+  ; CHECK:   [[COPY6:%[0-9]+]]:_(s32) = COPY $vgpr6
+  ; CHECK:   [[COPY7:%[0-9]+]]:_(s32) = COPY $vgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr8
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr9
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr10
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(s32) = COPY $vgpr11
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr12
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr13
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr14
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr15
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY $vgpr16
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY $vgpr17
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY $vgpr18
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY $vgpr19
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(s32) = COPY $vgpr20
+  ; CHECK:   [[COPY21:%[0-9]+]]:_(s32) = COPY $vgpr21
+  ; CHECK:   [[COPY22:%[0-9]+]]:_(s32) = COPY $vgpr22
+  ; CHECK:   [[COPY23:%[0-9]+]]:_(s32) = COPY $vgpr23
+  ; CHECK:   [[COPY24:%[0-9]+]]:_(s32) = COPY $vgpr24
+  ; CHECK:   [[COPY25:%[0-9]+]]:_(s32) = COPY $vgpr25
+  ; CHECK:   [[COPY26:%[0-9]+]]:_(s32) = COPY $vgpr26
+  ; CHECK:   [[COPY27:%[0-9]+]]:_(s32) = COPY $vgpr27
+  ; CHECK:   [[COPY28:%[0-9]+]]:_(s32) = COPY $vgpr28
+  ; CHECK:   [[COPY29:%[0-9]+]]:_(s32) = COPY $vgpr29
+  ; CHECK:   [[COPY30:%[0-9]+]]:_(s32) = COPY $vgpr30
+  ; CHECK:   [[COPY31:%[0-9]+]]:_(s32) = COPY $vgpr31
+  ; CHECK:   [[FRAME_INDEX:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.7
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX]](p5) :: (invariant load 4 from %fixed-stack.7, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX1:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.6
+  ; CHECK:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX1]](p5) :: (invariant load 4 from %fixed-stack.6, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX2:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.5
+  ; CHECK:   [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX2]](p5) :: (invariant load 4 from %fixed-stack.5, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX3:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.4
+  ; CHECK:   [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX3]](p5) :: (invariant load 4 from %fixed-stack.4, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX4:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.3
+  ; CHECK:   [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX4]](p5) :: (invariant load 4 from %fixed-stack.3, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX5:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.2
+  ; CHECK:   [[LOAD5:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX5]](p5) :: (invariant load 4 from %fixed-stack.2, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX6:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.1
+  ; CHECK:   [[LOAD6:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX6]](p5) :: (invariant load 4 from %fixed-stack.1, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX7:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.0
+  ; CHECK:   [[LOAD7:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX7]](p5) :: (invariant load 4 from %fixed-stack.0, align 1, addrspace 5)
+  ; CHECK:   [[COPY32:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<32 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32), [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32), [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[COPY15]](s32), [[COPY16]](s32), [[COPY17]](s32), [[COPY18]](s32), [[COPY19]](s32), [[COPY20]](s32), [[COPY21]](s32), [[COPY22]](s32), [[COPY23]](s32), [[COPY24]](s32), [[COPY25]](s32), [[COPY26]](s32), [[COPY27]](s32), [[COPY28]](s32), [[COPY29]](s32), [[COPY30]](s32), [[COPY31]](s32)
+  ; CHECK:   [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+  ; CHECK:   [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](s32), [[LOAD3]](s32)
+  ; CHECK:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64)
+  ; CHECK:   [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD4]](s32), [[LOAD5]](s32)
+  ; CHECK:   [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD6]](s32), [[LOAD7]](s32)
+  ; CHECK:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[MV2]](s64), [[MV3]](s64)
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[DEF1:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[DEF2:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   G_STORE [[BUILD_VECTOR]](<32 x s32>), [[DEF]](p1) :: (volatile store 128 into `<32 x i32> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   G_STORE [[BUILD_VECTOR1]](<2 x s64>), [[DEF1]](p1) :: (volatile store 16 into `<2 x i64> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   G_STORE [[BUILD_VECTOR2]](<2 x s64>), [[DEF2]](p1) :: (volatile store 16 into `<2 x double> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[COPY33:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY32]]
+  ; CHECK:   S_SETPC_B64_return [[COPY33]]
+  store volatile <32 x i32> %arg0, <32 x i32> addrspace(1)* undef
+  store volatile <2 x i64> %arg1, <2 x i64> addrspace(1)* undef
+  store volatile <2 x double> %arg2, <2 x double> addrspace(1)* undef
+  ret void
+}
+
+define void @void_func_v32i32_v4i32_v4f32(<32 x i32> %arg0, <4 x i32> %arg1, <4 x float> %arg2) #0 {
+  ; CHECK-LABEL: name: void_func_v32i32_v4i32_v4f32
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10, $vgpr11, $vgpr12, $vgpr13, $vgpr14, $vgpr15, $vgpr16, $vgpr17, $vgpr18, $vgpr19, $vgpr20, $vgpr21, $vgpr22, $vgpr23, $vgpr24, $vgpr25, $vgpr26, $vgpr27, $vgpr28, $vgpr29, $vgpr30, $vgpr31, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
+  ; CHECK:   [[COPY6:%[0-9]+]]:_(s32) = COPY $vgpr6
+  ; CHECK:   [[COPY7:%[0-9]+]]:_(s32) = COPY $vgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr8
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr9
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr10
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(s32) = COPY $vgpr11
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr12
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr13
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr14
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr15
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY $vgpr16
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY $vgpr17
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY $vgpr18
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY $vgpr19
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(s32) = COPY $vgpr20
+  ; CHECK:   [[COPY21:%[0-9]+]]:_(s32) = COPY $vgpr21
+  ; CHECK:   [[COPY22:%[0-9]+]]:_(s32) = COPY $vgpr22
+  ; CHECK:   [[COPY23:%[0-9]+]]:_(s32) = COPY $vgpr23
+  ; CHECK:   [[COPY24:%[0-9]+]]:_(s32) = COPY $vgpr24
+  ; CHECK:   [[COPY25:%[0-9]+]]:_(s32) = COPY $vgpr25
+  ; CHECK:   [[COPY26:%[0-9]+]]:_(s32) = COPY $vgpr26
+  ; CHECK:   [[COPY27:%[0-9]+]]:_(s32) = COPY $vgpr27
+  ; CHECK:   [[COPY28:%[0-9]+]]:_(s32) = COPY $vgpr28
+  ; CHECK:   [[COPY29:%[0-9]+]]:_(s32) = COPY $vgpr29
+  ; CHECK:   [[COPY30:%[0-9]+]]:_(s32) = COPY $vgpr30
+  ; CHECK:   [[COPY31:%[0-9]+]]:_(s32) = COPY $vgpr31
+  ; CHECK:   [[FRAME_INDEX:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.7
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX]](p5) :: (invariant load 4 from %fixed-stack.7, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX1:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.6
+  ; CHECK:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX1]](p5) :: (invariant load 4 from %fixed-stack.6, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX2:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.5
+  ; CHECK:   [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX2]](p5) :: (invariant load 4 from %fixed-stack.5, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX3:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.4
+  ; CHECK:   [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX3]](p5) :: (invariant load 4 from %fixed-stack.4, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX4:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.3
+  ; CHECK:   [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX4]](p5) :: (invariant load 4 from %fixed-stack.3, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX5:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.2
+  ; CHECK:   [[LOAD5:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX5]](p5) :: (invariant load 4 from %fixed-stack.2, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX6:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.1
+  ; CHECK:   [[LOAD6:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX6]](p5) :: (invariant load 4 from %fixed-stack.1, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX7:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.0
+  ; CHECK:   [[LOAD7:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX7]](p5) :: (invariant load 4 from %fixed-stack.0, align 1, addrspace 5)
+  ; CHECK:   [[COPY32:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<32 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32), [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32), [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[COPY15]](s32), [[COPY16]](s32), [[COPY17]](s32), [[COPY18]](s32), [[COPY19]](s32), [[COPY20]](s32), [[COPY21]](s32), [[COPY22]](s32), [[COPY23]](s32), [[COPY24]](s32), [[COPY25]](s32), [[COPY26]](s32), [[COPY27]](s32), [[COPY28]](s32), [[COPY29]](s32), [[COPY30]](s32), [[COPY31]](s32)
+  ; CHECK:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[LOAD]](s32), [[LOAD1]](s32), [[LOAD2]](s32), [[LOAD3]](s32)
+  ; CHECK:   [[BUILD_VECTOR2:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[LOAD4]](s32), [[LOAD5]](s32), [[LOAD6]](s32), [[LOAD7]](s32)
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[DEF1:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[DEF2:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   G_STORE [[BUILD_VECTOR]](<32 x s32>), [[DEF]](p1) :: (volatile store 128 into `<32 x i32> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   G_STORE [[BUILD_VECTOR1]](<4 x s32>), [[DEF1]](p1) :: (volatile store 16 into `<4 x i32> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   G_STORE [[BUILD_VECTOR2]](<4 x s32>), [[DEF2]](p1) :: (volatile store 16 into `<4 x float> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[COPY33:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY32]]
+  ; CHECK:   S_SETPC_B64_return [[COPY33]]
+  store volatile <32 x i32> %arg0, <32 x i32> addrspace(1)* undef
+  store volatile <4 x i32> %arg1, <4 x i32> addrspace(1)* undef
+  store volatile <4 x float> %arg2, <4 x float> addrspace(1)* undef
+  ret void
+}
+
+define void @void_func_v32i32_v8i32_v8f32(<32 x i32> %arg0, <8 x i32> %arg1, <8 x float> %arg2) #0 {
+  ; CHECK-LABEL: name: void_func_v32i32_v8i32_v8f32
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10, $vgpr11, $vgpr12, $vgpr13, $vgpr14, $vgpr15, $vgpr16, $vgpr17, $vgpr18, $vgpr19, $vgpr20, $vgpr21, $vgpr22, $vgpr23, $vgpr24, $vgpr25, $vgpr26, $vgpr27, $vgpr28, $vgpr29, $vgpr30, $vgpr31, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
+  ; CHECK:   [[COPY6:%[0-9]+]]:_(s32) = COPY $vgpr6
+  ; CHECK:   [[COPY7:%[0-9]+]]:_(s32) = COPY $vgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr8
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr9
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr10
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(s32) = COPY $vgpr11
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr12
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr13
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr14
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr15
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY $vgpr16
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY $vgpr17
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY $vgpr18
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY $vgpr19
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(s32) = COPY $vgpr20
+  ; CHECK:   [[COPY21:%[0-9]+]]:_(s32) = COPY $vgpr21
+  ; CHECK:   [[COPY22:%[0-9]+]]:_(s32) = COPY $vgpr22
+  ; CHECK:   [[COPY23:%[0-9]+]]:_(s32) = COPY $vgpr23
+  ; CHECK:   [[COPY24:%[0-9]+]]:_(s32) = COPY $vgpr24
+  ; CHECK:   [[COPY25:%[0-9]+]]:_(s32) = COPY $vgpr25
+  ; CHECK:   [[COPY26:%[0-9]+]]:_(s32) = COPY $vgpr26
+  ; CHECK:   [[COPY27:%[0-9]+]]:_(s32) = COPY $vgpr27
+  ; CHECK:   [[COPY28:%[0-9]+]]:_(s32) = COPY $vgpr28
+  ; CHECK:   [[COPY29:%[0-9]+]]:_(s32) = COPY $vgpr29
+  ; CHECK:   [[COPY30:%[0-9]+]]:_(s32) = COPY $vgpr30
+  ; CHECK:   [[COPY31:%[0-9]+]]:_(s32) = COPY $vgpr31
+  ; CHECK:   [[FRAME_INDEX:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.15
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX]](p5) :: (invariant load 4 from %fixed-stack.15, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX1:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.14
+  ; CHECK:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX1]](p5) :: (invariant load 4 from %fixed-stack.14, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX2:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.13
+  ; CHECK:   [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX2]](p5) :: (invariant load 4 from %fixed-stack.13, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX3:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.12
+  ; CHECK:   [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX3]](p5) :: (invariant load 4 from %fixed-stack.12, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX4:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.11
+  ; CHECK:   [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX4]](p5) :: (invariant load 4 from %fixed-stack.11, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX5:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.10
+  ; CHECK:   [[LOAD5:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX5]](p5) :: (invariant load 4 from %fixed-stack.10, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX6:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.9
+  ; CHECK:   [[LOAD6:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX6]](p5) :: (invariant load 4 from %fixed-stack.9, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX7:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.8
+  ; CHECK:   [[LOAD7:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX7]](p5) :: (invariant load 4 from %fixed-stack.8, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX8:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.7
+  ; CHECK:   [[LOAD8:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX8]](p5) :: (invariant load 4 from %fixed-stack.7, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX9:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.6
+  ; CHECK:   [[LOAD9:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX9]](p5) :: (invariant load 4 from %fixed-stack.6, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX10:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.5
+  ; CHECK:   [[LOAD10:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX10]](p5) :: (invariant load 4 from %fixed-stack.5, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX11:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.4
+  ; CHECK:   [[LOAD11:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX11]](p5) :: (invariant load 4 from %fixed-stack.4, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX12:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.3
+  ; CHECK:   [[LOAD12:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX12]](p5) :: (invariant load 4 from %fixed-stack.3, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX13:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.2
+  ; CHECK:   [[LOAD13:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX13]](p5) :: (invariant load 4 from %fixed-stack.2, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX14:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.1
+  ; CHECK:   [[LOAD14:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX14]](p5) :: (invariant load 4 from %fixed-stack.1, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX15:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.0
+  ; CHECK:   [[LOAD15:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX15]](p5) :: (invariant load 4 from %fixed-stack.0, align 1, addrspace 5)
+  ; CHECK:   [[COPY32:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<32 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32), [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32), [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[COPY15]](s32), [[COPY16]](s32), [[COPY17]](s32), [[COPY18]](s32), [[COPY19]](s32), [[COPY20]](s32), [[COPY21]](s32), [[COPY22]](s32), [[COPY23]](s32), [[COPY24]](s32), [[COPY25]](s32), [[COPY26]](s32), [[COPY27]](s32), [[COPY28]](s32), [[COPY29]](s32), [[COPY30]](s32), [[COPY31]](s32)
+  ; CHECK:   [[BUILD_VECTOR1:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[LOAD]](s32), [[LOAD1]](s32), [[LOAD2]](s32), [[LOAD3]](s32), [[LOAD4]](s32), [[LOAD5]](s32), [[LOAD6]](s32), [[LOAD7]](s32)
+  ; CHECK:   [[BUILD_VECTOR2:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[LOAD8]](s32), [[LOAD9]](s32), [[LOAD10]](s32), [[LOAD11]](s32), [[LOAD12]](s32), [[LOAD13]](s32), [[LOAD14]](s32), [[LOAD15]](s32)
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[DEF1:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[DEF2:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   G_STORE [[BUILD_VECTOR]](<32 x s32>), [[DEF]](p1) :: (volatile store 128 into `<32 x i32> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   G_STORE [[BUILD_VECTOR1]](<8 x s32>), [[DEF1]](p1) :: (volatile store 32 into `<8 x i32> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   G_STORE [[BUILD_VECTOR2]](<8 x s32>), [[DEF2]](p1) :: (volatile store 32 into `<8 x float> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[COPY33:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY32]]
+  ; CHECK:   S_SETPC_B64_return [[COPY33]]
+  store volatile <32 x i32> %arg0, <32 x i32> addrspace(1)* undef
+  store volatile <8 x i32> %arg1, <8 x i32> addrspace(1)* undef
+  store volatile <8 x float> %arg2, <8 x float> addrspace(1)* undef
+  ret void
+}
+
+define void @void_func_v32i32_v16i32_v16f32(<32 x i32> %arg0, <16 x i32> %arg1, <16 x float> %arg2) #0 {
+  ; CHECK-LABEL: name: void_func_v32i32_v16i32_v16f32
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10, $vgpr11, $vgpr12, $vgpr13, $vgpr14, $vgpr15, $vgpr16, $vgpr17, $vgpr18, $vgpr19, $vgpr20, $vgpr21, $vgpr22, $vgpr23, $vgpr24, $vgpr25, $vgpr26, $vgpr27, $vgpr28, $vgpr29, $vgpr30, $vgpr31, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
+  ; CHECK:   [[COPY6:%[0-9]+]]:_(s32) = COPY $vgpr6
+  ; CHECK:   [[COPY7:%[0-9]+]]:_(s32) = COPY $vgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr8
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr9
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr10
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(s32) = COPY $vgpr11
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr12
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr13
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr14
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr15
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY $vgpr16
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY $vgpr17
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY $vgpr18
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY $vgpr19
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(s32) = COPY $vgpr20
+  ; CHECK:   [[COPY21:%[0-9]+]]:_(s32) = COPY $vgpr21
+  ; CHECK:   [[COPY22:%[0-9]+]]:_(s32) = COPY $vgpr22
+  ; CHECK:   [[COPY23:%[0-9]+]]:_(s32) = COPY $vgpr23
+  ; CHECK:   [[COPY24:%[0-9]+]]:_(s32) = COPY $vgpr24
+  ; CHECK:   [[COPY25:%[0-9]+]]:_(s32) = COPY $vgpr25
+  ; CHECK:   [[COPY26:%[0-9]+]]:_(s32) = COPY $vgpr26
+  ; CHECK:   [[COPY27:%[0-9]+]]:_(s32) = COPY $vgpr27
+  ; CHECK:   [[COPY28:%[0-9]+]]:_(s32) = COPY $vgpr28
+  ; CHECK:   [[COPY29:%[0-9]+]]:_(s32) = COPY $vgpr29
+  ; CHECK:   [[COPY30:%[0-9]+]]:_(s32) = COPY $vgpr30
+  ; CHECK:   [[COPY31:%[0-9]+]]:_(s32) = COPY $vgpr31
+  ; CHECK:   [[FRAME_INDEX:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.31
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX]](p5) :: (invariant load 4 from %fixed-stack.31, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX1:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.30
+  ; CHECK:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX1]](p5) :: (invariant load 4 from %fixed-stack.30, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX2:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.29
+  ; CHECK:   [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX2]](p5) :: (invariant load 4 from %fixed-stack.29, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX3:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.28
+  ; CHECK:   [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX3]](p5) :: (invariant load 4 from %fixed-stack.28, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX4:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.27
+  ; CHECK:   [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX4]](p5) :: (invariant load 4 from %fixed-stack.27, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX5:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.26
+  ; CHECK:   [[LOAD5:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX5]](p5) :: (invariant load 4 from %fixed-stack.26, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX6:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.25
+  ; CHECK:   [[LOAD6:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX6]](p5) :: (invariant load 4 from %fixed-stack.25, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX7:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.24
+  ; CHECK:   [[LOAD7:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX7]](p5) :: (invariant load 4 from %fixed-stack.24, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX8:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.23
+  ; CHECK:   [[LOAD8:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX8]](p5) :: (invariant load 4 from %fixed-stack.23, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX9:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.22
+  ; CHECK:   [[LOAD9:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX9]](p5) :: (invariant load 4 from %fixed-stack.22, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX10:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.21
+  ; CHECK:   [[LOAD10:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX10]](p5) :: (invariant load 4 from %fixed-stack.21, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX11:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.20
+  ; CHECK:   [[LOAD11:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX11]](p5) :: (invariant load 4 from %fixed-stack.20, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX12:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.19
+  ; CHECK:   [[LOAD12:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX12]](p5) :: (invariant load 4 from %fixed-stack.19, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX13:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.18
+  ; CHECK:   [[LOAD13:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX13]](p5) :: (invariant load 4 from %fixed-stack.18, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX14:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.17
+  ; CHECK:   [[LOAD14:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX14]](p5) :: (invariant load 4 from %fixed-stack.17, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX15:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.16
+  ; CHECK:   [[LOAD15:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX15]](p5) :: (invariant load 4 from %fixed-stack.16, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX16:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.15
+  ; CHECK:   [[LOAD16:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX16]](p5) :: (invariant load 4 from %fixed-stack.15, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX17:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.14
+  ; CHECK:   [[LOAD17:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX17]](p5) :: (invariant load 4 from %fixed-stack.14, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX18:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.13
+  ; CHECK:   [[LOAD18:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX18]](p5) :: (invariant load 4 from %fixed-stack.13, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX19:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.12
+  ; CHECK:   [[LOAD19:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX19]](p5) :: (invariant load 4 from %fixed-stack.12, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX20:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.11
+  ; CHECK:   [[LOAD20:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX20]](p5) :: (invariant load 4 from %fixed-stack.11, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX21:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.10
+  ; CHECK:   [[LOAD21:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX21]](p5) :: (invariant load 4 from %fixed-stack.10, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX22:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.9
+  ; CHECK:   [[LOAD22:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX22]](p5) :: (invariant load 4 from %fixed-stack.9, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX23:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.8
+  ; CHECK:   [[LOAD23:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX23]](p5) :: (invariant load 4 from %fixed-stack.8, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX24:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.7
+  ; CHECK:   [[LOAD24:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX24]](p5) :: (invariant load 4 from %fixed-stack.7, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX25:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.6
+  ; CHECK:   [[LOAD25:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX25]](p5) :: (invariant load 4 from %fixed-stack.6, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX26:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.5
+  ; CHECK:   [[LOAD26:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX26]](p5) :: (invariant load 4 from %fixed-stack.5, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX27:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.4
+  ; CHECK:   [[LOAD27:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX27]](p5) :: (invariant load 4 from %fixed-stack.4, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX28:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.3
+  ; CHECK:   [[LOAD28:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX28]](p5) :: (invariant load 4 from %fixed-stack.3, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX29:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.2
+  ; CHECK:   [[LOAD29:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX29]](p5) :: (invariant load 4 from %fixed-stack.2, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX30:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.1
+  ; CHECK:   [[LOAD30:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX30]](p5) :: (invariant load 4 from %fixed-stack.1, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX31:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.0
+  ; CHECK:   [[LOAD31:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX31]](p5) :: (invariant load 4 from %fixed-stack.0, align 1, addrspace 5)
+  ; CHECK:   [[COPY32:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<32 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32), [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32), [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[COPY15]](s32), [[COPY16]](s32), [[COPY17]](s32), [[COPY18]](s32), [[COPY19]](s32), [[COPY20]](s32), [[COPY21]](s32), [[COPY22]](s32), [[COPY23]](s32), [[COPY24]](s32), [[COPY25]](s32), [[COPY26]](s32), [[COPY27]](s32), [[COPY28]](s32), [[COPY29]](s32), [[COPY30]](s32), [[COPY31]](s32)
+  ; CHECK:   [[BUILD_VECTOR1:%[0-9]+]]:_(<16 x s32>) = G_BUILD_VECTOR [[LOAD]](s32), [[LOAD1]](s32), [[LOAD2]](s32), [[LOAD3]](s32), [[LOAD4]](s32), [[LOAD5]](s32), [[LOAD6]](s32), [[LOAD7]](s32), [[LOAD8]](s32), [[LOAD9]](s32), [[LOAD10]](s32), [[LOAD11]](s32), [[LOAD12]](s32), [[LOAD13]](s32), [[LOAD14]](s32), [[LOAD15]](s32)
+  ; CHECK:   [[BUILD_VECTOR2:%[0-9]+]]:_(<16 x s32>) = G_BUILD_VECTOR [[LOAD16]](s32), [[LOAD17]](s32), [[LOAD18]](s32), [[LOAD19]](s32), [[LOAD20]](s32), [[LOAD21]](s32), [[LOAD22]](s32), [[LOAD23]](s32), [[LOAD24]](s32), [[LOAD25]](s32), [[LOAD26]](s32), [[LOAD27]](s32), [[LOAD28]](s32), [[LOAD29]](s32), [[LOAD30]](s32), [[LOAD31]](s32)
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[DEF1:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[DEF2:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   G_STORE [[BUILD_VECTOR]](<32 x s32>), [[DEF]](p1) :: (volatile store 128 into `<32 x i32> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   G_STORE [[BUILD_VECTOR1]](<16 x s32>), [[DEF1]](p1) :: (volatile store 64 into `<16 x i32> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   G_STORE [[BUILD_VECTOR2]](<16 x s32>), [[DEF2]](p1) :: (volatile store 64 into `<16 x float> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[COPY33:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY32]]
+  ; CHECK:   S_SETPC_B64_return [[COPY33]]
+  store volatile <32 x i32> %arg0, <32 x i32> addrspace(1)* undef
+  store volatile <16 x i32> %arg1, <16 x i32> addrspace(1)* undef
+  store volatile <16 x float> %arg2, <16 x float> addrspace(1)* undef
+  ret void
+}
+
+; Make sure v3 isn't a wasted register because of v3 types being promoted to v4
+define void @void_func_v3f32_wasted_reg(<3 x float> %arg0, i32 %arg1) #0 {
+  ; CHECK-LABEL: name: void_func_v3f32_wasted_reg
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32)
+  ; CHECK:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; CHECK:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p3) = G_IMPLICIT_DEF
+  ; CHECK:   [[DEF1:%[0-9]+]]:_(p3) = G_IMPLICIT_DEF
+  ; CHECK:   [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[BUILD_VECTOR]](<3 x s32>), [[C]](s32)
+  ; CHECK:   [[EVEC1:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[BUILD_VECTOR]](<3 x s32>), [[C1]](s32)
+  ; CHECK:   [[EVEC2:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[BUILD_VECTOR]](<3 x s32>), [[C2]](s32)
+  ; CHECK:   G_STORE [[EVEC]](s32), [[DEF]](p3) :: (volatile store 4 into `float addrspace(3)* undef`, addrspace 3)
+  ; CHECK:   G_STORE [[EVEC1]](s32), [[DEF]](p3) :: (volatile store 4 into `float addrspace(3)* undef`, addrspace 3)
+  ; CHECK:   G_STORE [[EVEC2]](s32), [[DEF]](p3) :: (volatile store 4 into `float addrspace(3)* undef`, addrspace 3)
+  ; CHECK:   G_STORE [[COPY3]](s32), [[DEF1]](p3) :: (volatile store 4 into `i32 addrspace(3)* undef`, addrspace 3)
+  ; CHECK:   [[COPY5:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY4]]
+  ; CHECK:   S_SETPC_B64_return [[COPY5]]
+  %arg0.0 = extractelement <3 x float> %arg0, i32 0
+  %arg0.1 = extractelement <3 x float> %arg0, i32 1
+  %arg0.2 = extractelement <3 x float> %arg0, i32 2
+  store volatile float %arg0.0, float addrspace(3)* undef
+  store volatile float %arg0.1, float addrspace(3)* undef
+  store volatile float %arg0.2, float addrspace(3)* undef
+  store volatile i32 %arg1, i32 addrspace(3)* undef
+  ret void
+}
+
+define void @void_func_v3i32_wasted_reg(<3 x i32> %arg0, i32 %arg1) #0 {
+  ; CHECK-LABEL: name: void_func_v3i32_wasted_reg
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32)
+  ; CHECK:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; CHECK:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p3) = G_IMPLICIT_DEF
+  ; CHECK:   [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[BUILD_VECTOR]](<3 x s32>), [[C]](s32)
+  ; CHECK:   [[EVEC1:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[BUILD_VECTOR]](<3 x s32>), [[C1]](s32)
+  ; CHECK:   [[EVEC2:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[BUILD_VECTOR]](<3 x s32>), [[C2]](s32)
+  ; CHECK:   G_STORE [[EVEC]](s32), [[DEF]](p3) :: (volatile store 4 into `i32 addrspace(3)* undef`, addrspace 3)
+  ; CHECK:   G_STORE [[EVEC1]](s32), [[DEF]](p3) :: (volatile store 4 into `i32 addrspace(3)* undef`, addrspace 3)
+  ; CHECK:   G_STORE [[EVEC2]](s32), [[DEF]](p3) :: (volatile store 4 into `i32 addrspace(3)* undef`, addrspace 3)
+  ; CHECK:   G_STORE [[COPY3]](s32), [[DEF]](p3) :: (volatile store 4 into `i32 addrspace(3)* undef`, addrspace 3)
+  ; CHECK:   [[COPY5:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY4]]
+  ; CHECK:   S_SETPC_B64_return [[COPY5]]
+  %arg0.0 = extractelement <3 x i32> %arg0, i32 0
+  %arg0.1 = extractelement <3 x i32> %arg0, i32 1
+  %arg0.2 = extractelement <3 x i32> %arg0, i32 2
+  store volatile i32 %arg0.0, i32 addrspace(3)* undef
+  store volatile i32 %arg0.1, i32 addrspace(3)* undef
+  store volatile i32 %arg0.2, i32 addrspace(3)* undef
+  store volatile i32 %arg1, i32 addrspace(3)* undef
+  ret void
+}
+
+; Check there is no crash.
+define void @void_func_v16i8(<16 x i8> %arg0) #0 {
+  ; CHECK-LABEL: name: void_func_v16i8
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10, $vgpr11, $vgpr12, $vgpr13, $vgpr14, $vgpr15, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
+  ; CHECK:   [[COPY6:%[0-9]+]]:_(s32) = COPY $vgpr6
+  ; CHECK:   [[COPY7:%[0-9]+]]:_(s32) = COPY $vgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr8
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr9
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr10
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(s32) = COPY $vgpr11
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr12
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr13
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr14
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr15
+  ; CHECK:   [[COPY16:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<16 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32), [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32), [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[COPY15]](s32)
+  ; CHECK:   [[TRUNC:%[0-9]+]]:_(<16 x s8>) = G_TRUNC [[BUILD_VECTOR]](<16 x s32>)
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   G_STORE [[TRUNC]](<16 x s8>), [[DEF]](p1) :: (volatile store 16 into `<16 x i8> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[COPY17:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY16]]
+  ; CHECK:   S_SETPC_B64_return [[COPY17]]
+  store volatile <16 x i8> %arg0, <16 x i8> addrspace(1)* undef
+  ret void
+}
+
+; Check there is no crash.
+define void @void_func_v32i32_v16i8(<32 x i32> %arg0, <16 x i8> %arg1) #0 {
+  ; CHECK-LABEL: name: void_func_v32i32_v16i8
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10, $vgpr11, $vgpr12, $vgpr13, $vgpr14, $vgpr15, $vgpr16, $vgpr17, $vgpr18, $vgpr19, $vgpr20, $vgpr21, $vgpr22, $vgpr23, $vgpr24, $vgpr25, $vgpr26, $vgpr27, $vgpr28, $vgpr29, $vgpr30, $vgpr31, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
+  ; CHECK:   [[COPY6:%[0-9]+]]:_(s32) = COPY $vgpr6
+  ; CHECK:   [[COPY7:%[0-9]+]]:_(s32) = COPY $vgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr8
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr9
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr10
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(s32) = COPY $vgpr11
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr12
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr13
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr14
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr15
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY $vgpr16
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY $vgpr17
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY $vgpr18
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY $vgpr19
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(s32) = COPY $vgpr20
+  ; CHECK:   [[COPY21:%[0-9]+]]:_(s32) = COPY $vgpr21
+  ; CHECK:   [[COPY22:%[0-9]+]]:_(s32) = COPY $vgpr22
+  ; CHECK:   [[COPY23:%[0-9]+]]:_(s32) = COPY $vgpr23
+  ; CHECK:   [[COPY24:%[0-9]+]]:_(s32) = COPY $vgpr24
+  ; CHECK:   [[COPY25:%[0-9]+]]:_(s32) = COPY $vgpr25
+  ; CHECK:   [[COPY26:%[0-9]+]]:_(s32) = COPY $vgpr26
+  ; CHECK:   [[COPY27:%[0-9]+]]:_(s32) = COPY $vgpr27
+  ; CHECK:   [[COPY28:%[0-9]+]]:_(s32) = COPY $vgpr28
+  ; CHECK:   [[COPY29:%[0-9]+]]:_(s32) = COPY $vgpr29
+  ; CHECK:   [[COPY30:%[0-9]+]]:_(s32) = COPY $vgpr30
+  ; CHECK:   [[COPY31:%[0-9]+]]:_(s32) = COPY $vgpr31
+  ; CHECK:   [[FRAME_INDEX:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.15
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX]](p5) :: (invariant load 4 from %fixed-stack.15, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX1:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.14
+  ; CHECK:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX1]](p5) :: (invariant load 4 from %fixed-stack.14, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX2:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.13
+  ; CHECK:   [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX2]](p5) :: (invariant load 4 from %fixed-stack.13, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX3:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.12
+  ; CHECK:   [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX3]](p5) :: (invariant load 4 from %fixed-stack.12, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX4:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.11
+  ; CHECK:   [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX4]](p5) :: (invariant load 4 from %fixed-stack.11, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX5:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.10
+  ; CHECK:   [[LOAD5:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX5]](p5) :: (invariant load 4 from %fixed-stack.10, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX6:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.9
+  ; CHECK:   [[LOAD6:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX6]](p5) :: (invariant load 4 from %fixed-stack.9, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX7:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.8
+  ; CHECK:   [[LOAD7:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX7]](p5) :: (invariant load 4 from %fixed-stack.8, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX8:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.7
+  ; CHECK:   [[LOAD8:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX8]](p5) :: (invariant load 4 from %fixed-stack.7, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX9:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.6
+  ; CHECK:   [[LOAD9:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX9]](p5) :: (invariant load 4 from %fixed-stack.6, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX10:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.5
+  ; CHECK:   [[LOAD10:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX10]](p5) :: (invariant load 4 from %fixed-stack.5, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX11:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.4
+  ; CHECK:   [[LOAD11:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX11]](p5) :: (invariant load 4 from %fixed-stack.4, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX12:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.3
+  ; CHECK:   [[LOAD12:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX12]](p5) :: (invariant load 4 from %fixed-stack.3, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX13:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.2
+  ; CHECK:   [[LOAD13:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX13]](p5) :: (invariant load 4 from %fixed-stack.2, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX14:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.1
+  ; CHECK:   [[LOAD14:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX14]](p5) :: (invariant load 4 from %fixed-stack.1, align 1, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX15:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.0
+  ; CHECK:   [[LOAD15:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX15]](p5) :: (invariant load 4 from %fixed-stack.0, align 1, addrspace 5)
+  ; CHECK:   [[COPY32:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<32 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32), [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32), [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[COPY15]](s32), [[COPY16]](s32), [[COPY17]](s32), [[COPY18]](s32), [[COPY19]](s32), [[COPY20]](s32), [[COPY21]](s32), [[COPY22]](s32), [[COPY23]](s32), [[COPY24]](s32), [[COPY25]](s32), [[COPY26]](s32), [[COPY27]](s32), [[COPY28]](s32), [[COPY29]](s32), [[COPY30]](s32), [[COPY31]](s32)
+  ; CHECK:   [[BUILD_VECTOR1:%[0-9]+]]:_(<16 x s32>) = G_BUILD_VECTOR [[LOAD]](s32), [[LOAD1]](s32), [[LOAD2]](s32), [[LOAD3]](s32), [[LOAD4]](s32), [[LOAD5]](s32), [[LOAD6]](s32), [[LOAD7]](s32), [[LOAD8]](s32), [[LOAD9]](s32), [[LOAD10]](s32), [[LOAD11]](s32), [[LOAD12]](s32), [[LOAD13]](s32), [[LOAD14]](s32), [[LOAD15]](s32)
+  ; CHECK:   [[TRUNC:%[0-9]+]]:_(<16 x s8>) = G_TRUNC [[BUILD_VECTOR1]](<16 x s32>)
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[DEF1:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   G_STORE [[BUILD_VECTOR]](<32 x s32>), [[DEF]](p1) :: (volatile store 128 into `<32 x i32> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   G_STORE [[TRUNC]](<16 x s8>), [[DEF1]](p1) :: (volatile store 16 into `<16 x i8> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[COPY33:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY32]]
+  ; CHECK:   S_SETPC_B64_return [[COPY33]]
+  store volatile <32 x i32> %arg0, <32 x i32> addrspace(1)* undef
+  store volatile <16 x i8> %arg1, <16 x i8> addrspace(1)* undef
+  ret void
+}
+
+attributes #0 = { nounwind }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-struct-return-intrinsics.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-struct-return-intrinsics.ll
index f45037027..66a04b45c 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-struct-return-intrinsics.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-struct-return-intrinsics.ll
@@ -9,14 +9,13 @@ define amdgpu_ps void @test_div_scale(float %arg0, float %arg1) {
   ; CHECK:   liveins: $vgpr0, $vgpr1
   ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; CHECK:   [[C:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
   ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; CHECK:   [[DEF1:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
-  ; CHECK:   [[INT:%[0-9]+]]:_(s32), [[INT1:%[0-9]+]]:_(s1) = G_INTRINSIC intrinsic(@llvm.amdgcn.div.scale), [[COPY]](s32), [[COPY1]](s32), [[C]](s1)
+  ; CHECK:   [[INT:%[0-9]+]]:_(s32), [[INT1:%[0-9]+]]:_(s1) = G_INTRINSIC intrinsic(@llvm.amdgcn.div.scale), [[COPY]](s32), [[COPY1]](s32), -1
   ; CHECK:   [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[INT1]](s1)
   ; CHECK:   G_STORE [[INT]](s32), [[DEF]](p1) :: (store 4 into `float addrspace(1)* undef`, addrspace 1)
   ; CHECK:   G_STORE [[SEXT]](s32), [[DEF1]](p1) :: (store 4 into `i32 addrspace(1)* undef`, addrspace 1)
-  ; CHECK:   S_ENDPGM
+  ; CHECK:   S_ENDPGM 0
   %call = call { float, i1 } @llvm.amdgcn.div.scale.f32(float %arg0, float %arg1, i1 true)
   %extract0 = extractvalue { float, i1 } %call, 0
   %extract1 = extractvalue { float, i1 } %call, 1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/lds-global-non-entry-func.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/lds-global-non-entry-func.ll
new file mode 100644
index 000000000..56571fd07
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/lds-global-non-entry-func.ll
@@ -0,0 +1,13 @@
+; Runs original SDAG test with -global-isel
+
+; RUN: not llc -global-isel -mtriple=amdgcn-amd-amdhsa -o /dev/null < %S/../lds-global-non-entry-func.ll 2>&1 | FileCheck %s
+
+@lds = internal addrspace(3) global float undef, align 4
+
+; CHECK: error: <unknown>:0:0: in function func_use_lds_global void (): local memory global used by non-kernel function
+; CHECK-NOT: error
+; CHECK-NOT: ERROR
+define void @func_use_lds_global() {
+  store float 0.0, float addrspace(3)* @lds, align 4
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/lds-global-value.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/lds-global-value.ll
new file mode 100644
index 000000000..3bbfed778
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/lds-global-value.ll
@@ -0,0 +1,35 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=bonaire -verify-machineinstrs < %s | FileCheck %s
+; TODO: Replace with existing DAG tests
+
+@lds_512_4 = internal unnamed_addr addrspace(3) global [128 x i32] undef, align 4
+@lds_4_8 = addrspace(3) global i32 undef, align 8
+
+define amdgpu_kernel void @use_lds_globals(i32 addrspace(1)* %out, i32 addrspace(3)* %in) #0 {
+; CHECK-LABEL: use_lds_globals:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; CHECK-NEXT:    s_add_u32 s2, 4, 4
+; CHECK-NEXT:    v_mov_b32_e32 v2, s2
+; CHECK-NEXT:    s_mov_b32 m0, -1
+; CHECK-NEXT:    ds_read_b32 v2, v2
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    s_add_u32 s0, s0, 4
+; CHECK-NEXT:    s_addc_u32 s1, s1, 0
+; CHECK-NEXT:    v_mov_b32_e32 v0, s0
+; CHECK-NEXT:    v_mov_b32_e32 v1, s1
+; CHECK-NEXT:    flat_store_dword v[0:1], v2
+; CHECK-NEXT:    v_mov_b32_e32 v0, 9
+; CHECK-NEXT:    v_mov_b32_e32 v1, 0
+; CHECK-NEXT:    ds_write_b32 v1, v0
+; CHECK-NEXT:    s_endpgm
+entry:
+  %tmp0 = getelementptr [128 x i32], [128 x i32] addrspace(3)* @lds_512_4, i32 0, i32 1
+  %tmp1 = load i32, i32 addrspace(3)* %tmp0
+  %tmp2 = getelementptr i32, i32 addrspace(1)* %out, i32 1
+  store i32 %tmp1, i32 addrspace(1)* %tmp2
+  store i32 9, i32 addrspace(3)* @lds_4_8
+  ret void
+}
+
+attributes #0 = { nounwind }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/lds-size.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/lds-size.ll
new file mode 100644
index 000000000..199289394
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/lds-size.ll
@@ -0,0 +1 @@
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa < %S/../lds-size.ll | FileCheck -check-prefix=ALL -check-prefix=HSA %S/../lds-size.ll
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/lds-zero-initializer.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/lds-zero-initializer.ll
new file mode 100644
index 000000000..e18895d1e
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/lds-zero-initializer.ll
@@ -0,0 +1,5 @@
+; RUN: not llc -global-isel -march=amdgcn -mcpu=tonga < %S/../lds-zero-initializer.ll 2>&1 | FileCheck %s
+
+; FIXME: Select should succeed
+; CHECK: error: <unknown>:0:0: in function load_zeroinit_lds_global void (i32 addrspace(1)*, i1): unsupported initializer for address space
+; CHECK: LLVM ERROR: cannot select: %16:sreg_32(p3) = G_GLOBAL_VALUE @lds (in function: load_zeroinit_lds_global)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ds.ordered.add.gfx10.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ds.ordered.add.gfx10.ll
new file mode 100644
index 000000000..9a287359d
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ds.ordered.add.gfx10.ll
@@ -0,0 +1 @@
+; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs -show-mc-encoding < %S/../llvm.amdgcn.ds.ordered.add.gfx10.ll | FileCheck -check-prefixes=GCN %S/../llvm.amdgcn.ds.ordered.add.gfx10.ll
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ds.ordered.add.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ds.ordered.add.ll
new file mode 100644
index 000000000..8cba08f01
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ds.ordered.add.ll
@@ -0,0 +1,5 @@
+; FIXME: Broken SI run line
+; XUN: llc -global-isel -march=amdgcn -verify-machineinstrs < %S/../llvm.amdgcn.ds.ordered.add.ll | FileCheck -check-prefixes=GCN,FUNC %S/../llvm.amdgcn.ds.ordered.add.ll
+; RUN: llc -global-isel -march=amdgcn -mcpu=bonaire -verify-machineinstrs < %S/../llvm.amdgcn.ds.ordered.add.ll | FileCheck -check-prefixes=GCN,FUNC %S/../llvm.amdgcn.ds.ordered.add.ll
+; RUN: llc -global-isel -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %S/../llvm.amdgcn.ds.ordered.add.ll | FileCheck -check-prefixes=GCN,VIGFX9,FUNC %S/../llvm.amdgcn.ds.ordered.add.ll
+; RUN: llc -global-isel -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %S/../llvm.amdgcn.ds.ordered.add.ll | FileCheck -check-prefixes=GCN,VIGFX9,FUNC %S/../llvm.amdgcn.ds.ordered.add.ll
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ds.ordered.swap.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ds.ordered.swap.ll
new file mode 100644
index 000000000..28c2c7a4e
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ds.ordered.swap.ll
@@ -0,0 +1,5 @@
+; FIXME: Broken SI run line
+; XUN: llc -global-isel -march=amdgcn -verify-machineinstrs < %S/../llvm.amdgcn.ds.ordered.swap.ll | FileCheck -check-prefixes=GCN,FUNC %S/../llvm.amdgcn.ds.ordered.swap.ll
+; RUN: llc -global-isel -march=amdgcn -mcpu=bonaire -verify-machineinstrs < %S/../llvm.amdgcn.ds.ordered.swap.ll | FileCheck -check-prefixes=GCN,FUNC %S/../llvm.amdgcn.ds.ordered.swap.ll
+; RUN: llc -global-isel -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %S/../llvm.amdgcn.ds.ordered.swap.ll | FileCheck -check-prefixes=GCN,VIGFX9,FUNC %S/../llvm.amdgcn.ds.ordered.swap.ll
+; RUN: llc -global-isel -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %S/../llvm.amdgcn.ds.ordered.swap.ll | FileCheck -check-prefixes=GCN,VIGFX9,FUNC %S/../llvm.amdgcn.ds.ordered.swap.ll
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.end.cf.i32.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.end.cf.i32.ll
index 0f04c0c44..29c9729d2 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.end.cf.i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.end.cf.i32.ll
@@ -9,6 +9,9 @@ define amdgpu_kernel void @test_wave32(i32 %arg0, [8 x i32], i32 %saved) {
 ; GCN-NEXT:    ; implicit-def: $vcc_hi
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    s_cmp_eq_u32 s1, 0
+; GCN-NEXT:    s_cselect_b32 s1, 1, 0
+; GCN-NEXT:    s_and_b32 s1, s1, 1
+; GCN-NEXT:    s_cmp_lg_u32 s1, 0
 ; GCN-NEXT:    s_cbranch_scc0 BB0_2
 ; GCN-NEXT:  ; %bb.1: ; %mid
 ; GCN-NEXT:    v_mov_b32_e32 v0, 0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.end.cf.i64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.end.cf.i64.ll
index 6172c9cee..c4ae325ba 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.end.cf.i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.end.cf.i64.ll
@@ -8,6 +8,9 @@ define amdgpu_kernel void @test_wave64(i32 %arg0, i64 %saved) {
 ; GCN-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    s_cmp_eq_u32 s2, 0
+; GCN-NEXT:    s_cselect_b32 s2, 1, 0
+; GCN-NEXT:    s_and_b32 s2, s2, 1
+; GCN-NEXT:    s_cmp_lg_u32 s2, 0
 ; GCN-NEXT:    s_cbranch_scc0 BB0_2
 ; GCN-NEXT:  ; %bb.1: ; %mid
 ; GCN-NEXT:    v_mov_b32_e32 v0, 0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.if.break.i32.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.if.break.i32.ll
index 0f259fcb8..ce186792a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.if.break.i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.if.break.i32.ll
@@ -9,7 +9,8 @@ define amdgpu_kernel void @test_wave32(i32 %arg0, [8 x i32], i32 %saved) {
 ; GCN-NEXT:    ; implicit-def: $vcc_hi
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    s_cmp_eq_u32 s0, 0
-; GCN-NEXT:    s_cselect_b32 s0, -1, 0
+; GCN-NEXT:    s_cselect_b32 s0, 1, 0
+; GCN-NEXT:    s_and_b32 s0, 1, s0
 ; GCN-NEXT:    v_cmp_ne_u32_e64 s0, 0, s0
 ; GCN-NEXT:    s_or_b32 s0, s0, s1
 ; GCN-NEXT:    v_mov_b32_e32 v0, s0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.if.break.i64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.if.break.i64.ll
index 77d1374c8..e1f22b720 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.if.break.i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.if.break.i64.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -mtriple=amdgcn--amdhsa -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -global-isel -mtriple=amdgcn--amdhsa -mcpu=hawaii -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
 
 define amdgpu_kernel void @test_wave64(i32 %arg0, [8 x i32], i64 %saved) {
 ; GCN-LABEL: test_wave64:
@@ -8,12 +8,12 @@ define amdgpu_kernel void @test_wave64(i32 %arg0, [8 x i32], i64 %saved) {
 ; GCN-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0xa
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    s_cmp_eq_u32 s2, 0
-; GCN-NEXT:    s_cselect_b32 s2, -1, 0
+; GCN-NEXT:    s_cselect_b32 s2, 1, 0
+; GCN-NEXT:    s_and_b32 s2, 1, s2
 ; GCN-NEXT:    v_cmp_ne_u32_e64 s[2:3], 0, s2
 ; GCN-NEXT:    s_or_b64 s[0:1], s[2:3], s[0:1]
 ; GCN-NEXT:    v_mov_b32_e32 v0, s0
 ; GCN-NEXT:    v_mov_b32_e32 v1, s1
-; GCN-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN-NEXT:    flat_store_dwordx2 v[0:1], v[0:1]
 ; GCN-NEXT:    s_endpgm
 entry:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.init.exec.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.init.exec.ll
new file mode 100644
index 000000000..77c2f80ac
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.init.exec.ll
@@ -0,0 +1,2 @@
+; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %S/../llvm.amdgcn.init.exec.ll | FileCheck -check-prefix=GCN %S/../llvm.amdgcn.init.exec.ll
+; RUN: llc -march=amdgcn -mcpu=gfx1010 -mattr=-wavefrontsize32,+wavefrontsize64 -verify-machineinstrs < %S/../llvm.amdgcn.init.exec.ll | FileCheck -check-prefix=GCN %S/../llvm.amdgcn.init.exec.ll
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.init.exec.wave32.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.init.exec.wave32.ll
new file mode 100644
index 000000000..1e8fce0e3
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.init.exec.wave32.ll
@@ -0,0 +1,2 @@
+; Runs original SDAG test with -global-isel
+; RUN: llc -march=amdgcn -mcpu=gfx1010 -mattr=+wavefrontsize32,-wavefrontsize64 -verify-machineinstrs < %S/../llvm.amdgcn.init.exec.wave32.ll | FileCheck -check-prefixes=GCN,GFX1032  %S/../llvm.amdgcn.init.exec.wave32.ll
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.is.private.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.is.private.ll
new file mode 100644
index 000000000..4409b4e5c
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.is.private.ll
@@ -0,0 +1,109 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=hawaii -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=CI %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GFX9 %s
+
+; TODO: Merge with DAG test
+
+define amdgpu_kernel void @is_private_vgpr(i8* addrspace(1)* %ptr.ptr) {
+; CI-LABEL: is_private_vgpr:
+; CI:       ; %bb.0:
+; CI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; CI-NEXT:    v_mul_lo_u32 v2, 0, v0
+; CI-NEXT:    v_mul_lo_u32 v1, 8, v1
+; CI-NEXT:    v_mul_lo_u32 v3, 8, v0
+; CI-NEXT:    s_load_dwordx2 s[0:1], s[6:7], 0x0
+; CI-NEXT:    v_mul_hi_u32 v0, 8, v0
+; CI-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
+; CI-NEXT:    v_add_i32_e32 v1, vcc, v1, v0
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    v_add_i32_e32 v0, vcc, s0, v3
+; CI-NEXT:    v_mov_b32_e32 v2, s1
+; CI-NEXT:    v_addc_u32_e32 v1, vcc, v2, v1, vcc
+; CI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; CI-NEXT:    s_load_dword s0, s[4:5], 0x11
+; CI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CI-NEXT:    v_cmp_eq_u32_e32 vcc, s0, v1
+; CI-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; CI-NEXT:    flat_store_dword v[0:1], v0
+; CI-NEXT:    s_endpgm
+;
+; GFX9-LABEL: is_private_vgpr:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; GFX9-NEXT:    v_mul_lo_u32 v2, 0, v0
+; GFX9-NEXT:    v_mul_lo_u32 v1, 8, v1
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[6:7], 0x0
+; GFX9-NEXT:    v_mul_hi_u32 v3, 8, v0
+; GFX9-NEXT:    v_mul_lo_u32 v0, 8, v0
+; GFX9-NEXT:    v_add_u32_e32 v1, v2, v1
+; GFX9-NEXT:    v_add_u32_e32 v1, v1, v3
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v2, s1
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s0, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, v2, v1, vcc
+; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off
+; GFX9-NEXT:    s_getreg_b32 s0, hwreg(HW_REG_SH_MEM_BASES, 0, 16)
+; GFX9-NEXT:    s_lshl_b32 s0, s0, 16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, s0, v1
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX9-NEXT:    global_store_dword v[0:1], v0, off
+; GFX9-NEXT:    s_endpgm
+  %id = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep = getelementptr inbounds i8*, i8* addrspace(1)* %ptr.ptr, i32 %id
+  %ptr = load volatile i8*, i8* addrspace(1)* %gep
+  %val = call i1 @llvm.amdgcn.is.private(i8* %ptr)
+  %ext = zext i1 %val to i32
+  store i32 %ext, i32 addrspace(1)* undef
+  ret void
+}
+
+define amdgpu_kernel void @is_private_sgpr(i8* %ptr) {
+; CI-LABEL: is_private_sgpr:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx2 s[0:1], s[6:7], 0x0
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    s_load_dword s0, s[4:5], 0x11
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    s_cmp_eq_u32 s1, s0
+; CI-NEXT:    s_cselect_b32 s0, 1, 0
+; CI-NEXT:    s_and_b32 s0, s0, 1
+; CI-NEXT:    s_cmp_lg_u32 s0, 0
+; CI-NEXT:    s_cbranch_scc0 BB1_2
+; CI-NEXT:  ; %bb.1: ; %bb0
+; CI-NEXT:    v_mov_b32_e32 v0, 0
+; CI-NEXT:    flat_store_dword v[0:1], v0
+; CI-NEXT:  BB1_2: ; %bb1
+; CI-NEXT:    s_endpgm
+;
+; GFX9-LABEL: is_private_sgpr:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[6:7], 0x0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_getreg_b32 s0, hwreg(HW_REG_SH_MEM_BASES, 0, 16)
+; GFX9-NEXT:    s_lshl_b32 s0, s0, 16
+; GFX9-NEXT:    s_cmp_eq_u32 s1, s0
+; GFX9-NEXT:    s_cselect_b32 s0, 1, 0
+; GFX9-NEXT:    s_and_b32 s0, s0, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s0, 0
+; GFX9-NEXT:    s_cbranch_scc0 BB1_2
+; GFX9-NEXT:  ; %bb.1: ; %bb0
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-NEXT:    global_store_dword v[0:1], v0, off
+; GFX9-NEXT:  BB1_2: ; %bb1
+; GFX9-NEXT:    s_endpgm
+  %val = call i1 @llvm.amdgcn.is.private(i8* %ptr)
+  br i1 %val, label %bb0, label %bb1
+
+bb0:
+  store volatile i32 0, i32 addrspace(1)* undef
+  br label %bb1
+
+bb1:
+  ret void
+}
+
+declare i32 @llvm.amdgcn.workitem.id.x() #0
+declare i1 @llvm.amdgcn.is.private(i8* nocapture) #0
+
+attributes #0 = { nounwind readnone speculatable }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.is.shared.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.is.shared.ll
new file mode 100644
index 000000000..bbabe63b5
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.is.shared.ll
@@ -0,0 +1,109 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=hawaii -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=CI %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GFX9 %s
+
+; TODO: Merge with DAG test
+
+define amdgpu_kernel void @is_local_vgpr(i8* addrspace(1)* %ptr.ptr) {
+; CI-LABEL: is_local_vgpr:
+; CI:       ; %bb.0:
+; CI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; CI-NEXT:    v_mul_lo_u32 v2, 0, v0
+; CI-NEXT:    v_mul_lo_u32 v1, 8, v1
+; CI-NEXT:    v_mul_lo_u32 v3, 8, v0
+; CI-NEXT:    s_load_dwordx2 s[0:1], s[6:7], 0x0
+; CI-NEXT:    v_mul_hi_u32 v0, 8, v0
+; CI-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
+; CI-NEXT:    v_add_i32_e32 v1, vcc, v1, v0
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    v_add_i32_e32 v0, vcc, s0, v3
+; CI-NEXT:    v_mov_b32_e32 v2, s1
+; CI-NEXT:    v_addc_u32_e32 v1, vcc, v2, v1, vcc
+; CI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; CI-NEXT:    s_load_dword s0, s[4:5], 0x10
+; CI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CI-NEXT:    v_cmp_eq_u32_e32 vcc, s0, v1
+; CI-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; CI-NEXT:    flat_store_dword v[0:1], v0
+; CI-NEXT:    s_endpgm
+;
+; GFX9-LABEL: is_local_vgpr:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; GFX9-NEXT:    v_mul_lo_u32 v2, 0, v0
+; GFX9-NEXT:    v_mul_lo_u32 v1, 8, v1
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[6:7], 0x0
+; GFX9-NEXT:    v_mul_hi_u32 v3, 8, v0
+; GFX9-NEXT:    v_mul_lo_u32 v0, 8, v0
+; GFX9-NEXT:    v_add_u32_e32 v1, v2, v1
+; GFX9-NEXT:    v_add_u32_e32 v1, v1, v3
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v2, s1
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s0, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, v2, v1, vcc
+; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off
+; GFX9-NEXT:    s_getreg_b32 s0, hwreg(HW_REG_SH_MEM_BASES, 16, 16)
+; GFX9-NEXT:    s_lshl_b32 s0, s0, 16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, s0, v1
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX9-NEXT:    global_store_dword v[0:1], v0, off
+; GFX9-NEXT:    s_endpgm
+  %id = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep = getelementptr inbounds i8*, i8* addrspace(1)* %ptr.ptr, i32 %id
+  %ptr = load volatile i8*, i8* addrspace(1)* %gep
+  %val = call i1 @llvm.amdgcn.is.shared(i8* %ptr)
+  %ext = zext i1 %val to i32
+  store i32 %ext, i32 addrspace(1)* undef
+  ret void
+}
+
+define amdgpu_kernel void @is_local_sgpr(i8* %ptr) {
+; CI-LABEL: is_local_sgpr:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx2 s[0:1], s[6:7], 0x0
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    s_load_dword s0, s[4:5], 0x10
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    s_cmp_eq_u32 s1, s0
+; CI-NEXT:    s_cselect_b32 s0, 1, 0
+; CI-NEXT:    s_and_b32 s0, s0, 1
+; CI-NEXT:    s_cmp_lg_u32 s0, 0
+; CI-NEXT:    s_cbranch_scc0 BB1_2
+; CI-NEXT:  ; %bb.1: ; %bb0
+; CI-NEXT:    v_mov_b32_e32 v0, 0
+; CI-NEXT:    flat_store_dword v[0:1], v0
+; CI-NEXT:  BB1_2: ; %bb1
+; CI-NEXT:    s_endpgm
+;
+; GFX9-LABEL: is_local_sgpr:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[6:7], 0x0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_getreg_b32 s0, hwreg(HW_REG_SH_MEM_BASES, 16, 16)
+; GFX9-NEXT:    s_lshl_b32 s0, s0, 16
+; GFX9-NEXT:    s_cmp_eq_u32 s1, s0
+; GFX9-NEXT:    s_cselect_b32 s0, 1, 0
+; GFX9-NEXT:    s_and_b32 s0, s0, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s0, 0
+; GFX9-NEXT:    s_cbranch_scc0 BB1_2
+; GFX9-NEXT:  ; %bb.1: ; %bb0
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-NEXT:    global_store_dword v[0:1], v0, off
+; GFX9-NEXT:  BB1_2: ; %bb1
+; GFX9-NEXT:    s_endpgm
+  %val = call i1 @llvm.amdgcn.is.shared(i8* %ptr)
+  br i1 %val, label %bb0, label %bb1
+
+bb0:
+  store volatile i32 0, i32 addrspace(1)* undef
+  br label %bb1
+
+bb1:
+  ret void
+}
+
+declare i32 @llvm.amdgcn.workitem.id.x() #0
+declare i1 @llvm.amdgcn.is.shared(i8* nocapture) #0
+
+attributes #0 = { nounwind readnone speculatable }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.kernarg.segment.ptr.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.kernarg.segment.ptr.ll
index ca28f6e00..55929202b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.kernarg.segment.ptr.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.kernarg.segment.ptr.ll
@@ -1,6 +1,6 @@
 ; RUN: llc -global-isel -mtriple=amdgcn--amdhsa -mattr=-code-object-v3 -mcpu=kaveri -verify-machineinstrs < %s | FileCheck -check-prefixes=CO-V2,HSA,ALL %s
 ; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=hawaii -mattr=+flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefixes=CO-V2,OS-MESA3D,MESA,ALL %s
-; RUN: llc -global-isel -mtriple=amdgcn-mesa-unknown -verify-machineinstrs < %s | FileCheck -check-prefixes=OS-UNKNOWN,MESA,ALL %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-unknown -mcpu=hawaii -verify-machineinstrs < %s | FileCheck -check-prefixes=OS-UNKNOWN,MESA,ALL %s
 
 ; ALL-LABEL: {{^}}test:
 ; CO-V2: enable_sgpr_kernarg_segment_ptr = 1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.format.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.format.f16.ll
new file mode 100644
index 000000000..ce71a89ad
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.format.f16.ll
@@ -0,0 +1,504 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=tonga -stop-after=instruction-select -o - %s | FileCheck -check-prefix=UNPACKED %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx810 -stop-after=instruction-select -o - %s | FileCheck -check-prefix=PACKED %s
+
+define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_f16(<4 x i32> inreg %rsrc, half %val, i32 %voffset, i32 inreg %soffset) {
+  ; UNPACKED-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_f16
+  ; UNPACKED: bb.1 (%ir-block.0):
+  ; UNPACKED:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; UNPACKED:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; UNPACKED:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; UNPACKED:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; UNPACKED:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; UNPACKED:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; UNPACKED:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; UNPACKED:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; UNPACKED:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; UNPACKED:   BUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 2 into custom "TargetCustom7", align 1, addrspace 4)
+  ; UNPACKED:   S_ENDPGM 0
+  ; PACKED-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_f16
+  ; PACKED: bb.1 (%ir-block.0):
+  ; PACKED:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; PACKED:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; PACKED:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; PACKED:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; PACKED:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; PACKED:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; PACKED:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; PACKED:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; PACKED:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; PACKED:   BUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 2 into custom "TargetCustom7", align 1, addrspace 4)
+  ; PACKED:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.format.f16(half %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__voffset_4095__sgpr_soffset_f16(<4 x i32> inreg %rsrc, half %val, i32 inreg %soffset) {
+  ; UNPACKED-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__voffset_4095__sgpr_soffset_f16
+  ; UNPACKED: bb.1 (%ir-block.0):
+  ; UNPACKED:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
+  ; UNPACKED:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; UNPACKED:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; UNPACKED:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; UNPACKED:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; UNPACKED:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; UNPACKED:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; UNPACKED:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; UNPACKED:   BUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY4]], $noreg, [[REG_SEQUENCE]], [[COPY5]], 4095, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 2 into custom "TargetCustom7" + 4095, align 1, addrspace 4)
+  ; UNPACKED:   S_ENDPGM 0
+  ; PACKED-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__voffset_4095__sgpr_soffset_f16
+  ; PACKED: bb.1 (%ir-block.0):
+  ; PACKED:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
+  ; PACKED:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; PACKED:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; PACKED:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; PACKED:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; PACKED:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; PACKED:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; PACKED:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; PACKED:   BUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY4]], $noreg, [[REG_SEQUENCE]], [[COPY5]], 4095, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 2 into custom "TargetCustom7" + 4095, align 1, addrspace 4)
+  ; PACKED:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.format.f16(half %val, <4 x i32> %rsrc, i32 4095, i32 %soffset, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16(<4 x i32> inreg %rsrc, <2 x half> %val, i32 %voffset, i32 inreg %soffset) {
+  ; UNPACKED-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16
+  ; UNPACKED: bb.1 (%ir-block.0):
+  ; UNPACKED:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; UNPACKED:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; UNPACKED:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; UNPACKED:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; UNPACKED:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; UNPACKED:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; UNPACKED:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; UNPACKED:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; UNPACKED:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; UNPACKED:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+  ; UNPACKED:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; UNPACKED:   [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY7]], [[COPY4]], implicit $exec
+  ; UNPACKED:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[V_LSHRREV_B32_e64_]], %subreg.sub1
+  ; UNPACKED:   BUFFER_STORE_FORMAT_D16_XY_gfx80_OFFEN_exact [[REG_SEQUENCE1]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom "TargetCustom7", align 1, addrspace 4)
+  ; UNPACKED:   S_ENDPGM 0
+  ; PACKED-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16
+  ; PACKED: bb.1 (%ir-block.0):
+  ; PACKED:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; PACKED:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; PACKED:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; PACKED:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; PACKED:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; PACKED:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; PACKED:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; PACKED:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; PACKED:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; PACKED:   BUFFER_STORE_FORMAT_D16_XY_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom "TargetCustom7", align 1, addrspace 4)
+  ; PACKED:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.format.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f16(<4 x i32> inreg %rsrc, <4 x half> %val, i32 %voffset, i32 inreg %soffset) {
+  ; UNPACKED-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f16
+  ; UNPACKED: bb.1 (%ir-block.0):
+  ; UNPACKED:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
+  ; UNPACKED:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; UNPACKED:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; UNPACKED:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; UNPACKED:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; UNPACKED:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; UNPACKED:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; UNPACKED:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; UNPACKED:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; UNPACKED:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; UNPACKED:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+  ; UNPACKED:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; UNPACKED:   [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY8]], [[COPY4]], implicit $exec
+  ; UNPACKED:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; UNPACKED:   [[V_LSHRREV_B32_e64_1:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY9]], [[COPY5]], implicit $exec
+  ; UNPACKED:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[V_LSHRREV_B32_e64_]], %subreg.sub1, [[COPY5]], %subreg.sub2, [[V_LSHRREV_B32_e64_1]], %subreg.sub3
+  ; UNPACKED:   BUFFER_STORE_FORMAT_D16_XYZW_gfx80_OFFEN_exact [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 8 into custom "TargetCustom7", align 1, addrspace 4)
+  ; UNPACKED:   S_ENDPGM 0
+  ; PACKED-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f16
+  ; PACKED: bb.1 (%ir-block.0):
+  ; PACKED:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
+  ; PACKED:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; PACKED:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; PACKED:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; PACKED:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; PACKED:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; PACKED:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; PACKED:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; PACKED:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; PACKED:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; PACKED:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; PACKED:   BUFFER_STORE_FORMAT_D16_XYZW_OFFEN_exact [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 8 into custom "TargetCustom7", align 1, addrspace 4)
+  ; PACKED:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.format.v4f16(<4 x half> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+; Make sure unpack code is emitted outside of loop
+define amdgpu_ps void @raw_buffer_store_format__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f16(<4 x i32> %rsrc, <4 x half> %val, i32 %voffset, i32 inreg %soffset) {
+  ; UNPACKED-LABEL: name: raw_buffer_store_format__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f16
+  ; UNPACKED: bb.1 (%ir-block.0):
+  ; UNPACKED:   successors: %bb.2(0x80000000)
+  ; UNPACKED:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; UNPACKED:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; UNPACKED:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; UNPACKED:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; UNPACKED:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; UNPACKED:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; UNPACKED:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; UNPACKED:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; UNPACKED:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; UNPACKED:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; UNPACKED:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+  ; UNPACKED:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; UNPACKED:   [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY8]], [[COPY4]], implicit $exec
+  ; UNPACKED:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; UNPACKED:   [[V_LSHRREV_B32_e64_1:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY9]], [[COPY5]], implicit $exec
+  ; UNPACKED:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[V_LSHRREV_B32_e64_]], %subreg.sub1, [[COPY5]], %subreg.sub2, [[V_LSHRREV_B32_e64_1]], %subreg.sub3
+  ; UNPACKED:   [[COPY10:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; UNPACKED:   [[COPY11:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; UNPACKED:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term $exec
+  ; UNPACKED: bb.2:
+  ; UNPACKED:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; UNPACKED:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]].sub0, implicit $exec
+  ; UNPACKED:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]].sub1, implicit $exec
+  ; UNPACKED:   [[REG_SEQUENCE2:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; UNPACKED:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[COPY10]], implicit $exec
+  ; UNPACKED:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY11]].sub0, implicit $exec
+  ; UNPACKED:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY11]].sub1, implicit $exec
+  ; UNPACKED:   [[REG_SEQUENCE3:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; UNPACKED:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE3]], [[COPY11]], implicit $exec
+  ; UNPACKED:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_1]], [[V_CMP_EQ_U64_e64_]], implicit-def $scc
+  ; UNPACKED:   [[REG_SEQUENCE4:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; UNPACKED:   BUFFER_STORE_FORMAT_D16_XYZW_gfx80_OFFEN_exact [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE4]], [[COPY7]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 8 into custom "TargetCustom7", align 1, addrspace 4)
+  ; UNPACKED:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; UNPACKED:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; UNPACKED:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; UNPACKED: bb.3:
+  ; UNPACKED:   successors: %bb.4(0x80000000)
+  ; UNPACKED:   $exec = S_MOV_B64_term [[S_MOV_B64_term]]
+  ; UNPACKED: bb.4:
+  ; UNPACKED:   S_ENDPGM 0
+  ; PACKED-LABEL: name: raw_buffer_store_format__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f16
+  ; PACKED: bb.1 (%ir-block.0):
+  ; PACKED:   successors: %bb.2(0x80000000)
+  ; PACKED:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; PACKED:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; PACKED:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; PACKED:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; PACKED:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; PACKED:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; PACKED:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; PACKED:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; PACKED:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; PACKED:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; PACKED:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; PACKED:   [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; PACKED:   [[COPY9:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; PACKED:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term $exec
+  ; PACKED: bb.2:
+  ; PACKED:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; PACKED:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]].sub0, implicit $exec
+  ; PACKED:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]].sub1, implicit $exec
+  ; PACKED:   [[REG_SEQUENCE2:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; PACKED:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[COPY8]], implicit $exec
+  ; PACKED:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]].sub0, implicit $exec
+  ; PACKED:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]].sub1, implicit $exec
+  ; PACKED:   [[REG_SEQUENCE3:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; PACKED:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE3]], [[COPY9]], implicit $exec
+  ; PACKED:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_1]], [[V_CMP_EQ_U64_e64_]], implicit-def $scc
+  ; PACKED:   [[REG_SEQUENCE4:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; PACKED:   BUFFER_STORE_FORMAT_D16_XYZW_OFFEN_exact [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE4]], [[COPY7]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 8 into custom "TargetCustom7", align 1, addrspace 4)
+  ; PACKED:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; PACKED:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; PACKED:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; PACKED: bb.3:
+  ; PACKED:   successors: %bb.4(0x80000000)
+  ; PACKED:   $exec = S_MOV_B64_term [[S_MOV_B64_term]]
+  ; PACKED: bb.4:
+  ; PACKED:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.format.v4f16(<4 x half> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_soffset4095(<4 x i32> inreg %rsrc, <2 x half> %val, i32 %voffset, i32 inreg %soffset) {
+  ; UNPACKED-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_soffset4095
+  ; UNPACKED: bb.1 (%ir-block.0):
+  ; UNPACKED:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; UNPACKED:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; UNPACKED:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; UNPACKED:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; UNPACKED:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; UNPACKED:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; UNPACKED:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; UNPACKED:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; UNPACKED:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4095
+  ; UNPACKED:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+  ; UNPACKED:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
+  ; UNPACKED:   [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY6]], [[COPY4]], implicit $exec
+  ; UNPACKED:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[V_LSHRREV_B32_e64_]], %subreg.sub1
+  ; UNPACKED:   BUFFER_STORE_FORMAT_D16_XY_gfx80_OFFEN_exact [[REG_SEQUENCE1]], [[COPY5]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom "TargetCustom7", align 1, addrspace 4)
+  ; UNPACKED:   S_ENDPGM 0
+  ; PACKED-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_soffset4095
+  ; PACKED: bb.1 (%ir-block.0):
+  ; PACKED:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; PACKED:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; PACKED:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; PACKED:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; PACKED:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; PACKED:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; PACKED:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; PACKED:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; PACKED:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4095
+  ; PACKED:   BUFFER_STORE_FORMAT_D16_XY_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom "TargetCustom7", align 1, addrspace 4)
+  ; PACKED:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.format.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %voffset, i32 4095, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_soffset4096(<4 x i32> inreg %rsrc, <2 x half> %val, i32 %voffset, i32 inreg %soffset) {
+  ; UNPACKED-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_soffset4096
+  ; UNPACKED: bb.1 (%ir-block.0):
+  ; UNPACKED:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; UNPACKED:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; UNPACKED:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; UNPACKED:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; UNPACKED:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; UNPACKED:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; UNPACKED:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; UNPACKED:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; UNPACKED:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4096
+  ; UNPACKED:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+  ; UNPACKED:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
+  ; UNPACKED:   [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY6]], [[COPY4]], implicit $exec
+  ; UNPACKED:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[V_LSHRREV_B32_e64_]], %subreg.sub1
+  ; UNPACKED:   BUFFER_STORE_FORMAT_D16_XY_gfx80_OFFEN_exact [[REG_SEQUENCE1]], [[COPY5]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom "TargetCustom7", align 1, addrspace 4)
+  ; UNPACKED:   S_ENDPGM 0
+  ; PACKED-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_soffset4096
+  ; PACKED: bb.1 (%ir-block.0):
+  ; PACKED:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; PACKED:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; PACKED:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; PACKED:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; PACKED:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; PACKED:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; PACKED:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; PACKED:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; PACKED:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4096
+  ; PACKED:   BUFFER_STORE_FORMAT_D16_XY_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom "TargetCustom7", align 1, addrspace 4)
+  ; PACKED:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.format.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %voffset, i32 4096, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_voffset_add_16(<4 x i32> inreg %rsrc, <2 x half> %val, i32 %voffset, i32 inreg %soffset) {
+  ; UNPACKED-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_voffset_add_16
+  ; UNPACKED: bb.1 (%ir-block.0):
+  ; UNPACKED:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; UNPACKED:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; UNPACKED:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; UNPACKED:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; UNPACKED:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; UNPACKED:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; UNPACKED:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; UNPACKED:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; UNPACKED:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; UNPACKED:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+  ; UNPACKED:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; UNPACKED:   [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY7]], [[COPY4]], implicit $exec
+  ; UNPACKED:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[V_LSHRREV_B32_e64_]], %subreg.sub1
+  ; UNPACKED:   BUFFER_STORE_FORMAT_D16_XY_gfx80_OFFEN_exact [[REG_SEQUENCE1]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 16, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom "TargetCustom7" + 16, align 1, addrspace 4)
+  ; UNPACKED:   S_ENDPGM 0
+  ; PACKED-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_voffset_add_16
+  ; PACKED: bb.1 (%ir-block.0):
+  ; PACKED:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; PACKED:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; PACKED:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; PACKED:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; PACKED:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; PACKED:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; PACKED:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; PACKED:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; PACKED:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; PACKED:   BUFFER_STORE_FORMAT_D16_XY_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 16, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom "TargetCustom7" + 16, align 1, addrspace 4)
+  ; PACKED:   S_ENDPGM 0
+  %voffset.add = add i32 %voffset, 16
+  call void @llvm.amdgcn.raw.buffer.store.format.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_voffset_add_4095(<4 x i32> inreg %rsrc, <2 x half> %val, i32 %voffset, i32 inreg %soffset) {
+  ; UNPACKED-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_voffset_add_4095
+  ; UNPACKED: bb.1 (%ir-block.0):
+  ; UNPACKED:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; UNPACKED:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; UNPACKED:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; UNPACKED:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; UNPACKED:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; UNPACKED:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; UNPACKED:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; UNPACKED:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; UNPACKED:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; UNPACKED:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+  ; UNPACKED:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; UNPACKED:   [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY7]], [[COPY4]], implicit $exec
+  ; UNPACKED:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[V_LSHRREV_B32_e64_]], %subreg.sub1
+  ; UNPACKED:   BUFFER_STORE_FORMAT_D16_XY_gfx80_OFFEN_exact [[REG_SEQUENCE1]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 4095, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom "TargetCustom7" + 4095, align 1, addrspace 4)
+  ; UNPACKED:   S_ENDPGM 0
+  ; PACKED-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_voffset_add_4095
+  ; PACKED: bb.1 (%ir-block.0):
+  ; PACKED:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; PACKED:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; PACKED:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; PACKED:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; PACKED:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; PACKED:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; PACKED:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; PACKED:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; PACKED:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; PACKED:   BUFFER_STORE_FORMAT_D16_XY_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 4095, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom "TargetCustom7" + 4095, align 1, addrspace 4)
+  ; PACKED:   S_ENDPGM 0
+  %voffset.add = add i32 %voffset, 4095
+  call void @llvm.amdgcn.raw.buffer.store.format.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_voffset_add_4096(<4 x i32> inreg %rsrc, <2 x half> %val, i32 %voffset, i32 inreg %soffset) {
+  ; UNPACKED-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_voffset_add_4096
+  ; UNPACKED: bb.1 (%ir-block.0):
+  ; UNPACKED:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; UNPACKED:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; UNPACKED:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; UNPACKED:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; UNPACKED:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; UNPACKED:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; UNPACKED:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; UNPACKED:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; UNPACKED:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; UNPACKED:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4096, implicit $exec
+  ; UNPACKED:   %23:vgpr_32, dead %24:sreg_64 = V_ADD_I32_e64 [[COPY5]], killed [[V_MOV_B32_e32_]], 0, implicit $exec
+  ; UNPACKED:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+  ; UNPACKED:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; UNPACKED:   [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY7]], [[COPY4]], implicit $exec
+  ; UNPACKED:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[V_LSHRREV_B32_e64_]], %subreg.sub1
+  ; UNPACKED:   BUFFER_STORE_FORMAT_D16_XY_gfx80_OFFEN_exact [[REG_SEQUENCE1]], %23, [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom "TargetCustom7" + 4096, align 1, addrspace 4)
+  ; UNPACKED:   S_ENDPGM 0
+  ; PACKED-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_voffset_add_4096
+  ; PACKED: bb.1 (%ir-block.0):
+  ; PACKED:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; PACKED:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; PACKED:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; PACKED:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; PACKED:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; PACKED:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; PACKED:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; PACKED:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; PACKED:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; PACKED:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4096, implicit $exec
+  ; PACKED:   %14:vgpr_32, dead %15:sreg_64 = V_ADD_I32_e64 [[COPY5]], killed [[V_MOV_B32_e32_]], 0, implicit $exec
+  ; PACKED:   BUFFER_STORE_FORMAT_D16_XY_OFFEN_exact [[COPY4]], %14, [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom "TargetCustom7" + 4096, align 1, addrspace 4)
+  ; PACKED:   S_ENDPGM 0
+  %voffset.add = add i32 %voffset, 4096
+  call void @llvm.amdgcn.raw.buffer.store.format.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
+  ret void
+}
+
+
+; Check what happens with offset add inside a waterfall loop
+define amdgpu_ps void @raw_buffer_store_format__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f16_add_4096(<4 x i32> %rsrc, <4 x half> %val, i32 %voffset, i32 inreg %soffset) {
+  ; UNPACKED-LABEL: name: raw_buffer_store_format__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f16_add_4096
+  ; UNPACKED: bb.1 (%ir-block.0):
+  ; UNPACKED:   successors: %bb.2(0x80000000)
+  ; UNPACKED:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; UNPACKED:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; UNPACKED:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; UNPACKED:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; UNPACKED:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; UNPACKED:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; UNPACKED:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; UNPACKED:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; UNPACKED:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; UNPACKED:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; UNPACKED:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4096, implicit $exec
+  ; UNPACKED:   %48:vgpr_32, dead %49:sreg_64 = V_ADD_I32_e64 [[COPY6]], killed [[V_MOV_B32_e32_]], 0, implicit $exec
+  ; UNPACKED:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+  ; UNPACKED:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; UNPACKED:   [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY8]], [[COPY4]], implicit $exec
+  ; UNPACKED:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; UNPACKED:   [[V_LSHRREV_B32_e64_1:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY9]], [[COPY5]], implicit $exec
+  ; UNPACKED:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[V_LSHRREV_B32_e64_]], %subreg.sub1, [[COPY5]], %subreg.sub2, [[V_LSHRREV_B32_e64_1]], %subreg.sub3
+  ; UNPACKED:   [[COPY10:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; UNPACKED:   [[COPY11:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; UNPACKED:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term $exec
+  ; UNPACKED: bb.2:
+  ; UNPACKED:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; UNPACKED:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]].sub0, implicit $exec
+  ; UNPACKED:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]].sub1, implicit $exec
+  ; UNPACKED:   [[REG_SEQUENCE2:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; UNPACKED:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[COPY10]], implicit $exec
+  ; UNPACKED:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY11]].sub0, implicit $exec
+  ; UNPACKED:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY11]].sub1, implicit $exec
+  ; UNPACKED:   [[REG_SEQUENCE3:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; UNPACKED:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE3]], [[COPY11]], implicit $exec
+  ; UNPACKED:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_1]], [[V_CMP_EQ_U64_e64_]], implicit-def $scc
+  ; UNPACKED:   [[REG_SEQUENCE4:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; UNPACKED:   BUFFER_STORE_FORMAT_D16_XYZW_gfx80_OFFEN_exact [[REG_SEQUENCE1]], %48, [[REG_SEQUENCE4]], [[COPY7]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 8 into custom "TargetCustom7" + 4096, align 1, addrspace 4)
+  ; UNPACKED:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; UNPACKED:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; UNPACKED:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; UNPACKED: bb.3:
+  ; UNPACKED:   successors: %bb.4(0x80000000)
+  ; UNPACKED:   $exec = S_MOV_B64_term [[S_MOV_B64_term]]
+  ; UNPACKED: bb.4:
+  ; UNPACKED:   S_ENDPGM 0
+  ; PACKED-LABEL: name: raw_buffer_store_format__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f16_add_4096
+  ; PACKED: bb.1 (%ir-block.0):
+  ; PACKED:   successors: %bb.2(0x80000000)
+  ; PACKED:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; PACKED:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; PACKED:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; PACKED:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; PACKED:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; PACKED:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; PACKED:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; PACKED:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; PACKED:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; PACKED:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; PACKED:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; PACKED:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4096, implicit $exec
+  ; PACKED:   %32:vgpr_32, dead %33:sreg_64 = V_ADD_I32_e64 [[COPY6]], killed [[V_MOV_B32_e32_]], 0, implicit $exec
+  ; PACKED:   [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; PACKED:   [[COPY9:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; PACKED:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term $exec
+  ; PACKED: bb.2:
+  ; PACKED:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; PACKED:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]].sub0, implicit $exec
+  ; PACKED:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]].sub1, implicit $exec
+  ; PACKED:   [[REG_SEQUENCE2:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; PACKED:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[COPY8]], implicit $exec
+  ; PACKED:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]].sub0, implicit $exec
+  ; PACKED:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]].sub1, implicit $exec
+  ; PACKED:   [[REG_SEQUENCE3:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; PACKED:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE3]], [[COPY9]], implicit $exec
+  ; PACKED:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_1]], [[V_CMP_EQ_U64_e64_]], implicit-def $scc
+  ; PACKED:   [[REG_SEQUENCE4:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; PACKED:   BUFFER_STORE_FORMAT_D16_XYZW_OFFEN_exact [[REG_SEQUENCE1]], %32, [[REG_SEQUENCE4]], [[COPY7]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 8 into custom "TargetCustom7" + 4096, align 1, addrspace 4)
+  ; PACKED:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; PACKED:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; PACKED:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; PACKED: bb.3:
+  ; PACKED:   successors: %bb.4(0x80000000)
+  ; PACKED:   $exec = S_MOV_B64_term [[S_MOV_B64_term]]
+  ; PACKED: bb.4:
+  ; PACKED:   S_ENDPGM 0
+  %voffset.add = add i32 %voffset, 4096
+  call void @llvm.amdgcn.raw.buffer.store.format.v4f16(<4 x half> %val, <4 x i32> %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
+  ret void
+}
+
+declare void @llvm.amdgcn.raw.buffer.store.format.f16(half, <4 x i32>, i32, i32, i32 immarg)
+declare void @llvm.amdgcn.raw.buffer.store.format.v2f16(<2 x half>, <4 x i32>, i32, i32, i32 immarg)
+declare void @llvm.amdgcn.raw.buffer.store.format.v4f16(<4 x half>, <4 x i32>, i32, i32, i32 immarg)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.format.f32.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.format.f32.ll
new file mode 100644
index 000000000..aea37fd08
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.format.f32.ll
@@ -0,0 +1,306 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=tonga -stop-after=instruction-select -o - %s | FileCheck %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx810 -stop-after=instruction-select -o - %s | FileCheck %s
+
+define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_f32(<4 x i32> inreg %rsrc, float %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_f32
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   BUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom "TargetCustom7", align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.format.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__voffset_4095__sgpr_soffset_f32(<4 x i32> inreg %rsrc, float %val, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__voffset_4095__sgpr_soffset_f32
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   BUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY4]], $noreg, [[REG_SEQUENCE]], [[COPY5]], 4095, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom "TargetCustom7" + 4095, align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.format.f32(float %val, <4 x i32> %rsrc, i32 4095, i32 %soffset, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f32(<4 x i32> inreg %rsrc, <2 x float> %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f32
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; CHECK:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; CHECK:   BUFFER_STORE_FORMAT_XY_OFFEN_exact [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 8 into custom "TargetCustom7", align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.format.v2f32(<2 x float> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v3f32(<4 x i32> inreg %rsrc, <3 x float> %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v3f32
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; CHECK:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; CHECK:   [[COPY8:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_96 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY6]], %subreg.sub2
+  ; CHECK:   BUFFER_STORE_FORMAT_XYZ_OFFEN_exact [[REG_SEQUENCE1]], [[COPY7]], [[REG_SEQUENCE]], [[COPY8]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 12 into custom "TargetCustom7", align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.format.v3f32(<3 x float> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f32(<4 x i32> inreg %rsrc, <4 x float> %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f32
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; CHECK:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; CHECK:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; CHECK:   [[COPY9:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY6]], %subreg.sub2, [[COPY7]], %subreg.sub3
+  ; CHECK:   BUFFER_STORE_FORMAT_XYZW_OFFEN_exact [[REG_SEQUENCE1]], [[COPY8]], [[REG_SEQUENCE]], [[COPY9]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 16 into custom "TargetCustom7", align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.format.v4f32(<4 x float> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store_format__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f32(<4 x i32> %rsrc, <4 x float> %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store_format__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f32
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; CHECK:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; CHECK:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+  ; CHECK:   [[COPY9:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY6]], %subreg.sub2, [[COPY7]], %subreg.sub3
+  ; CHECK:   [[COPY10:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; CHECK:   [[COPY11:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; CHECK:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term $exec
+  ; CHECK: bb.2:
+  ; CHECK:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]].sub0, implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]].sub1, implicit $exec
+  ; CHECK:   [[REG_SEQUENCE2:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; CHECK:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[COPY10]], implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY11]].sub0, implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY11]].sub1, implicit $exec
+  ; CHECK:   [[REG_SEQUENCE3:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; CHECK:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE3]], [[COPY11]], implicit $exec
+  ; CHECK:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_1]], [[V_CMP_EQ_U64_e64_]], implicit-def $scc
+  ; CHECK:   [[REG_SEQUENCE4:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; CHECK:   BUFFER_STORE_FORMAT_XYZW_OFFEN_exact [[REG_SEQUENCE1]], [[COPY8]], [[REG_SEQUENCE4]], [[COPY9]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 16 into custom "TargetCustom7", align 1, addrspace 4)
+  ; CHECK:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; CHECK:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; CHECK:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; CHECK: bb.3:
+  ; CHECK:   successors: %bb.4(0x80000000)
+  ; CHECK:   $exec = S_MOV_B64_term [[S_MOV_B64_term]]
+  ; CHECK: bb.4:
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.format.v4f32(<4 x float> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f32_soffset4095(<4 x i32> inreg %rsrc, <2 x float> %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f32_soffset4095
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; CHECK:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4095
+  ; CHECK:   BUFFER_STORE_FORMAT_XY_OFFEN_exact [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 8 into custom "TargetCustom7", align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.format.v2f32(<2 x float> %val, <4 x i32> %rsrc, i32 %voffset, i32 4095, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f32_soffset4096(<4 x i32> inreg %rsrc, <2 x float> %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f32_soffset4096
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; CHECK:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4096
+  ; CHECK:   BUFFER_STORE_FORMAT_XY_OFFEN_exact [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 8 into custom "TargetCustom7", align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.format.v2f32(<2 x float> %val, <4 x i32> %rsrc, i32 %voffset, i32 4096, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f32_voffset_add_16(<4 x i32> inreg %rsrc, <2 x float> %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f32_voffset_add_16
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; CHECK:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; CHECK:   BUFFER_STORE_FORMAT_XY_OFFEN_exact [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 16, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 8 into custom "TargetCustom7" + 16, align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  %voffset.add = add i32 %voffset, 16
+  call void @llvm.amdgcn.raw.buffer.store.format.v2f32(<2 x float> %val, <4 x i32> %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f32_voffset_add_4095(<4 x i32> inreg %rsrc, <2 x float> %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f32_voffset_add_4095
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; CHECK:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; CHECK:   BUFFER_STORE_FORMAT_XY_OFFEN_exact [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 4095, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 8 into custom "TargetCustom7" + 4095, align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  %voffset.add = add i32 %voffset, 4095
+  call void @llvm.amdgcn.raw.buffer.store.format.v2f32(<2 x float> %val, <4 x i32> %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f32_voffset_add_4096(<4 x i32> inreg %rsrc, <2 x float> %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f32_voffset_add_4096
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; CHECK:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; CHECK:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4096, implicit $exec
+  ; CHECK:   %16:vgpr_32, dead %17:sreg_64 = V_ADD_I32_e64 [[COPY6]], killed [[V_MOV_B32_e32_]], 0, implicit $exec
+  ; CHECK:   BUFFER_STORE_FORMAT_XY_OFFEN_exact [[REG_SEQUENCE1]], %16, [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 8 into custom "TargetCustom7" + 4096, align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  %voffset.add = add i32 %voffset, 4096
+  call void @llvm.amdgcn.raw.buffer.store.format.v2f32(<2 x float> %val, <4 x i32> %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
+  ret void
+}
+
+
+; Check what happens with offset add inside a waterfall loop
+define amdgpu_ps void @raw_buffer_store_format__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f32_add_4096(<4 x i32> %rsrc, <4 x float> %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store_format__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f32_add_4096
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; CHECK:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; CHECK:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+  ; CHECK:   [[COPY9:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY6]], %subreg.sub2, [[COPY7]], %subreg.sub3
+  ; CHECK:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4096, implicit $exec
+  ; CHECK:   %34:vgpr_32, dead %35:sreg_64 = V_ADD_I32_e64 [[COPY8]], killed [[V_MOV_B32_e32_]], 0, implicit $exec
+  ; CHECK:   [[COPY10:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; CHECK:   [[COPY11:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; CHECK:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term $exec
+  ; CHECK: bb.2:
+  ; CHECK:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]].sub0, implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]].sub1, implicit $exec
+  ; CHECK:   [[REG_SEQUENCE2:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; CHECK:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[COPY10]], implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY11]].sub0, implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY11]].sub1, implicit $exec
+  ; CHECK:   [[REG_SEQUENCE3:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; CHECK:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE3]], [[COPY11]], implicit $exec
+  ; CHECK:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_1]], [[V_CMP_EQ_U64_e64_]], implicit-def $scc
+  ; CHECK:   [[REG_SEQUENCE4:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; CHECK:   BUFFER_STORE_FORMAT_XYZW_OFFEN_exact [[REG_SEQUENCE1]], %34, [[REG_SEQUENCE4]], [[COPY9]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 16 into custom "TargetCustom7" + 4096, align 1, addrspace 4)
+  ; CHECK:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; CHECK:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; CHECK:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; CHECK: bb.3:
+  ; CHECK:   successors: %bb.4(0x80000000)
+  ; CHECK:   $exec = S_MOV_B64_term [[S_MOV_B64_term]]
+  ; CHECK: bb.4:
+  ; CHECK:   S_ENDPGM 0
+  %voffset.add = add i32 %voffset, 4096
+  call void @llvm.amdgcn.raw.buffer.store.format.v4f32(<4 x float> %val, <4 x i32> %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
+  ret void
+}
+
+declare void @llvm.amdgcn.raw.buffer.store.format.f32(float, <4 x i32>, i32, i32, i32 immarg)
+declare void @llvm.amdgcn.raw.buffer.store.format.v2f32(<2 x float>, <4 x i32>, i32, i32, i32 immarg)
+declare void @llvm.amdgcn.raw.buffer.store.format.v3f32(<3 x float>, <4 x i32>, i32, i32, i32 immarg)
+declare void @llvm.amdgcn.raw.buffer.store.format.v4f32(<4 x float>, <4 x i32>, i32, i32, i32 immarg)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.ll
new file mode 100644
index 000000000..c5aa36df8
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.ll
@@ -0,0 +1,776 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -stop-after=instruction-select -verify-machineinstrs -o - %s | FileCheck %s
+; FIXME: Test with SI when argument lowering not broken for f16
+
+; Natural mapping
+define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset(<4 x i32> inreg %rsrc, float %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom "TargetCustom7", align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+; Copies for VGPR arguments
+define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__sgpr_val__sgpr_voffset__sgpr_soffset(<4 x i32> inreg %rsrc, float inreg %val, i32 inreg %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store__sgpr_rsrc__sgpr_val__sgpr_voffset__sgpr_soffset
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr7
+  ; CHECK:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr8
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY4]]
+  ; CHECK:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
+  ; CHECK:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY7]], [[COPY8]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom "TargetCustom7", align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+; Waterfall for rsrc
+define amdgpu_ps void @raw_buffer_store__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset(<4 x i32> %rsrc, float %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; CHECK:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   [[COPY7:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; CHECK:   [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; CHECK:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term $exec
+  ; CHECK: bb.2:
+  ; CHECK:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]].sub0, implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]].sub1, implicit $exec
+  ; CHECK:   [[REG_SEQUENCE1:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; CHECK:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[COPY7]], implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]].sub0, implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]].sub1, implicit $exec
+  ; CHECK:   [[REG_SEQUENCE2:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; CHECK:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[COPY8]], implicit $exec
+  ; CHECK:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_1]], [[V_CMP_EQ_U64_e64_]], implicit-def $scc
+  ; CHECK:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; CHECK:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE3]], [[COPY6]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom "TargetCustom7", align 1, addrspace 4)
+  ; CHECK:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; CHECK:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; CHECK:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; CHECK: bb.3:
+  ; CHECK:   successors: %bb.4(0x80000000)
+  ; CHECK:   $exec = S_MOV_B64_term [[S_MOV_B64_term]]
+  ; CHECK: bb.4:
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+; Waterfall for soffset
+define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__vgpr_soffset(<4 x i32> inreg %rsrc, float %val, i32 %voffset, i32 %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__vgpr_soffset
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $vgpr0, $vgpr1, $vgpr2
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term $exec
+  ; CHECK: bb.2:
+  ; CHECK:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
+  ; CHECK:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]], [[COPY6]], implicit $exec
+  ; CHECK:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[V_READFIRSTLANE_B32_]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom "TargetCustom7", align 1, addrspace 4)
+  ; CHECK:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[V_CMP_EQ_U32_e64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; CHECK:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; CHECK:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; CHECK: bb.3:
+  ; CHECK:   successors: %bb.4(0x80000000)
+  ; CHECK:   $exec = S_MOV_B64_term [[S_MOV_B64_term]]
+  ; CHECK: bb.4:
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+; Waterfall for rsrc and soffset
+define amdgpu_ps void @raw_buffer_store__vgpr_rsrc__vgpr_val__vgpr_voffset__vgpr_soffset(<4 x i32> %rsrc, float %val, i32 %voffset, i32 %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store__vgpr_rsrc__vgpr_val__vgpr_voffset__vgpr_soffset
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; CHECK:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   [[COPY7:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; CHECK:   [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; CHECK:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term $exec
+  ; CHECK: bb.2:
+  ; CHECK:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]].sub0, implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]].sub1, implicit $exec
+  ; CHECK:   [[REG_SEQUENCE1:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; CHECK:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[COPY7]], implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]].sub0, implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]].sub1, implicit $exec
+  ; CHECK:   [[REG_SEQUENCE2:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; CHECK:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[COPY8]], implicit $exec
+  ; CHECK:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_1]], [[V_CMP_EQ_U64_e64_]], implicit-def $scc
+  ; CHECK:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; CHECK:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
+  ; CHECK:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY6]], implicit $exec
+  ; CHECK:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U32_e64_]], [[S_AND_B64_]], implicit-def $scc
+  ; CHECK:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE3]], [[V_READFIRSTLANE_B32_4]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom "TargetCustom7", align 1, addrspace 4)
+  ; CHECK:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; CHECK:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; CHECK:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; CHECK: bb.3:
+  ; CHECK:   successors: %bb.4(0x80000000)
+  ; CHECK:   $exec = S_MOV_B64_term [[S_MOV_B64_term]]
+  ; CHECK: bb.4:
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_glc(<4 x i32> inreg %rsrc, float %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_glc
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 1, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom "TargetCustom7", align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 1)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_slc(<4 x i32> inreg %rsrc, float %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_slc
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 1, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom "TargetCustom7", align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 2)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_glc_slc(<4 x i32> inreg %rsrc, float %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_glc_slc
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 1, 1, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom "TargetCustom7", align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 3)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_dlc(<4 x i32> inreg %rsrc, float %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_dlc
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, 0, 1, 0, implicit $exec :: (dereferenceable store 4 into custom "TargetCustom7", align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 4)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_slc_dlc(<4 x i32> inreg %rsrc, float %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_slc_dlc
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 1, 0, 1, 0, implicit $exec :: (dereferenceable store 4 into custom "TargetCustom7", align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 6)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_glc_dlc(<4 x i32> inreg %rsrc, float %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_glc_dlc
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 1, 0, 0, 1, 0, implicit $exec :: (dereferenceable store 4 into custom "TargetCustom7", align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 5)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_glc_slc_dlc(<4 x i32> inreg %rsrc, float %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_glc_slc_dlc
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 1, 1, 0, 1, 0, implicit $exec :: (dereferenceable store 4 into custom "TargetCustom7", align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 7)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f32(<4 x i32> inreg %rsrc, <2 x float> %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f32
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; CHECK:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; CHECK:   BUFFER_STORE_DWORDX2_OFFEN_exact [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 8 into custom "TargetCustom7", align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.v2f32(<2 x float> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v3f32(<4 x i32> inreg %rsrc, <3 x float> %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v3f32
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; CHECK:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; CHECK:   [[COPY8:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_96 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY6]], %subreg.sub2
+  ; CHECK:   BUFFER_STORE_DWORDX3_OFFEN_exact [[REG_SEQUENCE1]], [[COPY7]], [[REG_SEQUENCE]], [[COPY8]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 12 into custom "TargetCustom7", align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.v3f32(<3 x float> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f32(<4 x i32> inreg %rsrc, <4 x float> %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f32
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; CHECK:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; CHECK:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; CHECK:   [[COPY9:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY6]], %subreg.sub2, [[COPY7]], %subreg.sub3
+  ; CHECK:   BUFFER_STORE_DWORDX4_OFFEN_exact [[REG_SEQUENCE1]], [[COPY8]], [[REG_SEQUENCE]], [[COPY9]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 16 into custom "TargetCustom7", align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.v4f32(<4 x float> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_i8(<4 x i32> inreg %rsrc, i32 %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_i8
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   BUFFER_STORE_BYTE_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 1 into custom "TargetCustom7", addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  %val.trunc = trunc i32 %val to i8
+  call void @llvm.amdgcn.raw.buffer.store.i8(i8 %val.trunc, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_i16(<4 x i32> inreg %rsrc, i32 %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_i16
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   BUFFER_STORE_SHORT_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 2 into custom "TargetCustom7", align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  %val.trunc = trunc i32 %val to i16
+  call void @llvm.amdgcn.raw.buffer.store.i16(i16 %val.trunc, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_f16(<4 x i32> inreg %rsrc, half %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_f16
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   BUFFER_STORE_SHORT_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 2 into custom "TargetCustom7", align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.f16(half %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16(<4 x i32> inreg %rsrc, <2 x half> %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom "TargetCustom7", align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f16(<4 x i32> inreg %rsrc, <4 x half> %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f16
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; CHECK:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; CHECK:   BUFFER_STORE_DWORDX2_OFFEN_exact [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 8 into custom "TargetCustom7", align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.v4f16(<4 x half> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f16(<4 x i32> %rsrc, <4 x half> %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f16
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; CHECK:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; CHECK:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; CHECK:   [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; CHECK:   [[COPY9:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; CHECK:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term $exec
+  ; CHECK: bb.2:
+  ; CHECK:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]].sub0, implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]].sub1, implicit $exec
+  ; CHECK:   [[REG_SEQUENCE2:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; CHECK:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[COPY8]], implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]].sub0, implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]].sub1, implicit $exec
+  ; CHECK:   [[REG_SEQUENCE3:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; CHECK:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE3]], [[COPY9]], implicit $exec
+  ; CHECK:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_1]], [[V_CMP_EQ_U64_e64_]], implicit-def $scc
+  ; CHECK:   [[REG_SEQUENCE4:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; CHECK:   BUFFER_STORE_DWORDX2_OFFEN_exact [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE4]], [[COPY7]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 8 into custom "TargetCustom7", align 1, addrspace 4)
+  ; CHECK:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; CHECK:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; CHECK:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; CHECK: bb.3:
+  ; CHECK:   successors: %bb.4(0x80000000)
+  ; CHECK:   $exec = S_MOV_B64_term [[S_MOV_B64_term]]
+  ; CHECK: bb.4:
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.v4f16(<4 x half> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__sgpr_soffset_f32_voffset4095(<4 x i32> inreg %rsrc, float %val, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__sgpr_soffset_f32_voffset4095
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], $noreg, [[REG_SEQUENCE]], [[COPY5]], 4095, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom "TargetCustom7" + 4095, align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 4095, i32 %soffset, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__sgpr_soffset_f32_voffset4096(<4 x i32> inreg %rsrc, float %val, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__sgpr_soffset_f32_voffset4096
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4096, implicit $exec
+  ; CHECK:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[V_MOV_B32_e32_]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom "TargetCustom7" + 4096, align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 4096, i32 %soffset, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_f32_voffset_add_16(<4 x i32> inreg %rsrc, float %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_f32_voffset_add_16
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 16, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom "TargetCustom7" + 16, align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  %voffset.add = add i32 %voffset, 16
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_f32_voffset_add_4095(<4 x i32> inreg %rsrc, float %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_f32_voffset_add_4095
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 4095, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom "TargetCustom7" + 4095, align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  %voffset.add = add i32 %voffset, 4095
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_f32_voffset_add_4096(<4 x i32> inreg %rsrc, float %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_f32_voffset_add_4096
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4096, implicit $exec
+  ; CHECK:   %14:vgpr_32, dead %15:sreg_64 = V_ADD_I32_e64 [[COPY5]], killed [[V_MOV_B32_e32_]], 0, implicit $exec
+  ; CHECK:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], %14, [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom "TargetCustom7" + 4096, align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  %voffset.add = add i32 %voffset, 4096
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_soffset4095(<4 x i32> inreg %rsrc, <2 x half> %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_soffset4095
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4095
+  ; CHECK:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom "TargetCustom7", align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %voffset, i32 4095, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_soffset4096(<4 x i32> inreg %rsrc, <2 x half> %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_soffset4096
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4096
+  ; CHECK:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom "TargetCustom7", align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %voffset, i32 4096, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_soffset_add_16(<4 x i32> inreg %rsrc, <2 x half> %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_soffset_add_16
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 16, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom "TargetCustom7" + 16, align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  %voffset.add = add i32 %voffset, 16
+  call void @llvm.amdgcn.raw.buffer.store.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_soffset_add_4095(<4 x i32> inreg %rsrc, <2 x half> %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_soffset_add_4095
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 4095, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom "TargetCustom7" + 4095, align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  %voffset.add = add i32 %voffset, 4095
+  call void @llvm.amdgcn.raw.buffer.store.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_soffset_add_4096(<4 x i32> inreg %rsrc, <2 x half> %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_soffset_add_4096
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4096, implicit $exec
+  ; CHECK:   %14:vgpr_32, dead %15:sreg_64 = V_ADD_I32_e64 [[COPY5]], killed [[V_MOV_B32_e32_]], 0, implicit $exec
+  ; CHECK:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], %14, [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom "TargetCustom7" + 4096, align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  %voffset.add = add i32 %voffset, 4096
+  call void @llvm.amdgcn.raw.buffer.store.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
+  ret void
+}
+
+; An add of the offset is necessary, with a waterfall loop. Make sure the add is done outside of the waterfall loop.
+define amdgpu_ps void @raw_buffer_store__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_offset_add_5000(<4 x i32> %rsrc, float %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_offset_add_5000
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; CHECK:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4096, implicit $exec
+  ; CHECK:   %30:vgpr_32, dead %31:sreg_64 = V_ADD_I32_e64 [[COPY5]], killed [[V_MOV_B32_e32_]], 0, implicit $exec
+  ; CHECK:   [[COPY7:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; CHECK:   [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; CHECK:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term $exec
+  ; CHECK: bb.2:
+  ; CHECK:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]].sub0, implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]].sub1, implicit $exec
+  ; CHECK:   [[REG_SEQUENCE1:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; CHECK:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[COPY7]], implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]].sub0, implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]].sub1, implicit $exec
+  ; CHECK:   [[REG_SEQUENCE2:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; CHECK:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[COPY8]], implicit $exec
+  ; CHECK:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_1]], [[V_CMP_EQ_U64_e64_]], implicit-def $scc
+  ; CHECK:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; CHECK:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], %30, [[REG_SEQUENCE3]], [[COPY6]], 904, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom "TargetCustom7" + 5000, align 1, addrspace 4)
+  ; CHECK:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; CHECK:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; CHECK:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; CHECK: bb.3:
+  ; CHECK:   successors: %bb.4(0x80000000)
+  ; CHECK:   $exec = S_MOV_B64_term [[S_MOV_B64_term]]
+  ; CHECK: bb.4:
+  ; CHECK:   S_ENDPGM 0
+  %voffset.add = add i32 %voffset, 5000
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
+  ret void
+}
+
+; An add of the offset is necessary, with a waterfall loop. Make sure the add is done outside of the waterfall loop.
+define amdgpu_ps void @raw_buffer_store__vgpr_rsrc__vgpr_val__5000_voffset__sgpr_soffset_offset(<4 x i32> %rsrc, float %val, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store__vgpr_rsrc__vgpr_val__5000_voffset__sgpr_soffset_offset
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4096, implicit $exec
+  ; CHECK:   [[COPY6:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; CHECK:   [[COPY7:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; CHECK:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term $exec
+  ; CHECK: bb.2:
+  ; CHECK:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]].sub0, implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]].sub1, implicit $exec
+  ; CHECK:   [[REG_SEQUENCE1:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; CHECK:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[COPY6]], implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]].sub0, implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]].sub1, implicit $exec
+  ; CHECK:   [[REG_SEQUENCE2:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; CHECK:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[COPY7]], implicit $exec
+  ; CHECK:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_1]], [[V_CMP_EQ_U64_e64_]], implicit-def $scc
+  ; CHECK:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; CHECK:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[V_MOV_B32_e32_]], [[REG_SEQUENCE3]], [[COPY5]], 904, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom "TargetCustom7" + 5000, align 1, addrspace 4)
+  ; CHECK:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; CHECK:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; CHECK:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; CHECK: bb.3:
+  ; CHECK:   successors: %bb.4(0x80000000)
+  ; CHECK:   $exec = S_MOV_B64_term [[S_MOV_B64_term]]
+  ; CHECK: bb.4:
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 5000, i32 %soffset, i32 0)
+  ret void
+}
+
+declare void @llvm.amdgcn.raw.buffer.store.i8(i8, <4 x i32>, i32, i32, i32 immarg)
+declare void @llvm.amdgcn.raw.buffer.store.i16(i16, <4 x i32>, i32, i32, i32 immarg)
+
+declare void @llvm.amdgcn.raw.buffer.store.f16(half, <4 x i32>, i32, i32, i32 immarg)
+declare void @llvm.amdgcn.raw.buffer.store.v2f16(<2 x half>, <4 x i32>, i32, i32, i32 immarg)
+declare void @llvm.amdgcn.raw.buffer.store.v4f16(<4 x half>, <4 x i32>, i32, i32, i32 immarg)
+
+declare void @llvm.amdgcn.raw.buffer.store.f32(float, <4 x i32>, i32, i32, i32 immarg)
+declare void @llvm.amdgcn.raw.buffer.store.v2f32(<2 x float>, <4 x i32>, i32, i32, i32 immarg)
+declare void @llvm.amdgcn.raw.buffer.store.v3f32(<3 x float>, <4 x i32>, i32, i32, i32 immarg)
+declare void @llvm.amdgcn.raw.buffer.store.v4f32(<4 x float>, <4 x i32>, i32, i32, i32 immarg)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.s.sleep.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.s.sleep.ll
new file mode 100644
index 000000000..55d142cc3
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.s.sleep.ll
@@ -0,0 +1,45 @@
+; RUN: llc -global-isel -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -global-isel -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+
+declare void @llvm.amdgcn.s.sleep(i32) #0
+
+; GCN-LABEL: {{^}}test_s_sleep:
+; GCN: s_sleep 0{{$}}
+; GCN: s_sleep 1{{$}}
+; GCN: s_sleep 2{{$}}
+; GCN: s_sleep 3{{$}}
+; GCN: s_sleep 4{{$}}
+; GCN: s_sleep 5{{$}}
+; GCN: s_sleep 6{{$}}
+; GCN: s_sleep 7{{$}}
+; GCN: s_sleep 8{{$}}
+; GCN: s_sleep 9{{$}}
+; GCN: s_sleep 10{{$}}
+; GCN: s_sleep 11{{$}}
+; GCN: s_sleep 12{{$}}
+; GCN: s_sleep 13{{$}}
+; GCN: s_sleep 14{{$}}
+; GCN: s_sleep 15{{$}}
+define amdgpu_kernel void @test_s_sleep(i32 %x) #0 {
+  call void @llvm.amdgcn.s.sleep(i32 0)
+  call void @llvm.amdgcn.s.sleep(i32 1)
+  call void @llvm.amdgcn.s.sleep(i32 2)
+  call void @llvm.amdgcn.s.sleep(i32 3)
+  call void @llvm.amdgcn.s.sleep(i32 4)
+  call void @llvm.amdgcn.s.sleep(i32 5)
+  call void @llvm.amdgcn.s.sleep(i32 6)
+  call void @llvm.amdgcn.s.sleep(i32 7)
+
+  ; Values that might only work on VI
+  call void @llvm.amdgcn.s.sleep(i32 8)
+  call void @llvm.amdgcn.s.sleep(i32 9)
+  call void @llvm.amdgcn.s.sleep(i32 10)
+  call void @llvm.amdgcn.s.sleep(i32 11)
+  call void @llvm.amdgcn.s.sleep(i32 12)
+  call void @llvm.amdgcn.s.sleep(i32 13)
+  call void @llvm.amdgcn.s.sleep(i32 14)
+  call void @llvm.amdgcn.s.sleep(i32 15)
+  ret void
+}
+
+attributes #0 = { nounwind }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.workgroup.id.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.workgroup.id.ll
index ef87f0a2c..494dfb406 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.workgroup.id.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.workgroup.id.ll
@@ -1,8 +1,8 @@
 ; RUN: llc -global-isel -mtriple=amdgcn-unknown-amdhsa -mattr=-code-object-v3 -mcpu=kaveri -verify-machineinstrs < %s | FileCheck -check-prefix=ALL -check-prefix=CO-V2 -check-prefix=CI-HSA  %s
 ; RUN: llc -global-isel -mtriple=amdgcn-unknown-amdhsa -mattr=-code-object-v3 -mcpu=carrizo -verify-machineinstrs < %s | FileCheck -check-prefix=ALL -check-prefix=CO-V2 -check-prefix=VI-HSA  %s
-; RUN: llc -global-isel -mtriple=amdgcn-- -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefix=ALL -check-prefix=UNKNOWN-OS -check-prefix=SI-MESA %s
+; RUN: llc -global-isel -mtriple=amdgcn-- -mcpu=hawaii -verify-machineinstrs < %s | FileCheck -check-prefix=ALL -check-prefix=UNKNOWN-OS -check-prefix=SI-MESA %s
 ; RUN: llc -global-isel -mtriple=amdgcn-- -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=ALL -check-prefix=UNKNOWN-OS -check-prefix=VI-MESA %s
-; RUN: llc -global-isel -mtriple=amdgcn-unknown-mesa3d -mattr=-code-object-v3 -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefixes=ALL,CO-V2,SI-MESA %s
+; RUN: llc -global-isel -mtriple=amdgcn-unknown-mesa3d -mattr=-code-object-v3 -mcpu=hawaii -verify-machineinstrs < %s | FileCheck -check-prefixes=ALL,CO-V2,SI-MESA %s
 ; RUN: llc -global-isel -mtriple=amdgcn-unknown-mesa3d -mattr=-code-object-v3 -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefixes=ALL,CO-V2,VI-MESA %s
 
 declare i32 @llvm.amdgcn.workgroup.id.x() #0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.workitem.id.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.workitem.id.ll
index efd6cc758..946a138be 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.workitem.id.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.workitem.id.ll
@@ -1,8 +1,8 @@
 ; RUN: llc -global-isel -mtriple=amdgcn-unknown-amdhsa -mattr=-code-object-v3 -mcpu=kaveri -verify-machineinstrs < %s | FileCheck -check-prefix=ALL -check-prefix=CO-V2 -check-prefix=CI-HSA  %s
 ; RUN: llc -global-isel -mtriple=amdgcn-unknown-amdhsa -mattr=-code-object-v3 -mcpu=carrizo -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=ALL -check-prefix=CO-V2 -check-prefix=VI-HSA  %s
-; RUN: llc -global-isel -mtriple=amdgcn-- -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefix=ALL -check-prefix=MESA -check-prefix=SI-MESA %s
+; RUN: llc -global-isel -mtriple=amdgcn-- -mcpu=hawaii -verify-machineinstrs < %s | FileCheck -check-prefix=ALL -check-prefix=MESA -check-prefix=SI-MESA %s
 ; RUN: llc -global-isel -mtriple=amdgcn-- -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=ALL -check-prefix=MESA -check-prefix=VI-MESA %s
-; RUN: llc -global-isel -mtriple=amdgcn-unknown-mesa3d -mattr=-code-object-v3 -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefixes=ALL,CO-V2,SI-MESA %s
+; RUN: llc -global-isel -mtriple=amdgcn-unknown-mesa3d -mattr=-code-object-v3 -mcpu=hawaii -verify-machineinstrs < %s | FileCheck -check-prefixes=ALL,CO-V2,SI-MESA %s
 ; RUN: llc -global-isel -mtriple=amdgcn-unknown-mesa3d -mattr=-code-object-v3 -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefixes=ALL,CO-V2,VI-MESA %s
 
 declare i32 @llvm.amdgcn.workitem.id.x() #0
@@ -88,5 +88,32 @@ bb2:
   ret void
 }
 
+; ALL-LABEL: {{^}}test_workitem_id_x_func:
+; ALL: s_waitcnt
+; ALL-NEXT: v_and_b32_e32 v2, 0x3ff, v2
+define void @test_workitem_id_x_func(i32 addrspace(1)* %out) #1 {
+  %id = call i32 @llvm.amdgcn.workitem.id.x()
+  store i32 %id, i32 addrspace(1)* %out
+  ret void
+}
+
+; ALL-LABEL: {{^}}test_workitem_id_y_func:
+; ALL: v_lshrrev_b32_e32 v2, 10, v2
+; ALL-NEXT: v_and_b32_e32 v2, 0x3ff, v2
+define void @test_workitem_id_y_func(i32 addrspace(1)* %out) #1 {
+  %id = call i32 @llvm.amdgcn.workitem.id.y()
+  store i32 %id, i32 addrspace(1)* %out
+  ret void
+}
+
+; ALL-LABEL: {{^}}test_workitem_id_z_func:
+; ALL: v_lshrrev_b32_e32 v2, 20, v2
+; ALL-NEXT: v_and_b32_e32 v2, 0x3ff, v2
+define void @test_workitem_id_z_func(i32 addrspace(1)* %out) #1 {
+  %id = call i32 @llvm.amdgcn.workitem.id.z()
+  store i32 %id, i32 addrspace(1)* %out
+  ret void
+}
+
 attributes #0 = { nounwind readnone }
 attributes #1 = { nounwind }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.wqm.vote.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.wqm.vote.ll
new file mode 100644
index 000000000..a0913eeb9
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.wqm.vote.ll
@@ -0,0 +1,3 @@
+; Runs original SDAG test with -global-isel
+; RUN: llc -global-isel -march=amdgcn -mcpu=tonga -verify-machineinstrs < %S/../llvm.amdgcn.wqm.vote.ll | FileCheck -enable-var-scope -check-prefixes=CHECK,WAVE64  %S/../llvm.amdgcn.wqm.vote.ll
+; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %S/../llvm.amdgcn.wqm.vote.ll | FileCheck -enable-var-scope -check-prefixes=CHECK,WAVE32 %S/../llvm.amdgcn.wqm.vote.ll
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/read_register.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/read_register.ll
new file mode 100644
index 000000000..3bd16996f
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/read_register.ll
@@ -0,0 +1,2 @@
+; Runs original SDAG test with -global-isel
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=bonaire -verify-machineinstrs < %S/../read_register.ll | FileCheck -enable-var-scope %S/../read_register.ll
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/readcyclecounter.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/readcyclecounter.ll
new file mode 100644
index 000000000..824817469
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/readcyclecounter.ll
@@ -0,0 +1,3 @@
+; SI run line skipped since store not yet implemented.
+; RUN: llc -global-isel -march=amdgcn -mcpu=tonga -verify-machineinstrs < %S/../readcyclecounter.ll | FileCheck -enable-var-scope -check-prefix=MEMTIME -check-prefix=SIVI -check-prefix=GCN %S/../readcyclecounter.ll
+; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %S/../readcyclecounter.ll | FileCheck -enable-var-scope -check-prefix=MEMTIME -check-prefix=GCN %S/../readcyclecounter.ll
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.image.load.1d.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.image.load.1d.ll
new file mode 100644
index 000000000..d4a3f4025
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.image.load.1d.ll
@@ -0,0 +1,181 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -stop-after=regbankselect -regbankselect-fast -o - %s | FileCheck %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -stop-after=regbankselect -regbankselect-greedy -o - %s | FileCheck %s
+
+; Natural mapping
+define amdgpu_ps void @load_1d_vgpr_vaddr__sgpr_srsrc(<8 x i32> inreg %rsrc, i32 %s) {
+  ; CHECK-LABEL: name: load_1d_vgpr_vaddr__sgpr_srsrc
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr(s32) = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr(s32) = COPY $sgpr6
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr(s32) = COPY $sgpr7
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr(s32) = COPY $sgpr8
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr(s32) = COPY $sgpr9
+  ; CHECK:   [[COPY8:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:sgpr(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; CHECK:   [[DEF:%[0-9]+]]:sgpr(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[INT:%[0-9]+]]:vgpr(<4 x s32>) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.image.load.1d), 15, [[COPY8]](s32), [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
+  ; CHECK:   [[COPY9:%[0-9]+]]:vgpr(p1) = COPY [[DEF]](p1)
+  ; CHECK:   G_STORE [[INT]](<4 x s32>), [[COPY9]](p1) :: (store 16 into `<4 x float> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   S_ENDPGM 0
+  %v = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i32(i32 15, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  store <4 x float> %v, <4 x float> addrspace(1)* undef
+  ret void
+}
+
+; Copy needed for VGPR argument
+define amdgpu_ps void @load_1d_sgpr_vaddr__sgpr_srsrc(<8 x i32> inreg %rsrc, i32 inreg %s) {
+  ; CHECK-LABEL: name: load_1d_sgpr_vaddr__sgpr_srsrc
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr(s32) = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr(s32) = COPY $sgpr6
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr(s32) = COPY $sgpr7
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr(s32) = COPY $sgpr8
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr(s32) = COPY $sgpr9
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr(s32) = COPY $sgpr10
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:sgpr(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; CHECK:   [[DEF:%[0-9]+]]:sgpr(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[COPY9:%[0-9]+]]:vgpr(s32) = COPY [[COPY8]](s32)
+  ; CHECK:   [[INT:%[0-9]+]]:vgpr(<4 x s32>) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.image.load.1d), 15, [[COPY9]](s32), [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
+  ; CHECK:   [[COPY10:%[0-9]+]]:vgpr(p1) = COPY [[DEF]](p1)
+  ; CHECK:   G_STORE [[INT]](<4 x s32>), [[COPY10]](p1) :: (store 16 into `<4 x float> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   S_ENDPGM 0
+  %v = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i32(i32 15, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  store <4 x float> %v, <4 x float> addrspace(1)* undef
+  ret void
+}
+
+; Waterfall loop needed for rsrc
+define amdgpu_ps void @load_1d_vgpr_vaddr__vgpr_srsrc(<8 x i32> %rsrc, i32 %s) {
+  ; CHECK-LABEL: name: load_1d_vgpr_vaddr__vgpr_srsrc
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr(s32) = COPY $vgpr5
+  ; CHECK:   [[COPY6:%[0-9]+]]:vgpr(s32) = COPY $vgpr6
+  ; CHECK:   [[COPY7:%[0-9]+]]:vgpr(s32) = COPY $vgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:vgpr(s32) = COPY $vgpr8
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:vgpr(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; CHECK:   [[DEF:%[0-9]+]]:sgpr(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[DEF1:%[0-9]+]]:vgpr(<4 x s32>) = G_IMPLICIT_DEF
+  ; CHECK:   [[DEF2:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
+  ; CHECK:   [[UV:%[0-9]+]]:vreg_64(s64), [[UV1:%[0-9]+]]:vreg_64(s64), [[UV2:%[0-9]+]]:vreg_64(s64), [[UV3:%[0-9]+]]:vreg_64(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<8 x s32>)
+  ; CHECK:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term $exec
+  ; CHECK: bb.2:
+  ; CHECK:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   [[PHI:%[0-9]+]]:sreg_64_xexec = PHI [[DEF2]], %bb.1, %19, %bb.2
+  ; CHECK:   [[PHI1:%[0-9]+]]:vgpr(<4 x s32>) = G_PHI [[DEF1]](<4 x s32>), %bb.1, %12(<4 x s32>), %bb.2
+  ; CHECK:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV]].sub0(s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV]].sub1(s64), implicit $exec
+  ; CHECK:   [[MV:%[0-9]+]]:sreg_64_xexec(s64) = G_MERGE_VALUES [[V_READFIRSTLANE_B32_]](s32), [[V_READFIRSTLANE_B32_1]](s32)
+  ; CHECK:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[MV]](s64), [[UV]](s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV1]].sub0(s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV1]].sub1(s64), implicit $exec
+  ; CHECK:   [[MV1:%[0-9]+]]:sreg_64_xexec(s64) = G_MERGE_VALUES [[V_READFIRSTLANE_B32_2]](s32), [[V_READFIRSTLANE_B32_3]](s32)
+  ; CHECK:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[MV1]](s64), [[UV1]](s64), implicit $exec
+  ; CHECK:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_1]], [[V_CMP_EQ_U64_e64_]], implicit-def $scc
+  ; CHECK:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV2]].sub0(s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_5:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV2]].sub1(s64), implicit $exec
+  ; CHECK:   [[MV2:%[0-9]+]]:sreg_64_xexec(s64) = G_MERGE_VALUES [[V_READFIRSTLANE_B32_4]](s32), [[V_READFIRSTLANE_B32_5]](s32)
+  ; CHECK:   [[V_CMP_EQ_U64_e64_2:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[MV2]](s64), [[UV2]](s64), implicit $exec
+  ; CHECK:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_2]], [[S_AND_B64_]], implicit-def $scc
+  ; CHECK:   [[V_READFIRSTLANE_B32_6:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV3]].sub0(s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_7:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV3]].sub1(s64), implicit $exec
+  ; CHECK:   [[MV3:%[0-9]+]]:sreg_64_xexec(s64) = G_MERGE_VALUES [[V_READFIRSTLANE_B32_6]](s32), [[V_READFIRSTLANE_B32_7]](s32)
+  ; CHECK:   [[V_CMP_EQ_U64_e64_3:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[MV3]](s64), [[UV3]](s64), implicit $exec
+  ; CHECK:   [[S_AND_B64_2:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_3]], [[S_AND_B64_1]], implicit-def $scc
+  ; CHECK:   [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<8 x s32>) = G_BUILD_VECTOR [[V_READFIRSTLANE_B32_]](s32), [[V_READFIRSTLANE_B32_1]](s32), [[V_READFIRSTLANE_B32_2]](s32), [[V_READFIRSTLANE_B32_3]](s32), [[V_READFIRSTLANE_B32_4]](s32), [[V_READFIRSTLANE_B32_5]](s32), [[V_READFIRSTLANE_B32_6]](s32), [[V_READFIRSTLANE_B32_7]](s32)
+  ; CHECK:   [[INT:%[0-9]+]]:vgpr(<4 x s32>) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.image.load.1d), 15, [[COPY8]](s32), [[BUILD_VECTOR1]](<8 x s32>), 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
+  ; CHECK:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_2]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; CHECK:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; CHECK:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; CHECK: bb.3:
+  ; CHECK:   successors: %bb.4(0x80000000)
+  ; CHECK:   $exec = S_MOV_B64_term [[S_MOV_B64_term]]
+  ; CHECK: bb.4:
+  ; CHECK:   [[COPY9:%[0-9]+]]:vgpr(p1) = COPY [[DEF]](p1)
+  ; CHECK:   G_STORE [[INT]](<4 x s32>), [[COPY9]](p1) :: (store 16 into `<4 x float> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   S_ENDPGM 0
+  %v = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i32(i32 15, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  store <4 x float> %v, <4 x float> addrspace(1)* undef
+  ret void
+}
+
+; Waterfall loop needed for rsrc, copy needed for vaddr
+define amdgpu_ps void @load_1d_sgpr_vaddr__vgpr_srsrc(<8 x i32> %rsrc, i32 inreg %s) {
+  ; CHECK-LABEL: name: load_1d_sgpr_vaddr__vgpr_srsrc
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr(s32) = COPY $vgpr5
+  ; CHECK:   [[COPY6:%[0-9]+]]:vgpr(s32) = COPY $vgpr6
+  ; CHECK:   [[COPY7:%[0-9]+]]:vgpr(s32) = COPY $vgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:vgpr(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; CHECK:   [[DEF:%[0-9]+]]:sgpr(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[COPY9:%[0-9]+]]:vgpr(s32) = COPY [[COPY8]](s32)
+  ; CHECK:   [[DEF1:%[0-9]+]]:vgpr(<4 x s32>) = G_IMPLICIT_DEF
+  ; CHECK:   [[DEF2:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
+  ; CHECK:   [[UV:%[0-9]+]]:vreg_64(s64), [[UV1:%[0-9]+]]:vreg_64(s64), [[UV2:%[0-9]+]]:vreg_64(s64), [[UV3:%[0-9]+]]:vreg_64(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<8 x s32>)
+  ; CHECK:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term $exec
+  ; CHECK: bb.2:
+  ; CHECK:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   [[PHI:%[0-9]+]]:sreg_64_xexec = PHI [[DEF2]], %bb.1, %20, %bb.2
+  ; CHECK:   [[PHI1:%[0-9]+]]:vgpr(<4 x s32>) = G_PHI [[DEF1]](<4 x s32>), %bb.1, %12(<4 x s32>), %bb.2
+  ; CHECK:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV]].sub0(s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV]].sub1(s64), implicit $exec
+  ; CHECK:   [[MV:%[0-9]+]]:sreg_64_xexec(s64) = G_MERGE_VALUES [[V_READFIRSTLANE_B32_]](s32), [[V_READFIRSTLANE_B32_1]](s32)
+  ; CHECK:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[MV]](s64), [[UV]](s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV1]].sub0(s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV1]].sub1(s64), implicit $exec
+  ; CHECK:   [[MV1:%[0-9]+]]:sreg_64_xexec(s64) = G_MERGE_VALUES [[V_READFIRSTLANE_B32_2]](s32), [[V_READFIRSTLANE_B32_3]](s32)
+  ; CHECK:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[MV1]](s64), [[UV1]](s64), implicit $exec
+  ; CHECK:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_1]], [[V_CMP_EQ_U64_e64_]], implicit-def $scc
+  ; CHECK:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV2]].sub0(s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_5:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV2]].sub1(s64), implicit $exec
+  ; CHECK:   [[MV2:%[0-9]+]]:sreg_64_xexec(s64) = G_MERGE_VALUES [[V_READFIRSTLANE_B32_4]](s32), [[V_READFIRSTLANE_B32_5]](s32)
+  ; CHECK:   [[V_CMP_EQ_U64_e64_2:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[MV2]](s64), [[UV2]](s64), implicit $exec
+  ; CHECK:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_2]], [[S_AND_B64_]], implicit-def $scc
+  ; CHECK:   [[V_READFIRSTLANE_B32_6:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV3]].sub0(s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_7:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV3]].sub1(s64), implicit $exec
+  ; CHECK:   [[MV3:%[0-9]+]]:sreg_64_xexec(s64) = G_MERGE_VALUES [[V_READFIRSTLANE_B32_6]](s32), [[V_READFIRSTLANE_B32_7]](s32)
+  ; CHECK:   [[V_CMP_EQ_U64_e64_3:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[MV3]](s64), [[UV3]](s64), implicit $exec
+  ; CHECK:   [[S_AND_B64_2:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_3]], [[S_AND_B64_1]], implicit-def $scc
+  ; CHECK:   [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<8 x s32>) = G_BUILD_VECTOR [[V_READFIRSTLANE_B32_]](s32), [[V_READFIRSTLANE_B32_1]](s32), [[V_READFIRSTLANE_B32_2]](s32), [[V_READFIRSTLANE_B32_3]](s32), [[V_READFIRSTLANE_B32_4]](s32), [[V_READFIRSTLANE_B32_5]](s32), [[V_READFIRSTLANE_B32_6]](s32), [[V_READFIRSTLANE_B32_7]](s32)
+  ; CHECK:   [[INT:%[0-9]+]]:vgpr(<4 x s32>) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.image.load.1d), 15, [[COPY9]](s32), [[BUILD_VECTOR1]](<8 x s32>), 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
+  ; CHECK:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_2]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; CHECK:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; CHECK:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; CHECK: bb.3:
+  ; CHECK:   successors: %bb.4(0x80000000)
+  ; CHECK:   $exec = S_MOV_B64_term [[S_MOV_B64_term]]
+  ; CHECK: bb.4:
+  ; CHECK:   [[COPY10:%[0-9]+]]:vgpr(p1) = COPY [[DEF]](p1)
+  ; CHECK:   G_STORE [[INT]](<4 x s32>), [[COPY10]](p1) :: (store 16 into `<4 x float> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   S_ENDPGM 0
+  %v = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i32(i32 15, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  store <4 x float> %v, <4 x float> addrspace(1)* undef
+  ret void
+}
+
+declare <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i32(i32 immarg, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+
+attributes #0 = { nounwind readonly }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.image.sample.1d.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.image.sample.1d.ll
new file mode 100644
index 000000000..e5d67a3f8
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.image.sample.1d.ll
@@ -0,0 +1,268 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -stop-after=regbankselect -regbankselect-fast -o - %s | FileCheck %s
+; XUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -stop-after=regbankselect -regbankselect-greedy -o - %s | FileCheck %s
+
+; Natural mapping
+define amdgpu_ps void @sample_1d_vgpr_vaddr__sgpr_rsrc__sgpr_samp(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s) {
+  ; CHECK-LABEL: name: sample_1d_vgpr_vaddr__sgpr_rsrc__sgpr_samp
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr(s32) = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr(s32) = COPY $sgpr6
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr(s32) = COPY $sgpr7
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr(s32) = COPY $sgpr8
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr(s32) = COPY $sgpr9
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr(s32) = COPY $sgpr10
+  ; CHECK:   [[COPY9:%[0-9]+]]:sgpr(s32) = COPY $sgpr11
+  ; CHECK:   [[COPY10:%[0-9]+]]:sgpr(s32) = COPY $sgpr12
+  ; CHECK:   [[COPY11:%[0-9]+]]:sgpr(s32) = COPY $sgpr13
+  ; CHECK:   [[COPY12:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:sgpr(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; CHECK:   [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
+  ; CHECK:   [[DEF:%[0-9]+]]:sgpr(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[INT:%[0-9]+]]:vgpr(<4 x s32>) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.image.sample.1d), 15, [[COPY12]](s32), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
+  ; CHECK:   [[COPY13:%[0-9]+]]:vgpr(p1) = COPY [[DEF]](p1)
+  ; CHECK:   G_STORE [[INT]](<4 x s32>), [[COPY13]](p1) :: (store 16 into `<4 x float> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   S_ENDPGM 0
+  %v = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
+  store <4 x float> %v, <4 x float> addrspace(1)* undef
+  ret void
+}
+
+; Copy required for VGPR input
+define amdgpu_ps void @sample_1d_sgpr_vaddr__sgpr_rsrc__sgpr_samp(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float inreg %s) {
+  ; CHECK-LABEL: name: sample_1d_sgpr_vaddr__sgpr_rsrc__sgpr_samp
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $sgpr14
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr(s32) = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr(s32) = COPY $sgpr6
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr(s32) = COPY $sgpr7
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr(s32) = COPY $sgpr8
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr(s32) = COPY $sgpr9
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr(s32) = COPY $sgpr10
+  ; CHECK:   [[COPY9:%[0-9]+]]:sgpr(s32) = COPY $sgpr11
+  ; CHECK:   [[COPY10:%[0-9]+]]:sgpr(s32) = COPY $sgpr12
+  ; CHECK:   [[COPY11:%[0-9]+]]:sgpr(s32) = COPY $sgpr13
+  ; CHECK:   [[COPY12:%[0-9]+]]:sgpr(s32) = COPY $sgpr14
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:sgpr(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; CHECK:   [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
+  ; CHECK:   [[DEF:%[0-9]+]]:sgpr(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[COPY13:%[0-9]+]]:vgpr(s32) = COPY [[COPY12]](s32)
+  ; CHECK:   [[INT:%[0-9]+]]:vgpr(<4 x s32>) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.image.sample.1d), 15, [[COPY13]](s32), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
+  ; CHECK:   [[COPY14:%[0-9]+]]:vgpr(p1) = COPY [[DEF]](p1)
+  ; CHECK:   G_STORE [[INT]](<4 x s32>), [[COPY14]](p1) :: (store 16 into `<4 x float> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   S_ENDPGM 0
+  %v = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
+  store <4 x float> %v, <4 x float> addrspace(1)* undef
+  ret void
+}
+
+; Waterfall loop for rsrc
+define amdgpu_ps void @sample_1d_vgpr_vaddr__vgpr_rsrc__sgpr_samp(<8 x i32> %rsrc, <4 x i32> inreg %samp, float %s) {
+  ; CHECK-LABEL: name: sample_1d_vgpr_vaddr__vgpr_rsrc__sgpr_samp
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr(s32) = COPY $vgpr5
+  ; CHECK:   [[COPY6:%[0-9]+]]:vgpr(s32) = COPY $vgpr6
+  ; CHECK:   [[COPY7:%[0-9]+]]:vgpr(s32) = COPY $vgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+  ; CHECK:   [[COPY9:%[0-9]+]]:sgpr(s32) = COPY $sgpr3
+  ; CHECK:   [[COPY10:%[0-9]+]]:sgpr(s32) = COPY $sgpr4
+  ; CHECK:   [[COPY11:%[0-9]+]]:sgpr(s32) = COPY $sgpr5
+  ; CHECK:   [[COPY12:%[0-9]+]]:vgpr(s32) = COPY $vgpr8
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:vgpr(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; CHECK:   [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
+  ; CHECK:   [[DEF:%[0-9]+]]:sgpr(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[DEF1:%[0-9]+]]:vgpr(<4 x s32>) = G_IMPLICIT_DEF
+  ; CHECK:   [[DEF2:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
+  ; CHECK:   [[UV:%[0-9]+]]:vreg_64(s64), [[UV1:%[0-9]+]]:vreg_64(s64), [[UV2:%[0-9]+]]:vreg_64(s64), [[UV3:%[0-9]+]]:vreg_64(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<8 x s32>)
+  ; CHECK:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term $exec
+  ; CHECK: bb.2:
+  ; CHECK:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   [[PHI:%[0-9]+]]:sreg_64_xexec = PHI [[DEF2]], %bb.1, %24, %bb.2
+  ; CHECK:   [[PHI1:%[0-9]+]]:vgpr(<4 x s32>) = G_PHI [[DEF1]](<4 x s32>), %bb.1, %17(<4 x s32>), %bb.2
+  ; CHECK:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV]].sub0(s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV]].sub1(s64), implicit $exec
+  ; CHECK:   [[MV:%[0-9]+]]:sreg_64_xexec(s64) = G_MERGE_VALUES [[V_READFIRSTLANE_B32_]](s32), [[V_READFIRSTLANE_B32_1]](s32)
+  ; CHECK:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[MV]](s64), [[UV]](s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV1]].sub0(s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV1]].sub1(s64), implicit $exec
+  ; CHECK:   [[MV1:%[0-9]+]]:sreg_64_xexec(s64) = G_MERGE_VALUES [[V_READFIRSTLANE_B32_2]](s32), [[V_READFIRSTLANE_B32_3]](s32)
+  ; CHECK:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[MV1]](s64), [[UV1]](s64), implicit $exec
+  ; CHECK:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_1]], [[V_CMP_EQ_U64_e64_]], implicit-def $scc
+  ; CHECK:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV2]].sub0(s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_5:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV2]].sub1(s64), implicit $exec
+  ; CHECK:   [[MV2:%[0-9]+]]:sreg_64_xexec(s64) = G_MERGE_VALUES [[V_READFIRSTLANE_B32_4]](s32), [[V_READFIRSTLANE_B32_5]](s32)
+  ; CHECK:   [[V_CMP_EQ_U64_e64_2:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[MV2]](s64), [[UV2]](s64), implicit $exec
+  ; CHECK:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_2]], [[S_AND_B64_]], implicit-def $scc
+  ; CHECK:   [[V_READFIRSTLANE_B32_6:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV3]].sub0(s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_7:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV3]].sub1(s64), implicit $exec
+  ; CHECK:   [[MV3:%[0-9]+]]:sreg_64_xexec(s64) = G_MERGE_VALUES [[V_READFIRSTLANE_B32_6]](s32), [[V_READFIRSTLANE_B32_7]](s32)
+  ; CHECK:   [[V_CMP_EQ_U64_e64_3:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[MV3]](s64), [[UV3]](s64), implicit $exec
+  ; CHECK:   [[S_AND_B64_2:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_3]], [[S_AND_B64_1]], implicit-def $scc
+  ; CHECK:   [[BUILD_VECTOR2:%[0-9]+]]:sgpr(<8 x s32>) = G_BUILD_VECTOR [[V_READFIRSTLANE_B32_]](s32), [[V_READFIRSTLANE_B32_1]](s32), [[V_READFIRSTLANE_B32_2]](s32), [[V_READFIRSTLANE_B32_3]](s32), [[V_READFIRSTLANE_B32_4]](s32), [[V_READFIRSTLANE_B32_5]](s32), [[V_READFIRSTLANE_B32_6]](s32), [[V_READFIRSTLANE_B32_7]](s32)
+  ; CHECK:   [[INT:%[0-9]+]]:vgpr(<4 x s32>) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.image.sample.1d), 15, [[COPY12]](s32), [[BUILD_VECTOR2]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
+  ; CHECK:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_2]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; CHECK:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; CHECK:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; CHECK: bb.3:
+  ; CHECK:   successors: %bb.4(0x80000000)
+  ; CHECK:   $exec = S_MOV_B64_term [[S_MOV_B64_term]]
+  ; CHECK: bb.4:
+  ; CHECK:   [[COPY13:%[0-9]+]]:vgpr(p1) = COPY [[DEF]](p1)
+  ; CHECK:   G_STORE [[INT]](<4 x s32>), [[COPY13]](p1) :: (store 16 into `<4 x float> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   S_ENDPGM 0
+  %v = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
+  store <4 x float> %v, <4 x float> addrspace(1)* undef
+  ret void
+}
+
+; Waterfall loop for sampler
+define amdgpu_ps void @sample_1d_vgpr_vaddr__sgpr_rsrc__vgpr_samp(<8 x i32> inreg %rsrc, <4 x i32> %samp, float %s) {
+  ; CHECK-LABEL: name: sample_1d_vgpr_vaddr__sgpr_rsrc__vgpr_samp
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr(s32) = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr(s32) = COPY $sgpr6
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr(s32) = COPY $sgpr7
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr(s32) = COPY $sgpr8
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr(s32) = COPY $sgpr9
+  ; CHECK:   [[COPY8:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY9:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY10:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY11:%[0-9]+]]:vgpr(s32) = COPY $vgpr3
+  ; CHECK:   [[COPY12:%[0-9]+]]:vgpr(s32) = COPY $vgpr4
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:sgpr(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; CHECK:   [[BUILD_VECTOR1:%[0-9]+]]:vgpr(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
+  ; CHECK:   [[DEF:%[0-9]+]]:sgpr(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[DEF1:%[0-9]+]]:vgpr(<4 x s32>) = G_IMPLICIT_DEF
+  ; CHECK:   [[DEF2:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
+  ; CHECK:   [[UV:%[0-9]+]]:vreg_64(s64), [[UV1:%[0-9]+]]:vreg_64(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x s32>)
+  ; CHECK:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term $exec
+  ; CHECK: bb.2:
+  ; CHECK:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   [[PHI:%[0-9]+]]:sreg_64_xexec = PHI [[DEF2]], %bb.1, %24, %bb.2
+  ; CHECK:   [[PHI1:%[0-9]+]]:vgpr(<4 x s32>) = G_PHI [[DEF1]](<4 x s32>), %bb.1, %17(<4 x s32>), %bb.2
+  ; CHECK:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV]].sub0(s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV]].sub1(s64), implicit $exec
+  ; CHECK:   [[MV:%[0-9]+]]:sreg_64_xexec(s64) = G_MERGE_VALUES [[V_READFIRSTLANE_B32_]](s32), [[V_READFIRSTLANE_B32_1]](s32)
+  ; CHECK:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[MV]](s64), [[UV]](s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV1]].sub0(s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV1]].sub1(s64), implicit $exec
+  ; CHECK:   [[MV1:%[0-9]+]]:sreg_64_xexec(s64) = G_MERGE_VALUES [[V_READFIRSTLANE_B32_2]](s32), [[V_READFIRSTLANE_B32_3]](s32)
+  ; CHECK:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[MV1]](s64), [[UV1]](s64), implicit $exec
+  ; CHECK:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_1]], [[V_CMP_EQ_U64_e64_]], implicit-def $scc
+  ; CHECK:   [[BUILD_VECTOR2:%[0-9]+]]:sgpr(<4 x s32>) = G_BUILD_VECTOR [[V_READFIRSTLANE_B32_]](s32), [[V_READFIRSTLANE_B32_1]](s32), [[V_READFIRSTLANE_B32_2]](s32), [[V_READFIRSTLANE_B32_3]](s32)
+  ; CHECK:   [[INT:%[0-9]+]]:vgpr(<4 x s32>) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.image.sample.1d), 15, [[COPY12]](s32), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR2]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
+  ; CHECK:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; CHECK:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; CHECK:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; CHECK: bb.3:
+  ; CHECK:   successors: %bb.4(0x80000000)
+  ; CHECK:   $exec = S_MOV_B64_term [[S_MOV_B64_term]]
+  ; CHECK: bb.4:
+  ; CHECK:   [[COPY13:%[0-9]+]]:vgpr(p1) = COPY [[DEF]](p1)
+  ; CHECK:   G_STORE [[INT]](<4 x s32>), [[COPY13]](p1) :: (store 16 into `<4 x float> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   S_ENDPGM 0
+  %v = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
+  store <4 x float> %v, <4 x float> addrspace(1)* undef
+  ret void
+}
+
+; Waterfall loop for rsrc and sampler
+define amdgpu_ps void @sample_1d_vgpr_vaddr__vgpr_rsrc__vgpr_samp(<8 x i32> %rsrc, <4 x i32> %samp, float %s) {
+  ; CHECK-LABEL: name: sample_1d_vgpr_vaddr__vgpr_rsrc__vgpr_samp
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10, $vgpr11, $vgpr12
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr(s32) = COPY $vgpr5
+  ; CHECK:   [[COPY6:%[0-9]+]]:vgpr(s32) = COPY $vgpr6
+  ; CHECK:   [[COPY7:%[0-9]+]]:vgpr(s32) = COPY $vgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:vgpr(s32) = COPY $vgpr8
+  ; CHECK:   [[COPY9:%[0-9]+]]:vgpr(s32) = COPY $vgpr9
+  ; CHECK:   [[COPY10:%[0-9]+]]:vgpr(s32) = COPY $vgpr10
+  ; CHECK:   [[COPY11:%[0-9]+]]:vgpr(s32) = COPY $vgpr11
+  ; CHECK:   [[COPY12:%[0-9]+]]:vgpr(s32) = COPY $vgpr12
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:vgpr(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; CHECK:   [[BUILD_VECTOR1:%[0-9]+]]:vgpr(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
+  ; CHECK:   [[DEF:%[0-9]+]]:sgpr(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[DEF1:%[0-9]+]]:vgpr(<4 x s32>) = G_IMPLICIT_DEF
+  ; CHECK:   [[DEF2:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
+  ; CHECK:   [[UV:%[0-9]+]]:vreg_64(s64), [[UV1:%[0-9]+]]:vreg_64(s64), [[UV2:%[0-9]+]]:vreg_64(s64), [[UV3:%[0-9]+]]:vreg_64(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<8 x s32>)
+  ; CHECK:   [[UV4:%[0-9]+]]:vreg_64(s64), [[UV5:%[0-9]+]]:vreg_64(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x s32>)
+  ; CHECK:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term $exec
+  ; CHECK: bb.2:
+  ; CHECK:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   [[PHI:%[0-9]+]]:sreg_64_xexec = PHI [[DEF2]], %bb.1, %24, %bb.2
+  ; CHECK:   [[PHI1:%[0-9]+]]:vgpr(<4 x s32>) = G_PHI [[DEF1]](<4 x s32>), %bb.1, %17(<4 x s32>), %bb.2
+  ; CHECK:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV]].sub0(s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV]].sub1(s64), implicit $exec
+  ; CHECK:   [[MV:%[0-9]+]]:sreg_64_xexec(s64) = G_MERGE_VALUES [[V_READFIRSTLANE_B32_]](s32), [[V_READFIRSTLANE_B32_1]](s32)
+  ; CHECK:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[MV]](s64), [[UV]](s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV1]].sub0(s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV1]].sub1(s64), implicit $exec
+  ; CHECK:   [[MV1:%[0-9]+]]:sreg_64_xexec(s64) = G_MERGE_VALUES [[V_READFIRSTLANE_B32_2]](s32), [[V_READFIRSTLANE_B32_3]](s32)
+  ; CHECK:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[MV1]](s64), [[UV1]](s64), implicit $exec
+  ; CHECK:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_1]], [[V_CMP_EQ_U64_e64_]], implicit-def $scc
+  ; CHECK:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV2]].sub0(s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_5:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV2]].sub1(s64), implicit $exec
+  ; CHECK:   [[MV2:%[0-9]+]]:sreg_64_xexec(s64) = G_MERGE_VALUES [[V_READFIRSTLANE_B32_4]](s32), [[V_READFIRSTLANE_B32_5]](s32)
+  ; CHECK:   [[V_CMP_EQ_U64_e64_2:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[MV2]](s64), [[UV2]](s64), implicit $exec
+  ; CHECK:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_2]], [[S_AND_B64_]], implicit-def $scc
+  ; CHECK:   [[V_READFIRSTLANE_B32_6:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV3]].sub0(s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_7:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV3]].sub1(s64), implicit $exec
+  ; CHECK:   [[MV3:%[0-9]+]]:sreg_64_xexec(s64) = G_MERGE_VALUES [[V_READFIRSTLANE_B32_6]](s32), [[V_READFIRSTLANE_B32_7]](s32)
+  ; CHECK:   [[V_CMP_EQ_U64_e64_3:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[MV3]](s64), [[UV3]](s64), implicit $exec
+  ; CHECK:   [[S_AND_B64_2:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_3]], [[S_AND_B64_1]], implicit-def $scc
+  ; CHECK:   [[BUILD_VECTOR2:%[0-9]+]]:sgpr(<8 x s32>) = G_BUILD_VECTOR [[V_READFIRSTLANE_B32_]](s32), [[V_READFIRSTLANE_B32_1]](s32), [[V_READFIRSTLANE_B32_2]](s32), [[V_READFIRSTLANE_B32_3]](s32), [[V_READFIRSTLANE_B32_4]](s32), [[V_READFIRSTLANE_B32_5]](s32), [[V_READFIRSTLANE_B32_6]](s32), [[V_READFIRSTLANE_B32_7]](s32)
+  ; CHECK:   [[V_READFIRSTLANE_B32_8:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV4]].sub0(s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_9:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV4]].sub1(s64), implicit $exec
+  ; CHECK:   [[MV4:%[0-9]+]]:sreg_64_xexec(s64) = G_MERGE_VALUES [[V_READFIRSTLANE_B32_8]](s32), [[V_READFIRSTLANE_B32_9]](s32)
+  ; CHECK:   [[V_CMP_EQ_U64_e64_4:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[MV4]](s64), [[UV4]](s64), implicit $exec
+  ; CHECK:   [[S_AND_B64_3:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_4]], [[S_AND_B64_2]], implicit-def $scc
+  ; CHECK:   [[V_READFIRSTLANE_B32_10:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV5]].sub0(s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_11:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV5]].sub1(s64), implicit $exec
+  ; CHECK:   [[MV5:%[0-9]+]]:sreg_64_xexec(s64) = G_MERGE_VALUES [[V_READFIRSTLANE_B32_10]](s32), [[V_READFIRSTLANE_B32_11]](s32)
+  ; CHECK:   [[V_CMP_EQ_U64_e64_5:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[MV5]](s64), [[UV5]](s64), implicit $exec
+  ; CHECK:   [[S_AND_B64_4:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_5]], [[S_AND_B64_3]], implicit-def $scc
+  ; CHECK:   [[BUILD_VECTOR3:%[0-9]+]]:sgpr(<4 x s32>) = G_BUILD_VECTOR [[V_READFIRSTLANE_B32_8]](s32), [[V_READFIRSTLANE_B32_9]](s32), [[V_READFIRSTLANE_B32_10]](s32), [[V_READFIRSTLANE_B32_11]](s32)
+  ; CHECK:   [[INT:%[0-9]+]]:vgpr(<4 x s32>) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.image.sample.1d), 15, [[COPY12]](s32), [[BUILD_VECTOR2]](<8 x s32>), [[BUILD_VECTOR3]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
+  ; CHECK:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_4]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; CHECK:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; CHECK:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; CHECK: bb.3:
+  ; CHECK:   successors: %bb.4(0x80000000)
+  ; CHECK:   $exec = S_MOV_B64_term [[S_MOV_B64_term]]
+  ; CHECK: bb.4:
+  ; CHECK:   [[COPY13:%[0-9]+]]:vgpr(p1) = COPY [[DEF]](p1)
+  ; CHECK:   G_STORE [[INT]](<4 x s32>), [[COPY13]](p1) :: (store 16 into `<4 x float> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   S_ENDPGM 0
+  %v = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
+  store <4 x float> %v, <4 x float> addrspace(1)* undef
+  ret void
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 immarg, float, <8 x i32>, <4 x i32>, i1 immarg, i32 immarg, i32 immarg) #0
+
+attributes #0 = { nounwind readonly }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.raw.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.raw.buffer.load.ll
new file mode 100644
index 000000000..4443daba2
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.raw.buffer.load.ll
@@ -0,0 +1,173 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -stop-after=regbankselect -regbankselect-fast -o - %s | FileCheck %s
+; XUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -stop-after=regbankselect -regbankselect-greedy -o - %s | FileCheck %s
+
+; Natural mapping
+define amdgpu_ps float @raw_buffer_load__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset(<4 x i32> inreg %rsrc, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_load__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr(s32) = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr(s32) = COPY $sgpr6
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32)
+  ; CHECK:   [[INT:%[0-9]+]]:vgpr(s32) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.raw.buffer.load), [[BUILD_VECTOR]](<4 x s32>), [[COPY4]](s32), [[COPY5]](s32), 0 :: (dereferenceable load 4 from custom "TargetCustom7", align 1, addrspace 4)
+  ; CHECK:   $vgpr0 = COPY [[INT]](s32)
+  ; CHECK:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  %val = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret float %val
+}
+
+; Copies for VGPR arguments
+define amdgpu_ps float @raw_buffer_load__sgpr_rsrc__sgpr_val__sgpr_voffset__sgpr_soffset(<4 x i32> inreg %rsrc, i32 inreg %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_load__sgpr_rsrc__sgpr_val__sgpr_voffset__sgpr_soffset
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr(s32) = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr(s32) = COPY $sgpr6
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr(s32) = COPY $sgpr7
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32)
+  ; CHECK:   [[COPY6:%[0-9]+]]:vgpr(s32) = COPY [[COPY4]](s32)
+  ; CHECK:   [[INT:%[0-9]+]]:vgpr(s32) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.raw.buffer.load), [[BUILD_VECTOR]](<4 x s32>), [[COPY6]](s32), [[COPY5]](s32), 0 :: (dereferenceable load 4 from custom "TargetCustom7", align 1, addrspace 4)
+  ; CHECK:   $vgpr0 = COPY [[INT]](s32)
+  ; CHECK:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  %val = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret float %val
+}
+
+; Waterfall for rsrc
+define amdgpu_ps float @raw_buffer_load__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset(<4 x i32> %rsrc, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_load__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:vgpr(<4 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32)
+  ; CHECK:   [[DEF:%[0-9]+]]:vgpr(s32) = G_IMPLICIT_DEF
+  ; CHECK:   [[DEF1:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
+  ; CHECK:   [[UV:%[0-9]+]]:vreg_64(s64), [[UV1:%[0-9]+]]:vreg_64(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x s32>)
+  ; CHECK:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term $exec
+  ; CHECK: bb.2:
+  ; CHECK:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   [[PHI:%[0-9]+]]:sreg_64_xexec = PHI [[DEF1]], %bb.1, %15, %bb.2
+  ; CHECK:   [[PHI1:%[0-9]+]]:vgpr(s32) = G_PHI [[DEF]](s32), %bb.1, %9(s32), %bb.2
+  ; CHECK:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV]].sub0(s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV]].sub1(s64), implicit $exec
+  ; CHECK:   [[MV:%[0-9]+]]:sreg_64_xexec(s64) = G_MERGE_VALUES [[V_READFIRSTLANE_B32_]](s32), [[V_READFIRSTLANE_B32_1]](s32)
+  ; CHECK:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[MV]](s64), [[UV]](s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV1]].sub0(s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV1]].sub1(s64), implicit $exec
+  ; CHECK:   [[MV1:%[0-9]+]]:sreg_64_xexec(s64) = G_MERGE_VALUES [[V_READFIRSTLANE_B32_2]](s32), [[V_READFIRSTLANE_B32_3]](s32)
+  ; CHECK:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[MV1]](s64), [[UV1]](s64), implicit $exec
+  ; CHECK:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_1]], [[V_CMP_EQ_U64_e64_]], implicit-def $scc
+  ; CHECK:   [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x s32>) = G_BUILD_VECTOR [[V_READFIRSTLANE_B32_]](s32), [[V_READFIRSTLANE_B32_1]](s32), [[V_READFIRSTLANE_B32_2]](s32), [[V_READFIRSTLANE_B32_3]](s32)
+  ; CHECK:   [[INT:%[0-9]+]]:vgpr(s32) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.raw.buffer.load), [[BUILD_VECTOR1]](<4 x s32>), [[COPY4]](s32), [[COPY5]](s32), 0 :: (dereferenceable load 4 from custom "TargetCustom7", align 1, addrspace 4)
+  ; CHECK:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; CHECK:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; CHECK:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; CHECK: bb.3:
+  ; CHECK:   successors: %bb.4(0x80000000)
+  ; CHECK:   $exec = S_MOV_B64_term [[S_MOV_B64_term]]
+  ; CHECK: bb.4:
+  ; CHECK:   $vgpr0 = COPY [[INT]](s32)
+  ; CHECK:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  %val = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret float %val
+}
+
+; Waterfall for soffset
+define amdgpu_ps float @raw_buffer_load__sgpr_rsrc__vgpr_val__vgpr_voffset__vgpr_soffset(<4 x i32> inreg %rsrc, i32 %voffset, i32 %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_load__sgpr_rsrc__vgpr_val__vgpr_voffset__vgpr_soffset
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $vgpr0, $vgpr1
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr(s32) = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32)
+  ; CHECK:   [[DEF:%[0-9]+]]:vgpr(s32) = G_IMPLICIT_DEF
+  ; CHECK:   [[DEF1:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
+  ; CHECK:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term $exec
+  ; CHECK: bb.2:
+  ; CHECK:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   [[PHI:%[0-9]+]]:sreg_64_xexec = PHI [[DEF1]], %bb.1, %15, %bb.2
+  ; CHECK:   [[PHI1:%[0-9]+]]:vgpr(s32) = G_PHI [[DEF]](s32), %bb.1, %9(s32), %bb.2
+  ; CHECK:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[COPY5]](s32), implicit $exec
+  ; CHECK:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]](s32), [[COPY5]](s32), implicit $exec
+  ; CHECK:   [[INT:%[0-9]+]]:vgpr(s32) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.raw.buffer.load), [[BUILD_VECTOR]](<4 x s32>), [[COPY4]](s32), [[V_READFIRSTLANE_B32_]](s32), 0 :: (dereferenceable load 4 from custom "TargetCustom7", align 1, addrspace 4)
+  ; CHECK:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[V_CMP_EQ_U32_e64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; CHECK:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; CHECK:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; CHECK: bb.3:
+  ; CHECK:   successors: %bb.4(0x80000000)
+  ; CHECK:   $exec = S_MOV_B64_term [[S_MOV_B64_term]]
+  ; CHECK: bb.4:
+  ; CHECK:   $vgpr0 = COPY [[INT]](s32)
+  ; CHECK:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  %val = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret float %val
+}
+
+; Waterfall for rsrc and soffset
+define amdgpu_ps float @raw_buffer_load__vgpr_rsrc__vgpr_val__vgpr_voffset__vgpr_soffset(<4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_load__vgpr_rsrc__vgpr_val__vgpr_voffset__vgpr_soffset
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr5
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:vgpr(<4 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32)
+  ; CHECK:   [[DEF:%[0-9]+]]:vgpr(s32) = G_IMPLICIT_DEF
+  ; CHECK:   [[DEF1:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
+  ; CHECK:   [[UV:%[0-9]+]]:vreg_64(s64), [[UV1:%[0-9]+]]:vreg_64(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x s32>)
+  ; CHECK:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term $exec
+  ; CHECK: bb.2:
+  ; CHECK:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   [[PHI:%[0-9]+]]:sreg_64_xexec = PHI [[DEF1]], %bb.1, %15, %bb.2
+  ; CHECK:   [[PHI1:%[0-9]+]]:vgpr(s32) = G_PHI [[DEF]](s32), %bb.1, %9(s32), %bb.2
+  ; CHECK:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV]].sub0(s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV]].sub1(s64), implicit $exec
+  ; CHECK:   [[MV:%[0-9]+]]:sreg_64_xexec(s64) = G_MERGE_VALUES [[V_READFIRSTLANE_B32_]](s32), [[V_READFIRSTLANE_B32_1]](s32)
+  ; CHECK:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[MV]](s64), [[UV]](s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV1]].sub0(s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV1]].sub1(s64), implicit $exec
+  ; CHECK:   [[MV1:%[0-9]+]]:sreg_64_xexec(s64) = G_MERGE_VALUES [[V_READFIRSTLANE_B32_2]](s32), [[V_READFIRSTLANE_B32_3]](s32)
+  ; CHECK:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[MV1]](s64), [[UV1]](s64), implicit $exec
+  ; CHECK:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_1]], [[V_CMP_EQ_U64_e64_]], implicit-def $scc
+  ; CHECK:   [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x s32>) = G_BUILD_VECTOR [[V_READFIRSTLANE_B32_]](s32), [[V_READFIRSTLANE_B32_1]](s32), [[V_READFIRSTLANE_B32_2]](s32), [[V_READFIRSTLANE_B32_3]](s32)
+  ; CHECK:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[COPY5]](s32), implicit $exec
+  ; CHECK:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]](s32), [[COPY5]](s32), implicit $exec
+  ; CHECK:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U32_e64_]], [[S_AND_B64_]], implicit-def $scc
+  ; CHECK:   [[INT:%[0-9]+]]:vgpr(s32) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.raw.buffer.load), [[BUILD_VECTOR1]](<4 x s32>), [[COPY4]](s32), [[V_READFIRSTLANE_B32_4]](s32), 0 :: (dereferenceable load 4 from custom "TargetCustom7", align 1, addrspace 4)
+  ; CHECK:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; CHECK:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; CHECK:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; CHECK: bb.3:
+  ; CHECK:   successors: %bb.4(0x80000000)
+  ; CHECK:   $exec = S_MOV_B64_term [[S_MOV_B64_term]]
+  ; CHECK: bb.4:
+  ; CHECK:   $vgpr0 = COPY [[INT]](s32)
+  ; CHECK:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  %val = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret float %val
+}
+
+declare float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32>, i32, i32, i32 immarg)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.struct.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.struct.buffer.load.ll
new file mode 100644
index 000000000..a65748827
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.struct.buffer.load.ll
@@ -0,0 +1,179 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -stop-after=regbankselect -regbankselect-fast -o - %s | FileCheck %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -stop-after=regbankselect -regbankselect-greedy -o - %s | FileCheck %s
+
+; Natural mapping
+define amdgpu_ps float @struct_buffer_load__sgpr_rsrc__vgpr_val__vgpr_vindex__vgpr_voffset__sgpr_soffset(<4 x i32> inreg %rsrc, i32 %vindex, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: struct_buffer_load__sgpr_rsrc__vgpr_val__vgpr_vindex__vgpr_voffset__sgpr_soffset
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr(s32) = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr(s32) = COPY $sgpr6
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32)
+  ; CHECK:   [[INT:%[0-9]+]]:vgpr(s32) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.struct.buffer.load), [[BUILD_VECTOR]](<4 x s32>), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), 0 :: (dereferenceable load 4 from custom "TargetCustom7", align 1, addrspace 4)
+  ; CHECK:   $vgpr0 = COPY [[INT]](s32)
+  ; CHECK:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  %val = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
+  ret float %val
+}
+
+; Copies for VGPR arguments
+define amdgpu_ps float @struct_buffer_load__sgpr_rsrc__sgpr_val__sgpr_vindex__sgpr_voffset__sgpr_soffset(<4 x i32> inreg %rsrc, i32 inreg %vindex, i32 inreg %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: struct_buffer_load__sgpr_rsrc__sgpr_val__sgpr_vindex__sgpr_voffset__sgpr_soffset
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr(s32) = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr(s32) = COPY $sgpr6
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr(s32) = COPY $sgpr7
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr(s32) = COPY $sgpr8
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32)
+  ; CHECK:   [[COPY7:%[0-9]+]]:vgpr(s32) = COPY [[COPY4]](s32)
+  ; CHECK:   [[COPY8:%[0-9]+]]:vgpr(s32) = COPY [[COPY5]](s32)
+  ; CHECK:   [[INT:%[0-9]+]]:vgpr(s32) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.struct.buffer.load), [[BUILD_VECTOR]](<4 x s32>), [[COPY7]](s32), [[COPY8]](s32), [[COPY6]](s32), 0 :: (dereferenceable load 4 from custom "TargetCustom7", align 1, addrspace 4)
+  ; CHECK:   $vgpr0 = COPY [[INT]](s32)
+  ; CHECK:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  %val = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
+  ret float %val
+}
+
+; Waterfall for rsrc
+define amdgpu_ps float @struct_buffer_load__vgpr_rsrc__vgpr_val__vgpr_vindex__vgpr_voffset__sgpr_soffset(<4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: struct_buffer_load__vgpr_rsrc__vgpr_val__vgpr_vindex__vgpr_voffset__sgpr_soffset
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr(s32) = COPY $vgpr5
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:vgpr(<4 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32)
+  ; CHECK:   [[DEF:%[0-9]+]]:vgpr(s32) = G_IMPLICIT_DEF
+  ; CHECK:   [[DEF1:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
+  ; CHECK:   [[UV:%[0-9]+]]:vreg_64(s64), [[UV1:%[0-9]+]]:vreg_64(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x s32>)
+  ; CHECK:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term $exec
+  ; CHECK: bb.2:
+  ; CHECK:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   [[PHI:%[0-9]+]]:sreg_64_xexec = PHI [[DEF1]], %bb.1, %16, %bb.2
+  ; CHECK:   [[PHI1:%[0-9]+]]:vgpr(s32) = G_PHI [[DEF]](s32), %bb.1, %10(s32), %bb.2
+  ; CHECK:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV]].sub0(s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV]].sub1(s64), implicit $exec
+  ; CHECK:   [[MV:%[0-9]+]]:sreg_64_xexec(s64) = G_MERGE_VALUES [[V_READFIRSTLANE_B32_]](s32), [[V_READFIRSTLANE_B32_1]](s32)
+  ; CHECK:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[MV]](s64), [[UV]](s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV1]].sub0(s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV1]].sub1(s64), implicit $exec
+  ; CHECK:   [[MV1:%[0-9]+]]:sreg_64_xexec(s64) = G_MERGE_VALUES [[V_READFIRSTLANE_B32_2]](s32), [[V_READFIRSTLANE_B32_3]](s32)
+  ; CHECK:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[MV1]](s64), [[UV1]](s64), implicit $exec
+  ; CHECK:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_1]], [[V_CMP_EQ_U64_e64_]], implicit-def $scc
+  ; CHECK:   [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x s32>) = G_BUILD_VECTOR [[V_READFIRSTLANE_B32_]](s32), [[V_READFIRSTLANE_B32_1]](s32), [[V_READFIRSTLANE_B32_2]](s32), [[V_READFIRSTLANE_B32_3]](s32)
+  ; CHECK:   [[INT:%[0-9]+]]:vgpr(s32) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.struct.buffer.load), [[BUILD_VECTOR1]](<4 x s32>), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), 0 :: (dereferenceable load 4 from custom "TargetCustom7", align 1, addrspace 4)
+  ; CHECK:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; CHECK:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; CHECK:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; CHECK: bb.3:
+  ; CHECK:   successors: %bb.4(0x80000000)
+  ; CHECK:   $exec = S_MOV_B64_term [[S_MOV_B64_term]]
+  ; CHECK: bb.4:
+  ; CHECK:   $vgpr0 = COPY [[INT]](s32)
+  ; CHECK:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  %val = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
+  ret float %val
+}
+
+; Waterfall for soffset
+define amdgpu_ps float @struct_buffer_load__sgpr_rsrc__vgpr_val__vgpr_vindex_vgpr_voffset__vgpr_soffset(<4 x i32> inreg %rsrc, i32 %vindex, i32 %voffset, i32 %soffset) {
+  ; CHECK-LABEL: name: struct_buffer_load__sgpr_rsrc__vgpr_val__vgpr_vindex_vgpr_voffset__vgpr_soffset
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $vgpr0, $vgpr1, $vgpr2
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr(s32) = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32)
+  ; CHECK:   [[DEF:%[0-9]+]]:vgpr(s32) = G_IMPLICIT_DEF
+  ; CHECK:   [[DEF1:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
+  ; CHECK:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term $exec
+  ; CHECK: bb.2:
+  ; CHECK:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   [[PHI:%[0-9]+]]:sreg_64_xexec = PHI [[DEF1]], %bb.1, %16, %bb.2
+  ; CHECK:   [[PHI1:%[0-9]+]]:vgpr(s32) = G_PHI [[DEF]](s32), %bb.1, %10(s32), %bb.2
+  ; CHECK:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[COPY6]](s32), implicit $exec
+  ; CHECK:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]](s32), [[COPY6]](s32), implicit $exec
+  ; CHECK:   [[INT:%[0-9]+]]:vgpr(s32) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.struct.buffer.load), [[BUILD_VECTOR]](<4 x s32>), [[COPY4]](s32), [[COPY5]](s32), [[V_READFIRSTLANE_B32_]](s32), 0 :: (dereferenceable load 4 from custom "TargetCustom7", align 1, addrspace 4)
+  ; CHECK:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[V_CMP_EQ_U32_e64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; CHECK:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; CHECK:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; CHECK: bb.3:
+  ; CHECK:   successors: %bb.4(0x80000000)
+  ; CHECK:   $exec = S_MOV_B64_term [[S_MOV_B64_term]]
+  ; CHECK: bb.4:
+  ; CHECK:   $vgpr0 = COPY [[INT]](s32)
+  ; CHECK:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  %val = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
+  ret float %val
+}
+
+; Waterfall for rsrc and soffset
+define amdgpu_ps float @struct_buffer_load__vgpr_rsrc__vgpr_val__vgpr_vindex__vgpr_voffset__vgpr_soffset(<4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset) {
+  ; CHECK-LABEL: name: struct_buffer_load__vgpr_rsrc__vgpr_val__vgpr_vindex__vgpr_voffset__vgpr_soffset
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr(s32) = COPY $vgpr5
+  ; CHECK:   [[COPY6:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr6
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:vgpr(<4 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32)
+  ; CHECK:   [[DEF:%[0-9]+]]:vgpr(s32) = G_IMPLICIT_DEF
+  ; CHECK:   [[DEF1:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
+  ; CHECK:   [[UV:%[0-9]+]]:vreg_64(s64), [[UV1:%[0-9]+]]:vreg_64(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x s32>)
+  ; CHECK:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term $exec
+  ; CHECK: bb.2:
+  ; CHECK:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   [[PHI:%[0-9]+]]:sreg_64_xexec = PHI [[DEF1]], %bb.1, %16, %bb.2
+  ; CHECK:   [[PHI1:%[0-9]+]]:vgpr(s32) = G_PHI [[DEF]](s32), %bb.1, %10(s32), %bb.2
+  ; CHECK:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV]].sub0(s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV]].sub1(s64), implicit $exec
+  ; CHECK:   [[MV:%[0-9]+]]:sreg_64_xexec(s64) = G_MERGE_VALUES [[V_READFIRSTLANE_B32_]](s32), [[V_READFIRSTLANE_B32_1]](s32)
+  ; CHECK:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[MV]](s64), [[UV]](s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV1]].sub0(s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV1]].sub1(s64), implicit $exec
+  ; CHECK:   [[MV1:%[0-9]+]]:sreg_64_xexec(s64) = G_MERGE_VALUES [[V_READFIRSTLANE_B32_2]](s32), [[V_READFIRSTLANE_B32_3]](s32)
+  ; CHECK:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[MV1]](s64), [[UV1]](s64), implicit $exec
+  ; CHECK:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_1]], [[V_CMP_EQ_U64_e64_]], implicit-def $scc
+  ; CHECK:   [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x s32>) = G_BUILD_VECTOR [[V_READFIRSTLANE_B32_]](s32), [[V_READFIRSTLANE_B32_1]](s32), [[V_READFIRSTLANE_B32_2]](s32), [[V_READFIRSTLANE_B32_3]](s32)
+  ; CHECK:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[COPY6]](s32), implicit $exec
+  ; CHECK:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]](s32), [[COPY6]](s32), implicit $exec
+  ; CHECK:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U32_e64_]], [[S_AND_B64_]], implicit-def $scc
+  ; CHECK:   [[INT:%[0-9]+]]:vgpr(s32) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.struct.buffer.load), [[BUILD_VECTOR1]](<4 x s32>), [[COPY4]](s32), [[COPY5]](s32), [[V_READFIRSTLANE_B32_4]](s32), 0 :: (dereferenceable load 4 from custom "TargetCustom7", align 1, addrspace 4)
+  ; CHECK:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; CHECK:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; CHECK:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; CHECK: bb.3:
+  ; CHECK:   successors: %bb.4(0x80000000)
+  ; CHECK:   $exec = S_MOV_B64_term [[S_MOV_B64_term]]
+  ; CHECK: bb.4:
+  ; CHECK:   $vgpr0 = COPY [[INT]](s32)
+  ; CHECK:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  %val = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
+  ret float %val
+}
+
+declare float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32>, i32, i32, i32, i32 immarg)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.struct.buffer.store.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.struct.buffer.store.ll
new file mode 100644
index 000000000..a6ba55938
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.struct.buffer.store.ll
@@ -0,0 +1,174 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -stop-after=regbankselect -regbankselect-fast -o - %s | FileCheck %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -stop-after=regbankselect -regbankselect-greedy -o - %s | FileCheck %s
+
+; Natural mapping
+define amdgpu_ps void @struct_buffer_store__sgpr_rsrc__vgpr_val__vgpr_vindex__vgpr_voffset__sgpr_soffset(<4 x i32> inreg %rsrc, float %val, i32 %vindex, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: struct_buffer_store__sgpr_rsrc__vgpr_val__vgpr_vindex__vgpr_voffset__sgpr_soffset
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr(s32) = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr(s32) = COPY $sgpr6
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32)
+  ; CHECK:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.struct.buffer.store), [[COPY4]](s32), [[BUILD_VECTOR]](<4 x s32>), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32), 0 :: (dereferenceable store 4 into custom "TargetCustom7", align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.struct.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+; Copies for VGPR arguments
+define amdgpu_ps void @struct_buffer_store__sgpr_rsrc__sgpr_val__sgpr_vindex__sgpr_voffset__sgpr_soffset(<4 x i32> inreg %rsrc, float inreg %val, i32 inreg %vindex, i32 inreg %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: struct_buffer_store__sgpr_rsrc__sgpr_val__sgpr_vindex__sgpr_voffset__sgpr_soffset
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr(s32) = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr(s32) = COPY $sgpr6
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr(s32) = COPY $sgpr7
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr(s32) = COPY $sgpr8
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr(s32) = COPY $sgpr9
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32)
+  ; CHECK:   [[COPY8:%[0-9]+]]:vgpr(s32) = COPY [[COPY4]](s32)
+  ; CHECK:   [[COPY9:%[0-9]+]]:vgpr(s32) = COPY [[COPY5]](s32)
+  ; CHECK:   [[COPY10:%[0-9]+]]:vgpr(s32) = COPY [[COPY6]](s32)
+  ; CHECK:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.struct.buffer.store), [[COPY8]](s32), [[BUILD_VECTOR]](<4 x s32>), [[COPY9]](s32), [[COPY10]](s32), [[COPY7]](s32), 0 :: (dereferenceable store 4 into custom "TargetCustom7", align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.struct.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+; Waterfall for rsrc
+define amdgpu_ps void @struct_buffer_store__vgpr_rsrc__vgpr_val__vgpr_vindex__vgpr_voffset__sgpr_soffset(<4 x i32> %rsrc, float %val, i32 %vindex, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: struct_buffer_store__vgpr_rsrc__vgpr_val__vgpr_vindex__vgpr_voffset__sgpr_soffset
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr(s32) = COPY $vgpr5
+  ; CHECK:   [[COPY6:%[0-9]+]]:vgpr(s32) = COPY $vgpr6
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:vgpr(<4 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32)
+  ; CHECK:   [[DEF:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
+  ; CHECK:   [[UV:%[0-9]+]]:vreg_64(s64), [[UV1:%[0-9]+]]:vreg_64(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x s32>)
+  ; CHECK:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term $exec
+  ; CHECK: bb.2:
+  ; CHECK:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   [[PHI:%[0-9]+]]:sreg_64_xexec = PHI [[DEF]], %bb.1, %14, %bb.2
+  ; CHECK:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV]].sub0(s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV]].sub1(s64), implicit $exec
+  ; CHECK:   [[MV:%[0-9]+]]:sreg_64_xexec(s64) = G_MERGE_VALUES [[V_READFIRSTLANE_B32_]](s32), [[V_READFIRSTLANE_B32_1]](s32)
+  ; CHECK:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[MV]](s64), [[UV]](s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV1]].sub0(s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV1]].sub1(s64), implicit $exec
+  ; CHECK:   [[MV1:%[0-9]+]]:sreg_64_xexec(s64) = G_MERGE_VALUES [[V_READFIRSTLANE_B32_2]](s32), [[V_READFIRSTLANE_B32_3]](s32)
+  ; CHECK:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[MV1]](s64), [[UV1]](s64), implicit $exec
+  ; CHECK:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_1]], [[V_CMP_EQ_U64_e64_]], implicit-def $scc
+  ; CHECK:   [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x s32>) = G_BUILD_VECTOR [[V_READFIRSTLANE_B32_]](s32), [[V_READFIRSTLANE_B32_1]](s32), [[V_READFIRSTLANE_B32_2]](s32), [[V_READFIRSTLANE_B32_3]](s32)
+  ; CHECK:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.struct.buffer.store), [[COPY4]](s32), [[BUILD_VECTOR1]](<4 x s32>), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32), 0 :: (dereferenceable store 4 into custom "TargetCustom7", align 1, addrspace 4)
+  ; CHECK:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; CHECK:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; CHECK:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; CHECK: bb.3:
+  ; CHECK:   successors: %bb.4(0x80000000)
+  ; CHECK:   $exec = S_MOV_B64_term [[S_MOV_B64_term]]
+  ; CHECK: bb.4:
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.struct.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+; Waterfall for soffset
+define amdgpu_ps void @struct_buffer_store__sgpr_rsrc__vgpr_val__vgpr_vindex__vgpr_voffset__vgpr_soffset(<4 x i32> inreg %rsrc, float %val, i32 %vindex, i32 %voffset, i32 %soffset) {
+  ; CHECK-LABEL: name: struct_buffer_store__sgpr_rsrc__vgpr_val__vgpr_vindex__vgpr_voffset__vgpr_soffset
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr(s32) = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY7:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr3
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32)
+  ; CHECK:   [[DEF:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
+  ; CHECK:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term $exec
+  ; CHECK: bb.2:
+  ; CHECK:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   [[PHI:%[0-9]+]]:sreg_64_xexec = PHI [[DEF]], %bb.1, %14, %bb.2
+  ; CHECK:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[COPY7]](s32), implicit $exec
+  ; CHECK:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]](s32), [[COPY7]](s32), implicit $exec
+  ; CHECK:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.struct.buffer.store), [[COPY4]](s32), [[BUILD_VECTOR]](<4 x s32>), [[COPY5]](s32), [[COPY6]](s32), [[V_READFIRSTLANE_B32_]](s32), 0 :: (dereferenceable store 4 into custom "TargetCustom7", align 1, addrspace 4)
+  ; CHECK:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[V_CMP_EQ_U32_e64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; CHECK:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; CHECK:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; CHECK: bb.3:
+  ; CHECK:   successors: %bb.4(0x80000000)
+  ; CHECK:   $exec = S_MOV_B64_term [[S_MOV_B64_term]]
+  ; CHECK: bb.4:
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.struct.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+; Waterfall for rsrc and soffset
+define amdgpu_ps void @struct_buffer_store__vgpr_rsrc__vgpr_val__vgpr_vindex__vgpr_voffset__vgpr_soffset(<4 x i32> %rsrc, float %val, i32 %vindex, i32 %voffset, i32 %soffset) {
+  ; CHECK-LABEL: name: struct_buffer_store__vgpr_rsrc__vgpr_val__vgpr_vindex__vgpr_voffset__vgpr_soffset
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr(s32) = COPY $vgpr5
+  ; CHECK:   [[COPY6:%[0-9]+]]:vgpr(s32) = COPY $vgpr6
+  ; CHECK:   [[COPY7:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr7
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:vgpr(<4 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32)
+  ; CHECK:   [[DEF:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
+  ; CHECK:   [[UV:%[0-9]+]]:vreg_64(s64), [[UV1:%[0-9]+]]:vreg_64(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x s32>)
+  ; CHECK:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term $exec
+  ; CHECK: bb.2:
+  ; CHECK:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   [[PHI:%[0-9]+]]:sreg_64_xexec = PHI [[DEF]], %bb.1, %14, %bb.2
+  ; CHECK:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV]].sub0(s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV]].sub1(s64), implicit $exec
+  ; CHECK:   [[MV:%[0-9]+]]:sreg_64_xexec(s64) = G_MERGE_VALUES [[V_READFIRSTLANE_B32_]](s32), [[V_READFIRSTLANE_B32_1]](s32)
+  ; CHECK:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[MV]](s64), [[UV]](s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV1]].sub0(s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV1]].sub1(s64), implicit $exec
+  ; CHECK:   [[MV1:%[0-9]+]]:sreg_64_xexec(s64) = G_MERGE_VALUES [[V_READFIRSTLANE_B32_2]](s32), [[V_READFIRSTLANE_B32_3]](s32)
+  ; CHECK:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[MV1]](s64), [[UV1]](s64), implicit $exec
+  ; CHECK:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_1]], [[V_CMP_EQ_U64_e64_]], implicit-def $scc
+  ; CHECK:   [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x s32>) = G_BUILD_VECTOR [[V_READFIRSTLANE_B32_]](s32), [[V_READFIRSTLANE_B32_1]](s32), [[V_READFIRSTLANE_B32_2]](s32), [[V_READFIRSTLANE_B32_3]](s32)
+  ; CHECK:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[COPY7]](s32), implicit $exec
+  ; CHECK:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]](s32), [[COPY7]](s32), implicit $exec
+  ; CHECK:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U32_e64_]], [[S_AND_B64_]], implicit-def $scc
+  ; CHECK:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.struct.buffer.store), [[COPY4]](s32), [[BUILD_VECTOR1]](<4 x s32>), [[COPY5]](s32), [[COPY6]](s32), [[V_READFIRSTLANE_B32_4]](s32), 0 :: (dereferenceable store 4 into custom "TargetCustom7", align 1, addrspace 4)
+  ; CHECK:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; CHECK:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; CHECK:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; CHECK: bb.3:
+  ; CHECK:   successors: %bb.4(0x80000000)
+  ; CHECK:   $exec = S_MOV_B64_term [[S_MOV_B64_term]]
+  ; CHECK: bb.4:
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.struct.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+declare void @llvm.amdgcn.struct.buffer.store.f32(float, <4 x i32>, i32, i32, i32, i32 immarg)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/ret.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/ret.ll
new file mode 100644
index 000000000..c6c1a8717
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/ret.ll
@@ -0,0 +1,3 @@
+; Runs original SDAG test with -global-isel
+; RUN: llc -global-isel -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %S/../ret.ll | FileCheck -check-prefix=GCN %S/../ret.ll
+; RUN: llc -global-isel -march=amdgcn -mcpu=tonga -verify-machineinstrs < %S/../ret.ll | FileCheck -check-prefix=GCN %S/../ret.ll
diff --git a/llvm/test/CodeGen/AMDGPU/add.i16.ll b/llvm/test/CodeGen/AMDGPU/add.i16.ll
index 8da3401b3..98848295a 100644
--- a/llvm/test/CodeGen/AMDGPU/add.i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/add.i16.ll
@@ -105,7 +105,7 @@ define amdgpu_kernel void @v_test_add_i16_zext_to_i64(i64 addrspace(1)* %out, i1
 ; GCN-LABEL: {{^}}v_test_add_i16_sext_to_i32:
 ; VI: flat_load_ushort [[A:v[0-9]+]]
 ; VI: flat_load_ushort [[B:v[0-9]+]]
-; VI: v_add_u16_e32 [[ADD:v[0-9]+]],  [[B]], [[A]]
+; VI: v_add_u16_e32 [[ADD:v[0-9]+]],  [[A]], [[B]]
 ; VI-NEXT: v_bfe_i32 [[SEXT:v[0-9]+]], [[ADD]], 0, 16
 ; VI-NEXT: buffer_store_dword [[SEXT]]
 define amdgpu_kernel void @v_test_add_i16_sext_to_i32(i32 addrspace(1)* %out, i16 addrspace(1)* %in0, i16 addrspace(1)* %in1) #1 {
@@ -125,7 +125,7 @@ define amdgpu_kernel void @v_test_add_i16_sext_to_i32(i32 addrspace(1)* %out, i1
 ; GCN-LABEL: {{^}}v_test_add_i16_sext_to_i64:
 ; VI: flat_load_ushort [[A:v[0-9]+]]
 ; VI: flat_load_ushort [[B:v[0-9]+]]
-; VI: v_add_u16_e32 [[ADD:v[0-9]+]], [[B]], [[A]]
+; VI: v_add_u16_e32 [[ADD:v[0-9]+]], [[A]], [[B]]
 ; VI-NEXT: v_bfe_i32 v[[LO:[0-9]+]], [[ADD]], 0, 16
 ; VI-NEXT: v_ashrrev_i32_e32 v[[HI:[0-9]+]], 31, v[[LO]]
 ; VI-NEXT: buffer_store_dwordx2 v{{\[}}[[LO]]:[[HI]]{{\]}}
diff --git a/llvm/test/CodeGen/AMDGPU/addrspacecast-constantexpr.ll b/llvm/test/CodeGen/AMDGPU/addrspacecast-constantexpr.ll
index b40fcb3e4..1ed53194f 100644
--- a/llvm/test/CodeGen/AMDGPU/addrspacecast-constantexpr.ll
+++ b/llvm/test/CodeGen/AMDGPU/addrspacecast-constantexpr.ll
@@ -98,7 +98,7 @@ define i32 addrspace(3)* @ret_constant_cast_group_gv_gep_to_flat_to_group() #1 {
   ret i32 addrspace(3)* addrspacecast (i32 addrspace(4)* getelementptr ([256 x i32], [256 x i32] addrspace(4)* addrspacecast ([256 x i32] addrspace(3)* @lds.arr to [256 x i32] addrspace(4)*), i64 0, i64 8) to i32 addrspace(3)*)
 }
 
-; HSA: attributes #0 = { argmemonly nounwind }
+; HSA: attributes #0 = { argmemonly nounwind willreturn }
 ; HSA: attributes #1 = { nounwind }
 ; HSA: attributes #2 = { nounwind "amdgpu-queue-ptr" }
 
diff --git a/llvm/test/CodeGen/AMDGPU/agpr-register-count.ll b/llvm/test/CodeGen/AMDGPU/agpr-register-count.ll
index dfedd2402..8ad231c4c 100644
--- a/llvm/test/CodeGen/AMDGPU/agpr-register-count.ll
+++ b/llvm/test/CodeGen/AMDGPU/agpr-register-count.ll
@@ -1,15 +1,134 @@
-; RUN: llc -march=amdgcn -mcpu=gfx908 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx908 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN %s
 
-declare <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float, float, <32 x float>, i32, i32, i32)
+; GCN-LABEL: {{^}}kernel_32_agprs:
+; GCN:    .amdhsa_next_free_vgpr 32
+; GCN:    NumVgprs: 9
+; GCN:    NumAgprs: 32
+; GCN:    TotalNumVgprs: 32
+; GCN:    VGPRBlocks: 7
+; GCN:    NumVGPRsForWavesPerEU: 32
+; GCN:    Occupancy: 8
+define amdgpu_kernel void @kernel_32_agprs() {
+bb:
+  call void asm sideeffect "", "~{v8}" ()
+  call void asm sideeffect "", "~{a31}" ()
+  ret void
+}
+
+; GCN-LABEL: {{^}}kernel_0_agprs:
+; GCN:    .amdhsa_next_free_vgpr 1
+; GCN:    NumVgprs: 1
+; GCN:    NumAgprs: 0
+; GCN:    TotalNumVgprs: 1
+; GCN:    VGPRBlocks: 0
+; GCN:    NumVGPRsForWavesPerEU: 1
+; GCN:    Occupancy: 10
+define amdgpu_kernel void @kernel_0_agprs() {
+bb:
+  call void asm sideeffect "", "~{v0}" ()
+  ret void
+}
+
+; GCN-LABEL: {{^}}kernel_40_vgprs:
+; GCN:    .amdhsa_next_free_vgpr 40
+; GCN:    NumVgprs: 40
+; GCN:    NumAgprs: 16
+; GCN:    TotalNumVgprs: 40
+; GCN:    VGPRBlocks: 9
+; GCN:    NumVGPRsForWavesPerEU: 40
+; GCN:    Occupancy: 6
+define amdgpu_kernel void @kernel_40_vgprs() {
+bb:
+  call void asm sideeffect "", "~{v39}" ()
+  call void asm sideeffect "", "~{a15}" ()
+  ret void
+}
+
+; GCN-LABEL: {{^}}func_32_agprs:
+; GCN:    NumVgprs: 9
+; GCN:    NumAgprs: 32
+; GCN:    TotalNumVgprs: 32
+define void @func_32_agprs() #0 {
+bb:
+  call void asm sideeffect "", "~{v8}" ()
+  call void asm sideeffect "", "~{a31}" ()
+  ret void
+}
+
+; GCN-LABEL: {{^}}func_32_vgprs:
+; GCN:    NumVgprs: 32
+; GCN:    NumAgprs: 9
+; GCN:    TotalNumVgprs: 32
+define void @func_32_vgprs() {
+bb:
+  call void asm sideeffect "", "~{v31}" ()
+  call void asm sideeffect "", "~{a8}" ()
+  ret void
+}
 
-; GCN-LABEL: {{^}}test_32_agprs:
-; GCN: v_mfma_f32_32x32x1f32 a[0:31], {{v[0-9]+}}, {{v[0-9]+}}, 0
-; GCN-NOT: v28
-; GCN: NumVgprs: 32
-; GCN: VGPRBlocks: 7
-define amdgpu_kernel void @test_32_agprs(<32 x float> addrspace(1)* %arg) {
+; GCN-LABEL: {{^}}func_0_agprs:
+; GCN:    NumVgprs: 1
+; GCN:    NumAgprs: 0
+; GCN:    TotalNumVgprs: 1
+define amdgpu_kernel void @func_0_agprs() {
 bb:
-  %mai.1 = tail call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.0, float 2.0, <32 x float> <float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0>, i32 0, i32 0, i32 0)
-  store <32 x float> %mai.1, <32 x float> addrspace(1)* %arg
+  call void asm sideeffect "", "~{v0}" ()
   ret void
 }
+
+; GCN-LABEL: {{^}}kernel_max_gprs:
+; GCN:    .amdhsa_next_free_vgpr 256
+; GCN:    NumVgprs: 256
+; GCN:    NumAgprs: 256
+; GCN:    TotalNumVgprs: 256
+; GCN:    VGPRBlocks: 63
+; GCN:    NumVGPRsForWavesPerEU: 256
+; GCN:    Occupancy: 1
+define amdgpu_kernel void @kernel_max_gprs() {
+bb:
+  call void asm sideeffect "", "~{v255}" ()
+  call void asm sideeffect "", "~{a255}" ()
+  ret void
+}
+
+; GCN-LABEL: {{^}}kernel_call_func_32_agprs:
+; GCN:    .amdhsa_next_free_vgpr 32
+; GCN:    NumVgprs: 9
+; GCN:    NumAgprs: 32
+; GCN:    TotalNumVgprs: 32
+; GCN:    VGPRBlocks: 7
+; GCN:    NumVGPRsForWavesPerEU: 32
+; GCN:    Occupancy: 8
+define amdgpu_kernel void @kernel_call_func_32_agprs() {
+bb:
+  call void @func_32_agprs() #0
+  ret void
+}
+
+; GCN-LABEL: {{^}}func_call_func_32_agprs:
+; GCN:    NumVgprs: 9
+; GCN:    NumAgprs: 32
+; GCN:    TotalNumVgprs: 32
+define void @func_call_func_32_agprs() {
+bb:
+  call void @func_32_agprs() #0
+  ret void
+}
+
+declare void @undef_func()
+
+; GCN-LABEL: {{^}}kernel_call_undef_func:
+; GCN:    .amdhsa_next_free_vgpr 24
+; GCN:    NumVgprs: 24
+; GCN:    NumAgprs: 24
+; GCN:    TotalNumVgprs: 24
+; GCN:    VGPRBlocks: 5
+; GCN:    NumVGPRsForWavesPerEU: 24
+; GCN:    Occupancy: 10
+define amdgpu_kernel void @kernel_call_undef_func() {
+bb:
+  call void @undef_func()
+  ret void
+}
+
+attributes #0 = { nounwind noinline }
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.private-memory.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.private-memory.ll
index f93c96bc5..d38452a3a 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.private-memory.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.private-memory.ll
@@ -13,7 +13,14 @@ declare i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
 
 ; GCN-LABEL: {{^}}work_item_info:
 ; GCN-NOT: v0
-; GCN: v_add_{{[iu]}}32_e32 [[RESULT:v[0-9]+]], vcc, v{{[0-9]+}}, v0
+; GCN: s_load_dword [[IN:s[0-9]+]]
+; GCN-NOT: v0
+
+; GCN-ALLOCA: v_add_{{[iu]}}32_e32 [[RESULT:v[0-9]+]], vcc, v{{[0-9]+}}, v0
+
+; GCN-PROMOTE: v_cmp_eq_u32_e64 vcc, [[IN]], 1
+; GCN-PROMOTE-NEXT: v_addc_u32_e32 [[RESULT:v[0-9]+]], vcc, 0, v0, vcc
+
 ; GCN: buffer_store_dword [[RESULT]]
 define amdgpu_kernel void @work_item_info(i32 addrspace(1)* %out, i32 %in) {
 entry:
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll
index e76c70684..8d13354e1 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll
@@ -227,7 +227,7 @@ define amdgpu_kernel void @udiv_i16(i16 addrspace(1)* %out, i16 %x, i16 %y) {
 ; CHECK-NEXT:    [[TMP5:%.*]] = fdiv fast float 1.000000e+00, [[TMP4]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = fmul fast float [[TMP3]], [[TMP5]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = call fast float @llvm.trunc.f32(float [[TMP6]])
-; CHECK-NEXT:    [[TMP8:%.*]] = fsub fast float -0.000000e+00, [[TMP7]]
+; CHECK-NEXT:    [[TMP8:%.*]] = fneg fast float [[TMP7]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP8]], float [[TMP4]], float [[TMP3]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = fptoui float [[TMP7]] to i32
 ; CHECK-NEXT:    [[TMP11:%.*]] = call fast float @llvm.fabs.f32(float [[TMP9]])
@@ -254,7 +254,7 @@ define amdgpu_kernel void @urem_i16(i16 addrspace(1)* %out, i16 %x, i16 %y) {
 ; CHECK-NEXT:    [[TMP5:%.*]] = fdiv fast float 1.000000e+00, [[TMP4]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = fmul fast float [[TMP3]], [[TMP5]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = call fast float @llvm.trunc.f32(float [[TMP6]])
-; CHECK-NEXT:    [[TMP8:%.*]] = fsub fast float -0.000000e+00, [[TMP7]]
+; CHECK-NEXT:    [[TMP8:%.*]] = fneg fast float [[TMP7]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP8]], float [[TMP4]], float [[TMP3]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = fptoui float [[TMP7]] to i32
 ; CHECK-NEXT:    [[TMP11:%.*]] = call fast float @llvm.fabs.f32(float [[TMP9]])
@@ -286,7 +286,7 @@ define amdgpu_kernel void @sdiv_i16(i16 addrspace(1)* %out, i16 %x, i16 %y) {
 ; CHECK-NEXT:    [[TMP8:%.*]] = fdiv fast float 1.000000e+00, [[TMP7]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = fmul fast float [[TMP6]], [[TMP8]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = call fast float @llvm.trunc.f32(float [[TMP9]])
-; CHECK-NEXT:    [[TMP11:%.*]] = fsub fast float -0.000000e+00, [[TMP10]]
+; CHECK-NEXT:    [[TMP11:%.*]] = fneg fast float [[TMP10]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP11]], float [[TMP7]], float [[TMP6]])
 ; CHECK-NEXT:    [[TMP13:%.*]] = fptosi float [[TMP10]] to i32
 ; CHECK-NEXT:    [[TMP14:%.*]] = call fast float @llvm.fabs.f32(float [[TMP12]])
@@ -317,7 +317,7 @@ define amdgpu_kernel void @srem_i16(i16 addrspace(1)* %out, i16 %x, i16 %y) {
 ; CHECK-NEXT:    [[TMP8:%.*]] = fdiv fast float 1.000000e+00, [[TMP7]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = fmul fast float [[TMP6]], [[TMP8]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = call fast float @llvm.trunc.f32(float [[TMP9]])
-; CHECK-NEXT:    [[TMP11:%.*]] = fsub fast float -0.000000e+00, [[TMP10]]
+; CHECK-NEXT:    [[TMP11:%.*]] = fneg fast float [[TMP10]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP11]], float [[TMP7]], float [[TMP6]])
 ; CHECK-NEXT:    [[TMP13:%.*]] = fptosi float [[TMP10]] to i32
 ; CHECK-NEXT:    [[TMP14:%.*]] = call fast float @llvm.fabs.f32(float [[TMP12]])
@@ -347,7 +347,7 @@ define amdgpu_kernel void @udiv_i8(i8 addrspace(1)* %out, i8 %x, i8 %y) {
 ; CHECK-NEXT:    [[TMP5:%.*]] = fdiv fast float 1.000000e+00, [[TMP4]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = fmul fast float [[TMP3]], [[TMP5]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = call fast float @llvm.trunc.f32(float [[TMP6]])
-; CHECK-NEXT:    [[TMP8:%.*]] = fsub fast float -0.000000e+00, [[TMP7]]
+; CHECK-NEXT:    [[TMP8:%.*]] = fneg fast float [[TMP7]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP8]], float [[TMP4]], float [[TMP3]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = fptoui float [[TMP7]] to i32
 ; CHECK-NEXT:    [[TMP11:%.*]] = call fast float @llvm.fabs.f32(float [[TMP9]])
@@ -374,7 +374,7 @@ define amdgpu_kernel void @urem_i8(i8 addrspace(1)* %out, i8 %x, i8 %y) {
 ; CHECK-NEXT:    [[TMP5:%.*]] = fdiv fast float 1.000000e+00, [[TMP4]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = fmul fast float [[TMP3]], [[TMP5]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = call fast float @llvm.trunc.f32(float [[TMP6]])
-; CHECK-NEXT:    [[TMP8:%.*]] = fsub fast float -0.000000e+00, [[TMP7]]
+; CHECK-NEXT:    [[TMP8:%.*]] = fneg fast float [[TMP7]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP8]], float [[TMP4]], float [[TMP3]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = fptoui float [[TMP7]] to i32
 ; CHECK-NEXT:    [[TMP11:%.*]] = call fast float @llvm.fabs.f32(float [[TMP9]])
@@ -406,7 +406,7 @@ define amdgpu_kernel void @sdiv_i8(i8 addrspace(1)* %out, i8 %x, i8 %y) {
 ; CHECK-NEXT:    [[TMP8:%.*]] = fdiv fast float 1.000000e+00, [[TMP7]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = fmul fast float [[TMP6]], [[TMP8]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = call fast float @llvm.trunc.f32(float [[TMP9]])
-; CHECK-NEXT:    [[TMP11:%.*]] = fsub fast float -0.000000e+00, [[TMP10]]
+; CHECK-NEXT:    [[TMP11:%.*]] = fneg fast float [[TMP10]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP11]], float [[TMP7]], float [[TMP6]])
 ; CHECK-NEXT:    [[TMP13:%.*]] = fptosi float [[TMP10]] to i32
 ; CHECK-NEXT:    [[TMP14:%.*]] = call fast float @llvm.fabs.f32(float [[TMP12]])
@@ -437,7 +437,7 @@ define amdgpu_kernel void @srem_i8(i8 addrspace(1)* %out, i8 %x, i8 %y) {
 ; CHECK-NEXT:    [[TMP8:%.*]] = fdiv fast float 1.000000e+00, [[TMP7]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = fmul fast float [[TMP6]], [[TMP8]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = call fast float @llvm.trunc.f32(float [[TMP9]])
-; CHECK-NEXT:    [[TMP11:%.*]] = fsub fast float -0.000000e+00, [[TMP10]]
+; CHECK-NEXT:    [[TMP11:%.*]] = fneg fast float [[TMP10]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP11]], float [[TMP7]], float [[TMP6]])
 ; CHECK-NEXT:    [[TMP13:%.*]] = fptosi float [[TMP10]] to i32
 ; CHECK-NEXT:    [[TMP14:%.*]] = call fast float @llvm.fabs.f32(float [[TMP12]])
@@ -1265,7 +1265,7 @@ define amdgpu_kernel void @udiv_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> %x
 ; CHECK-NEXT:    [[TMP7:%.*]] = fdiv fast float 1.000000e+00, [[TMP6]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = fmul fast float [[TMP5]], [[TMP7]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = call fast float @llvm.trunc.f32(float [[TMP8]])
-; CHECK-NEXT:    [[TMP10:%.*]] = fsub fast float -0.000000e+00, [[TMP9]]
+; CHECK-NEXT:    [[TMP10:%.*]] = fneg fast float [[TMP9]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP10]], float [[TMP6]], float [[TMP5]])
 ; CHECK-NEXT:    [[TMP12:%.*]] = fptoui float [[TMP9]] to i32
 ; CHECK-NEXT:    [[TMP13:%.*]] = call fast float @llvm.fabs.f32(float [[TMP11]])
@@ -1285,7 +1285,7 @@ define amdgpu_kernel void @udiv_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> %x
 ; CHECK-NEXT:    [[TMP27:%.*]] = fdiv fast float 1.000000e+00, [[TMP26]]
 ; CHECK-NEXT:    [[TMP28:%.*]] = fmul fast float [[TMP25]], [[TMP27]]
 ; CHECK-NEXT:    [[TMP29:%.*]] = call fast float @llvm.trunc.f32(float [[TMP28]])
-; CHECK-NEXT:    [[TMP30:%.*]] = fsub fast float -0.000000e+00, [[TMP29]]
+; CHECK-NEXT:    [[TMP30:%.*]] = fneg fast float [[TMP29]]
 ; CHECK-NEXT:    [[TMP31:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP30]], float [[TMP26]], float [[TMP25]])
 ; CHECK-NEXT:    [[TMP32:%.*]] = fptoui float [[TMP29]] to i32
 ; CHECK-NEXT:    [[TMP33:%.*]] = call fast float @llvm.fabs.f32(float [[TMP31]])
@@ -1305,7 +1305,7 @@ define amdgpu_kernel void @udiv_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> %x
 ; CHECK-NEXT:    [[TMP47:%.*]] = fdiv fast float 1.000000e+00, [[TMP46]]
 ; CHECK-NEXT:    [[TMP48:%.*]] = fmul fast float [[TMP45]], [[TMP47]]
 ; CHECK-NEXT:    [[TMP49:%.*]] = call fast float @llvm.trunc.f32(float [[TMP48]])
-; CHECK-NEXT:    [[TMP50:%.*]] = fsub fast float -0.000000e+00, [[TMP49]]
+; CHECK-NEXT:    [[TMP50:%.*]] = fneg fast float [[TMP49]]
 ; CHECK-NEXT:    [[TMP51:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP50]], float [[TMP46]], float [[TMP45]])
 ; CHECK-NEXT:    [[TMP52:%.*]] = fptoui float [[TMP49]] to i32
 ; CHECK-NEXT:    [[TMP53:%.*]] = call fast float @llvm.fabs.f32(float [[TMP51]])
@@ -1325,7 +1325,7 @@ define amdgpu_kernel void @udiv_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> %x
 ; CHECK-NEXT:    [[TMP67:%.*]] = fdiv fast float 1.000000e+00, [[TMP66]]
 ; CHECK-NEXT:    [[TMP68:%.*]] = fmul fast float [[TMP65]], [[TMP67]]
 ; CHECK-NEXT:    [[TMP69:%.*]] = call fast float @llvm.trunc.f32(float [[TMP68]])
-; CHECK-NEXT:    [[TMP70:%.*]] = fsub fast float -0.000000e+00, [[TMP69]]
+; CHECK-NEXT:    [[TMP70:%.*]] = fneg fast float [[TMP69]]
 ; CHECK-NEXT:    [[TMP71:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP70]], float [[TMP66]], float [[TMP65]])
 ; CHECK-NEXT:    [[TMP72:%.*]] = fptoui float [[TMP69]] to i32
 ; CHECK-NEXT:    [[TMP73:%.*]] = call fast float @llvm.fabs.f32(float [[TMP71]])
@@ -1355,7 +1355,7 @@ define amdgpu_kernel void @urem_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> %x
 ; CHECK-NEXT:    [[TMP7:%.*]] = fdiv fast float 1.000000e+00, [[TMP6]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = fmul fast float [[TMP5]], [[TMP7]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = call fast float @llvm.trunc.f32(float [[TMP8]])
-; CHECK-NEXT:    [[TMP10:%.*]] = fsub fast float -0.000000e+00, [[TMP9]]
+; CHECK-NEXT:    [[TMP10:%.*]] = fneg fast float [[TMP9]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP10]], float [[TMP6]], float [[TMP5]])
 ; CHECK-NEXT:    [[TMP12:%.*]] = fptoui float [[TMP9]] to i32
 ; CHECK-NEXT:    [[TMP13:%.*]] = call fast float @llvm.fabs.f32(float [[TMP11]])
@@ -1377,7 +1377,7 @@ define amdgpu_kernel void @urem_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> %x
 ; CHECK-NEXT:    [[TMP29:%.*]] = fdiv fast float 1.000000e+00, [[TMP28]]
 ; CHECK-NEXT:    [[TMP30:%.*]] = fmul fast float [[TMP27]], [[TMP29]]
 ; CHECK-NEXT:    [[TMP31:%.*]] = call fast float @llvm.trunc.f32(float [[TMP30]])
-; CHECK-NEXT:    [[TMP32:%.*]] = fsub fast float -0.000000e+00, [[TMP31]]
+; CHECK-NEXT:    [[TMP32:%.*]] = fneg fast float [[TMP31]]
 ; CHECK-NEXT:    [[TMP33:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP32]], float [[TMP28]], float [[TMP27]])
 ; CHECK-NEXT:    [[TMP34:%.*]] = fptoui float [[TMP31]] to i32
 ; CHECK-NEXT:    [[TMP35:%.*]] = call fast float @llvm.fabs.f32(float [[TMP33]])
@@ -1399,7 +1399,7 @@ define amdgpu_kernel void @urem_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> %x
 ; CHECK-NEXT:    [[TMP51:%.*]] = fdiv fast float 1.000000e+00, [[TMP50]]
 ; CHECK-NEXT:    [[TMP52:%.*]] = fmul fast float [[TMP49]], [[TMP51]]
 ; CHECK-NEXT:    [[TMP53:%.*]] = call fast float @llvm.trunc.f32(float [[TMP52]])
-; CHECK-NEXT:    [[TMP54:%.*]] = fsub fast float -0.000000e+00, [[TMP53]]
+; CHECK-NEXT:    [[TMP54:%.*]] = fneg fast float [[TMP53]]
 ; CHECK-NEXT:    [[TMP55:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP54]], float [[TMP50]], float [[TMP49]])
 ; CHECK-NEXT:    [[TMP56:%.*]] = fptoui float [[TMP53]] to i32
 ; CHECK-NEXT:    [[TMP57:%.*]] = call fast float @llvm.fabs.f32(float [[TMP55]])
@@ -1421,7 +1421,7 @@ define amdgpu_kernel void @urem_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> %x
 ; CHECK-NEXT:    [[TMP73:%.*]] = fdiv fast float 1.000000e+00, [[TMP72]]
 ; CHECK-NEXT:    [[TMP74:%.*]] = fmul fast float [[TMP71]], [[TMP73]]
 ; CHECK-NEXT:    [[TMP75:%.*]] = call fast float @llvm.trunc.f32(float [[TMP74]])
-; CHECK-NEXT:    [[TMP76:%.*]] = fsub fast float -0.000000e+00, [[TMP75]]
+; CHECK-NEXT:    [[TMP76:%.*]] = fneg fast float [[TMP75]]
 ; CHECK-NEXT:    [[TMP77:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP76]], float [[TMP72]], float [[TMP71]])
 ; CHECK-NEXT:    [[TMP78:%.*]] = fptoui float [[TMP75]] to i32
 ; CHECK-NEXT:    [[TMP79:%.*]] = call fast float @llvm.fabs.f32(float [[TMP77]])
@@ -1456,7 +1456,7 @@ define amdgpu_kernel void @sdiv_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> %x
 ; CHECK-NEXT:    [[TMP10:%.*]] = fdiv fast float 1.000000e+00, [[TMP9]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = fmul fast float [[TMP8]], [[TMP10]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = call fast float @llvm.trunc.f32(float [[TMP11]])
-; CHECK-NEXT:    [[TMP13:%.*]] = fsub fast float -0.000000e+00, [[TMP12]]
+; CHECK-NEXT:    [[TMP13:%.*]] = fneg fast float [[TMP12]]
 ; CHECK-NEXT:    [[TMP14:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP13]], float [[TMP9]], float [[TMP8]])
 ; CHECK-NEXT:    [[TMP15:%.*]] = fptosi float [[TMP12]] to i32
 ; CHECK-NEXT:    [[TMP16:%.*]] = call fast float @llvm.fabs.f32(float [[TMP14]])
@@ -1480,7 +1480,7 @@ define amdgpu_kernel void @sdiv_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> %x
 ; CHECK-NEXT:    [[TMP34:%.*]] = fdiv fast float 1.000000e+00, [[TMP33]]
 ; CHECK-NEXT:    [[TMP35:%.*]] = fmul fast float [[TMP32]], [[TMP34]]
 ; CHECK-NEXT:    [[TMP36:%.*]] = call fast float @llvm.trunc.f32(float [[TMP35]])
-; CHECK-NEXT:    [[TMP37:%.*]] = fsub fast float -0.000000e+00, [[TMP36]]
+; CHECK-NEXT:    [[TMP37:%.*]] = fneg fast float [[TMP36]]
 ; CHECK-NEXT:    [[TMP38:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP37]], float [[TMP33]], float [[TMP32]])
 ; CHECK-NEXT:    [[TMP39:%.*]] = fptosi float [[TMP36]] to i32
 ; CHECK-NEXT:    [[TMP40:%.*]] = call fast float @llvm.fabs.f32(float [[TMP38]])
@@ -1504,7 +1504,7 @@ define amdgpu_kernel void @sdiv_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> %x
 ; CHECK-NEXT:    [[TMP58:%.*]] = fdiv fast float 1.000000e+00, [[TMP57]]
 ; CHECK-NEXT:    [[TMP59:%.*]] = fmul fast float [[TMP56]], [[TMP58]]
 ; CHECK-NEXT:    [[TMP60:%.*]] = call fast float @llvm.trunc.f32(float [[TMP59]])
-; CHECK-NEXT:    [[TMP61:%.*]] = fsub fast float -0.000000e+00, [[TMP60]]
+; CHECK-NEXT:    [[TMP61:%.*]] = fneg fast float [[TMP60]]
 ; CHECK-NEXT:    [[TMP62:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP61]], float [[TMP57]], float [[TMP56]])
 ; CHECK-NEXT:    [[TMP63:%.*]] = fptosi float [[TMP60]] to i32
 ; CHECK-NEXT:    [[TMP64:%.*]] = call fast float @llvm.fabs.f32(float [[TMP62]])
@@ -1528,7 +1528,7 @@ define amdgpu_kernel void @sdiv_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> %x
 ; CHECK-NEXT:    [[TMP82:%.*]] = fdiv fast float 1.000000e+00, [[TMP81]]
 ; CHECK-NEXT:    [[TMP83:%.*]] = fmul fast float [[TMP80]], [[TMP82]]
 ; CHECK-NEXT:    [[TMP84:%.*]] = call fast float @llvm.trunc.f32(float [[TMP83]])
-; CHECK-NEXT:    [[TMP85:%.*]] = fsub fast float -0.000000e+00, [[TMP84]]
+; CHECK-NEXT:    [[TMP85:%.*]] = fneg fast float [[TMP84]]
 ; CHECK-NEXT:    [[TMP86:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP85]], float [[TMP81]], float [[TMP80]])
 ; CHECK-NEXT:    [[TMP87:%.*]] = fptosi float [[TMP84]] to i32
 ; CHECK-NEXT:    [[TMP88:%.*]] = call fast float @llvm.fabs.f32(float [[TMP86]])
@@ -1562,7 +1562,7 @@ define amdgpu_kernel void @srem_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> %x
 ; CHECK-NEXT:    [[TMP10:%.*]] = fdiv fast float 1.000000e+00, [[TMP9]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = fmul fast float [[TMP8]], [[TMP10]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = call fast float @llvm.trunc.f32(float [[TMP11]])
-; CHECK-NEXT:    [[TMP13:%.*]] = fsub fast float -0.000000e+00, [[TMP12]]
+; CHECK-NEXT:    [[TMP13:%.*]] = fneg fast float [[TMP12]]
 ; CHECK-NEXT:    [[TMP14:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP13]], float [[TMP9]], float [[TMP8]])
 ; CHECK-NEXT:    [[TMP15:%.*]] = fptosi float [[TMP12]] to i32
 ; CHECK-NEXT:    [[TMP16:%.*]] = call fast float @llvm.fabs.f32(float [[TMP14]])
@@ -1588,7 +1588,7 @@ define amdgpu_kernel void @srem_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> %x
 ; CHECK-NEXT:    [[TMP36:%.*]] = fdiv fast float 1.000000e+00, [[TMP35]]
 ; CHECK-NEXT:    [[TMP37:%.*]] = fmul fast float [[TMP34]], [[TMP36]]
 ; CHECK-NEXT:    [[TMP38:%.*]] = call fast float @llvm.trunc.f32(float [[TMP37]])
-; CHECK-NEXT:    [[TMP39:%.*]] = fsub fast float -0.000000e+00, [[TMP38]]
+; CHECK-NEXT:    [[TMP39:%.*]] = fneg fast float [[TMP38]]
 ; CHECK-NEXT:    [[TMP40:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP39]], float [[TMP35]], float [[TMP34]])
 ; CHECK-NEXT:    [[TMP41:%.*]] = fptosi float [[TMP38]] to i32
 ; CHECK-NEXT:    [[TMP42:%.*]] = call fast float @llvm.fabs.f32(float [[TMP40]])
@@ -1614,7 +1614,7 @@ define amdgpu_kernel void @srem_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> %x
 ; CHECK-NEXT:    [[TMP62:%.*]] = fdiv fast float 1.000000e+00, [[TMP61]]
 ; CHECK-NEXT:    [[TMP63:%.*]] = fmul fast float [[TMP60]], [[TMP62]]
 ; CHECK-NEXT:    [[TMP64:%.*]] = call fast float @llvm.trunc.f32(float [[TMP63]])
-; CHECK-NEXT:    [[TMP65:%.*]] = fsub fast float -0.000000e+00, [[TMP64]]
+; CHECK-NEXT:    [[TMP65:%.*]] = fneg fast float [[TMP64]]
 ; CHECK-NEXT:    [[TMP66:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP65]], float [[TMP61]], float [[TMP60]])
 ; CHECK-NEXT:    [[TMP67:%.*]] = fptosi float [[TMP64]] to i32
 ; CHECK-NEXT:    [[TMP68:%.*]] = call fast float @llvm.fabs.f32(float [[TMP66]])
@@ -1640,7 +1640,7 @@ define amdgpu_kernel void @srem_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> %x
 ; CHECK-NEXT:    [[TMP88:%.*]] = fdiv fast float 1.000000e+00, [[TMP87]]
 ; CHECK-NEXT:    [[TMP89:%.*]] = fmul fast float [[TMP86]], [[TMP88]]
 ; CHECK-NEXT:    [[TMP90:%.*]] = call fast float @llvm.trunc.f32(float [[TMP89]])
-; CHECK-NEXT:    [[TMP91:%.*]] = fsub fast float -0.000000e+00, [[TMP90]]
+; CHECK-NEXT:    [[TMP91:%.*]] = fneg fast float [[TMP90]]
 ; CHECK-NEXT:    [[TMP92:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP91]], float [[TMP87]], float [[TMP86]])
 ; CHECK-NEXT:    [[TMP93:%.*]] = fptosi float [[TMP90]] to i32
 ; CHECK-NEXT:    [[TMP94:%.*]] = call fast float @llvm.fabs.f32(float [[TMP92]])
@@ -1671,7 +1671,7 @@ define amdgpu_kernel void @udiv_i3(i3 addrspace(1)* %out, i3 %x, i3 %y) {
 ; CHECK-NEXT:    [[TMP5:%.*]] = fdiv fast float 1.000000e+00, [[TMP4]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = fmul fast float [[TMP3]], [[TMP5]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = call fast float @llvm.trunc.f32(float [[TMP6]])
-; CHECK-NEXT:    [[TMP8:%.*]] = fsub fast float -0.000000e+00, [[TMP7]]
+; CHECK-NEXT:    [[TMP8:%.*]] = fneg fast float [[TMP7]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP8]], float [[TMP4]], float [[TMP3]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = fptoui float [[TMP7]] to i32
 ; CHECK-NEXT:    [[TMP11:%.*]] = call fast float @llvm.fabs.f32(float [[TMP9]])
@@ -1698,7 +1698,7 @@ define amdgpu_kernel void @urem_i3(i3 addrspace(1)* %out, i3 %x, i3 %y) {
 ; CHECK-NEXT:    [[TMP5:%.*]] = fdiv fast float 1.000000e+00, [[TMP4]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = fmul fast float [[TMP3]], [[TMP5]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = call fast float @llvm.trunc.f32(float [[TMP6]])
-; CHECK-NEXT:    [[TMP8:%.*]] = fsub fast float -0.000000e+00, [[TMP7]]
+; CHECK-NEXT:    [[TMP8:%.*]] = fneg fast float [[TMP7]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP8]], float [[TMP4]], float [[TMP3]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = fptoui float [[TMP7]] to i32
 ; CHECK-NEXT:    [[TMP11:%.*]] = call fast float @llvm.fabs.f32(float [[TMP9]])
@@ -1730,7 +1730,7 @@ define amdgpu_kernel void @sdiv_i3(i3 addrspace(1)* %out, i3 %x, i3 %y) {
 ; CHECK-NEXT:    [[TMP8:%.*]] = fdiv fast float 1.000000e+00, [[TMP7]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = fmul fast float [[TMP6]], [[TMP8]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = call fast float @llvm.trunc.f32(float [[TMP9]])
-; CHECK-NEXT:    [[TMP11:%.*]] = fsub fast float -0.000000e+00, [[TMP10]]
+; CHECK-NEXT:    [[TMP11:%.*]] = fneg fast float [[TMP10]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP11]], float [[TMP7]], float [[TMP6]])
 ; CHECK-NEXT:    [[TMP13:%.*]] = fptosi float [[TMP10]] to i32
 ; CHECK-NEXT:    [[TMP14:%.*]] = call fast float @llvm.fabs.f32(float [[TMP12]])
@@ -1761,7 +1761,7 @@ define amdgpu_kernel void @srem_i3(i3 addrspace(1)* %out, i3 %x, i3 %y) {
 ; CHECK-NEXT:    [[TMP8:%.*]] = fdiv fast float 1.000000e+00, [[TMP7]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = fmul fast float [[TMP6]], [[TMP8]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = call fast float @llvm.trunc.f32(float [[TMP9]])
-; CHECK-NEXT:    [[TMP11:%.*]] = fsub fast float -0.000000e+00, [[TMP10]]
+; CHECK-NEXT:    [[TMP11:%.*]] = fneg fast float [[TMP10]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP11]], float [[TMP7]], float [[TMP6]])
 ; CHECK-NEXT:    [[TMP13:%.*]] = fptosi float [[TMP10]] to i32
 ; CHECK-NEXT:    [[TMP14:%.*]] = call fast float @llvm.fabs.f32(float [[TMP12]])
@@ -1793,7 +1793,7 @@ define amdgpu_kernel void @udiv_v3i16(<3 x i16> addrspace(1)* %out, <3 x i16> %x
 ; CHECK-NEXT:    [[TMP7:%.*]] = fdiv fast float 1.000000e+00, [[TMP6]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = fmul fast float [[TMP5]], [[TMP7]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = call fast float @llvm.trunc.f32(float [[TMP8]])
-; CHECK-NEXT:    [[TMP10:%.*]] = fsub fast float -0.000000e+00, [[TMP9]]
+; CHECK-NEXT:    [[TMP10:%.*]] = fneg fast float [[TMP9]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP10]], float [[TMP6]], float [[TMP5]])
 ; CHECK-NEXT:    [[TMP12:%.*]] = fptoui float [[TMP9]] to i32
 ; CHECK-NEXT:    [[TMP13:%.*]] = call fast float @llvm.fabs.f32(float [[TMP11]])
@@ -1813,7 +1813,7 @@ define amdgpu_kernel void @udiv_v3i16(<3 x i16> addrspace(1)* %out, <3 x i16> %x
 ; CHECK-NEXT:    [[TMP27:%.*]] = fdiv fast float 1.000000e+00, [[TMP26]]
 ; CHECK-NEXT:    [[TMP28:%.*]] = fmul fast float [[TMP25]], [[TMP27]]
 ; CHECK-NEXT:    [[TMP29:%.*]] = call fast float @llvm.trunc.f32(float [[TMP28]])
-; CHECK-NEXT:    [[TMP30:%.*]] = fsub fast float -0.000000e+00, [[TMP29]]
+; CHECK-NEXT:    [[TMP30:%.*]] = fneg fast float [[TMP29]]
 ; CHECK-NEXT:    [[TMP31:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP30]], float [[TMP26]], float [[TMP25]])
 ; CHECK-NEXT:    [[TMP32:%.*]] = fptoui float [[TMP29]] to i32
 ; CHECK-NEXT:    [[TMP33:%.*]] = call fast float @llvm.fabs.f32(float [[TMP31]])
@@ -1833,7 +1833,7 @@ define amdgpu_kernel void @udiv_v3i16(<3 x i16> addrspace(1)* %out, <3 x i16> %x
 ; CHECK-NEXT:    [[TMP47:%.*]] = fdiv fast float 1.000000e+00, [[TMP46]]
 ; CHECK-NEXT:    [[TMP48:%.*]] = fmul fast float [[TMP45]], [[TMP47]]
 ; CHECK-NEXT:    [[TMP49:%.*]] = call fast float @llvm.trunc.f32(float [[TMP48]])
-; CHECK-NEXT:    [[TMP50:%.*]] = fsub fast float -0.000000e+00, [[TMP49]]
+; CHECK-NEXT:    [[TMP50:%.*]] = fneg fast float [[TMP49]]
 ; CHECK-NEXT:    [[TMP51:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP50]], float [[TMP46]], float [[TMP45]])
 ; CHECK-NEXT:    [[TMP52:%.*]] = fptoui float [[TMP49]] to i32
 ; CHECK-NEXT:    [[TMP53:%.*]] = call fast float @llvm.fabs.f32(float [[TMP51]])
@@ -1863,7 +1863,7 @@ define amdgpu_kernel void @urem_v3i16(<3 x i16> addrspace(1)* %out, <3 x i16> %x
 ; CHECK-NEXT:    [[TMP7:%.*]] = fdiv fast float 1.000000e+00, [[TMP6]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = fmul fast float [[TMP5]], [[TMP7]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = call fast float @llvm.trunc.f32(float [[TMP8]])
-; CHECK-NEXT:    [[TMP10:%.*]] = fsub fast float -0.000000e+00, [[TMP9]]
+; CHECK-NEXT:    [[TMP10:%.*]] = fneg fast float [[TMP9]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP10]], float [[TMP6]], float [[TMP5]])
 ; CHECK-NEXT:    [[TMP12:%.*]] = fptoui float [[TMP9]] to i32
 ; CHECK-NEXT:    [[TMP13:%.*]] = call fast float @llvm.fabs.f32(float [[TMP11]])
@@ -1885,7 +1885,7 @@ define amdgpu_kernel void @urem_v3i16(<3 x i16> addrspace(1)* %out, <3 x i16> %x
 ; CHECK-NEXT:    [[TMP29:%.*]] = fdiv fast float 1.000000e+00, [[TMP28]]
 ; CHECK-NEXT:    [[TMP30:%.*]] = fmul fast float [[TMP27]], [[TMP29]]
 ; CHECK-NEXT:    [[TMP31:%.*]] = call fast float @llvm.trunc.f32(float [[TMP30]])
-; CHECK-NEXT:    [[TMP32:%.*]] = fsub fast float -0.000000e+00, [[TMP31]]
+; CHECK-NEXT:    [[TMP32:%.*]] = fneg fast float [[TMP31]]
 ; CHECK-NEXT:    [[TMP33:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP32]], float [[TMP28]], float [[TMP27]])
 ; CHECK-NEXT:    [[TMP34:%.*]] = fptoui float [[TMP31]] to i32
 ; CHECK-NEXT:    [[TMP35:%.*]] = call fast float @llvm.fabs.f32(float [[TMP33]])
@@ -1907,7 +1907,7 @@ define amdgpu_kernel void @urem_v3i16(<3 x i16> addrspace(1)* %out, <3 x i16> %x
 ; CHECK-NEXT:    [[TMP51:%.*]] = fdiv fast float 1.000000e+00, [[TMP50]]
 ; CHECK-NEXT:    [[TMP52:%.*]] = fmul fast float [[TMP49]], [[TMP51]]
 ; CHECK-NEXT:    [[TMP53:%.*]] = call fast float @llvm.trunc.f32(float [[TMP52]])
-; CHECK-NEXT:    [[TMP54:%.*]] = fsub fast float -0.000000e+00, [[TMP53]]
+; CHECK-NEXT:    [[TMP54:%.*]] = fneg fast float [[TMP53]]
 ; CHECK-NEXT:    [[TMP55:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP54]], float [[TMP50]], float [[TMP49]])
 ; CHECK-NEXT:    [[TMP56:%.*]] = fptoui float [[TMP53]] to i32
 ; CHECK-NEXT:    [[TMP57:%.*]] = call fast float @llvm.fabs.f32(float [[TMP55]])
@@ -1942,7 +1942,7 @@ define amdgpu_kernel void @sdiv_v3i16(<3 x i16> addrspace(1)* %out, <3 x i16> %x
 ; CHECK-NEXT:    [[TMP10:%.*]] = fdiv fast float 1.000000e+00, [[TMP9]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = fmul fast float [[TMP8]], [[TMP10]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = call fast float @llvm.trunc.f32(float [[TMP11]])
-; CHECK-NEXT:    [[TMP13:%.*]] = fsub fast float -0.000000e+00, [[TMP12]]
+; CHECK-NEXT:    [[TMP13:%.*]] = fneg fast float [[TMP12]]
 ; CHECK-NEXT:    [[TMP14:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP13]], float [[TMP9]], float [[TMP8]])
 ; CHECK-NEXT:    [[TMP15:%.*]] = fptosi float [[TMP12]] to i32
 ; CHECK-NEXT:    [[TMP16:%.*]] = call fast float @llvm.fabs.f32(float [[TMP14]])
@@ -1966,7 +1966,7 @@ define amdgpu_kernel void @sdiv_v3i16(<3 x i16> addrspace(1)* %out, <3 x i16> %x
 ; CHECK-NEXT:    [[TMP34:%.*]] = fdiv fast float 1.000000e+00, [[TMP33]]
 ; CHECK-NEXT:    [[TMP35:%.*]] = fmul fast float [[TMP32]], [[TMP34]]
 ; CHECK-NEXT:    [[TMP36:%.*]] = call fast float @llvm.trunc.f32(float [[TMP35]])
-; CHECK-NEXT:    [[TMP37:%.*]] = fsub fast float -0.000000e+00, [[TMP36]]
+; CHECK-NEXT:    [[TMP37:%.*]] = fneg fast float [[TMP36]]
 ; CHECK-NEXT:    [[TMP38:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP37]], float [[TMP33]], float [[TMP32]])
 ; CHECK-NEXT:    [[TMP39:%.*]] = fptosi float [[TMP36]] to i32
 ; CHECK-NEXT:    [[TMP40:%.*]] = call fast float @llvm.fabs.f32(float [[TMP38]])
@@ -1990,7 +1990,7 @@ define amdgpu_kernel void @sdiv_v3i16(<3 x i16> addrspace(1)* %out, <3 x i16> %x
 ; CHECK-NEXT:    [[TMP58:%.*]] = fdiv fast float 1.000000e+00, [[TMP57]]
 ; CHECK-NEXT:    [[TMP59:%.*]] = fmul fast float [[TMP56]], [[TMP58]]
 ; CHECK-NEXT:    [[TMP60:%.*]] = call fast float @llvm.trunc.f32(float [[TMP59]])
-; CHECK-NEXT:    [[TMP61:%.*]] = fsub fast float -0.000000e+00, [[TMP60]]
+; CHECK-NEXT:    [[TMP61:%.*]] = fneg fast float [[TMP60]]
 ; CHECK-NEXT:    [[TMP62:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP61]], float [[TMP57]], float [[TMP56]])
 ; CHECK-NEXT:    [[TMP63:%.*]] = fptosi float [[TMP60]] to i32
 ; CHECK-NEXT:    [[TMP64:%.*]] = call fast float @llvm.fabs.f32(float [[TMP62]])
@@ -2024,7 +2024,7 @@ define amdgpu_kernel void @srem_v3i16(<3 x i16> addrspace(1)* %out, <3 x i16> %x
 ; CHECK-NEXT:    [[TMP10:%.*]] = fdiv fast float 1.000000e+00, [[TMP9]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = fmul fast float [[TMP8]], [[TMP10]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = call fast float @llvm.trunc.f32(float [[TMP11]])
-; CHECK-NEXT:    [[TMP13:%.*]] = fsub fast float -0.000000e+00, [[TMP12]]
+; CHECK-NEXT:    [[TMP13:%.*]] = fneg fast float [[TMP12]]
 ; CHECK-NEXT:    [[TMP14:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP13]], float [[TMP9]], float [[TMP8]])
 ; CHECK-NEXT:    [[TMP15:%.*]] = fptosi float [[TMP12]] to i32
 ; CHECK-NEXT:    [[TMP16:%.*]] = call fast float @llvm.fabs.f32(float [[TMP14]])
@@ -2050,7 +2050,7 @@ define amdgpu_kernel void @srem_v3i16(<3 x i16> addrspace(1)* %out, <3 x i16> %x
 ; CHECK-NEXT:    [[TMP36:%.*]] = fdiv fast float 1.000000e+00, [[TMP35]]
 ; CHECK-NEXT:    [[TMP37:%.*]] = fmul fast float [[TMP34]], [[TMP36]]
 ; CHECK-NEXT:    [[TMP38:%.*]] = call fast float @llvm.trunc.f32(float [[TMP37]])
-; CHECK-NEXT:    [[TMP39:%.*]] = fsub fast float -0.000000e+00, [[TMP38]]
+; CHECK-NEXT:    [[TMP39:%.*]] = fneg fast float [[TMP38]]
 ; CHECK-NEXT:    [[TMP40:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP39]], float [[TMP35]], float [[TMP34]])
 ; CHECK-NEXT:    [[TMP41:%.*]] = fptosi float [[TMP38]] to i32
 ; CHECK-NEXT:    [[TMP42:%.*]] = call fast float @llvm.fabs.f32(float [[TMP40]])
@@ -2076,7 +2076,7 @@ define amdgpu_kernel void @srem_v3i16(<3 x i16> addrspace(1)* %out, <3 x i16> %x
 ; CHECK-NEXT:    [[TMP62:%.*]] = fdiv fast float 1.000000e+00, [[TMP61]]
 ; CHECK-NEXT:    [[TMP63:%.*]] = fmul fast float [[TMP60]], [[TMP62]]
 ; CHECK-NEXT:    [[TMP64:%.*]] = call fast float @llvm.trunc.f32(float [[TMP63]])
-; CHECK-NEXT:    [[TMP65:%.*]] = fsub fast float -0.000000e+00, [[TMP64]]
+; CHECK-NEXT:    [[TMP65:%.*]] = fneg fast float [[TMP64]]
 ; CHECK-NEXT:    [[TMP66:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP65]], float [[TMP61]], float [[TMP60]])
 ; CHECK-NEXT:    [[TMP67:%.*]] = fptosi float [[TMP64]] to i32
 ; CHECK-NEXT:    [[TMP68:%.*]] = call fast float @llvm.fabs.f32(float [[TMP66]])
@@ -2109,7 +2109,7 @@ define amdgpu_kernel void @udiv_v3i15(<3 x i15> addrspace(1)* %out, <3 x i15> %x
 ; CHECK-NEXT:    [[TMP7:%.*]] = fdiv fast float 1.000000e+00, [[TMP6]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = fmul fast float [[TMP5]], [[TMP7]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = call fast float @llvm.trunc.f32(float [[TMP8]])
-; CHECK-NEXT:    [[TMP10:%.*]] = fsub fast float -0.000000e+00, [[TMP9]]
+; CHECK-NEXT:    [[TMP10:%.*]] = fneg fast float [[TMP9]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP10]], float [[TMP6]], float [[TMP5]])
 ; CHECK-NEXT:    [[TMP12:%.*]] = fptoui float [[TMP9]] to i32
 ; CHECK-NEXT:    [[TMP13:%.*]] = call fast float @llvm.fabs.f32(float [[TMP11]])
@@ -2129,7 +2129,7 @@ define amdgpu_kernel void @udiv_v3i15(<3 x i15> addrspace(1)* %out, <3 x i15> %x
 ; CHECK-NEXT:    [[TMP27:%.*]] = fdiv fast float 1.000000e+00, [[TMP26]]
 ; CHECK-NEXT:    [[TMP28:%.*]] = fmul fast float [[TMP25]], [[TMP27]]
 ; CHECK-NEXT:    [[TMP29:%.*]] = call fast float @llvm.trunc.f32(float [[TMP28]])
-; CHECK-NEXT:    [[TMP30:%.*]] = fsub fast float -0.000000e+00, [[TMP29]]
+; CHECK-NEXT:    [[TMP30:%.*]] = fneg fast float [[TMP29]]
 ; CHECK-NEXT:    [[TMP31:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP30]], float [[TMP26]], float [[TMP25]])
 ; CHECK-NEXT:    [[TMP32:%.*]] = fptoui float [[TMP29]] to i32
 ; CHECK-NEXT:    [[TMP33:%.*]] = call fast float @llvm.fabs.f32(float [[TMP31]])
@@ -2149,7 +2149,7 @@ define amdgpu_kernel void @udiv_v3i15(<3 x i15> addrspace(1)* %out, <3 x i15> %x
 ; CHECK-NEXT:    [[TMP47:%.*]] = fdiv fast float 1.000000e+00, [[TMP46]]
 ; CHECK-NEXT:    [[TMP48:%.*]] = fmul fast float [[TMP45]], [[TMP47]]
 ; CHECK-NEXT:    [[TMP49:%.*]] = call fast float @llvm.trunc.f32(float [[TMP48]])
-; CHECK-NEXT:    [[TMP50:%.*]] = fsub fast float -0.000000e+00, [[TMP49]]
+; CHECK-NEXT:    [[TMP50:%.*]] = fneg fast float [[TMP49]]
 ; CHECK-NEXT:    [[TMP51:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP50]], float [[TMP46]], float [[TMP45]])
 ; CHECK-NEXT:    [[TMP52:%.*]] = fptoui float [[TMP49]] to i32
 ; CHECK-NEXT:    [[TMP53:%.*]] = call fast float @llvm.fabs.f32(float [[TMP51]])
@@ -2179,7 +2179,7 @@ define amdgpu_kernel void @urem_v3i15(<3 x i15> addrspace(1)* %out, <3 x i15> %x
 ; CHECK-NEXT:    [[TMP7:%.*]] = fdiv fast float 1.000000e+00, [[TMP6]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = fmul fast float [[TMP5]], [[TMP7]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = call fast float @llvm.trunc.f32(float [[TMP8]])
-; CHECK-NEXT:    [[TMP10:%.*]] = fsub fast float -0.000000e+00, [[TMP9]]
+; CHECK-NEXT:    [[TMP10:%.*]] = fneg fast float [[TMP9]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP10]], float [[TMP6]], float [[TMP5]])
 ; CHECK-NEXT:    [[TMP12:%.*]] = fptoui float [[TMP9]] to i32
 ; CHECK-NEXT:    [[TMP13:%.*]] = call fast float @llvm.fabs.f32(float [[TMP11]])
@@ -2201,7 +2201,7 @@ define amdgpu_kernel void @urem_v3i15(<3 x i15> addrspace(1)* %out, <3 x i15> %x
 ; CHECK-NEXT:    [[TMP29:%.*]] = fdiv fast float 1.000000e+00, [[TMP28]]
 ; CHECK-NEXT:    [[TMP30:%.*]] = fmul fast float [[TMP27]], [[TMP29]]
 ; CHECK-NEXT:    [[TMP31:%.*]] = call fast float @llvm.trunc.f32(float [[TMP30]])
-; CHECK-NEXT:    [[TMP32:%.*]] = fsub fast float -0.000000e+00, [[TMP31]]
+; CHECK-NEXT:    [[TMP32:%.*]] = fneg fast float [[TMP31]]
 ; CHECK-NEXT:    [[TMP33:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP32]], float [[TMP28]], float [[TMP27]])
 ; CHECK-NEXT:    [[TMP34:%.*]] = fptoui float [[TMP31]] to i32
 ; CHECK-NEXT:    [[TMP35:%.*]] = call fast float @llvm.fabs.f32(float [[TMP33]])
@@ -2223,7 +2223,7 @@ define amdgpu_kernel void @urem_v3i15(<3 x i15> addrspace(1)* %out, <3 x i15> %x
 ; CHECK-NEXT:    [[TMP51:%.*]] = fdiv fast float 1.000000e+00, [[TMP50]]
 ; CHECK-NEXT:    [[TMP52:%.*]] = fmul fast float [[TMP49]], [[TMP51]]
 ; CHECK-NEXT:    [[TMP53:%.*]] = call fast float @llvm.trunc.f32(float [[TMP52]])
-; CHECK-NEXT:    [[TMP54:%.*]] = fsub fast float -0.000000e+00, [[TMP53]]
+; CHECK-NEXT:    [[TMP54:%.*]] = fneg fast float [[TMP53]]
 ; CHECK-NEXT:    [[TMP55:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP54]], float [[TMP50]], float [[TMP49]])
 ; CHECK-NEXT:    [[TMP56:%.*]] = fptoui float [[TMP53]] to i32
 ; CHECK-NEXT:    [[TMP57:%.*]] = call fast float @llvm.fabs.f32(float [[TMP55]])
@@ -2258,7 +2258,7 @@ define amdgpu_kernel void @sdiv_v3i15(<3 x i15> addrspace(1)* %out, <3 x i15> %x
 ; CHECK-NEXT:    [[TMP10:%.*]] = fdiv fast float 1.000000e+00, [[TMP9]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = fmul fast float [[TMP8]], [[TMP10]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = call fast float @llvm.trunc.f32(float [[TMP11]])
-; CHECK-NEXT:    [[TMP13:%.*]] = fsub fast float -0.000000e+00, [[TMP12]]
+; CHECK-NEXT:    [[TMP13:%.*]] = fneg fast float [[TMP12]]
 ; CHECK-NEXT:    [[TMP14:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP13]], float [[TMP9]], float [[TMP8]])
 ; CHECK-NEXT:    [[TMP15:%.*]] = fptosi float [[TMP12]] to i32
 ; CHECK-NEXT:    [[TMP16:%.*]] = call fast float @llvm.fabs.f32(float [[TMP14]])
@@ -2282,7 +2282,7 @@ define amdgpu_kernel void @sdiv_v3i15(<3 x i15> addrspace(1)* %out, <3 x i15> %x
 ; CHECK-NEXT:    [[TMP34:%.*]] = fdiv fast float 1.000000e+00, [[TMP33]]
 ; CHECK-NEXT:    [[TMP35:%.*]] = fmul fast float [[TMP32]], [[TMP34]]
 ; CHECK-NEXT:    [[TMP36:%.*]] = call fast float @llvm.trunc.f32(float [[TMP35]])
-; CHECK-NEXT:    [[TMP37:%.*]] = fsub fast float -0.000000e+00, [[TMP36]]
+; CHECK-NEXT:    [[TMP37:%.*]] = fneg fast float [[TMP36]]
 ; CHECK-NEXT:    [[TMP38:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP37]], float [[TMP33]], float [[TMP32]])
 ; CHECK-NEXT:    [[TMP39:%.*]] = fptosi float [[TMP36]] to i32
 ; CHECK-NEXT:    [[TMP40:%.*]] = call fast float @llvm.fabs.f32(float [[TMP38]])
@@ -2306,7 +2306,7 @@ define amdgpu_kernel void @sdiv_v3i15(<3 x i15> addrspace(1)* %out, <3 x i15> %x
 ; CHECK-NEXT:    [[TMP58:%.*]] = fdiv fast float 1.000000e+00, [[TMP57]]
 ; CHECK-NEXT:    [[TMP59:%.*]] = fmul fast float [[TMP56]], [[TMP58]]
 ; CHECK-NEXT:    [[TMP60:%.*]] = call fast float @llvm.trunc.f32(float [[TMP59]])
-; CHECK-NEXT:    [[TMP61:%.*]] = fsub fast float -0.000000e+00, [[TMP60]]
+; CHECK-NEXT:    [[TMP61:%.*]] = fneg fast float [[TMP60]]
 ; CHECK-NEXT:    [[TMP62:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP61]], float [[TMP57]], float [[TMP56]])
 ; CHECK-NEXT:    [[TMP63:%.*]] = fptosi float [[TMP60]] to i32
 ; CHECK-NEXT:    [[TMP64:%.*]] = call fast float @llvm.fabs.f32(float [[TMP62]])
@@ -2340,7 +2340,7 @@ define amdgpu_kernel void @srem_v3i15(<3 x i15> addrspace(1)* %out, <3 x i15> %x
 ; CHECK-NEXT:    [[TMP10:%.*]] = fdiv fast float 1.000000e+00, [[TMP9]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = fmul fast float [[TMP8]], [[TMP10]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = call fast float @llvm.trunc.f32(float [[TMP11]])
-; CHECK-NEXT:    [[TMP13:%.*]] = fsub fast float -0.000000e+00, [[TMP12]]
+; CHECK-NEXT:    [[TMP13:%.*]] = fneg fast float [[TMP12]]
 ; CHECK-NEXT:    [[TMP14:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP13]], float [[TMP9]], float [[TMP8]])
 ; CHECK-NEXT:    [[TMP15:%.*]] = fptosi float [[TMP12]] to i32
 ; CHECK-NEXT:    [[TMP16:%.*]] = call fast float @llvm.fabs.f32(float [[TMP14]])
@@ -2366,7 +2366,7 @@ define amdgpu_kernel void @srem_v3i15(<3 x i15> addrspace(1)* %out, <3 x i15> %x
 ; CHECK-NEXT:    [[TMP36:%.*]] = fdiv fast float 1.000000e+00, [[TMP35]]
 ; CHECK-NEXT:    [[TMP37:%.*]] = fmul fast float [[TMP34]], [[TMP36]]
 ; CHECK-NEXT:    [[TMP38:%.*]] = call fast float @llvm.trunc.f32(float [[TMP37]])
-; CHECK-NEXT:    [[TMP39:%.*]] = fsub fast float -0.000000e+00, [[TMP38]]
+; CHECK-NEXT:    [[TMP39:%.*]] = fneg fast float [[TMP38]]
 ; CHECK-NEXT:    [[TMP40:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP39]], float [[TMP35]], float [[TMP34]])
 ; CHECK-NEXT:    [[TMP41:%.*]] = fptosi float [[TMP38]] to i32
 ; CHECK-NEXT:    [[TMP42:%.*]] = call fast float @llvm.fabs.f32(float [[TMP40]])
@@ -2392,7 +2392,7 @@ define amdgpu_kernel void @srem_v3i15(<3 x i15> addrspace(1)* %out, <3 x i15> %x
 ; CHECK-NEXT:    [[TMP62:%.*]] = fdiv fast float 1.000000e+00, [[TMP61]]
 ; CHECK-NEXT:    [[TMP63:%.*]] = fmul fast float [[TMP60]], [[TMP62]]
 ; CHECK-NEXT:    [[TMP64:%.*]] = call fast float @llvm.trunc.f32(float [[TMP63]])
-; CHECK-NEXT:    [[TMP65:%.*]] = fsub fast float -0.000000e+00, [[TMP64]]
+; CHECK-NEXT:    [[TMP65:%.*]] = fneg fast float [[TMP64]]
 ; CHECK-NEXT:    [[TMP66:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP65]], float [[TMP61]], float [[TMP60]])
 ; CHECK-NEXT:    [[TMP67:%.*]] = fptosi float [[TMP64]] to i32
 ; CHECK-NEXT:    [[TMP68:%.*]] = call fast float @llvm.fabs.f32(float [[TMP66]])
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-mul24.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-mul24.ll
index cda1da825..91575f5f1 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-mul24.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-mul24.ll
@@ -1,18 +1,23 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -S -mtriple=amdgcn-- -mcpu=tahiti -amdgpu-codegenprepare %s | FileCheck -check-prefix=SI %s
 ; RUN: opt -S -mtriple=amdgcn-- -mcpu=fiji -amdgpu-codegenprepare %s | FileCheck -check-prefix=VI %s
+; RUN: opt -S -mtriple=amdgcn-- -mcpu=fiji -amdgpu-codegenprepare-mul24=0 -amdgpu-codegenprepare %s | FileCheck -check-prefix=DISABLED %s
 
 define i16 @mul_i16(i16 %lhs, i16 %rhs) {
 ; SI-LABEL: @mul_i16(
 ; SI-NEXT:    [[TMP1:%.*]] = zext i16 [[LHS:%.*]] to i32
 ; SI-NEXT:    [[TMP2:%.*]] = zext i16 [[RHS:%.*]] to i32
 ; SI-NEXT:    [[TMP3:%.*]] = call i32 @llvm.amdgcn.mul.u24(i32 [[TMP1]], i32 [[TMP2]])
-; SI-NEXT:    [[TMP4:%.*]] = trunc i32 [[TMP3]] to i16
-; SI-NEXT:    ret i16 [[TMP4]]
+; SI-NEXT:    [[MUL:%.*]] = trunc i32 [[TMP3]] to i16
+; SI-NEXT:    ret i16 [[MUL]]
 ;
 ; VI-LABEL: @mul_i16(
 ; VI-NEXT:    [[MUL:%.*]] = mul i16 [[LHS:%.*]], [[RHS:%.*]]
 ; VI-NEXT:    ret i16 [[MUL]]
+;
+; DISABLED-LABEL: @mul_i16(
+; DISABLED-NEXT:    [[MUL:%.*]] = mul i16 [[LHS:%.*]], [[RHS:%.*]]
+; DISABLED-NEXT:    ret i16 [[MUL]]
 ;
   %mul = mul i16 %lhs, %rhs
   ret i16 %mul
@@ -24,16 +29,24 @@ define i32 @smul24_i32(i32 %lhs, i32 %rhs) {
 ; SI-NEXT:    [[LHS24:%.*]] = ashr i32 [[SHL_LHS]], 8
 ; SI-NEXT:    [[LSHR_RHS:%.*]] = shl i32 [[RHS:%.*]], 8
 ; SI-NEXT:    [[RHS24:%.*]] = ashr i32 [[LHS]], 8
-; SI-NEXT:    [[TMP1:%.*]] = call i32 @llvm.amdgcn.mul.i24(i32 [[LHS24]], i32 [[RHS24]])
-; SI-NEXT:    ret i32 [[TMP1]]
+; SI-NEXT:    [[MUL:%.*]] = call i32 @llvm.amdgcn.mul.i24(i32 [[LHS24]], i32 [[RHS24]])
+; SI-NEXT:    ret i32 [[MUL]]
 ;
 ; VI-LABEL: @smul24_i32(
 ; VI-NEXT:    [[SHL_LHS:%.*]] = shl i32 [[LHS:%.*]], 8
 ; VI-NEXT:    [[LHS24:%.*]] = ashr i32 [[SHL_LHS]], 8
 ; VI-NEXT:    [[LSHR_RHS:%.*]] = shl i32 [[RHS:%.*]], 8
 ; VI-NEXT:    [[RHS24:%.*]] = ashr i32 [[LHS]], 8
-; VI-NEXT:    [[TMP1:%.*]] = call i32 @llvm.amdgcn.mul.i24(i32 [[LHS24]], i32 [[RHS24]])
-; VI-NEXT:    ret i32 [[TMP1]]
+; VI-NEXT:    [[MUL:%.*]] = call i32 @llvm.amdgcn.mul.i24(i32 [[LHS24]], i32 [[RHS24]])
+; VI-NEXT:    ret i32 [[MUL]]
+;
+; DISABLED-LABEL: @smul24_i32(
+; DISABLED-NEXT:    [[SHL_LHS:%.*]] = shl i32 [[LHS:%.*]], 8
+; DISABLED-NEXT:    [[LHS24:%.*]] = ashr i32 [[SHL_LHS]], 8
+; DISABLED-NEXT:    [[LSHR_RHS:%.*]] = shl i32 [[RHS:%.*]], 8
+; DISABLED-NEXT:    [[RHS24:%.*]] = ashr i32 [[LHS]], 8
+; DISABLED-NEXT:    [[MUL:%.*]] = mul i32 [[LHS24]], [[RHS24]]
+; DISABLED-NEXT:    ret i32 [[MUL]]
 ;
   %shl.lhs = shl i32 %lhs, 8
   %lhs24 = ashr i32 %shl.lhs, 8
@@ -56,8 +69,8 @@ define <2 x i32> @smul24_v2i32(<2 x i32> %lhs, <2 x i32> %rhs) {
 ; SI-NEXT:    [[TMP5:%.*]] = call i32 @llvm.amdgcn.mul.i24(i32 [[TMP1]], i32 [[TMP3]])
 ; SI-NEXT:    [[TMP6:%.*]] = call i32 @llvm.amdgcn.mul.i24(i32 [[TMP2]], i32 [[TMP4]])
 ; SI-NEXT:    [[TMP7:%.*]] = insertelement <2 x i32> undef, i32 [[TMP5]], i64 0
-; SI-NEXT:    [[TMP8:%.*]] = insertelement <2 x i32> [[TMP7]], i32 [[TMP6]], i64 1
-; SI-NEXT:    ret <2 x i32> [[TMP8]]
+; SI-NEXT:    [[MUL:%.*]] = insertelement <2 x i32> [[TMP7]], i32 [[TMP6]], i64 1
+; SI-NEXT:    ret <2 x i32> [[MUL]]
 ;
 ; VI-LABEL: @smul24_v2i32(
 ; VI-NEXT:    [[SHL_LHS:%.*]] = shl <2 x i32> [[LHS:%.*]], <i32 8, i32 8>
@@ -71,8 +84,16 @@ define <2 x i32> @smul24_v2i32(<2 x i32> %lhs, <2 x i32> %rhs) {
 ; VI-NEXT:    [[TMP5:%.*]] = call i32 @llvm.amdgcn.mul.i24(i32 [[TMP1]], i32 [[TMP3]])
 ; VI-NEXT:    [[TMP6:%.*]] = call i32 @llvm.amdgcn.mul.i24(i32 [[TMP2]], i32 [[TMP4]])
 ; VI-NEXT:    [[TMP7:%.*]] = insertelement <2 x i32> undef, i32 [[TMP5]], i64 0
-; VI-NEXT:    [[TMP8:%.*]] = insertelement <2 x i32> [[TMP7]], i32 [[TMP6]], i64 1
-; VI-NEXT:    ret <2 x i32> [[TMP8]]
+; VI-NEXT:    [[MUL:%.*]] = insertelement <2 x i32> [[TMP7]], i32 [[TMP6]], i64 1
+; VI-NEXT:    ret <2 x i32> [[MUL]]
+;
+; DISABLED-LABEL: @smul24_v2i32(
+; DISABLED-NEXT:    [[SHL_LHS:%.*]] = shl <2 x i32> [[LHS:%.*]], <i32 8, i32 8>
+; DISABLED-NEXT:    [[LHS24:%.*]] = ashr <2 x i32> [[SHL_LHS]], <i32 8, i32 8>
+; DISABLED-NEXT:    [[LSHR_RHS:%.*]] = shl <2 x i32> [[RHS:%.*]], <i32 8, i32 8>
+; DISABLED-NEXT:    [[RHS24:%.*]] = ashr <2 x i32> [[LHS]], <i32 8, i32 8>
+; DISABLED-NEXT:    [[MUL:%.*]] = mul <2 x i32> [[LHS24]], [[RHS24]]
+; DISABLED-NEXT:    ret <2 x i32> [[MUL]]
 ;
   %shl.lhs = shl <2 x i32> %lhs, <i32 8, i32 8>
   %lhs24 = ashr <2 x i32> %shl.lhs, <i32 8, i32 8>
@@ -86,14 +107,20 @@ define i32 @umul24_i32(i32 %lhs, i32 %rhs) {
 ; SI-LABEL: @umul24_i32(
 ; SI-NEXT:    [[LHS24:%.*]] = and i32 [[LHS:%.*]], 16777215
 ; SI-NEXT:    [[RHS24:%.*]] = and i32 [[RHS:%.*]], 16777215
-; SI-NEXT:    [[TMP1:%.*]] = call i32 @llvm.amdgcn.mul.u24(i32 [[LHS24]], i32 [[RHS24]])
-; SI-NEXT:    ret i32 [[TMP1]]
+; SI-NEXT:    [[MUL:%.*]] = call i32 @llvm.amdgcn.mul.u24(i32 [[LHS24]], i32 [[RHS24]])
+; SI-NEXT:    ret i32 [[MUL]]
 ;
 ; VI-LABEL: @umul24_i32(
 ; VI-NEXT:    [[LHS24:%.*]] = and i32 [[LHS:%.*]], 16777215
 ; VI-NEXT:    [[RHS24:%.*]] = and i32 [[RHS:%.*]], 16777215
-; VI-NEXT:    [[TMP1:%.*]] = call i32 @llvm.amdgcn.mul.u24(i32 [[LHS24]], i32 [[RHS24]])
-; VI-NEXT:    ret i32 [[TMP1]]
+; VI-NEXT:    [[MUL:%.*]] = call i32 @llvm.amdgcn.mul.u24(i32 [[LHS24]], i32 [[RHS24]])
+; VI-NEXT:    ret i32 [[MUL]]
+;
+; DISABLED-LABEL: @umul24_i32(
+; DISABLED-NEXT:    [[LHS24:%.*]] = and i32 [[LHS:%.*]], 16777215
+; DISABLED-NEXT:    [[RHS24:%.*]] = and i32 [[RHS:%.*]], 16777215
+; DISABLED-NEXT:    [[MUL:%.*]] = mul i32 [[LHS24]], [[RHS24]]
+; DISABLED-NEXT:    ret i32 [[MUL]]
 ;
   %lhs24 = and i32 %lhs, 16777215
   %rhs24 = and i32 %rhs, 16777215
@@ -112,8 +139,8 @@ define <2 x i32> @umul24_v2i32(<2 x i32> %lhs, <2 x i32> %rhs) {
 ; SI-NEXT:    [[TMP5:%.*]] = call i32 @llvm.amdgcn.mul.u24(i32 [[TMP1]], i32 [[TMP3]])
 ; SI-NEXT:    [[TMP6:%.*]] = call i32 @llvm.amdgcn.mul.u24(i32 [[TMP2]], i32 [[TMP4]])
 ; SI-NEXT:    [[TMP7:%.*]] = insertelement <2 x i32> undef, i32 [[TMP5]], i64 0
-; SI-NEXT:    [[TMP8:%.*]] = insertelement <2 x i32> [[TMP7]], i32 [[TMP6]], i64 1
-; SI-NEXT:    ret <2 x i32> [[TMP8]]
+; SI-NEXT:    [[MUL:%.*]] = insertelement <2 x i32> [[TMP7]], i32 [[TMP6]], i64 1
+; SI-NEXT:    ret <2 x i32> [[MUL]]
 ;
 ; VI-LABEL: @umul24_v2i32(
 ; VI-NEXT:    [[LHS24:%.*]] = and <2 x i32> [[LHS:%.*]], <i32 16777215, i32 16777215>
@@ -125,8 +152,14 @@ define <2 x i32> @umul24_v2i32(<2 x i32> %lhs, <2 x i32> %rhs) {
 ; VI-NEXT:    [[TMP5:%.*]] = call i32 @llvm.amdgcn.mul.u24(i32 [[TMP1]], i32 [[TMP3]])
 ; VI-NEXT:    [[TMP6:%.*]] = call i32 @llvm.amdgcn.mul.u24(i32 [[TMP2]], i32 [[TMP4]])
 ; VI-NEXT:    [[TMP7:%.*]] = insertelement <2 x i32> undef, i32 [[TMP5]], i64 0
-; VI-NEXT:    [[TMP8:%.*]] = insertelement <2 x i32> [[TMP7]], i32 [[TMP6]], i64 1
-; VI-NEXT:    ret <2 x i32> [[TMP8]]
+; VI-NEXT:    [[MUL:%.*]] = insertelement <2 x i32> [[TMP7]], i32 [[TMP6]], i64 1
+; VI-NEXT:    ret <2 x i32> [[MUL]]
+;
+; DISABLED-LABEL: @umul24_v2i32(
+; DISABLED-NEXT:    [[LHS24:%.*]] = and <2 x i32> [[LHS:%.*]], <i32 16777215, i32 16777215>
+; DISABLED-NEXT:    [[RHS24:%.*]] = and <2 x i32> [[RHS:%.*]], <i32 16777215, i32 16777215>
+; DISABLED-NEXT:    [[MUL:%.*]] = mul <2 x i32> [[LHS24]], [[RHS24]]
+; DISABLED-NEXT:    ret <2 x i32> [[MUL]]
 ;
   %lhs24 = and <2 x i32> %lhs, <i32 16777215, i32 16777215>
   %rhs24 = and <2 x i32> %rhs, <i32 16777215, i32 16777215>
@@ -143,8 +176,8 @@ define i64 @smul24_i64(i64 %lhs, i64 %rhs) {
 ; SI-NEXT:    [[TMP1:%.*]] = trunc i64 [[LHS24]] to i32
 ; SI-NEXT:    [[TMP2:%.*]] = trunc i64 [[RHS24]] to i32
 ; SI-NEXT:    [[TMP3:%.*]] = call i32 @llvm.amdgcn.mul.i24(i32 [[TMP1]], i32 [[TMP2]])
-; SI-NEXT:    [[TMP4:%.*]] = sext i32 [[TMP3]] to i64
-; SI-NEXT:    ret i64 [[TMP4]]
+; SI-NEXT:    [[MUL:%.*]] = sext i32 [[TMP3]] to i64
+; SI-NEXT:    ret i64 [[MUL]]
 ;
 ; VI-LABEL: @smul24_i64(
 ; VI-NEXT:    [[SHL_LHS:%.*]] = shl i64 [[LHS:%.*]], 40
@@ -154,8 +187,16 @@ define i64 @smul24_i64(i64 %lhs, i64 %rhs) {
 ; VI-NEXT:    [[TMP1:%.*]] = trunc i64 [[LHS24]] to i32
 ; VI-NEXT:    [[TMP2:%.*]] = trunc i64 [[RHS24]] to i32
 ; VI-NEXT:    [[TMP3:%.*]] = call i32 @llvm.amdgcn.mul.i24(i32 [[TMP1]], i32 [[TMP2]])
-; VI-NEXT:    [[TMP4:%.*]] = sext i32 [[TMP3]] to i64
-; VI-NEXT:    ret i64 [[TMP4]]
+; VI-NEXT:    [[MUL:%.*]] = sext i32 [[TMP3]] to i64
+; VI-NEXT:    ret i64 [[MUL]]
+;
+; DISABLED-LABEL: @smul24_i64(
+; DISABLED-NEXT:    [[SHL_LHS:%.*]] = shl i64 [[LHS:%.*]], 40
+; DISABLED-NEXT:    [[LHS24:%.*]] = ashr i64 [[SHL_LHS]], 40
+; DISABLED-NEXT:    [[LSHR_RHS:%.*]] = shl i64 [[RHS:%.*]], 40
+; DISABLED-NEXT:    [[RHS24:%.*]] = ashr i64 [[LHS]], 40
+; DISABLED-NEXT:    [[MUL:%.*]] = mul i64 [[LHS24]], [[RHS24]]
+; DISABLED-NEXT:    ret i64 [[MUL]]
 ;
   %shl.lhs = shl i64 %lhs, 40
   %lhs24 = ashr i64 %shl.lhs, 40
@@ -172,8 +213,8 @@ define i64 @umul24_i64(i64 %lhs, i64 %rhs) {
 ; SI-NEXT:    [[TMP1:%.*]] = trunc i64 [[LHS24]] to i32
 ; SI-NEXT:    [[TMP2:%.*]] = trunc i64 [[RHS24]] to i32
 ; SI-NEXT:    [[TMP3:%.*]] = call i32 @llvm.amdgcn.mul.u24(i32 [[TMP1]], i32 [[TMP2]])
-; SI-NEXT:    [[TMP4:%.*]] = zext i32 [[TMP3]] to i64
-; SI-NEXT:    ret i64 [[TMP4]]
+; SI-NEXT:    [[MUL:%.*]] = zext i32 [[TMP3]] to i64
+; SI-NEXT:    ret i64 [[MUL]]
 ;
 ; VI-LABEL: @umul24_i64(
 ; VI-NEXT:    [[LHS24:%.*]] = and i64 [[LHS:%.*]], 16777215
@@ -181,8 +222,14 @@ define i64 @umul24_i64(i64 %lhs, i64 %rhs) {
 ; VI-NEXT:    [[TMP1:%.*]] = trunc i64 [[LHS24]] to i32
 ; VI-NEXT:    [[TMP2:%.*]] = trunc i64 [[RHS24]] to i32
 ; VI-NEXT:    [[TMP3:%.*]] = call i32 @llvm.amdgcn.mul.u24(i32 [[TMP1]], i32 [[TMP2]])
-; VI-NEXT:    [[TMP4:%.*]] = zext i32 [[TMP3]] to i64
-; VI-NEXT:    ret i64 [[TMP4]]
+; VI-NEXT:    [[MUL:%.*]] = zext i32 [[TMP3]] to i64
+; VI-NEXT:    ret i64 [[MUL]]
+;
+; DISABLED-LABEL: @umul24_i64(
+; DISABLED-NEXT:    [[LHS24:%.*]] = and i64 [[LHS:%.*]], 16777215
+; DISABLED-NEXT:    [[RHS24:%.*]] = and i64 [[RHS:%.*]], 16777215
+; DISABLED-NEXT:    [[MUL:%.*]] = mul i64 [[LHS24]], [[RHS24]]
+; DISABLED-NEXT:    ret i64 [[MUL]]
 ;
   %lhs24 = and i64 %lhs, 16777215
   %rhs24 = and i64 %rhs, 16777215
@@ -199,8 +246,8 @@ define i31 @smul24_i31(i31 %lhs, i31 %rhs) {
 ; SI-NEXT:    [[TMP1:%.*]] = sext i31 [[LHS24]] to i32
 ; SI-NEXT:    [[TMP2:%.*]] = sext i31 [[RHS24]] to i32
 ; SI-NEXT:    [[TMP3:%.*]] = call i32 @llvm.amdgcn.mul.i24(i32 [[TMP1]], i32 [[TMP2]])
-; SI-NEXT:    [[TMP4:%.*]] = trunc i32 [[TMP3]] to i31
-; SI-NEXT:    ret i31 [[TMP4]]
+; SI-NEXT:    [[MUL:%.*]] = trunc i32 [[TMP3]] to i31
+; SI-NEXT:    ret i31 [[MUL]]
 ;
 ; VI-LABEL: @smul24_i31(
 ; VI-NEXT:    [[SHL_LHS:%.*]] = shl i31 [[LHS:%.*]], 7
@@ -210,8 +257,16 @@ define i31 @smul24_i31(i31 %lhs, i31 %rhs) {
 ; VI-NEXT:    [[TMP1:%.*]] = sext i31 [[LHS24]] to i32
 ; VI-NEXT:    [[TMP2:%.*]] = sext i31 [[RHS24]] to i32
 ; VI-NEXT:    [[TMP3:%.*]] = call i32 @llvm.amdgcn.mul.i24(i32 [[TMP1]], i32 [[TMP2]])
-; VI-NEXT:    [[TMP4:%.*]] = trunc i32 [[TMP3]] to i31
-; VI-NEXT:    ret i31 [[TMP4]]
+; VI-NEXT:    [[MUL:%.*]] = trunc i32 [[TMP3]] to i31
+; VI-NEXT:    ret i31 [[MUL]]
+;
+; DISABLED-LABEL: @smul24_i31(
+; DISABLED-NEXT:    [[SHL_LHS:%.*]] = shl i31 [[LHS:%.*]], 7
+; DISABLED-NEXT:    [[LHS24:%.*]] = ashr i31 [[SHL_LHS]], 7
+; DISABLED-NEXT:    [[LSHR_RHS:%.*]] = shl i31 [[RHS:%.*]], 7
+; DISABLED-NEXT:    [[RHS24:%.*]] = ashr i31 [[LHS]], 7
+; DISABLED-NEXT:    [[MUL:%.*]] = mul i31 [[LHS24]], [[RHS24]]
+; DISABLED-NEXT:    ret i31 [[MUL]]
 ;
   %shl.lhs = shl i31 %lhs, 7
   %lhs24 = ashr i31 %shl.lhs, 7
@@ -228,8 +283,8 @@ define i31 @umul24_i31(i31 %lhs, i31 %rhs) {
 ; SI-NEXT:    [[TMP1:%.*]] = zext i31 [[LHS24]] to i32
 ; SI-NEXT:    [[TMP2:%.*]] = zext i31 [[RHS24]] to i32
 ; SI-NEXT:    [[TMP3:%.*]] = call i32 @llvm.amdgcn.mul.u24(i32 [[TMP1]], i32 [[TMP2]])
-; SI-NEXT:    [[TMP4:%.*]] = trunc i32 [[TMP3]] to i31
-; SI-NEXT:    ret i31 [[TMP4]]
+; SI-NEXT:    [[MUL:%.*]] = trunc i32 [[TMP3]] to i31
+; SI-NEXT:    ret i31 [[MUL]]
 ;
 ; VI-LABEL: @umul24_i31(
 ; VI-NEXT:    [[LHS24:%.*]] = and i31 [[LHS:%.*]], 16777215
@@ -237,8 +292,14 @@ define i31 @umul24_i31(i31 %lhs, i31 %rhs) {
 ; VI-NEXT:    [[TMP1:%.*]] = zext i31 [[LHS24]] to i32
 ; VI-NEXT:    [[TMP2:%.*]] = zext i31 [[RHS24]] to i32
 ; VI-NEXT:    [[TMP3:%.*]] = call i32 @llvm.amdgcn.mul.u24(i32 [[TMP1]], i32 [[TMP2]])
-; VI-NEXT:    [[TMP4:%.*]] = trunc i32 [[TMP3]] to i31
-; VI-NEXT:    ret i31 [[TMP4]]
+; VI-NEXT:    [[MUL:%.*]] = trunc i32 [[TMP3]] to i31
+; VI-NEXT:    ret i31 [[MUL]]
+;
+; DISABLED-LABEL: @umul24_i31(
+; DISABLED-NEXT:    [[LHS24:%.*]] = and i31 [[LHS:%.*]], 16777215
+; DISABLED-NEXT:    [[RHS24:%.*]] = and i31 [[RHS:%.*]], 16777215
+; DISABLED-NEXT:    [[MUL:%.*]] = mul i31 [[LHS24]], [[RHS24]]
+; DISABLED-NEXT:    ret i31 [[MUL]]
 ;
   %lhs24 = and i31 %lhs, 16777215
   %rhs24 = and i31 %rhs, 16777215
@@ -263,8 +324,8 @@ define <2 x i31> @umul24_v2i31(<2 x i31> %lhs, <2 x i31> %rhs) {
 ; SI-NEXT:    [[TMP11:%.*]] = call i32 @llvm.amdgcn.mul.u24(i32 [[TMP9]], i32 [[TMP10]])
 ; SI-NEXT:    [[TMP12:%.*]] = trunc i32 [[TMP11]] to i31
 ; SI-NEXT:    [[TMP13:%.*]] = insertelement <2 x i31> undef, i31 [[TMP8]], i64 0
-; SI-NEXT:    [[TMP14:%.*]] = insertelement <2 x i31> [[TMP13]], i31 [[TMP12]], i64 1
-; SI-NEXT:    ret <2 x i31> [[TMP14]]
+; SI-NEXT:    [[MUL:%.*]] = insertelement <2 x i31> [[TMP13]], i31 [[TMP12]], i64 1
+; SI-NEXT:    ret <2 x i31> [[MUL]]
 ;
 ; VI-LABEL: @umul24_v2i31(
 ; VI-NEXT:    [[LHS24:%.*]] = and <2 x i31> [[LHS:%.*]], <i31 16777215, i31 16777215>
@@ -282,8 +343,14 @@ define <2 x i31> @umul24_v2i31(<2 x i31> %lhs, <2 x i31> %rhs) {
 ; VI-NEXT:    [[TMP11:%.*]] = call i32 @llvm.amdgcn.mul.u24(i32 [[TMP9]], i32 [[TMP10]])
 ; VI-NEXT:    [[TMP12:%.*]] = trunc i32 [[TMP11]] to i31
 ; VI-NEXT:    [[TMP13:%.*]] = insertelement <2 x i31> undef, i31 [[TMP8]], i64 0
-; VI-NEXT:    [[TMP14:%.*]] = insertelement <2 x i31> [[TMP13]], i31 [[TMP12]], i64 1
-; VI-NEXT:    ret <2 x i31> [[TMP14]]
+; VI-NEXT:    [[MUL:%.*]] = insertelement <2 x i31> [[TMP13]], i31 [[TMP12]], i64 1
+; VI-NEXT:    ret <2 x i31> [[MUL]]
+;
+; DISABLED-LABEL: @umul24_v2i31(
+; DISABLED-NEXT:    [[LHS24:%.*]] = and <2 x i31> [[LHS:%.*]], <i31 16777215, i31 16777215>
+; DISABLED-NEXT:    [[RHS24:%.*]] = and <2 x i31> [[RHS:%.*]], <i31 16777215, i31 16777215>
+; DISABLED-NEXT:    [[MUL:%.*]] = mul <2 x i31> [[LHS24]], [[RHS24]]
+; DISABLED-NEXT:    ret <2 x i31> [[MUL]]
 ;
   %lhs24 = and <2 x i31> %lhs, <i31 16777215, i31 16777215>
   %rhs24 = and <2 x i31> %rhs, <i31 16777215, i31 16777215>
@@ -310,8 +377,8 @@ define <2 x i31> @smul24_v2i31(<2 x i31> %lhs, <2 x i31> %rhs) {
 ; SI-NEXT:    [[TMP11:%.*]] = call i32 @llvm.amdgcn.mul.i24(i32 [[TMP9]], i32 [[TMP10]])
 ; SI-NEXT:    [[TMP12:%.*]] = trunc i32 [[TMP11]] to i31
 ; SI-NEXT:    [[TMP13:%.*]] = insertelement <2 x i31> undef, i31 [[TMP8]], i64 0
-; SI-NEXT:    [[TMP14:%.*]] = insertelement <2 x i31> [[TMP13]], i31 [[TMP12]], i64 1
-; SI-NEXT:    ret <2 x i31> [[TMP14]]
+; SI-NEXT:    [[MUL:%.*]] = insertelement <2 x i31> [[TMP13]], i31 [[TMP12]], i64 1
+; SI-NEXT:    ret <2 x i31> [[MUL]]
 ;
 ; VI-LABEL: @smul24_v2i31(
 ; VI-NEXT:    [[SHL_LHS:%.*]] = shl <2 x i31> [[LHS:%.*]], <i31 8, i31 8>
@@ -331,8 +398,16 @@ define <2 x i31> @smul24_v2i31(<2 x i31> %lhs, <2 x i31> %rhs) {
 ; VI-NEXT:    [[TMP11:%.*]] = call i32 @llvm.amdgcn.mul.i24(i32 [[TMP9]], i32 [[TMP10]])
 ; VI-NEXT:    [[TMP12:%.*]] = trunc i32 [[TMP11]] to i31
 ; VI-NEXT:    [[TMP13:%.*]] = insertelement <2 x i31> undef, i31 [[TMP8]], i64 0
-; VI-NEXT:    [[TMP14:%.*]] = insertelement <2 x i31> [[TMP13]], i31 [[TMP12]], i64 1
-; VI-NEXT:    ret <2 x i31> [[TMP14]]
+; VI-NEXT:    [[MUL:%.*]] = insertelement <2 x i31> [[TMP13]], i31 [[TMP12]], i64 1
+; VI-NEXT:    ret <2 x i31> [[MUL]]
+;
+; DISABLED-LABEL: @smul24_v2i31(
+; DISABLED-NEXT:    [[SHL_LHS:%.*]] = shl <2 x i31> [[LHS:%.*]], <i31 8, i31 8>
+; DISABLED-NEXT:    [[LHS24:%.*]] = ashr <2 x i31> [[SHL_LHS]], <i31 8, i31 8>
+; DISABLED-NEXT:    [[LSHR_RHS:%.*]] = shl <2 x i31> [[RHS:%.*]], <i31 8, i31 8>
+; DISABLED-NEXT:    [[RHS24:%.*]] = ashr <2 x i31> [[LHS]], <i31 8, i31 8>
+; DISABLED-NEXT:    [[MUL:%.*]] = mul <2 x i31> [[LHS24]], [[RHS24]]
+; DISABLED-NEXT:    ret <2 x i31> [[MUL]]
 ;
   %shl.lhs = shl <2 x i31> %lhs, <i31 8, i31 8>
   %lhs24 = ashr <2 x i31> %shl.lhs, <i31 8, i31 8>
@@ -351,8 +426,8 @@ define i33 @smul24_i33(i33 %lhs, i33 %rhs) {
 ; SI-NEXT:    [[TMP1:%.*]] = trunc i33 [[LHS24]] to i32
 ; SI-NEXT:    [[TMP2:%.*]] = trunc i33 [[RHS24]] to i32
 ; SI-NEXT:    [[TMP3:%.*]] = call i32 @llvm.amdgcn.mul.i24(i32 [[TMP1]], i32 [[TMP2]])
-; SI-NEXT:    [[TMP4:%.*]] = sext i32 [[TMP3]] to i33
-; SI-NEXT:    ret i33 [[TMP4]]
+; SI-NEXT:    [[MUL:%.*]] = sext i32 [[TMP3]] to i33
+; SI-NEXT:    ret i33 [[MUL]]
 ;
 ; VI-LABEL: @smul24_i33(
 ; VI-NEXT:    [[SHL_LHS:%.*]] = shl i33 [[LHS:%.*]], 9
@@ -362,8 +437,16 @@ define i33 @smul24_i33(i33 %lhs, i33 %rhs) {
 ; VI-NEXT:    [[TMP1:%.*]] = trunc i33 [[LHS24]] to i32
 ; VI-NEXT:    [[TMP2:%.*]] = trunc i33 [[RHS24]] to i32
 ; VI-NEXT:    [[TMP3:%.*]] = call i32 @llvm.amdgcn.mul.i24(i32 [[TMP1]], i32 [[TMP2]])
-; VI-NEXT:    [[TMP4:%.*]] = sext i32 [[TMP3]] to i33
-; VI-NEXT:    ret i33 [[TMP4]]
+; VI-NEXT:    [[MUL:%.*]] = sext i32 [[TMP3]] to i33
+; VI-NEXT:    ret i33 [[MUL]]
+;
+; DISABLED-LABEL: @smul24_i33(
+; DISABLED-NEXT:    [[SHL_LHS:%.*]] = shl i33 [[LHS:%.*]], 9
+; DISABLED-NEXT:    [[LHS24:%.*]] = ashr i33 [[SHL_LHS]], 9
+; DISABLED-NEXT:    [[LSHR_RHS:%.*]] = shl i33 [[RHS:%.*]], 9
+; DISABLED-NEXT:    [[RHS24:%.*]] = ashr i33 [[LHS]], 9
+; DISABLED-NEXT:    [[MUL:%.*]] = mul i33 [[LHS24]], [[RHS24]]
+; DISABLED-NEXT:    ret i33 [[MUL]]
 ;
   %shl.lhs = shl i33 %lhs, 9
   %lhs24 = ashr i33 %shl.lhs, 9
@@ -380,8 +463,8 @@ define i33 @umul24_i33(i33 %lhs, i33 %rhs) {
 ; SI-NEXT:    [[TMP1:%.*]] = trunc i33 [[LHS24]] to i32
 ; SI-NEXT:    [[TMP2:%.*]] = trunc i33 [[RHS24]] to i32
 ; SI-NEXT:    [[TMP3:%.*]] = call i32 @llvm.amdgcn.mul.u24(i32 [[TMP1]], i32 [[TMP2]])
-; SI-NEXT:    [[TMP4:%.*]] = zext i32 [[TMP3]] to i33
-; SI-NEXT:    ret i33 [[TMP4]]
+; SI-NEXT:    [[MUL:%.*]] = zext i32 [[TMP3]] to i33
+; SI-NEXT:    ret i33 [[MUL]]
 ;
 ; VI-LABEL: @umul24_i33(
 ; VI-NEXT:    [[LHS24:%.*]] = and i33 [[LHS:%.*]], 16777215
@@ -389,8 +472,14 @@ define i33 @umul24_i33(i33 %lhs, i33 %rhs) {
 ; VI-NEXT:    [[TMP1:%.*]] = trunc i33 [[LHS24]] to i32
 ; VI-NEXT:    [[TMP2:%.*]] = trunc i33 [[RHS24]] to i32
 ; VI-NEXT:    [[TMP3:%.*]] = call i32 @llvm.amdgcn.mul.u24(i32 [[TMP1]], i32 [[TMP2]])
-; VI-NEXT:    [[TMP4:%.*]] = zext i32 [[TMP3]] to i33
-; VI-NEXT:    ret i33 [[TMP4]]
+; VI-NEXT:    [[MUL:%.*]] = zext i32 [[TMP3]] to i33
+; VI-NEXT:    ret i33 [[MUL]]
+;
+; DISABLED-LABEL: @umul24_i33(
+; DISABLED-NEXT:    [[LHS24:%.*]] = and i33 [[LHS:%.*]], 16777215
+; DISABLED-NEXT:    [[RHS24:%.*]] = and i33 [[RHS:%.*]], 16777215
+; DISABLED-NEXT:    [[MUL:%.*]] = mul i33 [[LHS24]], [[RHS24]]
+; DISABLED-NEXT:    ret i33 [[MUL]]
 ;
   %lhs24 = and i33 %lhs, 16777215
   %rhs24 = and i33 %rhs, 16777215
@@ -414,6 +503,14 @@ define i32 @smul25_i32(i32 %lhs, i32 %rhs) {
 ; VI-NEXT:    [[RHS24:%.*]] = ashr i32 [[LHS]], 7
 ; VI-NEXT:    [[MUL:%.*]] = mul i32 [[LHS24]], [[RHS24]]
 ; VI-NEXT:    ret i32 [[MUL]]
+;
+; DISABLED-LABEL: @smul25_i32(
+; DISABLED-NEXT:    [[SHL_LHS:%.*]] = shl i32 [[LHS:%.*]], 7
+; DISABLED-NEXT:    [[LHS24:%.*]] = ashr i32 [[SHL_LHS]], 7
+; DISABLED-NEXT:    [[LSHR_RHS:%.*]] = shl i32 [[RHS:%.*]], 7
+; DISABLED-NEXT:    [[RHS24:%.*]] = ashr i32 [[LHS]], 7
+; DISABLED-NEXT:    [[MUL:%.*]] = mul i32 [[LHS24]], [[RHS24]]
+; DISABLED-NEXT:    ret i32 [[MUL]]
 ;
   %shl.lhs = shl i32 %lhs, 7
   %lhs24 = ashr i32 %shl.lhs, 7
@@ -435,6 +532,12 @@ define i32 @umul25_i32(i32 %lhs, i32 %rhs) {
 ; VI-NEXT:    [[RHS24:%.*]] = and i32 [[RHS:%.*]], 33554431
 ; VI-NEXT:    [[MUL:%.*]] = mul i32 [[LHS24]], [[RHS24]]
 ; VI-NEXT:    ret i32 [[MUL]]
+;
+; DISABLED-LABEL: @umul25_i32(
+; DISABLED-NEXT:    [[LHS24:%.*]] = and i32 [[LHS:%.*]], 33554431
+; DISABLED-NEXT:    [[RHS24:%.*]] = and i32 [[RHS:%.*]], 33554431
+; DISABLED-NEXT:    [[MUL:%.*]] = mul i32 [[LHS24]], [[RHS24]]
+; DISABLED-NEXT:    ret i32 [[MUL]]
 ;
   %lhs24 = and i32 %lhs, 33554431
   %rhs24 = and i32 %rhs, 33554431
@@ -461,8 +564,8 @@ define <2 x i33> @smul24_v2i33(<2 x i33> %lhs, <2 x i33> %rhs) {
 ; SI-NEXT:    [[TMP11:%.*]] = call i32 @llvm.amdgcn.mul.i24(i32 [[TMP9]], i32 [[TMP10]])
 ; SI-NEXT:    [[TMP12:%.*]] = sext i32 [[TMP11]] to i33
 ; SI-NEXT:    [[TMP13:%.*]] = insertelement <2 x i33> undef, i33 [[TMP8]], i64 0
-; SI-NEXT:    [[TMP14:%.*]] = insertelement <2 x i33> [[TMP13]], i33 [[TMP12]], i64 1
-; SI-NEXT:    ret <2 x i33> [[TMP14]]
+; SI-NEXT:    [[MUL:%.*]] = insertelement <2 x i33> [[TMP13]], i33 [[TMP12]], i64 1
+; SI-NEXT:    ret <2 x i33> [[MUL]]
 ;
 ; VI-LABEL: @smul24_v2i33(
 ; VI-NEXT:    [[SHL_LHS:%.*]] = shl <2 x i33> [[LHS:%.*]], <i33 9, i33 9>
@@ -482,8 +585,16 @@ define <2 x i33> @smul24_v2i33(<2 x i33> %lhs, <2 x i33> %rhs) {
 ; VI-NEXT:    [[TMP11:%.*]] = call i32 @llvm.amdgcn.mul.i24(i32 [[TMP9]], i32 [[TMP10]])
 ; VI-NEXT:    [[TMP12:%.*]] = sext i32 [[TMP11]] to i33
 ; VI-NEXT:    [[TMP13:%.*]] = insertelement <2 x i33> undef, i33 [[TMP8]], i64 0
-; VI-NEXT:    [[TMP14:%.*]] = insertelement <2 x i33> [[TMP13]], i33 [[TMP12]], i64 1
-; VI-NEXT:    ret <2 x i33> [[TMP14]]
+; VI-NEXT:    [[MUL:%.*]] = insertelement <2 x i33> [[TMP13]], i33 [[TMP12]], i64 1
+; VI-NEXT:    ret <2 x i33> [[MUL]]
+;
+; DISABLED-LABEL: @smul24_v2i33(
+; DISABLED-NEXT:    [[SHL_LHS:%.*]] = shl <2 x i33> [[LHS:%.*]], <i33 9, i33 9>
+; DISABLED-NEXT:    [[LHS24:%.*]] = ashr <2 x i33> [[SHL_LHS]], <i33 9, i33 9>
+; DISABLED-NEXT:    [[LSHR_RHS:%.*]] = shl <2 x i33> [[RHS:%.*]], <i33 9, i33 9>
+; DISABLED-NEXT:    [[RHS24:%.*]] = ashr <2 x i33> [[LHS]], <i33 9, i33 9>
+; DISABLED-NEXT:    [[MUL:%.*]] = mul <2 x i33> [[LHS24]], [[RHS24]]
+; DISABLED-NEXT:    ret <2 x i33> [[MUL]]
 ;
   %shl.lhs = shl <2 x i33> %lhs, <i33 9, i33 9>
   %lhs24 = ashr <2 x i33> %shl.lhs, <i33 9, i33 9>
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-inline.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-inline.ll
index c2f1836f4..243522e28 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-inline.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-inline.ll
@@ -28,8 +28,15 @@ if.end:                                           ; preds = %if.then, %entry
 define coldcc void @foo_private_ptr2(float addrspace(5)* nocapture %p1, float addrspace(5)* nocapture %p2) {
 entry:
   %tmp1 = load float, float addrspace(5)* %p1, align 4
+  %cmp = fcmp ogt float %tmp1, 1.000000e+00
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
   %div = fdiv float 2.000000e+00, %tmp1
   store float %div, float addrspace(5)* %p2, align 4
+  br label %if.end
+
+if.end:
   ret void
 }
 
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-mul24-knownbits.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-mul24-knownbits.ll
new file mode 100644
index 000000000..1f660df59
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-mul24-knownbits.ll
@@ -0,0 +1,34 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+
+; RUN: llc -mtriple amdgcn-amd-amdhsa -mcpu=gfx900 < %s | FileCheck --check-prefix=GCN %s
+define weak_odr amdgpu_kernel void @test_mul24_knownbits_kernel(float addrspace(1)* %p) #4 {
+; GCN-LABEL: test_mul24_knownbits_kernel:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    v_and_b32_e32 v0, 3, v0
+; GCN-NEXT:    v_mul_i32_i24_e32 v0, 0xfffffb, v0
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffffffe0, v0
+; GCN-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; GCN-NEXT:    v_lshlrev_b64 v[0:1], 2, v[0:1]
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v2, s1
+; GCN-NEXT:    v_add_co_u32_e32 v0, vcc, s0, v0
+; GCN-NEXT:    v_addc_co_u32_e32 v1, vcc, v2, v1, vcc
+; GCN-NEXT:    v_mov_b32_e32 v2, 0
+; GCN-NEXT:    global_store_dword v[0:1], v2, off
+; GCN-NEXT:    s_endpgm
+entry:
+  %0 = tail call i32 @llvm.amdgcn.workitem.id.x() #28, !range !4
+  %tid = and i32 %0, 3
+  %1 = mul nsw i32 %tid, -5
+  %v1 = and i32 %1, -32
+  %v2 = sext i32 %v1 to i64
+  %v3 = getelementptr inbounds float, float addrspace(1)* %p, i64 %v2
+  store float 0.000, float addrspace(1)* %v3, align 4
+  ret void
+}
+
+; Function Attrs: nounwind readnone speculatable
+declare i32 @llvm.amdgcn.workitem.id.x() #20
+
+!4 = !{i32 0, i32 1024}
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-unroll-threshold.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-unroll-threshold.ll
new file mode 100644
index 000000000..4da11ca77
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-unroll-threshold.ll
@@ -0,0 +1,52 @@
+; RUN: opt < %s -S -mtriple=amdgcn-- -basicaa -loop-unroll | FileCheck %s
+
+; Check that the loop in unroll_default is not fully unrolled using the default
+; unroll threshold
+; CHECK-LABEL: @unroll_default
+; CHECK: entry:
+; CHECK: br i1 %cmp
+; CHECK: ret void
+
+; Check that the same loop in unroll_full is fully unrolled when the default
+; unroll threshold is increased by use of the amdgpu-unroll-threshold attribute
+; CHECK-LABEL: @unroll_full
+; CHECK: entry:
+; CHECK-NOT: br i1 %cmp
+; CHECK: ret void
+
+@in = internal unnamed_addr global i32* null, align 8
+@out = internal unnamed_addr global i32* null, align 8
+
+define void @unroll_default() {
+entry:
+  br label %do.body
+
+do.body:                                          ; preds = %entry
+  %i.0 = phi i32 [ 0, %entry ], [ %inc, %do.body ]
+  %v1 = load i64, i64* bitcast (i32** @in to i64*), align 8
+  store i64 %v1, i64* bitcast (i32** @out to i64*), align 8
+  %inc = add nsw i32 %i.0, 1
+  %cmp = icmp slt i32 %inc, 100
+  br i1 %cmp, label %do.body, label %do.end
+
+do.end:                                           ; preds = %do.body
+  ret void
+}
+
+define void @unroll_full() #0 {
+entry:
+  br label %do.body
+
+do.body:                                          ; preds = %entry
+  %i.0 = phi i32 [ 0, %entry ], [ %inc, %do.body ]
+  %v1 = load i64, i64* bitcast (i32** @in to i64*), align 8
+  store i64 %v1, i64* bitcast (i32** @out to i64*), align 8
+  %inc = add nsw i32 %i.0, 1
+  %cmp = icmp slt i32 %inc, 100
+  br i1 %cmp, label %do.body, label %do.end
+
+do.end:                                           ; preds = %do.body
+  ret void
+}
+
+attributes #0 = { "amdgpu-unroll-threshold"="1000" }
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu.private-memory.ll b/llvm/test/CodeGen/AMDGPU/amdgpu.private-memory.ll
index 199a96c64..d26f51302 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu.private-memory.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu.private-memory.ll
@@ -412,7 +412,7 @@ define amdgpu_kernel void @ptrtoint(i32 addrspace(1)* %out, i32 %a, i32 %b) #0 {
 ; OPT-LABEL: @pointer_typed_alloca(
 ; OPT:  getelementptr inbounds [256 x i32 addrspace(1)*], [256 x i32 addrspace(1)*] addrspace(3)* @pointer_typed_alloca.A.addr, i32 0, i32 %{{[0-9]+}}
 ; OPT: load i32 addrspace(1)*, i32 addrspace(1)* addrspace(3)* %{{[0-9]+}}, align 4
-define amdgpu_kernel void @pointer_typed_alloca(i32 addrspace(1)* %A) {
+define amdgpu_kernel void @pointer_typed_alloca(i32 addrspace(1)* %A) #1 {
 entry:
   %A.addr = alloca i32 addrspace(1)*, align 4, addrspace(5)
   store i32 addrspace(1)* %A, i32 addrspace(1)* addrspace(5)* %A.addr, align 4
@@ -556,7 +556,8 @@ entry:
   ret void
 }
 
-attributes #0 = { nounwind "amdgpu-waves-per-eu"="1,2" }
+attributes #0 = { nounwind "amdgpu-waves-per-eu"="1,2" "amdgpu-flat-work-group-size"="1,256" }
+attributes #1 = { nounwind "amdgpu-flat-work-group-size"="1,256" }
 
 ; HSAOPT: !0 = !{}
 ; HSAOPT: !1 = !{i32 0, i32 257}
diff --git a/llvm/test/CodeGen/AMDGPU/annotate-kernel-features-hsa.ll b/llvm/test/CodeGen/AMDGPU/annotate-kernel-features-hsa.ll
index 5a9d72d36..7efd00719 100644
--- a/llvm/test/CodeGen/AMDGPU/annotate-kernel-features-hsa.ll
+++ b/llvm/test/CodeGen/AMDGPU/annotate-kernel-features-hsa.ll
@@ -12,6 +12,9 @@ declare i8 addrspace(4)* @llvm.amdgcn.dispatch.ptr() #0
 declare i8 addrspace(4)* @llvm.amdgcn.queue.ptr() #0
 declare i8 addrspace(4)* @llvm.amdgcn.kernarg.segment.ptr() #0
 
+declare i1 @llvm.amdgcn.is.shared(i8* nocapture) #2
+declare i1 @llvm.amdgcn.is.private(i8* nocapture) #2
+
 ; HSA: define amdgpu_kernel void @use_tgid_x(i32 addrspace(1)* %ptr) #1 {
 define amdgpu_kernel void @use_tgid_x(i32 addrspace(1)* %ptr) #1 {
   %val = call i32 @llvm.amdgcn.workgroup.id.x()
@@ -231,6 +234,22 @@ define amdgpu_kernel void @use_flat_to_constant_addrspacecast(i32* %ptr) #1 {
   ret void
 }
 
+; HSA: define amdgpu_kernel void @use_is_shared(i8* %ptr) #11 {
+define amdgpu_kernel void @use_is_shared(i8* %ptr) #1 {
+  %is.shared = call i1 @llvm.amdgcn.is.shared(i8* %ptr)
+  %ext = zext i1 %is.shared to i32
+  store i32 %ext, i32 addrspace(1)* undef
+  ret void
+}
+
+; HSA: define amdgpu_kernel void @use_is_private(i8* %ptr) #11 {
+define amdgpu_kernel void @use_is_private(i8* %ptr) #1 {
+  %is.private = call i1 @llvm.amdgcn.is.private(i8* %ptr)
+  %ext = zext i1 %is.private to i32
+  store i32 %ext, i32 addrspace(1)* undef
+  ret void
+}
+
 attributes #0 = { nounwind readnone speculatable }
 attributes #1 = { nounwind }
 
diff --git a/llvm/test/CodeGen/AMDGPU/array-ptr-calc-i32.ll b/llvm/test/CodeGen/AMDGPU/array-ptr-calc-i32.ll
index 8e7f40abc..b301384b7 100644
--- a/llvm/test/CodeGen/AMDGPU/array-ptr-calc-i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/array-ptr-calc-i32.ll
@@ -43,7 +43,7 @@ define amdgpu_kernel void @test_private_array_ptr_calc(i32 addrspace(1)* noalias
   ret void
 }
 
-attributes #0 = { nounwind "amdgpu-waves-per-eu"="1,1" }
+attributes #0 = { nounwind "amdgpu-waves-per-eu"="1,1" "amdgpu-flat-work-group-size"="1,256" }
 attributes #1 = { nounwind readnone }
 attributes #2 = { nounwind convergent }
 
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_buffer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_buffer.ll
index 88824aa64..f358dc259 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_buffer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_buffer.ll
@@ -1,6 +1,8 @@
-; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX7LESS %s
-; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=tonga -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -amdgpu-dpp-combine=false -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX8MORE %s
-; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=gfx900 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -amdgpu-dpp-combine=false -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX8MORE %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN64,GFX7LESS %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=tonga -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -amdgpu-dpp-combine=false -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN64,GFX8MORE,GFX8MORE64,GFX89 %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=gfx900 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -amdgpu-dpp-combine=false -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN64,GFX8MORE,GFX8MORE64,GFX89 %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=gfx1010 -mattr=-wavefrontsize32,+wavefrontsize64 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -amdgpu-dpp-combine=false -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN64,GFX8MORE,GFX8MORE64 %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=gfx1010 -mattr=+wavefrontsize32,-wavefrontsize64 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -amdgpu-dpp-combine=false -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN32,GFX8MORE,GFX8MORE32 %s
 
 declare i32 @llvm.amdgcn.workitem.id.x()
 declare i32 @llvm.amdgcn.buffer.atomic.add(i32, <4 x i32>, i32, i32, i1)
@@ -9,11 +11,13 @@ declare i32 @llvm.amdgcn.buffer.atomic.sub(i32, <4 x i32>, i32, i32, i1)
 ; Show that what the atomic optimization pass will do for raw buffers.
 
 ; GCN-LABEL: add_i32_constant:
-; GCN: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
-; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_lo:[0-9]+]], s[[exec_lo]], 0
-; GCN: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_hi:[0-9]+]], s[[exec_hi]], v[[mbcnt_lo]]
-; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc, 0, v[[mbcnt_hi]]
-; GCN: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
+; GCN32: v_cmp_ne_u32_e64 s[[exec_lo:[0-9]+]], 1, 0
+; GCN64: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
+; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt:[0-9]+]], s[[exec_lo]], 0
+; GCN64: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt]], s[[exec_hi]], v[[mbcnt]]
+; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc{{(_lo)?}}, 0, v[[mbcnt]]
+; GCN32: s_bcnt1_i32_b32 s[[popcount:[0-9]+]], s[[exec_lo]]
+; GCN64: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
 ; GCN: v_mul_u32_u24{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[popcount]], 5
 ; GCN: buffer_atomic_add v[[value]]
 define amdgpu_kernel void @add_i32_constant(i32 addrspace(1)* %out, <4 x i32> %inout) {
@@ -24,11 +28,13 @@ entry:
 }
 
 ; GCN-LABEL: add_i32_uniform:
-; GCN: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
-; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_lo:[0-9]+]], s[[exec_lo]], 0
-; GCN: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_hi:[0-9]+]], s[[exec_hi]], v[[mbcnt_lo]]
-; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc, 0, v[[mbcnt_hi]]
-; GCN: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
+; GCN32: v_cmp_ne_u32_e64 s[[exec_lo:[0-9]+]], 1, 0
+; GCN64: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
+; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt:[0-9]+]], s[[exec_lo]], 0
+; GCN64: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt]], s[[exec_hi]], v[[mbcnt]]
+; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc{{(_lo)?}}, 0, v[[mbcnt]]
+; GCN32: s_bcnt1_i32_b32 s[[popcount:[0-9]+]], s[[exec_lo]]
+; GCN64: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
 ; GCN: s_mul_i32 s[[scalar_value:[0-9]+]], s{{[0-9]+}}, s[[popcount]]
 ; GCN: v_mov_b32{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[scalar_value]]
 ; GCN: buffer_atomic_add v[[value]]
@@ -44,15 +50,15 @@ entry:
 ; GFX7LESS-NOT: v_mbcnt_hi_u32_b32
 ; GFX7LESS-NOT: s_bcnt1_i32_b64
 ; GFX7LESS: buffer_atomic_add v{{[0-9]+}}
-; GFX8MORE: v_mov_b32_dpp v[[wave_shr1:[0-9]+]], v{{[0-9]+}} wave_shr:1 row_mask:0xf bank_mask:0xf
-; GFX8MORE: v_mov_b32_dpp v{{[0-9]+}}, v[[wave_shr1]] row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX8MORE: v_mov_b32_dpp v{{[0-9]+}}, v[[wave_shr1]] row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX8MORE: v_mov_b32_dpp v{{[0-9]+}}, v[[wave_shr1]] row_shr:3 row_mask:0xf bank_mask:0xf
-; GFX8MORE: v_mov_b32_dpp v{{[0-9]+}}, v{{[0-9]+}} row_shr:4 row_mask:0xf bank_mask:0xe
-; GFX8MORE: v_mov_b32_dpp v{{[0-9]+}}, v{{[0-9]+}} row_shr:8 row_mask:0xf bank_mask:0xc
-; GFX8MORE: v_mov_b32_dpp v{{[0-9]+}}, v{{[0-9]+}} row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX8MORE: v_mov_b32_dpp v{{[0-9]+}}, v{{[0-9]+}} row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX8MORE: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 63
+; GFX8MORE: v_mov_b32_dpp v{{[0-9]+}}, v{{[0-9]+}} row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX8MORE: v_mov_b32_dpp v{{[0-9]+}}, v{{[0-9]+}} row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX8MORE: v_mov_b32_dpp v{{[0-9]+}}, v{{[0-9]+}} row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX8MORE: v_mov_b32_dpp v{{[0-9]+}}, v{{[0-9]+}} row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX89:    v_mov_b32_dpp v{{[0-9]+}}, v{{[0-9]+}} row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX89:    v_mov_b32_dpp v{{[0-9]+}}, v{{[0-9]+}} row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX8MORE32: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 31
+; GFX8MORE64: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 63
+; GFX89: v_mov_b32_dpp v{{[0-9]+}}, v{{[0-9]+}} wave_shr:1 row_mask:0xf bank_mask:0xf
 ; GFX8MORE: v_mov_b32{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[scalar_value]]
 ; GFX8MORE: buffer_atomic_add v[[value]]
 define amdgpu_kernel void @add_i32_varying_vdata(i32 addrspace(1)* %out, <4 x i32> %inout) {
@@ -77,11 +83,13 @@ entry:
 }
 
 ; GCN-LABEL: sub_i32_constant:
-; GCN: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
-; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_lo:[0-9]+]], s[[exec_lo]], 0
-; GCN: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_hi:[0-9]+]], s[[exec_hi]], v[[mbcnt_lo]]
-; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc, 0, v[[mbcnt_hi]]
-; GCN: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
+; GCN32: v_cmp_ne_u32_e64 s[[exec_lo:[0-9]+]], 1, 0
+; GCN64: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
+; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt:[0-9]+]], s[[exec_lo]], 0
+; GCN64: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt]], s[[exec_hi]], v[[mbcnt]]
+; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc{{(_lo)?}}, 0, v[[mbcnt]]
+; GCN32: s_bcnt1_i32_b32 s[[popcount:[0-9]+]], s[[exec_lo]]
+; GCN64: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
 ; GCN: v_mul_u32_u24{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[popcount]], 5
 ; GCN: buffer_atomic_sub v[[value]]
 define amdgpu_kernel void @sub_i32_constant(i32 addrspace(1)* %out, <4 x i32> %inout) {
@@ -92,11 +100,13 @@ entry:
 }
 
 ; GCN-LABEL: sub_i32_uniform:
-; GCN: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
-; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_lo:[0-9]+]], s[[exec_lo]], 0
-; GCN: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_hi:[0-9]+]], s[[exec_hi]], v[[mbcnt_lo]]
-; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc, 0, v[[mbcnt_hi]]
-; GCN: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
+; GCN32: v_cmp_ne_u32_e64 s[[exec_lo:[0-9]+]], 1, 0
+; GCN64: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
+; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt:[0-9]+]], s[[exec_lo]], 0
+; GCN64: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt]], s[[exec_hi]], v[[mbcnt]]
+; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc{{(_lo)?}}, 0, v[[mbcnt]]
+; GCN32: s_bcnt1_i32_b32 s[[popcount:[0-9]+]], s[[exec_lo]]
+; GCN64: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
 ; GCN: s_mul_i32 s[[scalar_value:[0-9]+]], s{{[0-9]+}}, s[[popcount]]
 ; GCN: v_mov_b32{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[scalar_value]]
 ; GCN: buffer_atomic_sub v[[value]]
@@ -112,15 +122,15 @@ entry:
 ; GFX7LESS-NOT: v_mbcnt_hi_u32_b32
 ; GFX7LESS-NOT: s_bcnt1_i32_b64
 ; GFX7LESS: buffer_atomic_sub v{{[0-9]+}}
-; GFX8MORE: v_mov_b32_dpp v[[wave_shr1:[0-9]+]], v{{[0-9]+}} wave_shr:1 row_mask:0xf bank_mask:0xf
-; GFX8MORE: v_mov_b32_dpp v{{[0-9]+}}, v[[wave_shr1]] row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX8MORE: v_mov_b32_dpp v{{[0-9]+}}, v[[wave_shr1]] row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX8MORE: v_mov_b32_dpp v{{[0-9]+}}, v[[wave_shr1]] row_shr:3 row_mask:0xf bank_mask:0xf
-; GFX8MORE: v_mov_b32_dpp v{{[0-9]+}}, v{{[0-9]+}} row_shr:4 row_mask:0xf bank_mask:0xe
-; GFX8MORE: v_mov_b32_dpp v{{[0-9]+}}, v{{[0-9]+}} row_shr:8 row_mask:0xf bank_mask:0xc
-; GFX8MORE: v_mov_b32_dpp v{{[0-9]+}}, v{{[0-9]+}} row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX8MORE: v_mov_b32_dpp v{{[0-9]+}}, v{{[0-9]+}} row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX8MORE: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 63
+; GFX8MORE: v_mov_b32_dpp v{{[0-9]+}}, v{{[0-9]+}} row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX8MORE: v_mov_b32_dpp v{{[0-9]+}}, v{{[0-9]+}} row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX8MORE: v_mov_b32_dpp v{{[0-9]+}}, v{{[0-9]+}} row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX8MORE: v_mov_b32_dpp v{{[0-9]+}}, v{{[0-9]+}} row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX89:    v_mov_b32_dpp v{{[0-9]+}}, v{{[0-9]+}} row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX89:    v_mov_b32_dpp v{{[0-9]+}}, v{{[0-9]+}} row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX8MORE32: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 31
+; GFX8MORE64: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 63
+; GFX89: v_mov_b32_dpp v{{[0-9]+}}, v{{[0-9]+}} wave_shr:1 row_mask:0xf bank_mask:0xf
 ; GFX8MORE: v_mov_b32{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[scalar_value]]
 ; GFX8MORE: buffer_atomic_sub v[[value]]
 define amdgpu_kernel void @sub_i32_varying_vdata(i32 addrspace(1)* %out, <4 x i32> %inout) {
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
index 6f52a5f06..24fe35797 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
@@ -1,17 +1,21 @@
-; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX7LESS %s
-; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=tonga -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX8MORE %s
-; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=gfx900 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX8MORE %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN64,GFX7LESS %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=tonga -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN64,GFX8MORE,GFX8MORE64,DPPCOMB %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=gfx900 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN64,GFX8MORE,GFX8MORE64,DPPCOMB %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=gfx1010 -mattr=-wavefrontsize32,+wavefrontsize64 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN64,GFX8MORE,GFX8MORE64 %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=gfx1010 -mattr=+wavefrontsize32,-wavefrontsize64 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN32,GFX8MORE,GFX8MORE32 %s
 
 declare i32 @llvm.amdgcn.workitem.id.x()
 
 ; Show that what the atomic optimization pass will do for global pointers.
 
 ; GCN-LABEL: add_i32_constant:
-; GCN: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
-; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_lo:[0-9]+]], s[[exec_lo]], 0
-; GCN: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_hi:[0-9]+]], s[[exec_hi]], v[[mbcnt_lo]]
-; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc, 0, v[[mbcnt_hi]]
-; GCN: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
+; GCN32: v_cmp_ne_u32_e64 s[[exec_lo:[0-9]+]], 1, 0
+; GCN64: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
+; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt:[0-9]+]], s[[exec_lo]], 0
+; GCN64: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt]], s[[exec_hi]], v[[mbcnt]]
+; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc{{(_lo)?}}, 0, v[[mbcnt]]
+; GCN32: s_bcnt1_i32_b32 s[[popcount:[0-9]+]], s[[exec_lo]]
+; GCN64: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
 ; GCN: v_mul_u32_u24{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[popcount]], 5
 ; GCN: {{flat|buffer|global}}_atomic_add v[[value]]
 define amdgpu_kernel void @add_i32_constant(i32 addrspace(1)* %out, i32 addrspace(1)* %inout) {
@@ -22,11 +26,13 @@ entry:
 }
 
 ; GCN-LABEL: add_i32_uniform:
-; GCN: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
-; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_lo:[0-9]+]], s[[exec_lo]], 0
-; GCN: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_hi:[0-9]+]], s[[exec_hi]], v[[mbcnt_lo]]
-; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc, 0, v[[mbcnt_hi]]
-; GCN: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
+; GCN32: v_cmp_ne_u32_e64 s[[exec_lo:[0-9]+]], 1, 0
+; GCN64: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
+; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt:[0-9]+]], s[[exec_lo]], 0
+; GCN64: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt]], s[[exec_hi]], v[[mbcnt]]
+; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc{{(_lo)?}}, 0, v[[mbcnt]]
+; GCN32: s_bcnt1_i32_b32 s[[popcount:[0-9]+]], s[[exec_lo]]
+; GCN64: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
 ; GCN: s_mul_i32 s[[scalar_value:[0-9]+]], s{{[0-9]+}}, s[[popcount]]
 ; GCN: v_mov_b32{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[scalar_value]]
 ; GCN: {{flat|buffer|global}}_atomic_add v[[value]]
@@ -42,9 +48,10 @@ entry:
 ; GFX7LESS-NOT: v_mbcnt_hi_u32_b32
 ; GFX7LESS-NOT: s_bcnt1_i32_b64
 ; GFX7LESS: buffer_atomic_add v{{[0-9]+}}
-; GFX8MORE: v_add_u32_dpp
-; GFX8MORE: v_add_u32_dpp
-; GFX8MORE: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 63
+; DPPCOMB: v_add_u32_dpp
+; DPPCOMB: v_add_u32_dpp
+; GFX8MORE32: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 31
+; GFX8MORE64: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 63
 ; GFX8MORE: v_mov_b32{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[scalar_value]]
 ; GFX8MORE: buffer_atomic_add v[[value]]
 define amdgpu_kernel void @add_i32_varying(i32 addrspace(1)* %out, i32 addrspace(1)* %inout) {
@@ -56,11 +63,13 @@ entry:
 }
 
 ; GCN-LABEL: add_i64_constant:
-; GCN: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
-; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_lo:[0-9]+]], s[[exec_lo]], 0
-; GCN: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_hi:[0-9]+]], s[[exec_hi]], v[[mbcnt_lo]]
-; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc, 0, v[[mbcnt_hi]]
-; GCN: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
+; GCN32: v_cmp_ne_u32_e64 s[[exec_lo:[0-9]+]], 1, 0
+; GCN64: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
+; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt:[0-9]+]], s[[exec_lo]], 0
+; GCN64: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt]], s[[exec_hi]], v[[mbcnt]]
+; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc{{(_lo)?}}, 0, v[[mbcnt]]
+; GCN32: s_bcnt1_i32_b32 s[[popcount:[0-9]+]], s[[exec_lo]]
+; GCN64: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
 ; GCN: v_mul_hi_u32_u24{{(_e[0-9]+)?}} v[[value_hi:[0-9]+]], s[[popcount]], 5
 ; GCN: v_mul_u32_u24{{(_e[0-9]+)?}} v[[value_lo:[0-9]+]], s[[popcount]], 5
 ; GCN: {{flat|buffer|global}}_atomic_add_x2 v{{\[}}[[value_lo]]:[[value_hi]]{{\]}}
@@ -72,11 +81,13 @@ entry:
 }
 
 ; GCN-LABEL: add_i64_uniform:
-; GCN: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
-; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_lo:[0-9]+]], s[[exec_lo]], 0
-; GCN: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_hi:[0-9]+]], s[[exec_hi]], v[[mbcnt_lo]]
-; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc, 0, v[[mbcnt_hi]]
-; GCN: s_bcnt1_i32_b64 s{{[0-9]+}}, s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
+; GCN32: v_cmp_ne_u32_e64 s[[exec_lo:[0-9]+]], 1, 0
+; GCN64: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
+; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt:[0-9]+]], s[[exec_lo]], 0
+; GCN64: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt]], s[[exec_hi]], v[[mbcnt]]
+; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc{{(_lo)?}}, 0, v[[mbcnt]]
+; GCN32: s_bcnt1_i32_b32 s{{[0-9]+}}, s[[exec_lo]]
+; GCN64: s_bcnt1_i32_b64 s{{[0-9]+}}, s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
 ; GCN: {{flat|buffer|global}}_atomic_add_x2 v{{\[}}{{[0-9]+}}:{{[0-9]+}}{{\]}}
 define amdgpu_kernel void @add_i64_uniform(i64 addrspace(1)* %out, i64 addrspace(1)* %inout, i64 %additive) {
 entry:
@@ -100,11 +111,13 @@ entry:
 }
 
 ; GCN-LABEL: sub_i32_constant:
-; GCN: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
-; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_lo:[0-9]+]], s[[exec_lo]], 0
-; GCN: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_hi:[0-9]+]], s[[exec_hi]], v[[mbcnt_lo]]
-; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc, 0, v[[mbcnt_hi]]
-; GCN: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
+; GCN32: v_cmp_ne_u32_e64 s[[exec_lo:[0-9]+]], 1, 0
+; GCN64: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
+; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt:[0-9]+]], s[[exec_lo]], 0
+; GCN64: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt]], s[[exec_hi]], v[[mbcnt]]
+; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc{{(_lo)?}}, 0, v[[mbcnt]]
+; GCN32: s_bcnt1_i32_b32 s[[popcount:[0-9]+]], s[[exec_lo]]
+; GCN64: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
 ; GCN: v_mul_u32_u24{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[popcount]], 5
 ; GCN: {{flat|buffer|global}}_atomic_sub v[[value]]
 define amdgpu_kernel void @sub_i32_constant(i32 addrspace(1)* %out, i32 addrspace(1)* %inout) {
@@ -115,11 +128,13 @@ entry:
 }
 
 ; GCN-LABEL: sub_i32_uniform:
-; GCN: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
-; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_lo:[0-9]+]], s[[exec_lo]], 0
-; GCN: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_hi:[0-9]+]], s[[exec_hi]], v[[mbcnt_lo]]
-; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc, 0, v[[mbcnt_hi]]
-; GCN: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
+; GCN32: v_cmp_ne_u32_e64 s[[exec_lo:[0-9]+]], 1, 0
+; GCN64: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
+; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt:[0-9]+]], s[[exec_lo]], 0
+; GCN64: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt]], s[[exec_hi]], v[[mbcnt]]
+; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc{{(_lo)?}}, 0, v[[mbcnt]]
+; GCN32: s_bcnt1_i32_b32 s[[popcount:[0-9]+]], s[[exec_lo]]
+; GCN64: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
 ; GCN: s_mul_i32 s[[scalar_value:[0-9]+]], s{{[0-9]+}}, s[[popcount]]
 ; GCN: v_mov_b32{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[scalar_value]]
 ; GCN: {{flat|buffer|global}}_atomic_sub v[[value]]
@@ -135,9 +150,10 @@ entry:
 ; GFX7LESS-NOT: v_mbcnt_hi_u32_b32
 ; GFX7LESS-NOT: s_bcnt1_i32_b64
 ; GFX7LESS: buffer_atomic_sub v{{[0-9]+}}
-; GFX8MORE: v_sub{{(rev)?}}_u32_dpp
-; GFX8MORE: v_sub{{(rev)?}}_u32_dpp
-; GFX8MORE: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 63
+; DPPCOMB: v_add_u32_dpp
+; DPPCOMB: v_add_u32_dpp
+; GFX8MORE32: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 31
+; GFX8MORE64: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 63
 ; GFX8MORE: v_mov_b32{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[scalar_value]]
 ; GFX8MORE: buffer_atomic_sub v[[value]]
 define amdgpu_kernel void @sub_i32_varying(i32 addrspace(1)* %out, i32 addrspace(1)* %inout) {
@@ -149,11 +165,13 @@ entry:
 }
 
 ; GCN-LABEL: sub_i64_constant:
-; GCN: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
-; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_lo:[0-9]+]], s[[exec_lo]], 0
-; GCN: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_hi:[0-9]+]], s[[exec_hi]], v[[mbcnt_lo]]
-; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc, 0, v[[mbcnt_hi]]
-; GCN: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
+; GCN32: v_cmp_ne_u32_e64 s[[exec_lo:[0-9]+]], 1, 0
+; GCN64: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
+; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt:[0-9]+]], s[[exec_lo]], 0
+; GCN64: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt]], s[[exec_hi]], v[[mbcnt]]
+; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc{{(_lo)?}}, 0, v[[mbcnt]]
+; GCN32: s_bcnt1_i32_b32 s[[popcount:[0-9]+]], s[[exec_lo]]
+; GCN64: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
 ; GCN: v_mul_hi_u32_u24{{(_e[0-9]+)?}} v[[value_hi:[0-9]+]], s[[popcount]], 5
 ; GCN: v_mul_u32_u24{{(_e[0-9]+)?}} v[[value_lo:[0-9]+]], s[[popcount]], 5
 ; GCN: {{flat|buffer|global}}_atomic_sub_x2 v{{\[}}[[value_lo]]:[[value_hi]]{{\]}}
@@ -165,11 +183,13 @@ entry:
 }
 
 ; GCN-LABEL: sub_i64_uniform:
-; GCN: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
-; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_lo:[0-9]+]], s[[exec_lo]], 0
-; GCN: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_hi:[0-9]+]], s[[exec_hi]], v[[mbcnt_lo]]
-; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc, 0, v[[mbcnt_hi]]
-; GCN: s_bcnt1_i32_b64 s{{[0-9]+}}, s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
+; GCN32: v_cmp_ne_u32_e64 s[[exec_lo:[0-9]+]], 1, 0
+; GCN64: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
+; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt:[0-9]+]], s[[exec_lo]], 0
+; GCN64: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt]], s[[exec_hi]], v[[mbcnt]]
+; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc{{(_lo)?}}, 0, v[[mbcnt]]
+; GCN32: s_bcnt1_i32_b32 s{{[0-9]+}}, s[[exec_lo]]
+; GCN64: s_bcnt1_i32_b64 s{{[0-9]+}}, s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
 ; GCN: {{flat|buffer|global}}_atomic_sub_x2 v{{\[}}{{[0-9]+}}:{{[0-9]+}}{{\]}}
 define amdgpu_kernel void @sub_i64_uniform(i64 addrspace(1)* %out, i64 addrspace(1)* %inout, i64 %subitive) {
 entry:
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
index 056207891..2fec729a3 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
@@ -1,6 +1,9 @@
-; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX7LESS %s
-; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=tonga -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX8MORE %s
-; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=gfx900 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX8MORE %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX7LESS %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=tonga -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX8 %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=gfx900 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX9 %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=gfx1010 -mattr=-wavefrontsize32,+wavefrontsize64 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX1064 %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=gfx1010 -mattr=+wavefrontsize32,-wavefrontsize64 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX1032 %s
 
 declare i32 @llvm.amdgcn.workitem.id.x()
 
@@ -9,48 +12,1082 @@ declare i32 @llvm.amdgcn.workitem.id.x()
 
 ; Show that what the atomic optimization pass will do for local pointers.
 
-; GCN-LABEL: add_i32_constant:
-; GCN: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
-; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_lo:[0-9]+]], s[[exec_lo]], 0
-; GCN: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_hi:[0-9]+]], s[[exec_hi]], v[[mbcnt_lo]]
-; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc, 0, v[[mbcnt_hi]]
-; GCN: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
-; GCN: v_mul_u32_u24{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[popcount]], 5
-; GCN: ds_add_rtn_u32 v{{[0-9]+}}, v{{[0-9]+}}, v[[value]]
 define amdgpu_kernel void @add_i32_constant(i32 addrspace(1)* %out) {
+;
+;
+; GFX7LESS-LABEL: add_i32_constant:
+; GFX7LESS:       ; %bb.0: ; %entry
+; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; GFX7LESS-NEXT:    v_cmp_ne_u32_e64 s[4:5], 1, 0
+; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s4, 0
+; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s5, v0
+; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT:    ; implicit-def: $vgpr1
+; GFX7LESS-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX7LESS-NEXT:    ; mask branch BB0_2
+; GFX7LESS-NEXT:    s_cbranch_execz BB0_2
+; GFX7LESS-NEXT:  BB0_1:
+; GFX7LESS-NEXT:    s_bcnt1_i32_b64 s4, s[4:5]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v1, local_var32@abs32@lo
+; GFX7LESS-NEXT:    v_mul_u32_u24_e64 v2, s4, 5
+; GFX7LESS-NEXT:    s_mov_b32 m0, -1
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7LESS-NEXT:    ds_add_rtn_u32 v1, v1, v2
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7LESS-NEXT:    buffer_wbinvl1
+; GFX7LESS-NEXT:  BB0_2:
+; GFX7LESS-NEXT:    s_or_b64 exec, exec, s[2:3]
+; GFX7LESS-NEXT:    v_readfirstlane_b32 s2, v1
+; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7LESS-NEXT:    v_mad_u32_u24 v0, v0, 5, s2
+; GFX7LESS-NEXT:    s_mov_b32 s2, -1
+; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX7LESS-NEXT:    s_endpgm
+;
+; GFX8-LABEL: add_i32_constant:
+; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX8-NEXT:    v_cmp_ne_u32_e64 s[4:5], 1, 0
+; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s4, 0
+; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s5, v0
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX8-NEXT:    ; implicit-def: $vgpr1
+; GFX8-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX8-NEXT:    ; mask branch BB0_2
+; GFX8-NEXT:    s_cbranch_execz BB0_2
+; GFX8-NEXT:  BB0_1:
+; GFX8-NEXT:    s_bcnt1_i32_b64 s4, s[4:5]
+; GFX8-NEXT:    v_mul_u32_u24_e64 v1, s4, 5
+; GFX8-NEXT:    v_mov_b32_e32 v2, local_var32@abs32@lo
+; GFX8-NEXT:    s_mov_b32 m0, -1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    ds_add_rtn_u32 v1, v2, v1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:  BB0_2:
+; GFX8-NEXT:    s_or_b64 exec, exec, s[2:3]
+; GFX8-NEXT:    v_readfirstlane_b32 s2, v1
+; GFX8-NEXT:    v_mad_u32_u24 v0, v0, 5, s2
+; GFX8-NEXT:    s_mov_b32 s3, 0xf000
+; GFX8-NEXT:    s_mov_b32 s2, -1
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX8-NEXT:    s_endpgm
+;
+; GFX9-LABEL: add_i32_constant:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    v_cmp_ne_u32_e64 s[4:5], 1, 0
+; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s4, 0
+; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s5, v0
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT:    ; implicit-def: $vgpr1
+; GFX9-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX9-NEXT:    ; mask branch BB0_2
+; GFX9-NEXT:    s_cbranch_execz BB0_2
+; GFX9-NEXT:  BB0_1:
+; GFX9-NEXT:    s_bcnt1_i32_b64 s4, s[4:5]
+; GFX9-NEXT:    v_mul_u32_u24_e64 v1, s4, 5
+; GFX9-NEXT:    v_mov_b32_e32 v2, local_var32@abs32@lo
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    ds_add_rtn_u32 v1, v2, v1
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:  BB0_2:
+; GFX9-NEXT:    s_or_b64 exec, exec, s[2:3]
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v1
+; GFX9-NEXT:    v_mad_u32_u24 v0, v0, 5, s2
+; GFX9-NEXT:    s_mov_b32 s3, 0xf000
+; GFX9-NEXT:    s_mov_b32 s2, -1
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX9-NEXT:    s_endpgm
+;
+; GFX1064-LABEL: add_i32_constant:
+; GFX1064:       ; %bb.0: ; %entry
+; GFX1064-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX1064-NEXT:    ; implicit-def: $vgpr1
+; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
+; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, s3, v0
+; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX1064-NEXT:    ; mask branch BB0_2
+; GFX1064-NEXT:    s_cbranch_execz BB0_2
+; GFX1064-NEXT:  BB0_1:
+; GFX1064-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
+; GFX1064-NEXT:    v_mov_b32_e32 v2, local_var32@abs32@lo
+; GFX1064-NEXT:    v_mul_u32_u24_e64 v1, s2, 5
+; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1064-NEXT:    ds_add_rtn_u32 v1, v2, v1
+; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1064-NEXT:    buffer_gl0_inv
+; GFX1064-NEXT:    buffer_gl1_inv
+; GFX1064-NEXT:  BB0_2:
+; GFX1064-NEXT:    v_nop
+; GFX1064-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX1064-NEXT:    v_readfirstlane_b32 s2, v1
+; GFX1064-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1064-NEXT:    v_mad_u32_u24 v0, v0, 5, s2
+; GFX1064-NEXT:    s_mov_b32 s2, -1
+; GFX1064-NEXT:    s_nop 1
+; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX1064-NEXT:    s_endpgm
+;
+; GFX1032-LABEL: add_i32_constant:
+; GFX1032:       ; %bb.0: ; %entry
+; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX1032-NEXT:    v_cmp_ne_u32_e64 s3, 1, 0
+; GFX1032-NEXT:    ; implicit-def: $vcc_hi
+; GFX1032-NEXT:    ; implicit-def: $vgpr1
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s3, 0
+; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-NEXT:    s_and_saveexec_b32 s2, vcc_lo
+; GFX1032-NEXT:    ; mask branch BB0_2
+; GFX1032-NEXT:    s_cbranch_execz BB0_2
+; GFX1032-NEXT:  BB0_1:
+; GFX1032-NEXT:    s_bcnt1_i32_b32 s3, s3
+; GFX1032-NEXT:    v_mov_b32_e32 v2, local_var32@abs32@lo
+; GFX1032-NEXT:    v_mul_u32_u24_e64 v1, s3, 5
+; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1032-NEXT:    ds_add_rtn_u32 v1, v2, v1
+; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1032-NEXT:    buffer_gl0_inv
+; GFX1032-NEXT:    buffer_gl1_inv
+; GFX1032-NEXT:  BB0_2:
+; GFX1032-NEXT:    v_nop
+; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT:    v_readfirstlane_b32 s2, v1
+; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1032-NEXT:    v_mad_u32_u24 v0, v0, 5, s2
+; GFX1032-NEXT:    s_mov_b32 s2, -1
+; GFX1032-NEXT:    s_nop 1
+; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX1032-NEXT:    s_endpgm
 entry:
   %old = atomicrmw add i32 addrspace(3)* @local_var32, i32 5 acq_rel
   store i32 %old, i32 addrspace(1)* %out
   ret void
 }
 
-; GCN-LABEL: add_i32_uniform:
-; GCN: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
-; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_lo:[0-9]+]], s[[exec_lo]], 0
-; GCN: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_hi:[0-9]+]], s[[exec_hi]], v[[mbcnt_lo]]
-; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc, 0, v[[mbcnt_hi]]
-; GCN: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
-; GCN: s_mul_i32 s[[scalar_value:[0-9]+]], s{{[0-9]+}}, s[[popcount]]
-; GCN: v_mov_b32{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[scalar_value]]
-; GCN: ds_add_rtn_u32 v{{[0-9]+}}, v{{[0-9]+}}, v[[value]]
 define amdgpu_kernel void @add_i32_uniform(i32 addrspace(1)* %out, i32 %additive) {
+;
+;
+; GFX7LESS-LABEL: add_i32_uniform:
+; GFX7LESS:       ; %bb.0: ; %entry
+; GFX7LESS-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX7LESS-NEXT:    s_load_dword s2, s[0:1], 0xb
+; GFX7LESS-NEXT:    v_cmp_ne_u32_e64 s[6:7], 1, 0
+; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s6, 0
+; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s7, v0
+; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT:    ; implicit-def: $vgpr1
+; GFX7LESS-NEXT:    s_and_saveexec_b64 s[0:1], vcc
+; GFX7LESS-NEXT:    ; mask branch BB1_2
+; GFX7LESS-NEXT:    s_cbranch_execz BB1_2
+; GFX7LESS-NEXT:  BB1_1:
+; GFX7LESS-NEXT:    s_bcnt1_i32_b64 s3, s[6:7]
+; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT:    s_mul_i32 s3, s2, s3
+; GFX7LESS-NEXT:    v_mov_b32_e32 v1, local_var32@abs32@lo
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s3
+; GFX7LESS-NEXT:    s_mov_b32 m0, -1
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7LESS-NEXT:    ds_add_rtn_u32 v1, v1, v2
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7LESS-NEXT:    buffer_wbinvl1
+; GFX7LESS-NEXT:  BB1_2:
+; GFX7LESS-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX7LESS-NEXT:    v_readfirstlane_b32 s0, v1
+; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT:    v_mul_lo_u32 v0, s2, v0
+; GFX7LESS-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7LESS-NEXT:    v_add_i32_e32 v0, vcc, s0, v0
+; GFX7LESS-NEXT:    s_mov_b32 s6, -1
+; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX7LESS-NEXT:    s_endpgm
+;
+; GFX8-LABEL: add_i32_uniform:
+; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
+; GFX8-NEXT:    s_load_dword s0, s[0:1], 0x2c
+; GFX8-NEXT:    v_cmp_ne_u32_e64 s[6:7], 1, 0
+; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s7, v0
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX8-NEXT:    ; implicit-def: $vgpr1
+; GFX8-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX8-NEXT:    ; mask branch BB1_2
+; GFX8-NEXT:    s_cbranch_execz BB1_2
+; GFX8-NEXT:  BB1_1:
+; GFX8-NEXT:    s_bcnt1_i32_b64 s1, s[6:7]
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_mul_i32 s1, s0, s1
+; GFX8-NEXT:    v_mov_b32_e32 v1, local_var32@abs32@lo
+; GFX8-NEXT:    v_mov_b32_e32 v2, s1
+; GFX8-NEXT:    s_mov_b32 m0, -1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    ds_add_rtn_u32 v1, v1, v2
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:  BB1_2:
+; GFX8-NEXT:    s_or_b64 exec, exec, s[2:3]
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    v_mul_lo_u32 v0, s0, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v1
+; GFX8-NEXT:    s_mov_b32 s7, 0xf000
+; GFX8-NEXT:    s_mov_b32 s6, -1
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
+; GFX8-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX8-NEXT:    s_endpgm
+;
+; GFX9-LABEL: add_i32_uniform:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
+; GFX9-NEXT:    s_load_dword s0, s[0:1], 0x2c
+; GFX9-NEXT:    v_cmp_ne_u32_e64 s[6:7], 1, 0
+; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s7, v0
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT:    ; implicit-def: $vgpr1
+; GFX9-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX9-NEXT:    ; mask branch BB1_2
+; GFX9-NEXT:    s_cbranch_execz BB1_2
+; GFX9-NEXT:  BB1_1:
+; GFX9-NEXT:    s_bcnt1_i32_b64 s1, s[6:7]
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_mul_i32 s1, s0, s1
+; GFX9-NEXT:    v_mov_b32_e32 v1, local_var32@abs32@lo
+; GFX9-NEXT:    v_mov_b32_e32 v2, s1
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    ds_add_rtn_u32 v1, v1, v2
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:  BB1_2:
+; GFX9-NEXT:    s_or_b64 exec, exec, s[2:3]
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mul_lo_u32 v0, s0, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v1
+; GFX9-NEXT:    s_mov_b32 s7, 0xf000
+; GFX9-NEXT:    s_mov_b32 s6, -1
+; GFX9-NEXT:    v_add_u32_e32 v0, s0, v0
+; GFX9-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX9-NEXT:    s_endpgm
+;
+; GFX1064-LABEL: add_i32_uniform:
+; GFX1064:       ; %bb.0: ; %entry
+; GFX1064-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
+; GFX1064-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX1064-NEXT:    s_load_dword s0, s[0:1], 0x2c
+; GFX1064-NEXT:    ; implicit-def: $vgpr1
+; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
+; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, s3, v0
+; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT:    s_and_saveexec_b64 s[6:7], vcc
+; GFX1064-NEXT:    ; mask branch BB1_2
+; GFX1064-NEXT:    s_cbranch_execz BB1_2
+; GFX1064-NEXT:  BB1_1:
+; GFX1064-NEXT:    s_bcnt1_i32_b64 s1, s[2:3]
+; GFX1064-NEXT:    v_mov_b32_e32 v1, local_var32@abs32@lo
+; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT:    s_mul_i32 s1, s0, s1
+; GFX1064-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1064-NEXT:    ds_add_rtn_u32 v1, v1, v2
+; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1064-NEXT:    buffer_gl0_inv
+; GFX1064-NEXT:    buffer_gl1_inv
+; GFX1064-NEXT:  BB1_2:
+; GFX1064-NEXT:    v_nop
+; GFX1064-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT:    v_mul_lo_u32 v0, s0, v0
+; GFX1064-NEXT:    v_readfirstlane_b32 s0, v1
+; GFX1064-NEXT:    s_mov_b32 s7, 0x31016000
+; GFX1064-NEXT:    s_mov_b32 s6, -1
+; GFX1064-NEXT:    v_add_nc_u32_e32 v0, s0, v0
+; GFX1064-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX1064-NEXT:    s_endpgm
+;
+; GFX1032-LABEL: add_i32_uniform:
+; GFX1032:       ; %bb.0: ; %entry
+; GFX1032-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
+; GFX1032-NEXT:    s_load_dword s0, s[0:1], 0x2c
+; GFX1032-NEXT:    v_cmp_ne_u32_e64 s2, 1, 0
+; GFX1032-NEXT:    ; implicit-def: $vcc_hi
+; GFX1032-NEXT:    ; implicit-def: $vgpr1
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
+; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-NEXT:    s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-NEXT:    ; mask branch BB1_2
+; GFX1032-NEXT:    s_cbranch_execz BB1_2
+; GFX1032-NEXT:  BB1_1:
+; GFX1032-NEXT:    s_bcnt1_i32_b32 s2, s2
+; GFX1032-NEXT:    v_mov_b32_e32 v1, local_var32@abs32@lo
+; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT:    s_mul_i32 s2, s0, s2
+; GFX1032-NEXT:    v_mov_b32_e32 v2, s2
+; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1032-NEXT:    ds_add_rtn_u32 v1, v1, v2
+; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1032-NEXT:    buffer_gl0_inv
+; GFX1032-NEXT:    buffer_gl1_inv
+; GFX1032-NEXT:  BB1_2:
+; GFX1032-NEXT:    v_nop
+; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s1
+; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT:    v_mul_lo_u32 v0, s0, v0
+; GFX1032-NEXT:    v_readfirstlane_b32 s0, v1
+; GFX1032-NEXT:    s_mov_b32 s7, 0x31016000
+; GFX1032-NEXT:    s_mov_b32 s6, -1
+; GFX1032-NEXT:    v_add_nc_u32_e32 v0, s0, v0
+; GFX1032-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX1032-NEXT:    s_endpgm
 entry:
   %old = atomicrmw add i32 addrspace(3)* @local_var32, i32 %additive acq_rel
   store i32 %old, i32 addrspace(1)* %out
   ret void
 }
 
-; GCN-LABEL: add_i32_varying:
 ; GFX7LESS-NOT: v_mbcnt_lo_u32_b32
 ; GFX7LESS-NOT: v_mbcnt_hi_u32_b32
 ; GFX7LESS-NOT: s_bcnt1_i32_b64
-; GFX7LESS: ds_add_rtn_u32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; GFX8MORE: v_add_u32_dpp
-; GFX8MORE: v_add_u32_dpp
-; GFX8MORE: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 63
+; DPPCOMB: v_add_u32_dpp
+; DPPCOMB: v_add_u32_dpp
+; GFX8MORE32: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 31
 ; GFX8MORE: v_mov_b32{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[scalar_value]]
 ; GFX8MORE: ds_add_rtn_u32 v{{[0-9]+}}, v{{[0-9]+}}, v[[value]]
 define amdgpu_kernel void @add_i32_varying(i32 addrspace(1)* %out) {
+;
+;
+; GFX7LESS-LABEL: add_i32_varying:
+; GFX7LESS:       ; %bb.0: ; %entry
+; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; GFX7LESS-NEXT:    v_mov_b32_e32 v1, local_var32@abs32@lo
+; GFX7LESS-NEXT:    s_mov_b32 m0, -1
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7LESS-NEXT:    ds_add_rtn_u32 v0, v1, v0
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7LESS-NEXT:    buffer_wbinvl1
+; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7LESS-NEXT:    s_mov_b32 s2, -1
+; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX7LESS-NEXT:    s_endpgm
+;
+; GFX8-LABEL: add_i32_varying:
+; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    v_mov_b32_e32 v2, v0
+; GFX8-NEXT:    s_or_saveexec_b64 s[2:3], -1
+; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX8-NEXT:    v_mov_b32_e32 v1, 0
+; GFX8-NEXT:    s_mov_b64 exec, s[2:3]
+; GFX8-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX8-NEXT:    s_not_b64 exec, exec
+; GFX8-NEXT:    v_mov_b32_e32 v2, 0
+; GFX8-NEXT:    s_not_b64 exec, exec
+; GFX8-NEXT:    s_or_saveexec_b64 s[4:5], -1
+; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX8-NEXT:    v_readlane_b32 s2, v2, 63
+; GFX8-NEXT:    s_nop 0
+; GFX8-NEXT:    v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf
+; GFX8-NEXT:    s_mov_b64 exec, s[4:5]
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX8-NEXT:    ; implicit-def: $vgpr0
+; GFX8-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX8-NEXT:    ; mask branch BB2_2
+; GFX8-NEXT:    s_cbranch_execz BB2_2
+; GFX8-NEXT:  BB2_1:
+; GFX8-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
+; GFX8-NEXT:    v_mov_b32_e32 v3, s2
+; GFX8-NEXT:    s_mov_b32 m0, -1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    ds_add_rtn_u32 v0, v0, v3
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:  BB2_2:
+; GFX8-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT:    v_readfirstlane_b32 s2, v0
+; GFX8-NEXT:    v_mov_b32_e32 v0, v1
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s2, v0
+; GFX8-NEXT:    s_mov_b32 s3, 0xf000
+; GFX8-NEXT:    s_mov_b32 s2, -1
+; GFX8-NEXT:    s_nop 0
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX8-NEXT:    s_endpgm
+;
+; GFX9-LABEL: add_i32_varying:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_or_saveexec_b64 s[2:3], -1
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0
+; GFX9-NEXT:    s_mov_b64 exec, s[2:3]
+; GFX9-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX9-NEXT:    s_not_b64 exec, exec
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-NEXT:    s_not_b64 exec, exec
+; GFX9-NEXT:    s_or_saveexec_b64 s[4:5], -1
+; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX9-NEXT:    v_readlane_b32 s2, v2, 63
+; GFX9-NEXT:    s_nop 0
+; GFX9-NEXT:    v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-NEXT:    s_mov_b64 exec, s[4:5]
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT:    ; implicit-def: $vgpr0
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    ; mask branch BB2_2
+; GFX9-NEXT:    s_cbranch_execz BB2_2
+; GFX9-NEXT:  BB2_1:
+; GFX9-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
+; GFX9-NEXT:    v_mov_b32_e32 v3, s2
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    ds_add_rtn_u32 v0, v0, v3
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:  BB2_2:
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v0
+; GFX9-NEXT:    v_mov_b32_e32 v0, v1
+; GFX9-NEXT:    v_add_u32_e32 v0, s2, v0
+; GFX9-NEXT:    s_mov_b32 s3, 0xf000
+; GFX9-NEXT:    s_mov_b32 s2, -1
+; GFX9-NEXT:    s_nop 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX9-NEXT:    s_endpgm
+;
+; GFX1064-LABEL: add_i32_varying:
+; GFX1064:       ; %bb.0: ; %entry
+; GFX1064-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1064-NEXT:    s_or_saveexec_b64 s[2:3], -1
+; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
+; GFX1064-NEXT:    s_mov_b64 exec, s[2:3]
+; GFX1064-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
+; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, s3, v0
+; GFX1064-NEXT:    s_not_b64 exec, exec
+; GFX1064-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1064-NEXT:    s_not_b64 exec, exec
+; GFX1064-NEXT:    s_or_saveexec_b64 s[4:5], -1
+; GFX1064-NEXT:    v_add_nc_u32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX1064-NEXT:    v_add_nc_u32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX1064-NEXT:    v_add_nc_u32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX1064-NEXT:    v_add_nc_u32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX1064-NEXT:    v_mov_b32_e32 v3, v2
+; GFX1064-NEXT:    v_permlanex16_b32 v3, v3, -1, -1
+; GFX1064-NEXT:    v_add_nc_u32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
+; GFX1064-NEXT:    v_readlane_b32 s2, v2, 31
+; GFX1064-NEXT:    v_mov_b32_e32 v3, s2
+; GFX1064-NEXT:    v_add_nc_u32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf
+; GFX1064-NEXT:    v_readlane_b32 s2, v2, 15
+; GFX1064-NEXT:    v_mov_b32_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-NEXT:    v_readlane_b32 s3, v2, 31
+; GFX1064-NEXT:    v_readlane_b32 s6, v2, 47
+; GFX1064-NEXT:    v_writelane_b32 v1, s2, 16
+; GFX1064-NEXT:    s_mov_b32 s2, -1
+; GFX1064-NEXT:    v_writelane_b32 v1, s3, 32
+; GFX1064-NEXT:    v_readlane_b32 s3, v2, 63
+; GFX1064-NEXT:    v_writelane_b32 v1, s6, 48
+; GFX1064-NEXT:    s_mov_b64 exec, s[4:5]
+; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT:    ; implicit-def: $vgpr0
+; GFX1064-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX1064-NEXT:    ; mask branch BB2_2
+; GFX1064-NEXT:    s_cbranch_execz BB2_2
+; GFX1064-NEXT:  BB2_1:
+; GFX1064-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
+; GFX1064-NEXT:    v_mov_b32_e32 v7, s3
+; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1064-NEXT:    ds_add_rtn_u32 v0, v0, v7
+; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1064-NEXT:    buffer_gl0_inv
+; GFX1064-NEXT:    buffer_gl1_inv
+; GFX1064-NEXT:  BB2_2:
+; GFX1064-NEXT:    v_nop
+; GFX1064-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX1064-NEXT:    v_readfirstlane_b32 s3, v0
+; GFX1064-NEXT:    v_mov_b32_e32 v0, v1
+; GFX1064-NEXT:    v_add_nc_u32_e32 v0, s3, v0
+; GFX1064-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1064-NEXT:    s_nop 1
+; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX1064-NEXT:    s_endpgm
+;
+; GFX1032-LABEL: add_i32_varying:
+; GFX1032:       ; %bb.0: ; %entry
+; GFX1032-NEXT:    ; implicit-def: $vcc_hi
+; GFX1032-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1032-NEXT:    s_or_saveexec_b32 s2, -1
+; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
+; GFX1032-NEXT:    s_mov_b32 exec_lo, s2
+; GFX1032-NEXT:    v_cmp_ne_u32_e64 s2, 1, 0
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
+; GFX1032-NEXT:    s_not_b32 exec_lo, exec_lo
+; GFX1032-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1032-NEXT:    s_not_b32 exec_lo, exec_lo
+; GFX1032-NEXT:    s_or_saveexec_b32 s4, -1
+; GFX1032-NEXT:    v_add_nc_u32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX1032-NEXT:    s_mov_b32 s2, -1
+; GFX1032-NEXT:    v_add_nc_u32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX1032-NEXT:    v_add_nc_u32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX1032-NEXT:    v_add_nc_u32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX1032-NEXT:    v_mov_b32_e32 v3, v2
+; GFX1032-NEXT:    v_permlanex16_b32 v3, v3, -1, -1
+; GFX1032-NEXT:    v_add_nc_u32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
+; GFX1032-NEXT:    v_readlane_b32 s3, v2, 31
+; GFX1032-NEXT:    v_mov_b32_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-NEXT:    v_readlane_b32 s5, v2, 15
+; GFX1032-NEXT:    v_writelane_b32 v1, s5, 16
+; GFX1032-NEXT:    s_mov_b32 exec_lo, s4
+; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-NEXT:    ; implicit-def: $vgpr0
+; GFX1032-NEXT:    s_and_saveexec_b32 s4, vcc_lo
+; GFX1032-NEXT:    ; mask branch BB2_2
+; GFX1032-NEXT:    s_cbranch_execz BB2_2
+; GFX1032-NEXT:  BB2_1:
+; GFX1032-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
+; GFX1032-NEXT:    v_mov_b32_e32 v7, s3
+; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1032-NEXT:    ds_add_rtn_u32 v0, v0, v7
+; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1032-NEXT:    buffer_gl0_inv
+; GFX1032-NEXT:    buffer_gl1_inv
+; GFX1032-NEXT:  BB2_2:
+; GFX1032-NEXT:    v_nop
+; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX1032-NEXT:    v_readfirstlane_b32 s3, v0
+; GFX1032-NEXT:    v_mov_b32_e32 v0, v1
+; GFX1032-NEXT:    v_add_nc_u32_e32 v0, s3, v0
+; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1032-NEXT:    s_nop 1
+; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX1032-NEXT:    s_endpgm
+entry:
+  %lane = call i32 @llvm.amdgcn.workitem.id.x()
+  %old = atomicrmw add i32 addrspace(3)* @local_var32, i32 %lane acq_rel
+  store i32 %old, i32 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @add_i32_varying_gfx1032(i32 addrspace(1)* %out) {
+;
+;
+; GFX7LESS-LABEL: add_i32_varying_gfx1032:
+; GFX7LESS:       ; %bb.0: ; %entry
+; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; GFX7LESS-NEXT:    v_mov_b32_e32 v1, local_var32@abs32@lo
+; GFX7LESS-NEXT:    s_mov_b32 m0, -1
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7LESS-NEXT:    ds_add_rtn_u32 v0, v1, v0
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7LESS-NEXT:    buffer_wbinvl1
+; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7LESS-NEXT:    s_mov_b32 s2, -1
+; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX7LESS-NEXT:    s_endpgm
+;
+; GFX8-LABEL: add_i32_varying_gfx1032:
+; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    v_mov_b32_e32 v2, v0
+; GFX8-NEXT:    s_or_saveexec_b64 s[2:3], -1
+; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX8-NEXT:    v_mov_b32_e32 v1, 0
+; GFX8-NEXT:    s_mov_b64 exec, s[2:3]
+; GFX8-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX8-NEXT:    s_not_b64 exec, exec
+; GFX8-NEXT:    v_mov_b32_e32 v2, 0
+; GFX8-NEXT:    s_not_b64 exec, exec
+; GFX8-NEXT:    s_or_saveexec_b64 s[4:5], -1
+; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX8-NEXT:    v_readlane_b32 s2, v2, 63
+; GFX8-NEXT:    s_nop 0
+; GFX8-NEXT:    v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf
+; GFX8-NEXT:    s_mov_b64 exec, s[4:5]
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX8-NEXT:    ; implicit-def: $vgpr0
+; GFX8-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX8-NEXT:    ; mask branch BB3_2
+; GFX8-NEXT:    s_cbranch_execz BB3_2
+; GFX8-NEXT:  BB3_1:
+; GFX8-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
+; GFX8-NEXT:    v_mov_b32_e32 v3, s2
+; GFX8-NEXT:    s_mov_b32 m0, -1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    ds_add_rtn_u32 v0, v0, v3
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:  BB3_2:
+; GFX8-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT:    v_readfirstlane_b32 s2, v0
+; GFX8-NEXT:    v_mov_b32_e32 v0, v1
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s2, v0
+; GFX8-NEXT:    s_mov_b32 s3, 0xf000
+; GFX8-NEXT:    s_mov_b32 s2, -1
+; GFX8-NEXT:    s_nop 0
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX8-NEXT:    s_endpgm
+;
+; GFX9-LABEL: add_i32_varying_gfx1032:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_or_saveexec_b64 s[2:3], -1
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0
+; GFX9-NEXT:    s_mov_b64 exec, s[2:3]
+; GFX9-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX9-NEXT:    s_not_b64 exec, exec
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-NEXT:    s_not_b64 exec, exec
+; GFX9-NEXT:    s_or_saveexec_b64 s[4:5], -1
+; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX9-NEXT:    v_readlane_b32 s2, v2, 63
+; GFX9-NEXT:    s_nop 0
+; GFX9-NEXT:    v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-NEXT:    s_mov_b64 exec, s[4:5]
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT:    ; implicit-def: $vgpr0
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    ; mask branch BB3_2
+; GFX9-NEXT:    s_cbranch_execz BB3_2
+; GFX9-NEXT:  BB3_1:
+; GFX9-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
+; GFX9-NEXT:    v_mov_b32_e32 v3, s2
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    ds_add_rtn_u32 v0, v0, v3
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:  BB3_2:
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v0
+; GFX9-NEXT:    v_mov_b32_e32 v0, v1
+; GFX9-NEXT:    v_add_u32_e32 v0, s2, v0
+; GFX9-NEXT:    s_mov_b32 s3, 0xf000
+; GFX9-NEXT:    s_mov_b32 s2, -1
+; GFX9-NEXT:    s_nop 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX9-NEXT:    s_endpgm
+;
+; GFX1064-LABEL: add_i32_varying_gfx1032:
+; GFX1064:       ; %bb.0: ; %entry
+; GFX1064-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1064-NEXT:    s_or_saveexec_b64 s[2:3], -1
+; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
+; GFX1064-NEXT:    s_mov_b64 exec, s[2:3]
+; GFX1064-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
+; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, s3, v0
+; GFX1064-NEXT:    s_not_b64 exec, exec
+; GFX1064-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1064-NEXT:    s_not_b64 exec, exec
+; GFX1064-NEXT:    s_or_saveexec_b64 s[4:5], -1
+; GFX1064-NEXT:    v_add_nc_u32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX1064-NEXT:    v_add_nc_u32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX1064-NEXT:    v_add_nc_u32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX1064-NEXT:    v_add_nc_u32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX1064-NEXT:    v_mov_b32_e32 v3, v2
+; GFX1064-NEXT:    v_permlanex16_b32 v3, v3, -1, -1
+; GFX1064-NEXT:    v_add_nc_u32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
+; GFX1064-NEXT:    v_readlane_b32 s2, v2, 31
+; GFX1064-NEXT:    v_mov_b32_e32 v3, s2
+; GFX1064-NEXT:    v_add_nc_u32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf
+; GFX1064-NEXT:    v_readlane_b32 s2, v2, 15
+; GFX1064-NEXT:    v_mov_b32_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-NEXT:    v_readlane_b32 s3, v2, 31
+; GFX1064-NEXT:    v_readlane_b32 s6, v2, 47
+; GFX1064-NEXT:    v_writelane_b32 v1, s2, 16
+; GFX1064-NEXT:    s_mov_b32 s2, -1
+; GFX1064-NEXT:    v_writelane_b32 v1, s3, 32
+; GFX1064-NEXT:    v_readlane_b32 s3, v2, 63
+; GFX1064-NEXT:    v_writelane_b32 v1, s6, 48
+; GFX1064-NEXT:    s_mov_b64 exec, s[4:5]
+; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT:    ; implicit-def: $vgpr0
+; GFX1064-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX1064-NEXT:    ; mask branch BB3_2
+; GFX1064-NEXT:    s_cbranch_execz BB3_2
+; GFX1064-NEXT:  BB3_1:
+; GFX1064-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
+; GFX1064-NEXT:    v_mov_b32_e32 v7, s3
+; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1064-NEXT:    ds_add_rtn_u32 v0, v0, v7
+; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1064-NEXT:    buffer_gl0_inv
+; GFX1064-NEXT:    buffer_gl1_inv
+; GFX1064-NEXT:  BB3_2:
+; GFX1064-NEXT:    v_nop
+; GFX1064-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX1064-NEXT:    v_readfirstlane_b32 s3, v0
+; GFX1064-NEXT:    v_mov_b32_e32 v0, v1
+; GFX1064-NEXT:    v_add_nc_u32_e32 v0, s3, v0
+; GFX1064-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1064-NEXT:    s_nop 1
+; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX1064-NEXT:    s_endpgm
+;
+; GFX1032-LABEL: add_i32_varying_gfx1032:
+; GFX1032:       ; %bb.0: ; %entry
+; GFX1032-NEXT:    ; implicit-def: $vcc_hi
+; GFX1032-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1032-NEXT:    s_or_saveexec_b32 s2, -1
+; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
+; GFX1032-NEXT:    s_mov_b32 exec_lo, s2
+; GFX1032-NEXT:    v_cmp_ne_u32_e64 s2, 1, 0
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
+; GFX1032-NEXT:    s_not_b32 exec_lo, exec_lo
+; GFX1032-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1032-NEXT:    s_not_b32 exec_lo, exec_lo
+; GFX1032-NEXT:    s_or_saveexec_b32 s4, -1
+; GFX1032-NEXT:    v_add_nc_u32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX1032-NEXT:    s_mov_b32 s2, -1
+; GFX1032-NEXT:    v_add_nc_u32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX1032-NEXT:    v_add_nc_u32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX1032-NEXT:    v_add_nc_u32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX1032-NEXT:    v_mov_b32_e32 v3, v2
+; GFX1032-NEXT:    v_permlanex16_b32 v3, v3, -1, -1
+; GFX1032-NEXT:    v_add_nc_u32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
+; GFX1032-NEXT:    v_readlane_b32 s3, v2, 31
+; GFX1032-NEXT:    v_mov_b32_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-NEXT:    v_readlane_b32 s5, v2, 15
+; GFX1032-NEXT:    v_writelane_b32 v1, s5, 16
+; GFX1032-NEXT:    s_mov_b32 exec_lo, s4
+; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-NEXT:    ; implicit-def: $vgpr0
+; GFX1032-NEXT:    s_and_saveexec_b32 s4, vcc_lo
+; GFX1032-NEXT:    ; mask branch BB3_2
+; GFX1032-NEXT:    s_cbranch_execz BB3_2
+; GFX1032-NEXT:  BB3_1:
+; GFX1032-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
+; GFX1032-NEXT:    v_mov_b32_e32 v7, s3
+; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1032-NEXT:    ds_add_rtn_u32 v0, v0, v7
+; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1032-NEXT:    buffer_gl0_inv
+; GFX1032-NEXT:    buffer_gl1_inv
+; GFX1032-NEXT:  BB3_2:
+; GFX1032-NEXT:    v_nop
+; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX1032-NEXT:    v_readfirstlane_b32 s3, v0
+; GFX1032-NEXT:    v_mov_b32_e32 v0, v1
+; GFX1032-NEXT:    v_add_nc_u32_e32 v0, s3, v0
+; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1032-NEXT:    s_nop 1
+; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX1032-NEXT:    s_endpgm
+entry:
+  %lane = call i32 @llvm.amdgcn.workitem.id.x()
+  %old = atomicrmw add i32 addrspace(3)* @local_var32, i32 %lane acq_rel
+  store i32 %old, i32 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @add_i32_varying_gfx1064(i32 addrspace(1)* %out) {
+;
+;
+; GFX7LESS-LABEL: add_i32_varying_gfx1064:
+; GFX7LESS:       ; %bb.0: ; %entry
+; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; GFX7LESS-NEXT:    v_mov_b32_e32 v1, local_var32@abs32@lo
+; GFX7LESS-NEXT:    s_mov_b32 m0, -1
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7LESS-NEXT:    ds_add_rtn_u32 v0, v1, v0
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7LESS-NEXT:    buffer_wbinvl1
+; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7LESS-NEXT:    s_mov_b32 s2, -1
+; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX7LESS-NEXT:    s_endpgm
+;
+; GFX8-LABEL: add_i32_varying_gfx1064:
+; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    v_mov_b32_e32 v2, v0
+; GFX8-NEXT:    s_or_saveexec_b64 s[2:3], -1
+; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX8-NEXT:    v_mov_b32_e32 v1, 0
+; GFX8-NEXT:    s_mov_b64 exec, s[2:3]
+; GFX8-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX8-NEXT:    s_not_b64 exec, exec
+; GFX8-NEXT:    v_mov_b32_e32 v2, 0
+; GFX8-NEXT:    s_not_b64 exec, exec
+; GFX8-NEXT:    s_or_saveexec_b64 s[4:5], -1
+; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX8-NEXT:    v_readlane_b32 s2, v2, 63
+; GFX8-NEXT:    s_nop 0
+; GFX8-NEXT:    v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf
+; GFX8-NEXT:    s_mov_b64 exec, s[4:5]
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX8-NEXT:    ; implicit-def: $vgpr0
+; GFX8-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX8-NEXT:    ; mask branch BB4_2
+; GFX8-NEXT:    s_cbranch_execz BB4_2
+; GFX8-NEXT:  BB4_1:
+; GFX8-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
+; GFX8-NEXT:    v_mov_b32_e32 v3, s2
+; GFX8-NEXT:    s_mov_b32 m0, -1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    ds_add_rtn_u32 v0, v0, v3
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:  BB4_2:
+; GFX8-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT:    v_readfirstlane_b32 s2, v0
+; GFX8-NEXT:    v_mov_b32_e32 v0, v1
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s2, v0
+; GFX8-NEXT:    s_mov_b32 s3, 0xf000
+; GFX8-NEXT:    s_mov_b32 s2, -1
+; GFX8-NEXT:    s_nop 0
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX8-NEXT:    s_endpgm
+;
+; GFX9-LABEL: add_i32_varying_gfx1064:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_or_saveexec_b64 s[2:3], -1
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0
+; GFX9-NEXT:    s_mov_b64 exec, s[2:3]
+; GFX9-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX9-NEXT:    s_not_b64 exec, exec
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-NEXT:    s_not_b64 exec, exec
+; GFX9-NEXT:    s_or_saveexec_b64 s[4:5], -1
+; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX9-NEXT:    v_readlane_b32 s2, v2, 63
+; GFX9-NEXT:    s_nop 0
+; GFX9-NEXT:    v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-NEXT:    s_mov_b64 exec, s[4:5]
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT:    ; implicit-def: $vgpr0
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    ; mask branch BB4_2
+; GFX9-NEXT:    s_cbranch_execz BB4_2
+; GFX9-NEXT:  BB4_1:
+; GFX9-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
+; GFX9-NEXT:    v_mov_b32_e32 v3, s2
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    ds_add_rtn_u32 v0, v0, v3
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:  BB4_2:
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v0
+; GFX9-NEXT:    v_mov_b32_e32 v0, v1
+; GFX9-NEXT:    v_add_u32_e32 v0, s2, v0
+; GFX9-NEXT:    s_mov_b32 s3, 0xf000
+; GFX9-NEXT:    s_mov_b32 s2, -1
+; GFX9-NEXT:    s_nop 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX9-NEXT:    s_endpgm
+;
+; GFX1064-LABEL: add_i32_varying_gfx1064:
+; GFX1064:       ; %bb.0: ; %entry
+; GFX1064-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1064-NEXT:    s_or_saveexec_b64 s[2:3], -1
+; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
+; GFX1064-NEXT:    s_mov_b64 exec, s[2:3]
+; GFX1064-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
+; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, s3, v0
+; GFX1064-NEXT:    s_not_b64 exec, exec
+; GFX1064-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1064-NEXT:    s_not_b64 exec, exec
+; GFX1064-NEXT:    s_or_saveexec_b64 s[4:5], -1
+; GFX1064-NEXT:    v_add_nc_u32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX1064-NEXT:    v_add_nc_u32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX1064-NEXT:    v_add_nc_u32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX1064-NEXT:    v_add_nc_u32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX1064-NEXT:    v_mov_b32_e32 v3, v2
+; GFX1064-NEXT:    v_permlanex16_b32 v3, v3, -1, -1
+; GFX1064-NEXT:    v_add_nc_u32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
+; GFX1064-NEXT:    v_readlane_b32 s2, v2, 31
+; GFX1064-NEXT:    v_mov_b32_e32 v3, s2
+; GFX1064-NEXT:    v_add_nc_u32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf
+; GFX1064-NEXT:    v_readlane_b32 s2, v2, 15
+; GFX1064-NEXT:    v_mov_b32_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-NEXT:    v_readlane_b32 s3, v2, 31
+; GFX1064-NEXT:    v_readlane_b32 s6, v2, 47
+; GFX1064-NEXT:    v_writelane_b32 v1, s2, 16
+; GFX1064-NEXT:    s_mov_b32 s2, -1
+; GFX1064-NEXT:    v_writelane_b32 v1, s3, 32
+; GFX1064-NEXT:    v_readlane_b32 s3, v2, 63
+; GFX1064-NEXT:    v_writelane_b32 v1, s6, 48
+; GFX1064-NEXT:    s_mov_b64 exec, s[4:5]
+; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT:    ; implicit-def: $vgpr0
+; GFX1064-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX1064-NEXT:    ; mask branch BB4_2
+; GFX1064-NEXT:    s_cbranch_execz BB4_2
+; GFX1064-NEXT:  BB4_1:
+; GFX1064-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
+; GFX1064-NEXT:    v_mov_b32_e32 v7, s3
+; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1064-NEXT:    ds_add_rtn_u32 v0, v0, v7
+; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1064-NEXT:    buffer_gl0_inv
+; GFX1064-NEXT:    buffer_gl1_inv
+; GFX1064-NEXT:  BB4_2:
+; GFX1064-NEXT:    v_nop
+; GFX1064-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX1064-NEXT:    v_readfirstlane_b32 s3, v0
+; GFX1064-NEXT:    v_mov_b32_e32 v0, v1
+; GFX1064-NEXT:    v_add_nc_u32_e32 v0, s3, v0
+; GFX1064-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1064-NEXT:    s_nop 1
+; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX1064-NEXT:    s_endpgm
+;
+; GFX1032-LABEL: add_i32_varying_gfx1064:
+; GFX1032:       ; %bb.0: ; %entry
+; GFX1032-NEXT:    ; implicit-def: $vcc_hi
+; GFX1032-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1032-NEXT:    s_or_saveexec_b32 s2, -1
+; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
+; GFX1032-NEXT:    s_mov_b32 exec_lo, s2
+; GFX1032-NEXT:    v_cmp_ne_u32_e64 s2, 1, 0
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
+; GFX1032-NEXT:    s_not_b32 exec_lo, exec_lo
+; GFX1032-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1032-NEXT:    s_not_b32 exec_lo, exec_lo
+; GFX1032-NEXT:    s_or_saveexec_b32 s4, -1
+; GFX1032-NEXT:    v_add_nc_u32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX1032-NEXT:    s_mov_b32 s2, -1
+; GFX1032-NEXT:    v_add_nc_u32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX1032-NEXT:    v_add_nc_u32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX1032-NEXT:    v_add_nc_u32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX1032-NEXT:    v_mov_b32_e32 v3, v2
+; GFX1032-NEXT:    v_permlanex16_b32 v3, v3, -1, -1
+; GFX1032-NEXT:    v_add_nc_u32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
+; GFX1032-NEXT:    v_readlane_b32 s3, v2, 31
+; GFX1032-NEXT:    v_mov_b32_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-NEXT:    v_readlane_b32 s5, v2, 15
+; GFX1032-NEXT:    v_writelane_b32 v1, s5, 16
+; GFX1032-NEXT:    s_mov_b32 exec_lo, s4
+; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-NEXT:    ; implicit-def: $vgpr0
+; GFX1032-NEXT:    s_and_saveexec_b32 s4, vcc_lo
+; GFX1032-NEXT:    ; mask branch BB4_2
+; GFX1032-NEXT:    s_cbranch_execz BB4_2
+; GFX1032-NEXT:  BB4_1:
+; GFX1032-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
+; GFX1032-NEXT:    v_mov_b32_e32 v7, s3
+; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1032-NEXT:    ds_add_rtn_u32 v0, v0, v7
+; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1032-NEXT:    buffer_gl0_inv
+; GFX1032-NEXT:    buffer_gl1_inv
+; GFX1032-NEXT:  BB4_2:
+; GFX1032-NEXT:    v_nop
+; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX1032-NEXT:    v_readfirstlane_b32 s3, v0
+; GFX1032-NEXT:    v_mov_b32_e32 v0, v1
+; GFX1032-NEXT:    v_add_nc_u32_e32 v0, s3, v0
+; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1032-NEXT:    s_nop 1
+; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX1032-NEXT:    s_endpgm
 entry:
   %lane = call i32 @llvm.amdgcn.workitem.id.x()
   %old = atomicrmw add i32 addrspace(3)* @local_var32, i32 %lane acq_rel
@@ -58,42 +1095,499 @@ entry:
   ret void
 }
 
-; GCN-LABEL: add_i64_constant:
-; GCN: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
-; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_lo:[0-9]+]], s[[exec_lo]], 0
-; GCN: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_hi:[0-9]+]], s[[exec_hi]], v[[mbcnt_lo]]
-; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc, 0, v[[mbcnt_hi]]
-; GCN: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
-; GCN: v_mul_hi_u32_u24{{(_e[0-9]+)?}} v[[value_hi:[0-9]+]], s[[popcount]], 5
-; GCN: v_mul_u32_u24{{(_e[0-9]+)?}} v[[value_lo:[0-9]+]], s[[popcount]], 5
-; GCN: ds_add_rtn_u64 v{{\[}}{{[0-9]+}}:{{[0-9]+}}{{\]}}, v{{[0-9]+}}, v{{\[}}[[value_lo]]:[[value_hi]]{{\]}}
 define amdgpu_kernel void @add_i64_constant(i64 addrspace(1)* %out) {
+;
+;
+; GFX7LESS-LABEL: add_i64_constant:
+; GFX7LESS:       ; %bb.0: ; %entry
+; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; GFX7LESS-NEXT:    v_cmp_ne_u32_e64 s[4:5], 1, 0
+; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s4, 0
+; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s5, v0
+; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT:    ; implicit-def: $vgpr1_vgpr2
+; GFX7LESS-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX7LESS-NEXT:    ; mask branch BB5_2
+; GFX7LESS-NEXT:    s_cbranch_execz BB5_2
+; GFX7LESS-NEXT:  BB5_1:
+; GFX7LESS-NEXT:    s_bcnt1_i32_b64 s4, s[4:5]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, local_var64@abs32@lo
+; GFX7LESS-NEXT:    v_mul_hi_u32_u24_e64 v2, s4, 5
+; GFX7LESS-NEXT:    v_mul_u32_u24_e64 v1, s4, 5
+; GFX7LESS-NEXT:    s_mov_b32 m0, -1
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7LESS-NEXT:    ds_add_rtn_u64 v[1:2], v3, v[1:2]
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7LESS-NEXT:    buffer_wbinvl1
+; GFX7LESS-NEXT:  BB5_2:
+; GFX7LESS-NEXT:    s_or_b64 exec, exec, s[2:3]
+; GFX7LESS-NEXT:    v_readfirstlane_b32 s2, v1
+; GFX7LESS-NEXT:    v_readfirstlane_b32 s4, v2
+; GFX7LESS-NEXT:    v_mul_hi_u32_u24_e32 v1, 5, v0
+; GFX7LESS-NEXT:    v_mul_u32_u24_e32 v0, 5, v0
+; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s4
+; GFX7LESS-NEXT:    v_add_i32_e32 v0, vcc, s2, v0
+; GFX7LESS-NEXT:    v_addc_u32_e32 v1, vcc, v2, v1, vcc
+; GFX7LESS-NEXT:    s_mov_b32 s2, -1
+; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX7LESS-NEXT:    s_endpgm
+;
+; GFX8-LABEL: add_i64_constant:
+; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX8-NEXT:    v_cmp_ne_u32_e64 s[4:5], 1, 0
+; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s4, 0
+; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s5, v0
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX8-NEXT:    ; implicit-def: $vgpr1_vgpr2
+; GFX8-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX8-NEXT:    ; mask branch BB5_2
+; GFX8-NEXT:    s_cbranch_execz BB5_2
+; GFX8-NEXT:  BB5_1:
+; GFX8-NEXT:    s_bcnt1_i32_b64 s4, s[4:5]
+; GFX8-NEXT:    v_mul_hi_u32_u24_e64 v2, s4, 5
+; GFX8-NEXT:    v_mul_u32_u24_e64 v1, s4, 5
+; GFX8-NEXT:    v_mov_b32_e32 v3, local_var64@abs32@lo
+; GFX8-NEXT:    s_mov_b32 m0, -1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    ds_add_rtn_u64 v[1:2], v3, v[1:2]
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:  BB5_2:
+; GFX8-NEXT:    s_or_b64 exec, exec, s[2:3]
+; GFX8-NEXT:    v_readfirstlane_b32 s2, v1
+; GFX8-NEXT:    v_readfirstlane_b32 s3, v2
+; GFX8-NEXT:    v_mov_b32_e32 v1, s2
+; GFX8-NEXT:    v_mov_b32_e32 v2, s3
+; GFX8-NEXT:    v_mad_u64_u32 v[0:1], s[2:3], v0, 5, v[1:2]
+; GFX8-NEXT:    s_mov_b32 s3, 0xf000
+; GFX8-NEXT:    s_mov_b32 s2, -1
+; GFX8-NEXT:    s_nop 2
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX8-NEXT:    s_endpgm
+;
+; GFX9-LABEL: add_i64_constant:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    v_cmp_ne_u32_e64 s[4:5], 1, 0
+; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s4, 0
+; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s5, v0
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT:    ; implicit-def: $vgpr1_vgpr2
+; GFX9-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX9-NEXT:    ; mask branch BB5_2
+; GFX9-NEXT:    s_cbranch_execz BB5_2
+; GFX9-NEXT:  BB5_1:
+; GFX9-NEXT:    s_bcnt1_i32_b64 s4, s[4:5]
+; GFX9-NEXT:    v_mul_hi_u32_u24_e64 v2, s4, 5
+; GFX9-NEXT:    v_mul_u32_u24_e64 v1, s4, 5
+; GFX9-NEXT:    v_mov_b32_e32 v3, local_var64@abs32@lo
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    ds_add_rtn_u64 v[1:2], v3, v[1:2]
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:  BB5_2:
+; GFX9-NEXT:    s_or_b64 exec, exec, s[2:3]
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s3, v2
+; GFX9-NEXT:    v_mov_b32_e32 v1, s2
+; GFX9-NEXT:    v_mov_b32_e32 v2, s3
+; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[2:3], v0, 5, v[1:2]
+; GFX9-NEXT:    s_mov_b32 s3, 0xf000
+; GFX9-NEXT:    s_mov_b32 s2, -1
+; GFX9-NEXT:    s_nop 2
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX9-NEXT:    s_endpgm
+;
+; GFX1064-LABEL: add_i64_constant:
+; GFX1064:       ; %bb.0: ; %entry
+; GFX1064-NEXT:    v_cmp_ne_u32_e64 s[4:5], 1, 0
+; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX1064-NEXT:    ; implicit-def: $vgpr1_vgpr2
+; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s4, 0
+; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, s5, v0
+; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX1064-NEXT:    ; mask branch BB5_2
+; GFX1064-NEXT:    s_cbranch_execz BB5_2
+; GFX1064-NEXT:  BB5_1:
+; GFX1064-NEXT:    s_bcnt1_i32_b64 s4, s[4:5]
+; GFX1064-NEXT:    v_mov_b32_e32 v3, local_var64@abs32@lo
+; GFX1064-NEXT:    v_mul_hi_u32_u24_e64 v2, s4, 5
+; GFX1064-NEXT:    v_mul_u32_u24_e64 v1, s4, 5
+; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1064-NEXT:    ds_add_rtn_u64 v[1:2], v3, v[1:2]
+; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1064-NEXT:    buffer_gl0_inv
+; GFX1064-NEXT:    buffer_gl1_inv
+; GFX1064-NEXT:  BB5_2:
+; GFX1064-NEXT:    v_nop
+; GFX1064-NEXT:    s_or_b64 exec, exec, s[2:3]
+; GFX1064-NEXT:    v_readfirstlane_b32 s2, v1
+; GFX1064-NEXT:    v_readfirstlane_b32 s3, v2
+; GFX1064-NEXT:    v_mad_u64_u32 v[0:1], s[2:3], v0, 5, s[2:3]
+; GFX1064-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1064-NEXT:    s_mov_b32 s2, -1
+; GFX1064-NEXT:    s_nop 2
+; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX1064-NEXT:    s_endpgm
+;
+; GFX1032-LABEL: add_i64_constant:
+; GFX1032:       ; %bb.0: ; %entry
+; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX1032-NEXT:    v_cmp_ne_u32_e64 s3, 1, 0
+; GFX1032-NEXT:    ; implicit-def: $vcc_hi
+; GFX1032-NEXT:    ; implicit-def: $vgpr1_vgpr2
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s3, 0
+; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-NEXT:    s_and_saveexec_b32 s2, vcc_lo
+; GFX1032-NEXT:    ; mask branch BB5_2
+; GFX1032-NEXT:    s_cbranch_execz BB5_2
+; GFX1032-NEXT:  BB5_1:
+; GFX1032-NEXT:    s_bcnt1_i32_b32 s3, s3
+; GFX1032-NEXT:    v_mov_b32_e32 v3, local_var64@abs32@lo
+; GFX1032-NEXT:    v_mul_hi_u32_u24_e64 v2, s3, 5
+; GFX1032-NEXT:    v_mul_u32_u24_e64 v1, s3, 5
+; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1032-NEXT:    ds_add_rtn_u64 v[1:2], v3, v[1:2]
+; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1032-NEXT:    buffer_gl0_inv
+; GFX1032-NEXT:    buffer_gl1_inv
+; GFX1032-NEXT:  BB5_2:
+; GFX1032-NEXT:    v_nop
+; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT:    v_readfirstlane_b32 s2, v1
+; GFX1032-NEXT:    v_readfirstlane_b32 s3, v2
+; GFX1032-NEXT:    v_mad_u64_u32 v[0:1], s2, v0, 5, s[2:3]
+; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1032-NEXT:    s_mov_b32 s2, -1
+; GFX1032-NEXT:    s_nop 2
+; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX1032-NEXT:    s_endpgm
 entry:
   %old = atomicrmw add i64 addrspace(3)* @local_var64, i64 5 acq_rel
   store i64 %old, i64 addrspace(1)* %out
   ret void
 }
 
-; GCN-LABEL: add_i64_uniform:
-; GCN: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
-; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_lo:[0-9]+]], s[[exec_lo]], 0
-; GCN: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_hi:[0-9]+]], s[[exec_hi]], v[[mbcnt_lo]]
-; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc, 0, v[[mbcnt_hi]]
-; GCN: s_bcnt1_i32_b64 s{{[0-9]+}}, s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
-; GCN: ds_add_rtn_u64 v{{\[}}{{[0-9]+}}:{{[0-9]+}}{{\]}}, v{{[0-9]+}}, v{{\[}}{{[0-9]+}}:{{[0-9]+}}{{\]}}
 define amdgpu_kernel void @add_i64_uniform(i64 addrspace(1)* %out, i64 %additive) {
+;
+;
+; GFX7LESS-LABEL: add_i64_uniform:
+; GFX7LESS:       ; %bb.0: ; %entry
+; GFX7LESS-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; GFX7LESS-NEXT:    v_cmp_ne_u32_e64 s[6:7], 1, 0
+; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s6, 0
+; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s7, v0
+; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT:    ; implicit-def: $vgpr1_vgpr2
+; GFX7LESS-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX7LESS-NEXT:    ; mask branch BB6_2
+; GFX7LESS-NEXT:    s_cbranch_execz BB6_2
+; GFX7LESS-NEXT:  BB6_1:
+; GFX7LESS-NEXT:    s_bcnt1_i32_b64 s6, s[6:7]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, local_var64@abs32@lo
+; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT:    s_mul_i32 s7, s3, s6
+; GFX7LESS-NEXT:    v_mov_b32_e32 v1, s6
+; GFX7LESS-NEXT:    v_mul_hi_u32 v1, s2, v1
+; GFX7LESS-NEXT:    s_mul_i32 s6, s2, s6
+; GFX7LESS-NEXT:    v_add_i32_e32 v2, vcc, s7, v1
+; GFX7LESS-NEXT:    v_mov_b32_e32 v1, s6
+; GFX7LESS-NEXT:    s_mov_b32 m0, -1
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7LESS-NEXT:    ds_add_rtn_u64 v[1:2], v3, v[1:2]
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7LESS-NEXT:    buffer_wbinvl1
+; GFX7LESS-NEXT:  BB6_2:
+; GFX7LESS-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX7LESS-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7LESS-NEXT:    s_mov_b32 s6, -1
+; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT:    s_mov_b32 s4, s0
+; GFX7LESS-NEXT:    s_mov_b32 s5, s1
+; GFX7LESS-NEXT:    v_readfirstlane_b32 s0, v1
+; GFX7LESS-NEXT:    v_readfirstlane_b32 s1, v2
+; GFX7LESS-NEXT:    v_mul_lo_u32 v1, s3, v0
+; GFX7LESS-NEXT:    v_mul_hi_u32 v2, s2, v0
+; GFX7LESS-NEXT:    v_mul_lo_u32 v0, s2, v0
+; GFX7LESS-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7LESS-NEXT:    v_add_i32_e32 v0, vcc, s0, v0
+; GFX7LESS-NEXT:    v_addc_u32_e32 v1, vcc, v2, v1, vcc
+; GFX7LESS-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GFX7LESS-NEXT:    s_endpgm
+;
+; GFX8-LABEL: add_i64_uniform:
+; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX8-NEXT:    v_cmp_ne_u32_e64 s[6:7], 1, 0
+; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s7, v0
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX8-NEXT:    ; implicit-def: $vgpr1_vgpr2
+; GFX8-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX8-NEXT:    ; mask branch BB6_2
+; GFX8-NEXT:    s_cbranch_execz BB6_2
+; GFX8-NEXT:  BB6_1:
+; GFX8-NEXT:    s_bcnt1_i32_b64 s6, s[6:7]
+; GFX8-NEXT:    v_mov_b32_e32 v1, s6
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    v_mul_hi_u32 v1, s2, v1
+; GFX8-NEXT:    s_mul_i32 s7, s3, s6
+; GFX8-NEXT:    s_mul_i32 s6, s2, s6
+; GFX8-NEXT:    v_mov_b32_e32 v3, local_var64@abs32@lo
+; GFX8-NEXT:    v_add_u32_e32 v2, vcc, s7, v1
+; GFX8-NEXT:    v_mov_b32_e32 v1, s6
+; GFX8-NEXT:    s_mov_b32 m0, -1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    ds_add_rtn_u64 v[1:2], v3, v[1:2]
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:  BB6_2:
+; GFX8-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_mov_b32 s4, s0
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v1
+; GFX8-NEXT:    v_mul_lo_u32 v1, s3, v0
+; GFX8-NEXT:    v_mul_hi_u32 v3, s2, v0
+; GFX8-NEXT:    v_mul_lo_u32 v0, s2, v0
+; GFX8-NEXT:    s_mov_b32 s5, s1
+; GFX8-NEXT:    v_readfirstlane_b32 s1, v2
+; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v3, v1
+; GFX8-NEXT:    v_mov_b32_e32 v2, s1
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
+; GFX8-NEXT:    s_mov_b32 s7, 0xf000
+; GFX8-NEXT:    s_mov_b32 s6, -1
+; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, v2, v1, vcc
+; GFX8-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GFX8-NEXT:    s_endpgm
+;
+; GFX9-LABEL: add_i64_uniform:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-NEXT:    v_cmp_ne_u32_e64 s[6:7], 1, 0
+; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s7, v0
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT:    ; implicit-def: $vgpr1_vgpr2
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    ; mask branch BB6_2
+; GFX9-NEXT:    s_cbranch_execz BB6_2
+; GFX9-NEXT:  BB6_1:
+; GFX9-NEXT:    s_bcnt1_i32_b64 s6, s[6:7]
+; GFX9-NEXT:    v_mov_b32_e32 v1, s6
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mul_hi_u32 v2, s2, v1
+; GFX9-NEXT:    s_mul_i32 s7, s3, s6
+; GFX9-NEXT:    s_mul_i32 s6, s2, s6
+; GFX9-NEXT:    v_mov_b32_e32 v1, s6
+; GFX9-NEXT:    v_add_u32_e32 v2, s7, v2
+; GFX9-NEXT:    v_mov_b32_e32 v3, local_var64@abs32@lo
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    ds_add_rtn_u64 v[1:2], v3, v[1:2]
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:  BB6_2:
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mul_lo_u32 v3, s3, v0
+; GFX9-NEXT:    v_mul_hi_u32 v4, s2, v0
+; GFX9-NEXT:    v_mul_lo_u32 v0, s2, v0
+; GFX9-NEXT:    s_mov_b32 s4, s0
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v1
+; GFX9-NEXT:    s_mov_b32 s5, s1
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v2
+; GFX9-NEXT:    v_add_u32_e32 v1, v4, v3
+; GFX9-NEXT:    v_mov_b32_e32 v2, s1
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s0, v0
+; GFX9-NEXT:    s_mov_b32 s7, 0xf000
+; GFX9-NEXT:    s_mov_b32 s6, -1
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, v2, v1, vcc
+; GFX9-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GFX9-NEXT:    s_endpgm
+;
+; GFX1064-LABEL: add_i64_uniform:
+; GFX1064:       ; %bb.0: ; %entry
+; GFX1064-NEXT:    v_cmp_ne_u32_e64 s[6:7], 1, 0
+; GFX1064-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX1064-NEXT:    ; implicit-def: $vgpr1_vgpr2
+; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s6, 0
+; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, s7, v0
+; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX1064-NEXT:    ; mask branch BB6_2
+; GFX1064-NEXT:    s_cbranch_execz BB6_2
+; GFX1064-NEXT:  BB6_1:
+; GFX1064-NEXT:    s_bcnt1_i32_b64 s6, s[6:7]
+; GFX1064-NEXT:    v_mov_b32_e32 v3, local_var64@abs32@lo
+; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT:    v_mul_hi_u32 v2, s2, s6
+; GFX1064-NEXT:    s_mul_i32 s7, s2, s6
+; GFX1064-NEXT:    s_mul_i32 s6, s3, s6
+; GFX1064-NEXT:    v_mov_b32_e32 v1, s7
+; GFX1064-NEXT:    v_add_nc_u32_e32 v2, s6, v2
+; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1064-NEXT:    ds_add_rtn_u64 v[1:2], v3, v[1:2]
+; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1064-NEXT:    buffer_gl0_inv
+; GFX1064-NEXT:    buffer_gl1_inv
+; GFX1064-NEXT:  BB6_2:
+; GFX1064-NEXT:    v_nop
+; GFX1064-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT:    v_mul_lo_u32 v3, s3, v0
+; GFX1064-NEXT:    v_mul_hi_u32 v4, s2, v0
+; GFX1064-NEXT:    v_mul_lo_u32 v0, s2, v0
+; GFX1064-NEXT:    v_readfirstlane_b32 s4, v1
+; GFX1064-NEXT:    v_readfirstlane_b32 s5, v2
+; GFX1064-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1064-NEXT:    s_mov_b32 s2, -1
+; GFX1064-NEXT:    v_add_nc_u32_e32 v1, v4, v3
+; GFX1064-NEXT:    v_add_co_u32_e64 v0, vcc, s4, v0
+; GFX1064-NEXT:    v_add_co_ci_u32_e32 v1, vcc, s5, v1, vcc
+; GFX1064-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX1064-NEXT:    s_endpgm
+;
+; GFX1032-LABEL: add_i64_uniform:
+; GFX1032:       ; %bb.0: ; %entry
+; GFX1032-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX1032-NEXT:    v_cmp_ne_u32_e64 s5, 1, 0
+; GFX1032-NEXT:    ; implicit-def: $vcc_hi
+; GFX1032-NEXT:    ; implicit-def: $vgpr1_vgpr2
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s5, 0
+; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-NEXT:    s_and_saveexec_b32 s4, vcc_lo
+; GFX1032-NEXT:    ; mask branch BB6_2
+; GFX1032-NEXT:    s_cbranch_execz BB6_2
+; GFX1032-NEXT:  BB6_1:
+; GFX1032-NEXT:    s_bcnt1_i32_b32 s5, s5
+; GFX1032-NEXT:    v_mov_b32_e32 v3, local_var64@abs32@lo
+; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT:    v_mul_hi_u32 v2, s2, s5
+; GFX1032-NEXT:    s_mul_i32 s6, s2, s5
+; GFX1032-NEXT:    s_mul_i32 s5, s3, s5
+; GFX1032-NEXT:    v_mov_b32_e32 v1, s6
+; GFX1032-NEXT:    v_add_nc_u32_e32 v2, s5, v2
+; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1032-NEXT:    ds_add_rtn_u64 v[1:2], v3, v[1:2]
+; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1032-NEXT:    buffer_gl0_inv
+; GFX1032-NEXT:    buffer_gl1_inv
+; GFX1032-NEXT:  BB6_2:
+; GFX1032-NEXT:    v_nop
+; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT:    v_mul_lo_u32 v3, s3, v0
+; GFX1032-NEXT:    v_mul_hi_u32 v4, s2, v0
+; GFX1032-NEXT:    v_mul_lo_u32 v0, s2, v0
+; GFX1032-NEXT:    v_readfirstlane_b32 s4, v1
+; GFX1032-NEXT:    v_readfirstlane_b32 s5, v2
+; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1032-NEXT:    s_mov_b32 s2, -1
+; GFX1032-NEXT:    v_add_nc_u32_e32 v1, v4, v3
+; GFX1032-NEXT:    v_add_co_u32_e64 v0, vcc_lo, s4, v0
+; GFX1032-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, s5, v1, vcc_lo
+; GFX1032-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX1032-NEXT:    s_endpgm
 entry:
   %old = atomicrmw add i64 addrspace(3)* @local_var64, i64 %additive acq_rel
   store i64 %old, i64 addrspace(1)* %out
   ret void
 }
 
-; GCN-LABEL: add_i64_varying:
 ; GCN-NOT: v_mbcnt_lo_u32_b32
 ; GCN-NOT: v_mbcnt_hi_u32_b32
 ; GCN-NOT: s_bcnt1_i32_b64
-; GCN: ds_add_rtn_u64 v{{\[}}{{[0-9]+}}:{{[0-9]+}}{{\]}}, v{{[0-9]+}}, v{{\[}}{{[0-9]+}}:{{[0-9]+}}{{\]}}
 define amdgpu_kernel void @add_i64_varying(i64 addrspace(1)* %out) {
+;
+;
+; GFX7LESS-LABEL: add_i64_varying:
+; GFX7LESS:       ; %bb.0: ; %entry
+; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; GFX7LESS-NEXT:    v_mov_b32_e32 v1, 0
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, local_var64@abs32@lo
+; GFX7LESS-NEXT:    s_mov_b32 m0, -1
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7LESS-NEXT:    ds_add_rtn_u64 v[0:1], v2, v[0:1]
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7LESS-NEXT:    buffer_wbinvl1
+; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7LESS-NEXT:    s_mov_b32 s2, -1
+; GFX7LESS-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX7LESS-NEXT:    s_endpgm
+;
+; GFX8-LABEL: add_i64_varying:
+; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    v_mov_b32_e32 v1, 0
+; GFX8-NEXT:    v_mov_b32_e32 v2, local_var64@abs32@lo
+; GFX8-NEXT:    s_mov_b32 m0, -1
+; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    ds_add_rtn_u64 v[0:1], v2, v[0:1]
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    s_mov_b32 s3, 0xf000
+; GFX8-NEXT:    s_mov_b32 s2, -1
+; GFX8-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX8-NEXT:    s_endpgm
+;
+; GFX9-LABEL: add_i64_varying:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0
+; GFX9-NEXT:    v_mov_b32_e32 v2, local_var64@abs32@lo
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    ds_add_rtn_u64 v[0:1], v2, v[0:1]
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    s_mov_b32 s3, 0xf000
+; GFX9-NEXT:    s_mov_b32 s2, -1
+; GFX9-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX9-NEXT:    s_endpgm
+;
+; GFX1064-LABEL: add_i64_varying:
+; GFX1064:       ; %bb.0: ; %entry
+; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
+; GFX1064-NEXT:    v_mov_b32_e32 v2, local_var64@abs32@lo
+; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX1064-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1064-NEXT:    s_mov_b32 s2, -1
+; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1064-NEXT:    ds_add_rtn_u64 v[0:1], v2, v[0:1]
+; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1064-NEXT:    buffer_gl0_inv
+; GFX1064-NEXT:    buffer_gl1_inv
+; GFX1064-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX1064-NEXT:    s_endpgm
+;
+; GFX1032-LABEL: add_i64_varying:
+; GFX1032:       ; %bb.0: ; %entry
+; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
+; GFX1032-NEXT:    v_mov_b32_e32 v2, local_var64@abs32@lo
+; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1032-NEXT:    s_mov_b32 s2, -1
+; GFX1032-NEXT:    ; implicit-def: $vcc_hi
+; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1032-NEXT:    ds_add_rtn_u64 v[0:1], v2, v[0:1]
+; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1032-NEXT:    buffer_gl0_inv
+; GFX1032-NEXT:    buffer_gl1_inv
+; GFX1032-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX1032-NEXT:    s_endpgm
 entry:
   %lane = call i32 @llvm.amdgcn.workitem.id.x()
   %zext = zext i32 %lane to i64
@@ -102,48 +1596,601 @@ entry:
   ret void
 }
 
-; GCN-LABEL: sub_i32_constant:
-; GCN: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
-; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_lo:[0-9]+]], s[[exec_lo]], 0
-; GCN: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_hi:[0-9]+]], s[[exec_hi]], v[[mbcnt_lo]]
-; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc, 0, v[[mbcnt_hi]]
-; GCN: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
-; GCN: v_mul_u32_u24{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[popcount]], 5
-; GCN: ds_sub_rtn_u32 v{{[0-9]+}}, v{{[0-9]+}}, v[[value]]
 define amdgpu_kernel void @sub_i32_constant(i32 addrspace(1)* %out) {
+;
+;
+; GFX7LESS-LABEL: sub_i32_constant:
+; GFX7LESS:       ; %bb.0: ; %entry
+; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; GFX7LESS-NEXT:    v_cmp_ne_u32_e64 s[4:5], 1, 0
+; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s4, 0
+; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s5, v0
+; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT:    ; implicit-def: $vgpr1
+; GFX7LESS-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX7LESS-NEXT:    ; mask branch BB8_2
+; GFX7LESS-NEXT:    s_cbranch_execz BB8_2
+; GFX7LESS-NEXT:  BB8_1:
+; GFX7LESS-NEXT:    s_bcnt1_i32_b64 s4, s[4:5]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v1, local_var32@abs32@lo
+; GFX7LESS-NEXT:    v_mul_u32_u24_e64 v2, s4, 5
+; GFX7LESS-NEXT:    s_mov_b32 m0, -1
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7LESS-NEXT:    ds_sub_rtn_u32 v1, v1, v2
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7LESS-NEXT:    buffer_wbinvl1
+; GFX7LESS-NEXT:  BB8_2:
+; GFX7LESS-NEXT:    s_or_b64 exec, exec, s[2:3]
+; GFX7LESS-NEXT:    v_readfirstlane_b32 s2, v1
+; GFX7LESS-NEXT:    v_mul_u32_u24_e32 v0, 5, v0
+; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7LESS-NEXT:    v_sub_i32_e32 v0, vcc, s2, v0
+; GFX7LESS-NEXT:    s_mov_b32 s2, -1
+; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX7LESS-NEXT:    s_endpgm
+;
+; GFX8-LABEL: sub_i32_constant:
+; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX8-NEXT:    v_cmp_ne_u32_e64 s[4:5], 1, 0
+; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s4, 0
+; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s5, v0
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX8-NEXT:    ; implicit-def: $vgpr1
+; GFX8-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX8-NEXT:    ; mask branch BB8_2
+; GFX8-NEXT:    s_cbranch_execz BB8_2
+; GFX8-NEXT:  BB8_1:
+; GFX8-NEXT:    s_bcnt1_i32_b64 s4, s[4:5]
+; GFX8-NEXT:    v_mul_u32_u24_e64 v1, s4, 5
+; GFX8-NEXT:    v_mov_b32_e32 v2, local_var32@abs32@lo
+; GFX8-NEXT:    s_mov_b32 m0, -1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    ds_sub_rtn_u32 v1, v2, v1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:  BB8_2:
+; GFX8-NEXT:    s_or_b64 exec, exec, s[2:3]
+; GFX8-NEXT:    v_readfirstlane_b32 s2, v1
+; GFX8-NEXT:    v_mul_u32_u24_e32 v0, 5, v0
+; GFX8-NEXT:    v_sub_u32_e32 v0, vcc, s2, v0
+; GFX8-NEXT:    s_mov_b32 s3, 0xf000
+; GFX8-NEXT:    s_mov_b32 s2, -1
+; GFX8-NEXT:    s_nop 0
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX8-NEXT:    s_endpgm
+;
+; GFX9-LABEL: sub_i32_constant:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    v_cmp_ne_u32_e64 s[4:5], 1, 0
+; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s4, 0
+; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s5, v0
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT:    ; implicit-def: $vgpr1
+; GFX9-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX9-NEXT:    ; mask branch BB8_2
+; GFX9-NEXT:    s_cbranch_execz BB8_2
+; GFX9-NEXT:  BB8_1:
+; GFX9-NEXT:    s_bcnt1_i32_b64 s4, s[4:5]
+; GFX9-NEXT:    v_mul_u32_u24_e64 v1, s4, 5
+; GFX9-NEXT:    v_mov_b32_e32 v2, local_var32@abs32@lo
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    ds_sub_rtn_u32 v1, v2, v1
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:  BB8_2:
+; GFX9-NEXT:    s_or_b64 exec, exec, s[2:3]
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v1
+; GFX9-NEXT:    v_mul_u32_u24_e32 v0, 5, v0
+; GFX9-NEXT:    v_sub_u32_e32 v0, s2, v0
+; GFX9-NEXT:    s_mov_b32 s3, 0xf000
+; GFX9-NEXT:    s_mov_b32 s2, -1
+; GFX9-NEXT:    s_nop 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX9-NEXT:    s_endpgm
+;
+; GFX1064-LABEL: sub_i32_constant:
+; GFX1064:       ; %bb.0: ; %entry
+; GFX1064-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX1064-NEXT:    ; implicit-def: $vgpr1
+; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
+; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, s3, v0
+; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX1064-NEXT:    ; mask branch BB8_2
+; GFX1064-NEXT:    s_cbranch_execz BB8_2
+; GFX1064-NEXT:  BB8_1:
+; GFX1064-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
+; GFX1064-NEXT:    v_mov_b32_e32 v2, local_var32@abs32@lo
+; GFX1064-NEXT:    v_mul_u32_u24_e64 v1, s2, 5
+; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1064-NEXT:    ds_sub_rtn_u32 v1, v2, v1
+; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1064-NEXT:    buffer_gl0_inv
+; GFX1064-NEXT:    buffer_gl1_inv
+; GFX1064-NEXT:  BB8_2:
+; GFX1064-NEXT:    v_nop
+; GFX1064-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX1064-NEXT:    v_readfirstlane_b32 s2, v1
+; GFX1064-NEXT:    v_mul_u32_u24_e32 v0, 5, v0
+; GFX1064-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1064-NEXT:    v_sub_nc_u32_e32 v0, s2, v0
+; GFX1064-NEXT:    s_mov_b32 s2, -1
+; GFX1064-NEXT:    s_nop 0
+; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX1064-NEXT:    s_endpgm
+;
+; GFX1032-LABEL: sub_i32_constant:
+; GFX1032:       ; %bb.0: ; %entry
+; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX1032-NEXT:    v_cmp_ne_u32_e64 s3, 1, 0
+; GFX1032-NEXT:    ; implicit-def: $vcc_hi
+; GFX1032-NEXT:    ; implicit-def: $vgpr1
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s3, 0
+; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-NEXT:    s_and_saveexec_b32 s2, vcc_lo
+; GFX1032-NEXT:    ; mask branch BB8_2
+; GFX1032-NEXT:    s_cbranch_execz BB8_2
+; GFX1032-NEXT:  BB8_1:
+; GFX1032-NEXT:    s_bcnt1_i32_b32 s3, s3
+; GFX1032-NEXT:    v_mov_b32_e32 v2, local_var32@abs32@lo
+; GFX1032-NEXT:    v_mul_u32_u24_e64 v1, s3, 5
+; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1032-NEXT:    ds_sub_rtn_u32 v1, v2, v1
+; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1032-NEXT:    buffer_gl0_inv
+; GFX1032-NEXT:    buffer_gl1_inv
+; GFX1032-NEXT:  BB8_2:
+; GFX1032-NEXT:    v_nop
+; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT:    v_readfirstlane_b32 s2, v1
+; GFX1032-NEXT:    v_mul_u32_u24_e32 v0, 5, v0
+; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1032-NEXT:    v_sub_nc_u32_e32 v0, s2, v0
+; GFX1032-NEXT:    s_mov_b32 s2, -1
+; GFX1032-NEXT:    s_nop 0
+; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX1032-NEXT:    s_endpgm
 entry:
   %old = atomicrmw sub i32 addrspace(3)* @local_var32, i32 5 acq_rel
   store i32 %old, i32 addrspace(1)* %out
   ret void
 }
 
-; GCN-LABEL: sub_i32_uniform:
-; GCN: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
-; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_lo:[0-9]+]], s[[exec_lo]], 0
-; GCN: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_hi:[0-9]+]], s[[exec_hi]], v[[mbcnt_lo]]
-; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc, 0, v[[mbcnt_hi]]
-; GCN: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
-; GCN: s_mul_i32 s[[scalar_value:[0-9]+]], s{{[0-9]+}}, s[[popcount]]
-; GCN: v_mov_b32{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[scalar_value]]
-; GCN: ds_sub_rtn_u32 v{{[0-9]+}}, v{{[0-9]+}}, v[[value]]
 define amdgpu_kernel void @sub_i32_uniform(i32 addrspace(1)* %out, i32 %subitive) {
+;
+;
+; GFX7LESS-LABEL: sub_i32_uniform:
+; GFX7LESS:       ; %bb.0: ; %entry
+; GFX7LESS-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX7LESS-NEXT:    s_load_dword s2, s[0:1], 0xb
+; GFX7LESS-NEXT:    v_cmp_ne_u32_e64 s[6:7], 1, 0
+; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s6, 0
+; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s7, v0
+; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT:    ; implicit-def: $vgpr1
+; GFX7LESS-NEXT:    s_and_saveexec_b64 s[0:1], vcc
+; GFX7LESS-NEXT:    ; mask branch BB9_2
+; GFX7LESS-NEXT:    s_cbranch_execz BB9_2
+; GFX7LESS-NEXT:  BB9_1:
+; GFX7LESS-NEXT:    s_bcnt1_i32_b64 s3, s[6:7]
+; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT:    s_mul_i32 s3, s2, s3
+; GFX7LESS-NEXT:    v_mov_b32_e32 v1, local_var32@abs32@lo
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s3
+; GFX7LESS-NEXT:    s_mov_b32 m0, -1
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7LESS-NEXT:    ds_sub_rtn_u32 v1, v1, v2
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7LESS-NEXT:    buffer_wbinvl1
+; GFX7LESS-NEXT:  BB9_2:
+; GFX7LESS-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX7LESS-NEXT:    v_readfirstlane_b32 s0, v1
+; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT:    v_mul_lo_u32 v0, s2, v0
+; GFX7LESS-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7LESS-NEXT:    v_sub_i32_e32 v0, vcc, s0, v0
+; GFX7LESS-NEXT:    s_mov_b32 s6, -1
+; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX7LESS-NEXT:    s_endpgm
+;
+; GFX8-LABEL: sub_i32_uniform:
+; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
+; GFX8-NEXT:    s_load_dword s0, s[0:1], 0x2c
+; GFX8-NEXT:    v_cmp_ne_u32_e64 s[6:7], 1, 0
+; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s7, v0
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX8-NEXT:    ; implicit-def: $vgpr1
+; GFX8-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX8-NEXT:    ; mask branch BB9_2
+; GFX8-NEXT:    s_cbranch_execz BB9_2
+; GFX8-NEXT:  BB9_1:
+; GFX8-NEXT:    s_bcnt1_i32_b64 s1, s[6:7]
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_mul_i32 s1, s0, s1
+; GFX8-NEXT:    v_mov_b32_e32 v1, local_var32@abs32@lo
+; GFX8-NEXT:    v_mov_b32_e32 v2, s1
+; GFX8-NEXT:    s_mov_b32 m0, -1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    ds_sub_rtn_u32 v1, v1, v2
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:  BB9_2:
+; GFX8-NEXT:    s_or_b64 exec, exec, s[2:3]
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    v_mul_lo_u32 v0, s0, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v1
+; GFX8-NEXT:    s_mov_b32 s7, 0xf000
+; GFX8-NEXT:    s_mov_b32 s6, -1
+; GFX8-NEXT:    v_sub_u32_e32 v0, vcc, s0, v0
+; GFX8-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX8-NEXT:    s_endpgm
+;
+; GFX9-LABEL: sub_i32_uniform:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
+; GFX9-NEXT:    s_load_dword s0, s[0:1], 0x2c
+; GFX9-NEXT:    v_cmp_ne_u32_e64 s[6:7], 1, 0
+; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s7, v0
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT:    ; implicit-def: $vgpr1
+; GFX9-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX9-NEXT:    ; mask branch BB9_2
+; GFX9-NEXT:    s_cbranch_execz BB9_2
+; GFX9-NEXT:  BB9_1:
+; GFX9-NEXT:    s_bcnt1_i32_b64 s1, s[6:7]
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_mul_i32 s1, s0, s1
+; GFX9-NEXT:    v_mov_b32_e32 v1, local_var32@abs32@lo
+; GFX9-NEXT:    v_mov_b32_e32 v2, s1
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    ds_sub_rtn_u32 v1, v1, v2
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:  BB9_2:
+; GFX9-NEXT:    s_or_b64 exec, exec, s[2:3]
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mul_lo_u32 v0, s0, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v1
+; GFX9-NEXT:    s_mov_b32 s7, 0xf000
+; GFX9-NEXT:    s_mov_b32 s6, -1
+; GFX9-NEXT:    v_sub_u32_e32 v0, s0, v0
+; GFX9-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX9-NEXT:    s_endpgm
+;
+; GFX1064-LABEL: sub_i32_uniform:
+; GFX1064:       ; %bb.0: ; %entry
+; GFX1064-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
+; GFX1064-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX1064-NEXT:    s_load_dword s0, s[0:1], 0x2c
+; GFX1064-NEXT:    ; implicit-def: $vgpr1
+; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
+; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, s3, v0
+; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT:    s_and_saveexec_b64 s[6:7], vcc
+; GFX1064-NEXT:    ; mask branch BB9_2
+; GFX1064-NEXT:    s_cbranch_execz BB9_2
+; GFX1064-NEXT:  BB9_1:
+; GFX1064-NEXT:    s_bcnt1_i32_b64 s1, s[2:3]
+; GFX1064-NEXT:    v_mov_b32_e32 v1, local_var32@abs32@lo
+; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT:    s_mul_i32 s1, s0, s1
+; GFX1064-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1064-NEXT:    ds_sub_rtn_u32 v1, v1, v2
+; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1064-NEXT:    buffer_gl0_inv
+; GFX1064-NEXT:    buffer_gl1_inv
+; GFX1064-NEXT:  BB9_2:
+; GFX1064-NEXT:    v_nop
+; GFX1064-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT:    v_mul_lo_u32 v0, s0, v0
+; GFX1064-NEXT:    v_readfirstlane_b32 s0, v1
+; GFX1064-NEXT:    s_mov_b32 s7, 0x31016000
+; GFX1064-NEXT:    s_mov_b32 s6, -1
+; GFX1064-NEXT:    v_sub_nc_u32_e32 v0, s0, v0
+; GFX1064-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX1064-NEXT:    s_endpgm
+;
+; GFX1032-LABEL: sub_i32_uniform:
+; GFX1032:       ; %bb.0: ; %entry
+; GFX1032-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
+; GFX1032-NEXT:    s_load_dword s0, s[0:1], 0x2c
+; GFX1032-NEXT:    v_cmp_ne_u32_e64 s2, 1, 0
+; GFX1032-NEXT:    ; implicit-def: $vcc_hi
+; GFX1032-NEXT:    ; implicit-def: $vgpr1
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
+; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-NEXT:    s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-NEXT:    ; mask branch BB9_2
+; GFX1032-NEXT:    s_cbranch_execz BB9_2
+; GFX1032-NEXT:  BB9_1:
+; GFX1032-NEXT:    s_bcnt1_i32_b32 s2, s2
+; GFX1032-NEXT:    v_mov_b32_e32 v1, local_var32@abs32@lo
+; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT:    s_mul_i32 s2, s0, s2
+; GFX1032-NEXT:    v_mov_b32_e32 v2, s2
+; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1032-NEXT:    ds_sub_rtn_u32 v1, v1, v2
+; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1032-NEXT:    buffer_gl0_inv
+; GFX1032-NEXT:    buffer_gl1_inv
+; GFX1032-NEXT:  BB9_2:
+; GFX1032-NEXT:    v_nop
+; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s1
+; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT:    v_mul_lo_u32 v0, s0, v0
+; GFX1032-NEXT:    v_readfirstlane_b32 s0, v1
+; GFX1032-NEXT:    s_mov_b32 s7, 0x31016000
+; GFX1032-NEXT:    s_mov_b32 s6, -1
+; GFX1032-NEXT:    v_sub_nc_u32_e32 v0, s0, v0
+; GFX1032-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX1032-NEXT:    s_endpgm
 entry:
   %old = atomicrmw sub i32 addrspace(3)* @local_var32, i32 %subitive acq_rel
   store i32 %old, i32 addrspace(1)* %out
   ret void
 }
 
-; GCN-LABEL: sub_i32_varying:
 ; GFX7LESS-NOT: v_mbcnt_lo_u32_b32
 ; GFX7LESS-NOT: v_mbcnt_hi_u32_b32
 ; GFX7LESS-NOT: s_bcnt1_i32_b64
-; GFX7LESS: ds_sub_rtn_u32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; GFX8MORE: v_sub{{(rev)?}}_u32_dpp
-; GFX8MORE: v_sub{{(rev)?}}_u32_dpp
-; GFX8MORE: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 63
+; DPPCOMB: v_add_u32_dpp
+; DPPCOMB: v_add_u32_dpp
+; GFX8MORE32: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 31
 ; GFX8MORE: v_mov_b32{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[scalar_value]]
 ; GFX8MORE: ds_sub_rtn_u32 v{{[0-9]+}}, v{{[0-9]+}}, v[[value]]
 define amdgpu_kernel void @sub_i32_varying(i32 addrspace(1)* %out) {
+;
+;
+; GFX7LESS-LABEL: sub_i32_varying:
+; GFX7LESS:       ; %bb.0: ; %entry
+; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; GFX7LESS-NEXT:    v_mov_b32_e32 v1, local_var32@abs32@lo
+; GFX7LESS-NEXT:    s_mov_b32 m0, -1
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7LESS-NEXT:    ds_sub_rtn_u32 v0, v1, v0
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7LESS-NEXT:    buffer_wbinvl1
+; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7LESS-NEXT:    s_mov_b32 s2, -1
+; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX7LESS-NEXT:    s_endpgm
+;
+; GFX8-LABEL: sub_i32_varying:
+; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    v_mov_b32_e32 v2, v0
+; GFX8-NEXT:    s_or_saveexec_b64 s[2:3], -1
+; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX8-NEXT:    v_mov_b32_e32 v1, 0
+; GFX8-NEXT:    s_mov_b64 exec, s[2:3]
+; GFX8-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX8-NEXT:    s_not_b64 exec, exec
+; GFX8-NEXT:    v_mov_b32_e32 v2, 0
+; GFX8-NEXT:    s_not_b64 exec, exec
+; GFX8-NEXT:    s_or_saveexec_b64 s[4:5], -1
+; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX8-NEXT:    v_readlane_b32 s2, v2, 63
+; GFX8-NEXT:    s_nop 0
+; GFX8-NEXT:    v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf
+; GFX8-NEXT:    s_mov_b64 exec, s[4:5]
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX8-NEXT:    ; implicit-def: $vgpr0
+; GFX8-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX8-NEXT:    ; mask branch BB10_2
+; GFX8-NEXT:    s_cbranch_execz BB10_2
+; GFX8-NEXT:  BB10_1:
+; GFX8-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
+; GFX8-NEXT:    v_mov_b32_e32 v3, s2
+; GFX8-NEXT:    s_mov_b32 m0, -1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    ds_sub_rtn_u32 v0, v0, v3
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:  BB10_2:
+; GFX8-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT:    v_readfirstlane_b32 s2, v0
+; GFX8-NEXT:    v_mov_b32_e32 v0, v1
+; GFX8-NEXT:    v_sub_u32_e32 v0, vcc, s2, v0
+; GFX8-NEXT:    s_mov_b32 s3, 0xf000
+; GFX8-NEXT:    s_mov_b32 s2, -1
+; GFX8-NEXT:    s_nop 0
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX8-NEXT:    s_endpgm
+;
+; GFX9-LABEL: sub_i32_varying:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_or_saveexec_b64 s[2:3], -1
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0
+; GFX9-NEXT:    s_mov_b64 exec, s[2:3]
+; GFX9-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX9-NEXT:    s_not_b64 exec, exec
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-NEXT:    s_not_b64 exec, exec
+; GFX9-NEXT:    s_or_saveexec_b64 s[4:5], -1
+; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX9-NEXT:    v_readlane_b32 s2, v2, 63
+; GFX9-NEXT:    s_nop 0
+; GFX9-NEXT:    v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-NEXT:    s_mov_b64 exec, s[4:5]
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT:    ; implicit-def: $vgpr0
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    ; mask branch BB10_2
+; GFX9-NEXT:    s_cbranch_execz BB10_2
+; GFX9-NEXT:  BB10_1:
+; GFX9-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
+; GFX9-NEXT:    v_mov_b32_e32 v3, s2
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    ds_sub_rtn_u32 v0, v0, v3
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:  BB10_2:
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v0
+; GFX9-NEXT:    v_mov_b32_e32 v0, v1
+; GFX9-NEXT:    v_sub_u32_e32 v0, s2, v0
+; GFX9-NEXT:    s_mov_b32 s3, 0xf000
+; GFX9-NEXT:    s_mov_b32 s2, -1
+; GFX9-NEXT:    s_nop 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX9-NEXT:    s_endpgm
+;
+; GFX1064-LABEL: sub_i32_varying:
+; GFX1064:       ; %bb.0: ; %entry
+; GFX1064-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1064-NEXT:    s_or_saveexec_b64 s[2:3], -1
+; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
+; GFX1064-NEXT:    s_mov_b64 exec, s[2:3]
+; GFX1064-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
+; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, s3, v0
+; GFX1064-NEXT:    s_not_b64 exec, exec
+; GFX1064-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1064-NEXT:    s_not_b64 exec, exec
+; GFX1064-NEXT:    s_or_saveexec_b64 s[4:5], -1
+; GFX1064-NEXT:    v_add_nc_u32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX1064-NEXT:    v_add_nc_u32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX1064-NEXT:    v_add_nc_u32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX1064-NEXT:    v_add_nc_u32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX1064-NEXT:    v_mov_b32_e32 v3, v2
+; GFX1064-NEXT:    v_permlanex16_b32 v3, v3, -1, -1
+; GFX1064-NEXT:    v_add_nc_u32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
+; GFX1064-NEXT:    v_readlane_b32 s2, v2, 31
+; GFX1064-NEXT:    v_mov_b32_e32 v3, s2
+; GFX1064-NEXT:    v_add_nc_u32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf
+; GFX1064-NEXT:    v_readlane_b32 s2, v2, 15
+; GFX1064-NEXT:    v_mov_b32_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-NEXT:    v_readlane_b32 s3, v2, 31
+; GFX1064-NEXT:    v_readlane_b32 s6, v2, 47
+; GFX1064-NEXT:    v_writelane_b32 v1, s2, 16
+; GFX1064-NEXT:    s_mov_b32 s2, -1
+; GFX1064-NEXT:    v_writelane_b32 v1, s3, 32
+; GFX1064-NEXT:    v_readlane_b32 s3, v2, 63
+; GFX1064-NEXT:    v_writelane_b32 v1, s6, 48
+; GFX1064-NEXT:    s_mov_b64 exec, s[4:5]
+; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT:    ; implicit-def: $vgpr0
+; GFX1064-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX1064-NEXT:    ; mask branch BB10_2
+; GFX1064-NEXT:    s_cbranch_execz BB10_2
+; GFX1064-NEXT:  BB10_1:
+; GFX1064-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
+; GFX1064-NEXT:    v_mov_b32_e32 v7, s3
+; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1064-NEXT:    ds_sub_rtn_u32 v0, v0, v7
+; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1064-NEXT:    buffer_gl0_inv
+; GFX1064-NEXT:    buffer_gl1_inv
+; GFX1064-NEXT:  BB10_2:
+; GFX1064-NEXT:    v_nop
+; GFX1064-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX1064-NEXT:    v_readfirstlane_b32 s3, v0
+; GFX1064-NEXT:    v_mov_b32_e32 v0, v1
+; GFX1064-NEXT:    v_sub_nc_u32_e32 v0, s3, v0
+; GFX1064-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1064-NEXT:    s_nop 1
+; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX1064-NEXT:    s_endpgm
+;
+; GFX1032-LABEL: sub_i32_varying:
+; GFX1032:       ; %bb.0: ; %entry
+; GFX1032-NEXT:    ; implicit-def: $vcc_hi
+; GFX1032-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1032-NEXT:    s_or_saveexec_b32 s2, -1
+; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
+; GFX1032-NEXT:    s_mov_b32 exec_lo, s2
+; GFX1032-NEXT:    v_cmp_ne_u32_e64 s2, 1, 0
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
+; GFX1032-NEXT:    s_not_b32 exec_lo, exec_lo
+; GFX1032-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1032-NEXT:    s_not_b32 exec_lo, exec_lo
+; GFX1032-NEXT:    s_or_saveexec_b32 s4, -1
+; GFX1032-NEXT:    v_add_nc_u32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX1032-NEXT:    s_mov_b32 s2, -1
+; GFX1032-NEXT:    v_add_nc_u32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX1032-NEXT:    v_add_nc_u32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX1032-NEXT:    v_add_nc_u32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX1032-NEXT:    v_mov_b32_e32 v3, v2
+; GFX1032-NEXT:    v_permlanex16_b32 v3, v3, -1, -1
+; GFX1032-NEXT:    v_add_nc_u32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
+; GFX1032-NEXT:    v_readlane_b32 s3, v2, 31
+; GFX1032-NEXT:    v_mov_b32_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-NEXT:    v_readlane_b32 s5, v2, 15
+; GFX1032-NEXT:    v_writelane_b32 v1, s5, 16
+; GFX1032-NEXT:    s_mov_b32 exec_lo, s4
+; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-NEXT:    ; implicit-def: $vgpr0
+; GFX1032-NEXT:    s_and_saveexec_b32 s4, vcc_lo
+; GFX1032-NEXT:    ; mask branch BB10_2
+; GFX1032-NEXT:    s_cbranch_execz BB10_2
+; GFX1032-NEXT:  BB10_1:
+; GFX1032-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
+; GFX1032-NEXT:    v_mov_b32_e32 v7, s3
+; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1032-NEXT:    ds_sub_rtn_u32 v0, v0, v7
+; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1032-NEXT:    buffer_gl0_inv
+; GFX1032-NEXT:    buffer_gl1_inv
+; GFX1032-NEXT:  BB10_2:
+; GFX1032-NEXT:    v_nop
+; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX1032-NEXT:    v_readfirstlane_b32 s3, v0
+; GFX1032-NEXT:    v_mov_b32_e32 v0, v1
+; GFX1032-NEXT:    v_sub_nc_u32_e32 v0, s3, v0
+; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1032-NEXT:    s_nop 1
+; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX1032-NEXT:    s_endpgm
 entry:
   %lane = call i32 @llvm.amdgcn.workitem.id.x()
   %old = atomicrmw sub i32 addrspace(3)* @local_var32, i32 %lane acq_rel
@@ -151,42 +2198,505 @@ entry:
   ret void
 }
 
-; GCN-LABEL: sub_i64_constant:
-; GCN: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
-; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_lo:[0-9]+]], s[[exec_lo]], 0
-; GCN: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_hi:[0-9]+]], s[[exec_hi]], v[[mbcnt_lo]]
-; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc, 0, v[[mbcnt_hi]]
-; GCN: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
-; GCN: v_mul_hi_u32_u24{{(_e[0-9]+)?}} v[[value_hi:[0-9]+]], s[[popcount]], 5
-; GCN: v_mul_u32_u24{{(_e[0-9]+)?}} v[[value_lo:[0-9]+]], s[[popcount]], 5
-; GCN: ds_sub_rtn_u64 v{{\[}}{{[0-9]+}}:{{[0-9]+}}{{\]}}, v{{[0-9]+}}, v{{\[}}[[value_lo]]:[[value_hi]]{{\]}}
 define amdgpu_kernel void @sub_i64_constant(i64 addrspace(1)* %out) {
+;
+;
+; GFX7LESS-LABEL: sub_i64_constant:
+; GFX7LESS:       ; %bb.0: ; %entry
+; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; GFX7LESS-NEXT:    v_cmp_ne_u32_e64 s[4:5], 1, 0
+; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s4, 0
+; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s5, v0
+; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT:    ; implicit-def: $vgpr1_vgpr2
+; GFX7LESS-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX7LESS-NEXT:    ; mask branch BB11_2
+; GFX7LESS-NEXT:    s_cbranch_execz BB11_2
+; GFX7LESS-NEXT:  BB11_1:
+; GFX7LESS-NEXT:    s_bcnt1_i32_b64 s4, s[4:5]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, local_var64@abs32@lo
+; GFX7LESS-NEXT:    v_mul_hi_u32_u24_e64 v2, s4, 5
+; GFX7LESS-NEXT:    v_mul_u32_u24_e64 v1, s4, 5
+; GFX7LESS-NEXT:    s_mov_b32 m0, -1
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7LESS-NEXT:    ds_sub_rtn_u64 v[1:2], v3, v[1:2]
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7LESS-NEXT:    buffer_wbinvl1
+; GFX7LESS-NEXT:  BB11_2:
+; GFX7LESS-NEXT:    s_or_b64 exec, exec, s[2:3]
+; GFX7LESS-NEXT:    v_readfirstlane_b32 s2, v1
+; GFX7LESS-NEXT:    v_readfirstlane_b32 s4, v2
+; GFX7LESS-NEXT:    v_mul_hi_u32_u24_e32 v1, 5, v0
+; GFX7LESS-NEXT:    v_mul_u32_u24_e32 v0, 5, v0
+; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s4
+; GFX7LESS-NEXT:    v_sub_i32_e32 v0, vcc, s2, v0
+; GFX7LESS-NEXT:    v_subb_u32_e32 v1, vcc, v2, v1, vcc
+; GFX7LESS-NEXT:    s_mov_b32 s2, -1
+; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX7LESS-NEXT:    s_endpgm
+;
+; GFX8-LABEL: sub_i64_constant:
+; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX8-NEXT:    v_cmp_ne_u32_e64 s[4:5], 1, 0
+; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s4, 0
+; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s5, v0
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX8-NEXT:    ; implicit-def: $vgpr1_vgpr2
+; GFX8-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX8-NEXT:    ; mask branch BB11_2
+; GFX8-NEXT:    s_cbranch_execz BB11_2
+; GFX8-NEXT:  BB11_1:
+; GFX8-NEXT:    s_bcnt1_i32_b64 s4, s[4:5]
+; GFX8-NEXT:    v_mul_hi_u32_u24_e64 v2, s4, 5
+; GFX8-NEXT:    v_mul_u32_u24_e64 v1, s4, 5
+; GFX8-NEXT:    v_mov_b32_e32 v3, local_var64@abs32@lo
+; GFX8-NEXT:    s_mov_b32 m0, -1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    ds_sub_rtn_u64 v[1:2], v3, v[1:2]
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:  BB11_2:
+; GFX8-NEXT:    s_or_b64 exec, exec, s[2:3]
+; GFX8-NEXT:    v_readfirstlane_b32 s3, v2
+; GFX8-NEXT:    v_readfirstlane_b32 s2, v1
+; GFX8-NEXT:    v_mul_hi_u32_u24_e32 v1, 5, v0
+; GFX8-NEXT:    v_mul_u32_u24_e32 v0, 5, v0
+; GFX8-NEXT:    v_mov_b32_e32 v2, s3
+; GFX8-NEXT:    v_sub_u32_e32 v0, vcc, s2, v0
+; GFX8-NEXT:    v_subb_u32_e32 v1, vcc, v2, v1, vcc
+; GFX8-NEXT:    s_mov_b32 s3, 0xf000
+; GFX8-NEXT:    s_mov_b32 s2, -1
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX8-NEXT:    s_endpgm
+;
+; GFX9-LABEL: sub_i64_constant:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    v_cmp_ne_u32_e64 s[4:5], 1, 0
+; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s4, 0
+; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s5, v0
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT:    ; implicit-def: $vgpr1_vgpr2
+; GFX9-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX9-NEXT:    ; mask branch BB11_2
+; GFX9-NEXT:    s_cbranch_execz BB11_2
+; GFX9-NEXT:  BB11_1:
+; GFX9-NEXT:    s_bcnt1_i32_b64 s4, s[4:5]
+; GFX9-NEXT:    v_mul_hi_u32_u24_e64 v2, s4, 5
+; GFX9-NEXT:    v_mul_u32_u24_e64 v1, s4, 5
+; GFX9-NEXT:    v_mov_b32_e32 v3, local_var64@abs32@lo
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    ds_sub_rtn_u64 v[1:2], v3, v[1:2]
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:  BB11_2:
+; GFX9-NEXT:    s_or_b64 exec, exec, s[2:3]
+; GFX9-NEXT:    v_readfirstlane_b32 s3, v2
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v1
+; GFX9-NEXT:    v_mul_hi_u32_u24_e32 v1, 5, v0
+; GFX9-NEXT:    v_mul_u32_u24_e32 v0, 5, v0
+; GFX9-NEXT:    v_mov_b32_e32 v2, s3
+; GFX9-NEXT:    v_sub_co_u32_e32 v0, vcc, s2, v0
+; GFX9-NEXT:    v_subb_co_u32_e32 v1, vcc, v2, v1, vcc
+; GFX9-NEXT:    s_mov_b32 s3, 0xf000
+; GFX9-NEXT:    s_mov_b32 s2, -1
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX9-NEXT:    s_endpgm
+;
+; GFX1064-LABEL: sub_i64_constant:
+; GFX1064:       ; %bb.0: ; %entry
+; GFX1064-NEXT:    v_cmp_ne_u32_e64 s[4:5], 1, 0
+; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX1064-NEXT:    ; implicit-def: $vgpr1_vgpr2
+; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s4, 0
+; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, s5, v0
+; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX1064-NEXT:    ; mask branch BB11_2
+; GFX1064-NEXT:    s_cbranch_execz BB11_2
+; GFX1064-NEXT:  BB11_1:
+; GFX1064-NEXT:    s_bcnt1_i32_b64 s4, s[4:5]
+; GFX1064-NEXT:    v_mov_b32_e32 v3, local_var64@abs32@lo
+; GFX1064-NEXT:    v_mul_hi_u32_u24_e64 v2, s4, 5
+; GFX1064-NEXT:    v_mul_u32_u24_e64 v1, s4, 5
+; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1064-NEXT:    ds_sub_rtn_u64 v[1:2], v3, v[1:2]
+; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1064-NEXT:    buffer_gl0_inv
+; GFX1064-NEXT:    buffer_gl1_inv
+; GFX1064-NEXT:  BB11_2:
+; GFX1064-NEXT:    v_nop
+; GFX1064-NEXT:    s_or_b64 exec, exec, s[2:3]
+; GFX1064-NEXT:    v_readfirstlane_b32 s2, v1
+; GFX1064-NEXT:    v_mul_u32_u24_e32 v1, 5, v0
+; GFX1064-NEXT:    v_readfirstlane_b32 s3, v2
+; GFX1064-NEXT:    v_mul_hi_u32_u24_e32 v2, 5, v0
+; GFX1064-NEXT:    v_sub_co_u32_e64 v0, vcc, s2, v1
+; GFX1064-NEXT:    s_mov_b32 s2, -1
+; GFX1064-NEXT:    v_sub_co_ci_u32_e32 v1, vcc, s3, v2, vcc
+; GFX1064-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX1064-NEXT:    s_endpgm
+;
+; GFX1032-LABEL: sub_i64_constant:
+; GFX1032:       ; %bb.0: ; %entry
+; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX1032-NEXT:    v_cmp_ne_u32_e64 s3, 1, 0
+; GFX1032-NEXT:    ; implicit-def: $vcc_hi
+; GFX1032-NEXT:    ; implicit-def: $vgpr1_vgpr2
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s3, 0
+; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-NEXT:    s_and_saveexec_b32 s2, vcc_lo
+; GFX1032-NEXT:    ; mask branch BB11_2
+; GFX1032-NEXT:    s_cbranch_execz BB11_2
+; GFX1032-NEXT:  BB11_1:
+; GFX1032-NEXT:    s_bcnt1_i32_b32 s3, s3
+; GFX1032-NEXT:    v_mov_b32_e32 v3, local_var64@abs32@lo
+; GFX1032-NEXT:    v_mul_hi_u32_u24_e64 v2, s3, 5
+; GFX1032-NEXT:    v_mul_u32_u24_e64 v1, s3, 5
+; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1032-NEXT:    ds_sub_rtn_u64 v[1:2], v3, v[1:2]
+; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1032-NEXT:    buffer_gl0_inv
+; GFX1032-NEXT:    buffer_gl1_inv
+; GFX1032-NEXT:  BB11_2:
+; GFX1032-NEXT:    v_nop
+; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT:    v_readfirstlane_b32 s2, v1
+; GFX1032-NEXT:    v_mul_u32_u24_e32 v1, 5, v0
+; GFX1032-NEXT:    v_readfirstlane_b32 s3, v2
+; GFX1032-NEXT:    v_mul_hi_u32_u24_e32 v2, 5, v0
+; GFX1032-NEXT:    v_sub_co_u32_e64 v0, vcc_lo, s2, v1
+; GFX1032-NEXT:    s_mov_b32 s2, -1
+; GFX1032-NEXT:    v_sub_co_ci_u32_e32 v1, vcc_lo, s3, v2, vcc_lo
+; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX1032-NEXT:    s_endpgm
 entry:
   %old = atomicrmw sub i64 addrspace(3)* @local_var64, i64 5 acq_rel
   store i64 %old, i64 addrspace(1)* %out
   ret void
 }
 
-; GCN-LABEL: sub_i64_uniform:
-; GCN: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
-; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_lo:[0-9]+]], s[[exec_lo]], 0
-; GCN: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_hi:[0-9]+]], s[[exec_hi]], v[[mbcnt_lo]]
-; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc, 0, v[[mbcnt_hi]]
-; GCN: s_bcnt1_i32_b64 s{{[0-9]+}}, s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
-; GCN: ds_sub_rtn_u64 v{{\[}}{{[0-9]+}}:{{[0-9]+}}{{\]}}, v{{[0-9]+}}, v{{\[}}{{[0-9]+}}:{{[0-9]+}}{{\]}}
 define amdgpu_kernel void @sub_i64_uniform(i64 addrspace(1)* %out, i64 %subitive) {
+;
+;
+; GFX7LESS-LABEL: sub_i64_uniform:
+; GFX7LESS:       ; %bb.0: ; %entry
+; GFX7LESS-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; GFX7LESS-NEXT:    v_cmp_ne_u32_e64 s[6:7], 1, 0
+; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s6, 0
+; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s7, v0
+; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT:    ; implicit-def: $vgpr1_vgpr2
+; GFX7LESS-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX7LESS-NEXT:    ; mask branch BB12_2
+; GFX7LESS-NEXT:    s_cbranch_execz BB12_2
+; GFX7LESS-NEXT:  BB12_1:
+; GFX7LESS-NEXT:    s_bcnt1_i32_b64 s6, s[6:7]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, local_var64@abs32@lo
+; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT:    s_mul_i32 s7, s3, s6
+; GFX7LESS-NEXT:    v_mov_b32_e32 v1, s6
+; GFX7LESS-NEXT:    v_mul_hi_u32 v1, s2, v1
+; GFX7LESS-NEXT:    s_mul_i32 s6, s2, s6
+; GFX7LESS-NEXT:    v_add_i32_e32 v2, vcc, s7, v1
+; GFX7LESS-NEXT:    v_mov_b32_e32 v1, s6
+; GFX7LESS-NEXT:    s_mov_b32 m0, -1
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7LESS-NEXT:    ds_sub_rtn_u64 v[1:2], v3, v[1:2]
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7LESS-NEXT:    buffer_wbinvl1
+; GFX7LESS-NEXT:  BB12_2:
+; GFX7LESS-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX7LESS-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7LESS-NEXT:    s_mov_b32 s6, -1
+; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT:    s_mov_b32 s4, s0
+; GFX7LESS-NEXT:    s_mov_b32 s5, s1
+; GFX7LESS-NEXT:    v_readfirstlane_b32 s0, v1
+; GFX7LESS-NEXT:    v_readfirstlane_b32 s1, v2
+; GFX7LESS-NEXT:    v_mul_lo_u32 v1, s3, v0
+; GFX7LESS-NEXT:    v_mul_hi_u32 v2, s2, v0
+; GFX7LESS-NEXT:    v_mul_lo_u32 v0, s2, v0
+; GFX7LESS-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7LESS-NEXT:    v_sub_i32_e32 v0, vcc, s0, v0
+; GFX7LESS-NEXT:    v_subb_u32_e32 v1, vcc, v2, v1, vcc
+; GFX7LESS-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GFX7LESS-NEXT:    s_endpgm
+;
+; GFX8-LABEL: sub_i64_uniform:
+; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX8-NEXT:    v_cmp_ne_u32_e64 s[6:7], 1, 0
+; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s7, v0
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX8-NEXT:    ; implicit-def: $vgpr1_vgpr2
+; GFX8-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX8-NEXT:    ; mask branch BB12_2
+; GFX8-NEXT:    s_cbranch_execz BB12_2
+; GFX8-NEXT:  BB12_1:
+; GFX8-NEXT:    s_bcnt1_i32_b64 s6, s[6:7]
+; GFX8-NEXT:    v_mov_b32_e32 v1, s6
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    v_mul_hi_u32 v1, s2, v1
+; GFX8-NEXT:    s_mul_i32 s7, s3, s6
+; GFX8-NEXT:    s_mul_i32 s6, s2, s6
+; GFX8-NEXT:    v_mov_b32_e32 v3, local_var64@abs32@lo
+; GFX8-NEXT:    v_add_u32_e32 v2, vcc, s7, v1
+; GFX8-NEXT:    v_mov_b32_e32 v1, s6
+; GFX8-NEXT:    s_mov_b32 m0, -1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    ds_sub_rtn_u64 v[1:2], v3, v[1:2]
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:  BB12_2:
+; GFX8-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_mov_b32 s4, s0
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v1
+; GFX8-NEXT:    v_mul_lo_u32 v1, s3, v0
+; GFX8-NEXT:    v_mul_hi_u32 v3, s2, v0
+; GFX8-NEXT:    v_mul_lo_u32 v0, s2, v0
+; GFX8-NEXT:    s_mov_b32 s5, s1
+; GFX8-NEXT:    v_readfirstlane_b32 s1, v2
+; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v3, v1
+; GFX8-NEXT:    v_mov_b32_e32 v2, s1
+; GFX8-NEXT:    v_sub_u32_e32 v0, vcc, s0, v0
+; GFX8-NEXT:    s_mov_b32 s7, 0xf000
+; GFX8-NEXT:    s_mov_b32 s6, -1
+; GFX8-NEXT:    v_subb_u32_e32 v1, vcc, v2, v1, vcc
+; GFX8-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GFX8-NEXT:    s_endpgm
+;
+; GFX9-LABEL: sub_i64_uniform:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-NEXT:    v_cmp_ne_u32_e64 s[6:7], 1, 0
+; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s7, v0
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT:    ; implicit-def: $vgpr1_vgpr2
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    ; mask branch BB12_2
+; GFX9-NEXT:    s_cbranch_execz BB12_2
+; GFX9-NEXT:  BB12_1:
+; GFX9-NEXT:    s_bcnt1_i32_b64 s6, s[6:7]
+; GFX9-NEXT:    v_mov_b32_e32 v1, s6
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mul_hi_u32 v2, s2, v1
+; GFX9-NEXT:    s_mul_i32 s7, s3, s6
+; GFX9-NEXT:    s_mul_i32 s6, s2, s6
+; GFX9-NEXT:    v_mov_b32_e32 v1, s6
+; GFX9-NEXT:    v_add_u32_e32 v2, s7, v2
+; GFX9-NEXT:    v_mov_b32_e32 v3, local_var64@abs32@lo
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    ds_sub_rtn_u64 v[1:2], v3, v[1:2]
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:  BB12_2:
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mul_lo_u32 v3, s3, v0
+; GFX9-NEXT:    v_mul_hi_u32 v4, s2, v0
+; GFX9-NEXT:    v_mul_lo_u32 v0, s2, v0
+; GFX9-NEXT:    s_mov_b32 s4, s0
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v1
+; GFX9-NEXT:    s_mov_b32 s5, s1
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v2
+; GFX9-NEXT:    v_add_u32_e32 v1, v4, v3
+; GFX9-NEXT:    v_mov_b32_e32 v2, s1
+; GFX9-NEXT:    v_sub_co_u32_e32 v0, vcc, s0, v0
+; GFX9-NEXT:    s_mov_b32 s7, 0xf000
+; GFX9-NEXT:    s_mov_b32 s6, -1
+; GFX9-NEXT:    v_subb_co_u32_e32 v1, vcc, v2, v1, vcc
+; GFX9-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GFX9-NEXT:    s_endpgm
+;
+; GFX1064-LABEL: sub_i64_uniform:
+; GFX1064:       ; %bb.0: ; %entry
+; GFX1064-NEXT:    v_cmp_ne_u32_e64 s[6:7], 1, 0
+; GFX1064-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX1064-NEXT:    ; implicit-def: $vgpr1_vgpr2
+; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s6, 0
+; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, s7, v0
+; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX1064-NEXT:    ; mask branch BB12_2
+; GFX1064-NEXT:    s_cbranch_execz BB12_2
+; GFX1064-NEXT:  BB12_1:
+; GFX1064-NEXT:    s_bcnt1_i32_b64 s6, s[6:7]
+; GFX1064-NEXT:    v_mov_b32_e32 v3, local_var64@abs32@lo
+; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT:    v_mul_hi_u32 v2, s2, s6
+; GFX1064-NEXT:    s_mul_i32 s7, s2, s6
+; GFX1064-NEXT:    s_mul_i32 s6, s3, s6
+; GFX1064-NEXT:    v_mov_b32_e32 v1, s7
+; GFX1064-NEXT:    v_add_nc_u32_e32 v2, s6, v2
+; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1064-NEXT:    ds_sub_rtn_u64 v[1:2], v3, v[1:2]
+; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1064-NEXT:    buffer_gl0_inv
+; GFX1064-NEXT:    buffer_gl1_inv
+; GFX1064-NEXT:  BB12_2:
+; GFX1064-NEXT:    v_nop
+; GFX1064-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT:    v_mul_lo_u32 v3, s3, v0
+; GFX1064-NEXT:    v_mul_hi_u32 v4, s2, v0
+; GFX1064-NEXT:    v_mul_lo_u32 v0, s2, v0
+; GFX1064-NEXT:    v_readfirstlane_b32 s4, v1
+; GFX1064-NEXT:    v_readfirstlane_b32 s5, v2
+; GFX1064-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1064-NEXT:    s_mov_b32 s2, -1
+; GFX1064-NEXT:    v_add_nc_u32_e32 v1, v4, v3
+; GFX1064-NEXT:    v_sub_co_u32_e64 v0, vcc, s4, v0
+; GFX1064-NEXT:    v_sub_co_ci_u32_e32 v1, vcc, s5, v1, vcc
+; GFX1064-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX1064-NEXT:    s_endpgm
+;
+; GFX1032-LABEL: sub_i64_uniform:
+; GFX1032:       ; %bb.0: ; %entry
+; GFX1032-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX1032-NEXT:    v_cmp_ne_u32_e64 s5, 1, 0
+; GFX1032-NEXT:    ; implicit-def: $vcc_hi
+; GFX1032-NEXT:    ; implicit-def: $vgpr1_vgpr2
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s5, 0
+; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-NEXT:    s_and_saveexec_b32 s4, vcc_lo
+; GFX1032-NEXT:    ; mask branch BB12_2
+; GFX1032-NEXT:    s_cbranch_execz BB12_2
+; GFX1032-NEXT:  BB12_1:
+; GFX1032-NEXT:    s_bcnt1_i32_b32 s5, s5
+; GFX1032-NEXT:    v_mov_b32_e32 v3, local_var64@abs32@lo
+; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT:    v_mul_hi_u32 v2, s2, s5
+; GFX1032-NEXT:    s_mul_i32 s6, s2, s5
+; GFX1032-NEXT:    s_mul_i32 s5, s3, s5
+; GFX1032-NEXT:    v_mov_b32_e32 v1, s6
+; GFX1032-NEXT:    v_add_nc_u32_e32 v2, s5, v2
+; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1032-NEXT:    ds_sub_rtn_u64 v[1:2], v3, v[1:2]
+; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1032-NEXT:    buffer_gl0_inv
+; GFX1032-NEXT:    buffer_gl1_inv
+; GFX1032-NEXT:  BB12_2:
+; GFX1032-NEXT:    v_nop
+; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT:    v_mul_lo_u32 v3, s3, v0
+; GFX1032-NEXT:    v_mul_hi_u32 v4, s2, v0
+; GFX1032-NEXT:    v_mul_lo_u32 v0, s2, v0
+; GFX1032-NEXT:    v_readfirstlane_b32 s4, v1
+; GFX1032-NEXT:    v_readfirstlane_b32 s5, v2
+; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1032-NEXT:    s_mov_b32 s2, -1
+; GFX1032-NEXT:    v_add_nc_u32_e32 v1, v4, v3
+; GFX1032-NEXT:    v_sub_co_u32_e64 v0, vcc_lo, s4, v0
+; GFX1032-NEXT:    v_sub_co_ci_u32_e32 v1, vcc_lo, s5, v1, vcc_lo
+; GFX1032-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX1032-NEXT:    s_endpgm
 entry:
   %old = atomicrmw sub i64 addrspace(3)* @local_var64, i64 %subitive acq_rel
   store i64 %old, i64 addrspace(1)* %out
   ret void
 }
 
-; GCN-LABEL: sub_i64_varying:
 ; GCN-NOT: v_mbcnt_lo_u32_b32
 ; GCN-NOT: v_mbcnt_hi_u32_b32
 ; GCN-NOT: s_bcnt1_i32_b64
-; GCN: ds_sub_rtn_u64 v{{\[}}{{[0-9]+}}:{{[0-9]+}}{{\]}}, v{{[0-9]+}}, v{{\[}}{{[0-9]+}}:{{[0-9]+}}{{\]}}
 define amdgpu_kernel void @sub_i64_varying(i64 addrspace(1)* %out) {
+;
+;
+; GFX7LESS-LABEL: sub_i64_varying:
+; GFX7LESS:       ; %bb.0: ; %entry
+; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; GFX7LESS-NEXT:    v_mov_b32_e32 v1, 0
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, local_var64@abs32@lo
+; GFX7LESS-NEXT:    s_mov_b32 m0, -1
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7LESS-NEXT:    ds_sub_rtn_u64 v[0:1], v2, v[0:1]
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7LESS-NEXT:    buffer_wbinvl1
+; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7LESS-NEXT:    s_mov_b32 s2, -1
+; GFX7LESS-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX7LESS-NEXT:    s_endpgm
+;
+; GFX8-LABEL: sub_i64_varying:
+; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    v_mov_b32_e32 v1, 0
+; GFX8-NEXT:    v_mov_b32_e32 v2, local_var64@abs32@lo
+; GFX8-NEXT:    s_mov_b32 m0, -1
+; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    ds_sub_rtn_u64 v[0:1], v2, v[0:1]
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    s_mov_b32 s3, 0xf000
+; GFX8-NEXT:    s_mov_b32 s2, -1
+; GFX8-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX8-NEXT:    s_endpgm
+;
+; GFX9-LABEL: sub_i64_varying:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0
+; GFX9-NEXT:    v_mov_b32_e32 v2, local_var64@abs32@lo
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    ds_sub_rtn_u64 v[0:1], v2, v[0:1]
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    s_mov_b32 s3, 0xf000
+; GFX9-NEXT:    s_mov_b32 s2, -1
+; GFX9-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX9-NEXT:    s_endpgm
+;
+; GFX1064-LABEL: sub_i64_varying:
+; GFX1064:       ; %bb.0: ; %entry
+; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
+; GFX1064-NEXT:    v_mov_b32_e32 v2, local_var64@abs32@lo
+; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX1064-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1064-NEXT:    s_mov_b32 s2, -1
+; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1064-NEXT:    ds_sub_rtn_u64 v[0:1], v2, v[0:1]
+; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1064-NEXT:    buffer_gl0_inv
+; GFX1064-NEXT:    buffer_gl1_inv
+; GFX1064-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX1064-NEXT:    s_endpgm
+;
+; GFX1032-LABEL: sub_i64_varying:
+; GFX1032:       ; %bb.0: ; %entry
+; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
+; GFX1032-NEXT:    v_mov_b32_e32 v2, local_var64@abs32@lo
+; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1032-NEXT:    s_mov_b32 s2, -1
+; GFX1032-NEXT:    ; implicit-def: $vcc_hi
+; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1032-NEXT:    ds_sub_rtn_u64 v[0:1], v2, v[0:1]
+; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1032-NEXT:    buffer_gl0_inv
+; GFX1032-NEXT:    buffer_gl1_inv
+; GFX1032-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX1032-NEXT:    s_endpgm
 entry:
   %lane = call i32 @llvm.amdgcn.workitem.id.x()
   %zext = zext i32 %lane to i64
@@ -195,11 +2705,245 @@ entry:
   ret void
 }
 
-; GCN-LABEL: and_i32_varying:
-; GFX8MORE: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 63
+; GFX8MORE32: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 31
 ; GFX8MORE: v_mov_b32{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[scalar_value]]
 ; GFX8MORE: ds_and_rtn_b32 v{{[0-9]+}}, v{{[0-9]+}}, v[[value]]
 define amdgpu_kernel void @and_i32_varying(i32 addrspace(1)* %out) {
+;
+;
+; GFX7LESS-LABEL: and_i32_varying:
+; GFX7LESS:       ; %bb.0: ; %entry
+; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; GFX7LESS-NEXT:    v_mov_b32_e32 v1, local_var32@abs32@lo
+; GFX7LESS-NEXT:    s_mov_b32 m0, -1
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7LESS-NEXT:    ds_and_rtn_b32 v0, v1, v0
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7LESS-NEXT:    buffer_wbinvl1
+; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7LESS-NEXT:    s_mov_b32 s2, -1
+; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX7LESS-NEXT:    s_endpgm
+;
+; GFX8-LABEL: and_i32_varying:
+; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX8-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v3, s2, 0
+; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v3, s3, v3
+; GFX8-NEXT:    v_mov_b32_e32 v2, v0
+; GFX8-NEXT:    s_or_saveexec_b64 s[2:3], -1
+; GFX8-NEXT:    v_mov_b32_e32 v1, -1
+; GFX8-NEXT:    s_mov_b64 exec, s[2:3]
+; GFX8-NEXT:    s_not_b64 exec, exec
+; GFX8-NEXT:    v_mov_b32_e32 v2, -1
+; GFX8-NEXT:    s_not_b64 exec, exec
+; GFX8-NEXT:    s_or_saveexec_b64 s[4:5], -1
+; GFX8-NEXT:    v_and_b32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_and_b32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_and_b32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_and_b32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_and_b32_dpp v2, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_and_b32_dpp v2, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX8-NEXT:    v_readlane_b32 s2, v2, 63
+; GFX8-NEXT:    s_nop 0
+; GFX8-NEXT:    v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf
+; GFX8-NEXT:    s_mov_b64 exec, s[4:5]
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; GFX8-NEXT:    ; implicit-def: $vgpr0
+; GFX8-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX8-NEXT:    ; mask branch BB14_2
+; GFX8-NEXT:    s_cbranch_execz BB14_2
+; GFX8-NEXT:  BB14_1:
+; GFX8-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
+; GFX8-NEXT:    v_mov_b32_e32 v3, s2
+; GFX8-NEXT:    s_mov_b32 m0, -1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    ds_and_rtn_b32 v0, v0, v3
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:  BB14_2:
+; GFX8-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT:    v_readfirstlane_b32 s2, v0
+; GFX8-NEXT:    v_mov_b32_e32 v0, v1
+; GFX8-NEXT:    v_and_b32_e32 v0, s2, v0
+; GFX8-NEXT:    s_mov_b32 s3, 0xf000
+; GFX8-NEXT:    s_mov_b32 s2, -1
+; GFX8-NEXT:    s_nop 0
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX8-NEXT:    s_endpgm
+;
+; GFX9-LABEL: and_i32_varying:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v3, s2, 0
+; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v3, s3, v3
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_or_saveexec_b64 s[2:3], -1
+; GFX9-NEXT:    v_mov_b32_e32 v1, -1
+; GFX9-NEXT:    s_mov_b64 exec, s[2:3]
+; GFX9-NEXT:    s_not_b64 exec, exec
+; GFX9-NEXT:    v_mov_b32_e32 v2, -1
+; GFX9-NEXT:    s_not_b64 exec, exec
+; GFX9-NEXT:    s_or_saveexec_b64 s[4:5], -1
+; GFX9-NEXT:    v_and_b32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_and_b32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_and_b32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_and_b32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_and_b32_dpp v2, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_and_b32_dpp v2, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX9-NEXT:    v_readlane_b32 s2, v2, 63
+; GFX9-NEXT:    s_nop 0
+; GFX9-NEXT:    v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-NEXT:    s_mov_b64 exec, s[4:5]
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; GFX9-NEXT:    ; implicit-def: $vgpr0
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    ; mask branch BB14_2
+; GFX9-NEXT:    s_cbranch_execz BB14_2
+; GFX9-NEXT:  BB14_1:
+; GFX9-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
+; GFX9-NEXT:    v_mov_b32_e32 v3, s2
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    ds_and_rtn_b32 v0, v0, v3
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:  BB14_2:
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v0
+; GFX9-NEXT:    v_mov_b32_e32 v0, v1
+; GFX9-NEXT:    v_and_b32_e32 v0, s2, v0
+; GFX9-NEXT:    s_mov_b32 s3, 0xf000
+; GFX9-NEXT:    s_mov_b32 s2, -1
+; GFX9-NEXT:    s_nop 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX9-NEXT:    s_endpgm
+;
+; GFX1064-LABEL: and_i32_varying:
+; GFX1064:       ; %bb.0: ; %entry
+; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX1064-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX1064-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v4, s2, 0
+; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v4, s3, v4
+; GFX1064-NEXT:    s_or_saveexec_b64 s[2:3], -1
+; GFX1064-NEXT:    v_mov_b32_e32 v1, -1
+; GFX1064-NEXT:    s_mov_b64 exec, s[2:3]
+; GFX1064-NEXT:    s_not_b64 exec, exec
+; GFX1064-NEXT:    v_mov_b32_e32 v2, -1
+; GFX1064-NEXT:    s_not_b64 exec, exec
+; GFX1064-NEXT:    s_or_saveexec_b64 s[4:5], -1
+; GFX1064-NEXT:    v_and_b32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-NEXT:    v_and_b32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064-NEXT:    v_and_b32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064-NEXT:    v_and_b32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-NEXT:    v_mov_b32_e32 v3, v2
+; GFX1064-NEXT:    v_permlanex16_b32 v3, v3, -1, -1
+; GFX1064-NEXT:    v_and_b32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
+; GFX1064-NEXT:    v_readlane_b32 s2, v2, 31
+; GFX1064-NEXT:    v_mov_b32_e32 v3, s2
+; GFX1064-NEXT:    v_and_b32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf
+; GFX1064-NEXT:    v_readlane_b32 s2, v2, 15
+; GFX1064-NEXT:    v_mov_b32_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-NEXT:    v_readlane_b32 s3, v2, 31
+; GFX1064-NEXT:    v_readlane_b32 s6, v2, 47
+; GFX1064-NEXT:    v_writelane_b32 v1, s2, 16
+; GFX1064-NEXT:    s_mov_b32 s2, -1
+; GFX1064-NEXT:    v_writelane_b32 v1, s3, 32
+; GFX1064-NEXT:    v_readlane_b32 s3, v2, 63
+; GFX1064-NEXT:    v_writelane_b32 v1, s6, 48
+; GFX1064-NEXT:    s_mov_b64 exec, s[4:5]
+; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
+; GFX1064-NEXT:    ; implicit-def: $vgpr0
+; GFX1064-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX1064-NEXT:    ; mask branch BB14_2
+; GFX1064-NEXT:    s_cbranch_execz BB14_2
+; GFX1064-NEXT:  BB14_1:
+; GFX1064-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
+; GFX1064-NEXT:    v_mov_b32_e32 v7, s3
+; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1064-NEXT:    ds_and_rtn_b32 v0, v0, v7
+; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1064-NEXT:    buffer_gl0_inv
+; GFX1064-NEXT:    buffer_gl1_inv
+; GFX1064-NEXT:  BB14_2:
+; GFX1064-NEXT:    v_nop
+; GFX1064-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX1064-NEXT:    v_readfirstlane_b32 s3, v0
+; GFX1064-NEXT:    v_mov_b32_e32 v0, v1
+; GFX1064-NEXT:    v_and_b32_e32 v0, s3, v0
+; GFX1064-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1064-NEXT:    s_nop 1
+; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX1064-NEXT:    s_endpgm
+;
+; GFX1032-LABEL: and_i32_varying:
+; GFX1032:       ; %bb.0: ; %entry
+; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX1032-NEXT:    v_cmp_ne_u32_e64 s2, 1, 0
+; GFX1032-NEXT:    ; implicit-def: $vcc_hi
+; GFX1032-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v4, s2, 0
+; GFX1032-NEXT:    s_or_saveexec_b32 s2, -1
+; GFX1032-NEXT:    v_mov_b32_e32 v1, -1
+; GFX1032-NEXT:    s_mov_b32 exec_lo, s2
+; GFX1032-NEXT:    s_not_b32 exec_lo, exec_lo
+; GFX1032-NEXT:    v_mov_b32_e32 v2, -1
+; GFX1032-NEXT:    s_not_b32 exec_lo, exec_lo
+; GFX1032-NEXT:    s_or_saveexec_b32 s4, -1
+; GFX1032-NEXT:    v_and_b32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-NEXT:    s_mov_b32 s2, -1
+; GFX1032-NEXT:    v_and_b32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032-NEXT:    v_and_b32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032-NEXT:    v_and_b32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032-NEXT:    v_mov_b32_e32 v3, v2
+; GFX1032-NEXT:    v_permlanex16_b32 v3, v3, -1, -1
+; GFX1032-NEXT:    v_and_b32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
+; GFX1032-NEXT:    v_readlane_b32 s3, v2, 31
+; GFX1032-NEXT:    v_mov_b32_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-NEXT:    v_readlane_b32 s5, v2, 15
+; GFX1032-NEXT:    v_writelane_b32 v1, s5, 16
+; GFX1032-NEXT:    s_mov_b32 exec_lo, s4
+; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v4
+; GFX1032-NEXT:    ; implicit-def: $vgpr0
+; GFX1032-NEXT:    s_and_saveexec_b32 s4, vcc_lo
+; GFX1032-NEXT:    ; mask branch BB14_2
+; GFX1032-NEXT:    s_cbranch_execz BB14_2
+; GFX1032-NEXT:  BB14_1:
+; GFX1032-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
+; GFX1032-NEXT:    v_mov_b32_e32 v7, s3
+; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1032-NEXT:    ds_and_rtn_b32 v0, v0, v7
+; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1032-NEXT:    buffer_gl0_inv
+; GFX1032-NEXT:    buffer_gl1_inv
+; GFX1032-NEXT:  BB14_2:
+; GFX1032-NEXT:    v_nop
+; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX1032-NEXT:    v_readfirstlane_b32 s3, v0
+; GFX1032-NEXT:    v_mov_b32_e32 v0, v1
+; GFX1032-NEXT:    v_and_b32_e32 v0, s3, v0
+; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1032-NEXT:    s_nop 1
+; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX1032-NEXT:    s_endpgm
 entry:
   %lane = call i32 @llvm.amdgcn.workitem.id.x()
   %old = atomicrmw and i32 addrspace(3)* @local_var32, i32 %lane acq_rel
@@ -207,11 +2951,245 @@ entry:
   ret void
 }
 
-; GCN-LABEL: or_i32_varying:
-; GFX8MORE: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 63
+; GFX8MORE32: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 31
 ; GFX8MORE: v_mov_b32{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[scalar_value]]
 ; GFX8MORE: ds_or_rtn_b32 v{{[0-9]+}}, v{{[0-9]+}}, v[[value]]
 define amdgpu_kernel void @or_i32_varying(i32 addrspace(1)* %out) {
+;
+;
+; GFX7LESS-LABEL: or_i32_varying:
+; GFX7LESS:       ; %bb.0: ; %entry
+; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; GFX7LESS-NEXT:    v_mov_b32_e32 v1, local_var32@abs32@lo
+; GFX7LESS-NEXT:    s_mov_b32 m0, -1
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7LESS-NEXT:    ds_or_rtn_b32 v0, v1, v0
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7LESS-NEXT:    buffer_wbinvl1
+; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7LESS-NEXT:    s_mov_b32 s2, -1
+; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX7LESS-NEXT:    s_endpgm
+;
+; GFX8-LABEL: or_i32_varying:
+; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    v_mov_b32_e32 v2, v0
+; GFX8-NEXT:    s_or_saveexec_b64 s[2:3], -1
+; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX8-NEXT:    v_mov_b32_e32 v1, 0
+; GFX8-NEXT:    s_mov_b64 exec, s[2:3]
+; GFX8-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX8-NEXT:    s_not_b64 exec, exec
+; GFX8-NEXT:    v_mov_b32_e32 v2, 0
+; GFX8-NEXT:    s_not_b64 exec, exec
+; GFX8-NEXT:    s_or_saveexec_b64 s[4:5], -1
+; GFX8-NEXT:    v_or_b32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_or_b32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_or_b32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_or_b32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_or_b32_dpp v2, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_or_b32_dpp v2, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX8-NEXT:    v_readlane_b32 s2, v2, 63
+; GFX8-NEXT:    s_nop 0
+; GFX8-NEXT:    v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf
+; GFX8-NEXT:    s_mov_b64 exec, s[4:5]
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX8-NEXT:    ; implicit-def: $vgpr0
+; GFX8-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX8-NEXT:    ; mask branch BB15_2
+; GFX8-NEXT:    s_cbranch_execz BB15_2
+; GFX8-NEXT:  BB15_1:
+; GFX8-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
+; GFX8-NEXT:    v_mov_b32_e32 v3, s2
+; GFX8-NEXT:    s_mov_b32 m0, -1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    ds_or_rtn_b32 v0, v0, v3
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:  BB15_2:
+; GFX8-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT:    v_readfirstlane_b32 s2, v0
+; GFX8-NEXT:    v_mov_b32_e32 v0, v1
+; GFX8-NEXT:    v_or_b32_e32 v0, s2, v0
+; GFX8-NEXT:    s_mov_b32 s3, 0xf000
+; GFX8-NEXT:    s_mov_b32 s2, -1
+; GFX8-NEXT:    s_nop 0
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX8-NEXT:    s_endpgm
+;
+; GFX9-LABEL: or_i32_varying:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_or_saveexec_b64 s[2:3], -1
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0
+; GFX9-NEXT:    s_mov_b64 exec, s[2:3]
+; GFX9-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX9-NEXT:    s_not_b64 exec, exec
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-NEXT:    s_not_b64 exec, exec
+; GFX9-NEXT:    s_or_saveexec_b64 s[4:5], -1
+; GFX9-NEXT:    v_or_b32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_or_b32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_or_b32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_or_b32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_or_b32_dpp v2, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_or_b32_dpp v2, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX9-NEXT:    v_readlane_b32 s2, v2, 63
+; GFX9-NEXT:    s_nop 0
+; GFX9-NEXT:    v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-NEXT:    s_mov_b64 exec, s[4:5]
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT:    ; implicit-def: $vgpr0
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    ; mask branch BB15_2
+; GFX9-NEXT:    s_cbranch_execz BB15_2
+; GFX9-NEXT:  BB15_1:
+; GFX9-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
+; GFX9-NEXT:    v_mov_b32_e32 v3, s2
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    ds_or_rtn_b32 v0, v0, v3
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:  BB15_2:
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v0
+; GFX9-NEXT:    v_mov_b32_e32 v0, v1
+; GFX9-NEXT:    v_or_b32_e32 v0, s2, v0
+; GFX9-NEXT:    s_mov_b32 s3, 0xf000
+; GFX9-NEXT:    s_mov_b32 s2, -1
+; GFX9-NEXT:    s_nop 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX9-NEXT:    s_endpgm
+;
+; GFX1064-LABEL: or_i32_varying:
+; GFX1064:       ; %bb.0: ; %entry
+; GFX1064-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1064-NEXT:    s_or_saveexec_b64 s[2:3], -1
+; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
+; GFX1064-NEXT:    s_mov_b64 exec, s[2:3]
+; GFX1064-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
+; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, s3, v0
+; GFX1064-NEXT:    s_not_b64 exec, exec
+; GFX1064-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1064-NEXT:    s_not_b64 exec, exec
+; GFX1064-NEXT:    s_or_saveexec_b64 s[4:5], -1
+; GFX1064-NEXT:    v_or_b32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX1064-NEXT:    v_or_b32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX1064-NEXT:    v_or_b32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX1064-NEXT:    v_or_b32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX1064-NEXT:    v_mov_b32_e32 v3, v2
+; GFX1064-NEXT:    v_permlanex16_b32 v3, v3, -1, -1
+; GFX1064-NEXT:    v_or_b32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
+; GFX1064-NEXT:    v_readlane_b32 s2, v2, 31
+; GFX1064-NEXT:    v_mov_b32_e32 v3, s2
+; GFX1064-NEXT:    v_or_b32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf
+; GFX1064-NEXT:    v_readlane_b32 s2, v2, 15
+; GFX1064-NEXT:    v_mov_b32_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-NEXT:    v_readlane_b32 s3, v2, 31
+; GFX1064-NEXT:    v_readlane_b32 s6, v2, 47
+; GFX1064-NEXT:    v_writelane_b32 v1, s2, 16
+; GFX1064-NEXT:    s_mov_b32 s2, -1
+; GFX1064-NEXT:    v_writelane_b32 v1, s3, 32
+; GFX1064-NEXT:    v_readlane_b32 s3, v2, 63
+; GFX1064-NEXT:    v_writelane_b32 v1, s6, 48
+; GFX1064-NEXT:    s_mov_b64 exec, s[4:5]
+; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT:    ; implicit-def: $vgpr0
+; GFX1064-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX1064-NEXT:    ; mask branch BB15_2
+; GFX1064-NEXT:    s_cbranch_execz BB15_2
+; GFX1064-NEXT:  BB15_1:
+; GFX1064-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
+; GFX1064-NEXT:    v_mov_b32_e32 v7, s3
+; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1064-NEXT:    ds_or_rtn_b32 v0, v0, v7
+; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1064-NEXT:    buffer_gl0_inv
+; GFX1064-NEXT:    buffer_gl1_inv
+; GFX1064-NEXT:  BB15_2:
+; GFX1064-NEXT:    v_nop
+; GFX1064-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX1064-NEXT:    v_readfirstlane_b32 s3, v0
+; GFX1064-NEXT:    v_mov_b32_e32 v0, v1
+; GFX1064-NEXT:    v_or_b32_e32 v0, s3, v0
+; GFX1064-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1064-NEXT:    s_nop 1
+; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX1064-NEXT:    s_endpgm
+;
+; GFX1032-LABEL: or_i32_varying:
+; GFX1032:       ; %bb.0: ; %entry
+; GFX1032-NEXT:    ; implicit-def: $vcc_hi
+; GFX1032-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1032-NEXT:    s_or_saveexec_b32 s2, -1
+; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
+; GFX1032-NEXT:    s_mov_b32 exec_lo, s2
+; GFX1032-NEXT:    v_cmp_ne_u32_e64 s2, 1, 0
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
+; GFX1032-NEXT:    s_not_b32 exec_lo, exec_lo
+; GFX1032-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1032-NEXT:    s_not_b32 exec_lo, exec_lo
+; GFX1032-NEXT:    s_or_saveexec_b32 s4, -1
+; GFX1032-NEXT:    v_or_b32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX1032-NEXT:    s_mov_b32 s2, -1
+; GFX1032-NEXT:    v_or_b32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX1032-NEXT:    v_or_b32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX1032-NEXT:    v_or_b32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX1032-NEXT:    v_mov_b32_e32 v3, v2
+; GFX1032-NEXT:    v_permlanex16_b32 v3, v3, -1, -1
+; GFX1032-NEXT:    v_or_b32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
+; GFX1032-NEXT:    v_readlane_b32 s3, v2, 31
+; GFX1032-NEXT:    v_mov_b32_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-NEXT:    v_readlane_b32 s5, v2, 15
+; GFX1032-NEXT:    v_writelane_b32 v1, s5, 16
+; GFX1032-NEXT:    s_mov_b32 exec_lo, s4
+; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-NEXT:    ; implicit-def: $vgpr0
+; GFX1032-NEXT:    s_and_saveexec_b32 s4, vcc_lo
+; GFX1032-NEXT:    ; mask branch BB15_2
+; GFX1032-NEXT:    s_cbranch_execz BB15_2
+; GFX1032-NEXT:  BB15_1:
+; GFX1032-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
+; GFX1032-NEXT:    v_mov_b32_e32 v7, s3
+; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1032-NEXT:    ds_or_rtn_b32 v0, v0, v7
+; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1032-NEXT:    buffer_gl0_inv
+; GFX1032-NEXT:    buffer_gl1_inv
+; GFX1032-NEXT:  BB15_2:
+; GFX1032-NEXT:    v_nop
+; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX1032-NEXT:    v_readfirstlane_b32 s3, v0
+; GFX1032-NEXT:    v_mov_b32_e32 v0, v1
+; GFX1032-NEXT:    v_or_b32_e32 v0, s3, v0
+; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1032-NEXT:    s_nop 1
+; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX1032-NEXT:    s_endpgm
 entry:
   %lane = call i32 @llvm.amdgcn.workitem.id.x()
   %old = atomicrmw or i32 addrspace(3)* @local_var32, i32 %lane acq_rel
@@ -219,11 +3197,245 @@ entry:
   ret void
 }
 
-; GCN-LABEL: xor_i32_varying:
-; GFX8MORE: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 63
+; GFX8MORE32: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 31
 ; GFX8MORE: v_mov_b32{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[scalar_value]]
 ; GFX8MORE: ds_xor_rtn_b32 v{{[0-9]+}}, v{{[0-9]+}}, v[[value]]
 define amdgpu_kernel void @xor_i32_varying(i32 addrspace(1)* %out) {
+;
+;
+; GFX7LESS-LABEL: xor_i32_varying:
+; GFX7LESS:       ; %bb.0: ; %entry
+; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; GFX7LESS-NEXT:    v_mov_b32_e32 v1, local_var32@abs32@lo
+; GFX7LESS-NEXT:    s_mov_b32 m0, -1
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7LESS-NEXT:    ds_xor_rtn_b32 v0, v1, v0
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7LESS-NEXT:    buffer_wbinvl1
+; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7LESS-NEXT:    s_mov_b32 s2, -1
+; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX7LESS-NEXT:    s_endpgm
+;
+; GFX8-LABEL: xor_i32_varying:
+; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    v_mov_b32_e32 v2, v0
+; GFX8-NEXT:    s_or_saveexec_b64 s[2:3], -1
+; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX8-NEXT:    v_mov_b32_e32 v1, 0
+; GFX8-NEXT:    s_mov_b64 exec, s[2:3]
+; GFX8-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX8-NEXT:    s_not_b64 exec, exec
+; GFX8-NEXT:    v_mov_b32_e32 v2, 0
+; GFX8-NEXT:    s_not_b64 exec, exec
+; GFX8-NEXT:    s_or_saveexec_b64 s[4:5], -1
+; GFX8-NEXT:    v_xor_b32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_xor_b32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_xor_b32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_xor_b32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_xor_b32_dpp v2, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_xor_b32_dpp v2, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX8-NEXT:    v_readlane_b32 s2, v2, 63
+; GFX8-NEXT:    s_nop 0
+; GFX8-NEXT:    v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf
+; GFX8-NEXT:    s_mov_b64 exec, s[4:5]
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX8-NEXT:    ; implicit-def: $vgpr0
+; GFX8-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX8-NEXT:    ; mask branch BB16_2
+; GFX8-NEXT:    s_cbranch_execz BB16_2
+; GFX8-NEXT:  BB16_1:
+; GFX8-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
+; GFX8-NEXT:    v_mov_b32_e32 v3, s2
+; GFX8-NEXT:    s_mov_b32 m0, -1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    ds_xor_rtn_b32 v0, v0, v3
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:  BB16_2:
+; GFX8-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT:    v_readfirstlane_b32 s2, v0
+; GFX8-NEXT:    v_mov_b32_e32 v0, v1
+; GFX8-NEXT:    v_xor_b32_e32 v0, s2, v0
+; GFX8-NEXT:    s_mov_b32 s3, 0xf000
+; GFX8-NEXT:    s_mov_b32 s2, -1
+; GFX8-NEXT:    s_nop 0
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX8-NEXT:    s_endpgm
+;
+; GFX9-LABEL: xor_i32_varying:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_or_saveexec_b64 s[2:3], -1
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0
+; GFX9-NEXT:    s_mov_b64 exec, s[2:3]
+; GFX9-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX9-NEXT:    s_not_b64 exec, exec
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-NEXT:    s_not_b64 exec, exec
+; GFX9-NEXT:    s_or_saveexec_b64 s[4:5], -1
+; GFX9-NEXT:    v_xor_b32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_xor_b32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_xor_b32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_xor_b32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_xor_b32_dpp v2, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_xor_b32_dpp v2, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX9-NEXT:    v_readlane_b32 s2, v2, 63
+; GFX9-NEXT:    s_nop 0
+; GFX9-NEXT:    v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-NEXT:    s_mov_b64 exec, s[4:5]
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT:    ; implicit-def: $vgpr0
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    ; mask branch BB16_2
+; GFX9-NEXT:    s_cbranch_execz BB16_2
+; GFX9-NEXT:  BB16_1:
+; GFX9-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
+; GFX9-NEXT:    v_mov_b32_e32 v3, s2
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    ds_xor_rtn_b32 v0, v0, v3
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:  BB16_2:
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v0
+; GFX9-NEXT:    v_mov_b32_e32 v0, v1
+; GFX9-NEXT:    v_xor_b32_e32 v0, s2, v0
+; GFX9-NEXT:    s_mov_b32 s3, 0xf000
+; GFX9-NEXT:    s_mov_b32 s2, -1
+; GFX9-NEXT:    s_nop 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX9-NEXT:    s_endpgm
+;
+; GFX1064-LABEL: xor_i32_varying:
+; GFX1064:       ; %bb.0: ; %entry
+; GFX1064-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1064-NEXT:    s_or_saveexec_b64 s[2:3], -1
+; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
+; GFX1064-NEXT:    s_mov_b64 exec, s[2:3]
+; GFX1064-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
+; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, s3, v0
+; GFX1064-NEXT:    s_not_b64 exec, exec
+; GFX1064-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1064-NEXT:    s_not_b64 exec, exec
+; GFX1064-NEXT:    s_or_saveexec_b64 s[4:5], -1
+; GFX1064-NEXT:    v_xor_b32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX1064-NEXT:    v_xor_b32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX1064-NEXT:    v_xor_b32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX1064-NEXT:    v_xor_b32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX1064-NEXT:    v_mov_b32_e32 v3, v2
+; GFX1064-NEXT:    v_permlanex16_b32 v3, v3, -1, -1
+; GFX1064-NEXT:    v_xor_b32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
+; GFX1064-NEXT:    v_readlane_b32 s2, v2, 31
+; GFX1064-NEXT:    v_mov_b32_e32 v3, s2
+; GFX1064-NEXT:    v_xor_b32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf
+; GFX1064-NEXT:    v_readlane_b32 s2, v2, 15
+; GFX1064-NEXT:    v_mov_b32_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-NEXT:    v_readlane_b32 s3, v2, 31
+; GFX1064-NEXT:    v_readlane_b32 s6, v2, 47
+; GFX1064-NEXT:    v_writelane_b32 v1, s2, 16
+; GFX1064-NEXT:    s_mov_b32 s2, -1
+; GFX1064-NEXT:    v_writelane_b32 v1, s3, 32
+; GFX1064-NEXT:    v_readlane_b32 s3, v2, 63
+; GFX1064-NEXT:    v_writelane_b32 v1, s6, 48
+; GFX1064-NEXT:    s_mov_b64 exec, s[4:5]
+; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT:    ; implicit-def: $vgpr0
+; GFX1064-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX1064-NEXT:    ; mask branch BB16_2
+; GFX1064-NEXT:    s_cbranch_execz BB16_2
+; GFX1064-NEXT:  BB16_1:
+; GFX1064-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
+; GFX1064-NEXT:    v_mov_b32_e32 v7, s3
+; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1064-NEXT:    ds_xor_rtn_b32 v0, v0, v7
+; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1064-NEXT:    buffer_gl0_inv
+; GFX1064-NEXT:    buffer_gl1_inv
+; GFX1064-NEXT:  BB16_2:
+; GFX1064-NEXT:    v_nop
+; GFX1064-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX1064-NEXT:    v_readfirstlane_b32 s3, v0
+; GFX1064-NEXT:    v_mov_b32_e32 v0, v1
+; GFX1064-NEXT:    v_xor_b32_e32 v0, s3, v0
+; GFX1064-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1064-NEXT:    s_nop 1
+; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX1064-NEXT:    s_endpgm
+;
+; GFX1032-LABEL: xor_i32_varying:
+; GFX1032:       ; %bb.0: ; %entry
+; GFX1032-NEXT:    ; implicit-def: $vcc_hi
+; GFX1032-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1032-NEXT:    s_or_saveexec_b32 s2, -1
+; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
+; GFX1032-NEXT:    s_mov_b32 exec_lo, s2
+; GFX1032-NEXT:    v_cmp_ne_u32_e64 s2, 1, 0
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
+; GFX1032-NEXT:    s_not_b32 exec_lo, exec_lo
+; GFX1032-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1032-NEXT:    s_not_b32 exec_lo, exec_lo
+; GFX1032-NEXT:    s_or_saveexec_b32 s4, -1
+; GFX1032-NEXT:    v_xor_b32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX1032-NEXT:    s_mov_b32 s2, -1
+; GFX1032-NEXT:    v_xor_b32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX1032-NEXT:    v_xor_b32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX1032-NEXT:    v_xor_b32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX1032-NEXT:    v_mov_b32_e32 v3, v2
+; GFX1032-NEXT:    v_permlanex16_b32 v3, v3, -1, -1
+; GFX1032-NEXT:    v_xor_b32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
+; GFX1032-NEXT:    v_readlane_b32 s3, v2, 31
+; GFX1032-NEXT:    v_mov_b32_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-NEXT:    v_readlane_b32 s5, v2, 15
+; GFX1032-NEXT:    v_writelane_b32 v1, s5, 16
+; GFX1032-NEXT:    s_mov_b32 exec_lo, s4
+; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-NEXT:    ; implicit-def: $vgpr0
+; GFX1032-NEXT:    s_and_saveexec_b32 s4, vcc_lo
+; GFX1032-NEXT:    ; mask branch BB16_2
+; GFX1032-NEXT:    s_cbranch_execz BB16_2
+; GFX1032-NEXT:  BB16_1:
+; GFX1032-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
+; GFX1032-NEXT:    v_mov_b32_e32 v7, s3
+; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1032-NEXT:    ds_xor_rtn_b32 v0, v0, v7
+; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1032-NEXT:    buffer_gl0_inv
+; GFX1032-NEXT:    buffer_gl1_inv
+; GFX1032-NEXT:  BB16_2:
+; GFX1032-NEXT:    v_nop
+; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX1032-NEXT:    v_readfirstlane_b32 s3, v0
+; GFX1032-NEXT:    v_mov_b32_e32 v0, v1
+; GFX1032-NEXT:    v_xor_b32_e32 v0, s3, v0
+; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1032-NEXT:    s_nop 1
+; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX1032-NEXT:    s_endpgm
 entry:
   %lane = call i32 @llvm.amdgcn.workitem.id.x()
   %old = atomicrmw xor i32 addrspace(3)* @local_var32, i32 %lane acq_rel
@@ -231,11 +3443,245 @@ entry:
   ret void
 }
 
-; GCN-LABEL: max_i32_varying:
-; GFX8MORE: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 63
+; GFX8MORE32: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 31
 ; GFX8MORE: v_mov_b32{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[scalar_value]]
 ; GFX8MORE: ds_max_rtn_i32 v{{[0-9]+}}, v{{[0-9]+}}, v[[value]]
 define amdgpu_kernel void @max_i32_varying(i32 addrspace(1)* %out) {
+;
+;
+; GFX7LESS-LABEL: max_i32_varying:
+; GFX7LESS:       ; %bb.0: ; %entry
+; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; GFX7LESS-NEXT:    v_mov_b32_e32 v1, local_var32@abs32@lo
+; GFX7LESS-NEXT:    s_mov_b32 m0, -1
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7LESS-NEXT:    ds_max_rtn_i32 v0, v1, v0
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7LESS-NEXT:    buffer_wbinvl1
+; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7LESS-NEXT:    s_mov_b32 s2, -1
+; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX7LESS-NEXT:    s_endpgm
+;
+; GFX8-LABEL: max_i32_varying:
+; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX8-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v3, s2, 0
+; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v3, s3, v3
+; GFX8-NEXT:    v_mov_b32_e32 v2, v0
+; GFX8-NEXT:    s_or_saveexec_b64 s[2:3], -1
+; GFX8-NEXT:    v_bfrev_b32_e32 v1, 1
+; GFX8-NEXT:    s_mov_b64 exec, s[2:3]
+; GFX8-NEXT:    s_not_b64 exec, exec
+; GFX8-NEXT:    v_mov_b32_e32 v2, v1
+; GFX8-NEXT:    s_not_b64 exec, exec
+; GFX8-NEXT:    s_or_saveexec_b64 s[4:5], -1
+; GFX8-NEXT:    v_max_i32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_max_i32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_max_i32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_max_i32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_max_i32_dpp v2, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_max_i32_dpp v2, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX8-NEXT:    v_readlane_b32 s2, v2, 63
+; GFX8-NEXT:    s_nop 0
+; GFX8-NEXT:    v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf
+; GFX8-NEXT:    s_mov_b64 exec, s[4:5]
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; GFX8-NEXT:    ; implicit-def: $vgpr0
+; GFX8-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX8-NEXT:    ; mask branch BB17_2
+; GFX8-NEXT:    s_cbranch_execz BB17_2
+; GFX8-NEXT:  BB17_1:
+; GFX8-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
+; GFX8-NEXT:    v_mov_b32_e32 v3, s2
+; GFX8-NEXT:    s_mov_b32 m0, -1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    ds_max_rtn_i32 v0, v0, v3
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:  BB17_2:
+; GFX8-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT:    v_readfirstlane_b32 s2, v0
+; GFX8-NEXT:    v_mov_b32_e32 v0, v1
+; GFX8-NEXT:    v_max_i32_e32 v0, s2, v0
+; GFX8-NEXT:    s_mov_b32 s3, 0xf000
+; GFX8-NEXT:    s_mov_b32 s2, -1
+; GFX8-NEXT:    s_nop 0
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX8-NEXT:    s_endpgm
+;
+; GFX9-LABEL: max_i32_varying:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v3, s2, 0
+; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v3, s3, v3
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_or_saveexec_b64 s[2:3], -1
+; GFX9-NEXT:    v_bfrev_b32_e32 v1, 1
+; GFX9-NEXT:    s_mov_b64 exec, s[2:3]
+; GFX9-NEXT:    s_not_b64 exec, exec
+; GFX9-NEXT:    v_mov_b32_e32 v2, v1
+; GFX9-NEXT:    s_not_b64 exec, exec
+; GFX9-NEXT:    s_or_saveexec_b64 s[4:5], -1
+; GFX9-NEXT:    v_max_i32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_max_i32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_max_i32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_max_i32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_max_i32_dpp v2, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_max_i32_dpp v2, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX9-NEXT:    v_readlane_b32 s2, v2, 63
+; GFX9-NEXT:    s_nop 0
+; GFX9-NEXT:    v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-NEXT:    s_mov_b64 exec, s[4:5]
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; GFX9-NEXT:    ; implicit-def: $vgpr0
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    ; mask branch BB17_2
+; GFX9-NEXT:    s_cbranch_execz BB17_2
+; GFX9-NEXT:  BB17_1:
+; GFX9-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
+; GFX9-NEXT:    v_mov_b32_e32 v3, s2
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    ds_max_rtn_i32 v0, v0, v3
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:  BB17_2:
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v0
+; GFX9-NEXT:    v_mov_b32_e32 v0, v1
+; GFX9-NEXT:    v_max_i32_e32 v0, s2, v0
+; GFX9-NEXT:    s_mov_b32 s3, 0xf000
+; GFX9-NEXT:    s_mov_b32 s2, -1
+; GFX9-NEXT:    s_nop 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX9-NEXT:    s_endpgm
+;
+; GFX1064-LABEL: max_i32_varying:
+; GFX1064:       ; %bb.0: ; %entry
+; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX1064-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX1064-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v4, s2, 0
+; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v4, s3, v4
+; GFX1064-NEXT:    s_or_saveexec_b64 s[2:3], -1
+; GFX1064-NEXT:    v_bfrev_b32_e32 v1, 1
+; GFX1064-NEXT:    s_mov_b64 exec, s[2:3]
+; GFX1064-NEXT:    s_not_b64 exec, exec
+; GFX1064-NEXT:    v_mov_b32_e32 v2, v1
+; GFX1064-NEXT:    s_not_b64 exec, exec
+; GFX1064-NEXT:    s_or_saveexec_b64 s[4:5], -1
+; GFX1064-NEXT:    v_max_i32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-NEXT:    v_max_i32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064-NEXT:    v_max_i32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064-NEXT:    v_max_i32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-NEXT:    v_mov_b32_e32 v3, v2
+; GFX1064-NEXT:    v_permlanex16_b32 v3, v3, -1, -1
+; GFX1064-NEXT:    v_max_i32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
+; GFX1064-NEXT:    v_readlane_b32 s2, v2, 31
+; GFX1064-NEXT:    v_mov_b32_e32 v3, s2
+; GFX1064-NEXT:    v_max_i32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf
+; GFX1064-NEXT:    v_readlane_b32 s2, v2, 15
+; GFX1064-NEXT:    v_mov_b32_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-NEXT:    v_readlane_b32 s3, v2, 31
+; GFX1064-NEXT:    v_readlane_b32 s6, v2, 47
+; GFX1064-NEXT:    v_writelane_b32 v1, s2, 16
+; GFX1064-NEXT:    s_mov_b32 s2, -1
+; GFX1064-NEXT:    v_writelane_b32 v1, s3, 32
+; GFX1064-NEXT:    v_readlane_b32 s3, v2, 63
+; GFX1064-NEXT:    v_writelane_b32 v1, s6, 48
+; GFX1064-NEXT:    s_mov_b64 exec, s[4:5]
+; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
+; GFX1064-NEXT:    ; implicit-def: $vgpr0
+; GFX1064-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX1064-NEXT:    ; mask branch BB17_2
+; GFX1064-NEXT:    s_cbranch_execz BB17_2
+; GFX1064-NEXT:  BB17_1:
+; GFX1064-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
+; GFX1064-NEXT:    v_mov_b32_e32 v7, s3
+; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1064-NEXT:    ds_max_rtn_i32 v0, v0, v7
+; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1064-NEXT:    buffer_gl0_inv
+; GFX1064-NEXT:    buffer_gl1_inv
+; GFX1064-NEXT:  BB17_2:
+; GFX1064-NEXT:    v_nop
+; GFX1064-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX1064-NEXT:    v_readfirstlane_b32 s3, v0
+; GFX1064-NEXT:    v_mov_b32_e32 v0, v1
+; GFX1064-NEXT:    v_max_i32_e32 v0, s3, v0
+; GFX1064-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1064-NEXT:    s_nop 1
+; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX1064-NEXT:    s_endpgm
+;
+; GFX1032-LABEL: max_i32_varying:
+; GFX1032:       ; %bb.0: ; %entry
+; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX1032-NEXT:    v_cmp_ne_u32_e64 s2, 1, 0
+; GFX1032-NEXT:    ; implicit-def: $vcc_hi
+; GFX1032-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v4, s2, 0
+; GFX1032-NEXT:    s_or_saveexec_b32 s2, -1
+; GFX1032-NEXT:    v_bfrev_b32_e32 v1, 1
+; GFX1032-NEXT:    s_mov_b32 exec_lo, s2
+; GFX1032-NEXT:    s_not_b32 exec_lo, exec_lo
+; GFX1032-NEXT:    v_mov_b32_e32 v2, v1
+; GFX1032-NEXT:    s_not_b32 exec_lo, exec_lo
+; GFX1032-NEXT:    s_or_saveexec_b32 s4, -1
+; GFX1032-NEXT:    v_max_i32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-NEXT:    s_mov_b32 s2, -1
+; GFX1032-NEXT:    v_max_i32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032-NEXT:    v_max_i32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032-NEXT:    v_max_i32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032-NEXT:    v_mov_b32_e32 v3, v2
+; GFX1032-NEXT:    v_permlanex16_b32 v3, v3, -1, -1
+; GFX1032-NEXT:    v_max_i32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
+; GFX1032-NEXT:    v_readlane_b32 s3, v2, 31
+; GFX1032-NEXT:    v_mov_b32_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-NEXT:    v_readlane_b32 s5, v2, 15
+; GFX1032-NEXT:    v_writelane_b32 v1, s5, 16
+; GFX1032-NEXT:    s_mov_b32 exec_lo, s4
+; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v4
+; GFX1032-NEXT:    ; implicit-def: $vgpr0
+; GFX1032-NEXT:    s_and_saveexec_b32 s4, vcc_lo
+; GFX1032-NEXT:    ; mask branch BB17_2
+; GFX1032-NEXT:    s_cbranch_execz BB17_2
+; GFX1032-NEXT:  BB17_1:
+; GFX1032-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
+; GFX1032-NEXT:    v_mov_b32_e32 v7, s3
+; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1032-NEXT:    ds_max_rtn_i32 v0, v0, v7
+; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1032-NEXT:    buffer_gl0_inv
+; GFX1032-NEXT:    buffer_gl1_inv
+; GFX1032-NEXT:  BB17_2:
+; GFX1032-NEXT:    v_nop
+; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX1032-NEXT:    v_readfirstlane_b32 s3, v0
+; GFX1032-NEXT:    v_mov_b32_e32 v0, v1
+; GFX1032-NEXT:    v_max_i32_e32 v0, s3, v0
+; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1032-NEXT:    s_nop 1
+; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX1032-NEXT:    s_endpgm
 entry:
   %lane = call i32 @llvm.amdgcn.workitem.id.x()
   %old = atomicrmw max i32 addrspace(3)* @local_var32, i32 %lane acq_rel
@@ -243,26 +3689,440 @@ entry:
   ret void
 }
 
-; GCN-LABEL: max_i64_constant:
-; GCN: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
-; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_lo:[0-9]+]], s[[exec_lo]], 0
-; GCN: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_hi:[0-9]+]], s[[exec_hi]], v[[mbcnt_lo]]
-; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc, 0, v[[mbcnt_hi]]
-; GCN: v_mov_b32{{(_e[0-9]+)?}} v[[value_lo:[0-9]+]], 5
-; GCN: v_mov_b32{{(_e[0-9]+)?}} v[[value_hi:[0-9]+]], 0
-; GCN: ds_max_rtn_i64 v{{\[}}{{[0-9]+}}:{{[0-9]+}}{{\]}}, v{{[0-9]+}}, v{{\[}}[[value_lo]]:[[value_hi]]{{\]}}
 define amdgpu_kernel void @max_i64_constant(i64 addrspace(1)* %out) {
+;
+;
+; GFX7LESS-LABEL: max_i64_constant:
+; GFX7LESS:       ; %bb.0: ; %entry
+; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; GFX7LESS-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
+; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s3, v0
+; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; GFX7LESS-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX7LESS-NEXT:    ; mask branch BB18_2
+; GFX7LESS-NEXT:    s_cbranch_execz BB18_2
+; GFX7LESS-NEXT:  BB18_1:
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, local_var64@abs32@lo
+; GFX7LESS-NEXT:    v_mov_b32_e32 v0, 5
+; GFX7LESS-NEXT:    v_mov_b32_e32 v1, 0
+; GFX7LESS-NEXT:    s_mov_b32 m0, -1
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7LESS-NEXT:    ds_max_rtn_i64 v[0:1], v2, v[0:1]
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7LESS-NEXT:    buffer_wbinvl1
+; GFX7LESS-NEXT:  BB18_2:
+; GFX7LESS-NEXT:    s_or_b64 exec, exec, s[2:3]
+; GFX7LESS-NEXT:    v_readfirstlane_b32 s4, v0
+; GFX7LESS-NEXT:    v_readfirstlane_b32 s5, v1
+; GFX7LESS-NEXT:    v_bfrev_b32_e32 v1, 1
+; GFX7LESS-NEXT:    v_cndmask_b32_e64 v0, 5, 0, vcc
+; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7LESS-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s5
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s4
+; GFX7LESS-NEXT:    v_cmp_gt_i64_e32 vcc, s[4:5], v[0:1]
+; GFX7LESS-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; GFX7LESS-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
+; GFX7LESS-NEXT:    s_mov_b32 s2, -1
+; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX7LESS-NEXT:    s_endpgm
+;
+; GFX8-LABEL: max_i64_constant:
+; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX8-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX8-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; GFX8-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX8-NEXT:    ; mask branch BB18_2
+; GFX8-NEXT:    s_cbranch_execz BB18_2
+; GFX8-NEXT:  BB18_1:
+; GFX8-NEXT:    v_mov_b32_e32 v0, 5
+; GFX8-NEXT:    v_mov_b32_e32 v2, local_var64@abs32@lo
+; GFX8-NEXT:    v_mov_b32_e32 v1, 0
+; GFX8-NEXT:    s_mov_b32 m0, -1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    ds_max_rtn_i64 v[0:1], v2, v[0:1]
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:  BB18_2:
+; GFX8-NEXT:    s_or_b64 exec, exec, s[2:3]
+; GFX8-NEXT:    v_readfirstlane_b32 s2, v0
+; GFX8-NEXT:    v_bfrev_b32_e32 v0, 1
+; GFX8-NEXT:    v_readfirstlane_b32 s3, v1
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, 0, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, 5, 0, vcc
+; GFX8-NEXT:    v_cmp_gt_i64_e32 vcc, s[2:3], v[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v2, s3
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; GFX8-NEXT:    v_mov_b32_e32 v2, s2
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX8-NEXT:    s_mov_b32 s3, 0xf000
+; GFX8-NEXT:    s_mov_b32 s2, -1
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX8-NEXT:    s_endpgm
+;
+; GFX9-LABEL: max_i64_constant:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; GFX9-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX9-NEXT:    ; mask branch BB18_2
+; GFX9-NEXT:    s_cbranch_execz BB18_2
+; GFX9-NEXT:  BB18_1:
+; GFX9-NEXT:    v_mov_b32_e32 v0, 5
+; GFX9-NEXT:    v_mov_b32_e32 v2, local_var64@abs32@lo
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    ds_max_rtn_i64 v[0:1], v2, v[0:1]
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:  BB18_2:
+; GFX9-NEXT:    s_or_b64 exec, exec, s[2:3]
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v0
+; GFX9-NEXT:    v_bfrev_b32_e32 v0, 1
+; GFX9-NEXT:    v_readfirstlane_b32 s3, v1
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, 0, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, 5, 0, vcc
+; GFX9-NEXT:    v_cmp_gt_i64_e32 vcc, s[2:3], v[0:1]
+; GFX9-NEXT:    v_mov_b32_e32 v2, s3
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v2, s2
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX9-NEXT:    s_mov_b32 s3, 0xf000
+; GFX9-NEXT:    s_mov_b32 s2, -1
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX9-NEXT:    s_endpgm
+;
+; GFX1064-LABEL: max_i64_constant:
+; GFX1064:       ; %bb.0: ; %entry
+; GFX1064-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
+; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, s3, v0
+; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; GFX1064-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX1064-NEXT:    ; mask branch BB18_2
+; GFX1064-NEXT:    s_cbranch_execz BB18_2
+; GFX1064-NEXT:  BB18_1:
+; GFX1064-NEXT:    v_mov_b32_e32 v0, 5
+; GFX1064-NEXT:    v_mov_b32_e32 v2, local_var64@abs32@lo
+; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
+; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1064-NEXT:    ds_max_rtn_i64 v[0:1], v2, v[0:1]
+; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1064-NEXT:    buffer_gl0_inv
+; GFX1064-NEXT:    buffer_gl1_inv
+; GFX1064-NEXT:  BB18_2:
+; GFX1064-NEXT:    v_nop
+; GFX1064-NEXT:    s_or_b64 exec, exec, s[2:3]
+; GFX1064-NEXT:    v_readfirstlane_b32 s4, v0
+; GFX1064-NEXT:    v_readfirstlane_b32 s5, v1
+; GFX1064-NEXT:    v_cndmask_b32_e64 v1, 0, 0x80000000, vcc
+; GFX1064-NEXT:    v_cndmask_b32_e64 v0, 5, 0, vcc
+; GFX1064-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1064-NEXT:    s_mov_b32 s2, -1
+; GFX1064-NEXT:    v_cmp_gt_i64_e32 vcc, s[4:5], v[0:1]
+; GFX1064-NEXT:    v_cndmask_b32_e64 v1, v1, s5, vcc
+; GFX1064-NEXT:    v_cndmask_b32_e64 v0, v0, s4, vcc
+; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX1064-NEXT:    s_endpgm
+;
+; GFX1032-LABEL: max_i64_constant:
+; GFX1032:       ; %bb.0: ; %entry
+; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX1032-NEXT:    v_cmp_ne_u32_e64 s2, 1, 0
+; GFX1032-NEXT:    ; implicit-def: $vcc_hi
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
+; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; GFX1032-NEXT:    s_and_saveexec_b32 s2, vcc_lo
+; GFX1032-NEXT:    ; mask branch BB18_2
+; GFX1032-NEXT:    s_cbranch_execz BB18_2
+; GFX1032-NEXT:  BB18_1:
+; GFX1032-NEXT:    v_mov_b32_e32 v0, 5
+; GFX1032-NEXT:    v_mov_b32_e32 v2, local_var64@abs32@lo
+; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
+; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1032-NEXT:    ds_max_rtn_i64 v[0:1], v2, v[0:1]
+; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1032-NEXT:    buffer_gl0_inv
+; GFX1032-NEXT:    buffer_gl1_inv
+; GFX1032-NEXT:  BB18_2:
+; GFX1032-NEXT:    v_nop
+; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT:    v_readfirstlane_b32 s4, v0
+; GFX1032-NEXT:    v_readfirstlane_b32 s5, v1
+; GFX1032-NEXT:    v_cndmask_b32_e64 v1, 0, 0x80000000, vcc_lo
+; GFX1032-NEXT:    v_cndmask_b32_e64 v0, 5, 0, vcc_lo
+; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1032-NEXT:    s_mov_b32 s2, -1
+; GFX1032-NEXT:    v_cmp_gt_i64_e32 vcc_lo, s[4:5], v[0:1]
+; GFX1032-NEXT:    v_cndmask_b32_e64 v1, v1, s5, vcc_lo
+; GFX1032-NEXT:    v_cndmask_b32_e64 v0, v0, s4, vcc_lo
+; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX1032-NEXT:    s_endpgm
 entry:
   %old = atomicrmw max i64 addrspace(3)* @local_var64, i64 5 acq_rel
   store i64 %old, i64 addrspace(1)* %out
   ret void
 }
 
-; GCN-LABEL: min_i32_varying:
-; GFX8MORE: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 63
+; GFX8MORE32: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 31
 ; GFX8MORE: v_mov_b32{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[scalar_value]]
 ; GFX8MORE: ds_min_rtn_i32 v{{[0-9]+}}, v{{[0-9]+}}, v[[value]]
 define amdgpu_kernel void @min_i32_varying(i32 addrspace(1)* %out) {
+;
+;
+; GFX7LESS-LABEL: min_i32_varying:
+; GFX7LESS:       ; %bb.0: ; %entry
+; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; GFX7LESS-NEXT:    v_mov_b32_e32 v1, local_var32@abs32@lo
+; GFX7LESS-NEXT:    s_mov_b32 m0, -1
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7LESS-NEXT:    ds_min_rtn_i32 v0, v1, v0
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7LESS-NEXT:    buffer_wbinvl1
+; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7LESS-NEXT:    s_mov_b32 s2, -1
+; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX7LESS-NEXT:    s_endpgm
+;
+; GFX8-LABEL: min_i32_varying:
+; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX8-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v3, s2, 0
+; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v3, s3, v3
+; GFX8-NEXT:    v_mov_b32_e32 v2, v0
+; GFX8-NEXT:    s_or_saveexec_b64 s[2:3], -1
+; GFX8-NEXT:    v_bfrev_b32_e32 v1, -2
+; GFX8-NEXT:    s_mov_b64 exec, s[2:3]
+; GFX8-NEXT:    s_not_b64 exec, exec
+; GFX8-NEXT:    v_mov_b32_e32 v2, v1
+; GFX8-NEXT:    s_not_b64 exec, exec
+; GFX8-NEXT:    s_or_saveexec_b64 s[4:5], -1
+; GFX8-NEXT:    v_min_i32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_min_i32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_min_i32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_min_i32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_min_i32_dpp v2, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_min_i32_dpp v2, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX8-NEXT:    v_readlane_b32 s2, v2, 63
+; GFX8-NEXT:    s_nop 0
+; GFX8-NEXT:    v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf
+; GFX8-NEXT:    s_mov_b64 exec, s[4:5]
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; GFX8-NEXT:    ; implicit-def: $vgpr0
+; GFX8-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX8-NEXT:    ; mask branch BB19_2
+; GFX8-NEXT:    s_cbranch_execz BB19_2
+; GFX8-NEXT:  BB19_1:
+; GFX8-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
+; GFX8-NEXT:    v_mov_b32_e32 v3, s2
+; GFX8-NEXT:    s_mov_b32 m0, -1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    ds_min_rtn_i32 v0, v0, v3
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:  BB19_2:
+; GFX8-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT:    v_readfirstlane_b32 s2, v0
+; GFX8-NEXT:    v_mov_b32_e32 v0, v1
+; GFX8-NEXT:    v_min_i32_e32 v0, s2, v0
+; GFX8-NEXT:    s_mov_b32 s3, 0xf000
+; GFX8-NEXT:    s_mov_b32 s2, -1
+; GFX8-NEXT:    s_nop 0
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX8-NEXT:    s_endpgm
+;
+; GFX9-LABEL: min_i32_varying:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v3, s2, 0
+; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v3, s3, v3
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_or_saveexec_b64 s[2:3], -1
+; GFX9-NEXT:    v_bfrev_b32_e32 v1, -2
+; GFX9-NEXT:    s_mov_b64 exec, s[2:3]
+; GFX9-NEXT:    s_not_b64 exec, exec
+; GFX9-NEXT:    v_mov_b32_e32 v2, v1
+; GFX9-NEXT:    s_not_b64 exec, exec
+; GFX9-NEXT:    s_or_saveexec_b64 s[4:5], -1
+; GFX9-NEXT:    v_min_i32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_min_i32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_min_i32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_min_i32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_min_i32_dpp v2, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_min_i32_dpp v2, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX9-NEXT:    v_readlane_b32 s2, v2, 63
+; GFX9-NEXT:    s_nop 0
+; GFX9-NEXT:    v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-NEXT:    s_mov_b64 exec, s[4:5]
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; GFX9-NEXT:    ; implicit-def: $vgpr0
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    ; mask branch BB19_2
+; GFX9-NEXT:    s_cbranch_execz BB19_2
+; GFX9-NEXT:  BB19_1:
+; GFX9-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
+; GFX9-NEXT:    v_mov_b32_e32 v3, s2
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    ds_min_rtn_i32 v0, v0, v3
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:  BB19_2:
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v0
+; GFX9-NEXT:    v_mov_b32_e32 v0, v1
+; GFX9-NEXT:    v_min_i32_e32 v0, s2, v0
+; GFX9-NEXT:    s_mov_b32 s3, 0xf000
+; GFX9-NEXT:    s_mov_b32 s2, -1
+; GFX9-NEXT:    s_nop 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX9-NEXT:    s_endpgm
+;
+; GFX1064-LABEL: min_i32_varying:
+; GFX1064:       ; %bb.0: ; %entry
+; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX1064-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX1064-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v4, s2, 0
+; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v4, s3, v4
+; GFX1064-NEXT:    s_or_saveexec_b64 s[2:3], -1
+; GFX1064-NEXT:    v_bfrev_b32_e32 v1, -2
+; GFX1064-NEXT:    s_mov_b64 exec, s[2:3]
+; GFX1064-NEXT:    s_not_b64 exec, exec
+; GFX1064-NEXT:    v_mov_b32_e32 v2, v1
+; GFX1064-NEXT:    s_not_b64 exec, exec
+; GFX1064-NEXT:    s_or_saveexec_b64 s[4:5], -1
+; GFX1064-NEXT:    v_min_i32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-NEXT:    v_min_i32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064-NEXT:    v_min_i32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064-NEXT:    v_min_i32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-NEXT:    v_mov_b32_e32 v3, v2
+; GFX1064-NEXT:    v_permlanex16_b32 v3, v3, -1, -1
+; GFX1064-NEXT:    v_min_i32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
+; GFX1064-NEXT:    v_readlane_b32 s2, v2, 31
+; GFX1064-NEXT:    v_mov_b32_e32 v3, s2
+; GFX1064-NEXT:    v_min_i32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf
+; GFX1064-NEXT:    v_readlane_b32 s2, v2, 15
+; GFX1064-NEXT:    v_mov_b32_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-NEXT:    v_readlane_b32 s3, v2, 31
+; GFX1064-NEXT:    v_readlane_b32 s6, v2, 47
+; GFX1064-NEXT:    v_writelane_b32 v1, s2, 16
+; GFX1064-NEXT:    s_mov_b32 s2, -1
+; GFX1064-NEXT:    v_writelane_b32 v1, s3, 32
+; GFX1064-NEXT:    v_readlane_b32 s3, v2, 63
+; GFX1064-NEXT:    v_writelane_b32 v1, s6, 48
+; GFX1064-NEXT:    s_mov_b64 exec, s[4:5]
+; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
+; GFX1064-NEXT:    ; implicit-def: $vgpr0
+; GFX1064-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX1064-NEXT:    ; mask branch BB19_2
+; GFX1064-NEXT:    s_cbranch_execz BB19_2
+; GFX1064-NEXT:  BB19_1:
+; GFX1064-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
+; GFX1064-NEXT:    v_mov_b32_e32 v7, s3
+; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1064-NEXT:    ds_min_rtn_i32 v0, v0, v7
+; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1064-NEXT:    buffer_gl0_inv
+; GFX1064-NEXT:    buffer_gl1_inv
+; GFX1064-NEXT:  BB19_2:
+; GFX1064-NEXT:    v_nop
+; GFX1064-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX1064-NEXT:    v_readfirstlane_b32 s3, v0
+; GFX1064-NEXT:    v_mov_b32_e32 v0, v1
+; GFX1064-NEXT:    v_min_i32_e32 v0, s3, v0
+; GFX1064-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1064-NEXT:    s_nop 1
+; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX1064-NEXT:    s_endpgm
+;
+; GFX1032-LABEL: min_i32_varying:
+; GFX1032:       ; %bb.0: ; %entry
+; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX1032-NEXT:    v_cmp_ne_u32_e64 s2, 1, 0
+; GFX1032-NEXT:    ; implicit-def: $vcc_hi
+; GFX1032-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v4, s2, 0
+; GFX1032-NEXT:    s_or_saveexec_b32 s2, -1
+; GFX1032-NEXT:    v_bfrev_b32_e32 v1, -2
+; GFX1032-NEXT:    s_mov_b32 exec_lo, s2
+; GFX1032-NEXT:    s_not_b32 exec_lo, exec_lo
+; GFX1032-NEXT:    v_mov_b32_e32 v2, v1
+; GFX1032-NEXT:    s_not_b32 exec_lo, exec_lo
+; GFX1032-NEXT:    s_or_saveexec_b32 s4, -1
+; GFX1032-NEXT:    v_min_i32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-NEXT:    s_mov_b32 s2, -1
+; GFX1032-NEXT:    v_min_i32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032-NEXT:    v_min_i32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032-NEXT:    v_min_i32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032-NEXT:    v_mov_b32_e32 v3, v2
+; GFX1032-NEXT:    v_permlanex16_b32 v3, v3, -1, -1
+; GFX1032-NEXT:    v_min_i32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
+; GFX1032-NEXT:    v_readlane_b32 s3, v2, 31
+; GFX1032-NEXT:    v_mov_b32_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-NEXT:    v_readlane_b32 s5, v2, 15
+; GFX1032-NEXT:    v_writelane_b32 v1, s5, 16
+; GFX1032-NEXT:    s_mov_b32 exec_lo, s4
+; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v4
+; GFX1032-NEXT:    ; implicit-def: $vgpr0
+; GFX1032-NEXT:    s_and_saveexec_b32 s4, vcc_lo
+; GFX1032-NEXT:    ; mask branch BB19_2
+; GFX1032-NEXT:    s_cbranch_execz BB19_2
+; GFX1032-NEXT:  BB19_1:
+; GFX1032-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
+; GFX1032-NEXT:    v_mov_b32_e32 v7, s3
+; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1032-NEXT:    ds_min_rtn_i32 v0, v0, v7
+; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1032-NEXT:    buffer_gl0_inv
+; GFX1032-NEXT:    buffer_gl1_inv
+; GFX1032-NEXT:  BB19_2:
+; GFX1032-NEXT:    v_nop
+; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX1032-NEXT:    v_readfirstlane_b32 s3, v0
+; GFX1032-NEXT:    v_mov_b32_e32 v0, v1
+; GFX1032-NEXT:    v_min_i32_e32 v0, s3, v0
+; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1032-NEXT:    s_nop 1
+; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX1032-NEXT:    s_endpgm
 entry:
   %lane = call i32 @llvm.amdgcn.workitem.id.x()
   %old = atomicrmw min i32 addrspace(3)* @local_var32, i32 %lane acq_rel
@@ -270,26 +4130,440 @@ entry:
   ret void
 }
 
-; GCN-LABEL: min_i64_constant:
-; GCN: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
-; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_lo:[0-9]+]], s[[exec_lo]], 0
-; GCN: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_hi:[0-9]+]], s[[exec_hi]], v[[mbcnt_lo]]
-; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc, 0, v[[mbcnt_hi]]
-; GCN: v_mov_b32{{(_e[0-9]+)?}} v[[value_lo:[0-9]+]], 5
-; GCN: v_mov_b32{{(_e[0-9]+)?}} v[[value_hi:[0-9]+]], 0
-; GCN: ds_min_rtn_i64 v{{\[}}{{[0-9]+}}:{{[0-9]+}}{{\]}}, v{{[0-9]+}}, v{{\[}}[[value_lo]]:[[value_hi]]{{\]}}
 define amdgpu_kernel void @min_i64_constant(i64 addrspace(1)* %out) {
+;
+;
+; GFX7LESS-LABEL: min_i64_constant:
+; GFX7LESS:       ; %bb.0: ; %entry
+; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; GFX7LESS-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
+; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s3, v0
+; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; GFX7LESS-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX7LESS-NEXT:    ; mask branch BB20_2
+; GFX7LESS-NEXT:    s_cbranch_execz BB20_2
+; GFX7LESS-NEXT:  BB20_1:
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, local_var64@abs32@lo
+; GFX7LESS-NEXT:    v_mov_b32_e32 v0, 5
+; GFX7LESS-NEXT:    v_mov_b32_e32 v1, 0
+; GFX7LESS-NEXT:    s_mov_b32 m0, -1
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7LESS-NEXT:    ds_min_rtn_i64 v[0:1], v2, v[0:1]
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7LESS-NEXT:    buffer_wbinvl1
+; GFX7LESS-NEXT:  BB20_2:
+; GFX7LESS-NEXT:    s_or_b64 exec, exec, s[2:3]
+; GFX7LESS-NEXT:    v_readfirstlane_b32 s4, v0
+; GFX7LESS-NEXT:    v_readfirstlane_b32 s5, v1
+; GFX7LESS-NEXT:    v_bfrev_b32_e32 v1, -2
+; GFX7LESS-NEXT:    s_mov_b32 s2, -1
+; GFX7LESS-NEXT:    v_cndmask_b32_e64 v0, 5, -1, vcc
+; GFX7LESS-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s5
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s4
+; GFX7LESS-NEXT:    v_cmp_lt_i64_e32 vcc, s[4:5], v[0:1]
+; GFX7LESS-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; GFX7LESS-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
+; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX7LESS-NEXT:    s_endpgm
+;
+; GFX8-LABEL: min_i64_constant:
+; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX8-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX8-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; GFX8-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX8-NEXT:    ; mask branch BB20_2
+; GFX8-NEXT:    s_cbranch_execz BB20_2
+; GFX8-NEXT:  BB20_1:
+; GFX8-NEXT:    v_mov_b32_e32 v0, 5
+; GFX8-NEXT:    v_mov_b32_e32 v2, local_var64@abs32@lo
+; GFX8-NEXT:    v_mov_b32_e32 v1, 0
+; GFX8-NEXT:    s_mov_b32 m0, -1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    ds_min_rtn_i64 v[0:1], v2, v[0:1]
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:  BB20_2:
+; GFX8-NEXT:    s_or_b64 exec, exec, s[2:3]
+; GFX8-NEXT:    v_readfirstlane_b32 s4, v0
+; GFX8-NEXT:    v_bfrev_b32_e32 v0, -2
+; GFX8-NEXT:    v_readfirstlane_b32 s5, v1
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, 0, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, 5, -1, vcc
+; GFX8-NEXT:    v_cmp_lt_i64_e32 vcc, s[4:5], v[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v2, s5
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; GFX8-NEXT:    v_mov_b32_e32 v2, s4
+; GFX8-NEXT:    s_mov_b32 s2, -1
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX8-NEXT:    s_mov_b32 s3, 0xf000
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX8-NEXT:    s_endpgm
+;
+; GFX9-LABEL: min_i64_constant:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; GFX9-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX9-NEXT:    ; mask branch BB20_2
+; GFX9-NEXT:    s_cbranch_execz BB20_2
+; GFX9-NEXT:  BB20_1:
+; GFX9-NEXT:    v_mov_b32_e32 v0, 5
+; GFX9-NEXT:    v_mov_b32_e32 v2, local_var64@abs32@lo
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    ds_min_rtn_i64 v[0:1], v2, v[0:1]
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:  BB20_2:
+; GFX9-NEXT:    s_or_b64 exec, exec, s[2:3]
+; GFX9-NEXT:    v_readfirstlane_b32 s4, v0
+; GFX9-NEXT:    v_bfrev_b32_e32 v0, -2
+; GFX9-NEXT:    v_readfirstlane_b32 s5, v1
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, 0, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, 5, -1, vcc
+; GFX9-NEXT:    v_cmp_lt_i64_e32 vcc, s[4:5], v[0:1]
+; GFX9-NEXT:    v_mov_b32_e32 v2, s5
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v2, s4
+; GFX9-NEXT:    s_mov_b32 s2, -1
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX9-NEXT:    s_mov_b32 s3, 0xf000
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX9-NEXT:    s_endpgm
+;
+; GFX1064-LABEL: min_i64_constant:
+; GFX1064:       ; %bb.0: ; %entry
+; GFX1064-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
+; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, s3, v0
+; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; GFX1064-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX1064-NEXT:    ; mask branch BB20_2
+; GFX1064-NEXT:    s_cbranch_execz BB20_2
+; GFX1064-NEXT:  BB20_1:
+; GFX1064-NEXT:    v_mov_b32_e32 v0, 5
+; GFX1064-NEXT:    v_mov_b32_e32 v2, local_var64@abs32@lo
+; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
+; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1064-NEXT:    ds_min_rtn_i64 v[0:1], v2, v[0:1]
+; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1064-NEXT:    buffer_gl0_inv
+; GFX1064-NEXT:    buffer_gl1_inv
+; GFX1064-NEXT:  BB20_2:
+; GFX1064-NEXT:    v_nop
+; GFX1064-NEXT:    s_or_b64 exec, exec, s[2:3]
+; GFX1064-NEXT:    v_readfirstlane_b32 s4, v0
+; GFX1064-NEXT:    v_readfirstlane_b32 s5, v1
+; GFX1064-NEXT:    v_cndmask_b32_e64 v1, 0, 0x7fffffff, vcc
+; GFX1064-NEXT:    v_cndmask_b32_e64 v0, 5, -1, vcc
+; GFX1064-NEXT:    s_mov_b32 s2, -1
+; GFX1064-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1064-NEXT:    v_cmp_lt_i64_e32 vcc, s[4:5], v[0:1]
+; GFX1064-NEXT:    v_cndmask_b32_e64 v1, v1, s5, vcc
+; GFX1064-NEXT:    v_cndmask_b32_e64 v0, v0, s4, vcc
+; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX1064-NEXT:    s_endpgm
+;
+; GFX1032-LABEL: min_i64_constant:
+; GFX1032:       ; %bb.0: ; %entry
+; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX1032-NEXT:    v_cmp_ne_u32_e64 s2, 1, 0
+; GFX1032-NEXT:    ; implicit-def: $vcc_hi
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
+; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; GFX1032-NEXT:    s_and_saveexec_b32 s2, vcc_lo
+; GFX1032-NEXT:    ; mask branch BB20_2
+; GFX1032-NEXT:    s_cbranch_execz BB20_2
+; GFX1032-NEXT:  BB20_1:
+; GFX1032-NEXT:    v_mov_b32_e32 v0, 5
+; GFX1032-NEXT:    v_mov_b32_e32 v2, local_var64@abs32@lo
+; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
+; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1032-NEXT:    ds_min_rtn_i64 v[0:1], v2, v[0:1]
+; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1032-NEXT:    buffer_gl0_inv
+; GFX1032-NEXT:    buffer_gl1_inv
+; GFX1032-NEXT:  BB20_2:
+; GFX1032-NEXT:    v_nop
+; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT:    v_readfirstlane_b32 s4, v0
+; GFX1032-NEXT:    v_readfirstlane_b32 s5, v1
+; GFX1032-NEXT:    v_cndmask_b32_e64 v1, 0, 0x7fffffff, vcc_lo
+; GFX1032-NEXT:    v_cndmask_b32_e64 v0, 5, -1, vcc_lo
+; GFX1032-NEXT:    s_mov_b32 s2, -1
+; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1032-NEXT:    v_cmp_lt_i64_e32 vcc_lo, s[4:5], v[0:1]
+; GFX1032-NEXT:    v_cndmask_b32_e64 v1, v1, s5, vcc_lo
+; GFX1032-NEXT:    v_cndmask_b32_e64 v0, v0, s4, vcc_lo
+; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX1032-NEXT:    s_endpgm
 entry:
   %old = atomicrmw min i64 addrspace(3)* @local_var64, i64 5 acq_rel
   store i64 %old, i64 addrspace(1)* %out
   ret void
 }
 
-; GCN-LABEL: umax_i32_varying:
-; GFX8MORE: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 63
+; GFX8MORE32: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 31
 ; GFX8MORE: v_mov_b32{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[scalar_value]]
 ; GFX8MORE: ds_max_rtn_u32 v{{[0-9]+}}, v{{[0-9]+}}, v[[value]]
 define amdgpu_kernel void @umax_i32_varying(i32 addrspace(1)* %out) {
+;
+;
+; GFX7LESS-LABEL: umax_i32_varying:
+; GFX7LESS:       ; %bb.0: ; %entry
+; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; GFX7LESS-NEXT:    v_mov_b32_e32 v1, local_var32@abs32@lo
+; GFX7LESS-NEXT:    s_mov_b32 m0, -1
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7LESS-NEXT:    ds_max_rtn_u32 v0, v1, v0
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7LESS-NEXT:    buffer_wbinvl1
+; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7LESS-NEXT:    s_mov_b32 s2, -1
+; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX7LESS-NEXT:    s_endpgm
+;
+; GFX8-LABEL: umax_i32_varying:
+; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    v_mov_b32_e32 v2, v0
+; GFX8-NEXT:    s_or_saveexec_b64 s[2:3], -1
+; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX8-NEXT:    v_mov_b32_e32 v1, 0
+; GFX8-NEXT:    s_mov_b64 exec, s[2:3]
+; GFX8-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX8-NEXT:    s_not_b64 exec, exec
+; GFX8-NEXT:    v_mov_b32_e32 v2, 0
+; GFX8-NEXT:    s_not_b64 exec, exec
+; GFX8-NEXT:    s_or_saveexec_b64 s[4:5], -1
+; GFX8-NEXT:    v_max_u32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_max_u32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_max_u32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_max_u32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_max_u32_dpp v2, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_max_u32_dpp v2, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX8-NEXT:    v_readlane_b32 s2, v2, 63
+; GFX8-NEXT:    s_nop 0
+; GFX8-NEXT:    v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf
+; GFX8-NEXT:    s_mov_b64 exec, s[4:5]
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX8-NEXT:    ; implicit-def: $vgpr0
+; GFX8-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX8-NEXT:    ; mask branch BB21_2
+; GFX8-NEXT:    s_cbranch_execz BB21_2
+; GFX8-NEXT:  BB21_1:
+; GFX8-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
+; GFX8-NEXT:    v_mov_b32_e32 v3, s2
+; GFX8-NEXT:    s_mov_b32 m0, -1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    ds_max_rtn_u32 v0, v0, v3
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:  BB21_2:
+; GFX8-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT:    v_readfirstlane_b32 s2, v0
+; GFX8-NEXT:    v_mov_b32_e32 v0, v1
+; GFX8-NEXT:    v_max_u32_e32 v0, s2, v0
+; GFX8-NEXT:    s_mov_b32 s3, 0xf000
+; GFX8-NEXT:    s_mov_b32 s2, -1
+; GFX8-NEXT:    s_nop 0
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX8-NEXT:    s_endpgm
+;
+; GFX9-LABEL: umax_i32_varying:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_or_saveexec_b64 s[2:3], -1
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0
+; GFX9-NEXT:    s_mov_b64 exec, s[2:3]
+; GFX9-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX9-NEXT:    s_not_b64 exec, exec
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-NEXT:    s_not_b64 exec, exec
+; GFX9-NEXT:    s_or_saveexec_b64 s[4:5], -1
+; GFX9-NEXT:    v_max_u32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_max_u32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_max_u32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_max_u32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_max_u32_dpp v2, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_max_u32_dpp v2, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX9-NEXT:    v_readlane_b32 s2, v2, 63
+; GFX9-NEXT:    s_nop 0
+; GFX9-NEXT:    v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-NEXT:    s_mov_b64 exec, s[4:5]
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT:    ; implicit-def: $vgpr0
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    ; mask branch BB21_2
+; GFX9-NEXT:    s_cbranch_execz BB21_2
+; GFX9-NEXT:  BB21_1:
+; GFX9-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
+; GFX9-NEXT:    v_mov_b32_e32 v3, s2
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    ds_max_rtn_u32 v0, v0, v3
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:  BB21_2:
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v0
+; GFX9-NEXT:    v_mov_b32_e32 v0, v1
+; GFX9-NEXT:    v_max_u32_e32 v0, s2, v0
+; GFX9-NEXT:    s_mov_b32 s3, 0xf000
+; GFX9-NEXT:    s_mov_b32 s2, -1
+; GFX9-NEXT:    s_nop 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX9-NEXT:    s_endpgm
+;
+; GFX1064-LABEL: umax_i32_varying:
+; GFX1064:       ; %bb.0: ; %entry
+; GFX1064-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1064-NEXT:    s_or_saveexec_b64 s[2:3], -1
+; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
+; GFX1064-NEXT:    s_mov_b64 exec, s[2:3]
+; GFX1064-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
+; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, s3, v0
+; GFX1064-NEXT:    s_not_b64 exec, exec
+; GFX1064-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1064-NEXT:    s_not_b64 exec, exec
+; GFX1064-NEXT:    s_or_saveexec_b64 s[4:5], -1
+; GFX1064-NEXT:    v_max_u32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX1064-NEXT:    v_max_u32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX1064-NEXT:    v_max_u32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX1064-NEXT:    v_max_u32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX1064-NEXT:    v_mov_b32_e32 v3, v2
+; GFX1064-NEXT:    v_permlanex16_b32 v3, v3, -1, -1
+; GFX1064-NEXT:    v_max_u32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
+; GFX1064-NEXT:    v_readlane_b32 s2, v2, 31
+; GFX1064-NEXT:    v_mov_b32_e32 v3, s2
+; GFX1064-NEXT:    v_max_u32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf
+; GFX1064-NEXT:    v_readlane_b32 s2, v2, 15
+; GFX1064-NEXT:    v_mov_b32_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-NEXT:    v_readlane_b32 s3, v2, 31
+; GFX1064-NEXT:    v_readlane_b32 s6, v2, 47
+; GFX1064-NEXT:    v_writelane_b32 v1, s2, 16
+; GFX1064-NEXT:    s_mov_b32 s2, -1
+; GFX1064-NEXT:    v_writelane_b32 v1, s3, 32
+; GFX1064-NEXT:    v_readlane_b32 s3, v2, 63
+; GFX1064-NEXT:    v_writelane_b32 v1, s6, 48
+; GFX1064-NEXT:    s_mov_b64 exec, s[4:5]
+; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT:    ; implicit-def: $vgpr0
+; GFX1064-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX1064-NEXT:    ; mask branch BB21_2
+; GFX1064-NEXT:    s_cbranch_execz BB21_2
+; GFX1064-NEXT:  BB21_1:
+; GFX1064-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
+; GFX1064-NEXT:    v_mov_b32_e32 v7, s3
+; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1064-NEXT:    ds_max_rtn_u32 v0, v0, v7
+; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1064-NEXT:    buffer_gl0_inv
+; GFX1064-NEXT:    buffer_gl1_inv
+; GFX1064-NEXT:  BB21_2:
+; GFX1064-NEXT:    v_nop
+; GFX1064-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX1064-NEXT:    v_readfirstlane_b32 s3, v0
+; GFX1064-NEXT:    v_mov_b32_e32 v0, v1
+; GFX1064-NEXT:    v_max_u32_e32 v0, s3, v0
+; GFX1064-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1064-NEXT:    s_nop 1
+; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX1064-NEXT:    s_endpgm
+;
+; GFX1032-LABEL: umax_i32_varying:
+; GFX1032:       ; %bb.0: ; %entry
+; GFX1032-NEXT:    ; implicit-def: $vcc_hi
+; GFX1032-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1032-NEXT:    s_or_saveexec_b32 s2, -1
+; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
+; GFX1032-NEXT:    s_mov_b32 exec_lo, s2
+; GFX1032-NEXT:    v_cmp_ne_u32_e64 s2, 1, 0
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
+; GFX1032-NEXT:    s_not_b32 exec_lo, exec_lo
+; GFX1032-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1032-NEXT:    s_not_b32 exec_lo, exec_lo
+; GFX1032-NEXT:    s_or_saveexec_b32 s4, -1
+; GFX1032-NEXT:    v_max_u32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX1032-NEXT:    s_mov_b32 s2, -1
+; GFX1032-NEXT:    v_max_u32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX1032-NEXT:    v_max_u32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX1032-NEXT:    v_max_u32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0
+; GFX1032-NEXT:    v_mov_b32_e32 v3, v2
+; GFX1032-NEXT:    v_permlanex16_b32 v3, v3, -1, -1
+; GFX1032-NEXT:    v_max_u32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
+; GFX1032-NEXT:    v_readlane_b32 s3, v2, 31
+; GFX1032-NEXT:    v_mov_b32_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-NEXT:    v_readlane_b32 s5, v2, 15
+; GFX1032-NEXT:    v_writelane_b32 v1, s5, 16
+; GFX1032-NEXT:    s_mov_b32 exec_lo, s4
+; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-NEXT:    ; implicit-def: $vgpr0
+; GFX1032-NEXT:    s_and_saveexec_b32 s4, vcc_lo
+; GFX1032-NEXT:    ; mask branch BB21_2
+; GFX1032-NEXT:    s_cbranch_execz BB21_2
+; GFX1032-NEXT:  BB21_1:
+; GFX1032-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
+; GFX1032-NEXT:    v_mov_b32_e32 v7, s3
+; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1032-NEXT:    ds_max_rtn_u32 v0, v0, v7
+; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1032-NEXT:    buffer_gl0_inv
+; GFX1032-NEXT:    buffer_gl1_inv
+; GFX1032-NEXT:  BB21_2:
+; GFX1032-NEXT:    v_nop
+; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX1032-NEXT:    v_readfirstlane_b32 s3, v0
+; GFX1032-NEXT:    v_mov_b32_e32 v0, v1
+; GFX1032-NEXT:    v_max_u32_e32 v0, s3, v0
+; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1032-NEXT:    s_nop 1
+; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX1032-NEXT:    s_endpgm
 entry:
   %lane = call i32 @llvm.amdgcn.workitem.id.x()
   %old = atomicrmw umax i32 addrspace(3)* @local_var32, i32 %lane acq_rel
@@ -297,26 +4571,437 @@ entry:
   ret void
 }
 
-; GCN-LABEL: umax_i64_constant:
-; GCN: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
-; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_lo:[0-9]+]], s[[exec_lo]], 0
-; GCN: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_hi:[0-9]+]], s[[exec_hi]], v[[mbcnt_lo]]
-; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc, 0, v[[mbcnt_hi]]
-; GCN: v_mov_b32{{(_e[0-9]+)?}} v[[value_lo:[0-9]+]], 5
-; GCN: v_mov_b32{{(_e[0-9]+)?}} v[[value_hi:[0-9]+]], 0
-; GCN: ds_max_rtn_u64 v{{\[}}{{[0-9]+}}:{{[0-9]+}}{{\]}}, v{{[0-9]+}}, v{{\[}}[[value_lo]]:[[value_hi]]{{\]}}
 define amdgpu_kernel void @umax_i64_constant(i64 addrspace(1)* %out) {
+;
+;
+; GFX7LESS-LABEL: umax_i64_constant:
+; GFX7LESS:       ; %bb.0: ; %entry
+; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; GFX7LESS-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
+; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s3, v0
+; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; GFX7LESS-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX7LESS-NEXT:    ; mask branch BB22_2
+; GFX7LESS-NEXT:    s_cbranch_execz BB22_2
+; GFX7LESS-NEXT:  BB22_1:
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, local_var64@abs32@lo
+; GFX7LESS-NEXT:    v_mov_b32_e32 v0, 5
+; GFX7LESS-NEXT:    v_mov_b32_e32 v1, 0
+; GFX7LESS-NEXT:    s_mov_b32 m0, -1
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7LESS-NEXT:    ds_max_rtn_u64 v[0:1], v2, v[0:1]
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7LESS-NEXT:    buffer_wbinvl1
+; GFX7LESS-NEXT:  BB22_2:
+; GFX7LESS-NEXT:    s_or_b64 exec, exec, s[2:3]
+; GFX7LESS-NEXT:    v_readfirstlane_b32 s4, v0
+; GFX7LESS-NEXT:    v_readfirstlane_b32 s5, v1
+; GFX7LESS-NEXT:    v_mov_b32_e32 v1, 0
+; GFX7LESS-NEXT:    v_cndmask_b32_e64 v0, 5, 0, vcc
+; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s4
+; GFX7LESS-NEXT:    v_cmp_gt_u64_e32 vcc, s[4:5], v[0:1]
+; GFX7LESS-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX7LESS-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7LESS-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX7LESS-NEXT:    s_mov_b32 s2, -1
+; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX7LESS-NEXT:    s_endpgm
+;
+; GFX8-LABEL: umax_i64_constant:
+; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX8-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX8-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; GFX8-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX8-NEXT:    ; mask branch BB22_2
+; GFX8-NEXT:    s_cbranch_execz BB22_2
+; GFX8-NEXT:  BB22_1:
+; GFX8-NEXT:    v_mov_b32_e32 v0, 5
+; GFX8-NEXT:    v_mov_b32_e32 v2, local_var64@abs32@lo
+; GFX8-NEXT:    v_mov_b32_e32 v1, 0
+; GFX8-NEXT:    s_mov_b32 m0, -1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    ds_max_rtn_u64 v[0:1], v2, v[0:1]
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:  BB22_2:
+; GFX8-NEXT:    s_or_b64 exec, exec, s[2:3]
+; GFX8-NEXT:    v_readfirstlane_b32 s2, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s3, v1
+; GFX8-NEXT:    v_mov_b32_e32 v1, 0
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, 5, 0, vcc
+; GFX8-NEXT:    v_cmp_gt_u64_e32 vcc, s[2:3], v[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v1, s3
+; GFX8-NEXT:    v_mov_b32_e32 v2, s2
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX8-NEXT:    s_mov_b32 s3, 0xf000
+; GFX8-NEXT:    s_mov_b32 s2, -1
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX8-NEXT:    s_endpgm
+;
+; GFX9-LABEL: umax_i64_constant:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; GFX9-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX9-NEXT:    ; mask branch BB22_2
+; GFX9-NEXT:    s_cbranch_execz BB22_2
+; GFX9-NEXT:  BB22_1:
+; GFX9-NEXT:    v_mov_b32_e32 v0, 5
+; GFX9-NEXT:    v_mov_b32_e32 v2, local_var64@abs32@lo
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    ds_max_rtn_u64 v[0:1], v2, v[0:1]
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:  BB22_2:
+; GFX9-NEXT:    s_or_b64 exec, exec, s[2:3]
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s3, v1
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, 5, 0, vcc
+; GFX9-NEXT:    v_cmp_gt_u64_e32 vcc, s[2:3], v[0:1]
+; GFX9-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NEXT:    v_mov_b32_e32 v2, s2
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX9-NEXT:    s_mov_b32 s3, 0xf000
+; GFX9-NEXT:    s_mov_b32 s2, -1
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX9-NEXT:    s_endpgm
+;
+; GFX1064-LABEL: umax_i64_constant:
+; GFX1064:       ; %bb.0: ; %entry
+; GFX1064-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
+; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, s3, v0
+; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; GFX1064-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX1064-NEXT:    ; mask branch BB22_2
+; GFX1064-NEXT:    s_cbranch_execz BB22_2
+; GFX1064-NEXT:  BB22_1:
+; GFX1064-NEXT:    v_mov_b32_e32 v0, 5
+; GFX1064-NEXT:    v_mov_b32_e32 v2, local_var64@abs32@lo
+; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
+; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1064-NEXT:    ds_max_rtn_u64 v[0:1], v2, v[0:1]
+; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1064-NEXT:    buffer_gl0_inv
+; GFX1064-NEXT:    buffer_gl1_inv
+; GFX1064-NEXT:  BB22_2:
+; GFX1064-NEXT:    v_nop
+; GFX1064-NEXT:    s_or_b64 exec, exec, s[2:3]
+; GFX1064-NEXT:    v_readfirstlane_b32 s4, v0
+; GFX1064-NEXT:    v_readfirstlane_b32 s5, v1
+; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
+; GFX1064-NEXT:    v_cndmask_b32_e64 v0, 5, 0, vcc
+; GFX1064-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1064-NEXT:    s_mov_b32 s2, -1
+; GFX1064-NEXT:    v_cmp_gt_u64_e32 vcc, s[4:5], v[0:1]
+; GFX1064-NEXT:    v_cndmask_b32_e64 v0, v0, s4, vcc
+; GFX1064-NEXT:    v_cndmask_b32_e64 v1, 0, s5, vcc
+; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX1064-NEXT:    s_endpgm
+;
+; GFX1032-LABEL: umax_i64_constant:
+; GFX1032:       ; %bb.0: ; %entry
+; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX1032-NEXT:    v_cmp_ne_u32_e64 s2, 1, 0
+; GFX1032-NEXT:    ; implicit-def: $vcc_hi
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
+; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; GFX1032-NEXT:    s_and_saveexec_b32 s2, vcc_lo
+; GFX1032-NEXT:    ; mask branch BB22_2
+; GFX1032-NEXT:    s_cbranch_execz BB22_2
+; GFX1032-NEXT:  BB22_1:
+; GFX1032-NEXT:    v_mov_b32_e32 v0, 5
+; GFX1032-NEXT:    v_mov_b32_e32 v2, local_var64@abs32@lo
+; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
+; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1032-NEXT:    ds_max_rtn_u64 v[0:1], v2, v[0:1]
+; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1032-NEXT:    buffer_gl0_inv
+; GFX1032-NEXT:    buffer_gl1_inv
+; GFX1032-NEXT:  BB22_2:
+; GFX1032-NEXT:    v_nop
+; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT:    v_readfirstlane_b32 s4, v0
+; GFX1032-NEXT:    v_readfirstlane_b32 s5, v1
+; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
+; GFX1032-NEXT:    v_cndmask_b32_e64 v0, 5, 0, vcc_lo
+; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1032-NEXT:    s_mov_b32 s2, -1
+; GFX1032-NEXT:    v_cmp_gt_u64_e32 vcc_lo, s[4:5], v[0:1]
+; GFX1032-NEXT:    v_cndmask_b32_e64 v0, v0, s4, vcc_lo
+; GFX1032-NEXT:    v_cndmask_b32_e64 v1, 0, s5, vcc_lo
+; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX1032-NEXT:    s_endpgm
 entry:
   %old = atomicrmw umax i64 addrspace(3)* @local_var64, i64 5 acq_rel
   store i64 %old, i64 addrspace(1)* %out
   ret void
 }
 
-; GCN-LABEL: umin_i32_varying:
-; GFX8MORE: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 63
+; GFX8MORE32: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 31
 ; GFX8MORE: v_mov_b32{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[scalar_value]]
 ; GFX8MORE: ds_min_rtn_u32 v{{[0-9]+}}, v{{[0-9]+}}, v[[value]]
 define amdgpu_kernel void @umin_i32_varying(i32 addrspace(1)* %out) {
+;
+;
+; GFX7LESS-LABEL: umin_i32_varying:
+; GFX7LESS:       ; %bb.0: ; %entry
+; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; GFX7LESS-NEXT:    v_mov_b32_e32 v1, local_var32@abs32@lo
+; GFX7LESS-NEXT:    s_mov_b32 m0, -1
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7LESS-NEXT:    ds_min_rtn_u32 v0, v1, v0
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7LESS-NEXT:    buffer_wbinvl1
+; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7LESS-NEXT:    s_mov_b32 s2, -1
+; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX7LESS-NEXT:    s_endpgm
+;
+; GFX8-LABEL: umin_i32_varying:
+; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX8-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v3, s2, 0
+; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v3, s3, v3
+; GFX8-NEXT:    v_mov_b32_e32 v2, v0
+; GFX8-NEXT:    s_or_saveexec_b64 s[2:3], -1
+; GFX8-NEXT:    v_mov_b32_e32 v1, -1
+; GFX8-NEXT:    s_mov_b64 exec, s[2:3]
+; GFX8-NEXT:    s_not_b64 exec, exec
+; GFX8-NEXT:    v_mov_b32_e32 v2, -1
+; GFX8-NEXT:    s_not_b64 exec, exec
+; GFX8-NEXT:    s_or_saveexec_b64 s[4:5], -1
+; GFX8-NEXT:    v_min_u32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_min_u32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_min_u32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_min_u32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_min_u32_dpp v2, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_min_u32_dpp v2, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX8-NEXT:    v_readlane_b32 s2, v2, 63
+; GFX8-NEXT:    s_nop 0
+; GFX8-NEXT:    v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf
+; GFX8-NEXT:    s_mov_b64 exec, s[4:5]
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; GFX8-NEXT:    ; implicit-def: $vgpr0
+; GFX8-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX8-NEXT:    ; mask branch BB23_2
+; GFX8-NEXT:    s_cbranch_execz BB23_2
+; GFX8-NEXT:  BB23_1:
+; GFX8-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
+; GFX8-NEXT:    v_mov_b32_e32 v3, s2
+; GFX8-NEXT:    s_mov_b32 m0, -1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    ds_min_rtn_u32 v0, v0, v3
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:  BB23_2:
+; GFX8-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT:    v_readfirstlane_b32 s2, v0
+; GFX8-NEXT:    v_mov_b32_e32 v0, v1
+; GFX8-NEXT:    v_min_u32_e32 v0, s2, v0
+; GFX8-NEXT:    s_mov_b32 s3, 0xf000
+; GFX8-NEXT:    s_mov_b32 s2, -1
+; GFX8-NEXT:    s_nop 0
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX8-NEXT:    s_endpgm
+;
+; GFX9-LABEL: umin_i32_varying:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v3, s2, 0
+; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v3, s3, v3
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_or_saveexec_b64 s[2:3], -1
+; GFX9-NEXT:    v_mov_b32_e32 v1, -1
+; GFX9-NEXT:    s_mov_b64 exec, s[2:3]
+; GFX9-NEXT:    s_not_b64 exec, exec
+; GFX9-NEXT:    v_mov_b32_e32 v2, -1
+; GFX9-NEXT:    s_not_b64 exec, exec
+; GFX9-NEXT:    s_or_saveexec_b64 s[4:5], -1
+; GFX9-NEXT:    v_min_u32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_min_u32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_min_u32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_min_u32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_min_u32_dpp v2, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_min_u32_dpp v2, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX9-NEXT:    v_readlane_b32 s2, v2, 63
+; GFX9-NEXT:    s_nop 0
+; GFX9-NEXT:    v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-NEXT:    s_mov_b64 exec, s[4:5]
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; GFX9-NEXT:    ; implicit-def: $vgpr0
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    ; mask branch BB23_2
+; GFX9-NEXT:    s_cbranch_execz BB23_2
+; GFX9-NEXT:  BB23_1:
+; GFX9-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
+; GFX9-NEXT:    v_mov_b32_e32 v3, s2
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    ds_min_rtn_u32 v0, v0, v3
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:  BB23_2:
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v0
+; GFX9-NEXT:    v_mov_b32_e32 v0, v1
+; GFX9-NEXT:    v_min_u32_e32 v0, s2, v0
+; GFX9-NEXT:    s_mov_b32 s3, 0xf000
+; GFX9-NEXT:    s_mov_b32 s2, -1
+; GFX9-NEXT:    s_nop 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX9-NEXT:    s_endpgm
+;
+; GFX1064-LABEL: umin_i32_varying:
+; GFX1064:       ; %bb.0: ; %entry
+; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX1064-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX1064-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v4, s2, 0
+; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v4, s3, v4
+; GFX1064-NEXT:    s_or_saveexec_b64 s[2:3], -1
+; GFX1064-NEXT:    v_mov_b32_e32 v1, -1
+; GFX1064-NEXT:    s_mov_b64 exec, s[2:3]
+; GFX1064-NEXT:    s_not_b64 exec, exec
+; GFX1064-NEXT:    v_mov_b32_e32 v2, -1
+; GFX1064-NEXT:    s_not_b64 exec, exec
+; GFX1064-NEXT:    s_or_saveexec_b64 s[4:5], -1
+; GFX1064-NEXT:    v_min_u32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-NEXT:    v_min_u32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064-NEXT:    v_min_u32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064-NEXT:    v_min_u32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-NEXT:    v_mov_b32_e32 v3, v2
+; GFX1064-NEXT:    v_permlanex16_b32 v3, v3, -1, -1
+; GFX1064-NEXT:    v_min_u32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
+; GFX1064-NEXT:    v_readlane_b32 s2, v2, 31
+; GFX1064-NEXT:    v_mov_b32_e32 v3, s2
+; GFX1064-NEXT:    v_min_u32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf
+; GFX1064-NEXT:    v_readlane_b32 s2, v2, 15
+; GFX1064-NEXT:    v_mov_b32_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-NEXT:    v_readlane_b32 s3, v2, 31
+; GFX1064-NEXT:    v_readlane_b32 s6, v2, 47
+; GFX1064-NEXT:    v_writelane_b32 v1, s2, 16
+; GFX1064-NEXT:    s_mov_b32 s2, -1
+; GFX1064-NEXT:    v_writelane_b32 v1, s3, 32
+; GFX1064-NEXT:    v_readlane_b32 s3, v2, 63
+; GFX1064-NEXT:    v_writelane_b32 v1, s6, 48
+; GFX1064-NEXT:    s_mov_b64 exec, s[4:5]
+; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
+; GFX1064-NEXT:    ; implicit-def: $vgpr0
+; GFX1064-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX1064-NEXT:    ; mask branch BB23_2
+; GFX1064-NEXT:    s_cbranch_execz BB23_2
+; GFX1064-NEXT:  BB23_1:
+; GFX1064-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
+; GFX1064-NEXT:    v_mov_b32_e32 v7, s3
+; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1064-NEXT:    ds_min_rtn_u32 v0, v0, v7
+; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1064-NEXT:    buffer_gl0_inv
+; GFX1064-NEXT:    buffer_gl1_inv
+; GFX1064-NEXT:  BB23_2:
+; GFX1064-NEXT:    v_nop
+; GFX1064-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX1064-NEXT:    v_readfirstlane_b32 s3, v0
+; GFX1064-NEXT:    v_mov_b32_e32 v0, v1
+; GFX1064-NEXT:    v_min_u32_e32 v0, s3, v0
+; GFX1064-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1064-NEXT:    s_nop 1
+; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX1064-NEXT:    s_endpgm
+;
+; GFX1032-LABEL: umin_i32_varying:
+; GFX1032:       ; %bb.0: ; %entry
+; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX1032-NEXT:    v_cmp_ne_u32_e64 s2, 1, 0
+; GFX1032-NEXT:    ; implicit-def: $vcc_hi
+; GFX1032-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v4, s2, 0
+; GFX1032-NEXT:    s_or_saveexec_b32 s2, -1
+; GFX1032-NEXT:    v_mov_b32_e32 v1, -1
+; GFX1032-NEXT:    s_mov_b32 exec_lo, s2
+; GFX1032-NEXT:    s_not_b32 exec_lo, exec_lo
+; GFX1032-NEXT:    v_mov_b32_e32 v2, -1
+; GFX1032-NEXT:    s_not_b32 exec_lo, exec_lo
+; GFX1032-NEXT:    s_or_saveexec_b32 s4, -1
+; GFX1032-NEXT:    v_min_u32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-NEXT:    s_mov_b32 s2, -1
+; GFX1032-NEXT:    v_min_u32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032-NEXT:    v_min_u32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032-NEXT:    v_min_u32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032-NEXT:    v_mov_b32_e32 v3, v2
+; GFX1032-NEXT:    v_permlanex16_b32 v3, v3, -1, -1
+; GFX1032-NEXT:    v_min_u32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
+; GFX1032-NEXT:    v_readlane_b32 s3, v2, 31
+; GFX1032-NEXT:    v_mov_b32_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-NEXT:    v_readlane_b32 s5, v2, 15
+; GFX1032-NEXT:    v_writelane_b32 v1, s5, 16
+; GFX1032-NEXT:    s_mov_b32 exec_lo, s4
+; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v4
+; GFX1032-NEXT:    ; implicit-def: $vgpr0
+; GFX1032-NEXT:    s_and_saveexec_b32 s4, vcc_lo
+; GFX1032-NEXT:    ; mask branch BB23_2
+; GFX1032-NEXT:    s_cbranch_execz BB23_2
+; GFX1032-NEXT:  BB23_1:
+; GFX1032-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
+; GFX1032-NEXT:    v_mov_b32_e32 v7, s3
+; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1032-NEXT:    ds_min_rtn_u32 v0, v0, v7
+; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1032-NEXT:    buffer_gl0_inv
+; GFX1032-NEXT:    buffer_gl1_inv
+; GFX1032-NEXT:  BB23_2:
+; GFX1032-NEXT:    v_nop
+; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX1032-NEXT:    v_readfirstlane_b32 s3, v0
+; GFX1032-NEXT:    v_mov_b32_e32 v0, v1
+; GFX1032-NEXT:    v_min_u32_e32 v0, s3, v0
+; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1032-NEXT:    s_nop 1
+; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX1032-NEXT:    s_endpgm
 entry:
   %lane = call i32 @llvm.amdgcn.workitem.id.x()
   %old = atomicrmw umin i32 addrspace(3)* @local_var32, i32 %lane acq_rel
@@ -324,15 +5009,192 @@ entry:
   ret void
 }
 
-; GCN-LABEL: umin_i64_constant:
-; GCN: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
-; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_lo:[0-9]+]], s[[exec_lo]], 0
-; GCN: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_hi:[0-9]+]], s[[exec_hi]], v[[mbcnt_lo]]
-; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc, 0, v[[mbcnt_hi]]
-; GCN: v_mov_b32{{(_e[0-9]+)?}} v[[value_lo:[0-9]+]], 5
-; GCN: v_mov_b32{{(_e[0-9]+)?}} v[[value_hi:[0-9]+]], 0
-; GCN: ds_min_rtn_u64 v{{\[}}{{[0-9]+}}:{{[0-9]+}}{{\]}}, v{{[0-9]+}}, v{{\[}}[[value_lo]]:[[value_hi]]{{\]}}
 define amdgpu_kernel void @umin_i64_constant(i64 addrspace(1)* %out) {
+;
+;
+; GFX7LESS-LABEL: umin_i64_constant:
+; GFX7LESS:       ; %bb.0: ; %entry
+; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; GFX7LESS-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
+; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s3, v0
+; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; GFX7LESS-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX7LESS-NEXT:    ; mask branch BB24_2
+; GFX7LESS-NEXT:    s_cbranch_execz BB24_2
+; GFX7LESS-NEXT:  BB24_1:
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, local_var64@abs32@lo
+; GFX7LESS-NEXT:    v_mov_b32_e32 v0, 5
+; GFX7LESS-NEXT:    v_mov_b32_e32 v1, 0
+; GFX7LESS-NEXT:    s_mov_b32 m0, -1
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7LESS-NEXT:    ds_min_rtn_u64 v[0:1], v2, v[0:1]
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7LESS-NEXT:    buffer_wbinvl1
+; GFX7LESS-NEXT:  BB24_2:
+; GFX7LESS-NEXT:    s_or_b64 exec, exec, s[2:3]
+; GFX7LESS-NEXT:    v_readfirstlane_b32 s4, v0
+; GFX7LESS-NEXT:    v_readfirstlane_b32 s5, v1
+; GFX7LESS-NEXT:    s_mov_b32 s2, -1
+; GFX7LESS-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc
+; GFX7LESS-NEXT:    v_cndmask_b32_e64 v0, 5, -1, vcc
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s5
+; GFX7LESS-NEXT:    v_cmp_lt_u64_e32 vcc, s[4:5], v[0:1]
+; GFX7LESS-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s4
+; GFX7LESS-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX7LESS-NEXT:    s_endpgm
+;
+; GFX8-LABEL: umin_i64_constant:
+; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX8-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX8-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; GFX8-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX8-NEXT:    ; mask branch BB24_2
+; GFX8-NEXT:    s_cbranch_execz BB24_2
+; GFX8-NEXT:  BB24_1:
+; GFX8-NEXT:    v_mov_b32_e32 v0, 5
+; GFX8-NEXT:    v_mov_b32_e32 v2, local_var64@abs32@lo
+; GFX8-NEXT:    v_mov_b32_e32 v1, 0
+; GFX8-NEXT:    s_mov_b32 m0, -1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    ds_min_rtn_u64 v[0:1], v2, v[0:1]
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:  BB24_2:
+; GFX8-NEXT:    s_or_b64 exec, exec, s[2:3]
+; GFX8-NEXT:    v_readfirstlane_b32 s5, v1
+; GFX8-NEXT:    v_readfirstlane_b32 s4, v0
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, 5, -1, vcc
+; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, s[4:5], v[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v2, s5
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; GFX8-NEXT:    v_mov_b32_e32 v2, s4
+; GFX8-NEXT:    s_mov_b32 s2, -1
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX8-NEXT:    s_mov_b32 s3, 0xf000
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX8-NEXT:    s_endpgm
+;
+; GFX9-LABEL: umin_i64_constant:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; GFX9-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX9-NEXT:    ; mask branch BB24_2
+; GFX9-NEXT:    s_cbranch_execz BB24_2
+; GFX9-NEXT:  BB24_1:
+; GFX9-NEXT:    v_mov_b32_e32 v0, 5
+; GFX9-NEXT:    v_mov_b32_e32 v2, local_var64@abs32@lo
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    ds_min_rtn_u64 v[0:1], v2, v[0:1]
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:  BB24_2:
+; GFX9-NEXT:    s_or_b64 exec, exec, s[2:3]
+; GFX9-NEXT:    v_readfirstlane_b32 s5, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s4, v0
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, 5, -1, vcc
+; GFX9-NEXT:    v_cmp_lt_u64_e32 vcc, s[4:5], v[0:1]
+; GFX9-NEXT:    v_mov_b32_e32 v2, s5
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v2, s4
+; GFX9-NEXT:    s_mov_b32 s2, -1
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX9-NEXT:    s_mov_b32 s3, 0xf000
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX9-NEXT:    s_endpgm
+;
+; GFX1064-LABEL: umin_i64_constant:
+; GFX1064:       ; %bb.0: ; %entry
+; GFX1064-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
+; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, s3, v0
+; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; GFX1064-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX1064-NEXT:    ; mask branch BB24_2
+; GFX1064-NEXT:    s_cbranch_execz BB24_2
+; GFX1064-NEXT:  BB24_1:
+; GFX1064-NEXT:    v_mov_b32_e32 v0, 5
+; GFX1064-NEXT:    v_mov_b32_e32 v2, local_var64@abs32@lo
+; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
+; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1064-NEXT:    ds_min_rtn_u64 v[0:1], v2, v[0:1]
+; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1064-NEXT:    buffer_gl0_inv
+; GFX1064-NEXT:    buffer_gl1_inv
+; GFX1064-NEXT:  BB24_2:
+; GFX1064-NEXT:    v_nop
+; GFX1064-NEXT:    s_or_b64 exec, exec, s[2:3]
+; GFX1064-NEXT:    v_readfirstlane_b32 s4, v0
+; GFX1064-NEXT:    v_readfirstlane_b32 s5, v1
+; GFX1064-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc
+; GFX1064-NEXT:    v_cndmask_b32_e64 v0, 5, -1, vcc
+; GFX1064-NEXT:    s_mov_b32 s2, -1
+; GFX1064-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1064-NEXT:    v_cmp_lt_u64_e32 vcc, s[4:5], v[0:1]
+; GFX1064-NEXT:    v_cndmask_b32_e64 v1, v1, s5, vcc
+; GFX1064-NEXT:    v_cndmask_b32_e64 v0, v0, s4, vcc
+; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX1064-NEXT:    s_endpgm
+;
+; GFX1032-LABEL: umin_i64_constant:
+; GFX1032:       ; %bb.0: ; %entry
+; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX1032-NEXT:    v_cmp_ne_u32_e64 s2, 1, 0
+; GFX1032-NEXT:    ; implicit-def: $vcc_hi
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
+; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; GFX1032-NEXT:    s_and_saveexec_b32 s2, vcc_lo
+; GFX1032-NEXT:    ; mask branch BB24_2
+; GFX1032-NEXT:    s_cbranch_execz BB24_2
+; GFX1032-NEXT:  BB24_1:
+; GFX1032-NEXT:    v_mov_b32_e32 v0, 5
+; GFX1032-NEXT:    v_mov_b32_e32 v2, local_var64@abs32@lo
+; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
+; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1032-NEXT:    ds_min_rtn_u64 v[0:1], v2, v[0:1]
+; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1032-NEXT:    buffer_gl0_inv
+; GFX1032-NEXT:    buffer_gl1_inv
+; GFX1032-NEXT:  BB24_2:
+; GFX1032-NEXT:    v_nop
+; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT:    v_readfirstlane_b32 s4, v0
+; GFX1032-NEXT:    v_readfirstlane_b32 s5, v1
+; GFX1032-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX1032-NEXT:    v_cndmask_b32_e64 v0, 5, -1, vcc_lo
+; GFX1032-NEXT:    s_mov_b32 s2, -1
+; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1032-NEXT:    v_cmp_lt_u64_e32 vcc_lo, s[4:5], v[0:1]
+; GFX1032-NEXT:    v_cndmask_b32_e64 v1, v1, s5, vcc_lo
+; GFX1032-NEXT:    v_cndmask_b32_e64 v0, v0, s4, vcc_lo
+; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX1032-NEXT:    s_endpgm
 entry:
   %old = atomicrmw umin i64 addrspace(3)* @local_var64, i64 5 acq_rel
   store i64 %old, i64 addrspace(1)* %out
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_pixelshader.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_pixelshader.ll
index a170f96e4..81d045b14 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_pixelshader.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_pixelshader.ll
@@ -1,6 +1,8 @@
-; RUN: llc -mtriple=amdgcn-- -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX7LESS %s
-; RUN: llc  -mtriple=amdgcn-- -mcpu=tonga -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX8MORE %s
-; RUN: llc -mtriple=amdgcn-- -mcpu=gfx900 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX8MORE %s
+; RUN: llc -mtriple=amdgcn-- -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN64,GFX7LESS %s
+; RUN: llc  -mtriple=amdgcn-- -mcpu=tonga -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN64,GFX8MORE,GFX8MORE64,DPPCOMB %s
+; RUN: llc -mtriple=amdgcn-- -mcpu=gfx900 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN64,GFX8MORE,GFX8MORE64,DPPCOMB %s
+; RUN: llc -mtriple=amdgcn-- -mcpu=gfx1010 -mattr=-wavefrontsize32,+wavefrontsize64 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN64,GFX8MORE,GFX8MORE64 %s
+; RUN: llc -mtriple=amdgcn-- -mcpu=gfx1010 -mattr=+wavefrontsize32,-wavefrontsize64 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN32,GFX8MORE,GFX8MORE32 %s
 
 declare i1 @llvm.amdgcn.wqm.vote(i1)
 declare i32 @llvm.amdgcn.buffer.atomic.add(i32, <4 x i32>, i32, i32, i1)
@@ -10,11 +12,13 @@ declare void @llvm.amdgcn.buffer.store.f32(float, <4 x i32>, i32, i32, i1, i1)
 
 ; GCN-LABEL: add_i32_constant:
 ; GCN-LABEL: BB0_1:
-; GCN: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
-; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_lo:[0-9]+]], s[[exec_lo]], 0
-; GCN: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_hi:[0-9]+]], s[[exec_hi]], v[[mbcnt_lo]]
-; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc, 0, v[[mbcnt_hi]]
-; GCN: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
+; GCN32: v_cmp_ne_u32_e64 s[[exec_lo:[0-9]+]], 1, 0
+; GCN64: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
+; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt:[0-9]+]], s[[exec_lo]], 0
+; GCN64: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt]], s[[exec_hi]], v[[mbcnt]]
+; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc{{(_lo)?}}, 0, v[[mbcnt]]
+; GCN32: s_bcnt1_i32_b32 s[[popcount:[0-9]+]], s[[exec_lo]]
+; GCN64: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
 ; GCN: v_mul_u32_u24{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[popcount]], 5
 ; GCN: buffer_atomic_add v[[value]]
 ; GCN: v_readfirstlane_b32 s{{[0-9]+}}, v[[value]]
@@ -36,13 +40,15 @@ else:
 ; GCN-LABEL: add_i32_varying:
 ; GFX7LESS-NOT: v_mbcnt_lo_u32_b32
 ; GFX7LESS-NOT: v_mbcnt_hi_u32_b32
-; GFX8MORE: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
-; GFX8MORE: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_lo:[0-9]+]], s[[exec_lo]], 0
-; GFX8MORE: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_hi:[0-9]+]], s[[exec_hi]], v[[mbcnt_lo]]
-; GFX8MORE: v_add_u32_dpp
-; GFX8MORE: v_add_u32_dpp
-; GFX8MORE: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 63
-; GFX8MORE: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc, 0, v[[mbcnt_hi]]
+; GFX8MORE32: v_cmp_ne_u32_e64 s[[exec_lo:[0-9]+]], 1, 0
+; GFX8MORE64: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
+; GFX8MORE: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt:[0-9]+]], s[[exec_lo]], 0
+; GFX8MORE64: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt]], s[[exec_hi]], v[[mbcnt]]
+; DPPCOMB: v_add_u32_dpp
+; DPPCOMB: v_add_u32_dpp
+; GFX8MORE32: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 31
+; GFX8MORE64: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 63
+; GFX8MORE: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc{{(_lo)?}}, 0, v[[mbcnt]]
 ; GFX8MORE: v_mov_b32{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[scalar_value]]
 ; GFX8MORE: buffer_atomic_add v[[value]]
 ; GFX8MORE: v_readfirstlane_b32 s{{[0-9]+}}, v[[value]]
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_raw_buffer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_raw_buffer.ll
index dd813df32..084d901a2 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_raw_buffer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_raw_buffer.ll
@@ -1,6 +1,8 @@
-; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX7LESS %s
-; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=tonga -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX8MORE %s
-; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=gfx900 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX8MORE %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN64,GFX7LESS %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=tonga -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN64,GFX8MORE,GFX8MORE64,DPPCOMB %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=gfx900 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN64,GFX8MORE,GFX8MORE64,DPPCOMB %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=gfx1010 -mattr=-wavefrontsize32,+wavefrontsize64 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN64,GFX8MORE,GFX8MORE64 %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=gfx1010 -mattr=+wavefrontsize32,-wavefrontsize64 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN32,GFX8MORE,GFX8MORE32 %s
 
 declare i32 @llvm.amdgcn.workitem.id.x()
 declare i32 @llvm.amdgcn.raw.buffer.atomic.add(i32, <4 x i32>, i32, i32, i32)
@@ -9,11 +11,13 @@ declare i32 @llvm.amdgcn.raw.buffer.atomic.sub(i32, <4 x i32>, i32, i32, i32)
 ; Show that what the atomic optimization pass will do for raw buffers.
 
 ; GCN-LABEL: add_i32_constant:
-; GCN: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
-; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_lo:[0-9]+]], s[[exec_lo]], 0
-; GCN: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_hi:[0-9]+]], s[[exec_hi]], v[[mbcnt_lo]]
-; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc, 0, v[[mbcnt_hi]]
-; GCN: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
+; GCN32: v_cmp_ne_u32_e64 s[[exec_lo:[0-9]+]], 1, 0
+; GCN64: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
+; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt:[0-9]+]], s[[exec_lo]], 0
+; GCN64: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt]], s[[exec_hi]], v[[mbcnt]]
+; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc{{(_lo)?}}, 0, v[[mbcnt]]
+; GCN32: s_bcnt1_i32_b32 s[[popcount:[0-9]+]], s[[exec_lo]]
+; GCN64: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
 ; GCN: v_mul_u32_u24{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[popcount]], 5
 ; GCN: buffer_atomic_add v[[value]]
 define amdgpu_kernel void @add_i32_constant(i32 addrspace(1)* %out, <4 x i32> %inout) {
@@ -24,11 +28,13 @@ entry:
 }
 
 ; GCN-LABEL: add_i32_uniform:
-; GCN: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
-; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_lo:[0-9]+]], s[[exec_lo]], 0
-; GCN: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_hi:[0-9]+]], s[[exec_hi]], v[[mbcnt_lo]]
-; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc, 0, v[[mbcnt_hi]]
-; GCN: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
+; GCN32: v_cmp_ne_u32_e64 s[[exec_lo:[0-9]+]], 1, 0
+; GCN64: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
+; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt:[0-9]+]], s[[exec_lo]], 0
+; GCN64: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt]], s[[exec_hi]], v[[mbcnt]]
+; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc{{(_lo)?}}, 0, v[[mbcnt]]
+; GCN32: s_bcnt1_i32_b32 s[[popcount:[0-9]+]], s[[exec_lo]]
+; GCN64: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
 ; GCN: s_mul_i32 s[[scalar_value:[0-9]+]], s{{[0-9]+}}, s[[popcount]]
 ; GCN: v_mov_b32{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[scalar_value]]
 ; GCN: buffer_atomic_add v[[value]]
@@ -44,9 +50,10 @@ entry:
 ; GFX7LESS-NOT: v_mbcnt_hi_u32_b32
 ; GFX7LESS-NOT: s_bcnt1_i32_b64
 ; GFX7LESS: buffer_atomic_add v{{[0-9]+}}
-; GFX8MORE: v_add_u32_dpp
-; GFX8MORE: v_add_u32_dpp
-; GFX8MORE: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 63
+; DPPCOMB: v_add_u32_dpp
+; DPPCOMB: v_add_u32_dpp
+; GFX8MORE32: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 31
+; GFX8MORE64: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 63
 ; GFX8MORE: v_mov_b32{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[scalar_value]]
 ; GFX8MORE: buffer_atomic_add v[[value]]
 define amdgpu_kernel void @add_i32_varying_vdata(i32 addrspace(1)* %out, <4 x i32> %inout) {
@@ -71,11 +78,13 @@ entry:
 }
 
 ; GCN-LABEL: sub_i32_constant:
-; GCN: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
-; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_lo:[0-9]+]], s[[exec_lo]], 0
-; GCN: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_hi:[0-9]+]], s[[exec_hi]], v[[mbcnt_lo]]
-; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc, 0, v[[mbcnt_hi]]
-; GCN: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
+; GCN32: v_cmp_ne_u32_e64 s[[exec_lo:[0-9]+]], 1, 0
+; GCN64: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
+; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt:[0-9]+]], s[[exec_lo]], 0
+; GCN64: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt]], s[[exec_hi]], v[[mbcnt]]
+; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc{{(_lo)?}}, 0, v[[mbcnt]]
+; GCN32: s_bcnt1_i32_b32 s[[popcount:[0-9]+]], s[[exec_lo]]
+; GCN64: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
 ; GCN: v_mul_u32_u24{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[popcount]], 5
 ; GCN: buffer_atomic_sub v[[value]]
 define amdgpu_kernel void @sub_i32_constant(i32 addrspace(1)* %out, <4 x i32> %inout) {
@@ -86,11 +95,13 @@ entry:
 }
 
 ; GCN-LABEL: sub_i32_uniform:
-; GCN: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
-; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_lo:[0-9]+]], s[[exec_lo]], 0
-; GCN: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_hi:[0-9]+]], s[[exec_hi]], v[[mbcnt_lo]]
-; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc, 0, v[[mbcnt_hi]]
-; GCN: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
+; GCN32: v_cmp_ne_u32_e64 s[[exec_lo:[0-9]+]], 1, 0
+; GCN64: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
+; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt:[0-9]+]], s[[exec_lo]], 0
+; GCN64: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt]], s[[exec_hi]], v[[mbcnt]]
+; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc{{(_lo)?}}, 0, v[[mbcnt]]
+; GCN32: s_bcnt1_i32_b32 s[[popcount:[0-9]+]], s[[exec_lo]]
+; GCN64: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
 ; GCN: s_mul_i32 s[[scalar_value:[0-9]+]], s{{[0-9]+}}, s[[popcount]]
 ; GCN: v_mov_b32{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[scalar_value]]
 ; GCN: buffer_atomic_sub v[[value]]
@@ -106,9 +117,10 @@ entry:
 ; GFX7LESS-NOT: v_mbcnt_hi_u32_b32
 ; GFX7LESS-NOT: s_bcnt1_i32_b64
 ; GFX7LESS: buffer_atomic_sub v{{[0-9]+}}
-; GFX8MORE: v_sub{{(rev)?}}_u32_dpp
-; GFX8MORE: v_sub{{(rev)?}}_u32_dpp
-; GFX8MORE: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 63
+; DPPCOMB: v_add_u32_dpp
+; DPPCOMB: v_add_u32_dpp
+; GFX8MORE32: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 31
+; GFX8MORE64: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 63
 ; GFX8MORE: v_mov_b32{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[scalar_value]]
 ; GFX8MORE: buffer_atomic_sub v[[value]]
 define amdgpu_kernel void @sub_i32_varying_vdata(i32 addrspace(1)* %out, <4 x i32> %inout) {
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_struct_buffer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_struct_buffer.ll
index c32a73ef2..ec9625131 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_struct_buffer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_struct_buffer.ll
@@ -1,6 +1,8 @@
-; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX7LESS %s
-; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=tonga -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX8MORE %s
-; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=gfx900 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX8MORE %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN64,GFX7LESS %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=tonga -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN64,GFX8MORE,GFX8MORE64,DPPCOMB %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=gfx900 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN64,GFX8MORE,GFX8MORE64,DPPCOMB %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=gfx1010 -mattr=-wavefrontsize32,+wavefrontsize64 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN64,GFX8MORE,GFX8MORE64 %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=gfx1010 -mattr=+wavefrontsize32,-wavefrontsize64 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN32,GFX8MORE,GFX8MORE32 %s
 
 declare i32 @llvm.amdgcn.workitem.id.x()
 declare i32 @llvm.amdgcn.struct.buffer.atomic.add(i32, <4 x i32>, i32, i32, i32, i32)
@@ -9,11 +11,13 @@ declare i32 @llvm.amdgcn.struct.buffer.atomic.sub(i32, <4 x i32>, i32, i32, i32,
 ; Show that what the atomic optimization pass will do for struct buffers.
 
 ; GCN-LABEL: add_i32_constant:
-; GCN: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
-; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_lo:[0-9]+]], s[[exec_lo]], 0
-; GCN: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_hi:[0-9]+]], s[[exec_hi]], v[[mbcnt_lo]]
-; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc, 0, v[[mbcnt_hi]]
-; GCN: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
+; GCN32: v_cmp_ne_u32_e64 s[[exec_lo:[0-9]+]], 1, 0
+; GCN64: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
+; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt:[0-9]+]], s[[exec_lo]], 0
+; GCN64: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt]], s[[exec_hi]], v[[mbcnt]]
+; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc{{(_lo)?}}, 0, v[[mbcnt]]
+; GCN32: s_bcnt1_i32_b32 s[[popcount:[0-9]+]], s[[exec_lo]]
+; GCN64: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
 ; GCN: v_mul_u32_u24{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[popcount]], 5
 ; GCN: buffer_atomic_add v[[value]]
 define amdgpu_kernel void @add_i32_constant(i32 addrspace(1)* %out, <4 x i32> %inout) {
@@ -24,11 +28,13 @@ entry:
 }
 
 ; GCN-LABEL: add_i32_uniform:
-; GCN: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
-; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_lo:[0-9]+]], s[[exec_lo]], 0
-; GCN: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_hi:[0-9]+]], s[[exec_hi]], v[[mbcnt_lo]]
-; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc, 0, v[[mbcnt_hi]]
-; GCN: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
+; GCN32: v_cmp_ne_u32_e64 s[[exec_lo:[0-9]+]], 1, 0
+; GCN64: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
+; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt:[0-9]+]], s[[exec_lo]], 0
+; GCN64: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt]], s[[exec_hi]], v[[mbcnt]]
+; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc{{(_lo)?}}, 0, v[[mbcnt]]
+; GCN32: s_bcnt1_i32_b32 s[[popcount:[0-9]+]], s[[exec_lo]]
+; GCN64: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
 ; GCN: s_mul_i32 s[[scalar_value:[0-9]+]], s{{[0-9]+}}, s[[popcount]]
 ; GCN: v_mov_b32{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[scalar_value]]
 ; GCN: buffer_atomic_add v[[value]]
@@ -44,9 +50,10 @@ entry:
 ; GFX7LESS-NOT: v_mbcnt_hi_u32_b32
 ; GFX7LESS-NOT: s_bcnt1_i32_b64
 ; GFX7LESS: buffer_atomic_add v{{[0-9]+}}
-; GFX8MORE: v_add_u32_dpp
-; GFX8MORE: v_add_u32_dpp
-; GFX8MORE: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 63
+; DPPCOMB: v_add_u32_dpp
+; DPPCOMB: v_add_u32_dpp
+; GFX8MORE32: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 31
+; GFX8MORE64: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 63
 ; GFX8MORE: v_mov_b32{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[scalar_value]]
 ; GFX8MORE: buffer_atomic_add v[[value]]
 define amdgpu_kernel void @add_i32_varying_vdata(i32 addrspace(1)* %out, <4 x i32> %inout) {
@@ -84,11 +91,13 @@ entry:
 }
 
 ; GCN-LABEL: sub_i32_constant:
-; GCN: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
-; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_lo:[0-9]+]], s[[exec_lo]], 0
-; GCN: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_hi:[0-9]+]], s[[exec_hi]], v[[mbcnt_lo]]
-; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc, 0, v[[mbcnt_hi]]
-; GCN: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
+; GCN32: v_cmp_ne_u32_e64 s[[exec_lo:[0-9]+]], 1, 0
+; GCN64: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
+; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt:[0-9]+]], s[[exec_lo]], 0
+; GCN64: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt]], s[[exec_hi]], v[[mbcnt]]
+; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc{{(_lo)?}}, 0, v[[mbcnt]]
+; GCN32: s_bcnt1_i32_b32 s[[popcount:[0-9]+]], s[[exec_lo]]
+; GCN64: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
 ; GCN: v_mul_u32_u24{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[popcount]], 5
 ; GCN: buffer_atomic_sub v[[value]]
 define amdgpu_kernel void @sub_i32_constant(i32 addrspace(1)* %out, <4 x i32> %inout) {
@@ -99,11 +108,13 @@ entry:
 }
 
 ; GCN-LABEL: sub_i32_uniform:
-; GCN: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
-; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_lo:[0-9]+]], s[[exec_lo]], 0
-; GCN: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_hi:[0-9]+]], s[[exec_hi]], v[[mbcnt_lo]]
-; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc, 0, v[[mbcnt_hi]]
-; GCN: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
+; GCN32: v_cmp_ne_u32_e64 s[[exec_lo:[0-9]+]], 1, 0
+; GCN64: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
+; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt:[0-9]+]], s[[exec_lo]], 0
+; GCN64: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt]], s[[exec_hi]], v[[mbcnt]]
+; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc{{(_lo)?}}, 0, v[[mbcnt]]
+; GCN32: s_bcnt1_i32_b32 s[[popcount:[0-9]+]], s[[exec_lo]]
+; GCN64: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
 ; GCN: s_mul_i32 s[[scalar_value:[0-9]+]], s{{[0-9]+}}, s[[popcount]]
 ; GCN: v_mov_b32{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[scalar_value]]
 ; GCN: buffer_atomic_sub v[[value]]
@@ -119,9 +130,10 @@ entry:
 ; GFX7LESS-NOT: v_mbcnt_hi_u32_b32
 ; GFX7LESS-NOT: s_bcnt1_i32_b64
 ; GFX7LESS: buffer_atomic_sub v{{[0-9]+}}
-; GFX8MORE: v_sub{{(rev)?}}_u32_dpp
-; GFX8MORE: v_sub{{(rev)?}}_u32_dpp
-; GFX8MORE: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 63
+; DPPCOMB: v_add_u32_dpp
+; DPPCOMB: v_add_u32_dpp
+; GFX8MORE32: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 31
+; GFX8MORE64: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 63
 ; GFX8MORE: v_mov_b32{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[scalar_value]]
 ; GFX8MORE: buffer_atomic_sub v[[value]]
 define amdgpu_kernel void @sub_i32_varying_vdata(i32 addrspace(1)* %out, <4 x i32> %inout) {
diff --git a/llvm/test/CodeGen/AMDGPU/atomicrmw-nand.ll b/llvm/test/CodeGen/AMDGPU/atomicrmw-nand.ll
index 6d2e04d73..04eac1a9b 100644
--- a/llvm/test/CodeGen/AMDGPU/atomicrmw-nand.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomicrmw-nand.ll
@@ -5,11 +5,12 @@ define i32 @atomic_nand_i32_lds(i32 addrspace(3)* %ptr) nounwind {
 ; GCN-LABEL: atomic_nand_i32_lds:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT:    ds_read_b32 v2, v0
+; GCN-NEXT:    ds_read_b32 v1, v0
 ; GCN-NEXT:    s_mov_b64 s[4:5], 0
 ; GCN-NEXT:  BB0_1: ; %atomicrmw.start
 ; GCN-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v2, v1
 ; GCN-NEXT:    v_not_b32_e32 v1, v2
 ; GCN-NEXT:    v_or_b32_e32 v1, -5, v1
 ; GCN-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -17,7 +18,6 @@ define i32 @atomic_nand_i32_lds(i32 addrspace(3)* %ptr) nounwind {
 ; GCN-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN-NEXT:    buffer_wbinvl1_vol
 ; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
-; GCN-NEXT:    v_mov_b32_e32 v2, v1
 ; GCN-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
 ; GCN-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GCN-NEXT:    s_cbranch_execnz BB0_1
@@ -33,11 +33,12 @@ define i32 @atomic_nand_i32_global(i32 addrspace(1)* %ptr) nounwind {
 ; GCN-LABEL: atomic_nand_i32_global:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT:    global_load_dword v3, v[0:1], off
+; GCN-NEXT:    global_load_dword v2, v[0:1], off
 ; GCN-NEXT:    s_mov_b64 s[4:5], 0
 ; GCN-NEXT:  BB1_1: ; %atomicrmw.start
 ; GCN-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v3, v2
 ; GCN-NEXT:    v_not_b32_e32 v2, v3
 ; GCN-NEXT:    v_or_b32_e32 v2, -5, v2
 ; GCN-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -45,7 +46,6 @@ define i32 @atomic_nand_i32_global(i32 addrspace(1)* %ptr) nounwind {
 ; GCN-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN-NEXT:    buffer_wbinvl1_vol
 ; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
-; GCN-NEXT:    v_mov_b32_e32 v3, v2
 ; GCN-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
 ; GCN-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GCN-NEXT:    s_cbranch_execnz BB1_1
@@ -61,11 +61,12 @@ define i32 @atomic_nand_i32_flat(i32* %ptr) nounwind {
 ; GCN-LABEL: atomic_nand_i32_flat:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT:    flat_load_dword v3, v[0:1]
+; GCN-NEXT:    flat_load_dword v2, v[0:1]
 ; GCN-NEXT:    s_mov_b64 s[4:5], 0
 ; GCN-NEXT:  BB2_1: ; %atomicrmw.start
 ; GCN-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GCN-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v3, v2
 ; GCN-NEXT:    v_not_b32_e32 v2, v3
 ; GCN-NEXT:    v_or_b32_e32 v2, -5, v2
 ; GCN-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -74,7 +75,6 @@ define i32 @atomic_nand_i32_flat(i32* %ptr) nounwind {
 ; GCN-NEXT:    buffer_wbinvl1_vol
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
-; GCN-NEXT:    v_mov_b32_e32 v3, v2
 ; GCN-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
 ; GCN-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GCN-NEXT:    s_cbranch_execnz BB2_1
diff --git a/llvm/test/CodeGen/AMDGPU/attr-amdgpu-flat-work-group-size-v3.ll b/llvm/test/CodeGen/AMDGPU/attr-amdgpu-flat-work-group-size-v3.ll
index e57ce963e..d11919929 100644
--- a/llvm/test/CodeGen/AMDGPU/attr-amdgpu-flat-work-group-size-v3.ll
+++ b/llvm/test/CodeGen/AMDGPU/attr-amdgpu-flat-work-group-size-v3.ll
@@ -34,13 +34,13 @@ entry:
 }
 attributes #2 = {"amdgpu-flat-work-group-size"="128,128"}
 
-; CHECK-LABEL: {{^}}min_1024_max_2048
-; CHECK: SGPRBlocks: 1
-; CHECK: VGPRBlocks: 7
-; CHECK: NumSGPRsForWavesPerEU: 12
-; CHECK: NumVGPRsForWavesPerEU: 32
+; CHECK-LABEL: {{^}}min_1024_max_1024
+; CHECK: SGPRBlocks: 0
+; CHECK: VGPRBlocks: 10
+; CHECK: NumSGPRsForWavesPerEU: 2{{$}}
+; CHECK: NumVGPRsForWavesPerEU: 43
 @var = addrspace(1) global float 0.0
-define amdgpu_kernel void @min_1024_max_2048() #3 {
+define amdgpu_kernel void @min_1024_max_1024() #3 {
   %val0 = load volatile float, float addrspace(1)* @var
   %val1 = load volatile float, float addrspace(1)* @var
   %val2 = load volatile float, float addrspace(1)* @var
@@ -127,7 +127,7 @@ define amdgpu_kernel void @min_1024_max_2048() #3 {
 
   ret void
 }
-attributes #3 = {"amdgpu-flat-work-group-size"="1024,2048"}
+attributes #3 = {"amdgpu-flat-work-group-size"="1024,1024"}
 
 ; CHECK: amdhsa.kernels:
 ; CHECK:   .max_flat_workgroup_size: 64
@@ -136,8 +136,8 @@ attributes #3 = {"amdgpu-flat-work-group-size"="1024,2048"}
 ; CHECK:   .name:                 min_64_max_128
 ; CHECK:   .max_flat_workgroup_size: 128
 ; CHECK:   .name:                 min_128_max_128
-; CHECK:   .max_flat_workgroup_size: 2048
-; CHECK:   .name:                 min_1024_max_2048
+; CHECK:   .max_flat_workgroup_size: 1024
+; CHECK:   .name:                 min_1024_max_1024
 ; CHECK: amdhsa.version:
 ; CHECK:   - 1
 ; CHECK:   - 0
diff --git a/llvm/test/CodeGen/AMDGPU/attr-amdgpu-flat-work-group-size.ll b/llvm/test/CodeGen/AMDGPU/attr-amdgpu-flat-work-group-size.ll
index d51e2d6e9..f372fcb42 100644
--- a/llvm/test/CodeGen/AMDGPU/attr-amdgpu-flat-work-group-size.ll
+++ b/llvm/test/CodeGen/AMDGPU/attr-amdgpu-flat-work-group-size.ll
@@ -34,13 +34,13 @@ entry:
 }
 attributes #2 = {"amdgpu-flat-work-group-size"="128,128"}
 
-; CHECK-LABEL: {{^}}min_1024_max_2048
-; CHECK: SGPRBlocks: 1
-; CHECK: VGPRBlocks: 7
-; CHECK: NumSGPRsForWavesPerEU: 12
-; CHECK: NumVGPRsForWavesPerEU: 32
+; CHECK-LABEL: {{^}}min_1024_max_1024
+; CHECK: SGPRBlocks: 0
+; CHECK: VGPRBlocks: 10
+; CHECK: NumSGPRsForWavesPerEU: 2{{$}}
+; CHECK: NumVGPRsForWavesPerEU: 43
 @var = addrspace(1) global float 0.0
-define amdgpu_kernel void @min_1024_max_2048() #3 {
+define amdgpu_kernel void @min_1024_max_1024() #3 {
   %val0 = load volatile float, float addrspace(1)* @var
   %val1 = load volatile float, float addrspace(1)* @var
   %val2 = load volatile float, float addrspace(1)* @var
@@ -127,7 +127,7 @@ define amdgpu_kernel void @min_1024_max_2048() #3 {
 
   ret void
 }
-attributes #3 = {"amdgpu-flat-work-group-size"="1024,2048"}
+attributes #3 = {"amdgpu-flat-work-group-size"="1024,1024"}
 
 ; HSAMD: NT_AMD_AMDGPU_HSA_METADATA (HSA Metadata)
 ; HSAMD: Version: [ 1, 0 ]
@@ -138,5 +138,5 @@ attributes #3 = {"amdgpu-flat-work-group-size"="1024,2048"}
 ; HSAMD:   MaxFlatWorkGroupSize: 128
 ; HSAMD: - Name:                 min_128_max_128
 ; HSAMD:   MaxFlatWorkGroupSize: 128
-; HSAMD: - Name:                 min_1024_max_2048
-; HSAMD:   MaxFlatWorkGroupSize: 2048
+; HSAMD: - Name:                 min_1024_max_1024
+; HSAMD:   MaxFlatWorkGroupSize: 1024
diff --git a/llvm/test/CodeGen/AMDGPU/attr-amdgpu-num-sgpr.ll b/llvm/test/CodeGen/AMDGPU/attr-amdgpu-num-sgpr.ll
index 4e3b326d1..9dc74cb83 100644
--- a/llvm/test/CodeGen/AMDGPU/attr-amdgpu-num-sgpr.ll
+++ b/llvm/test/CodeGen/AMDGPU/attr-amdgpu-num-sgpr.ll
@@ -1,10 +1,6 @@
-; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=fiji -amdgpu-spill-sgpr-to-smem=0 -verify-machineinstrs < %s | FileCheck -check-prefix=TOSGPR -check-prefix=ALL %s
-
-; If spilling to smem, additional registers are used for the resource
-; descriptor.
+; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=TOSGPR -check-prefix=ALL %s
 
 ; FIXME: Vectorization can increase required SGPR count beyond limit.
-; FIXME: SGPR-to-SMEM requires an additional SGPR always to scavenge m0
 
 ; ALL-LABEL: {{^}}max_9_sgprs:
 
@@ -55,13 +51,6 @@ define amdgpu_kernel void @max_9_sgprs() #0 {
 ; XTOSGPR: SGPRBlocks: 1
 ; XTOSGPR: NumSGPRsForWavesPerEU: 16
 
-; XTOSMEM: s_mov_b64 s[10:11], s[2:3]
-; XTOSMEM: s_mov_b64 s[8:9], s[0:1]
-; XTOSMEM: s_mov_b32 s7, s13
-
-; XTOSMEM: SGPRBlocks: 1
-; XTOSMEM: NumSGPRsForWavesPerEU: 16
-;
 ; This test case is disabled: When calculating the spillslot addresses AMDGPU
 ; creates an extra vreg to save/restore m0 which in a point of maximum register
 ; pressure would trigger an endless loop; the compiler aborts earlier with
@@ -101,10 +90,6 @@ define amdgpu_kernel void @max_9_sgprs() #0 {
 ; ; swapping the order the registers are copied from what normally
 ; ; happens.
 
-; XTOSMEM: s_mov_b32 s5, s11
-; XTOSMEM: s_add_u32 m0, s5,
-; XTOSMEM: s_buffer_store_dword vcc_lo, s[0:3], m0
-
 ; XALL: SGPRBlocks: 2
 ; XALL: NumSGPRsForWavesPerEU: 18
 ;define amdgpu_kernel void @max_12_sgprs_12_input_sgprs(i32 addrspace(1)* %out1,
diff --git a/llvm/test/CodeGen/AMDGPU/basic-branch.ll b/llvm/test/CodeGen/AMDGPU/basic-branch.ll
index 99a64ba03..14f207020 100644
--- a/llvm/test/CodeGen/AMDGPU/basic-branch.ll
+++ b/llvm/test/CodeGen/AMDGPU/basic-branch.ll
@@ -1,6 +1,6 @@
 ; RUN: llc -O0 -march=amdgcn -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCNNOOPT -check-prefix=GCN %s
-; RUN: llc -O0 -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -amdgpu-spill-sgpr-to-smem=0 -verify-machineinstrs < %s | FileCheck -enable-var-scope  -check-prefix=GCNNOOPT -check-prefix=GCN %s
-; RUN: llc -O0 -march=amdgcn -mcpu=gfx1010 -mattr=-flat-for-global,-WavefrontSize32,+WavefrontSize64 -amdgpu-spill-sgpr-to-smem=0 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCNNOOPT -check-prefix=GCN %s
+; RUN: llc -O0 -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope  -check-prefix=GCNNOOPT -check-prefix=GCN %s
+; RUN: llc -O0 -march=amdgcn -mcpu=gfx1010 -mattr=-flat-for-global,-WavefrontSize32,+WavefrontSize64 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCNNOOPT -check-prefix=GCN %s
 ; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCNOPT -check-prefix=GCN %s
 ; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCNOPT -check-prefix=GCN %s
 
diff --git a/llvm/test/CodeGen/AMDGPU/bitcast-constant-to-vector.ll b/llvm/test/CodeGen/AMDGPU/bitcast-constant-to-vector.ll
index ea5f01fbd..763829094 100644
--- a/llvm/test/CodeGen/AMDGPU/bitcast-constant-to-vector.ll
+++ b/llvm/test/CodeGen/AMDGPU/bitcast-constant-to-vector.ll
@@ -1,9 +1,9 @@
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
 
 ; GCN-LABEL: {{^}}cast_constant_i64_to_build_vector_v4i16:
-; GCN: global_store_dwordx2
-; GCN: global_store_dword v
 ; GCN: global_store_short
+; GCN: global_store_dword v
+; GCN: global_store_dwordx2
 define amdgpu_kernel void @cast_constant_i64_to_build_vector_v4i16(i8 addrspace(1)* nocapture %data) {
 entry:
   store i8 72, i8 addrspace(1)* %data, align 1
diff --git a/llvm/test/CodeGen/AMDGPU/bitreverse.ll b/llvm/test/CodeGen/AMDGPU/bitreverse.ll
index 9c537d5bc..40bb9be03 100644
--- a/llvm/test/CodeGen/AMDGPU/bitreverse.ll
+++ b/llvm/test/CodeGen/AMDGPU/bitreverse.ll
@@ -391,14 +391,13 @@ define amdgpu_kernel void @v_brev_i64(i64 addrspace(1)* noalias %out, i64 addrsp
 ; SI-NEXT:    v_mov_b32_e32 v1, 0
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    buffer_load_dwordx2 v[0:1], v[0:1], s[0:3], 0 addr64
-; SI-NEXT:    s_mov_b32 s0, 0xff0000
-; SI-NEXT:    s_mov_b32 s1, 0xff00
-; SI-NEXT:    s_mov_b32 s2, 0xf0f0f0f
-; SI-NEXT:    s_mov_b32 s3, 0xf0f0f0f0
-; SI-NEXT:    s_mov_b32 s6, 0x33333333
-; SI-NEXT:    s_mov_b32 s8, 0xcccccccc
-; SI-NEXT:    s_mov_b32 s9, 0x55555555
-; SI-NEXT:    s_mov_b32 s10, 0xaaaaaaaa
+; SI-NEXT:    s_mov_b32 s0, 0xff00
+; SI-NEXT:    s_mov_b32 s1, 0xf0f0f0f
+; SI-NEXT:    s_mov_b32 s2, 0xf0f0f0f0
+; SI-NEXT:    s_mov_b32 s3, 0x33333333
+; SI-NEXT:    s_mov_b32 s6, 0xcccccccc
+; SI-NEXT:    s_mov_b32 s8, 0x55555555
+; SI-NEXT:    s_mov_b32 s9, 0xaaaaaaaa
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_lshl_b64 v[2:3], v[0:1], 8
 ; SI-NEXT:    v_alignbit_b32 v4, v1, v0, 24
@@ -408,39 +407,39 @@ define amdgpu_kernel void @v_brev_i64(i64 addrspace(1)* noalias %out, i64 addrsp
 ; SI-NEXT:    v_lshl_b64 v[1:2], v[0:1], 24
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 24, v0
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
-; SI-NEXT:    v_and_b32_e32 v0, s0, v0
-; SI-NEXT:    v_and_b32_e32 v4, s0, v4
+; SI-NEXT:    v_and_b32_e32 v0, 0xff0000, v0
+; SI-NEXT:    v_and_b32_e32 v4, 0xff0000, v4
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff000000, v5
-; SI-NEXT:    v_and_b32_e32 v7, s1, v7
+; SI-NEXT:    v_and_b32_e32 v7, s0, v7
 ; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
-; SI-NEXT:    v_and_b32_e32 v2, s1, v2
+; SI-NEXT:    v_and_b32_e32 v2, s0, v2
 ; SI-NEXT:    v_or_b32_e32 v4, v5, v4
 ; SI-NEXT:    v_or_b32_e32 v5, v7, v6
 ; SI-NEXT:    v_or_b32_e32 v0, v1, v0
 ; SI-NEXT:    v_or_b32_e32 v2, v2, v3
 ; SI-NEXT:    v_or_b32_e32 v1, v4, v5
 ; SI-NEXT:    v_or_b32_e32 v3, v0, v2
-; SI-NEXT:    v_and_b32_e32 v0, s2, v1
-; SI-NEXT:    v_and_b32_e32 v2, s3, v1
-; SI-NEXT:    v_and_b32_e32 v1, s2, v3
-; SI-NEXT:    v_and_b32_e32 v3, s3, v3
+; SI-NEXT:    v_and_b32_e32 v0, s1, v1
+; SI-NEXT:    v_and_b32_e32 v2, s2, v1
+; SI-NEXT:    v_and_b32_e32 v1, s1, v3
+; SI-NEXT:    v_and_b32_e32 v3, s2, v3
 ; SI-NEXT:    v_lshl_b64 v[0:1], v[0:1], 4
 ; SI-NEXT:    v_lshr_b64 v[2:3], v[2:3], 4
 ; SI-NEXT:    v_or_b32_e32 v3, v3, v1
 ; SI-NEXT:    v_or_b32_e32 v2, v2, v0
-; SI-NEXT:    v_and_b32_e32 v1, s6, v3
-; SI-NEXT:    v_and_b32_e32 v0, s6, v2
-; SI-NEXT:    v_and_b32_e32 v3, s8, v3
-; SI-NEXT:    v_and_b32_e32 v2, s8, v2
+; SI-NEXT:    v_and_b32_e32 v1, s3, v3
+; SI-NEXT:    v_and_b32_e32 v0, s3, v2
+; SI-NEXT:    v_and_b32_e32 v3, s6, v3
+; SI-NEXT:    v_and_b32_e32 v2, s6, v2
 ; SI-NEXT:    v_lshl_b64 v[0:1], v[0:1], 2
 ; SI-NEXT:    v_lshr_b64 v[2:3], v[2:3], 2
 ; SI-NEXT:    s_mov_b32 s6, -1
 ; SI-NEXT:    v_or_b32_e32 v3, v3, v1
 ; SI-NEXT:    v_or_b32_e32 v2, v2, v0
-; SI-NEXT:    v_and_b32_e32 v1, s9, v3
-; SI-NEXT:    v_and_b32_e32 v0, s9, v2
-; SI-NEXT:    v_and_b32_e32 v3, s10, v3
-; SI-NEXT:    v_and_b32_e32 v2, s10, v2
+; SI-NEXT:    v_and_b32_e32 v1, s8, v3
+; SI-NEXT:    v_and_b32_e32 v0, s8, v2
+; SI-NEXT:    v_and_b32_e32 v3, s9, v3
+; SI-NEXT:    v_and_b32_e32 v2, s9, v2
 ; SI-NEXT:    v_lshl_b64 v[0:1], v[0:1], 1
 ; SI-NEXT:    v_lshr_b64 v[2:3], v[2:3], 1
 ; SI-NEXT:    v_or_b32_e32 v1, v3, v1
@@ -454,18 +453,17 @@ define amdgpu_kernel void @v_brev_i64(i64 addrspace(1)* noalias %out, i64 addrsp
 ; FLAT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2c
 ; FLAT-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
 ; FLAT-NEXT:    v_mov_b32_e32 v4, 8
-; FLAT-NEXT:    s_mov_b32 s2, 0xff0000
-; FLAT-NEXT:    s_mov_b32 s3, 0xf0f0f0f
+; FLAT-NEXT:    s_mov_b32 s2, 0xf0f0f0f
+; FLAT-NEXT:    s_mov_b32 s3, 0xf0f0f0f0
 ; FLAT-NEXT:    s_waitcnt lgkmcnt(0)
 ; FLAT-NEXT:    v_mov_b32_e32 v1, s1
 ; FLAT-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
 ; FLAT-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; FLAT-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
-; FLAT-NEXT:    s_mov_b32 s0, 0xf0f0f0f0
-; FLAT-NEXT:    s_mov_b32 s1, 0x33333333
-; FLAT-NEXT:    s_mov_b32 s6, 0xcccccccc
-; FLAT-NEXT:    s_mov_b32 s8, 0x55555555
-; FLAT-NEXT:    s_mov_b32 s9, 0xaaaaaaaa
+; FLAT-NEXT:    s_mov_b32 s0, 0x33333333
+; FLAT-NEXT:    s_mov_b32 s1, 0xcccccccc
+; FLAT-NEXT:    s_mov_b32 s6, 0x55555555
+; FLAT-NEXT:    s_mov_b32 s8, 0xaaaaaaaa
 ; FLAT-NEXT:    s_mov_b32 s7, 0xf000
 ; FLAT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; FLAT-NEXT:    v_lshlrev_b64 v[2:3], 24, v[0:1]
@@ -475,9 +473,9 @@ define amdgpu_kernel void @v_brev_i64(i64 addrspace(1)* noalias %out, i64 addrsp
 ; FLAT-NEXT:    v_lshlrev_b64 v[4:5], 8, v[0:1]
 ; FLAT-NEXT:    v_lshlrev_b32_e32 v4, 24, v0
 ; FLAT-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
-; FLAT-NEXT:    v_and_b32_e32 v2, s2, v2
+; FLAT-NEXT:    v_and_b32_e32 v2, 0xff0000, v2
 ; FLAT-NEXT:    v_and_b32_e32 v6, 0xff000000, v6
-; FLAT-NEXT:    v_and_b32_e32 v0, s2, v0
+; FLAT-NEXT:    v_and_b32_e32 v0, 0xff0000, v0
 ; FLAT-NEXT:    v_or_b32_sdwa v1, v7, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
 ; FLAT-NEXT:    v_or_b32_e32 v2, v6, v2
 ; FLAT-NEXT:    v_and_b32_e32 v3, 0xff00, v3
@@ -485,29 +483,29 @@ define amdgpu_kernel void @v_brev_i64(i64 addrspace(1)* noalias %out, i64 addrsp
 ; FLAT-NEXT:    v_or_b32_e32 v0, v4, v0
 ; FLAT-NEXT:    v_or_b32_sdwa v2, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; FLAT-NEXT:    v_or_b32_e32 v3, v0, v2
-; FLAT-NEXT:    v_and_b32_e32 v0, s3, v1
-; FLAT-NEXT:    v_and_b32_e32 v2, s0, v1
-; FLAT-NEXT:    v_and_b32_e32 v1, s3, v3
-; FLAT-NEXT:    v_and_b32_e32 v3, s0, v3
+; FLAT-NEXT:    v_and_b32_e32 v0, s2, v1
+; FLAT-NEXT:    v_and_b32_e32 v2, s3, v1
+; FLAT-NEXT:    v_and_b32_e32 v1, s2, v3
+; FLAT-NEXT:    v_and_b32_e32 v3, s3, v3
 ; FLAT-NEXT:    v_lshlrev_b64 v[0:1], 4, v[0:1]
 ; FLAT-NEXT:    v_lshrrev_b64 v[2:3], 4, v[2:3]
 ; FLAT-NEXT:    v_or_b32_e32 v3, v3, v1
 ; FLAT-NEXT:    v_or_b32_e32 v2, v2, v0
-; FLAT-NEXT:    v_and_b32_e32 v1, s1, v3
-; FLAT-NEXT:    v_and_b32_e32 v0, s1, v2
-; FLAT-NEXT:    v_and_b32_e32 v3, s6, v3
-; FLAT-NEXT:    v_and_b32_e32 v2, s6, v2
+; FLAT-NEXT:    v_and_b32_e32 v1, s0, v3
+; FLAT-NEXT:    v_and_b32_e32 v0, s0, v2
+; FLAT-NEXT:    v_and_b32_e32 v3, s1, v3
+; FLAT-NEXT:    v_and_b32_e32 v2, s1, v2
 ; FLAT-NEXT:    v_lshlrev_b64 v[0:1], 2, v[0:1]
 ; FLAT-NEXT:    v_lshrrev_b64 v[2:3], 2, v[2:3]
-; FLAT-NEXT:    s_mov_b32 s6, -1
 ; FLAT-NEXT:    v_or_b32_e32 v3, v3, v1
 ; FLAT-NEXT:    v_or_b32_e32 v2, v2, v0
-; FLAT-NEXT:    v_and_b32_e32 v1, s8, v3
-; FLAT-NEXT:    v_and_b32_e32 v0, s8, v2
-; FLAT-NEXT:    v_and_b32_e32 v3, s9, v3
-; FLAT-NEXT:    v_and_b32_e32 v2, s9, v2
+; FLAT-NEXT:    v_and_b32_e32 v1, s6, v3
+; FLAT-NEXT:    v_and_b32_e32 v0, s6, v2
+; FLAT-NEXT:    v_and_b32_e32 v3, s8, v3
+; FLAT-NEXT:    v_and_b32_e32 v2, s8, v2
 ; FLAT-NEXT:    v_lshlrev_b64 v[0:1], 1, v[0:1]
 ; FLAT-NEXT:    v_lshrrev_b64 v[2:3], 1, v[2:3]
+; FLAT-NEXT:    s_mov_b32 s6, -1
 ; FLAT-NEXT:    v_or_b32_e32 v1, v3, v1
 ; FLAT-NEXT:    v_or_b32_e32 v0, v2, v0
 ; FLAT-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
@@ -789,23 +787,20 @@ define amdgpu_kernel void @v_brev_v2i64(<2 x i64> addrspace(1)* noalias %out, <2
 ; SI-NEXT:    v_mov_b32_e32 v1, 0
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    buffer_load_dwordx4 v[0:3], v[0:1], s[0:3], 0 addr64
-; SI-NEXT:    s_mov_b32 s0, 0xff0000
-; SI-NEXT:    s_mov_b32 s1, 0xff000000
-; SI-NEXT:    s_mov_b32 s2, 0xff00
-; SI-NEXT:    s_movk_i32 s3, 0xff
-; SI-NEXT:    s_mov_b32 s8, 0xf0f0f0f
-; SI-NEXT:    s_mov_b32 s9, 0xf0f0f0f0
-; SI-NEXT:    s_mov_b32 s10, 0x33333333
-; SI-NEXT:    s_mov_b32 s11, 0xcccccccc
-; SI-NEXT:    s_mov_b32 s12, 0x55555555
-; SI-NEXT:    s_mov_b32 s13, 0xaaaaaaaa
+; SI-NEXT:    s_mov_b32 s0, 0xff00
+; SI-NEXT:    s_mov_b32 s1, 0xf0f0f0f
+; SI-NEXT:    s_mov_b32 s2, 0xf0f0f0f0
+; SI-NEXT:    s_mov_b32 s3, 0x33333333
+; SI-NEXT:    s_mov_b32 s8, 0xcccccccc
+; SI-NEXT:    s_mov_b32 s9, 0x55555555
+; SI-NEXT:    s_mov_b32 s10, 0xaaaaaaaa
 ; SI-NEXT:    s_mov_b32 s6, -1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_lshl_b64 v[4:5], v[2:3], 8
 ; SI-NEXT:    v_alignbit_b32 v6, v3, v2, 24
 ; SI-NEXT:    v_alignbit_b32 v7, v3, v2, 8
-; SI-NEXT:    v_lshrrev_b32_e32 v8, 24, v3
 ; SI-NEXT:    v_lshrrev_b32_e32 v9, 8, v3
+; SI-NEXT:    v_lshrrev_b32_e32 v8, 24, v3
 ; SI-NEXT:    v_lshl_b64 v[3:4], v[2:3], 24
 ; SI-NEXT:    v_lshlrev_b32_e32 v10, 24, v2
 ; SI-NEXT:    v_lshlrev_b32_e32 v11, 8, v2
@@ -817,78 +812,80 @@ define amdgpu_kernel void @v_brev_v2i64(<2 x i64> addrspace(1)* noalias %out, <2
 ; SI-NEXT:    v_lshlrev_b32_e32 v16, 24, v0
 ; SI-NEXT:    v_lshlrev_b32_e32 v17, 8, v0
 ; SI-NEXT:    v_lshl_b64 v[0:1], v[0:1], 24
-; SI-NEXT:    v_and_b32_e32 v0, s0, v6
-; SI-NEXT:    v_and_b32_e32 v2, s1, v7
-; SI-NEXT:    v_and_b32_e32 v6, s2, v9
-; SI-NEXT:    v_and_b32_e32 v7, s0, v11
-; SI-NEXT:    v_and_b32_e32 v9, s0, v12
-; SI-NEXT:    v_and_b32_e32 v11, s1, v13
-; SI-NEXT:    v_or_b32_e32 v0, v2, v0
-; SI-NEXT:    v_or_b32_e32 v2, v6, v8
-; SI-NEXT:    v_and_b32_e32 v12, s2, v15
-; SI-NEXT:    v_and_b32_e32 v13, s0, v17
-; SI-NEXT:    v_and_b32_e32 v5, s3, v5
-; SI-NEXT:    v_and_b32_e32 v4, s2, v4
-; SI-NEXT:    v_and_b32_e32 v3, s3, v3
-; SI-NEXT:    v_and_b32_e32 v1, s2, v1
-; SI-NEXT:    v_or_b32_e32 v6, v10, v7
-; SI-NEXT:    v_or_b32_e32 v7, v11, v9
-; SI-NEXT:    v_or_b32_e32 v2, v0, v2
-; SI-NEXT:    v_or_b32_e32 v8, v12, v14
-; SI-NEXT:    v_or_b32_e32 v0, v4, v5
+; SI-NEXT:    v_and_b32_e32 v6, 0xff0000, v6
+; SI-NEXT:    v_and_b32_e32 v7, 0xff000000, v7
+; SI-NEXT:    v_mov_b32_e32 v0, 0xff0000
+; SI-NEXT:    v_or_b32_e32 v6, v7, v6
+; SI-NEXT:    v_mov_b32_e32 v7, 0xff00
+; SI-NEXT:    v_and_b32_e32 v2, v0, v11
+; SI-NEXT:    v_and_b32_e32 v11, v0, v12
+; SI-NEXT:    v_and_b32_e32 v9, s0, v9
+; SI-NEXT:    v_and_b32_e32 v12, 0xff000000, v13
+; SI-NEXT:    v_and_b32_e32 v0, v0, v17
+; SI-NEXT:    v_and_b32_e32 v13, v7, v15
+; SI-NEXT:    v_and_b32_e32 v1, v7, v1
+; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
+; SI-NEXT:    v_or_b32_e32 v8, v9, v8
+; SI-NEXT:    v_or_b32_e32 v2, v10, v2
+; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
+; SI-NEXT:    v_and_b32_e32 v4, s0, v4
+; SI-NEXT:    v_or_b32_e32 v7, v16, v0
 ; SI-NEXT:    v_or_b32_e32 v1, v1, v3
-; SI-NEXT:    v_or_b32_e32 v9, v16, v13
-; SI-NEXT:    v_or_b32_e32 v5, v7, v8
-; SI-NEXT:    v_or_b32_e32 v3, v6, v0
-; SI-NEXT:    v_or_b32_e32 v7, v9, v1
-; SI-NEXT:    v_and_b32_e32 v0, s8, v2
-; SI-NEXT:    v_and_b32_e32 v1, s8, v3
-; SI-NEXT:    v_and_b32_e32 v2, s9, v2
-; SI-NEXT:    v_and_b32_e32 v3, s9, v3
-; SI-NEXT:    v_and_b32_e32 v4, s8, v5
-; SI-NEXT:    v_and_b32_e32 v6, s9, v5
-; SI-NEXT:    v_and_b32_e32 v5, s8, v7
-; SI-NEXT:    v_and_b32_e32 v7, s9, v7
-; SI-NEXT:    v_lshl_b64 v[0:1], v[0:1], 4
-; SI-NEXT:    v_lshr_b64 v[2:3], v[2:3], 4
+; SI-NEXT:    v_or_b32_e32 v9, v12, v11
+; SI-NEXT:    v_or_b32_e32 v10, v13, v14
+; SI-NEXT:    v_or_b32_e32 v0, v4, v5
+; SI-NEXT:    v_or_b32_e32 v5, v9, v10
+; SI-NEXT:    v_or_b32_e32 v6, v6, v8
+; SI-NEXT:    v_or_b32_e32 v7, v7, v1
+; SI-NEXT:    v_or_b32_e32 v3, v2, v0
+; SI-NEXT:    v_and_b32_e32 v0, s1, v6
+; SI-NEXT:    v_and_b32_e32 v2, s2, v6
+; SI-NEXT:    v_and_b32_e32 v4, s1, v5
+; SI-NEXT:    v_and_b32_e32 v6, s2, v5
+; SI-NEXT:    v_and_b32_e32 v5, s1, v7
+; SI-NEXT:    v_and_b32_e32 v7, s2, v7
+; SI-NEXT:    v_and_b32_e32 v1, s1, v3
+; SI-NEXT:    v_and_b32_e32 v3, s2, v3
 ; SI-NEXT:    v_lshl_b64 v[4:5], v[4:5], 4
 ; SI-NEXT:    v_lshr_b64 v[6:7], v[6:7], 4
-; SI-NEXT:    v_or_b32_e32 v3, v3, v1
-; SI-NEXT:    v_or_b32_e32 v2, v2, v0
+; SI-NEXT:    v_lshl_b64 v[0:1], v[0:1], 4
+; SI-NEXT:    v_lshr_b64 v[2:3], v[2:3], 4
 ; SI-NEXT:    v_or_b32_e32 v7, v7, v5
 ; SI-NEXT:    v_or_b32_e32 v6, v6, v4
-; SI-NEXT:    v_and_b32_e32 v1, s10, v3
-; SI-NEXT:    v_and_b32_e32 v0, s10, v2
-; SI-NEXT:    v_and_b32_e32 v5, s10, v7
-; SI-NEXT:    v_and_b32_e32 v4, s10, v6
-; SI-NEXT:    v_and_b32_e32 v3, s11, v3
-; SI-NEXT:    v_and_b32_e32 v2, s11, v2
-; SI-NEXT:    v_and_b32_e32 v7, s11, v7
-; SI-NEXT:    v_and_b32_e32 v6, s11, v6
-; SI-NEXT:    v_lshl_b64 v[0:1], v[0:1], 2
-; SI-NEXT:    v_lshr_b64 v[2:3], v[2:3], 2
+; SI-NEXT:    v_or_b32_e32 v3, v3, v1
+; SI-NEXT:    v_or_b32_e32 v2, v2, v0
+; SI-NEXT:    v_and_b32_e32 v5, s3, v7
+; SI-NEXT:    v_and_b32_e32 v4, s3, v6
+; SI-NEXT:    v_and_b32_e32 v7, s8, v7
+; SI-NEXT:    v_and_b32_e32 v6, s8, v6
+; SI-NEXT:    v_and_b32_e32 v1, s3, v3
+; SI-NEXT:    v_and_b32_e32 v0, s3, v2
+; SI-NEXT:    v_and_b32_e32 v3, s8, v3
+; SI-NEXT:    v_and_b32_e32 v2, s8, v2
 ; SI-NEXT:    v_lshl_b64 v[4:5], v[4:5], 2
 ; SI-NEXT:    v_lshr_b64 v[6:7], v[6:7], 2
+; SI-NEXT:    v_lshl_b64 v[0:1], v[0:1], 2
+; SI-NEXT:    v_lshr_b64 v[2:3], v[2:3], 2
+; SI-NEXT:    v_or_b32_e32 v7, v7, v5
+; SI-NEXT:    v_or_b32_e32 v9, v6, v4
 ; SI-NEXT:    v_or_b32_e32 v3, v3, v1
 ; SI-NEXT:    v_or_b32_e32 v2, v2, v0
-; SI-NEXT:    v_or_b32_e32 v7, v7, v5
-; SI-NEXT:    v_or_b32_e32 v6, v6, v4
-; SI-NEXT:    v_and_b32_e32 v1, s12, v3
-; SI-NEXT:    v_and_b32_e32 v0, s12, v2
-; SI-NEXT:    v_and_b32_e32 v5, s12, v7
-; SI-NEXT:    v_and_b32_e32 v4, s12, v6
-; SI-NEXT:    v_and_b32_e32 v3, s13, v3
-; SI-NEXT:    v_and_b32_e32 v2, s13, v2
-; SI-NEXT:    v_and_b32_e32 v7, s13, v7
-; SI-NEXT:    v_and_b32_e32 v6, s13, v6
+; SI-NEXT:    v_and_b32_e32 v5, s9, v7
+; SI-NEXT:    v_and_b32_e32 v4, s9, v9
+; SI-NEXT:    v_and_b32_e32 v1, s9, v3
+; SI-NEXT:    v_and_b32_e32 v0, s9, v2
+; SI-NEXT:    v_and_b32_e32 v6, s10, v7
+; SI-NEXT:    v_lshl_b64 v[7:8], v[4:5], 1
+; SI-NEXT:    v_and_b32_e32 v5, s10, v9
+; SI-NEXT:    v_and_b32_e32 v3, s10, v3
+; SI-NEXT:    v_and_b32_e32 v2, s10, v2
 ; SI-NEXT:    v_lshl_b64 v[0:1], v[0:1], 1
 ; SI-NEXT:    v_lshr_b64 v[2:3], v[2:3], 1
-; SI-NEXT:    v_lshl_b64 v[4:5], v[4:5], 1
-; SI-NEXT:    v_lshr_b64 v[6:7], v[6:7], 1
+; SI-NEXT:    v_lshr_b64 v[4:5], v[5:6], 1
 ; SI-NEXT:    v_or_b32_e32 v3, v3, v1
 ; SI-NEXT:    v_or_b32_e32 v2, v2, v0
-; SI-NEXT:    v_or_b32_e32 v1, v7, v5
-; SI-NEXT:    v_or_b32_e32 v0, v6, v4
+; SI-NEXT:    v_or_b32_e32 v1, v5, v8
+; SI-NEXT:    v_or_b32_e32 v0, v4, v7
 ; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
 ; SI-NEXT:    s_endpgm
 ;
@@ -898,65 +895,63 @@ define amdgpu_kernel void @v_brev_v2i64(<2 x i64> addrspace(1)* noalias %out, <2
 ; FLAT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2c
 ; FLAT-NEXT:    v_lshlrev_b32_e32 v0, 4, v0
 ; FLAT-NEXT:    v_mov_b32_e32 v8, 8
-; FLAT-NEXT:    s_mov_b32 s2, 0xff0000
-; FLAT-NEXT:    s_mov_b32 s3, 0xff000000
+; FLAT-NEXT:    v_mov_b32_e32 v10, 0xff0000
+; FLAT-NEXT:    s_mov_b32 s2, 0xf0f0f0f
 ; FLAT-NEXT:    s_waitcnt lgkmcnt(0)
 ; FLAT-NEXT:    v_mov_b32_e32 v1, s1
 ; FLAT-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
 ; FLAT-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; FLAT-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
-; FLAT-NEXT:    s_mov_b32 s0, 0xff00
-; FLAT-NEXT:    s_mov_b32 s1, 0xf0f0f0f
-; FLAT-NEXT:    s_mov_b32 s8, 0xf0f0f0f0
-; FLAT-NEXT:    s_mov_b32 s9, 0x33333333
-; FLAT-NEXT:    s_mov_b32 s10, 0xcccccccc
-; FLAT-NEXT:    s_mov_b32 s11, 0x55555555
-; FLAT-NEXT:    s_mov_b32 s12, 0xaaaaaaaa
+; FLAT-NEXT:    s_mov_b32 s0, 0xf0f0f0f0
+; FLAT-NEXT:    s_mov_b32 s1, 0x33333333
+; FLAT-NEXT:    s_mov_b32 s3, 0xcccccccc
+; FLAT-NEXT:    s_mov_b32 s8, 0x55555555
+; FLAT-NEXT:    s_mov_b32 s9, 0xaaaaaaaa
 ; FLAT-NEXT:    s_mov_b32 s7, 0xf000
 ; FLAT-NEXT:    s_mov_b32 s6, -1
 ; FLAT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; FLAT-NEXT:    v_lshlrev_b64 v[4:5], 24, v[2:3]
-; FLAT-NEXT:    v_lshlrev_b32_sdwa v11, v8, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
-; FLAT-NEXT:    v_lshlrev_b32_sdwa v14, v8, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
+; FLAT-NEXT:    v_lshlrev_b32_sdwa v12, v8, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
+; FLAT-NEXT:    v_lshlrev_b32_sdwa v15, v8, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
 ; FLAT-NEXT:    v_lshlrev_b64 v[8:9], 8, v[0:1]
 ; FLAT-NEXT:    v_lshlrev_b64 v[6:7], 8, v[2:3]
 ; FLAT-NEXT:    v_alignbit_b32 v4, v3, v2, 24
-; FLAT-NEXT:    v_alignbit_b32 v10, v3, v2, 8
-; FLAT-NEXT:    v_or_b32_sdwa v3, v11, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
-; FLAT-NEXT:    v_alignbit_b32 v12, v1, v0, 24
-; FLAT-NEXT:    v_alignbit_b32 v13, v1, v0, 8
+; FLAT-NEXT:    v_alignbit_b32 v11, v3, v2, 8
+; FLAT-NEXT:    v_or_b32_sdwa v3, v12, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+; FLAT-NEXT:    v_or_b32_sdwa v12, v15, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+; FLAT-NEXT:    v_alignbit_b32 v13, v1, v0, 24
+; FLAT-NEXT:    v_alignbit_b32 v14, v1, v0, 8
 ; FLAT-NEXT:    v_lshlrev_b32_e32 v8, 24, v0
 ; FLAT-NEXT:    v_lshlrev_b32_e32 v15, 8, v0
-; FLAT-NEXT:    v_or_b32_sdwa v11, v14, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
 ; FLAT-NEXT:    v_lshlrev_b64 v[0:1], 24, v[0:1]
 ; FLAT-NEXT:    v_lshlrev_b32_e32 v6, 24, v2
 ; FLAT-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
-; FLAT-NEXT:    v_and_b32_e32 v0, s2, v4
-; FLAT-NEXT:    v_and_b32_e32 v4, s3, v10
-; FLAT-NEXT:    v_and_b32_e32 v2, s2, v2
+; FLAT-NEXT:    v_and_b32_e32 v0, 0xff0000, v4
+; FLAT-NEXT:    v_and_b32_e32 v4, 0xff000000, v11
+; FLAT-NEXT:    v_and_b32_e32 v2, v10, v2
+; FLAT-NEXT:    v_and_b32_e32 v11, v10, v13
 ; FLAT-NEXT:    v_or_b32_e32 v0, v4, v0
-; FLAT-NEXT:    v_and_b32_e32 v1, s0, v1
-; FLAT-NEXT:    v_and_b32_e32 v10, s2, v12
-; FLAT-NEXT:    v_and_b32_e32 v12, s3, v13
-; FLAT-NEXT:    v_and_b32_e32 v4, s0, v5
-; FLAT-NEXT:    v_and_b32_e32 v13, s2, v15
-; FLAT-NEXT:    v_or_b32_e32 v5, v12, v10
+; FLAT-NEXT:    v_and_b32_e32 v1, 0xff00, v1
+; FLAT-NEXT:    v_and_b32_e32 v13, 0xff000000, v14
+; FLAT-NEXT:    v_and_b32_e32 v4, 0xff00, v5
+; FLAT-NEXT:    v_and_b32_e32 v10, v10, v15
+; FLAT-NEXT:    v_or_b32_e32 v5, v13, v11
 ; FLAT-NEXT:    v_or_b32_e32 v2, v6, v2
 ; FLAT-NEXT:    v_or_b32_e32 v3, v0, v3
 ; FLAT-NEXT:    v_or_b32_sdwa v0, v4, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; FLAT-NEXT:    v_or_b32_e32 v6, v8, v13
+; FLAT-NEXT:    v_or_b32_e32 v6, v8, v10
 ; FLAT-NEXT:    v_or_b32_sdwa v1, v1, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; FLAT-NEXT:    v_or_b32_e32 v7, v2, v0
-; FLAT-NEXT:    v_or_b32_e32 v5, v5, v11
+; FLAT-NEXT:    v_or_b32_e32 v5, v5, v12
 ; FLAT-NEXT:    v_or_b32_e32 v8, v6, v1
-; FLAT-NEXT:    v_and_b32_e32 v0, s1, v3
-; FLAT-NEXT:    v_and_b32_e32 v1, s1, v7
-; FLAT-NEXT:    v_and_b32_e32 v2, s8, v3
-; FLAT-NEXT:    v_and_b32_e32 v3, s8, v7
-; FLAT-NEXT:    v_and_b32_e32 v4, s1, v5
-; FLAT-NEXT:    v_and_b32_e32 v6, s8, v5
-; FLAT-NEXT:    v_and_b32_e32 v5, s1, v8
-; FLAT-NEXT:    v_and_b32_e32 v7, s8, v8
+; FLAT-NEXT:    v_and_b32_e32 v0, s2, v3
+; FLAT-NEXT:    v_and_b32_e32 v1, s2, v7
+; FLAT-NEXT:    v_and_b32_e32 v2, s0, v3
+; FLAT-NEXT:    v_and_b32_e32 v3, s0, v7
+; FLAT-NEXT:    v_and_b32_e32 v4, s2, v5
+; FLAT-NEXT:    v_and_b32_e32 v6, s0, v5
+; FLAT-NEXT:    v_and_b32_e32 v5, s2, v8
+; FLAT-NEXT:    v_and_b32_e32 v7, s0, v8
 ; FLAT-NEXT:    v_lshlrev_b64 v[0:1], 4, v[0:1]
 ; FLAT-NEXT:    v_lshrrev_b64 v[2:3], 4, v[2:3]
 ; FLAT-NEXT:    v_lshlrev_b64 v[4:5], 4, v[4:5]
@@ -965,14 +960,14 @@ define amdgpu_kernel void @v_brev_v2i64(<2 x i64> addrspace(1)* noalias %out, <2
 ; FLAT-NEXT:    v_or_b32_e32 v2, v2, v0
 ; FLAT-NEXT:    v_or_b32_e32 v7, v7, v5
 ; FLAT-NEXT:    v_or_b32_e32 v6, v6, v4
-; FLAT-NEXT:    v_and_b32_e32 v1, s9, v3
-; FLAT-NEXT:    v_and_b32_e32 v0, s9, v2
-; FLAT-NEXT:    v_and_b32_e32 v5, s9, v7
-; FLAT-NEXT:    v_and_b32_e32 v4, s9, v6
-; FLAT-NEXT:    v_and_b32_e32 v3, s10, v3
-; FLAT-NEXT:    v_and_b32_e32 v2, s10, v2
-; FLAT-NEXT:    v_and_b32_e32 v7, s10, v7
-; FLAT-NEXT:    v_and_b32_e32 v6, s10, v6
+; FLAT-NEXT:    v_and_b32_e32 v1, s1, v3
+; FLAT-NEXT:    v_and_b32_e32 v0, s1, v2
+; FLAT-NEXT:    v_and_b32_e32 v5, s1, v7
+; FLAT-NEXT:    v_and_b32_e32 v4, s1, v6
+; FLAT-NEXT:    v_and_b32_e32 v3, s3, v3
+; FLAT-NEXT:    v_and_b32_e32 v2, s3, v2
+; FLAT-NEXT:    v_and_b32_e32 v7, s3, v7
+; FLAT-NEXT:    v_and_b32_e32 v6, s3, v6
 ; FLAT-NEXT:    v_lshlrev_b64 v[0:1], 2, v[0:1]
 ; FLAT-NEXT:    v_lshrrev_b64 v[2:3], 2, v[2:3]
 ; FLAT-NEXT:    v_lshlrev_b64 v[4:5], 2, v[4:5]
@@ -981,14 +976,14 @@ define amdgpu_kernel void @v_brev_v2i64(<2 x i64> addrspace(1)* noalias %out, <2
 ; FLAT-NEXT:    v_or_b32_e32 v2, v2, v0
 ; FLAT-NEXT:    v_or_b32_e32 v7, v7, v5
 ; FLAT-NEXT:    v_or_b32_e32 v6, v6, v4
-; FLAT-NEXT:    v_and_b32_e32 v1, s11, v3
-; FLAT-NEXT:    v_and_b32_e32 v0, s11, v2
-; FLAT-NEXT:    v_and_b32_e32 v5, s11, v7
-; FLAT-NEXT:    v_and_b32_e32 v4, s11, v6
-; FLAT-NEXT:    v_and_b32_e32 v3, s12, v3
-; FLAT-NEXT:    v_and_b32_e32 v2, s12, v2
-; FLAT-NEXT:    v_and_b32_e32 v7, s12, v7
-; FLAT-NEXT:    v_and_b32_e32 v6, s12, v6
+; FLAT-NEXT:    v_and_b32_e32 v1, s8, v3
+; FLAT-NEXT:    v_and_b32_e32 v0, s8, v2
+; FLAT-NEXT:    v_and_b32_e32 v5, s8, v7
+; FLAT-NEXT:    v_and_b32_e32 v4, s8, v6
+; FLAT-NEXT:    v_and_b32_e32 v3, s9, v3
+; FLAT-NEXT:    v_and_b32_e32 v2, s9, v2
+; FLAT-NEXT:    v_and_b32_e32 v7, s9, v7
+; FLAT-NEXT:    v_and_b32_e32 v6, s9, v6
 ; FLAT-NEXT:    v_lshlrev_b64 v[0:1], 1, v[0:1]
 ; FLAT-NEXT:    v_lshrrev_b64 v[2:3], 1, v[2:3]
 ; FLAT-NEXT:    v_lshlrev_b64 v[4:5], 1, v[4:5]
diff --git a/llvm/test/CodeGen/AMDGPU/branch-relax-spill.ll b/llvm/test/CodeGen/AMDGPU/branch-relax-spill.ll
index c81128646..e4d3df91d 100644
--- a/llvm/test/CodeGen/AMDGPU/branch-relax-spill.ll
+++ b/llvm/test/CodeGen/AMDGPU/branch-relax-spill.ll
@@ -1,4 +1,4 @@
-; RUN: not llc -march=amdgcn -verify-machineinstrs  -amdgpu-s-branch-bits=4 < %s 2>&1 | FileCheck -check-prefix=FAIL %s
+; RUN: not llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs  -amdgpu-s-branch-bits=4 < %s 2>&1 | FileCheck -check-prefix=FAIL %s
 
 ; FIXME: This should be able to compile, but requires inserting an
 ; extra block to restore the scavenged register.
@@ -109,8 +109,6 @@ entry:
   %sgpr99 = tail call i32 asm sideeffect "s_mov_b32 s99, 0", "={s99}"() #0
   %sgpr100 = tail call i32 asm sideeffect "s_mov_b32 s100, 0", "={s100}"() #0
   %sgpr101 = tail call i32 asm sideeffect "s_mov_b32 s101, 0", "={s101}"() #0
-  %sgpr102 = tail call i32 asm sideeffect "s_mov_b32 s102, 0", "={s102}"() #0
-  %sgpr103 = tail call i32 asm sideeffect "s_mov_b32 s103, 0", "={s103}"() #0
   %vcc_lo = tail call i32 asm sideeffect "s_mov_b32 $0, 0", "={vcc_lo}"() #0
   %vcc_hi = tail call i32 asm sideeffect "s_mov_b32 $0, 0", "={vcc_hi}"() #0
   %cmp = icmp eq i32 %cnd, 0
@@ -228,8 +226,6 @@ bb3:
   tail call void asm sideeffect "; reg use $0", "{s99}"(i32 %sgpr99) #0
   tail call void asm sideeffect "; reg use $0", "{s100}"(i32 %sgpr100) #0
   tail call void asm sideeffect "; reg use $0", "{s101}"(i32 %sgpr101) #0
-  tail call void asm sideeffect "; reg use $0", "{s102}"(i32 %sgpr102) #0
-  tail call void asm sideeffect "; reg use $0", "{s103}"(i32 %sgpr103) #0
   tail call void asm sideeffect "; reg use $0", "{vcc_lo}"(i32 %vcc_lo) #0
   tail call void asm sideeffect "; reg use $0", "{vcc_hi}"(i32 %vcc_hi) #0
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/branch-relaxation.ll b/llvm/test/CodeGen/AMDGPU/branch-relaxation.ll
index 45ed05656..a2facaafb 100644
--- a/llvm/test/CodeGen/AMDGPU/branch-relaxation.ll
+++ b/llvm/test/CodeGen/AMDGPU/branch-relaxation.ll
@@ -99,7 +99,7 @@ bb3:
 
 ; GCN-LABEL: {{^}}uniform_conditional_min_long_forward_vcnd_branch:
 ; GCN: s_load_dword [[CND:s[0-9]+]]
-; GCN-DAG: v_mov_b32_e32 [[V_CND:v[0-9]+]], [[CND]]
+
 ; GCN-DAG: v_cmp_eq_f32_e64 [[UNMASKED:s\[[0-9]+:[0-9]+\]]], [[CND]], 0
 ; GCN-DAG: s_and_b64 vcc, exec, [[UNMASKED]]
 ; GCN: s_cbranch_vccz [[LONGBB:BB[0-9]+_[0-9]+]]
@@ -117,6 +117,7 @@ bb3:
 ; GCN: v_nop_e64
 
 ; GCN: [[ENDBB]]:
+; GCN: v_mov_b32_e32 [[V_CND:v[0-9]+]], [[CND]]
 ; GCN: buffer_store_dword [[V_CND]]
 ; GCN: s_endpgm
 define amdgpu_kernel void @uniform_conditional_min_long_forward_vcnd_branch(float addrspace(1)* %arg, float %cnd) #0 {
diff --git a/llvm/test/CodeGen/AMDGPU/branch-uniformity.ll b/llvm/test/CodeGen/AMDGPU/branch-uniformity.ll
index e6f684178..c9c801fb1 100644
--- a/llvm/test/CodeGen/AMDGPU/branch-uniformity.ll
+++ b/llvm/test/CodeGen/AMDGPU/branch-uniformity.ll
@@ -8,8 +8,8 @@
 ;
 ; CHECK-LABEL: {{^}}main:
 ; CHECK: ; %LOOP49
-; CHECK: v_cmp_ne_u32_e32 vcc,
-; CHECK: s_cbranch_vccnz
+; CHECK: s_cmp_lg_u32 s{{[0-9]+}}, 0
+; CHECK: s_cbranch_scc1
 ; CHECK: ; %ENDIF53
 define amdgpu_vs float @main(i32 %in) {
 main_body:
diff --git a/llvm/test/CodeGen/AMDGPU/bswap.ll b/llvm/test/CodeGen/AMDGPU/bswap.ll
index 9f24365e5..6f3cd3950 100644
--- a/llvm/test/CodeGen/AMDGPU/bswap.ll
+++ b/llvm/test/CodeGen/AMDGPU/bswap.ll
@@ -189,41 +189,41 @@ define amdgpu_kernel void @test_bswap_v8i32(<8 x i32> addrspace(1)* %out, <8 x i
 ;
 ; VI-LABEL: test_bswap_v8i32:
 ; VI:       ; %bb.0:
-; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; VI-NEXT:    s_mov_b32 s12, 0xff00ff
-; VI-NEXT:    s_mov_b32 s11, 0xf000
-; VI-NEXT:    s_mov_b32 s10, -1
+; VI-NEXT:    s_mov_b32 s3, 0xf000
+; VI-NEXT:    s_mov_b32 s2, -1
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
-; VI-NEXT:    s_mov_b32 s8, s0
-; VI-NEXT:    s_mov_b32 s9, s1
-; VI-NEXT:    s_load_dwordx8 s[0:7], s[2:3], 0x0
+; VI-NEXT:    s_mov_b32 s0, s4
+; VI-NEXT:    s_mov_b32 s1, s5
+; VI-NEXT:    s_load_dwordx8 s[4:11], s[6:7], 0x0
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
-; VI-NEXT:    v_alignbit_b32 v0, s3, s3, 8
-; VI-NEXT:    v_alignbit_b32 v1, s3, s3, 24
+; VI-NEXT:    v_alignbit_b32 v0, s7, s7, 8
+; VI-NEXT:    v_alignbit_b32 v1, s7, s7, 24
 ; VI-NEXT:    v_bfi_b32 v3, s12, v1, v0
-; VI-NEXT:    v_alignbit_b32 v2, s2, s2, 8
-; VI-NEXT:    v_alignbit_b32 v4, s2, s2, 24
-; VI-NEXT:    v_alignbit_b32 v0, s1, s1, 8
-; VI-NEXT:    v_alignbit_b32 v1, s1, s1, 24
+; VI-NEXT:    v_alignbit_b32 v2, s6, s6, 8
+; VI-NEXT:    v_alignbit_b32 v4, s6, s6, 24
+; VI-NEXT:    v_alignbit_b32 v0, s5, s5, 8
+; VI-NEXT:    v_alignbit_b32 v1, s5, s5, 24
 ; VI-NEXT:    v_bfi_b32 v2, s12, v4, v2
 ; VI-NEXT:    v_bfi_b32 v1, s12, v1, v0
-; VI-NEXT:    v_alignbit_b32 v0, s0, s0, 8
-; VI-NEXT:    v_alignbit_b32 v4, s0, s0, 24
+; VI-NEXT:    v_alignbit_b32 v0, s4, s4, 8
+; VI-NEXT:    v_alignbit_b32 v4, s4, s4, 24
 ; VI-NEXT:    v_bfi_b32 v0, s12, v4, v0
-; VI-NEXT:    v_alignbit_b32 v4, s7, s7, 8
-; VI-NEXT:    v_alignbit_b32 v5, s7, s7, 24
+; VI-NEXT:    v_alignbit_b32 v4, s11, s11, 8
+; VI-NEXT:    v_alignbit_b32 v5, s11, s11, 24
 ; VI-NEXT:    v_bfi_b32 v7, s12, v5, v4
-; VI-NEXT:    v_alignbit_b32 v4, s6, s6, 8
-; VI-NEXT:    v_alignbit_b32 v5, s6, s6, 24
+; VI-NEXT:    v_alignbit_b32 v4, s10, s10, 8
+; VI-NEXT:    v_alignbit_b32 v5, s10, s10, 24
 ; VI-NEXT:    v_bfi_b32 v6, s12, v5, v4
-; VI-NEXT:    v_alignbit_b32 v4, s5, s5, 8
-; VI-NEXT:    v_alignbit_b32 v5, s5, s5, 24
+; VI-NEXT:    v_alignbit_b32 v4, s9, s9, 8
+; VI-NEXT:    v_alignbit_b32 v5, s9, s9, 24
 ; VI-NEXT:    v_bfi_b32 v5, s12, v5, v4
-; VI-NEXT:    v_alignbit_b32 v4, s4, s4, 8
-; VI-NEXT:    v_alignbit_b32 v8, s4, s4, 24
+; VI-NEXT:    v_alignbit_b32 v4, s8, s8, 8
+; VI-NEXT:    v_alignbit_b32 v8, s8, s8, 24
 ; VI-NEXT:    v_bfi_b32 v4, s12, v8, v4
-; VI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[8:11], 0 offset:16
-; VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[8:11], 0
+; VI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:16
+; VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
   %val = load <8 x i32>, <8 x i32> addrspace(1)* %in, align 32
   %bswap = call <8 x i32> @llvm.bswap.v8i32(<8 x i32> %val) nounwind readnone
@@ -234,11 +234,11 @@ define amdgpu_kernel void @test_bswap_v8i32(<8 x i32> addrspace(1)* %out, <8 x i
 define amdgpu_kernel void @test_bswap_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %in) nounwind {
 ; SI-LABEL: test_bswap_i64:
 ; SI:       ; %bb.0:
-; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
-; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s3, 0xf000
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
-; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_load_dwordx2 s[6:7], s[6:7], 0x0
+; SI-NEXT:    s_mov_b32 s2, -1
 ; SI-NEXT:    s_mov_b32 s19, 0xff0000
 ; SI-NEXT:    s_mov_b32 s9, 0
 ; SI-NEXT:    s_mov_b32 s15, 0xff00
@@ -247,34 +247,34 @@ define amdgpu_kernel void @test_bswap_i64(i64 addrspace(1)* %out, i64 addrspace(
 ; SI-NEXT:    s_mov_b32 s14, s9
 ; SI-NEXT:    s_mov_b32 s16, s9
 ; SI-NEXT:    s_mov_b32 s18, s9
-; SI-NEXT:    s_mov_b32 s4, s0
-; SI-NEXT:    s_mov_b32 s5, s1
+; SI-NEXT:    s_mov_b32 s0, s4
+; SI-NEXT:    s_mov_b32 s1, s5
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NEXT:    v_mov_b32_e32 v0, s2
-; SI-NEXT:    v_alignbit_b32 v1, s3, v0, 24
-; SI-NEXT:    v_alignbit_b32 v0, s3, v0, 8
-; SI-NEXT:    s_lshr_b32 s8, s3, 24
-; SI-NEXT:    s_lshr_b32 s10, s3, 8
-; SI-NEXT:    s_lshl_b64 s[0:1], s[2:3], 8
-; SI-NEXT:    s_lshl_b64 s[20:21], s[2:3], 24
-; SI-NEXT:    s_lshl_b32 s17, s2, 24
-; SI-NEXT:    s_lshl_b32 s0, s2, 8
+; SI-NEXT:    v_mov_b32_e32 v0, s6
+; SI-NEXT:    v_alignbit_b32 v1, s7, v0, 24
+; SI-NEXT:    v_alignbit_b32 v0, s7, v0, 8
+; SI-NEXT:    s_lshr_b32 s8, s7, 24
+; SI-NEXT:    s_lshr_b32 s10, s7, 8
+; SI-NEXT:    s_lshl_b64 s[4:5], s[6:7], 8
+; SI-NEXT:    s_lshl_b64 s[20:21], s[6:7], 24
+; SI-NEXT:    s_lshl_b32 s17, s6, 24
+; SI-NEXT:    s_lshl_b32 s4, s6, 8
 ; SI-NEXT:    v_and_b32_e32 v1, s19, v1
 ; SI-NEXT:    v_and_b32_e32 v0, 0xff000000, v0
 ; SI-NEXT:    s_and_b32 s10, s10, s15
-; SI-NEXT:    s_and_b32 s13, s1, 0xff
+; SI-NEXT:    s_and_b32 s13, s5, 0xff
 ; SI-NEXT:    s_and_b32 s15, s21, s15
-; SI-NEXT:    s_and_b32 s19, s0, s19
+; SI-NEXT:    s_and_b32 s19, s4, s19
 ; SI-NEXT:    v_or_b32_e32 v0, v0, v1
-; SI-NEXT:    s_or_b64 s[0:1], s[10:11], s[8:9]
-; SI-NEXT:    s_or_b64 s[2:3], s[14:15], s[12:13]
+; SI-NEXT:    s_or_b64 s[4:5], s[10:11], s[8:9]
+; SI-NEXT:    s_or_b64 s[6:7], s[14:15], s[12:13]
 ; SI-NEXT:    s_or_b64 s[8:9], s[16:17], s[18:19]
-; SI-NEXT:    v_or_b32_e32 v0, s0, v0
-; SI-NEXT:    v_mov_b32_e32 v1, s1
-; SI-NEXT:    s_or_b64 s[0:1], s[8:9], s[2:3]
-; SI-NEXT:    v_or_b32_e32 v0, s0, v0
-; SI-NEXT:    v_or_b32_e32 v1, s1, v1
-; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; SI-NEXT:    v_or_b32_e32 v0, s4, v0
+; SI-NEXT:    v_mov_b32_e32 v1, s5
+; SI-NEXT:    s_or_b64 s[4:5], s[8:9], s[6:7]
+; SI-NEXT:    v_or_b32_e32 v0, s4, v0
+; SI-NEXT:    v_or_b32_e32 v1, s5, v1
+; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: test_bswap_i64:
@@ -328,47 +328,47 @@ define amdgpu_kernel void @test_bswap_i64(i64 addrspace(1)* %out, i64 addrspace(
 define amdgpu_kernel void @test_bswap_v2i64(<2 x i64> addrspace(1)* %out, <2 x i64> addrspace(1)* %in) nounwind {
 ; SI-LABEL: test_bswap_v2i64:
 ; SI:       ; %bb.0:
-; SI-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
+; SI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
 ; SI-NEXT:    s_mov_b32 s3, 0xf000
 ; SI-NEXT:    s_mov_b32 s2, -1
 ; SI-NEXT:    s_mov_b32 s31, 0xff0000
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NEXT:    s_load_dwordx4 s[4:7], s[10:11], 0x0
-; SI-NEXT:    s_mov_b32 s11, 0
+; SI-NEXT:    s_load_dwordx4 s[8:11], s[6:7], 0x0
+; SI-NEXT:    s_mov_b32 s7, 0
 ; SI-NEXT:    s_mov_b32 s22, 0xff000000
 ; SI-NEXT:    s_mov_b32 s27, 0xff00
 ; SI-NEXT:    s_movk_i32 s25, 0xff
-; SI-NEXT:    s_mov_b32 s13, s11
-; SI-NEXT:    s_mov_b32 s14, s11
-; SI-NEXT:    s_mov_b32 s16, s11
-; SI-NEXT:    s_mov_b32 s18, s11
-; SI-NEXT:    s_mov_b32 s20, s11
-; SI-NEXT:    s_mov_b32 s23, s11
-; SI-NEXT:    s_mov_b32 s24, s11
-; SI-NEXT:    s_mov_b32 s26, s11
-; SI-NEXT:    s_mov_b32 s28, s11
-; SI-NEXT:    s_mov_b32 s30, s11
-; SI-NEXT:    s_mov_b32 s0, s8
-; SI-NEXT:    s_mov_b32 s1, s9
+; SI-NEXT:    s_mov_b32 s13, s7
+; SI-NEXT:    s_mov_b32 s14, s7
+; SI-NEXT:    s_mov_b32 s16, s7
+; SI-NEXT:    s_mov_b32 s18, s7
+; SI-NEXT:    s_mov_b32 s20, s7
+; SI-NEXT:    s_mov_b32 s23, s7
+; SI-NEXT:    s_mov_b32 s24, s7
+; SI-NEXT:    s_mov_b32 s26, s7
+; SI-NEXT:    s_mov_b32 s28, s7
+; SI-NEXT:    s_mov_b32 s30, s7
+; SI-NEXT:    s_mov_b32 s0, s4
+; SI-NEXT:    s_mov_b32 s1, s5
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NEXT:    v_mov_b32_e32 v0, s6
-; SI-NEXT:    v_alignbit_b32 v1, s7, v0, 24
-; SI-NEXT:    v_alignbit_b32 v0, s7, v0, 8
-; SI-NEXT:    s_lshr_b32 s10, s7, 24
-; SI-NEXT:    s_lshr_b32 s12, s7, 8
-; SI-NEXT:    s_lshl_b64 s[8:9], s[6:7], 8
-; SI-NEXT:    s_lshl_b64 s[32:33], s[6:7], 24
-; SI-NEXT:    s_lshl_b32 s19, s6, 24
-; SI-NEXT:    s_lshl_b32 s21, s6, 8
-; SI-NEXT:    v_mov_b32_e32 v2, s4
-; SI-NEXT:    v_alignbit_b32 v3, s5, v2, 24
-; SI-NEXT:    v_alignbit_b32 v2, s5, v2, 8
-; SI-NEXT:    s_lshr_b32 s32, s5, 8
-; SI-NEXT:    s_lshl_b64 s[6:7], s[4:5], 8
-; SI-NEXT:    s_and_b32 s15, s9, s25
-; SI-NEXT:    s_lshl_b64 s[8:9], s[4:5], 24
-; SI-NEXT:    s_lshl_b32 s29, s4, 24
-; SI-NEXT:    s_lshl_b32 s4, s4, 8
+; SI-NEXT:    v_mov_b32_e32 v0, s10
+; SI-NEXT:    v_alignbit_b32 v1, s11, v0, 24
+; SI-NEXT:    v_alignbit_b32 v0, s11, v0, 8
+; SI-NEXT:    s_lshr_b32 s6, s11, 24
+; SI-NEXT:    s_lshr_b32 s12, s11, 8
+; SI-NEXT:    s_lshl_b64 s[4:5], s[10:11], 8
+; SI-NEXT:    s_lshl_b64 s[32:33], s[10:11], 24
+; SI-NEXT:    s_lshl_b32 s19, s10, 24
+; SI-NEXT:    s_lshl_b32 s21, s10, 8
+; SI-NEXT:    v_mov_b32_e32 v2, s8
+; SI-NEXT:    v_alignbit_b32 v3, s9, v2, 24
+; SI-NEXT:    v_alignbit_b32 v2, s9, v2, 8
+; SI-NEXT:    s_lshr_b32 s32, s9, 8
+; SI-NEXT:    s_lshl_b64 s[10:11], s[8:9], 8
+; SI-NEXT:    s_and_b32 s15, s5, s25
+; SI-NEXT:    s_lshl_b64 s[4:5], s[8:9], 24
+; SI-NEXT:    s_lshl_b32 s29, s8, 24
+; SI-NEXT:    s_lshl_b32 s4, s8, 8
 ; SI-NEXT:    v_and_b32_e32 v1, s31, v1
 ; SI-NEXT:    v_and_b32_e32 v0, s22, v0
 ; SI-NEXT:    s_and_b32 s12, s12, s27
@@ -377,28 +377,28 @@ define amdgpu_kernel void @test_bswap_v2i64(<2 x i64> addrspace(1)* %out, <2 x i
 ; SI-NEXT:    v_and_b32_e32 v3, s31, v3
 ; SI-NEXT:    v_and_b32_e32 v2, s22, v2
 ; SI-NEXT:    s_and_b32 s22, s32, s27
-; SI-NEXT:    s_and_b32 s25, s7, s25
-; SI-NEXT:    s_and_b32 s27, s9, s27
+; SI-NEXT:    s_and_b32 s25, s11, s25
+; SI-NEXT:    s_and_b32 s27, s5, s27
 ; SI-NEXT:    s_and_b32 s31, s4, s31
 ; SI-NEXT:    v_or_b32_e32 v0, v0, v1
-; SI-NEXT:    s_or_b64 s[6:7], s[12:13], s[10:11]
-; SI-NEXT:    s_or_b64 s[8:9], s[16:17], s[14:15]
+; SI-NEXT:    s_or_b64 s[4:5], s[12:13], s[6:7]
+; SI-NEXT:    s_or_b64 s[10:11], s[16:17], s[14:15]
 ; SI-NEXT:    s_or_b64 s[12:13], s[18:19], s[20:21]
 ; SI-NEXT:    v_or_b32_e32 v1, v2, v3
-; SI-NEXT:    s_lshr_b32 s10, s5, 24
-; SI-NEXT:    s_or_b64 s[4:5], s[26:27], s[24:25]
+; SI-NEXT:    s_lshr_b32 s6, s9, 24
+; SI-NEXT:    s_or_b64 s[8:9], s[26:27], s[24:25]
 ; SI-NEXT:    s_or_b64 s[14:15], s[28:29], s[30:31]
-; SI-NEXT:    v_or_b32_e32 v0, s6, v0
-; SI-NEXT:    v_mov_b32_e32 v3, s7
-; SI-NEXT:    s_or_b64 s[6:7], s[12:13], s[8:9]
-; SI-NEXT:    s_or_b64 s[8:9], s[22:23], s[10:11]
-; SI-NEXT:    s_or_b64 s[4:5], s[14:15], s[4:5]
-; SI-NEXT:    v_or_b32_e32 v2, s6, v0
-; SI-NEXT:    v_or_b32_e32 v3, s7, v3
-; SI-NEXT:    v_or_b32_e32 v0, s8, v1
-; SI-NEXT:    v_mov_b32_e32 v1, s9
 ; SI-NEXT:    v_or_b32_e32 v0, s4, v0
-; SI-NEXT:    v_or_b32_e32 v1, s5, v1
+; SI-NEXT:    v_mov_b32_e32 v3, s5
+; SI-NEXT:    s_or_b64 s[4:5], s[12:13], s[10:11]
+; SI-NEXT:    s_or_b64 s[6:7], s[22:23], s[6:7]
+; SI-NEXT:    s_or_b64 s[8:9], s[14:15], s[8:9]
+; SI-NEXT:    v_or_b32_e32 v2, s4, v0
+; SI-NEXT:    v_or_b32_e32 v3, s5, v3
+; SI-NEXT:    v_or_b32_e32 v0, s6, v1
+; SI-NEXT:    v_mov_b32_e32 v1, s7
+; SI-NEXT:    v_or_b32_e32 v0, s8, v0
+; SI-NEXT:    v_or_b32_e32 v1, s9, v1
 ; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
 ; SI-NEXT:    s_endpgm
 ;
@@ -732,7 +732,6 @@ define float @missing_truncate_promote_bswap(i32 %arg) {
 ; VI-LABEL: missing_truncate_promote_bswap:
 ; VI:       ; %bb.0: ; %bb
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; VI-NEXT:    v_alignbit_b32 v1, v0, v0, 8
 ; VI-NEXT:    v_alignbit_b32 v0, v0, v0, 24
 ; VI-NEXT:    s_mov_b32 s4, 0xff00ff
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-intrinsics-mmo-offsets.ll b/llvm/test/CodeGen/AMDGPU/buffer-intrinsics-mmo-offsets.ll
new file mode 100644
index 000000000..ce62e041a
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/buffer-intrinsics-mmo-offsets.ll
@@ -0,0 +1,414 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; RUN: llc -march=amdgcn -verify-machineinstrs -stop-after=amdgpu-isel -o - %s | FileCheck -check-prefix=GCN %s
+
+define amdgpu_cs void @mmo_offsets0(<4 x i32> addrspace(6)* inreg noalias dereferenceable(18446744073709551615) %arg0, i32 %arg1) {
+  ; GCN-LABEL: name: mmo_offsets0
+  ; GCN: bb.0.bb.0:
+  ; GCN:   liveins: $sgpr0, $vgpr0
+  ; GCN:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GCN:   [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+  ; GCN:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+  ; GCN:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[S_MOV_B32_]], %subreg.sub1
+  ; GCN:   [[S_LOAD_DWORDX4_IMM:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM killed [[REG_SEQUENCE]], 0, 0, 0 :: (dereferenceable invariant load 16 from %ir.arg0, addrspace 6)
+  ; GCN:   [[BUFFER_LOAD_DWORDX4_OFFSET:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 16, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 16 from custom "TargetCustom7" + 16, align 1, addrspace 4)
+  ; GCN:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+  ; GCN:   [[BUFFER_LOAD_DWORDX4_IDXEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 16, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 16 from custom "TargetCustom7", align 1, addrspace 4)
+  ; GCN:   [[BUFFER_LOAD_DWORDX4_OFFEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFEN [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 16 from custom "TargetCustom7", align 1, addrspace 4)
+  ; GCN:   [[BUFFER_LOAD_DWORDX4_IDXEN1:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 16, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 16 from custom "TargetCustom7", align 1, addrspace 4)
+  ; GCN:   INLINEASM &"", 1
+  ; GCN:   BUFFER_STORE_DWORDX4_OFFSET_exact killed [[BUFFER_LOAD_DWORDX4_OFFSET]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 32, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 16 into custom "TargetCustom7" + 32, align 1, addrspace 4)
+  ; GCN:   BUFFER_STORE_DWORDX4_OFFEN_exact killed [[BUFFER_LOAD_DWORDX4_OFFEN]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 16 into custom "TargetCustom7", align 1, addrspace 4)
+  ; GCN:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN]], [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 32, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 16 into custom "TargetCustom7", align 1, addrspace 4)
+  ; GCN:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN1]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 32, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 16 into custom "TargetCustom7", align 1, addrspace 4)
+  ; GCN:   INLINEASM &"", 1
+  ; GCN:   [[BUFFER_LOAD_FORMAT_XYZW_OFFSET:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFSET [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 48, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 16 from custom "TargetCustom7" + 48, align 1, addrspace 4)
+  ; GCN:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 48, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 16 from custom "TargetCustom7", align 1, addrspace 4)
+  ; GCN:   [[BUFFER_LOAD_FORMAT_XYZW_OFFEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFEN [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 16 from custom "TargetCustom7", align 1, addrspace 4)
+  ; GCN:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN1:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 48, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 16 from custom "TargetCustom7", align 1, addrspace 4)
+  ; GCN:   INLINEASM &"", 1
+  ; GCN:   BUFFER_STORE_FORMAT_XYZW_OFFSET_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFSET]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 64, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 16 into custom "TargetCustom7" + 64, align 1, addrspace 4)
+  ; GCN:   BUFFER_STORE_FORMAT_XYZW_OFFEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFEN]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 16 into custom "TargetCustom7", align 1, addrspace 4)
+  ; GCN:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN]], [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 64, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 16 into custom "TargetCustom7", align 1, addrspace 4)
+  ; GCN:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN1]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 64, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 16 into custom "TargetCustom7", align 1, addrspace 4)
+  ; GCN:   INLINEASM &"", 1
+  ; GCN:   BUFFER_ATOMIC_ADD_OFFSET [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 80, 0, implicit $exec :: (volatile dereferenceable load store 4 on custom "TargetCustom7" + 80, align 1, addrspace 4)
+  ; GCN:   BUFFER_ATOMIC_ADD_OFFEN [[COPY]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 0, 0, implicit $exec :: (volatile dereferenceable load store 4 on custom "TargetCustom7", align 1, addrspace 4)
+  ; GCN:   BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 80, 0, implicit $exec :: (volatile dereferenceable load store 4 on custom "TargetCustom7", align 1, addrspace 4)
+  ; GCN:   BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 80, 0, implicit $exec :: (volatile dereferenceable load store 4 on custom "TargetCustom7", align 1, addrspace 4)
+  ; GCN:   INLINEASM &"", 1
+  ; GCN:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY]], %subreg.sub1
+  ; GCN:   [[DEF:%[0-9]+]]:vreg_64 = IMPLICIT_DEF
+  ; GCN:   BUFFER_ATOMIC_CMPSWAP_OFFSET [[REG_SEQUENCE1]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 96, 0, implicit $exec :: (volatile dereferenceable load store 4 on custom "TargetCustom7" + 96, align 1, addrspace 4)
+  ; GCN:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[DEF]].sub0
+  ; GCN:   [[DEF1:%[0-9]+]]:vreg_64 = IMPLICIT_DEF
+  ; GCN:   BUFFER_ATOMIC_CMPSWAP_OFFEN [[REG_SEQUENCE1]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 0, 0, implicit $exec :: (volatile dereferenceable load store 4 on custom "TargetCustom7", align 1, addrspace 4)
+  ; GCN:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[DEF1]].sub0
+  ; GCN:   [[DEF2:%[0-9]+]]:vreg_64 = IMPLICIT_DEF
+  ; GCN:   BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE1]], [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 96, 0, implicit $exec :: (volatile dereferenceable load store 4 on custom "TargetCustom7", align 1, addrspace 4)
+  ; GCN:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[DEF2]].sub0
+  ; GCN:   [[DEF3:%[0-9]+]]:vreg_64 = IMPLICIT_DEF
+  ; GCN:   BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE1]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 96, 0, implicit $exec :: (volatile dereferenceable load store 4 on custom "TargetCustom7", align 1, addrspace 4)
+  ; GCN:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[DEF3]].sub0
+  ; GCN:   INLINEASM &"", 1
+  ; GCN:   [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1065353216, implicit $exec
+  ; GCN:   BUFFER_ATOMIC_ADD_F32_OFFSET [[V_MOV_B32_e32_1]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 112, 0, implicit $exec :: (load store 4 on custom "TargetCustom7" + 112, addrspace 4)
+  ; GCN:   BUFFER_ATOMIC_ADD_F32_OFFEN [[V_MOV_B32_e32_1]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 0, 0, implicit $exec :: (load store 4 on custom "TargetCustom7", addrspace 4)
+  ; GCN:   BUFFER_ATOMIC_ADD_F32_IDXEN [[V_MOV_B32_e32_1]], [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 112, 0, implicit $exec :: (load store 4 on custom "TargetCustom7", addrspace 4)
+  ; GCN:   BUFFER_ATOMIC_ADD_F32_IDXEN [[V_MOV_B32_e32_1]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 112, 0, implicit $exec :: (load store 4 on custom "TargetCustom7", addrspace 4)
+  ; GCN:   INLINEASM &"", 1
+  ; GCN:   [[BUFFER_LOAD_DWORDX4_OFFSET1:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 128, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 16 from custom "TargetCustom7" + 128, align 1, addrspace 4)
+  ; GCN:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 64
+  ; GCN:   [[BUFFER_LOAD_DWORDX4_OFFSET2:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET [[S_LOAD_DWORDX4_IMM]], killed [[S_MOV_B32_1]], 64, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 16 from custom "TargetCustom7" + 128, align 1, addrspace 4)
+  ; GCN:   [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 128
+  ; GCN:   [[BUFFER_LOAD_DWORDX4_OFFSET3:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_2]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 16 from custom "TargetCustom7" + 128, align 1, addrspace 4)
+  ; GCN:   [[BUFFER_LOAD_DWORDX4_OFFEN1:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFEN [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_2]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 16 from custom "TargetCustom7", align 1, addrspace 4)
+  ; GCN:   [[COPY6:%[0-9]+]]:sreg_32 = COPY [[COPY]]
+  ; GCN:   [[BUFFER_LOAD_DWORDX4_OFFSET4:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET [[S_LOAD_DWORDX4_IMM]], [[COPY6]], 128, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 16 from custom "TargetCustom7", align 1, addrspace 4)
+  ; GCN:   INLINEASM &"", 1
+  ; GCN:   [[BUFFER_LOAD_FORMAT_XYZW_OFFSET1:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFSET [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 144, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 16 from custom "TargetCustom7" + 144, align 1, addrspace 4)
+  ; GCN:   [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 72
+  ; GCN:   [[BUFFER_LOAD_FORMAT_XYZW_OFFSET2:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFSET [[S_LOAD_DWORDX4_IMM]], killed [[S_MOV_B32_3]], 72, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 16 from custom "TargetCustom7" + 144, align 1, addrspace 4)
+  ; GCN:   [[S_MOV_B32_4:%[0-9]+]]:sreg_32 = S_MOV_B32 144
+  ; GCN:   [[BUFFER_LOAD_FORMAT_XYZW_OFFSET3:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFSET [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_4]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 16 from custom "TargetCustom7" + 144, align 1, addrspace 4)
+  ; GCN:   [[BUFFER_LOAD_FORMAT_XYZW_OFFEN1:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFEN [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_4]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 16 from custom "TargetCustom7", align 1, addrspace 4)
+  ; GCN:   [[COPY7:%[0-9]+]]:sreg_32 = COPY [[COPY]]
+  ; GCN:   [[BUFFER_LOAD_FORMAT_XYZW_OFFSET4:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFSET [[S_LOAD_DWORDX4_IMM]], [[COPY7]], 144, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 16 from custom "TargetCustom7", align 1, addrspace 4)
+  ; GCN:   INLINEASM &"", 1
+  ; GCN:   BUFFER_ATOMIC_ADD_OFFSET [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 160, 0, implicit $exec :: (volatile dereferenceable load store 4 on custom "TargetCustom7" + 160, align 1, addrspace 4)
+  ; GCN:   [[S_MOV_B32_5:%[0-9]+]]:sreg_32 = S_MOV_B32 80
+  ; GCN:   BUFFER_ATOMIC_ADD_OFFSET [[COPY]], [[S_LOAD_DWORDX4_IMM]], killed [[S_MOV_B32_5]], 80, 0, implicit $exec :: (volatile dereferenceable load store 4 on custom "TargetCustom7" + 160, align 1, addrspace 4)
+  ; GCN:   [[S_MOV_B32_6:%[0-9]+]]:sreg_32 = S_MOV_B32 160
+  ; GCN:   BUFFER_ATOMIC_ADD_OFFSET [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_6]], 0, 0, implicit $exec :: (volatile dereferenceable load store 4 on custom "TargetCustom7" + 160, align 1, addrspace 4)
+  ; GCN:   BUFFER_ATOMIC_ADD_OFFEN [[COPY]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_6]], 0, 0, implicit $exec :: (volatile dereferenceable load store 4 on custom "TargetCustom7", align 1, addrspace 4)
+  ; GCN:   [[COPY8:%[0-9]+]]:sreg_32 = COPY [[COPY]]
+  ; GCN:   BUFFER_ATOMIC_ADD_OFFSET [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[COPY8]], 160, 0, implicit $exec :: (volatile dereferenceable load store 4 on custom "TargetCustom7", align 1, addrspace 4)
+  ; GCN:   INLINEASM &"", 1
+  ; GCN:   [[DEF4:%[0-9]+]]:vreg_64 = IMPLICIT_DEF
+  ; GCN:   BUFFER_ATOMIC_CMPSWAP_OFFSET [[REG_SEQUENCE1]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 176, 0, implicit $exec :: (volatile dereferenceable load store 4 on custom "TargetCustom7" + 176, align 1, addrspace 4)
+  ; GCN:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[DEF4]].sub0
+  ; GCN:   [[S_MOV_B32_7:%[0-9]+]]:sreg_32 = S_MOV_B32 88
+  ; GCN:   [[DEF5:%[0-9]+]]:vreg_64 = IMPLICIT_DEF
+  ; GCN:   BUFFER_ATOMIC_CMPSWAP_OFFSET [[REG_SEQUENCE1]], [[S_LOAD_DWORDX4_IMM]], killed [[S_MOV_B32_7]], 88, 0, implicit $exec :: (volatile dereferenceable load store 4 on custom "TargetCustom7" + 176, align 1, addrspace 4)
+  ; GCN:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[DEF5]].sub0
+  ; GCN:   [[S_MOV_B32_8:%[0-9]+]]:sreg_32 = S_MOV_B32 176
+  ; GCN:   [[DEF6:%[0-9]+]]:vreg_64 = IMPLICIT_DEF
+  ; GCN:   BUFFER_ATOMIC_CMPSWAP_OFFSET [[REG_SEQUENCE1]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_8]], 0, 0, implicit $exec :: (volatile dereferenceable load store 4 on custom "TargetCustom7" + 176, align 1, addrspace 4)
+  ; GCN:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[DEF6]].sub0
+  ; GCN:   [[DEF7:%[0-9]+]]:vreg_64 = IMPLICIT_DEF
+  ; GCN:   BUFFER_ATOMIC_CMPSWAP_OFFEN [[REG_SEQUENCE1]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_8]], 0, 0, implicit $exec :: (volatile dereferenceable load store 4 on custom "TargetCustom7", align 1, addrspace 4)
+  ; GCN:   [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[DEF7]].sub0
+  ; GCN:   [[COPY13:%[0-9]+]]:sreg_32 = COPY [[COPY]]
+  ; GCN:   [[DEF8:%[0-9]+]]:vreg_64 = IMPLICIT_DEF
+  ; GCN:   BUFFER_ATOMIC_CMPSWAP_OFFSET [[REG_SEQUENCE1]], [[S_LOAD_DWORDX4_IMM]], [[COPY13]], 176, 0, implicit $exec :: (volatile dereferenceable load store 4 on custom "TargetCustom7", align 1, addrspace 4)
+  ; GCN:   [[COPY14:%[0-9]+]]:vgpr_32 = COPY [[DEF8]].sub0
+  ; GCN:   INLINEASM &"", 1
+  ; GCN:   BUFFER_STORE_DWORDX4_OFFSET_exact killed [[BUFFER_LOAD_DWORDX4_OFFSET1]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 192, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 16 into custom "TargetCustom7" + 192, align 1, addrspace 4)
+  ; GCN:   [[S_MOV_B32_9:%[0-9]+]]:sreg_32 = S_MOV_B32 96
+  ; GCN:   BUFFER_STORE_DWORDX4_OFFSET_exact killed [[BUFFER_LOAD_DWORDX4_OFFSET2]], [[S_LOAD_DWORDX4_IMM]], killed [[S_MOV_B32_9]], 96, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 16 into custom "TargetCustom7" + 192, align 1, addrspace 4)
+  ; GCN:   [[S_MOV_B32_10:%[0-9]+]]:sreg_32 = S_MOV_B32 192
+  ; GCN:   BUFFER_STORE_DWORDX4_OFFSET_exact killed [[BUFFER_LOAD_DWORDX4_OFFSET3]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_10]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 16 into custom "TargetCustom7" + 192, align 1, addrspace 4)
+  ; GCN:   BUFFER_STORE_DWORDX4_OFFEN_exact killed [[BUFFER_LOAD_DWORDX4_OFFEN1]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_10]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 16 into custom "TargetCustom7", align 1, addrspace 4)
+  ; GCN:   [[COPY15:%[0-9]+]]:sreg_32 = COPY [[COPY]]
+  ; GCN:   BUFFER_STORE_DWORDX4_OFFSET_exact killed [[BUFFER_LOAD_DWORDX4_OFFSET4]], [[S_LOAD_DWORDX4_IMM]], [[COPY15]], 192, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 16 into custom "TargetCustom7", align 1, addrspace 4)
+  ; GCN:   INLINEASM &"", 1
+  ; GCN:   BUFFER_STORE_FORMAT_XYZW_OFFSET_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFSET1]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 208, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 16 into custom "TargetCustom7" + 208, align 1, addrspace 4)
+  ; GCN:   [[S_MOV_B32_11:%[0-9]+]]:sreg_32 = S_MOV_B32 104
+  ; GCN:   BUFFER_STORE_FORMAT_XYZW_OFFSET_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFSET2]], [[S_LOAD_DWORDX4_IMM]], killed [[S_MOV_B32_11]], 104, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 16 into custom "TargetCustom7" + 208, align 1, addrspace 4)
+  ; GCN:   [[S_MOV_B32_12:%[0-9]+]]:sreg_32 = S_MOV_B32 208
+  ; GCN:   BUFFER_STORE_FORMAT_XYZW_OFFSET_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFSET3]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_12]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 16 into custom "TargetCustom7" + 208, align 1, addrspace 4)
+  ; GCN:   BUFFER_STORE_FORMAT_XYZW_OFFEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFEN1]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_12]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 16 into custom "TargetCustom7", align 1, addrspace 4)
+  ; GCN:   [[COPY16:%[0-9]+]]:sreg_32 = COPY [[COPY]]
+  ; GCN:   BUFFER_STORE_FORMAT_XYZW_OFFSET_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFSET4]], [[S_LOAD_DWORDX4_IMM]], [[COPY16]], 208, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 16 into custom "TargetCustom7", align 1, addrspace 4)
+  ; GCN:   INLINEASM &"", 1
+  ; GCN:   [[COPY17:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; GCN:   [[BUFFER_LOAD_DWORDX4_IDXEN2:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY17]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 224, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 16 from custom "TargetCustom7" + 224, align 1, addrspace 4)
+  ; GCN:   [[S_MOV_B32_13:%[0-9]+]]:sreg_32 = S_MOV_B32 112
+  ; GCN:   [[COPY18:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; GCN:   [[BUFFER_LOAD_DWORDX4_IDXEN3:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY18]], [[S_LOAD_DWORDX4_IMM]], killed [[S_MOV_B32_13]], 112, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 16 from custom "TargetCustom7" + 224, align 1, addrspace 4)
+  ; GCN:   [[S_MOV_B32_14:%[0-9]+]]:sreg_32 = S_MOV_B32 224
+  ; GCN:   [[COPY19:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; GCN:   [[BUFFER_LOAD_DWORDX4_IDXEN4:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY19]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_14]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 16 from custom "TargetCustom7" + 224, align 1, addrspace 4)
+  ; GCN:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[COPY]], %subreg.sub1
+  ; GCN:   [[BUFFER_LOAD_DWORDX4_BOTHEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_BOTHEN [[REG_SEQUENCE2]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_14]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 16 from custom "TargetCustom7", align 1, addrspace 4)
+  ; GCN:   [[COPY20:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; GCN:   [[COPY21:%[0-9]+]]:sreg_32 = COPY [[COPY]]
+  ; GCN:   [[BUFFER_LOAD_DWORDX4_IDXEN5:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY20]], [[S_LOAD_DWORDX4_IMM]], [[COPY21]], 224, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 16 from custom "TargetCustom7", align 1, addrspace 4)
+  ; GCN:   [[BUFFER_LOAD_DWORDX4_IDXEN6:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 224, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 16 from custom "TargetCustom7", align 1, addrspace 4)
+  ; GCN:   [[BUFFER_LOAD_DWORDX4_IDXEN7:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 224, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 16 from custom "TargetCustom7", align 1, addrspace 4)
+  ; GCN:   INLINEASM &"", 1
+  ; GCN:   [[COPY22:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; GCN:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN2:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY22]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 240, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 16 from custom "TargetCustom7" + 240, align 1, addrspace 4)
+  ; GCN:   [[S_MOV_B32_15:%[0-9]+]]:sreg_32 = S_MOV_B32 120
+  ; GCN:   [[COPY23:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; GCN:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN3:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY23]], [[S_LOAD_DWORDX4_IMM]], killed [[S_MOV_B32_15]], 120, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 16 from custom "TargetCustom7" + 240, align 1, addrspace 4)
+  ; GCN:   [[S_MOV_B32_16:%[0-9]+]]:sreg_32 = S_MOV_B32 240
+  ; GCN:   [[COPY24:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; GCN:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN4:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY24]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_16]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 16 from custom "TargetCustom7" + 240, align 1, addrspace 4)
+  ; GCN:   [[BUFFER_LOAD_FORMAT_XYZW_BOTHEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_BOTHEN [[REG_SEQUENCE2]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_16]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 16 from custom "TargetCustom7", align 1, addrspace 4)
+  ; GCN:   [[COPY25:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; GCN:   [[COPY26:%[0-9]+]]:sreg_32 = COPY [[COPY]]
+  ; GCN:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN5:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY25]], [[S_LOAD_DWORDX4_IMM]], [[COPY26]], 240, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 16 from custom "TargetCustom7", align 1, addrspace 4)
+  ; GCN:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN6:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 240, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 16 from custom "TargetCustom7", align 1, addrspace 4)
+  ; GCN:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN7:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 240, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 16 from custom "TargetCustom7", align 1, addrspace 4)
+  ; GCN:   INLINEASM &"", 1
+  ; GCN:   [[COPY27:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; GCN:   BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[COPY27]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 256, 0, implicit $exec :: (volatile dereferenceable load store 4 on custom "TargetCustom7" + 256, align 1, addrspace 4)
+  ; GCN:   [[COPY28:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; GCN:   BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[COPY28]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_2]], 128, 0, implicit $exec :: (volatile dereferenceable load store 4 on custom "TargetCustom7" + 256, align 1, addrspace 4)
+  ; GCN:   [[S_MOV_B32_17:%[0-9]+]]:sreg_32 = S_MOV_B32 256
+  ; GCN:   [[COPY29:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; GCN:   BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[COPY29]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_17]], 0, 0, implicit $exec :: (volatile dereferenceable load store 4 on custom "TargetCustom7" + 256, align 1, addrspace 4)
+  ; GCN:   BUFFER_ATOMIC_ADD_BOTHEN [[COPY]], [[REG_SEQUENCE2]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_17]], 0, 0, implicit $exec :: (volatile dereferenceable load store 4 on custom "TargetCustom7", align 1, addrspace 4)
+  ; GCN:   [[COPY30:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; GCN:   [[COPY31:%[0-9]+]]:sreg_32 = COPY [[COPY]]
+  ; GCN:   BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[COPY30]], [[S_LOAD_DWORDX4_IMM]], [[COPY31]], 256, 0, implicit $exec :: (volatile dereferenceable load store 4 on custom "TargetCustom7", align 1, addrspace 4)
+  ; GCN:   BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 256, 0, implicit $exec :: (volatile dereferenceable load store 4 on custom "TargetCustom7", align 1, addrspace 4)
+  ; GCN:   BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 256, 0, implicit $exec :: (volatile dereferenceable load store 4 on custom "TargetCustom7", align 1, addrspace 4)
+  ; GCN:   INLINEASM &"", 1
+  ; GCN:   [[COPY32:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; GCN:   [[DEF9:%[0-9]+]]:vreg_64 = IMPLICIT_DEF
+  ; GCN:   BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE1]], [[COPY32]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 272, 0, implicit $exec :: (volatile dereferenceable load store 4 on custom "TargetCustom7" + 272, align 1, addrspace 4)
+  ; GCN:   [[COPY33:%[0-9]+]]:vgpr_32 = COPY [[DEF9]].sub0
+  ; GCN:   [[S_MOV_B32_18:%[0-9]+]]:sreg_32 = S_MOV_B32 136
+  ; GCN:   [[COPY34:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; GCN:   [[DEF10:%[0-9]+]]:vreg_64 = IMPLICIT_DEF
+  ; GCN:   BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE1]], [[COPY34]], [[S_LOAD_DWORDX4_IMM]], killed [[S_MOV_B32_18]], 136, 0, implicit $exec :: (volatile dereferenceable load store 4 on custom "TargetCustom7" + 272, align 1, addrspace 4)
+  ; GCN:   [[COPY35:%[0-9]+]]:vgpr_32 = COPY [[DEF10]].sub0
+  ; GCN:   [[S_MOV_B32_19:%[0-9]+]]:sreg_32 = S_MOV_B32 272
+  ; GCN:   [[COPY36:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; GCN:   [[DEF11:%[0-9]+]]:vreg_64 = IMPLICIT_DEF
+  ; GCN:   BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE1]], [[COPY36]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_19]], 0, 0, implicit $exec :: (volatile dereferenceable load store 4 on custom "TargetCustom7" + 272, align 1, addrspace 4)
+  ; GCN:   [[COPY37:%[0-9]+]]:vgpr_32 = COPY [[DEF11]].sub0
+  ; GCN:   [[DEF12:%[0-9]+]]:vreg_64 = IMPLICIT_DEF
+  ; GCN:   BUFFER_ATOMIC_CMPSWAP_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE2]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_19]], 0, 0, implicit $exec :: (volatile dereferenceable load store 4 on custom "TargetCustom7", align 1, addrspace 4)
+  ; GCN:   [[COPY38:%[0-9]+]]:vgpr_32 = COPY [[DEF12]].sub0
+  ; GCN:   [[COPY39:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; GCN:   [[COPY40:%[0-9]+]]:sreg_32 = COPY [[COPY]]
+  ; GCN:   [[DEF13:%[0-9]+]]:vreg_64 = IMPLICIT_DEF
+  ; GCN:   BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE1]], [[COPY39]], [[S_LOAD_DWORDX4_IMM]], [[COPY40]], 272, 0, implicit $exec :: (volatile dereferenceable load store 4 on custom "TargetCustom7", align 1, addrspace 4)
+  ; GCN:   [[COPY41:%[0-9]+]]:vgpr_32 = COPY [[DEF13]].sub0
+  ; GCN:   [[DEF14:%[0-9]+]]:vreg_64 = IMPLICIT_DEF
+  ; GCN:   BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE1]], [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 272, 0, implicit $exec :: (volatile dereferenceable load store 4 on custom "TargetCustom7", align 1, addrspace 4)
+  ; GCN:   [[COPY42:%[0-9]+]]:vgpr_32 = COPY [[DEF14]].sub0
+  ; GCN:   [[DEF15:%[0-9]+]]:vreg_64 = IMPLICIT_DEF
+  ; GCN:   BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE1]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 272, 0, implicit $exec :: (volatile dereferenceable load store 4 on custom "TargetCustom7", align 1, addrspace 4)
+  ; GCN:   [[COPY43:%[0-9]+]]:vgpr_32 = COPY [[DEF15]].sub0
+  ; GCN:   INLINEASM &"", 1
+  ; GCN:   [[COPY44:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; GCN:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN2]], [[COPY44]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 288, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 16 into custom "TargetCustom7" + 288, align 1, addrspace 4)
+  ; GCN:   [[COPY45:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; GCN:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN3]], [[COPY45]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_4]], 144, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 16 into custom "TargetCustom7" + 288, align 1, addrspace 4)
+  ; GCN:   [[S_MOV_B32_20:%[0-9]+]]:sreg_32 = S_MOV_B32 288
+  ; GCN:   [[COPY46:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; GCN:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN4]], [[COPY46]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_20]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 16 into custom "TargetCustom7" + 288, align 1, addrspace 4)
+  ; GCN:   BUFFER_STORE_DWORDX4_BOTHEN_exact killed [[BUFFER_LOAD_DWORDX4_BOTHEN]], [[REG_SEQUENCE2]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_20]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 16 into custom "TargetCustom7", align 1, addrspace 4)
+  ; GCN:   [[COPY47:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; GCN:   [[COPY48:%[0-9]+]]:sreg_32 = COPY [[COPY]]
+  ; GCN:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN5]], [[COPY47]], [[S_LOAD_DWORDX4_IMM]], [[COPY48]], 288, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 16 into custom "TargetCustom7", align 1, addrspace 4)
+  ; GCN:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN6]], [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 288, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 16 into custom "TargetCustom7", align 1, addrspace 4)
+  ; GCN:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN7]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 288, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 16 into custom "TargetCustom7", align 1, addrspace 4)
+  ; GCN:   INLINEASM &"", 1
+  ; GCN:   [[COPY49:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; GCN:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN2]], [[COPY49]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 304, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 16 into custom "TargetCustom7" + 304, align 1, addrspace 4)
+  ; GCN:   [[S_MOV_B32_21:%[0-9]+]]:sreg_32 = S_MOV_B32 152
+  ; GCN:   [[COPY50:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; GCN:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN3]], [[COPY50]], [[S_LOAD_DWORDX4_IMM]], killed [[S_MOV_B32_21]], 152, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 16 into custom "TargetCustom7" + 304, align 1, addrspace 4)
+  ; GCN:   [[S_MOV_B32_22:%[0-9]+]]:sreg_32 = S_MOV_B32 304
+  ; GCN:   [[COPY51:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; GCN:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN4]], [[COPY51]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_22]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 16 into custom "TargetCustom7" + 304, align 1, addrspace 4)
+  ; GCN:   BUFFER_STORE_FORMAT_XYZW_BOTHEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_BOTHEN]], [[REG_SEQUENCE2]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_22]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 16 into custom "TargetCustom7", align 1, addrspace 4)
+  ; GCN:   [[COPY52:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; GCN:   [[COPY53:%[0-9]+]]:sreg_32 = COPY [[COPY]]
+  ; GCN:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN5]], [[COPY52]], [[S_LOAD_DWORDX4_IMM]], [[COPY53]], 304, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 16 into custom "TargetCustom7", align 1, addrspace 4)
+  ; GCN:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN6]], [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 304, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 16 into custom "TargetCustom7", align 1, addrspace 4)
+  ; GCN:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN7]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 304, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 16 into custom "TargetCustom7", align 1, addrspace 4)
+  ; GCN:   S_ENDPGM 0
+bb.0:
+  %tmp0 = load <4 x i32>, <4 x i32> addrspace(6)* %arg0, align 16, !invariant.load !0
+  %buffer0 = call nsz <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %tmp0, i32 0, i32 16, i1 false, i1 false) #0
+  %buffer1 = call nsz <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %tmp0, i32 0, i32 %arg1, i1 false, i1 false) #0
+  %buffer2 = call nsz <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %tmp0, i32 1, i32 16, i1 false, i1 false) #0
+  %buffer3 = call nsz <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %tmp0, i32 %arg1, i32 16, i1 false, i1 false) #0
+
+  ; Insert inline asm to keep the different instruction types from being mixed.  This makes the output easier to read.
+  call void asm sideeffect "", "" ()
+
+  call void @llvm.amdgcn.buffer.store.v4f32(<4 x float> %buffer0, <4 x i32> %tmp0, i32 0, i32 32, i1 false, i1 false) #1
+  call void @llvm.amdgcn.buffer.store.v4f32(<4 x float> %buffer1, <4 x i32> %tmp0, i32 0, i32 %arg1, i1 false, i1 false) #1
+  call void @llvm.amdgcn.buffer.store.v4f32(<4 x float> %buffer2, <4 x i32> %tmp0, i32 1, i32 32, i1 false, i1 false) #1
+  call void @llvm.amdgcn.buffer.store.v4f32(<4 x float> %buffer3, <4 x i32> %tmp0, i32 %arg1, i32 32, i1 false, i1 false) #1
+
+  call void asm sideeffect "", "" ()
+
+  %buffer_format0 = call nsz <4 x float> @llvm.amdgcn.buffer.load.format.v4f32(<4 x i32> %tmp0, i32 0, i32 48, i1 false, i1 false) #0
+  %buffer_format1 = call nsz <4 x float> @llvm.amdgcn.buffer.load.format.v4f32(<4 x i32> %tmp0, i32 0, i32 %arg1, i1 false, i1 false) #0
+  %buffer_format2 = call nsz <4 x float> @llvm.amdgcn.buffer.load.format.v4f32(<4 x i32> %tmp0, i32 1, i32 48, i1 false, i1 false) #0
+  %buffer_format3 = call nsz <4 x float> @llvm.amdgcn.buffer.load.format.v4f32(<4 x i32> %tmp0, i32 %arg1, i32 48, i1 false, i1 false) #0
+
+  call void asm sideeffect "", "" ()
+
+  call void @llvm.amdgcn.buffer.store.format.v4f32(<4 x float> %buffer_format0, <4 x i32> %tmp0, i32 0, i32 64, i1 false, i1 false) #1
+  call void @llvm.amdgcn.buffer.store.format.v4f32(<4 x float> %buffer_format1, <4 x i32> %tmp0, i32 0, i32 %arg1, i1 false, i1 false) #1
+  call void @llvm.amdgcn.buffer.store.format.v4f32(<4 x float> %buffer_format2, <4 x i32> %tmp0, i32 1, i32 64, i1 false, i1 false) #1
+  call void @llvm.amdgcn.buffer.store.format.v4f32(<4 x float> %buffer_format3, <4 x i32> %tmp0, i32 %arg1, i32 64, i1 false, i1 false) #1
+
+  call void asm sideeffect "", "" ()
+
+  %atomic_add0 = call i32 @llvm.amdgcn.buffer.atomic.add.i32(i32 %arg1, <4 x i32> %tmp0, i32 0, i32 80, i1 false) #2
+  %atomic_add1 = call i32 @llvm.amdgcn.buffer.atomic.add.i32(i32 %arg1, <4 x i32> %tmp0, i32 0, i32 %arg1, i1 false) #2
+  %atomic_add2 = call i32 @llvm.amdgcn.buffer.atomic.add.i32(i32 %arg1, <4 x i32> %tmp0, i32 1, i32 80, i1 false) #2
+  %atomic_add3 = call i32 @llvm.amdgcn.buffer.atomic.add.i32(i32 %arg1, <4 x i32> %tmp0, i32 %arg1, i32 80, i1 false) #2
+
+  call void asm sideeffect "", "" ()
+
+  %atomic_cmpswap0 = call i32 @llvm.amdgcn.buffer.atomic.cmpswap(i32 %arg1, i32 %arg1, <4 x i32> %tmp0, i32 0, i32 96, i1 false) #2
+  %atomic_cmpswap1 = call i32 @llvm.amdgcn.buffer.atomic.cmpswap(i32 %arg1, i32 %arg1, <4 x i32> %tmp0, i32 0, i32 %arg1, i1 false) #2
+  %atomic_cmpswap2 = call i32 @llvm.amdgcn.buffer.atomic.cmpswap(i32 %arg1, i32 %arg1, <4 x i32> %tmp0, i32 1, i32 96, i1 false) #2
+  %atomic_cmpswap3 = call i32 @llvm.amdgcn.buffer.atomic.cmpswap(i32 %arg1, i32 %arg1, <4 x i32> %tmp0, i32 %arg1, i32 96, i1 false) #2
+
+  call void asm sideeffect "", "" ()
+
+  call void @llvm.amdgcn.buffer.atomic.fadd.f32(float 1.0, <4 x i32> %tmp0, i32 0, i32 112, i1 false) #2
+  call void @llvm.amdgcn.buffer.atomic.fadd.f32(float 1.0, <4 x i32> %tmp0, i32 0, i32 %arg1, i1 false) #2
+  call void @llvm.amdgcn.buffer.atomic.fadd.f32(float 1.0, <4 x i32> %tmp0, i32 1, i32 112, i1 false) #2
+  call void @llvm.amdgcn.buffer.atomic.fadd.f32(float 1.0, <4 x i32> %tmp0, i32 %arg1, i32 112, i1 false) #2
+
+  call void asm sideeffect "", "" ()
+
+  ; rsrc, offset, soffset, cachepolicy
+  %raw_buffer0 = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %tmp0, i32 128, i32 0, i32 0) #0
+  %raw_buffer1 = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %tmp0, i32 64, i32 64, i32 0) #0
+  %raw_buffer2 = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %tmp0, i32 0, i32 128, i32 0) #0
+  %raw_buffer3 = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %tmp0, i32 %arg1, i32 128, i32 0) #0
+  %raw_buffer4 = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %tmp0, i32 128, i32 %arg1, i32 0) #0
+
+  call void asm sideeffect "", "" ()
+
+  %raw_buffer_format0 = call <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32> %tmp0, i32 144, i32 0, i32 0) #0
+  %raw_buffer_format1 = call <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32> %tmp0, i32 72, i32 72, i32 0) #0
+  %raw_buffer_format2 = call <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32> %tmp0, i32 0, i32 144, i32 0) #0
+  %raw_buffer_format3 = call <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32> %tmp0, i32 %arg1, i32 144, i32 0) #0
+  %raw_buffer_format4 = call <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32> %tmp0, i32 144, i32 %arg1, i32 0) #0
+
+  call void asm sideeffect "", "" ()
+
+  %raw_atomic_add0 = call i32 @llvm.amdgcn.raw.buffer.atomic.add.i32(i32 %arg1, <4 x i32> %tmp0, i32 160, i32 0, i32 0) #2
+  %raw_atomic_add1 = call i32 @llvm.amdgcn.raw.buffer.atomic.add.i32(i32 %arg1, <4 x i32> %tmp0, i32 80, i32 80, i32 0) #2
+  %raw_atomic_add2 = call i32 @llvm.amdgcn.raw.buffer.atomic.add.i32(i32 %arg1, <4 x i32> %tmp0, i32 0, i32 160, i32 0) #2
+  %raw_atomic_add3 = call i32 @llvm.amdgcn.raw.buffer.atomic.add.i32(i32 %arg1, <4 x i32> %tmp0, i32 %arg1, i32 160, i32 0) #2
+  %raw_atomic_add4 = call i32 @llvm.amdgcn.raw.buffer.atomic.add.i32(i32 %arg1, <4 x i32> %tmp0, i32 160, i32 %arg1, i32 0) #2
+
+  call void asm sideeffect "", "" ()
+
+  %raw_atomic_cmpswap0 = call i32 @llvm.amdgcn.raw.buffer.atomic.cmpswap.i32(i32 %arg1, i32 %arg1, <4 x i32> %tmp0, i32 176, i32 0, i32 0) #2
+  %raw_atomic_cmpswap1 = call i32 @llvm.amdgcn.raw.buffer.atomic.cmpswap.i32(i32 %arg1, i32 %arg1, <4 x i32> %tmp0, i32 88, i32 88, i32 0) #2
+  %raw_atomic_cmpswap2 = call i32 @llvm.amdgcn.raw.buffer.atomic.cmpswap.i32(i32 %arg1, i32 %arg1, <4 x i32> %tmp0, i32 0, i32 176, i32 0) #2
+  %raw_atomic_cmpswap3 = call i32 @llvm.amdgcn.raw.buffer.atomic.cmpswap.i32(i32 %arg1, i32 %arg1, <4 x i32> %tmp0, i32 %arg1, i32 176, i32 0) #2
+  %raw_atomic_cmpswap4 = call i32 @llvm.amdgcn.raw.buffer.atomic.cmpswap.i32(i32 %arg1, i32 %arg1, <4 x i32> %tmp0, i32 176, i32 %arg1, i32 0) #2
+
+  call void asm sideeffect "", "" ()
+
+  call void @llvm.amdgcn.raw.buffer.store.v4f32(<4 x float> %raw_buffer0, <4 x i32> %tmp0, i32 192, i32 0, i32 0) #2
+  call void @llvm.amdgcn.raw.buffer.store.v4f32(<4 x float> %raw_buffer1, <4 x i32> %tmp0, i32 96, i32 96, i32 0) #2
+  call void @llvm.amdgcn.raw.buffer.store.v4f32(<4 x float> %raw_buffer2, <4 x i32> %tmp0, i32 0, i32 192, i32 0) #2
+  call void @llvm.amdgcn.raw.buffer.store.v4f32(<4 x float> %raw_buffer3, <4 x i32> %tmp0, i32 %arg1, i32 192, i32 0) #2
+  call void @llvm.amdgcn.raw.buffer.store.v4f32(<4 x float> %raw_buffer4, <4 x i32> %tmp0, i32 192, i32 %arg1, i32 0) #2
+
+  call void asm sideeffect "", "" ()
+
+  call void @llvm.amdgcn.raw.buffer.store.format.v4f32(<4 x float> %raw_buffer_format0, <4 x i32> %tmp0, i32 208, i32 0, i32 0) #2
+  call void @llvm.amdgcn.raw.buffer.store.format.v4f32(<4 x float> %raw_buffer_format1, <4 x i32> %tmp0, i32 104, i32 104, i32 0) #2
+  call void @llvm.amdgcn.raw.buffer.store.format.v4f32(<4 x float> %raw_buffer_format2, <4 x i32> %tmp0, i32 0, i32 208, i32 0) #2
+  call void @llvm.amdgcn.raw.buffer.store.format.v4f32(<4 x float> %raw_buffer_format3, <4 x i32> %tmp0, i32 %arg1, i32 208, i32 0) #2
+  call void @llvm.amdgcn.raw.buffer.store.format.v4f32(<4 x float> %raw_buffer_format4, <4 x i32> %tmp0, i32 208, i32 %arg1, i32 0) #2
+
+  call void asm sideeffect "", "" ()
+
+  ; rsrc, vindex, offset, soffset, cachepolicy
+  %struct_buffer0 = call <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32> %tmp0, i32 0, i32 224, i32 0, i32 0) #0
+  %struct_buffer1 = call <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32> %tmp0, i32 0, i32 112, i32 112, i32 0) #0
+  %struct_buffer2 = call <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32> %tmp0, i32 0, i32 0, i32 224, i32 0) #0
+  %struct_buffer3 = call <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32> %tmp0, i32 0, i32 %arg1, i32 224, i32 0) #0
+  %struct_buffer4 = call <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32> %tmp0, i32 0, i32 224, i32 %arg1, i32 0) #0
+  %struct_buffer5 = call <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32> %tmp0, i32 1, i32 224, i32 0, i32 0) #0
+  %struct_buffer6 = call <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32> %tmp0, i32 %arg1, i32 224, i32 0, i32 0) #0
+
+  call void asm sideeffect "", "" ()
+
+  %struct_buffer_format0 = call <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32> %tmp0, i32 0, i32 240, i32 0, i32 0) #0
+  %struct_buffer_format1 = call <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32> %tmp0, i32 0, i32 120, i32 120, i32 0) #0
+  %struct_buffer_format2 = call <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32> %tmp0, i32 0, i32 0, i32 240, i32 0) #0
+  %struct_buffer_format3 = call <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32> %tmp0, i32 0, i32 %arg1, i32 240, i32 0) #0
+  %struct_buffer_format4 = call <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32> %tmp0, i32 0, i32 240, i32 %arg1, i32 0) #0
+  %struct_buffer_format5 = call <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32> %tmp0, i32 1, i32 240, i32 0, i32 0) #0
+  %struct_buffer_format6 = call <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32> %tmp0, i32 %arg1, i32 240, i32 0, i32 0) #0
+
+  call void asm sideeffect "", "" ()
+
+  %struct_atomic_add0 = call i32 @llvm.amdgcn.struct.buffer.atomic.add.i32(i32 %arg1, <4 x i32> %tmp0, i32 0, i32 256, i32 0, i32 0) #2
+  %struct_atomic_add1 = call i32 @llvm.amdgcn.struct.buffer.atomic.add.i32(i32 %arg1, <4 x i32> %tmp0, i32 0, i32 128, i32 128, i32 0) #2
+  %struct_atomic_add2 = call i32 @llvm.amdgcn.struct.buffer.atomic.add.i32(i32 %arg1, <4 x i32> %tmp0, i32 0, i32 0, i32 256, i32 0) #2
+  %struct_atomic_add3 = call i32 @llvm.amdgcn.struct.buffer.atomic.add.i32(i32 %arg1, <4 x i32> %tmp0, i32 0, i32 %arg1, i32 256, i32 0) #2
+  %struct_atomic_add4 = call i32 @llvm.amdgcn.struct.buffer.atomic.add.i32(i32 %arg1, <4 x i32> %tmp0, i32 0, i32 256, i32 %arg1, i32 0) #2
+  %struct_atomic_add5 = call i32 @llvm.amdgcn.struct.buffer.atomic.add.i32(i32 %arg1, <4 x i32> %tmp0, i32 1, i32 256, i32 0, i32 0) #2
+  %struct_atomic_add6 = call i32 @llvm.amdgcn.struct.buffer.atomic.add.i32(i32 %arg1, <4 x i32> %tmp0, i32 %arg1, i32 256, i32 0, i32 0) #2
+
+  call void asm sideeffect "", "" ()
+
+  %struct_atomic_cmpswap0 = call i32 @llvm.amdgcn.struct.buffer.atomic.cmpswap.i32(i32 %arg1, i32 %arg1, <4 x i32> %tmp0, i32 0, i32 272, i32 0, i32 0) #2
+  %struct_atomic_cmpswap1 = call i32 @llvm.amdgcn.struct.buffer.atomic.cmpswap.i32(i32 %arg1, i32 %arg1, <4 x i32> %tmp0, i32 0, i32 136, i32 136, i32 0) #2
+  %struct_atomic_cmpswap2 = call i32 @llvm.amdgcn.struct.buffer.atomic.cmpswap.i32(i32 %arg1, i32 %arg1, <4 x i32> %tmp0, i32 0, i32 0, i32 272, i32 0) #2
+  %struct_atomic_cmpswap3 = call i32 @llvm.amdgcn.struct.buffer.atomic.cmpswap.i32(i32 %arg1, i32 %arg1, <4 x i32> %tmp0, i32 0, i32 %arg1, i32 272, i32 0) #2
+  %struct_atomic_cmpswap4 = call i32 @llvm.amdgcn.struct.buffer.atomic.cmpswap.i32(i32 %arg1, i32 %arg1, <4 x i32> %tmp0, i32 0, i32 272, i32 %arg1, i32 0) #2
+  %struct_atomic_cmpswap5 = call i32 @llvm.amdgcn.struct.buffer.atomic.cmpswap.i32(i32 %arg1, i32 %arg1, <4 x i32> %tmp0, i32 1, i32 272, i32 0, i32 0) #2
+  %struct_atomic_cmpswap6 = call i32 @llvm.amdgcn.struct.buffer.atomic.cmpswap.i32(i32 %arg1, i32 %arg1, <4 x i32> %tmp0, i32 %arg1, i32 272, i32 0, i32 0) #2
+
+  call void asm sideeffect "", "" ()
+
+  call void @llvm.amdgcn.struct.buffer.store.v4f32(<4 x float> %struct_buffer0, <4 x i32> %tmp0, i32 0, i32 288, i32 0, i32 0) #2
+  call void @llvm.amdgcn.struct.buffer.store.v4f32(<4 x float> %struct_buffer1, <4 x i32> %tmp0, i32 0, i32 144, i32 144, i32 0) #2
+  call void @llvm.amdgcn.struct.buffer.store.v4f32(<4 x float> %struct_buffer2, <4 x i32> %tmp0, i32 0, i32 0, i32 288, i32 0) #2
+  call void @llvm.amdgcn.struct.buffer.store.v4f32(<4 x float> %struct_buffer3, <4 x i32> %tmp0, i32 0, i32 %arg1, i32 288, i32 0) #2
+  call void @llvm.amdgcn.struct.buffer.store.v4f32(<4 x float> %struct_buffer4, <4 x i32> %tmp0, i32 0, i32 288, i32 %arg1, i32 0) #2
+  call void @llvm.amdgcn.struct.buffer.store.v4f32(<4 x float> %struct_buffer5, <4 x i32> %tmp0, i32 1, i32 288, i32 0, i32 0) #2
+  call void @llvm.amdgcn.struct.buffer.store.v4f32(<4 x float> %struct_buffer6, <4 x i32> %tmp0, i32 %arg1, i32 288, i32 0, i32 0) #2
+
+  call void asm sideeffect "", "" ()
+
+  call void @llvm.amdgcn.struct.buffer.store.format.v4f32(<4 x float> %struct_buffer_format0, <4 x i32> %tmp0, i32 0, i32 304, i32 0, i32 0) #2
+  call void @llvm.amdgcn.struct.buffer.store.format.v4f32(<4 x float> %struct_buffer_format1, <4 x i32> %tmp0, i32 0, i32 152, i32 152, i32 0) #2
+  call void @llvm.amdgcn.struct.buffer.store.format.v4f32(<4 x float> %struct_buffer_format2, <4 x i32> %tmp0, i32 0, i32 0, i32 304, i32 0) #2
+  call void @llvm.amdgcn.struct.buffer.store.format.v4f32(<4 x float> %struct_buffer_format3, <4 x i32> %tmp0, i32 0, i32 %arg1, i32 304, i32 0) #2
+  call void @llvm.amdgcn.struct.buffer.store.format.v4f32(<4 x float> %struct_buffer_format4, <4 x i32> %tmp0, i32 0, i32 304, i32 %arg1, i32 0) #2
+  call void @llvm.amdgcn.struct.buffer.store.format.v4f32(<4 x float> %struct_buffer_format5, <4 x i32> %tmp0, i32 1, i32 304, i32 0, i32 0) #2
+  call void @llvm.amdgcn.struct.buffer.store.format.v4f32(<4 x float> %struct_buffer_format6, <4 x i32> %tmp0, i32 %arg1, i32 304, i32 0, i32 0) #2
+
+  ret void
+}
+
+declare <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32>, i32, i32, i1, i1) #0
+declare void @llvm.amdgcn.buffer.store.v4f32(<4 x float>, <4 x i32>, i32, i32, i1, i1) #1
+declare <4 x float> @llvm.amdgcn.buffer.load.format.v4f32(<4 x i32>, i32, i32, i1, i1) #0
+declare void @llvm.amdgcn.buffer.store.format.v4f32(<4 x float>, <4 x i32>, i32, i32, i1, i1) #1
+declare i32 @llvm.amdgcn.buffer.atomic.add.i32(i32, <4 x i32>, i32, i32, i1) #2
+declare i32 @llvm.amdgcn.buffer.atomic.cmpswap(i32, i32, <4 x i32>, i32, i32, i1) #2
+declare void @llvm.amdgcn.buffer.atomic.fadd.f32(float, <4 x i32>, i32, i32, i1) #2
+declare <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32>, i32, i32, i32) #0
+declare <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32>, i32, i32, i32) #0
+declare i32 @llvm.amdgcn.raw.buffer.atomic.add.i32(i32, <4 x i32>, i32, i32, i32) #2
+declare i32 @llvm.amdgcn.raw.buffer.atomic.cmpswap.i32(i32, i32, <4 x i32>, i32, i32, i32) #2
+declare void @llvm.amdgcn.raw.buffer.store.v4f32(<4 x float>, <4 x i32>, i32, i32, i32) #2
+declare void @llvm.amdgcn.raw.buffer.store.format.v4f32(<4 x float>, <4 x i32>, i32, i32, i32) #2
+
+declare <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32>, i32, i32, i32, i32) #0
+declare <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32>, i32, i32, i32, i32) #0
+declare i32 @llvm.amdgcn.struct.buffer.atomic.add.i32(i32, <4 x i32>, i32, i32, i32, i32) #2
+declare i32 @llvm.amdgcn.struct.buffer.atomic.cmpswap.i32(i32, i32, <4 x i32>, i32, i32, i32, i32) #2
+declare void @llvm.amdgcn.struct.buffer.store.v4f32(<4 x float>, <4 x i32>, i32, i32, i32, i32) #2
+declare void @llvm.amdgcn.struct.buffer.store.format.v4f32(<4 x float>, <4 x i32>, i32, i32, i32, i32) #2
+
+attributes #0 = { nounwind readonly }
+attributes #1 = { nounwind writeonly }
+attributes #2 = { nounwind }
+
+!0 = !{}
diff --git a/llvm/test/CodeGen/AMDGPU/byval-frame-setup.ll b/llvm/test/CodeGen/AMDGPU/byval-frame-setup.ll
index 2ab3327d5..42192289a 100644
--- a/llvm/test/CodeGen/AMDGPU/byval-frame-setup.ll
+++ b/llvm/test/CodeGen/AMDGPU/byval-frame-setup.ll
@@ -263,10 +263,10 @@ entry:
 
 
 ; GCN-NOT: s_add_u32 s32, s32, 0x800
+; GCN-DAG: s_add_u32 s32, s33, 0xc00{{$}}
 
 ; GCN: buffer_load_dword [[LOAD0:v[0-9]+]], off, s[0:3], s33 offset:8
 ; GCN: buffer_load_dword [[LOAD1:v[0-9]+]], off, s[0:3], s33 offset:12
-; GCN-DAG: s_add_u32 s32, s33, 0xc00{{$}}
 ; GCN: buffer_load_dword [[LOAD2:v[0-9]+]], off, s[0:3], s33 offset:16
 ; GCN: buffer_load_dword [[LOAD3:v[0-9]+]], off, s[0:3], s33 offset:20
 
@@ -335,6 +335,7 @@ entry:
 ; GCN: buffer_load_dword [[LOAD5:v[0-9]+]], off, s[0:3], s34 offset:20
 ; GCN: buffer_load_dword [[LOAD6:v[0-9]+]], off, s[0:3], s34 offset:24
 ; GCN: buffer_load_dword [[LOAD7:v[0-9]+]], off, s[0:3], s34 offset:28
+
 ; GCN: s_waitcnt vmcnt(0)
 ; GCN-DAG: buffer_store_dword [[LOAD4]], off, s[0:3], s32 offset:16
 ; GCN-DAG: buffer_store_dword [[LOAD5]], off, s[0:3], s32 offset:20
diff --git a/llvm/test/CodeGen/AMDGPU/call-argument-types.ll b/llvm/test/CodeGen/AMDGPU/call-argument-types.ll
index 0e07a3419..a627bf004 100644
--- a/llvm/test/CodeGen/AMDGPU/call-argument-types.ll
+++ b/llvm/test/CodeGen/AMDGPU/call-argument-types.ll
@@ -250,13 +250,11 @@ define amdgpu_kernel void @test_call_external_void_func_i32_imm(i32) #0 {
 }
 
 ; GCN-LABEL: {{^}}test_call_external_void_func_i64_imm:
-; GCN-DAG: s_movk_i32 [[K0:s[0-9]+]], 0x7b{{$}}
-; GCN-DAG: s_mov_b32 [[K1:s[0-9]+]], 0{{$}}
-; GCN-DAG: v_mov_b32_e32 v0, [[K0]]
+; GCN-DAG: v_mov_b32_e32 v0, 0x7b{{$}}
+; GCN-DAG: v_mov_b32_e32 v1, 0{{$}}
 ; GCN-DAG: s_getpc_b64 s{{\[}}[[PC_LO:[0-9]+]]:[[PC_HI:[0-9]+]]{{\]}}
 ; GCN-DAG: s_add_u32 s[[PC_LO]], s[[PC_LO]], external_void_func_i64@rel32@lo+4
 ; GCN-DAG: s_addc_u32 s[[PC_HI]], s[[PC_HI]], external_void_func_i64@rel32@hi+4
-; GCN-DAG: v_mov_b32_e32 v1, [[K1]]
 ; GCN: s_swappc_b64 s[30:31], s{{\[}}[[PC_LO]]:[[PC_HI]]{{\]}}
 ; GCN-NEXT: s_endpgm
 define amdgpu_kernel void @test_call_external_void_func_i64_imm() #0 {
@@ -756,8 +754,8 @@ define amdgpu_kernel void @test_call_external_void_func_v16i8() #0 {
 }
 
 ; GCN-LABEL: {{^}}stack_passed_arg_alignment_v32i32_f64:
-; GCN: buffer_store_dword v{{[0-9]+}}, off, s{{\[[0-9]+:[0-9]+\]}}, s32 offset:4
 ; GCN: buffer_store_dword v{{[0-9]+}}, off, s{{\[[0-9]+:[0-9]+\]}}, s32{{$}}
+; GCN: buffer_store_dword v{{[0-9]+}}, off, s{{\[[0-9]+:[0-9]+\]}}, s32 offset:4
 ; GCN: s_swappc_b64
 define amdgpu_kernel void @stack_passed_arg_alignment_v32i32_f64(<32 x i32> %val, double %tmp) #0 {
 entry:
@@ -769,13 +767,13 @@ entry:
 ; GCN-NOT: s32
 ; GCN: buffer_store_dword v32, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
 ; GCN: buffer_store_dword v33, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
-; GCN: buffer_load_dword v32, off, s[0:3], s32 offset:16
-; GCN: buffer_load_dword v33, off, s[0:3], s32 offset:20
+; GCN: buffer_load_dword v32, off, s[0:3], s32 offset:20
+; GCN: buffer_load_dword v33, off, s[0:3], s32 offset:16
 
 ; GCN: s_getpc_b64
 
-; GCN: buffer_store_dword v33, off, s[0:3], s32 offset:4
-; GCN: buffer_store_dword v32, off, s[0:3], s32{{$}}
+; GCN: buffer_store_dword v32, off, s[0:3], s32 offset:4
+; GCN: buffer_store_dword v33, off, s[0:3], s32{{$}}
 ; GCN: buffer_load_dword v33, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
 ; GCN: buffer_load_dword v32, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
 ; GCN-NOT: s32
@@ -791,11 +789,11 @@ entry:
 ; GCN-NOT: s32
 ; GCN: buffer_store_dword v32, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
 ; GCN: buffer_store_dword v33, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
-; GCN: buffer_load_dword v32, off, s[0:3], s32{{$}}
-; GCN: buffer_load_dword v33, off, s[0:3], s32 offset:4
+; GCN: buffer_load_dword v32, off, s[0:3], s32 offset:4
+; GCN: buffer_load_dword v33, off, s[0:3], s32{{$}}
 ; GCN: s_getpc_b64
-; GCN: buffer_store_dword v32, off, s[0:3], s32{{$}}
-; GCN: buffer_store_dword v33, off, s[0:3], s32 offset:4
+; GCN: buffer_store_dword v33, off, s[0:3], s32{{$}}
+; GCN: buffer_store_dword v32, off, s[0:3], s32 offset:4
 ; GCN: buffer_load_dword v33, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
 ; GCN: buffer_load_dword v32, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
 ; GCN-NOT: s32
@@ -808,12 +806,12 @@ entry:
 
 ; GCN-LABEL: {{^}}stack_12xv3i32:
 ; GCN: v_mov_b32_e32 [[REG12:v[0-9]+]], 12
-; GCN: buffer_store_dword [[REG12]], {{.*$}}
 ; GCN: v_mov_b32_e32 [[REG13:v[0-9]+]], 13
-; GCN: buffer_store_dword [[REG13]], {{.*}} offset:4
 ; GCN: v_mov_b32_e32 [[REG14:v[0-9]+]], 14
-; GCN: buffer_store_dword [[REG14]], {{.*}} offset:8
 ; GCN: v_mov_b32_e32 [[REG15:v[0-9]+]], 15
+; GCN: buffer_store_dword [[REG12]], {{.*$}}
+; GCN: buffer_store_dword [[REG13]], {{.*}} offset:4
+; GCN: buffer_store_dword [[REG14]], {{.*}} offset:8
 ; GCN: buffer_store_dword [[REG15]], {{.*}} offset:12
 ; GCN: v_mov_b32_e32 v31, 11
 ; GCN: s_getpc
@@ -837,12 +835,12 @@ entry:
 
 ; GCN-LABEL: {{^}}stack_12xv3f32:
 ; GCN: v_mov_b32_e32 [[REG12:v[0-9]+]], 0x41400000
-; GCN: buffer_store_dword [[REG12]], {{.*$}}
 ; GCN: v_mov_b32_e32 [[REG13:v[0-9]+]], 0x41500000
-; GCN: buffer_store_dword [[REG13]], {{.*}} offset:4
 ; GCN: v_mov_b32_e32 [[REG14:v[0-9]+]], 0x41600000
-; GCN: buffer_store_dword [[REG14]], {{.*}} offset:8
 ; GCN: v_mov_b32_e32 [[REG15:v[0-9]+]], 0x41700000
+; GCN: buffer_store_dword [[REG12]], {{.*$}}
+; GCN: buffer_store_dword [[REG13]], {{.*}} offset:4
+; GCN: buffer_store_dword [[REG14]], {{.*}} offset:8
 ; GCN: buffer_store_dword [[REG15]], {{.*}} offset:12
 ; GCN: v_mov_b32_e32 v31, 0x41300000
 ; GCN: s_getpc
@@ -867,20 +865,20 @@ entry:
 ; GCN-LABEL: {{^}}stack_8xv5i32:
 
 ; GCN: v_mov_b32_e32 [[REG8:v[0-9]+]], 8
-; GCN: buffer_store_dword [[REG8]], {{.*$}}
 ; GCN: v_mov_b32_e32 [[REG9:v[0-9]+]], 9
-; GCN: buffer_store_dword [[REG9]], {{.*}} offset:4
 ; GCN: v_mov_b32_e32 [[REG10:v[0-9]+]], 10
-; GCN: buffer_store_dword [[REG10]], {{.*}} offset:8
 ; GCN: v_mov_b32_e32 [[REG11:v[0-9]+]], 11
-; GCN: buffer_store_dword [[REG11]], {{.*}} offset:12
 ; GCN: v_mov_b32_e32 [[REG12:v[0-9]+]], 12
+; GCN: buffer_store_dword [[REG8]], {{.*$}}
+; GCN: buffer_store_dword [[REG9]], {{.*}} offset:4
+; GCN: buffer_store_dword [[REG10]], {{.*}} offset:8
+; GCN: buffer_store_dword [[REG11]], {{.*}} offset:12
 ; GCN: buffer_store_dword [[REG12]], {{.*}} offset:16
 ; GCN: v_mov_b32_e32 [[REG13:v[0-9]+]], 13
-; GCN: buffer_store_dword [[REG13]], {{.*}} offset:20
 ; GCN: v_mov_b32_e32 [[REG14:v[0-9]+]], 14
-; GCN: buffer_store_dword [[REG14]], {{.*}} offset:24
 ; GCN: v_mov_b32_e32 [[REG15:v[0-9]+]], 15
+; GCN: buffer_store_dword [[REG13]], {{.*}} offset:20
+; GCN: buffer_store_dword [[REG14]], {{.*}} offset:24
 ; GCN: buffer_store_dword [[REG15]], {{.*}} offset:28
 
 ; GCN: v_mov_b32_e32 v31, 7
@@ -901,20 +899,20 @@ entry:
 
 ; GCN-LABEL: {{^}}stack_8xv5f32:
 ; GCN: v_mov_b32_e32 [[REG8:v[0-9]+]], 0x41000000
-; GCN: buffer_store_dword [[REG8]], {{.*$}}
 ; GCN: v_mov_b32_e32 [[REG9:v[0-9]+]], 0x41100000
-; GCN: buffer_store_dword [[REG9]], {{.*}} offset:4
 ; GCN: v_mov_b32_e32 [[REG10:v[0-9]+]], 0x41200000
-; GCN: buffer_store_dword [[REG10]], {{.*}} offset:8
 ; GCN: v_mov_b32_e32 [[REG11:v[0-9]+]], 0x41300000
-; GCN: buffer_store_dword [[REG11]], {{.*}} offset:12
 ; GCN: v_mov_b32_e32 [[REG12:v[0-9]+]], 0x41400000
+; GCN: buffer_store_dword [[REG8]], {{.*$}}
+; GCN: buffer_store_dword [[REG9]], {{.*}} offset:4
+; GCN: buffer_store_dword [[REG10]], {{.*}} offset:8
+; GCN: buffer_store_dword [[REG11]], {{.*}} offset:12
 ; GCN: buffer_store_dword [[REG12]], {{.*}} offset:16
 ; GCN: v_mov_b32_e32 [[REG13:v[0-9]+]], 0x41500000
-; GCN: buffer_store_dword [[REG13]], {{.*}} offset:20
 ; GCN: v_mov_b32_e32 [[REG14:v[0-9]+]], 0x41600000
-; GCN: buffer_store_dword [[REG14]], {{.*}} offset:24
 ; GCN: v_mov_b32_e32 [[REG15:v[0-9]+]], 0x41700000
+; GCN: buffer_store_dword [[REG13]], {{.*}} offset:20
+; GCN: buffer_store_dword [[REG14]], {{.*}} offset:24
 ; GCN: buffer_store_dword [[REG15]], {{.*}} offset:28
 
 ; GCN: v_mov_b32_e32 v31, 0x40e00000
diff --git a/llvm/test/CodeGen/AMDGPU/call-constant.ll b/llvm/test/CodeGen/AMDGPU/call-constant.ll
new file mode 100644
index 000000000..19aadfc96
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/call-constant.ll
@@ -0,0 +1,45 @@
+; RUN: llc -mtriple=amdgcn-amd-amdhsa < %s | FileCheck -check-prefix=GCN %s
+
+; FIXME: Emitting unnecessary flat_scratch setup
+
+; GCN-LABEL: {{^}}test_call_undef:
+; GCN: s_mov_b32 s8, s7
+; GCN: s_mov_b32 flat_scratch_lo, s5
+; GCN: s_add_u32 s4, s4, s8
+; GCN: s_lshr_b32
+; GCN: s_endpgm
+define amdgpu_kernel void @test_call_undef() #0 {
+  %val = call i32 undef(i32 1)
+  %op = add i32 %val, 1
+  store volatile i32 %op, i32 addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_tail_call_undef:
+; GCN: s_waitcnt
+; GCN-NEXT: .Lfunc_end
+define i32 @test_tail_call_undef() #0 {
+  %call = tail call i32 undef(i32 1)
+  ret i32 %call
+}
+
+; GCN-LABEL: {{^}}test_call_null:
+; GCN: s_mov_b32 s8, s7
+; GCN: s_mov_b32 flat_scratch_lo, s5
+; GCN: s_add_u32 s4, s4, s8
+; GCN: s_lshr_b32
+; GCN: s_endpgm
+define amdgpu_kernel void @test_call_null() #0 {
+  %val = call i32 null(i32 1)
+  %op = add i32 %val, 1
+  store volatile i32 %op, i32 addrspace(1)* null
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_tail_call_null:
+; GCN: s_waitcnt
+; GCN-NEXT: .Lfunc_end
+define i32 @test_tail_call_null() #0 {
+  %call = tail call i32 null(i32 1)
+  ret i32 %call
+}
diff --git a/llvm/test/CodeGen/AMDGPU/call-graph-register-usage.ll b/llvm/test/CodeGen/AMDGPU/call-graph-register-usage.ll
index bf17c6687..de0086495 100644
--- a/llvm/test/CodeGen/AMDGPU/call-graph-register-usage.ll
+++ b/llvm/test/CodeGen/AMDGPU/call-graph-register-usage.ll
@@ -238,6 +238,34 @@ entry:
   ret void
 }
 
+; Make sure there's no assert when a sgpr160 is used.
+; GCN-LABEL: {{^}}count_use_sgpr160_external_call
+; GCN: ; sgpr160 s[{{[0-9]+}}:{{[0-9]+}}]
+; CI: NumSgprs: 48
+; VI-NOBUG: NumSgprs: 48
+; VI-BUG: NumSgprs: 96
+; GCN: NumVgprs: 24
+define amdgpu_kernel void @count_use_sgpr160_external_call()  {
+entry:
+  tail call void asm sideeffect "; sgpr160 $0", "s"(<5 x i32> <i32 10, i32 11, i32 12, i32 13, i32 14>) #1
+  call void @external()
+  ret void
+}
+
+; Make sure there's no assert when a vgpr160 is used.
+; GCN-LABEL: {{^}}count_use_vgpr160_external_call
+; GCN: ; vgpr160 v[{{[0-9]+}}:{{[0-9]+}}]
+; CI: NumSgprs: 48
+; VI-NOBUG: NumSgprs: 48
+; VI-BUG: NumSgprs: 96
+; GCN: NumVgprs: 24
+define amdgpu_kernel void @count_use_vgpr160_external_call()  {
+entry:
+  tail call void asm sideeffect "; vgpr160 $0", "v"(<5 x i32> <i32 10, i32 11, i32 12, i32 13, i32 14>) #1
+  call void @external()
+  ret void
+}
+
 attributes #0 = { nounwind noinline norecurse }
 attributes #1 = { nounwind noinline norecurse }
 attributes #2 = { nounwind noinline }
diff --git a/llvm/test/CodeGen/AMDGPU/callee-special-input-vgprs.ll b/llvm/test/CodeGen/AMDGPU/callee-special-input-vgprs.ll
index 123ae561e..61528cf50 100644
--- a/llvm/test/CodeGen/AMDGPU/callee-special-input-vgprs.ll
+++ b/llvm/test/CodeGen/AMDGPU/callee-special-input-vgprs.ll
@@ -3,7 +3,7 @@
 ; GCN-LABEL: {{^}}use_workitem_id_x:
 ; GCN: s_waitcnt
 ; GCN: v_and_b32_e32 [[ID:v[0-9]+]], 0x3ff, v0
-; GCN-NEXT: flat_store_dword v{{\[[0-9]:[0-9]+\]}}, [[ID]]
+; GCN-NEXT: {{flat|global}}_store_dword v{{\[[0-9]:[0-9]+\]}}, [[ID]]
 ; GCN-NEXT: s_waitcnt
 ; GCN-NEXT: s_setpc_b64
 define void @use_workitem_id_x() #1 {
@@ -15,7 +15,7 @@ define void @use_workitem_id_x() #1 {
 ; GCN-LABEL: {{^}}use_workitem_id_y:
 ; GCN: s_waitcnt
 ; GCN: v_bfe_u32 [[ID:v[0-9]+]], v0, 10, 10
-; GCN-NEXT: flat_store_dword v{{\[[0-9]:[0-9]+\]}}, [[ID]]
+; GCN-NEXT: {{flat|global}}_store_dword v{{\[[0-9]:[0-9]+\]}}, [[ID]]
 ; GCN-NEXT: s_waitcnt
 ; GCN-NEXT: s_setpc_b64
 define void @use_workitem_id_y() #1 {
@@ -27,7 +27,7 @@ define void @use_workitem_id_y() #1 {
 ; GCN-LABEL: {{^}}use_workitem_id_z:
 ; GCN: s_waitcnt
 ; GCN: v_bfe_u32 [[ID:v[0-9]+]], v0, 20, 10
-; GCN-NEXT: flat_store_dword v{{\[[0-9]:[0-9]+\]}}, [[ID]]
+; GCN-NEXT: {{flat|global}}_store_dword v{{\[[0-9]:[0-9]+\]}}, [[ID]]
 ; GCN-NEXT: s_waitcnt
 ; GCN-NEXT: s_setpc_b64
 define void @use_workitem_id_z() #1 {
@@ -40,8 +40,8 @@ define void @use_workitem_id_z() #1 {
 ; GCN: s_waitcnt
 ; GCN-DAG: v_and_b32_e32 [[IDX:v[0-9]+]], 0x3ff, v0
 ; GCN-DAG: v_bfe_u32 [[IDY:v[0-9]+]], v0, 10, 10
-; GCN-DAG: flat_store_dword v{{\[[0-9]:[0-9]+\]}}, [[IDX]]
-; GCN-DAG: flat_store_dword v{{\[[0-9]:[0-9]+\]}}, [[IDY]]
+; GCN-DAG: {{flat|global}}_store_dword v{{\[[0-9]:[0-9]+\]}}, [[IDX]]
+; GCN-DAG: {{flat|global}}_store_dword v{{\[[0-9]:[0-9]+\]}}, [[IDY]]
 ; GCN-NEXT: s_waitcnt
 ; GCN-NEXT: s_setpc_b64
 define void @use_workitem_id_xy() #1 {
@@ -57,9 +57,9 @@ define void @use_workitem_id_xy() #1 {
 ; GCN-DAG: v_and_b32_e32 [[IDX:v[0-9]+]], 0x3ff, v0
 ; GCN-DAG: v_bfe_u32 [[IDY:v[0-9]+]], v0, 10, 10
 ; GCN-DAG: v_bfe_u32 [[IDZ:v[0-9]+]], v0, 20, 10
-; GCN-DAG: flat_store_dword v{{\[[0-9]:[0-9]+\]}}, [[IDX]]
-; GCN-DAG: flat_store_dword v{{\[[0-9]:[0-9]+\]}}, [[IDY]]
-; GCN-DAG: flat_store_dword v{{\[[0-9]:[0-9]+\]}}, [[IDZ]]
+; GCN-DAG: {{flat|global}}_store_dword v{{\[[0-9]:[0-9]+\]}}, [[IDX]]
+; GCN-DAG: {{flat|global}}_store_dword v{{\[[0-9]:[0-9]+\]}}, [[IDY]]
+; GCN-DAG: {{flat|global}}_store_dword v{{\[[0-9]:[0-9]+\]}}, [[IDZ]]
 ; GCN-NEXT: s_waitcnt
 ; GCN-NEXT: s_setpc_b64
 define void @use_workitem_id_xyz() #1 {
@@ -76,8 +76,8 @@ define void @use_workitem_id_xyz() #1 {
 ; GCN: s_waitcnt
 ; GCN-DAG: v_and_b32_e32 [[IDX:v[0-9]+]], 0x3ff, v0
 ; GCN-DAG: v_bfe_u32 [[IDZ:v[0-9]+]], v0, 20, 10
-; GCN-DAG: flat_store_dword v{{\[[0-9]:[0-9]+\]}}, [[IDX]]
-; GCN-DAG: flat_store_dword v{{\[[0-9]:[0-9]+\]}}, [[IDZ]]
+; GCN-DAG: {{flat|global}}_store_dword v{{\[[0-9]:[0-9]+\]}}, [[IDX]]
+; GCN-DAG: {{flat|global}}_store_dword v{{\[[0-9]:[0-9]+\]}}, [[IDZ]]
 ; GCN-NEXT: s_waitcnt
 ; GCN-NEXT: s_setpc_b64
 define void @use_workitem_id_xz() #1 {
@@ -92,8 +92,8 @@ define void @use_workitem_id_xz() #1 {
 ; GCN: s_waitcnt
 ; GCN-DAG: v_bfe_u32 [[IDY:v[0-9]+]], v0, 10, 10
 ; GCN-DAG: v_bfe_u32 [[IDZ:v[0-9]+]], v0, 20, 10
-; GCN-DAG: flat_store_dword v{{\[[0-9]:[0-9]+\]}}, [[IDY]]
-; GCN-DAG: flat_store_dword v{{\[[0-9]:[0-9]+\]}}, [[IDZ]]
+; GCN-DAG: {{flat|global}}_store_dword v{{\[[0-9]:[0-9]+\]}}, [[IDY]]
+; GCN-DAG: {{flat|global}}_store_dword v{{\[[0-9]:[0-9]+\]}}, [[IDZ]]
 ; GCN-NEXT: s_waitcnt
 ; GCN-NEXT: s_setpc_b64
 define void @use_workitem_id_yz() #1 {
@@ -230,8 +230,8 @@ define void @func_indirect_use_workitem_id_z() #1 {
 ; GCN-LABEL: {{^}}other_arg_use_workitem_id_x:
 ; GCN: s_waitcnt
 ; GCN-DAG: v_and_b32_e32 [[ID:v[0-9]+]], 0x3ff, v1
-; GCN-DAG: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, v0
-; GCN-DAG: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[ID]]
+; GCN-DAG: {{flat|global}}_store_dword v{{\[[0-9]+:[0-9]+\]}}, v0
+; GCN-DAG: {{flat|global}}_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[ID]]
 define void @other_arg_use_workitem_id_x(i32 %arg0) #1 {
   %val = call i32 @llvm.amdgcn.workitem.id.x()
   store volatile i32 %arg0, i32 addrspace(1)* undef
@@ -242,8 +242,8 @@ define void @other_arg_use_workitem_id_x(i32 %arg0) #1 {
 ; GCN-LABEL: {{^}}other_arg_use_workitem_id_y:
 ; GCN: s_waitcnt
 ; GCN-DAG: v_bfe_u32 [[ID:v[0-9]+]], v1, 10, 10
-; GCN-DAG: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, v0
-; GCN-DAG: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[ID]]
+; GCN-DAG: {{flat|global}}_store_dword v{{\[[0-9]+:[0-9]+\]}}, v0
+; GCN-DAG: {{flat|global}}_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[ID]]
 define void @other_arg_use_workitem_id_y(i32 %arg0) #1 {
   %val = call i32 @llvm.amdgcn.workitem.id.y()
   store volatile i32 %arg0, i32 addrspace(1)* undef
@@ -254,8 +254,8 @@ define void @other_arg_use_workitem_id_y(i32 %arg0) #1 {
 ; GCN-LABEL: {{^}}other_arg_use_workitem_id_z:
 ; GCN: s_waitcnt
 ; GCN-DAG: v_bfe_u32 [[ID:v[0-9]+]], v1, 20, 10
-; GCN-DAG: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, v0
-; GCN-DAG: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[ID]]
+; GCN-DAG: {{flat|global}}_store_dword v{{\[[0-9]+:[0-9]+\]}}, v0
+; GCN-DAG: {{flat|global}}_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[ID]]
 define void @other_arg_use_workitem_id_z(i32 %arg0) #1 {
   %val = call i32 @llvm.amdgcn.workitem.id.z()
   store volatile i32 %arg0, i32 addrspace(1)* undef
@@ -306,7 +306,7 @@ define amdgpu_kernel void @kern_indirect_other_arg_use_workitem_id_z() #1 {
 ; GCN: buffer_store_dword v32, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
 ; GCN: buffer_load_dword v32, off, s[0:3], s32{{$}}
 ; GCN: v_and_b32_e32 v32, 0x3ff, v32
-; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+]}}, v32
+; GCN: {{flat|global}}_store_dword v{{\[[0-9]+:[0-9]+]}}, v32
 
 ; GCN: buffer_load_dword v32, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
 ; GCN-NEXT: s_waitcnt
@@ -432,7 +432,7 @@ define void @too_many_args_call_too_many_args_use_workitem_id_x(
 ; GCN: buffer_load_dword v32, off, s[0:3], s32 offset:4
 ; GCN-NEXT: s_waitcnt
 ; GCN-NEXT: v_and_b32_e32 v32, 0x3ff, v32
-; GCN-NEXT: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, v32
+; GCN-NEXT: {{flat|global}}_store_dword v{{\[[0-9]+:[0-9]+\]}}, v32
 ; GCN: buffer_load_dword v0, off, s[0:3], s32{{$}}
 ; GCN: buffer_load_dword v32, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
 ; GCN: s_setpc_b64
@@ -489,16 +489,15 @@ define void @too_many_args_use_workitem_id_x_byval(
 
 ; GCN-LABEL: {{^}}kern_call_too_many_args_use_workitem_id_x_byval:
 ; GCN: enable_vgpr_workitem_id = 0
-
-; GCN: s_mov_b32 s33, s7
+; GCN-DAG: s_mov_b32 s33, s7
+; GCN-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 0x3e7{{$}}
+; GCN: buffer_store_dword [[K]], off, s[0:3], s33 offset:4
+; GCN: buffer_load_dword [[RELOAD_BYVAL:v[0-9]+]], off, s[0:3], s33 offset:4
 ; GCN: s_add_u32 s32, s33, 0x400{{$}}
 
 ; GCN-NOT: s32
-; GCN-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 0x3e7{{$}}
-; GCN: buffer_store_dword [[K]], off, s[0:3], s33 offset:4
 ; GCN: buffer_store_dword v0, off, s[0:3], s32 offset:4
 
-; GCN: buffer_load_dword [[RELOAD_BYVAL:v[0-9]+]], off, s[0:3], s33 offset:4
 ; GCN: buffer_store_dword [[RELOAD_BYVAL]], off, s[0:3], s32{{$}}
 ; GCN: v_mov_b32_e32 [[RELOAD_BYVAL]],
 ; GCN: s_swappc_b64
@@ -521,9 +520,8 @@ define amdgpu_kernel void @kern_call_too_many_args_use_workitem_id_x_byval() #1
 ; GCN-LABEL: {{^}}func_call_too_many_args_use_workitem_id_x_byval:
 ; GCN: v_mov_b32_e32 [[K:v[0-9]+]], 0x3e7{{$}}
 ; GCN: buffer_store_dword [[K]], off, s[0:3], s34{{$}}
-; GCN: buffer_store_dword v0, off, s[0:3], s32 offset:4
-
 ; GCN: buffer_load_dword [[RELOAD_BYVAL:v[0-9]+]], off, s[0:3], s34{{$}}
+; GCN: buffer_store_dword v0, off, s[0:3], s32 offset:4
 ; GCN: buffer_store_dword [[RELOAD_BYVAL]], off, s[0:3], s32{{$}}
 ; GCN: v_mov_b32_e32 [[RELOAD_BYVAL]],
 ; GCN: s_swappc_b64
@@ -543,19 +541,25 @@ define void @func_call_too_many_args_use_workitem_id_x_byval() #1 {
   ret void
 }
 
+; Only one stack load should be emitted for all 3 values.
 ; GCN-LABEL: {{^}}too_many_args_use_workitem_id_xyz:
-; GCN: buffer_store_dword v32, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
-; GCN: buffer_load_dword v32, off, s[0:3], s32{{$}}
-; GCN: v_and_b32_e32 v32, 0x3ff, v32
-; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+]}}, v32
-; GCN: buffer_load_dword v32, off, s[0:3], s32{{$}}
-; GCN: v_bfe_u32 v32, v32, 10, 10
-; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+]}}, v32
+; GCN: buffer_store_dword v32, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GCN: buffer_store_dword v33, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GCN-NOT: buffer_load_dword v{{[0-9]+}}, off, s[0:3], s32{{$}}
 ; GCN: buffer_load_dword v32, off, s[0:3], s32{{$}}
-; GCN: v_bfe_u32 v32, v32, 20, 10
-; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+]}}, v32
-
-; GCN: buffer_load_dword v32, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GCN-NOT: buffer_load_dword
+
+; GCN: v_and_b32_e32 [[AND_X:v[0-9]+]], 0x3ff, v32
+; GCN-NOT: buffer_load_dword
+; GCN: {{flat|global}}_store_dword v{{\[[0-9]+:[0-9]+]}}, [[AND_X]]
+; GCN-NOT: buffer_load_dword
+; GCN: v_bfe_u32 [[BFE_Y:v[0-9]+]], v32, 10, 10
+; GCN-NEXT: v_bfe_u32 [[BFE_Z:v[0-9]+]], v32, 20, 10
+; GCN-NEXT: {{flat|global}}_store_dword v{{\[[0-9]+:[0-9]+]}}, [[BFE_Y]]
+; GCN-NEXT: {{flat|global}}_store_dword v{{\[[0-9]+:[0-9]+]}}, [[BFE_Z]]
+
+; GCN: buffer_load_dword v33, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GCN: buffer_load_dword v32, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
 ; GCN-NEXT: s_waitcnt
 ; GCN-NEXT: s_setpc_b64
 define void @too_many_args_use_workitem_id_xyz(
@@ -642,11 +646,11 @@ define amdgpu_kernel void @kern_call_too_many_args_use_workitem_id_xyz() #1 {
 
 ; GCN-LABEL: {{^}}too_many_args_use_workitem_id_x_stack_yz:
 ; GCN-DAG: v_and_b32_e32 [[IDX:v[0-9]+]], 0x3ff, v31
-; GCN-DAG: flat_store_dword v[0:1], [[IDX]]
+; GCN-DAG: {{flat|global}}_store_dword v[0:1], [[IDX]]
 ; GCN-DAG: v_bfe_u32 [[IDY:v[0-9]+]], v31, 10, 10
-; GCN-DAG: flat_store_dword v{{\[[0-9]+:[0-9]+]}}, [[IDY]]
+; GCN-DAG: {{flat|global}}_store_dword v{{\[[0-9]+:[0-9]+]}}, [[IDY]]
 ; GCN-DAG: v_bfe_u32 [[IDZ:v[0-9]+]], v31, 20, 10
-; GCN-DAG: flat_store_dword v{{\[[0-9]+:[0-9]+]}}, [[IDZ]]
+; GCN-DAG: {{flat|global}}_store_dword v{{\[[0-9]+:[0-9]+]}}, [[IDZ]]
 
 ; GCN: s_waitcnt
 ; GCN-NEXT: s_setpc_b64
diff --git a/llvm/test/CodeGen/AMDGPU/calling-conventions.ll b/llvm/test/CodeGen/AMDGPU/calling-conventions.ll
index 9affc6a53..f97886110 100644
--- a/llvm/test/CodeGen/AMDGPU/calling-conventions.ll
+++ b/llvm/test/CodeGen/AMDGPU/calling-conventions.ll
@@ -199,6 +199,37 @@ define amdgpu_ps void @ps_mesa_inreg_v2i16(<2 x i16> inreg %arg0) {
   ret void
 }
 
+; FIXME: Differenet ABI for VI+
+; GCN-LABEL: {{^}}ps_mesa_v4f16:
+; SI: v_cvt_f16_f32_e32 v3, v3
+; SI: v_cvt_f16_f32_e32 v2, v2
+; SI: v_cvt_f16_f32_e32 v1, v1
+; SI: v_cvt_f16_f32_e32 v0, v0
+
+; VI: v_add_f16_e32 v2, 1.0, v1
+; VI: v_add_f16_sdwa v1, v1, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI: v_add_f16_e32 v4, 1.0, v0
+; VI: v_add_f16_sdwa v0, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+define amdgpu_ps <4 x half> @ps_mesa_v4f16(<4 x half> %arg0) {
+  %add = fadd <4 x half> %arg0, <half 1.0, half 1.0, half 1.0, half 1.0>
+  ret <4 x half> %add
+}
+
+; GCN-LABEL: {{^}}ps_mesa_inreg_v4f16:
+; SI: v_cvt_f16_f32_e32 v{{[0-9]+}}, s3
+; SI: v_cvt_f16_f32_e32 v{{[0-9]+}}, s2
+; SI: v_cvt_f16_f32_e32 v{{[0-9]+}}, s1
+; SI: v_cvt_f16_f32_e32 v{{[0-9]+}}, s0
+
+; VI: v_add_f16_e64
+; VI: v_add_f16_sdwa
+; VI: v_add_f16_e64
+; VI: v_add_f16_sdwa
+define amdgpu_ps <4 x half> @ps_mesa_inreg_v4f16(<4 x half> inreg %arg0) {
+  %add = fadd <4 x half> %arg0, <half 1.0, half 1.0, half 1.0, half 1.0>
+  ret <4 x half> %add
+}
+
 ; GCN-LABEL: {{^}}ps_mesa_inreg_v3i32:
 ; GCN-DAG: s_add_i32 s0, s0, 1
 ; GCN-DAG: s_add_i32 s{{[0-9]*}}, s1, 2
diff --git a/llvm/test/CodeGen/AMDGPU/cc-sgpr-limit.ll b/llvm/test/CodeGen/AMDGPU/cc-sgpr-limit.ll
new file mode 100644
index 000000000..4352e411d
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/cc-sgpr-limit.ll
@@ -0,0 +1,138 @@
+; RUN: llc < %s -march=amdgcn -mcpu=verde -verify-machineinstrs | FileCheck %s
+; RUN: llc < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck %s
+; RUN: llc < %s -march=amdgcn -mcpu=gfx900 -verify-machineinstrs | FileCheck %s
+
+; CHECK: s_add_i32 s0, s0, s1
+; CHECK: s_add_i32 s1, s0, s2
+; CHECK: s_add_i32 s2, s1, s3
+; CHECK: s_add_i32 s3, s2, s4
+; CHECK: s_add_i32 s4, s3, s5
+; CHECK: s_add_i32 s5, s4, s6
+; CHECK: s_add_i32 s6, s5, s7
+; CHECK: s_add_i32 s7, s6, s8
+; CHECK: s_add_i32 s8, s7, s9
+; CHECK: s_add_i32 s9, s8, s10
+; CHECK: s_add_i32 s10, s9, s11
+; CHECK: s_add_i32 s11, s10, s12
+; CHECK: s_add_i32 s12, s11, s13
+; CHECK: s_add_i32 s13, s12, s14
+; CHECK: s_add_i32 s14, s13, s15
+; CHECK: s_add_i32 s15, s14, s16
+; CHECK: s_add_i32 s16, s15, s17
+; CHECK: s_add_i32 s17, s16, s18
+; CHECK: s_add_i32 s18, s17, s19
+; CHECK: s_add_i32 s19, s18, s20
+; CHECK: s_add_i32 s20, s19, s21
+; CHECK: s_add_i32 s21, s20, s22
+; CHECK: s_add_i32 s22, s21, s23
+; CHECK: s_add_i32 s23, s22, s24
+; CHECK: s_add_i32 s24, s23, s25
+; CHECK: s_add_i32 s25, s24, s26
+; CHECK: s_add_i32 s26, s25, s27
+; CHECK: s_add_i32 s27, s26, s28
+; CHECK: s_add_i32 s28, s27, s29
+define amdgpu_gs { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } @_amdgpu_gs_sgpr_limit_i32 (i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg) {
+.entry:
+  %30 = add i32 %0, %1
+  %31 =  add i32 %30, %2
+  %32 =  add i32 %31, %3
+  %33 =  add i32 %32, %4
+  %34 =  add i32 %33, %5
+  %35 =  add i32 %34, %6
+  %36 =  add i32 %35, %7
+  %37 =  add i32 %36, %8
+  %38 =  add i32 %37, %9
+  %39 =  add i32 %38, %10
+  %40 =  add i32 %39, %11
+  %41 =  add i32 %40, %12
+  %42 =  add i32 %41, %13
+  %43 =  add i32 %42, %14
+  %44 =  add i32 %43, %15
+  %45 =  add i32 %44, %16
+  %46 =  add i32 %45, %17
+  %47 =  add i32 %46, %18
+  %48 =  add i32 %47, %19
+  %49 =  add i32 %48, %20
+  %50 =  add i32 %49, %21
+  %51 =  add i32 %50, %22
+  %52 =  add i32 %51, %23
+  %53 =  add i32 %52, %24
+  %54 =  add i32 %53, %25
+  %55 =  add i32 %54, %26
+  %56 =  add i32 %55, %27
+  %57 =  add i32 %56, %28
+  %58 =  add i32 %57, %29
+  %59 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } undef, i32 %30, 0
+  %60 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %59, i32 %31, 1
+  %61 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %60, i32 %32, 2
+  %62 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %61, i32 %33, 3
+  %63 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %62, i32 %34, 4
+  %64 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %63, i32 %35, 5
+  %65 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %64, i32 %36, 6
+  %66 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %65, i32 %37, 7
+  %67 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %66, i32 %38, 8
+  %68 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %67, i32 %39, 9
+  %69 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %68, i32 %40, 10
+  %70 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %69, i32 %41, 11
+  %71 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %70, i32 %42, 12
+  %72 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %71, i32 %43, 13
+  %73 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %72, i32 %44, 14
+  %74 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %73, i32 %45, 15
+  %75 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %74, i32 %46, 16
+  %76 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %75, i32 %47, 17
+  %77 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %76, i32 %48, 18
+  %78 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %77, i32 %49, 19
+  %79 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %78, i32 %50, 20
+  %80 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %79, i32 %51, 21
+  %81 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %80, i32 %52, 22
+  %82 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %81, i32 %53, 23
+  %83 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %82, i32 %54, 24
+  %84 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %83, i32 %55, 25
+  %85 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %84, i32 %56, 26
+  %86 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %85, i32 %57, 27
+  %87 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %86, i32 %58, 28
+  ret { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %87
+}
+
+; CHECK: s_xor_b64 s[0:1], s[0:1], s[2:3]
+; CHECK: s_xor_b64 s[0:1], s[0:1], s[4:5]
+; CHECK: s_xor_b64 s[0:1], s[0:1], s[6:7]
+; CHECK: s_xor_b64 s[0:1], s[0:1], s[8:9]
+; CHECK: s_xor_b64 s[0:1], s[0:1], s[10:11]
+; CHECK: s_xor_b64 s[0:1], s[0:1], s[12:13]
+; CHECK: s_xor_b64 s[0:1], s[0:1], s[14:15]
+; CHECK: s_xor_b64 s[0:1], s[0:1], s[16:17]
+; CHECK: s_xor_b64 s[0:1], s[0:1], s[18:19]
+; CHECK: s_xor_b64 s[0:1], s[0:1], s[20:21]
+; CHECK: s_xor_b64 s[0:1], s[0:1], s[22:23]
+; CHECK: s_xor_b64 s[0:1], s[0:1], s[24:25]
+; CHECK: s_xor_b64 s[0:1], s[0:1], s[26:27]
+; CHECK: s_xor_b64 s[0:1], s[0:1], s[28:29]
+; CHECK: s_xor_b64 s[0:1], s[0:1], s[30:31]
+; CHECK: s_xor_b64 s[0:1], s[0:1], s[32:33]
+; CHECK: s_xor_b64 s[0:1], s[0:1], s[34:35]
+define amdgpu_gs void @_amdgpu_gs_sgpr_limit_i64(i64 inreg, i64 inreg, i64 inreg, i64 inreg, i64 inreg, i64 inreg, i64 inreg, i64 inreg, i64 inreg, i64 inreg, i64 inreg, i64 inreg, i64 inreg, i64 inreg, i64 inreg, i64 inreg, i64 inreg, i64 inreg, i64 inreg, <4 x i32> inreg %addr) {
+.entry:
+  %19 = xor i64 %0, %1
+  %20 =  xor i64 %19, %2
+  %21 =  xor i64 %20, %3
+  %22 =  xor i64 %21, %4
+  %23 =  xor i64 %22, %5
+  %24 =  xor i64 %23, %6
+  %25 =  xor i64 %24, %7
+  %26 =  xor i64 %25, %8
+  %27 =  xor i64 %26, %9
+  %28 =  xor i64 %27, %10
+  %29 =  xor i64 %28, %11
+  %30 =  xor i64 %29, %12
+  %31 =  xor i64 %30, %13
+  %32 =  xor i64 %31, %14
+  %33 =  xor i64 %32, %15
+  %34 =  xor i64 %33, %16
+  %35 =  xor i64 %34, %17
+  %36 = bitcast i64 %35 to <2 x i32>
+  call void @llvm.amdgcn.raw.buffer.store.v2i32(<2 x i32> %36, <4 x i32> %addr, i32 4, i32 0, i32 0)
+  ret void
+}
+
+declare void @llvm.amdgcn.raw.buffer.store.v2i32(<2 x i32>, <4 x i32>, i32, i32, i32)
diff --git a/llvm/test/CodeGen/AMDGPU/cc-sgpr-over-limit.ll b/llvm/test/CodeGen/AMDGPU/cc-sgpr-over-limit.ll
new file mode 100644
index 000000000..72c6cfee2
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/cc-sgpr-over-limit.ll
@@ -0,0 +1,101 @@
+; RUN: not llc -march=amdgcn -mcpu=verde -verify-machineinstrs -o /dev/null %s 2>&1 | FileCheck %s
+; RUN: not llc -march=amdgcn -mcpu=tonga -verify-machineinstrs -o /dev/null %s 2>&1 | FileCheck %s
+; RUN: not llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs -o /dev/null %s 2>&1 | FileCheck %s
+
+;CHECK: LLVM ERROR: unable to allocate function argument
+define amdgpu_gs { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } @_amdgpu_gs_sgpr_i32 (i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg) {
+.entry:
+  %46 = add i32 %0, %1
+  %47 =  add i32 %46, %2
+  %48 =  add i32 %47, %3
+  %49 =  add i32 %48, %4
+  %50 =  add i32 %49, %5
+  %51 =  add i32 %50, %6
+  %52 =  add i32 %51, %7
+  %53 =  add i32 %52, %8
+  %54 =  add i32 %53, %9
+  %55 =  add i32 %54, %10
+  %56 =  add i32 %55, %11
+  %57 =  add i32 %56, %12
+  %58 =  add i32 %57, %13
+  %59 =  add i32 %58, %14
+  %60 =  add i32 %59, %15
+  %61 =  add i32 %60, %16
+  %62 =  add i32 %61, %17
+  %63 =  add i32 %62, %18
+  %64 =  add i32 %63, %19
+  %65 =  add i32 %64, %20
+  %66 =  add i32 %65, %21
+  %67 =  add i32 %66, %22
+  %68 =  add i32 %67, %23
+  %69 =  add i32 %68, %24
+  %70 =  add i32 %69, %25
+  %71 =  add i32 %70, %26
+  %72 =  add i32 %71, %27
+  %73 =  add i32 %72, %28
+  %74 =  add i32 %73, %29
+  %75 =  add i32 %74, %30
+  %76 =  add i32 %75, %31
+  %77 =  add i32 %76, %32
+  %78 =  add i32 %77, %33
+  %79 =  add i32 %78, %34
+  %80 =  add i32 %79, %35
+  %81 =  add i32 %80, %36
+  %82 =  add i32 %81, %37
+  %83 =  add i32 %82, %38
+  %84 =  add i32 %83, %39
+  %85 =  add i32 %84, %40
+  %86 =  add i32 %85, %41
+  %87 =  add i32 %86, %42
+  %88 =  add i32 %87, %43
+  %89 =  add i32 %88, %44
+  %90 =  add i32 %89, %45
+  %91 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } undef, i32 %46, 0
+  %92 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %91, i32 %47, 1
+  %93 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %92, i32 %48, 2
+  %94 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %93, i32 %49, 3
+  %95 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %94, i32 %50, 4
+  %96 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %95, i32 %51, 5
+  %97 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %96, i32 %52, 6
+  %98 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %97, i32 %53, 7
+  %99 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %98, i32 %54, 8
+  %100 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %99, i32 %55, 9
+  %101 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %100, i32 %56, 10
+  %102 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %101, i32 %57, 11
+  %103 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %102, i32 %58, 12
+  %104 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %103, i32 %59, 13
+  %105 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %104, i32 %60, 14
+  %106 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %105, i32 %61, 15
+  %107 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %106, i32 %62, 16
+  %108 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %107, i32 %63, 17
+  %109 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %108, i32 %64, 18
+  %110 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %109, i32 %65, 19
+  %111 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %110, i32 %66, 20
+  %112 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %111, i32 %67, 21
+  %113 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %112, i32 %68, 22
+  %114 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %113, i32 %69, 23
+  %115 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %114, i32 %70, 24
+  %116 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %115, i32 %71, 25
+  %117 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %116, i32 %72, 26
+  %118 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %117, i32 %73, 27
+  %119 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %118, i32 %74, 28
+  %120 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %119, i32 %75, 29
+  %121 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %120, i32 %76, 30
+  %122 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %121, i32 %77, 31
+  %123 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %122, i32 %78, 32
+  %124 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %123, i32 %79, 33
+  %125 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %124, i32 %80, 34
+  %126 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %125, i32 %81, 35
+  %127 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %126, i32 %82, 36
+  %128 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %127, i32 %83, 37
+  %129 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %128, i32 %84, 38
+  %130 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %129, i32 %85, 39
+  %131 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %130, i32 %86, 40
+  %132 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %131, i32 %87, 41
+  %133 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %132, i32 %88, 42
+  %134 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %133, i32 %89, 43
+  %135 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %134, i32 %90, 44
+  ret { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %135
+}
+
+declare void @llvm.amdgcn.raw.buffer.store.v2i32(<2 x i32>, <4 x i32>, i32, i32, i32)
diff --git a/llvm/test/CodeGen/AMDGPU/cgp-addressing-modes.ll b/llvm/test/CodeGen/AMDGPU/cgp-addressing-modes.ll
index 47ca6054f..1cdf1d391 100644
--- a/llvm/test/CodeGen/AMDGPU/cgp-addressing-modes.ll
+++ b/llvm/test/CodeGen/AMDGPU/cgp-addressing-modes.ll
@@ -44,7 +44,10 @@ done:
 ; GCN-LABEL: {{^}}test_sink_global_small_max_i32_ds_offset:
 ; GCN: s_and_saveexec_b64
 ; SICIVI: buffer_load_sbyte {{v[0-9]+}}, off, {{s\[[0-9]+:[0-9]+\]}}, s{{[0-9]+$}}
-; GFX9: global_load_sbyte {{v[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}}, off{{$}}
+
+; GFX9: v_add_co_u32_e32 v{{[0-9]+}}, vcc, 0xf000,
+; GFX9: v_addc_co_u32_e32 v{{[0-9]+}}, vcc, 0,
+; GFX9: global_load_sbyte {{v[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}}, off offset:4095{{$}}
 ; GCN: {{^}}BB1_2:
 ; GCN: s_or_b64 exec
 define amdgpu_kernel void @test_sink_global_small_max_i32_ds_offset(i32 addrspace(1)* %out, i8 addrspace(1)* %in) {
diff --git a/llvm/test/CodeGen/AMDGPU/chain-hi-to-lo.ll b/llvm/test/CodeGen/AMDGPU/chain-hi-to-lo.ll
index f4c06a8a9..9ec8b7573 100644
--- a/llvm/test/CodeGen/AMDGPU/chain-hi-to-lo.ll
+++ b/llvm/test/CodeGen/AMDGPU/chain-hi-to-lo.ll
@@ -1,10 +1,15 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX900 %s
 
-; GCN-LABEL: {{^}}chain_hi_to_lo_private:
-; GCN: buffer_load_ushort [[DST:v[0-9]+]], off, [[RSRC:s\[[0-9]+:[0-9]+\]]], [[SOFF:s[0-9]+]] offset:2
-; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: buffer_load_short_d16_hi [[DST]], off, [[RSRC]], [[SOFF]]
 define <2 x half> @chain_hi_to_lo_private() {
+; GCN-LABEL: chain_hi_to_lo_private:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_load_ushort v0, off, s[0:3], s33 offset:2
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_load_short_d16_hi v0, off, s[0:3], s33
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
 bb:
   %gep_lo = getelementptr inbounds half, half addrspace(5)* null, i64 1
   %load_lo = load half, half addrspace(5)* %gep_lo
@@ -17,11 +22,15 @@ bb:
   ret <2 x half> %result
 }
 
-; GCN-LABEL: {{^}}chain_hi_to_lo_private_different_bases:
-; GCN: buffer_load_ushort [[DST:v[0-9]+]], v{{[0-9]+}}, [[RSRC:s\[[0-9]+:[0-9]+\]]], [[SOFF:s[0-9]+]] offen
-; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: buffer_load_short_d16_hi [[DST]], v{{[0-9]+}}, [[RSRC]], [[SOFF]] offen
 define <2 x half> @chain_hi_to_lo_private_different_bases(half addrspace(5)* %base_lo, half addrspace(5)* %base_hi) {
+; GCN-LABEL: chain_hi_to_lo_private_different_bases:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_load_ushort v0, v0, s[0:3], s33 offen
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_load_short_d16_hi v0, v1, s[0:3], s33 offen
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
 bb:
   %load_lo = load half, half addrspace(5)* %base_lo
   %load_hi = load half, half addrspace(5)* %base_hi
@@ -32,10 +41,15 @@ bb:
   ret <2 x half> %result
 }
 
-; GCN-LABEL: {{^}}chain_hi_to_lo_arithmatic:
-; GCN: v_add_f16_e32 [[DST:v[0-9]+]], 1.0, v{{[0-9]+}}
-; GCN-NEXT: buffer_load_short_d16_hi [[DST]], v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offen
 define <2 x half> @chain_hi_to_lo_arithmatic(half addrspace(5)* %base, half %in) {
+; GCN-LABEL: chain_hi_to_lo_arithmatic:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_add_f16_e32 v1, 1.0, v1
+; GCN-NEXT:    buffer_load_short_d16_hi v1, v0, s[0:3], s33 offen
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v0, v1
+; GCN-NEXT:    s_setpc_b64 s[30:31]
 bb:
   %arith_lo = fadd half %in, 1.0
   %load_hi = load half, half addrspace(5)* %base
@@ -46,11 +60,16 @@ bb:
   ret <2 x half> %result
 }
 
-; GCN-LABEL: {{^}}chain_hi_to_lo_group:
-; GCN: ds_read_u16 [[DST:v[0-9]+]], [[ADDR:v[0-9]+]] offset:2
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: ds_read_u16_d16_hi [[DST]], [[ADDR]]
 define <2 x half> @chain_hi_to_lo_group() {
+; GCN-LABEL: chain_hi_to_lo_group:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v1, 0
+; GCN-NEXT:    ds_read_u16 v0, v1 offset:2
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    ds_read_u16_d16_hi v0, v1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
 bb:
   %gep_lo = getelementptr inbounds half, half addrspace(3)* null, i64 1
   %load_lo = load half, half addrspace(3)* %gep_lo
@@ -63,11 +82,15 @@ bb:
   ret <2 x half> %result
 }
 
-; GCN-LABEL: {{^}}chain_hi_to_lo_group_different_bases:
-; GCN: ds_read_u16 [[DST:v[0-9]+]], v{{[0-9]+}}
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: ds_read_u16_d16_hi [[DST]], v{{[0-9]+}}
 define <2 x half> @chain_hi_to_lo_group_different_bases(half addrspace(3)* %base_lo, half addrspace(3)* %base_hi) {
+; GCN-LABEL: chain_hi_to_lo_group_different_bases:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    ds_read_u16 v0, v0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    ds_read_u16_d16_hi v0, v1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
 bb:
   %load_lo = load half, half addrspace(3)* %base_lo
   %load_hi = load half, half addrspace(3)* %base_hi
@@ -78,10 +101,19 @@ bb:
   ret <2 x half> %result
 }
 
-; GCN-LABEL: {{^}}chain_hi_to_lo_global:
-; GCN: global_load_ushort [[DST:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, off
-; GCN: global_load_short_d16_hi [[DST]], v{{\[[0-9]+:[0-9]+\]}}, off
 define <2 x half> @chain_hi_to_lo_global() {
+; GCN-LABEL: chain_hi_to_lo_global:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v0, 2
+; GCN-NEXT:    v_mov_b32_e32 v1, 0
+; GCN-NEXT:    global_load_ushort v0, v[0:1], off
+; GCN-NEXT:    v_mov_b32_e32 v1, 0
+; GCN-NEXT:    v_mov_b32_e32 v2, 0
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    global_load_short_d16_hi v0, v[1:2], off
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
 bb:
   %gep_lo = getelementptr inbounds half, half addrspace(1)* null, i64 1
   %load_lo = load half, half addrspace(1)* %gep_lo
@@ -94,11 +126,15 @@ bb:
   ret <2 x half> %result
 }
 
-; GCN-LABEL: {{^}}chain_hi_to_lo_global_different_bases:
-; GCN: global_load_ushort [[DST:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, off
-; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: global_load_short_d16_hi [[DST]], v{{\[[0-9]+:[0-9]+\]}}, off
 define <2 x half> @chain_hi_to_lo_global_different_bases(half addrspace(1)* %base_lo, half addrspace(1)* %base_hi) {
+; GCN-LABEL: chain_hi_to_lo_global_different_bases:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    global_load_ushort v0, v[0:1], off
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    global_load_short_d16_hi v0, v[2:3], off
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
 bb:
   %load_lo = load half, half addrspace(1)* %base_lo
   %load_hi = load half, half addrspace(1)* %base_hi
@@ -109,10 +145,19 @@ bb:
   ret <2 x half> %result
 }
 
-; GCN-LABEL: {{^}}chain_hi_to_lo_flat:
-; GCN: flat_load_ushort [[DST:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}
-; GCN: flat_load_short_d16_hi [[DST]], v{{\[[0-9]+:[0-9]+\]}}
 define <2 x half> @chain_hi_to_lo_flat() {
+; GCN-LABEL: chain_hi_to_lo_flat:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v0, 2
+; GCN-NEXT:    v_mov_b32_e32 v1, 0
+; GCN-NEXT:    flat_load_ushort v0, v[0:1]
+; GCN-NEXT:    v_mov_b32_e32 v1, 0
+; GCN-NEXT:    v_mov_b32_e32 v2, 0
+; GCN-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN-NEXT:    flat_load_short_d16_hi v0, v[1:2]
+; GCN-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
 bb:
   %gep_lo = getelementptr inbounds half, half* null, i64 1
   %load_lo = load half, half* %gep_lo
@@ -125,11 +170,15 @@ bb:
   ret <2 x half> %result
 }
 
-; GCN-LABEL: {{^}}chain_hi_to_lo_flat_different_bases:
-; GCN: flat_load_ushort [[DST:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}
-; GCN-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN-NEXT: flat_load_short_d16_hi [[DST]], v{{\[[0-9]+:[0-9]+\]}}
 define <2 x half> @chain_hi_to_lo_flat_different_bases(half* %base_lo, half* %base_hi) {
+; GCN-LABEL: chain_hi_to_lo_flat_different_bases:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    flat_load_ushort v0, v[0:1]
+; GCN-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN-NEXT:    flat_load_short_d16_hi v0, v[2:3]
+; GCN-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
 bb:
   %load_lo = load half, half* %base_lo
   %load_hi = load half, half* %base_hi
@@ -141,15 +190,34 @@ bb:
 }
 
 ; Make sure we don't lose any of the private stores.
-; GCN-LABEL: {{^}}vload2_private:
-; GCN: buffer_store_short v{{[0-9]+}}, off, s[0:3], s{{[0-9]+}} offset:4
-; GCN: buffer_store_short_d16_hi v{{[0-9]+}}, off, s[0:3], s{{[0-9]+}} offset:6
-; GCN: buffer_store_short v{{[0-9]+}}, off, s[0:3], s{{[0-9]+}} offset:8
-
-; GCN: buffer_load_ushort v{{[0-9]+}}, off, s[0:3], s{{[0-9]+}} offset:4
-; GCN: buffer_load_ushort v{{[0-9]+}}, off, s[0:3], s{{[0-9]+}} offset:6
-; GCN: buffer_load_short_d16_hi v{{[0-9]+}}, off, s[0:3], s{{[0-9]+}} offset:8
 define amdgpu_kernel void @vload2_private(i16 addrspace(1)* nocapture readonly %in, <2 x i16> addrspace(1)* nocapture %out) #0 {
+; GCN-LABEL: vload2_private:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_add_u32 flat_scratch_lo, s6, s9
+; GCN-NEXT:    s_addc_u32 flat_scratch_hi, s7, 0
+; GCN-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v2, s4
+; GCN-NEXT:    v_mov_b32_e32 v3, s5
+; GCN-NEXT:    global_load_ushort v4, v[2:3], off offset:4
+; GCN-NEXT:    global_load_dword v2, v[2:3], off
+; GCN-NEXT:    v_mov_b32_e32 v0, s6
+; GCN-NEXT:    v_mov_b32_e32 v1, s7
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_short v2, off, s[0:3], s9 offset:4
+; GCN-NEXT:    buffer_store_short_d16_hi v2, off, s[0:3], s9 offset:6
+; GCN-NEXT:    buffer_store_short v4, off, s[0:3], s9 offset:8
+; GCN-NEXT:    buffer_load_ushort v2, off, s[0:3], s9 offset:4
+; GCN-NEXT:    buffer_load_ushort v4, off, s[0:3], s9 offset:6
+; GCN-NEXT:    s_waitcnt vmcnt(1)
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v3, v4
+; GCN-NEXT:    buffer_load_short_d16_hi v3, off, s[0:3], s9 offset:8
+; GCN-NEXT:    v_lshl_or_b32 v2, v4, 16, v2
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    global_store_dwordx2 v[0:1], v[2:3], off
+; GCN-NEXT:    s_endpgm
 entry:
   %loc = alloca [3 x i16], align 2, addrspace(5)
   %loc.0.sroa_cast1 = bitcast [3 x i16] addrspace(5)* %loc to i8 addrspace(5)*
@@ -178,15 +246,17 @@ entry:
 
 ; There is another instruction between the misordered instruction and
 ; the value dependent load, so a simple operand check is insufficient.
-; GCN-LABEL: {{^}}chain_hi_to_lo_group_other_dep:
-; GFX900: ds_read_u16_d16_hi v1, v0
-; GFX900-NEXT: s_waitcnt lgkmcnt(0)
-; GFX900-NEXT: v_pk_sub_u16 v1, v1, -12 op_sel_hi:[1,0]
-; GFX900-NEXT: ds_read_u16_d16 v1, v0 offset:2
-; GFX900-NEXT: s_waitcnt lgkmcnt(0)
-; GFX900-NEXT: v_mov_b32_e32 v0, v1
-; GFX900-NEXT: s_setpc_b64
 define <2 x i16> @chain_hi_to_lo_group_other_dep(i16 addrspace(3)* %ptr) {
+; GCN-LABEL: chain_hi_to_lo_group_other_dep:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    ds_read_u16_d16_hi v1, v0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_pk_sub_u16 v1, v1, -12 op_sel_hi:[1,0]
+; GCN-NEXT:    ds_read_u16_d16 v1, v0 offset:2
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v0, v1
+; GCN-NEXT:    s_setpc_b64 s[30:31]
 bb:
   %gep_lo = getelementptr inbounds i16, i16 addrspace(3)* %ptr, i64 1
   %load_lo = load i16, i16 addrspace(3)* %gep_lo
@@ -199,15 +269,17 @@ bb:
 }
 
 ; The volatile operations aren't put on the same chain
-; GCN-LABEL: {{^}}chain_hi_to_lo_group_other_dep_multi_chain:
-; GFX900: ds_read_u16 v1, v0 offset:2
-; GFX900-NEXT: ds_read_u16_d16_hi v0, v0
-; GFX900-NEXT: v_mov_b32_e32 [[MASK:v[0-9]+]], 0xffff
-; GFX900-NEXT: s_waitcnt lgkmcnt(0)
-; GFX900-NEXT: v_pk_sub_u16 v0, v0, -12 op_sel_hi:[1,0]
-; GFX900-NEXT: v_bfi_b32 v0, [[MASK]], v1, v0
-; GFX900-NEXT: s_setpc_b64
 define <2 x i16> @chain_hi_to_lo_group_other_dep_multi_chain(i16 addrspace(3)* %ptr) {
+; GCN-LABEL: chain_hi_to_lo_group_other_dep_multi_chain:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    ds_read_u16 v1, v0 offset:2
+; GCN-NEXT:    ds_read_u16_d16_hi v0, v0
+; GCN-NEXT:    v_mov_b32_e32 v2, 0xffff
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_pk_sub_u16 v0, v0, -12 op_sel_hi:[1,0]
+; GCN-NEXT:    v_bfi_b32 v0, v2, v1, v0
+; GCN-NEXT:    s_setpc_b64 s[30:31]
 bb:
   %gep_lo = getelementptr inbounds i16, i16 addrspace(3)* %ptr, i64 1
   %load_lo = load volatile i16, i16 addrspace(3)* %gep_lo
@@ -219,15 +291,17 @@ bb:
   ret <2 x i16> %result
 }
 
-; GCN-LABEL: {{^}}chain_hi_to_lo_private_other_dep:
-; GFX900: buffer_load_short_d16_hi v1, v0, s[0:3], s33 offen
-; GFX900-NEXT: s_waitcnt vmcnt(0)
-; GFX900-NEXT: v_pk_sub_u16 v1, v1, -12 op_sel_hi:[1,0]
-; GFX900-NEXT: buffer_load_short_d16 v1, v0, s[0:3], s33 offen offset:2
-; GFX900-NEXT: s_waitcnt vmcnt(0)
-; GFX900-NEXT: v_mov_b32_e32 v0, v1
-; GFX900-NEXT: s_setpc_b64
 define <2 x i16> @chain_hi_to_lo_private_other_dep(i16 addrspace(5)* %ptr) {
+; GCN-LABEL: chain_hi_to_lo_private_other_dep:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_load_short_d16_hi v1, v0, s[0:3], s33 offen
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_pk_sub_u16 v1, v1, -12 op_sel_hi:[1,0]
+; GCN-NEXT:    buffer_load_short_d16 v1, v0, s[0:3], s33 offen offset:2
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v0, v1
+; GCN-NEXT:    s_setpc_b64 s[30:31]
 bb:
   %gep_lo = getelementptr inbounds i16, i16 addrspace(5)* %ptr, i64 1
   %load_lo = load i16, i16 addrspace(5)* %gep_lo
@@ -239,15 +313,17 @@ bb:
   ret <2 x i16> %result
 }
 
-; GCN-LABEL: {{^}}chain_hi_to_lo_global_other_dep:
-; GFX900: global_load_ushort v2, v[0:1], off offset:2
-; GFX900-NEXT: global_load_short_d16_hi v0, v[0:1], off
-; GFX900-NEXT: v_mov_b32_e32 [[MASK:v[0-9]+]], 0xffff
-; GFX900-NEXT: s_waitcnt vmcnt(0)
-; GFX900-NEXT: v_pk_sub_u16 v0, v0, -12 op_sel_hi:[1,0]
-; GFX900-NEXT: v_bfi_b32 v0, [[MASK]], v2, v0
-; GFX900-NEXT: s_setpc_b64
 define <2 x i16> @chain_hi_to_lo_global_other_dep(i16 addrspace(1)* %ptr) {
+; GCN-LABEL: chain_hi_to_lo_global_other_dep:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    global_load_ushort v2, v[0:1], off offset:2
+; GCN-NEXT:    global_load_short_d16_hi v0, v[0:1], off
+; GCN-NEXT:    v_mov_b32_e32 v1, 0xffff
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_pk_sub_u16 v0, v0, -12 op_sel_hi:[1,0]
+; GCN-NEXT:    v_bfi_b32 v0, v1, v2, v0
+; GCN-NEXT:    s_setpc_b64 s[30:31]
 bb:
   %gep_lo = getelementptr inbounds i16, i16 addrspace(1)* %ptr, i64 1
   %load_lo = load volatile i16, i16 addrspace(1)* %gep_lo
@@ -259,15 +335,17 @@ bb:
   ret <2 x i16> %result
 }
 
-; GCN-LABEL: {{^}}chain_hi_to_lo_flat_other_dep:
-; GFX900: flat_load_ushort v2, v[0:1] offset:2
-; GFX900-NEXT: flat_load_short_d16_hi v0, v[0:1]
-; GFX900-NEXT: v_mov_b32_e32 [[MASK:v[0-9]+]], 0xffff
-; GFX900-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_pk_sub_u16 v0, v0, -12 op_sel_hi:[1,0]
-; GFX900-NEXT: v_bfi_b32 v0, v1, v2, v0
-; GFX900-NEXT: s_setpc_b64
 define <2 x i16> @chain_hi_to_lo_flat_other_dep(i16 addrspace(0)* %ptr) {
+; GCN-LABEL: chain_hi_to_lo_flat_other_dep:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    flat_load_ushort v2, v[0:1] offset:2
+; GCN-NEXT:    flat_load_short_d16_hi v0, v[0:1]
+; GCN-NEXT:    v_mov_b32_e32 v1, 0xffff
+; GCN-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_pk_sub_u16 v0, v0, -12 op_sel_hi:[1,0]
+; GCN-NEXT:    v_bfi_b32 v0, v1, v2, v0
+; GCN-NEXT:    s_setpc_b64 s[30:31]
 bb:
   %gep_lo = getelementptr inbounds i16, i16 addrspace(0)* %ptr, i64 1
   %load_lo = load volatile i16, i16 addrspace(0)* %gep_lo
@@ -279,16 +357,18 @@ bb:
   ret <2 x i16> %result
 }
 
-; GCN-LABEL: {{^}}chain_hi_to_lo_group_may_alias_store:
-; GFX900: v_mov_b32_e32 [[K:v[0-9]+]], 0x7b
-; GFX900-NEXT: ds_read_u16 v2, v0
-; GFX900-NEXT: ds_write_b16 v1, [[K]]
-; GFX900-NEXT: ds_read_u16 v0, v0 offset:2
-; GFX900-NEXT: s_waitcnt lgkmcnt(0)
-; GFX900-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX900-NEXT: v_lshl_or_b32 v0, v2, 16, v0
-; GFX900-NEXT: s_setpc_b64
 define <2 x i16> @chain_hi_to_lo_group_may_alias_store(i16 addrspace(3)* %ptr, i16 addrspace(3)* %may.alias) {
+; GCN-LABEL: chain_hi_to_lo_group_may_alias_store:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v3, 0x7b
+; GCN-NEXT:    ds_read_u16 v2, v0
+; GCN-NEXT:    ds_write_b16 v1, v3
+; GCN-NEXT:    ds_read_u16 v0, v0 offset:2
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GCN-NEXT:    v_lshl_or_b32 v0, v2, 16, v0
+; GCN-NEXT:    s_setpc_b64 s[30:31]
 bb:
   %gep_lo = getelementptr inbounds i16, i16 addrspace(3)* %ptr, i64 1
   %gep_hi = getelementptr inbounds i16, i16 addrspace(3)* %ptr, i64 0
diff --git a/llvm/test/CodeGen/AMDGPU/combine-cond-add-sub.ll b/llvm/test/CodeGen/AMDGPU/combine-cond-add-sub.ll
index ae30ad37c..617f280e0 100644
--- a/llvm/test/CodeGen/AMDGPU/combine-cond-add-sub.ll
+++ b/llvm/test/CodeGen/AMDGPU/combine-cond-add-sub.ll
@@ -1,5 +1,5 @@
-; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
-; RUN: llc -march=amdgcn -mcpu=gfx902  -verify-machineinstrs < %s | FileCheck -check-prefix=GFX9 %s
+; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN %s
+; RUN: llc -march=amdgcn -mcpu=gfx902  -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GFX9 %s
 
 ; GCN-LABEL: {{^}}add1:
 ; GCN: v_cmp_gt_u32_e{{32|64}} [[CC:[^,]+]], v{{[0-9]+}}, v{{[0-9]+}}
@@ -130,16 +130,18 @@ bb:
 ; GCN-LABEL: {{^}}sub_sube_commuted:
 ; GCN-DAG: v_cmp_gt_u32_e{{32|64}} [[CC:[^,]+]], v{{[0-9]+}}, v{{[0-9]+}}
 ; GCN-DAG: buffer_load_dword [[V:v[0-9]+]],
-; GCN:     v_subbrev_u32_e{{32|64}} [[SUBB:v[0-9]+]], {{[^,]+}}, 0, [[V]], [[CC]]
-; GCN:     v_sub_i32_e32 [[SUB:v[0-9]+]], vcc, s{{[0-9]+}}, [[SUBB]]
-; GCN:     v_add_i32_e32 {{.*}}, 0x64, [[SUB]]
+; GCN: v_cndmask_b32_e64 [[CCZEXT:v[0-9]+]], 0, 1, [[CC]]
+; GCN: v_sub_i32_e32 [[SUB:v[0-9]+]], vcc, [[CCZEXT]], v4
+; GCN: v_add_i32_e32 [[ADD:v[0-9]+]], vcc, {{.*}}, [[SUB]]
+; GCN: v_add_i32_e32 {{.*}}, 0x64, [[ADD]]
 
 ; GFX9-LABEL: {{^}}sub_sube_commuted:
 ; GFX9-DAG: v_cmp_gt_u32_e{{32|64}} [[CC:[^,]+]], v{{[0-9]+}}, v{{[0-9]+}}
 ; GFX9-DAG: global_load_dword [[V:v[0-9]+]],
-; GFX9:     v_subbrev_co_u32_e{{32|64}} [[SUBB:v[0-9]+]], {{[^,]+}}, 0, [[V]], [[CC]]
-; GFX9:     v_sub_u32_e32 [[SUB:v[0-9]+]], s{{[0-9]+}}, [[SUBB]]
-; GFX9:     v_add_u32_e32 {{.*}}, 0x64, [[SUB]]
+; GFX9-DAG: v_cndmask_b32_e64 [[CCZEXT:v[0-9]+]], 0, 1, [[CC]]
+; GFX9: v_sub_u32_e32 {{.*}}, [[CCZEXT]]
+; GFX9: v_add_u32_e32
+; GFX9: v_add_u32_e32 {{.*}}, 0x64,
 define amdgpu_kernel void @sub_sube_commuted(i32 addrspace(1)* nocapture %arg, i32 %a) {
 bb:
   %x = tail call i32 @llvm.amdgcn.workitem.id.x()
@@ -237,6 +239,124 @@ bb:
   ret void
 }
 
+; sub x, sext (setcc) => addcarry x, 0, setcc
+; GCN-LABEL: {{^}}cmp_sub_sext:
+; GCN: v_cmp_gt_u32_e32 vcc, v
+; GCN-NOT: vcc
+; GCN: v_addc_u32_e32 [[RESULT:v[0-9]+]], vcc, 0, v{{[0-9]+}}, vcc
+define amdgpu_kernel void @cmp_sub_sext(i32 addrspace(1)* nocapture %arg) {
+bb:
+  %x = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %y = tail call i32 @llvm.amdgcn.workitem.id.y()
+  %gep = getelementptr inbounds i32, i32 addrspace(1)* %arg, i32 %x
+  %v = load i32, i32 addrspace(1)* %gep, align 4
+  %cmp = icmp ugt i32 %x, %y
+  %ext = sext i1 %cmp to i32
+  %add = sub i32 %v, %ext
+  store i32 %add, i32 addrspace(1)* %gep, align 4
+  ret void
+}
+
+; sub x, zext (setcc) => subcarry x, 0, setcc
+; GCN-LABEL: {{^}}cmp_sub_zext:
+; GCN: v_cmp_gt_u32_e32 vcc, v
+; GCN-NOT: vcc
+; GCN: v_subbrev_u32_e32 [[RESULT:v[0-9]+]], vcc, 0, v{{[0-9]+}}, vcc
+define amdgpu_kernel void @cmp_sub_zext(i32 addrspace(1)* nocapture %arg) {
+bb:
+  %x = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %y = tail call i32 @llvm.amdgcn.workitem.id.y()
+  %gep = getelementptr inbounds i32, i32 addrspace(1)* %arg, i32 %x
+  %v = load i32, i32 addrspace(1)* %gep, align 4
+  %cmp = icmp ugt i32 %x, %y
+  %ext = zext i1 %cmp to i32
+  %add = sub i32 %v, %ext
+  store i32 %add, i32 addrspace(1)* %gep, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}sub_addcarry:
+; GCN: v_cmp_gt_u32_e32 vcc, v
+; GCN-NOT: vcc
+; GCN: v_addc_u32_e32 [[ADDC:v[0-9]+]], vcc, 0, v{{[0-9]+}}, vcc
+; GCN-NOT: vcc
+; GCN: v_subrev_i32_e32 [[RESULT:v[0-9]+]], vcc,
+define amdgpu_kernel void @sub_addcarry(i32 addrspace(1)* nocapture %arg, i32 %a) {
+bb:
+  %x = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %y = tail call i32 @llvm.amdgcn.workitem.id.y()
+  %gep = getelementptr inbounds i32, i32 addrspace(1)* %arg, i32 %x
+  %v = load i32, i32 addrspace(1)* %gep, align 4
+  %cmp = icmp ugt i32 %x, %y
+  %ext = zext i1 %cmp to i32
+  %adde = add i32 %v, %ext
+  %add2 = sub i32 %adde, %a
+  store i32 %add2, i32 addrspace(1)* %gep, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}sub_subcarry:
+; GCN: v_cmp_gt_u32_e32 vcc, v
+; GCN-NOT: vcc
+; GCN: v_subb_u32_e32 [[RESULT:v[0-9]+]], vcc, v{{[0-9]+}}, v{{[0-9]+}}, vcc
+define amdgpu_kernel void @sub_subcarry(i32 addrspace(1)* nocapture %arg, i32 %a) {
+bb:
+  %x = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %y = tail call i32 @llvm.amdgcn.workitem.id.y()
+  %gep = getelementptr inbounds i32, i32 addrspace(1)* %arg, i32 %x
+  %v = load i32, i32 addrspace(1)* %gep, align 4
+  %cmp = icmp ugt i32 %x, %y
+  %ext = zext i1 %cmp to i32
+  %adde = sub i32 %v, %ext
+  %add2 = sub i32 %adde, %a
+  store i32 %add2, i32 addrspace(1)* %gep, align 4
+  ret void
+}
+
+; Check case where sub is commuted with zext
+; GCN-LABEL: {{^}}sub_zext_setcc_commute:
+; GCN: v_cmp_gt_u32_e32 vcc, v
+; GCN: v_cndmask
+; GCN: v_sub_i32_e32
+; GCN: v_add_i32_e32 [[ADD:v[0-9]+]], vcc,
+; GCN: v_subrev_i32_e32 [[RESULT:v[0-9]+]], vcc, s{{[0-9]+}}, [[ADD]]
+define amdgpu_kernel void @sub_zext_setcc_commute(i32 addrspace(1)* nocapture %arg, i32 %a, i32%b) {
+bb:
+  %x = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %y = tail call i32 @llvm.amdgcn.workitem.id.y()
+  %gep = getelementptr inbounds i32, i32 addrspace(1)* %arg, i32 %x
+  %v = load i32, i32 addrspace(1)* %gep, align 4
+  %cmp = icmp ugt i32 %x, %y
+  %ext = zext i1 %cmp to i32
+  %adde = sub i32 %v, %ext
+  %sub = sub i32 %a, %adde
+  %sub2 = sub i32 %sub, %b
+  store i32 %sub2, i32 addrspace(1)* %gep, align 4
+  ret void
+}
+
+; Check case where sub is commuted with sext
+; GCN-LABEL: {{^}}sub_sext_setcc_commute:
+; GCN: v_cmp_gt_u32_e32 vcc, v
+; GCN: v_cndmask
+; GCN: v_sub_i32_e32
+; GCN: v_add_i32_e32 [[ADD:v[0-9]+]], vcc,
+; GCN: v_subrev_i32_e32 [[RESULT:v[0-9]+]], vcc, s{{[0-9]+}}, [[ADD]]
+define amdgpu_kernel void @sub_sext_setcc_commute(i32 addrspace(1)* nocapture %arg, i32 %a, i32%b) {
+bb:
+  %x = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %y = tail call i32 @llvm.amdgcn.workitem.id.y()
+  %gep = getelementptr inbounds i32, i32 addrspace(1)* %arg, i32 %x
+  %v = load i32, i32 addrspace(1)* %gep, align 4
+  %cmp = icmp ugt i32 %x, %y
+  %ext = sext i1 %cmp to i32
+  %adde = sub i32 %v, %ext
+  %sub = sub i32 %a, %adde
+  %sub2 = sub i32 %sub, %b
+  store i32 %sub2, i32 addrspace(1)* %gep, align 4
+  ret void
+}
+
 declare i1 @llvm.amdgcn.class.f32(float, i32) #0
 
 declare i32 @llvm.amdgcn.workitem.id.x() #0
diff --git a/llvm/test/CodeGen/AMDGPU/commute-shifts.ll b/llvm/test/CodeGen/AMDGPU/commute-shifts.ll
index db8523326..1f0643e85 100644
--- a/llvm/test/CodeGen/AMDGPU/commute-shifts.ll
+++ b/llvm/test/CodeGen/AMDGPU/commute-shifts.ll
@@ -16,12 +16,12 @@ define amdgpu_ps float @main(float %arg0, float %arg1) #0 {
 ; SI-NEXT:    s_mov_b32 s7, s0
 ; SI-NEXT:    image_load v2, v0, s[0:7] dmask:0x1 unorm
 ; SI-NEXT:    v_and_b32_e32 v0, 7, v0
-; SI-NEXT:    v_lshl_b32_e32 v0, 1, v0
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v0, v2, v0
-; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; SI-NEXT:    v_lshrrev_b32_e32 v0, v0, v2
+; SI-NEXT:    v_and_b32_e32 v0, 1, v0
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
 ; SI-NEXT:    v_cndmask_b32_e32 v0, 0, v1, vcc
-; SI-NEXT:    v_cvt_pkrtz_f16_f32_e32 v0, v0, v0
+; SI-NEXT:    v_cvt_pkrtz_f16_f32_e32 v0, s0, v0
 ; SI-NEXT:    ; return to shader part epilog
 ;
 ; VI-LABEL: main:
@@ -37,12 +37,12 @@ define amdgpu_ps float @main(float %arg0, float %arg1) #0 {
 ; VI-NEXT:    s_mov_b32 s7, s0
 ; VI-NEXT:    image_load v2, v0, s[0:7] dmask:0x1 unorm
 ; VI-NEXT:    v_and_b32_e32 v0, 7, v0
-; VI-NEXT:    v_lshlrev_b32_e64 v0, v0, 1
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_and_b32_e32 v0, v2, v0
-; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; VI-NEXT:    v_lshrrev_b32_e32 v0, v0, v2
+; VI-NEXT:    v_and_b32_e32 v0, 1, v0
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
 ; VI-NEXT:    v_cndmask_b32_e32 v0, 0, v1, vcc
-; VI-NEXT:    v_cvt_pkrtz_f16_f32 v0, v0, v0
+; VI-NEXT:    v_cvt_pkrtz_f16_f32 v0, s0, v0
 ; VI-NEXT:    ; return to shader part epilog
 bb:
   %tmp = fptosi float %arg0 to i32
diff --git a/llvm/test/CodeGen/AMDGPU/computeNumSignBits-mul.ll b/llvm/test/CodeGen/AMDGPU/computeNumSignBits-mul.ll
new file mode 100644
index 000000000..11795cca1
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/computeNumSignBits-mul.ll
@@ -0,0 +1,172 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc  -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -o - -amdgpu-codegenprepare-mul24=0 < %s | FileCheck -check-prefix=GFX9 %s
+
+define i16 @num_sign_bits_mul_i48_0(i8 %X, i8 %Y, i8 %Z, i8 %W) {
+; GFX9-LABEL: num_sign_bits_mul_i48_0:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mul_i32_i24_sdwa v0, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-NEXT:    v_mul_i32_i24_sdwa v1, sext(v2), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-NEXT:    v_mul_i32_i24_e32 v0, v0, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %A = sext i8 %X to i48
+  %B = sext i8 %Y to i48
+  %C = sext i8 %Z to i48
+  %D = sext i8 %W to i48
+  %mul0 = mul i48 %A, %B
+  %mul1 = mul i48 %C, %D
+  %mul2 = mul i48 %mul0, %mul1
+  %trunc = trunc i48 %mul2 to i16
+  ret i16 %trunc
+}
+
+define i16 @num_sign_bits_mul_i48_1(i8 %X, i8 %Y, i8 %Z, i8 %W) {
+; GFX9-LABEL: num_sign_bits_mul_i48_1:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mul_i32_i24_sdwa v0, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-NEXT:    v_mul_i32_i24_sdwa v2, sext(v2), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-NEXT:    v_mul_hi_i32_i24_e32 v1, v0, v2
+; GFX9-NEXT:    v_mul_i32_i24_e32 v0, v0, v2
+; GFX9-NEXT:    v_lshrrev_b64 v[0:1], 24, v[0:1]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %A = sext i8 %X to i48
+  %B = sext i8 %Y to i48
+  %C = sext i8 %Z to i48
+  %D = sext i8 %W to i48
+  %mul0 = mul i48 %A, %B
+  %mul1 = mul i48 %C, %D
+  %mul2 = mul i48 %mul0, %mul1
+  %ashr = ashr i48 %mul2, 24
+  %trunc = trunc i48 %ashr to i16
+  ret i16 %trunc
+}
+
+define i32 @num_sign_bits_mul_i32_7(i32 %x, i32 %y, i32 %z, i32 %w) {
+; GFX9-LABEL: num_sign_bits_mul_i32_7:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_bfe_i32 v0, v0, 0, 25
+; GFX9-NEXT:    v_bfe_i32 v1, v1, 0, 25
+; GFX9-NEXT:    v_bfe_i32 v2, v2, 0, 25
+; GFX9-NEXT:    v_bfe_i32 v3, v3, 0, 25
+; GFX9-NEXT:    v_mul_lo_u32 v0, v0, v1
+; GFX9-NEXT:    v_mul_lo_u32 v1, v2, v3
+; GFX9-NEXT:    v_mul_lo_u32 v0, v0, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %x.shl = shl i32 %x, 7
+  %x.bits = ashr i32 %x.shl, 7
+
+  %y.shl = shl i32 %y, 7
+  %y.bits = ashr i32 %y.shl, 7
+
+  %z.shl = shl i32 %z, 7
+  %z.bits = ashr i32 %z.shl, 7
+
+  %w.shl = shl i32 %w, 7
+  %w.bits = ashr i32 %w.shl, 7
+
+  %mul0 = mul i32 %x.bits, %y.bits
+  %mul1 = mul i32 %z.bits, %w.bits
+  %mul2 = mul i32 %mul0, %mul1
+  ret i32 %mul2
+}
+
+define i32 @num_sign_bits_mul_i32_8(i32 %x, i32 %y, i32 %z, i32 %w) {
+; GFX9-LABEL: num_sign_bits_mul_i32_8:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mul_i32_i24_e32 v0, v0, v1
+; GFX9-NEXT:    v_mul_i32_i24_e32 v1, v2, v3
+; GFX9-NEXT:    v_mul_lo_u32 v0, v0, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %x.shl = shl i32 %x, 8
+  %x.bits = ashr i32 %x.shl, 8
+
+  %y.shl = shl i32 %y, 8
+  %y.bits = ashr i32 %y.shl, 8
+
+  %z.shl = shl i32 %z, 8
+  %z.bits = ashr i32 %z.shl, 8
+
+  %w.shl = shl i32 %w, 8
+  %w.bits = ashr i32 %w.shl, 8
+
+  %mul0 = mul i32 %x.bits, %y.bits
+  %mul1 = mul i32 %z.bits, %w.bits
+  %mul2 = mul i32 %mul0, %mul1
+  ret i32 %mul2
+}
+
+define i32 @num_sign_bits_mul_i32_9(i32 %x, i32 %y, i32 %z, i32 %w) {
+; GFX9-LABEL: num_sign_bits_mul_i32_9:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_bfe_i32 v0, v0, 0, 23
+; GFX9-NEXT:    v_bfe_i32 v1, v1, 0, 23
+; GFX9-NEXT:    v_bfe_i32 v2, v2, 0, 23
+; GFX9-NEXT:    v_bfe_i32 v3, v3, 0, 23
+; GFX9-NEXT:    v_mul_i32_i24_e32 v0, v0, v1
+; GFX9-NEXT:    v_mul_i32_i24_e32 v1, v2, v3
+; GFX9-NEXT:    v_mul_lo_u32 v0, v0, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %x.shl = shl i32 %x, 9
+  %x.bits = ashr i32 %x.shl, 9
+
+  %y.shl = shl i32 %y, 9
+  %y.bits = ashr i32 %y.shl, 9
+
+  %z.shl = shl i32 %z, 9
+  %z.bits = ashr i32 %z.shl, 9
+
+  %w.shl = shl i32 %w, 9
+  %w.bits = ashr i32 %w.shl, 9
+
+  %mul0 = mul i32 %x.bits, %y.bits
+  %mul1 = mul i32 %z.bits, %w.bits
+  %mul2 = mul i32 %mul0, %mul1
+  ret i32 %mul2
+}
+
+define i32 @num_sign_bits_mul_i32_10(i32 %x, i32 %y, i32 %z, i32 %w) {
+; GFX9-LABEL: num_sign_bits_mul_i32_10:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_bfe_i32 v0, v0, 0, 22
+; GFX9-NEXT:    v_bfe_i32 v1, v1, 0, 22
+; GFX9-NEXT:    v_bfe_i32 v2, v2, 0, 22
+; GFX9-NEXT:    v_bfe_i32 v3, v3, 0, 22
+; GFX9-NEXT:    v_mul_i32_i24_e32 v0, v0, v1
+; GFX9-NEXT:    v_mul_i32_i24_e32 v1, v2, v3
+; GFX9-NEXT:    v_mul_lo_u32 v0, v0, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %x.shl = shl i32 %x, 10
+  %x.bits = ashr i32 %x.shl, 10
+
+  %y.shl = shl i32 %y, 10
+  %y.bits = ashr i32 %y.shl, 10
+
+  %z.shl = shl i32 %z, 10
+  %z.bits = ashr i32 %z.shl, 10
+
+  %w.shl = shl i32 %w, 10
+  %w.bits = ashr i32 %w.shl, 10
+
+  %mul0 = mul i32 %x.bits, %y.bits
+  %mul1 = mul i32 %z.bits, %w.bits
+  %mul2 = mul i32 %mul0, %mul1
+  ret i32 %mul2
+}
+
+define i32 @known_bits_mul24() {
+; GFX9-LABEL: known_bits_mul24:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %r0 = call i32 @llvm.amdgcn.mul.i24(i32 0, i32 -7)
+  %r1 = shl i32 %r0, 2
+  ret i32 %r1
+}
+
+declare i32 @llvm.amdgcn.mul.i24(i32, i32)
diff --git a/llvm/test/CodeGen/AMDGPU/control-flow-fastregalloc.ll b/llvm/test/CodeGen/AMDGPU/control-flow-fastregalloc.ll
index 41ecdd403..15e807a3e 100644
--- a/llvm/test/CodeGen/AMDGPU/control-flow-fastregalloc.ll
+++ b/llvm/test/CodeGen/AMDGPU/control-flow-fastregalloc.ll
@@ -89,7 +89,7 @@ endif:
 }
 
 ; GCN-LABEL: {{^}}divergent_loop:
-; VGPR: workitem_private_segment_byte_size = 16{{$}}
+; VGPR: workitem_private_segment_byte_size = 12{{$}}
 
 ; GCN: {{^}}; %bb.0:
 
@@ -123,10 +123,9 @@ endif:
 ; GCN: [[LOOP:BB[0-9]+_[0-9]+]]:
 ; GCN: buffer_load_dword v[[VAL_LOOP_RELOAD:[0-9]+]], off, s[0:3], s7 offset:[[LOAD0_OFFSET]] ; 4-byte Folded Reload
 ; GCN: v_subrev_i32_e32 [[VAL_LOOP:v[0-9]+]], vcc, v{{[0-9]+}}, v[[VAL_LOOP_RELOAD]]
-; GCN: v_cmp_ne_u32_e32 vcc,
-; GCN: s_and_b64 vcc, exec, vcc
+; GCN: s_cmp_lg_u32
 ; GCN: buffer_store_dword [[VAL_LOOP]], off, s[0:3], s7 offset:[[VAL_SUB_OFFSET:[0-9]+]] ; 4-byte Folded Spill
-; GCN-NEXT: s_cbranch_vccnz [[LOOP]]
+; GCN-NEXT: s_cbranch_scc1 [[LOOP]]
 
 
 ; GCN: [[END]]:
diff --git a/llvm/test/CodeGen/AMDGPU/control-flow-optnone.ll b/llvm/test/CodeGen/AMDGPU/control-flow-optnone.ll
index 2122af627..8dda45fee 100644
--- a/llvm/test/CodeGen/AMDGPU/control-flow-optnone.ll
+++ b/llvm/test/CodeGen/AMDGPU/control-flow-optnone.ll
@@ -15,8 +15,8 @@
 ; GCN: s_mov_b64 exec
 
 ; GCN: s_or_b64 exec, exec
-; GCN: v_cmp_eq_u32
-; GCN: s_cbranch_vccnz
+; GCN: s_cmp_eq_u32
+; GCN: s_cbranch_scc1
 ; GCN-NEXT: s_branch
 define amdgpu_kernel void @copytoreg_divergent_brcond(i32 %arg, i32 %arg1, i32 %arg2) #0 {
 bb:
diff --git a/llvm/test/CodeGen/AMDGPU/copy-illegal-type.ll b/llvm/test/CodeGen/AMDGPU/copy-illegal-type.ll
index 91070878b..3529db39d 100644
--- a/llvm/test/CodeGen/AMDGPU/copy-illegal-type.ll
+++ b/llvm/test/CodeGen/AMDGPU/copy-illegal-type.ll
@@ -1,14 +1,45 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -march=amdgcn -mcpu=tahiti < %s | FileCheck -allow-deprecated-dag-overlap -check-prefix=GCN -check-prefix=SI -check-prefix=FUNC %s
 ; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -amdgpu-sdwa-peephole=0 < %s | FileCheck -allow-deprecated-dag-overlap -check-prefix=GCN -check-prefix=VI -check-prefix=FUNC %s
 
 declare i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
 declare i32 @llvm.amdgcn.workitem.id.y() nounwind readnone
 
-; FUNC-LABEL: {{^}}test_copy_v4i8:
-; GCN: {{buffer|flat}}_load_dword [[REG:v[0-9]+]]
-; GCN: buffer_store_dword [[REG]]
-; GCN: s_endpgm
 define amdgpu_kernel void @test_copy_v4i8(<4 x i8> addrspace(1)* %out, <4 x i8> addrspace(1)* %in) nounwind {
+; SI-LABEL: test_copy_v4i8:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s10, 0
+; SI-NEXT:    s_mov_b32 s11, s7
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[8:9], s[2:3]
+; SI-NEXT:    v_mov_b32_e32 v1, 0
+; SI-NEXT:    buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_mov_b32 s4, s0
+; SI-NEXT:    s_mov_b32 s5, s1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: test_copy_v4i8:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; VI-NEXT:    s_mov_b32 s7, 0xf000
+; VI-NEXT:    s_mov_b32 s6, -1
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dword v0, v[0:1]
+; VI-NEXT:    s_mov_b32 s4, s0
+; VI-NEXT:    s_mov_b32 s5, s1
+; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; VI-NEXT:    s_endpgm
   %tid.x = call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %in, i32 %tid.x
   %val = load <4 x i8>, <4 x i8> addrspace(1)* %gep, align 4
@@ -16,12 +47,48 @@ define amdgpu_kernel void @test_copy_v4i8(<4 x i8> addrspace(1)* %out, <4 x i8>
   ret void
 }
 
-; FUNC-LABEL: {{^}}test_copy_v4i8_x2:
-; GCN: {{buffer|flat}}_load_dword [[REG:v[0-9]+]]
-; GCN: buffer_store_dword [[REG]]
-; GCN: buffer_store_dword [[REG]]
-; GCN: s_endpgm
 define amdgpu_kernel void @test_copy_v4i8_x2(<4 x i8> addrspace(1)* %out0, <4 x i8> addrspace(1)* %out1, <4 x i8> addrspace(1)* %in) nounwind {
+; SI-LABEL: test_copy_v4i8_x2:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
+; SI-NEXT:    s_mov_b32 s11, 0xf000
+; SI-NEXT:    s_mov_b32 s2, 0
+; SI-NEXT:    s_mov_b32 s3, s11
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-NEXT:    v_mov_b32_e32 v1, 0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], 0 addr64
+; SI-NEXT:    s_mov_b32 s10, -1
+; SI-NEXT:    s_mov_b32 s8, s6
+; SI-NEXT:    s_mov_b32 s9, s7
+; SI-NEXT:    s_mov_b32 s6, s10
+; SI-NEXT:    s_mov_b32 s7, s11
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SI-NEXT:    buffer_store_dword v0, off, s[8:11], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: test_copy_v4i8_x2:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; VI-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0x34
+; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; VI-NEXT:    s_mov_b32 s3, 0xf000
+; VI-NEXT:    s_mov_b32 s2, -1
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_mov_b32 s0, s6
+; VI-NEXT:    v_mov_b32_e32 v1, s9
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s8, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dword v0, v[0:1]
+; VI-NEXT:    s_mov_b32 s1, s7
+; VI-NEXT:    s_mov_b32 s6, s2
+; VI-NEXT:    s_mov_b32 s7, s3
+; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT:    s_endpgm
   %tid.x = call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %in, i32 %tid.x
   %val = load <4 x i8>, <4 x i8> addrspace(1)* %gep, align 4
@@ -30,13 +97,57 @@ define amdgpu_kernel void @test_copy_v4i8_x2(<4 x i8> addrspace(1)* %out0, <4 x
   ret void
 }
 
-; FUNC-LABEL: {{^}}test_copy_v4i8_x3:
-; GCN: {{buffer|flat}}_load_dword [[REG:v[0-9]+]]
-; GCN: buffer_store_dword [[REG]]
-; GCN: buffer_store_dword [[REG]]
-; GCN: buffer_store_dword [[REG]]
-; GCN: s_endpgm
 define amdgpu_kernel void @test_copy_v4i8_x3(<4 x i8> addrspace(1)* %out0, <4 x i8> addrspace(1)* %out1, <4 x i8> addrspace(1)* %out2, <4 x i8> addrspace(1)* %in) nounwind {
+; SI-LABEL: test_copy_v4i8_x3:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx8 s[0:7], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s11, 0xf000
+; SI-NEXT:    s_mov_b32 s14, 0
+; SI-NEXT:    s_mov_b32 s15, s11
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[12:13], s[6:7]
+; SI-NEXT:    v_mov_b32_e32 v1, 0
+; SI-NEXT:    buffer_load_dword v0, v[0:1], s[12:15], 0 addr64
+; SI-NEXT:    s_mov_b32 s10, -1
+; SI-NEXT:    s_mov_b32 s8, s4
+; SI-NEXT:    s_mov_b32 s9, s5
+; SI-NEXT:    s_mov_b32 s4, s2
+; SI-NEXT:    s_mov_b32 s5, s3
+; SI-NEXT:    s_mov_b32 s6, s10
+; SI-NEXT:    s_mov_b32 s7, s11
+; SI-NEXT:    s_mov_b32 s2, s10
+; SI-NEXT:    s_mov_b32 s3, s11
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SI-NEXT:    buffer_store_dword v0, off, s[8:11], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: test_copy_v4i8_x3:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx8 s[0:7], s[0:1], 0x24
+; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; VI-NEXT:    s_mov_b32 s11, 0xf000
+; VI-NEXT:    s_mov_b32 s10, -1
+; VI-NEXT:    s_mov_b32 s14, s10
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s7
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s6, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dword v0, v[0:1]
+; VI-NEXT:    s_mov_b32 s12, s2
+; VI-NEXT:    s_mov_b32 s13, s3
+; VI-NEXT:    s_mov_b32 s8, s4
+; VI-NEXT:    s_mov_b32 s9, s5
+; VI-NEXT:    s_mov_b32 s15, s11
+; VI-NEXT:    s_mov_b32 s2, s10
+; VI-NEXT:    s_mov_b32 s3, s11
+; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT:    buffer_store_dword v0, off, s[12:15], 0
+; VI-NEXT:    buffer_store_dword v0, off, s[8:11], 0
+; VI-NEXT:    s_endpgm
   %tid.x = call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %in, i32 %tid.x
   %val = load <4 x i8>, <4 x i8> addrspace(1)* %gep, align 4
@@ -46,14 +157,65 @@ define amdgpu_kernel void @test_copy_v4i8_x3(<4 x i8> addrspace(1)* %out0, <4 x
   ret void
 }
 
-; FUNC-LABEL: {{^}}test_copy_v4i8_x4:
-; GCN: {{buffer|flat}}_load_dword [[REG:v[0-9]+]]
-; GCN: buffer_store_dword [[REG]]
-; GCN: buffer_store_dword [[REG]]
-; GCN: buffer_store_dword [[REG]]
-; GCN: buffer_store_dword [[REG]]
-; GCN: s_endpgm
 define amdgpu_kernel void @test_copy_v4i8_x4(<4 x i8> addrspace(1)* %out0, <4 x i8> addrspace(1)* %out1, <4 x i8> addrspace(1)* %out2, <4 x i8> addrspace(1)* %out3, <4 x i8> addrspace(1)* %in) nounwind {
+; SI-LABEL: test_copy_v4i8_x4:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0x11
+; SI-NEXT:    s_mov_b32 s15, 0xf000
+; SI-NEXT:    s_mov_b32 s10, 0
+; SI-NEXT:    s_mov_b32 s11, s15
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-NEXT:    v_mov_b32_e32 v1, 0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
+; SI-NEXT:    s_load_dwordx8 s[0:7], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s14, -1
+; SI-NEXT:    s_mov_b32 s18, s14
+; SI-NEXT:    s_mov_b32 s19, s15
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b32 s12, s6
+; SI-NEXT:    s_mov_b32 s13, s7
+; SI-NEXT:    s_mov_b32 s16, s2
+; SI-NEXT:    s_mov_b32 s17, s3
+; SI-NEXT:    s_mov_b32 s6, s14
+; SI-NEXT:    s_mov_b32 s7, s15
+; SI-NEXT:    s_mov_b32 s2, s14
+; SI-NEXT:    s_mov_b32 s3, s15
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT:    buffer_store_dword v0, off, s[16:19], 0
+; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SI-NEXT:    buffer_store_dword v0, off, s[12:15], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: test_copy_v4i8_x4:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0x44
+; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; VI-NEXT:    s_load_dwordx8 s[0:7], s[0:1], 0x24
+; VI-NEXT:    s_mov_b32 s11, 0xf000
+; VI-NEXT:    s_mov_b32 s10, -1
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s9
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s8, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dword v0, v[0:1]
+; VI-NEXT:    s_mov_b32 s8, s6
+; VI-NEXT:    s_mov_b32 s9, s7
+; VI-NEXT:    s_mov_b32 s12, s2
+; VI-NEXT:    s_mov_b32 s13, s3
+; VI-NEXT:    s_mov_b32 s6, s10
+; VI-NEXT:    s_mov_b32 s7, s11
+; VI-NEXT:    s_mov_b32 s14, s10
+; VI-NEXT:    s_mov_b32 s15, s11
+; VI-NEXT:    s_mov_b32 s2, s10
+; VI-NEXT:    s_mov_b32 s3, s11
+; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT:    buffer_store_dword v0, off, s[12:15], 0
+; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; VI-NEXT:    buffer_store_dword v0, off, s[8:11], 0
+; VI-NEXT:    s_endpgm
   %tid.x = call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %in, i32 %tid.x
   %val = load <4 x i8>, <4 x i8> addrspace(1)* %gep, align 4
@@ -64,16 +226,81 @@ define amdgpu_kernel void @test_copy_v4i8_x4(<4 x i8> addrspace(1)* %out0, <4 x
   ret void
 }
 
-; FUNC-LABEL: {{^}}test_copy_v4i8_extra_use:
-; GCN: {{buffer|flat}}_load_dword
-; GCN-DAG: v_lshrrev_b32
-; GCN: v_and_b32
-; GCN: v_or_b32
-; GCN-DAG: buffer_store_dword
-; GCN-DAG: buffer_store_dword
-
-; GCN: s_endpgm
 define amdgpu_kernel void @test_copy_v4i8_extra_use(<4 x i8> addrspace(1)* %out0, <4 x i8> addrspace(1)* %out1, <4 x i8> addrspace(1)* %in) nounwind {
+; SI-LABEL: test_copy_v4i8_extra_use:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
+; SI-NEXT:    s_mov_b32 s11, 0xf000
+; SI-NEXT:    s_mov_b32 s2, 0
+; SI-NEXT:    s_mov_b32 s3, s11
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-NEXT:    v_mov_b32_e32 v1, 0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], 0 addr64
+; SI-NEXT:    s_mov_b32 s10, -1
+; SI-NEXT:    s_mov_b32 s0, 0xff00
+; SI-NEXT:    s_mov_b32 s8, s6
+; SI-NEXT:    s_mov_b32 s9, s7
+; SI-NEXT:    s_mov_b32 s6, s10
+; SI-NEXT:    s_mov_b32 s7, s11
+; SI-NEXT:    s_movk_i32 s1, 0xff
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
+; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SI-NEXT:    v_and_b32_e32 v2, s0, v0
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_add_i32_e32 v0, vcc, 9, v0
+; SI-NEXT:    v_and_b32_e32 v0, s1, v0
+; SI-NEXT:    v_and_b32_e32 v3, s0, v1
+; SI-NEXT:    v_add_i32_e32 v1, vcc, 9, v1
+; SI-NEXT:    v_or_b32_e32 v0, v2, v0
+; SI-NEXT:    v_and_b32_e32 v1, s1, v1
+; SI-NEXT:    v_add_i32_e32 v0, vcc, 0x900, v0
+; SI-NEXT:    v_or_b32_e32 v1, v3, v1
+; SI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT:    v_or_b32_e32 v0, v1, v0
+; SI-NEXT:    v_add_i32_e32 v0, vcc, 0x9000000, v0
+; SI-NEXT:    buffer_store_dword v0, off, s[8:11], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: test_copy_v4i8_extra_use:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; VI-NEXT:    s_movk_i32 s8, 0xff00
+; VI-NEXT:    s_mov_b32 s3, 0xf000
+; VI-NEXT:    s_mov_b32 s2, -1
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dword v0, v[0:1]
+; VI-NEXT:    s_mov_b32 s0, s6
+; VI-NEXT:    s_mov_b32 s1, s7
+; VI-NEXT:    s_movk_i32 s9, 0xff
+; VI-NEXT:    s_mov_b32 s6, s2
+; VI-NEXT:    s_mov_b32 s7, s3
+; VI-NEXT:    s_movk_i32 s10, 0x900
+; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
+; VI-NEXT:    v_and_b32_e32 v3, s8, v1
+; VI-NEXT:    v_add_u16_e32 v1, 9, v1
+; VI-NEXT:    v_and_b32_e32 v1, s9, v1
+; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; VI-NEXT:    v_and_b32_e32 v2, s8, v0
+; VI-NEXT:    v_add_u16_e32 v0, 9, v0
+; VI-NEXT:    v_and_b32_e32 v0, s9, v0
+; VI-NEXT:    v_or_b32_e32 v1, v3, v1
+; VI-NEXT:    v_or_b32_e32 v0, v2, v0
+; VI-NEXT:    v_add_u16_e32 v1, s10, v1
+; VI-NEXT:    v_add_u16_e32 v0, s10, v0
+; VI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; VI-NEXT:    v_or_b32_e32 v0, v0, v1
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT:    s_endpgm
   %tid.x = call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %in, i32 %tid.x
   %val = load <4 x i8>, <4 x i8> addrspace(1)* %gep, align 4
@@ -84,18 +311,87 @@ define amdgpu_kernel void @test_copy_v4i8_extra_use(<4 x i8> addrspace(1)* %out0
 }
 
 ; FIXME: Need to handle non-uniform case for function below (load without gep).
-; FUNC-LABEL: {{^}}test_copy_v4i8_x2_extra_use:
-; GCN: {{buffer|flat}}_load_dword
-; GCN-DAG: v_lshrrev_b32
-; SI-DAG: v_add_i32
-; VI-DAG: v_add_u16
-; GCN-DAG: v_and_b32
-; GCN-DAG: v_or_b32
-; GCN-DAG: {{buffer|flat}}_store_dword
-; GCN: {{buffer|flat}}_store_dword
-; GCN: {{buffer|flat}}_store_dword
-; GCN: s_endpgm
 define amdgpu_kernel void @test_copy_v4i8_x2_extra_use(<4 x i8> addrspace(1)* %out0, <4 x i8> addrspace(1)* %out1, <4 x i8> addrspace(1)* %out2, <4 x i8> addrspace(1)* %in) nounwind {
+; SI-LABEL: test_copy_v4i8_x2_extra_use:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx8 s[4:11], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s14, 0
+; SI-NEXT:    s_mov_b32 s15, s3
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[12:13], s[10:11]
+; SI-NEXT:    v_mov_b32_e32 v1, 0
+; SI-NEXT:    buffer_load_dword v0, v[0:1], s[12:15], 0 addr64
+; SI-NEXT:    s_mov_b32 s16, 0xff00
+; SI-NEXT:    s_movk_i32 s17, 0xff
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_mov_b32 s0, s8
+; SI-NEXT:    s_mov_b32 s1, s9
+; SI-NEXT:    s_mov_b32 s8, s6
+; SI-NEXT:    s_mov_b32 s9, s7
+; SI-NEXT:    s_mov_b32 s10, s2
+; SI-NEXT:    s_mov_b32 s11, s3
+; SI-NEXT:    s_mov_b32 s6, s2
+; SI-NEXT:    s_mov_b32 s7, s3
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_add_i32_e32 v3, vcc, 9, v0
+; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
+; SI-NEXT:    v_and_b32_e32 v4, s16, v1
+; SI-NEXT:    v_add_i32_e32 v1, vcc, 9, v1
+; SI-NEXT:    v_and_b32_e32 v2, s16, v0
+; SI-NEXT:    v_and_b32_e32 v3, s17, v3
+; SI-NEXT:    v_or_b32_e32 v2, v2, v3
+; SI-NEXT:    v_and_b32_e32 v1, s17, v1
+; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x900, v2
+; SI-NEXT:    v_or_b32_e32 v1, v4, v1
+; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT:    v_or_b32_e32 v1, v1, v2
+; SI-NEXT:    v_add_i32_e32 v1, vcc, 0x9000000, v1
+; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SI-NEXT:    buffer_store_dword v1, off, s[8:11], 0
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: test_copy_v4i8_x2_extra_use:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx8 s[0:7], s[0:1], 0x24
+; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; VI-NEXT:    s_movk_i32 s12, 0xff00
+; VI-NEXT:    s_movk_i32 s13, 0xff
+; VI-NEXT:    s_movk_i32 s14, 0x900
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s7
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s6, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dword v0, v[0:1]
+; VI-NEXT:    s_mov_b32 s7, 0xf000
+; VI-NEXT:    s_mov_b32 s6, -1
+; VI-NEXT:    s_mov_b32 s8, s2
+; VI-NEXT:    s_mov_b32 s9, s3
+; VI-NEXT:    s_mov_b32 s10, s6
+; VI-NEXT:    s_mov_b32 s11, s7
+; VI-NEXT:    s_mov_b32 s2, s6
+; VI-NEXT:    s_mov_b32 s3, s7
+; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
+; VI-NEXT:    v_and_b32_e32 v4, s12, v1
+; VI-NEXT:    v_add_u16_e32 v1, 9, v1
+; VI-NEXT:    v_add_u16_e32 v3, 9, v0
+; VI-NEXT:    v_and_b32_e32 v1, s13, v1
+; VI-NEXT:    v_or_b32_e32 v1, v4, v1
+; VI-NEXT:    v_and_b32_e32 v2, s12, v0
+; VI-NEXT:    v_and_b32_e32 v3, s13, v3
+; VI-NEXT:    v_or_b32_e32 v2, v2, v3
+; VI-NEXT:    v_add_u16_e32 v1, s14, v1
+; VI-NEXT:    v_add_u16_e32 v2, s14, v2
+; VI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; VI-NEXT:    v_or_b32_e32 v1, v2, v1
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT:    buffer_store_dword v1, off, s[8:11], 0
+; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; VI-NEXT:    s_endpgm
   %tid.x = call i32 @llvm.amdgcn.workitem.id.x()
   %in.ptr = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %in, i32 %tid.x
   %val = load <4 x i8>, <4 x i8> addrspace(1)* %in.ptr, align 4
@@ -106,12 +402,45 @@ define amdgpu_kernel void @test_copy_v4i8_x2_extra_use(<4 x i8> addrspace(1)* %o
   ret void
 }
 
-; FUNC-LABEL: {{^}}test_copy_v3i8_align4:
-; GCN: {{buffer|flat}}_load_dword
-; GCN-DAG: buffer_store_short v{{[0-9]+}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0{{$}}
-; GCN-DAG: buffer_store_byte v{{[0-9]+}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0 offset:2{{$}}
-; GCN: s_endpgm
 define amdgpu_kernel void @test_copy_v3i8_align4(<3 x i8> addrspace(1)* %out, <3 x i8> addrspace(1)* %in) nounwind {
+; SI-LABEL: test_copy_v3i8_align4:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s10, 0
+; SI-NEXT:    s_mov_b32 s11, s7
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[8:9], s[2:3]
+; SI-NEXT:    v_mov_b32_e32 v1, 0
+; SI-NEXT:    buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_mov_b32 s4, s0
+; SI-NEXT:    s_mov_b32 s5, s1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
+; SI-NEXT:    buffer_store_short v0, off, s[4:7], 0
+; SI-NEXT:    buffer_store_byte v1, off, s[4:7], 0 offset:2
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: test_copy_v3i8_align4:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; VI-NEXT:    s_mov_b32 s7, 0xf000
+; VI-NEXT:    s_mov_b32 s6, -1
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dword v0, v[0:1]
+; VI-NEXT:    s_mov_b32 s4, s0
+; VI-NEXT:    s_mov_b32 s5, s1
+; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_store_short v0, off, s[4:7], 0
+; VI-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; VI-NEXT:    buffer_store_byte v0, off, s[4:7], 0 offset:2
+; VI-NEXT:    s_endpgm
   %tid.x = call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr <3 x i8>, <3 x i8> addrspace(1)* %in, i32 %tid.x
   %val = load <3 x i8>, <3 x i8> addrspace(1)* %gep, align 4
@@ -119,54 +448,185 @@ define amdgpu_kernel void @test_copy_v3i8_align4(<3 x i8> addrspace(1)* %out, <3
   ret void
 }
 
-; FUNC-LABEL: {{^}}test_copy_v3i8_align2:
-; GCN-DAG: {{buffer|flat}}_load_ushort v{{[0-9]+}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0{{$}}
-; GCN-DAG: {{buffer|flat}}_load_ubyte v{{[0-9]+}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0 offset:2{{$}}
-; GCN-DAG: buffer_store_short v{{[0-9]+}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0{{$}}
-; GCN-DAG: buffer_store_byte v{{[0-9]+}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0 offset:2{{$}}
-; GCN: s_endpgm
 define amdgpu_kernel void @test_copy_v3i8_align2(<3 x i8> addrspace(1)* %out, <3 x i8> addrspace(1)* %in) nounwind {
+; SI-LABEL: test_copy_v3i8_align2:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_mov_b32 s10, s2
+; SI-NEXT:    s_mov_b32 s11, s3
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b32 s8, s6
+; SI-NEXT:    s_mov_b32 s9, s7
+; SI-NEXT:    buffer_load_ushort v0, off, s[8:11], 0
+; SI-NEXT:    buffer_load_ubyte v1, off, s[8:11], 0 offset:2
+; SI-NEXT:    s_mov_b32 s0, s4
+; SI-NEXT:    s_mov_b32 s1, s5
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_byte v1, off, s[0:3], 0 offset:2
+; SI-NEXT:    buffer_store_short v0, off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: test_copy_v3i8_align2:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; VI-NEXT:    s_mov_b32 s3, 0xf000
+; VI-NEXT:    s_mov_b32 s2, -1
+; VI-NEXT:    s_mov_b32 s10, s2
+; VI-NEXT:    s_mov_b32 s11, s3
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_mov_b32 s8, s6
+; VI-NEXT:    s_mov_b32 s9, s7
+; VI-NEXT:    buffer_load_ushort v0, off, s[8:11], 0
+; VI-NEXT:    buffer_load_ubyte v1, off, s[8:11], 0 offset:2
+; VI-NEXT:    s_mov_b32 s0, s4
+; VI-NEXT:    s_mov_b32 s1, s5
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_store_byte v1, off, s[0:3], 0 offset:2
+; VI-NEXT:    buffer_store_short v0, off, s[0:3], 0
+; VI-NEXT:    s_endpgm
   %val = load <3 x i8>, <3 x i8> addrspace(1)* %in, align 2
   store <3 x i8> %val, <3 x i8> addrspace(1)* %out, align 2
   ret void
 }
 
-; FUNC-LABEL: {{^}}test_copy_v3i8_align1:
-; GCN: {{buffer|flat}}_load_ubyte
-; GCN: {{buffer|flat}}_load_ubyte
-; GCN: {{buffer|flat}}_load_ubyte
-
-; GCN: buffer_store_byte
-; GCN: buffer_store_byte
-; GCN: buffer_store_byte
-; GCN: s_endpgm
 define amdgpu_kernel void @test_copy_v3i8_align1(<3 x i8> addrspace(1)* %out, <3 x i8> addrspace(1)* %in) nounwind {
+; SI-LABEL: test_copy_v3i8_align1:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_mov_b32 s10, s2
+; SI-NEXT:    s_mov_b32 s11, s3
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b32 s8, s6
+; SI-NEXT:    s_mov_b32 s9, s7
+; SI-NEXT:    buffer_load_ubyte v0, off, s[8:11], 0
+; SI-NEXT:    buffer_load_ubyte v1, off, s[8:11], 0 offset:1
+; SI-NEXT:    buffer_load_ubyte v2, off, s[8:11], 0 offset:2
+; SI-NEXT:    s_mov_b32 s0, s4
+; SI-NEXT:    s_mov_b32 s1, s5
+; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    buffer_store_byte v0, off, s[0:3], 0
+; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    buffer_store_byte v1, off, s[0:3], 0 offset:1
+; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    buffer_store_byte v2, off, s[0:3], 0 offset:2
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: test_copy_v3i8_align1:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; VI-NEXT:    s_mov_b32 s3, 0xf000
+; VI-NEXT:    s_mov_b32 s2, -1
+; VI-NEXT:    s_mov_b32 s10, s2
+; VI-NEXT:    s_mov_b32 s11, s3
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_mov_b32 s8, s6
+; VI-NEXT:    s_mov_b32 s9, s7
+; VI-NEXT:    buffer_load_ubyte v0, off, s[8:11], 0
+; VI-NEXT:    buffer_load_ubyte v1, off, s[8:11], 0 offset:1
+; VI-NEXT:    buffer_load_ubyte v2, off, s[8:11], 0 offset:2
+; VI-NEXT:    s_mov_b32 s0, s4
+; VI-NEXT:    s_mov_b32 s1, s5
+; VI-NEXT:    s_waitcnt vmcnt(2)
+; VI-NEXT:    buffer_store_byte v0, off, s[0:3], 0
+; VI-NEXT:    s_waitcnt vmcnt(2)
+; VI-NEXT:    buffer_store_byte v1, off, s[0:3], 0 offset:1
+; VI-NEXT:    s_waitcnt vmcnt(2)
+; VI-NEXT:    buffer_store_byte v2, off, s[0:3], 0 offset:2
+; VI-NEXT:    s_endpgm
   %val = load <3 x i8>, <3 x i8> addrspace(1)* %in, align 1
   store <3 x i8> %val, <3 x i8> addrspace(1)* %out, align 1
   ret void
 }
 
-; FUNC-LABEL: {{^}}test_copy_v4i8_volatile_load:
-; GCN: {{buffer|flat}}_load_dword
-; GCN: buffer_store_dword
-; GCN: s_endpgm
 define amdgpu_kernel void @test_copy_v4i8_volatile_load(<4 x i8> addrspace(1)* %out, <4 x i8> addrspace(1)* %in) nounwind {
+; SI-LABEL: test_copy_v4i8_volatile_load:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b32 s0, s4
+; SI-NEXT:    s_mov_b32 s1, s5
+; SI-NEXT:    s_mov_b32 s4, s6
+; SI-NEXT:    s_mov_b32 s5, s7
+; SI-NEXT:    s_mov_b32 s6, s2
+; SI-NEXT:    s_mov_b32 s7, s3
+; SI-NEXT:    buffer_load_dword v0, off, s[4:7], 0
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: test_copy_v4i8_volatile_load:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; VI-NEXT:    s_mov_b32 s3, 0xf000
+; VI-NEXT:    s_mov_b32 s2, -1
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_mov_b32 s0, s4
+; VI-NEXT:    s_mov_b32 s1, s5
+; VI-NEXT:    s_mov_b32 s4, s6
+; VI-NEXT:    s_mov_b32 s5, s7
+; VI-NEXT:    s_mov_b32 s6, s2
+; VI-NEXT:    s_mov_b32 s7, s3
+; VI-NEXT:    buffer_load_dword v0, off, s[4:7], 0
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT:    s_endpgm
   %val = load volatile <4 x i8>, <4 x i8> addrspace(1)* %in, align 4
   store <4 x i8> %val, <4 x i8> addrspace(1)* %out, align 4
   ret void
 }
 
-; FUNC-LABEL: {{^}}test_copy_v4i8_volatile_store:
-; GCN: {{buffer|flat}}_load_ubyte
-; GCN: {{buffer|flat}}_load_ubyte
-; GCN: {{buffer|flat}}_load_ubyte
-; GCN: {{buffer|flat}}_load_ubyte
-; GCN: buffer_store_byte
-; GCN: buffer_store_byte
-; GCN: buffer_store_byte
-; GCN: buffer_store_byte
-; GCN: s_endpgm
 define amdgpu_kernel void @test_copy_v4i8_volatile_store(<4 x i8> addrspace(1)* %out, <4 x i8> addrspace(1)* %in) nounwind {
+; SI-LABEL: test_copy_v4i8_volatile_store:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_mov_b32 s10, s2
+; SI-NEXT:    s_mov_b32 s11, s3
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b32 s8, s6
+; SI-NEXT:    s_mov_b32 s9, s7
+; SI-NEXT:    buffer_load_ubyte v0, off, s[8:11], 0
+; SI-NEXT:    buffer_load_ubyte v1, off, s[8:11], 0 offset:1
+; SI-NEXT:    buffer_load_ubyte v2, off, s[8:11], 0 offset:2
+; SI-NEXT:    buffer_load_ubyte v3, off, s[8:11], 0 offset:3
+; SI-NEXT:    s_mov_b32 s0, s4
+; SI-NEXT:    s_mov_b32 s1, s5
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_byte v3, off, s[0:3], 0 offset:3
+; SI-NEXT:    buffer_store_byte v2, off, s[0:3], 0 offset:2
+; SI-NEXT:    buffer_store_byte v1, off, s[0:3], 0 offset:1
+; SI-NEXT:    buffer_store_byte v0, off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: test_copy_v4i8_volatile_store:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; VI-NEXT:    s_mov_b32 s3, 0xf000
+; VI-NEXT:    s_mov_b32 s2, -1
+; VI-NEXT:    s_mov_b32 s10, s2
+; VI-NEXT:    s_mov_b32 s11, s3
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_mov_b32 s8, s6
+; VI-NEXT:    s_mov_b32 s9, s7
+; VI-NEXT:    buffer_load_ubyte v0, off, s[8:11], 0
+; VI-NEXT:    buffer_load_ubyte v1, off, s[8:11], 0 offset:1
+; VI-NEXT:    buffer_load_ubyte v2, off, s[8:11], 0 offset:2
+; VI-NEXT:    buffer_load_ubyte v3, off, s[8:11], 0 offset:3
+; VI-NEXT:    s_mov_b32 s0, s4
+; VI-NEXT:    s_mov_b32 s1, s5
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_store_byte v3, off, s[0:3], 0 offset:3
+; VI-NEXT:    buffer_store_byte v2, off, s[0:3], 0 offset:2
+; VI-NEXT:    buffer_store_byte v1, off, s[0:3], 0 offset:1
+; VI-NEXT:    buffer_store_byte v0, off, s[0:3], 0
+; VI-NEXT:    s_endpgm
   %val = load <4 x i8>, <4 x i8> addrspace(1)* %in, align 4
   store volatile <4 x i8> %val, <4 x i8> addrspace(1)* %out, align 4
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/cross-block-use-is-not-abi-copy.ll b/llvm/test/CodeGen/AMDGPU/cross-block-use-is-not-abi-copy.ll
index a0a52fbe3..019beea0a 100644
--- a/llvm/test/CodeGen/AMDGPU/cross-block-use-is-not-abi-copy.ll
+++ b/llvm/test/CodeGen/AMDGPU/cross-block-use-is-not-abi-copy.ll
@@ -165,11 +165,100 @@ bb1:
   ret { i32, half } %ins1
 }
 
+define amdgpu_kernel void @v3i16_registers(i1 %cond) #0 {
+; GCN-LABEL: v3i16_registers:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_load_dword s4, s[4:5], 0x0
+; GCN-NEXT:    s_mov_b32 s33, s9
+; GCN-NEXT:    s_add_u32 flat_scratch_lo, s6, s33
+; GCN-NEXT:    s_addc_u32 flat_scratch_hi, s7, 0
+; GCN-NEXT:    s_mov_b32 s32, s33
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_and_b32 s4, 1, s4
+; GCN-NEXT:    v_cmp_eq_u32_e64 s[4:5], s4, 1
+; GCN-NEXT:    s_and_b64 vcc, exec, s[4:5]
+; GCN-NEXT:    s_cbranch_vccz BB4_2
+; GCN-NEXT:  ; %bb.1:
+; GCN-NEXT:    s_mov_b32 s4, 0
+; GCN-NEXT:    s_mov_b32 s5, s4
+; GCN-NEXT:    v_mov_b32_e32 v0, s4
+; GCN-NEXT:    v_mov_b32_e32 v1, s5
+; GCN-NEXT:    s_branch BB4_3
+; GCN-NEXT:  BB4_2: ; %if.else
+; GCN-NEXT:    s_getpc_b64 s[4:5]
+; GCN-NEXT:    s_add_u32 s4, s4, func_v3i16@rel32@lo+4
+; GCN-NEXT:    s_addc_u32 s5, s5, func_v3i16@rel32@hi+4
+; GCN-NEXT:    s_swappc_b64 s[30:31], s[4:5]
+; GCN-NEXT:  BB4_3: ; %if.end
+; GCN-NEXT:    global_store_short v[0:1], v1, off
+; GCN-NEXT:    global_store_dword v[0:1], v0, off
+; GCN-NEXT:    s_endpgm
+entry:
+  br i1 %cond, label %if.then, label %if.else
+
+if.then:                                          ; preds = %entry
+  br label %if.end
+
+if.else:                                          ; preds = %entry
+  %call6 = tail call <3 x i16> @func_v3i16() #0
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  %call6.sink = phi <3 x i16> [ %call6, %if.else ], [ undef, %if.then ]
+  store <3 x i16> %call6.sink, <3 x i16> addrspace(1)* undef
+  ret void
+}
+
+define amdgpu_kernel void @v3f16_registers(i1 %cond) #0 {
+; GCN-LABEL: v3f16_registers:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_load_dword s4, s[4:5], 0x0
+; GCN-NEXT:    s_mov_b32 s33, s9
+; GCN-NEXT:    s_add_u32 flat_scratch_lo, s6, s33
+; GCN-NEXT:    s_addc_u32 flat_scratch_hi, s7, 0
+; GCN-NEXT:    s_mov_b32 s32, s33
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_and_b32 s4, 1, s4
+; GCN-NEXT:    v_cmp_eq_u32_e64 s[4:5], s4, 1
+; GCN-NEXT:    s_and_b64 vcc, exec, s[4:5]
+; GCN-NEXT:    s_cbranch_vccz BB5_2
+; GCN-NEXT:  ; %bb.1:
+; GCN-NEXT:    s_mov_b32 s4, 0
+; GCN-NEXT:    s_mov_b32 s5, s4
+; GCN-NEXT:    v_mov_b32_e32 v0, s4
+; GCN-NEXT:    v_mov_b32_e32 v1, s5
+; GCN-NEXT:    s_branch BB5_3
+; GCN-NEXT:  BB5_2: ; %if.else
+; GCN-NEXT:    s_getpc_b64 s[4:5]
+; GCN-NEXT:    s_add_u32 s4, s4, func_v3f16@rel32@lo+4
+; GCN-NEXT:    s_addc_u32 s5, s5, func_v3f16@rel32@hi+4
+; GCN-NEXT:    s_swappc_b64 s[30:31], s[4:5]
+; GCN-NEXT:  BB5_3: ; %if.end
+; GCN-NEXT:    global_store_short v[0:1], v1, off
+; GCN-NEXT:    global_store_dword v[0:1], v0, off
+; GCN-NEXT:    s_endpgm
+entry:
+  br i1 %cond, label %if.then, label %if.else
+
+if.then:                                          ; preds = %entry
+  br label %if.end
+
+if.else:                                          ; preds = %entry
+  %call6 = tail call <3 x half> @func_v3f16() #0
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  %call6.sink = phi <3 x half> [ %call6, %if.else ], [ undef, %if.then ]
+  store <3 x half> %call6.sink, <3 x half> addrspace(1)* undef
+  ret void
+}
 
 declare hidden <2 x float> @func_v2f32() #0
 declare hidden <3 x float> @func_v3f32() #0
 declare hidden <4 x float> @func_v4f32() #0
 declare hidden <4 x half> @func_v4f16() #0
+declare hidden <3 x i16> @func_v3i16()
+declare hidden <3 x half> @func_v3f16()
 
 declare hidden { <4 x i32>, <4 x half> } @func_struct() #0
 
diff --git a/llvm/test/CodeGen/AMDGPU/cse-phi-incoming-val.ll b/llvm/test/CodeGen/AMDGPU/cse-phi-incoming-val.ll
new file mode 100644
index 000000000..6b3491b0c
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/cse-phi-incoming-val.ll
@@ -0,0 +1,42 @@
+; RUN: llc < %s -mtriple=amdgcn--amdpal -mcpu=gfx900 -verify-machineinstrs | FileCheck %s
+
+; Check that the redundant immediate MOV instruction
+; (by-product of handling phi nodes) is not found
+; in the generated code.
+
+; CHECK-LABEL: {{^}}mov_opt:
+; CHECK: s_mov_b32 [[SREG:s[0-9]+]], 1.0
+; CHECK: %bb.1:
+; CHECK-NOT: v_mov_b32_e32 {{v[0-9]+}}, 1.0
+; CHECK: BB0_4:
+; CHECK: v_mov_b32_e32 v{{[0-9]+}}, [[SREG]]
+
+define amdgpu_ps void @mov_opt(i32 %arg, i32 inreg %arg1, i32 inreg %arg2) local_unnamed_addr #0 {
+bb:
+  %tmp = icmp eq i32 %arg1, 0
+  br i1 %tmp, label %bb3, label %bb10
+
+bb3:                                              ; preds = %bb
+  %tmp4 = icmp eq i32 %arg2, 0
+  br i1 %tmp4, label %bb5, label %bb10
+
+bb5:                                              ; preds = %bb3
+  %tmp6 = getelementptr <{ [4294967295 x i32] }>, <{ [4294967295 x i32] }> addrspace(6)* null, i32 0, i32 0, i32 %arg
+  %tmp7 = load i32, i32 addrspace(6)* %tmp6
+  %tmp8 = icmp eq i32 %tmp7, 1
+  br i1 %tmp8, label %bb10, label %bb9
+
+bb9:                                              ; preds = %bb5
+  br label %bb10
+
+bb10:                                             ; preds = %bb9, %bb5, %bb3, %bb
+  %tmp11 = phi float [ 1.000000e+00, %bb3 ], [ 0.000000e+00, %bb9 ], [ 1.000000e+00, %bb ], [ undef, %bb5 ]
+  call void @llvm.amdgcn.exp.f32(i32 immarg 40, i32 immarg 15, float %tmp11, float undef, float undef, float undef, i1 immarg false, i1 immarg false) #0
+  ret void
+}
+
+; Function Attrs: inaccessiblememonly nounwind
+declare void @llvm.amdgcn.exp.f32(i32 immarg, i32 immarg, float, float, float, float, i1 immarg, i1 immarg) #1
+
+attributes #0 = { nounwind }
+attributes #1 = { inaccessiblememonly nounwind }
diff --git a/llvm/test/CodeGen/AMDGPU/ctpop.ll b/llvm/test/CodeGen/AMDGPU/ctpop.ll
index df661b8f4..70ac38d94 100644
--- a/llvm/test/CodeGen/AMDGPU/ctpop.ll
+++ b/llvm/test/CodeGen/AMDGPU/ctpop.ll
@@ -284,7 +284,7 @@ define amdgpu_kernel void @v_ctpop_i32_add_var_inv(i32 addrspace(1)* noalias %ou
 ; SI: v_bcnt_u32_b32_e32 [[RESULT:v[0-9]+]], [[VAR]], [[VAL]]
 ; VI: flat_load_dword [[VAR:v[0-9]+]], v[{{[0-9]+:[0-9]+}}]
 ; VI: flat_load_dword [[VAL:v[0-9]+]], v[{{[0-9]+:[0-9]+}}]
-; VI: v_bcnt_u32_b32 [[RESULT:v[0-9]+]], [[VAL]], [[VAR]]
+; VI: v_bcnt_u32_b32 [[RESULT:v[0-9]+]], [[VAR]], [[VAL]]
 ; GCN: buffer_store_dword [[RESULT]],
 ; GCN: s_endpgm
 
diff --git a/llvm/test/CodeGen/AMDGPU/ctpop16.ll b/llvm/test/CodeGen/AMDGPU/ctpop16.ll
index 9e774d151..5990697fc 100644
--- a/llvm/test/CodeGen/AMDGPU/ctpop16.ll
+++ b/llvm/test/CodeGen/AMDGPU/ctpop16.ll
@@ -285,7 +285,7 @@ define amdgpu_kernel void @v_ctpop_i16_add_var_inv(i16 addrspace(1)* noalias %ou
 ; SI: v_bcnt_u32_b32_e32 [[RESULT:v[0-9]+]], [[VAR]], [[VAL]]
 ; VI: flat_load_ushort [[VAR:v[0-9]+]], v[{{[0-9]+:[0-9]+}}]
 ; VI: flat_load_ushort [[VAL:v[0-9]+]], v[{{[0-9]+:[0-9]+}}]
-; VI: v_bcnt_u32_b32 [[RESULT:v[0-9]+]], [[VAL]], [[VAR]]
+; VI: v_bcnt_u32_b32 [[RESULT:v[0-9]+]], [[VAR]], [[VAL]]
 ; GCN: buffer_store_short [[RESULT]],
 ; GCN: s_endpgm
 
diff --git a/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll b/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll
index 4f693e204..941e32aae 100644
--- a/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll
+++ b/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll
@@ -236,19 +236,21 @@ define amdgpu_kernel void @load_v4i8_to_v4f32_unaligned(<4 x float> addrspace(1)
 ; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
 ; VI-NEXT:    v_add_u32_e32 v6, vcc, 1, v0
 ; VI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v1, vcc
-; VI-NEXT:    flat_load_ubyte v0, v[0:1]
-; VI-NEXT:    flat_load_ubyte v1, v[6:7]
-; VI-NEXT:    flat_load_ubyte v4, v[4:5]
 ; VI-NEXT:    flat_load_ubyte v2, v[2:3]
+; VI-NEXT:    flat_load_ubyte v3, v[4:5]
+; VI-NEXT:    flat_load_ubyte v4, v[6:7]
+; VI-NEXT:    flat_load_ubyte v0, v[0:1]
+; VI-NEXT:    s_waitcnt vmcnt(3) lgkmcnt(3)
+; VI-NEXT:    v_lshlrev_b32_e32 v1, 8, v2
 ; VI-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(2)
-; VI-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
-; VI-NEXT:    v_or_b32_e32 v0, v1, v0
+; VI-NEXT:    v_or_b32_e32 v1, v1, v3
+; VI-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)
+; VI-NEXT:    v_lshlrev_b32_e32 v2, 8, v4
+; VI-NEXT:    v_lshlrev_b32_e32 v3, 16, v1
 ; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
-; VI-NEXT:    v_or_b32_e32 v2, v2, v4
-; VI-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
+; VI-NEXT:    v_or_b32_e32 v0, v2, v0
+; VI-NEXT:    v_cvt_f32_ubyte0_e32 v2, v1
 ; VI-NEXT:    v_cvt_f32_ubyte1_e32 v1, v0
-; VI-NEXT:    v_cvt_f32_ubyte0_e32 v2, v2
 ; VI-NEXT:    v_cvt_f32_ubyte0_e32 v0, v0
 ; VI-NEXT:    v_cvt_f32_ubyte3_e32 v3, v3
 ; VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
@@ -280,7 +282,6 @@ define amdgpu_kernel void @load_v4i8_to_v4f32_2_uses(<4 x float> addrspace(1)* n
 ; SI-NEXT:    s_mov_b32 s10, s2
 ; SI-NEXT:    s_mov_b32 s11, s3
 ; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
-; SI-NEXT:    s_movk_i32 s13, 0x900
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_lshrrev_b32_e32 v4, 16, v1
 ; SI-NEXT:    v_add_i32_e32 v7, vcc, 9, v1
@@ -298,7 +299,7 @@ define amdgpu_kernel void @load_v4i8_to_v4f32_2_uses(<4 x float> addrspace(1)* n
 ; SI-NEXT:    v_or_b32_e32 v0, v6, v7
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 8, v5
 ; SI-NEXT:    v_and_b32_e32 v1, s12, v4
-; SI-NEXT:    v_add_i32_e32 v0, vcc, s13, v0
+; SI-NEXT:    v_add_i32_e32 v0, vcc, 0x900, v0
 ; SI-NEXT:    v_or_b32_e32 v1, v5, v1
 ; SI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
@@ -415,35 +416,34 @@ define amdgpu_kernel void @load_v7i8_to_v7f32(<7 x float> addrspace(1)* noalias
 ; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v1, vcc
 ; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v0
 ; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_ubyte v9, v[2:3]
 ; VI-NEXT:    flat_load_ubyte v10, v[4:5]
-; VI-NEXT:    flat_load_ubyte v11, v[2:3]
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 2, v0
 ; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v1, vcc
 ; VI-NEXT:    v_add_u32_e32 v4, vcc, 5, v0
 ; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
 ; VI-NEXT:    v_add_u32_e32 v6, vcc, 4, v0
 ; VI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v1, vcc
-; VI-NEXT:    v_add_u32_e32 v8, vcc, 6, v0
-; VI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v1, vcc
-; VI-NEXT:    flat_load_ubyte v0, v[0:1]
-; VI-NEXT:    flat_load_ubyte v1, v[8:9]
-; VI-NEXT:    flat_load_ubyte v7, v[6:7]
-; VI-NEXT:    flat_load_ubyte v4, v[4:5]
+; VI-NEXT:    flat_load_ubyte v8, v[0:1]
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 6, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; VI-NEXT:    flat_load_ubyte v2, v[2:3]
+; VI-NEXT:    flat_load_ubyte v3, v[4:5]
+; VI-NEXT:    flat_load_ubyte v4, v[6:7]
+; VI-NEXT:    flat_load_ubyte v0, v[0:1]
 ; VI-NEXT:    s_waitcnt vmcnt(6) lgkmcnt(6)
-; VI-NEXT:    v_lshlrev_b32_e32 v5, 8, v10
+; VI-NEXT:    v_lshlrev_b32_e32 v1, 8, v9
 ; VI-NEXT:    s_waitcnt vmcnt(5) lgkmcnt(5)
-; VI-NEXT:    v_lshlrev_b32_e32 v3, 8, v11
-; VI-NEXT:    s_waitcnt vmcnt(4) lgkmcnt(4)
-; VI-NEXT:    v_or_b32_e32 v0, v3, v0
-; VI-NEXT:    s_waitcnt vmcnt(3) lgkmcnt(3)
-; VI-NEXT:    v_cvt_f32_ubyte0_e32 v6, v1
+; VI-NEXT:    v_lshlrev_b32_e32 v5, 8, v10
+; VI-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(2)
+; VI-NEXT:    v_lshlrev_b32_e32 v3, 8, v3
 ; VI-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)
-; VI-NEXT:    v_lshlrev_b32_e32 v4, 8, v4
+; VI-NEXT:    v_or_b32_e32 v4, v3, v4
 ; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cvt_f32_ubyte0_e32 v6, v0
+; VI-NEXT:    v_or_b32_e32 v0, v1, v8
 ; VI-NEXT:    v_or_b32_sdwa v1, v5, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v0, v1, v0
-; VI-NEXT:    v_or_b32_e32 v4, v4, v7
 ; VI-NEXT:    v_and_b32_e32 v5, 0xffff0000, v0
 ; VI-NEXT:    v_or_b32_e32 v4, v4, v5
 ; VI-NEXT:    v_cvt_f32_ubyte1_e32 v5, v4
@@ -700,23 +700,24 @@ define amdgpu_kernel void @v4i8_zext_v4i32_to_v4f32(<4 x float> addrspace(1)* no
 ; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v1, vcc
 ; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v0
 ; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; VI-NEXT:    v_add_u32_e32 v6, vcc, 2, v0
-; VI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v1, vcc
-; VI-NEXT:    flat_load_ubyte v0, v[0:1]
-; VI-NEXT:    flat_load_ubyte v1, v[6:7]
-; VI-NEXT:    flat_load_ubyte v4, v[4:5]
+; VI-NEXT:    flat_load_ubyte v6, v[0:1]
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 2, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; VI-NEXT:    flat_load_ubyte v2, v[2:3]
+; VI-NEXT:    flat_load_ubyte v3, v[4:5]
+; VI-NEXT:    flat_load_ubyte v0, v[0:1]
+; VI-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(2)
+; VI-NEXT:    v_lshlrev_b32_e32 v1, 8, v2
 ; VI-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)
-; VI-NEXT:    v_lshlrev_b32_e32 v3, 8, v4
+; VI-NEXT:    v_lshlrev_b32_e32 v2, 8, v3
+; VI-NEXT:    v_or_b32_e32 v1, v1, v6
 ; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
-; VI-NEXT:    v_or_b32_e32 v4, v2, v0
-; VI-NEXT:    v_or_b32_sdwa v0, v3, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    v_or_b32_e32 v0, v0, v4
+; VI-NEXT:    v_or_b32_sdwa v0, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v0, v0, v1
 ; VI-NEXT:    v_cvt_f32_ubyte3_e32 v3, v0
 ; VI-NEXT:    v_cvt_f32_ubyte2_e32 v2, v0
 ; VI-NEXT:    v_cvt_f32_ubyte0_e32 v0, v0
-; VI-NEXT:    v_cvt_f32_ubyte1_e32 v1, v4
+; VI-NEXT:    v_cvt_f32_ubyte1_e32 v1, v1
 ; VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
 ; VI-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
diff --git a/llvm/test/CodeGen/AMDGPU/dagcombine-setcc-select.ll b/llvm/test/CodeGen/AMDGPU/dagcombine-setcc-select.ll
index 9e1c58bce..c7b12e178 100644
--- a/llvm/test/CodeGen/AMDGPU/dagcombine-setcc-select.ll
+++ b/llvm/test/CodeGen/AMDGPU/dagcombine-setcc-select.ll
@@ -1,14 +1,15 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN %s
 
-; GCN-LABEL: {{^}}eq_t:
-; GCN-DAG: s_load_dword [[X:s[0-9]+]]
-; GCN:     v_cmp_lt_f32_e{{32|64}} [[CC:s\[[0-9]+:[0-9]+\]|vcc]], [[X]], 1.0{{$}}
-; GCN-NOT: 0xddd5
-; GCN-NOT: v_cndmask_b32
-; GCN-NOT: v_cmp_eq_u32
-; GCN:     v_cndmask_b32_e{{32|64}} [[RES:v[0-9]+]], 2.0, 4.0, [[CC]]
-; GCN:     store_dword v[{{[0-9:]+}}], [[RES]]{{$}}
 define amdgpu_kernel void @eq_t(float %x) {
+; GCN-LABEL: eq_t:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dword s0, s[0:1], 0x24
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_cmp_lt_f32_e64 s[0:1], s0, 1.0
+; GCN-NEXT:    v_cndmask_b32_e64 v0, 2.0, 4.0, s[0:1]
+; GCN-NEXT:    flat_store_dword v[0:1], v0
+; GCN-NEXT:    s_endpgm
   %c1 = fcmp olt float %x, 1.0
   %s1 = select i1 %c1, i32 56789, i32 1
   %c2 = icmp eq i32 %s1, 56789
@@ -17,15 +18,15 @@ define amdgpu_kernel void @eq_t(float %x) {
   ret void
 }
 
-; GCN-LABEL: {{^}}ne_t:
-; GCN-DAG: s_load_dword [[X:s[0-9]+]]
-; GCN:     v_cmp_lt_f32_e{{32|64}} [[CC:s\[[0-9]+:[0-9]+\]|vcc]], [[X]], 1.0{{$}}
-; GCN-NOT: 0xddd5
-; GCN-NOT: v_cndmask_b32
-; GCN-NOT: v_cmp_eq_u32
-; GCN:     v_cndmask_b32_e{{32|64}} [[RES:v[0-9]+]], 4.0, 2.0, [[CC]]
-; GCN:     store_dword v[{{[0-9:]+}}], [[RES]]{{$}}
 define amdgpu_kernel void @ne_t(float %x) {
+; GCN-LABEL: ne_t:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dword s0, s[0:1], 0x24
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_cmp_lt_f32_e64 s[0:1], s0, 1.0
+; GCN-NEXT:    v_cndmask_b32_e64 v0, 4.0, 2.0, s[0:1]
+; GCN-NEXT:    flat_store_dword v[0:1], v0
+; GCN-NEXT:    s_endpgm
   %c1 = fcmp olt float %x, 1.0
   %s1 = select i1 %c1, i32 56789, i32 1
   %c2 = icmp ne i32 %s1, 56789
@@ -34,15 +35,15 @@ define amdgpu_kernel void @ne_t(float %x) {
   ret void
 }
 
-; GCN-LABEL: {{^}}eq_f:
-; GCN-DAG: s_load_dword [[X:s[0-9]+]]
-; GCN:     v_cmp_lt_f32_e{{32|64}} [[CC:s\[[0-9]+:[0-9]+\]|vcc]], [[X]], 1.0{{$}}
-; GCN-NOT: 0xddd5
-; GCN-NOT: v_cndmask_b32
-; GCN-NOT: v_cmp_eq_u32
-; GCN:     v_cndmask_b32_e{{32|64}} [[RES:v[0-9]+]], 4.0, 2.0, [[CC]]
-; GCN:     store_dword v[{{[0-9:]+}}], [[RES]]{{$}}
 define amdgpu_kernel void @eq_f(float %x) {
+; GCN-LABEL: eq_f:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dword s0, s[0:1], 0x24
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_cmp_lt_f32_e64 s[0:1], s0, 1.0
+; GCN-NEXT:    v_cndmask_b32_e64 v0, 4.0, 2.0, s[0:1]
+; GCN-NEXT:    flat_store_dword v[0:1], v0
+; GCN-NEXT:    s_endpgm
   %c1 = fcmp olt float %x, 1.0
   %s1 = select i1 %c1, i32 1, i32 56789
   %c2 = icmp eq i32 %s1, 56789
@@ -51,15 +52,15 @@ define amdgpu_kernel void @eq_f(float %x) {
   ret void
 }
 
-; GCN-LABEL: {{^}}ne_f:
-; GCN-DAG: s_load_dword [[X:s[0-9]+]]
-; GCN:     v_cmp_lt_f32_e{{32|64}} [[CC:s\[[0-9]+:[0-9]+\]|vcc]], [[X]], 1.0{{$}}
-; GCN-NOT: 0xddd5
-; GCN-NOT: v_cndmask_b32
-; GCN-NOT: v_cmp_eq_u32
-; GCN:     v_cndmask_b32_e{{32|64}} [[RES:v[0-9]+]], 2.0, 4.0, [[CC]]
-; GCN:     store_dword v[{{[0-9:]+}}], [[RES]]{{$}}
 define amdgpu_kernel void @ne_f(float %x) {
+; GCN-LABEL: ne_f:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dword s0, s[0:1], 0x24
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_cmp_lt_f32_e64 s[0:1], s0, 1.0
+; GCN-NEXT:    v_cndmask_b32_e64 v0, 2.0, 4.0, s[0:1]
+; GCN-NEXT:    flat_store_dword v[0:1], v0
+; GCN-NEXT:    s_endpgm
   %c1 = fcmp olt float %x, 1.0
   %s1 = select i1 %c1, i32 1, i32 56789
   %c2 = icmp ne i32 %s1, 56789
@@ -68,10 +69,12 @@ define amdgpu_kernel void @ne_f(float %x) {
   ret void
 }
 
-; GCN-LABEL: {{^}}different_constants:
-; GCN: v_mov_b32_e32 [[RES:v[0-9]+]], 2.0
-; GCN: store_dword v[{{[0-9:]+}}], [[RES]]{{$}}
 define amdgpu_kernel void @different_constants(float %x) {
+; GCN-LABEL: different_constants:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    v_mov_b32_e32 v0, 2.0
+; GCN-NEXT:    flat_store_dword v[0:1], v0
+; GCN-NEXT:    s_endpgm
   %c1 = fcmp olt float %x, 1.0
   %s1 = select i1 %c1, i32 56789, i32 1
   %c2 = icmp eq i32 %s1, 5678
diff --git a/llvm/test/CodeGen/AMDGPU/dead-machine-elim-after-dead-lane.ll b/llvm/test/CodeGen/AMDGPU/dead-machine-elim-after-dead-lane.ll
new file mode 100644
index 000000000..da4bce27b
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/dead-machine-elim-after-dead-lane.ll
@@ -0,0 +1,28 @@
+; RUN: llc -march=amdgcn -verify-machineinstrs %s -o - | FileCheck %s
+
+; CHECK-LABEL: foo
+; CHECK-NOT: BUFFER_LOAD_DWORDX2_OFFSET
+; After dead code elimination, that buffer load should be eliminated finally
+; after dead lane detection.
+define amdgpu_kernel void @foo() {
+entry:
+  switch i8 undef, label %foo.exit [
+    i8 4, label %sw.bb4
+    i8 10, label %sw.bb10
+  ]
+
+sw.bb4:
+  %x = load i64, i64 addrspace(1)* undef, align 8
+  %c = sitofp i64 %x to float
+  %v = insertelement <2 x float> <float undef, float 0.000000e+00>, float %c, i32 0
+  br label %foo.exit
+
+sw.bb10:
+  unreachable
+
+foo.exit:
+  %agg = phi <2 x float> [ %v, %sw.bb4 ], [ zeroinitializer, %entry ]
+  %s = extractelement <2 x float> %agg, i32 1
+  store float %s, float addrspace(1)* undef, align 4
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/debug-value2.ll b/llvm/test/CodeGen/AMDGPU/debug-value2.ll
index 3a13a3348..18f525398 100644
--- a/llvm/test/CodeGen/AMDGPU/debug-value2.ll
+++ b/llvm/test/CodeGen/AMDGPU/debug-value2.ll
@@ -10,9 +10,13 @@ declare %struct.ShapeData addrspace(1)* @Scene_getSubShapeData(i32, i8 addrspace
 
 define <4 x float> @Scene_transformT(i32 %subshapeIdx, <4 x float> %v, float %time, i8 addrspace(1)* %gScene, i32 addrspace(1)* %gSceneOffsets) local_unnamed_addr !dbg !110 {
 entry:
-; CHECK: ;DEBUG_VALUE: Scene_transformT:gScene <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef] $vgpr6_vgpr7
+  ; CHECK: v_mov_b32_e32 v[[COPIED_ARG_PIECE:[0-9]+]], v9
+
+  ; CHECK: ;DEBUG_VALUE: Scene_transformT:gScene <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef, DW_OP_LLVM_fragment 0 32] $vgpr6
+  ; CHECK: ;DEBUG_VALUE: Scene_transformT:gScene <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef, DW_OP_LLVM_fragment 32 32] $vgpr7
   call void @llvm.dbg.value(metadata i8 addrspace(1)* %gScene, metadata !120, metadata !DIExpression(DW_OP_constu, 1, DW_OP_swap, DW_OP_xderef)), !dbg !154
-; CHECK: ;DEBUG_VALUE: Scene_transformT:gSceneOffsets <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef] $vgpr8_vgpr9
+  ; CHECK: ;DEBUG_VALUE: Scene_transformT:gSceneOffsets <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef, DW_OP_LLVM_fragment 0 32] $vgpr8
+  ; CHECK: ;DEBUG_VALUE: Scene_transformT:gSceneOffsets <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef, DW_OP_LLVM_fragment 32 32] $vgpr[[COPIED_ARG_PIECE]]
   call void @llvm.dbg.value(metadata i32 addrspace(1)* %gSceneOffsets, metadata !121, metadata !DIExpression(DW_OP_constu, 1, DW_OP_swap, DW_OP_xderef)), !dbg !155
   %call = tail call %struct.ShapeData addrspace(1)* @Scene_getSubShapeData(i32 %subshapeIdx, i8 addrspace(1)* %gScene, i32 addrspace(1)* %gSceneOffsets)
   %m_linearMotion = getelementptr inbounds %struct.ShapeData, %struct.ShapeData addrspace(1)* %call, i64 0, i32 2
diff --git a/llvm/test/CodeGen/AMDGPU/div_i128.ll b/llvm/test/CodeGen/AMDGPU/div_i128.ll
index 7c4c88b73..80acf6804 100644
--- a/llvm/test/CodeGen/AMDGPU/div_i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/div_i128.ll
@@ -1,4 +1,4 @@
-; RUN: not llc -mtriple=amdgcn-amd-amdhsa -verify-machineinstrs  %s 2>&1 | FileCheck %s
+; RUN: not llc -mtriple=amdgcn-amd-amdhsa -verify-machineinstrs -o - %s 2>&1 | FileCheck %s
 
 ; CHECK: LLVM ERROR: unsupported libcall legalization
 define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
diff --git a/llvm/test/CodeGen/AMDGPU/divergence-at-use.ll b/llvm/test/CodeGen/AMDGPU/divergence-at-use.ll
new file mode 100644
index 000000000..ab418b70e
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/divergence-at-use.ll
@@ -0,0 +1,20 @@
+; RUN: llc -march=amdgcn -mcpu=gfx900 -amdgpu-atomic-optimizations=true < %s | FileCheck %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -amdgpu-atomic-optimizations=true < %s -use-gpu-divergence-analysis | FileCheck %s
+
+@local = addrspace(3) global i32 undef
+
+define amdgpu_kernel void @reducible(i32 %x) {
+; CHECK-LABEL: reducible:
+; CHECK-NOT: dpp
+entry:
+  br label %loop
+loop:
+  %i = phi i32 [ 0, %entry ], [ %i1, %loop ]
+  %gep = getelementptr i32, i32 addrspace(3)* @local, i32 %i
+  %cond = icmp ult i32 %i, %x
+  %i1 = add i32 %i, 1
+  br i1 %cond, label %loop, label %exit
+exit:
+  %old = atomicrmw add i32 addrspace(3)* %gep, i32 %x acq_rel
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/divergent-branch-uniform-condition.ll b/llvm/test/CodeGen/AMDGPU/divergent-branch-uniform-condition.ll
index c903a0403..895539c00 100644
--- a/llvm/test/CodeGen/AMDGPU/divergent-branch-uniform-condition.ll
+++ b/llvm/test/CodeGen/AMDGPU/divergent-branch-uniform-condition.ll
@@ -13,13 +13,12 @@ define amdgpu_ps void @main(i32, float) {
 ; CHECK:       ; %bb.0: ; %start
 ; CHECK-NEXT:    v_readfirstlane_b32 s0, v0
 ; CHECK-NEXT:    s_mov_b32 m0, s0
-; CHECK-NEXT:    s_mov_b64 s[4:5], 0
+; CHECK-NEXT:    s_mov_b32 s0, 0
 ; CHECK-NEXT:    v_interp_p1_f32_e32 v0, v1, attr0.x
-; CHECK-NEXT:    v_cmp_nlt_f32_e64 s[0:1], 0, v0
-; CHECK-NEXT:    v_mov_b32_e32 v1, 0
-; CHECK-NEXT:    ; implicit-def: $sgpr8_sgpr9
+; CHECK-NEXT:    v_cmp_nlt_f32_e32 vcc, 0, v0
+; CHECK-NEXT:    s_mov_b64 s[2:3], 0
 ; CHECK-NEXT:    ; implicit-def: $sgpr6_sgpr7
-; CHECK-NEXT:    ; implicit-def: $sgpr2_sgpr3
+; CHECK-NEXT:    ; implicit-def: $sgpr4_sgpr5
 ; CHECK-NEXT:    s_branch BB0_3
 ; CHECK-NEXT:  BB0_1: ; %Flow1
 ; CHECK-NEXT:    ; in Loop: Header=BB0_3 Depth=1
@@ -28,36 +27,34 @@ define amdgpu_ps void @main(i32, float) {
 ; CHECK-NEXT:  BB0_2: ; %Flow
 ; CHECK-NEXT:    ; in Loop: Header=BB0_3 Depth=1
 ; CHECK-NEXT:    s_and_b64 s[10:11], exec, s[6:7]
-; CHECK-NEXT:    s_or_b64 s[10:11], s[10:11], s[4:5]
-; CHECK-NEXT:    s_andn2_b64 s[2:3], s[2:3], exec
-; CHECK-NEXT:    s_and_b64 s[4:5], s[8:9], exec
-; CHECK-NEXT:    s_or_b64 s[2:3], s[2:3], s[4:5]
-; CHECK-NEXT:    s_mov_b64 s[4:5], s[10:11]
-; CHECK-NEXT:    s_andn2_b64 exec, exec, s[10:11]
+; CHECK-NEXT:    s_or_b64 s[2:3], s[10:11], s[2:3]
+; CHECK-NEXT:    s_andn2_b64 s[4:5], s[4:5], exec
+; CHECK-NEXT:    s_and_b64 s[8:9], s[8:9], exec
+; CHECK-NEXT:    s_or_b64 s[4:5], s[4:5], s[8:9]
+; CHECK-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; CHECK-NEXT:    s_cbranch_execz BB0_6
 ; CHECK-NEXT:  BB0_3: ; %loop
 ; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    v_cmp_gt_u32_e32 vcc, 32, v1
-; CHECK-NEXT:    s_and_b64 vcc, exec, vcc
 ; CHECK-NEXT:    s_or_b64 s[6:7], s[6:7], exec
-; CHECK-NEXT:    s_or_b64 s[8:9], s[8:9], exec
-; CHECK-NEXT:    s_cbranch_vccz BB0_2
+; CHECK-NEXT:    s_cmp_lt_u32 s0, 32
+; CHECK-NEXT:    s_mov_b64 s[8:9], -1
+; CHECK-NEXT:    s_cbranch_scc0 BB0_2
 ; CHECK-NEXT:  ; %bb.4: ; %endif1
 ; CHECK-NEXT:    ; in Loop: Header=BB0_3 Depth=1
 ; CHECK-NEXT:    s_mov_b64 s[6:7], -1
-; CHECK-NEXT:    s_and_saveexec_b64 s[8:9], s[0:1]
+; CHECK-NEXT:    s_and_saveexec_b64 s[8:9], vcc
 ; CHECK-NEXT:    s_xor_b64 s[8:9], exec, s[8:9]
 ; CHECK-NEXT:    ; mask branch BB0_1
 ; CHECK-NEXT:    s_cbranch_execz BB0_1
 ; CHECK-NEXT:  BB0_5: ; %endif2
 ; CHECK-NEXT:    ; in Loop: Header=BB0_3 Depth=1
-; CHECK-NEXT:    v_add_u32_e32 v1, 1, v1
+; CHECK-NEXT:    s_add_i32 s0, s0, 1
 ; CHECK-NEXT:    s_xor_b64 s[6:7], exec, -1
 ; CHECK-NEXT:    s_branch BB0_1
 ; CHECK-NEXT:  BB0_6: ; %Flow2
-; CHECK-NEXT:    s_or_b64 exec, exec, s[10:11]
+; CHECK-NEXT:    s_or_b64 exec, exec, s[2:3]
 ; CHECK-NEXT:    v_mov_b32_e32 v1, 0
-; CHECK-NEXT:    s_and_saveexec_b64 s[0:1], s[2:3]
+; CHECK-NEXT:    s_and_saveexec_b64 s[0:1], s[4:5]
 ; CHECK-NEXT:    ; mask branch BB0_8
 ; CHECK-NEXT:  BB0_7: ; %if1
 ; CHECK-NEXT:    v_sqrt_f32_e32 v1, v0
@@ -65,6 +62,7 @@ define amdgpu_ps void @main(i32, float) {
 ; CHECK-NEXT:    s_or_b64 exec, exec, s[0:1]
 ; CHECK-NEXT:    exp mrt0 v1, v1, v1, v1 done vm
 ; CHECK-NEXT:    s_endpgm
+
 ; this is the divergent branch with the condition not marked as divergent
 start:
   %v0 = call float @llvm.amdgcn.interp.p1(float %1, i32 0, i32 0, i32 %0)
diff --git a/llvm/test/CodeGen/AMDGPU/divrem24-assume.ll b/llvm/test/CodeGen/AMDGPU/divrem24-assume.ll
index c46945398..e32c93bdf 100644
--- a/llvm/test/CodeGen/AMDGPU/divrem24-assume.ll
+++ b/llvm/test/CodeGen/AMDGPU/divrem24-assume.ll
@@ -12,7 +12,7 @@ define amdgpu_kernel void @divrem24_assume(i32 addrspace(1)* %arg, i32 %arg1) {
 ; CHECK-NEXT:    [[TMP2:%.*]] = fdiv fast float 1.000000e+00, [[TMP1]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = fmul fast float [[TMP0]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call fast float @llvm.trunc.f32(float [[TMP3]])
-; CHECK-NEXT:    [[TMP5:%.*]] = fsub fast float -0.000000e+00, [[TMP4]]
+; CHECK-NEXT:    [[TMP5:%.*]] = fneg fast float [[TMP4]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP5]], float [[TMP1]], float [[TMP0]])
 ; CHECK-NEXT:    [[TMP7:%.*]] = fptoui float [[TMP4]] to i32
 ; CHECK-NEXT:    [[TMP8:%.*]] = call fast float @llvm.fabs.f32(float [[TMP6]])
diff --git a/llvm/test/CodeGen/AMDGPU/dpp_combine.ll b/llvm/test/CodeGen/AMDGPU/dpp_combine.ll
new file mode 100644
index 000000000..2afc53888
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/dpp_combine.ll
@@ -0,0 +1,53 @@
+; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck %s -check-prefix=GCN
+; RUN: llc -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck %s -check-prefix=GCN
+
+; GCN-LABEL: {{^}}dpp_add:
+; GCN: global_load_dword [[V:v[0-9]+]],
+; GCN: v_add_{{(nc_)?}}u32_dpp [[V]], [[V]], [[V]] quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:0{{$}}
+define amdgpu_kernel void @dpp_add(i32 addrspace(1)* %arg) {
+  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %gep = getelementptr inbounds i32, i32 addrspace(1)* %arg, i32 %id
+  %load = load i32, i32 addrspace(1)* %gep
+  %tmp0 = call i32 @llvm.amdgcn.update.dpp.i32(i32 %load, i32 %load, i32 1, i32 15, i32 15, i1 1) #0
+  %add = add i32 %tmp0, %load
+  store i32 %add, i32 addrspace(1)* %gep
+  ret void
+}
+
+; GCN-LABEL: {{^}}dpp_ceil:
+; GCN: global_load_dword [[V:v[0-9]+]],
+; GCN: v_ceil_f32_dpp [[V]], [[V]] quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:0{{$}}
+define amdgpu_kernel void @dpp_ceil(i32 addrspace(1)* %arg) {
+  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %gep = getelementptr inbounds i32, i32 addrspace(1)* %arg, i32 %id
+  %load = load i32, i32 addrspace(1)* %gep
+  %tmp0 = call i32 @llvm.amdgcn.update.dpp.i32(i32 %load, i32 %load, i32 1, i32 15, i32 15, i1 1) #0
+  %tmp1 = bitcast i32 %tmp0 to float
+  %round = tail call float @llvm.ceil.f32(float %tmp1)
+  %tmp2 = bitcast float %round to i32
+  store i32 %tmp2, i32 addrspace(1)* %gep
+  ret void
+}
+
+; GCN-LABEL: {{^}}dpp_fadd:
+; GCN: global_load_dword [[V:v[0-9]+]],
+; GCN: v_add_f32_dpp [[V]], [[V]], [[V]] quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:0{{$}}
+define amdgpu_kernel void @dpp_fadd(i32 addrspace(1)* %arg) {
+  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %gep = getelementptr inbounds i32, i32 addrspace(1)* %arg, i32 %id
+  %load = load i32, i32 addrspace(1)* %gep
+  %tmp0 = call i32 @llvm.amdgcn.update.dpp.i32(i32 %load, i32 %load, i32 1, i32 15, i32 15, i1 1) #0
+  %tmp1 = bitcast i32 %tmp0 to float
+  %t = bitcast i32 %load to float
+  %add = fadd float %tmp1, %t
+  %tmp2 = bitcast float %add to i32
+  store i32 %tmp2, i32 addrspace(1)* %gep
+  ret void
+}
+
+
+declare i32 @llvm.amdgcn.workitem.id.x()
+declare i32 @llvm.amdgcn.update.dpp.i32(i32, i32, i32, i32, i32, i1) #0
+declare float @llvm.ceil.f32(float)
+
+attributes #0 = { nounwind readnone convergent }
diff --git a/llvm/test/CodeGen/AMDGPU/ds-negative-offset-addressing-mode-loop.ll b/llvm/test/CodeGen/AMDGPU/ds-negative-offset-addressing-mode-loop.ll
index e2c7f1c47..5997e27fd 100644
--- a/llvm/test/CodeGen/AMDGPU/ds-negative-offset-addressing-mode-loop.ll
+++ b/llvm/test/CodeGen/AMDGPU/ds-negative-offset-addressing-mode-loop.ll
@@ -7,8 +7,6 @@ declare void @llvm.amdgcn.s.barrier() #1
 
 ; Function Attrs: nounwind
 ; CHECK-LABEL: {{^}}signed_ds_offset_addressing_loop:
-; SI: s_movk_i32 [[K_0X88:s[0-9]+]], 0x
-; SI: s_movk_i32 [[K_0X100:s[0-9]+]], 0x100
 ; CHECK: BB0_1:
 ; CHECK: v_add_i32_e32 [[VADDR:v[0-9]+]],
 ; SI-DAG: ds_read_b32 v{{[0-9]+}}, [[VADDR]]
@@ -16,9 +14,9 @@ declare void @llvm.amdgcn.s.barrier() #1
 ; SI-DAG: ds_read_b32 v{{[0-9]+}}, [[VADDR8]]
 ; SI-DAG: v_add_i32_e32 [[VADDR0x80:v[0-9]+]], vcc, 0x80, [[VADDR]]
 ; SI-DAG: ds_read_b32 v{{[0-9]+}}, [[VADDR0x80]]
-; SI-DAG: v_add_i32_e32 [[VADDR0x88:v[0-9]+]], vcc, [[K_0X88]], [[VADDR]]
+; SI-DAG: v_add_i32_e32 [[VADDR0x88:v[0-9]+]], vcc, 0x88, [[VADDR]]
 ; SI-DAG: ds_read_b32 v{{[0-9]+}}, [[VADDR0x88]]
-; SI-DAG: v_add_i32_e32 [[VADDR0x100:v[0-9]+]], vcc, [[K_0X100]], [[VADDR]]
+; SI-DAG: v_add_i32_e32 [[VADDR0x100:v[0-9]+]], vcc, 0x100, [[VADDR]]
 ; SI-DAG: ds_read_b32 v{{[0-9]+}}, [[VADDR0x100]]
 
 ; CI-DAG: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[VADDR]] offset1:2
diff --git a/llvm/test/CodeGen/AMDGPU/ds_read2.ll b/llvm/test/CodeGen/AMDGPU/ds_read2.ll
index 9991eb3fc..e8a7a1ce5 100644
--- a/llvm/test/CodeGen/AMDGPU/ds_read2.ll
+++ b/llvm/test/CodeGen/AMDGPU/ds_read2.ll
@@ -473,8 +473,8 @@ define amdgpu_kernel void @load_constant_disjoint_offsets(i32 addrspace(1)* %out
 ; GFX9-NOT: m0
 
 ; GCN-DAG: v_mov_b32_e32 [[PTR:v[0-9]+]], bar@abs32@lo{{$}}
-; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[PTR]] offset1:1
 ; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[PTR]] offset0:2 offset1:3
+; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[PTR]] offset1:1
 define amdgpu_kernel void @load_misaligned64_constant_offsets(i64 addrspace(1)* %out) {
   %val0 = load i64, i64 addrspace(3)* getelementptr inbounds ([4 x i64], [4 x i64] addrspace(3)* @bar, i32 0, i32 0), align 4
   %val1 = load i64, i64 addrspace(3)* getelementptr inbounds ([4 x i64], [4 x i64] addrspace(3)* @bar, i32 0, i32 1), align 4
diff --git a/llvm/test/CodeGen/AMDGPU/enable-no-signed-zeros-fp-math.ll b/llvm/test/CodeGen/AMDGPU/enable-no-signed-zeros-fp-math.ll
index b7dfcd990..2565395b8 100644
--- a/llvm/test/CodeGen/AMDGPU/enable-no-signed-zeros-fp-math.ll
+++ b/llvm/test/CodeGen/AMDGPU/enable-no-signed-zeros-fp-math.ll
@@ -1,17 +1,29 @@
-; RUN: llc -march=amdgcn -enable-no-signed-zeros-fp-math=0 < %s | FileCheck -check-prefix=GCN -check-prefix=GCN-SAFE %s
-; RUN: llc -march=amdgcn -enable-no-signed-zeros-fp-math=1 < %s | FileCheck -check-prefix=GCN -check-prefix=GCN-UNSAFE %s
-; RUN: llc -march=amdgcn -enable-unsafe-fp-math < %s | FileCheck -check-prefix=GCN -check-prefix=GCN-UNSAFE %s
+; RUN: llc -march=amdgcn < %s | FileCheck --check-prefixes=GCN,GCN-FMF,GCN-SAFE %s
 
 declare i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
 
 ; Test that the -enable-no-signed-zeros-fp-math flag works
 
-; GCN-LABEL: {{^}}fneg_fsub_f32:
+; GCN-LABEL: {{^}}fneg_fsub_f32_fmf:
 ; GCN: v_sub_f32_e32 [[SUB:v[0-9]+]], {{v[0-9]+}}, {{v[0-9]+}}
-; GCN-SAFE: v_xor_b32_e32 v{{[0-9]+}}, 0x80000000, [[SUB]]
+; GCN-FMF-NOT: xor
+define amdgpu_kernel void @fneg_fsub_f32_fmf(float addrspace(1)* %out, float addrspace(1)* %in) #0 {
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %add = add i32 %tid, 1
+  %gep = getelementptr float, float addrspace(1)* %in, i32 %tid
+  %b_ptr = getelementptr float, float addrspace(1)* %in, i32 %add
+  %a = load float, float addrspace(1)* %gep, align 4
+  %b = load float, float addrspace(1)* %b_ptr, align 4
+  %result = fsub fast float %a, %b
+  %neg.result = fsub fast float -0.0, %result
+  store float %neg.result, float addrspace(1)* %out, align 4
+  ret void
+}
 
-; GCN-UNSAFE-NOT: xor
-define amdgpu_kernel void @fneg_fsub_f32(float addrspace(1)* %out, float addrspace(1)* %in) #0 {
+; GCN-LABEL: {{^}}fneg_fsub_f32_safe:
+; GCN: v_sub_f32_e32 [[SUB:v[0-9]+]], {{v[0-9]+}}, {{v[0-9]+}}
+; GCN-SAFE: v_xor_b32_e32 v{{[0-9]+}}, 0x80000000, [[SUB]]
+define amdgpu_kernel void @fneg_fsub_f32_safe(float addrspace(1)* %out, float addrspace(1)* %in) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %add = add i32 %tid, 1
   %gep = getelementptr float, float addrspace(1)* %in, i32 %tid
diff --git a/llvm/test/CodeGen/AMDGPU/extract-subvector-equal-length.ll b/llvm/test/CodeGen/AMDGPU/extract-subvector-equal-length.ll
index 6f7fb53f7..23c96373d 100644
--- a/llvm/test/CodeGen/AMDGPU/extract-subvector-equal-length.ll
+++ b/llvm/test/CodeGen/AMDGPU/extract-subvector-equal-length.ll
@@ -4,7 +4,7 @@
 ; Test for ICE in SelectionDAG::computeKnownBits when visiting EXTRACT_SUBVECTOR
 ; with DemandedElts already as wide as the source vector.
 
-define <3 x i32> @quux() #0 {
+define <3 x i32> @quux() {
 ; CHECK-LABEL: quux:
 ; CHECK:       ; %bb.0: ; %bb
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -26,5 +26,3 @@ bb:
   %tmp10 = lshr <3 x i32> %tmp9, <i32 1, i32 1, i32 1>
   ret <3 x i32> %tmp10
 }
-
-attributes #0 = { noinline optnone }
diff --git a/llvm/test/CodeGen/AMDGPU/extract-subvector.ll b/llvm/test/CodeGen/AMDGPU/extract-subvector.ll
new file mode 100644
index 000000000..19eb67569
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/extract-subvector.ll
@@ -0,0 +1,40 @@
+; RUN: llc -march=amdgcn -mtriple=amdgcn-- -verify-machineinstrs -o - %s | FileCheck %s
+
+; CHECK-LABEL: foo
+; CHECK: buffer_load_ushort
+; CHECK: buffer_load_ushort
+; CHECK: buffer_load_ushort
+; CHECK: buffer_load_ushort
+; CHECK: buffer_load_ushort
+; CHECK: buffer_load_ushort
+; CHECK: buffer_load_ushort
+; CHECK: buffer_load_ushort
+; CHECK: buffer_load_ushort
+; CHECK: buffer_load_ushort
+; CHECK: buffer_load_ushort
+; CHECK: buffer_load_ushort
+; CHECK: buffer_load_ushort
+; CHECK: buffer_load_ushort
+; CHECK: buffer_load_ushort
+; CHECK: buffer_load_ushort
+; CHECK: v_bfe_i32
+; CHECK: v_bfe_i32
+
+define <2 x i16> @foo(<8 x i16> addrspace(1) * %p0, <8 x i16> addrspace(1) * %p1) {
+  br i1 undef, label %T, label %F
+
+T:
+  %t = load volatile <8 x i16>, <8 x i16> addrspace(1) * %p0
+  br label %exit
+
+F:
+  %f = load volatile <8 x i16>, <8 x i16> addrspace(1) * %p1
+  br label %exit
+
+exit:
+  %m = phi <8 x i16> [ %t, %T ], [ %f, %F ]
+  %v2 = shufflevector <8 x i16> %m, <8 x i16> undef, <2 x i32> <i32 0, i32 1>
+  %b2 = icmp sgt <2 x i16> %v2, <i16 -1, i16 -1>
+  %r2 = select <2 x i1> %b2, <2 x i16> <i16 -32768, i16 -32768>, <2 x i16> <i16 -1, i16 -1>
+  ret <2 x i16> %r2
+}
diff --git a/llvm/test/CodeGen/AMDGPU/extract_subvector_vec4_vec3.ll b/llvm/test/CodeGen/AMDGPU/extract_subvector_vec4_vec3.ll
index a39833455..93322c7da 100644
--- a/llvm/test/CodeGen/AMDGPU/extract_subvector_vec4_vec3.ll
+++ b/llvm/test/CodeGen/AMDGPU/extract_subvector_vec4_vec3.ll
@@ -8,20 +8,20 @@
 define amdgpu_hs void @main([0 x i8] addrspace(6)* inreg %arg) {
   ; GCN-LABEL: name: main
   ; GCN: bb.0.main_body:
-  ; GCN:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0 = S_MOV_B32 0
-  ; GCN:   [[DEF:%[0-9]+]]:sreg_32_xm0 = IMPLICIT_DEF
+  ; GCN:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+  ; GCN:   [[DEF:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
   ; GCN:   [[COPY:%[0-9]+]]:vgpr_32 = COPY [[DEF]]
-  ; GCN:   [[DEF1:%[0-9]+]]:sreg_128 = IMPLICIT_DEF
-  ; GCN:   [[BUFFER_LOAD_DWORDX4_OFFEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFEN [[COPY]], [[DEF1]], [[S_MOV_B32_]], 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 16 from custom TargetCustom7, align 1, addrspace 4)
-  ; GCN:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_OFFEN]].sub2
-  ; GCN:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_OFFEN]].sub1
-  ; GCN:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_OFFEN]].sub0
+  ; GCN:   [[DEF1:%[0-9]+]]:sgpr_128 = IMPLICIT_DEF
+  ; GCN:   [[BUFFER_LOAD_DWORDX4_OFFEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFEN [[COPY]], [[DEF1]], [[S_MOV_B32_]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 16 from custom "TargetCustom7", align 1, addrspace 4)
+  ; GCN:   [[COPY1:%[0-9]+]]:sgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_OFFEN]].sub2
+  ; GCN:   [[COPY2:%[0-9]+]]:sgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_OFFEN]].sub1
+  ; GCN:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_OFFEN]].sub0
   ; GCN:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_96 = REG_SEQUENCE killed [[COPY3]], %subreg.sub0, killed [[COPY2]], %subreg.sub1, killed [[COPY1]], %subreg.sub2
   ; GCN:   [[COPY4:%[0-9]+]]:vreg_96 = COPY [[REG_SEQUENCE]]
-  ; GCN:   [[DEF2:%[0-9]+]]:sreg_32_xm0 = IMPLICIT_DEF
+  ; GCN:   [[DEF2:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
   ; GCN:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[DEF2]]
-  ; GCN:   [[DEF3:%[0-9]+]]:sreg_128 = IMPLICIT_DEF
-  ; GCN:   BUFFER_STORE_DWORDX3_OFFEN_exact killed [[COPY4]], [[COPY5]], [[DEF3]], [[S_MOV_B32_]], 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 12 into custom TargetCustom7, align 1, addrspace 4)
+  ; GCN:   [[DEF3:%[0-9]+]]:sgpr_128 = IMPLICIT_DEF
+  ; GCN:   BUFFER_STORE_DWORDX3_OFFEN_exact killed [[COPY4]], [[COPY5]], [[DEF3]], [[S_MOV_B32_]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 12 into custom "TargetCustom7", align 1, addrspace 4)
   ; GCN:   S_ENDPGM 0
 main_body:
   %tmp25 = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> undef, i32 undef, i32 0, i32 0)
@@ -34,4 +34,3 @@ main_body:
 
 declare void @llvm.amdgcn.raw.buffer.store.v3i32(<3 x i32>, <4 x i32>, i32, i32, i32 immarg)
 declare <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32>, i32, i32, i32 immarg)
-
diff --git a/llvm/test/CodeGen/AMDGPU/extract_vector_dynelt.ll b/llvm/test/CodeGen/AMDGPU/extract_vector_dynelt.ll
index a2b68cc42..eb95efd18 100644
--- a/llvm/test/CodeGen/AMDGPU/extract_vector_dynelt.ll
+++ b/llvm/test/CodeGen/AMDGPU/extract_vector_dynelt.ll
@@ -306,10 +306,10 @@ entry:
 
 ; GCN-LABEL: {{^}}bit128_extelt:
 ; GCN-NOT: buffer_
-; GCN-DAG: v_cndmask_b32_e{{32|64}} [[V1:v[0-9]+]], 1, 0,
+; GCN-DAG: v_cndmask_b32_e{{32|64}} [[V1:v[0-9]+]], 0, 1
 ; GCN-DAG: v_mov_b32_e32 [[LASTIDX:v[0-9]+]], 0x7f
 ; GCN-DAG: v_cmp_ne_u32_e32 [[CL:[^,]+]], s{{[0-9]+}}, [[LASTIDX]]
-; GCN-DAG: v_cndmask_b32_e{{32|64}} [[VL:v[0-9]+]], 0, v{{[0-9]+}}, [[CL]]
+; GCN-DAG: v_cndmask_b32_e{{32|64}} [[VL:v[0-9]+]], 0, [[V1]], [[CL]]
 ; GCN:     v_and_b32_e32 [[RES:v[0-9]+]], 1, [[VL]]
 ; GCN:     store_dword v[{{[0-9:]+}}], [[RES]]
 define amdgpu_kernel void @bit128_extelt(i32 addrspace(1)* %out, i32 %sel) {
diff --git a/llvm/test/CodeGen/AMDGPU/fabs.ll b/llvm/test/CodeGen/AMDGPU/fabs.ll
index f96019dba..badaa16bb 100644
--- a/llvm/test/CodeGen/AMDGPU/fabs.ll
+++ b/llvm/test/CodeGen/AMDGPU/fabs.ll
@@ -48,8 +48,8 @@ define amdgpu_kernel void @s_fabs_f32(float addrspace(1)* %out, float %in) {
 ; R600: |{{(PV|T[0-9])\.[XYZW]}}|
 ; R600: |{{(PV|T[0-9])\.[XYZW]}}|
 
-; GCN: v_and_b32
-; GCN: v_and_b32
+; GCN: s_and_b32
+; GCN: s_and_b32
 define amdgpu_kernel void @fabs_v2f32(<2 x float> addrspace(1)* %out, <2 x float> %in) {
   %fabs = call <2 x float> @llvm.fabs.v2f32(<2 x float> %in)
   store <2 x float> %fabs, <2 x float> addrspace(1)* %out
@@ -62,10 +62,10 @@ define amdgpu_kernel void @fabs_v2f32(<2 x float> addrspace(1)* %out, <2 x float
 ; R600: |{{(PV|T[0-9])\.[XYZW]}}|
 ; R600: |{{(PV|T[0-9])\.[XYZW]}}|
 
-; GCN: v_and_b32
-; GCN: v_and_b32
-; GCN: v_and_b32
-; GCN: v_and_b32
+; GCN: s_and_b32
+; GCN: s_and_b32
+; GCN: s_and_b32
+; GCN: s_and_b32
 define amdgpu_kernel void @fabs_v4f32(<4 x float> addrspace(1)* %out, <4 x float> %in) {
   %fabs = call <4 x float> @llvm.fabs.v4f32(<4 x float> %in)
   store <4 x float> %fabs, <4 x float> addrspace(1)* %out
diff --git a/llvm/test/CodeGen/AMDGPU/fadd-fma-fmul-combine.ll b/llvm/test/CodeGen/AMDGPU/fadd-fma-fmul-combine.ll
index 0c4a77964..f66ea1522 100644
--- a/llvm/test/CodeGen/AMDGPU/fadd-fma-fmul-combine.ll
+++ b/llvm/test/CodeGen/AMDGPU/fadd-fma-fmul-combine.ll
@@ -219,7 +219,7 @@ define amdgpu_kernel void @fast_sub_fmuladd_fmul_multi_use_mul() #0 {
   ret void
 }
 
-; GCN-LABEL: {{^}}fast_sub_fmuladd_fmul_multi_use_fmuladd:
+; GCN-LABEL: {{^}}fast_sub_fmuladd_fmul_multi_use_fmuladd_lhs:
 ; GCN: buffer_load_dword [[X:v[0-9]+]]
 ; GCN: buffer_load_dword [[Y:v[0-9]+]]
 ; GCN: buffer_load_dword [[Z:v[0-9]+]]
@@ -241,7 +241,7 @@ define amdgpu_kernel void @fast_sub_fmuladd_fmul_multi_use_mul() #0 {
 ; GCN-SLOWFMA-DAG: v_mul_f32_e32 v{{[0-9]+}}, [[X]], [[Y]]
 ; GCN-SLOWFMA: v_add_f32_e32
 ; GCN-SLOWFMA: v_sub_f32_e32
-define amdgpu_kernel void @fast_sub_fmuladd_fmul_multi_use_fmuladd() #0 {
+define amdgpu_kernel void @fast_sub_fmuladd_fmul_multi_use_fmuladd_lhs() #0 {
   %x = load volatile float, float addrspace(1)* undef
   %y = load volatile float, float addrspace(1)* undef
   %z = load volatile float, float addrspace(1)* undef
@@ -255,6 +255,120 @@ define amdgpu_kernel void @fast_sub_fmuladd_fmul_multi_use_fmuladd() #0 {
   ret void
 }
 
+; GCN-LABEL: {{^}}fast_sub_fmuladd_fmul_multi_use_fmuladd_rhs:
+; GCN: buffer_load_dword [[X:v[0-9]+]]
+; GCN: buffer_load_dword [[Y:v[0-9]+]]
+; GCN: buffer_load_dword [[Z:v[0-9]+]]
+; GCN: buffer_load_dword [[U:v[0-9]+]]
+; GCN: buffer_load_dword [[V:v[0-9]+]]
+
+; GCN-DAG: v_mul_f32_e32 [[MUL:v[0-9]+]], [[U]], [[V]]
+
+; GCN-FLUSH-NEXT: v_mac_f32_e32 [[MUL]], [[X]], [[Y]]
+; GCN-FLUSH-NEXT: v_sub_f32_e32 [[SUB:v[0-9]+]],  [[Z]], [[MUL]]
+; GCN-FLUSH-NEXT: buffer_store_dword [[MUL]]
+; GCN-FLUSH-NEXT: buffer_store_dword [[SUB]]
+
+; GCN-FASTFMA-NEXT: v_fma_f32 [[FMA:v[0-9]+]], [[X]], [[Y]], [[U]]
+; GCN-FASTFMA-NEXT: v_sub_f32_e32 [[SUB:v[0-9]+]], [[Z]], [[FMA]]
+; GCN-FASTFMA-NEXT: buffer_store_dword [[FMA]]
+; GCN-FASTFMA-NEXT: buffer_store_dword [[SUB]]
+
+; GCN-SLOWFMA-DAG: v_mul_f32_e32 v{{[0-9]+}}, [[X]], [[Y]]
+; GCN-SLOWFMA: v_add_f32_e32
+; GCN-SLOWFMA: v_sub_f32_e32
+define amdgpu_kernel void @fast_sub_fmuladd_fmul_multi_use_fmuladd_rhs() #0 {
+  %x = load volatile float, float addrspace(1)* undef
+  %y = load volatile float, float addrspace(1)* undef
+  %z = load volatile float, float addrspace(1)* undef
+  %u = load volatile float, float addrspace(1)* undef
+  %v = load volatile float, float addrspace(1)* undef
+  %mul.u.v = fmul fast float %u, %v
+  %fma = call fast float @llvm.fmuladd.f32(float %x, float %y, float %mul.u.v)
+  %add = fsub fast float %z, %fma
+  store volatile float %fma, float addrspace(1)* undef
+  store volatile float %add, float addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}fast_sub_fmuladd_fpext_fmul_multi_use_fmuladd_lhs:
+; GCN: buffer_load_dword [[X:v[0-9]+]]
+; GCN: buffer_load_dword [[Y:v[0-9]+]]
+; GCN: buffer_load_dword [[Z:v[0-9]+]]
+; GCN: buffer_load_ushort [[U:v[0-9]+]]
+; GCN: buffer_load_ushort [[V:v[0-9]+]]
+
+; GCN-DAG: v_cvt_f32_f16_e32 [[UFLOAT:v[0-9]+]], [[U]]
+; GCN-DAG: v_cvt_f32_f16_e32 [[VFLOAT:v[0-9]+]], [[V]]
+; GCN-DAG: v_mul_f32_e32 [[MUL:v[0-9]+]], [[UFLOAT]], [[VFLOAT]]
+
+; GCN-FLUSH-NEXT: v_mac_f32_e32 [[MUL]], [[X]], [[Y]]
+; GCN-FLUSH-NEXT: v_sub_f32_e32 [[SUB:v[0-9]+]],  [[MUL]], [[Z]]
+; GCN-FLUSH-NEXT: buffer_store_dword [[MUL]]
+; GCN-FLUSH-NEXT: buffer_store_dword [[SUB]]
+
+; GCN-FASTFMA-NEXT: v_fma_f32 [[FMA:v[0-9]+]], [[X]], [[Y]], [[UFLOAT]]
+; GCN-FASTFMA-NEXT: v_sub_f32_e32 [[SUB:v[0-9]+]], [[FMA]], [[Z]]
+; GCN-FASTFMA-NEXT: buffer_store_dword [[FMA]]
+; GCN-FASTFMA-NEXT: buffer_store_dword [[SUB]]
+
+; GCN-SLOWFMA-DAG: v_mul_f32_e32 v{{[0-9]+}}, [[X]], [[Y]]
+; GCN-SLOWFMA: v_add_f32_e32
+; GCN-SLOWFMA: v_sub_f32_e32
+define amdgpu_kernel void @fast_sub_fmuladd_fpext_fmul_multi_use_fmuladd_lhs() #0 {
+  %x = load volatile float, float addrspace(1)* undef
+  %y = load volatile float, float addrspace(1)* undef
+  %z = load volatile float, float addrspace(1)* undef
+  %u = load volatile half, half addrspace(1)* undef
+  %v = load volatile half, half addrspace(1)* undef
+  %mul.u.v.half = fmul fast half %u, %v
+  %mul.u.v = fpext half %mul.u.v.half to float
+  %fma = call fast float @llvm.fmuladd.f32(float %x, float %y, float %mul.u.v)
+  %add = fsub fast float %fma, %z
+  store volatile float %fma, float addrspace(1)* undef
+  store volatile float %add, float addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}fast_sub_fmuladd_fpext_fmul_multi_use_fmuladd_rhs:
+; GCN: buffer_load_dword [[X:v[0-9]+]]
+; GCN: buffer_load_dword [[Y:v[0-9]+]]
+; GCN: buffer_load_dword [[Z:v[0-9]+]]
+; GCN: buffer_load_ushort [[U:v[0-9]+]]
+; GCN: buffer_load_ushort [[V:v[0-9]+]]
+
+; GCN-DAG: v_cvt_f32_f16_e32 [[UFLOAT:v[0-9]+]], [[U]]
+; GCN-DAG: v_cvt_f32_f16_e32 [[VFLOAT:v[0-9]+]], [[V]]
+; GCN-DAG: v_mul_f32_e32 [[MUL:v[0-9]+]], [[UFLOAT]], [[VFLOAT]]
+
+; GCN-FLUSH-NEXT: v_mac_f32_e32 [[MUL]], [[X]], [[Y]]
+; GCN-FLUSH-NEXT: v_sub_f32_e32 [[SUB:v[0-9]+]],  [[Z]], [[MUL]]
+; GCN-FLUSH-NEXT: buffer_store_dword [[MUL]]
+; GCN-FLUSH-NEXT: buffer_store_dword [[SUB]]
+
+; GCN-FASTFMA-NEXT: v_fma_f32 [[FMA:v[0-9]+]], [[X]], [[Y]], [[UFLOAT]]
+; GCN-FASTFMA-NEXT: v_sub_f32_e32 [[SUB:v[0-9]+]], [[Z]], [[FMA]]
+; GCN-FASTFMA-NEXT: buffer_store_dword [[FMA]]
+; GCN-FASTFMA-NEXT: buffer_store_dword [[SUB]]
+
+; GCN-SLOWFMA-DAG: v_mul_f32_e32 v{{[0-9]+}}, [[X]], [[Y]]
+; GCN-SLOWFMA: v_add_f32_e32
+; GCN-SLOWFMA: v_sub_f32_e32
+define amdgpu_kernel void @fast_sub_fmuladd_fpext_fmul_multi_use_fmuladd_rhs() #0 {
+  %x = load volatile float, float addrspace(1)* undef
+  %y = load volatile float, float addrspace(1)* undef
+  %z = load volatile float, float addrspace(1)* undef
+  %u = load volatile half, half addrspace(1)* undef
+  %v = load volatile half, half addrspace(1)* undef
+  %mul.u.v.half = fmul fast half %u, %v
+  %mul.u.v = fpext half %mul.u.v.half to float
+  %fma = call fast float @llvm.fmuladd.f32(float %x, float %y, float %mul.u.v)
+  %add = fsub fast float %z, %fma
+  store volatile float %fma, float addrspace(1)* undef
+  store volatile float %add, float addrspace(1)* undef
+  ret void
+}
+
 declare float @llvm.fma.f32(float, float, float) #1
 declare float @llvm.fmuladd.f32(float, float, float) #1
 
diff --git a/llvm/test/CodeGen/AMDGPU/fadd.f16.ll b/llvm/test/CodeGen/AMDGPU/fadd.f16.ll
index 30fdb1df8..3d7964647 100644
--- a/llvm/test/CodeGen/AMDGPU/fadd.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fadd.f16.ll
@@ -62,8 +62,8 @@ entry:
 ; GCN-LABEL: {{^}}fadd_v2f16:
 ; SI: buffer_load_dword v[[A_V2_F16:[0-9]+]]
 ; SI: buffer_load_dword v[[B_V2_F16:[0-9]+]]
-; VI: flat_load_dword v[[B_V2_F16:[0-9]+]]
 ; VI: flat_load_dword v[[A_V2_F16:[0-9]+]]
+; VI: flat_load_dword v[[B_V2_F16:[0-9]+]]
 
 ; SI-DAG:  v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_V2_F16]]
 ; SI-DAG: v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_V2_F16]]
diff --git a/llvm/test/CodeGen/AMDGPU/fdiv.ll b/llvm/test/CodeGen/AMDGPU/fdiv.ll
index bc4894543..bd4deb14a 100644
--- a/llvm/test/CodeGen/AMDGPU/fdiv.ll
+++ b/llvm/test/CodeGen/AMDGPU/fdiv.ll
@@ -1,6 +1,7 @@
-; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=FUNC %s
-; RUN: llc -march=amdgcn -mcpu=hawaii -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=FUNC %s
-; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,PREGFX10,FUNC %s
+; RUN: llc -march=amdgcn -mcpu=hawaii -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,PREGFX10,FUNC %s
+; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,PREGFX10,FUNC %s
+; RUN: llc -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX10,FUNC %s
 ; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefix=R600 -check-prefix=FUNC %s
 
 ; These tests check that fdiv is expanded correctly and also test that the
@@ -17,14 +18,16 @@
 ; GCN-DAG: v_div_scale_f32 [[DEN_SCALE:v[0-9]+]]
 ; GCN-DAG: v_rcp_f32_e32 [[NUM_RCP:v[0-9]+]], [[NUM_SCALE]]
 
-; GCN: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
+; PREGFX10: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
+; GFX10: s_denorm_mode 15
 ; GCN: v_fma_f32 [[A:v[0-9]+]], -[[NUM_SCALE]], [[NUM_RCP]], 1.0
 ; GCN: v_fma_f32 [[B:v[0-9]+]], [[A]], [[NUM_RCP]], [[NUM_RCP]]
 ; GCN: v_mul_f32_e32 [[C:v[0-9]+]], [[DEN_SCALE]], [[B]]
 ; GCN: v_fma_f32 [[D:v[0-9]+]], -[[NUM_SCALE]], [[C]], [[DEN_SCALE]]
 ; GCN: v_fma_f32 [[E:v[0-9]+]], [[D]], [[B]], [[C]]
 ; GCN: v_fma_f32 [[F:v[0-9]+]], -[[NUM_SCALE]], [[E]], [[DEN_SCALE]]
-; GCN: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
+; PREGFX10: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
+; GFX10: s_denorm_mode 12
 ; GCN: v_div_fmas_f32 [[FMAS:v[0-9]+]], [[F]], [[B]], [[E]]
 ; GCN: v_div_fixup_f32 v{{[0-9]+}}, [[FMAS]],
 define amdgpu_kernel void @fdiv_f32(float addrspace(1)* %out, float %a, float %b) #0 {
@@ -39,17 +42,28 @@ entry:
 ; R600-DAG: MUL_IEEE {{\** *}}T{{[0-9]+\.[XYZW]}}, KC0[2].Z, PS
 
 ; GCN: v_div_scale_f32 [[NUM_SCALE:v[0-9]+]]
-; GCN-DAG: v_div_scale_f32 [[DEN_SCALE:v[0-9]+]]
 ; GCN-DAG: v_rcp_f32_e32 [[NUM_RCP:v[0-9]+]], [[NUM_SCALE]]
 
-; GCN-NOT: s_setreg
-; GCN: v_fma_f32 [[A:v[0-9]+]], -[[NUM_SCALE]], [[NUM_RCP]], 1.0
-; GCN: v_fma_f32 [[B:v[0-9]+]], [[A]], [[NUM_RCP]], [[NUM_RCP]]
-; GCN: v_mul_f32_e32 [[C:v[0-9]+]], [[DEN_SCALE]], [[B]]
-; GCN: v_fma_f32 [[D:v[0-9]+]], -[[NUM_SCALE]], [[C]], [[DEN_SCALE]]
-; GCN: v_fma_f32 [[E:v[0-9]+]], [[D]], [[B]], [[C]]
-; GCN: v_fma_f32 [[F:v[0-9]+]], -[[NUM_SCALE]], [[E]], [[DEN_SCALE]]
-; GCN-NOT: s_setreg
+; PREGFX10-DAG: v_div_scale_f32 [[DEN_SCALE:v[0-9]+]]
+; PREGFX10-NOT: s_setreg
+; PREGFX10: v_fma_f32 [[A:v[0-9]+]], -[[NUM_SCALE]], [[NUM_RCP]], 1.0
+; PREGFX10: v_fma_f32 [[B:v[0-9]+]], [[A]], [[NUM_RCP]], [[NUM_RCP]]
+; PREGFX10: v_mul_f32_e32 [[C:v[0-9]+]], [[DEN_SCALE]], [[B]]
+; PREGFX10: v_fma_f32 [[D:v[0-9]+]], -[[NUM_SCALE]], [[C]], [[DEN_SCALE]]
+; PREGFX10: v_fma_f32 [[E:v[0-9]+]], [[D]], [[B]], [[C]]
+; PREGFX10: v_fma_f32 [[F:v[0-9]+]], -[[NUM_SCALE]], [[E]], [[DEN_SCALE]]
+; PREGFX10-NOT: s_setreg
+
+; GFX10-NOT: s_denorm_mode
+; GFX10: v_fma_f32 [[A:v[0-9]+]], -[[NUM_SCALE]], [[NUM_RCP]], 1.0
+; GFX10: v_fmac_f32_e32 [[B:v[0-9]+]], [[A]], [[NUM_RCP]]
+; GFX10: v_div_scale_f32 [[DEN_SCALE:v[0-9]+]]
+; GFX10: v_mul_f32_e32 [[C:v[0-9]+]], [[DEN_SCALE]], [[B]]
+; GFX10: v_fma_f32 [[D:v[0-9]+]], [[C]], -[[NUM_SCALE]], [[DEN_SCALE]]
+; GFX10: v_fmac_f32_e32 [[E:v[0-9]+]], [[D]], [[B]]
+; GFX10: v_fmac_f32_e64 [[F:v[0-9]+]], -[[NUM_SCALE]], [[E]]
+; GFX10-NOT: s_denorm_mode
+
 ; GCN: v_div_fmas_f32 [[FMAS:v[0-9]+]], [[F]], [[B]], [[E]]
 ; GCN: v_div_fixup_f32 v{{[0-9]+}}, [[FMAS]],
 define amdgpu_kernel void @fdiv_f32_denormals(float addrspace(1)* %out, float %a, float %b) #2 {
@@ -88,7 +102,8 @@ entry:
 ; GCN: v_rcp_f32_e32 [[RCP:v[0-9]+]], s{{[0-9]+}}
 ; GCN: v_mul_f32_e32 [[RESULT:v[0-9]+]], s{{[0-9]+}}, [[RCP]]
 ; GCN-NOT: [[RESULT]]
-; GCN-NOT: s_setreg
+; PREGFX10-NOT: s_setreg
+; GFX10-NOT: s_denorm_mode
 ; GCN: buffer_store_dword [[RESULT]]
 define amdgpu_kernel void @fdiv_fast_denormals_f32(float addrspace(1)* %out, float %a, float %b) #2 {
 entry:
@@ -99,7 +114,7 @@ entry:
 
 ; FUNC-LABEL: {{^}}fdiv_f32_fast_math:
 ; R600-DAG: RECIP_IEEE * T{{[0-9]+\.[XYZW]}}, KC0[2].W
-; R600-DAG: MUL_IEEE {{\** *}}T{{[0-9]+\.[XYZW]}}, KC0[2].Z, PS
+; R600-DAG: MUL_IEEE {{\** *}}T{{[0-9]+\.[XYZW]}}, PS, KC0[2].Z,
 
 ; GCN: v_rcp_f32_e32 [[RCP:v[0-9]+]], s{{[0-9]+}}
 ; GCN: v_mul_f32_e32 [[RESULT:v[0-9]+]], s{{[0-9]+}}, [[RCP]]
@@ -114,7 +129,7 @@ entry:
 
 ; FUNC-LABEL: {{^}}fdiv_ulp25_f32_fast_math:
 ; R600-DAG: RECIP_IEEE * T{{[0-9]+\.[XYZW]}}, KC0[2].W
-; R600-DAG: MUL_IEEE {{\** *}}T{{[0-9]+\.[XYZW]}}, KC0[2].Z, PS
+; R600-DAG: MUL_IEEE {{\** *}}T{{[0-9]+\.[XYZW]}}, PS, KC0[2].Z,
 
 ; GCN: v_rcp_f32_e32 [[RCP:v[0-9]+]], s{{[0-9]+}}
 ; GCN: v_mul_f32_e32 [[RESULT:v[0-9]+]], s{{[0-9]+}}, [[RCP]]
@@ -129,7 +144,7 @@ entry:
 
 ; FUNC-LABEL: {{^}}fdiv_f32_arcp_math:
 ; R600-DAG: RECIP_IEEE * T{{[0-9]+\.[XYZW]}}, KC0[2].W
-; R600-DAG: MUL_IEEE {{\** *}}T{{[0-9]+\.[XYZW]}}, KC0[2].Z, PS
+; R600-DAG: MUL_IEEE {{\** *}}T{{[0-9]+\.[XYZW]}}, PS, KC0[2].Z,
 
 ; GCN: v_rcp_f32_e32 [[RCP:v[0-9]+]], s{{[0-9]+}}
 ; GCN: v_mul_f32_e32 [[RESULT:v[0-9]+]], s{{[0-9]+}}, [[RCP]]
@@ -173,8 +188,8 @@ entry:
 ; FUNC-LABEL: {{^}}fdiv_v2f32_fast_math:
 ; R600-DAG: RECIP_IEEE * T{{[0-9]+\.[XYZW]}}, KC0[3].Z
 ; R600-DAG: RECIP_IEEE * T{{[0-9]+\.[XYZW]}}, KC0[3].Y
-; R600-DAG: MUL_IEEE {{\** *}}T{{[0-9]+\.[XYZW]}}, KC0[3].X, PS
-; R600-DAG: MUL_IEEE {{\** *}}T{{[0-9]+\.[XYZW]}}, KC0[2].W, PS
+; R600-DAG: MUL_IEEE {{\** *}}T{{[0-9]+\.[XYZW]}}, PS, KC0[3].X,
+; R600-DAG: MUL_IEEE {{\** *}}T{{[0-9]+\.[XYZW]}}, PS, KC0[2].W,
 
 ; GCN: v_rcp_f32
 ; GCN: v_rcp_f32
@@ -188,8 +203,8 @@ entry:
 ; FUNC-LABEL: {{^}}fdiv_v2f32_arcp_math:
 ; R600-DAG: RECIP_IEEE * T{{[0-9]+\.[XYZW]}}, KC0[3].Z
 ; R600-DAG: RECIP_IEEE * T{{[0-9]+\.[XYZW]}}, KC0[3].Y
-; R600-DAG: MUL_IEEE {{\** *}}T{{[0-9]+\.[XYZW]}}, KC0[3].X, PS
-; R600-DAG: MUL_IEEE {{\** *}}T{{[0-9]+\.[XYZW]}}, KC0[2].W, PS
+; R600-DAG: MUL_IEEE {{\** *}}T{{[0-9]+\.[XYZW]}}, PS, KC0[3].X,
+; R600-DAG: MUL_IEEE {{\** *}}T{{[0-9]+\.[XYZW]}}, PS, KC0[2].W,
 
 ; GCN: v_rcp_f32
 ; GCN: v_rcp_f32
@@ -228,10 +243,10 @@ define amdgpu_kernel void @fdiv_v4f32(<4 x float> addrspace(1)* %out, <4 x float
 ; R600-DAG: RECIP_IEEE * T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
 ; R600-DAG: RECIP_IEEE * T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
 ; R600-DAG: RECIP_IEEE * T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
-; R600-DAG: MUL_IEEE {{\** *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}, PS
-; R600-DAG: MUL_IEEE {{\** *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}, PS
-; R600-DAG: MUL_IEEE {{\** *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}, PS
-; R600-DAG: MUL_IEEE {{\** *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}, PS
+; R600-DAG: MUL_IEEE {{\** *}}T{{[0-9]+\.[XYZW], PS, T[0-9]+\.[XYZW]}},
+; R600-DAG: MUL_IEEE {{\** *}}T{{[0-9]+\.[XYZW], PS, T[0-9]+\.[XYZW]}},
+; R600-DAG: MUL_IEEE {{\** *}}T{{[0-9]+\.[XYZW], PS, T[0-9]+\.[XYZW]}},
+; R600-DAG: MUL_IEEE {{\** *}}T{{[0-9]+\.[XYZW], PS, T[0-9]+\.[XYZW]}},
 
 ; GCN: v_rcp_f32
 ; GCN: v_rcp_f32
@@ -251,10 +266,10 @@ define amdgpu_kernel void @fdiv_v4f32_fast_math(<4 x float> addrspace(1)* %out,
 ; R600-DAG: RECIP_IEEE * T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
 ; R600-DAG: RECIP_IEEE * T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
 ; R600-DAG: RECIP_IEEE * T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
-; R600-DAG: MUL_IEEE {{\** *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}, PS
-; R600-DAG: MUL_IEEE {{\** *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}, PS
-; R600-DAG: MUL_IEEE {{\** *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}, PS
-; R600-DAG: MUL_IEEE {{\** *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}, PS
+; R600-DAG: MUL_IEEE {{\** *}}T{{[0-9]+\.[XYZW], PS, T[0-9]+\.[XYZW]}},
+; R600-DAG: MUL_IEEE {{\** *}}T{{[0-9]+\.[XYZW], PS, T[0-9]+\.[XYZW]}},
+; R600-DAG: MUL_IEEE {{\** *}}T{{[0-9]+\.[XYZW], PS, T[0-9]+\.[XYZW]}},
+; R600-DAG: MUL_IEEE {{\** *}}T{{[0-9]+\.[XYZW], PS, T[0-9]+\.[XYZW]}},
 
 ; GCN: v_rcp_f32
 ; GCN: v_rcp_f32
@@ -269,7 +284,7 @@ define amdgpu_kernel void @fdiv_v4f32_arcp_math(<4 x float> addrspace(1)* %out,
   ret void
 }
 
-attributes #0 = { nounwind "enable-unsafe-fp-math"="false" "target-features"="-fp32-denormals,-flat-for-global" }
+attributes #0 = { nounwind "enable-unsafe-fp-math"="false" "target-features"="-fp32-denormals,+fp64-fp16-denormals,-flat-for-global" }
 attributes #1 = { nounwind "enable-unsafe-fp-math"="true" "target-features"="-fp32-denormals,-flat-for-global" }
 attributes #2 = { nounwind "enable-unsafe-fp-math"="false" "target-features"="+fp32-denormals,-flat-for-global" }
 
diff --git a/llvm/test/CodeGen/AMDGPU/fdiv32-to-rcp-folding.ll b/llvm/test/CodeGen/AMDGPU/fdiv32-to-rcp-folding.ll
index a3f176b3e..01499e681 100644
--- a/llvm/test/CodeGen/AMDGPU/fdiv32-to-rcp-folding.ll
+++ b/llvm/test/CodeGen/AMDGPU/fdiv32-to-rcp-folding.ll
@@ -85,15 +85,15 @@ define amdgpu_kernel void @div_minus_1_by_minus_x_25ulp(float addrspace(1)* %arg
 
 ; GCN-LABEL: {{^}}div_v4_1_by_x_25ulp:
 ; GCN-DAG:        s_load_dwordx4 s{{\[}}[[VAL0:[0-9]+]]:[[VAL3:[0-9]+]]], s[{{[0-9:]+}}], 0x0{{$}}
-; GCN-DENORM-DAG: s_mov_b32 [[L:s[0-9]+]], 0x6f800000
+; GCN-DENORM-DAG: v_mov_b32_e32 [[L:v[0-9]+]], 0x6f800000
 ; GCN-DENORM-DAG: v_mov_b32_e32 [[S:v[0-9]+]], 0x2f800000
-; GCN-DENORM-DAG: v_cmp_gt_f32_e64 vcc, |v{{[0-9]+}}|, [[L]]
+; GCN-DENORM-DAG: v_cmp_gt_f32_e64 vcc, |s{{[0-9]+}}|, [[L]]
 ; GCN-DENORM-DAG: v_cndmask_b32_e32 v{{[0-9]+}}, 1.0, [[S]], vcc
-; GCN-DENORM-DAG: v_cmp_gt_f32_e64 vcc, |v{{[0-9]+}}|, [[L]]
+; GCN-DENORM-DAG: v_cmp_gt_f32_e64 vcc, |s{{[0-9]+}}|, [[L]]
 ; GCN-DENORM-DAG: v_cndmask_b32_e32 v{{[0-9]+}}, 1.0, [[S]], vcc
-; GCN-DENORM-DAG: v_cmp_gt_f32_e64 vcc, |v{{[0-9]+}}|, [[L]]
+; GCN-DENORM-DAG: v_cmp_gt_f32_e64 vcc, |s{{[0-9]+}}|, [[L]]
 ; GCN-DENORM-DAG: v_cndmask_b32_e32 v{{[0-9]+}}, 1.0, [[S]], vcc
-; GCN-DENORM-DAG: v_cmp_gt_f32_e64 vcc, |v{{[0-9]+}}|, [[L]]
+; GCN-DENORM-DAG: v_cmp_gt_f32_e64 vcc, |s{{[0-9]+}}|, [[L]]
 ; GCN-DENORM-DAG: v_cndmask_b32_e32 v{{[0-9]+}}, 1.0, [[S]], vcc
 ; GCN-DENORM-DAG: v_mul_f32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 ; GCN-DENORM-DAG: v_mul_f32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
@@ -121,15 +121,15 @@ define amdgpu_kernel void @div_v4_1_by_x_25ulp(<4 x float> addrspace(1)* %arg) {
 }
 
 ; GCN-LABEL: {{^}}div_v4_minus_1_by_x_25ulp:
-; GCN-DENORM-DAG: s_mov_b32 [[L:s[0-9]+]], 0x6f800000
+; GCN-DENORM-DAG: v_mov_b32_e32 [[L:v[0-9]+]], 0x6f800000
 ; GCN-DENORM-DAG: v_mov_b32_e32 [[S:v[0-9]+]], 0x2f800000
-; GCN-DENORM-DAG: v_cmp_gt_f32_e64 vcc, |v{{[0-9]+}}|, [[L]]
+; GCN-DENORM-DAG: v_cmp_gt_f32_e64 vcc, |s{{[0-9]+}}|, [[L]]
 ; GCN-DENORM-DAG: v_cndmask_b32_e32 v{{[0-9]+}}, 1.0, [[S]], vcc
-; GCN-DENORM-DAG: v_cmp_gt_f32_e64 vcc, |v{{[0-9]+}}|, [[L]]
+; GCN-DENORM-DAG: v_cmp_gt_f32_e64 vcc, |s{{[0-9]+}}|, [[L]]
 ; GCN-DENORM-DAG: v_cndmask_b32_e32 v{{[0-9]+}}, 1.0, [[S]], vcc
-; GCN-DENORM-DAG: v_cmp_gt_f32_e64 vcc, |v{{[0-9]+}}|, [[L]]
+; GCN-DENORM-DAG: v_cmp_gt_f32_e64 vcc, |s{{[0-9]+}}|, [[L]]
 ; GCN-DENORM-DAG: v_cndmask_b32_e32 v{{[0-9]+}}, 1.0, [[S]], vcc
-; GCN-DENORM-DAG: v_cmp_gt_f32_e64 vcc, |v{{[0-9]+}}|, [[L]]
+; GCN-DENORM-DAG: v_cmp_gt_f32_e64 vcc, |s{{[0-9]+}}|, [[L]]
 ; GCN-DENORM-DAG: v_cndmask_b32_e32 v{{[0-9]+}}, 1.0, [[S]], vcc
 ; GCN-DENORM-DAG: v_mul_f32_e64 v{{[0-9]+}}, s{{[0-9]+}}, -v{{[0-9]+}}
 ; GCN-DENORM-DAG: v_mul_f32_e64 v{{[0-9]+}}, s{{[0-9]+}}, -v{{[0-9]+}}
@@ -156,15 +156,15 @@ define amdgpu_kernel void @div_v4_minus_1_by_x_25ulp(<4 x float> addrspace(1)* %
 }
 
 ; GCN-LABEL: {{^}}div_v4_1_by_minus_x_25ulp:
-; GCN-DENORM-DAG: s_mov_b32 [[L:s[0-9]+]], 0x6f800000
+; GCN-DENORM-DAG: v_mov_b32_e32 [[L:v[0-9]+]], 0x6f800000
 ; GCN-DENORM-DAG: v_mov_b32_e32 [[S:v[0-9]+]], 0x2f800000
-; GCN-DENORM-DAG: v_cmp_gt_f32_e64 vcc, |v{{[0-9]+}}|, [[L]]
+; GCN-DENORM-DAG: v_cmp_gt_f32_e64 vcc, |s{{[0-9]+}}|, [[L]]
 ; GCN-DENORM-DAG: v_cndmask_b32_e32 v{{[0-9]+}}, 1.0, [[S]], vcc
-; GCN-DENORM-DAG: v_cmp_gt_f32_e64 vcc, |v{{[0-9]+}}|, [[L]]
+; GCN-DENORM-DAG: v_cmp_gt_f32_e64 vcc, |s{{[0-9]+}}|, [[L]]
 ; GCN-DENORM-DAG: v_cndmask_b32_e32 v{{[0-9]+}}, 1.0, [[S]], vcc
-; GCN-DENORM-DAG: v_cmp_gt_f32_e64 vcc, |v{{[0-9]+}}|, [[L]]
+; GCN-DENORM-DAG: v_cmp_gt_f32_e64 vcc, |s{{[0-9]+}}|, [[L]]
 ; GCN-DENORM-DAG: v_cndmask_b32_e32 v{{[0-9]+}}, 1.0, [[S]], vcc
-; GCN-DENORM-DAG: v_cmp_gt_f32_e64 vcc, |v{{[0-9]+}}|, [[L]]
+; GCN-DENORM-DAG: v_cmp_gt_f32_e64 vcc, |s{{[0-9]+}}|, [[L]]
 ; GCN-DENORM-DAG: v_cndmask_b32_e32 v{{[0-9]+}}, 1.0, [[S]], vcc
 ; GCN-DENORM-DAG: v_mul_f32_e64 v{{[0-9]+}}, -s{{[0-9]+}}, v{{[0-9]+}}
 ; GCN-DENORM-DAG: v_mul_f32_e64 v{{[0-9]+}}, -s{{[0-9]+}}, v{{[0-9]+}}
@@ -194,15 +194,15 @@ define amdgpu_kernel void @div_v4_1_by_minus_x_25ulp(<4 x float> addrspace(1)* %
 
 ; GCN-LABEL: {{^}}div_v4_minus_1_by_minus_x_25ulp:
 ; GCN-DAG:        s_load_dwordx4 s{{\[}}[[VAL0:[0-9]+]]:[[VAL3:[0-9]+]]], s[{{[0-9:]+}}], 0x0{{$}}
-; GCN-DENORM-DAG: s_mov_b32 [[L:s[0-9]+]], 0x6f800000
+; GCN-DENORM-DAG: v_mov_b32_e32 [[L:v[0-9]+]], 0x6f800000
 ; GCN-DENORM-DAG: v_mov_b32_e32 [[S:v[0-9]+]], 0x2f800000
-; GCN-DENORM-DAG: v_cmp_gt_f32_e64 vcc, |v{{[0-9]+}}|, [[L]]
+; GCN-DENORM-DAG: v_cmp_gt_f32_e64 vcc, |s{{[0-9]+}}|, [[L]]
 ; GCN-DENORM-DAG: v_cndmask_b32_e32 v{{[0-9]+}}, 1.0, [[S]], vcc
-; GCN-DENORM-DAG: v_cmp_gt_f32_e64 vcc, |v{{[0-9]+}}|, [[L]]
+; GCN-DENORM-DAG: v_cmp_gt_f32_e64 vcc, |s{{[0-9]+}}|, [[L]]
 ; GCN-DENORM-DAG: v_cndmask_b32_e32 v{{[0-9]+}}, 1.0, [[S]], vcc
-; GCN-DENORM-DAG: v_cmp_gt_f32_e64 vcc, |v{{[0-9]+}}|, [[L]]
+; GCN-DENORM-DAG: v_cmp_gt_f32_e64 vcc, |s{{[0-9]+}}|, [[L]]
 ; GCN-DENORM-DAG: v_cndmask_b32_e32 v{{[0-9]+}}, 1.0, [[S]], vcc
-; GCN-DENORM-DAG: v_cmp_gt_f32_e64 vcc, |v{{[0-9]+}}|, [[L]]
+; GCN-DENORM-DAG: v_cmp_gt_f32_e64 vcc, |s{{[0-9]+}}|, [[L]]
 ; GCN-DENORM-DAG: v_cndmask_b32_e32 v{{[0-9]+}}, 1.0, [[S]], vcc
 ; GCN-DENORM-DAG: v_mul_f32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 ; GCN-DENORM-DAG: v_mul_f32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
@@ -231,8 +231,6 @@ define amdgpu_kernel void @div_v4_minus_1_by_minus_x_25ulp(<4 x float> addrspace
 }
 
 ; GCN-LABEL: {{^}}div_v4_c_by_x_25ulp:
-; GCN-DAG:        s_mov_b32 [[L:s[0-9]+]], 0x6f800000
-; GCN-DAG:        v_mov_b32_e32 [[S:v[0-9]+]], 0x2f800000
 ; GCN-DENORM-DAG: v_div_scale_f32 {{.*}}, 2.0{{$}}
 ; GCN-DENORM-DAG: v_div_scale_f32 {{.*}}, 2.0{{$}}
 ; GCN-DENORM-DAG: v_div_scale_f32 {{.*}}, -2.0{{$}}
@@ -240,9 +238,12 @@ define amdgpu_kernel void @div_v4_minus_1_by_minus_x_25ulp(<4 x float> addrspace
 ; GCN-DENORM-DAG: v_rcp_f32_e32
 ; GCN-DENORM-DAG: v_rcp_f32_e32
 
-; GCN-DAG:        v_cmp_gt_f32_e64 vcc, |v{{[0-9]+}}|, [[L]]
+; GCN-DAG:        v_mov_b32_e32 [[L:v[0-9]+]], 0x6f800000
+; GCN-DAG:        v_mov_b32_e32 [[S:v[0-9]+]], 0x2f800000
+
+; GCN-DAG:        v_cmp_gt_f32_e64 vcc, |s{{[0-9]+}}|, [[L]]
 ; GCN-DAG:        v_cndmask_b32_e32 v{{[0-9]+}}, 1.0, [[S]], vcc
-; GCN-DAG:        v_cmp_gt_f32_e64 vcc, |v{{[0-9]+}}|, [[L]]
+; GCN-DAG:        v_cmp_gt_f32_e64 vcc, |s{{[0-9]+}}|, [[L]]
 ; GCN-DAG:        v_cndmask_b32_e32 v{{[0-9]+}}, 1.0, [[S]], vcc
 
 ; GCN-DENORM-DAG: v_mul_f32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
@@ -273,8 +274,6 @@ define amdgpu_kernel void @div_v4_c_by_x_25ulp(<4 x float> addrspace(1)* %arg) {
 }
 
 ; GCN-LABEL: {{^}}div_v4_c_by_minus_x_25ulp:
-; GCN-DAG:        s_mov_b32 [[L:s[0-9]+]], 0x6f800000
-; GCN-DAG:        v_mov_b32_e32 [[S:v[0-9]+]], 0x2f800000
 ; GCN-DENORM-DAG: v_div_scale_f32 {{.*}}, -2.0{{$}}
 ; GCN-DENORM-DAG: v_div_scale_f32 {{.*}}, -2.0{{$}}
 ; GCN-DENORM-DAG: v_div_scale_f32 {{.*}}, -2.0{{$}}
@@ -282,9 +281,12 @@ define amdgpu_kernel void @div_v4_c_by_x_25ulp(<4 x float> addrspace(1)* %arg) {
 ; GCN-DENORM-DAG: v_rcp_f32_e32
 ; GCN-DENORM-DAG: v_rcp_f32_e32
 
-; GCN-DAG:        v_cmp_gt_f32_e64 vcc, |v{{[0-9]+}}|, [[L]]
+; GCN-DAG:        v_mov_b32_e32 [[L:v[0-9]+]], 0x6f800000
+; GCN-DAG:        v_mov_b32_e32 [[S:v[0-9]+]], 0x2f800000
+
+; GCN-DAG:        v_cmp_gt_f32_e64 vcc, |s{{[0-9]+}}|, [[L]]
 ; GCN-DAG:        v_cndmask_b32_e32 v{{[0-9]+}}, 1.0, [[S]], vcc
-; GCN-DAG:        v_cmp_gt_f32_e64 vcc, |v{{[0-9]+}}|, [[L]]
+; GCN-DAG:        v_cmp_gt_f32_e64 vcc, |s{{[0-9]+}}|, [[L]]
 ; GCN-DAG:        v_cndmask_b32_e32 v{{[0-9]+}}, 1.0, [[S]], vcc
 
 ; GCN-DENORM-DAG: v_mul_f32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
diff --git a/llvm/test/CodeGen/AMDGPU/fence-barrier.ll b/llvm/test/CodeGen/AMDGPU/fence-barrier.ll
index 7de4f1796..8f5a06d01 100644
--- a/llvm/test/CodeGen/AMDGPU/fence-barrier.ll
+++ b/llvm/test/CodeGen/AMDGPU/fence-barrier.ll
@@ -54,8 +54,7 @@ define amdgpu_kernel void @test_local(i32 addrspace(1)*) {
 }
 
 ; GCN-LABEL: {{^}}test_global
-; GCN: s_movk_i32 [[K:s[0-9]+]], 0x888
-; GCN: v_add_u32_e32 v{{[0-9]+}}, vcc, [[K]], v{{[0-9]+}}
+; GCN: v_add_u32_e32 v{{[0-9]+}}, vcc, 0x888, v{{[0-9]+}}
 ; GCN: flat_store_dword
 ; GCN: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
 ; GCN-NEXT: s_barrier
diff --git a/llvm/test/CodeGen/AMDGPU/fexp.ll b/llvm/test/CodeGen/AMDGPU/fexp.ll
index 9dc2f7574..d03233b3a 100644
--- a/llvm/test/CodeGen/AMDGPU/fexp.ll
+++ b/llvm/test/CodeGen/AMDGPU/fexp.ll
@@ -1,4 +1,3 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ;RUN: llc -mtriple=amdgcn-- < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=SI %s
 ;RUN: llc -mtriple=amdgcn-- -mcpu=fiji < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=VI %s
 ;RUN: llc -mtriple=amdgcn-- -mcpu=gfx900 < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=GFX9 %s
diff --git a/llvm/test/CodeGen/AMDGPU/ffloor.f64.ll b/llvm/test/CodeGen/AMDGPU/ffloor.f64.ll
index 407cccb84..d03f9c9e3 100644
--- a/llvm/test/CodeGen/AMDGPU/ffloor.f64.ll
+++ b/llvm/test/CodeGen/AMDGPU/ffloor.f64.ll
@@ -1,6 +1,6 @@
-; RUN: llc -march=amdgcn -verify-machineinstrs -enable-unsafe-fp-math < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
-; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs -enable-unsafe-fp-math < %s | FileCheck -check-prefix=CI -check-prefix=FUNC %s
-; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs -enable-unsafe-fp-math < %s | FileCheck -check-prefix=CI -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs < %s | FileCheck -check-prefix=CI -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=CI -check-prefix=FUNC %s
 
 declare double @llvm.fabs.f64(double %Val)
 declare double @llvm.floor.f64(double) nounwind readnone
@@ -20,7 +20,7 @@ declare <16 x double> @llvm.floor.v16f64(<16 x double>) nounwind readnone
 ; SI: v_add_f64
 ; SI: s_endpgm
 define amdgpu_kernel void @ffloor_f64(double addrspace(1)* %out, double %x) {
-  %y = call double @llvm.floor.f64(double %x) nounwind readnone
+  %y = call fast double @llvm.floor.f64(double %x) nounwind readnone
   store double %y, double addrspace(1)* %out
   ret void
 }
@@ -35,8 +35,8 @@ define amdgpu_kernel void @ffloor_f64(double addrspace(1)* %out, double %x) {
 ; SI: v_add_f64 {{v[[0-9]+:[0-9]+]}}, -[[INPUT]]
 ; SI: s_endpgm
 define amdgpu_kernel void @ffloor_f64_neg(double addrspace(1)* %out, double %x) {
-  %neg = fsub double 0.0, %x
-  %y = call double @llvm.floor.f64(double %neg) nounwind readnone
+  %neg = fsub nsz double 0.0, %x
+  %y = call fast double @llvm.floor.f64(double %neg) nounwind readnone
   store double %y, double addrspace(1)* %out
   ret void
 }
@@ -51,9 +51,9 @@ define amdgpu_kernel void @ffloor_f64_neg(double addrspace(1)* %out, double %x)
 ; SI: v_add_f64 {{v[[0-9]+:[0-9]+]}}, -|[[INPUT]]|
 ; SI: s_endpgm
 define amdgpu_kernel void @ffloor_f64_neg_abs(double addrspace(1)* %out, double %x) {
-  %abs = call double @llvm.fabs.f64(double %x)
-  %neg = fsub double 0.0, %abs
-  %y = call double @llvm.floor.f64(double %neg) nounwind readnone
+  %abs = call fast double @llvm.fabs.f64(double %x)
+  %neg = fsub nsz double 0.0, %abs
+  %y = call fast double @llvm.floor.f64(double %neg) nounwind readnone
   store double %y, double addrspace(1)* %out
   ret void
 }
@@ -62,7 +62,7 @@ define amdgpu_kernel void @ffloor_f64_neg_abs(double addrspace(1)* %out, double
 ; CI: v_floor_f64_e32
 ; CI: v_floor_f64_e32
 define amdgpu_kernel void @ffloor_v2f64(<2 x double> addrspace(1)* %out, <2 x double> %x) {
-  %y = call <2 x double> @llvm.floor.v2f64(<2 x double> %x) nounwind readnone
+  %y = call fast <2 x double> @llvm.floor.v2f64(<2 x double> %x) nounwind readnone
   store <2 x double> %y, <2 x double> addrspace(1)* %out
   ret void
 }
@@ -73,7 +73,7 @@ define amdgpu_kernel void @ffloor_v2f64(<2 x double> addrspace(1)* %out, <2 x do
 ; CI: v_floor_f64_e32
 ; CI-NOT: v_floor_f64_e32
 define amdgpu_kernel void @ffloor_v3f64(<3 x double> addrspace(1)* %out, <3 x double> %x) {
-  %y = call <3 x double> @llvm.floor.v3f64(<3 x double> %x) nounwind readnone
+  %y = call fast <3 x double> @llvm.floor.v3f64(<3 x double> %x) nounwind readnone
   store <3 x double> %y, <3 x double> addrspace(1)* %out
   ret void
 }
@@ -84,7 +84,7 @@ define amdgpu_kernel void @ffloor_v3f64(<3 x double> addrspace(1)* %out, <3 x do
 ; CI: v_floor_f64_e32
 ; CI: v_floor_f64_e32
 define amdgpu_kernel void @ffloor_v4f64(<4 x double> addrspace(1)* %out, <4 x double> %x) {
-  %y = call <4 x double> @llvm.floor.v4f64(<4 x double> %x) nounwind readnone
+  %y = call fast <4 x double> @llvm.floor.v4f64(<4 x double> %x) nounwind readnone
   store <4 x double> %y, <4 x double> addrspace(1)* %out
   ret void
 }
@@ -99,7 +99,7 @@ define amdgpu_kernel void @ffloor_v4f64(<4 x double> addrspace(1)* %out, <4 x do
 ; CI: v_floor_f64_e32
 ; CI: v_floor_f64_e32
 define amdgpu_kernel void @ffloor_v8f64(<8 x double> addrspace(1)* %out, <8 x double> %x) {
-  %y = call <8 x double> @llvm.floor.v8f64(<8 x double> %x) nounwind readnone
+  %y = call fast <8 x double> @llvm.floor.v8f64(<8 x double> %x) nounwind readnone
   store <8 x double> %y, <8 x double> addrspace(1)* %out
   ret void
 }
@@ -122,7 +122,7 @@ define amdgpu_kernel void @ffloor_v8f64(<8 x double> addrspace(1)* %out, <8 x do
 ; CI: v_floor_f64_e32
 ; CI: v_floor_f64_e32
 define amdgpu_kernel void @ffloor_v16f64(<16 x double> addrspace(1)* %out, <16 x double> %x) {
-  %y = call <16 x double> @llvm.floor.v16f64(<16 x double> %x) nounwind readnone
+  %y = call fast <16 x double> @llvm.floor.v16f64(<16 x double> %x) nounwind readnone
   store <16 x double> %y, <16 x double> addrspace(1)* %out
   ret void
 }
diff --git a/llvm/test/CodeGen/AMDGPU/flat-address-space.ll b/llvm/test/CodeGen/AMDGPU/flat-address-space.ll
index 5c45528f9..e48e8c96c 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-address-space.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-address-space.ll
@@ -1,14 +1,7 @@
-; RUN: llc -O0 -mtriple=amdgcn-mesa-mesa3d -mcpu=bonaire < %s | FileCheck -check-prefixes=CHECK,CIVI %s
-; RUN: llc -O0 -mtriple=amdgcn-mesa-mesa3d -mcpu=tonga -mattr=-flat-for-global < %s | FileCheck -check-prefixes=CHECK,CIVI %s
-; RUN: llc -O0 -mtriple=amdgcn-amd-amdhsa -mcpu=fiji -mattr=-flat-for-global < %s | FileCheck -check-prefixes=CHECK,HSA %s
-; RUN: llc -O0 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -mattr=-flat-for-global < %s | FileCheck -check-prefixes=CHECK,HSA,GFX9 %s
-
-; Disable optimizations in case there are optimizations added that
-; specialize away generic pointer accesses.
-
-
-; These testcases might become useless when there are optimizations to
-; remove generic pointers.
+; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=bonaire < %s | FileCheck -check-prefixes=CHECK,CIVI %s
+; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tonga -mattr=-flat-for-global < %s | FileCheck -check-prefixes=CHECK,CIVI %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=fiji -mattr=-flat-for-global < %s | FileCheck -check-prefixes=CHECK,CIVI,HSA %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -mattr=-flat-for-global < %s | FileCheck -check-prefixes=CHECK,HSA,GFX9 %s
 
 ; CHECK-LABEL: {{^}}store_flat_i32:
 ; CHECK-DAG: s_load_dwordx2 s{{\[}}[[LO_SREG:[0-9]+]]:[[HI_SREG:[0-9]+]]],
@@ -191,7 +184,11 @@ define amdgpu_kernel void @store_flat_i8_max_offset_p1(i8* %fptr, i8 %x) #0 {
 }
 
 ; CHECK-LABEL: {{^}}store_flat_i8_neg_offset:
-; CHECK: flat_store_byte v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}{{$}}
+; CIVI: flat_store_byte v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}{{$}}
+
+; GFX9: v_add_co_u32_e32 v{{[0-9]+}}, vcc, 0xfffff000, v
+; GFX9: v_addc_co_u32_e32 v{{[0-9]+}}, vcc, -1,
+; GFX9: flat_store_byte v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset:4094{{$}}
 define amdgpu_kernel void @store_flat_i8_neg_offset(i8* %fptr, i8 %x) #0 {
   %fptr.offset = getelementptr inbounds i8, i8* %fptr, i64 -2
   store volatile i8 %x, i8* %fptr.offset
@@ -216,7 +213,11 @@ define amdgpu_kernel void @load_flat_i8_max_offset_p1(i8* %fptr) #0 {
 }
 
 ; CHECK-LABEL: {{^}}load_flat_i8_neg_offset:
-; CHECK: flat_load_ubyte v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}{{$}}
+; CIVI: flat_load_ubyte v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}{{$}}
+
+; GFX9: v_add_co_u32_e32 v{{[0-9]+}}, vcc, 0xfffff000, v
+; GFX9: v_addc_co_u32_e32 v{{[0-9]+}}, vcc, -1,
+; GFX9: flat_load_ubyte v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}} offset:4094{{$}}
 define amdgpu_kernel void @load_flat_i8_neg_offset(i8* %fptr) #0 {
   %fptr.offset = getelementptr inbounds i8, i8* %fptr, i64 -2
   %val = load volatile i8, i8* %fptr.offset
diff --git a/llvm/test/CodeGen/AMDGPU/fma.f64.ll b/llvm/test/CodeGen/AMDGPU/fma.f64.ll
index 907121f1c..4bc8f5ec4 100644
--- a/llvm/test/CodeGen/AMDGPU/fma.f64.ll
+++ b/llvm/test/CodeGen/AMDGPU/fma.f64.ll
@@ -4,7 +4,7 @@
 declare double @llvm.fma.f64(double, double, double) nounwind readnone
 declare <2 x double> @llvm.fma.v2f64(<2 x double>, <2 x double>, <2 x double>) nounwind readnone
 declare <4 x double> @llvm.fma.v4f64(<4 x double>, <4 x double>, <4 x double>) nounwind readnone
-
+declare double @llvm.fabs.f64(double) nounwind readnone
 
 ; FUNC-LABEL: {{^}}fma_f64:
 ; SI: v_fma_f64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\]}}
@@ -45,3 +45,156 @@ define amdgpu_kernel void @fma_v4f64(<4 x double> addrspace(1)* %out, <4 x doubl
    store <4 x double> %r3, <4 x double> addrspace(1)* %out
    ret void
 }
+
+; FUNC-LABEL: {{^}}fma_f64_abs_src0:
+; SI: v_fma_f64 {{v\[[0-9]+:[0-9]+\], |v\[[0-9]+:[0-9]+\]|, v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\]}}
+define amdgpu_kernel void @fma_f64_abs_src0(double addrspace(1)* %out, double addrspace(1)* %in1,
+                     double addrspace(1)* %in2, double addrspace(1)* %in3) {
+   %r0 = load double, double addrspace(1)* %in1
+   %r1 = load double, double addrspace(1)* %in2
+   %r2 = load double, double addrspace(1)* %in3
+   %fabs = call double @llvm.fabs.f64(double %r0)
+   %r3 = tail call double @llvm.fma.f64(double %fabs, double %r1, double %r2)
+   store double %r3, double addrspace(1)* %out
+   ret void
+}
+
+; FUNC-LABEL: {{^}}fma_f64_abs_src1:
+; SI: v_fma_f64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], \|v\[[0-9]+:[0-9]+\]\|, v\[[0-9]+:[0-9]+\]}}
+define amdgpu_kernel void @fma_f64_abs_src1(double addrspace(1)* %out, double addrspace(1)* %in1,
+                     double addrspace(1)* %in2, double addrspace(1)* %in3) {
+   %r0 = load double, double addrspace(1)* %in1
+   %r1 = load double, double addrspace(1)* %in2
+   %r2 = load double, double addrspace(1)* %in3
+   %fabs = call double @llvm.fabs.f64(double %r1)
+   %r3 = tail call double @llvm.fma.f64(double %r0, double %fabs, double %r2)
+   store double %r3, double addrspace(1)* %out
+   ret void
+}
+
+; FUNC-LABEL: {{^}}fma_f64_abs_src2:
+; SI: v_fma_f64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], \|v\[[0-9]+:[0-9]+\]\|}}
+define amdgpu_kernel void @fma_f64_abs_src2(double addrspace(1)* %out, double addrspace(1)* %in1,
+                     double addrspace(1)* %in2, double addrspace(1)* %in3) {
+   %r0 = load double, double addrspace(1)* %in1
+   %r1 = load double, double addrspace(1)* %in2
+   %r2 = load double, double addrspace(1)* %in3
+   %fabs = call double @llvm.fabs.f64(double %r2)
+   %r3 = tail call double @llvm.fma.f64(double %r0, double %r1, double %fabs)
+   store double %r3, double addrspace(1)* %out
+   ret void
+}
+
+; FUNC-LABEL: {{^}}fma_f64_neg_src0:
+; SI: v_fma_f64 {{v\[[0-9]+:[0-9]+\], -v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\]}}
+define amdgpu_kernel void @fma_f64_neg_src0(double addrspace(1)* %out, double addrspace(1)* %in1,
+                     double addrspace(1)* %in2, double addrspace(1)* %in3) {
+   %r0 = load double, double addrspace(1)* %in1
+   %r1 = load double, double addrspace(1)* %in2
+   %r2 = load double, double addrspace(1)* %in3
+   %fsub = fsub double -0.000000e+00, %r0
+   %r3 = tail call double @llvm.fma.f64(double %fsub, double %r1, double %r2)
+   store double %r3, double addrspace(1)* %out
+   ret void
+}
+
+; FUNC-LABEL: {{^}}fma_f64_neg_src1:
+; SI: v_fma_f64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], -v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\]}}
+define amdgpu_kernel void @fma_f64_neg_src1(double addrspace(1)* %out, double addrspace(1)* %in1,
+                     double addrspace(1)* %in2, double addrspace(1)* %in3) {
+   %r0 = load double, double addrspace(1)* %in1
+   %r1 = load double, double addrspace(1)* %in2
+   %r2 = load double, double addrspace(1)* %in3
+   %fsub = fsub double -0.000000e+00, %r1
+   %r3 = tail call double @llvm.fma.f64(double %r0, double %fsub, double %r2)
+   store double %r3, double addrspace(1)* %out
+   ret void
+}
+
+; FUNC-LABEL: {{^}}fma_f64_neg_src2:
+; SI: v_fma_f64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], -v\[[0-9]+:[0-9]+\]}}
+define amdgpu_kernel void @fma_f64_neg_src2(double addrspace(1)* %out, double addrspace(1)* %in1,
+                     double addrspace(1)* %in2, double addrspace(1)* %in3) {
+   %r0 = load double, double addrspace(1)* %in1
+   %r1 = load double, double addrspace(1)* %in2
+   %r2 = load double, double addrspace(1)* %in3
+   %fsub = fsub double -0.000000e+00, %r2
+   %r3 = tail call double @llvm.fma.f64(double %r0, double %r1, double %fsub)
+   store double %r3, double addrspace(1)* %out
+   ret void
+}
+
+; FUNC-LABEL: {{^}}fma_f64_abs_neg_src0:
+; SI: v_fma_f64 {{v\[[0-9]+:[0-9]+\], -\|v\[[0-9]+:[0-9]+\]\|, v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\]}}
+define amdgpu_kernel void @fma_f64_abs_neg_src0(double addrspace(1)* %out, double addrspace(1)* %in1,
+                     double addrspace(1)* %in2, double addrspace(1)* %in3) {
+   %r0 = load double, double addrspace(1)* %in1
+   %r1 = load double, double addrspace(1)* %in2
+   %r2 = load double, double addrspace(1)* %in3
+   %fabs = call double @llvm.fabs.f64(double %r0)
+   %fsub = fsub double -0.000000e+00, %fabs
+   %r3 = tail call double @llvm.fma.f64(double %fsub, double %r1, double %r2)
+   store double %r3, double addrspace(1)* %out
+   ret void
+}
+
+; FUNC-LABEL: {{^}}fma_f64_abs_neg_src1:
+; SI: v_fma_f64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], -\|v\[[0-9]+:[0-9]+\]\|, v\[[0-9]+:[0-9]+\]}}
+define amdgpu_kernel void @fma_f64_abs_neg_src1(double addrspace(1)* %out, double addrspace(1)* %in1,
+                     double addrspace(1)* %in2, double addrspace(1)* %in3) {
+   %r0 = load double, double addrspace(1)* %in1
+   %r1 = load double, double addrspace(1)* %in2
+   %r2 = load double, double addrspace(1)* %in3
+   %fabs = call double @llvm.fabs.f64(double %r1)
+   %fsub = fsub double -0.000000e+00, %fabs
+   %r3 = tail call double @llvm.fma.f64(double %r0, double %fsub, double %r2)
+   store double %r3, double addrspace(1)* %out
+   ret void
+}
+
+; FUNC-LABEL: {{^}}fma_f64_abs_neg_src2:
+; SI: v_fma_f64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], -\|v\[[0-9]+:[0-9]+\]\|}}
+define amdgpu_kernel void @fma_f64_abs_neg_src2(double addrspace(1)* %out, double addrspace(1)* %in1,
+                     double addrspace(1)* %in2, double addrspace(1)* %in3) {
+   %r0 = load double, double addrspace(1)* %in1
+   %r1 = load double, double addrspace(1)* %in2
+   %r2 = load double, double addrspace(1)* %in3
+   %fabs = call double @llvm.fabs.f64(double %r2)
+   %fsub = fsub double -0.000000e+00, %fabs
+   %r3 = tail call double @llvm.fma.f64(double %r0, double %r1, double %fsub)
+   store double %r3, double addrspace(1)* %out
+   ret void
+}
+
+; FUNC-LABEL: {{^}}fma_f64_lit_src0:
+; SI: v_fma_f64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], 2.0, v\[[0-9]+:[0-9]+\]}}
+define amdgpu_kernel void @fma_f64_lit_src0(double addrspace(1)* %out,
+                     double addrspace(1)* %in2, double addrspace(1)* %in3) {
+   %r1 = load double, double addrspace(1)* %in2
+   %r2 = load double, double addrspace(1)* %in3
+   %r3 = tail call double @llvm.fma.f64(double +2.0, double %r1, double %r2)
+   store double %r3, double addrspace(1)* %out
+   ret void
+}
+
+; FUNC-LABEL: {{^}}fma_f64_lit_src1:
+; SI: v_fma_f64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], 2.0, v\[[0-9]+:[0-9]+\]}}
+define amdgpu_kernel void @fma_f64_lit_src1(double addrspace(1)* %out, double addrspace(1)* %in1,
+                     double addrspace(1)* %in3) {
+   %r0 = load double, double addrspace(1)* %in1
+   %r2 = load double, double addrspace(1)* %in3
+   %r3 = tail call double @llvm.fma.f64(double %r0, double +2.0, double %r2)
+   store double %r3, double addrspace(1)* %out
+   ret void
+}
+
+; FUNC-LABEL: {{^}}fma_f64_lit_src2:
+; SI: v_fma_f64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], 2.0}}
+define amdgpu_kernel void @fma_f64_lit_src2(double addrspace(1)* %out, double addrspace(1)* %in1,
+                     double addrspace(1)* %in2) {
+   %r0 = load double, double addrspace(1)* %in1
+   %r1 = load double, double addrspace(1)* %in2
+   %r3 = tail call double @llvm.fma.f64(double %r0, double %r1, double +2.0)
+   store double %r3, double addrspace(1)* %out
+   ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/fmac.sdwa.ll b/llvm/test/CodeGen/AMDGPU/fmac.sdwa.ll
index 8151c9df3..fad4ad802 100644
--- a/llvm/test/CodeGen/AMDGPU/fmac.sdwa.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmac.sdwa.ll
@@ -3,74 +3,96 @@
 ; GCN-LABEL: {{^}}addMul2D:
 ; GFX1010: v_fmac_f16
 ; GFX1010: v_fmac_f16
-define hidden <4 x half> @addMul2D(<4 x i8>* nocapture readonly, float addrspace(4)* nocapture readonly, <2 x i32>, i32) local_unnamed_addr #0 {
-  %5 = extractelement <2 x i32> %2, i64 1
-  %6 = icmp sgt i32 %5, 0
-  br i1 %6, label %7, label %38
+define hidden <4 x half> @addMul2D(<4 x i8>* nocapture readonly %arg, float addrspace(4)* nocapture readonly %arg1, <2 x i32> %arg2, i32 %arg3) local_unnamed_addr #0 {
+bb:
+  %tmp = extractelement <2 x i32> %arg2, i64 1
+  %tmp4 = icmp sgt i32 %tmp, 0
+  br i1 %tmp4, label %bb5, label %bb36
 
-7:                                                ; preds = %4
-  %8 = extractelement <2 x i32> %2, i64 0
-  %9 = icmp sgt i32 %8, 0
-  br label %10
+bb5:                                              ; preds = %bb
+  %tmp6 = extractelement <2 x i32> %arg2, i64 0
+  %tmp7 = icmp sgt i32 %tmp6, 0
+  br label %bb8
 
-10:                                               ; preds = %34, %7
-  %11 = phi <4 x half> [ zeroinitializer, %7 ], [ %35, %34 ]
-  %12 = phi i32 [ 0, %7 ], [ %36, %34 ]
-  br i1 %9, label %13, label %34
+bb8:                                              ; preds = %bb32, %bb5
+  %tmp9 = phi <4 x half> [ zeroinitializer, %bb5 ], [ %tmp33, %bb32 ]
+  %tmp10 = phi i32 [ 0, %bb5 ], [ %tmp34, %bb32 ]
+  br i1 %tmp7, label %bb11, label %bb32
 
-13:                                               ; preds = %10
-  %14 = mul nsw i32 %12, %3
-  %15 = mul nsw i32 %12, %8
-  br label %16
+bb11:                                             ; preds = %bb8
+  %tmp12 = mul nsw i32 %tmp10, %arg3
+  %tmp13 = mul nsw i32 %tmp10, %tmp6
+  br label %bb14
 
-16:                                               ; preds = %16, %13
-  %17 = phi <4 x half> [ %11, %13 ], [ %31, %16 ]
-  %18 = phi i32 [ 0, %13 ], [ %32, %16 ]
-  %19 = add nsw i32 %18, %14
-  %20 = sext i32 %19 to i64
-  %21 = getelementptr inbounds <4 x i8>, <4 x i8>* %0, i64 %20
-  %22 = load <4 x i8>, <4 x i8>* %21, align 4
-  %23 = tail call <4 x half> @_Z13convert_half4Dv4_h(<4 x i8> %22) #8
-  %24 = add nsw i32 %18, %15
-  %25 = sext i32 %24 to i64
-  %26 = getelementptr inbounds float, float addrspace(4)* %1, i64 %25
-  %27 = load float, float addrspace(4)* %26, align 4
-  %28 = fptrunc float %27 to half
-  %29 = insertelement <4 x half> undef, half %28, i32 0
-  %30 = shufflevector <4 x half> %29, <4 x half> undef, <4 x i32> zeroinitializer
-  %31 = tail call <4 x half> @llvm.fmuladd.v4f16(<4 x half> %23, <4 x half> %30, <4 x half> %17)
-  %32 = add nuw nsw i32 %18, 1
-  %33 = icmp eq i32 %32, %8
-  br i1 %33, label %34, label %16
+bb14:                                             ; preds = %bb14, %bb11
+  %tmp15 = phi <4 x half> [ %tmp9, %bb11 ], [ %tmp29, %bb14 ]
+  %tmp16 = phi i32 [ 0, %bb11 ], [ %tmp30, %bb14 ]
+  %tmp17 = add nsw i32 %tmp16, %tmp12
+  %tmp18 = sext i32 %tmp17 to i64
+  %tmp19 = getelementptr inbounds <4 x i8>, <4 x i8>* %arg, i64 %tmp18
+  %tmp20 = load <4 x i8>, <4 x i8>* %tmp19, align 4
+  %tmp21 = tail call <4 x half> @_Z13convert_half4Dv4_h(<4 x i8> %tmp20)
+  %tmp22 = add nsw i32 %tmp16, %tmp13
+  %tmp23 = sext i32 %tmp22 to i64
+  %tmp24 = getelementptr inbounds float, float addrspace(4)* %arg1, i64 %tmp23
+  %tmp25 = load float, float addrspace(4)* %tmp24, align 4
+  %tmp26 = fptrunc float %tmp25 to half
+  %tmp27 = insertelement <4 x half> undef, half %tmp26, i32 0
+  %tmp28 = shufflevector <4 x half> %tmp27, <4 x half> undef, <4 x i32> zeroinitializer
+  %vec.A.0 = extractelement <4 x half> %tmp21, i32 0
+  %vec.B.0 = extractelement <4 x half> %tmp28, i32 0
+  %vec.C.0 = extractelement <4 x half> %tmp15, i32 0
+  %vec.res.0 = tail call half @llvm.fmuladd.f16(half %vec.A.0, half %vec.B.0, half %vec.C.0)
+  %vec.A.1 = extractelement <4 x half> %tmp21, i32 1
+  %vec.B.1 = extractelement <4 x half> %tmp28, i32 1
+  %vec.C.1 = extractelement <4 x half> %tmp15, i32 1
+  %vec.res.1 = tail call half @llvm.fmuladd.f16(half %vec.A.1, half %vec.B.1, half %vec.C.1)
+  %vec.A.2 = extractelement <4 x half> %tmp21, i32 2
+  %vec.B.2 = extractelement <4 x half> %tmp28, i32 2
+  %vec.C.2 = extractelement <4 x half> %tmp15, i32 2
+  %vec.res.2 = tail call half @llvm.fmuladd.f16(half %vec.A.2, half %vec.B.2, half %vec.C.2)
+  %vec.A.3 = extractelement <4 x half> %tmp21, i32 3
+  %vec.B.3 = extractelement <4 x half> %tmp28, i32 3
+  %vec.C.3 = extractelement <4 x half> %tmp15, i32 3
+  %vec.res.3 = tail call half @llvm.fmuladd.f16(half %vec.A.3, half %vec.B.3, half %vec.C.3)
+  %full.res.0 = insertelement <4 x half> undef, half %vec.res.0, i32 0
+  %full.res.1 = insertelement <4 x half> %full.res.0, half %vec.res.1, i32 1
+  %full.res.2 = insertelement <4 x half> %full.res.1, half %vec.res.2, i32 2
+  %tmp29 = insertelement <4 x half> %full.res.2, half %vec.res.3, i32 3
+  %tmp30 = add nuw nsw i32 %tmp16, 1
+  %tmp31 = icmp eq i32 %tmp30, %tmp6
+  br i1 %tmp31, label %bb32, label %bb14
 
-34:                                               ; preds = %16, %10
-  %35 = phi <4 x half> [ %11, %10 ], [ %31, %16 ]
-  %36 = add nuw nsw i32 %12, 1
-  %37 = icmp eq i32 %36, %5
-  br i1 %37, label %38, label %10
+bb32:                                             ; preds = %bb14, %bb8
+  %tmp33 = phi <4 x half> [ %tmp9, %bb8 ], [ %tmp29, %bb14 ]
+  %tmp34 = add nuw nsw i32 %tmp10, 1
+  %tmp35 = icmp eq i32 %tmp34, %tmp
+  br i1 %tmp35, label %bb36, label %bb8
 
-38:                                               ; preds = %34, %4
-  %39 = phi <4 x half> [ zeroinitializer, %4 ], [ %35, %34 ]
-  ret <4 x half> %39
+bb36:                                             ; preds = %bb32, %bb
+  %tmp37 = phi <4 x half> [ zeroinitializer, %bb ], [ %tmp33, %bb32 ]
+  ret <4 x half> %tmp37
 }
 
-define linkonce_odr hidden <4 x half> @_Z13convert_half4Dv4_h(<4 x i8>) local_unnamed_addr #1 {
-  %2 = extractelement <4 x i8> %0, i64 0
-  %3 = uitofp i8 %2 to half
-  %4 = insertelement <4 x half> undef, half %3, i32 0
-  %5 = extractelement <4 x i8> %0, i64 1
-  %6 = uitofp i8 %5 to half
-  %7 = insertelement <4 x half> %4, half %6, i32 1
-  %8 = extractelement <4 x i8> %0, i64 2
-  %9 = uitofp i8 %8 to half
-  %10 = insertelement <4 x half> %7, half %9, i32 2
-  %11 = extractelement <4 x i8> %0, i64 3
-  %12 = uitofp i8 %11 to half
-  %13 = insertelement <4 x half> %10, half %12, i32 3
-  ret <4 x half> %13
+; Function Attrs: norecurse nounwind readnone
+define linkonce_odr hidden <4 x half> @_Z13convert_half4Dv4_h(<4 x i8> %arg) local_unnamed_addr #1 {
+bb:
+  %tmp = extractelement <4 x i8> %arg, i64 0
+  %tmp1 = uitofp i8 %tmp to half
+  %tmp2 = insertelement <4 x half> undef, half %tmp1, i32 0
+  %tmp3 = extractelement <4 x i8> %arg, i64 1
+  %tmp4 = uitofp i8 %tmp3 to half
+  %tmp5 = insertelement <4 x half> %tmp2, half %tmp4, i32 1
+  %tmp6 = extractelement <4 x i8> %arg, i64 2
+  %tmp7 = uitofp i8 %tmp6 to half
+  %tmp8 = insertelement <4 x half> %tmp5, half %tmp7, i32 2
+  %tmp9 = extractelement <4 x i8> %arg, i64 3
+  %tmp10 = uitofp i8 %tmp9 to half
+  %tmp11 = insertelement <4 x half> %tmp8, half %tmp10, i32 3
+  ret <4 x half> %tmp11
 }
 
-declare <4 x half> @llvm.fmuladd.v4f16(<4 x half>, <4 x half>, <4 x half>)
+declare half @llvm.fmuladd.f16(half, half, half)
 
 attributes #0 = { convergent nounwind readonly}
 attributes #1 = { norecurse nounwind readnone }
diff --git a/llvm/test/CodeGen/AMDGPU/fmin_legacy.ll b/llvm/test/CodeGen/AMDGPU/fmin_legacy.ll
index ca80c4edb..075115a2e 100644
--- a/llvm/test/CodeGen/AMDGPU/fmin_legacy.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmin_legacy.ll
@@ -33,9 +33,13 @@ define amdgpu_kernel void @s_test_fmin_legacy_subreg_inputs_f32(float addrspace(
 ; FUNC-LABEL: {{^}}s_test_fmin_legacy_ule_f32:
 ; GCN-DAG: s_load_dwordx2 s{{\[}}[[A:[0-9]+]]:[[B:[0-9]+]]{{\]}}, s{{\[[0-9]+:[0-9]+\]}}, {{0xb|0x2c}}
 
-; GCN-DAG: v_mov_b32_e32 [[VB:v[0-9]+]], s[[B]]
+; SI-SAFE: v_mov_b32_e32 [[VA:v[0-9]+]], s[[A]]
 
-; SI-SAFE: v_min_legacy_f32_e64 {{v[0-9]+}}, [[VB]], s[[A]]
+; GCN-NONAN: v_mov_b32_e32 [[VB:v[0-9]+]], s[[B]]
+
+; VI-SAFE: v_mov_b32_e32 [[VB:v[0-9]+]], s[[B]]
+
+; SI-SAFE: v_min_legacy_f32_e32 {{v[0-9]+}}, s[[B]], [[VA]]
 
 ; VI-SAFE: v_mov_b32_e32 [[VA:v[0-9]+]], s[[A]]
 ; VI-SAFE: v_cmp_ngt_f32_e32 vcc, s[[A]], [[VB]]
diff --git a/llvm/test/CodeGen/AMDGPU/fmul-2-combine-multi-use.ll b/llvm/test/CodeGen/AMDGPU/fmul-2-combine-multi-use.ll
index d18705f1d..af54cbf9d 100644
--- a/llvm/test/CodeGen/AMDGPU/fmul-2-combine-multi-use.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmul-2-combine-multi-use.ll
@@ -59,7 +59,7 @@ define amdgpu_kernel void @multiple_use_fadd_fmac_f32(float addrspace(1)* %out,
 ; GCN-LABEL: {{^}}multiple_use_fadd_fmad_f32:
 ; GCN-DAG:   v_add_f32_e64 [[MUL2:v[0-9]+]], |[[X:s[0-9]+]]|, |s{{[0-9]+}}|
 ; SIVI-DAG:  v_mad_f32 [[MAD:v[0-9]+]], |[[X]]|, 2.0, v{{[0-9]+}}
-; GFX10-DAG: v_fma_f32 [[MAD:v[0-9]+]], 2.0, |[[X]]|, v{{[0-9]+}}
+; GFX10-DAG: v_fma_f32 [[MAD:v[0-9]+]], |[[X]]|, 2.0, s{{[0-9]+}}
 ; GCN-DAG:   buffer_store_dword [[MUL2]]
 ; GCN-DAG:   buffer_store_dword [[MAD]]
 ; GCN:       s_endpgm
@@ -126,7 +126,7 @@ define amdgpu_kernel void @fmul_x2_xn3_f32(float addrspace(1)* %out, float %x, f
 ; GFX8_10:      v_mul_f16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 ; VI-FLUSH:     v_mad_f16 v{{[0-9]+}}, -v{{[0-9]+}}, v{{[0-9]+}}, 1.0
 ; VI-DENORM:    v_fma_f16 v{{[0-9]+}}, -v{{[0-9]+}}, v{{[0-9]+}}, 1.0
-; GFX10-DENORM: v_fmac_f16_e64 v{{[0-9]+}}, -v{{[0-9]+}}, v{{[0-9]+}}
+; GFX10-DENORM: v_fma_f16 v{{[0-9]+}}, -v{{[0-9]+}}, v{{[0-9]+}}, 1.0
 ; GFX10-FLUSH:  v_sub_f16_e32 v{{[0-9]+}}, 1.0, v{{[0-9]+}}
 define amdgpu_kernel void @multiple_fadd_use_test_f16(half addrspace(1)* %out, i16 zeroext %x.arg, i16 zeroext %y.arg, i16 zeroext %z.arg) #0 {
   %x = bitcast i16 %x.arg to half
@@ -152,7 +152,7 @@ define amdgpu_kernel void @multiple_fadd_use_test_f16(half addrspace(1)* %out, i
 ; VI-FLUSH-DAG:     v_mac_f16_e64 [[MAD:v[0-9]+]], [[X]], 2.0
 ; VI-DENORM-DAG:    v_fma_f16 [[MAD:v[0-9]+]], [[X]], 2.0, v{{[0-9]+}}
 ; GFX10-FLUSH-DAG:  v_add_f16_e32 [[MAD:v[0-9]+]], s{{[0-9]+}}, [[MUL2]]
-; GFX10-DENORM-DAG: v_fmac_f16_e64 [[MAD:v[0-9]+]], [[X]], 2.0
+; GFX10-DENORM-DAG: v_fma_f16 [[MAD:v[0-9]+]], [[X]], 2.0, s{{[0-9]+}}
 
 ; GCN-DAG: buffer_store_short [[MUL2]]
 ; GCN-DAG: buffer_store_short [[MAD]]
@@ -174,7 +174,7 @@ define amdgpu_kernel void @multiple_use_fadd_fmac_f16(half addrspace(1)* %out, i
 ; VI-FLUSH-DAG:     v_mad_f16 [[MAD:v[0-9]+]], |[[X]]|, 2.0, v{{[0-9]+}}
 ; VI-DENORM-DAG:    v_fma_f16 [[MAD:v[0-9]+]], |[[X]]|, 2.0, v{{[0-9]+}}
 ; GFX10-FLUSH-DAG:  v_add_f16_e32 [[MAD:v[0-9]+]], s{{[0-9]+}}, [[MUL2]]
-; GFX10-DENORM-DAG: v_fmac_f16_e64 [[MAD:v[0-9]+]], |[[X]]|, 2.0
+; GFX10-DENORM-DAG: v_fma_f16 [[MAD:v[0-9]+]], |[[X]]|, 2.0, s{{[0-9]+}}
 
 ; GCN-DAG: buffer_store_short [[MUL2]]
 ; GCN-DAG: buffer_store_short [[MAD]]
@@ -201,8 +201,8 @@ define amdgpu_kernel void @multiple_use_fadd_fmad_f16(half addrspace(1)* %out, i
 ; GFX10-FLUSH:  v_add_f16_e64 [[MUL2:v[0-9]+]], |[[X:s[0-9]+]]|, |{{s[0-9]+}}|
 ; GFX10-FLUSH:  v_add_f16_e32 {{v[0-9]+}}, {{s[0-9]+}}, [[MUL2]]
 ; GFX10-FLUSH:  v_add_f16_e32 {{v[0-9]+}}, {{s[0-9]+}}, [[MUL2]]
-; GFX10-DENORM: v_fmac_f16_e64 {{v[0-9]+}}, |[[X:s[0-9]+]]|, 2.0
-; GFX10-DENORM: v_fmac_f16_e64 {{v[0-9]+}}, |[[X]]|, 2.0
+; GFX10-DENORM: v_fma_f16 {{v[0-9]+}}, |[[X:s[0-9]+]]|, 2.0, s{{[0-9]+}}
+; GFX10-DENORM: v_fma_f16 {{v[0-9]+}}, |[[X]]|, 2.0, s{{[0-9]+}}
 
 define amdgpu_kernel void @multiple_use_fadd_multi_fmad_f16(half addrspace(1)* %out, i16 zeroext %x.arg, i16 zeroext %y.arg, i16 zeroext %z.arg) #0 {
   %x = bitcast i16 %x.arg to half
diff --git a/llvm/test/CodeGen/AMDGPU/fmuladd.f16.ll b/llvm/test/CodeGen/AMDGPU/fmuladd.f16.ll
index 0ab2e6710..248cbe6ab 100644
--- a/llvm/test/CodeGen/AMDGPU/fmuladd.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmuladd.f16.ll
@@ -331,7 +331,8 @@ define amdgpu_kernel void @mad_sub_f16(half addrspace(1)* noalias nocapture %out
 ; GFX10-FLUSH: v_mul_f16_e32 [[TMP:v[0-9]+]], [[REGA]], [[REGB]]
 ; GFX10-FLUSH: v_sub_f16_e32 [[RESULT:v[0-9]+]], [[REGC]], [[TMP]]
 ; GFX10-FLUSH:  global_store_short v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]]
-; GFX10-DENORM: global_store_short v{{\[[0-9]+:[0-9]+\]}}, [[REGC]]
+; GFX10-DENORM-STRICT: global_store_short v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]]
+; GFX10-DENORM-CONTRACT: global_store_short v{{\[[0-9]+:[0-9]+\]}}, [[REGC]]
 define amdgpu_kernel void @mad_sub_inv_f16(half addrspace(1)* noalias nocapture %out, half addrspace(1)* noalias nocapture readonly %ptr) #1 {
   %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0
   %tid.ext = sext i32 %tid to i64
@@ -439,7 +440,8 @@ define amdgpu_kernel void @mad_sub_fabs_inv_f16(half addrspace(1)* noalias nocap
 ; GFX10-FLUSH: v_mul_f16_e32 [[TMP:v[0-9]+]], [[REGA]], [[REGB]]
 ; GFX10-FLUSH: v_add_f16_e32 [[RESULT:v[0-9]+]], [[REGC]], [[TMP]]
 ; GFX10-FLUSH:  global_store_short v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]]
-; GFX10-DENORM: global_store_short v{{\[[0-9]+:[0-9]+\]}}, [[REGC]]
+; GFX10-DENORM-STRICT: global_store_short v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]]
+; GFX10-DENORM-CONTRACT: global_store_short v{{\[[0-9]+:[0-9]+\]}}, [[REGC]]
 define amdgpu_kernel void @neg_neg_mad_f16(half addrspace(1)* noalias nocapture %out, half addrspace(1)* noalias nocapture readonly %ptr) #1 {
   %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0
   %tid.ext = sext i32 %tid to i64
diff --git a/llvm/test/CodeGen/AMDGPU/fneg-combines.ll b/llvm/test/CodeGen/AMDGPU/fneg-combines.ll
index e60bf4476..5dda92dbd 100644
--- a/llvm/test/CodeGen/AMDGPU/fneg-combines.ll
+++ b/llvm/test/CodeGen/AMDGPU/fneg-combines.ll
@@ -157,13 +157,13 @@ define amdgpu_kernel void @v_fneg_add_fneg_fneg_f32(float addrspace(1)* %out, fl
 }
 
 ; GCN-LABEL: {{^}}v_fneg_add_store_use_fneg_x_f32:
-; GCN-SAFE: v_bfrev_b32_e32 [[SIGNBIT:v[0-9]+]], 1{{$}}
+; GCN-SAFE: s_brev_b32 [[SIGNBIT:s[0-9]+]], 1{{$}}
 ; GCN: {{buffer|flat}}_load_dword [[A:v[0-9]+]]
 ; GCN: {{buffer|flat}}_load_dword [[B:v[0-9]+]]
 
-; GCN-SAFE: v_xor_b32_e32 [[NEG_A:v[0-9]+]], [[A]], [[SIGNBIT]]
+; GCN-SAFE: v_xor_b32_e32 [[NEG_A:v[0-9]+]], [[SIGNBIT]], [[A]]
 ; GCN-SAFE: v_sub_f32_e32 [[ADD:v[0-9]+]], [[B]], [[A]]
-; GCN-SAFE: v_xor_b32_e32 [[NEG_ADD:v[0-9]+]], [[ADD]], [[SIGNBIT]]
+; GCN-SAFE: v_xor_b32_e32 [[NEG_ADD:v[0-9]+]], [[SIGNBIT]], [[ADD]]
 
 ; GCN-NSZ-DAG: v_xor_b32_e32 [[NEG_A:v[0-9]+]], 0x80000000, [[A]]
 ; GCN-NSZ-DAG: v_sub_f32_e32 [[NEG_ADD:v[0-9]+]], [[A]], [[B]]
@@ -219,8 +219,11 @@ define amdgpu_kernel void @v_fneg_add_multi_use_fneg_x_f32(float addrspace(1)* %
 ; GCN-SAFE-DAG: v_mad_f32 [[A:v[0-9]+]],
 ; GCN-SAFE-DAG: v_cmp_ngt_f32_e32 {{.*}}, [[A]]
 ; GCN-SAFE-DAG: v_cndmask_b32_e64 v{{[0-9]+}}, -[[A]]
-; GCN-NSZ-DAG: v_mac_f32_e32 [[C:v[0-9]+]],
-; GCN-NSZ-DAG: v_cmp_nlt_f32_e64 {{.*}}, -[[C]]
+; GCN-NSZ-DAG: v_rcp_f32_e32 [[A:v[0-9]+]],
+; GCN-NSZ-DAG: v_mov_b32_e32 [[B:v[0-9]+]],
+; GCN-NSZ-DAG: v_mov_b32_e32 [[C:v[0-9]+]],
+; GCN-NSZ-DAG: v_mul_f32_e32 [[D:v[0-9]+]],
+; GCN-NSZ-DAG: v_cmp_nlt_f32_e64 {{.*}}, -[[D]]
 
 define amdgpu_ps float @fneg_fadd_0(float inreg %tmp2, float inreg %tmp6, <4 x i32> %arg) local_unnamed_addr #0 {
 .entry:
@@ -1202,7 +1205,7 @@ define amdgpu_kernel void @v_fneg_fma_x_fneg_y_f32(float addrspace(1)* %out, flo
 ; GCN: {{buffer|flat}}_load_dword [[B:v[0-9]+]]
 ; GCN: {{buffer|flat}}_load_dword [[C:v[0-9]+]]
 
-; GCN-SAFE: v_fma_f32 [[FMA:v[0-9]+]], -[[A]], -[[B]], [[C]]
+; GCN-SAFE: v_fma_f32 [[FMA:v[0-9]+]], [[A]], [[B]], [[C]]
 ; GCN-SAFE: v_xor_b32_e32 v{{[[0-9]+}}, 0x80000000, [[FMA]]
 
 ; GCN-NSZ: v_fma_f32 [[FMA:v[0-9]+]], [[A]], -[[B]], -[[C]]
diff --git a/llvm/test/CodeGen/AMDGPU/fneg-fabs.ll b/llvm/test/CodeGen/AMDGPU/fneg-fabs.ll
index 0ff5d9652..a621b04a3 100644
--- a/llvm/test/CodeGen/AMDGPU/fneg-fabs.ll
+++ b/llvm/test/CodeGen/AMDGPU/fneg-fabs.ll
@@ -4,7 +4,7 @@
 
 ; FUNC-LABEL: {{^}}fneg_fabs_fadd_f32:
 ; SI-NOT: and
-; SI: v_sub_f32_e64 {{v[0-9]+}}, {{v[0-9]+}}, |{{s[0-9]+}}|
+; SI: v_sub_f32_e64 {{v[0-9]+}}, {{s[0-9]+}}, |{{v[0-9]+}}|
 define amdgpu_kernel void @fneg_fabs_fadd_f32(float addrspace(1)* %out, float %x, float %y) {
   %fabs = call float @llvm.fabs.f32(float %x)
   %fsub = fsub float -0.000000e+00, %fabs
@@ -15,7 +15,7 @@ define amdgpu_kernel void @fneg_fabs_fadd_f32(float addrspace(1)* %out, float %x
 
 ; FUNC-LABEL: {{^}}fneg_fabs_fmul_f32:
 ; SI-NOT: and
-; SI: v_mul_f32_e64 {{v[0-9]+}}, {{v[0-9]+}}, -|{{s[0-9]+}}|
+; SI: v_mul_f32_e64 {{v[0-9]+}}, {{s[0-9]+}}, -|{{v[0-9]+}}|
 ; SI-NOT: and
 define amdgpu_kernel void @fneg_fabs_fmul_f32(float addrspace(1)* %out, float %x, float %y) {
   %fabs = call float @llvm.fabs.f32(float %x)
@@ -85,8 +85,8 @@ define amdgpu_kernel void @v_fneg_fabs_f32(float addrspace(1)* %out, float addrs
 
 ; FIXME: In this case two uses of the constant should be folded
 ; SI: s_brev_b32 [[SIGNBITK:s[0-9]+]], 1{{$}}
-; SI: v_or_b32_e32 v{{[0-9]+}}, [[SIGNBITK]], v{{[0-9]+}}
-; SI: v_or_b32_e32 v{{[0-9]+}}, [[SIGNBITK]], v{{[0-9]+}}
+; SI: s_or_b32 s{{[0-9]+}}, s{{[0-9]+}}, [[SIGNBITK]]
+; SI: s_or_b32 s{{[0-9]+}}, s{{[0-9]+}}, [[SIGNBITK]]
 define amdgpu_kernel void @fneg_fabs_v2f32(<2 x float> addrspace(1)* %out, <2 x float> %in) {
   %fabs = call <2 x float> @llvm.fabs.v2f32(<2 x float> %in)
   %fsub = fsub <2 x float> <float -0.000000e+00, float -0.000000e+00>, %fabs
@@ -96,10 +96,10 @@ define amdgpu_kernel void @fneg_fabs_v2f32(<2 x float> addrspace(1)* %out, <2 x
 
 ; FUNC-LABEL: {{^}}fneg_fabs_v4f32:
 ; SI: s_brev_b32 [[SIGNBITK:s[0-9]+]], 1{{$}}
-; SI: v_or_b32_e32 v{{[0-9]+}}, [[SIGNBITK]], v{{[0-9]+}}
-; SI: v_or_b32_e32 v{{[0-9]+}}, [[SIGNBITK]], v{{[0-9]+}}
-; SI: v_or_b32_e32 v{{[0-9]+}}, [[SIGNBITK]], v{{[0-9]+}}
-; SI: v_or_b32_e32 v{{[0-9]+}}, [[SIGNBITK]], v{{[0-9]+}}
+; SI: s_or_b32 s{{[0-9]+}}, s{{[0-9]+}}, [[SIGNBITK]]
+; SI: s_or_b32 s{{[0-9]+}}, s{{[0-9]+}}, [[SIGNBITK]]
+; SI: s_or_b32 s{{[0-9]+}}, s{{[0-9]+}}, [[SIGNBITK]]
+; SI: s_or_b32 s{{[0-9]+}}, s{{[0-9]+}}, [[SIGNBITK]]
 define amdgpu_kernel void @fneg_fabs_v4f32(<4 x float> addrspace(1)* %out, <4 x float> %in) {
   %fabs = call <4 x float> @llvm.fabs.v4f32(<4 x float> %in)
   %fsub = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %fabs
diff --git a/llvm/test/CodeGen/AMDGPU/fneg-fold-legalize-dag-increase-insts.ll b/llvm/test/CodeGen/AMDGPU/fneg-fold-legalize-dag-increase-insts.ll
new file mode 100644
index 000000000..76300eb13
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/fneg-fold-legalize-dag-increase-insts.ll
@@ -0,0 +1,24 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -enable-no-signed-zeros-fp-math=true < %s | FileCheck %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -enable-no-signed-zeros-fp-math=false < %s | FileCheck %s
+
+; no-signed-zeros-fp-math should not increase the number of
+; instructions emitted.
+
+define { double, double } @testfn(double %arg, double %arg1, double %arg2) {
+; CHECK-LABEL: testfn:
+; CHECK:       ; %bb.0: ; %bb
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_add_f64 v[4:5], v[4:5], -v[0:1]
+; CHECK-NEXT:    v_add_f64 v[0:1], v[4:5], -v[2:3]
+; CHECK-NEXT:    v_add_f64 v[2:3], -v[2:3], -v[4:5]
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+bb:
+  %tmp = fsub fast double 0.000000e+00, %arg1
+  %tmp3 = fsub fast double %arg2, %arg
+  %tmp4 = fadd fast double %tmp3, %tmp
+  %tmp5 = fsub fast double %tmp, %tmp3
+  %tmp6 = insertvalue { double, double } undef, double %tmp4, 0
+  %tmp7 = insertvalue { double, double } %tmp6, double %tmp5, 1
+  ret { double, double } %tmp7
+}
diff --git a/llvm/test/CodeGen/AMDGPU/fneg.ll b/llvm/test/CodeGen/AMDGPU/fneg.ll
index 94ec61622..9a8a8c08d 100644
--- a/llvm/test/CodeGen/AMDGPU/fneg.ll
+++ b/llvm/test/CodeGen/AMDGPU/fneg.ll
@@ -1,11 +1,13 @@
-; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=GCN -check-prefix=FUNC %s
-; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=VI -check-prefix=GCN -check-prefix=FUNC %s
-; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefix=R600 -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=SI -check-prefix=GCN -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=VI -check-prefix=GCN -check-prefix=FUNC %s
+; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -enable-var-scope -check-prefix=R600 -check-prefix=FUNC %s
 
 ; FUNC-LABEL: {{^}}s_fneg_f32:
 ; R600: -PV
 
-; GCN: v_xor_b32
+; GCN: s_load_dword [[VAL:s[0-9]+]]
+; GCN: s_xor_b32 [[NEG_VAL:s[0-9]+]], [[VAL]], 0x80000000
+; GCN: v_mov_b32_e32 v{{[0-9]+}}, [[NEG_VAL]]
 define amdgpu_kernel void @s_fneg_f32(float addrspace(1)* %out, float %in) {
   %fneg = fsub float -0.000000e+00, %in
   store float %fneg, float addrspace(1)* %out
@@ -16,8 +18,9 @@ define amdgpu_kernel void @s_fneg_f32(float addrspace(1)* %out, float %in) {
 ; R600: -PV
 ; R600: -PV
 
-; GCN: v_xor_b32
-; GCN: v_xor_b32
+; GCN: s_brev_b32 [[SIGNBIT:s[0-9]+]], 1
+; GCN: s_xor_b32
+; GCN: s_xor_b32
 define amdgpu_kernel void @s_fneg_v2f32(<2 x float> addrspace(1)* nocapture %out, <2 x float> %in) {
   %fneg = fsub <2 x float> <float -0.000000e+00, float -0.000000e+00>, %in
   store <2 x float> %fneg, <2 x float> addrspace(1)* %out
@@ -30,10 +33,10 @@ define amdgpu_kernel void @s_fneg_v2f32(<2 x float> addrspace(1)* nocapture %out
 ; R600: -PV
 ; R600: -PV
 
-; GCN: v_xor_b32
-; GCN: v_xor_b32
-; GCN: v_xor_b32
-; GCN: v_xor_b32
+; GCN: s_xor_b32
+; GCN: s_xor_b32
+; GCN: s_xor_b32
+; GCN: s_xor_b32
 define amdgpu_kernel void @s_fneg_v4f32(<4 x float> addrspace(1)* nocapture %out, <4 x float> %in) {
   %fneg = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %in
   store <4 x float> %fneg, <4 x float> addrspace(1)* %out
@@ -60,9 +63,9 @@ define amdgpu_kernel void @fsub0_f32(float addrspace(1)* %out, i32 %in) {
 ; SI: s_load_dword [[NEG_VALUE:s[0-9]+]], s[{{[0-9]+:[0-9]+}}], 0xb
 ; VI: s_load_dword [[NEG_VALUE:s[0-9]+]], s[{{[0-9]+:[0-9]+}}], 0x2c
 
-; GCN: v_bfrev_b32_e32 [[SIGNBIT:v[0-9]+]], 1{{$}}
-; GCN: v_xor_b32_e32 [[RES:v[0-9]+]], [[NEG_VALUE]], [[SIGNBIT]]
-; GCN: buffer_store_dword [[RES]]
+; GCN: s_xor_b32 [[RES:s[0-9]+]], [[NEG_VALUE]], 0x80000000
+; GCN: v_mov_b32_e32 [[V_RES:v[0-9]+]], [[RES]]
+; GCN: buffer_store_dword [[V_RES]]
 
 ; R600-NOT: XOR
 ; R600: -PV.W
diff --git a/llvm/test/CodeGen/AMDGPU/fpow.ll b/llvm/test/CodeGen/AMDGPU/fpow.ll
new file mode 100644
index 000000000..860ede271
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/fpow.ll
@@ -0,0 +1,562 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -march=amdgcn -mcpu=tahiti < %s | FileCheck -check-prefix=GFX6 %s
+; RUN: llc -march=amdgcn -mcpu=fiji < %s | FileCheck -check-prefix=GFX8 %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefix=GFX9 %s
+
+define float @v_pow_f32(float %x, float %y) {
+; GFX6-LABEL: v_pow_f32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_log_f32_e32 v0, v0
+; GFX6-NEXT:    v_mul_legacy_f32_e32 v0, v1, v0
+; GFX6-NEXT:    v_exp_f32_e32 v0, v0
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_pow_f32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_log_f32_e32 v0, v0
+; GFX8-NEXT:    v_mul_legacy_f32_e32 v0, v1, v0
+; GFX8-NEXT:    v_exp_f32_e32 v0, v0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_pow_f32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_log_f32_e32 v0, v0
+; GFX9-NEXT:    v_mul_legacy_f32_e32 v0, v1, v0
+; GFX9-NEXT:    v_exp_f32_e32 v0, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %pow = call float @llvm.pow.f32(float %x, float %y)
+  ret float %pow
+}
+
+define <2 x float> @v_pow_v2f32(<2 x float> %x, <2 x float> %y) {
+; GFX6-LABEL: v_pow_v2f32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_log_f32_e32 v0, v0
+; GFX6-NEXT:    v_log_f32_e32 v1, v1
+; GFX6-NEXT:    v_mul_legacy_f32_e32 v0, v2, v0
+; GFX6-NEXT:    v_mul_legacy_f32_e32 v1, v3, v1
+; GFX6-NEXT:    v_exp_f32_e32 v0, v0
+; GFX6-NEXT:    v_exp_f32_e32 v1, v1
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_pow_v2f32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_log_f32_e32 v0, v0
+; GFX8-NEXT:    v_log_f32_e32 v1, v1
+; GFX8-NEXT:    v_mul_legacy_f32_e32 v0, v2, v0
+; GFX8-NEXT:    v_mul_legacy_f32_e32 v1, v3, v1
+; GFX8-NEXT:    v_exp_f32_e32 v0, v0
+; GFX8-NEXT:    v_exp_f32_e32 v1, v1
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_pow_v2f32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_log_f32_e32 v0, v0
+; GFX9-NEXT:    v_log_f32_e32 v1, v1
+; GFX9-NEXT:    v_mul_legacy_f32_e32 v0, v2, v0
+; GFX9-NEXT:    v_mul_legacy_f32_e32 v1, v3, v1
+; GFX9-NEXT:    v_exp_f32_e32 v0, v0
+; GFX9-NEXT:    v_exp_f32_e32 v1, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %pow = call <2 x float> @llvm.pow.v2f32(<2 x float> %x, <2 x float> %y)
+  ret <2 x float> %pow
+}
+
+define half @v_pow_f16(half %x, half %y) {
+; GFX6-LABEL: v_pow_f16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX6-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GFX6-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX6-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GFX6-NEXT:    v_log_f32_e32 v0, v0
+; GFX6-NEXT:    v_mul_legacy_f32_e32 v0, v1, v0
+; GFX6-NEXT:    v_exp_f32_e32 v0, v0
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_pow_f16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX8-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GFX8-NEXT:    v_log_f32_e32 v0, v0
+; GFX8-NEXT:    v_mul_legacy_f32_e32 v0, v1, v0
+; GFX8-NEXT:    v_exp_f32_e32 v0, v0
+; GFX8-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_pow_f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX9-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GFX9-NEXT:    v_log_f32_e32 v0, v0
+; GFX9-NEXT:    v_mul_legacy_f32_e32 v0, v1, v0
+; GFX9-NEXT:    v_exp_f32_e32 v0, v0
+; GFX9-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %pow = call half @llvm.pow.f16(half %x, half %y)
+  ret half %pow
+}
+
+define <2 x half> @v_pow_v2f16(<2 x half> %x, <2 x half> %y) {
+; GFX6-LABEL: v_pow_v2f16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GFX6-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX6-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GFX6-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GFX6-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GFX6-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX6-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; GFX6-NEXT:    v_cvt_f32_f16_e32 v2, v2
+; GFX6-NEXT:    v_log_f32_e32 v1, v1
+; GFX6-NEXT:    v_log_f32_e32 v0, v0
+; GFX6-NEXT:    v_mul_legacy_f32_e32 v1, v3, v1
+; GFX6-NEXT:    v_mul_legacy_f32_e32 v0, v2, v0
+; GFX6-NEXT:    v_exp_f32_e32 v0, v0
+; GFX6-NEXT:    v_exp_f32_e32 v1, v1
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_pow_v2f16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cvt_f32_f16_sdwa v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX8-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX8-NEXT:    v_cvt_f32_f16_sdwa v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX8-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GFX8-NEXT:    v_log_f32_e32 v2, v2
+; GFX8-NEXT:    v_log_f32_e32 v0, v0
+; GFX8-NEXT:    v_mul_legacy_f32_e32 v2, v3, v2
+; GFX8-NEXT:    v_mul_legacy_f32_e32 v0, v1, v0
+; GFX8-NEXT:    v_exp_f32_e32 v1, v2
+; GFX8-NEXT:    v_exp_f32_e32 v0, v0
+; GFX8-NEXT:    v_cvt_f16_f32_sdwa v1, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; GFX8-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_pow_v2f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cvt_f32_f16_e32 v2, v0
+; GFX9-NEXT:    v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-NEXT:    v_cvt_f32_f16_e32 v3, v1
+; GFX9-NEXT:    v_cvt_f32_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-NEXT:    v_log_f32_e32 v2, v2
+; GFX9-NEXT:    v_log_f32_e32 v0, v0
+; GFX9-NEXT:    v_mul_legacy_f32_e32 v2, v3, v2
+; GFX9-NEXT:    v_mul_legacy_f32_e32 v0, v1, v0
+; GFX9-NEXT:    v_exp_f32_e32 v1, v2
+; GFX9-NEXT:    v_exp_f32_e32 v0, v0
+; GFX9-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GFX9-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX9-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GFX9-NEXT:    v_lshl_or_b32 v0, v0, 16, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %pow = call <2 x half> @llvm.pow.v2f16(<2 x half> %x, <2 x half> %y)
+  ret <2 x half> %pow
+}
+
+define <2 x half> @v_pow_v2f16_fneg_lhs(<2 x half> %x, <2 x half> %y) {
+; GFX6-LABEL: v_pow_v2f16_fneg_lhs:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GFX6-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX6-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    v_xor_b32_e32 v0, 0x80008000, v0
+; GFX6-NEXT:    v_cvt_f16_f32_e32 v1, v2
+; GFX6-NEXT:    v_cvt_f32_f16_e32 v2, v0
+; GFX6-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX6-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX6-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GFX6-NEXT:    v_log_f32_e32 v2, v2
+; GFX6-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; GFX6-NEXT:    v_log_f32_e32 v4, v0
+; GFX6-NEXT:    v_mul_legacy_f32_e32 v0, v1, v2
+; GFX6-NEXT:    v_exp_f32_e32 v0, v0
+; GFX6-NEXT:    v_mul_legacy_f32_e32 v1, v3, v4
+; GFX6-NEXT:    v_exp_f32_e32 v1, v1
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_pow_v2f16_fneg_lhs:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cvt_f32_f16_sdwa v2, -v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX8-NEXT:    v_cvt_f32_f16_e64 v0, -v0
+; GFX8-NEXT:    v_cvt_f32_f16_sdwa v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX8-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GFX8-NEXT:    v_log_f32_e32 v2, v2
+; GFX8-NEXT:    v_log_f32_e32 v0, v0
+; GFX8-NEXT:    v_mul_legacy_f32_e32 v2, v3, v2
+; GFX8-NEXT:    v_mul_legacy_f32_e32 v0, v1, v0
+; GFX8-NEXT:    v_exp_f32_e32 v1, v2
+; GFX8-NEXT:    v_exp_f32_e32 v0, v0
+; GFX8-NEXT:    v_cvt_f16_f32_sdwa v1, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; GFX8-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_pow_v2f16_fneg_lhs:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cvt_f32_f16_e64 v2, -v0
+; GFX9-NEXT:    v_cvt_f32_f16_sdwa v0, -v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-NEXT:    v_cvt_f32_f16_e32 v3, v1
+; GFX9-NEXT:    v_cvt_f32_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-NEXT:    v_log_f32_e32 v2, v2
+; GFX9-NEXT:    v_log_f32_e32 v0, v0
+; GFX9-NEXT:    v_mul_legacy_f32_e32 v2, v3, v2
+; GFX9-NEXT:    v_mul_legacy_f32_e32 v0, v1, v0
+; GFX9-NEXT:    v_exp_f32_e32 v1, v2
+; GFX9-NEXT:    v_exp_f32_e32 v0, v0
+; GFX9-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GFX9-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX9-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GFX9-NEXT:    v_lshl_or_b32 v0, v0, 16, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %x.fneg = fneg <2 x half> %x
+  %pow = call <2 x half> @llvm.pow.v2f16(<2 x half> %x.fneg, <2 x half> %y)
+  ret <2 x half> %pow
+}
+
+define <2 x half> @v_pow_v2f16_fneg_rhs(<2 x half> %x, <2 x half> %y) {
+; GFX6-LABEL: v_pow_v2f16_fneg_rhs:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GFX6-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GFX6-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX6-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GFX6-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GFX6-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX6-NEXT:    v_or_b32_e32 v2, v2, v3
+; GFX6-NEXT:    v_xor_b32_e32 v2, 0x80008000, v2
+; GFX6-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
+; GFX6-NEXT:    v_cvt_f32_f16_e32 v2, v2
+; GFX6-NEXT:    v_log_f32_e32 v0, v0
+; GFX6-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; GFX6-NEXT:    v_log_f32_e32 v1, v1
+; GFX6-NEXT:    v_mul_legacy_f32_e32 v0, v2, v0
+; GFX6-NEXT:    v_exp_f32_e32 v0, v0
+; GFX6-NEXT:    v_mul_legacy_f32_e32 v1, v3, v1
+; GFX6-NEXT:    v_exp_f32_e32 v1, v1
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_pow_v2f16_fneg_rhs:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cvt_f32_f16_sdwa v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX8-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX8-NEXT:    v_cvt_f32_f16_sdwa v3, -v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX8-NEXT:    v_cvt_f32_f16_e64 v1, -v1
+; GFX8-NEXT:    v_log_f32_e32 v2, v2
+; GFX8-NEXT:    v_log_f32_e32 v0, v0
+; GFX8-NEXT:    v_mul_legacy_f32_e32 v2, v3, v2
+; GFX8-NEXT:    v_mul_legacy_f32_e32 v0, v1, v0
+; GFX8-NEXT:    v_exp_f32_e32 v1, v2
+; GFX8-NEXT:    v_exp_f32_e32 v0, v0
+; GFX8-NEXT:    v_cvt_f16_f32_sdwa v1, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; GFX8-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_pow_v2f16_fneg_rhs:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cvt_f32_f16_e32 v2, v0
+; GFX9-NEXT:    v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-NEXT:    v_cvt_f32_f16_e64 v3, -v1
+; GFX9-NEXT:    v_cvt_f32_f16_sdwa v1, -v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-NEXT:    v_log_f32_e32 v2, v2
+; GFX9-NEXT:    v_log_f32_e32 v0, v0
+; GFX9-NEXT:    v_mul_legacy_f32_e32 v2, v3, v2
+; GFX9-NEXT:    v_mul_legacy_f32_e32 v0, v1, v0
+; GFX9-NEXT:    v_exp_f32_e32 v1, v2
+; GFX9-NEXT:    v_exp_f32_e32 v0, v0
+; GFX9-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GFX9-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX9-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GFX9-NEXT:    v_lshl_or_b32 v0, v0, 16, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %y.fneg = fneg <2 x half> %y
+  %pow = call <2 x half> @llvm.pow.v2f16(<2 x half> %x, <2 x half> %y.fneg)
+  ret <2 x half> %pow
+}
+
+define <2 x half> @v_pow_v2f16_fneg_lhs_rhs(<2 x half> %x, <2 x half> %y) {
+; GFX6-LABEL: v_pow_v2f16_fneg_lhs_rhs:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GFX6-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX6-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GFX6-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    s_mov_b32 s4, 0x80008000
+; GFX6-NEXT:    v_xor_b32_e32 v0, s4, v0
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
+; GFX6-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GFX6-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-NEXT:    v_or_b32_e32 v2, v2, v3
+; GFX6-NEXT:    v_xor_b32_e32 v2, s4, v2
+; GFX6-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
+; GFX6-NEXT:    v_cvt_f32_f16_e32 v2, v2
+; GFX6-NEXT:    v_log_f32_e32 v0, v0
+; GFX6-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; GFX6-NEXT:    v_log_f32_e32 v1, v1
+; GFX6-NEXT:    v_mul_legacy_f32_e32 v0, v2, v0
+; GFX6-NEXT:    v_exp_f32_e32 v0, v0
+; GFX6-NEXT:    v_mul_legacy_f32_e32 v1, v3, v1
+; GFX6-NEXT:    v_exp_f32_e32 v1, v1
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_pow_v2f16_fneg_lhs_rhs:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cvt_f32_f16_sdwa v2, -v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX8-NEXT:    v_cvt_f32_f16_e64 v0, -v0
+; GFX8-NEXT:    v_cvt_f32_f16_sdwa v3, -v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX8-NEXT:    v_cvt_f32_f16_e64 v1, -v1
+; GFX8-NEXT:    v_log_f32_e32 v2, v2
+; GFX8-NEXT:    v_log_f32_e32 v0, v0
+; GFX8-NEXT:    v_mul_legacy_f32_e32 v2, v3, v2
+; GFX8-NEXT:    v_mul_legacy_f32_e32 v0, v1, v0
+; GFX8-NEXT:    v_exp_f32_e32 v1, v2
+; GFX8-NEXT:    v_exp_f32_e32 v0, v0
+; GFX8-NEXT:    v_cvt_f16_f32_sdwa v1, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; GFX8-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_pow_v2f16_fneg_lhs_rhs:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cvt_f32_f16_e64 v2, -v0
+; GFX9-NEXT:    v_cvt_f32_f16_sdwa v0, -v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-NEXT:    v_cvt_f32_f16_e64 v3, -v1
+; GFX9-NEXT:    v_cvt_f32_f16_sdwa v1, -v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-NEXT:    v_log_f32_e32 v2, v2
+; GFX9-NEXT:    v_log_f32_e32 v0, v0
+; GFX9-NEXT:    v_mul_legacy_f32_e32 v2, v3, v2
+; GFX9-NEXT:    v_mul_legacy_f32_e32 v0, v1, v0
+; GFX9-NEXT:    v_exp_f32_e32 v1, v2
+; GFX9-NEXT:    v_exp_f32_e32 v0, v0
+; GFX9-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GFX9-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX9-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GFX9-NEXT:    v_lshl_or_b32 v0, v0, 16, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %x.fneg = fneg <2 x half> %x
+  %y.fneg = fneg <2 x half> %y
+  %pow = call <2 x half> @llvm.pow.v2f16(<2 x half> %x.fneg, <2 x half> %y.fneg)
+  ret <2 x half> %pow
+}
+
+; FIXME
+; define double @v_pow_f64(double %x, double %y) {
+;   %pow = call double @llvm.pow.f64(double %x, double %y)
+;   ret double %pow
+; }
+
+define float @v_pow_f32_fabs_lhs(float %x, float %y) {
+; GFX6-LABEL: v_pow_f32_fabs_lhs:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_and_b32_e32 v0, 0x7fffffff, v0
+; GFX6-NEXT:    v_log_f32_e32 v0, v0
+; GFX6-NEXT:    v_mul_legacy_f32_e32 v0, v1, v0
+; GFX6-NEXT:    v_exp_f32_e32 v0, v0
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_pow_f32_fabs_lhs:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_and_b32_e32 v0, 0x7fffffff, v0
+; GFX8-NEXT:    v_log_f32_e32 v0, v0
+; GFX8-NEXT:    v_mul_legacy_f32_e32 v0, v1, v0
+; GFX8-NEXT:    v_exp_f32_e32 v0, v0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_pow_f32_fabs_lhs:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v0, 0x7fffffff, v0
+; GFX9-NEXT:    v_log_f32_e32 v0, v0
+; GFX9-NEXT:    v_mul_legacy_f32_e32 v0, v1, v0
+; GFX9-NEXT:    v_exp_f32_e32 v0, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %fabs.x = call float @llvm.fabs.f32(float %x)
+  %pow = call float @llvm.pow.f32(float %fabs.x, float %y)
+  ret float %pow
+}
+
+define float @v_pow_f32_fabs_rhs(float %x, float %y) {
+; GFX6-LABEL: v_pow_f32_fabs_rhs:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_log_f32_e32 v0, v0
+; GFX6-NEXT:    v_and_b32_e32 v1, 0x7fffffff, v1
+; GFX6-NEXT:    v_mul_legacy_f32_e32 v0, v1, v0
+; GFX6-NEXT:    v_exp_f32_e32 v0, v0
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_pow_f32_fabs_rhs:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_log_f32_e32 v0, v0
+; GFX8-NEXT:    v_and_b32_e32 v1, 0x7fffffff, v1
+; GFX8-NEXT:    v_mul_legacy_f32_e32 v0, v1, v0
+; GFX8-NEXT:    v_exp_f32_e32 v0, v0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_pow_f32_fabs_rhs:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_log_f32_e32 v0, v0
+; GFX9-NEXT:    v_and_b32_e32 v1, 0x7fffffff, v1
+; GFX9-NEXT:    v_mul_legacy_f32_e32 v0, v1, v0
+; GFX9-NEXT:    v_exp_f32_e32 v0, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %fabs.y = call float @llvm.fabs.f32(float %y)
+  %pow = call float @llvm.pow.f32(float %x, float %fabs.y)
+  ret float %pow
+}
+
+define float @v_pow_f32_fabs_lhs_rhs(float %x, float %y) {
+; GFX6-LABEL: v_pow_f32_fabs_lhs_rhs:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    s_brev_b32 s4, -2
+; GFX6-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX6-NEXT:    v_log_f32_e32 v0, v0
+; GFX6-NEXT:    v_and_b32_e32 v1, s4, v1
+; GFX6-NEXT:    v_mul_legacy_f32_e32 v0, v1, v0
+; GFX6-NEXT:    v_exp_f32_e32 v0, v0
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_pow_f32_fabs_lhs_rhs:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    s_brev_b32 s4, -2
+; GFX8-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX8-NEXT:    v_log_f32_e32 v0, v0
+; GFX8-NEXT:    v_and_b32_e32 v1, s4, v1
+; GFX8-NEXT:    v_mul_legacy_f32_e32 v0, v1, v0
+; GFX8-NEXT:    v_exp_f32_e32 v0, v0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_pow_f32_fabs_lhs_rhs:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_brev_b32 s4, -2
+; GFX9-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX9-NEXT:    v_log_f32_e32 v0, v0
+; GFX9-NEXT:    v_and_b32_e32 v1, s4, v1
+; GFX9-NEXT:    v_mul_legacy_f32_e32 v0, v1, v0
+; GFX9-NEXT:    v_exp_f32_e32 v0, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %fabs.x = call float @llvm.fabs.f32(float %x)
+  %fabs.y = call float @llvm.fabs.f32(float %y)
+  %pow = call float @llvm.pow.f32(float %fabs.x, float %fabs.y)
+  ret float %pow
+}
+
+define amdgpu_ps float @v_pow_f32_sgpr_vgpr(float inreg %x, float %y) {
+; GFX6-LABEL: v_pow_f32_sgpr_vgpr:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    v_log_f32_e32 v1, s0
+; GFX6-NEXT:    v_mul_legacy_f32_e32 v0, v0, v1
+; GFX6-NEXT:    v_exp_f32_e32 v0, v0
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: v_pow_f32_sgpr_vgpr:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    v_log_f32_e32 v1, s0
+; GFX8-NEXT:    v_mul_legacy_f32_e32 v0, v0, v1
+; GFX8-NEXT:    v_exp_f32_e32 v0, v0
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: v_pow_f32_sgpr_vgpr:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    v_log_f32_e32 v1, s0
+; GFX9-NEXT:    v_mul_legacy_f32_e32 v0, v0, v1
+; GFX9-NEXT:    v_exp_f32_e32 v0, v0
+; GFX9-NEXT:    ; return to shader part epilog
+  %pow = call float @llvm.pow.f32(float %x, float %y)
+  ret float %pow
+}
+
+define amdgpu_ps float @v_pow_f32_vgpr_sgpr(float %x, float inreg %y) {
+; GFX6-LABEL: v_pow_f32_vgpr_sgpr:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    v_log_f32_e32 v0, v0
+; GFX6-NEXT:    v_mul_legacy_f32_e32 v0, s0, v0
+; GFX6-NEXT:    v_exp_f32_e32 v0, v0
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: v_pow_f32_vgpr_sgpr:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    v_log_f32_e32 v0, v0
+; GFX8-NEXT:    v_mul_legacy_f32_e32 v0, s0, v0
+; GFX8-NEXT:    v_exp_f32_e32 v0, v0
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: v_pow_f32_vgpr_sgpr:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    v_log_f32_e32 v0, v0
+; GFX9-NEXT:    v_mul_legacy_f32_e32 v0, s0, v0
+; GFX9-NEXT:    v_exp_f32_e32 v0, v0
+; GFX9-NEXT:    ; return to shader part epilog
+  %pow = call float @llvm.pow.f32(float %x, float %y)
+  ret float %pow
+}
+
+define amdgpu_ps float @v_pow_f32_sgpr_sgpr(float inreg %x, float inreg %y) {
+; GFX6-LABEL: v_pow_f32_sgpr_sgpr:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    v_log_f32_e32 v0, s0
+; GFX6-NEXT:    v_mul_legacy_f32_e32 v0, s1, v0
+; GFX6-NEXT:    v_exp_f32_e32 v0, v0
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: v_pow_f32_sgpr_sgpr:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    v_log_f32_e32 v0, s0
+; GFX8-NEXT:    v_mul_legacy_f32_e32 v0, s1, v0
+; GFX8-NEXT:    v_exp_f32_e32 v0, v0
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: v_pow_f32_sgpr_sgpr:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    v_log_f32_e32 v0, s0
+; GFX9-NEXT:    v_mul_legacy_f32_e32 v0, s1, v0
+; GFX9-NEXT:    v_exp_f32_e32 v0, v0
+; GFX9-NEXT:    ; return to shader part epilog
+  %pow = call float @llvm.pow.f32(float %x, float %y)
+  ret float %pow
+}
+
+declare half @llvm.pow.f16(half, half)
+declare float @llvm.pow.f32(float, float)
+declare double @llvm.pow.f64(double, double)
+
+declare half @llvm.fabs.f16(half)
+declare float @llvm.fabs.f32(float)
+
+declare <2 x half> @llvm.pow.v2f16(<2 x half>, <2 x half>)
+declare <2 x float> @llvm.pow.v2f32(<2 x float>, <2 x float>)
diff --git a/llvm/test/CodeGen/AMDGPU/fptrunc.ll b/llvm/test/CodeGen/AMDGPU/fptrunc.ll
index d9c5b7e6f..80f0a373f 100644
--- a/llvm/test/CodeGen/AMDGPU/fptrunc.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptrunc.ll
@@ -30,6 +30,16 @@ define amdgpu_kernel void @fptrunc_v2f64_to_v2f32(<2 x float> addrspace(1)* %out
   ret void
 }
 
+; FUNC-LABEL: {{^}}fptrunc_v3f64_to_v3f32:
+; GCN: v_cvt_f32_f64_e32
+; GCN: v_cvt_f32_f64_e32
+; GCN: v_cvt_f32_f64_e32
+define amdgpu_kernel void @fptrunc_v3f64_to_v3f32(<3 x float> addrspace(1)* %out, <3 x double> %in) {
+  %result = fptrunc <3 x double> %in to <3 x float>
+  store <3 x float> %result, <3 x float> addrspace(1)* %out
+  ret void
+}
+
 ; FUNC-LABEL: {{^}}fptrunc_v4f64_to_v4f32:
 ; GCN: v_cvt_f32_f64_e32
 ; GCN: v_cvt_f32_f64_e32
diff --git a/llvm/test/CodeGen/AMDGPU/frame-index-elimination.ll b/llvm/test/CodeGen/AMDGPU/frame-index-elimination.ll
index d49c34339..a92838445 100644
--- a/llvm/test/CodeGen/AMDGPU/frame-index-elimination.ll
+++ b/llvm/test/CodeGen/AMDGPU/frame-index-elimination.ll
@@ -7,7 +7,7 @@
 ; Materialize into a mov. Make sure there isn't an unnecessary copy.
 ; GCN-LABEL: {{^}}func_mov_fi_i32:
 ; GCN: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN: s_sub_u32 [[SUB:s[0-9]+]], s32, s33
+; GCN: s_sub_u32 [[SUB:s[0-9]+|vcc_lo|vcc_hi]], s32, s33
 
 ; CI-NEXT: v_lshr_b32_e64 v0, [[SUB]], 6
 ; GFX9-NEXT: v_lshrrev_b32_e64 v0, 6, [[SUB]]
@@ -24,22 +24,20 @@ define void @func_mov_fi_i32() #0 {
 ; GCN-LABEL: {{^}}func_mov_fi_i32_offset:
 ; GCN: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 
-; CI: s_sub_u32 [[SUB:s[0-9]+]], s32, s33
-; CI-NEXT: v_lshr_b32_e64 v0, [[SUB]], 6
-
-; CI: s_sub_u32 [[SUB:s[0-9]+]], s32, s33
-; CI-NEXT: v_lshr_b32_e64 [[SCALED:v[0-9]+]], [[SUB]], 6
-; CI-NEXT: v_add_i32_e64 v1, s{{\[[0-9]+:[0-9]+\]}}, 4, [[SCALED]]
+; CI: s_sub_u32 [[SUB0:s[0-9]+|vcc_lo|vcc_hi]], s32, s33
+; CI-NEXT: s_sub_u32 [[SUB1:s[0-9]+|vcc_lo|vcc_hi]], s32, s33
+; CI-DAG: v_lshr_b32_e64 v0, [[SUB0]], 6
+; CI-DAG: v_lshr_b32_e64 [[SCALED:v[0-9]+]], [[SUB1]], 6
 ; CI-NOT: v_mov
 ; CI: ds_write_b32 v0, v0
-; CI-NEXT: ds_write_b32 v0, v1
+; CI-NEXT: v_add_i32_e64 v0, s{{\[[0-9]+:[0-9]+\]}}, 4, [[SCALED]]
+; CI-NEXT: ds_write_b32 v0, v0
 
-; GFX9: s_sub_u32 [[SUB:s[0-9]+]], s32, s33
-; GFX9-NEXT: v_lshrrev_b32_e64 v0, 6, [[SUB]]
+; GFX9: s_sub_u32 [[SUB0:s[0-9]+|vcc_lo|vcc_hi]], s32, s33
+; GFX9-NEXT: s_sub_u32 [[SUB1:s[0-9]+|vcc_lo|vcc_hi]], s32, s33
+; GFX9-NEXT: v_lshrrev_b32_e64 v0, 6, [[SUB0]]
+; GFX9-NEXT: v_lshrrev_b32_e64 [[SCALED:v[0-9]+]], 6, [[SUB1]]
 ; GFX9-DAG: ds_write_b32 v0, v0
-
-; GFX9-DAG: s_sub_u32 [[SUB:s[0-9]+]], s32, s33
-; GFX9-NEXT: v_lshrrev_b32_e64 [[SCALED:v[0-9]+]], 6, [[SUB]]
 ; GFX9-NEXT: v_add_u32_e32 v0, 4, [[SCALED]]
 ; GFX9-NEXT: ds_write_b32 v0, v0
 define void @func_mov_fi_i32_offset() #0 {
@@ -55,7 +53,7 @@ define void @func_mov_fi_i32_offset() #0 {
 
 ; GCN-LABEL: {{^}}func_add_constant_to_fi_i32:
 ; GCN: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN: s_sub_u32 [[SUB:s[0-9]+]], s32, s33
+; GCN: s_sub_u32 [[SUB:s[0-9]+|vcc_lo|vcc_hi]], s32, s33
 
 ; CI-NEXT: v_lshr_b32_e64 [[SCALED:v[0-9]+]], [[SUB]], 6
 ; CI-NEXT: v_add_i32_e32 v0, vcc, 4, [[SCALED]]
@@ -77,7 +75,7 @@ define void @func_add_constant_to_fi_i32() #0 {
 ; into.
 
 ; GCN-LABEL: {{^}}func_other_fi_user_i32:
-; GCN: s_sub_u32 [[SUB:s[0-9]+]], s32, s33
+; GCN: s_sub_u32 [[SUB:s[0-9]+|vcc_lo|vcc_hi]], s32, s33
 
 ; CI-NEXT: v_lshr_b32_e64 v0, [[SUB]], 6
 
@@ -112,7 +110,7 @@ define void @func_load_private_arg_i32_ptr(i32 addrspace(5)* %ptr) #0 {
 
 ; GCN-LABEL: {{^}}void_func_byval_struct_i8_i32_ptr:
 ; GCN: s_waitcnt
-; GCN-NEXT: s_sub_u32 [[SUB_OFFSET:s[0-9]+]], s32, s33
+; GCN-NEXT: s_sub_u32 [[SUB_OFFSET:s[0-9]+|vcc_lo|vcc_hi]], s32, s33
 
 ; CI-NEXT: v_lshr_b32_e64 [[SHIFT:v[0-9]+]], [[SUB_OFFSET]], 6
 ; CI-NEXT: v_or_b32_e32 v0, 4, [[SHIFT]]
@@ -177,14 +175,14 @@ ret:
 
 ; Added offset can't be used with VOP3 add
 ; GCN-LABEL: {{^}}func_other_fi_user_non_inline_imm_offset_i32:
-; GCN: s_sub_u32 [[SUB:s[0-9]+]], s32, s33
-; GCN-DAG: s_movk_i32 [[K:s[0-9]+]], 0x200
+; GCN: s_sub_u32 [[SUB:s[0-9]+|vcc_lo|vcc_hi]], s32, s33
+; CI-DAG: s_movk_i32 [[K:s[0-9]+|vcc_lo|vcc_hi]], 0x200
 
 ; CI-DAG: v_lshr_b32_e64 [[SCALED:v[0-9]+]], [[SUB]], 6
-; CI: v_add_i32_e64 [[VZ:v[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, [[K]], [[SCALED]]
+; CI: v_add_i32_e32 [[VZ:v[0-9]+]], vcc, [[K]], [[SCALED]]
 
 ; GFX9-DAG: v_lshrrev_b32_e64 [[SCALED:v[0-9]+]], 6, [[SUB]]
-; GFX9: v_add_u32_e32 [[VZ:v[0-9]+]], [[K]], [[SCALED]]
+; GFX9: v_add_u32_e32 [[VZ:v[0-9]+]], 0x200, [[SCALED]]
 
 ; GCN: v_mul_u32_u24_e32 [[VZ]], 9, [[VZ]]
 ; GCN: ds_write_b32 v0, [[VZ]]
@@ -202,13 +200,13 @@ define void @func_other_fi_user_non_inline_imm_offset_i32() #0 {
 
 ; GCN-LABEL: {{^}}func_other_fi_user_non_inline_imm_offset_i32_vcc_live:
 ; GCN: s_sub_u32 [[DIFF:s[0-9]+]], s32, s33
-; GCN-DAG: s_movk_i32 [[OFFSET:s[0-9]+]], 0x200
+; CI-DAG: s_movk_i32 [[OFFSET:s[0-9]+]], 0x200
 
 ; CI-DAG: v_lshr_b32_e64 [[SCALED:v[0-9]+]], [[DIFF]], 6
 ; CI: v_add_i32_e64 [[VZ:v[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, [[OFFSET]], [[SCALED]]
 
 ; GFX9-DAG: v_lshrrev_b32_e64 [[SCALED:v[0-9]+]], 6, [[DIFF]]
-; GFX9: v_add_u32_e32 [[VZ:v[0-9]+]], [[OFFSET]], [[SCALED]]
+; GFX9: v_add_u32_e32 [[VZ:v[0-9]+]], 0x200, [[SCALED]]
 
 ; GCN: v_mul_u32_u24_e32 [[VZ]], 9, [[VZ]]
 ; GCN: ds_write_b32 v0, [[VZ]]
@@ -258,7 +256,7 @@ bb5:
 ; GCN-LABEL: {{^}}alloca_ptr_nonentry_block:
 ; GCN: s_and_saveexec_b64
 ; GCN: buffer_load_dword v{{[0-9]+}}, off, s[0:3], s32 offset:4
-; GCN: s_sub_u32 [[SUB_OFFSET:s[0-9]+]], s32, s33
+; GCN: s_sub_u32 [[SUB_OFFSET:s[0-9]+|vcc_lo|vcc_hi]], s32, s33
 
 ; CI: v_lshr_b32_e64 [[SHIFT:v[0-9]+]], [[SUB_OFFSET]], 6
 ; CI-NEXT: v_or_b32_e32 [[PTR:v[0-9]+]], 4, [[SHIFT]]
diff --git a/llvm/test/CodeGen/AMDGPU/fsqrt.ll b/llvm/test/CodeGen/AMDGPU/fsqrt.ll
index 6bd9a0db1..ba5f79d9b 100644
--- a/llvm/test/CodeGen/AMDGPU/fsqrt.ll
+++ b/llvm/test/CodeGen/AMDGPU/fsqrt.ll
@@ -27,8 +27,8 @@ define amdgpu_kernel void @v_unsafe_fsqrt_f32(float addrspace(1)* %out, float ad
 ; FUNC-LABEL: {{^}}s_sqrt_f32:
 ; GCN: v_sqrt_f32_e32
 
-; R600: RECIPSQRT_CLAMPED * T{{[0-9]\.[XYZW]}}, KC0[2].Z
-; R600: MUL NON-IEEE T{{[0-9]\.[XYZW]}}, KC0[2].Z, PS
+; R600: RECIPSQRT_IEEE * T{{[0-9]\.[XYZW]}}, KC0[2].Z
+; R600: RECIP_IEEE * T{{[0-9]\.[XYZW]}}, PS
 define amdgpu_kernel void @s_sqrt_f32(float addrspace(1)* %out, float %in) #1 {
 entry:
   %fdiv = call float @llvm.sqrt.f32(float %in)
@@ -40,10 +40,10 @@ entry:
 ; GCN: v_sqrt_f32_e32
 ; GCN: v_sqrt_f32_e32
 
-; R600-DAG: RECIPSQRT_CLAMPED * T{{[0-9]\.[XYZW]}}, KC0[2].W
-; R600-DAG: MUL NON-IEEE T{{[0-9]\.[XYZW]}}, KC0[2].W, PS
-; R600-DAG: RECIPSQRT_CLAMPED * T{{[0-9]\.[XYZW]}}, KC0[3].X
-; R600-DAG: MUL NON-IEEE T{{[0-9]\.[XYZW]}}, KC0[3].X, PS
+; R600-DAG: RECIPSQRT_IEEE * T{{[0-9]\.[XYZW]}}, KC0[2].W
+; R600-DAG: RECIP_IEEE * T{{[0-9]\.[XYZW]}}, PS
+; R600-DAG: RECIPSQRT_IEEE * T{{[0-9]\.[XYZW]}}, KC0[3].X
+; R600-DAG: RECIP_IEEE * T{{[0-9]\.[XYZW]}}, T{{[0-9]\.[XYZW]}}
 define amdgpu_kernel void @s_sqrt_v2f32(<2 x float> addrspace(1)* %out, <2 x float> %in) #1 {
 entry:
   %fdiv = call <2 x float> @llvm.sqrt.v2f32(<2 x float> %in)
@@ -57,14 +57,14 @@ entry:
 ; GCN: v_sqrt_f32_e32
 ; GCN: v_sqrt_f32_e32
 
-; R600-DAG: RECIPSQRT_CLAMPED * T{{[0-9]\.[XYZW]}}, KC0[3].Y
-; R600-DAG: MUL NON-IEEE T{{[0-9]\.[XYZW]}}, KC0[3].Y, PS
-; R600-DAG: RECIPSQRT_CLAMPED * T{{[0-9]\.[XYZW]}}, KC0[3].Z
-; R600-DAG: MUL NON-IEEE T{{[0-9]\.[XYZW]}}, KC0[3].Z, PS
-; R600-DAG: RECIPSQRT_CLAMPED * T{{[0-9]\.[XYZW]}}, KC0[3].W
-; R600-DAG: MUL NON-IEEE T{{[0-9]\.[XYZW]}}, KC0[3].W, PS
-; R600-DAG: RECIPSQRT_CLAMPED * T{{[0-9]\.[XYZW]}}, KC0[4].X
-; R600-DAG: MUL NON-IEEE T{{[0-9]\.[XYZW]}}, KC0[4].X, PS
+; R600-DAG: RECIPSQRT_IEEE * T{{[0-9]\.[XYZW]}}, KC0[3].Y
+; R600-DAG: RECIP_IEEE * T{{[0-9]\.[XYZW]}}, PS
+; R600-DAG: RECIPSQRT_IEEE * T{{[0-9]\.[XYZW]}}, KC0[3].Z
+; R600-DAG: RECIP_IEEE * T{{[0-9]\.[XYZW]}}, T{{[0-9]\.[XYZW]}}
+; R600-DAG: RECIPSQRT_IEEE * T{{[0-9]\.[XYZW]}}, KC0[3].W
+; R600-DAG: RECIP_IEEE * T{{[0-9]\.[XYZW]}}, T{{[0-9]\.[XYZW]}}
+; R600-DAG: RECIPSQRT_IEEE * T{{[0-9]\.[XYZW]}}, KC0[4].X
+; R600-DAG: RECIP_IEEE * T{{[0-9]\.[XYZW]}}, T{{[0-9]\.[XYZW]}}
 define amdgpu_kernel void @s_sqrt_v4f32(<4 x float> addrspace(1)* %out, <4 x float> %in) #1 {
 entry:
   %fdiv = call <4 x float> @llvm.sqrt.v4f32(<4 x float> %in)
@@ -134,6 +134,16 @@ entry:
   ret void
 }
 
+; FUNC-LABEL: {{^}}recip_sqrt:
+; R600: RECIPSQRT_IEEE
+; R600-NOT: RECIP_IEEE
+define amdgpu_kernel void @recip_sqrt(float addrspace(1)* %out, float %src) nounwind {
+  %sqrt = call float @llvm.sqrt.f32(float %src)
+  %recipsqrt = fdiv fast float 1.0, %sqrt
+  store float %recipsqrt, float addrspace(1)* %out, align 4
+  ret void
+}
+
 declare float @llvm.sqrt.f32(float %in) #0
 declare <2 x float> @llvm.sqrt.v2f32(<2 x float> %in) #0
 declare <4 x float> @llvm.sqrt.v4f32(<4 x float> %in) #0
diff --git a/llvm/test/CodeGen/AMDGPU/fsub.ll b/llvm/test/CodeGen/AMDGPU/fsub.ll
index 48647a2cd..6e4635ec4 100644
--- a/llvm/test/CodeGen/AMDGPU/fsub.ll
+++ b/llvm/test/CodeGen/AMDGPU/fsub.ll
@@ -27,8 +27,8 @@ define amdgpu_kernel void @s_fsub_f32(float addrspace(1)* %out, float %a, float
 ; R600-DAG: ADD {{\** *}}T{{[0-9]+\.[XYZW]}}, KC0[3].X, -KC0[3].Z
 ; R600-DAG: ADD {{\** *}}T{{[0-9]+\.[XYZW]}}, KC0[2].W, -KC0[3].Y
 
-; SI: v_subrev_f32_e32 {{v[0-9]+}}, {{s[0-9]+}}, {{v[0-9]+}}
-; SI: v_subrev_f32_e32 {{v[0-9]+}}, {{s[0-9]+}}, {{v[0-9]+}}
+; SI: v_sub_f32_e32 {{v[0-9]+}}, {{s[0-9]+}}, {{v[0-9]+}}
+; SI: v_sub_f32_e32 {{v[0-9]+}}, {{s[0-9]+}}, {{v[0-9]+}}
 define amdgpu_kernel void @fsub_v2f32(<2 x float> addrspace(1)* %out, <2 x float> %a, <2 x float> %b) {
   %sub = fsub <2 x float> %a, %b
   store <2 x float> %sub, <2 x float> addrspace(1)* %out, align 8
@@ -55,10 +55,10 @@ define amdgpu_kernel void @v_fsub_v4f32(<4 x float> addrspace(1)* %out, <4 x flo
 }
 
 ; FUNC-LABEL: {{^}}s_fsub_v4f32:
-; SI: v_subrev_f32_e32 {{v[0-9]+}}, {{s[0-9]+}}, {{v[0-9]+}}
-; SI: v_subrev_f32_e32 {{v[0-9]+}}, {{s[0-9]+}}, {{v[0-9]+}}
-; SI: v_subrev_f32_e32 {{v[0-9]+}}, {{s[0-9]+}}, {{v[0-9]+}}
-; SI: v_subrev_f32_e32 {{v[0-9]+}}, {{s[0-9]+}}, {{v[0-9]+}}
+; SI: v_sub_f32_e32 {{v[0-9]+}}, {{s[0-9]+}}, {{v[0-9]+}}
+; SI: v_sub_f32_e32 {{v[0-9]+}}, {{s[0-9]+}}, {{v[0-9]+}}
+; SI: v_sub_f32_e32 {{v[0-9]+}}, {{s[0-9]+}}, {{v[0-9]+}}
+; SI: v_sub_f32_e32 {{v[0-9]+}}, {{s[0-9]+}}, {{v[0-9]+}}
 ; SI: s_endpgm
 define amdgpu_kernel void @s_fsub_v4f32(<4 x float> addrspace(1)* %out, <4 x float> %a, <4 x float> %b) {
   %result = fsub <4 x float> %a, %b
diff --git a/llvm/test/CodeGen/AMDGPU/function-returns.ll b/llvm/test/CodeGen/AMDGPU/function-returns.ll
index be1a02d06..f40729dc0 100644
--- a/llvm/test/CodeGen/AMDGPU/function-returns.ll
+++ b/llvm/test/CodeGen/AMDGPU/function-returns.ll
@@ -94,6 +94,16 @@ define i32 @i32_func_void() #0 {
   ret i32 %val
 }
 
+; GCN-LABEL: {{^}}i48_func_void:
+; GCN: buffer_load_dword v0, off
+; GCN-NEXT: buffer_load_ushort v1, off
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: s_setpc_b64
+define i48 @i48_func_void() #0 {
+  %val = load i48, i48 addrspace(1)* undef, align 8
+  ret i48 %val
+}
+
 ; GCN-LABEL: {{^}}i64_func_void:
 ; GCN: buffer_load_dwordx2 v[0:1], off
 ; GCN-NEXT: s_waitcnt vmcnt(0)
@@ -103,6 +113,16 @@ define i64 @i64_func_void() #0 {
   ret i64 %val
 }
 
+; GCN-LABEL: {{^}}i65_func_void:
+; GCN-DAG: buffer_load_dwordx2 v[0:1], off
+; GCN-DAG: buffer_load_ubyte v2, off
+; GCN: s_waitcnt vmcnt(0)
+; GCN-NEXT: s_setpc_b64
+define i65 @i65_func_void() #0 {
+  %val = load i65, i65 addrspace(1)* undef
+  ret i65 %val
+}
+
 ; GCN-LABEL: {{^}}f32_func_void:
 ; GCN: buffer_load_dword v0, off, s[4:7], 0
 ; GCN-NEXT: s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomics-fp.ll b/llvm/test/CodeGen/AMDGPU/global-atomics-fp.ll
new file mode 100644
index 000000000..b91536ead
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/global-atomics-fp.ll
@@ -0,0 +1,29 @@
+; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX900 %s
+; RUN: llc -march=amdgcn -mcpu=gfx908 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX908 %s
+
+; GCN-LABEL: {{^}}global_atomic_fadd_ret_f32:
+; GCN: [[LOOP:BB[0-9]+_[0-9]+]]
+; GCN: v_add_f32_e32
+; GCN: global_atomic_cmpswap
+; GCN: s_andn2_b64 exec, exec,
+; GCN-NEXT: s_cbranch_execnz [[LOOP]]
+define amdgpu_kernel void @global_atomic_fadd_ret_f32(float addrspace(1)* %ptr) {
+  %result = atomicrmw fadd float addrspace(1)* %ptr, float 4.0 seq_cst
+  store float %result, float addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}global_atomic_fadd_noret_f32:
+; GFX900: [[LOOP:BB[0-9]+_[0-9]+]]
+; GFX900: v_add_f32_e32
+; GFX900: global_atomic_cmpswap
+; GFX900: s_andn2_b64 exec, exec,
+; GFX900-NEXT: s_cbranch_execnz [[LOOP]]
+
+; GFX908-NOT: v_add_f32
+; GFX908: global_atomic_add_f32 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, off
+; GFX908-NOT: s_cbranch_execnz
+define amdgpu_kernel void @global_atomic_fadd_noret_f32(float addrspace(1)* %ptr) {
+  %result = atomicrmw fadd float addrspace(1)* %ptr, float 4.0 seq_cst
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/global-constant.ll b/llvm/test/CodeGen/AMDGPU/global-constant.ll
index 3ebc8e1e7..479f026b8 100644
--- a/llvm/test/CodeGen/AMDGPU/global-constant.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-constant.ll
@@ -1,5 +1,8 @@
-; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=NOHSA %s
-; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=kaveri -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=HSA %s
+; RUN: llc -mtriple=amdgcn--amdpal -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=PAL %s
+; RUN: llc -mtriple=amdgcn-- -mcpu=kaveri -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=NOPAL %s
+; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=kaveri -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=NOPAL %s
+; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=kaveri -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=NOPAL %s
+; RUN: llc -mtriple=r600-- -mcpu=cypress < %s | FileCheck -check-prefix=R600 %s
 
 @private1 = private unnamed_addr addrspace(4) constant [4 x float] [float 0.0, float 1.0, float 2.0, float 3.0]
 @private2 = private unnamed_addr addrspace(4) constant [4 x float] [float 4.0, float 5.0, float 6.0, float 7.0]
@@ -8,24 +11,25 @@
 ; GCN-LABEL: {{^}}private_test:
 ; GCN: s_getpc_b64 s{{\[}}[[PC0_LO:[0-9]+]]:[[PC0_HI:[0-9]+]]{{\]}}
 
-; Non-HSA OSes use fixup into .text section.
-; NOHSA: s_add_u32 s{{[0-9]+}}, s[[PC0_LO]], private1
-; NOHSA: s_addc_u32 s{{[0-9]+}}, s[[PC0_HI]], 0
+; PAL OSes use fixup into .text section.
+; PAL: s_add_u32 s{{[0-9]+}}, s[[PC0_LO]], private1
+; PAL: s_addc_u32 s{{[0-9]+}}, s[[PC0_HI]], 0
 
-; HSA OSes use relocations.
-; HSA: s_add_u32 s{{[0-9]+}}, s[[PC0_LO]], private1@rel32@lo+4
-; HSA: s_addc_u32 s{{[0-9]+}}, s[[PC0_HI]], private1@rel32@hi+4
+; Non-PAL OSes use relocations.
+; NOPAL: s_add_u32 s{{[0-9]+}}, s[[PC0_LO]], private1@rel32@lo+4
+; NOPAL: s_addc_u32 s{{[0-9]+}}, s[[PC0_HI]], private1@rel32@hi+4
 
 ; GCN: s_getpc_b64 s{{\[}}[[PC1_LO:[0-9]+]]:[[PC1_HI:[0-9]+]]{{\]}}
 
-; Non-HSA OSes use fixup into .text section.
-; NOHSA: s_add_u32 s{{[0-9]+}}, s[[PC1_LO]], private2
-; NOHSA: s_addc_u32 s{{[0-9]+}}, s[[PC1_HI]], 0
+; PAL OSes use fixup into .text section.
+; PAL: s_add_u32 s{{[0-9]+}}, s[[PC1_LO]], private2
+; PAL: s_addc_u32 s{{[0-9]+}}, s[[PC1_HI]], 0
 
-; HSA OSes use relocations.
-; HSA: s_add_u32 s{{[0-9]+}}, s[[PC1_LO]], private2@rel32@lo+4
-; HSA: s_addc_u32 s{{[0-9]+}}, s[[PC1_HI]], private2@rel32@hi+4
+; Non-PAL OSes use relocations.
+; NOPAL: s_add_u32 s{{[0-9]+}}, s[[PC1_LO]], private2@rel32@lo+4
+; NOPAL: s_addc_u32 s{{[0-9]+}}, s[[PC1_HI]], private2@rel32@hi+4
 
+; R600-LABEL: private_test
 define amdgpu_kernel void @private_test(i32 %index, float addrspace(1)* %out) {
   %ptr = getelementptr [4 x float], [4 x float] addrspace(4) * @private1, i32 0, i32 %index
   %val = load float, float addrspace(4)* %ptr
@@ -36,10 +40,11 @@ define amdgpu_kernel void @private_test(i32 %index, float addrspace(1)* %out) {
   ret void
 }
 
-; HSA-LABEL: {{^}}available_externally_test:
-; HSA: s_getpc_b64 s{{\[}}[[PC0_LO:[0-9]+]]:[[PC0_HI:[0-9]+]]{{\]}}
-; HSA: s_add_u32 s{{[0-9]+}}, s[[PC0_LO]], available_externally@gotpcrel32@lo+4
-; HSA: s_addc_u32 s{{[0-9]+}}, s[[PC0_HI]], available_externally@gotpcrel32@hi+4
+; NOPAL-LABEL: {{^}}available_externally_test:
+; NOPAL: s_getpc_b64 s{{\[}}[[PC0_LO:[0-9]+]]:[[PC0_HI:[0-9]+]]{{\]}}
+; NOPAL: s_add_u32 s{{[0-9]+}}, s[[PC0_LO]], available_externally@gotpcrel32@lo+4
+; NOPAL: s_addc_u32 s{{[0-9]+}}, s[[PC0_HI]], available_externally@gotpcrel32@hi+4
+; R600-LABEL: available_externally_test
 define amdgpu_kernel void @available_externally_test(i32 addrspace(1)* %out) {
   %ptr = getelementptr [256 x i32], [256 x i32] addrspace(4)* @available_externally, i32 0, i32 1
   %val = load i32, i32 addrspace(4)* %ptr
@@ -47,8 +52,11 @@ define amdgpu_kernel void @available_externally_test(i32 addrspace(1)* %out) {
   ret void
 }
 
-; NOHSA: .text
-; HSA: .section .rodata
+; PAL: .text
+; NOPAL: .section .rodata
+; R600: .text
 
 ; GCN: private1:
 ; GCN: private2:
+; R600: private1:
+; R600: private2:
diff --git a/llvm/test/CodeGen/AMDGPU/global-saddr.ll b/llvm/test/CodeGen/AMDGPU/global-saddr.ll
index b21fd9852..4df1ad683 100644
--- a/llvm/test/CodeGen/AMDGPU/global-saddr.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-saddr.ll
@@ -46,7 +46,7 @@ entry:
 
 ; Test various offset boundaries.
 ; GFX9: global_load_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, s[{{[0-9]+}}:{{[0-9]+}}] offset:4088{{$}}
-; GFX9: global_load_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, off{{$}}
+; GFX9: global_load_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, off offset:4088{{$}}
 ; GFX9: global_load_dwordx4 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, s[{{[0-9]+}}:{{[0-9]+}}] offset:2040{{$}}
   %gep11 = getelementptr inbounds i64, i64 addrspace(1)* %gep, i64 511
   %load11 = load i64, i64 addrspace(1)* %gep11
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics.ll b/llvm/test/CodeGen/AMDGPU/global_atomics.ll
index 72f748578..618c70083 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics.ll
@@ -25,7 +25,9 @@ entry:
 ; SIVI: s_mov_b32 [[SREG:s[0-9]+]], 0x8ca0
 ; SIVI: buffer_atomic_add v{{[0-9]+}}, off, s[{{[0-9]+}}:{{[0-9]+}}], [[SREG]]{{$}}
 
-; GFX9: global_atomic_add v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}, off{{$}}
+; GFX9: v_add_co_u32_e32 v{{[0-9]+}}, vcc, 0x8000,
+; GFX9-NEXT: v_addc_co_u32_e32 v{{[0-9]+}}, vcc, 0, v{{[0-9]+}}, vcc
+; GFX9: global_atomic_add v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}, off offset:3232{{$}}
 define amdgpu_kernel void @atomic_add_i32_soffset(i32 addrspace(1)* %out, i32 %in) {
 entry:
   %gep = getelementptr i32, i32 addrspace(1)* %out, i64 9000
@@ -40,7 +42,10 @@ entry:
 
 ; VI: flat_atomic_add
 
-; GFX9: global_atomic_add v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}, off{{$}}
+; GFX9: v_mov_b32_e32 [[HIGH_K:v[0-9]+]], 0xabcd
+; GFX9: v_add_co_u32_e32 v{{[0-9]+}}, vcc, 0xd000,
+; GFX9-NEXT: v_addc_co_u32_e32 v{{[0-9]+}}, vcc, [[HIGH_K]], v{{[0-9]+}}, vcc
+; GFX9: global_atomic_add v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}, off offset:3756{{$}}
 define amdgpu_kernel void @atomic_add_i32_huge_offset(i32 addrspace(1)* %out, i32 %in) {
 entry:
   %gep = getelementptr i32, i32 addrspace(1)* %out, i64 47224239175595
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_i64.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_i64.ll
index 0ef58fc1f..b996646a0 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_i64.ll
@@ -991,7 +991,9 @@ entry:
 ; CIVI: s_mov_b32 [[SREG:s[0-9]+]], 0x11940
 ; CIVI: buffer_atomic_cmpswap_x2 v[{{[0-9]+}}:{{[0-9]+}}], off, s[{{[0-9]+}}:{{[0-9]+}}], [[SREG]]{{$}}
 
-; GFX9: global_atomic_cmpswap_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off{{$}}
+; GFX9: v_add_co_u32_e32 v{{[0-9]+}}, vcc, 0x11000,
+; GFX9: v_addc_co_u32_e32 v{{[0-9]+}}, vcc, 0, v{{[0-9]+}}, vcc
+; GFX9: global_atomic_cmpswap_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off offset:2368{{$}}
 define amdgpu_kernel void @atomic_cmpxchg_i64_soffset(i64 addrspace(1)* %out, i64 %in, i64 %old) {
 entry:
   %gep = getelementptr i64, i64 addrspace(1)* %out, i64 9000
diff --git a/llvm/test/CodeGen/AMDGPU/global_smrd.ll b/llvm/test/CodeGen/AMDGPU/global_smrd.ll
index e4ad729ff..070aa1342 100644
--- a/llvm/test/CodeGen/AMDGPU/global_smrd.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_smrd.ll
@@ -83,8 +83,8 @@ define amdgpu_kernel void @memdep(i32 addrspace(1)* %in, [8 x i32], i32 addrspac
 ; CHECK: s_getpc_b64 [[GET_PC:s\[[0-9]+:[0-9]+\]]]
 ; CHECK: s_load_dwordx2 [[A_ADDR:s\[[0-9]+:[0-9]+\]]], [[GET_PC]], 0x0
 ; CHECK: s_load_dwordx2 [[A_ADDR1:s\[[0-9]+:[0-9]+\]]], [[A_ADDR]], 0x0
-; CHECK: s_load_dword [[SVAL:s[0-9]+]], [[A_ADDR1]], 0x0
 ; CHECK: s_load_dwordx2 [[OUT:s\[[0-9]+:[0-9]+\]]], s[4:5], 0x0
+; CHECK: s_load_dword [[SVAL:s[0-9]+]], [[A_ADDR1]], 0x0
 ; CHECK: v_mov_b32_e32 [[VVAL:v[0-9]+]], [[SVAL]]
 ; CHECK: flat_store_dword v[{{[0-9]+:[0-9]+}}], [[VVAL]]
 @A = common local_unnamed_addr addrspace(1) global i32 addrspace(1)* null, align 4
diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-from-llvm-ir-full-v3.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-from-llvm-ir-full-v3.ll
index 72e96a196..9d34d417e 100644
--- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-from-llvm-ir-full-v3.ll
+++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-from-llvm-ir-full-v3.ll
@@ -42,6 +42,7 @@
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NOT:          .value_kind:     hidden_default_queue
 ; CHECK-NOT:          .value_kind:     hidden_completion_action
+; CHECK-NOT:          .value_kind:     hidden_hostcall_buffer
 ; CHECK-NEXT:         .value_kind:     hidden_printf_buffer
 ; CHECK-NEXT:         .value_type:     i8
 ; CHECK:              .value_kind:     hidden_multigrid_sync_arg
diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-from-llvm-ir-full.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-from-llvm-ir-full.ll
index db1d5c5ec..f5a47a862 100644
--- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-from-llvm-ir-full.ll
+++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-from-llvm-ir-full.ll
@@ -49,6 +49,7 @@
 ; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
+; CHECK-NOT:        ValueKind:     HiddenHostcallBuffer
 ; CHECK-NEXT:       ValueKind:     HiddenPrintfBuffer
 ; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-absent-v3.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-absent-v3.ll
new file mode 100644
index 000000000..8741bfbc1
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-absent-v3.ll
@@ -0,0 +1,55 @@
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mattr=+code-object-v3 -mcpu=gfx900 -filetype=obj -o - < %s | llvm-readelf --notes | FileCheck %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mattr=+code-object-v3 -mcpu=gfx900 -amdgpu-dump-hsa-metadata -amdgpu-verify-hsa-metadata -filetype=obj -o - < %s 2>&1 | FileCheck --check-prefix=PARSER %s
+
+; CHECK:              ---
+; CHECK:      amdhsa.kernels:
+; CHECK:        - .args:
+; CHECK-NEXT:       - .name:           a
+; CHECK-NEXT:         .offset:         0
+; CHECK-NEXT:         .size:           1
+; CHECK-NEXT:         .type_name:      char
+; CHECK-NEXT:         .value_kind:     by_value
+; CHECK-NEXT:         .value_type:     i8
+; CHECK-NEXT:       - .offset:         8
+; CHECK-NEXT:         .size:           8
+; CHECK-NEXT:         .value_kind:     hidden_global_offset_x
+; CHECK-NEXT:         .value_type:     i64
+; CHECK-NEXT:       - .offset:         16
+; CHECK-NEXT:         .size:           8
+; CHECK-NEXT:         .value_kind:     hidden_global_offset_y
+; CHECK-NEXT:         .value_type:     i64
+; CHECK-NEXT:       - .offset:         24
+; CHECK-NEXT:         .size:           8
+; CHECK-NEXT:         .value_kind:     hidden_global_offset_z
+; CHECK-NEXT:         .value_type:     i64
+
+; CHECK-NOT:          .value_kind:     hidden_hostcall_buffer
+
+; CHECK:          .language:       OpenCL C
+; CHECK-NEXT:     .language_version:
+; CHECK-NEXT:       - 2
+; CHECK-NEXT:       - 0
+; CHECK:          .name:           test_kernel
+; CHECK:          .symbol:         test_kernel.kd
+
+define amdgpu_kernel void @test_kernel(i8 %a) #0
+    !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !3
+    !kernel_arg_base_type !3 !kernel_arg_type_qual !4 {
+  ret void
+}
+
+; CHECK:  amdhsa.version:
+; CHECK-NEXT: - 1
+; CHECK-NEXT: - 0
+
+attributes #0 = { "amdgpu-implicitarg-num-bytes"="48" }
+
+!1 = !{i32 0}
+!2 = !{!"none"}
+!3 = !{!"char"}
+!4 = !{!""}
+
+!opencl.ocl.version = !{!90}
+!90 = !{i32 2, i32 0}
+
+; PARSER: AMDGPU HSA Metadata Parser Test: PASS
diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-absent.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-absent.ll
new file mode 100644
index 000000000..5f1cda0fd
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-absent.ll
@@ -0,0 +1,52 @@
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mattr=-code-object-v3 -mcpu=gfx900 -filetype=obj -o - < %s | llvm-readelf --notes | FileCheck %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mattr=-code-object-v3 -mcpu=gfx900 -amdgpu-dump-hsa-metadata -amdgpu-verify-hsa-metadata -filetype=obj -o - < %s 2>&1 | FileCheck --check-prefix=PARSER %s
+
+; CHECK: ---
+; CHECK:  Version: [ 1, 0 ]
+; CHECK:  Kernels:
+
+; CHECK:      - Name:            test_kernel
+; CHECK-NEXT:   SymbolName:      'test_kernel@kd'
+; CHECK-NEXT:   Language:        OpenCL C
+; CHECK-NEXT:   LanguageVersion: [ 2, 0 ]
+; CHECK-NEXT:   Args:
+; CHECK-NEXT:     - Name:          a
+; CHECK-NEXT:       TypeName:      char
+; CHECK-NEXT:       Size:          1
+; CHECK-NEXT:       Align:         1
+; CHECK-NEXT:       ValueKind:     ByValue
+; CHECK-NEXT:       ValueType:     I8
+; CHECK-NEXT:       AccQual:       Default
+; CHECK-NEXT:     - Size:          8
+; CHECK-NEXT:       Align:         8
+; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
+; CHECK-NEXT:       ValueType:     I64
+; CHECK-NEXT:     - Size:          8
+; CHECK-NEXT:       Align:         8
+; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetY
+; CHECK-NEXT:       ValueType:     I64
+; CHECK-NEXT:     - Size:          8
+; CHECK-NEXT:       Align:         8
+; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetZ
+; CHECK-NEXT:       ValueType:     I64
+; CHECK-NOT:        ValueKind:     HiddenHostcallBuffer
+; CHECK-NOT:        ValueKind:     HiddenDefaultQueue
+; CHECK-NOT:        ValueKind:     HiddenCompletionAction
+
+define amdgpu_kernel void @test_kernel(i8 %a) #0
+    !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !3
+    !kernel_arg_base_type !3 !kernel_arg_type_qual !4 {
+  ret void
+}
+
+attributes #0 = { "amdgpu-implicitarg-num-bytes"="48" }
+
+!1 = !{i32 0}
+!2 = !{!"none"}
+!3 = !{!"char"}
+!4 = !{!""}
+
+!opencl.ocl.version = !{!90}
+!90 = !{i32 2, i32 0}
+
+; PARSER: AMDGPU HSA Metadata Parser Test: PASS
diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-present-v3.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-present-v3.ll
new file mode 100644
index 000000000..1a75f3661
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-present-v3.ll
@@ -0,0 +1,59 @@
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mattr=+code-object-v3 -mcpu=gfx900 -filetype=obj -o - < %s | llvm-readelf --notes | FileCheck %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mattr=+code-object-v3 -mcpu=gfx900 -amdgpu-dump-hsa-metadata -amdgpu-verify-hsa-metadata -filetype=obj -o - < %s 2>&1 | FileCheck --check-prefix=PARSER %s
+
+; CHECK:              ---
+; CHECK:      amdhsa.kernels:
+; CHECK:        - .args:
+; CHECK-NEXT:       - .name:           a
+; CHECK-NEXT:         .offset:         0
+; CHECK-NEXT:         .size:           1
+; CHECK-NEXT:         .type_name:      char
+; CHECK-NEXT:         .value_kind:     by_value
+; CHECK-NEXT:         .value_type:     i8
+; CHECK-NEXT:       - .offset:         8
+; CHECK-NEXT:         .size:           8
+; CHECK-NEXT:         .value_kind:     hidden_global_offset_x
+; CHECK-NEXT:         .value_type:     i64
+; CHECK-NEXT:       - .offset:         16
+; CHECK-NEXT:         .size:           8
+; CHECK-NEXT:         .value_kind:     hidden_global_offset_y
+; CHECK-NEXT:         .value_type:     i64
+; CHECK-NEXT:       - .offset:         24
+; CHECK-NEXT:         .size:           8
+; CHECK-NEXT:         .value_kind:     hidden_global_offset_z
+; CHECK-NEXT:         .value_type:     i64
+; CHECK-NEXT:       - .address_space:  global
+; CHECK-NEXT:         .offset:         32
+; CHECK-NEXT:         .size:           8
+; CHECK-NEXT:         .value_kind:     hidden_hostcall_buffer
+; CHECK-NEXT:         .value_type:     i8
+; CHECK:          .language:       OpenCL C
+; CHECK-NEXT:     .language_version:
+; CHECK-NEXT:       - 2
+; CHECK-NEXT:       - 0
+; CHECK:          .name:           test_kernel
+; CHECK:          .symbol:         test_kernel.kd
+
+declare <2 x i64> @__ockl_hostcall_internal(i8*, i32, i64, i64, i64, i64, i64, i64, i64, i64)
+
+define amdgpu_kernel void @test_kernel(i8 %a) #0
+    !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !3
+    !kernel_arg_base_type !3 !kernel_arg_type_qual !4 {
+  ret void
+}
+
+; CHECK:  amdhsa.version:
+; CHECK-NEXT: - 1
+; CHECK-NEXT: - 0
+
+attributes #0 = { "amdgpu-implicitarg-num-bytes"="48" }
+
+!1 = !{i32 0}
+!2 = !{!"none"}
+!3 = !{!"char"}
+!4 = !{!""}
+
+!opencl.ocl.version = !{!90}
+!90 = !{i32 2, i32 0}
+
+; PARSER: AMDGPU HSA Metadata Parser Test: PASS
diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-present.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-present.ll
new file mode 100644
index 000000000..b0428638e
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-present.ll
@@ -0,0 +1,58 @@
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mattr=-code-object-v3 -mcpu=gfx900 -filetype=obj -o - < %s | llvm-readelf --notes | FileCheck %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mattr=-code-object-v3 -mcpu=gfx900 -amdgpu-dump-hsa-metadata -amdgpu-verify-hsa-metadata -filetype=obj -o - < %s 2>&1 | FileCheck --check-prefix=PARSER %s
+
+; CHECK: ---
+; CHECK:  Version: [ 1, 0 ]
+; CHECK:  Kernels:
+
+; CHECK:      - Name:            test_kernel
+; CHECK-NEXT:   SymbolName:      'test_kernel@kd'
+; CHECK-NEXT:   Language:        OpenCL C
+; CHECK-NEXT:   LanguageVersion: [ 2, 0 ]
+; CHECK-NEXT:   Args:
+; CHECK-NEXT:     - Name:          a
+; CHECK-NEXT:       TypeName:      char
+; CHECK-NEXT:       Size:          1
+; CHECK-NEXT:       Align:         1
+; CHECK-NEXT:       ValueKind:     ByValue
+; CHECK-NEXT:       ValueType:     I8
+; CHECK-NEXT:       AccQual:       Default
+; CHECK-NEXT:     - Size:          8
+; CHECK-NEXT:       Align:         8
+; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
+; CHECK-NEXT:       ValueType:     I64
+; CHECK-NEXT:     - Size:          8
+; CHECK-NEXT:       Align:         8
+; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetY
+; CHECK-NEXT:       ValueType:     I64
+; CHECK-NEXT:     - Size:          8
+; CHECK-NEXT:       Align:         8
+; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetZ
+; CHECK-NEXT:       ValueType:     I64
+; CHECK-NEXT:     - Size:            8
+; CHECK-NEXT:       Align:           8
+; CHECK-NEXT:       ValueKind:       HiddenHostcallBuffer
+; CHECK-NEXT:       ValueType:       I8
+; CHECK-NEXT:       AddrSpaceQual:   Global
+; CHECK-NOT:        ValueKind:     HiddenDefaultQueue
+; CHECK-NOT:        ValueKind:     HiddenCompletionAction
+
+declare <2 x i64> @__ockl_hostcall_internal(i8*, i32, i64, i64, i64, i64, i64, i64, i64, i64)
+
+define amdgpu_kernel void @test_kernel(i8 %a) #0
+    !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !3
+    !kernel_arg_base_type !3 !kernel_arg_type_qual !4 {
+  ret void
+}
+
+attributes #0 = { "amdgpu-implicitarg-num-bytes"="48" }
+
+!1 = !{i32 0}
+!2 = !{!"none"}
+!3 = !{!"char"}
+!4 = !{!""}
+
+!opencl.ocl.version = !{!90}
+!90 = !{i32 2, i32 0}
+
+; PARSER: AMDGPU HSA Metadata Parser Test: PASS
diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-kernel-code-props-v3.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-kernel-code-props-v3.ll
index 0eed325bc..14e8e609e 100644
--- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-kernel-code-props-v3.ll
+++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-kernel-code-props-v3.ll
@@ -8,11 +8,11 @@
 ; CHECK: ---
 ; CHECK:  amdhsa.kernels:
 
-; CHECK:   - .args:           
+; CHECK:   - .args:
 ; CHECK:     .group_segment_fixed_size: 0
 ; CHECK:     .kernarg_segment_align: 8
 ; CHECK:     .kernarg_segment_size: 24
-; CHECK:     .max_flat_workgroup_size: 256
+; CHECK:     .max_flat_workgroup_size: 1024
 ; CHECK:     .name:           test
 ; CHECK:     .private_segment_fixed_size: 0
 ; WAVE64:    .sgpr_count:     8
@@ -33,6 +33,20 @@ entry:
   ret void
 }
 
+; CHECK:   - .args:
+; CHECK:     .max_flat_workgroup_size: 256
+define amdgpu_kernel void @test_max_flat_workgroup_size(
+    half addrspace(1)* %r,
+    half addrspace(1)* %a,
+    half addrspace(1)* %b) #2 {
+entry:
+  %a.val = load half, half addrspace(1)* %a
+  %b.val = load half, half addrspace(1)* %b
+  %r.val = fadd half %a.val, %b.val
+  store half %r.val, half addrspace(1)* %r
+  ret void
+}
+
 ; CHECK:   .name:       num_spilled_sgprs
 ; GFX700:   .sgpr_spill_count: 40
 ; GFX803:   .sgpr_spill_count: 24
@@ -149,3 +163,4 @@ define amdgpu_kernel void @num_spilled_vgprs() #1 {
 
 attributes #0 = { "amdgpu-num-sgpr"="14" }
 attributes #1 = { "amdgpu-num-vgpr"="20" }
+attributes #2 = { "amdgpu-flat-work-group-size"="1,256" }
diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-kernel-code-props.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-kernel-code-props.ll
index 1678df8bc..11dc60a5e 100644
--- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-kernel-code-props.ll
+++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-kernel-code-props.ll
@@ -18,7 +18,7 @@
 ; CHECK:     WavefrontSize:           64
 ; CHECK:     NumSGPRs:                8
 ; CHECK:     NumVGPRs:                6
-; CHECK:     MaxFlatWorkGroupSize:    256
+; CHECK:     MaxFlatWorkGroupSize:    1024
 define amdgpu_kernel void @test(
     half addrspace(1)* %r,
     half addrspace(1)* %a,
@@ -31,6 +31,29 @@ entry:
   ret void
 }
 
+; CHECK-LABEL: - Name:       test_max_flat_workgroup_size
+; CHECK:   SymbolName: 'test_max_flat_workgroup_size@kd'
+; CHECK:   CodeProps:
+; CHECK:     KernargSegmentSize:      24
+; CHECK:     GroupSegmentFixedSize:   0
+; CHECK:     PrivateSegmentFixedSize: 0
+; CHECK:     KernargSegmentAlign:     8
+; CHECK:     WavefrontSize:           64
+; CHECK:     NumSGPRs:                8
+; CHECK:     NumVGPRs:                6
+; CHECK:     MaxFlatWorkGroupSize:    256
+define amdgpu_kernel void @test_max_flat_workgroup_size(
+    half addrspace(1)* %r,
+    half addrspace(1)* %a,
+    half addrspace(1)* %b) #2 {
+entry:
+  %a.val = load half, half addrspace(1)* %a
+  %b.val = load half, half addrspace(1)* %b
+  %r.val = fadd half %a.val, %b.val
+  store half %r.val, half addrspace(1)* %r
+  ret void
+}
+
 ; CHECK-LABEL: - Name:       num_spilled_sgprs
 ; CHECK:   SymbolName: 'num_spilled_sgprs@kd'
 ; CHECK:   CodeProps:
@@ -144,3 +167,4 @@ define amdgpu_kernel void @num_spilled_vgprs() #1 {
 
 attributes #0 = { "amdgpu-num-sgpr"="14" }
 attributes #1 = { "amdgpu-num-vgpr"="20" }
+attributes #2 = { "amdgpu-flat-work-group-size"="1,256" }
diff --git a/llvm/test/CodeGen/AMDGPU/i1-copy-from-loop.ll b/llvm/test/CodeGen/AMDGPU/i1-copy-from-loop.ll
index a050bfe29..51d1c091a 100644
--- a/llvm/test/CodeGen/AMDGPU/i1-copy-from-loop.ll
+++ b/llvm/test/CodeGen/AMDGPU/i1-copy-from-loop.ll
@@ -3,22 +3,21 @@
 
 ; SI-LABEL: {{^}}i1_copy_from_loop:
 ;
-; SI: ; %Flow
-; SI-DAG:  s_andn2_b64       [[LCSSA_ACCUM:s\[[0-9]+:[0-9]+\]]], [[LCSSA_ACCUM]], exec
-; SI-DAG:  s_and_b64         [[CC_MASK2:s\[[0-9]+:[0-9]+\]]], [[CC_ACCUM:s\[[0-9]+:[0-9]+\]]], exec
-; SI:      s_or_b64          [[LCSSA_ACCUM]], [[LCSSA_ACCUM]], [[CC_MASK2]]
+; SI:   ; %Flow
+; SI:  s_or_b64 [[EXIT_MASK:s\[[0-9]+:[0-9]+\]]]
+; SI:  s_and_b64 [[ACCUM_MASK:s\[[0-9]+:[0-9]+\]]], [[CC_MASK:s\[[0-9]+:[0-9]+\]]], exec
+; SI:  s_or_b64  [[I1_VALUE:s\[[0-9]+:[0-9]+\]]], {{s\[[0-9]+:[0-9]+\]}}, [[ACCUM_MASK]]
+; SI:  s_cbranch_execz [[FOR_END_LABEL:BB0_[0-9]+]]
 
 ; SI: ; %for.body
-; SI:      v_cmp_gt_u32_e64  [[CC_SREG:s\[[0-9]+:[0-9]+\]]], 4,
-; SI-DAG:  s_andn2_b64       [[CC_ACCUM]], [[CC_ACCUM]], exec
-; SI-DAG:  s_and_b64         [[CC_MASK:s\[[0-9]+:[0-9]+\]]], [[CC_SREG]], exec
-; SI:      s_or_b64          [[CC_ACCUM]], [[CC_ACCUM]], [[CC_MASK]]
-
-; SI: ; %Flow1
-; SI:      s_or_b64          [[CC_ACCUM]], [[CC_ACCUM]], exec
-
-; SI: ; %for.end
-; SI:      s_and_saveexec_b64 {{s\[[0-9]+:[0-9]+\]}}, [[LCSSA_ACCUM]]
+; SI:      v_cmp_lt_u32_e64  [[CC_MASK]], s{{[0-9]+}}, 4
+
+; SI: [[FOR_END_LABEL]]
+; SI:      s_or_b64 exec, exec, [[EXIT_MASK]]
+; SI:      s_and_saveexec_b64 {{s\[[0-9]+:[0-9]+\]}}, [[I1_VALUE]]
+; SI:      s_cbranch_execz [[EXIT:BB0_[0-9]+]]
+; SI: [[EXIT]]
+; SI-NEXT: s_endpgm
 
 define amdgpu_ps void @i1_copy_from_loop(<4 x i32> inreg %rsrc, i32 %tid) {
 entry:
diff --git a/llvm/test/CodeGen/AMDGPU/i1-copy-phi-uniform-branch.ll b/llvm/test/CodeGen/AMDGPU/i1-copy-phi-uniform-branch.ll
index 0aacbbfda..c65683d4f 100644
--- a/llvm/test/CodeGen/AMDGPU/i1-copy-phi-uniform-branch.ll
+++ b/llvm/test/CodeGen/AMDGPU/i1-copy-phi-uniform-branch.ll
@@ -7,7 +7,6 @@
 ; GCN:      s_cbranch_scc1  [[PREEXIT:BB[0-9_]+]]
 
 ; GCN: ; %blocka
-; GCN:      s_xor_b64       s[{{[0-9:]+}}], exec, -1
 ; GCN:      s_cmp_eq_u32    s1, 0
 ; GCN:      s_cbranch_scc1  [[EXIT:BB[0-9_]+]]
 
diff --git a/llvm/test/CodeGen/AMDGPU/idiv-licm.ll b/llvm/test/CodeGen/AMDGPU/idiv-licm.ll
index d46f96ef6..15f9fb2a6 100644
--- a/llvm/test/CodeGen/AMDGPU/idiv-licm.ll
+++ b/llvm/test/CodeGen/AMDGPU/idiv-licm.ll
@@ -1,24 +1,53 @@
-; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN %s
-
-; GCN-LABEL: {{^}}udiv32_invariant_denom:
-; GCN:     v_cvt_f32_u32
-; GCN:     v_rcp_iflag_f32
-; GCN:     v_mul_f32_e32 v{{[0-9]+}}, 0x4f800000,
-; GCN:     v_cvt_u32_f32_e32
-; GCN-DAG: v_mul_hi_u32
-; GCN-DAG: v_mul_lo_u32
-; GCN-DAG: v_sub_i32_e32
-; GCN-DAG: v_cmp_eq_u32_e64
-; GCN-DAG: v_cndmask_b32_e64
-; GCN-DAG: v_mul_hi_u32
-; GCN-DAG: v_add_i32_e32
-; GCN-DAG: v_subrev_i32_e32
-; GCN-DAG: v_cndmask_b32_e64
-; GCN:     [[LOOP:BB[0-9_]+]]:
-; GCN-NOT: v_rcp
-; GCN:     s_cbranch_scc0 [[LOOP]]
-; GCN:     s_endpgm
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GFX9 %s
+
 define amdgpu_kernel void @udiv32_invariant_denom(i32 addrspace(1)* nocapture %arg, i32 %arg1) {
+; GFX9-LABEL: udiv32_invariant_denom:
+; GFX9:       ; %bb.0: ; %bb
+; GFX9-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
+; GFX9-NEXT:    s_mov_b64 s[6:7], 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s2
+; GFX9-NEXT:    s_sub_i32 s3, 0, s2
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GFX9-NEXT:    v_mul_f32_e32 v0, 0x4f800000, v0
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GFX9-NEXT:    v_mul_lo_u32 v1, v0, s2
+; GFX9-NEXT:    v_mul_hi_u32 v2, v0, s2
+; GFX9-NEXT:    v_sub_u32_e32 v3, 0, v1
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX9-NEXT:    v_mul_hi_u32 v1, v1, v0
+; GFX9-NEXT:    v_add_u32_e32 v2, v0, v1
+; GFX9-NEXT:    v_sub_u32_e32 v0, v0, v1
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX9-NEXT:  BB0_1: ; %bb3
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    v_mul_lo_u32 v1, v0, s7
+; GFX9-NEXT:    v_mul_hi_u32 v2, v0, s6
+; GFX9-NEXT:    v_add_u32_e32 v3, v2, v1
+; GFX9-NEXT:    v_mul_lo_u32 v1, s3, v3
+; GFX9-NEXT:    v_mul_lo_u32 v4, v3, s2
+; GFX9-NEXT:    v_add_u32_e32 v7, 1, v3
+; GFX9-NEXT:    v_add_u32_e32 v6, -1, v3
+; GFX9-NEXT:    v_add_u32_e32 v5, s6, v1
+; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, s6, v4
+; GFX9-NEXT:    v_cmp_le_u32_e64 s[0:1], s2, v5
+; GFX9-NEXT:    s_and_b64 s[0:1], s[0:1], vcc
+; GFX9-NEXT:    s_add_u32 s6, s6, 1
+; GFX9-NEXT:    v_mov_b32_e32 v1, s4
+; GFX9-NEXT:    s_addc_u32 s7, s7, 0
+; GFX9-NEXT:    v_mov_b32_e32 v2, s5
+; GFX9-NEXT:    s_add_u32 s4, s4, 4
+; GFX9-NEXT:    v_cndmask_b32_e64 v3, v3, v7, s[0:1]
+; GFX9-NEXT:    s_addc_u32 s5, s5, 0
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v6, v3, vcc
+; GFX9-NEXT:    s_cmpk_eq_i32 s6, 0x400
+; GFX9-NEXT:    global_store_dword v[1:2], v3, off
+; GFX9-NEXT:    s_cbranch_scc0 BB0_1
+; GFX9-NEXT:  ; %bb.2: ; %bb2
+; GFX9-NEXT:    s_endpgm
 bb:
   br label %bb3
 
@@ -36,25 +65,57 @@ bb3:                                              ; preds = %bb3, %bb
   br i1 %tmp8, label %bb2, label %bb3
 }
 
-; GCN-LABEL: {{^}}urem32_invariant_denom:
-; GCN:     v_cvt_f32_u32
-; GCN:     v_rcp_iflag_f32
-; GCN:     v_mul_f32_e32 v{{[0-9]+}}, 0x4f800000,
-; GCN:     v_cvt_u32_f32_e32
-; GCN-DAG: v_mul_hi_u32
-; GCN-DAG: v_mul_lo_u32
-; GCN-DAG: v_sub_i32_e32
-; GCN-DAG: v_cmp_eq_u32_e64
-; GCN-DAG: v_cndmask_b32_e64
-; GCN-DAG: v_mul_hi_u32
-; GCN-DAG: v_add_i32_e32
-; GCN-DAG: v_subrev_i32_e32
-; GCN-DAG: v_cndmask_b32_e64
-; GCN:     [[LOOP:BB[0-9_]+]]:
-; GCN-NOT: v_rcp
-; GCN:     s_cbranch_scc0 [[LOOP]]
-; GCN:     s_endpgm
 define amdgpu_kernel void @urem32_invariant_denom(i32 addrspace(1)* nocapture %arg, i32 %arg1) {
+; GFX9-LABEL: urem32_invariant_denom:
+; GFX9:       ; %bb.0: ; %bb
+; GFX9-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
+; GFX9-NEXT:    s_mov_b64 s[6:7], 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s2
+; GFX9-NEXT:    s_sub_i32 s3, 0, s2
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GFX9-NEXT:    v_mul_f32_e32 v0, 0x4f800000, v0
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GFX9-NEXT:    v_mul_lo_u32 v1, v0, s2
+; GFX9-NEXT:    v_mul_hi_u32 v2, v0, s2
+; GFX9-NEXT:    v_sub_u32_e32 v3, 0, v1
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX9-NEXT:    v_mul_hi_u32 v1, v1, v0
+; GFX9-NEXT:    v_add_u32_e32 v2, v0, v1
+; GFX9-NEXT:    v_sub_u32_e32 v0, v0, v1
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX9-NEXT:  BB1_1: ; %bb3
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    v_mul_lo_u32 v1, v0, s7
+; GFX9-NEXT:    v_mul_hi_u32 v2, v0, s6
+; GFX9-NEXT:    v_add_u32_e32 v3, v2, v1
+; GFX9-NEXT:    v_mul_lo_u32 v4, s3, v3
+; GFX9-NEXT:    v_mul_lo_u32 v6, v3, s2
+; GFX9-NEXT:    v_sub_u32_e32 v5, 1, v3
+; GFX9-NEXT:    v_not_b32_e32 v3, v3
+; GFX9-NEXT:    v_mul_lo_u32 v5, s2, v5
+; GFX9-NEXT:    v_mul_lo_u32 v3, s2, v3
+; GFX9-NEXT:    v_add_u32_e32 v4, s6, v4
+; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, s2, v4
+; GFX9-NEXT:    v_cmp_ge_u32_e64 s[0:1], s6, v6
+; GFX9-NEXT:    s_and_b64 vcc, vcc, s[0:1]
+; GFX9-NEXT:    v_add_u32_e32 v3, s6, v3
+; GFX9-NEXT:    v_add_u32_e32 v5, s6, v5
+; GFX9-NEXT:    s_add_u32 s6, s6, 1
+; GFX9-NEXT:    v_mov_b32_e32 v1, s4
+; GFX9-NEXT:    s_addc_u32 s7, s7, 0
+; GFX9-NEXT:    v_mov_b32_e32 v2, s5
+; GFX9-NEXT:    s_add_u32 s4, s4, 4
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
+; GFX9-NEXT:    s_addc_u32 s5, s5, 0
+; GFX9-NEXT:    v_cndmask_b32_e64 v3, v5, v3, s[0:1]
+; GFX9-NEXT:    s_cmpk_eq_i32 s6, 0x400
+; GFX9-NEXT:    global_store_dword v[1:2], v3, off
+; GFX9-NEXT:    s_cbranch_scc0 BB1_1
+; GFX9-NEXT:  ; %bb.2: ; %bb2
+; GFX9-NEXT:    s_endpgm
 bb:
   br label %bb3
 
@@ -72,25 +133,53 @@ bb3:                                              ; preds = %bb3, %bb
   br i1 %tmp8, label %bb2, label %bb3
 }
 
-; GCN-LABEL: {{^}}sdiv32_invariant_denom:
-; GCN:     v_cvt_f32_u32
-; GCN:     v_rcp_iflag_f32
-; GCN:     v_mul_f32_e32 v{{[0-9]+}}, 0x4f800000,
-; GCN:     v_cvt_u32_f32_e32
-; GCN-DAG: v_mul_hi_u32
-; GCN-DAG: v_mul_lo_u32
-; GCN-DAG: v_sub_i32_e32
-; GCN-DAG: v_cmp_eq_u32_e64
-; GCN-DAG: v_cndmask_b32_e64
-; GCN-DAG: v_mul_hi_u32
-; GCN-DAG: v_add_i32_e32
-; GCN-DAG: v_subrev_i32_e32
-; GCN-DAG: v_cndmask_b32_e64
-; GCN:     [[LOOP:BB[0-9_]+]]:
-; GCN-NOT: v_rcp
-; GCN:     s_cbranch_scc0 [[LOOP]]
-; GCN:     s_endpgm
 define amdgpu_kernel void @sdiv32_invariant_denom(i32 addrspace(1)* nocapture %arg, i32 %arg1) {
+; GFX9-LABEL: sdiv32_invariant_denom:
+; GFX9:       ; %bb.0: ; %bb
+; GFX9-NEXT:    s_load_dword s3, s[0:1], 0x2c
+; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
+; GFX9-NEXT:    s_mov_b32 s6, 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_ashr_i32 s2, s3, 31
+; GFX9-NEXT:    s_add_i32 s3, s3, s2
+; GFX9-NEXT:    s_xor_b32 s3, s3, s2
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s3
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GFX9-NEXT:    v_mul_f32_e32 v0, 0x4f800000, v0
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GFX9-NEXT:    v_mul_lo_u32 v1, v0, s3
+; GFX9-NEXT:    v_mul_hi_u32 v2, v0, s3
+; GFX9-NEXT:    v_sub_u32_e32 v3, 0, v1
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX9-NEXT:    v_mul_hi_u32 v1, v1, v0
+; GFX9-NEXT:    v_add_u32_e32 v2, v0, v1
+; GFX9-NEXT:    v_sub_u32_e32 v0, v0, v1
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX9-NEXT:  BB2_1: ; %bb3
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    v_mul_hi_u32 v3, v0, s6
+; GFX9-NEXT:    v_mov_b32_e32 v1, s4
+; GFX9-NEXT:    v_mov_b32_e32 v2, s5
+; GFX9-NEXT:    v_mul_lo_u32 v4, v3, s3
+; GFX9-NEXT:    v_add_u32_e32 v6, 1, v3
+; GFX9-NEXT:    v_add_u32_e32 v7, -1, v3
+; GFX9-NEXT:    v_sub_u32_e32 v5, s6, v4
+; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, s6, v4
+; GFX9-NEXT:    v_cmp_le_u32_e64 s[0:1], s3, v5
+; GFX9-NEXT:    s_and_b64 s[0:1], s[0:1], vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v3, v3, v6, s[0:1]
+; GFX9-NEXT:    s_add_i32 s6, s6, 1
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
+; GFX9-NEXT:    s_add_u32 s4, s4, 4
+; GFX9-NEXT:    v_xor_b32_e32 v3, s2, v3
+; GFX9-NEXT:    s_addc_u32 s5, s5, 0
+; GFX9-NEXT:    v_subrev_u32_e32 v3, s2, v3
+; GFX9-NEXT:    s_cmpk_eq_i32 s6, 0x400
+; GFX9-NEXT:    global_store_dword v[1:2], v3, off
+; GFX9-NEXT:    s_cbranch_scc0 BB2_1
+; GFX9-NEXT:  ; %bb.2: ; %bb2
+; GFX9-NEXT:    s_endpgm
 bb:
   br label %bb3
 
@@ -108,25 +197,51 @@ bb3:                                              ; preds = %bb3, %bb
   br i1 %tmp8, label %bb2, label %bb3
 }
 
-; GCN-LABEL: {{^}}srem32_invariant_denom:
-; GCN:     v_cvt_f32_u32
-; GCN:     v_rcp_iflag_f32
-; GCN:     v_mul_f32_e32 v{{[0-9]+}}, 0x4f800000,
-; GCN:     v_cvt_u32_f32_e32
-; GCN-DAG: v_mul_hi_u32
-; GCN-DAG: v_mul_lo_u32
-; GCN-DAG: v_sub_i32_e32
-; GCN-DAG: v_cmp_eq_u32_e64
-; GCN-DAG: v_cndmask_b32_e64
-; GCN-DAG: v_mul_hi_u32
-; GCN-DAG: v_add_i32_e32
-; GCN-DAG: v_subrev_i32_e32
-; GCN-DAG: v_cndmask_b32_e64
-; GCN:     [[LOOP:BB[0-9_]+]]:
-; GCN-NOT: v_rcp
-; GCN:     s_cbranch_scc0 [[LOOP]]
-; GCN:     s_endpgm
 define amdgpu_kernel void @srem32_invariant_denom(i32 addrspace(1)* nocapture %arg, i32 %arg1) {
+; GFX9-LABEL: srem32_invariant_denom:
+; GFX9:       ; %bb.0: ; %bb
+; GFX9-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_ashr_i32 s3, s2, 31
+; GFX9-NEXT:    s_add_i32 s2, s2, s3
+; GFX9-NEXT:    s_xor_b32 s2, s2, s3
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s2
+; GFX9-NEXT:    s_mov_b32 s3, 0
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GFX9-NEXT:    v_mul_f32_e32 v0, 0x4f800000, v0
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GFX9-NEXT:    v_mul_lo_u32 v1, v0, s2
+; GFX9-NEXT:    v_mul_hi_u32 v2, v0, s2
+; GFX9-NEXT:    v_sub_u32_e32 v3, 0, v1
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX9-NEXT:    v_mul_hi_u32 v1, v1, v0
+; GFX9-NEXT:    v_add_u32_e32 v2, v0, v1
+; GFX9-NEXT:    v_sub_u32_e32 v0, v0, v1
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX9-NEXT:  BB3_1: ; %bb3
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    v_mul_hi_u32 v1, v0, s3
+; GFX9-NEXT:    v_mul_lo_u32 v3, v1, s2
+; GFX9-NEXT:    v_mov_b32_e32 v1, s4
+; GFX9-NEXT:    v_mov_b32_e32 v2, s5
+; GFX9-NEXT:    v_sub_u32_e32 v4, s3, v3
+; GFX9-NEXT:    v_cmp_ge_u32_e64 s[0:1], s3, v3
+; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, s2, v4
+; GFX9-NEXT:    s_add_i32 s3, s3, 1
+; GFX9-NEXT:    s_and_b64 vcc, vcc, s[0:1]
+; GFX9-NEXT:    v_subrev_u32_e32 v3, s2, v4
+; GFX9-NEXT:    s_add_u32 s4, s4, 4
+; GFX9-NEXT:    s_addc_u32 s5, s5, 0
+; GFX9-NEXT:    v_add_u32_e32 v5, s2, v4
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v3, v5, v3, s[0:1]
+; GFX9-NEXT:    s_cmpk_eq_i32 s3, 0x400
+; GFX9-NEXT:    global_store_dword v[1:2], v3, off
+; GFX9-NEXT:    s_cbranch_scc0 BB3_1
+; GFX9-NEXT:  ; %bb.2: ; %bb2
+; GFX9-NEXT:    s_endpgm
 bb:
   br label %bb3
 
@@ -144,14 +259,40 @@ bb3:                                              ; preds = %bb3, %bb
   br i1 %tmp8, label %bb2, label %bb3
 }
 
-; GCN-LABEL: {{^}}udiv16_invariant_denom:
-; GCN:     v_cvt_f32_u32
-; GCN:     v_rcp_iflag_f32
-; GCN:     [[LOOP:BB[0-9_]+]]:
-; GCN-NOT: v_rcp
-; GCN:     s_cbranch_scc0 [[LOOP]]
-; GCN:     s_endpgm
 define amdgpu_kernel void @udiv16_invariant_denom(i16 addrspace(1)* nocapture %arg, i16 %arg1) {
+; GFX9-LABEL: udiv16_invariant_denom:
+; GFX9:       ; %bb.0: ; %bb
+; GFX9-NEXT:    s_load_dword s3, s[0:1], 0x2c
+; GFX9-NEXT:    s_mov_b32 s2, 0xffff
+; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
+; GFX9-NEXT:    v_mov_b32_e32 v3, 0
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_and_b32 s3, s2, s3
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s3
+; GFX9-NEXT:    s_movk_i32 s3, 0x400
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v1, v0
+; GFX9-NEXT:  BB4_1: ; %bb3
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    v_and_b32_e32 v2, s2, v4
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v7, v2
+; GFX9-NEXT:    v_lshlrev_b64 v[5:6], 1, v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v2, s5
+; GFX9-NEXT:    v_add_co_u32_e64 v5, s[0:1], s4, v5
+; GFX9-NEXT:    v_addc_co_u32_e64 v6, s[0:1], v2, v6, s[0:1]
+; GFX9-NEXT:    v_mul_f32_e32 v2, v7, v1
+; GFX9-NEXT:    v_trunc_f32_e32 v2, v2
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v8, v2
+; GFX9-NEXT:    v_add_u16_e32 v4, 1, v4
+; GFX9-NEXT:    v_mad_f32 v2, -v2, v0, v7
+; GFX9-NEXT:    v_cmp_ge_f32_e64 s[0:1], |v2|, v0
+; GFX9-NEXT:    v_cmp_eq_u16_e32 vcc, s3, v4
+; GFX9-NEXT:    v_addc_co_u32_e64 v2, s[0:1], 0, v8, s[0:1]
+; GFX9-NEXT:    s_and_b64 vcc, exec, vcc
+; GFX9-NEXT:    global_store_short v[5:6], v2, off
+; GFX9-NEXT:    s_cbranch_vccz BB4_1
+; GFX9-NEXT:  ; %bb.2: ; %bb2
+; GFX9-NEXT:    s_endpgm
 bb:
   br label %bb3
 
@@ -169,14 +310,42 @@ bb3:                                              ; preds = %bb3, %bb
   br i1 %tmp8, label %bb2, label %bb3
 }
 
-; GCN-LABEL: {{^}}urem16_invariant_denom:
-; GCN:     v_cvt_f32_u32
-; GCN:     v_rcp_iflag_f32
-; GCN:     [[LOOP:BB[0-9_]+]]:
-; GCN-NOT: v_rcp
-; GCN:     s_cbranch_scc0 [[LOOP]]
-; GCN:     s_endpgm
 define amdgpu_kernel void @urem16_invariant_denom(i16 addrspace(1)* nocapture %arg, i16 %arg1) {
+; GFX9-LABEL: urem16_invariant_denom:
+; GFX9:       ; %bb.0: ; %bb
+; GFX9-NEXT:    s_load_dword s3, s[0:1], 0x2c
+; GFX9-NEXT:    s_mov_b32 s2, 0xffff
+; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
+; GFX9-NEXT:    v_mov_b32_e32 v3, 0
+; GFX9-NEXT:    s_movk_i32 s6, 0x400
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_and_b32 s3, s2, s3
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s3
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v1, v0
+; GFX9-NEXT:  BB5_1: ; %bb3
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    v_and_b32_e32 v2, s2, v4
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v7, v2
+; GFX9-NEXT:    v_lshlrev_b64 v[5:6], 1, v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v8, s5
+; GFX9-NEXT:    v_add_co_u32_e64 v5, s[0:1], s4, v5
+; GFX9-NEXT:    v_addc_co_u32_e64 v6, s[0:1], v8, v6, s[0:1]
+; GFX9-NEXT:    v_mul_f32_e32 v8, v7, v1
+; GFX9-NEXT:    v_trunc_f32_e32 v8, v8
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v9, v8
+; GFX9-NEXT:    v_mad_f32 v7, -v8, v0, v7
+; GFX9-NEXT:    v_cmp_ge_f32_e64 s[0:1], |v7|, v0
+; GFX9-NEXT:    v_add_u16_e32 v4, 1, v4
+; GFX9-NEXT:    v_addc_co_u32_e64 v7, s[0:1], 0, v9, s[0:1]
+; GFX9-NEXT:    v_mul_lo_u32 v7, v7, s3
+; GFX9-NEXT:    v_cmp_eq_u16_e32 vcc, s6, v4
+; GFX9-NEXT:    s_and_b64 vcc, exec, vcc
+; GFX9-NEXT:    v_sub_u32_e32 v2, v2, v7
+; GFX9-NEXT:    global_store_short v[5:6], v2, off
+; GFX9-NEXT:    s_cbranch_vccz BB5_1
+; GFX9-NEXT:  ; %bb.2: ; %bb2
+; GFX9-NEXT:    s_endpgm
 bb:
   br label %bb3
 
@@ -194,16 +363,44 @@ bb3:                                              ; preds = %bb3, %bb
   br i1 %tmp8, label %bb2, label %bb3
 }
 
-; GCN-LABEL: {{^}}sdiv16_invariant_denom:
-; GCN-DAG: s_sext_i32_i16
-; GCN-DAG: v_and_b32_e32 v{{[0-9]+}}, 0x7fffffff
-; GCN-DAG: v_cvt_f32_i32
-; GCN-DAG: v_rcp_iflag_f32
-; GCN:     [[LOOP:BB[0-9_]+]]:
-; GCN-NOT: v_rcp
-; GCN:     s_cbranch_scc0 [[LOOP]]
-; GCN:     s_endpgm
 define amdgpu_kernel void @sdiv16_invariant_denom(i16 addrspace(1)* nocapture %arg, i16 %arg1) {
+; GFX9-LABEL: sdiv16_invariant_denom:
+; GFX9:       ; %bb.0: ; %bb
+; GFX9-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
+; GFX9-NEXT:    v_mov_b32_e32 v3, 0
+; GFX9-NEXT:    s_movk_i32 s3, 0x400
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_sext_i32_i16 s2, s2
+; GFX9-NEXT:    v_cvt_f32_i32_e32 v0, s2
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v1, v0
+; GFX9-NEXT:  BB6_1: ; %bb3
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    v_bfe_i32 v5, v4, 0, 16
+; GFX9-NEXT:    v_and_b32_e32 v2, 0xffff, v4
+; GFX9-NEXT:    v_cvt_f32_i32_e32 v9, v5
+; GFX9-NEXT:    v_xor_b32_e32 v8, s2, v5
+; GFX9-NEXT:    v_lshlrev_b64 v[5:6], 1, v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v7, s5
+; GFX9-NEXT:    v_add_co_u32_e64 v5, s[0:1], s4, v5
+; GFX9-NEXT:    v_addc_co_u32_e64 v6, s[0:1], v7, v6, s[0:1]
+; GFX9-NEXT:    v_mul_f32_e32 v7, v9, v1
+; GFX9-NEXT:    v_trunc_f32_e32 v7, v7
+; GFX9-NEXT:    v_ashrrev_i32_e32 v2, 30, v8
+; GFX9-NEXT:    v_cvt_i32_f32_e32 v8, v7
+; GFX9-NEXT:    v_mad_f32 v7, -v7, v0, v9
+; GFX9-NEXT:    v_add_u16_e32 v4, 1, v4
+; GFX9-NEXT:    v_or_b32_e32 v2, 1, v2
+; GFX9-NEXT:    v_cmp_ge_f32_e64 s[0:1], |v7|, |v0|
+; GFX9-NEXT:    v_cmp_eq_u16_e32 vcc, s3, v4
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, v2, s[0:1]
+; GFX9-NEXT:    v_add_u32_e32 v2, v8, v2
+; GFX9-NEXT:    s_and_b64 vcc, exec, vcc
+; GFX9-NEXT:    global_store_short v[5:6], v2, off
+; GFX9-NEXT:    s_cbranch_vccz BB6_1
+; GFX9-NEXT:  ; %bb.2: ; %bb2
+; GFX9-NEXT:    s_endpgm
 bb:
   br label %bb3
 
@@ -221,16 +418,46 @@ bb3:                                              ; preds = %bb3, %bb
   br i1 %tmp8, label %bb2, label %bb3
 }
 
-; GCN-LABEL: {{^}}srem16_invariant_denom:
-; GCN-DAG: s_sext_i32_i16
-; GCN-DAG: v_and_b32_e32 v{{[0-9]+}}, 0x7fffffff
-; GCN-DAG: v_cvt_f32_i32
-; GCN-DAG: v_rcp_iflag_f32
-; GCN:     [[LOOP:BB[0-9_]+]]:
-; GCN-NOT: v_rcp
-; GCN:     s_cbranch_scc0 [[LOOP]]
-; GCN:     s_endpgm
 define amdgpu_kernel void @srem16_invariant_denom(i16 addrspace(1)* nocapture %arg, i16 %arg1) {
+; GFX9-LABEL: srem16_invariant_denom:
+; GFX9:       ; %bb.0: ; %bb
+; GFX9-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
+; GFX9-NEXT:    v_mov_b32_e32 v3, 0
+; GFX9-NEXT:    s_movk_i32 s3, 0x400
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_sext_i32_i16 s2, s2
+; GFX9-NEXT:    v_cvt_f32_i32_e32 v0, s2
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v1, v0
+; GFX9-NEXT:  BB7_1: ; %bb3
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    v_bfe_i32 v7, v4, 0, 16
+; GFX9-NEXT:    v_and_b32_e32 v2, 0xffff, v4
+; GFX9-NEXT:    v_cvt_f32_i32_e32 v10, v7
+; GFX9-NEXT:    v_lshlrev_b64 v[5:6], 1, v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v8, s5
+; GFX9-NEXT:    v_add_co_u32_e64 v5, s[0:1], s4, v5
+; GFX9-NEXT:    v_addc_co_u32_e64 v6, s[0:1], v8, v6, s[0:1]
+; GFX9-NEXT:    v_mul_f32_e32 v8, v10, v1
+; GFX9-NEXT:    v_xor_b32_e32 v9, s2, v7
+; GFX9-NEXT:    v_trunc_f32_e32 v8, v8
+; GFX9-NEXT:    v_ashrrev_i32_e32 v2, 30, v9
+; GFX9-NEXT:    v_cvt_i32_f32_e32 v9, v8
+; GFX9-NEXT:    v_mad_f32 v8, -v8, v0, v10
+; GFX9-NEXT:    v_or_b32_e32 v2, 1, v2
+; GFX9-NEXT:    v_cmp_ge_f32_e64 s[0:1], |v8|, |v0|
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, v2, s[0:1]
+; GFX9-NEXT:    v_add_u32_e32 v2, v9, v2
+; GFX9-NEXT:    v_mul_lo_u32 v2, v2, s2
+; GFX9-NEXT:    v_add_u16_e32 v4, 1, v4
+; GFX9-NEXT:    v_cmp_eq_u16_e32 vcc, s3, v4
+; GFX9-NEXT:    s_and_b64 vcc, exec, vcc
+; GFX9-NEXT:    v_sub_u32_e32 v2, v7, v2
+; GFX9-NEXT:    global_store_short v[5:6], v2, off
+; GFX9-NEXT:    s_cbranch_vccz BB7_1
+; GFX9-NEXT:  ; %bb.2: ; %bb2
+; GFX9-NEXT:    s_endpgm
 bb:
   br label %bb3
 
diff --git a/llvm/test/CodeGen/AMDGPU/idot2.ll b/llvm/test/CodeGen/AMDGPU/idot2.ll
index 200851712..1c97e2a2e 100644
--- a/llvm/test/CodeGen/AMDGPU/idot2.ll
+++ b/llvm/test/CodeGen/AMDGPU/idot2.ll
@@ -20,15 +20,16 @@ define amdgpu_kernel void @udot2(<2 x i16> addrspace(1)* %src1,
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
-; GFX7-NEXT:    s_load_dword s6, s[0:1], 0x0
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_lshr_b32 s7, s4, 16
-; GFX7-NEXT:    s_lshr_b32 s9, s5, 16
+; GFX7-NEXT:    s_lshr_b32 s6, s4, 16
+; GFX7-NEXT:    s_lshr_b32 s7, s5, 16
 ; GFX7-NEXT:    s_and_b32 s4, s4, s8
-; GFX7-NEXT:    v_mov_b32_e32 v0, s7
-; GFX7-NEXT:    v_mov_b32_e32 v1, s6
-; GFX7-NEXT:    v_mad_u32_u24 v0, s9, v0, v1
 ; GFX7-NEXT:    s_and_b32 s5, s5, s8
+; GFX7-NEXT:    s_load_dword s8, s[0:1], 0x0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s6
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v1, s8
+; GFX7-NEXT:    v_mad_u32_u24 v0, s7, v0, v1
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s4
 ; GFX7-NEXT:    v_mad_u32_u24 v0, s5, v1, v0
 ; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
@@ -88,14 +89,14 @@ define amdgpu_kernel void @udot2(<2 x i16> addrspace(1)* %src1,
 ; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
-; GFX9-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
+; GFX9-DL-NEXT:    s_load_dword s3, s[0:1], 0x0
+; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
+; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s2
+; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-DL-NEXT:    v_dot2_u32_u16 v2, s4, v0, v1
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
-; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s2
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s4
-; GFX9-DL-NEXT:    v_dot2_u32_u16 v2, s3, v2, v3
 ; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-DL-NEXT:    s_endpgm
 ;
@@ -105,14 +106,14 @@ define amdgpu_kernel void @udot2(<2 x i16> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX10-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
-; GFX10-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
+; GFX10-DL-NEXT:    s_load_dword s2, s[0:1], 0x0
+; GFX10-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
+; GFX10-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
+; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-DL-NEXT:    v_dot2_u32_u16 v2, s4, s3, v0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
-; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    v_mov_b32_e32 v2, s4
-; GFX10-DL-NEXT:    v_dot2_u32_u16 v2, s3, s2, v2
 ; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX10-DL-NEXT:    s_endpgm
                                  <2 x i16> addrspace(1)* %src2,
@@ -154,17 +155,18 @@ define amdgpu_kernel void @udot2_MulMul(<2 x i16> addrspace(1)* %src1,
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
-; GFX7-NEXT:    s_load_dword s6, s[0:1], 0x0
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_lshr_b32 s7, s4, 16
+; GFX7-NEXT:    s_lshr_b32 s6, s4, 16
 ; GFX7-NEXT:    s_and_b32 s4, s4, s8
-; GFX7-NEXT:    s_lshr_b32 s9, s5, 16
-; GFX7-NEXT:    s_and_b32 s5, s5, s8
 ; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    s_load_dword s4, s[0:1], 0x0
+; GFX7-NEXT:    s_lshr_b32 s7, s5, 16
+; GFX7-NEXT:    s_and_b32 s5, s5, s8
 ; GFX7-NEXT:    v_mul_u32_u24_e32 v0, s5, v0
-; GFX7-NEXT:    v_mov_b32_e32 v1, s7
-; GFX7-NEXT:    v_mad_u32_u24 v0, s9, v1, v0
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, s6, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s6
+; GFX7-NEXT:    v_mad_u32_u24 v0, s7, v1, v0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, s4, v0
 ; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; GFX7-NEXT:    s_endpgm
 ;
@@ -250,16 +252,16 @@ define amdgpu_kernel void @udot2_MulMul(<2 x i16> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
 ; GFX10-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
 ; GFX10-DL-NEXT:    s_load_dword s5, s[0:1], 0x0
+; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-DL-NEXT:    s_and_b32 s6, s3, s2
+; GFX10-DL-NEXT:    s_and_b32 s2, s4, s2
+; GFX10-DL-NEXT:    s_lshr_b32 s3, s3, 16
+; GFX10-DL-NEXT:    s_lshr_b32 s4, s4, 16
+; GFX10-DL-NEXT:    v_mul_u32_u24_e64 v0, s2, s6
+; GFX10-DL-NEXT:    v_mad_u32_u24 v0, s4, s3, v0
+; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, s5, v0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
-; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_and_b32 s0, s3, s2
-; GFX10-DL-NEXT:    s_and_b32 s1, s4, s2
-; GFX10-DL-NEXT:    s_lshr_b32 s2, s3, 16
-; GFX10-DL-NEXT:    s_lshr_b32 s3, s4, 16
-; GFX10-DL-NEXT:    v_mul_u32_u24_e64 v2, s1, s0
-; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s3, s2, v2
-; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, s5, v2
 ; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX10-DL-NEXT:    s_endpgm
                                         <2 x i16> addrspace(1)* %src2,
@@ -318,18 +320,18 @@ define amdgpu_kernel void @idot2(<2 x i16> addrspace(1)* %src1,
 ; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
 ; GFX8-NEXT:    s_load_dword s3, s[6:7], 0x0
 ; GFX8-NEXT:    s_load_dword s4, s[0:1], 0x0
-; GFX8-NEXT:    v_mov_b32_e32 v0, s0
-; GFX8-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_sext_i32_i16 s0, s2
+; GFX8-NEXT:    s_sext_i32_i16 s5, s2
 ; GFX8-NEXT:    s_ashr_i32 s2, s2, 16
-; GFX8-NEXT:    s_sext_i32_i16 s1, s3
+; GFX8-NEXT:    s_sext_i32_i16 s6, s3
 ; GFX8-NEXT:    s_ashr_i32 s3, s3, 16
-; GFX8-NEXT:    v_mov_b32_e32 v2, s4
-; GFX8-NEXT:    v_mov_b32_e32 v3, s2
-; GFX8-NEXT:    v_mad_i32_i24 v2, s3, v3, v2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s0
-; GFX8-NEXT:    v_mad_i32_i24 v2, s1, v3, v2
+; GFX8-NEXT:    v_mov_b32_e32 v0, s4
+; GFX8-NEXT:    v_mov_b32_e32 v1, s2
+; GFX8-NEXT:    v_mov_b32_e32 v2, s5
+; GFX8-NEXT:    v_mad_i32_i24 v0, s3, v1, v0
+; GFX8-NEXT:    v_mad_i32_i24 v2, s6, v2, v0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX8-NEXT:    flat_store_dword v[0:1], v2
 ; GFX8-NEXT:    s_endpgm
 ;
@@ -341,18 +343,18 @@ define amdgpu_kernel void @idot2(<2 x i16> addrspace(1)* %src1,
 ; GFX9-NODL-NEXT:    s_load_dword s2, s[4:5], 0x0
 ; GFX9-NODL-NEXT:    s_load_dword s3, s[6:7], 0x0
 ; GFX9-NODL-NEXT:    s_load_dword s4, s[0:1], 0x0
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT:    s_sext_i32_i16 s0, s2
+; GFX9-NODL-NEXT:    s_sext_i32_i16 s5, s2
 ; GFX9-NODL-NEXT:    s_ashr_i32 s2, s2, 16
-; GFX9-NODL-NEXT:    s_sext_i32_i16 s1, s3
+; GFX9-NODL-NEXT:    s_sext_i32_i16 s6, s3
 ; GFX9-NODL-NEXT:    s_ashr_i32 s3, s3, 16
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s4
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s2
-; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s3, v3, v2
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s0
-; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s1, v3, v2
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s2
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s5
+; GFX9-NODL-NEXT:    v_mad_i32_i24 v0, s3, v1, v0
+; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s6, v2, v0
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-NODL-NEXT:    s_endpgm
 ;
@@ -362,14 +364,14 @@ define amdgpu_kernel void @idot2(<2 x i16> addrspace(1)* %src1,
 ; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
-; GFX9-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
+; GFX9-DL-NEXT:    s_load_dword s3, s[0:1], 0x0
+; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
+; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s2
+; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-DL-NEXT:    v_dot2_i32_i16 v2, s4, v0, v1
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
-; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s2
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s4
-; GFX9-DL-NEXT:    v_dot2_i32_i16 v2, s3, v2, v3
 ; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-DL-NEXT:    s_endpgm
 ;
@@ -379,14 +381,14 @@ define amdgpu_kernel void @idot2(<2 x i16> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX10-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
-; GFX10-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
+; GFX10-DL-NEXT:    s_load_dword s2, s[0:1], 0x0
+; GFX10-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
+; GFX10-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
+; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-DL-NEXT:    v_dot2_i32_i16 v2, s4, s3, v0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
-; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    v_mov_b32_e32 v2, s4
-; GFX10-DL-NEXT:    v_dot2_i32_i16 v2, s3, s2, v2
 ; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX10-DL-NEXT:    s_endpgm
                                  <2 x i16> addrspace(1)* %src2,
@@ -446,18 +448,18 @@ define amdgpu_kernel void @idot2_MixedTypedMul(<2 x i16> addrspace(1)* %src1,
 ; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
 ; GFX8-NEXT:    s_load_dword s3, s[6:7], 0x0
 ; GFX8-NEXT:    s_load_dword s4, s[0:1], 0x0
-; GFX8-NEXT:    v_mov_b32_e32 v0, s0
-; GFX8-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_sext_i32_i16 s0, s2
+; GFX8-NEXT:    s_sext_i32_i16 s5, s2
 ; GFX8-NEXT:    s_lshr_b32 s2, s2, 16
-; GFX8-NEXT:    s_sext_i32_i16 s1, s3
+; GFX8-NEXT:    s_sext_i32_i16 s6, s3
 ; GFX8-NEXT:    s_lshr_b32 s3, s3, 16
-; GFX8-NEXT:    v_mov_b32_e32 v2, s4
-; GFX8-NEXT:    v_mov_b32_e32 v3, s2
-; GFX8-NEXT:    v_mad_u32_u24 v2, s3, v3, v2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s0
-; GFX8-NEXT:    v_mad_i32_i24 v2, s1, v3, v2
+; GFX8-NEXT:    v_mov_b32_e32 v0, s4
+; GFX8-NEXT:    v_mov_b32_e32 v1, s2
+; GFX8-NEXT:    v_mov_b32_e32 v2, s5
+; GFX8-NEXT:    v_mad_u32_u24 v0, s3, v1, v0
+; GFX8-NEXT:    v_mad_i32_i24 v2, s6, v2, v0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX8-NEXT:    flat_store_dword v[0:1], v2
 ; GFX8-NEXT:    s_endpgm
 ;
@@ -469,18 +471,18 @@ define amdgpu_kernel void @idot2_MixedTypedMul(<2 x i16> addrspace(1)* %src1,
 ; GFX9-NODL-NEXT:    s_load_dword s2, s[4:5], 0x0
 ; GFX9-NODL-NEXT:    s_load_dword s3, s[6:7], 0x0
 ; GFX9-NODL-NEXT:    s_load_dword s4, s[0:1], 0x0
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT:    s_sext_i32_i16 s0, s2
+; GFX9-NODL-NEXT:    s_sext_i32_i16 s5, s2
 ; GFX9-NODL-NEXT:    s_lshr_b32 s2, s2, 16
-; GFX9-NODL-NEXT:    s_sext_i32_i16 s1, s3
+; GFX9-NODL-NEXT:    s_sext_i32_i16 s6, s3
 ; GFX9-NODL-NEXT:    s_lshr_b32 s3, s3, 16
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s4
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s2
-; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s3, v3, v2
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s0
-; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s1, v3, v2
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s2
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s5
+; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s3, v1, v0
+; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s6, v2, v0
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-NODL-NEXT:    s_endpgm
 ;
@@ -492,18 +494,18 @@ define amdgpu_kernel void @idot2_MixedTypedMul(<2 x i16> addrspace(1)* %src1,
 ; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
 ; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
 ; GFX9-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
-; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    s_sext_i32_i16 s0, s2
+; GFX9-DL-NEXT:    s_sext_i32_i16 s5, s2
 ; GFX9-DL-NEXT:    s_lshr_b32 s2, s2, 16
-; GFX9-DL-NEXT:    s_sext_i32_i16 s1, s3
+; GFX9-DL-NEXT:    s_sext_i32_i16 s6, s3
 ; GFX9-DL-NEXT:    s_lshr_b32 s3, s3, 16
-; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s4
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s2
-; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s3, v3, v2
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s0
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s1, v3, v2
+; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s2
+; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s5
+; GFX9-DL-NEXT:    v_mad_u32_u24 v0, s3, v1, v0
+; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s6, v2, v0
+; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-DL-NEXT:    s_endpgm
 ;
@@ -516,16 +518,16 @@ define amdgpu_kernel void @idot2_MixedTypedMul(<2 x i16> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
 ; GFX10-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
 ; GFX10-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
-; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_lshr_b32 s0, s2, 16
-; GFX10-DL-NEXT:    s_lshr_b32 s1, s3, 16
-; GFX10-DL-NEXT:    v_mov_b32_e32 v2, s4
+; GFX10-DL-NEXT:    s_lshr_b32 s5, s2, 16
+; GFX10-DL-NEXT:    s_lshr_b32 s6, s3, 16
+; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s4
 ; GFX10-DL-NEXT:    s_sext_i32_i16 s2, s2
 ; GFX10-DL-NEXT:    s_sext_i32_i16 s3, s3
-; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s1, s0, v2
-; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s3, s2, v2
+; GFX10-DL-NEXT:    v_mad_u32_u24 v0, s6, s5, v0
+; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s3, s2, v0
+; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX10-DL-NEXT:    s_endpgm
                                                <2 x i16> addrspace(1)* %src2,
@@ -564,15 +566,16 @@ define amdgpu_kernel void @udot2_alt_AddOperands(<2 x i16> addrspace(1)* %src1,
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
-; GFX7-NEXT:    s_load_dword s6, s[0:1], 0x0
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_lshr_b32 s7, s4, 16
-; GFX7-NEXT:    s_lshr_b32 s9, s5, 16
+; GFX7-NEXT:    s_lshr_b32 s6, s4, 16
+; GFX7-NEXT:    s_lshr_b32 s7, s5, 16
 ; GFX7-NEXT:    s_and_b32 s4, s4, s8
-; GFX7-NEXT:    v_mov_b32_e32 v0, s7
-; GFX7-NEXT:    v_mov_b32_e32 v1, s6
-; GFX7-NEXT:    v_mad_u32_u24 v0, s9, v0, v1
 ; GFX7-NEXT:    s_and_b32 s5, s5, s8
+; GFX7-NEXT:    s_load_dword s8, s[0:1], 0x0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s6
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v1, s8
+; GFX7-NEXT:    v_mad_u32_u24 v0, s7, v0, v1
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s4
 ; GFX7-NEXT:    v_mad_u32_u24 v0, s5, v1, v0
 ; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
@@ -632,14 +635,14 @@ define amdgpu_kernel void @udot2_alt_AddOperands(<2 x i16> addrspace(1)* %src1,
 ; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
-; GFX9-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
+; GFX9-DL-NEXT:    s_load_dword s3, s[0:1], 0x0
+; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
+; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s2
+; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-DL-NEXT:    v_dot2_u32_u16 v2, s4, v0, v1
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
-; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s2
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s4
-; GFX9-DL-NEXT:    v_dot2_u32_u16 v2, s3, v2, v3
 ; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-DL-NEXT:    s_endpgm
 ;
@@ -649,14 +652,14 @@ define amdgpu_kernel void @udot2_alt_AddOperands(<2 x i16> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX10-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
-; GFX10-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
+; GFX10-DL-NEXT:    s_load_dword s2, s[0:1], 0x0
+; GFX10-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
+; GFX10-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
+; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-DL-NEXT:    v_dot2_u32_u16 v2, s4, s3, v0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
-; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    v_mov_b32_e32 v2, s4
-; GFX10-DL-NEXT:    v_dot2_u32_u16 v2, s3, s2, v2
 ; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX10-DL-NEXT:    s_endpgm
                                                  <2 x i16> addrspace(1)* %src2,
@@ -716,18 +719,18 @@ define amdgpu_kernel void @idot2_MixedExt(<2 x i16> addrspace(1)* %src1,
 ; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
 ; GFX8-NEXT:    s_load_dword s3, s[6:7], 0x0
 ; GFX8-NEXT:    s_load_dword s4, s[0:1], 0x0
-; GFX8-NEXT:    v_mov_b32_e32 v0, s0
-; GFX8-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_sext_i32_i16 s0, s2
+; GFX8-NEXT:    s_sext_i32_i16 s5, s2
 ; GFX8-NEXT:    s_ashr_i32 s2, s2, 16
-; GFX8-NEXT:    s_and_b32 s1, s3, 0xffff
+; GFX8-NEXT:    s_and_b32 s6, s3, 0xffff
 ; GFX8-NEXT:    s_ashr_i32 s3, s3, 16
-; GFX8-NEXT:    v_mov_b32_e32 v2, s4
-; GFX8-NEXT:    v_mov_b32_e32 v3, s2
-; GFX8-NEXT:    v_mad_i32_i24 v2, s3, v3, v2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s0
-; GFX8-NEXT:    v_mad_i32_i24 v2, s1, v3, v2
+; GFX8-NEXT:    v_mov_b32_e32 v0, s4
+; GFX8-NEXT:    v_mov_b32_e32 v1, s2
+; GFX8-NEXT:    v_mov_b32_e32 v2, s5
+; GFX8-NEXT:    v_mad_i32_i24 v0, s3, v1, v0
+; GFX8-NEXT:    v_mad_i32_i24 v2, s6, v2, v0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX8-NEXT:    flat_store_dword v[0:1], v2
 ; GFX8-NEXT:    s_endpgm
 ;
@@ -739,18 +742,18 @@ define amdgpu_kernel void @idot2_MixedExt(<2 x i16> addrspace(1)* %src1,
 ; GFX9-NODL-NEXT:    s_load_dword s2, s[4:5], 0x0
 ; GFX9-NODL-NEXT:    s_load_dword s3, s[6:7], 0x0
 ; GFX9-NODL-NEXT:    s_load_dword s4, s[0:1], 0x0
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT:    s_sext_i32_i16 s0, s2
+; GFX9-NODL-NEXT:    s_sext_i32_i16 s5, s2
 ; GFX9-NODL-NEXT:    s_ashr_i32 s2, s2, 16
-; GFX9-NODL-NEXT:    s_and_b32 s1, s3, 0xffff
+; GFX9-NODL-NEXT:    s_and_b32 s6, s3, 0xffff
 ; GFX9-NODL-NEXT:    s_ashr_i32 s3, s3, 16
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s4
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s2
-; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s3, v3, v2
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s0
-; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s1, v3, v2
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s2
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s5
+; GFX9-NODL-NEXT:    v_mad_i32_i24 v0, s3, v1, v0
+; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s6, v2, v0
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-NODL-NEXT:    s_endpgm
 ;
@@ -762,18 +765,18 @@ define amdgpu_kernel void @idot2_MixedExt(<2 x i16> addrspace(1)* %src1,
 ; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
 ; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
 ; GFX9-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
-; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    s_sext_i32_i16 s0, s2
+; GFX9-DL-NEXT:    s_sext_i32_i16 s5, s2
 ; GFX9-DL-NEXT:    s_ashr_i32 s2, s2, 16
-; GFX9-DL-NEXT:    s_and_b32 s1, s3, 0xffff
+; GFX9-DL-NEXT:    s_and_b32 s6, s3, 0xffff
 ; GFX9-DL-NEXT:    s_ashr_i32 s3, s3, 16
-; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s4
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s2
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s3, v3, v2
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s0
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s1, v3, v2
+; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s2
+; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s5
+; GFX9-DL-NEXT:    v_mad_i32_i24 v0, s3, v1, v0
+; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s6, v2, v0
+; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-DL-NEXT:    s_endpgm
 ;
@@ -786,16 +789,16 @@ define amdgpu_kernel void @idot2_MixedExt(<2 x i16> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
 ; GFX10-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
 ; GFX10-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
-; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_ashr_i32 s0, s2, 16
-; GFX10-DL-NEXT:    s_ashr_i32 s1, s3, 16
-; GFX10-DL-NEXT:    v_mov_b32_e32 v2, s4
+; GFX10-DL-NEXT:    s_ashr_i32 s5, s2, 16
+; GFX10-DL-NEXT:    s_ashr_i32 s6, s3, 16
+; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s4
 ; GFX10-DL-NEXT:    s_sext_i32_i16 s2, s2
 ; GFX10-DL-NEXT:    s_and_b32 s3, s3, 0xffff
-; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s1, s0, v2
-; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s3, s2, v2
+; GFX10-DL-NEXT:    v_mad_i32_i24 v0, s6, s5, v0
+; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s3, s2, v0
+; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX10-DL-NEXT:    s_endpgm
                                           <2 x i16> addrspace(1)* %src2,
@@ -831,13 +834,13 @@ define amdgpu_kernel void @notudot2_SameVec(<2 x i16> addrspace(1)* %src1,
 ; GFX7-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX7-NEXT:    s_mov_b32 s2, -1
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_load_dword s6, s[6:7], 0x0
+; GFX7-NEXT:    s_load_dword s7, s[0:1], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
-; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
-; GFX7-NEXT:    s_load_dword s6, s[0:1], 0x0
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_lshr_b32 s5, s6, 16
+; GFX7-NEXT:    v_mov_b32_e32 v0, s7
 ; GFX7-NEXT:    s_and_b32 s4, s4, 0xffff
-; GFX7-NEXT:    s_lshr_b32 s5, s5, 16
-; GFX7-NEXT:    v_mov_b32_e32 v0, s6
 ; GFX7-NEXT:    v_mad_u32_u24 v0, s5, s5, v0
 ; GFX7-NEXT:    v_mad_u32_u24 v0, s4, s4, v0
 ; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
@@ -848,17 +851,17 @@ define amdgpu_kernel void @notudot2_SameVec(<2 x i16> addrspace(1)* %src1,
 ; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX8-NEXT:    s_load_dword s3, s[6:7], 0x0
-; GFX8-NEXT:    s_load_dword s4, s[0:1], 0x0
+; GFX8-NEXT:    s_load_dword s2, s[6:7], 0x0
+; GFX8-NEXT:    s_load_dword s3, s[0:1], 0x0
+; GFX8-NEXT:    s_load_dword s4, s[4:5], 0x0
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_and_b32 s0, s2, 0xffff
-; GFX8-NEXT:    s_lshr_b32 s1, s3, 16
-; GFX8-NEXT:    v_mov_b32_e32 v2, s4
-; GFX8-NEXT:    v_mad_u32_u24 v2, s1, s1, v2
+; GFX8-NEXT:    s_lshr_b32 s0, s2, 16
+; GFX8-NEXT:    v_mov_b32_e32 v2, s3
+; GFX8-NEXT:    s_and_b32 s1, s4, 0xffff
 ; GFX8-NEXT:    v_mad_u32_u24 v2, s0, s0, v2
+; GFX8-NEXT:    v_mad_u32_u24 v2, s1, s1, v2
 ; GFX8-NEXT:    flat_store_dword v[0:1], v2
 ; GFX8-NEXT:    s_endpgm
 ;
@@ -867,17 +870,17 @@ define amdgpu_kernel void @notudot2_SameVec(<2 x i16> addrspace(1)* %src1,
 ; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX9-NODL-NEXT:    s_load_dword s3, s[6:7], 0x0
-; GFX9-NODL-NEXT:    s_load_dword s4, s[0:1], 0x0
+; GFX9-NODL-NEXT:    s_load_dword s2, s[6:7], 0x0
+; GFX9-NODL-NEXT:    s_load_dword s3, s[0:1], 0x0
+; GFX9-NODL-NEXT:    s_load_dword s4, s[4:5], 0x0
 ; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT:    s_and_b32 s0, s2, 0xffff
-; GFX9-NODL-NEXT:    s_lshr_b32 s1, s3, 16
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s4
-; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s1, s1, v2
+; GFX9-NODL-NEXT:    s_lshr_b32 s0, s2, 16
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s3
+; GFX9-NODL-NEXT:    s_and_b32 s1, s4, 0xffff
 ; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s0, s0, v2
+; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s1, s1, v2
 ; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-NODL-NEXT:    s_endpgm
 ;
@@ -886,17 +889,17 @@ define amdgpu_kernel void @notudot2_SameVec(<2 x i16> addrspace(1)* %src1,
 ; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
-; GFX9-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
+; GFX9-DL-NEXT:    s_load_dword s2, s[6:7], 0x0
+; GFX9-DL-NEXT:    s_load_dword s3, s[0:1], 0x0
+; GFX9-DL-NEXT:    s_load_dword s4, s[4:5], 0x0
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    s_and_b32 s0, s2, 0xffff
-; GFX9-DL-NEXT:    s_lshr_b32 s1, s3, 16
-; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s4
-; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s1, s1, v2
+; GFX9-DL-NEXT:    s_lshr_b32 s0, s2, 16
+; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s3
+; GFX9-DL-NEXT:    s_and_b32 s1, s4, 0xffff
 ; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s0, s0, v2
+; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s1, s1, v2
 ; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-DL-NEXT:    s_endpgm
 ;
@@ -906,16 +909,16 @@ define amdgpu_kernel void @notudot2_SameVec(<2 x i16> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX10-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
+; GFX10-DL-NEXT:    s_load_dword s2, s[6:7], 0x0
+; GFX10-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
 ; GFX10-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
+; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-DL-NEXT:    s_lshr_b32 s2, s2, 16
+; GFX10-DL-NEXT:    s_and_b32 s3, s3, 0xffff
+; GFX10-DL-NEXT:    v_mad_u32_u24 v0, s2, s2, s4
+; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s3, s3, v0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
-; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_and_b32 s0, s2, 0xffff
-; GFX10-DL-NEXT:    s_lshr_b32 s1, s3, 16
-; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s1, s1, s4
-; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s0, s0, v2
 ; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX10-DL-NEXT:    s_endpgm
                                             <2 x i16> addrspace(1)* %src2,
@@ -954,17 +957,18 @@ define amdgpu_kernel void @udot2_v4i16(<4 x i16> addrspace(1)* %src1,
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
-; GFX7-NEXT:    s_load_dword s6, s[0:1], 0x0
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_and_b32 s7, s4, s8
+; GFX7-NEXT:    s_and_b32 s6, s4, s8
+; GFX7-NEXT:    s_and_b32 s7, s5, s8
+; GFX7-NEXT:    s_load_dword s8, s[0:1], 0x0
 ; GFX7-NEXT:    s_lshr_b32 s4, s4, 16
-; GFX7-NEXT:    s_and_b32 s8, s5, s8
 ; GFX7-NEXT:    s_lshr_b32 s5, s5, 16
 ; GFX7-NEXT:    v_mov_b32_e32 v0, s4
-; GFX7-NEXT:    v_mov_b32_e32 v1, s6
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v1, s8
 ; GFX7-NEXT:    v_mad_u32_u24 v0, s5, v0, v1
-; GFX7-NEXT:    v_mov_b32_e32 v1, s7
-; GFX7-NEXT:    v_mad_u32_u24 v0, s8, v1, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s6
+; GFX7-NEXT:    v_mad_u32_u24 v0, s7, v1, v0
 ; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; GFX7-NEXT:    s_endpgm
 ;
@@ -1022,14 +1026,14 @@ define amdgpu_kernel void @udot2_v4i16(<4 x i16> addrspace(1)* %src1,
 ; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
-; GFX9-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
+; GFX9-DL-NEXT:    s_load_dword s3, s[0:1], 0x0
+; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
+; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s2
+; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-DL-NEXT:    v_dot2_u32_u16 v2, s4, v0, v1
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
-; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s2
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s4
-; GFX9-DL-NEXT:    v_dot2_u32_u16 v2, s3, v2, v3
 ; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-DL-NEXT:    s_endpgm
 ;
@@ -1039,14 +1043,14 @@ define amdgpu_kernel void @udot2_v4i16(<4 x i16> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX10-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
-; GFX10-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
+; GFX10-DL-NEXT:    s_load_dword s2, s[0:1], 0x0
+; GFX10-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
+; GFX10-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
+; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-DL-NEXT:    v_dot2_u32_u16 v2, s4, s3, v0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
-; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    v_mov_b32_e32 v2, s4
-; GFX10-DL-NEXT:    v_dot2_u32_u16 v2, s3, s2, v2
 ; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX10-DL-NEXT:    s_endpgm
                                        <4 x i16> addrspace(1)* %src2,
@@ -1085,17 +1089,18 @@ define amdgpu_kernel void @udot2_v4i16_Hi(<4 x i16> addrspace(1)* %src1,
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x1
 ; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x1
-; GFX7-NEXT:    s_load_dword s6, s[0:1], 0x0
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_and_b32 s7, s4, s8
+; GFX7-NEXT:    s_and_b32 s6, s4, s8
+; GFX7-NEXT:    s_and_b32 s7, s5, s8
+; GFX7-NEXT:    s_load_dword s8, s[0:1], 0x0
 ; GFX7-NEXT:    s_lshr_b32 s4, s4, 16
-; GFX7-NEXT:    s_and_b32 s8, s5, s8
 ; GFX7-NEXT:    s_lshr_b32 s5, s5, 16
 ; GFX7-NEXT:    v_mov_b32_e32 v0, s4
-; GFX7-NEXT:    v_mov_b32_e32 v1, s6
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v1, s8
 ; GFX7-NEXT:    v_mad_u32_u24 v0, s5, v0, v1
-; GFX7-NEXT:    v_mov_b32_e32 v1, s7
-; GFX7-NEXT:    v_mad_u32_u24 v0, s8, v1, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s6
+; GFX7-NEXT:    v_mad_u32_u24 v0, s7, v1, v0
 ; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; GFX7-NEXT:    s_endpgm
 ;
@@ -1153,14 +1158,14 @@ define amdgpu_kernel void @udot2_v4i16_Hi(<4 x i16> addrspace(1)* %src1,
 ; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x4
-; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x4
-; GFX9-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
+; GFX9-DL-NEXT:    s_load_dword s3, s[0:1], 0x0
+; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x4
+; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s2
+; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-DL-NEXT:    v_dot2_u32_u16 v2, s4, v0, v1
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
-; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s2
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s4
-; GFX9-DL-NEXT:    v_dot2_u32_u16 v2, s3, v2, v3
 ; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-DL-NEXT:    s_endpgm
 ;
@@ -1170,14 +1175,14 @@ define amdgpu_kernel void @udot2_v4i16_Hi(<4 x i16> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_load_dword s2, s[4:5], 0x4
-; GFX10-DL-NEXT:    s_load_dword s3, s[6:7], 0x4
-; GFX10-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
+; GFX10-DL-NEXT:    s_load_dword s2, s[0:1], 0x0
+; GFX10-DL-NEXT:    s_load_dword s3, s[4:5], 0x4
+; GFX10-DL-NEXT:    s_load_dword s4, s[6:7], 0x4
+; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-DL-NEXT:    v_dot2_u32_u16 v2, s4, s3, v0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
-; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    v_mov_b32_e32 v2, s4
-; GFX10-DL-NEXT:    v_dot2_u32_u16 v2, s3, s2, v2
 ; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX10-DL-NEXT:    s_endpgm
                                           <4 x i16> addrspace(1)* %src2,
@@ -1216,15 +1221,16 @@ define amdgpu_kernel void @notudot2_v4i16_Even(<4 x i16> addrspace(1)* %src1,
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[6:7], 0x0
-; GFX7-NEXT:    s_load_dword s9, s[0:1], 0x0
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    s_and_b32 s5, s5, s8
 ; GFX7-NEXT:    s_and_b32 s4, s4, s8
+; GFX7-NEXT:    s_and_b32 s6, s6, s8
 ; GFX7-NEXT:    s_and_b32 s7, s7, s8
+; GFX7-NEXT:    s_load_dword s8, s[0:1], 0x0
 ; GFX7-NEXT:    v_mov_b32_e32 v0, s5
-; GFX7-NEXT:    v_mov_b32_e32 v1, s9
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v1, s8
 ; GFX7-NEXT:    v_mad_u32_u24 v0, s7, v0, v1
-; GFX7-NEXT:    s_and_b32 s6, s6, s8
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s4
 ; GFX7-NEXT:    v_mad_u32_u24 v0, s6, v1, v0
 ; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
@@ -1312,16 +1318,16 @@ define amdgpu_kernel void @notudot2_v4i16_Even(<4 x i16> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-DL-NEXT:    s_load_dword s6, s[0:1], 0x0
-; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_and_b32 s0, s3, s8
-; GFX10-DL-NEXT:    s_and_b32 s1, s5, s8
-; GFX10-DL-NEXT:    v_mov_b32_e32 v2, s6
+; GFX10-DL-NEXT:    s_and_b32 s3, s3, s8
+; GFX10-DL-NEXT:    s_and_b32 s5, s5, s8
+; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s6
 ; GFX10-DL-NEXT:    s_and_b32 s2, s2, s8
-; GFX10-DL-NEXT:    s_and_b32 s3, s4, s8
-; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s1, s0, v2
-; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s3, s2, v2
+; GFX10-DL-NEXT:    s_and_b32 s4, s4, s8
+; GFX10-DL-NEXT:    v_mad_u32_u24 v0, s5, s3, v0
+; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s4, s2, v0
+; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX10-DL-NEXT:    s_endpgm
                                                <4 x i16> addrspace(1)* %src2,
@@ -1360,15 +1366,16 @@ define amdgpu_kernel void @notudot2_v4i16_Middle(<4 x i16> addrspace(1)* %src1,
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[6:7], 0x0
-; GFX7-NEXT:    s_load_dword s9, s[0:1], 0x0
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    s_and_b32 s5, s5, s8
-; GFX7-NEXT:    s_lshr_b32 s4, s4, 16
 ; GFX7-NEXT:    s_and_b32 s7, s7, s8
+; GFX7-NEXT:    s_load_dword s8, s[0:1], 0x0
+; GFX7-NEXT:    s_lshr_b32 s4, s4, 16
 ; GFX7-NEXT:    v_mov_b32_e32 v0, s5
-; GFX7-NEXT:    v_mov_b32_e32 v1, s9
-; GFX7-NEXT:    v_mad_u32_u24 v0, s7, v0, v1
 ; GFX7-NEXT:    s_lshr_b32 s6, s6, 16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v1, s8
+; GFX7-NEXT:    v_mad_u32_u24 v0, s7, v0, v1
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s4
 ; GFX7-NEXT:    v_mad_u32_u24 v0, s6, v1, v0
 ; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
@@ -1456,16 +1463,16 @@ define amdgpu_kernel void @notudot2_v4i16_Middle(<4 x i16> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-DL-NEXT:    s_load_dword s6, s[0:1], 0x0
-; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_and_b32 s0, s3, s8
-; GFX10-DL-NEXT:    s_and_b32 s1, s5, s8
-; GFX10-DL-NEXT:    v_mov_b32_e32 v2, s6
+; GFX10-DL-NEXT:    s_and_b32 s3, s3, s8
+; GFX10-DL-NEXT:    s_and_b32 s5, s5, s8
+; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s6
 ; GFX10-DL-NEXT:    s_lshr_b32 s2, s2, 16
-; GFX10-DL-NEXT:    s_lshr_b32 s3, s4, 16
-; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s1, s0, v2
-; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s3, s2, v2
+; GFX10-DL-NEXT:    s_lshr_b32 s4, s4, 16
+; GFX10-DL-NEXT:    v_mad_u32_u24 v0, s5, s3, v0
+; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s4, s2, v0
+; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX10-DL-NEXT:    s_endpgm
                                                  <4 x i16> addrspace(1)* %src2,
@@ -1504,17 +1511,18 @@ define amdgpu_kernel void @notudot2_DiffIndex(<2 x i16> addrspace(1)* %src1,
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
-; GFX7-NEXT:    s_load_dword s6, s[0:1], 0x0
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_lshr_b32 s7, s4, 16
-; GFX7-NEXT:    s_lshr_b32 s9, s5, 16
+; GFX7-NEXT:    s_lshr_b32 s6, s4, 16
+; GFX7-NEXT:    s_lshr_b32 s7, s5, 16
 ; GFX7-NEXT:    s_and_b32 s4, s4, s8
 ; GFX7-NEXT:    s_and_b32 s5, s5, s8
-; GFX7-NEXT:    v_mov_b32_e32 v0, s7
-; GFX7-NEXT:    v_mov_b32_e32 v1, s6
+; GFX7-NEXT:    s_load_dword s8, s[0:1], 0x0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s6
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v1, s8
 ; GFX7-NEXT:    v_mad_u32_u24 v0, s5, v0, v1
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s4
-; GFX7-NEXT:    v_mad_u32_u24 v0, s9, v1, v0
+; GFX7-NEXT:    v_mad_u32_u24 v0, s7, v1, v0
 ; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; GFX7-NEXT:    s_endpgm
 ;
@@ -1600,16 +1608,16 @@ define amdgpu_kernel void @notudot2_DiffIndex(<2 x i16> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
 ; GFX10-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
 ; GFX10-DL-NEXT:    s_load_dword s5, s[0:1], 0x0
-; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_lshr_b32 s0, s3, 16
-; GFX10-DL-NEXT:    s_and_b32 s1, s4, s2
-; GFX10-DL-NEXT:    v_mov_b32_e32 v2, s5
+; GFX10-DL-NEXT:    s_lshr_b32 s6, s3, 16
+; GFX10-DL-NEXT:    s_and_b32 s7, s4, s2
+; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s5
 ; GFX10-DL-NEXT:    s_and_b32 s2, s3, s2
 ; GFX10-DL-NEXT:    s_lshr_b32 s3, s4, 16
-; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s1, s0, v2
-; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s3, s2, v2
+; GFX10-DL-NEXT:    v_mad_u32_u24 v0, s7, s6, v0
+; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s3, s2, v0
+; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX10-DL-NEXT:    s_endpgm
                                               <2 x i16> addrspace(1)* %src2,
@@ -1648,15 +1656,16 @@ define amdgpu_kernel void @udot2_MultipleUses_add1(<2 x i16> addrspace(1)* %src1
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
-; GFX7-NEXT:    s_load_dword s6, s[0:1], 0x0
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_lshr_b32 s7, s4, 16
-; GFX7-NEXT:    s_lshr_b32 s9, s5, 16
+; GFX7-NEXT:    s_lshr_b32 s6, s4, 16
+; GFX7-NEXT:    s_lshr_b32 s7, s5, 16
 ; GFX7-NEXT:    s_and_b32 s4, s4, s8
-; GFX7-NEXT:    v_mov_b32_e32 v0, s7
-; GFX7-NEXT:    v_mov_b32_e32 v1, s6
-; GFX7-NEXT:    v_mad_u32_u24 v0, s9, v0, v1
 ; GFX7-NEXT:    s_and_b32 s5, s5, s8
+; GFX7-NEXT:    s_load_dword s8, s[0:1], 0x0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s6
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v1, s8
+; GFX7-NEXT:    v_mad_u32_u24 v0, s7, v0, v1
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s4
 ; GFX7-NEXT:    v_mad_u32_u24 v1, s5, v1, v0
 ; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
@@ -1748,17 +1757,17 @@ define amdgpu_kernel void @udot2_MultipleUses_add1(<2 x i16> addrspace(1)* %src1
 ; GFX10-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
 ; GFX10-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
 ; GFX10-DL-NEXT:    s_load_dword s5, s[0:1], 0x0
-; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_lshr_b32 s0, s3, 16
-; GFX10-DL-NEXT:    s_lshr_b32 s1, s4, 16
-; GFX10-DL-NEXT:    v_mov_b32_e32 v2, s5
+; GFX10-DL-NEXT:    s_lshr_b32 s6, s3, 16
+; GFX10-DL-NEXT:    s_lshr_b32 s7, s4, 16
+; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s5
 ; GFX10-DL-NEXT:    s_and_b32 s3, s3, s2
 ; GFX10-DL-NEXT:    s_and_b32 s2, s4, s2
-; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s1, s0, v2
-; GFX10-DL-NEXT:    v_mad_u32_u24 v3, s2, s3, v2
-; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v3, v2
+; GFX10-DL-NEXT:    v_mad_u32_u24 v0, s7, s6, v0
+; GFX10-DL-NEXT:    v_mad_u32_u24 v1, s2, s3, v0
+; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v1, v0
+; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX10-DL-NEXT:    s_endpgm
                                                    <2 x i16> addrspace(1)* %src2,
@@ -1821,19 +1830,19 @@ define amdgpu_kernel void @idot2_MultipleUses_add1(<2 x i16> addrspace(1)* %src1
 ; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
 ; GFX8-NEXT:    s_load_dword s3, s[6:7], 0x0
 ; GFX8-NEXT:    s_load_dword s4, s[0:1], 0x0
-; GFX8-NEXT:    v_mov_b32_e32 v0, s0
-; GFX8-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_sext_i32_i16 s0, s2
+; GFX8-NEXT:    s_sext_i32_i16 s5, s2
 ; GFX8-NEXT:    s_ashr_i32 s2, s2, 16
-; GFX8-NEXT:    s_sext_i32_i16 s1, s3
+; GFX8-NEXT:    s_sext_i32_i16 s6, s3
 ; GFX8-NEXT:    s_ashr_i32 s3, s3, 16
-; GFX8-NEXT:    v_mov_b32_e32 v2, s4
-; GFX8-NEXT:    v_mov_b32_e32 v3, s2
-; GFX8-NEXT:    v_mad_i32_i24 v2, s3, v3, v2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s0
-; GFX8-NEXT:    v_mad_i32_i24 v3, s1, v3, v2
-; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v2, v3
+; GFX8-NEXT:    v_mov_b32_e32 v0, s4
+; GFX8-NEXT:    v_mov_b32_e32 v1, s2
+; GFX8-NEXT:    v_mov_b32_e32 v2, s5
+; GFX8-NEXT:    v_mad_i32_i24 v0, s3, v1, v0
+; GFX8-NEXT:    v_mad_i32_i24 v1, s6, v2, v0
+; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v0, v1
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX8-NEXT:    flat_store_dword v[0:1], v2
 ; GFX8-NEXT:    s_endpgm
 ;
@@ -1845,19 +1854,19 @@ define amdgpu_kernel void @idot2_MultipleUses_add1(<2 x i16> addrspace(1)* %src1
 ; GFX9-NODL-NEXT:    s_load_dword s2, s[4:5], 0x0
 ; GFX9-NODL-NEXT:    s_load_dword s3, s[6:7], 0x0
 ; GFX9-NODL-NEXT:    s_load_dword s4, s[0:1], 0x0
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT:    s_sext_i32_i16 s0, s2
+; GFX9-NODL-NEXT:    s_sext_i32_i16 s5, s2
 ; GFX9-NODL-NEXT:    s_ashr_i32 s2, s2, 16
-; GFX9-NODL-NEXT:    s_sext_i32_i16 s1, s3
+; GFX9-NODL-NEXT:    s_sext_i32_i16 s6, s3
 ; GFX9-NODL-NEXT:    s_ashr_i32 s3, s3, 16
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s4
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s2
-; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s3, v3, v2
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s0
-; GFX9-NODL-NEXT:    v_mad_i32_i24 v3, s1, v3, v2
-; GFX9-NODL-NEXT:    v_add_u32_e32 v2, v3, v2
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s2
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s5
+; GFX9-NODL-NEXT:    v_mad_i32_i24 v0, s3, v1, v0
+; GFX9-NODL-NEXT:    v_mad_i32_i24 v1, s6, v2, v0
+; GFX9-NODL-NEXT:    v_add_u32_e32 v2, v1, v0
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-NODL-NEXT:    s_endpgm
 ;
@@ -1869,19 +1878,19 @@ define amdgpu_kernel void @idot2_MultipleUses_add1(<2 x i16> addrspace(1)* %src1
 ; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
 ; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
 ; GFX9-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
-; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    s_sext_i32_i16 s0, s2
+; GFX9-DL-NEXT:    s_sext_i32_i16 s5, s2
 ; GFX9-DL-NEXT:    s_ashr_i32 s2, s2, 16
-; GFX9-DL-NEXT:    s_sext_i32_i16 s1, s3
+; GFX9-DL-NEXT:    s_sext_i32_i16 s6, s3
 ; GFX9-DL-NEXT:    s_ashr_i32 s3, s3, 16
-; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s4
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s2
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s3, v3, v2
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s0
-; GFX9-DL-NEXT:    v_mad_i32_i24 v3, s1, v3, v2
-; GFX9-DL-NEXT:    v_add_u32_e32 v2, v3, v2
+; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s2
+; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s5
+; GFX9-DL-NEXT:    v_mad_i32_i24 v0, s3, v1, v0
+; GFX9-DL-NEXT:    v_mad_i32_i24 v1, s6, v2, v0
+; GFX9-DL-NEXT:    v_add_u32_e32 v2, v1, v0
+; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-DL-NEXT:    s_endpgm
 ;
@@ -1894,17 +1903,17 @@ define amdgpu_kernel void @idot2_MultipleUses_add1(<2 x i16> addrspace(1)* %src1
 ; GFX10-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
 ; GFX10-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
 ; GFX10-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
-; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_ashr_i32 s0, s2, 16
-; GFX10-DL-NEXT:    s_ashr_i32 s1, s3, 16
-; GFX10-DL-NEXT:    v_mov_b32_e32 v2, s4
+; GFX10-DL-NEXT:    s_ashr_i32 s5, s2, 16
+; GFX10-DL-NEXT:    s_ashr_i32 s6, s3, 16
+; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s4
 ; GFX10-DL-NEXT:    s_sext_i32_i16 s2, s2
 ; GFX10-DL-NEXT:    s_sext_i32_i16 s3, s3
-; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s1, s0, v2
-; GFX10-DL-NEXT:    v_mad_i32_i24 v3, s3, s2, v2
-; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v3, v2
+; GFX10-DL-NEXT:    v_mad_i32_i24 v0, s6, s5, v0
+; GFX10-DL-NEXT:    v_mad_i32_i24 v1, s3, s2, v0
+; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v1, v0
+; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX10-DL-NEXT:    s_endpgm
                                                    <2 x i16> addrspace(1)* %src2,
@@ -1945,17 +1954,18 @@ define amdgpu_kernel void @udot2_MultipleUses_mul1(<2 x i16> addrspace(1)* %src1
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
-; GFX7-NEXT:    s_load_dword s6, s[0:1], 0x0
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_lshr_b32 s7, s4, 16
+; GFX7-NEXT:    s_lshr_b32 s6, s4, 16
+; GFX7-NEXT:    s_lshr_b32 s7, s5, 16
 ; GFX7-NEXT:    s_and_b32 s4, s4, s8
-; GFX7-NEXT:    s_lshr_b32 s9, s5, 16
 ; GFX7-NEXT:    s_and_b32 s5, s5, s8
+; GFX7-NEXT:    s_load_dword s8, s[0:1], 0x0
 ; GFX7-NEXT:    v_mov_b32_e32 v0, s4
-; GFX7-NEXT:    v_mov_b32_e32 v1, s6
+; GFX7-NEXT:    v_mov_b32_e32 v2, s6
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v1, s8
 ; GFX7-NEXT:    v_mad_u32_u24 v1, s5, v0, v1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s7
-; GFX7-NEXT:    v_mad_u32_u24 v1, s9, v2, v1
+; GFX7-NEXT:    v_mad_u32_u24 v1, s7, v2, v1
 ; GFX7-NEXT:    v_mad_u32_u24 v0, s5, v0, v1
 ; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; GFX7-NEXT:    s_endpgm
@@ -2045,17 +2055,17 @@ define amdgpu_kernel void @udot2_MultipleUses_mul1(<2 x i16> addrspace(1)* %src1
 ; GFX10-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
 ; GFX10-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
 ; GFX10-DL-NEXT:    s_load_dword s5, s[0:1], 0x0
+; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-DL-NEXT:    s_and_b32 s6, s3, s2
+; GFX10-DL-NEXT:    s_and_b32 s2, s4, s2
+; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s5
+; GFX10-DL-NEXT:    s_lshr_b32 s3, s3, 16
+; GFX10-DL-NEXT:    s_lshr_b32 s4, s4, 16
+; GFX10-DL-NEXT:    v_mad_u32_u24 v0, s2, s6, v0
+; GFX10-DL-NEXT:    v_mad_u32_u24 v0, s4, s3, v0
+; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s2, s6, v0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
-; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_and_b32 s0, s3, s2
-; GFX10-DL-NEXT:    s_and_b32 s1, s4, s2
-; GFX10-DL-NEXT:    v_mov_b32_e32 v2, s5
-; GFX10-DL-NEXT:    s_lshr_b32 s2, s3, 16
-; GFX10-DL-NEXT:    s_lshr_b32 s3, s4, 16
-; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s1, s0, v2
-; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s3, s2, v2
-; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s1, s0, v2
 ; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX10-DL-NEXT:    s_endpgm
                                                    <2 x i16> addrspace(1)* %src2,
@@ -2119,19 +2129,19 @@ define amdgpu_kernel void @idot2_MultipleUses_mul1(<2 x i16> addrspace(1)* %src1
 ; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
 ; GFX8-NEXT:    s_load_dword s3, s[6:7], 0x0
 ; GFX8-NEXT:    s_load_dword s4, s[0:1], 0x0
-; GFX8-NEXT:    v_mov_b32_e32 v0, s0
-; GFX8-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_sext_i32_i16 s0, s2
-; GFX8-NEXT:    s_sext_i32_i16 s1, s3
+; GFX8-NEXT:    s_sext_i32_i16 s5, s2
+; GFX8-NEXT:    s_sext_i32_i16 s6, s3
 ; GFX8-NEXT:    s_ashr_i32 s2, s2, 16
-; GFX8-NEXT:    v_mov_b32_e32 v2, s4
-; GFX8-NEXT:    v_mov_b32_e32 v3, s0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s4
+; GFX8-NEXT:    v_mov_b32_e32 v1, s5
 ; GFX8-NEXT:    s_ashr_i32 s3, s3, 16
-; GFX8-NEXT:    v_mad_i32_i24 v2, s1, v3, v2
-; GFX8-NEXT:    v_mov_b32_e32 v4, s2
-; GFX8-NEXT:    v_mad_i32_i24 v2, s3, v4, v2
-; GFX8-NEXT:    v_mad_i32_i24 v2, s1, v3, v2
+; GFX8-NEXT:    v_mov_b32_e32 v2, s2
+; GFX8-NEXT:    v_mad_i32_i24 v0, s6, v1, v0
+; GFX8-NEXT:    v_mad_i32_i24 v0, s3, v2, v0
+; GFX8-NEXT:    v_mad_i32_i24 v2, s6, v1, v0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX8-NEXT:    flat_store_dword v[0:1], v2
 ; GFX8-NEXT:    s_endpgm
 ;
@@ -2143,19 +2153,19 @@ define amdgpu_kernel void @idot2_MultipleUses_mul1(<2 x i16> addrspace(1)* %src1
 ; GFX9-NODL-NEXT:    s_load_dword s2, s[4:5], 0x0
 ; GFX9-NODL-NEXT:    s_load_dword s3, s[6:7], 0x0
 ; GFX9-NODL-NEXT:    s_load_dword s4, s[0:1], 0x0
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT:    s_sext_i32_i16 s0, s2
-; GFX9-NODL-NEXT:    s_sext_i32_i16 s1, s3
+; GFX9-NODL-NEXT:    s_sext_i32_i16 s5, s2
+; GFX9-NODL-NEXT:    s_sext_i32_i16 s6, s3
 ; GFX9-NODL-NEXT:    s_ashr_i32 s2, s2, 16
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s4
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s0
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s5
 ; GFX9-NODL-NEXT:    s_ashr_i32 s3, s3, 16
-; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s1, v3, v2
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v4, s2
-; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s3, v4, v2
-; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s1, v3, v2
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s2
+; GFX9-NODL-NEXT:    v_mad_i32_i24 v0, s6, v1, v0
+; GFX9-NODL-NEXT:    v_mad_i32_i24 v0, s3, v2, v0
+; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s6, v1, v0
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-NODL-NEXT:    s_endpgm
 ;
@@ -2167,19 +2177,19 @@ define amdgpu_kernel void @idot2_MultipleUses_mul1(<2 x i16> addrspace(1)* %src1
 ; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
 ; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
 ; GFX9-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
-; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    s_sext_i32_i16 s0, s2
-; GFX9-DL-NEXT:    s_sext_i32_i16 s1, s3
+; GFX9-DL-NEXT:    s_sext_i32_i16 s5, s2
+; GFX9-DL-NEXT:    s_sext_i32_i16 s6, s3
 ; GFX9-DL-NEXT:    s_ashr_i32 s2, s2, 16
-; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s4
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s0
+; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s5
 ; GFX9-DL-NEXT:    s_ashr_i32 s3, s3, 16
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s1, v3, v2
-; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s2
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s3, v4, v2
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s1, v3, v2
+; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s2
+; GFX9-DL-NEXT:    v_mad_i32_i24 v0, s6, v1, v0
+; GFX9-DL-NEXT:    v_mad_i32_i24 v0, s3, v2, v0
+; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s6, v1, v0
+; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-DL-NEXT:    s_endpgm
 ;
@@ -2192,17 +2202,17 @@ define amdgpu_kernel void @idot2_MultipleUses_mul1(<2 x i16> addrspace(1)* %src1
 ; GFX10-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
 ; GFX10-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
 ; GFX10-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
-; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_sext_i32_i16 s0, s2
-; GFX10-DL-NEXT:    s_sext_i32_i16 s1, s3
-; GFX10-DL-NEXT:    v_mov_b32_e32 v2, s4
+; GFX10-DL-NEXT:    s_sext_i32_i16 s5, s2
+; GFX10-DL-NEXT:    s_sext_i32_i16 s6, s3
+; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s4
 ; GFX10-DL-NEXT:    s_ashr_i32 s2, s2, 16
 ; GFX10-DL-NEXT:    s_ashr_i32 s3, s3, 16
-; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s1, s0, v2
-; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s3, s2, v2
-; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s1, s0, v2
+; GFX10-DL-NEXT:    v_mad_i32_i24 v0, s6, s5, v0
+; GFX10-DL-NEXT:    v_mad_i32_i24 v0, s3, s2, v0
+; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s6, s5, v0
+; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX10-DL-NEXT:    s_endpgm
                                                    <2 x i16> addrspace(1)* %src2,
@@ -2244,16 +2254,17 @@ define amdgpu_kernel void @udot2_MultipleUses_mul2(<2 x i16> addrspace(1)* %src1
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
-; GFX7-NEXT:    s_load_dword s6, s[0:1], 0x0
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_lshr_b32 s7, s4, 16
-; GFX7-NEXT:    s_lshr_b32 s9, s5, 16
-; GFX7-NEXT:    v_mov_b32_e32 v0, s7
-; GFX7-NEXT:    v_mov_b32_e32 v1, s6
-; GFX7-NEXT:    v_mad_u32_u24 v1, s9, v0, v1
+; GFX7-NEXT:    s_lshr_b32 s6, s4, 16
+; GFX7-NEXT:    s_lshr_b32 s7, s5, 16
 ; GFX7-NEXT:    s_and_b32 s4, s4, s8
-; GFX7-NEXT:    v_mad_u32_u24 v0, s9, v0, v1
 ; GFX7-NEXT:    s_and_b32 s5, s5, s8
+; GFX7-NEXT:    s_load_dword s8, s[0:1], 0x0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s6
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v1, s8
+; GFX7-NEXT:    v_mad_u32_u24 v1, s7, v0, v1
+; GFX7-NEXT:    v_mad_u32_u24 v0, s7, v0, v1
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s4
 ; GFX7-NEXT:    v_mad_u32_u24 v0, s5, v1, v0
 ; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
@@ -2344,17 +2355,17 @@ define amdgpu_kernel void @udot2_MultipleUses_mul2(<2 x i16> addrspace(1)* %src1
 ; GFX10-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
 ; GFX10-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
 ; GFX10-DL-NEXT:    s_load_dword s5, s[0:1], 0x0
-; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_lshr_b32 s0, s3, 16
-; GFX10-DL-NEXT:    s_lshr_b32 s1, s4, 16
-; GFX10-DL-NEXT:    v_mov_b32_e32 v2, s5
+; GFX10-DL-NEXT:    s_lshr_b32 s6, s3, 16
+; GFX10-DL-NEXT:    s_lshr_b32 s7, s4, 16
+; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s5
 ; GFX10-DL-NEXT:    s_and_b32 s3, s3, s2
 ; GFX10-DL-NEXT:    s_and_b32 s2, s4, s2
-; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s1, s0, v2
-; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s1, s0, v2
-; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s2, s3, v2
+; GFX10-DL-NEXT:    v_mad_u32_u24 v0, s7, s6, v0
+; GFX10-DL-NEXT:    v_mad_u32_u24 v0, s7, s6, v0
+; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s2, s3, v0
+; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX10-DL-NEXT:    s_endpgm
                                                    <2 x i16> addrspace(1)* %src2,
@@ -2418,19 +2429,19 @@ define amdgpu_kernel void @idot2_MultipleUses_mul2(<2 x i16> addrspace(1)* %src1
 ; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
 ; GFX8-NEXT:    s_load_dword s3, s[6:7], 0x0
 ; GFX8-NEXT:    s_load_dword s4, s[0:1], 0x0
-; GFX8-NEXT:    v_mov_b32_e32 v0, s0
-; GFX8-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_sext_i32_i16 s0, s2
+; GFX8-NEXT:    s_sext_i32_i16 s5, s2
 ; GFX8-NEXT:    s_ashr_i32 s2, s2, 16
-; GFX8-NEXT:    s_sext_i32_i16 s1, s3
+; GFX8-NEXT:    s_sext_i32_i16 s6, s3
 ; GFX8-NEXT:    s_ashr_i32 s3, s3, 16
-; GFX8-NEXT:    v_mov_b32_e32 v2, s4
-; GFX8-NEXT:    v_mov_b32_e32 v3, s2
-; GFX8-NEXT:    v_mad_i32_i24 v2, s3, v3, v2
-; GFX8-NEXT:    v_mad_i32_i24 v2, s3, v3, v2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s0
-; GFX8-NEXT:    v_mad_i32_i24 v2, s1, v3, v2
+; GFX8-NEXT:    v_mov_b32_e32 v0, s4
+; GFX8-NEXT:    v_mov_b32_e32 v1, s2
+; GFX8-NEXT:    v_mad_i32_i24 v0, s3, v1, v0
+; GFX8-NEXT:    v_mov_b32_e32 v2, s5
+; GFX8-NEXT:    v_mad_i32_i24 v0, s3, v1, v0
+; GFX8-NEXT:    v_mad_i32_i24 v2, s6, v2, v0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX8-NEXT:    flat_store_dword v[0:1], v2
 ; GFX8-NEXT:    s_endpgm
 ;
@@ -2442,19 +2453,19 @@ define amdgpu_kernel void @idot2_MultipleUses_mul2(<2 x i16> addrspace(1)* %src1
 ; GFX9-NODL-NEXT:    s_load_dword s2, s[4:5], 0x0
 ; GFX9-NODL-NEXT:    s_load_dword s3, s[6:7], 0x0
 ; GFX9-NODL-NEXT:    s_load_dword s4, s[0:1], 0x0
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT:    s_sext_i32_i16 s0, s2
+; GFX9-NODL-NEXT:    s_sext_i32_i16 s5, s2
 ; GFX9-NODL-NEXT:    s_ashr_i32 s2, s2, 16
-; GFX9-NODL-NEXT:    s_sext_i32_i16 s1, s3
+; GFX9-NODL-NEXT:    s_sext_i32_i16 s6, s3
 ; GFX9-NODL-NEXT:    s_ashr_i32 s3, s3, 16
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s4
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s2
-; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s3, v3, v2
-; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s3, v3, v2
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s0
-; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s1, v3, v2
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s2
+; GFX9-NODL-NEXT:    v_mad_i32_i24 v0, s3, v1, v0
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s5
+; GFX9-NODL-NEXT:    v_mad_i32_i24 v0, s3, v1, v0
+; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s6, v2, v0
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-NODL-NEXT:    s_endpgm
 ;
@@ -2466,19 +2477,19 @@ define amdgpu_kernel void @idot2_MultipleUses_mul2(<2 x i16> addrspace(1)* %src1
 ; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
 ; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
 ; GFX9-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
-; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    s_sext_i32_i16 s0, s2
+; GFX9-DL-NEXT:    s_sext_i32_i16 s5, s2
 ; GFX9-DL-NEXT:    s_ashr_i32 s2, s2, 16
-; GFX9-DL-NEXT:    s_sext_i32_i16 s1, s3
+; GFX9-DL-NEXT:    s_sext_i32_i16 s6, s3
 ; GFX9-DL-NEXT:    s_ashr_i32 s3, s3, 16
-; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s4
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s2
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s3, v3, v2
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s3, v3, v2
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s0
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s1, v3, v2
+; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s2
+; GFX9-DL-NEXT:    v_mad_i32_i24 v0, s3, v1, v0
+; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s5
+; GFX9-DL-NEXT:    v_mad_i32_i24 v0, s3, v1, v0
+; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s6, v2, v0
+; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-DL-NEXT:    s_endpgm
 ;
@@ -2491,17 +2502,17 @@ define amdgpu_kernel void @idot2_MultipleUses_mul2(<2 x i16> addrspace(1)* %src1
 ; GFX10-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
 ; GFX10-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
 ; GFX10-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
-; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_ashr_i32 s0, s2, 16
-; GFX10-DL-NEXT:    s_ashr_i32 s1, s3, 16
-; GFX10-DL-NEXT:    v_mov_b32_e32 v2, s4
+; GFX10-DL-NEXT:    s_ashr_i32 s5, s2, 16
+; GFX10-DL-NEXT:    s_ashr_i32 s6, s3, 16
+; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s4
 ; GFX10-DL-NEXT:    s_sext_i32_i16 s2, s2
 ; GFX10-DL-NEXT:    s_sext_i32_i16 s3, s3
-; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s1, s0, v2
-; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s1, s0, v2
-; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s3, s2, v2
+; GFX10-DL-NEXT:    v_mad_i32_i24 v0, s6, s5, v0
+; GFX10-DL-NEXT:    v_mad_i32_i24 v0, s6, s5, v0
+; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s3, s2, v0
+; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX10-DL-NEXT:    s_endpgm
                                                    <2 x i16> addrspace(1)* %src2,
@@ -2561,23 +2572,23 @@ define amdgpu_kernel void @udot2_acc16(<2 x i16> addrspace(1)* %src1,
 ; GFX8:       ; %bb.0: ; %entry
 ; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX8-NEXT:    s_mov_b32 s2, 0xffff
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX8-NEXT:    flat_load_ushort v2, v[0:1]
-; GFX8-NEXT:    s_load_dword s0, s[4:5], 0x0
 ; GFX8-NEXT:    s_load_dword s1, s[6:7], 0x0
+; GFX8-NEXT:    s_mov_b32 s0, 0xffff
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_and_b32 s3, s1, s2
+; GFX8-NEXT:    s_and_b32 s3, s1, s0
 ; GFX8-NEXT:    s_lshr_b32 s1, s1, 16
-; GFX8-NEXT:    s_and_b32 s2, s0, s2
-; GFX8-NEXT:    s_lshr_b32 s0, s0, 16
+; GFX8-NEXT:    s_and_b32 s0, s2, s0
+; GFX8-NEXT:    s_lshr_b32 s2, s2, 16
 ; GFX8-NEXT:    v_mov_b32_e32 v3, s1
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
-; GFX8-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s3
 ; GFX8-NEXT:    v_mad_u32_u24 v2, s2, v3, v2
+; GFX8-NEXT:    v_mov_b32_e32 v3, s3
+; GFX8-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
 ; GFX8-NEXT:    flat_store_short v[0:1], v2
 ; GFX8-NEXT:    s_endpgm
 ;
@@ -2585,23 +2596,23 @@ define amdgpu_kernel void @udot2_acc16(<2 x i16> addrspace(1)* %src1,
 ; GFX9-NODL:       ; %bb.0: ; %entry
 ; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX9-NODL-NEXT:    s_mov_b32 s2, 0xffff
 ; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NODL-NEXT:    s_load_dword s2, s[4:5], 0x0
 ; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-NODL-NEXT:    global_load_ushort v2, v[0:1], off
-; GFX9-NODL-NEXT:    s_load_dword s0, s[4:5], 0x0
 ; GFX9-NODL-NEXT:    s_load_dword s1, s[6:7], 0x0
+; GFX9-NODL-NEXT:    s_mov_b32 s0, 0xffff
 ; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT:    s_and_b32 s3, s1, s2
+; GFX9-NODL-NEXT:    s_and_b32 s3, s1, s0
 ; GFX9-NODL-NEXT:    s_lshr_b32 s1, s1, 16
-; GFX9-NODL-NEXT:    s_and_b32 s2, s0, s2
-; GFX9-NODL-NEXT:    s_lshr_b32 s0, s0, 16
+; GFX9-NODL-NEXT:    s_and_b32 s0, s2, s0
+; GFX9-NODL-NEXT:    s_lshr_b32 s2, s2, 16
 ; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s1
 ; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s3
 ; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s2, v3, v2
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s3
+; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
 ; GFX9-NODL-NEXT:    global_store_short v[0:1], v2, off
 ; GFX9-NODL-NEXT:    s_endpgm
 ;
@@ -2610,15 +2621,15 @@ define amdgpu_kernel void @udot2_acc16(<2 x i16> addrspace(1)* %src1,
 ; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
+; GFX9-DL-NEXT:    s_load_dword s2, s[6:7], 0x0
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-DL-NEXT:    global_load_ushort v2, v[0:1], off
+; GFX9-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s3
+; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s2
 ; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-DL-NEXT:    v_dot2_u32_u16 v2, s2, v3, v2
+; GFX9-DL-NEXT:    v_dot2_u32_u16 v2, s0, v3, v2
 ; GFX9-DL-NEXT:    global_store_short v[0:1], v2, off
 ; GFX9-DL-NEXT:    s_endpgm
 ;
@@ -2699,19 +2710,19 @@ define amdgpu_kernel void @notsdot2_sext8(<2 x i8> addrspace(1)* %src1,
 ; GFX8-NEXT:    v_mov_b32_e32 v2, s4
 ; GFX8-NEXT:    v_mov_b32_e32 v3, s5
 ; GFX8-NEXT:    flat_load_ushort v2, v[2:3]
-; GFX8-NEXT:    flat_load_ushort v3, v[0:1]
-; GFX8-NEXT:    v_mov_b32_e32 v0, s0
-; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    flat_load_ushort v0, v[0:1]
 ; GFX8-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(0)
-; GFX8-NEXT:    v_lshrrev_b16_e32 v4, 8, v2
+; GFX8-NEXT:    v_bfe_i32 v1, v2, 0, 8
+; GFX8-NEXT:    v_lshrrev_b16_e32 v2, 8, v2
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
-; GFX8-NEXT:    v_bfe_i32 v5, v3, 0, 8
-; GFX8-NEXT:    v_lshrrev_b16_e32 v3, 8, v3
-; GFX8-NEXT:    v_bfe_i32 v4, v4, 0, 8
-; GFX8-NEXT:    v_bfe_i32 v3, v3, 0, 8
+; GFX8-NEXT:    v_bfe_i32 v3, v0, 0, 8
+; GFX8-NEXT:    v_lshrrev_b16_e32 v0, 8, v0
 ; GFX8-NEXT:    v_bfe_i32 v2, v2, 0, 8
-; GFX8-NEXT:    v_mad_i32_i24 v3, v3, v4, s2
-; GFX8-NEXT:    v_mad_i32_i24 v2, v5, v2, v3
+; GFX8-NEXT:    v_bfe_i32 v0, v0, 0, 8
+; GFX8-NEXT:    v_mad_i32_i24 v0, v0, v2, s2
+; GFX8-NEXT:    v_mad_i32_i24 v2, v3, v1, v0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX8-NEXT:    flat_store_dword v[0:1], v2
 ; GFX8-NEXT:    s_endpgm
 ;
@@ -2726,20 +2737,20 @@ define amdgpu_kernel void @notsdot2_sext8(<2 x i8> addrspace(1)* %src1,
 ; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s4
 ; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s5
 ; GFX9-NODL-NEXT:    global_load_ushort v2, v[2:3], off
-; GFX9-NODL-NEXT:    global_load_ushort v3, v[0:1], off
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NODL-NEXT:    global_load_ushort v0, v[0:1], off
 ; GFX9-NODL-NEXT:    s_waitcnt vmcnt(1)
-; GFX9-NODL-NEXT:    v_lshrrev_b16_e32 v4, 8, v2
+; GFX9-NODL-NEXT:    v_bfe_i32 v1, v2, 0, 8
+; GFX9-NODL-NEXT:    v_lshrrev_b16_e32 v2, 8, v2
 ; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NODL-NEXT:    v_bfe_i32 v5, v3, 0, 8
-; GFX9-NODL-NEXT:    v_lshrrev_b16_e32 v3, 8, v3
-; GFX9-NODL-NEXT:    v_bfe_i32 v4, v4, 0, 8
-; GFX9-NODL-NEXT:    v_bfe_i32 v3, v3, 0, 8
+; GFX9-NODL-NEXT:    v_bfe_i32 v3, v0, 0, 8
+; GFX9-NODL-NEXT:    v_lshrrev_b16_e32 v0, 8, v0
 ; GFX9-NODL-NEXT:    v_bfe_i32 v2, v2, 0, 8
+; GFX9-NODL-NEXT:    v_bfe_i32 v0, v0, 0, 8
 ; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT:    v_mad_i32_i24 v3, v3, v4, s2
-; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, v5, v2, v3
+; GFX9-NODL-NEXT:    v_mad_i32_i24 v0, v0, v2, s2
+; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, v3, v1, v0
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-NODL-NEXT:    s_endpgm
 ;
@@ -2754,20 +2765,20 @@ define amdgpu_kernel void @notsdot2_sext8(<2 x i8> addrspace(1)* %src1,
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s4
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s5
 ; GFX9-DL-NEXT:    global_load_ushort v2, v[2:3], off
-; GFX9-DL-NEXT:    global_load_ushort v3, v[0:1], off
-; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-DL-NEXT:    global_load_ushort v0, v[0:1], off
 ; GFX9-DL-NEXT:    s_waitcnt vmcnt(1)
-; GFX9-DL-NEXT:    v_lshrrev_b16_e32 v4, 8, v2
+; GFX9-DL-NEXT:    v_bfe_i32 v1, v2, 0, 8
+; GFX9-DL-NEXT:    v_lshrrev_b16_e32 v2, 8, v2
 ; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-DL-NEXT:    v_bfe_i32 v5, v3, 0, 8
-; GFX9-DL-NEXT:    v_lshrrev_b16_e32 v3, 8, v3
-; GFX9-DL-NEXT:    v_bfe_i32 v4, v4, 0, 8
-; GFX9-DL-NEXT:    v_bfe_i32 v3, v3, 0, 8
+; GFX9-DL-NEXT:    v_bfe_i32 v3, v0, 0, 8
+; GFX9-DL-NEXT:    v_lshrrev_b16_e32 v0, 8, v0
 ; GFX9-DL-NEXT:    v_bfe_i32 v2, v2, 0, 8
+; GFX9-DL-NEXT:    v_bfe_i32 v0, v0, 0, 8
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    v_mad_i32_i24 v3, v3, v4, s2
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, v5, v2, v3
+; GFX9-DL-NEXT:    v_mad_i32_i24 v0, v0, v2, s2
+; GFX9-DL-NEXT:    v_mad_i32_i24 v2, v3, v1, v0
+; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-DL-NEXT:    s_endpgm
 ;
@@ -2775,7 +2786,6 @@ define amdgpu_kernel void @notsdot2_sext8(<2 x i8> addrspace(1)* %src1,
 ; GFX10-DL:       ; %bb.0: ; %entry
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX10-DL-NEXT:    v_mov_b32_e32 v4, 0xffff
 ; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s6
@@ -2784,13 +2794,13 @@ define amdgpu_kernel void @notsdot2_sext8(<2 x i8> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s7
 ; GFX10-DL-NEXT:    s_load_dword s2, s[0:1], 0x0
 ; GFX10-DL-NEXT:    global_load_ushort v2, v[2:3], off
-; GFX10-DL-NEXT:    global_load_ushort v7, v[0:1], off
+; GFX10-DL-NEXT:    global_load_ushort v0, v[0:1], off
 ; GFX10-DL-NEXT:    s_waitcnt vmcnt(1)
-; GFX10-DL-NEXT:    v_and_b32_sdwa v1, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-DL-NEXT:    v_lshrrev_b16_e64 v1, 8, v2
 ; GFX10-DL-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-DL-NEXT:    v_and_b32_sdwa v3, v7, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-DL-NEXT:    v_lshrrev_b16_e64 v3, 8, v0
 ; GFX10-DL-NEXT:    v_bfe_i32 v2, v2, 0, 8
-; GFX10-DL-NEXT:    v_bfe_i32 v0, v7, 0, 8
+; GFX10-DL-NEXT:    v_bfe_i32 v0, v0, 0, 8
 ; GFX10-DL-NEXT:    v_bfe_i32 v1, v1, 0, 8
 ; GFX10-DL-NEXT:    v_bfe_i32 v3, v3, 0, 8
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/idot4s.ll b/llvm/test/CodeGen/AMDGPU/idot4s.ll
index ef292a361..28eb2717b 100644
--- a/llvm/test/CodeGen/AMDGPU/idot4s.ll
+++ b/llvm/test/CodeGen/AMDGPU/idot4s.ll
@@ -16,22 +16,22 @@ define amdgpu_kernel void @idot4_acc32(<4 x i8> addrspace(1)* %src1,
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
-; GFX7-NEXT:    s_load_dword s6, s[0:1], 0x0
+; GFX7-NEXT:    s_load_dword s12, s[0:1], 0x0
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_sext_i32_i8 s7, s4
-; GFX7-NEXT:    s_sext_i32_i8 s8, s5
-; GFX7-NEXT:    s_bfe_i32 s10, s5, 0x80008
-; GFX7-NEXT:    v_mov_b32_e32 v0, s8
-; GFX7-NEXT:    v_mov_b32_e32 v1, s6
-; GFX7-NEXT:    s_bfe_i32 s12, s5, 0x80010
-; GFX7-NEXT:    v_mad_i32_i24 v0, s7, v0, v1
-; GFX7-NEXT:    s_bfe_i32 s9, s4, 0x80008
-; GFX7-NEXT:    v_mov_b32_e32 v1, s10
-; GFX7-NEXT:    s_bfe_i32 s11, s4, 0x80010
-; GFX7-NEXT:    v_mad_i32_i24 v0, s9, v1, v0
+; GFX7-NEXT:    s_sext_i32_i8 s6, s4
+; GFX7-NEXT:    s_sext_i32_i8 s7, s5
+; GFX7-NEXT:    s_bfe_i32 s9, s5, 0x80008
+; GFX7-NEXT:    v_mov_b32_e32 v0, s7
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s12
+; GFX7-NEXT:    s_bfe_i32 s11, s5, 0x80010
+; GFX7-NEXT:    v_mad_i32_i24 v0, s6, v0, v1
+; GFX7-NEXT:    s_bfe_i32 s8, s4, 0x80008
+; GFX7-NEXT:    v_mov_b32_e32 v1, s9
+; GFX7-NEXT:    s_bfe_i32 s10, s4, 0x80010
+; GFX7-NEXT:    v_mad_i32_i24 v0, s8, v1, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s11
 ; GFX7-NEXT:    s_ashr_i32 s5, s5, 24
-; GFX7-NEXT:    v_mad_i32_i24 v0, s11, v1, v0
+; GFX7-NEXT:    v_mad_i32_i24 v0, s10, v1, v0
 ; GFX7-NEXT:    s_ashr_i32 s4, s4, 24
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s5
 ; GFX7-NEXT:    v_mad_i32_i24 v0, s4, v1, v0
@@ -45,27 +45,27 @@ define amdgpu_kernel void @idot4_acc32(<4 x i8> addrspace(1)* %src1,
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
 ; GFX8-NEXT:    s_load_dword s3, s[6:7], 0x0
-; GFX8-NEXT:    s_load_dword s4, s[0:1], 0x0
-; GFX8-NEXT:    v_mov_b32_e32 v0, s0
-; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    s_load_dword s10, s[0:1], 0x0
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_sext_i32_i8 s0, s2
-; GFX8-NEXT:    s_sext_i32_i8 s1, s3
-; GFX8-NEXT:    s_bfe_i32 s6, s3, 0x80008
-; GFX8-NEXT:    v_mov_b32_e32 v2, s1
-; GFX8-NEXT:    v_mov_b32_e32 v3, s4
-; GFX8-NEXT:    s_bfe_i32 s8, s3, 0x80010
-; GFX8-NEXT:    v_mad_i32_i24 v2, s0, v2, v3
-; GFX8-NEXT:    s_bfe_i32 s5, s2, 0x80008
-; GFX8-NEXT:    v_mov_b32_e32 v3, s6
-; GFX8-NEXT:    s_bfe_i32 s7, s2, 0x80010
-; GFX8-NEXT:    v_mad_i32_i24 v2, s5, v3, v2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s8
+; GFX8-NEXT:    s_sext_i32_i8 s4, s2
+; GFX8-NEXT:    s_sext_i32_i8 s5, s3
+; GFX8-NEXT:    s_bfe_i32 s7, s3, 0x80008
+; GFX8-NEXT:    v_mov_b32_e32 v0, s5
+; GFX8-NEXT:    v_mov_b32_e32 v1, s10
+; GFX8-NEXT:    s_bfe_i32 s9, s3, 0x80010
+; GFX8-NEXT:    v_mad_i32_i24 v0, s4, v0, v1
+; GFX8-NEXT:    s_bfe_i32 s6, s2, 0x80008
+; GFX8-NEXT:    v_mov_b32_e32 v1, s7
+; GFX8-NEXT:    s_bfe_i32 s8, s2, 0x80010
+; GFX8-NEXT:    v_mad_i32_i24 v0, s6, v1, v0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s9
 ; GFX8-NEXT:    s_ashr_i32 s3, s3, 24
-; GFX8-NEXT:    v_mad_i32_i24 v2, s7, v3, v2
+; GFX8-NEXT:    v_mad_i32_i24 v0, s8, v1, v0
 ; GFX8-NEXT:    s_ashr_i32 s2, s2, 24
-; GFX8-NEXT:    v_mov_b32_e32 v3, s3
-; GFX8-NEXT:    v_mad_i32_i24 v2, s2, v3, v2
+; GFX8-NEXT:    v_mov_b32_e32 v1, s3
+; GFX8-NEXT:    v_mad_i32_i24 v2, s2, v1, v0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX8-NEXT:    flat_store_dword v[0:1], v2
 ; GFX8-NEXT:    s_endpgm
 ;
@@ -76,27 +76,27 @@ define amdgpu_kernel void @idot4_acc32(<4 x i8> addrspace(1)* %src1,
 ; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NODL-NEXT:    s_load_dword s2, s[4:5], 0x0
 ; GFX9-NODL-NEXT:    s_load_dword s3, s[6:7], 0x0
-; GFX9-NODL-NEXT:    s_load_dword s4, s[0:1], 0x0
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NODL-NEXT:    s_load_dword s10, s[0:1], 0x0
 ; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT:    s_sext_i32_i8 s0, s2
-; GFX9-NODL-NEXT:    s_sext_i32_i8 s1, s3
-; GFX9-NODL-NEXT:    s_bfe_i32 s6, s3, 0x80008
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s1
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s4
-; GFX9-NODL-NEXT:    s_bfe_i32 s8, s3, 0x80010
-; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s0, v2, v3
-; GFX9-NODL-NEXT:    s_bfe_i32 s5, s2, 0x80008
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s6
-; GFX9-NODL-NEXT:    s_bfe_i32 s7, s2, 0x80010
-; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s5, v3, v2
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s8
+; GFX9-NODL-NEXT:    s_sext_i32_i8 s4, s2
+; GFX9-NODL-NEXT:    s_sext_i32_i8 s5, s3
+; GFX9-NODL-NEXT:    s_bfe_i32 s7, s3, 0x80008
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s5
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s10
+; GFX9-NODL-NEXT:    s_bfe_i32 s9, s3, 0x80010
+; GFX9-NODL-NEXT:    v_mad_i32_i24 v0, s4, v0, v1
+; GFX9-NODL-NEXT:    s_bfe_i32 s6, s2, 0x80008
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s7
+; GFX9-NODL-NEXT:    s_bfe_i32 s8, s2, 0x80010
+; GFX9-NODL-NEXT:    v_mad_i32_i24 v0, s6, v1, v0
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s9
 ; GFX9-NODL-NEXT:    s_ashr_i32 s3, s3, 24
-; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s7, v3, v2
+; GFX9-NODL-NEXT:    v_mad_i32_i24 v0, s8, v1, v0
 ; GFX9-NODL-NEXT:    s_ashr_i32 s2, s2, 24
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s3
-; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s2, v3, v2
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s2, v1, v0
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-NODL-NEXT:    s_endpgm
 ;
@@ -105,15 +105,15 @@ define amdgpu_kernel void @idot4_acc32(<4 x i8> addrspace(1)* %src1,
 ; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
-; GFX9-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
+; GFX9-DL-NEXT:    s_load_dword s2, s[6:7], 0x0
+; GFX9-DL-NEXT:    s_load_dword s3, s[0:1], 0x0
+; GFX9-DL-NEXT:    s_load_dword s4, s[4:5], 0x0
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s3
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s4
-; GFX9-DL-NEXT:    v_dot4_i32_i8 v2, s2, v2, v3
+; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s2
+; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s3
+; GFX9-DL-NEXT:    v_dot4_i32_i8 v2, s4, v2, v3
 ; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-DL-NEXT:    s_endpgm
 ;
@@ -123,14 +123,14 @@ define amdgpu_kernel void @idot4_acc32(<4 x i8> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX10-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
-; GFX10-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
+; GFX10-DL-NEXT:    s_load_dword s2, s[0:1], 0x0
+; GFX10-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
+; GFX10-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
+; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-DL-NEXT:    v_dot4_i32_i8 v2, s3, s4, v0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
-; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    v_mov_b32_e32 v2, s4
-; GFX10-DL-NEXT:    v_dot4_i32_i8 v2, s2, s3, v2
 ; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX10-DL-NEXT:    s_endpgm
                                        <4 x i8> addrspace(1)* %src2,
@@ -220,29 +220,29 @@ define amdgpu_kernel void @idot4_acc16(<4 x i8> addrspace(1)* %src1,
 ; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX8-NEXT:    s_load_dword s3, s[6:7], 0x0
+; GFX8-NEXT:    s_load_dword s2, s[6:7], 0x0
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX8-NEXT:    flat_load_ushort v2, v[0:1]
+; GFX8-NEXT:    s_load_dword s0, s[4:5], 0x0
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_sext_i32_i8 s0, s2
-; GFX8-NEXT:    s_sext_i32_i8 s1, s3
+; GFX8-NEXT:    s_sext_i32_i8 s1, s2
+; GFX8-NEXT:    s_bfe_i32 s3, s2, 0x80008
 ; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    s_bfe_i32 s4, s3, 0x80008
-; GFX8-NEXT:    s_bfe_i32 s5, s3, 0x80010
-; GFX8-NEXT:    v_mov_b32_e32 v4, s4
-; GFX8-NEXT:    s_bfe_i32 s1, s2, 0x80008
-; GFX8-NEXT:    s_bfe_i32 s4, s2, 0x80010
-; GFX8-NEXT:    s_ashr_i32 s3, s3, 24
-; GFX8-NEXT:    v_mov_b32_e32 v5, s5
+; GFX8-NEXT:    s_bfe_i32 s5, s2, 0x80010
+; GFX8-NEXT:    s_sext_i32_i8 s1, s0
+; GFX8-NEXT:    v_mov_b32_e32 v4, s3
+; GFX8-NEXT:    s_bfe_i32 s4, s0, 0x80008
+; GFX8-NEXT:    s_bfe_i32 s3, s0, 0x80010
 ; GFX8-NEXT:    s_ashr_i32 s2, s2, 24
+; GFX8-NEXT:    v_mov_b32_e32 v5, s5
+; GFX8-NEXT:    s_ashr_i32 s0, s0, 24
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    v_mad_i32_i24 v2, s1, v3, v2
+; GFX8-NEXT:    v_mad_i32_i24 v2, s4, v4, v2
+; GFX8-NEXT:    v_mad_i32_i24 v2, s3, v5, v2
+; GFX8-NEXT:    v_mov_b32_e32 v3, s2
 ; GFX8-NEXT:    v_mad_i32_i24 v2, s0, v3, v2
-; GFX8-NEXT:    v_mad_i32_i24 v2, s1, v4, v2
-; GFX8-NEXT:    v_mad_i32_i24 v2, s4, v5, v2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s3
-; GFX8-NEXT:    v_mad_i32_i24 v2, s2, v3, v2
 ; GFX8-NEXT:    flat_store_short v[0:1], v2
 ; GFX8-NEXT:    s_endpgm
 ;
@@ -251,29 +251,29 @@ define amdgpu_kernel void @idot4_acc16(<4 x i8> addrspace(1)* %src1,
 ; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX9-NODL-NEXT:    s_load_dword s3, s[6:7], 0x0
+; GFX9-NODL-NEXT:    s_load_dword s2, s[6:7], 0x0
 ; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-NODL-NEXT:    global_load_ushort v2, v[0:1], off
+; GFX9-NODL-NEXT:    s_load_dword s0, s[4:5], 0x0
 ; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT:    s_sext_i32_i8 s0, s2
-; GFX9-NODL-NEXT:    s_sext_i32_i8 s1, s3
+; GFX9-NODL-NEXT:    s_sext_i32_i8 s1, s2
+; GFX9-NODL-NEXT:    s_bfe_i32 s3, s2, 0x80008
 ; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s1
-; GFX9-NODL-NEXT:    s_bfe_i32 s4, s3, 0x80008
-; GFX9-NODL-NEXT:    s_bfe_i32 s5, s3, 0x80010
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v4, s4
-; GFX9-NODL-NEXT:    s_bfe_i32 s1, s2, 0x80008
-; GFX9-NODL-NEXT:    s_bfe_i32 s4, s2, 0x80010
-; GFX9-NODL-NEXT:    s_ashr_i32 s3, s3, 24
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v5, s5
+; GFX9-NODL-NEXT:    s_bfe_i32 s5, s2, 0x80010
+; GFX9-NODL-NEXT:    s_sext_i32_i8 s1, s0
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v4, s3
+; GFX9-NODL-NEXT:    s_bfe_i32 s4, s0, 0x80008
+; GFX9-NODL-NEXT:    s_bfe_i32 s3, s0, 0x80010
 ; GFX9-NODL-NEXT:    s_ashr_i32 s2, s2, 24
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v5, s5
+; GFX9-NODL-NEXT:    s_ashr_i32 s0, s0, 24
 ; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s1, v3, v2
+; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s4, v4, v2
+; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s3, v5, v2
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s2
 ; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s0, v3, v2
-; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s1, v4, v2
-; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s4, v5, v2
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s3
-; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s2, v3, v2
 ; GFX9-NODL-NEXT:    global_store_short v[0:1], v2, off
 ; GFX9-NODL-NEXT:    s_endpgm
 ;
@@ -282,15 +282,15 @@ define amdgpu_kernel void @idot4_acc16(<4 x i8> addrspace(1)* %src1,
 ; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
+; GFX9-DL-NEXT:    s_load_dword s2, s[6:7], 0x0
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-DL-NEXT:    global_load_ushort v2, v[0:1], off
+; GFX9-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s3
+; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s2
 ; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-DL-NEXT:    v_dot4_i32_i8 v2, s2, v3, v2
+; GFX9-DL-NEXT:    v_dot4_i32_i8 v2, s0, v3, v2
 ; GFX9-DL-NEXT:    global_store_short v[0:1], v2, off
 ; GFX9-DL-NEXT:    s_endpgm
 ;
@@ -356,28 +356,28 @@ define amdgpu_kernel void @idot4_acc8(<4 x i8> addrspace(1)* %src1,
 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
 ; GFX7-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX7-NEXT:    s_mov_b32 s2, -1
-; GFX7-NEXT:    s_movk_i32 s8, 0xff
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
 ; GFX7-NEXT:    buffer_load_ubyte v0, off, s[0:3], 0
-; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
+; GFX7-NEXT:    s_load_dword s6, s[6:7], 0x0
+; GFX7-NEXT:    s_movk_i32 s5, 0xff
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_and_b32 s7, s4, s8
 ; GFX7-NEXT:    s_bfe_u32 s9, s4, 0x80008
-; GFX7-NEXT:    s_and_b32 s6, s5, s8
-; GFX7-NEXT:    s_bfe_u32 s8, s5, 0x80008
-; GFX7-NEXT:    v_mov_b32_e32 v1, s6
-; GFX7-NEXT:    s_bfe_u32 s10, s5, 0x80010
-; GFX7-NEXT:    v_mov_b32_e32 v2, s8
 ; GFX7-NEXT:    s_bfe_u32 s11, s4, 0x80010
-; GFX7-NEXT:    s_lshr_b32 s5, s5, 24
+; GFX7-NEXT:    s_and_b32 s7, s6, s5
+; GFX7-NEXT:    s_and_b32 s5, s4, s5
+; GFX7-NEXT:    s_bfe_u32 s8, s6, 0x80008
+; GFX7-NEXT:    v_mov_b32_e32 v1, s7
+; GFX7-NEXT:    s_bfe_u32 s10, s6, 0x80010
+; GFX7-NEXT:    v_mov_b32_e32 v2, s8
+; GFX7-NEXT:    s_lshr_b32 s6, s6, 24
 ; GFX7-NEXT:    v_mov_b32_e32 v3, s10
 ; GFX7-NEXT:    s_lshr_b32 s4, s4, 24
 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
-; GFX7-NEXT:    v_mad_u32_u24 v0, s7, v1, v0
+; GFX7-NEXT:    v_mad_u32_u24 v0, s5, v1, v0
 ; GFX7-NEXT:    v_mad_u32_u24 v0, s9, v2, v0
 ; GFX7-NEXT:    v_mad_u32_u24 v0, s11, v3, v0
-; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v1, s6
 ; GFX7-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
 ; GFX7-NEXT:    buffer_store_byte v0, off, s[0:3], 0
 ; GFX7-NEXT:    s_endpgm
@@ -386,31 +386,31 @@ define amdgpu_kernel void @idot4_acc8(<4 x i8> addrspace(1)* %src1,
 ; GFX8:       ; %bb.0: ; %entry
 ; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX8-NEXT:    s_movk_i32 s2, 0xff
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX8-NEXT:    flat_load_ubyte v2, v[0:1]
-; GFX8-NEXT:    s_load_dword s0, s[4:5], 0x0
 ; GFX8-NEXT:    s_load_dword s1, s[6:7], 0x0
+; GFX8-NEXT:    s_movk_i32 s0, 0xff
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_bfe_u32 s5, s0, 0x80008
-; GFX8-NEXT:    s_and_b32 s3, s1, s2
+; GFX8-NEXT:    s_bfe_u32 s5, s2, 0x80008
+; GFX8-NEXT:    s_bfe_u32 s7, s2, 0x80010
+; GFX8-NEXT:    s_and_b32 s3, s1, s0
+; GFX8-NEXT:    s_and_b32 s0, s2, s0
 ; GFX8-NEXT:    s_bfe_u32 s4, s1, 0x80008
-; GFX8-NEXT:    s_and_b32 s2, s0, s2
 ; GFX8-NEXT:    v_mov_b32_e32 v3, s3
 ; GFX8-NEXT:    s_bfe_u32 s6, s1, 0x80010
 ; GFX8-NEXT:    v_mov_b32_e32 v4, s4
-; GFX8-NEXT:    s_bfe_u32 s7, s0, 0x80010
 ; GFX8-NEXT:    s_lshr_b32 s1, s1, 24
 ; GFX8-NEXT:    v_mov_b32_e32 v5, s6
-; GFX8-NEXT:    s_lshr_b32 s0, s0, 24
+; GFX8-NEXT:    s_lshr_b32 s2, s2, 24
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
-; GFX8-NEXT:    v_mad_u32_u24 v2, s2, v3, v2
+; GFX8-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
 ; GFX8-NEXT:    v_mad_u32_u24 v2, s5, v4, v2
 ; GFX8-NEXT:    v_mad_u32_u24 v2, s7, v5, v2
 ; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
+; GFX8-NEXT:    v_mad_u32_u24 v2, s2, v3, v2
 ; GFX8-NEXT:    flat_store_byte v[0:1], v2
 ; GFX8-NEXT:    s_endpgm
 ;
@@ -418,31 +418,31 @@ define amdgpu_kernel void @idot4_acc8(<4 x i8> addrspace(1)* %src1,
 ; GFX9-NODL:       ; %bb.0: ; %entry
 ; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX9-NODL-NEXT:    s_movk_i32 s2, 0xff
 ; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NODL-NEXT:    s_load_dword s2, s[4:5], 0x0
 ; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-NODL-NEXT:    global_load_ubyte v2, v[0:1], off
-; GFX9-NODL-NEXT:    s_load_dword s0, s[4:5], 0x0
 ; GFX9-NODL-NEXT:    s_load_dword s1, s[6:7], 0x0
+; GFX9-NODL-NEXT:    s_movk_i32 s0, 0xff
 ; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT:    s_bfe_u32 s5, s0, 0x80008
-; GFX9-NODL-NEXT:    s_and_b32 s3, s1, s2
+; GFX9-NODL-NEXT:    s_bfe_u32 s5, s2, 0x80008
+; GFX9-NODL-NEXT:    s_bfe_u32 s7, s2, 0x80010
+; GFX9-NODL-NEXT:    s_and_b32 s3, s1, s0
+; GFX9-NODL-NEXT:    s_and_b32 s0, s2, s0
 ; GFX9-NODL-NEXT:    s_bfe_u32 s4, s1, 0x80008
-; GFX9-NODL-NEXT:    s_and_b32 s2, s0, s2
 ; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s3
 ; GFX9-NODL-NEXT:    s_bfe_u32 s6, s1, 0x80010
 ; GFX9-NODL-NEXT:    v_mov_b32_e32 v4, s4
-; GFX9-NODL-NEXT:    s_bfe_u32 s7, s0, 0x80010
 ; GFX9-NODL-NEXT:    s_lshr_b32 s1, s1, 24
 ; GFX9-NODL-NEXT:    v_mov_b32_e32 v5, s6
-; GFX9-NODL-NEXT:    s_lshr_b32 s0, s0, 24
+; GFX9-NODL-NEXT:    s_lshr_b32 s2, s2, 24
 ; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s2, v3, v2
+; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
 ; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s5, v4, v2
 ; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s7, v5, v2
 ; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s1
-; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
+; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s2, v3, v2
 ; GFX9-NODL-NEXT:    global_store_byte v[0:1], v2, off
 ; GFX9-NODL-NEXT:    s_endpgm
 ;
@@ -451,15 +451,15 @@ define amdgpu_kernel void @idot4_acc8(<4 x i8> addrspace(1)* %src1,
 ; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
+; GFX9-DL-NEXT:    s_load_dword s2, s[6:7], 0x0
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-DL-NEXT:    global_load_ubyte v2, v[0:1], off
+; GFX9-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s3
+; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s2
 ; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-DL-NEXT:    v_dot4_u32_u8 v2, s2, v3, v2
+; GFX9-DL-NEXT:    v_dot4_u32_u8 v2, s0, v3, v2
 ; GFX9-DL-NEXT:    global_store_byte v[0:1], v2, off
 ; GFX9-DL-NEXT:    s_endpgm
 ;
@@ -520,23 +520,23 @@ define amdgpu_kernel void @idot4_multiuse_mul1(<4 x i8> addrspace(1)* %src1,
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
-; GFX7-NEXT:    s_load_dword s6, s[0:1], 0x0
+; GFX7-NEXT:    s_load_dword s12, s[0:1], 0x0
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_sext_i32_i8 s7, s4
-; GFX7-NEXT:    s_sext_i32_i8 s8, s5
-; GFX7-NEXT:    s_bfe_i32 s10, s5, 0x80008
-; GFX7-NEXT:    v_mov_b32_e32 v0, s8
-; GFX7-NEXT:    v_mov_b32_e32 v1, s6
-; GFX7-NEXT:    s_bfe_i32 s9, s4, 0x80008
-; GFX7-NEXT:    v_mad_i32_i24 v1, s7, v0, v1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s10
-; GFX7-NEXT:    s_bfe_i32 s12, s5, 0x80010
-; GFX7-NEXT:    v_mad_i32_i24 v1, s9, v2, v1
-; GFX7-NEXT:    s_bfe_i32 s11, s4, 0x80010
-; GFX7-NEXT:    v_mad_i32_i24 v0, s7, v0, v1
+; GFX7-NEXT:    s_sext_i32_i8 s6, s4
+; GFX7-NEXT:    s_sext_i32_i8 s7, s5
+; GFX7-NEXT:    s_bfe_i32 s9, s5, 0x80008
+; GFX7-NEXT:    v_mov_b32_e32 v0, s7
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s12
+; GFX7-NEXT:    s_bfe_i32 s8, s4, 0x80008
+; GFX7-NEXT:    v_mad_i32_i24 v1, s6, v0, v1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s9
+; GFX7-NEXT:    s_bfe_i32 s11, s5, 0x80010
+; GFX7-NEXT:    v_mad_i32_i24 v1, s8, v2, v1
+; GFX7-NEXT:    s_bfe_i32 s10, s4, 0x80010
+; GFX7-NEXT:    v_mad_i32_i24 v0, s6, v0, v1
+; GFX7-NEXT:    v_mov_b32_e32 v1, s11
 ; GFX7-NEXT:    s_ashr_i32 s5, s5, 24
-; GFX7-NEXT:    v_mad_i32_i24 v0, s11, v1, v0
+; GFX7-NEXT:    v_mad_i32_i24 v0, s10, v1, v0
 ; GFX7-NEXT:    s_ashr_i32 s4, s4, 24
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s5
 ; GFX7-NEXT:    v_mad_i32_i24 v0, s4, v1, v0
@@ -550,28 +550,28 @@ define amdgpu_kernel void @idot4_multiuse_mul1(<4 x i8> addrspace(1)* %src1,
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
 ; GFX8-NEXT:    s_load_dword s3, s[6:7], 0x0
-; GFX8-NEXT:    s_load_dword s4, s[0:1], 0x0
-; GFX8-NEXT:    v_mov_b32_e32 v0, s0
-; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    s_load_dword s10, s[0:1], 0x0
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_sext_i32_i8 s0, s2
-; GFX8-NEXT:    s_sext_i32_i8 s1, s3
-; GFX8-NEXT:    s_bfe_i32 s6, s3, 0x80008
-; GFX8-NEXT:    v_mov_b32_e32 v2, s1
-; GFX8-NEXT:    v_mov_b32_e32 v3, s4
-; GFX8-NEXT:    s_bfe_i32 s5, s2, 0x80008
-; GFX8-NEXT:    v_mad_i32_i24 v3, s0, v2, v3
-; GFX8-NEXT:    v_mov_b32_e32 v4, s6
-; GFX8-NEXT:    s_bfe_i32 s8, s3, 0x80010
-; GFX8-NEXT:    v_mad_i32_i24 v3, s5, v4, v3
-; GFX8-NEXT:    s_bfe_i32 s7, s2, 0x80010
-; GFX8-NEXT:    v_mad_i32_i24 v2, s0, v2, v3
-; GFX8-NEXT:    v_mov_b32_e32 v3, s8
+; GFX8-NEXT:    s_sext_i32_i8 s4, s2
+; GFX8-NEXT:    s_sext_i32_i8 s5, s3
+; GFX8-NEXT:    s_bfe_i32 s7, s3, 0x80008
+; GFX8-NEXT:    v_mov_b32_e32 v0, s5
+; GFX8-NEXT:    v_mov_b32_e32 v1, s10
+; GFX8-NEXT:    s_bfe_i32 s6, s2, 0x80008
+; GFX8-NEXT:    v_mad_i32_i24 v1, s4, v0, v1
+; GFX8-NEXT:    v_mov_b32_e32 v2, s7
+; GFX8-NEXT:    s_bfe_i32 s9, s3, 0x80010
+; GFX8-NEXT:    v_mad_i32_i24 v1, s6, v2, v1
+; GFX8-NEXT:    s_bfe_i32 s8, s2, 0x80010
+; GFX8-NEXT:    v_mad_i32_i24 v0, s4, v0, v1
+; GFX8-NEXT:    v_mov_b32_e32 v1, s9
 ; GFX8-NEXT:    s_ashr_i32 s3, s3, 24
-; GFX8-NEXT:    v_mad_i32_i24 v2, s7, v3, v2
+; GFX8-NEXT:    v_mad_i32_i24 v0, s8, v1, v0
 ; GFX8-NEXT:    s_ashr_i32 s2, s2, 24
-; GFX8-NEXT:    v_mov_b32_e32 v3, s3
-; GFX8-NEXT:    v_mad_i32_i24 v2, s2, v3, v2
+; GFX8-NEXT:    v_mov_b32_e32 v1, s3
+; GFX8-NEXT:    v_mad_i32_i24 v2, s2, v1, v0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX8-NEXT:    flat_store_dword v[0:1], v2
 ; GFX8-NEXT:    s_endpgm
 ;
@@ -582,28 +582,28 @@ define amdgpu_kernel void @idot4_multiuse_mul1(<4 x i8> addrspace(1)* %src1,
 ; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NODL-NEXT:    s_load_dword s2, s[4:5], 0x0
 ; GFX9-NODL-NEXT:    s_load_dword s3, s[6:7], 0x0
-; GFX9-NODL-NEXT:    s_load_dword s4, s[0:1], 0x0
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NODL-NEXT:    s_load_dword s10, s[0:1], 0x0
 ; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT:    s_sext_i32_i8 s0, s2
-; GFX9-NODL-NEXT:    s_sext_i32_i8 s1, s3
-; GFX9-NODL-NEXT:    s_bfe_i32 s6, s3, 0x80008
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s1
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s4
-; GFX9-NODL-NEXT:    s_bfe_i32 s5, s2, 0x80008
-; GFX9-NODL-NEXT:    v_mad_i32_i24 v3, s0, v2, v3
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v4, s6
-; GFX9-NODL-NEXT:    s_bfe_i32 s8, s3, 0x80010
-; GFX9-NODL-NEXT:    v_mad_i32_i24 v3, s5, v4, v3
-; GFX9-NODL-NEXT:    s_bfe_i32 s7, s2, 0x80010
-; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s0, v2, v3
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s8
+; GFX9-NODL-NEXT:    s_sext_i32_i8 s4, s2
+; GFX9-NODL-NEXT:    s_sext_i32_i8 s5, s3
+; GFX9-NODL-NEXT:    s_bfe_i32 s7, s3, 0x80008
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s5
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s10
+; GFX9-NODL-NEXT:    s_bfe_i32 s6, s2, 0x80008
+; GFX9-NODL-NEXT:    v_mad_i32_i24 v1, s4, v0, v1
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s7
+; GFX9-NODL-NEXT:    s_bfe_i32 s9, s3, 0x80010
+; GFX9-NODL-NEXT:    v_mad_i32_i24 v1, s6, v2, v1
+; GFX9-NODL-NEXT:    s_bfe_i32 s8, s2, 0x80010
+; GFX9-NODL-NEXT:    v_mad_i32_i24 v0, s4, v0, v1
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s9
 ; GFX9-NODL-NEXT:    s_ashr_i32 s3, s3, 24
-; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s7, v3, v2
+; GFX9-NODL-NEXT:    v_mad_i32_i24 v0, s8, v1, v0
 ; GFX9-NODL-NEXT:    s_ashr_i32 s2, s2, 24
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s3
-; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s2, v3, v2
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s2, v1, v0
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-NODL-NEXT:    s_endpgm
 ;
@@ -614,28 +614,28 @@ define amdgpu_kernel void @idot4_multiuse_mul1(<4 x i8> addrspace(1)* %src1,
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
 ; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
-; GFX9-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
-; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-DL-NEXT:    s_load_dword s10, s[0:1], 0x0
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    s_sext_i32_i8 s0, s2
-; GFX9-DL-NEXT:    s_sext_i32_i8 s1, s3
-; GFX9-DL-NEXT:    s_bfe_i32 s6, s3, 0x80008
-; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s1
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s4
-; GFX9-DL-NEXT:    s_bfe_i32 s5, s2, 0x80008
-; GFX9-DL-NEXT:    v_mad_i32_i24 v3, s0, v2, v3
-; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s6
-; GFX9-DL-NEXT:    s_bfe_i32 s8, s3, 0x80010
-; GFX9-DL-NEXT:    v_mad_i32_i24 v3, s5, v4, v3
-; GFX9-DL-NEXT:    s_bfe_i32 s7, s2, 0x80010
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s0, v2, v3
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s8
+; GFX9-DL-NEXT:    s_sext_i32_i8 s4, s2
+; GFX9-DL-NEXT:    s_sext_i32_i8 s5, s3
+; GFX9-DL-NEXT:    s_bfe_i32 s7, s3, 0x80008
+; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s5
+; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s10
+; GFX9-DL-NEXT:    s_bfe_i32 s6, s2, 0x80008
+; GFX9-DL-NEXT:    v_mad_i32_i24 v1, s4, v0, v1
+; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s7
+; GFX9-DL-NEXT:    s_bfe_i32 s9, s3, 0x80010
+; GFX9-DL-NEXT:    v_mad_i32_i24 v1, s6, v2, v1
+; GFX9-DL-NEXT:    s_bfe_i32 s8, s2, 0x80010
+; GFX9-DL-NEXT:    v_mad_i32_i24 v0, s4, v0, v1
+; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s9
 ; GFX9-DL-NEXT:    s_ashr_i32 s3, s3, 24
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s7, v3, v2
+; GFX9-DL-NEXT:    v_mad_i32_i24 v0, s8, v1, v0
 ; GFX9-DL-NEXT:    s_ashr_i32 s2, s2, 24
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s3
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s2, v3, v2
+; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s2, v1, v0
+; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-DL-NEXT:    s_endpgm
 ;
@@ -648,23 +648,23 @@ define amdgpu_kernel void @idot4_multiuse_mul1(<4 x i8> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
 ; GFX10-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
 ; GFX10-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
-; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_sext_i32_i8 s0, s2
-; GFX10-DL-NEXT:    s_sext_i32_i8 s1, s3
-; GFX10-DL-NEXT:    v_mov_b32_e32 v2, s4
+; GFX10-DL-NEXT:    s_sext_i32_i8 s5, s2
+; GFX10-DL-NEXT:    s_sext_i32_i8 s6, s3
+; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s4
 ; GFX10-DL-NEXT:    s_bfe_i32 s4, s2, 0x80008
-; GFX10-DL-NEXT:    s_bfe_i32 s5, s3, 0x80008
-; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s0, s1, v2
-; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s4, s5, v2
+; GFX10-DL-NEXT:    s_bfe_i32 s7, s3, 0x80008
+; GFX10-DL-NEXT:    v_mad_i32_i24 v0, s5, s6, v0
+; GFX10-DL-NEXT:    v_mad_i32_i24 v0, s4, s7, v0
 ; GFX10-DL-NEXT:    s_bfe_i32 s4, s2, 0x80010
-; GFX10-DL-NEXT:    s_bfe_i32 s5, s3, 0x80010
-; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s0, s1, v2
-; GFX10-DL-NEXT:    s_ashr_i32 s0, s2, 24
-; GFX10-DL-NEXT:    s_ashr_i32 s1, s3, 24
-; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s4, s5, v2
-; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s0, s1, v2
+; GFX10-DL-NEXT:    s_bfe_i32 s7, s3, 0x80010
+; GFX10-DL-NEXT:    s_ashr_i32 s2, s2, 24
+; GFX10-DL-NEXT:    s_ashr_i32 s3, s3, 24
+; GFX10-DL-NEXT:    v_mad_i32_i24 v0, s5, s6, v0
+; GFX10-DL-NEXT:    v_mad_i32_i24 v0, s4, s7, v0
+; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s2, s3, v0
+; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX10-DL-NEXT:    s_endpgm
                                                <4 x i8> addrspace(1)* %src2,
@@ -719,25 +719,25 @@ define amdgpu_kernel void @idot4_acc32_vecMul(<4 x i8> addrspace(1)* %src1,
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
-; GFX7-NEXT:    s_load_dword s6, s[0:1], 0x0
+; GFX7-NEXT:    s_load_dword s12, s[0:1], 0x0
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_ashr_i32 s7, s4, 24
-; GFX7-NEXT:    s_ashr_i32 s10, s5, 24
-; GFX7-NEXT:    s_bfe_i32 s11, s5, 0x80010
-; GFX7-NEXT:    s_bfe_i32 s12, s5, 0x80008
+; GFX7-NEXT:    s_ashr_i32 s6, s4, 24
+; GFX7-NEXT:    s_ashr_i32 s9, s5, 24
+; GFX7-NEXT:    s_bfe_i32 s10, s5, 0x80010
+; GFX7-NEXT:    s_bfe_i32 s11, s5, 0x80008
 ; GFX7-NEXT:    s_sext_i32_i8 s5, s5
-; GFX7-NEXT:    s_bfe_i32 s8, s4, 0x80010
-; GFX7-NEXT:    s_bfe_i32 s9, s4, 0x80008
+; GFX7-NEXT:    s_bfe_i32 s7, s4, 0x80010
+; GFX7-NEXT:    s_bfe_i32 s8, s4, 0x80008
 ; GFX7-NEXT:    s_sext_i32_i8 s4, s4
 ; GFX7-NEXT:    v_mov_b32_e32 v0, s5
-; GFX7-NEXT:    v_mov_b32_e32 v1, s6
-; GFX7-NEXT:    v_mad_i32_i24 v0, s4, v0, v1
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s12
-; GFX7-NEXT:    v_mad_i32_i24 v0, s9, v1, v0
+; GFX7-NEXT:    v_mad_i32_i24 v0, s4, v0, v1
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s11
 ; GFX7-NEXT:    v_mad_i32_i24 v0, s8, v1, v0
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s10
 ; GFX7-NEXT:    v_mad_i32_i24 v0, s7, v1, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s9
+; GFX7-NEXT:    v_mad_i32_i24 v0, s6, v1, v0
 ; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; GFX7-NEXT:    s_endpgm
 ;
@@ -748,28 +748,28 @@ define amdgpu_kernel void @idot4_acc32_vecMul(<4 x i8> addrspace(1)* %src1,
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
 ; GFX8-NEXT:    s_load_dword s3, s[6:7], 0x0
-; GFX8-NEXT:    s_load_dword s4, s[0:1], 0x0
-; GFX8-NEXT:    v_mov_b32_e32 v0, s0
-; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    s_load_dword s8, s[0:1], 0x0
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_lshrrev_b16_e64 v2, 8, s2
-; GFX8-NEXT:    v_lshrrev_b16_e64 v3, 8, s3
-; GFX8-NEXT:    s_ashr_i32 s5, s3, 24
-; GFX8-NEXT:    s_bfe_i32 s6, s3, 0x80010
+; GFX8-NEXT:    v_lshrrev_b16_e64 v0, 8, s2
+; GFX8-NEXT:    v_lshrrev_b16_e64 v1, 8, s3
+; GFX8-NEXT:    s_ashr_i32 s6, s3, 24
+; GFX8-NEXT:    s_bfe_i32 s7, s3, 0x80010
 ; GFX8-NEXT:    s_sext_i32_i8 s3, s3
-; GFX8-NEXT:    s_ashr_i32 s0, s2, 24
-; GFX8-NEXT:    s_bfe_i32 s1, s2, 0x80010
+; GFX8-NEXT:    s_ashr_i32 s4, s2, 24
+; GFX8-NEXT:    s_bfe_i32 s5, s2, 0x80010
 ; GFX8-NEXT:    s_sext_i32_i8 s2, s2
-; GFX8-NEXT:    v_mov_b32_e32 v4, s3
-; GFX8-NEXT:    v_mov_b32_e32 v5, s4
-; GFX8-NEXT:    v_bfe_i32 v2, v2, 0, 8
-; GFX8-NEXT:    v_bfe_i32 v3, v3, 0, 8
-; GFX8-NEXT:    v_mad_i32_i24 v4, s2, v4, v5
-; GFX8-NEXT:    v_mad_i32_i24 v2, v2, v3, v4
-; GFX8-NEXT:    v_mov_b32_e32 v3, s6
-; GFX8-NEXT:    v_mad_i32_i24 v2, s1, v3, v2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s5
-; GFX8-NEXT:    v_mad_i32_i24 v2, s0, v3, v2
+; GFX8-NEXT:    v_mov_b32_e32 v2, s3
+; GFX8-NEXT:    v_mov_b32_e32 v3, s8
+; GFX8-NEXT:    v_bfe_i32 v0, v0, 0, 8
+; GFX8-NEXT:    v_bfe_i32 v1, v1, 0, 8
+; GFX8-NEXT:    v_mad_i32_i24 v2, s2, v2, v3
+; GFX8-NEXT:    v_mad_i32_i24 v0, v0, v1, v2
+; GFX8-NEXT:    v_mov_b32_e32 v1, s7
+; GFX8-NEXT:    v_mad_i32_i24 v0, s5, v1, v0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s6
+; GFX8-NEXT:    v_mad_i32_i24 v2, s4, v1, v0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX8-NEXT:    flat_store_dword v[0:1], v2
 ; GFX8-NEXT:    s_endpgm
 ;
@@ -780,28 +780,28 @@ define amdgpu_kernel void @idot4_acc32_vecMul(<4 x i8> addrspace(1)* %src1,
 ; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NODL-NEXT:    s_load_dword s2, s[4:5], 0x0
 ; GFX9-NODL-NEXT:    s_load_dword s3, s[6:7], 0x0
-; GFX9-NODL-NEXT:    s_load_dword s4, s[0:1], 0x0
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NODL-NEXT:    s_load_dword s8, s[0:1], 0x0
 ; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT:    v_lshrrev_b16_e64 v2, 8, s2
-; GFX9-NODL-NEXT:    v_lshrrev_b16_e64 v3, 8, s3
-; GFX9-NODL-NEXT:    s_ashr_i32 s5, s3, 24
-; GFX9-NODL-NEXT:    s_bfe_i32 s6, s3, 0x80010
+; GFX9-NODL-NEXT:    v_lshrrev_b16_e64 v0, 8, s2
+; GFX9-NODL-NEXT:    v_lshrrev_b16_e64 v1, 8, s3
+; GFX9-NODL-NEXT:    s_ashr_i32 s6, s3, 24
+; GFX9-NODL-NEXT:    s_bfe_i32 s7, s3, 0x80010
 ; GFX9-NODL-NEXT:    s_sext_i32_i8 s3, s3
-; GFX9-NODL-NEXT:    s_ashr_i32 s0, s2, 24
-; GFX9-NODL-NEXT:    s_bfe_i32 s1, s2, 0x80010
+; GFX9-NODL-NEXT:    s_ashr_i32 s4, s2, 24
+; GFX9-NODL-NEXT:    s_bfe_i32 s5, s2, 0x80010
 ; GFX9-NODL-NEXT:    s_sext_i32_i8 s2, s2
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v4, s3
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v5, s4
-; GFX9-NODL-NEXT:    v_bfe_i32 v2, v2, 0, 8
-; GFX9-NODL-NEXT:    v_bfe_i32 v3, v3, 0, 8
-; GFX9-NODL-NEXT:    v_mad_i32_i24 v4, s2, v4, v5
-; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, v2, v3, v4
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s6
-; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s1, v3, v2
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s5
-; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s0, v3, v2
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s3
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s8
+; GFX9-NODL-NEXT:    v_bfe_i32 v0, v0, 0, 8
+; GFX9-NODL-NEXT:    v_bfe_i32 v1, v1, 0, 8
+; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s2, v2, v3
+; GFX9-NODL-NEXT:    v_mad_i32_i24 v0, v0, v1, v2
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s7
+; GFX9-NODL-NEXT:    v_mad_i32_i24 v0, s5, v1, v0
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s6
+; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s4, v1, v0
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-NODL-NEXT:    s_endpgm
 ;
@@ -812,28 +812,28 @@ define amdgpu_kernel void @idot4_acc32_vecMul(<4 x i8> addrspace(1)* %src1,
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
 ; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
-; GFX9-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
-; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-DL-NEXT:    s_load_dword s8, s[0:1], 0x0
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    v_lshrrev_b16_e64 v2, 8, s2
-; GFX9-DL-NEXT:    v_lshrrev_b16_e64 v3, 8, s3
-; GFX9-DL-NEXT:    s_ashr_i32 s5, s3, 24
-; GFX9-DL-NEXT:    s_bfe_i32 s6, s3, 0x80010
+; GFX9-DL-NEXT:    v_lshrrev_b16_e64 v0, 8, s2
+; GFX9-DL-NEXT:    v_lshrrev_b16_e64 v1, 8, s3
+; GFX9-DL-NEXT:    s_ashr_i32 s6, s3, 24
+; GFX9-DL-NEXT:    s_bfe_i32 s7, s3, 0x80010
 ; GFX9-DL-NEXT:    s_sext_i32_i8 s3, s3
-; GFX9-DL-NEXT:    s_ashr_i32 s0, s2, 24
-; GFX9-DL-NEXT:    s_bfe_i32 s1, s2, 0x80010
+; GFX9-DL-NEXT:    s_ashr_i32 s4, s2, 24
+; GFX9-DL-NEXT:    s_bfe_i32 s5, s2, 0x80010
 ; GFX9-DL-NEXT:    s_sext_i32_i8 s2, s2
-; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s3
-; GFX9-DL-NEXT:    v_mov_b32_e32 v5, s4
-; GFX9-DL-NEXT:    v_bfe_i32 v2, v2, 0, 8
-; GFX9-DL-NEXT:    v_bfe_i32 v3, v3, 0, 8
-; GFX9-DL-NEXT:    v_mad_i32_i24 v4, s2, v4, v5
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, v2, v3, v4
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s6
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s1, v3, v2
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s5
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s0, v3, v2
+; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s3
+; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s8
+; GFX9-DL-NEXT:    v_bfe_i32 v0, v0, 0, 8
+; GFX9-DL-NEXT:    v_bfe_i32 v1, v1, 0, 8
+; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s2, v2, v3
+; GFX9-DL-NEXT:    v_mad_i32_i24 v0, v0, v1, v2
+; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s7
+; GFX9-DL-NEXT:    v_mad_i32_i24 v0, s5, v1, v0
+; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s6
+; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s4, v1, v0
+; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-DL-NEXT:    s_endpgm
 ;
@@ -841,30 +841,29 @@ define amdgpu_kernel void @idot4_acc32_vecMul(<4 x i8> addrspace(1)* %src1,
 ; GFX10-DL:       ; %bb.0: ; %entry
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX10-DL-NEXT:    v_mov_b32_e32 v2, 0xffff
 ; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
 ; GFX10-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
 ; GFX10-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
-; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    v_and_b32_sdwa v3, s2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
-; GFX10-DL-NEXT:    v_and_b32_sdwa v2, s3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
-; GFX10-DL-NEXT:    v_mov_b32_e32 v4, s4
-; GFX10-DL-NEXT:    s_sext_i32_i8 s0, s2
-; GFX10-DL-NEXT:    s_sext_i32_i8 s1, s3
-; GFX10-DL-NEXT:    v_bfe_i32 v3, v3, 0, 8
-; GFX10-DL-NEXT:    v_bfe_i32 v2, v2, 0, 8
+; GFX10-DL-NEXT:    v_lshrrev_b16_e64 v0, 8, s2
+; GFX10-DL-NEXT:    v_lshrrev_b16_e64 v1, 8, s3
+; GFX10-DL-NEXT:    v_mov_b32_e32 v2, s4
+; GFX10-DL-NEXT:    s_sext_i32_i8 s5, s2
+; GFX10-DL-NEXT:    s_sext_i32_i8 s6, s3
+; GFX10-DL-NEXT:    v_bfe_i32 v0, v0, 0, 8
+; GFX10-DL-NEXT:    v_bfe_i32 v1, v1, 0, 8
 ; GFX10-DL-NEXT:    s_bfe_i32 s4, s2, 0x80010
+; GFX10-DL-NEXT:    s_ashr_i32 s2, s2, 24
+; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s5, s6, v2
 ; GFX10-DL-NEXT:    s_bfe_i32 s5, s3, 0x80010
-; GFX10-DL-NEXT:    v_mad_i32_i24 v4, s0, s1, v4
-; GFX10-DL-NEXT:    s_ashr_i32 s0, s2, 24
-; GFX10-DL-NEXT:    s_ashr_i32 s1, s3, 24
-; GFX10-DL-NEXT:    v_mad_i32_i24 v2, v3, v2, v4
-; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s4, s5, v2
-; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s0, s1, v2
+; GFX10-DL-NEXT:    s_ashr_i32 s3, s3, 24
+; GFX10-DL-NEXT:    v_mad_i32_i24 v0, v0, v1, v2
+; GFX10-DL-NEXT:    v_mad_i32_i24 v0, s4, s5, v0
+; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s2, s3, v0
+; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX10-DL-NEXT:    s_endpgm
                                               <4 x i8> addrspace(1)* %src2,
@@ -899,41 +898,28 @@ define amdgpu_kernel void @idot4_acc16_vecMul(<4 x i8> addrspace(1)* %src1,
 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
 ; GFX7-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX7-NEXT:    s_mov_b32 s2, -1
-; GFX7-NEXT:    s_mov_b32 s8, 0xffff
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
 ; GFX7-NEXT:    buffer_load_ushort v0, off, s[0:3], 0
 ; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_sext_i32_i8 s6, s4
-; GFX7-NEXT:    s_bfe_i32 s7, s4, 0x80008
-; GFX7-NEXT:    s_sext_i32_i8 s10, s5
+; GFX7-NEXT:    s_ashr_i32 s6, s4, 24
+; GFX7-NEXT:    s_bfe_i32 s7, s4, 0x80010
+; GFX7-NEXT:    s_bfe_i32 s10, s5, 0x80010
 ; GFX7-NEXT:    s_bfe_i32 s11, s5, 0x80008
-; GFX7-NEXT:    s_bfe_i32 s12, s5, 0x80010
-; GFX7-NEXT:    s_ashr_i32 s5, s5, 24
-; GFX7-NEXT:    v_mov_b32_e32 v3, s11
-; GFX7-NEXT:    v_mov_b32_e32 v4, s10
-; GFX7-NEXT:    s_bfe_i32 s9, s4, 0x80010
-; GFX7-NEXT:    v_mov_b32_e32 v2, s12
-; GFX7-NEXT:    s_ashr_i32 s4, s4, 24
+; GFX7-NEXT:    s_ashr_i32 s9, s5, 24
+; GFX7-NEXT:    s_sext_i32_i8 s5, s5
+; GFX7-NEXT:    s_bfe_i32 s8, s4, 0x80008
+; GFX7-NEXT:    s_sext_i32_i8 s4, s4
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s5
-; GFX7-NEXT:    v_mul_i32_i24_e32 v1, s4, v1
-; GFX7-NEXT:    v_mul_i32_i24_e32 v2, s9, v2
-; GFX7-NEXT:    v_mul_i32_i24_e32 v3, s7, v3
-; GFX7-NEXT:    v_mul_i32_i24_e32 v4, s6, v4
-; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX7-NEXT:    v_and_b32_e32 v2, s8, v2
-; GFX7-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX7-NEXT:    v_and_b32_e32 v4, s8, v4
-; GFX7-NEXT:    v_or_b32_e32 v1, v2, v1
-; GFX7-NEXT:    v_or_b32_e32 v2, v4, v3
-; GFX7-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
-; GFX7-NEXT:    v_lshrrev_b32_e32 v4, 16, v1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s11
+; GFX7-NEXT:    v_mov_b32_e32 v3, s10
 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v3, v0
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v4, v0
+; GFX7-NEXT:    v_mad_i32_i24 v0, s4, v1, v0
+; GFX7-NEXT:    v_mad_i32_i24 v0, s8, v2, v0
+; GFX7-NEXT:    v_mad_i32_i24 v0, s7, v3, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s9
+; GFX7-NEXT:    v_mad_i32_i24 v0, s6, v1, v0
 ; GFX7-NEXT:    buffer_store_short v0, off, s[0:3], 0
 ; GFX7-NEXT:    s_endpgm
 ;
@@ -941,33 +927,33 @@ define amdgpu_kernel void @idot4_acc16_vecMul(<4 x i8> addrspace(1)* %src1,
 ; GFX8:       ; %bb.0: ; %entry
 ; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX8-NEXT:    s_mov_b32 s2, 0xffff
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_load_dword s3, s[6:7], 0x0
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX8-NEXT:    flat_load_ushort v2, v[0:1]
-; GFX8-NEXT:    s_load_dword s1, s[4:5], 0x0
-; GFX8-NEXT:    s_load_dword s2, s[6:7], 0x0
-; GFX8-NEXT:    s_mov_b32 s0, 0xffff
+; GFX8-NEXT:    s_load_dword s0, s[4:5], 0x0
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_lshr_b32 s3, s1, 16
-; GFX8-NEXT:    s_bfe_i32 s6, s2, 0x80000
-; GFX8-NEXT:    s_lshr_b32 s4, s2, 16
-; GFX8-NEXT:    s_bfe_i32 s5, s1, 0x80000
-; GFX8-NEXT:    v_ashrrev_i16_e64 v4, 8, s1
-; GFX8-NEXT:    s_bfe_i32 s1, s3, 0x80000
-; GFX8-NEXT:    v_ashrrev_i16_e64 v6, 8, s3
-; GFX8-NEXT:    s_and_b32 s3, s0, s6
-; GFX8-NEXT:    v_ashrrev_i16_e64 v3, 8, s2
-; GFX8-NEXT:    s_bfe_i32 s2, s4, 0x80000
+; GFX8-NEXT:    s_bfe_i32 s6, s3, 0x80000
+; GFX8-NEXT:    s_lshr_b32 s4, s3, 16
+; GFX8-NEXT:    v_ashrrev_i16_e64 v3, 8, s3
+; GFX8-NEXT:    s_bfe_i32 s3, s4, 0x80000
+; GFX8-NEXT:    s_lshr_b32 s1, s0, 16
+; GFX8-NEXT:    s_bfe_i32 s5, s0, 0x80000
+; GFX8-NEXT:    v_ashrrev_i16_e64 v4, 8, s0
+; GFX8-NEXT:    s_bfe_i32 s0, s1, 0x80000
+; GFX8-NEXT:    v_ashrrev_i16_e64 v6, 8, s1
+; GFX8-NEXT:    s_and_b32 s1, s2, s6
 ; GFX8-NEXT:    v_ashrrev_i16_e64 v5, 8, s4
-; GFX8-NEXT:    s_and_b32 s4, s0, s5
-; GFX8-NEXT:    v_mov_b32_e32 v7, s3
-; GFX8-NEXT:    s_and_b32 s2, s0, s2
-; GFX8-NEXT:    s_and_b32 s0, s0, s1
+; GFX8-NEXT:    s_and_b32 s4, s2, s5
+; GFX8-NEXT:    v_mov_b32_e32 v7, s1
+; GFX8-NEXT:    s_and_b32 s3, s2, s3
+; GFX8-NEXT:    s_and_b32 s0, s2, s0
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    v_mad_u32_u24 v2, s4, v7, v2
 ; GFX8-NEXT:    v_mad_u32_u24 v2, v4, v3, v2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s2
+; GFX8-NEXT:    v_mov_b32_e32 v3, s3
 ; GFX8-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
 ; GFX8-NEXT:    v_mad_u32_u24 v2, v6, v5, v2
 ; GFX8-NEXT:    flat_store_short v[0:1], v2
@@ -1060,31 +1046,31 @@ define amdgpu_kernel void @idot4_acc16_vecMul(<4 x i8> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v2, 0xffff
 ; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX10-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
+; GFX10-DL-NEXT:    s_load_dword s1, s[6:7], 0x0
 ; GFX10-DL-NEXT:    global_load_ushort v3, v[0:1], off
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-DL-NEXT:    s_bfe_i32 s4, s0, 0x80000
+; GFX10-DL-NEXT:    s_bfe_i32 s3, s1, 0x80000
+; GFX10-DL-NEXT:    s_lshr_b32 s2, s0, 16
+; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v4, 8, s0
+; GFX10-DL-NEXT:    s_lshr_b32 s5, s1, 16
+; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v5, 8, s1
+; GFX10-DL-NEXT:    v_and_b32_e32 v6, s3, v2
+; GFX10-DL-NEXT:    v_and_b32_e32 v7, s4, v2
 ; GFX10-DL-NEXT:    s_bfe_i32 s0, s2, 0x80000
-; GFX10-DL-NEXT:    s_bfe_i32 s1, s3, 0x80000
-; GFX10-DL-NEXT:    s_lshr_b32 s4, s2, 16
-; GFX10-DL-NEXT:    s_lshr_b32 s5, s3, 16
-; GFX10-DL-NEXT:    v_and_b32_sdwa v4, sext(s2), v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
-; GFX10-DL-NEXT:    v_and_b32_e32 v7, s0, v2
-; GFX10-DL-NEXT:    v_and_b32_e32 v6, s1, v2
-; GFX10-DL-NEXT:    v_and_b32_sdwa v5, sext(s3), v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
-; GFX10-DL-NEXT:    s_bfe_i32 s0, s4, 0x80000
 ; GFX10-DL-NEXT:    s_bfe_i32 s1, s5, 0x80000
-; GFX10-DL-NEXT:    v_lshl_or_b32 v4, v4, 16, v7
-; GFX10-DL-NEXT:    v_and_b32_sdwa v8, sext(s4), v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v11, 8, s2
 ; GFX10-DL-NEXT:    v_lshl_or_b32 v5, v5, 16, v6
-; GFX10-DL-NEXT:    v_and_b32_sdwa v6, sext(s5), v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
-; GFX10-DL-NEXT:    v_and_b32_e32 v7, s1, v2
+; GFX10-DL-NEXT:    v_lshl_or_b32 v4, v4, 16, v7
+; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v7, 8, s5
+; GFX10-DL-NEXT:    v_and_b32_e32 v8, s1, v2
 ; GFX10-DL-NEXT:    v_and_b32_e32 v2, s0, v2
 ; GFX10-DL-NEXT:    v_pk_mul_lo_u16 v4, v4, v5
-; GFX10-DL-NEXT:    v_lshl_or_b32 v5, v6, 16, v7
-; GFX10-DL-NEXT:    v_lshl_or_b32 v2, v8, 16, v2
+; GFX10-DL-NEXT:    v_lshl_or_b32 v5, v7, 16, v8
+; GFX10-DL-NEXT:    v_lshl_or_b32 v2, v11, 16, v2
 ; GFX10-DL-NEXT:    v_pk_mul_lo_u16 v2, v2, v5
 ; GFX10-DL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-DL-NEXT:    v_add_nc_u32_e32 v3, v4, v3
diff --git a/llvm/test/CodeGen/AMDGPU/idot4u.ll b/llvm/test/CodeGen/AMDGPU/idot4u.ll
index 2af14ec6b..6172d54b8 100644
--- a/llvm/test/CodeGen/AMDGPU/idot4u.ll
+++ b/llvm/test/CodeGen/AMDGPU/idot4u.ll
@@ -17,22 +17,22 @@ define amdgpu_kernel void @udot4_acc32(<4 x i8> addrspace(1)* %src1,
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
-; GFX7-NEXT:    s_load_dword s6, s[0:1], 0x0
+; GFX7-NEXT:    s_load_dword s12, s[0:1], 0x0
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_and_b32 s7, s4, s8
-; GFX7-NEXT:    s_and_b32 s8, s5, s8
-; GFX7-NEXT:    s_bfe_u32 s10, s5, 0x80008
-; GFX7-NEXT:    v_mov_b32_e32 v0, s8
-; GFX7-NEXT:    v_mov_b32_e32 v1, s6
-; GFX7-NEXT:    s_bfe_u32 s12, s5, 0x80010
-; GFX7-NEXT:    v_mad_u32_u24 v0, s7, v0, v1
-; GFX7-NEXT:    s_bfe_u32 s9, s4, 0x80008
-; GFX7-NEXT:    v_mov_b32_e32 v1, s10
-; GFX7-NEXT:    s_bfe_u32 s11, s4, 0x80010
-; GFX7-NEXT:    v_mad_u32_u24 v0, s9, v1, v0
+; GFX7-NEXT:    s_and_b32 s6, s4, s8
+; GFX7-NEXT:    s_and_b32 s7, s5, s8
+; GFX7-NEXT:    s_bfe_u32 s9, s5, 0x80008
+; GFX7-NEXT:    v_mov_b32_e32 v0, s7
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s12
+; GFX7-NEXT:    s_bfe_u32 s11, s5, 0x80010
+; GFX7-NEXT:    v_mad_u32_u24 v0, s6, v0, v1
+; GFX7-NEXT:    s_bfe_u32 s8, s4, 0x80008
+; GFX7-NEXT:    v_mov_b32_e32 v1, s9
+; GFX7-NEXT:    s_bfe_u32 s10, s4, 0x80010
+; GFX7-NEXT:    v_mad_u32_u24 v0, s8, v1, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s11
 ; GFX7-NEXT:    s_lshr_b32 s5, s5, 24
-; GFX7-NEXT:    v_mad_u32_u24 v0, s11, v1, v0
+; GFX7-NEXT:    v_mad_u32_u24 v0, s10, v1, v0
 ; GFX7-NEXT:    s_lshr_b32 s4, s4, 24
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s5
 ; GFX7-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
@@ -47,22 +47,22 @@ define amdgpu_kernel void @udot4_acc32(<4 x i8> addrspace(1)* %src1,
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    s_load_dword s3, s[4:5], 0x0
 ; GFX8-NEXT:    s_load_dword s4, s[6:7], 0x0
-; GFX8-NEXT:    s_load_dword s5, s[0:1], 0x0
+; GFX8-NEXT:    s_load_dword s10, s[0:1], 0x0
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_and_b32 s6, s3, s2
+; GFX8-NEXT:    s_and_b32 s5, s3, s2
 ; GFX8-NEXT:    s_and_b32 s2, s4, s2
-; GFX8-NEXT:    s_bfe_u32 s8, s4, 0x80008
+; GFX8-NEXT:    s_bfe_u32 s7, s4, 0x80008
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s2
-; GFX8-NEXT:    v_mov_b32_e32 v1, s5
-; GFX8-NEXT:    s_bfe_u32 s10, s4, 0x80010
-; GFX8-NEXT:    v_mad_u32_u24 v0, s6, v0, v1
-; GFX8-NEXT:    s_bfe_u32 s7, s3, 0x80008
-; GFX8-NEXT:    v_mov_b32_e32 v1, s8
-; GFX8-NEXT:    s_bfe_u32 s9, s3, 0x80010
-; GFX8-NEXT:    v_mad_u32_u24 v0, s7, v1, v0
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s10
+; GFX8-NEXT:    s_bfe_u32 s9, s4, 0x80010
+; GFX8-NEXT:    v_mad_u32_u24 v0, s5, v0, v1
+; GFX8-NEXT:    s_bfe_u32 s6, s3, 0x80008
+; GFX8-NEXT:    v_mov_b32_e32 v1, s7
+; GFX8-NEXT:    s_bfe_u32 s8, s3, 0x80010
+; GFX8-NEXT:    v_mad_u32_u24 v0, s6, v1, v0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s9
 ; GFX8-NEXT:    s_lshr_b32 s4, s4, 24
-; GFX8-NEXT:    v_mad_u32_u24 v0, s9, v1, v0
+; GFX8-NEXT:    v_mad_u32_u24 v0, s8, v1, v0
 ; GFX8-NEXT:    s_lshr_b32 s3, s3, 24
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s4
 ; GFX8-NEXT:    v_mad_u32_u24 v2, s3, v1, v0
@@ -79,22 +79,22 @@ define amdgpu_kernel void @udot4_acc32(<4 x i8> addrspace(1)* %src1,
 ; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NODL-NEXT:    s_load_dword s3, s[4:5], 0x0
 ; GFX9-NODL-NEXT:    s_load_dword s4, s[6:7], 0x0
-; GFX9-NODL-NEXT:    s_load_dword s5, s[0:1], 0x0
+; GFX9-NODL-NEXT:    s_load_dword s10, s[0:1], 0x0
 ; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT:    s_and_b32 s6, s3, s2
+; GFX9-NODL-NEXT:    s_and_b32 s5, s3, s2
 ; GFX9-NODL-NEXT:    s_and_b32 s2, s4, s2
-; GFX9-NODL-NEXT:    s_bfe_u32 s8, s4, 0x80008
+; GFX9-NODL-NEXT:    s_bfe_u32 s7, s4, 0x80008
 ; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s2
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s5
-; GFX9-NODL-NEXT:    s_bfe_u32 s10, s4, 0x80010
-; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s6, v0, v1
-; GFX9-NODL-NEXT:    s_bfe_u32 s7, s3, 0x80008
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s8
-; GFX9-NODL-NEXT:    s_bfe_u32 s9, s3, 0x80010
-; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s7, v1, v0
 ; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s10
+; GFX9-NODL-NEXT:    s_bfe_u32 s9, s4, 0x80010
+; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s5, v0, v1
+; GFX9-NODL-NEXT:    s_bfe_u32 s6, s3, 0x80008
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s7
+; GFX9-NODL-NEXT:    s_bfe_u32 s8, s3, 0x80010
+; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s6, v1, v0
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s9
 ; GFX9-NODL-NEXT:    s_lshr_b32 s4, s4, 24
-; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s9, v1, v0
+; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s8, v1, v0
 ; GFX9-NODL-NEXT:    s_lshr_b32 s3, s3, 24
 ; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s4
 ; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s3, v1, v0
@@ -108,15 +108,15 @@ define amdgpu_kernel void @udot4_acc32(<4 x i8> addrspace(1)* %src1,
 ; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
-; GFX9-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
+; GFX9-DL-NEXT:    s_load_dword s2, s[6:7], 0x0
+; GFX9-DL-NEXT:    s_load_dword s3, s[0:1], 0x0
+; GFX9-DL-NEXT:    s_load_dword s4, s[4:5], 0x0
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s3
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s4
-; GFX9-DL-NEXT:    v_dot4_u32_u8 v2, s2, v2, v3
+; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s2
+; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s3
+; GFX9-DL-NEXT:    v_dot4_u32_u8 v2, s4, v2, v3
 ; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-DL-NEXT:    s_endpgm
 ;
@@ -126,14 +126,14 @@ define amdgpu_kernel void @udot4_acc32(<4 x i8> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX10-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
-; GFX10-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
+; GFX10-DL-NEXT:    s_load_dword s2, s[0:1], 0x0
+; GFX10-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
+; GFX10-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
+; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-DL-NEXT:    v_dot4_u32_u8 v2, s3, s4, v0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
-; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    v_mov_b32_e32 v2, s4
-; GFX10-DL-NEXT:    v_dot4_u32_u8 v2, s2, s3, v2
 ; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX10-DL-NEXT:    s_endpgm
                                        <4 x i8> addrspace(1)* %src2,
@@ -183,28 +183,28 @@ define amdgpu_kernel void @udot4_acc16(<4 x i8> addrspace(1)* %src1,
 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
 ; GFX7-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX7-NEXT:    s_mov_b32 s2, -1
-; GFX7-NEXT:    s_movk_i32 s8, 0xff
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
 ; GFX7-NEXT:    buffer_load_ushort v0, off, s[0:3], 0
-; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
+; GFX7-NEXT:    s_load_dword s6, s[6:7], 0x0
+; GFX7-NEXT:    s_movk_i32 s5, 0xff
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_and_b32 s7, s4, s8
 ; GFX7-NEXT:    s_bfe_u32 s9, s4, 0x80008
-; GFX7-NEXT:    s_and_b32 s6, s5, s8
-; GFX7-NEXT:    s_bfe_u32 s8, s5, 0x80008
-; GFX7-NEXT:    v_mov_b32_e32 v1, s6
-; GFX7-NEXT:    s_bfe_u32 s10, s5, 0x80010
-; GFX7-NEXT:    v_mov_b32_e32 v2, s8
 ; GFX7-NEXT:    s_bfe_u32 s11, s4, 0x80010
-; GFX7-NEXT:    s_lshr_b32 s5, s5, 24
+; GFX7-NEXT:    s_and_b32 s7, s6, s5
+; GFX7-NEXT:    s_and_b32 s5, s4, s5
+; GFX7-NEXT:    s_bfe_u32 s8, s6, 0x80008
+; GFX7-NEXT:    v_mov_b32_e32 v1, s7
+; GFX7-NEXT:    s_bfe_u32 s10, s6, 0x80010
+; GFX7-NEXT:    v_mov_b32_e32 v2, s8
+; GFX7-NEXT:    s_lshr_b32 s6, s6, 24
 ; GFX7-NEXT:    v_mov_b32_e32 v3, s10
 ; GFX7-NEXT:    s_lshr_b32 s4, s4, 24
 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
-; GFX7-NEXT:    v_mad_u32_u24 v0, s7, v1, v0
+; GFX7-NEXT:    v_mad_u32_u24 v0, s5, v1, v0
 ; GFX7-NEXT:    v_mad_u32_u24 v0, s9, v2, v0
 ; GFX7-NEXT:    v_mad_u32_u24 v0, s11, v3, v0
-; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v1, s6
 ; GFX7-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
 ; GFX7-NEXT:    buffer_store_short v0, off, s[0:3], 0
 ; GFX7-NEXT:    s_endpgm
@@ -214,20 +214,20 @@ define amdgpu_kernel void @udot4_acc16(<4 x i8> addrspace(1)* %src1,
 ; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_load_dword s2, s[6:7], 0x0
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX8-NEXT:    flat_load_ushort v2, v[0:1]
 ; GFX8-NEXT:    s_load_dword s1, s[4:5], 0x0
-; GFX8-NEXT:    s_load_dword s2, s[6:7], 0x0
 ; GFX8-NEXT:    s_movk_i32 s0, 0xff
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_bfe_u32 s5, s2, 0x80008
+; GFX8-NEXT:    s_bfe_u32 s7, s2, 0x80010
+; GFX8-NEXT:    v_mov_b32_e32 v4, s5
 ; GFX8-NEXT:    s_and_b32 s3, s1, s0
 ; GFX8-NEXT:    s_and_b32 s0, s2, s0
-; GFX8-NEXT:    s_bfe_u32 s5, s2, 0x80008
 ; GFX8-NEXT:    v_mov_b32_e32 v3, s0
-; GFX8-NEXT:    s_bfe_u32 s7, s2, 0x80010
 ; GFX8-NEXT:    s_bfe_u32 s4, s1, 0x80008
-; GFX8-NEXT:    v_mov_b32_e32 v4, s5
 ; GFX8-NEXT:    s_bfe_u32 s6, s1, 0x80010
 ; GFX8-NEXT:    s_lshr_b32 s2, s2, 24
 ; GFX8-NEXT:    v_mov_b32_e32 v5, s7
@@ -246,20 +246,20 @@ define amdgpu_kernel void @udot4_acc16(<4 x i8> addrspace(1)* %src1,
 ; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NODL-NEXT:    s_load_dword s2, s[6:7], 0x0
 ; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-NODL-NEXT:    global_load_ushort v2, v[0:1], off
 ; GFX9-NODL-NEXT:    s_load_dword s1, s[4:5], 0x0
-; GFX9-NODL-NEXT:    s_load_dword s2, s[6:7], 0x0
 ; GFX9-NODL-NEXT:    s_movk_i32 s0, 0xff
 ; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NODL-NEXT:    s_bfe_u32 s5, s2, 0x80008
+; GFX9-NODL-NEXT:    s_bfe_u32 s7, s2, 0x80010
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v4, s5
 ; GFX9-NODL-NEXT:    s_and_b32 s3, s1, s0
 ; GFX9-NODL-NEXT:    s_and_b32 s0, s2, s0
-; GFX9-NODL-NEXT:    s_bfe_u32 s5, s2, 0x80008
 ; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s0
-; GFX9-NODL-NEXT:    s_bfe_u32 s7, s2, 0x80010
 ; GFX9-NODL-NEXT:    s_bfe_u32 s4, s1, 0x80008
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v4, s5
 ; GFX9-NODL-NEXT:    s_bfe_u32 s6, s1, 0x80010
 ; GFX9-NODL-NEXT:    s_lshr_b32 s2, s2, 24
 ; GFX9-NODL-NEXT:    v_mov_b32_e32 v5, s7
@@ -278,15 +278,15 @@ define amdgpu_kernel void @udot4_acc16(<4 x i8> addrspace(1)* %src1,
 ; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
+; GFX9-DL-NEXT:    s_load_dword s2, s[6:7], 0x0
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-DL-NEXT:    global_load_ushort v2, v[0:1], off
+; GFX9-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s3
+; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s2
 ; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-DL-NEXT:    v_dot4_u32_u8 v2, s2, v3, v2
+; GFX9-DL-NEXT:    v_dot4_u32_u8 v2, s0, v3, v2
 ; GFX9-DL-NEXT:    global_store_short v[0:1], v2, off
 ; GFX9-DL-NEXT:    s_endpgm
 ;
@@ -353,28 +353,28 @@ define amdgpu_kernel void @udot4_acc8(<4 x i8> addrspace(1)* %src1,
 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
 ; GFX7-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX7-NEXT:    s_mov_b32 s2, -1
-; GFX7-NEXT:    s_movk_i32 s8, 0xff
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
 ; GFX7-NEXT:    buffer_load_ubyte v0, off, s[0:3], 0
-; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
+; GFX7-NEXT:    s_load_dword s6, s[6:7], 0x0
+; GFX7-NEXT:    s_movk_i32 s5, 0xff
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_and_b32 s7, s4, s8
 ; GFX7-NEXT:    s_bfe_u32 s9, s4, 0x80008
-; GFX7-NEXT:    s_and_b32 s6, s5, s8
-; GFX7-NEXT:    s_bfe_u32 s8, s5, 0x80008
-; GFX7-NEXT:    v_mov_b32_e32 v1, s6
-; GFX7-NEXT:    s_bfe_u32 s10, s5, 0x80010
-; GFX7-NEXT:    v_mov_b32_e32 v2, s8
 ; GFX7-NEXT:    s_bfe_u32 s11, s4, 0x80010
-; GFX7-NEXT:    s_lshr_b32 s5, s5, 24
+; GFX7-NEXT:    s_and_b32 s7, s6, s5
+; GFX7-NEXT:    s_and_b32 s5, s4, s5
+; GFX7-NEXT:    s_bfe_u32 s8, s6, 0x80008
+; GFX7-NEXT:    v_mov_b32_e32 v1, s7
+; GFX7-NEXT:    s_bfe_u32 s10, s6, 0x80010
+; GFX7-NEXT:    v_mov_b32_e32 v2, s8
+; GFX7-NEXT:    s_lshr_b32 s6, s6, 24
 ; GFX7-NEXT:    v_mov_b32_e32 v3, s10
 ; GFX7-NEXT:    s_lshr_b32 s4, s4, 24
 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
-; GFX7-NEXT:    v_mad_u32_u24 v0, s7, v1, v0
+; GFX7-NEXT:    v_mad_u32_u24 v0, s5, v1, v0
 ; GFX7-NEXT:    v_mad_u32_u24 v0, s9, v2, v0
 ; GFX7-NEXT:    v_mad_u32_u24 v0, s11, v3, v0
-; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v1, s6
 ; GFX7-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
 ; GFX7-NEXT:    buffer_store_byte v0, off, s[0:3], 0
 ; GFX7-NEXT:    s_endpgm
@@ -383,31 +383,31 @@ define amdgpu_kernel void @udot4_acc8(<4 x i8> addrspace(1)* %src1,
 ; GFX8:       ; %bb.0: ; %entry
 ; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX8-NEXT:    s_movk_i32 s2, 0xff
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX8-NEXT:    flat_load_ubyte v2, v[0:1]
-; GFX8-NEXT:    s_load_dword s0, s[4:5], 0x0
 ; GFX8-NEXT:    s_load_dword s1, s[6:7], 0x0
+; GFX8-NEXT:    s_movk_i32 s0, 0xff
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_bfe_u32 s5, s0, 0x80008
-; GFX8-NEXT:    s_and_b32 s3, s1, s2
+; GFX8-NEXT:    s_bfe_u32 s5, s2, 0x80008
+; GFX8-NEXT:    s_bfe_u32 s7, s2, 0x80010
+; GFX8-NEXT:    s_and_b32 s3, s1, s0
+; GFX8-NEXT:    s_and_b32 s0, s2, s0
 ; GFX8-NEXT:    s_bfe_u32 s4, s1, 0x80008
-; GFX8-NEXT:    s_and_b32 s2, s0, s2
 ; GFX8-NEXT:    v_mov_b32_e32 v3, s3
 ; GFX8-NEXT:    s_bfe_u32 s6, s1, 0x80010
 ; GFX8-NEXT:    v_mov_b32_e32 v4, s4
-; GFX8-NEXT:    s_bfe_u32 s7, s0, 0x80010
 ; GFX8-NEXT:    s_lshr_b32 s1, s1, 24
 ; GFX8-NEXT:    v_mov_b32_e32 v5, s6
-; GFX8-NEXT:    s_lshr_b32 s0, s0, 24
+; GFX8-NEXT:    s_lshr_b32 s2, s2, 24
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
-; GFX8-NEXT:    v_mad_u32_u24 v2, s2, v3, v2
+; GFX8-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
 ; GFX8-NEXT:    v_mad_u32_u24 v2, s5, v4, v2
 ; GFX8-NEXT:    v_mad_u32_u24 v2, s7, v5, v2
 ; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
+; GFX8-NEXT:    v_mad_u32_u24 v2, s2, v3, v2
 ; GFX8-NEXT:    flat_store_byte v[0:1], v2
 ; GFX8-NEXT:    s_endpgm
 ;
@@ -415,31 +415,31 @@ define amdgpu_kernel void @udot4_acc8(<4 x i8> addrspace(1)* %src1,
 ; GFX9-NODL:       ; %bb.0: ; %entry
 ; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX9-NODL-NEXT:    s_movk_i32 s2, 0xff
 ; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NODL-NEXT:    s_load_dword s2, s[4:5], 0x0
 ; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-NODL-NEXT:    global_load_ubyte v2, v[0:1], off
-; GFX9-NODL-NEXT:    s_load_dword s0, s[4:5], 0x0
 ; GFX9-NODL-NEXT:    s_load_dword s1, s[6:7], 0x0
+; GFX9-NODL-NEXT:    s_movk_i32 s0, 0xff
 ; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT:    s_bfe_u32 s5, s0, 0x80008
-; GFX9-NODL-NEXT:    s_and_b32 s3, s1, s2
+; GFX9-NODL-NEXT:    s_bfe_u32 s5, s2, 0x80008
+; GFX9-NODL-NEXT:    s_bfe_u32 s7, s2, 0x80010
+; GFX9-NODL-NEXT:    s_and_b32 s3, s1, s0
+; GFX9-NODL-NEXT:    s_and_b32 s0, s2, s0
 ; GFX9-NODL-NEXT:    s_bfe_u32 s4, s1, 0x80008
-; GFX9-NODL-NEXT:    s_and_b32 s2, s0, s2
 ; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s3
 ; GFX9-NODL-NEXT:    s_bfe_u32 s6, s1, 0x80010
 ; GFX9-NODL-NEXT:    v_mov_b32_e32 v4, s4
-; GFX9-NODL-NEXT:    s_bfe_u32 s7, s0, 0x80010
 ; GFX9-NODL-NEXT:    s_lshr_b32 s1, s1, 24
 ; GFX9-NODL-NEXT:    v_mov_b32_e32 v5, s6
-; GFX9-NODL-NEXT:    s_lshr_b32 s0, s0, 24
+; GFX9-NODL-NEXT:    s_lshr_b32 s2, s2, 24
 ; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s2, v3, v2
+; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
 ; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s5, v4, v2
 ; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s7, v5, v2
 ; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s1
-; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
+; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s2, v3, v2
 ; GFX9-NODL-NEXT:    global_store_byte v[0:1], v2, off
 ; GFX9-NODL-NEXT:    s_endpgm
 ;
@@ -448,15 +448,15 @@ define amdgpu_kernel void @udot4_acc8(<4 x i8> addrspace(1)* %src1,
 ; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
+; GFX9-DL-NEXT:    s_load_dword s2, s[6:7], 0x0
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-DL-NEXT:    global_load_ubyte v2, v[0:1], off
+; GFX9-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s3
+; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s2
 ; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-DL-NEXT:    v_dot4_u32_u8 v2, s2, v3, v2
+; GFX9-DL-NEXT:    v_dot4_u32_u8 v2, s0, v3, v2
 ; GFX9-DL-NEXT:    global_store_byte v[0:1], v2, off
 ; GFX9-DL-NEXT:    s_endpgm
 ;
@@ -613,19 +613,19 @@ define amdgpu_kernel void @udot2_8(<4 x i8> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_movk_i32 s2, 0xff
 ; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
-; GFX10-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-DL-NEXT:    s_load_dword s0, s[6:7], 0x0
+; GFX10-DL-NEXT:    s_load_dword s1, s[4:5], 0x0
 ; GFX10-DL-NEXT:    global_load_ubyte v2, v[0:1], off
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_and_b32 s0, s3, s2
-; GFX10-DL-NEXT:    s_and_b32 s1, s4, s2
-; GFX10-DL-NEXT:    s_bfe_u32 s2, s4, 0x80008
-; GFX10-DL-NEXT:    s_bfe_u32 s3, s3, 0x80008
+; GFX10-DL-NEXT:    s_and_b32 s3, s0, s2
+; GFX10-DL-NEXT:    s_and_b32 s2, s1, s2
+; GFX10-DL-NEXT:    s_bfe_u32 s0, s0, 0x80008
+; GFX10-DL-NEXT:    s_bfe_u32 s1, s1, 0x80008
 ; GFX10-DL-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s0, s1, v2
-; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s3, s2, v2
+; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s2, s3, v2
+; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s1, s0, v2
 ; GFX10-DL-NEXT:    global_store_byte v[0:1], v2, off
 ; GFX10-DL-NEXT:    s_endpgm
                                    <4 x i8> addrspace(1)* %src2,
@@ -752,14 +752,14 @@ define amdgpu_kernel void @udot4_CommutationInsideMAD(<4 x i8> addrspace(1)* %sr
 ; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-DL-NEXT:    global_load_ubyte v2, v[0:1], off
+; GFX9-DL-NEXT:    s_load_dword s0, s[6:7], 0x0
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s2
 ; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-DL-NEXT:    v_dot4_u32_u8 v2, s3, v3, v2
+; GFX9-DL-NEXT:    v_dot4_u32_u8 v2, s0, v3, v2
 ; GFX9-DL-NEXT:    global_store_byte v[0:1], v2, off
 ; GFX9-DL-NEXT:    s_endpgm
 ;
@@ -948,24 +948,24 @@ define amdgpu_kernel void @udot4_CommutationAccrossMADs(<4 x i8> addrspace(1)* %
 ; GFX10-DL-NEXT:    s_movk_i32 s2, 0xff
 ; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
-; GFX10-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
+; GFX10-DL-NEXT:    s_load_dword s1, s[6:7], 0x0
 ; GFX10-DL-NEXT:    global_load_ubyte v2, v[0:1], off
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_bfe_u32 s0, s3, 0x80008
-; GFX10-DL-NEXT:    s_bfe_u32 s1, s4, 0x80008
-; GFX10-DL-NEXT:    s_and_b32 s5, s3, s2
-; GFX10-DL-NEXT:    s_and_b32 s2, s4, s2
-; GFX10-DL-NEXT:    s_bfe_u32 s6, s3, 0x80010
-; GFX10-DL-NEXT:    s_bfe_u32 s7, s4, 0x80010
+; GFX10-DL-NEXT:    s_bfe_u32 s3, s0, 0x80008
+; GFX10-DL-NEXT:    s_bfe_u32 s4, s1, 0x80008
+; GFX10-DL-NEXT:    s_and_b32 s5, s0, s2
+; GFX10-DL-NEXT:    s_and_b32 s2, s1, s2
 ; GFX10-DL-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s1, s0, v2
-; GFX10-DL-NEXT:    s_lshr_b32 s0, s3, 24
-; GFX10-DL-NEXT:    s_lshr_b32 s1, s4, 24
+; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s4, s3, v2
+; GFX10-DL-NEXT:    s_bfe_u32 s3, s0, 0x80010
+; GFX10-DL-NEXT:    s_bfe_u32 s4, s1, 0x80010
+; GFX10-DL-NEXT:    s_lshr_b32 s0, s0, 24
+; GFX10-DL-NEXT:    s_lshr_b32 s1, s1, 24
 ; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s2, s5, v2
-; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s7, s6, v2
+; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s4, s3, v2
 ; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s1, s0, v2
 ; GFX10-DL-NEXT:    global_store_byte v[0:1], v2, off
 ; GFX10-DL-NEXT:    s_endpgm
@@ -1012,23 +1012,23 @@ define amdgpu_kernel void @udot4_multiuse_mul1(<4 x i8> addrspace(1)* %src1,
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
-; GFX7-NEXT:    s_load_dword s6, s[0:1], 0x0
+; GFX7-NEXT:    s_load_dword s12, s[0:1], 0x0
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_and_b32 s7, s4, s8
-; GFX7-NEXT:    s_and_b32 s8, s5, s8
-; GFX7-NEXT:    s_bfe_u32 s10, s5, 0x80008
-; GFX7-NEXT:    v_mov_b32_e32 v0, s8
-; GFX7-NEXT:    v_mov_b32_e32 v1, s6
-; GFX7-NEXT:    s_bfe_u32 s9, s4, 0x80008
-; GFX7-NEXT:    v_mad_u32_u24 v1, s7, v0, v1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s10
-; GFX7-NEXT:    s_bfe_u32 s12, s5, 0x80010
-; GFX7-NEXT:    v_mad_u32_u24 v1, s9, v2, v1
-; GFX7-NEXT:    s_bfe_u32 s11, s4, 0x80010
-; GFX7-NEXT:    v_mad_u32_u24 v0, s7, v0, v1
+; GFX7-NEXT:    s_and_b32 s6, s4, s8
+; GFX7-NEXT:    s_and_b32 s7, s5, s8
+; GFX7-NEXT:    s_bfe_u32 s9, s5, 0x80008
+; GFX7-NEXT:    v_mov_b32_e32 v0, s7
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s12
+; GFX7-NEXT:    s_bfe_u32 s8, s4, 0x80008
+; GFX7-NEXT:    v_mad_u32_u24 v1, s6, v0, v1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s9
+; GFX7-NEXT:    s_bfe_u32 s11, s5, 0x80010
+; GFX7-NEXT:    v_mad_u32_u24 v1, s8, v2, v1
+; GFX7-NEXT:    s_bfe_u32 s10, s4, 0x80010
+; GFX7-NEXT:    v_mad_u32_u24 v0, s6, v0, v1
+; GFX7-NEXT:    v_mov_b32_e32 v1, s11
 ; GFX7-NEXT:    s_lshr_b32 s5, s5, 24
-; GFX7-NEXT:    v_mad_u32_u24 v0, s11, v1, v0
+; GFX7-NEXT:    v_mad_u32_u24 v0, s10, v1, v0
 ; GFX7-NEXT:    s_lshr_b32 s4, s4, 24
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s5
 ; GFX7-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
@@ -1043,23 +1043,23 @@ define amdgpu_kernel void @udot4_multiuse_mul1(<4 x i8> addrspace(1)* %src1,
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    s_load_dword s3, s[4:5], 0x0
 ; GFX8-NEXT:    s_load_dword s4, s[6:7], 0x0
-; GFX8-NEXT:    s_load_dword s5, s[0:1], 0x0
+; GFX8-NEXT:    s_load_dword s10, s[0:1], 0x0
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_and_b32 s6, s3, s2
+; GFX8-NEXT:    s_and_b32 s5, s3, s2
 ; GFX8-NEXT:    s_and_b32 s2, s4, s2
-; GFX8-NEXT:    s_bfe_u32 s8, s4, 0x80008
+; GFX8-NEXT:    s_bfe_u32 s7, s4, 0x80008
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s2
-; GFX8-NEXT:    v_mov_b32_e32 v1, s5
-; GFX8-NEXT:    s_bfe_u32 s7, s3, 0x80008
-; GFX8-NEXT:    v_mad_u32_u24 v1, s6, v0, v1
-; GFX8-NEXT:    v_mov_b32_e32 v2, s8
-; GFX8-NEXT:    s_bfe_u32 s10, s4, 0x80010
-; GFX8-NEXT:    v_mad_u32_u24 v1, s7, v2, v1
-; GFX8-NEXT:    s_bfe_u32 s9, s3, 0x80010
-; GFX8-NEXT:    v_mad_u32_u24 v0, s6, v0, v1
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s10
+; GFX8-NEXT:    s_bfe_u32 s6, s3, 0x80008
+; GFX8-NEXT:    v_mad_u32_u24 v1, s5, v0, v1
+; GFX8-NEXT:    v_mov_b32_e32 v2, s7
+; GFX8-NEXT:    s_bfe_u32 s9, s4, 0x80010
+; GFX8-NEXT:    v_mad_u32_u24 v1, s6, v2, v1
+; GFX8-NEXT:    s_bfe_u32 s8, s3, 0x80010
+; GFX8-NEXT:    v_mad_u32_u24 v0, s5, v0, v1
+; GFX8-NEXT:    v_mov_b32_e32 v1, s9
 ; GFX8-NEXT:    s_lshr_b32 s4, s4, 24
-; GFX8-NEXT:    v_mad_u32_u24 v0, s9, v1, v0
+; GFX8-NEXT:    v_mad_u32_u24 v0, s8, v1, v0
 ; GFX8-NEXT:    s_lshr_b32 s3, s3, 24
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s4
 ; GFX8-NEXT:    v_mad_u32_u24 v2, s3, v1, v0
@@ -1076,23 +1076,23 @@ define amdgpu_kernel void @udot4_multiuse_mul1(<4 x i8> addrspace(1)* %src1,
 ; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NODL-NEXT:    s_load_dword s3, s[4:5], 0x0
 ; GFX9-NODL-NEXT:    s_load_dword s4, s[6:7], 0x0
-; GFX9-NODL-NEXT:    s_load_dword s5, s[0:1], 0x0
+; GFX9-NODL-NEXT:    s_load_dword s10, s[0:1], 0x0
 ; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT:    s_and_b32 s6, s3, s2
+; GFX9-NODL-NEXT:    s_and_b32 s5, s3, s2
 ; GFX9-NODL-NEXT:    s_and_b32 s2, s4, s2
-; GFX9-NODL-NEXT:    s_bfe_u32 s8, s4, 0x80008
+; GFX9-NODL-NEXT:    s_bfe_u32 s7, s4, 0x80008
 ; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s2
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s5
-; GFX9-NODL-NEXT:    s_bfe_u32 s7, s3, 0x80008
-; GFX9-NODL-NEXT:    v_mad_u32_u24 v1, s6, v0, v1
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s8
-; GFX9-NODL-NEXT:    s_bfe_u32 s10, s4, 0x80010
-; GFX9-NODL-NEXT:    v_mad_u32_u24 v1, s7, v2, v1
-; GFX9-NODL-NEXT:    s_bfe_u32 s9, s3, 0x80010
-; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s6, v0, v1
 ; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s10
+; GFX9-NODL-NEXT:    s_bfe_u32 s6, s3, 0x80008
+; GFX9-NODL-NEXT:    v_mad_u32_u24 v1, s5, v0, v1
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s7
+; GFX9-NODL-NEXT:    s_bfe_u32 s9, s4, 0x80010
+; GFX9-NODL-NEXT:    v_mad_u32_u24 v1, s6, v2, v1
+; GFX9-NODL-NEXT:    s_bfe_u32 s8, s3, 0x80010
+; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s5, v0, v1
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s9
 ; GFX9-NODL-NEXT:    s_lshr_b32 s4, s4, 24
-; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s9, v1, v0
+; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s8, v1, v0
 ; GFX9-NODL-NEXT:    s_lshr_b32 s3, s3, 24
 ; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s4
 ; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s3, v1, v0
@@ -1109,23 +1109,23 @@ define amdgpu_kernel void @udot4_multiuse_mul1(<4 x i8> addrspace(1)* %src1,
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
 ; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
-; GFX9-DL-NEXT:    s_load_dword s5, s[0:1], 0x0
+; GFX9-DL-NEXT:    s_load_dword s10, s[0:1], 0x0
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    s_and_b32 s6, s3, s2
+; GFX9-DL-NEXT:    s_and_b32 s5, s3, s2
 ; GFX9-DL-NEXT:    s_and_b32 s2, s4, s2
-; GFX9-DL-NEXT:    s_bfe_u32 s8, s4, 0x80008
+; GFX9-DL-NEXT:    s_bfe_u32 s7, s4, 0x80008
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s2
-; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s5
-; GFX9-DL-NEXT:    s_bfe_u32 s7, s3, 0x80008
-; GFX9-DL-NEXT:    v_mad_u32_u24 v1, s6, v0, v1
-; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s8
-; GFX9-DL-NEXT:    s_bfe_u32 s10, s4, 0x80010
-; GFX9-DL-NEXT:    v_mad_u32_u24 v1, s7, v2, v1
-; GFX9-DL-NEXT:    s_bfe_u32 s9, s3, 0x80010
-; GFX9-DL-NEXT:    v_mad_u32_u24 v0, s6, v0, v1
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s10
+; GFX9-DL-NEXT:    s_bfe_u32 s6, s3, 0x80008
+; GFX9-DL-NEXT:    v_mad_u32_u24 v1, s5, v0, v1
+; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s7
+; GFX9-DL-NEXT:    s_bfe_u32 s9, s4, 0x80010
+; GFX9-DL-NEXT:    v_mad_u32_u24 v1, s6, v2, v1
+; GFX9-DL-NEXT:    s_bfe_u32 s8, s3, 0x80010
+; GFX9-DL-NEXT:    v_mad_u32_u24 v0, s5, v0, v1
+; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s9
 ; GFX9-DL-NEXT:    s_lshr_b32 s4, s4, 24
-; GFX9-DL-NEXT:    v_mad_u32_u24 v0, s9, v1, v0
+; GFX9-DL-NEXT:    v_mad_u32_u24 v0, s8, v1, v0
 ; GFX9-DL-NEXT:    s_lshr_b32 s3, s3, 24
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s4
 ; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s3, v1, v0
@@ -1144,23 +1144,23 @@ define amdgpu_kernel void @udot4_multiuse_mul1(<4 x i8> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
 ; GFX10-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
 ; GFX10-DL-NEXT:    s_load_dword s5, s[0:1], 0x0
+; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-DL-NEXT:    s_and_b32 s6, s3, s2
+; GFX10-DL-NEXT:    s_and_b32 s2, s4, s2
+; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s5
+; GFX10-DL-NEXT:    s_bfe_u32 s5, s3, 0x80008
+; GFX10-DL-NEXT:    s_bfe_u32 s7, s4, 0x80008
+; GFX10-DL-NEXT:    v_mad_u32_u24 v0, s6, s2, v0
+; GFX10-DL-NEXT:    v_mad_u32_u24 v0, s5, s7, v0
+; GFX10-DL-NEXT:    s_bfe_u32 s5, s3, 0x80010
+; GFX10-DL-NEXT:    s_bfe_u32 s7, s4, 0x80010
+; GFX10-DL-NEXT:    v_mad_u32_u24 v0, s6, s2, v0
+; GFX10-DL-NEXT:    s_lshr_b32 s2, s3, 24
+; GFX10-DL-NEXT:    s_lshr_b32 s3, s4, 24
+; GFX10-DL-NEXT:    v_mad_u32_u24 v0, s5, s7, v0
+; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s2, s3, v0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
-; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_and_b32 s0, s3, s2
-; GFX10-DL-NEXT:    s_and_b32 s1, s4, s2
-; GFX10-DL-NEXT:    v_mov_b32_e32 v2, s5
-; GFX10-DL-NEXT:    s_bfe_u32 s2, s3, 0x80008
-; GFX10-DL-NEXT:    s_bfe_u32 s5, s4, 0x80008
-; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s0, s1, v2
-; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s2, s5, v2
-; GFX10-DL-NEXT:    s_bfe_u32 s2, s3, 0x80010
-; GFX10-DL-NEXT:    s_bfe_u32 s5, s4, 0x80010
-; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s0, s1, v2
-; GFX10-DL-NEXT:    s_lshr_b32 s0, s3, 24
-; GFX10-DL-NEXT:    s_lshr_b32 s1, s4, 24
-; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s2, s5, v2
-; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s0, s1, v2
 ; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX10-DL-NEXT:    s_endpgm
                                                <4 x i8> addrspace(1)* %src2,
@@ -1215,23 +1215,23 @@ define amdgpu_kernel void @udot4_multiuse_add1(<4 x i8> addrspace(1)* %src1,
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
-; GFX7-NEXT:    s_load_dword s6, s[0:1], 0x0
+; GFX7-NEXT:    s_load_dword s12, s[0:1], 0x0
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_and_b32 s7, s4, s8
-; GFX7-NEXT:    s_bfe_u32 s10, s5, 0x80008
-; GFX7-NEXT:    s_and_b32 s8, s5, s8
-; GFX7-NEXT:    s_bfe_u32 s9, s4, 0x80008
-; GFX7-NEXT:    v_mov_b32_e32 v0, s10
-; GFX7-NEXT:    v_mov_b32_e32 v1, s6
-; GFX7-NEXT:    v_mad_u32_u24 v0, s9, v0, v1
-; GFX7-NEXT:    s_bfe_u32 s12, s5, 0x80010
-; GFX7-NEXT:    v_mov_b32_e32 v2, s8
-; GFX7-NEXT:    s_bfe_u32 s11, s4, 0x80010
-; GFX7-NEXT:    v_add_i32_e32 v1, vcc, s6, v0
-; GFX7-NEXT:    v_mad_u32_u24 v0, s7, v2, v0
-; GFX7-NEXT:    v_mov_b32_e32 v2, s12
+; GFX7-NEXT:    s_and_b32 s6, s4, s8
+; GFX7-NEXT:    s_bfe_u32 s9, s5, 0x80008
+; GFX7-NEXT:    s_and_b32 s7, s5, s8
+; GFX7-NEXT:    s_bfe_u32 s8, s4, 0x80008
+; GFX7-NEXT:    v_mov_b32_e32 v0, s9
+; GFX7-NEXT:    v_mov_b32_e32 v1, s12
+; GFX7-NEXT:    v_mad_u32_u24 v0, s8, v0, v1
+; GFX7-NEXT:    s_bfe_u32 s11, s5, 0x80010
+; GFX7-NEXT:    v_mov_b32_e32 v2, s7
+; GFX7-NEXT:    s_bfe_u32 s10, s4, 0x80010
+; GFX7-NEXT:    v_add_i32_e32 v1, vcc, s12, v0
+; GFX7-NEXT:    v_mad_u32_u24 v0, s6, v2, v0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s11
 ; GFX7-NEXT:    s_lshr_b32 s5, s5, 24
-; GFX7-NEXT:    v_mad_u32_u24 v0, s11, v2, v0
+; GFX7-NEXT:    v_mad_u32_u24 v0, s10, v2, v0
 ; GFX7-NEXT:    s_lshr_b32 s4, s4, 24
 ; GFX7-NEXT:    v_mov_b32_e32 v2, s5
 ; GFX7-NEXT:    v_mad_u32_u24 v0, s4, v2, v0
@@ -1247,23 +1247,23 @@ define amdgpu_kernel void @udot4_multiuse_add1(<4 x i8> addrspace(1)* %src1,
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    s_load_dword s3, s[4:5], 0x0
 ; GFX8-NEXT:    s_load_dword s4, s[6:7], 0x0
-; GFX8-NEXT:    s_load_dword s5, s[0:1], 0x0
+; GFX8-NEXT:    s_load_dword s10, s[0:1], 0x0
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_and_b32 s6, s3, s2
-; GFX8-NEXT:    s_bfe_u32 s8, s4, 0x80008
+; GFX8-NEXT:    s_and_b32 s5, s3, s2
+; GFX8-NEXT:    s_bfe_u32 s7, s4, 0x80008
 ; GFX8-NEXT:    s_and_b32 s2, s4, s2
-; GFX8-NEXT:    s_bfe_u32 s7, s3, 0x80008
-; GFX8-NEXT:    v_mov_b32_e32 v0, s8
-; GFX8-NEXT:    v_mov_b32_e32 v1, s5
-; GFX8-NEXT:    v_mad_u32_u24 v0, s7, v0, v1
-; GFX8-NEXT:    s_bfe_u32 s10, s4, 0x80010
+; GFX8-NEXT:    s_bfe_u32 s6, s3, 0x80008
+; GFX8-NEXT:    v_mov_b32_e32 v0, s7
+; GFX8-NEXT:    v_mov_b32_e32 v1, s10
+; GFX8-NEXT:    v_mad_u32_u24 v0, s6, v0, v1
+; GFX8-NEXT:    s_bfe_u32 s9, s4, 0x80010
 ; GFX8-NEXT:    v_mov_b32_e32 v2, s2
-; GFX8-NEXT:    s_bfe_u32 s9, s3, 0x80010
-; GFX8-NEXT:    v_add_u32_e32 v1, vcc, s5, v0
-; GFX8-NEXT:    v_mad_u32_u24 v0, s6, v2, v0
-; GFX8-NEXT:    v_mov_b32_e32 v2, s10
+; GFX8-NEXT:    s_bfe_u32 s8, s3, 0x80010
+; GFX8-NEXT:    v_add_u32_e32 v1, vcc, s10, v0
+; GFX8-NEXT:    v_mad_u32_u24 v0, s5, v2, v0
+; GFX8-NEXT:    v_mov_b32_e32 v2, s9
 ; GFX8-NEXT:    s_lshr_b32 s4, s4, 24
-; GFX8-NEXT:    v_mad_u32_u24 v0, s9, v2, v0
+; GFX8-NEXT:    v_mad_u32_u24 v0, s8, v2, v0
 ; GFX8-NEXT:    s_lshr_b32 s3, s3, 24
 ; GFX8-NEXT:    v_mov_b32_e32 v2, s4
 ; GFX8-NEXT:    v_mad_u32_u24 v0, s3, v2, v0
@@ -1281,23 +1281,23 @@ define amdgpu_kernel void @udot4_multiuse_add1(<4 x i8> addrspace(1)* %src1,
 ; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NODL-NEXT:    s_load_dword s3, s[4:5], 0x0
 ; GFX9-NODL-NEXT:    s_load_dword s4, s[6:7], 0x0
-; GFX9-NODL-NEXT:    s_load_dword s5, s[0:1], 0x0
+; GFX9-NODL-NEXT:    s_load_dword s10, s[0:1], 0x0
 ; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT:    s_and_b32 s6, s3, s2
-; GFX9-NODL-NEXT:    s_bfe_u32 s8, s4, 0x80008
+; GFX9-NODL-NEXT:    s_and_b32 s5, s3, s2
+; GFX9-NODL-NEXT:    s_bfe_u32 s7, s4, 0x80008
 ; GFX9-NODL-NEXT:    s_and_b32 s2, s4, s2
-; GFX9-NODL-NEXT:    s_bfe_u32 s7, s3, 0x80008
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s8
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s5
-; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s7, v0, v1
-; GFX9-NODL-NEXT:    s_bfe_u32 s10, s4, 0x80010
+; GFX9-NODL-NEXT:    s_bfe_u32 s6, s3, 0x80008
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s7
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s10
+; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s6, v0, v1
+; GFX9-NODL-NEXT:    s_bfe_u32 s9, s4, 0x80010
 ; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s2
-; GFX9-NODL-NEXT:    s_bfe_u32 s9, s3, 0x80010
-; GFX9-NODL-NEXT:    v_add_u32_e32 v1, s5, v0
-; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s6, v2, v0
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s10
+; GFX9-NODL-NEXT:    s_bfe_u32 s8, s3, 0x80010
+; GFX9-NODL-NEXT:    v_add_u32_e32 v1, s10, v0
+; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s5, v2, v0
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s9
 ; GFX9-NODL-NEXT:    s_lshr_b32 s4, s4, 24
-; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s9, v2, v0
+; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s8, v2, v0
 ; GFX9-NODL-NEXT:    s_lshr_b32 s3, s3, 24
 ; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s4
 ; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s3, v2, v0
@@ -1315,23 +1315,23 @@ define amdgpu_kernel void @udot4_multiuse_add1(<4 x i8> addrspace(1)* %src1,
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
 ; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
-; GFX9-DL-NEXT:    s_load_dword s5, s[0:1], 0x0
+; GFX9-DL-NEXT:    s_load_dword s10, s[0:1], 0x0
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    s_and_b32 s6, s3, s2
-; GFX9-DL-NEXT:    s_bfe_u32 s8, s4, 0x80008
+; GFX9-DL-NEXT:    s_and_b32 s5, s3, s2
+; GFX9-DL-NEXT:    s_bfe_u32 s7, s4, 0x80008
 ; GFX9-DL-NEXT:    s_and_b32 s2, s4, s2
-; GFX9-DL-NEXT:    s_bfe_u32 s7, s3, 0x80008
-; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s8
-; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s5
-; GFX9-DL-NEXT:    v_mad_u32_u24 v0, s7, v0, v1
-; GFX9-DL-NEXT:    s_bfe_u32 s10, s4, 0x80010
+; GFX9-DL-NEXT:    s_bfe_u32 s6, s3, 0x80008
+; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s7
+; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s10
+; GFX9-DL-NEXT:    v_mad_u32_u24 v0, s6, v0, v1
+; GFX9-DL-NEXT:    s_bfe_u32 s9, s4, 0x80010
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s2
-; GFX9-DL-NEXT:    s_bfe_u32 s9, s3, 0x80010
-; GFX9-DL-NEXT:    v_add_u32_e32 v1, s5, v0
-; GFX9-DL-NEXT:    v_mad_u32_u24 v0, s6, v2, v0
-; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s10
+; GFX9-DL-NEXT:    s_bfe_u32 s8, s3, 0x80010
+; GFX9-DL-NEXT:    v_add_u32_e32 v1, s10, v0
+; GFX9-DL-NEXT:    v_mad_u32_u24 v0, s5, v2, v0
+; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s9
 ; GFX9-DL-NEXT:    s_lshr_b32 s4, s4, 24
-; GFX9-DL-NEXT:    v_mad_u32_u24 v0, s9, v2, v0
+; GFX9-DL-NEXT:    v_mad_u32_u24 v0, s8, v2, v0
 ; GFX9-DL-NEXT:    s_lshr_b32 s3, s3, 24
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s4
 ; GFX9-DL-NEXT:    v_mad_u32_u24 v0, s3, v2, v0
@@ -1351,24 +1351,24 @@ define amdgpu_kernel void @udot4_multiuse_add1(<4 x i8> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
 ; GFX10-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
 ; GFX10-DL-NEXT:    s_load_dword s5, s[0:1], 0x0
-; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_bfe_u32 s0, s3, 0x80008
-; GFX10-DL-NEXT:    s_bfe_u32 s1, s4, 0x80008
-; GFX10-DL-NEXT:    v_mov_b32_e32 v2, s5
-; GFX10-DL-NEXT:    s_and_b32 s6, s3, s2
+; GFX10-DL-NEXT:    s_bfe_u32 s6, s3, 0x80008
+; GFX10-DL-NEXT:    s_bfe_u32 s7, s4, 0x80008
+; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s5
+; GFX10-DL-NEXT:    s_and_b32 s8, s3, s2
 ; GFX10-DL-NEXT:    s_and_b32 s2, s4, s2
-; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s0, s1, v2
-; GFX10-DL-NEXT:    s_bfe_u32 s0, s3, 0x80010
-; GFX10-DL-NEXT:    s_bfe_u32 s1, s4, 0x80010
-; GFX10-DL-NEXT:    v_mad_u32_u24 v3, s6, s2, v2
+; GFX10-DL-NEXT:    v_mad_u32_u24 v0, s6, s7, v0
+; GFX10-DL-NEXT:    s_bfe_u32 s6, s3, 0x80010
+; GFX10-DL-NEXT:    s_bfe_u32 s7, s4, 0x80010
+; GFX10-DL-NEXT:    v_mad_u32_u24 v1, s8, s2, v0
 ; GFX10-DL-NEXT:    s_lshr_b32 s2, s3, 24
 ; GFX10-DL-NEXT:    s_lshr_b32 s3, s4, 24
-; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, s5, v2
-; GFX10-DL-NEXT:    v_mad_u32_u24 v3, s0, s1, v3
-; GFX10-DL-NEXT:    v_mad_u32_u24 v3, s2, s3, v3
-; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v3, v2
+; GFX10-DL-NEXT:    v_add_nc_u32_e32 v0, s5, v0
+; GFX10-DL-NEXT:    v_mad_u32_u24 v1, s6, s7, v1
+; GFX10-DL-NEXT:    v_mad_u32_u24 v1, s2, s3, v1
+; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v1, v0
+; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX10-DL-NEXT:    s_endpgm
                                                <4 x i8> addrspace(1)* %src2,
@@ -1421,28 +1421,28 @@ define amdgpu_kernel void @notdot4_mixedtypes(<4 x i8> addrspace(1)* %src1,
 ; GFX7-NEXT:    s_mov_b32 s2, -1
 ; GFX7-NEXT:    s_mov_b32 s8, 0xffff
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s6, s[6:7], 0x0
 ; GFX7-NEXT:    buffer_load_ushort v0, off, s[0:3], 0
-; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
+; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_sext_i32_i8 s6, s4
-; GFX7-NEXT:    s_bfe_u32 s10, s4, 0x80008
-; GFX7-NEXT:    s_sext_i32_i8 s7, s5
-; GFX7-NEXT:    s_bfe_u32 s9, s5, 0x80008
+; GFX7-NEXT:    s_sext_i32_i8 s7, s6
+; GFX7-NEXT:    s_bfe_u32 s9, s6, 0x80008
+; GFX7-NEXT:    s_sext_i32_i8 s5, s4
 ; GFX7-NEXT:    s_and_b32 s7, s7, s8
+; GFX7-NEXT:    s_bfe_u32 s10, s4, 0x80008
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s9
-; GFX7-NEXT:    s_bfe_u32 s11, s5, 0x80010
-; GFX7-NEXT:    s_and_b32 s6, s6, s8
+; GFX7-NEXT:    s_bfe_u32 s11, s6, 0x80010
+; GFX7-NEXT:    s_and_b32 s5, s5, s8
 ; GFX7-NEXT:    v_mov_b32_e32 v3, s7
 ; GFX7-NEXT:    s_bfe_u32 s12, s4, 0x80010
-; GFX7-NEXT:    s_lshr_b32 s5, s5, 24
+; GFX7-NEXT:    s_lshr_b32 s6, s6, 24
 ; GFX7-NEXT:    v_mov_b32_e32 v2, s11
 ; GFX7-NEXT:    s_lshr_b32 s4, s4, 24
 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7-NEXT:    v_mad_u32_u24 v0, s10, v1, v0
-; GFX7-NEXT:    v_mad_u32_u24 v0, s6, v3, v0
+; GFX7-NEXT:    v_mad_u32_u24 v0, s5, v3, v0
 ; GFX7-NEXT:    v_mad_u32_u24 v0, s12, v2, v0
-; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v1, s6
 ; GFX7-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
 ; GFX7-NEXT:    buffer_store_short v0, off, s[0:3], 0
 ; GFX7-NEXT:    s_endpgm
@@ -1452,29 +1452,29 @@ define amdgpu_kernel void @notdot4_mixedtypes(<4 x i8> addrspace(1)* %src1,
 ; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX8-NEXT:    s_load_dword s3, s[6:7], 0x0
+; GFX8-NEXT:    s_load_dword s2, s[6:7], 0x0
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX8-NEXT:    flat_load_ushort v2, v[0:1]
+; GFX8-NEXT:    s_load_dword s0, s[4:5], 0x0
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_bfe_u32 s0, s2, 0x80008
-; GFX8-NEXT:    s_bfe_u32 s1, s3, 0x80008
+; GFX8-NEXT:    s_bfe_u32 s1, s2, 0x80008
+; GFX8-NEXT:    s_sext_i32_i8 s3, s2
 ; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    s_sext_i32_i8 s4, s3
-; GFX8-NEXT:    s_bfe_u32 s5, s3, 0x80010
-; GFX8-NEXT:    v_mov_b32_e32 v4, s4
-; GFX8-NEXT:    s_sext_i32_i8 s1, s2
-; GFX8-NEXT:    s_bfe_u32 s4, s2, 0x80010
-; GFX8-NEXT:    s_lshr_b32 s3, s3, 24
-; GFX8-NEXT:    v_mov_b32_e32 v5, s5
+; GFX8-NEXT:    s_bfe_u32 s5, s2, 0x80010
+; GFX8-NEXT:    s_bfe_u32 s1, s0, 0x80008
+; GFX8-NEXT:    v_mov_b32_e32 v4, s3
+; GFX8-NEXT:    s_sext_i32_i8 s4, s0
+; GFX8-NEXT:    s_bfe_u32 s3, s0, 0x80010
 ; GFX8-NEXT:    s_lshr_b32 s2, s2, 24
+; GFX8-NEXT:    v_mov_b32_e32 v5, s5
+; GFX8-NEXT:    s_lshr_b32 s0, s0, 24
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    v_mad_u32_u24 v2, s1, v3, v2
+; GFX8-NEXT:    v_mad_i32_i24 v2, s4, v4, v2
+; GFX8-NEXT:    v_mad_u32_u24 v2, s3, v5, v2
+; GFX8-NEXT:    v_mov_b32_e32 v3, s2
 ; GFX8-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
-; GFX8-NEXT:    v_mad_i32_i24 v2, s1, v4, v2
-; GFX8-NEXT:    v_mad_u32_u24 v2, s4, v5, v2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s3
-; GFX8-NEXT:    v_mad_u32_u24 v2, s2, v3, v2
 ; GFX8-NEXT:    flat_store_short v[0:1], v2
 ; GFX8-NEXT:    s_endpgm
 ;
@@ -1483,29 +1483,29 @@ define amdgpu_kernel void @notdot4_mixedtypes(<4 x i8> addrspace(1)* %src1,
 ; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX9-NODL-NEXT:    s_load_dword s3, s[6:7], 0x0
+; GFX9-NODL-NEXT:    s_load_dword s2, s[6:7], 0x0
 ; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-NODL-NEXT:    global_load_ushort v2, v[0:1], off
+; GFX9-NODL-NEXT:    s_load_dword s0, s[4:5], 0x0
 ; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT:    s_bfe_u32 s0, s2, 0x80008
-; GFX9-NODL-NEXT:    s_bfe_u32 s1, s3, 0x80008
+; GFX9-NODL-NEXT:    s_bfe_u32 s1, s2, 0x80008
+; GFX9-NODL-NEXT:    s_sext_i32_i8 s3, s2
 ; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s1
-; GFX9-NODL-NEXT:    s_sext_i32_i8 s4, s3
-; GFX9-NODL-NEXT:    s_bfe_u32 s5, s3, 0x80010
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v4, s4
-; GFX9-NODL-NEXT:    s_sext_i32_i8 s1, s2
-; GFX9-NODL-NEXT:    s_bfe_u32 s4, s2, 0x80010
-; GFX9-NODL-NEXT:    s_lshr_b32 s3, s3, 24
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v5, s5
+; GFX9-NODL-NEXT:    s_bfe_u32 s5, s2, 0x80010
+; GFX9-NODL-NEXT:    s_bfe_u32 s1, s0, 0x80008
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v4, s3
+; GFX9-NODL-NEXT:    s_sext_i32_i8 s4, s0
+; GFX9-NODL-NEXT:    s_bfe_u32 s3, s0, 0x80010
 ; GFX9-NODL-NEXT:    s_lshr_b32 s2, s2, 24
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v5, s5
+; GFX9-NODL-NEXT:    s_lshr_b32 s0, s0, 24
 ; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s1, v3, v2
+; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s4, v4, v2
+; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s3, v5, v2
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s2
 ; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
-; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s1, v4, v2
-; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s4, v5, v2
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s3
-; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s2, v3, v2
 ; GFX9-NODL-NEXT:    global_store_short v[0:1], v2, off
 ; GFX9-NODL-NEXT:    s_endpgm
 ;
@@ -1514,29 +1514,29 @@ define amdgpu_kernel void @notdot4_mixedtypes(<4 x i8> addrspace(1)* %src1,
 ; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
+; GFX9-DL-NEXT:    s_load_dword s2, s[6:7], 0x0
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-DL-NEXT:    global_load_ushort v2, v[0:1], off
+; GFX9-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    s_bfe_u32 s0, s2, 0x80008
-; GFX9-DL-NEXT:    s_bfe_u32 s1, s3, 0x80008
+; GFX9-DL-NEXT:    s_bfe_u32 s1, s2, 0x80008
+; GFX9-DL-NEXT:    s_sext_i32_i8 s3, s2
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s1
-; GFX9-DL-NEXT:    s_sext_i32_i8 s4, s3
-; GFX9-DL-NEXT:    s_bfe_u32 s5, s3, 0x80010
-; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s4
-; GFX9-DL-NEXT:    s_sext_i32_i8 s1, s2
-; GFX9-DL-NEXT:    s_bfe_u32 s4, s2, 0x80010
-; GFX9-DL-NEXT:    s_lshr_b32 s3, s3, 24
-; GFX9-DL-NEXT:    v_mov_b32_e32 v5, s5
+; GFX9-DL-NEXT:    s_bfe_u32 s5, s2, 0x80010
+; GFX9-DL-NEXT:    s_bfe_u32 s1, s0, 0x80008
+; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s3
+; GFX9-DL-NEXT:    s_sext_i32_i8 s4, s0
+; GFX9-DL-NEXT:    s_bfe_u32 s3, s0, 0x80010
 ; GFX9-DL-NEXT:    s_lshr_b32 s2, s2, 24
+; GFX9-DL-NEXT:    v_mov_b32_e32 v5, s5
+; GFX9-DL-NEXT:    s_lshr_b32 s0, s0, 24
 ; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s1, v3, v2
+; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s4, v4, v2
+; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s3, v5, v2
+; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s2
 ; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s1, v4, v2
-; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s4, v5, v2
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s3
-; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s2, v3, v2
 ; GFX9-DL-NEXT:    global_store_short v[0:1], v2, off
 ; GFX9-DL-NEXT:    s_endpgm
 ;
@@ -1546,24 +1546,24 @@ define amdgpu_kernel void @notdot4_mixedtypes(<4 x i8> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX10-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
+; GFX10-DL-NEXT:    s_load_dword s1, s[6:7], 0x0
 ; GFX10-DL-NEXT:    global_load_ushort v2, v[0:1], off
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_bfe_u32 s0, s2, 0x80008
-; GFX10-DL-NEXT:    s_bfe_u32 s1, s3, 0x80008
-; GFX10-DL-NEXT:    s_sext_i32_i8 s4, s2
-; GFX10-DL-NEXT:    s_sext_i32_i8 s5, s3
-; GFX10-DL-NEXT:    s_bfe_u32 s6, s2, 0x80010
-; GFX10-DL-NEXT:    s_bfe_u32 s7, s3, 0x80010
+; GFX10-DL-NEXT:    s_bfe_u32 s2, s0, 0x80008
+; GFX10-DL-NEXT:    s_bfe_u32 s3, s1, 0x80008
+; GFX10-DL-NEXT:    s_sext_i32_i8 s4, s0
+; GFX10-DL-NEXT:    s_sext_i32_i8 s5, s1
 ; GFX10-DL-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s0, s1, v2
-; GFX10-DL-NEXT:    s_lshr_b32 s0, s2, 24
-; GFX10-DL-NEXT:    s_lshr_b32 s1, s3, 24
+; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s2, s3, v2
+; GFX10-DL-NEXT:    s_bfe_u32 s2, s0, 0x80010
+; GFX10-DL-NEXT:    s_bfe_u32 s3, s1, 0x80010
+; GFX10-DL-NEXT:    s_lshr_b32 s0, s0, 24
+; GFX10-DL-NEXT:    s_lshr_b32 s1, s1, 24
 ; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s4, s5, v2
-; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s6, s7, v2
+; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s2, s3, v2
 ; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s0, s1, v2
 ; GFX10-DL-NEXT:    global_store_short v[0:1], v2, off
 ; GFX10-DL-NEXT:    s_endpgm
@@ -1613,31 +1613,32 @@ define amdgpu_kernel void @udot4_acc32_vecMul(<4 x i8> addrspace(1)* %src1,
 ; GFX7:       ; %bb.0: ; %entry
 ; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
-; GFX7-NEXT:    s_movk_i32 s12, 0xff
+; GFX7-NEXT:    s_movk_i32 s11, 0xff
 ; GFX7-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX7-NEXT:    s_mov_b32 s2, -1
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
-; GFX7-NEXT:    s_load_dword s6, s[0:1], 0x0
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_lshr_b32 s7, s4, 24
-; GFX7-NEXT:    s_lshr_b32 s9, s5, 24
-; GFX7-NEXT:    s_bfe_u32 s10, s5, 0x80008
-; GFX7-NEXT:    s_bfe_u32 s13, s5, 0x80010
-; GFX7-NEXT:    s_and_b32 s5, s5, s12
-; GFX7-NEXT:    s_bfe_u32 s8, s4, 0x80008
-; GFX7-NEXT:    s_bfe_u32 s11, s4, 0x80010
-; GFX7-NEXT:    s_and_b32 s4, s4, s12
+; GFX7-NEXT:    s_lshr_b32 s6, s4, 24
+; GFX7-NEXT:    s_bfe_u32 s7, s4, 0x80008
+; GFX7-NEXT:    s_bfe_u32 s10, s4, 0x80010
+; GFX7-NEXT:    s_lshr_b32 s8, s5, 24
+; GFX7-NEXT:    s_bfe_u32 s9, s5, 0x80008
+; GFX7-NEXT:    s_bfe_u32 s12, s5, 0x80010
+; GFX7-NEXT:    s_and_b32 s5, s5, s11
+; GFX7-NEXT:    s_and_b32 s4, s4, s11
+; GFX7-NEXT:    s_load_dword s11, s[0:1], 0x0
 ; GFX7-NEXT:    v_mov_b32_e32 v0, s5
-; GFX7-NEXT:    v_mov_b32_e32 v1, s6
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v1, s11
 ; GFX7-NEXT:    v_mad_u32_u24 v0, s4, v0, v1
-; GFX7-NEXT:    v_mov_b32_e32 v1, s10
-; GFX7-NEXT:    v_mad_u32_u24 v0, s8, v1, v0
-; GFX7-NEXT:    v_mov_b32_e32 v1, s13
-; GFX7-NEXT:    v_mad_u32_u24 v0, s11, v1, v0
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s9
 ; GFX7-NEXT:    v_mad_u32_u24 v0, s7, v1, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s12
+; GFX7-NEXT:    v_mad_u32_u24 v0, s10, v1, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s8
+; GFX7-NEXT:    v_mad_u32_u24 v0, s6, v1, v0
 ; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; GFX7-NEXT:    s_endpgm
 ;
@@ -1649,24 +1650,25 @@ define amdgpu_kernel void @udot4_acc32_vecMul(<4 x i8> addrspace(1)* %src1,
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    s_load_dword s3, s[4:5], 0x0
 ; GFX8-NEXT:    s_load_dword s4, s[6:7], 0x0
-; GFX8-NEXT:    s_load_dword s5, s[0:1], 0x0
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_lshr_b32 s6, s3, 24
-; GFX8-NEXT:    s_bfe_u32 s8, s3, 0x80010
+; GFX8-NEXT:    s_lshr_b32 s5, s3, 24
+; GFX8-NEXT:    s_lshr_b32 s6, s4, 24
+; GFX8-NEXT:    s_bfe_u32 s7, s3, 0x80010
 ; GFX8-NEXT:    v_lshrrev_b16_e64 v0, 8, s3
 ; GFX8-NEXT:    s_and_b32 s3, s3, s2
 ; GFX8-NEXT:    s_and_b32 s2, s4, s2
-; GFX8-NEXT:    v_mov_b32_e32 v2, s2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s5
-; GFX8-NEXT:    s_bfe_u32 s9, s4, 0x80010
+; GFX8-NEXT:    s_bfe_u32 s8, s4, 0x80010
 ; GFX8-NEXT:    v_lshrrev_b16_e64 v1, 8, s4
+; GFX8-NEXT:    s_load_dword s4, s[0:1], 0x0
+; GFX8-NEXT:    v_mov_b32_e32 v2, s2
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v3, s4
 ; GFX8-NEXT:    v_mad_u32_u24 v2, s3, v2, v3
 ; GFX8-NEXT:    v_mad_u32_u24 v0, v0, v1, v2
-; GFX8-NEXT:    v_mov_b32_e32 v1, s9
-; GFX8-NEXT:    s_lshr_b32 s7, s4, 24
-; GFX8-NEXT:    v_mad_u32_u24 v0, s8, v1, v0
-; GFX8-NEXT:    v_mov_b32_e32 v1, s7
-; GFX8-NEXT:    v_mad_u32_u24 v2, s6, v1, v0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s8
+; GFX8-NEXT:    v_mad_u32_u24 v0, s7, v1, v0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s6
+; GFX8-NEXT:    v_mad_u32_u24 v2, s5, v1, v0
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX8-NEXT:    flat_store_dword v[0:1], v2
@@ -1680,24 +1682,25 @@ define amdgpu_kernel void @udot4_acc32_vecMul(<4 x i8> addrspace(1)* %src1,
 ; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NODL-NEXT:    s_load_dword s3, s[4:5], 0x0
 ; GFX9-NODL-NEXT:    s_load_dword s4, s[6:7], 0x0
-; GFX9-NODL-NEXT:    s_load_dword s5, s[0:1], 0x0
 ; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT:    s_lshr_b32 s6, s3, 24
-; GFX9-NODL-NEXT:    s_bfe_u32 s8, s3, 0x80010
+; GFX9-NODL-NEXT:    s_lshr_b32 s5, s3, 24
+; GFX9-NODL-NEXT:    s_lshr_b32 s6, s4, 24
+; GFX9-NODL-NEXT:    s_bfe_u32 s7, s3, 0x80010
 ; GFX9-NODL-NEXT:    v_lshrrev_b16_e64 v0, 8, s3
 ; GFX9-NODL-NEXT:    s_and_b32 s3, s3, s2
 ; GFX9-NODL-NEXT:    s_and_b32 s2, s4, s2
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s2
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s5
-; GFX9-NODL-NEXT:    s_bfe_u32 s9, s4, 0x80010
+; GFX9-NODL-NEXT:    s_bfe_u32 s8, s4, 0x80010
 ; GFX9-NODL-NEXT:    v_lshrrev_b16_e64 v1, 8, s4
+; GFX9-NODL-NEXT:    s_load_dword s4, s[0:1], 0x0
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s2
+; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s4
 ; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s3, v2, v3
 ; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, v0, v1, v2
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s9
-; GFX9-NODL-NEXT:    s_lshr_b32 s7, s4, 24
-; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s8, v1, v0
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s7
-; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s6, v1, v0
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s8
+; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s7, v1, v0
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s6
+; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s5, v1, v0
 ; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
@@ -1711,24 +1714,25 @@ define amdgpu_kernel void @udot4_acc32_vecMul(<4 x i8> addrspace(1)* %src1,
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
 ; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
-; GFX9-DL-NEXT:    s_load_dword s5, s[0:1], 0x0
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    s_lshr_b32 s6, s3, 24
-; GFX9-DL-NEXT:    s_bfe_u32 s8, s3, 0x80010
+; GFX9-DL-NEXT:    s_lshr_b32 s5, s3, 24
+; GFX9-DL-NEXT:    s_lshr_b32 s6, s4, 24
+; GFX9-DL-NEXT:    s_bfe_u32 s7, s3, 0x80010
 ; GFX9-DL-NEXT:    v_lshrrev_b16_e64 v0, 8, s3
 ; GFX9-DL-NEXT:    s_and_b32 s3, s3, s2
 ; GFX9-DL-NEXT:    s_and_b32 s2, s4, s2
-; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s2
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s5
-; GFX9-DL-NEXT:    s_bfe_u32 s9, s4, 0x80010
+; GFX9-DL-NEXT:    s_bfe_u32 s8, s4, 0x80010
 ; GFX9-DL-NEXT:    v_lshrrev_b16_e64 v1, 8, s4
+; GFX9-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
+; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s2
+; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s4
 ; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s3, v2, v3
 ; GFX9-DL-NEXT:    v_mad_u32_u24 v0, v0, v1, v2
-; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s9
-; GFX9-DL-NEXT:    s_lshr_b32 s7, s4, 24
-; GFX9-DL-NEXT:    v_mad_u32_u24 v0, s8, v1, v0
-; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s7
-; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s6, v1, v0
+; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s8
+; GFX9-DL-NEXT:    v_mad_u32_u24 v0, s7, v1, v0
+; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s6
+; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s5, v1, v0
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
@@ -1738,29 +1742,31 @@ define amdgpu_kernel void @udot4_acc32_vecMul(<4 x i8> addrspace(1)* %src1,
 ; GFX10-DL:       ; %bb.0: ; %entry
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX10-DL-NEXT:    s_movk_i32 s2, 0xff
-; GFX10-DL-NEXT:    v_mov_b32_e32 v2, 0xffff
+; GFX10-DL-NEXT:    s_movk_i32 s3, 0xff
+; GFX10-DL-NEXT:    s_mov_b32 s2, 0xffff
 ; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
-; GFX10-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
-; GFX10-DL-NEXT:    s_load_dword s5, s[0:1], 0x0
+; GFX10-DL-NEXT:    s_load_dword s4, s[4:5], 0x0
+; GFX10-DL-NEXT:    s_load_dword s5, s[6:7], 0x0
+; GFX10-DL-NEXT:    s_load_dword s6, s[0:1], 0x0
+; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-DL-NEXT:    s_and_b32 s7, s4, s3
+; GFX10-DL-NEXT:    s_and_b32 s3, s5, s3
+; GFX10-DL-NEXT:    v_mov_b32_e32 v2, s6
+; GFX10-DL-NEXT:    v_and_b32_sdwa v0, s2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_1
+; GFX10-DL-NEXT:    v_and_b32_sdwa v1, s2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_1
+; GFX10-DL-NEXT:    s_bfe_u32 s2, s4, 0x80010
+; GFX10-DL-NEXT:    s_lshr_b32 s4, s4, 24
+; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s7, s3, v2
+; GFX10-DL-NEXT:    s_bfe_u32 s3, s5, 0x80010
+; GFX10-DL-NEXT:    s_lshr_b32 s5, s5, 24
+; GFX10-DL-NEXT:    v_mad_u32_u24 v0, v0, v1, v2
+; GFX10-DL-NEXT:    v_mad_u32_u24 v0, s2, s3, v0
+; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s4, s5, v0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
-; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_and_b32 s0, s3, s2
-; GFX10-DL-NEXT:    s_and_b32 s1, s4, s2
-; GFX10-DL-NEXT:    v_mov_b32_e32 v3, s5
-; GFX10-DL-NEXT:    v_and_b32_sdwa v4, s3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
-; GFX10-DL-NEXT:    v_and_b32_sdwa v2, s4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
-; GFX10-DL-NEXT:    s_bfe_u32 s2, s3, 0x80010
-; GFX10-DL-NEXT:    s_bfe_u32 s5, s4, 0x80010
-; GFX10-DL-NEXT:    v_mad_u32_u24 v3, s0, s1, v3
-; GFX10-DL-NEXT:    s_lshr_b32 s0, s3, 24
-; GFX10-DL-NEXT:    s_lshr_b32 s1, s4, 24
-; GFX10-DL-NEXT:    v_mad_u32_u24 v2, v4, v2, v3
-; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s2, s5, v2
-; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s0, s1, v2
 ; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX10-DL-NEXT:    s_endpgm
                                               <4 x i8> addrspace(1)* %src2,
@@ -1796,39 +1802,29 @@ define amdgpu_kernel void @udot4_acc16_vecMul(<4 x i8> addrspace(1)* %src1,
 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
 ; GFX7-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX7-NEXT:    s_mov_b32 s2, -1
-; GFX7-NEXT:    s_movk_i32 s8, 0xff
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s6, s[6:7], 0x0
 ; GFX7-NEXT:    buffer_load_ushort v0, off, s[0:3], 0
-; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
+; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
+; GFX7-NEXT:    s_movk_i32 s7, 0xff
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_and_b32 s11, s4, s8
-; GFX7-NEXT:    s_bfe_u32 s6, s4, 0x80008
-; GFX7-NEXT:    s_bfe_u32 s9, s5, 0x80008
-; GFX7-NEXT:    s_lshr_b32 s10, s5, 24
-; GFX7-NEXT:    s_and_b32 s8, s5, s8
-; GFX7-NEXT:    v_mov_b32_e32 v4, s9
-; GFX7-NEXT:    s_lshr_b32 s7, s4, 24
+; GFX7-NEXT:    s_bfe_u32 s10, s6, 0x80008
+; GFX7-NEXT:    s_bfe_u32 s12, s6, 0x80010
+; GFX7-NEXT:    s_lshr_b32 s9, s6, 24
+; GFX7-NEXT:    s_and_b32 s6, s6, s7
+; GFX7-NEXT:    s_lshr_b32 s5, s4, 24
+; GFX7-NEXT:    s_bfe_u32 s8, s4, 0x80008
+; GFX7-NEXT:    s_bfe_u32 s11, s4, 0x80010
+; GFX7-NEXT:    s_and_b32 s4, s4, s7
+; GFX7-NEXT:    v_mov_b32_e32 v1, s6
 ; GFX7-NEXT:    v_mov_b32_e32 v2, s10
-; GFX7-NEXT:    s_bfe_u32 s5, s5, 0x80010
-; GFX7-NEXT:    v_mov_b32_e32 v3, s8
-; GFX7-NEXT:    v_mul_u32_u24_e32 v2, s7, v2
-; GFX7-NEXT:    v_mul_u32_u24_e32 v4, s6, v4
-; GFX7-NEXT:    s_bfe_u32 s4, s4, 0x80010
-; GFX7-NEXT:    v_mov_b32_e32 v1, s5
-; GFX7-NEXT:    v_mul_u32_u24_e32 v1, s4, v1
-; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT:    v_mul_u32_u24_e32 v3, s11, v3
-; GFX7-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX7-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX7-NEXT:    v_or_b32_e32 v2, v3, v4
-; GFX7-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
-; GFX7-NEXT:    v_lshrrev_b32_e32 v4, 16, v1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s12
 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v3, v0
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v4, v0
+; GFX7-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
+; GFX7-NEXT:    v_mad_u32_u24 v0, s8, v2, v0
+; GFX7-NEXT:    v_mad_u32_u24 v0, s11, v3, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s9
+; GFX7-NEXT:    v_mad_u32_u24 v0, s5, v1, v0
 ; GFX7-NEXT:    buffer_store_short v0, off, s[0:3], 0
 ; GFX7-NEXT:    s_endpgm
 ;
@@ -1942,27 +1938,27 @@ define amdgpu_kernel void @udot4_acc16_vecMul(<4 x i8> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v2, 0xffff
 ; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX10-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
+; GFX10-DL-NEXT:    s_load_dword s1, s[6:7], 0x0
 ; GFX10-DL-NEXT:    global_load_ushort v3, v[0:1], off
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    v_and_b32_sdwa v4, s2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
-; GFX10-DL-NEXT:    v_and_b32_sdwa v7, v2, s2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX10-DL-NEXT:    v_and_b32_sdwa v5, s3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
-; GFX10-DL-NEXT:    v_and_b32_sdwa v6, v2, s3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX10-DL-NEXT:    s_lshr_b32 s0, s2, 16
-; GFX10-DL-NEXT:    s_lshr_b32 s1, s3, 16
+; GFX10-DL-NEXT:    v_lshrrev_b16_e64 v4, 8, s0
+; GFX10-DL-NEXT:    v_and_b32_sdwa v7, v2, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX10-DL-NEXT:    v_lshrrev_b16_e64 v5, 8, s1
+; GFX10-DL-NEXT:    v_and_b32_sdwa v6, v2, s1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX10-DL-NEXT:    s_lshr_b32 s2, s0, 16
+; GFX10-DL-NEXT:    s_lshr_b32 s3, s1, 16
 ; GFX10-DL-NEXT:    v_lshl_or_b32 v4, v4, 16, v7
-; GFX10-DL-NEXT:    s_lshr_b32 s2, s2, 24
+; GFX10-DL-NEXT:    s_lshr_b32 s0, s0, 24
 ; GFX10-DL-NEXT:    v_lshl_or_b32 v5, v5, 16, v6
-; GFX10-DL-NEXT:    s_lshr_b32 s3, s3, 24
-; GFX10-DL-NEXT:    v_and_b32_sdwa v6, v2, s1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX10-DL-NEXT:    v_and_b32_sdwa v2, v2, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX10-DL-NEXT:    s_lshr_b32 s1, s1, 24
+; GFX10-DL-NEXT:    v_and_b32_sdwa v6, v2, s3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX10-DL-NEXT:    v_and_b32_sdwa v2, v2, s2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX10-DL-NEXT:    v_pk_mul_lo_u16 v4, v4, v5
-; GFX10-DL-NEXT:    v_lshl_or_b32 v5, s3, 16, v6
-; GFX10-DL-NEXT:    v_lshl_or_b32 v2, s2, 16, v2
+; GFX10-DL-NEXT:    v_lshl_or_b32 v5, s1, 16, v6
+; GFX10-DL-NEXT:    v_lshl_or_b32 v2, s0, 16, v2
 ; GFX10-DL-NEXT:    v_pk_mul_lo_u16 v2, v2, v5
 ; GFX10-DL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-DL-NEXT:    v_add_nc_u32_e32 v3, v4, v3
@@ -2023,23 +2019,23 @@ define amdgpu_kernel void @udot4_acc8_vecMul(<4 x i8> addrspace(1)* %src1,
 ; GFX7-NEXT:    v_mul_u32_u24_e32 v1, s9, v1
 ; GFX7-NEXT:    v_mul_u32_u24_e32 v2, s7, v2
 ; GFX7-NEXT:    v_mul_u32_u24_e32 v3, s6, v3
-; GFX7-NEXT:    s_and_b32 s4, s4, s8
+; GFX7-NEXT:    s_and_b32 s5, s4, s8
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
 ; GFX7-NEXT:    v_and_b32_e32 v2, s8, v2
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v3, 8, v3
 ; GFX7-NEXT:    v_or_b32_e32 v1, v2, v1
-; GFX7-NEXT:    v_or_b32_e32 v2, s4, v3
+; GFX7-NEXT:    v_or_b32_e32 v2, s5, v3
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX7-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; GFX7-NEXT:    v_or_b32_e32 v1, v2, v1
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v2, 8, v1
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
-; GFX7-NEXT:    v_lshrrev_b32_e32 v4, 24, v1
+; GFX7-NEXT:    v_lshrrev_b32_e32 v1, 24, v1
 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, s4, v0
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v3
 ; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v2, v0
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v3, v0
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v4, v0
 ; GFX7-NEXT:    buffer_store_byte v0, off, s[0:3], 0
 ; GFX7-NEXT:    s_endpgm
 ;
@@ -2055,31 +2051,32 @@ define amdgpu_kernel void @udot4_acc8_vecMul(<4 x i8> addrspace(1)* %src1,
 ; GFX8-NEXT:    s_load_dword s0, s[4:5], 0x0
 ; GFX8-NEXT:    s_load_dword s1, s[6:7], 0x0
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v3, s0
+; GFX8-NEXT:    v_mov_b32_e32 v4, s1
+; GFX8-NEXT:    s_and_b32 s7, s1, s8
 ; GFX8-NEXT:    s_lshr_b32 s2, s0, 24
 ; GFX8-NEXT:    s_lshr_b32 s3, s1, 24
 ; GFX8-NEXT:    s_bfe_u32 s6, s1, 0x80010
-; GFX8-NEXT:    s_and_b32 s7, s1, s8
-; GFX8-NEXT:    v_mov_b32_e32 v3, s0
-; GFX8-NEXT:    v_mov_b32_e32 v4, s1
 ; GFX8-NEXT:    v_mul_u32_u24_sdwa v3, v3, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
-; GFX8-NEXT:    s_bfe_u32 s4, s0, 0x80010
-; GFX8-NEXT:    v_mov_b32_e32 v5, s6
 ; GFX8-NEXT:    s_and_b32 s5, s0, s8
 ; GFX8-NEXT:    v_mov_b32_e32 v4, s7
+; GFX8-NEXT:    v_mul_u32_u24_e32 v4, s5, v4
+; GFX8-NEXT:    s_bfe_u32 s4, s0, 0x80010
+; GFX8-NEXT:    v_mov_b32_e32 v5, s6
 ; GFX8-NEXT:    v_mov_b32_e32 v6, s3
 ; GFX8-NEXT:    v_mov_b32_e32 v7, s2
-; GFX8-NEXT:    v_mul_u32_u24_e32 v4, s5, v4
+; GFX8-NEXT:    v_or_b32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX8-NEXT:    v_mul_u32_u24_e32 v5, s4, v5
 ; GFX8-NEXT:    v_mul_u32_u24_sdwa v6, v7, v6 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_and_b32_e32 v3, 0xffff, v3
 ; GFX8-NEXT:    v_or_b32_sdwa v5, v5, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT:    v_or_b32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT:    v_or_b32_sdwa v3, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT:    v_lshrrev_b32_e32 v4, 8, v3
+; GFX8-NEXT:    v_or_b32_e32 v4, v3, v5
+; GFX8-NEXT:    v_lshrrev_b32_e32 v5, 8, v4
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v2, v3
-; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v4, v2
-; GFX8-NEXT:    v_add_u32_sdwa v2, vcc, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT:    v_add_u32_sdwa v2, vcc, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v2, v5
+; GFX8-NEXT:    v_add_u32_sdwa v2, vcc, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_add_u32_sdwa v2, vcc, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
 ; GFX8-NEXT:    flat_store_byte v[0:1], v2
 ; GFX8-NEXT:    s_endpgm
 ;
@@ -2088,33 +2085,34 @@ define amdgpu_kernel void @udot4_acc8_vecMul(<4 x i8> addrspace(1)* %src1,
 ; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX9-NODL-NEXT:    s_load_dword s3, s[6:7], 0x0
+; GFX9-NODL-NEXT:    s_load_dword s2, s[6:7], 0x0
 ; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-NODL-NEXT:    global_load_ubyte v2, v[0:1], off
+; GFX9-NODL-NEXT:    s_load_dword s0, s[4:5], 0x0
 ; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT:    s_lshr_b32 s0, s2, 16
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s3
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v4, s3
-; GFX9-NODL-NEXT:    s_lshr_b32 s1, s3, 16
-; GFX9-NODL-NEXT:    s_lshr_b32 s4, s3, 24
-; GFX9-NODL-NEXT:    v_mul_lo_u16_e32 v3, s2, v3
-; GFX9-NODL-NEXT:    v_mul_lo_u16_sdwa v4, s2, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v5, s1
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s2
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v4, s2
+; GFX9-NODL-NEXT:    s_lshr_b32 s1, s2, 16
+; GFX9-NODL-NEXT:    s_lshr_b32 s3, s2, 24
+; GFX9-NODL-NEXT:    v_mul_lo_u16_e32 v3, s0, v3
+; GFX9-NODL-NEXT:    v_mul_lo_u16_sdwa v4, s0, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
 ; GFX9-NODL-NEXT:    v_or_b32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NODL-NEXT:    s_lshr_b32 s5, s2, 24
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v4, s4
-; GFX9-NODL-NEXT:    v_mul_lo_u16_sdwa v4, s5, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NODL-NEXT:    v_mul_lo_u16_e32 v5, s0, v5
+; GFX9-NODL-NEXT:    s_lshr_b32 s2, s0, 16
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v5, s1
+; GFX9-NODL-NEXT:    s_lshr_b32 s4, s0, 24
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v4, s3
+; GFX9-NODL-NEXT:    v_mul_lo_u16_sdwa v4, s4, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NODL-NEXT:    v_mul_lo_u16_e32 v5, s2, v5
+; GFX9-NODL-NEXT:    v_and_b32_e32 v3, 0xffff, v3
 ; GFX9-NODL-NEXT:    v_or_b32_sdwa v4, v5, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NODL-NEXT:    v_or_b32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NODL-NEXT:    v_lshrrev_b32_e32 v4, 8, v3
+; GFX9-NODL-NEXT:    v_or_b32_e32 v4, v3, v4
+; GFX9-NODL-NEXT:    v_lshrrev_b32_e32 v5, 8, v4
 ; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NODL-NEXT:    v_add_u32_e32 v2, v3, v2
-; GFX9-NODL-NEXT:    v_add_u32_e32 v2, v2, v4
-; GFX9-NODL-NEXT:    v_add_u32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NODL-NEXT:    v_add_u32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+; GFX9-NODL-NEXT:    v_add_u32_e32 v2, v2, v5
+; GFX9-NODL-NEXT:    v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NODL-NEXT:    v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
 ; GFX9-NODL-NEXT:    global_store_byte v[0:1], v2, off
 ; GFX9-NODL-NEXT:    s_endpgm
 ;
@@ -2123,33 +2121,34 @@ define amdgpu_kernel void @udot4_acc8_vecMul(<4 x i8> addrspace(1)* %src1,
 ; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
+; GFX9-DL-NEXT:    s_load_dword s2, s[6:7], 0x0
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-DL-NEXT:    global_load_ubyte v2, v[0:1], off
+; GFX9-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    s_lshr_b32 s0, s2, 16
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s3
-; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s3
-; GFX9-DL-NEXT:    s_lshr_b32 s1, s3, 16
-; GFX9-DL-NEXT:    s_lshr_b32 s4, s3, 24
-; GFX9-DL-NEXT:    v_mul_lo_u16_e32 v3, s2, v3
-; GFX9-DL-NEXT:    v_mul_lo_u16_sdwa v4, s2, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
-; GFX9-DL-NEXT:    v_mov_b32_e32 v5, s1
+; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s2
+; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s2
+; GFX9-DL-NEXT:    s_lshr_b32 s1, s2, 16
+; GFX9-DL-NEXT:    s_lshr_b32 s3, s2, 24
+; GFX9-DL-NEXT:    v_mul_lo_u16_e32 v3, s0, v3
+; GFX9-DL-NEXT:    v_mul_lo_u16_sdwa v4, s0, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
 ; GFX9-DL-NEXT:    v_or_b32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-DL-NEXT:    s_lshr_b32 s5, s2, 24
-; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s4
-; GFX9-DL-NEXT:    v_mul_lo_u16_sdwa v4, s5, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-DL-NEXT:    v_mul_lo_u16_e32 v5, s0, v5
+; GFX9-DL-NEXT:    s_lshr_b32 s2, s0, 16
+; GFX9-DL-NEXT:    v_mov_b32_e32 v5, s1
+; GFX9-DL-NEXT:    s_lshr_b32 s4, s0, 24
+; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s3
+; GFX9-DL-NEXT:    v_mul_lo_u16_sdwa v4, s4, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-DL-NEXT:    v_mul_lo_u16_e32 v5, s2, v5
+; GFX9-DL-NEXT:    v_and_b32_e32 v3, 0xffff, v3
 ; GFX9-DL-NEXT:    v_or_b32_sdwa v4, v5, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-DL-NEXT:    v_or_b32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-DL-NEXT:    v_lshrrev_b32_e32 v4, 8, v3
+; GFX9-DL-NEXT:    v_or_b32_e32 v4, v3, v4
+; GFX9-DL-NEXT:    v_lshrrev_b32_e32 v5, 8, v4
 ; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-DL-NEXT:    v_add_u32_e32 v2, v3, v2
-; GFX9-DL-NEXT:    v_add_u32_e32 v2, v2, v4
-; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+; GFX9-DL-NEXT:    v_add_u32_e32 v2, v2, v5
+; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
 ; GFX9-DL-NEXT:    global_store_byte v[0:1], v2, off
 ; GFX9-DL-NEXT:    s_endpgm
 ;
@@ -2157,39 +2156,36 @@ define amdgpu_kernel void @udot4_acc8_vecMul(<4 x i8> addrspace(1)* %src1,
 ; GFX10-DL:       ; %bb.0: ; %entry
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX10-DL-NEXT:    v_mov_b32_e32 v2, 0xffff
-; GFX10-DL-NEXT:    s_movk_i32 s2, 0xff
 ; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
-; GFX10-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
-; GFX10-DL-NEXT:    global_load_ubyte v3, v[0:1], off
+; GFX10-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
+; GFX10-DL-NEXT:    s_load_dword s1, s[6:7], 0x0
+; GFX10-DL-NEXT:    global_load_ubyte v2, v[0:1], off
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_lshr_b32 s0, s3, 24
-; GFX10-DL-NEXT:    s_lshr_b32 s5, s4, 24
-; GFX10-DL-NEXT:    s_lshr_b32 s1, s3, 16
-; GFX10-DL-NEXT:    s_lshr_b32 s6, s4, 16
-; GFX10-DL-NEXT:    v_and_b32_sdwa v4, s3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
-; GFX10-DL-NEXT:    v_and_b32_sdwa v5, s4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
-; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v6, s3, s4
-; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v7, s0, s5
-; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v8, s1, s6
-; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v4, v4, v5
-; GFX10-DL-NEXT:    v_and_b32_sdwa v5, v6, s2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX10-DL-NEXT:    v_and_b32_sdwa v6, v7, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-DL-NEXT:    v_and_b32_sdwa v7, v8, s2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX10-DL-NEXT:    v_and_b32_sdwa v2, v4, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-DL-NEXT:    v_or_b32_sdwa v4, v7, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-; GFX10-DL-NEXT:    v_or_b32_sdwa v2, v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-; GFX10-DL-NEXT:    v_or_b32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX10-DL-NEXT:    v_lshrrev_b32_e32 v4, 8, v2
+; GFX10-DL-NEXT:    v_lshrrev_b16_e64 v3, 8, s0
+; GFX10-DL-NEXT:    v_lshrrev_b16_e64 v4, 8, s1
+; GFX10-DL-NEXT:    s_lshr_b32 s2, s0, 24
+; GFX10-DL-NEXT:    s_lshr_b32 s3, s1, 24
+; GFX10-DL-NEXT:    s_lshr_b32 s4, s0, 16
+; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v3, v3, v4
+; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v4, s0, s1
+; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v5, s2, s3
+; GFX10-DL-NEXT:    s_lshr_b32 s0, s1, 16
+; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v3, 8, v3
+; GFX10-DL-NEXT:    v_or_b32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v4, 8, v5
+; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v5, s4, s0
+; GFX10-DL-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GFX10-DL-NEXT:    v_or_b32_sdwa v4, v5, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-DL-NEXT:    v_or_b32_e32 v4, v3, v4
+; GFX10-DL-NEXT:    v_lshrrev_b32_e32 v5, 8, v4
 ; GFX10-DL-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-DL-NEXT:    v_add_nc_u32_e32 v3, v2, v3
-; GFX10-DL-NEXT:    v_add_nc_u32_e32 v3, v3, v4
-; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v3, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v3, v2
+; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v2, v5
+; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
 ; GFX10-DL-NEXT:    global_store_byte v[0:1], v2, off
 ; GFX10-DL-NEXT:    s_endpgm
                                              <4 x i8> addrspace(1)* %src2,
diff --git a/llvm/test/CodeGen/AMDGPU/idot8s.ll b/llvm/test/CodeGen/AMDGPU/idot8s.ll
index 38cd11551..9ce54565b 100644
--- a/llvm/test/CodeGen/AMDGPU/idot8s.ll
+++ b/llvm/test/CodeGen/AMDGPU/idot8s.ll
@@ -16,38 +16,38 @@ define amdgpu_kernel void @idot8_acc32(<8 x i4> addrspace(1)* %src1,
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    s_load_dword s0, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s1, s[10:11], 0x0
-; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s21, s[4:5], 0x0
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_bfe_i32 s8, s0, 0x40000
-; GFX7-NEXT:    s_bfe_i32 s9, s1, 0x40000
-; GFX7-NEXT:    s_bfe_i32 s11, s1, 0x40004
-; GFX7-NEXT:    v_mov_b32_e32 v0, s9
-; GFX7-NEXT:    v_mov_b32_e32 v1, s2
-; GFX7-NEXT:    v_mad_i32_i24 v0, s8, v0, v1
-; GFX7-NEXT:    s_bfe_i32 s10, s0, 0x40004
-; GFX7-NEXT:    v_mov_b32_e32 v1, s11
-; GFX7-NEXT:    s_bfe_i32 s13, s1, 0x40008
-; GFX7-NEXT:    v_mad_i32_i24 v0, s10, v1, v0
-; GFX7-NEXT:    s_bfe_i32 s12, s0, 0x40008
-; GFX7-NEXT:    v_mov_b32_e32 v1, s13
-; GFX7-NEXT:    s_bfe_i32 s15, s1, 0x4000c
-; GFX7-NEXT:    v_mad_i32_i24 v0, s12, v1, v0
-; GFX7-NEXT:    s_bfe_i32 s14, s0, 0x4000c
-; GFX7-NEXT:    v_mov_b32_e32 v1, s15
-; GFX7-NEXT:    s_bfe_i32 s17, s1, 0x40010
-; GFX7-NEXT:    v_mad_i32_i24 v0, s14, v1, v0
-; GFX7-NEXT:    s_bfe_i32 s16, s0, 0x40010
-; GFX7-NEXT:    v_mov_b32_e32 v1, s17
-; GFX7-NEXT:    s_bfe_i32 s19, s1, 0x40014
-; GFX7-NEXT:    s_bfe_i32 s21, s1, 0x40018
-; GFX7-NEXT:    v_mad_i32_i24 v0, s16, v1, v0
-; GFX7-NEXT:    s_bfe_i32 s18, s0, 0x40014
-; GFX7-NEXT:    v_mov_b32_e32 v1, s19
-; GFX7-NEXT:    s_bfe_i32 s20, s0, 0x40018
-; GFX7-NEXT:    v_mad_i32_i24 v0, s18, v1, v0
+; GFX7-NEXT:    s_bfe_i32 s2, s0, 0x40000
+; GFX7-NEXT:    s_bfe_i32 s8, s1, 0x40000
+; GFX7-NEXT:    s_bfe_i32 s10, s1, 0x40004
+; GFX7-NEXT:    v_mov_b32_e32 v0, s8
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s21
+; GFX7-NEXT:    v_mad_i32_i24 v0, s2, v0, v1
+; GFX7-NEXT:    s_bfe_i32 s9, s0, 0x40004
+; GFX7-NEXT:    v_mov_b32_e32 v1, s10
+; GFX7-NEXT:    s_bfe_i32 s12, s1, 0x40008
+; GFX7-NEXT:    v_mad_i32_i24 v0, s9, v1, v0
+; GFX7-NEXT:    s_bfe_i32 s11, s0, 0x40008
+; GFX7-NEXT:    v_mov_b32_e32 v1, s12
+; GFX7-NEXT:    s_bfe_i32 s14, s1, 0x4000c
+; GFX7-NEXT:    v_mad_i32_i24 v0, s11, v1, v0
+; GFX7-NEXT:    s_bfe_i32 s13, s0, 0x4000c
+; GFX7-NEXT:    v_mov_b32_e32 v1, s14
+; GFX7-NEXT:    s_bfe_i32 s16, s1, 0x40010
+; GFX7-NEXT:    v_mad_i32_i24 v0, s13, v1, v0
+; GFX7-NEXT:    s_bfe_i32 s15, s0, 0x40010
+; GFX7-NEXT:    v_mov_b32_e32 v1, s16
+; GFX7-NEXT:    s_bfe_i32 s18, s1, 0x40014
+; GFX7-NEXT:    s_bfe_i32 s20, s1, 0x40018
+; GFX7-NEXT:    v_mad_i32_i24 v0, s15, v1, v0
+; GFX7-NEXT:    s_bfe_i32 s17, s0, 0x40014
+; GFX7-NEXT:    v_mov_b32_e32 v1, s18
+; GFX7-NEXT:    s_bfe_i32 s19, s0, 0x40018
+; GFX7-NEXT:    v_mad_i32_i24 v0, s17, v1, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s20
 ; GFX7-NEXT:    s_ashr_i32 s1, s1, 28
-; GFX7-NEXT:    v_mad_i32_i24 v0, s20, v1, v0
+; GFX7-NEXT:    v_mad_i32_i24 v0, s19, v1, v0
 ; GFX7-NEXT:    s_ashr_i32 s0, s0, 28
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX7-NEXT:    v_mad_i32_i24 v0, s0, v1, v0
@@ -61,43 +61,43 @@ define amdgpu_kernel void @idot8_acc32(<8 x i4> addrspace(1)* %src1,
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
 ; GFX8-NEXT:    s_load_dword s4, s[6:7], 0x0
-; GFX8-NEXT:    s_load_dword s5, s[0:1], 0x0
-; GFX8-NEXT:    v_mov_b32_e32 v0, s0
-; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    s_load_dword s19, s[0:1], 0x0
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_bfe_i32 s0, s2, 0x40000
-; GFX8-NEXT:    s_bfe_i32 s1, s4, 0x40000
-; GFX8-NEXT:    s_bfe_i32 s7, s4, 0x40004
-; GFX8-NEXT:    v_mov_b32_e32 v2, s1
-; GFX8-NEXT:    v_mov_b32_e32 v3, s5
-; GFX8-NEXT:    v_mad_i32_i24 v2, s0, v2, v3
-; GFX8-NEXT:    s_bfe_i32 s6, s2, 0x40004
-; GFX8-NEXT:    v_mov_b32_e32 v3, s7
-; GFX8-NEXT:    s_bfe_i32 s9, s4, 0x40008
-; GFX8-NEXT:    v_mad_i32_i24 v2, s6, v3, v2
-; GFX8-NEXT:    s_bfe_i32 s8, s2, 0x40008
-; GFX8-NEXT:    v_mov_b32_e32 v3, s9
-; GFX8-NEXT:    s_bfe_i32 s11, s4, 0x4000c
-; GFX8-NEXT:    v_mad_i32_i24 v2, s8, v3, v2
-; GFX8-NEXT:    s_bfe_i32 s10, s2, 0x4000c
-; GFX8-NEXT:    v_mov_b32_e32 v3, s11
-; GFX8-NEXT:    s_bfe_i32 s13, s4, 0x40010
-; GFX8-NEXT:    v_mad_i32_i24 v2, s10, v3, v2
-; GFX8-NEXT:    s_bfe_i32 s12, s2, 0x40010
-; GFX8-NEXT:    v_mov_b32_e32 v3, s13
-; GFX8-NEXT:    s_bfe_i32 s15, s4, 0x40014
-; GFX8-NEXT:    s_bfe_i32 s17, s4, 0x40018
-; GFX8-NEXT:    v_mad_i32_i24 v2, s12, v3, v2
-; GFX8-NEXT:    s_bfe_i32 s14, s2, 0x40014
-; GFX8-NEXT:    v_mov_b32_e32 v3, s15
-; GFX8-NEXT:    s_bfe_i32 s16, s2, 0x40018
-; GFX8-NEXT:    v_mad_i32_i24 v2, s14, v3, v2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s17
+; GFX8-NEXT:    s_bfe_i32 s5, s2, 0x40000
+; GFX8-NEXT:    s_bfe_i32 s6, s4, 0x40000
+; GFX8-NEXT:    s_bfe_i32 s8, s4, 0x40004
+; GFX8-NEXT:    v_mov_b32_e32 v0, s6
+; GFX8-NEXT:    v_mov_b32_e32 v1, s19
+; GFX8-NEXT:    v_mad_i32_i24 v0, s5, v0, v1
+; GFX8-NEXT:    s_bfe_i32 s7, s2, 0x40004
+; GFX8-NEXT:    v_mov_b32_e32 v1, s8
+; GFX8-NEXT:    s_bfe_i32 s10, s4, 0x40008
+; GFX8-NEXT:    v_mad_i32_i24 v0, s7, v1, v0
+; GFX8-NEXT:    s_bfe_i32 s9, s2, 0x40008
+; GFX8-NEXT:    v_mov_b32_e32 v1, s10
+; GFX8-NEXT:    s_bfe_i32 s12, s4, 0x4000c
+; GFX8-NEXT:    v_mad_i32_i24 v0, s9, v1, v0
+; GFX8-NEXT:    s_bfe_i32 s11, s2, 0x4000c
+; GFX8-NEXT:    v_mov_b32_e32 v1, s12
+; GFX8-NEXT:    s_bfe_i32 s14, s4, 0x40010
+; GFX8-NEXT:    v_mad_i32_i24 v0, s11, v1, v0
+; GFX8-NEXT:    s_bfe_i32 s13, s2, 0x40010
+; GFX8-NEXT:    v_mov_b32_e32 v1, s14
+; GFX8-NEXT:    s_bfe_i32 s16, s4, 0x40014
+; GFX8-NEXT:    s_bfe_i32 s18, s4, 0x40018
+; GFX8-NEXT:    v_mad_i32_i24 v0, s13, v1, v0
+; GFX8-NEXT:    s_bfe_i32 s15, s2, 0x40014
+; GFX8-NEXT:    v_mov_b32_e32 v1, s16
+; GFX8-NEXT:    s_bfe_i32 s17, s2, 0x40018
+; GFX8-NEXT:    v_mad_i32_i24 v0, s15, v1, v0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s18
 ; GFX8-NEXT:    s_ashr_i32 s4, s4, 28
-; GFX8-NEXT:    v_mad_i32_i24 v2, s16, v3, v2
+; GFX8-NEXT:    v_mad_i32_i24 v0, s17, v1, v0
 ; GFX8-NEXT:    s_ashr_i32 s2, s2, 28
-; GFX8-NEXT:    v_mov_b32_e32 v3, s4
-; GFX8-NEXT:    v_mad_i32_i24 v2, s2, v3, v2
+; GFX8-NEXT:    v_mov_b32_e32 v1, s4
+; GFX8-NEXT:    v_mad_i32_i24 v2, s2, v1, v0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX8-NEXT:    flat_store_dword v[0:1], v2
 ; GFX8-NEXT:    s_endpgm
 ;
@@ -108,43 +108,43 @@ define amdgpu_kernel void @idot8_acc32(<8 x i4> addrspace(1)* %src1,
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x0
 ; GFX9-NEXT:    s_load_dword s4, s[6:7], 0x0
-; GFX9-NEXT:    s_load_dword s5, s[0:1], 0x0
-; GFX9-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    s_load_dword s19, s[0:1], 0x0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_bfe_i32 s0, s2, 0x40000
-; GFX9-NEXT:    s_bfe_i32 s1, s4, 0x40000
-; GFX9-NEXT:    s_bfe_i32 s7, s4, 0x40004
-; GFX9-NEXT:    v_mov_b32_e32 v2, s1
-; GFX9-NEXT:    v_mov_b32_e32 v3, s5
-; GFX9-NEXT:    v_mad_i32_i24 v2, s0, v2, v3
-; GFX9-NEXT:    s_bfe_i32 s6, s2, 0x40004
-; GFX9-NEXT:    v_mov_b32_e32 v3, s7
-; GFX9-NEXT:    s_bfe_i32 s9, s4, 0x40008
-; GFX9-NEXT:    v_mad_i32_i24 v2, s6, v3, v2
-; GFX9-NEXT:    s_bfe_i32 s8, s2, 0x40008
-; GFX9-NEXT:    v_mov_b32_e32 v3, s9
-; GFX9-NEXT:    s_bfe_i32 s11, s4, 0x4000c
-; GFX9-NEXT:    v_mad_i32_i24 v2, s8, v3, v2
-; GFX9-NEXT:    s_bfe_i32 s10, s2, 0x4000c
-; GFX9-NEXT:    v_mov_b32_e32 v3, s11
-; GFX9-NEXT:    s_bfe_i32 s13, s4, 0x40010
-; GFX9-NEXT:    v_mad_i32_i24 v2, s10, v3, v2
-; GFX9-NEXT:    s_bfe_i32 s12, s2, 0x40010
-; GFX9-NEXT:    v_mov_b32_e32 v3, s13
-; GFX9-NEXT:    s_bfe_i32 s15, s4, 0x40014
-; GFX9-NEXT:    s_bfe_i32 s17, s4, 0x40018
-; GFX9-NEXT:    v_mad_i32_i24 v2, s12, v3, v2
-; GFX9-NEXT:    s_bfe_i32 s14, s2, 0x40014
-; GFX9-NEXT:    v_mov_b32_e32 v3, s15
-; GFX9-NEXT:    s_bfe_i32 s16, s2, 0x40018
-; GFX9-NEXT:    v_mad_i32_i24 v2, s14, v3, v2
-; GFX9-NEXT:    v_mov_b32_e32 v3, s17
+; GFX9-NEXT:    s_bfe_i32 s5, s2, 0x40000
+; GFX9-NEXT:    s_bfe_i32 s6, s4, 0x40000
+; GFX9-NEXT:    s_bfe_i32 s8, s4, 0x40004
+; GFX9-NEXT:    v_mov_b32_e32 v0, s6
+; GFX9-NEXT:    v_mov_b32_e32 v1, s19
+; GFX9-NEXT:    v_mad_i32_i24 v0, s5, v0, v1
+; GFX9-NEXT:    s_bfe_i32 s7, s2, 0x40004
+; GFX9-NEXT:    v_mov_b32_e32 v1, s8
+; GFX9-NEXT:    s_bfe_i32 s10, s4, 0x40008
+; GFX9-NEXT:    v_mad_i32_i24 v0, s7, v1, v0
+; GFX9-NEXT:    s_bfe_i32 s9, s2, 0x40008
+; GFX9-NEXT:    v_mov_b32_e32 v1, s10
+; GFX9-NEXT:    s_bfe_i32 s12, s4, 0x4000c
+; GFX9-NEXT:    v_mad_i32_i24 v0, s9, v1, v0
+; GFX9-NEXT:    s_bfe_i32 s11, s2, 0x4000c
+; GFX9-NEXT:    v_mov_b32_e32 v1, s12
+; GFX9-NEXT:    s_bfe_i32 s14, s4, 0x40010
+; GFX9-NEXT:    v_mad_i32_i24 v0, s11, v1, v0
+; GFX9-NEXT:    s_bfe_i32 s13, s2, 0x40010
+; GFX9-NEXT:    v_mov_b32_e32 v1, s14
+; GFX9-NEXT:    s_bfe_i32 s16, s4, 0x40014
+; GFX9-NEXT:    s_bfe_i32 s18, s4, 0x40018
+; GFX9-NEXT:    v_mad_i32_i24 v0, s13, v1, v0
+; GFX9-NEXT:    s_bfe_i32 s15, s2, 0x40014
+; GFX9-NEXT:    v_mov_b32_e32 v1, s16
+; GFX9-NEXT:    s_bfe_i32 s17, s2, 0x40018
+; GFX9-NEXT:    v_mad_i32_i24 v0, s15, v1, v0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s18
 ; GFX9-NEXT:    s_ashr_i32 s4, s4, 28
-; GFX9-NEXT:    v_mad_i32_i24 v2, s16, v3, v2
+; GFX9-NEXT:    v_mad_i32_i24 v0, s17, v1, v0
 ; GFX9-NEXT:    s_ashr_i32 s2, s2, 28
-; GFX9-NEXT:    v_mov_b32_e32 v3, s4
-; GFX9-NEXT:    v_mad_i32_i24 v2, s2, v3, v2
+; GFX9-NEXT:    v_mov_b32_e32 v1, s4
+; GFX9-NEXT:    v_mad_i32_i24 v2, s2, v1, v0
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-NEXT:    s_endpgm
 ;
@@ -153,15 +153,15 @@ define amdgpu_kernel void @idot8_acc32(<8 x i4> addrspace(1)* %src1,
 ; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
-; GFX9-DL-NEXT:    s_load_dword s5, s[0:1], 0x0
+; GFX9-DL-NEXT:    s_load_dword s2, s[6:7], 0x0
+; GFX9-DL-NEXT:    s_load_dword s6, s[0:1], 0x0
+; GFX9-DL-NEXT:    s_load_dword s4, s[4:5], 0x0
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s4
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s5
-; GFX9-DL-NEXT:    v_dot8_i32_i4 v2, s2, v2, v3
+; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s2
+; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s6
+; GFX9-DL-NEXT:    v_dot8_i32_i4 v2, s4, v2, v3
 ; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-DL-NEXT:    s_endpgm
 ;
@@ -171,14 +171,14 @@ define amdgpu_kernel void @idot8_acc32(<8 x i4> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX10-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
-; GFX10-DL-NEXT:    s_load_dword s5, s[0:1], 0x0
+; GFX10-DL-NEXT:    s_load_dword s2, s[0:1], 0x0
+; GFX10-DL-NEXT:    s_load_dword s4, s[4:5], 0x0
+; GFX10-DL-NEXT:    s_load_dword s5, s[6:7], 0x0
+; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-DL-NEXT:    v_dot8_i32_i4 v2, s4, s5, v0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
-; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    v_mov_b32_e32 v2, s5
-; GFX10-DL-NEXT:    v_dot8_i32_i4 v2, s2, s4, v2
 ; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX10-DL-NEXT:    s_endpgm
                                        <8 x i4> addrspace(1)* %src2,
@@ -321,49 +321,49 @@ define amdgpu_kernel void @idot8_acc16(<8 x i4> addrspace(1)* %src1,
 ; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX8-NEXT:    s_load_dword s4, s[6:7], 0x0
+; GFX8-NEXT:    s_load_dword s2, s[6:7], 0x0
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX8-NEXT:    flat_load_ushort v2, v[0:1]
+; GFX8-NEXT:    s_load_dword s0, s[4:5], 0x0
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_bfe_i32 s0, s2, 0x40000
-; GFX8-NEXT:    s_bfe_i32 s1, s4, 0x40000
+; GFX8-NEXT:    s_bfe_i32 s1, s2, 0x40000
+; GFX8-NEXT:    s_bfe_i32 s4, s2, 0x40004
+; GFX8-NEXT:    s_bfe_i32 s5, s2, 0x40008
 ; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    s_bfe_i32 s5, s4, 0x40004
-; GFX8-NEXT:    s_bfe_i32 s6, s4, 0x40008
-; GFX8-NEXT:    s_lshr_b32 s1, s2, 12
-; GFX8-NEXT:    s_lshr_b32 s7, s4, 12
-; GFX8-NEXT:    s_bfe_i32 s8, s2, 0x40004
-; GFX8-NEXT:    s_bfe_i32 s9, s2, 0x40008
-; GFX8-NEXT:    v_mov_b32_e32 v4, s6
-; GFX8-NEXT:    v_mov_b32_e32 v7, s5
-; GFX8-NEXT:    v_lshlrev_b16_e64 v5, 12, s1
-; GFX8-NEXT:    v_lshlrev_b16_e64 v6, 12, s7
+; GFX8-NEXT:    s_bfe_i32 s6, s0, 0x40000
+; GFX8-NEXT:    s_lshr_b32 s1, s0, 12
+; GFX8-NEXT:    s_lshr_b32 s7, s2, 12
+; GFX8-NEXT:    v_mov_b32_e32 v4, s5
+; GFX8-NEXT:    s_bfe_i32 s9, s0, 0x40008
+; GFX8-NEXT:    v_mov_b32_e32 v5, s4
+; GFX8-NEXT:    s_bfe_i32 s8, s0, 0x40004
+; GFX8-NEXT:    v_lshlrev_b16_e64 v6, 12, s1
+; GFX8-NEXT:    v_lshlrev_b16_e64 v7, 12, s7
 ; GFX8-NEXT:    v_mul_i32_i24_e32 v4, s9, v4
-; GFX8-NEXT:    s_bfe_i32 s10, s4, 0x40010
-; GFX8-NEXT:    v_ashrrev_i16_e32 v5, 12, v5
+; GFX8-NEXT:    s_bfe_i32 s1, s2, 0x40010
 ; GFX8-NEXT:    v_ashrrev_i16_e32 v6, 12, v6
-; GFX8-NEXT:    s_bfe_i32 s12, s4, 0x40014
-; GFX8-NEXT:    s_bfe_i32 s11, s2, 0x40010
-; GFX8-NEXT:    v_mov_b32_e32 v8, s10
-; GFX8-NEXT:    s_bfe_i32 s14, s4, 0x40018
-; GFX8-NEXT:    s_bfe_i32 s13, s2, 0x40014
-; GFX8-NEXT:    v_mov_b32_e32 v9, s12
-; GFX8-NEXT:    s_bfe_i32 s15, s2, 0x40018
-; GFX8-NEXT:    s_ashr_i32 s4, s4, 28
-; GFX8-NEXT:    v_mov_b32_e32 v10, s14
+; GFX8-NEXT:    v_ashrrev_i16_e32 v7, 12, v7
+; GFX8-NEXT:    s_bfe_i32 s5, s2, 0x40014
+; GFX8-NEXT:    v_mov_b32_e32 v8, s1
+; GFX8-NEXT:    s_bfe_i32 s4, s0, 0x40010
+; GFX8-NEXT:    s_bfe_i32 s7, s2, 0x40018
+; GFX8-NEXT:    v_mov_b32_e32 v9, s5
+; GFX8-NEXT:    s_bfe_i32 s1, s0, 0x40014
+; GFX8-NEXT:    s_bfe_i32 s5, s0, 0x40018
 ; GFX8-NEXT:    s_ashr_i32 s2, s2, 28
+; GFX8-NEXT:    v_mov_b32_e32 v10, s7
+; GFX8-NEXT:    s_ashr_i32 s0, s0, 28
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
-; GFX8-NEXT:    v_mad_i32_i24 v2, s0, v3, v2
-; GFX8-NEXT:    v_mad_i32_i24 v2, s8, v7, v2
+; GFX8-NEXT:    v_mad_i32_i24 v2, s6, v3, v2
+; GFX8-NEXT:    v_mad_i32_i24 v2, s8, v5, v2
 ; GFX8-NEXT:    v_add_u32_sdwa v2, vcc, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
-; GFX8-NEXT:    v_mad_u32_u24 v2, v5, v6, v2
-; GFX8-NEXT:    v_mad_i32_i24 v2, s11, v8, v2
-; GFX8-NEXT:    v_mad_i32_i24 v2, s13, v9, v2
-; GFX8-NEXT:    v_mad_i32_i24 v2, s15, v10, v2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s4
-; GFX8-NEXT:    v_mad_i32_i24 v2, s2, v3, v2
+; GFX8-NEXT:    v_mad_u32_u24 v2, v6, v7, v2
+; GFX8-NEXT:    v_mad_i32_i24 v2, s4, v8, v2
+; GFX8-NEXT:    v_mad_i32_i24 v2, s1, v9, v2
+; GFX8-NEXT:    v_mad_i32_i24 v2, s5, v10, v2
+; GFX8-NEXT:    v_mov_b32_e32 v3, s2
+; GFX8-NEXT:    v_mad_i32_i24 v2, s0, v3, v2
 ; GFX8-NEXT:    flat_store_short v[0:1], v2
 ; GFX8-NEXT:    s_endpgm
 ;
@@ -372,49 +372,49 @@ define amdgpu_kernel void @idot8_acc16(<8 x i4> addrspace(1)* %src1,
 ; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX9-NEXT:    s_load_dword s4, s[6:7], 0x0
+; GFX9-NEXT:    s_load_dword s2, s[6:7], 0x0
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-NEXT:    global_load_ushort v2, v[0:1], off
+; GFX9-NEXT:    s_load_dword s0, s[4:5], 0x0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_bfe_i32 s0, s2, 0x40000
-; GFX9-NEXT:    s_bfe_i32 s1, s4, 0x40000
+; GFX9-NEXT:    s_bfe_i32 s1, s2, 0x40000
+; GFX9-NEXT:    s_bfe_i32 s4, s2, 0x40004
+; GFX9-NEXT:    s_bfe_i32 s5, s2, 0x40008
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s1
-; GFX9-NEXT:    s_bfe_i32 s5, s4, 0x40004
-; GFX9-NEXT:    s_bfe_i32 s6, s4, 0x40008
-; GFX9-NEXT:    s_lshr_b32 s1, s2, 12
-; GFX9-NEXT:    s_lshr_b32 s7, s4, 12
-; GFX9-NEXT:    s_bfe_i32 s8, s2, 0x40004
-; GFX9-NEXT:    s_bfe_i32 s9, s2, 0x40008
-; GFX9-NEXT:    v_mov_b32_e32 v4, s6
-; GFX9-NEXT:    v_mov_b32_e32 v7, s5
-; GFX9-NEXT:    v_lshlrev_b16_e64 v5, 12, s1
-; GFX9-NEXT:    v_lshlrev_b16_e64 v6, 12, s7
+; GFX9-NEXT:    s_bfe_i32 s6, s0, 0x40000
+; GFX9-NEXT:    s_lshr_b32 s1, s0, 12
+; GFX9-NEXT:    s_lshr_b32 s7, s2, 12
+; GFX9-NEXT:    v_mov_b32_e32 v4, s5
+; GFX9-NEXT:    s_bfe_i32 s9, s0, 0x40008
+; GFX9-NEXT:    v_mov_b32_e32 v5, s4
+; GFX9-NEXT:    s_bfe_i32 s8, s0, 0x40004
+; GFX9-NEXT:    v_lshlrev_b16_e64 v6, 12, s1
+; GFX9-NEXT:    v_lshlrev_b16_e64 v7, 12, s7
 ; GFX9-NEXT:    v_mul_i32_i24_e32 v4, s9, v4
-; GFX9-NEXT:    s_bfe_i32 s10, s4, 0x40010
-; GFX9-NEXT:    v_ashrrev_i16_e32 v5, 12, v5
+; GFX9-NEXT:    s_bfe_i32 s1, s2, 0x40010
 ; GFX9-NEXT:    v_ashrrev_i16_e32 v6, 12, v6
-; GFX9-NEXT:    s_bfe_i32 s12, s4, 0x40014
-; GFX9-NEXT:    s_bfe_i32 s11, s2, 0x40010
-; GFX9-NEXT:    v_mov_b32_e32 v8, s10
-; GFX9-NEXT:    s_bfe_i32 s14, s4, 0x40018
-; GFX9-NEXT:    s_bfe_i32 s13, s2, 0x40014
-; GFX9-NEXT:    v_mov_b32_e32 v9, s12
-; GFX9-NEXT:    s_bfe_i32 s15, s2, 0x40018
-; GFX9-NEXT:    s_ashr_i32 s4, s4, 28
-; GFX9-NEXT:    v_mov_b32_e32 v10, s14
+; GFX9-NEXT:    v_ashrrev_i16_e32 v7, 12, v7
+; GFX9-NEXT:    s_bfe_i32 s5, s2, 0x40014
+; GFX9-NEXT:    v_mov_b32_e32 v8, s1
+; GFX9-NEXT:    s_bfe_i32 s4, s0, 0x40010
+; GFX9-NEXT:    s_bfe_i32 s7, s2, 0x40018
+; GFX9-NEXT:    v_mov_b32_e32 v9, s5
+; GFX9-NEXT:    s_bfe_i32 s1, s0, 0x40014
+; GFX9-NEXT:    s_bfe_i32 s5, s0, 0x40018
 ; GFX9-NEXT:    s_ashr_i32 s2, s2, 28
+; GFX9-NEXT:    v_mov_b32_e32 v10, s7
+; GFX9-NEXT:    s_ashr_i32 s0, s0, 28
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_mad_i32_i24 v2, s0, v3, v2
-; GFX9-NEXT:    v_mad_i32_i24 v2, s8, v7, v2
+; GFX9-NEXT:    v_mad_i32_i24 v2, s6, v3, v2
+; GFX9-NEXT:    v_mad_i32_i24 v2, s8, v5, v2
 ; GFX9-NEXT:    v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
-; GFX9-NEXT:    v_mad_u32_u24 v2, v5, v6, v2
-; GFX9-NEXT:    v_mad_i32_i24 v2, s11, v8, v2
-; GFX9-NEXT:    v_mad_i32_i24 v2, s13, v9, v2
-; GFX9-NEXT:    v_mad_i32_i24 v2, s15, v10, v2
-; GFX9-NEXT:    v_mov_b32_e32 v3, s4
-; GFX9-NEXT:    v_mad_i32_i24 v2, s2, v3, v2
+; GFX9-NEXT:    v_mad_u32_u24 v2, v6, v7, v2
+; GFX9-NEXT:    v_mad_i32_i24 v2, s4, v8, v2
+; GFX9-NEXT:    v_mad_i32_i24 v2, s1, v9, v2
+; GFX9-NEXT:    v_mad_i32_i24 v2, s5, v10, v2
+; GFX9-NEXT:    v_mov_b32_e32 v3, s2
+; GFX9-NEXT:    v_mad_i32_i24 v2, s0, v3, v2
 ; GFX9-NEXT:    global_store_short v[0:1], v2, off
 ; GFX9-NEXT:    s_endpgm
 ;
@@ -423,49 +423,49 @@ define amdgpu_kernel void @idot8_acc16(<8 x i4> addrspace(1)* %src1,
 ; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
+; GFX9-DL-NEXT:    s_load_dword s2, s[6:7], 0x0
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-DL-NEXT:    global_load_ushort v2, v[0:1], off
+; GFX9-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    s_bfe_i32 s0, s2, 0x40000
-; GFX9-DL-NEXT:    s_bfe_i32 s1, s4, 0x40000
+; GFX9-DL-NEXT:    s_bfe_i32 s1, s2, 0x40000
+; GFX9-DL-NEXT:    s_bfe_i32 s4, s2, 0x40004
+; GFX9-DL-NEXT:    s_bfe_i32 s5, s2, 0x40008
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s1
-; GFX9-DL-NEXT:    s_bfe_i32 s5, s4, 0x40004
-; GFX9-DL-NEXT:    s_bfe_i32 s6, s4, 0x40008
-; GFX9-DL-NEXT:    s_lshr_b32 s1, s2, 12
-; GFX9-DL-NEXT:    s_lshr_b32 s7, s4, 12
-; GFX9-DL-NEXT:    s_bfe_i32 s8, s2, 0x40004
-; GFX9-DL-NEXT:    s_bfe_i32 s9, s2, 0x40008
-; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s6
-; GFX9-DL-NEXT:    v_mov_b32_e32 v7, s5
-; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v5, 12, s1
-; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v6, 12, s7
+; GFX9-DL-NEXT:    s_bfe_i32 s6, s0, 0x40000
+; GFX9-DL-NEXT:    s_lshr_b32 s1, s0, 12
+; GFX9-DL-NEXT:    s_lshr_b32 s7, s2, 12
+; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s5
+; GFX9-DL-NEXT:    s_bfe_i32 s9, s0, 0x40008
+; GFX9-DL-NEXT:    v_mov_b32_e32 v5, s4
+; GFX9-DL-NEXT:    s_bfe_i32 s8, s0, 0x40004
+; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v6, 12, s1
+; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v7, 12, s7
 ; GFX9-DL-NEXT:    v_mul_i32_i24_e32 v4, s9, v4
-; GFX9-DL-NEXT:    s_bfe_i32 s10, s4, 0x40010
-; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v5, 12, v5
+; GFX9-DL-NEXT:    s_bfe_i32 s1, s2, 0x40010
 ; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v6, 12, v6
-; GFX9-DL-NEXT:    s_bfe_i32 s12, s4, 0x40014
-; GFX9-DL-NEXT:    s_bfe_i32 s11, s2, 0x40010
-; GFX9-DL-NEXT:    v_mov_b32_e32 v8, s10
-; GFX9-DL-NEXT:    s_bfe_i32 s14, s4, 0x40018
-; GFX9-DL-NEXT:    s_bfe_i32 s13, s2, 0x40014
-; GFX9-DL-NEXT:    v_mov_b32_e32 v9, s12
-; GFX9-DL-NEXT:    s_bfe_i32 s15, s2, 0x40018
-; GFX9-DL-NEXT:    s_ashr_i32 s4, s4, 28
-; GFX9-DL-NEXT:    v_mov_b32_e32 v10, s14
+; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v7, 12, v7
+; GFX9-DL-NEXT:    s_bfe_i32 s5, s2, 0x40014
+; GFX9-DL-NEXT:    v_mov_b32_e32 v8, s1
+; GFX9-DL-NEXT:    s_bfe_i32 s4, s0, 0x40010
+; GFX9-DL-NEXT:    s_bfe_i32 s7, s2, 0x40018
+; GFX9-DL-NEXT:    v_mov_b32_e32 v9, s5
+; GFX9-DL-NEXT:    s_bfe_i32 s1, s0, 0x40014
+; GFX9-DL-NEXT:    s_bfe_i32 s5, s0, 0x40018
 ; GFX9-DL-NEXT:    s_ashr_i32 s2, s2, 28
+; GFX9-DL-NEXT:    v_mov_b32_e32 v10, s7
+; GFX9-DL-NEXT:    s_ashr_i32 s0, s0, 28
 ; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s0, v3, v2
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s8, v7, v2
+; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s6, v3, v2
+; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s8, v5, v2
 ; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
-; GFX9-DL-NEXT:    v_mad_u32_u24 v2, v5, v6, v2
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s11, v8, v2
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s13, v9, v2
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s15, v10, v2
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s4
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s2, v3, v2
+; GFX9-DL-NEXT:    v_mad_u32_u24 v2, v6, v7, v2
+; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s4, v8, v2
+; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s1, v9, v2
+; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s5, v10, v2
+; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s2
+; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s0, v3, v2
 ; GFX9-DL-NEXT:    global_store_short v[0:1], v2, off
 ; GFX9-DL-NEXT:    s_endpgm
 ;
@@ -473,49 +473,47 @@ define amdgpu_kernel void @idot8_acc16(<8 x i4> addrspace(1)* %src1,
 ; GFX10-DL:       ; %bb.0: ; %entry
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX10-DL-NEXT:    v_mov_b32_e32 v2, 0xffff
+; GFX10-DL-NEXT:    s_mov_b32 s2, 0xffff
 ; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX10-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
-; GFX10-DL-NEXT:    global_load_ushort v3, v[0:1], off
+; GFX10-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
+; GFX10-DL-NEXT:    s_load_dword s1, s[6:7], 0x0
+; GFX10-DL-NEXT:    global_load_ushort v2, v[0:1], off
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_lshr_b32 s0, s2, 12
-; GFX10-DL-NEXT:    s_lshr_b32 s1, s4, 12
-; GFX10-DL-NEXT:    s_bfe_i32 s5, s2, 0x40000
-; GFX10-DL-NEXT:    s_bfe_i32 s6, s4, 0x40000
-; GFX10-DL-NEXT:    s_bfe_i32 s7, s2, 0x40004
-; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v4, 12, s0
-; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v5, 12, s1
-; GFX10-DL-NEXT:    s_bfe_i32 s0, s4, 0x40004
-; GFX10-DL-NEXT:    s_bfe_i32 s1, s2, 0x40008
-; GFX10-DL-NEXT:    s_bfe_i32 s8, s4, 0x40008
-; GFX10-DL-NEXT:    v_and_b32_e32 v4, v4, v2
-; GFX10-DL-NEXT:    v_and_b32_e32 v5, v5, v2
-; GFX10-DL-NEXT:    s_bfe_i32 s9, s2, 0x40010
-; GFX10-DL-NEXT:    s_bfe_i32 s10, s4, 0x40010
-; GFX10-DL-NEXT:    v_mul_i32_i24_e64 v6, s1, s8
+; GFX10-DL-NEXT:    s_lshr_b32 s4, s0, 12
+; GFX10-DL-NEXT:    s_lshr_b32 s5, s1, 12
+; GFX10-DL-NEXT:    s_bfe_i32 s6, s0, 0x40000
+; GFX10-DL-NEXT:    s_bfe_i32 s7, s1, 0x40000
+; GFX10-DL-NEXT:    s_bfe_i32 s8, s0, 0x40004
+; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v3, 12, s4
+; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v4, 12, s5
+; GFX10-DL-NEXT:    s_bfe_i32 s9, s1, 0x40004
+; GFX10-DL-NEXT:    s_bfe_i32 s10, s0, 0x40008
+; GFX10-DL-NEXT:    s_bfe_i32 s11, s1, 0x40008
+; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v3, 12, v3
 ; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v4, 12, v4
-; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v5, 12, v5
-; GFX10-DL-NEXT:    s_bfe_i32 s1, s2, 0x40014
-; GFX10-DL-NEXT:    s_bfe_i32 s8, s4, 0x40014
-; GFX10-DL-NEXT:    s_bfe_i32 s11, s2, 0x40018
-; GFX10-DL-NEXT:    v_and_b32_e32 v4, v4, v2
-; GFX10-DL-NEXT:    v_and_b32_e32 v2, v5, v2
-; GFX10-DL-NEXT:    s_bfe_i32 s12, s4, 0x40018
-; GFX10-DL-NEXT:    s_ashr_i32 s2, s2, 28
-; GFX10-DL-NEXT:    s_ashr_i32 s4, s4, 28
+; GFX10-DL-NEXT:    s_bfe_i32 s4, s1, 0x40010
+; GFX10-DL-NEXT:    s_bfe_i32 s5, s0, 0x40014
+; GFX10-DL-NEXT:    v_mul_i32_i24_e64 v5, s10, s11
+; GFX10-DL-NEXT:    v_and_b32_e32 v3, s2, v3
+; GFX10-DL-NEXT:    v_and_b32_e32 v4, s2, v4
+; GFX10-DL-NEXT:    s_bfe_i32 s2, s0, 0x40010
 ; GFX10-DL-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-DL-NEXT:    v_mad_i32_i24 v3, s5, s6, v3
-; GFX10-DL-NEXT:    v_mad_i32_i24 v3, s7, s0, v3
-; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v3, v3, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
-; GFX10-DL-NEXT:    v_mad_u32_u24 v2, v4, v2, v3
-; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s9, s10, v2
-; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s1, s8, v2
-; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s11, s12, v2
+; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s6, s7, v2
+; GFX10-DL-NEXT:    s_bfe_i32 s6, s1, 0x40014
+; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s8, s9, v2
+; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
+; GFX10-DL-NEXT:    v_mad_u32_u24 v2, v3, v4, v2
+; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s2, s4, v2
+; GFX10-DL-NEXT:    s_bfe_i32 s2, s0, 0x40018
+; GFX10-DL-NEXT:    s_bfe_i32 s4, s1, 0x40018
+; GFX10-DL-NEXT:    s_ashr_i32 s0, s0, 28
+; GFX10-DL-NEXT:    s_ashr_i32 s1, s1, 28
+; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s5, s6, v2
 ; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s2, s4, v2
+; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s0, s1, v2
 ; GFX10-DL-NEXT:    global_store_short v[0:1], v2, off
 ; GFX10-DL-NEXT:    s_endpgm
                                        <8 x i4> addrspace(1)* %src2,
@@ -658,21 +656,21 @@ define amdgpu_kernel void @idot8_acc8(<8 x i4> addrspace(1)* %src1,
 ; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX8-NEXT:    s_movk_i32 s2, 0xff
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_load_dword s6, s[6:7], 0x0
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX8-NEXT:    flat_load_ubyte v2, v[0:1]
 ; GFX8-NEXT:    s_load_dword s0, s[4:5], 0x0
-; GFX8-NEXT:    s_load_dword s1, s[6:7], 0x0
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_lshr_b32 s4, s0, 12
-; GFX8-NEXT:    s_bfe_i32 s7, s1, 0x40000
-; GFX8-NEXT:    s_lshr_b32 s5, s1, 12
-; GFX8-NEXT:    s_bfe_i32 s9, s1, 0x40004
-; GFX8-NEXT:    s_bfe_i32 s11, s1, 0x40008
-; GFX8-NEXT:    s_bfe_i32 s6, s0, 0x40000
+; GFX8-NEXT:    s_bfe_i32 s7, s6, 0x40000
+; GFX8-NEXT:    s_lshr_b32 s4, s6, 12
+; GFX8-NEXT:    s_bfe_i32 s9, s6, 0x40004
+; GFX8-NEXT:    s_bfe_i32 s11, s6, 0x40008
+; GFX8-NEXT:    s_lshr_b32 s1, s0, 12
+; GFX8-NEXT:    s_bfe_i32 s5, s0, 0x40000
 ; GFX8-NEXT:    v_mov_b32_e32 v6, s7
-; GFX8-NEXT:    v_lshlrev_b16_e64 v4, 12, s4
-; GFX8-NEXT:    v_lshlrev_b16_e64 v5, 12, s5
+; GFX8-NEXT:    v_lshlrev_b16_e64 v4, 12, s1
+; GFX8-NEXT:    v_lshlrev_b16_e64 v5, 12, s4
 ; GFX8-NEXT:    s_bfe_i32 s8, s0, 0x40004
 ; GFX8-NEXT:    s_bfe_i32 s10, s0, 0x40008
 ; GFX8-NEXT:    v_mov_b32_e32 v3, s11
@@ -680,28 +678,28 @@ define amdgpu_kernel void @idot8_acc8(<8 x i4> addrspace(1)* %src1,
 ; GFX8-NEXT:    v_ashrrev_i16_e32 v4, 12, v4
 ; GFX8-NEXT:    v_ashrrev_i16_e32 v5, 12, v5
 ; GFX8-NEXT:    v_mul_i32_i24_e32 v3, s10, v3
-; GFX8-NEXT:    s_bfe_i32 s13, s1, 0x40010
+; GFX8-NEXT:    s_bfe_i32 s13, s6, 0x40010
 ; GFX8-NEXT:    v_and_b32_e32 v4, s2, v4
 ; GFX8-NEXT:    v_and_b32_e32 v5, s2, v5
-; GFX8-NEXT:    s_bfe_i32 s15, s1, 0x40014
+; GFX8-NEXT:    s_bfe_i32 s15, s6, 0x40014
 ; GFX8-NEXT:    s_bfe_i32 s12, s0, 0x40010
 ; GFX8-NEXT:    v_mov_b32_e32 v8, s13
-; GFX8-NEXT:    s_bfe_i32 s17, s1, 0x40018
+; GFX8-NEXT:    s_bfe_i32 s17, s6, 0x40018
 ; GFX8-NEXT:    s_bfe_i32 s14, s0, 0x40014
 ; GFX8-NEXT:    v_mov_b32_e32 v9, s15
 ; GFX8-NEXT:    s_bfe_i32 s16, s0, 0x40018
-; GFX8-NEXT:    s_ashr_i32 s1, s1, 28
+; GFX8-NEXT:    s_ashr_i32 s6, s6, 28
 ; GFX8-NEXT:    v_mov_b32_e32 v10, s17
 ; GFX8-NEXT:    s_ashr_i32 s0, s0, 28
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
-; GFX8-NEXT:    v_mad_i32_i24 v2, s6, v6, v2
+; GFX8-NEXT:    v_mad_i32_i24 v2, s5, v6, v2
 ; GFX8-NEXT:    v_mad_i32_i24 v2, s8, v7, v2
 ; GFX8-NEXT:    v_add_u32_sdwa v2, vcc, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
 ; GFX8-NEXT:    v_mad_u32_u24 v2, v4, v5, v2
 ; GFX8-NEXT:    v_mad_i32_i24 v2, s12, v8, v2
 ; GFX8-NEXT:    v_mad_i32_i24 v2, s14, v9, v2
 ; GFX8-NEXT:    v_mad_i32_i24 v2, s16, v10, v2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s1
+; GFX8-NEXT:    v_mov_b32_e32 v3, s6
 ; GFX8-NEXT:    v_mad_i32_i24 v2, s0, v3, v2
 ; GFX8-NEXT:    flat_store_byte v[0:1], v2
 ; GFX8-NEXT:    s_endpgm
@@ -712,21 +710,21 @@ define amdgpu_kernel void @idot8_acc8(<8 x i4> addrspace(1)* %src1,
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX9-NEXT:    s_movk_i32 s2, 0xff
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_load_dword s6, s[6:7], 0x0
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-NEXT:    global_load_ubyte v2, v[0:1], off
 ; GFX9-NEXT:    s_load_dword s0, s[4:5], 0x0
-; GFX9-NEXT:    s_load_dword s1, s[6:7], 0x0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_lshr_b32 s4, s0, 12
-; GFX9-NEXT:    s_bfe_i32 s7, s1, 0x40000
-; GFX9-NEXT:    s_lshr_b32 s5, s1, 12
-; GFX9-NEXT:    s_bfe_i32 s9, s1, 0x40004
-; GFX9-NEXT:    s_bfe_i32 s11, s1, 0x40008
-; GFX9-NEXT:    s_bfe_i32 s6, s0, 0x40000
+; GFX9-NEXT:    s_bfe_i32 s7, s6, 0x40000
+; GFX9-NEXT:    s_lshr_b32 s4, s6, 12
+; GFX9-NEXT:    s_bfe_i32 s9, s6, 0x40004
+; GFX9-NEXT:    s_bfe_i32 s11, s6, 0x40008
+; GFX9-NEXT:    s_lshr_b32 s1, s0, 12
+; GFX9-NEXT:    s_bfe_i32 s5, s0, 0x40000
 ; GFX9-NEXT:    v_mov_b32_e32 v6, s7
-; GFX9-NEXT:    v_lshlrev_b16_e64 v4, 12, s4
-; GFX9-NEXT:    v_lshlrev_b16_e64 v5, 12, s5
+; GFX9-NEXT:    v_lshlrev_b16_e64 v4, 12, s1
+; GFX9-NEXT:    v_lshlrev_b16_e64 v5, 12, s4
 ; GFX9-NEXT:    s_bfe_i32 s8, s0, 0x40004
 ; GFX9-NEXT:    s_bfe_i32 s10, s0, 0x40008
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s11
@@ -734,28 +732,28 @@ define amdgpu_kernel void @idot8_acc8(<8 x i4> addrspace(1)* %src1,
 ; GFX9-NEXT:    v_ashrrev_i16_e32 v4, 12, v4
 ; GFX9-NEXT:    v_ashrrev_i16_e32 v5, 12, v5
 ; GFX9-NEXT:    v_mul_i32_i24_e32 v3, s10, v3
-; GFX9-NEXT:    s_bfe_i32 s13, s1, 0x40010
+; GFX9-NEXT:    s_bfe_i32 s13, s6, 0x40010
 ; GFX9-NEXT:    v_and_b32_e32 v4, s2, v4
 ; GFX9-NEXT:    v_and_b32_e32 v5, s2, v5
-; GFX9-NEXT:    s_bfe_i32 s15, s1, 0x40014
+; GFX9-NEXT:    s_bfe_i32 s15, s6, 0x40014
 ; GFX9-NEXT:    s_bfe_i32 s12, s0, 0x40010
 ; GFX9-NEXT:    v_mov_b32_e32 v8, s13
-; GFX9-NEXT:    s_bfe_i32 s17, s1, 0x40018
+; GFX9-NEXT:    s_bfe_i32 s17, s6, 0x40018
 ; GFX9-NEXT:    s_bfe_i32 s14, s0, 0x40014
 ; GFX9-NEXT:    v_mov_b32_e32 v9, s15
 ; GFX9-NEXT:    s_bfe_i32 s16, s0, 0x40018
-; GFX9-NEXT:    s_ashr_i32 s1, s1, 28
+; GFX9-NEXT:    s_ashr_i32 s6, s6, 28
 ; GFX9-NEXT:    v_mov_b32_e32 v10, s17
 ; GFX9-NEXT:    s_ashr_i32 s0, s0, 28
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_mad_i32_i24 v2, s6, v6, v2
+; GFX9-NEXT:    v_mad_i32_i24 v2, s5, v6, v2
 ; GFX9-NEXT:    v_mad_i32_i24 v2, s8, v7, v2
 ; GFX9-NEXT:    v_add_u32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
 ; GFX9-NEXT:    v_mad_u32_u24 v2, v4, v5, v2
 ; GFX9-NEXT:    v_mad_i32_i24 v2, s12, v8, v2
 ; GFX9-NEXT:    v_mad_i32_i24 v2, s14, v9, v2
 ; GFX9-NEXT:    v_mad_i32_i24 v2, s16, v10, v2
-; GFX9-NEXT:    v_mov_b32_e32 v3, s1
+; GFX9-NEXT:    v_mov_b32_e32 v3, s6
 ; GFX9-NEXT:    v_mad_i32_i24 v2, s0, v3, v2
 ; GFX9-NEXT:    global_store_byte v[0:1], v2, off
 ; GFX9-NEXT:    s_endpgm
@@ -766,21 +764,21 @@ define amdgpu_kernel void @idot8_acc8(<8 x i4> addrspace(1)* %src1,
 ; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX9-DL-NEXT:    s_movk_i32 s2, 0xff
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-DL-NEXT:    s_load_dword s6, s[6:7], 0x0
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-DL-NEXT:    global_load_ubyte v2, v[0:1], off
 ; GFX9-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
-; GFX9-DL-NEXT:    s_load_dword s1, s[6:7], 0x0
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    s_lshr_b32 s4, s0, 12
-; GFX9-DL-NEXT:    s_bfe_i32 s7, s1, 0x40000
-; GFX9-DL-NEXT:    s_lshr_b32 s5, s1, 12
-; GFX9-DL-NEXT:    s_bfe_i32 s9, s1, 0x40004
-; GFX9-DL-NEXT:    s_bfe_i32 s11, s1, 0x40008
-; GFX9-DL-NEXT:    s_bfe_i32 s6, s0, 0x40000
+; GFX9-DL-NEXT:    s_bfe_i32 s7, s6, 0x40000
+; GFX9-DL-NEXT:    s_lshr_b32 s4, s6, 12
+; GFX9-DL-NEXT:    s_bfe_i32 s9, s6, 0x40004
+; GFX9-DL-NEXT:    s_bfe_i32 s11, s6, 0x40008
+; GFX9-DL-NEXT:    s_lshr_b32 s1, s0, 12
+; GFX9-DL-NEXT:    s_bfe_i32 s5, s0, 0x40000
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v6, s7
-; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v4, 12, s4
-; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v5, 12, s5
+; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v4, 12, s1
+; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v5, 12, s4
 ; GFX9-DL-NEXT:    s_bfe_i32 s8, s0, 0x40004
 ; GFX9-DL-NEXT:    s_bfe_i32 s10, s0, 0x40008
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s11
@@ -788,28 +786,28 @@ define amdgpu_kernel void @idot8_acc8(<8 x i4> addrspace(1)* %src1,
 ; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v4, 12, v4
 ; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v5, 12, v5
 ; GFX9-DL-NEXT:    v_mul_i32_i24_e32 v3, s10, v3
-; GFX9-DL-NEXT:    s_bfe_i32 s13, s1, 0x40010
+; GFX9-DL-NEXT:    s_bfe_i32 s13, s6, 0x40010
 ; GFX9-DL-NEXT:    v_and_b32_e32 v4, s2, v4
 ; GFX9-DL-NEXT:    v_and_b32_e32 v5, s2, v5
-; GFX9-DL-NEXT:    s_bfe_i32 s15, s1, 0x40014
+; GFX9-DL-NEXT:    s_bfe_i32 s15, s6, 0x40014
 ; GFX9-DL-NEXT:    s_bfe_i32 s12, s0, 0x40010
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v8, s13
-; GFX9-DL-NEXT:    s_bfe_i32 s17, s1, 0x40018
+; GFX9-DL-NEXT:    s_bfe_i32 s17, s6, 0x40018
 ; GFX9-DL-NEXT:    s_bfe_i32 s14, s0, 0x40014
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v9, s15
 ; GFX9-DL-NEXT:    s_bfe_i32 s16, s0, 0x40018
-; GFX9-DL-NEXT:    s_ashr_i32 s1, s1, 28
+; GFX9-DL-NEXT:    s_ashr_i32 s6, s6, 28
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v10, s17
 ; GFX9-DL-NEXT:    s_ashr_i32 s0, s0, 28
 ; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s6, v6, v2
+; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s5, v6, v2
 ; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s8, v7, v2
 ; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
 ; GFX9-DL-NEXT:    v_mad_u32_u24 v2, v4, v5, v2
 ; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s12, v8, v2
 ; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s14, v9, v2
 ; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s16, v10, v2
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s1
+; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s6
 ; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s0, v3, v2
 ; GFX9-DL-NEXT:    global_store_byte v[0:1], v2, off
 ; GFX9-DL-NEXT:    s_endpgm
@@ -818,50 +816,47 @@ define amdgpu_kernel void @idot8_acc8(<8 x i4> addrspace(1)* %src1,
 ; GFX10-DL:       ; %bb.0: ; %entry
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX10-DL-NEXT:    v_mov_b32_e32 v2, 0xffff
 ; GFX10-DL-NEXT:    s_movk_i32 s2, 0xff
 ; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_load_dword s4, s[4:5], 0x0
-; GFX10-DL-NEXT:    s_load_dword s5, s[6:7], 0x0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
-; GFX10-DL-NEXT:    global_load_ubyte v3, v[0:1], off
+; GFX10-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
+; GFX10-DL-NEXT:    s_load_dword s1, s[6:7], 0x0
+; GFX10-DL-NEXT:    global_load_ubyte v2, v[0:1], off
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_lshr_b32 s0, s4, 12
-; GFX10-DL-NEXT:    s_lshr_b32 s1, s5, 12
-; GFX10-DL-NEXT:    s_bfe_i32 s6, s4, 0x40000
-; GFX10-DL-NEXT:    s_bfe_i32 s7, s5, 0x40000
-; GFX10-DL-NEXT:    s_bfe_i32 s8, s4, 0x40004
-; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v4, 12, s0
-; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v5, 12, s1
-; GFX10-DL-NEXT:    s_bfe_i32 s0, s5, 0x40004
-; GFX10-DL-NEXT:    s_bfe_i32 s1, s4, 0x40008
-; GFX10-DL-NEXT:    s_bfe_i32 s9, s5, 0x40008
-; GFX10-DL-NEXT:    v_and_b32_e32 v4, v4, v2
-; GFX10-DL-NEXT:    v_and_b32_e32 v2, v5, v2
-; GFX10-DL-NEXT:    s_bfe_i32 s10, s4, 0x40010
-; GFX10-DL-NEXT:    s_bfe_i32 s11, s5, 0x40010
-; GFX10-DL-NEXT:    v_mul_i32_i24_e64 v5, s1, s9
+; GFX10-DL-NEXT:    s_lshr_b32 s4, s0, 12
+; GFX10-DL-NEXT:    s_lshr_b32 s5, s1, 12
+; GFX10-DL-NEXT:    s_bfe_i32 s6, s0, 0x40000
+; GFX10-DL-NEXT:    s_bfe_i32 s7, s1, 0x40000
+; GFX10-DL-NEXT:    s_bfe_i32 s8, s0, 0x40004
+; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v3, 12, s4
+; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v4, 12, s5
+; GFX10-DL-NEXT:    s_bfe_i32 s9, s1, 0x40004
+; GFX10-DL-NEXT:    s_bfe_i32 s10, s0, 0x40008
+; GFX10-DL-NEXT:    s_bfe_i32 s11, s1, 0x40008
+; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v3, 12, v3
 ; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v4, 12, v4
-; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v2, 12, v2
-; GFX10-DL-NEXT:    s_bfe_i32 s1, s4, 0x40014
-; GFX10-DL-NEXT:    s_bfe_i32 s9, s5, 0x40014
-; GFX10-DL-NEXT:    s_bfe_i32 s12, s4, 0x40018
-; GFX10-DL-NEXT:    v_and_b32_sdwa v4, v4, s2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX10-DL-NEXT:    v_and_b32_sdwa v2, v2, s2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX10-DL-NEXT:    s_bfe_i32 s2, s5, 0x40018
-; GFX10-DL-NEXT:    s_ashr_i32 s4, s4, 28
-; GFX10-DL-NEXT:    s_ashr_i32 s5, s5, 28
+; GFX10-DL-NEXT:    s_bfe_i32 s4, s1, 0x40010
+; GFX10-DL-NEXT:    s_bfe_i32 s5, s0, 0x40014
+; GFX10-DL-NEXT:    v_mul_i32_i24_e64 v5, s10, s11
+; GFX10-DL-NEXT:    v_and_b32_e32 v3, s2, v3
+; GFX10-DL-NEXT:    v_and_b32_e32 v4, s2, v4
+; GFX10-DL-NEXT:    s_bfe_i32 s2, s0, 0x40010
 ; GFX10-DL-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-DL-NEXT:    v_mad_i32_i24 v3, s6, s7, v3
-; GFX10-DL-NEXT:    v_mad_i32_i24 v3, s8, s0, v3
-; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v3, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX10-DL-NEXT:    v_mad_u32_u24 v2, v4, v2, v3
-; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s10, s11, v2
-; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s1, s9, v2
-; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s12, s2, v2
-; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s4, s5, v2
+; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s6, s7, v2
+; GFX10-DL-NEXT:    s_bfe_i32 s6, s1, 0x40014
+; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s8, s9, v2
+; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX10-DL-NEXT:    v_mad_u32_u24 v2, v3, v4, v2
+; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s2, s4, v2
+; GFX10-DL-NEXT:    s_bfe_i32 s2, s0, 0x40018
+; GFX10-DL-NEXT:    s_bfe_i32 s4, s1, 0x40018
+; GFX10-DL-NEXT:    s_ashr_i32 s0, s0, 28
+; GFX10-DL-NEXT:    s_ashr_i32 s1, s1, 28
+; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s5, s6, v2
+; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s2, s4, v2
+; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s0, s1, v2
 ; GFX10-DL-NEXT:    global_store_byte v[0:1], v2, off
 ; GFX10-DL-NEXT:    s_endpgm
                                        <8 x i4> addrspace(1)* %src2,
@@ -944,39 +939,39 @@ define amdgpu_kernel void @idot8_multiuses_mul1(<8 x i4> addrspace(1)* %src1,
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    s_load_dword s0, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s1, s[10:11], 0x0
-; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s21, s[4:5], 0x0
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_bfe_i32 s8, s0, 0x40000
-; GFX7-NEXT:    s_bfe_i32 s9, s1, 0x40000
-; GFX7-NEXT:    v_mov_b32_e32 v0, s9
-; GFX7-NEXT:    v_mov_b32_e32 v1, s2
-; GFX7-NEXT:    v_mad_i32_i24 v1, s8, v0, v1
-; GFX7-NEXT:    s_bfe_i32 s11, s1, 0x40004
-; GFX7-NEXT:    s_bfe_i32 s10, s0, 0x40004
-; GFX7-NEXT:    s_bfe_i32 s13, s1, 0x40008
-; GFX7-NEXT:    v_mad_i32_i24 v0, s8, v0, v1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s11
-; GFX7-NEXT:    v_mad_i32_i24 v0, s10, v2, v0
-; GFX7-NEXT:    s_bfe_i32 s12, s0, 0x40008
-; GFX7-NEXT:    v_mov_b32_e32 v2, s13
-; GFX7-NEXT:    s_bfe_i32 s15, s1, 0x4000c
-; GFX7-NEXT:    v_mad_i32_i24 v0, s12, v2, v0
-; GFX7-NEXT:    s_bfe_i32 s14, s0, 0x4000c
-; GFX7-NEXT:    v_mov_b32_e32 v2, s15
-; GFX7-NEXT:    s_bfe_i32 s17, s1, 0x40010
-; GFX7-NEXT:    v_mad_i32_i24 v0, s14, v2, v0
-; GFX7-NEXT:    s_bfe_i32 s16, s0, 0x40010
-; GFX7-NEXT:    v_mov_b32_e32 v2, s17
-; GFX7-NEXT:    s_bfe_i32 s19, s1, 0x40014
-; GFX7-NEXT:    s_bfe_i32 s21, s1, 0x40018
-; GFX7-NEXT:    v_mad_i32_i24 v0, s16, v2, v0
-; GFX7-NEXT:    s_bfe_i32 s18, s0, 0x40014
-; GFX7-NEXT:    v_mov_b32_e32 v2, s19
-; GFX7-NEXT:    s_bfe_i32 s20, s0, 0x40018
-; GFX7-NEXT:    v_mad_i32_i24 v0, s18, v2, v0
-; GFX7-NEXT:    v_mov_b32_e32 v2, s21
+; GFX7-NEXT:    s_bfe_i32 s2, s0, 0x40000
+; GFX7-NEXT:    s_bfe_i32 s8, s1, 0x40000
+; GFX7-NEXT:    v_mov_b32_e32 v0, s8
+; GFX7-NEXT:    v_mov_b32_e32 v1, s21
+; GFX7-NEXT:    v_mad_i32_i24 v1, s2, v0, v1
+; GFX7-NEXT:    s_bfe_i32 s10, s1, 0x40004
+; GFX7-NEXT:    s_bfe_i32 s9, s0, 0x40004
+; GFX7-NEXT:    s_bfe_i32 s12, s1, 0x40008
+; GFX7-NEXT:    v_mad_i32_i24 v0, s2, v0, v1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s10
+; GFX7-NEXT:    v_mad_i32_i24 v0, s9, v2, v0
+; GFX7-NEXT:    s_bfe_i32 s11, s0, 0x40008
+; GFX7-NEXT:    v_mov_b32_e32 v2, s12
+; GFX7-NEXT:    s_bfe_i32 s14, s1, 0x4000c
+; GFX7-NEXT:    v_mad_i32_i24 v0, s11, v2, v0
+; GFX7-NEXT:    s_bfe_i32 s13, s0, 0x4000c
+; GFX7-NEXT:    v_mov_b32_e32 v2, s14
+; GFX7-NEXT:    s_bfe_i32 s16, s1, 0x40010
+; GFX7-NEXT:    v_mad_i32_i24 v0, s13, v2, v0
+; GFX7-NEXT:    s_bfe_i32 s15, s0, 0x40010
+; GFX7-NEXT:    v_mov_b32_e32 v2, s16
+; GFX7-NEXT:    s_bfe_i32 s18, s1, 0x40014
+; GFX7-NEXT:    s_bfe_i32 s20, s1, 0x40018
+; GFX7-NEXT:    v_mad_i32_i24 v0, s15, v2, v0
+; GFX7-NEXT:    s_bfe_i32 s17, s0, 0x40014
+; GFX7-NEXT:    v_mov_b32_e32 v2, s18
+; GFX7-NEXT:    s_bfe_i32 s19, s0, 0x40018
+; GFX7-NEXT:    v_mad_i32_i24 v0, s17, v2, v0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s20
 ; GFX7-NEXT:    s_ashr_i32 s1, s1, 28
-; GFX7-NEXT:    v_mad_i32_i24 v0, s20, v2, v0
+; GFX7-NEXT:    v_mad_i32_i24 v0, s19, v2, v0
 ; GFX7-NEXT:    s_ashr_i32 s0, s0, 28
 ; GFX7-NEXT:    v_mov_b32_e32 v2, s1
 ; GFX7-NEXT:    v_mad_i32_i24 v0, s0, v2, v0
@@ -991,45 +986,45 @@ define amdgpu_kernel void @idot8_multiuses_mul1(<8 x i4> addrspace(1)* %src1,
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
 ; GFX8-NEXT:    s_load_dword s4, s[6:7], 0x0
-; GFX8-NEXT:    s_load_dword s5, s[0:1], 0x0
-; GFX8-NEXT:    v_mov_b32_e32 v0, s0
-; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    s_load_dword s19, s[0:1], 0x0
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_bfe_i32 s0, s2, 0x40000
-; GFX8-NEXT:    s_bfe_i32 s1, s4, 0x40000
-; GFX8-NEXT:    v_mov_b32_e32 v2, s1
-; GFX8-NEXT:    v_mov_b32_e32 v3, s5
-; GFX8-NEXT:    v_mad_i32_i24 v3, s0, v2, v3
-; GFX8-NEXT:    s_bfe_i32 s7, s4, 0x40004
-; GFX8-NEXT:    s_bfe_i32 s6, s2, 0x40004
-; GFX8-NEXT:    s_bfe_i32 s9, s4, 0x40008
-; GFX8-NEXT:    v_mad_i32_i24 v2, s0, v2, v3
-; GFX8-NEXT:    v_mov_b32_e32 v4, s7
-; GFX8-NEXT:    v_mad_i32_i24 v2, s6, v4, v2
-; GFX8-NEXT:    s_bfe_i32 s8, s2, 0x40008
-; GFX8-NEXT:    v_mov_b32_e32 v4, s9
-; GFX8-NEXT:    s_bfe_i32 s11, s4, 0x4000c
-; GFX8-NEXT:    v_mad_i32_i24 v2, s8, v4, v2
-; GFX8-NEXT:    s_bfe_i32 s10, s2, 0x4000c
-; GFX8-NEXT:    v_mov_b32_e32 v4, s11
-; GFX8-NEXT:    s_bfe_i32 s13, s4, 0x40010
-; GFX8-NEXT:    v_mad_i32_i24 v2, s10, v4, v2
-; GFX8-NEXT:    s_bfe_i32 s12, s2, 0x40010
-; GFX8-NEXT:    v_mov_b32_e32 v4, s13
-; GFX8-NEXT:    s_bfe_i32 s15, s4, 0x40014
-; GFX8-NEXT:    s_bfe_i32 s17, s4, 0x40018
-; GFX8-NEXT:    v_mad_i32_i24 v2, s12, v4, v2
-; GFX8-NEXT:    s_bfe_i32 s14, s2, 0x40014
-; GFX8-NEXT:    v_mov_b32_e32 v4, s15
-; GFX8-NEXT:    s_bfe_i32 s16, s2, 0x40018
-; GFX8-NEXT:    v_mad_i32_i24 v2, s14, v4, v2
-; GFX8-NEXT:    v_mov_b32_e32 v4, s17
+; GFX8-NEXT:    s_bfe_i32 s5, s2, 0x40000
+; GFX8-NEXT:    s_bfe_i32 s6, s4, 0x40000
+; GFX8-NEXT:    v_mov_b32_e32 v0, s6
+; GFX8-NEXT:    v_mov_b32_e32 v1, s19
+; GFX8-NEXT:    v_mad_i32_i24 v1, s5, v0, v1
+; GFX8-NEXT:    s_bfe_i32 s8, s4, 0x40004
+; GFX8-NEXT:    s_bfe_i32 s7, s2, 0x40004
+; GFX8-NEXT:    s_bfe_i32 s10, s4, 0x40008
+; GFX8-NEXT:    v_mad_i32_i24 v0, s5, v0, v1
+; GFX8-NEXT:    v_mov_b32_e32 v2, s8
+; GFX8-NEXT:    v_mad_i32_i24 v0, s7, v2, v0
+; GFX8-NEXT:    s_bfe_i32 s9, s2, 0x40008
+; GFX8-NEXT:    v_mov_b32_e32 v2, s10
+; GFX8-NEXT:    s_bfe_i32 s12, s4, 0x4000c
+; GFX8-NEXT:    v_mad_i32_i24 v0, s9, v2, v0
+; GFX8-NEXT:    s_bfe_i32 s11, s2, 0x4000c
+; GFX8-NEXT:    v_mov_b32_e32 v2, s12
+; GFX8-NEXT:    s_bfe_i32 s14, s4, 0x40010
+; GFX8-NEXT:    v_mad_i32_i24 v0, s11, v2, v0
+; GFX8-NEXT:    s_bfe_i32 s13, s2, 0x40010
+; GFX8-NEXT:    v_mov_b32_e32 v2, s14
+; GFX8-NEXT:    s_bfe_i32 s16, s4, 0x40014
+; GFX8-NEXT:    s_bfe_i32 s18, s4, 0x40018
+; GFX8-NEXT:    v_mad_i32_i24 v0, s13, v2, v0
+; GFX8-NEXT:    s_bfe_i32 s15, s2, 0x40014
+; GFX8-NEXT:    v_mov_b32_e32 v2, s16
+; GFX8-NEXT:    s_bfe_i32 s17, s2, 0x40018
+; GFX8-NEXT:    v_mad_i32_i24 v0, s15, v2, v0
+; GFX8-NEXT:    v_mov_b32_e32 v2, s18
 ; GFX8-NEXT:    s_ashr_i32 s4, s4, 28
-; GFX8-NEXT:    v_mad_i32_i24 v2, s16, v4, v2
+; GFX8-NEXT:    v_mad_i32_i24 v0, s17, v2, v0
 ; GFX8-NEXT:    s_ashr_i32 s2, s2, 28
-; GFX8-NEXT:    v_mov_b32_e32 v4, s4
-; GFX8-NEXT:    v_mad_i32_i24 v2, s2, v4, v2
-; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v2, v3
+; GFX8-NEXT:    v_mov_b32_e32 v2, s4
+; GFX8-NEXT:    v_mad_i32_i24 v0, s2, v2, v0
+; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v0, v1
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX8-NEXT:    flat_store_dword v[0:1], v2
 ; GFX8-NEXT:    s_endpgm
 ;
@@ -1040,45 +1035,45 @@ define amdgpu_kernel void @idot8_multiuses_mul1(<8 x i4> addrspace(1)* %src1,
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x0
 ; GFX9-NEXT:    s_load_dword s4, s[6:7], 0x0
-; GFX9-NEXT:    s_load_dword s5, s[0:1], 0x0
-; GFX9-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    s_load_dword s19, s[0:1], 0x0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_bfe_i32 s0, s2, 0x40000
-; GFX9-NEXT:    s_bfe_i32 s1, s4, 0x40000
-; GFX9-NEXT:    v_mov_b32_e32 v2, s1
-; GFX9-NEXT:    v_mov_b32_e32 v3, s5
-; GFX9-NEXT:    v_mad_i32_i24 v3, s0, v2, v3
-; GFX9-NEXT:    s_bfe_i32 s7, s4, 0x40004
-; GFX9-NEXT:    s_bfe_i32 s6, s2, 0x40004
-; GFX9-NEXT:    s_bfe_i32 s9, s4, 0x40008
-; GFX9-NEXT:    v_mad_i32_i24 v2, s0, v2, v3
-; GFX9-NEXT:    v_mov_b32_e32 v4, s7
-; GFX9-NEXT:    v_mad_i32_i24 v2, s6, v4, v2
-; GFX9-NEXT:    s_bfe_i32 s8, s2, 0x40008
-; GFX9-NEXT:    v_mov_b32_e32 v4, s9
-; GFX9-NEXT:    s_bfe_i32 s11, s4, 0x4000c
-; GFX9-NEXT:    v_mad_i32_i24 v2, s8, v4, v2
-; GFX9-NEXT:    s_bfe_i32 s10, s2, 0x4000c
-; GFX9-NEXT:    v_mov_b32_e32 v4, s11
-; GFX9-NEXT:    s_bfe_i32 s13, s4, 0x40010
-; GFX9-NEXT:    v_mad_i32_i24 v2, s10, v4, v2
-; GFX9-NEXT:    s_bfe_i32 s12, s2, 0x40010
-; GFX9-NEXT:    v_mov_b32_e32 v4, s13
-; GFX9-NEXT:    s_bfe_i32 s15, s4, 0x40014
-; GFX9-NEXT:    s_bfe_i32 s17, s4, 0x40018
-; GFX9-NEXT:    v_mad_i32_i24 v2, s12, v4, v2
-; GFX9-NEXT:    s_bfe_i32 s14, s2, 0x40014
-; GFX9-NEXT:    v_mov_b32_e32 v4, s15
-; GFX9-NEXT:    s_bfe_i32 s16, s2, 0x40018
-; GFX9-NEXT:    v_mad_i32_i24 v2, s14, v4, v2
-; GFX9-NEXT:    v_mov_b32_e32 v4, s17
+; GFX9-NEXT:    s_bfe_i32 s5, s2, 0x40000
+; GFX9-NEXT:    s_bfe_i32 s6, s4, 0x40000
+; GFX9-NEXT:    v_mov_b32_e32 v0, s6
+; GFX9-NEXT:    v_mov_b32_e32 v1, s19
+; GFX9-NEXT:    v_mad_i32_i24 v1, s5, v0, v1
+; GFX9-NEXT:    s_bfe_i32 s8, s4, 0x40004
+; GFX9-NEXT:    s_bfe_i32 s7, s2, 0x40004
+; GFX9-NEXT:    s_bfe_i32 s10, s4, 0x40008
+; GFX9-NEXT:    v_mad_i32_i24 v0, s5, v0, v1
+; GFX9-NEXT:    v_mov_b32_e32 v2, s8
+; GFX9-NEXT:    v_mad_i32_i24 v0, s7, v2, v0
+; GFX9-NEXT:    s_bfe_i32 s9, s2, 0x40008
+; GFX9-NEXT:    v_mov_b32_e32 v2, s10
+; GFX9-NEXT:    s_bfe_i32 s12, s4, 0x4000c
+; GFX9-NEXT:    v_mad_i32_i24 v0, s9, v2, v0
+; GFX9-NEXT:    s_bfe_i32 s11, s2, 0x4000c
+; GFX9-NEXT:    v_mov_b32_e32 v2, s12
+; GFX9-NEXT:    s_bfe_i32 s14, s4, 0x40010
+; GFX9-NEXT:    v_mad_i32_i24 v0, s11, v2, v0
+; GFX9-NEXT:    s_bfe_i32 s13, s2, 0x40010
+; GFX9-NEXT:    v_mov_b32_e32 v2, s14
+; GFX9-NEXT:    s_bfe_i32 s16, s4, 0x40014
+; GFX9-NEXT:    s_bfe_i32 s18, s4, 0x40018
+; GFX9-NEXT:    v_mad_i32_i24 v0, s13, v2, v0
+; GFX9-NEXT:    s_bfe_i32 s15, s2, 0x40014
+; GFX9-NEXT:    v_mov_b32_e32 v2, s16
+; GFX9-NEXT:    s_bfe_i32 s17, s2, 0x40018
+; GFX9-NEXT:    v_mad_i32_i24 v0, s15, v2, v0
+; GFX9-NEXT:    v_mov_b32_e32 v2, s18
 ; GFX9-NEXT:    s_ashr_i32 s4, s4, 28
-; GFX9-NEXT:    v_mad_i32_i24 v2, s16, v4, v2
+; GFX9-NEXT:    v_mad_i32_i24 v0, s17, v2, v0
 ; GFX9-NEXT:    s_ashr_i32 s2, s2, 28
-; GFX9-NEXT:    v_mov_b32_e32 v4, s4
-; GFX9-NEXT:    v_mad_i32_i24 v2, s2, v4, v2
-; GFX9-NEXT:    v_add_u32_e32 v2, v3, v2
+; GFX9-NEXT:    v_mov_b32_e32 v2, s4
+; GFX9-NEXT:    v_mad_i32_i24 v0, s2, v2, v0
+; GFX9-NEXT:    v_add_u32_e32 v2, v1, v0
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-NEXT:    s_endpgm
 ;
@@ -1089,45 +1084,45 @@ define amdgpu_kernel void @idot8_multiuses_mul1(<8 x i4> addrspace(1)* %src1,
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
 ; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
-; GFX9-DL-NEXT:    s_load_dword s5, s[0:1], 0x0
-; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-DL-NEXT:    s_load_dword s19, s[0:1], 0x0
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    s_bfe_i32 s0, s2, 0x40000
-; GFX9-DL-NEXT:    s_bfe_i32 s1, s4, 0x40000
-; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s1
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s5
-; GFX9-DL-NEXT:    v_mad_i32_i24 v3, s0, v2, v3
-; GFX9-DL-NEXT:    s_bfe_i32 s7, s4, 0x40004
-; GFX9-DL-NEXT:    s_bfe_i32 s6, s2, 0x40004
-; GFX9-DL-NEXT:    s_bfe_i32 s9, s4, 0x40008
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s0, v2, v3
-; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s7
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s6, v4, v2
-; GFX9-DL-NEXT:    s_bfe_i32 s8, s2, 0x40008
-; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s9
-; GFX9-DL-NEXT:    s_bfe_i32 s11, s4, 0x4000c
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s8, v4, v2
-; GFX9-DL-NEXT:    s_bfe_i32 s10, s2, 0x4000c
-; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s11
-; GFX9-DL-NEXT:    s_bfe_i32 s13, s4, 0x40010
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s10, v4, v2
-; GFX9-DL-NEXT:    s_bfe_i32 s12, s2, 0x40010
-; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s13
-; GFX9-DL-NEXT:    s_bfe_i32 s15, s4, 0x40014
-; GFX9-DL-NEXT:    s_bfe_i32 s17, s4, 0x40018
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s12, v4, v2
-; GFX9-DL-NEXT:    s_bfe_i32 s14, s2, 0x40014
-; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s15
-; GFX9-DL-NEXT:    s_bfe_i32 s16, s2, 0x40018
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s14, v4, v2
-; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s17
+; GFX9-DL-NEXT:    s_bfe_i32 s5, s2, 0x40000
+; GFX9-DL-NEXT:    s_bfe_i32 s6, s4, 0x40000
+; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s6
+; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s19
+; GFX9-DL-NEXT:    v_mad_i32_i24 v1, s5, v0, v1
+; GFX9-DL-NEXT:    s_bfe_i32 s8, s4, 0x40004
+; GFX9-DL-NEXT:    s_bfe_i32 s7, s2, 0x40004
+; GFX9-DL-NEXT:    s_bfe_i32 s10, s4, 0x40008
+; GFX9-DL-NEXT:    v_mad_i32_i24 v0, s5, v0, v1
+; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s8
+; GFX9-DL-NEXT:    v_mad_i32_i24 v0, s7, v2, v0
+; GFX9-DL-NEXT:    s_bfe_i32 s9, s2, 0x40008
+; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s10
+; GFX9-DL-NEXT:    s_bfe_i32 s12, s4, 0x4000c
+; GFX9-DL-NEXT:    v_mad_i32_i24 v0, s9, v2, v0
+; GFX9-DL-NEXT:    s_bfe_i32 s11, s2, 0x4000c
+; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s12
+; GFX9-DL-NEXT:    s_bfe_i32 s14, s4, 0x40010
+; GFX9-DL-NEXT:    v_mad_i32_i24 v0, s11, v2, v0
+; GFX9-DL-NEXT:    s_bfe_i32 s13, s2, 0x40010
+; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s14
+; GFX9-DL-NEXT:    s_bfe_i32 s16, s4, 0x40014
+; GFX9-DL-NEXT:    s_bfe_i32 s18, s4, 0x40018
+; GFX9-DL-NEXT:    v_mad_i32_i24 v0, s13, v2, v0
+; GFX9-DL-NEXT:    s_bfe_i32 s15, s2, 0x40014
+; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s16
+; GFX9-DL-NEXT:    s_bfe_i32 s17, s2, 0x40018
+; GFX9-DL-NEXT:    v_mad_i32_i24 v0, s15, v2, v0
+; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s18
 ; GFX9-DL-NEXT:    s_ashr_i32 s4, s4, 28
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s16, v4, v2
+; GFX9-DL-NEXT:    v_mad_i32_i24 v0, s17, v2, v0
 ; GFX9-DL-NEXT:    s_ashr_i32 s2, s2, 28
-; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s4
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s2, v4, v2
-; GFX9-DL-NEXT:    v_add_u32_e32 v2, v3, v2
+; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s4
+; GFX9-DL-NEXT:    v_mad_i32_i24 v0, s2, v2, v0
+; GFX9-DL-NEXT:    v_add_u32_e32 v2, v1, v0
+; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-DL-NEXT:    s_endpgm
 ;
@@ -1140,36 +1135,36 @@ define amdgpu_kernel void @idot8_multiuses_mul1(<8 x i4> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
 ; GFX10-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
 ; GFX10-DL-NEXT:    s_load_dword s5, s[0:1], 0x0
-; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_bfe_i32 s0, s2, 0x40000
-; GFX10-DL-NEXT:    s_bfe_i32 s1, s4, 0x40000
-; GFX10-DL-NEXT:    v_mov_b32_e32 v2, s5
+; GFX10-DL-NEXT:    s_bfe_i32 s6, s2, 0x40000
+; GFX10-DL-NEXT:    s_bfe_i32 s7, s4, 0x40000
+; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s5
 ; GFX10-DL-NEXT:    s_bfe_i32 s5, s2, 0x40004
-; GFX10-DL-NEXT:    s_bfe_i32 s6, s4, 0x40004
-; GFX10-DL-NEXT:    s_bfe_i32 s7, s2, 0x40008
-; GFX10-DL-NEXT:    s_bfe_i32 s8, s4, 0x40008
-; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s0, s1, v2
-; GFX10-DL-NEXT:    s_bfe_i32 s9, s2, 0x4000c
-; GFX10-DL-NEXT:    s_bfe_i32 s10, s4, 0x4000c
-; GFX10-DL-NEXT:    s_bfe_i32 s11, s2, 0x40010
-; GFX10-DL-NEXT:    s_bfe_i32 s12, s4, 0x40010
-; GFX10-DL-NEXT:    v_mad_i32_i24 v3, s0, s1, v2
-; GFX10-DL-NEXT:    s_bfe_i32 s0, s2, 0x40014
-; GFX10-DL-NEXT:    s_bfe_i32 s1, s4, 0x40014
-; GFX10-DL-NEXT:    s_bfe_i32 s13, s2, 0x40018
-; GFX10-DL-NEXT:    s_bfe_i32 s14, s4, 0x40018
-; GFX10-DL-NEXT:    v_mad_i32_i24 v3, s5, s6, v3
-; GFX10-DL-NEXT:    v_mad_i32_i24 v3, s7, s8, v3
-; GFX10-DL-NEXT:    v_mad_i32_i24 v3, s9, s10, v3
-; GFX10-DL-NEXT:    v_mad_i32_i24 v3, s11, s12, v3
-; GFX10-DL-NEXT:    v_mad_i32_i24 v3, s0, s1, v3
-; GFX10-DL-NEXT:    s_ashr_i32 s0, s2, 28
-; GFX10-DL-NEXT:    s_ashr_i32 s1, s4, 28
-; GFX10-DL-NEXT:    v_mad_i32_i24 v3, s13, s14, v3
-; GFX10-DL-NEXT:    v_mad_i32_i24 v3, s0, s1, v3
-; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v2, v3
+; GFX10-DL-NEXT:    s_bfe_i32 s8, s4, 0x40004
+; GFX10-DL-NEXT:    v_mad_i32_i24 v0, s6, s7, v0
+; GFX10-DL-NEXT:    v_mad_i32_i24 v1, s6, s7, v0
+; GFX10-DL-NEXT:    s_bfe_i32 s6, s2, 0x40008
+; GFX10-DL-NEXT:    s_bfe_i32 s7, s4, 0x40008
+; GFX10-DL-NEXT:    v_mad_i32_i24 v1, s5, s8, v1
+; GFX10-DL-NEXT:    s_bfe_i32 s5, s2, 0x4000c
+; GFX10-DL-NEXT:    s_bfe_i32 s8, s4, 0x4000c
+; GFX10-DL-NEXT:    v_mad_i32_i24 v1, s6, s7, v1
+; GFX10-DL-NEXT:    s_bfe_i32 s6, s2, 0x40010
+; GFX10-DL-NEXT:    s_bfe_i32 s7, s4, 0x40010
+; GFX10-DL-NEXT:    v_mad_i32_i24 v1, s5, s8, v1
+; GFX10-DL-NEXT:    s_bfe_i32 s5, s2, 0x40014
+; GFX10-DL-NEXT:    s_bfe_i32 s8, s4, 0x40014
+; GFX10-DL-NEXT:    v_mad_i32_i24 v1, s6, s7, v1
+; GFX10-DL-NEXT:    s_bfe_i32 s6, s2, 0x40018
+; GFX10-DL-NEXT:    s_bfe_i32 s7, s4, 0x40018
+; GFX10-DL-NEXT:    s_ashr_i32 s2, s2, 28
+; GFX10-DL-NEXT:    s_ashr_i32 s4, s4, 28
+; GFX10-DL-NEXT:    v_mad_i32_i24 v1, s5, s8, v1
+; GFX10-DL-NEXT:    v_mad_i32_i24 v1, s6, s7, v1
+; GFX10-DL-NEXT:    v_mad_i32_i24 v1, s2, s4, v1
+; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v0, v1
+; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX10-DL-NEXT:    s_endpgm
                                                 <8 x i4> addrspace(1)* %src2,
@@ -1253,17 +1248,15 @@ define amdgpu_kernel void @idot8_acc32_vecMul(<8 x i4> addrspace(1)* %src1,
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    s_load_dword s1, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[10:11], 0x0
-; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    s_ashr_i64 s[10:11], s[0:1], 60
 ; GFX7-NEXT:    s_lshl_b32 s11, s1, 4
-; GFX7-NEXT:    s_ashr_i64 s[14:15], s[10:11], 60
-; GFX7-NEXT:    s_lshl_b32 s11, s1, 12
 ; GFX7-NEXT:    s_ashr_i64 s[16:17], s[10:11], 60
 ; GFX7-NEXT:    s_lshl_b32 s11, s1, 16
 ; GFX7-NEXT:    s_ashr_i64 s[18:19], s[10:11], 60
 ; GFX7-NEXT:    s_lshl_b32 s11, s1, 20
 ; GFX7-NEXT:    s_lshl_b32 s13, s1, 8
+; GFX7-NEXT:    s_lshl_b32 s15, s1, 12
 ; GFX7-NEXT:    s_ashr_i64 s[20:21], s[10:11], 60
 ; GFX7-NEXT:    s_lshl_b32 s11, s1, 24
 ; GFX7-NEXT:    s_lshl_b32 s1, s1, 28
@@ -1283,10 +1276,14 @@ define amdgpu_kernel void @idot8_acc32_vecMul(<8 x i4> addrspace(1)* %src1,
 ; GFX7-NEXT:    s_lshl_b32 s1, s9, 28
 ; GFX7-NEXT:    s_ashr_i64 s[24:25], s[8:9], 60
 ; GFX7-NEXT:    s_ashr_i64 s[8:9], s[0:1], 60
+; GFX7-NEXT:    s_load_dword s1, s[4:5], 0x0
 ; GFX7-NEXT:    v_mov_b32_e32 v0, s8
-; GFX7-NEXT:    v_mov_b32_e32 v1, s2
-; GFX7-NEXT:    v_mad_i32_i24 v0, s0, v0, v1
 ; GFX7-NEXT:    s_ashr_i64 s[22:23], s[10:11], 60
+; GFX7-NEXT:    s_ashr_i64 s[14:15], s[14:15], 60
+; GFX7-NEXT:    s_ashr_i64 s[12:13], s[12:13], 60
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mad_i32_i24 v0, s0, v0, v1
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s36
 ; GFX7-NEXT:    v_mad_i32_i24 v0, s22, v1, v0
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s34
@@ -1294,12 +1291,11 @@ define amdgpu_kernel void @idot8_acc32_vecMul(<8 x i4> addrspace(1)* %src1,
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s32
 ; GFX7-NEXT:    v_mad_i32_i24 v0, s18, v1, v0
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s30
-; GFX7-NEXT:    v_mad_i32_i24 v0, s16, v1, v0
-; GFX7-NEXT:    s_ashr_i64 s[12:13], s[12:13], 60
+; GFX7-NEXT:    v_mad_i32_i24 v0, s14, v1, v0
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s28
 ; GFX7-NEXT:    v_mad_i32_i24 v0, s12, v1, v0
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s26
-; GFX7-NEXT:    v_mad_i32_i24 v0, s14, v1, v0
+; GFX7-NEXT:    v_mad_i32_i24 v0, s16, v1, v0
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s24
 ; GFX7-NEXT:    v_mad_i32_i24 v0, s10, v1, v0
 ; GFX7-NEXT:    buffer_store_dword v0, off, s[4:7], 0
@@ -1313,56 +1309,56 @@ define amdgpu_kernel void @idot8_acc32_vecMul(<8 x i4> addrspace(1)* %src1,
 ; GFX8-NEXT:    s_load_dword s5, s[4:5], 0x0
 ; GFX8-NEXT:    s_load_dword s7, s[6:7], 0x0
 ; GFX8-NEXT:    s_load_dword s2, s[0:1], 0x0
-; GFX8-NEXT:    v_mov_b32_e32 v0, s0
-; GFX8-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_ashr_i64 s[0:1], s[4:5], 60
-; GFX8-NEXT:    s_lshl_b32 s1, s5, 4
-; GFX8-NEXT:    s_ashr_i64 s[12:13], s[0:1], 60
-; GFX8-NEXT:    s_lshl_b32 s1, s5, 16
-; GFX8-NEXT:    s_ashr_i64 s[14:15], s[0:1], 60
-; GFX8-NEXT:    s_lshl_b32 s1, s5, 20
-; GFX8-NEXT:    s_ashr_i64 s[16:17], s[0:1], 60
-; GFX8-NEXT:    s_lshl_b32 s1, s5, 24
-; GFX8-NEXT:    s_ashr_i64 s[18:19], s[0:1], 60
-; GFX8-NEXT:    s_lshl_b32 s1, s5, 28
-; GFX8-NEXT:    s_lshl_b32 s9, s5, 8
-; GFX8-NEXT:    s_lshl_b32 s11, s5, 12
-; GFX8-NEXT:    s_ashr_i64 s[4:5], s[0:1], 60
-; GFX8-NEXT:    s_lshl_b32 s1, s7, 4
-; GFX8-NEXT:    s_ashr_i64 s[22:23], s[0:1], 60
-; GFX8-NEXT:    s_lshl_b32 s1, s7, 8
-; GFX8-NEXT:    s_ashr_i64 s[24:25], s[0:1], 60
-; GFX8-NEXT:    s_lshl_b32 s1, s7, 12
-; GFX8-NEXT:    s_ashr_i64 s[26:27], s[0:1], 60
-; GFX8-NEXT:    s_lshl_b32 s1, s7, 16
-; GFX8-NEXT:    s_ashr_i64 s[28:29], s[0:1], 60
-; GFX8-NEXT:    s_lshl_b32 s1, s7, 20
-; GFX8-NEXT:    s_ashr_i64 s[30:31], s[0:1], 60
-; GFX8-NEXT:    s_lshl_b32 s1, s7, 24
-; GFX8-NEXT:    s_ashr_i64 s[32:33], s[0:1], 60
-; GFX8-NEXT:    s_lshl_b32 s1, s7, 28
-; GFX8-NEXT:    s_ashr_i64 s[20:21], s[6:7], 60
-; GFX8-NEXT:    s_ashr_i64 s[6:7], s[0:1], 60
-; GFX8-NEXT:    v_mov_b32_e32 v2, s6
-; GFX8-NEXT:    v_mov_b32_e32 v3, s2
-; GFX8-NEXT:    v_mad_i32_i24 v2, s4, v2, v3
-; GFX8-NEXT:    v_mov_b32_e32 v3, s32
-; GFX8-NEXT:    v_mad_i32_i24 v2, s18, v3, v2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s30
-; GFX8-NEXT:    v_mad_i32_i24 v2, s16, v3, v2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s28
-; GFX8-NEXT:    v_mad_i32_i24 v2, s14, v3, v2
+; GFX8-NEXT:    s_ashr_i64 s[8:9], s[4:5], 60
+; GFX8-NEXT:    s_lshl_b32 s9, s5, 4
+; GFX8-NEXT:    s_ashr_i64 s[16:17], s[8:9], 60
+; GFX8-NEXT:    s_lshl_b32 s9, s5, 20
+; GFX8-NEXT:    s_lshl_b32 s11, s5, 8
+; GFX8-NEXT:    s_lshl_b32 s13, s5, 12
+; GFX8-NEXT:    s_lshl_b32 s15, s5, 16
+; GFX8-NEXT:    s_ashr_i64 s[18:19], s[8:9], 60
+; GFX8-NEXT:    s_lshl_b32 s9, s5, 24
+; GFX8-NEXT:    s_lshl_b32 s5, s5, 28
+; GFX8-NEXT:    s_ashr_i64 s[4:5], s[4:5], 60
+; GFX8-NEXT:    s_lshl_b32 s5, s7, 4
+; GFX8-NEXT:    s_ashr_i64 s[24:25], s[4:5], 60
+; GFX8-NEXT:    s_lshl_b32 s5, s7, 8
+; GFX8-NEXT:    s_ashr_i64 s[26:27], s[4:5], 60
+; GFX8-NEXT:    s_lshl_b32 s5, s7, 12
+; GFX8-NEXT:    s_ashr_i64 s[28:29], s[4:5], 60
+; GFX8-NEXT:    s_lshl_b32 s5, s7, 16
+; GFX8-NEXT:    s_ashr_i64 s[30:31], s[4:5], 60
+; GFX8-NEXT:    s_lshl_b32 s5, s7, 20
+; GFX8-NEXT:    s_ashr_i64 s[32:33], s[4:5], 60
+; GFX8-NEXT:    s_lshl_b32 s5, s7, 24
+; GFX8-NEXT:    s_ashr_i64 s[34:35], s[4:5], 60
+; GFX8-NEXT:    s_lshl_b32 s5, s7, 28
+; GFX8-NEXT:    s_ashr_i64 s[22:23], s[6:7], 60
+; GFX8-NEXT:    s_ashr_i64 s[6:7], s[4:5], 60
+; GFX8-NEXT:    v_mov_b32_e32 v0, s6
+; GFX8-NEXT:    v_mov_b32_e32 v1, s2
+; GFX8-NEXT:    v_mad_i32_i24 v0, s4, v0, v1
+; GFX8-NEXT:    s_ashr_i64 s[20:21], s[8:9], 60
+; GFX8-NEXT:    v_mov_b32_e32 v1, s34
+; GFX8-NEXT:    v_mad_i32_i24 v0, s20, v1, v0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s32
+; GFX8-NEXT:    v_mad_i32_i24 v0, s18, v1, v0
+; GFX8-NEXT:    s_ashr_i64 s[14:15], s[14:15], 60
+; GFX8-NEXT:    v_mov_b32_e32 v1, s30
+; GFX8-NEXT:    v_mad_i32_i24 v0, s14, v1, v0
+; GFX8-NEXT:    s_ashr_i64 s[12:13], s[12:13], 60
+; GFX8-NEXT:    v_mov_b32_e32 v1, s28
+; GFX8-NEXT:    v_mad_i32_i24 v0, s12, v1, v0
 ; GFX8-NEXT:    s_ashr_i64 s[10:11], s[10:11], 60
-; GFX8-NEXT:    v_mov_b32_e32 v3, s26
-; GFX8-NEXT:    v_mad_i32_i24 v2, s10, v3, v2
-; GFX8-NEXT:    s_ashr_i64 s[8:9], s[8:9], 60
-; GFX8-NEXT:    v_mov_b32_e32 v3, s24
-; GFX8-NEXT:    v_mad_i32_i24 v2, s8, v3, v2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s22
-; GFX8-NEXT:    v_mad_i32_i24 v2, s12, v3, v2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s20
-; GFX8-NEXT:    v_mad_i32_i24 v2, s0, v3, v2
+; GFX8-NEXT:    v_mov_b32_e32 v1, s26
+; GFX8-NEXT:    v_mad_i32_i24 v0, s10, v1, v0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s24
+; GFX8-NEXT:    v_mad_i32_i24 v0, s16, v1, v0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s22
+; GFX8-NEXT:    v_mad_i32_i24 v2, s8, v1, v0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX8-NEXT:    flat_store_dword v[0:1], v2
 ; GFX8-NEXT:    s_endpgm
 ;
@@ -1374,56 +1370,56 @@ define amdgpu_kernel void @idot8_acc32_vecMul(<8 x i4> addrspace(1)* %src1,
 ; GFX9-NEXT:    s_load_dword s5, s[4:5], 0x0
 ; GFX9-NEXT:    s_load_dword s7, s[6:7], 0x0
 ; GFX9-NEXT:    s_load_dword s2, s[0:1], 0x0
-; GFX9-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_ashr_i64 s[0:1], s[4:5], 60
-; GFX9-NEXT:    s_lshl_b32 s1, s5, 4
-; GFX9-NEXT:    s_ashr_i64 s[12:13], s[0:1], 60
-; GFX9-NEXT:    s_lshl_b32 s1, s5, 16
-; GFX9-NEXT:    s_ashr_i64 s[14:15], s[0:1], 60
-; GFX9-NEXT:    s_lshl_b32 s1, s5, 20
-; GFX9-NEXT:    s_ashr_i64 s[16:17], s[0:1], 60
-; GFX9-NEXT:    s_lshl_b32 s1, s5, 24
-; GFX9-NEXT:    s_ashr_i64 s[18:19], s[0:1], 60
-; GFX9-NEXT:    s_lshl_b32 s1, s5, 28
-; GFX9-NEXT:    s_lshl_b32 s9, s5, 8
-; GFX9-NEXT:    s_lshl_b32 s11, s5, 12
-; GFX9-NEXT:    s_ashr_i64 s[4:5], s[0:1], 60
-; GFX9-NEXT:    s_lshl_b32 s1, s7, 4
-; GFX9-NEXT:    s_ashr_i64 s[22:23], s[0:1], 60
-; GFX9-NEXT:    s_lshl_b32 s1, s7, 8
-; GFX9-NEXT:    s_ashr_i64 s[24:25], s[0:1], 60
-; GFX9-NEXT:    s_lshl_b32 s1, s7, 12
-; GFX9-NEXT:    s_ashr_i64 s[26:27], s[0:1], 60
-; GFX9-NEXT:    s_lshl_b32 s1, s7, 16
-; GFX9-NEXT:    s_ashr_i64 s[28:29], s[0:1], 60
-; GFX9-NEXT:    s_lshl_b32 s1, s7, 20
-; GFX9-NEXT:    s_ashr_i64 s[30:31], s[0:1], 60
-; GFX9-NEXT:    s_lshl_b32 s1, s7, 24
-; GFX9-NEXT:    s_ashr_i64 s[32:33], s[0:1], 60
-; GFX9-NEXT:    s_lshl_b32 s1, s7, 28
-; GFX9-NEXT:    s_ashr_i64 s[20:21], s[6:7], 60
-; GFX9-NEXT:    s_ashr_i64 s[6:7], s[0:1], 60
-; GFX9-NEXT:    v_mov_b32_e32 v2, s6
-; GFX9-NEXT:    v_mov_b32_e32 v3, s2
-; GFX9-NEXT:    v_mad_i32_i24 v2, s4, v2, v3
-; GFX9-NEXT:    v_mov_b32_e32 v3, s32
-; GFX9-NEXT:    v_mad_i32_i24 v2, s18, v3, v2
-; GFX9-NEXT:    v_mov_b32_e32 v3, s30
-; GFX9-NEXT:    v_mad_i32_i24 v2, s16, v3, v2
-; GFX9-NEXT:    v_mov_b32_e32 v3, s28
-; GFX9-NEXT:    v_mad_i32_i24 v2, s14, v3, v2
+; GFX9-NEXT:    s_ashr_i64 s[8:9], s[4:5], 60
+; GFX9-NEXT:    s_lshl_b32 s9, s5, 4
+; GFX9-NEXT:    s_ashr_i64 s[16:17], s[8:9], 60
+; GFX9-NEXT:    s_lshl_b32 s9, s5, 20
+; GFX9-NEXT:    s_lshl_b32 s11, s5, 8
+; GFX9-NEXT:    s_lshl_b32 s13, s5, 12
+; GFX9-NEXT:    s_lshl_b32 s15, s5, 16
+; GFX9-NEXT:    s_ashr_i64 s[18:19], s[8:9], 60
+; GFX9-NEXT:    s_lshl_b32 s9, s5, 24
+; GFX9-NEXT:    s_lshl_b32 s5, s5, 28
+; GFX9-NEXT:    s_ashr_i64 s[4:5], s[4:5], 60
+; GFX9-NEXT:    s_lshl_b32 s5, s7, 4
+; GFX9-NEXT:    s_ashr_i64 s[24:25], s[4:5], 60
+; GFX9-NEXT:    s_lshl_b32 s5, s7, 8
+; GFX9-NEXT:    s_ashr_i64 s[26:27], s[4:5], 60
+; GFX9-NEXT:    s_lshl_b32 s5, s7, 12
+; GFX9-NEXT:    s_ashr_i64 s[28:29], s[4:5], 60
+; GFX9-NEXT:    s_lshl_b32 s5, s7, 16
+; GFX9-NEXT:    s_ashr_i64 s[30:31], s[4:5], 60
+; GFX9-NEXT:    s_lshl_b32 s5, s7, 20
+; GFX9-NEXT:    s_ashr_i64 s[32:33], s[4:5], 60
+; GFX9-NEXT:    s_lshl_b32 s5, s7, 24
+; GFX9-NEXT:    s_ashr_i64 s[34:35], s[4:5], 60
+; GFX9-NEXT:    s_lshl_b32 s5, s7, 28
+; GFX9-NEXT:    s_ashr_i64 s[22:23], s[6:7], 60
+; GFX9-NEXT:    s_ashr_i64 s[6:7], s[4:5], 60
+; GFX9-NEXT:    v_mov_b32_e32 v0, s6
+; GFX9-NEXT:    v_mov_b32_e32 v1, s2
+; GFX9-NEXT:    v_mad_i32_i24 v0, s4, v0, v1
+; GFX9-NEXT:    s_ashr_i64 s[20:21], s[8:9], 60
+; GFX9-NEXT:    v_mov_b32_e32 v1, s34
+; GFX9-NEXT:    v_mad_i32_i24 v0, s20, v1, v0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s32
+; GFX9-NEXT:    v_mad_i32_i24 v0, s18, v1, v0
+; GFX9-NEXT:    s_ashr_i64 s[14:15], s[14:15], 60
+; GFX9-NEXT:    v_mov_b32_e32 v1, s30
+; GFX9-NEXT:    v_mad_i32_i24 v0, s14, v1, v0
+; GFX9-NEXT:    s_ashr_i64 s[12:13], s[12:13], 60
+; GFX9-NEXT:    v_mov_b32_e32 v1, s28
+; GFX9-NEXT:    v_mad_i32_i24 v0, s12, v1, v0
 ; GFX9-NEXT:    s_ashr_i64 s[10:11], s[10:11], 60
-; GFX9-NEXT:    v_mov_b32_e32 v3, s26
-; GFX9-NEXT:    v_mad_i32_i24 v2, s10, v3, v2
-; GFX9-NEXT:    s_ashr_i64 s[8:9], s[8:9], 60
-; GFX9-NEXT:    v_mov_b32_e32 v3, s24
-; GFX9-NEXT:    v_mad_i32_i24 v2, s8, v3, v2
-; GFX9-NEXT:    v_mov_b32_e32 v3, s22
-; GFX9-NEXT:    v_mad_i32_i24 v2, s12, v3, v2
-; GFX9-NEXT:    v_mov_b32_e32 v3, s20
-; GFX9-NEXT:    v_mad_i32_i24 v2, s0, v3, v2
+; GFX9-NEXT:    v_mov_b32_e32 v1, s26
+; GFX9-NEXT:    v_mad_i32_i24 v0, s10, v1, v0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s24
+; GFX9-NEXT:    v_mad_i32_i24 v0, s16, v1, v0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s22
+; GFX9-NEXT:    v_mad_i32_i24 v2, s8, v1, v0
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-NEXT:    s_endpgm
 ;
@@ -1435,56 +1431,56 @@ define amdgpu_kernel void @idot8_acc32_vecMul(<8 x i4> addrspace(1)* %src1,
 ; GFX9-DL-NEXT:    s_load_dword s5, s[4:5], 0x0
 ; GFX9-DL-NEXT:    s_load_dword s7, s[6:7], 0x0
 ; GFX9-DL-NEXT:    s_load_dword s2, s[0:1], 0x0
-; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    s_ashr_i64 s[0:1], s[4:5], 60
-; GFX9-DL-NEXT:    s_lshl_b32 s1, s5, 4
-; GFX9-DL-NEXT:    s_ashr_i64 s[12:13], s[0:1], 60
-; GFX9-DL-NEXT:    s_lshl_b32 s1, s5, 16
-; GFX9-DL-NEXT:    s_ashr_i64 s[14:15], s[0:1], 60
-; GFX9-DL-NEXT:    s_lshl_b32 s1, s5, 20
-; GFX9-DL-NEXT:    s_ashr_i64 s[16:17], s[0:1], 60
-; GFX9-DL-NEXT:    s_lshl_b32 s1, s5, 24
-; GFX9-DL-NEXT:    s_ashr_i64 s[18:19], s[0:1], 60
-; GFX9-DL-NEXT:    s_lshl_b32 s1, s5, 28
-; GFX9-DL-NEXT:    s_lshl_b32 s9, s5, 8
-; GFX9-DL-NEXT:    s_lshl_b32 s11, s5, 12
-; GFX9-DL-NEXT:    s_ashr_i64 s[4:5], s[0:1], 60
-; GFX9-DL-NEXT:    s_lshl_b32 s1, s7, 4
-; GFX9-DL-NEXT:    s_ashr_i64 s[22:23], s[0:1], 60
-; GFX9-DL-NEXT:    s_lshl_b32 s1, s7, 8
-; GFX9-DL-NEXT:    s_ashr_i64 s[24:25], s[0:1], 60
-; GFX9-DL-NEXT:    s_lshl_b32 s1, s7, 12
-; GFX9-DL-NEXT:    s_ashr_i64 s[26:27], s[0:1], 60
-; GFX9-DL-NEXT:    s_lshl_b32 s1, s7, 16
-; GFX9-DL-NEXT:    s_ashr_i64 s[28:29], s[0:1], 60
-; GFX9-DL-NEXT:    s_lshl_b32 s1, s7, 20
-; GFX9-DL-NEXT:    s_ashr_i64 s[30:31], s[0:1], 60
-; GFX9-DL-NEXT:    s_lshl_b32 s1, s7, 24
-; GFX9-DL-NEXT:    s_ashr_i64 s[32:33], s[0:1], 60
-; GFX9-DL-NEXT:    s_lshl_b32 s1, s7, 28
-; GFX9-DL-NEXT:    s_ashr_i64 s[20:21], s[6:7], 60
-; GFX9-DL-NEXT:    s_ashr_i64 s[6:7], s[0:1], 60
-; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s6
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s2
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s4, v2, v3
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s32
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s18, v3, v2
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s30
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s16, v3, v2
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s28
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s14, v3, v2
+; GFX9-DL-NEXT:    s_ashr_i64 s[8:9], s[4:5], 60
+; GFX9-DL-NEXT:    s_lshl_b32 s9, s5, 4
+; GFX9-DL-NEXT:    s_ashr_i64 s[16:17], s[8:9], 60
+; GFX9-DL-NEXT:    s_lshl_b32 s9, s5, 20
+; GFX9-DL-NEXT:    s_lshl_b32 s11, s5, 8
+; GFX9-DL-NEXT:    s_lshl_b32 s13, s5, 12
+; GFX9-DL-NEXT:    s_lshl_b32 s15, s5, 16
+; GFX9-DL-NEXT:    s_ashr_i64 s[18:19], s[8:9], 60
+; GFX9-DL-NEXT:    s_lshl_b32 s9, s5, 24
+; GFX9-DL-NEXT:    s_lshl_b32 s5, s5, 28
+; GFX9-DL-NEXT:    s_ashr_i64 s[4:5], s[4:5], 60
+; GFX9-DL-NEXT:    s_lshl_b32 s5, s7, 4
+; GFX9-DL-NEXT:    s_ashr_i64 s[24:25], s[4:5], 60
+; GFX9-DL-NEXT:    s_lshl_b32 s5, s7, 8
+; GFX9-DL-NEXT:    s_ashr_i64 s[26:27], s[4:5], 60
+; GFX9-DL-NEXT:    s_lshl_b32 s5, s7, 12
+; GFX9-DL-NEXT:    s_ashr_i64 s[28:29], s[4:5], 60
+; GFX9-DL-NEXT:    s_lshl_b32 s5, s7, 16
+; GFX9-DL-NEXT:    s_ashr_i64 s[30:31], s[4:5], 60
+; GFX9-DL-NEXT:    s_lshl_b32 s5, s7, 20
+; GFX9-DL-NEXT:    s_ashr_i64 s[32:33], s[4:5], 60
+; GFX9-DL-NEXT:    s_lshl_b32 s5, s7, 24
+; GFX9-DL-NEXT:    s_ashr_i64 s[34:35], s[4:5], 60
+; GFX9-DL-NEXT:    s_lshl_b32 s5, s7, 28
+; GFX9-DL-NEXT:    s_ashr_i64 s[22:23], s[6:7], 60
+; GFX9-DL-NEXT:    s_ashr_i64 s[6:7], s[4:5], 60
+; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s6
+; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s2
+; GFX9-DL-NEXT:    v_mad_i32_i24 v0, s4, v0, v1
+; GFX9-DL-NEXT:    s_ashr_i64 s[20:21], s[8:9], 60
+; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s34
+; GFX9-DL-NEXT:    v_mad_i32_i24 v0, s20, v1, v0
+; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s32
+; GFX9-DL-NEXT:    v_mad_i32_i24 v0, s18, v1, v0
+; GFX9-DL-NEXT:    s_ashr_i64 s[14:15], s[14:15], 60
+; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s30
+; GFX9-DL-NEXT:    v_mad_i32_i24 v0, s14, v1, v0
+; GFX9-DL-NEXT:    s_ashr_i64 s[12:13], s[12:13], 60
+; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s28
+; GFX9-DL-NEXT:    v_mad_i32_i24 v0, s12, v1, v0
 ; GFX9-DL-NEXT:    s_ashr_i64 s[10:11], s[10:11], 60
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s26
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s10, v3, v2
-; GFX9-DL-NEXT:    s_ashr_i64 s[8:9], s[8:9], 60
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s24
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s8, v3, v2
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s22
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s12, v3, v2
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s20
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s0, v3, v2
+; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s26
+; GFX9-DL-NEXT:    v_mad_i32_i24 v0, s10, v1, v0
+; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s24
+; GFX9-DL-NEXT:    v_mad_i32_i24 v0, s16, v1, v0
+; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s22
+; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s8, v1, v0
+; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-DL-NEXT:    s_endpgm
 ;
@@ -1497,48 +1493,48 @@ define amdgpu_kernel void @idot8_acc32_vecMul(<8 x i4> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_load_dword s5, s[4:5], 0x0
 ; GFX10-DL-NEXT:    s_load_dword s7, s[6:7], 0x0
 ; GFX10-DL-NEXT:    s_load_dword s2, s[0:1], 0x0
-; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_lshl_b32 s1, s5, 28
-; GFX10-DL-NEXT:    s_lshl_b32 s9, s7, 28
-; GFX10-DL-NEXT:    s_lshl_b32 s11, s5, 24
-; GFX10-DL-NEXT:    s_lshl_b32 s13, s7, 24
-; GFX10-DL-NEXT:    v_mov_b32_e32 v2, s2
-; GFX10-DL-NEXT:    s_ashr_i64 s[0:1], s[0:1], 60
+; GFX10-DL-NEXT:    s_lshl_b32 s9, s5, 28
+; GFX10-DL-NEXT:    s_lshl_b32 s11, s7, 28
+; GFX10-DL-NEXT:    s_lshl_b32 s13, s5, 24
+; GFX10-DL-NEXT:    s_lshl_b32 s15, s7, 24
+; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX10-DL-NEXT:    s_ashr_i64 s[8:9], s[8:9], 60
-; GFX10-DL-NEXT:    s_lshl_b32 s1, s5, 20
 ; GFX10-DL-NEXT:    s_ashr_i64 s[10:11], s[10:11], 60
-; GFX10-DL-NEXT:    s_lshl_b32 s9, s7, 20
 ; GFX10-DL-NEXT:    s_ashr_i64 s[12:13], s[12:13], 60
-; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s0, s8, v2
-; GFX10-DL-NEXT:    s_ashr_i64 s[0:1], s[0:1], 60
-; GFX10-DL-NEXT:    s_lshl_b32 s11, s5, 16
+; GFX10-DL-NEXT:    s_ashr_i64 s[14:15], s[14:15], 60
+; GFX10-DL-NEXT:    s_lshl_b32 s9, s5, 20
+; GFX10-DL-NEXT:    s_lshl_b32 s11, s7, 20
+; GFX10-DL-NEXT:    v_mad_i32_i24 v0, s8, s10, v0
+; GFX10-DL-NEXT:    s_lshl_b32 s13, s5, 16
+; GFX10-DL-NEXT:    s_lshl_b32 s15, s7, 16
 ; GFX10-DL-NEXT:    s_ashr_i64 s[8:9], s[8:9], 60
-; GFX10-DL-NEXT:    s_lshl_b32 s1, s7, 16
-; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s10, s12, v2
-; GFX10-DL-NEXT:    s_lshl_b32 s9, s5, 12
 ; GFX10-DL-NEXT:    s_ashr_i64 s[10:11], s[10:11], 60
+; GFX10-DL-NEXT:    v_mad_i32_i24 v0, s12, s14, v0
+; GFX10-DL-NEXT:    s_ashr_i64 s[12:13], s[12:13], 60
+; GFX10-DL-NEXT:    s_ashr_i64 s[14:15], s[14:15], 60
+; GFX10-DL-NEXT:    s_lshl_b32 s9, s5, 12
 ; GFX10-DL-NEXT:    s_lshl_b32 s11, s7, 12
-; GFX10-DL-NEXT:    s_ashr_i64 s[12:13], s[0:1], 60
-; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s0, s8, v2
-; GFX10-DL-NEXT:    s_lshl_b32 s1, s5, 8
+; GFX10-DL-NEXT:    v_mad_i32_i24 v0, s8, s10, v0
+; GFX10-DL-NEXT:    s_lshl_b32 s13, s5, 8
+; GFX10-DL-NEXT:    s_lshl_b32 s15, s7, 8
+; GFX10-DL-NEXT:    s_ashr_i64 s[8:9], s[8:9], 60
+; GFX10-DL-NEXT:    s_ashr_i64 s[10:11], s[10:11], 60
+; GFX10-DL-NEXT:    v_mad_i32_i24 v0, s12, s14, v0
+; GFX10-DL-NEXT:    s_lshl_b32 s9, s5, 4
+; GFX10-DL-NEXT:    s_lshl_b32 s11, s7, 4
+; GFX10-DL-NEXT:    s_ashr_i64 s[12:13], s[12:13], 60
+; GFX10-DL-NEXT:    s_ashr_i64 s[14:15], s[14:15], 60
+; GFX10-DL-NEXT:    v_mad_i32_i24 v0, s8, s10, v0
 ; GFX10-DL-NEXT:    s_ashr_i64 s[8:9], s[8:9], 60
-; GFX10-DL-NEXT:    s_ashr_i64 s[14:15], s[10:11], 60
-; GFX10-DL-NEXT:    s_lshl_b32 s9, s7, 8
-; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s10, s12, v2
-; GFX10-DL-NEXT:    s_ashr_i64 s[0:1], s[0:1], 60
-; GFX10-DL-NEXT:    s_lshl_b32 s11, s5, 4
-; GFX10-DL-NEXT:    s_lshl_b32 s1, s7, 4
-; GFX10-DL-NEXT:    s_ashr_i64 s[12:13], s[8:9], 60
-; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s8, s14, v2
-; GFX10-DL-NEXT:    s_ashr_i64 s[8:9], s[10:11], 60
-; GFX10-DL-NEXT:    s_ashr_i64 s[10:11], s[0:1], 60
-; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s0, s12, v2
-; GFX10-DL-NEXT:    s_ashr_i64 s[0:1], s[4:5], 60
-; GFX10-DL-NEXT:    s_ashr_i64 s[4:5], s[6:7], 60
-; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s8, s10, v2
-; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s0, s4, v2
+; GFX10-DL-NEXT:    s_ashr_i64 s[10:11], s[10:11], 60
+; GFX10-DL-NEXT:    s_ashr_i64 s[4:5], s[4:5], 60
+; GFX10-DL-NEXT:    s_ashr_i64 s[6:7], s[6:7], 60
+; GFX10-DL-NEXT:    v_mad_i32_i24 v0, s12, s14, v0
+; GFX10-DL-NEXT:    v_mad_i32_i24 v0, s8, s10, v0
+; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s4, s6, v0
+; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX10-DL-NEXT:    s_endpgm
                                               <8 x i4> addrspace(1)* %src2,
@@ -1582,24 +1578,24 @@ define amdgpu_kernel void @idot8_acc16_vecMul(<8 x i4> addrspace(1)* %src1,
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xd
 ; GFX7-NEXT:    s_mov_b32 s7, 0xf000
 ; GFX7-NEXT:    s_mov_b32 s6, -1
-; GFX7-NEXT:    s_mov_b32 s0, 0xffff
+; GFX7-NEXT:    s_mov_b32 s2, 0xffff
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_load_dword s1, s[8:9], 0x0
+; GFX7-NEXT:    s_load_dword s0, s[10:11], 0x0
 ; GFX7-NEXT:    buffer_load_ushort v0, off, s[4:7], 0
-; GFX7-NEXT:    s_load_dword s2, s[10:11], 0x0
+; GFX7-NEXT:    s_load_dword s1, s[8:9], 0x0
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_bfe_i32 s8, s1, 0x40010
-; GFX7-NEXT:    s_bfe_i32 s9, s1, 0x40014
-; GFX7-NEXT:    s_bfe_i32 s15, s2, 0x40010
-; GFX7-NEXT:    s_bfe_i32 s16, s2, 0x40014
-; GFX7-NEXT:    s_bfe_i32 s17, s2, 0x40018
-; GFX7-NEXT:    s_ashr_i32 s18, s2, 28
-; GFX7-NEXT:    s_bfe_i32 s19, s2, 0x40000
-; GFX7-NEXT:    s_bfe_i32 s20, s2, 0x40004
-; GFX7-NEXT:    s_bfe_i32 s21, s2, 0x40008
-; GFX7-NEXT:    s_bfe_i32 s2, s2, 0x4000c
-; GFX7-NEXT:    s_bfe_i32 s10, s1, 0x40018
-; GFX7-NEXT:    s_ashr_i32 s11, s1, 28
+; GFX7-NEXT:    s_bfe_i32 s16, s0, 0x40018
+; GFX7-NEXT:    s_bfe_i32 s17, s0, 0x40014
+; GFX7-NEXT:    s_bfe_i32 s18, s0, 0x40010
+; GFX7-NEXT:    s_bfe_i32 s19, s0, 0x40000
+; GFX7-NEXT:    s_bfe_i32 s20, s0, 0x40004
+; GFX7-NEXT:    s_bfe_i32 s21, s0, 0x40008
+; GFX7-NEXT:    s_ashr_i32 s15, s0, 28
+; GFX7-NEXT:    s_bfe_i32 s0, s0, 0x4000c
+; GFX7-NEXT:    s_ashr_i32 s8, s1, 28
+; GFX7-NEXT:    s_bfe_i32 s9, s1, 0x40018
+; GFX7-NEXT:    s_bfe_i32 s10, s1, 0x40014
+; GFX7-NEXT:    s_bfe_i32 s11, s1, 0x40010
 ; GFX7-NEXT:    s_bfe_i32 s12, s1, 0x40000
 ; GFX7-NEXT:    v_mov_b32_e32 v4, s19
 ; GFX7-NEXT:    s_bfe_i32 s13, s1, 0x40004
@@ -1607,44 +1603,32 @@ define amdgpu_kernel void @idot8_acc16_vecMul(<8 x i4> addrspace(1)* %src1,
 ; GFX7-NEXT:    s_bfe_i32 s14, s1, 0x40008
 ; GFX7-NEXT:    v_mov_b32_e32 v2, s21
 ; GFX7-NEXT:    s_bfe_i32 s1, s1, 0x4000c
-; GFX7-NEXT:    v_mov_b32_e32 v1, s2
-; GFX7-NEXT:    v_mov_b32_e32 v5, s18
-; GFX7-NEXT:    v_mov_b32_e32 v6, s17
+; GFX7-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX7-NEXT:    v_mul_i32_i24_e32 v1, s1, v1
 ; GFX7-NEXT:    v_mul_i32_i24_e32 v2, s14, v2
 ; GFX7-NEXT:    v_mul_i32_i24_e32 v3, s13, v3
 ; GFX7-NEXT:    v_mul_i32_i24_e32 v4, s12, v4
-; GFX7-NEXT:    v_mul_i32_i24_e32 v5, s11, v5
-; GFX7-NEXT:    v_mul_i32_i24_e32 v6, s10, v6
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX7-NEXT:    v_and_b32_e32 v2, s0, v2
+; GFX7-NEXT:    v_and_b32_e32 v2, s2, v2
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX7-NEXT:    v_and_b32_e32 v4, s0, v4
+; GFX7-NEXT:    v_and_b32_e32 v4, s2, v4
 ; GFX7-NEXT:    v_or_b32_e32 v1, v2, v1
 ; GFX7-NEXT:    v_or_b32_e32 v2, v4, v3
-; GFX7-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GFX7-NEXT:    v_and_b32_e32 v6, s0, v6
+; GFX7-NEXT:    v_alignbit_b32 v3, v1, v2, 16
+; GFX7-NEXT:    v_lshrrev_b32_e32 v4, 16, v1
+; GFX7-NEXT:    v_mov_b32_e32 v5, s18
+; GFX7-NEXT:    v_mov_b32_e32 v6, s17
 ; GFX7-NEXT:    v_mov_b32_e32 v7, s16
-; GFX7-NEXT:    v_mov_b32_e32 v8, s15
-; GFX7-NEXT:    v_or_b32_e32 v3, v6, v5
-; GFX7-NEXT:    v_alignbit_b32 v5, v1, v2, 16
-; GFX7-NEXT:    v_mul_i32_i24_e32 v7, s9, v7
-; GFX7-NEXT:    v_mul_i32_i24_e32 v8, s8, v8
-; GFX7-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; GFX7-NEXT:    v_and_b32_e32 v8, s0, v8
-; GFX7-NEXT:    v_lshrrev_b32_e32 v6, 16, v1
-; GFX7-NEXT:    v_or_b32_e32 v4, v8, v7
-; GFX7-NEXT:    v_lshrrev_b32_e32 v7, 16, v4
-; GFX7-NEXT:    v_lshrrev_b32_e32 v8, 16, v3
 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v5, v0
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v3, v0
 ; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v6, v0
 ; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v4, v0
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v7, v0
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v3, v0
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v8, v0
+; GFX7-NEXT:    v_mad_i32_i24 v0, s11, v5, v0
+; GFX7-NEXT:    v_mad_i32_i24 v0, s10, v6, v0
+; GFX7-NEXT:    v_mad_i32_i24 v0, s9, v7, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s15
+; GFX7-NEXT:    v_mad_i32_i24 v0, s8, v1, v0
 ; GFX7-NEXT:    buffer_store_short v0, off, s[4:7], 0
 ; GFX7-NEXT:    s_endpgm
 ;
@@ -1653,68 +1637,68 @@ define amdgpu_kernel void @idot8_acc16_vecMul(<8 x i4> addrspace(1)* %src1,
 ; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX8-NEXT:    s_load_dword s4, s[6:7], 0x0
+; GFX8-NEXT:    s_load_dword s2, s[6:7], 0x0
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX8-NEXT:    flat_load_ushort v2, v[0:1]
+; GFX8-NEXT:    s_load_dword s0, s[4:5], 0x0
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_lshlrev_b16_e64 v3, 12, s2
-; GFX8-NEXT:    v_lshlrev_b16_e64 v4, 12, s4
-; GFX8-NEXT:    s_lshr_b32 s0, s2, 4
-; GFX8-NEXT:    s_lshr_b32 s1, s2, 8
-; GFX8-NEXT:    s_lshr_b32 s5, s4, 4
-; GFX8-NEXT:    s_lshr_b32 s6, s4, 8
-; GFX8-NEXT:    v_lshlrev_b16_e64 v5, 12, s1
-; GFX8-NEXT:    v_lshlrev_b16_e64 v6, 12, s0
-; GFX8-NEXT:    v_lshlrev_b16_e64 v7, 12, s6
-; GFX8-NEXT:    v_lshlrev_b16_e64 v8, 12, s5
+; GFX8-NEXT:    v_lshlrev_b16_e64 v4, 12, s2
+; GFX8-NEXT:    s_lshr_b32 s15, s2, 4
+; GFX8-NEXT:    s_lshr_b32 s16, s2, 8
+; GFX8-NEXT:    v_lshlrev_b16_e64 v12, 12, s16
+; GFX8-NEXT:    v_lshlrev_b16_e64 v3, 12, s0
+; GFX8-NEXT:    s_lshr_b32 s8, s0, 4
+; GFX8-NEXT:    s_lshr_b32 s9, s0, 8
+; GFX8-NEXT:    v_lshlrev_b16_e64 v5, 12, s9
+; GFX8-NEXT:    v_lshlrev_b16_e64 v6, 12, s8
+; GFX8-NEXT:    v_lshlrev_b16_e64 v13, 12, s15
 ; GFX8-NEXT:    v_ashrrev_i16_e32 v3, 12, v3
 ; GFX8-NEXT:    v_ashrrev_i16_e32 v4, 12, v4
-; GFX8-NEXT:    s_lshr_b32 s0, s2, 12
-; GFX8-NEXT:    s_lshr_b32 s1, s4, 12
+; GFX8-NEXT:    s_lshr_b32 s7, s0, 12
+; GFX8-NEXT:    s_lshr_b32 s14, s2, 12
 ; GFX8-NEXT:    v_ashrrev_i16_e32 v6, 12, v6
 ; GFX8-NEXT:    v_ashrrev_i16_e32 v5, 12, v5
-; GFX8-NEXT:    v_ashrrev_i16_e32 v7, 12, v7
-; GFX8-NEXT:    v_ashrrev_i16_e32 v8, 12, v8
-; GFX8-NEXT:    v_lshlrev_b16_e64 v9, 12, s0
-; GFX8-NEXT:    v_lshlrev_b16_e64 v10, 12, s1
-; GFX8-NEXT:    s_lshr_b32 s5, s2, 16
-; GFX8-NEXT:    s_lshr_b32 s6, s4, 16
-; GFX8-NEXT:    v_mul_u32_u24_e32 v5, v5, v7
-; GFX8-NEXT:    v_lshlrev_b16_e64 v11, 12, s5
-; GFX8-NEXT:    v_lshlrev_b16_e64 v12, 12, s6
-; GFX8-NEXT:    s_lshr_b32 s0, s2, 20
-; GFX8-NEXT:    s_lshr_b32 s1, s4, 20
-; GFX8-NEXT:    v_ashrrev_i16_e32 v9, 12, v9
-; GFX8-NEXT:    v_ashrrev_i16_e32 v10, 12, v10
-; GFX8-NEXT:    v_lshlrev_b16_e64 v13, 12, s0
-; GFX8-NEXT:    v_lshlrev_b16_e64 v14, 12, s1
-; GFX8-NEXT:    s_lshr_b32 s5, s2, 24
-; GFX8-NEXT:    s_lshr_b32 s6, s4, 24
-; GFX8-NEXT:    v_ashrrev_i16_e32 v11, 12, v11
 ; GFX8-NEXT:    v_ashrrev_i16_e32 v12, 12, v12
-; GFX8-NEXT:    v_lshlrev_b16_e64 v15, 12, s5
-; GFX8-NEXT:    v_lshlrev_b16_e64 v17, 12, s6
-; GFX8-NEXT:    s_lshr_b32 s0, s2, 28
-; GFX8-NEXT:    s_lshr_b32 s1, s4, 28
 ; GFX8-NEXT:    v_ashrrev_i16_e32 v13, 12, v13
+; GFX8-NEXT:    v_lshlrev_b16_e64 v7, 12, s7
+; GFX8-NEXT:    v_lshlrev_b16_e64 v14, 12, s14
+; GFX8-NEXT:    s_lshr_b32 s6, s0, 16
+; GFX8-NEXT:    s_lshr_b32 s13, s2, 16
+; GFX8-NEXT:    v_mul_u32_u24_e32 v5, v5, v12
+; GFX8-NEXT:    v_lshlrev_b16_e64 v8, 12, s6
+; GFX8-NEXT:    v_lshlrev_b16_e64 v15, 12, s13
+; GFX8-NEXT:    s_lshr_b32 s5, s0, 20
+; GFX8-NEXT:    s_lshr_b32 s12, s2, 20
+; GFX8-NEXT:    v_ashrrev_i16_e32 v7, 12, v7
 ; GFX8-NEXT:    v_ashrrev_i16_e32 v14, 12, v14
-; GFX8-NEXT:    v_lshlrev_b16_e64 v16, 12, s0
-; GFX8-NEXT:    v_lshlrev_b16_e64 v18, 12, s1
+; GFX8-NEXT:    v_lshlrev_b16_e64 v9, 12, s5
+; GFX8-NEXT:    v_lshlrev_b16_e64 v16, 12, s12
+; GFX8-NEXT:    s_lshr_b32 s4, s0, 24
+; GFX8-NEXT:    s_lshr_b32 s11, s2, 24
+; GFX8-NEXT:    v_ashrrev_i16_e32 v8, 12, v8
 ; GFX8-NEXT:    v_ashrrev_i16_e32 v15, 12, v15
-; GFX8-NEXT:    v_ashrrev_i16_e32 v17, 12, v17
+; GFX8-NEXT:    v_lshlrev_b16_e64 v10, 12, s4
+; GFX8-NEXT:    v_lshlrev_b16_e64 v17, 12, s11
+; GFX8-NEXT:    s_lshr_b32 s1, s0, 28
+; GFX8-NEXT:    s_lshr_b32 s10, s2, 28
+; GFX8-NEXT:    v_ashrrev_i16_e32 v9, 12, v9
 ; GFX8-NEXT:    v_ashrrev_i16_e32 v16, 12, v16
+; GFX8-NEXT:    v_lshlrev_b16_e64 v11, 12, s1
+; GFX8-NEXT:    v_lshlrev_b16_e64 v18, 12, s10
+; GFX8-NEXT:    v_ashrrev_i16_e32 v10, 12, v10
+; GFX8-NEXT:    v_ashrrev_i16_e32 v17, 12, v17
+; GFX8-NEXT:    v_ashrrev_i16_e32 v11, 12, v11
 ; GFX8-NEXT:    v_ashrrev_i16_e32 v18, 12, v18
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    v_mad_u32_u24 v2, v3, v4, v2
-; GFX8-NEXT:    v_mad_u32_u24 v2, v6, v8, v2
+; GFX8-NEXT:    v_mad_u32_u24 v2, v6, v13, v2
 ; GFX8-NEXT:    v_add_u32_sdwa v2, vcc, v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
-; GFX8-NEXT:    v_mad_u32_u24 v2, v9, v10, v2
-; GFX8-NEXT:    v_mad_u32_u24 v2, v11, v12, v2
-; GFX8-NEXT:    v_mad_u32_u24 v2, v13, v14, v2
-; GFX8-NEXT:    v_mad_u32_u24 v2, v15, v17, v2
-; GFX8-NEXT:    v_mad_u32_u24 v2, v16, v18, v2
+; GFX8-NEXT:    v_mad_u32_u24 v2, v7, v14, v2
+; GFX8-NEXT:    v_mad_u32_u24 v2, v8, v15, v2
+; GFX8-NEXT:    v_mad_u32_u24 v2, v9, v16, v2
+; GFX8-NEXT:    v_mad_u32_u24 v2, v10, v17, v2
+; GFX8-NEXT:    v_mad_u32_u24 v2, v11, v18, v2
 ; GFX8-NEXT:    flat_store_short v[0:1], v2
 ; GFX8-NEXT:    s_endpgm
 ;
@@ -1724,64 +1708,64 @@ define amdgpu_kernel void @idot8_acc16_vecMul(<8 x i4> addrspace(1)* %src1,
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX9-NEXT:    s_load_dword s4, s[6:7], 0x0
+; GFX9-NEXT:    s_load_dword s6, s[6:7], 0x0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_and_b32 s4, s2, 15
+; GFX9-NEXT:    s_bfe_u32 s5, s2, 0x40004
+; GFX9-NEXT:    s_bfe_u32 s8, s2, 0x40008
+; GFX9-NEXT:    s_bfe_u32 s9, s2, 0x4000c
+; GFX9-NEXT:    s_bfe_u32 s10, s2, 0x40010
+; GFX9-NEXT:    s_bfe_u32 s11, s2, 0x40018
+; GFX9-NEXT:    s_lshr_b32 s12, s2, 28
+; GFX9-NEXT:    s_bfe_u32 s2, s2, 0x40014
+; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s10, s2
+; GFX9-NEXT:    v_pk_lshlrev_b16 v2, 12, s2 op_sel_hi:[0,1]
+; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s11, s12
+; GFX9-NEXT:    s_bfe_u32 s7, s6, 0x40018
+; GFX9-NEXT:    s_lshr_b32 s13, s6, 28
+; GFX9-NEXT:    s_bfe_u32 s14, s6, 0x40010
+; GFX9-NEXT:    s_bfe_u32 s15, s6, 0x40014
+; GFX9-NEXT:    s_bfe_u32 s16, s6, 0x40008
+; GFX9-NEXT:    s_bfe_u32 s17, s6, 0x4000c
+; GFX9-NEXT:    s_and_b32 s18, s6, 15
+; GFX9-NEXT:    s_bfe_u32 s6, s6, 0x40004
+; GFX9-NEXT:    v_pk_lshlrev_b16 v3, 12, s2 op_sel_hi:[0,1]
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s5
+; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s18, s6
+; GFX9-NEXT:    v_pk_lshlrev_b16 v0, 12, s4 op_sel_hi:[0,1]
+; GFX9-NEXT:    v_pk_lshlrev_b16 v4, 12, s2 op_sel_hi:[0,1]
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s8, s9
+; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s16, s17
+; GFX9-NEXT:    v_pk_lshlrev_b16 v5, 12, s2 op_sel_hi:[0,1]
+; GFX9-NEXT:    v_pk_lshlrev_b16 v1, 12, s4 op_sel_hi:[0,1]
+; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s14, s15
+; GFX9-NEXT:    v_pk_ashrrev_i16 v0, 12, v0 op_sel_hi:[0,1]
+; GFX9-NEXT:    v_pk_ashrrev_i16 v4, 12, v4 op_sel_hi:[0,1]
+; GFX9-NEXT:    v_pk_ashrrev_i16 v1, 12, v1 op_sel_hi:[0,1]
+; GFX9-NEXT:    v_pk_ashrrev_i16 v5, 12, v5 op_sel_hi:[0,1]
+; GFX9-NEXT:    v_pk_lshlrev_b16 v6, 12, s2 op_sel_hi:[0,1]
+; GFX9-NEXT:    v_pk_mul_lo_u16 v5, v1, v5
+; GFX9-NEXT:    v_pk_mul_lo_u16 v4, v0, v4
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_pk_ashrrev_i16 v2, 12, v2 op_sel_hi:[0,1]
+; GFX9-NEXT:    v_pk_ashrrev_i16 v6, 12, v6 op_sel_hi:[0,1]
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s1
-; GFX9-NEXT:    global_load_ushort v2, v[0:1], off
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_and_b32 s0, s2, 15
-; GFX9-NEXT:    s_bfe_u32 s1, s2, 0x40004
-; GFX9-NEXT:    s_and_b32 s5, s4, 15
-; GFX9-NEXT:    s_bfe_u32 s6, s4, 0x40004
-; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s1
-; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s5, s6
-; GFX9-NEXT:    s_bfe_u32 s5, s2, 0x40008
-; GFX9-NEXT:    s_bfe_u32 s6, s2, 0x4000c
-; GFX9-NEXT:    v_pk_lshlrev_b16 v3, 12, s0 op_sel_hi:[0,1]
-; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s5, s6
-; GFX9-NEXT:    s_bfe_u32 s9, s2, 0x40010
-; GFX9-NEXT:    s_bfe_u32 s10, s2, 0x40014
-; GFX9-NEXT:    v_pk_lshlrev_b16 v4, 12, s0 op_sel_hi:[0,1]
-; GFX9-NEXT:    s_bfe_u32 s13, s2, 0x40018
-; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s9, s10
-; GFX9-NEXT:    s_lshr_b32 s2, s2, 28
-; GFX9-NEXT:    v_pk_lshlrev_b16 v5, 12, s0 op_sel_hi:[0,1]
-; GFX9-NEXT:    v_pk_lshlrev_b16 v7, 12, s1 op_sel_hi:[0,1]
-; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s13, s2
-; GFX9-NEXT:    s_bfe_u32 s7, s4, 0x40008
-; GFX9-NEXT:    s_bfe_u32 s8, s4, 0x4000c
-; GFX9-NEXT:    v_pk_lshlrev_b16 v6, 12, s0 op_sel_hi:[0,1]
-; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s7, s8
+; GFX9-NEXT:    v_pk_mul_lo_u16 v2, v2, v6
+; GFX9-NEXT:    global_load_ushort v6, v[0:1], off
+; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s7, s13
+; GFX9-NEXT:    v_pk_lshlrev_b16 v7, 12, s2 op_sel_hi:[0,1]
 ; GFX9-NEXT:    v_pk_ashrrev_i16 v3, 12, v3 op_sel_hi:[0,1]
 ; GFX9-NEXT:    v_pk_ashrrev_i16 v7, 12, v7 op_sel_hi:[0,1]
-; GFX9-NEXT:    v_pk_lshlrev_b16 v8, 12, s0 op_sel_hi:[0,1]
 ; GFX9-NEXT:    v_pk_mul_lo_u16 v3, v3, v7
-; GFX9-NEXT:    s_bfe_u32 s11, s4, 0x40010
-; GFX9-NEXT:    s_bfe_u32 s12, s4, 0x40014
-; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s11, s12
-; GFX9-NEXT:    v_pk_ashrrev_i16 v4, 12, v4 op_sel_hi:[0,1]
-; GFX9-NEXT:    v_pk_ashrrev_i16 v8, 12, v8 op_sel_hi:[0,1]
-; GFX9-NEXT:    v_pk_lshlrev_b16 v9, 12, s0 op_sel_hi:[0,1]
-; GFX9-NEXT:    s_bfe_u32 s14, s4, 0x40018
-; GFX9-NEXT:    s_lshr_b32 s4, s4, 28
-; GFX9-NEXT:    v_pk_mul_lo_u16 v4, v4, v8
-; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s14, s4
-; GFX9-NEXT:    v_pk_ashrrev_i16 v5, 12, v5 op_sel_hi:[0,1]
-; GFX9-NEXT:    v_pk_ashrrev_i16 v9, 12, v9 op_sel_hi:[0,1]
-; GFX9-NEXT:    v_pk_lshlrev_b16 v10, 12, s0 op_sel_hi:[0,1]
-; GFX9-NEXT:    v_pk_mul_lo_u16 v5, v5, v9
-; GFX9-NEXT:    v_pk_ashrrev_i16 v6, 12, v6 op_sel_hi:[0,1]
-; GFX9-NEXT:    v_pk_ashrrev_i16 v10, 12, v10 op_sel_hi:[0,1]
-; GFX9-NEXT:    v_pk_mul_lo_u16 v6, v6, v10
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_add_u32_e32 v2, v3, v2
+; GFX9-NEXT:    v_add_u32_e32 v6, v4, v6
+; GFX9-NEXT:    v_add_u32_sdwa v4, v6, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT:    v_add_u32_sdwa v4, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
+; GFX9-NEXT:    v_add_u32_sdwa v4, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT:    v_add_u32_e32 v4, v4, v2
+; GFX9-NEXT:    v_add_u32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT:    v_add_u32_e32 v2, v2, v3
 ; GFX9-NEXT:    v_add_u32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT:    v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
-; GFX9-NEXT:    v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT:    v_add_u32_e32 v2, v2, v5
-; GFX9-NEXT:    v_add_u32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT:    v_add_u32_e32 v2, v2, v6
-; GFX9-NEXT:    v_add_u32_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
 ; GFX9-NEXT:    global_store_short v[0:1], v2, off
 ; GFX9-NEXT:    s_endpgm
 ;
@@ -1791,64 +1775,64 @@ define amdgpu_kernel void @idot8_acc16_vecMul(<8 x i4> addrspace(1)* %src1,
 ; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
+; GFX9-DL-NEXT:    s_load_dword s6, s[6:7], 0x0
+; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-DL-NEXT:    s_and_b32 s4, s2, 15
+; GFX9-DL-NEXT:    s_bfe_u32 s5, s2, 0x40004
+; GFX9-DL-NEXT:    s_bfe_u32 s8, s2, 0x40008
+; GFX9-DL-NEXT:    s_bfe_u32 s9, s2, 0x4000c
+; GFX9-DL-NEXT:    s_bfe_u32 s10, s2, 0x40010
+; GFX9-DL-NEXT:    s_bfe_u32 s11, s2, 0x40018
+; GFX9-DL-NEXT:    s_lshr_b32 s12, s2, 28
+; GFX9-DL-NEXT:    s_bfe_u32 s2, s2, 0x40014
+; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s2, s10, s2
+; GFX9-DL-NEXT:    v_pk_lshlrev_b16 v2, 12, s2 op_sel_hi:[0,1]
+; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s2, s11, s12
+; GFX9-DL-NEXT:    s_bfe_u32 s7, s6, 0x40018
+; GFX9-DL-NEXT:    s_lshr_b32 s13, s6, 28
+; GFX9-DL-NEXT:    s_bfe_u32 s14, s6, 0x40010
+; GFX9-DL-NEXT:    s_bfe_u32 s15, s6, 0x40014
+; GFX9-DL-NEXT:    s_bfe_u32 s16, s6, 0x40008
+; GFX9-DL-NEXT:    s_bfe_u32 s17, s6, 0x4000c
+; GFX9-DL-NEXT:    s_and_b32 s18, s6, 15
+; GFX9-DL-NEXT:    s_bfe_u32 s6, s6, 0x40004
+; GFX9-DL-NEXT:    v_pk_lshlrev_b16 v3, 12, s2 op_sel_hi:[0,1]
+; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s4, s4, s5
+; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s2, s18, s6
+; GFX9-DL-NEXT:    v_pk_lshlrev_b16 v0, 12, s4 op_sel_hi:[0,1]
+; GFX9-DL-NEXT:    v_pk_lshlrev_b16 v4, 12, s2 op_sel_hi:[0,1]
+; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s4, s8, s9
+; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s2, s16, s17
+; GFX9-DL-NEXT:    v_pk_lshlrev_b16 v5, 12, s2 op_sel_hi:[0,1]
+; GFX9-DL-NEXT:    v_pk_lshlrev_b16 v1, 12, s4 op_sel_hi:[0,1]
+; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s2, s14, s15
+; GFX9-DL-NEXT:    v_pk_ashrrev_i16 v0, 12, v0 op_sel_hi:[0,1]
+; GFX9-DL-NEXT:    v_pk_ashrrev_i16 v4, 12, v4 op_sel_hi:[0,1]
+; GFX9-DL-NEXT:    v_pk_ashrrev_i16 v1, 12, v1 op_sel_hi:[0,1]
+; GFX9-DL-NEXT:    v_pk_ashrrev_i16 v5, 12, v5 op_sel_hi:[0,1]
+; GFX9-DL-NEXT:    v_pk_lshlrev_b16 v6, 12, s2 op_sel_hi:[0,1]
+; GFX9-DL-NEXT:    v_pk_mul_lo_u16 v5, v1, v5
+; GFX9-DL-NEXT:    v_pk_mul_lo_u16 v4, v0, v4
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-DL-NEXT:    v_pk_ashrrev_i16 v2, 12, v2 op_sel_hi:[0,1]
+; GFX9-DL-NEXT:    v_pk_ashrrev_i16 v6, 12, v6 op_sel_hi:[0,1]
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
-; GFX9-DL-NEXT:    global_load_ushort v2, v[0:1], off
-; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    s_and_b32 s0, s2, 15
-; GFX9-DL-NEXT:    s_bfe_u32 s1, s2, 0x40004
-; GFX9-DL-NEXT:    s_and_b32 s5, s4, 15
-; GFX9-DL-NEXT:    s_bfe_u32 s6, s4, 0x40004
-; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s0, s0, s1
-; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s1, s5, s6
-; GFX9-DL-NEXT:    s_bfe_u32 s5, s2, 0x40008
-; GFX9-DL-NEXT:    s_bfe_u32 s6, s2, 0x4000c
-; GFX9-DL-NEXT:    v_pk_lshlrev_b16 v3, 12, s0 op_sel_hi:[0,1]
-; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s0, s5, s6
-; GFX9-DL-NEXT:    s_bfe_u32 s9, s2, 0x40010
-; GFX9-DL-NEXT:    s_bfe_u32 s10, s2, 0x40014
-; GFX9-DL-NEXT:    v_pk_lshlrev_b16 v4, 12, s0 op_sel_hi:[0,1]
-; GFX9-DL-NEXT:    s_bfe_u32 s13, s2, 0x40018
-; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s0, s9, s10
-; GFX9-DL-NEXT:    s_lshr_b32 s2, s2, 28
-; GFX9-DL-NEXT:    v_pk_lshlrev_b16 v5, 12, s0 op_sel_hi:[0,1]
-; GFX9-DL-NEXT:    v_pk_lshlrev_b16 v7, 12, s1 op_sel_hi:[0,1]
-; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s0, s13, s2
-; GFX9-DL-NEXT:    s_bfe_u32 s7, s4, 0x40008
-; GFX9-DL-NEXT:    s_bfe_u32 s8, s4, 0x4000c
-; GFX9-DL-NEXT:    v_pk_lshlrev_b16 v6, 12, s0 op_sel_hi:[0,1]
-; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s0, s7, s8
+; GFX9-DL-NEXT:    v_pk_mul_lo_u16 v2, v2, v6
+; GFX9-DL-NEXT:    global_load_ushort v6, v[0:1], off
+; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s2, s7, s13
+; GFX9-DL-NEXT:    v_pk_lshlrev_b16 v7, 12, s2 op_sel_hi:[0,1]
 ; GFX9-DL-NEXT:    v_pk_ashrrev_i16 v3, 12, v3 op_sel_hi:[0,1]
 ; GFX9-DL-NEXT:    v_pk_ashrrev_i16 v7, 12, v7 op_sel_hi:[0,1]
-; GFX9-DL-NEXT:    v_pk_lshlrev_b16 v8, 12, s0 op_sel_hi:[0,1]
 ; GFX9-DL-NEXT:    v_pk_mul_lo_u16 v3, v3, v7
-; GFX9-DL-NEXT:    s_bfe_u32 s11, s4, 0x40010
-; GFX9-DL-NEXT:    s_bfe_u32 s12, s4, 0x40014
-; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s0, s11, s12
-; GFX9-DL-NEXT:    v_pk_ashrrev_i16 v4, 12, v4 op_sel_hi:[0,1]
-; GFX9-DL-NEXT:    v_pk_ashrrev_i16 v8, 12, v8 op_sel_hi:[0,1]
-; GFX9-DL-NEXT:    v_pk_lshlrev_b16 v9, 12, s0 op_sel_hi:[0,1]
-; GFX9-DL-NEXT:    s_bfe_u32 s14, s4, 0x40018
-; GFX9-DL-NEXT:    s_lshr_b32 s4, s4, 28
-; GFX9-DL-NEXT:    v_pk_mul_lo_u16 v4, v4, v8
-; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s0, s14, s4
-; GFX9-DL-NEXT:    v_pk_ashrrev_i16 v5, 12, v5 op_sel_hi:[0,1]
-; GFX9-DL-NEXT:    v_pk_ashrrev_i16 v9, 12, v9 op_sel_hi:[0,1]
-; GFX9-DL-NEXT:    v_pk_lshlrev_b16 v10, 12, s0 op_sel_hi:[0,1]
-; GFX9-DL-NEXT:    v_pk_mul_lo_u16 v5, v5, v9
-; GFX9-DL-NEXT:    v_pk_ashrrev_i16 v6, 12, v6 op_sel_hi:[0,1]
-; GFX9-DL-NEXT:    v_pk_ashrrev_i16 v10, 12, v10 op_sel_hi:[0,1]
-; GFX9-DL-NEXT:    v_pk_mul_lo_u16 v6, v6, v10
 ; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-DL-NEXT:    v_add_u32_e32 v2, v3, v2
+; GFX9-DL-NEXT:    v_add_u32_e32 v6, v4, v6
+; GFX9-DL-NEXT:    v_add_u32_sdwa v4, v6, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-DL-NEXT:    v_add_u32_sdwa v4, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
+; GFX9-DL-NEXT:    v_add_u32_sdwa v4, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-DL-NEXT:    v_add_u32_e32 v4, v4, v2
+; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-DL-NEXT:    v_add_u32_e32 v2, v2, v3
 ; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
-; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-DL-NEXT:    v_add_u32_e32 v2, v2, v5
-; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-DL-NEXT:    v_add_u32_e32 v2, v2, v6
-; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
 ; GFX9-DL-NEXT:    global_store_short v[0:1], v2, off
 ; GFX9-DL-NEXT:    s_endpgm
 ;
@@ -1858,65 +1842,65 @@ define amdgpu_kernel void @idot8_acc16_vecMul(<8 x i4> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX10-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
+; GFX10-DL-NEXT:    s_load_dword s1, s[6:7], 0x0
 ; GFX10-DL-NEXT:    global_load_ushort v2, v[0:1], off
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_and_b32 s0, s2, 15
-; GFX10-DL-NEXT:    s_bfe_u32 s1, s2, 0x40004
-; GFX10-DL-NEXT:    s_and_b32 s5, s4, 15
-; GFX10-DL-NEXT:    s_bfe_u32 s6, s4, 0x40004
-; GFX10-DL-NEXT:    s_bfe_u32 s7, s2, 0x40008
-; GFX10-DL-NEXT:    s_bfe_u32 s8, s2, 0x4000c
-; GFX10-DL-NEXT:    s_pack_ll_b32_b16 s0, s0, s1
-; GFX10-DL-NEXT:    s_bfe_u32 s1, s4, 0x40008
+; GFX10-DL-NEXT:    s_and_b32 s5, s0, 15
+; GFX10-DL-NEXT:    s_bfe_u32 s6, s0, 0x40004
+; GFX10-DL-NEXT:    s_and_b32 s7, s1, 15
+; GFX10-DL-NEXT:    s_bfe_u32 s8, s1, 0x40004
+; GFX10-DL-NEXT:    s_bfe_u32 s2, s0, 0x40018
+; GFX10-DL-NEXT:    s_lshr_b32 s4, s0, 28
 ; GFX10-DL-NEXT:    s_pack_ll_b32_b16 s5, s5, s6
-; GFX10-DL-NEXT:    s_bfe_u32 s6, s4, 0x4000c
+; GFX10-DL-NEXT:    s_bfe_u32 s9, s0, 0x40010
 ; GFX10-DL-NEXT:    s_pack_ll_b32_b16 s7, s7, s8
-; GFX10-DL-NEXT:    v_pk_lshlrev_b16 v3, 12, s0 op_sel_hi:[0,1]
-; GFX10-DL-NEXT:    s_bfe_u32 s0, s2, 0x40010
-; GFX10-DL-NEXT:    v_pk_lshlrev_b16 v4, 12, s5 op_sel_hi:[0,1]
-; GFX10-DL-NEXT:    s_pack_ll_b32_b16 s1, s1, s6
-; GFX10-DL-NEXT:    s_bfe_u32 s5, s2, 0x40014
+; GFX10-DL-NEXT:    s_bfe_u32 s10, s0, 0x40014
+; GFX10-DL-NEXT:    s_bfe_u32 s6, s0, 0x40008
+; GFX10-DL-NEXT:    v_pk_lshlrev_b16 v3, 12, s5 op_sel_hi:[0,1]
+; GFX10-DL-NEXT:    s_bfe_u32 s0, s0, 0x4000c
+; GFX10-DL-NEXT:    v_pk_lshlrev_b16 v4, 12, s7 op_sel_hi:[0,1]
+; GFX10-DL-NEXT:    s_bfe_u32 s8, s1, 0x40008
+; GFX10-DL-NEXT:    s_bfe_u32 s5, s1, 0x4000c
 ; GFX10-DL-NEXT:    v_pk_ashrrev_i16 v3, 12, v3 op_sel_hi:[0,1]
-; GFX10-DL-NEXT:    s_bfe_u32 s6, s4, 0x40010
+; GFX10-DL-NEXT:    s_pack_ll_b32_b16 s0, s6, s0
 ; GFX10-DL-NEXT:    v_pk_ashrrev_i16 v4, 12, v4 op_sel_hi:[0,1]
-; GFX10-DL-NEXT:    s_bfe_u32 s8, s4, 0x40014
-; GFX10-DL-NEXT:    v_pk_lshlrev_b16 v6, 12, s1 op_sel_hi:[0,1]
-; GFX10-DL-NEXT:    v_pk_lshlrev_b16 v5, 12, s7 op_sel_hi:[0,1]
-; GFX10-DL-NEXT:    s_pack_ll_b32_b16 s0, s0, s5
+; GFX10-DL-NEXT:    s_bfe_u32 s6, s1, 0x40010
+; GFX10-DL-NEXT:    s_pack_ll_b32_b16 s5, s8, s5
+; GFX10-DL-NEXT:    s_bfe_u32 s7, s1, 0x40018
+; GFX10-DL-NEXT:    v_pk_lshlrev_b16 v5, 12, s0 op_sel_hi:[0,1]
+; GFX10-DL-NEXT:    s_bfe_u32 s0, s1, 0x40014
 ; GFX10-DL-NEXT:    v_pk_mul_lo_u16 v3, v3, v4
-; GFX10-DL-NEXT:    s_pack_ll_b32_b16 s1, s6, s8
-; GFX10-DL-NEXT:    s_bfe_u32 s5, s2, 0x40018
+; GFX10-DL-NEXT:    v_pk_lshlrev_b16 v6, 12, s5 op_sel_hi:[0,1]
+; GFX10-DL-NEXT:    s_pack_ll_b32_b16 s5, s9, s10
 ; GFX10-DL-NEXT:    v_pk_ashrrev_i16 v4, 12, v5 op_sel_hi:[0,1]
+; GFX10-DL-NEXT:    s_pack_ll_b32_b16 s0, s6, s0
+; GFX10-DL-NEXT:    s_lshr_b32 s1, s1, 28
 ; GFX10-DL-NEXT:    v_pk_ashrrev_i16 v5, 12, v6 op_sel_hi:[0,1]
-; GFX10-DL-NEXT:    s_bfe_u32 s6, s4, 0x40018
-; GFX10-DL-NEXT:    s_lshr_b32 s2, s2, 28
-; GFX10-DL-NEXT:    v_pk_lshlrev_b16 v6, 12, s0 op_sel_hi:[0,1]
-; GFX10-DL-NEXT:    s_lshr_b32 s4, s4, 28
-; GFX10-DL-NEXT:    v_pk_lshlrev_b16 v7, 12, s1 op_sel_hi:[0,1]
-; GFX10-DL-NEXT:    v_pk_mul_lo_u16 v4, v4, v5
-; GFX10-DL-NEXT:    s_pack_ll_b32_b16 s0, s5, s2
-; GFX10-DL-NEXT:    v_pk_ashrrev_i16 v5, 12, v6 op_sel_hi:[0,1]
-; GFX10-DL-NEXT:    s_pack_ll_b32_b16 s1, s6, s4
-; GFX10-DL-NEXT:    v_pk_ashrrev_i16 v6, 12, v7 op_sel_hi:[0,1]
+; GFX10-DL-NEXT:    v_pk_lshlrev_b16 v6, 12, s5 op_sel_hi:[0,1]
 ; GFX10-DL-NEXT:    v_pk_lshlrev_b16 v7, 12, s0 op_sel_hi:[0,1]
-; GFX10-DL-NEXT:    v_pk_lshlrev_b16 v8, 12, s1 op_sel_hi:[0,1]
-; GFX10-DL-NEXT:    v_pk_mul_lo_u16 v5, v5, v6
-; GFX10-DL-NEXT:    v_pk_ashrrev_i16 v6, 12, v7 op_sel_hi:[0,1]
-; GFX10-DL-NEXT:    v_pk_ashrrev_i16 v7, 12, v8 op_sel_hi:[0,1]
-; GFX10-DL-NEXT:    v_pk_mul_lo_u16 v7, v6, v7
+; GFX10-DL-NEXT:    s_pack_ll_b32_b16 s0, s2, s4
+; GFX10-DL-NEXT:    v_pk_mul_lo_u16 v4, v4, v5
+; GFX10-DL-NEXT:    s_pack_ll_b32_b16 s1, s7, s1
+; GFX10-DL-NEXT:    v_pk_ashrrev_i16 v5, 12, v7 op_sel_hi:[0,1]
+; GFX10-DL-NEXT:    v_pk_lshlrev_b16 v7, 12, s1 op_sel_hi:[0,1]
 ; GFX10-DL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v3, v2
 ; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-DL-NEXT:    v_pk_ashrrev_i16 v3, 12, v6 op_sel_hi:[0,1]
+; GFX10-DL-NEXT:    v_pk_lshlrev_b16 v6, 12, s0 op_sel_hi:[0,1]
 ; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
+; GFX10-DL-NEXT:    v_pk_mul_lo_u16 v3, v3, v5
+; GFX10-DL-NEXT:    v_pk_ashrrev_i16 v5, 12, v7 op_sel_hi:[0,1]
+; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-DL-NEXT:    v_pk_ashrrev_i16 v4, 12, v6 op_sel_hi:[0,1]
+; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v2, v3
+; GFX10-DL-NEXT:    v_pk_mul_lo_u16 v4, v4, v5
+; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v2, v4
 ; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v2, v5
-; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v2, v7
-; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v2, v2, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
 ; GFX10-DL-NEXT:    global_store_short v[0:1], v2, off
 ; GFX10-DL-NEXT:    s_endpgm
                                               <8 x i4> addrspace(1)* %src2,
@@ -1994,7 +1978,7 @@ define amdgpu_kernel void @idot8_acc8_vecMul(<8 x i4> addrspace(1)* %src1,
 ; GFX7-NEXT:    v_mul_i32_i24_e32 v1, s2, v1
 ; GFX7-NEXT:    v_mul_i32_i24_e32 v2, s15, v2
 ; GFX7-NEXT:    v_mul_i32_i24_e32 v3, s14, v3
-; GFX7-NEXT:    v_mul_i32_i24_e32 v4, s13, v4
+; GFX7-NEXT:    v_mul_i32_i24_e32 v9, s13, v4
 ; GFX7-NEXT:    v_mul_i32_i24_e32 v5, s12, v5
 ; GFX7-NEXT:    v_mul_i32_i24_e32 v6, s11, v6
 ; GFX7-NEXT:    v_mul_i32_i24_e32 v7, s10, v7
@@ -2002,36 +1986,36 @@ define amdgpu_kernel void @idot8_acc8_vecMul(<8 x i4> addrspace(1)* %src1,
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
 ; GFX7-NEXT:    v_and_b32_e32 v2, s0, v2
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v3, 8, v3
-; GFX7-NEXT:    v_and_b32_e32 v4, s0, v4
+; GFX7-NEXT:    v_and_b32_e32 v9, s0, v9
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v5, 8, v5
 ; GFX7-NEXT:    v_and_b32_e32 v6, s0, v6
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v7, 8, v7
 ; GFX7-NEXT:    v_and_b32_e32 v8, s0, v8
 ; GFX7-NEXT:    v_or_b32_e32 v1, v2, v1
-; GFX7-NEXT:    v_or_b32_e32 v2, v4, v3
+; GFX7-NEXT:    v_or_b32_e32 v2, v9, v3
 ; GFX7-NEXT:    v_or_b32_e32 v3, v6, v5
-; GFX7-NEXT:    v_or_b32_e32 v4, v8, v7
+; GFX7-NEXT:    v_or_b32_e32 v5, v8, v7
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX7-NEXT:    v_and_b32_e32 v2, s1, v2
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX7-NEXT:    v_and_b32_e32 v4, s1, v4
+; GFX7-NEXT:    v_and_b32_e32 v5, s1, v5
 ; GFX7-NEXT:    v_or_b32_e32 v1, v2, v1
-; GFX7-NEXT:    v_or_b32_e32 v2, v4, v3
+; GFX7-NEXT:    v_or_b32_e32 v2, v5, v3
 ; GFX7-NEXT:    v_alignbit_b32 v3, v1, v2, 8
-; GFX7-NEXT:    v_alignbit_b32 v4, v1, v2, 16
-; GFX7-NEXT:    v_lshrrev_b32_e32 v5, 24, v2
-; GFX7-NEXT:    v_lshrrev_b32_e32 v6, 8, v1
-; GFX7-NEXT:    v_lshrrev_b32_e32 v7, 16, v1
-; GFX7-NEXT:    v_lshrrev_b32_e32 v8, 24, v1
+; GFX7-NEXT:    v_alignbit_b32 v5, v1, v2, 16
+; GFX7-NEXT:    v_lshrrev_b32_e32 v6, 24, v2
+; GFX7-NEXT:    v_lshrrev_b32_e32 v7, 8, v1
+; GFX7-NEXT:    v_lshrrev_b32_e32 v8, 16, v1
+; GFX7-NEXT:    v_lshrrev_b32_e32 v1, 24, v1
 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
 ; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v3, v0
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v4, v0
 ; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v5, v0
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
 ; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v6, v0
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v7, v0
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v8, v0
+; GFX7-NEXT:    v_mad_i32_i24 v0, s13, v4, v0
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v7
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v8
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
 ; GFX7-NEXT:    buffer_store_byte v0, off, s[4:7], 0
 ; GFX7-NEXT:    s_endpgm
 ;
@@ -2040,83 +2024,86 @@ define amdgpu_kernel void @idot8_acc8_vecMul(<8 x i4> addrspace(1)* %src1,
 ; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX8-NEXT:    s_load_dword s4, s[6:7], 0x0
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX8-NEXT:    flat_load_ubyte v2, v[0:1]
+; GFX8-NEXT:    s_load_dword s1, s[4:5], 0x0
+; GFX8-NEXT:    s_load_dword s2, s[6:7], 0x0
+; GFX8-NEXT:    s_mov_b32 s0, 0xffff
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_lshr_b32 s0, s2, 4
-; GFX8-NEXT:    s_lshr_b32 s1, s2, 12
-; GFX8-NEXT:    s_lshr_b32 s5, s2, 8
-; GFX8-NEXT:    s_lshr_b32 s6, s4, 4
-; GFX8-NEXT:    s_lshr_b32 s7, s4, 12
-; GFX8-NEXT:    s_lshr_b32 s8, s4, 8
-; GFX8-NEXT:    v_lshlrev_b16_e64 v3, 12, s5
-; GFX8-NEXT:    v_lshlrev_b16_e64 v4, 12, s1
-; GFX8-NEXT:    v_lshlrev_b16_e64 v5, 12, s0
+; GFX8-NEXT:    s_lshr_b32 s8, s1, 4
+; GFX8-NEXT:    s_lshr_b32 s9, s1, 12
+; GFX8-NEXT:    s_lshr_b32 s10, s1, 8
+; GFX8-NEXT:    s_lshr_b32 s15, s2, 4
+; GFX8-NEXT:    s_lshr_b32 s16, s2, 12
+; GFX8-NEXT:    s_lshr_b32 s17, s2, 8
+; GFX8-NEXT:    v_lshlrev_b16_e64 v3, 12, s1
+; GFX8-NEXT:    v_lshlrev_b16_e64 v4, 12, s2
+; GFX8-NEXT:    v_lshlrev_b16_e64 v5, 12, s10
+; GFX8-NEXT:    v_lshlrev_b16_e64 v6, 12, s9
 ; GFX8-NEXT:    v_lshlrev_b16_e64 v7, 12, s8
+; GFX8-NEXT:    v_lshlrev_b16_e64 v12, 12, s17
+; GFX8-NEXT:    v_lshlrev_b16_e64 v13, 12, s16
+; GFX8-NEXT:    v_lshlrev_b16_e64 v14, 12, s15
+; GFX8-NEXT:    s_lshr_b32 s4, s1, 20
+; GFX8-NEXT:    s_lshr_b32 s5, s1, 16
+; GFX8-NEXT:    s_lshr_b32 s6, s1, 28
+; GFX8-NEXT:    s_lshr_b32 s7, s1, 24
+; GFX8-NEXT:    s_lshr_b32 s11, s2, 20
+; GFX8-NEXT:    s_lshr_b32 s12, s2, 16
+; GFX8-NEXT:    s_lshr_b32 s13, s2, 28
+; GFX8-NEXT:    s_lshr_b32 s14, s2, 24
 ; GFX8-NEXT:    v_lshlrev_b16_e64 v8, 12, s7
 ; GFX8-NEXT:    v_lshlrev_b16_e64 v9, 12, s6
-; GFX8-NEXT:    v_lshlrev_b16_e64 v6, 12, s2
-; GFX8-NEXT:    v_lshlrev_b16_e64 v10, 12, s4
+; GFX8-NEXT:    v_lshlrev_b16_e64 v10, 12, s5
+; GFX8-NEXT:    v_lshlrev_b16_e64 v11, 12, s4
+; GFX8-NEXT:    v_lshlrev_b16_e64 v15, 12, s14
+; GFX8-NEXT:    v_lshlrev_b16_e64 v16, 12, s13
+; GFX8-NEXT:    v_lshlrev_b16_e64 v17, 12, s12
+; GFX8-NEXT:    v_lshlrev_b16_e64 v18, 12, s11
 ; GFX8-NEXT:    v_ashrrev_i16_e32 v3, 12, v3
-; GFX8-NEXT:    v_ashrrev_i16_e32 v7, 12, v7
 ; GFX8-NEXT:    v_ashrrev_i16_e32 v4, 12, v4
-; GFX8-NEXT:    v_ashrrev_i16_e32 v8, 12, v8
 ; GFX8-NEXT:    v_ashrrev_i16_e32 v5, 12, v5
-; GFX8-NEXT:    v_ashrrev_i16_e32 v9, 12, v9
 ; GFX8-NEXT:    v_ashrrev_i16_e32 v6, 12, v6
-; GFX8-NEXT:    v_ashrrev_i16_e32 v10, 12, v10
-; GFX8-NEXT:    v_mul_u32_u24_sdwa v3, v3, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT:    v_mul_u32_u24_sdwa v4, v4, v8 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT:    v_mul_u32_u24_sdwa v5, v5, v9 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT:    v_mul_u32_u24_sdwa v6, v6, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT:    s_lshr_b32 s0, s2, 20
-; GFX8-NEXT:    s_lshr_b32 s1, s2, 16
-; GFX8-NEXT:    s_lshr_b32 s5, s2, 28
-; GFX8-NEXT:    s_lshr_b32 s2, s2, 24
-; GFX8-NEXT:    s_lshr_b32 s6, s4, 20
-; GFX8-NEXT:    s_lshr_b32 s7, s4, 16
-; GFX8-NEXT:    s_lshr_b32 s8, s4, 28
-; GFX8-NEXT:    s_lshr_b32 s4, s4, 24
-; GFX8-NEXT:    v_or_b32_sdwa v5, v6, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT:    v_or_b32_sdwa v3, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT:    v_or_b32_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT:    v_lshlrev_b16_e64 v7, 12, s2
-; GFX8-NEXT:    v_lshlrev_b16_e64 v8, 12, s5
-; GFX8-NEXT:    v_lshlrev_b16_e64 v9, 12, s1
-; GFX8-NEXT:    v_lshlrev_b16_e64 v10, 12, s0
-; GFX8-NEXT:    v_lshlrev_b16_e64 v11, 12, s4
-; GFX8-NEXT:    v_lshlrev_b16_e64 v12, 12, s8
-; GFX8-NEXT:    v_lshlrev_b16_e64 v13, 12, s7
-; GFX8-NEXT:    v_lshlrev_b16_e64 v14, 12, s6
 ; GFX8-NEXT:    v_ashrrev_i16_e32 v7, 12, v7
-; GFX8-NEXT:    v_ashrrev_i16_e32 v11, 12, v11
-; GFX8-NEXT:    v_ashrrev_i16_e32 v8, 12, v8
 ; GFX8-NEXT:    v_ashrrev_i16_e32 v12, 12, v12
-; GFX8-NEXT:    v_ashrrev_i16_e32 v9, 12, v9
 ; GFX8-NEXT:    v_ashrrev_i16_e32 v13, 12, v13
-; GFX8-NEXT:    v_ashrrev_i16_e32 v10, 12, v10
 ; GFX8-NEXT:    v_ashrrev_i16_e32 v14, 12, v14
-; GFX8-NEXT:    v_lshrrev_b32_e32 v5, 8, v3
-; GFX8-NEXT:    v_mul_u32_u24_sdwa v7, v7, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT:    v_mul_u32_u24_sdwa v8, v8, v12 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT:    v_mul_u32_u24_sdwa v9, v9, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT:    v_mul_u32_u24_sdwa v10, v10, v14 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-NEXT:    v_mul_u32_u24_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-NEXT:    v_mul_u32_u24_sdwa v4, v5, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-NEXT:    v_mul_u32_u24_sdwa v5, v6, v13 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-NEXT:    v_mul_u32_u24_sdwa v6, v7, v14 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-NEXT:    v_ashrrev_i16_e32 v8, 12, v8
+; GFX8-NEXT:    v_ashrrev_i16_e32 v15, 12, v15
+; GFX8-NEXT:    v_ashrrev_i16_e32 v9, 12, v9
+; GFX8-NEXT:    v_ashrrev_i16_e32 v10, 12, v10
+; GFX8-NEXT:    v_ashrrev_i16_e32 v11, 12, v11
+; GFX8-NEXT:    v_ashrrev_i16_e32 v16, 12, v16
+; GFX8-NEXT:    v_ashrrev_i16_e32 v17, 12, v17
+; GFX8-NEXT:    v_ashrrev_i16_e32 v18, 12, v18
+; GFX8-NEXT:    v_or_b32_sdwa v3, v3, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-NEXT:    v_mul_u32_u24_sdwa v7, v8, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-NEXT:    v_mul_u32_u24_sdwa v8, v9, v16 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-NEXT:    v_mul_u32_u24_sdwa v9, v10, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-NEXT:    v_mul_u32_u24_sdwa v10, v11, v18 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-NEXT:    v_and_b32_e32 v3, s0, v3
+; GFX8-NEXT:    v_or_b32_sdwa v4, v4, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX8-NEXT:    v_or_b32_sdwa v9, v9, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX8-NEXT:    v_or_b32_sdwa v7, v7, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT:    v_or_b32_sdwa v4, v9, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT:    v_lshrrev_b32_e32 v6, 8, v4
+; GFX8-NEXT:    v_and_b32_e32 v5, s0, v9
+; GFX8-NEXT:    v_or_b32_e32 v4, v3, v4
+; GFX8-NEXT:    v_or_b32_e32 v6, v5, v7
+; GFX8-NEXT:    v_lshrrev_b32_e32 v7, 8, v4
+; GFX8-NEXT:    v_lshrrev_b32_e32 v8, 8, v6
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v2, v3
-; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v5, v2
-; GFX8-NEXT:    v_add_u32_sdwa v2, vcc, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_2
-; GFX8-NEXT:    v_add_u32_sdwa v2, vcc, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
-; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v2, v4
-; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v2, v6
-; GFX8-NEXT:    v_add_u32_sdwa v2, vcc, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT:    v_add_u32_sdwa v2, vcc, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v7, v2
+; GFX8-NEXT:    v_add_u32_sdwa v2, vcc, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_0
+; GFX8-NEXT:    v_add_u32_sdwa v2, vcc, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v2, v5
+; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v8, v2
+; GFX8-NEXT:    v_add_u32_sdwa v2, vcc, v6, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT:    v_add_u32_sdwa v2, vcc, v6, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
 ; GFX8-NEXT:    flat_store_byte v[0:1], v2
 ; GFX8-NEXT:    s_endpgm
 ;
@@ -2124,84 +2111,87 @@ define amdgpu_kernel void @idot8_acc8_vecMul(<8 x i4> addrspace(1)* %src1,
 ; GFX9:       ; %bb.0: ; %entry
 ; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX9-NEXT:    s_mov_b32 s2, 0xffff
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX9-NEXT:    s_load_dword s4, s[6:7], 0x0
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-NEXT:    global_load_ubyte v2, v[0:1], off
+; GFX9-NEXT:    s_load_dword s0, s[4:5], 0x0
+; GFX9-NEXT:    s_load_dword s1, s[6:7], 0x0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_lshr_b32 s0, s2, 4
-; GFX9-NEXT:    s_lshr_b32 s1, s2, 12
-; GFX9-NEXT:    s_lshr_b32 s5, s2, 8
-; GFX9-NEXT:    s_lshr_b32 s6, s4, 4
-; GFX9-NEXT:    s_lshr_b32 s7, s4, 12
-; GFX9-NEXT:    s_lshr_b32 s8, s4, 8
-; GFX9-NEXT:    v_lshlrev_b16_e64 v3, 12, s5
+; GFX9-NEXT:    s_lshr_b32 s8, s0, 4
+; GFX9-NEXT:    s_lshr_b32 s15, s1, 4
+; GFX9-NEXT:    v_lshlrev_b16_e64 v3, 12, s0
 ; GFX9-NEXT:    v_lshlrev_b16_e64 v4, 12, s1
-; GFX9-NEXT:    v_lshlrev_b16_e64 v5, 12, s0
 ; GFX9-NEXT:    v_lshlrev_b16_e64 v7, 12, s8
-; GFX9-NEXT:    v_lshlrev_b16_e64 v8, 12, s7
-; GFX9-NEXT:    v_lshlrev_b16_e64 v9, 12, s6
-; GFX9-NEXT:    v_lshlrev_b16_e64 v6, 12, s2
-; GFX9-NEXT:    v_lshlrev_b16_e64 v10, 12, s4
+; GFX9-NEXT:    v_lshlrev_b16_e64 v14, 12, s15
+; GFX9-NEXT:    s_lshr_b32 s9, s0, 12
+; GFX9-NEXT:    s_lshr_b32 s10, s0, 8
+; GFX9-NEXT:    s_lshr_b32 s16, s1, 12
+; GFX9-NEXT:    s_lshr_b32 s17, s1, 8
+; GFX9-NEXT:    v_lshlrev_b16_e64 v5, 12, s10
+; GFX9-NEXT:    v_lshlrev_b16_e64 v6, 12, s9
+; GFX9-NEXT:    v_lshlrev_b16_e64 v12, 12, s17
+; GFX9-NEXT:    v_lshlrev_b16_e64 v13, 12, s16
 ; GFX9-NEXT:    v_ashrrev_i16_e32 v3, 12, v3
-; GFX9-NEXT:    v_ashrrev_i16_e32 v7, 12, v7
 ; GFX9-NEXT:    v_ashrrev_i16_e32 v4, 12, v4
-; GFX9-NEXT:    v_ashrrev_i16_e32 v8, 12, v8
+; GFX9-NEXT:    v_ashrrev_i16_e32 v7, 12, v7
+; GFX9-NEXT:    v_ashrrev_i16_e32 v14, 12, v14
 ; GFX9-NEXT:    v_ashrrev_i16_e32 v5, 12, v5
-; GFX9-NEXT:    v_ashrrev_i16_e32 v9, 12, v9
+; GFX9-NEXT:    v_ashrrev_i16_e32 v12, 12, v12
 ; GFX9-NEXT:    v_ashrrev_i16_e32 v6, 12, v6
-; GFX9-NEXT:    v_ashrrev_i16_e32 v10, 12, v10
-; GFX9-NEXT:    v_mul_lo_u16_e32 v6, v6, v10
-; GFX9-NEXT:    v_mul_lo_u16_sdwa v5, v5, v9 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    v_mul_lo_u16_sdwa v4, v4, v8 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    v_mul_lo_u16_e32 v3, v3, v7
-; GFX9-NEXT:    s_lshr_b32 s0, s2, 20
-; GFX9-NEXT:    s_lshr_b32 s1, s2, 16
-; GFX9-NEXT:    s_lshr_b32 s5, s2, 28
-; GFX9-NEXT:    s_lshr_b32 s2, s2, 24
-; GFX9-NEXT:    s_lshr_b32 s6, s4, 20
-; GFX9-NEXT:    s_lshr_b32 s7, s4, 16
-; GFX9-NEXT:    s_lshr_b32 s8, s4, 28
-; GFX9-NEXT:    s_lshr_b32 s4, s4, 24
-; GFX9-NEXT:    v_or_b32_sdwa v5, v6, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v3, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    v_lshlrev_b16_e64 v9, 12, s2
-; GFX9-NEXT:    v_lshlrev_b16_e64 v10, 12, s5
-; GFX9-NEXT:    v_lshlrev_b16_e64 v11, 12, s1
-; GFX9-NEXT:    v_lshlrev_b16_e64 v12, 12, s0
-; GFX9-NEXT:    v_lshlrev_b16_e64 v13, 12, s4
-; GFX9-NEXT:    v_lshlrev_b16_e64 v14, 12, s8
-; GFX9-NEXT:    v_lshlrev_b16_e64 v15, 12, s7
-; GFX9-NEXT:    v_lshlrev_b16_e64 v16, 12, s6
-; GFX9-NEXT:    v_ashrrev_i16_e32 v9, 12, v9
 ; GFX9-NEXT:    v_ashrrev_i16_e32 v13, 12, v13
+; GFX9-NEXT:    v_mul_lo_u16_e32 v3, v3, v4
+; GFX9-NEXT:    v_mul_lo_u16_sdwa v7, v7, v14 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v3, v3, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    s_lshr_b32 s4, s0, 20
+; GFX9-NEXT:    s_lshr_b32 s5, s0, 16
+; GFX9-NEXT:    s_lshr_b32 s11, s1, 20
+; GFX9-NEXT:    s_lshr_b32 s12, s1, 16
+; GFX9-NEXT:    v_mul_lo_u16_sdwa v6, v6, v13 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT:    v_mul_lo_u16_e32 v5, v5, v12
+; GFX9-NEXT:    v_lshlrev_b16_e64 v10, 12, s5
+; GFX9-NEXT:    v_lshlrev_b16_e64 v11, 12, s4
+; GFX9-NEXT:    v_lshlrev_b16_e64 v17, 12, s12
+; GFX9-NEXT:    v_lshlrev_b16_e64 v18, 12, s11
+; GFX9-NEXT:    s_lshr_b32 s6, s0, 28
+; GFX9-NEXT:    s_lshr_b32 s7, s0, 24
+; GFX9-NEXT:    s_lshr_b32 s13, s1, 28
+; GFX9-NEXT:    s_lshr_b32 s14, s1, 24
+; GFX9-NEXT:    v_and_b32_e32 v3, s2, v3
+; GFX9-NEXT:    v_or_b32_sdwa v5, v5, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_lshlrev_b16_e64 v8, 12, s7
+; GFX9-NEXT:    v_lshlrev_b16_e64 v9, 12, s6
+; GFX9-NEXT:    v_lshlrev_b16_e64 v15, 12, s14
+; GFX9-NEXT:    v_lshlrev_b16_e64 v16, 12, s13
+; GFX9-NEXT:    v_or_b32_e32 v5, v3, v5
 ; GFX9-NEXT:    v_ashrrev_i16_e32 v10, 12, v10
-; GFX9-NEXT:    v_ashrrev_i16_e32 v14, 12, v14
+; GFX9-NEXT:    v_ashrrev_i16_e32 v17, 12, v17
 ; GFX9-NEXT:    v_ashrrev_i16_e32 v11, 12, v11
+; GFX9-NEXT:    v_ashrrev_i16_e32 v18, 12, v18
+; GFX9-NEXT:    v_ashrrev_i16_e32 v8, 12, v8
 ; GFX9-NEXT:    v_ashrrev_i16_e32 v15, 12, v15
-; GFX9-NEXT:    v_ashrrev_i16_e32 v12, 12, v12
+; GFX9-NEXT:    v_ashrrev_i16_e32 v9, 12, v9
 ; GFX9-NEXT:    v_ashrrev_i16_e32 v16, 12, v16
-; GFX9-NEXT:    v_lshrrev_b32_e32 v5, 8, v3
-; GFX9-NEXT:    v_mul_lo_u16_sdwa v12, v12, v16 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    v_mul_lo_u16_e32 v11, v11, v15
-; GFX9-NEXT:    v_mul_lo_u16_sdwa v10, v10, v14 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    v_mul_lo_u16_e32 v9, v9, v13
-; GFX9-NEXT:    v_or_b32_sdwa v7, v11, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v8, v9, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v4, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX9-NEXT:    v_mul_lo_u16_sdwa v4, v11, v18 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT:    v_mul_lo_u16_e32 v10, v10, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v7, 8, v5
+; GFX9-NEXT:    v_or_b32_sdwa v4, v10, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_mul_lo_u16_sdwa v9, v9, v16 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT:    v_mul_lo_u16_e32 v8, v8, v15
+; GFX9-NEXT:    v_or_b32_sdwa v8, v8, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_and_b32_e32 v4, s2, v4
+; GFX9-NEXT:    v_or_b32_e32 v6, v4, v8
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u32_e32 v2, v3, v2
-; GFX9-NEXT:    v_add_u32_e32 v2, v2, v5
-; GFX9-NEXT:    v_add_u32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_2
-; GFX9-NEXT:    v_add_u32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+; GFX9-NEXT:    v_add_u32_e32 v2, v2, v7
+; GFX9-NEXT:    v_add_u32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_2
+; GFX9-NEXT:    v_add_u32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
 ; GFX9-NEXT:    v_add_u32_e32 v2, v2, v4
-; GFX9-NEXT:    v_lshrrev_b32_e32 v3, 8, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v3, 8, v6
 ; GFX9-NEXT:    v_add_u32_e32 v2, v2, v3
-; GFX9-NEXT:    v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT:    v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+; GFX9-NEXT:    v_add_u32_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT:    v_add_u32_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
 ; GFX9-NEXT:    global_store_byte v[0:1], v2, off
 ; GFX9-NEXT:    s_endpgm
 ;
@@ -2209,84 +2199,87 @@ define amdgpu_kernel void @idot8_acc8_vecMul(<8 x i4> addrspace(1)* %src1,
 ; GFX9-DL:       ; %bb.0: ; %entry
 ; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX9-DL-NEXT:    s_mov_b32 s2, 0xffff
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-DL-NEXT:    global_load_ubyte v2, v[0:1], off
+; GFX9-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
+; GFX9-DL-NEXT:    s_load_dword s1, s[6:7], 0x0
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    s_lshr_b32 s0, s2, 4
-; GFX9-DL-NEXT:    s_lshr_b32 s1, s2, 12
-; GFX9-DL-NEXT:    s_lshr_b32 s5, s2, 8
-; GFX9-DL-NEXT:    s_lshr_b32 s6, s4, 4
-; GFX9-DL-NEXT:    s_lshr_b32 s7, s4, 12
-; GFX9-DL-NEXT:    s_lshr_b32 s8, s4, 8
-; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v3, 12, s5
+; GFX9-DL-NEXT:    s_lshr_b32 s8, s0, 4
+; GFX9-DL-NEXT:    s_lshr_b32 s15, s1, 4
+; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v3, 12, s0
 ; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v4, 12, s1
-; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v5, 12, s0
 ; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v7, 12, s8
-; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v8, 12, s7
-; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v9, 12, s6
-; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v6, 12, s2
-; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v10, 12, s4
+; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v14, 12, s15
+; GFX9-DL-NEXT:    s_lshr_b32 s9, s0, 12
+; GFX9-DL-NEXT:    s_lshr_b32 s10, s0, 8
+; GFX9-DL-NEXT:    s_lshr_b32 s16, s1, 12
+; GFX9-DL-NEXT:    s_lshr_b32 s17, s1, 8
+; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v5, 12, s10
+; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v6, 12, s9
+; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v12, 12, s17
+; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v13, 12, s16
 ; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v3, 12, v3
-; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v7, 12, v7
 ; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v4, 12, v4
-; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v8, 12, v8
+; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v7, 12, v7
+; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v14, 12, v14
 ; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v5, 12, v5
-; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v9, 12, v9
+; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v12, 12, v12
 ; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v6, 12, v6
-; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v10, 12, v10
-; GFX9-DL-NEXT:    v_mul_lo_u16_e32 v6, v6, v10
-; GFX9-DL-NEXT:    v_mul_lo_u16_sdwa v5, v5, v9 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-DL-NEXT:    v_mul_lo_u16_sdwa v4, v4, v8 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-DL-NEXT:    v_mul_lo_u16_e32 v3, v3, v7
-; GFX9-DL-NEXT:    s_lshr_b32 s0, s2, 20
-; GFX9-DL-NEXT:    s_lshr_b32 s1, s2, 16
-; GFX9-DL-NEXT:    s_lshr_b32 s5, s2, 28
-; GFX9-DL-NEXT:    s_lshr_b32 s2, s2, 24
-; GFX9-DL-NEXT:    s_lshr_b32 s6, s4, 20
-; GFX9-DL-NEXT:    s_lshr_b32 s7, s4, 16
-; GFX9-DL-NEXT:    s_lshr_b32 s8, s4, 28
-; GFX9-DL-NEXT:    s_lshr_b32 s4, s4, 24
-; GFX9-DL-NEXT:    v_or_b32_sdwa v5, v6, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-DL-NEXT:    v_or_b32_sdwa v3, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-DL-NEXT:    v_or_b32_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v9, 12, s2
-; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v10, 12, s5
-; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v11, 12, s1
-; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v12, 12, s0
-; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v13, 12, s4
-; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v14, 12, s8
-; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v15, 12, s7
-; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v16, 12, s6
-; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v9, 12, v9
 ; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v13, 12, v13
+; GFX9-DL-NEXT:    v_mul_lo_u16_e32 v3, v3, v4
+; GFX9-DL-NEXT:    v_mul_lo_u16_sdwa v7, v7, v14 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-DL-NEXT:    v_or_b32_sdwa v3, v3, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-DL-NEXT:    s_lshr_b32 s4, s0, 20
+; GFX9-DL-NEXT:    s_lshr_b32 s5, s0, 16
+; GFX9-DL-NEXT:    s_lshr_b32 s11, s1, 20
+; GFX9-DL-NEXT:    s_lshr_b32 s12, s1, 16
+; GFX9-DL-NEXT:    v_mul_lo_u16_sdwa v6, v6, v13 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-DL-NEXT:    v_mul_lo_u16_e32 v5, v5, v12
+; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v10, 12, s5
+; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v11, 12, s4
+; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v17, 12, s12
+; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v18, 12, s11
+; GFX9-DL-NEXT:    s_lshr_b32 s6, s0, 28
+; GFX9-DL-NEXT:    s_lshr_b32 s7, s0, 24
+; GFX9-DL-NEXT:    s_lshr_b32 s13, s1, 28
+; GFX9-DL-NEXT:    s_lshr_b32 s14, s1, 24
+; GFX9-DL-NEXT:    v_and_b32_e32 v3, s2, v3
+; GFX9-DL-NEXT:    v_or_b32_sdwa v5, v5, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v8, 12, s7
+; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v9, 12, s6
+; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v15, 12, s14
+; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v16, 12, s13
+; GFX9-DL-NEXT:    v_or_b32_e32 v5, v3, v5
 ; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v10, 12, v10
-; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v14, 12, v14
+; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v17, 12, v17
 ; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v11, 12, v11
+; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v18, 12, v18
+; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v8, 12, v8
 ; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v15, 12, v15
-; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v12, 12, v12
+; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v9, 12, v9
 ; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v16, 12, v16
-; GFX9-DL-NEXT:    v_lshrrev_b32_e32 v5, 8, v3
-; GFX9-DL-NEXT:    v_mul_lo_u16_sdwa v12, v12, v16 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-DL-NEXT:    v_mul_lo_u16_e32 v11, v11, v15
-; GFX9-DL-NEXT:    v_mul_lo_u16_sdwa v10, v10, v14 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-DL-NEXT:    v_mul_lo_u16_e32 v9, v9, v13
-; GFX9-DL-NEXT:    v_or_b32_sdwa v7, v11, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-DL-NEXT:    v_or_b32_sdwa v8, v9, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-DL-NEXT:    v_or_b32_sdwa v4, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX9-DL-NEXT:    v_mul_lo_u16_sdwa v4, v11, v18 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-DL-NEXT:    v_mul_lo_u16_e32 v10, v10, v17
+; GFX9-DL-NEXT:    v_lshrrev_b32_e32 v7, 8, v5
+; GFX9-DL-NEXT:    v_or_b32_sdwa v4, v10, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-DL-NEXT:    v_mul_lo_u16_sdwa v9, v9, v16 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-DL-NEXT:    v_mul_lo_u16_e32 v8, v8, v15
+; GFX9-DL-NEXT:    v_or_b32_sdwa v8, v8, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-DL-NEXT:    v_and_b32_e32 v4, s2, v4
+; GFX9-DL-NEXT:    v_or_b32_e32 v6, v4, v8
 ; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-DL-NEXT:    v_add_u32_e32 v2, v3, v2
-; GFX9-DL-NEXT:    v_add_u32_e32 v2, v2, v5
-; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_2
-; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+; GFX9-DL-NEXT:    v_add_u32_e32 v2, v2, v7
+; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_2
+; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
 ; GFX9-DL-NEXT:    v_add_u32_e32 v2, v2, v4
-; GFX9-DL-NEXT:    v_lshrrev_b32_e32 v3, 8, v4
+; GFX9-DL-NEXT:    v_lshrrev_b32_e32 v3, 8, v6
 ; GFX9-DL-NEXT:    v_add_u32_e32 v2, v2, v3
-; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
 ; GFX9-DL-NEXT:    global_store_byte v[0:1], v2, off
 ; GFX9-DL-NEXT:    s_endpgm
 ;
@@ -2294,127 +2287,92 @@ define amdgpu_kernel void @idot8_acc8_vecMul(<8 x i4> addrspace(1)* %src1,
 ; GFX10-DL:       ; %bb.0: ; %entry
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX10-DL-NEXT:    v_mov_b32_e32 v2, 0xffff
-; GFX10-DL-NEXT:    s_movk_i32 s2, 0xff
+; GFX10-DL-NEXT:    s_mov_b32 s2, 0xffff
 ; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_load_dword s4, s[4:5], 0x0
-; GFX10-DL-NEXT:    s_load_dword s5, s[6:7], 0x0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
-; GFX10-DL-NEXT:    global_load_ubyte v3, v[0:1], off
+; GFX10-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
+; GFX10-DL-NEXT:    s_load_dword s1, s[6:7], 0x0
+; GFX10-DL-NEXT:    global_load_ubyte v2, v[0:1], off
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_lshr_b32 s0, s4, 4
-; GFX10-DL-NEXT:    s_lshr_b32 s1, s4, 8
-; GFX10-DL-NEXT:    s_lshr_b32 s6, s4, 12
-; GFX10-DL-NEXT:    s_lshr_b32 s7, s5, 4
-; GFX10-DL-NEXT:    s_lshr_b32 s8, s5, 8
-; GFX10-DL-NEXT:    s_lshr_b32 s9, s5, 12
+; GFX10-DL-NEXT:    s_lshr_b32 s8, s0, 4
+; GFX10-DL-NEXT:    s_lshr_b32 s15, s1, 4
+; GFX10-DL-NEXT:    s_lshr_b32 s9, s0, 12
+; GFX10-DL-NEXT:    s_lshr_b32 s16, s1, 12
+; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v3, 12, s0
+; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v7, 12, s8
+; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v12, 12, s15
 ; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v4, 12, s1
-; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v5, 12, s6
-; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v15, 12, s0
-; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v9, 12, s8
-; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v8, 12, s9
-; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v7, 12, s4
-; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v19, 12, s7
-; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v11, 12, s5
-; GFX10-DL-NEXT:    v_and_b32_e32 v5, v5, v2
-; GFX10-DL-NEXT:    v_and_b32_e32 v4, v4, v2
-; GFX10-DL-NEXT:    v_and_b32_e32 v6, v15, v2
-; GFX10-DL-NEXT:    v_and_b32_e32 v7, v7, v2
-; GFX10-DL-NEXT:    v_and_b32_e32 v8, v8, v2
-; GFX10-DL-NEXT:    v_and_b32_e32 v9, v9, v2
-; GFX10-DL-NEXT:    v_and_b32_e32 v10, v19, v2
-; GFX10-DL-NEXT:    v_and_b32_e32 v11, v11, v2
-; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v5, 12, v5
-; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v4, 12, v4
-; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v8, 12, v8
-; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v9, 12, v9
-; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v19, 12, v10
-; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v11, 12, v11
-; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v15, 12, v6
+; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v14, 12, s16
+; GFX10-DL-NEXT:    s_lshr_b32 s10, s0, 8
+; GFX10-DL-NEXT:    s_lshr_b32 s17, s1, 8
+; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v6, 12, s9
 ; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v7, 12, v7
-; GFX10-DL-NEXT:    s_lshr_b32 s1, s4, 20
-; GFX10-DL-NEXT:    s_lshr_b32 s6, s4, 24
-; GFX10-DL-NEXT:    s_lshr_b32 s0, s4, 16
-; GFX10-DL-NEXT:    s_lshr_b32 s4, s4, 28
-; GFX10-DL-NEXT:    s_lshr_b32 s8, s5, 20
-; GFX10-DL-NEXT:    s_lshr_b32 s9, s5, 24
-; GFX10-DL-NEXT:    s_lshr_b32 s7, s5, 16
-; GFX10-DL-NEXT:    s_lshr_b32 s5, s5, 28
-; GFX10-DL-NEXT:    v_and_b32_e32 v23, v15, v2
-; GFX10-DL-NEXT:    v_and_b32_e32 v10, v19, v2
-; GFX10-DL-NEXT:    v_and_b32_e32 v5, v5, v2
-; GFX10-DL-NEXT:    v_and_b32_e32 v8, v8, v2
-; GFX10-DL-NEXT:    v_and_b32_e32 v4, v4, v2
-; GFX10-DL-NEXT:    v_and_b32_e32 v9, v9, v2
-; GFX10-DL-NEXT:    v_and_b32_e32 v22, v7, v2
-; GFX10-DL-NEXT:    v_and_b32_e32 v11, v11, v2
-; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v5, v5, v8
-; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v12, 12, s4
-; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v4, v4, v9
-; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v13, 12, s6
-; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v7, v22, v11
-; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v15, 12, s0
-; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v20, 12, s8
-; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v23, v23, v10
-; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v21, 12, s1
-; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v16, 12, s5
-; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v17, 12, s9
-; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v19, 12, s7
-; GFX10-DL-NEXT:    v_and_b32_e32 v8, v12, v2
-; GFX10-DL-NEXT:    v_and_b32_e32 v9, v13, v2
-; GFX10-DL-NEXT:    v_and_b32_e32 v11, v15, v2
-; GFX10-DL-NEXT:    v_and_b32_e32 v12, v16, v2
-; GFX10-DL-NEXT:    v_and_b32_e32 v13, v17, v2
-; GFX10-DL-NEXT:    v_and_b32_e32 v15, v19, v2
-; GFX10-DL-NEXT:    v_and_b32_e32 v10, v21, v2
-; GFX10-DL-NEXT:    v_and_b32_e32 v14, v20, v2
-; GFX10-DL-NEXT:    v_and_b32_sdwa v6, v23, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-DL-NEXT:    v_and_b32_sdwa v7, v7, s2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX10-DL-NEXT:    v_and_b32_sdwa v4, v4, s2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX10-DL-NEXT:    v_and_b32_sdwa v5, v5, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v12, 12, v12
-; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v13, 12, v13
-; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v8, 12, v8
-; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v9, 12, v9
-; GFX10-DL-NEXT:    v_or_b32_sdwa v6, v7, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-; GFX10-DL-NEXT:    v_or_b32_sdwa v4, v4, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v19, 12, v10
-; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v15, 12, v15
-; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v11, 12, v11
-; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v23, 12, v14
-; GFX10-DL-NEXT:    v_and_b32_e32 v5, v8, v2
-; GFX10-DL-NEXT:    v_and_b32_e32 v7, v9, v2
-; GFX10-DL-NEXT:    v_and_b32_e32 v13, v13, v2
-; GFX10-DL-NEXT:    v_and_b32_e32 v9, v11, v2
-; GFX10-DL-NEXT:    v_and_b32_e32 v12, v12, v2
-; GFX10-DL-NEXT:    v_or_b32_sdwa v4, v6, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX10-DL-NEXT:    v_and_b32_e32 v10, v15, v2
-; GFX10-DL-NEXT:    v_and_b32_e32 v8, v19, v2
-; GFX10-DL-NEXT:    v_and_b32_e32 v11, v23, v2
-; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v7, v7, v13
+; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v5, 12, s10
+; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v3, 12, v3
+; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v4, 12, v4
+; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v13, 12, s17
+; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v7, v7, v12
+; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v19, 12, v6
+; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v14, 12, v14
+; GFX10-DL-NEXT:    s_lshr_b32 s4, s0, 20
+; GFX10-DL-NEXT:    s_lshr_b32 s5, s0, 16
+; GFX10-DL-NEXT:    s_lshr_b32 s6, s0, 28
+; GFX10-DL-NEXT:    s_lshr_b32 s7, s0, 24
+; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v3, v3, v4
+; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v4, v19, v14
+; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v6, 8, v7
+; GFX10-DL-NEXT:    s_lshr_b32 s11, s1, 20
+; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v12, 12, v13
+; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v5, 12, v5
+; GFX10-DL-NEXT:    s_lshr_b32 s12, s1, 16
+; GFX10-DL-NEXT:    v_or_b32_sdwa v3, v3, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-DL-NEXT:    s_lshr_b32 s13, s1, 28
+; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v8, 12, s7
+; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v9, 12, s6
+; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v10, 12, s5
+; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v11, 12, s4
+; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v13, 12, s11
 ; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v5, v5, v12
-; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v6, v9, v10
-; GFX10-DL-NEXT:    v_lshrrev_b32_e32 v9, 8, v4
+; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v4, 8, v4
+; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v7, 12, s12
+; GFX10-DL-NEXT:    s_lshr_b32 s14, s1, 24
+; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v6, 12, v8
+; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v8, 12, v9
+; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v9, 12, v10
+; GFX10-DL-NEXT:    v_or_b32_sdwa v4, v5, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-DL-NEXT:    v_and_b32_e32 v3, s2, v3
+; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v16, 12, s13
+; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v5, 12, v11
+; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v10, 12, v13
+; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v15, 12, s14
+; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v7, 12, v7
+; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v11, 12, v16
+; GFX10-DL-NEXT:    v_or_b32_e32 v4, v3, v4
+; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v5, v5, v10
+; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v12, 12, v15
+; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v10, v9, v7
 ; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v8, v8, v11
-; GFX10-DL-NEXT:    v_and_b32_sdwa v7, v7, s2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX10-DL-NEXT:    v_and_b32_sdwa v6, v6, s2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX10-DL-NEXT:    v_and_b32_sdwa v8, v8, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-DL-NEXT:    v_and_b32_sdwa v2, v5, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-DL-NEXT:    v_or_b32_sdwa v5, v6, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-; GFX10-DL-NEXT:    v_or_b32_sdwa v2, v7, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-; GFX10-DL-NEXT:    v_or_b32_sdwa v2, v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX10-DL-NEXT:    v_lshrrev_b32_e32 v9, 8, v4
 ; GFX10-DL-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-DL-NEXT:    v_add_nc_u32_e32 v3, v4, v3
-; GFX10-DL-NEXT:    v_add_nc_u32_e32 v3, v3, v9
-; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_2
-; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
-; GFX10-DL-NEXT:    v_lshrrev_b32_e32 v4, 8, v2
-; GFX10-DL-NEXT:    v_add_nc_u32_e32 v3, v3, v2
-; GFX10-DL-NEXT:    v_add_nc_u32_e32 v3, v3, v4
-; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v3, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v3, v2
+; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v3, 8, v5
+; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v5, v6, v12
+; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v6, 8, v8
+; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v2, v9
+; GFX10-DL-NEXT:    v_or_b32_sdwa v3, v10, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-DL-NEXT:    v_or_b32_sdwa v5, v5, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_2
+; GFX10-DL-NEXT:    v_and_b32_e32 v3, s2, v3
+; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+; GFX10-DL-NEXT:    v_or_b32_e32 v4, v3, v5
+; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v2, v3
+; GFX10-DL-NEXT:    v_lshrrev_b32_e32 v3, 8, v4
+; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v2, v3
+; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
 ; GFX10-DL-NEXT:    global_store_byte v[0:1], v2, off
 ; GFX10-DL-NEXT:    s_endpgm
                                              <8 x i4> addrspace(1)* %src2,
diff --git a/llvm/test/CodeGen/AMDGPU/idot8u.ll b/llvm/test/CodeGen/AMDGPU/idot8u.ll
index 43099dbfb..79f827cc2 100644
--- a/llvm/test/CodeGen/AMDGPU/idot8u.ll
+++ b/llvm/test/CodeGen/AMDGPU/idot8u.ll
@@ -15,42 +15,42 @@ define amdgpu_kernel void @udot8_acc32(<8 x i4> addrspace(1)* %src1,
 ; GFX7-NEXT:    s_mov_b32 s6, -1
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    s_load_dword s0, s[8:9], 0x0
-; GFX7-NEXT:    s_load_dword s1, s[10:11], 0x0
-; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s10, s[10:11], 0x0
+; GFX7-NEXT:    s_load_dword s21, s[4:5], 0x0
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_lshr_b32 s8, s0, 28
-; GFX7-NEXT:    s_lshr_b32 s15, s1, 28
-; GFX7-NEXT:    s_bfe_u32 s16, s1, 0x40018
-; GFX7-NEXT:    s_bfe_u32 s17, s1, 0x40014
-; GFX7-NEXT:    s_bfe_u32 s18, s1, 0x40010
-; GFX7-NEXT:    s_bfe_u32 s19, s1, 0x4000c
-; GFX7-NEXT:    s_bfe_u32 s20, s1, 0x40008
-; GFX7-NEXT:    s_bfe_u32 s21, s1, 0x40004
-; GFX7-NEXT:    s_and_b32 s1, s1, 15
-; GFX7-NEXT:    s_bfe_u32 s9, s0, 0x40018
-; GFX7-NEXT:    s_bfe_u32 s10, s0, 0x40014
-; GFX7-NEXT:    s_bfe_u32 s11, s0, 0x40010
+; GFX7-NEXT:    s_lshr_b32 s1, s0, 28
+; GFX7-NEXT:    s_lshr_b32 s11, s10, 28
+; GFX7-NEXT:    s_bfe_u32 s15, s10, 0x40018
+; GFX7-NEXT:    s_bfe_u32 s16, s10, 0x40014
+; GFX7-NEXT:    s_bfe_u32 s17, s10, 0x40010
+; GFX7-NEXT:    s_bfe_u32 s18, s10, 0x4000c
+; GFX7-NEXT:    s_bfe_u32 s19, s10, 0x40008
+; GFX7-NEXT:    s_bfe_u32 s20, s10, 0x40004
+; GFX7-NEXT:    s_and_b32 s10, s10, 15
+; GFX7-NEXT:    s_bfe_u32 s2, s0, 0x40018
+; GFX7-NEXT:    s_bfe_u32 s8, s0, 0x40014
+; GFX7-NEXT:    s_bfe_u32 s9, s0, 0x40010
 ; GFX7-NEXT:    s_bfe_u32 s12, s0, 0x4000c
 ; GFX7-NEXT:    s_bfe_u32 s13, s0, 0x40008
 ; GFX7-NEXT:    s_bfe_u32 s14, s0, 0x40004
 ; GFX7-NEXT:    s_and_b32 s0, s0, 15
-; GFX7-NEXT:    v_mov_b32_e32 v0, s1
-; GFX7-NEXT:    v_mov_b32_e32 v1, s2
-; GFX7-NEXT:    v_mad_u32_u24 v0, s0, v0, v1
+; GFX7-NEXT:    v_mov_b32_e32 v0, s10
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s21
-; GFX7-NEXT:    v_mad_u32_u24 v0, s14, v1, v0
+; GFX7-NEXT:    v_mad_u32_u24 v0, s0, v0, v1
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s20
-; GFX7-NEXT:    v_mad_u32_u24 v0, s13, v1, v0
+; GFX7-NEXT:    v_mad_u32_u24 v0, s14, v1, v0
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s19
-; GFX7-NEXT:    v_mad_u32_u24 v0, s12, v1, v0
+; GFX7-NEXT:    v_mad_u32_u24 v0, s13, v1, v0
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s18
-; GFX7-NEXT:    v_mad_u32_u24 v0, s11, v1, v0
+; GFX7-NEXT:    v_mad_u32_u24 v0, s12, v1, v0
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s17
-; GFX7-NEXT:    v_mad_u32_u24 v0, s10, v1, v0
-; GFX7-NEXT:    v_mov_b32_e32 v1, s16
 ; GFX7-NEXT:    v_mad_u32_u24 v0, s9, v1, v0
-; GFX7-NEXT:    v_mov_b32_e32 v1, s15
+; GFX7-NEXT:    v_mov_b32_e32 v1, s16
 ; GFX7-NEXT:    v_mad_u32_u24 v0, s8, v1, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s15
+; GFX7-NEXT:    v_mad_u32_u24 v0, s2, v1, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s11
+; GFX7-NEXT:    v_mad_u32_u24 v0, s1, v1, v0
 ; GFX7-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 ; GFX7-NEXT:    s_endpgm
 ;
@@ -60,44 +60,44 @@ define amdgpu_kernel void @udot8_acc32(<8 x i4> addrspace(1)* %src1,
 ; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX8-NEXT:    s_load_dword s4, s[6:7], 0x0
-; GFX8-NEXT:    s_load_dword s5, s[0:1], 0x0
-; GFX8-NEXT:    v_mov_b32_e32 v0, s0
-; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    s_load_dword s6, s[6:7], 0x0
+; GFX8-NEXT:    s_load_dword s19, s[0:1], 0x0
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_lshr_b32 s0, s2, 28
-; GFX8-NEXT:    s_lshr_b32 s11, s4, 28
-; GFX8-NEXT:    s_bfe_u32 s12, s4, 0x40018
-; GFX8-NEXT:    s_bfe_u32 s13, s4, 0x40014
-; GFX8-NEXT:    s_bfe_u32 s14, s4, 0x40010
-; GFX8-NEXT:    s_bfe_u32 s15, s4, 0x4000c
-; GFX8-NEXT:    s_bfe_u32 s16, s4, 0x40008
-; GFX8-NEXT:    s_bfe_u32 s17, s4, 0x40004
-; GFX8-NEXT:    s_and_b32 s4, s4, 15
-; GFX8-NEXT:    s_bfe_u32 s1, s2, 0x40018
-; GFX8-NEXT:    s_bfe_u32 s6, s2, 0x40014
-; GFX8-NEXT:    s_bfe_u32 s7, s2, 0x40010
-; GFX8-NEXT:    s_bfe_u32 s8, s2, 0x4000c
-; GFX8-NEXT:    s_bfe_u32 s9, s2, 0x40008
-; GFX8-NEXT:    s_bfe_u32 s10, s2, 0x40004
+; GFX8-NEXT:    s_lshr_b32 s4, s2, 28
+; GFX8-NEXT:    s_lshr_b32 s7, s6, 28
+; GFX8-NEXT:    s_bfe_u32 s13, s6, 0x40018
+; GFX8-NEXT:    s_bfe_u32 s14, s6, 0x40014
+; GFX8-NEXT:    s_bfe_u32 s15, s6, 0x40010
+; GFX8-NEXT:    s_bfe_u32 s16, s6, 0x4000c
+; GFX8-NEXT:    s_bfe_u32 s17, s6, 0x40008
+; GFX8-NEXT:    s_bfe_u32 s18, s6, 0x40004
+; GFX8-NEXT:    s_and_b32 s6, s6, 15
+; GFX8-NEXT:    s_bfe_u32 s5, s2, 0x40018
+; GFX8-NEXT:    s_bfe_u32 s8, s2, 0x40014
+; GFX8-NEXT:    s_bfe_u32 s9, s2, 0x40010
+; GFX8-NEXT:    s_bfe_u32 s10, s2, 0x4000c
+; GFX8-NEXT:    s_bfe_u32 s11, s2, 0x40008
+; GFX8-NEXT:    s_bfe_u32 s12, s2, 0x40004
 ; GFX8-NEXT:    s_and_b32 s2, s2, 15
-; GFX8-NEXT:    v_mov_b32_e32 v2, s4
-; GFX8-NEXT:    v_mov_b32_e32 v3, s5
-; GFX8-NEXT:    v_mad_u32_u24 v2, s2, v2, v3
-; GFX8-NEXT:    v_mov_b32_e32 v3, s17
-; GFX8-NEXT:    v_mad_u32_u24 v2, s10, v3, v2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s16
-; GFX8-NEXT:    v_mad_u32_u24 v2, s9, v3, v2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s15
-; GFX8-NEXT:    v_mad_u32_u24 v2, s8, v3, v2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s14
-; GFX8-NEXT:    v_mad_u32_u24 v2, s7, v3, v2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s13
-; GFX8-NEXT:    v_mad_u32_u24 v2, s6, v3, v2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s12
-; GFX8-NEXT:    v_mad_u32_u24 v2, s1, v3, v2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s11
-; GFX8-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
+; GFX8-NEXT:    v_mov_b32_e32 v0, s6
+; GFX8-NEXT:    v_mov_b32_e32 v1, s19
+; GFX8-NEXT:    v_mad_u32_u24 v0, s2, v0, v1
+; GFX8-NEXT:    v_mov_b32_e32 v1, s18
+; GFX8-NEXT:    v_mad_u32_u24 v0, s12, v1, v0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s17
+; GFX8-NEXT:    v_mad_u32_u24 v0, s11, v1, v0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s16
+; GFX8-NEXT:    v_mad_u32_u24 v0, s10, v1, v0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s15
+; GFX8-NEXT:    v_mad_u32_u24 v0, s9, v1, v0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s14
+; GFX8-NEXT:    v_mad_u32_u24 v0, s8, v1, v0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s13
+; GFX8-NEXT:    v_mad_u32_u24 v0, s5, v1, v0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s7
+; GFX8-NEXT:    v_mad_u32_u24 v2, s4, v1, v0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX8-NEXT:    flat_store_dword v[0:1], v2
 ; GFX8-NEXT:    s_endpgm
 ;
@@ -107,44 +107,44 @@ define amdgpu_kernel void @udot8_acc32(<8 x i4> addrspace(1)* %src1,
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX9-NEXT:    s_load_dword s4, s[6:7], 0x0
-; GFX9-NEXT:    s_load_dword s5, s[0:1], 0x0
-; GFX9-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    s_load_dword s6, s[6:7], 0x0
+; GFX9-NEXT:    s_load_dword s19, s[0:1], 0x0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_lshr_b32 s0, s2, 28
-; GFX9-NEXT:    s_lshr_b32 s11, s4, 28
-; GFX9-NEXT:    s_bfe_u32 s12, s4, 0x40018
-; GFX9-NEXT:    s_bfe_u32 s13, s4, 0x40014
-; GFX9-NEXT:    s_bfe_u32 s14, s4, 0x40010
-; GFX9-NEXT:    s_bfe_u32 s15, s4, 0x4000c
-; GFX9-NEXT:    s_bfe_u32 s16, s4, 0x40008
-; GFX9-NEXT:    s_bfe_u32 s17, s4, 0x40004
-; GFX9-NEXT:    s_and_b32 s4, s4, 15
-; GFX9-NEXT:    s_bfe_u32 s1, s2, 0x40018
-; GFX9-NEXT:    s_bfe_u32 s6, s2, 0x40014
-; GFX9-NEXT:    s_bfe_u32 s7, s2, 0x40010
-; GFX9-NEXT:    s_bfe_u32 s8, s2, 0x4000c
-; GFX9-NEXT:    s_bfe_u32 s9, s2, 0x40008
-; GFX9-NEXT:    s_bfe_u32 s10, s2, 0x40004
+; GFX9-NEXT:    s_lshr_b32 s4, s2, 28
+; GFX9-NEXT:    s_lshr_b32 s7, s6, 28
+; GFX9-NEXT:    s_bfe_u32 s13, s6, 0x40018
+; GFX9-NEXT:    s_bfe_u32 s14, s6, 0x40014
+; GFX9-NEXT:    s_bfe_u32 s15, s6, 0x40010
+; GFX9-NEXT:    s_bfe_u32 s16, s6, 0x4000c
+; GFX9-NEXT:    s_bfe_u32 s17, s6, 0x40008
+; GFX9-NEXT:    s_bfe_u32 s18, s6, 0x40004
+; GFX9-NEXT:    s_and_b32 s6, s6, 15
+; GFX9-NEXT:    s_bfe_u32 s5, s2, 0x40018
+; GFX9-NEXT:    s_bfe_u32 s8, s2, 0x40014
+; GFX9-NEXT:    s_bfe_u32 s9, s2, 0x40010
+; GFX9-NEXT:    s_bfe_u32 s10, s2, 0x4000c
+; GFX9-NEXT:    s_bfe_u32 s11, s2, 0x40008
+; GFX9-NEXT:    s_bfe_u32 s12, s2, 0x40004
 ; GFX9-NEXT:    s_and_b32 s2, s2, 15
-; GFX9-NEXT:    v_mov_b32_e32 v2, s4
-; GFX9-NEXT:    v_mov_b32_e32 v3, s5
-; GFX9-NEXT:    v_mad_u32_u24 v2, s2, v2, v3
-; GFX9-NEXT:    v_mov_b32_e32 v3, s17
-; GFX9-NEXT:    v_mad_u32_u24 v2, s10, v3, v2
-; GFX9-NEXT:    v_mov_b32_e32 v3, s16
-; GFX9-NEXT:    v_mad_u32_u24 v2, s9, v3, v2
-; GFX9-NEXT:    v_mov_b32_e32 v3, s15
-; GFX9-NEXT:    v_mad_u32_u24 v2, s8, v3, v2
-; GFX9-NEXT:    v_mov_b32_e32 v3, s14
-; GFX9-NEXT:    v_mad_u32_u24 v2, s7, v3, v2
-; GFX9-NEXT:    v_mov_b32_e32 v3, s13
-; GFX9-NEXT:    v_mad_u32_u24 v2, s6, v3, v2
-; GFX9-NEXT:    v_mov_b32_e32 v3, s12
-; GFX9-NEXT:    v_mad_u32_u24 v2, s1, v3, v2
-; GFX9-NEXT:    v_mov_b32_e32 v3, s11
-; GFX9-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
+; GFX9-NEXT:    v_mov_b32_e32 v0, s6
+; GFX9-NEXT:    v_mov_b32_e32 v1, s19
+; GFX9-NEXT:    v_mad_u32_u24 v0, s2, v0, v1
+; GFX9-NEXT:    v_mov_b32_e32 v1, s18
+; GFX9-NEXT:    v_mad_u32_u24 v0, s12, v1, v0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s17
+; GFX9-NEXT:    v_mad_u32_u24 v0, s11, v1, v0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s16
+; GFX9-NEXT:    v_mad_u32_u24 v0, s10, v1, v0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s15
+; GFX9-NEXT:    v_mad_u32_u24 v0, s9, v1, v0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s14
+; GFX9-NEXT:    v_mad_u32_u24 v0, s8, v1, v0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s13
+; GFX9-NEXT:    v_mad_u32_u24 v0, s5, v1, v0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s7
+; GFX9-NEXT:    v_mad_u32_u24 v2, s4, v1, v0
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-NEXT:    s_endpgm
 ;
@@ -153,15 +153,15 @@ define amdgpu_kernel void @udot8_acc32(<8 x i4> addrspace(1)* %src1,
 ; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
-; GFX9-DL-NEXT:    s_load_dword s5, s[0:1], 0x0
+; GFX9-DL-NEXT:    s_load_dword s2, s[6:7], 0x0
+; GFX9-DL-NEXT:    s_load_dword s6, s[0:1], 0x0
+; GFX9-DL-NEXT:    s_load_dword s4, s[4:5], 0x0
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s4
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s5
-; GFX9-DL-NEXT:    v_dot8_u32_u4 v2, s2, v2, v3
+; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s2
+; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s6
+; GFX9-DL-NEXT:    v_dot8_u32_u4 v2, s4, v2, v3
 ; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-DL-NEXT:    s_endpgm
 ;
@@ -171,14 +171,14 @@ define amdgpu_kernel void @udot8_acc32(<8 x i4> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX10-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
-; GFX10-DL-NEXT:    s_load_dword s5, s[0:1], 0x0
+; GFX10-DL-NEXT:    s_load_dword s2, s[0:1], 0x0
+; GFX10-DL-NEXT:    s_load_dword s4, s[4:5], 0x0
+; GFX10-DL-NEXT:    s_load_dword s5, s[6:7], 0x0
+; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-DL-NEXT:    v_dot8_u32_u4 v2, s4, s5, v0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
-; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    v_mov_b32_e32 v2, s5
-; GFX10-DL-NEXT:    v_dot8_u32_u4 v2, s2, s4, v2
 ; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX10-DL-NEXT:    s_endpgm
                                        <8 x i4> addrspace(1)* %src2,
@@ -254,17 +254,17 @@ entry:
 define amdgpu_kernel void @udot8_acc16(<8 x i4> addrspace(1)* %src1,
 ; GFX7-LABEL: udot8_acc16:
 ; GFX7:       ; %bb.0: ; %entry
-; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
-; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xd
-; GFX7-NEXT:    s_mov_b32 s11, 0xf000
-; GFX7-NEXT:    s_mov_b32 s10, -1
+; GFX7-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
+; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xd
+; GFX7-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7-NEXT:    s_mov_b32 s6, -1
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_load_dword s0, s[4:5], 0x0
-; GFX7-NEXT:    buffer_load_ushort v0, off, s[8:11], 0
-; GFX7-NEXT:    s_load_dword s1, s[6:7], 0x0
+; GFX7-NEXT:    s_load_dword s0, s[8:9], 0x0
+; GFX7-NEXT:    buffer_load_ushort v0, off, s[4:7], 0
+; GFX7-NEXT:    s_load_dword s1, s[10:11], 0x0
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    s_lshr_b32 s2, s0, 28
-; GFX7-NEXT:    s_bfe_u32 s4, s0, 0x40018
+; GFX7-NEXT:    s_bfe_u32 s8, s0, 0x40018
 ; GFX7-NEXT:    s_bfe_u32 s15, s1, 0x40018
 ; GFX7-NEXT:    s_bfe_u32 s16, s1, 0x40014
 ; GFX7-NEXT:    s_bfe_u32 s17, s1, 0x40010
@@ -273,9 +273,9 @@ define amdgpu_kernel void @udot8_acc16(<8 x i4> addrspace(1)* %src1,
 ; GFX7-NEXT:    s_bfe_u32 s20, s1, 0x40004
 ; GFX7-NEXT:    s_lshr_b32 s14, s1, 28
 ; GFX7-NEXT:    s_and_b32 s1, s1, 15
-; GFX7-NEXT:    s_bfe_u32 s5, s0, 0x40014
-; GFX7-NEXT:    s_bfe_u32 s6, s0, 0x40010
-; GFX7-NEXT:    s_bfe_u32 s7, s0, 0x4000c
+; GFX7-NEXT:    s_bfe_u32 s9, s0, 0x40014
+; GFX7-NEXT:    s_bfe_u32 s10, s0, 0x40010
+; GFX7-NEXT:    s_bfe_u32 s11, s0, 0x4000c
 ; GFX7-NEXT:    s_bfe_u32 s12, s0, 0x40008
 ; GFX7-NEXT:    s_bfe_u32 s13, s0, 0x40004
 ; GFX7-NEXT:    s_and_b32 s0, s0, 15
@@ -290,13 +290,13 @@ define amdgpu_kernel void @udot8_acc16(<8 x i4> addrspace(1)* %src1,
 ; GFX7-NEXT:    v_mad_u32_u24 v0, s0, v1, v0
 ; GFX7-NEXT:    v_mad_u32_u24 v0, s13, v2, v0
 ; GFX7-NEXT:    v_mad_u32_u24 v0, s12, v3, v0
-; GFX7-NEXT:    v_mad_u32_u24 v0, s7, v4, v0
-; GFX7-NEXT:    v_mad_u32_u24 v0, s6, v5, v0
-; GFX7-NEXT:    v_mad_u32_u24 v0, s5, v6, v0
-; GFX7-NEXT:    v_mad_u32_u24 v0, s4, v7, v0
+; GFX7-NEXT:    v_mad_u32_u24 v0, s11, v4, v0
+; GFX7-NEXT:    v_mad_u32_u24 v0, s10, v5, v0
+; GFX7-NEXT:    v_mad_u32_u24 v0, s9, v6, v0
+; GFX7-NEXT:    v_mad_u32_u24 v0, s8, v7, v0
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s14
 ; GFX7-NEXT:    v_mad_u32_u24 v0, s2, v1, v0
-; GFX7-NEXT:    buffer_store_short v0, off, s[8:11], 0
+; GFX7-NEXT:    buffer_store_short v0, off, s[4:7], 0
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: udot8_acc16:
@@ -304,46 +304,46 @@ define amdgpu_kernel void @udot8_acc16(<8 x i4> addrspace(1)* %src1,
 ; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX8-NEXT:    s_load_dword s4, s[6:7], 0x0
+; GFX8-NEXT:    s_load_dword s2, s[6:7], 0x0
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX8-NEXT:    flat_load_ushort v2, v[0:1]
+; GFX8-NEXT:    s_load_dword s0, s[4:5], 0x0
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_and_b32 s0, s2, 15
-; GFX8-NEXT:    s_and_b32 s1, s4, 15
+; GFX8-NEXT:    s_and_b32 s1, s2, 15
+; GFX8-NEXT:    s_bfe_u32 s4, s2, 0x40004
 ; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    s_bfe_u32 s5, s4, 0x40004
-; GFX8-NEXT:    v_mov_b32_e32 v4, s5
-; GFX8-NEXT:    s_bfe_u32 s1, s2, 0x40004
-; GFX8-NEXT:    s_bfe_u32 s5, s4, 0x40008
-; GFX8-NEXT:    s_bfe_u32 s8, s4, 0x40010
-; GFX8-NEXT:    s_bfe_u32 s10, s4, 0x40014
-; GFX8-NEXT:    s_bfe_u32 s12, s4, 0x40018
-; GFX8-NEXT:    s_lshr_b32 s14, s4, 28
-; GFX8-NEXT:    s_bfe_u32 s4, s4, 0x4000c
-; GFX8-NEXT:    s_bfe_u32 s6, s2, 0x40008
-; GFX8-NEXT:    v_mov_b32_e32 v5, s5
+; GFX8-NEXT:    v_mov_b32_e32 v4, s4
+; GFX8-NEXT:    s_and_b32 s1, s0, 15
+; GFX8-NEXT:    s_bfe_u32 s5, s0, 0x40004
+; GFX8-NEXT:    s_bfe_u32 s4, s2, 0x40008
 ; GFX8-NEXT:    s_bfe_u32 s7, s2, 0x4000c
-; GFX8-NEXT:    v_mov_b32_e32 v6, s4
-; GFX8-NEXT:    s_bfe_u32 s9, s2, 0x40010
+; GFX8-NEXT:    v_mov_b32_e32 v5, s4
+; GFX8-NEXT:    s_bfe_u32 s6, s0, 0x40008
+; GFX8-NEXT:    s_bfe_u32 s8, s2, 0x40010
+; GFX8-NEXT:    v_mov_b32_e32 v6, s7
+; GFX8-NEXT:    s_bfe_u32 s4, s0, 0x4000c
+; GFX8-NEXT:    s_bfe_u32 s9, s2, 0x40014
+; GFX8-NEXT:    s_bfe_u32 s7, s0, 0x40010
 ; GFX8-NEXT:    v_mov_b32_e32 v7, s8
-; GFX8-NEXT:    s_bfe_u32 s11, s2, 0x40014
-; GFX8-NEXT:    v_mov_b32_e32 v8, s10
-; GFX8-NEXT:    s_bfe_u32 s13, s2, 0x40018
-; GFX8-NEXT:    v_mov_b32_e32 v9, s12
+; GFX8-NEXT:    s_bfe_u32 s11, s2, 0x40018
+; GFX8-NEXT:    s_bfe_u32 s10, s0, 0x40014
+; GFX8-NEXT:    v_mov_b32_e32 v8, s9
+; GFX8-NEXT:    s_bfe_u32 s12, s0, 0x40018
 ; GFX8-NEXT:    s_lshr_b32 s2, s2, 28
+; GFX8-NEXT:    v_mov_b32_e32 v9, s11
+; GFX8-NEXT:    s_lshr_b32 s0, s0, 28
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
-; GFX8-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
-; GFX8-NEXT:    v_mad_u32_u24 v2, s1, v4, v2
+; GFX8-NEXT:    v_mad_u32_u24 v2, s1, v3, v2
+; GFX8-NEXT:    v_mad_u32_u24 v2, s5, v4, v2
 ; GFX8-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; GFX8-NEXT:    v_mad_u32_u24 v2, s6, v5, v2
-; GFX8-NEXT:    v_mad_u32_u24 v2, s7, v6, v2
-; GFX8-NEXT:    v_mad_u32_u24 v2, s9, v7, v2
-; GFX8-NEXT:    v_mad_u32_u24 v2, s11, v8, v2
-; GFX8-NEXT:    v_mad_u32_u24 v2, s13, v9, v2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s14
-; GFX8-NEXT:    v_mad_u32_u24 v2, s2, v3, v2
+; GFX8-NEXT:    v_mad_u32_u24 v2, s4, v6, v2
+; GFX8-NEXT:    v_mad_u32_u24 v2, s7, v7, v2
+; GFX8-NEXT:    v_mad_u32_u24 v2, s10, v8, v2
+; GFX8-NEXT:    v_mad_u32_u24 v2, s12, v9, v2
+; GFX8-NEXT:    v_mov_b32_e32 v3, s2
+; GFX8-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
 ; GFX8-NEXT:    flat_store_short v[0:1], v2
 ; GFX8-NEXT:    s_endpgm
 ;
@@ -352,46 +352,46 @@ define amdgpu_kernel void @udot8_acc16(<8 x i4> addrspace(1)* %src1,
 ; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX9-NEXT:    s_load_dword s4, s[6:7], 0x0
+; GFX9-NEXT:    s_load_dword s2, s[6:7], 0x0
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-NEXT:    global_load_ushort v2, v[0:1], off
+; GFX9-NEXT:    s_load_dword s0, s[4:5], 0x0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_and_b32 s0, s2, 15
-; GFX9-NEXT:    s_and_b32 s1, s4, 15
+; GFX9-NEXT:    s_and_b32 s1, s2, 15
+; GFX9-NEXT:    s_bfe_u32 s4, s2, 0x40004
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s1
-; GFX9-NEXT:    s_bfe_u32 s5, s4, 0x40004
-; GFX9-NEXT:    v_mov_b32_e32 v4, s5
-; GFX9-NEXT:    s_bfe_u32 s1, s2, 0x40004
-; GFX9-NEXT:    s_bfe_u32 s5, s4, 0x40008
-; GFX9-NEXT:    s_bfe_u32 s8, s4, 0x40010
-; GFX9-NEXT:    s_bfe_u32 s10, s4, 0x40014
-; GFX9-NEXT:    s_bfe_u32 s12, s4, 0x40018
-; GFX9-NEXT:    s_lshr_b32 s14, s4, 28
-; GFX9-NEXT:    s_bfe_u32 s4, s4, 0x4000c
-; GFX9-NEXT:    s_bfe_u32 s6, s2, 0x40008
-; GFX9-NEXT:    v_mov_b32_e32 v5, s5
+; GFX9-NEXT:    v_mov_b32_e32 v4, s4
+; GFX9-NEXT:    s_and_b32 s1, s0, 15
+; GFX9-NEXT:    s_bfe_u32 s5, s0, 0x40004
+; GFX9-NEXT:    s_bfe_u32 s4, s2, 0x40008
 ; GFX9-NEXT:    s_bfe_u32 s7, s2, 0x4000c
-; GFX9-NEXT:    v_mov_b32_e32 v6, s4
-; GFX9-NEXT:    s_bfe_u32 s9, s2, 0x40010
+; GFX9-NEXT:    v_mov_b32_e32 v5, s4
+; GFX9-NEXT:    s_bfe_u32 s6, s0, 0x40008
+; GFX9-NEXT:    s_bfe_u32 s8, s2, 0x40010
+; GFX9-NEXT:    v_mov_b32_e32 v6, s7
+; GFX9-NEXT:    s_bfe_u32 s4, s0, 0x4000c
+; GFX9-NEXT:    s_bfe_u32 s9, s2, 0x40014
+; GFX9-NEXT:    s_bfe_u32 s7, s0, 0x40010
 ; GFX9-NEXT:    v_mov_b32_e32 v7, s8
-; GFX9-NEXT:    s_bfe_u32 s11, s2, 0x40014
-; GFX9-NEXT:    v_mov_b32_e32 v8, s10
-; GFX9-NEXT:    s_bfe_u32 s13, s2, 0x40018
-; GFX9-NEXT:    v_mov_b32_e32 v9, s12
+; GFX9-NEXT:    s_bfe_u32 s11, s2, 0x40018
+; GFX9-NEXT:    s_bfe_u32 s10, s0, 0x40014
+; GFX9-NEXT:    v_mov_b32_e32 v8, s9
+; GFX9-NEXT:    s_bfe_u32 s12, s0, 0x40018
 ; GFX9-NEXT:    s_lshr_b32 s2, s2, 28
+; GFX9-NEXT:    v_mov_b32_e32 v9, s11
+; GFX9-NEXT:    s_lshr_b32 s0, s0, 28
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
-; GFX9-NEXT:    v_mad_u32_u24 v2, s1, v4, v2
+; GFX9-NEXT:    v_mad_u32_u24 v2, s1, v3, v2
+; GFX9-NEXT:    v_mad_u32_u24 v2, s5, v4, v2
 ; GFX9-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; GFX9-NEXT:    v_mad_u32_u24 v2, s6, v5, v2
-; GFX9-NEXT:    v_mad_u32_u24 v2, s7, v6, v2
-; GFX9-NEXT:    v_mad_u32_u24 v2, s9, v7, v2
-; GFX9-NEXT:    v_mad_u32_u24 v2, s11, v8, v2
-; GFX9-NEXT:    v_mad_u32_u24 v2, s13, v9, v2
-; GFX9-NEXT:    v_mov_b32_e32 v3, s14
-; GFX9-NEXT:    v_mad_u32_u24 v2, s2, v3, v2
+; GFX9-NEXT:    v_mad_u32_u24 v2, s4, v6, v2
+; GFX9-NEXT:    v_mad_u32_u24 v2, s7, v7, v2
+; GFX9-NEXT:    v_mad_u32_u24 v2, s10, v8, v2
+; GFX9-NEXT:    v_mad_u32_u24 v2, s12, v9, v2
+; GFX9-NEXT:    v_mov_b32_e32 v3, s2
+; GFX9-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
 ; GFX9-NEXT:    global_store_short v[0:1], v2, off
 ; GFX9-NEXT:    s_endpgm
 ;
@@ -400,46 +400,46 @@ define amdgpu_kernel void @udot8_acc16(<8 x i4> addrspace(1)* %src1,
 ; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
+; GFX9-DL-NEXT:    s_load_dword s2, s[6:7], 0x0
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-DL-NEXT:    global_load_ushort v2, v[0:1], off
+; GFX9-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    s_and_b32 s0, s2, 15
-; GFX9-DL-NEXT:    s_and_b32 s1, s4, 15
+; GFX9-DL-NEXT:    s_and_b32 s1, s2, 15
+; GFX9-DL-NEXT:    s_bfe_u32 s4, s2, 0x40004
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s1
-; GFX9-DL-NEXT:    s_bfe_u32 s5, s4, 0x40004
-; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s5
-; GFX9-DL-NEXT:    s_bfe_u32 s1, s2, 0x40004
-; GFX9-DL-NEXT:    s_bfe_u32 s5, s4, 0x40008
-; GFX9-DL-NEXT:    s_bfe_u32 s8, s4, 0x40010
-; GFX9-DL-NEXT:    s_bfe_u32 s10, s4, 0x40014
-; GFX9-DL-NEXT:    s_bfe_u32 s12, s4, 0x40018
-; GFX9-DL-NEXT:    s_lshr_b32 s14, s4, 28
-; GFX9-DL-NEXT:    s_bfe_u32 s4, s4, 0x4000c
-; GFX9-DL-NEXT:    s_bfe_u32 s6, s2, 0x40008
-; GFX9-DL-NEXT:    v_mov_b32_e32 v5, s5
+; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s4
+; GFX9-DL-NEXT:    s_and_b32 s1, s0, 15
+; GFX9-DL-NEXT:    s_bfe_u32 s5, s0, 0x40004
+; GFX9-DL-NEXT:    s_bfe_u32 s4, s2, 0x40008
 ; GFX9-DL-NEXT:    s_bfe_u32 s7, s2, 0x4000c
-; GFX9-DL-NEXT:    v_mov_b32_e32 v6, s4
-; GFX9-DL-NEXT:    s_bfe_u32 s9, s2, 0x40010
+; GFX9-DL-NEXT:    v_mov_b32_e32 v5, s4
+; GFX9-DL-NEXT:    s_bfe_u32 s6, s0, 0x40008
+; GFX9-DL-NEXT:    s_bfe_u32 s8, s2, 0x40010
+; GFX9-DL-NEXT:    v_mov_b32_e32 v6, s7
+; GFX9-DL-NEXT:    s_bfe_u32 s4, s0, 0x4000c
+; GFX9-DL-NEXT:    s_bfe_u32 s9, s2, 0x40014
+; GFX9-DL-NEXT:    s_bfe_u32 s7, s0, 0x40010
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v7, s8
-; GFX9-DL-NEXT:    s_bfe_u32 s11, s2, 0x40014
-; GFX9-DL-NEXT:    v_mov_b32_e32 v8, s10
-; GFX9-DL-NEXT:    s_bfe_u32 s13, s2, 0x40018
-; GFX9-DL-NEXT:    v_mov_b32_e32 v9, s12
+; GFX9-DL-NEXT:    s_bfe_u32 s11, s2, 0x40018
+; GFX9-DL-NEXT:    s_bfe_u32 s10, s0, 0x40014
+; GFX9-DL-NEXT:    v_mov_b32_e32 v8, s9
+; GFX9-DL-NEXT:    s_bfe_u32 s12, s0, 0x40018
 ; GFX9-DL-NEXT:    s_lshr_b32 s2, s2, 28
+; GFX9-DL-NEXT:    v_mov_b32_e32 v9, s11
+; GFX9-DL-NEXT:    s_lshr_b32 s0, s0, 28
 ; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
-; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s1, v4, v2
+; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s1, v3, v2
+; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s5, v4, v2
 ; GFX9-DL-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s6, v5, v2
-; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s7, v6, v2
-; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s9, v7, v2
-; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s11, v8, v2
-; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s13, v9, v2
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s14
-; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s2, v3, v2
+; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s4, v6, v2
+; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s7, v7, v2
+; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s10, v8, v2
+; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s12, v9, v2
+; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s2
+; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
 ; GFX9-DL-NEXT:    global_store_short v[0:1], v2, off
 ; GFX9-DL-NEXT:    s_endpgm
 ;
@@ -449,38 +449,38 @@ define amdgpu_kernel void @udot8_acc16(<8 x i4> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX10-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
+; GFX10-DL-NEXT:    s_load_dword s1, s[6:7], 0x0
 ; GFX10-DL-NEXT:    global_load_ushort v2, v[0:1], off
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_and_b32 s0, s2, 15
-; GFX10-DL-NEXT:    s_and_b32 s1, s4, 15
-; GFX10-DL-NEXT:    s_bfe_u32 s5, s2, 0x40004
-; GFX10-DL-NEXT:    s_bfe_u32 s6, s4, 0x40004
-; GFX10-DL-NEXT:    s_bfe_u32 s7, s2, 0x40008
-; GFX10-DL-NEXT:    s_bfe_u32 s8, s4, 0x40008
-; GFX10-DL-NEXT:    s_bfe_u32 s9, s2, 0x4000c
-; GFX10-DL-NEXT:    s_bfe_u32 s10, s4, 0x4000c
-; GFX10-DL-NEXT:    s_bfe_u32 s11, s2, 0x40010
-; GFX10-DL-NEXT:    s_bfe_u32 s12, s4, 0x40010
-; GFX10-DL-NEXT:    s_bfe_u32 s13, s2, 0x40014
-; GFX10-DL-NEXT:    s_bfe_u32 s14, s4, 0x40014
+; GFX10-DL-NEXT:    s_and_b32 s2, s0, 15
+; GFX10-DL-NEXT:    s_and_b32 s4, s1, 15
+; GFX10-DL-NEXT:    s_bfe_u32 s5, s0, 0x40004
+; GFX10-DL-NEXT:    s_bfe_u32 s6, s1, 0x40004
 ; GFX10-DL-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s0, s1, v2
-; GFX10-DL-NEXT:    s_bfe_u32 s0, s2, 0x40018
-; GFX10-DL-NEXT:    s_bfe_u32 s1, s4, 0x40018
-; GFX10-DL-NEXT:    s_lshr_b32 s2, s2, 28
-; GFX10-DL-NEXT:    s_lshr_b32 s4, s4, 28
+; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s2, s4, v2
+; GFX10-DL-NEXT:    s_bfe_u32 s2, s0, 0x40008
+; GFX10-DL-NEXT:    s_bfe_u32 s4, s1, 0x40008
 ; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s5, s6, v2
+; GFX10-DL-NEXT:    s_bfe_u32 s5, s0, 0x4000c
+; GFX10-DL-NEXT:    s_bfe_u32 s6, s1, 0x4000c
 ; GFX10-DL-NEXT:    v_and_b32_e32 v2, 0xffff, v2
-; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s7, s8, v2
-; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s9, s10, v2
-; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s11, s12, v2
-; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s13, s14, v2
-; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s0, s1, v2
 ; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s2, s4, v2
+; GFX10-DL-NEXT:    s_bfe_u32 s2, s0, 0x40010
+; GFX10-DL-NEXT:    s_bfe_u32 s4, s1, 0x40010
+; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s5, s6, v2
+; GFX10-DL-NEXT:    s_bfe_u32 s5, s0, 0x40014
+; GFX10-DL-NEXT:    s_bfe_u32 s6, s1, 0x40014
+; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s2, s4, v2
+; GFX10-DL-NEXT:    s_bfe_u32 s2, s0, 0x40018
+; GFX10-DL-NEXT:    s_bfe_u32 s4, s1, 0x40018
+; GFX10-DL-NEXT:    s_lshr_b32 s0, s0, 28
+; GFX10-DL-NEXT:    s_lshr_b32 s1, s1, 28
+; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s5, s6, v2
+; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s2, s4, v2
+; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s0, s1, v2
 ; GFX10-DL-NEXT:    global_store_short v[0:1], v2, off
 ; GFX10-DL-NEXT:    s_endpgm
                                        <8 x i4> addrspace(1)* %src2,
@@ -556,17 +556,17 @@ entry:
 define amdgpu_kernel void @udot8_acc8(<8 x i4> addrspace(1)* %src1,
 ; GFX7-LABEL: udot8_acc8:
 ; GFX7:       ; %bb.0: ; %entry
-; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
-; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xd
-; GFX7-NEXT:    s_mov_b32 s11, 0xf000
-; GFX7-NEXT:    s_mov_b32 s10, -1
+; GFX7-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
+; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xd
+; GFX7-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7-NEXT:    s_mov_b32 s6, -1
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_load_dword s0, s[4:5], 0x0
-; GFX7-NEXT:    buffer_load_ubyte v0, off, s[8:11], 0
-; GFX7-NEXT:    s_load_dword s1, s[6:7], 0x0
+; GFX7-NEXT:    s_load_dword s0, s[8:9], 0x0
+; GFX7-NEXT:    buffer_load_ubyte v0, off, s[4:7], 0
+; GFX7-NEXT:    s_load_dword s1, s[10:11], 0x0
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    s_lshr_b32 s2, s0, 28
-; GFX7-NEXT:    s_bfe_u32 s4, s0, 0x40018
+; GFX7-NEXT:    s_bfe_u32 s8, s0, 0x40018
 ; GFX7-NEXT:    s_bfe_u32 s15, s1, 0x40018
 ; GFX7-NEXT:    s_bfe_u32 s16, s1, 0x40014
 ; GFX7-NEXT:    s_bfe_u32 s17, s1, 0x40010
@@ -575,9 +575,9 @@ define amdgpu_kernel void @udot8_acc8(<8 x i4> addrspace(1)* %src1,
 ; GFX7-NEXT:    s_bfe_u32 s20, s1, 0x40004
 ; GFX7-NEXT:    s_lshr_b32 s14, s1, 28
 ; GFX7-NEXT:    s_and_b32 s1, s1, 15
-; GFX7-NEXT:    s_bfe_u32 s5, s0, 0x40014
-; GFX7-NEXT:    s_bfe_u32 s6, s0, 0x40010
-; GFX7-NEXT:    s_bfe_u32 s7, s0, 0x4000c
+; GFX7-NEXT:    s_bfe_u32 s9, s0, 0x40014
+; GFX7-NEXT:    s_bfe_u32 s10, s0, 0x40010
+; GFX7-NEXT:    s_bfe_u32 s11, s0, 0x4000c
 ; GFX7-NEXT:    s_bfe_u32 s12, s0, 0x40008
 ; GFX7-NEXT:    s_bfe_u32 s13, s0, 0x40004
 ; GFX7-NEXT:    s_and_b32 s0, s0, 15
@@ -592,13 +592,13 @@ define amdgpu_kernel void @udot8_acc8(<8 x i4> addrspace(1)* %src1,
 ; GFX7-NEXT:    v_mad_u32_u24 v0, s0, v1, v0
 ; GFX7-NEXT:    v_mad_u32_u24 v0, s13, v2, v0
 ; GFX7-NEXT:    v_mad_u32_u24 v0, s12, v3, v0
-; GFX7-NEXT:    v_mad_u32_u24 v0, s7, v4, v0
-; GFX7-NEXT:    v_mad_u32_u24 v0, s6, v5, v0
-; GFX7-NEXT:    v_mad_u32_u24 v0, s5, v6, v0
-; GFX7-NEXT:    v_mad_u32_u24 v0, s4, v7, v0
+; GFX7-NEXT:    v_mad_u32_u24 v0, s11, v4, v0
+; GFX7-NEXT:    v_mad_u32_u24 v0, s10, v5, v0
+; GFX7-NEXT:    v_mad_u32_u24 v0, s9, v6, v0
+; GFX7-NEXT:    v_mad_u32_u24 v0, s8, v7, v0
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s14
 ; GFX7-NEXT:    v_mad_u32_u24 v0, s2, v1, v0
-; GFX7-NEXT:    buffer_store_byte v0, off, s[8:11], 0
+; GFX7-NEXT:    buffer_store_byte v0, off, s[4:7], 0
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: udot8_acc8:
@@ -606,46 +606,46 @@ define amdgpu_kernel void @udot8_acc8(<8 x i4> addrspace(1)* %src1,
 ; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX8-NEXT:    s_load_dword s4, s[6:7], 0x0
+; GFX8-NEXT:    s_load_dword s2, s[6:7], 0x0
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX8-NEXT:    flat_load_ubyte v2, v[0:1]
+; GFX8-NEXT:    s_load_dword s0, s[4:5], 0x0
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_and_b32 s0, s2, 15
-; GFX8-NEXT:    s_and_b32 s1, s4, 15
+; GFX8-NEXT:    s_and_b32 s1, s2, 15
+; GFX8-NEXT:    s_bfe_u32 s4, s2, 0x40004
 ; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    s_bfe_u32 s5, s4, 0x40004
-; GFX8-NEXT:    v_mov_b32_e32 v4, s5
-; GFX8-NEXT:    s_bfe_u32 s1, s2, 0x40004
-; GFX8-NEXT:    s_bfe_u32 s5, s4, 0x40008
-; GFX8-NEXT:    s_bfe_u32 s8, s4, 0x40010
-; GFX8-NEXT:    s_bfe_u32 s10, s4, 0x40014
-; GFX8-NEXT:    s_bfe_u32 s12, s4, 0x40018
-; GFX8-NEXT:    s_lshr_b32 s14, s4, 28
-; GFX8-NEXT:    s_bfe_u32 s4, s4, 0x4000c
-; GFX8-NEXT:    s_bfe_u32 s6, s2, 0x40008
-; GFX8-NEXT:    v_mov_b32_e32 v5, s5
+; GFX8-NEXT:    v_mov_b32_e32 v4, s4
+; GFX8-NEXT:    s_and_b32 s1, s0, 15
+; GFX8-NEXT:    s_bfe_u32 s5, s0, 0x40004
+; GFX8-NEXT:    s_bfe_u32 s4, s2, 0x40008
 ; GFX8-NEXT:    s_bfe_u32 s7, s2, 0x4000c
-; GFX8-NEXT:    v_mov_b32_e32 v6, s4
-; GFX8-NEXT:    s_bfe_u32 s9, s2, 0x40010
+; GFX8-NEXT:    v_mov_b32_e32 v5, s4
+; GFX8-NEXT:    s_bfe_u32 s6, s0, 0x40008
+; GFX8-NEXT:    s_bfe_u32 s8, s2, 0x40010
+; GFX8-NEXT:    v_mov_b32_e32 v6, s7
+; GFX8-NEXT:    s_bfe_u32 s4, s0, 0x4000c
+; GFX8-NEXT:    s_bfe_u32 s9, s2, 0x40014
+; GFX8-NEXT:    s_bfe_u32 s7, s0, 0x40010
 ; GFX8-NEXT:    v_mov_b32_e32 v7, s8
-; GFX8-NEXT:    s_bfe_u32 s11, s2, 0x40014
-; GFX8-NEXT:    v_mov_b32_e32 v8, s10
-; GFX8-NEXT:    s_bfe_u32 s13, s2, 0x40018
-; GFX8-NEXT:    v_mov_b32_e32 v9, s12
+; GFX8-NEXT:    s_bfe_u32 s11, s2, 0x40018
+; GFX8-NEXT:    s_bfe_u32 s10, s0, 0x40014
+; GFX8-NEXT:    v_mov_b32_e32 v8, s9
+; GFX8-NEXT:    s_bfe_u32 s12, s0, 0x40018
 ; GFX8-NEXT:    s_lshr_b32 s2, s2, 28
+; GFX8-NEXT:    v_mov_b32_e32 v9, s11
+; GFX8-NEXT:    s_lshr_b32 s0, s0, 28
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
-; GFX8-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
-; GFX8-NEXT:    v_mad_u32_u24 v2, s1, v4, v2
+; GFX8-NEXT:    v_mad_u32_u24 v2, s1, v3, v2
+; GFX8-NEXT:    v_mad_u32_u24 v2, s5, v4, v2
 ; GFX8-NEXT:    v_and_b32_e32 v2, 0xff, v2
 ; GFX8-NEXT:    v_mad_u32_u24 v2, s6, v5, v2
-; GFX8-NEXT:    v_mad_u32_u24 v2, s7, v6, v2
-; GFX8-NEXT:    v_mad_u32_u24 v2, s9, v7, v2
-; GFX8-NEXT:    v_mad_u32_u24 v2, s11, v8, v2
-; GFX8-NEXT:    v_mad_u32_u24 v2, s13, v9, v2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s14
-; GFX8-NEXT:    v_mad_u32_u24 v2, s2, v3, v2
+; GFX8-NEXT:    v_mad_u32_u24 v2, s4, v6, v2
+; GFX8-NEXT:    v_mad_u32_u24 v2, s7, v7, v2
+; GFX8-NEXT:    v_mad_u32_u24 v2, s10, v8, v2
+; GFX8-NEXT:    v_mad_u32_u24 v2, s12, v9, v2
+; GFX8-NEXT:    v_mov_b32_e32 v3, s2
+; GFX8-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
 ; GFX8-NEXT:    flat_store_byte v[0:1], v2
 ; GFX8-NEXT:    s_endpgm
 ;
@@ -654,46 +654,46 @@ define amdgpu_kernel void @udot8_acc8(<8 x i4> addrspace(1)* %src1,
 ; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX9-NEXT:    s_load_dword s4, s[6:7], 0x0
+; GFX9-NEXT:    s_load_dword s2, s[6:7], 0x0
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-NEXT:    global_load_ubyte v2, v[0:1], off
+; GFX9-NEXT:    s_load_dword s0, s[4:5], 0x0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_and_b32 s0, s2, 15
-; GFX9-NEXT:    s_and_b32 s1, s4, 15
+; GFX9-NEXT:    s_and_b32 s1, s2, 15
+; GFX9-NEXT:    s_bfe_u32 s4, s2, 0x40004
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s1
-; GFX9-NEXT:    s_bfe_u32 s5, s4, 0x40004
-; GFX9-NEXT:    v_mov_b32_e32 v4, s5
-; GFX9-NEXT:    s_bfe_u32 s1, s2, 0x40004
-; GFX9-NEXT:    s_bfe_u32 s5, s4, 0x40008
-; GFX9-NEXT:    s_bfe_u32 s8, s4, 0x40010
-; GFX9-NEXT:    s_bfe_u32 s10, s4, 0x40014
-; GFX9-NEXT:    s_bfe_u32 s12, s4, 0x40018
-; GFX9-NEXT:    s_lshr_b32 s14, s4, 28
-; GFX9-NEXT:    s_bfe_u32 s4, s4, 0x4000c
-; GFX9-NEXT:    s_bfe_u32 s6, s2, 0x40008
-; GFX9-NEXT:    v_mov_b32_e32 v5, s5
+; GFX9-NEXT:    v_mov_b32_e32 v4, s4
+; GFX9-NEXT:    s_and_b32 s1, s0, 15
+; GFX9-NEXT:    s_bfe_u32 s5, s0, 0x40004
+; GFX9-NEXT:    s_bfe_u32 s4, s2, 0x40008
 ; GFX9-NEXT:    s_bfe_u32 s7, s2, 0x4000c
-; GFX9-NEXT:    v_mov_b32_e32 v6, s4
-; GFX9-NEXT:    s_bfe_u32 s9, s2, 0x40010
+; GFX9-NEXT:    v_mov_b32_e32 v5, s4
+; GFX9-NEXT:    s_bfe_u32 s6, s0, 0x40008
+; GFX9-NEXT:    s_bfe_u32 s8, s2, 0x40010
+; GFX9-NEXT:    v_mov_b32_e32 v6, s7
+; GFX9-NEXT:    s_bfe_u32 s4, s0, 0x4000c
+; GFX9-NEXT:    s_bfe_u32 s9, s2, 0x40014
+; GFX9-NEXT:    s_bfe_u32 s7, s0, 0x40010
 ; GFX9-NEXT:    v_mov_b32_e32 v7, s8
-; GFX9-NEXT:    s_bfe_u32 s11, s2, 0x40014
-; GFX9-NEXT:    v_mov_b32_e32 v8, s10
-; GFX9-NEXT:    s_bfe_u32 s13, s2, 0x40018
-; GFX9-NEXT:    v_mov_b32_e32 v9, s12
+; GFX9-NEXT:    s_bfe_u32 s11, s2, 0x40018
+; GFX9-NEXT:    s_bfe_u32 s10, s0, 0x40014
+; GFX9-NEXT:    v_mov_b32_e32 v8, s9
+; GFX9-NEXT:    s_bfe_u32 s12, s0, 0x40018
 ; GFX9-NEXT:    s_lshr_b32 s2, s2, 28
+; GFX9-NEXT:    v_mov_b32_e32 v9, s11
+; GFX9-NEXT:    s_lshr_b32 s0, s0, 28
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
-; GFX9-NEXT:    v_mad_u32_u24 v2, s1, v4, v2
+; GFX9-NEXT:    v_mad_u32_u24 v2, s1, v3, v2
+; GFX9-NEXT:    v_mad_u32_u24 v2, s5, v4, v2
 ; GFX9-NEXT:    v_and_b32_e32 v2, 0xff, v2
 ; GFX9-NEXT:    v_mad_u32_u24 v2, s6, v5, v2
-; GFX9-NEXT:    v_mad_u32_u24 v2, s7, v6, v2
-; GFX9-NEXT:    v_mad_u32_u24 v2, s9, v7, v2
-; GFX9-NEXT:    v_mad_u32_u24 v2, s11, v8, v2
-; GFX9-NEXT:    v_mad_u32_u24 v2, s13, v9, v2
-; GFX9-NEXT:    v_mov_b32_e32 v3, s14
-; GFX9-NEXT:    v_mad_u32_u24 v2, s2, v3, v2
+; GFX9-NEXT:    v_mad_u32_u24 v2, s4, v6, v2
+; GFX9-NEXT:    v_mad_u32_u24 v2, s7, v7, v2
+; GFX9-NEXT:    v_mad_u32_u24 v2, s10, v8, v2
+; GFX9-NEXT:    v_mad_u32_u24 v2, s12, v9, v2
+; GFX9-NEXT:    v_mov_b32_e32 v3, s2
+; GFX9-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
 ; GFX9-NEXT:    global_store_byte v[0:1], v2, off
 ; GFX9-NEXT:    s_endpgm
 ;
@@ -702,46 +702,46 @@ define amdgpu_kernel void @udot8_acc8(<8 x i4> addrspace(1)* %src1,
 ; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
+; GFX9-DL-NEXT:    s_load_dword s2, s[6:7], 0x0
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-DL-NEXT:    global_load_ubyte v2, v[0:1], off
+; GFX9-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    s_and_b32 s0, s2, 15
-; GFX9-DL-NEXT:    s_and_b32 s1, s4, 15
+; GFX9-DL-NEXT:    s_and_b32 s1, s2, 15
+; GFX9-DL-NEXT:    s_bfe_u32 s4, s2, 0x40004
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s1
-; GFX9-DL-NEXT:    s_bfe_u32 s5, s4, 0x40004
-; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s5
-; GFX9-DL-NEXT:    s_bfe_u32 s1, s2, 0x40004
-; GFX9-DL-NEXT:    s_bfe_u32 s5, s4, 0x40008
-; GFX9-DL-NEXT:    s_bfe_u32 s8, s4, 0x40010
-; GFX9-DL-NEXT:    s_bfe_u32 s10, s4, 0x40014
-; GFX9-DL-NEXT:    s_bfe_u32 s12, s4, 0x40018
-; GFX9-DL-NEXT:    s_lshr_b32 s14, s4, 28
-; GFX9-DL-NEXT:    s_bfe_u32 s4, s4, 0x4000c
-; GFX9-DL-NEXT:    s_bfe_u32 s6, s2, 0x40008
-; GFX9-DL-NEXT:    v_mov_b32_e32 v5, s5
+; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s4
+; GFX9-DL-NEXT:    s_and_b32 s1, s0, 15
+; GFX9-DL-NEXT:    s_bfe_u32 s5, s0, 0x40004
+; GFX9-DL-NEXT:    s_bfe_u32 s4, s2, 0x40008
 ; GFX9-DL-NEXT:    s_bfe_u32 s7, s2, 0x4000c
-; GFX9-DL-NEXT:    v_mov_b32_e32 v6, s4
-; GFX9-DL-NEXT:    s_bfe_u32 s9, s2, 0x40010
+; GFX9-DL-NEXT:    v_mov_b32_e32 v5, s4
+; GFX9-DL-NEXT:    s_bfe_u32 s6, s0, 0x40008
+; GFX9-DL-NEXT:    s_bfe_u32 s8, s2, 0x40010
+; GFX9-DL-NEXT:    v_mov_b32_e32 v6, s7
+; GFX9-DL-NEXT:    s_bfe_u32 s4, s0, 0x4000c
+; GFX9-DL-NEXT:    s_bfe_u32 s9, s2, 0x40014
+; GFX9-DL-NEXT:    s_bfe_u32 s7, s0, 0x40010
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v7, s8
-; GFX9-DL-NEXT:    s_bfe_u32 s11, s2, 0x40014
-; GFX9-DL-NEXT:    v_mov_b32_e32 v8, s10
-; GFX9-DL-NEXT:    s_bfe_u32 s13, s2, 0x40018
-; GFX9-DL-NEXT:    v_mov_b32_e32 v9, s12
+; GFX9-DL-NEXT:    s_bfe_u32 s11, s2, 0x40018
+; GFX9-DL-NEXT:    s_bfe_u32 s10, s0, 0x40014
+; GFX9-DL-NEXT:    v_mov_b32_e32 v8, s9
+; GFX9-DL-NEXT:    s_bfe_u32 s12, s0, 0x40018
 ; GFX9-DL-NEXT:    s_lshr_b32 s2, s2, 28
+; GFX9-DL-NEXT:    v_mov_b32_e32 v9, s11
+; GFX9-DL-NEXT:    s_lshr_b32 s0, s0, 28
 ; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
-; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s1, v4, v2
+; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s1, v3, v2
+; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s5, v4, v2
 ; GFX9-DL-NEXT:    v_and_b32_e32 v2, 0xff, v2
 ; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s6, v5, v2
-; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s7, v6, v2
-; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s9, v7, v2
-; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s11, v8, v2
-; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s13, v9, v2
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s14
-; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s2, v3, v2
+; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s4, v6, v2
+; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s7, v7, v2
+; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s10, v8, v2
+; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s12, v9, v2
+; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s2
+; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
 ; GFX9-DL-NEXT:    global_store_byte v[0:1], v2, off
 ; GFX9-DL-NEXT:    s_endpgm
 ;
@@ -751,38 +751,38 @@ define amdgpu_kernel void @udot8_acc8(<8 x i4> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX10-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
+; GFX10-DL-NEXT:    s_load_dword s1, s[6:7], 0x0
 ; GFX10-DL-NEXT:    global_load_ubyte v2, v[0:1], off
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_and_b32 s0, s2, 15
-; GFX10-DL-NEXT:    s_and_b32 s1, s4, 15
-; GFX10-DL-NEXT:    s_bfe_u32 s5, s2, 0x40004
-; GFX10-DL-NEXT:    s_bfe_u32 s6, s4, 0x40004
-; GFX10-DL-NEXT:    s_bfe_u32 s7, s2, 0x40008
-; GFX10-DL-NEXT:    s_bfe_u32 s8, s4, 0x40008
-; GFX10-DL-NEXT:    s_bfe_u32 s9, s2, 0x4000c
-; GFX10-DL-NEXT:    s_bfe_u32 s10, s4, 0x4000c
-; GFX10-DL-NEXT:    s_bfe_u32 s11, s2, 0x40010
-; GFX10-DL-NEXT:    s_bfe_u32 s12, s4, 0x40010
-; GFX10-DL-NEXT:    s_bfe_u32 s13, s2, 0x40014
-; GFX10-DL-NEXT:    s_bfe_u32 s14, s4, 0x40014
+; GFX10-DL-NEXT:    s_and_b32 s2, s0, 15
+; GFX10-DL-NEXT:    s_and_b32 s4, s1, 15
+; GFX10-DL-NEXT:    s_bfe_u32 s5, s0, 0x40004
+; GFX10-DL-NEXT:    s_bfe_u32 s6, s1, 0x40004
 ; GFX10-DL-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s0, s1, v2
-; GFX10-DL-NEXT:    s_bfe_u32 s0, s2, 0x40018
-; GFX10-DL-NEXT:    s_bfe_u32 s1, s4, 0x40018
-; GFX10-DL-NEXT:    s_lshr_b32 s2, s2, 28
-; GFX10-DL-NEXT:    s_lshr_b32 s4, s4, 28
+; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s2, s4, v2
+; GFX10-DL-NEXT:    s_bfe_u32 s2, s0, 0x40008
+; GFX10-DL-NEXT:    s_bfe_u32 s4, s1, 0x40008
 ; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s5, s6, v2
+; GFX10-DL-NEXT:    s_bfe_u32 s5, s0, 0x4000c
+; GFX10-DL-NEXT:    s_bfe_u32 s6, s1, 0x4000c
 ; GFX10-DL-NEXT:    v_and_b32_e32 v2, 0xff, v2
-; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s7, s8, v2
-; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s9, s10, v2
-; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s11, s12, v2
-; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s13, s14, v2
-; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s0, s1, v2
 ; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s2, s4, v2
+; GFX10-DL-NEXT:    s_bfe_u32 s2, s0, 0x40010
+; GFX10-DL-NEXT:    s_bfe_u32 s4, s1, 0x40010
+; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s5, s6, v2
+; GFX10-DL-NEXT:    s_bfe_u32 s5, s0, 0x40014
+; GFX10-DL-NEXT:    s_bfe_u32 s6, s1, 0x40014
+; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s2, s4, v2
+; GFX10-DL-NEXT:    s_bfe_u32 s2, s0, 0x40018
+; GFX10-DL-NEXT:    s_bfe_u32 s4, s1, 0x40018
+; GFX10-DL-NEXT:    s_lshr_b32 s0, s0, 28
+; GFX10-DL-NEXT:    s_lshr_b32 s1, s1, 28
+; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s5, s6, v2
+; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s2, s4, v2
+; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s0, s1, v2
 ; GFX10-DL-NEXT:    global_store_byte v[0:1], v2, off
 ; GFX10-DL-NEXT:    s_endpgm
                                       <8 x i4> addrspace(1)* %src2,
@@ -858,17 +858,17 @@ entry:
 define amdgpu_kernel void @udot8_acc4(<8 x i4> addrspace(1)* %src1,
 ; GFX7-LABEL: udot8_acc4:
 ; GFX7:       ; %bb.0: ; %entry
-; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
-; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xd
-; GFX7-NEXT:    s_mov_b32 s11, 0xf000
-; GFX7-NEXT:    s_mov_b32 s10, -1
+; GFX7-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
+; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xd
+; GFX7-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7-NEXT:    s_mov_b32 s6, -1
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_load_dword s0, s[4:5], 0x0
-; GFX7-NEXT:    buffer_load_ubyte v0, off, s[8:11], 0
-; GFX7-NEXT:    s_load_dword s1, s[6:7], 0x0
+; GFX7-NEXT:    s_load_dword s0, s[8:9], 0x0
+; GFX7-NEXT:    buffer_load_ubyte v0, off, s[4:7], 0
+; GFX7-NEXT:    s_load_dword s1, s[10:11], 0x0
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    s_lshr_b32 s2, s0, 28
-; GFX7-NEXT:    s_bfe_u32 s4, s0, 0x40018
+; GFX7-NEXT:    s_bfe_u32 s8, s0, 0x40018
 ; GFX7-NEXT:    s_bfe_u32 s15, s1, 0x40018
 ; GFX7-NEXT:    s_bfe_u32 s16, s1, 0x40014
 ; GFX7-NEXT:    s_bfe_u32 s17, s1, 0x40010
@@ -877,9 +877,9 @@ define amdgpu_kernel void @udot8_acc4(<8 x i4> addrspace(1)* %src1,
 ; GFX7-NEXT:    s_bfe_u32 s20, s1, 0x40004
 ; GFX7-NEXT:    s_lshr_b32 s14, s1, 28
 ; GFX7-NEXT:    s_and_b32 s1, s1, 15
-; GFX7-NEXT:    s_bfe_u32 s5, s0, 0x40014
-; GFX7-NEXT:    s_bfe_u32 s6, s0, 0x40010
-; GFX7-NEXT:    s_bfe_u32 s7, s0, 0x4000c
+; GFX7-NEXT:    s_bfe_u32 s9, s0, 0x40014
+; GFX7-NEXT:    s_bfe_u32 s10, s0, 0x40010
+; GFX7-NEXT:    s_bfe_u32 s11, s0, 0x4000c
 ; GFX7-NEXT:    s_bfe_u32 s12, s0, 0x40008
 ; GFX7-NEXT:    s_bfe_u32 s13, s0, 0x40004
 ; GFX7-NEXT:    s_and_b32 s0, s0, 15
@@ -894,14 +894,14 @@ define amdgpu_kernel void @udot8_acc4(<8 x i4> addrspace(1)* %src1,
 ; GFX7-NEXT:    v_mad_u32_u24 v0, s0, v1, v0
 ; GFX7-NEXT:    v_mad_u32_u24 v0, s13, v2, v0
 ; GFX7-NEXT:    v_mad_u32_u24 v0, s12, v3, v0
-; GFX7-NEXT:    v_mad_u32_u24 v0, s7, v4, v0
-; GFX7-NEXT:    v_mad_u32_u24 v0, s6, v5, v0
-; GFX7-NEXT:    v_mad_u32_u24 v0, s5, v6, v0
-; GFX7-NEXT:    v_mad_u32_u24 v0, s4, v7, v0
+; GFX7-NEXT:    v_mad_u32_u24 v0, s11, v4, v0
+; GFX7-NEXT:    v_mad_u32_u24 v0, s10, v5, v0
+; GFX7-NEXT:    v_mad_u32_u24 v0, s9, v6, v0
+; GFX7-NEXT:    v_mad_u32_u24 v0, s8, v7, v0
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s14
 ; GFX7-NEXT:    v_mad_u32_u24 v0, s2, v1, v0
 ; GFX7-NEXT:    v_and_b32_e32 v0, 15, v0
-; GFX7-NEXT:    buffer_store_byte v0, off, s[8:11], 0
+; GFX7-NEXT:    buffer_store_byte v0, off, s[4:7], 0
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: udot8_acc4:
@@ -909,48 +909,48 @@ define amdgpu_kernel void @udot8_acc4(<8 x i4> addrspace(1)* %src1,
 ; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX8-NEXT:    s_load_dword s4, s[6:7], 0x0
+; GFX8-NEXT:    s_load_dword s2, s[6:7], 0x0
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX8-NEXT:    flat_load_ubyte v2, v[0:1]
+; GFX8-NEXT:    s_load_dword s0, s[4:5], 0x0
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_and_b32 s0, s2, 15
-; GFX8-NEXT:    s_and_b32 s1, s4, 15
+; GFX8-NEXT:    s_and_b32 s1, s2, 15
+; GFX8-NEXT:    s_bfe_u32 s4, s2, 0x40004
 ; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    s_bfe_u32 s5, s4, 0x40004
-; GFX8-NEXT:    s_bfe_u32 s6, s4, 0x40008
-; GFX8-NEXT:    s_bfe_u32 s7, s4, 0x4000c
-; GFX8-NEXT:    v_mov_b32_e32 v4, s5
-; GFX8-NEXT:    s_bfe_u32 s1, s2, 0x40004
-; GFX8-NEXT:    s_bfe_u32 s5, s2, 0x40008
-; GFX8-NEXT:    s_bfe_u32 s8, s2, 0x4000c
+; GFX8-NEXT:    s_bfe_u32 s6, s2, 0x40008
+; GFX8-NEXT:    s_and_b32 s1, s0, 15
+; GFX8-NEXT:    s_bfe_u32 s7, s2, 0x4000c
+; GFX8-NEXT:    v_mov_b32_e32 v4, s4
+; GFX8-NEXT:    s_bfe_u32 s5, s0, 0x40004
 ; GFX8-NEXT:    v_mov_b32_e32 v5, s7
+; GFX8-NEXT:    s_bfe_u32 s8, s0, 0x4000c
 ; GFX8-NEXT:    v_mov_b32_e32 v6, s6
+; GFX8-NEXT:    s_bfe_u32 s4, s0, 0x40008
 ; GFX8-NEXT:    v_mul_u32_u24_e32 v5, s8, v5
-; GFX8-NEXT:    s_bfe_u32 s9, s4, 0x40010
+; GFX8-NEXT:    s_bfe_u32 s6, s2, 0x40010
 ; GFX8-NEXT:    v_and_b32_e32 v5, 15, v5
-; GFX8-NEXT:    s_bfe_u32 s11, s4, 0x40014
-; GFX8-NEXT:    s_bfe_u32 s10, s2, 0x40010
-; GFX8-NEXT:    v_mov_b32_e32 v7, s9
-; GFX8-NEXT:    s_bfe_u32 s13, s4, 0x40018
-; GFX8-NEXT:    s_bfe_u32 s12, s2, 0x40014
-; GFX8-NEXT:    v_mov_b32_e32 v8, s11
-; GFX8-NEXT:    s_bfe_u32 s14, s2, 0x40018
-; GFX8-NEXT:    s_lshr_b32 s4, s4, 28
-; GFX8-NEXT:    v_mov_b32_e32 v9, s13
-; GFX8-NEXT:    s_lshr_b32 s2, s2, 28
+; GFX8-NEXT:    s_bfe_u32 s8, s2, 0x40014
+; GFX8-NEXT:    s_bfe_u32 s7, s0, 0x40010
+; GFX8-NEXT:    v_mov_b32_e32 v7, s6
+; GFX8-NEXT:    s_lshr_b32 s11, s2, 28
+; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x40018
+; GFX8-NEXT:    s_bfe_u32 s9, s0, 0x40014
+; GFX8-NEXT:    v_mov_b32_e32 v8, s8
+; GFX8-NEXT:    s_bfe_u32 s10, s0, 0x40018
+; GFX8-NEXT:    v_mov_b32_e32 v9, s2
+; GFX8-NEXT:    s_lshr_b32 s0, s0, 28
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
-; GFX8-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
-; GFX8-NEXT:    v_mad_u32_u24 v2, s1, v4, v2
-; GFX8-NEXT:    v_mad_u32_u24 v2, s5, v6, v2
+; GFX8-NEXT:    v_mad_u32_u24 v2, s1, v3, v2
+; GFX8-NEXT:    v_mad_u32_u24 v2, s5, v4, v2
+; GFX8-NEXT:    v_mad_u32_u24 v2, s4, v6, v2
 ; GFX8-NEXT:    v_and_b32_e32 v2, 15, v2
 ; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v5, v2
-; GFX8-NEXT:    v_mad_u32_u24 v2, s10, v7, v2
-; GFX8-NEXT:    v_mad_u32_u24 v2, s12, v8, v2
-; GFX8-NEXT:    v_mad_u32_u24 v2, s14, v9, v2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s4
-; GFX8-NEXT:    v_mad_u32_u24 v2, s2, v3, v2
+; GFX8-NEXT:    v_mad_u32_u24 v2, s7, v7, v2
+; GFX8-NEXT:    v_mad_u32_u24 v2, s9, v8, v2
+; GFX8-NEXT:    v_mad_u32_u24 v2, s10, v9, v2
+; GFX8-NEXT:    v_mov_b32_e32 v3, s11
+; GFX8-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
 ; GFX8-NEXT:    v_and_b32_e32 v2, 15, v2
 ; GFX8-NEXT:    flat_store_byte v[0:1], v2
 ; GFX8-NEXT:    s_endpgm
@@ -960,48 +960,48 @@ define amdgpu_kernel void @udot8_acc4(<8 x i4> addrspace(1)* %src1,
 ; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX9-NEXT:    s_load_dword s4, s[6:7], 0x0
+; GFX9-NEXT:    s_load_dword s2, s[6:7], 0x0
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-NEXT:    global_load_ubyte v2, v[0:1], off
+; GFX9-NEXT:    s_load_dword s0, s[4:5], 0x0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_and_b32 s0, s2, 15
-; GFX9-NEXT:    s_and_b32 s1, s4, 15
+; GFX9-NEXT:    s_and_b32 s1, s2, 15
+; GFX9-NEXT:    s_bfe_u32 s4, s2, 0x40004
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s1
-; GFX9-NEXT:    s_bfe_u32 s5, s4, 0x40004
-; GFX9-NEXT:    s_bfe_u32 s6, s4, 0x40008
-; GFX9-NEXT:    s_bfe_u32 s7, s4, 0x4000c
-; GFX9-NEXT:    v_mov_b32_e32 v4, s5
-; GFX9-NEXT:    s_bfe_u32 s1, s2, 0x40004
-; GFX9-NEXT:    s_bfe_u32 s5, s2, 0x40008
-; GFX9-NEXT:    s_bfe_u32 s8, s2, 0x4000c
+; GFX9-NEXT:    s_bfe_u32 s6, s2, 0x40008
+; GFX9-NEXT:    s_and_b32 s1, s0, 15
+; GFX9-NEXT:    s_bfe_u32 s7, s2, 0x4000c
+; GFX9-NEXT:    v_mov_b32_e32 v4, s4
+; GFX9-NEXT:    s_bfe_u32 s5, s0, 0x40004
 ; GFX9-NEXT:    v_mov_b32_e32 v5, s7
+; GFX9-NEXT:    s_bfe_u32 s8, s0, 0x4000c
 ; GFX9-NEXT:    v_mov_b32_e32 v6, s6
+; GFX9-NEXT:    s_bfe_u32 s4, s0, 0x40008
 ; GFX9-NEXT:    v_mul_u32_u24_e32 v5, s8, v5
-; GFX9-NEXT:    s_bfe_u32 s9, s4, 0x40010
+; GFX9-NEXT:    s_bfe_u32 s6, s2, 0x40010
 ; GFX9-NEXT:    v_and_b32_e32 v5, 15, v5
-; GFX9-NEXT:    s_bfe_u32 s11, s4, 0x40014
-; GFX9-NEXT:    s_bfe_u32 s10, s2, 0x40010
-; GFX9-NEXT:    v_mov_b32_e32 v7, s9
-; GFX9-NEXT:    s_bfe_u32 s13, s4, 0x40018
-; GFX9-NEXT:    s_bfe_u32 s12, s2, 0x40014
-; GFX9-NEXT:    v_mov_b32_e32 v8, s11
-; GFX9-NEXT:    s_bfe_u32 s14, s2, 0x40018
-; GFX9-NEXT:    s_lshr_b32 s4, s4, 28
-; GFX9-NEXT:    v_mov_b32_e32 v9, s13
-; GFX9-NEXT:    s_lshr_b32 s2, s2, 28
+; GFX9-NEXT:    s_bfe_u32 s8, s2, 0x40014
+; GFX9-NEXT:    s_bfe_u32 s7, s0, 0x40010
+; GFX9-NEXT:    v_mov_b32_e32 v7, s6
+; GFX9-NEXT:    s_lshr_b32 s11, s2, 28
+; GFX9-NEXT:    s_bfe_u32 s2, s2, 0x40018
+; GFX9-NEXT:    s_bfe_u32 s9, s0, 0x40014
+; GFX9-NEXT:    v_mov_b32_e32 v8, s8
+; GFX9-NEXT:    s_bfe_u32 s10, s0, 0x40018
+; GFX9-NEXT:    v_mov_b32_e32 v9, s2
+; GFX9-NEXT:    s_lshr_b32 s0, s0, 28
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
-; GFX9-NEXT:    v_mad_u32_u24 v2, s1, v4, v2
-; GFX9-NEXT:    v_mad_u32_u24 v2, s5, v6, v2
+; GFX9-NEXT:    v_mad_u32_u24 v2, s1, v3, v2
+; GFX9-NEXT:    v_mad_u32_u24 v2, s5, v4, v2
+; GFX9-NEXT:    v_mad_u32_u24 v2, s4, v6, v2
 ; GFX9-NEXT:    v_and_b32_e32 v2, 15, v2
 ; GFX9-NEXT:    v_add_u32_e32 v2, v2, v5
-; GFX9-NEXT:    v_mad_u32_u24 v2, s10, v7, v2
-; GFX9-NEXT:    v_mad_u32_u24 v2, s12, v8, v2
-; GFX9-NEXT:    v_mad_u32_u24 v2, s14, v9, v2
-; GFX9-NEXT:    v_mov_b32_e32 v3, s4
-; GFX9-NEXT:    v_mad_u32_u24 v2, s2, v3, v2
+; GFX9-NEXT:    v_mad_u32_u24 v2, s7, v7, v2
+; GFX9-NEXT:    v_mad_u32_u24 v2, s9, v8, v2
+; GFX9-NEXT:    v_mad_u32_u24 v2, s10, v9, v2
+; GFX9-NEXT:    v_mov_b32_e32 v3, s11
+; GFX9-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
 ; GFX9-NEXT:    v_and_b32_e32 v2, 15, v2
 ; GFX9-NEXT:    global_store_byte v[0:1], v2, off
 ; GFX9-NEXT:    s_endpgm
@@ -1011,48 +1011,48 @@ define amdgpu_kernel void @udot8_acc4(<8 x i4> addrspace(1)* %src1,
 ; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
+; GFX9-DL-NEXT:    s_load_dword s2, s[6:7], 0x0
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-DL-NEXT:    global_load_ubyte v2, v[0:1], off
+; GFX9-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    s_and_b32 s0, s2, 15
-; GFX9-DL-NEXT:    s_and_b32 s1, s4, 15
+; GFX9-DL-NEXT:    s_and_b32 s1, s2, 15
+; GFX9-DL-NEXT:    s_bfe_u32 s4, s2, 0x40004
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s1
-; GFX9-DL-NEXT:    s_bfe_u32 s5, s4, 0x40004
-; GFX9-DL-NEXT:    s_bfe_u32 s6, s4, 0x40008
-; GFX9-DL-NEXT:    s_bfe_u32 s7, s4, 0x4000c
-; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s5
-; GFX9-DL-NEXT:    s_bfe_u32 s1, s2, 0x40004
-; GFX9-DL-NEXT:    s_bfe_u32 s5, s2, 0x40008
-; GFX9-DL-NEXT:    s_bfe_u32 s8, s2, 0x4000c
+; GFX9-DL-NEXT:    s_bfe_u32 s6, s2, 0x40008
+; GFX9-DL-NEXT:    s_and_b32 s1, s0, 15
+; GFX9-DL-NEXT:    s_bfe_u32 s7, s2, 0x4000c
+; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s4
+; GFX9-DL-NEXT:    s_bfe_u32 s5, s0, 0x40004
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v5, s7
+; GFX9-DL-NEXT:    s_bfe_u32 s8, s0, 0x4000c
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v6, s6
+; GFX9-DL-NEXT:    s_bfe_u32 s4, s0, 0x40008
 ; GFX9-DL-NEXT:    v_mul_u32_u24_e32 v5, s8, v5
-; GFX9-DL-NEXT:    s_bfe_u32 s9, s4, 0x40010
+; GFX9-DL-NEXT:    s_bfe_u32 s6, s2, 0x40010
 ; GFX9-DL-NEXT:    v_and_b32_e32 v5, 15, v5
-; GFX9-DL-NEXT:    s_bfe_u32 s11, s4, 0x40014
-; GFX9-DL-NEXT:    s_bfe_u32 s10, s2, 0x40010
-; GFX9-DL-NEXT:    v_mov_b32_e32 v7, s9
-; GFX9-DL-NEXT:    s_bfe_u32 s13, s4, 0x40018
-; GFX9-DL-NEXT:    s_bfe_u32 s12, s2, 0x40014
-; GFX9-DL-NEXT:    v_mov_b32_e32 v8, s11
-; GFX9-DL-NEXT:    s_bfe_u32 s14, s2, 0x40018
-; GFX9-DL-NEXT:    s_lshr_b32 s4, s4, 28
-; GFX9-DL-NEXT:    v_mov_b32_e32 v9, s13
-; GFX9-DL-NEXT:    s_lshr_b32 s2, s2, 28
+; GFX9-DL-NEXT:    s_bfe_u32 s8, s2, 0x40014
+; GFX9-DL-NEXT:    s_bfe_u32 s7, s0, 0x40010
+; GFX9-DL-NEXT:    v_mov_b32_e32 v7, s6
+; GFX9-DL-NEXT:    s_lshr_b32 s11, s2, 28
+; GFX9-DL-NEXT:    s_bfe_u32 s2, s2, 0x40018
+; GFX9-DL-NEXT:    s_bfe_u32 s9, s0, 0x40014
+; GFX9-DL-NEXT:    v_mov_b32_e32 v8, s8
+; GFX9-DL-NEXT:    s_bfe_u32 s10, s0, 0x40018
+; GFX9-DL-NEXT:    v_mov_b32_e32 v9, s2
+; GFX9-DL-NEXT:    s_lshr_b32 s0, s0, 28
 ; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
-; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s1, v4, v2
-; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s5, v6, v2
+; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s1, v3, v2
+; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s5, v4, v2
+; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s4, v6, v2
 ; GFX9-DL-NEXT:    v_and_b32_e32 v2, 15, v2
 ; GFX9-DL-NEXT:    v_add_u32_e32 v2, v2, v5
-; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s10, v7, v2
-; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s12, v8, v2
-; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s14, v9, v2
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s4
-; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s2, v3, v2
+; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s7, v7, v2
+; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s9, v8, v2
+; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s10, v9, v2
+; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s11
+; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
 ; GFX9-DL-NEXT:    v_and_b32_e32 v2, 15, v2
 ; GFX9-DL-NEXT:    global_store_byte v[0:1], v2, off
 ; GFX9-DL-NEXT:    s_endpgm
@@ -1063,40 +1063,40 @@ define amdgpu_kernel void @udot8_acc4(<8 x i4> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX10-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
+; GFX10-DL-NEXT:    s_load_dword s1, s[6:7], 0x0
 ; GFX10-DL-NEXT:    global_load_ubyte v2, v[0:1], off
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_and_b32 s0, s2, 15
-; GFX10-DL-NEXT:    s_and_b32 s1, s4, 15
-; GFX10-DL-NEXT:    s_bfe_u32 s5, s2, 0x40004
-; GFX10-DL-NEXT:    s_bfe_u32 s6, s4, 0x40004
-; GFX10-DL-NEXT:    s_bfe_u32 s7, s2, 0x40008
-; GFX10-DL-NEXT:    s_bfe_u32 s8, s2, 0x4000c
-; GFX10-DL-NEXT:    s_bfe_u32 s9, s4, 0x40008
+; GFX10-DL-NEXT:    s_and_b32 s2, s0, 15
+; GFX10-DL-NEXT:    s_and_b32 s4, s1, 15
+; GFX10-DL-NEXT:    s_bfe_u32 s5, s0, 0x40004
+; GFX10-DL-NEXT:    s_bfe_u32 s6, s1, 0x40004
+; GFX10-DL-NEXT:    s_bfe_u32 s7, s1, 0x40008
 ; GFX10-DL-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s0, s1, v2
-; GFX10-DL-NEXT:    s_bfe_u32 s0, s4, 0x4000c
-; GFX10-DL-NEXT:    s_bfe_u32 s1, s4, 0x40010
+; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s2, s4, v2
+; GFX10-DL-NEXT:    s_bfe_u32 s2, s0, 0x40008
+; GFX10-DL-NEXT:    s_bfe_u32 s4, s0, 0x4000c
 ; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s5, s6, v2
-; GFX10-DL-NEXT:    v_mul_u32_u24_e64 v3, s8, s0
-; GFX10-DL-NEXT:    s_bfe_u32 s0, s2, 0x40010
-; GFX10-DL-NEXT:    s_bfe_u32 s5, s2, 0x40014
-; GFX10-DL-NEXT:    s_bfe_u32 s6, s4, 0x40014
-; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s7, s9, v2
-; GFX10-DL-NEXT:    v_and_b32_e32 v3, 15, v3
+; GFX10-DL-NEXT:    s_bfe_u32 s5, s1, 0x4000c
+; GFX10-DL-NEXT:    s_bfe_u32 s6, s1, 0x40014
+; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s2, s7, v2
+; GFX10-DL-NEXT:    v_mul_u32_u24_e64 v3, s4, s5
+; GFX10-DL-NEXT:    s_bfe_u32 s2, s0, 0x40010
+; GFX10-DL-NEXT:    s_bfe_u32 s4, s1, 0x40010
+; GFX10-DL-NEXT:    s_bfe_u32 s5, s0, 0x40014
 ; GFX10-DL-NEXT:    v_and_b32_e32 v2, 15, v2
+; GFX10-DL-NEXT:    v_and_b32_e32 v3, 15, v3
 ; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v2, v3
-; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s0, s1, v2
-; GFX10-DL-NEXT:    s_bfe_u32 s0, s2, 0x40018
-; GFX10-DL-NEXT:    s_bfe_u32 s1, s4, 0x40018
-; GFX10-DL-NEXT:    s_lshr_b32 s2, s2, 28
-; GFX10-DL-NEXT:    s_lshr_b32 s4, s4, 28
+; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s2, s4, v2
+; GFX10-DL-NEXT:    s_bfe_u32 s2, s0, 0x40018
+; GFX10-DL-NEXT:    s_bfe_u32 s4, s1, 0x40018
+; GFX10-DL-NEXT:    s_lshr_b32 s0, s0, 28
+; GFX10-DL-NEXT:    s_lshr_b32 s1, s1, 28
 ; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s5, s6, v2
-; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s0, s1, v2
 ; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s2, s4, v2
+; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s0, s1, v2
 ; GFX10-DL-NEXT:    v_and_b32_e32 v2, 15, v2
 ; GFX10-DL-NEXT:    global_store_byte v[0:1], v2, off
 ; GFX10-DL-NEXT:    s_endpgm
@@ -1157,17 +1157,17 @@ entry:
 define amdgpu_kernel void @udot8_CommutationInsideMAD(<8 x i4> addrspace(1)* %src1,
 ; GFX7-LABEL: udot8_CommutationInsideMAD:
 ; GFX7:       ; %bb.0: ; %entry
-; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
-; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xd
-; GFX7-NEXT:    s_mov_b32 s11, 0xf000
-; GFX7-NEXT:    s_mov_b32 s10, -1
+; GFX7-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
+; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xd
+; GFX7-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7-NEXT:    s_mov_b32 s6, -1
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_load_dword s0, s[4:5], 0x0
-; GFX7-NEXT:    buffer_load_ubyte v0, off, s[8:11], 0
-; GFX7-NEXT:    s_load_dword s1, s[6:7], 0x0
+; GFX7-NEXT:    s_load_dword s0, s[8:9], 0x0
+; GFX7-NEXT:    buffer_load_ubyte v0, off, s[4:7], 0
+; GFX7-NEXT:    s_load_dword s1, s[10:11], 0x0
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    s_lshr_b32 s2, s0, 28
-; GFX7-NEXT:    s_bfe_u32 s4, s0, 0x40018
+; GFX7-NEXT:    s_bfe_u32 s8, s0, 0x40018
 ; GFX7-NEXT:    s_bfe_u32 s15, s1, 0x40018
 ; GFX7-NEXT:    s_bfe_u32 s16, s1, 0x40014
 ; GFX7-NEXT:    s_bfe_u32 s17, s1, 0x40010
@@ -1176,9 +1176,9 @@ define amdgpu_kernel void @udot8_CommutationInsideMAD(<8 x i4> addrspace(1)* %sr
 ; GFX7-NEXT:    s_bfe_u32 s20, s1, 0x40004
 ; GFX7-NEXT:    s_lshr_b32 s14, s1, 28
 ; GFX7-NEXT:    s_and_b32 s1, s1, 15
-; GFX7-NEXT:    s_bfe_u32 s5, s0, 0x40014
-; GFX7-NEXT:    s_bfe_u32 s6, s0, 0x40010
-; GFX7-NEXT:    s_bfe_u32 s7, s0, 0x4000c
+; GFX7-NEXT:    s_bfe_u32 s9, s0, 0x40014
+; GFX7-NEXT:    s_bfe_u32 s10, s0, 0x40010
+; GFX7-NEXT:    s_bfe_u32 s11, s0, 0x4000c
 ; GFX7-NEXT:    s_bfe_u32 s12, s0, 0x40008
 ; GFX7-NEXT:    s_bfe_u32 s13, s0, 0x40004
 ; GFX7-NEXT:    s_and_b32 s0, s0, 15
@@ -1193,14 +1193,14 @@ define amdgpu_kernel void @udot8_CommutationInsideMAD(<8 x i4> addrspace(1)* %sr
 ; GFX7-NEXT:    v_mad_u32_u24 v0, s0, v1, v0
 ; GFX7-NEXT:    v_mad_u32_u24 v0, s13, v2, v0
 ; GFX7-NEXT:    v_mad_u32_u24 v0, s12, v3, v0
-; GFX7-NEXT:    v_mad_u32_u24 v0, s7, v4, v0
-; GFX7-NEXT:    v_mad_u32_u24 v0, s6, v5, v0
-; GFX7-NEXT:    v_mad_u32_u24 v0, s5, v6, v0
-; GFX7-NEXT:    v_mad_u32_u24 v0, s4, v7, v0
+; GFX7-NEXT:    v_mad_u32_u24 v0, s11, v4, v0
+; GFX7-NEXT:    v_mad_u32_u24 v0, s10, v5, v0
+; GFX7-NEXT:    v_mad_u32_u24 v0, s9, v6, v0
+; GFX7-NEXT:    v_mad_u32_u24 v0, s8, v7, v0
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s14
 ; GFX7-NEXT:    v_mad_u32_u24 v0, s2, v1, v0
 ; GFX7-NEXT:    v_and_b32_e32 v0, 15, v0
-; GFX7-NEXT:    buffer_store_byte v0, off, s[8:11], 0
+; GFX7-NEXT:    buffer_store_byte v0, off, s[4:7], 0
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: udot8_CommutationInsideMAD:
@@ -1208,48 +1208,48 @@ define amdgpu_kernel void @udot8_CommutationInsideMAD(<8 x i4> addrspace(1)* %sr
 ; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX8-NEXT:    s_load_dword s4, s[6:7], 0x0
+; GFX8-NEXT:    s_load_dword s2, s[6:7], 0x0
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX8-NEXT:    flat_load_ubyte v2, v[0:1]
+; GFX8-NEXT:    s_load_dword s0, s[4:5], 0x0
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_and_b32 s0, s2, 15
-; GFX8-NEXT:    s_and_b32 s1, s4, 15
+; GFX8-NEXT:    s_and_b32 s1, s2, 15
+; GFX8-NEXT:    s_bfe_u32 s4, s2, 0x40004
 ; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    s_bfe_u32 s5, s4, 0x40004
-; GFX8-NEXT:    s_bfe_u32 s6, s4, 0x40008
-; GFX8-NEXT:    s_bfe_u32 s7, s4, 0x4000c
-; GFX8-NEXT:    v_mov_b32_e32 v4, s5
-; GFX8-NEXT:    s_bfe_u32 s1, s2, 0x40004
-; GFX8-NEXT:    s_bfe_u32 s5, s2, 0x40008
-; GFX8-NEXT:    s_bfe_u32 s8, s2, 0x4000c
+; GFX8-NEXT:    s_bfe_u32 s6, s2, 0x40008
+; GFX8-NEXT:    s_and_b32 s1, s0, 15
+; GFX8-NEXT:    s_bfe_u32 s7, s2, 0x4000c
+; GFX8-NEXT:    v_mov_b32_e32 v4, s4
+; GFX8-NEXT:    s_bfe_u32 s5, s0, 0x40004
 ; GFX8-NEXT:    v_mov_b32_e32 v5, s7
+; GFX8-NEXT:    s_bfe_u32 s8, s0, 0x4000c
 ; GFX8-NEXT:    v_mov_b32_e32 v6, s6
+; GFX8-NEXT:    s_bfe_u32 s4, s0, 0x40008
 ; GFX8-NEXT:    v_mul_u32_u24_e32 v5, s8, v5
-; GFX8-NEXT:    s_bfe_u32 s9, s4, 0x40010
+; GFX8-NEXT:    s_bfe_u32 s6, s2, 0x40010
 ; GFX8-NEXT:    v_and_b32_e32 v5, 15, v5
-; GFX8-NEXT:    s_bfe_u32 s11, s4, 0x40014
-; GFX8-NEXT:    s_bfe_u32 s10, s2, 0x40010
-; GFX8-NEXT:    v_mov_b32_e32 v7, s9
-; GFX8-NEXT:    s_bfe_u32 s13, s4, 0x40018
-; GFX8-NEXT:    s_bfe_u32 s12, s2, 0x40014
-; GFX8-NEXT:    v_mov_b32_e32 v8, s11
-; GFX8-NEXT:    s_bfe_u32 s14, s2, 0x40018
-; GFX8-NEXT:    s_lshr_b32 s4, s4, 28
-; GFX8-NEXT:    v_mov_b32_e32 v9, s13
-; GFX8-NEXT:    s_lshr_b32 s2, s2, 28
+; GFX8-NEXT:    s_bfe_u32 s8, s2, 0x40014
+; GFX8-NEXT:    s_bfe_u32 s7, s0, 0x40010
+; GFX8-NEXT:    v_mov_b32_e32 v7, s6
+; GFX8-NEXT:    s_lshr_b32 s11, s2, 28
+; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x40018
+; GFX8-NEXT:    s_bfe_u32 s9, s0, 0x40014
+; GFX8-NEXT:    v_mov_b32_e32 v8, s8
+; GFX8-NEXT:    s_bfe_u32 s10, s0, 0x40018
+; GFX8-NEXT:    v_mov_b32_e32 v9, s2
+; GFX8-NEXT:    s_lshr_b32 s0, s0, 28
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
-; GFX8-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
-; GFX8-NEXT:    v_mad_u32_u24 v2, s1, v4, v2
-; GFX8-NEXT:    v_mad_u32_u24 v2, s5, v6, v2
+; GFX8-NEXT:    v_mad_u32_u24 v2, s1, v3, v2
+; GFX8-NEXT:    v_mad_u32_u24 v2, s5, v4, v2
+; GFX8-NEXT:    v_mad_u32_u24 v2, s4, v6, v2
 ; GFX8-NEXT:    v_and_b32_e32 v2, 15, v2
 ; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v2, v5
-; GFX8-NEXT:    v_mad_u32_u24 v2, s10, v7, v2
-; GFX8-NEXT:    v_mad_u32_u24 v2, s12, v8, v2
-; GFX8-NEXT:    v_mad_u32_u24 v2, s14, v9, v2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s4
-; GFX8-NEXT:    v_mad_u32_u24 v2, s2, v3, v2
+; GFX8-NEXT:    v_mad_u32_u24 v2, s7, v7, v2
+; GFX8-NEXT:    v_mad_u32_u24 v2, s9, v8, v2
+; GFX8-NEXT:    v_mad_u32_u24 v2, s10, v9, v2
+; GFX8-NEXT:    v_mov_b32_e32 v3, s11
+; GFX8-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
 ; GFX8-NEXT:    v_and_b32_e32 v2, 15, v2
 ; GFX8-NEXT:    flat_store_byte v[0:1], v2
 ; GFX8-NEXT:    s_endpgm
@@ -1259,48 +1259,48 @@ define amdgpu_kernel void @udot8_CommutationInsideMAD(<8 x i4> addrspace(1)* %sr
 ; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX9-NEXT:    s_load_dword s4, s[6:7], 0x0
+; GFX9-NEXT:    s_load_dword s2, s[6:7], 0x0
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-NEXT:    global_load_ubyte v2, v[0:1], off
+; GFX9-NEXT:    s_load_dword s0, s[4:5], 0x0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_and_b32 s0, s2, 15
-; GFX9-NEXT:    s_and_b32 s1, s4, 15
+; GFX9-NEXT:    s_and_b32 s1, s2, 15
+; GFX9-NEXT:    s_bfe_u32 s4, s2, 0x40004
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s1
-; GFX9-NEXT:    s_bfe_u32 s5, s4, 0x40004
-; GFX9-NEXT:    s_bfe_u32 s6, s4, 0x40008
-; GFX9-NEXT:    s_bfe_u32 s7, s4, 0x4000c
-; GFX9-NEXT:    v_mov_b32_e32 v4, s5
-; GFX9-NEXT:    s_bfe_u32 s1, s2, 0x40004
-; GFX9-NEXT:    s_bfe_u32 s5, s2, 0x40008
-; GFX9-NEXT:    s_bfe_u32 s8, s2, 0x4000c
+; GFX9-NEXT:    s_bfe_u32 s6, s2, 0x40008
+; GFX9-NEXT:    s_and_b32 s1, s0, 15
+; GFX9-NEXT:    s_bfe_u32 s7, s2, 0x4000c
+; GFX9-NEXT:    v_mov_b32_e32 v4, s4
+; GFX9-NEXT:    s_bfe_u32 s5, s0, 0x40004
 ; GFX9-NEXT:    v_mov_b32_e32 v5, s7
+; GFX9-NEXT:    s_bfe_u32 s8, s0, 0x4000c
 ; GFX9-NEXT:    v_mov_b32_e32 v6, s6
+; GFX9-NEXT:    s_bfe_u32 s4, s0, 0x40008
 ; GFX9-NEXT:    v_mul_u32_u24_e32 v5, s8, v5
-; GFX9-NEXT:    s_bfe_u32 s9, s4, 0x40010
+; GFX9-NEXT:    s_bfe_u32 s6, s2, 0x40010
 ; GFX9-NEXT:    v_and_b32_e32 v5, 15, v5
-; GFX9-NEXT:    s_bfe_u32 s11, s4, 0x40014
-; GFX9-NEXT:    s_bfe_u32 s10, s2, 0x40010
-; GFX9-NEXT:    v_mov_b32_e32 v7, s9
-; GFX9-NEXT:    s_bfe_u32 s13, s4, 0x40018
-; GFX9-NEXT:    s_bfe_u32 s12, s2, 0x40014
-; GFX9-NEXT:    v_mov_b32_e32 v8, s11
-; GFX9-NEXT:    s_bfe_u32 s14, s2, 0x40018
-; GFX9-NEXT:    s_lshr_b32 s4, s4, 28
-; GFX9-NEXT:    v_mov_b32_e32 v9, s13
-; GFX9-NEXT:    s_lshr_b32 s2, s2, 28
+; GFX9-NEXT:    s_bfe_u32 s8, s2, 0x40014
+; GFX9-NEXT:    s_bfe_u32 s7, s0, 0x40010
+; GFX9-NEXT:    v_mov_b32_e32 v7, s6
+; GFX9-NEXT:    s_lshr_b32 s11, s2, 28
+; GFX9-NEXT:    s_bfe_u32 s2, s2, 0x40018
+; GFX9-NEXT:    s_bfe_u32 s9, s0, 0x40014
+; GFX9-NEXT:    v_mov_b32_e32 v8, s8
+; GFX9-NEXT:    s_bfe_u32 s10, s0, 0x40018
+; GFX9-NEXT:    v_mov_b32_e32 v9, s2
+; GFX9-NEXT:    s_lshr_b32 s0, s0, 28
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
-; GFX9-NEXT:    v_mad_u32_u24 v2, s1, v4, v2
-; GFX9-NEXT:    v_mad_u32_u24 v2, s5, v6, v2
+; GFX9-NEXT:    v_mad_u32_u24 v2, s1, v3, v2
+; GFX9-NEXT:    v_mad_u32_u24 v2, s5, v4, v2
+; GFX9-NEXT:    v_mad_u32_u24 v2, s4, v6, v2
 ; GFX9-NEXT:    v_and_b32_e32 v2, 15, v2
 ; GFX9-NEXT:    v_add_u32_e32 v2, v5, v2
-; GFX9-NEXT:    v_mad_u32_u24 v2, s10, v7, v2
-; GFX9-NEXT:    v_mad_u32_u24 v2, s12, v8, v2
-; GFX9-NEXT:    v_mad_u32_u24 v2, s14, v9, v2
-; GFX9-NEXT:    v_mov_b32_e32 v3, s4
-; GFX9-NEXT:    v_mad_u32_u24 v2, s2, v3, v2
+; GFX9-NEXT:    v_mad_u32_u24 v2, s7, v7, v2
+; GFX9-NEXT:    v_mad_u32_u24 v2, s9, v8, v2
+; GFX9-NEXT:    v_mad_u32_u24 v2, s10, v9, v2
+; GFX9-NEXT:    v_mov_b32_e32 v3, s11
+; GFX9-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
 ; GFX9-NEXT:    v_and_b32_e32 v2, 15, v2
 ; GFX9-NEXT:    global_store_byte v[0:1], v2, off
 ; GFX9-NEXT:    s_endpgm
@@ -1310,48 +1310,48 @@ define amdgpu_kernel void @udot8_CommutationInsideMAD(<8 x i4> addrspace(1)* %sr
 ; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
+; GFX9-DL-NEXT:    s_load_dword s2, s[6:7], 0x0
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-DL-NEXT:    global_load_ubyte v2, v[0:1], off
+; GFX9-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    s_and_b32 s0, s2, 15
-; GFX9-DL-NEXT:    s_and_b32 s1, s4, 15
+; GFX9-DL-NEXT:    s_and_b32 s1, s2, 15
+; GFX9-DL-NEXT:    s_bfe_u32 s4, s2, 0x40004
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s1
-; GFX9-DL-NEXT:    s_bfe_u32 s5, s4, 0x40004
-; GFX9-DL-NEXT:    s_bfe_u32 s6, s4, 0x40008
-; GFX9-DL-NEXT:    s_bfe_u32 s7, s4, 0x4000c
-; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s5
-; GFX9-DL-NEXT:    s_bfe_u32 s1, s2, 0x40004
-; GFX9-DL-NEXT:    s_bfe_u32 s5, s2, 0x40008
-; GFX9-DL-NEXT:    s_bfe_u32 s8, s2, 0x4000c
+; GFX9-DL-NEXT:    s_bfe_u32 s6, s2, 0x40008
+; GFX9-DL-NEXT:    s_and_b32 s1, s0, 15
+; GFX9-DL-NEXT:    s_bfe_u32 s7, s2, 0x4000c
+; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s4
+; GFX9-DL-NEXT:    s_bfe_u32 s5, s0, 0x40004
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v5, s7
+; GFX9-DL-NEXT:    s_bfe_u32 s8, s0, 0x4000c
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v6, s6
+; GFX9-DL-NEXT:    s_bfe_u32 s4, s0, 0x40008
 ; GFX9-DL-NEXT:    v_mul_u32_u24_e32 v5, s8, v5
-; GFX9-DL-NEXT:    s_bfe_u32 s9, s4, 0x40010
+; GFX9-DL-NEXT:    s_bfe_u32 s6, s2, 0x40010
 ; GFX9-DL-NEXT:    v_and_b32_e32 v5, 15, v5
-; GFX9-DL-NEXT:    s_bfe_u32 s11, s4, 0x40014
-; GFX9-DL-NEXT:    s_bfe_u32 s10, s2, 0x40010
-; GFX9-DL-NEXT:    v_mov_b32_e32 v7, s9
-; GFX9-DL-NEXT:    s_bfe_u32 s13, s4, 0x40018
-; GFX9-DL-NEXT:    s_bfe_u32 s12, s2, 0x40014
-; GFX9-DL-NEXT:    v_mov_b32_e32 v8, s11
-; GFX9-DL-NEXT:    s_bfe_u32 s14, s2, 0x40018
-; GFX9-DL-NEXT:    s_lshr_b32 s4, s4, 28
-; GFX9-DL-NEXT:    v_mov_b32_e32 v9, s13
-; GFX9-DL-NEXT:    s_lshr_b32 s2, s2, 28
+; GFX9-DL-NEXT:    s_bfe_u32 s8, s2, 0x40014
+; GFX9-DL-NEXT:    s_bfe_u32 s7, s0, 0x40010
+; GFX9-DL-NEXT:    v_mov_b32_e32 v7, s6
+; GFX9-DL-NEXT:    s_lshr_b32 s11, s2, 28
+; GFX9-DL-NEXT:    s_bfe_u32 s2, s2, 0x40018
+; GFX9-DL-NEXT:    s_bfe_u32 s9, s0, 0x40014
+; GFX9-DL-NEXT:    v_mov_b32_e32 v8, s8
+; GFX9-DL-NEXT:    s_bfe_u32 s10, s0, 0x40018
+; GFX9-DL-NEXT:    v_mov_b32_e32 v9, s2
+; GFX9-DL-NEXT:    s_lshr_b32 s0, s0, 28
 ; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
-; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s1, v4, v2
-; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s5, v6, v2
+; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s1, v3, v2
+; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s5, v4, v2
+; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s4, v6, v2
 ; GFX9-DL-NEXT:    v_and_b32_e32 v2, 15, v2
 ; GFX9-DL-NEXT:    v_add_u32_e32 v2, v5, v2
-; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s10, v7, v2
-; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s12, v8, v2
-; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s14, v9, v2
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s4
-; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s2, v3, v2
+; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s7, v7, v2
+; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s9, v8, v2
+; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s10, v9, v2
+; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s11
+; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
 ; GFX9-DL-NEXT:    v_and_b32_e32 v2, 15, v2
 ; GFX9-DL-NEXT:    global_store_byte v[0:1], v2, off
 ; GFX9-DL-NEXT:    s_endpgm
@@ -1362,40 +1362,40 @@ define amdgpu_kernel void @udot8_CommutationInsideMAD(<8 x i4> addrspace(1)* %sr
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX10-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
+; GFX10-DL-NEXT:    s_load_dword s1, s[6:7], 0x0
 ; GFX10-DL-NEXT:    global_load_ubyte v2, v[0:1], off
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_and_b32 s0, s2, 15
-; GFX10-DL-NEXT:    s_and_b32 s1, s4, 15
-; GFX10-DL-NEXT:    s_bfe_u32 s5, s2, 0x40004
-; GFX10-DL-NEXT:    s_bfe_u32 s6, s4, 0x40004
-; GFX10-DL-NEXT:    s_bfe_u32 s7, s2, 0x40008
-; GFX10-DL-NEXT:    s_bfe_u32 s8, s2, 0x4000c
+; GFX10-DL-NEXT:    s_and_b32 s2, s0, 15
+; GFX10-DL-NEXT:    s_and_b32 s4, s1, 15
+; GFX10-DL-NEXT:    s_bfe_u32 s5, s0, 0x40004
+; GFX10-DL-NEXT:    s_bfe_u32 s6, s1, 0x40004
+; GFX10-DL-NEXT:    s_bfe_u32 s7, s1, 0x40008
+; GFX10-DL-NEXT:    s_bfe_u32 s8, s1, 0x4000c
 ; GFX10-DL-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s0, s1, v2
-; GFX10-DL-NEXT:    s_bfe_u32 s1, s4, 0x4000c
-; GFX10-DL-NEXT:    s_bfe_u32 s0, s4, 0x40008
+; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s2, s4, v2
+; GFX10-DL-NEXT:    s_bfe_u32 s4, s0, 0x4000c
+; GFX10-DL-NEXT:    s_bfe_u32 s2, s0, 0x40008
 ; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s5, s6, v2
-; GFX10-DL-NEXT:    v_mul_u32_u24_e64 v3, s8, s1
-; GFX10-DL-NEXT:    s_bfe_u32 s1, s4, 0x40010
-; GFX10-DL-NEXT:    s_bfe_u32 s5, s2, 0x40014
-; GFX10-DL-NEXT:    s_bfe_u32 s6, s4, 0x40014
-; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s7, s0, v2
+; GFX10-DL-NEXT:    v_mul_u32_u24_e64 v3, s4, s8
+; GFX10-DL-NEXT:    s_bfe_u32 s4, s1, 0x40010
+; GFX10-DL-NEXT:    s_bfe_u32 s5, s0, 0x40014
+; GFX10-DL-NEXT:    s_bfe_u32 s6, s1, 0x40014
+; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s2, s7, v2
 ; GFX10-DL-NEXT:    v_and_b32_e32 v3, 15, v3
-; GFX10-DL-NEXT:    s_bfe_u32 s0, s2, 0x40010
+; GFX10-DL-NEXT:    s_bfe_u32 s2, s0, 0x40010
 ; GFX10-DL-NEXT:    v_and_b32_e32 v2, 15, v2
 ; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v3, v2
-; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s0, s1, v2
-; GFX10-DL-NEXT:    s_bfe_u32 s0, s2, 0x40018
-; GFX10-DL-NEXT:    s_bfe_u32 s1, s4, 0x40018
-; GFX10-DL-NEXT:    s_lshr_b32 s2, s2, 28
-; GFX10-DL-NEXT:    s_lshr_b32 s4, s4, 28
+; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s2, s4, v2
+; GFX10-DL-NEXT:    s_bfe_u32 s2, s0, 0x40018
+; GFX10-DL-NEXT:    s_bfe_u32 s4, s1, 0x40018
+; GFX10-DL-NEXT:    s_lshr_b32 s0, s0, 28
+; GFX10-DL-NEXT:    s_lshr_b32 s1, s1, 28
 ; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s5, s6, v2
-; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s0, s1, v2
 ; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s2, s4, v2
+; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s0, s1, v2
 ; GFX10-DL-NEXT:    v_and_b32_e32 v2, 15, v2
 ; GFX10-DL-NEXT:    global_store_byte v[0:1], v2, off
 ; GFX10-DL-NEXT:    s_endpgm
@@ -1460,43 +1460,43 @@ define amdgpu_kernel void @udot8_multiuses_mul1(<8 x i4> addrspace(1)* %src1,
 ; GFX7-NEXT:    s_mov_b32 s6, -1
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    s_load_dword s0, s[8:9], 0x0
-; GFX7-NEXT:    s_load_dword s1, s[10:11], 0x0
-; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s10, s[10:11], 0x0
+; GFX7-NEXT:    s_load_dword s21, s[4:5], 0x0
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_lshr_b32 s8, s0, 28
-; GFX7-NEXT:    s_bfe_u32 s21, s1, 0x40004
-; GFX7-NEXT:    s_lshr_b32 s15, s1, 28
-; GFX7-NEXT:    s_bfe_u32 s16, s1, 0x40018
-; GFX7-NEXT:    s_bfe_u32 s17, s1, 0x40014
-; GFX7-NEXT:    s_bfe_u32 s18, s1, 0x40010
-; GFX7-NEXT:    s_bfe_u32 s19, s1, 0x4000c
-; GFX7-NEXT:    s_bfe_u32 s20, s1, 0x40008
-; GFX7-NEXT:    s_and_b32 s1, s1, 15
-; GFX7-NEXT:    s_bfe_u32 s9, s0, 0x40018
-; GFX7-NEXT:    s_bfe_u32 s10, s0, 0x40014
-; GFX7-NEXT:    s_bfe_u32 s11, s0, 0x40010
+; GFX7-NEXT:    s_lshr_b32 s1, s0, 28
+; GFX7-NEXT:    s_bfe_u32 s20, s10, 0x40004
+; GFX7-NEXT:    s_lshr_b32 s11, s10, 28
+; GFX7-NEXT:    s_bfe_u32 s15, s10, 0x40018
+; GFX7-NEXT:    s_bfe_u32 s16, s10, 0x40014
+; GFX7-NEXT:    s_bfe_u32 s17, s10, 0x40010
+; GFX7-NEXT:    s_bfe_u32 s18, s10, 0x4000c
+; GFX7-NEXT:    s_bfe_u32 s19, s10, 0x40008
+; GFX7-NEXT:    s_and_b32 s10, s10, 15
+; GFX7-NEXT:    s_bfe_u32 s2, s0, 0x40018
+; GFX7-NEXT:    s_bfe_u32 s8, s0, 0x40014
+; GFX7-NEXT:    s_bfe_u32 s9, s0, 0x40010
 ; GFX7-NEXT:    s_bfe_u32 s12, s0, 0x4000c
 ; GFX7-NEXT:    s_bfe_u32 s13, s0, 0x40008
 ; GFX7-NEXT:    s_bfe_u32 s14, s0, 0x40004
 ; GFX7-NEXT:    s_and_b32 s0, s0, 15
-; GFX7-NEXT:    v_mov_b32_e32 v0, s1
-; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s10
+; GFX7-NEXT:    v_mov_b32_e32 v1, s21
 ; GFX7-NEXT:    v_mad_u32_u24 v1, s0, v0, v1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s21
+; GFX7-NEXT:    v_mov_b32_e32 v2, s20
 ; GFX7-NEXT:    v_mad_u32_u24 v0, s0, v0, v1
 ; GFX7-NEXT:    v_mad_u32_u24 v1, s14, v2, v1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s20
-; GFX7-NEXT:    v_mad_u32_u24 v1, s13, v2, v1
 ; GFX7-NEXT:    v_mov_b32_e32 v2, s19
-; GFX7-NEXT:    v_mad_u32_u24 v1, s12, v2, v1
+; GFX7-NEXT:    v_mad_u32_u24 v1, s13, v2, v1
 ; GFX7-NEXT:    v_mov_b32_e32 v2, s18
-; GFX7-NEXT:    v_mad_u32_u24 v1, s11, v2, v1
+; GFX7-NEXT:    v_mad_u32_u24 v1, s12, v2, v1
 ; GFX7-NEXT:    v_mov_b32_e32 v2, s17
-; GFX7-NEXT:    v_mad_u32_u24 v1, s10, v2, v1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s16
 ; GFX7-NEXT:    v_mad_u32_u24 v1, s9, v2, v1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s15
+; GFX7-NEXT:    v_mov_b32_e32 v2, s16
 ; GFX7-NEXT:    v_mad_u32_u24 v1, s8, v2, v1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s15
+; GFX7-NEXT:    v_mad_u32_u24 v1, s2, v2, v1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s11
+; GFX7-NEXT:    v_mad_u32_u24 v1, s1, v2, v1
 ; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v1, v0
 ; GFX7-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 ; GFX7-NEXT:    s_endpgm
@@ -1507,46 +1507,46 @@ define amdgpu_kernel void @udot8_multiuses_mul1(<8 x i4> addrspace(1)* %src1,
 ; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX8-NEXT:    s_load_dword s4, s[6:7], 0x0
-; GFX8-NEXT:    s_load_dword s5, s[0:1], 0x0
-; GFX8-NEXT:    v_mov_b32_e32 v0, s0
-; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    s_load_dword s6, s[6:7], 0x0
+; GFX8-NEXT:    s_load_dword s19, s[0:1], 0x0
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_lshr_b32 s0, s2, 28
-; GFX8-NEXT:    s_bfe_u32 s17, s4, 0x40004
-; GFX8-NEXT:    s_lshr_b32 s11, s4, 28
-; GFX8-NEXT:    s_bfe_u32 s12, s4, 0x40018
-; GFX8-NEXT:    s_bfe_u32 s13, s4, 0x40014
-; GFX8-NEXT:    s_bfe_u32 s14, s4, 0x40010
-; GFX8-NEXT:    s_bfe_u32 s15, s4, 0x4000c
-; GFX8-NEXT:    s_bfe_u32 s16, s4, 0x40008
-; GFX8-NEXT:    s_and_b32 s4, s4, 15
-; GFX8-NEXT:    s_bfe_u32 s1, s2, 0x40018
-; GFX8-NEXT:    s_bfe_u32 s6, s2, 0x40014
-; GFX8-NEXT:    s_bfe_u32 s7, s2, 0x40010
-; GFX8-NEXT:    s_bfe_u32 s8, s2, 0x4000c
-; GFX8-NEXT:    s_bfe_u32 s9, s2, 0x40008
-; GFX8-NEXT:    s_bfe_u32 s10, s2, 0x40004
+; GFX8-NEXT:    s_lshr_b32 s4, s2, 28
+; GFX8-NEXT:    s_bfe_u32 s18, s6, 0x40004
+; GFX8-NEXT:    s_lshr_b32 s7, s6, 28
+; GFX8-NEXT:    s_bfe_u32 s13, s6, 0x40018
+; GFX8-NEXT:    s_bfe_u32 s14, s6, 0x40014
+; GFX8-NEXT:    s_bfe_u32 s15, s6, 0x40010
+; GFX8-NEXT:    s_bfe_u32 s16, s6, 0x4000c
+; GFX8-NEXT:    s_bfe_u32 s17, s6, 0x40008
+; GFX8-NEXT:    s_and_b32 s6, s6, 15
+; GFX8-NEXT:    s_bfe_u32 s5, s2, 0x40018
+; GFX8-NEXT:    s_bfe_u32 s8, s2, 0x40014
+; GFX8-NEXT:    s_bfe_u32 s9, s2, 0x40010
+; GFX8-NEXT:    s_bfe_u32 s10, s2, 0x4000c
+; GFX8-NEXT:    s_bfe_u32 s11, s2, 0x40008
+; GFX8-NEXT:    s_bfe_u32 s12, s2, 0x40004
 ; GFX8-NEXT:    s_and_b32 s2, s2, 15
-; GFX8-NEXT:    v_mov_b32_e32 v2, s4
-; GFX8-NEXT:    v_mov_b32_e32 v3, s5
-; GFX8-NEXT:    v_mad_u32_u24 v3, s2, v2, v3
-; GFX8-NEXT:    v_mov_b32_e32 v4, s17
-; GFX8-NEXT:    v_mad_u32_u24 v2, s2, v2, v3
-; GFX8-NEXT:    v_mad_u32_u24 v3, s10, v4, v3
-; GFX8-NEXT:    v_mov_b32_e32 v4, s16
-; GFX8-NEXT:    v_mad_u32_u24 v3, s9, v4, v3
-; GFX8-NEXT:    v_mov_b32_e32 v4, s15
-; GFX8-NEXT:    v_mad_u32_u24 v3, s8, v4, v3
-; GFX8-NEXT:    v_mov_b32_e32 v4, s14
-; GFX8-NEXT:    v_mad_u32_u24 v3, s7, v4, v3
-; GFX8-NEXT:    v_mov_b32_e32 v4, s13
-; GFX8-NEXT:    v_mad_u32_u24 v3, s6, v4, v3
-; GFX8-NEXT:    v_mov_b32_e32 v4, s12
-; GFX8-NEXT:    v_mad_u32_u24 v3, s1, v4, v3
-; GFX8-NEXT:    v_mov_b32_e32 v4, s11
-; GFX8-NEXT:    v_mad_u32_u24 v3, s0, v4, v3
-; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v3, v2
+; GFX8-NEXT:    v_mov_b32_e32 v0, s6
+; GFX8-NEXT:    v_mov_b32_e32 v1, s19
+; GFX8-NEXT:    v_mad_u32_u24 v1, s2, v0, v1
+; GFX8-NEXT:    v_mov_b32_e32 v2, s18
+; GFX8-NEXT:    v_mad_u32_u24 v0, s2, v0, v1
+; GFX8-NEXT:    v_mad_u32_u24 v1, s12, v2, v1
+; GFX8-NEXT:    v_mov_b32_e32 v2, s17
+; GFX8-NEXT:    v_mad_u32_u24 v1, s11, v2, v1
+; GFX8-NEXT:    v_mov_b32_e32 v2, s16
+; GFX8-NEXT:    v_mad_u32_u24 v1, s10, v2, v1
+; GFX8-NEXT:    v_mov_b32_e32 v2, s15
+; GFX8-NEXT:    v_mad_u32_u24 v1, s9, v2, v1
+; GFX8-NEXT:    v_mov_b32_e32 v2, s14
+; GFX8-NEXT:    v_mad_u32_u24 v1, s8, v2, v1
+; GFX8-NEXT:    v_mov_b32_e32 v2, s13
+; GFX8-NEXT:    v_mad_u32_u24 v1, s5, v2, v1
+; GFX8-NEXT:    v_mov_b32_e32 v2, s7
+; GFX8-NEXT:    v_mad_u32_u24 v1, s4, v2, v1
+; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v1, v0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX8-NEXT:    flat_store_dword v[0:1], v2
 ; GFX8-NEXT:    s_endpgm
 ;
@@ -1556,46 +1556,46 @@ define amdgpu_kernel void @udot8_multiuses_mul1(<8 x i4> addrspace(1)* %src1,
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX9-NEXT:    s_load_dword s4, s[6:7], 0x0
-; GFX9-NEXT:    s_load_dword s5, s[0:1], 0x0
-; GFX9-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    s_load_dword s6, s[6:7], 0x0
+; GFX9-NEXT:    s_load_dword s19, s[0:1], 0x0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_lshr_b32 s0, s2, 28
-; GFX9-NEXT:    s_bfe_u32 s17, s4, 0x40004
-; GFX9-NEXT:    s_lshr_b32 s11, s4, 28
-; GFX9-NEXT:    s_bfe_u32 s12, s4, 0x40018
-; GFX9-NEXT:    s_bfe_u32 s13, s4, 0x40014
-; GFX9-NEXT:    s_bfe_u32 s14, s4, 0x40010
-; GFX9-NEXT:    s_bfe_u32 s15, s4, 0x4000c
-; GFX9-NEXT:    s_bfe_u32 s16, s4, 0x40008
-; GFX9-NEXT:    s_and_b32 s4, s4, 15
-; GFX9-NEXT:    s_bfe_u32 s1, s2, 0x40018
-; GFX9-NEXT:    s_bfe_u32 s6, s2, 0x40014
-; GFX9-NEXT:    s_bfe_u32 s7, s2, 0x40010
-; GFX9-NEXT:    s_bfe_u32 s8, s2, 0x4000c
-; GFX9-NEXT:    s_bfe_u32 s9, s2, 0x40008
-; GFX9-NEXT:    s_bfe_u32 s10, s2, 0x40004
+; GFX9-NEXT:    s_lshr_b32 s4, s2, 28
+; GFX9-NEXT:    s_bfe_u32 s18, s6, 0x40004
+; GFX9-NEXT:    s_lshr_b32 s7, s6, 28
+; GFX9-NEXT:    s_bfe_u32 s13, s6, 0x40018
+; GFX9-NEXT:    s_bfe_u32 s14, s6, 0x40014
+; GFX9-NEXT:    s_bfe_u32 s15, s6, 0x40010
+; GFX9-NEXT:    s_bfe_u32 s16, s6, 0x4000c
+; GFX9-NEXT:    s_bfe_u32 s17, s6, 0x40008
+; GFX9-NEXT:    s_and_b32 s6, s6, 15
+; GFX9-NEXT:    s_bfe_u32 s5, s2, 0x40018
+; GFX9-NEXT:    s_bfe_u32 s8, s2, 0x40014
+; GFX9-NEXT:    s_bfe_u32 s9, s2, 0x40010
+; GFX9-NEXT:    s_bfe_u32 s10, s2, 0x4000c
+; GFX9-NEXT:    s_bfe_u32 s11, s2, 0x40008
+; GFX9-NEXT:    s_bfe_u32 s12, s2, 0x40004
 ; GFX9-NEXT:    s_and_b32 s2, s2, 15
-; GFX9-NEXT:    v_mov_b32_e32 v2, s4
-; GFX9-NEXT:    v_mov_b32_e32 v3, s5
-; GFX9-NEXT:    v_mad_u32_u24 v3, s2, v2, v3
-; GFX9-NEXT:    v_mov_b32_e32 v4, s17
-; GFX9-NEXT:    v_mad_u32_u24 v2, s2, v2, v3
-; GFX9-NEXT:    v_mad_u32_u24 v3, s10, v4, v3
-; GFX9-NEXT:    v_mov_b32_e32 v4, s16
-; GFX9-NEXT:    v_mad_u32_u24 v3, s9, v4, v3
-; GFX9-NEXT:    v_mov_b32_e32 v4, s15
-; GFX9-NEXT:    v_mad_u32_u24 v3, s8, v4, v3
-; GFX9-NEXT:    v_mov_b32_e32 v4, s14
-; GFX9-NEXT:    v_mad_u32_u24 v3, s7, v4, v3
-; GFX9-NEXT:    v_mov_b32_e32 v4, s13
-; GFX9-NEXT:    v_mad_u32_u24 v3, s6, v4, v3
-; GFX9-NEXT:    v_mov_b32_e32 v4, s12
-; GFX9-NEXT:    v_mad_u32_u24 v3, s1, v4, v3
-; GFX9-NEXT:    v_mov_b32_e32 v4, s11
-; GFX9-NEXT:    v_mad_u32_u24 v3, s0, v4, v3
-; GFX9-NEXT:    v_add_u32_e32 v2, v2, v3
+; GFX9-NEXT:    v_mov_b32_e32 v0, s6
+; GFX9-NEXT:    v_mov_b32_e32 v1, s19
+; GFX9-NEXT:    v_mad_u32_u24 v1, s2, v0, v1
+; GFX9-NEXT:    v_mov_b32_e32 v2, s18
+; GFX9-NEXT:    v_mad_u32_u24 v0, s2, v0, v1
+; GFX9-NEXT:    v_mad_u32_u24 v1, s12, v2, v1
+; GFX9-NEXT:    v_mov_b32_e32 v2, s17
+; GFX9-NEXT:    v_mad_u32_u24 v1, s11, v2, v1
+; GFX9-NEXT:    v_mov_b32_e32 v2, s16
+; GFX9-NEXT:    v_mad_u32_u24 v1, s10, v2, v1
+; GFX9-NEXT:    v_mov_b32_e32 v2, s15
+; GFX9-NEXT:    v_mad_u32_u24 v1, s9, v2, v1
+; GFX9-NEXT:    v_mov_b32_e32 v2, s14
+; GFX9-NEXT:    v_mad_u32_u24 v1, s8, v2, v1
+; GFX9-NEXT:    v_mov_b32_e32 v2, s13
+; GFX9-NEXT:    v_mad_u32_u24 v1, s5, v2, v1
+; GFX9-NEXT:    v_mov_b32_e32 v2, s7
+; GFX9-NEXT:    v_mad_u32_u24 v1, s4, v2, v1
+; GFX9-NEXT:    v_add_u32_e32 v2, v0, v1
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-NEXT:    s_endpgm
 ;
@@ -1605,46 +1605,46 @@ define amdgpu_kernel void @udot8_multiuses_mul1(<8 x i4> addrspace(1)* %src1,
 ; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
-; GFX9-DL-NEXT:    s_load_dword s5, s[0:1], 0x0
-; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-DL-NEXT:    s_load_dword s6, s[6:7], 0x0
+; GFX9-DL-NEXT:    s_load_dword s19, s[0:1], 0x0
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    s_lshr_b32 s0, s2, 28
-; GFX9-DL-NEXT:    s_bfe_u32 s17, s4, 0x40004
-; GFX9-DL-NEXT:    s_lshr_b32 s11, s4, 28
-; GFX9-DL-NEXT:    s_bfe_u32 s12, s4, 0x40018
-; GFX9-DL-NEXT:    s_bfe_u32 s13, s4, 0x40014
-; GFX9-DL-NEXT:    s_bfe_u32 s14, s4, 0x40010
-; GFX9-DL-NEXT:    s_bfe_u32 s15, s4, 0x4000c
-; GFX9-DL-NEXT:    s_bfe_u32 s16, s4, 0x40008
-; GFX9-DL-NEXT:    s_and_b32 s4, s4, 15
-; GFX9-DL-NEXT:    s_bfe_u32 s1, s2, 0x40018
-; GFX9-DL-NEXT:    s_bfe_u32 s6, s2, 0x40014
-; GFX9-DL-NEXT:    s_bfe_u32 s7, s2, 0x40010
-; GFX9-DL-NEXT:    s_bfe_u32 s8, s2, 0x4000c
-; GFX9-DL-NEXT:    s_bfe_u32 s9, s2, 0x40008
-; GFX9-DL-NEXT:    s_bfe_u32 s10, s2, 0x40004
+; GFX9-DL-NEXT:    s_lshr_b32 s4, s2, 28
+; GFX9-DL-NEXT:    s_bfe_u32 s18, s6, 0x40004
+; GFX9-DL-NEXT:    s_lshr_b32 s7, s6, 28
+; GFX9-DL-NEXT:    s_bfe_u32 s13, s6, 0x40018
+; GFX9-DL-NEXT:    s_bfe_u32 s14, s6, 0x40014
+; GFX9-DL-NEXT:    s_bfe_u32 s15, s6, 0x40010
+; GFX9-DL-NEXT:    s_bfe_u32 s16, s6, 0x4000c
+; GFX9-DL-NEXT:    s_bfe_u32 s17, s6, 0x40008
+; GFX9-DL-NEXT:    s_and_b32 s6, s6, 15
+; GFX9-DL-NEXT:    s_bfe_u32 s5, s2, 0x40018
+; GFX9-DL-NEXT:    s_bfe_u32 s8, s2, 0x40014
+; GFX9-DL-NEXT:    s_bfe_u32 s9, s2, 0x40010
+; GFX9-DL-NEXT:    s_bfe_u32 s10, s2, 0x4000c
+; GFX9-DL-NEXT:    s_bfe_u32 s11, s2, 0x40008
+; GFX9-DL-NEXT:    s_bfe_u32 s12, s2, 0x40004
 ; GFX9-DL-NEXT:    s_and_b32 s2, s2, 15
-; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s4
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s5
-; GFX9-DL-NEXT:    v_mad_u32_u24 v3, s2, v2, v3
-; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s17
-; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s2, v2, v3
-; GFX9-DL-NEXT:    v_mad_u32_u24 v3, s10, v4, v3
-; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s16
-; GFX9-DL-NEXT:    v_mad_u32_u24 v3, s9, v4, v3
-; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s15
-; GFX9-DL-NEXT:    v_mad_u32_u24 v3, s8, v4, v3
-; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s14
-; GFX9-DL-NEXT:    v_mad_u32_u24 v3, s7, v4, v3
-; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s13
-; GFX9-DL-NEXT:    v_mad_u32_u24 v3, s6, v4, v3
-; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s12
-; GFX9-DL-NEXT:    v_mad_u32_u24 v3, s1, v4, v3
-; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s11
-; GFX9-DL-NEXT:    v_mad_u32_u24 v3, s0, v4, v3
-; GFX9-DL-NEXT:    v_add_u32_e32 v2, v2, v3
+; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s6
+; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s19
+; GFX9-DL-NEXT:    v_mad_u32_u24 v1, s2, v0, v1
+; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s18
+; GFX9-DL-NEXT:    v_mad_u32_u24 v0, s2, v0, v1
+; GFX9-DL-NEXT:    v_mad_u32_u24 v1, s12, v2, v1
+; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s17
+; GFX9-DL-NEXT:    v_mad_u32_u24 v1, s11, v2, v1
+; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s16
+; GFX9-DL-NEXT:    v_mad_u32_u24 v1, s10, v2, v1
+; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s15
+; GFX9-DL-NEXT:    v_mad_u32_u24 v1, s9, v2, v1
+; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s14
+; GFX9-DL-NEXT:    v_mad_u32_u24 v1, s8, v2, v1
+; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s13
+; GFX9-DL-NEXT:    v_mad_u32_u24 v1, s5, v2, v1
+; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s7
+; GFX9-DL-NEXT:    v_mad_u32_u24 v1, s4, v2, v1
+; GFX9-DL-NEXT:    v_add_u32_e32 v2, v0, v1
+; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-DL-NEXT:    s_endpgm
 ;
@@ -1657,36 +1657,36 @@ define amdgpu_kernel void @udot8_multiuses_mul1(<8 x i4> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
 ; GFX10-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
 ; GFX10-DL-NEXT:    s_load_dword s5, s[0:1], 0x0
-; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_and_b32 s0, s2, 15
-; GFX10-DL-NEXT:    s_and_b32 s1, s4, 15
-; GFX10-DL-NEXT:    v_mov_b32_e32 v2, s5
+; GFX10-DL-NEXT:    s_and_b32 s6, s2, 15
+; GFX10-DL-NEXT:    s_and_b32 s7, s4, 15
+; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s5
 ; GFX10-DL-NEXT:    s_bfe_u32 s5, s2, 0x40004
-; GFX10-DL-NEXT:    s_bfe_u32 s6, s4, 0x40004
-; GFX10-DL-NEXT:    s_bfe_u32 s7, s2, 0x40008
-; GFX10-DL-NEXT:    s_bfe_u32 s8, s4, 0x40008
-; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s0, s1, v2
-; GFX10-DL-NEXT:    s_bfe_u32 s9, s2, 0x4000c
-; GFX10-DL-NEXT:    s_bfe_u32 s10, s4, 0x4000c
-; GFX10-DL-NEXT:    s_bfe_u32 s11, s2, 0x40010
-; GFX10-DL-NEXT:    s_bfe_u32 s12, s4, 0x40010
-; GFX10-DL-NEXT:    v_mad_u32_u24 v3, s5, s6, v2
+; GFX10-DL-NEXT:    s_bfe_u32 s8, s4, 0x40004
+; GFX10-DL-NEXT:    s_bfe_u32 s9, s2, 0x40008
+; GFX10-DL-NEXT:    s_bfe_u32 s10, s4, 0x40008
+; GFX10-DL-NEXT:    v_mad_u32_u24 v0, s6, s7, v0
+; GFX10-DL-NEXT:    v_mad_u32_u24 v1, s5, s8, v0
+; GFX10-DL-NEXT:    s_bfe_u32 s5, s2, 0x4000c
+; GFX10-DL-NEXT:    s_bfe_u32 s8, s4, 0x4000c
+; GFX10-DL-NEXT:    v_mad_u32_u24 v0, s6, s7, v0
+; GFX10-DL-NEXT:    v_mad_u32_u24 v1, s9, s10, v1
+; GFX10-DL-NEXT:    s_bfe_u32 s9, s2, 0x40010
+; GFX10-DL-NEXT:    s_bfe_u32 s10, s4, 0x40010
+; GFX10-DL-NEXT:    v_mad_u32_u24 v1, s5, s8, v1
 ; GFX10-DL-NEXT:    s_bfe_u32 s5, s2, 0x40014
-; GFX10-DL-NEXT:    s_bfe_u32 s6, s4, 0x40014
-; GFX10-DL-NEXT:    s_bfe_u32 s13, s2, 0x40018
-; GFX10-DL-NEXT:    s_bfe_u32 s14, s4, 0x40018
-; GFX10-DL-NEXT:    v_mad_u32_u24 v3, s7, s8, v3
+; GFX10-DL-NEXT:    s_bfe_u32 s8, s4, 0x40014
+; GFX10-DL-NEXT:    v_mad_u32_u24 v1, s9, s10, v1
+; GFX10-DL-NEXT:    s_bfe_u32 s9, s2, 0x40018
+; GFX10-DL-NEXT:    s_bfe_u32 s10, s4, 0x40018
 ; GFX10-DL-NEXT:    s_lshr_b32 s2, s2, 28
 ; GFX10-DL-NEXT:    s_lshr_b32 s4, s4, 28
-; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s0, s1, v2
-; GFX10-DL-NEXT:    v_mad_u32_u24 v3, s9, s10, v3
-; GFX10-DL-NEXT:    v_mad_u32_u24 v3, s11, s12, v3
-; GFX10-DL-NEXT:    v_mad_u32_u24 v3, s5, s6, v3
-; GFX10-DL-NEXT:    v_mad_u32_u24 v3, s13, s14, v3
-; GFX10-DL-NEXT:    v_mad_u32_u24 v3, s2, s4, v3
-; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v2, v3
+; GFX10-DL-NEXT:    v_mad_u32_u24 v1, s5, s8, v1
+; GFX10-DL-NEXT:    v_mad_u32_u24 v1, s9, s10, v1
+; GFX10-DL-NEXT:    v_mad_u32_u24 v1, s2, s4, v1
+; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v0, v1
+; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX10-DL-NEXT:    s_endpgm
                                                 <8 x i4> addrspace(1)* %src2,
@@ -1768,42 +1768,42 @@ define amdgpu_kernel void @udot8_acc32_vecMul(<8 x i4> addrspace(1)* %src1,
 ; GFX7-NEXT:    s_mov_b32 s6, -1
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    s_load_dword s0, s[8:9], 0x0
-; GFX7-NEXT:    s_load_dword s1, s[10:11], 0x0
-; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s10, s[10:11], 0x0
+; GFX7-NEXT:    s_load_dword s21, s[4:5], 0x0
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_lshr_b32 s8, s0, 28
-; GFX7-NEXT:    s_lshr_b32 s15, s1, 28
-; GFX7-NEXT:    s_bfe_u32 s16, s1, 0x40018
-; GFX7-NEXT:    s_bfe_u32 s17, s1, 0x40014
-; GFX7-NEXT:    s_bfe_u32 s18, s1, 0x40010
-; GFX7-NEXT:    s_bfe_u32 s19, s1, 0x4000c
-; GFX7-NEXT:    s_bfe_u32 s20, s1, 0x40008
-; GFX7-NEXT:    s_bfe_u32 s21, s1, 0x40004
-; GFX7-NEXT:    s_and_b32 s1, s1, 15
-; GFX7-NEXT:    s_bfe_u32 s9, s0, 0x40018
-; GFX7-NEXT:    s_bfe_u32 s10, s0, 0x40014
-; GFX7-NEXT:    s_bfe_u32 s11, s0, 0x40010
+; GFX7-NEXT:    s_lshr_b32 s1, s0, 28
+; GFX7-NEXT:    s_lshr_b32 s11, s10, 28
+; GFX7-NEXT:    s_bfe_u32 s15, s10, 0x40018
+; GFX7-NEXT:    s_bfe_u32 s16, s10, 0x40014
+; GFX7-NEXT:    s_bfe_u32 s17, s10, 0x40010
+; GFX7-NEXT:    s_bfe_u32 s18, s10, 0x4000c
+; GFX7-NEXT:    s_bfe_u32 s19, s10, 0x40008
+; GFX7-NEXT:    s_bfe_u32 s20, s10, 0x40004
+; GFX7-NEXT:    s_and_b32 s10, s10, 15
+; GFX7-NEXT:    s_bfe_u32 s2, s0, 0x40018
+; GFX7-NEXT:    s_bfe_u32 s8, s0, 0x40014
+; GFX7-NEXT:    s_bfe_u32 s9, s0, 0x40010
 ; GFX7-NEXT:    s_bfe_u32 s12, s0, 0x4000c
 ; GFX7-NEXT:    s_bfe_u32 s13, s0, 0x40008
 ; GFX7-NEXT:    s_bfe_u32 s14, s0, 0x40004
 ; GFX7-NEXT:    s_and_b32 s0, s0, 15
-; GFX7-NEXT:    v_mov_b32_e32 v0, s1
-; GFX7-NEXT:    v_mov_b32_e32 v1, s2
-; GFX7-NEXT:    v_mad_u32_u24 v0, s0, v0, v1
+; GFX7-NEXT:    v_mov_b32_e32 v0, s10
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s21
-; GFX7-NEXT:    v_mad_u32_u24 v0, s14, v1, v0
+; GFX7-NEXT:    v_mad_u32_u24 v0, s0, v0, v1
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s20
-; GFX7-NEXT:    v_mad_u32_u24 v0, s13, v1, v0
+; GFX7-NEXT:    v_mad_u32_u24 v0, s14, v1, v0
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s19
-; GFX7-NEXT:    v_mad_u32_u24 v0, s12, v1, v0
+; GFX7-NEXT:    v_mad_u32_u24 v0, s13, v1, v0
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s18
-; GFX7-NEXT:    v_mad_u32_u24 v0, s11, v1, v0
+; GFX7-NEXT:    v_mad_u32_u24 v0, s12, v1, v0
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s17
-; GFX7-NEXT:    v_mad_u32_u24 v0, s10, v1, v0
-; GFX7-NEXT:    v_mov_b32_e32 v1, s16
 ; GFX7-NEXT:    v_mad_u32_u24 v0, s9, v1, v0
-; GFX7-NEXT:    v_mov_b32_e32 v1, s15
+; GFX7-NEXT:    v_mov_b32_e32 v1, s16
 ; GFX7-NEXT:    v_mad_u32_u24 v0, s8, v1, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s15
+; GFX7-NEXT:    v_mad_u32_u24 v0, s2, v1, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s11
+; GFX7-NEXT:    v_mad_u32_u24 v0, s1, v1, v0
 ; GFX7-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 ; GFX7-NEXT:    s_endpgm
 ;
@@ -1813,44 +1813,44 @@ define amdgpu_kernel void @udot8_acc32_vecMul(<8 x i4> addrspace(1)* %src1,
 ; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX8-NEXT:    s_load_dword s4, s[6:7], 0x0
-; GFX8-NEXT:    s_load_dword s5, s[0:1], 0x0
-; GFX8-NEXT:    v_mov_b32_e32 v0, s0
-; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    s_load_dword s6, s[6:7], 0x0
+; GFX8-NEXT:    s_load_dword s19, s[0:1], 0x0
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_lshr_b32 s0, s2, 28
-; GFX8-NEXT:    s_lshr_b32 s11, s4, 28
-; GFX8-NEXT:    s_bfe_u32 s12, s4, 0x40018
-; GFX8-NEXT:    s_bfe_u32 s13, s4, 0x40014
-; GFX8-NEXT:    s_bfe_u32 s14, s4, 0x40010
-; GFX8-NEXT:    s_bfe_u32 s15, s4, 0x4000c
-; GFX8-NEXT:    s_bfe_u32 s16, s4, 0x40008
-; GFX8-NEXT:    s_bfe_u32 s17, s4, 0x40004
-; GFX8-NEXT:    s_and_b32 s4, s4, 15
-; GFX8-NEXT:    s_bfe_u32 s1, s2, 0x40018
-; GFX8-NEXT:    s_bfe_u32 s6, s2, 0x40014
-; GFX8-NEXT:    s_bfe_u32 s7, s2, 0x40010
-; GFX8-NEXT:    s_bfe_u32 s8, s2, 0x4000c
-; GFX8-NEXT:    s_bfe_u32 s9, s2, 0x40008
-; GFX8-NEXT:    s_bfe_u32 s10, s2, 0x40004
+; GFX8-NEXT:    s_lshr_b32 s4, s2, 28
+; GFX8-NEXT:    s_lshr_b32 s7, s6, 28
+; GFX8-NEXT:    s_bfe_u32 s13, s6, 0x40018
+; GFX8-NEXT:    s_bfe_u32 s14, s6, 0x40014
+; GFX8-NEXT:    s_bfe_u32 s15, s6, 0x40010
+; GFX8-NEXT:    s_bfe_u32 s16, s6, 0x4000c
+; GFX8-NEXT:    s_bfe_u32 s17, s6, 0x40008
+; GFX8-NEXT:    s_bfe_u32 s18, s6, 0x40004
+; GFX8-NEXT:    s_and_b32 s6, s6, 15
+; GFX8-NEXT:    s_bfe_u32 s5, s2, 0x40018
+; GFX8-NEXT:    s_bfe_u32 s8, s2, 0x40014
+; GFX8-NEXT:    s_bfe_u32 s9, s2, 0x40010
+; GFX8-NEXT:    s_bfe_u32 s10, s2, 0x4000c
+; GFX8-NEXT:    s_bfe_u32 s11, s2, 0x40008
+; GFX8-NEXT:    s_bfe_u32 s12, s2, 0x40004
 ; GFX8-NEXT:    s_and_b32 s2, s2, 15
-; GFX8-NEXT:    v_mov_b32_e32 v2, s4
-; GFX8-NEXT:    v_mov_b32_e32 v3, s5
-; GFX8-NEXT:    v_mad_u32_u24 v2, s2, v2, v3
-; GFX8-NEXT:    v_mov_b32_e32 v3, s17
-; GFX8-NEXT:    v_mad_u32_u24 v2, s10, v3, v2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s16
-; GFX8-NEXT:    v_mad_u32_u24 v2, s9, v3, v2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s15
-; GFX8-NEXT:    v_mad_u32_u24 v2, s8, v3, v2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s14
-; GFX8-NEXT:    v_mad_u32_u24 v2, s7, v3, v2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s13
-; GFX8-NEXT:    v_mad_u32_u24 v2, s6, v3, v2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s12
-; GFX8-NEXT:    v_mad_u32_u24 v2, s1, v3, v2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s11
-; GFX8-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
+; GFX8-NEXT:    v_mov_b32_e32 v0, s6
+; GFX8-NEXT:    v_mov_b32_e32 v1, s19
+; GFX8-NEXT:    v_mad_u32_u24 v0, s2, v0, v1
+; GFX8-NEXT:    v_mov_b32_e32 v1, s18
+; GFX8-NEXT:    v_mad_u32_u24 v0, s12, v1, v0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s17
+; GFX8-NEXT:    v_mad_u32_u24 v0, s11, v1, v0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s16
+; GFX8-NEXT:    v_mad_u32_u24 v0, s10, v1, v0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s15
+; GFX8-NEXT:    v_mad_u32_u24 v0, s9, v1, v0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s14
+; GFX8-NEXT:    v_mad_u32_u24 v0, s8, v1, v0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s13
+; GFX8-NEXT:    v_mad_u32_u24 v0, s5, v1, v0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s7
+; GFX8-NEXT:    v_mad_u32_u24 v2, s4, v1, v0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX8-NEXT:    flat_store_dword v[0:1], v2
 ; GFX8-NEXT:    s_endpgm
 ;
@@ -1860,44 +1860,44 @@ define amdgpu_kernel void @udot8_acc32_vecMul(<8 x i4> addrspace(1)* %src1,
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX9-NEXT:    s_load_dword s4, s[6:7], 0x0
-; GFX9-NEXT:    s_load_dword s5, s[0:1], 0x0
-; GFX9-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    s_load_dword s6, s[6:7], 0x0
+; GFX9-NEXT:    s_load_dword s19, s[0:1], 0x0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_lshr_b32 s0, s2, 28
-; GFX9-NEXT:    s_lshr_b32 s11, s4, 28
-; GFX9-NEXT:    s_bfe_u32 s12, s4, 0x40018
-; GFX9-NEXT:    s_bfe_u32 s13, s4, 0x40014
-; GFX9-NEXT:    s_bfe_u32 s14, s4, 0x40010
-; GFX9-NEXT:    s_bfe_u32 s15, s4, 0x4000c
-; GFX9-NEXT:    s_bfe_u32 s16, s4, 0x40008
-; GFX9-NEXT:    s_bfe_u32 s17, s4, 0x40004
-; GFX9-NEXT:    s_and_b32 s4, s4, 15
-; GFX9-NEXT:    s_bfe_u32 s1, s2, 0x40018
-; GFX9-NEXT:    s_bfe_u32 s6, s2, 0x40014
-; GFX9-NEXT:    s_bfe_u32 s7, s2, 0x40010
-; GFX9-NEXT:    s_bfe_u32 s8, s2, 0x4000c
-; GFX9-NEXT:    s_bfe_u32 s9, s2, 0x40008
-; GFX9-NEXT:    s_bfe_u32 s10, s2, 0x40004
+; GFX9-NEXT:    s_lshr_b32 s4, s2, 28
+; GFX9-NEXT:    s_lshr_b32 s7, s6, 28
+; GFX9-NEXT:    s_bfe_u32 s13, s6, 0x40018
+; GFX9-NEXT:    s_bfe_u32 s14, s6, 0x40014
+; GFX9-NEXT:    s_bfe_u32 s15, s6, 0x40010
+; GFX9-NEXT:    s_bfe_u32 s16, s6, 0x4000c
+; GFX9-NEXT:    s_bfe_u32 s17, s6, 0x40008
+; GFX9-NEXT:    s_bfe_u32 s18, s6, 0x40004
+; GFX9-NEXT:    s_and_b32 s6, s6, 15
+; GFX9-NEXT:    s_bfe_u32 s5, s2, 0x40018
+; GFX9-NEXT:    s_bfe_u32 s8, s2, 0x40014
+; GFX9-NEXT:    s_bfe_u32 s9, s2, 0x40010
+; GFX9-NEXT:    s_bfe_u32 s10, s2, 0x4000c
+; GFX9-NEXT:    s_bfe_u32 s11, s2, 0x40008
+; GFX9-NEXT:    s_bfe_u32 s12, s2, 0x40004
 ; GFX9-NEXT:    s_and_b32 s2, s2, 15
-; GFX9-NEXT:    v_mov_b32_e32 v2, s4
-; GFX9-NEXT:    v_mov_b32_e32 v3, s5
-; GFX9-NEXT:    v_mad_u32_u24 v2, s2, v2, v3
-; GFX9-NEXT:    v_mov_b32_e32 v3, s17
-; GFX9-NEXT:    v_mad_u32_u24 v2, s10, v3, v2
-; GFX9-NEXT:    v_mov_b32_e32 v3, s16
-; GFX9-NEXT:    v_mad_u32_u24 v2, s9, v3, v2
-; GFX9-NEXT:    v_mov_b32_e32 v3, s15
-; GFX9-NEXT:    v_mad_u32_u24 v2, s8, v3, v2
-; GFX9-NEXT:    v_mov_b32_e32 v3, s14
-; GFX9-NEXT:    v_mad_u32_u24 v2, s7, v3, v2
-; GFX9-NEXT:    v_mov_b32_e32 v3, s13
-; GFX9-NEXT:    v_mad_u32_u24 v2, s6, v3, v2
-; GFX9-NEXT:    v_mov_b32_e32 v3, s12
-; GFX9-NEXT:    v_mad_u32_u24 v2, s1, v3, v2
-; GFX9-NEXT:    v_mov_b32_e32 v3, s11
-; GFX9-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
+; GFX9-NEXT:    v_mov_b32_e32 v0, s6
+; GFX9-NEXT:    v_mov_b32_e32 v1, s19
+; GFX9-NEXT:    v_mad_u32_u24 v0, s2, v0, v1
+; GFX9-NEXT:    v_mov_b32_e32 v1, s18
+; GFX9-NEXT:    v_mad_u32_u24 v0, s12, v1, v0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s17
+; GFX9-NEXT:    v_mad_u32_u24 v0, s11, v1, v0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s16
+; GFX9-NEXT:    v_mad_u32_u24 v0, s10, v1, v0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s15
+; GFX9-NEXT:    v_mad_u32_u24 v0, s9, v1, v0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s14
+; GFX9-NEXT:    v_mad_u32_u24 v0, s8, v1, v0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s13
+; GFX9-NEXT:    v_mad_u32_u24 v0, s5, v1, v0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s7
+; GFX9-NEXT:    v_mad_u32_u24 v2, s4, v1, v0
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-NEXT:    s_endpgm
 ;
@@ -1906,15 +1906,15 @@ define amdgpu_kernel void @udot8_acc32_vecMul(<8 x i4> addrspace(1)* %src1,
 ; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
-; GFX9-DL-NEXT:    s_load_dword s5, s[0:1], 0x0
+; GFX9-DL-NEXT:    s_load_dword s2, s[6:7], 0x0
+; GFX9-DL-NEXT:    s_load_dword s6, s[0:1], 0x0
+; GFX9-DL-NEXT:    s_load_dword s4, s[4:5], 0x0
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s4
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s5
-; GFX9-DL-NEXT:    v_dot8_u32_u4 v2, s2, v2, v3
+; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s2
+; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s6
+; GFX9-DL-NEXT:    v_dot8_u32_u4 v2, s4, v2, v3
 ; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-DL-NEXT:    s_endpgm
 ;
@@ -1924,14 +1924,14 @@ define amdgpu_kernel void @udot8_acc32_vecMul(<8 x i4> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX10-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
-; GFX10-DL-NEXT:    s_load_dword s5, s[0:1], 0x0
+; GFX10-DL-NEXT:    s_load_dword s2, s[0:1], 0x0
+; GFX10-DL-NEXT:    s_load_dword s4, s[4:5], 0x0
+; GFX10-DL-NEXT:    s_load_dword s5, s[6:7], 0x0
+; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-DL-NEXT:    v_dot8_u32_u4 v2, s4, s5, v0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
-; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    v_mov_b32_e32 v2, s5
-; GFX10-DL-NEXT:    v_dot8_u32_u4 v2, s2, s4, v2
 ; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX10-DL-NEXT:    s_endpgm
                                               <8 x i4> addrspace(1)* %src2,
@@ -1987,53 +1987,43 @@ define amdgpu_kernel void @udot8_acc16_vecMul(<8 x i4> addrspace(1)* %src1,
 ; GFX7-NEXT:    s_bfe_u32 s20, s1, 0x4000c
 ; GFX7-NEXT:    v_mov_b32_e32 v2, s20
 ; GFX7-NEXT:    v_mov_b32_e32 v4, s18
-; GFX7-NEXT:    s_bfe_u32 s14, s1, 0x40014
-; GFX7-NEXT:    s_bfe_u32 s15, s1, 0x40010
-; GFX7-NEXT:    s_lshr_b32 s16, s1, 28
-; GFX7-NEXT:    s_bfe_u32 s17, s1, 0x40018
+; GFX7-NEXT:    s_bfe_u32 s15, s1, 0x40018
+; GFX7-NEXT:    s_bfe_u32 s16, s1, 0x40014
+; GFX7-NEXT:    s_bfe_u32 s17, s1, 0x40010
 ; GFX7-NEXT:    s_and_b32 s19, s1, 15
+; GFX7-NEXT:    s_lshr_b32 s14, s1, 28
 ; GFX7-NEXT:    s_bfe_u32 s1, s1, 0x40008
 ; GFX7-NEXT:    v_mul_u32_u24_e32 v2, s13, v2
 ; GFX7-NEXT:    v_mul_u32_u24_e32 v4, s11, v4
-; GFX7-NEXT:    s_bfe_u32 s2, s0, 0x40014
-; GFX7-NEXT:    s_bfe_u32 s8, s0, 0x40010
-; GFX7-NEXT:    s_lshr_b32 s9, s0, 28
-; GFX7-NEXT:    v_mov_b32_e32 v6, s16
-; GFX7-NEXT:    s_bfe_u32 s10, s0, 0x40018
+; GFX7-NEXT:    s_lshr_b32 s2, s0, 28
+; GFX7-NEXT:    s_bfe_u32 s8, s0, 0x40018
+; GFX7-NEXT:    s_bfe_u32 s9, s0, 0x40014
+; GFX7-NEXT:    s_bfe_u32 s10, s0, 0x40010
 ; GFX7-NEXT:    s_and_b32 s12, s0, 15
 ; GFX7-NEXT:    v_mov_b32_e32 v3, s19
 ; GFX7-NEXT:    s_bfe_u32 s0, s0, 0x40008
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
-; GFX7-NEXT:    v_mov_b32_e32 v5, s17
-; GFX7-NEXT:    v_mul_u32_u24_e32 v6, s9, v6
 ; GFX7-NEXT:    v_mul_u32_u24_e32 v1, s0, v1
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX7-NEXT:    v_mul_u32_u24_e32 v3, s12, v3
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
 ; GFX7-NEXT:    v_or_b32_e32 v1, v1, v2
 ; GFX7-NEXT:    v_or_b32_e32 v2, v3, v4
-; GFX7-NEXT:    v_mul_u32_u24_e32 v5, s10, v5
-; GFX7-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX7-NEXT:    v_mov_b32_e32 v8, s14
-; GFX7-NEXT:    v_or_b32_e32 v3, v5, v6
-; GFX7-NEXT:    v_alignbit_b32 v5, v1, v2, 16
+; GFX7-NEXT:    v_alignbit_b32 v3, v1, v2, 16
+; GFX7-NEXT:    v_lshrrev_b32_e32 v4, 16, v1
+; GFX7-NEXT:    v_mov_b32_e32 v5, s17
+; GFX7-NEXT:    v_mov_b32_e32 v6, s16
 ; GFX7-NEXT:    v_mov_b32_e32 v7, s15
-; GFX7-NEXT:    v_mul_u32_u24_e32 v8, s2, v8
-; GFX7-NEXT:    v_mul_u32_u24_e32 v7, s8, v7
-; GFX7-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; GFX7-NEXT:    v_lshrrev_b32_e32 v6, 16, v1
-; GFX7-NEXT:    v_or_b32_e32 v4, v7, v8
-; GFX7-NEXT:    v_lshrrev_b32_e32 v7, 16, v4
-; GFX7-NEXT:    v_lshrrev_b32_e32 v8, 16, v3
 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v5, v0
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v3, v0
 ; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v6, v0
 ; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v4, v0
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v7, v0
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v3, v0
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v8, v0
+; GFX7-NEXT:    v_mad_u32_u24 v0, s10, v5, v0
+; GFX7-NEXT:    v_mad_u32_u24 v0, s9, v6, v0
+; GFX7-NEXT:    v_mad_u32_u24 v0, s8, v7, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s14
+; GFX7-NEXT:    v_mad_u32_u24 v0, s2, v1, v0
 ; GFX7-NEXT:    buffer_store_short v0, off, s[4:7], 0
 ; GFX7-NEXT:    s_endpgm
 ;
@@ -2042,46 +2032,46 @@ define amdgpu_kernel void @udot8_acc16_vecMul(<8 x i4> addrspace(1)* %src1,
 ; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX8-NEXT:    s_load_dword s4, s[6:7], 0x0
+; GFX8-NEXT:    s_load_dword s2, s[6:7], 0x0
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX8-NEXT:    flat_load_ushort v2, v[0:1]
+; GFX8-NEXT:    s_load_dword s0, s[4:5], 0x0
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_and_b32 s0, s2, 15
-; GFX8-NEXT:    s_and_b32 s1, s4, 15
+; GFX8-NEXT:    s_and_b32 s1, s2, 15
+; GFX8-NEXT:    s_bfe_u32 s4, s2, 0x40004
 ; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    s_bfe_u32 s5, s4, 0x40004
-; GFX8-NEXT:    v_mov_b32_e32 v4, s5
-; GFX8-NEXT:    s_bfe_u32 s1, s2, 0x40004
-; GFX8-NEXT:    s_bfe_u32 s5, s4, 0x40008
-; GFX8-NEXT:    s_bfe_u32 s8, s4, 0x40010
-; GFX8-NEXT:    s_bfe_u32 s10, s4, 0x40014
-; GFX8-NEXT:    s_bfe_u32 s12, s4, 0x40018
-; GFX8-NEXT:    s_lshr_b32 s14, s4, 28
-; GFX8-NEXT:    s_bfe_u32 s4, s4, 0x4000c
-; GFX8-NEXT:    s_bfe_u32 s6, s2, 0x40008
-; GFX8-NEXT:    v_mov_b32_e32 v5, s5
+; GFX8-NEXT:    v_mov_b32_e32 v4, s4
+; GFX8-NEXT:    s_and_b32 s1, s0, 15
+; GFX8-NEXT:    s_bfe_u32 s5, s0, 0x40004
+; GFX8-NEXT:    s_bfe_u32 s4, s2, 0x40008
 ; GFX8-NEXT:    s_bfe_u32 s7, s2, 0x4000c
-; GFX8-NEXT:    v_mov_b32_e32 v6, s4
-; GFX8-NEXT:    s_bfe_u32 s9, s2, 0x40010
+; GFX8-NEXT:    v_mov_b32_e32 v5, s4
+; GFX8-NEXT:    s_bfe_u32 s6, s0, 0x40008
+; GFX8-NEXT:    s_bfe_u32 s8, s2, 0x40010
+; GFX8-NEXT:    v_mov_b32_e32 v6, s7
+; GFX8-NEXT:    s_bfe_u32 s4, s0, 0x4000c
+; GFX8-NEXT:    s_bfe_u32 s9, s2, 0x40014
+; GFX8-NEXT:    s_bfe_u32 s7, s0, 0x40010
 ; GFX8-NEXT:    v_mov_b32_e32 v7, s8
-; GFX8-NEXT:    s_bfe_u32 s11, s2, 0x40014
-; GFX8-NEXT:    v_mov_b32_e32 v8, s10
-; GFX8-NEXT:    s_bfe_u32 s13, s2, 0x40018
-; GFX8-NEXT:    v_mov_b32_e32 v9, s12
+; GFX8-NEXT:    s_bfe_u32 s11, s2, 0x40018
+; GFX8-NEXT:    s_bfe_u32 s10, s0, 0x40014
+; GFX8-NEXT:    v_mov_b32_e32 v8, s9
+; GFX8-NEXT:    s_bfe_u32 s12, s0, 0x40018
 ; GFX8-NEXT:    s_lshr_b32 s2, s2, 28
+; GFX8-NEXT:    v_mov_b32_e32 v9, s11
+; GFX8-NEXT:    s_lshr_b32 s0, s0, 28
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
-; GFX8-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
-; GFX8-NEXT:    v_mad_u32_u24 v2, s1, v4, v2
+; GFX8-NEXT:    v_mad_u32_u24 v2, s1, v3, v2
+; GFX8-NEXT:    v_mad_u32_u24 v2, s5, v4, v2
 ; GFX8-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; GFX8-NEXT:    v_mad_u32_u24 v2, s6, v5, v2
-; GFX8-NEXT:    v_mad_u32_u24 v2, s7, v6, v2
-; GFX8-NEXT:    v_mad_u32_u24 v2, s9, v7, v2
-; GFX8-NEXT:    v_mad_u32_u24 v2, s11, v8, v2
-; GFX8-NEXT:    v_mad_u32_u24 v2, s13, v9, v2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s14
-; GFX8-NEXT:    v_mad_u32_u24 v2, s2, v3, v2
+; GFX8-NEXT:    v_mad_u32_u24 v2, s4, v6, v2
+; GFX8-NEXT:    v_mad_u32_u24 v2, s7, v7, v2
+; GFX8-NEXT:    v_mad_u32_u24 v2, s10, v8, v2
+; GFX8-NEXT:    v_mad_u32_u24 v2, s12, v9, v2
+; GFX8-NEXT:    v_mov_b32_e32 v3, s2
+; GFX8-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
 ; GFX8-NEXT:    flat_store_short v[0:1], v2
 ; GFX8-NEXT:    s_endpgm
 ;
@@ -2090,53 +2080,53 @@ define amdgpu_kernel void @udot8_acc16_vecMul(<8 x i4> addrspace(1)* %src1,
 ; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX9-NEXT:    s_load_dword s4, s[6:7], 0x0
+; GFX9-NEXT:    s_load_dword s2, s[6:7], 0x0
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-NEXT:    global_load_ushort v2, v[0:1], off
+; GFX9-NEXT:    s_load_dword s0, s[4:5], 0x0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_and_b32 s0, s2, 15
-; GFX9-NEXT:    s_and_b32 s1, s4, 15
-; GFX9-NEXT:    s_bfe_u32 s5, s4, 0x40004
-; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s5
-; GFX9-NEXT:    s_bfe_u32 s6, s2, 0x40004
-; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s6
+; GFX9-NEXT:    s_and_b32 s1, s2, 15
+; GFX9-NEXT:    s_bfe_u32 s4, s2, 0x40004
+; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s4
+; GFX9-NEXT:    s_bfe_u32 s11, s2, 0x40018
+; GFX9-NEXT:    s_lshr_b32 s13, s2, 28
+; GFX9-NEXT:    s_and_b32 s4, s0, 15
+; GFX9-NEXT:    s_bfe_u32 s5, s0, 0x40004
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s5
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s1
-; GFX9-NEXT:    s_bfe_u32 s5, s4, 0x40008
-; GFX9-NEXT:    s_bfe_u32 s7, s4, 0x4000c
-; GFX9-NEXT:    s_pack_ll_b32_b16 s5, s5, s7
-; GFX9-NEXT:    v_pk_mul_lo_u16 v3, s0, v3
-; GFX9-NEXT:    s_bfe_u32 s1, s2, 0x40008
-; GFX9-NEXT:    s_bfe_u32 s6, s2, 0x4000c
-; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s6
-; GFX9-NEXT:    v_mov_b32_e32 v4, s5
-; GFX9-NEXT:    s_bfe_u32 s0, s4, 0x40010
-; GFX9-NEXT:    s_bfe_u32 s7, s4, 0x40014
-; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s7
-; GFX9-NEXT:    v_pk_mul_lo_u16 v4, s1, v4
-; GFX9-NEXT:    s_bfe_u32 s5, s2, 0x40010
-; GFX9-NEXT:    s_bfe_u32 s6, s2, 0x40014
-; GFX9-NEXT:    s_bfe_u32 s1, s4, 0x40018
-; GFX9-NEXT:    s_lshr_b32 s4, s4, 28
+; GFX9-NEXT:    s_pack_ll_b32_b16 s11, s11, s13
+; GFX9-NEXT:    s_bfe_u32 s1, s0, 0x40008
+; GFX9-NEXT:    s_bfe_u32 s5, s0, 0x4000c
+; GFX9-NEXT:    s_bfe_u32 s9, s0, 0x40010
+; GFX9-NEXT:    s_bfe_u32 s10, s0, 0x40014
+; GFX9-NEXT:    s_bfe_u32 s12, s0, 0x40018
+; GFX9-NEXT:    s_lshr_b32 s0, s0, 28
+; GFX9-NEXT:    s_bfe_u32 s6, s2, 0x40008
+; GFX9-NEXT:    s_bfe_u32 s7, s2, 0x4000c
+; GFX9-NEXT:    s_bfe_u32 s8, s2, 0x40010
+; GFX9-NEXT:    s_bfe_u32 s2, s2, 0x40014
+; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s12, s0
+; GFX9-NEXT:    v_mov_b32_e32 v4, s11
+; GFX9-NEXT:    v_pk_mul_lo_u16 v4, s0, v4
+; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s8, s2
 ; GFX9-NEXT:    v_mov_b32_e32 v5, s0
-; GFX9-NEXT:    s_pack_ll_b32_b16 s5, s5, s6
-; GFX9-NEXT:    s_bfe_u32 s0, s2, 0x40018
-; GFX9-NEXT:    s_lshr_b32 s2, s2, 28
-; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s4
-; GFX9-NEXT:    v_pk_mul_lo_u16 v5, s5, v5
-; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s2
-; GFX9-NEXT:    v_mov_b32_e32 v6, s1
-; GFX9-NEXT:    v_pk_mul_lo_u16 v6, s0, v6
+; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s6, s7
+; GFX9-NEXT:    v_pk_mul_lo_u16 v3, s4, v3
+; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s5
+; GFX9-NEXT:    v_mov_b32_e32 v6, s0
+; GFX9-NEXT:    v_pk_mul_lo_u16 v6, s1, v6
+; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s9, s10
+; GFX9-NEXT:    v_pk_mul_lo_u16 v5, s2, v5
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u32_e32 v2, v3, v2
 ; GFX9-NEXT:    v_add_u32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT:    v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT:    v_add_u32_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u32_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
 ; GFX9-NEXT:    v_add_u32_e32 v2, v2, v5
 ; GFX9-NEXT:    v_add_u32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT:    v_add_u32_e32 v2, v2, v6
-; GFX9-NEXT:    v_add_u32_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT:    v_add_u32_e32 v2, v2, v4
+; GFX9-NEXT:    v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
 ; GFX9-NEXT:    global_store_short v[0:1], v2, off
 ; GFX9-NEXT:    s_endpgm
 ;
@@ -2145,53 +2135,53 @@ define amdgpu_kernel void @udot8_acc16_vecMul(<8 x i4> addrspace(1)* %src1,
 ; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
+; GFX9-DL-NEXT:    s_load_dword s2, s[6:7], 0x0
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-DL-NEXT:    global_load_ushort v2, v[0:1], off
+; GFX9-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    s_and_b32 s0, s2, 15
-; GFX9-DL-NEXT:    s_and_b32 s1, s4, 15
-; GFX9-DL-NEXT:    s_bfe_u32 s5, s4, 0x40004
-; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s1, s1, s5
-; GFX9-DL-NEXT:    s_bfe_u32 s6, s2, 0x40004
-; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s0, s0, s6
+; GFX9-DL-NEXT:    s_and_b32 s1, s2, 15
+; GFX9-DL-NEXT:    s_bfe_u32 s4, s2, 0x40004
+; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s1, s1, s4
+; GFX9-DL-NEXT:    s_bfe_u32 s11, s2, 0x40018
+; GFX9-DL-NEXT:    s_lshr_b32 s13, s2, 28
+; GFX9-DL-NEXT:    s_and_b32 s4, s0, 15
+; GFX9-DL-NEXT:    s_bfe_u32 s5, s0, 0x40004
+; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s4, s4, s5
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s1
-; GFX9-DL-NEXT:    s_bfe_u32 s5, s4, 0x40008
-; GFX9-DL-NEXT:    s_bfe_u32 s7, s4, 0x4000c
-; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s5, s5, s7
-; GFX9-DL-NEXT:    v_pk_mul_lo_u16 v3, s0, v3
-; GFX9-DL-NEXT:    s_bfe_u32 s1, s2, 0x40008
-; GFX9-DL-NEXT:    s_bfe_u32 s6, s2, 0x4000c
-; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s1, s1, s6
-; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s5
-; GFX9-DL-NEXT:    s_bfe_u32 s0, s4, 0x40010
-; GFX9-DL-NEXT:    s_bfe_u32 s7, s4, 0x40014
-; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s0, s0, s7
-; GFX9-DL-NEXT:    v_pk_mul_lo_u16 v4, s1, v4
-; GFX9-DL-NEXT:    s_bfe_u32 s5, s2, 0x40010
-; GFX9-DL-NEXT:    s_bfe_u32 s6, s2, 0x40014
-; GFX9-DL-NEXT:    s_bfe_u32 s1, s4, 0x40018
-; GFX9-DL-NEXT:    s_lshr_b32 s4, s4, 28
+; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s11, s11, s13
+; GFX9-DL-NEXT:    s_bfe_u32 s1, s0, 0x40008
+; GFX9-DL-NEXT:    s_bfe_u32 s5, s0, 0x4000c
+; GFX9-DL-NEXT:    s_bfe_u32 s9, s0, 0x40010
+; GFX9-DL-NEXT:    s_bfe_u32 s10, s0, 0x40014
+; GFX9-DL-NEXT:    s_bfe_u32 s12, s0, 0x40018
+; GFX9-DL-NEXT:    s_lshr_b32 s0, s0, 28
+; GFX9-DL-NEXT:    s_bfe_u32 s6, s2, 0x40008
+; GFX9-DL-NEXT:    s_bfe_u32 s7, s2, 0x4000c
+; GFX9-DL-NEXT:    s_bfe_u32 s8, s2, 0x40010
+; GFX9-DL-NEXT:    s_bfe_u32 s2, s2, 0x40014
+; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s0, s12, s0
+; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s11
+; GFX9-DL-NEXT:    v_pk_mul_lo_u16 v4, s0, v4
+; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s0, s8, s2
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v5, s0
-; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s5, s5, s6
-; GFX9-DL-NEXT:    s_bfe_u32 s0, s2, 0x40018
-; GFX9-DL-NEXT:    s_lshr_b32 s2, s2, 28
-; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s1, s1, s4
-; GFX9-DL-NEXT:    v_pk_mul_lo_u16 v5, s5, v5
-; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s0, s0, s2
-; GFX9-DL-NEXT:    v_mov_b32_e32 v6, s1
-; GFX9-DL-NEXT:    v_pk_mul_lo_u16 v6, s0, v6
+; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s0, s6, s7
+; GFX9-DL-NEXT:    v_pk_mul_lo_u16 v3, s4, v3
+; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s1, s1, s5
+; GFX9-DL-NEXT:    v_mov_b32_e32 v6, s0
+; GFX9-DL-NEXT:    v_pk_mul_lo_u16 v6, s1, v6
+; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s2, s9, s10
+; GFX9-DL-NEXT:    v_pk_mul_lo_u16 v5, s2, v5
 ; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-DL-NEXT:    v_add_u32_e32 v2, v3, v2
 ; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:BYTE_0
-; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:BYTE_0
+; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
 ; GFX9-DL-NEXT:    v_add_u32_e32 v2, v2, v5
 ; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-DL-NEXT:    v_add_u32_e32 v2, v2, v6
-; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-DL-NEXT:    v_add_u32_e32 v2, v2, v4
+; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
 ; GFX9-DL-NEXT:    global_store_short v[0:1], v2, off
 ; GFX9-DL-NEXT:    s_endpgm
 ;
@@ -2201,42 +2191,42 @@ define amdgpu_kernel void @udot8_acc16_vecMul(<8 x i4> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX10-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
+; GFX10-DL-NEXT:    s_load_dword s1, s[6:7], 0x0
 ; GFX10-DL-NEXT:    global_load_ushort v2, v[0:1], off
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_and_b32 s0, s2, 15
-; GFX10-DL-NEXT:    s_bfe_u32 s1, s2, 0x40004
-; GFX10-DL-NEXT:    s_and_b32 s5, s4, 15
-; GFX10-DL-NEXT:    s_bfe_u32 s6, s4, 0x40004
-; GFX10-DL-NEXT:    s_bfe_u32 s7, s2, 0x40008
-; GFX10-DL-NEXT:    s_bfe_u32 s8, s2, 0x4000c
-; GFX10-DL-NEXT:    s_pack_ll_b32_b16 s0, s0, s1
-; GFX10-DL-NEXT:    s_bfe_u32 s1, s4, 0x40008
-; GFX10-DL-NEXT:    s_pack_ll_b32_b16 s5, s5, s6
-; GFX10-DL-NEXT:    s_bfe_u32 s6, s4, 0x4000c
-; GFX10-DL-NEXT:    s_pack_ll_b32_b16 s7, s7, s8
-; GFX10-DL-NEXT:    s_bfe_u32 s8, s2, 0x40010
-; GFX10-DL-NEXT:    v_pk_mul_lo_u16 v3, s0, s5
-; GFX10-DL-NEXT:    s_pack_ll_b32_b16 s0, s1, s6
-; GFX10-DL-NEXT:    s_bfe_u32 s1, s2, 0x40014
-; GFX10-DL-NEXT:    s_bfe_u32 s5, s4, 0x40010
-; GFX10-DL-NEXT:    s_bfe_u32 s6, s4, 0x40014
-; GFX10-DL-NEXT:    v_pk_mul_lo_u16 v4, s7, s0
-; GFX10-DL-NEXT:    s_pack_ll_b32_b16 s1, s8, s1
-; GFX10-DL-NEXT:    s_bfe_u32 s0, s2, 0x40018
-; GFX10-DL-NEXT:    s_pack_ll_b32_b16 s5, s5, s6
-; GFX10-DL-NEXT:    s_lshr_b32 s2, s2, 28
-; GFX10-DL-NEXT:    s_bfe_u32 s6, s4, 0x40018
-; GFX10-DL-NEXT:    s_lshr_b32 s4, s4, 28
-; GFX10-DL-NEXT:    s_pack_ll_b32_b16 s0, s0, s2
+; GFX10-DL-NEXT:    s_and_b32 s2, s0, 15
+; GFX10-DL-NEXT:    s_bfe_u32 s6, s0, 0x40004
+; GFX10-DL-NEXT:    s_and_b32 s4, s1, 15
+; GFX10-DL-NEXT:    s_bfe_u32 s5, s1, 0x40004
+; GFX10-DL-NEXT:    s_bfe_u32 s7, s1, 0x4000c
+; GFX10-DL-NEXT:    s_bfe_u32 s8, s0, 0x4000c
+; GFX10-DL-NEXT:    s_pack_ll_b32_b16 s2, s2, s6
+; GFX10-DL-NEXT:    s_bfe_u32 s6, s1, 0x40008
+; GFX10-DL-NEXT:    s_pack_ll_b32_b16 s4, s4, s5
+; GFX10-DL-NEXT:    s_bfe_u32 s5, s0, 0x40008
+; GFX10-DL-NEXT:    v_pk_mul_lo_u16 v3, s2, s4
+; GFX10-DL-NEXT:    s_pack_ll_b32_b16 s4, s6, s7
+; GFX10-DL-NEXT:    s_pack_ll_b32_b16 s5, s5, s8
+; GFX10-DL-NEXT:    s_bfe_u32 s2, s0, 0x40010
+; GFX10-DL-NEXT:    s_bfe_u32 s6, s0, 0x40014
+; GFX10-DL-NEXT:    s_bfe_u32 s7, s1, 0x40010
+; GFX10-DL-NEXT:    s_bfe_u32 s8, s1, 0x40014
+; GFX10-DL-NEXT:    v_pk_mul_lo_u16 v4, s5, s4
+; GFX10-DL-NEXT:    s_bfe_u32 s4, s0, 0x40018
+; GFX10-DL-NEXT:    s_pack_ll_b32_b16 s2, s2, s6
+; GFX10-DL-NEXT:    s_lshr_b32 s0, s0, 28
+; GFX10-DL-NEXT:    s_pack_ll_b32_b16 s5, s7, s8
+; GFX10-DL-NEXT:    s_bfe_u32 s6, s1, 0x40018
+; GFX10-DL-NEXT:    s_lshr_b32 s1, s1, 28
+; GFX10-DL-NEXT:    s_pack_ll_b32_b16 s0, s4, s0
+; GFX10-DL-NEXT:    s_pack_ll_b32_b16 s1, s6, s1
 ; GFX10-DL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v3, v2
 ; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-DL-NEXT:    v_pk_mul_lo_u16 v3, s1, s5
-; GFX10-DL-NEXT:    s_pack_ll_b32_b16 s1, s6, s4
+; GFX10-DL-NEXT:    v_pk_mul_lo_u16 v3, s2, s5
 ; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:BYTE_0
 ; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
 ; GFX10-DL-NEXT:    v_pk_mul_lo_u16 v4, s0, s1
@@ -2293,64 +2283,64 @@ define amdgpu_kernel void @udot8_acc8_vecMul(<8 x i4> addrspace(1)* %src1,
 ; GFX7-NEXT:    s_load_dword s1, s[10:11], 0x0
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    s_bfe_u32 s2, s0, 0x4000c
-; GFX7-NEXT:    s_lshr_b32 s11, s0, 28
+; GFX7-NEXT:    s_bfe_u32 s9, s0, 0x40004
 ; GFX7-NEXT:    s_bfe_u32 s14, s1, 0x4000c
+; GFX7-NEXT:    s_bfe_u32 s16, s1, 0x40004
 ; GFX7-NEXT:    s_lshr_b32 s18, s1, 28
-; GFX7-NEXT:    s_bfe_u32 s20, s1, 0x40014
-; GFX7-NEXT:    v_mov_b32_e32 v4, s18
+; GFX7-NEXT:    v_mov_b32_e32 v6, s16
 ; GFX7-NEXT:    v_mov_b32_e32 v8, s14
 ; GFX7-NEXT:    s_bfe_u32 s15, s1, 0x40008
-; GFX7-NEXT:    s_bfe_u32 s16, s1, 0x40004
 ; GFX7-NEXT:    s_and_b32 s17, s1, 15
 ; GFX7-NEXT:    s_bfe_u32 s19, s1, 0x40018
-; GFX7-NEXT:    s_bfe_u32 s1, s1, 0x40010
-; GFX7-NEXT:    s_bfe_u32 s13, s0, 0x40014
-; GFX7-NEXT:    v_mov_b32_e32 v2, s20
-; GFX7-NEXT:    v_mul_u32_u24_e32 v2, s13, v2
+; GFX7-NEXT:    s_bfe_u32 s20, s1, 0x40014
+; GFX7-NEXT:    s_lshr_b32 s11, s0, 28
+; GFX7-NEXT:    v_mov_b32_e32 v4, s18
 ; GFX7-NEXT:    v_mul_u32_u24_e32 v4, s11, v4
+; GFX7-NEXT:    v_mul_u32_u24_e32 v6, s9, v6
 ; GFX7-NEXT:    v_mul_u32_u24_e32 v8, s2, v8
+; GFX7-NEXT:    s_bfe_u32 s1, s1, 0x40010
 ; GFX7-NEXT:    s_bfe_u32 s8, s0, 0x40008
 ; GFX7-NEXT:    v_mov_b32_e32 v7, s15
-; GFX7-NEXT:    s_bfe_u32 s9, s0, 0x40004
-; GFX7-NEXT:    v_mov_b32_e32 v6, s16
 ; GFX7-NEXT:    s_and_b32 s10, s0, 15
+; GFX7-NEXT:    v_mov_b32_e32 v5, s17
 ; GFX7-NEXT:    s_bfe_u32 s12, s0, 0x40018
 ; GFX7-NEXT:    v_mov_b32_e32 v3, s19
+; GFX7-NEXT:    s_bfe_u32 s13, s0, 0x40014
+; GFX7-NEXT:    v_mov_b32_e32 v2, s20
+; GFX7-NEXT:    v_mul_u32_u24_e32 v2, s13, v2
 ; GFX7-NEXT:    s_bfe_u32 s0, s0, 0x40010
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
-; GFX7-NEXT:    v_mov_b32_e32 v5, s17
-; GFX7-NEXT:    v_mul_u32_u24_e32 v6, s9, v6
-; GFX7-NEXT:    v_mul_u32_u24_e32 v1, s0, v1
-; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
 ; GFX7-NEXT:    v_mul_u32_u24_e32 v3, s12, v3
-; GFX7-NEXT:    v_mul_u32_u24_e32 v7, s8, v7
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v4, 8, v4
-; GFX7-NEXT:    v_lshlrev_b32_e32 v8, 8, v8
-; GFX7-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX7-NEXT:    v_or_b32_e32 v2, v3, v4
-; GFX7-NEXT:    v_or_b32_e32 v4, v7, v8
 ; GFX7-NEXT:    v_mul_u32_u24_e32 v5, s10, v5
+; GFX7-NEXT:    v_mul_u32_u24_e32 v7, s8, v7
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v6, 8, v6
-; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT:    v_or_b32_e32 v3, v5, v6
-; GFX7-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX7-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX7-NEXT:    v_or_b32_e32 v2, v3, v4
-; GFX7-NEXT:    v_alignbit_b32 v3, v1, v2, 8
-; GFX7-NEXT:    v_alignbit_b32 v4, v1, v2, 16
-; GFX7-NEXT:    v_lshrrev_b32_e32 v5, 24, v2
-; GFX7-NEXT:    v_lshrrev_b32_e32 v6, 8, v1
-; GFX7-NEXT:    v_lshrrev_b32_e32 v7, 16, v1
-; GFX7-NEXT:    v_lshrrev_b32_e32 v8, 24, v1
+; GFX7-NEXT:    v_lshlrev_b32_e32 v8, 8, v8
+; GFX7-NEXT:    v_or_b32_e32 v3, v3, v4
+; GFX7-NEXT:    v_or_b32_e32 v4, v5, v6
+; GFX7-NEXT:    v_or_b32_e32 v5, v7, v8
+; GFX7-NEXT:    v_mul_u32_u24_e32 v9, s0, v1
+; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
+; GFX7-NEXT:    v_or_b32_e32 v2, v9, v2
+; GFX7-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX7-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GFX7-NEXT:    v_or_b32_e32 v2, v2, v3
+; GFX7-NEXT:    v_or_b32_e32 v3, v4, v5
+; GFX7-NEXT:    v_alignbit_b32 v4, v2, v3, 8
+; GFX7-NEXT:    v_alignbit_b32 v5, v2, v3, 16
+; GFX7-NEXT:    v_lshrrev_b32_e32 v6, 24, v3
+; GFX7-NEXT:    v_lshrrev_b32_e32 v7, 8, v2
+; GFX7-NEXT:    v_lshrrev_b32_e32 v8, 16, v2
+; GFX7-NEXT:    v_lshrrev_b32_e32 v2, 24, v2
 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v3, v0
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v3
 ; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v4, v0
 ; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v5, v0
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
 ; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v6, v0
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v7, v0
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v8, v0
+; GFX7-NEXT:    v_mad_u32_u24 v0, s0, v1, v0
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v7
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v8
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
 ; GFX7-NEXT:    buffer_store_byte v0, off, s[4:7], 0
 ; GFX7-NEXT:    s_endpgm
 ;
@@ -2359,65 +2349,68 @@ define amdgpu_kernel void @udot8_acc8_vecMul(<8 x i4> addrspace(1)* %src1,
 ; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX8-NEXT:    s_load_dword s4, s[6:7], 0x0
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX8-NEXT:    flat_load_ubyte v2, v[0:1]
+; GFX8-NEXT:    s_load_dword s1, s[4:5], 0x0
+; GFX8-NEXT:    s_load_dword s2, s[6:7], 0x0
+; GFX8-NEXT:    s_mov_b32 s0, 0xffff
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_bfe_u32 s0, s2, 0x40004
-; GFX8-NEXT:    s_bfe_u32 s1, s2, 0x4000c
-; GFX8-NEXT:    s_bfe_u32 s5, s4, 0x40004
-; GFX8-NEXT:    s_and_b32 s6, s4, 15
-; GFX8-NEXT:    s_bfe_u32 s7, s4, 0x4000c
-; GFX8-NEXT:    s_bfe_u32 s8, s4, 0x40008
-; GFX8-NEXT:    v_mov_b32_e32 v3, s8
-; GFX8-NEXT:    s_bfe_u32 s10, s2, 0x40008
-; GFX8-NEXT:    v_mov_b32_e32 v4, s7
-; GFX8-NEXT:    v_mov_b32_e32 v5, s1
-; GFX8-NEXT:    v_mov_b32_e32 v6, s6
-; GFX8-NEXT:    v_mov_b32_e32 v7, s5
-; GFX8-NEXT:    v_mov_b32_e32 v8, s0
-; GFX8-NEXT:    s_and_b32 s9, s2, 15
+; GFX8-NEXT:    s_bfe_u32 s8, s1, 0x40004
+; GFX8-NEXT:    s_bfe_u32 s10, s1, 0x4000c
+; GFX8-NEXT:    s_bfe_u32 s15, s2, 0x40004
+; GFX8-NEXT:    s_and_b32 s16, s2, 15
+; GFX8-NEXT:    s_bfe_u32 s17, s2, 0x4000c
+; GFX8-NEXT:    s_bfe_u32 s4, s1, 0x40014
+; GFX8-NEXT:    s_lshr_b32 s6, s1, 28
+; GFX8-NEXT:    s_bfe_u32 s11, s2, 0x40014
+; GFX8-NEXT:    s_bfe_u32 s12, s2, 0x40010
+; GFX8-NEXT:    s_lshr_b32 s13, s2, 28
+; GFX8-NEXT:    s_bfe_u32 s14, s2, 0x40018
+; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x40008
+; GFX8-NEXT:    s_and_b32 s9, s1, 15
+; GFX8-NEXT:    v_mov_b32_e32 v4, s17
+; GFX8-NEXT:    v_mov_b32_e32 v5, s10
+; GFX8-NEXT:    v_mov_b32_e32 v6, s16
+; GFX8-NEXT:    v_mov_b32_e32 v7, s15
+; GFX8-NEXT:    v_mov_b32_e32 v8, s8
 ; GFX8-NEXT:    v_mul_u32_u24_sdwa v4, v5, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT:    v_mul_u32_u24_e32 v3, s10, v3
 ; GFX8-NEXT:    v_mul_u32_u24_e32 v5, s9, v6
 ; GFX8-NEXT:    v_mul_u32_u24_sdwa v6, v8, v7 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    s_bfe_u32 s5, s1, 0x40010
+; GFX8-NEXT:    s_bfe_u32 s7, s1, 0x40018
+; GFX8-NEXT:    v_mov_b32_e32 v9, s14
+; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x40008
+; GFX8-NEXT:    v_mov_b32_e32 v3, s2
+; GFX8-NEXT:    v_mov_b32_e32 v10, s13
+; GFX8-NEXT:    v_mov_b32_e32 v11, s6
+; GFX8-NEXT:    v_mov_b32_e32 v12, s12
+; GFX8-NEXT:    v_mov_b32_e32 v13, s11
+; GFX8-NEXT:    v_mov_b32_e32 v14, s4
+; GFX8-NEXT:    v_mul_u32_u24_e32 v3, s1, v3
 ; GFX8-NEXT:    v_or_b32_e32 v5, v5, v6
+; GFX8-NEXT:    v_mul_u32_u24_e32 v7, s7, v9
+; GFX8-NEXT:    v_mul_u32_u24_sdwa v8, v11, v10 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_mul_u32_u24_e32 v9, s5, v12
+; GFX8-NEXT:    v_mul_u32_u24_sdwa v10, v14, v13 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_and_b32_e32 v5, s0, v5
 ; GFX8-NEXT:    v_or_b32_sdwa v3, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT:    v_or_b32_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT:    s_bfe_u32 s0, s2, 0x40014
-; GFX8-NEXT:    s_lshr_b32 s1, s2, 28
-; GFX8-NEXT:    s_bfe_u32 s5, s4, 0x40014
-; GFX8-NEXT:    s_bfe_u32 s6, s4, 0x40010
-; GFX8-NEXT:    s_lshr_b32 s7, s4, 28
-; GFX8-NEXT:    s_bfe_u32 s4, s4, 0x40018
-; GFX8-NEXT:    s_bfe_u32 s8, s2, 0x40010
-; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x40018
-; GFX8-NEXT:    v_mov_b32_e32 v6, s4
-; GFX8-NEXT:    v_mov_b32_e32 v7, s7
-; GFX8-NEXT:    v_mov_b32_e32 v8, s1
-; GFX8-NEXT:    v_mov_b32_e32 v9, s6
-; GFX8-NEXT:    v_mov_b32_e32 v10, s5
-; GFX8-NEXT:    v_mov_b32_e32 v11, s0
-; GFX8-NEXT:    v_lshrrev_b32_e32 v5, 8, v3
-; GFX8-NEXT:    v_mul_u32_u24_sdwa v7, v8, v7 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT:    v_mul_u32_u24_e32 v6, s2, v6
-; GFX8-NEXT:    v_mul_u32_u24_e32 v8, s8, v9
-; GFX8-NEXT:    v_mul_u32_u24_sdwa v9, v11, v10 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT:    v_or_b32_e32 v8, v8, v9
-; GFX8-NEXT:    v_or_b32_sdwa v6, v6, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT:    v_or_b32_sdwa v4, v8, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT:    v_lshrrev_b32_e32 v6, 8, v4
+; GFX8-NEXT:    v_or_b32_e32 v9, v9, v10
+; GFX8-NEXT:    v_or_b32_sdwa v7, v7, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_and_b32_e32 v4, s0, v9
+; GFX8-NEXT:    v_or_b32_e32 v3, v5, v3
+; GFX8-NEXT:    v_or_b32_e32 v6, v4, v7
+; GFX8-NEXT:    v_lshrrev_b32_e32 v7, 8, v3
+; GFX8-NEXT:    v_lshrrev_b32_e32 v8, 8, v6
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
-; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v2, v3
-; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v5, v2
-; GFX8-NEXT:    v_add_u32_sdwa v2, vcc, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_2
-; GFX8-NEXT:    v_add_u32_sdwa v2, vcc, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v2, v5
+; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v7, v2
+; GFX8-NEXT:    v_add_u32_sdwa v2, vcc, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_0
+; GFX8-NEXT:    v_add_u32_sdwa v2, vcc, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
 ; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v2, v4
-; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v2, v6
-; GFX8-NEXT:    v_add_u32_sdwa v2, vcc, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT:    v_add_u32_sdwa v2, vcc, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v8, v2
+; GFX8-NEXT:    v_add_u32_sdwa v2, vcc, v6, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT:    v_add_u32_sdwa v2, vcc, v6, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
 ; GFX8-NEXT:    flat_store_byte v[0:1], v2
 ; GFX8-NEXT:    s_endpgm
 ;
@@ -2425,58 +2418,61 @@ define amdgpu_kernel void @udot8_acc8_vecMul(<8 x i4> addrspace(1)* %src1,
 ; GFX9:       ; %bb.0: ; %entry
 ; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX9-NEXT:    s_mov_b32 s2, 0xffff
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX9-NEXT:    s_load_dword s4, s[6:7], 0x0
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-NEXT:    global_load_ubyte v2, v[0:1], off
+; GFX9-NEXT:    s_load_dword s0, s[4:5], 0x0
+; GFX9-NEXT:    s_load_dword s1, s[6:7], 0x0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_and_b32 s0, s2, 15
-; GFX9-NEXT:    s_and_b32 s1, s4, 15
-; GFX9-NEXT:    s_bfe_u32 s5, s4, 0x40004
-; GFX9-NEXT:    s_bfe_u32 s6, s4, 0x40008
-; GFX9-NEXT:    s_bfe_u32 s7, s4, 0x4000c
-; GFX9-NEXT:    v_mov_b32_e32 v3, s1
-; GFX9-NEXT:    v_mov_b32_e32 v4, s5
-; GFX9-NEXT:    s_bfe_u32 s8, s2, 0x40004
-; GFX9-NEXT:    v_mov_b32_e32 v5, s6
-; GFX9-NEXT:    s_bfe_u32 s9, s2, 0x40008
-; GFX9-NEXT:    v_mov_b32_e32 v6, s7
-; GFX9-NEXT:    s_bfe_u32 s10, s2, 0x4000c
-; GFX9-NEXT:    v_mul_lo_u16_e32 v3, s0, v3
-; GFX9-NEXT:    v_mul_lo_u16_sdwa v4, s8, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    v_mul_lo_u16_e32 v5, s9, v5
-; GFX9-NEXT:    v_mul_lo_u16_sdwa v6, s10, v6 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT:    s_bfe_u32 s4, s0, 0x40010
+; GFX9-NEXT:    s_bfe_u32 s11, s1, 0x40010
+; GFX9-NEXT:    s_bfe_u32 s12, s1, 0x40014
+; GFX9-NEXT:    s_bfe_u32 s13, s1, 0x40018
+; GFX9-NEXT:    s_lshr_b32 s14, s1, 28
+; GFX9-NEXT:    s_and_b32 s15, s1, 15
+; GFX9-NEXT:    s_bfe_u32 s16, s1, 0x40004
+; GFX9-NEXT:    s_bfe_u32 s17, s1, 0x40008
+; GFX9-NEXT:    v_mov_b32_e32 v3, s11
+; GFX9-NEXT:    s_bfe_u32 s1, s1, 0x4000c
+; GFX9-NEXT:    s_bfe_u32 s5, s0, 0x40014
+; GFX9-NEXT:    v_mov_b32_e32 v4, s12
+; GFX9-NEXT:    s_bfe_u32 s6, s0, 0x40018
+; GFX9-NEXT:    v_mov_b32_e32 v5, s13
+; GFX9-NEXT:    s_lshr_b32 s7, s0, 28
+; GFX9-NEXT:    v_mov_b32_e32 v6, s14
+; GFX9-NEXT:    s_and_b32 s8, s0, 15
+; GFX9-NEXT:    v_mov_b32_e32 v7, s15
+; GFX9-NEXT:    s_bfe_u32 s9, s0, 0x40004
+; GFX9-NEXT:    v_mov_b32_e32 v8, s16
+; GFX9-NEXT:    s_bfe_u32 s10, s0, 0x40008
+; GFX9-NEXT:    v_mov_b32_e32 v9, s17
+; GFX9-NEXT:    s_bfe_u32 s0, s0, 0x4000c
+; GFX9-NEXT:    v_mov_b32_e32 v10, s1
+; GFX9-NEXT:    v_mul_lo_u16_e32 v3, s4, v3
+; GFX9-NEXT:    v_mul_lo_u16_sdwa v4, s5, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT:    v_mul_lo_u16_e32 v5, s6, v5
+; GFX9-NEXT:    v_mul_lo_u16_sdwa v6, s7, v6 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT:    v_mul_lo_u16_e32 v7, s8, v7
+; GFX9-NEXT:    v_mul_lo_u16_sdwa v8, s9, v8 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v3, v3, v4
 ; GFX9-NEXT:    v_or_b32_sdwa v4, v5, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    s_bfe_u32 s1, s4, 0x40014
-; GFX9-NEXT:    s_bfe_u32 s5, s4, 0x40018
-; GFX9-NEXT:    s_bfe_u32 s0, s4, 0x40010
-; GFX9-NEXT:    s_lshr_b32 s4, s4, 28
-; GFX9-NEXT:    v_or_b32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    s_bfe_u32 s6, s2, 0x40010
-; GFX9-NEXT:    v_mov_b32_e32 v4, s0
-; GFX9-NEXT:    s_bfe_u32 s7, s2, 0x40014
-; GFX9-NEXT:    v_mov_b32_e32 v5, s1
-; GFX9-NEXT:    s_bfe_u32 s8, s2, 0x40018
-; GFX9-NEXT:    v_mov_b32_e32 v6, s5
-; GFX9-NEXT:    s_lshr_b32 s2, s2, 28
-; GFX9-NEXT:    v_mov_b32_e32 v7, s4
-; GFX9-NEXT:    v_mul_lo_u16_e32 v4, s6, v4
-; GFX9-NEXT:    v_mul_lo_u16_sdwa v5, s7, v5 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    v_mul_lo_u16_e32 v6, s8, v6
-; GFX9-NEXT:    v_mul_lo_u16_sdwa v7, s2, v7 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_e32 v4, v4, v5
-; GFX9-NEXT:    v_or_b32_sdwa v5, v6, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v4, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    v_lshrrev_b32_e32 v5, 8, v3
+; GFX9-NEXT:    v_or_b32_e32 v5, v7, v8
+; GFX9-NEXT:    v_mul_lo_u16_e32 v9, s10, v9
+; GFX9-NEXT:    v_mul_lo_u16_sdwa v10, s0, v10 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT:    v_and_b32_e32 v5, s2, v5
+; GFX9-NEXT:    v_or_b32_sdwa v6, v9, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_e32 v6, v5, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v7, 8, v6
+; GFX9-NEXT:    v_and_b32_e32 v3, s2, v3
+; GFX9-NEXT:    v_or_b32_e32 v4, v3, v4
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_add_u32_e32 v2, v3, v2
-; GFX9-NEXT:    v_add_u32_e32 v2, v2, v5
-; GFX9-NEXT:    v_add_u32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_2
-; GFX9-NEXT:    v_add_u32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
-; GFX9-NEXT:    v_add_u32_e32 v2, v2, v4
+; GFX9-NEXT:    v_add_u32_e32 v2, v5, v2
+; GFX9-NEXT:    v_add_u32_e32 v2, v2, v7
+; GFX9-NEXT:    v_add_u32_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_2
+; GFX9-NEXT:    v_add_u32_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+; GFX9-NEXT:    v_add_u32_e32 v2, v2, v3
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v3, 8, v4
 ; GFX9-NEXT:    v_add_u32_e32 v2, v2, v3
 ; GFX9-NEXT:    v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
@@ -2488,58 +2484,61 @@ define amdgpu_kernel void @udot8_acc8_vecMul(<8 x i4> addrspace(1)* %src1,
 ; GFX9-DL:       ; %bb.0: ; %entry
 ; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX9-DL-NEXT:    s_mov_b32 s2, 0xffff
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-DL-NEXT:    global_load_ubyte v2, v[0:1], off
+; GFX9-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
+; GFX9-DL-NEXT:    s_load_dword s1, s[6:7], 0x0
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    s_and_b32 s0, s2, 15
-; GFX9-DL-NEXT:    s_and_b32 s1, s4, 15
-; GFX9-DL-NEXT:    s_bfe_u32 s5, s4, 0x40004
-; GFX9-DL-NEXT:    s_bfe_u32 s6, s4, 0x40008
-; GFX9-DL-NEXT:    s_bfe_u32 s7, s4, 0x4000c
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s1
-; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s5
-; GFX9-DL-NEXT:    s_bfe_u32 s8, s2, 0x40004
-; GFX9-DL-NEXT:    v_mov_b32_e32 v5, s6
-; GFX9-DL-NEXT:    s_bfe_u32 s9, s2, 0x40008
-; GFX9-DL-NEXT:    v_mov_b32_e32 v6, s7
-; GFX9-DL-NEXT:    s_bfe_u32 s10, s2, 0x4000c
-; GFX9-DL-NEXT:    v_mul_lo_u16_e32 v3, s0, v3
-; GFX9-DL-NEXT:    v_mul_lo_u16_sdwa v4, s8, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-DL-NEXT:    v_mul_lo_u16_e32 v5, s9, v5
-; GFX9-DL-NEXT:    v_mul_lo_u16_sdwa v6, s10, v6 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-DL-NEXT:    s_bfe_u32 s4, s0, 0x40010
+; GFX9-DL-NEXT:    s_bfe_u32 s11, s1, 0x40010
+; GFX9-DL-NEXT:    s_bfe_u32 s12, s1, 0x40014
+; GFX9-DL-NEXT:    s_bfe_u32 s13, s1, 0x40018
+; GFX9-DL-NEXT:    s_lshr_b32 s14, s1, 28
+; GFX9-DL-NEXT:    s_and_b32 s15, s1, 15
+; GFX9-DL-NEXT:    s_bfe_u32 s16, s1, 0x40004
+; GFX9-DL-NEXT:    s_bfe_u32 s17, s1, 0x40008
+; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s11
+; GFX9-DL-NEXT:    s_bfe_u32 s1, s1, 0x4000c
+; GFX9-DL-NEXT:    s_bfe_u32 s5, s0, 0x40014
+; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s12
+; GFX9-DL-NEXT:    s_bfe_u32 s6, s0, 0x40018
+; GFX9-DL-NEXT:    v_mov_b32_e32 v5, s13
+; GFX9-DL-NEXT:    s_lshr_b32 s7, s0, 28
+; GFX9-DL-NEXT:    v_mov_b32_e32 v6, s14
+; GFX9-DL-NEXT:    s_and_b32 s8, s0, 15
+; GFX9-DL-NEXT:    v_mov_b32_e32 v7, s15
+; GFX9-DL-NEXT:    s_bfe_u32 s9, s0, 0x40004
+; GFX9-DL-NEXT:    v_mov_b32_e32 v8, s16
+; GFX9-DL-NEXT:    s_bfe_u32 s10, s0, 0x40008
+; GFX9-DL-NEXT:    v_mov_b32_e32 v9, s17
+; GFX9-DL-NEXT:    s_bfe_u32 s0, s0, 0x4000c
+; GFX9-DL-NEXT:    v_mov_b32_e32 v10, s1
+; GFX9-DL-NEXT:    v_mul_lo_u16_e32 v3, s4, v3
+; GFX9-DL-NEXT:    v_mul_lo_u16_sdwa v4, s5, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-DL-NEXT:    v_mul_lo_u16_e32 v5, s6, v5
+; GFX9-DL-NEXT:    v_mul_lo_u16_sdwa v6, s7, v6 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-DL-NEXT:    v_mul_lo_u16_e32 v7, s8, v7
+; GFX9-DL-NEXT:    v_mul_lo_u16_sdwa v8, s9, v8 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-DL-NEXT:    v_or_b32_e32 v3, v3, v4
 ; GFX9-DL-NEXT:    v_or_b32_sdwa v4, v5, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-DL-NEXT:    s_bfe_u32 s1, s4, 0x40014
-; GFX9-DL-NEXT:    s_bfe_u32 s5, s4, 0x40018
-; GFX9-DL-NEXT:    s_bfe_u32 s0, s4, 0x40010
-; GFX9-DL-NEXT:    s_lshr_b32 s4, s4, 28
-; GFX9-DL-NEXT:    v_or_b32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-DL-NEXT:    s_bfe_u32 s6, s2, 0x40010
-; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s0
-; GFX9-DL-NEXT:    s_bfe_u32 s7, s2, 0x40014
-; GFX9-DL-NEXT:    v_mov_b32_e32 v5, s1
-; GFX9-DL-NEXT:    s_bfe_u32 s8, s2, 0x40018
-; GFX9-DL-NEXT:    v_mov_b32_e32 v6, s5
-; GFX9-DL-NEXT:    s_lshr_b32 s2, s2, 28
-; GFX9-DL-NEXT:    v_mov_b32_e32 v7, s4
-; GFX9-DL-NEXT:    v_mul_lo_u16_e32 v4, s6, v4
-; GFX9-DL-NEXT:    v_mul_lo_u16_sdwa v5, s7, v5 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-DL-NEXT:    v_mul_lo_u16_e32 v6, s8, v6
-; GFX9-DL-NEXT:    v_mul_lo_u16_sdwa v7, s2, v7 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-DL-NEXT:    v_or_b32_e32 v4, v4, v5
-; GFX9-DL-NEXT:    v_or_b32_sdwa v5, v6, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-DL-NEXT:    v_or_b32_sdwa v4, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-DL-NEXT:    v_lshrrev_b32_e32 v5, 8, v3
+; GFX9-DL-NEXT:    v_or_b32_e32 v5, v7, v8
+; GFX9-DL-NEXT:    v_mul_lo_u16_e32 v9, s10, v9
+; GFX9-DL-NEXT:    v_mul_lo_u16_sdwa v10, s0, v10 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-DL-NEXT:    v_and_b32_e32 v5, s2, v5
+; GFX9-DL-NEXT:    v_or_b32_sdwa v6, v9, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-DL-NEXT:    v_or_b32_e32 v6, v5, v6
+; GFX9-DL-NEXT:    v_lshrrev_b32_e32 v7, 8, v6
+; GFX9-DL-NEXT:    v_and_b32_e32 v3, s2, v3
+; GFX9-DL-NEXT:    v_or_b32_e32 v4, v3, v4
 ; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-DL-NEXT:    v_add_u32_e32 v2, v3, v2
-; GFX9-DL-NEXT:    v_add_u32_e32 v2, v2, v5
-; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_2
-; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
-; GFX9-DL-NEXT:    v_add_u32_e32 v2, v2, v4
+; GFX9-DL-NEXT:    v_add_u32_e32 v2, v5, v2
+; GFX9-DL-NEXT:    v_add_u32_e32 v2, v2, v7
+; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_2
+; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+; GFX9-DL-NEXT:    v_add_u32_e32 v2, v2, v3
 ; GFX9-DL-NEXT:    v_lshrrev_b32_e32 v3, 8, v4
 ; GFX9-DL-NEXT:    v_add_u32_e32 v2, v2, v3
 ; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
@@ -2551,60 +2550,62 @@ define amdgpu_kernel void @udot8_acc8_vecMul(<8 x i4> addrspace(1)* %src1,
 ; GFX10-DL:       ; %bb.0: ; %entry
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX10-DL-NEXT:    v_mov_b32_e32 v2, 0xffff
+; GFX10-DL-NEXT:    s_mov_b32 s2, 0xffff
 ; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX10-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
-; GFX10-DL-NEXT:    global_load_ubyte v3, v[0:1], off
+; GFX10-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
+; GFX10-DL-NEXT:    s_load_dword s1, s[6:7], 0x0
+; GFX10-DL-NEXT:    global_load_ubyte v2, v[0:1], off
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_bfe_u32 s0, s2, 0x40004
-; GFX10-DL-NEXT:    s_bfe_u32 s5, s4, 0x40004
-; GFX10-DL-NEXT:    s_bfe_u32 s1, s2, 0x4000c
-; GFX10-DL-NEXT:    s_bfe_u32 s6, s4, 0x4000c
-; GFX10-DL-NEXT:    s_and_b32 s7, s2, 15
-; GFX10-DL-NEXT:    s_and_b32 s9, s4, 15
-; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v4, s0, s5
-; GFX10-DL-NEXT:    s_bfe_u32 s8, s2, 0x40008
-; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v5, s1, s6
-; GFX10-DL-NEXT:    s_bfe_u32 s10, s4, 0x40008
-; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v6, s7, s9
-; GFX10-DL-NEXT:    v_and_b32_sdwa v4, v4, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-DL-NEXT:    s_bfe_u32 s0, s2, 0x40014
-; GFX10-DL-NEXT:    v_and_b32_sdwa v5, v5, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v7, s8, s10
-; GFX10-DL-NEXT:    s_lshr_b32 s1, s2, 28
-; GFX10-DL-NEXT:    v_or_b32_sdwa v4, v6, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
-; GFX10-DL-NEXT:    s_lshr_b32 s6, s4, 28
-; GFX10-DL-NEXT:    s_bfe_u32 s5, s4, 0x40014
-; GFX10-DL-NEXT:    v_or_b32_sdwa v5, v7, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
-; GFX10-DL-NEXT:    s_bfe_u32 s7, s2, 0x40010
-; GFX10-DL-NEXT:    s_bfe_u32 s8, s4, 0x40010
-; GFX10-DL-NEXT:    s_bfe_u32 s2, s2, 0x40018
-; GFX10-DL-NEXT:    s_bfe_u32 s4, s4, 0x40018
-; GFX10-DL-NEXT:    v_or_b32_sdwa v4, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v5, s0, s5
-; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v11, s1, s6
-; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v8, s7, s8
-; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v9, s2, s4
-; GFX10-DL-NEXT:    v_lshrrev_b32_e32 v7, 8, v4
-; GFX10-DL-NEXT:    v_and_b32_sdwa v5, v5, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-DL-NEXT:    v_and_b32_sdwa v2, v11, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-DL-NEXT:    v_or_b32_sdwa v5, v8, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
-; GFX10-DL-NEXT:    v_or_b32_sdwa v2, v9, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
-; GFX10-DL-NEXT:    v_or_b32_sdwa v2, v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX10-DL-NEXT:    s_bfe_u32 s4, s0, 0x40004
+; GFX10-DL-NEXT:    s_bfe_u32 s5, s1, 0x40004
+; GFX10-DL-NEXT:    s_and_b32 s6, s0, 15
+; GFX10-DL-NEXT:    s_and_b32 s8, s1, 15
+; GFX10-DL-NEXT:    s_bfe_u32 s7, s0, 0x4000c
+; GFX10-DL-NEXT:    s_bfe_u32 s9, s1, 0x4000c
+; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v3, s4, s5
+; GFX10-DL-NEXT:    s_bfe_u32 s4, s0, 0x40008
+; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v4, s6, s8
+; GFX10-DL-NEXT:    s_bfe_u32 s5, s1, 0x40008
+; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v5, s7, s9
+; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v3, 8, v3
+; GFX10-DL-NEXT:    s_bfe_u32 s7, s1, 0x40014
+; GFX10-DL-NEXT:    s_bfe_u32 s6, s0, 0x40018
+; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v6, s4, s5
+; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v5, 8, v5
+; GFX10-DL-NEXT:    v_or_b32_e32 v3, v4, v3
+; GFX10-DL-NEXT:    s_bfe_u32 s5, s0, 0x40014
+; GFX10-DL-NEXT:    s_bfe_u32 s4, s0, 0x40010
+; GFX10-DL-NEXT:    s_lshr_b32 s0, s0, 28
+; GFX10-DL-NEXT:    v_or_b32_sdwa v4, v6, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-DL-NEXT:    v_and_b32_e32 v3, s2, v3
+; GFX10-DL-NEXT:    s_bfe_u32 s8, s1, 0x40010
+; GFX10-DL-NEXT:    s_lshr_b32 s9, s1, 28
+; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v5, s5, s7
+; GFX10-DL-NEXT:    s_bfe_u32 s1, s1, 0x40018
+; GFX10-DL-NEXT:    v_or_b32_e32 v4, v3, v4
+; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v6, s4, s8
+; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v7, s0, s9
+; GFX10-DL-NEXT:    v_lshrrev_b32_e32 v8, 8, v4
+; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v7, 8, v7
 ; GFX10-DL-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-DL-NEXT:    v_add_nc_u32_e32 v6, v4, v3
-; GFX10-DL-NEXT:    v_add_nc_u32_e32 v3, v6, v7
-; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_2
-; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
-; GFX10-DL-NEXT:    v_lshrrev_b32_e32 v4, 8, v2
-; GFX10-DL-NEXT:    v_add_nc_u32_e32 v3, v3, v2
-; GFX10-DL-NEXT:    v_add_nc_u32_e32 v3, v3, v4
-; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v3, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v3, v2
+; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v3, 8, v5
+; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v5, s6, s1
+; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v2, v8
+; GFX10-DL-NEXT:    v_or_b32_e32 v3, v6, v3
+; GFX10-DL-NEXT:    v_or_b32_sdwa v5, v5, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_2
+; GFX10-DL-NEXT:    v_and_b32_e32 v3, s2, v3
+; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+; GFX10-DL-NEXT:    v_or_b32_e32 v4, v3, v5
+; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v2, v3
+; GFX10-DL-NEXT:    v_lshrrev_b32_e32 v3, 8, v4
+; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v2, v3
+; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
 ; GFX10-DL-NEXT:    global_store_byte v[0:1], v2, off
 ; GFX10-DL-NEXT:    s_endpgm
                                              <8 x i4> addrspace(1)* %src2,
@@ -2644,17 +2645,17 @@ entry:
 define amdgpu_kernel void @udot8_acc4_vecMul(<8 x i4> addrspace(1)* %src1,
 ; GFX7-LABEL: udot8_acc4_vecMul:
 ; GFX7:       ; %bb.0: ; %entry
-; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
-; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xd
-; GFX7-NEXT:    s_mov_b32 s11, 0xf000
-; GFX7-NEXT:    s_mov_b32 s10, -1
+; GFX7-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
+; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xd
+; GFX7-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7-NEXT:    s_mov_b32 s6, -1
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_load_dword s0, s[4:5], 0x0
-; GFX7-NEXT:    buffer_load_ubyte v0, off, s[8:11], 0
-; GFX7-NEXT:    s_load_dword s1, s[6:7], 0x0
+; GFX7-NEXT:    s_load_dword s0, s[8:9], 0x0
+; GFX7-NEXT:    buffer_load_ubyte v0, off, s[4:7], 0
+; GFX7-NEXT:    s_load_dword s1, s[10:11], 0x0
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    s_lshr_b32 s2, s0, 28
-; GFX7-NEXT:    s_bfe_u32 s4, s0, 0x40018
+; GFX7-NEXT:    s_bfe_u32 s8, s0, 0x40018
 ; GFX7-NEXT:    s_bfe_u32 s15, s1, 0x40018
 ; GFX7-NEXT:    s_bfe_u32 s16, s1, 0x40014
 ; GFX7-NEXT:    s_bfe_u32 s17, s1, 0x40010
@@ -2663,9 +2664,9 @@ define amdgpu_kernel void @udot8_acc4_vecMul(<8 x i4> addrspace(1)* %src1,
 ; GFX7-NEXT:    s_bfe_u32 s20, s1, 0x40004
 ; GFX7-NEXT:    s_lshr_b32 s14, s1, 28
 ; GFX7-NEXT:    s_and_b32 s1, s1, 15
-; GFX7-NEXT:    s_bfe_u32 s5, s0, 0x40014
-; GFX7-NEXT:    s_bfe_u32 s6, s0, 0x40010
-; GFX7-NEXT:    s_bfe_u32 s7, s0, 0x4000c
+; GFX7-NEXT:    s_bfe_u32 s9, s0, 0x40014
+; GFX7-NEXT:    s_bfe_u32 s10, s0, 0x40010
+; GFX7-NEXT:    s_bfe_u32 s11, s0, 0x4000c
 ; GFX7-NEXT:    s_bfe_u32 s12, s0, 0x40008
 ; GFX7-NEXT:    s_bfe_u32 s13, s0, 0x40004
 ; GFX7-NEXT:    s_and_b32 s0, s0, 15
@@ -2680,14 +2681,14 @@ define amdgpu_kernel void @udot8_acc4_vecMul(<8 x i4> addrspace(1)* %src1,
 ; GFX7-NEXT:    v_mad_u32_u24 v0, s0, v1, v0
 ; GFX7-NEXT:    v_mad_u32_u24 v0, s13, v2, v0
 ; GFX7-NEXT:    v_mad_u32_u24 v0, s12, v3, v0
-; GFX7-NEXT:    v_mad_u32_u24 v0, s7, v4, v0
-; GFX7-NEXT:    v_mad_u32_u24 v0, s6, v5, v0
-; GFX7-NEXT:    v_mad_u32_u24 v0, s5, v6, v0
-; GFX7-NEXT:    v_mad_u32_u24 v0, s4, v7, v0
+; GFX7-NEXT:    v_mad_u32_u24 v0, s11, v4, v0
+; GFX7-NEXT:    v_mad_u32_u24 v0, s10, v5, v0
+; GFX7-NEXT:    v_mad_u32_u24 v0, s9, v6, v0
+; GFX7-NEXT:    v_mad_u32_u24 v0, s8, v7, v0
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s14
 ; GFX7-NEXT:    v_mad_u32_u24 v0, s2, v1, v0
 ; GFX7-NEXT:    v_and_b32_e32 v0, 15, v0
-; GFX7-NEXT:    buffer_store_byte v0, off, s[8:11], 0
+; GFX7-NEXT:    buffer_store_byte v0, off, s[4:7], 0
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: udot8_acc4_vecMul:
@@ -2695,48 +2696,48 @@ define amdgpu_kernel void @udot8_acc4_vecMul(<8 x i4> addrspace(1)* %src1,
 ; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX8-NEXT:    s_load_dword s4, s[6:7], 0x0
+; GFX8-NEXT:    s_load_dword s2, s[6:7], 0x0
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX8-NEXT:    flat_load_ubyte v2, v[0:1]
+; GFX8-NEXT:    s_load_dword s0, s[4:5], 0x0
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_and_b32 s0, s2, 15
-; GFX8-NEXT:    s_and_b32 s1, s4, 15
+; GFX8-NEXT:    s_and_b32 s1, s2, 15
+; GFX8-NEXT:    s_bfe_u32 s4, s2, 0x40004
 ; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    s_bfe_u32 s5, s4, 0x40004
-; GFX8-NEXT:    s_bfe_u32 s6, s4, 0x40008
-; GFX8-NEXT:    s_bfe_u32 s7, s4, 0x4000c
-; GFX8-NEXT:    v_mov_b32_e32 v4, s5
-; GFX8-NEXT:    s_bfe_u32 s1, s2, 0x40004
-; GFX8-NEXT:    s_bfe_u32 s5, s2, 0x40008
-; GFX8-NEXT:    s_bfe_u32 s8, s2, 0x4000c
+; GFX8-NEXT:    s_bfe_u32 s6, s2, 0x40008
+; GFX8-NEXT:    s_and_b32 s1, s0, 15
+; GFX8-NEXT:    s_bfe_u32 s7, s2, 0x4000c
+; GFX8-NEXT:    v_mov_b32_e32 v4, s4
+; GFX8-NEXT:    s_bfe_u32 s5, s0, 0x40004
 ; GFX8-NEXT:    v_mov_b32_e32 v5, s7
+; GFX8-NEXT:    s_bfe_u32 s8, s0, 0x4000c
 ; GFX8-NEXT:    v_mov_b32_e32 v6, s6
+; GFX8-NEXT:    s_bfe_u32 s4, s0, 0x40008
 ; GFX8-NEXT:    v_mul_u32_u24_e32 v5, s8, v5
-; GFX8-NEXT:    s_bfe_u32 s9, s4, 0x40010
+; GFX8-NEXT:    s_bfe_u32 s6, s2, 0x40010
 ; GFX8-NEXT:    v_and_b32_e32 v5, 15, v5
-; GFX8-NEXT:    s_bfe_u32 s11, s4, 0x40014
-; GFX8-NEXT:    s_bfe_u32 s10, s2, 0x40010
-; GFX8-NEXT:    v_mov_b32_e32 v7, s9
-; GFX8-NEXT:    s_bfe_u32 s13, s4, 0x40018
-; GFX8-NEXT:    s_bfe_u32 s12, s2, 0x40014
-; GFX8-NEXT:    v_mov_b32_e32 v8, s11
-; GFX8-NEXT:    s_bfe_u32 s14, s2, 0x40018
-; GFX8-NEXT:    s_lshr_b32 s4, s4, 28
-; GFX8-NEXT:    v_mov_b32_e32 v9, s13
-; GFX8-NEXT:    s_lshr_b32 s2, s2, 28
+; GFX8-NEXT:    s_bfe_u32 s8, s2, 0x40014
+; GFX8-NEXT:    s_bfe_u32 s7, s0, 0x40010
+; GFX8-NEXT:    v_mov_b32_e32 v7, s6
+; GFX8-NEXT:    s_lshr_b32 s11, s2, 28
+; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x40018
+; GFX8-NEXT:    s_bfe_u32 s9, s0, 0x40014
+; GFX8-NEXT:    v_mov_b32_e32 v8, s8
+; GFX8-NEXT:    s_bfe_u32 s10, s0, 0x40018
+; GFX8-NEXT:    v_mov_b32_e32 v9, s2
+; GFX8-NEXT:    s_lshr_b32 s0, s0, 28
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
-; GFX8-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
-; GFX8-NEXT:    v_mad_u32_u24 v2, s1, v4, v2
-; GFX8-NEXT:    v_mad_u32_u24 v2, s5, v6, v2
+; GFX8-NEXT:    v_mad_u32_u24 v2, s1, v3, v2
+; GFX8-NEXT:    v_mad_u32_u24 v2, s5, v4, v2
+; GFX8-NEXT:    v_mad_u32_u24 v2, s4, v6, v2
 ; GFX8-NEXT:    v_and_b32_e32 v2, 15, v2
 ; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v5, v2
-; GFX8-NEXT:    v_mad_u32_u24 v2, s10, v7, v2
-; GFX8-NEXT:    v_mad_u32_u24 v2, s12, v8, v2
-; GFX8-NEXT:    v_mad_u32_u24 v2, s14, v9, v2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s4
-; GFX8-NEXT:    v_mad_u32_u24 v2, s2, v3, v2
+; GFX8-NEXT:    v_mad_u32_u24 v2, s7, v7, v2
+; GFX8-NEXT:    v_mad_u32_u24 v2, s9, v8, v2
+; GFX8-NEXT:    v_mad_u32_u24 v2, s10, v9, v2
+; GFX8-NEXT:    v_mov_b32_e32 v3, s11
+; GFX8-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
 ; GFX8-NEXT:    v_and_b32_e32 v2, 15, v2
 ; GFX8-NEXT:    flat_store_byte v[0:1], v2
 ; GFX8-NEXT:    s_endpgm
@@ -2746,48 +2747,48 @@ define amdgpu_kernel void @udot8_acc4_vecMul(<8 x i4> addrspace(1)* %src1,
 ; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX9-NEXT:    s_load_dword s4, s[6:7], 0x0
+; GFX9-NEXT:    s_load_dword s2, s[6:7], 0x0
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-NEXT:    global_load_ubyte v2, v[0:1], off
+; GFX9-NEXT:    s_load_dword s0, s[4:5], 0x0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_and_b32 s0, s2, 15
-; GFX9-NEXT:    s_and_b32 s1, s4, 15
+; GFX9-NEXT:    s_and_b32 s1, s2, 15
+; GFX9-NEXT:    s_bfe_u32 s4, s2, 0x40004
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s1
-; GFX9-NEXT:    s_bfe_u32 s5, s4, 0x40004
-; GFX9-NEXT:    s_bfe_u32 s6, s4, 0x40008
-; GFX9-NEXT:    s_bfe_u32 s7, s4, 0x4000c
-; GFX9-NEXT:    v_mov_b32_e32 v4, s5
-; GFX9-NEXT:    s_bfe_u32 s1, s2, 0x40004
-; GFX9-NEXT:    s_bfe_u32 s5, s2, 0x40008
-; GFX9-NEXT:    s_bfe_u32 s8, s2, 0x4000c
+; GFX9-NEXT:    s_bfe_u32 s6, s2, 0x40008
+; GFX9-NEXT:    s_and_b32 s1, s0, 15
+; GFX9-NEXT:    s_bfe_u32 s7, s2, 0x4000c
+; GFX9-NEXT:    v_mov_b32_e32 v4, s4
+; GFX9-NEXT:    s_bfe_u32 s5, s0, 0x40004
 ; GFX9-NEXT:    v_mov_b32_e32 v5, s7
+; GFX9-NEXT:    s_bfe_u32 s8, s0, 0x4000c
 ; GFX9-NEXT:    v_mov_b32_e32 v6, s6
+; GFX9-NEXT:    s_bfe_u32 s4, s0, 0x40008
 ; GFX9-NEXT:    v_mul_u32_u24_e32 v5, s8, v5
-; GFX9-NEXT:    s_bfe_u32 s9, s4, 0x40010
+; GFX9-NEXT:    s_bfe_u32 s6, s2, 0x40010
 ; GFX9-NEXT:    v_and_b32_e32 v5, 15, v5
-; GFX9-NEXT:    s_bfe_u32 s11, s4, 0x40014
-; GFX9-NEXT:    s_bfe_u32 s10, s2, 0x40010
-; GFX9-NEXT:    v_mov_b32_e32 v7, s9
-; GFX9-NEXT:    s_bfe_u32 s13, s4, 0x40018
-; GFX9-NEXT:    s_bfe_u32 s12, s2, 0x40014
-; GFX9-NEXT:    v_mov_b32_e32 v8, s11
-; GFX9-NEXT:    s_bfe_u32 s14, s2, 0x40018
-; GFX9-NEXT:    s_lshr_b32 s4, s4, 28
-; GFX9-NEXT:    v_mov_b32_e32 v9, s13
-; GFX9-NEXT:    s_lshr_b32 s2, s2, 28
+; GFX9-NEXT:    s_bfe_u32 s8, s2, 0x40014
+; GFX9-NEXT:    s_bfe_u32 s7, s0, 0x40010
+; GFX9-NEXT:    v_mov_b32_e32 v7, s6
+; GFX9-NEXT:    s_lshr_b32 s11, s2, 28
+; GFX9-NEXT:    s_bfe_u32 s2, s2, 0x40018
+; GFX9-NEXT:    s_bfe_u32 s9, s0, 0x40014
+; GFX9-NEXT:    v_mov_b32_e32 v8, s8
+; GFX9-NEXT:    s_bfe_u32 s10, s0, 0x40018
+; GFX9-NEXT:    v_mov_b32_e32 v9, s2
+; GFX9-NEXT:    s_lshr_b32 s0, s0, 28
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
-; GFX9-NEXT:    v_mad_u32_u24 v2, s1, v4, v2
-; GFX9-NEXT:    v_mad_u32_u24 v2, s5, v6, v2
+; GFX9-NEXT:    v_mad_u32_u24 v2, s1, v3, v2
+; GFX9-NEXT:    v_mad_u32_u24 v2, s5, v4, v2
+; GFX9-NEXT:    v_mad_u32_u24 v2, s4, v6, v2
 ; GFX9-NEXT:    v_and_b32_e32 v2, 15, v2
 ; GFX9-NEXT:    v_add_u32_e32 v2, v2, v5
-; GFX9-NEXT:    v_mad_u32_u24 v2, s10, v7, v2
-; GFX9-NEXT:    v_mad_u32_u24 v2, s12, v8, v2
-; GFX9-NEXT:    v_mad_u32_u24 v2, s14, v9, v2
-; GFX9-NEXT:    v_mov_b32_e32 v3, s4
-; GFX9-NEXT:    v_mad_u32_u24 v2, s2, v3, v2
+; GFX9-NEXT:    v_mad_u32_u24 v2, s7, v7, v2
+; GFX9-NEXT:    v_mad_u32_u24 v2, s9, v8, v2
+; GFX9-NEXT:    v_mad_u32_u24 v2, s10, v9, v2
+; GFX9-NEXT:    v_mov_b32_e32 v3, s11
+; GFX9-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
 ; GFX9-NEXT:    v_and_b32_e32 v2, 15, v2
 ; GFX9-NEXT:    global_store_byte v[0:1], v2, off
 ; GFX9-NEXT:    s_endpgm
@@ -2797,48 +2798,48 @@ define amdgpu_kernel void @udot8_acc4_vecMul(<8 x i4> addrspace(1)* %src1,
 ; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
+; GFX9-DL-NEXT:    s_load_dword s2, s[6:7], 0x0
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-DL-NEXT:    global_load_ubyte v2, v[0:1], off
+; GFX9-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    s_and_b32 s0, s2, 15
-; GFX9-DL-NEXT:    s_and_b32 s1, s4, 15
+; GFX9-DL-NEXT:    s_and_b32 s1, s2, 15
+; GFX9-DL-NEXT:    s_bfe_u32 s4, s2, 0x40004
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s1
-; GFX9-DL-NEXT:    s_bfe_u32 s5, s4, 0x40004
-; GFX9-DL-NEXT:    s_bfe_u32 s6, s4, 0x40008
-; GFX9-DL-NEXT:    s_bfe_u32 s7, s4, 0x4000c
-; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s5
-; GFX9-DL-NEXT:    s_bfe_u32 s1, s2, 0x40004
-; GFX9-DL-NEXT:    s_bfe_u32 s5, s2, 0x40008
-; GFX9-DL-NEXT:    s_bfe_u32 s8, s2, 0x4000c
+; GFX9-DL-NEXT:    s_bfe_u32 s6, s2, 0x40008
+; GFX9-DL-NEXT:    s_and_b32 s1, s0, 15
+; GFX9-DL-NEXT:    s_bfe_u32 s7, s2, 0x4000c
+; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s4
+; GFX9-DL-NEXT:    s_bfe_u32 s5, s0, 0x40004
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v5, s7
+; GFX9-DL-NEXT:    s_bfe_u32 s8, s0, 0x4000c
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v6, s6
+; GFX9-DL-NEXT:    s_bfe_u32 s4, s0, 0x40008
 ; GFX9-DL-NEXT:    v_mul_u32_u24_e32 v5, s8, v5
-; GFX9-DL-NEXT:    s_bfe_u32 s9, s4, 0x40010
+; GFX9-DL-NEXT:    s_bfe_u32 s6, s2, 0x40010
 ; GFX9-DL-NEXT:    v_and_b32_e32 v5, 15, v5
-; GFX9-DL-NEXT:    s_bfe_u32 s11, s4, 0x40014
-; GFX9-DL-NEXT:    s_bfe_u32 s10, s2, 0x40010
-; GFX9-DL-NEXT:    v_mov_b32_e32 v7, s9
-; GFX9-DL-NEXT:    s_bfe_u32 s13, s4, 0x40018
-; GFX9-DL-NEXT:    s_bfe_u32 s12, s2, 0x40014
-; GFX9-DL-NEXT:    v_mov_b32_e32 v8, s11
-; GFX9-DL-NEXT:    s_bfe_u32 s14, s2, 0x40018
-; GFX9-DL-NEXT:    s_lshr_b32 s4, s4, 28
-; GFX9-DL-NEXT:    v_mov_b32_e32 v9, s13
-; GFX9-DL-NEXT:    s_lshr_b32 s2, s2, 28
+; GFX9-DL-NEXT:    s_bfe_u32 s8, s2, 0x40014
+; GFX9-DL-NEXT:    s_bfe_u32 s7, s0, 0x40010
+; GFX9-DL-NEXT:    v_mov_b32_e32 v7, s6
+; GFX9-DL-NEXT:    s_lshr_b32 s11, s2, 28
+; GFX9-DL-NEXT:    s_bfe_u32 s2, s2, 0x40018
+; GFX9-DL-NEXT:    s_bfe_u32 s9, s0, 0x40014
+; GFX9-DL-NEXT:    v_mov_b32_e32 v8, s8
+; GFX9-DL-NEXT:    s_bfe_u32 s10, s0, 0x40018
+; GFX9-DL-NEXT:    v_mov_b32_e32 v9, s2
+; GFX9-DL-NEXT:    s_lshr_b32 s0, s0, 28
 ; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
-; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s1, v4, v2
-; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s5, v6, v2
+; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s1, v3, v2
+; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s5, v4, v2
+; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s4, v6, v2
 ; GFX9-DL-NEXT:    v_and_b32_e32 v2, 15, v2
 ; GFX9-DL-NEXT:    v_add_u32_e32 v2, v2, v5
-; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s10, v7, v2
-; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s12, v8, v2
-; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s14, v9, v2
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s4
-; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s2, v3, v2
+; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s7, v7, v2
+; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s9, v8, v2
+; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s10, v9, v2
+; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s11
+; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
 ; GFX9-DL-NEXT:    v_and_b32_e32 v2, 15, v2
 ; GFX9-DL-NEXT:    global_store_byte v[0:1], v2, off
 ; GFX9-DL-NEXT:    s_endpgm
@@ -2849,40 +2850,40 @@ define amdgpu_kernel void @udot8_acc4_vecMul(<8 x i4> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX10-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
+; GFX10-DL-NEXT:    s_load_dword s1, s[6:7], 0x0
 ; GFX10-DL-NEXT:    global_load_ubyte v2, v[0:1], off
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_and_b32 s0, s2, 15
-; GFX10-DL-NEXT:    s_and_b32 s1, s4, 15
-; GFX10-DL-NEXT:    s_bfe_u32 s5, s2, 0x40004
-; GFX10-DL-NEXT:    s_bfe_u32 s6, s4, 0x40004
-; GFX10-DL-NEXT:    s_bfe_u32 s7, s2, 0x40008
-; GFX10-DL-NEXT:    s_bfe_u32 s8, s2, 0x4000c
-; GFX10-DL-NEXT:    s_bfe_u32 s9, s4, 0x40008
+; GFX10-DL-NEXT:    s_and_b32 s2, s0, 15
+; GFX10-DL-NEXT:    s_and_b32 s4, s1, 15
+; GFX10-DL-NEXT:    s_bfe_u32 s5, s0, 0x40004
+; GFX10-DL-NEXT:    s_bfe_u32 s6, s1, 0x40004
+; GFX10-DL-NEXT:    s_bfe_u32 s7, s1, 0x40008
 ; GFX10-DL-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s0, s1, v2
-; GFX10-DL-NEXT:    s_bfe_u32 s0, s4, 0x4000c
-; GFX10-DL-NEXT:    s_bfe_u32 s1, s4, 0x40010
+; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s2, s4, v2
+; GFX10-DL-NEXT:    s_bfe_u32 s2, s0, 0x40008
+; GFX10-DL-NEXT:    s_bfe_u32 s4, s0, 0x4000c
 ; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s5, s6, v2
-; GFX10-DL-NEXT:    v_mul_u32_u24_e64 v3, s8, s0
-; GFX10-DL-NEXT:    s_bfe_u32 s0, s2, 0x40010
-; GFX10-DL-NEXT:    s_bfe_u32 s5, s2, 0x40014
-; GFX10-DL-NEXT:    s_bfe_u32 s6, s4, 0x40014
-; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s7, s9, v2
-; GFX10-DL-NEXT:    v_and_b32_e32 v3, 15, v3
+; GFX10-DL-NEXT:    s_bfe_u32 s5, s1, 0x4000c
+; GFX10-DL-NEXT:    s_bfe_u32 s6, s1, 0x40014
+; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s2, s7, v2
+; GFX10-DL-NEXT:    v_mul_u32_u24_e64 v3, s4, s5
+; GFX10-DL-NEXT:    s_bfe_u32 s2, s0, 0x40010
+; GFX10-DL-NEXT:    s_bfe_u32 s4, s1, 0x40010
+; GFX10-DL-NEXT:    s_bfe_u32 s5, s0, 0x40014
 ; GFX10-DL-NEXT:    v_and_b32_e32 v2, 15, v2
+; GFX10-DL-NEXT:    v_and_b32_e32 v3, 15, v3
 ; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v2, v3
-; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s0, s1, v2
-; GFX10-DL-NEXT:    s_bfe_u32 s0, s2, 0x40018
-; GFX10-DL-NEXT:    s_bfe_u32 s1, s4, 0x40018
-; GFX10-DL-NEXT:    s_lshr_b32 s2, s2, 28
-; GFX10-DL-NEXT:    s_lshr_b32 s4, s4, 28
+; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s2, s4, v2
+; GFX10-DL-NEXT:    s_bfe_u32 s2, s0, 0x40018
+; GFX10-DL-NEXT:    s_bfe_u32 s4, s1, 0x40018
+; GFX10-DL-NEXT:    s_lshr_b32 s0, s0, 28
+; GFX10-DL-NEXT:    s_lshr_b32 s1, s1, 28
 ; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s5, s6, v2
-; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s0, s1, v2
 ; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s2, s4, v2
+; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s0, s1, v2
 ; GFX10-DL-NEXT:    v_and_b32_e32 v2, 15, v2
 ; GFX10-DL-NEXT:    global_store_byte v[0:1], v2, off
 ; GFX10-DL-NEXT:    s_endpgm
@@ -2926,41 +2927,41 @@ define amdgpu_kernel void @udot8_variant1(i32 addrspace(1)* %v1addr,
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
-; GFX7-NEXT:    s_load_dword s6, s[0:1], 0x0
+; GFX7-NEXT:    s_load_dword s20, s[0:1], 0x0
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_and_b32 s7, s4, 15
-; GFX7-NEXT:    s_and_b32 s8, s5, 15
-; GFX7-NEXT:    s_bfe_u32 s9, s4, 0x40004
-; GFX7-NEXT:    s_bfe_u32 s11, s4, 0x40008
-; GFX7-NEXT:    s_bfe_u32 s13, s4, 0x4000c
-; GFX7-NEXT:    s_bfe_u32 s15, s4, 0x40010
-; GFX7-NEXT:    s_bfe_u32 s17, s4, 0x40014
-; GFX7-NEXT:    s_bfe_u32 s19, s4, 0x40018
+; GFX7-NEXT:    s_and_b32 s6, s4, 15
+; GFX7-NEXT:    s_and_b32 s7, s5, 15
+; GFX7-NEXT:    s_bfe_u32 s8, s4, 0x40004
+; GFX7-NEXT:    s_bfe_u32 s10, s4, 0x40008
+; GFX7-NEXT:    s_bfe_u32 s12, s4, 0x4000c
+; GFX7-NEXT:    s_bfe_u32 s14, s4, 0x40010
+; GFX7-NEXT:    s_bfe_u32 s16, s4, 0x40014
+; GFX7-NEXT:    s_bfe_u32 s18, s4, 0x40018
 ; GFX7-NEXT:    s_lshr_b32 s4, s4, 28
-; GFX7-NEXT:    v_mov_b32_e32 v0, s7
-; GFX7-NEXT:    v_mov_b32_e32 v1, s6
-; GFX7-NEXT:    v_mad_u32_u24 v0, s8, v0, v1
-; GFX7-NEXT:    s_bfe_u32 s10, s5, 0x40004
-; GFX7-NEXT:    s_bfe_u32 s12, s5, 0x40008
-; GFX7-NEXT:    s_bfe_u32 s14, s5, 0x4000c
-; GFX7-NEXT:    s_bfe_u32 s16, s5, 0x40010
-; GFX7-NEXT:    s_bfe_u32 s18, s5, 0x40014
-; GFX7-NEXT:    s_bfe_u32 s20, s5, 0x40018
+; GFX7-NEXT:    v_mov_b32_e32 v0, s6
+; GFX7-NEXT:    v_mov_b32_e32 v1, s20
+; GFX7-NEXT:    v_mad_u32_u24 v0, s7, v0, v1
+; GFX7-NEXT:    s_bfe_u32 s9, s5, 0x40004
+; GFX7-NEXT:    s_bfe_u32 s11, s5, 0x40008
+; GFX7-NEXT:    s_bfe_u32 s13, s5, 0x4000c
+; GFX7-NEXT:    s_bfe_u32 s15, s5, 0x40010
+; GFX7-NEXT:    s_bfe_u32 s17, s5, 0x40014
+; GFX7-NEXT:    s_bfe_u32 s19, s5, 0x40018
 ; GFX7-NEXT:    s_lshr_b32 s5, s5, 28
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s4
 ; GFX7-NEXT:    v_mad_u32_u24 v0, s5, v1, v0
-; GFX7-NEXT:    v_mov_b32_e32 v1, s9
-; GFX7-NEXT:    v_mad_u32_u24 v0, s10, v1, v0
-; GFX7-NEXT:    v_mov_b32_e32 v1, s11
-; GFX7-NEXT:    v_mad_u32_u24 v0, s12, v1, v0
-; GFX7-NEXT:    v_mov_b32_e32 v1, s13
-; GFX7-NEXT:    v_mad_u32_u24 v0, s14, v1, v0
-; GFX7-NEXT:    v_mov_b32_e32 v1, s15
-; GFX7-NEXT:    v_mad_u32_u24 v0, s16, v1, v0
-; GFX7-NEXT:    v_mov_b32_e32 v1, s17
-; GFX7-NEXT:    v_mad_u32_u24 v0, s18, v1, v0
-; GFX7-NEXT:    v_mov_b32_e32 v1, s19
-; GFX7-NEXT:    v_mad_u32_u24 v0, s20, v1, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s8
+; GFX7-NEXT:    v_mad_u32_u24 v0, s9, v1, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s10
+; GFX7-NEXT:    v_mad_u32_u24 v0, s11, v1, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s12
+; GFX7-NEXT:    v_mad_u32_u24 v0, s13, v1, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s14
+; GFX7-NEXT:    v_mad_u32_u24 v0, s15, v1, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s16
+; GFX7-NEXT:    v_mad_u32_u24 v0, s17, v1, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s18
+; GFX7-NEXT:    v_mad_u32_u24 v0, s19, v1, v0
 ; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; GFX7-NEXT:    s_endpgm
 ;
@@ -2971,43 +2972,43 @@ define amdgpu_kernel void @udot8_variant1(i32 addrspace(1)* %v1addr,
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
 ; GFX8-NEXT:    s_load_dword s3, s[6:7], 0x0
-; GFX8-NEXT:    s_load_dword s4, s[0:1], 0x0
-; GFX8-NEXT:    v_mov_b32_e32 v0, s0
-; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    s_load_dword s18, s[0:1], 0x0
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_and_b32 s0, s2, 15
-; GFX8-NEXT:    s_and_b32 s1, s3, 15
-; GFX8-NEXT:    s_bfe_u32 s5, s2, 0x40004
-; GFX8-NEXT:    s_bfe_u32 s7, s2, 0x40008
-; GFX8-NEXT:    s_bfe_u32 s9, s2, 0x4000c
-; GFX8-NEXT:    s_bfe_u32 s11, s2, 0x40010
-; GFX8-NEXT:    s_bfe_u32 s13, s2, 0x40014
-; GFX8-NEXT:    s_bfe_u32 s15, s2, 0x40018
+; GFX8-NEXT:    s_and_b32 s4, s2, 15
+; GFX8-NEXT:    s_and_b32 s5, s3, 15
+; GFX8-NEXT:    s_bfe_u32 s6, s2, 0x40004
+; GFX8-NEXT:    s_bfe_u32 s8, s2, 0x40008
+; GFX8-NEXT:    s_bfe_u32 s10, s2, 0x4000c
+; GFX8-NEXT:    s_bfe_u32 s12, s2, 0x40010
+; GFX8-NEXT:    s_bfe_u32 s14, s2, 0x40014
+; GFX8-NEXT:    s_bfe_u32 s16, s2, 0x40018
 ; GFX8-NEXT:    s_lshr_b32 s2, s2, 28
-; GFX8-NEXT:    v_mov_b32_e32 v2, s0
-; GFX8-NEXT:    v_mov_b32_e32 v3, s4
-; GFX8-NEXT:    v_mad_u32_u24 v2, s1, v2, v3
-; GFX8-NEXT:    s_bfe_u32 s6, s3, 0x40004
-; GFX8-NEXT:    s_bfe_u32 s8, s3, 0x40008
-; GFX8-NEXT:    s_bfe_u32 s10, s3, 0x4000c
-; GFX8-NEXT:    s_bfe_u32 s12, s3, 0x40010
-; GFX8-NEXT:    s_bfe_u32 s14, s3, 0x40014
-; GFX8-NEXT:    s_bfe_u32 s16, s3, 0x40018
+; GFX8-NEXT:    v_mov_b32_e32 v0, s4
+; GFX8-NEXT:    v_mov_b32_e32 v1, s18
+; GFX8-NEXT:    v_mad_u32_u24 v0, s5, v0, v1
+; GFX8-NEXT:    s_bfe_u32 s7, s3, 0x40004
+; GFX8-NEXT:    s_bfe_u32 s9, s3, 0x40008
+; GFX8-NEXT:    s_bfe_u32 s11, s3, 0x4000c
+; GFX8-NEXT:    s_bfe_u32 s13, s3, 0x40010
+; GFX8-NEXT:    s_bfe_u32 s15, s3, 0x40014
+; GFX8-NEXT:    s_bfe_u32 s17, s3, 0x40018
 ; GFX8-NEXT:    s_lshr_b32 s3, s3, 28
-; GFX8-NEXT:    v_mov_b32_e32 v3, s2
-; GFX8-NEXT:    v_mad_u32_u24 v2, s3, v3, v2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s5
-; GFX8-NEXT:    v_mad_u32_u24 v2, s6, v3, v2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s7
-; GFX8-NEXT:    v_mad_u32_u24 v2, s8, v3, v2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s9
-; GFX8-NEXT:    v_mad_u32_u24 v2, s10, v3, v2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s11
-; GFX8-NEXT:    v_mad_u32_u24 v2, s12, v3, v2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s13
-; GFX8-NEXT:    v_mad_u32_u24 v2, s14, v3, v2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s15
-; GFX8-NEXT:    v_mad_u32_u24 v2, s16, v3, v2
+; GFX8-NEXT:    v_mov_b32_e32 v1, s2
+; GFX8-NEXT:    v_mad_u32_u24 v0, s3, v1, v0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s6
+; GFX8-NEXT:    v_mad_u32_u24 v0, s7, v1, v0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s8
+; GFX8-NEXT:    v_mad_u32_u24 v0, s9, v1, v0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s10
+; GFX8-NEXT:    v_mad_u32_u24 v0, s11, v1, v0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s12
+; GFX8-NEXT:    v_mad_u32_u24 v0, s13, v1, v0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s14
+; GFX8-NEXT:    v_mad_u32_u24 v0, s15, v1, v0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s16
+; GFX8-NEXT:    v_mad_u32_u24 v2, s17, v1, v0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX8-NEXT:    flat_store_dword v[0:1], v2
 ; GFX8-NEXT:    s_endpgm
 ;
@@ -3018,43 +3019,43 @@ define amdgpu_kernel void @udot8_variant1(i32 addrspace(1)* %v1addr,
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x0
 ; GFX9-NEXT:    s_load_dword s3, s[6:7], 0x0
-; GFX9-NEXT:    s_load_dword s4, s[0:1], 0x0
-; GFX9-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    s_load_dword s18, s[0:1], 0x0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_and_b32 s0, s2, 15
-; GFX9-NEXT:    s_and_b32 s1, s3, 15
-; GFX9-NEXT:    s_bfe_u32 s5, s2, 0x40004
-; GFX9-NEXT:    s_bfe_u32 s7, s2, 0x40008
-; GFX9-NEXT:    s_bfe_u32 s9, s2, 0x4000c
-; GFX9-NEXT:    s_bfe_u32 s11, s2, 0x40010
-; GFX9-NEXT:    s_bfe_u32 s13, s2, 0x40014
-; GFX9-NEXT:    s_bfe_u32 s15, s2, 0x40018
+; GFX9-NEXT:    s_and_b32 s4, s2, 15
+; GFX9-NEXT:    s_and_b32 s5, s3, 15
+; GFX9-NEXT:    s_bfe_u32 s6, s2, 0x40004
+; GFX9-NEXT:    s_bfe_u32 s8, s2, 0x40008
+; GFX9-NEXT:    s_bfe_u32 s10, s2, 0x4000c
+; GFX9-NEXT:    s_bfe_u32 s12, s2, 0x40010
+; GFX9-NEXT:    s_bfe_u32 s14, s2, 0x40014
+; GFX9-NEXT:    s_bfe_u32 s16, s2, 0x40018
 ; GFX9-NEXT:    s_lshr_b32 s2, s2, 28
-; GFX9-NEXT:    v_mov_b32_e32 v2, s0
-; GFX9-NEXT:    v_mov_b32_e32 v3, s4
-; GFX9-NEXT:    v_mad_u32_u24 v2, s1, v2, v3
-; GFX9-NEXT:    s_bfe_u32 s6, s3, 0x40004
-; GFX9-NEXT:    s_bfe_u32 s8, s3, 0x40008
-; GFX9-NEXT:    s_bfe_u32 s10, s3, 0x4000c
-; GFX9-NEXT:    s_bfe_u32 s12, s3, 0x40010
-; GFX9-NEXT:    s_bfe_u32 s14, s3, 0x40014
-; GFX9-NEXT:    s_bfe_u32 s16, s3, 0x40018
+; GFX9-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-NEXT:    v_mov_b32_e32 v1, s18
+; GFX9-NEXT:    v_mad_u32_u24 v0, s5, v0, v1
+; GFX9-NEXT:    s_bfe_u32 s7, s3, 0x40004
+; GFX9-NEXT:    s_bfe_u32 s9, s3, 0x40008
+; GFX9-NEXT:    s_bfe_u32 s11, s3, 0x4000c
+; GFX9-NEXT:    s_bfe_u32 s13, s3, 0x40010
+; GFX9-NEXT:    s_bfe_u32 s15, s3, 0x40014
+; GFX9-NEXT:    s_bfe_u32 s17, s3, 0x40018
 ; GFX9-NEXT:    s_lshr_b32 s3, s3, 28
-; GFX9-NEXT:    v_mov_b32_e32 v3, s2
-; GFX9-NEXT:    v_mad_u32_u24 v2, s3, v3, v2
-; GFX9-NEXT:    v_mov_b32_e32 v3, s5
-; GFX9-NEXT:    v_mad_u32_u24 v2, s6, v3, v2
-; GFX9-NEXT:    v_mov_b32_e32 v3, s7
-; GFX9-NEXT:    v_mad_u32_u24 v2, s8, v3, v2
-; GFX9-NEXT:    v_mov_b32_e32 v3, s9
-; GFX9-NEXT:    v_mad_u32_u24 v2, s10, v3, v2
-; GFX9-NEXT:    v_mov_b32_e32 v3, s11
-; GFX9-NEXT:    v_mad_u32_u24 v2, s12, v3, v2
-; GFX9-NEXT:    v_mov_b32_e32 v3, s13
-; GFX9-NEXT:    v_mad_u32_u24 v2, s14, v3, v2
-; GFX9-NEXT:    v_mov_b32_e32 v3, s15
-; GFX9-NEXT:    v_mad_u32_u24 v2, s16, v3, v2
+; GFX9-NEXT:    v_mov_b32_e32 v1, s2
+; GFX9-NEXT:    v_mad_u32_u24 v0, s3, v1, v0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s6
+; GFX9-NEXT:    v_mad_u32_u24 v0, s7, v1, v0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s8
+; GFX9-NEXT:    v_mad_u32_u24 v0, s9, v1, v0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s10
+; GFX9-NEXT:    v_mad_u32_u24 v0, s11, v1, v0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s12
+; GFX9-NEXT:    v_mad_u32_u24 v0, s13, v1, v0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s14
+; GFX9-NEXT:    v_mad_u32_u24 v0, s15, v1, v0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s16
+; GFX9-NEXT:    v_mad_u32_u24 v2, s17, v1, v0
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-NEXT:    s_endpgm
 ;
@@ -3064,14 +3065,14 @@ define amdgpu_kernel void @udot8_variant1(i32 addrspace(1)* %v1addr,
 ; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
-; GFX9-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
+; GFX9-DL-NEXT:    s_load_dword s3, s[0:1], 0x0
+; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
+; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s2
+; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-DL-NEXT:    v_dot8_u32_u4 v2, s4, v0, v1
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
-; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s2
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s4
-; GFX9-DL-NEXT:    v_dot8_u32_u4 v2, s3, v2, v3
 ; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-DL-NEXT:    s_endpgm
 ;
@@ -3081,14 +3082,14 @@ define amdgpu_kernel void @udot8_variant1(i32 addrspace(1)* %v1addr,
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX10-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
-; GFX10-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
+; GFX10-DL-NEXT:    s_load_dword s2, s[0:1], 0x0
+; GFX10-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
+; GFX10-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
+; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-DL-NEXT:    v_dot8_u32_u4 v2, s4, s3, v0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
-; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    v_mov_b32_e32 v2, s4
-; GFX10-DL-NEXT:    v_dot8_u32_u4 v2, s3, s2, v2
 ; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX10-DL-NEXT:    s_endpgm
                                           i32 addrspace(1)* %v2addr,
diff --git a/llvm/test/CodeGen/AMDGPU/illegal-sgpr-to-vgpr-copy.ll b/llvm/test/CodeGen/AMDGPU/illegal-sgpr-to-vgpr-copy.ll
index abcf3342f..e7ad0bd01 100644
--- a/llvm/test/CodeGen/AMDGPU/illegal-sgpr-to-vgpr-copy.ll
+++ b/llvm/test/CodeGen/AMDGPU/illegal-sgpr-to-vgpr-copy.ll
@@ -1,5 +1,5 @@
 ; RUN: not llc -march=amdgcn < %s 2>&1 | FileCheck -check-prefix=ERR %s
-; RUN: not llc -march=amdgcn < %s | FileCheck -check-prefix=GCN %s
+; RUN: not llc -march=amdgcn < %s 2>&1 | FileCheck -check-prefix=GCN %s
 
 ; ERR: error: <unknown>:0:0: in function illegal_vgpr_to_sgpr_copy_i32 void (): illegal SGPR to VGPR copy
 ; GCN: ; illegal copy v1 to s9
@@ -43,7 +43,8 @@ define amdgpu_kernel void @illegal_vgpr_to_sgpr_copy_v16i32() #0 {
 }
 
 ; ERR: error: <unknown>:0:0: in function illegal_agpr_to_sgpr_copy_i32 void (): illegal SGPR to VGPR copy
-; GCN: ; illegal copy a1 to s9
+; GCN: v_accvgpr_read_b32 [[COPY1:v[0-9]+]], a1
+; GCN: ; illegal copy [[COPY1]] to s9
 define amdgpu_kernel void @illegal_agpr_to_sgpr_copy_i32() #1 {
   %agpr = call i32 asm sideeffect "; def $0", "=${a1}"()
   call void asm sideeffect "; use $0", "${s9}"(i32 %agpr)
@@ -51,7 +52,9 @@ define amdgpu_kernel void @illegal_agpr_to_sgpr_copy_i32() #1 {
 }
 
 ; ERR: error: <unknown>:0:0: in function illegal_agpr_to_sgpr_copy_v2i32 void (): illegal SGPR to VGPR copy
-; GCN: ; illegal copy a[0:1] to s[10:11]
+; GCN-DAG: v_accvgpr_read_b32 v[[COPY1L:[0-9]+]], a0
+; GCN-DAG: v_accvgpr_read_b32 v[[COPY1H:[0-9]+]], a1
+; GCN: ; illegal copy v{{\[}}[[COPY1L]]:[[COPY1H]]] to s[10:11]
 define amdgpu_kernel void @illegal_agpr_to_sgpr_copy_v2i32() #1 {
   %vgpr = call <2 x i32> asm sideeffect "; def $0", "=${a[0:1]}"()
   call void asm sideeffect "; use $0", "${s[10:11]}"(<2 x i32> %vgpr)
diff --git a/llvm/test/CodeGen/AMDGPU/implicit-def-muse.ll b/llvm/test/CodeGen/AMDGPU/implicit-def-muse.ll
index fb540c3a6..427df9139 100644
--- a/llvm/test/CodeGen/AMDGPU/implicit-def-muse.ll
+++ b/llvm/test/CodeGen/AMDGPU/implicit-def-muse.ll
@@ -1,10 +1,11 @@
 ; RUN: llc -march=amdgcn -stop-after=amdgpu-isel -verify-machineinstrs -o - %s | FileCheck %s
 
 ; CHECK-LABEL: vcopy_i1_undef
-; CHECK: IMPLICIT_DEF
-; CHECK-NOT: COPY
-; CHECK: IMPLICIT_DEF
+; CHECK: [[IMPDEF0:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
 ; CHECK-NOT: COPY
+; CHECK: [[IMPDEF1:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+; CHECK-NOT: COPY [[IMPDEF0]]
+; CHECK-NOT: COPY [[IMPDEF1]]
 ; CHECK: .false:
 define <2 x float> @vcopy_i1_undef(<2 x float> addrspace(1)* %p) {
 entry:
diff --git a/llvm/test/CodeGen/AMDGPU/indirect-addressing-si-noopt.ll b/llvm/test/CodeGen/AMDGPU/indirect-addressing-si-noopt.ll
index 693d06ce6..70964348e 100644
--- a/llvm/test/CodeGen/AMDGPU/indirect-addressing-si-noopt.ll
+++ b/llvm/test/CodeGen/AMDGPU/indirect-addressing-si-noopt.ll
@@ -29,15 +29,11 @@ entry:
 ; CHECK: s_cbranch_scc1 [[BB4:BB[0-9]+_[0-9]+]]
 
 ; CHECK: buffer_load_dwordx4
-; CHECK: s_mov_b32 m0,
-; CHECK: v_movrels_b32_e32
 
 ; CHECK: s_branch [[ENDBB:BB[0-9]+_[0-9]+]]
 
 ; CHECK: [[BB4]]:
 ; CHECK: buffer_load_dwordx4
-; CHECK: s_mov_b32 m0,
-; CHECK: v_movrels_b32_e32
 
 ; CHECK: [[ENDBB]]:
 ; CHECK: buffer_store_dword
diff --git a/llvm/test/CodeGen/AMDGPU/indirect-addressing-term.ll b/llvm/test/CodeGen/AMDGPU/indirect-addressing-term.ll
index 053a68440..563e38cfe 100644
--- a/llvm/test/CodeGen/AMDGPU/indirect-addressing-term.ll
+++ b/llvm/test/CodeGen/AMDGPU/indirect-addressing-term.ll
@@ -73,14 +73,14 @@ define amdgpu_kernel void @extract_w_offset_vgpr(i32 addrspace(1)* %out) {
   ; GCN:   renamable $vgpr1 = IMPLICIT_DEF
   ; GCN:   renamable $sgpr24_sgpr25 = IMPLICIT_DEF
   ; GCN:   SI_SPILL_V32_SAVE killed $vgpr0, %stack.0, $sgpr96_sgpr97_sgpr98_sgpr99, $sgpr3, 0, implicit $exec :: (store 4 into %stack.0, addrspace 5)
-  ; GCN:   SI_SPILL_S128_SAVE killed $sgpr8_sgpr9_sgpr10_sgpr11, %stack.1, implicit $exec, implicit $sgpr96_sgpr97_sgpr98_sgpr99, implicit $sgpr3, implicit-def dead $m0 :: (store 16 into %stack.1, align 4, addrspace 5)
+  ; GCN:   SI_SPILL_S128_SAVE killed $sgpr8_sgpr9_sgpr10_sgpr11, %stack.1, implicit $exec, implicit $sgpr96_sgpr97_sgpr98_sgpr99, implicit $sgpr3 :: (store 16 into %stack.1, align 4, addrspace 5)
   ; GCN:   SI_SPILL_V512_SAVE killed $vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32, %stack.2, $sgpr96_sgpr97_sgpr98_sgpr99, $sgpr3, 0, implicit $exec :: (store 64 into %stack.2, align 4, addrspace 5)
-  ; GCN:   SI_SPILL_S64_SAVE killed $sgpr22_sgpr23, %stack.3, implicit $exec, implicit $sgpr96_sgpr97_sgpr98_sgpr99, implicit $sgpr3, implicit-def dead $m0 :: (store 8 into %stack.3, align 4, addrspace 5)
+  ; GCN:   SI_SPILL_S64_SAVE killed $sgpr22_sgpr23, %stack.3, implicit $exec, implicit $sgpr96_sgpr97_sgpr98_sgpr99, implicit $sgpr3 :: (store 8 into %stack.3, align 4, addrspace 5)
   ; GCN:   SI_SPILL_V32_SAVE killed $vgpr1, %stack.4, $sgpr96_sgpr97_sgpr98_sgpr99, $sgpr3, 0, implicit $exec :: (store 4 into %stack.4, addrspace 5)
-  ; GCN:   SI_SPILL_S64_SAVE killed $sgpr24_sgpr25, %stack.5, implicit $exec, implicit $sgpr96_sgpr97_sgpr98_sgpr99, implicit $sgpr3, implicit-def dead $m0 :: (store 8 into %stack.5, align 4, addrspace 5)
+  ; GCN:   SI_SPILL_S64_SAVE killed $sgpr24_sgpr25, %stack.5, implicit $exec, implicit $sgpr96_sgpr97_sgpr98_sgpr99, implicit $sgpr3 :: (store 8 into %stack.5, align 4, addrspace 5)
   ; GCN: bb.1:
   ; GCN:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
-  ; GCN:   $sgpr0_sgpr1 = SI_SPILL_S64_RESTORE %stack.5, implicit $exec, implicit $sgpr96_sgpr97_sgpr98_sgpr99, implicit $sgpr3, implicit-def dead $m0 :: (load 8 from %stack.5, align 4, addrspace 5)
+  ; GCN:   $sgpr0_sgpr1 = SI_SPILL_S64_RESTORE %stack.5, implicit $exec, implicit $sgpr96_sgpr97_sgpr98_sgpr99, implicit $sgpr3 :: (load 8 from %stack.5, align 4, addrspace 5)
   ; GCN:   $vgpr0 = SI_SPILL_V32_RESTORE %stack.4, $sgpr96_sgpr97_sgpr98_sgpr99, $sgpr3, 0, implicit $exec :: (load 4 from %stack.4, addrspace 5)
   ; GCN:   $vgpr1 = SI_SPILL_V32_RESTORE %stack.0, $sgpr96_sgpr97_sgpr98_sgpr99, $sgpr3, 0, implicit $exec :: (load 4 from %stack.0, addrspace 5)
   ; GCN:   renamable $sgpr2 = V_READFIRSTLANE_B32 $vgpr1, implicit $exec
@@ -92,19 +92,19 @@ define amdgpu_kernel void @extract_w_offset_vgpr(i32 addrspace(1)* %out) {
   ; GCN:   S_SET_GPR_IDX_OFF
   ; GCN:   renamable $vgpr19 = COPY renamable $vgpr18
   ; GCN:   renamable $sgpr6_sgpr7 = COPY renamable $sgpr4_sgpr5
-  ; GCN:   SI_SPILL_S64_SAVE killed $sgpr6_sgpr7, %stack.5, implicit $exec, implicit $sgpr96_sgpr97_sgpr98_sgpr99, implicit $sgpr3, implicit-def dead $m0 :: (store 8 into %stack.5, align 4, addrspace 5)
-  ; GCN:   SI_SPILL_S64_SAVE killed $sgpr0_sgpr1, %stack.6, implicit $exec, implicit $sgpr96_sgpr97_sgpr98_sgpr99, implicit $sgpr3, implicit-def dead $m0 :: (store 8 into %stack.6, align 4, addrspace 5)
+  ; GCN:   SI_SPILL_S64_SAVE killed $sgpr6_sgpr7, %stack.5, implicit $exec, implicit $sgpr96_sgpr97_sgpr98_sgpr99, implicit $sgpr3 :: (store 8 into %stack.5, align 4, addrspace 5)
+  ; GCN:   SI_SPILL_S64_SAVE killed $sgpr0_sgpr1, %stack.6, implicit $exec, implicit $sgpr96_sgpr97_sgpr98_sgpr99, implicit $sgpr3 :: (store 8 into %stack.6, align 4, addrspace 5)
   ; GCN:   SI_SPILL_V32_SAVE killed $vgpr19, %stack.4, $sgpr96_sgpr97_sgpr98_sgpr99, $sgpr3, 0, implicit $exec :: (store 4 into %stack.4, addrspace 5)
   ; GCN:   SI_SPILL_V32_SAVE killed $vgpr0, %stack.7, $sgpr96_sgpr97_sgpr98_sgpr99, $sgpr3, 0, implicit $exec :: (store 4 into %stack.7, addrspace 5)
   ; GCN:   SI_SPILL_V32_SAVE killed $vgpr18, %stack.8, $sgpr96_sgpr97_sgpr98_sgpr99, $sgpr3, 0, implicit $exec :: (store 4 into %stack.8, addrspace 5)
   ; GCN:   $exec = S_XOR_B64_term $exec, killed renamable $sgpr4_sgpr5, implicit-def $scc
   ; GCN:   S_CBRANCH_EXECNZ %bb.1, implicit $exec
   ; GCN: bb.2:
-  ; GCN:   $sgpr0_sgpr1 = SI_SPILL_S64_RESTORE %stack.3, implicit $exec, implicit $sgpr96_sgpr97_sgpr98_sgpr99, implicit $sgpr3, implicit-def dead $m0 :: (load 8 from %stack.3, align 4, addrspace 5)
+  ; GCN:   $sgpr0_sgpr1 = SI_SPILL_S64_RESTORE %stack.3, implicit $exec, implicit $sgpr96_sgpr97_sgpr98_sgpr99, implicit $sgpr3 :: (load 8 from %stack.3, align 4, addrspace 5)
   ; GCN:   $exec = S_MOV_B64 renamable $sgpr0_sgpr1
   ; GCN:   $vgpr0 = SI_SPILL_V32_RESTORE %stack.8, $sgpr96_sgpr97_sgpr98_sgpr99, $sgpr3, 0, implicit $exec :: (load 4 from %stack.8, addrspace 5)
-  ; GCN:   $sgpr4_sgpr5_sgpr6_sgpr7 = SI_SPILL_S128_RESTORE %stack.1, implicit $exec, implicit $sgpr96_sgpr97_sgpr98_sgpr99, implicit $sgpr3, implicit-def dead $m0 :: (load 16 from %stack.1, align 4, addrspace 5)
-  ; GCN:   BUFFER_STORE_DWORD_OFFSET renamable $vgpr0, renamable $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 4 into %ir.out.load, addrspace 1)
+  ; GCN:   $sgpr4_sgpr5_sgpr6_sgpr7 = SI_SPILL_S128_RESTORE %stack.1, implicit $exec, implicit $sgpr96_sgpr97_sgpr98_sgpr99, implicit $sgpr3 :: (load 16 from %stack.1, align 4, addrspace 5)
+  ; GCN:   BUFFER_STORE_DWORD_OFFSET renamable $vgpr0, renamable $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 4 into %ir.out.load, addrspace 1)
   ; GCN:   S_ENDPGM 0
 entry:
   %id = call i32 @llvm.amdgcn.workitem.id.x() #1
diff --git a/llvm/test/CodeGen/AMDGPU/inline-asm.ll b/llvm/test/CodeGen/AMDGPU/inline-asm.ll
index a964dedb2..cb06eb043 100644
--- a/llvm/test/CodeGen/AMDGPU/inline-asm.ll
+++ b/llvm/test/CodeGen/AMDGPU/inline-asm.ll
@@ -198,8 +198,7 @@ entry:
 }
 
 ; CHECK-LABEL: {{^}}i1_imm_input_phys_vgpr:
-; CHECK: s_mov_b64 [[MASK:s\[[0-9]+:[0-9]+\]]], -1
-; CHECK-NEXT: v_cndmask_b32_e64 v0, 0, -1, [[MASK]]
+; CHECK: v_mov_b32_e32 v0, 1{{$}}
 ; CHECK: ; use v0
 define amdgpu_kernel void @i1_imm_input_phys_vgpr() {
 entry:
@@ -207,14 +206,14 @@ entry:
   ret void
 }
 
+
+; FIXME: This behavior is nonsense. We should probably disallow i1 asm
+
 ; CHECK-LABEL: {{^}}i1_input_phys_vgpr:
 ; CHECK: {{buffer|flat}}_load_ubyte [[LOAD:v[0-9]+]]
-; CHECK: v_and_b32_e32 [[LOAD]], 1, [[LOAD]]
-; CHECK-NEXT: v_cmp_eq_u32_e32 vcc, 1, [[LOAD]]
-; CHECK-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc
+; CHECK-NOT: [[LOAD]]
 ; CHECK: ; use v0
-; CHECK: v_cmp_ne_u32_e32 vcc, 0, v1
-; CHECK: v_cndmask_b32_e64 [[STORE:v[0-9]+]], 0, 1, vcc
+; CHECK: v_and_b32_e32 [[STORE:v[0-9]+]], 1, v1
 ; CHECK: {{buffer|flat}}_store_byte [[STORE]],
 define amdgpu_kernel void @i1_input_phys_vgpr() {
 entry:
@@ -224,12 +223,12 @@ entry:
   ret void
 }
 
-; FIXME: Should be scheduled to shrink vcc
+; FIXME: Should prodbably be masking high bits of load.
 ; CHECK-LABEL: {{^}}i1_input_phys_vgpr_x2:
-; CHECK: v_cmp_eq_u32_e32 vcc, 1, v0
-; CHECK: v_cndmask_b32_e64 v0, 0, -1, vcc
-; CHECK: v_cmp_eq_u32_e32 vcc, 1, v1
-; CHECK: v_cndmask_b32_e64 v1, 0, -1, vcc
+; CHECK: buffer_load_ubyte v0
+; CHECK-NEXT: buffer_load_ubyte v1
+; CHECK-NEXT: s_waitcnt
+; CHECK-NEXT: ASMSTART
 define amdgpu_kernel void @i1_input_phys_vgpr_x2() {
 entry:
   %val0 = load volatile i1, i1 addrspace(1)* undef
diff --git a/llvm/test/CodeGen/AMDGPU/inline-constraints.ll b/llvm/test/CodeGen/AMDGPU/inline-constraints.ll
index 8a4af42fa..6f1d35519 100644
--- a/llvm/test/CodeGen/AMDGPU/inline-constraints.ll
+++ b/llvm/test/CodeGen/AMDGPU/inline-constraints.ll
@@ -30,11 +30,10 @@ entry:
   ret void
 }
 
-; FIXME: Should be able to avoid copy
 ; GCN-LABEL: {{^}}inline_sreg_constraint_m0:
 ; GCN: s_mov_b32 m0, -1
-; GCN: s_mov_b32 [[COPY_M0:s[0-9]+]], m0
-; GCN: ; use [[COPY_M0]]
+; GCN-NOT: m0
+; GCN: ; use m0
 define amdgpu_kernel void @inline_sreg_constraint_m0() {
   %m0 = tail call i32 asm sideeffect "s_mov_b32 m0, -1", "={m0}"()
   tail call void asm sideeffect "; use $0", "s"(i32 %m0)
diff --git a/llvm/test/CodeGen/AMDGPU/insert-subvector-unused-scratch.ll b/llvm/test/CodeGen/AMDGPU/insert-subvector-unused-scratch.ll
index d29bcef05..9a23df97d 100644
--- a/llvm/test/CodeGen/AMDGPU/insert-subvector-unused-scratch.ll
+++ b/llvm/test/CodeGen/AMDGPU/insert-subvector-unused-scratch.ll
@@ -4,8 +4,8 @@
 ; an unused stack slot, causing ScratchSize to be non-zero.
 
 ; GCN-LABEL: store_v3i32:
-; GCN:        ds_read_b64
 ; GCN:        ds_read_b32
+; GCN:        ds_read_b64
 ; GCN:        ds_write_b32
 ; GCN:        ds_write_b64
 ; GCN: ScratchSize: 0
@@ -17,8 +17,8 @@ define amdgpu_kernel void @store_v3i32(<3 x i32> addrspace(3)* %out, <3 x i32> %
 }
 
 ; GCN-LABEL: store_v5i32:
-; GCN:        ds_read2_b64
 ; GCN:        ds_read_b32
+; GCN:        ds_read2_b64
 ; GCN:        ds_write_b32
 ; GCN:        ds_write2_b64
 ; GCN: ScratchSize: 0
diff --git a/llvm/test/CodeGen/AMDGPU/insert_vector_elt.ll b/llvm/test/CodeGen/AMDGPU/insert_vector_elt.ll
index 47e080a94..04b8be9ab 100644
--- a/llvm/test/CodeGen/AMDGPU/insert_vector_elt.ll
+++ b/llvm/test/CodeGen/AMDGPU/insert_vector_elt.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -verify-machineinstrs -mtriple=amdgcn-amd-amdhsa -mcpu=kaveri -mattr=-flat-for-global,+max-private-element-size-16 < %s | FileCheck -enable-var-scope -check-prefixes=GCN,SI,GCN-NO-TONGA %s
 ; RUN: llc -verify-machineinstrs -mtriple=amdgcn-amd-amdhsa -mcpu=tonga -mattr=-flat-for-global -mattr=+max-private-element-size-16 < %s | FileCheck -enable-var-scope -check-prefixes=GCN,VI,GCN-TONGA %s
 
@@ -8,246 +9,973 @@
 
 ; FIXME: Why is the constant moved into the intermediate register and
 ; not just directly into the vector component?
-
-; GCN-LABEL: {{^}}insertelement_v4f32_0:
-; GCN: s_load_dwordx4
-; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, s{{[0-9]+}}
-; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, s{{[0-9]+}}
-; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, s{{[0-9]+}}
-; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, s{{[0-9]+}}
-; GCN-DAG: s_mov_b32 [[CONSTREG:s[0-9]+]], 0x40a00000
-; GCN-DAG: v_mov_b32_e32 v[[LOW_REG:[0-9]+]], [[CONSTREG]]
-; GCN: buffer_store_dwordx4 v{{\[}}[[LOW_REG]]:
 define amdgpu_kernel void @insertelement_v4f32_0(<4 x float> addrspace(1)* %out, <4 x float> %a) nounwind {
+; SI-LABEL: insertelement_v4f32_0:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; SI-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x4
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b32 s4, 0x40a00000
+; SI-NEXT:    v_mov_b32_e32 v0, s4
+; SI-NEXT:    s_mov_b32 s3, 0x100f000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    v_mov_b32_e32 v1, s5
+; SI-NEXT:    v_mov_b32_e32 v2, s6
+; SI-NEXT:    v_mov_b32_e32 v3, s7
+; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: insertelement_v4f32_0:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; VI-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x10
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_mov_b32 s4, 0x40a00000
+; VI-NEXT:    v_mov_b32_e32 v0, s4
+; VI-NEXT:    s_mov_b32 s3, 0x1100f000
+; VI-NEXT:    s_mov_b32 s2, -1
+; VI-NEXT:    v_mov_b32_e32 v1, s5
+; VI-NEXT:    v_mov_b32_e32 v2, s6
+; VI-NEXT:    v_mov_b32_e32 v3, s7
+; VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; VI-NEXT:    s_endpgm
   %vecins = insertelement <4 x float> %a, float 5.000000e+00, i32 0
   store <4 x float> %vecins, <4 x float> addrspace(1)* %out, align 16
   ret void
 }
 
-; GCN-LABEL: {{^}}insertelement_v4f32_1:
 define amdgpu_kernel void @insertelement_v4f32_1(<4 x float> addrspace(1)* %out, <4 x float> %a) nounwind {
+; SI-LABEL: insertelement_v4f32_1:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; SI-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x4
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b32 s5, 0x40a00000
+; SI-NEXT:    s_mov_b32 s3, 0x100f000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    v_mov_b32_e32 v0, s4
+; SI-NEXT:    v_mov_b32_e32 v1, s5
+; SI-NEXT:    v_mov_b32_e32 v2, s6
+; SI-NEXT:    v_mov_b32_e32 v3, s7
+; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: insertelement_v4f32_1:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; VI-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x10
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_mov_b32 s5, 0x40a00000
+; VI-NEXT:    s_mov_b32 s3, 0x1100f000
+; VI-NEXT:    s_mov_b32 s2, -1
+; VI-NEXT:    v_mov_b32_e32 v0, s4
+; VI-NEXT:    v_mov_b32_e32 v1, s5
+; VI-NEXT:    v_mov_b32_e32 v2, s6
+; VI-NEXT:    v_mov_b32_e32 v3, s7
+; VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; VI-NEXT:    s_endpgm
   %vecins = insertelement <4 x float> %a, float 5.000000e+00, i32 1
   store <4 x float> %vecins, <4 x float> addrspace(1)* %out, align 16
   ret void
 }
 
-; GCN-LABEL: {{^}}insertelement_v4f32_2:
 define amdgpu_kernel void @insertelement_v4f32_2(<4 x float> addrspace(1)* %out, <4 x float> %a) nounwind {
+; SI-LABEL: insertelement_v4f32_2:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; SI-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x4
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b32 s6, 0x40a00000
+; SI-NEXT:    s_mov_b32 s3, 0x100f000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    v_mov_b32_e32 v0, s4
+; SI-NEXT:    v_mov_b32_e32 v1, s5
+; SI-NEXT:    v_mov_b32_e32 v2, s6
+; SI-NEXT:    v_mov_b32_e32 v3, s7
+; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: insertelement_v4f32_2:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; VI-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x10
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_mov_b32 s6, 0x40a00000
+; VI-NEXT:    s_mov_b32 s3, 0x1100f000
+; VI-NEXT:    s_mov_b32 s2, -1
+; VI-NEXT:    v_mov_b32_e32 v0, s4
+; VI-NEXT:    v_mov_b32_e32 v1, s5
+; VI-NEXT:    v_mov_b32_e32 v2, s6
+; VI-NEXT:    v_mov_b32_e32 v3, s7
+; VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; VI-NEXT:    s_endpgm
   %vecins = insertelement <4 x float> %a, float 5.000000e+00, i32 2
   store <4 x float> %vecins, <4 x float> addrspace(1)* %out, align 16
   ret void
 }
 
-; GCN-LABEL: {{^}}insertelement_v4f32_3:
 define amdgpu_kernel void @insertelement_v4f32_3(<4 x float> addrspace(1)* %out, <4 x float> %a) nounwind {
+; SI-LABEL: insertelement_v4f32_3:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; SI-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x4
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b32 s7, 0x40a00000
+; SI-NEXT:    s_mov_b32 s3, 0x100f000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    v_mov_b32_e32 v0, s4
+; SI-NEXT:    v_mov_b32_e32 v1, s5
+; SI-NEXT:    v_mov_b32_e32 v2, s6
+; SI-NEXT:    v_mov_b32_e32 v3, s7
+; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: insertelement_v4f32_3:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; VI-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x10
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_mov_b32 s7, 0x40a00000
+; VI-NEXT:    s_mov_b32 s3, 0x1100f000
+; VI-NEXT:    s_mov_b32 s2, -1
+; VI-NEXT:    v_mov_b32_e32 v0, s4
+; VI-NEXT:    v_mov_b32_e32 v1, s5
+; VI-NEXT:    v_mov_b32_e32 v2, s6
+; VI-NEXT:    v_mov_b32_e32 v3, s7
+; VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; VI-NEXT:    s_endpgm
   %vecins = insertelement <4 x float> %a, float 5.000000e+00, i32 3
   store <4 x float> %vecins, <4 x float> addrspace(1)* %out, align 16
   ret void
 }
 
-; GCN-LABEL: {{^}}insertelement_v4i32_0:
 define amdgpu_kernel void @insertelement_v4i32_0(<4 x i32> addrspace(1)* %out, <4 x i32> %a) nounwind {
+; SI-LABEL: insertelement_v4i32_0:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; SI-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x4
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_movk_i32 s4, 0x3e7
+; SI-NEXT:    v_mov_b32_e32 v0, s4
+; SI-NEXT:    s_mov_b32 s3, 0x100f000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    v_mov_b32_e32 v1, s5
+; SI-NEXT:    v_mov_b32_e32 v2, s6
+; SI-NEXT:    v_mov_b32_e32 v3, s7
+; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: insertelement_v4i32_0:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; VI-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x10
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_movk_i32 s4, 0x3e7
+; VI-NEXT:    v_mov_b32_e32 v0, s4
+; VI-NEXT:    s_mov_b32 s3, 0x1100f000
+; VI-NEXT:    s_mov_b32 s2, -1
+; VI-NEXT:    v_mov_b32_e32 v1, s5
+; VI-NEXT:    v_mov_b32_e32 v2, s6
+; VI-NEXT:    v_mov_b32_e32 v3, s7
+; VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; VI-NEXT:    s_endpgm
   %vecins = insertelement <4 x i32> %a, i32 999, i32 0
   store <4 x i32> %vecins, <4 x i32> addrspace(1)* %out, align 16
   ret void
 }
 
-; GCN-LABEL: {{^}}insertelement_v3f32_1:
 define amdgpu_kernel void @insertelement_v3f32_1(<3 x float> addrspace(1)* %out, <3 x float> %a) nounwind {
+; SI-LABEL: insertelement_v3f32_1:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; SI-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x4
+; SI-NEXT:    s_mov_b32 s3, 0x100f000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    v_mov_b32_e32 v1, 0x40a00000
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v0, s4
+; SI-NEXT:    v_mov_b32_e32 v2, s6
+; SI-NEXT:    buffer_store_dwordx3 v[0:2], off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: insertelement_v3f32_1:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; VI-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x10
+; VI-NEXT:    s_mov_b32 s3, 0x1100f000
+; VI-NEXT:    s_mov_b32 s2, -1
+; VI-NEXT:    v_mov_b32_e32 v1, 0x40a00000
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v0, s4
+; VI-NEXT:    v_mov_b32_e32 v2, s6
+; VI-NEXT:    buffer_store_dwordx3 v[0:2], off, s[0:3], 0
+; VI-NEXT:    s_endpgm
   %vecins = insertelement <3 x float> %a, float 5.000000e+00, i32 1
   store <3 x float> %vecins, <3 x float> addrspace(1)* %out, align 16
   ret void
 }
 
-; GCN-LABEL: {{^}}insertelement_v3f32_2:
 define amdgpu_kernel void @insertelement_v3f32_2(<3 x float> addrspace(1)* %out, <3 x float> %a) nounwind {
+; SI-LABEL: insertelement_v3f32_2:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; SI-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x4
+; SI-NEXT:    s_mov_b32 s3, 0x100f000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    v_mov_b32_e32 v2, 0x40a00000
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v0, s4
+; SI-NEXT:    v_mov_b32_e32 v1, s5
+; SI-NEXT:    buffer_store_dwordx3 v[0:2], off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: insertelement_v3f32_2:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; VI-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x10
+; VI-NEXT:    s_mov_b32 s3, 0x1100f000
+; VI-NEXT:    s_mov_b32 s2, -1
+; VI-NEXT:    v_mov_b32_e32 v2, 0x40a00000
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v0, s4
+; VI-NEXT:    v_mov_b32_e32 v1, s5
+; VI-NEXT:    buffer_store_dwordx3 v[0:2], off, s[0:3], 0
+; VI-NEXT:    s_endpgm
   %vecins = insertelement <3 x float> %a, float 5.000000e+00, i32 2
   store <3 x float> %vecins, <3 x float> addrspace(1)* %out, align 16
   ret void
 }
 
-; GCN-LABEL: {{^}}insertelement_v3f32_3:
 define amdgpu_kernel void @insertelement_v3f32_3(<3 x float> addrspace(1)* %out, <3 x float> %a) nounwind {
+; GCN-LABEL: insertelement_v3f32_3:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_endpgm
   %vecins = insertelement <3 x float> %a, float 5.000000e+00, i32 3
   store <3 x float> %vecins, <3 x float> addrspace(1)* %out, align 16
   ret void
 }
 
-; GCN-LABEL: {{^}}insertelement_to_sgpr:
-; GCN-NOT: v_readfirstlane
 define <4 x float> @insertelement_to_sgpr() nounwind {
+; GCN-LABEL: insertelement_to_sgpr:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    s_load_dwordx4 s[12:15], s[4:5], 0x0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_mov_b32 s12, 0
+; GCN-NEXT:    s_mov_b32 s4, s12
+; GCN-NEXT:    s_mov_b32 s5, s12
+; GCN-NEXT:    s_mov_b32 s6, s12
+; GCN-NEXT:    s_mov_b32 s7, s12
+; GCN-NEXT:    s_mov_b32 s8, s12
+; GCN-NEXT:    s_mov_b32 s9, s12
+; GCN-NEXT:    s_mov_b32 s10, s12
+; GCN-NEXT:    s_mov_b32 s11, s12
+; GCN-NEXT:    image_gather4_lz v[0:3], v[0:1], s[4:11], s[12:15] dmask:0x1
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
   %tmp = load <4 x i32>, <4 x i32> addrspace(4)* undef
   %tmp1 = insertelement <4 x i32> %tmp, i32 0, i32 0
   %tmp2 = call <4 x float> @llvm.amdgcn.image.gather4.lz.2d.v4f32.f32(i32 1, float undef, float undef, <8 x i32> undef, <4 x i32> %tmp1, i1 0, i32 0, i32 0)
   ret <4 x float> %tmp2
 }
 
-; GCN-LABEL: {{^}}dynamic_insertelement_v2f32:
-; GCN-DAG: v_mov_b32_e32 [[CONST:v[0-9]+]], 0x40a00000
-; GCN-DAG: v_cmp_ne_u32_e64 [[CC2:[^,]+]], [[IDX:s[0-9]+]], 1
-; GCN-DAG: v_cndmask_b32_e32 v{{[0-9]+}}, [[CONST]], v{{[0-9]+}}, [[CC2]]
-; GCN-DAG: v_cmp_ne_u32_e64 [[CC1:[^,]+]], [[IDX]], 0
-; GCN-DAG: v_cndmask_b32_e32 v[[LOW_RESULT_REG:[0-9]+]], [[CONST]], v{{[0-9]+}}, [[CC1]]
-; GCN: buffer_store_dwordx2 {{v\[}}[[LOW_RESULT_REG]]:
 define amdgpu_kernel void @dynamic_insertelement_v2f32(<2 x float> addrspace(1)* %out, <2 x float> %a, i32 %b) nounwind {
+; SI-LABEL: dynamic_insertelement_v2f32:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; SI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
+; SI-NEXT:    s_load_dword s4, s[4:5], 0x4
+; SI-NEXT:    v_mov_b32_e32 v0, 0x40a00000
+; SI-NEXT:    s_mov_b32 s3, 0x100f000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v1, s7
+; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 1
+; SI-NEXT:    v_cndmask_b32_e32 v1, v0, v1, vcc
+; SI-NEXT:    v_mov_b32_e32 v2, s6
+; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 0
+; SI-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: dynamic_insertelement_v2f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; VI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x8
+; VI-NEXT:    s_load_dword s4, s[4:5], 0x10
+; VI-NEXT:    v_mov_b32_e32 v0, 0x40a00000
+; VI-NEXT:    s_mov_b32 s3, 0x1100f000
+; VI-NEXT:    s_mov_b32 s2, -1
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s7
+; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 1
+; VI-NEXT:    v_cndmask_b32_e32 v1, v0, v1, vcc
+; VI-NEXT:    v_mov_b32_e32 v2, s6
+; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 0
+; VI-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; VI-NEXT:    s_endpgm
   %vecins = insertelement <2 x float> %a, float 5.000000e+00, i32 %b
   store <2 x float> %vecins, <2 x float> addrspace(1)* %out, align 8
   ret void
 }
 
-; GCN-LABEL: {{^}}dynamic_insertelement_v3f32:
-; GCN-DAG: v_mov_b32_e32 [[CONST:v[0-9]+]], 0x40a00000
-; GCN-DAG: v_cmp_ne_u32_e64 [[CC3:[^,]+]], [[IDX:s[0-9]+]], 2
-; GCN-DAG: v_cndmask_b32_e32 v{{[0-9]+}}, [[CONST]], v{{[0-9]+}}, [[CC3]]
-; GCN-DAG: v_cmp_ne_u32_e64 [[CC2:[^,]+]], [[IDX]], 1
-; GCN-DAG: v_cndmask_b32_e32 v{{[0-9]+}}, [[CONST]], v{{[0-9]+}}, [[CC2]]
-; GCN-DAG: v_cmp_ne_u32_e64 [[CC1:[^,]+]], [[IDX]], 0
-; GCN-DAG: v_cndmask_b32_e32 v{{[0-9]+}}, [[CONST]], v{{[0-9]+}}, [[CC1]]
-; GCN-DAG: buffer_store_dwordx3 v
 define amdgpu_kernel void @dynamic_insertelement_v3f32(<3 x float> addrspace(1)* %out, <3 x float> %a, i32 %b) nounwind {
+; SI-LABEL: dynamic_insertelement_v3f32:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; SI-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x4
+; SI-NEXT:    s_load_dword s4, s[4:5], 0x8
+; SI-NEXT:    v_mov_b32_e32 v0, 0x40a00000
+; SI-NEXT:    s_mov_b32 s3, 0x100f000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v1, s10
+; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 2
+; SI-NEXT:    v_cndmask_b32_e32 v2, v0, v1, vcc
+; SI-NEXT:    v_mov_b32_e32 v1, s9
+; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 1
+; SI-NEXT:    v_cndmask_b32_e32 v1, v0, v1, vcc
+; SI-NEXT:    v_mov_b32_e32 v3, s8
+; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 0
+; SI-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
+; SI-NEXT:    buffer_store_dwordx3 v[0:2], off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: dynamic_insertelement_v3f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; VI-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x10
+; VI-NEXT:    s_load_dword s4, s[4:5], 0x20
+; VI-NEXT:    v_mov_b32_e32 v0, 0x40a00000
+; VI-NEXT:    s_mov_b32 s3, 0x1100f000
+; VI-NEXT:    s_mov_b32 s2, -1
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s10
+; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 2
+; VI-NEXT:    v_cndmask_b32_e32 v2, v0, v1, vcc
+; VI-NEXT:    v_mov_b32_e32 v1, s9
+; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 1
+; VI-NEXT:    v_cndmask_b32_e32 v1, v0, v1, vcc
+; VI-NEXT:    v_mov_b32_e32 v3, s8
+; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 0
+; VI-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
+; VI-NEXT:    buffer_store_dwordx3 v[0:2], off, s[0:3], 0
+; VI-NEXT:    s_endpgm
   %vecins = insertelement <3 x float> %a, float 5.000000e+00, i32 %b
   store <3 x float> %vecins, <3 x float> addrspace(1)* %out, align 16
   ret void
 }
 
-; GCN-LABEL: {{^}}dynamic_insertelement_v4f32:
-; GCN-DAG: v_mov_b32_e32 [[CONST:v[0-9]+]], 0x40a00000
-; GCN-DAG: v_cmp_ne_u32_e64 [[CC4:[^,]+]], [[IDX:s[0-9]+]], 3
-; GCN-DAG: v_cndmask_b32_e32 v{{[0-9]+}}, [[CONST]], v{{[0-9]+}}, [[CC4]]
-; GCN-DAG: v_cmp_ne_u32_e64 [[CC3:[^,]+]], [[IDX]], 2
-; GCN-DAG: v_cndmask_b32_e32 v{{[0-9]+}}, [[CONST]], v{{[0-9]+}}, [[CC3]]
-; GCN-DAG: v_cmp_ne_u32_e64 [[CC2:[^,]+]], [[IDX]], 1
-; GCN-DAG: v_cndmask_b32_e32 v{{[0-9]+}}, [[CONST]], v{{[0-9]+}}, [[CC2]]
-; GCN-DAG: v_cmp_ne_u32_e64 [[CC1:[^,]+]], [[IDX]], 0
-; GCN-DAG: v_cndmask_b32_e32 v[[LOW_RESULT_REG:[0-9]+]], [[CONST]], v{{[0-9]+}}, [[CC1]]
-; GCN: buffer_store_dwordx4 {{v\[}}[[LOW_RESULT_REG]]:
 define amdgpu_kernel void @dynamic_insertelement_v4f32(<4 x float> addrspace(1)* %out, <4 x float> %a, i32 %b) nounwind {
+; SI-LABEL: dynamic_insertelement_v4f32:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; SI-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x4
+; SI-NEXT:    s_load_dword s4, s[4:5], 0x8
+; SI-NEXT:    v_mov_b32_e32 v0, 0x40a00000
+; SI-NEXT:    s_mov_b32 s3, 0x100f000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v1, s11
+; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 3
+; SI-NEXT:    v_cndmask_b32_e32 v3, v0, v1, vcc
+; SI-NEXT:    v_mov_b32_e32 v1, s10
+; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 2
+; SI-NEXT:    v_cndmask_b32_e32 v2, v0, v1, vcc
+; SI-NEXT:    v_mov_b32_e32 v1, s9
+; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 1
+; SI-NEXT:    v_cndmask_b32_e32 v1, v0, v1, vcc
+; SI-NEXT:    v_mov_b32_e32 v4, s8
+; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 0
+; SI-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: dynamic_insertelement_v4f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; VI-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x10
+; VI-NEXT:    s_load_dword s4, s[4:5], 0x20
+; VI-NEXT:    v_mov_b32_e32 v0, 0x40a00000
+; VI-NEXT:    s_mov_b32 s3, 0x1100f000
+; VI-NEXT:    s_mov_b32 s2, -1
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s11
+; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 3
+; VI-NEXT:    v_cndmask_b32_e32 v3, v0, v1, vcc
+; VI-NEXT:    v_mov_b32_e32 v1, s10
+; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 2
+; VI-NEXT:    v_cndmask_b32_e32 v2, v0, v1, vcc
+; VI-NEXT:    v_mov_b32_e32 v1, s9
+; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 1
+; VI-NEXT:    v_cndmask_b32_e32 v1, v0, v1, vcc
+; VI-NEXT:    v_mov_b32_e32 v4, s8
+; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 0
+; VI-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; VI-NEXT:    s_endpgm
   %vecins = insertelement <4 x float> %a, float 5.000000e+00, i32 %b
   store <4 x float> %vecins, <4 x float> addrspace(1)* %out, align 16
   ret void
 }
 
-; GCN-LABEL: {{^}}dynamic_insertelement_v8f32:
-; GCN-DAG: v_mov_b32_e32 [[CONST:v[0-9]+]], 0x40a00000
-; GCN-DAG: v_cmp_ne_u32_e64 [[CCL:[^,]+]], [[IDX:s[0-9]+]], 7
-; GCN-DAG: v_cndmask_b32_e32 v{{[0-9]+}}, [[CONST]], v{{[0-9]+}}, [[CCL]]
-; GCN-DAG: v_cmp_ne_u32_e64 [[CC1:[^,]+]], [[IDX]], 0
-; GCN-DAG: v_cndmask_b32_e32 v{{[0-9]+}}, [[CONST]], v{{[0-9]+}}, [[CC1]]
-; GCN: buffer_store_dwordx4
-; GCN: buffer_store_dwordx4
 define amdgpu_kernel void @dynamic_insertelement_v8f32(<8 x float> addrspace(1)* %out, <8 x float> %a, i32 %b) nounwind {
+; SI-LABEL: dynamic_insertelement_v8f32:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; SI-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x8
+; SI-NEXT:    s_load_dword s4, s[4:5], 0x10
+; SI-NEXT:    v_mov_b32_e32 v4, 0x40a00000
+; SI-NEXT:    s_mov_b32 s3, 0x100f000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v0, s11
+; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 3
+; SI-NEXT:    v_cndmask_b32_e32 v3, v4, v0, vcc
+; SI-NEXT:    v_mov_b32_e32 v0, s10
+; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 2
+; SI-NEXT:    v_cndmask_b32_e32 v2, v4, v0, vcc
+; SI-NEXT:    v_mov_b32_e32 v0, s9
+; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 1
+; SI-NEXT:    v_cndmask_b32_e32 v1, v4, v0, vcc
+; SI-NEXT:    v_mov_b32_e32 v0, s8
+; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 0
+; SI-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; SI-NEXT:    v_mov_b32_e32 v5, s15
+; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 7
+; SI-NEXT:    v_cndmask_b32_e32 v7, v4, v5, vcc
+; SI-NEXT:    v_mov_b32_e32 v5, s14
+; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 6
+; SI-NEXT:    v_cndmask_b32_e32 v6, v4, v5, vcc
+; SI-NEXT:    v_mov_b32_e32 v5, s13
+; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 5
+; SI-NEXT:    v_cndmask_b32_e32 v5, v4, v5, vcc
+; SI-NEXT:    v_mov_b32_e32 v8, s12
+; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 4
+; SI-NEXT:    v_cndmask_b32_e32 v4, v4, v8, vcc
+; SI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:16
+; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: dynamic_insertelement_v8f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; VI-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x20
+; VI-NEXT:    s_load_dword s4, s[4:5], 0x40
+; VI-NEXT:    v_mov_b32_e32 v4, 0x40a00000
+; VI-NEXT:    s_mov_b32 s3, 0x1100f000
+; VI-NEXT:    s_mov_b32 s2, -1
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v0, s11
+; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 3
+; VI-NEXT:    v_cndmask_b32_e32 v3, v4, v0, vcc
+; VI-NEXT:    v_mov_b32_e32 v0, s10
+; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 2
+; VI-NEXT:    v_cndmask_b32_e32 v2, v4, v0, vcc
+; VI-NEXT:    v_mov_b32_e32 v0, s9
+; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 1
+; VI-NEXT:    v_cndmask_b32_e32 v1, v4, v0, vcc
+; VI-NEXT:    v_mov_b32_e32 v0, s8
+; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 0
+; VI-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; VI-NEXT:    v_mov_b32_e32 v5, s15
+; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 7
+; VI-NEXT:    v_cndmask_b32_e32 v7, v4, v5, vcc
+; VI-NEXT:    v_mov_b32_e32 v5, s14
+; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 6
+; VI-NEXT:    v_cndmask_b32_e32 v6, v4, v5, vcc
+; VI-NEXT:    v_mov_b32_e32 v5, s13
+; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 5
+; VI-NEXT:    v_cndmask_b32_e32 v5, v4, v5, vcc
+; VI-NEXT:    v_mov_b32_e32 v8, s12
+; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 4
+; VI-NEXT:    v_cndmask_b32_e32 v4, v4, v8, vcc
+; VI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:16
+; VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; VI-NEXT:    s_endpgm
   %vecins = insertelement <8 x float> %a, float 5.000000e+00, i32 %b
   store <8 x float> %vecins, <8 x float> addrspace(1)* %out, align 32
   ret void
 }
 
-; GCN-LABEL: {{^}}dynamic_insertelement_v16f32:
-; GCN: v_movreld_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}
-; GCN: buffer_store_dwordx4
-; GCN: buffer_store_dwordx4
-; GCN: buffer_store_dwordx4
-; GCN: buffer_store_dwordx4
 define amdgpu_kernel void @dynamic_insertelement_v16f32(<16 x float> addrspace(1)* %out, <16 x float> %a, i32 %b) nounwind {
+; SI-LABEL: dynamic_insertelement_v16f32:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; SI-NEXT:    s_load_dwordx16 s[8:23], s[4:5], 0x10
+; SI-NEXT:    s_load_dword s4, s[4:5], 0x20
+; SI-NEXT:    s_mov_b32 s3, 0x100f000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    v_mov_b32_e32 v16, 0x40a00000
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v0, s8
+; SI-NEXT:    v_mov_b32_e32 v1, s9
+; SI-NEXT:    v_mov_b32_e32 v2, s10
+; SI-NEXT:    v_mov_b32_e32 v3, s11
+; SI-NEXT:    v_mov_b32_e32 v4, s12
+; SI-NEXT:    v_mov_b32_e32 v5, s13
+; SI-NEXT:    v_mov_b32_e32 v6, s14
+; SI-NEXT:    v_mov_b32_e32 v7, s15
+; SI-NEXT:    v_mov_b32_e32 v8, s16
+; SI-NEXT:    v_mov_b32_e32 v9, s17
+; SI-NEXT:    v_mov_b32_e32 v10, s18
+; SI-NEXT:    v_mov_b32_e32 v11, s19
+; SI-NEXT:    v_mov_b32_e32 v12, s20
+; SI-NEXT:    v_mov_b32_e32 v13, s21
+; SI-NEXT:    v_mov_b32_e32 v14, s22
+; SI-NEXT:    v_mov_b32_e32 v15, s23
+; SI-NEXT:    s_mov_b32 m0, s4
+; SI-NEXT:    v_movreld_b32_e32 v0, v16
+; SI-NEXT:    buffer_store_dwordx4 v[12:15], off, s[0:3], 0 offset:48
+; SI-NEXT:    buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:32
+; SI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:16
+; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: dynamic_insertelement_v16f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; VI-NEXT:    s_load_dwordx16 s[8:23], s[4:5], 0x40
+; VI-NEXT:    s_load_dword s4, s[4:5], 0x80
+; VI-NEXT:    s_mov_b32 s3, 0x1100f000
+; VI-NEXT:    s_mov_b32 s2, -1
+; VI-NEXT:    v_mov_b32_e32 v16, 0x40a00000
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v0, s8
+; VI-NEXT:    v_mov_b32_e32 v1, s9
+; VI-NEXT:    v_mov_b32_e32 v2, s10
+; VI-NEXT:    v_mov_b32_e32 v3, s11
+; VI-NEXT:    v_mov_b32_e32 v4, s12
+; VI-NEXT:    v_mov_b32_e32 v5, s13
+; VI-NEXT:    v_mov_b32_e32 v6, s14
+; VI-NEXT:    v_mov_b32_e32 v7, s15
+; VI-NEXT:    v_mov_b32_e32 v8, s16
+; VI-NEXT:    v_mov_b32_e32 v9, s17
+; VI-NEXT:    v_mov_b32_e32 v10, s18
+; VI-NEXT:    v_mov_b32_e32 v11, s19
+; VI-NEXT:    v_mov_b32_e32 v12, s20
+; VI-NEXT:    v_mov_b32_e32 v13, s21
+; VI-NEXT:    v_mov_b32_e32 v14, s22
+; VI-NEXT:    v_mov_b32_e32 v15, s23
+; VI-NEXT:    s_mov_b32 m0, s4
+; VI-NEXT:    v_movreld_b32_e32 v0, v16
+; VI-NEXT:    buffer_store_dwordx4 v[12:15], off, s[0:3], 0 offset:48
+; VI-NEXT:    buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:32
+; VI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:16
+; VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; VI-NEXT:    s_endpgm
   %vecins = insertelement <16 x float> %a, float 5.000000e+00, i32 %b
   store <16 x float> %vecins, <16 x float> addrspace(1)* %out, align 64
   ret void
 }
 
-; GCN-LABEL: {{^}}dynamic_insertelement_v2i32:
-; GCN-DAG: v_cmp_ne_u32_e64 [[CC2:[^,]+]], [[IDX:s[0-9]+]], 1
-; GCN-DAG: v_cndmask_b32_e32 v{{[0-9]+}}, 5, v{{[0-9]+}}, [[CC2]]
-; GCN-DAG: v_cmp_ne_u32_e64 [[CC1:[^,]+]], [[IDX]], 0
-; GCN-DAG: v_cndmask_b32_e32 v[[LOW_RESULT_REG:[0-9]+]], 5, v{{[0-9]+}}, [[CC1]]
-; GCN: buffer_store_dwordx2 {{v\[}}[[LOW_RESULT_REG]]:
 define amdgpu_kernel void @dynamic_insertelement_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32> %a, i32 %b) nounwind {
+; SI-LABEL: dynamic_insertelement_v2i32:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; SI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
+; SI-NEXT:    s_load_dword s4, s[4:5], 0x4
+; SI-NEXT:    s_mov_b32 s3, 0x100f000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v0, s7
+; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 1
+; SI-NEXT:    v_cndmask_b32_e32 v1, 5, v0, vcc
+; SI-NEXT:    v_mov_b32_e32 v0, s6
+; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 0
+; SI-NEXT:    v_cndmask_b32_e32 v0, 5, v0, vcc
+; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: dynamic_insertelement_v2i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; VI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x8
+; VI-NEXT:    s_load_dword s4, s[4:5], 0x10
+; VI-NEXT:    s_mov_b32 s3, 0x1100f000
+; VI-NEXT:    s_mov_b32 s2, -1
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v0, s7
+; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 1
+; VI-NEXT:    v_cndmask_b32_e32 v1, 5, v0, vcc
+; VI-NEXT:    v_mov_b32_e32 v0, s6
+; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 0
+; VI-NEXT:    v_cndmask_b32_e32 v0, 5, v0, vcc
+; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; VI-NEXT:    s_endpgm
   %vecins = insertelement <2 x i32> %a, i32 5, i32 %b
   store <2 x i32> %vecins, <2 x i32> addrspace(1)* %out, align 8
   ret void
 }
 
-; GCN-LABEL: {{^}}dynamic_insertelement_v3i32:
-; GCN-DAG: v_cmp_ne_u32_e64 [[CC3:[^,]+]], [[IDX:s[0-9]+]], 2
-; GCN-DAG: v_cndmask_b32_e32 v{{[0-9]+}}, 5, v{{[0-9]+}}, [[CC3]]
-; GCN-DAG: v_cmp_ne_u32_e64 [[CC2:[^,]+]], [[IDX]], 1
-; GCN-DAG: v_cndmask_b32_e32 v{{[0-9]+}}, 5, v{{[0-9]+}}, [[CC2]]
-; GCN-DAG: v_cmp_ne_u32_e64 [[CC1:[^,]+]], [[IDX]], 0
-; GCN-DAG: v_cndmask_b32_e32 v{{[0-9]+}}, 5, v{{[0-9]+}}, [[CC1]]
-; GCN-DAG: buffer_store_dwordx3 v
 define amdgpu_kernel void @dynamic_insertelement_v3i32(<3 x i32> addrspace(1)* %out, <3 x i32> %a, i32 %b) nounwind {
+; SI-LABEL: dynamic_insertelement_v3i32:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; SI-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x4
+; SI-NEXT:    s_load_dword s4, s[4:5], 0x8
+; SI-NEXT:    s_mov_b32 s3, 0x100f000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v0, s10
+; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 2
+; SI-NEXT:    v_cndmask_b32_e32 v2, 5, v0, vcc
+; SI-NEXT:    v_mov_b32_e32 v0, s9
+; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 1
+; SI-NEXT:    v_cndmask_b32_e32 v1, 5, v0, vcc
+; SI-NEXT:    v_mov_b32_e32 v0, s8
+; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 0
+; SI-NEXT:    v_cndmask_b32_e32 v0, 5, v0, vcc
+; SI-NEXT:    buffer_store_dwordx3 v[0:2], off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: dynamic_insertelement_v3i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; VI-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x10
+; VI-NEXT:    s_load_dword s4, s[4:5], 0x20
+; VI-NEXT:    s_mov_b32 s3, 0x1100f000
+; VI-NEXT:    s_mov_b32 s2, -1
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v0, s10
+; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 2
+; VI-NEXT:    v_cndmask_b32_e32 v2, 5, v0, vcc
+; VI-NEXT:    v_mov_b32_e32 v0, s9
+; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 1
+; VI-NEXT:    v_cndmask_b32_e32 v1, 5, v0, vcc
+; VI-NEXT:    v_mov_b32_e32 v0, s8
+; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 0
+; VI-NEXT:    v_cndmask_b32_e32 v0, 5, v0, vcc
+; VI-NEXT:    buffer_store_dwordx3 v[0:2], off, s[0:3], 0
+; VI-NEXT:    s_endpgm
   %vecins = insertelement <3 x i32> %a, i32 5, i32 %b
   store <3 x i32> %vecins, <3 x i32> addrspace(1)* %out, align 16
   ret void
 }
 
-; GCN-LABEL: {{^}}dynamic_insertelement_v4i32:
-; GCN: s_load_dword [[SVAL:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, {{0x11|0x44}}
-; GCN-DAG: v_mov_b32_e32 [[VVAL:v[0-9]+]], [[SVAL]]
-; GCN-DAG: v_cmp_eq_u32_e64 [[CC4:[^,]+]], [[IDX:s[0-9]+]], 3
-; GCN-DAG: v_cndmask_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, [[VVAL]], [[CC4]]
-; GCN-DAG: v_cmp_eq_u32_e64 [[CC3:[^,]+]], [[IDX]], 2
-; GCN-DAG: v_cndmask_b32_e32 v{{[0-9]+}},  v{{[0-9]+}}, [[VVAL]], [[CC3]]
-; GCN-DAG: v_cmp_eq_u32_e64 [[CC2:[^,]+]], [[IDX]], 1
-; GCN-DAG: v_cndmask_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, [[VVAL]], [[CC2]]
-; GCN-DAG: v_cmp_eq_u32_e64 [[CC1:[^,]+]], [[IDX]], 0
-; GCN-DAG: v_cndmask_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, [[VVAL]], [[CC1]]
-; GCN: buffer_store_dwordx4
 define amdgpu_kernel void @dynamic_insertelement_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> %a, i32 %b, [8 x i32], i32 %val) nounwind {
+; SI-LABEL: dynamic_insertelement_v4i32:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; SI-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x4
+; SI-NEXT:    s_load_dword s6, s[4:5], 0x8
+; SI-NEXT:    s_load_dword s4, s[4:5], 0x11
+; SI-NEXT:    s_mov_b32 s3, 0x100f000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v0, s11
+; SI-NEXT:    v_cmp_eq_u32_e64 vcc, s6, 3
+; SI-NEXT:    v_mov_b32_e32 v4, s4
+; SI-NEXT:    v_cndmask_b32_e32 v3, v0, v4, vcc
+; SI-NEXT:    v_mov_b32_e32 v0, s10
+; SI-NEXT:    v_cmp_eq_u32_e64 vcc, s6, 2
+; SI-NEXT:    v_cndmask_b32_e32 v2, v0, v4, vcc
+; SI-NEXT:    v_mov_b32_e32 v0, s9
+; SI-NEXT:    v_cmp_eq_u32_e64 vcc, s6, 1
+; SI-NEXT:    v_cndmask_b32_e32 v1, v0, v4, vcc
+; SI-NEXT:    v_mov_b32_e32 v0, s8
+; SI-NEXT:    v_cmp_eq_u32_e64 vcc, s6, 0
+; SI-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: dynamic_insertelement_v4i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; VI-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x10
+; VI-NEXT:    s_load_dword s6, s[4:5], 0x20
+; VI-NEXT:    s_load_dword s4, s[4:5], 0x44
+; VI-NEXT:    s_mov_b32 s3, 0x1100f000
+; VI-NEXT:    s_mov_b32 s2, -1
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v0, s11
+; VI-NEXT:    v_cmp_eq_u32_e64 vcc, s6, 3
+; VI-NEXT:    v_mov_b32_e32 v4, s4
+; VI-NEXT:    v_cndmask_b32_e32 v3, v0, v4, vcc
+; VI-NEXT:    v_mov_b32_e32 v0, s10
+; VI-NEXT:    v_cmp_eq_u32_e64 vcc, s6, 2
+; VI-NEXT:    v_cndmask_b32_e32 v2, v0, v4, vcc
+; VI-NEXT:    v_mov_b32_e32 v0, s9
+; VI-NEXT:    v_cmp_eq_u32_e64 vcc, s6, 1
+; VI-NEXT:    v_cndmask_b32_e32 v1, v0, v4, vcc
+; VI-NEXT:    v_mov_b32_e32 v0, s8
+; VI-NEXT:    v_cmp_eq_u32_e64 vcc, s6, 0
+; VI-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; VI-NEXT:    s_endpgm
   %vecins = insertelement <4 x i32> %a, i32 %val, i32 %b
   store <4 x i32> %vecins, <4 x i32> addrspace(1)* %out, align 16
   ret void
 }
 
-; GCN-LABEL: {{^}}dynamic_insertelement_v8i32:
-; GCN-DAG: v_cmp_ne_u32_e64 [[CCL:[^,]+]], [[IDX:s[0-9]+]], 7
-; GCN-DAG: v_cndmask_b32_e32 v{{[0-9]+}}, 5, v{{[0-9]+}}, [[CCL]]
-; GCN-DAG: v_cmp_ne_u32_e64 [[CC1:[^,]+]], [[IDX]], 0
-; GCN-DAG: v_cndmask_b32_e32 v{{[0-9]+}}, 5, v{{[0-9]+}}, [[CC1]]
-; GCN: buffer_store_dwordx4
-; GCN: buffer_store_dwordx4
 define amdgpu_kernel void @dynamic_insertelement_v8i32(<8 x i32> addrspace(1)* %out, <8 x i32> %a, i32 %b) nounwind {
+; SI-LABEL: dynamic_insertelement_v8i32:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; SI-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x8
+; SI-NEXT:    s_load_dword s4, s[4:5], 0x10
+; SI-NEXT:    s_mov_b32 s3, 0x100f000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v0, s11
+; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 3
+; SI-NEXT:    v_cndmask_b32_e32 v3, 5, v0, vcc
+; SI-NEXT:    v_mov_b32_e32 v0, s10
+; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 2
+; SI-NEXT:    v_cndmask_b32_e32 v2, 5, v0, vcc
+; SI-NEXT:    v_mov_b32_e32 v0, s9
+; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 1
+; SI-NEXT:    v_cndmask_b32_e32 v1, 5, v0, vcc
+; SI-NEXT:    v_mov_b32_e32 v0, s8
+; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 0
+; SI-NEXT:    v_cndmask_b32_e32 v0, 5, v0, vcc
+; SI-NEXT:    v_mov_b32_e32 v4, s15
+; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 7
+; SI-NEXT:    v_cndmask_b32_e32 v7, 5, v4, vcc
+; SI-NEXT:    v_mov_b32_e32 v4, s14
+; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 6
+; SI-NEXT:    v_cndmask_b32_e32 v6, 5, v4, vcc
+; SI-NEXT:    v_mov_b32_e32 v4, s13
+; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 5
+; SI-NEXT:    v_cndmask_b32_e32 v5, 5, v4, vcc
+; SI-NEXT:    v_mov_b32_e32 v4, s12
+; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 4
+; SI-NEXT:    v_cndmask_b32_e32 v4, 5, v4, vcc
+; SI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:16
+; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: dynamic_insertelement_v8i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; VI-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x20
+; VI-NEXT:    s_load_dword s4, s[4:5], 0x40
+; VI-NEXT:    s_mov_b32 s3, 0x1100f000
+; VI-NEXT:    s_mov_b32 s2, -1
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v0, s11
+; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 3
+; VI-NEXT:    v_cndmask_b32_e32 v3, 5, v0, vcc
+; VI-NEXT:    v_mov_b32_e32 v0, s10
+; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 2
+; VI-NEXT:    v_cndmask_b32_e32 v2, 5, v0, vcc
+; VI-NEXT:    v_mov_b32_e32 v0, s9
+; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 1
+; VI-NEXT:    v_cndmask_b32_e32 v1, 5, v0, vcc
+; VI-NEXT:    v_mov_b32_e32 v0, s8
+; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 0
+; VI-NEXT:    v_cndmask_b32_e32 v0, 5, v0, vcc
+; VI-NEXT:    v_mov_b32_e32 v4, s15
+; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 7
+; VI-NEXT:    v_cndmask_b32_e32 v7, 5, v4, vcc
+; VI-NEXT:    v_mov_b32_e32 v4, s14
+; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 6
+; VI-NEXT:    v_cndmask_b32_e32 v6, 5, v4, vcc
+; VI-NEXT:    v_mov_b32_e32 v4, s13
+; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 5
+; VI-NEXT:    v_cndmask_b32_e32 v5, 5, v4, vcc
+; VI-NEXT:    v_mov_b32_e32 v4, s12
+; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 4
+; VI-NEXT:    v_cndmask_b32_e32 v4, 5, v4, vcc
+; VI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:16
+; VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; VI-NEXT:    s_endpgm
   %vecins = insertelement <8 x i32> %a, i32 5, i32 %b
   store <8 x i32> %vecins, <8 x i32> addrspace(1)* %out, align 32
   ret void
 }
 
-; GCN-LABEL: {{^}}dynamic_insertelement_v16i32:
-; GCN: v_movreld_b32
-; GCN: buffer_store_dwordx4
-; GCN: buffer_store_dwordx4
-; GCN: buffer_store_dwordx4
-; GCN: buffer_store_dwordx4
 define amdgpu_kernel void @dynamic_insertelement_v16i32(<16 x i32> addrspace(1)* %out, <16 x i32> %a, i32 %b) nounwind {
+; SI-LABEL: dynamic_insertelement_v16i32:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; SI-NEXT:    s_load_dwordx16 s[8:23], s[4:5], 0x10
+; SI-NEXT:    s_load_dword s4, s[4:5], 0x20
+; SI-NEXT:    s_mov_b32 s3, 0x100f000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v0, s8
+; SI-NEXT:    v_mov_b32_e32 v1, s9
+; SI-NEXT:    v_mov_b32_e32 v2, s10
+; SI-NEXT:    v_mov_b32_e32 v3, s11
+; SI-NEXT:    v_mov_b32_e32 v4, s12
+; SI-NEXT:    v_mov_b32_e32 v5, s13
+; SI-NEXT:    v_mov_b32_e32 v6, s14
+; SI-NEXT:    v_mov_b32_e32 v7, s15
+; SI-NEXT:    v_mov_b32_e32 v8, s16
+; SI-NEXT:    v_mov_b32_e32 v9, s17
+; SI-NEXT:    v_mov_b32_e32 v10, s18
+; SI-NEXT:    v_mov_b32_e32 v11, s19
+; SI-NEXT:    v_mov_b32_e32 v12, s20
+; SI-NEXT:    v_mov_b32_e32 v13, s21
+; SI-NEXT:    v_mov_b32_e32 v14, s22
+; SI-NEXT:    v_mov_b32_e32 v15, s23
+; SI-NEXT:    s_mov_b32 m0, s4
+; SI-NEXT:    v_movreld_b32_e32 v0, 5
+; SI-NEXT:    buffer_store_dwordx4 v[12:15], off, s[0:3], 0 offset:48
+; SI-NEXT:    buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:32
+; SI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:16
+; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: dynamic_insertelement_v16i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; VI-NEXT:    s_load_dwordx16 s[8:23], s[4:5], 0x40
+; VI-NEXT:    s_load_dword s4, s[4:5], 0x80
+; VI-NEXT:    s_mov_b32 s3, 0x1100f000
+; VI-NEXT:    s_mov_b32 s2, -1
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v0, s8
+; VI-NEXT:    v_mov_b32_e32 v1, s9
+; VI-NEXT:    v_mov_b32_e32 v2, s10
+; VI-NEXT:    v_mov_b32_e32 v3, s11
+; VI-NEXT:    v_mov_b32_e32 v4, s12
+; VI-NEXT:    v_mov_b32_e32 v5, s13
+; VI-NEXT:    v_mov_b32_e32 v6, s14
+; VI-NEXT:    v_mov_b32_e32 v7, s15
+; VI-NEXT:    v_mov_b32_e32 v8, s16
+; VI-NEXT:    v_mov_b32_e32 v9, s17
+; VI-NEXT:    v_mov_b32_e32 v10, s18
+; VI-NEXT:    v_mov_b32_e32 v11, s19
+; VI-NEXT:    v_mov_b32_e32 v12, s20
+; VI-NEXT:    v_mov_b32_e32 v13, s21
+; VI-NEXT:    v_mov_b32_e32 v14, s22
+; VI-NEXT:    v_mov_b32_e32 v15, s23
+; VI-NEXT:    s_mov_b32 m0, s4
+; VI-NEXT:    v_movreld_b32_e32 v0, 5
+; VI-NEXT:    buffer_store_dwordx4 v[12:15], off, s[0:3], 0 offset:48
+; VI-NEXT:    buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:32
+; VI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:16
+; VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; VI-NEXT:    s_endpgm
   %vecins = insertelement <16 x i32> %a, i32 5, i32 %b
   store <16 x i32> %vecins, <16 x i32> addrspace(1)* %out, align 64
   ret void
 }
 
-; GCN-LABEL: {{^}}dynamic_insertelement_v2i16:
 define amdgpu_kernel void @dynamic_insertelement_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> %a, i32 %b) nounwind {
+; SI-LABEL: dynamic_insertelement_v2i16:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; SI-NEXT:    s_load_dword s6, s[4:5], 0x2
+; SI-NEXT:    s_load_dword s4, s[4:5], 0x3
+; SI-NEXT:    v_mov_b32_e32 v0, 0x50005
+; SI-NEXT:    s_mov_b32 s3, 0x100f000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v1, s6
+; SI-NEXT:    s_lshl_b32 s4, s4, 4
+; SI-NEXT:    s_lshl_b32 s4, 0xffff, s4
+; SI-NEXT:    v_bfi_b32 v0, s4, v0, v1
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: dynamic_insertelement_v2i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; VI-NEXT:    s_load_dword s6, s[4:5], 0x8
+; VI-NEXT:    s_load_dword s4, s[4:5], 0xc
+; VI-NEXT:    v_mov_b32_e32 v0, 0x50005
+; VI-NEXT:    s_mov_b32 s3, 0x1100f000
+; VI-NEXT:    s_mov_b32 s2, -1
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s6
+; VI-NEXT:    s_lshl_b32 s4, s4, 4
+; VI-NEXT:    s_lshl_b32 s4, 0xffff, s4
+; VI-NEXT:    v_bfi_b32 v0, s4, v0, v1
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT:    s_endpgm
   %vecins = insertelement <2 x i16> %a, i16 5, i32 %b
   store <2 x i16> %vecins, <2 x i16> addrspace(1)* %out, align 8
   ret void
 }
 
-; GCN-LABEL: {{^}}dynamic_insertelement_v3i16:
 define amdgpu_kernel void @dynamic_insertelement_v3i16(<3 x i16> addrspace(1)* %out, <3 x i16> %a, i32 %b) nounwind {
+; SI-LABEL: dynamic_insertelement_v3i16:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; SI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
+; SI-NEXT:    s_load_dword s4, s[4:5], 0x4
+; SI-NEXT:    s_mov_b32 s5, 0
+; SI-NEXT:    s_mov_b32 s3, 0x100f000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_lshl_b32 s8, s4, 4
+; SI-NEXT:    s_mov_b32 s4, 0xffff
+; SI-NEXT:    s_lshl_b64 s[4:5], s[4:5], s8
+; SI-NEXT:    s_mov_b32 s8, 0x50005
+; SI-NEXT:    s_and_b32 s9, s5, s8
+; SI-NEXT:    s_and_b32 s8, s4, s8
+; SI-NEXT:    s_andn2_b64 s[4:5], s[6:7], s[4:5]
+; SI-NEXT:    s_or_b64 s[4:5], s[8:9], s[4:5]
+; SI-NEXT:    v_mov_b32_e32 v0, s5
+; SI-NEXT:    buffer_store_short v0, off, s[0:3], 0 offset:4
+; SI-NEXT:    v_mov_b32_e32 v0, s4
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: dynamic_insertelement_v3i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; VI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x8
+; VI-NEXT:    s_load_dword s4, s[4:5], 0x10
+; VI-NEXT:    s_mov_b32 s5, 0
+; VI-NEXT:    s_mov_b32 s3, 0x1100f000
+; VI-NEXT:    s_mov_b32 s2, -1
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s7
+; VI-NEXT:    s_lshl_b32 s8, s4, 4
+; VI-NEXT:    s_mov_b32 s4, 0xffff
+; VI-NEXT:    s_lshl_b64 s[4:5], s[4:5], s8
+; VI-NEXT:    s_mov_b32 s8, 0x50005
+; VI-NEXT:    v_mov_b32_e32 v0, s8
+; VI-NEXT:    v_bfi_b32 v0, s5, v0, v1
+; VI-NEXT:    v_mov_b32_e32 v1, s8
+; VI-NEXT:    v_mov_b32_e32 v2, s6
+; VI-NEXT:    v_bfi_b32 v1, s4, v1, v2
+; VI-NEXT:    buffer_store_short v0, off, s[0:3], 0 offset:4
+; VI-NEXT:    buffer_store_dword v1, off, s[0:3], 0
+; VI-NEXT:    s_endpgm
   %vecins = insertelement <3 x i16> %a, i16 5, i32 %b
   store <3 x i16> %vecins, <3 x i16> addrspace(1)* %out, align 8
   ret void
 }
 
-; GCN-LABEL: {{^}}dynamic_insertelement_v2i8:
-; VI: s_load_dword [[LOAD:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x28
-; VI-NEXT: s_load_dword [[IDX:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x4c
-; VI-NOT: _load
-; VI: s_lshl_b32 [[SCALED_IDX:s[0-9]+]], [[IDX]], 3
-; VI: v_lshlrev_b16_e64 [[MASK:v[0-9]+]], [[SCALED_IDX]], -1
-; VI: v_and_b32_e32 [[INSERT:v[0-9]+]], 0x505, [[MASK]]
-; VI: v_xor_b32_e32 [[NOT_MASK:v[0-9]+]], -1, [[MASK]]
-; VI: v_and_b32_e32 [[AND_NOT_MASK:v[0-9]+]], [[LOAD]], [[NOT_MASK]]
-; VI: v_or_b32_e32 [[OR:v[0-9]+]], [[INSERT]], [[AND_NOT_MASK]]
-; VI: buffer_store_short [[OR]]
 define amdgpu_kernel void @dynamic_insertelement_v2i8(<2 x i8> addrspace(1)* %out, [8 x i32], <2 x i8> %a, [8 x i32], i32 %b) nounwind {
+; SI-LABEL: dynamic_insertelement_v2i8:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; SI-NEXT:    s_load_dword s6, s[4:5], 0xa
+; SI-NEXT:    s_load_dword s4, s[4:5], 0x13
+; SI-NEXT:    v_mov_b32_e32 v0, 0x505
+; SI-NEXT:    s_mov_b32 s3, 0x100f000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v1, s6
+; SI-NEXT:    s_lshl_b32 s4, s4, 3
+; SI-NEXT:    s_lshl_b32 s4, -1, s4
+; SI-NEXT:    v_bfi_b32 v0, s4, v0, v1
+; SI-NEXT:    buffer_store_short v0, off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: dynamic_insertelement_v2i8:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; VI-NEXT:    s_load_dword s6, s[4:5], 0x28
+; VI-NEXT:    s_load_dword s4, s[4:5], 0x4c
+; VI-NEXT:    s_mov_b32 s3, 0x1100f000
+; VI-NEXT:    s_mov_b32 s2, -1
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_lshl_b32 s4, s4, 3
+; VI-NEXT:    v_lshlrev_b16_e64 v0, s4, -1
+; VI-NEXT:    v_and_b32_e32 v1, 0x505, v0
+; VI-NEXT:    v_xor_b32_e32 v0, -1, v0
+; VI-NEXT:    v_and_b32_e32 v0, s6, v0
+; VI-NEXT:    v_or_b32_e32 v0, v1, v0
+; VI-NEXT:    buffer_store_short v0, off, s[0:3], 0
+; VI-NEXT:    s_endpgm
   %vecins = insertelement <2 x i8> %a, i8 5, i32 %b
   store <2 x i8> %vecins, <2 x i8> addrspace(1)* %out, align 8
   ret void
@@ -255,83 +983,340 @@ define amdgpu_kernel void @dynamic_insertelement_v2i8(<2 x i8> addrspace(1)* %ou
 
 ; FIXME: post legalize i16 and i32 shifts aren't merged because of
 ; isTypeDesirableForOp in SimplifyDemandedBits
-
-; GCN-LABEL: {{^}}dynamic_insertelement_v3i8:
-; VI: s_load_dword [[LOAD:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x28
-; VI-NEXT: s_load_dword [[IDX:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x4c
-; VI-NOT: _load
-
-; VI: v_mov_b32_e32 [[VAL:v[0-9]+]], 0x5050505
-; VI: v_mov_b32_e32 [[V_LOAD:v[0-9]+]], [[LOAD]]
-; VI: s_lshl_b32 [[SCALED_IDX:s[0-9]+]], [[IDX]], 3
-; VI: s_lshl_b32 [[SHIFTED_MASK:s[0-9]+]], 0xffff, [[SCALED_IDX]]
-; VI: v_bfi_b32 [[BFI:v[0-9]+]], [[SHIFTED_MASK]], [[VAL]], [[V_LOAD]]
-; VI: v_lshrrev_b32_e32 [[V_HI2:v[0-9]+]], 16, [[BFI]]
-
-; VI: buffer_store_short [[BFI]]
-; VI: buffer_store_byte [[V_HI2]]
 define amdgpu_kernel void @dynamic_insertelement_v3i8(<3 x i8> addrspace(1)* %out, [8 x i32], <3 x i8> %a, [8 x i32], i32 %b) nounwind {
+; SI-LABEL: dynamic_insertelement_v3i8:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; SI-NEXT:    s_load_dword s6, s[4:5], 0xa
+; SI-NEXT:    s_load_dword s4, s[4:5], 0x13
+; SI-NEXT:    v_mov_b32_e32 v0, 0x5050505
+; SI-NEXT:    s_mov_b32 s3, 0x100f000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v1, s6
+; SI-NEXT:    s_lshl_b32 s4, s4, 3
+; SI-NEXT:    s_lshl_b32 s4, 0xffff, s4
+; SI-NEXT:    v_bfi_b32 v0, s4, v0, v1
+; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
+; SI-NEXT:    buffer_store_short v0, off, s[0:3], 0
+; SI-NEXT:    buffer_store_byte v1, off, s[0:3], 0 offset:2
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: dynamic_insertelement_v3i8:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; VI-NEXT:    s_load_dword s6, s[4:5], 0x28
+; VI-NEXT:    s_load_dword s4, s[4:5], 0x4c
+; VI-NEXT:    v_mov_b32_e32 v0, 0x5050505
+; VI-NEXT:    s_mov_b32 s3, 0x1100f000
+; VI-NEXT:    s_mov_b32 s2, -1
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s6
+; VI-NEXT:    s_lshl_b32 s4, s4, 3
+; VI-NEXT:    s_lshl_b32 s4, 0xffff, s4
+; VI-NEXT:    v_bfi_b32 v0, s4, v0, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
+; VI-NEXT:    buffer_store_short v0, off, s[0:3], 0
+; VI-NEXT:    buffer_store_byte v1, off, s[0:3], 0 offset:2
+; VI-NEXT:    s_endpgm
   %vecins = insertelement <3 x i8> %a, i8 5, i32 %b
   store <3 x i8> %vecins, <3 x i8> addrspace(1)* %out, align 4
   ret void
 }
 
-; GCN-LABEL: {{^}}dynamic_insertelement_v4i8:
-; VI: s_load_dword [[LOAD:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x28
-; VI-NEXT: s_load_dword [[IDX:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x4c
-; VI-NOT: _load
-
-; VI: v_mov_b32_e32 [[VAL:v[0-9]+]], 0x5050505
-; VI: v_mov_b32_e32 [[V_LOAD:v[0-9]+]], [[LOAD]]
-; VI-DAG: s_lshl_b32 [[SCALED_IDX:s[0-9]+]], [[IDX]], 3
-; VI: s_lshl_b32 [[SHIFTED_MASK:s[0-9]+]], 0xffff, [[SCALED_IDX]]
-; VI: v_bfi_b32 [[BFI:v[0-9]+]], [[SHIFTED_MASK]], [[VAL]], [[V_LOAD]]
-; VI: buffer_store_dword [[BFI]]
 define amdgpu_kernel void @dynamic_insertelement_v4i8(<4 x i8> addrspace(1)* %out, [8 x i32], <4 x i8> %a, [8 x i32], i32 %b) nounwind {
+; SI-LABEL: dynamic_insertelement_v4i8:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; SI-NEXT:    s_load_dword s6, s[4:5], 0xa
+; SI-NEXT:    s_load_dword s4, s[4:5], 0x13
+; SI-NEXT:    v_mov_b32_e32 v0, 0x5050505
+; SI-NEXT:    s_mov_b32 s3, 0x100f000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v1, s6
+; SI-NEXT:    s_lshl_b32 s4, s4, 3
+; SI-NEXT:    s_lshl_b32 s4, 0xffff, s4
+; SI-NEXT:    v_bfi_b32 v0, s4, v0, v1
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: dynamic_insertelement_v4i8:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; VI-NEXT:    s_load_dword s6, s[4:5], 0x28
+; VI-NEXT:    s_load_dword s4, s[4:5], 0x4c
+; VI-NEXT:    v_mov_b32_e32 v0, 0x5050505
+; VI-NEXT:    s_mov_b32 s3, 0x1100f000
+; VI-NEXT:    s_mov_b32 s2, -1
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s6
+; VI-NEXT:    s_lshl_b32 s4, s4, 3
+; VI-NEXT:    s_lshl_b32 s4, 0xffff, s4
+; VI-NEXT:    v_bfi_b32 v0, s4, v0, v1
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT:    s_endpgm
   %vecins = insertelement <4 x i8> %a, i8 5, i32 %b
   store <4 x i8> %vecins, <4 x i8> addrspace(1)* %out, align 4
   ret void
 }
 
-; GCN-LABEL: {{^}}s_dynamic_insertelement_v8i8:
-; VI-NOT: {{buffer|flat|global}}_load
-; VI-DAG: s_load_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s[4:5], 0x0
-; VI-DAG: s_load_dword [[IDX:s[0-9]]], s[4:5], 0x10
-; VI-DAG: s_mov_b32 s[[MASK_HI:[0-9]+]], 0{{$}}
-; VI-DAG: s_load_dwordx2 [[VEC:s\[[0-9]+:[0-9]+\]]], s{{\[[0-9]+:[0-9]+\]}}, 0x0
-
-; VI-DAG: s_lshl_b32 [[SCALED_IDX:s[0-9]+]], [[IDX]], 3
-; VI-DAG: s_mov_b32 s[[MASK_LO:[0-9]+]], 0xffff
-; VI: s_lshl_b64 s{{\[}}[[MASK_SHIFT_LO:[0-9]+]]:[[MASK_SHIFT_HI:[0-9]+]]{{\]}}, s{{\[}}[[MASK_LO]]:[[MASK_HI]]{{\]}}, [[SCALED_IDX]]
-; VI: s_mov_b32 [[VAL:s[0-9]+]], 0x5050505
-; VI: s_and_b32 s[[INS_HI:[0-9]+]], s[[MASK_SHIFT_HI]], [[VAL]]
-; VI: s_and_b32 s[[INS_LO:[0-9]+]], s[[MASK_SHIFT_LO]], [[VAL]]
-; VI: s_andn2_b64 [[AND:s\[[0-9]+:[0-9]+\]]], [[VEC]], s{{\[}}[[MASK_SHIFT_LO]]:[[MASK_SHIFT_HI]]{{\]}}
-; VI: s_or_b64 s{{\[}}[[RESULT0:[0-9]+]]:[[RESULT1:[0-9]+]]{{\]}}, s{{\[}}[[INS_LO]]:[[INS_HI]]{{\]}}, [[AND]]
-; VI: v_mov_b32_e32 v[[V_RESULT0:[0-9]+]], s[[RESULT0]]
-; VI: v_mov_b32_e32 v[[V_RESULT1:[0-9]+]], s[[RESULT1]]
-; VI: buffer_store_dwordx2 v{{\[}}[[V_RESULT0]]:[[V_RESULT1]]{{\]}}
 define amdgpu_kernel void @s_dynamic_insertelement_v8i8(<8 x i8> addrspace(1)* %out, <8 x i8> addrspace(4)* %a.ptr, i32 %b) nounwind {
+; SI-LABEL: s_dynamic_insertelement_v8i8:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x0
+; SI-NEXT:    s_load_dword s6, s[4:5], 0x4
+; SI-NEXT:    s_mov_b32 s7, 0
+; SI-NEXT:    s_mov_b32 s3, 0x100f000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[10:11], 0x0
+; SI-NEXT:    s_mov_b32 s0, s8
+; SI-NEXT:    s_lshl_b32 s8, s6, 3
+; SI-NEXT:    s_mov_b32 s6, 0xffff
+; SI-NEXT:    s_lshl_b64 s[6:7], s[6:7], s8
+; SI-NEXT:    s_mov_b32 s8, 0x5050505
+; SI-NEXT:    s_mov_b32 s1, s9
+; SI-NEXT:    s_and_b32 s9, s7, s8
+; SI-NEXT:    s_and_b32 s8, s6, s8
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_andn2_b64 s[4:5], s[4:5], s[6:7]
+; SI-NEXT:    s_or_b64 s[4:5], s[8:9], s[4:5]
+; SI-NEXT:    v_mov_b32_e32 v0, s4
+; SI-NEXT:    v_mov_b32_e32 v1, s5
+; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: s_dynamic_insertelement_v8i8:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x0
+; VI-NEXT:    s_load_dword s6, s[4:5], 0x10
+; VI-NEXT:    s_mov_b32 s7, 0
+; VI-NEXT:    s_mov_b32 s3, 0x1100f000
+; VI-NEXT:    s_mov_b32 s2, -1
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_load_dwordx2 s[4:5], s[10:11], 0x0
+; VI-NEXT:    s_mov_b32 s0, s8
+; VI-NEXT:    s_lshl_b32 s8, s6, 3
+; VI-NEXT:    s_mov_b32 s6, 0xffff
+; VI-NEXT:    s_lshl_b64 s[6:7], s[6:7], s8
+; VI-NEXT:    s_mov_b32 s8, 0x5050505
+; VI-NEXT:    s_mov_b32 s1, s9
+; VI-NEXT:    s_and_b32 s9, s7, s8
+; VI-NEXT:    s_and_b32 s8, s6, s8
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_andn2_b64 s[4:5], s[4:5], s[6:7]
+; VI-NEXT:    s_or_b64 s[4:5], s[8:9], s[4:5]
+; VI-NEXT:    v_mov_b32_e32 v0, s4
+; VI-NEXT:    v_mov_b32_e32 v1, s5
+; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; VI-NEXT:    s_endpgm
   %a = load <8 x i8>, <8 x i8> addrspace(4)* %a.ptr, align 4
   %vecins = insertelement <8 x i8> %a, i8 5, i32 %b
   store <8 x i8> %vecins, <8 x i8> addrspace(1)* %out, align 8
   ret void
 }
 
-; GCN-LABEL: {{^}}dynamic_insertelement_v16i8:
-; GCN: s_load_dwordx2
-; GCN: s_load_dwordx4
-; GCN: s_load_dword s
-
-; GCN-NOT: buffer_store_byte
-
-; GCN-DAG: v_cmp_ne_u32_e64 [[CCL:[^,]+]], [[IDX:s[0-9]+]], 15
-; GCN-DAG: v_cndmask_b32_e32 v{{[0-9]+}}, 5, v{{[0-9]+}}, [[CCL]]
-; GCN-DAG: v_cmp_ne_u32_e64 [[CC1:[^,]+]], [[IDX]], 0
-; GCN-DAG: v_cndmask_b32_e32 v{{[0-9]+}}, 5, v{{[0-9]+}}, [[CC1]]
-
-; GCN: buffer_store_dwordx4
 define amdgpu_kernel void @dynamic_insertelement_v16i8(<16 x i8> addrspace(1)* %out, <16 x i8> %a, i32 %b) nounwind {
+; SI-LABEL: dynamic_insertelement_v16i8:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; SI-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x4
+; SI-NEXT:    s_load_dword s4, s[4:5], 0x8
+; SI-NEXT:    s_mov_b32 s3, 0x100f000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_lshr_b32 s5, s11, 24
+; SI-NEXT:    v_mov_b32_e32 v0, s5
+; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 15
+; SI-NEXT:    s_lshr_b32 s5, s11, 16
+; SI-NEXT:    v_cndmask_b32_e32 v0, 5, v0, vcc
+; SI-NEXT:    v_mov_b32_e32 v1, s5
+; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 14
+; SI-NEXT:    v_cndmask_b32_e32 v1, 5, v1, vcc
+; SI-NEXT:    s_movk_i32 s5, 0xff
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
+; SI-NEXT:    v_and_b32_e32 v1, s5, v1
+; SI-NEXT:    s_lshr_b32 s6, s11, 8
+; SI-NEXT:    v_or_b32_e32 v0, v1, v0
+; SI-NEXT:    v_mov_b32_e32 v1, s6
+; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 13
+; SI-NEXT:    v_cndmask_b32_e32 v1, 5, v1, vcc
+; SI-NEXT:    v_mov_b32_e32 v2, s11
+; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 12
+; SI-NEXT:    v_cndmask_b32_e32 v2, 5, v2, vcc
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; SI-NEXT:    v_and_b32_e32 v2, s5, v2
+; SI-NEXT:    v_or_b32_e32 v1, v2, v1
+; SI-NEXT:    s_mov_b32 s6, 0xffff
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT:    v_and_b32_e32 v1, s6, v1
+; SI-NEXT:    s_lshr_b32 s7, s10, 24
+; SI-NEXT:    v_or_b32_e32 v3, v1, v0
+; SI-NEXT:    v_mov_b32_e32 v0, s7
+; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 11
+; SI-NEXT:    s_lshr_b32 s7, s10, 16
+; SI-NEXT:    v_cndmask_b32_e32 v0, 5, v0, vcc
+; SI-NEXT:    v_mov_b32_e32 v1, s7
+; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 10
+; SI-NEXT:    v_cndmask_b32_e32 v1, 5, v1, vcc
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
+; SI-NEXT:    v_and_b32_e32 v1, s5, v1
+; SI-NEXT:    s_lshr_b32 s7, s10, 8
+; SI-NEXT:    v_or_b32_e32 v0, v1, v0
+; SI-NEXT:    v_mov_b32_e32 v1, s7
+; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 9
+; SI-NEXT:    v_cndmask_b32_e32 v1, 5, v1, vcc
+; SI-NEXT:    v_mov_b32_e32 v2, s10
+; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 8
+; SI-NEXT:    v_cndmask_b32_e32 v2, 5, v2, vcc
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; SI-NEXT:    v_and_b32_e32 v2, s5, v2
+; SI-NEXT:    v_or_b32_e32 v1, v2, v1
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT:    v_and_b32_e32 v1, s6, v1
+; SI-NEXT:    s_lshr_b32 s7, s9, 24
+; SI-NEXT:    v_or_b32_e32 v2, v1, v0
+; SI-NEXT:    v_mov_b32_e32 v0, s7
+; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 7
+; SI-NEXT:    s_lshr_b32 s7, s9, 16
+; SI-NEXT:    v_cndmask_b32_e32 v0, 5, v0, vcc
+; SI-NEXT:    v_mov_b32_e32 v1, s7
+; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 6
+; SI-NEXT:    v_cndmask_b32_e32 v1, 5, v1, vcc
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
+; SI-NEXT:    v_and_b32_e32 v1, s5, v1
+; SI-NEXT:    s_lshr_b32 s7, s9, 8
+; SI-NEXT:    v_or_b32_e32 v0, v1, v0
+; SI-NEXT:    v_mov_b32_e32 v1, s7
+; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 5
+; SI-NEXT:    v_cndmask_b32_e32 v1, 5, v1, vcc
+; SI-NEXT:    v_mov_b32_e32 v4, s9
+; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 4
+; SI-NEXT:    v_cndmask_b32_e32 v4, 5, v4, vcc
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; SI-NEXT:    v_and_b32_e32 v4, s5, v4
+; SI-NEXT:    v_or_b32_e32 v1, v4, v1
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT:    v_and_b32_e32 v1, s6, v1
+; SI-NEXT:    s_lshr_b32 s7, s8, 24
+; SI-NEXT:    v_or_b32_e32 v1, v1, v0
+; SI-NEXT:    v_mov_b32_e32 v0, s7
+; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 3
+; SI-NEXT:    s_lshr_b32 s7, s8, 16
+; SI-NEXT:    v_cndmask_b32_e32 v0, 5, v0, vcc
+; SI-NEXT:    v_mov_b32_e32 v4, s7
+; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 2
+; SI-NEXT:    v_cndmask_b32_e32 v4, 5, v4, vcc
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
+; SI-NEXT:    v_and_b32_e32 v4, s5, v4
+; SI-NEXT:    s_lshr_b32 s7, s8, 8
+; SI-NEXT:    v_or_b32_e32 v0, v4, v0
+; SI-NEXT:    v_mov_b32_e32 v4, s7
+; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 1
+; SI-NEXT:    v_cndmask_b32_e32 v4, 5, v4, vcc
+; SI-NEXT:    v_mov_b32_e32 v5, s8
+; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 0
+; SI-NEXT:    v_cndmask_b32_e32 v5, 5, v5, vcc
+; SI-NEXT:    v_lshlrev_b32_e32 v4, 8, v4
+; SI-NEXT:    v_and_b32_e32 v5, s5, v5
+; SI-NEXT:    v_or_b32_e32 v4, v5, v4
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT:    v_and_b32_e32 v4, s6, v4
+; SI-NEXT:    v_or_b32_e32 v0, v4, v0
+; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: dynamic_insertelement_v16i8:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; VI-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x10
+; VI-NEXT:    s_load_dword s4, s[4:5], 0x20
+; VI-NEXT:    s_mov_b32 s3, 0x1100f000
+; VI-NEXT:    s_mov_b32 s2, -1
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_lshr_b32 s5, s11, 24
+; VI-NEXT:    v_mov_b32_e32 v0, s5
+; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 15
+; VI-NEXT:    s_lshr_b32 s5, s11, 16
+; VI-NEXT:    v_cndmask_b32_e32 v0, 5, v0, vcc
+; VI-NEXT:    v_mov_b32_e32 v1, s5
+; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 14
+; VI-NEXT:    v_cndmask_b32_e32 v1, 5, v1, vcc
+; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
+; VI-NEXT:    s_lshr_b32 s5, s11, 8
+; VI-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_mov_b32_e32 v1, s5
+; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 13
+; VI-NEXT:    v_cndmask_b32_e32 v1, 5, v1, vcc
+; VI-NEXT:    v_mov_b32_e32 v2, s11
+; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 12
+; VI-NEXT:    v_cndmask_b32_e32 v2, 5, v2, vcc
+; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT:    v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    s_lshr_b32 s5, s10, 24
+; VI-NEXT:    v_or_b32_sdwa v3, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_mov_b32_e32 v0, s5
+; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 11
+; VI-NEXT:    s_lshr_b32 s5, s10, 16
+; VI-NEXT:    v_cndmask_b32_e32 v0, 5, v0, vcc
+; VI-NEXT:    v_mov_b32_e32 v1, s5
+; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 10
+; VI-NEXT:    v_cndmask_b32_e32 v1, 5, v1, vcc
+; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
+; VI-NEXT:    s_lshr_b32 s5, s10, 8
+; VI-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_mov_b32_e32 v1, s5
+; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 9
+; VI-NEXT:    v_cndmask_b32_e32 v1, 5, v1, vcc
+; VI-NEXT:    v_mov_b32_e32 v2, s10
+; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 8
+; VI-NEXT:    v_cndmask_b32_e32 v2, 5, v2, vcc
+; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT:    v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    s_lshr_b32 s5, s9, 24
+; VI-NEXT:    v_or_b32_sdwa v2, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_mov_b32_e32 v0, s5
+; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 7
+; VI-NEXT:    s_lshr_b32 s5, s9, 16
+; VI-NEXT:    v_cndmask_b32_e32 v0, 5, v0, vcc
+; VI-NEXT:    v_mov_b32_e32 v1, s5
+; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 6
+; VI-NEXT:    v_cndmask_b32_e32 v1, 5, v1, vcc
+; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
+; VI-NEXT:    s_lshr_b32 s5, s9, 8
+; VI-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_mov_b32_e32 v1, s5
+; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 5
+; VI-NEXT:    v_cndmask_b32_e32 v1, 5, v1, vcc
+; VI-NEXT:    v_mov_b32_e32 v4, s9
+; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 4
+; VI-NEXT:    v_cndmask_b32_e32 v4, 5, v4, vcc
+; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT:    v_or_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    s_lshr_b32 s5, s8, 24
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_mov_b32_e32 v0, s5
+; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 3
+; VI-NEXT:    s_lshr_b32 s5, s8, 16
+; VI-NEXT:    v_cndmask_b32_e32 v0, 5, v0, vcc
+; VI-NEXT:    v_mov_b32_e32 v4, s5
+; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 2
+; VI-NEXT:    v_cndmask_b32_e32 v4, 5, v4, vcc
+; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
+; VI-NEXT:    s_lshr_b32 s5, s8, 8
+; VI-NEXT:    v_or_b32_sdwa v0, v4, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_mov_b32_e32 v4, s5
+; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 1
+; VI-NEXT:    v_cndmask_b32_e32 v4, 5, v4, vcc
+; VI-NEXT:    v_mov_b32_e32 v5, s8
+; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 0
+; VI-NEXT:    v_lshlrev_b16_e32 v4, 8, v4
+; VI-NEXT:    v_cndmask_b32_e32 v5, 5, v5, vcc
+; VI-NEXT:    v_or_b32_sdwa v4, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; VI-NEXT:    s_endpgm
   %vecins = insertelement <16 x i8> %a, i8 5, i32 %b
   store <16 x i8> %vecins, <16 x i8> addrspace(1)* %out, align 16
   ret void
@@ -339,8 +1324,48 @@ define amdgpu_kernel void @dynamic_insertelement_v16i8(<16 x i8> addrspace(1)* %
 
 ; This test requires handling INSERT_SUBREG in SIFixSGPRCopies.  Check that
 ; the compiler doesn't crash.
-; GCN-LABEL: {{^}}insert_split_bb:
 define amdgpu_kernel void @insert_split_bb(<2 x i32> addrspace(1)* %out, i32 addrspace(1)* %in, i32 %a, i32 %b) {
+; SI-LABEL: insert_split_bb:
+; SI:       ; %bb.0: ; %entry
+; SI-NEXT:    s_load_dword s0, s[4:5], 0x4
+; SI-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_cmp_lg_u32 s0, 0
+; SI-NEXT:    s_cbranch_scc0 BB26_2
+; SI-NEXT:  ; %bb.1: ; %else
+; SI-NEXT:    s_load_dword s1, s[6:7], 0x1
+; SI-NEXT:    s_branch BB26_3
+; SI-NEXT:  BB26_2: ; %if
+; SI-NEXT:    s_load_dword s1, s[6:7], 0x0
+; SI-NEXT:  BB26_3: ; %endif
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v0, s0
+; SI-NEXT:    s_mov_b32 s7, 0x100f000
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    v_mov_b32_e32 v1, s1
+; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: insert_split_bb:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_load_dword s0, s[4:5], 0x10
+; VI-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_cmp_lg_u32 s0, 0
+; VI-NEXT:    s_cbranch_scc0 BB26_2
+; VI-NEXT:  ; %bb.1: ; %else
+; VI-NEXT:    s_load_dword s1, s[6:7], 0x4
+; VI-NEXT:    s_branch BB26_3
+; VI-NEXT:  BB26_2: ; %if
+; VI-NEXT:    s_load_dword s1, s[6:7], 0x0
+; VI-NEXT:  BB26_3: ; %endif
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    s_mov_b32 s7, 0x1100f000
+; VI-NEXT:    s_mov_b32 s6, -1
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; VI-NEXT:    s_endpgm
 entry:
   %0 = insertelement <2 x i32> undef, i32 %a, i32 0
   %1 = icmp eq i32 %a, 0
@@ -363,111 +1388,324 @@ endif:
   ret void
 }
 
-; GCN-LABEL: {{^}}dynamic_insertelement_v2f64:
-; GCN-DAG: s_load_dwordx4 s{{\[}}[[A_ELT0:[0-9]+]]:[[A_ELT3:[0-9]+]]{{\]}}
-; GCN-DAG: s_load_dword [[IDX:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, {{0x18|0x60}}{{$}}
-
-; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, s{{[0-9]+}}
-; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, s{{[0-9]+}}
-; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, s{{[0-9]+}}
-; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, s{{[0-9]+}}
-; GCN-DAG: v_mov_b32_e32 [[ELT1:v[0-9]+]], 0x40200000
-
-; GCN-DAG: v_cmp_eq_u32_e64 [[CC2:[^,]+]], [[IDX:s[0-9]+]], 1
-; GCN-DAG: v_cndmask_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, [[ELT1]], [[CC2]]
-; GCN-DAG: v_cndmask_b32_e64 v{{[0-9]+}}, v{{[0-9]+}}, 0, [[CC2]]
-; GCN-DAG: v_cmp_eq_u32_e64 [[CC1:[^,]+]], [[IDX]], 0
-; GCN-DAG: v_cndmask_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, [[ELT1]], [[CC1]]
-; GCN-DAG: v_cndmask_b32_e64 v{{[0-9]+}}, v{{[0-9]+}}, 0, [[CC1]]
-
-; GCN: buffer_store_dwordx4
-; GCN: s_endpgm
 define amdgpu_kernel void @dynamic_insertelement_v2f64(<2 x double> addrspace(1)* %out, [8 x i32], <2 x double> %a, [8 x i32], i32 %b) nounwind {
+; SI-LABEL: dynamic_insertelement_v2f64:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; SI-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0xc
+; SI-NEXT:    s_load_dword s4, s[4:5], 0x18
+; SI-NEXT:    v_mov_b32_e32 v1, 0x40200000
+; SI-NEXT:    s_mov_b32 s3, 0x100f000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v0, s11
+; SI-NEXT:    v_cmp_eq_u32_e64 vcc, s4, 1
+; SI-NEXT:    v_cndmask_b32_e32 v3, v0, v1, vcc
+; SI-NEXT:    v_mov_b32_e32 v0, s10
+; SI-NEXT:    v_cndmask_b32_e64 v2, v0, 0, vcc
+; SI-NEXT:    v_mov_b32_e32 v0, s9
+; SI-NEXT:    v_cmp_eq_u32_e64 vcc, s4, 0
+; SI-NEXT:    v_cndmask_b32_e32 v1, v0, v1, vcc
+; SI-NEXT:    v_mov_b32_e32 v0, s8
+; SI-NEXT:    v_cndmask_b32_e64 v0, v0, 0, vcc
+; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: dynamic_insertelement_v2f64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; VI-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x30
+; VI-NEXT:    s_load_dword s4, s[4:5], 0x60
+; VI-NEXT:    v_mov_b32_e32 v1, 0x40200000
+; VI-NEXT:    s_mov_b32 s3, 0x1100f000
+; VI-NEXT:    s_mov_b32 s2, -1
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v0, s11
+; VI-NEXT:    v_cmp_eq_u32_e64 vcc, s4, 1
+; VI-NEXT:    v_cndmask_b32_e32 v3, v0, v1, vcc
+; VI-NEXT:    v_mov_b32_e32 v0, s10
+; VI-NEXT:    v_cndmask_b32_e64 v2, v0, 0, vcc
+; VI-NEXT:    v_mov_b32_e32 v0, s9
+; VI-NEXT:    v_cmp_eq_u32_e64 vcc, s4, 0
+; VI-NEXT:    v_cndmask_b32_e32 v1, v0, v1, vcc
+; VI-NEXT:    v_mov_b32_e32 v0, s8
+; VI-NEXT:    v_cndmask_b32_e64 v0, v0, 0, vcc
+; VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; VI-NEXT:    s_endpgm
   %vecins = insertelement <2 x double> %a, double 8.0, i32 %b
   store <2 x double> %vecins, <2 x double> addrspace(1)* %out, align 16
   ret void
 }
 
-; GCN-LABEL: {{^}}dynamic_insertelement_v2i64:
-
-; GCN-DAG: v_cmp_eq_u32_e64 [[CC2:[^,]+]], [[IDX:s[0-9]+]], 1
-; GCN-DAG: v_cndmask_b32_e64 v{{[0-9]+}}, v{{[0-9]+}}, 5, [[CC2]]
-; GCN-DAG: v_cndmask_b32_e64 v{{[0-9]+}}, v{{[0-9]+}}, 0, [[CC2]]
-; GCN-DAG: v_cmp_eq_u32_e64 [[CC1:[^,]+]], [[IDX]], 0
-; GCN-DAG: v_cndmask_b32_e64 v{{[0-9]+}}, v{{[0-9]+}}, 5, [[CC1]]
-; GCN-DAG: v_cndmask_b32_e64 v{{[0-9]+}}, v{{[0-9]+}}, 0, [[CC1]]
-
-; GCN: buffer_store_dwordx4
-; GCN: s_endpgm
 define amdgpu_kernel void @dynamic_insertelement_v2i64(<2 x i64> addrspace(1)* %out, <2 x i64> %a, i32 %b) nounwind {
+; SI-LABEL: dynamic_insertelement_v2i64:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; SI-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x4
+; SI-NEXT:    s_load_dword s6, s[4:5], 0x8
+; SI-NEXT:    s_mov_b32 s3, 0x100f000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v0, s11
+; SI-NEXT:    v_cmp_eq_u32_e64 s[4:5], s6, 1
+; SI-NEXT:    v_cndmask_b32_e64 v3, v0, 0, s[4:5]
+; SI-NEXT:    v_mov_b32_e32 v0, s10
+; SI-NEXT:    v_cndmask_b32_e64 v2, v0, 5, s[4:5]
+; SI-NEXT:    v_mov_b32_e32 v0, s9
+; SI-NEXT:    v_cmp_eq_u32_e64 s[4:5], s6, 0
+; SI-NEXT:    v_cndmask_b32_e64 v1, v0, 0, s[4:5]
+; SI-NEXT:    v_mov_b32_e32 v0, s8
+; SI-NEXT:    v_cndmask_b32_e64 v0, v0, 5, s[4:5]
+; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: dynamic_insertelement_v2i64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; VI-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x10
+; VI-NEXT:    s_load_dword s6, s[4:5], 0x20
+; VI-NEXT:    s_mov_b32 s3, 0x1100f000
+; VI-NEXT:    s_mov_b32 s2, -1
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v0, s11
+; VI-NEXT:    v_cmp_eq_u32_e64 s[4:5], s6, 1
+; VI-NEXT:    v_cndmask_b32_e64 v3, v0, 0, s[4:5]
+; VI-NEXT:    v_mov_b32_e32 v0, s10
+; VI-NEXT:    v_cndmask_b32_e64 v2, v0, 5, s[4:5]
+; VI-NEXT:    v_mov_b32_e32 v0, s9
+; VI-NEXT:    v_cmp_eq_u32_e64 s[4:5], s6, 0
+; VI-NEXT:    v_cndmask_b32_e64 v1, v0, 0, s[4:5]
+; VI-NEXT:    v_mov_b32_e32 v0, s8
+; VI-NEXT:    v_cndmask_b32_e64 v0, v0, 5, s[4:5]
+; VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; VI-NEXT:    s_endpgm
   %vecins = insertelement <2 x i64> %a, i64 5, i32 %b
   store <2 x i64> %vecins, <2 x i64> addrspace(1)* %out, align 8
   ret void
 }
 
-; GCN-LABEL: {{^}}dynamic_insertelement_v3i64:
-; GCN-DAG: v_cmp_eq_u32_e64 [[CC3:[^,]+]], [[IDX:s[0-9]+]], 2
-; GCN-DAG: v_cndmask_b32_e64 v{{[0-9]+}},  v{{[0-9]+}}, 5, [[CC3]]
-; GCN-DAG: v_cndmask_b32_e64 v{{[0-9]+}},  v{{[0-9]+}}, 0, [[CC3]]
-; GCN-DAG: v_cmp_eq_u32_e64 [[CC2:[^,]+]], [[IDX]], 1
-; GCN-DAG: v_cndmask_b32_e64 v{{[0-9]+}}, v{{[0-9]+}}, 5, [[CC2]]
-; GCN-DAG: v_cndmask_b32_e64 v{{[0-9]+}}, v{{[0-9]+}}, 0, [[CC2]]
-; GCN-DAG: v_cmp_eq_u32_e64 [[CC1:[^,]+]], [[IDX]], 0
-; GCN-DAG: v_cndmask_b32_e64 v{{[0-9]+}}, v{{[0-9]+}}, 5, [[CC1]]
-; GCN-DAG: v_cndmask_b32_e64 v{{[0-9]+}}, v{{[0-9]+}}, 0, [[CC1]]
 define amdgpu_kernel void @dynamic_insertelement_v3i64(<3 x i64> addrspace(1)* %out, <3 x i64> %a, i32 %b) nounwind {
+; SI-LABEL: dynamic_insertelement_v3i64:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; SI-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x8
+; SI-NEXT:    s_load_dword s6, s[4:5], 0x10
+; SI-NEXT:    s_mov_b32 s3, 0x100f000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v0, s13
+; SI-NEXT:    v_cmp_eq_u32_e64 s[4:5], s6, 2
+; SI-NEXT:    v_cndmask_b32_e64 v5, v0, 0, s[4:5]
+; SI-NEXT:    v_mov_b32_e32 v0, s12
+; SI-NEXT:    v_cndmask_b32_e64 v4, v0, 5, s[4:5]
+; SI-NEXT:    v_mov_b32_e32 v0, s11
+; SI-NEXT:    v_cmp_eq_u32_e64 s[4:5], s6, 1
+; SI-NEXT:    v_cndmask_b32_e64 v3, v0, 0, s[4:5]
+; SI-NEXT:    v_mov_b32_e32 v0, s10
+; SI-NEXT:    v_cndmask_b32_e64 v2, v0, 5, s[4:5]
+; SI-NEXT:    v_mov_b32_e32 v0, s9
+; SI-NEXT:    v_cmp_eq_u32_e64 s[4:5], s6, 0
+; SI-NEXT:    v_cndmask_b32_e64 v1, v0, 0, s[4:5]
+; SI-NEXT:    v_mov_b32_e32 v0, s8
+; SI-NEXT:    v_cndmask_b32_e64 v0, v0, 5, s[4:5]
+; SI-NEXT:    buffer_store_dwordx2 v[4:5], off, s[0:3], 0 offset:16
+; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: dynamic_insertelement_v3i64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; VI-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x20
+; VI-NEXT:    s_load_dword s6, s[4:5], 0x40
+; VI-NEXT:    s_mov_b32 s3, 0x1100f000
+; VI-NEXT:    s_mov_b32 s2, -1
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v0, s13
+; VI-NEXT:    v_cmp_eq_u32_e64 s[4:5], s6, 2
+; VI-NEXT:    v_cndmask_b32_e64 v5, v0, 0, s[4:5]
+; VI-NEXT:    v_mov_b32_e32 v0, s12
+; VI-NEXT:    v_cndmask_b32_e64 v4, v0, 5, s[4:5]
+; VI-NEXT:    v_mov_b32_e32 v0, s11
+; VI-NEXT:    v_cmp_eq_u32_e64 s[4:5], s6, 1
+; VI-NEXT:    v_cndmask_b32_e64 v3, v0, 0, s[4:5]
+; VI-NEXT:    v_mov_b32_e32 v0, s10
+; VI-NEXT:    v_cndmask_b32_e64 v2, v0, 5, s[4:5]
+; VI-NEXT:    v_mov_b32_e32 v0, s9
+; VI-NEXT:    v_cmp_eq_u32_e64 s[4:5], s6, 0
+; VI-NEXT:    v_cndmask_b32_e64 v1, v0, 0, s[4:5]
+; VI-NEXT:    v_mov_b32_e32 v0, s8
+; VI-NEXT:    v_cndmask_b32_e64 v0, v0, 5, s[4:5]
+; VI-NEXT:    buffer_store_dwordx2 v[4:5], off, s[0:3], 0 offset:16
+; VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; VI-NEXT:    s_endpgm
   %vecins = insertelement <3 x i64> %a, i64 5, i32 %b
   store <3 x i64> %vecins, <3 x i64> addrspace(1)* %out, align 32
   ret void
 }
 
-; GCN-LABEL: {{^}}dynamic_insertelement_v4f64:
-
-; GCN-DAG: v_mov_b32_e32 [[CONST:v[0-9]+]], 0x40200000
-; GCN-DAG: v_cmp_eq_u32_e64 [[CC4:[^,]+]], [[IDX:s[0-9]+]], 3
-; GCN-DAG: v_cndmask_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, [[CONST]], [[CC4]]
-; GCN-DAG: v_cndmask_b32_e64 v{{[0-9]+}}, v{{[0-9]+}}, 0, [[CC4]]
-; GCN-DAG: v_cmp_eq_u32_e64 [[CC3:[^,]+]], [[IDX]], 2
-; GCN-DAG: v_cndmask_b32_e32 v{{[0-9]+}},  v{{[0-9]+}}, [[CONST]], [[CC3]]
-; GCN-DAG: v_cndmask_b32_e64 v{{[0-9]+}},  v{{[0-9]+}}, 0, [[CC3]]
-; GCN-DAG: v_cmp_eq_u32_e64 [[CC2:[^,]+]], [[IDX]], 1
-; GCN-DAG: v_cndmask_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, [[CONST]], [[CC2]]
-; GCN-DAG: v_cndmask_b32_e64 v{{[0-9]+}}, v{{[0-9]+}}, 0, [[CC2]]
-; GCN-DAG: v_cmp_eq_u32_e64 [[CC1:[^,]+]], [[IDX]], 0
-; GCN-DAG: v_cndmask_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, [[CONST]], [[CC1]]
-; GCN-DAG: v_cndmask_b32_e64 v{{[0-9]+}}, v{{[0-9]+}}, 0, [[CC1]]
-
-; GCN: buffer_store_dwordx4
-; GCN: buffer_store_dwordx4
-; GCN: s_endpgm
-; GCN: ScratchSize: 0
-
 define amdgpu_kernel void @dynamic_insertelement_v4f64(<4 x double> addrspace(1)* %out, <4 x double> %a, i32 %b) nounwind {
+; SI-LABEL: dynamic_insertelement_v4f64:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; SI-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x8
+; SI-NEXT:    s_load_dword s4, s[4:5], 0x10
+; SI-NEXT:    v_mov_b32_e32 v4, 0x40200000
+; SI-NEXT:    s_mov_b32 s3, 0x100f000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v0, s11
+; SI-NEXT:    v_cmp_eq_u32_e64 vcc, s4, 1
+; SI-NEXT:    v_cndmask_b32_e32 v3, v0, v4, vcc
+; SI-NEXT:    v_mov_b32_e32 v0, s10
+; SI-NEXT:    v_cndmask_b32_e64 v2, v0, 0, vcc
+; SI-NEXT:    v_mov_b32_e32 v0, s9
+; SI-NEXT:    v_cmp_eq_u32_e64 vcc, s4, 0
+; SI-NEXT:    v_cndmask_b32_e32 v1, v0, v4, vcc
+; SI-NEXT:    v_mov_b32_e32 v0, s8
+; SI-NEXT:    v_cndmask_b32_e64 v0, v0, 0, vcc
+; SI-NEXT:    v_mov_b32_e32 v5, s15
+; SI-NEXT:    v_cmp_eq_u32_e64 vcc, s4, 3
+; SI-NEXT:    v_cndmask_b32_e32 v7, v5, v4, vcc
+; SI-NEXT:    v_mov_b32_e32 v5, s14
+; SI-NEXT:    v_cndmask_b32_e64 v6, v5, 0, vcc
+; SI-NEXT:    v_mov_b32_e32 v5, s13
+; SI-NEXT:    v_cmp_eq_u32_e64 vcc, s4, 2
+; SI-NEXT:    v_cndmask_b32_e32 v5, v5, v4, vcc
+; SI-NEXT:    v_mov_b32_e32 v4, s12
+; SI-NEXT:    v_cndmask_b32_e64 v4, v4, 0, vcc
+; SI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:16
+; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: dynamic_insertelement_v4f64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; VI-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x20
+; VI-NEXT:    s_load_dword s4, s[4:5], 0x40
+; VI-NEXT:    v_mov_b32_e32 v4, 0x40200000
+; VI-NEXT:    s_mov_b32 s3, 0x1100f000
+; VI-NEXT:    s_mov_b32 s2, -1
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v0, s11
+; VI-NEXT:    v_cmp_eq_u32_e64 vcc, s4, 1
+; VI-NEXT:    v_cndmask_b32_e32 v3, v0, v4, vcc
+; VI-NEXT:    v_mov_b32_e32 v0, s10
+; VI-NEXT:    v_cndmask_b32_e64 v2, v0, 0, vcc
+; VI-NEXT:    v_mov_b32_e32 v0, s9
+; VI-NEXT:    v_cmp_eq_u32_e64 vcc, s4, 0
+; VI-NEXT:    v_cndmask_b32_e32 v1, v0, v4, vcc
+; VI-NEXT:    v_mov_b32_e32 v0, s8
+; VI-NEXT:    v_cndmask_b32_e64 v0, v0, 0, vcc
+; VI-NEXT:    v_mov_b32_e32 v5, s15
+; VI-NEXT:    v_cmp_eq_u32_e64 vcc, s4, 3
+; VI-NEXT:    v_cndmask_b32_e32 v7, v5, v4, vcc
+; VI-NEXT:    v_mov_b32_e32 v5, s14
+; VI-NEXT:    v_cndmask_b32_e64 v6, v5, 0, vcc
+; VI-NEXT:    v_mov_b32_e32 v5, s13
+; VI-NEXT:    v_cmp_eq_u32_e64 vcc, s4, 2
+; VI-NEXT:    v_cndmask_b32_e32 v5, v5, v4, vcc
+; VI-NEXT:    v_mov_b32_e32 v4, s12
+; VI-NEXT:    v_cndmask_b32_e64 v4, v4, 0, vcc
+; VI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:16
+; VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; VI-NEXT:    s_endpgm
   %vecins = insertelement <4 x double> %a, double 8.0, i32 %b
   store <4 x double> %vecins, <4 x double> addrspace(1)* %out, align 16
   ret void
 }
 
-; GCN-LABEL: {{^}}dynamic_insertelement_v8f64:
-; GCN-DAG: buffer_store_dwordx4 v{{\[[0-9]+:[0-9]+\]}}, off, s[0:3], {{s[0-9]+}} offset:64{{$}}
-; GCN-DAG: buffer_store_dwordx4 v{{\[[0-9]+:[0-9]+\]}}, off, s[0:3], {{s[0-9]+}} offset:80{{$}}
-; GCN-DAG: buffer_store_dwordx4 v{{\[[0-9]+:[0-9]+\]}}, off, s[0:3], {{s[0-9]+}} offset:96{{$}}
-; GCN-DAG: buffer_store_dwordx4 v{{\[[0-9]+:[0-9]+\]}}, off, s[0:3], {{s[0-9]+}} offset:112{{$}}
-
-; GCN: buffer_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, s[0:3], {{s[0-9]+}} offen{{$}}
-
-; GCN-DAG: buffer_load_dwordx4 v{{\[[0-9]+:[0-9]+\]}}, off, s[0:3], {{s[0-9]+}} offset:64{{$}}
-; GCN-DAG: buffer_load_dwordx4 v{{\[[0-9]+:[0-9]+\]}}, off, s[0:3], {{s[0-9]+}} offset:80{{$}}
-; GCN-DAG: buffer_load_dwordx4 v{{\[[0-9]+:[0-9]+\]}}, off, s[0:3], {{s[0-9]+}} offset:96{{$}}
-; GCN-DAG: buffer_load_dwordx4 v{{\[[0-9]+:[0-9]+\]}}, off, s[0:3], {{s[0-9]+}} offset:112{{$}}
-
-; GCN: buffer_store_dwordx4
-; GCN: buffer_store_dwordx4
-; GCN: buffer_store_dwordx4
-; GCN: buffer_store_dwordx4
-; GCN: s_endpgm
-; GCN: ScratchSize: 128
 define amdgpu_kernel void @dynamic_insertelement_v8f64(<8 x double> addrspace(1)* %out, <8 x double> %a, i32 %b) #0 {
+; SI-LABEL: dynamic_insertelement_v8f64:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
+; SI-NEXT:    s_load_dwordx16 s[12:27], s[4:5], 0x10
+; SI-NEXT:    s_load_dword s4, s[4:5], 0x20
+; SI-NEXT:    v_mov_b32_e32 v16, 64
+; SI-NEXT:    s_mov_b32 s11, 0x100f000
+; SI-NEXT:    s_mov_b32 s10, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v0, s12
+; SI-NEXT:    s_and_b32 s4, s4, 7
+; SI-NEXT:    s_lshl_b32 s4, s4, 3
+; SI-NEXT:    v_mov_b32_e32 v1, s13
+; SI-NEXT:    v_mov_b32_e32 v2, s14
+; SI-NEXT:    v_mov_b32_e32 v3, s15
+; SI-NEXT:    v_mov_b32_e32 v4, s16
+; SI-NEXT:    v_mov_b32_e32 v5, s17
+; SI-NEXT:    v_mov_b32_e32 v6, s18
+; SI-NEXT:    v_mov_b32_e32 v7, s19
+; SI-NEXT:    v_mov_b32_e32 v8, s20
+; SI-NEXT:    v_mov_b32_e32 v9, s21
+; SI-NEXT:    v_mov_b32_e32 v10, s22
+; SI-NEXT:    v_mov_b32_e32 v11, s23
+; SI-NEXT:    v_mov_b32_e32 v12, s24
+; SI-NEXT:    v_mov_b32_e32 v13, s25
+; SI-NEXT:    v_mov_b32_e32 v14, s26
+; SI-NEXT:    v_mov_b32_e32 v15, s27
+; SI-NEXT:    buffer_store_dwordx4 v[12:15], off, s[0:3], s7 offset:112
+; SI-NEXT:    buffer_store_dwordx4 v[8:11], off, s[0:3], s7 offset:96
+; SI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], s7 offset:80
+; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], s7 offset:64
+; SI-NEXT:    v_or_b32_e32 v16, s4, v16
+; SI-NEXT:    v_mov_b32_e32 v0, 0
+; SI-NEXT:    v_mov_b32_e32 v1, 0x40200000
+; SI-NEXT:    buffer_store_dwordx2 v[0:1], v16, s[0:3], s7 offen
+; SI-NEXT:    buffer_load_dwordx4 v[0:3], off, s[0:3], s7 offset:64
+; SI-NEXT:    buffer_load_dwordx4 v[4:7], off, s[0:3], s7 offset:80
+; SI-NEXT:    buffer_load_dwordx4 v[8:11], off, s[0:3], s7 offset:96
+; SI-NEXT:    buffer_load_dwordx4 v[12:15], off, s[0:3], s7 offset:112
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dwordx4 v[12:15], off, s[8:11], 0 offset:48
+; SI-NEXT:    buffer_store_dwordx4 v[8:11], off, s[8:11], 0 offset:32
+; SI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[8:11], 0 offset:16
+; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[8:11], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: dynamic_insertelement_v8f64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
+; VI-NEXT:    s_load_dwordx16 s[12:27], s[4:5], 0x40
+; VI-NEXT:    s_load_dword s4, s[4:5], 0x80
+; VI-NEXT:    v_mov_b32_e32 v16, 64
+; VI-NEXT:    s_mov_b32 s11, 0x1100f000
+; VI-NEXT:    s_mov_b32 s10, -1
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v0, s12
+; VI-NEXT:    s_and_b32 s4, s4, 7
+; VI-NEXT:    s_lshl_b32 s4, s4, 3
+; VI-NEXT:    v_mov_b32_e32 v1, s13
+; VI-NEXT:    v_mov_b32_e32 v2, s14
+; VI-NEXT:    v_mov_b32_e32 v3, s15
+; VI-NEXT:    v_mov_b32_e32 v4, s16
+; VI-NEXT:    v_mov_b32_e32 v5, s17
+; VI-NEXT:    v_mov_b32_e32 v6, s18
+; VI-NEXT:    v_mov_b32_e32 v7, s19
+; VI-NEXT:    v_mov_b32_e32 v8, s20
+; VI-NEXT:    v_mov_b32_e32 v9, s21
+; VI-NEXT:    v_mov_b32_e32 v10, s22
+; VI-NEXT:    v_mov_b32_e32 v11, s23
+; VI-NEXT:    v_mov_b32_e32 v12, s24
+; VI-NEXT:    v_mov_b32_e32 v13, s25
+; VI-NEXT:    v_mov_b32_e32 v14, s26
+; VI-NEXT:    v_mov_b32_e32 v15, s27
+; VI-NEXT:    buffer_store_dwordx4 v[12:15], off, s[0:3], s7 offset:112
+; VI-NEXT:    buffer_store_dwordx4 v[8:11], off, s[0:3], s7 offset:96
+; VI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], s7 offset:80
+; VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], s7 offset:64
+; VI-NEXT:    v_or_b32_e32 v16, s4, v16
+; VI-NEXT:    v_mov_b32_e32 v0, 0
+; VI-NEXT:    v_mov_b32_e32 v1, 0x40200000
+; VI-NEXT:    buffer_store_dwordx2 v[0:1], v16, s[0:3], s7 offen
+; VI-NEXT:    buffer_load_dwordx4 v[0:3], off, s[0:3], s7 offset:64
+; VI-NEXT:    buffer_load_dwordx4 v[4:7], off, s[0:3], s7 offset:80
+; VI-NEXT:    buffer_load_dwordx4 v[8:11], off, s[0:3], s7 offset:96
+; VI-NEXT:    buffer_load_dwordx4 v[12:15], off, s[0:3], s7 offset:112
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_store_dwordx4 v[12:15], off, s[8:11], 0 offset:48
+; VI-NEXT:    buffer_store_dwordx4 v[8:11], off, s[8:11], 0 offset:32
+; VI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[8:11], 0 offset:16
+; VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[8:11], 0
+; VI-NEXT:    s_endpgm
   %vecins = insertelement <8 x double> %a, double 8.0, i32 %b
   store <8 x double> %vecins, <8 x double> addrspace(1)* %out, align 16
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/insert_vector_elt.v2i16.ll b/llvm/test/CodeGen/AMDGPU/insert_vector_elt.v2i16.ll
index a8a298045..f31c1b5c8 100644
--- a/llvm/test/CodeGen/AMDGPU/insert_vector_elt.v2i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/insert_vector_elt.v2i16.ll
@@ -1,64 +1,152 @@
-; RUN: llc -verify-machineinstrs -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -enable-amdgpu-aa=0 -mattr=+flat-for-global,-fp64-fp16-denormals < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9,GFX89 %s
-; RUN: llc -verify-machineinstrs -mtriple=amdgcn-amd-amdhsa -mcpu=fiji -enable-amdgpu-aa=0 -mattr=+flat-for-global < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CIVI,VI,GFX89 %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -verify-machineinstrs -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -enable-amdgpu-aa=0 -mattr=+flat-for-global,-fp64-fp16-denormals < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX89,GFX9 %s
+; RUN: llc -verify-machineinstrs -mtriple=amdgcn-amd-amdhsa -mcpu=fiji -enable-amdgpu-aa=0 -mattr=+flat-for-global < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX89,CIVI,VI %s
 ; RUN: llc -verify-machineinstrs -mtriple=amdgcn-amd-amdhsa -mcpu=hawaii -enable-amdgpu-aa=0 -mattr=+flat-for-global < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CIVI,CI %s
 
-; GCN-LABEL: {{^}}s_insertelement_v2i16_0:
-; GCN: s_load_dword [[VEC:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x0
-
-; CIVI: s_and_b32 [[ELT1:s[0-9]+]], [[VEC]], 0xffff0000{{$}}
-; CIVI: s_or_b32 s{{[0-9]+}}, [[ELT1]], 0x3e7{{$}}
-
-; GFX9-NOT: lshr
-; GFX9: s_pack_lh_b32_b16 s{{[0-9]+}}, 0x3e7, [[VEC]]
 define amdgpu_kernel void @s_insertelement_v2i16_0(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(4)* %vec.ptr) #0 {
+; GFX9-LABEL: s_insertelement_v2i16_0:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_load_dword s2, s[2:3], 0x0
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_pack_lh_b32_b16 s0, 0x3e7, s2
+; GFX9-NEXT:    v_mov_b32_e32 v2, s0
+; GFX9-NEXT:    global_store_dword v[0:1], v2, off
+; GFX9-NEXT:    s_endpgm
+;
+; CIVI-LABEL: s_insertelement_v2i16_0:
+; CIVI:       ; %bb.0:
+; CIVI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; CIVI-NEXT:    s_waitcnt lgkmcnt(0)
+; CIVI-NEXT:    s_load_dword s2, s[2:3], 0x0
+; CIVI-NEXT:    v_mov_b32_e32 v0, s0
+; CIVI-NEXT:    v_mov_b32_e32 v1, s1
+; CIVI-NEXT:    s_waitcnt lgkmcnt(0)
+; CIVI-NEXT:    s_and_b32 s0, s2, 0xffff0000
+; CIVI-NEXT:    s_or_b32 s0, s0, 0x3e7
+; CIVI-NEXT:    v_mov_b32_e32 v2, s0
+; CIVI-NEXT:    flat_store_dword v[0:1], v2
+; CIVI-NEXT:    s_endpgm
   %vec = load <2 x i16>, <2 x i16> addrspace(4)* %vec.ptr
   %vecins = insertelement <2 x i16> %vec, i16 999, i32 0
   store <2 x i16> %vecins, <2 x i16> addrspace(1)* %out
   ret void
 }
 
-; GCN-LABEL: {{^}}s_insertelement_v2i16_0_reg:
-; GCN-DAG: s_load_dword [[ELT_LOAD:s[0-9]+]], s[4:5],
-; GCN-DAG: s_load_dword [[VEC:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x0
-
-; CIVI-DAG: s_and_b32 [[ELT0:s[0-9]+]], [[ELT_LOAD]], 0xffff{{$}}
-; CIVI-DAG: s_and_b32 [[ELT1:s[0-9]+]], [[VEC]], 0xffff0000{{$}}
-; CIVI: s_or_b32 s{{[0-9]+}}, [[ELT0]], [[ELT1]]
 
-; GFX9-NOT: [[ELT0]]
-; GFX9-NOT: [[VEC]]
-; GFX9: s_pack_lh_b32_b16 s{{[0-9]+}}, [[ELT_LOAD]], [[VEC]]
 define amdgpu_kernel void @s_insertelement_v2i16_0_reg(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(4)* %vec.ptr, [8 x i32], i16 %elt) #0 {
+; GFX9-LABEL: s_insertelement_v2i16_0_reg:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX9-NEXT:    s_load_dword s4, s[4:5], 0x30
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_load_dword s2, s[2:3], 0x0
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_pack_lh_b32_b16 s0, s4, s2
+; GFX9-NEXT:    v_mov_b32_e32 v2, s0
+; GFX9-NEXT:    global_store_dword v[0:1], v2, off
+; GFX9-NEXT:    s_endpgm
+;
+; VI-LABEL: s_insertelement_v2i16_0_reg:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; VI-NEXT:    s_load_dword s4, s[4:5], 0x30
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_load_dword s2, s[2:3], 0x0
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    s_and_b32 s0, s4, 0xffff
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_and_b32 s1, s2, 0xffff0000
+; VI-NEXT:    s_or_b32 s0, s0, s1
+; VI-NEXT:    v_mov_b32_e32 v2, s0
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; CI-LABEL: s_insertelement_v2i16_0_reg:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; CI-NEXT:    s_load_dword s4, s[4:5], 0xc
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    s_load_dword s2, s[2:3], 0x0
+; CI-NEXT:    v_mov_b32_e32 v0, s0
+; CI-NEXT:    v_mov_b32_e32 v1, s1
+; CI-NEXT:    s_and_b32 s1, s4, 0xffff
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    s_and_b32 s0, s2, 0xffff0000
+; CI-NEXT:    s_or_b32 s0, s1, s0
+; CI-NEXT:    v_mov_b32_e32 v2, s0
+; CI-NEXT:    flat_store_dword v[0:1], v2
+; CI-NEXT:    s_endpgm
   %vec = load <2 x i16>, <2 x i16> addrspace(4)* %vec.ptr
   %vecins = insertelement <2 x i16> %vec, i16 %elt, i32 0
   store <2 x i16> %vecins, <2 x i16> addrspace(1)* %out
   ret void
 }
 
-; GCN-LABEL: {{^}}s_insertelement_v2i16_0_multi_use_hi_reg:
-; GCN-DAG: s_load_dword [[ELT_LOAD:s[0-9]+]], s[4:5],
-; GCN-DAG: s_load_dword [[VEC:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x0
-
-; CI-DAG: s_and_b32 [[ELT0_MASKED:s[0-9]+]], [[ELT_LOAD]], 0xffff{{$}}
-; CI: s_lshr_b32 [[SHR:s[0-9]+]], [[VEC]], 16
-; CI: s_lshl_b32 [[ELT1:s[0-9]+]], [[SHR]], 16
-; CI-DAG: s_or_b32 s{{[0-9]+}}, [[ELT0_MASKED]], [[ELT1]]
-; CI-DAG: ; use [[SHR]]
-
-
-; FIXME: Should be able to void mask of upper bits
-; VI-DAG: s_and_b32 [[ELT_MASKED:s[0-9]+]], [[ELT_LOAD]], 0xffff{{$}}
-; VI-DAG: s_and_b32 [[VEC_HIMASK:s[0-9]+]], [[VEC]], 0xffff0000{{$}}
-; VI-DAG: s_or_b32 [[OR:s[0-9]+]], [[ELT_MASKED]], [[VEC_HIMASK]]
-; VI-DAG: s_lshr_b32 [[SHR:s[0-9]+]], [[VEC]], 16
-
-; VI-DAG: ; use [[SHR]]
-
-
-; GFX9: s_lshr_b32 [[ELT1:s[0-9]+]], [[VEC]], 16
-; GFX9-DAG: s_pack_ll_b32_b16 s{{[0-9]+}}, [[ELT_LOAD]], [[ELT1]]
-; GFX9-DAG: ; use [[ELT1]]
 define amdgpu_kernel void @s_insertelement_v2i16_0_multi_use_hi_reg(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(4)* %vec.ptr, [8 x i32], i16 %elt) #0 {
+; GFX9-LABEL: s_insertelement_v2i16_0_multi_use_hi_reg:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX9-NEXT:    s_load_dword s4, s[4:5], 0x30
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_load_dword s2, s[2:3], 0x0
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_lshr_b32 s0, s2, 16
+; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s4, s0
+; GFX9-NEXT:    v_mov_b32_e32 v2, s1
+; GFX9-NEXT:    global_store_dword v[0:1], v2, off
+; GFX9-NEXT:    ;;#ASMSTART
+; GFX9-NEXT:    ; use s0
+; GFX9-NEXT:    ;;#ASMEND
+; GFX9-NEXT:    s_endpgm
+;
+; VI-LABEL: s_insertelement_v2i16_0_multi_use_hi_reg:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; VI-NEXT:    s_load_dword s4, s[4:5], 0x30
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_load_dword s2, s[2:3], 0x0
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    s_and_b32 s0, s4, 0xffff
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_lshr_b32 s1, s2, 16
+; VI-NEXT:    s_and_b32 s2, s2, 0xffff0000
+; VI-NEXT:    s_or_b32 s0, s0, s2
+; VI-NEXT:    v_mov_b32_e32 v2, s0
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    ;;#ASMSTART
+; VI-NEXT:    ; use s1
+; VI-NEXT:    ;;#ASMEND
+; VI-NEXT:    s_endpgm
+;
+; CI-LABEL: s_insertelement_v2i16_0_multi_use_hi_reg:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; CI-NEXT:    s_load_dword s4, s[4:5], 0xc
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    s_load_dword s2, s[2:3], 0x0
+; CI-NEXT:    v_mov_b32_e32 v1, s1
+; CI-NEXT:    v_mov_b32_e32 v0, s0
+; CI-NEXT:    s_and_b32 s0, s4, 0xffff
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    s_lshr_b32 s1, s2, 16
+; CI-NEXT:    s_lshl_b32 s2, s1, 16
+; CI-NEXT:    s_or_b32 s0, s0, s2
+; CI-NEXT:    v_mov_b32_e32 v2, s0
+; CI-NEXT:    flat_store_dword v[0:1], v2
+; CI-NEXT:    ;;#ASMSTART
+; CI-NEXT:    ; use s1
+; CI-NEXT:    ;;#ASMEND
+; CI-NEXT:    s_endpgm
   %vec = load <2 x i16>, <2 x i16> addrspace(4)* %vec.ptr
   %elt1 = extractelement <2 x i16> %vec, i32 1
   %vecins = insertelement <2 x i16> %vec, i16 %elt, i32 0
@@ -68,18 +156,52 @@ define amdgpu_kernel void @s_insertelement_v2i16_0_multi_use_hi_reg(<2 x i16> ad
   ret void
 }
 
-; GCN-LABEL: {{^}}s_insertelement_v2i16_0_reghi:
-; GCN-DAG: s_load_dword [[ELT_ARG:s[0-9]+]], s[4:5],
-; GCN-DAG: s_load_dword [[VEC:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x0
-
-; CIVI: s_lshr_b32 [[ELT_HI:s[0-9]+]], [[ELT_ARG]], 16
-; CIVI-DAG: s_and_b32 [[ELT1:s[0-9]+]], [[VEC]], 0xffff0000{{$}}
-; CIVI: s_or_b32 s{{[0-9]+}}, [[ELT_HI]], [[ELT1]]
-
-; GFX9-NOT: [[ELT0]]
-; GFX9-NOT: [[VEC]]
-; GFX9: s_pack_hh_b32_b16 s{{[0-9]+}}, [[ELT_ARG]], [[VEC]]
 define amdgpu_kernel void @s_insertelement_v2i16_0_reghi(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(4)* %vec.ptr, [8 x i32], i32 %elt.arg) #0 {
+; GFX9-LABEL: s_insertelement_v2i16_0_reghi:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX9-NEXT:    s_load_dword s4, s[4:5], 0x30
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_load_dword s2, s[2:3], 0x0
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_pack_hh_b32_b16 s0, s4, s2
+; GFX9-NEXT:    v_mov_b32_e32 v2, s0
+; GFX9-NEXT:    global_store_dword v[0:1], v2, off
+; GFX9-NEXT:    s_endpgm
+;
+; VI-LABEL: s_insertelement_v2i16_0_reghi:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; VI-NEXT:    s_load_dword s4, s[4:5], 0x30
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_load_dword s2, s[2:3], 0x0
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    s_lshr_b32 s0, s4, 16
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_and_b32 s1, s2, 0xffff0000
+; VI-NEXT:    s_or_b32 s0, s0, s1
+; VI-NEXT:    v_mov_b32_e32 v2, s0
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; CI-LABEL: s_insertelement_v2i16_0_reghi:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; CI-NEXT:    s_load_dword s4, s[4:5], 0xc
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    s_load_dword s2, s[2:3], 0x0
+; CI-NEXT:    v_mov_b32_e32 v0, s0
+; CI-NEXT:    v_mov_b32_e32 v1, s1
+; CI-NEXT:    s_lshr_b32 s1, s4, 16
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    s_and_b32 s0, s2, 0xffff0000
+; CI-NEXT:    s_or_b32 s0, s1, s0
+; CI-NEXT:    v_mov_b32_e32 v2, s0
+; CI-NEXT:    flat_store_dword v[0:1], v2
+; CI-NEXT:    s_endpgm
   %vec = load <2 x i16>, <2 x i16> addrspace(4)* %vec.ptr
   %elt.hi = lshr i32 %elt.arg, 16
   %elt = trunc i32 %elt.hi to i16
@@ -88,18 +210,62 @@ define amdgpu_kernel void @s_insertelement_v2i16_0_reghi(<2 x i16> addrspace(1)*
   ret void
 }
 
-; GCN-LABEL: {{^}}s_insertelement_v2i16_0_reghi_multi_use_1:
-; GCN: s_load_dword [[ELT_ARG:s[0-9]+]],
-; GCN: s_load_dword [[VEC:s[0-9]+]],
-
-; CIVI-DAG: s_lshr_b32 [[ELT1:s[0-9]+]], [[ELT_ARG]], 16
-; CIVI-DAG: s_and_b32 [[ELT0:s[0-9]+]], [[VEC]], 0xffff0000{{$}}
-; CIVI: s_or_b32 s{{[0-9]+}}, [[ELT1]], [[ELT0]]
-
-; GFX9: s_lshr_b32 [[ELT1:s[0-9]+]], [[ELT_ARG]], 16
-; GFX9: s_pack_lh_b32_b16 s{{[0-9]+}}, [[ELT1]], [[VEC]]
-; GFX9: ; use [[ELT1]]
 define amdgpu_kernel void @s_insertelement_v2i16_0_reghi_multi_use_1(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(4)* %vec.ptr, i32 %elt.arg) #0 {
+; GFX9-LABEL: s_insertelement_v2i16_0_reghi_multi_use_1:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX9-NEXT:    s_load_dword s4, s[4:5], 0x10
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_load_dword s2, s[2:3], 0x0
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    s_lshr_b32 s0, s4, 16
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_pack_lh_b32_b16 s1, s0, s2
+; GFX9-NEXT:    v_mov_b32_e32 v2, s1
+; GFX9-NEXT:    global_store_dword v[0:1], v2, off
+; GFX9-NEXT:    ;;#ASMSTART
+; GFX9-NEXT:    ; use s0
+; GFX9-NEXT:    ;;#ASMEND
+; GFX9-NEXT:    s_endpgm
+;
+; VI-LABEL: s_insertelement_v2i16_0_reghi_multi_use_1:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; VI-NEXT:    s_load_dword s4, s[4:5], 0x10
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_load_dword s2, s[2:3], 0x0
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    s_lshr_b32 s0, s4, 16
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_and_b32 s1, s2, 0xffff0000
+; VI-NEXT:    s_or_b32 s1, s0, s1
+; VI-NEXT:    v_mov_b32_e32 v2, s1
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    ;;#ASMSTART
+; VI-NEXT:    ; use s0
+; VI-NEXT:    ;;#ASMEND
+; VI-NEXT:    s_endpgm
+;
+; CI-LABEL: s_insertelement_v2i16_0_reghi_multi_use_1:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; CI-NEXT:    s_load_dword s4, s[4:5], 0x4
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    s_load_dword s2, s[2:3], 0x0
+; CI-NEXT:    v_mov_b32_e32 v0, s0
+; CI-NEXT:    v_mov_b32_e32 v1, s1
+; CI-NEXT:    s_lshr_b32 s0, s4, 16
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    s_and_b32 s1, s2, 0xffff0000
+; CI-NEXT:    s_or_b32 s1, s0, s1
+; CI-NEXT:    v_mov_b32_e32 v2, s1
+; CI-NEXT:    flat_store_dword v[0:1], v2
+; CI-NEXT:    ;;#ASMSTART
+; CI-NEXT:    ; use s0
+; CI-NEXT:    ;;#ASMEND
+; CI-NEXT:    s_endpgm
   %vec = load <2 x i16>, <2 x i16> addrspace(4)* %vec.ptr
   %elt.hi = lshr i32 %elt.arg, 16
   %elt = trunc i32 %elt.hi to i16
@@ -110,27 +276,74 @@ define amdgpu_kernel void @s_insertelement_v2i16_0_reghi_multi_use_1(<2 x i16> a
   ret void
 }
 
-; GCN-LABEL: {{^}}s_insertelement_v2i16_0_reghi_both_multi_use_1:
-; GCN: s_load_dword [[ELT_ARG:s[0-9]+]],
-; GCN: s_load_dword [[VEC:s[0-9]+]],
-
-; CI-DAG: s_lshr_b32 [[ELT_HI:s[0-9]+]], [[ELT_ARG]], 16
-; CI-DAG: s_lshr_b32 [[SHR:s[0-9]+]], [[VEC]], 16
-; CI-DAG: s_lshl_b32 [[VEC_HI:s[0-9]+]], [[SHR]], 16
-; CI: s_or_b32 s{{[0-9]+}}, [[ELT_HI]], [[VEC_HI]]
-
-
-; VI-DAG: s_lshr_b32 [[ELT_HI:s[0-9]+]], [[ELT_ARG]], 16
-; VI-DAG: s_lshr_b32 [[VEC_HI:s[0-9]+]], [[VEC]], 16
-; VI: s_and_b32 [[MASK_HI:s[0-9]+]], [[VEC]], 0xffff0000
-; VI: s_or_b32 s{{[0-9]+}}, [[ELT_HI]], [[MASK_HI]]
-
-; GFX9-DAG: s_lshr_b32 [[ELT_HI:s[0-9]+]], [[ELT_ARG]], 16
-; GFX9-DAG: s_lshr_b32 [[VEC_HI:s[0-9]+]], [[VEC]], 16
-; GFX9: s_pack_ll_b32_b16 s{{[0-9]+}}, [[ELT_HI]], [[VEC_HI]]
-; GFX9: ; use [[ELT_HI]]
-; GFX9: ; use [[VEC_HI]]
 define amdgpu_kernel void @s_insertelement_v2i16_0_reghi_both_multi_use_1(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(4)* %vec.ptr, i32 %elt.arg) #0 {
+; GFX9-LABEL: s_insertelement_v2i16_0_reghi_both_multi_use_1:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX9-NEXT:    s_load_dword s4, s[4:5], 0x10
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_load_dword s2, s[2:3], 0x0
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    s_lshr_b32 s0, s4, 16
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_lshr_b32 s1, s2, 16
+; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s0, s1
+; GFX9-NEXT:    v_mov_b32_e32 v2, s2
+; GFX9-NEXT:    global_store_dword v[0:1], v2, off
+; GFX9-NEXT:    ;;#ASMSTART
+; GFX9-NEXT:    ; use s0
+; GFX9-NEXT:    ;;#ASMEND
+; GFX9-NEXT:    ;;#ASMSTART
+; GFX9-NEXT:    ; use s1
+; GFX9-NEXT:    ;;#ASMEND
+; GFX9-NEXT:    s_endpgm
+;
+; VI-LABEL: s_insertelement_v2i16_0_reghi_both_multi_use_1:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; VI-NEXT:    s_load_dword s4, s[4:5], 0x10
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_load_dword s2, s[2:3], 0x0
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    s_lshr_b32 s0, s4, 16
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_lshr_b32 s1, s2, 16
+; VI-NEXT:    s_and_b32 s2, s2, 0xffff0000
+; VI-NEXT:    s_or_b32 s2, s0, s2
+; VI-NEXT:    v_mov_b32_e32 v2, s2
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    ;;#ASMSTART
+; VI-NEXT:    ; use s0
+; VI-NEXT:    ;;#ASMEND
+; VI-NEXT:    ;;#ASMSTART
+; VI-NEXT:    ; use s1
+; VI-NEXT:    ;;#ASMEND
+; VI-NEXT:    s_endpgm
+;
+; CI-LABEL: s_insertelement_v2i16_0_reghi_both_multi_use_1:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; CI-NEXT:    s_load_dword s4, s[4:5], 0x4
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    s_load_dword s2, s[2:3], 0x0
+; CI-NEXT:    v_mov_b32_e32 v1, s1
+; CI-NEXT:    v_mov_b32_e32 v0, s0
+; CI-NEXT:    s_lshr_b32 s0, s4, 16
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    s_lshr_b32 s1, s2, 16
+; CI-NEXT:    s_lshl_b32 s2, s1, 16
+; CI-NEXT:    s_or_b32 s2, s0, s2
+; CI-NEXT:    v_mov_b32_e32 v2, s2
+; CI-NEXT:    flat_store_dword v[0:1], v2
+; CI-NEXT:    ;;#ASMSTART
+; CI-NEXT:    ; use s0
+; CI-NEXT:    ;;#ASMEND
+; CI-NEXT:    ;;#ASMSTART
+; CI-NEXT:    ; use s1
+; CI-NEXT:    ;;#ASMEND
+; CI-NEXT:    s_endpgm
   %vec = load <2 x i16>, <2 x i16> addrspace(4)* %vec.ptr
   %elt.hi = lshr i32 %elt.arg, 16
   %elt = trunc i32 %elt.hi to i16
@@ -145,78 +358,213 @@ define amdgpu_kernel void @s_insertelement_v2i16_0_reghi_both_multi_use_1(<2 x i
   ret void
 }
 
-; GCN-LABEL: {{^}}s_insertelement_v2i16_1:
-; GCN: s_load_dword [[VEC:s[0-9]+]]
-
-; GCN-NOT: s_lshr
-
-; CIVI: s_and_b32 [[ELT0:s[0-9]+]], [[VEC]], 0xffff{{$}}
-; CIVI: s_or_b32 [[INS:s[0-9]+]], [[ELT0]], 0x3e70000
-
-; GFX9: s_pack_ll_b32_b16 s{{[0-9]+}}, [[VEC]], 0x3e7
 define amdgpu_kernel void @s_insertelement_v2i16_1(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(4)* %vec.ptr) #0 {
+; GFX9-LABEL: s_insertelement_v2i16_1:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_load_dword s2, s[2:3], 0x0
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s2, 0x3e7
+; GFX9-NEXT:    v_mov_b32_e32 v2, s0
+; GFX9-NEXT:    global_store_dword v[0:1], v2, off
+; GFX9-NEXT:    s_endpgm
+;
+; CIVI-LABEL: s_insertelement_v2i16_1:
+; CIVI:       ; %bb.0:
+; CIVI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; CIVI-NEXT:    s_waitcnt lgkmcnt(0)
+; CIVI-NEXT:    s_load_dword s2, s[2:3], 0x0
+; CIVI-NEXT:    v_mov_b32_e32 v0, s0
+; CIVI-NEXT:    v_mov_b32_e32 v1, s1
+; CIVI-NEXT:    s_waitcnt lgkmcnt(0)
+; CIVI-NEXT:    s_and_b32 s0, s2, 0xffff
+; CIVI-NEXT:    s_or_b32 s0, s0, 0x3e70000
+; CIVI-NEXT:    v_mov_b32_e32 v2, s0
+; CIVI-NEXT:    flat_store_dword v[0:1], v2
+; CIVI-NEXT:    s_endpgm
   %vec = load <2 x i16>, <2 x i16> addrspace(4)* %vec.ptr
   %vecins = insertelement <2 x i16> %vec, i16 999, i32 1
   store <2 x i16> %vecins, <2 x i16> addrspace(1)* %out
   ret void
 }
 
-; GCN-LABEL: {{^}}s_insertelement_v2i16_1_reg:
-; GCN-DAG: s_load_dword [[ELT1_LOAD:s[0-9]+]], s[4:5],
-; GCN-DAG: s_load_dword [[VEC:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x0
-
-; CIVI: s_lshl_b32 [[ELT1:s[0-9]+]], [[ELT1_LOAD]], 16
-; CIVI: s_and_b32 [[ELT0:s[0-9]+]], [[VEC]], 0xffff{{$}}
-; CIVI: s_or_b32 s{{[0-9]+}}, [[ELT0]], [[ELT1]]
-
-; GCN-NOT: shlr
-; GFX9: s_pack_ll_b32_b16 s{{[0-9]+}}, [[VEC]], [[ELT1_LOAD]]
 define amdgpu_kernel void @s_insertelement_v2i16_1_reg(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(4)* %vec.ptr, [8 x i32], i16 %elt) #0 {
+; GFX9-LABEL: s_insertelement_v2i16_1_reg:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX9-NEXT:    s_load_dword s4, s[4:5], 0x30
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_load_dword s2, s[2:3], 0x0
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s2, s4
+; GFX9-NEXT:    v_mov_b32_e32 v2, s0
+; GFX9-NEXT:    global_store_dword v[0:1], v2, off
+; GFX9-NEXT:    s_endpgm
+;
+; VI-LABEL: s_insertelement_v2i16_1_reg:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; VI-NEXT:    s_load_dword s4, s[4:5], 0x30
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_load_dword s2, s[2:3], 0x0
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    s_lshl_b32 s0, s4, 16
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_and_b32 s1, s2, 0xffff
+; VI-NEXT:    s_or_b32 s0, s1, s0
+; VI-NEXT:    v_mov_b32_e32 v2, s0
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; CI-LABEL: s_insertelement_v2i16_1_reg:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; CI-NEXT:    s_load_dword s4, s[4:5], 0xc
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    s_load_dword s2, s[2:3], 0x0
+; CI-NEXT:    v_mov_b32_e32 v0, s0
+; CI-NEXT:    v_mov_b32_e32 v1, s1
+; CI-NEXT:    s_lshl_b32 s1, s4, 16
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    s_and_b32 s0, s2, 0xffff
+; CI-NEXT:    s_or_b32 s0, s0, s1
+; CI-NEXT:    v_mov_b32_e32 v2, s0
+; CI-NEXT:    flat_store_dword v[0:1], v2
+; CI-NEXT:    s_endpgm
   %vec = load <2 x i16>, <2 x i16> addrspace(4)* %vec.ptr
   %vecins = insertelement <2 x i16> %vec, i16 %elt, i32 1
   store <2 x i16> %vecins, <2 x i16> addrspace(1)* %out
   ret void
 }
 
-; GCN-LABEL: {{^}}s_insertelement_v2f16_0:
-; GCN: s_load_dword [[VEC:s[0-9]+]]
-; CIVI: s_and_b32 [[ELT1:s[0-9]+]], [[VEC:s[0-9]+]], 0xffff0000
-; CIVI: s_or_b32 s{{[0-9]+}}, [[ELT1]], 0x4500
-
-; GFX9: s_lshr_b32 [[ELT1:s[0-9]+]], [[VEC]], 16
-; GFX9: s_pack_ll_b32_b16 s{{[0-9]+}}, 0x4500, [[ELT1]]
 define amdgpu_kernel void @s_insertelement_v2f16_0(<2 x half> addrspace(1)* %out, <2 x half> addrspace(4)* %vec.ptr) #0 {
+; GFX9-LABEL: s_insertelement_v2f16_0:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_load_dword s2, s[2:3], 0x0
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_lshr_b32 s0, s2, 16
+; GFX9-NEXT:    s_pack_ll_b32_b16 s0, 0x4500, s0
+; GFX9-NEXT:    v_mov_b32_e32 v2, s0
+; GFX9-NEXT:    global_store_dword v[0:1], v2, off
+; GFX9-NEXT:    s_endpgm
+;
+; CIVI-LABEL: s_insertelement_v2f16_0:
+; CIVI:       ; %bb.0:
+; CIVI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; CIVI-NEXT:    s_waitcnt lgkmcnt(0)
+; CIVI-NEXT:    s_load_dword s2, s[2:3], 0x0
+; CIVI-NEXT:    v_mov_b32_e32 v0, s0
+; CIVI-NEXT:    v_mov_b32_e32 v1, s1
+; CIVI-NEXT:    s_waitcnt lgkmcnt(0)
+; CIVI-NEXT:    s_and_b32 s0, s2, 0xffff0000
+; CIVI-NEXT:    s_or_b32 s0, s0, 0x4500
+; CIVI-NEXT:    v_mov_b32_e32 v2, s0
+; CIVI-NEXT:    flat_store_dword v[0:1], v2
+; CIVI-NEXT:    s_endpgm
   %vec = load <2 x half>, <2 x half> addrspace(4)* %vec.ptr
   %vecins = insertelement <2 x half> %vec, half 5.000000e+00, i32 0
   store <2 x half> %vecins, <2 x half> addrspace(1)* %out
   ret void
 }
 
-; GCN-LABEL: {{^}}s_insertelement_v2f16_1:
-; GCN: s_load_dword [[VEC:s[0-9]+]]
-; GCN-NOT: s_lshr
-
-; CIVI: s_and_b32 [[ELT0:s[0-9]+]], [[VEC]], 0xffff{{$}}
-; CIVI: s_or_b32 [[INS:s[0-9]+]], [[ELT0]], 0x45000000
-
-; GFX9: s_pack_ll_b32_b16 s{{[0-9]+}}, [[VEC]], 0x4500
 define amdgpu_kernel void @s_insertelement_v2f16_1(<2 x half> addrspace(1)* %out, <2 x half> addrspace(4)* %vec.ptr) #0 {
+; GFX9-LABEL: s_insertelement_v2f16_1:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_load_dword s2, s[2:3], 0x0
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s2, 0x4500
+; GFX9-NEXT:    v_mov_b32_e32 v2, s0
+; GFX9-NEXT:    global_store_dword v[0:1], v2, off
+; GFX9-NEXT:    s_endpgm
+;
+; CIVI-LABEL: s_insertelement_v2f16_1:
+; CIVI:       ; %bb.0:
+; CIVI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; CIVI-NEXT:    s_waitcnt lgkmcnt(0)
+; CIVI-NEXT:    s_load_dword s2, s[2:3], 0x0
+; CIVI-NEXT:    v_mov_b32_e32 v0, s0
+; CIVI-NEXT:    v_mov_b32_e32 v1, s1
+; CIVI-NEXT:    s_waitcnt lgkmcnt(0)
+; CIVI-NEXT:    s_and_b32 s0, s2, 0xffff
+; CIVI-NEXT:    s_or_b32 s0, s0, 0x45000000
+; CIVI-NEXT:    v_mov_b32_e32 v2, s0
+; CIVI-NEXT:    flat_store_dword v[0:1], v2
+; CIVI-NEXT:    s_endpgm
   %vec = load <2 x half>, <2 x half> addrspace(4)* %vec.ptr
   %vecins = insertelement <2 x half> %vec, half 5.000000e+00, i32 1
   store <2 x half> %vecins, <2 x half> addrspace(1)* %out
   ret void
 }
 
-; GCN-LABEL: {{^}}v_insertelement_v2i16_0:
-; GCN-DAG: {{flat|global}}_load_dword [[VEC:v[0-9]+]]
-; CIVI: v_and_b32_e32 [[ELT1:v[0-9]+]], 0xffff0000, [[VEC]]
-; CIVI: v_or_b32_e32 [[RES:v[0-9]+]], 0x3e7, [[ELT1]]
-
-; GFX9-DAG: s_movk_i32 [[ELT0:s[0-9]+]], 0x3e7{{$}}
-; GFX9-DAG: v_mov_b32_e32 [[MASK:v[0-9]+]], 0xffff{{$}}
-; GFX9: v_bfi_b32 [[RES:v[0-9]+]], [[MASK]], [[ELT0]], [[VEC]]
-; GCN: {{flat|global}}_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RES]]
 define amdgpu_kernel void @v_insertelement_v2i16_0(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %in) #0 {
+; GFX9-LABEL: v_insertelement_v2i16_0:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX9-NEXT:    s_movk_i32 s4, 0x3e7
+; GFX9-NEXT:    v_mov_b32_e32 v3, 0xffff
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s2, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    global_load_dword v4, v[0:1], off
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s0, v2
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_bfi_b32 v2, v3, s4, v4
+; GFX9-NEXT:    global_store_dword v[0:1], v2, off
+; GFX9-NEXT:    s_endpgm
+;
+; VI-LABEL: v_insertelement_v2i16_0:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; VI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dword v3, v[0:1]
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v2
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_and_b32_e32 v2, 0xffff0000, v3
+; VI-NEXT:    v_or_b32_e32 v2, 0x3e7, v2
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; CI-LABEL: v_insertelement_v2i16_0:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; CI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    v_mov_b32_e32 v1, s3
+; CI-NEXT:    v_add_i32_e32 v0, vcc, s2, v2
+; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; CI-NEXT:    flat_load_dword v3, v[0:1]
+; CI-NEXT:    v_add_i32_e32 v0, vcc, s0, v2
+; CI-NEXT:    v_mov_b32_e32 v1, s1
+; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; CI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CI-NEXT:    v_and_b32_e32 v2, 0xffff0000, v3
+; CI-NEXT:    v_or_b32_e32 v2, 0x3e7, v2
+; CI-NEXT:    flat_store_dword v[0:1], v2
+; CI-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x() #1
   %tid.ext = sext i32 %tid to i64
   %in.gep = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %in, i64 %tid.ext
@@ -227,20 +575,66 @@ define amdgpu_kernel void @v_insertelement_v2i16_0(<2 x i16> addrspace(1)* %out,
   ret void
 }
 
-; GCN-LABEL: {{^}}v_insertelement_v2i16_0_reghi:
-; GCN-DAG: {{flat|global}}_load_dword [[VEC:v[0-9]+]]
-; GCN-DAG: s_load_dword [[ELT0:s[0-9]+]]
-
-; CIVI-DAG: s_lshr_b32 [[ELT0_SHIFT:s[0-9]+]], [[ELT0]], 16
-; CIVI-DAG: v_and_b32_e32 [[ELT1:v[0-9]+]], 0xffff0000, [[VEC]]
-; CIVI: v_or_b32_e32 [[RES:v[0-9]+]], [[ELT0_SHIFT]], [[ELT1]]
-
-; GFX9-DAG: v_mov_b32_e32 [[MASK:v[0-9]+]], 0xffff0000{{$}}
-; GFX9-DAG: v_lshrrev_b32_e64 [[ELT0_SHIFT:v[0-9]+]], 16, [[ELT0]]
-; GFX9: v_and_or_b32 [[RES:v[0-9]+]], [[VEC]], [[MASK]], [[ELT0_SHIFT]]
-
-; GCN: {{flat|global}}_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RES]]
 define amdgpu_kernel void @v_insertelement_v2i16_0_reghi(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %in, i32 %elt.arg) #0 {
+; GFX9-LABEL: v_insertelement_v2i16_0_reghi:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX9-NEXT:    s_load_dword s4, s[4:5], 0x10
+; GFX9-NEXT:    v_mov_b32_e32 v3, 0xffff0000
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s2, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    global_load_dword v4, v[0:1], off
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s0, v2
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_lshrrev_b32_e64 v2, 16, s4
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_and_or_b32 v2, v4, v3, v2
+; GFX9-NEXT:    global_store_dword v[0:1], v2, off
+; GFX9-NEXT:    s_endpgm
+;
+; VI-LABEL: v_insertelement_v2i16_0_reghi:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; VI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; VI-NEXT:    s_load_dword s4, s[4:5], 0x10
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dword v3, v[0:1]
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v2
+; VI-NEXT:    s_lshr_b32 s1, s4, 16
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_and_b32_e32 v2, 0xffff0000, v3
+; VI-NEXT:    v_or_b32_e32 v2, s1, v2
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; CI-LABEL: v_insertelement_v2i16_0_reghi:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; CI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; CI-NEXT:    s_load_dword s4, s[4:5], 0x4
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    v_mov_b32_e32 v1, s3
+; CI-NEXT:    v_add_i32_e32 v0, vcc, s2, v2
+; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; CI-NEXT:    flat_load_dword v3, v[0:1]
+; CI-NEXT:    v_mov_b32_e32 v1, s1
+; CI-NEXT:    v_add_i32_e32 v0, vcc, s0, v2
+; CI-NEXT:    s_lshr_b32 s1, s4, 16
+; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; CI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CI-NEXT:    v_and_b32_e32 v2, 0xffff0000, v3
+; CI-NEXT:    v_or_b32_e32 v2, s1, v2
+; CI-NEXT:    flat_store_dword v[0:1], v2
+; CI-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x() #1
   %tid.ext = sext i32 %tid to i64
   %in.gep = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %in, i64 %tid.ext
@@ -253,17 +647,60 @@ define amdgpu_kernel void @v_insertelement_v2i16_0_reghi(<2 x i16> addrspace(1)*
   ret void
 }
 
-; GCN-LABEL: {{^}}v_insertelement_v2i16_0_inlineimm:
-; GCN-DAG: {{flat|global}}_load_dword [[VEC:v[0-9]+]]
-
-; CIVI: v_and_b32_e32 [[ELT1:v[0-9]+]], 0xffff0000, [[VEC]]
-; CIVI: v_or_b32_e32 [[RES:v[0-9]+]], 53, [[ELT1]]
-
-; GFX9-DAG: v_mov_b32_e32 [[MASK:v[0-9]+]], 0xffff{{$}}
-; GFX9: v_bfi_b32 [[RES:v[0-9]+]], [[MASK]], 53, [[VEC]]
-
-; GCN: {{flat|global}}_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RES]]
 define amdgpu_kernel void @v_insertelement_v2i16_0_inlineimm(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %in) #0 {
+; GFX9-LABEL: v_insertelement_v2i16_0_inlineimm:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX9-NEXT:    v_mov_b32_e32 v3, 0xffff
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s2, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    global_load_dword v4, v[0:1], off
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s0, v2
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_bfi_b32 v2, v3, 53, v4
+; GFX9-NEXT:    global_store_dword v[0:1], v2, off
+; GFX9-NEXT:    s_endpgm
+;
+; VI-LABEL: v_insertelement_v2i16_0_inlineimm:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; VI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dword v3, v[0:1]
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v2
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_and_b32_e32 v2, 0xffff0000, v3
+; VI-NEXT:    v_or_b32_e32 v2, 53, v2
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; CI-LABEL: v_insertelement_v2i16_0_inlineimm:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; CI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    v_mov_b32_e32 v1, s3
+; CI-NEXT:    v_add_i32_e32 v0, vcc, s2, v2
+; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; CI-NEXT:    flat_load_dword v3, v[0:1]
+; CI-NEXT:    v_add_i32_e32 v0, vcc, s0, v2
+; CI-NEXT:    v_mov_b32_e32 v1, s1
+; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; CI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CI-NEXT:    v_and_b32_e32 v2, 0xffff0000, v3
+; CI-NEXT:    v_or_b32_e32 v2, 53, v2
+; CI-NEXT:    flat_store_dword v[0:1], v2
+; CI-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x() #1
   %tid.ext = sext i32 %tid to i64
   %in.gep = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %in, i64 %tid.ext
@@ -275,21 +712,61 @@ define amdgpu_kernel void @v_insertelement_v2i16_0_inlineimm(<2 x i16> addrspace
 }
 
 ; FIXME: fold lshl_or c0, c1, v0 -> or (c0 << c1), v0
-
-; GCN-LABEL: {{^}}v_insertelement_v2i16_1:
-; VI: v_mov_b32_e32 [[K:v[0-9]+]], 0x3e70000
-; GCN-DAG: {{flat|global}}_load_dword [[VEC:v[0-9]+]]
-
-; GFX9-DAG: s_movk_i32 [[K:s[0-9]+]], 0x3e7
-; GFX9-DAG: v_and_b32_e32 [[ELT0:v[0-9]+]], 0xffff, [[VEC]]
-; GFX9: v_lshl_or_b32 [[RES:v[0-9]+]], [[K]], 16, [[ELT0]]
-
-; CI: v_and_b32_e32 [[AND:v[0-9]+]], 0xffff, [[VEC]]
-; CI: v_or_b32_e32 [[RES:v[0-9]+]], 0x3e70000, [[AND]]
-; VI: v_or_b32_sdwa [[RES:v[0-9]+]], [[K]], [[VEC]] dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-
-; GCN: {{flat|global}}_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RES]]
 define amdgpu_kernel void @v_insertelement_v2i16_1(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %in) #0 {
+; GFX9-LABEL: v_insertelement_v2i16_1:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX9-NEXT:    s_movk_i32 s4, 0x3e7
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s2, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    global_load_dword v3, v[0:1], off
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s0, v2
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v2, 0xffff, v3
+; GFX9-NEXT:    v_lshl_or_b32 v2, s4, 16, v2
+; GFX9-NEXT:    global_store_dword v[0:1], v2, off
+; GFX9-NEXT:    s_endpgm
+;
+; VI-LABEL: v_insertelement_v2i16_1:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; VI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; VI-NEXT:    v_mov_b32_e32 v3, 0x3e70000
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dword v4, v[0:1]
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v2
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_or_b32_sdwa v2, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; CI-LABEL: v_insertelement_v2i16_1:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; CI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    v_mov_b32_e32 v1, s3
+; CI-NEXT:    v_add_i32_e32 v0, vcc, s2, v2
+; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; CI-NEXT:    flat_load_dword v3, v[0:1]
+; CI-NEXT:    v_add_i32_e32 v0, vcc, s0, v2
+; CI-NEXT:    v_mov_b32_e32 v1, s1
+; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; CI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CI-NEXT:    v_and_b32_e32 v2, 0xffff, v3
+; CI-NEXT:    v_or_b32_e32 v2, 0x3e70000, v2
+; CI-NEXT:    flat_store_dword v[0:1], v2
+; CI-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x() #1
   %tid.ext = sext i32 %tid to i64
   %in.gep = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %in, i64 %tid.ext
@@ -300,16 +777,60 @@ define amdgpu_kernel void @v_insertelement_v2i16_1(<2 x i16> addrspace(1)* %out,
   ret void
 }
 
-; GCN-LABEL: {{^}}v_insertelement_v2i16_1_inlineimm:
-; VI: v_mov_b32_e32 [[K:v[0-9]+]], 0xfff10000
-; GCN: {{flat|global}}_load_dword [[VEC:v[0-9]+]]
-; CI:   v_and_b32_e32 [[ELT0:v[0-9]+]], 0xffff, [[VEC]]
-; GFX9: v_and_b32_e32 [[ELT0:v[0-9]+]], 0xffff, [[VEC]]
-; CI: v_or_b32_e32 [[RES:v[0-9]+]], 0xfff10000, [[ELT0]]
-; VI: v_or_b32_sdwa [[RES:v[0-9]+]], [[K]], [[VEC]] dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-; GFX9: v_lshl_or_b32 [[RES:v[0-9]+]], -15, 16, [[ELT0]]
-; GCN: {{flat|global}}_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RES]]
 define amdgpu_kernel void @v_insertelement_v2i16_1_inlineimm(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %in) #0 {
+; GFX9-LABEL: v_insertelement_v2i16_1_inlineimm:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s2, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    global_load_dword v3, v[0:1], off
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s0, v2
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v2, 0xffff, v3
+; GFX9-NEXT:    v_lshl_or_b32 v2, -15, 16, v2
+; GFX9-NEXT:    global_store_dword v[0:1], v2, off
+; GFX9-NEXT:    s_endpgm
+;
+; VI-LABEL: v_insertelement_v2i16_1_inlineimm:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; VI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; VI-NEXT:    v_mov_b32_e32 v3, 0xfff10000
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dword v4, v[0:1]
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v2
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_or_b32_sdwa v2, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; CI-LABEL: v_insertelement_v2i16_1_inlineimm:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; CI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    v_mov_b32_e32 v1, s3
+; CI-NEXT:    v_add_i32_e32 v0, vcc, s2, v2
+; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; CI-NEXT:    flat_load_dword v3, v[0:1]
+; CI-NEXT:    v_add_i32_e32 v0, vcc, s0, v2
+; CI-NEXT:    v_mov_b32_e32 v1, s1
+; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; CI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CI-NEXT:    v_and_b32_e32 v2, 0xffff, v3
+; CI-NEXT:    v_or_b32_e32 v2, 0xfff10000, v2
+; CI-NEXT:    flat_store_dword v[0:1], v2
+; CI-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x() #1
   %tid.ext = sext i32 %tid to i64
   %in.gep = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %in, i64 %tid.ext
@@ -320,18 +841,61 @@ define amdgpu_kernel void @v_insertelement_v2i16_1_inlineimm(<2 x i16> addrspace
   ret void
 }
 
-; GCN-LABEL: {{^}}v_insertelement_v2f16_0:
-; GCN-DAG: {{flat|global}}_load_dword [[VEC:v[0-9]+]]
-
-; CIVI: v_and_b32_e32 [[ELT1:v[0-9]+]], 0xffff0000, [[VEC]]
-; CIVI: v_or_b32_e32 [[RES:v[0-9]+]], 0x4500, [[ELT1]]
-
-; GFX9-DAG: v_mov_b32_e32 [[ELT0:v[0-9]+]], 0x4500{{$}}
-; GFX9-DAG: v_lshrrev_b32_e32 [[ELT1:v[0-9]+]], 16, [[VEC]]
-; GFX9: v_lshl_or_b32 [[RES:v[0-9]+]], [[ELT1]], 16, [[ELT0]]
-
-; GCN: {{flat|global}}_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RES]]
 define amdgpu_kernel void @v_insertelement_v2f16_0(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %in) #0 {
+; GFX9-LABEL: v_insertelement_v2f16_0:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX9-NEXT:    v_mov_b32_e32 v3, 0x4500
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s2, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    global_load_dword v4, v[0:1], off
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s0, v2
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_lshrrev_b32_e32 v2, 16, v4
+; GFX9-NEXT:    v_lshl_or_b32 v2, v2, 16, v3
+; GFX9-NEXT:    global_store_dword v[0:1], v2, off
+; GFX9-NEXT:    s_endpgm
+;
+; VI-LABEL: v_insertelement_v2f16_0:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; VI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dword v3, v[0:1]
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v2
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_and_b32_e32 v2, 0xffff0000, v3
+; VI-NEXT:    v_or_b32_e32 v2, 0x4500, v2
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; CI-LABEL: v_insertelement_v2f16_0:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; CI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    v_mov_b32_e32 v1, s3
+; CI-NEXT:    v_add_i32_e32 v0, vcc, s2, v2
+; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; CI-NEXT:    flat_load_dword v3, v[0:1]
+; CI-NEXT:    v_add_i32_e32 v0, vcc, s0, v2
+; CI-NEXT:    v_mov_b32_e32 v1, s1
+; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; CI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CI-NEXT:    v_and_b32_e32 v2, 0xffff0000, v3
+; CI-NEXT:    v_or_b32_e32 v2, 0x4500, v2
+; CI-NEXT:    flat_store_dword v[0:1], v2
+; CI-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x() #1
   %tid.ext = sext i32 %tid to i64
   %in.gep = getelementptr inbounds <2 x half>, <2 x half> addrspace(1)* %in, i64 %tid.ext
@@ -342,16 +906,60 @@ define amdgpu_kernel void @v_insertelement_v2f16_0(<2 x half> addrspace(1)* %out
   ret void
 }
 
-; GCN-LABEL: {{^}}v_insertelement_v2f16_0_inlineimm:
-; GCN: {{flat|global}}_load_dword [[VEC:v[0-9]+]]
-
-; CIVI: v_and_b32_e32 [[ELT1:v[0-9]+]], 0xffff0000, [[VEC]]
-; CIVI: v_or_b32_e32 [[RES:v[0-9]+]], 53, [[ELT1]]
-
-; GFX9: v_lshrrev_b32_e32 [[ELT1:v[0-9]+]], 16, [[VEC]]
-; GFX9: v_lshl_or_b32 [[RES:v[0-9]+]], [[ELT1]], 16, 53
-; GCN: {{flat|global}}_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RES]]
 define amdgpu_kernel void @v_insertelement_v2f16_0_inlineimm(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %in) #0 {
+; GFX9-LABEL: v_insertelement_v2f16_0_inlineimm:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s2, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    global_load_dword v3, v[0:1], off
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s0, v2
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_lshrrev_b32_e32 v2, 16, v3
+; GFX9-NEXT:    v_lshl_or_b32 v2, v2, 16, 53
+; GFX9-NEXT:    global_store_dword v[0:1], v2, off
+; GFX9-NEXT:    s_endpgm
+;
+; VI-LABEL: v_insertelement_v2f16_0_inlineimm:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; VI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dword v3, v[0:1]
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v2
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_and_b32_e32 v2, 0xffff0000, v3
+; VI-NEXT:    v_or_b32_e32 v2, 53, v2
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; CI-LABEL: v_insertelement_v2f16_0_inlineimm:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; CI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    v_mov_b32_e32 v1, s3
+; CI-NEXT:    v_add_i32_e32 v0, vcc, s2, v2
+; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; CI-NEXT:    flat_load_dword v3, v[0:1]
+; CI-NEXT:    v_add_i32_e32 v0, vcc, s0, v2
+; CI-NEXT:    v_mov_b32_e32 v1, s1
+; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; CI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CI-NEXT:    v_and_b32_e32 v2, 0xffff0000, v3
+; CI-NEXT:    v_or_b32_e32 v2, 53, v2
+; CI-NEXT:    flat_store_dword v[0:1], v2
+; CI-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x() #1
   %tid.ext = sext i32 %tid to i64
   %in.gep = getelementptr inbounds <2 x half>, <2 x half> addrspace(1)* %in, i64 %tid.ext
@@ -362,21 +970,61 @@ define amdgpu_kernel void @v_insertelement_v2f16_0_inlineimm(<2 x half> addrspac
   ret void
 }
 
-; GCN-LABEL: {{^}}v_insertelement_v2f16_1:
-; VI: v_mov_b32_e32 [[K:v[0-9]+]], 0x45000000
-; GCN-DAG: {{flat|global}}_load_dword [[VEC:v[0-9]+]]
-
-; GFX9-DAG: s_movk_i32 [[K:s[0-9]+]], 0x4500
-; GFX9-DAG: v_and_b32_e32 [[ELT0:v[0-9]+]], 0xffff, [[VEC]]
-; GFX9: v_lshl_or_b32 [[RES:v[0-9]+]], [[K]], 16, [[ELT0]]
-
-; CI: v_and_b32_e32 [[AND:v[0-9]+]], 0xffff, [[VEC]]
-; CI: v_or_b32_e32 [[RES:v[0-9]+]], 0x45000000, [[AND]]
-
-; VI: v_or_b32_sdwa [[RES:v[0-9]+]], [[K]], [[VEC]] dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-
-; GCN: {{flat|global}}_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RES]]
 define amdgpu_kernel void @v_insertelement_v2f16_1(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %in) #0 {
+; GFX9-LABEL: v_insertelement_v2f16_1:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX9-NEXT:    s_movk_i32 s4, 0x4500
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s2, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    global_load_dword v3, v[0:1], off
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s0, v2
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v2, 0xffff, v3
+; GFX9-NEXT:    v_lshl_or_b32 v2, s4, 16, v2
+; GFX9-NEXT:    global_store_dword v[0:1], v2, off
+; GFX9-NEXT:    s_endpgm
+;
+; VI-LABEL: v_insertelement_v2f16_1:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; VI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; VI-NEXT:    v_mov_b32_e32 v3, 0x45000000
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dword v4, v[0:1]
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v2
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_or_b32_sdwa v2, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; CI-LABEL: v_insertelement_v2f16_1:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; CI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    v_mov_b32_e32 v1, s3
+; CI-NEXT:    v_add_i32_e32 v0, vcc, s2, v2
+; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; CI-NEXT:    flat_load_dword v3, v[0:1]
+; CI-NEXT:    v_add_i32_e32 v0, vcc, s0, v2
+; CI-NEXT:    v_mov_b32_e32 v1, s1
+; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; CI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CI-NEXT:    v_and_b32_e32 v2, 0xffff, v3
+; CI-NEXT:    v_or_b32_e32 v2, 0x45000000, v2
+; CI-NEXT:    flat_store_dword v[0:1], v2
+; CI-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x() #1
   %tid.ext = sext i32 %tid to i64
   %in.gep = getelementptr inbounds <2 x half>, <2 x half> addrspace(1)* %in, i64 %tid.ext
@@ -387,16 +1035,60 @@ define amdgpu_kernel void @v_insertelement_v2f16_1(<2 x half> addrspace(1)* %out
   ret void
 }
 
-; GCN-LABEL: {{^}}v_insertelement_v2f16_1_inlineimm:
-; VI: v_mov_b32_e32 [[K:v[0-9]+]], 0x230000
-; GCN: {{flat|global}}_load_dword [[VEC:v[0-9]+]]
-; CI: v_and_b32_e32 [[ELT0:v[0-9]+]], 0xffff, [[VEC]]
-; GFX9: v_and_b32_e32 [[ELT0:v[0-9]+]], 0xffff, [[VEC]]
-; CI: v_or_b32_e32 [[RES:v[0-9]+]], 0x230000, [[ELT0]]
-; VI: v_or_b32_sdwa [[RES:v[0-9]+]], [[K]], [[VEC]] dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-; GFX9: v_lshl_or_b32 [[RES:v[0-9]+]], 35, 16, [[ELT0]]
-; GCN: {{flat|global}}_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RES]]
 define amdgpu_kernel void @v_insertelement_v2f16_1_inlineimm(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %in) #0 {
+; GFX9-LABEL: v_insertelement_v2f16_1_inlineimm:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s2, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    global_load_dword v3, v[0:1], off
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s0, v2
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v2, 0xffff, v3
+; GFX9-NEXT:    v_lshl_or_b32 v2, 35, 16, v2
+; GFX9-NEXT:    global_store_dword v[0:1], v2, off
+; GFX9-NEXT:    s_endpgm
+;
+; VI-LABEL: v_insertelement_v2f16_1_inlineimm:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; VI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; VI-NEXT:    v_mov_b32_e32 v3, 0x230000
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dword v4, v[0:1]
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v2
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_or_b32_sdwa v2, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; CI-LABEL: v_insertelement_v2f16_1_inlineimm:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; CI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    v_mov_b32_e32 v1, s3
+; CI-NEXT:    v_add_i32_e32 v0, vcc, s2, v2
+; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; CI-NEXT:    flat_load_dword v3, v[0:1]
+; CI-NEXT:    v_add_i32_e32 v0, vcc, s0, v2
+; CI-NEXT:    v_mov_b32_e32 v1, s1
+; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; CI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CI-NEXT:    v_and_b32_e32 v2, 0xffff, v3
+; CI-NEXT:    v_or_b32_e32 v2, 0x230000, v2
+; CI-NEXT:    flat_store_dword v[0:1], v2
+; CI-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x() #1
   %tid.ext = sext i32 %tid to i64
   %in.gep = getelementptr inbounds <2 x half>, <2 x half> addrspace(1)* %in, i64 %tid.ext
@@ -408,16 +1100,60 @@ define amdgpu_kernel void @v_insertelement_v2f16_1_inlineimm(<2 x half> addrspac
 }
 
 ; FIXME: Enable for others when argument load not split
-; GCN-LABEL: {{^}}s_insertelement_v2i16_dynamic:
-; GCN-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 0x3e7
-; GCN: s_load_dword [[IDX:s[0-9]+]]
-; GCN: s_load_dword [[VEC:s[0-9]+]]
-; GCN-DAG: v_mov_b32_e32 [[VVEC:v[0-9]+]], [[VEC]]
-; GCN-DAG: s_lshl_b32 [[SCALED_IDX:s[0-9]+]], [[IDX]], 4
-; GCN-DAG: s_lshl_b32 [[MASK:s[0-9]+]], 0xffff, [[SCALED_IDX]]
-; GCN: v_bfi_b32 [[RESULT:v[0-9]+]], [[MASK]], [[K]], [[VVEC]]
-; GCN: {{flat|global}}_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]]
 define amdgpu_kernel void @s_insertelement_v2i16_dynamic(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(4)* %vec.ptr, i32 addrspace(4)* %idx.ptr) #0 {
+; GFX9-LABEL: s_insertelement_v2i16_dynamic:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x10
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0x3e703e7
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    s_load_dword s0, s[4:5], 0x0
+; GFX9-NEXT:    s_load_dword s1, s[2:3], 0x0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_lshl_b32 s0, s0, 4
+; GFX9-NEXT:    s_lshl_b32 s0, 0xffff, s0
+; GFX9-NEXT:    v_mov_b32_e32 v3, s1
+; GFX9-NEXT:    v_bfi_b32 v2, s0, v2, v3
+; GFX9-NEXT:    global_store_dword v[0:1], v2, off
+; GFX9-NEXT:    s_endpgm
+;
+; VI-LABEL: s_insertelement_v2i16_dynamic:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; VI-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x10
+; VI-NEXT:    v_mov_b32_e32 v2, 0x3e703e7
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    s_load_dword s0, s[4:5], 0x0
+; VI-NEXT:    s_load_dword s1, s[2:3], 0x0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_lshl_b32 s0, s0, 4
+; VI-NEXT:    s_lshl_b32 s0, 0xffff, s0
+; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    v_bfi_b32 v2, s0, v2, v3
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; CI-LABEL: s_insertelement_v2i16_dynamic:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; CI-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x4
+; CI-NEXT:    v_mov_b32_e32 v2, 0x3e703e7
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    v_mov_b32_e32 v0, s0
+; CI-NEXT:    v_mov_b32_e32 v1, s1
+; CI-NEXT:    s_load_dword s0, s[4:5], 0x0
+; CI-NEXT:    s_load_dword s1, s[2:3], 0x0
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    s_lshl_b32 s0, s0, 4
+; CI-NEXT:    s_lshl_b32 s0, 0xffff, s0
+; CI-NEXT:    v_mov_b32_e32 v3, s1
+; CI-NEXT:    v_bfi_b32 v2, s0, v2, v3
+; CI-NEXT:    flat_store_dword v[0:1], v2
+; CI-NEXT:    s_endpgm
   %idx = load volatile i32, i32 addrspace(4)* %idx.ptr
   %vec = load <2 x i16>, <2 x i16> addrspace(4)* %vec.ptr
   %vecins = insertelement <2 x i16> %vec, i16 999, i32 %idx
@@ -425,15 +1161,69 @@ define amdgpu_kernel void @s_insertelement_v2i16_dynamic(<2 x i16> addrspace(1)*
   ret void
 }
 
-; GCN-LABEL: {{^}}v_insertelement_v2i16_dynamic_sgpr:
-; GCN-DAG: {{flat|global}}_load_dword [[VEC:v[0-9]+]]
-; GCN-DAG: s_load_dword [[IDX:s[0-9]+]]
-; GCN-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 0x3e7
-; GCN-DAG: s_lshl_b32 [[SCALED_IDX:s[0-9]+]], [[IDX]], 4
-; GCN-DAG: s_lshl_b32 [[MASK:s[0-9]+]], 0xffff, [[SCALED_IDX]]
-; GCN: v_bfi_b32 [[RESULT:v[0-9]+]], [[MASK]], [[K]], [[VEC]]
-; GCN: {{flat|global}}_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]]
 define amdgpu_kernel void @v_insertelement_v2i16_dynamic_sgpr(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %in, i32 %idx) #0 {
+; GFX9-LABEL: v_insertelement_v2i16_dynamic_sgpr:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX9-NEXT:    s_load_dword s4, s[4:5], 0x10
+; GFX9-NEXT:    v_mov_b32_e32 v3, 0x3e703e7
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s2, v2
+; GFX9-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    global_load_dword v4, v[0:1], off
+; GFX9-NEXT:    s_lshl_b32 s2, s4, 4
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s0, v2
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    s_lshl_b32 s0, 0xffff, s2
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_bfi_b32 v2, s0, v3, v4
+; GFX9-NEXT:    global_store_dword v[0:1], v2, off
+; GFX9-NEXT:    s_endpgm
+;
+; VI-LABEL: v_insertelement_v2i16_dynamic_sgpr:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; VI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; VI-NEXT:    s_load_dword s4, s[4:5], 0x10
+; VI-NEXT:    v_mov_b32_e32 v3, 0x3e703e7
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dword v4, v[0:1]
+; VI-NEXT:    s_lshl_b32 s2, s4, 4
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v2
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    s_lshl_b32 s0, 0xffff, s2
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_bfi_b32 v2, s0, v3, v4
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; CI-LABEL: v_insertelement_v2i16_dynamic_sgpr:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; CI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; CI-NEXT:    s_load_dword s4, s[4:5], 0x4
+; CI-NEXT:    v_mov_b32_e32 v3, 0x3e703e7
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    v_add_i32_e32 v0, vcc, s2, v2
+; CI-NEXT:    v_mov_b32_e32 v1, s3
+; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; CI-NEXT:    flat_load_dword v4, v[0:1]
+; CI-NEXT:    s_lshl_b32 s2, s4, 4
+; CI-NEXT:    v_add_i32_e32 v0, vcc, s0, v2
+; CI-NEXT:    v_mov_b32_e32 v1, s1
+; CI-NEXT:    s_lshl_b32 s0, 0xffff, s2
+; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; CI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CI-NEXT:    v_bfi_b32 v2, s0, v3, v4
+; CI-NEXT:    flat_store_dword v[0:1], v2
+; CI-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x() #1
   %tid.ext = sext i32 %tid to i64
   %in.gep = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %in, i64 %tid.ext
@@ -444,22 +1234,86 @@ define amdgpu_kernel void @v_insertelement_v2i16_dynamic_sgpr(<2 x i16> addrspac
   ret void
 }
 
-; GCN-LABEL: {{^}}v_insertelement_v2f16_dynamic_vgpr:
-; GFX89-DAG: s_mov_b32 [[MASKK:s[0-9]+]], 0xffff{{$}}
-; GCN-DAG: s_mov_b32 [[K:s[0-9]+]], 0x12341234
-
-; GCN-DAG: {{flat|global}}_load_dword [[IDX:v[0-9]+]]
-; GCN-DAG: {{flat|global}}_load_dword [[VEC:v[0-9]+]]
-
-; GFX89-DAG: v_lshlrev_b32_e32 [[SCALED_IDX:v[0-9]+]], 4, [[IDX]]
-; GFX89-DAG: v_lshlrev_b32_e64 [[MASK:v[0-9]+]], [[SCALED_IDX]], [[MASKK]]
-
-; CI-DAG: v_lshlrev_b32_e32 [[SCALED_IDX:v[0-9]+]], 4, [[IDX]]
-; CI-DAG: v_lshl_b32_e32 [[MASK:v[0-9]+]], 0xffff, [[SCALED_IDX]]
-
-; GCN: v_bfi_b32 [[RESULT:v[0-9]+]], [[MASK]], [[K]], [[VEC]]
-; GCN: {{flat|global}}_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]]
 define amdgpu_kernel void @v_insertelement_v2f16_dynamic_vgpr(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %in, i32 addrspace(1)* %idx.ptr) #0 {
+; GFX9-LABEL: v_insertelement_v2f16_dynamic_vgpr:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x10
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_mov_b32 s6, 0xffff
+; GFX9-NEXT:    s_mov_b32 s7, 0x12341234
+; GFX9-NEXT:    v_mov_b32_e32 v3, s3
+; GFX9-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s4, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    global_load_dword v4, v[0:1], off
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s2, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
+; GFX9-NEXT:    global_load_dword v3, v[0:1], off
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s0, v2
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 4, v4
+; GFX9-NEXT:    v_lshlrev_b32_e64 v2, v2, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_bfi_b32 v2, v2, s7, v3
+; GFX9-NEXT:    global_store_dword v[0:1], v2, off
+; GFX9-NEXT:    s_endpgm
+;
+; VI-LABEL: v_insertelement_v2f16_dynamic_vgpr:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; VI-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x10
+; VI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_mov_b32 s6, 0xffff
+; VI-NEXT:    s_mov_b32 s7, 0x12341234
+; VI-NEXT:    v_mov_b32_e32 v3, s3
+; VI-NEXT:    v_mov_b32_e32 v1, s5
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s4, v2
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dword v4, v[0:1]
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v3, vcc
+; VI-NEXT:    flat_load_dword v3, v[0:1]
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v2
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)
+; VI-NEXT:    v_lshlrev_b32_e32 v2, 4, v4
+; VI-NEXT:    v_lshlrev_b32_e64 v2, v2, s6
+; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_bfi_b32 v2, v2, s7, v3
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; CI-LABEL: v_insertelement_v2f16_dynamic_vgpr:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; CI-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x4
+; CI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    s_mov_b32 s6, 0x12341234
+; CI-NEXT:    v_mov_b32_e32 v3, s3
+; CI-NEXT:    v_mov_b32_e32 v1, s5
+; CI-NEXT:    v_add_i32_e32 v0, vcc, s4, v2
+; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; CI-NEXT:    flat_load_dword v4, v[0:1]
+; CI-NEXT:    v_add_i32_e32 v0, vcc, s2, v2
+; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v3, vcc
+; CI-NEXT:    flat_load_dword v3, v[0:1]
+; CI-NEXT:    v_add_i32_e32 v0, vcc, s0, v2
+; CI-NEXT:    v_mov_b32_e32 v1, s1
+; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; CI-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)
+; CI-NEXT:    v_lshlrev_b32_e32 v2, 4, v4
+; CI-NEXT:    v_lshl_b32_e32 v2, 0xffff, v2
+; CI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CI-NEXT:    v_bfi_b32 v2, v2, s6, v3
+; CI-NEXT:    flat_store_dword v[0:1], v2
+; CI-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x() #1
   %tid.ext = sext i32 %tid to i64
   %in.gep = getelementptr inbounds <2 x half>, <2 x half> addrspace(1)* %in, i64 %tid.ext
@@ -472,19 +1326,65 @@ define amdgpu_kernel void @v_insertelement_v2f16_dynamic_vgpr(<2 x half> addrspa
   ret void
 }
 
-; GCN-LABEL: {{^}}v_insertelement_v4f16_0:
-; GCN-DAG: s_load_dword [[VAL:s[0-9]+]], s[4:5],
-; GCN-DAG: {{flat|global}}_load_dwordx2 v{{\[}}[[LO:[0-9]+]]:[[HI:[0-9]+]]{{\]}}
-
-; GFX9-DAG: v_mov_b32_e32 [[BFI_MASK:v[0-9]+]], 0xffff{{$}}
-; GFX9: v_bfi_b32 v[[INS_LO:[0-9]+]], [[BFI_MASK]], [[VAL]], v[[LO]]
-
-; CIVI: s_and_b32 [[VAL_MASKED:s[0-9]+]], [[VAL]], 0xffff{{$}}
-; CIVI: v_and_b32_e32 [[AND:v[0-9]+]], 0xffff0000, v[[LO]]
-; CIVI: v_or_b32_e32 v[[INS_LO:[0-9]+]], [[VAL_MASKED]], [[AND]]
-
-; GCN: {{flat|global}}_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[INS_LO]]:[[HI]]{{\]}}
 define amdgpu_kernel void @v_insertelement_v4f16_0(<4 x half> addrspace(1)* %out, <4 x half> addrspace(1)* %in, [8 x i32], i32 %val) #0 {
+; GFX9-LABEL: v_insertelement_v4f16_0:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; GFX9-NEXT:    s_load_dword s4, s[4:5], 0x30
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0xffff
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s2, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off
+; GFX9-NEXT:    v_mov_b32_e32 v3, s1
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, s0, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_bfi_b32 v0, v4, s4, v0
+; GFX9-NEXT:    global_store_dwordx2 v[2:3], v[0:1], off
+; GFX9-NEXT:    s_endpgm
+;
+; VI-LABEL: v_insertelement_v4f16_0:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; VI-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; VI-NEXT:    s_load_dword s4, s[4:5], 0x30
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    v_add_u32_e32 v2, vcc, s0, v2
+; VI-NEXT:    s_and_b32 s1, s4, 0xffff
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
+; VI-NEXT:    v_or_b32_e32 v0, s1, v0
+; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT:    s_endpgm
+;
+; CI-LABEL: v_insertelement_v4f16_0:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; CI-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; CI-NEXT:    s_load_dword s4, s[4:5], 0xc
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    v_mov_b32_e32 v1, s3
+; CI-NEXT:    v_add_i32_e32 v0, vcc, s2, v2
+; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; CI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; CI-NEXT:    v_mov_b32_e32 v3, s1
+; CI-NEXT:    v_add_i32_e32 v2, vcc, s0, v2
+; CI-NEXT:    s_and_b32 s1, s4, 0xffff
+; CI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; CI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CI-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
+; CI-NEXT:    v_or_b32_e32 v0, s1, v0
+; CI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; CI-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x() #1
   %tid.ext = sext i32 %tid to i64
   %in.gep = getelementptr inbounds <4 x half>, <4 x half> addrspace(1)* %in, i64 %tid.ext
@@ -497,23 +1397,65 @@ define amdgpu_kernel void @v_insertelement_v4f16_0(<4 x half> addrspace(1)* %out
   ret void
 }
 
-; GCN-LABEL: {{^}}v_insertelement_v4f16_1:
-; GCN-DAG: s_load_dword [[VAL:s[0-9]+]]
-; GCN-DAG: {{flat|global}}_load_dwordx2 v{{\[}}[[LO:[0-9]+]]:[[HI:[0-9]+]]{{\]}}
-
-; GFX9: v_and_b32_e32 [[AND:v[0-9]+]], 0xffff, v[[LO]]
-; GFX9: v_lshl_or_b32 v[[INS_HALF:[0-9]+]], [[VAL]], 16, [[AND]]
-
-; VI: s_lshl_b32 [[VAL_HI:s[0-9]+]], [[VAL]], 16
-; VI-DAG: v_mov_b32_e32 [[COPY_VAL:v[0-9]+]], [[VAL_HI]]
-; VI: v_or_b32_sdwa v[[INS_HALF:[0-9]+]], [[COPY_VAL]], v[[LO]] dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-
-; CI: s_lshl_b32 [[VAL_HI:s[0-9]+]], [[VAL]], 16
-; CI: v_and_b32_e32 [[AND:v[0-9]+]], 0xffff, v[[LO]]
-; CI: v_or_b32_e32 v[[INS_HALF:[0-9]+]], [[VAL_HI]], [[AND]]
-
-; GCN: {{flat|global}}_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[INS_HALF]]:[[HI]]{{\]}}
 define amdgpu_kernel void @v_insertelement_v4f16_1(<4 x half> addrspace(1)* %out, <4 x half> addrspace(1)* %in, i32 %val) #0 {
+; GFX9-LABEL: v_insertelement_v4f16_1:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; GFX9-NEXT:    s_load_dword s4, s[4:5], 0x10
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s2, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off
+; GFX9-NEXT:    v_mov_b32_e32 v3, s1
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, s0, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT:    v_lshl_or_b32 v0, s4, 16, v0
+; GFX9-NEXT:    global_store_dwordx2 v[2:3], v[0:1], off
+; GFX9-NEXT:    s_endpgm
+;
+; VI-LABEL: v_insertelement_v4f16_1:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; VI-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; VI-NEXT:    s_load_dword s4, s[4:5], 0x10
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; VI-NEXT:    s_lshl_b32 s2, s4, 16
+; VI-NEXT:    v_mov_b32_e32 v4, s2
+; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    v_add_u32_e32 v2, vcc, s0, v2
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_or_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT:    s_endpgm
+;
+; CI-LABEL: v_insertelement_v4f16_1:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; CI-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; CI-NEXT:    s_load_dword s4, s[4:5], 0x4
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    v_mov_b32_e32 v1, s3
+; CI-NEXT:    v_add_i32_e32 v0, vcc, s2, v2
+; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; CI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; CI-NEXT:    v_mov_b32_e32 v3, s1
+; CI-NEXT:    v_add_i32_e32 v2, vcc, s0, v2
+; CI-NEXT:    s_lshl_b32 s1, s4, 16
+; CI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; CI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; CI-NEXT:    v_or_b32_e32 v0, s1, v0
+; CI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; CI-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x() #1
   %tid.ext = sext i32 %tid to i64
   %in.gep = getelementptr inbounds <4 x half>, <4 x half> addrspace(1)* %in, i64 %tid.ext
@@ -526,19 +1468,65 @@ define amdgpu_kernel void @v_insertelement_v4f16_1(<4 x half> addrspace(1)* %out
   ret void
 }
 
-; GCN-LABEL: {{^}}v_insertelement_v4f16_2:
-; GCN-DAG: s_load_dword [[VAL:s[0-9]+]], s[4:5],
-; GCN-DAG: {{flat|global}}_load_dwordx2 v{{\[}}[[LO:[0-9]+]]:[[HI:[0-9]+]]{{\]}}
-
-; GFX9-DAG: v_mov_b32_e32 [[BFI_MASK:v[0-9]+]], 0xffff{{$}}
-; GFX9: v_bfi_b32 v[[INS_HI:[0-9]+]], [[BFI_MASK]], [[VAL]], v[[HI]]
-
-; CIVI: s_and_b32 [[VAL_MASKED:s[0-9]+]], [[VAL]], 0xffff{{$}}
-; CIVI: v_and_b32_e32 [[AND:v[0-9]+]], 0xffff0000, v[[HI]]
-; CIVI: v_or_b32_e32 v[[INS_HI:[0-9]+]], [[VAL_MASKED]], [[AND]]
-
-; GCN: {{flat|global}}_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[LO]]:[[INS_HI]]{{\]}}
 define amdgpu_kernel void @v_insertelement_v4f16_2(<4 x half> addrspace(1)* %out, <4 x half> addrspace(1)* %in, [8 x i32], i32 %val) #0 {
+; GFX9-LABEL: v_insertelement_v4f16_2:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; GFX9-NEXT:    s_load_dword s4, s[4:5], 0x30
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0xffff
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s2, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off
+; GFX9-NEXT:    v_mov_b32_e32 v3, s1
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, s0, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_bfi_b32 v1, v4, s4, v1
+; GFX9-NEXT:    global_store_dwordx2 v[2:3], v[0:1], off
+; GFX9-NEXT:    s_endpgm
+;
+; VI-LABEL: v_insertelement_v4f16_2:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; VI-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; VI-NEXT:    s_load_dword s4, s[4:5], 0x30
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    v_add_u32_e32 v2, vcc, s0, v2
+; VI-NEXT:    s_and_b32 s1, s4, 0xffff
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_and_b32_e32 v1, 0xffff0000, v1
+; VI-NEXT:    v_or_b32_e32 v1, s1, v1
+; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT:    s_endpgm
+;
+; CI-LABEL: v_insertelement_v4f16_2:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; CI-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; CI-NEXT:    s_load_dword s4, s[4:5], 0xc
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    v_mov_b32_e32 v1, s3
+; CI-NEXT:    v_add_i32_e32 v0, vcc, s2, v2
+; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; CI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; CI-NEXT:    v_mov_b32_e32 v3, s1
+; CI-NEXT:    v_add_i32_e32 v2, vcc, s0, v2
+; CI-NEXT:    s_and_b32 s1, s4, 0xffff
+; CI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; CI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CI-NEXT:    v_and_b32_e32 v1, 0xffff0000, v1
+; CI-NEXT:    v_or_b32_e32 v1, s1, v1
+; CI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; CI-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x() #1
   %tid.ext = sext i32 %tid to i64
   %in.gep = getelementptr inbounds <4 x half>, <4 x half> addrspace(1)* %in, i64 %tid.ext
@@ -551,23 +1539,65 @@ define amdgpu_kernel void @v_insertelement_v4f16_2(<4 x half> addrspace(1)* %out
   ret void
 }
 
-; GCN-LABEL: {{^}}v_insertelement_v4f16_3:
-; GCN-DAG: s_load_dword [[VAL:s[0-9]+]]
-; GCN-DAG: {{flat|global}}_load_dwordx2 v{{\[}}[[LO:[0-9]+]]:[[HI:[0-9]+]]{{\]}}
-
-; GFX9: v_and_b32_e32 [[AND:v[0-9]+]], 0xffff, v[[HI]]
-; GFX9: v_lshl_or_b32 v[[INS_HI:[0-9]+]], [[VAL]], 16, [[AND]]
-
-; VI: s_lshl_b32 [[VAL_HI:s[0-9]+]], [[VAL]], 16
-; VI-DAG: v_mov_b32_e32 [[COPY_VAL:v[0-9]+]], [[VAL_HI]]
-; VI: v_or_b32_sdwa v[[INS_HI:[0-9]+]], [[COPY_VAL]], v[[HI]] dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-
-; CI: s_lshl_b32 [[VAL_HI:s[0-9]+]], [[VAL]], 16
-; CI: v_and_b32_e32 [[AND:v[0-9]+]], 0xffff, v[[HI]]
-; CI: v_or_b32_e32 v[[INS_HI:[0-9]+]], [[VAL_HI]], [[AND]]
-
-; GCN: {{flat|global}}_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[LO]]:[[INS_HI]]{{\]}}
 define amdgpu_kernel void @v_insertelement_v4f16_3(<4 x half> addrspace(1)* %out, <4 x half> addrspace(1)* %in, i32 %val) #0 {
+; GFX9-LABEL: v_insertelement_v4f16_3:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; GFX9-NEXT:    s_load_dword s4, s[4:5], 0x10
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s2, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off
+; GFX9-NEXT:    v_mov_b32_e32 v3, s1
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, s0, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GFX9-NEXT:    v_lshl_or_b32 v1, s4, 16, v1
+; GFX9-NEXT:    global_store_dwordx2 v[2:3], v[0:1], off
+; GFX9-NEXT:    s_endpgm
+;
+; VI-LABEL: v_insertelement_v4f16_3:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; VI-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; VI-NEXT:    s_load_dword s4, s[4:5], 0x10
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; VI-NEXT:    s_lshl_b32 s2, s4, 16
+; VI-NEXT:    v_mov_b32_e32 v4, s2
+; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    v_add_u32_e32 v2, vcc, s0, v2
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_or_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT:    s_endpgm
+;
+; CI-LABEL: v_insertelement_v4f16_3:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; CI-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; CI-NEXT:    s_load_dword s4, s[4:5], 0x4
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    v_mov_b32_e32 v1, s3
+; CI-NEXT:    v_add_i32_e32 v0, vcc, s2, v2
+; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; CI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; CI-NEXT:    v_mov_b32_e32 v3, s1
+; CI-NEXT:    v_add_i32_e32 v2, vcc, s0, v2
+; CI-NEXT:    s_lshl_b32 s1, s4, 16
+; CI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; CI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; CI-NEXT:    v_or_b32_e32 v1, s1, v1
+; CI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; CI-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x() #1
   %tid.ext = sext i32 %tid to i64
   %in.gep = getelementptr inbounds <4 x half>, <4 x half> addrspace(1)* %in, i64 %tid.ext
@@ -580,19 +1610,65 @@ define amdgpu_kernel void @v_insertelement_v4f16_3(<4 x half> addrspace(1)* %out
   ret void
 }
 
-; GCN-LABEL: {{^}}v_insertelement_v4i16_2:
-; GCN-DAG: s_load_dword [[VAL:s[0-9]+]]
-; GCN-DAG: {{flat|global}}_load_dwordx2 v{{\[}}[[LO:[0-9]+]]:[[HI:[0-9]+]]{{\]}}
-
-; GFX9-DAG: v_mov_b32_e32 [[BFI_MASK:v[0-9]+]], 0xffff{{$}}
-; GFX9: v_bfi_b32 v[[INS_HI:[0-9]+]], [[BFI_MASK]], [[VAL]], v[[HI]]
-
-; CIVI: s_and_b32 [[VAL_MASKED:s[0-9]+]], [[VAL]], 0xffff{{$}}
-; CIVI: v_and_b32_e32 [[AND:v[0-9]+]], 0xffff0000, v[[HI]]
-; CIVI: v_or_b32_e32 v[[INS_HI:[0-9]+]], [[VAL_MASKED]], [[AND]]
-
-; GCN: {{flat|global}}_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[LO]]:[[INS_HI]]{{\]}}
 define amdgpu_kernel void @v_insertelement_v4i16_2(<4 x i16> addrspace(1)* %out, <4 x i16> addrspace(1)* %in, i32 %val) #0 {
+; GFX9-LABEL: v_insertelement_v4i16_2:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; GFX9-NEXT:    s_load_dword s4, s[4:5], 0x10
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0xffff
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s2, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off
+; GFX9-NEXT:    v_mov_b32_e32 v3, s1
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, s0, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_bfi_b32 v1, v4, s4, v1
+; GFX9-NEXT:    global_store_dwordx2 v[2:3], v[0:1], off
+; GFX9-NEXT:    s_endpgm
+;
+; VI-LABEL: v_insertelement_v4i16_2:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; VI-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; VI-NEXT:    s_load_dword s4, s[4:5], 0x10
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    v_add_u32_e32 v2, vcc, s0, v2
+; VI-NEXT:    s_and_b32 s1, s4, 0xffff
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_and_b32_e32 v1, 0xffff0000, v1
+; VI-NEXT:    v_or_b32_e32 v1, s1, v1
+; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT:    s_endpgm
+;
+; CI-LABEL: v_insertelement_v4i16_2:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; CI-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; CI-NEXT:    s_load_dword s4, s[4:5], 0x4
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    v_mov_b32_e32 v1, s3
+; CI-NEXT:    v_add_i32_e32 v0, vcc, s2, v2
+; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; CI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; CI-NEXT:    v_mov_b32_e32 v3, s1
+; CI-NEXT:    v_add_i32_e32 v2, vcc, s0, v2
+; CI-NEXT:    s_and_b32 s1, s4, 0xffff
+; CI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; CI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CI-NEXT:    v_and_b32_e32 v1, 0xffff0000, v1
+; CI-NEXT:    v_or_b32_e32 v1, s1, v1
+; CI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; CI-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x() #1
   %tid.ext = sext i32 %tid to i64
   %in.gep = getelementptr inbounds <4 x i16>, <4 x i16> addrspace(1)* %in, i64 %tid.ext
@@ -606,26 +1682,90 @@ define amdgpu_kernel void @v_insertelement_v4i16_2(<4 x i16> addrspace(1)* %out,
 }
 
 ; FIXME: Better code on CI?
-; GCN-LABEL: {{^}}v_insertelement_v4i16_dynamic_vgpr:
-; GCN-DAG: {{flat|global}}_load_dword [[IDX:v[0-9]+]],
-; GCN-DAG: s_load_dword [[VAL:s[0-9]+]]
-; GCN-DAG: {{flat|global}}_load_dwordx2 v{{\[}}[[LO:[0-9]+]]:[[HI:[0-9]+]]{{\]}}
-
-; GCN-DAG: s_mov_b32 s[[MASK_LO:[0-9]+]], 0xffff
-; GCN-DAG: s_mov_b32 s[[MASK_HI:[0-9]+]], 0
-; CIVI-DAG: s_and_b32 [[MASKED_VAL:s[0-9]+]], [[VAL]], s[[MASK_LO]]
-; VI-DAG: s_lshl_b32 [[SHIFTED_VAL:s[0-9]+]], [[MASKED_VAL]], 16
-; CI-DAG: s_lshl_b32 [[SHIFTED_VAL:s[0-9]+]], [[VAL]], 16
-; CIVI: s_or_b32 [[DUP_VAL:s[0-9]+]], [[MASKED_VAL]], [[SHIFTED_VAL]]
-; GCN-DAG: v_lshlrev_b32_e32 [[SCALED_IDX:v[0-9]+]], 4, [[IDX]]
-; GFX9-DAG: s_pack_ll_b32_b16 [[DUP_VAL:s[0-9]+]], [[VAL]], [[VAL]]
-; GFX89: v_lshlrev_b64 v[{{[0-9:]+}}], [[SCALED_IDX]], s{{\[}}[[MASK_LO]]:[[MASK_HI]]{{\]}}
-; CI: v_lshl_b64 v[{{[0-9:]+}}], s[{{[0-9:]+}}], [[SCALED_IDX]]
-; GCN: v_bfi_b32 v{{[0-9]+}}, v{{[0-9]+}}, [[DUP_VAL]], v{{[0-9]+}}
-; GCN: v_bfi_b32 v{{[0-9]+}}, v{{[0-9]+}}, [[DUP_VAL]], v{{[0-9]+}}
-
-; GCN: {{flat|global}}_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}
 define amdgpu_kernel void @v_insertelement_v4i16_dynamic_vgpr(<4 x i16> addrspace(1)* %out, <4 x i16> addrspace(1)* %in, i32 %val) #0 {
+; GFX9-LABEL: v_insertelement_v4i16_dynamic_vgpr:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX9-NEXT:    global_load_dword v4, v[0:1], off
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; GFX9-NEXT:    s_load_dword s6, s[4:5], 0x10
+; GFX9-NEXT:    s_mov_b32 s5, 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s2, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off
+; GFX9-NEXT:    s_mov_b32 s4, 0xffff
+; GFX9-NEXT:    v_mov_b32_e32 v3, s1
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, s0, v2
+; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s6, s6
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    v_lshlrev_b32_e32 v4, 4, v4
+; GFX9-NEXT:    v_lshlrev_b64 v[4:5], v4, s[4:5]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_bfi_b32 v1, v5, s1, v1
+; GFX9-NEXT:    v_bfi_b32 v0, v4, s1, v0
+; GFX9-NEXT:    global_store_dwordx2 v[2:3], v[0:1], off
+; GFX9-NEXT:    s_endpgm
+;
+; VI-LABEL: v_insertelement_v4i16_dynamic_vgpr:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    flat_load_dword v4, v[0:1]
+; VI-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; VI-NEXT:    s_load_dword s6, s[4:5], 0x10
+; VI-NEXT:    s_mov_b32 s4, 0xffff
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    s_mov_b32 s5, 0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_and_b32 s1, s6, s4
+; VI-NEXT:    v_add_u32_e32 v2, vcc, s0, v2
+; VI-NEXT:    s_lshl_b32 s0, s1, 16
+; VI-NEXT:    s_or_b32 s0, s1, s0
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_lshlrev_b32_e32 v4, 4, v4
+; VI-NEXT:    v_lshlrev_b64 v[4:5], v4, s[4:5]
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_bfi_b32 v1, v5, s0, v1
+; VI-NEXT:    v_bfi_b32 v0, v4, s0, v0
+; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT:    s_endpgm
+;
+; CI-LABEL: v_insertelement_v4i16_dynamic_vgpr:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    flat_load_dword v4, v[0:1]
+; CI-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; CI-NEXT:    s_load_dword s6, s[4:5], 0x4
+; CI-NEXT:    s_mov_b32 s4, 0xffff
+; CI-NEXT:    v_mov_b32_e32 v1, s3
+; CI-NEXT:    v_add_i32_e32 v0, vcc, s2, v2
+; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; CI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; CI-NEXT:    s_mov_b32 s5, 0
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    s_lshl_b32 s2, s6, 16
+; CI-NEXT:    s_and_b32 s3, s6, s4
+; CI-NEXT:    v_mov_b32_e32 v3, s1
+; CI-NEXT:    v_add_i32_e32 v2, vcc, s0, v2
+; CI-NEXT:    s_or_b32 s1, s3, s2
+; CI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; CI-NEXT:    s_waitcnt vmcnt(1)
+; CI-NEXT:    v_lshlrev_b32_e32 v4, 4, v4
+; CI-NEXT:    v_lshl_b64 v[4:5], s[4:5], v4
+; CI-NEXT:    s_waitcnt vmcnt(0)
+; CI-NEXT:    v_bfi_b32 v1, v5, s1, v1
+; CI-NEXT:    v_bfi_b32 v0, v4, s1, v0
+; CI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; CI-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x() #1
   %tid.ext = sext i32 %tid to i64
   %in.gep = getelementptr inbounds <4 x i16>, <4 x i16> addrspace(1)* %in, i64 %tid.ext
@@ -639,8 +1779,88 @@ define amdgpu_kernel void @v_insertelement_v4i16_dynamic_vgpr(<4 x i16> addrspac
   ret void
 }
 
-; GCN-LABEL: {{^}}v_insertelement_v4f16_dynamic_sgpr:
 define amdgpu_kernel void @v_insertelement_v4f16_dynamic_sgpr(<4 x half> addrspace(1)* %out, <4 x half> addrspace(1)* %in, i32 %val, i32 %idxval) #0 {
+; GFX9-LABEL: v_insertelement_v4f16_dynamic_sgpr:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x10
+; GFX9-NEXT:    s_mov_b32 s7, 0
+; GFX9-NEXT:    s_mov_b32 s6, 0xffff
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s2, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off
+; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s4, s4
+; GFX9-NEXT:    s_lshl_b32 s2, s5, 4
+; GFX9-NEXT:    v_mov_b32_e32 v3, s1
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, s0, v2
+; GFX9-NEXT:    s_lshl_b64 s[0:1], s[6:7], s2
+; GFX9-NEXT:    v_mov_b32_e32 v4, s3
+; GFX9-NEXT:    v_mov_b32_e32 v5, s3
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_bfi_b32 v1, s1, v4, v1
+; GFX9-NEXT:    v_bfi_b32 v0, s0, v5, v0
+; GFX9-NEXT:    global_store_dwordx2 v[2:3], v[0:1], off
+; GFX9-NEXT:    s_endpgm
+;
+; VI-LABEL: v_insertelement_v4f16_dynamic_sgpr:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; VI-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; VI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x10
+; VI-NEXT:    s_mov_b32 s6, 0xffff
+; VI-NEXT:    s_mov_b32 s7, 0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; VI-NEXT:    s_and_b32 s2, s4, s6
+; VI-NEXT:    s_lshl_b32 s3, s2, 16
+; VI-NEXT:    s_or_b32 s2, s2, s3
+; VI-NEXT:    s_lshl_b32 s4, s5, 4
+; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    v_add_u32_e32 v2, vcc, s0, v2
+; VI-NEXT:    s_lshl_b64 s[0:1], s[6:7], s4
+; VI-NEXT:    v_mov_b32_e32 v4, s2
+; VI-NEXT:    v_mov_b32_e32 v5, s2
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_bfi_b32 v1, s1, v4, v1
+; VI-NEXT:    v_bfi_b32 v0, s0, v5, v0
+; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT:    s_endpgm
+;
+; CI-LABEL: v_insertelement_v4f16_dynamic_sgpr:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; CI-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; CI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x4
+; CI-NEXT:    s_mov_b32 s6, 0xffff
+; CI-NEXT:    s_mov_b32 s7, 0
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    v_mov_b32_e32 v1, s3
+; CI-NEXT:    v_add_i32_e32 v0, vcc, s2, v2
+; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; CI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; CI-NEXT:    s_and_b32 s2, s4, s6
+; CI-NEXT:    s_lshl_b32 s3, s4, 16
+; CI-NEXT:    s_or_b32 s2, s2, s3
+; CI-NEXT:    s_lshl_b32 s4, s5, 4
+; CI-NEXT:    v_mov_b32_e32 v3, s1
+; CI-NEXT:    v_add_i32_e32 v2, vcc, s0, v2
+; CI-NEXT:    s_lshl_b64 s[0:1], s[6:7], s4
+; CI-NEXT:    v_mov_b32_e32 v4, s2
+; CI-NEXT:    v_mov_b32_e32 v5, s2
+; CI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; CI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CI-NEXT:    v_bfi_b32 v1, s1, v4, v1
+; CI-NEXT:    v_bfi_b32 v0, s0, v5, v0
+; CI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; CI-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x() #1
   %tid.ext = sext i32 %tid to i64
   %in.gep = getelementptr inbounds <4 x half>, <4 x half> addrspace(1)* %in, i64 %tid.ext
diff --git a/llvm/test/CodeGen/AMDGPU/kernel-args.ll b/llvm/test/CodeGen/AMDGPU/kernel-args.ll
index a657b8d01..d6ececb1b 100644
--- a/llvm/test/CodeGen/AMDGPU/kernel-args.ll
+++ b/llvm/test/CodeGen/AMDGPU/kernel-args.ll
@@ -855,8 +855,8 @@ define amdgpu_kernel void @struct_argument_alignment({i32, i64} %arg0, i8, {i32,
 ; multiple.
 ; FUNC-LABEL: {{^}}packed_struct_argument_alignment:
 ; HSA-GFX9: kernarg_segment_byte_size = 28
-; HSA-GFX9: global_load_dword v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, off offset:13
 ; HSA-GFX9: global_load_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, off offset:17
+; HSA-GFX9: global_load_dword v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, off offset:13
 ; HSA-GFX9: s_load_dword s{{[0-9]+}}, s[4:5], 0x0
 ; HSA-GFX9: s_load_dwordx2 s{{\[[0-9]+:[0-9]+\]}}, s[4:5], 0x4
 define amdgpu_kernel void @packed_struct_argument_alignment(<{i32, i64}> %arg0, i8, <{i32, i64}> %arg1) {
diff --git a/llvm/test/CodeGen/AMDGPU/kernel-argument-dag-lowering.ll b/llvm/test/CodeGen/AMDGPU/kernel-argument-dag-lowering.ll
index b7344cfb3..669169692 100644
--- a/llvm/test/CodeGen/AMDGPU/kernel-argument-dag-lowering.ll
+++ b/llvm/test/CodeGen/AMDGPU/kernel-argument-dag-lowering.ll
@@ -75,8 +75,8 @@ define amdgpu_kernel void @struct_argument_alignment({i32, i64} %arg0, i8, {i32,
 ; multiple.
 ; FUNC-LABEL: {{^}}packed_struct_argument_alignment:
 ; HSA-VI: kernarg_segment_byte_size = 28
-; HSA-VI: global_load_dword v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, off offset:13
 ; HSA-VI: global_load_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, off offset:17
+; HSA-VI: global_load_dword v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, off offset:13
 ; HSA-VI: s_load_dword s{{[0-9]+}}, s[4:5], 0x0
 ; HSA-VI: s_load_dwordx2 s{{\[[0-9]+:[0-9]+\]}}, s[4:5], 0x4
 define amdgpu_kernel void @packed_struct_argument_alignment(<{i32, i64}> %arg0, i8, <{i32, i64}> %arg1) {
diff --git a/llvm/test/CodeGen/AMDGPU/kill-infinite-loop.ll b/llvm/test/CodeGen/AMDGPU/kill-infinite-loop.ll
new file mode 100644
index 000000000..30280b967
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/kill-infinite-loop.ll
@@ -0,0 +1,68 @@
+; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -enable-var-scope %s
+; Although it's modeled without any control flow in order to get better code
+; out of the structurizer, @llvm.amdgcn.kill actually ends the thread that calls
+; it with "true". In case it's called in a provably infinite loop, we still
+; need to successfully exit and export something, even if we can't know where
+; to jump to in the LLVM IR. Therefore we insert a null export ourselves in
+; this case right before the s_endpgm to avoid GPU hangs, which is what this
+; tests.
+
+; CHECK-LABEL: return_void
+; Make sure that we remove the done bit from the original export
+; CHECK: exp mrt0 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} vm
+; CHECK: exp null off, off, off, off done vm
+; CHECK-NEXT: s_endpgm
+define amdgpu_ps void @return_void(float %0) #0 {
+main_body:
+  %cmp = fcmp olt float %0, 1.000000e+01
+  br i1 %cmp, label %end, label %loop
+
+loop:
+  call void @llvm.amdgcn.kill(i1 false) #3
+  br label %loop
+
+end:
+  call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float 0., float 0., float 0., float 1., i1 true, i1 true) #3
+  ret void
+}
+
+; Check that we also remove the done bit from compressed exports correctly.
+; CHECK-LABEL: return_void_compr
+; CHECK: exp mrt0 v{{[0-9]+}}, off, v{{[0-9]+}}, off compr vm
+; CHECK: exp null off, off, off, off done vm
+; CHECK-NEXT: s_endpgm
+define amdgpu_ps void @return_void_compr(float %0) #0 {
+main_body:
+  %cmp = fcmp olt float %0, 1.000000e+01
+  br i1 %cmp, label %end, label %loop
+
+loop:
+  call void @llvm.amdgcn.kill(i1 false) #3
+  br label %loop
+
+end:
+  call void @llvm.amdgcn.exp.compr.v2i16(i32 0, i32 5, <2 x i16> < i16 0, i16 0 >, <2 x i16> < i16 0, i16 0 >, i1 true, i1 true) #3
+  ret void
+}
+
+; In case there's an epilog, we shouldn't have to do this.
+; CHECK-LABEL: return_nonvoid
+; CHECK-NOT: exp null off, off, off, off done vm
+define amdgpu_ps float @return_nonvoid(float %0) #0 {
+main_body:
+  %cmp = fcmp olt float %0, 1.000000e+01
+  br i1 %cmp, label %end, label %loop
+
+loop:
+  call void @llvm.amdgcn.kill(i1 false) #3
+  br label %loop
+
+end:
+  ret float 0.
+}
+
+declare void @llvm.amdgcn.kill(i1) #0
+declare void @llvm.amdgcn.exp.f32(i32 immarg, i32 immarg, float, float, float, float, i1 immarg, i1 immarg) #0
+declare void @llvm.amdgcn.exp.compr.v2i16(i32 immarg, i32 immarg, <2 x i16>, <2 x i16>, i1 immarg, i1 immarg) #0
+
+attributes #0 = { nounwind }
diff --git a/llvm/test/CodeGen/AMDGPU/large-work-group-promote-alloca.ll b/llvm/test/CodeGen/AMDGPU/large-work-group-promote-alloca.ll
index 0702f4091..778e9ed9a 100644
--- a/llvm/test/CodeGen/AMDGPU/large-work-group-promote-alloca.ll
+++ b/llvm/test/CodeGen/AMDGPU/large-work-group-promote-alloca.ll
@@ -47,8 +47,9 @@ entry:
   ret void
 }
 
-; SICI: @promote_alloca_size_1600.stack = internal unnamed_addr addrspace(3) global [1600 x [5 x i32]] undef, align 4
-; GFX10: alloca [5 x i32]
+; SI-NOT: @promote_alloca_size_1600.stack
+; CI: @promote_alloca_size_1600.stack = internal unnamed_addr addrspace(3) global [1024 x [5 x i32]] undef, align 4
+; GFX10: @promote_alloca_size_1600.stack = internal unnamed_addr addrspace(3) global [1024 x [5 x i32]] undef, align 4
 
 define amdgpu_kernel void @promote_alloca_size_1600(i32 addrspace(1)* nocapture %out, i32 addrspace(1)* nocapture %in) #2 {
 entry:
@@ -274,7 +275,7 @@ entry:
 
 attributes #0 = { nounwind "amdgpu-flat-work-group-size"="63,63" }
 attributes #1 = { nounwind "amdgpu-waves-per-eu"="1,3" "amdgpu-flat-work-group-size"="256,256" }
-attributes #2 = { nounwind "amdgpu-waves-per-eu"="1,1" "amdgpu-flat-work-group-size"="1600,1600" }
+attributes #2 = { nounwind "amdgpu-waves-per-eu"="1,1" "amdgpu-flat-work-group-size"="1024,1024" }
 attributes #3 = { nounwind "amdgpu-waves-per-eu"="1,10" }
 attributes #4 = { nounwind "amdgpu-waves-per-eu"="1,10" }
 attributes #5 = { nounwind "amdgpu-waves-per-eu"="1,6" "amdgpu-flat-work-group-size"="64,64" }
diff --git a/llvm/test/CodeGen/AMDGPU/lcssa-optnone.ll b/llvm/test/CodeGen/AMDGPU/lcssa-optnone.ll
new file mode 100644
index 000000000..bda1412cb
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/lcssa-optnone.ll
@@ -0,0 +1,26 @@
+; RUN: llc -march=amdgcn -O0 -o - %s | FileCheck %s
+
+; CHECK-LABEL: non_uniform_loop
+; CHECK: s_endpgm
+define amdgpu_kernel void @non_uniform_loop(float addrspace(1)* %array) {
+entry:
+  %w = tail call i32 @llvm.amdgcn.workitem.id.x()
+  br label %for.cond
+
+for.cond:
+  %i = phi i32 [0, %entry], [%i.next, %for.inc]
+  %cmp = icmp ult i32 %i, %w
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:
+  br label %for.inc
+
+for.inc:
+  %i.next = add i32 %i, 1
+  br label %for.cond
+
+for.end:
+  ret void
+}
+
+declare i32 @llvm.amdgcn.workitem.id.x()
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.pkrtz.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.pkrtz.ll
index 06e209ab0..90d7b693b 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.pkrtz.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.pkrtz.ll
@@ -1,42 +1,159 @@
-; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=SI %s
-; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=GFX89 -check-prefix=VI %s
-; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=GFX89 -check-prefix=GFX9 %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck %s -enable-var-scope -check-prefixes=GCN,SI
+; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck %s -enable-var-scope -check-prefixes=GCN,GFX89,VI
+; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck %s -enable-var-scope -check-prefixes=GCN,GFX89,GFX9
 
-; GCN-LABEL: {{^}}s_cvt_pkrtz_v2f16_f32:
-; GCN-DAG: s_load_dwordx2 s{{\[}}[[SX:[0-9]+]]:[[SY:[0-9]+]]{{\]}}, s[0:1], 0x{{b|2c}}
-; GCN: v_mov_b32_e32 [[VY:v[0-9]+]], s[[SY]]
-; SI: v_cvt_pkrtz_f16_f32_e32 v{{[0-9]+}}, s[[SX]], [[VY]]
-; GFX89: v_cvt_pkrtz_f16_f32 v{{[0-9]+}}, s[[SX]], [[VY]]
 define amdgpu_kernel void @s_cvt_pkrtz_v2f16_f32(<2 x half> addrspace(1)* %out, float %x, float %y) #0 {
+; SI-LABEL: s_cvt_pkrtz_v2f16_f32:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0xb
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v0, s3
+; SI-NEXT:    v_cvt_pkrtz_f16_f32_e32 v0, s2, v0
+; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: s_cvt_pkrtz_v2f16_f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2c
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v0, s2
+; VI-NEXT:    v_mov_b32_e32 v2, s1
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_cvt_pkrtz_f16_f32 v2, s0, v2
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; GFX9-LABEL: s_cvt_pkrtz_v2f16_f32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2c
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, s2
+; GFX9-NEXT:    v_mov_b32_e32 v2, s1
+; GFX9-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NEXT:    v_cvt_pkrtz_f16_f32 v2, s0, v2
+; GFX9-NEXT:    global_store_dword v[0:1], v2, off
+; GFX9-NEXT:    s_endpgm
   %result = call <2 x half> @llvm.amdgcn.cvt.pkrtz(float %x, float %y)
   store <2 x half> %result, <2 x half> addrspace(1)* %out
   ret void
 }
 
-; GCN-LABEL: {{^}}s_cvt_pkrtz_samereg_v2f16_f32:
-; GCN: s_load_dword [[X:s[0-9]+]]
-; GCN: v_cvt_pkrtz_f16_f32{{(_e64)*}} v{{[0-9]+}}, [[X]], [[X]]
 define amdgpu_kernel void @s_cvt_pkrtz_samereg_v2f16_f32(<2 x half> addrspace(1)* %out, float %x) #0 {
+; SI-LABEL: s_cvt_pkrtz_samereg_v2f16_f32:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dword s2, s[0:1], 0xb
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    v_cvt_pkrtz_f16_f32_e64 v0, s2, s2
+; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: s_cvt_pkrtz_samereg_v2f16_f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
+; VI-NEXT:    s_load_dword s0, s[0:1], 0x2c
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v0, s2
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_cvt_pkrtz_f16_f32 v2, s0, s0
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; GFX9-LABEL: s_cvt_pkrtz_samereg_v2f16_f32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
+; GFX9-NEXT:    s_load_dword s0, s[0:1], 0x2c
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, s2
+; GFX9-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NEXT:    v_cvt_pkrtz_f16_f32 v2, s0, s0
+; GFX9-NEXT:    global_store_dword v[0:1], v2, off
+; GFX9-NEXT:    s_endpgm
   %result = call <2 x half> @llvm.amdgcn.cvt.pkrtz(float %x, float %x)
   store <2 x half> %result, <2 x half> addrspace(1)* %out
   ret void
 }
 
-; GCN-LABEL: {{^}}s_cvt_pkrtz_undef_undef:
-; GCN-NEXT: ; %bb.0
-; GCN-NEXT: s_endpgm
 define amdgpu_kernel void @s_cvt_pkrtz_undef_undef(<2 x half> addrspace(1)* %out) #0 {
+; GCN-LABEL: s_cvt_pkrtz_undef_undef:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_endpgm
   %result = call <2 x half> @llvm.amdgcn.cvt.pkrtz(float undef, float undef)
   store <2 x half> %result, <2 x half> addrspace(1)* %out
   ret void
 }
 
-; GCN-LABEL: {{^}}v_cvt_pkrtz_v2f16_f32:
-; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
-; GCN: {{buffer|flat|global}}_load_dword [[B:v[0-9]+]]
-; SI: v_cvt_pkrtz_f16_f32_e32 v{{[0-9]+}}, [[A]], [[B]]
-; GFX89: v_cvt_pkrtz_f16_f32 v{{[0-9]+}}, [[A]], [[B]]
 define amdgpu_kernel void @v_cvt_pkrtz_v2f16_f32(<2 x half> addrspace(1)* %out, float addrspace(1)* %a.ptr, float addrspace(1)* %b.ptr) #0 {
+; SI-LABEL: v_cvt_pkrtz_v2f16_f32:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xd
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-NEXT:    v_mov_b32_e32 v1, 0
+; SI-NEXT:    s_mov_b64 s[6:7], s[2:3]
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[0:1], s[10:11]
+; SI-NEXT:    buffer_load_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
+; SI-NEXT:    s_mov_b64 s[10:11], s[2:3]
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cvt_pkrtz_f16_f32_e32 v2, v2, v3
+; SI-NEXT:    buffer_store_dword v2, v[0:1], s[8:11], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: v_cvt_pkrtz_v2f16_f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; VI-NEXT:    v_lshlrev_b32_e32 v4, 2, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s7
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s6, v4
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    v_add_u32_e32 v2, vcc, s0, v4
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    flat_load_dword v5, v[0:1]
+; VI-NEXT:    flat_load_dword v2, v[2:3]
+; VI-NEXT:    v_mov_b32_e32 v1, s5
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s4, v4
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cvt_pkrtz_f16_f32 v2, v5, v2
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; GFX9-LABEL: v_cvt_pkrtz_v2f16_f32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX9-NEXT:    v_lshlrev_b32_e32 v4, 2, v0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v1, s7
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s6, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v3, s1
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, s0, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    global_load_dword v5, v[0:1], off
+; GFX9-NEXT:    global_load_dword v2, v[2:3], off
+; GFX9-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s4, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cvt_pkrtz_f16_f32 v2, v5, v2
+; GFX9-NEXT:    global_store_dword v[0:1], v2, off
+; GFX9-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
   %a.gep = getelementptr inbounds float, float addrspace(1)* %a.ptr, i64 %tid.ext
@@ -49,10 +166,56 @@ define amdgpu_kernel void @v_cvt_pkrtz_v2f16_f32(<2 x half> addrspace(1)* %out,
   ret void
 }
 
-; GCN-LABEL: {{^}}v_cvt_pkrtz_v2f16_f32_reg_imm:
-; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
-; GCN: v_cvt_pkrtz_f16_f32{{(_e64)*}} v{{[0-9]+}}, [[A]], 1.0
 define amdgpu_kernel void @v_cvt_pkrtz_v2f16_f32_reg_imm(<2 x half> addrspace(1)* %out, float addrspace(1)* %a.ptr) #0 {
+; SI-LABEL: v_cvt_pkrtz_v2f16_f32_reg_imm:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-NEXT:    v_mov_b32_e32 v1, 0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT:    buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
+; SI-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cvt_pkrtz_f16_f32_e64 v2, v2, 1.0
+; SI-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: v_cvt_pkrtz_v2f16_f32_reg_imm:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dword v3, v[0:1]
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v2
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cvt_pkrtz_f16_f32 v2, v3, 1.0
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; GFX9-LABEL: v_cvt_pkrtz_v2f16_f32_reg_imm:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s2, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    global_load_dword v3, v[0:1], off
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s0, v2
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cvt_pkrtz_f16_f32 v2, v3, 1.0
+; GFX9-NEXT:    global_store_dword v[0:1], v2, off
+; GFX9-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
   %a.gep = getelementptr inbounds float, float addrspace(1)* %a.ptr, i64 %tid.ext
@@ -63,11 +226,56 @@ define amdgpu_kernel void @v_cvt_pkrtz_v2f16_f32_reg_imm(<2 x half> addrspace(1)
   ret void
 }
 
-; GCN-LABEL: {{^}}v_cvt_pkrtz_v2f16_f32_imm_reg:
-; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
-; SI: v_cvt_pkrtz_f16_f32_e32 v{{[0-9]+}}, 1.0, [[A]]
-; GFX89: v_cvt_pkrtz_f16_f32 v{{[0-9]+}}, 1.0, [[A]]
 define amdgpu_kernel void @v_cvt_pkrtz_v2f16_f32_imm_reg(<2 x half> addrspace(1)* %out, float addrspace(1)* %a.ptr) #0 {
+; SI-LABEL: v_cvt_pkrtz_v2f16_f32_imm_reg:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-NEXT:    v_mov_b32_e32 v1, 0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT:    buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
+; SI-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cvt_pkrtz_f16_f32_e32 v2, 1.0, v2
+; SI-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: v_cvt_pkrtz_v2f16_f32_imm_reg:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dword v3, v[0:1]
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v2
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cvt_pkrtz_f16_f32 v2, 1.0, v3
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; GFX9-LABEL: v_cvt_pkrtz_v2f16_f32_imm_reg:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s2, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    global_load_dword v3, v[0:1], off
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s0, v2
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cvt_pkrtz_f16_f32 v2, 1.0, v3
+; GFX9-NEXT:    global_store_dword v[0:1], v2, off
+; GFX9-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
   %a.gep = getelementptr inbounds float, float addrspace(1)* %a.ptr, i64 %tid.ext
@@ -78,11 +286,69 @@ define amdgpu_kernel void @v_cvt_pkrtz_v2f16_f32_imm_reg(<2 x half> addrspace(1)
   ret void
 }
 
-; GCN-LABEL: {{^}}v_cvt_pkrtz_v2f16_f32_fneg_lo:
-; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
-; GCN: {{buffer|flat|global}}_load_dword [[B:v[0-9]+]]
-; GCN: v_cvt_pkrtz_f16_f32{{(_e64)*}} v{{[0-9]+}}, -[[A]], [[B]]
 define amdgpu_kernel void @v_cvt_pkrtz_v2f16_f32_fneg_lo(<2 x half> addrspace(1)* %out, float addrspace(1)* %a.ptr, float addrspace(1)* %b.ptr) #0 {
+; SI-LABEL: v_cvt_pkrtz_v2f16_f32_fneg_lo:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xd
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-NEXT:    v_mov_b32_e32 v1, 0
+; SI-NEXT:    s_mov_b64 s[6:7], s[2:3]
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[0:1], s[10:11]
+; SI-NEXT:    buffer_load_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
+; SI-NEXT:    s_mov_b64 s[10:11], s[2:3]
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cvt_pkrtz_f16_f32_e64 v2, -v2, v3
+; SI-NEXT:    buffer_store_dword v2, v[0:1], s[8:11], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: v_cvt_pkrtz_v2f16_f32_fneg_lo:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; VI-NEXT:    v_lshlrev_b32_e32 v4, 2, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s7
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s6, v4
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    v_add_u32_e32 v2, vcc, s0, v4
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    flat_load_dword v5, v[0:1]
+; VI-NEXT:    flat_load_dword v2, v[2:3]
+; VI-NEXT:    v_mov_b32_e32 v1, s5
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s4, v4
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cvt_pkrtz_f16_f32 v2, -v5, v2
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; GFX9-LABEL: v_cvt_pkrtz_v2f16_f32_fneg_lo:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX9-NEXT:    v_lshlrev_b32_e32 v4, 2, v0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v1, s7
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s6, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v3, s1
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, s0, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    global_load_dword v5, v[0:1], off
+; GFX9-NEXT:    global_load_dword v2, v[2:3], off
+; GFX9-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s4, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cvt_pkrtz_f16_f32 v2, -v5, v2
+; GFX9-NEXT:    global_store_dword v[0:1], v2, off
+; GFX9-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
   %a.gep = getelementptr inbounds float, float addrspace(1)* %a.ptr, i64 %tid.ext
@@ -96,11 +362,69 @@ define amdgpu_kernel void @v_cvt_pkrtz_v2f16_f32_fneg_lo(<2 x half> addrspace(1)
   ret void
 }
 
-; GCN-LABEL: {{^}}v_cvt_pkrtz_v2f16_f32_fneg_hi:
-; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
-; GCN: {{buffer|flat|global}}_load_dword [[B:v[0-9]+]]
-; GCN: v_cvt_pkrtz_f16_f32{{(_e64)*}} v{{[0-9]+}}, [[A]], -[[B]]
 define amdgpu_kernel void @v_cvt_pkrtz_v2f16_f32_fneg_hi(<2 x half> addrspace(1)* %out, float addrspace(1)* %a.ptr, float addrspace(1)* %b.ptr) #0 {
+; SI-LABEL: v_cvt_pkrtz_v2f16_f32_fneg_hi:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xd
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-NEXT:    v_mov_b32_e32 v1, 0
+; SI-NEXT:    s_mov_b64 s[6:7], s[2:3]
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[0:1], s[10:11]
+; SI-NEXT:    buffer_load_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
+; SI-NEXT:    s_mov_b64 s[10:11], s[2:3]
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cvt_pkrtz_f16_f32_e64 v2, v2, -v3
+; SI-NEXT:    buffer_store_dword v2, v[0:1], s[8:11], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: v_cvt_pkrtz_v2f16_f32_fneg_hi:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; VI-NEXT:    v_lshlrev_b32_e32 v4, 2, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s7
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s6, v4
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    v_add_u32_e32 v2, vcc, s0, v4
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    flat_load_dword v5, v[0:1]
+; VI-NEXT:    flat_load_dword v2, v[2:3]
+; VI-NEXT:    v_mov_b32_e32 v1, s5
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s4, v4
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cvt_pkrtz_f16_f32 v2, v5, -v2
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; GFX9-LABEL: v_cvt_pkrtz_v2f16_f32_fneg_hi:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX9-NEXT:    v_lshlrev_b32_e32 v4, 2, v0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v1, s7
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s6, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v3, s1
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, s0, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    global_load_dword v5, v[0:1], off
+; GFX9-NEXT:    global_load_dword v2, v[2:3], off
+; GFX9-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s4, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cvt_pkrtz_f16_f32 v2, v5, -v2
+; GFX9-NEXT:    global_store_dword v[0:1], v2, off
+; GFX9-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
   %a.gep = getelementptr inbounds float, float addrspace(1)* %a.ptr, i64 %tid.ext
@@ -114,11 +438,69 @@ define amdgpu_kernel void @v_cvt_pkrtz_v2f16_f32_fneg_hi(<2 x half> addrspace(1)
   ret void
 }
 
-; GCN-LABEL: {{^}}v_cvt_pkrtz_v2f16_f32_fneg_lo_hi:
-; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
-; GCN: {{buffer|flat|global}}_load_dword [[B:v[0-9]+]]
-; GCN: v_cvt_pkrtz_f16_f32{{(_e64)*}} v{{[0-9]+}}, -[[A]], -[[B]]
 define amdgpu_kernel void @v_cvt_pkrtz_v2f16_f32_fneg_lo_hi(<2 x half> addrspace(1)* %out, float addrspace(1)* %a.ptr, float addrspace(1)* %b.ptr) #0 {
+; SI-LABEL: v_cvt_pkrtz_v2f16_f32_fneg_lo_hi:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xd
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-NEXT:    v_mov_b32_e32 v1, 0
+; SI-NEXT:    s_mov_b64 s[6:7], s[2:3]
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[0:1], s[10:11]
+; SI-NEXT:    buffer_load_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
+; SI-NEXT:    s_mov_b64 s[10:11], s[2:3]
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cvt_pkrtz_f16_f32_e64 v2, -v2, -v3
+; SI-NEXT:    buffer_store_dword v2, v[0:1], s[8:11], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: v_cvt_pkrtz_v2f16_f32_fneg_lo_hi:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; VI-NEXT:    v_lshlrev_b32_e32 v4, 2, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s7
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s6, v4
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    v_add_u32_e32 v2, vcc, s0, v4
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    flat_load_dword v5, v[0:1]
+; VI-NEXT:    flat_load_dword v2, v[2:3]
+; VI-NEXT:    v_mov_b32_e32 v1, s5
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s4, v4
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cvt_pkrtz_f16_f32 v2, -v5, -v2
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; GFX9-LABEL: v_cvt_pkrtz_v2f16_f32_fneg_lo_hi:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX9-NEXT:    v_lshlrev_b32_e32 v4, 2, v0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v1, s7
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s6, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v3, s1
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, s0, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    global_load_dword v5, v[0:1], off
+; GFX9-NEXT:    global_load_dword v2, v[2:3], off
+; GFX9-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s4, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cvt_pkrtz_f16_f32 v2, -v5, -v2
+; GFX9-NEXT:    global_store_dword v[0:1], v2, off
+; GFX9-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
   %a.gep = getelementptr inbounds float, float addrspace(1)* %a.ptr, i64 %tid.ext
@@ -133,11 +515,69 @@ define amdgpu_kernel void @v_cvt_pkrtz_v2f16_f32_fneg_lo_hi(<2 x half> addrspace
   ret void
 }
 
-; GCN-LABEL: {{^}}v_cvt_pkrtz_v2f16_f32_fneg_fabs_lo_fneg_hi:
-; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
-; GCN: {{buffer|flat|global}}_load_dword [[B:v[0-9]+]]
-; GCN: v_cvt_pkrtz_f16_f32{{(_e64)*}} v{{[0-9]+}}, -|[[A]]|, -[[B]]
 define amdgpu_kernel void @v_cvt_pkrtz_v2f16_f32_fneg_fabs_lo_fneg_hi(<2 x half> addrspace(1)* %out, float addrspace(1)* %a.ptr, float addrspace(1)* %b.ptr) #0 {
+; SI-LABEL: v_cvt_pkrtz_v2f16_f32_fneg_fabs_lo_fneg_hi:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xd
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-NEXT:    v_mov_b32_e32 v1, 0
+; SI-NEXT:    s_mov_b64 s[6:7], s[2:3]
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[0:1], s[10:11]
+; SI-NEXT:    buffer_load_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
+; SI-NEXT:    s_mov_b64 s[10:11], s[2:3]
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cvt_pkrtz_f16_f32_e64 v2, -|v2|, -v3
+; SI-NEXT:    buffer_store_dword v2, v[0:1], s[8:11], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: v_cvt_pkrtz_v2f16_f32_fneg_fabs_lo_fneg_hi:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; VI-NEXT:    v_lshlrev_b32_e32 v4, 2, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s7
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s6, v4
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    v_add_u32_e32 v2, vcc, s0, v4
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    flat_load_dword v5, v[0:1]
+; VI-NEXT:    flat_load_dword v2, v[2:3]
+; VI-NEXT:    v_mov_b32_e32 v1, s5
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s4, v4
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cvt_pkrtz_f16_f32 v2, -|v5|, -v2
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; GFX9-LABEL: v_cvt_pkrtz_v2f16_f32_fneg_fabs_lo_fneg_hi:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX9-NEXT:    v_lshlrev_b32_e32 v4, 2, v0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v1, s7
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s6, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v3, s1
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, s0, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    global_load_dword v5, v[0:1], off
+; GFX9-NEXT:    global_load_dword v2, v[2:3], off
+; GFX9-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s4, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cvt_pkrtz_f16_f32 v2, -|v5|, -v2
+; GFX9-NEXT:    global_store_dword v[0:1], v2, off
+; GFX9-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
   %a.gep = getelementptr inbounds float, float addrspace(1)* %a.ptr, i64 %tid.ext
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.div.scale.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.div.scale.ll
index 2a5e81a6d..60ec52c22 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.div.scale.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.div.scale.ll
@@ -387,7 +387,7 @@ define amdgpu_kernel void @test_div_scale_f32_undef_val_val(float addrspace(1)*
 
 ; SI-LABEL: {{^}}test_div_scale_f32_undef_undef_val:
 ; SI-NOT: v0
-; SI: v_div_scale_f32 v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}, v0, v0, v0
+; SI: v_div_scale_f32 v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}, s0, s0, v0
 define amdgpu_kernel void @test_div_scale_f32_undef_undef_val(float addrspace(1)* %out) #0 {
   %result = call { float, i1 } @llvm.amdgcn.div.scale.f32(float undef, float undef, i1 false)
   %result0 = extractvalue { float, i1 } %result, 0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.barrier.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.barrier.ll
index a1affccd9..a781fb500 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.barrier.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.barrier.ll
@@ -2,33 +2,47 @@
 ; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=hawaii -o - -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,LOOP %s
 ; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -o - -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,LOOP %s
 ; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 -o - -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,NOLOOP %s
-; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1010 -o - -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,NOLOOP,GFX10 %s
+; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1010 -asm-verbose=0 -o - -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,NOLOOP,GFX10 %s
+
+; Make sure the op is emitted bundled with a waitcnt with and without the retry loop, and the bundle is not removed by ExpandPostRAPseudos.
+; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -stop-after=postrapseudos -o - -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=MIR %s
+; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 -stop-after=postrapseudos -o - -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=MIR %s
+
 
 ; Minimum offset
 ; GCN-LABEL: {{^}}gws_barrier_offset0:
 ; NOLOOP-DAG: s_load_dword [[BAR_NUM:s[0-9]+]]
-; NOLOOP-DAG: s_mov_b32 m0, -1{{$}}
+; NOLOOP-DAG: s_mov_b32 m0, 0{{$}}
 ; NOLOOP: v_mov_b32_e32 v0, [[BAR_NUM]]
-; NOLOOP: ds_gws_barrier v0 offset:1 gds{{$}}
+; NOLOOP: ds_gws_barrier v0 gds{{$}}
 
+; LOOP: s_mov_b32 m0, 0{{$}}
 ; LOOP: [[LOOP:BB[0-9]+_[0-9]+]]:
 ; LOOP-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_TRAPSTS, 8, 1), 0
-; LOOP-NEXT: ds_gws_barrier v0 offset:1 gds
+; LOOP-NEXT: ds_gws_barrier v0 gds
 ; LOOP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; LOOP-NEXT: s_getreg_b32 [[GETREG:s[0-9]+]], hwreg(HW_REG_TRAPSTS, 8, 1)
 ; LOOP-NEXT: s_cmp_lg_u32 [[GETREG]], 0
 ; LOOP-NEXT: s_cbranch_scc1 [[LOOP]]
+
+; MIR-LABEL: name: gws_barrier_offset0{{$}}
+; MIR: BUNDLE implicit{{( killed)?( renamable)?}} $vgpr0, implicit $m0, implicit $exec {
+; MIR-NEXT: DS_GWS_BARRIER renamable $vgpr0, 0, -1, implicit $m0, implicit $exec :: (load 4 from custom "GWSResource")
+; MIR-NEXT: S_WAITCNT 0
+; MIR-NEXT: }
 define amdgpu_kernel void @gws_barrier_offset0(i32 %val) #0 {
   call void @llvm.amdgcn.ds.gws.barrier(i32 %val, i32 0)
   ret void
 }
 
+; MIR-LABEL: name: gws_barrier_offset63{{$}}
+
 ; Maximum offset
 ; GCN-LABEL: {{^}}gws_barrier_offset63:
 ; NOLOOP-DAG: s_load_dword [[BAR_NUM:s[0-9]+]]
-; NOLOOP-DAG: s_mov_b32 m0, -1{{$}}
+; NOLOOP-DAG: s_mov_b32 m0, 0{{$}}
 ; NOLOOP-DAG: v_mov_b32_e32 v0, [[BAR_NUM]]
-; NOLOOP: ds_gws_barrier v0 offset:64 gds{{$}}
+; NOLOOP: ds_gws_barrier v0 offset:63 gds{{$}}
 define amdgpu_kernel void @gws_barrier_offset63(i32 %val) #0 {
   call void @llvm.amdgcn.ds.gws.barrier(i32 %val, i32 63)
   ret void
@@ -39,8 +53,8 @@ define amdgpu_kernel void @gws_barrier_offset63(i32 %val) #0 {
 ; NOLOOP-DAG: s_load_dwordx2 s{{\[}}[[BAR_NUM:[0-9]+]]:[[OFFSET:[0-9]+]]{{\]}}
 ; NOLOOP-DAG: s_lshl_b32 [[SHL:s[0-9]+]], s[[OFFSET]], 16
 ; NOLOOP-DAG: s_mov_b32 m0, [[SHL]]{{$}}
-; NOLOOP-DAG: v_mov_b32_e32 v0, s[[BAR_NUM]]
-; NOLOOP: ds_gws_barrier v0 gds{{$}}
+; NOLOOP-DAG: v_mov_b32_e32 [[GWS_VAL:v[0-9]+]], s[[BAR_NUM]]
+; NOLOOP: ds_gws_barrier [[GWS_VAL]] gds{{$}}
 define amdgpu_kernel void @gws_barrier_sgpr_offset(i32 %val, i32 %offset) #0 {
   call void @llvm.amdgcn.ds.gws.barrier(i32 %val, i32 %offset)
   ret void
@@ -51,8 +65,8 @@ define amdgpu_kernel void @gws_barrier_sgpr_offset(i32 %val, i32 %offset) #0 {
 ; NOLOOP-DAG: s_load_dwordx2 s{{\[}}[[BAR_NUM:[0-9]+]]:[[OFFSET:[0-9]+]]{{\]}}
 ; NOLOOP-DAG: s_lshl_b32 [[SHL:s[0-9]+]], s[[OFFSET]], 16
 ; NOLOOP-DAG: s_mov_b32 m0, [[SHL]]{{$}}
-; NOLOOP-DAG: v_mov_b32_e32 v0, s[[BAR_NUM]]
-; NOLOOP: ds_gws_barrier v0 offset:1 gds{{$}}
+; NOLOOP-DAG: v_mov_b32_e32 [[GWS_VAL:v[0-9]+]], s[[BAR_NUM]]
+; NOLOOP: ds_gws_barrier [[GWS_VAL]] offset:1 gds{{$}}
 define amdgpu_kernel void @gws_barrier_sgpr_offset_add1(i32 %val, i32 %offset.base) #0 {
   %offset = add i32 %offset.base, 1
   call void @llvm.amdgcn.ds.gws.barrier(i32 %val, i32 %offset)
@@ -64,8 +78,8 @@ define amdgpu_kernel void @gws_barrier_sgpr_offset_add1(i32 %val, i32 %offset.ba
 ; NOLOOP-DAG: v_readfirstlane_b32 [[READLANE:s[0-9]+]], v0
 ; NOLOOP-DAG: s_lshl_b32 [[SHL:s[0-9]+]], [[READLANE]], 16
 ; NOLOOP-DAG: s_mov_b32 m0, [[SHL]]{{$}}
-; NOLOOP-DAG: v_mov_b32_e32 v0, [[BAR_NUM]]
-; NOLOOP: ds_gws_barrier v0 gds{{$}}
+; NOLOOP-DAG: v_mov_b32_e32 [[GWS_VAL:v[0-9]+]], [[BAR_NUM]]
+; NOLOOP: ds_gws_barrier [[GWS_VAL]] gds{{$}}
 define amdgpu_kernel void @gws_barrier_vgpr_offset(i32 %val) #0 {
   %vgpr.offset = call i32 @llvm.amdgcn.workitem.id.x()
   call void @llvm.amdgcn.ds.gws.barrier(i32 %val, i32 %vgpr.offset)
@@ -78,8 +92,8 @@ define amdgpu_kernel void @gws_barrier_vgpr_offset(i32 %val) #0 {
 ; NOLOOP-DAG: v_readfirstlane_b32 [[READLANE:s[0-9]+]], v0
 ; NOLOOP-DAG: s_lshl_b32 [[SHL:s[0-9]+]], [[READLANE]], 16
 ; NOLOOP-DAG: s_mov_b32 m0, [[SHL]]{{$}}
-; NOLOOP-DAG: v_mov_b32_e32 v0, [[BAR_NUM]]
-; NOLOOP: ds_gws_barrier v0 offset:3 gds{{$}}
+; NOLOOP-DAG: v_mov_b32_e32 [[GWS_VAL:v[0-9]+]], [[BAR_NUM]]
+; NOLOOP: ds_gws_barrier [[GWS_VAL]] offset:3 gds{{$}}
 define amdgpu_kernel void @gws_barrier_vgpr_offset_add(i32 %val) #0 {
   %vgpr.offset.base = call i32 @llvm.amdgcn.workitem.id.x()
   %vgpr.offset = add i32 %vgpr.offset.base, 3
@@ -91,8 +105,18 @@ define amdgpu_kernel void @gws_barrier_vgpr_offset_add(i32 %val) #0 {
 
 ; Check if m0 initialization is shared
 ; GCN-LABEL: {{^}}gws_barrier_save_m0_barrier_constant_offset:
-; NOLOOP: s_mov_b32 m0, -1
-; NOLOOP-NOT: s_mov_b32 m0
+; NOLOOP: s_mov_b32 m0, 0
+; NOLOOP: ds_gws_barrier v{{[0-9]+}} offset:10 gds
+
+; LOOP: s_mov_b32 m0, -1
+; LOOP: ds_write_b32
+; LOOP: s_mov_b32 m0, 0
+; LOOP: s_setreg_imm32_b32
+; LOOP: ds_gws_barrier v{{[0-9]+}} offset:10 gds
+; LOOP: s_cbranch_scc1
+
+; LOOP: s_mov_b32 m0, -1
+; LOOP: ds_write_b32
 define amdgpu_kernel void @gws_barrier_save_m0_barrier_constant_offset(i32 %val) #0 {
   store i32 1, i32 addrspace(3)* @lds
   call void @llvm.amdgcn.ds.gws.barrier(i32 %val, i32 10)
@@ -102,8 +126,9 @@ define amdgpu_kernel void @gws_barrier_save_m0_barrier_constant_offset(i32 %val)
 
 ; Make sure this increments lgkmcnt
 ; GCN-LABEL: {{^}}gws_barrier_lgkmcnt:
-; NOLOOP: ds_gws_barrier v0 offset:1 gds{{$}}
-; NOLOOP-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; NOLOOP: s_mov_b32 m0, 0{{$}}
+; NOLOOP: ds_gws_barrier v0 gds{{$}}
+; NOLOOP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; NOLOOP-NEXT: s_setpc_b64
 define void @gws_barrier_lgkmcnt(i32 %val) {
   call void @llvm.amdgcn.ds.gws.barrier(i32 %val, i32 0)
@@ -121,8 +146,9 @@ define amdgpu_kernel void @gws_barrier_wait_before(i32 %val, i32 addrspace(1)* %
 }
 
 ; GCN-LABEL: {{^}}gws_barrier_wait_after:
-; NOLOOP: ds_gws_barrier v0 offset:8 gds
-; NOLOOP-NEXT: s_waitcnt expcnt(0){{$}}
+; NOLOOP: s_mov_b32 m0, 0{{$}}
+; NOLOOP: ds_gws_barrier v{{[0-9]+}} offset:7 gds
+; NOLOOP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; NOLOOP-NEXT: load_dword
 define amdgpu_kernel void @gws_barrier_wait_after(i32 %val, i32 addrspace(1)* %ptr) #0 {
   call void @llvm.amdgcn.ds.gws.barrier(i32 %val, i32 7)
@@ -132,9 +158,11 @@ define amdgpu_kernel void @gws_barrier_wait_after(i32 %val, i32 addrspace(1)* %p
 
 ; Does not imply memory fence on its own
 ; GCN-LABEL: {{^}}gws_barrier_fence_before:
+; NOLOOP: s_mov_b32 m0, 0{{$}}
 ; NOLOOP: store_dword
 ; NOLOOP: s_waitcnt vmcnt(0) lgkmcnt(0)
-; NOLOOP: ds_gws_barrier v0 offset:8 gds
+; NOLOOP: ds_gws_barrier v{{[0-9]+}} offset:7 gds
+; NOLOOP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 define amdgpu_kernel void @gws_barrier_fence_before(i32 %val, i32 addrspace(1)* %ptr) #0 {
   store i32 0, i32 addrspace(1)* %ptr
   fence release
@@ -142,9 +170,12 @@ define amdgpu_kernel void @gws_barrier_fence_before(i32 %val, i32 addrspace(1)*
   ret void
 }
 
+; FIXME: Extra waitcnt
 ; GCN-LABEL: {{^}}gws_barrier_fence_after:
-; NOLOOP: ds_gws_barrier v0 offset:8 gds
+; NOLOOP: s_mov_b32 m0, 0{{$}}
+; NOLOOP: ds_gws_barrier v{{[0-9]+}} offset:7 gds
 ; NOLOOP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; NOLOOP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
 ; NOLOOP-NEXT: load_dword
 define amdgpu_kernel void @gws_barrier_fence_after(i32 %val, i32 addrspace(1)* %ptr) #0 {
@@ -156,9 +187,11 @@ define amdgpu_kernel void @gws_barrier_fence_after(i32 %val, i32 addrspace(1)* %
 
 ; FIXME: Should a wait be inserted here, or is an explicit fence needed?
 ; GCN-LABEL: {{^}}gws_init_barrier:
-; NOLOOP: s_mov_b32 m0, -1
-; NOLOOP: ds_gws_init v0 offset:8 gds
-; NOLOOP-NEXT: ds_gws_barrier v0 offset:8 gds
+; NOLOOP: s_mov_b32 m0, 0
+; NOLOOP: ds_gws_init v0 offset:7 gds
+; NOLOOP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; NOLOOP-NEXT: ds_gws_barrier v0 offset:7 gds
+; NOLOOP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 define amdgpu_kernel void @gws_init_barrier(i32 %val) #0 {
   call void @llvm.amdgcn.ds.gws.init(i32 %val, i32 7)
   call void @llvm.amdgcn.ds.gws.barrier(i32 %val, i32 7)
@@ -167,11 +200,13 @@ define amdgpu_kernel void @gws_init_barrier(i32 %val) #0 {
 
 ; FIXME: Why vmcnt, not expcnt?
 ; GCN-LABEL: {{^}}gws_init_fence_barrier:
-; NOLOOP: s_mov_b32 m0, -1
-; NOLOOP: ds_gws_init v0 offset:8 gds
+; NOLOOP: s_mov_b32 m0, 0
+; NOLOOP: ds_gws_init v0 offset:7 gds
+; NOLOOP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; NOLOOP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; NOLOOP-NEXT: ds_gws_barrier v0 offset:8 gds
+; NOLOOP-NEXT: ds_gws_barrier v0 offset:7 gds
+; NOLOOP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 define amdgpu_kernel void @gws_init_fence_barrier(i32 %val) #0 {
   call void @llvm.amdgcn.ds.gws.init(i32 %val, i32 7)
   fence release
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.init.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.init.ll
index 075ec5073..aa490f15d 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.init.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.init.ll
@@ -2,18 +2,18 @@
 ; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=hawaii -o - -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,LOOP %s
 ; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -o - -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,LOOP %s
 ; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 -o - -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,NOLOOP %s
-; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1010 -o - -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,NOLOOP %s
+; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1010 -asm-verbose=0 -o - -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,NOLOOP %s
 
 ; Minimum offset
 ; GCN-LABEL: {{^}}gws_init_offset0:
 ; GCN-DAG: s_load_dword [[BAR_NUM:s[0-9]+]]
-; GCN-DAG: s_mov_b32 m0, -1{{$}}
+; GCN-DAG: s_mov_b32 m0, 0{{$}}
 ; GCN: v_mov_b32_e32 v0, [[BAR_NUM]]
-; NOLOOP: ds_gws_init v0 offset:1 gds{{$}}
+; NOLOOP: ds_gws_init v0 gds{{$}}
 
 ; LOOP: [[LOOP:BB[0-9]+_[0-9]+]]:
 ; LOOP-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_TRAPSTS, 8, 1), 0
-; LOOP-NEXT: ds_gws_init v0 offset:1 gds
+; LOOP-NEXT: ds_gws_init v0 gds
 ; LOOP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; LOOP-NEXT: s_getreg_b32 [[GETREG:s[0-9]+]], hwreg(HW_REG_TRAPSTS, 8, 1)
 ; LOOP-NEXT: s_cmp_lg_u32 [[GETREG]], 0
@@ -26,14 +26,15 @@ define amdgpu_kernel void @gws_init_offset0(i32 %val) #0 {
 ; Maximum offset
 ; GCN-LABEL: {{^}}gws_init_offset63:
 ; NOLOOP-DAG: s_load_dword [[BAR_NUM:s[0-9]+]]
-; NOLOOP-DAG: s_mov_b32 m0, -1{{$}}
+; NOLOOP-DAG: s_mov_b32 m0, 0{{$}}
 ; NOLOOP-DAG: v_mov_b32_e32 v0, [[BAR_NUM]]
-; NOLOOP: ds_gws_init v0 offset:64 gds{{$}}
+; NOLOOP: ds_gws_init v0 offset:63 gds{{$}}
 
 
+; LOOP: s_mov_b32 m0, 0{{$}}
 ; LOOP: [[LOOP:BB[0-9]+_[0-9]+]]:
 ; LOOP-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_TRAPSTS, 8, 1), 0
-; LOOP-NEXT: ds_gws_init v0 offset:64 gds
+; LOOP-NEXT: ds_gws_init v0 offset:63 gds
 ; LOOP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; LOOP-NEXT: s_getreg_b32 [[GETREG:s[0-9]+]], hwreg(HW_REG_TRAPSTS, 8, 1)
 ; LOOP-NEXT: s_cmp_lg_u32 [[GETREG]], 0
@@ -48,8 +49,8 @@ define amdgpu_kernel void @gws_init_offset63(i32 %val) #0 {
 ; NOLOOP-DAG: s_load_dwordx2 s{{\[}}[[BAR_NUM:[0-9]+]]:[[OFFSET:[0-9]+]]{{\]}}
 ; NOLOOP-DAG: s_lshl_b32 [[SHL:s[0-9]+]], s[[OFFSET]], 16
 ; NOLOOP-DAG: s_mov_b32 m0, [[SHL]]{{$}}
-; NOLOOP-DAG: v_mov_b32_e32 v0, s[[BAR_NUM]]
-; NOLOOP: ds_gws_init v0 gds{{$}}
+; NOLOOP-DAG: v_mov_b32_e32 [[GWS_VAL:v[0-9]+]], s[[BAR_NUM]]
+; NOLOOP: ds_gws_init [[GWS_VAL]] gds{{$}}
 define amdgpu_kernel void @gws_init_sgpr_offset(i32 %val, i32 %offset) #0 {
   call void @llvm.amdgcn.ds.gws.init(i32 %val, i32 %offset)
   ret void
@@ -60,8 +61,8 @@ define amdgpu_kernel void @gws_init_sgpr_offset(i32 %val, i32 %offset) #0 {
 ; NOLOOP-DAG: s_load_dwordx2 s{{\[}}[[BAR_NUM:[0-9]+]]:[[OFFSET:[0-9]+]]{{\]}}
 ; NOLOOP-DAG: s_lshl_b32 [[SHL:s[0-9]+]], s[[OFFSET]], 16
 ; NOLOOP-DAG: s_mov_b32 m0, [[SHL]]{{$}}
-; NOLOOP-DAG: v_mov_b32_e32 v0, s[[BAR_NUM]]
-; NOLOOP: ds_gws_init v0 offset:1 gds{{$}}
+; NOLOOP-DAG: v_mov_b32_e32 [[GWS_VAL:v[0-9]+]], s[[BAR_NUM]]
+; NOLOOP: ds_gws_init [[GWS_VAL]] offset:1 gds{{$}}
 define amdgpu_kernel void @gws_init_sgpr_offset_add1(i32 %val, i32 %offset.base) #0 {
   %offset = add i32 %offset.base, 1
   call void @llvm.amdgcn.ds.gws.init(i32 %val, i32 %offset)
@@ -100,18 +101,29 @@ define amdgpu_kernel void @gws_init_vgpr_offset_add(i32 %val) #0 {
 
 ; Check if m0 initialization is shared.
 ; GCN-LABEL: {{^}}gws_init_save_m0_init_constant_offset:
-; NOLOOP: s_mov_b32 m0, -1
-; NOLOOP-NOT: s_mov_b32 m0
+; NOLOOP: s_mov_b32 m0, 0
+; NOLOOP: ds_gws_init v{{[0-9]+}} offset:10 gds
+
+; LOOP: s_mov_b32 m0, -1
+; LOOP: ds_write_b32
+; LOOP: s_mov_b32 m0, 0
+; LOOP: s_setreg_imm32_b32
+; LOOP: ds_gws_init v{{[0-9]+}} offset:10 gds
+; LOOP: s_cbranch_scc1
+
+; LOOP: s_mov_b32 m0, -1
+; LOOP: ds_write_b32
 define amdgpu_kernel void @gws_init_save_m0_init_constant_offset(i32 %val) #0 {
-  store i32 1, i32 addrspace(3)* @lds
+  store volatile i32 1, i32 addrspace(3)* @lds
   call void @llvm.amdgcn.ds.gws.init(i32 %val, i32 10)
   store i32 2, i32 addrspace(3)* @lds
   ret void
 }
 
 ; GCN-LABEL: {{^}}gws_init_lgkmcnt:
-; NOLOOP: ds_gws_init v0 offset:1 gds{{$}}
-; NOLOOP-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; NOLOOP: s_mov_b32 m0, 0{{$}}
+; NOLOOP: ds_gws_init v0 gds{{$}}
+; NOLOOP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; NOLOOP-NEXT: s_setpc_b64
 define void @gws_init_lgkmcnt(i32 %val) {
   call void @llvm.amdgcn.ds.gws.init(i32 %val, i32 0)
@@ -120,8 +132,10 @@ define void @gws_init_lgkmcnt(i32 %val) {
 
 ; Does not imply memory fence on its own
 ; GCN-LABEL: {{^}}gws_init_wait_before:
-; NOLOOP: s_waitcnt
+; NOLOOP: s_waitcnt lgkmcnt(0)
 ; NOLOOP-NOT: s_waitcnt
+; NOLOOP: ds_gws_init
+; NOLOOP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 define amdgpu_kernel void @gws_init_wait_before(i32 %val, i32 addrspace(1)* %ptr) #0 {
   store i32 0, i32 addrspace(1)* %ptr
   call void @llvm.amdgcn.ds.gws.init(i32 %val, i32 7)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.sema.br.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.sema.br.ll
index 7363115e8..21e09c16f 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.sema.br.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.sema.br.ll
@@ -6,13 +6,14 @@
 
 ; GCN-LABEL: {{^}}gws_sema_br_offset0:
 ; NOLOOP-DAG: s_load_dword [[BAR_NUM:s[0-9]+]]
-; NOLOOP-DAG: s_mov_b32 m0, -1{{$}}
+; NOLOOP-DAG: s_mov_b32 m0, 0{{$}}
 ; NOLOOP: v_mov_b32_e32 v0, [[BAR_NUM]]
-; NOLOOP: ds_gws_sema_br v0 offset:1 gds{{$}}
+; NOLOOP: ds_gws_sema_br v0 gds{{$}}
 
+; LOOP: s_mov_b32 m0, 0{{$}}
 ; LOOP: [[LOOP:BB[0-9]+_[0-9]+]]:
 ; LOOP-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_TRAPSTS, 8, 1), 0
-; LOOP-NEXT: ds_gws_sema_br v0 offset:1 gds
+; LOOP-NEXT: ds_gws_sema_br v0 gds
 ; LOOP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; LOOP-NEXT: s_getreg_b32 [[GETREG:s[0-9]+]], hwreg(HW_REG_TRAPSTS, 8, 1)
 ; LOOP-NEXT: s_cmp_lg_u32 [[GETREG]], 0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.sema.p.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.sema.p.ll
index ba231c3df..dd1a801dd 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.sema.p.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.sema.p.ll
@@ -5,13 +5,14 @@
 ; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1010 -o - -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,NOLOOP,GFX10 %s
 
 ; GCN-LABEL: {{^}}gws_sema_p_offset0:
-; NOLOOP-DAG: s_mov_b32 m0, -1{{$}}
-; NOLOOP: ds_gws_sema_p offset:1 gds{{$}}
+; NOLOOP-DAG: s_mov_b32 m0, 0{{$}}
+; NOLOOP: ds_gws_sema_p gds{{$}}
 
+; LOOP: s_mov_b32 m0, 0{{$}}
 ; LOOP: [[LOOP:BB[0-9]+_[0-9]+]]:
 ; LOOP-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_TRAPSTS, 8, 1), 0
 ; GFX8-NEXT: s_nop 0
-; LOOP-NEXT: ds_gws_sema_p offset:1 gds
+; LOOP-NEXT: ds_gws_sema_p gds
 ; LOOP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; LOOP-NEXT: s_getreg_b32 [[GETREG:s[0-9]+]], hwreg(HW_REG_TRAPSTS, 8, 1)
 ; LOOP-NEXT: s_cmp_lg_u32 [[GETREG]], 0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.sema.release.all.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.sema.release.all.ll
index ab5f95b1b..2d690ec76 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.sema.release.all.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.sema.release.all.ll
@@ -7,13 +7,14 @@
 ; GFX6ERR: LLVM ERROR: Cannot select: intrinsic %llvm.amdgcn.ds.gws.sema.release.all
 
 ; GCN-LABEL: {{^}}gws_sema_release_all_offset0:
-; NOLOOP-DAG: s_mov_b32 m0, -1{{$}}
-; NOLOOP: ds_gws_sema_release_all offset:1 gds{{$}}
+; NOLOOP-DAG: s_mov_b32 m0, 0{{$}}
+; NOLOOP: ds_gws_sema_release_all gds{{$}}
 
+; LOOP: s_mov_b32 m0, 0{{$}}
 ; LOOP: [[LOOP:BB[0-9]+_[0-9]+]]:
 ; LOOP-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_TRAPSTS, 8, 1), 0
 ; GFX8-NEXT: s_nop 0
-; LOOP-NEXT: ds_gws_sema_release_all offset:1 gds
+; LOOP-NEXT: ds_gws_sema_release_all gds
 ; LOOP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; LOOP-NEXT: s_getreg_b32 [[GETREG:s[0-9]+]], hwreg(HW_REG_TRAPSTS, 8, 1)
 ; LOOP-NEXT: s_cmp_lg_u32 [[GETREG]], 0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.sema.v.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.sema.v.ll
index ed55db286..652646128 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.sema.v.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.sema.v.ll
@@ -5,13 +5,14 @@
 ; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1010 -o - -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,NOLOOP,GFX10 %s
 
 ; GCN-LABEL: {{^}}gws_sema_v_offset0:
-; NOLOOP-DAG: s_mov_b32 m0, -1{{$}}
-; NOLOOP: ds_gws_sema_v offset:1 gds{{$}}
+; NOLOOP-DAG: s_mov_b32 m0, 0{{$}}
+; NOLOOP: ds_gws_sema_v gds{{$}}
 
+; LOOP: s_mov_b32 m0, 0{{$}}
 ; LOOP: [[LOOP:BB[0-9]+_[0-9]+]]:
 ; LOOP-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_TRAPSTS, 8, 1), 0
 ; GFX8-NEXT: s_nop 0
-; LOOP-NEXT: ds_gws_sema_v offset:1 gds
+; LOOP-NEXT: ds_gws_sema_v gds
 ; LOOP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; LOOP-NEXT: s_getreg_b32 [[GETREG:s[0-9]+]], hwreg(HW_REG_TRAPSTS, 8, 1)
 ; LOOP-NEXT: s_cmp_lg_u32 [[GETREG]], 0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.ordered.swap.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.ordered.swap.ll
index acb1133c6..924a4cd04 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.ordered.swap.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.ordered.swap.ll
@@ -15,13 +15,13 @@ define amdgpu_cs float @ds_ordered_swap(i32 addrspace(2)* inreg %gds, i32 %value
 }
 
 ; FUNC-LABEL: {{^}}ds_ordered_swap_conditional:
-; GCN: v_cmp_ne_u32_e32 vcc, 0, v0
+; GCN: v_cmp_ne_u32_e32 vcc, 0, v[[VALUE:[0-9]+]]
 ; GCN: s_and_saveexec_b64 s[[SAVED:\[[0-9]+:[0-9]+\]]], vcc
 ; // We have to use s_cbranch, because ds_ordered_count has side effects with EXEC=0
 ; GCN: s_cbranch_execz [[BB:BB._.]]
 ; GCN: s_mov_b32 m0, s0
 ; VIGFX9-NEXT: s_nop 0
-; GCN-NEXT: ds_ordered_count v{{[0-9]+}}, v0 offset:4868 gds
+; GCN-NEXT: ds_ordered_count v{{[0-9]+}}, v[[VALUE]] offset:4868 gds
 ; GCN-NEXT: [[BB]]:
 ; // Wait for expcnt(0) before modifying EXEC
 ; GCN-NEXT: s_waitcnt expcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fmed3.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fmed3.ll
index c47d02f71..05b074bfe 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fmed3.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fmed3.ll
@@ -53,8 +53,8 @@ define amdgpu_kernel void @test_fabs_fmed3(float addrspace(1)* %out, float %src0
 }
 
 ; GCN-LABEL: {{^}}test_fneg_fmed3_rr_0:
-; GCN: s_brev_b32 [[NEG0:s[0-9]+]], 1
-; GCN: v_med3_f32 v{{[0-9]+}}, -v{{[0-9]+}}, -v{{[0-9]+}}, [[NEG0]]
+; GCN: v_bfrev_b32_e32 [[NEG0:v[0-9]+]], 1
+; GCN: v_med3_f32 v{{[0-9]+}}, -s{{[0-9]+}}, -v{{[0-9]+}}, [[NEG0]]
 define amdgpu_kernel void @test_fneg_fmed3_rr_0(float addrspace(1)* %out, float %src0, float %src1) #1 {
   %med3 = call float @llvm.amdgcn.fmed3.f32(float %src0, float %src1, float 0.0)
   %neg.med3 = fsub float -0.0, %med3
@@ -88,8 +88,8 @@ define amdgpu_kernel void @test_fneg_fmed3_r_inv2pi_0(float addrspace(1)* %out,
 
 ; GCN-LABEL: {{^}}test_fneg_fmed3_r_inv2pi_0_foldable_user:
 ; GCN-DAG: v_bfrev_b32_e32 [[NEG0:v[0-9]+]], 1
-; GCN-DAG: s_mov_b32 [[NEG_INV:s[0-9]+]], 0xbe22f983
-; GCN: v_med3_f32 [[MED3:v[0-9]+]], -v{{[0-9]+}}, [[NEG_INV]], [[NEG0]]
+; GCN-DAG: v_mov_b32_e32 [[NEG_INV:v[0-9]+]], 0xbe22f983
+; GCN: v_med3_f32 [[MED3:v[0-9]+]], -s{{[0-9]+}}, [[NEG_INV]], [[NEG0]]
 ; GCN: v_mul_f32_e32 v{{[0-9]+}}, s{{[0-9]+}}, [[MED3]]
 define amdgpu_kernel void @test_fneg_fmed3_r_inv2pi_0_foldable_user(float addrspace(1)* %out, float %src0, float %mul.arg) #1 {
   %med3 = call float @llvm.amdgcn.fmed3.f32(float %src0, float 0x3FC45F3060000000, float 0.0)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.init.exec.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.init.exec.ll
index fb79ba8ae..f3998d9a8 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.init.exec.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.init.exec.ll
@@ -1,4 +1,5 @@
-;RUN: llc < %s -march=amdgcn -mcpu=gfx900 -verify-machineinstrs | FileCheck %s --check-prefix=GCN
+; RUN: llc < %s -march=amdgcn -mcpu=gfx900 -verify-machineinstrs | FileCheck %s --check-prefix=GCN
+; RUN: llc -march=amdgcn -mcpu=gfx1010 -mattr=-wavefrontsize32,+wavefrontsize64 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
 
 ; GCN-LABEL: {{^}}full_mask:
 ; GCN: s_mov_b64 exec, -1
@@ -51,7 +52,7 @@ main_body:
 ; GCN: s_bfm_b64 exec, s1, 0
 ; GCN: s_cmp_eq_u32 s1, 64
 ; GCN: s_cmov_b64 exec, -1
-; GCN: v_add_u32_e32 v0, s0, v0
+; GCN: v_add{{(_nc)?}}_u32_e32 v0, s0, v0
 define amdgpu_ps float @reuse_input(i32 inreg %count, i32 %a) {
 main_body:
   call void @llvm.amdgcn.init.exec.from.input(i32 %count, i32 19)
@@ -65,7 +66,7 @@ main_body:
 ; GCN: s_bfm_b64 exec, s1, 0
 ; GCN: s_cmp_eq_u32 s1, 64
 ; GCN: s_cmov_b64 exec, -1
-; GCN: v_add_u32_e32 v0, s0, v0
+; GCN: v_add{{(_nc)?}}_u32_e32 v0, s0, v0
 define amdgpu_ps float @reuse_input2(i32 inreg %count, i32 %a) {
 main_body:
   %s = add i32 %a, %count
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.init.exec.wave32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.init.exec.wave32.ll
new file mode 100644
index 000000000..3c66f6e01
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.init.exec.wave32.ll
@@ -0,0 +1,31 @@
+; RUN: llc -march=amdgcn -mcpu=gfx1010 -mattr=+wavefrontsize32,-wavefrontsize64 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX1032 %s
+
+; GCN-LABEL: {{^}}test_init_exec:
+; GFX1032: s_mov_b32 exec_lo, 0x12345
+; GFX1064: s_mov_b64 exec, 0x12345
+; GCN: v_add_f32_e32 v0,
+define amdgpu_ps float @test_init_exec(float %a, float %b) {
+main_body:
+  %s = fadd float %a, %b
+  call void @llvm.amdgcn.init.exec(i64 74565)
+  ret float %s
+}
+
+; GCN-LABEL: {{^}}test_init_exec_from_input:
+; GCN: s_bfe_u32 s0, s3, 0x70008
+; GFX1032: s_bfm_b32 exec_lo, s0, 0
+; GFX1032: s_cmp_eq_u32 s0, 32
+; GFX1032: s_cmov_b32 exec_lo, -1
+; GFX1064: s_bfm_b64 exec, s0, 0
+; GFX1064: s_cmp_eq_u32 s0, 64
+; GFX1064: s_cmov_b64 exec, -1
+; GCN: v_add_f32_e32 v0,
+define amdgpu_ps float @test_init_exec_from_input(i32 inreg, i32 inreg, i32 inreg, i32 inreg %count, float %a, float %b) {
+main_body:
+  %s = fadd float %a, %b
+  call void @llvm.amdgcn.init.exec.from.input(i32 %count, i32 8)
+  ret float %s
+}
+
+declare void @llvm.amdgcn.init.exec(i64)
+declare void @llvm.amdgcn.init.exec.from.input(i32, i32)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.interp.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.interp.f16.ll
index 37417a9bc..5d2e10756 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.interp.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.interp.f16.ll
@@ -6,8 +6,8 @@
 define amdgpu_ps half @interp_f16(float inreg %i, float inreg %j, i32 inreg %m0) #0 {
 ; GFX9-32BANK-LABEL: interp_f16:
 ; GFX9-32BANK:       ; %bb.0: ; %main_body
-; GFX9-32BANK-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-32BANK-NEXT:    s_mov_b32 m0, s2
+; GFX9-32BANK-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-32BANK-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 3
 ; GFX9-32BANK-NEXT:    v_interp_p1ll_f16 v1, v0, attr2.y
 ; GFX9-32BANK-NEXT:    v_mov_b32_e32 v2, s1
@@ -20,8 +20,8 @@ define amdgpu_ps half @interp_f16(float inreg %i, float inreg %j, i32 inreg %m0)
 ;
 ; GFX8-32BANK-LABEL: interp_f16:
 ; GFX8-32BANK:       ; %bb.0: ; %main_body
-; GFX8-32BANK-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX8-32BANK-NEXT:    s_mov_b32 m0, s2
+; GFX8-32BANK-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX8-32BANK-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 3
 ; GFX8-32BANK-NEXT:    v_interp_p1ll_f16 v1, v0, attr2.y
 ; GFX8-32BANK-NEXT:    v_mov_b32_e32 v2, s1
@@ -119,8 +119,8 @@ main_body:
 define amdgpu_ps half @interp_p2_m0_setup(float inreg %i, float inreg %j, i32 inreg %m0) #0 {
 ; GFX9-32BANK-LABEL: interp_p2_m0_setup:
 ; GFX9-32BANK:       ; %bb.0: ; %main_body
-; GFX9-32BANK-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-32BANK-NEXT:    s_mov_b32 m0, s2
+; GFX9-32BANK-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-32BANK-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 3
 ; GFX9-32BANK-NEXT:    v_interp_p1ll_f16 v0, v0, attr2.y
 ; GFX9-32BANK-NEXT:    ;;#ASMSTART
@@ -136,8 +136,8 @@ define amdgpu_ps half @interp_p2_m0_setup(float inreg %i, float inreg %j, i32 in
 ;
 ; GFX8-32BANK-LABEL: interp_p2_m0_setup:
 ; GFX8-32BANK:       ; %bb.0: ; %main_body
-; GFX8-32BANK-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX8-32BANK-NEXT:    s_mov_b32 m0, s2
+; GFX8-32BANK-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX8-32BANK-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 3
 ; GFX8-32BANK-NEXT:    v_interp_p1ll_f16 v0, v0, attr2.y
 ; GFX8-32BANK-NEXT:    ;;#ASMSTART
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.is.private.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.is.private.ll
new file mode 100644
index 000000000..f324ba98e
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.is.private.ll
@@ -0,0 +1,50 @@
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=hawaii -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CI %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9 %s
+
+; GCN-LABEL: {{^}}is_private_vgpr:
+; GCN-DAG: {{flat|global}}_load_dwordx2 v{{\[[0-9]+}}:[[PTR_HI:[0-9]+]]{{\]}}
+; CI-DAG: s_load_dword [[APERTURE:s[0-9]+]], s[4:5], 0x11
+; GFX9-DAG: s_getreg_b32 [[APERTURE:s[0-9]+]], hwreg(HW_REG_SH_MEM_BASES, 0, 16)
+; GFX9: s_lshl_b32 [[APERTURE]], [[APERTURE]], 16
+; GCN: v_cmp_eq_u32_e32 vcc, [[APERTURE]], v[[PTR_HI]]
+; GCN: v_cndmask_b32_e64 v{{[0-9]+}}, 0, 1, vcc
+define amdgpu_kernel void @is_private_vgpr(i8* addrspace(1)* %ptr.ptr) {
+  %id = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep = getelementptr inbounds i8*, i8* addrspace(1)* %ptr.ptr, i32 %id
+  %ptr = load volatile i8*, i8* addrspace(1)* %gep
+  %val = call i1 @llvm.amdgcn.is.private(i8* %ptr)
+  %ext = zext i1 %val to i32
+  store i32 %ext, i32 addrspace(1)* undef
+  ret void
+}
+
+; FIXME: setcc (zero_extend (setcc)), 1) not folded out, resulting in
+; select and vcc branch.
+
+; GCN-LABEL: {{^}}is_private_sgpr:
+; CI-DAG: s_load_dword [[APERTURE:s[0-9]+]], s[4:5], 0x11{{$}}
+; GFX9-DAG: s_getreg_b32 [[APERTURE:s[0-9]+]], hwreg(HW_REG_SH_MEM_BASES, 0, 16)
+
+; CI-DAG: s_load_dword [[PTR_HI:s[0-9]+]], s[6:7], 0x1{{$}}
+; GFX9-DAG: s_load_dword [[PTR_HI:s[0-9]+]], s[6:7], 0x4{{$}}
+; GFX9: s_lshl_b32 [[APERTURE]], [[APERTURE]], 16
+
+; GCN: v_mov_b32_e32 [[V_APERTURE:v[0-9]+]], [[APERTURE]]
+; GCN: v_cmp_eq_u32_e32 vcc, [[PTR_HI]], [[V_APERTURE]]
+; GCN: s_cbranch_vccnz
+define amdgpu_kernel void @is_private_sgpr(i8* %ptr) {
+  %val = call i1 @llvm.amdgcn.is.private(i8* %ptr)
+  br i1 %val, label %bb0, label %bb1
+
+bb0:
+  store volatile i32 0, i32 addrspace(1)* undef
+  br label %bb1
+
+bb1:
+  ret void
+}
+
+declare i32 @llvm.amdgcn.workitem.id.x() #0
+declare i1 @llvm.amdgcn.is.private(i8* nocapture) #0
+
+attributes #0 = { nounwind readnone speculatable }
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.is.shared.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.is.shared.ll
new file mode 100644
index 000000000..1371392eb
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.is.shared.ll
@@ -0,0 +1,51 @@
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=hawaii -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CI %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9 %s
+
+; GCN-LABEL: {{^}}is_local_vgpr:
+; GCN-DAG: {{flat|global}}_load_dwordx2 v{{\[[0-9]+}}:[[PTR_HI:[0-9]+]]{{\]}}
+; CI-DAG: s_load_dword [[APERTURE:s[0-9]+]], s[4:5], 0x10
+; GFX9-DAG: s_getreg_b32 [[APERTURE:s[0-9]+]], hwreg(HW_REG_SH_MEM_BASES, 16, 16)
+; GFX9: s_lshl_b32 [[APERTURE]], [[APERTURE]], 16
+
+; GCN: v_cmp_eq_u32_e32 vcc, [[APERTURE]], v[[PTR_HI]]
+; GCN: v_cndmask_b32_e64 v{{[0-9]+}}, 0, 1, vcc
+define amdgpu_kernel void @is_local_vgpr(i8* addrspace(1)* %ptr.ptr) {
+  %id = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep = getelementptr inbounds i8*, i8* addrspace(1)* %ptr.ptr, i32 %id
+  %ptr = load volatile i8*, i8* addrspace(1)* %gep
+  %val = call i1 @llvm.amdgcn.is.shared(i8* %ptr)
+  %ext = zext i1 %val to i32
+  store i32 %ext, i32 addrspace(1)* undef
+  ret void
+}
+
+; FIXME: setcc (zero_extend (setcc)), 1) not folded out, resulting in
+; select and vcc branch.
+
+; GCN-LABEL: {{^}}is_local_sgpr:
+; CI-DAG: s_load_dword [[APERTURE:s[0-9]+]], s[4:5], 0x10{{$}}
+; GFX9-DAG: s_getreg_b32 [[APERTURE:s[0-9]+]], hwreg(HW_REG_SH_MEM_BASES, 16, 16)
+; GFX9-DAG: s_lshl_b32 [[APERTURE]], [[APERTURE]], 16
+
+; CI-DAG: s_load_dword [[PTR_HI:s[0-9]+]], s[6:7], 0x1{{$}}
+; GFX9-DAG: s_load_dword [[PTR_HI:s[0-9]+]], s[6:7], 0x4{{$}}
+
+; GCN: v_mov_b32_e32 [[V_APERTURE:v[0-9]+]], [[APERTURE]]
+; GCN: v_cmp_eq_u32_e32 vcc, [[PTR_HI]], [[V_APERTURE]]
+; GCN: s_cbranch_vccnz
+define amdgpu_kernel void @is_local_sgpr(i8* %ptr) {
+  %val = call i1 @llvm.amdgcn.is.shared(i8* %ptr)
+  br i1 %val, label %bb0, label %bb1
+
+bb0:
+  store volatile i32 0, i32 addrspace(1)* undef
+  br label %bb1
+
+bb1:
+  ret void
+}
+
+declare i32 @llvm.amdgcn.workitem.id.x() #0
+declare i1 @llvm.amdgcn.is.shared(i8* nocapture) #0
+
+attributes #0 = { nounwind readnone speculatable }
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll
index 5ac03632f..25742666a 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll
@@ -1,4 +1,5 @@
-; RUN: llc -march=amdgcn -mcpu=gfx908 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -march=amdgcn -mcpu=gfx908 -verify-machineinstrs < %s | FileCheck -enable-var-scope --check-prefixes=GCN,NOLIT-SRCC %s
+; RUN: llc -march=amdgcn -mcpu=gfx908 -mattr=-mfma-inline-literal-bug -verify-machineinstrs < %s | FileCheck -enable-var-scope --check-prefixes=GCN,LIT-SRCC %s
 
 declare <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float, float, <32 x float>, i32, i32, i32)
 declare <16 x float> @llvm.amdgcn.mfma.f32.16x16x1f32(float, float, <16 x float>, i32, i32, i32)
@@ -993,7 +994,12 @@ bb:
 ; GCN-LABEL: {{^}}test_mfma_f32_4x4x1f32_imm_splat:
 ; GCN-DAG: v_mov_b32_e32 [[TWO:v[0-9]+]], 2.0
 ; GCN-DAG: v_mov_b32_e32 [[ONE:v[0-9]+]], 1.0
-; GCN: v_mfma_f32_4x4x1f32 a[{{[0-9]+:[0-9]+}}], [[ONE]], [[TWO]], 1.0
+; NOLIT-SRCC-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, 1.0
+; NOLIT-SRCC-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, 1.0
+; NOLIT-SRCC-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, 1.0
+; NOLIT-SRCC-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, 1.0
+; NOLIT-SRCC: v_mfma_f32_4x4x1f32 a[{{[0-9]+:[0-9]+}}], [[ONE]], [[TWO]], a[{{[0-9:]+}}]
+; LIT-SRCC: v_mfma_f32_4x4x1f32 a[{{[0-9]+:[0-9]+}}], [[ONE]], [[TWO]], 1.0
 ; GCN: v_accvgpr_read_b32
 ; GCN: v_accvgpr_read_b32
 ; GCN: v_accvgpr_read_b32
@@ -1009,7 +1015,9 @@ bb:
 ; GCN-LABEL: {{^}}test_mfma_f32_16x16x1f32_imm_splat:
 ; GCN-DAG: v_mov_b32_e32 [[TWO:v[0-9]+]], 2.0
 ; GCN-DAG: v_mov_b32_e32 [[ONE:v[0-9]+]], 1.0
-; GCN: v_mfma_f32_16x16x1f32 a[{{[0-9]+:[0-9]+}}], [[ONE]], [[TWO]], 1.0
+; NOLIT-SRCC-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, 1.0
+; NOLIT-SRCC: v_mfma_f32_16x16x1f32 a[{{[0-9]+:[0-9]+}}], [[ONE]], [[TWO]], a[{{[0-9:]+}}]
+; LIT-SRCC: v_mfma_f32_16x16x1f32 a[{{[0-9]+:[0-9]+}}], [[ONE]], [[TWO]], 1.0
 ; GCN-DAG: v_accvgpr_read_b32
 ; GCN-DAG: v_accvgpr_read_b32
 ; GCN-DAG: v_accvgpr_read_b32
@@ -1040,7 +1048,9 @@ bb:
 ; GCN-LABEL: {{^}}test_mfma_f32_32x32x8f16_imm_splat:
 ; GCN-DAG: v_mov_b32_e32 v[[TWO:[0-9]+]], 0x40004000
 ; GCN-DAG: v_mov_b32_e32 v[[ONE:[0-9]+]], 0x3c003c00
-; GCN: v_mfma_f32_32x32x8f16 a[{{[0-9]+:[0-9]+}}], v{{\[}}[[ONE]]:{{[0-9]+}}], v{{\[}}[[TWO]]:{{[0-9]+}}], 1.0
+; NOLIT-SRCC-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, 1.0
+; NOLIT-SRCC: v_mfma_f32_32x32x8f16 a[{{[0-9]+:[0-9]+}}], v{{\[}}[[ONE]]:{{[0-9]+}}], v{{\[}}[[TWO]]:{{[0-9]+}}], a[{{[0-9:]+}}]
+; LIT-SRCC: v_mfma_f32_32x32x8f16 a[{{[0-9]+:[0-9]+}}], v{{\[}}[[ONE]]:{{[0-9]+}}], v{{\[}}[[TWO]]:{{[0-9]+}}], 1.0
 ; GCN-DAG: v_accvgpr_read_b32
 ; GCN-DAG: v_accvgpr_read_b32
 ; GCN-DAG: v_accvgpr_read_b32
@@ -1071,7 +1081,9 @@ bb:
 ; GCN-LABEL: {{^}}test_mfma_f32_32x32x1f32_imm_splat:
 ; GCN-DAG: v_mov_b32_e32 [[TWO:v[0-9]+]], 2.0
 ; GCN-DAG: v_mov_b32_e32 [[ONE:v[0-9]+]], 1.0
-; GCN: v_mfma_f32_32x32x1f32 a[{{[0-9]+:[0-9]+}}], [[ONE]], [[TWO]], 0
+; NOLIT-SRCC-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, 0
+; NOLIT-SRCC: v_mfma_f32_32x32x1f32 a[{{[0-9]+:[0-9]+}}], [[ONE]], [[TWO]], a[{{[0-9:]+}}]
+; LIT-SRCC: v_mfma_f32_32x32x1f32 a[{{[0-9]+:[0-9]+}}], [[ONE]], [[TWO]], 0
 ; GCN-DAG: v_accvgpr_read_b32
 ; GCN-DAG: v_accvgpr_read_b32
 ; GCN-DAG: v_accvgpr_read_b32
@@ -1283,73 +1295,35 @@ bb:
 }
 
 ; GCN-LABEL: {{^}}test_mfma_f32_32x32x1f32_vecarg:
-; GCN-DAG: v_mov_b32_e32 [[TWO:v[0-9]+]], 2.0
-; GCN-DAG: v_mov_b32_e32 [[ONE:v[0-9]+]], 1.0
-; GDN-DAG: global_load_dwordx4
-; GDN-DAG: global_load_dwordx4
-; GDN-DAG: global_load_dwordx4
-; GDN-DAG: global_load_dwordx4
-; GDN-DAG: global_load_dwordx4
-; GDN-DAG: global_load_dwordx4
-; GDN-DAG: global_load_dwordx4
-; GDN-DAG: global_load_dwordx4
-; GCN-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}}
-; GCN-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}}
-; GCN-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}}
-; GCN-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}}
-; GCN-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}}
-; GCN-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}}
-; GCN-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}}
-; GCN-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}}
-; GCN-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}}
-; GCN-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}}
-; GCN-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}}
-; GCN-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}}
-; GCN-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}}
-; GCN-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}}
-; GCN-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}}
-; GCN-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}}
-; GCN: v_mfma_f32_32x32x1f32 a[{{[0-9]+:[0-9]+}}], [[ONE]], [[TWO]], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3
-; GCN-DAG: v_accvgpr_read_b32
-; GCN-DAG: v_accvgpr_read_b32
-; GCN-DAG: v_accvgpr_read_b32
-; GCN-DAG: v_accvgpr_read_b32
-; GCN-DAG: v_accvgpr_read_b32
-; GCN-DAG: v_accvgpr_read_b32
-; GCN-DAG: v_accvgpr_read_b32
-; GCN-DAG: v_accvgpr_read_b32
-; GCN-DAG: v_accvgpr_read_b32
-; GCN-DAG: v_accvgpr_read_b32
-; GCN-DAG: v_accvgpr_read_b32
-; GCN-DAG: v_accvgpr_read_b32
-; GCN-DAG: v_accvgpr_read_b32
-; GCN-DAG: v_accvgpr_read_b32
-; GCN-DAG: v_accvgpr_read_b32
-; GCN-DAG: v_accvgpr_read_b32
-; GCN-DAG: v_accvgpr_read_b32
-; GCN-DAG: v_accvgpr_read_b32
-; GCN-DAG: v_accvgpr_read_b32
-; GCN-DAG: v_accvgpr_read_b32
-; GCN-DAG: v_accvgpr_read_b32
-; GCN-DAG: v_accvgpr_read_b32
-; GCN-DAG: v_accvgpr_read_b32
-; GCN-DAG: v_accvgpr_read_b32
-; GCN-DAG: v_accvgpr_read_b32
-; GCN-DAG: v_accvgpr_read_b32
-; GCN-DAG: v_accvgpr_read_b32
-; GCN-DAG: v_accvgpr_read_b32
-; GCN-DAG: v_accvgpr_read_b32
-; GCN-DAG: v_accvgpr_read_b32
-; GCN-DAG: v_accvgpr_read_b32
-; GCN-DAG: v_accvgpr_read_b32
-; GCN-DAG: global_store_dwordx4
-; GCN-DAG: global_store_dwordx4
-; GCN-DAG: global_store_dwordx4
-; GCN-DAG: global_store_dwordx4
-; GCN-DAG: global_store_dwordx4
-; GCN-DAG: global_store_dwordx4
-; GCN-DAG: global_store_dwordx4
-; GCN-DAG: global_store_dwordx4
+; GCN-DAG:         v_mov_b32_e32 [[TWO:v[0-9]+]], 2.0
+; GCN-DAG:         v_mov_b32_e32 [[ONE:v[0-9]+]], 1.0
+; GCN-DAG:         global_load_dwordx4
+; GCN-DAG:         global_load_dwordx4
+; GCN-DAG:         global_load_dwordx4
+; GCN-DAG:         global_load_dwordx4
+; GCN-DAG:         global_load_dwordx4
+; GCN-DAG:         global_load_dwordx4
+; GCN-DAG:         global_load_dwordx4
+; GCN-DAG:         global_load_dwordx4
+; GCN-DAG:         v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}}
+; GCN-DAG:         v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}}
+; GCN-DAG:         v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}}
+; GCN-DAG:         v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}}
+; GCN-DAG:         v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}}
+; GCN-DAG:         v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}}
+; GCN-DAG:         v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}}
+; GCN-DAG:         v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}}
+; GCN-DAG:         v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}}
+; GCN-DAG:         v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}}
+; GCN-DAG:         v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}}
+; GCN-DAG:         v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}}
+; GCN-DAG:         v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}}
+; GCN-DAG:         v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}}
+; GCN-DAG:         v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}}
+; GCN-DAG:         v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}}
+; GCN:             v_mfma_f32_32x32x1f32 a[{{[0-9]+:[0-9]+}}], [[ONE]], [[TWO]], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3
+; GCN-COUNT-32:    v_accvgpr_read_b32
+; GCN-COUNT-8:     global_store_dwordx4
 define amdgpu_kernel void @test_mfma_f32_32x32x1f32_vecarg(<32 x float> addrspace(1)* %arg) {
 bb:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mov.dpp.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mov.dpp.ll
index 752357998..31a54f1e0 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mov.dpp.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mov.dpp.ll
@@ -43,6 +43,7 @@ define amdgpu_kernel void @dpp_wait_states(i32 addrspace(1)* %out, i32 %in) {
 ; PREGFX10-OPT: s_mov_b32
 ; PREGFX10-OPT: s_mov_b32
 ; PREGFX10-NOOPT: s_waitcnt
+; PREGFX10-NOOPT: v_mov_b32_e32
 ; PREGFX10-NOOPT-NEXT: s_nop 0
 ; VI: v_mov_b32_dpp [[VGPR0:v[0-9]+]], v{{[0-9]+}} quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1 bound_ctrl:0
 ; PREGFX10-OPT: s_nop 1
@@ -76,7 +77,31 @@ endif:
   ret void
 }
 
+; VI-LABEL: {{^}}mov_dpp64_test:
+; VI: v_mov_b32_dpp v{{[0-9]+}}, v{{[0-9]+}} quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1
+; VI: v_mov_b32_dpp v{{[0-9]+}}, v{{[0-9]+}} quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1
+define amdgpu_kernel void @mov_dpp64_test(i64 addrspace(1)* %out, i64 %in1) {
+  %tmp0 = call i64 @llvm.amdgcn.mov.dpp.i64(i64 %in1, i32 1, i32 1, i32 1, i1 0) #0
+  store i64 %tmp0, i64 addrspace(1)* %out
+  ret void
+}
+
+; VI-LABEL: {{^}}mov_dpp64_imm_test:
+; VI-OPT-DAG: s_mov_b32 s[[SOLD_LO:[0-9]+]], 0x3afaedd9
+; VI-OPT-DAG: s_movk_i32 s[[SOLD_HI:[0-9]+]], 0x7047
+; VI-OPT-DAG: v_mov_b32_e32 v[[OLD_LO:[0-9]+]], s[[SOLD_LO]]
+; VI-OPT-DAG: v_mov_b32_e32 v[[OLD_HI:[0-9]+]], s[[SOLD_HI]]
+; VI-OPT-DAG: v_mov_b32_dpp v[[OLD_LO]], v[[OLD_LO]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1
+; VI-OPT-DAG: v_mov_b32_dpp v[[OLD_HI]], v[[OLD_HI]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1
+; VI-NOOPT-COUNT-2: v_mov_b32_dpp v{{[0-9]+}}, v{{[0-9]+}} quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1
+define amdgpu_kernel void @mov_dpp64_imm_test(i64 addrspace(1)* %out) {
+  %tmp0 = call i64 @llvm.amdgcn.mov.dpp.i64(i64 123451234512345, i32 1, i32 1, i32 1, i1 0) #0
+  store i64 %tmp0, i64 addrspace(1)* %out
+  ret void
+}
+
 declare i32 @llvm.amdgcn.mov.dpp.i32(i32, i32, i32, i32, i1) #0
+declare i64 @llvm.amdgcn.mov.dpp.i64(i64, i32, i32, i32, i1) #0
 
 attributes #0 = { nounwind readnone convergent }
 
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mqsad.pk.u16.u8.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mqsad.pk.u16.u8.ll
index bc04f6f28..83bc8b234 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mqsad.pk.u16.u8.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mqsad.pk.u16.u8.ll
@@ -4,7 +4,7 @@
 declare i64 @llvm.amdgcn.mqsad.pk.u16.u8(i64, i32, i64) #0
 
 ; GCN-LABEL: {{^}}v_mqsad_pk_u16_u8:
-; GCN: v_mqsad_pk_u16_u8 v[0:1], v[4:5], v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}]
+; GCN: v_mqsad_pk_u16_u8 v[0:1], v[4:5], s{{[0-9]+}}, v[{{[0-9]+:[0-9]+}}]
 ; GCN-DAG: v_mov_b32_e32 v5, v1
 ; GCN-DAG: v_mov_b32_e32 v4, v0
 define amdgpu_kernel void @v_mqsad_pk_u16_u8(i64 addrspace(1)* %out, i64 %src) {
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.qsad.pk.u16.u8.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.qsad.pk.u16.u8.ll
index 2cab9c28d..1f46613a8 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.qsad.pk.u16.u8.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.qsad.pk.u16.u8.ll
@@ -4,7 +4,7 @@
 declare i64 @llvm.amdgcn.qsad.pk.u16.u8(i64, i32, i64) #0
 
 ; GCN-LABEL: {{^}}v_qsad_pk_u16_u8:
-; GCN: v_qsad_pk_u16_u8 v[0:1], v[4:5], v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}]
+; GCN: v_qsad_pk_u16_u8 v[0:1], v[4:5], s{{[0-9]+}}, v[{{[0-9]+:[0-9]+}}]
 ; GCN-DAG: v_mov_b32_e32 v5, v1
 ; GCN-DAG: v_mov_b32_e32 v4, v0
 define amdgpu_kernel void @v_qsad_pk_u16_u8(i64 addrspace(1)* %out, i64 %src) {
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.atomic.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.atomic.ll
index 08edaf989..3c5395029 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.atomic.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.atomic.ll
@@ -44,6 +44,10 @@ main_body:
 ;CHECK: buffer_atomic_or v0, v1, s[0:3], 0 offen glc slc
 ;CHECK: s_waitcnt vmcnt(0)
 ;CHECK: buffer_atomic_xor v0, v1, s[0:3], 0 offen glc
+;CHECK: s_waitcnt vmcnt(0)
+;CHECK: buffer_atomic_inc v0, v1, s[0:3], 0 offen glc
+;CHECK: s_waitcnt vmcnt(0)
+;CHECK: buffer_atomic_dec v0, v1, s[0:3], 0 offen glc
 define amdgpu_ps float @test2(<4 x i32> inreg %rsrc, i32 %data, i32 %voffset) {
 main_body:
   %t1 = call i32 @llvm.amdgcn.raw.buffer.atomic.add.i32(i32 %data, <4 x i32> %rsrc, i32 %voffset, i32 0, i32 0)
@@ -55,7 +59,9 @@ main_body:
   %t7 = call i32 @llvm.amdgcn.raw.buffer.atomic.and.i32(i32 %t6, <4 x i32> %rsrc, i32 %voffset, i32 0, i32 0)
   %t8 = call i32 @llvm.amdgcn.raw.buffer.atomic.or.i32(i32 %t7, <4 x i32> %rsrc, i32 %voffset, i32 0, i32 2)
   %t9 = call i32 @llvm.amdgcn.raw.buffer.atomic.xor.i32(i32 %t8, <4 x i32> %rsrc, i32 %voffset, i32 0, i32 0)
-  %out = bitcast i32 %t9 to float
+  %t10 = call i32 @llvm.amdgcn.raw.buffer.atomic.inc.i32(i32 %t9, <4 x i32> %rsrc, i32 %voffset, i32 0, i32 0)
+  %t11 = call i32 @llvm.amdgcn.raw.buffer.atomic.dec.i32(i32 %t10, <4 x i32> %rsrc, i32 %voffset, i32 0, i32 0)
+  %out = bitcast i32 %t11 to float
   ret float %out
 }
 
@@ -110,6 +116,8 @@ declare i32 @llvm.amdgcn.raw.buffer.atomic.umax.i32(i32, <4 x i32>, i32, i32, i3
 declare i32 @llvm.amdgcn.raw.buffer.atomic.and.i32(i32, <4 x i32>, i32, i32, i32) #0
 declare i32 @llvm.amdgcn.raw.buffer.atomic.or.i32(i32, <4 x i32>, i32, i32, i32) #0
 declare i32 @llvm.amdgcn.raw.buffer.atomic.xor.i32(i32, <4 x i32>, i32, i32, i32) #0
+declare i32 @llvm.amdgcn.raw.buffer.atomic.inc.i32(i32, <4 x i32>, i32, i32, i32) #0
+declare i32 @llvm.amdgcn.raw.buffer.atomic.dec.i32(i32, <4 x i32>, i32, i32, i32) #0
 declare i32 @llvm.amdgcn.raw.buffer.atomic.cmpswap.i32(i32, i32, <4 x i32>, i32, i32, i32) #0
 
 attributes #0 = { nounwind }
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.load.ll
index b241f8b98..9f18f4df4 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.load.ll
@@ -298,13 +298,13 @@ main_body:
   ret float %val
 }
 
-;CHECK-LABEL: {{^}}raw_buffer_load_ushort:
+;CHECK-LABEL: {{^}}raw_buffer_load_i16:
 ;CHECK-NEXT: %bb.
 ;CHECK-NEXT: buffer_load_ushort v{{[0-9]}}, off, s[0:3], 0
 ;CHECK: s_waitcnt vmcnt(0)
 ;CHECK-NEXT: v_cvt_f32_u32_e32 v0, v0
 ;CHECK-NEXT: ; return to shader part epilog
-define amdgpu_ps float @raw_buffer_load_ushort(<4 x i32> inreg %rsrc) {
+define amdgpu_ps float @raw_buffer_load_i16(<4 x i32> inreg %rsrc) {
 main_body:
   %tmp = call i16 @llvm.amdgcn.raw.buffer.load.i16(<4 x i32> %rsrc, i32 0, i32 0, i32 0)
   %tmp2 = zext i16 %tmp to i32
@@ -340,6 +340,106 @@ main_body:
   ret float %val
 }
 
+;CHECK-LABEL: {{^}}raw_buffer_load_f16:
+;CHECK-NEXT: %bb.
+;CHECK-NEXT: buffer_load_ushort [[VAL:v[0-9]+]], off, s[0:3], 0
+;CHECK: s_waitcnt vmcnt(0)
+;CHECK: ds_write_b16 v0, [[VAL]]
+define amdgpu_ps void @raw_buffer_load_f16(<4 x i32> inreg %rsrc, half addrspace(3)* %ptr) {
+main_body:
+  %val = call half @llvm.amdgcn.raw.buffer.load.f16(<4 x i32> %rsrc, i32 0, i32 0, i32 0)
+  store half %val, half addrspace(3)* %ptr
+  ret void
+}
+
+;CHECK-LABEL: {{^}}raw_buffer_load_v2f16:
+;CHECK-NEXT: %bb.
+;CHECK-NEXT: buffer_load_dword [[VAL:v[0-9]+]], off, s[0:3], 0
+;CHECK: s_waitcnt vmcnt(0)
+;CHECK: ds_write_b32 v0, [[VAL]]
+define amdgpu_ps void @raw_buffer_load_v2f16(<4 x i32> inreg %rsrc, <2 x half> addrspace(3)* %ptr) {
+main_body:
+  %val = call <2 x half> @llvm.amdgcn.raw.buffer.load.v2f16(<4 x i32> %rsrc, i32 0, i32 0, i32 0)
+  store <2 x half> %val, <2 x half> addrspace(3)* %ptr
+  ret void
+}
+
+;CHECK-LABEL: {{^}}raw_buffer_load_v4f16:
+;CHECK-NEXT: %bb.
+;CHECK-NEXT: buffer_load_dwordx2 [[VAL:v\[[0-9]+:[0-9]+\]]], off, s[0:3], 0
+;CHECK: s_waitcnt vmcnt(0)
+;CHECK: ds_write_b64 v0, [[VAL]]
+define amdgpu_ps void @raw_buffer_load_v4f16(<4 x i32> inreg %rsrc, <4 x half> addrspace(3)* %ptr) {
+main_body:
+  %val = call <4 x half> @llvm.amdgcn.raw.buffer.load.v4f16(<4 x i32> %rsrc, i32 0, i32 0, i32 0)
+  store <4 x half> %val, <4 x half> addrspace(3)* %ptr
+  ret void
+}
+
+;CHECK-LABEL: {{^}}raw_buffer_load_v2i16:
+;CHECK-NEXT: %bb.
+;CHECK-NEXT: buffer_load_dword [[VAL:v[0-9]+]], off, s[0:3], 0
+;CHECK: s_waitcnt vmcnt(0)
+;CHECK: ds_write_b32 v0, [[VAL]]
+define amdgpu_ps void @raw_buffer_load_v2i16(<4 x i32> inreg %rsrc, <2 x i16> addrspace(3)* %ptr) {
+main_body:
+  %val = call <2 x i16> @llvm.amdgcn.raw.buffer.load.v2i16(<4 x i32> %rsrc, i32 0, i32 0, i32 0)
+  store <2 x i16> %val, <2 x i16> addrspace(3)* %ptr
+  ret void
+}
+
+;CHECK-LABEL: {{^}}raw_buffer_load_v4i16:
+;CHECK-NEXT: %bb.
+;CHECK-NEXT: buffer_load_dwordx2 [[VAL:v\[[0-9]+:[0-9]+\]]], off, s[0:3], 0
+;CHECK: s_waitcnt vmcnt(0)
+;CHECK: ds_write_b64 v0, [[VAL]]
+define amdgpu_ps void @raw_buffer_load_v4i16(<4 x i32> inreg %rsrc, <4 x i16> addrspace(3)* %ptr) {
+main_body:
+  %val = call <4 x i16> @llvm.amdgcn.raw.buffer.load.v4i16(<4 x i32> %rsrc, i32 0, i32 0, i32 0)
+  store <4 x i16> %val, <4 x i16> addrspace(3)* %ptr
+  ret void
+}
+
+;CHECK-LABEL: {{^}}raw_buffer_load_x1_offset_merged:
+;CHECK-NEXT: %bb.
+;CHECK-NEXT: buffer_load_dwordx4 v[{{[0-9]}}:{{[0-9]}}], off, s[0:3], 0 offset:4
+;CHECK-NEXT: buffer_load_dwordx2 v[{{[0-9]}}:{{[0-9]}}], off, s[0:3], 0 offset:28
+;CHECK: s_waitcnt
+define amdgpu_ps void @raw_buffer_load_x1_offset_merged(<4 x i32> inreg %rsrc) {
+main_body:
+  %r1 = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 4, i32 0, i32 0)
+  %r2 = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 8, i32 0, i32 0)
+  %r3 = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 12, i32 0, i32 0)
+  %r4 = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 16, i32 0, i32 0)
+  %r5 = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 28, i32 0, i32 0)
+  %r6 = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 32, i32 0, i32 0)
+  call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r1, float %r2, float %r3, float %r4, i1 true, i1 true)
+  call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r5, float %r6, float undef, float undef, i1 true, i1 true)
+  ret void
+}
+
+;CHECK-LABEL: {{^}}raw_buffer_load_x1_offset_swizzled_not_merged:
+;CHECK-NEXT: %bb.
+;CHECK-NEXT: buffer_load_dword v{{[0-9]}}, off, s[0:3], 0 offset:4
+;CHECK-NEXT: buffer_load_dword v{{[0-9]}}, off, s[0:3], 0 offset:8
+;CHECK-NEXT: buffer_load_dword v{{[0-9]}}, off, s[0:3], 0 offset:12
+;CHECK-NEXT: buffer_load_dword v{{[0-9]}}, off, s[0:3], 0 offset:16
+;CHECK-NEXT: buffer_load_dword v{{[0-9]}}, off, s[0:3], 0 offset:28
+;CHECK-NEXT: buffer_load_dword v{{[0-9]}}, off, s[0:3], 0 offset:32
+;CHECK: s_waitcnt
+define amdgpu_ps void @raw_buffer_load_x1_offset_swizzled_not_merged(<4 x i32> inreg %rsrc) {
+main_body:
+  %r1 = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 4, i32 0, i32 8)
+  %r2 = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 8, i32 0, i32 8)
+  %r3 = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 12, i32 0, i32 8)
+  %r4 = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 16, i32 0, i32 8)
+  %r5 = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 28, i32 0, i32 8)
+  %r6 = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 32, i32 0, i32 8)
+  call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r1, float %r2, float %r3, float %r4, i1 true, i1 true)
+  call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r5, float %r6, float undef, float undef, i1 true, i1 true)
+  ret void
+}
+
 declare float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32>, i32, i32, i32) #0
 declare <2 x float> @llvm.amdgcn.raw.buffer.load.v2f32(<4 x i32>, i32, i32, i32) #0
 declare <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32>, i32, i32, i32) #0
@@ -349,5 +449,10 @@ declare <4 x i32> @llvm.amdgcn.raw.buffer.load.v4i32(<4 x i32>, i32, i32, i32) #
 declare void @llvm.amdgcn.exp.f32(i32, i32, float, float, float, float, i1, i1) #0
 declare i8 @llvm.amdgcn.raw.buffer.load.i8(<4 x i32>, i32, i32, i32) #0
 declare i16 @llvm.amdgcn.raw.buffer.load.i16(<4 x i32>, i32, i32, i32) #0
+declare <2 x i16> @llvm.amdgcn.raw.buffer.load.v2i16(<4 x i32>, i32, i32, i32) #0
+declare <4 x i16> @llvm.amdgcn.raw.buffer.load.v4i16(<4 x i32>, i32, i32, i32) #0
+declare half @llvm.amdgcn.raw.buffer.load.f16(<4 x i32>, i32, i32, i32) #0
+declare <2 x half> @llvm.amdgcn.raw.buffer.load.v2f16(<4 x i32>, i32, i32, i32) #0
+declare <4 x half> @llvm.amdgcn.raw.buffer.load.v4f16(<4 x i32>, i32, i32, i32) #0
 
 attributes #0 = { nounwind readonly }
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.store.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.store.ll
index 89728f255..1bfe0aa40 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.store.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.store.ll
@@ -215,6 +215,98 @@ main_body:
   ret void
 }
 
+;CHECK-LABEL: {{^}}raw_buffer_store_f16:
+;CHECK-NEXT: %bb.
+;CHECK-NOT: v0
+;CHECK-NEXT: buffer_store_short v0, off, s[0:3], 0
+;CHECK-NEXT: s_endpgm
+define amdgpu_ps void @raw_buffer_store_f16(<4 x i32> inreg %rsrc, i32 %v1) {
+main_body:
+  %trunc = trunc i32 %v1 to i16
+  %cast = bitcast i16 %trunc to half
+  call void @llvm.amdgcn.raw.buffer.store.f16(half %cast, <4 x i32> %rsrc, i32 0, i32 0, i32 0)
+  ret void
+}
+
+;CHECK-LABEL: {{^}}buffer_store_v2f16:
+;CHECK-NOT: s_waitcnt
+;CHECK: buffer_store_dword v0, v1, s[0:3], 0 offen
+define amdgpu_ps void @buffer_store_v2f16(<4 x i32> inreg %rsrc, <2 x half> %data, i32 %offset) {
+main_body:
+  call void @llvm.amdgcn.raw.buffer.store.v2f16(<2 x half> %data, <4 x i32> %rsrc, i32 %offset, i32 0, i32 0)
+  ret void
+}
+
+;CHECK-LABEL: {{^}}buffer_store_v4f16:
+;CHECK-NOT: s_waitcnt
+;CHECK: buffer_store_dwordx2 v[0:1], v2, s[0:3], 0 offen
+define amdgpu_ps void @buffer_store_v4f16(<4 x i32> inreg %rsrc, <4 x half> %data, i32 %offset) #0 {
+main_body:
+  call void @llvm.amdgcn.raw.buffer.store.v4f16(<4 x half> %data, <4 x i32> %rsrc, i32 %offset, i32 0, i32 0)
+  ret void
+}
+
+;CHECK-LABEL: {{^}}raw_buffer_store_i16:
+;CHECK-NEXT: %bb.
+;CHECK-NOT: v0
+;CHECK-NEXT: buffer_store_short v0, off, s[0:3], 0
+;CHECK-NEXT: s_endpgm
+define amdgpu_ps void @raw_buffer_store_i16(<4 x i32> inreg %rsrc, i32 %v1) {
+main_body:
+  %trunc = trunc i32 %v1 to i16
+  call void @llvm.amdgcn.raw.buffer.store.i16(i16 %trunc, <4 x i32> %rsrc, i32 0, i32 0, i32 0)
+  ret void
+}
+
+;CHECK-LABEL: {{^}}buffer_store_v2i16:
+;CHECK-NOT: s_waitcnt
+;CHECK: buffer_store_dword v0, v1, s[0:3], 0 offen
+define amdgpu_ps void @buffer_store_v2i16(<4 x i32> inreg %rsrc, <2 x i16> %data, i32 %offset) {
+main_body:
+  call void @llvm.amdgcn.raw.buffer.store.v2i16(<2 x i16> %data, <4 x i32> %rsrc, i32 %offset, i32 0, i32 0)
+  ret void
+}
+
+;CHECK-LABEL: {{^}}buffer_store_v4i16:
+;CHECK-NOT: s_waitcnt
+;CHECK: buffer_store_dwordx2 v[0:1], v2, s[0:3], 0 offen
+define amdgpu_ps void @buffer_store_v4i16(<4 x i32> inreg %rsrc, <4 x i16> %data, i32 %offset) #0 {
+main_body:
+  call void @llvm.amdgcn.raw.buffer.store.v4i16(<4 x i16> %data, <4 x i32> %rsrc, i32 %offset, i32 0, i32 0)
+  ret void
+}
+
+;CHECK-LABEL: {{^}}raw_buffer_store_x1_offset_merged:
+;CHECK-NOT: s_waitcnt
+;CHECK-DAG: buffer_store_dwordx4 v[{{[0-9]}}:{{[0-9]}}], off, s[0:3], 0 offset:4
+;CHECK-DAG: buffer_store_dwordx2 v[{{[0-9]}}:{{[0-9]}}], off, s[0:3], 0 offset:28
+define amdgpu_ps void @raw_buffer_store_x1_offset_merged(<4 x i32> inreg %rsrc, float %v1, float %v2, float %v3, float %v4, float %v5, float %v6) {
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %v1, <4 x i32> %rsrc, i32 4, i32 0, i32 0)
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %v2, <4 x i32> %rsrc, i32 8, i32 0, i32 0)
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %v3, <4 x i32> %rsrc, i32 12, i32 0, i32 0)
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %v4, <4 x i32> %rsrc, i32 16, i32 0, i32 0)
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %v5, <4 x i32> %rsrc, i32 28, i32 0, i32 0)
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %v6, <4 x i32> %rsrc, i32 32, i32 0, i32 0)
+  ret void
+}
+
+;CHECK-LABEL: {{^}}raw_buffer_store_x1_offset_swizzled_not_merged:
+;CHECK-DAG: buffer_store_dword v{{[0-9]}}, off, s[0:3], 0 offset:4
+;CHECK-DAG: buffer_store_dword v{{[0-9]}}, off, s[0:3], 0 offset:8
+;CHECK-DAG: buffer_store_dword v{{[0-9]}}, off, s[0:3], 0 offset:12
+;CHECK-DAG: buffer_store_dword v{{[0-9]}}, off, s[0:3], 0 offset:16
+;CHECK-DAG: buffer_store_dword v{{[0-9]}}, off, s[0:3], 0 offset:28
+;CHECK-DAG: buffer_store_dword v{{[0-9]}}, off, s[0:3], 0 offset:32
+define amdgpu_ps void @raw_buffer_store_x1_offset_swizzled_not_merged(<4 x i32> inreg %rsrc, float %v1, float %v2, float %v3, float %v4, float %v5, float %v6) {
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %v1, <4 x i32> %rsrc, i32 4, i32 0, i32 8)
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %v2, <4 x i32> %rsrc, i32 8, i32 0, i32 8)
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %v3, <4 x i32> %rsrc, i32 12, i32 0, i32 8)
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %v4, <4 x i32> %rsrc, i32 16, i32 0, i32 8)
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %v5, <4 x i32> %rsrc, i32 28, i32 0, i32 8)
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %v6, <4 x i32> %rsrc, i32 32, i32 0, i32 8)
+  ret void
+}
+
 declare void @llvm.amdgcn.raw.buffer.store.f32(float, <4 x i32>, i32, i32, i32) #0
 declare void @llvm.amdgcn.raw.buffer.store.v2f32(<2 x float>, <4 x i32>, i32, i32, i32) #0
 declare void @llvm.amdgcn.raw.buffer.store.v4f32(<4 x float>, <4 x i32>, i32, i32, i32) #0
@@ -223,7 +315,12 @@ declare void @llvm.amdgcn.raw.buffer.store.v2i32(<2 x i32>, <4 x i32>, i32, i32,
 declare void @llvm.amdgcn.raw.buffer.store.v4i32(<4 x i32>, <4 x i32>, i32, i32, i32) #0
 declare <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32>, i32, i32, i32) #1
 declare void @llvm.amdgcn.raw.buffer.store.i8(i8, <4 x i32>, i32, i32, i32) #0
+declare void @llvm.amdgcn.raw.buffer.store.f16(half, <4 x i32>, i32, i32, i32) #0
+declare void @llvm.amdgcn.raw.buffer.store.v2f16(<2 x half>, <4 x i32>, i32, i32, i32) #0
+declare void @llvm.amdgcn.raw.buffer.store.v4f16(<4 x half>, <4 x i32>, i32, i32, i32) #0
 declare void @llvm.amdgcn.raw.buffer.store.i16(i16, <4 x i32>, i32, i32, i32) #0
+declare void @llvm.amdgcn.raw.buffer.store.v2i16(<2 x i16>, <4 x i32>, i32, i32, i32) #0
+declare void @llvm.amdgcn.raw.buffer.store.v4i16(<4 x i16>, <4 x i32>, i32, i32, i32) #0
 
 attributes #0 = { nounwind }
 attributes #1 = { nounwind readonly }
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.readfirstlane.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.readfirstlane.ll
index f28f40c32..991afdb6c 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.readfirstlane.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.readfirstlane.ll
@@ -30,11 +30,9 @@ define amdgpu_kernel void @test_readfirstlane_imm_fold(i32 addrspace(1)* %out) #
   ret void
 }
 
-; TODO: m0 should be folded.
 ; CHECK-LABEL: {{^}}test_readfirstlane_m0:
 ; CHECK: s_mov_b32 m0, -1
-; CHECK: s_mov_b32 [[COPY_M0:s[0-9]+]], m0
-; CHECK: v_mov_b32_e32 [[VVAL:v[0-9]]], [[COPY_M0]]
+; CHECK: v_mov_b32_e32 [[VVAL:v[0-9]]], m0
 ; CHECK: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[VVAL]]
 define amdgpu_kernel void @test_readfirstlane_m0(i32 addrspace(1)* %out) #1 {
   %m0 = call i32 asm "s_mov_b32 m0, -1", "={m0}"()
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.readlane.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.readlane.ll
index f4f48d3b2..230d7aff2 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.readlane.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.readlane.ll
@@ -44,8 +44,7 @@ define amdgpu_kernel void @test_readlane_vregs(i32 addrspace(1)* %out, <2 x i32>
 ; TODO: m0 should be folded.
 ; CHECK-LABEL: {{^}}test_readlane_m0_sreg:
 ; CHECK: s_mov_b32 m0, -1
-; CHECK: s_mov_b32 [[COPY_M0:s[0-9]+]], m0
-; CHECK: v_mov_b32_e32 [[VVAL:v[0-9]]], [[COPY_M0]]
+; CHECK: v_mov_b32_e32 [[VVAL:v[0-9]]], m0
 ; CHECK: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[VVAL]]
 define amdgpu_kernel void @test_readlane_m0_sreg(i32 addrspace(1)* %out, i32 %src1) #1 {
   %m0 = call i32 asm "s_mov_b32 m0, -1", "={m0}"()
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.barrier.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.barrier.ll
index 2dd7e20c0..0bf3125b2 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.barrier.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.barrier.ll
@@ -16,9 +16,9 @@ define amdgpu_kernel void @test_barrier(i32 addrspace(1)* %out, i32 %size) #0 {
 ; VARIANT0-NEXT:    v_mov_b32_e32 v2, 0
 ; VARIANT0-NEXT:    s_waitcnt lgkmcnt(0)
 ; VARIANT0-NEXT:    buffer_store_dword v0, v[1:2], s[4:7], 0 addr64
-; VARIANT0-NEXT:    v_add_i32_e32 v3, vcc, s2, v3
 ; VARIANT0-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
 ; VARIANT0-NEXT:    s_barrier
+; VARIANT0-NEXT:    v_add_i32_e32 v3, vcc, s2, v3
 ; VARIANT0-NEXT:    v_ashrrev_i32_e32 v4, 31, v3
 ; VARIANT0-NEXT:    v_lshl_b64 v[3:4], v[3:4], 2
 ; VARIANT0-NEXT:    buffer_load_dword v0, v[3:4], s[4:7], 0 addr64
@@ -37,8 +37,8 @@ define amdgpu_kernel void @test_barrier(i32 addrspace(1)* %out, i32 %size) #0 {
 ; VARIANT1-NEXT:    v_mov_b32_e32 v2, 0
 ; VARIANT1-NEXT:    s_waitcnt lgkmcnt(0)
 ; VARIANT1-NEXT:    buffer_store_dword v0, v[1:2], s[4:7], 0 addr64
-; VARIANT1-NEXT:    v_add_i32_e32 v3, vcc, s2, v3
 ; VARIANT1-NEXT:    s_barrier
+; VARIANT1-NEXT:    v_add_i32_e32 v3, vcc, s2, v3
 ; VARIANT1-NEXT:    v_ashrrev_i32_e32 v4, 31, v3
 ; VARIANT1-NEXT:    v_lshl_b64 v[3:4], v[3:4], 2
 ; VARIANT1-NEXT:    s_waitcnt expcnt(0)
@@ -50,46 +50,46 @@ define amdgpu_kernel void @test_barrier(i32 addrspace(1)* %out, i32 %size) #0 {
 ; VARIANT2-LABEL: test_barrier:
 ; VARIANT2:       ; %bb.0: ; %entry
 ; VARIANT2-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
-; VARIANT2-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
 ; VARIANT2-NEXT:    s_load_dword s0, s[0:1], 0x2c
+; VARIANT2-NEXT:    v_lshlrev_b32_e32 v3, 2, v0
 ; VARIANT2-NEXT:    s_waitcnt lgkmcnt(0)
-; VARIANT2-NEXT:    v_mov_b32_e32 v2, s3
-; VARIANT2-NEXT:    v_add_co_u32_e32 v1, vcc, s2, v1
-; VARIANT2-NEXT:    v_addc_co_u32_e32 v2, vcc, 0, v2, vcc
-; VARIANT2-NEXT:    global_store_dword v[1:2], v0, off
+; VARIANT2-NEXT:    v_mov_b32_e32 v4, s3
+; VARIANT2-NEXT:    v_xad_u32 v1, v0, -1, s0
+; VARIANT2-NEXT:    v_ashrrev_i32_e32 v2, 31, v1
+; VARIANT2-NEXT:    v_add_co_u32_e32 v3, vcc, s2, v3
+; VARIANT2-NEXT:    v_lshlrev_b64 v[1:2], 2, v[1:2]
+; VARIANT2-NEXT:    v_addc_co_u32_e32 v4, vcc, 0, v4, vcc
+; VARIANT2-NEXT:    global_store_dword v[3:4], v0, off
+; VARIANT2-NEXT:    v_mov_b32_e32 v5, s3
+; VARIANT2-NEXT:    v_add_co_u32_e32 v0, vcc, s2, v1
+; VARIANT2-NEXT:    v_addc_co_u32_e32 v1, vcc, v5, v2, vcc
 ; VARIANT2-NEXT:    s_waitcnt vmcnt(0)
 ; VARIANT2-NEXT:    s_barrier
-; VARIANT2-NEXT:    v_xad_u32 v3, v0, -1, s0
-; VARIANT2-NEXT:    v_ashrrev_i32_e32 v4, 31, v3
-; VARIANT2-NEXT:    v_lshlrev_b64 v[3:4], 2, v[3:4]
-; VARIANT2-NEXT:    v_mov_b32_e32 v0, s3
-; VARIANT2-NEXT:    v_add_co_u32_e32 v3, vcc, s2, v3
-; VARIANT2-NEXT:    v_addc_co_u32_e32 v4, vcc, v0, v4, vcc
-; VARIANT2-NEXT:    global_load_dword v0, v[3:4], off
+; VARIANT2-NEXT:    global_load_dword v0, v[0:1], off
 ; VARIANT2-NEXT:    s_waitcnt vmcnt(0)
-; VARIANT2-NEXT:    global_store_dword v[1:2], v0, off
+; VARIANT2-NEXT:    global_store_dword v[3:4], v0, off
 ; VARIANT2-NEXT:    s_endpgm
 ;
 ; VARIANT3-LABEL: test_barrier:
 ; VARIANT3:       ; %bb.0: ; %entry
 ; VARIANT3-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
-; VARIANT3-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
 ; VARIANT3-NEXT:    s_load_dword s0, s[0:1], 0x2c
+; VARIANT3-NEXT:    v_lshlrev_b32_e32 v3, 2, v0
 ; VARIANT3-NEXT:    s_waitcnt lgkmcnt(0)
-; VARIANT3-NEXT:    v_mov_b32_e32 v2, s3
-; VARIANT3-NEXT:    v_add_co_u32_e32 v1, vcc, s2, v1
-; VARIANT3-NEXT:    v_addc_co_u32_e32 v2, vcc, 0, v2, vcc
-; VARIANT3-NEXT:    global_store_dword v[1:2], v0, off
-; VARIANT3-NEXT:    s_barrier
-; VARIANT3-NEXT:    v_xad_u32 v3, v0, -1, s0
-; VARIANT3-NEXT:    v_ashrrev_i32_e32 v4, 31, v3
-; VARIANT3-NEXT:    v_lshlrev_b64 v[3:4], 2, v[3:4]
-; VARIANT3-NEXT:    v_mov_b32_e32 v0, s3
+; VARIANT3-NEXT:    v_mov_b32_e32 v4, s3
+; VARIANT3-NEXT:    v_xad_u32 v1, v0, -1, s0
+; VARIANT3-NEXT:    v_ashrrev_i32_e32 v2, 31, v1
 ; VARIANT3-NEXT:    v_add_co_u32_e32 v3, vcc, s2, v3
-; VARIANT3-NEXT:    v_addc_co_u32_e32 v4, vcc, v0, v4, vcc
-; VARIANT3-NEXT:    global_load_dword v0, v[3:4], off
+; VARIANT3-NEXT:    v_lshlrev_b64 v[1:2], 2, v[1:2]
+; VARIANT3-NEXT:    v_addc_co_u32_e32 v4, vcc, 0, v4, vcc
+; VARIANT3-NEXT:    global_store_dword v[3:4], v0, off
+; VARIANT3-NEXT:    v_mov_b32_e32 v5, s3
+; VARIANT3-NEXT:    v_add_co_u32_e32 v0, vcc, s2, v1
+; VARIANT3-NEXT:    v_addc_co_u32_e32 v1, vcc, v5, v2, vcc
+; VARIANT3-NEXT:    s_barrier
+; VARIANT3-NEXT:    global_load_dword v0, v[0:1], off
 ; VARIANT3-NEXT:    s_waitcnt vmcnt(0)
-; VARIANT3-NEXT:    global_store_dword v[1:2], v0, off
+; VARIANT3-NEXT:    global_store_dword v[3:4], v0, off
 ; VARIANT3-NEXT:    s_endpgm
 entry:
   %tmp = call i32 @llvm.amdgcn.workitem.id.x()
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.buffer.load.ll
index 1c450e7c0..4c25ebb61 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.buffer.load.ll
@@ -1,8 +1,10 @@
-;RUN: llc < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck %s
+;RUN: llc < %s -march=amdgcn -mcpu=tahiti -verify-machineinstrs | FileCheck %s -check-prefixes=GCN,SI
+;RUN: llc < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck %s -check-prefixes=GCN,VI
 
-;CHECK-LABEL: {{^}}s_buffer_load_imm:
-;CHECK-NOT: s_waitcnt;
-;CHECK: s_buffer_load_dword s{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], 0x4
+;GCN-LABEL: {{^}}s_buffer_load_imm:
+;GCN-NOT: s_waitcnt;
+;SI: s_buffer_load_dword s{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], 0x1
+;VI: s_buffer_load_dword s{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], 0x4
 define amdgpu_ps void @s_buffer_load_imm(<4 x i32> inreg %desc) {
 main_body:
   %load = call i32 @llvm.amdgcn.s.buffer.load.i32(<4 x i32> %desc, i32 4, i32 0)
@@ -11,9 +13,9 @@ main_body:
   ret void
 }
 
-;CHECK-LABEL: {{^}}s_buffer_load_index:
-;CHECK-NOT: s_waitcnt;
-;CHECK: s_buffer_load_dword s{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], s{{[0-9]+}}
+;GCN-LABEL: {{^}}s_buffer_load_index:
+;GCN-NOT: s_waitcnt;
+;GCN: s_buffer_load_dword s{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], s{{[0-9]+}}
 define amdgpu_ps void @s_buffer_load_index(<4 x i32> inreg %desc, i32 inreg %index) {
 main_body:
   %load = call i32 @llvm.amdgcn.s.buffer.load.i32(<4 x i32> %desc, i32 %index, i32 0)
@@ -22,9 +24,21 @@ main_body:
   ret void
 }
 
-;CHECK-LABEL: {{^}}s_buffer_loadx2_imm:
-;CHECK-NOT: s_waitcnt;
-;CHECK: s_buffer_load_dwordx2 s[{{[0-9]+:[0-9]+}}], s[{{[0-9]+:[0-9]+}}], 0x40
+;GCN-LABEL: {{^}}s_buffer_load_index_divergent:
+;GCN-NOT: s_waitcnt;
+;GCN: buffer_load_dword v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], 0 offen
+define amdgpu_ps void @s_buffer_load_index_divergent(<4 x i32> inreg %desc, i32 %index) {
+main_body:
+  %load = call i32 @llvm.amdgcn.s.buffer.load.i32(<4 x i32> %desc, i32 %index, i32 0)
+  %bitcast = bitcast i32 %load to float
+  call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %bitcast, float undef, float undef, float undef, i1 true, i1 true)
+  ret void
+}
+
+;GCN-LABEL: {{^}}s_buffer_loadx2_imm:
+;GCN-NOT: s_waitcnt;
+;SI: s_buffer_load_dwordx2 s[{{[0-9]+:[0-9]+}}], s[{{[0-9]+:[0-9]+}}], 0x10
+;VI: s_buffer_load_dwordx2 s[{{[0-9]+:[0-9]+}}], s[{{[0-9]+:[0-9]+}}], 0x40
 define amdgpu_ps void @s_buffer_loadx2_imm(<4 x i32> inreg %desc) {
 main_body:
   %load = call <2 x i32> @llvm.amdgcn.s.buffer.load.v2i32(<4 x i32> %desc, i32 64, i32 0)
@@ -35,9 +49,9 @@ main_body:
   ret void
 }
 
-;CHECK-LABEL: {{^}}s_buffer_loadx2_index:
-;CHECK-NOT: s_waitcnt;
-;CHECK: s_buffer_load_dwordx2 s[{{[0-9]+:[0-9]+}}], s[{{[0-9]+:[0-9]+}}], s{{[0-9]+}}
+;GCN-LABEL: {{^}}s_buffer_loadx2_index:
+;GCN-NOT: s_waitcnt;
+;GCN: s_buffer_load_dwordx2 s[{{[0-9]+:[0-9]+}}], s[{{[0-9]+:[0-9]+}}], s{{[0-9]+}}
 define amdgpu_ps void @s_buffer_loadx2_index(<4 x i32> inreg %desc, i32 inreg %index) {
 main_body:
   %load = call <2 x i32> @llvm.amdgcn.s.buffer.load.v2i32(<4 x i32> %desc, i32 %index, i32 0)
@@ -48,9 +62,67 @@ main_body:
   ret void
 }
 
-;CHECK-LABEL: {{^}}s_buffer_loadx4_imm:
-;CHECK-NOT: s_waitcnt;
-;CHECK: s_buffer_load_dwordx4 s[{{[0-9]+:[0-9]+}}], s[{{[0-9]+:[0-9]+}}], 0xc8
+;GCN-LABEL: {{^}}s_buffer_loadx2_index_divergent:
+;GCN-NOT: s_waitcnt;
+;GCN: buffer_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], 0 offen
+define amdgpu_ps void @s_buffer_loadx2_index_divergent(<4 x i32> inreg %desc, i32 %index) {
+main_body:
+  %load = call <2 x i32> @llvm.amdgcn.s.buffer.load.v2i32(<4 x i32> %desc, i32 %index, i32 0)
+  %bitcast = bitcast <2 x i32> %load to <2 x float>
+  %x = extractelement <2 x float> %bitcast, i32 0
+  %y = extractelement <2 x float> %bitcast, i32 1
+  call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %x, float %y, float undef, float undef, i1 true, i1 true)
+  ret void
+}
+
+;GCN-LABEL: {{^}}s_buffer_loadx3_imm:
+;GCN-NOT: s_waitcnt;
+;SI: s_buffer_load_dwordx4 s[{{[0-9]+:[0-9]+}}], s[{{[0-9]+:[0-9]+}}], 0x10
+;VI: s_buffer_load_dwordx4 s[{{[0-9]+:[0-9]+}}], s[{{[0-9]+:[0-9]+}}], 0x40
+define amdgpu_ps void @s_buffer_loadx3_imm(<4 x i32> inreg %desc) {
+main_body:
+  %load = call <3 x i32> @llvm.amdgcn.s.buffer.load.v3i32(<4 x i32> %desc, i32 64, i32 0)
+  %bitcast = bitcast <3 x i32> %load to <3 x float>
+  %x = extractelement <3 x float> %bitcast, i32 0
+  %y = extractelement <3 x float> %bitcast, i32 1
+  %z = extractelement <3 x float> %bitcast, i32 2
+  call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %x, float %y, float %z, float undef, i1 true, i1 true)
+  ret void
+}
+
+;GCN-LABEL: {{^}}s_buffer_loadx3_index:
+;GCN-NOT: s_waitcnt;
+;GCN: s_buffer_load_dwordx4 s[{{[0-9]+:[0-9]+}}], s[{{[0-9]+:[0-9]+}}], s{{[0-9]+}}
+define amdgpu_ps void @s_buffer_loadx3_index(<4 x i32> inreg %desc, i32 inreg %index) {
+main_body:
+  %load = call <3 x i32> @llvm.amdgcn.s.buffer.load.v3i32(<4 x i32> %desc, i32 %index, i32 0)
+  %bitcast = bitcast <3 x i32> %load to <3 x float>
+  %x = extractelement <3 x float> %bitcast, i32 0
+  %y = extractelement <3 x float> %bitcast, i32 1
+  %z = extractelement <3 x float> %bitcast, i32 2
+  call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %x, float %y, float %z, float undef, i1 true, i1 true)
+  ret void
+}
+
+;GCN-LABEL: {{^}}s_buffer_loadx3_index_divergent:
+;GCN-NOT: s_waitcnt;
+;SI: buffer_load_dwordx4 v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], 0 offen
+;VI: buffer_load_dwordx3 v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], 0 offen
+define amdgpu_ps void @s_buffer_loadx3_index_divergent(<4 x i32> inreg %desc, i32 %index) {
+main_body:
+  %load = call <3 x i32> @llvm.amdgcn.s.buffer.load.v3i32(<4 x i32> %desc, i32 %index, i32 0)
+  %bitcast = bitcast <3 x i32> %load to <3 x float>
+  %x = extractelement <3 x float> %bitcast, i32 0
+  %y = extractelement <3 x float> %bitcast, i32 1
+  %z = extractelement <3 x float> %bitcast, i32 2
+  call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %x, float %y, float %z, float undef, i1 true, i1 true)
+  ret void
+}
+
+;GCN-LABEL: {{^}}s_buffer_loadx4_imm:
+;GCN-NOT: s_waitcnt;
+;SI: s_buffer_load_dwordx4 s[{{[0-9]+:[0-9]+}}], s[{{[0-9]+:[0-9]+}}], 0x32
+;VI: s_buffer_load_dwordx4 s[{{[0-9]+:[0-9]+}}], s[{{[0-9]+:[0-9]+}}], 0xc8
 define amdgpu_ps void @s_buffer_loadx4_imm(<4 x i32> inreg %desc) {
 main_body:
   %load = call <4 x i32> @llvm.amdgcn.s.buffer.load.v4i32(<4 x i32> %desc, i32 200, i32 0)
@@ -63,9 +135,9 @@ main_body:
   ret void
 }
 
-;CHECK-LABEL: {{^}}s_buffer_loadx4_index:
-;CHECK-NOT: s_waitcnt;
-;CHECK: s_buffer_load_dwordx4 s[{{[0-9]+:[0-9]+}}], s[{{[0-9]+:[0-9]+}}], s{{[0-9]+}}
+;GCN-LABEL: {{^}}s_buffer_loadx4_index:
+;GCN-NOT: s_waitcnt;
+;GCN: buffer_load_dwordx4 s[{{[0-9]+:[0-9]+}}], s[{{[0-9]+:[0-9]+}}], s{{[0-9]+}}
 define amdgpu_ps void @s_buffer_loadx4_index(<4 x i32> inreg %desc, i32 inreg %index) {
 main_body:
   %load = call <4 x i32> @llvm.amdgcn.s.buffer.load.v4i32(<4 x i32> %desc, i32 %index, i32 0)
@@ -78,9 +150,25 @@ main_body:
   ret void
 }
 
-;CHECK-LABEL: {{^}}s_buffer_load_imm_mergex2:
-;CHECK-NOT: s_waitcnt;
-;CHECK: s_buffer_load_dwordx2 s[{{[0-9]+:[0-9]+}}], s[{{[0-9]+:[0-9]+}}], 0x4
+;GCN-LABEL: {{^}}s_buffer_loadx4_index_divergent:
+;GCN-NOT: s_waitcnt;
+;GCN: buffer_load_dwordx4 v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], 0 offen
+define amdgpu_ps void @s_buffer_loadx4_index_divergent(<4 x i32> inreg %desc, i32 %index) {
+main_body:
+  %load = call <4 x i32> @llvm.amdgcn.s.buffer.load.v4i32(<4 x i32> %desc, i32 %index, i32 0)
+  %bitcast = bitcast <4 x i32> %load to <4 x float>
+  %x = extractelement <4 x float> %bitcast, i32 0
+  %y = extractelement <4 x float> %bitcast, i32 1
+  %z = extractelement <4 x float> %bitcast, i32 2
+  %w = extractelement <4 x float> %bitcast, i32 3
+  call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %x, float %y, float %z, float %w, i1 true, i1 true)
+  ret void
+}
+
+;GCN-LABEL: {{^}}s_buffer_load_imm_mergex2:
+;GCN-NOT: s_waitcnt;
+;SI: s_buffer_load_dwordx2 s[{{[0-9]+:[0-9]+}}], s[{{[0-9]+:[0-9]+}}], 0x1
+;VI: s_buffer_load_dwordx2 s[{{[0-9]+:[0-9]+}}], s[{{[0-9]+:[0-9]+}}], 0x4
 define amdgpu_ps void @s_buffer_load_imm_mergex2(<4 x i32> inreg %desc) {
 main_body:
   %load0 = call i32 @llvm.amdgcn.s.buffer.load.i32(<4 x i32> %desc, i32 4, i32 0)
@@ -91,9 +179,10 @@ main_body:
   ret void
 }
 
-;CHECK-LABEL: {{^}}s_buffer_load_imm_mergex4:
-;CHECK-NOT: s_waitcnt;
-;CHECK: s_buffer_load_dwordx4 s[{{[0-9]+:[0-9]+}}], s[{{[0-9]+:[0-9]+}}], 0x8
+;GCN-LABEL: {{^}}s_buffer_load_imm_mergex4:
+;GCN-NOT: s_waitcnt;
+;SI: s_buffer_load_dwordx4 s[{{[0-9]+:[0-9]+}}], s[{{[0-9]+:[0-9]+}}], 0x2
+;VI: s_buffer_load_dwordx4 s[{{[0-9]+:[0-9]+}}], s[{{[0-9]+:[0-9]+}}], 0x8
 define amdgpu_ps void @s_buffer_load_imm_mergex4(<4 x i32> inreg %desc) {
 main_body:
   %load0 = call i32 @llvm.amdgcn.s.buffer.load.i32(<4 x i32> %desc, i32 8, i32 0)
@@ -108,10 +197,10 @@ main_body:
   ret void
 }
 
-;CHECK-LABEL: {{^}}s_buffer_load_index_across_bb:
-;CHECK-NOT: s_waitcnt;
-;CHECK: v_or_b32
-;CHECK: buffer_load_dword v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], 0 offen
+;GCN-LABEL: {{^}}s_buffer_load_index_across_bb:
+;GCN-NOT: s_waitcnt;
+;GCN: v_or_b32
+;GCN: buffer_load_dword v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], 0 offen
 define amdgpu_ps void @s_buffer_load_index_across_bb(<4 x i32> inreg %desc, i32 %index) {
 main_body:
   %tmp = shl i32 %index, 4
@@ -125,12 +214,12 @@ bb1:                                              ; preds = %main_body
   ret void
 }
 
-;CHECK-LABEL: {{^}}s_buffer_load_index_across_bb_merged:
-;CHECK-NOT: s_waitcnt;
-;CHECK: v_or_b32
-;CHECK: v_or_b32
-;CHECK: buffer_load_dword v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], 0 offen
-;CHECK: buffer_load_dword v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], 0 offen
+;GCN-LABEL: {{^}}s_buffer_load_index_across_bb_merged:
+;GCN-NOT: s_waitcnt;
+;GCN: v_or_b32
+;GCN: v_or_b32
+;GCN: buffer_load_dword v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], 0 offen
+;GCN: buffer_load_dword v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], 0 offen
 define amdgpu_ps void @s_buffer_load_index_across_bb_merged(<4 x i32> inreg %desc, i32 %index) {
 main_body:
   %tmp = shl i32 %index, 4
@@ -150,4 +239,5 @@ bb1:                                              ; preds = %main_body
 declare void @llvm.amdgcn.exp.f32(i32, i32, float, float, float, float, i1, i1)
 declare i32 @llvm.amdgcn.s.buffer.load.i32(<4 x i32>, i32, i32)
 declare <2 x i32> @llvm.amdgcn.s.buffer.load.v2i32(<4 x i32>, i32, i32)
+declare <3 x i32> @llvm.amdgcn.s.buffer.load.v3i32(<4 x i32>, i32, i32)
 declare <4 x i32> @llvm.amdgcn.s.buffer.load.v4i32(<4 x i32>, i32, i32)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.softwqm.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.softwqm.ll
new file mode 100644
index 000000000..3d35e32cf
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.softwqm.ll
@@ -0,0 +1,188 @@
+; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=CHECK %s
+
+; Check that WQM is not triggered by the softwqm intrinsic alone.
+;
+;CHECK-LABEL: {{^}}test1:
+;CHECK-NOT: s_wqm_b64 exec, exec
+;CHECK: buffer_load_dword
+;CHECK: buffer_load_dword
+;CHECK: v_add_f32_e32
+define amdgpu_ps float @test1(i32 inreg %idx0, i32 inreg %idx1) {
+main_body:
+  %src0 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> undef, i32 %idx0, i32 0, i1 0, i1 0)
+  %src1 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> undef, i32 %idx1, i32 0, i1 0, i1 0)
+  %out = fadd float %src0, %src1
+  %out.0 = call float @llvm.amdgcn.softwqm.f32(float %out)
+  ret float %out.0
+}
+
+; Check that the softwqm intrinsic works correctly for integers.
+;
+;CHECK-LABEL: {{^}}test2:
+;CHECK-NOT: s_wqm_b64 exec, exec
+;CHECK: buffer_load_dword
+;CHECK: buffer_load_dword
+;CHECK: v_add_f32_e32
+define amdgpu_ps float @test2(i32 inreg %idx0, i32 inreg %idx1) {
+main_body:
+  %src0 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> undef, i32 %idx0, i32 0, i1 0, i1 0)
+  %src1 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> undef, i32 %idx1, i32 0, i1 0, i1 0)
+  %out = fadd float %src0, %src1
+  %out.0 = bitcast float %out to i32
+  %out.1 = call i32 @llvm.amdgcn.softwqm.i32(i32 %out.0)
+  %out.2 = bitcast i32 %out.1 to float
+  ret float %out.2
+}
+
+; Make sure the transition from WQM to Exact to softwqm does not trigger WQM.
+;
+;CHECK-LABEL: {{^}}test_softwqm1:
+;CHECK-NOT: s_wqm_b64 exec, exec
+;CHECK: buffer_load_dword
+;CHECK: buffer_load_dword
+;CHECK: buffer_store_dword
+;CHECK-NOT; s_wqm_b64 exec, exec
+;CHECK: v_add_f32_e32
+define amdgpu_ps float @test_softwqm1(i32 inreg %idx0, i32 inreg %idx1) {
+main_body:
+  %src0 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> undef, i32 %idx0, i32 0, i1 0, i1 0)
+  %src1 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> undef, i32 %idx1, i32 0, i1 0, i1 0)
+  %temp = fadd float %src0, %src1
+  call void @llvm.amdgcn.buffer.store.f32(float %temp, <4 x i32> undef, i32 %idx0, i32 0, i1 0, i1 0)
+  %out = fadd float %temp, %temp
+  %out.0 = call float @llvm.amdgcn.softwqm.f32(float %out)
+  ret float %out.0
+}
+
+; Make sure the transition from WQM to Exact to softwqm does trigger WQM.
+;
+;CHECK-LABEL: {{^}}test_softwqm2:
+;CHECK: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec
+;CHECK: s_wqm_b64 exec, exec
+;CHECK: buffer_load_dword
+;CHECK: buffer_load_dword
+;CHECK: s_and_b64 exec, exec, [[ORIG]]
+;CHECK: buffer_store_dword
+;CHECK; s_wqm_b64 exec, exec
+;CHECK: v_add_f32_e32
+define amdgpu_ps float @test_softwqm2(i32 inreg %idx0, i32 inreg %idx1) {
+main_body:
+  %src0 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> undef, i32 %idx0, i32 0, i1 0, i1 0)
+  %src1 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> undef, i32 %idx1, i32 0, i1 0, i1 0)
+  %temp = fadd float %src0, %src1
+  %temp.0 = call float @llvm.amdgcn.wqm.f32(float %temp)
+  call void @llvm.amdgcn.buffer.store.f32(float %temp.0, <4 x i32> undef, i32 %idx0, i32 0, i1 0, i1 0)
+  %out = fadd float %temp, %temp
+  %out.0 = call float @llvm.amdgcn.softwqm.f32(float %out)
+  ret float %out.0
+}
+
+; Make sure the transition from Exact to WWM then softwqm does not trigger WQM.
+;
+;CHECK-LABEL: {{^}}test_wwm1:
+;CHECK: buffer_load_dword
+;CHECK: buffer_store_dword
+;CHECK: s_or_saveexec_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], -1
+;CHECK: buffer_load_dword
+;CHECK: v_add_f32_e32
+;CHECK: s_mov_b64 exec, [[ORIG]]
+;CHECK-NOT: s_wqm_b64
+define amdgpu_ps float @test_wwm1(i32 inreg %idx0, i32 inreg %idx1) {
+main_body:
+  %src0 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> undef, i32 %idx0, i32 0, i1 0, i1 0)
+  call void @llvm.amdgcn.buffer.store.f32(float %src0, <4 x i32> undef, i32 %idx0, i32 0, i1 0, i1 0)
+  %src1 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> undef, i32 %idx1, i32 0, i1 0, i1 0)
+  %temp = fadd float %src0, %src1
+  %temp.0 = call float @llvm.amdgcn.wwm.f32(float %temp)
+  %out = fadd float %temp.0, %temp.0
+  %out.0 = call float @llvm.amdgcn.softwqm.f32(float %out)
+  ret float %out.0
+}
+
+; Check that softwqm on one case of branch does not trigger WQM for shader.
+;
+;CHECK-LABEL: {{^}}test_control_flow_0:
+;CHECK-NEXT: ; %main_body
+;CHECK-NOT: s_wqm_b64 exec, exec
+;CHECK: %ELSE
+;CHECK: store
+;CHECK: %IF
+;CHECK: buffer_load
+;CHECK: buffer_load
+define amdgpu_ps float @test_control_flow_0(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, i32 inreg %idx0, i32 inreg %idx1, i32 %c, i32 %z, float %data) {
+main_body:
+  %cmp = icmp eq i32 %z, 0
+  br i1 %cmp, label %IF, label %ELSE
+
+IF:
+  %src0 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> undef, i32 %idx0, i32 0, i1 0, i1 0)
+  %src1 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> undef, i32 %idx1, i32 0, i1 0, i1 0)
+  %out = fadd float %src0, %src1
+  %data.if = call float @llvm.amdgcn.softwqm.f32(float %out)
+  br label %END
+
+ELSE:
+  call void @llvm.amdgcn.buffer.store.f32(float %data, <4 x i32> undef, i32 %c, i32 0, i1 0, i1 0)
+  br label %END
+
+END:
+  %r = phi float [ %data.if, %IF ], [ %data, %ELSE ]
+  ret float %r
+}
+
+; Check that softwqm on one case of branch is treated as WQM in WQM shader.
+;
+;CHECK-LABEL: {{^}}test_control_flow_1:
+;CHECK-NEXT: ; %main_body
+;CHECK-NEXT: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec
+;CHECK-NEXT: s_wqm_b64 exec, exec
+;CHECK: %ELSE
+;CHECK: s_and_saveexec_b64 [[SAVED:s\[[0-9]+:[0-9]+\]]], [[ORIG]]
+;CHECK: store
+;CHECK: s_mov_b64 exec, [[SAVED]]
+;CHECK: %IF
+;CHECK-NOT: s_and_saveexec_b64
+;CHECK-NOT: s_and_b64 exec
+;CHECK: buffer_load
+;CHECK: buffer_load
+define amdgpu_ps float @test_control_flow_1(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, i32 inreg %idx0, i32 inreg %idx1, i32 %c, i32 %z, float %data) {
+main_body:
+  %c.bc = bitcast i32 %c to float
+  %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %c.bc, <8 x i32> %rsrc, <4 x i32> %sampler, i1 0, i32 0, i32 0) #0
+  %tex0 = extractelement <4 x float> %tex, i32 0
+  %dtex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %tex0, <8 x i32> %rsrc, <4 x i32> %sampler, i1 0, i32 0, i32 0) #0
+  %data.sample = extractelement <4 x float> %dtex, i32 0
+
+  %cmp = icmp eq i32 %z, 0
+  br i1 %cmp, label %IF, label %ELSE
+
+IF:
+  %src0 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> undef, i32 %idx0, i32 0, i1 0, i1 0)
+  %src1 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> undef, i32 %idx1, i32 0, i1 0, i1 0)
+  %out = fadd float %src0, %src1
+  %data.if = call float @llvm.amdgcn.softwqm.f32(float %out)
+  br label %END
+
+ELSE:
+  call void @llvm.amdgcn.buffer.store.f32(float %data.sample, <4 x i32> undef, i32 %c, i32 0, i1 0, i1 0)
+  br label %END
+
+END:
+  %r = phi float [ %data.if, %IF ], [ %data, %ELSE ]
+  ret float %r
+}
+
+declare void @llvm.amdgcn.buffer.store.f32(float, <4 x i32>, i32, i32, i1, i1) #2
+declare void @llvm.amdgcn.buffer.store.v4f32(<4 x float>, <4 x i32>, i32, i32, i1, i1) #2
+declare float @llvm.amdgcn.buffer.load.f32(<4 x i32>, i32, i32, i1, i1) #3
+declare <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #3
+declare <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #3
+declare void @llvm.amdgcn.kill(i1) #1
+declare float @llvm.amdgcn.wqm.f32(float) #3
+declare float @llvm.amdgcn.softwqm.f32(float) #3
+declare i32 @llvm.amdgcn.softwqm.i32(i32) #3
+declare float @llvm.amdgcn.wwm.f32(float) #3
+
+attributes #1 = { nounwind }
+attributes #2 = { nounwind readonly }
+attributes #3 = { nounwind readnone }
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.atomic.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.atomic.ll
index 6c9ca9b17..e7668f0f7 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.atomic.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.atomic.ll
@@ -50,6 +50,10 @@ main_body:
 ;CHECK: buffer_atomic_or v0, v1, s[0:3], 0 idxen glc slc
 ;CHECK: s_waitcnt vmcnt(0)
 ;CHECK: buffer_atomic_xor v0, v1, s[0:3], 0 idxen glc
+;CHECK: s_waitcnt vmcnt(0)
+;CHECK: buffer_atomic_inc v0, v1, s[0:3], 0 idxen glc
+;CHECK: s_waitcnt vmcnt(0)
+;CHECK: buffer_atomic_dec v0, v1, s[0:3], 0 idxen glc
 define amdgpu_ps float @test2(<4 x i32> inreg %rsrc, i32 %data, i32 %vindex) {
 main_body:
   %t1 = call i32 @llvm.amdgcn.struct.buffer.atomic.add.i32(i32 %data, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 0, i32 0)
@@ -61,7 +65,9 @@ main_body:
   %t7 = call i32 @llvm.amdgcn.struct.buffer.atomic.and.i32(i32 %t6, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 0, i32 0)
   %t8 = call i32 @llvm.amdgcn.struct.buffer.atomic.or.i32(i32 %t7, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 0, i32 2)
   %t9 = call i32 @llvm.amdgcn.struct.buffer.atomic.xor.i32(i32 %t8, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 0, i32 0)
-  %out = bitcast i32 %t9 to float
+  %t10 = call i32 @llvm.amdgcn.struct.buffer.atomic.inc.i32(i32 %t9, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 0, i32 0)
+  %t11 = call i32 @llvm.amdgcn.struct.buffer.atomic.dec.i32(i32 %t10, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 0, i32 0)
+  %out = bitcast i32 %t11 to float
   ret float %out
 }
 
@@ -122,6 +128,8 @@ declare i32 @llvm.amdgcn.struct.buffer.atomic.umax.i32(i32, <4 x i32>, i32, i32,
 declare i32 @llvm.amdgcn.struct.buffer.atomic.and.i32(i32, <4 x i32>, i32, i32, i32, i32) #0
 declare i32 @llvm.amdgcn.struct.buffer.atomic.or.i32(i32, <4 x i32>, i32, i32, i32, i32) #0
 declare i32 @llvm.amdgcn.struct.buffer.atomic.xor.i32(i32, <4 x i32>, i32, i32, i32, i32) #0
+declare i32 @llvm.amdgcn.struct.buffer.atomic.inc.i32(i32, <4 x i32>, i32, i32, i32, i32) #0
+declare i32 @llvm.amdgcn.struct.buffer.atomic.dec.i32(i32, <4 x i32>, i32, i32, i32, i32) #0
 declare i32 @llvm.amdgcn.struct.buffer.atomic.cmpswap.i32(i32, i32, <4 x i32>, i32, i32, i32, i32) #0
 
 attributes #0 = { nounwind }
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.ll
index 4ac34286b..496f8079f 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.ll
@@ -200,6 +200,78 @@ main_body:
   ret float %val
 }
 
+;CHECK-LABEL: {{^}}struct_buffer_load_f16:
+;CHECK-NEXT: %bb.
+;CHECK-NEXT: buffer_load_ushort [[VAL:v[0-9]+]], v1, s[0:3], 0 idxen
+;CHECK: s_waitcnt vmcnt(0)
+;CHECK: ds_write_b16 v0, [[VAL]]
+define amdgpu_ps void @struct_buffer_load_f16(<4 x i32> inreg %rsrc, half addrspace(3)* %ptr, i32 %idx) {
+main_body:
+  %val = call half @llvm.amdgcn.struct.buffer.load.f16(<4 x i32> %rsrc, i32 %idx, i32 0, i32 0, i32 0)
+  store half %val, half addrspace(3)* %ptr
+  ret void
+}
+
+;CHECK-LABEL: {{^}}struct_buffer_load_v2f16:
+;CHECK-NEXT: %bb.
+;CHECK-NEXT: buffer_load_dword [[VAL:v[0-9]+]], v1, s[0:3], 0 idxen
+;CHECK: s_waitcnt vmcnt(0)
+;CHECK: ds_write_b32 v0, [[VAL]]
+define amdgpu_ps void @struct_buffer_load_v2f16(<4 x i32> inreg %rsrc, <2 x half> addrspace(3)* %ptr, i32 %idx) {
+main_body:
+  %val = call <2 x half> @llvm.amdgcn.struct.buffer.load.v2f16(<4 x i32> %rsrc, i32 %idx, i32 0, i32 0, i32 0)
+  store <2 x half> %val, <2 x half> addrspace(3)* %ptr
+  ret void
+}
+
+;CHECK-LABEL: {{^}}struct_buffer_load_v4f16:
+;CHECK-NEXT: %bb.
+;CHECK-NEXT: buffer_load_dwordx2 [[VAL:v\[[0-9]+:[0-9]+\]]], v1, s[0:3], 0 idxen
+;CHECK: s_waitcnt vmcnt(0)
+;CHECK: ds_write_b64 v0, [[VAL]]
+define amdgpu_ps void @struct_buffer_load_v4f16(<4 x i32> inreg %rsrc, <4 x half> addrspace(3)* %ptr, i32 %idx) {
+main_body:
+  %val = call <4 x half> @llvm.amdgcn.struct.buffer.load.v4f16(<4 x i32> %rsrc, i32 %idx, i32 0, i32 0, i32 0)
+  store <4 x half> %val, <4 x half> addrspace(3)* %ptr
+  ret void
+}
+
+;CHECK-LABEL: {{^}}struct_buffer_load_i16:
+;CHECK-NEXT: %bb.
+;CHECK-NEXT: buffer_load_ushort [[VAL:v[0-9]+]], v1, s[0:3], 0 idxen
+;CHECK: s_waitcnt vmcnt(0)
+;CHECK: ds_write_b16 v0, [[VAL]]
+define amdgpu_ps void @struct_buffer_load_i16(<4 x i32> inreg %rsrc, i16 addrspace(3)* %ptr, i32 %idx) {
+main_body:
+  %val = call i16 @llvm.amdgcn.struct.buffer.load.i16(<4 x i32> %rsrc, i32 %idx, i32 0, i32 0, i32 0)
+  store i16 %val, i16 addrspace(3)* %ptr
+  ret void
+}
+
+;CHECK-LABEL: {{^}}struct_buffer_load_v2i16:
+;CHECK-NEXT: %bb.
+;CHECK-NEXT: buffer_load_dword [[VAL:v[0-9]+]], v1, s[0:3], 0 idxen
+;CHECK: s_waitcnt vmcnt(0)
+;CHECK: ds_write_b32 v0, [[VAL]]
+define amdgpu_ps void @struct_buffer_load_v2i16(<4 x i32> inreg %rsrc, <2 x i16> addrspace(3)* %ptr, i32 %idx) {
+main_body:
+  %val = call <2 x i16> @llvm.amdgcn.struct.buffer.load.v2i16(<4 x i32> %rsrc, i32 %idx, i32 0, i32 0, i32 0)
+  store <2 x i16> %val, <2 x i16> addrspace(3)* %ptr
+  ret void
+}
+
+;CHECK-LABEL: {{^}}struct_buffer_load_v4i16:
+;CHECK-NEXT: %bb.
+;CHECK-NEXT: buffer_load_dwordx2 [[VAL:v\[[0-9]+:[0-9]+\]]], v1, s[0:3], 0 idxen
+;CHECK: s_waitcnt vmcnt(0)
+;CHECK: ds_write_b64 v0, [[VAL]]
+define amdgpu_ps void @struct_buffer_load_v4i16(<4 x i32> inreg %rsrc, <4 x i16> addrspace(3)* %ptr, i32 %idx) {
+main_body:
+  %val = call <4 x i16> @llvm.amdgcn.struct.buffer.load.v4i16(<4 x i32> %rsrc, i32 %idx, i32 0, i32 0, i32 0)
+  store <4 x i16> %val, <4 x i16> addrspace(3)* %ptr
+  ret void
+}
+
 declare float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32>, i32, i32, i32, i32) #0
 declare <2 x float> @llvm.amdgcn.struct.buffer.load.v2f32(<4 x i32>, i32, i32, i32, i32) #0
 declare <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32>, i32, i32, i32, i32) #0
@@ -208,6 +280,13 @@ declare <2 x i32> @llvm.amdgcn.struct.buffer.load.v2i32(<4 x i32>, i32, i32, i32
 declare <4 x i32> @llvm.amdgcn.struct.buffer.load.v4i32(<4 x i32>, i32, i32, i32, i32) #0
 declare void @llvm.amdgcn.exp.f32(i32, i32, float, float, float, float, i1, i1) #0
 declare i8 @llvm.amdgcn.struct.buffer.load.i8(<4 x i32>, i32, i32, i32, i32) #0
+
+declare half @llvm.amdgcn.struct.buffer.load.f16(<4 x i32>, i32, i32, i32, i32) #0
+declare <2 x half> @llvm.amdgcn.struct.buffer.load.v2f16(<4 x i32>, i32, i32, i32, i32) #0
+declare <4 x half> @llvm.amdgcn.struct.buffer.load.v4f16(<4 x i32>, i32, i32, i32, i32) #0
+
 declare i16 @llvm.amdgcn.struct.buffer.load.i16(<4 x i32>, i32, i32, i32, i32) #0
+declare <2 x i16> @llvm.amdgcn.struct.buffer.load.v2i16(<4 x i32>, i32, i32, i32, i32) #0
+declare <4 x i16> @llvm.amdgcn.struct.buffer.load.v4i16(<4 x i32>, i32, i32, i32, i32) #0
 
 attributes #0 = { nounwind readonly }
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.store.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.store.ll
index 826030cd9..4c629e227 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.store.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.store.ll
@@ -121,12 +121,39 @@ main_body:
   ret void
 }
 
-;CHECK-LABEL: {{^}}struct_buffer_store_short:
+;CHECK-LABEL: {{^}}struct_buffer_store_f16:
+;CHECK-NEXT: %bb.
+;CHECK-NEXT: v_cvt_f16_f32_e32 v{{[0-9]}}, v{{[0-9]}}
+;CHECK-NEXT: buffer_store_short v{{[0-9]}}, v{{[0-9]}}, s[0:3], 0 idxen
+;CHECK-NEXT: s_endpgm
+define amdgpu_ps void @struct_buffer_store_f16(<4 x i32> inreg %rsrc, float %v1, i32 %index) {
+  %v2 = fptrunc float %v1 to half
+  call void @llvm.amdgcn.struct.buffer.store.f16(half %v2, <4 x i32> %rsrc, i32 %index, i32 0, i32 0, i32 0)
+  ret void
+}
+
+;CHECK-LABEL: {{^}}struct_buffer_store_v2f16:
+;CHECK-NEXT: %bb.
+;CHECK: buffer_store_dword v0, {{v[0-9]+}}, s[0:3], 0 idxen
+define amdgpu_ps void @struct_buffer_store_v2f16(<4 x i32> inreg %rsrc, <2 x half> %v1, i32 %index) {
+  call void @llvm.amdgcn.struct.buffer.store.v2f16(<2 x half> %v1, <4 x i32> %rsrc, i32 %index, i32 0, i32 0, i32 0)
+  ret void
+}
+
+;CHECK-LABEL: {{^}}struct_buffer_store_v4f16:
+;CHECK-NEXT: %bb.
+;CHECK: buffer_store_dwordx2 v[0:1], {{v[0-9]+}}, s[0:3], 0 idxen
+define amdgpu_ps void @struct_buffer_store_v4f16(<4 x i32> inreg %rsrc, <4 x half> %v1, i32 %index) {
+  call void @llvm.amdgcn.struct.buffer.store.v4f16(<4 x half> %v1, <4 x i32> %rsrc, i32 %index, i32 0, i32 0, i32 0)
+  ret void
+}
+
+;CHECK-LABEL: {{^}}struct_buffer_store_i16:
 ;CHECK-NEXT: %bb.
 ;CHECK-NEXT: v_cvt_u32_f32_e32 v{{[0-9]}}, v{{[0-9]}}
 ;CHECK-NEXT: buffer_store_short v{{[0-9]}}, v{{[0-9]}}, s[0:3], 0 idxen
 ;CHECK-NEXT: s_endpgm
-define amdgpu_ps void @struct_buffer_store_short(<4 x i32> inreg %rsrc, float %v1, i32 %index) {
+define amdgpu_ps void @struct_buffer_store_i16(<4 x i32> inreg %rsrc, float %v1, i32 %index) {
 main_body:
   %v2 = fptoui float %v1 to i32
   %v3 = trunc i32 %v2 to i16
@@ -134,6 +161,22 @@ main_body:
   ret void
 }
 
+;CHECK-LABEL: {{^}}struct_buffer_store_vif16:
+;CHECK-NEXT: %bb.
+;CHECK: buffer_store_dword v0, {{v[0-9]+}}, s[0:3], 0 idxen
+define amdgpu_ps void @struct_buffer_store_vif16(<4 x i32> inreg %rsrc, <2 x i16> %v1, i32 %index) {
+  call void @llvm.amdgcn.struct.buffer.store.v2i16(<2 x i16> %v1, <4 x i32> %rsrc, i32 %index, i32 0, i32 0, i32 0)
+  ret void
+}
+
+;CHECK-LABEL: {{^}}struct_buffer_store_v4i16:
+;CHECK-NEXT: %bb.
+;CHECK: buffer_store_dwordx2 v[0:1], {{v[0-9]+}}, s[0:3], 0 idxen
+define amdgpu_ps void @struct_buffer_store_v4i16(<4 x i32> inreg %rsrc, <4 x i16> %v1, i32 %index) {
+  call void @llvm.amdgcn.struct.buffer.store.v4i16(<4 x i16> %v1, <4 x i32> %rsrc, i32 %index, i32 0, i32 0, i32 0)
+  ret void
+}
+
 declare void @llvm.amdgcn.struct.buffer.store.f32(float, <4 x i32>, i32, i32, i32, i32) #0
 declare void @llvm.amdgcn.struct.buffer.store.v2f32(<2 x float>, <4 x i32>, i32, i32, i32, i32) #0
 declare void @llvm.amdgcn.struct.buffer.store.v4f32(<4 x float>, <4 x i32>, i32, i32, i32, i32) #0
@@ -143,6 +186,12 @@ declare void @llvm.amdgcn.struct.buffer.store.v4i32(<4 x i32>, <4 x i32>, i32, i
 declare <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32>, i32, i32, i32, i32) #1
 declare void @llvm.amdgcn.struct.buffer.store.i8(i8, <4 x i32>, i32, i32, i32, i32) #0
 declare void @llvm.amdgcn.struct.buffer.store.i16(i16, <4 x i32>, i32, i32, i32, i32) #0
+declare void @llvm.amdgcn.struct.buffer.store.v2i16(<2 x i16>, <4 x i32>, i32, i32, i32, i32) #0
+declare void @llvm.amdgcn.struct.buffer.store.v4i16(<4 x i16>, <4 x i32>, i32, i32, i32, i32) #0
+declare void @llvm.amdgcn.struct.buffer.store.f16(half, <4 x i32>, i32, i32, i32, i32) #0
+declare void @llvm.amdgcn.struct.buffer.store.v2f16(<2 x half>, <4 x i32>, i32, i32, i32, i32) #0
+declare void @llvm.amdgcn.struct.buffer.store.v4f16(<4 x half>, <4 x i32>, i32, i32, i32, i32) #0
+
 
 attributes #0 = { nounwind }
 attributes #1 = { nounwind readonly }
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.update.dpp.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.update.dpp.ll
index bfe491d9e..52562cc73 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.update.dpp.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.update.dpp.ll
@@ -1,10 +1,13 @@
-; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -amdgpu-dpp-combine=false -verify-machineinstrs < %s | FileCheck --check-prefixes=GCN,GFX8 %s
-; RUN: llc -march=amdgcn -mcpu=gfx1010 -mattr=-flat-for-global -amdgpu-dpp-combine=false -verify-machineinstrs < %s | FileCheck --check-prefixes=GCN,GFX10 %s
+; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -amdgpu-dpp-combine=false -verify-machineinstrs < %s | FileCheck --check-prefixes=GCN,GFX8,GFX8-OPT,GCN-OPT %s
+; RUN: llc -march=amdgcn -mcpu=tonga -O0 -mattr=-flat-for-global -amdgpu-dpp-combine=false -verify-machineinstrs < %s | FileCheck --check-prefixes=GCN,GFX8,GFX8-NOOPT %s
+; RUN: llc -march=amdgcn -mcpu=gfx1010 -mattr=-flat-for-global -amdgpu-dpp-combine=false -verify-machineinstrs < %s | FileCheck --check-prefixes=GCN,GFX10,GCN-OPT %s
 
 ; GCN-LABEL: {{^}}dpp_test:
 ; GCN:  v_mov_b32_e32 [[DST:v[0-9]+]], s{{[0-9]+}}
 ; GCN:  v_mov_b32_e32 [[SRC:v[0-9]+]], s{{[0-9]+}}
-; GFX8: s_nop 1
+; GFX8-OPT: s_nop 1
+; GFX8-NOOPT: s_nop 0
+; GFX8-NOOPT-NEXT: s_nop 0
 ; GCN:  v_mov_b32_dpp [[DST]], [[SRC]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1{{$}}
 define amdgpu_kernel void @dpp_test(i32 addrspace(1)* %out, i32 %in1, i32 %in2) {
   %tmp0 = call i32 @llvm.amdgcn.update.dpp.i32(i32 %in1, i32 %in2, i32 1, i32 1, i32 1, i1 0) #0
@@ -15,7 +18,9 @@ define amdgpu_kernel void @dpp_test(i32 addrspace(1)* %out, i32 %in1, i32 %in2)
 ; GCN-LABEL: {{^}}dpp_test_bc:
 ; GCN:  v_mov_b32_e32 [[DST:v[0-9]+]], s{{[0-9]+}}
 ; GCN:  v_mov_b32_e32 [[SRC:v[0-9]+]], s{{[0-9]+}}
-; GFX8: s_nop 1
+; GFX8-OPT: s_nop 1
+; GFX8-NOOPT: s_nop 0
+; GFX8-NOOPT-NEXT: s_nop 0
 ; GCN:  v_mov_b32_dpp [[DST]], [[SRC]] quad_perm:[2,0,0,0] row_mask:0x1 bank_mask:0x1 bound_ctrl:0{{$}}
 define amdgpu_kernel void @dpp_test_bc(i32 addrspace(1)* %out, i32 %in1, i32 %in2) {
   %tmp0 = call i32 @llvm.amdgcn.update.dpp.i32(i32 %in1, i32 %in2, i32 2, i32 1, i32 1, i1 1) #0
@@ -24,14 +29,14 @@ define amdgpu_kernel void @dpp_test_bc(i32 addrspace(1)* %out, i32 %in1, i32 %in
 }
 
 
-; VI-LABEL: {{^}}dpp_test1:
+; GCN-LABEL: {{^}}dpp_test1:
 ; GFX10: v_add_nc_u32_e32 [[REG:v[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}}
 ; GFX8-OPT: v_add_u32_e32 [[REG:v[0-9]+]], vcc, v{{[0-9]+}}, v{{[0-9]+}}
 ; GFX8-NOOPT: v_add_u32_e64 [[REG:v[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, v{{[0-9]+}}
 ; GFX8-NOOPT: v_mov_b32_e32 v{{[0-9]+}}, 0
 ; GFX8: s_nop 0
 ; GFX8-NEXT: s_nop 0
-; GFX8-OPT-NEXT: v_mov_b32_dpp {{v[0-9]+}}, [[REG]] quad_perm:[1,0,3,2] row_mask:0xf bank_mask:0xf
+; GFX8-NEXT: v_mov_b32_dpp {{v[0-9]+}}, [[REG]] quad_perm:[1,0,3,2] row_mask:0xf bank_mask:0xf
 @0 = internal unnamed_addr addrspace(3) global [448 x i32] undef, align 4
 define weak_odr amdgpu_kernel void @dpp_test1(i32* %arg) local_unnamed_addr {
 bb:
@@ -50,8 +55,56 @@ bb:
   ret void
 }
 
+; GCN-LABEL: {{^}}update_dpp64_test:
+; GCN:     load_dwordx2 v{{\[}}[[SRC_LO:[0-9]+]]:[[SRC_HI:[0-9]+]]]
+; GCN-DAG: v_mov_b32_dpp v{{[0-9]+}}, v[[SRC_LO]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1{{$}}
+; GCN-DAG: v_mov_b32_dpp v{{[0-9]+}}, v[[SRC_HI]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1{{$}}
+define amdgpu_kernel void @update_dpp64_test(i64 addrspace(1)* %arg, i64 %in1, i64 %in2) {
+  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %gep = getelementptr inbounds i64, i64 addrspace(1)* %arg, i32 %id
+  %load = load i64, i64 addrspace(1)* %gep
+  %tmp0 = call i64 @llvm.amdgcn.update.dpp.i64(i64 %in1, i64 %load, i32 1, i32 1, i32 1, i1 0) #0
+  store i64 %tmp0, i64 addrspace(1)* %gep
+  ret void
+}
+
+; GCN-LABEL: {{^}}update_dpp64_imm_old_test:
+; GCN-OPT-DAG: v_mov_b32_e32 v[[OLD_LO:[0-9]+]], 0x3afaedd9
+; GCN-OPT-DAG: v_mov_b32_e32 v[[OLD_HI:[0-9]+]], 0x7047
+; GFX8-NOOPT-DAG: s_mov_b32 s[[SOLD_LO:[0-9]+]], 0x3afaedd9
+; GFX8-NOOPT-DAG: s_movk_i32 s[[SOLD_HI:[0-9]+]], 0x7047
+; GCN-DAG: load_dwordx2 v{{\[}}[[SRC_LO:[0-9]+]]:[[SRC_HI:[0-9]+]]]
+; GCN-OPT-DAG: v_mov_b32_dpp v[[OLD_LO]], v[[SRC_LO]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1{{$}}
+; GCN-OPT-DAG: v_mov_b32_dpp v[[OLD_HI]], v[[SRC_HI]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1{{$}}
+; GCN-NOOPT-DAG: v_mov_b32_dpp v{{[0-9]+}}, v[[SRC_LO]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1{{$}}
+; GCN-NOOPT-DAG: v_mov_b32_dpp v{{[0-9]+}}, v[[SRC_HI]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1{{$}}
+define amdgpu_kernel void @update_dpp64_imm_old_test(i64 addrspace(1)* %arg, i64 %in2) {
+  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %gep = getelementptr inbounds i64, i64 addrspace(1)* %arg, i32 %id
+  %load = load i64, i64 addrspace(1)* %gep
+  %tmp0 = call i64 @llvm.amdgcn.update.dpp.i64(i64 123451234512345, i64 %load, i32 1, i32 1, i32 1, i1 0) #0
+  store i64 %tmp0, i64 addrspace(1)* %gep
+  ret void
+}
+
+; GCN-LABEL: {{^}}update_dpp64_imm_src_test:
+; GCN-OPT-DAG: v_mov_b32_e32 v[[OLD_LO:[0-9]+]], 0x3afaedd9
+; GCN-OPT-DAG: v_mov_b32_e32 v[[OLD_HI:[0-9]+]], 0x7047
+; GFX8-NOOPT-DAG: s_mov_b32 s[[SOLD_LO:[0-9]+]], 0x3afaedd9
+; GFX8-NOOPT-DAG: s_movk_i32 s[[SOLD_HI:[0-9]+]], 0x7047
+; GCN-OPT-DAG: v_mov_b32_dpp v{{[0-9]+}}, v[[OLD_LO]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1{{$}}
+; GCN-OPT-DAG: v_mov_b32_dpp v{{[0-9]+}}, v[[OLD_HI]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1{{$}}
+; GCN-NOOPT-DAG: v_mov_b32_dpp v{{[0-9]+}}, v[[SRC_LO]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1{{$}}
+; GCN-NOOPT-DAG: v_mov_b32_dpp v{{[0-9]+}}, v[[SRC_HI]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1{{$}}
+define amdgpu_kernel void @update_dpp64_imm_src_test(i64 addrspace(1)* %out, i64 %in1) {
+  %tmp0 = call i64 @llvm.amdgcn.update.dpp.i64(i64 %in1, i64 123451234512345, i32 1, i32 1, i32 1, i1 0) #0
+  store i64 %tmp0, i64 addrspace(1)* %out
+  ret void
+}
+
 declare i32 @llvm.amdgcn.workitem.id.x()
 declare void @llvm.amdgcn.s.barrier()
 declare i32 @llvm.amdgcn.update.dpp.i32(i32, i32, i32, i32, i32, i1) #0
+declare i64 @llvm.amdgcn.update.dpp.i64(i64, i64, i32, i32, i32, i1) #0
 
 attributes #0 = { nounwind readnone convergent }
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wqm.vote.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wqm.vote.ll
index 57af85b67..07aefa2a7 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wqm.vote.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wqm.vote.ll
@@ -1,8 +1,10 @@
-; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=CHECK %s
+; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=CHECK,WAVE64  %s
+; RUN: llc -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=CHECK,WAVE32 %s
 
 ;CHECK-LABEL: {{^}}ret:
 ;CHECK: v_cmp_eq_u32_e32 [[CMP:[^,]+]], v0, v1
-;CHECK: s_wqm_b64 [[WQM:[^,]+]], [[CMP]]
+;WAVE64: s_wqm_b64 [[WQM:[^,]+]], [[CMP]]
+;WAVE32: s_wqm_b32 [[WQM:[^,]+]], [[CMP]]
 ;CHECK: v_cndmask_b32_e64 v0, 0, 1.0, [[WQM]]
 define amdgpu_ps float @ret(i32 %v0, i32 %v1) #1 {
 main_body:
@@ -13,7 +15,8 @@ main_body:
 }
 
 ;CHECK-LABEL: {{^}}true:
-;CHECK: s_wqm_b64
+;WAVE64: s_wqm_b64
+;WAVE32: s_wqm_b32
 define amdgpu_ps float @true() #1 {
 main_body:
   %w = call i1 @llvm.amdgcn.wqm.vote(i1 true)
@@ -22,7 +25,8 @@ main_body:
 }
 
 ;CHECK-LABEL: {{^}}false:
-;CHECK: s_wqm_b64
+;WAVE64: s_wqm_b64
+;WAVE32: s_wqm_b32
 define amdgpu_ps float @false() #1 {
 main_body:
   %w = call i1 @llvm.amdgcn.wqm.vote(i1 false)
@@ -32,8 +36,13 @@ main_body:
 
 ;CHECK-LABEL: {{^}}kill:
 ;CHECK: v_cmp_eq_u32_e32 [[CMP:[^,]+]], v0, v1
-;CHECK: s_wqm_b64 [[WQM:[^,]+]], [[CMP]]
-;CHECK: s_and_b64 exec, exec, [[WQM]]
+
+;WAVE64: s_wqm_b64 [[WQM:[^,]+]], [[CMP]]
+;WAVE64: s_and_b64 exec, exec, [[WQM]]
+
+;WAVE32: s_wqm_b32 [[WQM:[^,]+]], [[CMP]]
+;WAVE32: s_and_b32 exec_lo, exec_lo, [[WQM]]
+
 ;CHECK: s_endpgm
 define amdgpu_ps void @kill(i32 %v0, i32 %v1) #1 {
 main_body:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.writelane.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.writelane.ll
index 2d0ebe8ed..fd93fbc43 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.writelane.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.writelane.ll
@@ -1,10 +1,12 @@
-; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx802 -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck -check-prefixes=CHECK,CI,CIGFX9 %s
+; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx802 -verify-machineinstrs < %s | FileCheck -check-prefixes=CHECK,GFX9,CIGFX9 %s
+; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefixes=CHECK,GFX10 %s
 
 declare i32 @llvm.amdgcn.writelane(i32, i32, i32) #0
 
 ; CHECK-LABEL: {{^}}test_writelane_sreg:
-; CHECK: v_writelane_b32 v{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}}
+; CIGFX9: v_writelane_b32 v{{[0-9]+}}, s{{[0-9]+}}, m0
+; GFX10: v_writelane_b32 v{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}}
 define amdgpu_kernel void @test_writelane_sreg(i32 addrspace(1)* %out, i32 %src0, i32 %src1) #1 {
   %oldval = load i32, i32 addrspace(1)* %out
   %writelane = call i32 @llvm.amdgcn.writelane(i32 %src0, i32 %src1, i32 %oldval)
@@ -35,11 +37,11 @@ define amdgpu_kernel void @test_writelane_vreg_lane(i32 addrspace(1)* %out, <2 x
   ret void
 }
 
-; TODO: m0 should be folded.
 ; CHECK-LABEL: {{^}}test_writelane_m0_sreg:
 ; CHECK: s_mov_b32 m0, -1
-; CHECK: s_mov_b32 [[COPY_M0:s[0-9]+]], m0
-; CHECK: v_writelane_b32 v{{[0-9]+}}, [[COPY_M0]], s{{[0-9]+}}
+; CIGFX9: s_mov_b32 [[COPY_M0:s[0-9]+]], m0
+; CIGFX9: v_writelane_b32 v{{[0-9]+}}, [[COPY_M0]], m0
+; GFX10: v_writelane_b32 v{{[0-9]+}}, m0, s{{[0-9]+}}
 define amdgpu_kernel void @test_writelane_m0_sreg(i32 addrspace(1)* %out, i32 %src1) #1 {
   %oldval = load i32, i32 addrspace(1)* %out
   %m0 = call i32 asm "s_mov_b32 m0, -1", "={m0}"()
@@ -59,7 +61,8 @@ define amdgpu_kernel void @test_writelane_imm(i32 addrspace(1)* %out, i32 %src0)
 
 ; CHECK-LABEL: {{^}}test_writelane_sreg_oldval:
 ; CHECK: v_mov_b32_e32 [[OLDVAL:v[0-9]+]], s{{[0-9]+}}
-; CHECK: v_writelane_b32 [[OLDVAL]], s{{[0-9]+}}, s{{[0-9]+}}
+; CIGFX9: v_writelane_b32 [[OLDVAL]], s{{[0-9]+}}, m0
+; GFX10: v_writelane_b32 [[OLDVAL]], s{{[0-9]+}}, s{{[0-9]+}}
 define amdgpu_kernel void @test_writelane_sreg_oldval(i32 inreg %oldval, i32 addrspace(1)* %out, i32 %src0, i32 %src1) #1 {
   %writelane = call i32 @llvm.amdgcn.writelane(i32 %src0, i32 %src1, i32 %oldval)
   store i32 %writelane, i32 addrspace(1)* %out, align 4
@@ -68,7 +71,8 @@ define amdgpu_kernel void @test_writelane_sreg_oldval(i32 inreg %oldval, i32 add
 
 ; CHECK-LABEL: {{^}}test_writelane_imm_oldval:
 ; CHECK: v_mov_b32_e32 [[OLDVAL:v[0-9]+]], 42
-; CHECK: v_writelane_b32 [[OLDVAL]], s{{[0-9]+}}, s{{[0-9]+}}
+; CIGFX9: v_writelane_b32 [[OLDVAL]], s{{[0-9]+}}, m0
+; GFX10: v_writelane_b32 [[OLDVAL]], s{{[0-9]+}}, s{{[0-9]+}}
 define amdgpu_kernel void @test_writelane_imm_oldval(i32 addrspace(1)* %out, i32 %src0, i32 %src1) #1 {
   %writelane = call i32 @llvm.amdgcn.writelane(i32 %src0, i32 %src1, i32 42)
   store i32 %writelane, i32 addrspace(1)* %out, align 4
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.fma.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.fma.f16.ll
index e5713b5bf..7e7af3e58 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.fma.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.fma.f16.ll
@@ -1,8 +1,10 @@
-; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=SI %s
-; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=VI %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,SI,SIVI %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,VI,SIVI,VIGFX9 %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9,VIGFX9 %s
 
 declare half @llvm.fma.f16(half %a, half %b, half %c)
 declare <2 x half> @llvm.fma.v2f16(<2 x half> %a, <2 x half> %b, <2 x half> %c)
+declare <4 x half> @llvm.fma.v4f16(<4 x half> %a, <4 x half> %b, <4 x half> %c)
 
 ; GCN-LABEL: {{^}}fma_f16
 ; GCN: buffer_load_ushort v[[A_F16:[0-9]+]]
@@ -13,7 +15,7 @@ declare <2 x half> @llvm.fma.v2f16(<2 x half> %a, <2 x half> %b, <2 x half> %c)
 ; SI:  v_cvt_f32_f16_e32 v[[C_F32:[0-9]+]], v[[C_F16]]
 ; SI:  v_fma_f32 v[[R_F32:[0-9]+]], v[[A_F32:[0-9]]], v[[B_F32:[0-9]]], v[[C_F32:[0-9]]]
 ; SI:  v_cvt_f16_f32_e32 v[[R_F16:[0-9]+]], v[[R_F32]]
-; VI:  v_fma_f16 v[[R_F16:[0-9]+]], v[[A_F16]], v[[B_F16]], v[[C_F16]]
+; VIGFX9:  v_fma_f16 v[[R_F16:[0-9]+]], v[[A_F16]], v[[B_F16]], v[[C_F16]]
 ; GCN: buffer_store_short v[[R_F16]]
 ; GCN: s_endpgm
 define amdgpu_kernel void @fma_f16(
@@ -38,8 +40,8 @@ define amdgpu_kernel void @fma_f16(
 ; SI:  v_cvt_f32_f16_e32 v[[C_F32:[0-9]+]], v[[C_F16]]
 ; SI:  v_fma_f32 v[[R_F32:[0-9]+]], v[[B_F32:[0-9]]], s[[A_F32:[0-9]]], v[[C_F32:[0-9]]]
 ; SI:  v_cvt_f16_f32_e32 v[[R_F16:[0-9]+]], v[[R_F32]]
-; VI:  s_movk_i32 s[[A_F16:[0-9]+]], 0x4200{{$}}
-; VI:  v_fma_f16 v[[R_F16:[0-9]+]], v[[B_F16]], s[[A_F16]], v[[C_F16]]
+; VIGFX9:  s_movk_i32 s[[A_F16:[0-9]+]], 0x4200{{$}}
+; VIGFX9:  v_fma_f16 v[[R_F16:[0-9]+]], v[[B_F16]], s[[A_F16]], v[[C_F16]]
 ; GCN: buffer_store_short v[[R_F16]]
 ; GCN: s_endpgm
 define amdgpu_kernel void @fma_f16_imm_a(
@@ -61,8 +63,8 @@ define amdgpu_kernel void @fma_f16_imm_a(
 ; SI:  v_cvt_f32_f16_e32 v[[C_F32:[0-9]+]], v[[C_F16]]
 ; SI:  v_fma_f32 v[[R_F32:[0-9]+]], v[[A_F32:[0-9]]], s[[B_F32:[0-9]]], v[[C_F32:[0-9]]]
 ; SI:  v_cvt_f16_f32_e32 v[[R_F16:[0-9]+]], v[[R_F32]]
-; VI:  s_movk_i32 s[[B_F16:[0-9]+]], 0x4200{{$}}
-; VI:  v_fma_f16 v[[R_F16:[0-9]+]], v[[A_F16]], s[[B_F16]], v[[C_F16]]
+; VIGFX9:  s_movk_i32 s[[B_F16:[0-9]+]], 0x4200{{$}}
+; VIGFX9:  v_fma_f16 v[[R_F16:[0-9]+]], v[[A_F16]], s[[B_F16]], v[[C_F16]]
 ; GCN: buffer_store_short v[[R_F16]]
 ; GCN: s_endpgm
 define amdgpu_kernel void @fma_f16_imm_b(
@@ -84,8 +86,8 @@ define amdgpu_kernel void @fma_f16_imm_b(
 ; SI:  v_cvt_f32_f16_e32 v[[B_F32:[0-9]+]], v[[B_F16]]
 ; SI:  v_fma_f32 v[[R_F32:[0-9]+]], v[[A_F32:[0-9]]], v[[B_F32:[0-9]]], s[[C_F32:[0-9]]]
 ; SI:  v_cvt_f16_f32_e32 v[[R_F16:[0-9]+]], v[[R_F32]]
-; VI:  s_movk_i32 s[[C_F16:[0-9]+]], 0x4200{{$}}
-; VI:  v_fma_f16 v[[R_F16:[0-9]+]], v[[A_F16]], v[[B_F16]], s[[C_F16]]
+; VIGFX9:  s_movk_i32 s[[C_F16:[0-9]+]], 0x4200{{$}}
+; VIGFX9:  v_fma_f16 v[[R_F16:[0-9]+]], v[[A_F16]], v[[B_F16]], s[[C_F16]]
 ; GCN: buffer_store_short v[[R_F16]]
 ; GCN: s_endpgm
 define amdgpu_kernel void @fma_f16_imm_c(
@@ -127,9 +129,11 @@ define amdgpu_kernel void @fma_f16_imm_c(
 ; VI-DAG: v_fma_f16 v[[R_F16_0:[0-9]+]], v[[A_V2_F16]], v[[B_V2_F16]], v[[C_V2_F16]]
 ; VI-DAG: v_fma_f16 v[[R_F16_1:[0-9]+]], v[[A_F16_1]], v[[B_F16_1]], v[[C_F16_1]]
 
-; GCN-DAG: v_lshlrev_b32_e32 v[[R_F16_HI:[0-9]+]], 16, v[[R_F16_1]]
+; GFX9: v_pk_fma_f16 v[[R_V2_F16:[0-9]+]], v[[A_V2_F16]], v[[B_V2_F16]], v[[C_V2_F16]]
+
+; SIVI-DAG: v_lshlrev_b32_e32 v[[R_F16_HI:[0-9]+]], 16, v[[R_F16_1]]
 ; GCN-NOT: and
-; GCN: v_or_b32_e32 v[[R_V2_F16:[0-9]+]], v[[R_F16_0]], v[[R_F16_HI]]
+; SIVI: v_or_b32_e32 v[[R_V2_F16:[0-9]+]], v[[R_F16_0]], v[[R_F16_HI]]
 ; GCN: buffer_store_dword v[[R_V2_F16]]
 ; GCN: s_endpgm
 define amdgpu_kernel void @fma_v2f16(
@@ -150,14 +154,14 @@ define amdgpu_kernel void @fma_v2f16(
 ; SI: buffer_load_dword v[[B_V2_F16:[0-9]+]]
 
 
-; VI: buffer_load_dword v[[C_V2_F16:[0-9]+]]
-; VI: buffer_load_dword v[[B_V2_F16:[0-9]+]]
+; VIGFX9: buffer_load_dword v[[C_V2_F16:[0-9]+]]
+; VIGFX9: buffer_load_dword v[[B_V2_F16:[0-9]+]]
 
 
 ; SI:  s_mov_b32 s[[A_F32:[0-9]+]], 0x40400000{{$}}
-; VI:  s_movk_i32 s[[A_F16:[0-9]+]], 0x4200{{$}}
-; GCN-DAG: v_lshrrev_b32_e32 v[[B_F16_1:[0-9]+]], 16, v[[B_V2_F16]]
-; GCN-DAG: v_lshrrev_b32_e32 v[[C_F16_1:[0-9]+]], 16, v[[C_V2_F16]]
+; VIGFX9:  s_movk_i32 s[[A_F16:[0-9]+]], 0x4200{{$}}
+; SIVI-DAG: v_lshrrev_b32_e32 v[[B_F16_1:[0-9]+]], 16, v[[B_V2_F16]]
+; SIVI-DAG: v_lshrrev_b32_e32 v[[C_F16_1:[0-9]+]], 16, v[[C_V2_F16]]
 
 ; SI: v_cvt_f32_f16_e32 v[[C_F32_1:[0-9]+]], v[[C_F16_1]]
 ; SI: v_cvt_f32_f16_e32 v[[B_F32_1:[0-9]+]], v[[B_F16_1]]
@@ -172,9 +176,11 @@ define amdgpu_kernel void @fma_v2f16(
 ; VI-DAG:  v_fma_f16 v[[R_F16_1:[0-9]+]], v[[C_F16_1]], s[[A_F16]], v[[B_F16_1]]
 ; VI-DAG:  v_fma_f16 v[[R_F16_0:[0-9]+]], v[[C_V2_F16]], s[[A_F16]], v[[B_V2_F16]]
 
-; GCN-DAG: v_lshlrev_b32_e32 v[[R_F16_HI:[0-9]+]], 16, v[[R_F16_1]]
+; GFX9: v_pk_fma_f16 v[[R_V2_F16:[0-9]+]], v[[C_V2_F16]], s[[A_F16]], v[[B_V2_F16]]
+
+; SIVI-DAG: v_lshlrev_b32_e32 v[[R_F16_HI:[0-9]+]], 16, v[[R_F16_1]]
 ; GCN-NOT: and
-; GCN: v_or_b32_e32 v[[R_V2_F16:[0-9]+]], v[[R_F16_0]], v[[R_F16_HI]]
+; SIVI: v_or_b32_e32 v[[R_V2_F16:[0-9]+]], v[[R_F16_0]], v[[R_F16_HI]]
 ; GCN: buffer_store_dword v[[R_V2_F16]]
 ; GCN: s_endpgm
 define amdgpu_kernel void @fma_v2f16_imm_a(
@@ -192,11 +198,11 @@ define amdgpu_kernel void @fma_v2f16_imm_a(
 ; SI: buffer_load_dword v[[C_V2_F16:[0-9]+]]
 ; SI: buffer_load_dword v[[A_V2_F16:[0-9]+]]
 
-; VI: buffer_load_dword v[[A_V2_F16:[0-9]+]]
-; VI: buffer_load_dword v[[C_V2_F16:[0-9]+]]
+; VIGFX9: buffer_load_dword v[[A_V2_F16:[0-9]+]]
+; VIGFX9: buffer_load_dword v[[C_V2_F16:[0-9]+]]
 
 ; SI:  s_mov_b32 s[[B_F32:[0-9]+]], 0x40400000{{$}}
-; VI:  s_movk_i32 s[[B_F16:[0-9]+]], 0x4200{{$}}
+; VIGFX9:  s_movk_i32 s[[B_F16:[0-9]+]], 0x4200{{$}}
 
 ; SI-DAG:  v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_V2_F16]]
 ; SI-DAG:  v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_V2_F16]]
@@ -215,9 +221,11 @@ define amdgpu_kernel void @fma_v2f16_imm_a(
 ; VI-DAG:  v_fma_f16 v[[R_F16_0:[0-9]+]], v[[A_V2_F16]], s[[B_F16]], v[[C_V2_F16]]
 ; VI-DAG:  v_fma_f16 v[[R_F16_1:[0-9]+]], v[[A_F16_1]], s[[B_F16]], v[[C_F16_1]]
 
-; GCN-DAG: v_lshlrev_b32_e32 v[[R_F16_HI:[0-9]+]], 16, v[[R_F16_1]]
+; GFX9: v_pk_fma_f16 v[[R_V2_F16:[0-9]+]], v[[A_V2_F16]], s[[B_F16]], v[[C_V2_F16]]
+
+; SIVI-DAG: v_lshlrev_b32_e32 v[[R_F16_HI:[0-9]+]], 16, v[[R_F16_1]]
 ; GCN-NOT: and
-; GCN: v_or_b32_e32 v[[R_V2_F16:[0-9]+]], v[[R_F16_0]], v[[R_F16_HI]]
+; SIVI: v_or_b32_e32 v[[R_V2_F16:[0-9]+]], v[[R_F16_0]], v[[R_F16_HI]]
 ; GCN: buffer_store_dword v[[R_V2_F16]]
 ; GCN: s_endpgm
 define amdgpu_kernel void @fma_v2f16_imm_b(
@@ -235,11 +243,11 @@ define amdgpu_kernel void @fma_v2f16_imm_b(
 ; SI: buffer_load_dword v[[B_V2_F16:[0-9]+]]
 ; SI: buffer_load_dword v[[A_V2_F16:[0-9]+]]
 
-; VI: buffer_load_dword v[[A_V2_F16:[0-9]+]]
-; VI: buffer_load_dword v[[B_V2_F16:[0-9]+]]
+; VIGFX9: buffer_load_dword v[[A_V2_F16:[0-9]+]]
+; VIGFX9: buffer_load_dword v[[B_V2_F16:[0-9]+]]
 
 ; SI:  s_mov_b32 s[[C_F32:[0-9]+]], 0x40400000{{$}}
-; VI:  s_movk_i32 s[[C_F16:[0-9]+]], 0x4200{{$}}
+; VIGFX9:  s_movk_i32 s[[C_F16:[0-9]+]], 0x4200{{$}}
 
 ; SI:  v_lshrrev_b32_e32 v[[B_F16_1:[0-9]+]], 16, v[[B_V2_F16]]
 ; SI:  v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_V2_F16]]
@@ -265,6 +273,7 @@ define amdgpu_kernel void @fma_v2f16_imm_b(
 ; GCN-NOT: and
 ; VI: v_or_b32_e32 v[[R_V2_F16:[0-9]+]], v[[R_F16_0]], v[[R_F16_1]]
 
+; GFX9: v_pk_fma_f16 v[[R_V2_F16:[0-9]+]], v[[A_V2_F16]], v[[B_V2_F16]], s[[C_F16]]
 
 ; GCN: buffer_store_dword v[[R_V2_F16]]
 ; GCN: s_endpgm
@@ -278,3 +287,74 @@ define amdgpu_kernel void @fma_v2f16_imm_c(
   store <2 x half> %r.val, <2 x half> addrspace(1)* %r
   ret void
 }
+
+; GCN-LABEL: {{^}}fma_v4f16
+; GCN: buffer_load_dwordx2 v{{\[}}[[A_V4_F16_LO:[0-9]+]]:[[A_V4_F16_HI:[0-9]+]]{{\]}}
+; GCN: buffer_load_dwordx2 v{{\[}}[[B_V4_F16_LO:[0-9]+]]:[[B_V4_F16_HI:[0-9]+]]{{\]}}
+; GCN: buffer_load_dwordx2 v{{\[}}[[C_V4_F16_LO:[0-9]+]]:[[C_V4_F16_HI:[0-9]+]]{{\]}}
+
+; SI-DAG: v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_V4_F16_LO]]
+; SI-DAG: v_lshrrev_b32_e32 v[[A_F16_0:[0-9]+]], 16, v[[A_V4_F16_LO]]
+; SI-DAG: v_cvt_f32_f16_e32 v[[A_F32_1:[0-9]+]], v[[A_V4_F16_HI]]
+; SI-DAG: v_lshrrev_b32_e32 v[[A_F16_2:[0-9]+]], 16, v[[A_V4_F16_HI]]
+; SI-DAG: v_cvt_f32_f16_e32 v[[B_F32_0:[0-9]+]], v[[B_V4_F16_LO]]
+; SI-DAG: v_cvt_f32_f16_e32 v[[C_F32_0:[0-9]+]], v[[C_V4_F16_LO]]
+; SI-DAG: v_lshrrev_b32_e32 v[[B_F16_0:[0-9]+]], 16, v[[B_V4_F16_LO]]
+; SI-DAG: v_lshrrev_b32_e32 v[[C_F16_0:[0-9]+]], 16, v[[C_V4_F16_LO]]
+; SI-DAG: v_cvt_f32_f16_e32 v[[B_F32_1:[0-9]+]], v[[B_V4_F16_HI]]
+; SI-DAG: v_cvt_f32_f16_e32 v[[C_F32_1:[0-9]+]], v[[C_V4_F16_HI]]
+; SI-DAG: v_lshrrev_b32_e32 v[[B_F16_1:[0-9]+]], 16, v[[B_V4_F16_HI]]
+; SI-DAG: v_lshrrev_b32_e32 v[[C_F16_1:[0-9]+]], 16, v[[C_V4_F16_HI]]
+; SI-DAG: v_cvt_f32_f16_e32 v[[A_F32_2:[0-9]+]], v[[A_V4_F16_LO]]
+; SI-DAG: v_cvt_f32_f16_e32 v[[A_F32_3:[0-9]+]], v[[A_V4_F16_HI]]
+; SI-DAG: v_cvt_f32_f16_e32 v[[B_F32_2:[0-9]+]], v[[B_V4_F16_LO]]
+; SI-DAG: v_cvt_f32_f16_e32 v[[B_F32_3:[0-9]+]], v[[B_V4_F16_HI]]
+; SI-DAG: v_cvt_f32_f16_e32 v[[C_F32_2:[0-9]+]], v[[C_V4_F16_LO]]
+; SI-DAG: v_cvt_f32_f16_e32 v[[C_F32_3:[0-9]+]], v[[C_V4_F16_HI]]
+
+; SI-DAG: v_fma_f32 v[[R_F32_0:[0-9]+]], v[[A_F32_0]], v[[B_F32_0]], v[[C_F32_0]]
+; SI-DAG: v_fma_f32 v[[R_F32_1:[0-9]+]], v[[A_F32_1]], v[[B_F32_1]], v[[C_F32_1]]
+; SI-DAG: v_fma_f32 v[[R_F32_2:[0-9]+]], v[[A_F32_2]], v[[B_F32_2]], v[[C_F32_2]]
+; SI-DAG: v_fma_f32 v[[R_F32_3:[0-9]+]], v[[A_F32_3]], v[[B_F32_3]], v[[C_F32_3]]
+
+; SI-DAG: v_cvt_f16_f32_e32 v[[R_F16_0:[0-9]+]], v[[R_F32_0]]
+; SI-DAG: v_cvt_f16_f32_e32 v[[R_F16_1:[0-9]+]], v[[R_F32_1]]
+; SI-DAG: v_cvt_f16_f32_e32 v[[R_F16_2:[0-9]+]], v[[R_F32_2]]
+; SI-DAG: v_cvt_f16_f32_e32 v[[R_F16_3:[0-9]+]], v[[R_F32_3]]
+
+; SI-DAG: v_lshlrev_b32_e32 v[[R1_F16_0:[0-9]]], 16, v[[R_F16_2]]
+; SI-DAG: v_lshlrev_b32_e32 v[[R1_F16_1:[0-9]]], 16, v[[R_F16_3]]
+
+; VI-DAG: v_lshrrev_b32_e32 v[[A_F16_0:[0-9]+]], 16, v[[A_V4_F16_LO]]
+; VI-DAG: v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_V4_F16_HI]]
+; VI-DAG: v_lshrrev_b32_e32 v[[B_F16_0:[0-9]+]], 16, v[[B_V4_F16_LO]]
+; VI-DAG: v_lshrrev_b32_e32 v[[B_F16_1:[0-9]+]], 16, v[[B_V4_F16_HI]]
+; VI-DAG: v_lshrrev_b32_e32 v[[C_F16_0:[0-9]+]], 16, v[[C_V4_F16_LO]]
+; VI-DAG: v_lshrrev_b32_e32 v[[C_F16_1:[0-9]+]], 16, v[[C_V4_F16_HI]]
+
+; VI-DAG: v_fma_f16 v[[R_F16_0:[0-9]+]], v[[A_V4_F16_LO]], v[[B_V4_F16_LO]], v[[C_V4_F16_LO]]
+; VI-DAG: v_fma_f16 v[[R1_F16_0:[0-9]+]], v[[A_F16_0]], v[[B_F16_0]], v[[C_F16_0]]
+; VI-DAG: v_fma_f16 v[[R_F16_1:[0-9]+]], v[[A_V4_F16_HI]], v[[B_V4_F16_HI]], v[[C_V4_F16_HI]]
+; VI-DAG: v_fma_f16 v[[R1_F16_1:[0-9]+]], v[[A_F16_1]], v[[B_F16_1]], v[[C_F16_1]]
+
+; SIVI-DAG: v_or_b32_e32 v[[R_V4_F16_LO:[0-9]+]], v[[R_F16_0]], v[[R1_F16_0]]
+; SIVI-DAG: v_or_b32_e32 v[[R_V4_F16_HI:[0-9]+]], v[[R_F16_1]], v[[R1_F16_1]]
+
+; GFX9-DAG: v_pk_fma_f16 v[[R_V4_F16_LO:[0-9]+]], v[[A_V4_F16_LO]], v[[B_V4_F16_LO]], v[[C_V4_F16_LO]]
+; GFX9-DAG: v_pk_fma_f16 v[[R_V4_F16_HI:[0-9]+]], v[[A_V4_F16_HI]], v[[B_V4_F16_HI]], v[[C_V4_F16_HI]]
+
+; GCN: buffer_store_dwordx2 v{{\[}}[[R_V4_F16_LO]]:[[R_V4_F16_HI]]{{\]}}
+; GCN: s_endpgm
+
+define amdgpu_kernel void @fma_v4f16(
+    <4 x half> addrspace(1)* %r,
+    <4 x half> addrspace(1)* %a,
+    <4 x half> addrspace(1)* %b,
+    <4 x half> addrspace(1)* %c) {
+  %a.val = load <4 x half>, <4 x half> addrspace(1)* %a
+  %b.val = load <4 x half>, <4 x half> addrspace(1)* %b
+  %c.val = load <4 x half>, <4 x half> addrspace(1)* %c
+  %r.val = call <4 x half> @llvm.fma.v4f16(<4 x half> %a.val, <4 x half> %b.val, <4 x half> %c.val)
+  store <4 x half> %r.val, <4 x half> addrspace(1)* %r
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.maxnum.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.maxnum.f16.ll
index 12573a5fe..4f32340a0 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.maxnum.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.maxnum.f16.ll
@@ -562,31 +562,32 @@ define amdgpu_kernel void @maxnum_v4f16(
 ; SI-NEXT:    s_mov_b32 s0, s4
 ; SI-NEXT:    s_mov_b32 s1, s5
 ; SI-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
-; SI-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v0, s4
+; SI-NEXT:    v_cvt_f32_f16_e32 v1, s5
 ; SI-NEXT:    s_lshr_b32 s4, s4, 16
+; SI-NEXT:    s_lshr_b32 s5, s5, 16
 ; SI-NEXT:    v_cvt_f32_f16_e32 v2, s4
-; SI-NEXT:    s_lshr_b32 s4, s5, 16
-; SI-NEXT:    v_cvt_f32_f16_e32 v3, s4
-; SI-NEXT:    s_lshr_b32 s4, s7, 16
-; SI-NEXT:    v_cvt_f32_f16_e32 v5, s4
-; SI-NEXT:    v_cvt_f32_f16_e32 v1, s5
-; SI-NEXT:    s_lshr_b32 s4, s6, 16
-; SI-NEXT:    v_cvt_f32_f16_e32 v7, s7
+; SI-NEXT:    v_cvt_f32_f16_e32 v3, s5
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
+; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v1
+; SI-NEXT:    v_mul_f32_e32 v2, 1.0, v2
+; SI-NEXT:    v_mul_f32_e32 v3, 1.0, v3
+; SI-NEXT:    v_mul_f32_e32 v0, 1.0, v0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_lshr_b32 s6, s5, 16
+; SI-NEXT:    v_cvt_f32_f16_e32 v5, s6
+; SI-NEXT:    v_cvt_f32_f16_e32 v4, s4
+; SI-NEXT:    s_lshr_b32 s4, s4, 16
+; SI-NEXT:    v_cvt_f32_f16_e32 v7, s5
 ; SI-NEXT:    v_cvt_f32_f16_e32 v6, s4
-; SI-NEXT:    v_cvt_f32_f16_e32 v4, s6
 ; SI-NEXT:    v_mul_f32_e32 v5, 1.0, v5
-; SI-NEXT:    v_mul_f32_e32 v3, 1.0, v3
 ; SI-NEXT:    v_max_f32_e32 v3, v3, v5
 ; SI-NEXT:    v_mul_f32_e32 v5, 1.0, v7
-; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v1
 ; SI-NEXT:    v_max_f32_e32 v1, v1, v5
 ; SI-NEXT:    v_mul_f32_e32 v5, 1.0, v6
-; SI-NEXT:    v_mul_f32_e32 v2, 1.0, v2
 ; SI-NEXT:    v_max_f32_e32 v2, v2, v5
 ; SI-NEXT:    v_mul_f32_e32 v4, 1.0, v4
-; SI-NEXT:    v_mul_f32_e32 v0, 1.0, v0
 ; SI-NEXT:    v_cvt_f16_f32_e32 v3, v3
 ; SI-NEXT:    v_cvt_f16_f32_e32 v2, v2
 ; SI-NEXT:    v_max_f32_e32 v0, v0, v4
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.minnum.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.minnum.f16.ll
index cdf05094f..923383cf5 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.minnum.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.minnum.f16.ll
@@ -615,31 +615,32 @@ define amdgpu_kernel void @minnum_v4f16(
 ; SI-NEXT:    s_mov_b32 s0, s4
 ; SI-NEXT:    s_mov_b32 s1, s5
 ; SI-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
-; SI-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v0, s4
+; SI-NEXT:    v_cvt_f32_f16_e32 v1, s5
 ; SI-NEXT:    s_lshr_b32 s4, s4, 16
+; SI-NEXT:    s_lshr_b32 s5, s5, 16
 ; SI-NEXT:    v_cvt_f32_f16_e32 v2, s4
-; SI-NEXT:    s_lshr_b32 s4, s5, 16
-; SI-NEXT:    v_cvt_f32_f16_e32 v3, s4
-; SI-NEXT:    s_lshr_b32 s4, s7, 16
-; SI-NEXT:    v_cvt_f32_f16_e32 v5, s4
-; SI-NEXT:    v_cvt_f32_f16_e32 v1, s5
-; SI-NEXT:    s_lshr_b32 s4, s6, 16
-; SI-NEXT:    v_cvt_f32_f16_e32 v7, s7
+; SI-NEXT:    v_cvt_f32_f16_e32 v3, s5
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
+; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v1
+; SI-NEXT:    v_mul_f32_e32 v2, 1.0, v2
+; SI-NEXT:    v_mul_f32_e32 v3, 1.0, v3
+; SI-NEXT:    v_mul_f32_e32 v0, 1.0, v0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_lshr_b32 s6, s5, 16
+; SI-NEXT:    v_cvt_f32_f16_e32 v5, s6
+; SI-NEXT:    v_cvt_f32_f16_e32 v4, s4
+; SI-NEXT:    s_lshr_b32 s4, s4, 16
+; SI-NEXT:    v_cvt_f32_f16_e32 v7, s5
 ; SI-NEXT:    v_cvt_f32_f16_e32 v6, s4
-; SI-NEXT:    v_cvt_f32_f16_e32 v4, s6
 ; SI-NEXT:    v_mul_f32_e32 v5, 1.0, v5
-; SI-NEXT:    v_mul_f32_e32 v3, 1.0, v3
 ; SI-NEXT:    v_min_f32_e32 v3, v3, v5
 ; SI-NEXT:    v_mul_f32_e32 v5, 1.0, v7
-; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v1
 ; SI-NEXT:    v_min_f32_e32 v1, v1, v5
 ; SI-NEXT:    v_mul_f32_e32 v5, 1.0, v6
-; SI-NEXT:    v_mul_f32_e32 v2, 1.0, v2
 ; SI-NEXT:    v_min_f32_e32 v2, v2, v5
 ; SI-NEXT:    v_mul_f32_e32 v4, 1.0, v4
-; SI-NEXT:    v_mul_f32_e32 v0, 1.0, v0
 ; SI-NEXT:    v_cvt_f16_f32_e32 v3, v3
 ; SI-NEXT:    v_cvt_f16_f32_e32 v2, v2
 ; SI-NEXT:    v_min_f32_e32 v0, v0, v4
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.round.f64.ll b/llvm/test/CodeGen/AMDGPU/llvm.round.f64.ll
index c58b9b4d9..8d04002d2 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.round.f64.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.round.f64.ll
@@ -1,32 +1,148 @@
-; RUN: llc -march=amdgcn < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -march=amdgcn -mcpu=tahiti < %s | FileCheck -check-prefix=SI %s
+; RUN: llc -march=amdgcn -mcpu=hawaii < %s | FileCheck -check-prefix=CI %s
 
-; FUNC-LABEL: {{^}}round_f64:
-; SI: s_endpgm
 define amdgpu_kernel void @round_f64(double addrspace(1)* %out, double %x) #0 {
+; SI-LABEL: round_f64:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_mov_b32 s11, 0x80000
+; SI-NEXT:    s_mov_b32 s10, 0
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b32 s4, s0
+; SI-NEXT:    s_bfe_u32 s0, s3, 0xb0014
+; SI-NEXT:    s_add_i32 s12, s0, 0xfffffc01
+; SI-NEXT:    s_mov_b32 s5, s1
+; SI-NEXT:    s_mov_b32 s1, 0xfffff
+; SI-NEXT:    s_mov_b32 s0, s6
+; SI-NEXT:    s_lshr_b64 s[0:1], s[0:1], s12
+; SI-NEXT:    s_and_b64 s[8:9], s[2:3], s[0:1]
+; SI-NEXT:    s_lshr_b64 s[10:11], s[10:11], s12
+; SI-NEXT:    v_cmp_ne_u64_e64 vcc, s[8:9], 0
+; SI-NEXT:    v_mov_b32_e32 v1, s10
+; SI-NEXT:    v_mov_b32_e32 v0, s11
+; SI-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; SI-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; SI-NEXT:    v_mov_b32_e32 v2, s3
+; SI-NEXT:    v_add_i32_e32 v1, vcc, s2, v1
+; SI-NEXT:    v_addc_u32_e32 v0, vcc, v2, v0, vcc
+; SI-NEXT:    v_not_b32_e32 v2, s0
+; SI-NEXT:    v_and_b32_e32 v2, v1, v2
+; SI-NEXT:    v_not_b32_e32 v1, s1
+; SI-NEXT:    v_and_b32_e32 v0, v0, v1
+; SI-NEXT:    v_mov_b32_e32 v1, 0x3ff00000
+; SI-NEXT:    v_cmp_eq_u32_e64 vcc, s12, -1
+; SI-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; SI-NEXT:    s_brev_b32 s0, -2
+; SI-NEXT:    v_mov_b32_e32 v3, s3
+; SI-NEXT:    v_bfi_b32 v1, s0, v1, v3
+; SI-NEXT:    v_cmp_lt_i32_e64 vcc, s12, 0
+; SI-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; SI-NEXT:    v_mov_b32_e32 v1, s3
+; SI-NEXT:    v_cmp_gt_i32_e64 s[0:1], s12, 51
+; SI-NEXT:    v_cndmask_b32_e64 v1, v0, v1, s[0:1]
+; SI-NEXT:    v_cndmask_b32_e64 v0, v2, 0, vcc
+; SI-NEXT:    v_mov_b32_e32 v2, s2
+; SI-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[0:1]
+; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; SI-NEXT:    s_endpgm
+;
+; CI-LABEL: round_f64:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; CI-NEXT:    s_brev_b32 s8, -2
+; CI-NEXT:    v_mov_b32_e32 v4, 0x3ff00000
+; CI-NEXT:    s_mov_b32 s3, 0xf000
+; CI-NEXT:    s_mov_b32 s2, -1
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    v_trunc_f64_e32 v[0:1], s[6:7]
+; CI-NEXT:    v_mov_b32_e32 v5, s7
+; CI-NEXT:    v_add_f64 v[2:3], s[6:7], -v[0:1]
+; CI-NEXT:    v_bfi_b32 v4, s8, v4, v5
+; CI-NEXT:    v_cmp_ge_f64_e64 vcc, |v[2:3]|, 0.5
+; CI-NEXT:    v_mov_b32_e32 v2, 0
+; CI-NEXT:    v_cndmask_b32_e32 v3, 0, v4, vcc
+; CI-NEXT:    v_add_f64 v[0:1], v[0:1], v[2:3]
+; CI-NEXT:    s_mov_b32 s0, s4
+; CI-NEXT:    s_mov_b32 s1, s5
+; CI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; CI-NEXT:    s_endpgm
   %result = call double @llvm.round.f64(double %x) #1
   store double %result, double addrspace(1)* %out
   ret void
 }
 
-; This is a pretty large function, so just test a few of the
-; instructions that are necessary.
-
-; FUNC-LABEL: {{^}}v_round_f64:
-; SI: buffer_load_dwordx2
-; SI-DAG: v_bfe_u32 [[EXP:v[0-9]+]], v{{[0-9]+}}, 20, 11
-
-; SI-DAG: v_not_b32_e32
-; SI-DAG: v_not_b32_e32
-
-; SI-DAG: v_cmp_eq_u32
-
-; SI-DAG: s_brev_b32 [[BFIMASK:s[0-9]+]], -2{{$}}
-; SI-DAG: v_cmp_gt_i32
-; SI-DAG: v_bfi_b32 [[COPYSIGN:v[0-9]+]], [[BFIMASK]]
-
-; SI: buffer_store_dwordx2
-; SI: s_endpgm
 define amdgpu_kernel void @v_round_f64(double addrspace(1)* %out, double addrspace(1)* %in) #0 {
+; SI-LABEL: v_round_f64:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
+; SI-NEXT:    v_mov_b32_e32 v1, 0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT:    buffer_load_dwordx2 v[2:3], v[0:1], s[4:7], 0 addr64
+; SI-NEXT:    s_movk_i32 s9, 0xfc01
+; SI-NEXT:    s_mov_b32 s5, 0xfffff
+; SI-NEXT:    s_mov_b32 s4, -1
+; SI-NEXT:    v_mov_b32_e32 v8, 0x3ff00000
+; SI-NEXT:    s_brev_b32 s8, -2
+; SI-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-NEXT:    s_mov_b32 s7, 0x80000
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_bfe_u32 v4, v3, 20, 11
+; SI-NEXT:    v_add_i32_e32 v10, vcc, s9, v4
+; SI-NEXT:    v_lshr_b64 v[4:5], s[4:5], v10
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, -1, v10
+; SI-NEXT:    v_cndmask_b32_e32 v8, 0, v8, vcc
+; SI-NEXT:    v_bfi_b32 v11, s8, v8, v3
+; SI-NEXT:    v_and_b32_e32 v9, v3, v5
+; SI-NEXT:    v_and_b32_e32 v8, v2, v4
+; SI-NEXT:    v_lshr_b64 v[6:7], s[6:7], v10
+; SI-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[8:9]
+; SI-NEXT:    v_not_b32_e32 v4, v4
+; SI-NEXT:    v_cndmask_b32_e32 v6, 0, v6, vcc
+; SI-NEXT:    v_cndmask_b32_e32 v7, 0, v7, vcc
+; SI-NEXT:    v_add_i32_e32 v6, vcc, v2, v6
+; SI-NEXT:    v_addc_u32_e32 v7, vcc, v3, v7, vcc
+; SI-NEXT:    v_not_b32_e32 v5, v5
+; SI-NEXT:    v_and_b32_e32 v5, v7, v5
+; SI-NEXT:    v_cmp_gt_i32_e32 vcc, 0, v10
+; SI-NEXT:    v_and_b32_e32 v4, v6, v4
+; SI-NEXT:    v_cndmask_b32_e32 v5, v5, v11, vcc
+; SI-NEXT:    v_cndmask_b32_e64 v4, v4, 0, vcc
+; SI-NEXT:    v_cmp_lt_i32_e32 vcc, 51, v10
+; SI-NEXT:    v_cndmask_b32_e32 v3, v5, v3, vcc
+; SI-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
+; SI-NEXT:    buffer_store_dwordx2 v[2:3], v[0:1], s[0:3], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; CI-LABEL: v_round_f64:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; CI-NEXT:    s_mov_b32 s7, 0xf000
+; CI-NEXT:    s_mov_b32 s6, 0
+; CI-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
+; CI-NEXT:    v_mov_b32_e32 v1, 0
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; CI-NEXT:    buffer_load_dwordx2 v[2:3], v[0:1], s[4:7], 0 addr64
+; CI-NEXT:    s_brev_b32 s2, -2
+; CI-NEXT:    v_mov_b32_e32 v5, 0x3ff00000
+; CI-NEXT:    v_mov_b32_e32 v4, 0
+; CI-NEXT:    s_waitcnt vmcnt(0)
+; CI-NEXT:    v_trunc_f64_e32 v[6:7], v[2:3]
+; CI-NEXT:    v_add_f64 v[8:9], v[2:3], -v[6:7]
+; CI-NEXT:    v_bfi_b32 v2, s2, v5, v3
+; CI-NEXT:    v_cmp_ge_f64_e64 vcc, |v[8:9]|, 0.5
+; CI-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; CI-NEXT:    v_cndmask_b32_e32 v5, 0, v2, vcc
+; CI-NEXT:    v_add_f64 v[2:3], v[6:7], v[4:5]
+; CI-NEXT:    buffer_store_dwordx2 v[2:3], v[0:1], s[0:3], 0 addr64
+; CI-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x() #1
   %gep = getelementptr double, double addrspace(1)* %in, i32 %tid
   %out.gep = getelementptr double, double addrspace(1)* %out, i32 %tid
@@ -36,25 +152,617 @@ define amdgpu_kernel void @v_round_f64(double addrspace(1)* %out, double addrspa
   ret void
 }
 
-; FUNC-LABEL: {{^}}round_v2f64:
-; SI: s_endpgm
 define amdgpu_kernel void @round_v2f64(<2 x double> addrspace(1)* %out, <2 x double> %in) #0 {
+; SI-LABEL: round_v2f64:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SI-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0xd
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_movk_i32 s18, 0xfc01
+; SI-NEXT:    s_mov_b32 s3, 0xfffff
+; SI-NEXT:    s_mov_b32 s2, s6
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_bfe_u32 s0, s11, 0xb0014
+; SI-NEXT:    s_add_i32 s19, s0, s18
+; SI-NEXT:    s_lshr_b64 s[0:1], s[2:3], s19
+; SI-NEXT:    s_and_b64 s[12:13], s[10:11], s[0:1]
+; SI-NEXT:    s_mov_b32 s15, 0x80000
+; SI-NEXT:    s_mov_b32 s14, 0
+; SI-NEXT:    s_lshr_b64 s[16:17], s[14:15], s19
+; SI-NEXT:    v_cmp_ne_u64_e64 vcc, s[12:13], 0
+; SI-NEXT:    v_mov_b32_e32 v1, s16
+; SI-NEXT:    v_mov_b32_e32 v0, s17
+; SI-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; SI-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; SI-NEXT:    v_mov_b32_e32 v2, s11
+; SI-NEXT:    v_add_i32_e32 v1, vcc, s10, v1
+; SI-NEXT:    v_addc_u32_e32 v0, vcc, v2, v0, vcc
+; SI-NEXT:    v_not_b32_e32 v2, s0
+; SI-NEXT:    v_and_b32_e32 v1, v1, v2
+; SI-NEXT:    v_not_b32_e32 v2, s1
+; SI-NEXT:    v_mov_b32_e32 v4, 0x3ff00000
+; SI-NEXT:    v_cmp_eq_u32_e64 vcc, s19, -1
+; SI-NEXT:    v_and_b32_e32 v0, v0, v2
+; SI-NEXT:    v_cndmask_b32_e32 v2, 0, v4, vcc
+; SI-NEXT:    s_brev_b32 s12, -2
+; SI-NEXT:    v_mov_b32_e32 v3, s11
+; SI-NEXT:    v_bfi_b32 v2, s12, v2, v3
+; SI-NEXT:    v_cmp_lt_i32_e64 vcc, s19, 0
+; SI-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; SI-NEXT:    v_mov_b32_e32 v2, s11
+; SI-NEXT:    v_cmp_gt_i32_e64 s[0:1], s19, 51
+; SI-NEXT:    v_cndmask_b32_e64 v3, v0, v2, s[0:1]
+; SI-NEXT:    v_cndmask_b32_e64 v0, v1, 0, vcc
+; SI-NEXT:    v_mov_b32_e32 v1, s10
+; SI-NEXT:    v_cndmask_b32_e64 v2, v0, v1, s[0:1]
+; SI-NEXT:    s_bfe_u32 s0, s9, 0xb0014
+; SI-NEXT:    s_add_i32 s13, s0, s18
+; SI-NEXT:    s_lshr_b64 s[0:1], s[2:3], s13
+; SI-NEXT:    s_and_b64 s[2:3], s[8:9], s[0:1]
+; SI-NEXT:    s_lshr_b64 s[10:11], s[14:15], s13
+; SI-NEXT:    v_cmp_ne_u64_e64 vcc, s[2:3], 0
+; SI-NEXT:    v_mov_b32_e32 v1, s10
+; SI-NEXT:    v_mov_b32_e32 v0, s11
+; SI-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; SI-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; SI-NEXT:    v_mov_b32_e32 v5, s9
+; SI-NEXT:    v_add_i32_e32 v1, vcc, s8, v1
+; SI-NEXT:    v_addc_u32_e32 v0, vcc, v5, v0, vcc
+; SI-NEXT:    v_not_b32_e32 v5, s0
+; SI-NEXT:    v_and_b32_e32 v5, v1, v5
+; SI-NEXT:    v_not_b32_e32 v1, s1
+; SI-NEXT:    v_cmp_eq_u32_e64 vcc, s13, -1
+; SI-NEXT:    v_and_b32_e32 v0, v0, v1
+; SI-NEXT:    v_cndmask_b32_e32 v1, 0, v4, vcc
+; SI-NEXT:    v_mov_b32_e32 v4, s9
+; SI-NEXT:    v_bfi_b32 v1, s12, v1, v4
+; SI-NEXT:    v_cmp_lt_i32_e64 vcc, s13, 0
+; SI-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; SI-NEXT:    v_mov_b32_e32 v1, s9
+; SI-NEXT:    v_cmp_gt_i32_e64 s[0:1], s13, 51
+; SI-NEXT:    v_cndmask_b32_e64 v1, v0, v1, s[0:1]
+; SI-NEXT:    v_cndmask_b32_e64 v0, v5, 0, vcc
+; SI-NEXT:    v_mov_b32_e32 v4, s8
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    v_cndmask_b32_e64 v0, v0, v4, s[0:1]
+; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; SI-NEXT:    s_endpgm
+;
+; CI-LABEL: round_v2f64:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; CI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0xd
+; CI-NEXT:    s_brev_b32 s6, -2
+; CI-NEXT:    v_mov_b32_e32 v6, 0x3ff00000
+; CI-NEXT:    s_mov_b32 s7, 0xf000
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    v_trunc_f64_e32 v[0:1], s[2:3]
+; CI-NEXT:    v_mov_b32_e32 v4, s3
+; CI-NEXT:    v_add_f64 v[2:3], s[2:3], -v[0:1]
+; CI-NEXT:    v_bfi_b32 v4, s6, v6, v4
+; CI-NEXT:    v_cmp_ge_f64_e64 vcc, |v[2:3]|, 0.5
+; CI-NEXT:    v_mov_b32_e32 v2, 0
+; CI-NEXT:    v_cndmask_b32_e32 v3, 0, v4, vcc
+; CI-NEXT:    v_trunc_f64_e32 v[4:5], s[0:1]
+; CI-NEXT:    v_add_f64 v[2:3], v[0:1], v[2:3]
+; CI-NEXT:    v_add_f64 v[0:1], s[0:1], -v[4:5]
+; CI-NEXT:    v_mov_b32_e32 v7, s1
+; CI-NEXT:    v_cmp_ge_f64_e64 vcc, |v[0:1]|, 0.5
+; CI-NEXT:    v_bfi_b32 v6, s6, v6, v7
+; CI-NEXT:    v_cndmask_b32_e32 v1, 0, v6, vcc
+; CI-NEXT:    v_mov_b32_e32 v0, 0
+; CI-NEXT:    v_add_f64 v[0:1], v[4:5], v[0:1]
+; CI-NEXT:    s_mov_b32 s6, -1
+; CI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; CI-NEXT:    s_endpgm
   %result = call <2 x double> @llvm.round.v2f64(<2 x double> %in) #1
   store <2 x double> %result, <2 x double> addrspace(1)* %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}round_v4f64:
-; SI: s_endpgm
 define amdgpu_kernel void @round_v4f64(<4 x double> addrspace(1)* %out, <4 x double> %in) #0 {
+; SI-LABEL: round_v4f64:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SI-NEXT:    s_load_dwordx8 s[8:15], s[0:1], 0x11
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_movk_i32 s22, 0xfc01
+; SI-NEXT:    s_mov_b32 s3, 0xfffff
+; SI-NEXT:    s_mov_b32 s2, s6
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_bfe_u32 s0, s11, 0xb0014
+; SI-NEXT:    s_add_i32 s23, s0, s22
+; SI-NEXT:    s_lshr_b64 s[0:1], s[2:3], s23
+; SI-NEXT:    s_and_b64 s[16:17], s[10:11], s[0:1]
+; SI-NEXT:    s_mov_b32 s19, 0x80000
+; SI-NEXT:    s_mov_b32 s18, 0
+; SI-NEXT:    s_lshr_b64 s[20:21], s[18:19], s23
+; SI-NEXT:    v_cmp_ne_u64_e64 vcc, s[16:17], 0
+; SI-NEXT:    v_mov_b32_e32 v1, s20
+; SI-NEXT:    v_mov_b32_e32 v0, s21
+; SI-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; SI-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; SI-NEXT:    v_mov_b32_e32 v2, s11
+; SI-NEXT:    v_add_i32_e32 v1, vcc, s10, v1
+; SI-NEXT:    v_addc_u32_e32 v0, vcc, v2, v0, vcc
+; SI-NEXT:    v_not_b32_e32 v2, s0
+; SI-NEXT:    v_and_b32_e32 v1, v1, v2
+; SI-NEXT:    v_not_b32_e32 v2, s1
+; SI-NEXT:    v_mov_b32_e32 v4, 0x3ff00000
+; SI-NEXT:    v_cmp_eq_u32_e64 vcc, s23, -1
+; SI-NEXT:    v_and_b32_e32 v0, v0, v2
+; SI-NEXT:    v_cndmask_b32_e32 v2, 0, v4, vcc
+; SI-NEXT:    s_brev_b32 s20, -2
+; SI-NEXT:    v_mov_b32_e32 v3, s11
+; SI-NEXT:    v_bfi_b32 v2, s20, v2, v3
+; SI-NEXT:    v_cmp_lt_i32_e64 vcc, s23, 0
+; SI-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; SI-NEXT:    v_cmp_gt_i32_e64 s[0:1], s23, 51
+; SI-NEXT:    v_cndmask_b32_e64 v3, v0, v3, s[0:1]
+; SI-NEXT:    v_cndmask_b32_e64 v0, v1, 0, vcc
+; SI-NEXT:    v_mov_b32_e32 v1, s10
+; SI-NEXT:    v_cndmask_b32_e64 v2, v0, v1, s[0:1]
+; SI-NEXT:    s_bfe_u32 s0, s9, 0xb0014
+; SI-NEXT:    s_add_i32 s21, s0, s22
+; SI-NEXT:    s_lshr_b64 s[0:1], s[2:3], s21
+; SI-NEXT:    s_and_b64 s[10:11], s[8:9], s[0:1]
+; SI-NEXT:    s_lshr_b64 s[16:17], s[18:19], s21
+; SI-NEXT:    v_cmp_ne_u64_e64 vcc, s[10:11], 0
+; SI-NEXT:    v_mov_b32_e32 v1, s16
+; SI-NEXT:    v_mov_b32_e32 v0, s17
+; SI-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; SI-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; SI-NEXT:    v_mov_b32_e32 v5, s9
+; SI-NEXT:    v_add_i32_e32 v1, vcc, s8, v1
+; SI-NEXT:    v_addc_u32_e32 v0, vcc, v5, v0, vcc
+; SI-NEXT:    v_not_b32_e32 v5, s0
+; SI-NEXT:    v_and_b32_e32 v5, v1, v5
+; SI-NEXT:    v_not_b32_e32 v1, s1
+; SI-NEXT:    v_cmp_eq_u32_e64 vcc, s21, -1
+; SI-NEXT:    v_and_b32_e32 v0, v0, v1
+; SI-NEXT:    v_cndmask_b32_e32 v1, 0, v4, vcc
+; SI-NEXT:    v_mov_b32_e32 v6, s9
+; SI-NEXT:    v_bfi_b32 v1, s20, v1, v6
+; SI-NEXT:    v_cmp_lt_i32_e64 vcc, s21, 0
+; SI-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; SI-NEXT:    v_cmp_gt_i32_e64 s[0:1], s21, 51
+; SI-NEXT:    v_cndmask_b32_e64 v1, v0, v6, s[0:1]
+; SI-NEXT:    v_cndmask_b32_e64 v0, v5, 0, vcc
+; SI-NEXT:    v_mov_b32_e32 v5, s8
+; SI-NEXT:    v_cndmask_b32_e64 v0, v0, v5, s[0:1]
+; SI-NEXT:    s_bfe_u32 s0, s15, 0xb0014
+; SI-NEXT:    s_add_i32 s16, s0, s22
+; SI-NEXT:    s_lshr_b64 s[0:1], s[2:3], s16
+; SI-NEXT:    s_and_b64 s[8:9], s[14:15], s[0:1]
+; SI-NEXT:    s_lshr_b64 s[10:11], s[18:19], s16
+; SI-NEXT:    v_cmp_ne_u64_e64 vcc, s[8:9], 0
+; SI-NEXT:    v_mov_b32_e32 v6, s10
+; SI-NEXT:    v_mov_b32_e32 v5, s11
+; SI-NEXT:    v_cndmask_b32_e32 v6, 0, v6, vcc
+; SI-NEXT:    v_cndmask_b32_e32 v5, 0, v5, vcc
+; SI-NEXT:    v_mov_b32_e32 v7, s15
+; SI-NEXT:    v_add_i32_e32 v6, vcc, s14, v6
+; SI-NEXT:    v_addc_u32_e32 v5, vcc, v7, v5, vcc
+; SI-NEXT:    v_not_b32_e32 v7, s0
+; SI-NEXT:    v_and_b32_e32 v6, v6, v7
+; SI-NEXT:    v_not_b32_e32 v7, s1
+; SI-NEXT:    v_cmp_eq_u32_e64 vcc, s16, -1
+; SI-NEXT:    v_and_b32_e32 v5, v5, v7
+; SI-NEXT:    v_cndmask_b32_e32 v7, 0, v4, vcc
+; SI-NEXT:    v_mov_b32_e32 v8, s15
+; SI-NEXT:    v_bfi_b32 v7, s20, v7, v8
+; SI-NEXT:    v_cmp_lt_i32_e64 vcc, s16, 0
+; SI-NEXT:    v_cndmask_b32_e32 v5, v5, v7, vcc
+; SI-NEXT:    v_cmp_gt_i32_e64 s[0:1], s16, 51
+; SI-NEXT:    v_cndmask_b32_e64 v7, v5, v8, s[0:1]
+; SI-NEXT:    v_cndmask_b32_e64 v5, v6, 0, vcc
+; SI-NEXT:    v_mov_b32_e32 v6, s14
+; SI-NEXT:    v_cndmask_b32_e64 v6, v5, v6, s[0:1]
+; SI-NEXT:    s_bfe_u32 s0, s13, 0xb0014
+; SI-NEXT:    s_add_i32 s10, s0, s22
+; SI-NEXT:    s_lshr_b64 s[0:1], s[2:3], s10
+; SI-NEXT:    s_and_b64 s[2:3], s[12:13], s[0:1]
+; SI-NEXT:    s_lshr_b64 s[8:9], s[18:19], s10
+; SI-NEXT:    v_cmp_ne_u64_e64 vcc, s[2:3], 0
+; SI-NEXT:    v_mov_b32_e32 v8, s8
+; SI-NEXT:    v_mov_b32_e32 v5, s9
+; SI-NEXT:    v_cndmask_b32_e32 v8, 0, v8, vcc
+; SI-NEXT:    v_cndmask_b32_e32 v5, 0, v5, vcc
+; SI-NEXT:    v_mov_b32_e32 v9, s13
+; SI-NEXT:    v_add_i32_e32 v8, vcc, s12, v8
+; SI-NEXT:    v_addc_u32_e32 v5, vcc, v9, v5, vcc
+; SI-NEXT:    v_not_b32_e32 v9, s0
+; SI-NEXT:    v_and_b32_e32 v8, v8, v9
+; SI-NEXT:    v_not_b32_e32 v9, s1
+; SI-NEXT:    v_cmp_eq_u32_e64 vcc, s10, -1
+; SI-NEXT:    v_and_b32_e32 v5, v5, v9
+; SI-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; SI-NEXT:    v_mov_b32_e32 v9, s13
+; SI-NEXT:    v_bfi_b32 v4, s20, v4, v9
+; SI-NEXT:    v_cmp_lt_i32_e64 vcc, s10, 0
+; SI-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
+; SI-NEXT:    v_cmp_gt_i32_e64 s[0:1], s10, 51
+; SI-NEXT:    v_cndmask_b32_e64 v5, v4, v9, s[0:1]
+; SI-NEXT:    v_cndmask_b32_e64 v4, v8, 0, vcc
+; SI-NEXT:    v_mov_b32_e32 v8, s12
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    v_cndmask_b32_e64 v4, v4, v8, s[0:1]
+; SI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[4:7], 0 offset:16
+; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; SI-NEXT:    s_endpgm
+;
+; CI-LABEL: round_v4f64:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; CI-NEXT:    s_load_dwordx8 s[8:15], s[0:1], 0x11
+; CI-NEXT:    s_brev_b32 s2, -2
+; CI-NEXT:    v_mov_b32_e32 v12, 0x3ff00000
+; CI-NEXT:    s_mov_b32 s7, 0xf000
+; CI-NEXT:    s_mov_b32 s6, -1
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    v_trunc_f64_e32 v[0:1], s[10:11]
+; CI-NEXT:    v_mov_b32_e32 v4, s11
+; CI-NEXT:    v_add_f64 v[2:3], s[10:11], -v[0:1]
+; CI-NEXT:    v_bfi_b32 v4, s2, v12, v4
+; CI-NEXT:    v_cmp_ge_f64_e64 vcc, |v[2:3]|, 0.5
+; CI-NEXT:    v_trunc_f64_e32 v[8:9], s[8:9]
+; CI-NEXT:    v_cndmask_b32_e32 v3, 0, v4, vcc
+; CI-NEXT:    v_mov_b32_e32 v2, 0
+; CI-NEXT:    v_add_f64 v[2:3], v[0:1], v[2:3]
+; CI-NEXT:    v_add_f64 v[0:1], s[8:9], -v[8:9]
+; CI-NEXT:    v_mov_b32_e32 v4, s9
+; CI-NEXT:    v_cmp_ge_f64_e64 vcc, |v[0:1]|, 0.5
+; CI-NEXT:    v_bfi_b32 v4, s2, v12, v4
+; CI-NEXT:    v_cndmask_b32_e32 v1, 0, v4, vcc
+; CI-NEXT:    v_trunc_f64_e32 v[4:5], s[14:15]
+; CI-NEXT:    v_mov_b32_e32 v10, s15
+; CI-NEXT:    v_add_f64 v[6:7], s[14:15], -v[4:5]
+; CI-NEXT:    v_bfi_b32 v10, s2, v12, v10
+; CI-NEXT:    v_cmp_ge_f64_e64 vcc, |v[6:7]|, 0.5
+; CI-NEXT:    v_mov_b32_e32 v6, 0
+; CI-NEXT:    v_cndmask_b32_e32 v7, 0, v10, vcc
+; CI-NEXT:    v_trunc_f64_e32 v[10:11], s[12:13]
+; CI-NEXT:    v_add_f64 v[6:7], v[4:5], v[6:7]
+; CI-NEXT:    v_add_f64 v[4:5], s[12:13], -v[10:11]
+; CI-NEXT:    v_mov_b32_e32 v13, s13
+; CI-NEXT:    v_cmp_ge_f64_e64 vcc, |v[4:5]|, 0.5
+; CI-NEXT:    v_bfi_b32 v12, s2, v12, v13
+; CI-NEXT:    v_mov_b32_e32 v0, 0
+; CI-NEXT:    v_cndmask_b32_e32 v5, 0, v12, vcc
+; CI-NEXT:    v_mov_b32_e32 v4, 0
+; CI-NEXT:    v_add_f64 v[4:5], v[10:11], v[4:5]
+; CI-NEXT:    v_add_f64 v[0:1], v[8:9], v[0:1]
+; CI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[4:7], 0 offset:16
+; CI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; CI-NEXT:    s_endpgm
   %result = call <4 x double> @llvm.round.v4f64(<4 x double> %in) #1
   store <4 x double> %result, <4 x double> addrspace(1)* %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}round_v8f64:
-; SI: s_endpgm
 define amdgpu_kernel void @round_v8f64(<8 x double> addrspace(1)* %out, <8 x double> %in) #0 {
+; SI-LABEL: round_v8f64:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SI-NEXT:    s_load_dwordx16 s[8:23], s[0:1], 0x19
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_movk_i32 s30, 0xfc01
+; SI-NEXT:    s_mov_b32 s3, 0xfffff
+; SI-NEXT:    s_mov_b32 s2, s6
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_bfe_u32 s0, s11, 0xb0014
+; SI-NEXT:    s_add_i32 s31, s0, s30
+; SI-NEXT:    s_lshr_b64 s[0:1], s[2:3], s31
+; SI-NEXT:    s_and_b64 s[26:27], s[10:11], s[0:1]
+; SI-NEXT:    s_mov_b32 s25, 0x80000
+; SI-NEXT:    s_mov_b32 s24, 0
+; SI-NEXT:    s_lshr_b64 s[28:29], s[24:25], s31
+; SI-NEXT:    v_cmp_ne_u64_e64 vcc, s[26:27], 0
+; SI-NEXT:    v_mov_b32_e32 v1, s28
+; SI-NEXT:    v_mov_b32_e32 v0, s29
+; SI-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; SI-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; SI-NEXT:    v_mov_b32_e32 v2, s11
+; SI-NEXT:    v_add_i32_e32 v1, vcc, s10, v1
+; SI-NEXT:    v_addc_u32_e32 v0, vcc, v2, v0, vcc
+; SI-NEXT:    v_not_b32_e32 v2, s0
+; SI-NEXT:    v_and_b32_e32 v1, v1, v2
+; SI-NEXT:    v_not_b32_e32 v2, s1
+; SI-NEXT:    v_mov_b32_e32 v12, 0x3ff00000
+; SI-NEXT:    v_cmp_eq_u32_e64 vcc, s31, -1
+; SI-NEXT:    v_and_b32_e32 v0, v0, v2
+; SI-NEXT:    v_cndmask_b32_e32 v2, 0, v12, vcc
+; SI-NEXT:    s_brev_b32 s28, -2
+; SI-NEXT:    v_mov_b32_e32 v3, s11
+; SI-NEXT:    v_bfi_b32 v2, s28, v2, v3
+; SI-NEXT:    v_cmp_lt_i32_e64 vcc, s31, 0
+; SI-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; SI-NEXT:    v_cmp_gt_i32_e64 s[0:1], s31, 51
+; SI-NEXT:    v_cndmask_b32_e64 v3, v0, v3, s[0:1]
+; SI-NEXT:    v_cndmask_b32_e64 v0, v1, 0, vcc
+; SI-NEXT:    v_mov_b32_e32 v1, s10
+; SI-NEXT:    v_cndmask_b32_e64 v2, v0, v1, s[0:1]
+; SI-NEXT:    s_bfe_u32 s0, s9, 0xb0014
+; SI-NEXT:    s_add_i32 s29, s0, s30
+; SI-NEXT:    s_lshr_b64 s[0:1], s[2:3], s29
+; SI-NEXT:    s_and_b64 s[10:11], s[8:9], s[0:1]
+; SI-NEXT:    s_lshr_b64 s[26:27], s[24:25], s29
+; SI-NEXT:    v_cmp_ne_u64_e64 vcc, s[10:11], 0
+; SI-NEXT:    v_mov_b32_e32 v1, s26
+; SI-NEXT:    v_mov_b32_e32 v0, s27
+; SI-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; SI-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; SI-NEXT:    v_mov_b32_e32 v4, s9
+; SI-NEXT:    v_add_i32_e32 v1, vcc, s8, v1
+; SI-NEXT:    v_addc_u32_e32 v0, vcc, v4, v0, vcc
+; SI-NEXT:    v_not_b32_e32 v4, s0
+; SI-NEXT:    v_and_b32_e32 v4, v1, v4
+; SI-NEXT:    v_not_b32_e32 v1, s1
+; SI-NEXT:    v_cmp_eq_u32_e64 vcc, s29, -1
+; SI-NEXT:    v_and_b32_e32 v0, v0, v1
+; SI-NEXT:    v_cndmask_b32_e32 v1, 0, v12, vcc
+; SI-NEXT:    v_mov_b32_e32 v5, s9
+; SI-NEXT:    v_bfi_b32 v1, s28, v1, v5
+; SI-NEXT:    v_cmp_lt_i32_e64 vcc, s29, 0
+; SI-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; SI-NEXT:    v_cmp_gt_i32_e64 s[0:1], s29, 51
+; SI-NEXT:    v_cndmask_b32_e64 v1, v0, v5, s[0:1]
+; SI-NEXT:    v_cndmask_b32_e64 v0, v4, 0, vcc
+; SI-NEXT:    v_mov_b32_e32 v4, s8
+; SI-NEXT:    v_cndmask_b32_e64 v0, v0, v4, s[0:1]
+; SI-NEXT:    s_bfe_u32 s0, s15, 0xb0014
+; SI-NEXT:    s_add_i32 s26, s0, s30
+; SI-NEXT:    s_lshr_b64 s[0:1], s[2:3], s26
+; SI-NEXT:    s_and_b64 s[8:9], s[14:15], s[0:1]
+; SI-NEXT:    s_lshr_b64 s[10:11], s[24:25], s26
+; SI-NEXT:    v_cmp_ne_u64_e64 vcc, s[8:9], 0
+; SI-NEXT:    v_mov_b32_e32 v5, s10
+; SI-NEXT:    v_mov_b32_e32 v4, s11
+; SI-NEXT:    v_cndmask_b32_e32 v5, 0, v5, vcc
+; SI-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; SI-NEXT:    v_mov_b32_e32 v6, s15
+; SI-NEXT:    v_add_i32_e32 v5, vcc, s14, v5
+; SI-NEXT:    v_addc_u32_e32 v4, vcc, v6, v4, vcc
+; SI-NEXT:    v_not_b32_e32 v6, s0
+; SI-NEXT:    v_and_b32_e32 v5, v5, v6
+; SI-NEXT:    v_not_b32_e32 v6, s1
+; SI-NEXT:    v_cmp_eq_u32_e64 vcc, s26, -1
+; SI-NEXT:    v_and_b32_e32 v4, v4, v6
+; SI-NEXT:    v_cndmask_b32_e32 v6, 0, v12, vcc
+; SI-NEXT:    v_mov_b32_e32 v7, s15
+; SI-NEXT:    v_bfi_b32 v6, s28, v6, v7
+; SI-NEXT:    v_cmp_lt_i32_e64 vcc, s26, 0
+; SI-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
+; SI-NEXT:    v_cmp_gt_i32_e64 s[0:1], s26, 51
+; SI-NEXT:    v_cndmask_b32_e64 v7, v4, v7, s[0:1]
+; SI-NEXT:    v_cndmask_b32_e64 v4, v5, 0, vcc
+; SI-NEXT:    v_mov_b32_e32 v5, s14
+; SI-NEXT:    v_cndmask_b32_e64 v6, v4, v5, s[0:1]
+; SI-NEXT:    s_bfe_u32 s0, s13, 0xb0014
+; SI-NEXT:    s_add_i32 s14, s0, s30
+; SI-NEXT:    s_lshr_b64 s[0:1], s[2:3], s14
+; SI-NEXT:    s_and_b64 s[8:9], s[12:13], s[0:1]
+; SI-NEXT:    s_lshr_b64 s[10:11], s[24:25], s14
+; SI-NEXT:    v_cmp_ne_u64_e64 vcc, s[8:9], 0
+; SI-NEXT:    v_mov_b32_e32 v5, s10
+; SI-NEXT:    v_mov_b32_e32 v4, s11
+; SI-NEXT:    v_cndmask_b32_e32 v5, 0, v5, vcc
+; SI-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; SI-NEXT:    v_mov_b32_e32 v8, s13
+; SI-NEXT:    v_add_i32_e32 v5, vcc, s12, v5
+; SI-NEXT:    v_addc_u32_e32 v4, vcc, v8, v4, vcc
+; SI-NEXT:    v_not_b32_e32 v8, s0
+; SI-NEXT:    v_and_b32_e32 v8, v5, v8
+; SI-NEXT:    v_not_b32_e32 v5, s1
+; SI-NEXT:    v_cmp_eq_u32_e64 vcc, s14, -1
+; SI-NEXT:    v_and_b32_e32 v4, v4, v5
+; SI-NEXT:    v_cndmask_b32_e32 v5, 0, v12, vcc
+; SI-NEXT:    v_mov_b32_e32 v9, s13
+; SI-NEXT:    v_bfi_b32 v5, s28, v5, v9
+; SI-NEXT:    v_cmp_lt_i32_e64 vcc, s14, 0
+; SI-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc
+; SI-NEXT:    v_cmp_gt_i32_e64 s[0:1], s14, 51
+; SI-NEXT:    v_cndmask_b32_e64 v5, v4, v9, s[0:1]
+; SI-NEXT:    v_cndmask_b32_e64 v4, v8, 0, vcc
+; SI-NEXT:    v_mov_b32_e32 v8, s12
+; SI-NEXT:    v_cndmask_b32_e64 v4, v4, v8, s[0:1]
+; SI-NEXT:    s_bfe_u32 s0, s19, 0xb0014
+; SI-NEXT:    s_add_i32 s12, s0, s30
+; SI-NEXT:    s_lshr_b64 s[0:1], s[2:3], s12
+; SI-NEXT:    s_and_b64 s[8:9], s[18:19], s[0:1]
+; SI-NEXT:    s_lshr_b64 s[10:11], s[24:25], s12
+; SI-NEXT:    v_cmp_ne_u64_e64 vcc, s[8:9], 0
+; SI-NEXT:    v_mov_b32_e32 v9, s10
+; SI-NEXT:    v_mov_b32_e32 v8, s11
+; SI-NEXT:    v_cndmask_b32_e32 v9, 0, v9, vcc
+; SI-NEXT:    v_cndmask_b32_e32 v8, 0, v8, vcc
+; SI-NEXT:    v_mov_b32_e32 v10, s19
+; SI-NEXT:    v_add_i32_e32 v9, vcc, s18, v9
+; SI-NEXT:    v_addc_u32_e32 v8, vcc, v10, v8, vcc
+; SI-NEXT:    v_not_b32_e32 v10, s0
+; SI-NEXT:    v_and_b32_e32 v9, v9, v10
+; SI-NEXT:    v_not_b32_e32 v10, s1
+; SI-NEXT:    v_cmp_eq_u32_e64 vcc, s12, -1
+; SI-NEXT:    v_and_b32_e32 v8, v8, v10
+; SI-NEXT:    v_cndmask_b32_e32 v10, 0, v12, vcc
+; SI-NEXT:    v_mov_b32_e32 v11, s19
+; SI-NEXT:    v_bfi_b32 v10, s28, v10, v11
+; SI-NEXT:    v_cmp_lt_i32_e64 vcc, s12, 0
+; SI-NEXT:    v_cndmask_b32_e32 v8, v8, v10, vcc
+; SI-NEXT:    v_cmp_gt_i32_e64 s[0:1], s12, 51
+; SI-NEXT:    v_cndmask_b32_e64 v11, v8, v11, s[0:1]
+; SI-NEXT:    v_cndmask_b32_e64 v8, v9, 0, vcc
+; SI-NEXT:    v_mov_b32_e32 v9, s18
+; SI-NEXT:    v_cndmask_b32_e64 v10, v8, v9, s[0:1]
+; SI-NEXT:    s_bfe_u32 s0, s17, 0xb0014
+; SI-NEXT:    s_add_i32 s12, s0, s30
+; SI-NEXT:    s_lshr_b64 s[0:1], s[2:3], s12
+; SI-NEXT:    s_and_b64 s[8:9], s[16:17], s[0:1]
+; SI-NEXT:    s_lshr_b64 s[10:11], s[24:25], s12
+; SI-NEXT:    v_cmp_ne_u64_e64 vcc, s[8:9], 0
+; SI-NEXT:    v_mov_b32_e32 v9, s10
+; SI-NEXT:    v_mov_b32_e32 v8, s11
+; SI-NEXT:    v_cndmask_b32_e32 v9, 0, v9, vcc
+; SI-NEXT:    v_cndmask_b32_e32 v8, 0, v8, vcc
+; SI-NEXT:    v_mov_b32_e32 v13, s17
+; SI-NEXT:    v_add_i32_e32 v9, vcc, s16, v9
+; SI-NEXT:    v_addc_u32_e32 v8, vcc, v13, v8, vcc
+; SI-NEXT:    v_not_b32_e32 v13, s0
+; SI-NEXT:    v_and_b32_e32 v13, v9, v13
+; SI-NEXT:    v_not_b32_e32 v9, s1
+; SI-NEXT:    v_cmp_eq_u32_e64 vcc, s12, -1
+; SI-NEXT:    v_and_b32_e32 v8, v8, v9
+; SI-NEXT:    v_cndmask_b32_e32 v9, 0, v12, vcc
+; SI-NEXT:    v_mov_b32_e32 v14, s17
+; SI-NEXT:    v_bfi_b32 v9, s28, v9, v14
+; SI-NEXT:    v_cmp_lt_i32_e64 vcc, s12, 0
+; SI-NEXT:    v_cndmask_b32_e32 v8, v8, v9, vcc
+; SI-NEXT:    v_cmp_gt_i32_e64 s[0:1], s12, 51
+; SI-NEXT:    v_cndmask_b32_e64 v9, v8, v14, s[0:1]
+; SI-NEXT:    v_cndmask_b32_e64 v8, v13, 0, vcc
+; SI-NEXT:    v_mov_b32_e32 v13, s16
+; SI-NEXT:    v_cndmask_b32_e64 v8, v8, v13, s[0:1]
+; SI-NEXT:    s_bfe_u32 s0, s23, 0xb0014
+; SI-NEXT:    s_add_i32 s12, s0, s30
+; SI-NEXT:    s_lshr_b64 s[0:1], s[2:3], s12
+; SI-NEXT:    s_and_b64 s[8:9], s[22:23], s[0:1]
+; SI-NEXT:    s_lshr_b64 s[10:11], s[24:25], s12
+; SI-NEXT:    v_cmp_ne_u64_e64 vcc, s[8:9], 0
+; SI-NEXT:    v_mov_b32_e32 v14, s10
+; SI-NEXT:    v_mov_b32_e32 v13, s11
+; SI-NEXT:    v_cndmask_b32_e32 v14, 0, v14, vcc
+; SI-NEXT:    v_cndmask_b32_e32 v13, 0, v13, vcc
+; SI-NEXT:    v_mov_b32_e32 v15, s23
+; SI-NEXT:    v_add_i32_e32 v14, vcc, s22, v14
+; SI-NEXT:    v_addc_u32_e32 v13, vcc, v15, v13, vcc
+; SI-NEXT:    v_not_b32_e32 v15, s0
+; SI-NEXT:    v_and_b32_e32 v14, v14, v15
+; SI-NEXT:    v_not_b32_e32 v15, s1
+; SI-NEXT:    v_cmp_eq_u32_e64 vcc, s12, -1
+; SI-NEXT:    v_and_b32_e32 v13, v13, v15
+; SI-NEXT:    v_cndmask_b32_e32 v15, 0, v12, vcc
+; SI-NEXT:    v_mov_b32_e32 v16, s23
+; SI-NEXT:    v_bfi_b32 v15, s28, v15, v16
+; SI-NEXT:    v_cmp_lt_i32_e64 vcc, s12, 0
+; SI-NEXT:    v_cndmask_b32_e32 v13, v13, v15, vcc
+; SI-NEXT:    v_cmp_gt_i32_e64 s[0:1], s12, 51
+; SI-NEXT:    v_cndmask_b32_e64 v15, v13, v16, s[0:1]
+; SI-NEXT:    v_cndmask_b32_e64 v13, v14, 0, vcc
+; SI-NEXT:    v_mov_b32_e32 v14, s22
+; SI-NEXT:    v_cndmask_b32_e64 v14, v13, v14, s[0:1]
+; SI-NEXT:    s_bfe_u32 s0, s21, 0xb0014
+; SI-NEXT:    s_add_i32 s10, s0, s30
+; SI-NEXT:    s_lshr_b64 s[0:1], s[2:3], s10
+; SI-NEXT:    s_and_b64 s[2:3], s[20:21], s[0:1]
+; SI-NEXT:    s_lshr_b64 s[8:9], s[24:25], s10
+; SI-NEXT:    v_cmp_ne_u64_e64 vcc, s[2:3], 0
+; SI-NEXT:    v_mov_b32_e32 v16, s8
+; SI-NEXT:    v_mov_b32_e32 v13, s9
+; SI-NEXT:    v_cndmask_b32_e32 v16, 0, v16, vcc
+; SI-NEXT:    v_cndmask_b32_e32 v13, 0, v13, vcc
+; SI-NEXT:    v_mov_b32_e32 v17, s21
+; SI-NEXT:    v_add_i32_e32 v16, vcc, s20, v16
+; SI-NEXT:    v_addc_u32_e32 v13, vcc, v17, v13, vcc
+; SI-NEXT:    v_not_b32_e32 v17, s0
+; SI-NEXT:    v_and_b32_e32 v16, v16, v17
+; SI-NEXT:    v_not_b32_e32 v17, s1
+; SI-NEXT:    v_cmp_eq_u32_e64 vcc, s10, -1
+; SI-NEXT:    v_and_b32_e32 v13, v13, v17
+; SI-NEXT:    v_cndmask_b32_e32 v12, 0, v12, vcc
+; SI-NEXT:    v_mov_b32_e32 v17, s21
+; SI-NEXT:    v_bfi_b32 v12, s28, v12, v17
+; SI-NEXT:    v_cmp_lt_i32_e64 vcc, s10, 0
+; SI-NEXT:    v_cndmask_b32_e32 v12, v13, v12, vcc
+; SI-NEXT:    v_cmp_gt_i32_e64 s[0:1], s10, 51
+; SI-NEXT:    v_cndmask_b32_e64 v13, v12, v17, s[0:1]
+; SI-NEXT:    v_cndmask_b32_e64 v12, v16, 0, vcc
+; SI-NEXT:    v_mov_b32_e32 v16, s20
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    v_cndmask_b32_e64 v12, v12, v16, s[0:1]
+; SI-NEXT:    buffer_store_dwordx4 v[12:15], off, s[4:7], 0 offset:48
+; SI-NEXT:    buffer_store_dwordx4 v[8:11], off, s[4:7], 0 offset:32
+; SI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[4:7], 0 offset:16
+; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; SI-NEXT:    s_endpgm
+;
+; CI-LABEL: round_v8f64:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; CI-NEXT:    s_load_dwordx16 s[8:23], s[0:1], 0x19
+; CI-NEXT:    s_brev_b32 s2, -2
+; CI-NEXT:    v_mov_b32_e32 v16, 0x3ff00000
+; CI-NEXT:    s_mov_b32 s7, 0xf000
+; CI-NEXT:    s_mov_b32 s6, -1
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    v_trunc_f64_e32 v[0:1], s[10:11]
+; CI-NEXT:    v_mov_b32_e32 v4, s11
+; CI-NEXT:    v_add_f64 v[2:3], s[10:11], -v[0:1]
+; CI-NEXT:    v_bfi_b32 v4, s2, v16, v4
+; CI-NEXT:    v_cmp_ge_f64_e64 vcc, |v[2:3]|, 0.5
+; CI-NEXT:    v_mov_b32_e32 v2, 0
+; CI-NEXT:    v_cndmask_b32_e32 v3, 0, v4, vcc
+; CI-NEXT:    v_trunc_f64_e32 v[4:5], s[8:9]
+; CI-NEXT:    v_add_f64 v[2:3], v[0:1], v[2:3]
+; CI-NEXT:    v_add_f64 v[0:1], s[8:9], -v[4:5]
+; CI-NEXT:    v_mov_b32_e32 v6, s9
+; CI-NEXT:    v_cmp_ge_f64_e64 vcc, |v[0:1]|, 0.5
+; CI-NEXT:    v_bfi_b32 v6, s2, v16, v6
+; CI-NEXT:    v_cndmask_b32_e32 v1, 0, v6, vcc
+; CI-NEXT:    v_trunc_f64_e32 v[6:7], s[14:15]
+; CI-NEXT:    v_mov_b32_e32 v0, 0
+; CI-NEXT:    v_add_f64 v[0:1], v[4:5], v[0:1]
+; CI-NEXT:    v_add_f64 v[4:5], s[14:15], -v[6:7]
+; CI-NEXT:    v_mov_b32_e32 v8, s15
+; CI-NEXT:    v_cmp_ge_f64_e64 vcc, |v[4:5]|, 0.5
+; CI-NEXT:    v_bfi_b32 v8, s2, v16, v8
+; CI-NEXT:    v_cndmask_b32_e32 v5, 0, v8, vcc
+; CI-NEXT:    v_trunc_f64_e32 v[8:9], s[12:13]
+; CI-NEXT:    v_mov_b32_e32 v4, 0
+; CI-NEXT:    v_add_f64 v[6:7], v[6:7], v[4:5]
+; CI-NEXT:    v_add_f64 v[4:5], s[12:13], -v[8:9]
+; CI-NEXT:    v_mov_b32_e32 v10, s13
+; CI-NEXT:    v_cmp_ge_f64_e64 vcc, |v[4:5]|, 0.5
+; CI-NEXT:    v_bfi_b32 v10, s2, v16, v10
+; CI-NEXT:    v_cndmask_b32_e32 v5, 0, v10, vcc
+; CI-NEXT:    v_mov_b32_e32 v4, 0
+; CI-NEXT:    v_add_f64 v[4:5], v[8:9], v[4:5]
+; CI-NEXT:    v_mov_b32_e32 v8, s19
+; CI-NEXT:    v_bfi_b32 v18, s2, v16, v8
+; CI-NEXT:    v_trunc_f64_e32 v[8:9], s[20:21]
+; CI-NEXT:    v_trunc_f64_e32 v[10:11], s[22:23]
+; CI-NEXT:    v_add_f64 v[14:15], s[20:21], -v[8:9]
+; CI-NEXT:    v_mov_b32_e32 v19, s23
+; CI-NEXT:    v_cmp_ge_f64_e64 vcc, |v[14:15]|, 0.5
+; CI-NEXT:    v_add_f64 v[14:15], s[22:23], -v[10:11]
+; CI-NEXT:    v_mov_b32_e32 v17, s21
+; CI-NEXT:    v_cmp_ge_f64_e64 s[0:1], |v[14:15]|, 0.5
+; CI-NEXT:    v_bfi_b32 v19, s2, v16, v19
+; CI-NEXT:    v_trunc_f64_e32 v[12:13], s[16:17]
+; CI-NEXT:    v_bfi_b32 v17, s2, v16, v17
+; CI-NEXT:    v_cndmask_b32_e64 v15, 0, v19, s[0:1]
+; CI-NEXT:    v_mov_b32_e32 v14, 0
+; CI-NEXT:    v_add_f64 v[10:11], v[10:11], v[14:15]
+; CI-NEXT:    v_cndmask_b32_e32 v15, 0, v17, vcc
+; CI-NEXT:    v_mov_b32_e32 v14, 0
+; CI-NEXT:    v_mov_b32_e32 v17, s17
+; CI-NEXT:    v_add_f64 v[8:9], v[8:9], v[14:15]
+; CI-NEXT:    v_add_f64 v[14:15], s[16:17], -v[12:13]
+; CI-NEXT:    v_bfi_b32 v19, s2, v16, v17
+; CI-NEXT:    v_trunc_f64_e32 v[16:17], s[18:19]
+; CI-NEXT:    v_cmp_ge_f64_e64 vcc, |v[14:15]|, 0.5
+; CI-NEXT:    v_add_f64 v[14:15], s[18:19], -v[16:17]
+; CI-NEXT:    v_cmp_ge_f64_e64 s[0:1], |v[14:15]|, 0.5
+; CI-NEXT:    v_mov_b32_e32 v14, 0
+; CI-NEXT:    v_cndmask_b32_e64 v15, 0, v18, s[0:1]
+; CI-NEXT:    v_add_f64 v[14:15], v[16:17], v[14:15]
+; CI-NEXT:    v_cndmask_b32_e32 v17, 0, v19, vcc
+; CI-NEXT:    v_mov_b32_e32 v16, 0
+; CI-NEXT:    v_add_f64 v[12:13], v[12:13], v[16:17]
+; CI-NEXT:    buffer_store_dwordx4 v[8:11], off, s[4:7], 0 offset:48
+; CI-NEXT:    buffer_store_dwordx4 v[12:15], off, s[4:7], 0 offset:32
+; CI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[4:7], 0 offset:16
+; CI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; CI-NEXT:    s_endpgm
   %result = call <8 x double> @llvm.round.v8f64(<8 x double> %in) #1
   store <8 x double> %result, <8 x double> addrspace(1)* %out
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/load-lo16.ll b/llvm/test/CodeGen/AMDGPU/load-lo16.ll
index 3ab3d92f5..08ab3faf4 100644
--- a/llvm/test/CodeGen/AMDGPU/load-lo16.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-lo16.ll
@@ -1,28 +1,64 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -march=amdgcn -mcpu=gfx900 -amdgpu-sroa=0 -mattr=-promote-alloca -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX900 %s
 ; RUN: llc -march=amdgcn -mcpu=gfx906 -amdgpu-sroa=0 -mattr=-promote-alloca,+sram-ecc -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX906,NO-D16-HI %s
 ; RUN: llc -march=amdgcn -mcpu=fiji -amdgpu-sroa=0 -mattr=-promote-alloca -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX803,NO-D16-HI %s
 
-; GCN-LABEL: {{^}}load_local_lo_v2i16_undeflo:
-; GCN: s_waitcnt
-; GFX900-NEXT: ds_read_u16_d16 v0, v0
-; GFX900-NEXT: s_waitcnt
-; GFX900-NEXT: s_setpc_b64
-
-; NO-D16-HI: ds_read_u16
 define <2 x i16> @load_local_lo_v2i16_undeflo(i16 addrspace(3)* %in) #0 {
+; GFX900-LABEL: load_local_lo_v2i16_undeflo:
+; GFX900:       ; %bb.0: ; %entry
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    ds_read_u16_d16 v0, v0
+; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX906-LABEL: load_local_lo_v2i16_undeflo:
+; GFX906:       ; %bb.0: ; %entry
+; GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-NEXT:    ds_read_u16 v0, v0
+; GFX906-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX906-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX803-LABEL: load_local_lo_v2i16_undeflo:
+; GFX803:       ; %bb.0: ; %entry
+; GFX803-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    s_mov_b32 m0, -1
+; GFX803-NEXT:    ds_read_u16 v0, v0
+; GFX803-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX803-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %load = load i16, i16 addrspace(3)* %in
   %build = insertelement <2 x i16> undef, i16 %load, i32 0
   ret <2 x i16> %build
 }
 
-; GCN-LABEL: {{^}}load_local_lo_v2i16_reglo:
-; GCN: s_waitcnt
-; GCN: ds_read_u16 v0, v0
-; GFX9: v_and_b32_e32 v0, 0xffff, v0
-; GFX9: v_lshl_or_b32 v0, v1, 16, v0
-; GFX9: s_setpc_b64
 define <2 x i16> @load_local_lo_v2i16_reglo(i16 addrspace(3)* %in, i16 %reg) #0 {
+; GFX900-LABEL: load_local_lo_v2i16_reglo:
+; GFX900:       ; %bb.0: ; %entry
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    ds_read_u16 v0, v0
+; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX900-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX900-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX906-LABEL: load_local_lo_v2i16_reglo:
+; GFX906:       ; %bb.0: ; %entry
+; GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-NEXT:    ds_read_u16 v0, v0
+; GFX906-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX906-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX906-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; GFX906-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX803-LABEL: load_local_lo_v2i16_reglo:
+; GFX803:       ; %bb.0: ; %entry
+; GFX803-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    s_mov_b32 m0, -1
+; GFX803-NEXT:    ds_read_u16 v0, v0
+; GFX803-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX803-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX803-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX803-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %load = load i16, i16 addrspace(3)* %in
   %build0 = insertelement <2 x i16> undef, i16 %reg, i32 1
@@ -31,14 +67,40 @@ entry:
 }
 
 ; Show that we get reasonable regalloc without physreg constraints.
-; GCN-LABEL: {{^}}load_local_lo_v2i16_reglo_vreg:
-; GCN: s_waitcnt
-; GCN: ds_read_u16 v0, v0
-; GCN: s_waitcnt
-; GFX9: v_and_b32_e32 v0, 0xffff, v0
-; GFX9: v_lshl_or_b32 v0, v1, 16, v0
-; GFX9: global_store_dword v
 define void @load_local_lo_v2i16_reglo_vreg(i16 addrspace(3)* %in, i16 %reg) #0 {
+; GFX900-LABEL: load_local_lo_v2i16_reglo_vreg:
+; GFX900:       ; %bb.0: ; %entry
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    ds_read_u16 v0, v0
+; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX900-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX900-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; GFX900-NEXT:    global_store_dword v[0:1], v0, off
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX906-LABEL: load_local_lo_v2i16_reglo_vreg:
+; GFX906:       ; %bb.0: ; %entry
+; GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-NEXT:    ds_read_u16 v0, v0
+; GFX906-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX906-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX906-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; GFX906-NEXT:    global_store_dword v[0:1], v0, off
+; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX803-LABEL: load_local_lo_v2i16_reglo_vreg:
+; GFX803:       ; %bb.0: ; %entry
+; GFX803-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    s_mov_b32 m0, -1
+; GFX803-NEXT:    ds_read_u16 v0, v0
+; GFX803-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX803-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX803-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX803-NEXT:    flat_store_dword v[0:1], v0
+; GFX803-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %load = load i16, i16 addrspace(3)* %in
   %build0 = insertelement <2 x i16> undef, i16 %reg, i32 1
@@ -47,48 +109,104 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}load_local_lo_v2i16_zerolo:
-; GCN: s_waitcnt
-; GFX900-NEXT: v_mov_b32_e32 v1, 0
-; GFX900-NEXT: ds_read_u16_d16 v1, v0
-; GFX900-NEXT: s_waitcnt
-; GFX900-NEXT: v_mov_b32_e32 v0, v1
-; GFX900-NEXT: s_setpc_b64
-
-; NO-D16-HI: ds_read_u16 v
 define <2 x i16> @load_local_lo_v2i16_zerolo(i16 addrspace(3)* %in) #0 {
+; GFX900-LABEL: load_local_lo_v2i16_zerolo:
+; GFX900:       ; %bb.0: ; %entry
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    v_mov_b32_e32 v1, 0
+; GFX900-NEXT:    ds_read_u16_d16 v1, v0
+; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX900-NEXT:    v_mov_b32_e32 v0, v1
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX906-LABEL: load_local_lo_v2i16_zerolo:
+; GFX906:       ; %bb.0: ; %entry
+; GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-NEXT:    ds_read_u16 v0, v0
+; GFX906-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX906-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX906-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX803-LABEL: load_local_lo_v2i16_zerolo:
+; GFX803:       ; %bb.0: ; %entry
+; GFX803-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    s_mov_b32 m0, -1
+; GFX803-NEXT:    ds_read_u16 v0, v0
+; GFX803-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX803-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %load = load i16, i16 addrspace(3)* %in
   %build = insertelement <2 x i16> zeroinitializer, i16 %load, i32 0
   ret <2 x i16> %build
 }
 
-; GCN-LABEL: {{^}}load_local_lo_v2f16_fpimm:
-; GCN: s_waitcnt
-; GFX900-NEXT: v_mov_b32_e32 v1, 2.0
-; GFX900-NEXT: ds_read_u16_d16 v1, v0
-; GFX900-NEXT: s_waitcnt
-; GFX900-NEXT: v_mov_b32_e32 v0, v1
-; GFX900-NEXT: s_setpc_b64
-
-; NO-D16-HI: ds_read_u16 v
 define <2 x half> @load_local_lo_v2f16_fpimm(half addrspace(3)* %in) #0 {
+; GFX900-LABEL: load_local_lo_v2f16_fpimm:
+; GFX900:       ; %bb.0: ; %entry
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    v_mov_b32_e32 v1, 2.0
+; GFX900-NEXT:    ds_read_u16_d16 v1, v0
+; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX900-NEXT:    v_mov_b32_e32 v0, v1
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX906-LABEL: load_local_lo_v2f16_fpimm:
+; GFX906:       ; %bb.0: ; %entry
+; GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-NEXT:    ds_read_u16 v0, v0
+; GFX906-NEXT:    s_movk_i32 s4, 0x4000
+; GFX906-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX906-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX906-NEXT:    v_lshl_or_b32 v0, s4, 16, v0
+; GFX906-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX803-LABEL: load_local_lo_v2f16_fpimm:
+; GFX803:       ; %bb.0: ; %entry
+; GFX803-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    s_mov_b32 m0, -1
+; GFX803-NEXT:    ds_read_u16 v0, v0
+; GFX803-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX803-NEXT:    v_or_b32_e32 v0, 2.0, v0
+; GFX803-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %load = load half, half addrspace(3)* %in
   %build = insertelement <2 x half> <half 0.0, half 2.0>, half %load, i32 0
   ret <2 x half> %build
 }
 
-; GCN-LABEL: {{^}}load_local_lo_v2f16_reghi_vreg:
-; GCN: s_waitcnt
-; GFX900-NEXT: ds_read_u16_d16 v1, v0
-; GFX900-NEXT: s_waitcnt
-; GFX900-NEXT: global_store_dword v[0:1], v1, off{{$}}
-; GFX900-NEXT: s_waitcnt
-; GFX900-NEXT: s_setpc_b64
-
-; NO-D16-HI: ds_read_u16 v
 define void @load_local_lo_v2f16_reghi_vreg(half addrspace(3)* %in, i32 %reg) #0 {
+; GFX900-LABEL: load_local_lo_v2f16_reghi_vreg:
+; GFX900:       ; %bb.0: ; %entry
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    ds_read_u16_d16 v1, v0
+; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX900-NEXT:    global_store_dword v[0:1], v1, off
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX906-LABEL: load_local_lo_v2f16_reghi_vreg:
+; GFX906:       ; %bb.0: ; %entry
+; GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-NEXT:    ds_read_u16 v0, v0
+; GFX906-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX906-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX906-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX906-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; GFX906-NEXT:    global_store_dword v[0:1], v0, off
+; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX803-LABEL: load_local_lo_v2f16_reghi_vreg:
+; GFX803:       ; %bb.0: ; %entry
+; GFX803-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    s_mov_b32 m0, -1
+; GFX803-NEXT:    ds_read_u16 v0, v0
+; GFX803-NEXT:    v_and_b32_e32 v1, 0xffff0000, v1
+; GFX803-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX803-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX803-NEXT:    flat_store_dword v[0:1], v0
+; GFX803-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %reg.bc = bitcast i32 %reg to <2 x half>
   %load = load half, half addrspace(3)* %in
@@ -96,15 +214,41 @@ entry:
   store <2 x half> %build1, <2 x half> addrspace(1)* undef
   ret void
 }
-; GCN-LABEL: {{^}}load_local_lo_v2f16_reglo_vreg:
 
-; GFX900: ds_read_u16 v
-; GFX900: v_and_b32_e32 v{{[0-9]+}}, 0xffff, v{{[0-9]+}}
-; GFX900: v_lshl_or_b32 v{{[0-9]+}}, v{{[0-9]+}}, 16, v{{[0-9]+}}
-; GFX900: global_store_dword
-
-; NO-D16-HI: ds_read_u16 v
 define void @load_local_lo_v2f16_reglo_vreg(half addrspace(3)* %in, half %reg) #0 {
+; GFX900-LABEL: load_local_lo_v2f16_reglo_vreg:
+; GFX900:       ; %bb.0: ; %entry
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    ds_read_u16 v0, v0
+; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX900-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX900-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; GFX900-NEXT:    global_store_dword v[0:1], v0, off
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX906-LABEL: load_local_lo_v2f16_reglo_vreg:
+; GFX906:       ; %bb.0: ; %entry
+; GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-NEXT:    ds_read_u16 v0, v0
+; GFX906-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX906-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX906-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; GFX906-NEXT:    global_store_dword v[0:1], v0, off
+; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX803-LABEL: load_local_lo_v2f16_reglo_vreg:
+; GFX803:       ; %bb.0: ; %entry
+; GFX803-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    s_mov_b32 m0, -1
+; GFX803-NEXT:    ds_read_u16 v0, v0
+; GFX803-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX803-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX803-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX803-NEXT:    flat_store_dword v[0:1], v0
+; GFX803-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %load = load half, half addrspace(3)* %in
   %build0 = insertelement <2 x half> undef, half %reg, i32 1
@@ -113,16 +257,39 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}load_local_lo_v2i16_reghi_vreg_zexti8:
-; GCN: s_waitcnt
-; GFX900-NEXT: ds_read_u8_d16 v1, v0
-; GFX900-NEXT: s_waitcnt
-; GFX900-NEXT: global_store_dword v[0:1], v1, off{{$}}
-; GFX900-NEXT: s_waitcnt
-; GFX900-NEXT: s_setpc_b64
-
-; NO-D16-HI: ds_read_u8 v
 define void @load_local_lo_v2i16_reghi_vreg_zexti8(i8 addrspace(3)* %in, i32 %reg) #0 {
+; GFX900-LABEL: load_local_lo_v2i16_reghi_vreg_zexti8:
+; GFX900:       ; %bb.0: ; %entry
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    ds_read_u8_d16 v1, v0
+; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX900-NEXT:    global_store_dword v[0:1], v1, off
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX906-LABEL: load_local_lo_v2i16_reghi_vreg_zexti8:
+; GFX906:       ; %bb.0: ; %entry
+; GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-NEXT:    ds_read_u8 v0, v0
+; GFX906-NEXT:    v_mov_b32_e32 v2, 0xffff
+; GFX906-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX906-NEXT:    v_bfi_b32 v0, v2, v0, v1
+; GFX906-NEXT:    global_store_dword v[0:1], v0, off
+; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX803-LABEL: load_local_lo_v2i16_reghi_vreg_zexti8:
+; GFX803:       ; %bb.0: ; %entry
+; GFX803-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    s_mov_b32 m0, -1
+; GFX803-NEXT:    ds_read_u8 v0, v0
+; GFX803-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX803-NEXT:    s_mov_b32 s4, 0x5040c00
+; GFX803-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX803-NEXT:    v_perm_b32 v0, v1, v0, s4
+; GFX803-NEXT:    flat_store_dword v[0:1], v0
+; GFX803-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %reg.bc = bitcast i32 %reg to <2 x i16>
   %load = load i8, i8 addrspace(3)* %in
@@ -132,15 +299,40 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}load_local_lo_v2i16_reglo_vreg_zexti8:
-; GCN: s_waitcnt
-; GFX900: ds_read_u8 v
-; GFX900: global_store_dword
-; GFX900-NEXT: s_waitcnt
-; GFX900-NEXT: s_setpc_b64
-
-; NO-D16-HI: ds_read_u8 v
 define void @load_local_lo_v2i16_reglo_vreg_zexti8(i8 addrspace(3)* %in, i16 %reg) #0 {
+; GFX900-LABEL: load_local_lo_v2i16_reglo_vreg_zexti8:
+; GFX900:       ; %bb.0: ; %entry
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    ds_read_u8 v0, v0
+; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX900-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX900-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; GFX900-NEXT:    global_store_dword v[0:1], v0, off
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX906-LABEL: load_local_lo_v2i16_reglo_vreg_zexti8:
+; GFX906:       ; %bb.0: ; %entry
+; GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-NEXT:    ds_read_u8 v0, v0
+; GFX906-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX906-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX906-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; GFX906-NEXT:    global_store_dword v[0:1], v0, off
+; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX803-LABEL: load_local_lo_v2i16_reglo_vreg_zexti8:
+; GFX803:       ; %bb.0: ; %entry
+; GFX803-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    s_mov_b32 m0, -1
+; GFX803-NEXT:    ds_read_u8 v0, v0
+; GFX803-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX803-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX803-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX803-NEXT:    flat_store_dword v[0:1], v0
+; GFX803-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %load = load i8, i8 addrspace(3)* %in
   %ext = zext i8 %load to i16
@@ -150,16 +342,38 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}load_local_lo_v2i16_reghi_vreg_sexti8:
-; GCN: s_waitcnt
-; GFX900-NEXT: ds_read_i8_d16 v1, v0
-; GFX900-NEXT: s_waitcnt
-; GFX900-NEXT: global_store_dword v[0:1], v1, off{{$}}
-; GFX900-NEXT: s_waitcnt
-; GFX900-NEXT: s_setpc_b64
-
-; NO-D16-HI: ds_read_i8 v
 define void @load_local_lo_v2i16_reghi_vreg_sexti8(i8 addrspace(3)* %in, i32 %reg) #0 {
+; GFX900-LABEL: load_local_lo_v2i16_reghi_vreg_sexti8:
+; GFX900:       ; %bb.0: ; %entry
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    ds_read_i8_d16 v1, v0
+; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX900-NEXT:    global_store_dword v[0:1], v1, off
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX906-LABEL: load_local_lo_v2i16_reghi_vreg_sexti8:
+; GFX906:       ; %bb.0: ; %entry
+; GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-NEXT:    ds_read_i8 v0, v0
+; GFX906-NEXT:    v_mov_b32_e32 v2, 0xffff
+; GFX906-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX906-NEXT:    v_bfi_b32 v0, v2, v0, v1
+; GFX906-NEXT:    global_store_dword v[0:1], v0, off
+; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX803-LABEL: load_local_lo_v2i16_reghi_vreg_sexti8:
+; GFX803:       ; %bb.0: ; %entry
+; GFX803-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    s_mov_b32 m0, -1
+; GFX803-NEXT:    ds_read_i8 v0, v0
+; GFX803-NEXT:    v_and_b32_e32 v1, 0xffff0000, v1
+; GFX803-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX803-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX803-NEXT:    flat_store_dword v[0:1], v0
+; GFX803-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %reg.bc = bitcast i32 %reg to <2 x i16>
   %load = load i8, i8 addrspace(3)* %in
@@ -169,14 +383,40 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}load_local_lo_v2i16_reglo_vreg_sexti8:
-; GCN: s_waitcnt
-; GFX900: ds_read_i8 v
-; GFX900: v_and_b32_e32 v{{[0-9]+}}, 0xffff, v{{[0-9]+}}
-; GFX900: v_lshl_or_b32 v{{[0-9]+}}, v{{[0-9]+}}, 16, v{{[0-9]+}}
-
-; NO-D16-HI: ds_read_i8 v
 define void @load_local_lo_v2i16_reglo_vreg_sexti8(i8 addrspace(3)* %in, i16 %reg) #0 {
+; GFX900-LABEL: load_local_lo_v2i16_reglo_vreg_sexti8:
+; GFX900:       ; %bb.0: ; %entry
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    ds_read_i8 v0, v0
+; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX900-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX900-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; GFX900-NEXT:    global_store_dword v[0:1], v0, off
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX906-LABEL: load_local_lo_v2i16_reglo_vreg_sexti8:
+; GFX906:       ; %bb.0: ; %entry
+; GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-NEXT:    ds_read_i8 v0, v0
+; GFX906-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX906-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX906-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; GFX906-NEXT:    global_store_dword v[0:1], v0, off
+; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX803-LABEL: load_local_lo_v2i16_reglo_vreg_sexti8:
+; GFX803:       ; %bb.0: ; %entry
+; GFX803-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    s_mov_b32 m0, -1
+; GFX803-NEXT:    ds_read_i8 v0, v0
+; GFX803-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX803-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX803-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX803-NEXT:    flat_store_dword v[0:1], v0
+; GFX803-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %load = load i8, i8 addrspace(3)* %in
   %ext = sext i8 %load to i16
@@ -186,15 +426,40 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}load_local_lo_v2f16_reglo_vreg_zexti8:
-; GCN: s_waitcnt
-; GFX900: ds_read_u8 v
-; GFX900: global_store_dword
-; GFX900-NEXT: s_waitcnt
-; GFX900-NEXT: s_setpc_b64
-
-; NO-D16-HI: ds_read_u8 v
 define void @load_local_lo_v2f16_reglo_vreg_zexti8(i8 addrspace(3)* %in, half %reg) #0 {
+; GFX900-LABEL: load_local_lo_v2f16_reglo_vreg_zexti8:
+; GFX900:       ; %bb.0: ; %entry
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    ds_read_u8 v0, v0
+; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX900-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX900-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; GFX900-NEXT:    global_store_dword v[0:1], v0, off
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX906-LABEL: load_local_lo_v2f16_reglo_vreg_zexti8:
+; GFX906:       ; %bb.0: ; %entry
+; GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-NEXT:    ds_read_u8 v0, v0
+; GFX906-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX906-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX906-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; GFX906-NEXT:    global_store_dword v[0:1], v0, off
+; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX803-LABEL: load_local_lo_v2f16_reglo_vreg_zexti8:
+; GFX803:       ; %bb.0: ; %entry
+; GFX803-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    s_mov_b32 m0, -1
+; GFX803-NEXT:    ds_read_u8 v0, v0
+; GFX803-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX803-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX803-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX803-NEXT:    flat_store_dword v[0:1], v0
+; GFX803-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %load = load i8, i8 addrspace(3)* %in
   %ext = zext i8 %load to i16
@@ -205,14 +470,40 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}load_local_lo_v2f16_reglo_vreg_sexti8:
-; GCN: s_waitcnt
-; GFX900: ds_read_i8 v
-; GFX900: v_and_b32_e32 v{{[0-9]+}}, 0xffff, v{{[0-9]+}}
-; GFX900: v_lshl_or_b32 v{{[0-9]+}}, v{{[0-9]+}}, 16, v{{[0-9]+}}
-
-; NO-D16-HI: ds_read_i8 v
 define void @load_local_lo_v2f16_reglo_vreg_sexti8(i8 addrspace(3)* %in, half %reg) #0 {
+; GFX900-LABEL: load_local_lo_v2f16_reglo_vreg_sexti8:
+; GFX900:       ; %bb.0: ; %entry
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    ds_read_i8 v0, v0
+; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX900-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX900-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; GFX900-NEXT:    global_store_dword v[0:1], v0, off
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX906-LABEL: load_local_lo_v2f16_reglo_vreg_sexti8:
+; GFX906:       ; %bb.0: ; %entry
+; GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-NEXT:    ds_read_i8 v0, v0
+; GFX906-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX906-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX906-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; GFX906-NEXT:    global_store_dword v[0:1], v0, off
+; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX803-LABEL: load_local_lo_v2f16_reglo_vreg_sexti8:
+; GFX803:       ; %bb.0: ; %entry
+; GFX803-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    s_mov_b32 m0, -1
+; GFX803-NEXT:    ds_read_i8 v0, v0
+; GFX803-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX803-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX803-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX803-NEXT:    flat_store_dword v[0:1], v0
+; GFX803-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %load = load i8, i8 addrspace(3)* %in
   %ext = sext i8 %load to i16
@@ -223,20 +514,46 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}load_local_lo_v2i16_reghi_vreg_multi_use_lo:
-; GFX900: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900: ds_read_u16 v0, v0
-; GFX900: v_mov_b32_e32 v3, 0
-; GFX900: v_mov_b32_e32 v2, 0xffff
-; GFX900: s_waitcnt lgkmcnt(0)
-; GFX900: ds_write_b16 v3, v0
-; GFX900: v_bfi_b32 v0, v2, v0, v1
-; GFX900: global_store_dword v[0:1], v0, off
-; GFX900: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX900: s_setpc_b64 s[30:31]
-
-; NO-D16-HI: ds_read_u16 v
 define void @load_local_lo_v2i16_reghi_vreg_multi_use_lo(i16 addrspace(3)* %in, <2 x i16> %reg) #0 {
+; GFX900-LABEL: load_local_lo_v2i16_reghi_vreg_multi_use_lo:
+; GFX900:       ; %bb.0: ; %entry
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    ds_read_u16 v0, v0
+; GFX900-NEXT:    v_mov_b32_e32 v3, 0
+; GFX900-NEXT:    v_mov_b32_e32 v2, 0xffff
+; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX900-NEXT:    ds_write_b16 v3, v0
+; GFX900-NEXT:    v_bfi_b32 v0, v2, v0, v1
+; GFX900-NEXT:    global_store_dword v[0:1], v0, off
+; GFX900-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX906-LABEL: load_local_lo_v2i16_reghi_vreg_multi_use_lo:
+; GFX906:       ; %bb.0: ; %entry
+; GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-NEXT:    ds_read_u16 v0, v0
+; GFX906-NEXT:    v_mov_b32_e32 v3, 0
+; GFX906-NEXT:    v_mov_b32_e32 v2, 0xffff
+; GFX906-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX906-NEXT:    ds_write_b16 v3, v0
+; GFX906-NEXT:    v_bfi_b32 v0, v2, v0, v1
+; GFX906-NEXT:    global_store_dword v[0:1], v0, off
+; GFX906-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX906-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX803-LABEL: load_local_lo_v2i16_reghi_vreg_multi_use_lo:
+; GFX803:       ; %bb.0: ; %entry
+; GFX803-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    s_mov_b32 m0, -1
+; GFX803-NEXT:    v_mov_b32_e32 v2, 0
+; GFX803-NEXT:    ds_read_u16 v0, v0
+; GFX803-NEXT:    v_and_b32_e32 v1, 0xffff0000, v1
+; GFX803-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX803-NEXT:    ds_write_b16 v2, v0
+; GFX803-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX803-NEXT:    flat_store_dword v[0:1], v0
+; GFX803-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %load = load i16, i16 addrspace(3)* %in
   %elt1 = extractelement <2 x i16> %reg, i32 1
@@ -246,19 +563,47 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}load_local_lo_v2i16_reghi_vreg_multi_use_hi:
-; GFX900: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900: v_lshrrev_b32_e32 v2, 16, v1
-; GFX900: ds_read_u16_d16 v1, v0
-; GFX900: v_mov_b32_e32 v0, 0
-; GFX900: ds_write_b16 v0, v2
-; GFX900: s_waitcnt lgkmcnt(1)
-; GFX900: global_store_dword v[0:1], v1, off
-; GFX900: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX900: s_setpc_b64 s[30:31]
-
-; NO-D16-HI: ds_read_u16 v
 define void @load_local_lo_v2i16_reghi_vreg_multi_use_hi(i16 addrspace(3)* %in, <2 x i16> %reg) #0 {
+; GFX900-LABEL: load_local_lo_v2i16_reghi_vreg_multi_use_hi:
+; GFX900:       ; %bb.0: ; %entry
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    v_lshrrev_b32_e32 v2, 16, v1
+; GFX900-NEXT:    ds_read_u16_d16 v1, v0
+; GFX900-NEXT:    v_mov_b32_e32 v0, 0
+; GFX900-NEXT:    ds_write_b16 v0, v2
+; GFX900-NEXT:    s_waitcnt lgkmcnt(1)
+; GFX900-NEXT:    global_store_dword v[0:1], v1, off
+; GFX900-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX906-LABEL: load_local_lo_v2i16_reghi_vreg_multi_use_hi:
+; GFX906:       ; %bb.0: ; %entry
+; GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-NEXT:    ds_read_u16 v0, v0
+; GFX906-NEXT:    v_mov_b32_e32 v2, 0xffff
+; GFX906-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
+; GFX906-NEXT:    v_mov_b32_e32 v4, 0
+; GFX906-NEXT:    ds_write_b16 v4, v3
+; GFX906-NEXT:    s_waitcnt lgkmcnt(1)
+; GFX906-NEXT:    v_bfi_b32 v0, v2, v0, v1
+; GFX906-NEXT:    global_store_dword v[0:1], v0, off
+; GFX906-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX906-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX803-LABEL: load_local_lo_v2i16_reghi_vreg_multi_use_hi:
+; GFX803:       ; %bb.0: ; %entry
+; GFX803-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    s_mov_b32 m0, -1
+; GFX803-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX803-NEXT:    v_mov_b32_e32 v3, 0
+; GFX803-NEXT:    ds_read_u16 v0, v0
+; GFX803-NEXT:    v_lshlrev_b32_e32 v2, 16, v1
+; GFX803-NEXT:    ds_write_b16 v3, v1
+; GFX803-NEXT:    s_waitcnt lgkmcnt(1)
+; GFX803-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX803-NEXT:    flat_store_dword v[0:1], v0
+; GFX803-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %load = load i16, i16 addrspace(3)* %in
   %elt1 = extractelement <2 x i16> %reg, i32 1
@@ -268,21 +613,49 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}load_local_lo_v2i16_reghi_vreg_multi_use_lohi:
-; GFX900: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900: ds_read_u16 v0, v0
-; GFX900: v_lshrrev_b32_e32 v[[A_F16:[0-9]+]], 16, v1
-; GFX900: v_mov_b32_e32 v[[A_F32:[0-9]+]], 0xffff
-; GFX900: s_waitcnt lgkmcnt(0)
-; GFX900: ds_write_b16 v2, v0
-; GFX900: ds_write_b16 v3, v[[A_F16]]
-; GFX900: v_bfi_b32 v0, v[[A_F32]], v0, v1
-; GFX900: global_store_dword v[0:1], v0, off
-; GFX900: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX900: s_setpc_b64 s[30:31]
-
-; NO-D16-HI: ds_read_u16 v
 define void @load_local_lo_v2i16_reghi_vreg_multi_use_lohi(i16 addrspace(3)* noalias %in, <2 x i16> %reg, i16 addrspace(3)* noalias %out0, i16 addrspace(3)* noalias %out1) #0 {
+; GFX900-LABEL: load_local_lo_v2i16_reghi_vreg_multi_use_lohi:
+; GFX900:       ; %bb.0: ; %entry
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    ds_read_u16 v0, v0
+; GFX900-NEXT:    v_lshrrev_b32_e32 v5, 16, v1
+; GFX900-NEXT:    v_mov_b32_e32 v4, 0xffff
+; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX900-NEXT:    ds_write_b16 v2, v0
+; GFX900-NEXT:    ds_write_b16 v3, v5
+; GFX900-NEXT:    v_bfi_b32 v0, v4, v0, v1
+; GFX900-NEXT:    global_store_dword v[0:1], v0, off
+; GFX900-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX906-LABEL: load_local_lo_v2i16_reghi_vreg_multi_use_lohi:
+; GFX906:       ; %bb.0: ; %entry
+; GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-NEXT:    ds_read_u16 v0, v0
+; GFX906-NEXT:    v_lshrrev_b32_e32 v5, 16, v1
+; GFX906-NEXT:    v_mov_b32_e32 v4, 0xffff
+; GFX906-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX906-NEXT:    ds_write_b16 v2, v0
+; GFX906-NEXT:    ds_write_b16 v3, v5
+; GFX906-NEXT:    v_bfi_b32 v0, v4, v0, v1
+; GFX906-NEXT:    global_store_dword v[0:1], v0, off
+; GFX906-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX906-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX803-LABEL: load_local_lo_v2i16_reghi_vreg_multi_use_lohi:
+; GFX803:       ; %bb.0: ; %entry
+; GFX803-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    s_mov_b32 m0, -1
+; GFX803-NEXT:    ds_read_u16 v0, v0
+; GFX803-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX803-NEXT:    v_lshlrev_b32_e32 v4, 16, v1
+; GFX803-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX803-NEXT:    ds_write_b16 v2, v0
+; GFX803-NEXT:    ds_write_b16 v3, v1
+; GFX803-NEXT:    v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX803-NEXT:    flat_store_dword v[0:1], v0
+; GFX803-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %load = load i16, i16 addrspace(3)* %in
   %elt1 = extractelement <2 x i16> %reg, i32 1
@@ -293,17 +666,39 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}load_global_lo_v2i16_reglo_vreg:
-; GCN: s_waitcnt
-; GFX900-NEXT: global_load_short_d16 v2, v[0:1], off offset:-4094
-; GFX900-NEXT: s_waitcnt
-; GFX900-NEXT: global_store_dword
-; GFX900-NEXT: s_waitcnt
-; GFX900-NEXT: s_setpc_b64
-
-; GFX906: global_load_ushort v0, v[0:1], off offset:-4094
-; GFX906: v_bfi_b32
 define void @load_global_lo_v2i16_reglo_vreg(i16 addrspace(1)* %in, i32 %reg) #0 {
+; GFX900-LABEL: load_global_lo_v2i16_reglo_vreg:
+; GFX900:       ; %bb.0: ; %entry
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    global_load_short_d16 v2, v[0:1], off offset:-4094
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    global_store_dword v[0:1], v2, off
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX906-LABEL: load_global_lo_v2i16_reglo_vreg:
+; GFX906:       ; %bb.0: ; %entry
+; GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-NEXT:    global_load_ushort v0, v[0:1], off offset:-4094
+; GFX906-NEXT:    v_mov_b32_e32 v1, 0xffff
+; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    v_bfi_b32 v0, v1, v0, v2
+; GFX906-NEXT:    global_store_dword v[0:1], v0, off
+; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX803-LABEL: load_global_lo_v2i16_reglo_vreg:
+; GFX803:       ; %bb.0: ; %entry
+; GFX803-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    v_add_u32_e32 v0, vcc, 0xfffff002, v0
+; GFX803-NEXT:    v_addc_u32_e32 v1, vcc, -1, v1, vcc
+; GFX803-NEXT:    flat_load_ushort v0, v[0:1]
+; GFX803-NEXT:    v_and_b32_e32 v1, 0xffff0000, v2
+; GFX803-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX803-NEXT:    flat_store_dword v[0:1], v0
+; GFX803-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %reg.bc = bitcast i32 %reg to <2 x i16>
   %gep = getelementptr inbounds i16, i16 addrspace(1)* %in, i64 -2047
@@ -313,21 +708,40 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}load_global_lo_v2f16_reglo_vreg:
-; GCN: s_waitcnt
-; GFX900-NEXT: global_load_short_d16 v2, v[0:1], off offset:-4094
-; GFX900-NEXT: s_waitcnt
-; GFX900-NEXT: global_store_dword
-; GFX900-NEXT: s_waitcnt
-; GFX900-NEXT: s_setpc_b64
-
-; GFX906: global_load_ushort v0, v[0:1], off offset:-4094
-; GFX906: v_lshrrev_b32
-; GFX906: v_and_b32_e32
-; GFX906: v_lshl_or_b32
-
-; GFX803: flat_load_ushort
 define void @load_global_lo_v2f16_reglo_vreg(half addrspace(1)* %in, i32 %reg) #0 {
+; GFX900-LABEL: load_global_lo_v2f16_reglo_vreg:
+; GFX900:       ; %bb.0: ; %entry
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    global_load_short_d16 v2, v[0:1], off offset:-4094
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    global_store_dword v[0:1], v2, off
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX906-LABEL: load_global_lo_v2f16_reglo_vreg:
+; GFX906:       ; %bb.0: ; %entry
+; GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-NEXT:    global_load_ushort v0, v[0:1], off offset:-4094
+; GFX906-NEXT:    v_lshrrev_b32_e32 v1, 16, v2
+; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX906-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; GFX906-NEXT:    global_store_dword v[0:1], v0, off
+; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX803-LABEL: load_global_lo_v2f16_reglo_vreg:
+; GFX803:       ; %bb.0: ; %entry
+; GFX803-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    v_add_u32_e32 v0, vcc, 0xfffff002, v0
+; GFX803-NEXT:    v_addc_u32_e32 v1, vcc, -1, v1, vcc
+; GFX803-NEXT:    flat_load_ushort v0, v[0:1]
+; GFX803-NEXT:    v_and_b32_e32 v1, 0xffff0000, v2
+; GFX803-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX803-NEXT:    flat_store_dword v[0:1], v0
+; GFX803-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %reg.bc = bitcast i32 %reg to <2 x half>
   %gep = getelementptr inbounds half, half addrspace(1)* %in, i64 -2047
@@ -337,19 +751,40 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}load_global_lo_v2i16_reglo_vreg_zexti8:
-; GCN: s_waitcnt
-; GFX900-NEXT: global_load_ubyte_d16 v2, v[0:1], off offset:-4095
-; GFX900-NEXT: s_waitcnt
-; GFX900-NEXT: global_store_dword
-; GFX900-NEXT: s_waitcnt
-; GFX900-NEXT: s_setpc_b64
-
-; GFX906: global_load_ubyte v0, v[0:1], off offset:-4095
-; GFX906: v_bfi_b32
-
-; GFX803: flat_load_ubyte
 define void @load_global_lo_v2i16_reglo_vreg_zexti8(i8 addrspace(1)* %in, i32 %reg) #0 {
+; GFX900-LABEL: load_global_lo_v2i16_reglo_vreg_zexti8:
+; GFX900:       ; %bb.0: ; %entry
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    global_load_ubyte_d16 v2, v[0:1], off offset:-4095
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    global_store_dword v[0:1], v2, off
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX906-LABEL: load_global_lo_v2i16_reglo_vreg_zexti8:
+; GFX906:       ; %bb.0: ; %entry
+; GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-NEXT:    global_load_ubyte v0, v[0:1], off offset:-4095
+; GFX906-NEXT:    v_mov_b32_e32 v1, 0xffff
+; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    v_bfi_b32 v0, v1, v0, v2
+; GFX906-NEXT:    global_store_dword v[0:1], v0, off
+; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX803-LABEL: load_global_lo_v2i16_reglo_vreg_zexti8:
+; GFX803:       ; %bb.0: ; %entry
+; GFX803-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    v_add_u32_e32 v0, vcc, 0xfffff001, v0
+; GFX803-NEXT:    v_addc_u32_e32 v1, vcc, -1, v1, vcc
+; GFX803-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX803-NEXT:    v_lshrrev_b32_e32 v1, 16, v2
+; GFX803-NEXT:    s_mov_b32 s4, 0x5040c00
+; GFX803-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    v_perm_b32 v0, v1, v0, s4
+; GFX803-NEXT:    flat_store_dword v[0:1], v0
+; GFX803-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %reg.bc = bitcast i32 %reg to <2 x i16>
   %gep = getelementptr inbounds i8, i8 addrspace(1)* %in, i64 -4095
@@ -360,19 +795,39 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}load_global_lo_v2i16_reglo_vreg_sexti8:
-; GCN: s_waitcnt
-; GFX900-NEXT: global_load_sbyte_d16 v2, v[0:1], off offset:-4095
-; GFX900-NEXT: s_waitcnt
-; GFX900-NEXT: global_store_dword
-; GFX900-NEXT: s_waitcnt
-; GFX900-NEXT: s_setpc_b64
-
-; GFX906: global_load_sbyte v0, v[0:1], off offset:-4095
-; GFX906: v_bfi_b32
-
-; GFX803: flat_load_sbyte
 define void @load_global_lo_v2i16_reglo_vreg_sexti8(i8 addrspace(1)* %in, i32 %reg) #0 {
+; GFX900-LABEL: load_global_lo_v2i16_reglo_vreg_sexti8:
+; GFX900:       ; %bb.0: ; %entry
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    global_load_sbyte_d16 v2, v[0:1], off offset:-4095
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    global_store_dword v[0:1], v2, off
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX906-LABEL: load_global_lo_v2i16_reglo_vreg_sexti8:
+; GFX906:       ; %bb.0: ; %entry
+; GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-NEXT:    global_load_sbyte v0, v[0:1], off offset:-4095
+; GFX906-NEXT:    v_mov_b32_e32 v1, 0xffff
+; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    v_bfi_b32 v0, v1, v0, v2
+; GFX906-NEXT:    global_store_dword v[0:1], v0, off
+; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX803-LABEL: load_global_lo_v2i16_reglo_vreg_sexti8:
+; GFX803:       ; %bb.0: ; %entry
+; GFX803-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    v_add_u32_e32 v0, vcc, 0xfffff001, v0
+; GFX803-NEXT:    v_addc_u32_e32 v1, vcc, -1, v1, vcc
+; GFX803-NEXT:    flat_load_sbyte v0, v[0:1]
+; GFX803-NEXT:    v_and_b32_e32 v1, 0xffff0000, v2
+; GFX803-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX803-NEXT:    flat_store_dword v[0:1], v0
+; GFX803-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %reg.bc = bitcast i32 %reg to <2 x i16>
   %gep = getelementptr inbounds i8, i8 addrspace(1)* %in, i64 -4095
@@ -383,20 +838,41 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}load_global_lo_v2f16_reglo_vreg_zexti8:
-; GCN: s_waitcnt
-; GFX900-NEXT: global_load_ubyte_d16 v2, v[0:1], off offset:-4095
-; GFX900-NEXT: s_waitcnt
-; GFX900-NEXT: global_store_dword
-; GFX900-NEXT: s_waitcnt
-; GFX900-NEXT: s_setpc_b64
-
-; GFX906: global_load_ubyte v0, v[0:1], off offset:-4095
-; GFX906: v_and_b32_e32
-; GFX906: v_lshl_or_b32
-
-; GFX803: flat_load_ubyte
 define void @load_global_lo_v2f16_reglo_vreg_zexti8(i8 addrspace(1)* %in, i32 %reg) #0 {
+; GFX900-LABEL: load_global_lo_v2f16_reglo_vreg_zexti8:
+; GFX900:       ; %bb.0: ; %entry
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    global_load_ubyte_d16 v2, v[0:1], off offset:-4095
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    global_store_dword v[0:1], v2, off
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX906-LABEL: load_global_lo_v2f16_reglo_vreg_zexti8:
+; GFX906:       ; %bb.0: ; %entry
+; GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-NEXT:    global_load_ubyte v0, v[0:1], off offset:-4095
+; GFX906-NEXT:    v_lshrrev_b32_e32 v1, 16, v2
+; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX906-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; GFX906-NEXT:    global_store_dword v[0:1], v0, off
+; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX803-LABEL: load_global_lo_v2f16_reglo_vreg_zexti8:
+; GFX803:       ; %bb.0: ; %entry
+; GFX803-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    v_add_u32_e32 v0, vcc, 0xfffff001, v0
+; GFX803-NEXT:    v_addc_u32_e32 v1, vcc, -1, v1, vcc
+; GFX803-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX803-NEXT:    v_lshrrev_b32_e32 v1, 16, v2
+; GFX803-NEXT:    s_mov_b32 s4, 0x5040c00
+; GFX803-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    v_perm_b32 v0, v1, v0, s4
+; GFX803-NEXT:    flat_store_dword v[0:1], v0
+; GFX803-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %reg.bc = bitcast i32 %reg to <2 x half>
   %gep = getelementptr inbounds i8, i8 addrspace(1)* %in, i64 -4095
@@ -408,21 +884,40 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}load_global_lo_v2f16_reglo_vreg_sexti8:
-; GCN: s_waitcnt
-; GFX900-NEXT: global_load_sbyte_d16 v2, v[0:1], off offset:-4095
-; GFX900-NEXT: s_waitcnt
-; GFX900-NEXT: global_store_dword
-; GFX900-NEXT: s_waitcnt
-; GFX900-NEXT: s_setpc_b64
-
-; GFX906: global_load_sbyte v0, v[0:1], off offset:-4095
-; GFX906: v_lshrrev_b32
-; GFX906: v_and_b32
-; GFX906: v_lshl_or_b32
-
-; GFX803: flat_load_sbyte
 define void @load_global_lo_v2f16_reglo_vreg_sexti8(i8 addrspace(1)* %in, i32 %reg) #0 {
+; GFX900-LABEL: load_global_lo_v2f16_reglo_vreg_sexti8:
+; GFX900:       ; %bb.0: ; %entry
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    global_load_sbyte_d16 v2, v[0:1], off offset:-4095
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    global_store_dword v[0:1], v2, off
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX906-LABEL: load_global_lo_v2f16_reglo_vreg_sexti8:
+; GFX906:       ; %bb.0: ; %entry
+; GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-NEXT:    global_load_sbyte v0, v[0:1], off offset:-4095
+; GFX906-NEXT:    v_lshrrev_b32_e32 v1, 16, v2
+; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX906-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; GFX906-NEXT:    global_store_dword v[0:1], v0, off
+; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX803-LABEL: load_global_lo_v2f16_reglo_vreg_sexti8:
+; GFX803:       ; %bb.0: ; %entry
+; GFX803-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    v_add_u32_e32 v0, vcc, 0xfffff001, v0
+; GFX803-NEXT:    v_addc_u32_e32 v1, vcc, -1, v1, vcc
+; GFX803-NEXT:    flat_load_sbyte v0, v[0:1]
+; GFX803-NEXT:    v_and_b32_e32 v1, 0xffff0000, v2
+; GFX803-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX803-NEXT:    flat_store_dword v[0:1], v0
+; GFX803-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %reg.bc = bitcast i32 %reg to <2 x half>
   %gep = getelementptr inbounds i8, i8 addrspace(1)* %in, i64 -4095
@@ -434,22 +929,37 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}load_flat_lo_v2i16_reghi_vreg:
-; GCN: s_waitcnt
-; GFX900-NEXT: flat_load_short_d16 v2, v[0:1]
-; GFX900-NEXT: s_waitcnt
-; GFX900-NEXT: global_store_dword v[0:1], v2
-; GFX900-NEXT: s_waitcnt
-; GFX900-NEXT: s_setpc_b64
-
-; GFX803: flat_load_ushort v{{[0-9]+}}
-; GFX803: v_or_b32_e32
-
-; GFX906: flat_load_ushort [[LOAD:v[0-9]+]]
-; GFX906: v_mov_b32_e32 [[MASK:v[0-9]+]], 0xffff{{$}}
-; GFX906: v_bfi_b32 v{{[0-9]+}}, [[MASK]], [[LOAD]], v2
-; GFX906: global_store_dword
 define void @load_flat_lo_v2i16_reghi_vreg(i16* %in, i32 %reg) #0 {
+; GFX900-LABEL: load_flat_lo_v2i16_reghi_vreg:
+; GFX900:       ; %bb.0: ; %entry
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    flat_load_short_d16 v2, v[0:1]
+; GFX900-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    global_store_dword v[0:1], v2, off
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX906-LABEL: load_flat_lo_v2i16_reghi_vreg:
+; GFX906:       ; %bb.0: ; %entry
+; GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-NEXT:    flat_load_ushort v0, v[0:1]
+; GFX906-NEXT:    v_mov_b32_e32 v1, 0xffff
+; GFX906-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX906-NEXT:    v_bfi_b32 v0, v1, v0, v2
+; GFX906-NEXT:    global_store_dword v[0:1], v0, off
+; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX803-LABEL: load_flat_lo_v2i16_reghi_vreg:
+; GFX803:       ; %bb.0: ; %entry
+; GFX803-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    flat_load_ushort v0, v[0:1]
+; GFX803-NEXT:    v_and_b32_e32 v1, 0xffff0000, v2
+; GFX803-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX803-NEXT:    flat_store_dword v[0:1], v0
+; GFX803-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %reg.bc = bitcast i32 %reg to <2 x i16>
   %load = load i16, i16* %in
@@ -458,24 +968,40 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}load_flat_lo_v2f16_reghi_vreg:
-; GCN: s_waitcnt
-; GFX900-NEXT: flat_load_short_d16 v2, v[0:1]
-; GFX900-NEXT: s_waitcnt
-; GFX900-NEXT: global_store_dword v[0:1], v2
-; GFX900-NEXT: s_waitcnt
-; GFX900-NEXT: s_setpc_b64
-
-; GFX803: flat_load_ushort v{{[0-9]+}}
-; GFX803: v_or_b32_e32
-
-; FIXME: and should be removable
-; GFX906: flat_load_ushort [[LOAD:v[0-9]+]]
-; GFX906: v_lshrrev_b32_e32 [[SHR:v[0-9]+]], 16, v2
-; GFX906: v_and_b32_e32 [[AND:v[0-9]+]], 0xffff, [[LOAD]]
-; GFX906: v_lshl_or_b32 [[LSHL_OR:v[0-9]+]], [[SHR]], 16, [[AND]]
-; GFX906: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[LSHL_OR]]
 define void @load_flat_lo_v2f16_reghi_vreg(half* %in, i32 %reg) #0 {
+; GFX900-LABEL: load_flat_lo_v2f16_reghi_vreg:
+; GFX900:       ; %bb.0: ; %entry
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    flat_load_short_d16 v2, v[0:1]
+; GFX900-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    global_store_dword v[0:1], v2, off
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX906-LABEL: load_flat_lo_v2f16_reghi_vreg:
+; GFX906:       ; %bb.0: ; %entry
+; GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-NEXT:    flat_load_ushort v0, v[0:1]
+; GFX906-NEXT:    v_lshrrev_b32_e32 v1, 16, v2
+; GFX906-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX906-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX906-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; GFX906-NEXT:    global_store_dword v[0:1], v0, off
+; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX803-LABEL: load_flat_lo_v2f16_reghi_vreg:
+; GFX803:       ; %bb.0: ; %entry
+; GFX803-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    flat_load_ushort v0, v[0:1]
+; GFX803-NEXT:    v_and_b32_e32 v1, 0xffff0000, v2
+; GFX803-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX803-NEXT:    flat_store_dword v[0:1], v0
+; GFX803-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    s_setpc_b64 s[30:31]
+
+; FIXME: the and above should be removable
 entry:
   %reg.bc = bitcast i32 %reg to <2 x half>
   %load = load half, half* %in
@@ -484,23 +1010,38 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}load_flat_lo_v2i16_reglo_vreg_zexti8:
-; GCN: s_waitcnt
-; GFX900-NEXT: flat_load_ubyte_d16 v2, v[0:1]
-; GFX900-NEXT: s_waitcnt
-; GFX900-NEXT: global_store_dword v[0:1], v2
-; GFX900-NEXT: s_waitcnt
-; GFX900-NEXT: s_setpc_b64
-
-; GFX803: flat_load_ubyte [[LO:v[0-9]+]]
-; GFX803: v_lshrrev_b32_e32 [[HI:v[0-9]+]], 16, v2
-; GFX803: s_mov_b32 [[MASK:s[0-9]+]], 0x5040c00
-; GFX803: v_perm_b32 [[RES:v[0-9]+]], [[HI]], [[LO]], [[MASK]]
-; GFX803: flat_store_dword v[0:1], [[RES]]
-
-; GFX906: flat_load_ubyte
-; GFX906: v_bfi_b32
 define void @load_flat_lo_v2i16_reglo_vreg_zexti8(i8* %in, i32 %reg) #0 {
+; GFX900-LABEL: load_flat_lo_v2i16_reglo_vreg_zexti8:
+; GFX900:       ; %bb.0: ; %entry
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    flat_load_ubyte_d16 v2, v[0:1]
+; GFX900-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    global_store_dword v[0:1], v2, off
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX906-LABEL: load_flat_lo_v2i16_reglo_vreg_zexti8:
+; GFX906:       ; %bb.0: ; %entry
+; GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX906-NEXT:    v_mov_b32_e32 v1, 0xffff
+; GFX906-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX906-NEXT:    v_bfi_b32 v0, v1, v0, v2
+; GFX906-NEXT:    global_store_dword v[0:1], v0, off
+; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX803-LABEL: load_flat_lo_v2i16_reglo_vreg_zexti8:
+; GFX803:       ; %bb.0: ; %entry
+; GFX803-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX803-NEXT:    v_lshrrev_b32_e32 v1, 16, v2
+; GFX803-NEXT:    s_mov_b32 s4, 0x5040c00
+; GFX803-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    v_perm_b32 v0, v1, v0, s4
+; GFX803-NEXT:    flat_store_dword v[0:1], v0
+; GFX803-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %reg.bc = bitcast i32 %reg to <2 x i16>
   %load = load i8, i8* %in
@@ -510,20 +1051,37 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}load_flat_lo_v2i16_reglo_vreg_sexti8:
-; GCN: s_waitcnt
-; GFX900-NEXT: flat_load_sbyte_d16 v2, v[0:1]
-; GFX900-NEXT: s_waitcnt
-; GFX900-NEXT: global_store_dword v[0:1], v2
-; GFX900-NEXT: s_waitcnt
-; GFX900-NEXT: s_setpc_b64
-
-; GFX803: flat_load_sbyte v{{[0-9]+}}
-; GFX803: v_or_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-
-; GFX906: flat_load_sbyte
-; GFX906: v_bfi_b32
 define void @load_flat_lo_v2i16_reglo_vreg_sexti8(i8* %in, i32 %reg) #0 {
+; GFX900-LABEL: load_flat_lo_v2i16_reglo_vreg_sexti8:
+; GFX900:       ; %bb.0: ; %entry
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    flat_load_sbyte_d16 v2, v[0:1]
+; GFX900-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    global_store_dword v[0:1], v2, off
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX906-LABEL: load_flat_lo_v2i16_reglo_vreg_sexti8:
+; GFX906:       ; %bb.0: ; %entry
+; GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-NEXT:    flat_load_sbyte v0, v[0:1]
+; GFX906-NEXT:    v_mov_b32_e32 v1, 0xffff
+; GFX906-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX906-NEXT:    v_bfi_b32 v0, v1, v0, v2
+; GFX906-NEXT:    global_store_dword v[0:1], v0, off
+; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX803-LABEL: load_flat_lo_v2i16_reglo_vreg_sexti8:
+; GFX803:       ; %bb.0: ; %entry
+; GFX803-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    flat_load_sbyte v0, v[0:1]
+; GFX803-NEXT:    v_and_b32_e32 v1, 0xffff0000, v2
+; GFX803-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX803-NEXT:    flat_store_dword v[0:1], v0
+; GFX803-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %reg.bc = bitcast i32 %reg to <2 x i16>
   %load = load i8, i8* %in
@@ -533,25 +1091,39 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}load_flat_lo_v2f16_reglo_vreg_zexti8:
-; GCN: s_waitcnt
-; GFX900-NEXT: flat_load_ubyte_d16 v2, v[0:1]
-; GFX900-NEXT: s_waitcnt
-; GFX900-NEXT: global_store_dword v[0:1], v2
-; GFX900-NEXT: s_waitcnt
-; GFX900-NEXT: s_setpc_b64
-
-; GFX803: flat_load_ubyte [[LO:v[0-9]+]]
-; GFX803: v_lshrrev_b32_e32 [[HI:v[0-9]+]], 16, v2
-; GFX803: s_mov_b32 [[MASK:s[0-9]+]], 0x5040c00
-; GFX803: v_perm_b32 [[RES:v[0-9]+]], [[HI]], [[LO]], [[MASK]]
-; GFX803: flat_store_dword v[0:1], [[RES]]
-
-; GFX906: flat_load_ubyte
-; GFX906: v_lshrrev_b32
-; GFX906: v_and_b32
-; GFX906: v_lshl_or_b32
 define void @load_flat_lo_v2f16_reglo_vreg_zexti8(i8* %in, i32 %reg) #0 {
+; GFX900-LABEL: load_flat_lo_v2f16_reglo_vreg_zexti8:
+; GFX900:       ; %bb.0: ; %entry
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    flat_load_ubyte_d16 v2, v[0:1]
+; GFX900-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    global_store_dword v[0:1], v2, off
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX906-LABEL: load_flat_lo_v2f16_reglo_vreg_zexti8:
+; GFX906:       ; %bb.0: ; %entry
+; GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX906-NEXT:    v_lshrrev_b32_e32 v1, 16, v2
+; GFX906-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX906-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX906-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; GFX906-NEXT:    global_store_dword v[0:1], v0, off
+; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX803-LABEL: load_flat_lo_v2f16_reglo_vreg_zexti8:
+; GFX803:       ; %bb.0: ; %entry
+; GFX803-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX803-NEXT:    v_lshrrev_b32_e32 v1, 16, v2
+; GFX803-NEXT:    s_mov_b32 s4, 0x5040c00
+; GFX803-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    v_perm_b32 v0, v1, v0, s4
+; GFX803-NEXT:    flat_store_dword v[0:1], v0
+; GFX803-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %reg.bc = bitcast i32 %reg to <2 x half>
   %load = load i8, i8* %in
@@ -562,22 +1134,38 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}load_flat_lo_v2f16_reglo_vreg_sexti8:
-; GCN: s_waitcnt
-; GFX900-NEXT: flat_load_sbyte_d16 v2, v[0:1]
-; GFX900-NEXT: s_waitcnt
-; GFX900-NEXT: global_store_dword v[0:1], v2
-; GFX900-NEXT: s_waitcnt
-; GFX900-NEXT: s_setpc_b64
-
-; GFX803: flat_load_sbyte v{{[0-9]+}}
-; GFX803: v_or_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-
-; GFX906: flat_load_sbyte
-; GFX906: v_lshrrev_b32
-; GFX906: v_and_b32
-; GFX906: v_lshl_or_b32
 define void @load_flat_lo_v2f16_reglo_vreg_sexti8(i8* %in, i32 %reg) #0 {
+; GFX900-LABEL: load_flat_lo_v2f16_reglo_vreg_sexti8:
+; GFX900:       ; %bb.0: ; %entry
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    flat_load_sbyte_d16 v2, v[0:1]
+; GFX900-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    global_store_dword v[0:1], v2, off
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX906-LABEL: load_flat_lo_v2f16_reglo_vreg_sexti8:
+; GFX906:       ; %bb.0: ; %entry
+; GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-NEXT:    flat_load_sbyte v0, v[0:1]
+; GFX906-NEXT:    v_lshrrev_b32_e32 v1, 16, v2
+; GFX906-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX906-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX906-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; GFX906-NEXT:    global_store_dword v[0:1], v0, off
+; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX803-LABEL: load_flat_lo_v2f16_reglo_vreg_sexti8:
+; GFX803:       ; %bb.0: ; %entry
+; GFX803-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    flat_load_sbyte v0, v[0:1]
+; GFX803-NEXT:    v_and_b32_e32 v1, 0xffff0000, v2
+; GFX803-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX803-NEXT:    flat_store_dword v[0:1], v0
+; GFX803-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %reg.bc = bitcast i32 %reg to <2 x half>
   %load = load i8, i8* %in
@@ -588,16 +1176,37 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}load_private_lo_v2i16_reglo_vreg:
-; GCN: s_waitcnt
-; GFX900: buffer_load_short_d16 v0, off, s[0:3], s32 offset:4094{{$}}
-; GFX900-NEXT: s_waitcnt
-; GFX900-NEXT: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v0
-; GFX900-NEXT: s_waitcnt
-; GFX900-NEXT: s_setpc_b64
-
-; NO-D16-HI: buffer_load_ushort v{{[0-9]+}}, off, s[0:3], s32 offset:4094{{$}}
 define void @load_private_lo_v2i16_reglo_vreg(i16 addrspace(5)* byval %in, i32 %reg) #0 {
+; GFX900-LABEL: load_private_lo_v2i16_reglo_vreg:
+; GFX900:       ; %bb.0: ; %entry
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    buffer_load_short_d16 v0, off, s[0:3], s32 offset:4094
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    global_store_dword v[0:1], v0, off
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX906-LABEL: load_private_lo_v2i16_reglo_vreg:
+; GFX906:       ; %bb.0: ; %entry
+; GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:4094
+; GFX906-NEXT:    v_mov_b32_e32 v2, 0xffff
+; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    v_bfi_b32 v0, v2, v1, v0
+; GFX906-NEXT:    global_store_dword v[0:1], v0, off
+; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX803-LABEL: load_private_lo_v2i16_reglo_vreg:
+; GFX803:       ; %bb.0: ; %entry
+; GFX803-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:4094
+; GFX803-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
+; GFX803-NEXT:    s_waitcnt vmcnt(0)
+; GFX803-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX803-NEXT:    flat_store_dword v[0:1], v0
+; GFX803-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %reg.bc = bitcast i32 %reg to <2 x i16>
   %gep = getelementptr inbounds i16, i16 addrspace(5)* %in, i64 2047
@@ -607,19 +1216,39 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}load_private_lo_v2i16_reghi_vreg:
-; GCN: s_waitcnt
-; GFX900: buffer_load_ushort v1, off, s[0:3], s32 offset:4094{{$}}
-; GFX900-NEXT: s_waitcnt
-; GFX900: v_and_b32
-; GFX900: v_lshl_or_b32
-
-; GFX900-NEXT: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}
-; GFX900-NEXT: s_waitcnt
-; GFX900-NEXT: s_setpc_b64
-
-; NO-D16-HI: buffer_load_ushort v{{[0-9]+}}, off, s[0:3], s32 offset:4094{{$}}
 define void @load_private_lo_v2i16_reghi_vreg(i16 addrspace(5)* byval %in, i16 %reg) #0 {
+; GFX900-LABEL: load_private_lo_v2i16_reghi_vreg:
+; GFX900:       ; %bb.0: ; %entry
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:4094
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GFX900-NEXT:    v_lshl_or_b32 v0, v0, 16, v1
+; GFX900-NEXT:    global_store_dword v[0:1], v0, off
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX906-LABEL: load_private_lo_v2i16_reghi_vreg:
+; GFX906:       ; %bb.0: ; %entry
+; GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:4094
+; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GFX906-NEXT:    v_lshl_or_b32 v0, v0, 16, v1
+; GFX906-NEXT:    global_store_dword v[0:1], v0, off
+; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX803-LABEL: load_private_lo_v2i16_reghi_vreg:
+; GFX803:       ; %bb.0: ; %entry
+; GFX803-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:4094
+; GFX803-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX803-NEXT:    s_waitcnt vmcnt(0)
+; GFX803-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX803-NEXT:    flat_store_dword v[0:1], v0
+; GFX803-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %gep = getelementptr inbounds i16, i16 addrspace(5)* %in, i64 2047
   %load = load i16, i16 addrspace(5)* %gep
@@ -629,16 +1258,38 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}load_private_lo_v2f16_reglo_vreg:
-; GCN: s_waitcnt
-; GFX900: buffer_load_short_d16 v0, off, s[0:3], s32 offset:4094{{$}}
-; GFX900-NEXT: s_waitcnt
-; GFX900-NEXT: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v0
-; GFX900-NEXT: s_waitcnt
-; GFX900-NEXT: s_setpc_b64
-
-; NO-D16-HI: buffer_load_ushort v{{[0-9]+}}, off, s[0:3], s32 offset:4094{{$}}
 define void @load_private_lo_v2f16_reglo_vreg(half addrspace(5)* byval %in, i32 %reg) #0 {
+; GFX900-LABEL: load_private_lo_v2f16_reglo_vreg:
+; GFX900:       ; %bb.0: ; %entry
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    buffer_load_short_d16 v0, off, s[0:3], s32 offset:4094
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    global_store_dword v[0:1], v0, off
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX906-LABEL: load_private_lo_v2f16_reglo_vreg:
+; GFX906:       ; %bb.0: ; %entry
+; GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:4094
+; GFX906-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GFX906-NEXT:    v_lshl_or_b32 v0, v0, 16, v1
+; GFX906-NEXT:    global_store_dword v[0:1], v0, off
+; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX803-LABEL: load_private_lo_v2f16_reglo_vreg:
+; GFX803:       ; %bb.0: ; %entry
+; GFX803-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:4094
+; GFX803-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
+; GFX803-NEXT:    s_waitcnt vmcnt(0)
+; GFX803-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX803-NEXT:    flat_store_dword v[0:1], v0
+; GFX803-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %reg.bc = bitcast i32 %reg to <2 x half>
   %gep = getelementptr inbounds half, half addrspace(5)* %in, i64 2047
@@ -648,16 +1299,37 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}load_private_lo_v2i16_reglo_vreg_nooff:
-; GCN: s_waitcnt
-; GFX900-NEXT: buffer_load_short_d16 v1, off, s[0:3], s33 offset:4094{{$}}
-; GFX900-NEXT: s_waitcnt
-; GFX900-NEXT: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v1
-; GFX900-NEXT: s_waitcnt
-; GFX900-NEXT: s_setpc_b64
-
-; NO-D16-HI: buffer_load_ushort v{{[0-9]+}}, off, s[0:3], s33 offset:4094{{$}}
 define void @load_private_lo_v2i16_reglo_vreg_nooff(i16 addrspace(5)* %in, i32 %reg) #0 {
+; GFX900-LABEL: load_private_lo_v2i16_reglo_vreg_nooff:
+; GFX900:       ; %bb.0: ; %entry
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    buffer_load_short_d16 v1, off, s[0:3], s33 offset:4094
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    global_store_dword v[0:1], v1, off
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX906-LABEL: load_private_lo_v2i16_reglo_vreg_nooff:
+; GFX906:       ; %bb.0: ; %entry
+; GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-NEXT:    buffer_load_ushort v0, off, s[0:3], s33 offset:4094
+; GFX906-NEXT:    v_mov_b32_e32 v2, 0xffff
+; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    v_bfi_b32 v0, v2, v0, v1
+; GFX906-NEXT:    global_store_dword v[0:1], v0, off
+; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX803-LABEL: load_private_lo_v2i16_reglo_vreg_nooff:
+; GFX803:       ; %bb.0: ; %entry
+; GFX803-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    buffer_load_ushort v0, off, s[0:3], s33 offset:4094
+; GFX803-NEXT:    v_and_b32_e32 v1, 0xffff0000, v1
+; GFX803-NEXT:    s_waitcnt vmcnt(0)
+; GFX803-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX803-NEXT:    flat_store_dword v[0:1], v0
+; GFX803-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %reg.bc = bitcast i32 %reg to <2 x i16>
   %load = load volatile i16, i16 addrspace(5)* inttoptr (i32 4094 to i16 addrspace(5)*)
@@ -666,16 +1338,37 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}load_private_lo_v2i16_reghi_vreg_nooff:
-; GCN: s_waitcnt
-; GFX900-NEXT: buffer_load_short_d16 v1, off, s[0:3], s33 offset:4094{{$}}
-; GFX900-NEXT: s_waitcnt
-; GFX900-NEXT: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v1
-; GFX900-NEXT: s_waitcnt
-; GFX900-NEXT: s_setpc_b64
-
-; NO-D16-HI: buffer_load_ushort v{{[0-9]+}}, off, s[0:3], s33 offset:4094{{$}}
 define void @load_private_lo_v2i16_reghi_vreg_nooff(i16 addrspace(5)* %in, i32 %reg) #0 {
+; GFX900-LABEL: load_private_lo_v2i16_reghi_vreg_nooff:
+; GFX900:       ; %bb.0: ; %entry
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    buffer_load_short_d16 v1, off, s[0:3], s33 offset:4094
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    global_store_dword v[0:1], v1, off
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX906-LABEL: load_private_lo_v2i16_reghi_vreg_nooff:
+; GFX906:       ; %bb.0: ; %entry
+; GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-NEXT:    buffer_load_ushort v0, off, s[0:3], s33 offset:4094
+; GFX906-NEXT:    v_mov_b32_e32 v2, 0xffff
+; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    v_bfi_b32 v0, v2, v0, v1
+; GFX906-NEXT:    global_store_dword v[0:1], v0, off
+; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX803-LABEL: load_private_lo_v2i16_reghi_vreg_nooff:
+; GFX803:       ; %bb.0: ; %entry
+; GFX803-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    buffer_load_ushort v0, off, s[0:3], s33 offset:4094
+; GFX803-NEXT:    v_and_b32_e32 v1, 0xffff0000, v1
+; GFX803-NEXT:    s_waitcnt vmcnt(0)
+; GFX803-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX803-NEXT:    flat_store_dword v[0:1], v0
+; GFX803-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %reg.bc = bitcast i32 %reg to <2 x i16>
   %load = load volatile i16, i16 addrspace(5)* inttoptr (i32 4094 to i16 addrspace(5)*)
@@ -684,16 +1377,38 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}load_private_lo_v2f16_reglo_vreg_nooff:
-; GCN: s_waitcnt
-; GFX900-NEXT: buffer_load_short_d16 v1, off, s[0:3], s33 offset:4094{{$}}
-; GFX900-NEXT: s_waitcnt
-; GFX900-NEXT: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v1
-; GFX900-NEXT: s_waitcnt
-; GFX900-NEXT: s_setpc_b64
-
-; NO-D16-HI: buffer_load_ushort v{{[0-9]+}}, off, s[0:3], s33 offset:4094{{$}}
 define void @load_private_lo_v2f16_reglo_vreg_nooff(half addrspace(5)* %in, i32 %reg) #0 {
+; GFX900-LABEL: load_private_lo_v2f16_reglo_vreg_nooff:
+; GFX900:       ; %bb.0: ; %entry
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    buffer_load_short_d16 v1, off, s[0:3], s33 offset:4094
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    global_store_dword v[0:1], v1, off
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX906-LABEL: load_private_lo_v2f16_reglo_vreg_nooff:
+; GFX906:       ; %bb.0: ; %entry
+; GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-NEXT:    buffer_load_ushort v0, off, s[0:3], s33 offset:4094
+; GFX906-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX906-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; GFX906-NEXT:    global_store_dword v[0:1], v0, off
+; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX803-LABEL: load_private_lo_v2f16_reglo_vreg_nooff:
+; GFX803:       ; %bb.0: ; %entry
+; GFX803-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    buffer_load_ushort v0, off, s[0:3], s33 offset:4094
+; GFX803-NEXT:    v_and_b32_e32 v1, 0xffff0000, v1
+; GFX803-NEXT:    s_waitcnt vmcnt(0)
+; GFX803-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX803-NEXT:    flat_store_dword v[0:1], v0
+; GFX803-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %reg.bc = bitcast i32 %reg to <2 x half>
   %load = load volatile half, half addrspace(5)* inttoptr (i32 4094 to half addrspace(5)*)
@@ -702,16 +1417,38 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}load_private_lo_v2i16_reglo_vreg_zexti8:
-; GCN: s_waitcnt
-; GFX900: buffer_load_ubyte_d16 v0, off, s[0:3], s32 offset:4095{{$}}
-; GFX900-NEXT: s_waitcnt
-; GFX900-NEXT: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v0
-; GFX900-NEXT: s_waitcnt
-; GFX900-NEXT: s_setpc_b64
-
-; NO-D16-HI: buffer_load_ubyte v{{[0-9]+}}, off, s[0:3], s32 offset:4095{{$}}
 define void @load_private_lo_v2i16_reglo_vreg_zexti8(i8 addrspace(5)* byval %in, i32 %reg) #0 {
+; GFX900-LABEL: load_private_lo_v2i16_reglo_vreg_zexti8:
+; GFX900:       ; %bb.0: ; %entry
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    buffer_load_ubyte_d16 v0, off, s[0:3], s32 offset:4095
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    global_store_dword v[0:1], v0, off
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX906-LABEL: load_private_lo_v2i16_reglo_vreg_zexti8:
+; GFX906:       ; %bb.0: ; %entry
+; GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-NEXT:    buffer_load_ubyte v1, off, s[0:3], s32 offset:4095
+; GFX906-NEXT:    v_mov_b32_e32 v2, 0xffff
+; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    v_bfi_b32 v0, v2, v1, v0
+; GFX906-NEXT:    global_store_dword v[0:1], v0, off
+; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX803-LABEL: load_private_lo_v2i16_reglo_vreg_zexti8:
+; GFX803:       ; %bb.0: ; %entry
+; GFX803-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    buffer_load_ubyte v1, off, s[0:3], s32 offset:4095
+; GFX803-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX803-NEXT:    s_mov_b32 s4, 0x5040c00
+; GFX803-NEXT:    s_waitcnt vmcnt(0)
+; GFX803-NEXT:    v_perm_b32 v0, v0, v1, s4
+; GFX803-NEXT:    flat_store_dword v[0:1], v0
+; GFX803-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %reg.bc = bitcast i32 %reg to <2 x i16>
   %gep = getelementptr inbounds i8, i8 addrspace(5)* %in, i64 4095
@@ -722,16 +1459,37 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}load_private_lo_v2i16_reglo_vreg_sexti8:
-; GCN: s_waitcnt
-; GFX900: buffer_load_sbyte_d16 v0, off, s[0:3], s32 offset:4095{{$}}
-; GFX900-NEXT: s_waitcnt
-; GFX900-NEXT: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v0
-; GFX900-NEXT: s_waitcnt
-; GFX900-NEXT: s_setpc_b64
-
-; NO-D16-HI: buffer_load_sbyte v{{[0-9]+}}, off, s[0:3], s32 offset:4095{{$}}
 define void @load_private_lo_v2i16_reglo_vreg_sexti8(i8 addrspace(5)* byval %in, i32 %reg) #0 {
+; GFX900-LABEL: load_private_lo_v2i16_reglo_vreg_sexti8:
+; GFX900:       ; %bb.0: ; %entry
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    buffer_load_sbyte_d16 v0, off, s[0:3], s32 offset:4095
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    global_store_dword v[0:1], v0, off
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX906-LABEL: load_private_lo_v2i16_reglo_vreg_sexti8:
+; GFX906:       ; %bb.0: ; %entry
+; GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-NEXT:    buffer_load_sbyte v1, off, s[0:3], s32 offset:4095
+; GFX906-NEXT:    v_mov_b32_e32 v2, 0xffff
+; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    v_bfi_b32 v0, v2, v1, v0
+; GFX906-NEXT:    global_store_dword v[0:1], v0, off
+; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX803-LABEL: load_private_lo_v2i16_reglo_vreg_sexti8:
+; GFX803:       ; %bb.0: ; %entry
+; GFX803-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    buffer_load_sbyte v1, off, s[0:3], s32 offset:4095
+; GFX803-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
+; GFX803-NEXT:    s_waitcnt vmcnt(0)
+; GFX803-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX803-NEXT:    flat_store_dword v[0:1], v0
+; GFX803-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %reg.bc = bitcast i32 %reg to <2 x i16>
   %gep = getelementptr inbounds i8, i8 addrspace(5)* %in, i64 4095
@@ -742,16 +1500,38 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}load_private_lo_v2i16_reglo_vreg_nooff_zexti8:
-; GCN: s_waitcnt
-; GFX900-NEXT: buffer_load_ubyte_d16 v1, off, s[0:3], s33 offset:4094{{$}}
-; GFX900-NEXT: s_waitcnt
-; GFX900-NEXT: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v1
-; GFX900-NEXT: s_waitcnt
-; GFX900-NEXT: s_setpc_b64
-
-; NO-D16-HI: buffer_load_ubyte v0, off, s[0:3], s33 offset:4094{{$}}
 define void @load_private_lo_v2i16_reglo_vreg_nooff_zexti8(i8 addrspace(5)* %in, i32 %reg) #0 {
+; GFX900-LABEL: load_private_lo_v2i16_reglo_vreg_nooff_zexti8:
+; GFX900:       ; %bb.0: ; %entry
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    buffer_load_ubyte_d16 v1, off, s[0:3], s33 offset:4094
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    global_store_dword v[0:1], v1, off
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX906-LABEL: load_private_lo_v2i16_reglo_vreg_nooff_zexti8:
+; GFX906:       ; %bb.0: ; %entry
+; GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-NEXT:    buffer_load_ubyte v0, off, s[0:3], s33 offset:4094
+; GFX906-NEXT:    v_mov_b32_e32 v2, 0xffff
+; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    v_bfi_b32 v0, v2, v0, v1
+; GFX906-NEXT:    global_store_dword v[0:1], v0, off
+; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX803-LABEL: load_private_lo_v2i16_reglo_vreg_nooff_zexti8:
+; GFX803:       ; %bb.0: ; %entry
+; GFX803-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    buffer_load_ubyte v0, off, s[0:3], s33 offset:4094
+; GFX803-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX803-NEXT:    s_mov_b32 s4, 0x5040c00
+; GFX803-NEXT:    s_waitcnt vmcnt(0)
+; GFX803-NEXT:    v_perm_b32 v0, v1, v0, s4
+; GFX803-NEXT:    flat_store_dword v[0:1], v0
+; GFX803-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %reg.bc = bitcast i32 %reg to <2 x i16>
   %load = load volatile i8, i8 addrspace(5)* inttoptr (i32 4094 to i8 addrspace(5)*)
@@ -761,16 +1541,37 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}load_private_lo_v2i16_reglo_vreg_nooff_sexti8:
-; GCN: s_waitcnt
-; GFX900-NEXT: buffer_load_sbyte_d16 v1, off, s[0:3], s33 offset:4094{{$}}
-; GFX900-NEXT: s_waitcnt
-; GFX900-NEXT: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v1
-; GFX900-NEXT: s_waitcnt
-; GFX900-NEXT: s_setpc_b64
-
-; NO-D16-HI: buffer_load_sbyte v0, off, s[0:3], s33 offset:4094{{$}}
 define void @load_private_lo_v2i16_reglo_vreg_nooff_sexti8(i8 addrspace(5)* %in, i32 %reg) #0 {
+; GFX900-LABEL: load_private_lo_v2i16_reglo_vreg_nooff_sexti8:
+; GFX900:       ; %bb.0: ; %entry
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    buffer_load_sbyte_d16 v1, off, s[0:3], s33 offset:4094
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    global_store_dword v[0:1], v1, off
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX906-LABEL: load_private_lo_v2i16_reglo_vreg_nooff_sexti8:
+; GFX906:       ; %bb.0: ; %entry
+; GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-NEXT:    buffer_load_sbyte v0, off, s[0:3], s33 offset:4094
+; GFX906-NEXT:    v_mov_b32_e32 v2, 0xffff
+; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    v_bfi_b32 v0, v2, v0, v1
+; GFX906-NEXT:    global_store_dword v[0:1], v0, off
+; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX803-LABEL: load_private_lo_v2i16_reglo_vreg_nooff_sexti8:
+; GFX803:       ; %bb.0: ; %entry
+; GFX803-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    buffer_load_sbyte v0, off, s[0:3], s33 offset:4094
+; GFX803-NEXT:    v_and_b32_e32 v1, 0xffff0000, v1
+; GFX803-NEXT:    s_waitcnt vmcnt(0)
+; GFX803-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX803-NEXT:    flat_store_dword v[0:1], v0
+; GFX803-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %reg.bc = bitcast i32 %reg to <2 x i16>
   %load = load volatile i8, i8 addrspace(5)* inttoptr (i32 4094 to i8 addrspace(5)*)
@@ -780,16 +1581,39 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}load_private_lo_v2f16_reglo_vreg_nooff_zexti8:
-; GCN: s_waitcnt
-; GFX900-NEXT: buffer_load_ubyte_d16 v1, off, s[0:3], s33 offset:4094{{$}}
-; GFX900-NEXT: s_waitcnt
-; GFX900-NEXT: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v1
-; GFX900-NEXT: s_waitcnt
-; GFX900-NEXT: s_setpc_b64
-
-; NO-D16-HI: buffer_load_ubyte v0, off, s[0:3], s33 offset:4094{{$}}
 define void @load_private_lo_v2f16_reglo_vreg_nooff_zexti8(i8 addrspace(5)* %in, i32 %reg) #0 {
+; GFX900-LABEL: load_private_lo_v2f16_reglo_vreg_nooff_zexti8:
+; GFX900:       ; %bb.0: ; %entry
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    buffer_load_ubyte_d16 v1, off, s[0:3], s33 offset:4094
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    global_store_dword v[0:1], v1, off
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX906-LABEL: load_private_lo_v2f16_reglo_vreg_nooff_zexti8:
+; GFX906:       ; %bb.0: ; %entry
+; GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-NEXT:    buffer_load_ubyte v0, off, s[0:3], s33 offset:4094
+; GFX906-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX906-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; GFX906-NEXT:    global_store_dword v[0:1], v0, off
+; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX803-LABEL: load_private_lo_v2f16_reglo_vreg_nooff_zexti8:
+; GFX803:       ; %bb.0: ; %entry
+; GFX803-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    buffer_load_ubyte v0, off, s[0:3], s33 offset:4094
+; GFX803-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX803-NEXT:    s_mov_b32 s4, 0x5040c00
+; GFX803-NEXT:    s_waitcnt vmcnt(0)
+; GFX803-NEXT:    v_perm_b32 v0, v1, v0, s4
+; GFX803-NEXT:    flat_store_dword v[0:1], v0
+; GFX803-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %reg.bc = bitcast i32 %reg to <2 x half>
   %load = load volatile i8, i8 addrspace(5)* inttoptr (i32 4094 to i8 addrspace(5)*)
@@ -800,18 +1624,39 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}load_constant_lo_v2i16_reglo_vreg:
-; GCN: s_waitcnt
-; GFX900-NEXT: global_load_short_d16 v2, v[0:1], off offset:-4094
-; GFX900-NEXT: s_waitcnt
-; GFX900-NEXT: global_store_dword
-; GFX900-NEXT: s_waitcnt
-; GFX900-NEXT: s_setpc_b64
-
-; GFX803: flat_load_ushort
-
-; GFX906: global_load_ushort
 define void @load_constant_lo_v2i16_reglo_vreg(i16 addrspace(4)* %in, i32 %reg) #0 {
+; GFX900-LABEL: load_constant_lo_v2i16_reglo_vreg:
+; GFX900:       ; %bb.0: ; %entry
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    global_load_short_d16 v2, v[0:1], off offset:-4094
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    global_store_dword v[0:1], v2, off
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX906-LABEL: load_constant_lo_v2i16_reglo_vreg:
+; GFX906:       ; %bb.0: ; %entry
+; GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-NEXT:    global_load_ushort v0, v[0:1], off offset:-4094
+; GFX906-NEXT:    v_mov_b32_e32 v1, 0xffff
+; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    v_bfi_b32 v0, v1, v0, v2
+; GFX906-NEXT:    global_store_dword v[0:1], v0, off
+; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX803-LABEL: load_constant_lo_v2i16_reglo_vreg:
+; GFX803:       ; %bb.0: ; %entry
+; GFX803-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    v_add_u32_e32 v0, vcc, 0xfffff002, v0
+; GFX803-NEXT:    v_addc_u32_e32 v1, vcc, -1, v1, vcc
+; GFX803-NEXT:    flat_load_ushort v0, v[0:1]
+; GFX803-NEXT:    v_and_b32_e32 v1, 0xffff0000, v2
+; GFX803-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX803-NEXT:    flat_store_dword v[0:1], v0
+; GFX803-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %reg.bc = bitcast i32 %reg to <2 x i16>
   %gep = getelementptr inbounds i16, i16 addrspace(4)* %in, i64 -2047
@@ -821,18 +1666,40 @@ entry:
   ret void
 }
 
-; GCN-LABEL: load_constant_lo_v2f16_reglo_vreg
-; GCN: s_waitcnt
-; GFX900-NEXT: global_load_short_d16 v2, v[0:1], off offset:-4094
-; GFX900-NEXT: s_waitcnt
-; GFX900-NEXT: global_store_dword
-; GFX900-NEXT: s_waitcnt
-; GFX900-NEXT: s_setpc_b64
-
-; GFX803: flat_load_ushort
-
-; GFX906: global_load_ushort
 define void @load_constant_lo_v2f16_reglo_vreg(half addrspace(4)* %in, i32 %reg) #0 {
+; GFX900-LABEL: load_constant_lo_v2f16_reglo_vreg:
+; GFX900:       ; %bb.0: ; %entry
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    global_load_short_d16 v2, v[0:1], off offset:-4094
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    global_store_dword v[0:1], v2, off
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX906-LABEL: load_constant_lo_v2f16_reglo_vreg:
+; GFX906:       ; %bb.0: ; %entry
+; GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-NEXT:    global_load_ushort v0, v[0:1], off offset:-4094
+; GFX906-NEXT:    v_lshrrev_b32_e32 v1, 16, v2
+; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX906-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; GFX906-NEXT:    global_store_dword v[0:1], v0, off
+; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX803-LABEL: load_constant_lo_v2f16_reglo_vreg:
+; GFX803:       ; %bb.0: ; %entry
+; GFX803-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    v_add_u32_e32 v0, vcc, 0xfffff002, v0
+; GFX803-NEXT:    v_addc_u32_e32 v1, vcc, -1, v1, vcc
+; GFX803-NEXT:    flat_load_ushort v0, v[0:1]
+; GFX803-NEXT:    v_and_b32_e32 v1, 0xffff0000, v2
+; GFX803-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX803-NEXT:    flat_store_dword v[0:1], v0
+; GFX803-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %reg.bc = bitcast i32 %reg to <2 x half>
   %gep = getelementptr inbounds half, half addrspace(4)* %in, i64 -2047
@@ -842,20 +1709,41 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}load_constant_lo_v2f16_reglo_vreg_zexti8:
-; GCN: s_waitcnt
-; GFX900-NEXT: global_load_ubyte_d16 v2, v[0:1], off offset:-4095
-; GFX900-NEXT: s_waitcnt
-; GFX900-NEXT: global_store_dword
-; GFX900-NEXT: s_waitcnt
-; GFX900-NEXT: s_setpc_b64
-
-; GFX906: global_load_ubyte v0, v[0:1], off offset:-4095
-; GFX906: v_and_b32_e32
-; GFX906: v_lshl_or_b32
-
-; GFX803: flat_load_ubyte
 define void @load_constant_lo_v2f16_reglo_vreg_zexti8(i8 addrspace(4)* %in, i32 %reg) #0 {
+; GFX900-LABEL: load_constant_lo_v2f16_reglo_vreg_zexti8:
+; GFX900:       ; %bb.0: ; %entry
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    global_load_ubyte_d16 v2, v[0:1], off offset:-4095
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    global_store_dword v[0:1], v2, off
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX906-LABEL: load_constant_lo_v2f16_reglo_vreg_zexti8:
+; GFX906:       ; %bb.0: ; %entry
+; GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-NEXT:    global_load_ubyte v0, v[0:1], off offset:-4095
+; GFX906-NEXT:    v_lshrrev_b32_e32 v1, 16, v2
+; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX906-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; GFX906-NEXT:    global_store_dword v[0:1], v0, off
+; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX803-LABEL: load_constant_lo_v2f16_reglo_vreg_zexti8:
+; GFX803:       ; %bb.0: ; %entry
+; GFX803-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    v_add_u32_e32 v0, vcc, 0xfffff001, v0
+; GFX803-NEXT:    v_addc_u32_e32 v1, vcc, -1, v1, vcc
+; GFX803-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX803-NEXT:    v_lshrrev_b32_e32 v1, 16, v2
+; GFX803-NEXT:    s_mov_b32 s4, 0x5040c00
+; GFX803-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    v_perm_b32 v0, v1, v0, s4
+; GFX803-NEXT:    flat_store_dword v[0:1], v0
+; GFX803-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %reg.bc = bitcast i32 %reg to <2 x half>
   %gep = getelementptr inbounds i8, i8 addrspace(4)* %in, i64 -4095
@@ -867,21 +1755,40 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}load_constant_lo_v2f16_reglo_vreg_sexti8:
-; GCN: s_waitcnt
-; GFX900-NEXT: global_load_sbyte_d16 v2, v[0:1], off offset:-4095
-; GFX900-NEXT: s_waitcnt
-; GFX900-NEXT: global_store_dword
-; GFX900-NEXT: s_waitcnt
-; GFX900-NEXT: s_setpc_b64
-
-; GFX906: global_load_sbyte v0, v[0:1], off offset:-4095
-; GFX906: v_lshrrev_b32
-; GFX906: v_and_b32
-; GFX906: v_lshl_or_b32
-
-; GFX803: flat_load_sbyte
 define void @load_constant_lo_v2f16_reglo_vreg_sexti8(i8 addrspace(4)* %in, i32 %reg) #0 {
+; GFX900-LABEL: load_constant_lo_v2f16_reglo_vreg_sexti8:
+; GFX900:       ; %bb.0: ; %entry
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    global_load_sbyte_d16 v2, v[0:1], off offset:-4095
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    global_store_dword v[0:1], v2, off
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX906-LABEL: load_constant_lo_v2f16_reglo_vreg_sexti8:
+; GFX906:       ; %bb.0: ; %entry
+; GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-NEXT:    global_load_sbyte v0, v[0:1], off offset:-4095
+; GFX906-NEXT:    v_lshrrev_b32_e32 v1, 16, v2
+; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX906-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; GFX906-NEXT:    global_store_dword v[0:1], v0, off
+; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX803-LABEL: load_constant_lo_v2f16_reglo_vreg_sexti8:
+; GFX803:       ; %bb.0: ; %entry
+; GFX803-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    v_add_u32_e32 v0, vcc, 0xfffff001, v0
+; GFX803-NEXT:    v_addc_u32_e32 v1, vcc, -1, v1, vcc
+; GFX803-NEXT:    flat_load_sbyte v0, v[0:1]
+; GFX803-NEXT:    v_and_b32_e32 v1, 0xffff0000, v2
+; GFX803-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX803-NEXT:    flat_store_dword v[0:1], v0
+; GFX803-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %reg.bc = bitcast i32 %reg to <2 x half>
   %gep = getelementptr inbounds i8, i8 addrspace(4)* %in, i64 -4095
@@ -893,12 +1800,43 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}load_private_lo_v2i16_reglo_vreg_to_offset:
-; GFX900: buffer_store_dword
-; GFX900-NEXT: buffer_load_short_d16 v0, off, s[0:3], s32 offset:4094
-
-; NO-D16-HI: buffer_load_ushort v
 define void @load_private_lo_v2i16_reglo_vreg_to_offset(i32 %reg) #0 {
+; GFX900-LABEL: load_private_lo_v2i16_reglo_vreg_to_offset:
+; GFX900:       ; %bb.0: ; %entry
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    v_mov_b32_e32 v1, 0x7b
+; GFX900-NEXT:    buffer_store_dword v1, off, s[0:3], s32
+; GFX900-NEXT:    buffer_load_short_d16 v0, off, s[0:3], s32 offset:4094
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    global_store_dword v[0:1], v0, off
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX906-LABEL: load_private_lo_v2i16_reglo_vreg_to_offset:
+; GFX906:       ; %bb.0: ; %entry
+; GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-NEXT:    v_mov_b32_e32 v1, 0x7b
+; GFX906-NEXT:    buffer_store_dword v1, off, s[0:3], s32
+; GFX906-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:4094
+; GFX906-NEXT:    v_mov_b32_e32 v2, 0xffff
+; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    v_bfi_b32 v0, v2, v1, v0
+; GFX906-NEXT:    global_store_dword v[0:1], v0, off
+; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX803-LABEL: load_private_lo_v2i16_reglo_vreg_to_offset:
+; GFX803:       ; %bb.0: ; %entry
+; GFX803-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    v_mov_b32_e32 v1, 0x7b
+; GFX803-NEXT:    buffer_store_dword v1, off, s[0:3], s32
+; GFX803-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:4094
+; GFX803-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
+; GFX803-NEXT:    s_waitcnt vmcnt(0)
+; GFX803-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX803-NEXT:    flat_store_dword v[0:1], v0
+; GFX803-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %obj0 = alloca [10 x i32], align 4, addrspace(5)
   %obj1 = alloca [4096 x i16], align 2, addrspace(5)
@@ -912,12 +1850,43 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}load_private_lo_v2i16_reglo_vreg_sexti8_to_offset:
-; GFX900: buffer_store_dword
-; GFX900-NEXT: buffer_load_sbyte_d16 v0, off, s[0:3], s32 offset:4095
-
-; NO-D16-HI: buffer_load_sbyte v
 define void @load_private_lo_v2i16_reglo_vreg_sexti8_to_offset(i32 %reg) #0 {
+; GFX900-LABEL: load_private_lo_v2i16_reglo_vreg_sexti8_to_offset:
+; GFX900:       ; %bb.0: ; %entry
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    v_mov_b32_e32 v1, 0x7b
+; GFX900-NEXT:    buffer_store_dword v1, off, s[0:3], s32
+; GFX900-NEXT:    buffer_load_sbyte_d16 v0, off, s[0:3], s32 offset:4095
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    global_store_dword v[0:1], v0, off
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX906-LABEL: load_private_lo_v2i16_reglo_vreg_sexti8_to_offset:
+; GFX906:       ; %bb.0: ; %entry
+; GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-NEXT:    v_mov_b32_e32 v1, 0x7b
+; GFX906-NEXT:    buffer_store_dword v1, off, s[0:3], s32
+; GFX906-NEXT:    buffer_load_sbyte v1, off, s[0:3], s32 offset:4095
+; GFX906-NEXT:    v_mov_b32_e32 v2, 0xffff
+; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    v_bfi_b32 v0, v2, v1, v0
+; GFX906-NEXT:    global_store_dword v[0:1], v0, off
+; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX803-LABEL: load_private_lo_v2i16_reglo_vreg_sexti8_to_offset:
+; GFX803:       ; %bb.0: ; %entry
+; GFX803-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    v_mov_b32_e32 v1, 0x7b
+; GFX803-NEXT:    buffer_store_dword v1, off, s[0:3], s32
+; GFX803-NEXT:    buffer_load_sbyte v1, off, s[0:3], s32 offset:4095
+; GFX803-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
+; GFX803-NEXT:    s_waitcnt vmcnt(0)
+; GFX803-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX803-NEXT:    flat_store_dword v[0:1], v0
+; GFX803-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %obj0 = alloca [10 x i32], align 4, addrspace(5)
   %obj1 = alloca [4096 x i8], align 2, addrspace(5)
@@ -932,12 +1901,44 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}load_private_lo_v2i16_reglo_vreg_zexti8_to_offset:
-; GFX900: buffer_store_dword
-; GFX900-NEXT: buffer_load_ubyte_d16 v0, off, s[0:3], s32 offset:4095
-
-; NO-D16-HI: buffer_load_ubyte v
 define void @load_private_lo_v2i16_reglo_vreg_zexti8_to_offset(i32 %reg) #0 {
+; GFX900-LABEL: load_private_lo_v2i16_reglo_vreg_zexti8_to_offset:
+; GFX900:       ; %bb.0: ; %entry
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    v_mov_b32_e32 v1, 0x7b
+; GFX900-NEXT:    buffer_store_dword v1, off, s[0:3], s32
+; GFX900-NEXT:    buffer_load_ubyte_d16 v0, off, s[0:3], s32 offset:4095
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    global_store_dword v[0:1], v0, off
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX906-LABEL: load_private_lo_v2i16_reglo_vreg_zexti8_to_offset:
+; GFX906:       ; %bb.0: ; %entry
+; GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-NEXT:    v_mov_b32_e32 v1, 0x7b
+; GFX906-NEXT:    buffer_store_dword v1, off, s[0:3], s32
+; GFX906-NEXT:    buffer_load_ubyte v1, off, s[0:3], s32 offset:4095
+; GFX906-NEXT:    v_mov_b32_e32 v2, 0xffff
+; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    v_bfi_b32 v0, v2, v1, v0
+; GFX906-NEXT:    global_store_dword v[0:1], v0, off
+; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX803-LABEL: load_private_lo_v2i16_reglo_vreg_zexti8_to_offset:
+; GFX803:       ; %bb.0: ; %entry
+; GFX803-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    v_mov_b32_e32 v1, 0x7b
+; GFX803-NEXT:    buffer_store_dword v1, off, s[0:3], s32
+; GFX803-NEXT:    buffer_load_ubyte v1, off, s[0:3], s32 offset:4095
+; GFX803-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX803-NEXT:    s_mov_b32 s4, 0x5040c00
+; GFX803-NEXT:    s_waitcnt vmcnt(0)
+; GFX803-NEXT:    v_perm_b32 v0, v0, v1, s4
+; GFX803-NEXT:    flat_store_dword v[0:1], v0
+; GFX803-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %obj0 = alloca [10 x i32], align 4, addrspace(5)
   %obj1 = alloca [4096 x i8], align 2, addrspace(5)
@@ -952,12 +1953,44 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}load_private_lo_v2f16_reglo_vreg_sexti8_to_offset:
-; GFX900: buffer_store_dword
-; GFX900-NEXT: buffer_load_sbyte_d16 v0, off, s[0:3], s32 offset:4095
-
-; NO-D16-HI: buffer_load_sbyte v
 define void @load_private_lo_v2f16_reglo_vreg_sexti8_to_offset(i32 %reg) #0 {
+; GFX900-LABEL: load_private_lo_v2f16_reglo_vreg_sexti8_to_offset:
+; GFX900:       ; %bb.0: ; %entry
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    v_mov_b32_e32 v1, 0x7b
+; GFX900-NEXT:    buffer_store_dword v1, off, s[0:3], s32
+; GFX900-NEXT:    buffer_load_sbyte_d16 v0, off, s[0:3], s32 offset:4095
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    global_store_dword v[0:1], v0, off
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX906-LABEL: load_private_lo_v2f16_reglo_vreg_sexti8_to_offset:
+; GFX906:       ; %bb.0: ; %entry
+; GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-NEXT:    v_mov_b32_e32 v1, 0x7b
+; GFX906-NEXT:    buffer_store_dword v1, off, s[0:3], s32
+; GFX906-NEXT:    buffer_load_sbyte v1, off, s[0:3], s32 offset:4095
+; GFX906-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GFX906-NEXT:    v_lshl_or_b32 v0, v0, 16, v1
+; GFX906-NEXT:    global_store_dword v[0:1], v0, off
+; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX803-LABEL: load_private_lo_v2f16_reglo_vreg_sexti8_to_offset:
+; GFX803:       ; %bb.0: ; %entry
+; GFX803-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    v_mov_b32_e32 v1, 0x7b
+; GFX803-NEXT:    buffer_store_dword v1, off, s[0:3], s32
+; GFX803-NEXT:    buffer_load_sbyte v1, off, s[0:3], s32 offset:4095
+; GFX803-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
+; GFX803-NEXT:    s_waitcnt vmcnt(0)
+; GFX803-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX803-NEXT:    flat_store_dword v[0:1], v0
+; GFX803-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %obj0 = alloca [10 x i32], align 4, addrspace(5)
   %obj1 = alloca [4096 x i8], align 2, addrspace(5)
@@ -973,12 +2006,45 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}load_private_lo_v2f16_reglo_vreg_zexti8_to_offset:
-; GFX900: buffer_store_dword
-; GFX900-NEXT: buffer_load_ubyte_d16 v0, off, s[0:3], s32 offset:4095
-
-; NO-D16-HI: buffer_load_ubyte v
 define void @load_private_lo_v2f16_reglo_vreg_zexti8_to_offset(i32 %reg) #0 {
+; GFX900-LABEL: load_private_lo_v2f16_reglo_vreg_zexti8_to_offset:
+; GFX900:       ; %bb.0: ; %entry
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    v_mov_b32_e32 v1, 0x7b
+; GFX900-NEXT:    buffer_store_dword v1, off, s[0:3], s32
+; GFX900-NEXT:    buffer_load_ubyte_d16 v0, off, s[0:3], s32 offset:4095
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    global_store_dword v[0:1], v0, off
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX906-LABEL: load_private_lo_v2f16_reglo_vreg_zexti8_to_offset:
+; GFX906:       ; %bb.0: ; %entry
+; GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-NEXT:    v_mov_b32_e32 v1, 0x7b
+; GFX906-NEXT:    buffer_store_dword v1, off, s[0:3], s32
+; GFX906-NEXT:    buffer_load_ubyte v1, off, s[0:3], s32 offset:4095
+; GFX906-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GFX906-NEXT:    v_lshl_or_b32 v0, v0, 16, v1
+; GFX906-NEXT:    global_store_dword v[0:1], v0, off
+; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX803-LABEL: load_private_lo_v2f16_reglo_vreg_zexti8_to_offset:
+; GFX803:       ; %bb.0: ; %entry
+; GFX803-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    v_mov_b32_e32 v1, 0x7b
+; GFX803-NEXT:    buffer_store_dword v1, off, s[0:3], s32
+; GFX803-NEXT:    buffer_load_ubyte v1, off, s[0:3], s32 offset:4095
+; GFX803-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX803-NEXT:    s_mov_b32 s4, 0x5040c00
+; GFX803-NEXT:    s_waitcnt vmcnt(0)
+; GFX803-NEXT:    v_perm_b32 v0, v0, v1, s4
+; GFX803-NEXT:    flat_store_dword v[0:1], v0
+; GFX803-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX803-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %obj0 = alloca [10 x i32], align 4, addrspace(5)
   %obj1 = alloca [4096 x i8], align 2, addrspace(5)
diff --git a/llvm/test/CodeGen/AMDGPU/local-memory.amdgcn.ll b/llvm/test/CodeGen/AMDGPU/local-memory.amdgcn.ll
index a070488a4..f6611ec26 100644
--- a/llvm/test/CodeGen/AMDGPU/local-memory.amdgcn.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-memory.amdgcn.ll
@@ -38,18 +38,16 @@ entry:
 
 ; GCN-LABEL: {{^}}local_memory_two_objects:
 ; GCN: v_lshlrev_b32_e32 [[ADDRW:v[0-9]+]], 2, v0
+; CI-DAG: v_sub_i32_e32 [[SUB:v[0-9]+]], vcc, 0, [[ADDRW]]
 ; CI-DAG: ds_write2_b32 [[ADDRW]], {{v[0-9]+}}, {{v[0-9]+}} offset1:4
 ; SI-DAG: ds_write2_b32 [[ADDRW]], {{v[0-9]+}}, {{v[0-9]+}} offset1:4
-
-; GCN: s_barrier
-
 ; SI-DAG: v_sub_i32_e32 [[SUB0:v[0-9]+]], vcc, 28, [[ADDRW]]
 ; SI-DAG: v_sub_i32_e32 [[SUB1:v[0-9]+]], vcc, 12, [[ADDRW]]
 
+; GCN: s_barrier
+
 ; SI-DAG: ds_read_b32 v{{[0-9]+}}, [[SUB0]]
 ; SI-DAG: ds_read_b32 v{{[0-9]+}}, [[SUB1]]
-
-; CI: v_sub_i32_e32 [[SUB:v[0-9]+]], vcc, 0, [[ADDRW]]
 ; CI: ds_read2_b32 {{v\[[0-9]+:[0-9]+\]}}, [[SUB]] offset0:3 offset1:7
 
 define amdgpu_kernel void @local_memory_two_objects(i32 addrspace(1)* %out) #0 {
diff --git a/llvm/test/CodeGen/AMDGPU/loop_break.ll b/llvm/test/CodeGen/AMDGPU/loop_break.ll
index fd634689f..ae4578511 100644
--- a/llvm/test/CodeGen/AMDGPU/loop_break.ll
+++ b/llvm/test/CodeGen/AMDGPU/loop_break.ll
@@ -23,26 +23,26 @@
 ; OPT: call void @llvm.amdgcn.end.cf.i64(i64
 
 ; GCN-LABEL: {{^}}break_loop:
-; GCN:      s_mov_b64         [[OUTER_MASK:s\[[0-9]+:[0-9]+\]]], 0{{$}}
+; GCN:      s_mov_b64         [[ACCUM_MASK:s\[[0-9]+:[0-9]+\]]], 0{{$}}
 
 ; GCN: [[LOOP_ENTRY:BB[0-9]+_[0-9]+]]: ; %bb1
-; GCN:      v_cmp_lt_i32_e32  vcc, -1
-; GCN:      s_and_b64         vcc, exec, vcc
-; GCN:      s_or_b64          [[INNER_MASK:s\[[0-9]+:[0-9]+\]]], [[INNER_MASK]], exec
-; GCN:      s_cbranch_vccnz   [[FLOW:BB[0-9]+_[0-9]+]]
+; GCN:     s_add_i32 s6, s6, 1
+; GCN:     s_or_b64 [[INNER_MASK:s\[[0-9]+:[0-9]+\]]], [[INNER_MASK]], exec
+; GCN:     s_cmp_gt_i32 s6, -1
+; GCN:     s_cbranch_scc1   [[FLOW:BB[0-9]+_[0-9]+]]
 
 ; GCN: ; %bb4
 ; GCN:      buffer_load_dword
-; GCN:      v_cmp_ge_i32_e32  vcc,
-; GCN:      s_andn2_b64       [[INNER_MASK]], [[INNER_MASK]], exec
-; GCN:      s_and_b64         [[TMP0:s\[[0-9]+:[0-9]+\]]], vcc, exec
-; GCN:      s_or_b64          [[INNER_MASK]], [[INNER_MASK]], [[TMP0]]
+; GCN:      v_cmp_ge_i32_e32  vcc
+; GCN:      s_andn2_b64 [[INNER_MASK]], [[INNER_MASK]], exec
+; GCN:      s_and_b64 [[BROKEN_MASK:s\[[0-9]+:[0-9]+\]]], vcc, exec
+; GCN:      s_or_b64  [[INNER_MASK]], [[INNER_MASK]], [[BROKEN_MASK]]
 
 ; GCN: [[FLOW]]: ; %Flow
-; GCN:      s_and_b64         [[TMP1:s\[[0-9]+:[0-9]+\]]], exec, [[INNER_MASK]]
-; GCN:      s_or_b64          [[TMP1]], [[TMP1]], [[OUTER_MASK]]
-; GCN:      s_mov_b64         [[OUTER_MASK]], [[TMP1]]
-; GCN:      s_andn2_b64       exec, exec, [[TMP1]]
+; GCN:           ;   in Loop: Header=BB0_1 Depth=1
+; GCN:      s_and_b64         [[AND_MASK:s\[[0-9]+:[0-9]+\]]], exec, [[INNER_MASK]]
+; GCN-NEXT: s_or_b64          [[ACCUM_MASK]], [[AND_MASK]], [[ACCUM_MASK]]
+; GCN-NEXT: s_andn2_b64       exec, exec, [[ACCUM_MASK]]
 ; GCN-NEXT: s_cbranch_execnz  [[LOOP_ENTRY]]
 
 ; GCN: ; %bb.4: ; %bb9
diff --git a/llvm/test/CodeGen/AMDGPU/lower-range-metadata-intrinsic-call.ll b/llvm/test/CodeGen/AMDGPU/lower-range-metadata-intrinsic-call.ll
index 6f5f4ca13..669988d38 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-range-metadata-intrinsic-call.ll
+++ b/llvm/test/CodeGen/AMDGPU/lower-range-metadata-intrinsic-call.ll
@@ -39,7 +39,7 @@ entry:
 
 declare i32 @llvm.amdgcn.workitem.id.x() #1
 
-attributes #0 = { norecurse nounwind }
+attributes #0 = { norecurse nounwind "amdgpu-flat-work-group-size"="1,256" }
 attributes #1 = { nounwind readnone }
 
 !0 = !{i32 0, i32 1024}
diff --git a/llvm/test/CodeGen/AMDGPU/lshr.v2i16.ll b/llvm/test/CodeGen/AMDGPU/lshr.v2i16.ll
index e88550fb9..95e88ebfc 100644
--- a/llvm/test/CodeGen/AMDGPU/lshr.v2i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/lshr.v2i16.ll
@@ -1,46 +1,134 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9 %s
-; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,VI,CIVI %s
-; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CI,CIVI %s
+; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CIVI,VI %s
+; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CIVI,CI %s
 
-; GCN-LABEL: {{^}}s_lshr_v2i16:
-; GFX9: s_load_dword [[LHS:s[0-9]+]]
-; GFX9: s_load_dword [[RHS:s[0-9]+]]
-; GFX9: v_mov_b32_e32 [[VLHS:v[0-9]+]], [[LHS]]
-; GFX9: v_pk_lshrrev_b16 [[RESULT:v[0-9]+]], [[RHS]], [[VLHS]]
-
-; CIVI: s_load_dword [[LHS:s[0-9]+]]
-; CIVI: s_load_dword [[RHS:s[0-9]+]]
-; CIVI: s_lshr_b32 s{{[0-9]+}}, s{{[0-9]+}}, 16
-; CIVI: s_lshr_b32 s{{[0-9]+}}, s{{[0-9]+}}, 16
-; CIVI: s_lshr_b32 s{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}}
-; CIVI-DAG: v_bfe_u32 v{{[0-9]+}}, s{{[0-9]+}}, v{{[0-9]+}}, 16
-; CIVI-DAG: s_lshl_b32
-; CIVI: v_or_b32_e32
 define amdgpu_kernel void @s_lshr_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> %lhs, <2 x i16> %rhs) #0 {
+; GFX9-LABEL: s_lshr_v2i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
+; GFX9-NEXT:    s_load_dword s4, s[0:1], 0x2c
+; GFX9-NEXT:    s_load_dword s0, s[0:1], 0x30
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, s2
+; GFX9-NEXT:    v_mov_b32_e32 v2, s4
+; GFX9-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NEXT:    v_pk_lshrrev_b16 v2, s0, v2
+; GFX9-NEXT:    global_store_dword v[0:1], v2, off
+; GFX9-NEXT:    s_endpgm
+;
+; VI-LABEL: s_lshr_v2i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
+; VI-NEXT:    s_load_dword s5, s[0:1], 0x2c
+; VI-NEXT:    s_load_dword s0, s[0:1], 0x30
+; VI-NEXT:    s_mov_b32 s4, 0xffff
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v0, s2
+; VI-NEXT:    s_lshr_b32 s1, s5, 16
+; VI-NEXT:    s_lshr_b32 s6, s0, 16
+; VI-NEXT:    s_lshr_b32 s1, s1, s6
+; VI-NEXT:    s_and_b32 s5, s5, s4
+; VI-NEXT:    s_and_b32 s0, s0, s4
+; VI-NEXT:    s_lshr_b32 s0, s5, s0
+; VI-NEXT:    s_lshl_b32 s1, s1, 16
+; VI-NEXT:    s_or_b32 s0, s0, s1
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_mov_b32_e32 v2, s0
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; CI-LABEL: s_lshr_v2i16:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; CI-NEXT:    s_load_dword s2, s[0:1], 0xb
+; CI-NEXT:    s_load_dword s0, s[0:1], 0xc
+; CI-NEXT:    s_mov_b32 s3, 0xffff
+; CI-NEXT:    s_mov_b32 s7, 0xf000
+; CI-NEXT:    s_mov_b32 s6, -1
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    s_lshr_b32 s1, s2, 16
+; CI-NEXT:    s_lshr_b32 s8, s0, 16
+; CI-NEXT:    s_lshr_b32 s1, s1, s8
+; CI-NEXT:    s_and_b32 s2, s2, s3
+; CI-NEXT:    s_and_b32 s0, s0, s3
+; CI-NEXT:    s_lshr_b32 s0, s2, s0
+; CI-NEXT:    s_lshl_b32 s1, s1, 16
+; CI-NEXT:    s_or_b32 s0, s0, s1
+; CI-NEXT:    v_mov_b32_e32 v0, s0
+; CI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; CI-NEXT:    s_endpgm
   %result = lshr <2 x i16> %lhs, %rhs
   store <2 x i16> %result, <2 x i16> addrspace(1)* %out
   ret void
 }
 
-; GCN-LABEL: {{^}}v_lshr_v2i16:
-; GCN: {{buffer|flat|global}}_load_dword [[LHS:v[0-9]+]]
-; GCN: {{buffer|flat|global}}_load_dword [[RHS:v[0-9]+]]
-; GFX9: v_pk_lshrrev_b16 [[RESULT:v[0-9]+]], [[RHS]], [[LHS]]
-
-; VI: v_lshrrev_b16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; VI: v_lshrrev_b16_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; VI: v_or_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-
-; CI: s_mov_b32 [[MASK:s[0-9]+]], 0xffff{{$}}
-; CI-DAG: v_lshrrev_b32_e32 v{{[0-9]+}}, 16, [[LHS]]
-; CI-DAG: v_lshrrev_b32_e32 v{{[0-9]+}}, 16, [[RHS]]
-; CI: v_and_b32_e32 v{{[0-9]+}}, [[MASK]], v{{[0-9]+}}
-; CI: v_and_b32_e32 v{{[0-9]+}}, [[MASK]], v{{[0-9]+}}
-; CI: v_bfe_u32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}, 16
-; CI: v_lshrrev_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; CI: v_lshlrev_b32_e32 v{{[0-9]+}}, 16, v{{[0-9]+}}
-; CI: v_or_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 define amdgpu_kernel void @v_lshr_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %in) #0 {
+; GFX9-LABEL: v_lshr_v2i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s2, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    global_load_dword v3, v[0:1], off
+; GFX9-NEXT:    global_load_dword v4, v[0:1], off offset:4
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s0, v2
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_pk_lshrrev_b16 v2, v4, v3
+; GFX9-NEXT:    global_store_dword v[0:1], v2, off
+; GFX9-NEXT:    s_endpgm
+;
+; VI-LABEL: v_lshr_v2i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-NEXT:    v_lshlrev_b32_e32 v4, 2, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v4
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 4, v0
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dword v5, v[0:1]
+; VI-NEXT:    flat_load_dword v2, v[2:3]
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v4
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_lshrrev_b16_e32 v3, v2, v5
+; VI-NEXT:    v_lshrrev_b16_sdwa v2, v2, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; VI-NEXT:    v_or_b32_e32 v2, v3, v2
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; CI-LABEL: v_lshr_v2i16:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; CI-NEXT:    s_mov_b32 s7, 0xf000
+; CI-NEXT:    s_mov_b32 s6, 0
+; CI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; CI-NEXT:    v_mov_b32_e32 v1, 0
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; CI-NEXT:    buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
+; CI-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:4
+; CI-NEXT:    s_mov_b32 s8, 0xffff
+; CI-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; CI-NEXT:    s_waitcnt vmcnt(1)
+; CI-NEXT:    v_lshrrev_b32_e32 v4, 16, v2
+; CI-NEXT:    s_waitcnt vmcnt(0)
+; CI-NEXT:    v_lshrrev_b32_e32 v5, 16, v3
+; CI-NEXT:    v_and_b32_e32 v2, s8, v2
+; CI-NEXT:    v_and_b32_e32 v3, s8, v3
+; CI-NEXT:    v_lshrrev_b32_e32 v2, v3, v2
+; CI-NEXT:    v_lshrrev_b32_e32 v3, v5, v4
+; CI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; CI-NEXT:    v_or_b32_e32 v2, v2, v3
+; CI-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; CI-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
   %in.gep = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %in, i64 %tid.ext
@@ -53,11 +141,71 @@ define amdgpu_kernel void @v_lshr_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16>
   ret void
 }
 
-; GCN-LABEL: {{^}}lshr_v_s_v2i16:
-; GFX9: s_load_dword [[RHS:s[0-9]+]]
-; GFX9: {{buffer|flat|global}}_load_dword [[LHS:v[0-9]+]]
-; GFX9: v_pk_lshrrev_b16 [[RESULT:v[0-9]+]], [[RHS]], [[LHS]]
 define amdgpu_kernel void @lshr_v_s_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %in, <2 x i16> %sgpr) #0 {
+; GFX9-LABEL: lshr_v_s_v2i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX9-NEXT:    s_load_dword s0, s[0:1], 0x34
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v1, s7
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s6, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    global_load_dword v3, v[0:1], off
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s4, v2
+; GFX9-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_pk_lshrrev_b16 v2, s0, v3
+; GFX9-NEXT:    global_store_dword v[0:1], v2, off
+; GFX9-NEXT:    s_endpgm
+;
+; VI-LABEL: lshr_v_s_v2i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; VI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; VI-NEXT:    s_load_dword s0, s[0:1], 0x34
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s7
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s6, v2
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dword v3, v[0:1]
+; VI-NEXT:    s_lshr_b32 s1, s0, 16
+; VI-NEXT:    v_mov_b32_e32 v4, s1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s4, v2
+; VI-NEXT:    v_mov_b32_e32 v1, s5
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_lshrrev_b16_e32 v2, s0, v3
+; VI-NEXT:    v_lshrrev_b16_sdwa v3, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_or_b32_e32 v2, v2, v3
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; CI-LABEL: lshr_v_s_v2i16:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; CI-NEXT:    s_load_dword s8, s[0:1], 0xd
+; CI-NEXT:    s_mov_b32 s3, 0xf000
+; CI-NEXT:    s_mov_b32 s2, 0
+; CI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    s_mov_b64 s[0:1], s[6:7]
+; CI-NEXT:    v_mov_b32_e32 v1, 0
+; CI-NEXT:    buffer_load_dword v2, v[0:1], s[0:3], 0 addr64
+; CI-NEXT:    s_lshr_b32 s9, s8, 16
+; CI-NEXT:    s_mov_b32 s10, 0xffff
+; CI-NEXT:    s_and_b32 s8, s8, s10
+; CI-NEXT:    s_mov_b64 s[6:7], s[2:3]
+; CI-NEXT:    s_waitcnt vmcnt(0)
+; CI-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
+; CI-NEXT:    v_and_b32_e32 v2, s10, v2
+; CI-NEXT:    v_lshrrev_b32_e32 v3, s9, v3
+; CI-NEXT:    v_lshrrev_b32_e32 v2, s8, v2
+; CI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; CI-NEXT:    v_or_b32_e32 v2, v2, v3
+; CI-NEXT:    buffer_store_dword v2, v[0:1], s[4:7], 0 addr64
+; CI-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
   %in.gep = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %in, i64 %tid.ext
@@ -68,11 +216,71 @@ define amdgpu_kernel void @lshr_v_s_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16
   ret void
 }
 
-; GCN-LABEL: {{^}}lshr_s_v_v2i16:
-; GFX9: s_load_dword [[LHS:s[0-9]+]]
-; GFX9: {{buffer|flat|global}}_load_dword [[RHS:v[0-9]+]]
-; GFX9: v_pk_lshrrev_b16 [[RESULT:v[0-9]+]], [[RHS]], [[LHS]]
 define amdgpu_kernel void @lshr_s_v_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %in, <2 x i16> %sgpr) #0 {
+; GFX9-LABEL: lshr_s_v_v2i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX9-NEXT:    s_load_dword s0, s[0:1], 0x34
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v1, s7
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s6, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    global_load_dword v3, v[0:1], off
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s4, v2
+; GFX9-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_pk_lshrrev_b16 v2, v3, s0
+; GFX9-NEXT:    global_store_dword v[0:1], v2, off
+; GFX9-NEXT:    s_endpgm
+;
+; VI-LABEL: lshr_s_v_v2i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; VI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; VI-NEXT:    s_load_dword s0, s[0:1], 0x34
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s7
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s6, v2
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dword v3, v[0:1]
+; VI-NEXT:    s_lshr_b32 s1, s0, 16
+; VI-NEXT:    v_mov_b32_e32 v4, s1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s4, v2
+; VI-NEXT:    v_mov_b32_e32 v1, s5
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_lshrrev_b16_e64 v2, v3, s0
+; VI-NEXT:    v_lshrrev_b16_sdwa v3, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v2, v2, v3
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; CI-LABEL: lshr_s_v_v2i16:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; CI-NEXT:    s_load_dword s8, s[0:1], 0xd
+; CI-NEXT:    s_mov_b32 s3, 0xf000
+; CI-NEXT:    s_mov_b32 s2, 0
+; CI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    s_mov_b64 s[0:1], s[6:7]
+; CI-NEXT:    v_mov_b32_e32 v1, 0
+; CI-NEXT:    buffer_load_dword v2, v[0:1], s[0:3], 0 addr64
+; CI-NEXT:    s_lshr_b32 s9, s8, 16
+; CI-NEXT:    s_mov_b32 s10, 0xffff
+; CI-NEXT:    s_and_b32 s8, s8, s10
+; CI-NEXT:    s_mov_b64 s[6:7], s[2:3]
+; CI-NEXT:    s_waitcnt vmcnt(0)
+; CI-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
+; CI-NEXT:    v_and_b32_e32 v2, s10, v2
+; CI-NEXT:    v_lshr_b32_e32 v3, s9, v3
+; CI-NEXT:    v_lshr_b32_e32 v2, s8, v2
+; CI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; CI-NEXT:    v_or_b32_e32 v2, v2, v3
+; CI-NEXT:    buffer_store_dword v2, v[0:1], s[4:7], 0 addr64
+; CI-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
   %in.gep = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %in, i64 %tid.ext
@@ -83,10 +291,64 @@ define amdgpu_kernel void @lshr_s_v_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16
   ret void
 }
 
-; GCN-LABEL: {{^}}lshr_imm_v_v2i16:
-; GCN: {{buffer|flat|global}}_load_dword [[RHS:v[0-9]+]]
-; GFX9: v_pk_lshrrev_b16 [[RESULT:v[0-9]+]], [[RHS]], 8
 define amdgpu_kernel void @lshr_imm_v_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %in) #0 {
+; GFX9-LABEL: lshr_imm_v_v2i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s2, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    global_load_dword v3, v[0:1], off
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s0, v2
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_pk_lshrrev_b16 v2, v3, 8 op_sel_hi:[1,0]
+; GFX9-NEXT:    global_store_dword v[0:1], v2, off
+; GFX9-NEXT:    s_endpgm
+;
+; VI-LABEL: lshr_imm_v_v2i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; VI-NEXT:    v_mov_b32_e32 v3, 8
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dword v4, v[0:1]
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v2
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_lshrrev_b16_e64 v2, v4, 8
+; VI-NEXT:    v_lshrrev_b16_sdwa v3, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v2, v2, v3
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; CI-LABEL: lshr_imm_v_v2i16:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; CI-NEXT:    s_mov_b32 s7, 0xf000
+; CI-NEXT:    s_mov_b32 s6, 0
+; CI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; CI-NEXT:    v_mov_b32_e32 v1, 0
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; CI-NEXT:    buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
+; CI-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; CI-NEXT:    s_waitcnt vmcnt(0)
+; CI-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
+; CI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; CI-NEXT:    v_lshr_b32_e32 v3, 8, v3
+; CI-NEXT:    v_lshr_b32_e32 v2, 8, v2
+; CI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; CI-NEXT:    v_or_b32_e32 v2, v2, v3
+; CI-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; CI-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
   %in.gep = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %in, i64 %tid.ext
@@ -97,10 +359,59 @@ define amdgpu_kernel void @lshr_imm_v_v2i16(<2 x i16> addrspace(1)* %out, <2 x i
   ret void
 }
 
-; GCN-LABEL: {{^}}lshr_v_imm_v2i16:
-; GCN: {{buffer|flat|global}}_load_dword [[LHS:v[0-9]+]]
-; GFX9: v_pk_lshrrev_b16 [[RESULT:v[0-9]+]], 8, [[LHS]]
 define amdgpu_kernel void @lshr_v_imm_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %in) #0 {
+; GFX9-LABEL: lshr_v_imm_v2i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s2, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    global_load_dword v3, v[0:1], off
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s0, v2
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_pk_lshrrev_b16 v2, 8, v3 op_sel_hi:[0,1]
+; GFX9-NEXT:    global_store_dword v[0:1], v2, off
+; GFX9-NEXT:    s_endpgm
+;
+; VI-LABEL: lshr_v_imm_v2i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dword v3, v[0:1]
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v2
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_lshrrev_b32_e32 v2, 24, v3
+; VI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; VI-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; CI-LABEL: lshr_v_imm_v2i16:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; CI-NEXT:    s_mov_b32 s7, 0xf000
+; CI-NEXT:    s_mov_b32 s6, 0
+; CI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; CI-NEXT:    v_mov_b32_e32 v1, 0
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; CI-NEXT:    buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
+; CI-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; CI-NEXT:    s_waitcnt vmcnt(0)
+; CI-NEXT:    v_lshrrev_b32_e32 v2, 8, v2
+; CI-NEXT:    v_and_b32_e32 v2, 0xff00ff, v2
+; CI-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; CI-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
   %in.gep = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %in, i64 %tid.ext
@@ -111,13 +422,84 @@ define amdgpu_kernel void @lshr_v_imm_v2i16(<2 x i16> addrspace(1)* %out, <2 x i
   ret void
 }
 
-; GCN-LABEL: {{^}}v_lshr_v4i16:
-; GCN: {{buffer|flat|global}}_load_dwordx2
-; GCN: {{buffer|flat|global}}_load_dwordx2
-; GFX9: v_pk_lshrrev_b16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; GFX9: v_pk_lshrrev_b16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; GCN: {{buffer|flat|global}}_store_dwordx2
 define amdgpu_kernel void @v_lshr_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> addrspace(1)* %in) #0 {
+; GFX9-LABEL: v_lshr_v4i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-NEXT:    v_lshlrev_b32_e32 v4, 3, v0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s2, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    global_load_dwordx2 v[2:3], v[0:1], off
+; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off offset:8
+; GFX9-NEXT:    v_mov_b32_e32 v5, s1
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, s0, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_pk_lshrrev_b16 v1, v1, v3
+; GFX9-NEXT:    v_pk_lshrrev_b16 v0, v0, v2
+; GFX9-NEXT:    global_store_dwordx2 v[4:5], v[0:1], off
+; GFX9-NEXT:    s_endpgm
+;
+; VI-LABEL: v_lshr_v4i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-NEXT:    v_lshlrev_b32_e32 v4, 3, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v4
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 8, v0
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; VI-NEXT:    flat_load_dwordx2 v[2:3], v[2:3]
+; VI-NEXT:    v_mov_b32_e32 v5, s1
+; VI-NEXT:    v_add_u32_e32 v4, vcc, s0, v4
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_lshrrev_b16_e32 v6, v3, v1
+; VI-NEXT:    v_lshrrev_b16_sdwa v1, v3, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; VI-NEXT:    v_lshrrev_b16_e32 v3, v2, v0
+; VI-NEXT:    v_lshrrev_b16_sdwa v0, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; VI-NEXT:    v_or_b32_e32 v1, v6, v1
+; VI-NEXT:    v_or_b32_e32 v0, v3, v0
+; VI-NEXT:    flat_store_dwordx2 v[4:5], v[0:1]
+; VI-NEXT:    s_endpgm
+;
+; CI-LABEL: v_lshr_v4i16:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; CI-NEXT:    s_mov_b32 s7, 0xf000
+; CI-NEXT:    s_mov_b32 s6, 0
+; CI-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
+; CI-NEXT:    v_mov_b32_e32 v1, 0
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; CI-NEXT:    buffer_load_dwordx2 v[2:3], v[0:1], s[4:7], 0 addr64
+; CI-NEXT:    buffer_load_dwordx2 v[4:5], v[0:1], s[4:7], 0 addr64 offset:8
+; CI-NEXT:    s_mov_b32 s8, 0xffff
+; CI-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; CI-NEXT:    s_waitcnt vmcnt(1)
+; CI-NEXT:    v_lshrrev_b32_e32 v6, 16, v2
+; CI-NEXT:    v_lshrrev_b32_e32 v7, 16, v3
+; CI-NEXT:    s_waitcnt vmcnt(0)
+; CI-NEXT:    v_lshrrev_b32_e32 v8, 16, v4
+; CI-NEXT:    v_lshrrev_b32_e32 v9, 16, v5
+; CI-NEXT:    v_and_b32_e32 v2, s8, v2
+; CI-NEXT:    v_and_b32_e32 v4, s8, v4
+; CI-NEXT:    v_and_b32_e32 v3, s8, v3
+; CI-NEXT:    v_and_b32_e32 v5, s8, v5
+; CI-NEXT:    v_lshrrev_b32_e32 v3, v5, v3
+; CI-NEXT:    v_lshrrev_b32_e32 v5, v9, v7
+; CI-NEXT:    v_lshrrev_b32_e32 v2, v4, v2
+; CI-NEXT:    v_lshrrev_b32_e32 v4, v8, v6
+; CI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; CI-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; CI-NEXT:    v_or_b32_e32 v3, v3, v5
+; CI-NEXT:    v_or_b32_e32 v2, v2, v4
+; CI-NEXT:    buffer_store_dwordx2 v[2:3], v[0:1], s[0:3], 0 addr64
+; CI-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
   %in.gep = getelementptr inbounds <4 x i16>, <4 x i16> addrspace(1)* %in, i64 %tid.ext
@@ -130,12 +512,66 @@ define amdgpu_kernel void @v_lshr_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16>
   ret void
 }
 
-; GCN-LABEL: {{^}}lshr_v_imm_v4i16:
-; GCN: {{buffer|flat|global}}_load_dwordx2
-; GFX9: v_pk_lshrrev_b16 v{{[0-9]+}}, 8, v{{[0-9]+}}
-; GFX9: v_pk_lshrrev_b16 v{{[0-9]+}}, 8, v{{[0-9]+}}
-; GCN: {{buffer|flat|global}}_store_dwordx2
 define amdgpu_kernel void @lshr_v_imm_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> addrspace(1)* %in) #0 {
+; GFX9-LABEL: lshr_v_imm_v4i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s2, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off
+; GFX9-NEXT:    v_mov_b32_e32 v3, s1
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, s0, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_pk_lshrrev_b16 v1, 8, v1 op_sel_hi:[0,1]
+; GFX9-NEXT:    v_pk_lshrrev_b16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX9-NEXT:    global_store_dwordx2 v[2:3], v[0:1], off
+; GFX9-NEXT:    s_endpgm
+;
+; VI-LABEL: lshr_v_imm_v4i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    v_add_u32_e32 v2, vcc, s0, v2
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_lshrrev_b32_e32 v4, 24, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v5, 24, v0
+; VI-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; VI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT:    s_endpgm
+;
+; CI-LABEL: lshr_v_imm_v4i16:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; CI-NEXT:    s_mov_b32 s7, 0xf000
+; CI-NEXT:    s_mov_b32 s6, 0
+; CI-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
+; CI-NEXT:    v_mov_b32_e32 v1, 0
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; CI-NEXT:    buffer_load_dwordx2 v[2:3], v[0:1], s[4:7], 0 addr64
+; CI-NEXT:    s_mov_b32 s8, 0xff00ff
+; CI-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; CI-NEXT:    s_waitcnt vmcnt(0)
+; CI-NEXT:    v_lshrrev_b32_e32 v3, 8, v3
+; CI-NEXT:    v_lshrrev_b32_e32 v2, 8, v2
+; CI-NEXT:    v_and_b32_e32 v3, s8, v3
+; CI-NEXT:    v_and_b32_e32 v2, s8, v2
+; CI-NEXT:    buffer_store_dwordx2 v[2:3], v[0:1], s[0:3], 0 addr64
+; CI-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
   %in.gep = getelementptr inbounds <4 x i16>, <4 x i16> addrspace(1)* %in, i64 %tid.ext
diff --git a/llvm/test/CodeGen/AMDGPU/mad_int24.ll b/llvm/test/CodeGen/AMDGPU/mad_int24.ll
index af0159aa9..7ac2ff4be 100644
--- a/llvm/test/CodeGen/AMDGPU/mad_int24.ll
+++ b/llvm/test/CodeGen/AMDGPU/mad_int24.ll
@@ -1,5 +1,5 @@
-; RUN: llc < %s -march=amdgcn -verify-machineinstrs | FileCheck %s --check-prefix=SI --check-prefix=FUNC
-; RUN: llc < %s -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs | FileCheck %s --check-prefix=SI --check-prefix=FUNC
+; RUN: llc < %s -march=amdgcn -verify-machineinstrs | FileCheck %s --check-prefix=GCN --check-prefix=FUNC
+; RUN: llc < %s -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs | FileCheck %s --check-prefix=GCN --check-prefix=FUNC
 ; RUN: llc < %s -march=r600 -mcpu=redwood | FileCheck %s --check-prefix=EG --check-prefix=FUNC
 ; RUN: llc < %s -march=r600 -mcpu=cayman | FileCheck %s --check-prefix=CM --check-prefix=FUNC
 
@@ -9,8 +9,8 @@
 ; Make sure we aren't masking the inputs.
 ; CM-NOT: AND
 ; CM: MULADD_INT24
-; SI-NOT: and
-; SI: v_mad_i32_i24
+; GCN-NOT: and
+; GCN: v_mad_i32_i24
 define amdgpu_kernel void @i32_mad24(i32 addrspace(1)* %out, i32 %a, i32 %b, i32 %c) {
 entry:
   %0 = shl i32 %a, 8
@@ -22,3 +22,100 @@ entry:
   store i32 %3, i32 addrspace(1)* %out
   ret void
 }
+
+; GCN-LABEL: {{^}}mad24_known_bits_destroyed:
+; GCN: s_waitcnt
+; GCN-NEXT: v_mad_i32_i24
+; GCN-NEXT: v_mul_i32_i24
+; GCN-NEXT: s_setpc_b64
+define i32 @mad24_known_bits_destroyed(i32 %a, i32 %b, i32 %c) {
+
+  %shl.0 = shl i32 %a, 8
+  %sra.0 = ashr i32 %shl.0, 8
+  %shl.1 = shl i32 %b, 8
+  %sra.1 = ashr i32 %shl.1, 8
+
+  %mul0 = mul nsw i32 %sra.0, %sra.1
+  %add0 = add nsw i32 %mul0, %c
+
+  %shl.2 = shl i32 %add0, 8
+  %sra.2 = ashr i32 %shl.2, 8
+
+  %shl.3 = shl i32 %sra.0, 8
+  %sra.3 = ashr i32 %shl.3, 8
+
+  %mul1 = mul nsw i32 %sra.2, %sra.3
+  ret i32 %mul1
+}
+
+; GCN-LABEL: {{^}}mad24_intrin_known_bits_destroyed:
+; GCN: s_waitcnt
+; GCN-NEXT: v_mad_i32_i24
+; GCN-NEXT: v_mul_i32_i24
+; GCN-NEXT: s_setpc_b64
+define i32 @mad24_intrin_known_bits_destroyed(i32 %a, i32 %b, i32 %c) {
+  %shl.0 = shl i32 %a, 8
+  %sra.0 = ashr i32 %shl.0, 8
+  %shl.1 = shl i32 %b, 8
+  %sra.1 = ashr i32 %shl.1, 8
+
+  %mul0 = call i32 @llvm.amdgcn.mul.i24(i32 %sra.0, i32 %sra.1)
+  %add0 = add nsw i32 %mul0, %c
+
+  %shl.2 = shl i32 %add0, 8
+  %sra.2 = ashr i32 %shl.2, 8
+
+  %shl.3 = shl i32 %sra.0, 8
+  %sra.3 = ashr i32 %shl.3, 8
+
+  %mul1 = mul nsw i32 %sra.2, %sra.3
+  ret i32 %mul1
+}
+
+; Make sure no unnecessary BFEs are emitted in the loop.
+; GCN-LABEL: {{^}}mad24_destroyed_knownbits_2:
+; GCN-NOT: v_bfe
+; GCN: v_mad_i32_i24
+; GCN-NOT: v_bfe
+; GCN: v_mad_i32_i24
+; GCN-NOT: v_bfe
+; GCN: v_mad_i32_i24
+; GCN-NOT: v_bfe
+; GCN: v_mad_i32_i24
+; GCN-NOT: v_bfe
+define void @mad24_destroyed_knownbits_2(i32 %arg, i32 %arg1, i32 %arg2, i32 addrspace(1)* %arg3) {
+bb:
+  br label %bb6
+
+bb5:                                              ; preds = %bb6
+  ret void
+
+bb6:                                              ; preds = %bb6, %bb
+  %tmp = phi i32 [ %tmp27, %bb6 ], [ 0, %bb ]
+  %tmp7 = phi i32 [ %arg2, %bb6 ], [ 1, %bb ]
+  %tmp8 = phi i32 [ %tmp26, %bb6 ], [ %arg, %bb ]
+  %tmp9 = shl i32 %tmp7, 8
+  %tmp10 = ashr exact i32 %tmp9, 8
+  %tmp11 = shl i32 %tmp8, 8
+  %tmp12 = ashr exact i32 %tmp11, 8
+  %tmp13 = mul nsw i32 %tmp12, %tmp10
+  %tmp14 = add nsw i32 %tmp13, %tmp7
+  %tmp15 = shl i32 %tmp14, 8
+  %tmp16 = ashr exact i32 %tmp15, 8
+  %tmp17 = mul nsw i32 %tmp16, %tmp10
+  %tmp18 = add nsw i32 %tmp17, %tmp14
+  %tmp19 = shl i32 %tmp18, 8
+  %tmp20 = ashr exact i32 %tmp19, 8
+  %tmp21 = mul nsw i32 %tmp20, %tmp16
+  %tmp22 = add nsw i32 %tmp21, %tmp18
+  %tmp23 = shl i32 %tmp22, 8
+  %tmp24 = ashr exact i32 %tmp23, 8
+  %tmp25 = mul nsw i32 %tmp24, %tmp20
+  %tmp26 = add nsw i32 %tmp25, %tmp22
+  store i32 %tmp26, i32 addrspace(1)* %arg3
+  %tmp27 = add nuw i32 %tmp, 1
+  %tmp28 = icmp eq i32 %tmp27, %arg1
+  br i1 %tmp28, label %bb5, label %bb6
+}
+
+declare i32 @llvm.amdgcn.mul.i24(i32, i32)
diff --git a/llvm/test/CodeGen/AMDGPU/madak.ll b/llvm/test/CodeGen/AMDGPU/madak.ll
index eed021876..fe710f7c7 100644
--- a/llvm/test/CodeGen/AMDGPU/madak.ll
+++ b/llvm/test/CodeGen/AMDGPU/madak.ll
@@ -1,7 +1,7 @@
 ; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX6,GFX6_8_9,MAD %s
 ; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX8,GFX6_8_9,GFX8_9,GFX8_9_10,MAD %s
 ; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs  -amdgpu-enable-global-sgpr-addr < %s | FileCheck -check-prefixes=GCN,GFX9,GFX6_8_9,GFX8_9,GFX8_9_10,MAD %s
-; RUN: llc -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs -amdgpu-enable-global-sgpr-addr < %s | FileCheck -check-prefixes=GCN,GFX10,GFX8_9_10,MAD,GFX10-MAD %s
+; RUN: llc -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs -amdgpu-enable-global-sgpr-addr < %s | FileCheck -check-prefixes=GCN,GFX10,GFX8_9_10,GFX10-MAD %s
 ; RUN: llc -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs -fp-contract=fast -amdgpu-enable-global-sgpr-addr < %s | FileCheck -check-prefixes=GCN,GFX10,GFX8_9_10,FMA %s
 
 declare i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
@@ -10,13 +10,14 @@ declare float @llvm.fabs.f32(float) nounwind readnone
 ; GCN-LABEL: {{^}}madak_f32:
 ; GFX6:   buffer_load_dword [[VA:v[0-9]+]]
 ; GFX6:   buffer_load_dword [[VB:v[0-9]+]]
-; GFX8: {{flat|global}}_load_dword [[VB:v[0-9]+]]
 ; GFX8: {{flat|global}}_load_dword [[VA:v[0-9]+]]
+; GFX8: {{flat|global}}_load_dword [[VB:v[0-9]+]]
 ; GFX9: {{flat|global}}_load_dword [[VA:v[0-9]+]]
 ; GFX9: {{flat|global}}_load_dword [[VB:v[0-9]+]]
 ; GFX10: {{flat|global}}_load_dword [[VA:v[0-9]+]]
 ; GFX10: {{flat|global}}_load_dword [[VB:v[0-9]+]]
 ; MAD:   v_madak_f32 {{v[0-9]+}}, [[VA]], [[VB]], 0x41200000
+; GFX10-MAD:   v_madak_f32 {{v[0-9]+}}, [[VA]], [[VB]], 0x41200000
 ; FMA:   v_fmaak_f32 {{v[0-9]+}}, [[VA]], [[VB]], 0x41200000
 define amdgpu_kernel void @madak_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in.a, float addrspace(1)* noalias %in.b) nounwind {
   %tid = tail call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
@@ -79,6 +80,7 @@ define amdgpu_kernel void @madak_2_use_f32(float addrspace(1)* noalias %out, flo
 ; GCN-LABEL: {{^}}madak_m_inline_imm_f32:
 ; GCN: {{buffer|flat|global}}_load_dword [[VA:v[0-9]+]]
 ; MAD: v_madak_f32 {{v[0-9]+}}, 4.0, [[VA]], 0x41200000
+; GFX10-MAD: v_madak_f32 {{v[0-9]+}}, 4.0, [[VA]], 0x41200000
 ; FMA: v_fmaak_f32 {{v[0-9]+}}, 4.0, [[VA]], 0x41200000
 define amdgpu_kernel void @madak_m_inline_imm_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in.a) nounwind {
   %tid = tail call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
@@ -99,13 +101,14 @@ define amdgpu_kernel void @madak_m_inline_imm_f32(float addrspace(1)* noalias %o
 ; GCN-LABEL: {{^}}madak_inline_imm_f32:
 ; GFX6:   buffer_load_dword [[VA:v[0-9]+]]
 ; GFX6:   buffer_load_dword [[VB:v[0-9]+]]
-; GFX8: {{flat|global}}_load_dword [[VB:v[0-9]+]]
 ; GFX8: {{flat|global}}_load_dword [[VA:v[0-9]+]]
+; GFX8: {{flat|global}}_load_dword [[VB:v[0-9]+]]
 ; GFX9: {{flat|global}}_load_dword [[VA:v[0-9]+]]
 ; GFX9: {{flat|global}}_load_dword [[VB:v[0-9]+]]
 ; GFX10: {{flat|global}}_load_dword [[VA:v[0-9]+]]
 ; GFX10: {{flat|global}}_load_dword [[VB:v[0-9]+]]
 ; MAD:   v_mad_f32 {{v[0-9]+}}, [[VA]], [[VB]], 4.0
+; GFX10-MAD:   v_mad_f32 {{v[0-9]+}}, [[VA]], [[VB]], 4.0
 ; FMA:   v_fma_f32 {{v[0-9]+}}, [[VA]], [[VB]], 4.0
 define amdgpu_kernel void @madak_inline_imm_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in.a, float addrspace(1)* noalias %in.b) nounwind {
   %tid = tail call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
@@ -234,9 +237,12 @@ define amdgpu_kernel void @no_madak_src1_modifier_f32(float addrspace(1)* noalia
 ; On GFX10+ we can use two scalar operands.
 ; GCN-LABEL: {{^}}madak_constant_bus_violation:
 ; GCN:       s_load_dword [[SGPR0:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, {{0x12|0x48}}
-; GCN:       v_mov_b32_e32 [[SGPR0_VCOPY:v[0-9]+]], [[SGPR0]]
+
 ; GCN:       {{buffer|flat|global}}_load_dword [[VGPR:v[0-9]+]]
-; MAD:       v_madak_f32 [[MADAK:v[0-9]+]], 0.5, [[SGPR0_VCOPY]], 0x42280000
+; MAD:       v_mov_b32_e32 [[MADAK:v[0-9]+]], 0x42280000
+; MAD:       v_mac_f32_e64 [[MADAK]], [[SGPR0]], 0.5
+; GFX10:     v_mov_b32_e32 [[SGPR0_VCOPY:v[0-9]+]], [[SGPR0]]
+; GFX10-MAD: v_madak_f32 [[MADAK:v[0-9]+]], 0.5, [[SGPR0_VCOPY]], 0x42280000
 ; FMA:       v_fmaak_f32 [[MADAK:v[0-9]+]], 0.5, [[SGPR0_VCOPY]], 0x42280000
 ; GCN:       v_mul_f32_e32 [[MUL:v[0-9]+]], [[MADAK]], [[VGPR]]
 ; GFX6:      buffer_store_dword [[MUL]]
diff --git a/llvm/test/CodeGen/AMDGPU/max.i16.ll b/llvm/test/CodeGen/AMDGPU/max.i16.ll
index 371831f28..3a87b5859 100644
--- a/llvm/test/CodeGen/AMDGPU/max.i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/max.i16.ll
@@ -16,13 +16,13 @@ define amdgpu_kernel void @v_test_imax_sge_i16(i16 addrspace(1)* %out, i16 addrs
 ; VI-NEXT:    v_mov_b32_e32 v3, s1
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, s0, v4
 ; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    flat_load_ushort v5, v[0:1]
 ; VI-NEXT:    flat_load_ushort v2, v[2:3]
-; VI-NEXT:    flat_load_ushort v3, v[0:1]
 ; VI-NEXT:    v_mov_b32_e32 v1, s5
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s4, v4
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT:    v_max_i16_e32 v2, v3, v2
+; VI-NEXT:    v_max_i16_e32 v2, v5, v2
 ; VI-NEXT:    flat_store_short v[0:1], v2
 ; VI-NEXT:    s_endpgm
 ;
@@ -38,13 +38,13 @@ define amdgpu_kernel void @v_test_imax_sge_i16(i16 addrspace(1)* %out, i16 addrs
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s1
 ; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, s0, v4
 ; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    global_load_ushort v5, v[0:1], off
 ; GFX9-NEXT:    global_load_ushort v2, v[2:3], off
-; GFX9-NEXT:    global_load_ushort v3, v[0:1], off
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s5
 ; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s4, v4
 ; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_max_i16_e32 v2, v3, v2
+; GFX9-NEXT:    v_max_i16_e32 v2, v5, v2
 ; GFX9-NEXT:    global_store_short v[0:1], v2, off
 ; GFX9-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
@@ -73,15 +73,15 @@ define amdgpu_kernel void @v_test_imax_sge_v2i16(<2 x i16> addrspace(1)* %out, <
 ; VI-NEXT:    v_mov_b32_e32 v3, s1
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, s0, v4
 ; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    flat_load_dword v5, v[0:1]
 ; VI-NEXT:    flat_load_dword v2, v[2:3]
-; VI-NEXT:    flat_load_dword v3, v[0:1]
-; VI-NEXT:    v_add_u32_e32 v0, vcc, s4, v4
 ; VI-NEXT:    v_mov_b32_e32 v1, s5
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s4, v4
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT:    v_max_i16_e32 v4, v3, v2
-; VI-NEXT:    v_max_i16_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; VI-NEXT:    v_or_b32_e32 v2, v4, v2
+; VI-NEXT:    v_max_i16_e32 v3, v5, v2
+; VI-NEXT:    v_max_i16_sdwa v2, v5, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; VI-NEXT:    v_or_b32_e32 v2, v3, v2
 ; VI-NEXT:    flat_store_dword v[0:1], v2
 ; VI-NEXT:    s_endpgm
 ;
@@ -97,13 +97,13 @@ define amdgpu_kernel void @v_test_imax_sge_v2i16(<2 x i16> addrspace(1)* %out, <
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s1
 ; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, s0, v4
 ; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    global_load_dword v5, v[0:1], off
 ; GFX9-NEXT:    global_load_dword v2, v[2:3], off
-; GFX9-NEXT:    global_load_dword v3, v[0:1], off
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s5
 ; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s4, v4
 ; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_pk_max_i16 v2, v3, v2
+; GFX9-NEXT:    v_pk_max_i16 v2, v5, v2
 ; GFX9-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
@@ -124,35 +124,35 @@ define amdgpu_kernel void @v_test_imax_sge_v3i16(<3 x i16> addrspace(1)* %out, <
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
-; VI-NEXT:    v_lshlrev_b32_e32 v8, 3, v0
+; VI-NEXT:    v_lshlrev_b32_e32 v6, 3, v0
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
-; VI-NEXT:    v_mov_b32_e32 v3, s7
-; VI-NEXT:    v_mov_b32_e32 v1, s1
-; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v8
+; VI-NEXT:    v_mov_b32_e32 v1, s7
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s6, v6
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT:    v_add_u32_e32 v2, vcc, s6, v8
+; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    v_add_u32_e32 v2, vcc, s0, v6
 ; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
 ; VI-NEXT:    v_add_u32_e32 v4, vcc, 4, v0
 ; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; VI-NEXT:    v_add_u32_e32 v6, vcc, 4, v2
+; VI-NEXT:    v_mov_b32_e32 v7, s5
+; VI-NEXT:    v_add_u32_e32 v6, vcc, s4, v6
+; VI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; VI-NEXT:    flat_load_ushort v8, v[4:5]
 ; VI-NEXT:    flat_load_dword v9, v[0:1]
-; VI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v3, vcc
-; VI-NEXT:    flat_load_ushort v4, v[4:5]
-; VI-NEXT:    flat_load_dword v5, v[2:3]
-; VI-NEXT:    flat_load_ushort v6, v[6:7]
-; VI-NEXT:    v_mov_b32_e32 v1, s5
-; VI-NEXT:    v_add_u32_e32 v0, vcc, s4, v8
-; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT:    v_add_u32_e32 v2, vcc, 4, v0
-; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v1, vcc
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 4, v2
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v3, vcc
+; VI-NEXT:    flat_load_ushort v0, v[0:1]
+; VI-NEXT:    flat_load_dword v1, v[2:3]
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 4, v6
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v7, vcc
 ; VI-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)
-; VI-NEXT:    v_max_i16_e32 v7, v5, v9
-; VI-NEXT:    v_max_i16_sdwa v5, v5, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; VI-NEXT:    v_max_i16_e32 v0, v8, v0
 ; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT:    v_max_i16_e32 v4, v6, v4
-; VI-NEXT:    v_or_b32_e32 v5, v7, v5
-; VI-NEXT:    flat_store_dword v[0:1], v5
-; VI-NEXT:    flat_store_short v[2:3], v4
+; VI-NEXT:    v_max_i16_e32 v2, v9, v1
+; VI-NEXT:    v_max_i16_sdwa v1, v9, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; VI-NEXT:    v_or_b32_e32 v1, v2, v1
+; VI-NEXT:    flat_store_short v[4:5], v0
+; VI-NEXT:    flat_store_dword v[6:7], v1
 ; VI-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: v_test_imax_sge_v3i16:
@@ -167,19 +167,21 @@ define amdgpu_kernel void @v_test_imax_sge_v3i16(<3 x i16> addrspace(1)* %out, <
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s1
 ; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, s0, v4
 ; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX9-NEXT:    global_load_dword v6, v[2:3], off
-; GFX9-NEXT:    global_load_dword v7, v[0:1], off
+; GFX9-NEXT:    global_load_dword v6, v[0:1], off
+; GFX9-NEXT:    global_load_dword v7, v[2:3], off
 ; GFX9-NEXT:    v_mov_b32_e32 v5, s5
 ; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, s4, v4
 ; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    v_mov_b32_e32 v8, v6
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v8, v7
-; GFX9-NEXT:    v_pk_max_i16 v7, v7, v6
-; GFX9-NEXT:    global_load_short_d16 v6, v[2:3], off offset:4
+; GFX9-NEXT:    v_mov_b32_e32 v9, v7
 ; GFX9-NEXT:    global_load_short_d16 v8, v[0:1], off offset:4
+; GFX9-NEXT:    global_load_short_d16 v9, v[2:3], off offset:4
+; GFX9-NEXT:    v_pk_max_i16 v6, v6, v7
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_pk_max_i16 v0, v8, v6
-; GFX9-NEXT:    global_store_dword v[4:5], v7, off
+; GFX9-NEXT:    v_pk_max_i16 v0, v8, v9
+; GFX9-NEXT:    global_store_dword v[4:5], v6, off
 ; GFX9-NEXT:    global_store_short v[4:5], v0, off offset:4
 ; GFX9-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
@@ -208,8 +210,8 @@ define amdgpu_kernel void @v_test_imax_sge_v4i16(<4 x i16> addrspace(1)* %out, <
 ; VI-NEXT:    v_mov_b32_e32 v3, s1
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, s0, v4
 ; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; VI-NEXT:    flat_load_dwordx2 v[2:3], v[2:3]
 ; VI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; VI-NEXT:    flat_load_dwordx2 v[2:3], v[2:3]
 ; VI-NEXT:    v_mov_b32_e32 v5, s5
 ; VI-NEXT:    v_add_u32_e32 v4, vcc, s4, v4
 ; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
@@ -235,8 +237,8 @@ define amdgpu_kernel void @v_test_imax_sge_v4i16(<4 x i16> addrspace(1)* %out, <
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s1
 ; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, s0, v4
 ; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX9-NEXT:    global_load_dwordx2 v[2:3], v[2:3], off
 ; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off
+; GFX9-NEXT:    global_load_dwordx2 v[2:3], v[2:3], off
 ; GFX9-NEXT:    v_mov_b32_e32 v5, s5
 ; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, s4, v4
 ; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
@@ -271,13 +273,13 @@ define amdgpu_kernel void @v_test_imax_sgt_i16(i16 addrspace(1)* %out, i16 addrs
 ; VI-NEXT:    v_mov_b32_e32 v3, s1
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, s0, v4
 ; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    flat_load_ushort v5, v[0:1]
 ; VI-NEXT:    flat_load_ushort v2, v[2:3]
-; VI-NEXT:    flat_load_ushort v3, v[0:1]
 ; VI-NEXT:    v_mov_b32_e32 v1, s5
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s4, v4
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT:    v_max_i16_e32 v2, v3, v2
+; VI-NEXT:    v_max_i16_e32 v2, v5, v2
 ; VI-NEXT:    flat_store_short v[0:1], v2
 ; VI-NEXT:    s_endpgm
 ;
@@ -293,13 +295,13 @@ define amdgpu_kernel void @v_test_imax_sgt_i16(i16 addrspace(1)* %out, i16 addrs
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s1
 ; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, s0, v4
 ; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    global_load_ushort v5, v[0:1], off
 ; GFX9-NEXT:    global_load_ushort v2, v[2:3], off
-; GFX9-NEXT:    global_load_ushort v3, v[0:1], off
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s5
 ; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s4, v4
 ; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_max_i16_e32 v2, v3, v2
+; GFX9-NEXT:    v_max_i16_e32 v2, v5, v2
 ; GFX9-NEXT:    global_store_short v[0:1], v2, off
 ; GFX9-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
@@ -328,13 +330,13 @@ define amdgpu_kernel void @v_test_umax_uge_i16(i16 addrspace(1)* %out, i16 addrs
 ; VI-NEXT:    v_mov_b32_e32 v3, s1
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, s0, v4
 ; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    flat_load_ushort v5, v[0:1]
 ; VI-NEXT:    flat_load_ushort v2, v[2:3]
-; VI-NEXT:    flat_load_ushort v3, v[0:1]
 ; VI-NEXT:    v_mov_b32_e32 v1, s5
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s4, v4
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT:    v_max_u16_e32 v2, v3, v2
+; VI-NEXT:    v_max_u16_e32 v2, v5, v2
 ; VI-NEXT:    flat_store_short v[0:1], v2
 ; VI-NEXT:    s_endpgm
 ;
@@ -350,13 +352,13 @@ define amdgpu_kernel void @v_test_umax_uge_i16(i16 addrspace(1)* %out, i16 addrs
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s1
 ; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, s0, v4
 ; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    global_load_ushort v5, v[0:1], off
 ; GFX9-NEXT:    global_load_ushort v2, v[2:3], off
-; GFX9-NEXT:    global_load_ushort v3, v[0:1], off
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s5
 ; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s4, v4
 ; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_max_u16_e32 v2, v3, v2
+; GFX9-NEXT:    v_max_u16_e32 v2, v5, v2
 ; GFX9-NEXT:    global_store_short v[0:1], v2, off
 ; GFX9-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
@@ -385,13 +387,13 @@ define amdgpu_kernel void @v_test_umax_ugt_i16(i16 addrspace(1)* %out, i16 addrs
 ; VI-NEXT:    v_mov_b32_e32 v3, s1
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, s0, v4
 ; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    flat_load_ushort v5, v[0:1]
 ; VI-NEXT:    flat_load_ushort v2, v[2:3]
-; VI-NEXT:    flat_load_ushort v3, v[0:1]
 ; VI-NEXT:    v_mov_b32_e32 v1, s5
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s4, v4
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT:    v_max_u16_e32 v2, v3, v2
+; VI-NEXT:    v_max_u16_e32 v2, v5, v2
 ; VI-NEXT:    flat_store_short v[0:1], v2
 ; VI-NEXT:    s_endpgm
 ;
@@ -407,13 +409,13 @@ define amdgpu_kernel void @v_test_umax_ugt_i16(i16 addrspace(1)* %out, i16 addrs
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s1
 ; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, s0, v4
 ; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    global_load_ushort v5, v[0:1], off
 ; GFX9-NEXT:    global_load_ushort v2, v[2:3], off
-; GFX9-NEXT:    global_load_ushort v3, v[0:1], off
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s5
 ; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s4, v4
 ; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_max_u16_e32 v2, v3, v2
+; GFX9-NEXT:    v_max_u16_e32 v2, v5, v2
 ; GFX9-NEXT:    global_store_short v[0:1], v2, off
 ; GFX9-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
@@ -441,15 +443,15 @@ define amdgpu_kernel void @v_test_umax_ugt_v2i16(<2 x i16> addrspace(1)* %out, <
 ; VI-NEXT:    v_mov_b32_e32 v3, s1
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, s0, v4
 ; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    flat_load_dword v5, v[0:1]
 ; VI-NEXT:    flat_load_dword v2, v[2:3]
-; VI-NEXT:    flat_load_dword v3, v[0:1]
-; VI-NEXT:    v_add_u32_e32 v0, vcc, s4, v4
 ; VI-NEXT:    v_mov_b32_e32 v1, s5
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s4, v4
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT:    v_max_u16_e32 v4, v3, v2
-; VI-NEXT:    v_max_u16_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; VI-NEXT:    v_or_b32_e32 v2, v4, v2
+; VI-NEXT:    v_max_u16_e32 v3, v5, v2
+; VI-NEXT:    v_max_u16_sdwa v2, v5, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; VI-NEXT:    v_or_b32_e32 v2, v3, v2
 ; VI-NEXT:    flat_store_dword v[0:1], v2
 ; VI-NEXT:    s_endpgm
 ;
@@ -465,13 +467,13 @@ define amdgpu_kernel void @v_test_umax_ugt_v2i16(<2 x i16> addrspace(1)* %out, <
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s1
 ; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, s0, v4
 ; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    global_load_dword v5, v[0:1], off
 ; GFX9-NEXT:    global_load_dword v2, v[2:3], off
-; GFX9-NEXT:    global_load_dword v3, v[0:1], off
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s5
 ; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s4, v4
 ; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_pk_max_u16 v2, v3, v2
+; GFX9-NEXT:    v_pk_max_u16 v2, v5, v2
 ; GFX9-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
diff --git a/llvm/test/CodeGen/AMDGPU/max.ll b/llvm/test/CodeGen/AMDGPU/max.ll
index 716caf252..b53d65e1b 100644
--- a/llvm/test/CodeGen/AMDGPU/max.ll
+++ b/llvm/test/CodeGen/AMDGPU/max.ll
@@ -1,5 +1,5 @@
-; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=pitcairn < %s | FileCheck -enable-var-scope -check-prefix=SI -check-prefix=FUNC %s
-; RUN: llc -amdgpu-scalarize-global-loads=false -march=r600 -mcpu=cypress < %s | FileCheck -enable-var-scope -check-prefix=EG -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -mcpu=pitcairn < %s | FileCheck -enable-var-scope -check-prefix=SI -check-prefix=FUNC %s
+; RUN: llc -march=r600 -mcpu=cypress < %s | FileCheck -enable-var-scope -check-prefix=EG -check-prefix=FUNC %s
 
 
 ; FUNC-LABEL: {{^}}v_test_imax_sge_i32:
@@ -7,8 +7,10 @@
 
 ; EG: MAX_INT
 define amdgpu_kernel void @v_test_imax_sge_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %aptr, i32 addrspace(1)* %bptr) nounwind {
+  %tid = call i32 @llvm.r600.read.tidig.x()
+  %gep.in = getelementptr inbounds i32, i32 addrspace(1)* %bptr, i32 %tid
   %a = load i32, i32 addrspace(1)* %aptr, align 4
-  %b = load i32, i32 addrspace(1)* %bptr, align 4
+  %b = load i32, i32 addrspace(1)* %gep.in, align 4
   %cmp = icmp sge i32 %a, %b
   %val = select i1 %cmp, i32 %a, i32 %b
   store i32 %val, i32 addrspace(1)* %out, align 4
@@ -27,8 +29,10 @@ define amdgpu_kernel void @v_test_imax_sge_i32(i32 addrspace(1)* %out, i32 addrs
 ; EG: MAX_INT
 ; EG: MAX_INT
 define amdgpu_kernel void @v_test_imax_sge_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> addrspace(1)* %aptr, <4 x i32> addrspace(1)* %bptr) nounwind {
+  %tid = call i32 @llvm.r600.read.tidig.x()
+  %gep.in = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* %bptr, i32 %tid
   %a = load <4 x i32>, <4 x i32> addrspace(1)* %aptr, align 4
-  %b = load <4 x i32>, <4 x i32> addrspace(1)* %bptr, align 4
+  %b = load <4 x i32>, <4 x i32> addrspace(1)* %gep.in, align 4
   %cmp = icmp sge <4 x i32> %a, %b
   %val = select <4 x i1> %cmp, <4 x i32> %a, <4 x i32> %b
   store <4 x i32> %val, <4 x i32> addrspace(1)* %out, align 4
@@ -101,8 +105,10 @@ define amdgpu_kernel void @s_test_imax_sgt_imm_v2i32(<2 x i32> addrspace(1)* %ou
 
 ; EG: MAX_INT
 define amdgpu_kernel void @v_test_imax_sgt_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %aptr, i32 addrspace(1)* %bptr) nounwind {
+  %tid = call i32 @llvm.r600.read.tidig.x()
+  %gep.in = getelementptr inbounds i32, i32 addrspace(1)* %bptr, i32 %tid
   %a = load i32, i32 addrspace(1)* %aptr, align 4
-  %b = load i32, i32 addrspace(1)* %bptr, align 4
+  %b = load i32, i32 addrspace(1)* %gep.in, align 4
   %cmp = icmp sgt i32 %a, %b
   %val = select i1 %cmp, i32 %a, i32 %b
   store i32 %val, i32 addrspace(1)* %out, align 4
@@ -125,8 +131,10 @@ define amdgpu_kernel void @s_test_imax_sgt_i32(i32 addrspace(1)* %out, i32 %a, i
 
 ; EG: MAX_UINT
 define amdgpu_kernel void @v_test_umax_uge_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %aptr, i32 addrspace(1)* %bptr) nounwind {
+  %tid = call i32 @llvm.r600.read.tidig.x()
+  %gep.in = getelementptr inbounds i32, i32 addrspace(1)* %bptr, i32 %tid
   %a = load i32, i32 addrspace(1)* %aptr, align 4
-  %b = load i32, i32 addrspace(1)* %bptr, align 4
+  %b = load i32, i32 addrspace(1)* %gep.in, align 4
   %cmp = icmp uge i32 %a, %b
   %val = select i1 %cmp, i32 %a, i32 %b
   store i32 %val, i32 addrspace(1)* %out, align 4
@@ -182,7 +190,9 @@ define amdgpu_kernel void @v_test_umax_uge_i8(i8 addrspace(1)* %out, i8 addrspac
 
 ; EG: MAX_UINT
 define amdgpu_kernel void @v_test_umax_ugt_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %aptr, i32 addrspace(1)* %bptr) nounwind {
-  %a = load i32, i32 addrspace(1)* %aptr, align 4
+  %tid = call i32 @llvm.r600.read.tidig.x()
+  %gep.in = getelementptr inbounds i32, i32 addrspace(1)* %bptr, i32 %tid
+  %a = load i32, i32 addrspace(1)* %gep.in, align 4
   %b = load i32, i32 addrspace(1)* %bptr, align 4
   %cmp = icmp ugt i32 %a, %b
   %val = select i1 %cmp, i32 %a, i32 %b
@@ -320,3 +330,9 @@ define amdgpu_kernel void @test_imax_sge_i64(i64 addrspace(1)* %out, i64 %a, i64
   store i64 %val, i64 addrspace(1)* %out, align 8
   ret void
 }
+
+
+declare i32 @llvm.r600.read.tidig.x() #0
+
+attributes #0 = { nounwind readnone }
+attributes #1 = { nounwind }
diff --git a/llvm/test/CodeGen/AMDGPU/memory_clause.ll b/llvm/test/CodeGen/AMDGPU/memory_clause.ll
index 9ae068a43..c721de658 100644
--- a/llvm/test/CodeGen/AMDGPU/memory_clause.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory_clause.ll
@@ -1,12 +1,29 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -march=amdgcn -mcpu=gfx902 -verify-machineinstrs -amdgpu-enable-global-sgpr-addr < %s | FileCheck -check-prefix=GCN %s
 
-; GCN-LABEL: {{^}}vector_clause:
-; GCN:      global_load_dwordx4
-; GCN-NEXT: global_load_dwordx4
-; GCN-NEXT: global_load_dwordx4
-; GCN-NEXT: global_load_dwordx4
-; GCN-NEXT: s_nop
 define amdgpu_kernel void @vector_clause(<4 x i32> addrspace(1)* noalias nocapture readonly %arg, <4 x i32> addrspace(1)* noalias nocapture %arg1) {
+; GCN-LABEL: vector_clause:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
+; GCN-NEXT:    v_mov_b32_e32 v17, 0
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 4, v0
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2c
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    global_load_dwordx4 v[0:3], v[16:17], s[2:3]
+; GCN-NEXT:    global_load_dwordx4 v[4:7], v[16:17], s[2:3] offset:16
+; GCN-NEXT:    global_load_dwordx4 v[8:11], v[16:17], s[2:3] offset:32
+; GCN-NEXT:    global_load_dwordx4 v[12:15], v[16:17], s[2:3] offset:48
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    s_waitcnt vmcnt(3)
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    global_store_dwordx4 v[16:17], v[0:3], s[4:5]
+; GCN-NEXT:    s_waitcnt vmcnt(3)
+; GCN-NEXT:    global_store_dwordx4 v[16:17], v[4:7], s[4:5] offset:16
+; GCN-NEXT:    s_waitcnt vmcnt(3)
+; GCN-NEXT:    global_store_dwordx4 v[16:17], v[8:11], s[4:5] offset:32
+; GCN-NEXT:    s_waitcnt vmcnt(3)
+; GCN-NEXT:    global_store_dwordx4 v[16:17], v[12:15], s[4:5] offset:48
+; GCN-NEXT:    s_endpgm
 bb:
   %tmp = tail call i32 @llvm.amdgcn.workitem.id.x()
   %tmp2 = zext i32 %tmp to i64
@@ -32,16 +49,45 @@ bb:
   ret void
 }
 
-; GCN-LABEL: {{^}}scalar_clause:
-; GCN:      s_load_dwordx2
-; GCN-NEXT: s_load_dwordx2
-; GCN-NEXT: s_nop
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_load_dwordx4
-; GCN-NEXT: s_load_dwordx4
-; GCN-NEXT: s_load_dwordx4
-; GCN-NEXT: s_load_dwordx4
 define amdgpu_kernel void @scalar_clause(<4 x i32> addrspace(1)* noalias nocapture readonly %arg, <4 x i32> addrspace(1)* noalias nocapture %arg1) {
+; GCN-LABEL: scalar_clause:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x24
+; GCN-NEXT:    s_load_dwordx2 s[18:19], s[0:1], 0x2c
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_load_dwordx4 s[0:3], s[16:17], 0x0
+; GCN-NEXT:    s_load_dwordx4 s[4:7], s[16:17], 0x10
+; GCN-NEXT:    s_load_dwordx4 s[8:11], s[16:17], 0x20
+; GCN-NEXT:    s_load_dwordx4 s[12:15], s[16:17], 0x30
+; GCN-NEXT:    v_mov_b32_e32 v12, s18
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-NEXT:    v_mov_b32_e32 v4, s4
+; GCN-NEXT:    v_mov_b32_e32 v13, s19
+; GCN-NEXT:    v_mov_b32_e32 v1, s1
+; GCN-NEXT:    v_mov_b32_e32 v2, s2
+; GCN-NEXT:    v_mov_b32_e32 v3, s3
+; GCN-NEXT:    v_mov_b32_e32 v5, s5
+; GCN-NEXT:    v_mov_b32_e32 v6, s6
+; GCN-NEXT:    v_mov_b32_e32 v7, s7
+; GCN-NEXT:    v_mov_b32_e32 v8, s8
+; GCN-NEXT:    v_mov_b32_e32 v9, s9
+; GCN-NEXT:    v_mov_b32_e32 v10, s10
+; GCN-NEXT:    v_mov_b32_e32 v11, s11
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    global_store_dwordx4 v[12:13], v[0:3], off
+; GCN-NEXT:    global_store_dwordx4 v[12:13], v[4:7], off offset:16
+; GCN-NEXT:    v_mov_b32_e32 v0, s12
+; GCN-NEXT:    v_mov_b32_e32 v1, s13
+; GCN-NEXT:    v_mov_b32_e32 v2, s14
+; GCN-NEXT:    v_mov_b32_e32 v3, s15
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    global_store_dwordx4 v[12:13], v[8:11], off offset:32
+; GCN-NEXT:    global_store_dwordx4 v[12:13], v[0:3], off offset:48
+; GCN-NEXT:    s_endpgm
 bb:
   %tmp = load <4 x i32>, <4 x i32> addrspace(1)* %arg, align 16
   %tmp2 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* %arg, i64 1
@@ -60,26 +106,70 @@ bb:
   ret void
 }
 
-; GCN-LABEL: {{^}}mubuf_clause:
-; GCN:      buffer_load_dword
-; GCN-NEXT: buffer_load_dword
-; GCN-NEXT: buffer_load_dword
-; GCN-NEXT: buffer_load_dword
-; GCN-NEXT: buffer_load_dword
-; GCN-NEXT: buffer_load_dword
-; GCN-NEXT: buffer_load_dword
-; GCN-NEXT: buffer_load_dword
-; GCN-NEXT: buffer_load_dword
-; GCN-NEXT: buffer_load_dword
-; GCN-NEXT: buffer_load_dword
-; GCN-NEXT: buffer_load_dword
-; GCN-NEXT: buffer_load_dword
-; GCN-NEXT: buffer_load_dword
-; GCN-NEXT: buffer_load_dword
-; GCN-NEXT: s_nop
-; GCN-NEXT: s_nop
-; GCN-NEXT: buffer_load_dword
 define void @mubuf_clause(<4 x i32> addrspace(5)* noalias nocapture readonly %arg, <4 x i32> addrspace(5)* noalias nocapture %arg1) {
+; GCN-LABEL: mubuf_clause:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v2, 0x3ff, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 4, v2
+; GCN-NEXT:    v_add_u32_e32 v0, v0, v2
+; GCN-NEXT:    v_add_u32_e32 v1, v1, v2
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    buffer_load_dword v3, v0, s[0:3], s33 offen
+; GCN-NEXT:    buffer_load_dword v4, v0, s[0:3], s33 offen offset:4
+; GCN-NEXT:    buffer_load_dword v5, v0, s[0:3], s33 offen offset:8
+; GCN-NEXT:    buffer_load_dword v6, v0, s[0:3], s33 offen offset:12
+; GCN-NEXT:    buffer_load_dword v7, v0, s[0:3], s33 offen offset:16
+; GCN-NEXT:    buffer_load_dword v8, v0, s[0:3], s33 offen offset:20
+; GCN-NEXT:    buffer_load_dword v9, v0, s[0:3], s33 offen offset:24
+; GCN-NEXT:    buffer_load_dword v10, v0, s[0:3], s33 offen offset:28
+; GCN-NEXT:    buffer_load_dword v11, v0, s[0:3], s33 offen offset:32
+; GCN-NEXT:    buffer_load_dword v12, v0, s[0:3], s33 offen offset:36
+; GCN-NEXT:    buffer_load_dword v13, v0, s[0:3], s33 offen offset:40
+; GCN-NEXT:    buffer_load_dword v14, v0, s[0:3], s33 offen offset:44
+; GCN-NEXT:    buffer_load_dword v15, v0, s[0:3], s33 offen offset:48
+; GCN-NEXT:    buffer_load_dword v16, v0, s[0:3], s33 offen offset:52
+; GCN-NEXT:    buffer_load_dword v17, v0, s[0:3], s33 offen offset:56
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    buffer_load_dword v0, v0, s[0:3], s33 offen offset:60
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    s_waitcnt vmcnt(15)
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    buffer_store_dword v3, v1, s[0:3], s33 offen
+; GCN-NEXT:    s_waitcnt vmcnt(15)
+; GCN-NEXT:    buffer_store_dword v4, v1, s[0:3], s33 offen offset:4
+; GCN-NEXT:    s_waitcnt vmcnt(15)
+; GCN-NEXT:    buffer_store_dword v5, v1, s[0:3], s33 offen offset:8
+; GCN-NEXT:    s_waitcnt vmcnt(15)
+; GCN-NEXT:    buffer_store_dword v6, v1, s[0:3], s33 offen offset:12
+; GCN-NEXT:    s_waitcnt vmcnt(15)
+; GCN-NEXT:    buffer_store_dword v7, v1, s[0:3], s33 offen offset:16
+; GCN-NEXT:    s_waitcnt vmcnt(15)
+; GCN-NEXT:    buffer_store_dword v8, v1, s[0:3], s33 offen offset:20
+; GCN-NEXT:    s_waitcnt vmcnt(15)
+; GCN-NEXT:    buffer_store_dword v9, v1, s[0:3], s33 offen offset:24
+; GCN-NEXT:    s_waitcnt vmcnt(15)
+; GCN-NEXT:    buffer_store_dword v10, v1, s[0:3], s33 offen offset:28
+; GCN-NEXT:    s_waitcnt vmcnt(15)
+; GCN-NEXT:    buffer_store_dword v11, v1, s[0:3], s33 offen offset:32
+; GCN-NEXT:    s_waitcnt vmcnt(15)
+; GCN-NEXT:    buffer_store_dword v12, v1, s[0:3], s33 offen offset:36
+; GCN-NEXT:    s_waitcnt vmcnt(15)
+; GCN-NEXT:    buffer_store_dword v13, v1, s[0:3], s33 offen offset:40
+; GCN-NEXT:    s_waitcnt vmcnt(15)
+; GCN-NEXT:    buffer_store_dword v14, v1, s[0:3], s33 offen offset:44
+; GCN-NEXT:    s_waitcnt vmcnt(15)
+; GCN-NEXT:    buffer_store_dword v15, v1, s[0:3], s33 offen offset:48
+; GCN-NEXT:    s_waitcnt vmcnt(15)
+; GCN-NEXT:    buffer_store_dword v16, v1, s[0:3], s33 offen offset:52
+; GCN-NEXT:    s_waitcnt vmcnt(15)
+; GCN-NEXT:    buffer_store_dword v17, v1, s[0:3], s33 offen offset:56
+; GCN-NEXT:    s_waitcnt vmcnt(15)
+; GCN-NEXT:    buffer_store_dword v0, v1, s[0:3], s33 offen offset:60
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
 bb:
   %tmp = tail call i32 @llvm.amdgcn.workitem.id.x()
   %tmp2 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(5)* %arg, i32 %tmp
@@ -104,14 +194,29 @@ bb:
   ret void
 }
 
-; GCN-LABEL: {{^}}vector_clause_indirect:
-; GCN: global_load_dwordx2 [[ADDR:v\[[0-9:]+\]]], v[{{[0-9:]+}}], s[{{[0-9:]+}}]
-; GCN-NEXT: s_nop 0
-; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: s_nop 0
-; GCN-NEXT: global_load_dwordx4 v[{{[0-9:]+}}], [[ADDR]], off
-; GCN-NEXT: global_load_dwordx4 v[{{[0-9:]+}}], [[ADDR]], off offset:16
 define amdgpu_kernel void @vector_clause_indirect(i64 addrspace(1)* noalias nocapture readonly %arg, <4 x i32> addrspace(1)* noalias nocapture readnone %arg1, <4 x i32> addrspace(1)* noalias nocapture %arg2) {
+; GCN-LABEL: vector_clause_indirect:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
+; GCN-NEXT:    v_mov_b32_e32 v1, 0
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    global_load_dwordx2 v[8:9], v[0:1], s[2:3]
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    global_load_dwordx4 v[0:3], v[8:9], off
+; GCN-NEXT:    global_load_dwordx4 v[4:7], v[8:9], off offset:16
+; GCN-NEXT:    v_mov_b32_e32 v9, s5
+; GCN-NEXT:    v_mov_b32_e32 v8, s4
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    s_waitcnt vmcnt(1)
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    global_store_dwordx4 v[8:9], v[0:3], off
+; GCN-NEXT:    s_waitcnt vmcnt(1)
+; GCN-NEXT:    global_store_dwordx4 v[8:9], v[4:7], off offset:16
+; GCN-NEXT:    s_endpgm
 bb:
   %tmp = tail call i32 @llvm.amdgcn.workitem.id.x()
   %tmp3 = zext i32 %tmp to i64
@@ -127,12 +232,25 @@ bb:
   ret void
 }
 
-; GCN-LABEL: {{^}}load_global_d16_hi:
-; GCN:      global_load_short_d16_hi v
-; GCN-NEXT: s_nop
-; GCN-NEXT: s_nop
-; GCN-NEXT: global_load_short_d16_hi v
 define void @load_global_d16_hi(i16 addrspace(1)* %in, i16 %reg, <2 x i16> addrspace(1)* %out) {
+; GCN-LABEL: load_global_d16_hi:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v5, v2
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    global_load_short_d16_hi v5, v[0:1], off
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    global_load_short_d16_hi v2, v[0:1], off offset:64
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    s_waitcnt vmcnt(1)
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    global_store_dword v[3:4], v5, off
+; GCN-NEXT:    s_waitcnt vmcnt(1)
+; GCN-NEXT:    global_store_dword v[3:4], v2, off offset:128
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %gep = getelementptr inbounds i16, i16 addrspace(1)* %in, i64 32
   %load1 = load i16, i16 addrspace(1)* %in
@@ -147,12 +265,25 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}load_global_d16_lo:
-; GCN:      global_load_short_d16 v
-; GCN-NEXT: s_nop
-; GCN-NEXT: s_nop
-; GCN-NEXT: global_load_short_d16 v
 define void @load_global_d16_lo(i16 addrspace(1)* %in, i32 %reg, <2 x i16> addrspace(1)* %out) {
+; GCN-LABEL: load_global_d16_lo:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v5, v2
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    global_load_short_d16 v5, v[0:1], off
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    global_load_short_d16 v2, v[0:1], off offset:64
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    s_waitcnt vmcnt(1)
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    global_store_dword v[3:4], v5, off
+; GCN-NEXT:    s_waitcnt vmcnt(1)
+; GCN-NEXT:    global_store_dword v[3:4], v2, off offset:128
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %gep = getelementptr inbounds i16, i16 addrspace(1)* %in, i64 32
   %reg.bc1 = bitcast i32 %reg to <2 x i16>
diff --git a/llvm/test/CodeGen/AMDGPU/mfma-loop.ll b/llvm/test/CodeGen/AMDGPU/mfma-loop.ll
new file mode 100644
index 000000000..b66c9d414
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/mfma-loop.ll
@@ -0,0 +1,491 @@
+; RUN: llc -march=amdgcn -mcpu=gfx908 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN %s
+
+; GCN-LABEL: {{^}}test_mfma_loop_zeroinit:
+
+; GCN-COUNT-32: v_accvgpr_write_b32 a{{[0-9]+}}, 0{{$}}
+
+; Check that we do not copy agprs to vgprs and back inside the loop.
+
+; GCN: [[LOOP:BB[0-9_]+]]:
+; GCN-NOT: v_accvgpr
+; GCN: v_mfma_f32_32x32x1f32
+; GCN-NOT: v_accvgpr
+; GCN: s_cbranch_scc1 [[LOOP]]
+
+; Final result should be read only once after the loop.
+
+; GCN-COUNT-32: v_accvgpr_read_b32
+
+define amdgpu_kernel void @test_mfma_loop_zeroinit(<32 x float> addrspace(1)* %arg) {
+entry:
+  br label %for.cond.preheader
+
+for.cond.preheader:
+  %phi = phi <32 x float> [ zeroinitializer, %entry ], [ %mai.1, %for.cond.preheader ]
+  %c = phi i32 [ 0, %entry ], [ %inc, %for.cond.preheader ]
+  %mai.1 = tail call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.0, float 2.0, <32 x float> %phi, i32 0, i32 0, i32 0)
+  %inc = add nuw nsw i32 %c, 1
+  %cc = icmp eq i32 %inc, 16
+  br i1 %cc, label %exit, label %for.cond.preheader
+
+exit:
+  store <32 x float> %mai.1, <32 x float> addrspace(1)* %arg
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_mfma_loop_unfoldable_splat:
+
+; Check that we do not use 32 temp vgprs, but rotate 3 vgprs only.
+; 3 vgprs are needed to avoid wait states between writes.
+; Check that we do not use 32 temp sgprs as well.
+
+; GCN:         v_mov_b32_e32 [[TMP:v[0-9]+]], 0x42f60000
+; GCN-COUNT-32: v_accvgpr_write_b32 a{{[0-9]+}}, [[TMP]]
+
+; GCN: [[LOOP:BB[0-9_]+]]:
+; GCN-NOT: v_accvgpr
+; GCN: v_mfma_f32_32x32x1f32
+; GCN-NOT: v_accvgpr
+; GCN: s_cbranch_scc1 [[LOOP]]
+
+; GCN-COUNT-32: v_accvgpr_read_b32
+
+define amdgpu_kernel void @test_mfma_loop_unfoldable_splat(<32 x float> addrspace(1)* %arg) {
+entry:
+  br label %for.cond.preheader
+
+for.cond.preheader:
+  %phi = phi <32 x float> [ <float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0>, %entry ], [ %mai.1, %for.cond.preheader ]
+  %c = phi i32 [ 0, %entry ], [ %inc, %for.cond.preheader ]
+  %mai.1 = tail call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.0, float 2.0, <32 x float> %phi, i32 0, i32 0, i32 0)
+  %inc = add nuw nsw i32 %c, 1
+  %cc = icmp eq i32 %inc, 16
+  br i1 %cc, label %exit, label %for.cond.preheader
+
+exit:
+  store <32 x float> %mai.1, <32 x float> addrspace(1)* %arg
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_mfma_loop_non_splat:
+
+; GCN:         v_accvgpr_write_b32 a{{[0-9]+}}, 0{{$}}
+; GCN:         v_accvgpr_write_b32 a{{[0-9]+}}, 1.0{{$}}
+; GCN-COUNT-30: v_accvgpr_write_b32 a{{[0-9]+}}, 0{{$}}
+
+; GCN: [[LOOP:BB[0-9_]+]]:
+; GCN-NOT: v_accvgpr
+; GCN: v_mfma_f32_32x32x1f32
+; GCN-NOT: v_accvgpr
+; GCN: s_cbranch_scc1 [[LOOP]]
+
+; GCN-COUNT-32: v_accvgpr_read_b32
+
+define amdgpu_kernel void @test_mfma_loop_non_splat(<32 x float> addrspace(1)* %arg) {
+entry:
+  br label %for.cond.preheader
+
+for.cond.preheader:
+  %phi = phi <32 x float> [ <float 0.0, float 1.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0>, %entry ], [ %mai.1, %for.cond.preheader ]
+  %c = phi i32 [ 0, %entry ], [ %inc, %for.cond.preheader ]
+  %mai.1 = tail call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.0, float 2.0, <32 x float> %phi, i32 0, i32 0, i32 0)
+  %inc = add nuw nsw i32 %c, 1
+  %cc = icmp eq i32 %inc, 16
+  br i1 %cc, label %exit, label %for.cond.preheader
+
+exit:
+  store <32 x float> %mai.1, <32 x float> addrspace(1)* %arg
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_mfma_loop_unfoldable_seq:
+
+; Check that we do not use 32 temp vgprs, but rotate 3 vgprs only.
+; 3 vgprs are needed to avoid wait states between writes.
+
+; GCN: v_mov_b32_e32 [[TMP1:v[0-9]+]], 0x42f60000
+; GCN: v_mov_b32_e32 [[TMP2:v[0-9]+]], 0x42f80000
+; GCN: v_mov_b32_e32 [[TMP3:v[0-9]+]], 0x42fe0000
+; GCN: v_accvgpr_write_b32 a{{[0-9]+}}, [[TMP1]]
+; GCN: v_accvgpr_write_b32 a{{[0-9]+}}, [[TMP2]]
+; GCN: v_mov_b32_e32 [[TMP1]], 0x4{{[0-9a-f]+}}
+; GCN: v_mov_b32_e32 [[TMP2]], 0x4{{[0-9a-f]+}}
+; GCN: v_accvgpr_write_b32 a{{[0-9]+}}, [[TMP3]]
+; GCN: v_accvgpr_write_b32 a{{[0-9]+}}, [[TMP1]]
+; GCN: v_accvgpr_write_b32 a{{[0-9]+}}, [[TMP2]]
+; GCN: v_mov_b32_e32 [[TMP1]], 0x4{{[0-9a-f]+}}
+; GCN: v_mov_b32_e32 [[TMP2]], 0x4{{[0-9a-f]+}}
+; GCN: v_mov_b32_e32 [[TMP3]], 0x4{{[0-9a-f]+}}
+; GCN: v_accvgpr_write_b32 a{{[0-9]+}}, [[TMP1]]
+; GCN: v_accvgpr_write_b32 a{{[0-9]+}}, [[TMP2]]
+; GCN: v_accvgpr_write_b32 a{{[0-9]+}}, [[TMP3]]
+; GCN: v_mov_b32_e32 [[TMP1]], 0x4{{[0-9a-f]+}}
+; GCN: v_mov_b32_e32 [[TMP2]], 0x4{{[0-9a-f]+}}
+; GCN: v_mov_b32_e32 [[TMP3]], 0x4{{[0-9a-f]+}}
+; GCN: v_accvgpr_write_b32 a{{[0-9]+}}, [[TMP1]]
+; GCN: v_accvgpr_write_b32 a{{[0-9]+}}, [[TMP2]]
+; GCN: v_accvgpr_write_b32 a{{[0-9]+}}, [[TMP3]]
+; GCN: v_mov_b32_e32 [[TMP1]], 0x4{{[0-9a-f]+}}
+; GCN: v_mov_b32_e32 [[TMP2]], 0x4{{[0-9a-f]+}}
+; GCN: v_mov_b32_e32 [[TMP3]], 0x4{{[0-9a-f]+}}
+; GCN: v_accvgpr_write_b32 a{{[0-9]+}}, [[TMP1]]
+; GCN: v_accvgpr_write_b32 a{{[0-9]+}}, [[TMP2]]
+; GCN: v_accvgpr_write_b32 a{{[0-9]+}}, [[TMP3]]
+; GCN: v_mov_b32_e32 [[TMP1]], 0x4{{[0-9a-f]+}}
+; GCN: v_mov_b32_e32 [[TMP2]], 0x4{{[0-9a-f]+}}
+; GCN: v_mov_b32_e32 [[TMP3]], 0x4{{[0-9a-f]+}}
+; GCN: v_accvgpr_write_b32 a{{[0-9]+}}, [[TMP1]]
+; GCN: v_accvgpr_write_b32 a{{[0-9]+}}, [[TMP2]]
+; GCN: v_accvgpr_write_b32 a{{[0-9]+}}, [[TMP3]]
+; GCN: v_mov_b32_e32 [[TMP1]], 0x4{{[0-9a-f]+}}
+; GCN: v_mov_b32_e32 [[TMP2]], 0x4{{[0-9a-f]+}}
+; GCN: v_mov_b32_e32 [[TMP3]], 0x4{{[0-9a-f]+}}
+; GCN: v_accvgpr_write_b32 a{{[0-9]+}}, [[TMP1]]
+; GCN: v_accvgpr_write_b32 a{{[0-9]+}}, [[TMP2]]
+; GCN: v_accvgpr_write_b32 a{{[0-9]+}}, [[TMP3]]
+; GCN: v_mov_b32_e32 [[TMP1]], 0x4{{[0-9a-f]+}}
+; GCN: v_mov_b32_e32 [[TMP2]], 0x4{{[0-9a-f]+}}
+; GCN: v_mov_b32_e32 [[TMP3]], 0x4{{[0-9a-f]+}}
+; GCN: v_accvgpr_write_b32 a{{[0-9]+}}, [[TMP1]]
+; GCN: v_accvgpr_write_b32 a{{[0-9]+}}, [[TMP2]]
+; GCN: v_accvgpr_write_b32 a{{[0-9]+}}, [[TMP3]]
+; GCN: v_mov_b32_e32 [[TMP1]], 0x4{{[0-9a-f]+}}
+; GCN: v_mov_b32_e32 [[TMP2]], 0x4{{[0-9a-f]+}}
+; GCN: v_mov_b32_e32 [[TMP3]], 0x4{{[0-9a-f]+}}
+; GCN: v_accvgpr_write_b32 a{{[0-9]+}}, [[TMP1]]
+; GCN: v_accvgpr_write_b32 a{{[0-9]+}}, [[TMP2]]
+; GCN: v_accvgpr_write_b32 a{{[0-9]+}}, [[TMP3]]
+; GCN: v_mov_b32_e32 [[TMP1]], 0x4{{[0-9a-f]+}}
+; GCN: v_mov_b32_e32 [[TMP2]], 0x4{{[0-9a-f]+}}
+; GCN: v_mov_b32_e32 [[TMP3]], 0x4{{[0-9a-f]+}}
+; GCN: v_accvgpr_write_b32 a{{[0-9]+}}, [[TMP1]]
+; GCN: v_accvgpr_write_b32 a{{[0-9]+}}, [[TMP2]]
+; GCN: v_accvgpr_write_b32 a{{[0-9]+}}, [[TMP3]]
+; GCN: v_mov_b32_e32 [[TMP1]], 0x4{{[0-9a-f]+}}
+; GCN: v_mov_b32_e32 [[TMP2]], 0x4{{[0-9a-f]+}}
+; GCN: v_mov_b32_e32 [[TMP3]], 0x4{{[0-9a-f]+}}
+; GCN: v_accvgpr_write_b32 a{{[0-9]+}}, [[TMP1]]
+; GCN: v_accvgpr_write_b32 a{{[0-9]+}}, [[TMP2]]
+; GCN: v_accvgpr_write_b32 a{{[0-9]+}}, [[TMP3]]
+
+; GCN: [[LOOP:BB[0-9_]+]]:
+; GCN-NOT: v_accvgpr
+; GCN: v_mfma_f32_32x32x1f32
+; GCN-NOT: v_accvgpr
+; GCN: s_cbranch_scc1 [[LOOP]]
+
+; GCN-COUNT-32: v_accvgpr_read_b32
+
+define amdgpu_kernel void @test_mfma_loop_unfoldable_seq(<32 x float> addrspace(1)* %arg) {
+entry:
+  br label %for.cond.preheader
+
+for.cond.preheader:
+  %phi = phi <32 x float> [ <float 123.0, float 124.0, float 125.0, float 126.0, float 127.0, float 128.0, float 129.0, float 130.0, float 131.0, float 132.0, float 133.0, float 134.0, float 135.0, float 136.0, float 137.0, float 138.0, float 139.0, float 140.0, float 141.0, float 142.0, float 143.0, float 144.0, float 145.0, float 146.0, float 147.0, float 148.0, float 149.0, float 150.0, float 151.0, float 152.0, float 153.0, float 154.0>, %entry ], [ %mai.1, %for.cond.preheader ]
+  %c = phi i32 [ 0, %entry ], [ %inc, %for.cond.preheader ]
+  %mai.1 = tail call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.0, float 2.0, <32 x float> %phi, i32 0, i32 0, i32 0)
+  %inc = add nuw nsw i32 %c, 1
+  %cc = icmp eq i32 %inc, 16
+  br i1 %cc, label %exit, label %for.cond.preheader
+
+exit:
+  store <32 x float> %mai.1, <32 x float> addrspace(1)* %arg
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_mfma_loop_vgpr_init:
+
+; GCN-COUNT-32: v_accvgpr_write_b32 a{{[0-9]+}}, v0{{$}}
+
+; GCN: [[LOOP:BB[0-9_]+]]:
+; GCN-NOT: v_accvgpr
+; GCN: v_mfma_f32_32x32x1f32
+; GCN-NOT: v_accvgpr
+; GCN: s_cbranch_scc1 [[LOOP]]
+
+; GCN-COUNT-32: v_accvgpr_read_b32
+
+define amdgpu_kernel void @test_mfma_loop_vgpr_init(<32 x float> addrspace(1)* %arg) {
+entry:
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %init = bitcast i32 %tid to float
+  %tmp0 = insertelement <32 x float> undef, float %init, i32 0
+  %tmp1 = insertelement <32 x float> %tmp0, float %init, i32 1
+  %tmp2 = insertelement <32 x float> %tmp1, float %init, i32 2
+  %tmp3 = insertelement <32 x float> %tmp2, float %init, i32 3
+  %tmp4 = insertelement <32 x float> %tmp3, float %init, i32 4
+  %tmp5 = insertelement <32 x float> %tmp4, float %init, i32 5
+  %tmp6 = insertelement <32 x float> %tmp5, float %init, i32 6
+  %tmp7 = insertelement <32 x float> %tmp6, float %init, i32 7
+  %tmp8 = insertelement <32 x float> %tmp7, float %init, i32 8
+  %tmp9 = insertelement <32 x float> %tmp8, float %init, i32 9
+  %tmp10 = insertelement <32 x float> %tmp9, float %init, i32 10
+  %tmp11 = insertelement <32 x float> %tmp10, float %init, i32 11
+  %tmp12 = insertelement <32 x float> %tmp11, float %init, i32 12
+  %tmp13 = insertelement <32 x float> %tmp12, float %init, i32 13
+  %tmp14 = insertelement <32 x float> %tmp13, float %init, i32 14
+  %tmp15 = insertelement <32 x float> %tmp14, float %init, i32 15
+  %tmp16 = insertelement <32 x float> %tmp15, float %init, i32 16
+  %tmp17 = insertelement <32 x float> %tmp16, float %init, i32 17
+  %tmp18 = insertelement <32 x float> %tmp17, float %init, i32 18
+  %tmp19 = insertelement <32 x float> %tmp18, float %init, i32 19
+  %tmp20 = insertelement <32 x float> %tmp19, float %init, i32 20
+  %tmp21 = insertelement <32 x float> %tmp20, float %init, i32 21
+  %tmp22 = insertelement <32 x float> %tmp21, float %init, i32 22
+  %tmp23 = insertelement <32 x float> %tmp22, float %init, i32 23
+  %tmp24 = insertelement <32 x float> %tmp23, float %init, i32 24
+  %tmp25 = insertelement <32 x float> %tmp24, float %init, i32 25
+  %tmp26 = insertelement <32 x float> %tmp25, float %init, i32 26
+  %tmp27 = insertelement <32 x float> %tmp26, float %init, i32 27
+  %tmp28 = insertelement <32 x float> %tmp27, float %init, i32 28
+  %tmp29 = insertelement <32 x float> %tmp28, float %init, i32 29
+  %tmp30 = insertelement <32 x float> %tmp29, float %init, i32 30
+  %tmp31 = insertelement <32 x float> %tmp30, float %init, i32 31
+
+  br label %for.cond.preheader
+
+for.cond.preheader:
+  %phi = phi <32 x float> [ %tmp31, %entry ], [ %mai.1, %for.cond.preheader ]
+  %c = phi i32 [ 0, %entry ], [ %inc, %for.cond.preheader ]
+  %mai.1 = tail call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.0, float 2.0, <32 x float> %phi, i32 0, i32 0, i32 0)
+  %inc = add nuw nsw i32 %c, 1
+  %cc = icmp eq i32 %inc, 16
+  br i1 %cc, label %exit, label %for.cond.preheader
+
+exit:
+  store <32 x float> %mai.1, <32 x float> addrspace(1)* %arg
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_mfma_loop_sgpr_init:
+
+; GCN:         v_mov_b32_e32 [[TMP:v[0-9]+]], s{{[0-9]+}}
+; GCN-COUNT-32: v_accvgpr_write_b32 a{{[0-9]+}}, [[TMP]]
+
+; GCN: [[LOOP:BB[0-9_]+]]:
+; GCN-NOT: v_accvgpr
+; GCN: v_mfma_f32_32x32x1f32
+; GCN-NOT: v_accvgpr
+; GCN: s_cbranch_scc1 [[LOOP]]
+
+; GCN-COUNT-32: v_accvgpr_read_b32
+
+define amdgpu_kernel void @test_mfma_loop_sgpr_init(<32 x float> addrspace(1)* %arg, float %init) {
+entry:
+  %tmp0 = insertelement <32 x float> undef, float %init, i32 0
+  %tmp1 = insertelement <32 x float> %tmp0, float %init, i32 1
+  %tmp2 = insertelement <32 x float> %tmp1, float %init, i32 2
+  %tmp3 = insertelement <32 x float> %tmp2, float %init, i32 3
+  %tmp4 = insertelement <32 x float> %tmp3, float %init, i32 4
+  %tmp5 = insertelement <32 x float> %tmp4, float %init, i32 5
+  %tmp6 = insertelement <32 x float> %tmp5, float %init, i32 6
+  %tmp7 = insertelement <32 x float> %tmp6, float %init, i32 7
+  %tmp8 = insertelement <32 x float> %tmp7, float %init, i32 8
+  %tmp9 = insertelement <32 x float> %tmp8, float %init, i32 9
+  %tmp10 = insertelement <32 x float> %tmp9, float %init, i32 10
+  %tmp11 = insertelement <32 x float> %tmp10, float %init, i32 11
+  %tmp12 = insertelement <32 x float> %tmp11, float %init, i32 12
+  %tmp13 = insertelement <32 x float> %tmp12, float %init, i32 13
+  %tmp14 = insertelement <32 x float> %tmp13, float %init, i32 14
+  %tmp15 = insertelement <32 x float> %tmp14, float %init, i32 15
+  %tmp16 = insertelement <32 x float> %tmp15, float %init, i32 16
+  %tmp17 = insertelement <32 x float> %tmp16, float %init, i32 17
+  %tmp18 = insertelement <32 x float> %tmp17, float %init, i32 18
+  %tmp19 = insertelement <32 x float> %tmp18, float %init, i32 19
+  %tmp20 = insertelement <32 x float> %tmp19, float %init, i32 20
+  %tmp21 = insertelement <32 x float> %tmp20, float %init, i32 21
+  %tmp22 = insertelement <32 x float> %tmp21, float %init, i32 22
+  %tmp23 = insertelement <32 x float> %tmp22, float %init, i32 23
+  %tmp24 = insertelement <32 x float> %tmp23, float %init, i32 24
+  %tmp25 = insertelement <32 x float> %tmp24, float %init, i32 25
+  %tmp26 = insertelement <32 x float> %tmp25, float %init, i32 26
+  %tmp27 = insertelement <32 x float> %tmp26, float %init, i32 27
+  %tmp28 = insertelement <32 x float> %tmp27, float %init, i32 28
+  %tmp29 = insertelement <32 x float> %tmp28, float %init, i32 29
+  %tmp30 = insertelement <32 x float> %tmp29, float %init, i32 30
+  %tmp31 = insertelement <32 x float> %tmp30, float %init, i32 31
+
+  br label %for.cond.preheader
+
+for.cond.preheader:
+  %phi = phi <32 x float> [ %tmp31, %entry ], [ %mai.1, %for.cond.preheader ]
+  %c = phi i32 [ 0, %entry ], [ %inc, %for.cond.preheader ]
+  %mai.1 = tail call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.0, float 2.0, <32 x float> %phi, i32 0, i32 0, i32 0)
+  %inc = add nuw nsw i32 %c, 1
+  %cc = icmp eq i32 %inc, 16
+  br i1 %cc, label %exit, label %for.cond.preheader
+
+exit:
+  store <32 x float> %mai.1, <32 x float> addrspace(1)* %arg
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_mfma_loop_mixed_init:
+
+; GCN-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, v0
+; GCN-DAG: v_mov_b32_e32 [[TMP:v[0-9]+]], s{{[0-9]+}}
+; GCN-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, [[TMP]]
+; GCN-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, 0{{$}}
+; GCN-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, 0{{$}}
+; GCN-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, 0{{$}}
+; GCN-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, 0{{$}}
+; GCN-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, 0{{$}}
+; GCN-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, 0{{$}}
+; GCN-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, 0{{$}}
+; GCN-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, 0{{$}}
+; GCN-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, 0{{$}}
+; GCN-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, 0{{$}}
+; GCN-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, 0{{$}}
+; GCN-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, 0{{$}}
+; GCN-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, 0{{$}}
+; GCN-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, 0{{$}}
+; GCN-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, 0{{$}}
+; GCN-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, 0{{$}}
+; GCN-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, 0{{$}}
+; GCN-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, 0{{$}}
+; GCN-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, 0{{$}}
+; GCN-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, 0{{$}}
+; GCN-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, 0{{$}}
+; GCN-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, 0{{$}}
+; GCN-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, 0{{$}}
+; GCN-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, 0{{$}}
+; GCN-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, 0{{$}}
+; GCN-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, 0{{$}}
+; GCN-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, 0{{$}}
+; GCN-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, 0{{$}}
+; GCN-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, 0{{$}}
+; GCN-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, 0{{$}}
+
+; GCN: [[LOOP:BB[0-9_]+]]:
+; GCN-NOT: v_accvgpr
+; GCN: v_mfma_f32_32x32x1f32
+; GCN-NOT: v_accvgpr
+; GCN: s_cbranch_scc1 [[LOOP]]
+
+; GCN-COUNT-32: v_accvgpr_read_b32
+
+define amdgpu_kernel void @test_mfma_loop_mixed_init(<32 x float> addrspace(1)* %arg, float %x) {
+entry:
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %init = bitcast i32 %tid to float
+  %tmp0 = insertelement <32 x float> zeroinitializer, float %init, i32 0
+  %tmp1 = insertelement <32 x float> %tmp0, float %x, i32 1
+
+  br label %for.cond.preheader
+
+for.cond.preheader:
+  %phi = phi <32 x float> [ %tmp1, %entry ], [ %mai.1, %for.cond.preheader ]
+  %c = phi i32 [ 0, %entry ], [ %inc, %for.cond.preheader ]
+  %mai.1 = tail call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.0, float 2.0, <32 x float> %phi, i32 0, i32 0, i32 0)
+  %inc = add nuw nsw i32 %c, 1
+  %cc = icmp eq i32 %inc, 16
+  br i1 %cc, label %exit, label %for.cond.preheader
+
+exit:
+  store <32 x float> %mai.1, <32 x float> addrspace(1)* %arg
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_mfma_loop_mfma_forward_init:
+
+; GCN-COUNT-32: v_accvgpr_write_b32 a{{[0-9]+}}, 0
+; GCN: v_mfma_f32_32x32x1f32
+; GCN-NOT: v_accvgpr
+
+; GCN: [[LOOP:BB[0-9_]+]]:
+; GCN-NOT: v_accvgpr
+; GCN: v_mfma_f32_32x32x1f32
+; GCN-NOT: v_accvgpr
+; GCN: s_cbranch_scc1 [[LOOP]]
+
+; GCN-COUNT-32: v_accvgpr_read_b32
+define amdgpu_kernel void @test_mfma_loop_mfma_forward_init(<32 x float> addrspace(1)* %arg) {
+entry:
+  %mai.0 = tail call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.0, float 2.0, <32 x float> zeroinitializer, i32 0, i32 0, i32 0)
+
+  br label %for.cond.preheader
+
+for.cond.preheader:
+  %phi = phi <32 x float> [ %mai.0, %entry ], [ %mai.1, %for.cond.preheader ]
+  %c = phi i32 [ 0, %entry ], [ %inc, %for.cond.preheader ]
+  %mai.1 = tail call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.0, float 2.0, <32 x float> %phi, i32 0, i32 0, i32 0)
+  %inc = add nuw nsw i32 %c, 1
+  %cc = icmp eq i32 %inc, 16
+  br i1 %cc, label %exit, label %for.cond.preheader
+
+exit:
+  store <32 x float> %mai.1, <32 x float> addrspace(1)* %arg
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_mfma_loop_agpr_init:
+
+; GCN-COUNT-32: v_accvgpr_write_b32 a{{[0-9]+}}, 0
+; GCN: v_mfma_f32_32x32x1f32
+
+; Check that we are using only one tmp VGPR.
+
+; GCN: v_accvgpr_read_b32 [[TMP:v[0-9]+]], a{{[0-9]+}}
+; GCN-COUNT-31: v_accvgpr_write_b32 a{{[0-9]+}}, [[TMP]]{{$}}
+
+; GCN: [[LOOP:BB[0-9_]+]]:
+; GCN-NOT: v_accvgpr
+; GCN: v_mfma_f32_32x32x1f32
+; GCN-NOT: v_accvgpr
+; GCN: s_cbranch_scc1 [[LOOP]]
+
+; GCN-COUNT-32: v_accvgpr_read_b32
+define amdgpu_kernel void @test_mfma_loop_agpr_init(<32 x float> addrspace(1)* %arg) {
+entry:
+  %mai.0 = tail call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.0, float 2.0, <32 x float> zeroinitializer, i32 0, i32 0, i32 0)
+  %init = extractelement <32 x float> %mai.0, i32 0
+  %tmp0 = insertelement <32 x float> undef, float %init, i32 0
+  %tmp1 = insertelement <32 x float> %tmp0, float %init, i32 1
+  %tmp2 = insertelement <32 x float> %tmp1, float %init, i32 2
+  %tmp3 = insertelement <32 x float> %tmp2, float %init, i32 3
+  %tmp4 = insertelement <32 x float> %tmp3, float %init, i32 4
+  %tmp5 = insertelement <32 x float> %tmp4, float %init, i32 5
+  %tmp6 = insertelement <32 x float> %tmp5, float %init, i32 6
+  %tmp7 = insertelement <32 x float> %tmp6, float %init, i32 7
+  %tmp8 = insertelement <32 x float> %tmp7, float %init, i32 8
+  %tmp9 = insertelement <32 x float> %tmp8, float %init, i32 9
+  %tmp10 = insertelement <32 x float> %tmp9, float %init, i32 10
+  %tmp11 = insertelement <32 x float> %tmp10, float %init, i32 11
+  %tmp12 = insertelement <32 x float> %tmp11, float %init, i32 12
+  %tmp13 = insertelement <32 x float> %tmp12, float %init, i32 13
+  %tmp14 = insertelement <32 x float> %tmp13, float %init, i32 14
+  %tmp15 = insertelement <32 x float> %tmp14, float %init, i32 15
+  %tmp16 = insertelement <32 x float> %tmp15, float %init, i32 16
+  %tmp17 = insertelement <32 x float> %tmp16, float %init, i32 17
+  %tmp18 = insertelement <32 x float> %tmp17, float %init, i32 18
+  %tmp19 = insertelement <32 x float> %tmp18, float %init, i32 19
+  %tmp20 = insertelement <32 x float> %tmp19, float %init, i32 20
+  %tmp21 = insertelement <32 x float> %tmp20, float %init, i32 21
+  %tmp22 = insertelement <32 x float> %tmp21, float %init, i32 22
+  %tmp23 = insertelement <32 x float> %tmp22, float %init, i32 23
+  %tmp24 = insertelement <32 x float> %tmp23, float %init, i32 24
+  %tmp25 = insertelement <32 x float> %tmp24, float %init, i32 25
+  %tmp26 = insertelement <32 x float> %tmp25, float %init, i32 26
+  %tmp27 = insertelement <32 x float> %tmp26, float %init, i32 27
+  %tmp28 = insertelement <32 x float> %tmp27, float %init, i32 28
+  %tmp29 = insertelement <32 x float> %tmp28, float %init, i32 29
+  %tmp30 = insertelement <32 x float> %tmp29, float %init, i32 30
+  %tmp31 = insertelement <32 x float> %tmp30, float %init, i32 31
+
+  br label %for.cond.preheader
+
+for.cond.preheader:
+  %phi = phi <32 x float> [ %tmp31, %entry ], [ %mai.1, %for.cond.preheader ]
+  %c = phi i32 [ 0, %entry ], [ %inc, %for.cond.preheader ]
+  %mai.1 = tail call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.0, float 2.0, <32 x float> %phi, i32 0, i32 0, i32 0)
+  %inc = add nuw nsw i32 %c, 1
+  %cc = icmp eq i32 %inc, 16
+  br i1 %cc, label %exit, label %for.cond.preheader
+
+exit:
+  store <32 x float> %mai.1, <32 x float> addrspace(1)* %arg
+  ret void
+}
+
+declare <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float, float, <32 x float>, i32, i32, i32)
+declare i32 @llvm.amdgcn.workitem.id.x()
diff --git a/llvm/test/CodeGen/AMDGPU/min.ll b/llvm/test/CodeGen/AMDGPU/min.ll
index 2a16be4d6..a2c802915 100644
--- a/llvm/test/CodeGen/AMDGPU/min.ll
+++ b/llvm/test/CodeGen/AMDGPU/min.ll
@@ -404,7 +404,7 @@ define amdgpu_kernel void @v_test_umin_ult_i32_multi_use(i32 addrspace(1)* %out0
 ; FUNC-LABEL: @v_test_umin_ult_i16_multi_use
 ; GCN-NOT: v_min
 ; GCN: v_cmp_lt_u32
-; GCN-NEXT: v_cndmask_b32
+; GCN: v_cndmask_b32
 ; GCN-NOT: v_min
 ; GCN: s_endpgm
 
diff --git a/llvm/test/CodeGen/AMDGPU/movreld-bug.ll b/llvm/test/CodeGen/AMDGPU/movreld-bug.ll
index d07cb5f3f..3071f18c4 100644
--- a/llvm/test/CodeGen/AMDGPU/movreld-bug.ll
+++ b/llvm/test/CodeGen/AMDGPU/movreld-bug.ll
@@ -1,10 +1,18 @@
-; RUN: llc -march=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
-; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -march=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,MOVREL %s
+; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,MOVREL %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GPRIDX %s
 
 ; GCN-LABEL: {{^}}main:
-; GCN: v_movreld_b32_e32 v0,
-; GCN: v_mov_b32_e32 v0, v1
-; GCN: ; return
+
+; MOVREL: s_mov_b32 m0, s0
+; MOVREL-NEXT: v_movreld_b32_e32 v0,
+
+; GPRIDX: s_set_gpr_idx_on s0, gpr_idx(DST)
+; GPRIDX-NEXT: v_mov_b32_e32 v0, 0
+; GPRIDX-NEXT: s_set_gpr_idx_off
+
+; GCN-NEXT: v_mov_b32_e32 v0, v1
+; GCN-NEXT: ; return
 define amdgpu_ps float @main(i32 inreg %arg) #0 {
 main_body:
   %tmp24 = insertelement <16 x float> undef, float 0.000000e+00, i32 %arg
diff --git a/llvm/test/CodeGen/AMDGPU/mul24-pass-ordering.ll b/llvm/test/CodeGen/AMDGPU/mul24-pass-ordering.ll
new file mode 100644
index 000000000..af34b0f39
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/mul24-pass-ordering.ll
@@ -0,0 +1,261 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX9 %s
+
+; Make sure that AMDGPUCodeGenPrepare introduces mul24 intrinsics
+; after SLSR, as the intrinsics would interfere. It's unclear if these
+; should be introduced before LSR or not. It seems to help in some
+; cases, and hurt others.
+
+define void @lsr_order_mul24_0(i32 %arg, i32 %arg2, i32 %arg6, i32 %arg13, i32 %arg16) #0 {
+; GFX9-LABEL: lsr_order_mul24_0:
+; GFX9:       ; %bb.0: ; %bb
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dword v5, v[0:1], off
+; GFX9-NEXT:    v_and_b32_e32 v2, 0xffffff, v2
+; GFX9-NEXT:    v_sub_u32_e32 v4, v4, v1
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ds_write_b32 v0, v5
+; GFX9-NEXT:  BB0_1: ; %bb23
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    v_mul_u32_u24_e32 v5, v0, v2
+; GFX9-NEXT:    v_add_u32_e32 v0, v0, v1
+; GFX9-NEXT:    v_sub_u32_e32 v5, v4, v5
+; GFX9-NEXT:    v_add_u32_e32 v5, v5, v0
+; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v5, v3
+; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execnz BB0_1
+; GFX9-NEXT:  ; %bb.2: ; %.loopexit
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+bb:
+  %tmp22 = and i32 %arg6, 16777215
+  br label %bb23
+
+.loopexit:                                        ; preds = %bb23
+  ret void
+
+bb23:                                             ; preds = %bb23, %bb
+  %tmp24 = phi i32 [ %arg, %bb ], [ %tmp47, %bb23 ]
+  %tmp28 = and i32 %tmp24, 16777215
+  %tmp29 = mul i32 %tmp28, %tmp22
+  %tmp30 = sub i32 %tmp24, %tmp29
+  %tmp31 = add i32 %tmp30, %arg16
+  %tmp37 = icmp ult i32 %tmp31, %arg13
+  %tmp44 = load float, float addrspace(1)* undef, align 4
+  store float %tmp44, float addrspace(3)* undef, align 4
+  %tmp47 = add i32 %tmp24, %arg2
+  br i1 %tmp37, label %bb23, label %.loopexit
+}
+
+define void @lsr_order_mul24_1(i32 %arg, i32 %arg1, i32 %arg2, float addrspace(3)* nocapture %arg3, i32 %arg4, i32 %arg5, i32 %arg6, i32 %arg7, i32 %arg8, i32 %arg9, float addrspace(1)* nocapture readonly %arg10, i32 %arg11, i32 %arg12, i32 %arg13, i32 %arg14, i32 %arg15, i32 %arg16, i1 zeroext %arg17, i1 zeroext %arg18) #0 {
+; GFX9-LABEL: lsr_order_mul24_1:
+; GFX9:       ; %bb.0: ; %bb
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v5, 1, v18
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v5
+; GFX9-NEXT:    v_cmp_lt_u32_e64 s[4:5], v0, v1
+; GFX9-NEXT:    s_and_saveexec_b64 s[10:11], s[4:5]
+; GFX9-NEXT:    ; mask branch BB1_4
+; GFX9-NEXT:    s_cbranch_execz BB1_4
+; GFX9-NEXT:  BB1_1: ; %bb19
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v7, v6
+; GFX9-NEXT:    v_and_b32_e32 v5, 0xffffff, v6
+; GFX9-NEXT:    v_add_u32_e32 v6, v4, v0
+; GFX9-NEXT:    v_lshl_add_u32 v3, v6, 2, v3
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v4, v7
+; GFX9-NEXT:    v_lshlrev_b32_e32 v6, 2, v2
+; GFX9-NEXT:    v_add_u32_e32 v7, v17, v12
+; GFX9-NEXT:    s_mov_b64 s[12:13], 0
+; GFX9-NEXT:  BB1_2: ; %bb23
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v8, v0
+; GFX9-NEXT:    v_add_u32_e32 v9, v17, v0
+; GFX9-NEXT:    v_add_u32_e32 v12, v7, v0
+; GFX9-NEXT:    v_add_u32_e32 v0, v0, v2
+; GFX9-NEXT:    v_madak_f32 v8, v8, v4, 0x3727c5ac
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v8, v8
+; GFX9-NEXT:    v_cmp_ge_u32_e64 s[4:5], v0, v1
+; GFX9-NEXT:    v_mul_u32_u24_e32 v18, v8, v5
+; GFX9-NEXT:    v_add_u32_e32 v8, v8, v16
+; GFX9-NEXT:    v_cmp_lt_u32_e64 s[6:7], v8, v13
+; GFX9-NEXT:    v_mul_lo_u32 v8, v8, v15
+; GFX9-NEXT:    v_sub_u32_e32 v19, v9, v18
+; GFX9-NEXT:    v_cmp_lt_u32_e64 s[8:9], v19, v14
+; GFX9-NEXT:    s_and_b64 s[6:7], s[6:7], s[8:9]
+; GFX9-NEXT:    v_sub_u32_e32 v12, v12, v18
+; GFX9-NEXT:    s_and_b64 s[6:7], s[6:7], vcc
+; GFX9-NEXT:    v_add_u32_e32 v8, v12, v8
+; GFX9-NEXT:    v_mov_b32_e32 v9, 0
+; GFX9-NEXT:    v_cndmask_b32_e64 v8, 0, v8, s[6:7]
+; GFX9-NEXT:    v_lshlrev_b64 v[8:9], 2, v[8:9]
+; GFX9-NEXT:    s_or_b64 s[12:13], s[4:5], s[12:13]
+; GFX9-NEXT:    v_add_co_u32_e64 v8, s[4:5], v10, v8
+; GFX9-NEXT:    v_addc_co_u32_e64 v9, s[4:5], v11, v9, s[4:5]
+; GFX9-NEXT:    global_load_dword v8, v[8:9], off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cndmask_b32_e64 v8, 0, v8, s[6:7]
+; GFX9-NEXT:    ds_write_b32 v3, v8
+; GFX9-NEXT:    v_add_u32_e32 v3, v3, v6
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[12:13]
+; GFX9-NEXT:    s_cbranch_execnz BB1_2
+; GFX9-NEXT:  ; %bb.3: ; %Flow
+; GFX9-NEXT:    s_or_b64 exec, exec, s[12:13]
+; GFX9-NEXT:  BB1_4: ; %Flow3
+; GFX9-NEXT:    s_or_b64 exec, exec, s[10:11]
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+bb:
+  %tmp = icmp ult i32 %arg, %arg1
+  br i1 %tmp, label %bb19, label %.loopexit
+
+bb19:                                             ; preds = %bb
+  %tmp20 = uitofp i32 %arg6 to float
+  %tmp21 = fdiv float 1.000000e+00, %tmp20
+  %tmp22 = and i32 %arg6, 16777215
+  br label %bb23
+
+.loopexit:                                        ; preds = %bb23, %bb
+  ret void
+
+bb23:                                             ; preds = %bb19, %bb23
+  %tmp24 = phi i32 [ %arg, %bb19 ], [ %tmp47, %bb23 ]
+  %tmp25 = uitofp i32 %tmp24 to float
+  %tmp26 = tail call float @llvm.fmuladd.f32(float %tmp25, float %tmp21, float 0x3EE4F8B580000000) #2
+  %tmp27 = fptoui float %tmp26 to i32
+  %tmp28 = and i32 %tmp27, 16777215
+  %tmp29 = mul i32 %tmp28, %tmp22
+  %tmp30 = sub i32 %tmp24, %tmp29
+  %tmp31 = add i32 %tmp30, %arg16
+  %tmp32 = add i32 %tmp27, %arg15
+  %tmp33 = mul i32 %tmp32, %arg14
+  %tmp34 = add i32 %tmp33, %arg11
+  %tmp35 = add i32 %tmp34, %tmp31
+  %tmp36 = add i32 %tmp24, %arg4
+  %tmp37 = icmp ult i32 %tmp31, %arg13
+  %tmp38 = icmp ult i32 %tmp32, %arg12
+  %tmp39 = and i1 %tmp38, %tmp37
+  %tmp40 = and i1 %tmp39, %arg17
+  %tmp41 = zext i32 %tmp35 to i64
+  %tmp42 = select i1 %tmp40, i64 %tmp41, i64 0
+  %tmp43 = getelementptr inbounds float, float addrspace(1)* %arg10, i64 %tmp42
+  %tmp44 = load float, float addrspace(1)* %tmp43, align 4
+  %tmp45 = select i1 %tmp40, float %tmp44, float 0.000000e+00
+  %tmp46 = getelementptr inbounds float, float addrspace(3)* %arg3, i32 %tmp36
+  store float %tmp45, float addrspace(3)* %tmp46, align 4
+  %tmp47 = add i32 %tmp24, %arg2
+  %tmp48 = icmp ult i32 %tmp47, %arg1
+  br i1 %tmp48, label %bb23, label %.loopexit
+}
+
+define void @slsr1_0(i32 %b.arg, i32 %s.arg) #0 {
+; GFX9-LABEL: slsr1_0:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mul_u32_u24_e32 v3, v0, v1
+; GFX9-NEXT:    v_and_b32_e32 v2, 0xffffff, v1
+; GFX9-NEXT:    global_store_dword v[0:1], v3, off
+; GFX9-NEXT:    v_mad_u32_u24 v0, v0, v1, v2
+; GFX9-NEXT:    global_store_dword v[0:1], v0, off
+; GFX9-NEXT:    v_add_u32_e32 v0, v0, v2
+; GFX9-NEXT:    global_store_dword v[0:1], v0, off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %b = and i32 %b.arg, 16777215
+  %s = and i32 %s.arg, 16777215
+
+; CHECK-LABEL: @slsr1(
+  ; foo(b * s);
+  %mul0 = mul i32 %b, %s
+; CHECK: mul i32
+; CHECK-NOT: mul i32
+  store volatile i32 %mul0, i32 addrspace(1)* undef
+
+  ; foo((b + 1) * s);
+  %b1 = add i32 %b, 1
+  %mul1 = mul i32 %b1, %s
+  store volatile i32 %mul1, i32 addrspace(1)* undef
+
+  ; foo((b + 2) * s);
+  %b2 = add i32 %b, 2
+  %mul2 = mul i32 %b2, %s
+  store volatile i32 %mul2, i32 addrspace(1)* undef
+  ret void
+}
+
+define void @slsr1_1(i32 %b.arg, i32 %s.arg) #0 {
+; GFX9-LABEL: slsr1_1:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_or_saveexec_b64 s[4:5], -1
+; GFX9-NEXT:    buffer_store_dword v35, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    s_mov_b64 exec, s[4:5]
+; GFX9-NEXT:    v_writelane_b32 v35, s34, 4
+; GFX9-NEXT:    s_mov_b32 s34, s32
+; GFX9-NEXT:    s_add_u32 s32, s32, 0x800
+; GFX9-NEXT:    buffer_store_dword v32, off, s[0:3], s34 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v33, off, s[0:3], s34 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v34, off, s[0:3], s34 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_writelane_b32 v35, s36, 0
+; GFX9-NEXT:    s_getpc_b64 s[4:5]
+; GFX9-NEXT:    s_add_u32 s4, s4, foo@gotpcrel32@lo+4
+; GFX9-NEXT:    s_addc_u32 s5, s5, foo@gotpcrel32@hi+4
+; GFX9-NEXT:    v_writelane_b32 v35, s37, 1
+; GFX9-NEXT:    s_load_dwordx2 s[36:37], s[4:5], 0x0
+; GFX9-NEXT:    v_mov_b32_e32 v32, v1
+; GFX9-NEXT:    v_mov_b32_e32 v33, v0
+; GFX9-NEXT:    v_writelane_b32 v35, s30, 2
+; GFX9-NEXT:    v_mul_u32_u24_e32 v0, v33, v32
+; GFX9-NEXT:    v_writelane_b32 v35, s31, 3
+; GFX9-NEXT:    v_and_b32_e32 v34, 0xffffff, v32
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_swappc_b64 s[30:31], s[36:37]
+; GFX9-NEXT:    v_mad_u32_u24 v32, v33, v32, v34
+; GFX9-NEXT:    v_mov_b32_e32 v0, v32
+; GFX9-NEXT:    s_swappc_b64 s[30:31], s[36:37]
+; GFX9-NEXT:    v_add_u32_e32 v0, v32, v34
+; GFX9-NEXT:    s_swappc_b64 s[30:31], s[36:37]
+; GFX9-NEXT:    v_readlane_b32 s4, v35, 2
+; GFX9-NEXT:    v_readlane_b32 s5, v35, 3
+; GFX9-NEXT:    v_readlane_b32 s37, v35, 1
+; GFX9-NEXT:    v_readlane_b32 s36, v35, 0
+; GFX9-NEXT:    buffer_load_dword v34, off, s[0:3], s34 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v33, off, s[0:3], s34 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s34 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_sub_u32 s32, s32, 0x800
+; GFX9-NEXT:    v_readlane_b32 s34, v35, 4
+; GFX9-NEXT:    s_or_saveexec_b64 s[6:7], -1
+; GFX9-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b64 exec, s[6:7]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[4:5]
+  %b = and i32 %b.arg, 16777215
+  %s = and i32 %s.arg, 16777215
+
+; CHECK-LABEL: @slsr1(
+  ; foo(b * s);
+  %mul0 = mul i32 %b, %s
+; CHECK: mul i32
+; CHECK-NOT: mul i32
+  call void @foo(i32 %mul0)
+
+  ; foo((b + 1) * s);
+  %b1 = add i32 %b, 1
+  %mul1 = mul i32 %b1, %s
+  call void @foo(i32 %mul1)
+
+  ; foo((b + 2) * s);
+  %b2 = add i32 %b, 2
+  %mul2 = mul i32 %b2, %s
+  call void @foo(i32 %mul2)
+
+  ret void
+}
+
+declare void @foo(i32) #0
+declare float @llvm.fmuladd.f32(float, float, float) #1
+
+attributes #0 = { nounwind willreturn }
+attributes #1 = { nounwind readnone speculatable }
diff --git a/llvm/test/CodeGen/AMDGPU/mul_uint24-amdgcn.ll b/llvm/test/CodeGen/AMDGPU/mul_uint24-amdgcn.ll
index 3ced47085..686c775a5 100644
--- a/llvm/test/CodeGen/AMDGPU/mul_uint24-amdgcn.ll
+++ b/llvm/test/CodeGen/AMDGPU/mul_uint24-amdgcn.ll
@@ -205,7 +205,7 @@ entry:
 ; GCN-NOT: and
 ; GCN-NOT: lshr
 ; GCN: v_mul_hi_u32_u24_e32 v[[MUL_HI:[0-9]+]],
-; GCN-NEXT: v_and_b32_e32 v[[HI:[0-9]+]], 1, v[[MUL_HI]]
+; GCN: v_and_b32_e32 v[[HI:[0-9]+]], 1, v[[MUL_HI]]
 ; GCN-NEXT: buffer_store_dword v[[HI]]
 define amdgpu_kernel void @test_umulhi24_i33(i32 addrspace(1)* %out, i33 %a, i33 %b) {
 entry:
diff --git a/llvm/test/CodeGen/AMDGPU/multilevel-break.ll b/llvm/test/CodeGen/AMDGPU/multilevel-break.ll
index 0ae28c6ef..5222ae56d 100644
--- a/llvm/test/CodeGen/AMDGPU/multilevel-break.ll
+++ b/llvm/test/CodeGen/AMDGPU/multilevel-break.ll
@@ -25,39 +25,29 @@
 ; GCN:      s_mov_b64           [[LEFT_OUTER:s\[[0-9]+:[0-9]+\]]], 0{{$}}
 
 ; GCN: [[FLOW2:BB[0-9]+_[0-9]+]]: ; %Flow2
-; GCN:      s_or_b64            exec, exec, [[TMP0:s\[[0-9]+:[0-9]+\]]]
+; GCN:      s_or_b64            exec, exec, [[LEFT_INNER:s\[[0-9]+:[0-9]+\]]]
 ; GCN:      s_and_b64           [[TMP1:s\[[0-9]+:[0-9]+\]]], exec, [[BREAK_OUTER:s\[[0-9]+:[0-9]+\]]]
-; GCN:      s_or_b64            [[TMP1]], [[TMP1]], [[LEFT_OUTER]]
-; GCN:      s_mov_b64           [[LEFT_OUTER]], [[TMP1]]
-; GCN:      s_andn2_b64         exec, exec, [[TMP1]]
+; GCN:      s_or_b64            [[LEFT_OUTER:s\[[0-9]+:[0-9]+\]]], [[TMP1]], [[LEFT_OUTER]]
+; GCN:      s_andn2_b64         exec, exec, [[LEFT_OUTER]]
 ; GCN:      s_cbranch_execz    [[IF_BLOCK:BB[0-9]+_[0-9]+]]
 
 ; GCN: [[OUTER_LOOP:BB[0-9]+_[0-9]+]]: ; %LOOP.outer{{$}}
-; GCN:      s_mov_b64           [[LEFT_INNER:s\[[0-9]+:[0-9]+\]]], 0{{$}}
+; GCN:      s_mov_b64           [[LEFT_INNER]], 0{{$}}
 
 ; GCN: ; %Flow
 ; GCN:      s_or_b64            exec, exec, [[SAVE_EXEC:s\[[0-9]+:[0-9]+\]]]
-; GCN:      s_and_b64           [[TMP0]], exec, [[BREAK_INNER:s\[[0-9]+:[0-9]+\]]]
-; GCN:      s_or_b64            [[TMP0]], [[TMP0]], [[LEFT_INNER]]
-; GCN:      s_mov_b64           [[LEFT_INNER]], [[TMP0]]
-; GCN:      s_andn2_b64         exec, exec, [[TMP0]]
+; GCN:      s_and_b64           [[TMP0:s\[[0-9]+:[0-9]+\]]], exec, [[BREAK_INNER:s\[[0-9]+:[0-9]+\]]]
+; GCN:      s_or_b64            [[LEFT_INNER]], [[TMP0]], [[LEFT_INNER]]
+; GCN:      s_andn2_b64         exec, exec, [[LEFT_INNER]]
 ; GCN:      s_cbranch_execz    [[FLOW2]]
 
 ; GCN: [[INNER_LOOP:BB[0-9]+_[0-9]+]]: ; %LOOP{{$}}
-; GCN:      s_or_b64            [[BREAK_OUTER]], [[BREAK_OUTER]], exec
-; GCN:      s_or_b64            [[BREAK_INNER]], [[BREAK_INNER]], exec
 ; GCN:      s_and_saveexec_b64  [[SAVE_EXEC]], vcc
 
 ; FIXME: duplicate comparison
 ; GCN: ; %ENDIF
 ; GCN-DAG:  v_cmp_eq_u32_e32    vcc,
 ; GCN-DAG:  v_cmp_ne_u32_e64    [[TMP51NEG:s\[[0-9]+:[0-9]+\]]],
-; GCN-DAG:  s_andn2_b64         [[BREAK_OUTER]], [[BREAK_OUTER]], exec
-; GCN-DAG:  s_andn2_b64         [[BREAK_INNER]], [[BREAK_INNER]], exec
-; GCN-DAG:  s_and_b64           [[TMP_EQ:s\[[0-9]+:[0-9]+\]]], vcc, exec
-; GCN-DAG:  s_and_b64           [[TMP_NE:s\[[0-9]+:[0-9]+\]]], [[TMP51NEG]], exec
-; GCN-DAG:  s_or_b64            [[BREAK_OUTER]], [[BREAK_OUTER]], [[TMP_EQ]]
-; GCN-DAG:  s_or_b64            [[BREAK_INNER]], [[BREAK_INNER]], [[TMP_NE]]
 
 ; GCN: [[IF_BLOCK]]: ; %IF
 ; GCN-NEXT: s_endpgm
@@ -90,39 +80,47 @@ ENDIF:                                            ; preds = %LOOP
 ; OPT: llvm.amdgcn.end.cf
 
 ; GCN-LABEL: {{^}}multi_if_break_loop:
-; GCN:      s_mov_b64          [[LEFT:s\[[0-9]+:[0-9]+\]]], 0{{$}}
+; GCN:      s_mov_b64          [[SAVED_MASK:s\[[0-9]+:[0-9]+\]]], 0{{$}}
 
-; GCN: ; %Flow4
-; GCN:      s_and_b64          [[BREAK:s\[[0-9]+:[0-9]+\]]], exec, [[BREAK]]
-; GCN:      s_or_b64           [[LEFT]], [[BREAK]], [[OLD_LEFT:s\[[0-9]+:[0-9]+\]]]
-; GCN:      s_andn2_b64        exec, exec, [[LEFT]]
-; GCN-NEXT: s_cbranch_execz
+; GCN: [[LOOP:BB[0-9]+_[0-9]+]]: ; %Flow4
+; GCN:      s_and_b64          [[ANDTMP0:s\[[0-9]+:[0-9]+\]]], exec, {{s\[[0-9]+:[0-9]+\]}}
+; GCN:      s_or_b64           [[MASK1:s\[[0-9]+:[0-9]+\]]], [[ANDTMP0]], [[SAVED_MASK]]
+; GCN:      s_and_b64          [[BROKEN_THREADS_MASK:s\[[0-9]+:[0-9]+\]]], {{s\[[0-9]+:[0-9]+\]}}, exec
+; GCN:      s_andn2_b64        exec, exec, [[MASK1]]
+; GCN-NEXT: s_cbranch_execz [[LOOP_EXIT:BB[0-9]+_[0-9]+]]
 
-; GCN: [[LOOP:BB[0-9]+_[0-9]+]]: ; %bb1{{$}}
-; GCN:      s_mov_b64          [[OLD_LEFT]], [[LEFT]]
+; GCN: ; %bb1{{$}}
+; GCN:      buffer_load_dword  [[LOAD0:v[0-9]+]],
 
 ; GCN: ; %LeafBlock1
-; GCN:      s_mov_b64
-; GCN:      s_mov_b64          [[BREAK]], -1{{$}}
+; GCN:      v_cmp_eq_u32_e32 vcc, 1, [[LOAD0]]
+; GCN:      s_and_b64 vcc, exec, vcc
+; GCN:      s_cbranch_vccz [[FLOW:BB[0-9]+_[0-9]+]]
 
 ; GCN: ; %case1
 ; GCN:      buffer_load_dword  [[LOAD2:v[0-9]+]],
 ; GCN:      v_cmp_ge_i32_e32   vcc, {{v[0-9]+}}, [[LOAD2]]
-; GCN:      s_orn2_b64         [[BREAK]], vcc, exec
+; GCN:      s_orn2_b64 [[BROKEN_THREADS_MASK]], vcc, exec
+; GCN:  BB1_{{[0-9]+}}:
+; GCN:      s_mov_b64 [[FALSE_MASK:s\[[0-9]+:[0-9]+\]]], 0
+; GCN:      s_and_b64 vcc, exec, [[FALSE_MASK]]
+; GCN:      s_cbranch_vccz [[LOOP]]
 
-; GCN: ; %Flow3
-; GCN:      s_branch           [[FLOW:BB[0-9]+_[0-9]+]]
-
-; GCN:      s_mov_b64          [[BREAK]], -1{{$}}
-
-; GCN: [[FLOW]]: ; %Flow
+; GCN: ; %LeafBlock
+; GCN:     v_cmp_eq_u32_e32 vcc, 0, [[LOAD0]]
+; GCN:     s_and_b64 vcc, exec, vcc
+; GCN:     s_cbranch_vccz [[LOOP]]
 
 ; GCN: ; %case0
 ; GCN:      buffer_load_dword  [[LOAD1:v[0-9]+]],
-; GCN-DAG:  s_andn2_b64        [[BREAK]], [[BREAK]], exec
 ; GCN-DAG:  v_cmp_ge_i32_e32   vcc, {{v[0-9]+}}, [[LOAD1]]
-; GCN-DAG:  s_and_b64          [[TMP:s\[[0-9]+:[0-9]+\]]], vcc, exec
-; GCN:      s_or_b64           [[BREAK]], [[BREAK]], [[TMP]]
+; GCN:      s_andn2_b64 [[BROKEN_THREADS_MASK]], [[BROKEN_THREADS_MASK]], exec
+; GCN:      s_and_b64 [[TMP_MASK:s\[[0-9]+:[0-9]+\]]], vcc, exec
+; GCN:      s_or_b64 [[BROKEN_THREADS_MASK]], [[BROKEN_THREADS_MASK]], [[TMP_MASK]]
+; GCN:      s_branch [[LOOP]]
+
+; GCN: [[LOOP_EXIT]]: ; %Flow6
+; GCN: 	s_or_b64 exec, exec, [[SAVED_MASK]]
 
 define amdgpu_kernel void @multi_if_break_loop(i32 %arg) #0 {
 bb:
diff --git a/llvm/test/CodeGen/AMDGPU/no-initializer-constant-addrspace.ll b/llvm/test/CodeGen/AMDGPU/no-initializer-constant-addrspace.ll
index 886144a4a..2312112b7 100644
--- a/llvm/test/CodeGen/AMDGPU/no-initializer-constant-addrspace.ll
+++ b/llvm/test/CodeGen/AMDGPU/no-initializer-constant-addrspace.ll
@@ -1,8 +1,8 @@
-; RUN: llc -march=amdgcn -mcpu=tahiti -filetype=obj < %s | llvm-readobj -r --symbols | FileCheck %s -check-prefix=GCN
-; RUN: llc -march=amdgcn -mcpu=tonga -filetype=obj < %s | llvm-readobj -r --symbols | FileCheck %s -check-prefix=GCN
-; RUN: llc -march=r600 -mcpu=cypress -filetype=obj < %s | llvm-readobj -r --symbols | FileCheck %s -check-prefix=EG
+; RUN: llc -mtriple=amdgcn- -mcpu=tahiti -filetype=obj < %s | llvm-readobj -r --symbols | FileCheck %s -check-prefix=GCN
+; RUN: llc -mtriple=amdgcn- -mcpu=tonga -filetype=obj < %s | llvm-readobj -r --symbols | FileCheck %s -check-prefix=GCN
+; RUN: llc -mtriple=r600- -mcpu=cypress -filetype=obj < %s | llvm-readobj -r --symbols | FileCheck %s -check-prefix=EG
 
-; GCN: R_AMDGPU_REL32_LO extern_const_addrspace
+; GCN: R_AMDGPU_GOTPCREL32_LO extern_const_addrspace 0x0
 ; EG: R_AMDGPU_ABS32 extern_const_addrspace
 
 ; CHECK-DAG: Name: extern_const_addrspace
diff --git a/llvm/test/CodeGen/AMDGPU/occupancy-levels.ll b/llvm/test/CodeGen/AMDGPU/occupancy-levels.ll
new file mode 100644
index 000000000..eae3f11ba
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/occupancy-levels.ll
@@ -0,0 +1,288 @@
+; RUN: llc -march=amdgcn -mcpu=gfx900 < %s | FileCheck --check-prefixes=GCN,GFX9 %s
+; RUN: llc -march=amdgcn -mcpu=gfx1010 < %s | FileCheck --check-prefixes=GCN,GFX1010,GFX1010W32 %s
+; RUN: llc -march=amdgcn -mcpu=gfx1010 -mattr=+wavefrontsize64 < %s | FileCheck --check-prefixes=GCN,GFX1010,GFX1010W64 %s
+
+; GCN-LABEL: {{^}}max_occupancy:
+; GFX9:       ; Occupancy: 10
+; GFX1010:    ; Occupancy: 20
+define amdgpu_kernel void @max_occupancy() {
+  ret void
+}
+
+; GCN-LABEL: {{^}}limited_occupancy_3:
+; GFX9:       ; Occupancy: 3
+; GFX1010W64: ; Occupancy: 3
+; GFX1010W32: ; Occupancy: 4
+define amdgpu_kernel void @limited_occupancy_3() #0 {
+  ret void
+}
+
+; GCN-LABEL: {{^}}limited_occupancy_18:
+; GFX9:       ; Occupancy: 10
+; GFX1010:    ; Occupancy: 18
+define amdgpu_kernel void @limited_occupancy_18() #1 {
+  ret void
+}
+
+; GCN-LABEL: {{^}}limited_occupancy_19:
+; GFX9:       ; Occupancy: 10
+; GFX1010:    ; Occupancy: 18
+define amdgpu_kernel void @limited_occupancy_19() #2 {
+  ret void
+}
+
+; GCN-LABEL: {{^}}used_24_vgprs:
+; GFX9:       ; Occupancy: 10
+; GFX1010:    ; Occupancy: 20
+define amdgpu_kernel void @used_24_vgprs() {
+  call void asm sideeffect "", "~{v23}" ()
+  ret void
+}
+
+; GCN-LABEL: {{^}}used_28_vgprs:
+; GFX9:       ; Occupancy: 9
+; GFX1010W64: ; Occupancy: 18
+; GFX1010W32: ; Occupancy: 20
+define amdgpu_kernel void @used_28_vgprs() {
+  call void asm sideeffect "", "~{v27}" ()
+  ret void
+}
+
+; GCN-LABEL: {{^}}used_32_vgprs:
+; GFX9:       ; Occupancy: 8
+; GFX1010W64: ; Occupancy: 16
+; GFX1010W32: ; Occupancy: 20
+define amdgpu_kernel void @used_32_vgprs() {
+  call void asm sideeffect "", "~{v31}" ()
+  ret void
+}
+
+; GCN-LABEL: {{^}}used_36_vgprs:
+; GFX9:       ; Occupancy: 7
+; GFX1010W64: ; Occupancy: 14
+; GFX1010W32: ; Occupancy: 20
+define amdgpu_kernel void @used_36_vgprs() {
+  call void asm sideeffect "", "~{v35}" ()
+  ret void
+}
+
+; GCN-LABEL: {{^}}used_40_vgprs:
+; GFX9:       ; Occupancy: 6
+; GFX1010W64: ; Occupancy: 12
+; GFX1010W32: ; Occupancy: 20
+define amdgpu_kernel void @used_40_vgprs() {
+  call void asm sideeffect "", "~{v39}" ()
+  ret void
+}
+
+; GCN-LABEL: {{^}}used_44_vgprs:
+; GFX9:       ; Occupancy: 5
+; GFX1010W64: ; Occupancy: 11
+; GFX1010W32: ; Occupancy: 20
+define amdgpu_kernel void @used_44_vgprs() {
+  call void asm sideeffect "", "~{v43}" ()
+  ret void
+}
+
+; GCN-LABEL: {{^}}used_48_vgprs:
+; GFX9:       ; Occupancy: 5
+; GFX1010W64: ; Occupancy: 10
+; GFX1010W32: ; Occupancy: 20
+define amdgpu_kernel void @used_48_vgprs() {
+  call void asm sideeffect "", "~{v47}" ()
+  ret void
+}
+
+; GCN-LABEL: {{^}}used_56_vgprs:
+; GFX9:       ; Occupancy: 4
+; GFX1010W64: ; Occupancy: 9
+; GFX1010W32: ; Occupancy: 18
+define amdgpu_kernel void @used_56_vgprs() {
+  call void asm sideeffect "", "~{v55}" ()
+  ret void
+}
+
+; GCN-LABEL: {{^}}used_64_vgprs:
+; GFX9:       ; Occupancy: 4
+; GFX1010W64: ; Occupancy: 8
+; GFX1010W32: ; Occupancy: 16
+define amdgpu_kernel void @used_64_vgprs() {
+  call void asm sideeffect "", "~{v63}" ()
+  ret void
+}
+
+; GCN-LABEL: {{^}}used_72_vgprs:
+; GFX9:       ; Occupancy: 3
+; GFX1010W64: ; Occupancy: 7
+; GFX1010W32: ; Occupancy: 14
+define amdgpu_kernel void @used_72_vgprs() {
+  call void asm sideeffect "", "~{v71}" ()
+  ret void
+}
+
+; GCN-LABEL: {{^}}used_80_vgprs:
+; GFX9:       ; Occupancy: 3
+; GFX1010W64: ; Occupancy: 6
+; GFX1010W32: ; Occupancy: 12
+define amdgpu_kernel void @used_80_vgprs() {
+  call void asm sideeffect "", "~{v79}" ()
+  ret void
+}
+
+; GCN-LABEL: {{^}}used_84_vgprs:
+; GFX9:       ; Occupancy: 3
+; GFX1010W64: ; Occupancy: 6
+; GFX1010W32: ; Occupancy: 11
+define amdgpu_kernel void @used_84_vgprs() {
+  call void asm sideeffect "", "~{v83}" ()
+  ret void
+}
+
+; GCN-LABEL: {{^}}used_88_vgprs:
+; GFX9:       ; Occupancy: 2
+; GFX1010W64: ; Occupancy: 5
+; GFX1010W32: ; Occupancy: 11
+define amdgpu_kernel void @used_88_vgprs() {
+  call void asm sideeffect "", "~{v87}" ()
+  ret void
+}
+
+; GCN-LABEL: {{^}}used_96_vgprs:
+; GFX9:       ; Occupancy: 2
+; GFX1010W64: ; Occupancy: 5
+; GFX1010W32: ; Occupancy: 10
+define amdgpu_kernel void @used_96_vgprs() {
+  call void asm sideeffect "", "~{v95}" ()
+  ret void
+}
+
+; GCN-LABEL: {{^}}used_100_vgprs:
+; GFX9:       ; Occupancy: 2
+; GFX1010W64: ; Occupancy: 5
+; GFX1010W32: ; Occupancy: 9
+define amdgpu_kernel void @used_100_vgprs() {
+  call void asm sideeffect "", "~{v99}" ()
+  ret void
+}
+
+; GCN-LABEL: {{^}}used_112_vgprs:
+; GFX9:       ; Occupancy: 2
+; GFX1010W64: ; Occupancy: 4
+; GFX1010W32: ; Occupancy: 9
+define amdgpu_kernel void @used_112_vgprs() {
+  call void asm sideeffect "", "~{v111}" ()
+  ret void
+}
+
+; GCN-LABEL: {{^}}used_128_vgprs:
+; GFX9:       ; Occupancy: 2
+; GFX1010W64: ; Occupancy: 4
+; GFX1010W32: ; Occupancy: 8
+define amdgpu_kernel void @used_128_vgprs() {
+  call void asm sideeffect "", "~{v127}" ()
+  ret void
+}
+
+; GCN-LABEL: {{^}}used_144_vgprs:
+; GFX9:       ; Occupancy: 1
+; GFX1010W64: ; Occupancy: 3
+; GFX1010W32: ; Occupancy: 7
+define amdgpu_kernel void @used_144_vgprs() {
+  call void asm sideeffect "", "~{v143}" ()
+  ret void
+}
+
+; GCN-LABEL: {{^}}used_168_vgprs:
+; GFX9:       ; Occupancy: 1
+; GFX1010W64: ; Occupancy: 3
+; GFX1010W32: ; Occupancy: 6
+define amdgpu_kernel void @used_168_vgprs() {
+  call void asm sideeffect "", "~{v167}" ()
+  ret void
+}
+
+; GCN-LABEL: {{^}}used_200_vgprs:
+; GFX9:       ; Occupancy: 1
+; GFX1010W64: ; Occupancy: 2
+; GFX1010W32: ; Occupancy: 5
+define amdgpu_kernel void @used_200_vgprs() {
+  call void asm sideeffect "", "~{v199}" ()
+  ret void
+}
+
+; GCN-LABEL: {{^}}used_256_vgprs:
+; GFX9:       ; Occupancy: 1
+; GFX1010W64: ; Occupancy: 2
+; GFX1010W32: ; Occupancy: 4
+define amdgpu_kernel void @used_256_vgprs() {
+  call void asm sideeffect "", "~{v255}" ()
+  ret void
+}
+
+; GCN-LABEL: {{^}}used_80_sgprs:
+; GFX9:       ; Occupancy: 10
+; GFX1010:    ; Occupancy: 20
+define amdgpu_kernel void @used_80_sgprs() {
+  call void asm sideeffect "", "~{s79}" ()
+  ret void
+}
+
+; GCN-LABEL: {{^}}used_88_sgprs:
+; GFX9:       ; Occupancy: 9
+; GFX1010:    ; Occupancy: 20
+define amdgpu_kernel void @used_88_sgprs() {
+  call void asm sideeffect "", "~{s87}" ()
+  ret void
+}
+
+; GCN-LABEL: {{^}}used_100_sgprs:
+; GFX9:       ; Occupancy: 8
+; GFX1010:    ; Occupancy: 20
+define amdgpu_kernel void @used_100_sgprs() {
+  call void asm sideeffect "", "~{s99}" ()
+  ret void
+}
+
+; GCN-LABEL: {{^}}used_101_sgprs:
+; GFX9:       ; Occupancy: 7
+; GFX1010:    ; Occupancy: 20
+define amdgpu_kernel void @used_101_sgprs() {
+  call void asm sideeffect "", "~{s100}" ()
+  ret void
+}
+
+; GCN-LABEL: {{^}}used_lds_6552:
+; GFX9:       ; Occupancy: 10
+; GFX1010:    ; Occupancy: 20
+@lds6552 = internal addrspace(3) global [6552 x i8] undef, align 4
+define amdgpu_kernel void @used_lds_6552() {
+  %p = bitcast [6552 x i8] addrspace(3)* @lds6552 to i8 addrspace(3)*
+  store volatile i8 1, i8 addrspace(3)* %p
+  ret void
+}
+
+; GCN-LABEL: {{^}}used_lds_6556:
+; GFX9:       ; Occupancy: 10
+; GFX1010W64: ; Occupancy: 20
+; GFX1010W32: ; Occupancy: 20
+@lds6556 = internal addrspace(3) global [6556 x i8] undef, align 4
+define amdgpu_kernel void @used_lds_6556() {
+  %p = bitcast [6556 x i8] addrspace(3)* @lds6556 to i8 addrspace(3)*
+  store volatile i8 1, i8 addrspace(3)* %p
+  ret void
+}
+
+; GCN-LABEL: {{^}}used_lds_13112:
+; GFX9:       ; Occupancy: 10
+; GFX1010W64: ; Occupancy: 20
+; GFX1010W32: ; Occupancy: 20
+@lds13112 = internal addrspace(3) global [13112 x i8] undef, align 4
+define amdgpu_kernel void @used_lds_13112() {
+  %p = bitcast [13112 x i8] addrspace(3)* @lds13112 to i8 addrspace(3)*
+  store volatile i8 1, i8 addrspace(3)* %p
+  ret void
+}
+
+attributes #0 = { "amdgpu-waves-per-eu"="2,3" }
+attributes #1 = { "amdgpu-waves-per-eu"="18,18" }
+attributes #2 = { "amdgpu-waves-per-eu"="19,19" }
diff --git a/llvm/test/CodeGen/AMDGPU/offset-split-flat.ll b/llvm/test/CodeGen/AMDGPU/offset-split-flat.ll
new file mode 100644
index 000000000..f7538c081
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/offset-split-flat.ll
@@ -0,0 +1,1470 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX9 %s
+; RUN: llc -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX10 %s
+
+; Test splitting flat instruction offsets into the low and high bits
+; when the offset doesn't fit in the offset field.
+
+define i8 @flat_inst_valu_offset_1(i8* %p) {
+; GFX9-LABEL: flat_inst_valu_offset_1:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    flat_load_ubyte v0, v[0:1] offset:1
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: flat_inst_valu_offset_1:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, v0, 1
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %gep = getelementptr i8, i8* %p, i64 1
+  %load = load i8, i8* %gep, align 4
+  ret i8 %load
+}
+
+define i8 @flat_inst_valu_offset_11bit_max(i8* %p) {
+; GFX9-LABEL: flat_inst_valu_offset_11bit_max:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    flat_load_ubyte v0, v[0:1] offset:2047
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: flat_inst_valu_offset_11bit_max:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x7ff, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %gep = getelementptr i8, i8* %p, i64 2047
+  %load = load i8, i8* %gep, align 4
+  ret i8 %load
+}
+
+define i8 @flat_inst_valu_offset_12bit_max(i8* %p) {
+; GFX9-LABEL: flat_inst_valu_offset_12bit_max:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    flat_load_ubyte v0, v[0:1] offset:4095
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: flat_inst_valu_offset_12bit_max:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0xfff, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %gep = getelementptr i8, i8* %p, i64 4095
+  %load = load i8, i8* %gep, align 4
+  ret i8 %load
+}
+
+define i8 @flat_inst_valu_offset_13bit_max(i8* %p) {
+; GFX9-LABEL: flat_inst_valu_offset_13bit_max:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0x1000, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    flat_load_ubyte v0, v[0:1] offset:4095
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: flat_inst_valu_offset_13bit_max:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x1fff, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %gep = getelementptr i8, i8* %p, i64 8191
+  %load = load i8, i8* %gep, align 4
+  ret i8 %load
+}
+
+define i8 @flat_inst_valu_offset_neg_11bit_max(i8* %p) {
+; GFX9-LABEL: flat_inst_valu_offset_neg_11bit_max:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffff000, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX9-NEXT:    flat_load_ubyte v0, v[0:1] offset:2048
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: flat_inst_valu_offset_neg_11bit_max:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0xfffff800, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
+; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %gep = getelementptr i8, i8* %p, i64 -2048
+  %load = load i8, i8* %gep, align 4
+  ret i8 %load
+}
+
+define i8 @flat_inst_valu_offset_neg_12bit_max(i8* %p) {
+; GFX9-LABEL: flat_inst_valu_offset_neg_12bit_max:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffff000, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX9-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: flat_inst_valu_offset_neg_12bit_max:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0xfffff000, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
+; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %gep = getelementptr i8, i8* %p, i64 -4096
+  %load = load i8, i8* %gep, align 4
+  ret i8 %load
+}
+
+define i8 @flat_inst_valu_offset_neg_13bit_max(i8* %p) {
+; GFX9-LABEL: flat_inst_valu_offset_neg_13bit_max:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0xffffe000, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX9-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: flat_inst_valu_offset_neg_13bit_max:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0xffffe000, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
+; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %gep = getelementptr i8, i8* %p, i64 -8192
+  %load = load i8, i8* %gep, align 4
+  ret i8 %load
+}
+
+define i8 @flat_inst_valu_offset_2x_11bit_max(i8* %p) {
+; GFX9-LABEL: flat_inst_valu_offset_2x_11bit_max:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    flat_load_ubyte v0, v[0:1] offset:4095
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: flat_inst_valu_offset_2x_11bit_max:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0xfff, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %gep = getelementptr i8, i8* %p, i64 4095
+  %load = load i8, i8* %gep, align 4
+  ret i8 %load
+}
+
+define i8 @flat_inst_valu_offset_2x_12bit_max(i8* %p) {
+; GFX9-LABEL: flat_inst_valu_offset_2x_12bit_max:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0x1000, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    flat_load_ubyte v0, v[0:1] offset:4095
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: flat_inst_valu_offset_2x_12bit_max:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x1fff, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %gep = getelementptr i8, i8* %p, i64 8191
+  %load = load i8, i8* %gep, align 4
+  ret i8 %load
+}
+
+define i8 @flat_inst_valu_offset_2x_13bit_max(i8* %p) {
+; GFX9-LABEL: flat_inst_valu_offset_2x_13bit_max:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0x3000, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    flat_load_ubyte v0, v[0:1] offset:4095
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: flat_inst_valu_offset_2x_13bit_max:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x3fff, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %gep = getelementptr i8, i8* %p, i64 16383
+  %load = load i8, i8* %gep, align 4
+  ret i8 %load
+}
+
+define i8 @flat_inst_valu_offset_2x_neg_11bit_max(i8* %p) {
+; GFX9-LABEL: flat_inst_valu_offset_2x_neg_11bit_max:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffff000, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX9-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: flat_inst_valu_offset_2x_neg_11bit_max:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0xfffff000, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
+; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %gep = getelementptr i8, i8* %p, i64 -4096
+  %load = load i8, i8* %gep, align 4
+  ret i8 %load
+}
+
+define i8 @flat_inst_valu_offset_2x_neg_12bit_max(i8* %p) {
+; GFX9-LABEL: flat_inst_valu_offset_2x_neg_12bit_max:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0xffffe000, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX9-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: flat_inst_valu_offset_2x_neg_12bit_max:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0xffffe000, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
+; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %gep = getelementptr i8, i8* %p, i64 -8192
+  %load = load i8, i8* %gep, align 4
+  ret i8 %load
+}
+
+define i8 @flat_inst_valu_offset_2x_neg_13bit_max(i8* %p) {
+; GFX9-LABEL: flat_inst_valu_offset_2x_neg_13bit_max:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0xffffc000, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX9-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: flat_inst_valu_offset_2x_neg_13bit_max:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0xffffc000, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
+; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %gep = getelementptr i8, i8* %p, i64 -16384
+  %load = load i8, i8* %gep, align 4
+  ret i8 %load
+}
+
+; Fill 11-bit low-bits (1ull << 33) | 2047
+define i8 @flat_inst_valu_offset_64bit_11bit_split0(i8* %p) {
+; GFX9-LABEL: flat_inst_valu_offset_64bit_11bit_split0:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 2, v1, vcc
+; GFX9-NEXT:    flat_load_ubyte v0, v[0:1] offset:2047
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: flat_inst_valu_offset_64bit_11bit_split0:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x7ff, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 2, v1, vcc_lo
+; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %gep = getelementptr i8, i8* %p, i64 8589936639
+  %load = load i8, i8* %gep, align 4
+  ret i8 %load
+}
+
+; Fill 11-bit low-bits (1ull << 33) | 2048
+define i8 @flat_inst_valu_offset_64bit_11bit_split1(i8* %p) {
+; GFX9-LABEL: flat_inst_valu_offset_64bit_11bit_split1:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 2, v1, vcc
+; GFX9-NEXT:    flat_load_ubyte v0, v[0:1] offset:2048
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: flat_inst_valu_offset_64bit_11bit_split1:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x800, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 2, v1, vcc_lo
+; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %gep = getelementptr i8, i8* %p, i64 8589936640
+  %load = load i8, i8* %gep, align 4
+  ret i8 %load
+}
+
+; Fill 12-bit low-bits (1ull << 33) | 4095
+define i8 @flat_inst_valu_offset_64bit_12bit_split0(i8* %p) {
+; GFX9-LABEL: flat_inst_valu_offset_64bit_12bit_split0:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 2, v1, vcc
+; GFX9-NEXT:    flat_load_ubyte v0, v[0:1] offset:4095
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: flat_inst_valu_offset_64bit_12bit_split0:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0xfff, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 2, v1, vcc_lo
+; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %gep = getelementptr i8, i8* %p, i64 8589938687
+  %load = load i8, i8* %gep, align 4
+  ret i8 %load
+}
+
+; Fill 12-bit low-bits (1ull << 33) | 4096
+define i8 @flat_inst_valu_offset_64bit_12bit_split1(i8* %p) {
+; GFX9-LABEL: flat_inst_valu_offset_64bit_12bit_split1:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0x1000, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 2, v1, vcc
+; GFX9-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: flat_inst_valu_offset_64bit_12bit_split1:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x1000, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 2, v1, vcc_lo
+; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %gep = getelementptr i8, i8* %p, i64 8589938688
+  %load = load i8, i8* %gep, align 4
+  ret i8 %load
+}
+
+; Fill 13-bit low-bits (1ull << 33) | 8191
+define i8 @flat_inst_valu_offset_64bit_13bit_split0(i8* %p) {
+; GFX9-LABEL: flat_inst_valu_offset_64bit_13bit_split0:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0x1000, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 2, v1, vcc
+; GFX9-NEXT:    flat_load_ubyte v0, v[0:1] offset:4095
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: flat_inst_valu_offset_64bit_13bit_split0:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x1fff, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 2, v1, vcc_lo
+; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %gep = getelementptr i8, i8* %p, i64 8589942783
+  %load = load i8, i8* %gep, align 4
+  ret i8 %load
+}
+
+; Fill 13-bit low-bits (1ull << 33) | 8192
+define i8 @flat_inst_valu_offset_64bit_13bit_split1(i8* %p) {
+; GFX9-LABEL: flat_inst_valu_offset_64bit_13bit_split1:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0x2000, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 2, v1, vcc
+; GFX9-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: flat_inst_valu_offset_64bit_13bit_split1:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x2000, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 2, v1, vcc_lo
+; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %gep = getelementptr i8, i8* %p, i64 8589942784
+  %load = load i8, i8* %gep, align 4
+  ret i8 %load
+}
+
+; Fill 11-bit low-bits, negative high bits (1ull << 63) | 2047
+define i8 @flat_inst_valu_offset_64bit_11bit_neg_high_split0(i8* %p) {
+; GFX9-LABEL: flat_inst_valu_offset_64bit_11bit_neg_high_split0:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_bfrev_b32_e32 v2, 1
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, v2, v1, vcc
+; GFX9-NEXT:    flat_load_ubyte v0, v[0:1] offset:2047
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: flat_inst_valu_offset_64bit_11bit_neg_high_split0:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x7ff, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0x80000000, v1, vcc_lo
+; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %gep = getelementptr i8, i8* %p, i64 -9223372036854773761
+  %load = load i8, i8* %gep, align 4
+  ret i8 %load
+}
+
+; Fill 11-bit low-bits, negative high bits (1ull << 63) | 2048
+define i8 @flat_inst_valu_offset_64bit_11bit_neg_high_split1(i8* %p) {
+; GFX9-LABEL: flat_inst_valu_offset_64bit_11bit_neg_high_split1:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_bfrev_b32_e32 v2, 1
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, v2, v1, vcc
+; GFX9-NEXT:    flat_load_ubyte v0, v[0:1] offset:2048
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: flat_inst_valu_offset_64bit_11bit_neg_high_split1:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x800, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0x80000000, v1, vcc_lo
+; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %gep = getelementptr i8, i8* %p, i64 -9223372036854773760
+  %load = load i8, i8* %gep, align 4
+  ret i8 %load
+}
+
+; Fill 12-bit low-bits, negative high bits (1ull << 63) | 4095
+define i8 @flat_inst_valu_offset_64bit_12bit_neg_high_split0(i8* %p) {
+; GFX9-LABEL: flat_inst_valu_offset_64bit_12bit_neg_high_split0:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_bfrev_b32_e32 v2, 1
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, v2, v1, vcc
+; GFX9-NEXT:    flat_load_ubyte v0, v[0:1] offset:4095
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: flat_inst_valu_offset_64bit_12bit_neg_high_split0:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0xfff, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0x80000000, v1, vcc_lo
+; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %gep = getelementptr i8, i8* %p, i64 -9223372036854771713
+  %load = load i8, i8* %gep, align 4
+  ret i8 %load
+}
+
+; Fill 12-bit low-bits, negative high bits (1ull << 63) | 4096
+define i8 @flat_inst_valu_offset_64bit_12bit_neg_high_split1(i8* %p) {
+; GFX9-LABEL: flat_inst_valu_offset_64bit_12bit_neg_high_split1:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0x1000, v0
+; GFX9-NEXT:    v_bfrev_b32_e32 v2, 1
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, v2, v1, vcc
+; GFX9-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: flat_inst_valu_offset_64bit_12bit_neg_high_split1:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x1000, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0x80000000, v1, vcc_lo
+; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %gep = getelementptr i8, i8* %p, i64 -9223372036854771712
+  %load = load i8, i8* %gep, align 4
+  ret i8 %load
+}
+
+; Fill 13-bit low-bits, negative high bits (1ull << 63) | 8191
+define i8 @flat_inst_valu_offset_64bit_13bit_neg_high_split0(i8* %p) {
+; GFX9-LABEL: flat_inst_valu_offset_64bit_13bit_neg_high_split0:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0x1000, v0
+; GFX9-NEXT:    v_bfrev_b32_e32 v2, 1
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, v2, v1, vcc
+; GFX9-NEXT:    flat_load_ubyte v0, v[0:1] offset:4095
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: flat_inst_valu_offset_64bit_13bit_neg_high_split0:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x1fff, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0x80000000, v1, vcc_lo
+; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %gep = getelementptr i8, i8* %p, i64 -9223372036854767617
+  %load = load i8, i8* %gep, align 4
+  ret i8 %load
+}
+
+; Fill 13-bit low-bits, negative high bits (1ull << 63) | 8192
+define i8 @flat_inst_valu_offset_64bit_13bit_neg_high_split1(i8* %p) {
+; GFX9-LABEL: flat_inst_valu_offset_64bit_13bit_neg_high_split1:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0x2000, v0
+; GFX9-NEXT:    v_bfrev_b32_e32 v2, 1
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, v2, v1, vcc
+; GFX9-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: flat_inst_valu_offset_64bit_13bit_neg_high_split1:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x2000, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0x80000000, v1, vcc_lo
+; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %gep = getelementptr i8, i8* %p, i64 -9223372036854767616
+  %load = load i8, i8* %gep, align 4
+  ret i8 %load
+}
+
+define amdgpu_kernel void @flat_inst_salu_offset_1(i8* %p) {
+; GFX9-LABEL: flat_inst_salu_offset_1:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    flat_load_ubyte v0, v[0:1] offset:1
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    flat_store_byte v[0:1], v0
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: flat_inst_salu_offset_1:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_add_u32 s0, s0, 1
+; GFX10-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    flat_store_byte v[0:1], v0
+; GFX10-NEXT:    s_endpgm
+  %gep = getelementptr i8, i8* %p, i64 1
+  %load = load volatile i8, i8* %gep, align 1
+  store i8 %load, i8* undef
+  ret void
+}
+
+define amdgpu_kernel void @flat_inst_salu_offset_11bit_max(i8* %p) {
+; GFX9-LABEL: flat_inst_salu_offset_11bit_max:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    flat_load_ubyte v0, v[0:1] offset:2047
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    flat_store_byte v[0:1], v0
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: flat_inst_salu_offset_11bit_max:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_add_u32 s0, s0, 0x7ff
+; GFX10-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    flat_store_byte v[0:1], v0
+; GFX10-NEXT:    s_endpgm
+  %gep = getelementptr i8, i8* %p, i64 2047
+  %load = load volatile i8, i8* %gep, align 1
+  store i8 %load, i8* undef
+  ret void
+}
+
+define amdgpu_kernel void @flat_inst_salu_offset_12bit_max(i8* %p) {
+; GFX9-LABEL: flat_inst_salu_offset_12bit_max:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    flat_load_ubyte v0, v[0:1] offset:4095
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    flat_store_byte v[0:1], v0
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: flat_inst_salu_offset_12bit_max:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_add_u32 s0, s0, 0xfff
+; GFX10-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    flat_store_byte v[0:1], v0
+; GFX10-NEXT:    s_endpgm
+  %gep = getelementptr i8, i8* %p, i64 4095
+  %load = load volatile i8, i8* %gep, align 1
+  store i8 %load, i8* undef
+  ret void
+}
+
+define amdgpu_kernel void @flat_inst_salu_offset_13bit_max(i8* %p) {
+; GFX9-LABEL: flat_inst_salu_offset_13bit_max:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0x1000, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    flat_load_ubyte v0, v[0:1] offset:4095
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    flat_store_byte v[0:1], v0
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: flat_inst_salu_offset_13bit_max:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_add_u32 s0, s0, 0x1fff
+; GFX10-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    flat_store_byte v[0:1], v0
+; GFX10-NEXT:    s_endpgm
+  %gep = getelementptr i8, i8* %p, i64 8191
+  %load = load volatile i8, i8* %gep, align 1
+  store i8 %load, i8* undef
+  ret void
+}
+
+define amdgpu_kernel void @flat_inst_salu_offset_neg_11bit_max(i8* %p) {
+; GFX9-LABEL: flat_inst_salu_offset_neg_11bit_max:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffff000, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX9-NEXT:    flat_load_ubyte v0, v[0:1] offset:2048
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    flat_store_byte v[0:1], v0
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: flat_inst_salu_offset_neg_11bit_max:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_add_u32 s0, s0, 0xfffff800
+; GFX10-NEXT:    s_addc_u32 s1, s1, -1
+; GFX10-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    flat_store_byte v[0:1], v0
+; GFX10-NEXT:    s_endpgm
+  %gep = getelementptr i8, i8* %p, i64 -2048
+  %load = load volatile i8, i8* %gep, align 1
+  store i8 %load, i8* undef
+  ret void
+}
+
+define amdgpu_kernel void @flat_inst_salu_offset_neg_12bit_max(i8* %p) {
+; GFX9-LABEL: flat_inst_salu_offset_neg_12bit_max:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffff000, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX9-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    flat_store_byte v[0:1], v0
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: flat_inst_salu_offset_neg_12bit_max:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_add_u32 s0, s0, 0xfffff000
+; GFX10-NEXT:    s_addc_u32 s1, s1, -1
+; GFX10-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    flat_store_byte v[0:1], v0
+; GFX10-NEXT:    s_endpgm
+  %gep = getelementptr i8, i8* %p, i64 -4096
+  %load = load volatile i8, i8* %gep, align 1
+  store i8 %load, i8* undef
+  ret void
+}
+
+define amdgpu_kernel void @flat_inst_salu_offset_neg_13bit_max(i8* %p) {
+; GFX9-LABEL: flat_inst_salu_offset_neg_13bit_max:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0xffffe000, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX9-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    flat_store_byte v[0:1], v0
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: flat_inst_salu_offset_neg_13bit_max:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_add_u32 s0, s0, 0xffffe000
+; GFX10-NEXT:    s_addc_u32 s1, s1, -1
+; GFX10-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    flat_store_byte v[0:1], v0
+; GFX10-NEXT:    s_endpgm
+  %gep = getelementptr i8, i8* %p, i64 -8192
+  %load = load volatile i8, i8* %gep, align 1
+  store i8 %load, i8* undef
+  ret void
+}
+
+define amdgpu_kernel void @flat_inst_salu_offset_2x_11bit_max(i8* %p) {
+; GFX9-LABEL: flat_inst_salu_offset_2x_11bit_max:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    flat_load_ubyte v0, v[0:1] offset:4095
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    flat_store_byte v[0:1], v0
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: flat_inst_salu_offset_2x_11bit_max:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_add_u32 s0, s0, 0xfff
+; GFX10-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    flat_store_byte v[0:1], v0
+; GFX10-NEXT:    s_endpgm
+  %gep = getelementptr i8, i8* %p, i64 4095
+  %load = load volatile i8, i8* %gep, align 1
+  store i8 %load, i8* undef
+  ret void
+}
+
+define amdgpu_kernel void @flat_inst_salu_offset_2x_12bit_max(i8* %p) {
+; GFX9-LABEL: flat_inst_salu_offset_2x_12bit_max:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0x1000, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    flat_load_ubyte v0, v[0:1] offset:4095
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    flat_store_byte v[0:1], v0
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: flat_inst_salu_offset_2x_12bit_max:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_add_u32 s0, s0, 0x1fff
+; GFX10-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    flat_store_byte v[0:1], v0
+; GFX10-NEXT:    s_endpgm
+  %gep = getelementptr i8, i8* %p, i64 8191
+  %load = load volatile i8, i8* %gep, align 1
+  store i8 %load, i8* undef
+  ret void
+}
+
+define amdgpu_kernel void @flat_inst_salu_offset_2x_13bit_max(i8* %p) {
+; GFX9-LABEL: flat_inst_salu_offset_2x_13bit_max:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0x3000, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    flat_load_ubyte v0, v[0:1] offset:4095
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    flat_store_byte v[0:1], v0
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: flat_inst_salu_offset_2x_13bit_max:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_add_u32 s0, s0, 0x3fff
+; GFX10-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    flat_store_byte v[0:1], v0
+; GFX10-NEXT:    s_endpgm
+  %gep = getelementptr i8, i8* %p, i64 16383
+  %load = load volatile i8, i8* %gep, align 1
+  store i8 %load, i8* undef
+  ret void
+}
+
+define amdgpu_kernel void @flat_inst_salu_offset_2x_neg_11bit_max(i8* %p) {
+; GFX9-LABEL: flat_inst_salu_offset_2x_neg_11bit_max:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffff000, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX9-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    flat_store_byte v[0:1], v0
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: flat_inst_salu_offset_2x_neg_11bit_max:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_add_u32 s0, s0, 0xfffff000
+; GFX10-NEXT:    s_addc_u32 s1, s1, -1
+; GFX10-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    flat_store_byte v[0:1], v0
+; GFX10-NEXT:    s_endpgm
+  %gep = getelementptr i8, i8* %p, i64 -4096
+  %load = load volatile i8, i8* %gep, align 1
+  store i8 %load, i8* undef
+  ret void
+}
+
+define amdgpu_kernel void @flat_inst_salu_offset_2x_neg_12bit_max(i8* %p) {
+; GFX9-LABEL: flat_inst_salu_offset_2x_neg_12bit_max:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0xffffe000, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX9-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    flat_store_byte v[0:1], v0
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: flat_inst_salu_offset_2x_neg_12bit_max:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_add_u32 s0, s0, 0xffffe000
+; GFX10-NEXT:    s_addc_u32 s1, s1, -1
+; GFX10-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    flat_store_byte v[0:1], v0
+; GFX10-NEXT:    s_endpgm
+  %gep = getelementptr i8, i8* %p, i64 -8192
+  %load = load volatile i8, i8* %gep, align 1
+  store i8 %load, i8* undef
+  ret void
+}
+
+define amdgpu_kernel void @flat_inst_salu_offset_2x_neg_13bit_max(i8* %p) {
+; GFX9-LABEL: flat_inst_salu_offset_2x_neg_13bit_max:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0xffffc000, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX9-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    flat_store_byte v[0:1], v0
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: flat_inst_salu_offset_2x_neg_13bit_max:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_add_u32 s0, s0, 0xffffc000
+; GFX10-NEXT:    s_addc_u32 s1, s1, -1
+; GFX10-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    flat_store_byte v[0:1], v0
+; GFX10-NEXT:    s_endpgm
+  %gep = getelementptr i8, i8* %p, i64 -16384
+  %load = load volatile i8, i8* %gep, align 1
+  store i8 %load, i8* undef
+  ret void
+}
+
+; Fill 11-bit low-bits (1ull << 33) | 2047
+define amdgpu_kernel void @flat_inst_salu_offset_64bit_11bit_split0(i8* %p) {
+; GFX9-LABEL: flat_inst_salu_offset_64bit_11bit_split0:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_add_co_u32_e64 v0, vcc, 0, s0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 2, v1, vcc
+; GFX9-NEXT:    flat_load_ubyte v0, v[0:1] offset:2047
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    flat_store_byte v[0:1], v0
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: flat_inst_salu_offset_64bit_11bit_split0:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_add_u32 s0, s0, 0x7ff
+; GFX10-NEXT:    s_addc_u32 s1, s1, 2
+; GFX10-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    flat_store_byte v[0:1], v0
+; GFX10-NEXT:    s_endpgm
+  %gep = getelementptr i8, i8* %p, i64 8589936639
+  %load = load volatile i8, i8* %gep, align 1
+  store i8 %load, i8* undef
+  ret void
+}
+
+; Fill 11-bit low-bits (1ull << 33) | 2048
+define amdgpu_kernel void @flat_inst_salu_offset_64bit_11bit_split1(i8* %p) {
+; GFX9-LABEL: flat_inst_salu_offset_64bit_11bit_split1:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_add_co_u32_e64 v0, vcc, 0, s0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 2, v1, vcc
+; GFX9-NEXT:    flat_load_ubyte v0, v[0:1] offset:2048
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    flat_store_byte v[0:1], v0
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: flat_inst_salu_offset_64bit_11bit_split1:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_add_u32 s0, s0, 0x800
+; GFX10-NEXT:    s_addc_u32 s1, s1, 2
+; GFX10-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    flat_store_byte v[0:1], v0
+; GFX10-NEXT:    s_endpgm
+  %gep = getelementptr i8, i8* %p, i64 8589936640
+  %load = load volatile i8, i8* %gep, align 1
+  store i8 %load, i8* undef
+  ret void
+}
+
+; Fill 12-bit low-bits (1ull << 33) | 4095
+define amdgpu_kernel void @flat_inst_salu_offset_64bit_12bit_split0(i8* %p) {
+; GFX9-LABEL: flat_inst_salu_offset_64bit_12bit_split0:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_add_co_u32_e64 v0, vcc, 0, s0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 2, v1, vcc
+; GFX9-NEXT:    flat_load_ubyte v0, v[0:1] offset:4095
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    flat_store_byte v[0:1], v0
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: flat_inst_salu_offset_64bit_12bit_split0:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_add_u32 s0, s0, 0xfff
+; GFX10-NEXT:    s_addc_u32 s1, s1, 2
+; GFX10-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    flat_store_byte v[0:1], v0
+; GFX10-NEXT:    s_endpgm
+  %gep = getelementptr i8, i8* %p, i64 8589938687
+  %load = load volatile i8, i8* %gep, align 1
+  store i8 %load, i8* undef
+  ret void
+}
+
+; Fill 12-bit low-bits (1ull << 33) | 4096
+define amdgpu_kernel void @flat_inst_salu_offset_64bit_12bit_split1(i8* %p) {
+; GFX9-LABEL: flat_inst_salu_offset_64bit_12bit_split1:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0x1000, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 2, v1, vcc
+; GFX9-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    flat_store_byte v[0:1], v0
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: flat_inst_salu_offset_64bit_12bit_split1:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_add_u32 s0, s0, 0x1000
+; GFX10-NEXT:    s_addc_u32 s1, s1, 2
+; GFX10-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    flat_store_byte v[0:1], v0
+; GFX10-NEXT:    s_endpgm
+  %gep = getelementptr i8, i8* %p, i64 8589938688
+  %load = load volatile i8, i8* %gep, align 1
+  store i8 %load, i8* undef
+  ret void
+}
+
+; Fill 13-bit low-bits (1ull << 33) | 8191
+define amdgpu_kernel void @flat_inst_salu_offset_64bit_13bit_split0(i8* %p) {
+; GFX9-LABEL: flat_inst_salu_offset_64bit_13bit_split0:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0x1000, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 2, v1, vcc
+; GFX9-NEXT:    flat_load_ubyte v0, v[0:1] offset:4095
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    flat_store_byte v[0:1], v0
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: flat_inst_salu_offset_64bit_13bit_split0:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_add_u32 s0, s0, 0x1fff
+; GFX10-NEXT:    s_addc_u32 s1, s1, 2
+; GFX10-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    flat_store_byte v[0:1], v0
+; GFX10-NEXT:    s_endpgm
+  %gep = getelementptr i8, i8* %p, i64 8589942783
+  %load = load volatile i8, i8* %gep, align 1
+  store i8 %load, i8* undef
+  ret void
+}
+
+; Fill 13-bit low-bits (1ull << 33) | 8192
+define amdgpu_kernel void @flat_inst_salu_offset_64bit_13bit_split1(i8* %p) {
+; GFX9-LABEL: flat_inst_salu_offset_64bit_13bit_split1:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0x2000, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 2, v1, vcc
+; GFX9-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    flat_store_byte v[0:1], v0
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: flat_inst_salu_offset_64bit_13bit_split1:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_add_u32 s0, s0, 0x2000
+; GFX10-NEXT:    s_addc_u32 s1, s1, 2
+; GFX10-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    flat_store_byte v[0:1], v0
+; GFX10-NEXT:    s_endpgm
+  %gep = getelementptr i8, i8* %p, i64 8589942784
+  %load = load volatile i8, i8* %gep, align 1
+  store i8 %load, i8* undef
+  ret void
+}
+
+; Fill 11-bit low-bits, negative high bits (1ull << 63) | 2047
+define amdgpu_kernel void @flat_inst_salu_offset_64bit_11bit_neg_high_split0(i8* %p) {
+; GFX9-LABEL: flat_inst_salu_offset_64bit_11bit_neg_high_split0:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    v_bfrev_b32_e32 v1, 1
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v2, s1
+; GFX9-NEXT:    v_add_co_u32_e64 v0, vcc, 0, s0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, v1, v2, vcc
+; GFX9-NEXT:    flat_load_ubyte v0, v[0:1] offset:2047
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    flat_store_byte v[0:1], v0
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: flat_inst_salu_offset_64bit_11bit_neg_high_split0:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_add_u32 s0, s0, 0x7ff
+; GFX10-NEXT:    s_addc_u32 s1, s1, 0x80000000
+; GFX10-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    flat_store_byte v[0:1], v0
+; GFX10-NEXT:    s_endpgm
+  %gep = getelementptr i8, i8* %p, i64 -9223372036854773761
+  %load = load volatile i8, i8* %gep, align 1
+  store i8 %load, i8* undef
+  ret void
+}
+
+; Fill 11-bit low-bits, negative high bits (1ull << 63) | 2048
+define amdgpu_kernel void @flat_inst_salu_offset_64bit_11bit_neg_high_split1(i8* %p) {
+; GFX9-LABEL: flat_inst_salu_offset_64bit_11bit_neg_high_split1:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    v_bfrev_b32_e32 v1, 1
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v2, s1
+; GFX9-NEXT:    v_add_co_u32_e64 v0, vcc, 0, s0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, v1, v2, vcc
+; GFX9-NEXT:    flat_load_ubyte v0, v[0:1] offset:2048
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    flat_store_byte v[0:1], v0
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: flat_inst_salu_offset_64bit_11bit_neg_high_split1:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_add_u32 s0, s0, 0x800
+; GFX10-NEXT:    s_addc_u32 s1, s1, 0x80000000
+; GFX10-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    flat_store_byte v[0:1], v0
+; GFX10-NEXT:    s_endpgm
+  %gep = getelementptr i8, i8* %p, i64 -9223372036854773760
+  %load = load volatile i8, i8* %gep, align 1
+  store i8 %load, i8* undef
+  ret void
+}
+
+; Fill 12-bit low-bits, negative high bits (1ull << 63) | 4095
+define amdgpu_kernel void @flat_inst_salu_offset_64bit_12bit_neg_high_split0(i8* %p) {
+; GFX9-LABEL: flat_inst_salu_offset_64bit_12bit_neg_high_split0:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    v_bfrev_b32_e32 v1, 1
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v2, s1
+; GFX9-NEXT:    v_add_co_u32_e64 v0, vcc, 0, s0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, v1, v2, vcc
+; GFX9-NEXT:    flat_load_ubyte v0, v[0:1] offset:4095
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    flat_store_byte v[0:1], v0
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: flat_inst_salu_offset_64bit_12bit_neg_high_split0:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_add_u32 s0, s0, 0xfff
+; GFX10-NEXT:    s_addc_u32 s1, s1, 0x80000000
+; GFX10-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    flat_store_byte v[0:1], v0
+; GFX10-NEXT:    s_endpgm
+  %gep = getelementptr i8, i8* %p, i64 -9223372036854771713
+  %load = load volatile i8, i8* %gep, align 1
+  store i8 %load, i8* undef
+  ret void
+}
+
+; Fill 12-bit low-bits, negative high bits (1ull << 63) | 4096
+define amdgpu_kernel void @flat_inst_salu_offset_64bit_12bit_neg_high_split1(i8* %p) {
+; GFX9-LABEL: flat_inst_salu_offset_64bit_12bit_neg_high_split1:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    v_bfrev_b32_e32 v1, 1
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v2, s1
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0x1000, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, v1, v2, vcc
+; GFX9-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    flat_store_byte v[0:1], v0
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: flat_inst_salu_offset_64bit_12bit_neg_high_split1:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_add_u32 s0, s0, 0x1000
+; GFX10-NEXT:    s_addc_u32 s1, s1, 0x80000000
+; GFX10-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    flat_store_byte v[0:1], v0
+; GFX10-NEXT:    s_endpgm
+  %gep = getelementptr i8, i8* %p, i64 -9223372036854771712
+  %load = load volatile i8, i8* %gep, align 1
+  store i8 %load, i8* undef
+  ret void
+}
+
+; Fill 13-bit low-bits, negative high bits (1ull << 63) | 8191
+define amdgpu_kernel void @flat_inst_salu_offset_64bit_13bit_neg_high_split0(i8* %p) {
+; GFX9-LABEL: flat_inst_salu_offset_64bit_13bit_neg_high_split0:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    v_bfrev_b32_e32 v1, 1
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v2, s1
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0x1000, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, v1, v2, vcc
+; GFX9-NEXT:    flat_load_ubyte v0, v[0:1] offset:4095
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    flat_store_byte v[0:1], v0
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: flat_inst_salu_offset_64bit_13bit_neg_high_split0:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_add_u32 s0, s0, 0x1fff
+; GFX10-NEXT:    s_addc_u32 s1, s1, 0x80000000
+; GFX10-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    flat_store_byte v[0:1], v0
+; GFX10-NEXT:    s_endpgm
+  %gep = getelementptr i8, i8* %p, i64 -9223372036854767617
+  %load = load volatile i8, i8* %gep, align 1
+  store i8 %load, i8* undef
+  ret void
+}
+
+; Fill 13-bit low-bits, negative high bits (1ull << 63) | 8192
+define amdgpu_kernel void @flat_inst_salu_offset_64bit_13bit_neg_high_split1(i8* %p) {
+; GFX9-LABEL: flat_inst_salu_offset_64bit_13bit_neg_high_split1:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    v_bfrev_b32_e32 v1, 1
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v2, s1
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0x2000, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, v1, v2, vcc
+; GFX9-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    flat_store_byte v[0:1], v0
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: flat_inst_salu_offset_64bit_13bit_neg_high_split1:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_add_u32 s0, s0, 0x2000
+; GFX10-NEXT:    s_addc_u32 s1, s1, 0x80000000
+; GFX10-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    flat_store_byte v[0:1], v0
+; GFX10-NEXT:    s_endpgm
+  %gep = getelementptr i8, i8* %p, i64 -9223372036854767616
+  %load = load volatile i8, i8* %gep, align 1
+  store i8 %load, i8* undef
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/offset-split-global.ll b/llvm/test/CodeGen/AMDGPU/offset-split-global.ll
new file mode 100644
index 000000000..add4e6879
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/offset-split-global.ll
@@ -0,0 +1,1408 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX9 %s
+; RUN: llc -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX10 %s
+
+; Test splitting flat instruction offsets into the low and high bits
+; when the offset doesn't fit in the offset field.
+
+define i8 @global_inst_valu_offset_1(i8 addrspace(1)* %p) {
+; GFX9-LABEL: global_inst_valu_offset_1:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_ubyte v0, v[0:1], off offset:1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: global_inst_valu_offset_1:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off offset:1
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %gep = getelementptr i8, i8 addrspace(1)* %p, i64 1
+  %load = load i8, i8 addrspace(1)* %gep, align 4
+  ret i8 %load
+}
+
+define i8 @global_inst_valu_offset_11bit_max(i8 addrspace(1)* %p) {
+; GFX9-LABEL: global_inst_valu_offset_11bit_max:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_ubyte v0, v[0:1], off offset:2047
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: global_inst_valu_offset_11bit_max:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off offset:2047
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %gep = getelementptr i8, i8 addrspace(1)* %p, i64 2047
+  %load = load i8, i8 addrspace(1)* %gep, align 4
+  ret i8 %load
+}
+
+define i8 @global_inst_valu_offset_12bit_max(i8 addrspace(1)* %p) {
+; GFX9-LABEL: global_inst_valu_offset_12bit_max:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_ubyte v0, v[0:1], off offset:4095
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: global_inst_valu_offset_12bit_max:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x800, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off offset:2047
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %gep = getelementptr i8, i8 addrspace(1)* %p, i64 4095
+  %load = load i8, i8 addrspace(1)* %gep, align 4
+  ret i8 %load
+}
+
+define i8 @global_inst_valu_offset_13bit_max(i8 addrspace(1)* %p) {
+; GFX9-LABEL: global_inst_valu_offset_13bit_max:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0x1000, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    global_load_ubyte v0, v[0:1], off offset:4095
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: global_inst_valu_offset_13bit_max:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x1800, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off offset:2047
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %gep = getelementptr i8, i8 addrspace(1)* %p, i64 8191
+  %load = load i8, i8 addrspace(1)* %gep, align 4
+  ret i8 %load
+}
+
+define i8 @global_inst_valu_offset_neg_11bit_max(i8 addrspace(1)* %p) {
+; GFX9-LABEL: global_inst_valu_offset_neg_11bit_max:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_ubyte v0, v[0:1], off offset:-2048
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: global_inst_valu_offset_neg_11bit_max:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off offset:-2048
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %gep = getelementptr i8, i8 addrspace(1)* %p, i64 -2048
+  %load = load i8, i8 addrspace(1)* %gep, align 4
+  ret i8 %load
+}
+
+define i8 @global_inst_valu_offset_neg_12bit_max(i8 addrspace(1)* %p) {
+; GFX9-LABEL: global_inst_valu_offset_neg_12bit_max:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_ubyte v0, v[0:1], off offset:-4096
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: global_inst_valu_offset_neg_12bit_max:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0xfffff000, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
+; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %gep = getelementptr i8, i8 addrspace(1)* %p, i64 -4096
+  %load = load i8, i8 addrspace(1)* %gep, align 4
+  ret i8 %load
+}
+
+define i8 @global_inst_valu_offset_neg_13bit_max(i8 addrspace(1)* %p) {
+; GFX9-LABEL: global_inst_valu_offset_neg_13bit_max:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0xffffe000, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX9-NEXT:    global_load_ubyte v0, v[0:1], off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: global_inst_valu_offset_neg_13bit_max:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0xffffe000, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
+; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %gep = getelementptr i8, i8 addrspace(1)* %p, i64 -8192
+  %load = load i8, i8 addrspace(1)* %gep, align 4
+  ret i8 %load
+}
+
+define i8 @global_inst_valu_offset_2x_11bit_max(i8 addrspace(1)* %p) {
+; GFX9-LABEL: global_inst_valu_offset_2x_11bit_max:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_ubyte v0, v[0:1], off offset:4095
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: global_inst_valu_offset_2x_11bit_max:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x800, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off offset:2047
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %gep = getelementptr i8, i8 addrspace(1)* %p, i64 4095
+  %load = load i8, i8 addrspace(1)* %gep, align 4
+  ret i8 %load
+}
+
+define i8 @global_inst_valu_offset_2x_12bit_max(i8 addrspace(1)* %p) {
+; GFX9-LABEL: global_inst_valu_offset_2x_12bit_max:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0x1000, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    global_load_ubyte v0, v[0:1], off offset:4095
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: global_inst_valu_offset_2x_12bit_max:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x1800, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off offset:2047
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %gep = getelementptr i8, i8 addrspace(1)* %p, i64 8191
+  %load = load i8, i8 addrspace(1)* %gep, align 4
+  ret i8 %load
+}
+
+define i8 @global_inst_valu_offset_2x_13bit_max(i8 addrspace(1)* %p) {
+; GFX9-LABEL: global_inst_valu_offset_2x_13bit_max:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0x3000, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    global_load_ubyte v0, v[0:1], off offset:4095
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: global_inst_valu_offset_2x_13bit_max:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x3800, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off offset:2047
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %gep = getelementptr i8, i8 addrspace(1)* %p, i64 16383
+  %load = load i8, i8 addrspace(1)* %gep, align 4
+  ret i8 %load
+}
+
+define i8 @global_inst_valu_offset_2x_neg_11bit_max(i8 addrspace(1)* %p) {
+; GFX9-LABEL: global_inst_valu_offset_2x_neg_11bit_max:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_ubyte v0, v[0:1], off offset:-4096
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: global_inst_valu_offset_2x_neg_11bit_max:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0xfffff000, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
+; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %gep = getelementptr i8, i8 addrspace(1)* %p, i64 -4096
+  %load = load i8, i8 addrspace(1)* %gep, align 4
+  ret i8 %load
+}
+
+define i8 @global_inst_valu_offset_2x_neg_12bit_max(i8 addrspace(1)* %p) {
+; GFX9-LABEL: global_inst_valu_offset_2x_neg_12bit_max:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0xffffe000, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX9-NEXT:    global_load_ubyte v0, v[0:1], off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: global_inst_valu_offset_2x_neg_12bit_max:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0xffffe000, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
+; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %gep = getelementptr i8, i8 addrspace(1)* %p, i64 -8192
+  %load = load i8, i8 addrspace(1)* %gep, align 4
+  ret i8 %load
+}
+
+define i8 @global_inst_valu_offset_2x_neg_13bit_max(i8 addrspace(1)* %p) {
+; GFX9-LABEL: global_inst_valu_offset_2x_neg_13bit_max:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0xffffc000, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX9-NEXT:    global_load_ubyte v0, v[0:1], off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: global_inst_valu_offset_2x_neg_13bit_max:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0xffffc000, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
+; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %gep = getelementptr i8, i8 addrspace(1)* %p, i64 -16384
+  %load = load i8, i8 addrspace(1)* %gep, align 4
+  ret i8 %load
+}
+
+; Fill 11-bit low-bits (1ull << 33) | 2047
+define i8 @global_inst_valu_offset_64bit_11bit_split0(i8 addrspace(1)* %p) {
+; GFX9-LABEL: global_inst_valu_offset_64bit_11bit_split0:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 2, v1, vcc
+; GFX9-NEXT:    global_load_ubyte v0, v[0:1], off offset:2047
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: global_inst_valu_offset_64bit_11bit_split0:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 2, v1, vcc_lo
+; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off offset:2047
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %gep = getelementptr i8, i8 addrspace(1)* %p, i64 8589936639
+  %load = load i8, i8 addrspace(1)* %gep, align 4
+  ret i8 %load
+}
+
+; Fill 11-bit low-bits (1ull << 33) | 2048
+define i8 @global_inst_valu_offset_64bit_11bit_split1(i8 addrspace(1)* %p) {
+; GFX9-LABEL: global_inst_valu_offset_64bit_11bit_split1:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 2, v1, vcc
+; GFX9-NEXT:    global_load_ubyte v0, v[0:1], off offset:2048
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: global_inst_valu_offset_64bit_11bit_split1:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x800, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 2, v1, vcc_lo
+; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %gep = getelementptr i8, i8 addrspace(1)* %p, i64 8589936640
+  %load = load i8, i8 addrspace(1)* %gep, align 4
+  ret i8 %load
+}
+
+; Fill 12-bit low-bits (1ull << 33) | 4095
+define i8 @global_inst_valu_offset_64bit_12bit_split0(i8 addrspace(1)* %p) {
+; GFX9-LABEL: global_inst_valu_offset_64bit_12bit_split0:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 2, v1, vcc
+; GFX9-NEXT:    global_load_ubyte v0, v[0:1], off offset:4095
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: global_inst_valu_offset_64bit_12bit_split0:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x800, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 2, v1, vcc_lo
+; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off offset:2047
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %gep = getelementptr i8, i8 addrspace(1)* %p, i64 8589938687
+  %load = load i8, i8 addrspace(1)* %gep, align 4
+  ret i8 %load
+}
+
+; Fill 12-bit low-bits (1ull << 33) | 4096
+define i8 @global_inst_valu_offset_64bit_12bit_split1(i8 addrspace(1)* %p) {
+; GFX9-LABEL: global_inst_valu_offset_64bit_12bit_split1:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0x1000, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 2, v1, vcc
+; GFX9-NEXT:    global_load_ubyte v0, v[0:1], off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: global_inst_valu_offset_64bit_12bit_split1:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x1000, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 2, v1, vcc_lo
+; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %gep = getelementptr i8, i8 addrspace(1)* %p, i64 8589938688
+  %load = load i8, i8 addrspace(1)* %gep, align 4
+  ret i8 %load
+}
+
+; Fill 13-bit low-bits (1ull << 33) | 8191
+define i8 @global_inst_valu_offset_64bit_13bit_split0(i8 addrspace(1)* %p) {
+; GFX9-LABEL: global_inst_valu_offset_64bit_13bit_split0:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0x1000, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 2, v1, vcc
+; GFX9-NEXT:    global_load_ubyte v0, v[0:1], off offset:4095
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: global_inst_valu_offset_64bit_13bit_split0:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x1800, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 2, v1, vcc_lo
+; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off offset:2047
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %gep = getelementptr i8, i8 addrspace(1)* %p, i64 8589942783
+  %load = load i8, i8 addrspace(1)* %gep, align 4
+  ret i8 %load
+}
+
+; Fill 13-bit low-bits (1ull << 33) | 8192
+define i8 @global_inst_valu_offset_64bit_13bit_split1(i8 addrspace(1)* %p) {
+; GFX9-LABEL: global_inst_valu_offset_64bit_13bit_split1:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0x2000, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 2, v1, vcc
+; GFX9-NEXT:    global_load_ubyte v0, v[0:1], off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: global_inst_valu_offset_64bit_13bit_split1:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x2000, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 2, v1, vcc_lo
+; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %gep = getelementptr i8, i8 addrspace(1)* %p, i64 8589942784
+  %load = load i8, i8 addrspace(1)* %gep, align 4
+  ret i8 %load
+}
+
+; Fill 11-bit low-bits, negative high bits (1ull << 63) | 2047
+define i8 @global_inst_valu_offset_64bit_11bit_neg_high_split0(i8 addrspace(1)* %p) {
+; GFX9-LABEL: global_inst_valu_offset_64bit_11bit_neg_high_split0:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_bfrev_b32_e32 v2, 1
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, v2, v1, vcc
+; GFX9-NEXT:    global_load_ubyte v0, v[0:1], off offset:2047
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: global_inst_valu_offset_64bit_11bit_neg_high_split0:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0x80000000, v1, vcc_lo
+; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off offset:2047
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %gep = getelementptr i8, i8 addrspace(1)* %p, i64 -9223372036854773761
+  %load = load i8, i8 addrspace(1)* %gep, align 4
+  ret i8 %load
+}
+
+; Fill 11-bit low-bits, negative high bits (1ull << 63) | 2048
+define i8 @global_inst_valu_offset_64bit_11bit_neg_high_split1(i8 addrspace(1)* %p) {
+; GFX9-LABEL: global_inst_valu_offset_64bit_11bit_neg_high_split1:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_bfrev_b32_e32 v2, 1
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, v2, v1, vcc
+; GFX9-NEXT:    global_load_ubyte v0, v[0:1], off offset:2048
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: global_inst_valu_offset_64bit_11bit_neg_high_split1:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x1000, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0x80000000, v1, vcc_lo
+; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off offset:-2048
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %gep = getelementptr i8, i8 addrspace(1)* %p, i64 -9223372036854773760
+  %load = load i8, i8 addrspace(1)* %gep, align 4
+  ret i8 %load
+}
+
+; Fill 12-bit low-bits, negative high bits (1ull << 63) | 4095
+define i8 @global_inst_valu_offset_64bit_12bit_neg_high_split0(i8 addrspace(1)* %p) {
+; GFX9-LABEL: global_inst_valu_offset_64bit_12bit_neg_high_split0:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_bfrev_b32_e32 v2, 1
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, v2, v1, vcc
+; GFX9-NEXT:    global_load_ubyte v0, v[0:1], off offset:4095
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: global_inst_valu_offset_64bit_12bit_neg_high_split0:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x1000, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0x80000000, v1, vcc_lo
+; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off offset:-1
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %gep = getelementptr i8, i8 addrspace(1)* %p, i64 -9223372036854771713
+  %load = load i8, i8 addrspace(1)* %gep, align 4
+  ret i8 %load
+}
+
+; Fill 12-bit low-bits, negative high bits (1ull << 63) | 4096
+define i8 @global_inst_valu_offset_64bit_12bit_neg_high_split1(i8 addrspace(1)* %p) {
+; GFX9-LABEL: global_inst_valu_offset_64bit_12bit_neg_high_split1:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0x2000, v0
+; GFX9-NEXT:    v_bfrev_b32_e32 v2, 1
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, v2, v1, vcc
+; GFX9-NEXT:    global_load_ubyte v0, v[0:1], off offset:-4096
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: global_inst_valu_offset_64bit_12bit_neg_high_split1:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x1000, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0x80000000, v1, vcc_lo
+; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %gep = getelementptr i8, i8 addrspace(1)* %p, i64 -9223372036854771712
+  %load = load i8, i8 addrspace(1)* %gep, align 4
+  ret i8 %load
+}
+
+; Fill 13-bit low-bits, negative high bits (1ull << 63) | 8191
+define i8 @global_inst_valu_offset_64bit_13bit_neg_high_split0(i8 addrspace(1)* %p) {
+; GFX9-LABEL: global_inst_valu_offset_64bit_13bit_neg_high_split0:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0x2000, v0
+; GFX9-NEXT:    v_bfrev_b32_e32 v2, 1
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, v2, v1, vcc
+; GFX9-NEXT:    global_load_ubyte v0, v[0:1], off offset:-1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: global_inst_valu_offset_64bit_13bit_neg_high_split0:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x2000, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0x80000000, v1, vcc_lo
+; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off offset:-1
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %gep = getelementptr i8, i8 addrspace(1)* %p, i64 -9223372036854767617
+  %load = load i8, i8 addrspace(1)* %gep, align 4
+  ret i8 %load
+}
+
+; Fill 13-bit low-bits, negative high bits (1ull << 63) | 8192
+define i8 @global_inst_valu_offset_64bit_13bit_neg_high_split1(i8 addrspace(1)* %p) {
+; GFX9-LABEL: global_inst_valu_offset_64bit_13bit_neg_high_split1:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0x2000, v0
+; GFX9-NEXT:    v_bfrev_b32_e32 v2, 1
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, v2, v1, vcc
+; GFX9-NEXT:    global_load_ubyte v0, v[0:1], off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: global_inst_valu_offset_64bit_13bit_neg_high_split1:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x2000, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0x80000000, v1, vcc_lo
+; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %gep = getelementptr i8, i8 addrspace(1)* %p, i64 -9223372036854767616
+  %load = load i8, i8 addrspace(1)* %gep, align 4
+  ret i8 %load
+}
+
+define amdgpu_kernel void @global_inst_salu_offset_1(i8 addrspace(1)* %p) {
+; GFX9-LABEL: global_inst_salu_offset_1:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    global_load_ubyte v0, v[0:1], off offset:1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    global_store_byte v[0:1], v0, off
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: global_inst_salu_offset_1:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off offset:1
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    global_store_byte v[0:1], v0, off
+; GFX10-NEXT:    s_endpgm
+  %gep = getelementptr i8, i8 addrspace(1)* %p, i64 1
+  %load = load volatile i8, i8 addrspace(1)* %gep, align 1
+  store i8 %load, i8 addrspace(1)* undef
+  ret void
+}
+
+define amdgpu_kernel void @global_inst_salu_offset_11bit_max(i8 addrspace(1)* %p) {
+; GFX9-LABEL: global_inst_salu_offset_11bit_max:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    global_load_ubyte v0, v[0:1], off offset:2047
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    global_store_byte v[0:1], v0, off
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: global_inst_salu_offset_11bit_max:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off offset:2047
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    global_store_byte v[0:1], v0, off
+; GFX10-NEXT:    s_endpgm
+  %gep = getelementptr i8, i8 addrspace(1)* %p, i64 2047
+  %load = load volatile i8, i8 addrspace(1)* %gep, align 1
+  store i8 %load, i8 addrspace(1)* undef
+  ret void
+}
+
+define amdgpu_kernel void @global_inst_salu_offset_12bit_max(i8 addrspace(1)* %p) {
+; GFX9-LABEL: global_inst_salu_offset_12bit_max:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    global_load_ubyte v0, v[0:1], off offset:4095
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    global_store_byte v[0:1], v0, off
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: global_inst_salu_offset_12bit_max:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s0, 0x800, s0
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, 0, s1, s0
+; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off offset:2047
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    global_store_byte v[0:1], v0, off
+; GFX10-NEXT:    s_endpgm
+  %gep = getelementptr i8, i8 addrspace(1)* %p, i64 4095
+  %load = load volatile i8, i8 addrspace(1)* %gep, align 1
+  store i8 %load, i8 addrspace(1)* undef
+  ret void
+}
+
+define amdgpu_kernel void @global_inst_salu_offset_13bit_max(i8 addrspace(1)* %p) {
+; GFX9-LABEL: global_inst_salu_offset_13bit_max:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0x1000, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    global_load_ubyte v0, v[0:1], off offset:4095
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    global_store_byte v[0:1], v0, off
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: global_inst_salu_offset_13bit_max:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s0, 0x1800, s0
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, 0, s1, s0
+; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off offset:2047
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    global_store_byte v[0:1], v0, off
+; GFX10-NEXT:    s_endpgm
+  %gep = getelementptr i8, i8 addrspace(1)* %p, i64 8191
+  %load = load volatile i8, i8 addrspace(1)* %gep, align 1
+  store i8 %load, i8 addrspace(1)* undef
+  ret void
+}
+
+define amdgpu_kernel void @global_inst_salu_offset_neg_11bit_max(i8 addrspace(1)* %p) {
+; GFX9-LABEL: global_inst_salu_offset_neg_11bit_max:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    global_load_ubyte v0, v[0:1], off offset:-2048
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    global_store_byte v[0:1], v0, off
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: global_inst_salu_offset_neg_11bit_max:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off offset:-2048
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    global_store_byte v[0:1], v0, off
+; GFX10-NEXT:    s_endpgm
+  %gep = getelementptr i8, i8 addrspace(1)* %p, i64 -2048
+  %load = load volatile i8, i8 addrspace(1)* %gep, align 1
+  store i8 %load, i8 addrspace(1)* undef
+  ret void
+}
+
+define amdgpu_kernel void @global_inst_salu_offset_neg_12bit_max(i8 addrspace(1)* %p) {
+; GFX9-LABEL: global_inst_salu_offset_neg_12bit_max:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    global_load_ubyte v0, v[0:1], off offset:-4096
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    global_store_byte v[0:1], v0, off
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: global_inst_salu_offset_neg_12bit_max:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s0, 0xfffff000, s0
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, -1, s1, s0
+; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    global_store_byte v[0:1], v0, off
+; GFX10-NEXT:    s_endpgm
+  %gep = getelementptr i8, i8 addrspace(1)* %p, i64 -4096
+  %load = load volatile i8, i8 addrspace(1)* %gep, align 1
+  store i8 %load, i8 addrspace(1)* undef
+  ret void
+}
+
+define amdgpu_kernel void @global_inst_salu_offset_neg_13bit_max(i8 addrspace(1)* %p) {
+; GFX9-LABEL: global_inst_salu_offset_neg_13bit_max:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0xffffe000, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX9-NEXT:    global_load_ubyte v0, v[0:1], off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    global_store_byte v[0:1], v0, off
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: global_inst_salu_offset_neg_13bit_max:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s0, 0xffffe000, s0
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, -1, s1, s0
+; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    global_store_byte v[0:1], v0, off
+; GFX10-NEXT:    s_endpgm
+  %gep = getelementptr i8, i8 addrspace(1)* %p, i64 -8192
+  %load = load volatile i8, i8 addrspace(1)* %gep, align 1
+  store i8 %load, i8 addrspace(1)* undef
+  ret void
+}
+
+define amdgpu_kernel void @global_inst_salu_offset_2x_11bit_max(i8 addrspace(1)* %p) {
+; GFX9-LABEL: global_inst_salu_offset_2x_11bit_max:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    global_load_ubyte v0, v[0:1], off offset:4095
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    global_store_byte v[0:1], v0, off
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: global_inst_salu_offset_2x_11bit_max:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s0, 0x800, s0
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, 0, s1, s0
+; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off offset:2047
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    global_store_byte v[0:1], v0, off
+; GFX10-NEXT:    s_endpgm
+  %gep = getelementptr i8, i8 addrspace(1)* %p, i64 4095
+  %load = load volatile i8, i8 addrspace(1)* %gep, align 1
+  store i8 %load, i8 addrspace(1)* undef
+  ret void
+}
+
+define amdgpu_kernel void @global_inst_salu_offset_2x_12bit_max(i8 addrspace(1)* %p) {
+; GFX9-LABEL: global_inst_salu_offset_2x_12bit_max:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0x1000, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    global_load_ubyte v0, v[0:1], off offset:4095
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    global_store_byte v[0:1], v0, off
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: global_inst_salu_offset_2x_12bit_max:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s0, 0x1800, s0
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, 0, s1, s0
+; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off offset:2047
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    global_store_byte v[0:1], v0, off
+; GFX10-NEXT:    s_endpgm
+  %gep = getelementptr i8, i8 addrspace(1)* %p, i64 8191
+  %load = load volatile i8, i8 addrspace(1)* %gep, align 1
+  store i8 %load, i8 addrspace(1)* undef
+  ret void
+}
+
+define amdgpu_kernel void @global_inst_salu_offset_2x_13bit_max(i8 addrspace(1)* %p) {
+; GFX9-LABEL: global_inst_salu_offset_2x_13bit_max:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0x3000, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    global_load_ubyte v0, v[0:1], off offset:4095
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    global_store_byte v[0:1], v0, off
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: global_inst_salu_offset_2x_13bit_max:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s0, 0x3800, s0
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, 0, s1, s0
+; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off offset:2047
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    global_store_byte v[0:1], v0, off
+; GFX10-NEXT:    s_endpgm
+  %gep = getelementptr i8, i8 addrspace(1)* %p, i64 16383
+  %load = load volatile i8, i8 addrspace(1)* %gep, align 1
+  store i8 %load, i8 addrspace(1)* undef
+  ret void
+}
+
+define amdgpu_kernel void @global_inst_salu_offset_2x_neg_11bit_max(i8 addrspace(1)* %p) {
+; GFX9-LABEL: global_inst_salu_offset_2x_neg_11bit_max:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    global_load_ubyte v0, v[0:1], off offset:-4096
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    global_store_byte v[0:1], v0, off
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: global_inst_salu_offset_2x_neg_11bit_max:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s0, 0xfffff000, s0
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, -1, s1, s0
+; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    global_store_byte v[0:1], v0, off
+; GFX10-NEXT:    s_endpgm
+  %gep = getelementptr i8, i8 addrspace(1)* %p, i64 -4096
+  %load = load volatile i8, i8 addrspace(1)* %gep, align 1
+  store i8 %load, i8 addrspace(1)* undef
+  ret void
+}
+
+define amdgpu_kernel void @global_inst_salu_offset_2x_neg_12bit_max(i8 addrspace(1)* %p) {
+; GFX9-LABEL: global_inst_salu_offset_2x_neg_12bit_max:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0xffffe000, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX9-NEXT:    global_load_ubyte v0, v[0:1], off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    global_store_byte v[0:1], v0, off
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: global_inst_salu_offset_2x_neg_12bit_max:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s0, 0xffffe000, s0
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, -1, s1, s0
+; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    global_store_byte v[0:1], v0, off
+; GFX10-NEXT:    s_endpgm
+  %gep = getelementptr i8, i8 addrspace(1)* %p, i64 -8192
+  %load = load volatile i8, i8 addrspace(1)* %gep, align 1
+  store i8 %load, i8 addrspace(1)* undef
+  ret void
+}
+
+define amdgpu_kernel void @global_inst_salu_offset_2x_neg_13bit_max(i8 addrspace(1)* %p) {
+; GFX9-LABEL: global_inst_salu_offset_2x_neg_13bit_max:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0xffffc000, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX9-NEXT:    global_load_ubyte v0, v[0:1], off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    global_store_byte v[0:1], v0, off
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: global_inst_salu_offset_2x_neg_13bit_max:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s0, 0xffffc000, s0
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, -1, s1, s0
+; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    global_store_byte v[0:1], v0, off
+; GFX10-NEXT:    s_endpgm
+  %gep = getelementptr i8, i8 addrspace(1)* %p, i64 -16384
+  %load = load volatile i8, i8 addrspace(1)* %gep, align 1
+  store i8 %load, i8 addrspace(1)* undef
+  ret void
+}
+
+; Fill 11-bit low-bits (1ull << 33) | 2047
+define amdgpu_kernel void @global_inst_salu_offset_64bit_11bit_split0(i8 addrspace(1)* %p) {
+; GFX9-LABEL: global_inst_salu_offset_64bit_11bit_split0:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_add_co_u32_e64 v0, vcc, 0, s0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 2, v1, vcc
+; GFX9-NEXT:    global_load_ubyte v0, v[0:1], off offset:2047
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    global_store_byte v[0:1], v0, off
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: global_inst_salu_offset_64bit_11bit_split0:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s0, 0, s0
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, 2, s1, s0
+; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off offset:2047
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    global_store_byte v[0:1], v0, off
+; GFX10-NEXT:    s_endpgm
+  %gep = getelementptr i8, i8 addrspace(1)* %p, i64 8589936639
+  %load = load volatile i8, i8 addrspace(1)* %gep, align 1
+  store i8 %load, i8 addrspace(1)* undef
+  ret void
+}
+
+; Fill 11-bit low-bits (1ull << 33) | 2048
+define amdgpu_kernel void @global_inst_salu_offset_64bit_11bit_split1(i8 addrspace(1)* %p) {
+; GFX9-LABEL: global_inst_salu_offset_64bit_11bit_split1:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_add_co_u32_e64 v0, vcc, 0, s0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 2, v1, vcc
+; GFX9-NEXT:    global_load_ubyte v0, v[0:1], off offset:2048
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    global_store_byte v[0:1], v0, off
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: global_inst_salu_offset_64bit_11bit_split1:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s0, 0x800, s0
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, 2, s1, s0
+; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    global_store_byte v[0:1], v0, off
+; GFX10-NEXT:    s_endpgm
+  %gep = getelementptr i8, i8 addrspace(1)* %p, i64 8589936640
+  %load = load volatile i8, i8 addrspace(1)* %gep, align 1
+  store i8 %load, i8 addrspace(1)* undef
+  ret void
+}
+
+; Fill 12-bit low-bits (1ull << 33) | 4095
+define amdgpu_kernel void @global_inst_salu_offset_64bit_12bit_split0(i8 addrspace(1)* %p) {
+; GFX9-LABEL: global_inst_salu_offset_64bit_12bit_split0:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_add_co_u32_e64 v0, vcc, 0, s0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 2, v1, vcc
+; GFX9-NEXT:    global_load_ubyte v0, v[0:1], off offset:4095
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    global_store_byte v[0:1], v0, off
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: global_inst_salu_offset_64bit_12bit_split0:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s0, 0x800, s0
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, 2, s1, s0
+; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off offset:2047
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    global_store_byte v[0:1], v0, off
+; GFX10-NEXT:    s_endpgm
+  %gep = getelementptr i8, i8 addrspace(1)* %p, i64 8589938687
+  %load = load volatile i8, i8 addrspace(1)* %gep, align 1
+  store i8 %load, i8 addrspace(1)* undef
+  ret void
+}
+
+; Fill 12-bit low-bits (1ull << 33) | 4096
+define amdgpu_kernel void @global_inst_salu_offset_64bit_12bit_split1(i8 addrspace(1)* %p) {
+; GFX9-LABEL: global_inst_salu_offset_64bit_12bit_split1:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0x1000, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 2, v1, vcc
+; GFX9-NEXT:    global_load_ubyte v0, v[0:1], off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    global_store_byte v[0:1], v0, off
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: global_inst_salu_offset_64bit_12bit_split1:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s0, 0x1000, s0
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, 2, s1, s0
+; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    global_store_byte v[0:1], v0, off
+; GFX10-NEXT:    s_endpgm
+  %gep = getelementptr i8, i8 addrspace(1)* %p, i64 8589938688
+  %load = load volatile i8, i8 addrspace(1)* %gep, align 1
+  store i8 %load, i8 addrspace(1)* undef
+  ret void
+}
+
+; Fill 13-bit low-bits (1ull << 33) | 8191
+define amdgpu_kernel void @global_inst_salu_offset_64bit_13bit_split0(i8 addrspace(1)* %p) {
+; GFX9-LABEL: global_inst_salu_offset_64bit_13bit_split0:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0x1000, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 2, v1, vcc
+; GFX9-NEXT:    global_load_ubyte v0, v[0:1], off offset:4095
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    global_store_byte v[0:1], v0, off
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: global_inst_salu_offset_64bit_13bit_split0:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s0, 0x1800, s0
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, 2, s1, s0
+; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off offset:2047
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    global_store_byte v[0:1], v0, off
+; GFX10-NEXT:    s_endpgm
+  %gep = getelementptr i8, i8 addrspace(1)* %p, i64 8589942783
+  %load = load volatile i8, i8 addrspace(1)* %gep, align 1
+  store i8 %load, i8 addrspace(1)* undef
+  ret void
+}
+
+; Fill 13-bit low-bits (1ull << 33) | 8192
+define amdgpu_kernel void @global_inst_salu_offset_64bit_13bit_split1(i8 addrspace(1)* %p) {
+; GFX9-LABEL: global_inst_salu_offset_64bit_13bit_split1:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0x2000, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 2, v1, vcc
+; GFX9-NEXT:    global_load_ubyte v0, v[0:1], off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    global_store_byte v[0:1], v0, off
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: global_inst_salu_offset_64bit_13bit_split1:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s0, 0x2000, s0
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, 2, s1, s0
+; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    global_store_byte v[0:1], v0, off
+; GFX10-NEXT:    s_endpgm
+  %gep = getelementptr i8, i8 addrspace(1)* %p, i64 8589942784
+  %load = load volatile i8, i8 addrspace(1)* %gep, align 1
+  store i8 %load, i8 addrspace(1)* undef
+  ret void
+}
+
+; Fill 11-bit low-bits, negative high bits (1ull << 63) | 2047
+define amdgpu_kernel void @global_inst_salu_offset_64bit_11bit_neg_high_split0(i8 addrspace(1)* %p) {
+; GFX9-LABEL: global_inst_salu_offset_64bit_11bit_neg_high_split0:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    v_bfrev_b32_e32 v1, 1
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v2, s1
+; GFX9-NEXT:    v_add_co_u32_e64 v0, vcc, 0, s0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, v1, v2, vcc
+; GFX9-NEXT:    global_load_ubyte v0, v[0:1], off offset:2047
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    global_store_byte v[0:1], v0, off
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: global_inst_salu_offset_64bit_11bit_neg_high_split0:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0, s0
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0x80000000, v1, vcc_lo
+; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off offset:2047
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    global_store_byte v[0:1], v0, off
+; GFX10-NEXT:    s_endpgm
+  %gep = getelementptr i8, i8 addrspace(1)* %p, i64 -9223372036854773761
+  %load = load volatile i8, i8 addrspace(1)* %gep, align 1
+  store i8 %load, i8 addrspace(1)* undef
+  ret void
+}
+
+; Fill 11-bit low-bits, negative high bits (1ull << 63) | 2048
+define amdgpu_kernel void @global_inst_salu_offset_64bit_11bit_neg_high_split1(i8 addrspace(1)* %p) {
+; GFX9-LABEL: global_inst_salu_offset_64bit_11bit_neg_high_split1:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    v_bfrev_b32_e32 v1, 1
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v2, s1
+; GFX9-NEXT:    v_add_co_u32_e64 v0, vcc, 0, s0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, v1, v2, vcc
+; GFX9-NEXT:    global_load_ubyte v0, v[0:1], off offset:2048
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    global_store_byte v[0:1], v0, off
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: global_inst_salu_offset_64bit_11bit_neg_high_split1:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x1000, s0
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0x80000000, v1, vcc_lo
+; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off offset:-2048
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    global_store_byte v[0:1], v0, off
+; GFX10-NEXT:    s_endpgm
+  %gep = getelementptr i8, i8 addrspace(1)* %p, i64 -9223372036854773760
+  %load = load volatile i8, i8 addrspace(1)* %gep, align 1
+  store i8 %load, i8 addrspace(1)* undef
+  ret void
+}
+
+; Fill 12-bit low-bits, negative high bits (1ull << 63) | 4095
+define amdgpu_kernel void @global_inst_salu_offset_64bit_12bit_neg_high_split0(i8 addrspace(1)* %p) {
+; GFX9-LABEL: global_inst_salu_offset_64bit_12bit_neg_high_split0:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    v_bfrev_b32_e32 v1, 1
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v2, s1
+; GFX9-NEXT:    v_add_co_u32_e64 v0, vcc, 0, s0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, v1, v2, vcc
+; GFX9-NEXT:    global_load_ubyte v0, v[0:1], off offset:4095
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    global_store_byte v[0:1], v0, off
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: global_inst_salu_offset_64bit_12bit_neg_high_split0:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x1000, s0
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0x80000000, v1, vcc_lo
+; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off offset:-1
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    global_store_byte v[0:1], v0, off
+; GFX10-NEXT:    s_endpgm
+  %gep = getelementptr i8, i8 addrspace(1)* %p, i64 -9223372036854771713
+  %load = load volatile i8, i8 addrspace(1)* %gep, align 1
+  store i8 %load, i8 addrspace(1)* undef
+  ret void
+}
+
+; Fill 12-bit low-bits, negative high bits (1ull << 63) | 4096
+define amdgpu_kernel void @global_inst_salu_offset_64bit_12bit_neg_high_split1(i8 addrspace(1)* %p) {
+; GFX9-LABEL: global_inst_salu_offset_64bit_12bit_neg_high_split1:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    v_bfrev_b32_e32 v1, 1
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v2, s1
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0x2000, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, v1, v2, vcc
+; GFX9-NEXT:    global_load_ubyte v0, v[0:1], off offset:-4096
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    global_store_byte v[0:1], v0, off
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: global_inst_salu_offset_64bit_12bit_neg_high_split1:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x1000, s0
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0x80000000, v1, vcc_lo
+; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    global_store_byte v[0:1], v0, off
+; GFX10-NEXT:    s_endpgm
+  %gep = getelementptr i8, i8 addrspace(1)* %p, i64 -9223372036854771712
+  %load = load volatile i8, i8 addrspace(1)* %gep, align 1
+  store i8 %load, i8 addrspace(1)* undef
+  ret void
+}
+
+; Fill 13-bit low-bits, negative high bits (1ull << 63) | 8191
+define amdgpu_kernel void @global_inst_salu_offset_64bit_13bit_neg_high_split0(i8 addrspace(1)* %p) {
+; GFX9-LABEL: global_inst_salu_offset_64bit_13bit_neg_high_split0:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    v_bfrev_b32_e32 v1, 1
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v2, s1
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0x2000, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, v1, v2, vcc
+; GFX9-NEXT:    global_load_ubyte v0, v[0:1], off offset:-1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    global_store_byte v[0:1], v0, off
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: global_inst_salu_offset_64bit_13bit_neg_high_split0:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x2000, s0
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0x80000000, v1, vcc_lo
+; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off offset:-1
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    global_store_byte v[0:1], v0, off
+; GFX10-NEXT:    s_endpgm
+  %gep = getelementptr i8, i8 addrspace(1)* %p, i64 -9223372036854767617
+  %load = load volatile i8, i8 addrspace(1)* %gep, align 1
+  store i8 %load, i8 addrspace(1)* undef
+  ret void
+}
+
+; Fill 13-bit low-bits, negative high bits (1ull << 63) | 8192
+define amdgpu_kernel void @global_inst_salu_offset_64bit_13bit_neg_high_split1(i8 addrspace(1)* %p) {
+; GFX9-LABEL: global_inst_salu_offset_64bit_13bit_neg_high_split1:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    v_bfrev_b32_e32 v1, 1
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v2, s1
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0x2000, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, v1, v2, vcc
+; GFX9-NEXT:    global_load_ubyte v0, v[0:1], off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    global_store_byte v[0:1], v0, off
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: global_inst_salu_offset_64bit_13bit_neg_high_split1:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x2000, s0
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0x80000000, v1, vcc_lo
+; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    global_store_byte v[0:1], v0, off
+; GFX10-NEXT:    s_endpgm
+  %gep = getelementptr i8, i8 addrspace(1)* %p, i64 -9223372036854767616
+  %load = load volatile i8, i8 addrspace(1)* %gep, align 1
+  store i8 %load, i8 addrspace(1)* undef
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/opencl-printf-no-hostcall.ll b/llvm/test/CodeGen/AMDGPU/opencl-printf-no-hostcall.ll
new file mode 100644
index 000000000..14c29760e
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/opencl-printf-no-hostcall.ll
@@ -0,0 +1,18 @@
+; RUN: not opt -S -mtriple=amdgcn-unknown-unknown -amdgpu-printf-runtime-binding < %s 2>&1 | FileCheck %s
+
+@.str = private unnamed_addr addrspace(2) constant [6 x i8] c"%s:%d\00", align 1
+
+define amdgpu_kernel void @test_kernel(i32 %n) {
+entry:
+  %str = alloca [9 x i8], align 1
+  %arraydecay = getelementptr inbounds [9 x i8], [9 x i8]* %str, i32 0, i32 0
+  %call1 = call i32 (i8 addrspace(2)*, ...) @printf(i8 addrspace(2)* getelementptr inbounds ([6 x i8], [6 x i8] addrspace(2)* @.str, i32 0, i32 0), i8* %arraydecay, i32 %n)
+  %call2 = call <2 x i64> (i8*, i32, i64, i64, i64, i64, i64, i64, i64, i64) @__ockl_hostcall_internal(i8* undef, i32 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9)
+  ret void
+}
+
+declare i32 @printf(i8 addrspace(2)*, ...)
+
+declare <2 x i64> @__ockl_hostcall_internal(i8*, i32, i64, i64, i64, i64, i64, i64, i64, i64)
+
+; CHECK: error: Cannot use both printf and hostcall in the same module
diff --git a/llvm/test/CodeGen/AMDGPU/opencl-printf.ll b/llvm/test/CodeGen/AMDGPU/opencl-printf.ll
new file mode 100644
index 000000000..f1056d3d5
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/opencl-printf.ll
@@ -0,0 +1,34 @@
+; RUN: opt -mtriple=r600-- -amdgpu-printf-runtime-binding -mcpu=r600 -S < %s | FileCheck --check-prefix=FUNC --check-prefix=R600 %s
+; RUN: opt -mtriple=amdgcn-- -amdgpu-printf-runtime-binding -mcpu=fiji -S < %s | FileCheck --check-prefix=FUNC --check-prefix=GCN %s
+; RUN: opt -mtriple=amdgcn--amdhsa -amdgpu-printf-runtime-binding -mcpu=fiji -S < %s | FileCheck --check-prefix=FUNC --check-prefix=GCN %s
+
+; FUNC-LABEL: @test_kernel(
+; R600-LABEL: entry
+; R600-NOT: call i8 addrspace(1)* @__printf_alloc
+; R600: call i32 (i8 addrspace(2)*, ...) @printf(i8 addrspace(2)* getelementptr inbounds ([6 x i8], [6 x i8] addrspace(2)* @.str, i32 0, i32 0), i8* %arraydecay, i32 %n)
+; GCN-LABEL: entry
+; GCN: call i8 addrspace(1)* @__printf_alloc
+; GCN-LABEL: entry.split
+; GCN: icmp ne i8 addrspace(1)* %printf_alloc_fn, null
+; GCN: %PrintBuffID = getelementptr i8, i8 addrspace(1)* %printf_alloc_fn, i32 0
+; GCN: %PrintBuffIdCast = bitcast i8 addrspace(1)* %PrintBuffID to i32 addrspace(1)*
+; GCN: store i32 1, i32 addrspace(1)* %PrintBuffIdCast
+; GCN: %PrintBuffGep = getelementptr i8, i8 addrspace(1)* %printf_alloc_fn, i32 4
+; GCN: %PrintArgPtr = ptrtoint i8* %arraydecay to i64
+; GCN: %PrintBuffPtrCast = bitcast i8 addrspace(1)* %PrintBuffGep to i64 addrspace(1)*
+; GCN: store i64 %PrintArgPtr, i64 addrspace(1)* %PrintBuffPtrCast
+; GCN: %PrintBuffNextPtr = getelementptr i8, i8 addrspace(1)* %PrintBuffGep, i32 8
+; GCN: %PrintBuffPtrCast1 = bitcast i8 addrspace(1)* %PrintBuffNextPtr to i32 addrspace(1)*
+; GCN: store i32 %n, i32 addrspace(1)* %PrintBuffPtrCast1
+
+@.str = private unnamed_addr addrspace(2) constant [6 x i8] c"%s:%d\00", align 1
+
+define amdgpu_kernel void @test_kernel(i32 %n) {
+entry:
+  %str = alloca [9 x i8], align 1
+  %arraydecay = getelementptr inbounds [9 x i8], [9 x i8]* %str, i32 0, i32 0
+  %call1 = call i32 (i8 addrspace(2)*, ...) @printf(i8 addrspace(2)* getelementptr inbounds ([6 x i8], [6 x i8] addrspace(2)* @.str, i32 0, i32 0), i8* %arraydecay, i32 %n)
+  ret void
+}
+
+declare i32 @printf(i8 addrspace(2)*, ...)
diff --git a/llvm/test/CodeGen/AMDGPU/operand-folding.ll b/llvm/test/CodeGen/AMDGPU/operand-folding.ll
index 3836a2b7e..9c99a01ee 100644
--- a/llvm/test/CodeGen/AMDGPU/operand-folding.ll
+++ b/llvm/test/CodeGen/AMDGPU/operand-folding.ll
@@ -124,6 +124,30 @@ define amdgpu_kernel void @no_fold_tied_subregister() {
   ret void
 }
 
+; There should be exact one folding on the same operand.
+; CHECK-LABEL: {{^}}no_extra_fold_on_same_opnd
+; CHECK: v_xor_b32_e32 v{{[0-9]+}}, 0, v{{[0-9]+}}
+; CHECK: v_xor_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
+define void @no_extra_fold_on_same_opnd() {
+entry:
+  %s0 = load i32, i32 addrspace(5)* undef, align 4
+  %s0.i64= zext i32 %s0 to i64
+  br label %for.body.i.i
+
+for.body.i.i:
+  %s1 = load i32, i32 addrspace(1)* undef, align 8
+  %s1.i64 = sext i32 %s1 to i64
+  %xor = xor i64 %s1.i64, %s0.i64
+  %flag = icmp ult i64 %xor, 8
+  br i1 %flag, label %if.then, label %if.else
+
+if.then:
+  unreachable
+
+if.else:
+  unreachable
+}
+
 declare i32 @llvm.amdgcn.workitem.id.x() #0
 
 attributes #0 = { nounwind readnone }
diff --git a/llvm/test/CodeGen/AMDGPU/packed-op-sel.ll b/llvm/test/CodeGen/AMDGPU/packed-op-sel.ll
index a9d7f43a6..62838ecfa 100644
--- a/llvm/test/CodeGen/AMDGPU/packed-op-sel.ll
+++ b/llvm/test/CodeGen/AMDGPU/packed-op-sel.ll
@@ -204,8 +204,8 @@ bb:
 ; GCN: ds_read_u16 [[SCALAR1:v[0-9]+]]
 
 ; FIXME: Remove and
-; GCN: v_and_b32_e32 [[SCALAR0]], 0xffff, [[SCALAR0]]
-; GCN: v_xor_b32_e32 [[SCALAR1]], 0x8000, [[SCALAR1]]
+; GCN-DAG: v_and_b32_e32 [[SCALAR0]], 0xffff, [[SCALAR0]]
+; GCN-DAG: v_xor_b32_e32 [[SCALAR1]], 0x8000, [[SCALAR1]]
 ; GCN: v_lshl_or_b32 [[PACKED:v[0-9]+]], [[SCALAR1]], 16, [[SCALAR0]]
 
 ; GCN: v_pk_fma_f16 v{{[0-9]+}}, [[VEC0]], [[VEC1]], [[PACKED]]{{$}}
@@ -660,7 +660,7 @@ bb:
 ; GCN-NOT: _or
 
 ; GCN: v_pk_add_f16 [[FADD:v[0-9]+]]
-; GCN-NEXT: v_pk_fma_f16 v{{[0-9]+}}, [[VEC0]], [[VEC1]], [[FADD]] op_sel:[0,0,1] op_sel_hi:[1,1,0]{{$}}
+; GCN: v_pk_fma_f16 v{{[0-9]+}}, [[VEC0]], [[VEC1]], [[FADD]] op_sel:[0,0,1] op_sel_hi:[1,1,0]{{$}}
 define amdgpu_kernel void @mix_elt_types_op_sel(<2 x half> addrspace(1)* %out, <2 x half> addrspace(3)* %lds) #0 {
 bb:
   %lds.gep1 = getelementptr inbounds <2 x half>, <2 x half> addrspace(3)* %lds, i32 1
diff --git a/llvm/test/CodeGen/AMDGPU/preserve-hi16.ll b/llvm/test/CodeGen/AMDGPU/preserve-hi16.ll
index 4eeb43e92..a0b033639 100644
--- a/llvm/test/CodeGen/AMDGPU/preserve-hi16.ll
+++ b/llvm/test/CodeGen/AMDGPU/preserve-hi16.ll
@@ -3,8 +3,8 @@
 
 ; GCN-LABEL: {{^}}shl_i16:
 ; GCN: v_lshlrev_b16_e{{32|64}} [[OP:v[0-9]+]],
-; GFX9-NEXT: s_setpc_b64
-; GFX10: v_and_b32_e32 v{{[0-9]+}}, 0xffff, [[OP]]
+; GFX10-NEXT: ; implicit-def: $vcc_hi
+; GCN-NEXT: s_setpc_b64
 define i16 @shl_i16(i16 %x, i16 %y) {
   %res = shl i16 %x, %y
   ret i16 %res
@@ -12,8 +12,8 @@ define i16 @shl_i16(i16 %x, i16 %y) {
 
 ; GCN-LABEL: {{^}}lshr_i16:
 ; GCN: v_lshrrev_b16_e{{32|64}} [[OP:v[0-9]+]],
-; GFX9-NEXT: s_setpc_b64
-; GFX10: v_and_b32_e32 v{{[0-9]+}}, 0xffff, [[OP]]
+; GFX10-NEXT: ; implicit-def: $vcc_hi
+; GCN-NEXT: s_setpc_b64
 define i16 @lshr_i16(i16 %x, i16 %y) {
   %res = lshr i16 %x, %y
   ret i16 %res
@@ -21,8 +21,8 @@ define i16 @lshr_i16(i16 %x, i16 %y) {
 
 ; GCN-LABEL: {{^}}ashr_i16:
 ; GCN: v_ashrrev_i16_e{{32|64}} [[OP:v[0-9]+]],
-; GFX9-NEXT: s_setpc_b64
-; GFX10: v_and_b32_e32 v{{[0-9]+}}, 0xffff, [[OP]]
+; GFX10-NEXT: ; implicit-def: $vcc_hi
+; GCN-NEXT: s_setpc_b64
 define i16 @ashr_i16(i16 %x, i16 %y) {
   %res = ashr i16 %x, %y
   ret i16 %res
@@ -30,8 +30,8 @@ define i16 @ashr_i16(i16 %x, i16 %y) {
 
 ; GCN-LABEL: {{^}}add_u16:
 ; GCN: v_add_{{(nc_)*}}u16_e{{32|64}} [[OP:v[0-9]+]],
-; GFX9-NEXT: s_setpc_b64
-; GFX10: v_and_b32_e32 v{{[0-9]+}}, 0xffff, [[OP]]
+; GFX10-NEXT: ; implicit-def: $vcc_hi
+; GCN-NEXT: s_setpc_b64
 define i16 @add_u16(i16 %x, i16 %y) {
   %res = add i16 %x, %y
   ret i16 %res
@@ -39,8 +39,8 @@ define i16 @add_u16(i16 %x, i16 %y) {
 
 ; GCN-LABEL: {{^}}sub_u16:
 ; GCN: v_sub_{{(nc_)*}}u16_e{{32|64}} [[OP:v[0-9]+]],
-; GFX9-NEXT: s_setpc_b64
-; GFX10: v_and_b32_e32 v{{[0-9]+}}, 0xffff, [[OP]]
+; GFX10-NEXT: ; implicit-def: $vcc_hi
+; GCN-NEXT: s_setpc_b64
 define i16 @sub_u16(i16 %x, i16 %y) {
   %res = sub i16 %x, %y
   ret i16 %res
@@ -48,8 +48,8 @@ define i16 @sub_u16(i16 %x, i16 %y) {
 
 ; GCN-LABEL: {{^}}mul_lo_u16:
 ; GCN: v_mul_lo_u16_e{{32|64}} [[OP:v[0-9]+]],
-; GFX9-NEXT: s_setpc_b64
-; GFX10: v_and_b32_e32 v{{[0-9]+}}, 0xffff, [[OP]]
+; GFX10-NEXT: ; implicit-def: $vcc_hi
+; GCN-NEXT: s_setpc_b64
 define i16 @mul_lo_u16(i16 %x, i16 %y) {
   %res = mul i16 %x, %y
   ret i16 %res
@@ -57,8 +57,8 @@ define i16 @mul_lo_u16(i16 %x, i16 %y) {
 
 ; GCN-LABEL: {{^}}min_u16:
 ; GCN: v_min_u16_e{{32|64}} [[OP:v[0-9]+]],
-; GFX9-NEXT: s_setpc_b64
-; GFX10: v_and_b32_e32 v{{[0-9]+}}, 0xffff, [[OP]]
+; GFX10-NEXT: ; implicit-def: $vcc_hi
+; GCN-NEXT: s_setpc_b64
 define i16 @min_u16(i16 %x, i16 %y) {
   %cmp = icmp ule i16 %x, %y
   %res = select i1 %cmp, i16 %x, i16 %y
@@ -67,8 +67,8 @@ define i16 @min_u16(i16 %x, i16 %y) {
 
 ; GCN-LABEL: {{^}}min_i16:
 ; GCN: v_min_i16_e{{32|64}} [[OP:v[0-9]+]],
-; GFX9-NEXT: s_setpc_b64
-; GFX10: v_and_b32_e32 v{{[0-9]+}}, 0xffff, [[OP]]
+; GFX10-NEXT: ; implicit-def: $vcc_hi
+; GCN-NEXT: s_setpc_b64
 define i16 @min_i16(i16 %x, i16 %y) {
   %cmp = icmp sle i16 %x, %y
   %res = select i1 %cmp, i16 %x, i16 %y
@@ -77,8 +77,8 @@ define i16 @min_i16(i16 %x, i16 %y) {
 
 ; GCN-LABEL: {{^}}max_u16:
 ; GCN: v_max_u16_e{{32|64}} [[OP:v[0-9]+]],
-; GFX9-NEXT: s_setpc_b64
-; GFX10: v_and_b32_e32 v{{[0-9]+}}, 0xffff, [[OP]]
+; GFX10-NEXT: ; implicit-def: $vcc_hi
+; GCN-NEXT: s_setpc_b64
 define i16 @max_u16(i16 %x, i16 %y) {
   %cmp = icmp uge i16 %x, %y
   %res = select i1 %cmp, i16 %x, i16 %y
@@ -87,10 +87,124 @@ define i16 @max_u16(i16 %x, i16 %y) {
 
 ; GCN-LABEL: {{^}}max_i16:
 ; GCN: v_max_i16_e{{32|64}} [[OP:v[0-9]+]],
-; GFX9-NEXT: s_setpc_b64
-; GFX10: v_and_b32_e32 v{{[0-9]+}}, 0xffff, [[OP]]
+; GFX10-NEXT: ; implicit-def: $vcc_hi
+; GCN-NEXT: s_setpc_b64
 define i16 @max_i16(i16 %x, i16 %y) {
   %cmp = icmp sge i16 %x, %y
   %res = select i1 %cmp, i16 %x, i16 %y
   ret i16 %res
 }
+
+; GCN-LABEL: {{^}}shl_i16_zext_i32:
+; GCN: v_lshlrev_b16_e{{32|64}} [[OP:v[0-9]+]],
+; GFX10-NEXT: ; implicit-def: $vcc_hi
+; GFX10-NEXT: v_and_b32_e32 v{{[0-9]+}}, 0xffff, [[OP]]
+; GCN-NEXT: s_setpc_b64
+define i32 @shl_i16_zext_i32(i16 %x, i16 %y) {
+  %res = shl i16 %x, %y
+  %zext = zext i16 %res to i32
+  ret i32 %zext
+}
+
+; GCN-LABEL: {{^}}lshr_i16_zext_i32:
+; GCN: v_lshrrev_b16_e{{32|64}} [[OP:v[0-9]+]],
+; GFX10-NEXT: ; implicit-def: $vcc_hi
+; GFX10-NEXT: v_and_b32_e32 v{{[0-9]+}}, 0xffff, [[OP]]
+; GCN-NEXT: s_setpc_b64
+define i32 @lshr_i16_zext_i32(i16 %x, i16 %y) {
+  %res = lshr i16 %x, %y
+  %zext = zext i16 %res to i32
+  ret i32 %zext
+}
+
+; GCN-LABEL: {{^}}ashr_i16_zext_i32:
+; GCN: v_ashrrev_i16_e{{32|64}} [[OP:v[0-9]+]],
+; GFX10-NEXT: ; implicit-def: $vcc_hi
+; GFX10-NEXT: v_and_b32_e32 v{{[0-9]+}}, 0xffff, [[OP]]
+; GCN-NEXT: s_setpc_b64
+define i32 @ashr_i16_zext_i32(i16 %x, i16 %y) {
+  %res = ashr i16 %x, %y
+  %zext = zext i16 %res to i32
+  ret i32 %zext
+}
+
+; GCN-LABEL: {{^}}add_u16_zext_i32:
+; GCN: v_add_{{(nc_)*}}u16_e{{32|64}} [[OP:v[0-9]+]],
+; GFX10-NEXT: ; implicit-def: $vcc_hi
+; GFX10-NEXT: v_and_b32_e32 v{{[0-9]+}}, 0xffff, [[OP]]
+; GCN-NEXT: s_setpc_b64
+define i32 @add_u16_zext_i32(i16 %x, i16 %y) {
+  %res = add i16 %x, %y
+  %zext = zext i16 %res to i32
+  ret i32 %zext
+}
+
+; GCN-LABEL: {{^}}sub_u16_zext_i32:
+; GCN: v_sub_{{(nc_)*}}u16_e{{32|64}} [[OP:v[0-9]+]],
+; GFX10-NEXT: ; implicit-def: $vcc_hi
+; GFX10-NEXT: v_and_b32_e32 v{{[0-9]+}}, 0xffff, [[OP]]
+; GCN-NEXT: s_setpc_b64
+define i32 @sub_u16_zext_i32(i16 %x, i16 %y) {
+  %res = sub i16 %x, %y
+  %zext = zext i16 %res to i32
+  ret i32 %zext
+}
+
+; GCN-LABEL: {{^}}mul_lo_u16_zext_i32:
+; GCN: v_mul_lo_u16_e{{32|64}} [[OP:v[0-9]+]],
+; GFX10-NEXT: ; implicit-def: $vcc_hi
+; GFX10-NEXT: v_and_b32_e32 v{{[0-9]+}}, 0xffff, [[OP]]
+; GCN-NEXT: s_setpc_b64
+define i32 @mul_lo_u16_zext_i32(i16 %x, i16 %y) {
+  %res = mul i16 %x, %y
+  %zext = zext i16 %res to i32
+  ret i32 %zext
+}
+
+; GCN-LABEL: {{^}}min_u16_zext_i32:
+; GCN: v_min_u16_e{{32|64}} [[OP:v[0-9]+]],
+; GFX10-NEXT: ; implicit-def: $vcc_hi
+; GFX10-NEXT: v_and_b32_e32 v{{[0-9]+}}, 0xffff, [[OP]]
+; GCN-NEXT: s_setpc_b64
+define i32 @min_u16_zext_i32(i16 %x, i16 %y) {
+  %cmp = icmp ule i16 %x, %y
+  %res = select i1 %cmp, i16 %x, i16 %y
+  %zext = zext i16 %res to i32
+  ret i32 %zext
+}
+
+; GCN-LABEL: {{^}}min_i16_zext_i32:
+; GCN: v_min_i16_e{{32|64}} [[OP:v[0-9]+]],
+; GFX10-NEXT: ; implicit-def: $vcc_hi
+; GFX10-NEXT: v_and_b32_e32 v{{[0-9]+}}, 0xffff, [[OP]]
+; GCN-NEXT: s_setpc_b64
+define i32 @min_i16_zext_i32(i16 %x, i16 %y) {
+  %cmp = icmp sle i16 %x, %y
+  %res = select i1 %cmp, i16 %x, i16 %y
+  %zext = zext i16 %res to i32
+  ret i32 %zext
+}
+
+; GCN-LABEL: {{^}}max_u16_zext_i32:
+; GCN: v_max_u16_e{{32|64}} [[OP:v[0-9]+]],
+; GFX10-NEXT: ; implicit-def: $vcc_hi
+; GFX10-NEXT: v_and_b32_e32 v{{[0-9]+}}, 0xffff, [[OP]]
+; GCN-NEXT: s_setpc_b64
+define i32 @max_u16_zext_i32(i16 %x, i16 %y) {
+  %cmp = icmp uge i16 %x, %y
+  %res = select i1 %cmp, i16 %x, i16 %y
+  %zext = zext i16 %res to i32
+  ret i32 %zext
+}
+
+; GCN-LABEL: {{^}}max_i16_zext_i32:
+; GCN: v_max_i16_e{{32|64}} [[OP:v[0-9]+]],
+; GFX10-NEXT: ; implicit-def: $vcc_hi
+; GFX10-NEXT: v_and_b32_e32 v{{[0-9]+}}, 0xffff, [[OP]]
+; GCN-NEXT: s_setpc_b64
+define i32 @max_i16_zext_i32(i16 %x, i16 %y) {
+  %cmp = icmp sge i16 %x, %y
+  %res = select i1 %cmp, i16 %x, i16 %y
+  %zext = zext i16 %res to i32
+  ret i32 %zext
+}
diff --git a/llvm/test/CodeGen/AMDGPU/private-memory-r600.ll b/llvm/test/CodeGen/AMDGPU/private-memory-r600.ll
index 5e03b8a6e..ddd44fd17 100644
--- a/llvm/test/CodeGen/AMDGPU/private-memory-r600.ll
+++ b/llvm/test/CodeGen/AMDGPU/private-memory-r600.ll
@@ -300,4 +300,4 @@ define amdgpu_kernel void @ptrtoint(i32 addrspace(1)* %out, i32 %a, i32 %b) #0 {
 ; OPT: !0 = !{i32 0, i32 257}
 ; OPT: !1 = !{i32 0, i32 256}
 
-attributes #0 = { nounwind "amdgpu-waves-per-eu"="1,2" }
+attributes #0 = { nounwind "amdgpu-waves-per-eu"="1,2" "amdgpu-flat-work-group-size"="1,256" }
diff --git a/llvm/test/CodeGen/AMDGPU/promote-alloca-addrspacecast.ll b/llvm/test/CodeGen/AMDGPU/promote-alloca-addrspacecast.ll
index f00b3de85..32395a177 100644
--- a/llvm/test/CodeGen/AMDGPU/promote-alloca-addrspacecast.ll
+++ b/llvm/test/CodeGen/AMDGPU/promote-alloca-addrspacecast.ll
@@ -18,4 +18,4 @@ entry:
   ret void
 }
 
-attributes #0 = { nounwind }
+attributes #0 = { nounwind "amdgpu-flat-work-group-size"="1,256" }
diff --git a/llvm/test/CodeGen/AMDGPU/promote-alloca-to-lds-icmp.ll b/llvm/test/CodeGen/AMDGPU/promote-alloca-to-lds-icmp.ll
index 8d12a7255..2bf668d61 100644
--- a/llvm/test/CodeGen/AMDGPU/promote-alloca-to-lds-icmp.ll
+++ b/llvm/test/CodeGen/AMDGPU/promote-alloca-to-lds-icmp.ll
@@ -64,4 +64,4 @@ define amdgpu_kernel void @lds_promoted_alloca_icmp_unknown_ptr(i32 addrspace(1)
 
 declare i32* @get_unknown_pointer() #0
 
-attributes #0 = { nounwind "amdgpu-waves-per-eu"="1,1" }
+attributes #0 = { nounwind "amdgpu-waves-per-eu"="1,1" "amdgpu-flat-work-group-size"="1,256" }
diff --git a/llvm/test/CodeGen/AMDGPU/promote-alloca-to-lds-phi.ll b/llvm/test/CodeGen/AMDGPU/promote-alloca-to-lds-phi.ll
index d196897d6..2fecbcb9d 100644
--- a/llvm/test/CodeGen/AMDGPU/promote-alloca-to-lds-phi.ll
+++ b/llvm/test/CodeGen/AMDGPU/promote-alloca-to-lds-phi.ll
@@ -201,4 +201,4 @@ for.body:                                         ; preds = %for.body, %for.body
 
 declare i32* @get_unknown_pointer() #0
 
-attributes #0 = { nounwind "amdgpu-waves-per-eu"="1,1" }
+attributes #0 = { nounwind "amdgpu-waves-per-eu"="1,1" "amdgpu-flat-work-group-size"="1,256" }
diff --git a/llvm/test/CodeGen/AMDGPU/promote-alloca-to-lds-select.ll b/llvm/test/CodeGen/AMDGPU/promote-alloca-to-lds-select.ll
index 28e4925f9..007cc6f3c 100644
--- a/llvm/test/CodeGen/AMDGPU/promote-alloca-to-lds-select.ll
+++ b/llvm/test/CodeGen/AMDGPU/promote-alloca-to-lds-select.ll
@@ -131,5 +131,5 @@ bb:
   ret void
 }
 
-attributes #0 = { norecurse nounwind "amdgpu-waves-per-eu"="1,1" }
+attributes #0 = { norecurse nounwind "amdgpu-waves-per-eu"="1,1" "amdgpu-flat-work-group-size"="1,256" }
 attributes #1 = { norecurse nounwind }
diff --git a/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm.ll b/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm.ll
index a9fc318ce..b3ceb43a5 100644
--- a/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm.ll
+++ b/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm.ll
@@ -1,5 +1,6 @@
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX8 %s
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX10 %s
 
 declare i64 @_Z13get_global_idj(i32)
 
@@ -14,14 +15,24 @@ define amdgpu_kernel void @clmem_read_simplified(i8 addrspace(1)*  %buffer) {
 ; GFX8:    flat_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}]
 ; GFX8:    flat_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}]
 ;
-; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off
+; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:2048
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:-4096
-; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:-2048
-; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off
+; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:2048
-; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:-2048
-; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off
+; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:2048
+; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
+; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:2048
+;
+; GFX10:   global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
+; GFX10:   global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:-2048
+; GFX10:   global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
+; GFX10:   global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:-2048
+; GFX10:   global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
+; GFX10:   global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:-2048
+; GFX10:   global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
+; GFX10:   global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
+
 entry:
   %call = tail call i64 @_Z13get_global_idj(i32 0)
   %conv = and i64 %call, 255
@@ -75,17 +86,29 @@ define hidden amdgpu_kernel void @clmem_read(i8 addrspace(1)*  %buffer) {
 ; GFX8:    flat_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}]
 ; GFX8:    flat_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}]
 ;
-; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:-4096
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:-2048
-; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off
-; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:-2048
-; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off
+; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:2048
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:-4096
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:-2048
+; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
+; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:2048
+; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:-4096
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off
+;
+; GFX10:   global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:-2048
+; GFX10:   global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
+; GFX10:   global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
+; GFX10:   global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
+; GFX10:   global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:-2048
+; GFX10:   global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:-2048
+; GFX10:   global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
+; GFX10:   global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:-2048
+; GFX10:   global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off
+; GFX10:   global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:-2048
+; GFX10:   global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off
 entry:
   %call = tail call i64 @_Z13get_global_idj(i32 0)
   %conv = and i64 %call, 255
@@ -184,7 +207,7 @@ while.end:                                        ; preds = %while.cond.loopexit
 }
 
 ; using 32bit address.
-define amdgpu_kernel void @Address32(i8 addrspace(1)*  %buffer) {
+define amdgpu_kernel void @Address32(i8 addrspace(1)* %buffer) {
 ; GCN-LABEL: Address32:
 ; GFX8:    flat_load_dword {{v[0-9]+}}, v[{{[0-9]+:[0-9]+}}]
 ; GFX8:    flat_load_dword {{v[0-9]+}}, v[{{[0-9]+:[0-9]+}}]
@@ -197,16 +220,27 @@ define amdgpu_kernel void @Address32(i8 addrspace(1)*  %buffer) {
 ; GFX8:    flat_load_dword {{v[0-9]+}}, v[{{[0-9]+:[0-9]+}}]
 ; GFX8:    flat_load_dword {{v[0-9]+}}, v[{{[0-9]+:[0-9]+}}]
 ;
-; GFX9:    global_load_dword {{v[0-9]+}}, v[{{[0-9]+:[0-9]+}}], off
 ; GFX9:    global_load_dword {{v[0-9]+}}, v[{{[0-9]+:[0-9]+}}], off offset:1024
 ; GFX9:    global_load_dword {{v[0-9]+}}, v[{{[0-9]+:[0-9]+}}], off offset:2048
 ; GFX9:    global_load_dword {{v[0-9]+}}, v[{{[0-9]+:[0-9]+}}], off offset:3072
-; GFX9:    global_load_dword {{v[0-9]+}}, v[{{[0-9]+:[0-9]+}}], off
-; GFX9:    global_load_dword {{v[0-9]+}}, v[{{[0-9]+:[0-9]+}}], off offset:-4096
-; GFX9:    global_load_dword {{v[0-9]+}}, v[{{[0-9]+:[0-9]+}}], off offset:-3072
-; GFX9:    global_load_dword {{v[0-9]+}}, v[{{[0-9]+:[0-9]+}}], off offset:-2048
-; GFX9:    global_load_dword {{v[0-9]+}}, v[{{[0-9]+:[0-9]+}}], off offset:-1024
-; GFX9:    global_load_dword {{v[0-9]+}}, v[{{[0-9]+:[0-9]+}}], off
+; GFX9:    global_load_dword {{v[0-9]+}}, v[{{[0-9]+:[0-9]+}}], off{{$}}
+; GFX9:    global_load_dword {{v[0-9]+}}, v[{{[0-9]+:[0-9]+}}], off offset:1024
+; GFX9:    global_load_dword {{v[0-9]+}}, v[{{[0-9]+:[0-9]+}}], off{{$}}
+; GFX9:    global_load_dword {{v[0-9]+}}, v[{{[0-9]+:[0-9]+}}], off offset:1024
+; GFX9:    global_load_dword {{v[0-9]+}}, v[{{[0-9]+:[0-9]+}}], off offset:2048
+; GFX9:    global_load_dword {{v[0-9]+}}, v[{{[0-9]+:[0-9]+}}], off offset:3072
+; GFX9:    global_load_dword {{v[0-9]+}}, v[{{[0-9]+:[0-9]+}}], off{{$}}
+;
+; GFX10:   global_load_dword {{v[0-9]+}}, v[{{[0-9]+:[0-9]+}}], off{{$}}
+; GFX10:   global_load_dword {{v[0-9]+}}, v[{{[0-9]+:[0-9]+}}], off offset:1024
+; GFX10:   global_load_dword {{v[0-9]+}}, v[{{[0-9]+:[0-9]+}}], off offset:1024
+; GFX10:   global_load_dword {{v[0-9]+}}, v[{{[0-9]+:[0-9]+}}], off offset:-2048
+; GFX10:   global_load_dword {{v[0-9]+}}, v[{{[0-9]+:[0-9]+}}], off{{$}}
+; GFX10:   global_load_dword {{v[0-9]+}}, v[{{[0-9]+:[0-9]+}}], off offset:1024
+; GFX10:   global_load_dword {{v[0-9]+}}, v[{{[0-9]+:[0-9]+}}], off offset:1024
+; GFX10:   global_load_dword {{v[0-9]+}}, v[{{[0-9]+:[0-9]+}}], off offset:-2048
+; GFX10:   global_load_dword {{v[0-9]+}}, v[{{[0-9]+:[0-9]+}}], off{{$}}
+; GFX10:   global_load_dword {{v[0-9]+}}, v[{{[0-9]+:[0-9]+}}], off offset:1024
 entry:
    %call = tail call i64 @_Z13get_global_idj(i32 0)
    %conv = and i64 %call, 255
@@ -265,10 +299,15 @@ define amdgpu_kernel void @Offset64(i8 addrspace(1)*  %buffer) {
 ; GFX8:    flat_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}]
 ; GFX8:    flat_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}]
 ;
-; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:-4096
-; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:-2048
-; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off
+; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
+; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:2048
+; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
+;
+; GFX10:   global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
+; GFX10:   global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:-2048
+; GFX10:   global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
+; GFX10:   global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
 entry:
   %call = tail call i64 @_Z13get_global_idj(i32 0)
   %conv = and i64 %call, 255
@@ -306,10 +345,15 @@ define amdgpu_kernel void @p32Offset64(i8 addrspace(1)*  %buffer) {
 ; GFX8:    flat_load_dword {{v[0-9]+}}, v[{{[0-9]+:[0-9]+}}]
 ; GFX8:    flat_load_dword {{v[0-9]+}}, v[{{[0-9]+:[0-9]+}}]
 ;
-; GFX9:    global_load_dword {{v[0-9]+}}, v[{{[0-9]+:[0-9]+}}], off
-; GFX9:    global_load_dword {{v[0-9]+}}, v[{{[0-9]+:[0-9]+}}], off
-; GFX9:    global_load_dword {{v[0-9]+}}, v[{{[0-9]+:[0-9]+}}], off offset:-1024
-; GFX9:    global_load_dword {{v[0-9]+}}, v[{{[0-9]+:[0-9]+}}], off
+; GFX9:    global_load_dword {{v[0-9]+}}, v[{{[0-9]+:[0-9]+}}], off offset:2048
+; GFX9:    global_load_dword {{v[0-9]+}}, v[{{[0-9]+:[0-9]+}}], off offset:3072
+; GFX9:    global_load_dword {{v[0-9]+}}, v[{{[0-9]+:[0-9]+}}], off{{$}}
+; GFX9:    global_load_dword {{v[0-9]+}}, v[{{[0-9]+:[0-9]+}}], off{{$}}
+;
+; GFX10:   global_load_dword {{v[0-9]+}}, v[{{[0-9]+:[0-9]+}}], off{{$}}
+; GFX10:   global_load_dword {{v[0-9]+}}, v[{{[0-9]+:[0-9]+}}], off offset:-2048
+; GFX10:   global_load_dword {{v[0-9]+}}, v[{{[0-9]+:[0-9]+}}], off{{$}}
+; GFX10:   global_load_dword {{v[0-9]+}}, v[{{[0-9]+:[0-9]+}}], off offset:1024
 entry:
   %call = tail call i64 @_Z13get_global_idj(i32 0)
   %conv = and i64 %call, 255
@@ -348,12 +392,19 @@ define amdgpu_kernel void @DiffBase(i8 addrspace(1)* %buffer1,
 ; GFX8:    flat_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}]
 ; GFX8:    flat_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}]
 ;
+; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:2048
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:-4096
-; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:-2048
-; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:-4096
-; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:-2048
-; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off
-; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off
+; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
+; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:2048
+; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
+; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:2048
+;
+; GFX10:   global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:-2048
+; GFX10:   global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
+; GFX10:   global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:-2048
+; GFX10:   global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
+; GFX10:   global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
+; GFX10:   global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
                                     i8 addrspace(1)* %buffer2) {
 entry:
   %call = tail call i64 @_Z13get_global_idj(i32 0)
@@ -403,14 +454,23 @@ define amdgpu_kernel void @ReverseOrder(i8 addrspace(1)* %buffer) {
 ; GFX8:    flat_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}]
 ; GFX8:    flat_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}]
 ;
-; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:-2048
-; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:2048
-; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off
+; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
+; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:2048
+; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
+; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
+; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:2048
-; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:-4096
-; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:2048
+;
+; GFX10:   global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
+; GFX10:   global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
+; GFX10:   global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
+; GFX10:   global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
+; GFX10:   global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
+; GFX10:   global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
+; GFX10:   global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
+; GFX10:   global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
 entry:
   %call = tail call i64 @_Z13get_global_idj(i32 0)
   %conv = and i64 %call, 255
@@ -459,8 +519,11 @@ define hidden amdgpu_kernel void @negativeoffset(i8 addrspace(1)* nocapture %buf
 ; GFX8:    flat_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}]
 ; GFX8:    flat_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}]
 ;
-; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:2048
+; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off
+;
+; GFX10:   global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:-2048
+; GFX10:   global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off
 entry:
   %call = tail call i64 @_Z13get_global_idj(i32 0) #2
   %conv = and i64 %call, 255
diff --git a/llvm/test/CodeGen/AMDGPU/r600-constant-array-fixup.ll b/llvm/test/CodeGen/AMDGPU/r600-constant-array-fixup.ll
index 9383bfb23..7166a88b4 100644
--- a/llvm/test/CodeGen/AMDGPU/r600-constant-array-fixup.ll
+++ b/llvm/test/CodeGen/AMDGPU/r600-constant-array-fixup.ll
@@ -1,4 +1,4 @@
-; RUN: llc -filetype=obj -march=r600 -mcpu=cypress -verify-machineinstrs < %s | llvm-readobj -r --symbols | FileCheck %s
+; RUN: llc -filetype=obj -mtriple=r600-mesa-mesa3d -mcpu=cypress -verify-machineinstrs < %s | llvm-readobj -r --symbols | FileCheck %s
 
 @arr = internal unnamed_addr addrspace(4) constant [4 x i32] [i32 4, i32 5, i32 6, i32 7], align 4
 
diff --git a/llvm/test/CodeGen/AMDGPU/r600-export-fix.ll b/llvm/test/CodeGen/AMDGPU/r600-export-fix.ll
index 665feeb67..4dafe7d09 100644
--- a/llvm/test/CodeGen/AMDGPU/r600-export-fix.ll
+++ b/llvm/test/CodeGen/AMDGPU/r600-export-fix.ll
@@ -1,16 +1,47 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -march=r600 -mcpu=cedar | FileCheck %s
 
-;CHECK:	EXPORT T{{[0-9]}}.XYZW
-;CHECK:	EXPORT T{{[0-9]}}.0000
-;CHECK: EXPORT T{{[0-9]}}.0000
-;CHECK: EXPORT T{{[0-9]}}.0YZW
-;CHECK: EXPORT T{{[0-9]}}.XYZW
-;CHECK: EXPORT T{{[0-9]}}.XY00
-;CHECK: EXPORT T{{[0-9]}}.0000
-;CHECK: EXPORT T{{[0-9]}}.0000
-
-
 define amdgpu_vs void @main(<4 x float> inreg %reg0, <4 x float> inreg %reg1) {
+; CHECK-LABEL: main:
+; CHECK:       ; %bb.0: ; %main_body
+; CHECK-NEXT:    CALL_FS
+; CHECK-NEXT:    ALU 24, @12, KC0[CB0:0-32], KC1[]
+; CHECK-NEXT:    EXPORT T0.XYZW
+; CHECK-NEXT:    EXPORT T0.0000
+; CHECK-NEXT:    EXPORT T0.0000
+; CHECK-NEXT:    EXPORT T4.0YZW
+; CHECK-NEXT:    EXPORT T3.XYZW
+; CHECK-NEXT:    EXPORT T2.XY00
+; CHECK-NEXT:    EXPORT T0.0000
+; CHECK-NEXT:    EXPORT T0.0000
+; CHECK-NEXT:    CF_END
+; CHECK-NEXT:    PAD
+; CHECK-NEXT:    ALU clause starting at 12:
+; CHECK-NEXT:     MUL_IEEE * T0.W, KC0[4].X, T1.X,
+; CHECK-NEXT:     MULADD_IEEE T0.W, KC0[5].X, T1.Y, PV.W,
+; CHECK-NEXT:     MUL_IEEE * T2.W, KC0[4].Y, T1.X,
+; CHECK-NEXT:     MULADD_IEEE * T0.W, KC0[6].X, T1.Z, PV.W,
+; CHECK-NEXT:     MULADD_IEEE T0.X, KC0[7].X, T1.W, PV.W,
+; CHECK-NEXT:     MULADD_IEEE * T0.W, KC0[5].Y, T1.Y, T2.W,
+; CHECK-NEXT:     MUL_IEEE * T2.W, KC0[4].Z, T1.X,
+; CHECK-NEXT:     MOV T2.Y, KC0[2].Z,
+; CHECK-NEXT:     MULADD_IEEE * T2.W, KC0[5].Z, T1.Y, PV.W,
+; CHECK-NEXT:     MULADD_IEEE * T0.W, KC0[6].Y, T1.Z, T0.W,
+; CHECK-NEXT:     MOV T2.X, KC0[2].Y,
+; CHECK-NEXT:     MULADD_IEEE * T0.Y, KC0[7].Y, T1.W, PV.W,
+; CHECK-NEXT:     MULADD_IEEE * T0.W, KC0[6].Z, T1.Z, T2.W,
+; CHECK-NEXT:     MULADD_IEEE T0.Z, KC0[7].Z, T1.W, PV.W,
+; CHECK-NEXT:     MUL_IEEE * T0.W, KC0[4].W, T1.X,
+; CHECK-NEXT:     MOV * T3.W, KC0[2].X,
+; CHECK-NEXT:     MOV T3.Z, KC0[3].Z,
+; CHECK-NEXT:     MULADD_IEEE * T0.W, KC0[5].W, T1.Y, T0.W,
+; CHECK-NEXT:     MOV * T4.W, KC0[0].Z,
+; CHECK-NEXT:     MOV T3.Y, KC0[3].Y,
+; CHECK-NEXT:     MOV * T4.Z, KC0[0].Y,
+; CHECK-NEXT:     MULADD_IEEE * T0.W, KC0[6].W, T1.Z, T0.W,
+; CHECK-NEXT:     MOV T3.X, KC0[3].X,
+; CHECK-NEXT:     MOV * T4.Y, KC0[0].X,
+; CHECK-NEXT:     MULADD_IEEE * T0.W, KC0[7].W, T1.W, T0.W,
 main_body:
   %0 = extractelement <4 x float> %reg1, i32 0
   %1 = extractelement <4 x float> %reg1, i32 1
diff --git a/llvm/test/CodeGen/AMDGPU/read_register.ll b/llvm/test/CodeGen/AMDGPU/read_register.ll
index 8fe9e7f3f..7224a8576 100644
--- a/llvm/test/CodeGen/AMDGPU/read_register.ll
+++ b/llvm/test/CodeGen/AMDGPU/read_register.ll
@@ -1,14 +1,12 @@
-; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=bonaire -verify-machineinstrs < %s | FileCheck -enable-var-scope %s
 
 declare i32 @llvm.read_register.i32(metadata) #0
 declare i64 @llvm.read_register.i64(metadata) #0
 
-; FIXME: Should be able to eliminate copy
 ; CHECK-LABEL: {{^}}test_read_m0:
 ; CHECK: s_mov_b32 m0, -1
-; CHECK: s_mov_b32 [[COPY_M0:s[0-9]+]], m0
-; CHECK: v_mov_b32_e32 [[COPY:v[0-9]+]], [[COPY_M0]]
-; CHECK: buffer_store_dword [[COPY]]
+; CHECK: v_mov_b32_e32 [[COPY:v[0-9]+]], m0
+; CHECK: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[COPY]]
 define amdgpu_kernel void @test_read_m0(i32 addrspace(1)* %out) #0 {
   store volatile i32 0, i32 addrspace(3)* undef
   %m0 = call i32 @llvm.read_register.i32(metadata !0)
@@ -19,7 +17,7 @@ define amdgpu_kernel void @test_read_m0(i32 addrspace(1)* %out) #0 {
 ; CHECK-LABEL: {{^}}test_read_exec:
 ; CHECK: v_mov_b32_e32 v[[LO:[0-9]+]], exec_lo
 ; CHECK: v_mov_b32_e32 v[[HI:[0-9]+]], exec_hi
-; CHECK: buffer_store_dwordx2 v{{\[}}[[LO]]:[[HI]]{{\]}}
+; CHECK: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[LO]]:[[HI]]{{\]}}
 define amdgpu_kernel void @test_read_exec(i64 addrspace(1)* %out) #0 {
   %exec = call i64 @llvm.read_register.i64(metadata !1)
   store i64 %exec, i64 addrspace(1)* %out
@@ -29,7 +27,7 @@ define amdgpu_kernel void @test_read_exec(i64 addrspace(1)* %out) #0 {
 ; CHECK-LABEL: {{^}}test_read_flat_scratch:
 ; CHECK: v_mov_b32_e32 v[[LO:[0-9]+]], flat_scratch_lo
 ; CHECK: v_mov_b32_e32 v[[HI:[0-9]+]], flat_scratch_hi
-; CHECK: buffer_store_dwordx2 v{{\[}}[[LO]]:[[HI]]{{\]}}
+; CHECK: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[LO]]:[[HI]]{{\]}}
 define amdgpu_kernel void @test_read_flat_scratch(i64 addrspace(1)* %out) #0 {
   %flat_scratch = call i64 @llvm.read_register.i64(metadata !2)
   store i64 %flat_scratch, i64 addrspace(1)* %out
@@ -38,7 +36,7 @@ define amdgpu_kernel void @test_read_flat_scratch(i64 addrspace(1)* %out) #0 {
 
 ; CHECK-LABEL: {{^}}test_read_flat_scratch_lo:
 ; CHECK: v_mov_b32_e32 [[COPY:v[0-9]+]], flat_scratch_lo
-; CHECK: buffer_store_dword [[COPY]]
+; CHECK: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[COPY]]
 define amdgpu_kernel void @test_read_flat_scratch_lo(i32 addrspace(1)* %out) #0 {
   %flat_scratch_lo = call i32 @llvm.read_register.i32(metadata !3)
   store i32 %flat_scratch_lo, i32 addrspace(1)* %out
@@ -47,7 +45,7 @@ define amdgpu_kernel void @test_read_flat_scratch_lo(i32 addrspace(1)* %out) #0
 
 ; CHECK-LABEL: {{^}}test_read_flat_scratch_hi:
 ; CHECK: v_mov_b32_e32 [[COPY:v[0-9]+]], flat_scratch_hi
-; CHECK: buffer_store_dword [[COPY]]
+; CHECK: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[COPY]]
 define amdgpu_kernel void @test_read_flat_scratch_hi(i32 addrspace(1)* %out) #0 {
   %flat_scratch_hi = call i32 @llvm.read_register.i32(metadata !4)
   store i32 %flat_scratch_hi, i32 addrspace(1)* %out
@@ -56,7 +54,7 @@ define amdgpu_kernel void @test_read_flat_scratch_hi(i32 addrspace(1)* %out) #0
 
 ; CHECK-LABEL: {{^}}test_read_exec_lo:
 ; CHECK: v_mov_b32_e32 [[COPY:v[0-9]+]], exec_lo
-; CHECK: buffer_store_dword [[COPY]]
+; CHECK: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[COPY]]
 define amdgpu_kernel void @test_read_exec_lo(i32 addrspace(1)* %out) #0 {
   %exec_lo = call i32 @llvm.read_register.i32(metadata !5)
   store i32 %exec_lo, i32 addrspace(1)* %out
@@ -65,7 +63,7 @@ define amdgpu_kernel void @test_read_exec_lo(i32 addrspace(1)* %out) #0 {
 
 ; CHECK-LABEL: {{^}}test_read_exec_hi:
 ; CHECK: v_mov_b32_e32 [[COPY:v[0-9]+]], exec_hi
-; CHECK: buffer_store_dword [[COPY]]
+; CHECK: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[COPY]]
 define amdgpu_kernel void @test_read_exec_hi(i32 addrspace(1)* %out) #0 {
   %exec_hi = call i32 @llvm.read_register.i32(metadata !6)
   store i32 %exec_hi, i32 addrspace(1)* %out
diff --git a/llvm/test/CodeGen/AMDGPU/reduce-build-vec-ext-to-ext-build-vec.ll b/llvm/test/CodeGen/AMDGPU/reduce-build-vec-ext-to-ext-build-vec.ll
index 79b817ac0..02eb400c0 100644
--- a/llvm/test/CodeGen/AMDGPU/reduce-build-vec-ext-to-ext-build-vec.ll
+++ b/llvm/test/CodeGen/AMDGPU/reduce-build-vec-ext-to-ext-build-vec.ll
@@ -8,9 +8,9 @@
 
 ; GCN-LABEL: {{^}}v2i16_to_i64:
 ; GFX9: s_waitcnt
-; GFX9-NEXT: v_pk_add_u16 v0, v0, v1
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT: v_pk_add_u16 v1, v0, v1
+; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
 ; GFX9-NEXT: s_setpc_b64
 define i64 @v2i16_to_i64(<2 x i16> %x, <2 x i16> %y) {
   %x.add = add <2 x i16> %x, %y
diff --git a/llvm/test/CodeGen/AMDGPU/ret.ll b/llvm/test/CodeGen/AMDGPU/ret.ll
index e77e21220..cee224bfa 100644
--- a/llvm/test/CodeGen/AMDGPU/ret.ll
+++ b/llvm/test/CodeGen/AMDGPU/ret.ll
@@ -79,7 +79,7 @@ bb:
 ; GCN-LABEL: {{^}}ps_input_ena_pos_w:
 ; GCN-DAG: v_mov_b32_e32 v0, v4
 ; GCN-DAG: v_mov_b32_e32 v1, v2
-; GCN: v_mov_b32_e32 v2, v3
+; GCN-DAG: v_mov_b32_e32 v2, v3
 ; GCN-NOT: s_endpgm
 define amdgpu_ps { float, <2 x float> } @ps_input_ena_pos_w([9 x <16 x i8>] addrspace(4)* inreg %arg, i32 inreg %arg1, i32 inreg %arg2, <2 x i32> %arg3, <2 x i32> %arg4, <2 x i32> %arg5, <3 x i32> %arg6, <2 x i32> %arg7, <2 x i32> %arg8, <2 x i32> %arg9, float %arg10, float %arg11, float %arg12, float %arg13, float %arg14, float %arg15, float %arg16, float %arg17, float %arg18) #1 {
 bb:
@@ -177,8 +177,8 @@ bb:
 }
 
 ; GCN-LABEL: {{^}}sgpr:
-; GCN: s_mov_b32 s2, s3
-; GCN: s_add_i32 s0, s3, 2
+; GCN-DAG: s_mov_b32 s2, s3
+; GCN-DAG: s_add_{{i|u}}32 s0, s3, 2
 ; GCN-NOT: s_endpgm
 define amdgpu_vs { i32, i32, i32 } @sgpr([9 x <16 x i8>] addrspace(4)* inreg %arg, i32 inreg %arg1, i32 inreg %arg2, float %arg3) #0 {
 bb:
@@ -206,9 +206,9 @@ bb:
 ; GCN-DAG: exp mrt0 v0, v0, v0, v0 done vm
 ; GCN-DAG: v_mov_b32_e32 v1, v0
 ; GCN-DAG: s_mov_b32 s1, s2
-; GCN: s_waitcnt expcnt(0)
-; GCN: v_add_f32_e32 v0, 1.0, v1
-; GCN-DAG: s_add_i32 s0, s3, 2
+; GCN-DAG: s_waitcnt expcnt(0)
+; GCN-DAG: v_add_f32_e32 v0, 1.0, v1
+; GCN-DAG: s_add_{{i|u}}32 s0, s3, 2
 ; GCN-DAG: s_mov_b32 s2, s3
 ; GCN-NOT: s_endpgm
 define amdgpu_vs { float, i32, float, i32, i32 } @both([9 x <16 x i8>] addrspace(4)* inreg %arg, i32 inreg %arg1, i32 inreg %arg2, float %arg3) #0 {
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-out-arguments-address-space.ll b/llvm/test/CodeGen/AMDGPU/rewrite-out-arguments-address-space.ll
index e0aed8aa3..1ab38398a 100644
--- a/llvm/test/CodeGen/AMDGPU/rewrite-out-arguments-address-space.ll
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-out-arguments-address-space.ll
@@ -6,7 +6,7 @@
 ; CHECK-LABEL: define private %void_one_out_non_private_arg_i32_1_use @void_one_out_non_private_arg_i32_1_use.body(i32 addrspace(1)* %val) #0 {
 ; CHECK-NEXT: ret %void_one_out_non_private_arg_i32_1_use zeroinitializer
 
-; CHECK-LABEL: define void @void_one_out_non_private_arg_i32_1_use(i32 addrspace(1)*) #1 {
+; CHECK-LABEL: define void @void_one_out_non_private_arg_i32_1_use(i32 addrspace(1)* %0) #1 {
 ; CHECK-NEXT: %2 = call %void_one_out_non_private_arg_i32_1_use @void_one_out_non_private_arg_i32_1_use.body(i32 addrspace(1)* undef)
 ; CHECK-NEXT: %3 = extractvalue %void_one_out_non_private_arg_i32_1_use %2, 0
 ; CHECK-NEXT: store i32 %3, i32 addrspace(1)* %0, align 4
@@ -23,7 +23,7 @@ define void @void_one_out_non_private_arg_i32_1_use(i32 addrspace(1)* %val) #0 {
 ; CHECK-NEXT: %2 = insertvalue %bitcast_pointer_as1 undef, <3 x i32> %1, 0
 ; CHECK-NEXT: ret %bitcast_pointer_as1 %2
 
-; CHECK-LABEL: define void @bitcast_pointer_as1(<3 x i32> addrspace(1)*) #1 {
+; CHECK-LABEL: define void @bitcast_pointer_as1(<3 x i32> addrspace(1)* %0) #1 {
 ; CHECK-NEXT: %2 = call %bitcast_pointer_as1 @bitcast_pointer_as1.body(<3 x i32> addrspace(1)* undef)
 define void @bitcast_pointer_as1(<3 x i32> addrspace(1)* %out) #0 {
   %load = load volatile <4 x i32>, <4 x i32> addrspace(1)* undef
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-out-arguments.ll b/llvm/test/CodeGen/AMDGPU/rewrite-out-arguments.ll
index f79bbfb57..396f34bbd 100644
--- a/llvm/test/CodeGen/AMDGPU/rewrite-out-arguments.ll
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-out-arguments.ll
@@ -153,7 +153,7 @@ define void @void_one_out_arg_i32_1_use_align(i32* align 8 %val) #0 {
 ; CHECK: ret1:
 ; CHECK-NEXT: ret %void_one_out_arg_i32_2_use { i32 9 }
 
-; CHECK-LABEL: define void @void_one_out_arg_i32_2_use(i1, i32*) #2 {
+; CHECK-LABEL: define void @void_one_out_arg_i32_2_use(i1 %0, i32* %1) #2 {
 ; CHECK-NEXT: %3 = call %void_one_out_arg_i32_2_use @void_one_out_arg_i32_2_use.body(i1 %0, i32* undef)
 ; CHECK-NEXT: %4 = extractvalue %void_one_out_arg_i32_2_use %3, 0
 ; CHECK-NEXT: store i32 %4, i32* %1, align 4
@@ -176,7 +176,7 @@ declare void @may.clobber()
 ; CHECK-NEXT: store i32 0, i32* %val
 ; CHECK-NEXT: ret %void_one_out_arg_i32_2_stores { i32 1 }
 
-; CHECK-LABEL: define void @void_one_out_arg_i32_2_stores(i32*) #2 {
+; CHECK-LABEL: define void @void_one_out_arg_i32_2_stores(i32* %0) #2 {
 ; CHECK-NEXT: %2 = call %void_one_out_arg_i32_2_stores @void_one_out_arg_i32_2_stores.body(i32* undef)
 ; CHECK-NEXT: %3 = extractvalue %void_one_out_arg_i32_2_stores %2, 0
 ; CHECK-NEXT: store i32 %3, i32* %0, align 4
@@ -191,7 +191,7 @@ define void @void_one_out_arg_i32_2_stores(i32* %val) #0 {
 ; CHECK-NEXT: call void @may.clobber()
 ; CHECK-NEXT: ret %void_one_out_arg_i32_2_stores_clobber { i32 1 }
 
-; CHECK-LABEL: define void @void_one_out_arg_i32_2_stores_clobber(i32*) #2 {
+; CHECK-LABEL: define void @void_one_out_arg_i32_2_stores_clobber(i32* %0) #2 {
 ; CHECK-NEXT: %2 = call %void_one_out_arg_i32_2_stores_clobber @void_one_out_arg_i32_2_stores_clobber.body(i32* undef)
 ; CHECK-NEXT: %3 = extractvalue %void_one_out_arg_i32_2_stores_clobber %2, 0
 ; CHECK-NEXT: store i32 %3, i32* %0, align 4
@@ -219,7 +219,7 @@ define void @void_one_out_arg_i32_call_may_clobber(i32* %val) #0 {
 ; CHECK-NEXT: call void @may.clobber()
 ; CHECK-NEXT: ret %void_one_out_arg_i32_pre_call_may_clobber zeroinitializer
 
-; CHECK-LABEL: @void_one_out_arg_i32_pre_call_may_clobber(i32*) #2 {
+; CHECK-LABEL: @void_one_out_arg_i32_pre_call_may_clobber(i32* %0) #2 {
 ; CHECK-NEXT: %2 = call %void_one_out_arg_i32_pre_call_may_clobber @void_one_out_arg_i32_pre_call_may_clobber.body(i32* undef)
 ; CHECK-NEXT: %3 = extractvalue %void_one_out_arg_i32_pre_call_may_clobber %2, 0
 ; CHECK-NEXT: store i32 %3, i32* %0, align 4
@@ -278,7 +278,7 @@ ret1:
 ; CHECK-LABEL: define private %void_one_out_arg_v2i32_1_use @void_one_out_arg_v2i32_1_use.body(<2 x i32>* %val) #0 {
 ; CHECK-NEXT: ret %void_one_out_arg_v2i32_1_use { <2 x i32> <i32 17, i32 9> }
 
-; CHECK-LABEL: define void @void_one_out_arg_v2i32_1_use(<2 x i32>*) #2 {
+; CHECK-LABEL: define void @void_one_out_arg_v2i32_1_use(<2 x i32>* %0) #2 {
 ; CHECK-NEXT: %2 = call %void_one_out_arg_v2i32_1_use @void_one_out_arg_v2i32_1_use.body(<2 x i32>* undef)
 ; CHECK-NEXT: %3 = extractvalue %void_one_out_arg_v2i32_1_use %2, 0
 ; CHECK-NEXT: store <2 x i32> %3, <2 x i32>* %0, align 8
@@ -294,7 +294,7 @@ define void @void_one_out_arg_v2i32_1_use(<2 x i32>* %val) #0 {
 ; CHECK-NEXT: ret %void_one_out_arg_struct_1_use { %struct { i32 9, i8 99, float 4.000000e+00 } }
 
 ; Normally this is split into element accesses which we don't handle.
-; CHECK-LABEL: define void @void_one_out_arg_struct_1_use(%struct*) #2 {
+; CHECK-LABEL: define void @void_one_out_arg_struct_1_use(%struct* %0) #2 {
 ; CHECK-NEXT: %2 = call %void_one_out_arg_struct_1_use @void_one_out_arg_struct_1_use.body(%struct* undef)
 ; CHECK-NEXT: %3 = extractvalue %void_one_out_arg_struct_1_use %2, 0
 ; CHECK-NEXT: store %struct %3, %struct* %0, align 4
@@ -307,7 +307,7 @@ define void @void_one_out_arg_struct_1_use(%struct* %out) #0 {
 ; CHECK-LABEL: define private %i32_one_out_arg_i32_1_use @i32_one_out_arg_i32_1_use.body(i32* %val) #0 {
 ; CHECK-NEXT: ret %i32_one_out_arg_i32_1_use { i32 9, i32 24 }
 
-; CHECK-LABEL: define i32 @i32_one_out_arg_i32_1_use(i32*) #2 {
+; CHECK-LABEL: define i32 @i32_one_out_arg_i32_1_use(i32* %0) #2 {
 ; CHECK-NEXT: %2 = call %i32_one_out_arg_i32_1_use @i32_one_out_arg_i32_1_use.body(i32* undef)
 ; CHECK-NEXT: %3 = extractvalue %i32_one_out_arg_i32_1_use %2, 1
 ; CHECK-NEXT: store i32 %3, i32* %0, align 4
@@ -321,7 +321,7 @@ define i32 @i32_one_out_arg_i32_1_use(i32* %val) #0 {
 ; CHECK-LABEL: define private %unused_different_type @unused_different_type.body(i32* %arg0, float* nocapture %arg1) #0 {
 ; CHECK-NEXT: ret %unused_different_type { float 4.000000e+00 }
 
-; CHECK-LABEL: define void @unused_different_type(i32*, float* nocapture) #2 {
+; CHECK-LABEL: define void @unused_different_type(i32* %0, float* nocapture %1) #2 {
 ; CHECK-NEXT: %3 = call %unused_different_type @unused_different_type.body(i32* %0, float* undef)
 ; CHECK-NEXT: %4 = extractvalue %unused_different_type %3, 0
 ; CHECK-NEXT: store float %4, float* %1, align 4
@@ -350,7 +350,7 @@ define void @multiple_same_return_noalias(i32* noalias %out0, i32* noalias %out1
 ; CHECK-LABEL: define private %multiple_same_return_mayalias @multiple_same_return_mayalias.body(i32* %out0, i32* %out1) #0 {
 ; CHECK-NEXT: ret %multiple_same_return_mayalias { i32 2, i32 1 }
 
-; CHECK-LABEL: define void @multiple_same_return_mayalias(i32*, i32*) #2 {
+; CHECK-LABEL: define void @multiple_same_return_mayalias(i32* %0, i32* %1) #2 {
 ; CHECK-NEXT: %3 = call %multiple_same_return_mayalias @multiple_same_return_mayalias.body(i32* undef, i32* undef)
 ; CHECK-NEXT: %4 = extractvalue %multiple_same_return_mayalias %3, 0
 ; CHECK-NEXT: store i32 %4, i32* %0, align 4
@@ -366,7 +366,7 @@ define void @multiple_same_return_mayalias(i32* %out0, i32* %out1) #0 {
 ; CHECK-LABEL: define private %multiple_same_return_mayalias_order @multiple_same_return_mayalias_order.body(i32* %out0, i32* %out1) #0 {
 ; CHECK-NEXT: ret %multiple_same_return_mayalias_order { i32 1, i32 2 }
 
-; CHECK-LABEL: define void @multiple_same_return_mayalias_order(i32*, i32*) #2 {
+; CHECK-LABEL: define void @multiple_same_return_mayalias_order(i32* %0, i32* %1) #2 {
 ; CHECK-NEXT: %3 = call %multiple_same_return_mayalias_order @multiple_same_return_mayalias_order.body(i32* undef, i32* undef)
 ; CHECK-NEXT: %4 = extractvalue %multiple_same_return_mayalias_order %3, 0
 ; CHECK-NEXT: store i32 %4, i32* %0, align 4
@@ -401,7 +401,7 @@ endif:
 ; CHECK-LABEL: define private %i1_one_out_arg_i32_1_use @i1_one_out_arg_i32_1_use.body(i32* %val) #0 {
 ; CHECK-NEXT: ret %i1_one_out_arg_i32_1_use { i1 true, i32 24 }
 
-; CHECK-LABEL: define i1 @i1_one_out_arg_i32_1_use(i32*) #2 {
+; CHECK-LABEL: define i1 @i1_one_out_arg_i32_1_use(i32* %0) #2 {
 ; CHECK: %2 = call %i1_one_out_arg_i32_1_use @i1_one_out_arg_i32_1_use.body(i32* undef)
 ; CHECK: %3 = extractvalue %i1_one_out_arg_i32_1_use %2, 1
 ; CHECK: store i32 %3, i32* %0, align 4
@@ -418,7 +418,7 @@ define i1 @i1_one_out_arg_i32_1_use(i32* %val) #0 {
 ; CHECK-LABEL: define private %i1_zeroext_one_out_arg_i32_1_use @i1_zeroext_one_out_arg_i32_1_use.body(i32* %val) #0 {
 ; CHECK-NEXT: ret %i1_zeroext_one_out_arg_i32_1_use { i1 true, i32 24 }
 
-; CHECK-LABEL: define zeroext i1 @i1_zeroext_one_out_arg_i32_1_use(i32*) #2 {
+; CHECK-LABEL: define zeroext i1 @i1_zeroext_one_out_arg_i32_1_use(i32* %0) #2 {
 ; CHECK-NEXT: %2 = call %i1_zeroext_one_out_arg_i32_1_use @i1_zeroext_one_out_arg_i32_1_use.body(i32* undef)
 ; CHECK-NEXT: %3 = extractvalue %i1_zeroext_one_out_arg_i32_1_use %2, 1
 ; CHECK-NEXT: store i32 %3, i32* %0, align 4
@@ -432,7 +432,7 @@ define zeroext i1 @i1_zeroext_one_out_arg_i32_1_use(i32* %val) #0 {
 ; CHECK-LABEL: define private %i1_signext_one_out_arg_i32_1_use @i1_signext_one_out_arg_i32_1_use.body(i32* %val) #0 {
 ; CHECK-NEXT: ret %i1_signext_one_out_arg_i32_1_use { i1 true, i32 24 }
 
-; CHECK-LABEL: define signext i1 @i1_signext_one_out_arg_i32_1_use(i32*) #2 {
+; CHECK-LABEL: define signext i1 @i1_signext_one_out_arg_i32_1_use(i32* %0) #2 {
 ; CHECK-NEXT: %2 = call %i1_signext_one_out_arg_i32_1_use @i1_signext_one_out_arg_i32_1_use.body(i32* undef)
 ; CHECK-NEXT: %3 = extractvalue %i1_signext_one_out_arg_i32_1_use %2, 1
 ; CHECK-NEXT: store i32 %3, i32* %0, align 4
@@ -446,7 +446,7 @@ define signext i1 @i1_signext_one_out_arg_i32_1_use(i32* %val) #0 {
 ; CHECK-LABEL: define private %p1i32_noalias_one_out_arg_i32_1_use @p1i32_noalias_one_out_arg_i32_1_use.body(i32* %val) #0 {
 ; CHECK-NEXT: ret %p1i32_noalias_one_out_arg_i32_1_use { i32 addrspace(1)* null, i32 24 }
 
-; CHECK-LABEL: define noalias i32 addrspace(1)* @p1i32_noalias_one_out_arg_i32_1_use(i32*) #2 {
+; CHECK-LABEL: define noalias i32 addrspace(1)* @p1i32_noalias_one_out_arg_i32_1_use(i32* %0) #2 {
 ; CHECK-NEXT: %2 = call %p1i32_noalias_one_out_arg_i32_1_use @p1i32_noalias_one_out_arg_i32_1_use.body(i32* undef)
 ; CHECK-NEXT: %3 = extractvalue %p1i32_noalias_one_out_arg_i32_1_use %2, 1
 ; CHECK-NEXT: store i32 %3, i32* %0, align 4
@@ -466,7 +466,7 @@ define void @void_one_out_non_private_arg_i32_1_use(i32 addrspace(1)* %val) #0 {
 }
 
 ; CHECK-LABEL: define private %func_ptr_type @func_ptr_type.body(void ()** %out) #0 {
-; CHECK-LABEL: define void @func_ptr_type(void ()**) #2 {
+; CHECK-LABEL: define void @func_ptr_type(void ()** %0) #2 {
 ; CHECK: %2 = call %func_ptr_type @func_ptr_type.body(void ()** undef)
 define void @func_ptr_type(void()** %out) #0 {
   %func = load void()*, void()** undef
@@ -475,7 +475,7 @@ define void @func_ptr_type(void()** %out) #0 {
 }
 
 ; CHECK-LABEL: define private %bitcast_func_ptr_type @bitcast_func_ptr_type.body(void ()** %out) #0 {
-; CHECK-LABEL: define void @bitcast_func_ptr_type(void ()**) #2 {
+; CHECK-LABEL: define void @bitcast_func_ptr_type(void ()** %0) #2 {
 define void @bitcast_func_ptr_type(void()** %out) #0 {
   %func = load i32()*, i32()** undef
   %cast = bitcast void()** %out to i32()**
@@ -486,7 +486,7 @@ define void @bitcast_func_ptr_type(void()** %out) #0 {
 ; CHECK-LABEL: define private %out_arg_small_array @out_arg_small_array.body([4 x i32]* %val) #0 {
 ; CHECK-NEXT: ret %out_arg_small_array { [4 x i32] [i32 0, i32 1, i32 2, i32 3] }
 
-; CHECK-LABEL: define void @out_arg_small_array([4 x i32]*) #2 {
+; CHECK-LABEL: define void @out_arg_small_array([4 x i32]* %0) #2 {
 define void @out_arg_small_array([4 x i32]* %val) #0 {
   store [4 x i32] [i32 0, i32 1, i32 2, i32 3], [4 x i32]* %val
   ret void
@@ -510,7 +510,7 @@ define <16 x i32> @num_regs_return_limit(i32* %out, i32 %val) #0 {
 }
 
 ; CHECK-LABEL: define private %num_regs_reach_limit @num_regs_reach_limit.body(i32* %out, i32 %val) #0 {
-; CHECK: define [15 x i32] @num_regs_reach_limit(i32*, i32) #2 {
+; CHECK: define [15 x i32] @num_regs_reach_limit(i32* %0, i32 %1) #2 {
 ; CHECK-NEXT: call %num_regs_reach_limit @num_regs_reach_limit.body(i32* undef, i32 %1)
 define [15 x i32] @num_regs_reach_limit(i32* %out, i32 %val) #0 {
   %load = load volatile [15 x i32], [15 x i32] addrspace(1)* undef
@@ -526,7 +526,7 @@ define [15 x i32] @num_regs_reach_limit(i32* %out, i32 %val) #0 {
 ; CHECK-NEXT: %3 = insertvalue %num_regs_reach_limit_leftover %2, i32 %val0, 2
 ; CHECK-NEXT: ret %num_regs_reach_limit_leftover %3
 
-; CHECK-LABEL: define [15 x i32] @num_regs_reach_limit_leftover(i32*, i32*, i32) #2 {
+; CHECK-LABEL: define [15 x i32] @num_regs_reach_limit_leftover(i32* %0, i32* %1, i32 %2) #2 {
 ; CHECK-NEXT: %4 = call %num_regs_reach_limit_leftover @num_regs_reach_limit_leftover.body(i32* undef, i32* undef, i32 %2)
 ; CHECK-NEXT: %5 = extractvalue %num_regs_reach_limit_leftover %4, 1
 ; CHECK-NEXT: store i32 %5, i32* %0, align 4
@@ -547,7 +547,7 @@ define [15 x i32] @num_regs_reach_limit_leftover(i32* %out0, i32* %out1, i32 %va
 ; CHECK-NEXT: %1 = insertvalue %preserve_debug_info undef, i32 %arg0, 0, !dbg !11
 ; CHECK-NEXT: ret %preserve_debug_info %1, !dbg !11
 
-; CHECK-LABEL: define void @preserve_debug_info(i32, i32*) #2 !dbg !6 {
+; CHECK-LABEL: define void @preserve_debug_info(i32 %0, i32* %1) #2 !dbg !6 {
 ; CHECK-NEXT: %3 = call %preserve_debug_info @preserve_debug_info.body(i32 %0, i32* undef){{$}}
 ; CHECK-NEXT: %4 = extractvalue %preserve_debug_info %3, 0{{$}}
 ; CHECK-NEXT: store i32 %4, i32* %1, align 4{{$}}
@@ -572,7 +572,7 @@ define void @preserve_metadata(i32 %arg0, i32* %val) #0 !kernel_arg_access_qual
 ; CHECK-NEXT: %2 = insertvalue %bitcast_pointer_v4i32_v3i32 undef, <3 x i32> %1, 0
 ; CHECK-NEXT: ret %bitcast_pointer_v4i32_v3i32 %2
 
-; CHECK-LABEL: define void @bitcast_pointer_v4i32_v3i32(<3 x i32>*) #2 {
+; CHECK-LABEL: define void @bitcast_pointer_v4i32_v3i32(<3 x i32>* %0) #2 {
 ; CHECK-NEXT: %2 = call %bitcast_pointer_v4i32_v3i32 @bitcast_pointer_v4i32_v3i32.body(<3 x i32>* undef)
 ; CHECK-NEXT: %3 = extractvalue %bitcast_pointer_v4i32_v3i32 %2, 0
 ; CHECK-NEXT: store <3 x i32> %3, <3 x i32>* %0, align 16
@@ -609,7 +609,7 @@ define void @bitcast_pointer_v4i32_v3f32(<3 x float>* %out) #0 {
 ; CHECK-NEXT: %2 = insertvalue %bitcast_pointer_i32_f32 undef, float %1, 0
 ; CHECK-NEXT: ret %bitcast_pointer_i32_f32 %2
 
-; CHECK-LABEL: define void @bitcast_pointer_i32_f32(float*) #2 {
+; CHECK-LABEL: define void @bitcast_pointer_i32_f32(float* %0) #2 {
 ; CHECK-NEXT: %2 = call %bitcast_pointer_i32_f32 @bitcast_pointer_i32_f32.body(float* undef)
 ; CHECK-NEXT: %3 = extractvalue %bitcast_pointer_i32_f32 %2, 0
 ; CHECK-NEXT: store float %3, float* %0, align 4
@@ -652,7 +652,7 @@ define void @bitcast_pointer_f16_i32(i32* %out) #0 {
 ; CHECK-NEXT: %3 = insertvalue %bitcast_struct_v3f32_v3f32 undef, %struct.v3f32 %2, 0
 ; CHECK-NEXT: ret %bitcast_struct_v3f32_v3f32 %3
 
-; CHECK-LABEL: define void @bitcast_struct_v3f32_v3f32(%struct.v3f32*, <3 x float>) #2 {
+; CHECK-LABEL: define void @bitcast_struct_v3f32_v3f32(%struct.v3f32* %0, <3 x float> %1) #2 {
 ; CHECK-NEXT: %3 = call %bitcast_struct_v3f32_v3f32 @bitcast_struct_v3f32_v3f32.body(%struct.v3f32* undef, <3 x float> %1)
 ; CHECK-NEXT: %4 = extractvalue %bitcast_struct_v3f32_v3f32 %3, 0
 ; CHECK-NEXT: store %struct.v3f32 %4, %struct.v3f32* %0, align 16
@@ -673,7 +673,7 @@ define void @bitcast_struct_v3f32_v3f32(%struct.v3f32* %out, <3 x float> %value)
 ; CHECK-NEXT: %4 = insertvalue %bitcast_struct_v3f32_v3i32 undef, %struct.v3f32 %3, 0
 ; CHECK-NEXT: ret %bitcast_struct_v3f32_v3i32 %4
 
-; CHECK-LABEL: define void @bitcast_struct_v3f32_v3i32(%struct.v3f32*, <3 x i32>) #2 {
+; CHECK-LABEL: define void @bitcast_struct_v3f32_v3i32(%struct.v3f32* %0, <3 x i32> %1) #2 {
 ; CHECK-NEXT: %3 = call %bitcast_struct_v3f32_v3i32 @bitcast_struct_v3f32_v3i32.body(%struct.v3f32* undef, <3 x i32> %1)
 ; CHECK-NEXT: %4 = extractvalue %bitcast_struct_v3f32_v3i32 %3, 0
 ; CHECK-NEXT: store %struct.v3f32 %4, %struct.v3f32* %0, align 16
@@ -690,7 +690,7 @@ define void @bitcast_struct_v3f32_v3i32(%struct.v3f32* %out, <3 x i32> %value) #
 ; CHECK-NEXT: %2 = insertvalue %bitcast_struct_v4f32_v4f32 undef, %struct.v4f32 %1, 0
 ; CHECK-NEXT: ret %bitcast_struct_v4f32_v4f32 %2
 
-; CHECK-LABEL: define void @bitcast_struct_v4f32_v4f32(%struct.v4f32*, <4 x float>) #2 {
+; CHECK-LABEL: define void @bitcast_struct_v4f32_v4f32(%struct.v4f32* %0, <4 x float> %1) #2 {
 ; CHECK-NEXT: %3 = call %bitcast_struct_v4f32_v4f32 @bitcast_struct_v4f32_v4f32.body(%struct.v4f32* undef, <4 x float> %1)
 define void @bitcast_struct_v4f32_v4f32(%struct.v4f32* %out, <4 x float> %value) #0 {
   %cast = bitcast %struct.v4f32* %out to <4 x float>*
@@ -699,7 +699,7 @@ define void @bitcast_struct_v4f32_v4f32(%struct.v4f32* %out, <4 x float> %value)
 }
 
 ; CHECK-LABEL: define private %bitcast_struct_v3f32_v4i32 @bitcast_struct_v3f32_v4i32.body(%struct.v3f32* %out, <4 x i32> %value) #0 {
-; CHECK-LABEL: define void @bitcast_struct_v3f32_v4i32(%struct.v3f32*, <4 x i32>) #2 {
+; CHECK-LABEL: define void @bitcast_struct_v3f32_v4i32(%struct.v3f32* %0, <4 x i32> %1) #2 {
 define void @bitcast_struct_v3f32_v4i32(%struct.v3f32* %out, <4 x i32> %value) #0 {
   %cast = bitcast %struct.v3f32* %out to <4 x i32>*
   store <4 x i32> %value, <4 x i32>* %cast, align 16
@@ -707,7 +707,7 @@ define void @bitcast_struct_v3f32_v4i32(%struct.v3f32* %out, <4 x i32> %value) #
 }
 
 ; CHECK-LABEL: define private %bitcast_struct_v4f32_v3f32 @bitcast_struct_v4f32_v3f32.body(%struct.v4f32* %out, <3 x float> %value) #0 {
-; CHECK-LABEL: define void @bitcast_struct_v4f32_v3f32(%struct.v4f32*, <3 x float>) #2 {
+; CHECK-LABEL: define void @bitcast_struct_v4f32_v3f32(%struct.v4f32* %0, <3 x float> %1) #2 {
 define void @bitcast_struct_v4f32_v3f32(%struct.v4f32* %out, <3 x float> %value) #0 {
   %extractVec = shufflevector <3 x float> %value, <3 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
   %cast = bitcast %struct.v4f32* %out to <4 x float>*
@@ -755,7 +755,7 @@ define void @bitcast_struct_i128_v4f32(%struct.i128* %out, <4 x float> %value) #
   ret void
 }
 
-; CHECK-LABEL: define void @bitcast_struct_i128_v4f32(%struct.i128*, <4 x float>) #2 {
+; CHECK-LABEL: define void @bitcast_struct_i128_v4f32(%struct.i128* %0, <4 x float> %1) #2 {
 ; CHECK-NEXT: %3 = call %bitcast_struct_i128_v4f32 @bitcast_struct_i128_v4f32.body(%struct.i128* undef, <4 x float> %1)
 define void @bitcast_array_v4i32_v4f32([4 x i32]* %out, [4 x float] %value) #0 {
   %cast = bitcast [4 x i32]* %out to [4 x float]*
diff --git a/llvm/test/CodeGen/AMDGPU/rsq.ll b/llvm/test/CodeGen/AMDGPU/rsq.ll
index 204eeb998..40c3c9424 100644
--- a/llvm/test/CodeGen/AMDGPU/rsq.ll
+++ b/llvm/test/CodeGen/AMDGPU/rsq.ll
@@ -48,7 +48,7 @@ define amdgpu_kernel void @rsq_f32_sgpr(float addrspace(1)* noalias %out, float
 
 ; SI-UNSAFE-DAG: v_rsq_f32_e32 [[RSQA:v[0-9]+]], [[A]]
 ; SI-UNSAFE-DAG: v_rcp_f32_e32 [[RCPB:v[0-9]+]], [[B]]
-; SI-UNSAFE-DAG: v_mul_f32_e32 [[TMP:v[0-9]+]], [[RSQA]], [[RCPB]]
+; SI-UNSAFE-DAG: v_mul_f32_e32 [[TMP:v[0-9]+]], [[RCPB]], [[RSQA]]
 ; SI-UNSAFE: v_mul_f32_e32 [[RESULT:v[0-9]+]], [[C]], [[TMP]]
 ; SI-UNSAFE: buffer_store_dword [[RESULT]]
 
diff --git a/llvm/test/CodeGen/AMDGPU/s_code_end.ll b/llvm/test/CodeGen/AMDGPU/s_code_end.ll
index 2e87098a4..0cf2276b2 100644
--- a/llvm/test/CodeGen/AMDGPU/s_code_end.ll
+++ b/llvm/test/CodeGen/AMDGPU/s_code_end.ll
@@ -35,47 +35,14 @@ define amdgpu_kernel void @a_kernel2() {
 ; GCN-ASM-NEXT:   [[END_LABEL3:\.Lfunc_end.*]]:
 ; GCN-ASM-NEXT:           .size   a_function, [[END_LABEL3]]-a_function
 ; GFX10END-ASM:           .p2alignl 6, 3214868480
-; GFX10END-ASM-NEXT:      .fill 32, 4, 3214868480
+; GFX10END-ASM-NEXT:      .fill 48, 4, 3214868480
 ; GFX10NOEND-NOT:         .fill
 
 ; GFX10NOEND-OBJ-NOT:     s_code_end
 ; GFX10END-OBJ-NEXT:      s_code_end
 
 ; GFX10END-OBJ:           s_code_end // 000000000140:
-; GFX10END-OBJ-NEXT:      s_code_end
-; GFX10END-OBJ-NEXT:      s_code_end
-; GFX10END-OBJ-NEXT:      s_code_end
-; GFX10END-OBJ-NEXT:      s_code_end
-; GFX10END-OBJ-NEXT:      s_code_end
-; GFX10END-OBJ-NEXT:      s_code_end
-; GFX10END-OBJ-NEXT:      s_code_end
-
-; GFX10END-OBJ-NEXT:      s_code_end
-; GFX10END-OBJ-NEXT:      s_code_end
-; GFX10END-OBJ-NEXT:      s_code_end
-; GFX10END-OBJ-NEXT:      s_code_end
-; GFX10END-OBJ-NEXT:      s_code_end
-; GFX10END-OBJ-NEXT:      s_code_end
-; GFX10END-OBJ-NEXT:      s_code_end
-; GFX10END-OBJ-NEXT:      s_code_end
-
-; GFX10END-OBJ-NEXT:      s_code_end
-; GFX10END-OBJ-NEXT:      s_code_end
-; GFX10END-OBJ-NEXT:      s_code_end
-; GFX10END-OBJ-NEXT:      s_code_end
-; GFX10END-OBJ-NEXT:      s_code_end
-; GFX10END-OBJ-NEXT:      s_code_end
-; GFX10END-OBJ-NEXT:      s_code_end
-; GFX10END-OBJ-NEXT:      s_code_end
-
-; GFX10END-OBJ-NEXT:      s_code_end
-; GFX10END-OBJ-NEXT:      s_code_end
-; GFX10END-OBJ-NEXT:      s_code_end
-; GFX10END-OBJ-NEXT:      s_code_end
-; GFX10END-OBJ-NEXT:      s_code_end
-; GFX10END-OBJ-NEXT:      s_code_end
-; GFX10END-OBJ-NEXT:      s_code_end
-; GFX10END-OBJ-NEXT:      s_code_end
+; GFX10END-OBJ-COUNT-47:  s_code_end
 
 define void @a_function() {
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/saddo.ll b/llvm/test/CodeGen/AMDGPU/saddo.ll
index a5d7592fa..219ea42a7 100644
--- a/llvm/test/CodeGen/AMDGPU/saddo.ll
+++ b/llvm/test/CodeGen/AMDGPU/saddo.ll
@@ -1,6 +1,7 @@
-; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tahiti -verify-machineinstrs< %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC %s
-; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tonga -verify-machineinstrs< %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC %s
-; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=gfx900 -verify-machineinstrs< %s | FileCheck -check-prefixes=GCN,GFX9,FUNC %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tahiti -verify-machineinstrs | FileCheck %s -check-prefixes=FUNC,GCN,SICIVI,SI
+; RUN: llc < %s -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck %s -check-prefixes=FUNC,GCN,SICIVI,VI
+; RUN: llc < %s -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=gfx900 -verify-machineinstrs | FileCheck %s -check-prefixes=FUNC,GCN,GFX9
 
 
 declare { i32, i1 } @llvm.sadd.with.overflow.i32(i32, i32) nounwind readnone
@@ -9,8 +10,71 @@ declare { i64, i1 } @llvm.sadd.with.overflow.i64(i64, i64) nounwind readnone
 
 declare { <2 x i32>, <2 x i1> } @llvm.sadd.with.overflow.v2i32(<2 x i32>, <2 x i32>) nounwind readnone
 
-; FUNC-LABEL: {{^}}saddo_i64_zext:
 define amdgpu_kernel void @saddo_i64_zext(i64 addrspace(1)* %out, i64 %a, i64 %b) nounwind {
+; SI-LABEL: saddo_i64_zext:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SI-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xd
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v0, s6
+; SI-NEXT:    s_add_u32 s10, s6, s8
+; SI-NEXT:    s_addc_u32 s11, s7, s9
+; SI-NEXT:    v_mov_b32_e32 v1, s7
+; SI-NEXT:    v_cmp_lt_i64_e32 vcc, s[10:11], v[0:1]
+; SI-NEXT:    v_cmp_lt_i64_e64 s[6:7], s[8:9], 0
+; SI-NEXT:    s_mov_b32 s0, s4
+; SI-NEXT:    s_mov_b32 s1, s5
+; SI-NEXT:    s_xor_b64 s[4:5], s[6:7], vcc
+; SI-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; SI-NEXT:    v_mov_b32_e32 v1, s11
+; SI-NEXT:    v_add_i32_e32 v0, vcc, s10, v0
+; SI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: saddo_i64_zext:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s6
+; VI-NEXT:    s_add_u32 s2, s6, s0
+; VI-NEXT:    s_addc_u32 s3, s7, s1
+; VI-NEXT:    v_mov_b32_e32 v2, s7
+; VI-NEXT:    v_cmp_lt_i64_e32 vcc, s[2:3], v[1:2]
+; VI-NEXT:    v_cmp_lt_i64_e64 s[8:9], s[0:1], 0
+; VI-NEXT:    v_mov_b32_e32 v3, s3
+; VI-NEXT:    s_xor_b64 s[0:1], s[8:9], vcc
+; VI-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
+; VI-NEXT:    v_add_u32_e32 v2, vcc, s2, v2
+; VI-NEXT:    v_mov_b32_e32 v0, s4
+; VI-NEXT:    v_mov_b32_e32 v1, s5
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-NEXT:    s_endpgm
+;
+; GFX9-LABEL: saddo_i64_zext:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v1, s6
+; GFX9-NEXT:    s_add_u32 s2, s6, s0
+; GFX9-NEXT:    s_addc_u32 s3, s7, s1
+; GFX9-NEXT:    v_mov_b32_e32 v2, s7
+; GFX9-NEXT:    v_cmp_lt_i64_e32 vcc, s[2:3], v[1:2]
+; GFX9-NEXT:    v_cmp_lt_i64_e64 s[8:9], s[0:1], 0
+; GFX9-NEXT:    v_mov_b32_e32 v3, s3
+; GFX9-NEXT:    s_xor_b64 s[0:1], s[8:9], vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, s2, v2
+; GFX9-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    global_store_dwordx2 v[0:1], v[2:3], off
+; GFX9-NEXT:    s_endpgm
   %sadd = call { i64, i1 } @llvm.sadd.with.overflow.i64(i64 %a, i64 %b) nounwind
   %val = extractvalue { i64, i1 } %sadd, 0
   %carry = extractvalue { i64, i1 } %sadd, 1
@@ -20,8 +84,71 @@ define amdgpu_kernel void @saddo_i64_zext(i64 addrspace(1)* %out, i64 %a, i64 %b
   ret void
 }
 
-; FUNC-LABEL: {{^}}s_saddo_i32:
 define amdgpu_kernel void @s_saddo_i32(i32 addrspace(1)* %out, i1 addrspace(1)* %carryout, i32 %a, i32 %b) nounwind {
+; SI-LABEL: s_saddo_i32:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SI-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xd
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b32 s0, s4
+; SI-NEXT:    v_cmp_lt_i32_e64 s[10:11], s9, 0
+; SI-NEXT:    s_add_i32 s9, s8, s9
+; SI-NEXT:    v_mov_b32_e32 v0, s8
+; SI-NEXT:    s_mov_b32 s1, s5
+; SI-NEXT:    v_cmp_lt_i32_e32 vcc, s9, v0
+; SI-NEXT:    v_mov_b32_e32 v0, s9
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT:    s_xor_b64 s[0:1], s[10:11], vcc
+; SI-NEXT:    s_mov_b32 s4, s6
+; SI-NEXT:    s_mov_b32 s5, s7
+; SI-NEXT:    s_mov_b32 s6, s2
+; SI-NEXT:    s_mov_b32 s7, s3
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; SI-NEXT:    buffer_store_byte v0, off, s[4:7], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: s_saddo_i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v0, s4
+; VI-NEXT:    v_cmp_lt_i32_e64 s[2:3], s1, 0
+; VI-NEXT:    s_add_i32 s1, s0, s1
+; VI-NEXT:    v_mov_b32_e32 v4, s0
+; VI-NEXT:    v_cmp_lt_i32_e32 vcc, s1, v4
+; VI-NEXT:    v_mov_b32_e32 v4, s1
+; VI-NEXT:    v_mov_b32_e32 v1, s5
+; VI-NEXT:    s_xor_b64 s[0:1], s[2:3], vcc
+; VI-NEXT:    flat_store_dword v[0:1], v4
+; VI-NEXT:    v_mov_b32_e32 v2, s6
+; VI-NEXT:    v_mov_b32_e32 v3, s7
+; VI-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; VI-NEXT:    flat_store_byte v[2:3], v0
+; VI-NEXT:    s_endpgm
+;
+; GFX9-LABEL: s_saddo_i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-NEXT:    v_cmp_lt_i32_e64 s[2:3], s1, 0
+; GFX9-NEXT:    s_add_i32 s1, s0, s1
+; GFX9-NEXT:    v_mov_b32_e32 v4, s0
+; GFX9-NEXT:    v_cmp_lt_i32_e32 vcc, s1, v4
+; GFX9-NEXT:    v_mov_b32_e32 v4, s1
+; GFX9-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-NEXT:    s_xor_b64 s[0:1], s[2:3], vcc
+; GFX9-NEXT:    global_store_dword v[0:1], v4, off
+; GFX9-NEXT:    v_mov_b32_e32 v2, s6
+; GFX9-NEXT:    v_mov_b32_e32 v3, s7
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; GFX9-NEXT:    global_store_byte v[2:3], v0, off
+; GFX9-NEXT:    s_endpgm
   %sadd = call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 %a, i32 %b) nounwind
   %val = extractvalue { i32, i1 } %sadd, 0
   %carry = extractvalue { i32, i1 } %sadd, 1
@@ -30,8 +157,82 @@ define amdgpu_kernel void @s_saddo_i32(i32 addrspace(1)* %out, i1 addrspace(1)*
   ret void
 }
 
-; FUNC-LABEL: {{^}}v_saddo_i32:
 define amdgpu_kernel void @v_saddo_i32(i32 addrspace(1)* %out, i1 addrspace(1)* %carryout, i32 addrspace(1)* %aptr, i32 addrspace(1)* %bptr) nounwind {
+; SI-LABEL: v_saddo_i32:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx8 s[4:11], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s15, 0xf000
+; SI-NEXT:    s_mov_b32 s14, -1
+; SI-NEXT:    s_mov_b32 s2, s14
+; SI-NEXT:    s_mov_b32 s3, s15
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b32 s0, s10
+; SI-NEXT:    s_mov_b32 s1, s11
+; SI-NEXT:    s_mov_b32 s10, s14
+; SI-NEXT:    s_mov_b32 s11, s15
+; SI-NEXT:    buffer_load_dword v0, off, s[8:11], 0
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], 0
+; SI-NEXT:    s_mov_b32 s12, s6
+; SI-NEXT:    s_mov_b32 s13, s7
+; SI-NEXT:    s_mov_b32 s6, s14
+; SI-NEXT:    s_mov_b32 s7, s15
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_add_i32_e32 v2, vcc, v1, v0
+; SI-NEXT:    v_cmp_gt_i32_e32 vcc, 0, v1
+; SI-NEXT:    v_cmp_lt_i32_e64 s[0:1], v2, v0
+; SI-NEXT:    s_xor_b64 s[0:1], vcc, s[0:1]
+; SI-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; SI-NEXT:    buffer_store_dword v2, off, s[4:7], 0
+; SI-NEXT:    buffer_store_byte v0, off, s[12:15], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: v_saddo_i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx8 s[0:7], s[0:1], 0x24
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v4, s6
+; VI-NEXT:    v_mov_b32_e32 v5, s7
+; VI-NEXT:    v_mov_b32_e32 v6, s4
+; VI-NEXT:    v_mov_b32_e32 v7, s5
+; VI-NEXT:    flat_load_dword v6, v[6:7]
+; VI-NEXT:    flat_load_dword v4, v[4:5]
+; VI-NEXT:    v_mov_b32_e32 v2, s0
+; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    v_mov_b32_e32 v0, s2
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_add_u32_e32 v5, vcc, v4, v6
+; VI-NEXT:    v_cmp_gt_i32_e32 vcc, 0, v4
+; VI-NEXT:    v_cmp_lt_i32_e64 s[0:1], v5, v6
+; VI-NEXT:    s_xor_b64 s[0:1], vcc, s[0:1]
+; VI-NEXT:    flat_store_dword v[2:3], v5
+; VI-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
+; VI-NEXT:    flat_store_byte v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; GFX9-LABEL: v_saddo_i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx8 s[0:7], s[0:1], 0x24
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v4, s6
+; GFX9-NEXT:    v_mov_b32_e32 v5, s7
+; GFX9-NEXT:    v_mov_b32_e32 v6, s4
+; GFX9-NEXT:    v_mov_b32_e32 v7, s5
+; GFX9-NEXT:    global_load_dword v6, v[6:7], off
+; GFX9-NEXT:    global_load_dword v4, v[4:5], off
+; GFX9-NEXT:    v_mov_b32_e32 v2, s0
+; GFX9-NEXT:    v_mov_b32_e32 v3, s1
+; GFX9-NEXT:    v_mov_b32_e32 v0, s2
+; GFX9-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_add_u32_e32 v5, v6, v4
+; GFX9-NEXT:    v_cmp_gt_i32_e32 vcc, 0, v4
+; GFX9-NEXT:    v_cmp_lt_i32_e64 s[0:1], v5, v6
+; GFX9-NEXT:    s_xor_b64 s[0:1], vcc, s[0:1]
+; GFX9-NEXT:    global_store_dword v[2:3], v5, off
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
+; GFX9-NEXT:    global_store_byte v[0:1], v2, off
+; GFX9-NEXT:    s_endpgm
   %a = load i32, i32 addrspace(1)* %aptr, align 4
   %b = load i32, i32 addrspace(1)* %bptr, align 4
   %sadd = call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 %a, i32 %b) nounwind
@@ -42,8 +243,75 @@ define amdgpu_kernel void @v_saddo_i32(i32 addrspace(1)* %out, i1 addrspace(1)*
   ret void
 }
 
-; FUNC-LABEL: {{^}}s_saddo_i64:
 define amdgpu_kernel void @s_saddo_i64(i64 addrspace(1)* %out, i1 addrspace(1)* %carryout, i64 %a, i64 %b) nounwind {
+; SI-LABEL: s_saddo_i64:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx8 s[0:7], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s11, 0xf000
+; SI-NEXT:    s_mov_b32 s10, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_add_u32 s12, s4, s6
+; SI-NEXT:    v_mov_b32_e32 v0, s4
+; SI-NEXT:    s_addc_u32 s13, s5, s7
+; SI-NEXT:    v_mov_b32_e32 v1, s5
+; SI-NEXT:    v_cmp_lt_i64_e32 vcc, s[12:13], v[0:1]
+; SI-NEXT:    v_mov_b32_e32 v0, s12
+; SI-NEXT:    v_cmp_lt_i64_e64 s[4:5], s[6:7], 0
+; SI-NEXT:    s_mov_b32 s8, s2
+; SI-NEXT:    s_mov_b32 s9, s3
+; SI-NEXT:    s_mov_b32 s2, s10
+; SI-NEXT:    s_mov_b32 s3, s11
+; SI-NEXT:    v_mov_b32_e32 v1, s13
+; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; SI-NEXT:    s_xor_b64 s[0:1], s[4:5], vcc
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; SI-NEXT:    buffer_store_byte v0, off, s[8:11], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: s_saddo_i64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx8 s[0:7], s[0:1], 0x24
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v2, s0
+; VI-NEXT:    v_mov_b32_e32 v4, s4
+; VI-NEXT:    s_add_u32 s0, s4, s6
+; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    s_addc_u32 s1, s5, s7
+; VI-NEXT:    v_mov_b32_e32 v5, s5
+; VI-NEXT:    v_cmp_lt_i64_e32 vcc, s[0:1], v[4:5]
+; VI-NEXT:    v_mov_b32_e32 v0, s2
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_cmp_lt_i64_e64 s[2:3], s[6:7], 0
+; VI-NEXT:    v_mov_b32_e32 v5, s1
+; VI-NEXT:    v_mov_b32_e32 v4, s0
+; VI-NEXT:    s_xor_b64 s[0:1], s[2:3], vcc
+; VI-NEXT:    flat_store_dwordx2 v[2:3], v[4:5]
+; VI-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
+; VI-NEXT:    flat_store_byte v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; GFX9-LABEL: s_saddo_i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx8 s[0:7], s[0:1], 0x24
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v2, s0
+; GFX9-NEXT:    v_mov_b32_e32 v4, s4
+; GFX9-NEXT:    s_add_u32 s0, s4, s6
+; GFX9-NEXT:    v_mov_b32_e32 v3, s1
+; GFX9-NEXT:    s_addc_u32 s1, s5, s7
+; GFX9-NEXT:    v_mov_b32_e32 v5, s5
+; GFX9-NEXT:    v_cmp_lt_i64_e32 vcc, s[0:1], v[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v0, s2
+; GFX9-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NEXT:    v_cmp_lt_i64_e64 s[2:3], s[6:7], 0
+; GFX9-NEXT:    v_mov_b32_e32 v5, s1
+; GFX9-NEXT:    v_mov_b32_e32 v4, s0
+; GFX9-NEXT:    s_xor_b64 s[0:1], s[2:3], vcc
+; GFX9-NEXT:    global_store_dwordx2 v[2:3], v[4:5], off
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
+; GFX9-NEXT:    global_store_byte v[0:1], v2, off
+; GFX9-NEXT:    s_endpgm
   %sadd = call { i64, i1 } @llvm.sadd.with.overflow.i64(i64 %a, i64 %b) nounwind
   %val = extractvalue { i64, i1 } %sadd, 0
   %carry = extractvalue { i64, i1 } %sadd, 1
@@ -52,13 +320,85 @@ define amdgpu_kernel void @s_saddo_i64(i64 addrspace(1)* %out, i1 addrspace(1)*
   ret void
 }
 
-; FUNC-LABEL: {{^}}v_saddo_i64:
-; SICIVI: v_add_{{[iu]}}32_e32 v{{[0-9]+}}, vcc
-; SICIVI: v_addc_u32_e32 v{{[0-9]+}}, vcc
-
-; GFX9: v_add_co_u32_e32 v{{[0-9]+}}, vcc
-; GFX9: v_addc_co_u32_e32 v{{[0-9]+}}, vcc
 define amdgpu_kernel void @v_saddo_i64(i64 addrspace(1)* %out, i1 addrspace(1)* %carryout, i64 addrspace(1)* %aptr, i64 addrspace(1)* %bptr) nounwind {
+; SI-LABEL: v_saddo_i64:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx8 s[4:11], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s15, 0xf000
+; SI-NEXT:    s_mov_b32 s14, -1
+; SI-NEXT:    s_mov_b32 s2, s14
+; SI-NEXT:    s_mov_b32 s3, s15
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b32 s0, s10
+; SI-NEXT:    s_mov_b32 s1, s11
+; SI-NEXT:    s_mov_b32 s10, s14
+; SI-NEXT:    s_mov_b32 s11, s15
+; SI-NEXT:    buffer_load_dwordx2 v[0:1], off, s[8:11], 0
+; SI-NEXT:    buffer_load_dwordx2 v[2:3], off, s[0:3], 0
+; SI-NEXT:    s_mov_b32 s12, s6
+; SI-NEXT:    s_mov_b32 s13, s7
+; SI-NEXT:    s_mov_b32 s6, s14
+; SI-NEXT:    s_mov_b32 s7, s15
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_add_i32_e32 v4, vcc, v0, v2
+; SI-NEXT:    v_addc_u32_e32 v5, vcc, v1, v3, vcc
+; SI-NEXT:    v_cmp_gt_i64_e32 vcc, 0, v[2:3]
+; SI-NEXT:    v_cmp_lt_i64_e64 s[0:1], v[4:5], v[0:1]
+; SI-NEXT:    buffer_store_dwordx2 v[4:5], off, s[4:7], 0
+; SI-NEXT:    s_xor_b64 s[0:1], vcc, s[0:1]
+; SI-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; SI-NEXT:    buffer_store_byte v0, off, s[12:15], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: v_saddo_i64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx8 s[0:7], s[0:1], 0x24
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v4, s6
+; VI-NEXT:    v_mov_b32_e32 v5, s7
+; VI-NEXT:    v_mov_b32_e32 v6, s4
+; VI-NEXT:    v_mov_b32_e32 v7, s5
+; VI-NEXT:    flat_load_dwordx2 v[6:7], v[6:7]
+; VI-NEXT:    flat_load_dwordx2 v[4:5], v[4:5]
+; VI-NEXT:    v_mov_b32_e32 v2, s0
+; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    v_mov_b32_e32 v0, s2
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_add_u32_e32 v8, vcc, v6, v4
+; VI-NEXT:    v_addc_u32_e32 v9, vcc, v7, v5, vcc
+; VI-NEXT:    v_cmp_gt_i64_e32 vcc, 0, v[4:5]
+; VI-NEXT:    v_cmp_lt_i64_e64 s[0:1], v[8:9], v[6:7]
+; VI-NEXT:    flat_store_dwordx2 v[2:3], v[8:9]
+; VI-NEXT:    s_xor_b64 s[0:1], vcc, s[0:1]
+; VI-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
+; VI-NEXT:    flat_store_byte v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; GFX9-LABEL: v_saddo_i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx8 s[0:7], s[0:1], 0x24
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v4, s6
+; GFX9-NEXT:    v_mov_b32_e32 v5, s7
+; GFX9-NEXT:    v_mov_b32_e32 v6, s4
+; GFX9-NEXT:    v_mov_b32_e32 v7, s5
+; GFX9-NEXT:    global_load_dwordx2 v[6:7], v[6:7], off
+; GFX9-NEXT:    global_load_dwordx2 v[4:5], v[4:5], off
+; GFX9-NEXT:    v_mov_b32_e32 v2, s0
+; GFX9-NEXT:    v_mov_b32_e32 v3, s1
+; GFX9-NEXT:    v_mov_b32_e32 v0, s2
+; GFX9-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, v6, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, v7, v5, vcc
+; GFX9-NEXT:    v_cmp_gt_i64_e32 vcc, 0, v[4:5]
+; GFX9-NEXT:    v_cmp_lt_i64_e64 s[0:1], v[8:9], v[6:7]
+; GFX9-NEXT:    global_store_dwordx2 v[2:3], v[8:9], off
+; GFX9-NEXT:    s_xor_b64 s[0:1], vcc, s[0:1]
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
+; GFX9-NEXT:    global_store_byte v[0:1], v2, off
+; GFX9-NEXT:    s_endpgm
   %a = load i64, i64 addrspace(1)* %aptr, align 4
   %b = load i64, i64 addrspace(1)* %bptr, align 4
   %sadd = call { i64, i1 } @llvm.sadd.with.overflow.i64(i64 %a, i64 %b) nounwind
@@ -69,14 +409,97 @@ define amdgpu_kernel void @v_saddo_i64(i64 addrspace(1)* %out, i1 addrspace(1)*
   ret void
 }
 
-; FUNC-LABEL: {{^}}v_saddo_v2i32:
-; SICIVI: v_cmp_lt_i32
-; SICIVI: v_cmp_lt_i32
-; SICIVI: v_add_{{[iu]}}32
-; SICIVI: v_cmp_lt_i32
-; SICIVI: v_cmp_lt_i32
-; SICIVI: v_add_{{[iu]}}32
 define amdgpu_kernel void @v_saddo_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32> addrspace(1)* %carryout, <2 x i32> addrspace(1)* %aptr, <2 x i32> addrspace(1)* %bptr) nounwind {
+; SI-LABEL: v_saddo_v2i32:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx8 s[8:15], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s19, 0xf000
+; SI-NEXT:    s_mov_b32 s18, -1
+; SI-NEXT:    s_mov_b32 s2, s18
+; SI-NEXT:    s_mov_b32 s3, s19
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b32 s0, s14
+; SI-NEXT:    s_mov_b32 s1, s15
+; SI-NEXT:    s_mov_b32 s14, s18
+; SI-NEXT:    s_mov_b32 s15, s19
+; SI-NEXT:    buffer_load_dwordx2 v[0:1], off, s[12:15], 0
+; SI-NEXT:    buffer_load_dwordx2 v[2:3], off, s[0:3], 0
+; SI-NEXT:    s_mov_b32 s16, s10
+; SI-NEXT:    s_mov_b32 s17, s11
+; SI-NEXT:    s_mov_b32 s10, s18
+; SI-NEXT:    s_mov_b32 s11, s19
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_add_i32_e32 v5, vcc, v1, v3
+; SI-NEXT:    v_add_i32_e32 v4, vcc, v0, v2
+; SI-NEXT:    v_cmp_gt_i32_e64 s[0:1], 0, v3
+; SI-NEXT:    v_cmp_lt_i32_e64 s[4:5], v5, v1
+; SI-NEXT:    s_xor_b64 s[0:1], s[0:1], s[4:5]
+; SI-NEXT:    v_cmp_gt_i32_e32 vcc, 0, v2
+; SI-NEXT:    v_cmp_lt_i32_e64 s[2:3], v4, v0
+; SI-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s[0:1]
+; SI-NEXT:    s_xor_b64 s[0:1], vcc, s[2:3]
+; SI-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; SI-NEXT:    buffer_store_dwordx2 v[4:5], off, s[8:11], 0
+; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[16:19], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: v_saddo_v2i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx8 s[0:7], s[0:1], 0x24
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v4, s6
+; VI-NEXT:    v_mov_b32_e32 v5, s7
+; VI-NEXT:    v_mov_b32_e32 v6, s4
+; VI-NEXT:    v_mov_b32_e32 v7, s5
+; VI-NEXT:    flat_load_dwordx2 v[6:7], v[6:7]
+; VI-NEXT:    flat_load_dwordx2 v[4:5], v[4:5]
+; VI-NEXT:    v_mov_b32_e32 v2, s0
+; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    v_mov_b32_e32 v0, s2
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_add_u32_e32 v9, vcc, v7, v5
+; VI-NEXT:    v_add_u32_e32 v8, vcc, v6, v4
+; VI-NEXT:    v_cmp_gt_i32_e64 s[0:1], 0, v5
+; VI-NEXT:    v_cmp_lt_i32_e64 s[4:5], v9, v7
+; VI-NEXT:    v_cmp_gt_i32_e32 vcc, 0, v4
+; VI-NEXT:    v_cmp_lt_i32_e64 s[2:3], v8, v6
+; VI-NEXT:    s_xor_b64 s[0:1], s[0:1], s[4:5]
+; VI-NEXT:    flat_store_dwordx2 v[2:3], v[8:9]
+; VI-NEXT:    v_cndmask_b32_e64 v3, 0, 1, s[0:1]
+; VI-NEXT:    s_xor_b64 s[0:1], vcc, s[2:3]
+; VI-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
+; VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-NEXT:    s_endpgm
+;
+; GFX9-LABEL: v_saddo_v2i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx8 s[0:7], s[0:1], 0x24
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v4, s6
+; GFX9-NEXT:    v_mov_b32_e32 v5, s7
+; GFX9-NEXT:    v_mov_b32_e32 v6, s4
+; GFX9-NEXT:    v_mov_b32_e32 v7, s5
+; GFX9-NEXT:    global_load_dwordx2 v[6:7], v[6:7], off
+; GFX9-NEXT:    global_load_dwordx2 v[4:5], v[4:5], off
+; GFX9-NEXT:    v_mov_b32_e32 v2, s0
+; GFX9-NEXT:    v_mov_b32_e32 v3, s1
+; GFX9-NEXT:    v_mov_b32_e32 v0, s2
+; GFX9-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_add_u32_e32 v9, v7, v5
+; GFX9-NEXT:    v_add_u32_e32 v8, v6, v4
+; GFX9-NEXT:    v_cmp_gt_i32_e64 s[0:1], 0, v5
+; GFX9-NEXT:    v_cmp_lt_i32_e64 s[4:5], v9, v7
+; GFX9-NEXT:    v_cmp_gt_i32_e32 vcc, 0, v4
+; GFX9-NEXT:    v_cmp_lt_i32_e64 s[2:3], v8, v6
+; GFX9-NEXT:    s_xor_b64 s[0:1], s[0:1], s[4:5]
+; GFX9-NEXT:    global_store_dwordx2 v[2:3], v[8:9], off
+; GFX9-NEXT:    v_cndmask_b32_e64 v3, 0, 1, s[0:1]
+; GFX9-NEXT:    s_xor_b64 s[0:1], vcc, s[2:3]
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
+; GFX9-NEXT:    global_store_dwordx2 v[0:1], v[2:3], off
+; GFX9-NEXT:    s_endpgm
   %a = load <2 x i32>, <2 x i32> addrspace(1)* %aptr, align 4
   %b = load <2 x i32>, <2 x i32> addrspace(1)* %bptr, align 4
   %sadd = call { <2 x i32>, <2 x i1> } @llvm.sadd.with.overflow.v2i32(<2 x i32> %a, <2 x i32> %b) nounwind
diff --git a/llvm/test/CodeGen/AMDGPU/scalar_to_vector.ll b/llvm/test/CodeGen/AMDGPU/scalar_to_vector.ll
index ff634c609..73d98ab5b 100644
--- a/llvm/test/CodeGen/AMDGPU/scalar_to_vector.ll
+++ b/llvm/test/CodeGen/AMDGPU/scalar_to_vector.ll
@@ -1,15 +1,50 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -amdgpu-scalarize-global-loads=false  -march=amdgcn -mattr=-flat-for-global -verify-machineinstrs | FileCheck %s -check-prefixes=GCN,SI
+; RUN: llc < %s -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs | FileCheck %s -check-prefixes=GCN,VI
 
 ; XXX - Why the packing?
-; GCN-LABEL: {{^}}scalar_to_vector_v2i32:
-; GCN: buffer_load_dword [[VAL:v[0-9]+]],
-; GCN: v_lshrrev_b32_e32 [[SHR:v[0-9]+]], 16, [[VAL]]
-; GCN: v_lshlrev_b32_e32 [[SHL:v[0-9]+]], 16, [[SHR]]
-; GCN: v_or_b32_e32 v[[OR:[0-9]+]], [[SHR]], [[SHL]]
-; GCN: v_mov_b32_e32 v[[COPY:[0-9]+]], v[[OR]]
-; GCN: buffer_store_dwordx2 v{{\[}}[[OR]]:[[COPY]]{{\]}}
 define amdgpu_kernel void @scalar_to_vector_v2i32(<4 x i16> addrspace(1)* %out, i32 addrspace(1)* %in) nounwind {
+; SI-LABEL: scalar_to_vector_v2i32:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_mov_b32 s10, s6
+; SI-NEXT:    s_mov_b32 s11, s7
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b32 s8, s2
+; SI-NEXT:    s_mov_b32 s9, s3
+; SI-NEXT:    buffer_load_dword v0, off, s[8:11], 0
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v0
+; SI-NEXT:    v_or_b32_e32 v0, v0, v1
+; SI-NEXT:    s_mov_b32 s4, s0
+; SI-NEXT:    s_mov_b32 s5, s1
+; SI-NEXT:    v_mov_b32_e32 v1, v0
+; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: scalar_to_vector_v2i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; VI-NEXT:    s_mov_b32 s3, 0xf000
+; VI-NEXT:    s_mov_b32 s2, -1
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_mov_b32 s0, s4
+; VI-NEXT:    s_mov_b32 s1, s5
+; VI-NEXT:    s_mov_b32 s4, s6
+; VI-NEXT:    s_mov_b32 s5, s7
+; VI-NEXT:    s_mov_b32 s6, s2
+; VI-NEXT:    s_mov_b32 s7, s3
+; VI-NEXT:    buffer_load_dword v0, off, s[4:7], 0
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; VI-NEXT:    v_lshlrev_b32_e32 v1, 16, v0
+; VI-NEXT:    v_or_b32_e32 v0, v0, v1
+; VI-NEXT:    v_mov_b32_e32 v1, v0
+; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; VI-NEXT:    s_endpgm
   %tmp1 = load i32, i32 addrspace(1)* %in, align 4
   %bc = bitcast i32 %tmp1 to <2 x i16>
   %tmp2 = shufflevector <2 x i16> %bc, <2 x i16> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
@@ -17,11 +52,48 @@ define amdgpu_kernel void @scalar_to_vector_v2i32(<4 x i16> addrspace(1)* %out,
   ret void
 }
 
-; GCN-LABEL: {{^}}scalar_to_vector_v2f32:
-; GCN: buffer_load_dword [[VAL:v[0-9]+]],
-; GCN: v_lshrrev_b32_e32 [[RESULT:v[0-9]+]], 16, [[VAL]]
-; GCN: buffer_store_dwordx2
 define amdgpu_kernel void @scalar_to_vector_v2f32(<4 x i16> addrspace(1)* %out, float addrspace(1)* %in) nounwind {
+; SI-LABEL: scalar_to_vector_v2f32:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_mov_b32 s10, s6
+; SI-NEXT:    s_mov_b32 s11, s7
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b32 s8, s2
+; SI-NEXT:    s_mov_b32 s9, s3
+; SI-NEXT:    buffer_load_dword v0, off, s[8:11], 0
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v0
+; SI-NEXT:    v_or_b32_e32 v0, v0, v1
+; SI-NEXT:    s_mov_b32 s4, s0
+; SI-NEXT:    s_mov_b32 s5, s1
+; SI-NEXT:    v_mov_b32_e32 v1, v0
+; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: scalar_to_vector_v2f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; VI-NEXT:    s_mov_b32 s3, 0xf000
+; VI-NEXT:    s_mov_b32 s2, -1
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_mov_b32 s0, s4
+; VI-NEXT:    s_mov_b32 s1, s5
+; VI-NEXT:    s_mov_b32 s4, s6
+; VI-NEXT:    s_mov_b32 s5, s7
+; VI-NEXT:    s_mov_b32 s6, s2
+; VI-NEXT:    s_mov_b32 s7, s3
+; VI-NEXT:    buffer_load_dword v0, off, s[4:7], 0
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; VI-NEXT:    v_lshlrev_b32_e32 v1, 16, v0
+; VI-NEXT:    v_or_b32_e32 v0, v0, v1
+; VI-NEXT:    v_mov_b32_e32 v1, v0
+; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; VI-NEXT:    s_endpgm
   %tmp1 = load float, float addrspace(1)* %in, align 4
   %bc = bitcast float %tmp1 to <2 x i16>
   %tmp2 = shufflevector <2 x i16> %bc, <2 x i16> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
@@ -29,14 +101,40 @@ define amdgpu_kernel void @scalar_to_vector_v2f32(<4 x i16> addrspace(1)* %out,
   ret void
 }
 
-; GCN-LABEL: {{^}}scalar_to_vector_v4i16:
-; VI: v_lshlrev_b16_e32
-; VI: v_lshlrev_b16_e32
-; VI: v_or_b32_e32
-; VI: v_lshlrev_b32
-; VI: v_or_b32_sdwa
-; VI: v_or_b32_sdwa
 define amdgpu_kernel void @scalar_to_vector_v4i16() {
+; SI-LABEL: scalar_to_vector_v4i16:
+; SI:       ; %bb.0: ; %bb
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    buffer_load_ubyte v0, off, s[0:3], 0
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v0
+; SI-NEXT:    v_or_b32_e32 v0, v1, v0
+; SI-NEXT:    v_lshrrev_b32_e32 v1, 8, v0
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 8, v1
+; SI-NEXT:    v_or_b32_e32 v1, v1, v2
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v1
+; SI-NEXT:    v_or_b32_e32 v1, v1, v2
+; SI-NEXT:    v_or_b32_e32 v0, v0, v2
+; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: scalar_to_vector_v4i16:
+; VI:       ; %bb.0: ; %bb
+; VI-NEXT:    s_mov_b32 s3, 0xf000
+; VI-NEXT:    s_mov_b32 s2, -1
+; VI-NEXT:    buffer_load_ubyte v0, off, s[0:3], 0
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v0
+; VI-NEXT:    v_or_b32_e32 v0, v1, v0
+; VI-NEXT:    v_lshrrev_b16_e32 v1, 8, v0
+; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v1
+; VI-NEXT:    v_or_b32_e32 v1, v1, v2
+; VI-NEXT:    v_lshlrev_b32_e32 v2, 16, v1
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; VI-NEXT:    s_endpgm
 bb:
   %tmp = load <2 x i8>, <2 x i8> addrspace(1)* undef, align 1
   %tmp1 = shufflevector <2 x i8> %tmp, <2 x i8> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
@@ -45,14 +143,40 @@ bb:
   ret void
 }
 
-; GCN-LABEL: {{^}}scalar_to_vector_v4f16:
-; VI: v_lshlrev_b16_e32
-; VI: v_lshlrev_b16_e32
-; VI: v_or_b32_e32
-; VI: v_lshlrev_b32
-; VI: v_or_b32_sdwa
-; VI: v_or_b32_sdwa
 define amdgpu_kernel void @scalar_to_vector_v4f16() {
+; SI-LABEL: scalar_to_vector_v4f16:
+; SI:       ; %bb.0: ; %bb
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    buffer_load_ubyte v0, off, s[0:3], 0
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v0
+; SI-NEXT:    v_or_b32_e32 v0, v1, v0
+; SI-NEXT:    v_lshrrev_b32_e32 v1, 8, v0
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 8, v1
+; SI-NEXT:    v_or_b32_e32 v1, v1, v2
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v1
+; SI-NEXT:    v_or_b32_e32 v1, v1, v2
+; SI-NEXT:    v_or_b32_e32 v0, v0, v2
+; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: scalar_to_vector_v4f16:
+; VI:       ; %bb.0: ; %bb
+; VI-NEXT:    s_mov_b32 s3, 0xf000
+; VI-NEXT:    s_mov_b32 s2, -1
+; VI-NEXT:    buffer_load_ubyte v0, off, s[0:3], 0
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v0
+; VI-NEXT:    v_or_b32_e32 v0, v1, v0
+; VI-NEXT:    v_lshrrev_b16_e32 v1, 8, v0
+; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v1
+; VI-NEXT:    v_or_b32_e32 v1, v1, v2
+; VI-NEXT:    v_lshlrev_b32_e32 v2, 16, v1
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; VI-NEXT:    s_endpgm
 bb:
   %load = load half, half addrspace(1)* undef, align 1
   %tmp = bitcast half %load to <2 x i8>
@@ -101,6 +225,29 @@ bb:
 ; }
 
 define amdgpu_kernel void @scalar_to_vector_test6(<2 x half> addrspace(1)* %out, i8 zeroext %val) nounwind {
+; SI-LABEL: scalar_to_vector_test6:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dword s2, s[0:1], 0xb
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_and_b32 s0, s2, 0xff
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    v_mov_b32_e32 v0, s0
+; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: scalar_to_vector_test6:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
+; VI-NEXT:    s_load_dword s0, s[0:1], 0x2c
+; VI-NEXT:    s_mov_b32 s7, 0xf000
+; VI-NEXT:    s_mov_b32 s6, -1
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_and_b32 s0, s0, 0xff
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; VI-NEXT:    s_endpgm
   %newvec0 = insertelement <4 x i8> undef, i8 %val, i32 0
   %bc = bitcast <4 x i8> %newvec0 to <2 x half>
   store <2 x half> %bc, <2 x half> addrspace(1)* %out
diff --git a/llvm/test/CodeGen/AMDGPU/scratch-simple.ll b/llvm/test/CodeGen/AMDGPU/scratch-simple.ll
index 627d39a98..670610f4b 100644
--- a/llvm/test/CodeGen/AMDGPU/scratch-simple.ll
+++ b/llvm/test/CodeGen/AMDGPU/scratch-simple.ll
@@ -22,9 +22,9 @@
 ; GFX9-DAG: s_mov_b32 s7, 0xe00000
 ; GFX10_W32-DAG: s_mov_b32 s7, 0x31c16000
 ; GFX10_W64-DAG: s_mov_b32 s7, 0x31e16000
-; GCN-NOT: s_mov_b32 s0
 ; GCN-DAG: v_lshlrev_b32_e32 [[BYTES:v[0-9]+]], 2, v0
 ; GCN-DAG: v_and_b32_e32 [[CLAMP_IDX:v[0-9]+]], 0x1fc, [[BYTES]]
+; GCN-NOT: s_mov_b32 s0
 
 ; GCN-DAG: v_or_b32_e32 [[LO_OFF:v[0-9]+]], 0x200, [[CLAMP_IDX]]
 ; GCN-DAG: v_or_b32_e32 [[HI_OFF:v[0-9]+]], 0x400, [[CLAMP_IDX]]
diff --git a/llvm/test/CodeGen/AMDGPU/sdiv.ll b/llvm/test/CodeGen/AMDGPU/sdiv.ll
index 03784dcba..d47e50194 100644
--- a/llvm/test/CodeGen/AMDGPU/sdiv.ll
+++ b/llvm/test/CodeGen/AMDGPU/sdiv.ll
@@ -1931,19 +1931,19 @@ define amdgpu_kernel void @v_sdiv_i24(i32 addrspace(1)* %out, i24 addrspace(1)*
 ; GCN-NEXT:    s_waitcnt vmcnt(1)
 ; GCN-NEXT:    v_or_b32_e32 v0, v0, v1
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
-; GCN-NEXT:    v_or_b32_e32 v1, v2, v3
-; GCN-NEXT:    v_xor_b32_e32 v2, v0, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v3
+; GCN-NEXT:    v_xor_b32_e32 v1, v1, v3
+; GCN-NEXT:    v_ashrrev_i32_e32 v1, 30, v1
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v0, v0
-; GCN-NEXT:    v_cvt_f32_i32_e32 v1, v1
-; GCN-NEXT:    v_ashrrev_i32_e32 v2, 30, v2
-; GCN-NEXT:    v_rcp_iflag_f32_e32 v3, v1
-; GCN-NEXT:    v_or_b32_e32 v2, 1, v2
+; GCN-NEXT:    v_cvt_f32_i32_e32 v2, v2
+; GCN-NEXT:    v_or_b32_e32 v1, 1, v1
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v3, v2
 ; GCN-NEXT:    v_mul_f32_e32 v3, v0, v3
 ; GCN-NEXT:    v_trunc_f32_e32 v3, v3
-; GCN-NEXT:    v_mad_f32 v0, -v3, v1, v0
+; GCN-NEXT:    v_mad_f32 v0, -v3, v2, v0
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v3, v3
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v0|, |v1|
-; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v2, vcc
+; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v0|, |v2|
+; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v1, vcc
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v3
 ; GCN-NEXT:    v_bfe_i32 v0, v0, 0, 24
 ; GCN-NEXT:    buffer_store_dword v0, off, s[4:7], 0
@@ -1970,21 +1970,21 @@ define amdgpu_kernel void @v_sdiv_i24(i32 addrspace(1)* %out, i24 addrspace(1)*
 ; TONGA-NEXT:    s_waitcnt vmcnt(1)
 ; TONGA-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; TONGA-NEXT:    v_or_b32_e32 v1, v1, v2
-; TONGA-NEXT:    v_cvt_f32_i32_e32 v2, v1
+; TONGA-NEXT:    v_cvt_f32_i32_e32 v1, v1
 ; TONGA-NEXT:    s_waitcnt vmcnt(0)
-; TONGA-NEXT:    v_or_b32_e32 v0, v3, v0
-; TONGA-NEXT:    v_cvt_f32_i32_e32 v3, v0
-; TONGA-NEXT:    v_xor_b32_e32 v0, v0, v1
-; TONGA-NEXT:    v_rcp_iflag_f32_e32 v4, v2
+; TONGA-NEXT:    v_or_b32_e32 v3, v3, v0
+; TONGA-NEXT:    v_cvt_f32_i32_e32 v3, v3
+; TONGA-NEXT:    v_xor_b32_e32 v0, v0, v2
+; TONGA-NEXT:    v_rcp_iflag_f32_e32 v4, v1
 ; TONGA-NEXT:    v_ashrrev_i32_e32 v0, 30, v0
 ; TONGA-NEXT:    v_or_b32_e32 v0, 1, v0
-; TONGA-NEXT:    v_mul_f32_e32 v1, v3, v4
-; TONGA-NEXT:    v_trunc_f32_e32 v1, v1
-; TONGA-NEXT:    v_mad_f32 v3, -v1, v2, v3
-; TONGA-NEXT:    v_cvt_i32_f32_e32 v1, v1
-; TONGA-NEXT:    v_cmp_ge_f32_e64 vcc, |v3|, |v2|
+; TONGA-NEXT:    v_mul_f32_e32 v2, v3, v4
+; TONGA-NEXT:    v_trunc_f32_e32 v2, v2
+; TONGA-NEXT:    v_mad_f32 v3, -v2, v1, v3
+; TONGA-NEXT:    v_cvt_i32_f32_e32 v2, v2
+; TONGA-NEXT:    v_cmp_ge_f32_e64 vcc, |v3|, |v1|
 ; TONGA-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
-; TONGA-NEXT:    v_add_u32_e32 v0, vcc, v0, v1
+; TONGA-NEXT:    v_add_u32_e32 v0, vcc, v0, v2
 ; TONGA-NEXT:    v_bfe_i32 v0, v0, 0, 24
 ; TONGA-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; TONGA-NEXT:    s_endpgm
@@ -2011,18 +2011,18 @@ define amdgpu_kernel void @v_sdiv_i24(i32 addrspace(1)* %out, i24 addrspace(1)*
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
 ; GFX9-NEXT:    v_or_b32_e32 v2, v2, v3
-; GFX9-NEXT:    v_cvt_f32_i32_e32 v3, v2
-; GFX9-NEXT:    v_cvt_f32_i32_e32 v1, v0
-; GFX9-NEXT:    v_xor_b32_e32 v0, v0, v2
-; GFX9-NEXT:    v_ashrrev_i32_e32 v0, 30, v0
-; GFX9-NEXT:    v_rcp_iflag_f32_e32 v4, v3
-; GFX9-NEXT:    v_or_b32_e32 v0, 1, v0
-; GFX9-NEXT:    v_mul_f32_e32 v2, v1, v4
-; GFX9-NEXT:    v_trunc_f32_e32 v2, v2
-; GFX9-NEXT:    v_cvt_i32_f32_e32 v4, v2
-; GFX9-NEXT:    v_mad_f32 v1, -v2, v3, v1
-; GFX9-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v3|
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; GFX9-NEXT:    v_cvt_f32_i32_e32 v2, v2
+; GFX9-NEXT:    v_cvt_f32_i32_e32 v0, v0
+; GFX9-NEXT:    v_xor_b32_e32 v1, v1, v3
+; GFX9-NEXT:    v_ashrrev_i32_e32 v1, 30, v1
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v4, v2
+; GFX9-NEXT:    v_or_b32_e32 v1, 1, v1
+; GFX9-NEXT:    v_mul_f32_e32 v3, v0, v4
+; GFX9-NEXT:    v_trunc_f32_e32 v3, v3
+; GFX9-NEXT:    v_cvt_i32_f32_e32 v4, v3
+; GFX9-NEXT:    v_mad_f32 v0, -v3, v2, v0
+; GFX9-NEXT:    v_cmp_ge_f32_e64 vcc, |v0|, |v2|
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, 0, v1, vcc
 ; GFX9-NEXT:    v_add_u32_e32 v0, v4, v0
 ; GFX9-NEXT:    v_bfe_i32 v0, v0, 0, 24
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], 0
diff --git a/llvm/test/CodeGen/AMDGPU/sdwa-peephole.ll b/llvm/test/CodeGen/AMDGPU/sdwa-peephole.ll
index 92a3ab207..97d5bcce6 100644
--- a/llvm/test/CodeGen/AMDGPU/sdwa-peephole.ll
+++ b/llvm/test/CodeGen/AMDGPU/sdwa-peephole.ll
@@ -73,7 +73,7 @@ entry:
 ; GCN-LABEL: {{^}}mul_v2i16:
 ; NOSDWA: v_lshrrev_b32_e32 v[[DST0:[0-9]+]], 16, v{{[0-9]+}}
 ; NOSDWA: v_lshrrev_b32_e32 v[[DST1:[0-9]+]], 16, v{{[0-9]+}}
-; NOSDWA: v_mul_u32_u24_e32 v[[DST_MUL:[0-9]+]], v[[DST0]], v[[DST1]]
+; NOSDWA: v_mul_u32_u24_e32 v[[DST_MUL:[0-9]+]], v[[DST1]], v[[DST0]]
 ; NOSDWA: v_lshlrev_b32_e32 v[[DST_SHL:[0-9]+]], 16, v[[DST_MUL]]
 ; NOSDWA: v_or_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v[[DST_SHL]]
 ; NOSDWA-NOT: v_mul_u32_u24_sdwa
@@ -283,11 +283,8 @@ entry:
 
 ; GFX9: v_or_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 
-; GFX10-DAG: v_and_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX10-DAG: v_and_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10: v_or_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-
-
+; GFX10: v_lshlrev_b16_e64 v{{[0-9]+}}, 8, v
+; GFX10: v_or_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 define amdgpu_kernel void @mul_v2i8(<2 x i8> addrspace(1)* %out, <2 x i8> addrspace(1)* %ina, <2 x i8> addrspace(1)* %inb) {
 entry:
   %a = load <2 x i8>, <2 x i8> addrspace(1)* %ina, align 4
@@ -368,15 +365,15 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}sitofp_v2i16_to_v2f16:
-; NOSDWA-DAG: v_bfe_i32 v{{[0-9]+}}, v{{[0-9]+}}, 0, 16
-; NOSDWA-DAG: v_ashrrev_i32_e32 v{{[0-9]+}}, 16, v{{[0-9]+}}
-; NOSDWA-DAG: v_cvt_f32_i32_e32 v{{[0-9]+}}, v{{[0-9]+}}
-; NOSDWA-DAG: v_cvt_f32_i32_e32 v{{[0-9]+}}, v{{[0-9]+}}
-; NOSDWA-NOT: v_cvt_f32_i32_sdwa
+; NOSDWA-DAG: v_cvt_f16_i16_e32 v{{[0-9]+}}, v{{[0-9]+}}
+; NOSDWA-DAG: v_lshrrev_b32_e32 v{{[0-9]+}}, 16, v{{[0-9]+}}
+; NOSDWA-DAG: v_cvt_f16_i16_e32 v{{[0-9]+}}, v{{[0-9]+}}
+; NOSDWA-NOT: v_cvt_f16_i16_sdwa
 
-; SDWA-DAG: v_cvt_f32_i32_sdwa v{{[0-9]+}}, sext(v{{[0-9]+}}) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
-; SDWA-DAG: v_cvt_f32_i32_sdwa v{{[0-9]+}}, sext(v{{[0-9]+}}) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDWA-DAG: v_cvt_f16_i16_e32 v{{[0-9]+}}, v{{[0-9]+}}
+; SDWA-DAG: v_cvt_f16_i16_sdwa v{{[0-9]+}}, v{{[0-9]+}} dst_sel:{{(WORD_1|DWORD)?}} dst_unused:UNUSED_PAD src0_sel:WORD_1
 
+; FIXME: Should be able to avoid or
 define amdgpu_kernel void @sitofp_v2i16_to_v2f16(
     <2 x half> addrspace(1)* %r,
     <2 x i16> addrspace(1)* %a) {
@@ -501,10 +498,10 @@ store_label:
 ;
 ; GFX89: v_or_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ;
-; GFX10: v_or_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-; GFX10: v_or_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-; GFX10: v_or_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX10: v_or_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX10: v_or_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10: v_or_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10: v_or_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10: v_or_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 
 define amdgpu_kernel void @pulled_out_test(<8 x i8> addrspace(1)* %sourceA, <8 x i8> addrspace(1)* %destValues) {
 entry:
diff --git a/llvm/test/CodeGen/AMDGPU/select-opt.ll b/llvm/test/CodeGen/AMDGPU/select-opt.ll
index f77335797..24df126e4 100644
--- a/llvm/test/CodeGen/AMDGPU/select-opt.ll
+++ b/llvm/test/CodeGen/AMDGPU/select-opt.ll
@@ -135,8 +135,8 @@ define amdgpu_kernel void @opt_select_i64_or_cmp_f32(i64 addrspace(1)* %out, flo
 
 ; GCN-LABEL: {{^}}regression:
 ; GCN: v_cmp_neq_f32_e64 s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}}, 1.0
-; GCN: v_cmp_neq_f32_e32 vcc, 0, v{{[0-9]+}}
-; GCN: v_cmp_eq_f32_e32 vcc, 0, v{{[0-9]+}}
+; GCN: v_cmp_neq_f32_e64 s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}}, 0
+; GCN: v_cmp_eq_f32_e64 s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}}, 0
 
 define amdgpu_kernel void @regression(float addrspace(1)* %out, float %c0, float %c1) #0 {
 entry:
diff --git a/llvm/test/CodeGen/AMDGPU/select.f16.ll b/llvm/test/CodeGen/AMDGPU/select.f16.ll
index 16f48ae9d..16125bbcd 100644
--- a/llvm/test/CodeGen/AMDGPU/select.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/select.f16.ll
@@ -395,28 +395,28 @@ define amdgpu_kernel void @select_v2f16(
 ; SI-LABEL: select_v2f16:
 ; SI:       ; %bb.0: ; %entry
 ; SI-NEXT:    s_load_dwordx8 s[4:11], s[0:1], 0x9
-; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x11
-; SI-NEXT:    s_mov_b32 s15, 0xf000
-; SI-NEXT:    s_mov_b32 s14, -1
-; SI-NEXT:    s_mov_b32 s22, s14
+; SI-NEXT:    s_load_dwordx2 s[12:13], s[0:1], 0x11
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_mov_b32 s22, s2
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    s_mov_b32 s16, s10
 ; SI-NEXT:    s_mov_b32 s17, s11
-; SI-NEXT:    s_mov_b32 s10, s14
-; SI-NEXT:    s_mov_b32 s11, s15
+; SI-NEXT:    s_mov_b32 s10, s2
+; SI-NEXT:    s_mov_b32 s11, s3
 ; SI-NEXT:    s_mov_b32 s20, s6
 ; SI-NEXT:    s_mov_b32 s21, s7
-; SI-NEXT:    s_mov_b32 s23, s15
-; SI-NEXT:    s_mov_b32 s2, s14
-; SI-NEXT:    s_mov_b32 s3, s15
+; SI-NEXT:    s_mov_b32 s23, s3
+; SI-NEXT:    s_mov_b32 s14, s2
+; SI-NEXT:    s_mov_b32 s15, s3
 ; SI-NEXT:    buffer_load_dword v0, off, s[20:23], 0
-; SI-NEXT:    s_mov_b32 s18, s14
-; SI-NEXT:    s_mov_b32 s19, s15
+; SI-NEXT:    s_mov_b32 s18, s2
+; SI-NEXT:    s_mov_b32 s19, s3
 ; SI-NEXT:    buffer_load_dword v1, off, s[8:11], 0
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], 0
+; SI-NEXT:    buffer_load_dword v2, off, s[12:15], 0
 ; SI-NEXT:    buffer_load_dword v3, off, s[16:19], 0
-; SI-NEXT:    s_mov_b32 s12, s4
-; SI-NEXT:    s_mov_b32 s13, s5
+; SI-NEXT:    s_mov_b32 s0, s4
+; SI-NEXT:    s_mov_b32 s1, s5
 ; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_lshrrev_b32_e32 v5, 16, v0
 ; SI-NEXT:    v_cvt_f32_f16_e32 v5, v5
@@ -441,7 +441,7 @@ define amdgpu_kernel void @select_v2f16(
 ; SI-NEXT:    v_cvt_f16_f32_e32 v0, v0
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v2
 ; SI-NEXT:    v_or_b32_e32 v0, v0, v1
-; SI-NEXT:    buffer_store_dword v0, off, s[12:15], 0
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: select_v2f16:
diff --git a/llvm/test/CodeGen/AMDGPU/selectcc-opt.ll b/llvm/test/CodeGen/AMDGPU/selectcc-opt.ll
index 8fef3f8b3..fe7a350a1 100644
--- a/llvm/test/CodeGen/AMDGPU/selectcc-opt.ll
+++ b/llvm/test/CodeGen/AMDGPU/selectcc-opt.ll
@@ -69,7 +69,7 @@ entry:
 
 ; FUNC-LABEL: {{^}}selectcc_bool:
 ; SI: v_cmp_ne_u32
-; SI-NEXT: v_cndmask_b32_e64
+; SI: v_cndmask_b32_e64
 ; SI-NOT: cmp
 ; SI-NOT: cndmask
 define amdgpu_kernel void @selectcc_bool(i32 addrspace(1)* %out, i32 %a, i32 %b) nounwind {
diff --git a/llvm/test/CodeGen/AMDGPU/setcc-opt.ll b/llvm/test/CodeGen/AMDGPU/setcc-opt.ll
index 64e8bdc97..d194240ff 100644
--- a/llvm/test/CodeGen/AMDGPU/setcc-opt.ll
+++ b/llvm/test/CodeGen/AMDGPU/setcc-opt.ll
@@ -5,7 +5,7 @@
 ; FUNC-LABEL: {{^}}sext_bool_icmp_eq_0:
 ; GCN-NOT: v_cmp
 ; GCN: v_cmp_ne_u32_e32 vcc,
-; GCN-NEXT: v_cndmask_b32_e64 [[RESULT:v[0-9]+]], 0, 1, vcc
+; GCN: v_cndmask_b32_e64 [[RESULT:v[0-9]+]], 0, 1, vcc
 ; GCN-NEXT:buffer_store_byte [[RESULT]]
 ; GCN-NEXT: s_endpgm
 
@@ -22,7 +22,7 @@ define amdgpu_kernel void @sext_bool_icmp_eq_0(i1 addrspace(1)* %out, i32 %a, i3
 ; FUNC-LABEL: {{^}}sext_bool_icmp_ne_0:
 ; GCN-NOT: v_cmp
 ; GCN: v_cmp_ne_u32_e32 vcc,
-; GCN-NEXT: v_cndmask_b32_e64 [[RESULT:v[0-9]+]], 0, 1, vcc
+; GCN: v_cndmask_b32_e64 [[RESULT:v[0-9]+]], 0, 1, vcc
 ; GCN-NEXT: buffer_store_byte [[RESULT]]
 ; GCN-NEXT: s_endpgm
 
@@ -39,7 +39,7 @@ define amdgpu_kernel void @sext_bool_icmp_ne_0(i1 addrspace(1)* %out, i32 %a, i3
 ; FUNC-LABEL: {{^}}sext_bool_icmp_eq_neg1:
 ; GCN-NOT: v_cmp
 ; GCN: v_cmp_eq_u32_e32 vcc,
-; GCN-NEXT: v_cndmask_b32_e64 [[RESULT:v[0-9]+]], 0, 1, vcc
+; GCN: v_cndmask_b32_e64 [[RESULT:v[0-9]+]], 0, 1, vcc
 ; GCN-NEXT: buffer_store_byte [[RESULT]]
 ; GCN-NEXT: s_endpgm
 define amdgpu_kernel void @sext_bool_icmp_eq_neg1(i1 addrspace(1)* %out, i32 %a, i32 %b) nounwind {
@@ -53,7 +53,7 @@ define amdgpu_kernel void @sext_bool_icmp_eq_neg1(i1 addrspace(1)* %out, i32 %a,
 ; FUNC-LABEL: {{^}}sext_bool_icmp_ne_neg1:
 ; GCN-NOT: v_cmp
 ; GCN: v_cmp_eq_u32_e32 vcc,
-; GCN-NEXT: v_cndmask_b32_e64 [[RESULT:v[0-9]+]], 0, 1, vcc
+; GCN: v_cndmask_b32_e64 [[RESULT:v[0-9]+]], 0, 1, vcc
 ; GCN-NEXT: buffer_store_byte [[RESULT]]
 ; GCN-NEXT: s_endpgm
 define amdgpu_kernel void @sext_bool_icmp_ne_neg1(i1 addrspace(1)* %out, i32 %a, i32 %b) nounwind {
@@ -67,7 +67,7 @@ define amdgpu_kernel void @sext_bool_icmp_ne_neg1(i1 addrspace(1)* %out, i32 %a,
 ; FUNC-LABEL: {{^}}zext_bool_icmp_eq_0:
 ; GCN-NOT: v_cmp
 ; GCN: v_cmp_ne_u32_e32 vcc,
-; GCN-NEXT: v_cndmask_b32_e64 [[RESULT:v[0-9]+]], 0, 1, vcc
+; GCN: v_cndmask_b32_e64 [[RESULT:v[0-9]+]], 0, 1, vcc
 ; GCN-NEXT: buffer_store_byte [[RESULT]]
 ; GCN-NEXT: s_endpgm
 define amdgpu_kernel void @zext_bool_icmp_eq_0(i1 addrspace(1)* %out, i32 %a, i32 %b) nounwind {
@@ -81,7 +81,7 @@ define amdgpu_kernel void @zext_bool_icmp_eq_0(i1 addrspace(1)* %out, i32 %a, i3
 ; FUNC-LABEL: {{^}}zext_bool_icmp_ne_0:
 ; GCN-NOT: v_cmp
 ; GCN: v_cmp_ne_u32_e32 vcc,
-; GCN-NEXT: v_cndmask_b32_e64 [[RESULT:v[0-9]+]], 0, 1, vcc
+; GCN: v_cndmask_b32_e64 [[RESULT:v[0-9]+]], 0, 1, vcc
 ; GCN-NEXT: buffer_store_byte [[RESULT]]
 ; GCN-NEXT: s_endpgm
 define amdgpu_kernel void @zext_bool_icmp_ne_0(i1 addrspace(1)* %out, i32 %a, i32 %b) nounwind {
@@ -95,7 +95,7 @@ define amdgpu_kernel void @zext_bool_icmp_ne_0(i1 addrspace(1)* %out, i32 %a, i3
 ; FUNC-LABEL: {{^}}zext_bool_icmp_eq_1:
 ; GCN-NOT: v_cmp
 ; GCN: v_cmp_eq_u32_e32 vcc,
-; GCN-NEXT: v_cndmask_b32_e64 [[RESULT:v[0-9]+]], 0, 1, vcc
+; GCN: v_cndmask_b32_e64 [[RESULT:v[0-9]+]], 0, 1, vcc
 ; GCN-NEXT: buffer_store_byte [[RESULT]]
 ; GCN-NEXT: s_endpgm
 define amdgpu_kernel void @zext_bool_icmp_eq_1(i1 addrspace(1)* %out, i32 %a, i32 %b) nounwind {
@@ -109,7 +109,7 @@ define amdgpu_kernel void @zext_bool_icmp_eq_1(i1 addrspace(1)* %out, i32 %a, i3
 ; FUNC-LABEL: {{^}}zext_bool_icmp_ne_1:
 ; GCN-NOT: v_cmp
 ; GCN: v_cmp_eq_u32_e32 vcc,
-; GCN-NEXT: v_cndmask_b32_e64 [[RESULT:v[0-9]+]], 0, 1, vcc
+; GCN: v_cndmask_b32_e64 [[RESULT:v[0-9]+]], 0, 1, vcc
 ; GCN-NEXT: buffer_store_byte [[RESULT]]
 define amdgpu_kernel void @zext_bool_icmp_ne_1(i1 addrspace(1)* %out, i32 %a, i32 %b) nounwind {
   %icmp0 = icmp ne i32 %a, %b
diff --git a/llvm/test/CodeGen/AMDGPU/sgpr-control-flow.ll b/llvm/test/CodeGen/AMDGPU/sgpr-control-flow.ll
index e0971b845..3d5c3285c 100644
--- a/llvm/test/CodeGen/AMDGPU/sgpr-control-flow.ll
+++ b/llvm/test/CodeGen/AMDGPU/sgpr-control-flow.ll
@@ -104,7 +104,8 @@ endif:
 
 ; SI: ; %else
 ; SI:      buffer_load_dword  [[AVAL:v[0-9]+]]
-; SI:      v_cmp_gt_i32_e64   [[PHI:s\[[0-9]+:[0-9]+\]]], 0, [[AVAL]]
+; SI:      v_cmp_gt_i32_e32   vcc, 0, [[AVAL]]
+; SI:      s_and_b64 [[PHI:s\[[0-9]+:[0-9]+\]]], vcc, exec
 
 ; SI: ; %if
 ; SI:      buffer_load_dword  [[AVAL:v[0-9]+]]
diff --git a/llvm/test/CodeGen/AMDGPU/sgpr-copy.ll b/llvm/test/CodeGen/AMDGPU/sgpr-copy.ll
index cac9163aa..d1e5c389e 100644
--- a/llvm/test/CodeGen/AMDGPU/sgpr-copy.ll
+++ b/llvm/test/CodeGen/AMDGPU/sgpr-copy.ll
@@ -3,7 +3,8 @@
 
 ; CHECK-LABEL: {{^}}phi1:
 ; CHECK: s_buffer_load_dword [[DST:s[0-9]]], {{s\[[0-9]+:[0-9]+\]}}, 0x0
-; CHECK: v_mov_b32_e32 v{{[0-9]}}, [[DST]]
+; CHECK: ; %bb.1: ; %ELSE
+; CHECK: s_xor_b32 s{{[0-9]}}, [[DST]]
 define amdgpu_ps void @phi1(<4 x i32> addrspace(4)* inreg %arg, <4 x i32> addrspace(4)* inreg %arg1, <8 x i32> addrspace(4)* inreg %arg2, i32 inreg %arg3, <2 x i32> %arg4, <2 x i32> %arg5, <2 x i32> %arg6, <3 x i32> %arg7, <2 x i32> %arg8, <2 x i32> %arg9, <2 x i32> %arg10, float %arg11, float %arg12, float %arg13, float %arg14, float %arg15, float %arg16, float %arg17, float %arg18, float %arg19) #0 {
 main_body:
   %tmp = getelementptr <4 x i32>, <4 x i32> addrspace(4)* %arg, i32 0
diff --git a/llvm/test/CodeGen/AMDGPU/shift-i128.ll b/llvm/test/CodeGen/AMDGPU/shift-i128.ll
index 9fb3ede34..b46e68689 100644
--- a/llvm/test/CodeGen/AMDGPU/shift-i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/shift-i128.ll
@@ -15,12 +15,12 @@ define i128 @v_shl_i128_vv(i128 %lhs, i128 %rhs) {
 ; GCN-NEXT:    v_lshl_b64 v[5:6], v[0:1], v5
 ; GCN-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v4
 ; GCN-NEXT:    v_lshl_b64 v[0:1], v[0:1], v4
-; GCN-NEXT:    v_cndmask_b32_e32 v6, v6, v8, vcc
 ; GCN-NEXT:    v_cndmask_b32_e32 v5, v5, v7, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v3, v6, v3, s[4:5]
 ; GCN-NEXT:    v_cndmask_b32_e64 v2, v5, v2, s[4:5]
-; GCN-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GCN-NEXT:    v_cndmask_b32_e32 v5, v6, v8, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v3, v5, v3, s[4:5]
 ; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; GCN-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
   %shl = shl i128 %lhs, %rhs
   ret i128 %shl
@@ -40,12 +40,12 @@ define i128 @v_lshr_i128_vv(i128 %lhs, i128 %rhs) {
 ; GCN-NEXT:    v_lshr_b64 v[5:6], v[2:3], v5
 ; GCN-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v4
 ; GCN-NEXT:    v_lshr_b64 v[2:3], v[2:3], v4
-; GCN-NEXT:    v_cndmask_b32_e32 v6, v6, v8, vcc
 ; GCN-NEXT:    v_cndmask_b32_e32 v5, v5, v7, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v1, v6, v1, s[4:5]
 ; GCN-NEXT:    v_cndmask_b32_e64 v0, v5, v0, s[4:5]
-; GCN-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc
+; GCN-NEXT:    v_cndmask_b32_e32 v5, v6, v8, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s[4:5]
 ; GCN-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GCN-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
 
   %shl = lshr i128 %lhs, %rhs
@@ -56,25 +56,23 @@ define i128 @v_ashr_i128_vv(i128 %lhs, i128 %rhs) {
 ; GCN-LABEL: v_ashr_i128_vv:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT:    v_sub_i32_e32 v9, vcc, 64, v4
-; GCN-NEXT:    v_lshr_b64 v[7:8], v[0:1], v4
-; GCN-NEXT:    v_lshl_b64 v[9:10], v[2:3], v9
-; GCN-NEXT:    v_ashrrev_i32_e32 v11, 31, v3
-; GCN-NEXT:    v_or_b32_e32 v8, v8, v10
-; GCN-NEXT:    v_subrev_i32_e32 v10, vcc, 64, v4
-; GCN-NEXT:    v_ashr_i64 v[5:6], v[2:3], v4
-; GCN-NEXT:    v_ashr_i64 v[2:3], v[2:3], v10
-; GCN-NEXT:    v_cmp_gt_u32_e64 s[4:5], 64, v4
-; GCN-NEXT:    v_or_b32_e32 v7, v7, v9
-; GCN-NEXT:    v_cndmask_b32_e64 v5, v11, v5, s[4:5]
-; GCN-NEXT:    v_cndmask_b32_e64 v3, v3, v8, s[4:5]
-; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v2, v7, s[4:5]
-; GCN-NEXT:    v_cndmask_b32_e64 v6, v11, v6, s[4:5]
-; GCN-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GCN-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GCN-NEXT:    v_mov_b32_e32 v2, v5
-; GCN-NEXT:    v_mov_b32_e32 v3, v6
+; GCN-NEXT:    v_sub_i32_e32 v7, vcc, 64, v4
+; GCN-NEXT:    v_lshr_b64 v[5:6], v[0:1], v4
+; GCN-NEXT:    v_lshl_b64 v[7:8], v[2:3], v7
+; GCN-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v4
+; GCN-NEXT:    v_or_b32_e32 v7, v5, v7
+; GCN-NEXT:    v_subrev_i32_e32 v5, vcc, 64, v4
+; GCN-NEXT:    v_or_b32_e32 v8, v6, v8
+; GCN-NEXT:    v_ashr_i64 v[5:6], v[2:3], v5
+; GCN-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v4
+; GCN-NEXT:    v_cndmask_b32_e32 v5, v5, v7, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v0, v5, v0, s[4:5]
+; GCN-NEXT:    v_cndmask_b32_e32 v5, v6, v8, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s[4:5]
+; GCN-NEXT:    v_ashr_i64 v[4:5], v[2:3], v4
+; GCN-NEXT:    v_ashrrev_i32_e32 v3, 31, v3
+; GCN-NEXT:    v_cndmask_b32_e32 v2, v3, v4, vcc
+; GCN-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
   %shl = ashr i128 %lhs, %rhs
   ret i128 %shl
@@ -85,10 +83,13 @@ define i128 @v_shl_i128_vk(i128 %lhs) {
 ; GCN-LABEL: v_shl_i128_vk:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT:    v_lshl_b64 v[2:3], v[2:3], 17
 ; GCN-NEXT:    v_lshrrev_b32_e32 v4, 15, v1
-; GCN-NEXT:    v_lshl_b64 v[0:1], v[0:1], 17
-; GCN-NEXT:    v_or_b32_e32 v2, v2, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 17, v2
+; GCN-NEXT:    v_or_b32_e32 v4, v5, v4
+; GCN-NEXT:    v_alignbit_b32 v1, v1, v0, 15
+; GCN-NEXT:    v_alignbit_b32 v3, v3, v2, 15
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 17, v0
+; GCN-NEXT:    v_mov_b32_e32 v2, v4
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
   %shl = shl i128 %lhs, 17
   ret i128 %shl
@@ -98,7 +99,8 @@ define i128 @v_lshr_i128_vk(i128 %lhs) {
 ; GCN-LABEL: v_lshr_i128_vk:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT:    v_lshr_b64 v[0:1], v[2:3], 1
+; GCN-NEXT:    v_alignbit_b32 v0, v3, v2, 1
+; GCN-NEXT:    v_lshrrev_b32_e32 v1, 1, v3
 ; GCN-NEXT:    v_mov_b32_e32 v2, 0
 ; GCN-NEXT:    v_mov_b32_e32 v3, 0
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
@@ -110,12 +112,13 @@ define i128 @v_ashr_i128_vk(i128 %lhs) {
 ; GCN-LABEL: v_ashr_i128_vk:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT:    v_lshl_b64 v[4:5], v[2:3], 31
-; GCN-NEXT:    v_lshrrev_b32_e32 v0, 1, v1
-; GCN-NEXT:    v_or_b32_e32 v4, v0, v4
-; GCN-NEXT:    v_mov_b32_e32 v0, v4
-; GCN-NEXT:    v_ashr_i64 v[2:3], v[2:3], 33
-; GCN-NEXT:    v_mov_b32_e32 v1, v5
+; GCN-NEXT:    v_ashr_i64 v[4:5], v[2:3], 33
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 31, v2
+; GCN-NEXT:    v_lshrrev_b32_e32 v1, 1, v1
+; GCN-NEXT:    v_or_b32_e32 v0, v1, v0
+; GCN-NEXT:    v_alignbit_b32 v1, v3, v2, 1
+; GCN-NEXT:    v_mov_b32_e32 v2, v4
+; GCN-NEXT:    v_mov_b32_e32 v3, v5
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
   %shl = ashr i128 %lhs, 33
   ret i128 %shl
@@ -126,18 +129,18 @@ define i128 @v_shl_i128_kv(i128 %rhs) {
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN-NEXT:    v_sub_i32_e32 v1, vcc, 64, v0
-; GCN-NEXT:    v_subrev_i32_e32 v3, vcc, 64, v0
-; GCN-NEXT:    v_lshr_b64 v[1:2], 17, v1
-; GCN-NEXT:    v_lshl_b64 v[4:5], 17, v3
+; GCN-NEXT:    v_lshr_b64 v[2:3], 17, v1
+; GCN-NEXT:    v_subrev_i32_e32 v1, vcc, 64, v0
+; GCN-NEXT:    v_lshl_b64 v[4:5], 17, v1
 ; GCN-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v0
-; GCN-NEXT:    v_cndmask_b32_e32 v2, v5, v2, vcc
+; GCN-NEXT:    v_cndmask_b32_e32 v1, v4, v2, vcc
 ; GCN-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, v0
-; GCN-NEXT:    v_cndmask_b32_e64 v3, 0, v2, s[4:5]
-; GCN-NEXT:    v_cndmask_b32_e32 v2, v4, v1, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v2, 0, v1, s[4:5]
 ; GCN-NEXT:    v_lshl_b64 v[0:1], 17, v0
-; GCN-NEXT:    v_cndmask_b32_e64 v2, 0, v2, s[4:5]
-; GCN-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GCN-NEXT:    v_cndmask_b32_e32 v3, v5, v3, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v3, 0, v3, s[4:5]
 ; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; GCN-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
   %shl = shl i128 17, %rhs
   ret i128 %shl
@@ -147,17 +150,17 @@ define i128 @v_lshr_i128_kv(i128 %rhs) {
 ; GCN-LABEL: v_lshr_i128_kv:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT:    s_mov_b32 s7, 0
-; GCN-NEXT:    s_movk_i32 s6, 0x41
-; GCN-NEXT:    v_lshr_b64 v[2:3], s[6:7], v0
+; GCN-NEXT:    s_movk_i32 s4, 0x41
+; GCN-NEXT:    s_mov_b32 s5, 0
+; GCN-NEXT:    v_lshr_b64 v[1:2], s[4:5], v0
 ; GCN-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v0
+; GCN-NEXT:    v_mov_b32_e32 v3, s4
 ; GCN-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, v0
-; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v2, vcc
-; GCN-NEXT:    v_mov_b32_e32 v2, s6
-; GCN-NEXT:    v_cndmask_b32_e32 v1, 0, v3, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v0, v2, v0, s[4:5]
+; GCN-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GCN-NEXT:    s_and_b64 vcc, s[4:5], vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v0, v3, v1, s[4:5]
+; GCN-NEXT:    v_cndmask_b32_e32 v1, 0, v2, vcc
 ; GCN-NEXT:    v_mov_b32_e32 v2, 0
-; GCN-NEXT:    v_cndmask_b32_e64 v1, 0, v1, s[4:5]
 ; GCN-NEXT:    v_mov_b32_e32 v3, 0
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
   %shl = lshr i128 65, %rhs
@@ -168,14 +171,14 @@ define i128 @v_ashr_i128_kv(i128 %rhs) {
 ; GCN-LABEL: v_ashr_i128_kv:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT:    v_lshr_b64 v[2:3], 33, v0
+; GCN-NEXT:    v_lshr_b64 v[1:2], 33, v0
 ; GCN-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v0
 ; GCN-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, v0
-; GCN-NEXT:    v_cndmask_b32_e32 v1, 0, v3, vcc
-; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v2, vcc
+; GCN-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GCN-NEXT:    s_and_b64 vcc, s[4:5], vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v0, 33, v1, s[4:5]
+; GCN-NEXT:    v_cndmask_b32_e32 v1, 0, v2, vcc
 ; GCN-NEXT:    v_mov_b32_e32 v2, 0
-; GCN-NEXT:    v_cndmask_b32_e64 v1, 0, v1, s[4:5]
-; GCN-NEXT:    v_cndmask_b32_e64 v0, 33, v0, s[4:5]
 ; GCN-NEXT:    v_mov_b32_e32 v3, 0
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
   %shl = ashr i128 33, %rhs
@@ -312,33 +315,33 @@ define <2 x i128> @v_shl_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
 ; GCN-NEXT:    v_lshl_b64 v[16:17], v[0:1], v9
 ; GCN-NEXT:    s_and_b64 s[4:5], s[6:7], s[4:5]
 ; GCN-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[10:11]
-; GCN-NEXT:    v_cndmask_b32_e64 v9, v17, v19, s[4:5]
-; GCN-NEXT:    v_cndmask_b32_e32 v3, v9, v3, vcc
 ; GCN-NEXT:    v_cndmask_b32_e64 v9, v16, v18, s[4:5]
+; GCN-NEXT:    v_sub_i32_e64 v16, s[6:7], 64, v12
+; GCN-NEXT:    v_cndmask_b32_e64 v11, v17, v19, s[4:5]
 ; GCN-NEXT:    v_cndmask_b32_e32 v2, v9, v2, vcc
-; GCN-NEXT:    v_sub_i32_e32 v11, vcc, 64, v12
 ; GCN-NEXT:    v_lshl_b64 v[9:10], v[6:7], v12
-; GCN-NEXT:    v_lshr_b64 v[16:17], v[4:5], v11
-; GCN-NEXT:    v_cmp_gt_u64_e64 s[6:7], 64, v[12:13]
-; GCN-NEXT:    v_or_b32_e32 v16, v9, v16
+; GCN-NEXT:    v_lshr_b64 v[16:17], v[4:5], v16
 ; GCN-NEXT:    v_cmp_eq_u64_e64 s[8:9], 0, v[14:15]
+; GCN-NEXT:    v_cndmask_b32_e32 v3, v11, v3, vcc
+; GCN-NEXT:    v_or_b32_e32 v16, v9, v16
+; GCN-NEXT:    v_cmp_gt_u64_e64 s[6:7], 64, v[12:13]
 ; GCN-NEXT:    v_subrev_i32_e32 v9, vcc, 64, v12
 ; GCN-NEXT:    v_or_b32_e32 v11, v10, v17
 ; GCN-NEXT:    v_lshl_b64 v[9:10], v[4:5], v9
+; GCN-NEXT:    v_or_b32_e32 v15, v13, v15
+; GCN-NEXT:    v_or_b32_e32 v14, v12, v14
 ; GCN-NEXT:    s_and_b64 vcc, s[8:9], s[6:7]
-; GCN-NEXT:    v_cndmask_b32_e32 v17, v10, v11, vcc
-; GCN-NEXT:    v_or_b32_e32 v11, v13, v15
-; GCN-NEXT:    v_or_b32_e32 v10, v12, v14
+; GCN-NEXT:    v_cmp_eq_u64_e64 s[6:7], 0, v[14:15]
 ; GCN-NEXT:    v_lshl_b64 v[0:1], v[0:1], v8
 ; GCN-NEXT:    v_lshl_b64 v[4:5], v[4:5], v12
-; GCN-NEXT:    v_cmp_eq_u64_e64 s[6:7], 0, v[10:11]
 ; GCN-NEXT:    v_cndmask_b32_e32 v9, v9, v16, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v7, v17, v7, s[6:7]
 ; GCN-NEXT:    v_cndmask_b32_e64 v6, v9, v6, s[6:7]
-; GCN-NEXT:    v_cndmask_b32_e64 v1, 0, v1, s[4:5]
+; GCN-NEXT:    v_cndmask_b32_e32 v9, v10, v11, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v7, v9, v7, s[6:7]
 ; GCN-NEXT:    v_cndmask_b32_e64 v0, 0, v0, s[4:5]
-; GCN-NEXT:    v_cndmask_b32_e32 v5, 0, v5, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v1, 0, v1, s[4:5]
 ; GCN-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GCN-NEXT:    v_cndmask_b32_e32 v5, 0, v5, vcc
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
   %shl = shl <2 x i128> %lhs, %rhs
   ret <2 x i128> %shl
@@ -361,33 +364,33 @@ define <2 x i128> @v_lshr_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
 ; GCN-NEXT:    v_lshr_b64 v[16:17], v[2:3], v9
 ; GCN-NEXT:    s_and_b64 s[4:5], s[6:7], s[4:5]
 ; GCN-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[10:11]
-; GCN-NEXT:    v_cndmask_b32_e64 v9, v17, v19, s[4:5]
-; GCN-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc
 ; GCN-NEXT:    v_cndmask_b32_e64 v9, v16, v18, s[4:5]
+; GCN-NEXT:    v_sub_i32_e64 v16, s[6:7], 64, v12
+; GCN-NEXT:    v_cndmask_b32_e64 v11, v17, v19, s[4:5]
 ; GCN-NEXT:    v_cndmask_b32_e32 v0, v9, v0, vcc
-; GCN-NEXT:    v_sub_i32_e32 v11, vcc, 64, v12
 ; GCN-NEXT:    v_lshr_b64 v[9:10], v[4:5], v12
-; GCN-NEXT:    v_lshl_b64 v[16:17], v[6:7], v11
-; GCN-NEXT:    v_cmp_gt_u64_e64 s[6:7], 64, v[12:13]
-; GCN-NEXT:    v_or_b32_e32 v16, v9, v16
+; GCN-NEXT:    v_lshl_b64 v[16:17], v[6:7], v16
 ; GCN-NEXT:    v_cmp_eq_u64_e64 s[8:9], 0, v[14:15]
+; GCN-NEXT:    v_cndmask_b32_e32 v1, v11, v1, vcc
+; GCN-NEXT:    v_or_b32_e32 v16, v9, v16
+; GCN-NEXT:    v_cmp_gt_u64_e64 s[6:7], 64, v[12:13]
 ; GCN-NEXT:    v_subrev_i32_e32 v9, vcc, 64, v12
 ; GCN-NEXT:    v_or_b32_e32 v11, v10, v17
 ; GCN-NEXT:    v_lshr_b64 v[9:10], v[6:7], v9
+; GCN-NEXT:    v_or_b32_e32 v15, v13, v15
+; GCN-NEXT:    v_or_b32_e32 v14, v12, v14
 ; GCN-NEXT:    s_and_b64 vcc, s[8:9], s[6:7]
-; GCN-NEXT:    v_cndmask_b32_e32 v17, v10, v11, vcc
-; GCN-NEXT:    v_or_b32_e32 v11, v13, v15
-; GCN-NEXT:    v_or_b32_e32 v10, v12, v14
+; GCN-NEXT:    v_cmp_eq_u64_e64 s[6:7], 0, v[14:15]
 ; GCN-NEXT:    v_lshr_b64 v[2:3], v[2:3], v8
 ; GCN-NEXT:    v_lshr_b64 v[6:7], v[6:7], v12
-; GCN-NEXT:    v_cmp_eq_u64_e64 s[6:7], 0, v[10:11]
 ; GCN-NEXT:    v_cndmask_b32_e32 v9, v9, v16, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v5, v17, v5, s[6:7]
 ; GCN-NEXT:    v_cndmask_b32_e64 v4, v9, v4, s[6:7]
-; GCN-NEXT:    v_cndmask_b32_e64 v3, 0, v3, s[4:5]
+; GCN-NEXT:    v_cndmask_b32_e32 v9, v10, v11, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v5, v9, v5, s[6:7]
 ; GCN-NEXT:    v_cndmask_b32_e64 v2, 0, v2, s[4:5]
-; GCN-NEXT:    v_cndmask_b32_e32 v7, 0, v7, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v3, 0, v3, s[4:5]
 ; GCN-NEXT:    v_cndmask_b32_e32 v6, 0, v6, vcc
+; GCN-NEXT:    v_cndmask_b32_e32 v7, 0, v7, vcc
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
   %shl = lshr <2 x i128> %lhs, %rhs
   ret <2 x i128> %shl
@@ -410,35 +413,35 @@ define <2 x i128> @v_ashr_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
 ; GCN-NEXT:    v_ashr_i64 v[16:17], v[2:3], v9
 ; GCN-NEXT:    s_and_b64 s[4:5], s[6:7], s[4:5]
 ; GCN-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[10:11]
-; GCN-NEXT:    v_cndmask_b32_e64 v9, v17, v19, s[4:5]
-; GCN-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc
 ; GCN-NEXT:    v_cndmask_b32_e64 v9, v16, v18, s[4:5]
+; GCN-NEXT:    v_sub_i32_e64 v16, s[6:7], 64, v12
+; GCN-NEXT:    v_cndmask_b32_e64 v11, v17, v19, s[4:5]
 ; GCN-NEXT:    v_cndmask_b32_e32 v0, v9, v0, vcc
-; GCN-NEXT:    v_sub_i32_e32 v11, vcc, 64, v12
 ; GCN-NEXT:    v_lshr_b64 v[9:10], v[4:5], v12
-; GCN-NEXT:    v_lshl_b64 v[16:17], v[6:7], v11
-; GCN-NEXT:    v_cmp_gt_u64_e64 s[6:7], 64, v[12:13]
-; GCN-NEXT:    v_or_b32_e32 v16, v9, v16
+; GCN-NEXT:    v_lshl_b64 v[16:17], v[6:7], v16
 ; GCN-NEXT:    v_cmp_eq_u64_e64 s[8:9], 0, v[14:15]
+; GCN-NEXT:    v_cndmask_b32_e32 v1, v11, v1, vcc
+; GCN-NEXT:    v_or_b32_e32 v16, v9, v16
+; GCN-NEXT:    v_cmp_gt_u64_e64 s[6:7], 64, v[12:13]
 ; GCN-NEXT:    v_subrev_i32_e32 v9, vcc, 64, v12
 ; GCN-NEXT:    v_or_b32_e32 v11, v10, v17
 ; GCN-NEXT:    v_ashr_i64 v[9:10], v[6:7], v9
+; GCN-NEXT:    v_or_b32_e32 v15, v13, v15
+; GCN-NEXT:    v_or_b32_e32 v14, v12, v14
 ; GCN-NEXT:    s_and_b64 vcc, s[8:9], s[6:7]
-; GCN-NEXT:    v_cndmask_b32_e32 v17, v10, v11, vcc
-; GCN-NEXT:    v_or_b32_e32 v11, v13, v15
-; GCN-NEXT:    v_or_b32_e32 v10, v12, v14
-; GCN-NEXT:    v_cmp_eq_u64_e64 s[6:7], 0, v[10:11]
+; GCN-NEXT:    v_cmp_eq_u64_e64 s[6:7], 0, v[14:15]
 ; GCN-NEXT:    v_cndmask_b32_e32 v9, v9, v16, vcc
 ; GCN-NEXT:    v_cndmask_b32_e64 v4, v9, v4, s[6:7]
+; GCN-NEXT:    v_cndmask_b32_e32 v9, v10, v11, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v5, v9, v5, s[6:7]
 ; GCN-NEXT:    v_ashr_i64 v[8:9], v[2:3], v8
-; GCN-NEXT:    v_ashrrev_i32_e32 v2, 31, v3
-; GCN-NEXT:    v_cndmask_b32_e64 v3, v2, v9, s[4:5]
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v2, v8, s[4:5]
+; GCN-NEXT:    v_ashrrev_i32_e32 v3, 31, v3
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v3, v8, s[4:5]
+; GCN-NEXT:    v_cndmask_b32_e64 v3, v3, v9, s[4:5]
 ; GCN-NEXT:    v_ashr_i64 v[8:9], v[6:7], v12
-; GCN-NEXT:    v_ashrrev_i32_e32 v6, 31, v7
-; GCN-NEXT:    v_cndmask_b32_e32 v7, v6, v9, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v5, v17, v5, s[6:7]
-; GCN-NEXT:    v_cndmask_b32_e32 v6, v6, v8, vcc
+; GCN-NEXT:    v_ashrrev_i32_e32 v7, 31, v7
+; GCN-NEXT:    v_cndmask_b32_e32 v6, v7, v8, vcc
+; GCN-NEXT:    v_cndmask_b32_e32 v7, v7, v9, vcc
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
   %shl = ashr <2 x i128> %lhs, %rhs
   ret <2 x i128> %shl
@@ -506,8 +509,8 @@ define amdgpu_kernel void @s_shl_v2i128ss(<2 x i128> %lhs, <2 x i128> %rhs) {
 ; GCN-NEXT:    v_mov_b32_e32 v10, 16
 ; GCN-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
 ; GCN-NEXT:    v_mov_b32_e32 v11, 0
-; GCN-NEXT:    flat_store_dwordx4 v[8:9], v[0:3]
 ; GCN-NEXT:    flat_store_dwordx4 v[10:11], v[4:7]
+; GCN-NEXT:    flat_store_dwordx4 v[8:9], v[0:3]
 ; GCN-NEXT:    s_endpgm
   %shift = shl <2 x i128> %lhs, %rhs
   store <2 x i128> %shift, <2 x i128> addrspace(1)* null
@@ -576,8 +579,8 @@ define amdgpu_kernel void @s_lshr_v2i128_ss(<2 x i128> %lhs, <2 x i128> %rhs) {
 ; GCN-NEXT:    v_cndmask_b32_e64 v4, v4, v10, s[0:1]
 ; GCN-NEXT:    v_mov_b32_e32 v10, 16
 ; GCN-NEXT:    v_mov_b32_e32 v11, 0
-; GCN-NEXT:    flat_store_dwordx4 v[8:9], v[0:3]
 ; GCN-NEXT:    flat_store_dwordx4 v[10:11], v[4:7]
+; GCN-NEXT:    flat_store_dwordx4 v[8:9], v[0:3]
 ; GCN-NEXT:    s_endpgm
   %shift = lshr <2 x i128> %lhs, %rhs
   store <2 x i128> %shift, <2 x i128> addrspace(1)* null
@@ -650,8 +653,8 @@ define amdgpu_kernel void @s_ashr_v2i128_ss(<2 x i128> %lhs, <2 x i128> %rhs) {
 ; GCN-NEXT:    v_cndmask_b32_e64 v4, v4, v10, s[0:1]
 ; GCN-NEXT:    v_mov_b32_e32 v10, 16
 ; GCN-NEXT:    v_mov_b32_e32 v11, 0
-; GCN-NEXT:    flat_store_dwordx4 v[8:9], v[0:3]
 ; GCN-NEXT:    flat_store_dwordx4 v[10:11], v[4:7]
+; GCN-NEXT:    flat_store_dwordx4 v[8:9], v[0:3]
 ; GCN-NEXT:    s_endpgm
   %shift = ashr <2 x i128> %lhs, %rhs
   store <2 x i128> %shift, <2 x i128> addrspace(1)* null
diff --git a/llvm/test/CodeGen/AMDGPU/shl.ll b/llvm/test/CodeGen/AMDGPU/shl.ll
index 8997e6bc7..4adacbc12 100644
--- a/llvm/test/CodeGen/AMDGPU/shl.ll
+++ b/llvm/test/CodeGen/AMDGPU/shl.ll
@@ -1,25 +1,50 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mtriple=amdgcn-- -mcpu=verde -verify-machineinstrs < %s | FileCheck -allow-deprecated-dag-overlap -check-prefix=GCN -check-prefix=SI -check-prefix=FUNC %s
-; XUN: llc -march=amdgcn -mtriple=amdgcn-- -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI -check-prefix=FUNC %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=r600 -mtriple=r600-- -mcpu=redwood -verify-machineinstrs < %s | FileCheck -allow-deprecated-dag-overlap -check-prefix=EG -check-prefix=FUNC %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -amdgpu-scalarize-global-loads=false  -march=amdgcn -mtriple=amdgcn-- -mcpu=verde -verify-machineinstrs | FileCheck %s -allow-deprecated-dag-overlap -check-prefixes=FUNC,GCN,SI
+; XUN: llc < %s -march=amdgcn -mtriple=amdgcn-- -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs | FileCheck %s -check-prefixes=FUNC,GCN,VI
+; RUN: llc < %s -amdgpu-scalarize-global-loads=false  -march=r600 -mtriple=r600-- -mcpu=redwood -verify-machineinstrs | FileCheck %s -allow-deprecated-dag-overlap -check-prefixes=FUNC,EG
 
 declare i32 @llvm.r600.read.tidig.x() #0
 
 declare i32 @llvm.r600.read.tgid.x() #0
 
-
-;EG: {{^}}shl_v2i32:
-;EG: LSHL {{\*? *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
-;EG: LSHL {{\*? *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
-
-;SI: {{^}}shl_v2i32:
-;SI: v_lshl_b32_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
-;SI: v_lshl_b32_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
-
-;VI: {{^}}shl_v2i32:
-;VI: v_lshlrev_b32_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
-;VI: v_lshlrev_b32_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
-
 define amdgpu_kernel void @shl_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32> addrspace(1)* %in) {
+; GCN-LABEL: shl_v2i32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GCN-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NEXT:    s_mov_b32 s2, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_mov_b32 s0, s4
+; GCN-NEXT:    s_mov_b32 s1, s5
+; GCN-NEXT:    s_mov_b32 s4, s6
+; GCN-NEXT:    s_mov_b32 s5, s7
+; GCN-NEXT:    s_mov_b32 s6, s2
+; GCN-NEXT:    s_mov_b32 s7, s3
+; GCN-NEXT:    buffer_load_dwordx4 v[0:3], off, s[4:7], 0
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshl_b32_e32 v1, v1, v3
+; GCN-NEXT:    v_lshl_b32_e32 v0, v0, v2
+; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GCN-NEXT:    s_endpgm
+;
+; EG-LABEL: shl_v2i32:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 0, @10, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 1 @6
+; EG-NEXT:    ALU 3, @11, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+; EG-NEXT:    Fetch clause starting at 6:
+; EG-NEXT:     VTX_READ_64 T1.XY, T0.X, 8, #1
+; EG-NEXT:     VTX_READ_64 T0.XY, T0.X, 0, #1
+; EG-NEXT:    ALU clause starting at 10:
+; EG-NEXT:     MOV * T0.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 11:
+; EG-NEXT:     LSHL * T0.Y, T0.Y, T1.Y,
+; EG-NEXT:     LSHL T0.X, T0.X, T1.X,
+; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
   %b_ptr = getelementptr <2 x i32>, <2 x i32> addrspace(1)* %in, i32 1
   %a = load <2 x i32>, <2 x i32> addrspace(1)* %in
   %b = load <2 x i32>, <2 x i32> addrspace(1)* %b_ptr
@@ -28,25 +53,49 @@ define amdgpu_kernel void @shl_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32> add
   ret void
 }
 
-;EG: {{^}}shl_v4i32:
-;EG: LSHL {{\*? *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
-;EG: LSHL {{\*? *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
-;EG: LSHL {{\*? *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
-;EG: LSHL {{\*? *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
-
-;SI: {{^}}shl_v4i32:
-;SI: v_lshl_b32_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
-;SI: v_lshl_b32_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
-;SI: v_lshl_b32_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
-;SI: v_lshl_b32_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
-
-;VI: {{^}}shl_v4i32:
-;VI: v_lshlrev_b32_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
-;VI: v_lshlrev_b32_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
-;VI: v_lshlrev_b32_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
-;VI: v_lshlrev_b32_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
-
 define amdgpu_kernel void @shl_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> addrspace(1)* %in) {
+; GCN-LABEL: shl_v4i32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GCN-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NEXT:    s_mov_b32 s2, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_mov_b32 s0, s4
+; GCN-NEXT:    s_mov_b32 s1, s5
+; GCN-NEXT:    s_mov_b32 s4, s6
+; GCN-NEXT:    s_mov_b32 s5, s7
+; GCN-NEXT:    s_mov_b32 s6, s2
+; GCN-NEXT:    s_mov_b32 s7, s3
+; GCN-NEXT:    buffer_load_dwordx4 v[0:3], off, s[4:7], 0
+; GCN-NEXT:    buffer_load_dwordx4 v[4:7], off, s[4:7], 0 offset:16
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshl_b32_e32 v3, v3, v7
+; GCN-NEXT:    v_lshl_b32_e32 v2, v2, v6
+; GCN-NEXT:    v_lshl_b32_e32 v1, v1, v5
+; GCN-NEXT:    v_lshl_b32_e32 v0, v0, v4
+; GCN-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; GCN-NEXT:    s_endpgm
+;
+; EG-LABEL: shl_v4i32:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 0, @10, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 1 @6
+; EG-NEXT:    ALU 5, @11, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XYZW, T1.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+; EG-NEXT:    Fetch clause starting at 6:
+; EG-NEXT:     VTX_READ_128 T1.XYZW, T0.X, 16, #1
+; EG-NEXT:     VTX_READ_128 T0.XYZW, T0.X, 0, #1
+; EG-NEXT:    ALU clause starting at 10:
+; EG-NEXT:     MOV * T0.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 11:
+; EG-NEXT:     LSHL * T0.W, T0.W, T1.W,
+; EG-NEXT:     LSHL * T0.Z, T0.Z, T1.Z,
+; EG-NEXT:     LSHL * T0.Y, T0.Y, T1.Y,
+; EG-NEXT:     LSHL T0.X, T0.X, T1.X,
+; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
   %b_ptr = getelementptr <4 x i32>, <4 x i32> addrspace(1)* %in, i32 1
   %a = load <4 x i32>, <4 x i32> addrspace(1)* %in
   %b = load <4 x i32>, <4 x i32> addrspace(1)* %b_ptr
@@ -55,11 +104,53 @@ define amdgpu_kernel void @shl_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> add
   ret void
 }
 
-; GCN-LABEL: {{^}}shl_i16:
-; SI: v_lshlrev_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-
-; VI: v_lshlrev_b16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 define amdgpu_kernel void @shl_i16(i16 addrspace(1)* %out, i16 addrspace(1)* %in) {
+; GCN-LABEL: shl_i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GCN-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NEXT:    s_mov_b32 s2, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_mov_b32 s0, s4
+; GCN-NEXT:    s_mov_b32 s1, s5
+; GCN-NEXT:    s_mov_b32 s4, s6
+; GCN-NEXT:    s_mov_b32 s5, s7
+; GCN-NEXT:    s_mov_b32 s6, s2
+; GCN-NEXT:    s_mov_b32 s7, s3
+; GCN-NEXT:    buffer_load_ushort v0, off, s[4:7], 0
+; GCN-NEXT:    buffer_load_ushort v1, off, s[4:7], 0 offset:2
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, v1, v0
+; GCN-NEXT:    buffer_store_short v0, off, s[0:3], 0
+; GCN-NEXT:    s_endpgm
+;
+; EG-LABEL: shl_i16:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 0, @10, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 1 @6
+; EG-NEXT:    ALU 12, @11, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT MSKOR T0.XW, T1.X
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+; EG-NEXT:    Fetch clause starting at 6:
+; EG-NEXT:     VTX_READ_16 T1.X, T0.X, 2, #1
+; EG-NEXT:     VTX_READ_16 T0.X, T0.X, 0, #1
+; EG-NEXT:    ALU clause starting at 10:
+; EG-NEXT:     MOV * T0.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 11:
+; EG-NEXT:     AND_INT T0.W, KC0[2].Y, literal.x,
+; EG-NEXT:     LSHL * T1.W, T0.X, T1.X,
+; EG-NEXT:    3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT:     AND_INT T1.W, PS, literal.x,
+; EG-NEXT:     LSHL * T0.W, PV.W, literal.y,
+; EG-NEXT:    65535(9.183409e-41), 3(4.203895e-45)
+; EG-NEXT:     LSHL T0.X, PV.W, PS,
+; EG-NEXT:     LSHL * T0.W, literal.x, PS,
+; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
+; EG-NEXT:     MOV T0.Y, 0.0,
+; EG-NEXT:     MOV * T0.Z, 0.0,
+; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
   %b_ptr = getelementptr i16, i16 addrspace(1)* %in, i16 1
   %a = load i16, i16 addrspace(1)* %in
   %b = load i16, i16 addrspace(1)* %b_ptr
@@ -68,22 +159,118 @@ define amdgpu_kernel void @shl_i16(i16 addrspace(1)* %out, i16 addrspace(1)* %in
   ret void
 }
 
-; GCN-LABEL: {{^}}shl_i16_v_s:
-; VI: v_lshlrev_b32_e64 v{{[0-9]+}}, v{{[0-9]+}}, s{{[0-9]+}}
-
-; VI: v_lshlrev_b16_e64 v{{[0-9]+}}, v{{[0-9]+}}, s{{[0-9]+}}
 define amdgpu_kernel void @shl_i16_v_s(i16 addrspace(1)* %out, i16 addrspace(1)* %in, i16 %b) {
+; GCN-LABEL: shl_i16_v_s:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GCN-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NEXT:    s_mov_b32 s2, -1
+; GCN-NEXT:    s_load_dword s8, s[0:1], 0xd
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_mov_b32 s0, s4
+; GCN-NEXT:    s_mov_b32 s1, s5
+; GCN-NEXT:    s_mov_b32 s4, s6
+; GCN-NEXT:    s_mov_b32 s5, s7
+; GCN-NEXT:    s_mov_b32 s6, s2
+; GCN-NEXT:    s_mov_b32 s7, s3
+; GCN-NEXT:    buffer_load_ushort v0, off, s[4:7], 0
+; GCN-NEXT:    s_and_b32 s8, s8, 0xffff
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, s8, v0
+; GCN-NEXT:    buffer_store_short v0, off, s[0:3], 0
+; GCN-NEXT:    s_endpgm
+;
+; EG-LABEL: shl_i16_v_s:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 1, @10, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 1 @6
+; EG-NEXT:    ALU 12, @12, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT MSKOR T0.XW, T1.X
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+; EG-NEXT:    Fetch clause starting at 6:
+; EG-NEXT:     VTX_READ_16 T1.X, T1.X, 0, #1
+; EG-NEXT:     VTX_READ_16 T0.X, T0.X, 44, #3
+; EG-NEXT:    ALU clause starting at 10:
+; EG-NEXT:     MOV T0.X, 0.0,
+; EG-NEXT:     MOV * T1.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 12:
+; EG-NEXT:     AND_INT T0.W, KC0[2].Y, literal.x,
+; EG-NEXT:     LSHL * T1.W, T1.X, T0.X,
+; EG-NEXT:    3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT:     AND_INT T1.W, PS, literal.x,
+; EG-NEXT:     LSHL * T0.W, PV.W, literal.y,
+; EG-NEXT:    65535(9.183409e-41), 3(4.203895e-45)
+; EG-NEXT:     LSHL T0.X, PV.W, PS,
+; EG-NEXT:     LSHL * T0.W, literal.x, PS,
+; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
+; EG-NEXT:     MOV T0.Y, 0.0,
+; EG-NEXT:     MOV * T0.Z, 0.0,
+; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
   %a = load i16, i16 addrspace(1)* %in
   %result = shl i16 %a, %b
   store i16 %result, i16 addrspace(1)* %out
   ret void
 }
 
-; GCN-LABEL: {{^}}shl_i16_v_compute_s:
-; SI: v_lshlrev_b32_e32 v{{[0-9]+}}, s{{[0-9]+}}, v{{[0-9]+}}
-
-; VI: v_lshlrev_b16_e64 v{{[0-9]+}}, v{{[0-9]+}}, s{{[0-9]+}}
 define amdgpu_kernel void @shl_i16_v_compute_s(i16 addrspace(1)* %out, i16 addrspace(1)* %in, i16 %b) {
+; GCN-LABEL: shl_i16_v_compute_s:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GCN-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NEXT:    s_mov_b32 s2, -1
+; GCN-NEXT:    s_load_dword s8, s[0:1], 0xd
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_mov_b32 s0, s4
+; GCN-NEXT:    s_mov_b32 s1, s5
+; GCN-NEXT:    s_mov_b32 s4, s6
+; GCN-NEXT:    s_mov_b32 s5, s7
+; GCN-NEXT:    s_mov_b32 s6, s2
+; GCN-NEXT:    s_mov_b32 s7, s3
+; GCN-NEXT:    buffer_load_ushort v0, off, s[4:7], 0
+; GCN-NEXT:    s_add_i32 s8, s8, 3
+; GCN-NEXT:    s_and_b32 s4, s8, 0xffff
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, s4, v0
+; GCN-NEXT:    buffer_store_short v0, off, s[0:3], 0
+; GCN-NEXT:    s_endpgm
+;
+; EG-LABEL: shl_i16_v_compute_s:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 0, @12, KC0[], KC1[]
+; EG-NEXT:    TEX 0 @8
+; EG-NEXT:    ALU 0, @13, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 0 @10
+; EG-NEXT:    ALU 15, @14, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT MSKOR T0.XW, T1.X
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+; EG-NEXT:    Fetch clause starting at 8:
+; EG-NEXT:     VTX_READ_16 T0.X, T0.X, 44, #3
+; EG-NEXT:    Fetch clause starting at 10:
+; EG-NEXT:     VTX_READ_16 T1.X, T1.X, 0, #1
+; EG-NEXT:    ALU clause starting at 12:
+; EG-NEXT:     MOV * T0.X, 0.0,
+; EG-NEXT:    ALU clause starting at 13:
+; EG-NEXT:     MOV * T1.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 14:
+; EG-NEXT:     ADD_INT * T0.W, T0.X, literal.x,
+; EG-NEXT:    3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT:     AND_INT T0.W, PV.W, literal.x,
+; EG-NEXT:     AND_INT * T1.W, KC0[2].Y, literal.y,
+; EG-NEXT:    65535(9.183409e-41), 3(4.203895e-45)
+; EG-NEXT:     LSHL * T0.W, T1.X, PV.W,
+; EG-NEXT:     AND_INT T0.W, PV.W, literal.x,
+; EG-NEXT:     LSHL * T1.W, T1.W, literal.y,
+; EG-NEXT:    65535(9.183409e-41), 3(4.203895e-45)
+; EG-NEXT:     LSHL T0.X, PV.W, PS,
+; EG-NEXT:     LSHL * T0.W, literal.x, PS,
+; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
+; EG-NEXT:     MOV T0.Y, 0.0,
+; EG-NEXT:     MOV * T0.Z, 0.0,
+; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
   %a = load i16, i16 addrspace(1)* %in
   %b.add = add i16 %b, 3
   %result = shl i16 %a, %b.add
@@ -91,10 +278,69 @@ define amdgpu_kernel void @shl_i16_v_compute_s(i16 addrspace(1)* %out, i16 addrs
   ret void
 }
 
-; GCN-LABEL: {{^}}shl_i16_computed_amount:
-; VI: v_add_u16_e32 [[ADD:v[0-9]+]], 3, v{{[0-9]+}}
-; VI: v_lshlrev_b16_e32 v{{[0-9]+}}, [[ADD]], v{{[0-9]+}}
 define amdgpu_kernel void @shl_i16_computed_amount(i16 addrspace(1)* %out, i16 addrspace(1)* %in) {
+; GCN-LABEL: shl_i16_computed_amount:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_mov_b32 s10, s6
+; GCN-NEXT:    s_mov_b32 s11, s7
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_mov_b32 s8, s2
+; GCN-NEXT:    s_mov_b32 s9, s3
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
+; GCN-NEXT:    v_mov_b32_e32 v1, 0
+; GCN-NEXT:    s_mov_b32 s14, 0
+; GCN-NEXT:    s_mov_b32 s15, s7
+; GCN-NEXT:    s_mov_b64 s[12:13], s[2:3]
+; GCN-NEXT:    buffer_load_ushort v2, off, s[8:11], 0
+; GCN-NEXT:    buffer_load_ushort v0, v[0:1], s[12:15], 0 addr64 offset:2
+; GCN-NEXT:    s_mov_b32 s4, s0
+; GCN-NEXT:    s_mov_b32 s5, s1
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, v0, v2
+; GCN-NEXT:    buffer_store_short v0, off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
+;
+; EG-LABEL: shl_i16_computed_amount:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 0, @12, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 0 @8
+; EG-NEXT:    ALU 1, @13, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 0 @10
+; EG-NEXT:    ALU 15, @15, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT MSKOR T0.XW, T1.X
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+; EG-NEXT:    Fetch clause starting at 8:
+; EG-NEXT:     VTX_READ_16 T1.X, T1.X, 0, #1
+; EG-NEXT:    Fetch clause starting at 10:
+; EG-NEXT:     VTX_READ_16 T0.X, T0.X, 2, #1
+; EG-NEXT:    ALU clause starting at 12:
+; EG-NEXT:     MOV * T1.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 13:
+; EG-NEXT:     LSHL * T0.W, T0.X, 1,
+; EG-NEXT:     ADD_INT * T0.X, KC0[2].Z, PV.W,
+; EG-NEXT:    ALU clause starting at 15:
+; EG-NEXT:     ADD_INT * T0.W, T0.X, literal.x,
+; EG-NEXT:    3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT:     AND_INT T0.W, PV.W, literal.x,
+; EG-NEXT:     AND_INT * T1.W, KC0[2].Y, literal.y,
+; EG-NEXT:    65535(9.183409e-41), 3(4.203895e-45)
+; EG-NEXT:     LSHL * T0.W, T1.X, PV.W,
+; EG-NEXT:     AND_INT T0.W, PV.W, literal.x,
+; EG-NEXT:     LSHL * T1.W, T1.W, literal.y,
+; EG-NEXT:    65535(9.183409e-41), 3(4.203895e-45)
+; EG-NEXT:     LSHL T0.X, PV.W, PS,
+; EG-NEXT:     LSHL * T0.W, literal.x, PS,
+; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
+; EG-NEXT:     MOV T0.Y, 0.0,
+; EG-NEXT:     MOV * T0.Z, 0.0,
+; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
   %tid = call i32 @llvm.r600.read.tidig.x() #0
   %gep = getelementptr inbounds i16, i16 addrspace(1)* %in, i32 %tid
   %gep.out = getelementptr inbounds i16, i16 addrspace(1)* %out, i32 %tid
@@ -107,18 +353,120 @@ define amdgpu_kernel void @shl_i16_computed_amount(i16 addrspace(1)* %out, i16 a
   ret void
 }
 
-; GCN-LABEL: {{^}}shl_i16_i_s:
-; GCN: s_lshl_b32 s{{[0-9]+}}, s{{[0-9]+}}, 12
 define amdgpu_kernel void @shl_i16_i_s(i16 addrspace(1)* %out, i16 zeroext %a) {
+; GCN-LABEL: shl_i16_i_s:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_lshl_b32 s0, s0, 12
+; GCN-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-NEXT:    buffer_store_short v0, off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
+;
+; EG-LABEL: shl_i16_i_s:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 0, @8, KC0[], KC1[]
+; EG-NEXT:    TEX 0 @6
+; EG-NEXT:    ALU 14, @9, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT MSKOR T0.XW, T1.X
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+; EG-NEXT:    Fetch clause starting at 6:
+; EG-NEXT:     VTX_READ_16 T0.X, T0.X, 40, #3
+; EG-NEXT:    ALU clause starting at 8:
+; EG-NEXT:     MOV * T0.X, 0.0,
+; EG-NEXT:    ALU clause starting at 9:
+; EG-NEXT:     BFE_INT T0.W, T0.X, 0.0, literal.x,
+; EG-NEXT:     AND_INT * T1.W, KC0[2].Y, literal.y,
+; EG-NEXT:    16(2.242078e-44), 3(4.203895e-45)
+; EG-NEXT:     LSHL * T0.W, PV.W, literal.x,
+; EG-NEXT:    12(1.681558e-44), 0(0.000000e+00)
+; EG-NEXT:     AND_INT T0.W, PV.W, literal.x,
+; EG-NEXT:     LSHL * T1.W, T1.W, literal.y,
+; EG-NEXT:    61440(8.609578e-41), 3(4.203895e-45)
+; EG-NEXT:     LSHL T0.X, PV.W, PS,
+; EG-NEXT:     LSHL * T0.W, literal.x, PS,
+; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
+; EG-NEXT:     MOV T0.Y, 0.0,
+; EG-NEXT:     MOV * T0.Z, 0.0,
+; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
   %result = shl i16 %a, 12
   store i16 %result, i16 addrspace(1)* %out
   ret void
 }
 
-; GCN-LABEL: {{^}}shl_v2i16:
-; VI: v_lshlrev_b16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; VI: v_lshlrev_b16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 define amdgpu_kernel void @shl_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %in) {
+; GCN-LABEL: shl_v2i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_mov_b32 s10, s6
+; GCN-NEXT:    s_mov_b32 s11, s7
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_mov_b32 s8, s2
+; GCN-NEXT:    s_mov_b32 s9, s3
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GCN-NEXT:    v_mov_b32_e32 v1, 0
+; GCN-NEXT:    s_mov_b32 s14, 0
+; GCN-NEXT:    s_mov_b32 s15, s7
+; GCN-NEXT:    s_mov_b64 s[12:13], s[2:3]
+; GCN-NEXT:    buffer_load_dword v0, v[0:1], s[12:15], 0 addr64 offset:4
+; GCN-NEXT:    buffer_load_dword v1, off, s[8:11], 0
+; GCN-NEXT:    s_mov_b32 s4, s0
+; GCN-NEXT:    s_mov_b32 s0, 0xffff
+; GCN-NEXT:    s_mov_b32 s5, s1
+; GCN-NEXT:    s_waitcnt vmcnt(1)
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
+; GCN-NEXT:    v_and_b32_e32 v0, s0, v0
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshrrev_b32_e32 v2, 16, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, v0, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, v3, v2
+; GCN-NEXT:    v_and_b32_e32 v0, s0, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v1
+; GCN-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
+;
+; EG-LABEL: shl_v2i16:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 2, @12, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 0 @8
+; EG-NEXT:    ALU 0, @15, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 0 @10
+; EG-NEXT:    ALU 12, @16, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.X, T7.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+; EG-NEXT:    Fetch clause starting at 8:
+; EG-NEXT:     VTX_READ_32 T0.X, T0.X, 4, #1
+; EG-NEXT:    Fetch clause starting at 10:
+; EG-NEXT:     VTX_READ_32 T7.X, T7.X, 0, #1
+; EG-NEXT:    ALU clause starting at 12:
+; EG-NEXT:     LSHL * T0.W, T0.X, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+; EG-NEXT:     ADD_INT * T0.X, KC0[2].Z, PV.W,
+; EG-NEXT:    ALU clause starting at 15:
+; EG-NEXT:     MOV * T7.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 16:
+; EG-NEXT:     AND_INT T0.Y, T0.X, literal.x,
+; EG-NEXT:     AND_INT T0.Z, T7.X, literal.x, BS:VEC_120/SCL_212
+; EG-NEXT:     LSHR T0.W, T0.X, literal.y,
+; EG-NEXT:     LSHR * T1.W, T7.X, literal.y,
+; EG-NEXT:    65535(9.183409e-41), 16(2.242078e-44)
+; EG-NEXT:     LSHL T0.W, PS, PV.W,
+; EG-NEXT:     LSHL * T1.W, PV.Z, PV.Y,
+; EG-NEXT:     AND_INT T1.W, PS, literal.x,
+; EG-NEXT:     LSHL * T0.W, PV.W, literal.y,
+; EG-NEXT:    65535(9.183409e-41), 16(2.242078e-44)
+; EG-NEXT:     OR_INT T0.X, PV.W, PS,
+; EG-NEXT:     LSHR * T7.X, KC0[2].Y, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
   %tid = call i32 @llvm.r600.read.tidig.x() #0
   %gep = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %in, i32 %tid
   %gep.out = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %out, i32 %tid
@@ -130,12 +478,115 @@ define amdgpu_kernel void @shl_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> add
   ret void
 }
 
-; GCN-LABEL: {{^}}shl_v4i16:
-; VI: v_lshlrev_b16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; VI: v_lshlrev_b16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; VI: v_lshlrev_b16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; VI: v_lshlrev_b16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 define amdgpu_kernel void @shl_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> addrspace(1)* %in) {
+; GCN-LABEL: shl_v4i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, 0
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
+; GCN-NEXT:    v_mov_b32_e32 v1, 0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; GCN-NEXT:    buffer_load_dwordx2 v[2:3], v[0:1], s[4:7], 0 addr64
+; GCN-NEXT:    buffer_load_dwordx2 v[4:5], v[0:1], s[4:7], 0 addr64 offset:8
+; GCN-NEXT:    s_mov_b32 s8, 0xffff
+; GCN-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; GCN-NEXT:    s_waitcnt vmcnt(1)
+; GCN-NEXT:    v_lshrrev_b32_e32 v6, 16, v2
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v8, s8, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    v_and_b32_e32 v9, s8, v5
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, v5, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, v9, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, v4, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, v8, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GCN-NEXT:    v_and_b32_e32 v3, s8, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    v_and_b32_e32 v2, s8, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v5
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v4
+; GCN-NEXT:    buffer_store_dwordx2 v[2:3], v[0:1], s[0:3], 0 addr64
+; GCN-NEXT:    s_endpgm
+;
+; EG-LABEL: shl_v4i16:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 2, @12, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 0 @8
+; EG-NEXT:    ALU 3, @15, KC0[], KC1[]
+; EG-NEXT:    TEX 0 @10
+; EG-NEXT:    ALU 49, @19, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T10.XY, T0.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+; EG-NEXT:    Fetch clause starting at 8:
+; EG-NEXT:     VTX_READ_64 T10.XY, T0.X, 0, #1
+; EG-NEXT:    Fetch clause starting at 10:
+; EG-NEXT:     VTX_READ_64 T10.XY, T0.X, 8, #1
+; EG-NEXT:    ALU clause starting at 12:
+; EG-NEXT:     LSHL * T0.W, T0.X, literal.x,
+; EG-NEXT:    3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT:     ADD_INT * T0.X, KC0[2].Z, PV.W,
+; EG-NEXT:    ALU clause starting at 15:
+; EG-NEXT:     MOV T4.X, T10.X,
+; EG-NEXT:     MOV * T5.X, T10.Y,
+; EG-NEXT:     MOV T0.Y, PV.X,
+; EG-NEXT:     MOV * T0.Z, PS,
+; EG-NEXT:    ALU clause starting at 19:
+; EG-NEXT:     MOV T2.X, T10.X,
+; EG-NEXT:     MOV * T3.X, T10.Y,
+; EG-NEXT:     MOV T0.X, T6.X,
+; EG-NEXT:     MOV * T1.Y, PV.X,
+; EG-NEXT:     AND_INT T1.W, PV.Y, literal.x,
+; EG-NEXT:     AND_INT * T2.W, T0.Y, literal.x,
+; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
+; EG-NEXT:     LSHL * T1.W, PS, PV.W,
+; EG-NEXT:     AND_INT T1.W, PV.W, literal.x,
+; EG-NEXT:     AND_INT * T2.W, T0.X, literal.y,
+; EG-NEXT:    65535(9.183409e-41), -65536(nan)
+; EG-NEXT:     OR_INT * T1.W, PS, PV.W,
+; EG-NEXT:     MOV T0.X, T3.X,
+; EG-NEXT:     MOV * T6.X, PV.W,
+; EG-NEXT:     MOV T1.Z, PS,
+; EG-NEXT:     LSHR T1.W, T1.Y, literal.x,
+; EG-NEXT:     LSHR * T2.W, T0.Y, literal.x,
+; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
+; EG-NEXT:     LSHL T1.W, PS, PV.W,
+; EG-NEXT:     AND_INT * T2.W, PV.Z, literal.x,
+; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
+; EG-NEXT:     LSHL * T1.W, PV.W, literal.x,
+; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
+; EG-NEXT:     OR_INT * T1.W, T2.W, PV.W,
+; EG-NEXT:     MOV T6.X, PV.W,
+; EG-NEXT:     MOV T0.Y, T7.X,
+; EG-NEXT:     AND_INT T1.W, T0.X, literal.x, BS:VEC_120/SCL_212
+; EG-NEXT:     AND_INT * T2.W, T0.Z, literal.x,
+; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
+; EG-NEXT:     LSHL T1.W, PS, PV.W,
+; EG-NEXT:     AND_INT * T2.W, PV.Y, literal.x,
+; EG-NEXT:    -65536(nan), 0(0.000000e+00)
+; EG-NEXT:     AND_INT * T1.W, PV.W, literal.x,
+; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
+; EG-NEXT:     OR_INT * T1.W, T2.W, PV.W,
+; EG-NEXT:     MOV * T7.X, PV.W,
+; EG-NEXT:     MOV T0.Y, PV.X,
+; EG-NEXT:     LSHR T1.W, T0.X, literal.x,
+; EG-NEXT:     LSHR * T2.W, T0.Z, literal.x,
+; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
+; EG-NEXT:     LSHL * T1.W, PS, PV.W,
+; EG-NEXT:     AND_INT T0.Z, T0.Y, literal.x,
+; EG-NEXT:     LSHL T1.W, PV.W, literal.y,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, T0.W,
+; EG-NEXT:    65535(9.183409e-41), 16(2.242078e-44)
+; EG-NEXT:     LSHR T0.X, PS, literal.x,
+; EG-NEXT:     OR_INT * T10.Y, PV.Z, PV.W,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+; EG-NEXT:     MOV T7.X, PV.Y,
+; EG-NEXT:     MOV * T10.X, T6.X,
   %tid = call i32 @llvm.r600.read.tidig.x() #0
   %gep = getelementptr inbounds <4 x i16>, <4 x i16> addrspace(1)* %in, i32 %tid
   %gep.out = getelementptr inbounds <4 x i16>, <4 x i16> addrspace(1)* %out, i32 %tid
@@ -147,22 +598,54 @@ define amdgpu_kernel void @shl_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> add
   ret void
 }
 
-;EG-LABEL: {{^}}shl_i64:
-;EG: SUB_INT {{\*? *}}[[COMPSH:T[0-9]+\.[XYZW]]], {{literal.[xy]}}, [[SHIFT:T[0-9]+\.[XYZW]]]
-;EG: LSHR {{\* *}}[[TEMP:T[0-9]+\.[XYZW]]], [[OPLO:T[0-9]+\.[XYZW]]], {{[[COMPSH]]|PV.[XYZW]}}
-;EG-DAG: ADD_INT {{\*? *}}[[BIGSH:T[0-9]+\.[XYZW]]], [[SHIFT]], literal
-;EG-DAG: LSHR {{\*? *}}[[OVERF:T[0-9]+\.[XYZW]]], {{[[TEMP]]|PV.[XYZW]}}, 1
-;EG-DAG: LSHL {{\*? *}}[[HISMTMP:T[0-9]+\.[XYZW]]], [[OPHI:T[0-9]+\.[XYZW]]], [[SHIFT]]
-;EG-DAG: OR_INT {{\*? *}}[[HISM:T[0-9]+\.[XYZW]]], {{[[HISMTMP]]|PV.[XYZW]|PS}}, {{[[OVERF]]|PV.[XYZW]}}
-;EG-DAG: LSHL {{\*? *}}[[LOSM:T[0-9]+\.[XYZW]]], [[OPLO]], {{PS|[[SHIFT]]|PV.[XYZW]}}
-;EG-DAG: SETGT_UINT {{\*? *}}[[RESC:T[0-9]+\.[XYZW]]], [[SHIFT]], literal
-;EG-DAG: CNDE_INT {{\*? *}}[[RESLO:T[0-9]+\.[XYZW]]], {{T[0-9]+\.[XYZW]}}
-;EG-DAG: CNDE_INT {{\*? *}}[[RESHI:T[0-9]+\.[XYZW]]], {{T[0-9]+\.[XYZW], .*}}, 0.0
-
-; GCN-LABEL: {{^}}shl_i64:
-; SI: v_lshl_b64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v[0-9]+}}
-; VI: v_lshlrev_b64 {{v\[[0-9]+:[0-9]+\], v[0-9]+, v\[[0-9]+:[0-9]+\]}}
 define amdgpu_kernel void @shl_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %in) {
+; GCN-LABEL: shl_i64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GCN-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NEXT:    s_mov_b32 s2, -1
+; GCN-NEXT:    s_mov_b32 s10, s2
+; GCN-NEXT:    s_mov_b32 s11, s3
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_mov_b32 s8, s6
+; GCN-NEXT:    s_mov_b32 s9, s7
+; GCN-NEXT:    buffer_load_dwordx4 v[0:3], off, s[8:11], 0
+; GCN-NEXT:    s_mov_b32 s0, s4
+; GCN-NEXT:    s_mov_b32 s1, s5
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshl_b64 v[0:1], v[0:1], v2
+; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GCN-NEXT:    s_endpgm
+;
+; EG-LABEL: shl_i64:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 0 @6
+; EG-NEXT:    ALU 15, @9, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+; EG-NEXT:    Fetch clause starting at 6:
+; EG-NEXT:     VTX_READ_128 T0.XYZW, T0.X, 0, #1
+; EG-NEXT:    ALU clause starting at 8:
+; EG-NEXT:     MOV * T0.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 9:
+; EG-NEXT:     SUB_INT * T0.W, literal.x, T0.Z,
+; EG-NEXT:    31(4.344025e-44), 0(0.000000e+00)
+; EG-NEXT:     LSHR * T0.W, T0.X, PV.W,
+; EG-NEXT:     ADD_INT T1.Z, T0.Z, literal.x,
+; EG-NEXT:     LSHR T0.W, PV.W, 1,
+; EG-NEXT:     LSHL * T1.W, T0.Y, T0.Z,
+; EG-NEXT:    -32(nan), 0(0.000000e+00)
+; EG-NEXT:     OR_INT T2.Z, PS, PV.W,
+; EG-NEXT:     LSHL T0.W, T0.X, PV.Z,
+; EG-NEXT:     SETGT_UINT * T1.W, T0.Z, literal.x,
+; EG-NEXT:    31(4.344025e-44), 0(0.000000e+00)
+; EG-NEXT:     CNDE_INT T0.Y, PS, PV.Z, PV.W,
+; EG-NEXT:     LSHL * T0.W, T0.X, T0.Z,
+; EG-NEXT:     CNDE_INT T0.X, T1.W, PV.W, 0.0,
+; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
   %b_ptr = getelementptr i64, i64 addrspace(1)* %in, i64 1
   %a = load i64, i64 addrspace(1)* %in
   %b = load i64, i64 addrspace(1)* %b_ptr
@@ -171,37 +654,70 @@ define amdgpu_kernel void @shl_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %in
   ret void
 }
 
-;EG-LABEL: {{^}}shl_v2i64:
-;EG-DAG: SUB_INT {{\*? *}}[[COMPSHA:T[0-9]+\.[XYZW]]], {{literal.[xy]}}, [[SHA:T[0-9]+\.[XYZW]]]
-;EG-DAG: SUB_INT {{\*? *}}[[COMPSHB:T[0-9]+\.[XYZW]]], {{literal.[xy]}}, [[SHB:T[0-9]+\.[XYZW]]]
-;EG-DAG: LSHR {{\*? *}}[[COMPSHA]]
-;EG-DAG: LSHR {{\*? *}}[[COMPSHB]]
-;EG-DAG: LSHR {{.*}}, 1
-;EG-DAG: LSHR {{.*}}, 1
-;EG-DAG: ADD_INT  {{\*? *}}[[BIGSHA:T[0-9]+\.[XYZW]]]{{.*}}, literal
-;EG-DAG: ADD_INT  {{\*? *}}[[BIGSHB:T[0-9]+\.[XYZW]]]{{.*}}, literal
-;EG-DAG: LSHL {{.*}}, [[SHA]]
-;EG-DAG: LSHL {{.*}}, [[SHB]]
-;EG-DAG: LSHL {{.*}}, [[SHA]]
-;EG-DAG: LSHL {{.*}}, [[SHB]]
-;EG-DAG: LSHL
-;EG-DAG: LSHL
-;EG-DAG: SETGT_UINT {{\*? *T[0-9]\.[XYZW]}}, [[SHA]], literal
-;EG-DAG: SETGT_UINT {{\*? *T[0-9]\.[XYZW]}}, [[SHB]], literal
-;EG-DAG: CNDE_INT {{.*}}, 0.0
-;EG-DAG: CNDE_INT {{.*}}, 0.0
-;EG-DAG: CNDE_INT
-;EG-DAG: CNDE_INT
-
-;SI: {{^}}shl_v2i64:
-;SI: v_lshl_b64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v[0-9]+}}
-;SI: v_lshl_b64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v[0-9]+}}
-
-;VI: {{^}}shl_v2i64:
-;VI: v_lshlrev_b64 {{v\[[0-9]+:[0-9]+\], v[0-9]+, v\[[0-9]+:[0-9]+\]}}
-;VI: v_lshlrev_b64 {{v\[[0-9]+:[0-9]+\], v[0-9]+, v\[[0-9]+:[0-9]+\]}}
-
 define amdgpu_kernel void @shl_v2i64(<2 x i64> addrspace(1)* %out, <2 x i64> addrspace(1)* %in) {
+; GCN-LABEL: shl_v2i64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GCN-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NEXT:    s_mov_b32 s2, -1
+; GCN-NEXT:    s_mov_b32 s10, s2
+; GCN-NEXT:    s_mov_b32 s11, s3
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_mov_b32 s8, s6
+; GCN-NEXT:    s_mov_b32 s9, s7
+; GCN-NEXT:    buffer_load_dwordx4 v[0:3], off, s[8:11], 0
+; GCN-NEXT:    buffer_load_dwordx4 v[4:7], off, s[8:11], 0 offset:16
+; GCN-NEXT:    s_mov_b32 s0, s4
+; GCN-NEXT:    s_mov_b32 s1, s5
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshl_b64 v[2:3], v[2:3], v6
+; GCN-NEXT:    v_lshl_b64 v[0:1], v[0:1], v4
+; GCN-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; GCN-NEXT:    s_endpgm
+;
+; EG-LABEL: shl_v2i64:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 0, @10, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 1 @6
+; EG-NEXT:    ALU 28, @11, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T2.XYZW, T0.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+; EG-NEXT:    Fetch clause starting at 6:
+; EG-NEXT:     VTX_READ_128 T1.XYZW, T0.X, 16, #1
+; EG-NEXT:     VTX_READ_128 T0.XYZW, T0.X, 0, #1
+; EG-NEXT:    ALU clause starting at 10:
+; EG-NEXT:     MOV * T0.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 11:
+; EG-NEXT:     SUB_INT * T1.W, literal.x, T1.Z,
+; EG-NEXT:    31(4.344025e-44), 0(0.000000e+00)
+; EG-NEXT:     LSHR * T1.W, T0.Z, PV.W,
+; EG-NEXT:     SUB_INT T2.Z, literal.x, T1.X,
+; EG-NEXT:     LSHR T1.W, PV.W, 1,
+; EG-NEXT:     LSHL * T0.W, T0.W, T1.Z,
+; EG-NEXT:    31(4.344025e-44), 0(0.000000e+00)
+; EG-NEXT:     OR_INT T3.Z, PS, PV.W,
+; EG-NEXT:     LSHR T0.W, T0.X, PV.Z,
+; EG-NEXT:     ADD_INT * T1.W, T1.Z, literal.x,
+; EG-NEXT:    -32(nan), 0(0.000000e+00)
+; EG-NEXT:     LSHL T2.X, T0.Z, PS,
+; EG-NEXT:     SETGT_UINT T1.Y, T1.Z, literal.x, BS:VEC_120/SCL_212
+; EG-NEXT:     ADD_INT T2.Z, T1.X, literal.y,
+; EG-NEXT:     LSHR T0.W, PV.W, 1,
+; EG-NEXT:     LSHL * T1.W, T0.Y, T1.X,
+; EG-NEXT:    31(4.344025e-44), -32(nan)
+; EG-NEXT:     OR_INT T0.Y, PS, PV.W,
+; EG-NEXT:     LSHL T2.Z, T0.X, PV.Z,
+; EG-NEXT:     SETGT_UINT T0.W, T1.X, literal.x, BS:VEC_120/SCL_212
+; EG-NEXT:     CNDE_INT * T2.W, PV.Y, T3.Z, PV.X,
+; EG-NEXT:    31(4.344025e-44), 0(0.000000e+00)
+; EG-NEXT:     CNDE_INT T2.Y, PV.W, PV.Y, PV.Z,
+; EG-NEXT:     LSHL * T1.W, T0.Z, T1.Z,
+; EG-NEXT:     CNDE_INT T2.Z, T1.Y, PV.W, 0.0,
+; EG-NEXT:     LSHL * T1.W, T0.X, T1.X,
+; EG-NEXT:     CNDE_INT T2.X, T0.W, PV.W, 0.0,
+; EG-NEXT:     LSHR * T0.X, KC0[2].Y, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
   %b_ptr = getelementptr <2 x i64>, <2 x i64> addrspace(1)* %in, i64 1
   %a = load <2 x i64>, <2 x i64> addrspace(1)* %in
   %b = load <2 x i64>, <2 x i64> addrspace(1)* %b_ptr
@@ -210,61 +726,108 @@ define amdgpu_kernel void @shl_v2i64(<2 x i64> addrspace(1)* %out, <2 x i64> add
   ret void
 }
 
-;EG: {{^}}shl_v4i64:
-;EG-DAG: SUB_INT {{\*? *}}[[COMPSHA:T[0-9]+\.[XYZW]]], {{literal.[xy]}}, [[SHA:T[0-9]+\.[XYZW]]]
-;EG-DAG: SUB_INT {{\*? *}}[[COMPSHB:T[0-9]+\.[XYZW]]], {{literal.[xy]}}, [[SHB:T[0-9]+\.[XYZW]]]
-;EG-DAG: SUB_INT {{\*? *}}[[COMPSHC:T[0-9]+\.[XYZW]]], {{literal.[xy]}}, [[SHC:T[0-9]+\.[XYZW]]]
-;EG-DAG: SUB_INT {{\*? *}}[[COMPSHD:T[0-9]+\.[XYZW]]], {{literal.[xy]}}, [[SHD:T[0-9]+\.[XYZW]]]
-;EG-DAG: LSHR {{\*? *}}[[COMPSHA]]
-;EG-DAG: LSHR {{\*? *}}[[COMPSHB]]
-;EG-DAG: LSHR {{\*? *}}[[COMPSHC]]
-;EG-DAG: LSHR {{\*? *}}[[COMPSHD]]
-;EG-DAG: LSHR {{.*}}, 1
-;EG-DAG: LSHR {{.*}}, 1
-;EG-DAG: LSHR {{.*}}, 1
-;EG-DAG: LSHR {{.*}}, 1
-;EG-DAG: ADD_INT  {{\*? *}}[[BIGSHA:T[0-9]+\.[XYZW]]]{{.*}}, literal
-;EG-DAG: ADD_INT  {{\*? *}}[[BIGSHB:T[0-9]+\.[XYZW]]]{{.*}}, literal
-;EG-DAG: ADD_INT  {{\*? *}}[[BIGSHC:T[0-9]+\.[XYZW]]]{{.*}}, literal
-;EG-DAG: ADD_INT  {{\*? *}}[[BIGSHD:T[0-9]+\.[XYZW]]]{{.*}}, literal
-;EG-DAG: LSHL {{.*}}, [[SHA]]
-;EG-DAG: LSHL {{.*}}, [[SHB]]
-;EG-DAG: LSHL {{.*}}, [[SHC]]
-;EG-DAG: LSHL {{.*}}, [[SHD]]
-;EG-DAG: LSHL {{.*}}, [[SHA]]
-;EG-DAG: LSHL {{.*}}, [[SHB]]
-;EG-DAG: LSHL {{.*}}, [[SHC]]
-;EG-DAG: LSHL {{.*}}, [[SHD]]
-;EG-DAG: LSHL
-;EG-DAG: LSHL
-;EG-DAG: LSHL
-;EG-DAG: LSHL
-;EG-DAG: SETGT_UINT {{\*? *T[0-9]\.[XYZW]}}, [[SHA]], literal
-;EG-DAG: SETGT_UINT {{\*? *T[0-9]\.[XYZW]}}, [[SHB]], literal
-;EG-DAG: SETGT_UINT {{\*? *T[0-9]\.[XYZW]}}, [[SHC]], literal
-;EG-DAG: SETGT_UINT {{\*? *T[0-9]\.[XYZW]}}, [[SHD]], literal
-;EG-DAG: CNDE_INT {{.*}}, 0.0
-;EG-DAG: CNDE_INT {{.*}}, 0.0
-;EG-DAG: CNDE_INT {{.*}}, 0.0
-;EG-DAG: CNDE_INT {{.*}}, 0.0
-;EG-DAG: CNDE_INT
-;EG-DAG: CNDE_INT
-;EG-DAG: CNDE_INT
-;EG-DAG: CNDE_INT
-
-;SI: {{^}}shl_v4i64:
-;SI: v_lshl_b64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v[0-9]+}}
-;SI: v_lshl_b64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v[0-9]+}}
-;SI: v_lshl_b64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v[0-9]+}}
-;SI: v_lshl_b64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v[0-9]+}}
-
-;VI: {{^}}shl_v4i64:
-;VI: v_lshlrev_b64 {{v\[[0-9]+:[0-9]+\], v[0-9]+, v\[[0-9]+:[0-9]+\]}}
-;VI: v_lshlrev_b64 {{v\[[0-9]+:[0-9]+\], v[0-9]+, v\[[0-9]+:[0-9]+\]}}
-;VI: v_lshlrev_b64 {{v\[[0-9]+:[0-9]+\], v[0-9]+, v\[[0-9]+:[0-9]+\]}}
-;VI: v_lshlrev_b64 {{v\[[0-9]+:[0-9]+\], v[0-9]+, v\[[0-9]+:[0-9]+\]}}
-
 define amdgpu_kernel void @shl_v4i64(<4 x i64> addrspace(1)* %out, <4 x i64> addrspace(1)* %in) {
+; GCN-LABEL: shl_v4i64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GCN-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NEXT:    s_mov_b32 s2, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_mov_b32 s0, s4
+; GCN-NEXT:    s_mov_b32 s1, s5
+; GCN-NEXT:    s_mov_b32 s4, s6
+; GCN-NEXT:    s_mov_b32 s5, s7
+; GCN-NEXT:    s_mov_b32 s6, s2
+; GCN-NEXT:    s_mov_b32 s7, s3
+; GCN-NEXT:    buffer_load_dwordx4 v[0:3], off, s[4:7], 0
+; GCN-NEXT:    buffer_load_dwordx4 v[4:7], off, s[4:7], 0 offset:16
+; GCN-NEXT:    buffer_load_dwordx4 v[8:11], off, s[4:7], 0 offset:32
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_load_dwordx4 v[11:14], off, s[4:7], 0 offset:48
+; GCN-NEXT:    v_lshl_b64 v[2:3], v[2:3], v10
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshl_b64 v[6:7], v[6:7], v13
+; GCN-NEXT:    v_lshl_b64 v[4:5], v[4:5], v11
+; GCN-NEXT:    v_lshl_b64 v[0:1], v[0:1], v8
+; GCN-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:16
+; GCN-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; GCN-NEXT:    s_endpgm
+;
+; EG-LABEL: shl_v4i64:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 0, @14, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 3 @6
+; EG-NEXT:    ALU 58, @15, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T1.XYZW, T0.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T3.XYZW, T4.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    Fetch clause starting at 6:
+; EG-NEXT:     VTX_READ_128 T1.XYZW, T0.X, 48, #1
+; EG-NEXT:     VTX_READ_128 T2.XYZW, T0.X, 0, #1
+; EG-NEXT:     VTX_READ_128 T3.XYZW, T0.X, 16, #1
+; EG-NEXT:     VTX_READ_128 T0.XYZW, T0.X, 32, #1
+; EG-NEXT:    ALU clause starting at 14:
+; EG-NEXT:     MOV * T0.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 15:
+; EG-NEXT:     SUB_INT * T0.W, literal.x, T1.Z,
+; EG-NEXT:    31(4.344025e-44), 0(0.000000e+00)
+; EG-NEXT:     SUB_INT T4.Z, literal.x, T0.Z,
+; EG-NEXT:     SUB_INT T1.W, literal.x, T0.X,
+; EG-NEXT:     LSHR * T0.W, T3.Z, PV.W,
+; EG-NEXT:    31(4.344025e-44), 0(0.000000e+00)
+; EG-NEXT:     SUB_INT T0.Y, literal.x, T1.X,
+; EG-NEXT:     LSHR T5.Z, PS, 1,
+; EG-NEXT:     LSHR T0.W, T2.X, PV.W,
+; EG-NEXT:     LSHR * T1.W, T2.Z, PV.Z,
+; EG-NEXT:    31(4.344025e-44), 0(0.000000e+00)
+; EG-NEXT:     LSHL T4.X, T3.W, T1.Z,
+; EG-NEXT:     LSHR T1.Y, PS, 1,
+; EG-NEXT:     LSHL T4.Z, T2.W, T0.Z, BS:VEC_120/SCL_212
+; EG-NEXT:     LSHR T0.W, PV.W, 1,
+; EG-NEXT:     LSHL * T1.W, T2.Y, T0.X,
+; EG-NEXT:     OR_INT T5.X, PS, PV.W,
+; EG-NEXT:     OR_INT T1.Y, PV.Z, PV.Y,
+; EG-NEXT:     OR_INT T4.Z, PV.X, T5.Z,
+; EG-NEXT:     LSHR T0.W, T3.X, T0.Y,
+; EG-NEXT:     ADD_INT * T1.W, T1.Z, literal.x,
+; EG-NEXT:    -32(nan), 0(0.000000e+00)
+; EG-NEXT:     LSHL T4.X, T3.Z, PS,
+; EG-NEXT:     SETGT_UINT T0.Y, T1.Z, literal.x, BS:VEC_120/SCL_212
+; EG-NEXT:     ADD_INT T5.Z, T1.X, literal.y,
+; EG-NEXT:     LSHR T0.W, PV.W, 1,
+; EG-NEXT:     LSHL * T1.W, T3.Y, T1.X,
+; EG-NEXT:    31(4.344025e-44), -32(nan)
+; EG-NEXT:     OR_INT T6.X, PS, PV.W,
+; EG-NEXT:     LSHL T2.Y, T3.X, PV.Z,
+; EG-NEXT:     SETGT_UINT T5.Z, T1.X, literal.x, BS:VEC_120/SCL_212
+; EG-NEXT:     ADD_INT T0.W, T0.Z, literal.y,
+; EG-NEXT:     CNDE_INT * T3.W, PV.Y, T4.Z, PV.X,
+; EG-NEXT:    31(4.344025e-44), -32(nan)
+; EG-NEXT:     LSHL T4.X, T2.Z, PV.W,
+; EG-NEXT:     CNDE_INT T3.Y, PV.Z, PV.X, PV.Y,
+; EG-NEXT:     SETGT_UINT * T4.Z, T0.Z, literal.x, BS:VEC_120/SCL_212
+; EG-NEXT:    31(4.344025e-44), 0(0.000000e+00)
+; EG-NEXT:     LSHL T0.W, T3.Z, T1.Z,
+; EG-NEXT:     ADD_INT * T1.W, T0.X, literal.x,
+; EG-NEXT:    -32(nan), 0(0.000000e+00)
+; EG-NEXT:     LSHL T6.X, T2.X, PS,
+; EG-NEXT:     SETGT_UINT T2.Y, T0.X, literal.x, BS:VEC_120/SCL_212
+; EG-NEXT:     CNDE_INT * T3.Z, T0.Y, PV.W, 0.0,
+; EG-NEXT:    31(4.344025e-44), 0(0.000000e+00)
+; EG-NEXT:     LSHL T0.W, T3.X, T1.X, BS:VEC_120/SCL_212
+; EG-NEXT:     CNDE_INT * T1.W, T4.Z, T1.Y, T4.X,
+; EG-NEXT:     CNDE_INT T3.X, T5.Z, PV.W, 0.0,
+; EG-NEXT:     CNDE_INT T1.Y, T2.Y, T5.X, T6.X,
+; EG-NEXT:     LSHL T0.W, T2.Z, T0.Z, BS:VEC_120/SCL_212
+; EG-NEXT:     ADD_INT * T2.W, KC0[2].Y, literal.x,
+; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
+; EG-NEXT:     LSHR T4.X, PS, literal.x,
+; EG-NEXT:     CNDE_INT T1.Z, T4.Z, PV.W, 0.0,
+; EG-NEXT:     LSHL * T0.W, T2.X, T0.X,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+; EG-NEXT:     CNDE_INT T1.X, T2.Y, PV.W, 0.0,
+; EG-NEXT:     LSHR * T0.X, KC0[2].Y, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
   %b_ptr = getelementptr <4 x i64>, <4 x i64> addrspace(1)* %in, i64 1
   %a = load <4 x i64>, <4 x i64> addrspace(1)* %in
   %b = load <4 x i64>, <4 x i64> addrspace(1)* %b_ptr
@@ -274,22 +837,74 @@ define amdgpu_kernel void @shl_v4i64(<4 x i64> addrspace(1)* %out, <4 x i64> add
 }
 
 ; Make sure load width gets reduced to i32 load.
-; GCN-LABEL: {{^}}s_shl_32_i64:
-; GCN-DAG: s_load_dword [[LO_A:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x13{{$}}
-; GCN-DAG: v_mov_b32_e32 v[[VLO:[0-9]+]], 0{{$}}
-; GCN-DAG: v_mov_b32_e32 v[[VHI:[0-9]+]], [[LO_A]]
-; GCN: buffer_store_dwordx2 v{{\[}}[[VLO]]:[[VHI]]{{\]}}
 define amdgpu_kernel void @s_shl_32_i64(i64 addrspace(1)* %out, [8 x i32], i64 %a) {
+; GCN-LABEL: s_shl_32_i64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_load_dword s0, s[0:1], 0x13
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    v_mov_b32_e32 v0, 0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v1, s0
+; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
+;
+; EG-LABEL: s_shl_32_i64:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 3, @4, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+; EG-NEXT:    ALU clause starting at 4:
+; EG-NEXT:     MOV * T0.Y, KC0[4].W,
+; EG-NEXT:     MOV T0.X, 0.0,
+; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
   %result = shl i64 %a, 32
   store i64 %result, i64 addrspace(1)* %out
   ret void
 }
 
-; GCN-LABEL: {{^}}v_shl_32_i64:
-; GCN-DAG: buffer_load_dword v[[LO_A:[0-9]+]],
-; GCN-DAG: v_mov_b32_e32 v[[VLO:[0-9]+]], 0{{$}}
-; GCN: buffer_store_dwordx2 v{{\[}}[[VLO]]:[[LO_A]]{{\]}}
 define amdgpu_kernel void @v_shl_32_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %in) {
+; GCN-LABEL: v_shl_32_i64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GCN-NEXT:    s_ashr_i32 s3, s2, 31
+; GCN-NEXT:    s_lshl_b64 s[8:9], s[2:3], 3
+; GCN-NEXT:    v_mov_b32_e32 v0, s8
+; GCN-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NEXT:    s_mov_b32 s2, 0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_mov_b64 s[0:1], s[6:7]
+; GCN-NEXT:    v_mov_b32_e32 v1, s9
+; GCN-NEXT:    buffer_load_dword v3, v[0:1], s[0:3], 0 addr64
+; GCN-NEXT:    s_mov_b64 s[6:7], s[2:3]
+; GCN-NEXT:    v_mov_b32_e32 v2, 0
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dwordx2 v[2:3], v[0:1], s[4:7], 0 addr64
+; GCN-NEXT:    s_endpgm
+;
+; EG-LABEL: v_shl_32_i64:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 2, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 0 @6
+; EG-NEXT:    ALU 4, @11, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T1.XY, T2.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+; EG-NEXT:    Fetch clause starting at 6:
+; EG-NEXT:     VTX_READ_32 T0.X, T0.X, 0, #1
+; EG-NEXT:    ALU clause starting at 8:
+; EG-NEXT:     LSHL * T0.W, T1.X, literal.x,
+; EG-NEXT:    3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT:     ADD_INT * T0.X, KC0[2].Z, PV.W,
+; EG-NEXT:    ALU clause starting at 11:
+; EG-NEXT:     MOV T1.X, 0.0,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, T0.W,
+; EG-NEXT:     LSHR T2.X, PV.W, literal.x,
+; EG-NEXT:     MOV * T1.Y, T0.X,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
   %tid = call i32 @llvm.r600.read.tgid.x() #0
   %gep.in = getelementptr i64, i64 addrspace(1)* %in, i32 %tid
   %gep.out = getelementptr i64, i64 addrspace(1)* %out, i32 %tid
@@ -299,123 +914,554 @@ define amdgpu_kernel void @v_shl_32_i64(i64 addrspace(1)* %out, i64 addrspace(1)
   ret void
 }
 
-; FUNC-LABEL: {{^}}s_shl_constant_i64
-; SI: s_lshl_b64 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}}
 define amdgpu_kernel void @s_shl_constant_i64(i64 addrspace(1)* %out, i64 %a) {
+; GCN-LABEL: s_shl_constant_i64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GCN-NEXT:    s_mov_b32 s2, -1
+; GCN-NEXT:    s_mov_b32 s9, 0xffff
+; GCN-NEXT:    s_mov_b32 s8, s2
+; GCN-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_mov_b32 s0, s4
+; GCN-NEXT:    s_mov_b32 s1, s5
+; GCN-NEXT:    s_lshl_b64 s[4:5], s[8:9], s6
+; GCN-NEXT:    v_mov_b32_e32 v0, s4
+; GCN-NEXT:    v_mov_b32_e32 v1, s5
+; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GCN-NEXT:    s_endpgm
+;
+; EG-LABEL: s_shl_constant_i64:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 17, @4, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+; EG-NEXT:    ALU clause starting at 4:
+; EG-NEXT:     SUB_INT * T0.W, literal.x, KC0[2].W,
+; EG-NEXT:    31(4.344025e-44), 0(0.000000e+00)
+; EG-NEXT:     LSHR * T0.W, literal.x, PV.W,
+; EG-NEXT:    -1(nan), 0(0.000000e+00)
+; EG-NEXT:     ADD_INT T0.Z, KC0[2].W, literal.x,
+; EG-NEXT:     LSHR T0.W, PV.W, 1,
+; EG-NEXT:     LSHL * T1.W, literal.y, KC0[2].W,
+; EG-NEXT:    -32(nan), 65535(9.183409e-41)
+; EG-NEXT:     OR_INT T1.Z, PS, PV.W,
+; EG-NEXT:     LSHL T0.W, literal.x, PV.Z,
+; EG-NEXT:     SETGT_UINT * T1.W, KC0[2].W, literal.y,
+; EG-NEXT:    -1(nan), 31(4.344025e-44)
+; EG-NEXT:     CNDE_INT T0.Y, PS, PV.Z, PV.W,
+; EG-NEXT:     LSHL * T0.W, literal.x, KC0[2].W,
+; EG-NEXT:    -1(nan), 0(0.000000e+00)
+; EG-NEXT:     CNDE_INT T0.X, T1.W, PV.W, 0.0,
+; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
   %shl = shl i64 281474976710655, %a
   store i64 %shl, i64 addrspace(1)* %out, align 8
   ret void
 }
 
-; FUNC-LABEL: {{^}}v_shl_constant_i64:
-; SI-DAG: buffer_load_dword [[VAL:v[0-9]+]]
-; SI-DAG: s_mov_b32 s[[KLO:[0-9]+]], 0xab19b207
-; SI-DAG: s_movk_i32 s[[KHI:[0-9]+]], 0x11e{{$}}
-; SI: v_lshl_b64 {{v\[[0-9]+:[0-9]+\]}}, s{{\[}}[[KLO]]:[[KHI]]{{\]}}, [[VAL]]
-; SI: buffer_store_dwordx2
 define amdgpu_kernel void @v_shl_constant_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %aptr) {
+; GCN-LABEL: v_shl_constant_i64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GCN-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NEXT:    s_mov_b32 s2, -1
+; GCN-NEXT:    s_mov_b32 s10, s2
+; GCN-NEXT:    s_mov_b32 s11, s3
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_mov_b32 s8, s6
+; GCN-NEXT:    s_mov_b32 s9, s7
+; GCN-NEXT:    buffer_load_dword v0, off, s[8:11], 0
+; GCN-NEXT:    s_movk_i32 s7, 0x11e
+; GCN-NEXT:    s_mov_b32 s6, 0xab19b207
+; GCN-NEXT:    s_mov_b32 s0, s4
+; GCN-NEXT:    s_mov_b32 s1, s5
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshl_b64 v[0:1], s[6:7], v0
+; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GCN-NEXT:    s_endpgm
+;
+; EG-LABEL: v_shl_constant_i64:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 0 @6
+; EG-NEXT:    ALU 17, @9, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+; EG-NEXT:    Fetch clause starting at 6:
+; EG-NEXT:     VTX_READ_32 T0.X, T0.X, 0, #1
+; EG-NEXT:    ALU clause starting at 8:
+; EG-NEXT:     MOV * T0.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 9:
+; EG-NEXT:     SUB_INT * T0.W, literal.x, T0.X,
+; EG-NEXT:    31(4.344025e-44), 0(0.000000e+00)
+; EG-NEXT:     LSHR * T0.W, literal.x, PV.W,
+; EG-NEXT:    -1424379385(-5.460358e-13), 0(0.000000e+00)
+; EG-NEXT:     ADD_INT T0.Z, T0.X, literal.x,
+; EG-NEXT:     LSHR T0.W, PV.W, 1,
+; EG-NEXT:     LSHL * T1.W, literal.y, T0.X,
+; EG-NEXT:    -32(nan), 286(4.007714e-43)
+; EG-NEXT:     OR_INT T1.Z, PS, PV.W,
+; EG-NEXT:     SETGT_UINT T0.W, T0.X, literal.x,
+; EG-NEXT:     LSHL * T1.W, literal.y, PV.Z,
+; EG-NEXT:    31(4.344025e-44), -1424379385(-5.460358e-13)
+; EG-NEXT:     CNDE_INT T0.Y, PV.W, PV.Z, PS,
+; EG-NEXT:     LSHL * T1.W, literal.x, T0.X,
+; EG-NEXT:    -1424379385(-5.460358e-13), 0(0.000000e+00)
+; EG-NEXT:     CNDE_INT T0.X, T0.W, PV.W, 0.0,
+; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
   %a = load i64, i64 addrspace(1)* %aptr, align 8
   %shl = shl i64 1231231234567, %a
   store i64 %shl, i64 addrspace(1)* %out, align 8
   ret void
 }
 
-; FUNC-LABEL: {{^}}v_shl_i64_32_bit_constant:
-; SI-DAG: buffer_load_dword [[VAL:v[0-9]+]]
-; SI-DAG: s_mov_b32 s[[KLO:[0-9]+]], 0x12d687{{$}}
-; SI-DAG: s_mov_b32 s[[KHI:[0-9]+]], 0{{$}}
-; SI: v_lshl_b64 {{v\[[0-9]+:[0-9]+\]}}, s{{\[}}[[KLO]]:[[KHI]]{{\]}}, [[VAL]]
 define amdgpu_kernel void @v_shl_i64_32_bit_constant(i64 addrspace(1)* %out, i64 addrspace(1)* %aptr) {
+; GCN-LABEL: v_shl_i64_32_bit_constant:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GCN-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NEXT:    s_mov_b32 s2, -1
+; GCN-NEXT:    s_mov_b32 s10, s2
+; GCN-NEXT:    s_mov_b32 s11, s3
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_mov_b32 s8, s6
+; GCN-NEXT:    s_mov_b32 s9, s7
+; GCN-NEXT:    buffer_load_dword v0, off, s[8:11], 0
+; GCN-NEXT:    s_mov_b32 s7, 0
+; GCN-NEXT:    s_mov_b32 s6, 0x12d687
+; GCN-NEXT:    s_mov_b32 s0, s4
+; GCN-NEXT:    s_mov_b32 s1, s5
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshl_b64 v[0:1], s[6:7], v0
+; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GCN-NEXT:    s_endpgm
+;
+; EG-LABEL: v_shl_i64_32_bit_constant:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 0 @6
+; EG-NEXT:    ALU 14, @9, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+; EG-NEXT:    Fetch clause starting at 6:
+; EG-NEXT:     VTX_READ_32 T0.X, T0.X, 0, #1
+; EG-NEXT:    ALU clause starting at 8:
+; EG-NEXT:     MOV * T0.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 9:
+; EG-NEXT:     SUB_INT T0.W, literal.x, T0.X,
+; EG-NEXT:     ADD_INT * T1.W, T0.X, literal.y,
+; EG-NEXT:    31(4.344025e-44), -32(nan)
+; EG-NEXT:     LSHR * T0.W, literal.x, PV.W,
+; EG-NEXT:    1234567(1.729997e-39), 0(0.000000e+00)
+; EG-NEXT:     LSHR T0.Z, PV.W, 1,
+; EG-NEXT:     LSHL T0.W, literal.x, T1.W,
+; EG-NEXT:     SETGT_UINT * T1.W, T0.X, literal.y,
+; EG-NEXT:    1234567(1.729997e-39), 31(4.344025e-44)
+; EG-NEXT:     CNDE_INT T0.Y, PS, PV.Z, PV.W,
+; EG-NEXT:     LSHL * T0.W, literal.x, T0.X,
+; EG-NEXT:    1234567(1.729997e-39), 0(0.000000e+00)
+; EG-NEXT:     CNDE_INT T0.X, T1.W, PV.W, 0.0,
+; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
   %a = load i64, i64 addrspace(1)* %aptr, align 8
   %shl = shl i64 1234567, %a
   store i64 %shl, i64 addrspace(1)* %out, align 8
   ret void
 }
 
-; FUNC-LABEL: {{^}}v_shl_inline_imm_64_i64:
-; SI: v_lshl_b64 {{v\[[0-9]+:[0-9]+\]}}, 64, {{v[0-9]+}}
 define amdgpu_kernel void @v_shl_inline_imm_64_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %aptr) {
+; GCN-LABEL: v_shl_inline_imm_64_i64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GCN-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NEXT:    s_mov_b32 s2, -1
+; GCN-NEXT:    s_mov_b32 s10, s2
+; GCN-NEXT:    s_mov_b32 s11, s3
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_mov_b32 s8, s6
+; GCN-NEXT:    s_mov_b32 s9, s7
+; GCN-NEXT:    buffer_load_dword v0, off, s[8:11], 0
+; GCN-NEXT:    s_mov_b32 s0, s4
+; GCN-NEXT:    s_mov_b32 s1, s5
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshl_b64 v[0:1], 64, v0
+; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GCN-NEXT:    s_endpgm
+;
+; EG-LABEL: v_shl_inline_imm_64_i64:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 0 @6
+; EG-NEXT:    ALU 14, @9, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+; EG-NEXT:    Fetch clause starting at 6:
+; EG-NEXT:     VTX_READ_32 T0.X, T0.X, 0, #1
+; EG-NEXT:    ALU clause starting at 8:
+; EG-NEXT:     MOV * T0.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 9:
+; EG-NEXT:     SUB_INT T0.W, literal.x, T0.X,
+; EG-NEXT:     ADD_INT * T1.W, T0.X, literal.y,
+; EG-NEXT:    31(4.344025e-44), -32(nan)
+; EG-NEXT:     LSHR * T0.W, literal.x, PV.W,
+; EG-NEXT:    64(8.968310e-44), 0(0.000000e+00)
+; EG-NEXT:     LSHR T0.Z, PV.W, 1,
+; EG-NEXT:     LSHL T0.W, literal.x, T1.W,
+; EG-NEXT:     SETGT_UINT * T1.W, T0.X, literal.y,
+; EG-NEXT:    64(8.968310e-44), 31(4.344025e-44)
+; EG-NEXT:     CNDE_INT T0.Y, PS, PV.Z, PV.W,
+; EG-NEXT:     LSHL * T0.W, literal.x, T0.X,
+; EG-NEXT:    64(8.968310e-44), 0(0.000000e+00)
+; EG-NEXT:     CNDE_INT T0.X, T1.W, PV.W, 0.0,
+; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
   %a = load i64, i64 addrspace(1)* %aptr, align 8
   %shl = shl i64 64, %a
   store i64 %shl, i64 addrspace(1)* %out, align 8
   ret void
 }
 
-; FUNC-LABEL: {{^}}s_shl_inline_imm_64_i64:
-; SI: s_lshl_b64 s{{\[[0-9]+:[0-9]+\]}}, 64, s{{[0-9]+}}
 define amdgpu_kernel void @s_shl_inline_imm_64_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %aptr, i64 %a) {
+; GCN-LABEL: s_shl_inline_imm_64_i64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_load_dword s0, s[0:1], 0xd
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_lshl_b64 s[0:1], 64, s0
+; GCN-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-NEXT:    v_mov_b32_e32 v1, s1
+; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
+;
+; EG-LABEL: s_shl_inline_imm_64_i64:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 14, @4, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+; EG-NEXT:    ALU clause starting at 4:
+; EG-NEXT:     SUB_INT * T0.W, literal.x, KC0[2].W,
+; EG-NEXT:    31(4.344025e-44), 0(0.000000e+00)
+; EG-NEXT:     LSHR T0.W, literal.x, PV.W,
+; EG-NEXT:     ADD_INT * T1.W, KC0[2].W, literal.y,
+; EG-NEXT:    64(8.968310e-44), -32(nan)
+; EG-NEXT:     LSHL T0.Z, literal.x, PS,
+; EG-NEXT:     LSHR T0.W, PV.W, 1,
+; EG-NEXT:     SETGT_UINT * T1.W, KC0[2].W, literal.y,
+; EG-NEXT:    64(8.968310e-44), 31(4.344025e-44)
+; EG-NEXT:     CNDE_INT T0.Y, PS, PV.W, PV.Z,
+; EG-NEXT:     LSHL * T0.W, literal.x, KC0[2].W,
+; EG-NEXT:    64(8.968310e-44), 0(0.000000e+00)
+; EG-NEXT:     CNDE_INT T0.X, T1.W, PV.W, 0.0,
+; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
   %shl = shl i64 64, %a
   store i64 %shl, i64 addrspace(1)* %out, align 8
   ret void
 }
 
-; FUNC-LABEL: {{^}}s_shl_inline_imm_1_i64:
-; SI: s_lshl_b64 s{{\[[0-9]+:[0-9]+\]}}, 1, s{{[0-9]+}}
 define amdgpu_kernel void @s_shl_inline_imm_1_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %aptr, i64 %a) {
+; GCN-LABEL: s_shl_inline_imm_1_i64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_load_dword s0, s[0:1], 0xd
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_lshl_b64 s[0:1], 1, s0
+; GCN-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-NEXT:    v_mov_b32_e32 v1, s1
+; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
+;
+; EG-LABEL: s_shl_inline_imm_1_i64:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 8, @4, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+; EG-NEXT:    ALU clause starting at 4:
+; EG-NEXT:     ADD_INT T0.Z, KC0[2].W, literal.x,
+; EG-NEXT:     SETGT_UINT T0.W, KC0[2].W, literal.y,
+; EG-NEXT:     LSHL * T1.W, 1, KC0[2].W,
+; EG-NEXT:    -32(nan), 31(4.344025e-44)
+; EG-NEXT:     CNDE_INT T0.X, PV.W, PS, 0.0,
+; EG-NEXT:     LSHL T1.W, 1, PV.Z,
+; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+; EG-NEXT:     CNDE_INT * T0.Y, T0.W, 0.0, PV.W,
   %shl = shl i64 1, %a
   store i64 %shl, i64 addrspace(1)* %out, align 8
   ret void
 }
 
-; FUNC-LABEL: {{^}}s_shl_inline_imm_1.0_i64:
-; SI: s_lshl_b64 s{{\[[0-9]+:[0-9]+\]}}, 1.0, s{{[0-9]+}}
-define amdgpu_kernel void @s_shl_inline_imm_1.0_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %aptr, i64 %a) {
+define amdgpu_kernel void @s_shl_inline_imm_1_0_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %aptr, i64 %a) {
+; GCN-LABEL: s_shl_inline_imm_1_0_i64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_load_dword s0, s[0:1], 0xd
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_lshl_b64 s[0:1], 1.0, s0
+; GCN-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-NEXT:    v_mov_b32_e32 v1, s1
+; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
+;
+; EG-LABEL: s_shl_inline_imm_1_0_i64:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 6, @4, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+; EG-NEXT:    ALU clause starting at 4:
+; EG-NEXT:     SETGT_UINT T0.W, KC0[2].W, literal.x,
+; EG-NEXT:     LSHL * T1.W, literal.y, KC0[2].W,
+; EG-NEXT:    31(4.344025e-44), 1072693248(1.875000e+00)
+; EG-NEXT:     CNDE_INT * T0.Y, PV.W, PS, 0.0,
+; EG-NEXT:     MOV T0.X, 0.0,
+; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
   %shl = shl i64 4607182418800017408, %a
   store i64 %shl, i64 addrspace(1)* %out, align 8
   ret void
 }
 
-; FUNC-LABEL: {{^}}s_shl_inline_imm_neg_1.0_i64:
-; SI: s_lshl_b64 s{{\[[0-9]+:[0-9]+\]}}, -1.0, s{{[0-9]+}}
-define amdgpu_kernel void @s_shl_inline_imm_neg_1.0_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %aptr, i64 %a) {
+define amdgpu_kernel void @s_shl_inline_imm_neg_1_0_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %aptr, i64 %a) {
+; GCN-LABEL: s_shl_inline_imm_neg_1_0_i64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_load_dword s0, s[0:1], 0xd
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_lshl_b64 s[0:1], -1.0, s0
+; GCN-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-NEXT:    v_mov_b32_e32 v1, s1
+; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
+;
+; EG-LABEL: s_shl_inline_imm_neg_1_0_i64:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 6, @4, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+; EG-NEXT:    ALU clause starting at 4:
+; EG-NEXT:     SETGT_UINT T0.W, KC0[2].W, literal.x,
+; EG-NEXT:     LSHL * T1.W, literal.y, KC0[2].W,
+; EG-NEXT:    31(4.344025e-44), -1074790400(-1.875000e+00)
+; EG-NEXT:     CNDE_INT * T0.Y, PV.W, PS, 0.0,
+; EG-NEXT:     MOV T0.X, 0.0,
+; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
   %shl = shl i64 13830554455654793216, %a
   store i64 %shl, i64 addrspace(1)* %out, align 8
   ret void
 }
 
-; FUNC-LABEL: {{^}}s_shl_inline_imm_0.5_i64:
-; SI: s_lshl_b64 s{{\[[0-9]+:[0-9]+\]}}, 0.5, s{{[0-9]+}}
-define amdgpu_kernel void @s_shl_inline_imm_0.5_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %aptr, i64 %a) {
+define amdgpu_kernel void @s_shl_inline_imm_0_5_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %aptr, i64 %a) {
+; GCN-LABEL: s_shl_inline_imm_0_5_i64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_load_dword s0, s[0:1], 0xd
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_lshl_b64 s[0:1], 0.5, s0
+; GCN-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-NEXT:    v_mov_b32_e32 v1, s1
+; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
+;
+; EG-LABEL: s_shl_inline_imm_0_5_i64:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 6, @4, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+; EG-NEXT:    ALU clause starting at 4:
+; EG-NEXT:     SETGT_UINT T0.W, KC0[2].W, literal.x,
+; EG-NEXT:     LSHL * T1.W, literal.y, KC0[2].W,
+; EG-NEXT:    31(4.344025e-44), 1071644672(1.750000e+00)
+; EG-NEXT:     CNDE_INT * T0.Y, PV.W, PS, 0.0,
+; EG-NEXT:     MOV T0.X, 0.0,
+; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
   %shl = shl i64 4602678819172646912, %a
   store i64 %shl, i64 addrspace(1)* %out, align 8
   ret void
 }
 
-; FUNC-LABEL: {{^}}s_shl_inline_imm_neg_0.5_i64:
-; SI: s_lshl_b64 s{{\[[0-9]+:[0-9]+\]}}, -0.5, s{{[0-9]+}}
-define amdgpu_kernel void @s_shl_inline_imm_neg_0.5_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %aptr, i64 %a) {
+define amdgpu_kernel void @s_shl_inline_imm_neg_0_5_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %aptr, i64 %a) {
+; GCN-LABEL: s_shl_inline_imm_neg_0_5_i64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_load_dword s0, s[0:1], 0xd
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_lshl_b64 s[0:1], -0.5, s0
+; GCN-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-NEXT:    v_mov_b32_e32 v1, s1
+; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
+;
+; EG-LABEL: s_shl_inline_imm_neg_0_5_i64:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 6, @4, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+; EG-NEXT:    ALU clause starting at 4:
+; EG-NEXT:     SETGT_UINT T0.W, KC0[2].W, literal.x,
+; EG-NEXT:     LSHL * T1.W, literal.y, KC0[2].W,
+; EG-NEXT:    31(4.344025e-44), -1075838976(-1.750000e+00)
+; EG-NEXT:     CNDE_INT * T0.Y, PV.W, PS, 0.0,
+; EG-NEXT:     MOV T0.X, 0.0,
+; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
   %shl = shl i64 13826050856027422720, %a
   store i64 %shl, i64 addrspace(1)* %out, align 8
   ret void
 }
 
-; FUNC-LABEL: {{^}}s_shl_inline_imm_2.0_i64:
-; SI: s_lshl_b64 s{{\[[0-9]+:[0-9]+\]}}, 2.0, s{{[0-9]+}}
-define amdgpu_kernel void @s_shl_inline_imm_2.0_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %aptr, i64 %a) {
+define amdgpu_kernel void @s_shl_inline_imm_2_0_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %aptr, i64 %a) {
+; GCN-LABEL: s_shl_inline_imm_2_0_i64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_load_dword s0, s[0:1], 0xd
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_lshl_b64 s[0:1], 2.0, s0
+; GCN-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-NEXT:    v_mov_b32_e32 v1, s1
+; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
+;
+; EG-LABEL: s_shl_inline_imm_2_0_i64:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 6, @4, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+; EG-NEXT:    ALU clause starting at 4:
+; EG-NEXT:     SETGT_UINT T0.W, KC0[2].W, literal.x,
+; EG-NEXT:     LSHL * T1.W, literal.y, KC0[2].W,
+; EG-NEXT:    31(4.344025e-44), 1073741824(2.000000e+00)
+; EG-NEXT:     CNDE_INT * T0.Y, PV.W, PS, 0.0,
+; EG-NEXT:     MOV T0.X, 0.0,
+; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
   %shl = shl i64 4611686018427387904, %a
   store i64 %shl, i64 addrspace(1)* %out, align 8
   ret void
 }
 
-; FUNC-LABEL: {{^}}s_shl_inline_imm_neg_2.0_i64:
-; SI: s_lshl_b64 s{{\[[0-9]+:[0-9]+\]}}, -2.0, s{{[0-9]+}}
-define amdgpu_kernel void @s_shl_inline_imm_neg_2.0_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %aptr, i64 %a) {
+define amdgpu_kernel void @s_shl_inline_imm_neg_2_0_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %aptr, i64 %a) {
+; GCN-LABEL: s_shl_inline_imm_neg_2_0_i64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_load_dword s0, s[0:1], 0xd
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_lshl_b64 s[0:1], -2.0, s0
+; GCN-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-NEXT:    v_mov_b32_e32 v1, s1
+; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
+;
+; EG-LABEL: s_shl_inline_imm_neg_2_0_i64:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 6, @4, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+; EG-NEXT:    ALU clause starting at 4:
+; EG-NEXT:     SETGT_UINT T0.W, KC0[2].W, literal.x,
+; EG-NEXT:     LSHL * T1.W, literal.y, KC0[2].W,
+; EG-NEXT:    31(4.344025e-44), -1073741824(-2.000000e+00)
+; EG-NEXT:     CNDE_INT * T0.Y, PV.W, PS, 0.0,
+; EG-NEXT:     MOV T0.X, 0.0,
+; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
   %shl = shl i64 13835058055282163712, %a
   store i64 %shl, i64 addrspace(1)* %out, align 8
   ret void
 }
 
-; FUNC-LABEL: {{^}}s_shl_inline_imm_4.0_i64:
-; SI: s_lshl_b64 s{{\[[0-9]+:[0-9]+\]}}, 4.0, s{{[0-9]+}}
-define amdgpu_kernel void @s_shl_inline_imm_4.0_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %aptr, i64 %a) {
+define amdgpu_kernel void @s_shl_inline_imm_4_0_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %aptr, i64 %a) {
+; GCN-LABEL: s_shl_inline_imm_4_0_i64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_load_dword s0, s[0:1], 0xd
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_lshl_b64 s[0:1], 4.0, s0
+; GCN-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-NEXT:    v_mov_b32_e32 v1, s1
+; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
+;
+; EG-LABEL: s_shl_inline_imm_4_0_i64:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 6, @4, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+; EG-NEXT:    ALU clause starting at 4:
+; EG-NEXT:     SETGT_UINT T0.W, KC0[2].W, literal.x,
+; EG-NEXT:     LSHL * T1.W, literal.y, KC0[2].W,
+; EG-NEXT:    31(4.344025e-44), 1074790400(2.250000e+00)
+; EG-NEXT:     CNDE_INT * T0.Y, PV.W, PS, 0.0,
+; EG-NEXT:     MOV T0.X, 0.0,
+; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
   %shl = shl i64 4616189618054758400, %a
   store i64 %shl, i64 addrspace(1)* %out, align 8
   ret void
 }
 
-; FUNC-LABEL: {{^}}s_shl_inline_imm_neg_4.0_i64:
-; SI: s_lshl_b64 s{{\[[0-9]+:[0-9]+\]}}, -4.0, s{{[0-9]+}}
-define amdgpu_kernel void @s_shl_inline_imm_neg_4.0_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %aptr, i64 %a) {
+define amdgpu_kernel void @s_shl_inline_imm_neg_4_0_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %aptr, i64 %a) {
+; GCN-LABEL: s_shl_inline_imm_neg_4_0_i64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_load_dword s0, s[0:1], 0xd
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_lshl_b64 s[0:1], -4.0, s0
+; GCN-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-NEXT:    v_mov_b32_e32 v1, s1
+; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
+;
+; EG-LABEL: s_shl_inline_imm_neg_4_0_i64:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 6, @4, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+; EG-NEXT:    ALU clause starting at 4:
+; EG-NEXT:     SETGT_UINT T0.W, KC0[2].W, literal.x,
+; EG-NEXT:     LSHL * T1.W, literal.y, KC0[2].W,
+; EG-NEXT:    31(4.344025e-44), -1072693248(-2.250000e+00)
+; EG-NEXT:     CNDE_INT * T0.Y, PV.W, PS, 0.0,
+; EG-NEXT:     MOV T0.X, 0.0,
+; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
   %shl = shl i64 13839561654909534208, %a
   store i64 %shl, i64 addrspace(1)* %out, align 8
   ret void
@@ -424,75 +1470,253 @@ define amdgpu_kernel void @s_shl_inline_imm_neg_4.0_i64(i64 addrspace(1)* %out,
 
 ; Test with the 64-bit integer bitpattern for a 32-bit float in the
 ; low 32-bits, which is not a valid 64-bit inline immmediate.
-
-; FUNC-LABEL: {{^}}s_shl_inline_imm_f32_4.0_i64:
-; SI-DAG: s_mov_b32 s[[K_LO:[0-9]+]], 4.0
-; SI-DAG: s_mov_b32 s[[K_HI:[0-9]+]], 0{{$}}
-; SI: s_lshl_b64 s{{\[[0-9]+:[0-9]+\]}}, s{{\[}}[[K_LO]]:[[K_HI]]{{\]}}, s{{[0-9]+}}
-define amdgpu_kernel void @s_shl_inline_imm_f32_4.0_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %aptr, i64 %a) {
+define amdgpu_kernel void @s_shl_inline_imm_f32_4_0_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %aptr, i64 %a) {
+; GCN-LABEL: s_shl_inline_imm_f32_4_0_i64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_load_dword s2, s[0:1], 0xd
+; GCN-NEXT:    s_mov_b32 s1, 0
+; GCN-NEXT:    s_mov_b32 s0, 4.0
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_lshl_b64 s[0:1], s[0:1], s2
+; GCN-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-NEXT:    v_mov_b32_e32 v1, s1
+; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
+;
+; EG-LABEL: s_shl_inline_imm_f32_4_0_i64:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 14, @4, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+; EG-NEXT:    ALU clause starting at 4:
+; EG-NEXT:     SUB_INT * T0.W, literal.x, KC0[2].W,
+; EG-NEXT:    31(4.344025e-44), 0(0.000000e+00)
+; EG-NEXT:     LSHR T0.W, literal.x, PV.W,
+; EG-NEXT:     ADD_INT * T1.W, KC0[2].W, literal.y,
+; EG-NEXT:    1082130432(4.000000e+00), -32(nan)
+; EG-NEXT:     LSHL T0.Z, literal.x, PS,
+; EG-NEXT:     LSHR T0.W, PV.W, 1,
+; EG-NEXT:     SETGT_UINT * T1.W, KC0[2].W, literal.y,
+; EG-NEXT:    1082130432(4.000000e+00), 31(4.344025e-44)
+; EG-NEXT:     CNDE_INT T0.Y, PS, PV.W, PV.Z,
+; EG-NEXT:     LSHL * T0.W, literal.x, KC0[2].W,
+; EG-NEXT:    1082130432(4.000000e+00), 0(0.000000e+00)
+; EG-NEXT:     CNDE_INT T0.X, T1.W, PV.W, 0.0,
+; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
   %shl = shl i64 1082130432, %a
   store i64 %shl, i64 addrspace(1)* %out, align 8
   ret void
 }
 
 ; FIXME: Copy of -1 register
-; FUNC-LABEL: {{^}}s_shl_inline_imm_f32_neg_4.0_i64:
-; SI-DAG: s_mov_b32 s[[K_LO:[0-9]+]], -4.0
-; SI-DAG: s_mov_b32 s[[K_HI:[0-9]+]], -1{{$}}
-; SI-DAG: s_mov_b32 s[[K_HI_COPY:[0-9]+]], s[[K_HI]]
-; SI: s_lshl_b64 s{{\[[0-9]+:[0-9]+\]}}, s{{\[}}[[K_LO]]:[[K_HI_COPY]]{{\]}}, s{{[0-9]+}}
-define amdgpu_kernel void @s_shl_inline_imm_f32_neg_4.0_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %aptr, i64 %a) {
+define amdgpu_kernel void @s_shl_inline_imm_f32_neg_4_0_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %aptr, i64 %a) {
+; GCN-LABEL: s_shl_inline_imm_f32_neg_4_0_i64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_load_dword s2, s[0:1], 0xd
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_mov_b32 s0, -4.0
+; GCN-NEXT:    s_mov_b32 s1, s6
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_lshl_b64 s[0:1], s[0:1], s2
+; GCN-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-NEXT:    v_mov_b32_e32 v1, s1
+; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
+;
+; EG-LABEL: s_shl_inline_imm_f32_neg_4_0_i64:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 17, @4, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+; EG-NEXT:    ALU clause starting at 4:
+; EG-NEXT:     SUB_INT * T0.W, literal.x, KC0[2].W,
+; EG-NEXT:    31(4.344025e-44), 0(0.000000e+00)
+; EG-NEXT:     LSHR * T0.W, literal.x, PV.W,
+; EG-NEXT:    -1065353216(-4.000000e+00), 0(0.000000e+00)
+; EG-NEXT:     ADD_INT T0.Z, KC0[2].W, literal.x,
+; EG-NEXT:     LSHR T0.W, PV.W, 1,
+; EG-NEXT:     LSHL * T1.W, literal.y, KC0[2].W,
+; EG-NEXT:    -32(nan), -1(nan)
+; EG-NEXT:     OR_INT T1.Z, PS, PV.W,
+; EG-NEXT:     LSHL T0.W, literal.x, PV.Z,
+; EG-NEXT:     SETGT_UINT * T1.W, KC0[2].W, literal.y,
+; EG-NEXT:    -1065353216(-4.000000e+00), 31(4.344025e-44)
+; EG-NEXT:     CNDE_INT T0.Y, PS, PV.Z, PV.W,
+; EG-NEXT:     LSHL * T0.W, literal.x, KC0[2].W,
+; EG-NEXT:    -1065353216(-4.000000e+00), 0(0.000000e+00)
+; EG-NEXT:     CNDE_INT T0.X, T1.W, PV.W, 0.0,
+; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
   %shl = shl i64 -1065353216, %a
   store i64 %shl, i64 addrspace(1)* %out, align 8
   ret void
 }
 
-; Shift into upper 32-bits
-; FUNC-LABEL: {{^}}s_shl_inline_high_imm_f32_4.0_i64:
-; SI-DAG: s_mov_b32 s[[K_HI:[0-9]+]], 4.0
-; SI-DAG: s_mov_b32 s[[K_LO:[0-9]+]], 0{{$}}
-; SI: s_lshl_b64 s{{\[[0-9]+:[0-9]+\]}}, s{{\[}}[[K_LO]]:[[K_HI]]{{\]}}, s{{[0-9]+}}
-define amdgpu_kernel void @s_shl_inline_high_imm_f32_4.0_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %aptr, i64 %a) {
+define amdgpu_kernel void @s_shl_inline_high_imm_f32_4_0_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %aptr, i64 %a) {
+; GCN-LABEL: s_shl_inline_high_imm_f32_4_0_i64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_load_dword s2, s[0:1], 0xd
+; GCN-NEXT:    s_mov_b32 s1, 4.0
+; GCN-NEXT:    s_mov_b32 s0, 0
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_lshl_b64 s[0:1], s[0:1], s2
+; GCN-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-NEXT:    v_mov_b32_e32 v1, s1
+; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
+;
+; EG-LABEL: s_shl_inline_high_imm_f32_4_0_i64:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 6, @4, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+; EG-NEXT:    ALU clause starting at 4:
+; EG-NEXT:     SETGT_UINT T0.W, KC0[2].W, literal.x,
+; EG-NEXT:     LSHL * T1.W, literal.y, KC0[2].W,
+; EG-NEXT:    31(4.344025e-44), 1082130432(4.000000e+00)
+; EG-NEXT:     CNDE_INT * T0.Y, PV.W, PS, 0.0,
+; EG-NEXT:     MOV T0.X, 0.0,
+; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
   %shl = shl i64 4647714815446351872, %a
   store i64 %shl, i64 addrspace(1)* %out, align 8
   ret void
 }
 
-; FUNC-LABEL: {{^}}s_shl_inline_high_imm_f32_neg_4.0_i64:
-; SI-DAG: s_mov_b32 s[[K_HI:[0-9]+]], -4.0
-; SI-DAG: s_mov_b32 s[[K_LO:[0-9]+]], 0{{$}}
-; SI: s_lshl_b64 s{{\[[0-9]+:[0-9]+\]}}, s{{\[}}[[K_LO]]:[[K_HI]]{{\]}}, s{{[0-9]+}}
-define amdgpu_kernel void @s_shl_inline_high_imm_f32_neg_4.0_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %aptr, i64 %a) {
+define amdgpu_kernel void @s_shl_inline_high_imm_f32_neg_4_0_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %aptr, i64 %a) {
+; GCN-LABEL: s_shl_inline_high_imm_f32_neg_4_0_i64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_load_dword s2, s[0:1], 0xd
+; GCN-NEXT:    s_mov_b32 s1, -4.0
+; GCN-NEXT:    s_mov_b32 s0, 0
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_lshl_b64 s[0:1], s[0:1], s2
+; GCN-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-NEXT:    v_mov_b32_e32 v1, s1
+; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
+;
+; EG-LABEL: s_shl_inline_high_imm_f32_neg_4_0_i64:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 6, @4, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+; EG-NEXT:    ALU clause starting at 4:
+; EG-NEXT:     SETGT_UINT T0.W, KC0[2].W, literal.x,
+; EG-NEXT:     LSHL * T1.W, literal.y, KC0[2].W,
+; EG-NEXT:    31(4.344025e-44), -1065353216(-4.000000e+00)
+; EG-NEXT:     CNDE_INT * T0.Y, PV.W, PS, 0.0,
+; EG-NEXT:     MOV T0.X, 0.0,
+; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
   %shl = shl i64 13871086852301127680, %a
   store i64 %shl, i64 addrspace(1)* %out, align 8
   ret void
 }
 
-; FUNC-LABEL: {{^}}test_mul2:
-; GCN: s_lshl_b32 s{{[0-9]}}, s{{[0-9]}}, 1
 define amdgpu_kernel void @test_mul2(i32 %p) {
+; GCN-LABEL: test_mul2:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dword s0, s[0:1], 0x9
+; GCN-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NEXT:    s_mov_b32 s2, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_lshl_b32 s0, s0, 1
+; GCN-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GCN-NEXT:    s_endpgm
+;
+; EG-LABEL: test_mul2:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 2, @4, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T1.X, T0.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+; EG-NEXT:    ALU clause starting at 4:
+; EG-NEXT:     MOV T0.X, literal.x,
+; EG-NEXT:     LSHL * T1.X, KC0[2].Y, 1,
+; EG-NEXT:    0(0.000000e+00), 0(0.000000e+00)
    %i = mul i32 %p, 2
    store volatile i32 %i, i32 addrspace(1)* undef
    ret void
 }
 
-; FUNC-LABEL: {{^}}shl_or_k:
-; SI: v_lshlrev_b32_e32 [[SHL:v[0-9]+]], 2, v{{[0-9]+}}
-; SI: v_or_b32_e32 [[OR:v[0-9]+]], 4, [[SHL]]
-; SI: buffer_store_dword [[OR]]
 define void @shl_or_k(i32 addrspace(1)* %out, i32 %in) {
+; GCN-LABEL: shl_or_k:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    s_mov_b32 s6, 0
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 2, v2
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s4, s6
+; GCN-NEXT:    s_mov_b32 s5, s6
+; GCN-NEXT:    v_or_b32_e32 v2, 4, v2
+; GCN-NEXT:    buffer_store_dword v2, v[0:1], s[4:7], 0 addr64
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; EG-LABEL: shl_or_k:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 4, @4, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.X, T1.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+; EG-NEXT:    ALU clause starting at 4:
+; EG-NEXT:     LSHL * T0.W, KC0[2].Z, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+; EG-NEXT:     OR_INT T0.X, PV.W, literal.x,
+; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.y,
+; EG-NEXT:    4(5.605194e-45), 2(2.802597e-45)
   %tmp0 = or i32 %in, 1
   %tmp2 = shl i32 %tmp0, 2
   store i32 %tmp2, i32 addrspace(1)* %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}shl_or_k_two_uses:
-; SI: v_or_b32_e32 [[OR:v[0-9]+]], 1, v{{[0-9]+}}
-; SI: v_lshlrev_b32_e32 [[SHL:v[0-9]+]], 2, [[OR]]
-; SI-DAG: buffer_store_dword [[OR]]
-; SI-DAG: buffer_store_dword [[SHL]]
 define void @shl_or_k_two_uses(i32 addrspace(1)* %out0, i32 addrspace(1)* %out1, i32 %in) {
+; GCN-LABEL: shl_or_k_two_uses:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    s_mov_b32 s6, 0
+; GCN-NEXT:    v_or_b32_e32 v4, 1, v4
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s4, s6
+; GCN-NEXT:    s_mov_b32 s5, s6
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 2, v4
+; GCN-NEXT:    buffer_store_dword v5, v[0:1], s[4:7], 0 addr64
+; GCN-NEXT:    buffer_store_dword v4, v[2:3], s[4:7], 0 addr64
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; EG-LABEL: shl_or_k_two_uses:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 5, @4, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T2.X, T3.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T1.X, T0.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    ALU clause starting at 4:
+; EG-NEXT:     LSHR T0.X, KC0[2].Z, literal.x,
+; EG-NEXT:     OR_INT * T1.X, KC0[2].W, 1,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+; EG-NEXT:     LSHL T2.X, PS, literal.x,
+; EG-NEXT:     LSHR * T3.X, KC0[2].Y, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
   %tmp0 = or i32 %in, 1
   %tmp2 = shl i32 %tmp0, 2
   store i32 %tmp2, i32 addrspace(1)* %out0
diff --git a/llvm/test/CodeGen/AMDGPU/shl.v2i16.ll b/llvm/test/CodeGen/AMDGPU/shl.v2i16.ll
index b116e2ee6..7968aa4c0 100644
--- a/llvm/test/CodeGen/AMDGPU/shl.v2i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/shl.v2i16.ll
@@ -1,60 +1,135 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9 %s
-; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,VI,CIVI %s
-; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CI,CIVI %s
+; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CIVI,VI %s
+; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CIVI,CI %s
 
-; GCN-LABEL: {{^}}s_shl_v2i16:
-; GFX9: s_load_dword [[LHS:s[0-9]+]]
-; GFX9: s_load_dword [[RHS:s[0-9]+]]
-; GFX9: v_mov_b32_e32 [[VLHS:v[0-9]+]], [[LHS]]
-; GFX9: v_pk_lshlrev_b16 [[RESULT:v[0-9]+]], [[RHS]], [[VLHS]]
-
-; VI: s_load_dword s
-; VI: s_load_dword s
-; VI: s_lshr_b32
-; VI: s_lshr_b32
-; VI: s_and_b32
-; VI: s_and_b32
-; VI: s_lshl_b32
-; VI: s_lshl_b32
-; VI: s_lshl_b32
-; VI: s_and_b32
-; VI: s_or_b32
-
-; CI: s_load_dword s
-; CI: s_load_dword s
-; CI: s_lshr_b32
-; CI: s_and_b32
-; CI: s_lshr_b32
-; CI: s_lshl_b32
-; CI: s_lshl_b32
-; CI: s_lshl_b32
-; CI: s_and_b32
-; CI: s_or_b32
-; CI: _store_dword
 define amdgpu_kernel void @s_shl_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> %lhs, <2 x i16> %rhs) #0 {
+; GFX9-LABEL: s_shl_v2i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
+; GFX9-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; GFX9-NEXT:    s_load_dword s0, s[0:1], 0x30
+; GFX9-NEXT:    s_mov_b32 s7, 0xf000
+; GFX9-NEXT:    s_mov_b32 s6, -1
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, s2
+; GFX9-NEXT:    v_pk_lshlrev_b16 v0, s0, v0
+; GFX9-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX9-NEXT:    s_endpgm
+;
+; VI-LABEL: s_shl_v2i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
+; VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-NEXT:    s_load_dword s0, s[0:1], 0x30
+; VI-NEXT:    s_mov_b32 s3, 0xffff
+; VI-NEXT:    s_mov_b32 s7, 0xf000
+; VI-NEXT:    s_mov_b32 s6, -1
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_lshr_b32 s1, s2, 16
+; VI-NEXT:    s_lshr_b32 s8, s0, 16
+; VI-NEXT:    s_and_b32 s2, s2, s3
+; VI-NEXT:    s_and_b32 s0, s0, s3
+; VI-NEXT:    s_lshl_b32 s0, s2, s0
+; VI-NEXT:    s_lshl_b32 s1, s1, s8
+; VI-NEXT:    s_lshl_b32 s1, s1, 16
+; VI-NEXT:    s_and_b32 s0, s0, s3
+; VI-NEXT:    s_or_b32 s0, s0, s1
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; VI-NEXT:    s_endpgm
+;
+; CI-LABEL: s_shl_v2i16:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; CI-NEXT:    s_load_dword s2, s[0:1], 0xb
+; CI-NEXT:    s_load_dword s0, s[0:1], 0xc
+; CI-NEXT:    s_mov_b32 s3, 0xffff
+; CI-NEXT:    s_mov_b32 s7, 0xf000
+; CI-NEXT:    s_mov_b32 s6, -1
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    s_lshr_b32 s1, s2, 16
+; CI-NEXT:    s_and_b32 s8, s0, s3
+; CI-NEXT:    s_lshr_b32 s0, s0, 16
+; CI-NEXT:    s_lshl_b32 s0, s1, s0
+; CI-NEXT:    s_lshl_b32 s1, s2, s8
+; CI-NEXT:    s_lshl_b32 s0, s0, 16
+; CI-NEXT:    s_and_b32 s1, s1, s3
+; CI-NEXT:    s_or_b32 s0, s1, s0
+; CI-NEXT:    v_mov_b32_e32 v0, s0
+; CI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; CI-NEXT:    s_endpgm
   %result = shl <2 x i16> %lhs, %rhs
   store <2 x i16> %result, <2 x i16> addrspace(1)* %out
   ret void
 }
 
-; GCN-LABEL: {{^}}v_shl_v2i16:
-; GCN: {{buffer|flat|global}}_load_dword [[LHS:v[0-9]+]]
-; GCN: {{buffer|flat|global}}_load_dword [[RHS:v[0-9]+]]
-; GFX9: v_pk_lshlrev_b16 [[RESULT:v[0-9]+]], [[RHS]], [[LHS]]
-
-; VI: v_lshlrev_b16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; VI: v_lshlrev_b16_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; VI: v_or_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-
-; CI: s_mov_b32 [[MASK:s[0-9]+]], 0xffff{{$}}
-; CI: v_lshrrev_b32_e32 v{{[0-9]+}}, 16, [[LHS]]
-; CI: v_lshrrev_b32_e32 v{{[0-9]+}}, 16, v{{[0-9]+}}
-; CI: v_lshlrev_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; CI: v_lshlrev_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; CI: v_lshlrev_b32_e32 v{{[0-9]+}}, 16, v{{[0-9]+}}
-; CI: v_and_b32_e32 v{{[0-9]+}}, [[MASK]], v{{[0-9]+}}
-; CI: v_or_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 define amdgpu_kernel void @v_shl_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %in) #0 {
+; GFX9-LABEL: v_shl_v2i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s2, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    global_load_dword v3, v[0:1], off
+; GFX9-NEXT:    global_load_dword v4, v[0:1], off offset:4
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s0, v2
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_pk_lshlrev_b16 v2, v4, v3
+; GFX9-NEXT:    global_store_dword v[0:1], v2, off
+; GFX9-NEXT:    s_endpgm
+;
+; VI-LABEL: v_shl_v2i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-NEXT:    v_lshlrev_b32_e32 v4, 2, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v4
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 4, v0
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dword v5, v[0:1]
+; VI-NEXT:    flat_load_dword v2, v[2:3]
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v4
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_lshlrev_b16_e32 v3, v2, v5
+; VI-NEXT:    v_lshlrev_b16_sdwa v2, v2, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; VI-NEXT:    v_or_b32_e32 v2, v3, v2
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; CI-LABEL: v_shl_v2i16:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; CI-NEXT:    s_mov_b32 s7, 0xf000
+; CI-NEXT:    s_mov_b32 s6, 0
+; CI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; CI-NEXT:    v_mov_b32_e32 v1, 0
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; CI-NEXT:    buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
+; CI-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:4
+; CI-NEXT:    s_mov_b32 s8, 0xffff
+; CI-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; CI-NEXT:    s_waitcnt vmcnt(1)
+; CI-NEXT:    v_lshrrev_b32_e32 v4, 16, v2
+; CI-NEXT:    s_waitcnt vmcnt(0)
+; CI-NEXT:    v_and_b32_e32 v5, s8, v3
+; CI-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
+; CI-NEXT:    v_lshlrev_b32_e32 v3, v3, v4
+; CI-NEXT:    v_lshlrev_b32_e32 v2, v5, v2
+; CI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; CI-NEXT:    v_and_b32_e32 v2, s8, v2
+; CI-NEXT:    v_or_b32_e32 v2, v2, v3
+; CI-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; CI-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
   %in.gep = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %in, i64 %tid.ext
@@ -67,11 +142,71 @@ define amdgpu_kernel void @v_shl_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> a
   ret void
 }
 
-; GCN-LABEL: {{^}}shl_v_s_v2i16:
-; GFX9: s_load_dword [[RHS:s[0-9]+]]
-; GFX9: {{buffer|flat|global}}_load_dword [[LHS:v[0-9]+]]
-; GFX9: v_pk_lshlrev_b16 [[RESULT:v[0-9]+]], [[RHS]], [[LHS]]
 define amdgpu_kernel void @shl_v_s_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %in, <2 x i16> %sgpr) #0 {
+; GFX9-LABEL: shl_v_s_v2i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX9-NEXT:    s_load_dword s0, s[0:1], 0x34
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v1, s7
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s6, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    global_load_dword v3, v[0:1], off
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s4, v2
+; GFX9-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_pk_lshlrev_b16 v2, s0, v3
+; GFX9-NEXT:    global_store_dword v[0:1], v2, off
+; GFX9-NEXT:    s_endpgm
+;
+; VI-LABEL: shl_v_s_v2i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; VI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; VI-NEXT:    s_load_dword s0, s[0:1], 0x34
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s7
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s6, v2
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dword v3, v[0:1]
+; VI-NEXT:    s_lshr_b32 s1, s0, 16
+; VI-NEXT:    v_mov_b32_e32 v4, s1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s4, v2
+; VI-NEXT:    v_mov_b32_e32 v1, s5
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_lshlrev_b16_e32 v2, s0, v3
+; VI-NEXT:    v_lshlrev_b16_sdwa v3, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_or_b32_e32 v2, v2, v3
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; CI-LABEL: shl_v_s_v2i16:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; CI-NEXT:    s_load_dword s8, s[0:1], 0xd
+; CI-NEXT:    s_mov_b32 s3, 0xf000
+; CI-NEXT:    s_mov_b32 s2, 0
+; CI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    s_mov_b64 s[0:1], s[6:7]
+; CI-NEXT:    v_mov_b32_e32 v1, 0
+; CI-NEXT:    buffer_load_dword v2, v[0:1], s[0:3], 0 addr64
+; CI-NEXT:    s_mov_b32 s9, 0xffff
+; CI-NEXT:    s_lshr_b32 s10, s8, 16
+; CI-NEXT:    s_and_b32 s8, s8, s9
+; CI-NEXT:    s_mov_b64 s[6:7], s[2:3]
+; CI-NEXT:    s_waitcnt vmcnt(0)
+; CI-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
+; CI-NEXT:    v_lshlrev_b32_e32 v2, s8, v2
+; CI-NEXT:    v_lshlrev_b32_e32 v3, s10, v3
+; CI-NEXT:    v_and_b32_e32 v2, s9, v2
+; CI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; CI-NEXT:    v_or_b32_e32 v2, v2, v3
+; CI-NEXT:    buffer_store_dword v2, v[0:1], s[4:7], 0 addr64
+; CI-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
   %in.gep = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %in, i64 %tid.ext
@@ -82,11 +217,71 @@ define amdgpu_kernel void @shl_v_s_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16>
   ret void
 }
 
-; GCN-LABEL: {{^}}shl_s_v_v2i16:
-; GFX9: s_load_dword [[LHS:s[0-9]+]]
-; GFX9: {{buffer|flat|global}}_load_dword [[RHS:v[0-9]+]]
-; GFX9: v_pk_lshlrev_b16 [[RESULT:v[0-9]+]], [[RHS]], [[LHS]]
 define amdgpu_kernel void @shl_s_v_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %in, <2 x i16> %sgpr) #0 {
+; GFX9-LABEL: shl_s_v_v2i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX9-NEXT:    s_load_dword s0, s[0:1], 0x34
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v1, s7
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s6, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    global_load_dword v3, v[0:1], off
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s4, v2
+; GFX9-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_pk_lshlrev_b16 v2, v3, s0
+; GFX9-NEXT:    global_store_dword v[0:1], v2, off
+; GFX9-NEXT:    s_endpgm
+;
+; VI-LABEL: shl_s_v_v2i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; VI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; VI-NEXT:    s_load_dword s0, s[0:1], 0x34
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s7
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s6, v2
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dword v3, v[0:1]
+; VI-NEXT:    s_lshr_b32 s1, s0, 16
+; VI-NEXT:    v_mov_b32_e32 v4, s1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s4, v2
+; VI-NEXT:    v_mov_b32_e32 v1, s5
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_lshlrev_b16_e64 v2, v3, s0
+; VI-NEXT:    v_lshlrev_b16_sdwa v3, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v2, v2, v3
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; CI-LABEL: shl_s_v_v2i16:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; CI-NEXT:    s_load_dword s8, s[0:1], 0xd
+; CI-NEXT:    s_mov_b32 s3, 0xf000
+; CI-NEXT:    s_mov_b32 s2, 0
+; CI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    s_mov_b64 s[0:1], s[6:7]
+; CI-NEXT:    v_mov_b32_e32 v1, 0
+; CI-NEXT:    buffer_load_dword v2, v[0:1], s[0:3], 0 addr64
+; CI-NEXT:    s_mov_b32 s0, 0xffff
+; CI-NEXT:    s_lshr_b32 s1, s8, 16
+; CI-NEXT:    s_mov_b64 s[6:7], s[2:3]
+; CI-NEXT:    s_waitcnt vmcnt(0)
+; CI-NEXT:    v_and_b32_e32 v3, s0, v2
+; CI-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
+; CI-NEXT:    v_lshl_b32_e32 v2, s1, v2
+; CI-NEXT:    v_lshl_b32_e32 v3, s8, v3
+; CI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; CI-NEXT:    v_and_b32_e32 v3, s0, v3
+; CI-NEXT:    v_or_b32_e32 v2, v3, v2
+; CI-NEXT:    buffer_store_dword v2, v[0:1], s[4:7], 0 addr64
+; CI-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
   %in.gep = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %in, i64 %tid.ext
@@ -97,10 +292,65 @@ define amdgpu_kernel void @shl_s_v_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16>
   ret void
 }
 
-; GCN-LABEL: {{^}}shl_imm_v_v2i16:
-; GCN: {{buffer|flat|global}}_load_dword [[RHS:v[0-9]+]]
-; GFX9: v_pk_lshlrev_b16 [[RESULT:v[0-9]+]], [[RHS]], 8
 define amdgpu_kernel void @shl_imm_v_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %in) #0 {
+; GFX9-LABEL: shl_imm_v_v2i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s2, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    global_load_dword v3, v[0:1], off
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s0, v2
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_pk_lshlrev_b16 v2, v3, 8 op_sel_hi:[1,0]
+; GFX9-NEXT:    global_store_dword v[0:1], v2, off
+; GFX9-NEXT:    s_endpgm
+;
+; VI-LABEL: shl_imm_v_v2i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; VI-NEXT:    v_mov_b32_e32 v3, 8
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dword v4, v[0:1]
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v2
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_lshlrev_b16_e64 v2, v4, 8
+; VI-NEXT:    v_lshlrev_b16_sdwa v3, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v2, v2, v3
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; CI-LABEL: shl_imm_v_v2i16:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; CI-NEXT:    s_mov_b32 s7, 0xf000
+; CI-NEXT:    s_mov_b32 s6, 0
+; CI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; CI-NEXT:    v_mov_b32_e32 v1, 0
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; CI-NEXT:    buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
+; CI-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; CI-NEXT:    s_waitcnt vmcnt(0)
+; CI-NEXT:    v_and_b32_e32 v3, 0xffff, v2
+; CI-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
+; CI-NEXT:    v_lshl_b32_e32 v2, 8, v2
+; CI-NEXT:    v_lshl_b32_e32 v3, 8, v3
+; CI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; CI-NEXT:    v_and_b32_e32 v3, 0xfff8, v3
+; CI-NEXT:    v_or_b32_e32 v2, v3, v2
+; CI-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; CI-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
   %in.gep = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %in, i64 %tid.ext
@@ -111,10 +361,60 @@ define amdgpu_kernel void @shl_imm_v_v2i16(<2 x i16> addrspace(1)* %out, <2 x i1
   ret void
 }
 
-; GCN-LABEL: {{^}}shl_v_imm_v2i16:
-; GCN: {{buffer|flat|global}}_load_dword [[LHS:v[0-9]+]]
-; GFX9: v_pk_lshlrev_b16 [[RESULT:v[0-9]+]], 8, [[LHS]]
 define amdgpu_kernel void @shl_v_imm_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %in) #0 {
+; GFX9-LABEL: shl_v_imm_v2i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s2, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    global_load_dword v3, v[0:1], off
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s0, v2
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_pk_lshlrev_b16 v2, 8, v3 op_sel_hi:[0,1]
+; GFX9-NEXT:    global_store_dword v[0:1], v2, off
+; GFX9-NEXT:    s_endpgm
+;
+; VI-LABEL: shl_v_imm_v2i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dword v3, v[0:1]
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v2
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_lshlrev_b32_e32 v2, 8, v3
+; VI-NEXT:    v_and_b32_e32 v2, 0xff000000, v2
+; VI-NEXT:    v_lshlrev_b16_e32 v3, 8, v3
+; VI-NEXT:    v_or_b32_e32 v2, v3, v2
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; CI-LABEL: shl_v_imm_v2i16:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; CI-NEXT:    s_mov_b32 s7, 0xf000
+; CI-NEXT:    s_mov_b32 s6, 0
+; CI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; CI-NEXT:    v_mov_b32_e32 v1, 0
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; CI-NEXT:    buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
+; CI-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; CI-NEXT:    s_waitcnt vmcnt(0)
+; CI-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
+; CI-NEXT:    v_and_b32_e32 v2, 0xff00ff00, v2
+; CI-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; CI-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
   %in.gep = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %in, i64 %tid.ext
@@ -125,13 +425,84 @@ define amdgpu_kernel void @shl_v_imm_v2i16(<2 x i16> addrspace(1)* %out, <2 x i1
   ret void
 }
 
-; GCN-LABEL: {{^}}v_shl_v4i16:
-; GCN: {{buffer|flat|global}}_load_dwordx2
-; GCN: {{buffer|flat|global}}_load_dwordx2
-; GFX9: v_pk_lshlrev_b16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; GFX9: v_pk_lshlrev_b16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; GCN: {{buffer|flat|global}}_store_dwordx2
 define amdgpu_kernel void @v_shl_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> addrspace(1)* %in) #0 {
+; GFX9-LABEL: v_shl_v4i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-NEXT:    v_lshlrev_b32_e32 v4, 3, v0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s2, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    global_load_dwordx2 v[2:3], v[0:1], off
+; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off offset:8
+; GFX9-NEXT:    v_mov_b32_e32 v5, s1
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, s0, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_pk_lshlrev_b16 v1, v1, v3
+; GFX9-NEXT:    v_pk_lshlrev_b16 v0, v0, v2
+; GFX9-NEXT:    global_store_dwordx2 v[4:5], v[0:1], off
+; GFX9-NEXT:    s_endpgm
+;
+; VI-LABEL: v_shl_v4i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-NEXT:    v_lshlrev_b32_e32 v4, 3, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v4
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 8, v0
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; VI-NEXT:    flat_load_dwordx2 v[2:3], v[2:3]
+; VI-NEXT:    v_mov_b32_e32 v5, s1
+; VI-NEXT:    v_add_u32_e32 v4, vcc, s0, v4
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_lshlrev_b16_e32 v6, v3, v1
+; VI-NEXT:    v_lshlrev_b16_sdwa v1, v3, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b16_e32 v3, v2, v0
+; VI-NEXT:    v_lshlrev_b16_sdwa v0, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; VI-NEXT:    v_or_b32_e32 v1, v6, v1
+; VI-NEXT:    v_or_b32_e32 v0, v3, v0
+; VI-NEXT:    flat_store_dwordx2 v[4:5], v[0:1]
+; VI-NEXT:    s_endpgm
+;
+; CI-LABEL: v_shl_v4i16:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; CI-NEXT:    s_mov_b32 s7, 0xf000
+; CI-NEXT:    s_mov_b32 s6, 0
+; CI-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
+; CI-NEXT:    v_mov_b32_e32 v1, 0
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; CI-NEXT:    buffer_load_dwordx2 v[2:3], v[0:1], s[4:7], 0 addr64
+; CI-NEXT:    buffer_load_dwordx2 v[4:5], v[0:1], s[4:7], 0 addr64 offset:8
+; CI-NEXT:    s_mov_b32 s8, 0xffff
+; CI-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; CI-NEXT:    s_waitcnt vmcnt(1)
+; CI-NEXT:    v_lshrrev_b32_e32 v6, 16, v2
+; CI-NEXT:    s_waitcnt vmcnt(0)
+; CI-NEXT:    v_and_b32_e32 v8, s8, v4
+; CI-NEXT:    v_lshrrev_b32_e32 v4, 16, v4
+; CI-NEXT:    v_and_b32_e32 v9, s8, v5
+; CI-NEXT:    v_lshrrev_b32_e32 v7, 16, v3
+; CI-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
+; CI-NEXT:    v_lshlrev_b32_e32 v5, v5, v7
+; CI-NEXT:    v_lshlrev_b32_e32 v3, v9, v3
+; CI-NEXT:    v_lshlrev_b32_e32 v4, v4, v6
+; CI-NEXT:    v_lshlrev_b32_e32 v2, v8, v2
+; CI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; CI-NEXT:    v_and_b32_e32 v3, s8, v3
+; CI-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; CI-NEXT:    v_and_b32_e32 v2, s8, v2
+; CI-NEXT:    v_or_b32_e32 v3, v3, v5
+; CI-NEXT:    v_or_b32_e32 v2, v2, v4
+; CI-NEXT:    buffer_store_dwordx2 v[2:3], v[0:1], s[0:3], 0 addr64
+; CI-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
   %in.gep = getelementptr inbounds <4 x i16>, <4 x i16> addrspace(1)* %in, i64 %tid.ext
@@ -144,12 +515,73 @@ define amdgpu_kernel void @v_shl_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> a
   ret void
 }
 
-; GCN-LABEL: {{^}}shl_v_imm_v4i16:
-; GCN: {{buffer|flat|global}}_load_dwordx2
-; GFX9: v_pk_lshlrev_b16 v{{[0-9]+}}, 8, v{{[0-9]+}}
-; GFX9: v_pk_lshlrev_b16 v{{[0-9]+}}, 8, v{{[0-9]+}}
-; GCN: {{buffer|flat|global}}_store_dwordx2
 define amdgpu_kernel void @shl_v_imm_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> addrspace(1)* %in) #0 {
+; GFX9-LABEL: shl_v_imm_v4i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s2, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off
+; GFX9-NEXT:    v_mov_b32_e32 v3, s1
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, s0, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
+; GFX9-NEXT:    v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX9-NEXT:    global_store_dwordx2 v[2:3], v[0:1], off
+; GFX9-NEXT:    s_endpgm
+;
+; VI-LABEL: shl_v_imm_v4i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; VI-NEXT:    s_mov_b32 s4, 0xff000000
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    v_add_u32_e32 v2, vcc, s0, v2
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_lshlrev_b32_e32 v4, 8, v1
+; VI-NEXT:    v_lshlrev_b16_e32 v5, 8, v0
+; VI-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
+; VI-NEXT:    v_and_b32_e32 v0, s4, v0
+; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT:    v_and_b32_e32 v4, s4, v4
+; VI-NEXT:    v_or_b32_e32 v1, v1, v4
+; VI-NEXT:    v_or_b32_e32 v0, v5, v0
+; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT:    s_endpgm
+;
+; CI-LABEL: shl_v_imm_v4i16:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; CI-NEXT:    s_mov_b32 s7, 0xf000
+; CI-NEXT:    s_mov_b32 s6, 0
+; CI-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
+; CI-NEXT:    v_mov_b32_e32 v1, 0
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; CI-NEXT:    buffer_load_dwordx2 v[2:3], v[0:1], s[4:7], 0 addr64
+; CI-NEXT:    s_mov_b32 s8, 0xff00
+; CI-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; CI-NEXT:    s_waitcnt vmcnt(0)
+; CI-NEXT:    v_lshrrev_b32_e32 v4, 8, v3
+; CI-NEXT:    v_lshlrev_b32_e32 v3, 8, v3
+; CI-NEXT:    v_and_b32_e32 v4, s8, v4
+; CI-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
+; CI-NEXT:    v_and_b32_e32 v3, s8, v3
+; CI-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; CI-NEXT:    v_or_b32_e32 v3, v3, v4
+; CI-NEXT:    v_and_b32_e32 v2, 0xff00ff00, v2
+; CI-NEXT:    buffer_store_dwordx2 v[2:3], v[0:1], s[0:3], 0 addr64
+; CI-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
   %in.gep = getelementptr inbounds <4 x i16>, <4 x i16> addrspace(1)* %in, i64 %tid.ext
diff --git a/llvm/test/CodeGen/AMDGPU/shl_add_ptr.ll b/llvm/test/CodeGen/AMDGPU/shl_add_ptr.ll
index 6c8891d28..ed57ec6cc 100644
--- a/llvm/test/CodeGen/AMDGPU/shl_add_ptr.ll
+++ b/llvm/test/CodeGen/AMDGPU/shl_add_ptr.ll
@@ -78,7 +78,7 @@ define amdgpu_kernel void @load_shl_base_lds_max_offset(i8 addrspace(1)* %out, i
 ; GCN: v_lshlrev_b32_e32 [[OFS:v[0-9]+]], 2, {{v[0-9]+}}
 ; GCN-DAG: v_add_{{[iu]}}32_e32 [[PTR0:v[0-9]+]], vcc, lds0@abs32@lo, [[OFS]]
 ; GCN-DAG: v_add_{{[iu]}}32_e32 [[PTR1:v[0-9]+]], vcc, lds1@abs32@lo, [[OFS]]
-; GCN: s_mov_b32 m0, -1
+; GCN-DAG: s_mov_b32 m0, -1
 
 ; GCN-DAG: ds_read_b32 {{v[0-9]+}}, [[PTR0]] offset:256
 ; GCN-DAG: ds_read_b32 {{v[0-9]+}}, [[PTR1]] offset:256
diff --git a/llvm/test/CodeGen/AMDGPU/shrink-add-sub-constant.ll b/llvm/test/CodeGen/AMDGPU/shrink-add-sub-constant.ll
index 163f919b8..e73003b83 100644
--- a/llvm/test/CodeGen/AMDGPU/shrink-add-sub-constant.ll
+++ b/llvm/test/CodeGen/AMDGPU/shrink-add-sub-constant.ll
@@ -2,6 +2,7 @@
 ; RUN: llc < %s -march=amdgcn -verify-machineinstrs | FileCheck %s -check-prefixes=GCN,SI
 ; RUN: llc < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck %s -check-prefixes=GCN,VI
 ; RUN: llc < %s -march=amdgcn -mcpu=gfx900 -verify-machineinstrs | FileCheck %s -check-prefixes=GCN,GFX9
+; RUN: llc < %s -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs | FileCheck %s -check-prefixes=GCN,GFX10
 
 ; Test that add/sub with a constant is swapped to sub/add with negated
 ; constant to minimize code size.
@@ -56,6 +57,22 @@ define amdgpu_kernel void @v_test_i32_x_sub_64(i32 addrspace(1)* %out, i32 addrs
 ; GFX9-NEXT:    v_subrev_u32_e32 v2, 64, v3
 ; GFX9-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: v_test_i32_x_sub_64:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s2, s2, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s2, s3, 0, s2
+; GFX10-NEXT:    global_load_dword v3, v[0:1], off
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s0, s0, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, s1, 0, s0
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v2, 64, v3
+; GFX10-NEXT:    global_store_dword v[0:1], v2, off
+; GFX10-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
   %gep = getelementptr inbounds i32, i32 addrspace(1)* %in, i64 %tid.ext
@@ -128,6 +145,26 @@ define amdgpu_kernel void @v_test_i32_x_sub_64_multi_use(i32 addrspace(1)* %out,
 ; GFX9-NEXT:    global_store_dword v[2:3], v1, off
 ; GFX9-NEXT:    global_store_dword v[2:3], v0, off
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: v_test_i32_x_sub_64_multi_use:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s2, s2, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s2, s3, 0, s2
+; GFX10-NEXT:    global_load_dword v3, v[0:1], off
+; GFX10-NEXT:    global_load_dword v4, v[0:1], off
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s0, s0, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, s1, 0, s0
+; GFX10-NEXT:    s_waitcnt vmcnt(1)
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v2, 64, v3
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v3, 64, v4
+; GFX10-NEXT:    global_store_dword v[0:1], v2, off
+; GFX10-NEXT:    global_store_dword v[0:1], v3, off
+; GFX10-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
   %gep = getelementptr inbounds i32, i32 addrspace(1)* %in, i64 %tid.ext
@@ -191,6 +228,22 @@ define amdgpu_kernel void @v_test_i32_64_sub_x(i32 addrspace(1)* %out, i32 addrs
 ; GFX9-NEXT:    v_sub_u32_e32 v2, 64, v3
 ; GFX9-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: v_test_i32_64_sub_x:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s2, s2, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s2, s3, 0, s2
+; GFX10-NEXT:    global_load_dword v3, v[0:1], off
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s0, s0, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, s1, 0, s0
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_sub_nc_u32_e32 v2, 64, v3
+; GFX10-NEXT:    global_store_dword v[0:1], v2, off
+; GFX10-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
   %gep = getelementptr inbounds i32, i32 addrspace(1)* %in, i64 %tid.ext
@@ -251,6 +304,22 @@ define amdgpu_kernel void @v_test_i32_x_sub_65(i32 addrspace(1)* %out, i32 addrs
 ; GFX9-NEXT:    v_add_u32_e32 v2, 0xffffffbf, v3
 ; GFX9-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: v_test_i32_x_sub_65:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s2, s2, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s2, s3, 0, s2
+; GFX10-NEXT:    global_load_dword v3, v[0:1], off
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s0, s0, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, s1, 0, s0
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_add_nc_u32_e32 v2, 0xffffffbf, v3
+; GFX10-NEXT:    global_store_dword v[0:1], v2, off
+; GFX10-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
   %gep = getelementptr inbounds i32, i32 addrspace(1)* %in, i64 %tid.ext
@@ -311,6 +380,22 @@ define amdgpu_kernel void @v_test_i32_65_sub_x(i32 addrspace(1)* %out, i32 addrs
 ; GFX9-NEXT:    v_sub_u32_e32 v2, 0x41, v3
 ; GFX9-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: v_test_i32_65_sub_x:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s2, s2, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s2, s3, 0, s2
+; GFX10-NEXT:    global_load_dword v3, v[0:1], off
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s0, s0, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, s1, 0, s0
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_sub_nc_u32_e32 v2, 0x41, v3
+; GFX10-NEXT:    global_store_dword v[0:1], v2, off
+; GFX10-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
   %gep = getelementptr inbounds i32, i32 addrspace(1)* %in, i64 %tid.ext
@@ -371,6 +456,22 @@ define amdgpu_kernel void @v_test_i32_x_sub_neg16(i32 addrspace(1)* %out, i32 ad
 ; GFX9-NEXT:    v_add_u32_e32 v2, 16, v3
 ; GFX9-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: v_test_i32_x_sub_neg16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s2, s2, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s2, s3, 0, s2
+; GFX10-NEXT:    global_load_dword v3, v[0:1], off
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s0, s0, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, s1, 0, s0
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_add_nc_u32_e32 v2, 16, v3
+; GFX10-NEXT:    global_store_dword v[0:1], v2, off
+; GFX10-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
   %gep = getelementptr inbounds i32, i32 addrspace(1)* %in, i64 %tid.ext
@@ -431,6 +532,22 @@ define amdgpu_kernel void @v_test_i32_neg16_sub_x(i32 addrspace(1)* %out, i32 ad
 ; GFX9-NEXT:    v_sub_u32_e32 v2, -16, v3
 ; GFX9-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: v_test_i32_neg16_sub_x:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s2, s2, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s2, s3, 0, s2
+; GFX10-NEXT:    global_load_dword v3, v[0:1], off
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s0, s0, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, s1, 0, s0
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_sub_nc_u32_e32 v2, -16, v3
+; GFX10-NEXT:    global_store_dword v[0:1], v2, off
+; GFX10-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
   %gep = getelementptr inbounds i32, i32 addrspace(1)* %in, i64 %tid.ext
@@ -491,6 +608,22 @@ define amdgpu_kernel void @v_test_i32_x_sub_neg17(i32 addrspace(1)* %out, i32 ad
 ; GFX9-NEXT:    v_add_u32_e32 v2, 17, v3
 ; GFX9-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: v_test_i32_x_sub_neg17:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s2, s2, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s2, s3, 0, s2
+; GFX10-NEXT:    global_load_dword v3, v[0:1], off
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s0, s0, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, s1, 0, s0
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_add_nc_u32_e32 v2, 17, v3
+; GFX10-NEXT:    global_store_dword v[0:1], v2, off
+; GFX10-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
   %gep = getelementptr inbounds i32, i32 addrspace(1)* %in, i64 %tid.ext
@@ -551,6 +684,22 @@ define amdgpu_kernel void @v_test_i32_neg17_sub_x(i32 addrspace(1)* %out, i32 ad
 ; GFX9-NEXT:    v_sub_u32_e32 v2, 0xffffffef, v3
 ; GFX9-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: v_test_i32_neg17_sub_x:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s2, s2, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s2, s3, 0, s2
+; GFX10-NEXT:    global_load_dword v3, v[0:1], off
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s0, s0, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, s1, 0, s0
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_sub_nc_u32_e32 v2, 0xffffffef, v3
+; GFX10-NEXT:    global_store_dword v[0:1], v2, off
+; GFX10-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
   %gep = getelementptr inbounds i32, i32 addrspace(1)* %in, i64 %tid.ext
@@ -591,6 +740,17 @@ define amdgpu_kernel void @s_test_i32_x_sub_64(i32 %x) #0 {
 ; GFX9-NEXT:    ; use s0
 ; GFX9-NEXT:    ;;#ASMEND
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: s_test_i32_x_sub_64:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dword s0, s[0:1], 0x24
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_sub_i32 s0, s0, 64
+; GFX10-NEXT:    ;;#ASMSTART
+; GFX10-NEXT:    ; use s0
+; GFX10-NEXT:    ;;#ASMEND
+; GFX10-NEXT:    s_endpgm
   %result = sub i32 %x, 64
   call void asm sideeffect "; use $0", "s"(i32 %result)
   ret void
@@ -646,6 +806,22 @@ define amdgpu_kernel void @v_test_i16_x_sub_64(i16 addrspace(1)* %out, i16 addrs
 ; GFX9-NEXT:    v_subrev_u16_e32 v2, 64, v3
 ; GFX9-NEXT:    global_store_short v[0:1], v2, off
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: v_test_i16_x_sub_64:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 1, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s2, s2, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s2, s3, 0, s2
+; GFX10-NEXT:    global_load_ushort v3, v[0:1], off
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s0, s0, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, s1, 0, s0
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_sub_nc_u16_e64 v2, v3, 64
+; GFX10-NEXT:    global_store_short v[0:1], v2, off
+; GFX10-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
   %gep = getelementptr inbounds i16, i16 addrspace(1)* %in, i64 %tid.ext
@@ -656,6 +832,89 @@ define amdgpu_kernel void @v_test_i16_x_sub_64(i16 addrspace(1)* %out, i16 addrs
   ret void
 }
 
+define amdgpu_kernel void @v_test_i16_x_sub_64_zext_to_i32(i32 addrspace(1)* %out, i16 addrspace(1)* %in) #0 {
+; SI-LABEL: v_test_i16_x_sub_64_zext_to_i32:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; SI-NEXT:    v_mov_b32_e32 v2, 0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT:    buffer_load_ushort v3, v[1:2], s[4:7], 0 addr64
+; SI-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_subrev_i32_e32 v0, vcc, 64, v3
+; SI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT:    buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: v_test_i16_x_sub_64_zext_to_i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; VI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v3, s3
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v1
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v3, vcc
+; VI-NEXT:    flat_load_ushort v3, v[0:1]
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v2
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_subrev_u16_e32 v2, 64, v3
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; GFX9-LABEL: v_test_i16_x_sub_64_zext_to_i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v3, s3
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s2, v1
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
+; GFX9-NEXT:    global_load_ushort v3, v[0:1], off
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s0, v2
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_subrev_u16_e32 v2, 64, v3
+; GFX9-NEXT:    global_store_dword v[0:1], v2, off
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: v_test_i16_x_sub_64_zext_to_i32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_add_co_u32_e64 v1, s2, s2, v1
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s0, s0, v0
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v2, s2, s3, 0, s2
+; GFX10-NEXT:    global_load_ushort v1, v[1:2], off
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_sub_nc_u16_e64 v2, v1, 64
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, s1, 0, s0
+; GFX10-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GFX10-NEXT:    global_store_dword v[0:1], v2, off
+; GFX10-NEXT:    s_endpgm
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %tid.ext = sext i32 %tid to i64
+  %gep = getelementptr inbounds i16, i16 addrspace(1)* %in, i64 %tid.ext
+  %gep.out = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 %tid.ext
+  %x = load i16, i16 addrspace(1)* %gep
+  %result = sub i16 %x, 64
+  %zext = zext i16 %result to i32
+  store i32 %zext, i32 addrspace(1)* %gep.out
+  ret void
+}
+
 define amdgpu_kernel void @v_test_i16_x_sub_64_multi_use(i16 addrspace(1)* %out, i16 addrspace(1)* %in) #0 {
 ; SI-LABEL: v_test_i16_x_sub_64_multi_use:
 ; SI:       ; %bb.0:
@@ -718,6 +977,26 @@ define amdgpu_kernel void @v_test_i16_x_sub_64_multi_use(i16 addrspace(1)* %out,
 ; GFX9-NEXT:    global_store_short v[2:3], v1, off
 ; GFX9-NEXT:    global_store_short v[2:3], v0, off
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: v_test_i16_x_sub_64_multi_use:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 1, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s2, s2, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s2, s3, 0, s2
+; GFX10-NEXT:    global_load_ushort v3, v[0:1], off
+; GFX10-NEXT:    global_load_ushort v4, v[0:1], off
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s0, s0, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, s1, 0, s0
+; GFX10-NEXT:    s_waitcnt vmcnt(1)
+; GFX10-NEXT:    v_sub_nc_u16_e64 v2, v3, 64
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_sub_nc_u16_e64 v3, v4, 64
+; GFX10-NEXT:    global_store_short v[0:1], v2, off
+; GFX10-NEXT:    global_store_short v[0:1], v3, off
+; GFX10-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
   %gep = getelementptr inbounds i16, i16 addrspace(1)* %in, i64 %tid.ext
@@ -766,9 +1045,9 @@ define amdgpu_kernel void @v_test_v2i16_x_sub_64_64(<2 x i16> addrspace(1)* %out
 ; VI-NEXT:    v_mov_b32_e32 v1, s1
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT:    v_add_u16_e32 v2, 0xffffffc0, v4
-; VI-NEXT:    v_sub_u16_sdwa v3, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_e32 v2, v2, v3
+; VI-NEXT:    v_sub_u16_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_subrev_u16_e32 v3, 64, v4
+; VI-NEXT:    v_or_b32_e32 v2, v3, v2
 ; VI-NEXT:    flat_store_dword v[0:1], v2
 ; VI-NEXT:    s_endpgm
 ;
@@ -788,6 +1067,22 @@ define amdgpu_kernel void @v_test_v2i16_x_sub_64_64(<2 x i16> addrspace(1)* %out
 ; GFX9-NEXT:    v_pk_sub_i16 v2, v3, 64 op_sel_hi:[1,0]
 ; GFX9-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: v_test_v2i16_x_sub_64_64:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s2, s2, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s2, s3, 0, s2
+; GFX10-NEXT:    global_load_dword v3, v[0:1], off
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s0, s0, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, s1, 0, s0
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_pk_sub_i16 v2, v3, 64 op_sel_hi:[1,0]
+; GFX10-NEXT:    global_store_dword v[0:1], v2, off
+; GFX10-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
   %gep = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %in, i64 %tid.ext
@@ -856,6 +1151,22 @@ define amdgpu_kernel void @v_test_v2i16_x_sub_7_64(<2 x i16> addrspace(1)* %out,
 ; GFX9-NEXT:    v_pk_sub_i16 v2, v3, s4
 ; GFX9-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: v_test_v2i16_x_sub_7_64:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s2, s2, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s2, s3, 0, s2
+; GFX10-NEXT:    global_load_dword v3, v[0:1], off
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s0, s0, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, s1, 0, s0
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_pk_sub_i16 v2, v3, 7 op_sel_hi:[1,0]
+; GFX10-NEXT:    global_store_dword v[0:1], v2, off
+; GFX10-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
   %gep = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %in, i64 %tid.ext
@@ -901,9 +1212,9 @@ define amdgpu_kernel void @v_test_v2i16_x_sub_64_123(<2 x i16> addrspace(1)* %ou
 ; VI-NEXT:    v_mov_b32_e32 v1, s1
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT:    v_add_u16_e32 v2, 0xffffffc0, v4
-; VI-NEXT:    v_add_u16_sdwa v3, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_e32 v2, v2, v3
+; VI-NEXT:    v_add_u16_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_subrev_u16_e32 v3, 64, v4
+; VI-NEXT:    v_or_b32_e32 v2, v3, v2
 ; VI-NEXT:    flat_store_dword v[0:1], v2
 ; VI-NEXT:    s_endpgm
 ;
@@ -924,6 +1235,22 @@ define amdgpu_kernel void @v_test_v2i16_x_sub_64_123(<2 x i16> addrspace(1)* %ou
 ; GFX9-NEXT:    v_pk_sub_i16 v2, v3, s4
 ; GFX9-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: v_test_v2i16_x_sub_64_123:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s2, s2, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s2, s3, 0, s2
+; GFX10-NEXT:    global_load_dword v3, v[0:1], off
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s0, s0, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, s1, 0, s0
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_pk_sub_i16 v2, v3, 64 op_sel_hi:[1,0]
+; GFX10-NEXT:    global_store_dword v[0:1], v2, off
+; GFX10-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
   %gep = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %in, i64 %tid.ext
@@ -990,6 +1317,22 @@ define amdgpu_kernel void @v_test_v2i16_x_sub_7_0(<2 x i16> addrspace(1)* %out,
 ; GFX9-NEXT:    v_pk_sub_i16 v2, v3, 7
 ; GFX9-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: v_test_v2i16_x_sub_7_0:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s2, s2, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s2, s3, 0, s2
+; GFX10-NEXT:    global_load_dword v3, v[0:1], off
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s0, s0, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, s1, 0, s0
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_pk_sub_i16 v2, v3, 7
+; GFX10-NEXT:    global_store_dword v[0:1], v2, off
+; GFX10-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
   %gep = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %in, i64 %tid.ext
@@ -1053,6 +1396,22 @@ define amdgpu_kernel void @v_test_v2i16_x_sub_0_16(<2 x i16> addrspace(1)* %out,
 ; GFX9-NEXT:    v_pk_sub_i16 v2, v3, 16 op_sel:[0,1] op_sel_hi:[1,0]
 ; GFX9-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: v_test_v2i16_x_sub_0_16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s2, s2, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s2, s3, 0, s2
+; GFX10-NEXT:    global_load_dword v3, v[0:1], off
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s0, s0, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, s1, 0, s0
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_pk_sub_i16 v2, v3, 16 op_sel:[0,1] op_sel_hi:[1,0]
+; GFX10-NEXT:    global_store_dword v[0:1], v2, off
+; GFX10-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
   %gep = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %in, i64 %tid.ext
@@ -1115,6 +1474,22 @@ define amdgpu_kernel void @v_test_v2i16_x_sub_0_1_0(<2 x i16> addrspace(1)* %out
 ; GFX9-NEXT:    v_pk_sub_i16 v2, v3, -4.0 op_sel:[0,1] op_sel_hi:[1,0]
 ; GFX9-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: v_test_v2i16_x_sub_0_1_0:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s2, s2, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s2, s3, 0, s2
+; GFX10-NEXT:    global_load_dword v3, v[0:1], off
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s0, s0, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, s1, 0, s0
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_pk_sub_i16 v2, v3, -4.0 op_sel:[0,1] op_sel_hi:[1,0]
+; GFX10-NEXT:    global_store_dword v[0:1], v2, off
+; GFX10-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
   %gep = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %in, i64 %tid.ext
@@ -1177,6 +1552,22 @@ define amdgpu_kernel void @v_test_v2i16_x_sub_0_neg1_0(<2 x i16> addrspace(1)* %
 ; GFX9-NEXT:    v_pk_sub_i16 v2, v3, 4.0 op_sel:[0,1] op_sel_hi:[1,0]
 ; GFX9-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: v_test_v2i16_x_sub_0_neg1_0:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s2, s2, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s2, s3, 0, s2
+; GFX10-NEXT:    global_load_dword v3, v[0:1], off
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s0, s0, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, s1, 0, s0
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_pk_sub_i16 v2, v3, 4.0 op_sel:[0,1] op_sel_hi:[1,0]
+; GFX10-NEXT:    global_store_dword v[0:1], v2, off
+; GFX10-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
   %gep = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %in, i64 %tid.ext
@@ -1223,9 +1614,9 @@ define amdgpu_kernel void @v_test_v2i16_x_add_neg32_neg32(<2 x i16> addrspace(1)
 ; VI-NEXT:    v_mov_b32_e32 v1, s1
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT:    v_add_u16_e32 v2, 0xffffffe0, v4
-; VI-NEXT:    v_sub_u16_sdwa v3, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_e32 v2, v2, v3
+; VI-NEXT:    v_sub_u16_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_subrev_u16_e32 v3, 32, v4
+; VI-NEXT:    v_or_b32_e32 v2, v3, v2
 ; VI-NEXT:    flat_store_dword v[0:1], v2
 ; VI-NEXT:    s_endpgm
 ;
@@ -1245,6 +1636,22 @@ define amdgpu_kernel void @v_test_v2i16_x_add_neg32_neg32(<2 x i16> addrspace(1)
 ; GFX9-NEXT:    v_pk_sub_u16 v2, v3, 32 op_sel_hi:[1,0]
 ; GFX9-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: v_test_v2i16_x_add_neg32_neg32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s2, s2, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s2, s3, 0, s2
+; GFX10-NEXT:    global_load_dword v3, v[0:1], off
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s0, s0, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, s1, 0, s0
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_pk_sub_u16 v2, v3, 32 op_sel_hi:[1,0]
+; GFX10-NEXT:    global_store_dword v[0:1], v2, off
+; GFX10-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
   %gep = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %in, i64 %tid.ext
@@ -1307,6 +1714,22 @@ define amdgpu_kernel void @v_test_v2i16_x_add_0_neg32(<2 x i16> addrspace(1)* %o
 ; GFX9-NEXT:    v_pk_sub_u16 v2, v3, 32 op_sel:[0,1] op_sel_hi:[1,0]
 ; GFX9-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: v_test_v2i16_x_add_0_neg32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s2, s2, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s2, s3, 0, s2
+; GFX10-NEXT:    global_load_dword v3, v[0:1], off
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s0, s0, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, s1, 0, s0
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_pk_sub_u16 v2, v3, 32 op_sel:[0,1] op_sel_hi:[1,0]
+; GFX10-NEXT:    global_store_dword v[0:1], v2, off
+; GFX10-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
   %gep = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %in, i64 %tid.ext
@@ -1351,7 +1774,7 @@ define amdgpu_kernel void @v_test_v2i16_x_add_neg32_0(<2 x i16> addrspace(1)* %o
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; VI-NEXT:    v_and_b32_e32 v2, 0xffff0000, v3
-; VI-NEXT:    v_add_u16_e32 v3, 0xffffffe0, v3
+; VI-NEXT:    v_subrev_u16_e32 v3, 32, v3
 ; VI-NEXT:    v_or_b32_e32 v2, v3, v2
 ; VI-NEXT:    flat_store_dword v[0:1], v2
 ; VI-NEXT:    s_endpgm
@@ -1372,6 +1795,22 @@ define amdgpu_kernel void @v_test_v2i16_x_add_neg32_0(<2 x i16> addrspace(1)* %o
 ; GFX9-NEXT:    v_pk_sub_u16 v2, v3, 32
 ; GFX9-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: v_test_v2i16_x_add_neg32_0:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s2, s2, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s2, s3, 0, s2
+; GFX10-NEXT:    global_load_dword v3, v[0:1], off
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s0, s0, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, s1, 0, s0
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_pk_sub_u16 v2, v3, 32
+; GFX10-NEXT:    global_store_dword v[0:1], v2, off
+; GFX10-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
   %gep = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %in, i64 %tid.ext
@@ -1440,6 +1879,22 @@ define amdgpu_kernel void @v_test_v2i16_x_add_neg16_neg16(<2 x i16> addrspace(1)
 ; GFX9-NEXT:    v_pk_sub_u16 v2, v3, 16 op_sel_hi:[1,0]
 ; GFX9-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: v_test_v2i16_x_add_neg16_neg16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s2, s2, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s2, s3, 0, s2
+; GFX10-NEXT:    global_load_dword v3, v[0:1], off
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s0, s0, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, s1, 0, s0
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_pk_sub_u16 v2, v3, 16 op_sel_hi:[1,0]
+; GFX10-NEXT:    global_store_dword v[0:1], v2, off
+; GFX10-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
   %gep = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %in, i64 %tid.ext
@@ -1502,6 +1957,22 @@ define amdgpu_kernel void @v_test_v2i16_x_add_0_neg16(<2 x i16> addrspace(1)* %o
 ; GFX9-NEXT:    v_pk_sub_u16 v2, v3, 16 op_sel:[0,1] op_sel_hi:[1,0]
 ; GFX9-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: v_test_v2i16_x_add_0_neg16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s2, s2, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s2, s3, 0, s2
+; GFX10-NEXT:    global_load_dword v3, v[0:1], off
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s0, s0, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, s1, 0, s0
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_pk_sub_u16 v2, v3, 16 op_sel:[0,1] op_sel_hi:[1,0]
+; GFX10-NEXT:    global_store_dword v[0:1], v2, off
+; GFX10-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
   %gep = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %in, i64 %tid.ext
@@ -1567,6 +2038,22 @@ define amdgpu_kernel void @v_test_v2i16_x_add_neg16_0(<2 x i16> addrspace(1)* %o
 ; GFX9-NEXT:    v_pk_sub_u16 v2, v3, 16
 ; GFX9-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: v_test_v2i16_x_add_neg16_0:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s2, s2, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s2, s3, 0, s2
+; GFX10-NEXT:    global_load_dword v3, v[0:1], off
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s0, s0, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, s1, 0, s0
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_pk_sub_u16 v2, v3, 16
+; GFX10-NEXT:    global_store_dword v[0:1], v2, off
+; GFX10-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
   %gep = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %in, i64 %tid.ext
@@ -1634,6 +2121,22 @@ define amdgpu_kernel void @v_test_v2i16_x_add_neg_fpone(<2 x i16> addrspace(1)*
 ; GFX9-NEXT:    v_pk_sub_u16 v2, v3, 1.0 op_sel_hi:[1,0]
 ; GFX9-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: v_test_v2i16_x_add_neg_fpone:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s2, s2, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s2, s3, 0, s2
+; GFX10-NEXT:    global_load_dword v3, v[0:1], off
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s0, s0, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, s1, 0, s0
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_pk_sub_u16 v2, v3, 1.0 op_sel_hi:[1,0]
+; GFX10-NEXT:    global_store_dword v[0:1], v2, off
+; GFX10-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
   %gep = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %in, i64 %tid.ext
@@ -1701,6 +2204,22 @@ define amdgpu_kernel void @v_test_v2i16_x_add_neg_negfpone(<2 x i16> addrspace(1
 ; GFX9-NEXT:    v_pk_sub_u16 v2, v3, -1.0 op_sel_hi:[1,0]
 ; GFX9-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: v_test_v2i16_x_add_neg_negfpone:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s2, s2, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s2, s3, 0, s2
+; GFX10-NEXT:    global_load_dword v3, v[0:1], off
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s0, s0, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, s1, 0, s0
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_pk_sub_u16 v2, v3, -1.0 op_sel_hi:[1,0]
+; GFX10-NEXT:    global_store_dword v[0:1], v2, off
+; GFX10-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
   %gep = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %in, i64 %tid.ext
@@ -1768,6 +2287,22 @@ define amdgpu_kernel void @v_test_v2i16_x_add_neg_fptwo(<2 x i16> addrspace(1)*
 ; GFX9-NEXT:    v_pk_sub_u16 v2, v3, -2.0 op_sel_hi:[1,0]
 ; GFX9-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: v_test_v2i16_x_add_neg_fptwo:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s2, s2, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s2, s3, 0, s2
+; GFX10-NEXT:    global_load_dword v3, v[0:1], off
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s0, s0, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, s1, 0, s0
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_pk_sub_u16 v2, v3, -2.0 op_sel_hi:[1,0]
+; GFX10-NEXT:    global_store_dword v[0:1], v2, off
+; GFX10-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
   %gep = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %in, i64 %tid.ext
@@ -1835,6 +2370,22 @@ define amdgpu_kernel void @v_test_v2i16_x_add_neg_negfptwo(<2 x i16> addrspace(1
 ; GFX9-NEXT:    v_pk_sub_u16 v2, v3, 2.0 op_sel_hi:[1,0]
 ; GFX9-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: v_test_v2i16_x_add_neg_negfptwo:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s2, s2, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s2, s3, 0, s2
+; GFX10-NEXT:    global_load_dword v3, v[0:1], off
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s0, s0, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, s1, 0, s0
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_pk_sub_u16 v2, v3, 2.0 op_sel_hi:[1,0]
+; GFX10-NEXT:    global_store_dword v[0:1], v2, off
+; GFX10-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
   %gep = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %in, i64 %tid.ext
@@ -1897,6 +2448,22 @@ define amdgpu_kernel void @v_test_v2i16_x_add_undef_neg32(<2 x i16> addrspace(1)
 ; GFX9-NEXT:    v_pk_sub_u16 v2, v3, 32 op_sel:[0,1] op_sel_hi:[1,0]
 ; GFX9-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: v_test_v2i16_x_add_undef_neg32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s2, s2, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s2, s3, 0, s2
+; GFX10-NEXT:    global_load_dword v3, v[0:1], off
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s0, s0, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, s1, 0, s0
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_pk_sub_u16 v2, v3, 32 op_sel:[0,1] op_sel_hi:[1,0]
+; GFX10-NEXT:    global_store_dword v[0:1], v2, off
+; GFX10-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
   %gep = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %in, i64 %tid.ext
@@ -1958,6 +2525,22 @@ define amdgpu_kernel void @v_test_v2i16_x_add_neg32_undef(<2 x i16> addrspace(1)
 ; GFX9-NEXT:    v_pk_sub_u16 v2, v3, 32
 ; GFX9-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: v_test_v2i16_x_add_neg32_undef:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s2, s2, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s2, s3, 0, s2
+; GFX10-NEXT:    global_load_dword v3, v[0:1], off
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s0, s0, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, s1, 0, s0
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_pk_sub_u16 v2, v3, 32
+; GFX10-NEXT:    global_store_dword v[0:1], v2, off
+; GFX10-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
   %gep = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %in, i64 %tid.ext
diff --git a/llvm/test/CodeGen/AMDGPU/si-annotate-cf.ll b/llvm/test/CodeGen/AMDGPU/si-annotate-cf.ll
index 14928749e..23bb18e73 100644
--- a/llvm/test/CodeGen/AMDGPU/si-annotate-cf.ll
+++ b/llvm/test/CodeGen/AMDGPU/si-annotate-cf.ll
@@ -33,14 +33,12 @@ ENDIF:
 
 ; SI: ; %else
 ; SI:     v_cmp_eq_u32_e64  [[TMP:s\[[0-9]+:[0-9]+\]]],
-; SI:     s_and_b64         [[PHI]], [[TMP]], exec
 
 ; SI: ; %endif
 
 ; SI: [[LOOP_LABEL:BB[0-9]+_[0-9]+]]: ; %loop
-; SI:     s_mov_b64         [[TMP:s\[[0-9]+:[0-9]+\]]], [[LEFT]]
 ; SI:     s_and_b64         [[TMP1:s\[[0-9]+:[0-9]+\]]], exec, [[PHI]]
-; SI:     s_or_b64          [[LEFT]], [[TMP1]], [[TMP]]
+; SI:     s_or_b64          [[LEFT]], [[TMP1]], [[LEFT]]
 ; SI:     s_andn2_b64       exec, exec, [[LEFT]]
 ; SI:     s_cbranch_execnz  [[LOOP_LABEL]]
 ; SI:     s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/si-scheduler.ll b/llvm/test/CodeGen/AMDGPU/si-scheduler.ll
index b76eb7c3f..d4087a34f 100644
--- a/llvm/test/CodeGen/AMDGPU/si-scheduler.ll
+++ b/llvm/test/CodeGen/AMDGPU/si-scheduler.ll
@@ -10,8 +10,8 @@
 
 ; CHECK-LABEL: {{^}}main:
 ; CHECK: s_wqm
-; CHECK: s_load_dwordx4
 ; CHECK: s_load_dwordx8
+; CHECK: s_load_dwordx4
 ; CHECK: s_waitcnt lgkmcnt(0)
 ; CHECK: image_sample
 ; CHECK: s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/si-spill-sgpr-stack.ll b/llvm/test/CodeGen/AMDGPU/si-spill-sgpr-stack.ll
index a57e7b134..b93658665 100644
--- a/llvm/test/CodeGen/AMDGPU/si-spill-sgpr-stack.ll
+++ b/llvm/test/CodeGen/AMDGPU/si-spill-sgpr-stack.ll
@@ -1,5 +1,4 @@
-; RUN: llc -march=amdgcn -mcpu=fiji -mattr=-flat-for-global -amdgpu-spill-sgpr-to-smem=0 -verify-machineinstrs < %s | FileCheck -check-prefix=ALL -check-prefix=SGPR %s
-; RUN: llc -march=amdgcn -mcpu=fiji -mattr=-flat-for-global -amdgpu-spill-sgpr-to-smem=1 -verify-machineinstrs < %s | FileCheck -check-prefix=ALL -check-prefix=SMEM %s
+; RUN: llc -march=amdgcn -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=ALL -check-prefix=SGPR %s
 
 ; Make sure this doesn't crash.
 ; ALL-LABEL: {{^}}test:
@@ -14,15 +13,6 @@
 ; SGPR-NEXT: s_nop 4
 ; SGPR-NEXT: buffer_store_dword v0, off, s[0:[[HI]]{{\]}}, 0
 
-; Make sure scratch wave offset register is correctly incremented and
-; then restored.
-; SMEM: s_add_u32 m0, s[[OFF]], 0x100{{$}}
-; SMEM: s_buffer_store_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s{{\[}}[[LO]]:[[HI]]], m0 ; 16-byte Folded Spill
-
-; SMEM: s_add_u32 m0, s[[OFF]], 0x100{{$}}
-; SMEM: s_buffer_load_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s{{\[}}[[LO]]:[[HI]]], m0 ; 16-byte Folded Reload
-
-; SMEM: s_dcache_wb
 ; ALL: s_endpgm
 define amdgpu_kernel void @test(i32 addrspace(1)* %out, i32 %in) {
   call void asm sideeffect "", "~{s[0:7]}" ()
diff --git a/llvm/test/CodeGen/AMDGPU/simplify-libcalls.ll b/llvm/test/CodeGen/AMDGPU/simplify-libcalls.ll
index 859f848d2..682c0679f 100644
--- a/llvm/test/CodeGen/AMDGPU/simplify-libcalls.ll
+++ b/llvm/test/CodeGen/AMDGPU/simplify-libcalls.ll
@@ -3,17 +3,17 @@
 ; RUN: opt -S -O1 -mtriple=amdgcn-- -amdgpu-use-native -amdgpu-prelink < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=GCN-NATIVE %s
 
 ; GCN-LABEL: {{^}}define amdgpu_kernel void @test_sincos
-; GCN-POSTLINK: tail call fast float @_Z3sinf(
-; GCN-POSTLINK: tail call fast float @_Z3cosf(
+; GCN-POSTLINK: call fast float @_Z3sinf(
+; GCN-POSTLINK: call fast float @_Z3cosf(
 ; GCN-PRELINK: call fast float @_Z6sincosfPf(
-; GCN-NATIVE: tail call fast float @_Z10native_sinf(
-; GCN-NATIVE: tail call fast float @_Z10native_cosf(
+; GCN-NATIVE: call fast float @_Z10native_sinf(
+; GCN-NATIVE: call fast float @_Z10native_cosf(
 define amdgpu_kernel void @test_sincos(float addrspace(1)* nocapture %a) {
 entry:
   %tmp = load float, float addrspace(1)* %a, align 4
-  %call = tail call fast float @_Z3sinf(float %tmp)
+  %call = call fast float @_Z3sinf(float %tmp)
   store float %call, float addrspace(1)* %a, align 4
-  %call2 = tail call fast float @_Z3cosf(float %tmp)
+  %call2 = call fast float @_Z3cosf(float %tmp)
   %arrayidx3 = getelementptr inbounds float, float addrspace(1)* %a, i64 1
   store float %call2, float addrspace(1)* %arrayidx3, align 4
   ret void
@@ -24,17 +24,17 @@ declare float @_Z3sinf(float)
 declare float @_Z3cosf(float)
 
 ; GCN-LABEL: {{^}}define amdgpu_kernel void @test_sincos_v2
-; GCN-POSTLINK: tail call fast <2 x float> @_Z3sinDv2_f(
-; GCN-POSTLINK: tail call fast <2 x float> @_Z3cosDv2_f(
+; GCN-POSTLINK: call fast <2 x float> @_Z3sinDv2_f(
+; GCN-POSTLINK: call fast <2 x float> @_Z3cosDv2_f(
 ; GCN-PRELINK: call fast <2 x float> @_Z6sincosDv2_fPS_(
-; GCN-NATIVE: tail call fast <2 x float> @_Z10native_sinDv2_f(
-; GCN-NATIVE: tail call fast <2 x float> @_Z10native_cosDv2_f(
+; GCN-NATIVE: call fast <2 x float> @_Z10native_sinDv2_f(
+; GCN-NATIVE: call fast <2 x float> @_Z10native_cosDv2_f(
 define amdgpu_kernel void @test_sincos_v2(<2 x float> addrspace(1)* nocapture %a) {
 entry:
   %tmp = load <2 x float>, <2 x float> addrspace(1)* %a, align 8
-  %call = tail call fast <2 x float> @_Z3sinDv2_f(<2 x float> %tmp)
+  %call = call fast <2 x float> @_Z3sinDv2_f(<2 x float> %tmp)
   store <2 x float> %call, <2 x float> addrspace(1)* %a, align 8
-  %call2 = tail call fast <2 x float> @_Z3cosDv2_f(<2 x float> %tmp)
+  %call2 = call fast <2 x float> @_Z3cosDv2_f(<2 x float> %tmp)
   %arrayidx3 = getelementptr inbounds <2 x float>, <2 x float> addrspace(1)* %a, i64 1
   store <2 x float> %call2, <2 x float> addrspace(1)* %arrayidx3, align 8
   ret void
@@ -45,20 +45,20 @@ declare <2 x float> @_Z3sinDv2_f(<2 x float>)
 declare <2 x float> @_Z3cosDv2_f(<2 x float>)
 
 ; GCN-LABEL: {{^}}define amdgpu_kernel void @test_sincos_v3
-; GCN-POSTLINK: tail call fast <3 x float> @_Z3sinDv3_f(
-; GCN-POSTLINK: tail call fast <3 x float> @_Z3cosDv3_f(
+; GCN-POSTLINK: call fast <3 x float> @_Z3sinDv3_f(
+; GCN-POSTLINK: call fast <3 x float> @_Z3cosDv3_f(
 ; GCN-PRELINK: call fast <3 x float> @_Z6sincosDv3_fPS_(
-; GCN-NATIVE: tail call fast <3 x float> @_Z10native_sinDv3_f(
-; GCN-NATIVE: tail call fast <3 x float> @_Z10native_cosDv3_f(
+; GCN-NATIVE: call fast <3 x float> @_Z10native_sinDv3_f(
+; GCN-NATIVE: call fast <3 x float> @_Z10native_cosDv3_f(
 define amdgpu_kernel void @test_sincos_v3(<3 x float> addrspace(1)* nocapture %a) {
 entry:
   %castToVec4 = bitcast <3 x float> addrspace(1)* %a to <4 x float> addrspace(1)*
   %loadVec4 = load <4 x float>, <4 x float> addrspace(1)* %castToVec4, align 16
   %extractVec4 = shufflevector <4 x float> %loadVec4, <4 x float> undef, <3 x i32> <i32 0, i32 1, i32 2>
-  %call = tail call fast <3 x float> @_Z3sinDv3_f(<3 x float> %extractVec4)
+  %call = call fast <3 x float> @_Z3sinDv3_f(<3 x float> %extractVec4)
   %extractVec6 = shufflevector <3 x float> %call, <3 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
   store <4 x float> %extractVec6, <4 x float> addrspace(1)* %castToVec4, align 16
-  %call11 = tail call fast <3 x float> @_Z3cosDv3_f(<3 x float> %extractVec4)
+  %call11 = call fast <3 x float> @_Z3cosDv3_f(<3 x float> %extractVec4)
   %arrayidx12 = getelementptr inbounds <3 x float>, <3 x float> addrspace(1)* %a, i64 1
   %extractVec13 = shufflevector <3 x float> %call11, <3 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
   %storetmp14 = bitcast <3 x float> addrspace(1)* %arrayidx12 to <4 x float> addrspace(1)*
@@ -71,17 +71,17 @@ declare <3 x float> @_Z3sinDv3_f(<3 x float>)
 declare <3 x float> @_Z3cosDv3_f(<3 x float>)
 
 ; GCN-LABEL: {{^}}define amdgpu_kernel void @test_sincos_v4
-; GCN-POSTLINK: tail call fast <4 x float> @_Z3sinDv4_f(
-; GCN-POSTLINK: tail call fast <4 x float> @_Z3cosDv4_f(
+; GCN-POSTLINK: call fast <4 x float> @_Z3sinDv4_f(
+; GCN-POSTLINK: call fast <4 x float> @_Z3cosDv4_f(
 ; GCN-PRELINK: call fast <4 x float> @_Z6sincosDv4_fPS_(
-; GCN-NATIVE: tail call fast <4 x float> @_Z10native_sinDv4_f(
-; GCN-NATIVE: tail call fast <4 x float> @_Z10native_cosDv4_f(
+; GCN-NATIVE: call fast <4 x float> @_Z10native_sinDv4_f(
+; GCN-NATIVE: call fast <4 x float> @_Z10native_cosDv4_f(
 define amdgpu_kernel void @test_sincos_v4(<4 x float> addrspace(1)* nocapture %a) {
 entry:
   %tmp = load <4 x float>, <4 x float> addrspace(1)* %a, align 16
-  %call = tail call fast <4 x float> @_Z3sinDv4_f(<4 x float> %tmp)
+  %call = call fast <4 x float> @_Z3sinDv4_f(<4 x float> %tmp)
   store <4 x float> %call, <4 x float> addrspace(1)* %a, align 16
-  %call2 = tail call fast <4 x float> @_Z3cosDv4_f(<4 x float> %tmp)
+  %call2 = call fast <4 x float> @_Z3cosDv4_f(<4 x float> %tmp)
   %arrayidx3 = getelementptr inbounds <4 x float>, <4 x float> addrspace(1)* %a, i64 1
   store <4 x float> %call2, <4 x float> addrspace(1)* %arrayidx3, align 16
   ret void
@@ -92,17 +92,17 @@ declare <4 x float> @_Z3sinDv4_f(<4 x float>)
 declare <4 x float> @_Z3cosDv4_f(<4 x float>)
 
 ; GCN-LABEL: {{^}}define amdgpu_kernel void @test_sincos_v8
-; GCN-POSTLINK: tail call fast <8 x float> @_Z3sinDv8_f(
-; GCN-POSTLINK: tail call fast <8 x float> @_Z3cosDv8_f(
+; GCN-POSTLINK: call fast <8 x float> @_Z3sinDv8_f(
+; GCN-POSTLINK: call fast <8 x float> @_Z3cosDv8_f(
 ; GCN-PRELINK: call fast <8 x float> @_Z6sincosDv8_fPS_(
-; GCN-NATIVE: tail call fast <8 x float> @_Z10native_sinDv8_f(
-; GCN-NATIVE: tail call fast <8 x float> @_Z10native_cosDv8_f(
+; GCN-NATIVE: call fast <8 x float> @_Z10native_sinDv8_f(
+; GCN-NATIVE: call fast <8 x float> @_Z10native_cosDv8_f(
 define amdgpu_kernel void @test_sincos_v8(<8 x float> addrspace(1)* nocapture %a) {
 entry:
   %tmp = load <8 x float>, <8 x float> addrspace(1)* %a, align 32
-  %call = tail call fast <8 x float> @_Z3sinDv8_f(<8 x float> %tmp)
+  %call = call fast <8 x float> @_Z3sinDv8_f(<8 x float> %tmp)
   store <8 x float> %call, <8 x float> addrspace(1)* %a, align 32
-  %call2 = tail call fast <8 x float> @_Z3cosDv8_f(<8 x float> %tmp)
+  %call2 = call fast <8 x float> @_Z3cosDv8_f(<8 x float> %tmp)
   %arrayidx3 = getelementptr inbounds <8 x float>, <8 x float> addrspace(1)* %a, i64 1
   store <8 x float> %call2, <8 x float> addrspace(1)* %arrayidx3, align 32
   ret void
@@ -113,17 +113,17 @@ declare <8 x float> @_Z3sinDv8_f(<8 x float>)
 declare <8 x float> @_Z3cosDv8_f(<8 x float>)
 
 ; GCN-LABEL: {{^}}define amdgpu_kernel void @test_sincos_v16
-; GCN-POSTLINK: tail call fast <16 x float> @_Z3sinDv16_f(
-; GCN-POSTLINK: tail call fast <16 x float> @_Z3cosDv16_f(
+; GCN-POSTLINK: call fast <16 x float> @_Z3sinDv16_f(
+; GCN-POSTLINK: call fast <16 x float> @_Z3cosDv16_f(
 ; GCN-PRELINK: call fast <16 x float> @_Z6sincosDv16_fPS_(
-; GCN-NATIVE: tail call fast <16 x float> @_Z10native_sinDv16_f(
-; GCN-NATIVE: tail call fast <16 x float> @_Z10native_cosDv16_f(
+; GCN-NATIVE: call fast <16 x float> @_Z10native_sinDv16_f(
+; GCN-NATIVE: call fast <16 x float> @_Z10native_cosDv16_f(
 define amdgpu_kernel void @test_sincos_v16(<16 x float> addrspace(1)* nocapture %a) {
 entry:
   %tmp = load <16 x float>, <16 x float> addrspace(1)* %a, align 64
-  %call = tail call fast <16 x float> @_Z3sinDv16_f(<16 x float> %tmp)
+  %call = call fast <16 x float> @_Z3sinDv16_f(<16 x float> %tmp)
   store <16 x float> %call, <16 x float> addrspace(1)* %a, align 64
-  %call2 = tail call fast <16 x float> @_Z3cosDv16_f(<16 x float> %tmp)
+  %call2 = call fast <16 x float> @_Z3cosDv16_f(<16 x float> %tmp)
   %arrayidx3 = getelementptr inbounds <16 x float>, <16 x float> addrspace(1)* %a, i64 1
   store <16 x float> %call2, <16 x float> addrspace(1)* %arrayidx3, align 64
   ret void
@@ -137,7 +137,7 @@ declare <16 x float> @_Z3cosDv16_f(<16 x float>)
 ; GCN: store float 0x3FD5555560000000, float addrspace(1)* %a
 define amdgpu_kernel void @test_native_recip(float addrspace(1)* nocapture %a) {
 entry:
-  %call = tail call fast float @_Z12native_recipf(float 3.000000e+00)
+  %call = call fast float @_Z12native_recipf(float 3.000000e+00)
   store float %call, float addrspace(1)* %a, align 4
   ret void
 }
@@ -148,7 +148,7 @@ declare float @_Z12native_recipf(float)
 ; GCN: store float 0x3FD5555560000000, float addrspace(1)* %a
 define amdgpu_kernel void @test_half_recip(float addrspace(1)* nocapture %a) {
 entry:
-  %call = tail call fast float @_Z10half_recipf(float 3.000000e+00)
+  %call = call fast float @_Z10half_recipf(float 3.000000e+00)
   store float %call, float addrspace(1)* %a, align 4
   ret void
 }
@@ -160,7 +160,7 @@ declare float @_Z10half_recipf(float)
 define amdgpu_kernel void @test_native_divide(float addrspace(1)* nocapture %a) {
 entry:
   %tmp = load float, float addrspace(1)* %a, align 4
-  %call = tail call fast float @_Z13native_divideff(float %tmp, float 3.000000e+00)
+  %call = call fast float @_Z13native_divideff(float %tmp, float 3.000000e+00)
   store float %call, float addrspace(1)* %a, align 4
   ret void
 }
@@ -172,7 +172,7 @@ declare float @_Z13native_divideff(float, float)
 define amdgpu_kernel void @test_half_divide(float addrspace(1)* nocapture %a) {
 entry:
   %tmp = load float, float addrspace(1)* %a, align 4
-  %call = tail call fast float @_Z11half_divideff(float %tmp, float 3.000000e+00)
+  %call = call fast float @_Z11half_divideff(float %tmp, float 3.000000e+00)
   store float %call, float addrspace(1)* %a, align 4
   ret void
 }
@@ -184,7 +184,7 @@ declare float @_Z11half_divideff(float, float)
 define amdgpu_kernel void @test_pow_0f(float addrspace(1)* nocapture %a) {
 entry:
   %tmp = load float, float addrspace(1)* %a, align 4
-  %call = tail call fast float @_Z3powff(float %tmp, float 0.000000e+00)
+  %call = call fast float @_Z3powff(float %tmp, float 0.000000e+00)
   store float %call, float addrspace(1)* %a, align 4
   ret void
 }
@@ -196,7 +196,7 @@ declare float @_Z3powff(float, float)
 define amdgpu_kernel void @test_pow_0i(float addrspace(1)* nocapture %a) {
 entry:
   %tmp = load float, float addrspace(1)* %a, align 4
-  %call = tail call fast float @_Z3powff(float %tmp, float 0.000000e+00)
+  %call = call fast float @_Z3powff(float %tmp, float 0.000000e+00)
   store float %call, float addrspace(1)* %a, align 4
   ret void
 }
@@ -208,7 +208,7 @@ define amdgpu_kernel void @test_pow_1f(float addrspace(1)* nocapture %a) {
 entry:
   %arrayidx = getelementptr inbounds float, float addrspace(1)* %a, i64 1
   %tmp = load float, float addrspace(1)* %arrayidx, align 4
-  %call = tail call fast float @_Z3powff(float %tmp, float 1.000000e+00)
+  %call = call fast float @_Z3powff(float %tmp, float 1.000000e+00)
   store float %call, float addrspace(1)* %a, align 4
   ret void
 }
@@ -220,7 +220,7 @@ define amdgpu_kernel void @test_pow_1i(float addrspace(1)* nocapture %a) {
 entry:
   %arrayidx = getelementptr inbounds float, float addrspace(1)* %a, i64 1
   %tmp = load float, float addrspace(1)* %arrayidx, align 4
-  %call = tail call fast float @_Z3powff(float %tmp, float 1.000000e+00)
+  %call = call fast float @_Z3powff(float %tmp, float 1.000000e+00)
   store float %call, float addrspace(1)* %a, align 4
   ret void
 }
@@ -231,7 +231,7 @@ entry:
 define amdgpu_kernel void @test_pow_2f(float addrspace(1)* nocapture %a) {
 entry:
   %tmp = load float, float addrspace(1)* %a, align 4
-  %call = tail call fast float @_Z3powff(float %tmp, float 2.000000e+00)
+  %call = call fast float @_Z3powff(float %tmp, float 2.000000e+00)
   store float %call, float addrspace(1)* %a, align 4
   ret void
 }
@@ -242,7 +242,7 @@ entry:
 define amdgpu_kernel void @test_pow_2i(float addrspace(1)* nocapture %a) {
 entry:
   %tmp = load float, float addrspace(1)* %a, align 4
-  %call = tail call fast float @_Z3powff(float %tmp, float 2.000000e+00)
+  %call = call fast float @_Z3powff(float %tmp, float 2.000000e+00)
   store float %call, float addrspace(1)* %a, align 4
   ret void
 }
@@ -254,7 +254,7 @@ define amdgpu_kernel void @test_pow_m1f(float addrspace(1)* nocapture %a) {
 entry:
   %arrayidx = getelementptr inbounds float, float addrspace(1)* %a, i64 1
   %tmp = load float, float addrspace(1)* %arrayidx, align 4
-  %call = tail call fast float @_Z3powff(float %tmp, float -1.000000e+00)
+  %call = call fast float @_Z3powff(float %tmp, float -1.000000e+00)
   store float %call, float addrspace(1)* %a, align 4
   ret void
 }
@@ -266,31 +266,31 @@ define amdgpu_kernel void @test_pow_m1i(float addrspace(1)* nocapture %a) {
 entry:
   %arrayidx = getelementptr inbounds float, float addrspace(1)* %a, i64 1
   %tmp = load float, float addrspace(1)* %arrayidx, align 4
-  %call = tail call fast float @_Z3powff(float %tmp, float -1.000000e+00)
+  %call = call fast float @_Z3powff(float %tmp, float -1.000000e+00)
   store float %call, float addrspace(1)* %a, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}define amdgpu_kernel void @test_pow_half
-; GCN-POSTLINK: tail call fast float @_Z3powff(float %tmp, float 5.000000e-01)
-; GCN-PRELINK: %__pow2sqrt = tail call fast float @_Z4sqrtf(float %tmp)
+; GCN-POSTLINK: call fast float @_Z3powff(float %tmp, float 5.000000e-01)
+; GCN-PRELINK: %__pow2sqrt = call fast float @_Z4sqrtf(float %tmp)
 define amdgpu_kernel void @test_pow_half(float addrspace(1)* nocapture %a) {
 entry:
   %arrayidx = getelementptr inbounds float, float addrspace(1)* %a, i64 1
   %tmp = load float, float addrspace(1)* %arrayidx, align 4
-  %call = tail call fast float @_Z3powff(float %tmp, float 5.000000e-01)
+  %call = call fast float @_Z3powff(float %tmp, float 5.000000e-01)
   store float %call, float addrspace(1)* %a, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}define amdgpu_kernel void @test_pow_mhalf
-; GCN-POSTLINK: tail call fast float @_Z3powff(float %tmp, float -5.000000e-01)
-; GCN-PRELINK: %__pow2rsqrt = tail call fast float @_Z5rsqrtf(float %tmp)
+; GCN-POSTLINK: call fast float @_Z3powff(float %tmp, float -5.000000e-01)
+; GCN-PRELINK: %__pow2rsqrt = call fast float @_Z5rsqrtf(float %tmp)
 define amdgpu_kernel void @test_pow_mhalf(float addrspace(1)* nocapture %a) {
 entry:
   %arrayidx = getelementptr inbounds float, float addrspace(1)* %a, i64 1
   %tmp = load float, float addrspace(1)* %arrayidx, align 4
-  %call = tail call fast float @_Z3powff(float %tmp, float -5.000000e-01)
+  %call = call fast float @_Z3powff(float %tmp, float -5.000000e-01)
   store float %call, float addrspace(1)* %a, align 4
   ret void
 }
@@ -305,7 +305,7 @@ define amdgpu_kernel void @test_pow_c(float addrspace(1)* nocapture %a) {
 entry:
   %arrayidx = getelementptr inbounds float, float addrspace(1)* %a, i64 1
   %tmp = load float, float addrspace(1)* %arrayidx, align 4
-  %call = tail call fast float @_Z3powff(float %tmp, float 1.100000e+01)
+  %call = call fast float @_Z3powff(float %tmp, float 1.100000e+01)
   store float %call, float addrspace(1)* %a, align 4
   ret void
 }
@@ -320,7 +320,7 @@ define amdgpu_kernel void @test_powr_c(float addrspace(1)* nocapture %a) {
 entry:
   %arrayidx = getelementptr inbounds float, float addrspace(1)* %a, i64 1
   %tmp = load float, float addrspace(1)* %arrayidx, align 4
-  %call = tail call fast float @_Z4powrff(float %tmp, float 1.100000e+01)
+  %call = call fast float @_Z4powrff(float %tmp, float 1.100000e+01)
   store float %call, float addrspace(1)* %a, align 4
   ret void
 }
@@ -337,7 +337,7 @@ define amdgpu_kernel void @test_pown_c(float addrspace(1)* nocapture %a) {
 entry:
   %arrayidx = getelementptr inbounds float, float addrspace(1)* %a, i64 1
   %tmp = load float, float addrspace(1)* %arrayidx, align 4
-  %call = tail call fast float @_Z4pownfi(float %tmp, i32 11)
+  %call = call fast float @_Z4pownfi(float %tmp, i32 11)
   store float %call, float addrspace(1)* %a, align 4
   ret void
 }
@@ -345,11 +345,11 @@ entry:
 declare float @_Z4pownfi(float, i32)
 
 ; GCN-LABEL: {{^}}define amdgpu_kernel void @test_pow
-; GCN-POSTLINK: tail call fast float @_Z3powff(float %tmp, float 1.013000e+03)
-; GCN-PRELINK: %__fabs = tail call fast float @_Z4fabsf(float %tmp)
-; GCN-PRELINK: %__log2 = tail call fast float @_Z4log2f(float %__fabs)
+; GCN-POSTLINK: call fast float @_Z3powff(float %tmp, float 1.013000e+03)
+; GCN-PRELINK: %__fabs = call fast float @_Z4fabsf(float %tmp)
+; GCN-PRELINK: %__log2 = call fast float @_Z4log2f(float %__fabs)
 ; GCN-PRELINK: %__ylogx = fmul fast float %__log2, 1.013000e+03
-; GCN-PRELINK: %__exp2 = tail call fast float @_Z4exp2f(float %__ylogx)
+; GCN-PRELINK: %__exp2 = call fast float @_Z4exp2f(float %__ylogx)
 ; GCN-PRELINK: %[[r0:.*]] = bitcast float %tmp to i32
 ; GCN-PRELINK: %__pow_sign = and i32 %[[r0]], -2147483648
 ; GCN-PRELINK: %[[r1:.*]] = bitcast float %__exp2 to i32
@@ -359,39 +359,39 @@ declare float @_Z4pownfi(float, i32)
 define amdgpu_kernel void @test_pow(float addrspace(1)* nocapture %a) {
 entry:
   %tmp = load float, float addrspace(1)* %a, align 4
-  %call = tail call fast float @_Z3powff(float %tmp, float 1.013000e+03)
+  %call = call fast float @_Z3powff(float %tmp, float 1.013000e+03)
   store float %call, float addrspace(1)* %a, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}define amdgpu_kernel void @test_powr
-; GCN-POSTLINK: tail call fast float @_Z4powrff(float %tmp, float %tmp1)
-; GCN-PRELINK: %__log2 = tail call fast float @_Z4log2f(float %tmp)
+; GCN-POSTLINK: call fast float @_Z4powrff(float %tmp, float %tmp1)
+; GCN-PRELINK: %__log2 = call fast float @_Z4log2f(float %tmp)
 ; GCN-PRELINK: %__ylogx = fmul fast float %__log2, %tmp1
-; GCN-PRELINK: %__exp2 = tail call fast float @_Z4exp2f(float %__ylogx)
+; GCN-PRELINK: %__exp2 = call fast float @_Z4exp2f(float %__ylogx)
 ; GCN-PRELINK: store float %__exp2, float addrspace(1)* %a, align 4
-; GCN-NATIVE:  %__log2 = tail call fast float @_Z11native_log2f(float %tmp)
+; GCN-NATIVE:  %__log2 = call fast float @_Z11native_log2f(float %tmp)
 ; GCN-NATIVE:  %__ylogx = fmul fast float %__log2, %tmp1
-; GCN-NATIVE:  %__exp2 = tail call fast float @_Z11native_exp2f(float %__ylogx)
+; GCN-NATIVE:  %__exp2 = call fast float @_Z11native_exp2f(float %__ylogx)
 ; GCN-NATIVE:  store float %__exp2, float addrspace(1)* %a, align 4
 define amdgpu_kernel void @test_powr(float addrspace(1)* nocapture %a) {
 entry:
   %tmp = load float, float addrspace(1)* %a, align 4
   %arrayidx1 = getelementptr inbounds float, float addrspace(1)* %a, i64 1
   %tmp1 = load float, float addrspace(1)* %arrayidx1, align 4
-  %call = tail call fast float @_Z4powrff(float %tmp, float %tmp1)
+  %call = call fast float @_Z4powrff(float %tmp, float %tmp1)
   store float %call, float addrspace(1)* %a, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}define amdgpu_kernel void @test_pown
-; GCN-POSTLINK: tail call fast float @_Z4pownfi(float %tmp, i32 %conv)
+; GCN-POSTLINK: call fast float @_Z4pownfi(float %tmp, i32 %conv)
 ; GCN-PRELINK: %conv = fptosi float %tmp1 to i32
-; GCN-PRELINK: %__fabs = tail call fast float @_Z4fabsf(float %tmp)
-; GCN-PRELINK: %__log2 = tail call fast float @_Z4log2f(float %__fabs)
+; GCN-PRELINK: %__fabs = call fast float @_Z4fabsf(float %tmp)
+; GCN-PRELINK: %__log2 = call fast float @_Z4log2f(float %__fabs)
 ; GCN-PRELINK: %pownI2F = sitofp i32 %conv to float
 ; GCN-PRELINK: %__ylogx = fmul fast float %__log2, %pownI2F
-; GCN-PRELINK: %__exp2 = tail call fast float @_Z4exp2f(float %__ylogx)
+; GCN-PRELINK: %__exp2 = call fast float @_Z4exp2f(float %__ylogx)
 ; GCN-PRELINK: %__yeven = shl i32 %conv, 31
 ; GCN-PRELINK: %[[r0:.*]] = bitcast float %tmp to i32
 ; GCN-PRELINK: %__pow_sign = and i32 %__yeven, %[[r0]]
@@ -405,7 +405,7 @@ entry:
   %arrayidx1 = getelementptr inbounds float, float addrspace(1)* %a, i64 1
   %tmp1 = load float, float addrspace(1)* %arrayidx1, align 4
   %conv = fptosi float %tmp1 to i32
-  %call = tail call fast float @_Z4pownfi(float %tmp, i32 %conv)
+  %call = call fast float @_Z4pownfi(float %tmp, i32 %conv)
   store float %call, float addrspace(1)* %a, align 4
   ret void
 }
@@ -417,7 +417,7 @@ define amdgpu_kernel void @test_rootn_1(float addrspace(1)* nocapture %a) {
 entry:
   %arrayidx = getelementptr inbounds float, float addrspace(1)* %a, i64 1
   %tmp = load float, float addrspace(1)* %arrayidx, align 4
-  %call = tail call fast float @_Z5rootnfi(float %tmp, i32 1)
+  %call = call fast float @_Z5rootnfi(float %tmp, i32 1)
   store float %call, float addrspace(1)* %a, align 4
   ret void
 }
@@ -425,23 +425,23 @@ entry:
 declare float @_Z5rootnfi(float, i32)
 
 ; GCN-LABEL: {{^}}define amdgpu_kernel void @test_rootn_2
-; GCN-POSTLINK: tail call fast float @_Z5rootnfi(float %tmp, i32 2)
-; GCN-PRELINK: %__rootn2sqrt = tail call fast float @_Z4sqrtf(float %tmp)
+; GCN-POSTLINK: call fast float @_Z5rootnfi(float %tmp, i32 2)
+; GCN-PRELINK: %__rootn2sqrt = call fast float @_Z4sqrtf(float %tmp)
 define amdgpu_kernel void @test_rootn_2(float addrspace(1)* nocapture %a) {
 entry:
   %tmp = load float, float addrspace(1)* %a, align 4
-  %call = tail call fast float @_Z5rootnfi(float %tmp, i32 2)
+  %call = call fast float @_Z5rootnfi(float %tmp, i32 2)
   store float %call, float addrspace(1)* %a, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}define amdgpu_kernel void @test_rootn_3
-; GCN-POSTLINK: tail call fast float @_Z5rootnfi(float %tmp, i32 3)
-; GCN-PRELINK: %__rootn2cbrt = tail call fast float @_Z4cbrtf(float %tmp)
+; GCN-POSTLINK: call fast float @_Z5rootnfi(float %tmp, i32 3)
+; GCN-PRELINK: %__rootn2cbrt = call fast float @_Z4cbrtf(float %tmp)
 define amdgpu_kernel void @test_rootn_3(float addrspace(1)* nocapture %a) {
 entry:
   %tmp = load float, float addrspace(1)* %a, align 4
-  %call = tail call fast float @_Z5rootnfi(float %tmp, i32 3)
+  %call = call fast float @_Z5rootnfi(float %tmp, i32 3)
   store float %call, float addrspace(1)* %a, align 4
   ret void
 }
@@ -451,18 +451,18 @@ entry:
 define amdgpu_kernel void @test_rootn_m1(float addrspace(1)* nocapture %a) {
 entry:
   %tmp = load float, float addrspace(1)* %a, align 4
-  %call = tail call fast float @_Z5rootnfi(float %tmp, i32 -1)
+  %call = call fast float @_Z5rootnfi(float %tmp, i32 -1)
   store float %call, float addrspace(1)* %a, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}define amdgpu_kernel void @test_rootn_m2
-; GCN-POSTLINK: tail call fast float @_Z5rootnfi(float %tmp, i32 -2)
-; GCN-PRELINK: %__rootn2rsqrt = tail call fast float @_Z5rsqrtf(float %tmp)
+; GCN-POSTLINK: call fast float @_Z5rootnfi(float %tmp, i32 -2)
+; GCN-PRELINK: %__rootn2rsqrt = call fast float @_Z5rsqrtf(float %tmp)
 define amdgpu_kernel void @test_rootn_m2(float addrspace(1)* nocapture %a) {
 entry:
   %tmp = load float, float addrspace(1)* %a, align 4
-  %call = tail call fast float @_Z5rootnfi(float %tmp, i32 -2)
+  %call = call fast float @_Z5rootnfi(float %tmp, i32 -2)
   store float %call, float addrspace(1)* %a, align 4
   ret void
 }
@@ -472,7 +472,7 @@ entry:
 define amdgpu_kernel void @test_fma_0x(float addrspace(1)* nocapture %a, float %y) {
 entry:
   %tmp = load float, float addrspace(1)* %a, align 4
-  %call = tail call fast float @_Z3fmafff(float 0.000000e+00, float %tmp, float %y)
+  %call = call fast float @_Z3fmafff(float 0.000000e+00, float %tmp, float %y)
   store float %call, float addrspace(1)* %a, align 4
   ret void
 }
@@ -484,7 +484,7 @@ declare float @_Z3fmafff(float, float, float)
 define amdgpu_kernel void @test_fma_x0(float addrspace(1)* nocapture %a, float %y) {
 entry:
   %tmp = load float, float addrspace(1)* %a, align 4
-  %call = tail call fast float @_Z3fmafff(float %tmp, float 0.000000e+00, float %y)
+  %call = call fast float @_Z3fmafff(float %tmp, float 0.000000e+00, float %y)
   store float %call, float addrspace(1)* %a, align 4
   ret void
 }
@@ -494,7 +494,7 @@ entry:
 define amdgpu_kernel void @test_mad_0x(float addrspace(1)* nocapture %a, float %y) {
 entry:
   %tmp = load float, float addrspace(1)* %a, align 4
-  %call = tail call fast float @_Z3madfff(float 0.000000e+00, float %tmp, float %y)
+  %call = call fast float @_Z3madfff(float 0.000000e+00, float %tmp, float %y)
   store float %call, float addrspace(1)* %a, align 4
   ret void
 }
@@ -506,7 +506,7 @@ declare float @_Z3madfff(float, float, float)
 define amdgpu_kernel void @test_mad_x0(float addrspace(1)* nocapture %a, float %y) {
 entry:
   %tmp = load float, float addrspace(1)* %a, align 4
-  %call = tail call fast float @_Z3madfff(float %tmp, float 0.000000e+00, float %y)
+  %call = call fast float @_Z3madfff(float %tmp, float 0.000000e+00, float %y)
   store float %call, float addrspace(1)* %a, align 4
   ret void
 }
@@ -516,7 +516,7 @@ entry:
 define amdgpu_kernel void @test_fma_x1y(float addrspace(1)* nocapture %a, float %y) {
 entry:
   %tmp = load float, float addrspace(1)* %a, align 4
-  %call = tail call fast float @_Z3fmafff(float %tmp, float 1.000000e+00, float %y)
+  %call = call fast float @_Z3fmafff(float %tmp, float 1.000000e+00, float %y)
   store float %call, float addrspace(1)* %a, align 4
   ret void
 }
@@ -526,7 +526,7 @@ entry:
 define amdgpu_kernel void @test_fma_1xy(float addrspace(1)* nocapture %a, float %y) {
 entry:
   %tmp = load float, float addrspace(1)* %a, align 4
-  %call = tail call fast float @_Z3fmafff(float 1.000000e+00, float %tmp, float %y)
+  %call = call fast float @_Z3fmafff(float 1.000000e+00, float %tmp, float %y)
   store float %call, float addrspace(1)* %a, align 4
   ret void
 }
@@ -538,17 +538,17 @@ entry:
   %arrayidx = getelementptr inbounds float, float addrspace(1)* %a, i64 1
   %tmp = load float, float addrspace(1)* %arrayidx, align 4
   %tmp1 = load float, float addrspace(1)* %a, align 4
-  %call = tail call fast float @_Z3fmafff(float %tmp, float %tmp1, float 0.000000e+00)
+  %call = call fast float @_Z3fmafff(float %tmp, float %tmp1, float 0.000000e+00)
   store float %call, float addrspace(1)* %a, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}define amdgpu_kernel void @test_use_native_exp
-; GCN-NATIVE: tail call fast float @_Z10native_expf(float %tmp)
+; GCN-NATIVE: call fast float @_Z10native_expf(float %tmp)
 define amdgpu_kernel void @test_use_native_exp(float addrspace(1)* nocapture %a) {
 entry:
   %tmp = load float, float addrspace(1)* %a, align 4
-  %call = tail call fast float @_Z3expf(float %tmp)
+  %call = call fast float @_Z3expf(float %tmp)
   store float %call, float addrspace(1)* %a, align 4
   ret void
 }
@@ -556,11 +556,11 @@ entry:
 declare float @_Z3expf(float)
 
 ; GCN-LABEL: {{^}}define amdgpu_kernel void @test_use_native_exp2
-; GCN-NATIVE: tail call fast float @_Z11native_exp2f(float %tmp)
+; GCN-NATIVE: call fast float @_Z11native_exp2f(float %tmp)
 define amdgpu_kernel void @test_use_native_exp2(float addrspace(1)* nocapture %a) {
 entry:
   %tmp = load float, float addrspace(1)* %a, align 4
-  %call = tail call fast float @_Z4exp2f(float %tmp)
+  %call = call fast float @_Z4exp2f(float %tmp)
   store float %call, float addrspace(1)* %a, align 4
   ret void
 }
@@ -568,11 +568,11 @@ entry:
 declare float @_Z4exp2f(float)
 
 ; GCN-LABEL: {{^}}define amdgpu_kernel void @test_use_native_exp10
-; GCN-NATIVE: tail call fast float @_Z12native_exp10f(float %tmp)
+; GCN-NATIVE: call fast float @_Z12native_exp10f(float %tmp)
 define amdgpu_kernel void @test_use_native_exp10(float addrspace(1)* nocapture %a) {
 entry:
   %tmp = load float, float addrspace(1)* %a, align 4
-  %call = tail call fast float @_Z5exp10f(float %tmp)
+  %call = call fast float @_Z5exp10f(float %tmp)
   store float %call, float addrspace(1)* %a, align 4
   ret void
 }
@@ -580,11 +580,11 @@ entry:
 declare float @_Z5exp10f(float)
 
 ; GCN-LABEL: {{^}}define amdgpu_kernel void @test_use_native_log
-; GCN-NATIVE: tail call fast float @_Z10native_logf(float %tmp)
+; GCN-NATIVE: call fast float @_Z10native_logf(float %tmp)
 define amdgpu_kernel void @test_use_native_log(float addrspace(1)* nocapture %a) {
 entry:
   %tmp = load float, float addrspace(1)* %a, align 4
-  %call = tail call fast float @_Z3logf(float %tmp)
+  %call = call fast float @_Z3logf(float %tmp)
   store float %call, float addrspace(1)* %a, align 4
   ret void
 }
@@ -592,11 +592,11 @@ entry:
 declare float @_Z3logf(float)
 
 ; GCN-LABEL: {{^}}define amdgpu_kernel void @test_use_native_log2
-; GCN-NATIVE: tail call fast float @_Z11native_log2f(float %tmp)
+; GCN-NATIVE: call fast float @_Z11native_log2f(float %tmp)
 define amdgpu_kernel void @test_use_native_log2(float addrspace(1)* nocapture %a) {
 entry:
   %tmp = load float, float addrspace(1)* %a, align 4
-  %call = tail call fast float @_Z4log2f(float %tmp)
+  %call = call fast float @_Z4log2f(float %tmp)
   store float %call, float addrspace(1)* %a, align 4
   ret void
 }
@@ -604,11 +604,11 @@ entry:
 declare float @_Z4log2f(float)
 
 ; GCN-LABEL: {{^}}define amdgpu_kernel void @test_use_native_log10
-; GCN-NATIVE: tail call fast float @_Z12native_log10f(float %tmp)
+; GCN-NATIVE: call fast float @_Z12native_log10f(float %tmp)
 define amdgpu_kernel void @test_use_native_log10(float addrspace(1)* nocapture %a) {
 entry:
   %tmp = load float, float addrspace(1)* %a, align 4
-  %call = tail call fast float @_Z5log10f(float %tmp)
+  %call = call fast float @_Z5log10f(float %tmp)
   store float %call, float addrspace(1)* %a, align 4
   ret void
 }
@@ -617,36 +617,36 @@ declare float @_Z5log10f(float)
 
 ; GCN-LABEL: {{^}}define amdgpu_kernel void @test_use_native_powr
 ; GCN-NATIVE: %tmp1 = load float, float addrspace(1)* %arrayidx1, align 4
-; GCN-NATIVE: %__log2 = tail call fast float @_Z11native_log2f(float %tmp)
+; GCN-NATIVE: %__log2 = call fast float @_Z11native_log2f(float %tmp)
 ; GCN-NATIVE: %__ylogx = fmul fast float %__log2, %tmp1
-; GCN-NATIVE: %__exp2 = tail call fast float @_Z11native_exp2f(float %__ylogx)
+; GCN-NATIVE: %__exp2 = call fast float @_Z11native_exp2f(float %__ylogx)
 ; GCN-NATIVE: store float %__exp2, float addrspace(1)* %a, align 4
 define amdgpu_kernel void @test_use_native_powr(float addrspace(1)* nocapture %a) {
 entry:
   %tmp = load float, float addrspace(1)* %a, align 4
   %arrayidx1 = getelementptr inbounds float, float addrspace(1)* %a, i64 1
   %tmp1 = load float, float addrspace(1)* %arrayidx1, align 4
-  %call = tail call fast float @_Z4powrff(float %tmp, float %tmp1)
+  %call = call fast float @_Z4powrff(float %tmp, float %tmp1)
   store float %call, float addrspace(1)* %a, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}define amdgpu_kernel void @test_use_native_sqrt
-; GCN-NATIVE: tail call fast float @_Z11native_sqrtf(float %tmp)
+; GCN-NATIVE: call fast float @_Z11native_sqrtf(float %tmp)
 define amdgpu_kernel void @test_use_native_sqrt(float addrspace(1)* nocapture %a) {
 entry:
   %tmp = load float, float addrspace(1)* %a, align 4
-  %call = tail call fast float @_Z4sqrtf(float %tmp)
+  %call = call fast float @_Z4sqrtf(float %tmp)
   store float %call, float addrspace(1)* %a, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}define amdgpu_kernel void @test_dont_use_native_sqrt_fast_f64
-; GCN: tail call fast double @_Z4sqrtd(double %tmp)
+; GCN: call fast double @_Z4sqrtd(double %tmp)
 define amdgpu_kernel void @test_dont_use_native_sqrt_fast_f64(double addrspace(1)* nocapture %a) {
 entry:
   %tmp = load double, double addrspace(1)* %a, align 8
-  %call = tail call fast double @_Z4sqrtd(double %tmp)
+  %call = call fast double @_Z4sqrtd(double %tmp)
   store double %call, double addrspace(1)* %a, align 8
   ret void
 }
@@ -655,11 +655,11 @@ declare float @_Z4sqrtf(float)
 declare double @_Z4sqrtd(double)
 
 ; GCN-LABEL: {{^}}define amdgpu_kernel void @test_use_native_rsqrt
-; GCN-NATIVE: tail call fast float @_Z12native_rsqrtf(float %tmp)
+; GCN-NATIVE: call fast float @_Z12native_rsqrtf(float %tmp)
 define amdgpu_kernel void @test_use_native_rsqrt(float addrspace(1)* nocapture %a) {
 entry:
   %tmp = load float, float addrspace(1)* %a, align 4
-  %call = tail call fast float @_Z5rsqrtf(float %tmp)
+  %call = call fast float @_Z5rsqrtf(float %tmp)
   store float %call, float addrspace(1)* %a, align 4
   ret void
 }
@@ -667,11 +667,11 @@ entry:
 declare float @_Z5rsqrtf(float)
 
 ; GCN-LABEL: {{^}}define amdgpu_kernel void @test_use_native_tan
-; GCN-NATIVE: tail call fast float @_Z10native_tanf(float %tmp)
+; GCN-NATIVE: call fast float @_Z10native_tanf(float %tmp)
 define amdgpu_kernel void @test_use_native_tan(float addrspace(1)* nocapture %a) {
 entry:
   %tmp = load float, float addrspace(1)* %a, align 4
-  %call = tail call fast float @_Z3tanf(float %tmp)
+  %call = call fast float @_Z3tanf(float %tmp)
   store float %call, float addrspace(1)* %a, align 4
   ret void
 }
@@ -679,14 +679,14 @@ entry:
 declare float @_Z3tanf(float)
 
 ; GCN-LABEL: {{^}}define amdgpu_kernel void @test_use_native_sincos
-; GCN-NATIVE: tail call float @_Z10native_sinf(float %tmp)
-; GCN-NATIVE: tail call float @_Z10native_cosf(float %tmp)
+; GCN-NATIVE: call float @_Z10native_sinf(float %tmp)
+; GCN-NATIVE: call float @_Z10native_cosf(float %tmp)
 define amdgpu_kernel void @test_use_native_sincos(float addrspace(1)* %a) {
 entry:
   %tmp = load float, float addrspace(1)* %a, align 4
   %arrayidx1 = getelementptr inbounds float, float addrspace(1)* %a, i64 1
   %tmp1 = addrspacecast float addrspace(1)* %arrayidx1 to float*
-  %call = tail call fast float @_Z6sincosfPf(float %tmp, float* %tmp1)
+  %call = call fast float @_Z6sincosfPf(float %tmp, float* %tmp1)
   store float %call, float addrspace(1)* %a, align 4
   ret void
 }
@@ -703,10 +703,10 @@ define amdgpu_kernel void @test_read_pipe(%opencl.pipe_t addrspace(1)* %p, i32 a
 entry:
   %tmp = bitcast i32 addrspace(1)* %ptr to i8 addrspace(1)*
   %tmp1 = addrspacecast i8 addrspace(1)* %tmp to i8*
-  %tmp2 = tail call i32 @__read_pipe_2(%opencl.pipe_t addrspace(1)* %p, i8* %tmp1, i32 4, i32 4) #0
-  %tmp3 = tail call %opencl.reserve_id_t addrspace(5)* @__reserve_read_pipe(%opencl.pipe_t addrspace(1)* %p, i32 2, i32 4, i32 4)
-  %tmp4 = tail call i32 @__read_pipe_4(%opencl.pipe_t addrspace(1)* %p, %opencl.reserve_id_t addrspace(5)* %tmp3, i32 2, i8* %tmp1, i32 4, i32 4) #0
-  tail call void @__commit_read_pipe(%opencl.pipe_t addrspace(1)* %p, %opencl.reserve_id_t addrspace(5)* %tmp3, i32 4, i32 4)
+  %tmp2 = call i32 @__read_pipe_2(%opencl.pipe_t addrspace(1)* %p, i8* %tmp1, i32 4, i32 4) #0
+  %tmp3 = call %opencl.reserve_id_t addrspace(5)* @__reserve_read_pipe(%opencl.pipe_t addrspace(1)* %p, i32 2, i32 4, i32 4)
+  %tmp4 = call i32 @__read_pipe_4(%opencl.pipe_t addrspace(1)* %p, %opencl.reserve_id_t addrspace(5)* %tmp3, i32 2, i8* %tmp1, i32 4, i32 4) #0
+  call void @__commit_read_pipe(%opencl.pipe_t addrspace(1)* %p, %opencl.reserve_id_t addrspace(5)* %tmp3, i32 4, i32 4)
   ret void
 }
 
@@ -725,10 +725,10 @@ define amdgpu_kernel void @test_write_pipe(%opencl.pipe_t addrspace(1)* %p, i32
 entry:
   %tmp = bitcast i32 addrspace(1)* %ptr to i8 addrspace(1)*
   %tmp1 = addrspacecast i8 addrspace(1)* %tmp to i8*
-  %tmp2 = tail call i32 @__write_pipe_2(%opencl.pipe_t addrspace(1)* %p, i8* %tmp1, i32 4, i32 4) #0
-  %tmp3 = tail call %opencl.reserve_id_t addrspace(5)* @__reserve_write_pipe(%opencl.pipe_t addrspace(1)* %p, i32 2, i32 4, i32 4) #0
-  %tmp4 = tail call i32 @__write_pipe_4(%opencl.pipe_t addrspace(1)* %p, %opencl.reserve_id_t addrspace(5)* %tmp3, i32 2, i8* %tmp1, i32 4, i32 4) #0
-  tail call void @__commit_write_pipe(%opencl.pipe_t addrspace(1)* %p, %opencl.reserve_id_t addrspace(5)* %tmp3, i32 4, i32 4) #0
+  %tmp2 = call i32 @__write_pipe_2(%opencl.pipe_t addrspace(1)* %p, i8* %tmp1, i32 4, i32 4) #0
+  %tmp3 = call %opencl.reserve_id_t addrspace(5)* @__reserve_write_pipe(%opencl.pipe_t addrspace(1)* %p, i32 2, i32 4, i32 4) #0
+  %tmp4 = call i32 @__write_pipe_4(%opencl.pipe_t addrspace(1)* %p, %opencl.reserve_id_t addrspace(5)* %tmp3, i32 2, i8* %tmp1, i32 4, i32 4) #0
+  call void @__commit_write_pipe(%opencl.pipe_t addrspace(1)* %p, %opencl.reserve_id_t addrspace(5)* %tmp3, i32 4, i32 4) #0
   ret void
 }
 
@@ -755,31 +755,31 @@ declare void @__commit_write_pipe(%opencl.pipe_t addrspace(1)*, %opencl.reserve_
 define amdgpu_kernel void @test_pipe_size(%opencl.pipe_t addrspace(1)* %p1, i8 addrspace(1)* %ptr1, %opencl.pipe_t addrspace(1)* %p2, i16 addrspace(1)* %ptr2, %opencl.pipe_t addrspace(1)* %p4, i32 addrspace(1)* %ptr4, %opencl.pipe_t addrspace(1)* %p8, i64 addrspace(1)* %ptr8, %opencl.pipe_t addrspace(1)* %p16, <2 x i64> addrspace(1)* %ptr16, %opencl.pipe_t addrspace(1)* %p32, <4 x i64> addrspace(1)* %ptr32, %opencl.pipe_t addrspace(1)* %p64, <8 x i64> addrspace(1)* %ptr64, %opencl.pipe_t addrspace(1)* %p128, <16 x i64> addrspace(1)* %ptr128, %opencl.pipe_t addrspace(1)* %pu, %struct.S addrspace(1)* %ptru) local_unnamed_addr #0 {
 entry:
   %tmp = addrspacecast i8 addrspace(1)* %ptr1 to i8*
-  %tmp1 = tail call i32 @__read_pipe_2(%opencl.pipe_t addrspace(1)* %p1, i8* %tmp, i32 1, i32 1) #0
+  %tmp1 = call i32 @__read_pipe_2(%opencl.pipe_t addrspace(1)* %p1, i8* %tmp, i32 1, i32 1) #0
   %tmp2 = bitcast i16 addrspace(1)* %ptr2 to i8 addrspace(1)*
   %tmp3 = addrspacecast i8 addrspace(1)* %tmp2 to i8*
-  %tmp4 = tail call i32 @__read_pipe_2(%opencl.pipe_t addrspace(1)* %p2, i8* %tmp3, i32 2, i32 2) #0
+  %tmp4 = call i32 @__read_pipe_2(%opencl.pipe_t addrspace(1)* %p2, i8* %tmp3, i32 2, i32 2) #0
   %tmp5 = bitcast i32 addrspace(1)* %ptr4 to i8 addrspace(1)*
   %tmp6 = addrspacecast i8 addrspace(1)* %tmp5 to i8*
-  %tmp7 = tail call i32 @__read_pipe_2(%opencl.pipe_t addrspace(1)* %p4, i8* %tmp6, i32 4, i32 4) #0
+  %tmp7 = call i32 @__read_pipe_2(%opencl.pipe_t addrspace(1)* %p4, i8* %tmp6, i32 4, i32 4) #0
   %tmp8 = bitcast i64 addrspace(1)* %ptr8 to i8 addrspace(1)*
   %tmp9 = addrspacecast i8 addrspace(1)* %tmp8 to i8*
-  %tmp10 = tail call i32 @__read_pipe_2(%opencl.pipe_t addrspace(1)* %p8, i8* %tmp9, i32 8, i32 8) #0
+  %tmp10 = call i32 @__read_pipe_2(%opencl.pipe_t addrspace(1)* %p8, i8* %tmp9, i32 8, i32 8) #0
   %tmp11 = bitcast <2 x i64> addrspace(1)* %ptr16 to i8 addrspace(1)*
   %tmp12 = addrspacecast i8 addrspace(1)* %tmp11 to i8*
-  %tmp13 = tail call i32 @__read_pipe_2(%opencl.pipe_t addrspace(1)* %p16, i8* %tmp12, i32 16, i32 16) #0
+  %tmp13 = call i32 @__read_pipe_2(%opencl.pipe_t addrspace(1)* %p16, i8* %tmp12, i32 16, i32 16) #0
   %tmp14 = bitcast <4 x i64> addrspace(1)* %ptr32 to i8 addrspace(1)*
   %tmp15 = addrspacecast i8 addrspace(1)* %tmp14 to i8*
-  %tmp16 = tail call i32 @__read_pipe_2(%opencl.pipe_t addrspace(1)* %p32, i8* %tmp15, i32 32, i32 32) #0
+  %tmp16 = call i32 @__read_pipe_2(%opencl.pipe_t addrspace(1)* %p32, i8* %tmp15, i32 32, i32 32) #0
   %tmp17 = bitcast <8 x i64> addrspace(1)* %ptr64 to i8 addrspace(1)*
   %tmp18 = addrspacecast i8 addrspace(1)* %tmp17 to i8*
-  %tmp19 = tail call i32 @__read_pipe_2(%opencl.pipe_t addrspace(1)* %p64, i8* %tmp18, i32 64, i32 64) #0
+  %tmp19 = call i32 @__read_pipe_2(%opencl.pipe_t addrspace(1)* %p64, i8* %tmp18, i32 64, i32 64) #0
   %tmp20 = bitcast <16 x i64> addrspace(1)* %ptr128 to i8 addrspace(1)*
   %tmp21 = addrspacecast i8 addrspace(1)* %tmp20 to i8*
-  %tmp22 = tail call i32 @__read_pipe_2(%opencl.pipe_t addrspace(1)* %p128, i8* %tmp21, i32 128, i32 128) #0
+  %tmp22 = call i32 @__read_pipe_2(%opencl.pipe_t addrspace(1)* %p128, i8* %tmp21, i32 128, i32 128) #0
   %tmp23 = bitcast %struct.S addrspace(1)* %ptru to i8 addrspace(1)*
   %tmp24 = addrspacecast i8 addrspace(1)* %tmp23 to i8*
-  %tmp25 = tail call i32 @__read_pipe_2(%opencl.pipe_t addrspace(1)* %pu, i8* %tmp24, i32 400, i32 4) #0
+  %tmp25 = call i32 @__read_pipe_2(%opencl.pipe_t addrspace(1)* %pu, i8* %tmp24, i32 400, i32 4) #0
   ret void
 }
 
diff --git a/llvm/test/CodeGen/AMDGPU/sint_to_fp.f64.ll b/llvm/test/CodeGen/AMDGPU/sint_to_fp.f64.ll
index f98a716b4..663831f06 100644
--- a/llvm/test/CodeGen/AMDGPU/sint_to_fp.f64.ll
+++ b/llvm/test/CodeGen/AMDGPU/sint_to_fp.f64.ll
@@ -1,9 +1,10 @@
-; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=SI %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=hawaii -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,SI %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=fiji -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,VI %s
 
 declare i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
 
-; SI-LABEL: {{^}}sint_to_fp_i32_to_f64
-; SI: v_cvt_f64_i32_e32
+; GCN-LABEL: {{^}}sint_to_fp_i32_to_f64
+; GCN: v_cvt_f64_i32_e32
 define amdgpu_kernel void @sint_to_fp_i32_to_f64(double addrspace(1)* %out, i32 %in) {
   %result = sitofp i32 %in to double
   store double %result, double addrspace(1)* %out
@@ -13,12 +14,12 @@ define amdgpu_kernel void @sint_to_fp_i32_to_f64(double addrspace(1)* %out, i32
 ; We can't fold the SGPRs into v_cndmask_b32_e64, because it already
 ; uses an SGPR (implicit vcc).
 
-; SI-LABEL: {{^}}sint_to_fp_i1_f64:
-; SI-DAG: v_cmp_eq_u32_e64 vcc,
-; SI-DAG: v_cndmask_b32_e32 v[[SEL:[0-9]+]], 0, v{{[0-9]+}}
-; SI-DAG: v_mov_b32_e32 v[[ZERO:[0-9]+]], 0{{$}}
-; SI: buffer_store_dwordx2 v{{\[}}[[ZERO]]:[[SEL]]{{\]}}
-; SI: s_endpgm
+; GCN-LABEL: {{^}}sint_to_fp_i1_f64:
+; GCN-DAG: v_cmp_eq_u32_e64 vcc,
+; GCN-DAG: v_cndmask_b32_e32 v[[SEL:[0-9]+]], 0, v{{[0-9]+}}
+; GCN-DAG: v_mov_b32_e32 v[[ZERO:[0-9]+]], 0{{$}}
+; GCN: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[ZERO]]:[[SEL]]{{\]}}
+; GCN: s_endpgm
 define amdgpu_kernel void @sint_to_fp_i1_f64(double addrspace(1)* %out, i32 %in) {
   %cmp = icmp eq i32 %in, 0
   %fp = sitofp i1 %cmp to double
@@ -26,31 +27,31 @@ define amdgpu_kernel void @sint_to_fp_i1_f64(double addrspace(1)* %out, i32 %in)
   ret void
 }
 
-; SI-LABEL: {{^}}sint_to_fp_i1_f64_load:
-; SI: v_cndmask_b32_e64 [[IRESULT:v[0-9]]], 0, -1
-; SI-NEXT: v_cvt_f64_i32_e32 [[RESULT:v\[[0-9]+:[0-9]\]]], [[IRESULT]]
-; SI: buffer_store_dwordx2 [[RESULT]]
-; SI: s_endpgm
+; GCN-LABEL: {{^}}sint_to_fp_i1_f64_load:
+; GCN: v_cndmask_b32_e64 [[IRESULT:v[0-9]]], 0, -1
+; GCN: v_cvt_f64_i32_e32 [[RESULT:v\[[0-9]+:[0-9]\]]], [[IRESULT]]
+; GCN: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]]
+; GCN: s_endpgm
 define amdgpu_kernel void @sint_to_fp_i1_f64_load(double addrspace(1)* %out, i1 %in) {
   %fp = sitofp i1 %in to double
   store double %fp, double addrspace(1)* %out, align 8
   ret void
 }
 
-; SI-LABEL: @s_sint_to_fp_i64_to_f64
+; GCN-LABEL: @s_sint_to_fp_i64_to_f64
 define amdgpu_kernel void @s_sint_to_fp_i64_to_f64(double addrspace(1)* %out, i64 %in) {
   %result = sitofp i64 %in to double
   store double %result, double addrspace(1)* %out
   ret void
 }
 
-; SI-LABEL: @v_sint_to_fp_i64_to_f64
-; SI: buffer_load_dwordx2 v{{\[}}[[LO:[0-9]+]]:[[HI:[0-9]+]]{{\]}}
-; SI-DAG: v_cvt_f64_i32_e32 [[HI_CONV:v\[[0-9]+:[0-9]+\]]], v[[HI]]
-; SI-DAG: v_cvt_f64_u32_e32 [[LO_CONV:v\[[0-9]+:[0-9]+\]]], v[[LO]]
-; SI-DAG: v_ldexp_f64 [[LDEXP:v\[[0-9]+:[0-9]+\]]], [[HI_CONV]], 32
-; SI: v_add_f64 [[RESULT:v\[[0-9]+:[0-9]+\]]], [[LDEXP]], [[LO_CONV]]
-; SI: buffer_store_dwordx2 [[RESULT]]
+; GCN-LABEL: @v_sint_to_fp_i64_to_f64
+; GCN: flat_load_dwordx2 v{{\[}}[[LO:[0-9]+]]:[[HI:[0-9]+]]{{\]}}
+; GCN-DAG: v_cvt_f64_i32_e32 [[HI_CONV:v\[[0-9]+:[0-9]+\]]], v[[HI]]
+; GCN-DAG: v_cvt_f64_u32_e32 [[LO_CONV:v\[[0-9]+:[0-9]+\]]], v[[LO]]
+; GCN-DAG: v_ldexp_f64 [[LDEXP:v\[[0-9]+:[0-9]+\]]], [[HI_CONV]], 32
+; GCN: v_add_f64 [[RESULT:v\[[0-9]+:[0-9]+\]]], [[LDEXP]], [[LO_CONV]]
+; GCN: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]]
 define amdgpu_kernel void @v_sint_to_fp_i64_to_f64(double addrspace(1)* %out, i64 addrspace(1)* %in) {
   %tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
   %gep = getelementptr i64, i64 addrspace(1)* %in, i32 %tid
@@ -59,3 +60,27 @@ define amdgpu_kernel void @v_sint_to_fp_i64_to_f64(double addrspace(1)* %out, i6
   store double %result, double addrspace(1)* %out
   ret void
 }
+
+; FIXME: bfe and sext on VI+
+; GCN-LABEL: {{^}}s_sint_to_fp_i8_to_f64:
+; GCN: s_load_dword [[VAL:s[0-9]+]]
+; SI-NOT: bfe
+; SI: s_sext_i32_i8 [[SEXT:s[0-9]+]], [[VAL]]
+
+; VI: s_bfe_i32 [[BFE:s[0-9]+]], [[VAL]], 0x80000
+; VI: s_sext_i32_i16 [[SEXT:s[0-9]+]], [[BFE]]
+
+; GCN: v_cvt_f64_i32_e32 v{{\[[0-9]+:[0-9]+\]}}, [[SEXT]]
+define amdgpu_kernel void @s_sint_to_fp_i8_to_f64(double addrspace(1)* %out, i8 %in) {
+  %fp = sitofp i8 %in to double
+  store double %fp, double addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}v_sint_to_fp_i8_to_f64:
+; GCN: v_bfe_i32 [[SEXT:v[0-9]+]]
+; GCN: v_cvt_f64_i32_e32 v{{\[[0-9]+:[0-9]+\]}}, [[SEXT]]
+define double @v_sint_to_fp_i8_to_f64(i8 %in) {
+  %fp = sitofp i8 %in to double
+  ret double %fp
+}
diff --git a/llvm/test/CodeGen/AMDGPU/sint_to_fp.ll b/llvm/test/CodeGen/AMDGPU/sint_to_fp.ll
index 8e85d9998..e9fd773a1 100644
--- a/llvm/test/CodeGen/AMDGPU/sint_to_fp.ll
+++ b/llvm/test/CodeGen/AMDGPU/sint_to_fp.ll
@@ -78,7 +78,7 @@ define amdgpu_kernel void @v_sint_to_fp_v4i32(<4 x float> addrspace(1)* %out, <4
 
 ; FUNC-LABEL: {{^}}s_sint_to_fp_i1_f32:
 ; SI: v_cmp_eq_u32_e64 [[CMP:s\[[0-9]+:[0-9]\]]],
-; SI-NEXT: v_cndmask_b32_e64 [[RESULT:v[0-9]+]], 0, 1.0, [[CMP]]
+; SI: v_cndmask_b32_e64 [[RESULT:v[0-9]+]], 0, 1.0, [[CMP]]
 ; SI: buffer_store_dword [[RESULT]],
 ; SI: s_endpgm
 define amdgpu_kernel void @s_sint_to_fp_i1_f32(float addrspace(1)* %out, i32 %in) #0 {
diff --git a/llvm/test/CodeGen/AMDGPU/sitofp.f16.ll b/llvm/test/CodeGen/AMDGPU/sitofp.f16.ll
index 231d3a6d1..3a3e0def3 100644
--- a/llvm/test/CodeGen/AMDGPU/sitofp.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/sitofp.f16.ll
@@ -3,8 +3,12 @@
 
 ; GCN-LABEL: {{^}}sitofp_i16_to_f16
 ; GCN: buffer_load_{{sshort|ushort}} v[[A_I16:[0-9]+]]
-; GCN: v_cvt_f32_i32_e32 v[[A_F32:[0-9]+]], v[[A_I16]]
-; GCN: v_cvt_f16_f32_e32 v[[R_F16:[0-9]+]], v[[A_F32]]
+
+; SI: v_cvt_f32_i32_e32 v[[A_F32:[0-9]+]], v[[A_I16]]
+; SI: v_cvt_f16_f32_e32 v[[R_F16:[0-9]+]], v[[A_F32]]
+
+; VI: v_cvt_f16_i16_e32 v[[R_F16:[0-9]+]], v[[A_I16]]
+
 ; GCN: buffer_store_short v[[R_F16]]
 ; GCN: s_endpgm
 define amdgpu_kernel void @sitofp_i16_to_f16(
@@ -45,10 +49,8 @@ entry:
 ; SI-DAG: v_lshlrev_b32_e32
 ; SI: v_or_b32_e32
 
-; VI-DAG: v_cvt_f32_i32_sdwa
-; VI-DAG: v_cvt_f32_i32_sdwa
-; VI-DAG: v_cvt_f16_f32_e32
-; VI-DAG: v_cvt_f16_f32_sdwa
+; VI-DAG: v_cvt_f16_i16_sdwa v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-DAG: v_cvt_f16_i16_e32
 ; VI:     v_or_b32_e32
 
 ; GCN: buffer_store_dword
diff --git a/llvm/test/CodeGen/AMDGPU/sminmax.ll b/llvm/test/CodeGen/AMDGPU/sminmax.ll
index ea4b70012..e56db7edb 100644
--- a/llvm/test/CodeGen/AMDGPU/sminmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/sminmax.ll
@@ -1,7 +1,7 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,SIVI,FUNC %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,SIVI,FUNC %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9,FUNC %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=r600 -mcpu=cypress -verify-machineinstrs < %s | FileCheck -check-prefixes=EG,FUNC %s
+; RUN:  llc -march=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,SIVI,FUNC %s
+; RUN:  llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,SIVI,FUNC %s
+; RUN:  llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9,FUNC %s
+; RUN:  llc -march=r600 -mcpu=cypress -verify-machineinstrs < %s | FileCheck -check-prefixes=EG,FUNC %s
 
 ; FUNC-LABEL: {{^}}s_abs_i32:
 ; GCN: s_abs_i32
@@ -28,7 +28,9 @@ define amdgpu_kernel void @s_abs_i32(i32 addrspace(1)* %out, i32 %val) nounwind
 
 ; EG: MAX_INT
 define amdgpu_kernel void @v_abs_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %src) nounwind {
-  %val = load i32, i32 addrspace(1)* %src, align 4
+  %tid = call i32 @llvm.r600.read.tidig.x()
+  %gep.in = getelementptr inbounds i32, i32 addrspace(1)* %src, i32 %tid
+  %val = load i32, i32 addrspace(1)* %gep.in, align 4
   %neg = sub i32 0, %val
   %cond = icmp sgt i32 %val, %neg
   %res = select i1 %cond, i32 %val, i32 %neg
@@ -43,7 +45,9 @@ define amdgpu_kernel void @v_abs_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %
 ; GCN: v_max_i32_e32 [[MAX:v[0-9]+]], [[SRC]], [[NEG]]
 ; GCN: v_mul_lo_u32 v{{[0-9]+}}, [[MAX]], [[MAX]]
 define amdgpu_kernel void @v_abs_i32_repeat_user(i32 addrspace(1)* %out, i32 addrspace(1)* %src) nounwind {
-  %val = load i32, i32 addrspace(1)* %src, align 4
+  %tid = call i32 @llvm.r600.read.tidig.x()
+  %gep.in = getelementptr inbounds i32, i32 addrspace(1)* %src, i32 %tid
+  %val = load i32, i32 addrspace(1)* %gep.in, align 4
   %neg = sub i32 0, %val
   %cond = icmp sgt i32 %val, %neg
   %res = select i1 %cond, i32 %val, i32 %neg
@@ -96,7 +100,9 @@ define amdgpu_kernel void @v_abs_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32> a
   %z1 = insertelement <2 x i32> %z0, i32 0, i32 1
   %t0 = insertelement <2 x i32> undef, i32 2, i32 0
   %t1 = insertelement <2 x i32> %t0, i32 2, i32 1
-  %val = load <2 x i32>, <2 x i32> addrspace(1)* %src, align 4
+  %tid = call i32 @llvm.r600.read.tidig.x()
+  %gep.in = getelementptr inbounds <2 x i32>, <2 x i32> addrspace(1)* %src, i32 %tid
+  %val = load <2 x i32>, <2 x i32> addrspace(1)* %gep.in, align 4
   %neg = sub <2 x i32> %z1, %val
   %cond = icmp sgt <2 x i32> %val, %neg
   %res = select <2 x i1> %cond, <2 x i32> %val, <2 x i32> %neg
@@ -178,7 +184,9 @@ define amdgpu_kernel void @v_abs_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> a
   %t1 = insertelement <4 x i32> %t0, i32 2, i32 1
   %t2 = insertelement <4 x i32> %t1, i32 2, i32 2
   %t3 = insertelement <4 x i32> %t2, i32 2, i32 3
-  %val = load <4 x i32>, <4 x i32> addrspace(1)* %src, align 4
+  %tid = call i32 @llvm.r600.read.tidig.x()
+  %gep.in = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* %src, i32 %tid
+  %val = load <4 x i32>, <4 x i32> addrspace(1)* %gep.in, align 4
   %neg = sub <4 x i32> %z3, %val
   %cond = icmp sgt <4 x i32> %val, %neg
   %res = select <4 x i1> %cond, <4 x i32> %val, <4 x i32> %neg
@@ -259,3 +267,8 @@ define amdgpu_kernel void @v_min_max_i32_user(i32 addrspace(1)* %out0, i32 addrs
   store volatile i1 %cond0, i1 addrspace(1)* undef
   ret void
 }
+
+declare i32 @llvm.r600.read.tidig.x() #0
+
+attributes #0 = { nounwind readnone }
+attributes #1 = { nounwind }
diff --git a/llvm/test/CodeGen/AMDGPU/sminmax.v2i16.ll b/llvm/test/CodeGen/AMDGPU/sminmax.v2i16.ll
index 8efc1ce22..184082dd3 100644
--- a/llvm/test/CodeGen/AMDGPU/sminmax.v2i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/sminmax.v2i16.ll
@@ -1,6 +1,6 @@
-; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX9,GCN %s
-; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=VI,CIVI,GCN %s
-; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=bonaire -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=CI,CIVI,GCN %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX9,GCN %s
+; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=VI,CIVI,GCN %s
+; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=CI,CIVI,GCN %s
 
 ; GCN-LABEL: {{^}}s_abs_v2i16:
 ; GFX9: s_load_dword [[VAL:s[0-9]+]]
@@ -85,7 +85,7 @@ define amdgpu_kernel void @s_abs_v2i16_2(<2 x i16> addrspace(1)* %out, <2 x i16>
 }
 
 ; GCN-LABEL: {{^}}v_abs_v2i16_2:
-; GFX9: buffer_load_dword [[VAL:v[0-9]+]]
+; GFX9: global_load_dword [[VAL:v[0-9]+]]
 ; GFX9: v_pk_sub_i16 [[SUB:v[0-9]+]], 0, [[VAL]]
 ; GFX9: v_pk_max_i16 [[MAX:v[0-9]+]], [[VAL]], [[SUB]]
 ; GFX9: v_pk_sub_u16 [[ADD:v[0-9]+]], [[MAX]], -2 op_sel_hi:[1,0]
@@ -94,7 +94,9 @@ define amdgpu_kernel void @v_abs_v2i16_2(<2 x i16> addrspace(1)* %out, <2 x i16>
   %z1 = insertelement <2 x i16> %z0, i16 0, i16 1
   %t0 = insertelement <2 x i16> undef, i16 2, i16 0
   %t1 = insertelement <2 x i16> %t0, i16 2, i16 1
-  %val = load <2 x i16>, <2 x i16> addrspace(1)* %src, align 4
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep.in = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %src, i32 %tid
+  %val = load <2 x i16>, <2 x i16> addrspace(1)* %gep.in, align 4
   %neg = sub <2 x i16> %z1, %val
   %cond = icmp sgt <2 x i16> %val, %neg
   %res = select <2 x i1> %cond, <2 x i16> %val, <2 x i16> %neg
@@ -129,7 +131,7 @@ define amdgpu_kernel void @s_abs_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> %
 }
 
 ; GCN-LABEL: {{^}}v_abs_v4i16:
-; GFX9: buffer_load_dwordx2 v{{\[}}[[VAL0:[0-9]+]]:[[VAL1:[0-9]+]]{{\]}}
+; GFX9: global_load_dwordx2 v{{\[}}[[VAL0:[0-9]+]]:[[VAL1:[0-9]+]]{{\]}}
 
 ; GFX9-DAG: v_pk_sub_i16 [[SUB0:v[0-9]+]], 0, v[[VAL0]]
 ; GFX9-DAG: v_pk_max_i16 [[MAX0:v[0-9]+]], v[[VAL0]], [[SUB0]]
@@ -147,7 +149,9 @@ define amdgpu_kernel void @v_abs_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> a
   %t1 = insertelement <4 x i16> %t0, i16 2, i16 1
   %t2 = insertelement <4 x i16> %t1, i16 2, i16 2
   %t3 = insertelement <4 x i16> %t2, i16 2, i16 3
-  %val = load <4 x i16>, <4 x i16> addrspace(1)* %src, align 4
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep.in = getelementptr inbounds <4 x i16>, <4 x i16> addrspace(1)* %src, i32 %tid
+  %val = load <4 x i16>, <4 x i16> addrspace(1)* %gep.in, align 4
   %neg = sub <4 x i16> %z3, %val
   %cond = icmp sgt <4 x i16> %val, %neg
   %res = select <4 x i1> %cond, <4 x i16> %val, <4 x i16> %neg
diff --git a/llvm/test/CodeGen/AMDGPU/smrd-vccz-bug.ll b/llvm/test/CodeGen/AMDGPU/smrd-vccz-bug.ll
index 823785dc7..146e631ee 100644
--- a/llvm/test/CodeGen/AMDGPU/smrd-vccz-bug.ll
+++ b/llvm/test/CodeGen/AMDGPU/smrd-vccz-bug.ll
@@ -1,13 +1,13 @@
 ; RUN: llc -march=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VCCZ-BUG %s
 ; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VCCZ-BUG %s
-; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=NOVCCZ-BUG %s
+; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
 
 ; GCN-FUNC: {{^}}vccz_workaround:
 ; GCN: s_load_dword s{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], 0x0
 ; GCN: v_cmp_neq_f32_e64 {{[^,]*}}, s{{[0-9]+}}, 0{{$}}
 ; VCCZ-BUG: s_waitcnt lgkmcnt(0)
 ; VCCZ-BUG: s_mov_b64 vcc, vcc
-; NOVCCZ-BUG-NOT: s_mov_b64 vcc, vcc
+; GCN-NOT: s_mov_b64 vcc, vcc
 ; GCN: s_cbranch_vccnz [[EXIT:[0-9A-Za-z_]+]]
 ; GCN: buffer_store_dword
 ; GCN: [[EXIT]]:
@@ -28,6 +28,8 @@ endif:
 
 ; GCN-FUNC: {{^}}vccz_noworkaround:
 ; GCN: v_cmp_neq_f32_e32 vcc, 0, v{{[0-9]+}}
+; GCN-NOT: s_waitcnt lgkmcnt(0)
+; GCN-NOT: s_mov_b64 vcc, vcc
 ; GCN: s_cbranch_vccnz [[EXIT:[0-9A-Za-z_]+]]
 ; GCN: buffer_store_dword
 ; GCN: [[EXIT]]:
diff --git a/llvm/test/CodeGen/AMDGPU/smrd.ll b/llvm/test/CodeGen/AMDGPU/smrd.ll
index a76239940..ca4832c5e 100644
--- a/llvm/test/CodeGen/AMDGPU/smrd.ll
+++ b/llvm/test/CodeGen/AMDGPU/smrd.ll
@@ -573,7 +573,6 @@ main_body:
 ;
 ; TODO: we should keep the loop counter in an SGPR
 ;
-; GCN: v_readfirstlane_b32
 ; GCN: s_buffer_load_dword
 define amdgpu_ps float @smrd_uniform_loop(<4 x i32> inreg %desc, i32 %bound) #0 {
 main_body:
diff --git a/llvm/test/CodeGen/AMDGPU/smrd_vmem_war.ll b/llvm/test/CodeGen/AMDGPU/smrd_vmem_war.ll
new file mode 100644
index 000000000..4ba16b4eb
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/smrd_vmem_war.ll
@@ -0,0 +1,29 @@
+; RUN: llc  -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck %s -check-prefix=GCN
+
+; GCN-LABEL: BB0_1
+; GCN: s_load_dword s{{[0-9]+}}, s{{\[}}[[ADDR_LO:[0-9]+]]{{\:}}[[ADDR_HI:[0-9]+]]{{\]}}, 0x0
+; GCN: s_waitcnt lgkmcnt(0)
+; GCN: global_store_dword v{{\[}}[[ADDR_LO]]{{\:}}[[ADDR_HI]]{{\]}}, v{{[0-9]+}}, off
+
+define amdgpu_kernel void @zot(i32 addrspace(1)* nocapture %arg, i64 addrspace(1)* nocapture %arg1) {
+bb:
+  %tmp = call i32 @llvm.amdgcn.workitem.id.x()
+  %tmp2 = icmp eq i32 %tmp, 0
+  br i1 %tmp2, label %bb3, label %bb8
+
+bb3:                                              ; preds = %bb
+  %tmp4 = load i32, i32 addrspace(1)* %arg, align 4
+  store i32 0, i32 addrspace(1)* %arg, align 4
+  %tmp5 = zext i32 %tmp4 to i64
+  %tmp6 = load i64, i64 addrspace(1)* %arg1, align 8
+  %tmp7 = add i64 %tmp6, %tmp5
+  store i64 %tmp7, i64 addrspace(1)* %arg1, align 8
+  br label %bb8
+
+bb8:                                              ; preds = %bb3, %bb
+  ret void
+}
+; Function Attrs: nounwind readnone speculatable
+declare i32 @llvm.amdgcn.workitem.id.x() #0
+
+attributes #0 = { nounwind readnone speculatable }
diff --git a/llvm/test/CodeGen/AMDGPU/spill-m0.ll b/llvm/test/CodeGen/AMDGPU/spill-m0.ll
index 641b1c90f..2d7f7f9e3 100644
--- a/llvm/test/CodeGen/AMDGPU/spill-m0.ll
+++ b/llvm/test/CodeGen/AMDGPU/spill-m0.ll
@@ -1,14 +1,11 @@
 ; RUN: llc -O0 -amdgpu-spill-sgpr-to-vgpr=1 -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=TOVGPR -check-prefix=GCN %s
-; RUN: llc -O0 -amdgpu-spill-sgpr-to-vgpr=1 -amdgpu-spill-sgpr-to-smem=0 -march=amdgcn -mcpu=tonga  -verify-machineinstrs < %s | FileCheck -check-prefix=TOVGPR -check-prefix=GCN %s
+; RUN: llc -O0 -amdgpu-spill-sgpr-to-vgpr=1 -march=amdgcn -mcpu=tonga  -verify-machineinstrs < %s | FileCheck -check-prefix=TOVGPR -check-prefix=GCN %s
 ; RUN: llc -O0 -amdgpu-spill-sgpr-to-vgpr=0 -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=TOVMEM -check-prefix=GCN %s
-; RUN: llc -O0 -amdgpu-spill-sgpr-to-vgpr=0 -amdgpu-spill-sgpr-to-smem=0 -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=TOVMEM -check-prefix=GCN %s
-; RUN: llc -O0 -amdgpu-spill-sgpr-to-vgpr=0 -amdgpu-spill-sgpr-to-smem=1 -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=TOSMEM -check-prefix=GCN %s
+; RUN: llc -O0 -amdgpu-spill-sgpr-to-vgpr=0 -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=TOVMEM -check-prefix=GCN %s
 
 ; XXX - Why does it like to use vcc?
 
 ; GCN-LABEL: {{^}}spill_m0:
-; TOSMEM: s_mov_b32 s[[LO:[0-9]+]], SCRATCH_RSRC_DWORD0
-; TOSMEM: s_mov_b32 s[[HI:[0-9]+]], 0xe80000
 
 ; GCN-DAG: s_cmp_lg_u32
 
@@ -19,11 +16,6 @@
 ; TOVMEM-DAG: v_mov_b32_e32 [[SPILL_VREG:v[0-9]+]], [[M0_COPY]]
 ; TOVMEM: buffer_store_dword [[SPILL_VREG]], off, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offset:12 ; 4-byte Folded Spill
 
-; TOSMEM-DAG: s_mov_b32 [[M0_COPY:s[0-9]+]], m0
-; TOSMEM: s_add_u32 m0, s3, 0x300{{$}}
-; TOSMEM-NOT: [[M0_COPY]]
-; TOSMEM: s_buffer_store_dword [[M0_COPY]], s{{\[}}[[LO]]:[[HI]]], m0 ; 4-byte Folded Spill
-
 ; GCN: s_cbranch_scc1 [[ENDIF:BB[0-9]+_[0-9]+]]
 
 ; GCN: [[ENDIF]]:
@@ -35,11 +27,6 @@
 ; TOVMEM: v_readfirstlane_b32 [[M0_RESTORE:s[0-9]+]], [[RELOAD_VREG]]
 ; TOVMEM: s_mov_b32 m0, [[M0_RESTORE]]
 
-; TOSMEM: s_add_u32 m0, s3, 0x300{{$}}
-; TOSMEM: s_buffer_load_dword [[M0_RESTORE:s[0-9]+]], s{{\[}}[[LO]]:[[HI]]], m0 ; 4-byte Folded Reload
-; TOSMEM-NOT: [[M0_RESTORE]]
-; TOSMEM: s_mov_b32 m0, [[M0_RESTORE]]
-
 ; GCN: s_add_i32 s{{[0-9]+}}, m0, 1
 define amdgpu_kernel void @spill_m0(i32 %cond, i32 addrspace(1)* %out) #0 {
 entry:
@@ -64,26 +51,6 @@ endif:
 ; GCN: s_mov_b32 m0, s6
 ; GCN: v_interp_mov_f32
 
-; TOSMEM-NOT: s_m0
-; TOSMEM: s_add_u32 m0, s7, 0x100
-; TOSMEM-NEXT: s_buffer_store_dwordx2 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, m0 ; 8-byte Folded Spill
-; FIXME: RegScavenger::isRegUsed() always returns true if m0 is reserved, so we have to save and restore it
-; FIXME-TOSMEM-NOT: m0
-
-; FIXME-TOSMEM-NOT: m0
-; TOSMEM: s_add_u32 m0, s7, 0x300
-; TOSMEM: s_buffer_store_dword s{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}, m0 ; 4-byte Folded Spill
-; FIXME-TOSMEM-NOT: m0
-
-; TOSMEM: s_mov_b64 exec,
-; TOSMEM: s_cbranch_execz
-; TOSMEM: s_branch
-
-; TOSMEM: BB{{[0-9]+_[0-9]+}}:
-; TOSMEM: s_add_u32 m0, s7, 0x500
-; TOSMEM-NEXT: s_buffer_load_dwordx2 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, m0 ; 8-byte Folded Reload
-
-
 ; GCN-NOT: v_readlane_b32 m0
 ; GCN-NOT: s_buffer_store_dword m0
 ; GCN-NOT: s_buffer_load_dword m0
@@ -110,101 +77,6 @@ endif:                                            ; preds = %else, %if
   ret void
 }
 
-; Force save and restore of m0 during SMEM spill
-; GCN-LABEL: {{^}}m0_unavailable_spill:
-
-; GCN: ; def m0, 1
-
-; GCN: s_mov_b32 m0, s2
-; GCN: v_interp_mov_f32
-
-; GCN: ; clobber m0
-
-; TOSMEM: s_mov_b32 s2, m0
-; TOSMEM: s_add_u32 m0, s3, 0x100
-; TOSMEM-NEXT: s_buffer_store_dwordx2 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, m0 ; 8-byte Folded Spill
-; TOSMEM: s_mov_b32 m0, s2
-
-; TOSMEM: s_mov_b64 exec,
-; TOSMEM: s_cbranch_execz
-; TOSMEM: s_branch
-
-; TOSMEM: BB{{[0-9]+_[0-9]+}}:
-; TOSMEM: s_add_u32 m0, s3, 0x100
-; TOSMEM-NEXT: s_buffer_load_dwordx2 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, m0 ; 8-byte Folded Reload
-
-; GCN-NOT: v_readlane_b32 m0
-; GCN-NOT: s_buffer_store_dword m0
-; GCN-NOT: s_buffer_load_dword m0
-define amdgpu_kernel void @m0_unavailable_spill(i32 %m0.arg) #0 {
-main_body:
-  %m0 = call i32 asm sideeffect "; def $0, 1", "={m0}"() #0
-  %tmp = call float @llvm.amdgcn.interp.mov(i32 2, i32 0, i32 0, i32 %m0.arg)
-  call void asm sideeffect "; clobber $0", "~{m0}"() #0
-  %cmp = fcmp ueq float 0.000000e+00, %tmp
-   br i1 %cmp, label %if, label %else
-
-if:                                               ; preds = %main_body
-  store volatile i32 8, i32 addrspace(1)* undef
-  br label %endif
-
-else:                                             ; preds = %main_body
-  store volatile i32 11, i32 addrspace(1)* undef
-  br label %endif
-
-endif:
-  ret void
-}
-
-; GCN-LABEL: {{^}}restore_m0_lds:
-; TOSMEM: s_load_dwordx2 [[REG:s\[[0-9]+:[0-9]+\]]]
-; TOSMEM: s_cmp_eq_u32
-; FIXME: RegScavenger::isRegUsed() always returns true if m0 is reserved, so we have to save and restore it
-; FIXME-TOSMEM-NOT: m0
-; TOSMEM: s_add_u32 m0, s3, 0x100
-; TOSMEM: s_buffer_store_dword s{{[0-9]+}}, s[88:91], m0 ; 4-byte Folded Spill
-; FIXME-TOSMEM-NOT: m0
-; TOSMEM: s_add_u32 m0, s3, 0x200
-; TOSMEM: s_buffer_store_dwordx2 [[REG]], s[88:91], m0 ; 8-byte Folded Spill
-; FIXME-TOSMEM-NOT: m0
-; TOSMEM: s_cbranch_scc1
-
-; TOSMEM: s_mov_b32 m0, -1
-
-; TOSMEM: s_mov_b32 s0, m0
-; TOSMEM: s_add_u32 m0, s3, 0x200
-; TOSMEM: s_buffer_load_dwordx2 s{{\[[0-9]+:[0-9]+\]}}, s[88:91], m0 ; 8-byte Folded Reload
-; TOSMEM: s_mov_b32 m0, s0
-; TOSMEM: s_waitcnt lgkmcnt(0)
-
-; TOSMEM: ds_write_b64
-
-; FIXME-TOSMEM-NOT: m0
-; TOSMEM: s_add_u32 m0, s3, 0x100
-; TOSMEM: s_buffer_load_dword s0, s[88:91], m0 ; 4-byte Folded Reload
-; FIXME-TOSMEM-NOT: m0
-; TOSMEM: s_waitcnt lgkmcnt(0)
-; TOSMEM-NOT: m0
-; TOSMEM: s_mov_b32 m0, s0
-; TOSMEM: ; use m0
-
-; TOSMEM: s_dcache_wb
-; TOSMEM: s_endpgm
-define amdgpu_kernel void @restore_m0_lds(i32 %arg) {
-  %m0 = call i32 asm sideeffect "s_mov_b32 m0, 0", "={m0}"() #0
-  %sval = load volatile i64, i64 addrspace(4)* undef
-  %cmp = icmp eq i32 %arg, 0
-  br i1 %cmp, label %ret, label %bb
-
-bb:
-  store volatile i64 %sval, i64 addrspace(3)* undef
-  call void asm sideeffect "; use $0", "{m0}"(i32 %m0) #0
-  br label %ret
-
-ret:
-  ret void
-}
-
 declare float @llvm.amdgcn.interp.mov(i32, i32, i32, i32) #1
 declare void @llvm.amdgcn.exp.f32(i32, i32, float, float, float, float, i1, i1) #0
 declare void @llvm.amdgcn.exp.compr.v2f16(i32, i32, <2 x half>, <2 x half>, i1, i1) #0
diff --git a/llvm/test/CodeGen/AMDGPU/spill-vgpr-to-agpr.ll b/llvm/test/CodeGen/AMDGPU/spill-vgpr-to-agpr.ll
index 6eef782d1..78e2885c5 100644
--- a/llvm/test/CodeGen/AMDGPU/spill-vgpr-to-agpr.ll
+++ b/llvm/test/CodeGen/AMDGPU/spill-vgpr-to-agpr.ll
@@ -1,5 +1,5 @@
 ; RUN: llc -march=amdgcn -mcpu=gfx908 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX908 %s
-; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX900 %s
+; RUN: not llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s 2>&1 | FileCheck -check-prefixes=GCN,GFX900 %s
 
 ; GCN-LABEL: {{^}}max_10_vgprs:
 ; GFX900-DAG: s_mov_b32 s{{[0-9]+}}, SCRATCH_RSRC_DWORD0
@@ -57,23 +57,21 @@ define amdgpu_kernel void @max_10_vgprs(i32 addrspace(1)* %p) #0 {
 }
 
 ; GCN-LABEL: {{^}}max_10_vgprs_used_9a:
-; GCN-DAG:    s_mov_b32 s{{[0-9]+}}, SCRATCH_RSRC_DWORD0
-; GCN-DAG:    s_mov_b32 s{{[0-9]+}}, SCRATCH_RSRC_DWORD1
-; GFX908:     v_accvgpr_write_b32 a9, v{{[0-9]}}
-; GCN:        buffer_store_dword v{{[0-9]}},
-; GFX900:     buffer_store_dword v{{[0-9]}},
-; GFX900:     buffer_load_dword v{{[0-9]}},
-; GFX900:     buffer_load_dword v{{[0-9]}},
+; GFX908-DAG: s_mov_b32 s{{[0-9]+}}, SCRATCH_RSRC_DWORD0
+; GFX908-DAG: s_mov_b32 s{{[0-9]+}}, SCRATCH_RSRC_DWORD1
+; GFX908-DAG: v_accvgpr_write_b32 a9, v{{[0-9]}}
+; GFX908:     buffer_store_dword v{{[0-9]}},
 ; GFX908-NOT: buffer_
 ; GFX908:     v_accvgpr_read_b32 v{{[0-9]}}, a9
 ; GFX908:     buffer_load_dword v{{[0-9]}},
 ; GFX908-NOT: buffer_
 
-; GCN:    NumVgprs: 10
-; GFX900: ScratchSize: 12
+; GFX900:     couldn't allocate input reg for constraint 'a'
+
+; GFX908: NumVgprs: 10
 ; GFX908: ScratchSize: 8
-; GCN:    VGPRBlocks: 2
-; GCN:    NumVGPRsForWavesPerEU: 10
+; GFX908: VGPRBlocks: 2
+; GFX908: NumVGPRsForWavesPerEU: 10
 define amdgpu_kernel void @max_10_vgprs_used_9a(i32 addrspace(1)* %p) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   call void asm sideeffect "", "a,a,a,a,a,a,a,a,a"(i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9)
@@ -248,7 +246,7 @@ define amdgpu_kernel void @max_256_vgprs_spill_9x32(<32 x float> addrspace(1)* %
 ; GFX908-DAG  v_accvgpr_read_b32
 
 ; GCN:    NumVgprs: 256
-; GFX900: ScratchSize: 580
+; GFX900: ScratchSize: 644
 ; GFX908-FIXME: ScratchSize: 0
 ; GCN:    VGPRBlocks: 63
 ; GCN:    NumVGPRsForWavesPerEU: 256
diff --git a/llvm/test/CodeGen/AMDGPU/spill-wide-sgpr.ll b/llvm/test/CodeGen/AMDGPU/spill-wide-sgpr.ll
index 920f503fc..c262f353d 100644
--- a/llvm/test/CodeGen/AMDGPU/spill-wide-sgpr.ll
+++ b/llvm/test/CodeGen/AMDGPU/spill-wide-sgpr.ll
@@ -1,21 +1,7 @@
-; RUN: llc -O0 -march=amdgcn -mcpu=fiji -amdgpu-spill-sgpr-to-smem=0 -verify-machineinstrs < %s | FileCheck -check-prefix=ALL -check-prefix=VGPR %s
-; RUN: llc -O0 -march=amdgcn -mcpu=fiji -amdgpu-spill-sgpr-to-smem=1 -verify-machineinstrs < %s | FileCheck -check-prefix=ALL -check-prefix=SMEM %s
-; RUN: llc -O0 -march=amdgcn -mcpu=fiji -amdgpu-spill-sgpr-to-smem=0 -amdgpu-spill-sgpr-to-vgpr=0 -verify-machineinstrs < %s | FileCheck -check-prefix=ALL -check-prefix=VMEM %s
+; RUN: llc -O0 -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=ALL -check-prefix=VGPR %s
+; RUN: llc -O0 -march=amdgcn -mcpu=fiji -amdgpu-spill-sgpr-to-vgpr=0 -verify-machineinstrs < %s | FileCheck -check-prefix=ALL -check-prefix=VMEM %s
 
 ; ALL-LABEL: {{^}}spill_sgpr_x2:
-; SMEM: s_add_u32 m0, s3, 0x100{{$}}
-; SMEM: s_buffer_store_dwordx2 s{{\[[0-9]+:[0-9]+\]}}, s[8:11], m0 ; 8-byte Folded Spill
-; SMEM: s_cbranch_scc1
-
-; SMEM: s_add_u32 m0, s3, 0x100{{$}}
-; SMEM: s_buffer_load_dwordx2 s{{\[[0-9]+:[0-9]+\]}}, s[8:11], m0 ; 8-byte Folded Reload
-
-; SMEM: s_dcache_wb
-; SMEM: s_endpgm
-
-; FIXME: Should only need 4 bytes
-; SMEM: ScratchSize: 12
-
 
 ; VGPR: v_writelane_b32 v{{[0-9]+}}, s{{[0-9]+}}, 0
 ; VGPR: v_writelane_b32 v{{[0-9]+}}, s{{[0-9]+}}, 1
@@ -24,6 +10,7 @@
 ; VGPR: v_readlane_b32 s{{[0-9]+}}, v{{[0-9]+}}, 0
 ; VGPR: v_readlane_b32 s{{[0-9]+}}, v{{[0-9]+}}, 1
 
+
 ; VMEM: buffer_store_dword
 ; VMEM: buffer_store_dword
 ; VMEM: s_cbranch_scc1
@@ -44,21 +31,6 @@ ret:
 }
 
 ; ALL-LABEL: {{^}}spill_sgpr_x3:
-; SMEM: s_add_u32 m0, s3, 0x100{{$}}
-; SMEM: s_buffer_store_dword s
-; SMEM: s_buffer_store_dword s
-; SMEM: s_buffer_store_dword s
-; SMEM: s_cbranch_scc1
-
-; SMEM: s_add_u32 m0, s3, 0x100{{$}}
-; SMEM: s_buffer_load_dword s
-; SMEM: s_buffer_load_dword s
-; SMEM: s_buffer_load_dword s
-; SMEM: s_dcache_wb
-; SMEM: s_endpgm
-
-; FIXME: Should only need 4 bytes
-; SMEM: ScratchSize: 16
 
 ; VGPR: v_writelane_b32 v{{[0-9]+}}, s{{[0-9]+}}, 0
 ; VGPR: v_writelane_b32 v{{[0-9]+}}, s{{[0-9]+}}, 1
@@ -92,17 +64,6 @@ ret:
 }
 
 ; ALL-LABEL: {{^}}spill_sgpr_x4:
-; SMEM: s_add_u32 m0, s3, 0x100{{$}}
-; SMEM: s_buffer_store_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s{{\[}}[[VALS:[0-9]+:[0-9]+]]{{\]}}, m0 ; 16-byte Folded Spill
-; SMEM: s_cbranch_scc1
-
-; SMEM: s_add_u32 m0, s3, 0x100{{$}}
-; SMEM: s_buffer_load_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s{{\[}}[[VALS]]{{\]}}, m0 ; 16-byte Folded Reload
-; SMEM: s_dcache_wb
-; SMEM: s_endpgm
-
-; FIXME: Should only need 4 bytes
-; SMEM: ScratchSize: 20
 
 ; VGPR: v_writelane_b32 v{{[0-9]+}}, s{{[0-9]+}}, 0
 ; VGPR: v_writelane_b32 v{{[0-9]+}}, s{{[0-9]+}}, 1
@@ -140,25 +101,6 @@ ret:
 }
 
 ; ALL-LABEL: {{^}}spill_sgpr_x5:
-; SMEM: s_add_u32 m0, s3, 0x100{{$}}
-; SMEM: s_buffer_store_dword s
-; SMEM: s_buffer_store_dword s
-; SMEM: s_buffer_store_dword s
-; SMEM: s_buffer_store_dword s
-; SMEM: s_buffer_store_dword s
-; SMEM: s_cbranch_scc1
-
-; SMEM: s_add_u32 m0, s3, 0x100{{$}}
-; SMEM: s_buffer_load_dword s
-; SMEM: s_buffer_load_dword s
-; SMEM: s_buffer_load_dword s
-; SMEM: s_buffer_load_dword s
-; SMEM: s_buffer_load_dword s
-; SMEM: s_dcache_wb
-; SMEM: s_endpgm
-
-; FIXME: Should only need 4 bytes
-; SMEM: ScratchSize: 24
 
 ; VGPR: v_writelane_b32 v{{[0-9]+}}, s{{[0-9]+}}, 0
 ; VGPR: v_writelane_b32 v{{[0-9]+}}, s{{[0-9]+}}, 1
@@ -201,22 +143,6 @@ ret:
 
 ; ALL-LABEL: {{^}}spill_sgpr_x8:
 
-; SMEM: s_add_u32 m0, s3, 0x100{{$}}
-; SMEM: s_buffer_store_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s{{\[}}[[VALS:[0-9]+:[0-9]+]]{{\]}}, m0 ; 16-byte Folded Spill
-; SMEM: s_add_u32 m0, s3, 0x110{{$}}
-; SMEM: s_buffer_store_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s{{\[}}[[VALS]]{{\]}}, m0 ; 16-byte Folded Spill
-; SMEM: s_cbranch_scc1
-
-; SMEM: s_add_u32 m0, s3, 0x100{{$}}
-; SMEM: s_buffer_load_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s{{\[}}[[VALS]]{{\]}}, m0 ; 16-byte Folded Reload
-; SMEM: s_add_u32 m0, s3, 0x110{{$}}
-; SMEM: s_buffer_load_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s{{\[}}[[VALS]]{{\]}}, m0 ; 16-byte Folded Reload
-
-; SMEM: s_dcache_wb
-; SMEM: s_endpgm
-
-; SMEM: ScratchSize: 36
-
 ; VGPR: v_writelane_b32 v{{[0-9]+}}, s{{[0-9]+}}, 0
 ; VGPR: v_writelane_b32 v{{[0-9]+}}, s{{[0-9]+}}, 1
 ; VGPR: v_writelane_b32 v{{[0-9]+}}, s{{[0-9]+}}, 2
diff --git a/llvm/test/CodeGen/AMDGPU/split-arg-dbg-value.ll b/llvm/test/CodeGen/AMDGPU/split-arg-dbg-value.ll
new file mode 100644
index 000000000..06a4d90df
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/split-arg-dbg-value.ll
@@ -0,0 +1,223 @@
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 < %s | FileCheck -check-prefix=GCN %s
+; Make sure dbg_value reports something for argument registers when they are split into multiple registers
+
+define hidden <4 x float> @split_v4f32_arg(<4 x float> returned %arg) local_unnamed_addr #0 !dbg !7 {
+; GCN-LABEL: split_v4f32_arg:
+; GCN:       .Lfunc_begin0:
+; GCN-NEXT:    .file 0
+; GCN-NEXT:    .loc 0 3 0 ; /tmp/dbg.cl:3:0
+; GCN-NEXT:  ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:  .Ltmp0:
+; GCN-NEXT:    ;DEBUG_VALUE: split_v4f32_arg:arg <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef, DW_OP_LLVM_fragment 96 32] $vgpr3
+; GCN-NEXT:    ;DEBUG_VALUE: split_v4f32_arg:arg <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef, DW_OP_LLVM_fragment 64 32] $vgpr2
+; GCN-NEXT:    ;DEBUG_VALUE: split_v4f32_arg:arg <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef, DW_OP_LLVM_fragment 32 32] $vgpr1
+; GCN-NEXT:    ;DEBUG_VALUE: split_v4f32_arg:arg <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef, DW_OP_LLVM_fragment 0 32] $vgpr0
+; GCN-NEXT:    .loc 0 4 5 prologue_end ; /tmp/dbg.cl:4:5
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .Ltmp1:
+  call void @llvm.dbg.value(metadata <4 x float> %arg, metadata !18, metadata !DIExpression(DW_OP_constu, 1, DW_OP_swap, DW_OP_xderef)), !dbg !19
+  ret <4 x float> %arg, !dbg !20
+}
+
+define hidden <4 x float> @split_v4f32_multi_arg(<4 x float> %arg0, <2 x float> %arg1) local_unnamed_addr #0 !dbg !21 {
+; GCN-LABEL: split_v4f32_multi_arg:
+; GCN:       .Lfunc_begin1:
+; GCN-NEXT:    .loc 0 7 0 ; /tmp/dbg.cl:7:0
+; GCN-NEXT:  ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:  .Ltmp2:
+; GCN-NEXT:    ;DEBUG_VALUE: split_v4f32_multi_arg:arg1 <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef, DW_OP_LLVM_fragment 32 32] $vgpr5
+; GCN-NEXT:    ;DEBUG_VALUE: split_v4f32_multi_arg:arg1 <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef, DW_OP_LLVM_fragment 0 32] $vgpr4
+; GCN-NEXT:    ;DEBUG_VALUE: split_v4f32_multi_arg:arg0 <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef, DW_OP_LLVM_fragment 96 32] $vgpr3
+; GCN-NEXT:    ;DEBUG_VALUE: split_v4f32_multi_arg:arg0 <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef, DW_OP_LLVM_fragment 64 32] $vgpr2
+; GCN-NEXT:    ;DEBUG_VALUE: split_v4f32_multi_arg:arg0 <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef, DW_OP_LLVM_fragment 32 32] $vgpr1
+; GCN-NEXT:    ;DEBUG_VALUE: split_v4f32_multi_arg:arg0 <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef, DW_OP_LLVM_fragment 0 32] $vgpr0
+; GCN-NEXT:    .loc 0 8 17 prologue_end ; /tmp/dbg.cl:8:17
+; GCN-NEXT:    v_add_f32_e32 v0, v4, v0
+; GCN-NEXT:  .Ltmp3:
+; GCN-NEXT:    v_add_f32_e32 v1, v5, v1
+; GCN-NEXT:  .Ltmp4:
+; GCN-NEXT:    v_add_f32_e32 v2, v4, v2
+; GCN-NEXT:  .Ltmp5:
+; GCN-NEXT:    v_add_f32_e32 v3, v5, v3
+; GCN-NEXT:  .Ltmp6:
+; GCN-NEXT:    .loc 0 8 5 is_stmt 0 ; /tmp/dbg.cl:8:5
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .Ltmp7:
+  call void @llvm.dbg.value(metadata <4 x float> %arg0, metadata !29, metadata !DIExpression(DW_OP_constu, 1, DW_OP_swap, DW_OP_xderef)), !dbg !31
+  call void @llvm.dbg.value(metadata <2 x float> %arg1, metadata !30, metadata !DIExpression(DW_OP_constu, 1, DW_OP_swap, DW_OP_xderef)), !dbg !31
+  %tmp = shufflevector <2 x float> %arg1, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>, !dbg !32
+  %add = fadd <4 x float> %tmp, %arg0, !dbg !33
+  ret <4 x float> %add, !dbg !34
+}
+
+define hidden <4 x half> @split_v4f16_arg(<4 x half> returned %arg) local_unnamed_addr #0 !dbg !35 {
+; GCN-LABEL: split_v4f16_arg:
+; GCN:       .Lfunc_begin2:
+; GCN-NEXT:    .loc 0 11 0 is_stmt 1 ; /tmp/dbg.cl:11:0
+; GCN-NEXT:  ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:  .Ltmp8:
+; GCN-NEXT:    ;DEBUG_VALUE: split_v4f16_arg:arg <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef, DW_OP_LLVM_fragment 32 32] $vgpr1
+; GCN-NEXT:    ;DEBUG_VALUE: split_v4f16_arg:arg <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef, DW_OP_LLVM_fragment 0 32] $vgpr0
+; GCN-NEXT:    .loc 0 12 5 prologue_end ; /tmp/dbg.cl:12:5
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .Ltmp9:
+  call void @llvm.dbg.value(metadata <4 x half> %arg, metadata !42, metadata !DIExpression(DW_OP_constu, 1, DW_OP_swap, DW_OP_xderef)), !dbg !43
+  ret <4 x half> %arg, !dbg !44
+}
+
+define hidden double @split_f64_arg(double returned %arg) local_unnamed_addr #0 !dbg !45 {
+; GCN-LABEL: split_f64_arg:
+; GCN:       .Lfunc_begin3:
+; GCN-NEXT:    .loc 0 15 0 ; /tmp/dbg.cl:15:0
+; GCN-NEXT:  ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:  .Ltmp10:
+; GCN-NEXT:    ;DEBUG_VALUE: split_f64_arg:arg <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef, DW_OP_LLVM_fragment 32 32] $vgpr1
+; GCN-NEXT:    ;DEBUG_VALUE: split_f64_arg:arg <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef, DW_OP_LLVM_fragment 0 32] $vgpr0
+; GCN-NEXT:    .loc 0 16 5 prologue_end ; /tmp/dbg.cl:16:5
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .Ltmp11:
+  call void @llvm.dbg.value(metadata double %arg, metadata !50, metadata !DIExpression(DW_OP_constu, 1, DW_OP_swap, DW_OP_xderef)), !dbg !51
+  ret double %arg, !dbg !52
+}
+
+define hidden <2 x double> @split_v2f64_arg(<2 x double> returned %arg) local_unnamed_addr #0 !dbg !53 {
+; GCN-LABEL: split_v2f64_arg:
+; GCN:       .Lfunc_begin4:
+; GCN-NEXT:    .loc 0 19 0 ; /tmp/dbg.cl:19:0
+; GCN-NEXT:  ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:  .Ltmp12:
+; GCN-NEXT:    ;DEBUG_VALUE: split_v2f64_arg:arg <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef, DW_OP_LLVM_fragment 96 32] $vgpr3
+; GCN-NEXT:    ;DEBUG_VALUE: split_v2f64_arg:arg <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef, DW_OP_LLVM_fragment 64 32] $vgpr2
+; GCN-NEXT:    ;DEBUG_VALUE: split_v2f64_arg:arg <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef, DW_OP_LLVM_fragment 32 32] $vgpr1
+; GCN-NEXT:    ;DEBUG_VALUE: split_v2f64_arg:arg <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef, DW_OP_LLVM_fragment 0 32] $vgpr0
+; GCN-NEXT:    .loc 0 20 5 prologue_end ; /tmp/dbg.cl:20:5
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .Ltmp13:
+  call void @llvm.dbg.value(metadata <2 x double> %arg, metadata !59, metadata !DIExpression(DW_OP_constu, 1, DW_OP_swap, DW_OP_xderef)), !dbg !60
+  ret <2 x double> %arg, !dbg !61
+}
+
+define hidden i64 @split_i64_arg(i64 returned %arg) local_unnamed_addr #0 !dbg !62 {
+; GCN-LABEL: split_i64_arg:
+; GCN:       .Lfunc_begin5:
+; GCN-NEXT:    .loc 0 23 0 ; /tmp/dbg.cl:23:0
+; GCN-NEXT:  ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:  .Ltmp14:
+; GCN-NEXT:    ;DEBUG_VALUE: split_i64_arg:arg <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef, DW_OP_LLVM_fragment 32 32] $vgpr1
+; GCN-NEXT:    ;DEBUG_VALUE: split_i64_arg:arg <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef, DW_OP_LLVM_fragment 0 32] $vgpr0
+; GCN-NEXT:    .loc 0 24 5 prologue_end ; /tmp/dbg.cl:24:5
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .Ltmp15:
+  call void @llvm.dbg.value(metadata i64 %arg, metadata !67, metadata !DIExpression(DW_OP_constu, 1, DW_OP_swap, DW_OP_xderef)), !dbg !68
+  ret i64 %arg, !dbg !69
+}
+
+define hidden i8 addrspace(1)* @split_ptr_arg(i8 addrspace(1)* readnone returned %arg) local_unnamed_addr #0 !dbg !70 {
+; GCN-LABEL: split_ptr_arg:
+; GCN:       .Lfunc_begin6:
+; GCN-NEXT:    .loc 0 27 0 ; /tmp/dbg.cl:27:0
+; GCN-NEXT:  ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:  .Ltmp16:
+; GCN-NEXT:    ;DEBUG_VALUE: split_ptr_arg:arg <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef, DW_OP_LLVM_fragment 32 32] $vgpr1
+; GCN-NEXT:    ;DEBUG_VALUE: split_ptr_arg:arg <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef, DW_OP_LLVM_fragment 0 32] $vgpr0
+; GCN-NEXT:    .loc 0 28 5 prologue_end ; /tmp/dbg.cl:28:5
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .Ltmp17:
+  call void @llvm.dbg.value(metadata i8 addrspace(1)* %arg, metadata !76, metadata !DIExpression(DW_OP_constu, 1, DW_OP_swap, DW_OP_xderef)), !dbg !77
+  ret i8 addrspace(1)* %arg, !dbg !78
+}
+
+declare void @llvm.dbg.value(metadata, metadata, metadata) #1
+
+attributes #0 = { norecurse nounwind readnone }
+attributes #1 = { nounwind readnone speculatable }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5, !6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 9.0.0 (trunk 365209) (llvm/trunk 365206)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "/tmp/dbg.cl", directory: "/Users/matt/src/llvm", checksumkind: CSK_MD5, checksum: "0f834f91e91489a5ff6308040ddbd175")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 5}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{i32 7, !"PIC Level", i32 1}
+!7 = distinct !DISubprogram(name: "split_v4f32_arg", scope: !8, file: !8, line: 3, type: !9, scopeLine: 3, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !17)
+!8 = !DIFile(filename: "/tmp/dbg.cl", directory: "", checksumkind: CSK_MD5, checksum: "0f834f91e91489a5ff6308040ddbd175")
+!9 = !DISubroutineType(types: !10)
+!10 = !{!11, !11}
+!11 = !DIDerivedType(tag: DW_TAG_typedef, name: "float4", file: !12, line: 107, baseType: !13)
+!12 = !DIFile(filename: "build_debug/lib/clang/9.0.0/include/opencl-c-base.h", directory: "/Users/matt/src/llvm", checksumkind: CSK_MD5, checksum: "9526a66ac52220225f05e11186d7e461")
+!13 = !DICompositeType(tag: DW_TAG_array_type, baseType: !14, size: 128, flags: DIFlagVector, elements: !15)
+!14 = !DIBasicType(name: "float", size: 32, encoding: DW_ATE_float)
+!15 = !{!16}
+!16 = !DISubrange(count: 4)
+!17 = !{!18}
+!18 = !DILocalVariable(name: "arg", arg: 1, scope: !7, file: !8, line: 3, type: !11)
+!19 = !DILocation(line: 0, scope: !7)
+!20 = !DILocation(line: 4, column: 5, scope: !7)
+!21 = distinct !DISubprogram(name: "split_v4f32_multi_arg", scope: !8, file: !8, line: 7, type: !22, scopeLine: 7, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !28)
+!22 = !DISubroutineType(types: !23)
+!23 = !{!11, !11, !24}
+!24 = !DIDerivedType(tag: DW_TAG_typedef, name: "float2", file: !12, line: 105, baseType: !25)
+!25 = !DICompositeType(tag: DW_TAG_array_type, baseType: !14, size: 64, flags: DIFlagVector, elements: !26)
+!26 = !{!27}
+!27 = !DISubrange(count: 2)
+!28 = !{!29, !30}
+!29 = !DILocalVariable(name: "arg0", arg: 1, scope: !21, file: !8, line: 7, type: !11)
+!30 = !DILocalVariable(name: "arg1", arg: 2, scope: !21, file: !8, line: 7, type: !24)
+!31 = !DILocation(line: 0, scope: !21)
+!32 = !DILocation(line: 8, column: 19, scope: !21)
+!33 = !DILocation(line: 8, column: 17, scope: !21)
+!34 = !DILocation(line: 8, column: 5, scope: !21)
+!35 = distinct !DISubprogram(name: "split_v4f16_arg", scope: !8, file: !8, line: 11, type: !36, scopeLine: 11, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !41)
+!36 = !DISubroutineType(types: !37)
+!37 = !{!38, !38}
+!38 = !DIDerivedType(tag: DW_TAG_typedef, name: "half4", file: !12, line: 114, baseType: !39)
+!39 = !DICompositeType(tag: DW_TAG_array_type, baseType: !40, size: 64, flags: DIFlagVector, elements: !15)
+!40 = !DIBasicType(name: "half", size: 16, encoding: DW_ATE_float)
+!41 = !{!42}
+!42 = !DILocalVariable(name: "arg", arg: 1, scope: !35, file: !8, line: 11, type: !38)
+!43 = !DILocation(line: 0, scope: !35)
+!44 = !DILocation(line: 12, column: 5, scope: !35)
+!45 = distinct !DISubprogram(name: "split_f64_arg", scope: !8, file: !8, line: 15, type: !46, scopeLine: 15, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !49)
+!46 = !DISubroutineType(types: !47)
+!47 = !{!48, !48}
+!48 = !DIBasicType(name: "double", size: 64, encoding: DW_ATE_float)
+!49 = !{!50}
+!50 = !DILocalVariable(name: "arg", arg: 1, scope: !45, file: !8, line: 15, type: !48)
+!51 = !DILocation(line: 0, scope: !45)
+!52 = !DILocation(line: 16, column: 5, scope: !45)
+!53 = distinct !DISubprogram(name: "split_v2f64_arg", scope: !8, file: !8, line: 19, type: !54, scopeLine: 19, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !58)
+!54 = !DISubroutineType(types: !55)
+!55 = !{!56, !56}
+!56 = !DIDerivedType(tag: DW_TAG_typedef, name: "double2", file: !12, line: 122, baseType: !57)
+!57 = !DICompositeType(tag: DW_TAG_array_type, baseType: !48, size: 128, flags: DIFlagVector, elements: !26)
+!58 = !{!59}
+!59 = !DILocalVariable(name: "arg", arg: 1, scope: !53, file: !8, line: 19, type: !56)
+!60 = !DILocation(line: 0, scope: !53)
+!61 = !DILocation(line: 20, column: 5, scope: !53)
+!62 = distinct !DISubprogram(name: "split_i64_arg", scope: !8, file: !8, line: 23, type: !63, scopeLine: 23, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !66)
+!63 = !DISubroutineType(types: !64)
+!64 = !{!65, !65}
+!65 = !DIBasicType(name: "long int", size: 64, encoding: DW_ATE_signed)
+!66 = !{!67}
+!67 = !DILocalVariable(name: "arg", arg: 1, scope: !62, file: !8, line: 23, type: !65)
+!68 = !DILocation(line: 0, scope: !62)
+!69 = !DILocation(line: 24, column: 5, scope: !62)
+!70 = distinct !DISubprogram(name: "split_ptr_arg", scope: !8, file: !8, line: 27, type: !71, scopeLine: 27, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !75)
+!71 = !DISubroutineType(types: !72)
+!72 = !{!73, !73}
+!73 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !74, size: 64)
+!74 = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_signed_char)
+!75 = !{!76}
+!76 = !DILocalVariable(name: "arg", arg: 1, scope: !70, file: !8, line: 27, type: !73)
+!77 = !DILocation(line: 0, scope: !70)
+!78 = !DILocation(line: 28, column: 5, scope: !70)
diff --git a/llvm/test/CodeGen/AMDGPU/stack-pointer-offset-relative-frameindex.ll b/llvm/test/CodeGen/AMDGPU/stack-pointer-offset-relative-frameindex.ll
new file mode 100644
index 000000000..00ae166a6
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/stack-pointer-offset-relative-frameindex.ll
@@ -0,0 +1,66 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs | FileCheck -check-prefix=GCN %s
+
+; An assert was hit when frame offset register was used to address FrameIndex.
+define amdgpu_kernel void @kernel_background_evaluate(float addrspace(5)* %kg, <4 x i32> addrspace(1)* %input, <4 x float> addrspace(1)* %output, i32 %i) {
+; GCN-LABEL: kernel_background_evaluate:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_load_dword s6, s[0:1], 0x24
+; GCN-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GCN-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GCN-NEXT:    s_mov_b32 s38, -1
+; GCN-NEXT:    s_mov_b32 s39, 0x31c16000
+; GCN-NEXT:    s_mov_b32 s33, s3
+; GCN-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GCN-NEXT:    v_mov_b32_e32 v1, 0x2000
+; GCN-NEXT:    v_mov_b32_e32 v2, 0x4000
+; GCN-NEXT:    v_mov_b32_e32 v3, 0
+; GCN-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GCN-NEXT:    v_mov_b32_e32 v4, 0x400000
+; GCN-NEXT:    s_add_u32 s32, s33, 0xc0000
+; GCN-NEXT:    v_add_nc_u32_e64 v32, 4, 0x4000
+; GCN-NEXT:    ; implicit-def: $vcc_hi
+; GCN-NEXT:    s_getpc_b64 s[4:5]
+; GCN-NEXT:    s_add_u32 s4, s4, svm_eval_nodes@rel32@lo+4
+; GCN-NEXT:    s_addc_u32 s5, s5, svm_eval_nodes@rel32@hi+4
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v0, s6
+; GCN-NEXT:    s_swappc_b64 s[30:31], s[4:5]
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v0
+; GCN-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GCN-NEXT:    ; mask branch BB0_2
+; GCN-NEXT:    s_cbranch_execz BB0_2
+; GCN-NEXT:  BB0_1: ; %if.then4.i
+; GCN-NEXT:    buffer_load_dword v0, v32, s[36:39], s32 offen
+; GCN-NEXT:    buffer_load_dword v1, v32, s[36:39], s32 offen offset:4
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_nc_u32_e32 v0, v1, v0
+; GCN-NEXT:    v_mul_lo_u32 v0, 0x41c64e6d, v0
+; GCN-NEXT:    v_add_nc_u32_e32 v0, 0x3039, v0
+; GCN-NEXT:    buffer_store_dword v0, v0, s[36:39], s33 offen
+; GCN-NEXT:  BB0_2: ; %shader_eval_surface.exit
+; GCN-NEXT:    s_endpgm
+entry:
+  %sd = alloca < 1339 x i32>, align 16, addrspace(5)
+  %state = alloca <4 x i32>, align 4, addrspace(5)
+  %rslt = call i32 @svm_eval_nodes(float addrspace(5)* %kg, <1339 x i32> addrspace(5)* %sd, <4 x i32> addrspace(5)* %state, i32 0, i32 4194304)
+  %cmp = icmp eq i32 %rslt, 0
+  br i1 %cmp, label %shader_eval_surface.exit, label %if.then4.i
+
+if.then4.i:                                       ; preds = %entry
+  %rng_hash.i.i = getelementptr inbounds < 4 x i32>, <4 x i32> addrspace(5)* %state, i32 0, i32 1
+  %tmp0 = load i32, i32 addrspace(5)* %rng_hash.i.i, align 4
+  %rng_offset.i.i = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(5)* %state, i32 0, i32 2
+  %tmp1 = load i32, i32 addrspace(5)* %rng_offset.i.i, align 4
+  %add.i.i = add i32 %tmp1, %tmp0
+  %add1.i.i = add i32 %add.i.i, 0
+  %mul.i.i.i.i = mul i32 %add1.i.i, 1103515245
+  %add.i.i.i.i = add i32 %mul.i.i.i.i, 12345
+  store i32 %add.i.i.i.i, i32 addrspace(5)* undef, align 16
+  br label %shader_eval_surface.exit
+
+shader_eval_surface.exit:                         ; preds = %entry
+  ret void
+}
+
+declare hidden i32 @svm_eval_nodes(float addrspace(5)*, <1339 x i32> addrspace(5)*, <4 x i32> addrspace(5)*, i32, i32) local_unnamed_addr
diff --git a/llvm/test/CodeGen/AMDGPU/stack-realign-kernel.ll b/llvm/test/CodeGen/AMDGPU/stack-realign-kernel.ll
index 037ed1a31..deb94f521 100644
--- a/llvm/test/CodeGen/AMDGPU/stack-realign-kernel.ll
+++ b/llvm/test/CodeGen/AMDGPU/stack-realign-kernel.ll
@@ -1,4 +1,3 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=fiji < %s | FileCheck -check-prefix=VI %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900  < %s | FileCheck -check-prefix=GFX9 %s
 
diff --git a/llvm/test/CodeGen/AMDGPU/store-hi16.ll b/llvm/test/CodeGen/AMDGPU/store-hi16.ll
index 379133744..e8d6b24ef 100644
--- a/llvm/test/CodeGen/AMDGPU/store-hi16.ll
+++ b/llvm/test/CodeGen/AMDGPU/store-hi16.ll
@@ -1,5 +1,5 @@
-; RUN: llc -march=amdgcn -mcpu=gfx900 -amdgpu-sroa=0 -mattr=-promote-alloca -verify-machineinstrs < %s | FileCheck -allow-deprecated-dag-overlap -check-prefixes=GCN,GFX900 %s
-; RUN: llc -march=amdgcn -mcpu=gfx906 -amdgpu-sroa=0 -mattr=-promote-alloca,+sram-ecc -verify-machineinstrs < %s | FileCheck -allow-deprecated-dag-overlap -check-prefixes=GCN,GFX906,NO-D16-HI %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -amdgpu-sroa=0 -mattr=-promote-alloca -verify-machineinstrs < %s | FileCheck -allow-deprecated-dag-overlap -check-prefixes=GCN,GFX900,GFX9 %s
+; RUN: llc -march=amdgcn -mcpu=gfx906 -amdgpu-sroa=0 -mattr=-promote-alloca,+sram-ecc -verify-machineinstrs < %s | FileCheck -allow-deprecated-dag-overlap -check-prefixes=GCN,GFX906,GFX9,NO-D16-HI %s
 ; RUN: llc -march=amdgcn -mcpu=fiji -amdgpu-sroa=0 -mattr=-promote-alloca -verify-machineinstrs < %s | FileCheck -allow-deprecated-dag-overlap -check-prefixes=GCN,GFX803,NO-D16-HI %s
 
 ; GCN-LABEL: {{^}}store_global_hi_v2i16:
@@ -311,16 +311,16 @@ entry:
 
 ; GCN-LABEL: {{^}}store_flat_hi_v2i16_neg_offset:
 ; GCN: s_waitcnt
-; GCN: v_add{{(_co)?}}_{{i|u}}32_e32
-
+; GFX803: v_add{{(_co)?}}_{{i|u}}32_e32
 ; GFX803: v_addc_u32_e32
-; GFX900: v_addc_co_u32_e32
 
-; GFX906-NEXT: v_lshrrev_b32_e32
-; GFX906-NEXT: v_addc_co_u32_e32
-; GFX906: flat_store_short v[0:1], v2
+; GFX9-DAG: v_add_co_u32_e32 v{{[0-9]+}}, vcc, 0xfffff000, v
+; GFX9-DAG: v_addc_co_u32_e32 v{{[0-9]+}}, vcc, -1, v
 
-; GFX900-NEXT: flat_store_short_d16_hi v[0:1], v2{{$}}
+; GFX906-DAG: v_lshrrev_b32_e32
+; GFX906: flat_store_short v[0:1], v2 offset:2050{{$}}
+
+; GFX900-NEXT: flat_store_short_d16_hi v[0:1], v2 offset:2050{{$}}
 ; GFX803: flat_store_short v[0:1], v2{{$}}
 ; GCN-NEXT: s_waitcnt
 ; GCN-NEXT: s_setpc_b64
@@ -359,17 +359,17 @@ entry:
 
 ; GCN-LABEL: {{^}}store_flat_hi_v2i16_i8_neg_offset:
 ; GCN: s_waitcnt
-; GCN-DAG: v_add{{(_co)?}}_{{i|u}}32_e32
 
+; GFX803-DAG: v_add_u32_e32
 ; GFX803-DAG: v_addc_u32_e32
-; GFX900-DAG: v_addc_co_u32_e32
-; GFX906-DAG: v_add_co_u32_e32
 
-; GFX900-NEXT: flat_store_byte_d16_hi v[0:1], v2{{$}}
+; GFX9-DAG: v_add_co_u32_e32 v{{[0-9]+}}, vcc, 0xfffff000, v
+; GFX9-DAG: v_addc_co_u32_e32 v{{[0-9]+}}, vcc, -1, v{{[0-9]+}}, vcc
 
-; GFX906-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX906-NEXT: v_addc_co_u32_e32
-; GFX906-NEXT: flat_store_byte v[0:1], v2{{$}}
+; GFX900-NEXT: flat_store_byte_d16_hi v[0:1], v2 offset:1{{$}}
+
+; GFX906-DAG: v_lshrrev_b32_e32 v2, 16, v2
+; GFX906: flat_store_byte v[0:1], v2 offset:1{{$}}
 
 ; GFX803-DAG: v_lshrrev_b32_e32 v2, 16, v2
 ; GFX803: flat_store_byte v[0:1], v2{{$}}
diff --git a/llvm/test/CodeGen/AMDGPU/sub-zext-cc-zext-cc.ll b/llvm/test/CodeGen/AMDGPU/sub-zext-cc-zext-cc.ll
new file mode 100644
index 000000000..911677478
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/sub-zext-cc-zext-cc.ll
@@ -0,0 +1,34 @@
+; RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx900 -verify-machineinstrs <%s | FileCheck -check-prefixes=GCN %s
+;
+; This test checks that we have the correct fold for zext(cc1) - zext(cc2).
+;
+; GCN-LABEL: sub_zext_zext:
+; GCN: ds_read_b32 [[VAL:v[0-9]+]],
+; GCN-DAG: v_cmp_lt_f32{{.*}} [[CC1:s\[[0-9]+:[0-9]+\]]], 0, [[VAL]]
+; GCN-DAG: v_cmp_gt_f32{{.*}} vcc, 0, [[VAL]]
+; GCN: v_cndmask_{{.*}} [[ZEXTCC1:v[0-9]+]], 0, 1, [[CC1]]
+; GCN: v_subbrev{{.*}} {{v[0-9]+}}, vcc, 0, [[ZEXTCC1]], vcc
+;
+; Before the reversion that this test is attached to, the compiler commuted
+; the operands to the sub and used different logic to select the addc/subc
+; instruction:
+;    sub zext (setcc), x => addcarry 0, x, setcc
+;    sub sext (setcc), x => subcarry 0, x, setcc
+;
+; ... but that is bogus. I believe it is not possible to fold those commuted
+; patterns into any form of addcarry or subcarry.
+
+define amdgpu_cs float @sub_zext_zext() {
+.entry:
+
+  %t519 = load float, float addrspace(3)* null
+
+  %t524 = fcmp ogt float %t519, 0.000000e+00
+  %t525 = fcmp olt float %t519, 0.000000e+00
+  %t526 = zext i1 %t524 to i32
+  %t527 = zext i1 %t525 to i32
+  %t528 = sub nsw i32 %t526, %t527
+  %t529 = sitofp i32 %t528 to float
+  ret float %t529
+}
+
diff --git a/llvm/test/CodeGen/AMDGPU/sub.i16.ll b/llvm/test/CodeGen/AMDGPU/sub.i16.ll
index 6ff620c87..5f98b7b1f 100644
--- a/llvm/test/CodeGen/AMDGPU/sub.i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/sub.i16.ll
@@ -107,8 +107,8 @@ define amdgpu_kernel void @v_test_sub_i16_zext_to_i64(i64 addrspace(1)* %out, i1
 ; GCN-LABEL: {{^}}v_test_sub_i16_sext_to_i32:
 ; VI: flat_load_ushort [[A:v[0-9]+]]
 ; VI: flat_load_ushort [[B:v[0-9]+]]
-; VI: v_sub_u16_e32 [[ADD:v[0-9]+]], [[B]], [[A]]
-; VI-NEXT: v_bfe_i32 [[SEXT:v[0-9]+]], [[ADD]], 0, 16
+; VI: v_sub_u16_e32 [[ADD:v[0-9]+]], [[A]], [[B]]
+; VI: v_bfe_i32 [[SEXT:v[0-9]+]], [[ADD]], 0, 16
 ; VI-NEXT: buffer_store_dword [[SEXT]]
 define amdgpu_kernel void @v_test_sub_i16_sext_to_i32(i32 addrspace(1)* %out, i16 addrspace(1)* %in0, i16 addrspace(1)* %in1) #1 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -127,9 +127,9 @@ define amdgpu_kernel void @v_test_sub_i16_sext_to_i32(i32 addrspace(1)* %out, i1
 ; GCN-LABEL: {{^}}v_test_sub_i16_sext_to_i64:
 ; VI: flat_load_ushort [[A:v[0-9]+]]
 ; VI: flat_load_ushort [[B:v[0-9]+]]
-; VI: v_sub_u16_e32 [[ADD:v[0-9]+]], [[B]], [[A]]
+; VI: v_sub_u16_e32 [[ADD:v[0-9]+]], [[A]], [[B]]
 ; VI-NEXT: v_bfe_i32 v[[LO:[0-9]+]], [[ADD]], 0, 16
-; VI-NEXT: v_ashrrev_i32_e32 v[[HI:[0-9]+]], 31, v[[LO]]
+; VI:      v_ashrrev_i32_e32 v[[HI:[0-9]+]], 31, v[[LO]]
 ; VI-NEXT: buffer_store_dwordx2 v{{\[}}[[LO]]:[[HI]]{{\]}}
 define amdgpu_kernel void @v_test_sub_i16_sext_to_i64(i64 addrspace(1)* %out, i16 addrspace(1)* %in0, i16 addrspace(1)* %in1) #1 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
diff --git a/llvm/test/CodeGen/AMDGPU/sub.v2i16.ll b/llvm/test/CodeGen/AMDGPU/sub.v2i16.ll
index c7a08a665..495e79bb7 100644
--- a/llvm/test/CodeGen/AMDGPU/sub.v2i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/sub.v2i16.ll
@@ -66,39 +66,39 @@ define amdgpu_kernel void @s_test_sub_v2i16(<2 x i16> addrspace(1)* %out, <2 x i
 ; GFX9-LABEL: s_test_sub_v2i16:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX9-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0x34
 ; GFX9-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX9-NEXT:    s_mov_b32 s2, -1
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_load_dword s6, s[6:7], 0x0
-; GFX9-NEXT:    s_load_dword s7, s[0:1], 0x0
 ; GFX9-NEXT:    s_mov_b32 s0, s4
 ; GFX9-NEXT:    s_mov_b32 s1, s5
+; GFX9-NEXT:    s_load_dword s4, s[6:7], 0x0
+; GFX9-NEXT:    s_load_dword s5, s[8:9], 0x0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, s7
-; GFX9-NEXT:    v_pk_sub_i16 v0, s6, v0
+; GFX9-NEXT:    v_mov_b32_e32 v0, s5
+; GFX9-NEXT:    v_pk_sub_i16 v0, s4, v0
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; GFX9-NEXT:    s_endpgm
 ;
 ; VI-LABEL: s_test_sub_v2i16:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
-; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; VI-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0x34
 ; VI-NEXT:    s_mov_b32 s3, 0xf000
 ; VI-NEXT:    s_mov_b32 s2, -1
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
-; VI-NEXT:    s_load_dword s6, s[6:7], 0x0
-; VI-NEXT:    s_load_dword s7, s[0:1], 0x0
 ; VI-NEXT:    s_mov_b32 s0, s4
 ; VI-NEXT:    s_mov_b32 s1, s5
+; VI-NEXT:    s_load_dword s4, s[6:7], 0x0
+; VI-NEXT:    s_load_dword s5, s[8:9], 0x0
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
-; VI-NEXT:    s_lshr_b32 s4, s6, 16
-; VI-NEXT:    s_lshr_b32 s5, s7, 16
+; VI-NEXT:    s_lshr_b32 s6, s4, 16
+; VI-NEXT:    s_lshr_b32 s7, s5, 16
 ; VI-NEXT:    s_sub_i32 s4, s4, s5
-; VI-NEXT:    s_sub_i32 s6, s6, s7
-; VI-NEXT:    s_and_b32 s5, s6, 0xffff
-; VI-NEXT:    s_lshl_b32 s4, s4, 16
-; VI-NEXT:    s_or_b32 s4, s5, s4
+; VI-NEXT:    s_sub_i32 s5, s6, s7
+; VI-NEXT:    s_and_b32 s4, s4, 0xffff
+; VI-NEXT:    s_lshl_b32 s5, s5, 16
+; VI-NEXT:    s_or_b32 s4, s4, s5
 ; VI-NEXT:    v_mov_b32_e32 v0, s4
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
@@ -344,7 +344,7 @@ define amdgpu_kernel void @v_test_sub_v2i16_inline_lo_zero_hi(<2 x i16> addrspac
 ; VI-NEXT:    s_mov_b32 s5, s1
 ; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; VI-NEXT:    v_and_b32_e32 v1, 0xffff0000, v0
-; VI-NEXT:    v_add_u16_e32 v0, 0xffffffe0, v0
+; VI-NEXT:    v_subrev_u16_e32 v0, 32, v0
 ; VI-NEXT:    v_or_b32_e32 v0, v0, v1
 ; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 ; VI-NEXT:    s_endpgm
@@ -614,12 +614,12 @@ define amdgpu_kernel void @v_test_sub_v2i16_sext_to_v2i64(<2 x i64> addrspace(1)
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s1
 ; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, s0, v2
 ; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX9-NEXT:    global_load_dword v2, v[2:3], off
 ; GFX9-NEXT:    global_load_dword v0, v[0:1], off
+; GFX9-NEXT:    global_load_dword v1, v[2:3], off
 ; GFX9-NEXT:    s_mov_b32 s0, s4
 ; GFX9-NEXT:    s_mov_b32 s1, s5
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_pk_sub_i16 v1, v0, v2
+; GFX9-NEXT:    v_pk_sub_i16 v1, v0, v1
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v2, 16, v1
 ; GFX9-NEXT:    v_bfe_i32 v0, v1, 0, 16
 ; GFX9-NEXT:    v_bfe_i32 v2, v2, 0, 16
@@ -642,15 +642,15 @@ define amdgpu_kernel void @v_test_sub_v2i16_sext_to_v2i64(<2 x i64> addrspace(1)
 ; VI-NEXT:    v_mov_b32_e32 v3, s1
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, s0, v2
 ; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; VI-NEXT:    flat_load_dword v2, v[2:3]
 ; VI-NEXT:    flat_load_dword v0, v[0:1]
+; VI-NEXT:    flat_load_dword v1, v[2:3]
 ; VI-NEXT:    s_mov_b32 s0, s4
 ; VI-NEXT:    s_mov_b32 s1, s5
 ; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT:    v_sub_u16_sdwa v1, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; VI-NEXT:    v_sub_u16_e32 v0, v0, v2
-; VI-NEXT:    v_bfe_i32 v2, v1, 0, 16
+; VI-NEXT:    v_sub_u16_sdwa v2, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; VI-NEXT:    v_sub_u16_e32 v0, v0, v1
 ; VI-NEXT:    v_bfe_i32 v0, v0, 0, 16
+; VI-NEXT:    v_bfe_i32 v2, v2, 0, 16
 ; VI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; VI-NEXT:    v_ashrrev_i32_e32 v3, 31, v2
 ; VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
diff --git a/llvm/test/CodeGen/AMDGPU/subreg-coalescer-undef-use.ll b/llvm/test/CodeGen/AMDGPU/subreg-coalescer-undef-use.ll
index 80071e340..3127201e9 100644
--- a/llvm/test/CodeGen/AMDGPU/subreg-coalescer-undef-use.ll
+++ b/llvm/test/CodeGen/AMDGPU/subreg-coalescer-undef-use.ll
@@ -1,28 +1,41 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -march=amdgcn -mcpu=tahiti -amdgpu-dce-in-ra=0 -o - %s | FileCheck %s
 ; Don't crash when the use of an undefined value is only detected by the
 ; register coalescer because it is hidden with subregister insert/extract.
 target triple="amdgcn--"
 
+define amdgpu_kernel void @foobar(float %a0, float %a1, float addrspace(1)* %out) nounwind {
 ; CHECK-LABEL: foobar:
-; CHECK: s_load_dwordx2 s[4:5], s[0:1], 0x9
-; CHECK-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xb
-; CHECK-NEXT: v_mbcnt_lo_u32_b32_e64
-; CHECK-NEXT: s_mov_b32 s2, -1
-; CHECK-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: v_mov_b32_e32 v1, s5
-; CHECK-NEXT: s_and_saveexec_b64 s[4:5], vcc
-
-; CHECK: BB0_1:
-; CHECK-NEXT: ; kill: def $vgpr0_vgpr1 killed $sgpr4_sgpr5 killed $exec
-; CHECK-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; CHECK-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; CHECK-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, -1, 0
+; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; CHECK-NEXT:    s_mov_b32 s2, -1
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    v_mov_b32_e32 v0, s4
+; CHECK-NEXT:    v_mov_b32_e32 v1, s5
+; CHECK-NEXT:    v_mov_b32_e32 v2, s6
+; CHECK-NEXT:    v_mov_b32_e32 v3, s7
+; CHECK-NEXT:    s_and_saveexec_b64 s[6:7], vcc
+; CHECK-NEXT:    ; mask branch BB0_2
+; CHECK-NEXT:  BB0_1: ; %ift
+; CHECK-NEXT:    s_mov_b32 s4, s5
+; CHECK-NEXT:    v_mov_b32_e32 v0, s4
+; CHECK-NEXT:    v_mov_b32_e32 v1, s5
+; CHECK-NEXT:    v_mov_b32_e32 v2, s6
+; CHECK-NEXT:    v_mov_b32_e32 v3, s7
+; CHECK-NEXT:  BB0_2: ; %ife
+; CHECK-NEXT:    s_or_b64 exec, exec, s[6:7]
+; CHECK-NEXT:    s_mov_b32 s3, 0xf000
+; CHECK-NEXT:    buffer_store_dword v1, off, s[0:3], 0
+; CHECK-NEXT:    s_endpgm
 
-; CHECK: BB0_2:
-; CHECK: s_or_b64 exec, exec, s[4:5]
-; CHECK-NEXT: s_mov_b32 s3, 0xf000
-; CHECK-NEXT: buffer_store_dword v1, off, s[0:3], 0
-; CHECK-NEXT: s_endpgm
-define amdgpu_kernel void @foobar(float %a0, float %a1, float addrspace(1)* %out) nounwind {
+; FIXME: The change related to the fact that
+; DetectDeadLanes pass hit "Copy across incompatible class" SGPR -> VGPR in analysis
+; and hence it cannot derive the fact that the vector element in the "ift" block is unused.
+; Such a copies appear because the float4 vectors and their elements in the test are uniform
+; but the PHI node in "ife" block is divergent because of the CF dependency (divergent branch in bb0)
 entry:
   %v0 = insertelement <4 x float> undef, float %a0, i32 0
   %tid = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #0
diff --git a/llvm/test/CodeGen/AMDGPU/trunc-combine.ll b/llvm/test/CodeGen/AMDGPU/trunc-combine.ll
index 8b7791905..200026494 100644
--- a/llvm/test/CodeGen/AMDGPU/trunc-combine.ll
+++ b/llvm/test/CodeGen/AMDGPU/trunc-combine.ll
@@ -1,33 +1,48 @@
-; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs< %s | FileCheck -enable-var-scope -check-prefixes=GCN,SI %s
-; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs< %s | FileCheck -enable-var-scope -check-prefixes=GCN,VI %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -march=amdgcn -mcpu=tahiti -verify-machineinstrs | FileCheck %s -enable-var-scope -check-prefixes=GCN,SI
+; RUN: llc < %s -march=amdgcn -mcpu=fiji -verify-machineinstrs | FileCheck %s -enable-var-scope -check-prefixes=GCN,VI
 
 ; Make sure high constant 0 isn't pointlessly materialized
-; GCN-LABEL: {{^}}trunc_bitcast_i64_lshr_32_i16:
-; GCN: s_waitcnt
-; GCN-NEXT: v_mov_b32_e32 v0, v1
-; GCN-NEXT: s_setpc_b64
 define i16 @trunc_bitcast_i64_lshr_32_i16(i64 %bar) {
+; GCN-LABEL: trunc_bitcast_i64_lshr_32_i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v0, v1
+; GCN-NEXT:    s_setpc_b64 s[30:31]
   %srl = lshr i64 %bar, 32
   %trunc = trunc i64 %srl to i16
   ret i16 %trunc
 }
 
-; GCN-LABEL: {{^}}trunc_bitcast_i64_lshr_32_i32:
-; GCN: s_waitcnt
-; GCN-NEXT: v_mov_b32_e32 v0, v1
-; GCN-NEXT: s_setpc_b64
 define i32 @trunc_bitcast_i64_lshr_32_i32(i64 %bar) {
+; GCN-LABEL: trunc_bitcast_i64_lshr_32_i32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v0, v1
+; GCN-NEXT:    s_setpc_b64 s[30:31]
   %srl = lshr i64 %bar, 32
   %trunc = trunc i64 %srl to i32
   ret i32 %trunc
 }
 
-; GCN-LABEL: {{^}}trunc_bitcast_v2i32_to_i16:
-; GCN: _load_dword
-; GCN-NOT: _load_dword
-; GCN-NOT: v_mov_b32
-; VI: v_add_u16_e32 v0, 4, v0
 define i16 @trunc_bitcast_v2i32_to_i16(<2 x i32> %bar) {
+; SI-LABEL: trunc_bitcast_v2i32_to_i16:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    buffer_load_dword v0, off, s[4:7], 0
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_add_i32_e32 v0, vcc, 4, v0
+; SI-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: trunc_bitcast_v2i32_to_i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    flat_load_dword v0, v[0:1]
+; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_add_u16_e32 v0, 4, v0
+; VI-NEXT:    s_setpc_b64 s[30:31]
   %load0 = load i32, i32 addrspace(1)* undef
   %load1 = load i32, i32 addrspace(1)* null
   %insert.0 = insertelement <2 x i32> undef, i32 %load0, i32 0
@@ -39,12 +54,24 @@ define i16 @trunc_bitcast_v2i32_to_i16(<2 x i32> %bar) {
 }
 
 ; Make sure there's no crash if the source vector type is FP
-; GCN-LABEL: {{^}}trunc_bitcast_v2f32_to_i16:
-; GCN: _load_dword
-; GCN-NOT: _load_dword
-; GCN-NOT: v_mov_b32
-; VI: v_add_u16_e32 v0, 4, v0
 define i16 @trunc_bitcast_v2f32_to_i16(<2 x float> %bar) {
+; SI-LABEL: trunc_bitcast_v2f32_to_i16:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    buffer_load_dword v0, off, s[4:7], 0
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_add_i32_e32 v0, vcc, 4, v0
+; SI-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: trunc_bitcast_v2f32_to_i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    flat_load_dword v0, v[0:1]
+; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_add_u16_e32 v0, 4, v0
+; VI-NEXT:    s_setpc_b64 s[30:31]
   %load0 = load float, float addrspace(1)* undef
   %load1 = load float, float addrspace(1)* null
   %insert.0 = insertelement <2 x float> undef, float %load0, i32 0
@@ -55,14 +82,42 @@ define i16 @trunc_bitcast_v2f32_to_i16(<2 x float> %bar) {
   ret i16 %add
 }
 
-; GCN-LABEL: {{^}}truncate_high_elt_extract_vector:
-; GCN: s_load_dword s
-; GCN: s_load_dword s
-; GCN: s_sext_i32_i16
-; GCN: s_sext_i32_i16
-; GCN: v_mul_i32_i24
-; GCN: v_lshrrev_b32_e32
 define amdgpu_kernel void @truncate_high_elt_extract_vector(<2 x i16> addrspace(1)* nocapture readonly %arg, <2 x i16> addrspace(1)* nocapture readonly %arg1, <2 x i16> addrspace(1)* nocapture %arg2) local_unnamed_addr {
+; SI-LABEL: truncate_high_elt_extract_vector:
+; SI:       ; %bb.0: ; %bb
+; SI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_load_dword s4, s[4:5], 0x0
+; SI-NEXT:    s_load_dword s5, s[6:7], 0x0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_sext_i32_i16 s4, s4
+; SI-NEXT:    s_sext_i32_i16 s5, s5
+; SI-NEXT:    v_mov_b32_e32 v0, s4
+; SI-NEXT:    v_mul_i32_i24_e32 v0, s5, v0
+; SI-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: truncate_high_elt_extract_vector:
+; VI:       ; %bb.0: ; %bb
+; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    s_load_dword s0, s[4:5], 0x0
+; VI-NEXT:    s_load_dword s1, s[6:7], 0x0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_sext_i32_i16 s0, s0
+; VI-NEXT:    s_sext_i32_i16 s1, s1
+; VI-NEXT:    v_mov_b32_e32 v2, s0
+; VI-NEXT:    v_mul_i32_i24_e32 v2, s1, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
 bb:
   %tmp = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %arg, i64 undef
   %tmp3 = load <2 x i16>, <2 x i16> addrspace(1)* %tmp, align 4
@@ -82,17 +137,22 @@ bb:
   ret void
 }
 
-; GCN-LABEL: {{^}}trunc_v2i64_arg_to_v2i16:
-; GCN: v_lshlrev_b32_e32 v1, 16, v2
-
-; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; SI-NEXT: v_or_b32_e32 v0, v0, v1
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-
-; VI-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-
-; GCN-NEXT: s_setpc_b64
 define <2 x i16> @trunc_v2i64_arg_to_v2i16(<2 x i64> %arg0) #0 {
+; SI-LABEL: trunc_v2i64_arg_to_v2i16:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v2
+; SI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT:    v_or_b32_e32 v0, v0, v1
+; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
+; SI-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: trunc_v2i64_arg_to_v2i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_lshlrev_b32_e32 v1, 16, v2
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_setpc_b64 s[30:31]
   %trunc = trunc <2 x i64> %arg0 to <2 x i16>
   ret <2 x i16> %trunc
 }
diff --git a/llvm/test/CodeGen/AMDGPU/uint_to_fp.f64.ll b/llvm/test/CodeGen/AMDGPU/uint_to_fp.f64.ll
index 62943aeef..75d6eb57c 100644
--- a/llvm/test/CodeGen/AMDGPU/uint_to_fp.f64.ll
+++ b/llvm/test/CodeGen/AMDGPU/uint_to_fp.f64.ll
@@ -1,14 +1,15 @@
-; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=SI %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=hawaii -verify-machineinstrs < %s | FileCheck -enable-var-scope  -check-prefixes=GCN,SI %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=fiji -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,VI %s
 
 declare i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
 
-; SI-LABEL: {{^}}v_uint_to_fp_i64_to_f64
-; SI: buffer_load_dwordx2 v{{\[}}[[LO:[0-9]+]]:[[HI:[0-9]+]]{{\]}}
-; SI-DAG: v_cvt_f64_u32_e32 [[HI_CONV:v\[[0-9]+:[0-9]+\]]], v[[HI]]
-; SI-DAG: v_cvt_f64_u32_e32 [[LO_CONV:v\[[0-9]+:[0-9]+\]]], v[[LO]]
-; SI-DAG: v_ldexp_f64 [[LDEXP:v\[[0-9]+:[0-9]+\]]], [[HI_CONV]], 32
-; SI: v_add_f64 [[RESULT:v\[[0-9]+:[0-9]+\]]], [[LDEXP]], [[LO_CONV]]
-; SI: buffer_store_dwordx2 [[RESULT]]
+; GCN-LABEL: {{^}}v_uint_to_fp_i64_to_f64
+; GCN: flat_load_dwordx2 v{{\[}}[[LO:[0-9]+]]:[[HI:[0-9]+]]{{\]}}
+; GCN-DAG: v_cvt_f64_u32_e32 [[HI_CONV:v\[[0-9]+:[0-9]+\]]], v[[HI]]
+; GCN-DAG: v_cvt_f64_u32_e32 [[LO_CONV:v\[[0-9]+:[0-9]+\]]], v[[LO]]
+; GCN-DAG: v_ldexp_f64 [[LDEXP:v\[[0-9]+:[0-9]+\]]], [[HI_CONV]], 32
+; GCN: v_add_f64 [[RESULT:v\[[0-9]+:[0-9]+\]]], [[LDEXP]], [[LO_CONV]]
+; GCN: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]]
 define amdgpu_kernel void @v_uint_to_fp_i64_to_f64(double addrspace(1)* %out, i64 addrspace(1)* %in) {
   %tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
   %gep = getelementptr i64, i64 addrspace(1)* %in, i32 %tid
@@ -18,52 +19,52 @@ define amdgpu_kernel void @v_uint_to_fp_i64_to_f64(double addrspace(1)* %out, i6
   ret void
 }
 
-; SI-LABEL: {{^}}s_uint_to_fp_i64_to_f64
+; GCN-LABEL: {{^}}s_uint_to_fp_i64_to_f64
 define amdgpu_kernel void @s_uint_to_fp_i64_to_f64(double addrspace(1)* %out, i64 %in) {
   %cast = uitofp i64 %in to double
   store double %cast, double addrspace(1)* %out, align 8
   ret void
 }
 
-; SI-LABEL: {{^}}s_uint_to_fp_v2i64_to_v2f64
+; GCN-LABEL: {{^}}s_uint_to_fp_v2i64_to_v2f64
 define amdgpu_kernel void @s_uint_to_fp_v2i64_to_v2f64(<2 x double> addrspace(1)* %out, <2 x i64> %in) {
   %cast = uitofp <2 x i64> %in to <2 x double>
   store <2 x double> %cast, <2 x double> addrspace(1)* %out, align 16
   ret void
 }
 
-; SI-LABEL: {{^}}s_uint_to_fp_v4i64_to_v4f64
+; GCN-LABEL: {{^}}s_uint_to_fp_v4i64_to_v4f64
 define amdgpu_kernel void @s_uint_to_fp_v4i64_to_v4f64(<4 x double> addrspace(1)* %out, <4 x i64> %in) {
   %cast = uitofp <4 x i64> %in to <4 x double>
   store <4 x double> %cast, <4 x double> addrspace(1)* %out, align 16
   ret void
 }
 
-; SI-LABEL: {{^}}s_uint_to_fp_i32_to_f64
-; SI: v_cvt_f64_u32_e32
-; SI: s_endpgm
+; GCN-LABEL: {{^}}s_uint_to_fp_i32_to_f64
+; GCN: v_cvt_f64_u32_e32
+; GCN: s_endpgm
 define amdgpu_kernel void @s_uint_to_fp_i32_to_f64(double addrspace(1)* %out, i32 %in) {
   %cast = uitofp i32 %in to double
   store double %cast, double addrspace(1)* %out, align 8
   ret void
 }
 
-; SI-LABEL: {{^}}s_uint_to_fp_v2i32_to_v2f64
-; SI: v_cvt_f64_u32_e32
-; SI: v_cvt_f64_u32_e32
-; SI: s_endpgm
+; GCN-LABEL: {{^}}s_uint_to_fp_v2i32_to_v2f64
+; GCN: v_cvt_f64_u32_e32
+; GCN: v_cvt_f64_u32_e32
+; GCN: s_endpgm
 define amdgpu_kernel void @s_uint_to_fp_v2i32_to_v2f64(<2 x double> addrspace(1)* %out, <2 x i32> %in) {
   %cast = uitofp <2 x i32> %in to <2 x double>
   store <2 x double> %cast, <2 x double> addrspace(1)* %out, align 16
   ret void
 }
 
-; SI-LABEL: {{^}}s_uint_to_fp_v4i32_to_v4f64
-; SI: v_cvt_f64_u32_e32
-; SI: v_cvt_f64_u32_e32
-; SI: v_cvt_f64_u32_e32
-; SI: v_cvt_f64_u32_e32
-; SI: s_endpgm
+; GCN-LABEL: {{^}}s_uint_to_fp_v4i32_to_v4f64
+; GCN: v_cvt_f64_u32_e32
+; GCN: v_cvt_f64_u32_e32
+; GCN: v_cvt_f64_u32_e32
+; GCN: v_cvt_f64_u32_e32
+; GCN: s_endpgm
 define amdgpu_kernel void @s_uint_to_fp_v4i32_to_v4f64(<4 x double> addrspace(1)* %out, <4 x i32> %in) {
   %cast = uitofp <4 x i32> %in to <4 x double>
   store <4 x double> %cast, <4 x double> addrspace(1)* %out, align 16
@@ -73,12 +74,12 @@ define amdgpu_kernel void @s_uint_to_fp_v4i32_to_v4f64(<4 x double> addrspace(1)
 ; We can't fold the SGPRs into v_cndmask_b32_e32, because it already
 ; uses an SGPR (implicit vcc).
 
-; SI-LABEL: {{^}}uint_to_fp_i1_to_f64:
-; SI-DAG: v_cmp_eq_u32_e64 vcc
-; SI-DAG: v_cndmask_b32_e32 v[[SEL:[0-9]+]], 0, v{{[0-9]+}}
-; SI-DAG: v_mov_b32_e32 v[[ZERO:[0-9]+]], 0{{$}}
-; SI: buffer_store_dwordx2 v{{\[}}[[ZERO]]:[[SEL]]{{\]}}
-; SI: s_endpgm
+; GCN-LABEL: {{^}}uint_to_fp_i1_to_f64:
+; GCN-DAG: v_cmp_eq_u32_e64 vcc
+; GCN-DAG: v_cndmask_b32_e32 v[[SEL:[0-9]+]], 0, v{{[0-9]+}}
+; GCN-DAG: v_mov_b32_e32 v[[ZERO:[0-9]+]], 0{{$}}
+; GCN: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[ZERO]]:[[SEL]]{{\]}}
+; GCN: s_endpgm
 define amdgpu_kernel void @uint_to_fp_i1_to_f64(double addrspace(1)* %out, i32 %in) {
   %cmp = icmp eq i32 %in, 0
   %fp = uitofp i1 %cmp to double
@@ -86,13 +87,39 @@ define amdgpu_kernel void @uint_to_fp_i1_to_f64(double addrspace(1)* %out, i32 %
   ret void
 }
 
-; SI-LABEL: {{^}}uint_to_fp_i1_to_f64_load:
-; SI: v_cndmask_b32_e64 [[IRESULT:v[0-9]]], 0, 1
-; SI-NEXT: v_cvt_f64_u32_e32 [[RESULT:v\[[0-9]+:[0-9]\]]], [[IRESULT]]
-; SI: buffer_store_dwordx2 [[RESULT]]
-; SI: s_endpgm
+; GCN-LABEL: {{^}}uint_to_fp_i1_to_f64_load:
+; GCN: v_cndmask_b32_e64 [[IRESULT:v[0-9]]], 0, 1
+; GCN: v_cvt_f64_u32_e32 [[RESULT:v\[[0-9]+:[0-9]\]]], [[IRESULT]]
+; GCN: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]]
+; GCN: s_endpgm
 define amdgpu_kernel void @uint_to_fp_i1_to_f64_load(double addrspace(1)* %out, i1 %in) {
   %fp = uitofp i1 %in to double
   store double %fp, double addrspace(1)* %out, align 8
   ret void
 }
+
+; GCN-LABEL: {{^}}s_uint_to_fp_i8_to_f64:
+; GCN: s_load_dword [[VAL:s[0-9]+]]
+; SI: s_and_b32 [[ZEXT:s[0-9]+]], [[VAL]], 0xff{{$}}
+; SI: v_cvt_f64_u32_e32 v{{\[[0-9]+:[0-9]+\]}}, [[ZEXT]]
+
+; VI: s_and_b32 [[ZEXT:s[0-9]+]], [[VAL]], 0xff{{$}}
+; VI: v_cvt_f64_u32_e32 v{{\[[0-9]+:[0-9]+\]}}, [[ZEXT]]
+define amdgpu_kernel void @s_uint_to_fp_i8_to_f64(double addrspace(1)* %out, i8 %in) {
+  %fp = uitofp i8 %in to double
+  store double %fp, double addrspace(1)* %out
+  ret void
+}
+
+; FIXME: Worse on VI
+; GCN-LABEL: {{^}}v_uint_to_fp_i8_to_f64:
+; SI: v_and_b32_e32 [[ZEXT:v[0-9]+]], 0xff, v0
+; SI: v_cvt_f64_u32_e32 v{{\[[0-9]+:[0-9]+\]}}, [[ZEXT]]
+
+; VI: v_mov_b32_e32 v{{[0-9]+}}
+; VI: v_and_b32_sdwa
+; VI: v_cvt_f64_u32_e32 v{{\[[0-9]+:[0-9]+\]}},
+define double @v_uint_to_fp_i8_to_f64(i8 %in) {
+  %fp = uitofp i8 %in to double
+  ret double %fp
+}
diff --git a/llvm/test/CodeGen/AMDGPU/uint_to_fp.ll b/llvm/test/CodeGen/AMDGPU/uint_to_fp.ll
index 763f48e7c..88a43abdc 100644
--- a/llvm/test/CodeGen/AMDGPU/uint_to_fp.ll
+++ b/llvm/test/CodeGen/AMDGPU/uint_to_fp.ll
@@ -78,7 +78,7 @@ define amdgpu_kernel void @v_uint_to_fp_v4i32(<4 x float> addrspace(1)* %out, <4
 
 ; FUNC-LABEL: {{^}}s_uint_to_fp_i1_to_f32:
 ; SI: v_cmp_eq_u32_e64 [[CMP:s\[[0-9]+:[0-9]\]]],
-; SI-NEXT: v_cndmask_b32_e64 [[RESULT:v[0-9]+]], 0, 1.0, [[CMP]]
+; SI: v_cndmask_b32_e64 [[RESULT:v[0-9]+]], 0, 1.0, [[CMP]]
 ; SI: buffer_store_dword [[RESULT]],
 ; SI: s_endpgm
 define amdgpu_kernel void @s_uint_to_fp_i1_to_f32(float addrspace(1)* %out, i32 %in) #0 {
diff --git a/llvm/test/CodeGen/AMDGPU/uitofp.f16.ll b/llvm/test/CodeGen/AMDGPU/uitofp.f16.ll
index 9cedce70d..1a42618d5 100644
--- a/llvm/test/CodeGen/AMDGPU/uitofp.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/uitofp.f16.ll
@@ -4,8 +4,10 @@
 ; GCN-LABEL: {{^}}uitofp_i16_to_f16
 ; GCN: buffer_load_ushort v[[A_I16:[0-9]+]]
 ; SI:  v_cvt_f32_u32_e32 v[[A_F32:[0-9]+]], v[[A_I16]]
-; VI:  v_cvt_f32_i32_e32 v[[A_F32:[0-9]+]], v[[A_I16]]
-; GCN: v_cvt_f16_f32_e32 v[[R_F16:[0-9]+]], v[[A_F32]]
+; SI: v_cvt_f16_f32_e32 v[[R_F16:[0-9]+]], v[[A_F32]]
+
+; VI:  v_cvt_f16_u16_e32 v[[R_F16:[0-9]+]], v[[A_I16]]
+
 ; GCN: buffer_store_short v[[R_F16]]
 ; GCN: s_endpgm
 define amdgpu_kernel void @uitofp_i16_to_f16(
@@ -46,10 +48,9 @@ entry:
 ; SI-DAG: v_lshlrev_b32_e32
 ; SI: v_or_b32_e32
 
-; VI-DAG: v_cvt_f16_f32_e32
-; VI-DAG: v_cvt_f32_i32_sdwa
-; VI-DAG: v_cvt_f32_i32_sdwa
-; VI-DAG: v_cvt_f16_f32_sdwa
+
+; VI-DAG: v_cvt_f16_u16_e32
+; VI-DAG: v_cvt_f16_u16_sdwa v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
 ; VI:     v_or_b32_e32
 
 ; GCN: buffer_store_dword
diff --git a/llvm/test/CodeGen/AMDGPU/uniform-cfg.ll b/llvm/test/CodeGen/AMDGPU/uniform-cfg.ll
index b5435ef14..965526340 100644
--- a/llvm/test/CodeGen/AMDGPU/uniform-cfg.ll
+++ b/llvm/test/CodeGen/AMDGPU/uniform-cfg.ll
@@ -330,13 +330,14 @@ endif:
 
 ; GCN-LABEL: {{^}}divergent_inside_uniform:
 ; GCN: s_cmp_lg_u32 s{{[0-9]+}}, 0
-; GCN: s_cbranch_scc1 [[ENDIF_LABEL:[0-9_A-Za-z]+]]
+; GCN: s_cbranch_scc0 [[IF_LABEL:[0-9_A-Za-z]+]]
+; GCN: [[ENDIF_LABEL:[0-9_A-Za-z]+]]:
+; GCN: [[IF_LABEL]]:
 ; GCN: v_cmp_gt_u32_e32 vcc, 16, v{{[0-9]+}}
 ; GCN: s_and_saveexec_b64 [[MASK:s\[[0-9]+:[0-9]+\]]], vcc
 ; GCN: ; mask branch [[ENDIF_LABEL]]
 ; GCN: v_mov_b32_e32 [[ONE:v[0-9]+]], 1
 ; GCN: buffer_store_dword [[ONE]]
-; GCN: [[ENDIF_LABEL]]:
 ; GCN: s_endpgm
 define amdgpu_kernel void @divergent_inside_uniform(i32 addrspace(1)* %out, i32 %cond) {
 entry:
diff --git a/llvm/test/CodeGen/AMDGPU/uniform-loop-inside-nonuniform.ll b/llvm/test/CodeGen/AMDGPU/uniform-loop-inside-nonuniform.ll
index 82283f397..a1cf6cf63 100644
--- a/llvm/test/CodeGen/AMDGPU/uniform-loop-inside-nonuniform.ll
+++ b/llvm/test/CodeGen/AMDGPU/uniform-loop-inside-nonuniform.ll
@@ -7,10 +7,9 @@
 ; CHECK: s_and_saveexec_b64
 ; CHECK-NEXT: ; mask branch
 ; CHECK-NEXT: s_cbranch_execz BB{{[0-9]+_[0-9]+}}
-; CHECK-NEXT: BB{{[0-9]+_[0-9]+}}: ; %loop_body.preheader
 
-; CHECK: [[LOOP_BODY_LABEL:BB[0-9]+_[0-9]+]]:
-; CHECK: s_cbranch_vccz [[LOOP_BODY_LABEL]]
+; CHECK: [[LOOP_BODY_LABEL:BB[0-9]+_[0-9]+]]: ; %loop_body
+; CHECK: s_cbranch_scc0 [[LOOP_BODY_LABEL]]
 
 ; CHECK: s_endpgm
 define amdgpu_ps void @test1(<8 x i32> inreg %rsrc, <2 x i32> %addr.base, i32 %y, i32 %p) {
diff --git a/llvm/test/CodeGen/AMDGPU/unsupported-calls.ll b/llvm/test/CodeGen/AMDGPU/unsupported-calls.ll
index 303a0d6a1..3cff62735 100644
--- a/llvm/test/CodeGen/AMDGPU/unsupported-calls.ll
+++ b/llvm/test/CodeGen/AMDGPU/unsupported-calls.ll
@@ -74,3 +74,13 @@ define amdgpu_ps i32 @test_call_from_shader() {
   %call = call i32 @defined_function(i32 0)
   ret i32 %call
 }
+
+; FIXME: Bad error message
+; GCN: error: <unknown>:0:0: in function test_call_absolute void (): unsupported indirect call to function <unknown>
+; R600: error: <unknown>:0:0: in function test_call_absolute void (): unsupported call to function <unknown>
+define amdgpu_kernel void @test_call_absolute() #0 {
+  %val = call i32 inttoptr (i64 1234 to i32(i32)*) (i32 1)
+  %op = add i32 %val, 1
+  store volatile i32 %op, i32 addrspace(1)* undef
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/use-sgpr-multiple-times.ll b/llvm/test/CodeGen/AMDGPU/use-sgpr-multiple-times.ll
index 50cf85e28..fbf7364bf 100644
--- a/llvm/test/CodeGen/AMDGPU/use-sgpr-multiple-times.ll
+++ b/llvm/test/CodeGen/AMDGPU/use-sgpr-multiple-times.ll
@@ -226,13 +226,12 @@ define amdgpu_kernel void @test_literal_use_twice_ternary_op_s_k_k_x2(float addr
 ; GCN-LABEL: {{^}}test_s0_s1_k_f32:
 ; SI-DAG: s_load_dwordx2 s{{\[}}[[SGPR0:[0-9]+]]:[[SGPR1:[0-9]+]]{{\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0xb
 ; VI-DAG: s_load_dwordx2 s{{\[}}[[SGPR0:[0-9]+]]:[[SGPR1:[0-9]+]]{{\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x2c
-; GCN-DAG: s_mov_b32 [[SK0:s[0-9]+]], 0x44800000
+; GCN-DAG: v_mov_b32_e32 [[VK0:v[0-9]+]], 0x44800000
 ; GCN-DAG: v_mov_b32_e32 [[VS1:v[0-9]+]], s[[SGPR1]]
-; GCN-DAG: v_mov_b32_e32 [[VS0:v[0-9]+]], s[[SGPR0]]
 
-; GCN-DAG: v_fma_f32 [[RESULT0:v[0-9]+]], [[VS0]], [[VS1]], [[SK0]]
-; GCN-DAG: s_mov_b32 [[SK1:s[0-9]+]], 0x45800000
-; GCN-DAG: v_fma_f32 [[RESULT1:v[0-9]+]], [[VS0]], [[VS1]], [[SK1]]
+; GCN-DAG: v_fma_f32 [[RESULT0:v[0-9]+]], s[[SGPR0]], [[VS1]], [[VK0]]
+; GCN-DAG: v_mov_b32_e32 [[VK1:v[0-9]+]], 0x45800000
+; GCN-DAG: v_fma_f32 [[RESULT1:v[0-9]+]], s[[SGPR0]], [[VS1]], [[VK1]]
 
 ; GCN: buffer_store_dword [[RESULT0]]
 ; GCN: buffer_store_dword [[RESULT1]]
diff --git a/llvm/test/CodeGen/AMDGPU/v_mac_f16.ll b/llvm/test/CodeGen/AMDGPU/v_mac_f16.ll
index f2c2cc847..e503fbb51 100644
--- a/llvm/test/CodeGen/AMDGPU/v_mac_f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/v_mac_f16.ll
@@ -304,14 +304,17 @@ entry:
 ; GCN: {{buffer|flat}}_load_dword v[[C_V2_F16:[0-9]+]]
 
 ; SI:  v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_V2_F16]]
-; SI:  v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_V2_F16]]
-; SI:  v_cvt_f32_f16_e32 v[[A_F32_1:[0-9]+]], v[[A_F16_1]]
-; SI:  v_lshrrev_b32_e32 v[[B_F16_1:[0-9]+]], 16, v[[B_V2_F16]]
-; SI:  v_cvt_f32_f16_e32 v[[B_F32_1:[0-9]+]], v[[B_F16_1]]
-; SI:  v_cvt_f32_f16_e32 v[[B_F32_0:[0-9]+]], v[[B_V2_F16]]
-; SI:  v_lshrrev_b32_e32 v[[C_F16_1:[0-9]+]], 16, v[[C_V2_F16]]
-; SI:  v_cvt_f32_f16_e32 v[[C_F32_1:[0-9]+]], v[[C_F16_1]]
+; SI-DAG:  v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_V2_F16]]
+; SI-DAG:  v_cvt_f32_f16_e32 v[[A_F32_1:[0-9]+]], v[[A_F16_1]]
+
+; SI-DAG:  v_lshrrev_b32_e32 v[[B_F16_1:[0-9]+]], 16, v[[B_V2_F16]]
+; SI-DAG:  v_cvt_f32_f16_e32 v[[B_F32_1:[0-9]+]], v[[B_F16_1]]
+; SI-DAG:  v_cvt_f32_f16_e32 v[[B_F32_0:[0-9]+]], v[[B_V2_F16]]
+
+; SI-DAG:  v_lshrrev_b32_e32 v[[C_F16_1:[0-9]+]], 16, v[[C_V2_F16]]
+; SI-DAG:  v_cvt_f32_f16_e32 v[[C_F32_1:[0-9]+]], v[[C_F16_1]]
 ; SI-DAG:  v_cvt_f32_f16_e32 v[[C_F32_0:[0-9]+]], v[[C_V2_F16]]
+
 ; SI-DAG:  v_mac_f32_e32 v[[C_F32_0]], v[[A_F32_0]], v[[B_F32_0]]
 ; SI-DAG:  v_cvt_f16_f32_e32 v[[R_F16_LO:[0-9]+]], v[[C_F32_0]]
 ; SI-DAG:  v_mac_f32_e32 v[[C_F32_1]], v[[A_F32_1]], v[[B_F32_1]]
diff --git a/llvm/test/CodeGen/AMDGPU/valu-i1.ll b/llvm/test/CodeGen/AMDGPU/valu-i1.ll
index 718b1b523..ea74268db 100644
--- a/llvm/test/CodeGen/AMDGPU/valu-i1.ll
+++ b/llvm/test/CodeGen/AMDGPU/valu-i1.ll
@@ -165,8 +165,8 @@ exit:
 ; SI: [[LABEL_LOOP:BB[0-9]+_[0-9]+]]:
 ; SI: buffer_load_dword
 ; SI-DAG: buffer_store_dword
-; SI-DAG: v_cmp_eq_u32_e32 vcc, 0x100
-; SI: s_cbranch_vccz [[LABEL_LOOP]]
+; SI-DAG: s_cmpk_eq_i32 s{{[0-9+]}}, 0x100
+; SI: s_cbranch_scc0 [[LABEL_LOOP]]
 ; SI: [[LABEL_EXIT]]:
 ; SI: s_endpgm
 
@@ -214,7 +214,7 @@ exit:
 ; SI-DAG: v_cmp_ne_u32_e32 [[NEG1_CHECK_1:vcc]], -1, [[B]]
 ; SI: s_and_b64 [[ORNEG1:s\[[0-9]+:[0-9]+\]]], [[NEG1_CHECK_1]], [[NEG1_CHECK_0]]
 ; SI: s_and_saveexec_b64 [[ORNEG2:s\[[0-9]+:[0-9]+\]]], [[ORNEG1]]
-; SI: s_cbranch_execz [[LABEL_FLOW:BB[0-9]+_[0-9]+]]
+; SI: ; mask branch [[LABEL_FLOW:BB[0-9]+_[0-9]+]]
 
 ; SI: BB{{[0-9]+_[0-9]+}}: ; %bb20
 ; SI: buffer_store_dword
@@ -223,9 +223,8 @@ exit:
 ; SI-NEXT: ; in Loop: Header=[[LABEL_LOOP]]
 ; SI-NEXT: s_or_b64 exec, exec, [[ORNEG2]]
 ; SI-NEXT: s_and_b64 [[TMP1:s\[[0-9]+:[0-9]+\]]],
-; SI-NEXT: s_or_b64 [[TMP2:s\[[0-9]+:[0-9]+\]]], [[TMP1]], [[COND_STATE]]
-; SI-NEXT: s_mov_b64 [[COND_STATE]], [[TMP2]]
-; SI-NEXT: s_andn2_b64 exec, exec, [[TMP2]]
+; SI-NEXT: s_or_b64 [[COND_STATE]], [[TMP1]], [[COND_STATE]]
+; SI-NEXT: s_andn2_b64 exec, exec, [[COND_STATE]]
 ; SI-NEXT: s_cbranch_execnz [[LABEL_LOOP]]
 
 ; SI: [[LABEL_EXIT]]:
diff --git a/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll b/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll
index 50f06f9e8..3e8384ad3 100644
--- a/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll
@@ -96,12 +96,12 @@ define <4 x half> @shuffle_v4f16_3uu7(<4 x half> addrspace(1)* %arg0, <4 x half>
 ; GFX9-LABEL: shuffle_v4f16_3uu7:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dword v2, v[2:3], off offset:4
 ; GFX9-NEXT:    global_load_dword v0, v[0:1], off offset:4
-; GFX9-NEXT:    global_load_dword v1, v[2:3], off offset:4
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
-; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v1, 16, v2
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
@@ -114,15 +114,15 @@ define <4 x half> @shuffle_v4f16_35u5(<4 x half> addrspace(1)* %arg0, <4 x half>
 ; GFX9-LABEL: shuffle_v4f16_35u5:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dword v2, v[2:3], off
 ; GFX9-NEXT:    global_load_dword v0, v[0:1], off offset:4
-; GFX9-NEXT:    global_load_dword v1, v[2:3], off
-; GFX9-NEXT:    v_mov_b32_e32 v2, 0xffff
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0xffff
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
-; GFX9-NEXT:    v_and_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
-; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT:    v_and_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT:    v_lshl_or_b32 v0, v2, 16, v0
+; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 16, v2
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
   %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
@@ -134,15 +134,15 @@ define <4 x half> @shuffle_v4f16_357u(<4 x half> addrspace(1)* %arg0, <4 x half>
 ; GFX9-LABEL: shuffle_v4f16_357u:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_load_dword v4, v[0:1], off offset:4
-; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[2:3], off
-; GFX9-NEXT:    v_mov_b32_e32 v2, 0xffff
+; GFX9-NEXT:    global_load_dwordx2 v[2:3], v[2:3], off
+; GFX9-NEXT:    global_load_dword v0, v[0:1], off offset:4
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0xffff
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
-; GFX9-NEXT:    v_and_b32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT:    v_lshl_or_b32 v0, v0, 16, v2
+; GFX9-NEXT:    v_and_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v1, 16, v3
+; GFX9-NEXT:    v_lshl_or_b32 v0, v2, 16, v0
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
   %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
@@ -387,12 +387,13 @@ define <4 x half> @shuffle_v4f16_2356(<4 x half> addrspace(1)* %arg0, <4 x half>
 ; GFX9-LABEL: shuffle_v4f16_2356:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dwordx2 v[2:3], v[2:3], off
 ; GFX9-NEXT:    global_load_dwordx2 v[4:5], v[0:1], off
-; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[2:3], off
-; GFX9-NEXT:    v_mov_b32_e32 v2, 0xffff
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0xffff
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    v_and_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT:    v_lshl_or_b32 v1, v3, 16, v0
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_and_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT:    v_lshl_or_b32 v1, v1, 16, v0
 ; GFX9-NEXT:    v_mov_b32_e32 v0, v5
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
@@ -405,11 +406,10 @@ define <4 x half> @shuffle_v4f16_5623(<4 x half> addrspace(1)* %arg0, <4 x half>
 ; GFX9-LABEL: shuffle_v4f16_5623:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off
 ; GFX9-NEXT:    global_load_dwordx2 v[2:3], v[2:3], off
-; GFX9-NEXT:    s_waitcnt vmcnt(1)
-; GFX9-NEXT:    v_mov_b32_e32 v0, 0xffff
+; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0xffff
 ; GFX9-NEXT:    v_and_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
 ; GFX9-NEXT:    v_lshl_or_b32 v0, v3, 16, v0
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
@@ -423,15 +423,15 @@ define <4 x half> @shuffle_v4f16_3456(<4 x half> addrspace(1)* %arg0, <4 x half>
 ; GFX9-LABEL: shuffle_v4f16_3456:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_load_dword v4, v[0:1], off offset:4
-; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[2:3], off
-; GFX9-NEXT:    v_mov_b32_e32 v2, 0xffff
+; GFX9-NEXT:    global_load_dwordx2 v[2:3], v[2:3], off
+; GFX9-NEXT:    global_load_dword v0, v[0:1], off offset:4
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0xffff
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
-; GFX9-NEXT:    v_and_b32_sdwa v3, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT:    v_and_b32_sdwa v4, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_and_b32_sdwa v2, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT:    v_lshl_or_b32 v0, v0, 16, v3
-; GFX9-NEXT:    v_lshl_or_b32 v1, v1, 16, v2
+; GFX9-NEXT:    v_and_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT:    v_lshl_or_b32 v0, v2, 16, v0
+; GFX9-NEXT:    v_lshl_or_b32 v1, v3, 16, v4
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
   %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
@@ -443,15 +443,15 @@ define <4 x half> @shuffle_v4f16_5634(<4 x half> addrspace(1)* %arg0, <4 x half>
 ; GFX9-LABEL: shuffle_v4f16_5634:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dwordx2 v[2:3], v[2:3], off
 ; GFX9-NEXT:    global_load_dword v0, v[0:1], off offset:4
-; GFX9-NEXT:    global_load_dwordx2 v[1:2], v[2:3], off
-; GFX9-NEXT:    v_mov_b32_e32 v3, 0xffff
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0xffff
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
-; GFX9-NEXT:    v_and_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT:    v_and_b32_sdwa v4, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_and_b32_sdwa v3, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT:    v_lshl_or_b32 v1, v1, 16, v0
-; GFX9-NEXT:    v_lshl_or_b32 v0, v2, 16, v3
+; GFX9-NEXT:    v_and_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT:    v_lshl_or_b32 v1, v2, 16, v0
+; GFX9-NEXT:    v_lshl_or_b32 v0, v3, 16, v4
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
   %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
@@ -463,16 +463,16 @@ define <4 x half> @shuffle_v4f16_5734(<4 x half> addrspace(1)* %arg0, <4 x half>
 ; GFX9-LABEL: shuffle_v4f16_5734:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dwordx2 v[2:3], v[2:3], off
 ; GFX9-NEXT:    global_load_dword v0, v[0:1], off offset:4
-; GFX9-NEXT:    global_load_dwordx2 v[1:2], v[2:3], off
-; GFX9-NEXT:    v_mov_b32_e32 v3, 0xffff
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0xffff
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
-; GFX9-NEXT:    v_and_b32_sdwa v4, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v2
-; GFX9-NEXT:    v_and_b32_sdwa v2, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT:    v_lshl_or_b32 v0, v0, 16, v2
-; GFX9-NEXT:    v_lshl_or_b32 v1, v1, 16, v4
+; GFX9-NEXT:    v_and_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT:    v_and_b32_sdwa v4, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT:    v_lshl_or_b32 v1, v2, 16, v0
+; GFX9-NEXT:    v_lshl_or_b32 v0, v3, 16, v4
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
   %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
@@ -484,12 +484,13 @@ define <4 x i16> @shuffle_v4i16_2356(<4 x i16> addrspace(1)* %arg0, <4 x i16> ad
 ; GFX9-LABEL: shuffle_v4i16_2356:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dwordx2 v[2:3], v[2:3], off
 ; GFX9-NEXT:    global_load_dwordx2 v[4:5], v[0:1], off
-; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[2:3], off
-; GFX9-NEXT:    v_mov_b32_e32 v2, 0xffff
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0xffff
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    v_and_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT:    v_lshl_or_b32 v1, v3, 16, v0
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_and_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT:    v_lshl_or_b32 v1, v1, 16, v0
 ; GFX9-NEXT:    v_mov_b32_e32 v0, v5
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %val0 = load <4 x i16>, <4 x i16> addrspace(1)* %arg0
@@ -552,13 +553,13 @@ define <4 x half> @shuffle_v4f16_1100(<4 x half> addrspace(1)* %arg0, <4 x half>
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0xffff
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v1, 0xffff
-; GFX9-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
-; GFX9-NEXT:    v_and_b32_e32 v3, v1, v0
-; GFX9-NEXT:    v_and_b32_e32 v4, v1, v2
-; GFX9-NEXT:    v_lshl_or_b32 v1, v0, 16, v3
-; GFX9-NEXT:    v_lshl_or_b32 v0, v2, 16, v4
+; GFX9-NEXT:    v_and_b32_e32 v1, v2, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
+; GFX9-NEXT:    v_lshl_or_b32 v1, v0, 16, v1
+; GFX9-NEXT:    v_and_b32_e32 v0, v2, v3
+; GFX9-NEXT:    v_lshl_or_b32 v0, v3, 16, v0
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
   %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
@@ -570,12 +571,12 @@ define <4 x half> @shuffle_v4f16_6161(<4 x half> addrspace(1)* %arg0, <4 x half>
 ; GFX9-LABEL: shuffle_v4f16_6161:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dword v2, v[2:3], off offset:4
 ; GFX9-NEXT:    global_load_dword v0, v[0:1], off
-; GFX9-NEXT:    global_load_dword v1, v[2:3], off offset:4
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
-; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-NEXT:    v_and_b32_e32 v1, 0xffff, v2
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
 ; GFX9-NEXT:    v_lshl_or_b32 v0, v0, 16, v1
 ; GFX9-NEXT:    v_mov_b32_e32 v1, v0
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
@@ -678,12 +679,13 @@ define <4 x half> @shuffle_v8f16_13_14_2_3(<8 x half> addrspace(1)* %arg0, <8 x
 ; GFX9-LABEL: shuffle_v8f16_13_14_2_3:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_load_dword v1, v[0:1], off offset:4
 ; GFX9-NEXT:    global_load_dwordx4 v[2:5], v[2:3], off
+; GFX9-NEXT:    global_load_dword v1, v[0:1], off offset:4
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 0xffff
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_and_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
 ; GFX9-NEXT:    v_lshl_or_b32 v0, v5, 16, v0
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %val0 = load <8 x half>, <8 x half> addrspace(1)* %arg0
   %val1 = load <8 x half>, <8 x half> addrspace(1)* %arg1
@@ -757,9 +759,9 @@ define amdgpu_kernel void @fma_shuffle(<4 x half> addrspace(1)* nocapture readon
 ; GFX9-NEXT:    v_mov_b32_e32 v5, s5
 ; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, s4, v4
 ; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
-; GFX9-NEXT:    global_load_dwordx2 v[6:7], v[4:5], off
-; GFX9-NEXT:    global_load_dwordx2 v[2:3], v[2:3], off
 ; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off
+; GFX9-NEXT:    global_load_dwordx2 v[2:3], v[2:3], off
+; GFX9-NEXT:    global_load_dwordx2 v[6:7], v[4:5], off
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_pk_fma_f16 v6, v0, v2, v6 op_sel_hi:[0,1,1]
 ; GFX9-NEXT:    v_pk_fma_f16 v2, v1, v2, v7 op_sel_hi:[0,1,1]
diff --git a/llvm/test/CodeGen/AMDGPU/vgpr-descriptor-waterfall-loop-idom-update.ll b/llvm/test/CodeGen/AMDGPU/vgpr-descriptor-waterfall-loop-idom-update.ll
new file mode 100644
index 000000000..d3cb4ec3b
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/vgpr-descriptor-waterfall-loop-idom-update.ll
@@ -0,0 +1,47 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -march=amdgcn -mcpu=gfx1010 | FileCheck %s --check-prefix=GCN
+
+define void @vgpr_descriptor_waterfall_loop_idom_update(<4 x i32>* %arg) {
+; GCN-LABEL: vgpr_descriptor_waterfall_loop_idom_update:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    s_waitcnt_vscnt null, 0x0
+; GCN-NEXT:    ; implicit-def: $vcc_hi
+; GCN-NEXT:  BB0_1: ; %bb0
+; GCN-NEXT:    ; =>This Loop Header: Depth=1
+; GCN-NEXT:    ; Child Loop BB0_2 Depth 2
+; GCN-NEXT:    v_add_co_u32_e64 v2, vcc_lo, v0, 8
+; GCN-NEXT:    s_mov_b32 s5, exec_lo
+; GCN-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v1, vcc_lo
+; GCN-NEXT:    flat_load_dwordx2 v[2:3], v[2:3]
+; GCN-NEXT:    flat_load_dwordx2 v[4:5], v[0:1]
+; GCN-NEXT:  BB0_2: ; Parent Loop BB0_1 Depth=1
+; GCN-NEXT:    ; => This Inner Loop Header: Depth=2
+; GCN-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_readfirstlane_b32 s8, v4
+; GCN-NEXT:    v_readfirstlane_b32 s9, v5
+; GCN-NEXT:    v_readfirstlane_b32 s10, v2
+; GCN-NEXT:    v_readfirstlane_b32 s11, v3
+; GCN-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[4:5]
+; GCN-NEXT:    v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
+; GCN-NEXT:    s_and_b32 s4, vcc_lo, s4
+; GCN-NEXT:    s_and_saveexec_b32 s4, s4
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    buffer_store_dword v0, v0, s[8:11], 0 offen
+; GCN-NEXT:    v_nop
+; GCN-NEXT:    s_xor_b32 exec_lo, exec_lo, s4
+; GCN-NEXT:    s_cbranch_execnz BB0_2
+; GCN-NEXT:  ; %bb.3: ; in Loop: Header=BB0_1 Depth=1
+; GCN-NEXT:    s_mov_b32 exec_lo, s5
+; GCN-NEXT:    s_branch BB0_1
+entry:
+  br label %bb0
+
+bb0:
+  %desc = load <4 x i32>, <4 x i32>* %arg, align 8
+  tail call void @llvm.amdgcn.buffer.store.f32(float undef, <4 x i32> %desc, i32 0, i32 undef, i1 zeroext false, i1 zeroext false)
+  br label %bb0
+}
+
+; Function Attrs: nounwind writeonly
+declare void @llvm.amdgcn.buffer.store.f32(float, <4 x i32>, i32, i32, i1 immarg, i1 immarg)
diff --git a/llvm/test/CodeGen/AMDGPU/vgpr-spill-emergency-stack-slot-compute.ll b/llvm/test/CodeGen/AMDGPU/vgpr-spill-emergency-stack-slot-compute.ll
index 0c52daca0..b0e9171cb 100644
--- a/llvm/test/CodeGen/AMDGPU/vgpr-spill-emergency-stack-slot-compute.ll
+++ b/llvm/test/CodeGen/AMDGPU/vgpr-spill-emergency-stack-slot-compute.ll
@@ -1,3 +1,4 @@
+; XFAIL: *
 ; RUN: llc -march=amdgcn -mtriple=amdgcn-- -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GCNMESA -check-prefix=SIMESA %s
 ; RUN: llc -march=amdgcn -mtriple=amdgcn-- -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GCNMESA -check-prefix=VIMESA %s
 ; RUN: llc -march=amdgcn -mtriple=amdgcn-- -mcpu=gfx900 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GCNMESA -check-prefix=GFX9MESA %s
diff --git a/llvm/test/CodeGen/AMDGPU/wait.ll b/llvm/test/CodeGen/AMDGPU/wait.ll
index 12dc7cc7a..c06c16e29 100644
--- a/llvm/test/CodeGen/AMDGPU/wait.ll
+++ b/llvm/test/CodeGen/AMDGPU/wait.ll
@@ -7,10 +7,14 @@
 ; DEFAULT-LABEL: {{^}}main:
 ; DEFAULT: s_load_dwordx4
 ; DEFAULT: s_load_dwordx4
+; DEFAULT: s_waitcnt lgkmcnt(0)
+; DEFAULT: buffer_load_format_xyzw
+; DEFAULT: s_waitcnt vmcnt(0)
+; DEFAULT: buffer_load_format_xyzw
 ; DEFAULT: s_waitcnt vmcnt(0)
 ; DEFAULT: exp
-; DEFAULT: s_waitcnt lgkmcnt(0)
-; DEFAULT: s_endpgm
+; DEFAULT: exp
+; DEFAULT-NEXT: s_endpgm
 define amdgpu_vs void @main(<16 x i8> addrspace(4)* inreg %arg, <16 x i8> addrspace(4)* inreg %arg1, <32 x i8> addrspace(4)* inreg %arg2, <16 x i8> addrspace(4)* inreg %arg3, <16 x i8> addrspace(4)* inreg %arg4, i32 inreg %arg5, i32 %arg6, i32 %arg7, i32 %arg8, i32 %arg9, float addrspace(4)* inreg %constptr) #0 {
 main_body:
   %tmp = getelementptr <16 x i8>, <16 x i8> addrspace(4)* %arg3, i32 0
diff --git a/llvm/test/CodeGen/AMDGPU/waitcnt-vscnt.ll b/llvm/test/CodeGen/AMDGPU/waitcnt-vscnt.ll
index c8777f441..4cbd89147 100644
--- a/llvm/test/CodeGen/AMDGPU/waitcnt-vscnt.ll
+++ b/llvm/test/CodeGen/AMDGPU/waitcnt-vscnt.ll
@@ -5,8 +5,8 @@
 ; GCN-LABEL: barrier_vmcnt_global:
 ; GFX8:         flat_load_dword
 ; GFX9_10:      global_load_dword
-; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX9_10-NEXT: s_waitcnt vmcnt(0){{$}}
+; GFX8:         s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX9_10:      s_waitcnt vmcnt(0){{$}}
 ; GCN-NEXT:     s_barrier
 define amdgpu_kernel void @barrier_vmcnt_global(i32 addrspace(1)* %arg) {
 bb:
@@ -28,9 +28,9 @@ bb:
 ; GCN-LABEL: barrier_vscnt_global:
 ; GFX8:       flat_store_dword
 ; GFX9_10:    global_store_dword
-; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX9-NEXT:  s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX9:       s_waitcnt vmcnt(0){{$}}
+; GFX10:      s_waitcnt_vscnt null, 0x0
 ; GCN-NEXT:   s_barrier
 define amdgpu_kernel void @barrier_vscnt_global(i32 addrspace(1)* %arg) {
 bb:
@@ -54,9 +54,9 @@ bb:
 ; GCN-LABEL: barrier_vmcnt_vscnt_global:
 ; GFX8:         flat_load_dword
 ; GFX9_10:      global_load_dword
-; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX9_10-NEXT: s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT:   s_waitcnt_vscnt null, 0x0
+; GFX8:         s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX9_10:      s_waitcnt vmcnt(0){{$}}
+; GFX10:        s_waitcnt_vscnt null, 0x0
 ; GCN-NEXT:     s_barrier
 define amdgpu_kernel void @barrier_vmcnt_vscnt_global(i32 addrspace(1)* %arg) {
 bb:
@@ -81,7 +81,7 @@ bb:
 
 ; GCN-LABEL: barrier_vmcnt_flat:
 ; GCN:      flat_load_dword
-; GCN-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
 ; GCN-NEXT: s_barrier
 define amdgpu_kernel void @barrier_vmcnt_flat(i32* %arg) {
 bb:
@@ -102,9 +102,9 @@ bb:
 
 ; GCN-LABEL: barrier_vscnt_flat:
 ; GCN:         flat_store_dword
-; GFX8_9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10-NEXT:  s_waitcnt lgkmcnt(0){{$}}
-; GFX10-NEXT:  s_waitcnt_vscnt null, 0x0
+; GFX8_9:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10:       s_waitcnt lgkmcnt(0){{$}}
+; GFX10:       s_waitcnt_vscnt null, 0x0
 ; GCN-NEXT:    s_barrier
 define amdgpu_kernel void @barrier_vscnt_flat(i32* %arg) {
 bb:
@@ -127,8 +127,8 @@ bb:
 
 ; GCN-LABEL: barrier_vmcnt_vscnt_flat:
 ; GCN:        flat_load_dword
-; GCN-NEXT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt_vscnt null, 0x0
 ; GCN-NEXT:   s_barrier
 define amdgpu_kernel void @barrier_vmcnt_vscnt_flat(i32* %arg) {
 bb:
@@ -153,8 +153,8 @@ bb:
 
 ; GCN-LABEL: barrier_vmcnt_vscnt_flat_workgroup:
 ; GCN:        flat_load_dword
-; GCN-NEXT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt_vscnt null, 0x0
 ; GCN-NEXT:   s_barrier
 define amdgpu_kernel void @barrier_vmcnt_vscnt_flat_workgroup(i32* %arg) {
 bb:
@@ -218,8 +218,8 @@ bb:
 
 ; GCN-LABEL: store_vscnt_private:
 ; GCN:         buffer_store_dword
-; GFX8_9-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT:  s_waitcnt_vscnt null, 0x0
+; GFX8_9:      s_waitcnt vmcnt(0)
+; GFX10:       s_waitcnt_vscnt null, 0x0
 ; GCN-NEXT:    s_setpc_b64
 define void @store_vscnt_private(i32 addrspace(5)* %p) {
   store i32 0, i32 addrspace(5)* %p
@@ -229,8 +229,8 @@ define void @store_vscnt_private(i32 addrspace(5)* %p) {
 ; GCN-LABEL: store_vscnt_global:
 ; GFX8:        flat_store_dword
 ; GFX9_10:     global_store_dword
-; GFX8_9-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT:  s_waitcnt_vscnt null, 0x0
+; GFX8_9:      s_waitcnt vmcnt(0)
+; GFX10:       s_waitcnt_vscnt null, 0x0
 ; GCN-NEXT:    s_setpc_b64
 define void @store_vscnt_global(i32 addrspace(1)* %p) {
   store i32 0, i32 addrspace(1)* %p
@@ -239,9 +239,9 @@ define void @store_vscnt_global(i32 addrspace(1)* %p) {
 
 ; GCN-LABEL: store_vscnt_flat:
 ; GCN:         flat_store_dword
-; GFX8_9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10-NEXT:  s_waitcnt lgkmcnt(0){{$}}
-; GFX10-NEXT:  s_waitcnt_vscnt null, 0x0
+; GFX8_9:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10:       s_waitcnt lgkmcnt(0){{$}}
+; GFX10:       s_waitcnt_vscnt null, 0x0
 ; GCN-NEXT:    s_setpc_b64
 define void @store_vscnt_flat(i32* %p) {
   store i32 0, i32* %p
@@ -249,8 +249,8 @@ define void @store_vscnt_flat(i32* %p) {
 }
 
 ; GCN-LABEL: function_prologue:
-; GCN-NEXT:   s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GCN:        s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt_vscnt null, 0x0
 ; GCN-NEXT:   s_setpc_b64
 define void @function_prologue() {
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/wave32.ll b/llvm/test/CodeGen/AMDGPU/wave32.ll
index a71ca5db7..ce73f9c89 100644
--- a/llvm/test/CodeGen/AMDGPU/wave32.ll
+++ b/llvm/test/CodeGen/AMDGPU/wave32.ll
@@ -232,14 +232,29 @@ bb13:
 ; GCN:     s_cbranch_execz
 ; GCN:   BB{{.*}}:
 ; GCN:   BB{{.*}}:
-; GFX1032: s_andn2_b32 s{{[0-9]+}}, s{{[0-9]+}}, exec_lo
-; GFX1064: s_andn2_b64 s[{{[0-9:]+}}], s[{{[0-9:]+}}], exec
-; GFX1032: s_or_b32 s{{[0-9]+}}, vcc_lo, s{{[0-9]+}}
-; GFX1032: s_or_b32 s{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}}
-; GFX1064: s_or_b64 s[{{[0-9:]+}}], vcc, s[{{[0-9:]+}}]
-; GFX1064: s_or_b64 s[{{[0-9:]+}}], s[{{[0-9:]+}}], s[{{[0-9:]+}}]
+
+; GFX1032: s_or_b32 [[MASK0:s[0-9]+]], [[MASK0]], vcc_lo
+; GFX1064: s_or_b64 [[MASK0:s\[[0-9:]+\]]], [[MASK0]], vcc
+; GFX1032: s_andn2_b32 [[MASK1:s[0-9]+]], [[MASK1]], exec_lo
+; GFX1064: s_andn2_b64 [[MASK1:s\[[0-9:]+\]]], [[MASK1]], exec
+; GCN:     global_store_dword
+; GFX1032: s_and_b32 [[MASK0]], [[MASK0]], exec_lo
+; GFX1064: s_and_b64 [[MASK0]], [[MASK0]], exec
+; GFX1032: s_or_b32 [[MASK1]], [[MASK1]], [[MASK0]]
+; GFX1064: s_or_b64 [[MASK1]], [[MASK1]], [[MASK0]]
+; GCN:   BB{{.*}}: ; %Flow
+; GFX1032: s_and_b32 [[TMP0:s[0-9]+]], exec_lo, [[MASK1]]
+; GFX1064: s_and_b64 [[TMP0:s\[[0-9:]+\]]], exec, [[MASK1]]
+; GFX1032: s_or_b32  [[ACC:s[0-9]+]], [[TMP0]], [[ACC]]
+; GFX1064: s_or_b64  [[ACC:s\[[0-9:]+\]]], [[TMP0]], [[ACC]]
+; GFX1032: s_andn2_b32 exec_lo, exec_lo, [[ACC]]
+; GFX1064: s_andn2_b64 exec, exec, [[ACC]]
 ; GCN:     s_cbranch_execz
 ; GCN:   BB{{.*}}:
+; GCN: s_load_dword [[LOAD:s[0-9]+]]
+; GFX1032: s_or_b32 [[MASK1]], [[MASK1]], exec_lo
+; GFX1064: s_or_b64 [[MASK1]], [[MASK1]], exec
+; GCN: s_cmp_lt_i32 [[LOAD]], 11
 define amdgpu_kernel void @test_loop_with_if_else_break(i32 addrspace(1)* %arg) #0 {
 bb:
   %tmp = tail call i32 @llvm.amdgcn.workitem.id.x()
@@ -476,10 +491,10 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}test_br_cc_f16:
-; GFX1032:      v_cmp_nlt_f16_e32 vcc_lo,
-; GFX1032-NEXT: s_and_b32 vcc_lo, exec_lo, vcc_lo
-; GFX1064:      v_cmp_nlt_f16_e32 vcc,
-; GFX1064-NEXT: s_and_b64 vcc, exec, vcc{{$}}
+; GFX1032:  v_cmp_nlt_f16_e32 vcc_lo,
+; GFX1032:  s_and_b32 vcc_lo, exec_lo, vcc_lo
+; GFX1064:  v_cmp_nlt_f16_e32 vcc,
+; GFX1064:  s_and_b64 vcc, exec, vcc{{$}}
 ; GCN-NEXT: s_cbranch_vccnz
 define amdgpu_kernel void @test_br_cc_f16(
     half addrspace(1)* %r,
@@ -871,33 +886,6 @@ endif:
   ret void
 }
 
-; GCN-LABEL: {{^}}test_init_exec:
-; GFX1032: s_mov_b32 exec_lo, 0x12345
-; GFX1064: s_mov_b64 exec, 0x12345
-; GCN: v_add_f32_e32 v0,
-define amdgpu_ps float @test_init_exec(float %a, float %b) {
-main_body:
-  %s = fadd float %a, %b
-  call void @llvm.amdgcn.init.exec(i64 74565)
-  ret float %s
-}
-
-; GCN-LABEL: {{^}}test_init_exec_from_input:
-; GCN: s_bfe_u32 s0, s3, 0x70008
-; GFX1032: s_bfm_b32 exec_lo, s0, 0
-; GFX1032: s_cmp_eq_u32 s0, 32
-; GFX1032: s_cmov_b32 exec_lo, -1
-; GFX1064: s_bfm_b64 exec, s0, 0
-; GFX1064: s_cmp_eq_u32 s0, 64
-; GFX1064: s_cmov_b64 exec, -1
-; GCN: v_add_f32_e32 v0,
-define amdgpu_ps float @test_init_exec_from_input(i32 inreg, i32 inreg, i32 inreg, i32 inreg %count, float %a, float %b) {
-main_body:
-  %s = fadd float %a, %b
-  call void @llvm.amdgcn.init.exec.from.input(i32 %count, i32 8)
-  ret float %s
-}
-
 ; GCN-LABEL: {{^}}test_vgprblocks_w32_attr:
 ; Test that the wave size can be overridden in function attributes and that the block size is correct as a result
 ; GFX10DEFWAVE: ; VGPRBlocks: 1
@@ -1132,8 +1120,6 @@ declare i32 @llvm.amdgcn.icmp.i32.i32(i32, i32, i32)
 declare void @llvm.amdgcn.kill(i1)
 declare i1 @llvm.amdgcn.wqm.vote(i1)
 declare i1 @llvm.amdgcn.ps.live()
-declare void @llvm.amdgcn.init.exec(i64)
-declare void @llvm.amdgcn.init.exec.from.input(i32, i32)
 declare i64 @llvm.cttz.i64(i64, i1)
 declare i32 @llvm.cttz.i32(i32, i1)
 
diff --git a/llvm/test/CodeGen/AMDGPU/wqm.ll b/llvm/test/CodeGen/AMDGPU/wqm.ll
index b82766895..b799c2b59 100644
--- a/llvm/test/CodeGen/AMDGPU/wqm.ll
+++ b/llvm/test/CodeGen/AMDGPU/wqm.ll
@@ -117,6 +117,9 @@ main_body:
 ;CHECK: buffer_load_dword
 ;CHECK: buffer_load_dword
 ;CHECK: v_add_f32_e32
+; WQM was inserting an unecessary v_mov to self after the v_add. Make sure this
+; does not happen - the v_add should write the return reg directly.
+;CHECK-NOT: v_mov_b32_e32
 define amdgpu_ps float @test5(i32 inreg %idx0, i32 inreg %idx1) {
 main_body:
   %src0 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> undef, i32 %idx0, i32 0, i1 0, i1 0)
diff --git a/llvm/test/CodeGen/AMDGPU/write_register.ll b/llvm/test/CodeGen/AMDGPU/write_register.ll
index 0ca92f90c..eb9b10377 100644
--- a/llvm/test/CodeGen/AMDGPU/write_register.ll
+++ b/llvm/test/CodeGen/AMDGPU/write_register.ll
@@ -24,13 +24,25 @@ define amdgpu_kernel void @test_write_exec(i64 %val) #0 {
   ret void
 }
 
-; CHECK-LABEL: {{^}}test_write_flat_scratch:
+; CHECK-LABEL: {{^}}test_write_flat_scratch_0:
 ; CHECK: s_mov_b64 flat_scratch, 0
-; CHECK: s_mov_b64 flat_scratch, -1
-; CHECK: s_mov_b64 flat_scratch, s{{\[[0-9]+:[0-9]+\]}}
-define amdgpu_kernel void @test_write_flat_scratch(i64 %val) #0 {
+define amdgpu_kernel void @test_write_flat_scratch_0(i64 %val) #0 {
   call void @llvm.write_register.i64(metadata !2, i64 0)
+  call void @llvm.amdgcn.wave.barrier() #1
+  ret void
+}
+
+; CHECK-LABEL: {{^}}test_write_flat_scratch_neg1:
+; CHECK: s_mov_b64 flat_scratch, -1
+define amdgpu_kernel void @test_write_flat_scratch_neg1(i64 %val) #0 {
   call void @llvm.write_register.i64(metadata !2, i64 -1)
+  call void @llvm.amdgcn.wave.barrier() #1
+  ret void
+}
+
+; CHECK-LABEL: {{^}}test_write_flat_scratch_val:
+; CHECK: s_load_dwordx2 flat_scratch, s{{\[[0-9]+:[0-9]+\]}}
+define amdgpu_kernel void @test_write_flat_scratch_val(i64 %val) #0 {
   call void @llvm.write_register.i64(metadata !2, i64 %val)
   call void @llvm.amdgcn.wave.barrier() #1
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/wwm-reserved.ll b/llvm/test/CodeGen/AMDGPU/wwm-reserved.ll
index 358888316..1b1118e78 100644
--- a/llvm/test/CodeGen/AMDGPU/wwm-reserved.ll
+++ b/llvm/test/CodeGen/AMDGPU/wwm-reserved.ll
@@ -136,8 +136,8 @@ define amdgpu_kernel void @call_i64(<4 x i32> inreg %tmp14, i64 inreg %arg) {
 ; GFX9-O0: buffer_store_dword v1
 ; GFX9: s_swappc_b64
   %tmp134 = call i64 @called_i64(i64 %tmp107)
-; GFX9-O0: buffer_load_dword v3
-; GFX9-O0: buffer_load_dword v4
+; GFX9-O0: buffer_load_dword v6
+; GFX9-O0: buffer_load_dword v7
   %tmp136 = add i64 %tmp134, %tmp107
   %tmp137 = tail call i64 @llvm.amdgcn.wwm.i64(i64 %tmp136)
   %tmp138 = bitcast i64 %tmp137 to <2 x i32>
@@ -183,6 +183,6 @@ declare void @llvm.amdgcn.raw.buffer.store.f32(float, <4 x i32>, i32, i32, i32)
 declare void @llvm.amdgcn.raw.buffer.store.i32(i32, <4 x i32>, i32, i32, i32)
 declare void @llvm.amdgcn.raw.buffer.store.v2i32(<2 x i32>, <4 x i32>, i32, i32, i32)
 declare void @llvm.amdgcn.raw.buffer.store.v2f32(<2 x float>, <4 x i32>, i32, i32, i32)
-declare void @llvm.amdgcn.raw.buffer.store.v4f32(<4 x float>, <4 x i32>, i32, i32, i32) 
+declare void @llvm.amdgcn.raw.buffer.store.v4f32(<4 x float>, <4 x i32>, i32, i32, i32)
 declare <2 x i32> @llvm.amdgcn.s.buffer.load.v2i32(<4 x i32>, i32, i32)
 declare <4 x i32> @llvm.amdgcn.s.buffer.load.v4i32(<4 x i32>, i32, i32)
diff --git a/llvm/test/CodeGen/AMDGPU/zero_extend.ll b/llvm/test/CodeGen/AMDGPU/zero_extend.ll
index a49ff3330..d5d954eac 100644
--- a/llvm/test/CodeGen/AMDGPU/zero_extend.ll
+++ b/llvm/test/CodeGen/AMDGPU/zero_extend.ll
@@ -36,9 +36,9 @@ define amdgpu_kernel void @s_arg_zext_i1_to_i64(i64 addrspace(1)* %out, i1 zeroe
 }
 
 ; GCN-LABEL: {{^}}s_cmp_zext_i1_to_i64:
-; GCN: s_mov_b32 s{{[0-9]+}}, 0
-; GCN: v_cmp_eq_u32
-; GCN: v_cndmask_b32
+; GCN-DAG: s_mov_b32 s{{[0-9]+}}, 0
+; GCN-DAG: v_cmp_eq_u32
+; GCN:     v_cndmask_b32
 define amdgpu_kernel void @s_cmp_zext_i1_to_i64(i64 addrspace(1)* %out, i32 %a, i32 %b) #0 {
   %cmp = icmp eq i32 %a, %b
   %ext = zext i1 %cmp to i64
diff --git a/llvm/test/CodeGen/ARM/2007-01-19-InfiniteLoop.ll b/llvm/test/CodeGen/ARM/2007-01-19-InfiniteLoop.ll
index 991051cae..a35ded5b5 100644
--- a/llvm/test/CodeGen/ARM/2007-01-19-InfiniteLoop.ll
+++ b/llvm/test/CodeGen/ARM/2007-01-19-InfiniteLoop.ll
@@ -9,7 +9,7 @@
 @A = external global [4 x [4 x i32]]		; <[4 x [4 x i32]]*> [#uses=1]
 
 ; CHECK-LABEL: dct_luma_sp:
-define fastcc i32 @dct_luma_sp(i32 %block_x, i32 %block_y, i32* %coeff_cost) "no-frame-pointer-elim"="true" {
+define fastcc i32 @dct_luma_sp(i32 %block_x, i32 %block_y, i32* %coeff_cost) "frame-pointer"="all" {
 entry:
 ; Make sure to use base-updating stores for saving callee-saved registers.
 ; CHECK: push
diff --git a/llvm/test/CodeGen/ARM/2009-07-18-RewriterBug.ll b/llvm/test/CodeGen/ARM/2009-07-18-RewriterBug.ll
index 10b5ae4e2..9eae0d75e 100644
--- a/llvm/test/CodeGen/ARM/2009-07-18-RewriterBug.ll
+++ b/llvm/test/CodeGen/ARM/2009-07-18-RewriterBug.ll
@@ -1317,19 +1317,19 @@ bb15:
 }
 
 ; CHECK-LABEL: _build_delaunay:
-; CHECK: vcmpe
-; CHECK: vcmpe
-; CHECK: vcmpe
-; CHECK: vcmpe
-; CHECK: vcmpe
-; CHECK: vcmpe
-; CHECK: vcmpe
-; CHECK: vcmpe
-; CHECK: vcmpe
-; CHECK: vcmpe
-; CHECK: vcmpe
-; CHECK: vcmpe
-; CHECK: vcmpe
+; CHECK: vcmp
+; CHECK: vcmp
+; CHECK: vcmp
+; CHECK: vcmp
+; CHECK: vcmp
+; CHECK: vcmp
+; CHECK: vcmp
+; CHECK: vcmp
+; CHECK: vcmp
+; CHECK: vcmp
+; CHECK: vcmp
+; CHECK: vcmp
+; CHECK: vcmp
 
 declare i32 @puts(i8* nocapture) nounwind
 
diff --git a/llvm/test/CodeGen/ARM/2010-11-29-PrologueBug.ll b/llvm/test/CodeGen/ARM/2010-11-29-PrologueBug.ll
index 79315ab59..4b89ea325 100644
--- a/llvm/test/CodeGen/ARM/2010-11-29-PrologueBug.ll
+++ b/llvm/test/CodeGen/ARM/2010-11-29-PrologueBug.ll
@@ -2,7 +2,7 @@
 ; RUN: llc < %s -mtriple=thumbv7-apple-ios | FileCheck %s
 ; rdar://8690640
 
-define i32* @t(i32* %x) nounwind "no-frame-pointer-elim"="true" {
+define i32* @t(i32* %x) nounwind "frame-pointer"="all" {
 entry:
 ; CHECK-LABEL: t:
 ; CHECK: push
diff --git a/llvm/test/CodeGen/ARM/2010-12-07-PEIBug.ll b/llvm/test/CodeGen/ARM/2010-12-07-PEIBug.ll
index 340e3f8a8..e0802a64a 100644
--- a/llvm/test/CodeGen/ARM/2010-12-07-PEIBug.ll
+++ b/llvm/test/CodeGen/ARM/2010-12-07-PEIBug.ll
@@ -1,7 +1,7 @@
 ; RUN: llc < %s -mtriple=thumbv7-apple-ios -mcpu=cortex-a9 | FileCheck %s
 ; rdar://8728956
 
-define hidden void @foo() nounwind ssp "no-frame-pointer-elim"="true" {
+define hidden void @foo() nounwind ssp "frame-pointer"="all" {
 entry:
 ; CHECK-LABEL: foo:
 ; CHECK: mov r7, sp
diff --git a/llvm/test/CodeGen/ARM/2011-08-25-ldmia_ret.ll b/llvm/test/CodeGen/ARM/2011-08-25-ldmia_ret.ll
index f4d1b4de7..a568c04f1 100644
--- a/llvm/test/CodeGen/ARM/2011-08-25-ldmia_ret.ll
+++ b/llvm/test/CodeGen/ARM/2011-08-25-ldmia_ret.ll
@@ -14,7 +14,7 @@ declare i1 @getbool()
 declare void @foo(i32)
 declare i32 @bar(i32)
 
-define i32 @test(i32 %in1, i32 %in2) nounwind "no-frame-pointer-elim"="true" {
+define i32 @test(i32 %in1, i32 %in2) nounwind "frame-pointer"="all" {
 entry:
   %call = tail call zeroext i1 @getbool() nounwind
   br i1 %call, label %sw.bb18, label %sw.bb2
diff --git a/llvm/test/CodeGen/ARM/2014-05-14-DwarfEHCrash.ll b/llvm/test/CodeGen/ARM/2014-05-14-DwarfEHCrash.ll
index feae48646..b0c81c099 100644
--- a/llvm/test/CodeGen/ARM/2014-05-14-DwarfEHCrash.ll
+++ b/llvm/test/CodeGen/ARM/2014-05-14-DwarfEHCrash.ll
@@ -45,6 +45,6 @@ declare i8* @__cxa_begin_catch(i8*)
 
 declare void @__cxa_end_catch()
 
-attributes #0 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="true" }
+attributes #0 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="true" }
 attributes #1 = { nounwind readnone }
 attributes #2 = { nounwind }
diff --git a/llvm/test/CodeGen/ARM/2016-05-01-RegScavengerAssert.ll b/llvm/test/CodeGen/ARM/2016-05-01-RegScavengerAssert.ll
index 15e0dad18..8e11d5921 100644
--- a/llvm/test/CodeGen/ARM/2016-05-01-RegScavengerAssert.ll
+++ b/llvm/test/CodeGen/ARM/2016-05-01-RegScavengerAssert.ll
@@ -183,7 +183,7 @@ if.then343:                    ; preds = %entry
  ret void
 }
 
-attributes #0 = { sspstrong uwtable "no-frame-pointer-elim"="false" "target-cpu"="cortex-a7" }
+attributes #0 = { sspstrong uwtable "frame-pointer"="none" "target-cpu"="cortex-a7" }
 
 !1 = !{!2, !2, i64 0}
 !2 = !{!"float", !3, i64 0}
diff --git a/llvm/test/CodeGen/ARM/GlobalISel/arm-irtranslator.ll b/llvm/test/CodeGen/ARM/GlobalISel/arm-irtranslator.ll
index 2677a4cfd..2c286374a 100644
--- a/llvm/test/CodeGen/ARM/GlobalISel/arm-irtranslator.ll
+++ b/llvm/test/CodeGen/ARM/GlobalISel/arm-irtranslator.ll
@@ -438,9 +438,7 @@ define i32 @test_shufflevector_s32_v2s32(i32 %arg) {
 ; CHECK-LABEL: name: test_shufflevector_s32_v2s32
 ; CHECK: [[ARG:%[0-9]+]]:_(s32) = COPY $r0
 ; CHECK-DAG: [[UNDEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-; CHECK-DAG: [[C0:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-; CHECK-DAG: [[MASK:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[C0]](s32), [[C0]](s32)
-; CHECK: [[VEC:%[0-9]+]]:_(<2 x s32>) = G_SHUFFLE_VECTOR [[ARG]](s32), [[UNDEF]], [[MASK]](<2 x s32>)
+; CHECK: [[VEC:%[0-9]+]]:_(<2 x s32>) = G_SHUFFLE_VECTOR [[ARG]](s32), [[UNDEF]], shufflemask(0, 0)
 ; CHECK: G_EXTRACT_VECTOR_ELT [[VEC]](<2 x s32>)
   %vec = insertelement <1 x i32> undef, i32 %arg, i32 0
   %shuffle = shufflevector <1 x i32> %vec, <1 x i32> undef, <2 x i32> zeroinitializer
@@ -448,6 +446,17 @@ define i32 @test_shufflevector_s32_v2s32(i32 %arg) {
   ret i32 %res
 }
 
+define i32 @test_shufflevector_s32_s32_s32(i32 %arg) {
+; CHECK-LABEL: name: test_shufflevector_s32_s32_s32
+; CHECK: [[ARG:%[0-9]+]]:_(s32) = COPY $r0
+; CHECK-DAG: [[UNDEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+; CHECK: [[VEC:%[0-9]+]]:_(s32) = G_SHUFFLE_VECTOR [[ARG]](s32), [[UNDEF]], shufflemask(0)
+  %vec = insertelement <1 x i32> undef, i32 %arg, i32 0
+  %shuffle = shufflevector <1 x i32> %vec, <1 x i32> undef, <1 x i32> zeroinitializer
+  %res = extractelement <1 x i32> %shuffle, i32 0
+  ret i32 %res
+}
+
 define i32 @test_shufflevector_v2s32_v3s32(i32 %arg1, i32 %arg2) {
 ; CHECK-LABEL: name: test_shufflevector_v2s32_v3s32
 ; CHECK: [[ARG1:%[0-9]+]]:_(s32) = COPY $r0
@@ -455,10 +464,9 @@ define i32 @test_shufflevector_v2s32_v3s32(i32 %arg1, i32 %arg2) {
 ; CHECK-DAG: [[UNDEF:%[0-9]+]]:_(<2 x s32>) = G_IMPLICIT_DEF
 ; CHECK-DAG: [[C0:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
 ; CHECK-DAG: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-; CHECK-DAG: [[MASK:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[C1]](s32), [[C0]](s32), [[C1]](s32)
 ; CHECK-DAG: [[V1:%[0-9]+]]:_(<2 x s32>) = G_INSERT_VECTOR_ELT [[UNDEF]], [[ARG1]](s32), [[C0]](s32)
 ; CHECK-DAG: [[V2:%[0-9]+]]:_(<2 x s32>) = G_INSERT_VECTOR_ELT [[V1]], [[ARG2]](s32), [[C1]](s32)
-; CHECK: [[VEC:%[0-9]+]]:_(<3 x s32>) = G_SHUFFLE_VECTOR [[V2]](<2 x s32>), [[UNDEF]], [[MASK]](<3 x s32>)
+; CHECK: [[VEC:%[0-9]+]]:_(<3 x s32>) = G_SHUFFLE_VECTOR [[V2]](<2 x s32>), [[UNDEF]], shufflemask(1, 0, 1)
 ; CHECK: G_EXTRACT_VECTOR_ELT [[VEC]](<3 x s32>)
   %v1 = insertelement <2 x i32> undef, i32 %arg1, i32 0
   %v2 = insertelement <2 x i32> %v1, i32 %arg2, i32 1
@@ -475,10 +483,9 @@ define i32 @test_shufflevector_v2s32_v4s32(i32 %arg1, i32 %arg2) {
 ; CHECK-DAG: [[UNDEF:%[0-9]+]]:_(<2 x s32>) = G_IMPLICIT_DEF
 ; CHECK-DAG: [[C0:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
 ; CHECK-DAG: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-; CHECK-DAG: [[MASK:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[C0]](s32), [[C0]](s32), [[C0]](s32), [[C0]](s32)
 ; CHECK-DAG: [[V1:%[0-9]+]]:_(<2 x s32>) = G_INSERT_VECTOR_ELT [[UNDEF]], [[ARG1]](s32), [[C0]](s32)
 ; CHECK-DAG: [[V2:%[0-9]+]]:_(<2 x s32>) = G_INSERT_VECTOR_ELT [[V1]], [[ARG2]](s32), [[C1]](s32)
-; CHECK: [[VEC:%[0-9]+]]:_(<4 x s32>) = G_SHUFFLE_VECTOR [[V2]](<2 x s32>), [[UNDEF]], [[MASK]](<4 x s32>)
+; CHECK: [[VEC:%[0-9]+]]:_(<4 x s32>) = G_SHUFFLE_VECTOR [[V2]](<2 x s32>), [[UNDEF]], shufflemask(0, 0, 0, 0)
 ; CHECK: G_EXTRACT_VECTOR_ELT [[VEC]](<4 x s32>)
   %v1 = insertelement <2 x i32> undef, i32 %arg1, i32 0
   %v2 = insertelement <2 x i32> %v1, i32 %arg2, i32 1
@@ -498,12 +505,11 @@ define i32 @test_shufflevector_v4s32_v2s32(i32 %arg1, i32 %arg2, i32 %arg3, i32
 ; CHECK-DAG: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
 ; CHECK-DAG: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
 ; CHECK-DAG: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
-; CHECK-DAG: [[MASK:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[C1]](s32), [[C3]](s32)
 ; CHECK-DAG: [[V1:%[0-9]+]]:_(<4 x s32>) = G_INSERT_VECTOR_ELT [[UNDEF]], [[ARG1]](s32), [[C0]](s32)
 ; CHECK-DAG: [[V2:%[0-9]+]]:_(<4 x s32>) = G_INSERT_VECTOR_ELT [[V1]], [[ARG2]](s32), [[C1]](s32)
 ; CHECK-DAG: [[V3:%[0-9]+]]:_(<4 x s32>) = G_INSERT_VECTOR_ELT [[V2]], [[ARG3]](s32), [[C2]](s32)
 ; CHECK-DAG: [[V4:%[0-9]+]]:_(<4 x s32>) = G_INSERT_VECTOR_ELT [[V3]], [[ARG4]](s32), [[C3]](s32)
-; CHECK: [[VEC:%[0-9]+]]:_(<2 x s32>) = G_SHUFFLE_VECTOR [[V4]](<4 x s32>), [[UNDEF]], [[MASK]](<2 x s32>)
+; CHECK: [[VEC:%[0-9]+]]:_(<2 x s32>) = G_SHUFFLE_VECTOR [[V4]](<4 x s32>), [[UNDEF]], shufflemask(1, 3)
 ; CHECK: G_EXTRACT_VECTOR_ELT [[VEC]](<2 x s32>)
   %v1 = insertelement <4 x i32> undef, i32 %arg1, i32 0
   %v2 = insertelement <4 x i32> %v1, i32 %arg2, i32 1
@@ -551,11 +557,11 @@ define void @test_load_store_struct({i32, i32} *%addr) {
 ; CHECK: [[ADDR1:%[0-9]+]]:_(p0) = COPY $r0
 ; CHECK-DAG: [[VAL1:%[0-9]+]]:_(s32) = G_LOAD [[ADDR1]](p0) :: (load 4 from %ir.addr)
 ; CHECK-DAG: [[OFFSET:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-; CHECK-DAG: [[ADDR2:%[0-9]+]]:_(p0) = G_GEP [[ADDR1]], [[OFFSET]](s32)
+; CHECK-DAG: [[ADDR2:%[0-9]+]]:_(p0) = G_PTR_ADD [[ADDR1]], [[OFFSET]](s32)
 ; CHECK-DAG: [[VAL2:%[0-9]+]]:_(s32) = G_LOAD [[ADDR2]](p0) :: (load 4 from %ir.addr + 4)
 ; CHECK-DAG: G_STORE [[VAL1]](s32), [[ADDR1]](p0) :: (store 4 into %ir.addr)
-; CHECK-DAG: [[ADDR2:%[0-9]+]]:_(p0) = G_GEP [[ADDR1]], [[OFFSET]](s32)
-; CHECK-DAG: G_STORE [[VAL2]](s32), [[ADDR2]](p0) :: (store 4 into %ir.addr + 4)
+; CHECK-DAG: [[ADDR3:%[0-9]+]]:_(p0) = COPY [[ADDR2]]
+; CHECK-DAG: G_STORE [[VAL2]](s32), [[ADDR3]](p0) :: (store 4 into %ir.addr + 4)
   %val = load {i32, i32}, {i32, i32} *%addr, align 4
   store {i32, i32} %val, {i32, i32} *%addr, align 4
   ret void
diff --git a/llvm/test/CodeGen/ARM/GlobalISel/arm-param-lowering.ll b/llvm/test/CodeGen/ARM/GlobalISel/arm-param-lowering.ll
index e4f6c1ebe..e86496760 100644
--- a/llvm/test/CodeGen/ARM/GlobalISel/arm-param-lowering.ll
+++ b/llvm/test/CodeGen/ARM/GlobalISel/arm-param-lowering.ll
@@ -36,11 +36,11 @@ define arm_aapcscc i32* @test_call_simple_stack_params(i32 *%a, i32 %b) {
 ; CHECK-DAG: $r3 = COPY [[AVREG]]
 ; CHECK: [[SP1:%[0-9]+]]:_(p0) = COPY $sp
 ; CHECK: [[OFF1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-; CHECK: [[FI1:%[0-9]+]]:_(p0) = G_GEP [[SP1]], [[OFF1]](s32)
+; CHECK: [[FI1:%[0-9]+]]:_(p0) = G_PTR_ADD [[SP1]], [[OFF1]](s32)
 ; CHECK: G_STORE [[BVREG]](s32), [[FI1]](p0){{.*}}store 4
 ; CHECK: [[SP2:%[0-9]+]]:_(p0) = COPY $sp
 ; CHECK: [[OFF2:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-; CHECK: [[FI2:%[0-9]+]]:_(p0) = G_GEP [[SP2]], [[OFF2]](s32)
+; CHECK: [[FI2:%[0-9]+]]:_(p0) = G_PTR_ADD [[SP2]], [[OFF2]](s32)
 ; CHECK: G_STORE [[AVREG]](p0), [[FI2]](p0){{.*}}store 4
 ; ARM: BL @simple_stack_params_target, csr_aapcs, implicit-def $lr, implicit $sp, implicit $r0, implicit $r1, implicit $r2, implicit $r3, implicit-def $r0
 ; THUMB: tBL 14, $noreg, @simple_stack_params_target, csr_aapcs, implicit-def $lr, implicit $sp, implicit $r0, implicit $r1, implicit $r2, implicit $r3, implicit-def $r0
@@ -75,27 +75,27 @@ define arm_aapcscc signext i16 @test_call_ext_params(i8 %a, i16 %b, i1 %c) {
 ; CHECK: $r3 = COPY [[ZEXTB]]
 ; CHECK: [[SP1:%[0-9]+]]:_(p0) = COPY $sp
 ; CHECK: [[OFF1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-; CHECK: [[FI1:%[0-9]+]]:_(p0) = G_GEP [[SP1]], [[OFF1]](s32)
+; CHECK: [[FI1:%[0-9]+]]:_(p0) = G_PTR_ADD [[SP1]], [[OFF1]](s32)
 ; CHECK: [[SEXTA2:%[0-9]+]]:_(s32) = G_SEXT [[AVREG]]
 ; CHECK: G_STORE [[SEXTA2]](s32), [[FI1]](p0){{.*}}store 4
 ; CHECK: [[SP2:%[0-9]+]]:_(p0) = COPY $sp
 ; CHECK: [[OFF2:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-; CHECK: [[FI2:%[0-9]+]]:_(p0) = G_GEP [[SP2]], [[OFF2]](s32)
+; CHECK: [[FI2:%[0-9]+]]:_(p0) = G_PTR_ADD [[SP2]], [[OFF2]](s32)
 ; CHECK: [[ZEXTA2:%[0-9]+]]:_(s32) = G_ZEXT [[AVREG]]
 ; CHECK: G_STORE [[ZEXTA2]](s32), [[FI2]](p0){{.*}}store 4
 ; CHECK: [[SP3:%[0-9]+]]:_(p0) = COPY $sp
 ; CHECK: [[OFF3:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-; CHECK: [[FI3:%[0-9]+]]:_(p0) = G_GEP [[SP3]], [[OFF3]](s32)
+; CHECK: [[FI3:%[0-9]+]]:_(p0) = G_PTR_ADD [[SP3]], [[OFF3]](s32)
 ; CHECK: [[SEXTB2:%[0-9]+]]:_(s32) = G_SEXT [[BVREG]]
 ; CHECK: G_STORE [[SEXTB2]](s32), [[FI3]](p0){{.*}}store 4
 ; CHECK: [[SP4:%[0-9]+]]:_(p0) = COPY $sp
 ; CHECK: [[OFF4:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
-; CHECK: [[FI4:%[0-9]+]]:_(p0) = G_GEP [[SP4]], [[OFF4]](s32)
+; CHECK: [[FI4:%[0-9]+]]:_(p0) = G_PTR_ADD [[SP4]], [[OFF4]](s32)
 ; CHECK: [[ZEXTB2:%[0-9]+]]:_(s32) = G_ZEXT [[BVREG]]
 ; CHECK: G_STORE [[ZEXTB2]](s32), [[FI4]](p0){{.*}}store 4
 ; CHECK: [[SP5:%[0-9]+]]:_(p0) = COPY $sp
 ; CHECK: [[OFF5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-; CHECK: [[FI5:%[0-9]+]]:_(p0) = G_GEP [[SP5]], [[OFF5]](s32)
+; CHECK: [[FI5:%[0-9]+]]:_(p0) = G_PTR_ADD [[SP5]], [[OFF5]](s32)
 ; CHECK: [[ZEXTC:%[0-9]+]]:_(s32) = G_ZEXT [[CVREG]]
 ; CHECK: G_STORE [[ZEXTC]](s32), [[FI5]](p0){{.*}}store 4
 ; ARM: BL @ext_target, csr_aapcs, implicit-def $lr, implicit $sp, implicit $r0, implicit $r1, implicit $r2, implicit $r3, implicit-def $r0
@@ -151,11 +151,11 @@ define arm_aapcscc double @test_call_aapcs_fp_params(double %a, float %b) {
 ; BIG-DAG: $r3 = COPY [[A1]]
 ; CHECK: [[SP1:%[0-9]+]]:_(p0) = COPY $sp
 ; CHECK: [[OFF1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-; CHECK: [[FI1:%[0-9]+]]:_(p0) = G_GEP [[SP1]], [[OFF1]](s32)
+; CHECK: [[FI1:%[0-9]+]]:_(p0) = G_PTR_ADD [[SP1]], [[OFF1]](s32)
 ; CHECK: G_STORE [[BVREG]](s32), [[FI1]](p0){{.*}}store 4
 ; CHECK: [[SP2:%[0-9]+]]:_(p0) = COPY $sp
 ; CHECK: [[OFF2:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-; CHECK: [[FI2:%[0-9]+]]:_(p0) = G_GEP [[SP2]], [[OFF2]](s32)
+; CHECK: [[FI2:%[0-9]+]]:_(p0) = G_PTR_ADD [[SP2]], [[OFF2]](s32)
 ; CHECK: G_STORE [[AVREG]](s64), [[FI2]](p0){{.*}}store 8
 ; ARM: BL @aapcscc_fp_target, csr_aapcs, implicit-def $lr, implicit $sp, implicit $r0, implicit $r2, implicit $r3, implicit-def $r0, implicit-def $r1
 ; THUMB: tBL 14, $noreg, @aapcscc_fp_target, csr_aapcs, implicit-def $lr, implicit $sp, implicit $r0, implicit $r2, implicit $r3, implicit-def $r0, implicit-def $r1
@@ -273,13 +273,13 @@ define arm_aapcscc void @test_large_int_arrays([20 x i32] %arr) {
 ; CHECK: $r3 = COPY [[R3]]
 ; CHECK: [[SP:%[0-9]+]]:_(p0) = COPY $sp
 ; CHECK: [[OFF_FIRST_ELEMENT:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-; CHECK: [[FIRST_STACK_ARG_ADDR:%[0-9]+]]:_(p0) = G_GEP [[SP]], [[OFF_FIRST_ELEMENT]](s32)
+; CHECK: [[FIRST_STACK_ARG_ADDR:%[0-9]+]]:_(p0) = G_PTR_ADD [[SP]], [[OFF_FIRST_ELEMENT]](s32)
 ; CHECK: G_STORE [[FIRST_STACK_ELEMENT]](s32), [[FIRST_STACK_ARG_ADDR]]{{.*}}store 4
 ; Match the second-to-last offset, so we can get the correct SP for the last element
 ; CHECK: G_CONSTANT i32 56
 ; CHECK: [[SP:%[0-9]+]]:_(p0) = COPY $sp
 ; CHECK: [[OFF_LAST_ELEMENT:%[0-9]+]]:_(s32) = G_CONSTANT i32 60
-; CHECK: [[LAST_STACK_ARG_ADDR:%[0-9]+]]:_(p0) = G_GEP [[SP]], [[OFF_LAST_ELEMENT]](s32)
+; CHECK: [[LAST_STACK_ARG_ADDR:%[0-9]+]]:_(p0) = G_PTR_ADD [[SP]], [[OFF_LAST_ELEMENT]](s32)
 ; CHECK: G_STORE [[LAST_STACK_ELEMENT]](s32), [[LAST_STACK_ARG_ADDR]]{{.*}}store 4
 ; ARM: BL @large_int_arrays_target, csr_aapcs, implicit-def $lr, implicit $sp, implicit $r0, implicit $r1, implicit $r2, implicit $r3
 ; THUMB: tBL 14, $noreg, @large_int_arrays_target, csr_aapcs, implicit-def $lr, implicit $sp, implicit $r0, implicit $r1, implicit $r2, implicit $r3
@@ -321,7 +321,7 @@ define arm_aapcscc [2 x float] @test_fp_arrays_aapcs([3 x double] %arr) {
 ; BIG: $r3 = COPY [[ARR1_0]](s32)
 ; CHECK: [[SP:%[0-9]+]]:_(p0) = COPY $sp
 ; CHECK: [[ARR2_OFFSET:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-; CHECK: [[ARR2_ADDR:%[0-9]+]]:_(p0) = G_GEP [[SP]], [[ARR2_OFFSET]](s32)
+; CHECK: [[ARR2_ADDR:%[0-9]+]]:_(p0) = G_PTR_ADD [[SP]], [[ARR2_OFFSET]](s32)
 ; CHECK: G_STORE [[ARR2]](s64), [[ARR2_ADDR]](p0){{.*}}store 8
 ; ARM: BL @fp_arrays_aapcs_target, csr_aapcs, implicit-def $lr, implicit $sp, implicit $r0, implicit $r1, implicit $r2, implicit $r3, implicit-def $r0, implicit-def $r1
 ; THUMB: tBL 14, $noreg, @fp_arrays_aapcs_target, csr_aapcs, implicit-def $lr, implicit $sp, implicit $r0, implicit $r1, implicit $r2, implicit $r3, implicit-def $r0, implicit-def $r1
@@ -370,19 +370,19 @@ define arm_aapcs_vfpcc [4 x float] @test_fp_arrays_aapcs_vfp([3 x double] %x, [3
 ; CHECK: $s8 = COPY [[Y2]](s32)
 ; CHECK: [[SP:%[0-9]+]]:_(p0) = COPY $sp
 ; CHECK: [[Z0_OFFSET:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-; CHECK: [[Z0_ADDR:%[0-9]+]]:_(p0) = G_GEP [[SP]], [[Z0_OFFSET]](s32)
+; CHECK: [[Z0_ADDR:%[0-9]+]]:_(p0) = G_PTR_ADD [[SP]], [[Z0_OFFSET]](s32)
 ; CHECK: G_STORE [[Z0]](s64), [[Z0_ADDR]](p0){{.*}}store 8
 ; CHECK: [[SP:%[0-9]+]]:_(p0) = COPY $sp
 ; CHECK: [[Z1_OFFSET:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-; CHECK: [[Z1_ADDR:%[0-9]+]]:_(p0) = G_GEP [[SP]], [[Z1_OFFSET]](s32)
+; CHECK: [[Z1_ADDR:%[0-9]+]]:_(p0) = G_PTR_ADD [[SP]], [[Z1_OFFSET]](s32)
 ; CHECK: G_STORE [[Z1]](s64), [[Z1_ADDR]](p0){{.*}}store 8
 ; CHECK: [[SP:%[0-9]+]]:_(p0) = COPY $sp
 ; CHECK: [[Z2_OFFSET:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-; CHECK: [[Z2_ADDR:%[0-9]+]]:_(p0) = G_GEP [[SP]], [[Z2_OFFSET]](s32)
+; CHECK: [[Z2_ADDR:%[0-9]+]]:_(p0) = G_PTR_ADD [[SP]], [[Z2_OFFSET]](s32)
 ; CHECK: G_STORE [[Z2]](s64), [[Z2_ADDR]](p0){{.*}}store 8
 ; CHECK: [[SP:%[0-9]+]]:_(p0) = COPY $sp
 ; CHECK: [[Z3_OFFSET:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
-; CHECK: [[Z3_ADDR:%[0-9]+]]:_(p0) = G_GEP [[SP]], [[Z3_OFFSET]](s32)
+; CHECK: [[Z3_ADDR:%[0-9]+]]:_(p0) = G_PTR_ADD [[SP]], [[Z3_OFFSET]](s32)
 ; CHECK: G_STORE [[Z3]](s64), [[Z3_ADDR]](p0){{.*}}store 8
 ; ARM: BL @fp_arrays_aapcs_vfp_target, csr_aapcs, implicit-def $lr, implicit $sp, implicit $d0, implicit $d1, implicit $d2, implicit $s6, implicit $s7, implicit $s8, implicit-def $s0, implicit-def $s1, implicit-def $s2, implicit-def $s3
 ; THUMB: tBL 14, $noreg, @fp_arrays_aapcs_vfp_target, csr_aapcs, implicit-def $lr, implicit $sp, implicit $d0, implicit $d1, implicit $d2, implicit $s6, implicit $s7, implicit $s8, implicit-def $s0, implicit-def $s1, implicit-def $s2, implicit-def $s3
@@ -427,13 +427,13 @@ define arm_aapcscc [2 x i32*] @test_tough_arrays([6 x [4 x i32]] %arr) {
 ; CHECK: $r3 = COPY [[R3]]
 ; CHECK: [[SP:%[0-9]+]]:_(p0) = COPY $sp
 ; CHECK: [[OFF_FIRST_ELEMENT:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-; CHECK: [[FIRST_STACK_ARG_ADDR:%[0-9]+]]:_(p0) = G_GEP [[SP]], [[OFF_FIRST_ELEMENT]](s32)
+; CHECK: [[FIRST_STACK_ARG_ADDR:%[0-9]+]]:_(p0) = G_PTR_ADD [[SP]], [[OFF_FIRST_ELEMENT]](s32)
 ; CHECK: G_STORE [[FIRST_STACK_ELEMENT]](s32), [[FIRST_STACK_ARG_ADDR]]{{.*}}store 4
 ; Match the second-to-last offset, so we can get the correct SP for the last element
 ; CHECK: G_CONSTANT i32 72
 ; CHECK: [[SP:%[0-9]+]]:_(p0) = COPY $sp
 ; CHECK: [[OFF_LAST_ELEMENT:%[0-9]+]]:_(s32) = G_CONSTANT i32 76
-; CHECK: [[LAST_STACK_ARG_ADDR:%[0-9]+]]:_(p0) = G_GEP [[SP]], [[OFF_LAST_ELEMENT]](s32)
+; CHECK: [[LAST_STACK_ARG_ADDR:%[0-9]+]]:_(p0) = G_PTR_ADD [[SP]], [[OFF_LAST_ELEMENT]](s32)
 ; CHECK: G_STORE [[LAST_STACK_ELEMENT]](s32), [[LAST_STACK_ARG_ADDR]]{{.*}}store 4
 ; ARM: BL @tough_arrays_target, csr_aapcs, implicit-def $lr, implicit $sp, implicit $r0, implicit $r1, implicit $r2, implicit $r3, implicit-def $r0, implicit-def $r1
 ; THUMB: tBL 14, $noreg, @tough_arrays_target, csr_aapcs, implicit-def $lr, implicit $sp, implicit $r0, implicit $r1, implicit $r2, implicit $r3, implicit-def $r0, implicit-def $r1
diff --git a/llvm/test/CodeGen/ARM/GlobalISel/irtranslator-varargs-lowering.ll b/llvm/test/CodeGen/ARM/GlobalISel/irtranslator-varargs-lowering.ll
index 351594d1f..e303b7b1f 100644
--- a/llvm/test/CodeGen/ARM/GlobalISel/irtranslator-varargs-lowering.ll
+++ b/llvm/test/CodeGen/ARM/GlobalISel/irtranslator-varargs-lowering.ll
@@ -14,11 +14,11 @@ define arm_aapcscc i32 @test_call_to_varargs_with_ints(i32 *%a, i32 %b) {
 ; CHECK-DAG: $r3 = COPY [[AVREG]]
 ; CHECK: [[SP1:%[0-9]+]]:_(p0) = COPY $sp
 ; CHECK: [[OFF1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-; CHECK: [[FI1:%[0-9]+]]:_(p0) = G_GEP [[SP1]], [[OFF1]](s32)
+; CHECK: [[FI1:%[0-9]+]]:_(p0) = G_PTR_ADD [[SP1]], [[OFF1]](s32)
 ; CHECK: G_STORE [[BVREG]](s32), [[FI1]](p0){{.*}}store 4
 ; CHECK: [[SP2:%[0-9]+]]:_(p0) = COPY $sp
 ; CHECK: [[OFF2:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-; CHECK: [[FI2:%[0-9]+]]:_(p0) = G_GEP [[SP2]], [[OFF2]](s32)
+; CHECK: [[FI2:%[0-9]+]]:_(p0) = G_PTR_ADD [[SP2]], [[OFF2]](s32)
 ; CHECK: G_STORE [[AVREG]](p0), [[FI2]](p0){{.*}}store 4
 ; ARM: BL @int_varargs_target, csr_aapcs, implicit-def $lr, implicit $sp, implicit $r0, implicit $r1, implicit $r2, implicit $r3, implicit-def $r0
 ; THUMB: tBL 14, $noreg, @int_varargs_target, csr_aapcs, implicit-def $lr, implicit $sp, implicit $r0, implicit $r1, implicit $r2, implicit $r3, implicit-def $r0
@@ -45,7 +45,7 @@ define arm_aapcs_vfpcc float @test_call_to_varargs_with_floats(float %a, double
 ; CHECK-DAG: $r3 = COPY [[B2]]
 ; CHECK: [[SP1:%[0-9]+]]:_(p0) = COPY $sp
 ; CHECK: [[OFF1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-; CHECK: [[FI1:%[0-9]+]]:_(p0) = G_GEP [[SP1]], [[OFF1]](s32)
+; CHECK: [[FI1:%[0-9]+]]:_(p0) = G_PTR_ADD [[SP1]], [[OFF1]](s32)
 ; CHECK: G_STORE [[BVREG]](s64), [[FI1]](p0){{.*}}store 8
 ; ARM: BL @float_varargs_target, csr_aapcs, implicit-def $lr, implicit $sp, implicit $r0, implicit $r2, implicit $r3, implicit-def $r0
 ; THUMB: tBL 14, $noreg, @float_varargs_target, csr_aapcs, implicit-def $lr, implicit $sp, implicit $r0, implicit $r2, implicit $r3, implicit-def $r0
@@ -71,7 +71,7 @@ define arm_aapcs_vfpcc float @test_indirect_call_to_varargs(float (float, double
 ; CHECK-DAG: $r3 = COPY [[B2]]
 ; CHECK: [[SP1:%[0-9]+]]:_(p0) = COPY $sp
 ; CHECK: [[OFF1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-; CHECK: [[FI1:%[0-9]+]]:_(p0) = G_GEP [[SP1]], [[OFF1]](s32)
+; CHECK: [[FI1:%[0-9]+]]:_(p0) = G_PTR_ADD [[SP1]], [[OFF1]](s32)
 ; CHECK: G_STORE [[BVREG]](s64), [[FI1]](p0){{.*}}store 8
 ; ARM: BLX [[FPTRVREG]](p0), csr_aapcs, implicit-def $lr, implicit $sp, implicit $r0, implicit $r2, implicit $r3, implicit-def $r0
 ; THUMB: tBLXr 14, $noreg, [[FPTRVREG]](p0), csr_aapcs, implicit-def $lr, implicit $sp, implicit $r0, implicit $r2, implicit $r3, implicit-def $r0
diff --git a/llvm/test/CodeGen/ARM/O3-pipeline.ll b/llvm/test/CodeGen/ARM/O3-pipeline.ll
index ec96f055a..8a9f84d3f 100644
--- a/llvm/test/CodeGen/ARM/O3-pipeline.ll
+++ b/llvm/test/CodeGen/ARM/O3-pipeline.ll
@@ -7,6 +7,7 @@
 ; CHECK-NEXT:    FunctionPass Manager
 ; CHECK-NEXT:      Expand Atomic instructions
 ; CHECK-NEXT:      Simplify the CFG
+; CHECK-NEXT:      MVE gather/scatter lowering
 ; CHECK-NEXT:      Dominator Tree Construction
 ; CHECK-NEXT:      Basic Alias Analysis (stateless AA impl)
 ; CHECK-NEXT:      Module Verifier
@@ -19,9 +20,13 @@
 ; CHECK-NEXT:      Basic Alias Analysis (stateless AA impl)
 ; CHECK-NEXT:      Function Alias Analysis Results
 ; CHECK-NEXT:      Merge contiguous icmps into a memcmp
+; CHECK-NEXT:      Natural Loop Information
+; CHECK-NEXT:      Lazy Branch Probability Analysis
+; CHECK-NEXT:      Lazy Block Frequency Analysis
 ; CHECK-NEXT:      Expand memcmp() to load/stores
 ; CHECK-NEXT:      Lower Garbage Collection Instructions
 ; CHECK-NEXT:      Shadow Stack GC Lowering
+; CHECK-NEXT:      Lower constant intrinsics
 ; CHECK-NEXT:      Remove unreachable blocks from the CFG
 ; CHECK-NEXT:      Dominator Tree Construction
 ; CHECK-NEXT:      Natural Loop Information
@@ -37,10 +42,9 @@
 ; CHECK-NEXT:      Scalar Evolution Analysis
 ; CHECK-NEXT:      Basic Alias Analysis (stateless AA impl)
 ; CHECK-NEXT:      Function Alias Analysis Results
-; CHECK-NEXT:      Loop Pass Manager
-; CHECK-NEXT:        Transform loops to use DSP intrinsics
+; CHECK-NEXT:      Transform functions to use DSP intrinsics
 ; CHECK-NEXT:      Interleaved Access Pass
-; CHECK-NEXT:      ARM IR optimizations
+; CHECK-NEXT:      Type Promotion
 ; CHECK-NEXT:      Dominator Tree Construction
 ; CHECK-NEXT:      Natural Loop Information
 ; CHECK-NEXT:      CodeGen Prepare
@@ -52,7 +56,13 @@
 ; CHECK-NEXT:      Dominator Tree Construction
 ; CHECK-NEXT:      Natural Loop Information
 ; CHECK-NEXT:      Scalar Evolution Analysis
+; CHECK-NEXT:      Lazy Branch Probability Analysis
+; CHECK-NEXT:      Lazy Block Frequency Analysis
+; CHECK-NEXT:      Optimization Remark Emitter
 ; CHECK-NEXT:      Hardware Loop Insertion
+; CHECK-NEXT:      Scalar Evolution Analysis
+; CHECK-NEXT:      Loop Pass Manager
+; CHECK-NEXT:        Transform predicated vector loops to use MVE tail predication
 ; CHECK-NEXT:      Safe Stack instrumentation pass
 ; CHECK-NEXT:      Insert stack protectors
 ; CHECK-NEXT:      Module Verifier
@@ -61,8 +71,11 @@
 ; CHECK-NEXT:      Function Alias Analysis Results
 ; CHECK-NEXT:      Natural Loop Information
 ; CHECK-NEXT:      Branch Probability Analysis
+; CHECK-NEXT:      Lazy Branch Probability Analysis
+; CHECK-NEXT:      Lazy Block Frequency Analysis
 ; CHECK-NEXT:      ARM Instruction Selection
 ; CHECK-NEXT:      Finalize ISel and expand pseudo-instructions
+; CHECK-NEXT:      Lazy Machine Block Frequency Analysis
 ; CHECK-NEXT:      Early Tail Duplication
 ; CHECK-NEXT:      Optimize machine instruction PHIs
 ; CHECK-NEXT:      Slot index numbering
@@ -72,6 +85,7 @@
 ; CHECK-NEXT:      MachineDominator Tree Construction
 ; CHECK-NEXT:      Machine Natural Loop Construction
 ; CHECK-NEXT:      Early Machine Loop Invariant Code Motion
+; CHECK-NEXT:      MachineDominator Tree Construction
 ; CHECK-NEXT:      Machine Block Frequency Analysis
 ; CHECK-NEXT:      Machine Common Subexpression Elimination
 ; CHECK-NEXT:      MachinePostDominator Tree Construction
@@ -110,12 +124,14 @@
 ; CHECK-NEXT:      Machine Loop Invariant Code Motion
 ; CHECK-NEXT:      PostRA Machine Sink
 ; CHECK-NEXT:      Machine Block Frequency Analysis
+; CHECK-NEXT:      MachineDominator Tree Construction
 ; CHECK-NEXT:      MachinePostDominator Tree Construction
 ; CHECK-NEXT:      Lazy Machine Block Frequency Analysis
 ; CHECK-NEXT:      Machine Optimization Remark Emitter
 ; CHECK-NEXT:      Shrink Wrapping analysis
 ; CHECK-NEXT:      Prologue/Epilogue Insertion & Frame Finalization
 ; CHECK-NEXT:      Control Flow Optimizer
+; CHECK-NEXT:      Lazy Machine Block Frequency Analysis
 ; CHECK-NEXT:      Tail Duplication
 ; CHECK-NEXT:      Machine Copy Propagation Pass
 ; CHECK-NEXT:      Post-RA pseudo instruction expansion pass
@@ -129,28 +145,32 @@
 ; CHECK-NEXT:      Machine Natural Loop Construction
 ; CHECK-NEXT:      Machine Block Frequency Analysis
 ; CHECK-NEXT:      If Converter
+; CHECK-NEXT:      ReachingDefAnalysis
 ; CHECK-NEXT:      MVE VPT block insertion pass
 ; CHECK-NEXT:      Thumb IT blocks insertion pass
 ; CHECK-NEXT:      MachineDominator Tree Construction
 ; CHECK-NEXT:      Machine Natural Loop Construction
+; CHECK-NEXT:      PostRA Machine Instruction Scheduler
 ; CHECK-NEXT:      Post RA top-down list latency scheduler
 ; CHECK-NEXT:      Analyze Machine Code For Garbage Collection
 ; CHECK-NEXT:      Machine Block Frequency Analysis
 ; CHECK-NEXT:      MachinePostDominator Tree Construction
 ; CHECK-NEXT:      Branch Probability Basic Block Placement
+; CHECK-NEXT:      Insert fentry calls
+; CHECK-NEXT:      Insert XRay ops
+; CHECK-NEXT:      Implement the 'patchable-function' attribute
 ; CHECK-NEXT:      Thumb2 instruction size reduce pass
 ; CHECK-NEXT:      Unpack machine instruction bundles
 ; CHECK-NEXT:      optimise barriers pass
+; CHECK-NEXT:      MachineDominator Tree Construction
 ; CHECK-NEXT:      ARM constant island placement and branch shortening pass
 ; CHECK-NEXT:      MachineDominator Tree Construction
 ; CHECK-NEXT:      Machine Natural Loop Construction
+; CHECK-NEXT:      ReachingDefAnalysis
 ; CHECK-NEXT:      ARM Low Overhead Loops pass
 ; CHECK-NEXT:      Contiguously Lay Out Funclets
 ; CHECK-NEXT:      StackMap Liveness Analysis
 ; CHECK-NEXT:      Live DEBUG_VALUE analysis
-; CHECK-NEXT:      Insert fentry calls
-; CHECK-NEXT:      Insert XRay ops
-; CHECK-NEXT:      Implement the 'patchable-function' attribute
 ; CHECK-NEXT:      Lazy Machine Block Frequency Analysis
 ; CHECK-NEXT:      Machine Optimization Remark Emitter
 ; CHECK-NEXT:      ARM Assembly Printer
diff --git a/llvm/test/CodeGen/ARM/ParallelDSP/blocks.ll b/llvm/test/CodeGen/ARM/ParallelDSP/blocks.ll
new file mode 100644
index 000000000..adcb81eca
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/ParallelDSP/blocks.ll
@@ -0,0 +1,295 @@
+; RUN: opt -arm-parallel-dsp -dce -mtriple=armv7-a -S %s -o - | FileCheck %s
+
+; CHECK-LABEL: single_block
+; CHECK: [[CAST_A:%[^ ]+]] = bitcast i16* %a to i32*
+; CHECK: [[A:%[^ ]+]] = load i32, i32* [[CAST_A]]
+; CHECK: [[CAST_B:%[^ ]+]] = bitcast i16* %b to i32*
+; CHECK: [[B:%[^ ]+]] = load i32, i32* [[CAST_B]]
+; CHECK: call i32 @llvm.arm.smlad(i32 [[A]], i32 [[B]], i32 %acc)
+define i32 @single_block(i16* %a, i16* %b, i32 %acc) {
+entry:
+  %ld.a.0 = load i16, i16* %a
+  %sext.a.0 = sext i16 %ld.a.0 to i32
+  %ld.b.0 = load i16, i16* %b
+  %sext.b.0 = sext i16 %ld.b.0 to i32
+  %mul.0 = mul i32 %sext.a.0, %sext.b.0
+  %addr.a.1 = getelementptr i16, i16* %a, i32 1
+  %addr.b.1 = getelementptr i16, i16* %b, i32 1
+  %ld.a.1 = load i16, i16* %addr.a.1
+  %sext.a.1 = sext i16 %ld.a.1 to i32
+  %ld.b.1 = load i16, i16* %addr.b.1
+  %sext.b.1 = sext i16 %ld.b.1 to i32
+  %mul.1 = mul i32 %sext.a.1, %sext.b.1
+  %add = add i32 %mul.0, %mul.1
+  %res = add i32 %add, %acc
+  ret i32 %res
+}
+
+; CHECK-LABEL: single_block_64
+; CHECK: [[CAST_A:%[^ ]+]] = bitcast i16* %a to i32*
+; CHECK: [[A:%[^ ]+]] = load i32, i32* [[CAST_A]]
+; CHECK: [[CAST_B:%[^ ]+]] = bitcast i16* %b to i32*
+; CHECK: [[B:%[^ ]+]] = load i32, i32* [[CAST_B]]
+; CHECK: call i64 @llvm.arm.smlald(i32 [[A]], i32 [[B]], i64 %acc)
+define i64 @single_block_64(i16* %a, i16* %b, i64 %acc) {
+entry:
+  %ld.a.0 = load i16, i16* %a
+  %sext.a.0 = sext i16 %ld.a.0 to i32
+  %ld.b.0 = load i16, i16* %b
+  %sext.b.0 = sext i16 %ld.b.0 to i32
+  %mul.0 = mul i32 %sext.a.0, %sext.b.0
+  %addr.a.1 = getelementptr i16, i16* %a, i32 1
+  %addr.b.1 = getelementptr i16, i16* %b, i32 1
+  %ld.a.1 = load i16, i16* %addr.a.1
+  %sext.a.1 = sext i16 %ld.a.1 to i32
+  %ld.b.1 = load i16, i16* %addr.b.1
+  %sext.b.1 = sext i16 %ld.b.1 to i32
+  %mul.1 = mul i32 %sext.a.1, %sext.b.1
+  %sext.mul.0 = sext i32 %mul.0 to i64
+  %sext.mul.1 = sext i32 %mul.1 to i64
+  %add = add i64 %sext.mul.0, %sext.mul.1
+  %res = add i64 %add, %acc
+  ret i64 %res
+}
+
+; CHECK-LABEL: multi_block
+; CHECK: [[CAST_A:%[^ ]+]] = bitcast i16* %a to i32*
+; CHECK: [[A:%[^ ]+]] = load i32, i32* [[CAST_A]]
+; CHECK: [[CAST_B:%[^ ]+]] = bitcast i16* %b to i32*
+; CHECK: [[B:%[^ ]+]] = load i32, i32* [[CAST_B]]
+; CHECK:  call i32 @llvm.arm.smlad(i32 [[A]], i32 [[B]], i32 0)
+define i32 @multi_block(i16* %a, i16* %b, i32 %acc) {
+entry:
+  %ld.a.0 = load i16, i16* %a
+  %sext.a.0 = sext i16 %ld.a.0 to i32
+  %ld.b.0 = load i16, i16* %b
+  %sext.b.0 = sext i16 %ld.b.0 to i32
+  %mul.0 = mul i32 %sext.a.0, %sext.b.0
+  %addr.a.1 = getelementptr i16, i16* %a, i32 1
+  %addr.b.1 = getelementptr i16, i16* %b, i32 1
+  %ld.a.1 = load i16, i16* %addr.a.1
+  %sext.a.1 = sext i16 %ld.a.1 to i32
+  %ld.b.1 = load i16, i16* %addr.b.1
+  %sext.b.1 = sext i16 %ld.b.1 to i32
+  %mul.1 = mul i32 %sext.a.1, %sext.b.1
+  %add = add i32 %mul.0, %mul.1
+  br label %bb.1
+
+bb.1:
+  %res = add i32 %add, %acc
+  ret i32 %res
+}
+
+; CHECK-LABEL: multi_block_64
+; CHECK: [[CAST_A:%[^ ]+]] = bitcast i16* %a to i32*
+; CHECK: [[A:%[^ ]+]] = load i32, i32* [[CAST_A]]
+; CHECK: [[CAST_B:%[^ ]+]] = bitcast i16* %b to i32*
+; CHECK: [[B:%[^ ]+]] = load i32, i32* [[CAST_B]]
+; CHECK:  call i64 @llvm.arm.smlald(i32 [[A]], i32 [[B]], i64 0)
+define i64 @multi_block_64(i16* %a, i16* %b, i64 %acc) {
+entry:
+  %ld.a.0 = load i16, i16* %a
+  %sext.a.0 = sext i16 %ld.a.0 to i32
+  %ld.b.0 = load i16, i16* %b
+  %sext.b.0 = sext i16 %ld.b.0 to i32
+  %mul.0 = mul i32 %sext.a.0, %sext.b.0
+  %addr.a.1 = getelementptr i16, i16* %a, i32 1
+  %addr.b.1 = getelementptr i16, i16* %b, i32 1
+  %ld.a.1 = load i16, i16* %addr.a.1
+  %sext.a.1 = sext i16 %ld.a.1 to i32
+  %ld.b.1 = load i16, i16* %addr.b.1
+  %sext.b.1 = sext i16 %ld.b.1 to i32
+  %mul.1 = mul i32 %sext.a.1, %sext.b.1
+  %sext.mul.0 = sext i32 %mul.0 to i64
+  %sext.mul.1 = sext i32 %mul.1 to i64
+  %add = add i64 %sext.mul.0, %sext.mul.1
+  br label %bb.1
+
+bb.1:
+  %res = add i64 %add, %acc
+  ret i64 %res
+}
+
+; CHECK-LABEL: multi_block_1
+; CHECK-NOT: call i32 @llvm.arm.smlad
+define i32 @multi_block_1(i16* %a, i16* %b, i32 %acc) {
+entry:
+  %ld.a.0 = load i16, i16* %a
+  %sext.a.0 = sext i16 %ld.a.0 to i32
+  %ld.b.0 = load i16, i16* %b
+  %sext.b.0 = sext i16 %ld.b.0 to i32
+  %mul.0 = mul i32 %sext.a.0, %sext.b.0
+  br label %bb.1
+
+bb.1:
+  %addr.a.1 = getelementptr i16, i16* %a, i32 1
+  %addr.b.1 = getelementptr i16, i16* %b, i32 1
+  %ld.a.1 = load i16, i16* %addr.a.1
+  %sext.a.1 = sext i16 %ld.a.1 to i32
+  %ld.b.1 = load i16, i16* %addr.b.1
+  %sext.b.1 = sext i16 %ld.b.1 to i32
+  %mul.1 = mul i32 %sext.a.1, %sext.b.1
+  %add = add i32 %mul.0, %mul.1
+  %res = add i32 %add, %acc
+  ret i32 %res
+}
+
+; TODO: Four smlads should be generated here, but mul.0 and mul.3 remain as
+; scalars.
+; CHECK-LABEL: num_load_limit
+; CHECK: call i32 @llvm.arm.smlad
+; CHECK: call i32 @llvm.arm.smlad
+; CHECK: call i32 @llvm.arm.smlad
+; CHECK-NOT: call i32 @llvm.arm.smlad
+define i32 @num_load_limit(i16* %a, i16* %b, i32 %acc) {
+entry:
+  %ld.a.0 = load i16, i16* %a
+  %sext.a.0 = sext i16 %ld.a.0 to i32
+  %ld.b.0 = load i16, i16* %b
+  %sext.b.0 = sext i16 %ld.b.0 to i32
+  %mul.0 = mul i32 %sext.a.0, %sext.b.0
+  %addr.a.1 = getelementptr i16, i16* %a, i32 1
+  %addr.b.1 = getelementptr i16, i16* %b, i32 1
+  %ld.a.1 = load i16, i16* %addr.a.1
+  %sext.a.1 = sext i16 %ld.a.1 to i32
+  %ld.b.1 = load i16, i16* %addr.b.1
+  %sext.b.1 = sext i16 %ld.b.1 to i32
+  %mul.1 = mul i32 %sext.a.1, %sext.b.1
+  %add.0 = add i32 %mul.0, %mul.1
+
+  %addr.a.2 = getelementptr i16, i16* %a, i32 2
+  %addr.b.2 = getelementptr i16, i16* %b, i32 2
+  %ld.a.2 = load i16, i16* %addr.a.2
+  %sext.a.2 = sext i16 %ld.a.2 to i32
+  %ld.b.2 = load i16, i16* %addr.b.2
+  %sext.b.2 = sext i16 %ld.b.2 to i32
+  %mul.2 = mul i32 %sext.a.0, %sext.b.0
+  %addr.a.3 = getelementptr i16, i16* %a, i32 3
+  %addr.b.3 = getelementptr i16, i16* %b, i32 3
+  %ld.a.3 = load i16, i16* %addr.a.3
+  %sext.a.3 = sext i16 %ld.a.3 to i32
+  %ld.b.3 = load i16, i16* %addr.b.3
+  %sext.b.3 = sext i16 %ld.b.3 to i32
+  %mul.3 = mul i32 %sext.a.1, %sext.b.3
+  %add.3 = add i32 %mul.2, %mul.3
+
+  %addr.a.4 = getelementptr i16, i16* %a, i32 4
+  %addr.b.4 = getelementptr i16, i16* %b, i32 4
+  %ld.a.4 = load i16, i16* %addr.a.4
+  %sext.a.4 = sext i16 %ld.a.4 to i32
+  %ld.b.4 = load i16, i16* %addr.b.4
+  %sext.b.4 = sext i16 %ld.b.4 to i32
+  %mul.4 = mul i32 %sext.a.4, %sext.b.4
+  %addr.a.5 = getelementptr i16, i16* %a, i32 5
+  %addr.b.5 = getelementptr i16, i16* %b, i32 5
+  %ld.a.5 = load i16, i16* %addr.a.5
+  %sext.a.5 = sext i16 %ld.a.5 to i32
+  %ld.b.5 = load i16, i16* %addr.b.5
+  %sext.b.5 = sext i16 %ld.b.5 to i32
+  %mul.5 = mul i32 %sext.a.5, %sext.b.5
+  %add.5 = add i32 %mul.4, %mul.5
+
+  %addr.a.6 = getelementptr i16, i16* %a, i32 6
+  %addr.b.6 = getelementptr i16, i16* %b, i32 6
+  %ld.a.6 = load i16, i16* %addr.a.6
+  %sext.a.6 = sext i16 %ld.a.6 to i32
+  %ld.b.6 = load i16, i16* %addr.b.6
+  %sext.b.6 = sext i16 %ld.b.6 to i32
+  %mul.6 = mul i32 %sext.a.6, %sext.b.6
+  %addr.a.7 = getelementptr i16, i16* %a, i32 7
+  %addr.b.7 = getelementptr i16, i16* %b, i32 7
+  %ld.a.7 = load i16, i16* %addr.a.7
+  %sext.a.7 = sext i16 %ld.a.7 to i32
+  %ld.b.7 = load i16, i16* %addr.b.7
+  %sext.b.7 = sext i16 %ld.b.7 to i32
+  %mul.7 = mul i32 %sext.a.7, %sext.b.7
+  %add.7 = add i32 %mul.6, %mul.7
+
+  %add.10 = add i32 %add.7, %add.5
+  %add.11 = add i32 %add.3, %add.0
+  %add.12 = add i32 %add.10, %add.11
+  %res = add i32 %add.12, %acc
+  ret i32 %res
+}
+
+; CHECK-LABEL: too_many_loads
+; CHECK-NOT: call i32 @llvm.arm.smlad
+define i32 @too_many_loads(i16* %a, i16* %b, i32 %acc) {
+entry:
+  %ld.a.0 = load i16, i16* %a
+  %sext.a.0 = sext i16 %ld.a.0 to i32
+  %ld.b.0 = load i16, i16* %b
+  %sext.b.0 = sext i16 %ld.b.0 to i32
+  %mul.0 = mul i32 %sext.a.0, %sext.b.0
+  %addr.a.1 = getelementptr i16, i16* %a, i32 1
+  %addr.b.1 = getelementptr i16, i16* %b, i32 1
+  %ld.a.1 = load i16, i16* %addr.a.1
+  %sext.a.1 = sext i16 %ld.a.1 to i32
+  %ld.b.1 = load i16, i16* %addr.b.1
+  %sext.b.1 = sext i16 %ld.b.1 to i32
+  %mul.1 = mul i32 %sext.a.1, %sext.b.1
+  %add.0 = add i32 %mul.0, %mul.1
+
+  %addr.a.2 = getelementptr i16, i16* %a, i32 2
+  %addr.b.2 = getelementptr i16, i16* %b, i32 2
+  %ld.a.2 = load i16, i16* %addr.a.2
+  %sext.a.2 = sext i16 %ld.a.2 to i32
+  %ld.b.2 = load i16, i16* %addr.b.2
+  %sext.b.2 = sext i16 %ld.b.2 to i32
+  %mul.2 = mul i32 %sext.a.0, %sext.b.0
+  %addr.a.3 = getelementptr i16, i16* %a, i32 3
+  %addr.b.3 = getelementptr i16, i16* %b, i32 3
+  %ld.a.3 = load i16, i16* %addr.a.3
+  %sext.a.3 = sext i16 %ld.a.3 to i32
+  %ld.b.3 = load i16, i16* %addr.b.3
+  %sext.b.3 = sext i16 %ld.b.3 to i32
+  %mul.3 = mul i32 %sext.a.1, %sext.b.3
+  %add.3 = add i32 %mul.2, %mul.3
+
+  %addr.a.4 = getelementptr i16, i16* %a, i32 4
+  %addr.b.4 = getelementptr i16, i16* %b, i32 4
+  %ld.a.4 = load i16, i16* %addr.a.4
+  %sext.a.4 = sext i16 %ld.a.4 to i32
+  %ld.b.4 = load i16, i16* %addr.b.4
+  %sext.b.4 = sext i16 %ld.b.4 to i32
+  %mul.4 = mul i32 %sext.a.4, %sext.b.4
+  %addr.a.5 = getelementptr i16, i16* %a, i32 5
+  %addr.b.5 = getelementptr i16, i16* %b, i32 5
+  %ld.a.5 = load i16, i16* %addr.a.5
+  %sext.a.5 = sext i16 %ld.a.5 to i32
+  %ld.b.5 = load i16, i16* %addr.b.5
+  %sext.b.5 = sext i16 %ld.b.5 to i32
+  %mul.5 = mul i32 %sext.a.5, %sext.b.5
+  %add.5 = add i32 %mul.4, %mul.5
+
+  %addr.a.6 = getelementptr i16, i16* %a, i32 6
+  %addr.b.6 = getelementptr i16, i16* %b, i32 6
+  %ld.a.6 = load i16, i16* %addr.a.6
+  %sext.a.6 = sext i16 %ld.a.6 to i32
+  %ld.b.6 = load i16, i16* %addr.b.6
+  %sext.b.6 = sext i16 %ld.b.6 to i32
+  %mul.6 = mul i32 %sext.a.6, %sext.b.6
+  %addr.a.7 = getelementptr i16, i16* %a, i32 7
+  %addr.b.7 = getelementptr i16, i16* %b, i32 7
+  %ld.a.7 = load i16, i16* %addr.a.7
+  %sext.a.7 = sext i16 %ld.a.7 to i32
+  %ld.b.7 = load i16, i16* %addr.b.7
+  %sext.b.7 = sext i16 %ld.b.7 to i32
+  %mul.7 = mul i32 %sext.a.7, %sext.b.7
+  %add.7 = add i32 %mul.6, %mul.7
+
+  %addr.a.8 = getelementptr i16, i16* %a, i32 7
+  %addr.b.8 = getelementptr i16, i16* %b, i32 7
+  %ld.a.8 = load i16, i16* %addr.a.8
+  %sext.a.8 = sext i16 %ld.a.8 to i32
+  %ld.b.8 = load i16, i16* %addr.b.8
+  %sext.b.8 = sext i16 %ld.b.8 to i32
+  %mul.8 = mul i32 %sext.a.8, %sext.b.8
+
+  %add.10 = add i32 %add.7, %add.5
+  %add.11 = add i32 %add.3, %add.0
+  %add.12 = add i32 %add.10, %add.11
+  %add.13 = add i32 %add.12, %acc
+  %res = add i32 %add.13, %mul.8
+  ret i32 %res
+}
diff --git a/llvm/test/CodeGen/ARM/ParallelDSP/complex_dot_prod.ll b/llvm/test/CodeGen/ARM/ParallelDSP/complex_dot_prod.ll
new file mode 100644
index 000000000..fab5aba46
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/ParallelDSP/complex_dot_prod.ll
@@ -0,0 +1,150 @@
+; RUN: llc -mtriple=thumbv7em -mcpu=cortex-m4 -O3 %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-LLC
+; RUN: opt -S -mtriple=armv7-a -arm-parallel-dsp -dce %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-OPT
+
+; TODO: Think we should be able to use smlsdx/smlsldx here.
+
+; CHECK-LABEL: complex_dot_prod
+
+; CHECK-LLC: smlaldx
+; CHECK-LLC: smulbb
+; CHECK-LLC: smultt
+; CHECK-LLC: smlaldx
+; CHECK-LLC: smlalbb
+; CHECK-LLC: smultt
+; CHECK-LLC: smlalbb
+; CHECK-LLC: smultt
+; CHECK-LLC: smlaldx
+; CHECK-LLC: smlalbb
+; CHECK-LLC: smultt
+; CHECK-LLC: smlaldx
+; CHECK-LCC: pop.w {r4, r5, r6, r7, r8, r9, r10, pc}
+
+; CHECK-OPT: [[ADDR_A:%[^ ]+]] = bitcast i16* %pSrcA to i32*
+; CHECK-OPT: [[A:%[^ ]+]] = load i32, i32* [[ADDR_A]], align 2
+; CHECK-OPT: [[ADDR_A_2:%[^ ]+]] = getelementptr inbounds i16, i16* %pSrcA, i32 2
+; CHECK-OPT: [[ADDR_B:%[^ ]+]] = bitcast i16* %pSrcB to i32*
+; CHECK-OPT: [[B:%[^ ]+]] = load i32, i32* [[ADDR_B]], align 2
+; CHECK-OPT: [[ACC0:%[^ ]+]] = call i64 @llvm.arm.smlaldx(i32 [[A]], i32 [[B]], i64 0)
+; CHECK-OPT: [[ADDR_B_2:%[^ ]+]] = getelementptr inbounds i16, i16* %pSrcB, i32 2
+; CHECK-OPT: [[CAST_ADDR_A_2:%[^ ]+]] = bitcast i16* [[ADDR_A_2]] to i32*
+; CHECK-OPT: [[A_2:%[^ ]+]] = load i32, i32* [[CAST_ADDR_A_2]], align 2
+; CHECK-OPT: [[ADDR_A_4:%[^ ]+]] = getelementptr inbounds i16, i16* %pSrcA, i32 4
+; CHECK-OPT: [[CAST_ADDR_B_2:%[^ ]+]] = bitcast i16* [[ADDR_B_2]] to i32*
+; CHECK-OPT: [[B_2:%[^ ]+]] = load i32, i32* [[CAST_ADDR_B_2]], align 2
+; CHECK-OPT: [[ACC1:%[^ ]+]] = call i64 @llvm.arm.smlaldx(i32 [[A_2]], i32 [[B_2]], i64 [[ACC0]])
+; CHECK-OPT: [[ADDR_B_4:%[^ ]+]] = getelementptr inbounds i16, i16* %pSrcB, i32 4
+; CHECK-OPT: [[CAST_ADDR_A_4:%[^ ]+]] = bitcast i16* [[ADDR_A_4]] to i32*
+; CHECK-OPT: [[A_4:%[^ ]+]] = load i32, i32* [[CAST_ADDR_A_4]], align 2
+; CHECK-OPT: [[ADDR_A_6:%[^ ]+]] = getelementptr inbounds i16, i16* %pSrcA, i32 6
+; CHECK-OPT: [[CAST_ADDR_B_4:%[^ ]+]] = bitcast i16* [[ADDR_B_4]] to i32*
+; CHECK-OPT: [[B_4:%[^ ]+]] = load i32, i32* [[CAST_ADDR_B_4]], align 2
+; CHECK-OPT: [[ACC2:%[^ ]+]] = call i64 @llvm.arm.smlaldx(i32 [[A_4]], i32 [[B_4]], i64 [[ACC1]])
+; CHECK-OPT: [[ADDR_B_6:%[^ ]+]] = getelementptr inbounds i16, i16* %pSrcB, i32 6
+; CHECK-OPT: [[CAST_ADDR_A_6:%[^ ]+]] = bitcast i16* [[ADDR_A_6]] to i32*
+; CHECK-OPT: [[A_6:%[^ ]+]] = load i32, i32* [[CAST_ADDR_A_6]], align 2
+; CHECK-OPT: [[CAST_ADDR_B_6:%[^ ]+]] = bitcast i16* [[ADDR_B_6]] to i32*
+; CHECK-OPT: [[B_6:%[^ ]+]] = load i32, i32* [[CAST_ADDR_B_6]], align 2
+; CHECK-OPT: call i64 @llvm.arm.smlaldx(i32 [[A_6]], i32 [[B_6]], i64 [[ACC2]])
+
+define dso_local arm_aapcscc void @complex_dot_prod(i16* nocapture readonly %pSrcA, i16* nocapture readonly %pSrcB, i32* nocapture %realResult, i32* nocapture %imagResult) {
+entry:
+  %incdec.ptr = getelementptr inbounds i16, i16* %pSrcA, i32 1
+  %0 = load i16, i16* %pSrcA, align 2
+  %incdec.ptr1 = getelementptr inbounds i16, i16* %pSrcA, i32 2
+  %1 = load i16, i16* %incdec.ptr, align 2
+  %incdec.ptr2 = getelementptr inbounds i16, i16* %pSrcB, i32 1
+  %2 = load i16, i16* %pSrcB, align 2
+  %incdec.ptr3 = getelementptr inbounds i16, i16* %pSrcB, i32 2
+  %3 = load i16, i16* %incdec.ptr2, align 2
+  %conv = sext i16 %0 to i32
+  %conv4 = sext i16 %2 to i32
+  %mul = mul nsw i32 %conv4, %conv
+  %conv5 = sext i32 %mul to i64
+  %conv7 = sext i16 %3 to i32
+  %mul8 = mul nsw i32 %conv7, %conv
+  %conv9 = sext i32 %mul8 to i64
+  %conv11 = sext i16 %1 to i32
+  %mul13 = mul nsw i32 %conv7, %conv11
+  %conv14 = sext i32 %mul13 to i64
+  %sub = sub nsw i64 %conv5, %conv14
+  %mul17 = mul nsw i32 %conv4, %conv11
+  %conv18 = sext i32 %mul17 to i64
+  %add19 = add nsw i64 %conv9, %conv18
+  %incdec.ptr20 = getelementptr inbounds i16, i16* %pSrcA, i32 3
+  %4 = load i16, i16* %incdec.ptr1, align 2
+  %incdec.ptr21 = getelementptr inbounds i16, i16* %pSrcA, i32 4
+  %5 = load i16, i16* %incdec.ptr20, align 2
+  %incdec.ptr22 = getelementptr inbounds i16, i16* %pSrcB, i32 3
+  %6 = load i16, i16* %incdec.ptr3, align 2
+  %incdec.ptr23 = getelementptr inbounds i16, i16* %pSrcB, i32 4
+  %7 = load i16, i16* %incdec.ptr22, align 2
+  %conv24 = sext i16 %4 to i32
+  %conv25 = sext i16 %6 to i32
+  %mul26 = mul nsw i32 %conv25, %conv24
+  %conv27 = sext i32 %mul26 to i64
+  %add28 = add nsw i64 %sub, %conv27
+  %conv30 = sext i16 %7 to i32
+  %mul31 = mul nsw i32 %conv30, %conv24
+  %conv32 = sext i32 %mul31 to i64
+  %conv34 = sext i16 %5 to i32
+  %mul36 = mul nsw i32 %conv30, %conv34
+  %conv37 = sext i32 %mul36 to i64
+  %sub38 = sub nsw i64 %add28, %conv37
+  %mul41 = mul nsw i32 %conv25, %conv34
+  %conv42 = sext i32 %mul41 to i64
+  %add33 = add nsw i64 %add19, %conv42
+  %add43 = add nsw i64 %add33, %conv32
+  %incdec.ptr44 = getelementptr inbounds i16, i16* %pSrcA, i32 5
+  %8 = load i16, i16* %incdec.ptr21, align 2
+  %incdec.ptr45 = getelementptr inbounds i16, i16* %pSrcA, i32 6
+  %9 = load i16, i16* %incdec.ptr44, align 2
+  %incdec.ptr46 = getelementptr inbounds i16, i16* %pSrcB, i32 5
+  %10 = load i16, i16* %incdec.ptr23, align 2
+  %incdec.ptr47 = getelementptr inbounds i16, i16* %pSrcB, i32 6
+  %11 = load i16, i16* %incdec.ptr46, align 2
+  %conv48 = sext i16 %8 to i32
+  %conv49 = sext i16 %10 to i32
+  %mul50 = mul nsw i32 %conv49, %conv48
+  %conv51 = sext i32 %mul50 to i64
+  %add52 = add nsw i64 %sub38, %conv51
+  %conv54 = sext i16 %11 to i32
+  %mul55 = mul nsw i32 %conv54, %conv48
+  %conv56 = sext i32 %mul55 to i64
+  %conv58 = sext i16 %9 to i32
+  %mul60 = mul nsw i32 %conv54, %conv58
+  %conv61 = sext i32 %mul60 to i64
+  %sub62 = sub nsw i64 %add52, %conv61
+  %mul65 = mul nsw i32 %conv49, %conv58
+  %conv66 = sext i32 %mul65 to i64
+  %add57 = add nsw i64 %add43, %conv66
+  %add67 = add nsw i64 %add57, %conv56
+  %incdec.ptr68 = getelementptr inbounds i16, i16* %pSrcA, i32 7
+  %12 = load i16, i16* %incdec.ptr45, align 2
+  %13 = load i16, i16* %incdec.ptr68, align 2
+  %incdec.ptr70 = getelementptr inbounds i16, i16* %pSrcB, i32 7
+  %14 = load i16, i16* %incdec.ptr47, align 2
+  %15 = load i16, i16* %incdec.ptr70, align 2
+  %conv72 = sext i16 %12 to i32
+  %conv73 = sext i16 %14 to i32
+  %mul74 = mul nsw i32 %conv73, %conv72
+  %conv75 = sext i32 %mul74 to i64
+  %add76 = add nsw i64 %sub62, %conv75
+  %conv78 = sext i16 %15 to i32
+  %mul79 = mul nsw i32 %conv78, %conv72
+  %conv80 = sext i32 %mul79 to i64
+  %conv82 = sext i16 %13 to i32
+  %mul84 = mul nsw i32 %conv78, %conv82
+  %conv85 = sext i32 %mul84 to i64
+  %sub86 = sub nsw i64 %add76, %conv85
+  %mul89 = mul nsw i32 %conv73, %conv82
+  %conv90 = sext i32 %mul89 to i64
+  %add81 = add nsw i64 %add67, %conv90
+  %add91 = add nsw i64 %add81, %conv80
+  %16 = lshr i64 %sub86, 6
+  %conv92 = trunc i64 %16 to i32
+  store i32 %conv92, i32* %realResult, align 4
+  %17 = lshr i64 %add91, 6
+  %conv94 = trunc i64 %17 to i32
+  store i32 %conv94, i32* %imagResult, align 4
+  ret void
+}
diff --git a/llvm/test/CodeGen/ARM/ParallelDSP/exchange.ll b/llvm/test/CodeGen/ARM/ParallelDSP/exchange.ll
new file mode 100644
index 000000000..e90284b86
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/ParallelDSP/exchange.ll
@@ -0,0 +1,452 @@
+; RUN: opt -arm-parallel-dsp -mtriple=armv7-a -S %s -o - | FileCheck %s
+
+; CHECK-LABEL: exchange_1
+; CHECK: [[CAST_A:%[^ ]+]] = bitcast i16* %a to i32*
+; CHECK: [[LD_A:%[^ ]+]] = load i32, i32* [[CAST_A]]
+; CHECK: [[CAST_B:%[^ ]+]] = bitcast i16* %b to i32*
+; CHECK: [[LD_B:%[^ ]+]] = load i32, i32* [[CAST_B]]
+; CHECK: call i32 @llvm.arm.smladx(i32 [[LD_A]], i32 [[LD_B]]
+define i32 @exchange_1(i16* %a, i16* %b, i32 %acc) {
+entry:
+  %addr.a.1 = getelementptr i16, i16* %a, i32 1
+  %addr.b.1 = getelementptr i16, i16* %b, i32 1
+  %ld.a.0 = load i16, i16* %a
+  %sext.a.0 = sext i16 %ld.a.0 to i32
+  %ld.b.0 = load i16, i16* %b
+  %ld.a.1 = load i16, i16* %addr.a.1
+  %ld.b.1 = load i16, i16* %addr.b.1
+  %sext.a.1 = sext i16 %ld.a.1 to i32
+  %sext.b.1 = sext i16 %ld.b.1 to i32
+  %sext.b.0 = sext i16 %ld.b.0 to i32
+  %mul.0 = mul i32 %sext.a.0, %sext.b.1
+  %mul.1 = mul i32 %sext.a.1, %sext.b.0
+  %add = add i32 %mul.0, %mul.1
+  %res = add i32 %add, %acc
+  ret i32 %res
+}
+
+; CHECK-LABEL: exchange_2
+; CHECK: [[CAST_A:%[^ ]+]] = bitcast i16* %a to i32*
+; CHECK: [[LD_A:%[^ ]+]] = load i32, i32* [[CAST_A]]
+; CHECK: [[CAST_B:%[^ ]+]] = bitcast i16* %b to i32*
+; CHECK: [[LD_B:%[^ ]+]] = load i32, i32* [[CAST_B]]
+; CHECK: call i32 @llvm.arm.smladx(i32 [[LD_A]], i32 [[LD_B]]
+define i32 @exchange_2(i16* %a, i16* %b, i32 %acc) {
+entry:
+  %addr.a.1 = getelementptr i16, i16* %a, i32 1
+  %addr.b.1 = getelementptr i16, i16* %b, i32 1
+  %ld.a.0 = load i16, i16* %a
+  %sext.a.0 = sext i16 %ld.a.0 to i32
+  %ld.b.0 = load i16, i16* %b
+  %ld.a.1 = load i16, i16* %addr.a.1
+  %ld.b.1 = load i16, i16* %addr.b.1
+  %sext.a.1 = sext i16 %ld.a.1 to i32
+  %sext.b.1 = sext i16 %ld.b.1 to i32
+  %sext.b.0 = sext i16 %ld.b.0 to i32
+  %mul.0 = mul i32 %sext.b.1, %sext.a.0
+  %mul.1 = mul i32 %sext.b.0, %sext.a.1
+  %add = add i32 %mul.0, %mul.1
+  %res = add i32 %add, %acc
+  ret i32 %res
+}
+
+; CHECK-LABEL: exchange_3
+; CHECK: [[CAST_A:%[^ ]+]] = bitcast i16* %a to i32*
+; CHECK: [[LD_A:%[^ ]+]] = load i32, i32* [[CAST_A]]
+; CHECK: [[CAST_B:%[^ ]+]] = bitcast i16* %b to i32*
+; CHECK: [[LD_B:%[^ ]+]] = load i32, i32* [[CAST_B]]
+; CHECK: call i32 @llvm.arm.smladx(i32 [[LD_B]], i32 [[LD_A]]
+define i32 @exchange_3(i16* %a, i16* %b, i32 %acc) {
+entry:
+  %addr.a.1 = getelementptr i16, i16* %a, i32 1
+  %addr.b.1 = getelementptr i16, i16* %b, i32 1
+  %ld.a.0 = load i16, i16* %a
+  %sext.a.0 = sext i16 %ld.a.0 to i32
+  %ld.b.0 = load i16, i16* %b
+  %ld.a.1 = load i16, i16* %addr.a.1
+  %ld.b.1 = load i16, i16* %addr.b.1
+  %sext.a.1 = sext i16 %ld.a.1 to i32
+  %sext.b.1 = sext i16 %ld.b.1 to i32
+  %sext.b.0 = sext i16 %ld.b.0 to i32
+  %mul.0 = mul i32 %sext.a.0, %sext.b.1
+  %mul.1 = mul i32 %sext.a.1, %sext.b.0
+  %add = add i32 %mul.1, %mul.0
+  %res = add i32 %add, %acc
+  ret i32 %res
+}
+
+; CHECK-LABEL: exchange_4
+; CHECK: [[CAST_A:%[^ ]+]] = bitcast i16* %a to i32*
+; CHECK: [[LD_A:%[^ ]+]] = load i32, i32* [[CAST_A]]
+; CHECK: [[CAST_B:%[^ ]+]] = bitcast i16* %b to i32*
+; CHECK: [[LD_B:%[^ ]+]] = load i32, i32* [[CAST_B]]
+; CHECK: call i32 @llvm.arm.smladx(i32 [[LD_B]], i32 [[LD_A]]
+define i32 @exchange_4(i16* %a, i16* %b, i32 %acc) {
+entry:
+  %addr.a.1 = getelementptr i16, i16* %a, i32 1
+  %addr.b.1 = getelementptr i16, i16* %b, i32 1
+  %ld.a.0 = load i16, i16* %a
+  %sext.a.0 = sext i16 %ld.a.0 to i32
+  %ld.b.0 = load i16, i16* %b
+  %ld.a.1 = load i16, i16* %addr.a.1
+  %ld.b.1 = load i16, i16* %addr.b.1
+  %sext.a.1 = sext i16 %ld.a.1 to i32
+  %sext.b.1 = sext i16 %ld.b.1 to i32
+  %sext.b.0 = sext i16 %ld.b.0 to i32
+  %mul.0 = mul i32 %sext.b.1, %sext.a.0
+  %mul.1 = mul i32 %sext.b.0, %sext.a.1
+  %add = add i32 %mul.1, %mul.0
+  %res = add i32 %add, %acc
+  ret i32 %res
+}
+
+; CHECK-LABEL: exchange_multi_use_1
+; CHECK: [[CAST_A:%[^ ]+]] = bitcast i16* %a to i32*
+; CHECK: [[LD_A:%[^ ]+]] = load i32, i32* [[CAST_A]]
+; CHECK: [[CAST_B:%[^ ]+]] = bitcast i16* %b to i32*
+; CHECK: [[LD_B:%[^ ]+]] = load i32, i32* [[CAST_B]]
+; CHECK: [[X:%[^ ]+]] = call i32 @llvm.arm.smladx(i32 [[LD_A]], i32 [[LD_B]], i32 %acc
+; CHECK: [[GEP:%[^ ]+]] = getelementptr i16, i16* %a, i32 2
+; CHECK: [[CAST_A_2:%[^ ]+]] = bitcast i16* [[GEP]] to i32*
+; CHECK: [[LD_A_2:%[^ ]+]] = load i32, i32* [[CAST_A_2]]
+; CHECK: call i32 @llvm.arm.smlad(i32 [[LD_A_2]], i32 [[LD_B]], i32 [[X]])
+define i32 @exchange_multi_use_1(i16* %a, i16* %b, i32 %acc) {
+entry:
+  %addr.a.1 = getelementptr i16, i16* %a, i32 1
+  %addr.b.1 = getelementptr i16, i16* %b, i32 1
+  %ld.a.0 = load i16, i16* %a
+  %sext.a.0 = sext i16 %ld.a.0 to i32
+  %ld.b.0 = load i16, i16* %b
+  %ld.a.1 = load i16, i16* %addr.a.1
+  %ld.b.1 = load i16, i16* %addr.b.1
+  %sext.a.1 = sext i16 %ld.a.1 to i32
+  %sext.b.1 = sext i16 %ld.b.1 to i32
+  %sext.b.0 = sext i16 %ld.b.0 to i32
+  %mul.0 = mul i32 %sext.a.0, %sext.b.1
+  %mul.1 = mul i32 %sext.a.1, %sext.b.0
+  %add = add i32 %mul.0, %mul.1
+  %addr.a.2 = getelementptr i16, i16* %a, i32 2
+  %addr.a.3 = getelementptr i16, i16* %a, i32 3
+  %ld.a.2 = load i16, i16* %addr.a.2
+  %ld.a.3 = load i16, i16* %addr.a.3
+  %sext.a.2 = sext i16 %ld.a.2 to i32
+  %sext.a.3 = sext i16 %ld.a.3 to i32
+  %mul.2 = mul i32 %sext.a.3, %sext.b.1
+  %mul.3 = mul i32 %sext.a.2, %sext.b.0
+  %add.1 = add i32 %mul.2, %mul.3
+  %add.2 = add i32 %add, %add.1
+  %res = add i32 %add.2, %acc
+  ret i32 %res
+}
+
+; CHECK-LABEL: exchange_multi_use_64_1
+; CHECK: [[CAST_A:%[^ ]+]] = bitcast i16* %a to i32*
+; CHECK: [[LD_A:%[^ ]+]] = load i32, i32* [[CAST_A]]
+; CHECK: [[CAST_B:%[^ ]+]] = bitcast i16* %b to i32*
+; CHECK: [[LD_B:%[^ ]+]] = load i32, i32* [[CAST_B]]
+; CHECK: [[X:%[^ ]+]] = call i64 @llvm.arm.smlaldx(i32 [[LD_A]], i32 [[LD_B]], i64 %acc
+; CHECK: [[GEP:%[^ ]+]] = getelementptr i16, i16* %a, i32 2
+; CHECK: [[CAST_A_2:%[^ ]+]] = bitcast i16* [[GEP]] to i32*
+; CHECK: [[LD_A_2:%[^ ]+]] = load i32, i32* [[CAST_A_2]]
+; CHECK: call i64 @llvm.arm.smlald(i32 [[LD_A_2]], i32 [[LD_B]], i64 [[X]])
+define i64 @exchange_multi_use_64_1(i16* %a, i16* %b, i64 %acc) {
+entry:
+  %addr.a.1 = getelementptr i16, i16* %a, i32 1
+  %addr.b.1 = getelementptr i16, i16* %b, i32 1
+  %ld.a.0 = load i16, i16* %a
+  %sext.a.0 = sext i16 %ld.a.0 to i32
+  %ld.b.0 = load i16, i16* %b
+  %ld.a.1 = load i16, i16* %addr.a.1
+  %ld.b.1 = load i16, i16* %addr.b.1
+  %sext.a.1 = sext i16 %ld.a.1 to i32
+  %sext.b.1 = sext i16 %ld.b.1 to i32
+  %sext.b.0 = sext i16 %ld.b.0 to i32
+  %mul.0 = mul i32 %sext.a.0, %sext.b.1
+  %mul.1 = mul i32 %sext.a.1, %sext.b.0
+  %add = add i32 %mul.0, %mul.1
+  %addr.a.2 = getelementptr i16, i16* %a, i32 2
+  %addr.a.3 = getelementptr i16, i16* %a, i32 3
+  %ld.a.2 = load i16, i16* %addr.a.2
+  %ld.a.3 = load i16, i16* %addr.a.3
+  %sext.a.2 = sext i16 %ld.a.2 to i32
+  %sext.a.3 = sext i16 %ld.a.3 to i32
+  %mul.2 = mul i32 %sext.a.3, %sext.b.1
+  %mul.3 = mul i32 %sext.a.2, %sext.b.0
+  %add.1 = add i32 %mul.2, %mul.3
+  %add.2 = add i32 %add, %add.1
+  %sext.add.2 = sext i32 %add.2 to i64
+  %res = add i64 %sext.add.2, %acc
+  ret i64 %res
+}
+
+; CHECK-LABEL: exchange_multi_use_64_2
+; CHECK: [[CAST_A:%[^ ]+]] = bitcast i16* %a to i32*
+; CHECK: [[LD_A:%[^ ]+]] = load i32, i32* [[CAST_A]]
+; CHECK: [[CAST_B:%[^ ]+]] = bitcast i16* %b to i32*
+; CHECK: [[LD_B:%[^ ]+]] = load i32, i32* [[CAST_B]]
+; CHECK: [[X:%[^ ]+]] = call i64 @llvm.arm.smlaldx(i32 [[LD_A]], i32 [[LD_B]], i64 %acc
+; CHECK: [[GEP:%[^ ]+]] = getelementptr i16, i16* %a, i32 2
+; CHECK: [[CAST_A_2:%[^ ]+]] = bitcast i16* [[GEP]] to i32*
+; CHECK: [[LD_A_2:%[^ ]+]] = load i32, i32* [[CAST_A_2]]
+; CHECK: call i64 @llvm.arm.smlald(i32 [[LD_A_2]], i32 [[LD_B]], i64 [[X]])
+define i64 @exchange_multi_use_64_2(i16* %a, i16* %b, i64 %acc) {
+entry:
+  %addr.a.1 = getelementptr i16, i16* %a, i32 1
+  %addr.b.1 = getelementptr i16, i16* %b, i32 1
+  %ld.a.0 = load i16, i16* %a
+  %sext.a.0 = sext i16 %ld.a.0 to i32
+  %ld.b.0 = load i16, i16* %b
+  %ld.a.1 = load i16, i16* %addr.a.1
+  %ld.b.1 = load i16, i16* %addr.b.1
+  %sext.a.1 = sext i16 %ld.a.1 to i32
+  %sext.b.1 = sext i16 %ld.b.1 to i32
+  %sext.b.0 = sext i16 %ld.b.0 to i32
+  %mul.0 = mul i32 %sext.a.0, %sext.b.1
+  %mul.1 = mul i32 %sext.a.1, %sext.b.0
+  %add = add i32 %mul.0, %mul.1
+  %sext.add = sext i32 %add to i64
+  %addr.a.2 = getelementptr i16, i16* %a, i32 2
+  %addr.a.3 = getelementptr i16, i16* %a, i32 3
+  %ld.a.2 = load i16, i16* %addr.a.2
+  %ld.a.3 = load i16, i16* %addr.a.3
+  %sext.a.2 = sext i16 %ld.a.2 to i32
+  %sext.a.3 = sext i16 %ld.a.3 to i32
+  %mul.2 = mul i32 %sext.a.3, %sext.b.1
+  %mul.3 = mul i32 %sext.a.2, %sext.b.0
+  %add.1 = add i32 %mul.2, %mul.3
+  %sext.add.1 = sext i32 %add.1 to i64
+  %add.2 = add i64 %sext.add, %sext.add.1
+  %res = add i64 %add.2, %acc
+  ret i64 %res
+}
+
+; CHECK-LABEL: exchange_multi_use_2
+; CHECK: [[CAST_A:%[^ ]+]] = bitcast i16* %a to i32*
+; CHECK: [[LD_A:%[^ ]+]] = load i32, i32* [[CAST_A]]
+; CHECK: [[CAST_B:%[^ ]+]] = bitcast i16* %b to i32*
+; CHECK: [[LD_B:%[^ ]+]] = load i32, i32* [[CAST_B]]
+; CHECK: [[X:%[^ ]+]] = call i32 @llvm.arm.smlad(i32 [[LD_A]], i32 [[LD_B]], i32 %acc
+; CHECK: [[GEP:%[^ ]+]] = getelementptr i16, i16* %a, i32 2
+; CHECK: [[CAST_A_2:%[^ ]+]] = bitcast i16* [[GEP]] to i32*
+; CHECK: [[LD_A_2:%[^ ]+]] = load i32, i32* [[CAST_A_2]]
+; CHECK: call i32 @llvm.arm.smladx(i32 [[LD_B]], i32 [[LD_A_2]], i32 [[X]])
+define i32 @exchange_multi_use_2(i16* %a, i16* %b, i32 %acc) {
+entry:
+  %addr.a.1 = getelementptr i16, i16* %a, i32 1
+  %addr.b.1 = getelementptr i16, i16* %b, i32 1
+  %ld.a.0 = load i16, i16* %a
+  %sext.a.0 = sext i16 %ld.a.0 to i32
+  %ld.b.0 = load i16, i16* %b
+  %ld.a.1 = load i16, i16* %addr.a.1
+  %ld.b.1 = load i16, i16* %addr.b.1
+  %sext.a.1 = sext i16 %ld.a.1 to i32
+  %sext.b.1 = sext i16 %ld.b.1 to i32
+  %sext.b.0 = sext i16 %ld.b.0 to i32
+  %mul.0 = mul i32 %sext.a.0, %sext.b.0
+  %mul.1 = mul i32 %sext.a.1, %sext.b.1
+  %add = add i32 %mul.0, %mul.1
+  %addr.a.2 = getelementptr i16, i16* %a, i32 2
+  %addr.a.3 = getelementptr i16, i16* %a, i32 3
+  %ld.a.2 = load i16, i16* %addr.a.2
+  %ld.a.3 = load i16, i16* %addr.a.3
+  %sext.a.2 = sext i16 %ld.a.2 to i32
+  %sext.a.3 = sext i16 %ld.a.3 to i32
+  %mul.2 = mul i32 %sext.b.0, %sext.a.3
+  %mul.3 = mul i32 %sext.b.1, %sext.a.2
+  %add.1 = add i32 %mul.2, %mul.3
+  %add.2 = add i32 %add, %add.1
+  %res = add i32 %add.2, %acc
+  ret i32 %res
+}
+
+; TODO: Why aren't two intrinsics generated?
+; CHECK-LABEL: exchange_multi_use_3
+; CHECK: [[CAST_B:%[^ ]+]] = bitcast i16* %b to i32*
+; CHECK: [[LD_B:%[^ ]+]] = load i32, i32* [[CAST_B]]
+; CHECK: [[GEP:%[^ ]+]] = getelementptr i16, i16* %a, i32 2
+; CHECK: [[CAST_A_2:%[^ ]+]] = bitcast i16* [[GEP]] to i32*
+; CHECK: [[LD_A_2:%[^ ]+]] = load i32, i32* [[CAST_A_2]]
+; CHECK-NOT: call i32 @llvm.arm.smlad
+; CHECK: [[X:%[^ ]+]] = call i32 @llvm.arm.smladx(i32 [[LD_B]], i32 [[LD_A_2]], i32 0
+define i32 @exchange_multi_use_3(i16* %a, i16* %b, i32 %acc) {
+entry:
+  %addr.a.1 = getelementptr i16, i16* %a, i32 1
+  %addr.b.1 = getelementptr i16, i16* %b, i32 1
+  %ld.a.0 = load i16, i16* %a
+  %sext.a.0 = sext i16 %ld.a.0 to i32
+  %ld.b.0 = load i16, i16* %b
+  %ld.a.1 = load i16, i16* %addr.a.1
+  %ld.b.1 = load i16, i16* %addr.b.1
+  %sext.a.1 = sext i16 %ld.a.1 to i32
+  %sext.b.1 = sext i16 %ld.b.1 to i32
+  %sext.b.0 = sext i16 %ld.b.0 to i32
+  %addr.a.2 = getelementptr i16, i16* %a, i32 2
+  %addr.a.3 = getelementptr i16, i16* %a, i32 3
+  %ld.a.2 = load i16, i16* %addr.a.2
+  %ld.a.3 = load i16, i16* %addr.a.3
+  %sext.a.2 = sext i16 %ld.a.2 to i32
+  %sext.a.3 = sext i16 %ld.a.3 to i32
+  %mul.2 = mul i32 %sext.b.0, %sext.a.3
+  %mul.3 = mul i32 %sext.b.1, %sext.a.2
+  %mul.0 = mul i32 %sext.a.0, %sext.b.0
+  %mul.1 = mul i32 %sext.a.1, %sext.b.1
+  %add = add i32 %mul.0, %mul.1
+  %add.1 = add i32 %mul.2, %mul.3
+  %sub = sub i32 %add, %add.1
+  %res = add i32 %acc, %sub
+  ret i32 %res
+}
+
+; TODO: Would it be better to generate a smlad and then sign extend it?
+; CHECK-LABEL: exchange_multi_use_64_3
+; CHECK: [[CAST_A:%[^ ]+]] = bitcast i16* %a to i32*
+; CHECK: [[LD_A:%[^ ]+]] = load i32, i32* [[CAST_A]]
+; CHECK: [[CAST_B:%[^ ]+]] = bitcast i16* %b to i32*
+; CHECK: [[LD_B:%[^ ]+]] = load i32, i32* [[CAST_B]]
+; CHECK: [[GEP:%[^ ]+]] = getelementptr i16, i16* %a, i32 2
+; CHECK: [[CAST_A_2:%[^ ]+]] = bitcast i16* [[GEP]] to i32*
+; CHECK: [[LD_A_2:%[^ ]+]] = load i32, i32* [[CAST_A_2]]
+; CHECK: [[ACC:%[^ ]+]] = call i64 @llvm.arm.smlaldx(i32 [[LD_B]], i32 [[LD_A_2]], i64 0)
+; CHECK: [[X:%[^ ]+]] = call i64 @llvm.arm.smlald(i32 [[LD_A]], i32 [[LD_B]], i64 [[ACC]])
+define i64 @exchange_multi_use_64_3(i16* %a, i16* %b, i64 %acc) {
+entry:
+  %addr.a.1 = getelementptr i16, i16* %a, i32 1
+  %addr.b.1 = getelementptr i16, i16* %b, i32 1
+  %ld.a.0 = load i16, i16* %a
+  %sext.a.0 = sext i16 %ld.a.0 to i32
+  %ld.b.0 = load i16, i16* %b
+  %ld.a.1 = load i16, i16* %addr.a.1
+  %ld.b.1 = load i16, i16* %addr.b.1
+  %sext.a.1 = sext i16 %ld.a.1 to i32
+  %sext.b.1 = sext i16 %ld.b.1 to i32
+  %sext.b.0 = sext i16 %ld.b.0 to i32
+  %addr.a.2 = getelementptr i16, i16* %a, i32 2
+  %addr.a.3 = getelementptr i16, i16* %a, i32 3
+  %ld.a.2 = load i16, i16* %addr.a.2
+  %ld.a.3 = load i16, i16* %addr.a.3
+  %sext.a.2 = sext i16 %ld.a.2 to i32
+  %sext.a.3 = sext i16 %ld.a.3 to i32
+  %mul.2 = mul i32 %sext.b.0, %sext.a.3
+  %mul.3 = mul i32 %sext.b.1, %sext.a.2
+  %mul.0 = mul i32 %sext.a.0, %sext.b.0
+  %mul.1 = mul i32 %sext.a.1, %sext.b.1
+  %add = add i32 %mul.0, %mul.1
+  %add.1 = add i32 %mul.2, %mul.3
+  %sext.add = sext i32 %add to i64
+  %sext.add.1 = sext i32 %add.1 to i64
+  %add.2 = add i64 %sext.add, %sext.add.1
+  %res = sub i64 %acc, %add.2
+  ret i64 %res
+}
+
+; TODO: Why isn't smladx generated too?
+; CHECK-LABEL: exchange_multi_use_4
+; CHECK: [[CAST_A:%[^ ]+]] = bitcast i16* %a to i32*
+; CHECK: [[LD_A:%[^ ]+]] = load i32, i32* [[CAST_A]]
+; CHECK: [[CAST_B:%[^ ]+]] = bitcast i16* %b to i32*
+; CHECK: [[LD_B:%[^ ]+]] = load i32, i32* [[CAST_B]]
+; CHECK: [[X:%[^ ]+]] = call i32 @llvm.arm.smlad(i32 [[LD_A]], i32 [[LD_B]], i32 0
+; CHECK-NOT: call i32 @llvm.arm.smlad
+define i32 @exchange_multi_use_4(i16* %a, i16* %b, i32 %acc) {
+entry:
+  %addr.a.1 = getelementptr i16, i16* %a, i32 1
+  %addr.b.1 = getelementptr i16, i16* %b, i32 1
+  %ld.a.0 = load i16, i16* %a
+  %sext.a.0 = sext i16 %ld.a.0 to i32
+  %ld.b.0 = load i16, i16* %b
+  %ld.a.1 = load i16, i16* %addr.a.1
+  %ld.b.1 = load i16, i16* %addr.b.1
+  %sext.a.1 = sext i16 %ld.a.1 to i32
+  %sext.b.1 = sext i16 %ld.b.1 to i32
+  %sext.b.0 = sext i16 %ld.b.0 to i32
+  %addr.a.2 = getelementptr i16, i16* %a, i32 2
+  %addr.a.3 = getelementptr i16, i16* %a, i32 3
+  %ld.a.2 = load i16, i16* %addr.a.2
+  %ld.a.3 = load i16, i16* %addr.a.3
+  %sext.a.2 = sext i16 %ld.a.2 to i32
+  %sext.a.3 = sext i16 %ld.a.3 to i32
+  %mul.2 = mul i32 %sext.b.0, %sext.a.3
+  %mul.3 = mul i32 %sext.b.1, %sext.a.2
+  %mul.0 = mul i32 %sext.a.0, %sext.b.0
+  %mul.1 = mul i32 %sext.a.1, %sext.b.1
+  %add.1 = add i32 %mul.2, %mul.3
+  %add = add i32 %mul.0, %mul.1
+  %sub = sub i32 %add, %add.1
+  %res = add i32 %acc, %sub
+  ret i32 %res
+}
+
+; CHECK-LABEL: exchange_swap
+; CHECK: [[CAST_A:%[^ ]+]] = bitcast i16* %a to i32*
+; CHECK: [[LD_A:%[^ ]+]] = load i32, i32* [[CAST_A]]
+; CHECK: [[CAST_B:%[^ ]+]] = bitcast i16* %b to i32*
+; CHECK: [[LD_B:%[^ ]+]] = load i32, i32* [[CAST_B]]
+; CHECK: call i32 @llvm.arm.smladx(i32 [[LD_B]], i32 [[LD_A]]
+define i32 @exchange_swap(i16* %a, i16* %b, i32 %acc) {
+entry:
+  %addr.a.1 = getelementptr i16, i16* %a, i32 1
+  %addr.b.1 = getelementptr i16, i16* %b, i32 1
+  %ld.a.0 = load i16, i16* %a
+  %sext.a.0 = sext i16 %ld.a.0 to i32
+  %ld.b.0 = load i16, i16* %b
+  %ld.a.1 = load i16, i16* %addr.a.1
+  %ld.b.1 = load i16, i16* %addr.b.1
+  %sext.a.1 = sext i16 %ld.a.1 to i32
+  %sext.b.1 = sext i16 %ld.b.1 to i32
+  %sext.b.0 = sext i16 %ld.b.0 to i32
+  %mul.0 = mul i32 %sext.a.1, %sext.b.0
+  %mul.1 = mul i32 %sext.a.0, %sext.b.1
+  %add = add i32 %mul.0, %mul.1
+  %res = add i32 %add, %acc
+  ret i32 %res
+}
+
+; CHECK-LABEL: exchange_swap_2
+; CHECK: [[CAST_A:%[^ ]+]] = bitcast i16* %a to i32*
+; CHECK: [[LD_A:%[^ ]+]] = load i32, i32* [[CAST_A]]
+; CHECK: [[CAST_B:%[^ ]+]] = bitcast i16* %b to i32*
+; CHECK: [[LD_B:%[^ ]+]] = load i32, i32* [[CAST_B]]
+; CHECK: call i32 @llvm.arm.smladx(i32 [[LD_A]], i32 [[LD_B]]
+define i32 @exchange_swap_2(i16* %a, i16* %b, i32 %acc) {
+entry:
+  %addr.a.1 = getelementptr i16, i16* %a, i32 1
+  %addr.b.1 = getelementptr i16, i16* %b, i32 1
+  %ld.a.0 = load i16, i16* %a
+  %sext.a.0 = sext i16 %ld.a.0 to i32
+  %ld.b.0 = load i16, i16* %b
+  %ld.a.1 = load i16, i16* %addr.a.1
+  %ld.b.1 = load i16, i16* %addr.b.1
+  %sext.a.1 = sext i16 %ld.a.1 to i32
+  %sext.b.1 = sext i16 %ld.b.1 to i32
+  %sext.b.0 = sext i16 %ld.b.0 to i32
+  %mul.0 = mul i32 %sext.a.1, %sext.b.0
+  %mul.1 = mul i32 %sext.a.0, %sext.b.1
+  %add = add i32 %mul.1, %mul.0
+  %res = add i32 %add, %acc
+  ret i32 %res
+}
+
+; CHECK-LABEL: exchange_swap_3
+; CHECK: [[CAST_A:%[^ ]+]] = bitcast i16* %a to i32*
+; CHECK: [[LD_A:%[^ ]+]] = load i32, i32* [[CAST_A]]
+; CHECK: [[CAST_B:%[^ ]+]] = bitcast i16* %b to i32*
+; CHECK: [[LD_B:%[^ ]+]] = load i32, i32* [[CAST_B]]
+; CHECK: call i32 @llvm.arm.smladx(i32 [[LD_A]], i32 [[LD_B]]
+define i32 @exchange_swap_3(i16* %a, i16* %b, i32 %acc) {
+entry:
+  %addr.a.1 = getelementptr i16, i16* %a, i32 1
+  %addr.b.1 = getelementptr i16, i16* %b, i32 1
+  %ld.a.0 = load i16, i16* %a
+  %sext.a.0 = sext i16 %ld.a.0 to i32
+  %ld.b.0 = load i16, i16* %b
+  %ld.a.1 = load i16, i16* %addr.a.1
+  %ld.b.1 = load i16, i16* %addr.b.1
+  %sext.a.1 = sext i16 %ld.a.1 to i32
+  %sext.b.1 = sext i16 %ld.b.1 to i32
+  %sext.b.0 = sext i16 %ld.b.0 to i32
+  %mul.0 = mul i32 %sext.b.0, %sext.a.1
+  %mul.1 = mul i32 %sext.b.1, %sext.a.0
+  %add = add i32 %mul.1, %mul.0
+  %res = add i32 %add, %acc
+  ret i32 %res
+}
diff --git a/llvm/test/CodeGen/ARM/ParallelDSP/inner-full-unroll.ll b/llvm/test/CodeGen/ARM/ParallelDSP/inner-full-unroll.ll
index 052fb51a8..a75dd591d 100644
--- a/llvm/test/CodeGen/ARM/ParallelDSP/inner-full-unroll.ll
+++ b/llvm/test/CodeGen/ARM/ParallelDSP/inner-full-unroll.ll
@@ -11,14 +11,14 @@
 ; CHECK: [[BIJ_LD:%[^ ]+]] = load i32, i32* [[BIJ_CAST]], align 2
 ; CHECK: [[CIJ_CAST:%[^ ]+]] = bitcast i16* [[CIJ]] to i32*
 ; CHECK: [[CIJ_LD:%[^ ]+]] = load i32, i32* [[CIJ_CAST]], align 2
+; CHECK: [[SMLAD0:%[^ ]+]] = call i32 @llvm.arm.smlad(i32 [[CIJ_LD]], i32 [[BIJ_LD]], i32 0)
 ; CHECK: [[BIJ_2:%[^ ]+]] = getelementptr inbounds i16, i16* [[BIJ]], i32 2
 ; CHECK: [[BIJ_2_CAST:%[^ ]+]] = bitcast i16* [[BIJ_2]] to i32*
 ; CHECK: [[BIJ_2_LD:%[^ ]+]] = load i32, i32* [[BIJ_2_CAST]], align 2
 ; CHECK: [[CIJ_2:%[^ ]+]] = getelementptr inbounds i16, i16* [[CIJ]], i32 2
 ; CHECK: [[CIJ_2_CAST:%[^ ]+]] = bitcast i16* [[CIJ_2]] to i32*
 ; CHECK: [[CIJ_2_LD:%[^ ]+]] = load i32, i32* [[CIJ_2_CAST]], align 2
-; CHECK: [[SMLAD0:%[^ ]+]] = call i32 @llvm.arm.smlad(i32 [[CIJ_2_LD]], i32 [[BIJ_2_LD]], i32 0)
-; CHECK: [[SMLAD1:%[^ ]+]] = call i32 @llvm.arm.smlad(i32 [[CIJ_LD]], i32 [[BIJ_LD]], i32 [[SMLAD0]])
+; CHECK: [[SMLAD1:%[^ ]+]] = call i32 @llvm.arm.smlad(i32 [[CIJ_2_LD]], i32 [[BIJ_2_LD]], i32 [[SMLAD0]])
 ; CHECK: store i32 [[SMLAD1]], i32* %arrayidx, align 4
 
 define void @full_unroll(i32* noalias nocapture %a, i16** noalias nocapture readonly %b, i16** noalias nocapture readonly %c, i32 %N) {
diff --git a/llvm/test/CodeGen/ARM/ParallelDSP/multi-use-loads.ll b/llvm/test/CodeGen/ARM/ParallelDSP/multi-use-loads.ll
index 40fd39e6e..6949b4a70 100644
--- a/llvm/test/CodeGen/ARM/ParallelDSP/multi-use-loads.ll
+++ b/llvm/test/CodeGen/ARM/ParallelDSP/multi-use-loads.ll
@@ -1,15 +1,68 @@
-; RUN: llc -O3 -mtriple=arm-arm-eabi -mcpu=cortex-m33 < %s | FileCheck %s
-; RUN: llc -O3 -mtriple=armeb-arm-eabi -mcpu=cortex-m33 < %s | FileCheck %s --check-prefix=CHECK-UNSUPPORTED
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -O3 -mtriple=arm-arm-eabi -mcpu=cortex-m33 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-LE
+; RUN: llc -O3 -mtriple=armeb-arm-eabi -mcpu=cortex-m33 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-BE
 
-; CHECK-UNSUPPORTED-NOT: smlad
-
-; CHECK-LABEL: add_user
-; CHECK: %for.body
-; CHECK: ldr [[A:[rl0-9]+]],{{.*}}, #2]!
-; CHECK: ldr [[B:[rl0-9]+]],{{.*}}, #2]!
-; CHECK: sxtah [[COUNT:r[0-9]+]], [[COUNT]], [[A]]
-; CHECK: smlad [[ACC:r[0-9]+]], [[B]], [[A]], [[ACC]]
 define i32 @add_user(i32 %arg, i32* nocapture readnone %arg1, i16* nocapture readonly %arg2, i16* nocapture readonly %arg3) {
+; CHECK-LE-LABEL: add_user:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    .save {r4, lr}
+; CHECK-LE-NEXT:    push {r4, lr}
+; CHECK-LE-NEXT:    cmp r0, #1
+; CHECK-LE-NEXT:    blt .LBB0_4
+; CHECK-LE-NEXT:  @ %bb.1: @ %for.body.preheader
+; CHECK-LE-NEXT:    subs r2, #2
+; CHECK-LE-NEXT:    subs r3, #2
+; CHECK-LE-NEXT:    mov.w r12, #0
+; CHECK-LE-NEXT:    movs r1, #0
+; CHECK-LE-NEXT:    .p2align 2
+; CHECK-LE-NEXT:  .LBB0_2: @ %for.body
+; CHECK-LE-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-LE-NEXT:    ldr lr, [r3, #2]!
+; CHECK-LE-NEXT:    ldr r4, [r2, #2]!
+; CHECK-LE-NEXT:    subs r0, #1
+; CHECK-LE-NEXT:    smlad r12, r4, lr, r12
+; CHECK-LE-NEXT:    sxtah r1, r1, lr
+; CHECK-LE-NEXT:    bne .LBB0_2
+; CHECK-LE-NEXT:  @ %bb.3: @ %for.cond.cleanup
+; CHECK-LE-NEXT:    add.w r0, r12, r1
+; CHECK-LE-NEXT:    pop {r4, pc}
+; CHECK-LE-NEXT:  .LBB0_4:
+; CHECK-LE-NEXT:    mov.w r12, #0
+; CHECK-LE-NEXT:    movs r1, #0
+; CHECK-LE-NEXT:    add.w r0, r12, r1
+; CHECK-LE-NEXT:    pop {r4, pc}
+;
+; CHECK-BE-LABEL: add_user:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    .save {r4, r5, r6, lr}
+; CHECK-BE-NEXT:    push {r4, r5, r6, lr}
+; CHECK-BE-NEXT:    cmp r0, #1
+; CHECK-BE-NEXT:    blt .LBB0_4
+; CHECK-BE-NEXT:  @ %bb.1: @ %for.body.preheader
+; CHECK-BE-NEXT:    subs r2, #2
+; CHECK-BE-NEXT:    subs r3, #2
+; CHECK-BE-NEXT:    mov.w r12, #0
+; CHECK-BE-NEXT:    movs r1, #0
+; CHECK-BE-NEXT:    .p2align 2
+; CHECK-BE-NEXT:  .LBB0_2: @ %for.body
+; CHECK-BE-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-BE-NEXT:    ldrsh lr, [r3, #2]!
+; CHECK-BE-NEXT:    ldrsh r5, [r2, #2]!
+; CHECK-BE-NEXT:    ldrsh.w r4, [r3, #2]
+; CHECK-BE-NEXT:    ldrsh.w r6, [r2, #2]
+; CHECK-BE-NEXT:    smlabb r5, r5, lr, r12
+; CHECK-BE-NEXT:    add r1, lr
+; CHECK-BE-NEXT:    subs r0, #1
+; CHECK-BE-NEXT:    smlabb r12, r6, r4, r5
+; CHECK-BE-NEXT:    bne .LBB0_2
+; CHECK-BE-NEXT:  @ %bb.3: @ %for.cond.cleanup
+; CHECK-BE-NEXT:    add.w r0, r12, r1
+; CHECK-BE-NEXT:    pop {r4, r5, r6, pc}
+; CHECK-BE-NEXT:  .LBB0_4:
+; CHECK-BE-NEXT:    mov.w r12, #0
+; CHECK-BE-NEXT:    movs r1, #0
+; CHECK-BE-NEXT:    add.w r0, r12, r1
+; CHECK-BE-NEXT:    pop {r4, r5, r6, pc}
 entry:
   %cmp24 = icmp sgt i32 %arg, 0
   br i1 %cmp24, label %for.body.preheader, label %for.cond.cleanup
@@ -51,14 +104,68 @@ for.body:
   br i1 %exitcond, label %for.body, label %for.cond.cleanup
 }
 
-; CHECK-LABEL: mul_bottom_user
-; CHECK: %for.body
-; CHECK: ldr [[A:[rl0-9]+]],{{.*}}, #2]!
-; CHECK: ldr [[B:[rl0-9]+]],{{.*}}, #2]!
-; CHECK: sxth [[SXT:r[0-9]+]], [[A]]
-; CHECK: smlad [[ACC:r[0-9]+]], [[B]], [[A]], [[ACC]]
-; CHECK: mul [[COUNT:r[0-9]+]],{{.*}}[[SXT]]
 define i32 @mul_bottom_user(i32 %arg, i32* nocapture readnone %arg1, i16* nocapture readonly %arg2, i16* nocapture readonly %arg3) {
+; CHECK-LE-LABEL: mul_bottom_user:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    .save {r4, r5, r7, lr}
+; CHECK-LE-NEXT:    push {r4, r5, r7, lr}
+; CHECK-LE-NEXT:    cmp r0, #1
+; CHECK-LE-NEXT:    blt .LBB1_4
+; CHECK-LE-NEXT:  @ %bb.1: @ %for.body.preheader
+; CHECK-LE-NEXT:    sub.w lr, r2, #2
+; CHECK-LE-NEXT:    subs r3, #2
+; CHECK-LE-NEXT:    mov.w r12, #0
+; CHECK-LE-NEXT:    movs r1, #0
+; CHECK-LE-NEXT:    .p2align 2
+; CHECK-LE-NEXT:  .LBB1_2: @ %for.body
+; CHECK-LE-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-LE-NEXT:    ldr r2, [r3, #2]!
+; CHECK-LE-NEXT:    ldr r4, [lr, #2]!
+; CHECK-LE-NEXT:    sxth r5, r2
+; CHECK-LE-NEXT:    smlad r12, r4, r2, r12
+; CHECK-LE-NEXT:    subs r0, #1
+; CHECK-LE-NEXT:    mul r1, r5, r1
+; CHECK-LE-NEXT:    bne .LBB1_2
+; CHECK-LE-NEXT:  @ %bb.3: @ %for.cond.cleanup
+; CHECK-LE-NEXT:    add.w r0, r12, r1
+; CHECK-LE-NEXT:    pop {r4, r5, r7, pc}
+; CHECK-LE-NEXT:  .LBB1_4:
+; CHECK-LE-NEXT:    mov.w r12, #0
+; CHECK-LE-NEXT:    movs r1, #0
+; CHECK-LE-NEXT:    add.w r0, r12, r1
+; CHECK-LE-NEXT:    pop {r4, r5, r7, pc}
+;
+; CHECK-BE-LABEL: mul_bottom_user:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    .save {r4, r5, r6, lr}
+; CHECK-BE-NEXT:    push {r4, r5, r6, lr}
+; CHECK-BE-NEXT:    cmp r0, #1
+; CHECK-BE-NEXT:    blt .LBB1_4
+; CHECK-BE-NEXT:  @ %bb.1: @ %for.body.preheader
+; CHECK-BE-NEXT:    subs r2, #2
+; CHECK-BE-NEXT:    subs r3, #2
+; CHECK-BE-NEXT:    mov.w r12, #0
+; CHECK-BE-NEXT:    movs r1, #0
+; CHECK-BE-NEXT:    .p2align 2
+; CHECK-BE-NEXT:  .LBB1_2: @ %for.body
+; CHECK-BE-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-BE-NEXT:    ldrsh lr, [r3, #2]!
+; CHECK-BE-NEXT:    ldrsh r4, [r2, #2]!
+; CHECK-BE-NEXT:    ldrsh.w r5, [r3, #2]
+; CHECK-BE-NEXT:    ldrsh.w r6, [r2, #2]
+; CHECK-BE-NEXT:    smlabb r4, r4, lr, r12
+; CHECK-BE-NEXT:    smlabb r12, r6, r5, r4
+; CHECK-BE-NEXT:    subs r0, #1
+; CHECK-BE-NEXT:    mul r1, lr, r1
+; CHECK-BE-NEXT:    bne .LBB1_2
+; CHECK-BE-NEXT:  @ %bb.3: @ %for.cond.cleanup
+; CHECK-BE-NEXT:    add.w r0, r12, r1
+; CHECK-BE-NEXT:    pop {r4, r5, r6, pc}
+; CHECK-BE-NEXT:  .LBB1_4:
+; CHECK-BE-NEXT:    mov.w r12, #0
+; CHECK-BE-NEXT:    movs r1, #0
+; CHECK-BE-NEXT:    add.w r0, r12, r1
+; CHECK-BE-NEXT:    pop {r4, r5, r6, pc}
 entry:
   %cmp24 = icmp sgt i32 %arg, 0
   br i1 %cmp24, label %for.body.preheader, label %for.cond.cleanup
@@ -100,14 +207,68 @@ for.body:
   br i1 %exitcond, label %for.body, label %for.cond.cleanup
 }
 
-; CHECK-LABEL: mul_top_user
-; CHECK: %for.body
-; CHECK: ldr [[A:[rl0-9]+]],{{.*}}, #2]!
-; CHECK: ldr [[B:[rl0-9]+]],{{.*}}, #2]!
-; CHECK: asrs [[ASR:[rl0-9]+]], [[A]], #16
-; CHECK: smlad [[ACC:[rl0-9]+]], [[A]], [[B]], [[ACC]]
-; CHECK: mul [[COUNT:[rl0-9]+]],{{.}}[[ASR]]
 define i32 @mul_top_user(i32 %arg, i32* nocapture readnone %arg1, i16* nocapture readonly %arg2, i16* nocapture readonly %arg3) {
+; CHECK-LE-LABEL: mul_top_user:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    .save {r4, lr}
+; CHECK-LE-NEXT:    push {r4, lr}
+; CHECK-LE-NEXT:    cmp r0, #1
+; CHECK-LE-NEXT:    blt .LBB2_4
+; CHECK-LE-NEXT:  @ %bb.1: @ %for.body.preheader
+; CHECK-LE-NEXT:    subs r2, #2
+; CHECK-LE-NEXT:    subs r3, #2
+; CHECK-LE-NEXT:    mov.w r12, #0
+; CHECK-LE-NEXT:    movs r1, #0
+; CHECK-LE-NEXT:    .p2align 2
+; CHECK-LE-NEXT:  .LBB2_2: @ %for.body
+; CHECK-LE-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-LE-NEXT:    ldr lr, [r3, #2]!
+; CHECK-LE-NEXT:    ldr r4, [r2, #2]!
+; CHECK-LE-NEXT:    subs r0, #1
+; CHECK-LE-NEXT:    smlad r12, r4, lr, r12
+; CHECK-LE-NEXT:    asr.w r4, r4, #16
+; CHECK-LE-NEXT:    mul r1, r4, r1
+; CHECK-LE-NEXT:    bne .LBB2_2
+; CHECK-LE-NEXT:  @ %bb.3: @ %for.cond.cleanup
+; CHECK-LE-NEXT:    add.w r0, r12, r1
+; CHECK-LE-NEXT:    pop {r4, pc}
+; CHECK-LE-NEXT:  .LBB2_4:
+; CHECK-LE-NEXT:    mov.w r12, #0
+; CHECK-LE-NEXT:    movs r1, #0
+; CHECK-LE-NEXT:    add.w r0, r12, r1
+; CHECK-LE-NEXT:    pop {r4, pc}
+;
+; CHECK-BE-LABEL: mul_top_user:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    .save {r4, r5, r6, lr}
+; CHECK-BE-NEXT:    push {r4, r5, r6, lr}
+; CHECK-BE-NEXT:    cmp r0, #1
+; CHECK-BE-NEXT:    blt .LBB2_4
+; CHECK-BE-NEXT:  @ %bb.1: @ %for.body.preheader
+; CHECK-BE-NEXT:    subs r2, #2
+; CHECK-BE-NEXT:    subs r3, #2
+; CHECK-BE-NEXT:    mov.w r12, #0
+; CHECK-BE-NEXT:    movs r1, #0
+; CHECK-BE-NEXT:    .p2align 2
+; CHECK-BE-NEXT:  .LBB2_2: @ %for.body
+; CHECK-BE-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-BE-NEXT:    ldrsh lr, [r3, #2]!
+; CHECK-BE-NEXT:    ldrsh r4, [r2, #2]!
+; CHECK-BE-NEXT:    ldrsh.w r5, [r3, #2]
+; CHECK-BE-NEXT:    ldrsh.w r6, [r2, #2]
+; CHECK-BE-NEXT:    smlabb r4, r4, lr, r12
+; CHECK-BE-NEXT:    smlabb r12, r6, r5, r4
+; CHECK-BE-NEXT:    subs r0, #1
+; CHECK-BE-NEXT:    mul r1, r6, r1
+; CHECK-BE-NEXT:    bne .LBB2_2
+; CHECK-BE-NEXT:  @ %bb.3: @ %for.cond.cleanup
+; CHECK-BE-NEXT:    add.w r0, r12, r1
+; CHECK-BE-NEXT:    pop {r4, r5, r6, pc}
+; CHECK-BE-NEXT:  .LBB2_4:
+; CHECK-BE-NEXT:    mov.w r12, #0
+; CHECK-BE-NEXT:    movs r1, #0
+; CHECK-BE-NEXT:    add.w r0, r12, r1
+; CHECK-BE-NEXT:    pop {r4, r5, r6, pc}
 entry:
   %cmp24 = icmp sgt i32 %arg, 0
   br i1 %cmp24, label %for.body.preheader, label %for.cond.cleanup
@@ -149,14 +310,69 @@ for.body:
   br i1 %exitcond, label %for.body, label %for.cond.cleanup
 }
 
-; CHECK-LABEL: and_user
-; CHECK: %for.body
-; CHECK: ldr [[A:[rl0-9]+]],{{.*}}, #2]!
-; CHECK: ldr [[B:[rl0-9]+]],{{.*}}, #2]!
-; CHECK: uxth [[UXT:r[0-9]+]], [[A]]
-; CHECK: smlad [[ACC:r[0-9]+]], [[B]], [[A]], [[ACC]]
-; CHECK: mul [[MUL:r[0-9]+]],{{.*}}[[UXT]]
 define i32 @and_user(i32 %arg, i32* nocapture readnone %arg1, i16* nocapture readonly %arg2, i16* nocapture readonly %arg3) {
+; CHECK-LE-LABEL: and_user:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    .save {r4, lr}
+; CHECK-LE-NEXT:    push {r4, lr}
+; CHECK-LE-NEXT:    cmp r0, #1
+; CHECK-LE-NEXT:    blt .LBB3_4
+; CHECK-LE-NEXT:  @ %bb.1: @ %for.body.preheader
+; CHECK-LE-NEXT:    sub.w lr, r2, #2
+; CHECK-LE-NEXT:    subs r3, #2
+; CHECK-LE-NEXT:    mov.w r12, #0
+; CHECK-LE-NEXT:    movs r1, #0
+; CHECK-LE-NEXT:    .p2align 2
+; CHECK-LE-NEXT:  .LBB3_2: @ %for.body
+; CHECK-LE-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-LE-NEXT:    ldr r2, [r3, #2]!
+; CHECK-LE-NEXT:    ldr r4, [lr, #2]!
+; CHECK-LE-NEXT:    subs r0, #1
+; CHECK-LE-NEXT:    smlad r12, r4, r2, r12
+; CHECK-LE-NEXT:    uxth r2, r2
+; CHECK-LE-NEXT:    mul r1, r2, r1
+; CHECK-LE-NEXT:    bne .LBB3_2
+; CHECK-LE-NEXT:  @ %bb.3: @ %for.cond.cleanup
+; CHECK-LE-NEXT:    add.w r0, r12, r1
+; CHECK-LE-NEXT:    pop {r4, pc}
+; CHECK-LE-NEXT:  .LBB3_4:
+; CHECK-LE-NEXT:    mov.w r12, #0
+; CHECK-LE-NEXT:    movs r1, #0
+; CHECK-LE-NEXT:    add.w r0, r12, r1
+; CHECK-LE-NEXT:    pop {r4, pc}
+;
+; CHECK-BE-LABEL: and_user:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    .save {r4, r5, r6, r7, lr}
+; CHECK-BE-NEXT:    push {r4, r5, r6, r7, lr}
+; CHECK-BE-NEXT:    cmp r0, #1
+; CHECK-BE-NEXT:    blt .LBB3_4
+; CHECK-BE-NEXT:  @ %bb.1: @ %for.body.preheader
+; CHECK-BE-NEXT:    subs r2, #2
+; CHECK-BE-NEXT:    subs r3, #2
+; CHECK-BE-NEXT:    mov.w r12, #0
+; CHECK-BE-NEXT:    movs r1, #0
+; CHECK-BE-NEXT:    .p2align 2
+; CHECK-BE-NEXT:  .LBB3_2: @ %for.body
+; CHECK-BE-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-BE-NEXT:    ldrsh lr, [r3, #2]!
+; CHECK-BE-NEXT:    ldrsh r5, [r2, #2]!
+; CHECK-BE-NEXT:    ldrsh.w r4, [r3, #2]
+; CHECK-BE-NEXT:    ldrsh.w r7, [r2, #2]
+; CHECK-BE-NEXT:    uxth.w r6, lr
+; CHECK-BE-NEXT:    smlabb r5, r5, lr, r12
+; CHECK-BE-NEXT:    smlabb r12, r7, r4, r5
+; CHECK-BE-NEXT:    subs r0, #1
+; CHECK-BE-NEXT:    mul r1, r6, r1
+; CHECK-BE-NEXT:    bne .LBB3_2
+; CHECK-BE-NEXT:  @ %bb.3: @ %for.cond.cleanup
+; CHECK-BE-NEXT:    add.w r0, r12, r1
+; CHECK-BE-NEXT:    pop {r4, r5, r6, r7, pc}
+; CHECK-BE-NEXT:  .LBB3_4:
+; CHECK-BE-NEXT:    mov.w r12, #0
+; CHECK-BE-NEXT:    movs r1, #0
+; CHECK-BE-NEXT:    add.w r0, r12, r1
+; CHECK-BE-NEXT:    pop {r4, r5, r6, r7, pc}
 entry:
   %cmp24 = icmp sgt i32 %arg, 0
   br i1 %cmp24, label %for.body.preheader, label %for.cond.cleanup
@@ -199,16 +415,71 @@ for.body:
   br i1 %exitcond, label %for.body, label %for.cond.cleanup
 }
 
-; CHECK-LABEL: multi_uses
-; CHECK: %for.body
-; CHECK: ldr [[A:[rl0-9]+]], [{{.*}}, #2]!
-; CHECK: ldr [[B:[rl0-9]+]], [{{.*}}, #2]!
-; CHECK: sxth [[SXT:r[0-9]+]], [[A]]
-; CHECK: smlad [[ACC:[rl0-9]+]], [[B]], [[A]], [[ACC]]
-; CHECK: eor.w [[EOR:r[0-9]+]], [[SXT]], [[SHIFT:r[0-9]+]]
-; CHECK: muls [[MUL:r[0-9]+]],{{.*}}[[SXT]]
-; CHECK: lsl.w [[SHIFT]], [[MUL]], #16
 define i32 @multi_uses(i32 %arg, i32* nocapture readnone %arg1, i16* nocapture readonly %arg2, i16* nocapture readonly %arg3) {
+; CHECK-LE-LABEL: multi_uses:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    .save {r4, lr}
+; CHECK-LE-NEXT:    push {r4, lr}
+; CHECK-LE-NEXT:    cmp r0, #1
+; CHECK-LE-NEXT:    blt .LBB4_4
+; CHECK-LE-NEXT:  @ %bb.1: @ %for.body.preheader
+; CHECK-LE-NEXT:    subs r2, #2
+; CHECK-LE-NEXT:    subs r3, #2
+; CHECK-LE-NEXT:    mov.w lr, #0
+; CHECK-LE-NEXT:    mov.w r12, #0
+; CHECK-LE-NEXT:    .p2align 2
+; CHECK-LE-NEXT:  .LBB4_2: @ %for.body
+; CHECK-LE-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-LE-NEXT:    ldr r1, [r3, #2]!
+; CHECK-LE-NEXT:    ldr r4, [r2, #2]!
+; CHECK-LE-NEXT:    subs r0, #1
+; CHECK-LE-NEXT:    smlad lr, r4, r1, lr
+; CHECK-LE-NEXT:    eor.w r4, r1, r12
+; CHECK-LE-NEXT:    mul r1, r4, r1
+; CHECK-LE-NEXT:    lsl.w r12, r1, #16
+; CHECK-LE-NEXT:    bne .LBB4_2
+; CHECK-LE-NEXT:  @ %bb.3: @ %for.cond.cleanup
+; CHECK-LE-NEXT:    add.w r0, lr, r12
+; CHECK-LE-NEXT:    pop {r4, pc}
+; CHECK-LE-NEXT:  .LBB4_4:
+; CHECK-LE-NEXT:    mov.w lr, #0
+; CHECK-LE-NEXT:    mov.w r12, #0
+; CHECK-LE-NEXT:    add.w r0, lr, r12
+; CHECK-LE-NEXT:    pop {r4, pc}
+;
+; CHECK-BE-LABEL: multi_uses:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    .save {r4, r5, r6, lr}
+; CHECK-BE-NEXT:    push {r4, r5, r6, lr}
+; CHECK-BE-NEXT:    cmp r0, #1
+; CHECK-BE-NEXT:    blt .LBB4_4
+; CHECK-BE-NEXT:  @ %bb.1: @ %for.body.preheader
+; CHECK-BE-NEXT:    subs r2, #2
+; CHECK-BE-NEXT:    subs r3, #2
+; CHECK-BE-NEXT:    mov.w r12, #0
+; CHECK-BE-NEXT:    mov.w lr, #0
+; CHECK-BE-NEXT:    .p2align 2
+; CHECK-BE-NEXT:  .LBB4_2: @ %for.body
+; CHECK-BE-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-BE-NEXT:    ldrsh r1, [r3, #2]!
+; CHECK-BE-NEXT:    ldrsh r4, [r2, #2]!
+; CHECK-BE-NEXT:    ldrsh.w r5, [r3, #2]
+; CHECK-BE-NEXT:    ldrsh.w r6, [r2, #2]
+; CHECK-BE-NEXT:    smlabb r4, r4, r1, r12
+; CHECK-BE-NEXT:    smlabb r12, r6, r5, r4
+; CHECK-BE-NEXT:    eor.w r6, r1, lr
+; CHECK-BE-NEXT:    muls r1, r6, r1
+; CHECK-BE-NEXT:    subs r0, #1
+; CHECK-BE-NEXT:    lsl.w lr, r1, #16
+; CHECK-BE-NEXT:    bne .LBB4_2
+; CHECK-BE-NEXT:  @ %bb.3: @ %for.cond.cleanup
+; CHECK-BE-NEXT:    add.w r0, r12, lr
+; CHECK-BE-NEXT:    pop {r4, r5, r6, pc}
+; CHECK-BE-NEXT:  .LBB4_4:
+; CHECK-BE-NEXT:    mov.w r12, #0
+; CHECK-BE-NEXT:    mov.w lr, #0
+; CHECK-BE-NEXT:    add.w r0, r12, lr
+; CHECK-BE-NEXT:    pop {r4, r5, r6, pc}
 entry:
   %cmp24 = icmp sgt i32 %arg, 0
   br i1 %cmp24, label %for.body.preheader, label %for.cond.cleanup
diff --git a/llvm/test/CodeGen/ARM/ParallelDSP/overlapping.ll b/llvm/test/CodeGen/ARM/ParallelDSP/overlapping.ll
new file mode 100644
index 000000000..f807149d2
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/ParallelDSP/overlapping.ll
@@ -0,0 +1,220 @@
+; RUN: opt -arm-parallel-dsp -mtriple=armv7-a -S %s -o - | FileCheck %s
+
+; CHECK-LABEL: overlap_1
+; CHECK: [[ADDR_A_1:%[^ ]+]] = getelementptr i16, i16* %a, i32 1
+; CHECK: [[ADDR_B_1:%[^ ]+]] = getelementptr i16, i16* %b, i32 1
+; CHECK: [[CAST_A:%[^ ]+]] = bitcast i16* %a to i32*
+; CHECK: [[LD_A:%[^ ]+]] = load i32, i32* [[CAST_A]]
+; CHECK: [[CAST_B:%[^ ]+]] = bitcast i16* %b to i32*
+; CHECK: [[LD_B:%[^ ]+]] = load i32, i32* [[CAST_B]]
+; CHECK: [[ACC:%[^ ]+]] = call i32 @llvm.arm.smlad(i32 [[LD_A]], i32 [[LD_B]], i32 %acc)
+; CHECK: [[CAST_A_1:%[^ ]+]] = bitcast i16* [[ADDR_A_1]] to i32*
+; CHECK: [[LD_A_1:%[^ ]+]] = load i32, i32* [[CAST_A_1]]
+; CHECK: [[CAST_B_1:%[^ ]+]] = bitcast i16* [[ADDR_B_1]] to i32*
+; CHECK: [[LD_B_1:%[^ ]+]] = load i32, i32* [[CAST_B_1]]
+; CHECK: [[RES:%[^ ]+]] = call i32 @llvm.arm.smlad(i32 [[LD_A_1]], i32 [[LD_B_1]], i32 [[ACC]])
+; CHECK: ret i32 [[RES]]
+define i32 @overlap_1(i16* %a, i16* %b, i32 %acc) {
+entry:
+  %addr.a.1 = getelementptr i16, i16* %a, i32 1
+  %addr.b.1 = getelementptr i16, i16* %b, i32 1
+  %ld.a.0 = load i16, i16* %a
+  %sext.a.0 = sext i16 %ld.a.0 to i32
+  %ld.b.0 = load i16, i16* %b
+  %ld.a.1 = load i16, i16* %addr.a.1
+  %ld.b.1 = load i16, i16* %addr.b.1
+  %sext.a.1 = sext i16 %ld.a.1 to i32
+  %sext.b.1 = sext i16 %ld.b.1 to i32
+  %sext.b.0 = sext i16 %ld.b.0 to i32
+  %mul.0 = mul i32 %sext.a.0, %sext.b.0
+  %mul.1 = mul i32 %sext.a.1, %sext.b.1
+  %addr.a.2 = getelementptr i16, i16* %a, i32 2
+  %addr.b.2 = getelementptr i16, i16* %b, i32 2
+  %ld.a.2 = load i16, i16* %addr.a.2
+  %ld.b.2 = load i16, i16* %addr.b.2
+  %sext.a.2 = sext i16 %ld.a.2 to i32
+  %sext.b.2 = sext i16 %ld.b.2 to i32
+  %mul.2 = mul i32 %sext.a.2, %sext.b.2
+  %add = add i32 %mul.0, %mul.1
+  %add.1 = add i32 %mul.1, %mul.2
+  %add.2 = add i32 %add.1, %add
+  %res = add i32 %add.2, %acc
+  ret i32 %res
+}
+
+; TODO: Is it really best to generate smlald for the first instruction? Does
+; this just increase register pressure unnecessarily?
+; CHECK-LABEL: overlap_64_1
+; CHECK: [[ADDR_A_1:%[^ ]+]] = getelementptr i16, i16* %a, i32 1
+; CHECK: [[ADDR_B_1:%[^ ]+]] = getelementptr i16, i16* %b, i32 1
+; CHECK: [[CAST_A:%[^ ]+]] = bitcast i16* %a to i32*
+; CHECK: [[LD_A:%[^ ]+]] = load i32, i32* [[CAST_A]]
+; CHECK: [[CAST_B:%[^ ]+]] = bitcast i16* %b to i32*
+; CHECK: [[LD_B:%[^ ]+]] = load i32, i32* [[CAST_B]]
+; CHECK: [[ACC:%[^ ]+]] = call i64 @llvm.arm.smlald(i32 [[LD_A]], i32 [[LD_B]], i64 %acc)
+; CHECK: [[CAST_A_1:%[^ ]+]] = bitcast i16* [[ADDR_A_1]] to i32*
+; CHECK: [[LD_A_1:%[^ ]+]] = load i32, i32* [[CAST_A_1]]
+; CHECK: [[CAST_B_1:%[^ ]+]] = bitcast i16* [[ADDR_B_1]] to i32*
+; CHECK: [[LD_B_1:%[^ ]+]] = load i32, i32* [[CAST_B_1]]
+; CHECK: [[RES:%[^ ]+]] = call i64 @llvm.arm.smlald(i32 [[LD_A_1]], i32 [[LD_B_1]], i64 [[ACC]])
+; CHECK: ret i64 [[RES]]
+define i64 @overlap_64_1(i16* %a, i16* %b, i64 %acc) {
+entry:
+  %addr.a.1 = getelementptr i16, i16* %a, i32 1
+  %addr.b.1 = getelementptr i16, i16* %b, i32 1
+  %ld.a.0 = load i16, i16* %a
+  %sext.a.0 = sext i16 %ld.a.0 to i32
+  %ld.b.0 = load i16, i16* %b
+  %ld.a.1 = load i16, i16* %addr.a.1
+  %ld.b.1 = load i16, i16* %addr.b.1
+  %sext.a.1 = sext i16 %ld.a.1 to i32
+  %sext.b.1 = sext i16 %ld.b.1 to i32
+  %sext.b.0 = sext i16 %ld.b.0 to i32
+  %mul.0 = mul i32 %sext.a.0, %sext.b.0
+  %mul.1 = mul i32 %sext.a.1, %sext.b.1
+  %addr.a.2 = getelementptr i16, i16* %a, i32 2
+  %addr.b.2 = getelementptr i16, i16* %b, i32 2
+  %ld.a.2 = load i16, i16* %addr.a.2
+  %ld.b.2 = load i16, i16* %addr.b.2
+  %sext.a.2 = sext i16 %ld.a.2 to i32
+  %sext.b.2 = sext i16 %ld.b.2 to i32
+  %mul.2 = mul i32 %sext.a.2, %sext.b.2
+  %add = add i32 %mul.0, %mul.1
+  %add.1 = add i32 %mul.1, %mul.2
+  %sext.add = sext i32 %add to i64
+  %sext.add.1 = sext i32 %add.1 to i64
+  %add.2 = add i64 %sext.add.1, %sext.add
+  %res = add i64 %add.2, %acc
+  ret i64 %res
+}
+
+; CHECK-LABEL: overlap_2
+; CHECK: [[ADDR_A_1:%[^ ]+]] = getelementptr i16, i16* %a, i32 1
+; CHECK: [[ADDR_B_1:%[^ ]+]] = getelementptr i16, i16* %b, i32 1
+; CHECK: [[CAST_A:%[^ ]+]] = bitcast i16* %a to i32*
+; CHECK: [[LD_A:%[^ ]+]] = load i32, i32* [[CAST_A]]
+; CHECK: [[CAST_B:%[^ ]+]] = bitcast i16* %b to i32*
+; CHECK: [[LD_B:%[^ ]+]] = load i32, i32* [[CAST_B]]
+; CHECK: [[ACC1:%[^ ]+]] = add i32 %mul.1, %acc
+; CHECK: [[ACC2:%[^ ]+]] = add i32 %mul.2, [[ACC1]]
+; CHECK: [[RES:%[^ ]+]] = call i32 @llvm.arm.smlad(i32 [[LD_A]], i32 [[LD_B]], i32 [[ACC2]])
+; CHECK: ret i32 [[RES]]
+define i32 @overlap_2(i16* %a, i16* %b, i32 %acc) {
+entry:
+  %addr.a.1 = getelementptr i16, i16* %a, i32 1
+  %addr.b.1 = getelementptr i16, i16* %b, i32 1
+  %ld.a.0 = load i16, i16* %a
+  %sext.a.0 = sext i16 %ld.a.0 to i32
+  %ld.b.0 = load i16, i16* %b
+  %ld.a.1 = load i16, i16* %addr.a.1
+  %ld.b.1 = load i16, i16* %addr.b.1
+  %sext.a.1 = sext i16 %ld.a.1 to i32
+  %sext.b.1 = sext i16 %ld.b.1 to i32
+  %sext.b.0 = sext i16 %ld.b.0 to i32
+  %mul.0 = mul i32 %sext.a.0, %sext.b.0
+  %mul.1 = mul i32 %sext.a.1, %sext.b.1
+  %addr.a.2 = getelementptr i16, i16* %a, i32 2
+  %addr.b.2 = getelementptr i16, i16* %b, i32 2
+  %ld.a.2 = load i16, i16* %addr.a.2
+  %ld.b.2 = load i16, i16* %addr.b.2
+  %sext.a.2 = sext i16 %ld.a.2 to i32
+  %sext.b.2 = sext i16 %ld.b.2 to i32
+  %mul.2 = mul i32 %sext.b.2, %sext.a.2
+  %add = add i32 %mul.0, %mul.1
+  %add.1 = add i32 %mul.1, %mul.2
+  %add.2 = add i32 %add, %add.1
+  %res = add i32 %add.2, %acc
+  ret i32 %res
+}
+
+; CHECK-LABEL: overlap_3
+; CHECK: [[GEP_B:%[^ ]+]] = getelementptr i16, i16* %b, i32 1
+; CHECK: [[CAST_A:%[^ ]+]] = bitcast i16* %a to i32*
+; CHECK: [[LD_A:%[^ ]+]] = load i32, i32* [[CAST_A]]
+; CHECK: [[CAST_B:%[^ ]+]] = bitcast i16* %b to i32*
+; CHECK: [[LD_B:%[^ ]+]] = load i32, i32* [[CAST_B]]
+; CHECK: [[SMLAD:%[^ ]+]] = call i32 @llvm.arm.smlad(i32 [[LD_A]], i32 [[LD_B]], i32 %acc)
+; CHECK: [[CAST_B_1:%[^ ]+]] = bitcast i16* [[GEP_B]] to i32*
+; CHECK: [[LD_B_1:%[^ ]+]] = load i32, i32* [[CAST_B_1]]
+; CHECK: [[GEP_A:%[^ ]+]] = getelementptr i16, i16* %a, i32 2
+; CHECK: [[CAST_A_2:%[^ ]+]] = bitcast i16* [[GEP_A]] to i32*
+; CHECK: [[LD_A_2:%[^ ]+]] = load i32, i32* [[CAST_A_2]]
+; CHECK: [[RES:%[^ ]+]] = call i32 @llvm.arm.smlad(i32 [[LD_A_2]], i32 [[LD_B_1]], i32 [[SMLAD]])
+; CHECK: ret i32 [[RES]]
+define i32 @overlap_3(i16* %a, i16* %b, i32 %acc) {
+entry:
+  %addr.a.1 = getelementptr i16, i16* %a, i32 1
+  %addr.b.1 = getelementptr i16, i16* %b, i32 1
+  %ld.a.0 = load i16, i16* %a
+  %sext.a.0 = sext i16 %ld.a.0 to i32
+  %ld.b.0 = load i16, i16* %b
+  %ld.a.1 = load i16, i16* %addr.a.1
+  %ld.b.1 = load i16, i16* %addr.b.1
+  %sext.a.1 = sext i16 %ld.a.1 to i32
+  %sext.b.1 = sext i16 %ld.b.1 to i32
+  %sext.b.0 = sext i16 %ld.b.0 to i32
+  %mul.0 = mul i32 %sext.a.0, %sext.b.0
+  %mul.1 = mul i32 %sext.a.1, %sext.b.1
+  %addr.a.2 = getelementptr i16, i16* %a, i32 2
+  %addr.b.2 = getelementptr i16, i16* %b, i32 2
+  %addr.a.3 = getelementptr i16, i16* %a, i32 3
+  %ld.a.2 = load i16, i16* %addr.a.2
+  %ld.b.2 = load i16, i16* %addr.b.2
+  %ld.a.3 = load i16, i16* %addr.a.3
+  %sext.a.2 = sext i16 %ld.a.2 to i32
+  %sext.b.2 = sext i16 %ld.b.2 to i32
+  %sext.a.3 = sext i16 %ld.a.3 to i32
+  %mul.2 = mul i32 %sext.a.2, %sext.b.1
+  %mul.3 = mul i32 %sext.a.3, %sext.b.2
+  %add = add i32 %mul.0, %mul.1
+  %add.1 = add i32 %mul.2, %mul.3
+  %add.2 = add i32 %add.1, %add
+  %res = add i32 %add.2, %acc
+  ret i32 %res
+}
+
+; CHECK-LABEL: overlap_4
+; CHECK: [[GEP_B:%[^ ]+]] = getelementptr i16, i16* %b, i32 1
+; CHECK: [[CAST_A:%[^ ]+]] = bitcast i16* %a to i32*
+; CHECK: [[LD_A:%[^ ]+]] = load i32, i32* [[CAST_A]]
+; CHECK: [[CAST_B:%[^ ]+]] = bitcast i16* %b to i32*
+; CHECK: [[LD_B:%[^ ]+]] = load i32, i32* [[CAST_B]]
+; CHECK: [[SMLAD:%[^ ]+]] = call i32 @llvm.arm.smlad(i32 [[LD_A]], i32 [[LD_B]], i32 %acc)
+; CHECK: [[CAST_B_1:%[^ ]+]] = bitcast i16* [[GEP_B]] to i32*
+; CHECK: [[LD_B_1:%[^ ]+]] = load i32, i32* [[CAST_B_1]]
+; CHECK: [[GEP_A:%[^ ]+]] = getelementptr i16, i16* %a, i32 2
+; CHECK: [[CAST_A_2:%[^ ]+]] = bitcast i16* [[GEP_A]] to i32*
+; CHECK: [[LD_A_2:%[^ ]+]] = load i32, i32* [[CAST_A_2]]
+; CHECK: [[RES:%[^ ]+]] = call i32 @llvm.arm.smladx(i32 [[LD_A_2]], i32 [[LD_B_1]], i32 [[SMLAD]])
+; CHECK: ret i32 [[RES]]
+define i32 @overlap_4(i16* %a, i16* %b, i32 %acc) {
+entry:
+  %addr.a.1 = getelementptr i16, i16* %a, i32 1
+  %addr.b.1 = getelementptr i16, i16* %b, i32 1
+  %ld.a.0 = load i16, i16* %a
+  %sext.a.0 = sext i16 %ld.a.0 to i32
+  %ld.b.0 = load i16, i16* %b
+  %ld.a.1 = load i16, i16* %addr.a.1
+  %ld.b.1 = load i16, i16* %addr.b.1
+  %sext.a.1 = sext i16 %ld.a.1 to i32
+  %sext.b.1 = sext i16 %ld.b.1 to i32
+  %sext.b.0 = sext i16 %ld.b.0 to i32
+  %mul.0 = mul i32 %sext.a.0, %sext.b.0
+  %mul.1 = mul i32 %sext.a.1, %sext.b.1
+  %addr.a.2 = getelementptr i16, i16* %a, i32 2
+  %addr.b.2 = getelementptr i16, i16* %b, i32 2
+  %addr.a.3 = getelementptr i16, i16* %a, i32 3
+  %ld.a.2 = load i16, i16* %addr.a.2
+  %ld.b.2 = load i16, i16* %addr.b.2
+  %ld.a.3 = load i16, i16* %addr.a.3
+  %sext.a.2 = sext i16 %ld.a.2 to i32
+  %sext.b.2 = sext i16 %ld.b.2 to i32
+  %sext.a.3 = sext i16 %ld.a.3 to i32
+  %mul.2 = mul i32 %sext.b.2, %sext.a.2
+  %mul.3 = mul i32 %sext.b.1, %sext.a.3
+  %add = add i32 %mul.0, %mul.1
+  %add.1 = add i32 %mul.2, %mul.3
+  %add.2 = add i32 %add.1, %add
+  %res = add i32 %add.2, %acc
+  ret i32 %res
+}
diff --git a/llvm/test/CodeGen/ARM/ParallelDSP/pr42729.ll b/llvm/test/CodeGen/ARM/ParallelDSP/pr42729.ll
new file mode 100644
index 000000000..9f032cd24
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/ParallelDSP/pr42729.ll
@@ -0,0 +1,84 @@
+; RUN: opt -mtriple=thumbv7-unknown-linux-android -arm-parallel-dsp -S %s -o - | FileCheck %s
+
+; CHECK-LABEL: undef_no_return
+; CHECK: [[CAST_A:%[^ ]+]] = bitcast i16* %incdec.ptr21 to i32*
+; CHECK: [[LOAD_A:%[^ ]+]] = load i32, i32* [[CAST_A]], align 2
+; CHECK: %uglygep15 = getelementptr i8, i8* undef, i32 undef
+; CHECK: [[GEP8:%[^ ]+]] = getelementptr i8, i8* undef, i32 undef
+; CHECK: [[CAST_GEP8:%[^ ]+]] = bitcast i8* [[GEP8]] to i16*
+; CHECK: [[GEP16:%[^ ]+]] = getelementptr i16, i16* [[CAST_GEP8]], i32 6
+; CHECK: [[CAST_GEP16:%[^ ]+]] = bitcast i16* [[GEP16]] to i32*
+; CHECK: [[LOAD_UNDEF:%[^ ]+]] = load i32, i32* [[CAST_GEP16]], align 2
+; CHECK: call i32 @llvm.arm.smladx(i32 [[LOAD_UNDEF]], i32 [[LOAD_A]], i32 undef)
+define void @undef_no_return(i16* %a) {
+entry:
+  %incdec.ptr21 = getelementptr inbounds i16, i16* %a, i32 3
+  %incdec.ptr29 = getelementptr inbounds i16, i16* %a, i32 4
+  br label %for.body
+
+for.body:
+  %0 = load i16, i16* %incdec.ptr21, align 2
+  %conv25 = sext i16 %0 to i32
+  %uglygep15 = getelementptr i8, i8* undef, i32 undef
+  %uglygep1516 = bitcast i8* %uglygep15 to i16*
+  %scevgep17 = getelementptr i16, i16* %uglygep1516, i32 7
+  %1 = load i16, i16* %scevgep17, align 2
+  %conv31 = sext i16 %1 to i32
+  %2 = load i16, i16* %incdec.ptr29, align 2
+  %conv33 = sext i16 %2 to i32
+  %uglygep12 = getelementptr i8, i8* undef, i32 undef
+  %uglygep1213 = bitcast i8* %uglygep12 to i16*
+  %scevgep14 = getelementptr i16, i16* %uglygep1213, i32 6
+  %3 = load i16, i16* %scevgep14, align 2
+  %conv39 = sext i16 %3 to i32
+  %mul.i287.neg.neg = mul nsw i32 %conv31, %conv25
+  %mul.i283.neg.neg = mul nsw i32 %conv39, %conv33
+  %reass.add408 = add i32 undef, %mul.i287.neg.neg
+  %reass.add409 = add i32 %reass.add408, %mul.i283.neg.neg
+  br label %for.body
+}
+
+; CHECK-LABEL: return
+; CHECK: phi i32 [ %N, %entry ]
+; CHECK: [[ACC:%[^ ]+]] = phi i32 [ 0, %entry ], [ [[ACC_NEXT:%[^ ]+]], %for.body ]
+; CHECK: [[CAST_A:%[^ ]+]] = bitcast i16* %incdec.ptr21 to i32*
+; CHECK: [[LOAD_A:%[^ ]+]] = load i32, i32* [[CAST_A]], align 2
+; CHECK: [[GEP8:%[^ ]+]] = getelementptr i8, i8* %b, i32 0
+; CHECK: [[CAST_GEP8:%[^ ]+]] = bitcast i8* [[GEP8]] to i16*
+; CHECK: [[GEP16:%[^ ]+]] = getelementptr i16, i16* [[CAST_GEP8]], i32 %iv
+; CHECK: [[CAST_GEP16:%[^ ]+]] = bitcast i16* [[GEP16]] to i32*
+; CHECK: [[LOAD_B:%[^ ]+]] = load i32, i32* [[CAST_GEP16]], align 2
+; CHECK: [[ACC_NEXT]] = call i32 @llvm.arm.smladx(i32 [[LOAD_B]], i32 [[LOAD_A]], i32 [[ACC]])
+define i32 @return(i16* %a, i8* %b, i32 %N) {
+entry:
+  %incdec.ptr21 = getelementptr inbounds i16, i16* %a, i32 3
+  %incdec.ptr29 = getelementptr inbounds i16, i16* %a, i32 4
+  br label %for.body
+
+for.body:
+  %iv = phi i32 [ %N, %entry ], [ %iv.next, %for.body ]
+  %acc = phi i32 [ 0, %entry ], [ %reass.add409, %for.body ]
+  %0 = load i16, i16* %incdec.ptr21, align 2
+  %conv25 = sext i16 %0 to i32
+  %uglygep15 = getelementptr i8, i8* %b, i32 0
+  %uglygep1516 = bitcast i8* %uglygep15 to i16*
+  %b.idx = add nuw nsw i32 %iv, 1
+  %scevgep17 = getelementptr i16, i16* %uglygep1516, i32 %b.idx
+  %scevgep14 = getelementptr i16, i16* %uglygep1516, i32 %iv
+  %1 = load i16, i16* %scevgep17, align 2
+  %conv31 = sext i16 %1 to i32
+  %2 = load i16, i16* %incdec.ptr29, align 2
+  %conv33 = sext i16 %2 to i32
+  %3 = load i16, i16* %scevgep14, align 2
+  %conv39 = sext i16 %3 to i32
+  %mul.i287.neg.neg = mul nsw i32 %conv31, %conv25
+  %mul.i283.neg.neg = mul nsw i32 %conv39, %conv33
+  %reass.add408 = add i32 %acc, %mul.i287.neg.neg
+  %reass.add409 = add i32 %reass.add408, %mul.i283.neg.neg
+  %iv.next = add nuw nsw i32 %iv, -1
+  %cmp = icmp ne i32 %iv.next, 0
+  br i1 %cmp, label %for.body, label %exit
+
+exit:
+  ret i32 %reass.add409
+}
diff --git a/llvm/test/CodeGen/ARM/ParallelDSP/pr43073.ll b/llvm/test/CodeGen/ARM/ParallelDSP/pr43073.ll
new file mode 100644
index 000000000..7620b64f2
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/ParallelDSP/pr43073.ll
@@ -0,0 +1,294 @@
+; RUN: opt -mtriple=thumbv7-unknown-linux-gnueabihf -arm-parallel-dsp -dce %s -S -o - | FileCheck %s
+
+; CHECK-LABEL: first_mul_invalid
+; CHECK: [[ADDR_IN_MINUS_1:%[^ ]+]] = getelementptr inbounds i16, i16* %in, i32 -1
+; CHECK: [[LD_IN_MINUS_1:%[^ ]+]] = load i16, i16* [[ADDR_IN_MINUS_1]], align 2
+; CHECK: [[IN_MINUS_1:%[^ ]+]] = sext i16 [[LD_IN_MINUS_1]] to i32
+; CHECK: [[ADDR_B_PLUS_1:%[^ ]+]] = getelementptr inbounds i16, i16* %b, i32 1
+; CHECK: [[LD_B_PLUS_1:%[^ ]+]] = load i16, i16* [[ADDR_B_PLUS_1]], align 2
+; CHECK: [[B_PLUS_1:%[^ ]+]] = sext i16 [[LD_B_PLUS_1]] to i32
+; CHECK: [[MUL0:%[^ ]+]] = mul nsw i32 [[B_PLUS_1]], [[IN_MINUS_1]]
+; CHECK: [[ADD0:%[^ ]+]] = add i32 [[MUL0]], %call
+; CHECK: [[ADDR_IN_MINUS_3:%[^ ]+]] = getelementptr inbounds i16, i16* %in, i32 -3
+; CHECK: [[CAST_ADDR_IN_MINUS_3:%[^ ]+]] = bitcast i16* [[ADDR_IN_MINUS_3]] to i32*
+; CHECK: [[IN_MINUS_3:%[^ ]+]] = load i32, i32* [[CAST_ADDR_IN_MINUS_3]], align 2
+; CHECK: [[ADDR_B_PLUS_2:%[^ ]+]] = getelementptr inbounds i16, i16* %b, i32 2
+; CHECK: [[CAST_ADDR_B_PLUS_2:%[^ ]+]] = bitcast i16* [[ADDR_B_PLUS_2]] to i32*
+; CHECK: [[B_PLUS_2:%[^ ]+]] = load i32, i32* [[CAST_ADDR_B_PLUS_2]], align 2
+; CHECK: [[ACC:%[^ ]+]] = call i32 @llvm.arm.smladx(i32 [[IN_MINUS_3]], i32 [[B_PLUS_2]], i32 [[ADD0]])
+; CHECK: [[ADDR_IN_MINUS_5:%[^ ]+]] = getelementptr inbounds i16, i16* %in, i32 -5
+; CHECK: [[CAST_ADDR_IN_MINUS_5:%[^ ]+]] = bitcast i16* [[ADDR_IN_MINUS_5]] to i32*
+; CHECK: [[IN_MINUS_5:%[^ ]+]] = load i32, i32* [[CAST_ADDR_IN_MINUS_5]], align 2
+; CHECK: [[ADDR_B_PLUS_4:%[^ ]+]] = getelementptr inbounds i16, i16* %b, i32 4
+; CHECK: [[CAST_ADDR_B_PLUS_4:%[^ ]+]] = bitcast i16* [[ADDR_B_PLUS_4]] to i32*
+; CHECK: [[B_PLUS_4:%[^ ]+]] = load i32, i32* [[CAST_ADDR_B_PLUS_4]], align 2
+; CHECK: [[RES:%[^ ]+]] = call i32 @llvm.arm.smladx(i32 [[IN_MINUS_5]], i32 [[B_PLUS_4]], i32 [[ACC]])
+; CHECK: ret i32 [[RES]]
+define i32 @first_mul_invalid(i16* nocapture readonly %in, i16* nocapture readonly %b) {
+entry:
+  %0 = load i16, i16* %in, align 2
+  %conv = sext i16 %0 to i32
+  %1 = load i16, i16* %b, align 2
+  %conv2 = sext i16 %1 to i32
+  %call = tail call i32 @bar(i32 %conv, i32 %conv2)
+  %arrayidx3 = getelementptr inbounds i16, i16* %in, i32 -1
+  %2 = load i16, i16* %arrayidx3, align 2
+  %conv4 = sext i16 %2 to i32
+  %arrayidx5 = getelementptr inbounds i16, i16* %b, i32 1
+  %3 = load i16, i16* %arrayidx5, align 2
+  %conv6 = sext i16 %3 to i32
+  %mul = mul nsw i32 %conv6, %conv4
+  %add = add i32 %mul, %call
+  %arrayidx7 = getelementptr inbounds i16, i16* %in, i32 -2
+  %4 = load i16, i16* %arrayidx7, align 2
+  %conv8 = sext i16 %4 to i32
+  %arrayidx9 = getelementptr inbounds i16, i16* %b, i32 2
+  %5 = load i16, i16* %arrayidx9, align 2
+  %conv10 = sext i16 %5 to i32
+  %mul11 = mul nsw i32 %conv10, %conv8
+  %add12 = add i32 %add, %mul11
+  %arrayidx13 = getelementptr inbounds i16, i16* %in, i32 -3
+  %6 = load i16, i16* %arrayidx13, align 2
+  %conv14 = sext i16 %6 to i32
+  %arrayidx15 = getelementptr inbounds i16, i16* %b, i32 3
+  %7 = load i16, i16* %arrayidx15, align 2
+  %conv16 = sext i16 %7 to i32
+  %mul17 = mul nsw i32 %conv16, %conv14
+  %add18 = add i32 %add12, %mul17
+  %arrayidx19 = getelementptr inbounds i16, i16* %in, i32 -4
+  %8 = load i16, i16* %arrayidx19, align 2
+  %conv20 = sext i16 %8 to i32
+  %arrayidx21 = getelementptr inbounds i16, i16* %b, i32 4
+  %9 = load i16, i16* %arrayidx21, align 2
+  %conv22 = sext i16 %9 to i32
+  %mul23 = mul nsw i32 %conv22, %conv20
+  %add24 = add i32 %add18, %mul23
+  %arrayidx25 = getelementptr inbounds i16, i16* %in, i32 -5
+  %10 = load i16, i16* %arrayidx25, align 2
+  %conv26 = sext i16 %10 to i32
+  %arrayidx27 = getelementptr inbounds i16, i16* %b, i32 5
+  %11 = load i16, i16* %arrayidx27, align 2
+  %conv28 = sext i16 %11 to i32
+  %mul29 = mul nsw i32 %conv28, %conv26
+  %add30 = add i32 %add24, %mul29
+  ret i32 %add30
+}
+
+; CHECK-LABEL: with_no_acc_input
+; CHECK: [[ADDR_IN_MINUS_1:%[^ ]+]] = getelementptr inbounds i16, i16* %in, i32 -1
+; CHECK: [[LD_IN_MINUS_1:%[^ ]+]] = load i16, i16* [[ADDR_IN_MINUS_1]], align 2
+; CHECK: [[IN_MINUS_1:%[^ ]+]] = sext i16 [[LD_IN_MINUS_1]] to i32
+; CHECK: [[ADDR_B_PLUS_1:%[^ ]+]] = getelementptr inbounds i16, i16* %b, i32 1
+; CHECK: [[LD_B_PLUS_1:%[^ ]+]] = load i16, i16* [[ADDR_B_PLUS_1]], align 2
+; CHECK: [[B_PLUS_1:%[^ ]+]] = sext i16 [[LD_B_PLUS_1]] to i32
+; CHECK: [[MUL0:%[^ ]+]] = mul nsw i32 [[B_PLUS_1]], [[IN_MINUS_1]]
+; CHECK: [[ADDR_IN_MINUS_3:%[^ ]+]] = getelementptr inbounds i16, i16* %in, i32 -3
+; CHECK: [[CAST_ADDR_IN_MINUS_3:%[^ ]+]] = bitcast i16* [[ADDR_IN_MINUS_3]] to i32*
+; CHECK: [[IN_MINUS_3:%[^ ]+]] = load i32, i32* [[CAST_ADDR_IN_MINUS_3]], align 2
+; CHECK: [[ADDR_B_PLUS_2:%[^ ]+]] = getelementptr inbounds i16, i16* %b, i32 2
+; CHECK: [[CAST_ADDR_B_PLUS_2:%[^ ]+]] = bitcast i16* [[ADDR_B_PLUS_2]] to i32*
+; CHECK: [[B_PLUS_2:%[^ ]+]] = load i32, i32* [[CAST_ADDR_B_PLUS_2]], align 2
+; CHECK: [[ACC:%[^ ]+]] = call i32 @llvm.arm.smladx(i32 [[IN_MINUS_3]], i32 [[B_PLUS_2]], i32 [[MUL0]])
+; CHECK: [[ADDR_IN_MINUS_5:%[^ ]+]] = getelementptr inbounds i16, i16* %in, i32 -5
+; CHECK: [[CAST_ADDR_IN_MINUS_5:%[^ ]+]] = bitcast i16* [[ADDR_IN_MINUS_5]] to i32*
+; CHECK: [[IN_MINUS_5:%[^ ]+]] = load i32, i32* [[CAST_ADDR_IN_MINUS_5]], align 2
+; CHECK: [[ADDR_B_PLUS_4:%[^ ]+]] = getelementptr inbounds i16, i16* %b, i32 4
+; CHECK: [[CAST_ADDR_B_PLUS_4:%[^ ]+]] = bitcast i16* [[ADDR_B_PLUS_4]] to i32*
+; CHECK: [[B_PLUS_4:%[^ ]+]] = load i32, i32* [[CAST_ADDR_B_PLUS_4]], align 2
+; CHECK: [[RES:%[^ ]+]] = call i32 @llvm.arm.smladx(i32 [[IN_MINUS_5]], i32 [[B_PLUS_4]], i32 [[ACC]])
+; CHECK: ret i32 [[RES]]
+define i32 @with_no_acc_input(i16* nocapture readonly %in, i16* nocapture readonly %b) {
+entry:
+  %arrayidx3 = getelementptr inbounds i16, i16* %in, i32 -1
+  %ld.2 = load i16, i16* %arrayidx3, align 2
+  %conv4 = sext i16 %ld.2 to i32
+  %arrayidx5 = getelementptr inbounds i16, i16* %b, i32 1
+  %ld.3 = load i16, i16* %arrayidx5, align 2
+  %conv6 = sext i16 %ld.3 to i32
+  %mul = mul nsw i32 %conv6, %conv4
+  %arrayidx7 = getelementptr inbounds i16, i16* %in, i32 -2
+  %ld.4 = load i16, i16* %arrayidx7, align 2
+  %conv8 = sext i16 %ld.4 to i32
+  %arrayidx9 = getelementptr inbounds i16, i16* %b, i32 2
+  %ld.5 = load i16, i16* %arrayidx9, align 2
+  %conv10 = sext i16 %ld.5 to i32
+  %mul11 = mul nsw i32 %conv10, %conv8
+  %add12 = add i32 %mul, %mul11
+  %arrayidx13 = getelementptr inbounds i16, i16* %in, i32 -3
+  %ld.6 = load i16, i16* %arrayidx13, align 2
+  %conv14 = sext i16 %ld.6 to i32
+  %arrayidx15 = getelementptr inbounds i16, i16* %b, i32 3
+  %ld.7 = load i16, i16* %arrayidx15, align 2
+  %conv16 = sext i16 %ld.7 to i32
+  %mul17 = mul nsw i32 %conv16, %conv14
+  %add18 = add i32 %add12, %mul17
+  %arrayidx19 = getelementptr inbounds i16, i16* %in, i32 -4
+  %ld.8 = load i16, i16* %arrayidx19, align 2
+  %conv20 = sext i16 %ld.8 to i32
+  %arrayidx21 = getelementptr inbounds i16, i16* %b, i32 4
+  %ld.9 = load i16, i16* %arrayidx21, align 2
+  %conv22 = sext i16 %ld.9 to i32
+  %mul23 = mul nsw i32 %conv22, %conv20
+  %add24 = add i32 %add18, %mul23
+  %arrayidx25 = getelementptr inbounds i16, i16* %in, i32 -5
+  %ld.10 = load i16, i16* %arrayidx25, align 2
+  %conv26 = sext i16 %ld.10 to i32
+  %arrayidx27 = getelementptr inbounds i16, i16* %b, i32 5
+  %ld.11 = load i16, i16* %arrayidx27, align 2
+  %conv28 = sext i16 %ld.11 to i32
+  %mul29 = mul nsw i32 %conv28, %conv26
+  %add30 = add i32 %add24, %mul29
+  ret i32 %add30
+}
+
+; CHECK-LABEL: with_64bit_acc
+; CHECK: [[ADDR_IN_MINUS_1:%[^ ]+]] = getelementptr inbounds i16, i16* %in, i32 -1
+; CHECK: [[LD_IN_MINUS_1:%[^ ]+]] = load i16, i16* [[ADDR_IN_MINUS_1]], align 2
+; CHECK: [[IN_MINUS_1:%[^ ]+]] = sext i16 [[LD_IN_MINUS_1]] to i32
+; CHECK: [[ADDR_B_PLUS_1:%[^ ]+]] = getelementptr inbounds i16, i16* %b, i32 1
+; CHECK: [[LD_B_PLUS_1:%[^ ]+]] = load i16, i16* [[ADDR_B_PLUS_1]], align 2
+; CHECK: [[B_PLUS_1:%[^ ]+]] = sext i16 [[LD_B_PLUS_1]] to i32
+; CHECK: [[MUL0:%[^ ]+]] = mul nsw i32 [[B_PLUS_1]], [[IN_MINUS_1]]
+; CHECK: [[SEXT1:%[^ ]+]] = sext i32 [[MUL0]] to i64
+; CHECK: [[ADD0:%[^ ]+]] = add i64 %sext.0, [[SEXT1]]
+; CHECK: [[ADDR_IN_MINUS_3:%[^ ]+]] = getelementptr inbounds i16, i16* %in, i32 -3
+; CHECK: [[CAST_ADDR_IN_MINUS_3:%[^ ]+]] = bitcast i16* [[ADDR_IN_MINUS_3]] to i32*
+; CHECK: [[IN_MINUS_3:%[^ ]+]] = load i32, i32* [[CAST_ADDR_IN_MINUS_3]], align 2
+; CHECK: [[ADDR_B_PLUS_2:%[^ ]+]] = getelementptr inbounds i16, i16* %b, i32 2
+; CHECK: [[CAST_ADDR_B_PLUS_2:%[^ ]+]] = bitcast i16* [[ADDR_B_PLUS_2]] to i32*
+; CHECK: [[B_PLUS_2:%[^ ]+]] = load i32, i32* [[CAST_ADDR_B_PLUS_2]], align 2
+; CHECK: [[ACC:%[^ ]+]] = call i64 @llvm.arm.smlaldx(i32 [[IN_MINUS_3]], i32 [[B_PLUS_2]], i64 [[ADD0]])
+; CHECK: [[ADDR_IN_MINUS_5:%[^ ]+]] = getelementptr inbounds i16, i16* %in, i32 -5
+; CHECK: [[CAST_ADDR_IN_MINUS_5:%[^ ]+]] = bitcast i16* [[ADDR_IN_MINUS_5]] to i32*
+; CHECK: [[IN_MINUS_5:%[^ ]+]] = load i32, i32* [[CAST_ADDR_IN_MINUS_5]], align 2
+; CHECK: [[ADDR_B_PLUS_4:%[^ ]+]] = getelementptr inbounds i16, i16* %b, i32 4
+; CHECK: [[CAST_ADDR_B_PLUS_4:%[^ ]+]] = bitcast i16* [[ADDR_B_PLUS_4]] to i32*
+; CHECK: [[B_PLUS_4:%[^ ]+]] = load i32, i32* [[CAST_ADDR_B_PLUS_4]], align 2
+; CHECK: [[RES:%[^ ]+]] = call i64 @llvm.arm.smlaldx(i32 [[IN_MINUS_5]], i32 [[B_PLUS_4]], i64 [[ACC]])
+; CHECK: ret i64 [[RES]]
+define i64 @with_64bit_acc(i16* nocapture readonly %in, i16* nocapture readonly %b) {
+entry:
+  %0 = load i16, i16* %in, align 2
+  %conv = sext i16 %0 to i32
+  %1 = load i16, i16* %b, align 2
+  %conv2 = sext i16 %1 to i32
+  %call = tail call i32 @bar(i32 %conv, i32 %conv2)
+  %sext.0 = sext i32 %call to i64
+  %arrayidx3 = getelementptr inbounds i16, i16* %in, i32 -1
+  %2 = load i16, i16* %arrayidx3, align 2
+  %conv4 = sext i16 %2 to i32
+  %arrayidx5 = getelementptr inbounds i16, i16* %b, i32 1
+  %3 = load i16, i16* %arrayidx5, align 2
+  %conv6 = sext i16 %3 to i32
+  %mul = mul nsw i32 %conv6, %conv4
+  %sext.1 = sext i32 %mul to i64
+  %add = add i64 %sext.0, %sext.1
+  %arrayidx7 = getelementptr inbounds i16, i16* %in, i32 -2
+  %4 = load i16, i16* %arrayidx7, align 2
+  %conv8 = sext i16 %4 to i32
+  %arrayidx9 = getelementptr inbounds i16, i16* %b, i32 2
+  %5 = load i16, i16* %arrayidx9, align 2
+  %conv10 = sext i16 %5 to i32
+  %mul11 = mul nsw i32 %conv10, %conv8
+  %sext.2 = sext i32 %mul11 to i64
+  %add12 = add i64 %add, %sext.2
+  %arrayidx13 = getelementptr inbounds i16, i16* %in, i32 -3
+  %6 = load i16, i16* %arrayidx13, align 2
+  %conv14 = sext i16 %6 to i32
+  %arrayidx15 = getelementptr inbounds i16, i16* %b, i32 3
+  %7 = load i16, i16* %arrayidx15, align 2
+  %conv16 = sext i16 %7 to i32
+  %mul17 = mul nsw i32 %conv16, %conv14
+  %sext.3 = sext i32 %mul17 to i64
+  %add18 = add i64 %add12, %sext.3
+  %arrayidx19 = getelementptr inbounds i16, i16* %in, i32 -4
+  %8 = load i16, i16* %arrayidx19, align 2
+  %conv20 = sext i16 %8 to i32
+  %arrayidx21 = getelementptr inbounds i16, i16* %b, i32 4
+  %9 = load i16, i16* %arrayidx21, align 2
+  %conv22 = sext i16 %9 to i32
+  %mul23 = mul nsw i32 %conv22, %conv20
+  %sext.4 = sext i32 %mul23 to i64
+  %add24 = add i64 %add18, %sext.4
+  %arrayidx25 = getelementptr inbounds i16, i16* %in, i32 -5
+  %10 = load i16, i16* %arrayidx25, align 2
+  %conv26 = sext i16 %10 to i32
+  %arrayidx27 = getelementptr inbounds i16, i16* %b, i32 5
+  %11 = load i16, i16* %arrayidx27, align 2
+  %conv28 = sext i16 %11 to i32
+  %mul29 = mul nsw i32 %conv28, %conv26
+  %sext.5 = sext i32 %mul29 to i64
+  %add30 = add i64 %add24, %sext.5
+  ret i64 %add30
+}
+
+; CHECK: with_64bit_add_acc
+; CHECK: [[ADDR_X_PLUS_1:%[^ ]+]] = getelementptr inbounds i16, i16* %px.10756.unr, i32 1
+; CHECK: [[X:%[^ ]+]] = load i16, i16* %px.10756.unr, align 2
+; CHECK: [[SEXT_X:%[^ ]+]] = sext i16 [[X]] to i32
+; CHECK: [[ADDR_Y_MINUS_1:%[^ ]+]] = getelementptr inbounds i16, i16* %py.8757.unr, i32 -1
+; CHECK: [[Y:%[^ ]+]] = load i16, i16* %py.8757.unr, align 2
+; CHECK: [[SEXT_Y:%[^ ]+]] = sext i16 [[Y]] to i32
+; CHECK: [[MUL0:%[^ ]+]] = mul nsw i32 [[SEXT_Y]], [[SEXT_X]]
+; CHECK: [[SEXT_MUL0:%[^ ]+]] = sext i32 [[MUL0]] to i64
+; CHECK: [[ADD_1:%[^ ]+]] = add nsw i64 %sum.3758.unr, [[SEXT_MUL0]]
+; CHECK: [[X_PLUS_2:%[^ ]+]] = getelementptr inbounds i16, i16* %px.10756.unr, i32 2
+; CHECK: [[X_1:%[^ ]+]] = load i16, i16* [[ADDR_X_PLUS_1]], align 2
+; CHECK: [[SEXT_X_1:%[^ ]+]] = sext i16 [[X_1]] to i32
+; CHECK: [[Y_1:%[^ ]+]] = load i16, i16* [[ADDR_Y_MINUS_1]], align 2
+; CHECK: [[SEXT_Y_1:%[^ ]+]] = sext i16 [[Y_1]] to i32
+; CHECK: [[UNPAIRED:%[^ ]+]] = mul nsw i32 [[SEXT_Y_1]], [[SEXT_X_1]]
+; CHECK: [[SEXT:%[^ ]+]] = sext i32 [[UNPAIRED]] to i64
+; CHECK: [[ACC:%[^ ]+]] = add i64 [[SEXT]], [[ADD_1]]
+; CHECK: [[ADDR_X_PLUS_2:%[^ ]+]] = bitcast i16* [[X_PLUS_2]] to i32*
+; CHECK: [[X_2:%[^ ]+]] = load i32, i32* [[ADDR_X_PLUS_2]], align 2
+; CHECK: [[Y_MINUS_3:%[^ ]+]] = getelementptr inbounds i16, i16* %py.8757.unr, i32 -3
+; CHECK: [[ADDR_Y_MINUS_3:%[^ ]+]] = bitcast i16* [[Y_MINUS_3]] to i32*
+; CHECK: [[Y_3:%[^ ]+]] = load i32, i32* [[ADDR_Y_MINUS_3]], align 2
+; CHECK: [[RES:%[^ ]+]] = call i64 @llvm.arm.smlaldx(i32 [[Y_3]], i32 [[X_2]], i64 [[ACC]])
+; CHECK: ret i64 [[RES]]
+define i64 @with_64bit_add_acc(i16* nocapture readonly %px.10756.unr, i16* nocapture readonly %py.8757.unr, i32 %acc) {
+entry:
+  %sum.3758.unr = sext i32 %acc to i64
+  br label %bb.1
+
+bb.1:
+  %incdec.ptr184.epil = getelementptr inbounds i16, i16* %px.10756.unr, i32 1
+  %tmp216 = load i16, i16* %px.10756.unr, align 2
+  %conv185.epil = sext i16 %tmp216 to i32
+  %incdec.ptr186.epil = getelementptr inbounds i16, i16* %py.8757.unr, i32 -1
+  %tmp217 = load i16, i16* %py.8757.unr, align 2
+  %conv187.epil = sext i16 %tmp217 to i32
+  %mul.epil = mul nsw i32 %conv187.epil, %conv185.epil
+  %conv188.epil = sext i32 %mul.epil to i64
+  %add189.epil = add nsw i64 %sum.3758.unr, %conv188.epil
+  %incdec.ptr190.epil = getelementptr inbounds i16, i16* %px.10756.unr, i32 2
+  %tmp218 = load i16, i16* %incdec.ptr184.epil, align 2
+  %conv191.epil = sext i16 %tmp218 to i32
+  %incdec.ptr192.epil = getelementptr inbounds i16, i16* %py.8757.unr, i32 -2
+  %tmp219 = load i16, i16* %incdec.ptr186.epil, align 2
+  %conv193.epil = sext i16 %tmp219 to i32
+  %mul194.epil = mul nsw i32 %conv193.epil, %conv191.epil
+  %conv195.epil = sext i32 %mul194.epil to i64
+  %add196.epil = add nsw i64 %add189.epil, %conv195.epil
+  %incdec.ptr197.epil = getelementptr inbounds i16, i16* %px.10756.unr, i32 3
+  %tmp220 = load i16, i16* %incdec.ptr190.epil, align 2
+  %conv198.epil = sext i16 %tmp220 to i32
+  %incdec.ptr199.epil = getelementptr inbounds i16, i16* %py.8757.unr, i32 -3
+  %tmp221 = load i16, i16* %incdec.ptr192.epil, align 2
+  %conv200.epil = sext i16 %tmp221 to i32
+  %mul201.epil = mul nsw i32 %conv200.epil, %conv198.epil
+  %conv202.epil = sext i32 %mul201.epil to i64
+  %add203.epil = add nsw i64 %add196.epil, %conv202.epil
+  %tmp222 = load i16, i16* %incdec.ptr197.epil, align 2
+  %conv205.epil = sext i16 %tmp222 to i32
+  %tmp223 = load i16, i16* %incdec.ptr199.epil, align 2
+  %conv207.epil = sext i16 %tmp223 to i32
+  %mul208.epil = mul nsw i32 %conv207.epil, %conv205.epil
+  %conv209.epil = sext i32 %mul208.epil to i64
+  %add210.epil = add nsw i64 %add203.epil, %conv209.epil
+  ret i64 %add210.epil
+}
+
+declare dso_local i32 @bar(i32, i32) local_unnamed_addr
+
diff --git a/llvm/test/CodeGen/ARM/ParallelDSP/sext-acc.ll b/llvm/test/CodeGen/ARM/ParallelDSP/sext-acc.ll
new file mode 100644
index 000000000..192309a6d
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/ParallelDSP/sext-acc.ll
@@ -0,0 +1,186 @@
+; RUN: opt -arm-parallel-dsp -dce -mtriple=armv7-a -S %s -o - | FileCheck %s
+
+; CHECK-LABEL: sext_acc_1
+; CHECK: [[CAST_A:%[^ ]+]] = bitcast i16* %a to i32*
+; CHECK: [[A:%[^ ]+]] = load i32, i32* [[CAST_A]]
+; CHECK: [[CAST_B:%[^ ]+]] = bitcast i16* %b to i32*
+; CHECK: [[B:%[^ ]+]] = load i32, i32* [[CAST_B]]
+; CHECK: [[ACC:%[^ ]+]] = sext i32 %acc to i64
+; CHECK: call i64 @llvm.arm.smlald(i32 [[A]], i32 [[B]], i64 [[ACC]])
+define i64 @sext_acc_1(i16* %a, i16* %b, i32 %acc) {
+entry:
+  %ld.a.0 = load i16, i16* %a
+  %sext.a.0 = sext i16 %ld.a.0 to i32
+  %ld.b.0 = load i16, i16* %b
+  %sext.b.0 = sext i16 %ld.b.0 to i32
+  %mul.0 = mul i32 %sext.a.0, %sext.b.0
+  %addr.a.1 = getelementptr i16, i16* %a, i32 1
+  %addr.b.1 = getelementptr i16, i16* %b, i32 1
+  %ld.a.1 = load i16, i16* %addr.a.1
+  %sext.a.1 = sext i16 %ld.a.1 to i32
+  %ld.b.1 = load i16, i16* %addr.b.1
+  %sext.b.1 = sext i16 %ld.b.1 to i32
+  %mul.1 = mul i32 %sext.a.1, %sext.b.1
+  %sext.mul.0 = sext i32 %mul.0 to i64
+  %sext.mul.1 = sext i32 %mul.1 to i64
+  %add = add i64 %sext.mul.0, %sext.mul.1
+  %sext.acc = sext i32 %acc to i64
+  %res = add i64 %add, %sext.acc
+  ret i64 %res
+}
+
+; CHECK-LABEL: sext_acc_2
+; CHECK: [[CAST_A:%[^ ]+]] = bitcast i16* %a to i32*
+; CHECK: [[A:%[^ ]+]] = load i32, i32* [[CAST_A]]
+; CHECK: [[CAST_B:%[^ ]+]] = bitcast i16* %b to i32*
+; CHECK: [[B:%[^ ]+]] = load i32, i32* [[CAST_B]]
+; CHECK: [[CAST_A_2:%[^ ]+]] = bitcast i16* %addr.a.2 to i32*
+; CHECK: [[A_2:%[^ ]+]] = load i32, i32* %4
+; CHECK: [[CAST_B_2:%[^ ]+]] = bitcast i16* %addr.b.2 to i32*
+; CHECK: [[B_2:%[^ ]+]] = load i32, i32* %6
+; CHECK: [[ACC:%[^ ]+]] = sext i32 %acc to i64
+; CHECK: [[SMLALD:%[^ ]+]] = call i64 @llvm.arm.smlald(i32 [[A]], i32 [[B]], i64 [[ACC]])
+; CHECK: call i64 @llvm.arm.smlald(i32 [[A_2]], i32 [[B_2]], i64 [[SMLALD]])
+define i64 @sext_acc_2(i16* %a, i16* %b, i32 %acc) {
+entry:
+  %ld.a.0 = load i16, i16* %a
+  %sext.a.0 = sext i16 %ld.a.0 to i32
+  %ld.b.0 = load i16, i16* %b
+  %sext.b.0 = sext i16 %ld.b.0 to i32
+  %mul.0 = mul i32 %sext.a.0, %sext.b.0
+  %addr.a.1 = getelementptr i16, i16* %a, i32 1
+  %addr.b.1 = getelementptr i16, i16* %b, i32 1
+  %ld.a.1 = load i16, i16* %addr.a.1
+  %sext.a.1 = sext i16 %ld.a.1 to i32
+  %ld.b.1 = load i16, i16* %addr.b.1
+  %sext.b.1 = sext i16 %ld.b.1 to i32
+  %mul.1 = mul i32 %sext.a.1, %sext.b.1
+  %sext.mul.0 = sext i32 %mul.0 to i64
+  %sext.mul.1 = sext i32 %mul.1 to i64
+  %add = add i64 %sext.mul.0, %sext.mul.1
+  %sext.acc = sext i32 %acc to i64
+  %add.1 = add i64 %add, %sext.acc
+  %addr.a.2 = getelementptr i16, i16* %a, i32 2
+  %addr.b.2 = getelementptr i16, i16* %b, i32 2
+  %ld.a.2 = load i16, i16* %addr.a.2
+  %sext.a.2 = sext i16 %ld.a.2 to i32
+  %ld.b.2 = load i16, i16* %addr.b.2
+  %sext.b.2 = sext i16 %ld.b.2 to i32
+  %mul.2 = mul i32 %sext.a.2, %sext.b.2
+  %sext.mul.2 = sext i32 %mul.2 to i64
+  %addr.a.3 = getelementptr i16, i16* %a, i32 3
+  %addr.b.3 = getelementptr i16, i16* %b, i32 3
+  %ld.a.3 = load i16, i16* %addr.a.3
+  %sext.a.3 = sext i16 %ld.a.3 to i32
+  %ld.b.3 = load i16, i16* %addr.b.3
+  %sext.b.3 = sext i16 %ld.b.3 to i32
+  %mul.3 = mul i32 %sext.a.3, %sext.b.3
+  %sext.mul.3 = sext i32 %mul.3 to i64
+  %add.2 = add i64 %sext.mul.2, %sext.mul.3
+  %add.3 = add i64 %add.1, %add.2
+  ret i64 %add.3
+}
+
+; CHECK-LABEL: sext_acc_3
+; CHECK: [[CAST_A:%[^ ]+]] = bitcast i16* %a to i32*
+; CHECK: [[A:%[^ ]+]] = load i32, i32* [[CAST_A]]
+; CHECK: [[CAST_B:%[^ ]+]] = bitcast i16* %b to i32*
+; CHECK: [[B:%[^ ]+]] = load i32, i32* [[CAST_B]]
+; CHECK: [[CAST_A_2:%[^ ]+]] = bitcast i16* %addr.a.2 to i32*
+; CHECK: [[A_2:%[^ ]+]] = load i32, i32* %4
+; CHECK: [[CAST_B_2:%[^ ]+]] = bitcast i16* %addr.b.2 to i32*
+; CHECK: [[B_2:%[^ ]+]] = load i32, i32* %6
+; CHECK: [[ACC:%[^ ]+]] = sext i32 %acc to i64
+; CHECK: [[SMLALD:%[^ ]+]] = call i64 @llvm.arm.smlald(i32 [[A]], i32 [[B]], i64 [[ACC]])
+; CHECK: call i64 @llvm.arm.smlald(i32 [[A_2]], i32 [[B_2]], i64 [[SMLALD]])
+define i64 @sext_acc_3(i16* %a, i16* %b, i32 %acc) {
+entry:
+  %ld.a.0 = load i16, i16* %a
+  %sext.a.0 = sext i16 %ld.a.0 to i32
+  %ld.b.0 = load i16, i16* %b
+  %sext.b.0 = sext i16 %ld.b.0 to i32
+  %mul.0 = mul i32 %sext.a.0, %sext.b.0
+  %addr.a.1 = getelementptr i16, i16* %a, i32 1
+  %addr.b.1 = getelementptr i16, i16* %b, i32 1
+  %ld.a.1 = load i16, i16* %addr.a.1
+  %sext.a.1 = sext i16 %ld.a.1 to i32
+  %ld.b.1 = load i16, i16* %addr.b.1
+  %sext.b.1 = sext i16 %ld.b.1 to i32
+  %mul.1 = mul i32 %sext.a.1, %sext.b.1
+  %sext.mul.0 = sext i32 %mul.0 to i64
+  %sext.mul.1 = sext i32 %mul.1 to i64
+  %add = add i64 %sext.mul.0, %sext.mul.1
+  %addr.a.2 = getelementptr i16, i16* %a, i32 2
+  %addr.b.2 = getelementptr i16, i16* %b, i32 2
+  %ld.a.2 = load i16, i16* %addr.a.2
+  %sext.a.2 = sext i16 %ld.a.2 to i32
+  %ld.b.2 = load i16, i16* %addr.b.2
+  %sext.b.2 = sext i16 %ld.b.2 to i32
+  %mul.2 = mul i32 %sext.a.2, %sext.b.2
+  %sext.mul.2 = sext i32 %mul.2 to i64
+  %addr.a.3 = getelementptr i16, i16* %a, i32 3
+  %addr.b.3 = getelementptr i16, i16* %b, i32 3
+  %ld.a.3 = load i16, i16* %addr.a.3
+  %sext.a.3 = sext i16 %ld.a.3 to i32
+  %ld.b.3 = load i16, i16* %addr.b.3
+  %sext.b.3 = sext i16 %ld.b.3 to i32
+  %mul.3 = mul i32 %sext.a.3, %sext.b.3
+  %sext.mul.3 = sext i32 %mul.3 to i64
+  %add.1 = add i64 %sext.mul.2, %sext.mul.3
+  %add.2 = add i64 %add, %add.1
+  %sext.acc = sext i32 %acc to i64
+  %add.3 = add i64 %add.2, %sext.acc
+  ret i64 %add.3
+}
+
+; CHECK-LABEL: sext_acc_4
+; CHECK: [[CAST_A:%[^ ]+]] = bitcast i16* %a to i32*
+; CHECK: [[A:%[^ ]+]] = load i32, i32* [[CAST_A]]
+; CHECK: [[CAST_B:%[^ ]+]] = bitcast i16* %b to i32*
+; CHECK: [[B:%[^ ]+]] = load i32, i32* [[CAST_B]]
+; CHECK: [[CAST_A_2:%[^ ]+]] = bitcast i16* %addr.a.2 to i32*
+; CHECK: [[A_2:%[^ ]+]] = load i32, i32* %4
+; CHECK: [[CAST_B_2:%[^ ]+]] = bitcast i16* %addr.b.2 to i32*
+; CHECK: [[B_2:%[^ ]+]] = load i32, i32* %6
+; CHECK: [[ACC:%[^ ]+]] = sext i32 %acc to i64
+; CHECK: [[SMLALD:%[^ ]+]] = call i64 @llvm.arm.smlald(i32 [[A]], i32 [[B]], i64 [[ACC]])
+; CHECK: call i64 @llvm.arm.smlald(i32 [[A_2]], i32 [[B_2]], i64 [[SMLALD]])
+define i64 @sext_acc_4(i16* %a, i16* %b, i32 %acc) {
+entry:
+  %ld.a.0 = load i16, i16* %a
+  %sext.a.0 = sext i16 %ld.a.0 to i32
+  %ld.b.0 = load i16, i16* %b
+  %sext.b.0 = sext i16 %ld.b.0 to i32
+  %mul.0 = mul i32 %sext.a.0, %sext.b.0
+  %addr.a.1 = getelementptr i16, i16* %a, i32 1
+  %addr.b.1 = getelementptr i16, i16* %b, i32 1
+  %ld.a.1 = load i16, i16* %addr.a.1
+  %sext.a.1 = sext i16 %ld.a.1 to i32
+  %ld.b.1 = load i16, i16* %addr.b.1
+  %sext.b.1 = sext i16 %ld.b.1 to i32
+  %mul.1 = mul i32 %sext.a.1, %sext.b.1
+  %add = add i32 %mul.0, %mul.1
+  %sext.add = sext i32 %add to i64
+  %addr.a.2 = getelementptr i16, i16* %a, i32 2
+  %addr.b.2 = getelementptr i16, i16* %b, i32 2
+  %ld.a.2 = load i16, i16* %addr.a.2
+  %sext.a.2 = sext i16 %ld.a.2 to i32
+  %ld.b.2 = load i16, i16* %addr.b.2
+  %sext.b.2 = sext i16 %ld.b.2 to i32
+  %mul.2 = mul i32 %sext.a.2, %sext.b.2
+  %sext.mul.2 = sext i32 %mul.2 to i64
+  %addr.a.3 = getelementptr i16, i16* %a, i32 3
+  %addr.b.3 = getelementptr i16, i16* %b, i32 3
+  %ld.a.3 = load i16, i16* %addr.a.3
+  %sext.a.3 = sext i16 %ld.a.3 to i32
+  %ld.b.3 = load i16, i16* %addr.b.3
+  %sext.b.3 = sext i16 %ld.b.3 to i32
+  %mul.3 = mul i32 %sext.a.3, %sext.b.3
+  %sext.mul.3 = sext i32 %mul.3 to i64
+  %sext.acc = sext i32 %acc to i64
+  %add.1 = add i64 %sext.mul.2, %sext.add
+  %add.2 = add i64 %sext.add, %add.1
+  %add.3 = add i64 %add.2, %sext.mul.3
+  %add.4 = add i64 %add.3, %sext.acc
+  ret i64 %add.4
+}
diff --git a/llvm/test/CodeGen/ARM/ParallelDSP/smlad11.ll b/llvm/test/CodeGen/ARM/ParallelDSP/smlad11.ll
index b17106e70..2f47c2ce7 100644
--- a/llvm/test/CodeGen/ARM/ParallelDSP/smlad11.ll
+++ b/llvm/test/CodeGen/ARM/ParallelDSP/smlad11.ll
@@ -10,10 +10,10 @@
 ; CHECK:  [[V16:%[0-9]+]] = load i32, i32* [[V15]], align 2
 ; CHECK:  [[V8:%[0-9]+]] = bitcast i16* %arrayidx8 to i32*
 ; CHECK:  [[V9:%[0-9]+]] = load i32, i32* [[V8]], align 2
+; CHECK:  [[ACC:%[0-9]+]] = call i32 @llvm.arm.smlad(i32 [[V9]], i32 [[V11]], i32 %mac1{{\.}}054)
 ; CHECK:  [[V13:%[0-9]+]] = bitcast i16* %arrayidx17 to i32*
 ; CHECK:  [[V14:%[0-9]+]] = load i32, i32* [[V13]], align 2
-; CHECK:  [[V12:%[0-9]+]] = call i32 @llvm.arm.smlad(i32 [[V9]], i32 [[V11]], i32 %mac1{{\.}}054)
-; CHECK:  [[V17:%[0-9]+]] = call i32 @llvm.arm.smlad(i32 [[V14]], i32 [[V16]], i32 [[V12]])
+; CHECK:  [[V12:%[0-9]+]] = call i32 @llvm.arm.smlad(i32 [[V14]], i32 [[V16]], i32 [[ACC]])
 ;
 ; And we don't want to see a 3rd smlad:
 ; CHECK-NOT: call i32 @llvm.arm.smlad
diff --git a/llvm/test/CodeGen/ARM/ParallelDSP/smlad12.ll b/llvm/test/CodeGen/ARM/ParallelDSP/smlad12.ll
index d4e09ca3f..637fc3d37 100644
--- a/llvm/test/CodeGen/ARM/ParallelDSP/smlad12.ll
+++ b/llvm/test/CodeGen/ARM/ParallelDSP/smlad12.ll
@@ -2,7 +2,7 @@
 ;
 ; The loop header is not the loop latch.
 ;
-; CHECK-NOT:  call i32 @llvm.arm.smlad
+; CHECK:  call i32 @llvm.arm.smlad
 ;
 define dso_local i32 @test(i32 %arg, i32* nocapture readnone %arg1, i16* nocapture readonly %arg2, i16* nocapture readonly %arg3) {
 entry:
diff --git a/llvm/test/CodeGen/ARM/ParallelDSP/smladx-1.ll b/llvm/test/CodeGen/ARM/ParallelDSP/smladx-1.ll
index aa0125736..96e0667db 100644
--- a/llvm/test/CodeGen/ARM/ParallelDSP/smladx-1.ll
+++ b/llvm/test/CodeGen/ARM/ParallelDSP/smladx-1.ll
@@ -12,12 +12,13 @@ define i32 @smladx(i16* nocapture readonly %pIn1, i16* nocapture readonly %pIn2,
 ; CHECK: [[IN21:%[^ ]+]] = load i32, i32* [[PIN21]], align 2
 ; CHECK: [[PIN10:%[^ ]+]] = bitcast i16* %pIn1.0 to i32*
 ; CHECK: [[IN10:%[^ ]+]] = load i32, i32* [[PIN10]], align 2
+; CHECK: [[ACC1:%[^ ]+]] = call i32 @llvm.arm.smladx(i32 [[IN21]], i32 [[IN10]], i32 [[ACC0]])
+
 ; CHECK: [[PIN23:%[^ ]+]] = bitcast i16* %pIn2.3 to i32*
 ; CHECK: [[IN23:%[^ ]+]] = load i32, i32* [[PIN23]], align 2
 ; CHECK: [[PIN12:%[^ ]+]] = bitcast i16* %pIn1.2 to i32*
 ; CHECK: [[IN12:%[^ ]+]] = load i32, i32* [[PIN12]], align 2
-; CHECK: [[ACC1:%[^ ]+]] = call i32 @llvm.arm.smladx(i32 [[IN23]], i32 [[IN12]], i32 [[ACC0]])
-; CHECK: [[ACC2]] = call i32 @llvm.arm.smladx(i32 [[IN21]], i32 [[IN10]], i32 [[ACC1]])
+; CHECK: [[ACC2]] = call i32 @llvm.arm.smladx(i32 [[IN23]], i32 [[IN12]], i32 [[ACC1]])
 ; CHECK-NOT: call i32 @llvm.arm.smlad
 ; CHECK-UNSUPPORTED-NOT:  call i32 @llvm.arm.smlad
 
@@ -130,6 +131,7 @@ define i32 @smladx_swap(i16* nocapture readonly %pIn1, i16* nocapture readonly %
 ; CHECK: [[PIN1_2:%[^ ]+]] = getelementptr i16, i16* [[PIN1]], i32 -2
 ; CHECK: [[PIN1_2_CAST:%[^ ]+]] = bitcast i16* [[PIN1_2]] to i32*
 ; CHECK: [[IN1_2:%[^ ]+]] = load i32, i32* [[PIN1_2_CAST]], align 2
+; CHECK: [[ACC1:%[^ ]+]] = call i32 @llvm.arm.smladx(i32 [[IN2]], i32 [[IN1_2]], i32 [[ACC0]])
 
 ; CHECK: [[PIN2_2:%[^ ]+]] = getelementptr i16, i16* [[PIN2]], i32 -2
 ; CHECK: [[PIN2_2_CAST:%[^ ]+]] = bitcast i16* [[PIN2_2]] to i32*
@@ -138,8 +140,7 @@ define i32 @smladx_swap(i16* nocapture readonly %pIn1, i16* nocapture readonly %
 ; CHECK: [[PIN1_CAST:%[^ ]+]] = bitcast i16* [[PIN1]] to i32*
 ; CHECK: [[IN1:%[^ ]+]] = load i32, i32* [[PIN1_CAST]], align 2
 
-; CHECK: [[ACC1:%[^ ]+]] = call i32 @llvm.arm.smladx(i32 [[IN2_2]], i32 [[IN1]], i32 [[ACC0]])
-; CHECK: [[ACC2]] = call i32 @llvm.arm.smladx(i32 [[IN2]], i32 [[IN1_2]], i32 [[ACC1]])
+; CHECK: [[ACC2]] = call i32 @llvm.arm.smladx(i32 [[IN2_2]], i32 [[IN1]], i32 [[ACC1]])
 
 ; CHECK: [[PIN1_NEXT]] = getelementptr i16, i16* [[PIN1]], i32 4
 ; CHECK: [[PIN2_NEXT]] = getelementptr i16, i16* [[PIN2]], i32 -4
diff --git a/llvm/test/CodeGen/ARM/ParallelDSP/smlaldx-1.ll b/llvm/test/CodeGen/ARM/ParallelDSP/smlaldx-1.ll
index e325fe2f2..8fcfe66e8 100644
--- a/llvm/test/CodeGen/ARM/ParallelDSP/smlaldx-1.ll
+++ b/llvm/test/CodeGen/ARM/ParallelDSP/smlaldx-1.ll
@@ -11,12 +11,12 @@ define i64 @smlaldx(i16* nocapture readonly %pIn1, i16* nocapture readonly %pIn2
 ; CHECK: [[IN21:%[^ ]+]] = load i32, i32* [[PIN21]], align 2
 ; CHECK: [[PIN10:%[^ ]+]] = bitcast i16* %pIn1.0 to i32*
 ; CHECK: [[IN10:%[^ ]+]] = load i32, i32* [[PIN10]], align 2
+; CHECK: [[ACC1:%[^ ]+]] = call i64 @llvm.arm.smlaldx(i32 [[IN21]], i32 [[IN10]], i64 [[ACC0]])
 ; CHECK: [[PIN23:%[^ ]+]] = bitcast i16* %pIn2.3 to i32*
 ; CHECK: [[IN23:%[^ ]+]] = load i32, i32* [[PIN23]], align 2
 ; CHECK: [[PIN12:%[^ ]+]] = bitcast i16* %pIn1.2 to i32*
 ; CHECK: [[IN12:%[^ ]+]] = load i32, i32* [[PIN12]], align 2
-; CHECK: [[ACC1:%[^ ]+]] = call i64 @llvm.arm.smlaldx(i32 [[IN23]], i32 [[IN12]], i64 [[ACC0]])
-; CHECK: [[ACC2]] = call i64 @llvm.arm.smlaldx(i32 [[IN21]], i32 [[IN10]], i64 [[ACC1]])
+; CHECK: [[ACC2]] = call i64 @llvm.arm.smlaldx(i32 [[IN23]], i32 [[IN12]], i64 [[ACC1]])
 ; CHECK-NOT: call i64 @llvm.arm.smlad
 ; CHECK-UNSUPPORTED-NOT:  call i64 @llvm.arm.smlad
 
@@ -187,6 +187,7 @@ for.cond.cleanup:
 ; CHECK: [[PIN1_2:%[^ ]+]] = getelementptr i16, i16* [[PIN1]], i32 -2
 ; CHECK: [[PIN1_2_CAST:%[^ ]+]] = bitcast i16* [[PIN1_2]] to i32*
 ; CHECK: [[IN1_2:%[^ ]+]] = load i32, i32* [[PIN1_2_CAST]], align 2
+; CHECK: [[ACC1:%[^ ]+]] = call i64 @llvm.arm.smlaldx(i32 [[IN2]], i32 [[IN1_2]], i64 [[ACC0]])
 
 ; CHECK: [[PIN2_2:%[^ ]+]] = getelementptr i16, i16* [[PIN2]], i32 -2
 ; CHECK: [[PIN2_2_CAST:%[^ ]+]] = bitcast i16* [[PIN2_2]] to i32*
@@ -194,9 +195,7 @@ for.cond.cleanup:
 
 ; CHECK: [[PIN1_CAST:%[^ ]+]] = bitcast i16* [[PIN1]] to i32*
 ; CHECK: [[IN1:%[^ ]+]] = load i32, i32* [[PIN1_CAST]], align 2
-
-; CHECK: [[ACC1:%[^ ]+]] = call i64 @llvm.arm.smlaldx(i32 [[IN2_2]], i32 [[IN1]], i64 [[ACC0]])
-; CHECK: [[ACC2]] = call i64 @llvm.arm.smlaldx(i32 [[IN2]], i32 [[IN1_2]], i64 [[ACC1]])
+; CHECK: [[ACC2]] = call i64 @llvm.arm.smlaldx(i32 [[IN2_2]], i32 [[IN1]], i64 [[ACC1]])
 
 ; CHECK: [[PIN1_NEXT]] = getelementptr i16, i16* [[PIN1]], i32 4
 ; CHECK: [[PIN2_NEXT]] = getelementptr i16, i16* [[PIN2]], i32 -4
diff --git a/llvm/test/CodeGen/ARM/ParallelDSP/smlaldx-2.ll b/llvm/test/CodeGen/ARM/ParallelDSP/smlaldx-2.ll
index 971c85f1b..683562e54 100644
--- a/llvm/test/CodeGen/ARM/ParallelDSP/smlaldx-2.ll
+++ b/llvm/test/CodeGen/ARM/ParallelDSP/smlaldx-2.ll
@@ -11,12 +11,12 @@ define i64 @smlaldx(i16* nocapture readonly %pIn1, i16* nocapture readonly %pIn2
 ; CHECK: [[IN21:%[^ ]+]] = load i32, i32* [[PIN21]], align 2
 ; CHECK: [[PIN10:%[^ ]+]] = bitcast i16* %pIn1.0 to i32*
 ; CHECK: [[IN10:%[^ ]+]] = load i32, i32* [[PIN10]], align 2
+; CHECK: [[ACC1:%[^ ]+]] = call i64 @llvm.arm.smlaldx(i32 [[IN21]], i32 [[IN10]], i64 [[ACC0]])
 ; CHECK: [[PIN23:%[^ ]+]] = bitcast i16* %pIn2.3 to i32*
 ; CHECK: [[IN23:%[^ ]+]] = load i32, i32* [[PIN23]], align 2
 ; CHECK: [[PIN12:%[^ ]+]] = bitcast i16* %pIn1.2 to i32*
 ; CHECK: [[IN12:%[^ ]+]] = load i32, i32* [[PIN12]], align 2
-; CHECK: [[ACC1:%[^ ]+]] = call i64 @llvm.arm.smlaldx(i32 [[IN23]], i32 [[IN12]], i64 [[ACC0]])
-; CHECK: [[ACC2]] = call i64 @llvm.arm.smlaldx(i32 [[IN21]], i32 [[IN10]], i64 [[ACC1]])
+; CHECK: [[ACC2]] = call i64 @llvm.arm.smlaldx(i32 [[IN23]], i32 [[IN12]], i64 [[ACC1]])
 ; CHECK-NOT: call i64 @llvm.arm.smlad
 ; CHECK-UNSUPPORTED-NOT:  call i64 @llvm.arm.smlad
 
@@ -187,6 +187,7 @@ for.cond.cleanup:
 ; CHECK: [[PIN1_2:%[^ ]+]] = getelementptr i16, i16* [[PIN1]], i32 -2
 ; CHECK: [[PIN1_2_CAST:%[^ ]+]] = bitcast i16* [[PIN1_2]] to i32*
 ; CHECK: [[IN1_2:%[^ ]+]] = load i32, i32* [[PIN1_2_CAST]], align 2
+; CHECK: [[ACC1:%[^ ]+]] = call i64 @llvm.arm.smlaldx(i32 [[IN2]], i32 [[IN1_2]], i64 [[ACC0]])
 
 ; CHECK: [[PIN2_2:%[^ ]+]] = getelementptr i16, i16* [[PIN2]], i32 -2
 ; CHECK: [[PIN2_2_CAST:%[^ ]+]] = bitcast i16* [[PIN2_2]] to i32*
@@ -194,9 +195,7 @@ for.cond.cleanup:
 
 ; CHECK: [[PIN1_CAST:%[^ ]+]] = bitcast i16* [[PIN1]] to i32*
 ; CHECK: [[IN1:%[^ ]+]] = load i32, i32* [[PIN1_CAST]], align 2
-
-; CHECK: [[ACC1:%[^ ]+]] = call i64 @llvm.arm.smlaldx(i32 [[IN2]], i32 [[IN1_2]], i64 [[ACC0]])
-; CHECK: [[ACC2]] = call i64 @llvm.arm.smlaldx(i32 [[IN1]], i32 [[IN2_2]], i64 [[ACC1]])
+; CHECK: [[ACC2]] = call i64 @llvm.arm.smlaldx(i32 [[IN2_2]], i32 [[IN1]], i64 [[ACC1]])
 
 ; CHECK: [[PIN1_NEXT]] = getelementptr i16, i16* [[PIN1]], i32 4
 ; CHECK: [[PIN2_NEXT]] = getelementptr i16, i16* [[PIN2]], i32 -4
diff --git a/llvm/test/CodeGen/ARM/ParallelDSP/unroll-n-jam-smlad.ll b/llvm/test/CodeGen/ARM/ParallelDSP/unroll-n-jam-smlad.ll
index 03c77eb80..1e988fe34 100644
--- a/llvm/test/CodeGen/ARM/ParallelDSP/unroll-n-jam-smlad.ll
+++ b/llvm/test/CodeGen/ARM/ParallelDSP/unroll-n-jam-smlad.ll
@@ -1,6 +1,6 @@
-; RUN: llc -O3 -mtriple=thumbv7em %s -o - | FileCheck %s
+; RUN: llc -O3 -mtriple=thumbv7em -mcpu=cortex-m4 %s -o - | FileCheck %s --check-prefix=CHECK-REG-PRESSURE
 ; RUN: llc -O3 -mtriple=thumbv7eb %s -o - | FileCheck %s --check-prefix=CHECK-UNSUPPORTED
-; RUN: llc -O3 -mtriple=thumbv8m.main -mattr=+dsp %s -o - | FileCheck %s
+; RUN: llc -O3 -mtriple=thumbv8m.main -mattr=+dsp -arm-parallel-dsp-load-limit=20 %s -o - | FileCheck %s --check-prefix=CHECK
 
 ; CHECK-UNSUPPORTED-LABEL: unroll_n_jam_smlad
 ; CHECK-UNSUPPORTED-NOT: smlad r{{.}}
@@ -37,6 +37,16 @@ entry:
 ; CHECK: smlad
 ; CHECK-NOT: smlad r{{.*}}
 
+; CHECK-REG-PRESSURE: .LBB0_1:
+; CHECK-REG-PRESSURE-NOT: call i32 @llvm.arm.smlad
+; CHECK-REG-PRESSURE: ldr{{.*}}, [sp
+; CHECK-REG-PRESSURE: ldr{{.*}}, [sp
+; CHECK-REG-PRESSURE: ldr{{.*}}, [sp
+; CHECK-REG-PRESSURE: ldr{{.*}}, [sp
+; CHECK-REG-PRESSURE: ldr{{.*}}, [sp
+; CHECK-REG-PRESSURE-NOT: ldr{{.*}}, [sp
+; CHECK-REG-PRESSURE: bne .LBB0_1
+
 for.body:
   %A3 = phi i32 [ %add9.us.i.3361.i, %for.body ], [ 0, %entry ]
   %j.026.us.i.i = phi i32 [ %inc.us.i.3362.i, %for.body ], [ 0, %entry ]
diff --git a/llvm/test/CodeGen/ARM/Windows/wineh-basic.ll b/llvm/test/CodeGen/ARM/Windows/wineh-basic.ll
index 848ffbf50..c4e809f6d 100644
--- a/llvm/test/CodeGen/ARM/Windows/wineh-basic.ll
+++ b/llvm/test/CodeGen/ARM/Windows/wineh-basic.ll
@@ -36,8 +36,8 @@ declare arm_aapcs_vfpcc i32 @__CxxFrameHandler3(...)
 
 declare arm_aapcs_vfpcc void @__std_terminate() local_unnamed_addr
 
-attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="cortex-a9" "target-features"="+dsp,+fp16,+neon,+strict-align,+vfp3" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="cortex-a9" "target-features"="+dsp,+fp16,+neon,+strict-align,+vfp3" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="cortex-a9" "target-features"="+dsp,+fp16,+neon,+strict-align,+vfp3" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="cortex-a9" "target-features"="+dsp,+fp16,+neon,+strict-align,+vfp3" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { noreturn nounwind }
 
 !llvm.module.flags = !{!0, !1}
diff --git a/llvm/test/CodeGen/ARM/a15-partial-update.ll b/llvm/test/CodeGen/ARM/a15-partial-update.ll
index 576eb7a24..a3971d6a0 100644
--- a/llvm/test/CodeGen/ARM/a15-partial-update.ll
+++ b/llvm/test/CodeGen/ARM/a15-partial-update.ll
@@ -1,7 +1,6 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -O1 -mcpu=cortex-a15 -mtriple=armv7-linux-gnueabi -verify-machineinstrs < %s  | FileCheck %s
 
-; CHECK-LABEL: t1:
-define <2 x float> @t1(float* %A, <2 x float> %B) {
 ; The generated code for this test uses a vld1.32 instruction
 ; to write the lane 1 of a D register containing the value of
 ; <2 x float> %B. Since the D register is defined, it would
@@ -9,22 +8,78 @@ define <2 x float> @t1(float* %A, <2 x float> %B) {
 ; vld1.32 instruction. The test checks that a vmov.f64 was not
 ; generated.
 
-; CHECK-NOT: vmov.{{.*}} d{{[0-9]+}},
+define <2 x float> @t1(float* %A, <2 x float> %B) {
+; CHECK-LABEL: t1:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmov d16, r2, r3
+; CHECK-NEXT:    vld1.32 {d16[1]}, [r0:32]
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    bx lr
   %tmp2 = load float, float* %A, align 4
   %tmp3 = insertelement <2 x float> %B, float %tmp2, i32 1
   ret <2 x float> %tmp3
 }
 
-; CHECK-LABEL: t2:
+; The code generated by this test uses a vld1.32 instruction.
+; We check that a dependency breaking vmov* instruction was
+; generated.
+
 define void @t2(<4 x i8> *%in, <4 x i8> *%out, i32 %n) {
+; CHECK-LABEL: t2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    add r0, r0, #4
+; CHECK-NEXT:    add r1, r1, #4
+; CHECK-NEXT:  .LBB1_1: @ %loop
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmov.f64 d16, #5.000000e-01
+; CHECK-NEXT:    vld1.32 {d16[0]}, [r0:32]
+; CHECK-NEXT:    vmovl.u8 q8, d16
+; CHECK-NEXT:    vuzp.8 d16, d18
+; CHECK-NEXT:    vst1.32 {d16[0]}, [r1:32]!
+; CHECK-NEXT:    add r0, r0, #4
+; CHECK-NEXT:    subs r2, r2, #1
+; CHECK-NEXT:    beq .LBB1_1
+; CHECK-NEXT:  @ %bb.2: @ %ret
+; CHECK-NEXT:    bx lr
 entry:
   br label %loop
 loop:
-; The code generated by this test uses a vld1.32 instruction.
-; We check that a dependency breaking vmov* instruction was
-; generated.
+  %oldcount = phi i32 [0, %entry], [%newcount, %loop]
+  %newcount = add i32 %oldcount, 1
+  %p1 = getelementptr <4 x i8>, <4 x i8> *%in, i32 %newcount
+  %p2 = getelementptr <4 x i8>, <4 x i8> *%out, i32 %newcount
+  %tmp1 = load <4 x i8> , <4 x i8> *%p1, align 4
+  store <4 x i8> %tmp1, <4 x i8> *%p2
+  %cmp = icmp eq i32 %newcount, %n
+  br i1 %cmp, label %loop, label %ret
+ret:
+  ret void
+}
+
+; If minimizing size, that overrides perf, so no extra vmov.f64 here.
+
+; TODO: This (and above) could use a splat load to remove the false
+;       dependence with no extra instruction.
 
-; CHECK: vmov.{{.*}} d{{[0-9]+}},
+define void @t2_minsize(<4 x i8> *%in, <4 x i8> *%out, i32 %n) minsize {
+; CHECK-LABEL: t2_minsize:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    add r0, r0, #4
+; CHECK-NEXT:    add r1, r1, #4
+; CHECK-NEXT:  .LBB2_1: @ %loop
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vld1.32 {d16[0]}, [r0:32]
+; CHECK-NEXT:    vmovl.u8 q8, d16
+; CHECK-NEXT:    vuzp.8 d16, d18
+; CHECK-NEXT:    vst1.32 {d16[0]}, [r1:32]!
+; CHECK-NEXT:    add r0, r0, #4
+; CHECK-NEXT:    subs r2, r2, #1
+; CHECK-NEXT:    beq .LBB2_1
+; CHECK-NEXT:  @ %bb.2: @ %ret
+; CHECK-NEXT:    bx lr
+entry:
+  br label %loop
+loop:
   %oldcount = phi i32 [0, %entry], [%newcount, %loop]
   %newcount = add i32 %oldcount, 1
   %p1 = getelementptr <4 x i8>, <4 x i8> *%in, i32 %newcount
diff --git a/llvm/test/CodeGen/ARM/addsubcarry-promotion.ll b/llvm/test/CodeGen/ARM/addsubcarry-promotion.ll
index 2dfd21705..aaf7189a6 100644
--- a/llvm/test/CodeGen/ARM/addsubcarry-promotion.ll
+++ b/llvm/test/CodeGen/ARM/addsubcarry-promotion.ll
@@ -10,11 +10,11 @@
 define void @fn1(i32 %a, i32 %b, i32 %c) local_unnamed_addr #0 {
 ; ARM-LABEL: fn1:
 ; ARM:       @ %bb.0: @ %entry
+; ARM-NEXT:    rsb r2, r2, #0
 ; ARM-NEXT:    adds r0, r1, r0
-; ARM-NEXT:    mov r3, #0
-; ARM-NEXT:    adc r0, r3, #0
 ; ARM-NEXT:    movw r1, #65535
-; ARM-NEXT:    sub r0, r0, r2
+; ARM-NEXT:    sxth r2, r2
+; ARM-NEXT:    adc r0, r2, #0
 ; ARM-NEXT:    uxth r0, r0
 ; ARM-NEXT:    cmp r0, r1
 ; ARM-NEXT:    bxeq lr
@@ -24,11 +24,12 @@ define void @fn1(i32 %a, i32 %b, i32 %c) local_unnamed_addr #0 {
 ;
 ; THUMBV6M-LABEL: fn1:
 ; THUMBV6M:       @ %bb.0: @ %entry
+; THUMBV6M-NEXT:    rsbs r2, r2, #0
+; THUMBV6M-NEXT:    sxth r2, r2
 ; THUMBV6M-NEXT:    movs r3, #0
 ; THUMBV6M-NEXT:    adds r0, r1, r0
-; THUMBV6M-NEXT:    adcs r3, r3
-; THUMBV6M-NEXT:    subs r0, r3, r2
-; THUMBV6M-NEXT:    uxth r0, r0
+; THUMBV6M-NEXT:    adcs r3, r2
+; THUMBV6M-NEXT:    uxth r0, r3
 ; THUMBV6M-NEXT:    ldr r1, .LCPI0_0
 ; THUMBV6M-NEXT:    cmp r0, r1
 ; THUMBV6M-NEXT:    beq .LBB0_2
@@ -44,11 +45,12 @@ define void @fn1(i32 %a, i32 %b, i32 %c) local_unnamed_addr #0 {
 ;
 ; THUMBV8M-BASE-LABEL: fn1:
 ; THUMBV8M-BASE:       @ %bb.0: @ %entry
+; THUMBV8M-BASE-NEXT:    rsbs r2, r2, #0
+; THUMBV8M-BASE-NEXT:    sxth r2, r2
 ; THUMBV8M-BASE-NEXT:    movs r3, #0
 ; THUMBV8M-BASE-NEXT:    adds r0, r1, r0
-; THUMBV8M-BASE-NEXT:    adcs r3, r3
-; THUMBV8M-BASE-NEXT:    subs r0, r3, r2
-; THUMBV8M-BASE-NEXT:    uxth r0, r0
+; THUMBV8M-BASE-NEXT:    adcs r3, r2
+; THUMBV8M-BASE-NEXT:    uxth r0, r3
 ; THUMBV8M-BASE-NEXT:    movw r1, #65535
 ; THUMBV8M-BASE-NEXT:    cmp r0, r1
 ; THUMBV8M-BASE-NEXT:    beq .LBB0_2
@@ -60,11 +62,11 @@ define void @fn1(i32 %a, i32 %b, i32 %c) local_unnamed_addr #0 {
 ;
 ; THUMB-LABEL: fn1:
 ; THUMB:       @ %bb.0: @ %entry
+; THUMB-NEXT:    rsbs r2, r2, #0
 ; THUMB-NEXT:    adds r0, r0, r1
-; THUMB-NEXT:    mov.w r3, #0
-; THUMB-NEXT:    adc r0, r3, #0
 ; THUMB-NEXT:    movw r1, #65535
-; THUMB-NEXT:    subs r0, r0, r2
+; THUMB-NEXT:    sxth r2, r2
+; THUMB-NEXT:    adc r0, r2, #0
 ; THUMB-NEXT:    uxth r0, r0
 ; THUMB-NEXT:    cmp r0, r1
 ; THUMB-NEXT:    it eq
diff --git a/llvm/test/CodeGen/ARM/addsubo-legalization.ll b/llvm/test/CodeGen/ARM/addsubo-legalization.ll
index c899e5f73..e3a48ed0c 100644
--- a/llvm/test/CodeGen/ARM/addsubo-legalization.ll
+++ b/llvm/test/CodeGen/ARM/addsubo-legalization.ll
@@ -95,76 +95,19 @@ define <2 x i1> @usubo(<2 x i64> *%ptr, <2 x i64> *%ptr2) {
 define <2 x i1> @saddo(<2 x i64> *%ptr, <2 x i64> *%ptr2) {
 ; CHECK-LABEL: saddo:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    push {r4, r5, r6, r7, lr}
 ; CHECK-NEXT:    vld1.64 {d16, d17}, [r1]
-; CHECK-NEXT:    movs r5, #0
-; CHECK-NEXT:    movs r6, #0
-; CHECK-NEXT:    movs r3, #0
-; CHECK-NEXT:    vmov.32 r1, d16[1]
 ; CHECK-NEXT:    vld1.64 {d18, d19}, [r0]
-; CHECK-NEXT:    vmov.32 r2, d17[1]
+; CHECK-NEXT:    vqadd.s64 q10, q9, q8
 ; CHECK-NEXT:    vadd.i64 q8, q9, q8
-; CHECK-NEXT:    vmov.32 r12, d18[1]
-; CHECK-NEXT:    vmov.32 r4, d19[1]
-; CHECK-NEXT:    vmov.32 lr, d16[1]
-; CHECK-NEXT:    vmov.32 r7, d17[1]
-; CHECK-NEXT:    cmp.w r1, #-1
-; CHECK-NEXT:    mov.w r1, #0
-; CHECK-NEXT:    it gt
-; CHECK-NEXT:    movgt r1, #1
-; CHECK-NEXT:    cmp r1, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r1, #-1
-; CHECK-NEXT:    cmp.w r2, #-1
-; CHECK-NEXT:    mov.w r2, #0
-; CHECK-NEXT:    it gt
-; CHECK-NEXT:    movgt r2, #1
-; CHECK-NEXT:    cmp.w r12, #-1
-; CHECK-NEXT:    it gt
-; CHECK-NEXT:    movgt r5, #1
-; CHECK-NEXT:    cmp r5, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r5, #-1
-; CHECK-NEXT:    cmp.w r4, #-1
-; CHECK-NEXT:    mov.w r4, #0
-; CHECK-NEXT:    it gt
-; CHECK-NEXT:    movgt r4, #1
-; CHECK-NEXT:    cmp.w lr, #-1
-; CHECK-NEXT:    it gt
-; CHECK-NEXT:    movgt r6, #1
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r6, #-1
-; CHECK-NEXT:    cmp.w r7, #-1
-; CHECK-NEXT:    it gt
-; CHECK-NEXT:    movgt r3, #1
-; CHECK-NEXT:    cmp r3, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r3, #-1
-; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    vdup.32 d19, r3
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r4, #-1
-; CHECK-NEXT:    cmp r2, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r2, #-1
-; CHECK-NEXT:    vdup.32 d23, r2
-; CHECK-NEXT:    vdup.32 d21, r4
-; CHECK-NEXT:    vdup.32 d18, r6
-; CHECK-NEXT:    vdup.32 d22, r1
-; CHECK-NEXT:    vdup.32 d20, r5
-; CHECK-NEXT:    vceq.i32 q9, q10, q9
+; CHECK-NEXT:    vceq.i32 q9, q8, q10
 ; CHECK-NEXT:    vst1.64 {d16, d17}, [r0]
-; CHECK-NEXT:    vceq.i32 q10, q10, q11
-; CHECK-NEXT:    vrev64.32 q11, q9
-; CHECK-NEXT:    vrev64.32 q12, q10
-; CHECK-NEXT:    vand q9, q9, q11
-; CHECK-NEXT:    vand q10, q10, q12
-; CHECK-NEXT:    vbic q9, q10, q9
+; CHECK-NEXT:    vrev64.32 q10, q9
+; CHECK-NEXT:    vand q9, q9, q10
+; CHECK-NEXT:    vmvn q9, q9
 ; CHECK-NEXT:    vmovn.i64 d18, q9
 ; CHECK-NEXT:    vmov r2, r1, d18
 ; CHECK-NEXT:    mov r0, r2
-; CHECK-NEXT:    pop {r4, r5, r6, r7, pc}
+; CHECK-NEXT:    bx lr
   %x = load <2 x i64>, <2 x i64>* %ptr, align 8
   %y = load <2 x i64>, <2 x i64>* %ptr2, align 8
   %s = call {<2 x i64>, <2 x i1>} @llvm.sadd.with.overflow.v2i64(<2 x i64> %x, <2 x i64> %y)
@@ -177,77 +120,19 @@ define <2 x i1> @saddo(<2 x i64> *%ptr, <2 x i64> *%ptr2) {
 define <2 x i1> @ssubo(<2 x i64> *%ptr, <2 x i64> *%ptr2) {
 ; CHECK-LABEL: ssubo:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    push {r4, r5, r6, r7, lr}
-; CHECK-NEXT:    vld1.64 {d18, d19}, [r1]
-; CHECK-NEXT:    movs r5, #0
-; CHECK-NEXT:    movs r6, #0
-; CHECK-NEXT:    movs r3, #0
-; CHECK-NEXT:    vld1.64 {d20, d21}, [r0]
-; CHECK-NEXT:    vsub.i64 q8, q10, q9
-; CHECK-NEXT:    vmov.32 r12, d20[1]
-; CHECK-NEXT:    vmov.32 lr, d21[1]
-; CHECK-NEXT:    vmov.32 r1, d16[1]
-; CHECK-NEXT:    vmov.32 r2, d17[1]
-; CHECK-NEXT:    vmov.32 r4, d18[1]
-; CHECK-NEXT:    vmov.32 r7, d19[1]
-; CHECK-NEXT:    cmp.w r1, #-1
-; CHECK-NEXT:    mov.w r1, #0
-; CHECK-NEXT:    it gt
-; CHECK-NEXT:    movgt r1, #1
-; CHECK-NEXT:    cmp r1, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r1, #-1
-; CHECK-NEXT:    cmp.w r2, #-1
-; CHECK-NEXT:    mov.w r2, #0
-; CHECK-NEXT:    it gt
-; CHECK-NEXT:    movgt r2, #1
-; CHECK-NEXT:    cmp.w r12, #-1
-; CHECK-NEXT:    it gt
-; CHECK-NEXT:    movgt r5, #1
-; CHECK-NEXT:    cmp r5, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r5, #-1
-; CHECK-NEXT:    cmp.w lr, #-1
-; CHECK-NEXT:    it gt
-; CHECK-NEXT:    movgt r6, #1
-; CHECK-NEXT:    cmp.w r4, #-1
-; CHECK-NEXT:    mov.w r4, #0
-; CHECK-NEXT:    it gt
-; CHECK-NEXT:    movgt r4, #1
-; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r4, #-1
-; CHECK-NEXT:    cmp.w r7, #-1
-; CHECK-NEXT:    it gt
-; CHECK-NEXT:    movgt r3, #1
-; CHECK-NEXT:    cmp r3, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r3, #-1
-; CHECK-NEXT:    vdup.32 d19, r3
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r6, #-1
-; CHECK-NEXT:    vdup.32 d21, r6
-; CHECK-NEXT:    cmp r2, #0
-; CHECK-NEXT:    vdup.32 d18, r4
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r2, #-1
-; CHECK-NEXT:    vdup.32 d23, r2
-; CHECK-NEXT:    vdup.32 d20, r5
-; CHECK-NEXT:    vdup.32 d22, r1
-; CHECK-NEXT:    vceq.i32 q9, q10, q9
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r1]
+; CHECK-NEXT:    vld1.64 {d18, d19}, [r0]
+; CHECK-NEXT:    vqsub.s64 q10, q9, q8
+; CHECK-NEXT:    vsub.i64 q8, q9, q8
+; CHECK-NEXT:    vceq.i32 q9, q8, q10
 ; CHECK-NEXT:    vst1.64 {d16, d17}, [r0]
-; CHECK-NEXT:    vceq.i32 q10, q10, q11
-; CHECK-NEXT:    vrev64.32 q11, q9
-; CHECK-NEXT:    vrev64.32 q12, q10
-; CHECK-NEXT:    vand q9, q9, q11
-; CHECK-NEXT:    vand q10, q10, q12
+; CHECK-NEXT:    vrev64.32 q10, q9
+; CHECK-NEXT:    vand q9, q9, q10
 ; CHECK-NEXT:    vmvn q9, q9
-; CHECK-NEXT:    vbic q9, q9, q10
 ; CHECK-NEXT:    vmovn.i64 d18, q9
 ; CHECK-NEXT:    vmov r2, r1, d18
 ; CHECK-NEXT:    mov r0, r2
-; CHECK-NEXT:    pop {r4, r5, r6, r7, pc}
+; CHECK-NEXT:    bx lr
   %x = load <2 x i64>, <2 x i64>* %ptr, align 8
   %y = load <2 x i64>, <2 x i64>* %ptr2, align 8
   %s = call {<2 x i64>, <2 x i1>} @llvm.ssub.with.overflow.v2i64(<2 x i64> %x, <2 x i64> %y)
diff --git a/llvm/test/CodeGen/ARM/arm-shrink-wrapping.ll b/llvm/test/CodeGen/ARM/arm-shrink-wrapping.ll
index 4b043362a..e6fc02970 100644
--- a/llvm/test/CodeGen/ARM/arm-shrink-wrapping.ll
+++ b/llvm/test/CodeGen/ARM/arm-shrink-wrapping.ll
@@ -60,7 +60,7 @@
 ; pop {r7, pc}
 ;
 ; bx lr
-define i32 @foo(i32 %a, i32 %b) "no-frame-pointer-elim"="true" {
+define i32 @foo(i32 %a, i32 %b) "frame-pointer"="all" {
 ; ARM-ENABLE-LABEL: foo:
 ; ARM-ENABLE:       @ %bb.0:
 ; ARM-ENABLE-NEXT:    cmp r0, r1
@@ -188,7 +188,7 @@ declare i32 @doSomething(i32, i32*)
 ; pop {r4, r7, pc}
 ;
 ; bx lr
-define i32 @freqSaveAndRestoreOutsideLoop(i32 %cond, i32 %N) "no-frame-pointer-elim"="true" {
+define i32 @freqSaveAndRestoreOutsideLoop(i32 %cond, i32 %N) "frame-pointer"="all" {
 ; ARM-ENABLE-LABEL: freqSaveAndRestoreOutsideLoop:
 ; ARM-ENABLE:       @ %bb.0: @ %entry
 ; ARM-ENABLE-NEXT:    cmp r0, #0
@@ -347,7 +347,7 @@ declare i32 @something(...)
 ; @ %for.exit
 ; nop
 ; pop {r4
-define i32 @freqSaveAndRestoreOutsideLoop2(i32 %cond) "no-frame-pointer-elim"="true" {
+define i32 @freqSaveAndRestoreOutsideLoop2(i32 %cond) "frame-pointer"="all" {
 ; ARM-LABEL: freqSaveAndRestoreOutsideLoop2:
 ; ARM:       @ %bb.0: @ %entry
 ; ARM-NEXT:    push {r4, r7, lr}
@@ -553,7 +553,7 @@ for.end:                                          ; preds = %for.body
 ; pop {r4, r7, pc}
 ;
 ; bx lr
-define i32 @loopInfoSaveOutsideLoop(i32 %cond, i32 %N) "no-frame-pointer-elim"="true" {
+define i32 @loopInfoSaveOutsideLoop(i32 %cond, i32 %N) "frame-pointer"="all" {
 ; ARM-ENABLE-LABEL: loopInfoSaveOutsideLoop:
 ; ARM-ENABLE:       @ %bb.0: @ %entry
 ; ARM-ENABLE-NEXT:    cmp r0, #0
@@ -749,7 +749,7 @@ declare void @somethingElse(...)
 ; pop {r4, r7, pc}
 ;
 ; bx lr
-define i32 @loopInfoRestoreOutsideLoop(i32 %cond, i32 %N) "no-frame-pointer-elim"="true" nounwind {
+define i32 @loopInfoRestoreOutsideLoop(i32 %cond, i32 %N) "frame-pointer"="all" nounwind {
 ; ARM-ENABLE-LABEL: loopInfoRestoreOutsideLoop:
 ; ARM-ENABLE:       @ %bb.0: @ %entry
 ; ARM-ENABLE-NEXT:    cmp r0, #0
@@ -960,7 +960,7 @@ entry:
 ; pop {r4, r7, pc}
 ;
 ; bx lr
-define i32 @inlineAsm(i32 %cond, i32 %N) "no-frame-pointer-elim"="true" {
+define i32 @inlineAsm(i32 %cond, i32 %N) "frame-pointer"="all" {
 ; ARM-ENABLE-LABEL: inlineAsm:
 ; ARM-ENABLE:       @ %bb.0: @ %entry
 ; ARM-ENABLE-NEXT:    cmp r0, #0
@@ -1138,7 +1138,7 @@ if.end:                                           ; preds = %for.body, %if.else
 ; mov sp, r7
 ; add sp, #12
 ; pop {r7, pc}
-define i32 @callVariadicFunc(i32 %cond, i32 %N) "no-frame-pointer-elim"="true" {
+define i32 @callVariadicFunc(i32 %cond, i32 %N) "frame-pointer"="all" {
 ; ARM-ENABLE-LABEL: callVariadicFunc:
 ; ARM-ENABLE:       @ %bb.0: @ %entry
 ; ARM-ENABLE-NEXT:    cmp r0, #0
@@ -1270,7 +1270,7 @@ declare i32 @someVariadicFunc(i32, ...)
 ;
 ; bl{{x?}} _abort
 ; pop
-define i32 @noreturn(i8 signext %bad_thing) "no-frame-pointer-elim"="true" {
+define i32 @noreturn(i8 signext %bad_thing) "frame-pointer"="all" {
 ; ARM-ENABLE-LABEL: noreturn:
 ; ARM-ENABLE:       @ %bb.0: @ %entry
 ; ARM-ENABLE-NEXT:    cmp r0, #0
@@ -1354,7 +1354,7 @@ attributes #0 = { noreturn nounwind }
 ; The only condition for this test is the compilation finishes correctly.
 ; infiniteloop
 ; pop
-define void @infiniteloop() "no-frame-pointer-elim"="true" {
+define void @infiniteloop() "frame-pointer"="all" {
 ; ARM-LABEL: infiniteloop:
 ; ARM:       @ %bb.0: @ %entry
 ; ARM-NEXT:    push {r4, r5, r7, lr}
@@ -1509,7 +1509,7 @@ if.end:
 ; Another infinite loop test this time with a body bigger than just one block.
 ; infiniteloop2
 ; pop
-define void @infiniteloop2() "no-frame-pointer-elim"="true" {
+define void @infiniteloop2() "frame-pointer"="all" {
 entry:
   br i1 undef, label %if.then, label %if.end
 
@@ -1539,7 +1539,7 @@ if.end:
 ; Another infinite loop test this time with two nested infinite loop.
 ; infiniteloop3
 ; bx lr
-define void @infiniteloop3() "no-frame-pointer-elim"="true" {
+define void @infiniteloop3() "frame-pointer"="all" {
 ; ARM-LABEL: infiniteloop3:
 ; ARM:       @ %bb.0: @ %entry
 ; ARM-NEXT:    mov r0, #0
@@ -1759,7 +1759,7 @@ declare double @llvm.pow.f64(double, double)
 ; info (like labels named 'line_table) not because it's found a bl instruction.
 ;
 ; bl
-define float @debug_info(float %gamma, float %slopeLimit, i1 %or.cond, double %tmp) "no-frame-pointer-elim"="true" {
+define float @debug_info(float %gamma, float %slopeLimit, i1 %or.cond, double %tmp) "frame-pointer"="all" {
 ; ARM-LABEL: debug_info:
 ; ARM:       @ %bb.0: @ %bb
 ; ARM-NEXT:    push {r4, r7, lr}
@@ -1781,7 +1781,7 @@ define float @debug_info(float %gamma, float %slopeLimit, i1 %or.cond, double %t
 ; ARM-NEXT:    vmov.f32 s0, #1.000000e+00
 ; ARM-NEXT:    vmov.f64 d16, #1.000000e+00
 ; ARM-NEXT:    vadd.f64 d16, d9, d16
-; ARM-NEXT:    vcmpe.f32 s16, s0
+; ARM-NEXT:    vcmp.f32 s16, s0
 ; ARM-NEXT:    vmrs APSR_nzcv, fpscr
 ; ARM-NEXT:    vmov d17, r0, r1
 ; ARM-NEXT:    vmov.f64 d18, d9
@@ -1828,7 +1828,7 @@ define float @debug_info(float %gamma, float %slopeLimit, i1 %or.cond, double %t
 ; THUMB-NEXT:    vmov.f32 s0, #1.000000e+00
 ; THUMB-NEXT:    vmov.f64 d16, #1.000000e+00
 ; THUMB-NEXT:    vmov.f64 d18, d9
-; THUMB-NEXT:    vcmpe.f32 s16, s0
+; THUMB-NEXT:    vcmp.f32 s16, s0
 ; THUMB-NEXT:    vadd.f64 d16, d9, d16
 ; THUMB-NEXT:    vmrs APSR_nzcv, fpscr
 ; THUMB-NEXT:    it gt
diff --git a/llvm/test/CodeGen/ARM/arm-storebytesmerge.ll b/llvm/test/CodeGen/ARM/arm-storebytesmerge.ll
index c159ca49c..fec6ea7ae 100644
--- a/llvm/test/CodeGen/ARM/arm-storebytesmerge.ll
+++ b/llvm/test/CodeGen/ARM/arm-storebytesmerge.ll
@@ -337,5 +337,5 @@ define arm_aapcs_vfpcc void @test(i8* %v50) #0 {
   ret void
   }
 
-attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "denormal-fp-math"="preserve-sign" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="true" "no-jump-tables"="false" "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="true" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="cortex-m7" "target-features"="-d32,+dsp,+fp-armv8,+hwdiv,+thumb-mode,-crc,-crypto,-dotprod,-fullfp16,-hwdiv-arm,-neon,-ras" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "denormal-fp-math"="preserve-sign" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="true" "no-jump-tables"="false" "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="true" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="cortex-m7" "target-features"="-d32,+dsp,+fp-armv8,+hwdiv,+thumb-mode,-crc,-crypto,-dotprod,-fullfp16,-hwdiv-arm,-neon,-ras" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
diff --git a/llvm/test/CodeGen/ARM/bfi.ll b/llvm/test/CodeGen/ARM/bfi.ll
index 31eff16fc..725e17332 100644
--- a/llvm/test/CodeGen/ARM/bfi.ll
+++ b/llvm/test/CodeGen/ARM/bfi.ll
@@ -1,14 +1,21 @@
-; RUN: llc -mtriple=arm -mattr=+v6t2 %s -o - | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=arm -mattr=+v6t2 | FileCheck %s
 
 %struct.F = type { [3 x i8], i8 }
 
 @X = common global %struct.F zeroinitializer, align 4 ; <%struct.F*> [#uses=1]
 
 define void @f1([1 x i32] %f.coerce0) nounwind {
+; CHECK-LABEL: f1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    movw r0, :lower16:X
+; CHECK-NEXT:    mov r2, #10
+; CHECK-NEXT:    movt r0, :upper16:X
+; CHECK-NEXT:    ldr r1, [r0]
+; CHECK-NEXT:    bfi r1, r2, #22, #4
+; CHECK-NEXT:    str r1, [r0]
+; CHECK-NEXT:    bx lr
 entry:
-; CHECK: f1
-; CHECK: mov r2, #10
-; CHECK: bfi r1, r2, #22, #4
   %0 = load i32, i32* bitcast (%struct.F* @X to i32*), align 4 ; <i32> [#uses=1]
   %1 = and i32 %0, -62914561                      ; <i32> [#uses=1]
   %2 = or i32 %1, 41943040                        ; <i32> [#uses=1]
@@ -17,10 +24,12 @@ entry:
 }
 
 define i32 @f2(i32 %A, i32 %B) nounwind {
+; CHECK-LABEL: f2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    lsr r1, r1, #7
+; CHECK-NEXT:    bfi r0, r1, #7, #16
+; CHECK-NEXT:    bx lr
 entry:
-; CHECK: f2
-; CHECK: lsr{{.*}}#7
-; CHECK: bfi r0, r1, #7, #16
   %and = and i32 %A, -8388481                     ; <i32> [#uses=1]
   %and2 = and i32 %B, 8388480                     ; <i32> [#uses=1]
   %or = or i32 %and2, %and                        ; <i32> [#uses=1]
@@ -28,10 +37,13 @@ entry:
 }
 
 define i32 @f3(i32 %A, i32 %B) nounwind {
+; CHECK-LABEL: f3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    lsr r0, r0, #7
+; CHECK-NEXT:    bfi r1, r0, #7, #16
+; CHECK-NEXT:    mov r0, r1
+; CHECK-NEXT:    bx lr
 entry:
-; CHECK: f3
-; CHECK: lsr{{.*}} #7
-; CHECK: bfi {{.*}}, #7, #16
   %and = and i32 %A, 8388480                      ; <i32> [#uses=1]
   %and2 = and i32 %B, -8388481                    ; <i32> [#uses=1]
   %or = or i32 %and2, %and                        ; <i32> [#uses=1]
@@ -40,9 +52,12 @@ entry:
 
 ; rdar://8752056
 define i32 @f4(i32 %a) nounwind {
-; CHECK: f4
-; CHECK: movw [[R1:r[0-9]+]], #3137
-; CHECK: bfi [[R1]], {{r[0-9]+}}, #15, #5
+; CHECK-LABEL: f4:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    movw r1, #3137
+; CHECK-NEXT:    bfi r1, r0, #15, #5
+; CHECK-NEXT:    mov r0, r1
+; CHECK-NEXT:    bx lr
   %1 = shl i32 %a, 15
   %ins7 = and i32 %1, 1015808
   %ins12 = or i32 %ins7, 3137
@@ -51,10 +66,11 @@ define i32 @f4(i32 %a) nounwind {
 
 ; rdar://8458663
 define i32 @f5(i32 %a, i32 %b) nounwind {
-entry:
 ; CHECK-LABEL: f5:
-; CHECK-NOT: bfc
-; CHECK: bfi r0, r1, #20, #4
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    bfi r0, r1, #20, #4
+; CHECK-NEXT:    bx lr
+entry:
   %0 = and i32 %a, -15728641
   %1 = shl i32 %b, 20
   %2 = and i32 %1, 15728640
@@ -64,10 +80,11 @@ entry:
 
 ; rdar://9609030
 define i32 @f6(i32 %a, i32 %b) nounwind readnone {
-entry:
 ; CHECK-LABEL: f6:
-; CHECK-NOT: bic
-; CHECK: bfi r0, r1, #8, #9
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    bfi r0, r1, #8, #9
+; CHECK-NEXT:    bx lr
+entry:
   %and = and i32 %a, -130817
   %and2 = shl i32 %b, 8
   %shl = and i32 %and2, 130816
@@ -77,7 +94,11 @@ entry:
 
 define i32 @f7(i32 %x, i32 %y) {
 ; CHECK-LABEL: f7:
-; CHECK: bfi r0, r2, #4, #1
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    lsr r2, r0, #2
+; CHECK-NEXT:    bic r0, r1, #255
+; CHECK-NEXT:    bfi r0, r2, #4, #1
+; CHECK-NEXT:    bx lr
   %y2 = and i32 %y, 4294967040 ; 0xFFFFFF00
   %and = and i32 %x, 4
   %or = or i32 %y2, 16
@@ -88,8 +109,12 @@ define i32 @f7(i32 %x, i32 %y) {
 
 define i32 @f8(i32 %x, i32 %y) {
 ; CHECK-LABEL: f8:
-; CHECK: bfi r0, r2, #4, #1
-; CHECK: bfi r0, r2, #5, #1
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    lsr r2, r0, #2
+; CHECK-NEXT:    bic r0, r1, #255
+; CHECK-NEXT:    bfi r0, r2, #4, #1
+; CHECK-NEXT:    bfi r0, r2, #5, #1
+; CHECK-NEXT:    bx lr
   %y2 = and i32 %y, 4294967040 ; 0xFFFFFF00
   %and = and i32 %x, 4
   %or = or i32 %y2, 48
@@ -100,7 +125,12 @@ define i32 @f8(i32 %x, i32 %y) {
 
 define i32 @f9(i32 %x, i32 %y) {
 ; CHECK-LABEL: f9:
-; CHECK-NOT: bfi
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    bic r1, r1, #255
+; CHECK-NEXT:    tst r0, #4
+; CHECK-NEXT:    orreq r1, r1, #48
+; CHECK-NEXT:    mov r0, r1
+; CHECK-NEXT:    bx lr
   %y2 = and i32 %y, 4294967040 ; 0xFFFFFF00
   %and = and i32 %x, 4
   %or = or i32 %y2, 48
@@ -111,7 +141,11 @@ define i32 @f9(i32 %x, i32 %y) {
 
 define i32 @f10(i32 %x, i32 %y) {
 ; CHECK-LABEL: f10:
-; CHECK: bfi r0, r2, #4, #2
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    lsr r2, r0, #1
+; CHECK-NEXT:    bic r0, r1, #255
+; CHECK-NEXT:    bfi r0, r2, #4, #2
+; CHECK-NEXT:    bx lr
   %y2 = and i32 %y, 4294967040 ; 0xFFFFFF00
   %and = and i32 %x, 4
   %or = or i32 %y2, 32
@@ -128,7 +162,11 @@ define i32 @f10(i32 %x, i32 %y) {
 
 define i32 @f11(i32 %x, i32 %y) {
 ; CHECK-LABEL: f11:
-; CHECK: bfi r0, r2, #4, #3
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    lsr r2, r0, #1
+; CHECK-NEXT:    bic r0, r1, #255
+; CHECK-NEXT:    bfi r0, r2, #4, #3
+; CHECK-NEXT:    bx lr
   %y2 = and i32 %y, 4294967040 ; 0xFFFFFF00
   %and = and i32 %x, 4
   %or = or i32 %y2, 32
@@ -150,7 +188,11 @@ define i32 @f11(i32 %x, i32 %y) {
 
 define i32 @f12(i32 %x, i32 %y) {
 ; CHECK-LABEL: f12:
-; CHECK: bfi r0, r2, #4, #1
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    lsr r2, r0, #2
+; CHECK-NEXT:    bic r0, r1, #255
+; CHECK-NEXT:    bfi r0, r2, #4, #1
+; CHECK-NEXT:    bx lr
   %y2 = and i32 %y, 4294967040 ; 0xFFFFFF00
   %and = and i32 %x, 4
   %or = or i32 %y2, 16
@@ -161,7 +203,12 @@ define i32 @f12(i32 %x, i32 %y) {
 
 define i32 @f13(i32 %x, i32 %y) {
 ; CHECK-LABEL: f13:
-; CHECK-NOT: bfi
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    and r2, r0, #4
+; CHECK-NEXT:    bic r0, r1, #255
+; CHECK-NEXT:    cmp r2, #42
+; CHECK-NEXT:    orrne r0, r0, #16
+; CHECK-NEXT:    bx lr
   %y2 = and i32 %y, 4294967040 ; 0xFFFFFF00
   %and = and i32 %x, 4
   %or = or i32 %y2, 16
diff --git a/llvm/test/CodeGen/ARM/bswap16.ll b/llvm/test/CodeGen/ARM/bswap16.ll
index dc0e468b7..dcaa029c2 100644
--- a/llvm/test/CodeGen/ARM/bswap16.ll
+++ b/llvm/test/CodeGen/ARM/bswap16.ll
@@ -1,42 +1,43 @@
-; RUN: llc -mtriple=arm-darwin  -mattr=v6 < %s | FileCheck %s
-; RUN: llc -mtriple=thumb-darwin  -mattr=v6 < %s | FileCheck %s
-
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=arm-darwin -mattr=v6 | FileCheck %s --check-prefixes=CHECK,ARM
+; RUN: llc < %s -mtriple=thumb-darwin -mattr=v6 | FileCheck %s --check-prefixes=CHECK,THUMB
 
 define void @test1(i16* nocapture %data) {
+; CHECK-LABEL: test1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    ldrh r1, [r0]
+; CHECK-NEXT:    rev16 r1, r1
+; CHECK-NEXT:    strh r1, [r0]
+; CHECK-NEXT:    bx lr
 entry:
   %0 = load i16, i16* %data, align 2
   %1 = tail call i16 @llvm.bswap.i16(i16 %0)
   store i16 %1, i16* %data, align 2
   ret void
-
-  ; CHECK-LABEL: test1:
-  ; CHECK: ldrh r[[R1:[0-9]+]], [r0]
-  ; CHECK: rev16 r[[R1]], r[[R1]]
-  ; CHECK: strh r[[R1]], [r0]
 }
 
-
 define void @test2(i16* nocapture %data, i16 zeroext %in) {
+; CHECK-LABEL: test2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    rev16 r1, r1
+; CHECK-NEXT:    strh r1, [r0]
+; CHECK-NEXT:    bx lr
 entry:
   %0 = tail call i16 @llvm.bswap.i16(i16 %in)
   store i16 %0, i16* %data, align 2
   ret void
-
-  ; CHECK-LABEL: test2:
-  ; CHECK: rev16 r[[R1:[0-9]+]], r1
-  ; CHECK: strh r[[R1]], [r0]
 }
 
-
 define i16 @test3(i16* nocapture %data) {
+; CHECK-LABEL: test3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    ldrh r0, [r0]
+; CHECK-NEXT:    rev16 r0, r0
+; CHECK-NEXT:    bx lr
 entry:
   %0 = load i16, i16* %data, align 2
   %1 = tail call i16 @llvm.bswap.i16(i16 %0)
   ret i16 %1
-
-  ; CHECK-LABEL: test3:
-  ; CHECK: ldrh r[[R0:[0-9]+]], [r0]
-  ; CHECK: rev16 r[[R0]], r0
 }
 
 declare i16 @llvm.bswap.i16(i16)
diff --git a/llvm/test/CodeGen/ARM/build-attributes-fn-attr6.ll b/llvm/test/CodeGen/ARM/build-attributes-fn-attr6.ll
index 61741c000..b2f12b984 100644
--- a/llvm/test/CodeGen/ARM/build-attributes-fn-attr6.ll
+++ b/llvm/test/CodeGen/ARM/build-attributes-fn-attr6.ll
@@ -5,6 +5,7 @@
 ; functions have inconsistent values, and that a default is returned. 
 
 ; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a15  | FileCheck %s
+; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a15 -enable-ipra | FileCheck %s
 
 ; CHECK: .eabi_attribute 20, 1
 
diff --git a/llvm/test/CodeGen/ARM/build-attributes-optimization-minsize.ll b/llvm/test/CodeGen/ARM/build-attributes-optimization-minsize.ll
index c008222a9..3e2124e4b 100644
--- a/llvm/test/CodeGen/ARM/build-attributes-optimization-minsize.ll
+++ b/llvm/test/CodeGen/ARM/build-attributes-optimization-minsize.ll
@@ -2,9 +2,9 @@
 ; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O1 | FileCheck %s
 ; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O3 | FileCheck %s
 
-; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O0 -filetype obj -o - | llvm-readobj --arm-attributes - | FileCheck %s --check-prefix=CHECK-OBJ
-; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O1 -filetype obj -o - | llvm-readobj --arm-attributes - | FileCheck %s --check-prefix=CHECK-OBJ
-; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O3 -filetype obj -o - | llvm-readobj --arm-attributes - | FileCheck %s --check-prefix=CHECK-OBJ
+; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O0 -filetype obj -o - | llvm-readobj --arch-specific - | FileCheck %s --check-prefix=CHECK-OBJ
+; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O1 -filetype obj -o - | llvm-readobj --arch-specific - | FileCheck %s --check-prefix=CHECK-OBJ
+; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O3 -filetype obj -o - | llvm-readobj --arch-specific - | FileCheck %s --check-prefix=CHECK-OBJ
 
 ; CHECK: .eabi_attribute 30, 4	@ Tag_ABI_optimization_goals
 ; CHECK-OBJ:          TagName: ABI_optimization_goals
diff --git a/llvm/test/CodeGen/ARM/build-attributes-optimization-mixed.ll b/llvm/test/CodeGen/ARM/build-attributes-optimization-mixed.ll
index c144e1846..734223488 100644
--- a/llvm/test/CodeGen/ARM/build-attributes-optimization-mixed.ll
+++ b/llvm/test/CodeGen/ARM/build-attributes-optimization-mixed.ll
@@ -2,9 +2,9 @@
 ; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O1 | FileCheck %s
 ; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O3 | FileCheck %s
 
-; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O0 -filetype obj -o - | llvm-readobj --arm-attributes - | FileCheck %s
-; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O1 -filetype obj -o - | llvm-readobj --arm-attributes - | FileCheck %s
-; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O3 -filetype obj -o - | llvm-readobj --arm-attributes - | FileCheck %s
+; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O0 -filetype obj -o - | llvm-readobj --arch-specific - | FileCheck %s
+; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O1 -filetype obj -o - | llvm-readobj --arch-specific - | FileCheck %s
+; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O3 -filetype obj -o - | llvm-readobj --arch-specific - | FileCheck %s
 
 ; CHECK-NOT: .eabi_attribute 30
 ; CHECK-NOT: Tag_ABI_optimization_goals
diff --git a/llvm/test/CodeGen/ARM/build-attributes-optimization-optnone.ll b/llvm/test/CodeGen/ARM/build-attributes-optimization-optnone.ll
index f779402be..7f52b869c 100644
--- a/llvm/test/CodeGen/ARM/build-attributes-optimization-optnone.ll
+++ b/llvm/test/CodeGen/ARM/build-attributes-optimization-optnone.ll
@@ -2,9 +2,9 @@
 ; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O1 | FileCheck %s
 ; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O3 | FileCheck %s
 
-; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O0 -filetype obj -o - | llvm-readobj --arm-attributes - | FileCheck %s --check-prefix=CHECK-OBJ
-; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O1 -filetype obj -o - | llvm-readobj --arm-attributes - | FileCheck %s --check-prefix=CHECK-OBJ
-; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O3 -filetype obj -o - | llvm-readobj --arm-attributes - | FileCheck %s --check-prefix=CHECK-OBJ
+; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O0 -filetype obj -o - | llvm-readobj --arch-specific - | FileCheck %s --check-prefix=CHECK-OBJ
+; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O1 -filetype obj -o - | llvm-readobj --arch-specific - | FileCheck %s --check-prefix=CHECK-OBJ
+; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O3 -filetype obj -o - | llvm-readobj --arch-specific - | FileCheck %s --check-prefix=CHECK-OBJ
 
 ; CHECK: .eabi_attribute 30, 6	@ Tag_ABI_optimization_goals
 ; CHECK-OBJ:          TagName: ABI_optimization_goals
diff --git a/llvm/test/CodeGen/ARM/build-attributes-optimization-optsize.ll b/llvm/test/CodeGen/ARM/build-attributes-optimization-optsize.ll
index 3c7df7cd7..89b1973a8 100644
--- a/llvm/test/CodeGen/ARM/build-attributes-optimization-optsize.ll
+++ b/llvm/test/CodeGen/ARM/build-attributes-optimization-optsize.ll
@@ -2,9 +2,9 @@
 ; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O1 | FileCheck %s
 ; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O3 | FileCheck %s
 
-; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O0 -filetype obj -o - | llvm-readobj --arm-attributes - | FileCheck %s --check-prefix=CHECK-OBJ
-; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O1 -filetype obj -o - | llvm-readobj --arm-attributes - | FileCheck %s --check-prefix=CHECK-OBJ
-; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O3 -filetype obj -o - | llvm-readobj --arm-attributes - | FileCheck %s --check-prefix=CHECK-OBJ
+; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O0 -filetype obj -o - | llvm-readobj --arch-specific - | FileCheck %s --check-prefix=CHECK-OBJ
+; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O1 -filetype obj -o - | llvm-readobj --arch-specific - | FileCheck %s --check-prefix=CHECK-OBJ
+; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O3 -filetype obj -o - | llvm-readobj --arch-specific - | FileCheck %s --check-prefix=CHECK-OBJ
 
 ; CHECK: .eabi_attribute 30, 3	@ Tag_ABI_optimization_goals
 ; CHECK-OBJ:          TagName: ABI_optimization_goals
diff --git a/llvm/test/CodeGen/ARM/build-attributes-optimization.ll b/llvm/test/CodeGen/ARM/build-attributes-optimization.ll
index 6c1abae6b..85a123f7b 100644
--- a/llvm/test/CodeGen/ARM/build-attributes-optimization.ll
+++ b/llvm/test/CodeGen/ARM/build-attributes-optimization.ll
@@ -2,9 +2,9 @@
 ; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O1 | FileCheck %s --check-prefix=SPEED
 ; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O3 | FileCheck %s --check-prefix=MAXSPEED
 
-; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O0 -filetype obj -o - | llvm-readobj --arm-attributes - | FileCheck %s --check-prefix=NONE-OBJ
-; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O1 -filetype obj -o - | llvm-readobj --arm-attributes - | FileCheck %s --check-prefix=SPEED-OBJ
-; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O3 -filetype obj -o - | llvm-readobj --arm-attributes - | FileCheck %s --check-prefix=MAXSPEED-OBJ
+; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O0 -filetype obj -o - | llvm-readobj --arch-specific - | FileCheck %s --check-prefix=NONE-OBJ
+; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O1 -filetype obj -o - | llvm-readobj --arch-specific - | FileCheck %s --check-prefix=SPEED-OBJ
+; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O3 -filetype obj -o - | llvm-readobj --arch-specific - | FileCheck %s --check-prefix=MAXSPEED-OBJ
 
 ; NONE:     .eabi_attribute 30, 5	@ Tag_ABI_optimization_goals
 ; SPEED:    .eabi_attribute 30, 1	@ Tag_ABI_optimization_goals
diff --git a/llvm/test/CodeGen/ARM/build-attributes.ll b/llvm/test/CodeGen/ARM/build-attributes.ll
index 39d173c4f..7308b2124 100644
--- a/llvm/test/CodeGen/ARM/build-attributes.ll
+++ b/llvm/test/CodeGen/ARM/build-attributes.ll
@@ -34,8 +34,8 @@
 ; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a5  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A5-DEFAULT-FAST
 ; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a5 -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
 ; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a5 -mattr=-neon,-d32 | FileCheck %s --check-prefix=CORTEX-A5-NONEON
-; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a5 -mattr=-vfp2d16sp | FileCheck %s --check-prefix=CORTEX-A5-NOFPU
-; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a5 -mattr=-vfp2d16sp  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A5-NOFPU-FAST
+; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a5 -mattr=-vfp2sp | FileCheck %s --check-prefix=CORTEX-A5-NOFPU
+; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a5 -mattr=-vfp2sp  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A5-NOFPU-FAST
 ; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a8 -float-abi=soft | FileCheck %s --check-prefix=CORTEX-A8-SOFT
 ; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a8 -float-abi=soft  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A8-SOFT-FAST
 ; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a8 -float-abi=hard | FileCheck %s --check-prefix=CORTEX-A8-HARD
@@ -50,16 +50,16 @@
 ; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a12 | FileCheck %s --check-prefix=CORTEX-A12-DEFAULT
 ; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a9 -float-abi=soft | FileCheck %s --check-prefix=CORTEX-A9-SOFT
 ; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a12  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A12-DEFAULT-FAST
-; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a12 -mattr=-vfp2d16sp | FileCheck %s --check-prefix=CORTEX-A12-NOFPU
-; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a12 -mattr=-vfp2d16sp  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A12-NOFPU-FAST
+; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a12 -mattr=-vfp2sp | FileCheck %s --check-prefix=CORTEX-A12-NOFPU
+; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a12 -mattr=-vfp2sp  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A12-NOFPU-FAST
 ; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a12 -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
 ; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a15 | FileCheck %s --check-prefix=CORTEX-A15
 ; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a15  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A15-FAST
 ; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a15 -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
 ; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a17 | FileCheck %s --check-prefix=CORTEX-A17-DEFAULT
 ; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a17  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A17-FAST
-; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a17 -mattr=-vfp2d16sp | FileCheck %s --check-prefix=CORTEX-A17-NOFPU
-; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a17 -mattr=-vfp2d16sp  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A17-NOFPU-FAST
+; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a17 -mattr=-vfp2sp | FileCheck %s --check-prefix=CORTEX-A17-NOFPU
+; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a17 -mattr=-vfp2sp  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A17-NOFPU-FAST
 
 ; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a15 -enable-no-trapping-fp-math | FileCheck %s --check-prefix=NO-TRAPPING-MATH
 ; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a15 -denormal-fp-math=ieee | FileCheck %s --check-prefix=DENORMAL-IEEE
@@ -96,8 +96,8 @@
 ; RUN: llc < %s -mtriple=thumbv7m-linux-gnueabi -mcpu=cortex-m4 -float-abi=hard | FileCheck %s --check-prefix=CORTEX-M4-HARD
 ; RUN: llc < %s -mtriple=thumbv7m-linux-gnueabi -mcpu=cortex-m4 -float-abi=hard  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-M4-HARD-FAST
 ; RUN: llc < %s -mtriple=thumbv7m-linux-gnueabi -mcpu=cortex-m4 -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
-; RUN: llc < %s -mtriple=thumbv7em-linux-gnueabi -mcpu=cortex-m7 -mattr=-vfp2d16sp | FileCheck %s --check-prefix=CORTEX-M7 --check-prefix=CORTEX-M7-SOFT
-; RUN: llc < %s -mtriple=thumbv7em-linux-gnueabi -mcpu=cortex-m7 -mattr=-vfp2d16sp  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-M7-NOFPU-FAST
+; RUN: llc < %s -mtriple=thumbv7em-linux-gnueabi -mcpu=cortex-m7 -mattr=-vfp2sp | FileCheck %s --check-prefix=CORTEX-M7 --check-prefix=CORTEX-M7-SOFT
+; RUN: llc < %s -mtriple=thumbv7em-linux-gnueabi -mcpu=cortex-m7 -mattr=-vfp2sp  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-M7-NOFPU-FAST
 ; RUN: llc < %s -mtriple=thumbv7em-linux-gnueabi -mcpu=cortex-m7 -mattr=-fp64 | FileCheck %s --check-prefix=CORTEX-M7 --check-prefix=CORTEX-M7-SINGLE
 ; RUN: llc < %s -mtriple=thumbv7em-linux-gnueabi -mcpu=cortex-m7 -mattr=-fp64  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-M7-FAST
 ; RUN: llc < %s -mtriple=thumbv7em-linux-gnueabi -mcpu=cortex-m7 | FileCheck %s --check-prefix=CORTEX-M7-DOUBLE
@@ -138,27 +138,21 @@
 ; RUN: llc < %s -mtriple=armv8-linux-gnueabi -mcpu=cortex-a72 -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
 ; RUN: llc < %s -mtriple=armv8-linux-gnueabi -mcpu=cortex-a73 | FileCheck %s --check-prefix=CORTEX-A73
 ; RUN: llc < %s -mtriple=armv8.1a-linux-gnueabi | FileCheck %s --check-prefix=GENERIC-ARMV8_1-A
-; RUN: llc < %s -mtriple=armv8-linux-gnueabi -mcpu=exynos-m1 | FileCheck %s --check-prefix=EXYNOS-M1
-; RUN: llc < %s -mtriple=armv8-linux-gnueabi -mcpu=exynos-m1  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=EXYNOS-M1-FAST
-; RUN: llc < %s -mtriple=armv8-linux-gnueabi -mcpu=exynos-m1 -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
-; RUN: llc < %s -mtriple=armv8-linux-gnueabi -mcpu=exynos-m2 | FileCheck %s --check-prefix=EXYNOS-M2
-; RUN: llc < %s -mtriple=armv8-linux-gnueabi -mcpu=exynos-m2  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=EXYNOS-M1-FAST
-; RUN: llc < %s -mtriple=armv8-linux-gnueabi -mcpu=exynos-m2 -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
 ; RUN: llc < %s -mtriple=armv8-linux-gnueabi -mcpu=exynos-m3 | FileCheck %s --check-prefix=EXYNOS-M3
-; RUN: llc < %s -mtriple=armv8-linux-gnueabi -mcpu=exynos-m3  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=EXYNOS-M1-FAST
+; RUN: llc < %s -mtriple=armv8-linux-gnueabi -mcpu=exynos-m3  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=EXYNOS-FAST
 ; RUN: llc < %s -mtriple=armv8-linux-gnueabi -mcpu=exynos-m3 -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
 ; RUN: llc < %s -mtriple=armv8-linux-gnueabi -mcpu=exynos-m4 | FileCheck %s --check-prefix=EXYNOS-M4
-; RUN: llc < %s -mtriple=armv8-linux-gnueabi -mcpu=exynos-m4  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=EXYNOS-M1-FAST
+; RUN: llc < %s -mtriple=armv8-linux-gnueabi -mcpu=exynos-m4  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=EXYNOS-FAST
 ; RUN: llc < %s -mtriple=armv8-linux-gnueabi -mcpu=exynos-m4 -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
 ; RUN: llc < %s -mtriple=armv8-linux-gnueabi -mcpu=exynos-m5 | FileCheck %s --check-prefix=EXYNOS-M5
-; RUN: llc < %s -mtriple=armv8-linux-gnueabi -mcpu=exynos-m5  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=EXYNOS-M1-FAST
+; RUN: llc < %s -mtriple=armv8-linux-gnueabi -mcpu=exynos-m5  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=EXYNOS-FAST
 ; RUN: llc < %s -mtriple=armv8-linux-gnueabi -mcpu=exynos-m5 -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
 ; RUN: llc < %s -mtriple=armv8.1a-linux-gnueabi  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=GENERIC-ARMV8_1-A-FAST
 ; RUN: llc < %s -mtriple=armv8.1a-linux-gnueabi -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
 ; RUN: llc < %s -mtriple=armv7-none-linux-gnueabi -mcpu=cortex-a7 | FileCheck %s  --check-prefix=CORTEX-A7-CHECK
 ; RUN: llc < %s -mtriple=armv7-none-linux-gnueabi -mcpu=cortex-a7  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s  --check-prefix=CORTEX-A7-CHECK-FAST
-; RUN: llc < %s -mtriple=armv7-none-linux-gnueabi -mcpu=cortex-a7 -mattr=-vfp2d16sp,-vfp3,-vfp4,-neon,-fp16 | FileCheck %s --check-prefix=CORTEX-A7-NOFPU
-; RUN: llc < %s -mtriple=armv7-none-linux-gnueabi -mcpu=cortex-a7 -mattr=-vfp2d16sp,-vfp3,-vfp4,-neon,-fp16  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A7-NOFPU-FAST
+; RUN: llc < %s -mtriple=armv7-none-linux-gnueabi -mcpu=cortex-a7 -mattr=-vfp2sp,-vfp3,-vfp4,-neon,-fp16 | FileCheck %s --check-prefix=CORTEX-A7-NOFPU
+; RUN: llc < %s -mtriple=armv7-none-linux-gnueabi -mcpu=cortex-a7 -mattr=-vfp2sp,-vfp3,-vfp4,-neon,-fp16  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A7-NOFPU-FAST
 ; RUN: llc < %s -mtriple=armv7-none-linux-gnueabi -mcpu=cortex-a7 -mattr=+vfp4,-neon | FileCheck %s --check-prefix=CORTEX-A7-FPUV4
 ; RUN: llc < %s -mtriple=armv7-none-linux-gnueabi -mcpu=cortex-a7 -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
 ; RUN: llc < %s -mtriple=armv7-none-linux-gnueabi -mcpu=cortex-a7 -mattr=+vfp4,-neon  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A7-FPUV4-FAST
@@ -186,10 +180,6 @@
 ; RUN: llc < %s -mtriple=armv8-none-linux-gnueabi -mcpu=cortex-a57 -mattr=+strict-align | FileCheck %s --check-prefix=STRICT-ALIGN
 ; RUN: llc < %s -mtriple=armv8-none-linux-gnueabi -mcpu=cortex-a72 | FileCheck %s --check-prefix=NO-STRICT-ALIGN
 ; RUN: llc < %s -mtriple=armv8-none-linux-gnueabi -mcpu=cortex-a72 -mattr=+strict-align | FileCheck %s --check-prefix=STRICT-ALIGN
-; RUN: llc < %s -mtriple=armv8-none-linux-gnueabi -mcpu=exynos-m1 | FileCheck %s --check-prefix=NO-STRICT-ALIGN
-; RUN: llc < %s -mtriple=armv8-none-linux-gnueabi -mcpu=exynos-m1 -mattr=+strict-align | FileCheck %s --check-prefix=STRICT-ALIGN
-; RUN: llc < %s -mtriple=armv8-none-linux-gnueabi -mcpu=exynos-m2 | FileCheck %s --check-prefix=NO-STRICT-ALIGN
-; RUN: llc < %s -mtriple=armv8-none-linux-gnueabi -mcpu=exynos-m2 -mattr=+strict-align | FileCheck %s --check-prefix=STRICT-ALIGN
 ; RUN: llc < %s -mtriple=armv8-none-linux-gnueabi -mcpu=exynos-m3 | FileCheck %s --check-prefix=NO-STRICT-ALIGN
 ; RUN: llc < %s -mtriple=armv8-none-linux-gnueabi -mcpu=exynos-m3 -mattr=+strict-align | FileCheck %s --check-prefix=STRICT-ALIGN
 ; RUN: llc < %s -mtriple=armv8-none-linux-gnueabi -mcpu=exynos-m4 | FileCheck %s --check-prefix=NO-STRICT-ALIGN
@@ -230,7 +220,7 @@
 ; RUN: llc < %s -mtriple=armv5-none-linux-gnueabi -mcpu=arm1022e -mattr=+strict-align | FileCheck %s --check-prefix=STRICT-ALIGN
 
 ; ARMv8-R
-; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-r52 -mattr=-vfp2d16sp,-fp16 | FileCheck %s --check-prefix=ARMv8R --check-prefix=ARMv8R-NOFPU
+; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-r52 -mattr=-vfp2sp,-fp16 | FileCheck %s --check-prefix=ARMv8R --check-prefix=ARMv8R-NOFPU
 ; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-r52 -mattr=-neon,-fp64,-d32 | FileCheck %s --check-prefix=ARMv8R --check-prefix=ARMv8R-SP
 ; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-r52 | FileCheck %s --check-prefix=ARMv8R --check-prefix=ARMv8R-NEON
 
@@ -1576,58 +1566,12 @@
 ; CORTEX-A73:  .eabi_attribute 38, 1
 ; CORTEX-A73:  .eabi_attribute 14, 0
 
-; EXYNOS-M1:  .cpu exynos-m1
-; EXYNOS-M1:  .eabi_attribute 6, 14
-; EXYNOS-M1:  .eabi_attribute 7, 65
-; EXYNOS-M1:  .eabi_attribute 8, 1
-; EXYNOS-M1:  .eabi_attribute 9, 2
-; EXYNOS-M1:  .fpu crypto-neon-fp-armv8
-; EXYNOS-M1:  .eabi_attribute 12, 3
-; EXYNOS-M1-NOT:  .eabi_attribute 27
-; EXYNOS-M1:  .eabi_attribute 36, 1
-; EXYNOS-M1:  .eabi_attribute 42, 1
-; EXYNOS-M1-NOT:  .eabi_attribute 44
-; EXYNOS-M1:  .eabi_attribute 68, 3
-; EXYNOS-M1-NOT:   .eabi_attribute 19
-;; We default to IEEE 754 compliance
-; EXYNOS-M1:  .eabi_attribute 20, 1
-; EXYNOS-M1:  .eabi_attribute 21, 1
-; EXYNOS-M1-NOT:  .eabi_attribute 22
-; EXYNOS-M1:  .eabi_attribute 23, 3
-; EXYNOS-M1:  .eabi_attribute 24, 1
-; EXYNOS-M1:  .eabi_attribute 25, 1
-; EXYNOS-M1-NOT:  .eabi_attribute 28
-; EXYNOS-M1:  .eabi_attribute 38, 1
-
-; EXYNOS-M1-FAST-NOT:   .eabi_attribute 19
-;; The exynos-m1 has the ARMv8 FP unit, which always flushes preserving sign.
-; EXYNOS-M1-FAST:  .eabi_attribute 20, 2
-; EXYNOS-M1-FAST-NOT:  .eabi_attribute 21
-; EXYNOS-M1-FAST-NOT:  .eabi_attribute 22
-; EXYNOS-M1-FAST:  .eabi_attribute 23, 1
-
-; EXYNOS-M2:  .cpu exynos-m2
-; EXYNOS-M2:  .eabi_attribute 6, 14
-; EXYNOS-M2:  .eabi_attribute 7, 65
-; EXYNOS-M2:  .eabi_attribute 8, 1
-; EXYNOS-M2:  .eabi_attribute 9, 2
-; EXYNOS-M2:  .fpu crypto-neon-fp-armv8
-; EXYNOS-M2:  .eabi_attribute 12, 3
-; EXYNOS-M2-NOT:  .eabi_attribute 27
-; EXYNOS-M2:  .eabi_attribute 36, 1
-; EXYNOS-M2:  .eabi_attribute 42, 1
-; EXYNOS-M2-NOT:  .eabi_attribute 44
-; EXYNOS-M2:  .eabi_attribute 68, 3
-; EXYNOS-M2-NOT:   .eabi_attribute 19
-;; We default to IEEE 754 compliance
-; EXYNOS-M2:  .eabi_attribute 20, 1
-; EXYNOS-M2:  .eabi_attribute 21, 1
-; EXYNOS-M2-NOT:  .eabi_attribute 22
-; EXYNOS-M2:  .eabi_attribute 23, 3
-; EXYNOS-M2:  .eabi_attribute 24, 1
-; EXYNOS-M2:  .eabi_attribute 25, 1
-; EXYNOS-M2-NOT:  .eabi_attribute 28
-; EXYNOS-M2:  .eabi_attribute 38, 1
+; EXYNOS-FAST-NOT:   .eabi_attribute 19
+;; The Exynos processors have the ARMv8 FP unit, which always flushes preserving sign.
+; EXYNOS-FAST:  .eabi_attribute 20, 2
+; EXYNOS-FAST-NOT:  .eabi_attribute 21
+; EXYNOS-FAST-NOT:  .eabi_attribute 22
+; EXYNOS-FAST:  .eabi_attribute 23, 1
 
 ; EXYNOS-M3:  .cpu exynos-m3
 ; EXYNOS-M3:  .eabi_attribute 6, 14
diff --git a/llvm/test/CodeGen/ARM/byval_load_align.ll b/llvm/test/CodeGen/ARM/byval_load_align.ll
index d00d926c7..141ead6c0 100644
--- a/llvm/test/CodeGen/ARM/byval_load_align.ll
+++ b/llvm/test/CodeGen/ARM/byval_load_align.ll
@@ -22,6 +22,6 @@ entry:
 
 declare void @Logger(i8 signext, %struct.ModuleID* byval) #1
 
-attributes #0 = { nounwind ssp "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind ssp "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind }
diff --git a/llvm/test/CodeGen/ARM/call-tc.ll b/llvm/test/CodeGen/ARM/call-tc.ll
index c5cfb9def..4256cc879 100644
--- a/llvm/test/CodeGen/ARM/call-tc.ll
+++ b/llvm/test/CodeGen/ARM/call-tc.ll
@@ -10,14 +10,14 @@
 
 declare void @g(i32, i32, i32, i32)
 
-define void @t1() "no-frame-pointer-elim"="true" {
+define void @t1() "frame-pointer"="all" {
 ; CHECKELF-LABEL: t1:
 ; CHECKELF: bl g
         call void @g( i32 1, i32 2, i32 3, i32 4 )
         ret void
 }
 
-define void @t2() "no-frame-pointer-elim"="true" {
+define void @t2() "frame-pointer"="all" {
 ; CHECKV6-LABEL: t2:
 ; CHECKV6: bx r0
 ; CHECKT2D-LABEL: t2:
@@ -29,7 +29,7 @@ define void @t2() "no-frame-pointer-elim"="true" {
         ret void
 }
 
-define void @t3() "no-frame-pointer-elim"="true" {
+define void @t3() "frame-pointer"="all" {
 ; CHECKV6-LABEL: t3:
 ; CHECKV6: b _t2
 ; CHECKELF-LABEL: t3:
@@ -42,7 +42,7 @@ define void @t3() "no-frame-pointer-elim"="true" {
 }
 
 ; Sibcall optimization of expanded libcalls. rdar://8707777
-define double @t4(double %a) nounwind readonly ssp "no-frame-pointer-elim"="true" {
+define double @t4(double %a) nounwind readonly ssp "frame-pointer"="all" {
 entry:
 ; CHECKV6-LABEL: t4:
 ; CHECKV6: b _sin
@@ -52,7 +52,7 @@ entry:
   ret double %0
 }
 
-define float @t5(float %a) nounwind readonly ssp "no-frame-pointer-elim"="true" {
+define float @t5(float %a) nounwind readonly ssp "frame-pointer"="all" {
 entry:
 ; CHECKV6-LABEL: t5:
 ; CHECKV6: b _sinf
@@ -66,7 +66,7 @@ declare float @sinf(float) nounwind readonly
 
 declare double @sin(double) nounwind readonly
 
-define i32 @t6(i32 %a, i32 %b) nounwind readnone "no-frame-pointer-elim"="true" {
+define i32 @t6(i32 %a, i32 %b) nounwind readnone "frame-pointer"="all" {
 entry:
 ; CHECKV6-LABEL: t6:
 ; CHECKV6: b ___divsi3
@@ -80,7 +80,7 @@ entry:
 ; rdar://8309338
 declare void @foo() nounwind
 
-define void @t7() nounwind "no-frame-pointer-elim"="true" {
+define void @t7() nounwind "frame-pointer"="all" {
 entry:
 ; CHECKT2D-LABEL: t7:
 ; CHECKT2D: it ne
@@ -101,7 +101,7 @@ bb:
 
 ; Make sure codegenprep is duplicating ret instructions to enable tail calls.
 ; rdar://11140249
-define i32 @t8(i32 %x) nounwind ssp "no-frame-pointer-elim"="true" {
+define i32 @t8(i32 %x) nounwind ssp "frame-pointer"="all" {
 entry:
 ; CHECKT2D-LABEL: t8:
 ; CHECKT2D-NOT: push
@@ -148,7 +148,7 @@ declare i32 @c(i32)
 
 @x = external global i32, align 4
 
-define i32 @t9() nounwind "no-frame-pointer-elim"="true" {
+define i32 @t9() nounwind "frame-pointer"="all" {
 ; CHECKT2D-LABEL: t9:
 ; CHECKT2D: bl __ZN9MutexLockC1Ev
 ; CHECKT2D: bl __ZN9MutexLockD1Ev
@@ -168,7 +168,7 @@ declare %class.MutexLock* @_ZN9MutexLockD1Ev(%class.MutexLock*) unnamed_addr nou
 ; rdar://13827621
 ; Correctly preserve the input chain for the tailcall node in the bitcast case,
 ; otherwise the call to floorf is lost.
-define float @libcall_tc_test2(float* nocapture %a, float %b) "no-frame-pointer-elim"="true" {
+define float @libcall_tc_test2(float* nocapture %a, float %b) "frame-pointer"="all" {
 ; CHECKT2D-LABEL: libcall_tc_test2:
 ; CHECKT2D: bl _floorf
 ; CHECKT2D: b.w _truncf
diff --git a/llvm/test/CodeGen/ARM/cfguard-checks.ll b/llvm/test/CodeGen/ARM/cfguard-checks.ll
new file mode 100644
index 000000000..3fab04eb1
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/cfguard-checks.ll
@@ -0,0 +1,151 @@
+; RUN: llc < %s -mtriple=arm-pc-windows-msvc | FileCheck %s
+; Control Flow Guard is currently only available on Windows
+
+; Test that Control Flow Guard checks are correctly added when required.
+
+
+declare i32 @target_func()
+
+
+; Test that Control Flow Guard checks are not added on calls with the "guard_nocf" attribute.
+define i32 @func_guard_nocf() #0 {
+entry:
+  %func_ptr = alloca i32 ()*, align 8
+  store i32 ()* @target_func, i32 ()** %func_ptr, align 8
+  %0 = load i32 ()*, i32 ()** %func_ptr, align 8
+  %1 = call arm_aapcs_vfpcc i32 %0() #1
+  ret i32 %1
+
+  ; CHECK-LABEL: func_guard_nocf
+  ; CHECK:       movw r0, :lower16:target_func
+	; CHECK:       movt r0, :upper16:target_func
+  ; CHECK-NOT:   __guard_check_icall_fptr
+	; CHECK:       blx r0
+}
+attributes #0 = { "target-cpu"="cortex-a9" "target-features"="+armv7-a,+dsp,+fp16,+neon,+strict-align,+thumb-mode,+vfp3"}
+attributes #1 = { "guard_nocf" }
+
+
+; Test that Control Flow Guard checks are added even at -O0.
+define i32 @func_optnone_cf() #2 {
+entry:
+  %func_ptr = alloca i32 ()*, align 8
+  store i32 ()* @target_func, i32 ()** %func_ptr, align 8
+  %0 = load i32 ()*, i32 ()** %func_ptr, align 8
+  %1 = call i32 %0()
+  ret i32 %1
+
+  ; The call to __guard_check_icall_fptr should come immediately before the call to the target function.
+  ; CHECK-LABEL: func_optnone_cf
+	; CHECK:       movw r0, :lower16:target_func
+	; CHECK:       movt r0, :upper16:target_func
+	; CHECK:       str r0, [sp]
+	; CHECK:       ldr r4, [sp]
+	; CHECK:       movw r0, :lower16:__guard_check_icall_fptr
+	; CHECK:       movt r0, :upper16:__guard_check_icall_fptr
+	; CHECK:       ldr r1, [r0]
+	; CHECK:       mov r0, r4
+	; CHECK:       blx r1
+	; CHECK-NEXT:  blx r4
+}
+attributes #2 = { noinline optnone "target-cpu"="cortex-a9" "target-features"="+armv7-a,+dsp,+fp16,+neon,+strict-align,+thumb-mode,+vfp3"}
+
+
+; Test that Control Flow Guard checks are correctly added in optimized code (common case).
+define i32 @func_cf() #0 {
+entry:
+  %func_ptr = alloca i32 ()*, align 8
+  store i32 ()* @target_func, i32 ()** %func_ptr, align 8
+  %0 = load i32 ()*, i32 ()** %func_ptr, align 8
+  %1 = call i32 %0()
+  ret i32 %1
+
+  ; The call to __guard_check_icall_fptr should come immediately before the call to the target function.
+  ; CHECK-LABEL: func_cf
+  ; CHECK:       movw r0, :lower16:__guard_check_icall_fptr
+	; CHECK:       movt r0, :upper16:__guard_check_icall_fptr
+	; CHECK:       ldr r1, [r0]
+  ; CHECK:       movw r4, :lower16:target_func
+	; CHECK:       movt r4, :upper16:target_func
+	; CHECK:       mov r0, r4
+	; CHECK:       blx r1
+	; CHECK-NEXT:  blx r4
+}
+
+
+; Test that Control Flow Guard checks are correctly added on invoke instructions.
+define i32 @func_cf_invoke() #0 personality i8* bitcast (void ()* @h to i8*) {
+entry:
+  %0 = alloca i32, align 4
+  %func_ptr = alloca i32 ()*, align 8
+  store i32 ()* @target_func, i32 ()** %func_ptr, align 8
+  %1 = load i32 ()*, i32 ()** %func_ptr, align 8
+  %2 = invoke i32 %1()
+          to label %invoke.cont unwind label %lpad
+invoke.cont:                                      ; preds = %entry
+  ret i32 %2
+
+lpad:                                             ; preds = %entry
+  %tmp = landingpad { i8*, i32 }
+          catch i8* null
+  ret i32 -1
+
+  ; The call to __guard_check_icall_fptr should come immediately before the call to the target function.
+  ; CHECK-LABEL: func_cf_invoke
+  ; CHECK:       movw r0, :lower16:__guard_check_icall_fptr
+	; CHECK:       movt r0, :upper16:__guard_check_icall_fptr
+	; CHECK:       ldr r1, [r0]
+  ; CHECK:       movw r4, :lower16:target_func
+	; CHECK:       movt r4, :upper16:target_func
+	; CHECK:       mov r0, r4
+	; CHECK:       blx r1
+  ; CHECK-NEXT:  $Mtmp0:
+	; CHECK-NEXT:  blx r4
+  ; CHECK:       ; %invoke.cont
+  ; CHECK:       ; %lpad
+}
+
+declare void @h()
+
+
+; Test that longjmp targets have public labels and are included in the .gljmp section.
+%struct._SETJMP_FLOAT128 = type { [2 x i64] }
+@buf1 = internal global [16 x %struct._SETJMP_FLOAT128] zeroinitializer, align 16
+
+define i32 @func_cf_setjmp() #0 {
+  %1 = alloca i32, align 4
+  %2 = alloca i32, align 4
+  store i32 0, i32* %1, align 4
+  store i32 -1, i32* %2, align 4
+  %3 = call i8* @llvm.frameaddress(i32 0)
+  %4 = call i32 @_setjmp(i8* bitcast ([16 x %struct._SETJMP_FLOAT128]* @buf1 to i8*), i8* %3) #3
+
+  ; CHECK-LABEL: func_cf_setjmp
+  ; CHECK:       bl _setjmp
+  ; CHECK-NEXT:  $cfgsj_func_cf_setjmp0:
+
+  %5 = call i8* @llvm.frameaddress(i32 0)
+  %6 = call i32 @_setjmp(i8* bitcast ([16 x %struct._SETJMP_FLOAT128]* @buf1 to i8*), i8* %5) #3
+
+  ; CHECK:       bl _setjmp
+  ; CHECK-NEXT:  $cfgsj_func_cf_setjmp1:
+
+  store i32 1, i32* %2, align 4
+  %7 = load i32, i32* %2, align 4
+  ret i32 %7
+
+  ; CHECK:       .section .gljmp$y,"dr"
+  ; CHECK-NEXT:  .symidx $cfgsj_func_cf_setjmp0
+  ; CHECK-NEXT:  .symidx $cfgsj_func_cf_setjmp1
+}
+
+declare i8* @llvm.frameaddress(i32)
+
+; Function Attrs: returns_twice
+declare dso_local i32 @_setjmp(i8*, i8*) #3
+
+attributes #3 = { returns_twice }
+
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 2, !"cfguard", i32 2}
diff --git a/llvm/test/CodeGen/ARM/cfguard-module-flag.ll b/llvm/test/CodeGen/ARM/cfguard-module-flag.ll
new file mode 100644
index 000000000..c8a4256ac
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/cfguard-module-flag.ll
@@ -0,0 +1,26 @@
+
+; RUN: llc < %s -mtriple=arm-pc-windows-msvc | FileCheck %s
+; Control Flow Guard is currently only available on Windows
+
+; Test that Control Flow Guard checks are not added in modules with the
+; cfguard=1 flag (emit tables but no checks).
+
+
+declare void @target_func()
+
+define void @func_in_module_without_cfguard() #0 {
+entry:
+  %func_ptr = alloca void ()*, align 8
+  store void ()* @target_func, void ()** %func_ptr, align 8
+  %0 = load void ()*, void ()** %func_ptr, align 8
+
+  call void %0()
+  ret void
+
+  ; CHECK-NOT: __guard_check_icall_fptr
+  ; CHECK-NOT: __guard_dispatch_icall_fptr
+}
+attributes #0 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="cortex-a9" "target-features"="+armv7-a,+dsp,+fp16,+neon,+strict-align,+thumb-mode,+vfp3" "unsafe-fp-math"="false" "use-soft-float"="false"}
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 2, !"cfguard", i32 1}
diff --git a/llvm/test/CodeGen/ARM/clang-section.ll b/llvm/test/CodeGen/ARM/clang-section.ll
index 343f0e721..e3fd7276d 100644
--- a/llvm/test/CodeGen/ARM/clang-section.ll
+++ b/llvm/test/CodeGen/ARM/clang-section.ll
@@ -49,10 +49,10 @@ attributes #0 = { "bss-section"="my_bss.1" "data-section"="my_data.1" "rodata-se
 attributes #1 = { "data-section"="my_data.1" "rodata-section"="my_rodata.1" }
 attributes #2 = { "bss-section"="my_bss.2" "rodata-section"="my_rodata.1" }
 attributes #3 = { "bss-section"="my_bss.2" "data-section"="my_data.2" "rodata-section"="my_rodata.2" }
-attributes #4 = { noinline nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "denormal-fp-math"="preserve-sign" "disable-tail-calls"="false" "implicit-section-name"="my_text.1" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="true" "no-jump-tables"="false" "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="true" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="cortex-a9" "target-features"="+dsp,+fp16,+neon,+vfp3" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #5 = { noinline "correctly-rounded-divide-sqrt-fp-math"="false" "denormal-fp-math"="preserve-sign" "disable-tail-calls"="false" "implicit-section-name"="my_text.2" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="true" "no-jump-tables"="false" "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="true" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="cortex-a9" "target-features"="+dsp,+fp16,+neon,+vfp3" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #6 = { "correctly-rounded-divide-sqrt-fp-math"="false" "denormal-fp-math"="preserve-sign" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="true" "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="true" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="cortex-a9" "target-features"="+dsp,+fp16,+neon,+vfp3" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #7 = { noinline nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "denormal-fp-math"="preserve-sign" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="true" "no-jump-tables"="false" "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="true" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="cortex-a9" "target-features"="+dsp,+fp16,+neon,+vfp3" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #4 = { noinline nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "denormal-fp-math"="preserve-sign" "disable-tail-calls"="false" "implicit-section-name"="my_text.1" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="true" "no-jump-tables"="false" "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="true" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="cortex-a9" "target-features"="+dsp,+fp16,+neon,+vfp3" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #5 = { noinline "correctly-rounded-divide-sqrt-fp-math"="false" "denormal-fp-math"="preserve-sign" "disable-tail-calls"="false" "implicit-section-name"="my_text.2" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="true" "no-jump-tables"="false" "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="true" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="cortex-a9" "target-features"="+dsp,+fp16,+neon,+vfp3" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #6 = { "correctly-rounded-divide-sqrt-fp-math"="false" "denormal-fp-math"="preserve-sign" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="true" "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="true" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="cortex-a9" "target-features"="+dsp,+fp16,+neon,+vfp3" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #7 = { noinline nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "denormal-fp-math"="preserve-sign" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="true" "no-jump-tables"="false" "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="true" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="cortex-a9" "target-features"="+dsp,+fp16,+neon,+vfp3" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.module.flags = !{!0, !1, !2, !3}
 
diff --git a/llvm/test/CodeGen/ARM/cls.ll b/llvm/test/CodeGen/ARM/cls.ll
new file mode 100644
index 000000000..cccb38d07
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/cls.ll
@@ -0,0 +1,27 @@
+; RUN: llc -mtriple=armv5 %s -o - | FileCheck %s
+
+; CHECK:      eor [[T:r[0-9]+]], [[T]], [[T]], asr #31
+; CHECK-NEXT: mov [[C1:r[0-9]+]], #1
+; CHECK-NEXT: orr [[T]], [[C1]], [[T]], lsl #1
+; CHECK-NEXT: clz [[T]], [[T]]
+define i32 @cls(i32 %t) {
+  %cls.i = call i32 @llvm.arm.cls(i32 %t)
+  ret i32 %cls.i
+}
+
+; CHECK: cmp r1, #0
+; CHECK: mvnne [[ADJUSTEDLO:r[0-9]+]], r0
+; CHECK: clz [[CLZLO:r[0-9]+]], [[ADJUSTEDLO]]
+; CHECK: eor [[A:r[0-9]+]], r1, r1, asr #31
+; CHECK: mov r1, #1
+; CHECK: orr [[A]], r1, [[A]], lsl #1
+; CHECK: clz [[CLSHI:r[0-9]+]], [[A]]
+; CHECK: cmp [[CLSHI]], #31
+; CHECK: addeq r0, [[CLZLO]], #31
+define i32 @cls64(i64 %t) {
+  %cls.i = call i32 @llvm.arm.cls64(i64 %t)
+  ret i32 %cls.i
+}
+
+declare i32 @llvm.arm.cls(i32) nounwind
+declare i32 @llvm.arm.cls64(i64) nounwind
diff --git a/llvm/test/CodeGen/ARM/cmov_fp16.ll b/llvm/test/CodeGen/ARM/cmov_fp16.ll
new file mode 100644
index 000000000..925fed582
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/cmov_fp16.ll
@@ -0,0 +1,261 @@
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+fullfp16 %s -o - | FileCheck %s --check-prefixes CHECK-THUMB,CHECK
+; RUN: llc -mtriple=armv8.2a-arm-none-eabi -mattr=+fullfp16 %s -o - | FileCheck %s --check-prefixes CHECK-ARM,CHECK
+
+define i32 @test_ne(i32 %x, i32 %y, i32 %a, i32 %b) {
+; CHECK-LABEL: test_ne:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov s2, r0
+; CHECK-NEXT:    cmp r2, r3
+; CHECK-NEXT:    vmov s0, r1
+; CHECK-NEXT:    vcvt.f16.u32 s2, s2
+; CHECK-NEXT:    vcvt.f16.u32 s0, s0
+; CHECK-NEXT:    vseleq.f16 s0, s0, s2
+; CHECK-NEXT:    vmov.f16 r0, s0
+; CHECK-NEXT:    bx lr
+entry:
+  %x.half = uitofp i32 %x to half
+  %y.half = uitofp i32 %y to half
+  %cmp = icmp ne i32 %a, %b
+  %cond = select i1 %cmp, half %x.half, half %y.half
+  %0 = bitcast half %cond to i16
+  %1 = zext i16 %0 to i32
+  ret i32 %1
+}
+
+define i32 @test_eq(i32 %x, i32 %y, i32 %a, i32 %b) {
+; CHECK-LABEL: test_eq:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov s2, r1
+; CHECK-NEXT:    cmp r2, r3
+; CHECK-NEXT:    vmov s0, r0
+; CHECK-NEXT:    vcvt.f16.u32 s2, s2
+; CHECK-NEXT:    vcvt.f16.u32 s0, s0
+; CHECK-NEXT:    vseleq.f16 s0, s0, s2
+; CHECK-NEXT:    vmov.f16 r0, s0
+; CHECK-NEXT:    bx lr
+entry:
+  %x.half = uitofp i32 %x to half
+  %y.half = uitofp i32 %y to half
+  %cmp = icmp eq i32 %a, %b
+  %cond = select i1 %cmp, half %x.half, half %y.half
+  %0 = bitcast half %cond to i16
+  %1 = zext i16 %0 to i32
+  ret i32 %1
+}
+
+define i32 @test_gt(i32 %x, i32 %y, i32 %a, i32 %b) {
+; CHECK-LABEL: test_gt:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov s2, r1
+; CHECK-NEXT:    cmp r2, r3
+; CHECK-NEXT:    vmov s0, r0
+; CHECK-NEXT:    vcvt.f16.u32 s2, s2
+; CHECK-NEXT:    vcvt.f16.u32 s0, s0
+; CHECK-NEXT:    vselgt.f16 s0, s0, s2
+; CHECK-NEXT:    vmov.f16 r0, s0
+; CHECK-NEXT:    bx lr
+entry:
+  %x.half = uitofp i32 %x to half
+  %y.half = uitofp i32 %y to half
+  %cmp = icmp sgt i32 %a, %b
+  %cond = select i1 %cmp, half %x.half, half %y.half
+  %0 = bitcast half %cond to i16
+  %1 = zext i16 %0 to i32
+  ret i32 %1
+}
+
+define i32 @test_ge(i32 %x, i32 %y, i32 %a, i32 %b) {
+; CHECK-LABEL: test_ge:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov s2, r1
+; CHECK-NEXT:    cmp r2, r3
+; CHECK-NEXT:    vmov s0, r0
+; CHECK-NEXT:    vcvt.f16.u32 s2, s2
+; CHECK-NEXT:    vcvt.f16.u32 s0, s0
+; CHECK-NEXT:    vselge.f16 s0, s0, s2
+; CHECK-NEXT:    vmov.f16 r0, s0
+; CHECK-NEXT:    bx lr
+entry:
+  %x.half = uitofp i32 %x to half
+  %y.half = uitofp i32 %y to half
+  %cmp = icmp sge i32 %a, %b
+  %cond = select i1 %cmp, half %x.half, half %y.half
+  %0 = bitcast half %cond to i16
+  %1 = zext i16 %0 to i32
+  ret i32 %1
+}
+
+define i32 @test_lt(i32 %x, i32 %y, i32 %a, i32 %b) {
+; CHECK-LABEL: test_lt:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov s2, r0
+; CHECK-NEXT:    cmp r2, r3
+; CHECK-NEXT:    vmov s0, r1
+; CHECK-NEXT:    vcvt.f16.u32 s2, s2
+; CHECK-NEXT:    vcvt.f16.u32 s0, s0
+; CHECK-NEXT:    vselge.f16 s0, s0, s2
+; CHECK-NEXT:    vmov.f16 r0, s0
+; CHECK-NEXT:    bx lr
+entry:
+  %x.half = uitofp i32 %x to half
+  %y.half = uitofp i32 %y to half
+  %cmp = icmp slt i32 %a, %b
+  %cond = select i1 %cmp, half %x.half, half %y.half
+  %0 = bitcast half %cond to i16
+  %1 = zext i16 %0 to i32
+  ret i32 %1
+}
+
+define i32 @test_le(i32 %x, i32 %y, i32 %a, i32 %b) {
+; CHECK-LABEL: test_le:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov s2, r0
+; CHECK-NEXT:    cmp r2, r3
+; CHECK-NEXT:    vmov s0, r1
+; CHECK-NEXT:    vcvt.f16.u32 s2, s2
+; CHECK-NEXT:    vcvt.f16.u32 s0, s0
+; CHECK-NEXT:    vselgt.f16 s0, s0, s2
+; CHECK-NEXT:    vmov.f16 r0, s0
+; CHECK-NEXT:    bx lr
+entry:
+  %x.half = uitofp i32 %x to half
+  %y.half = uitofp i32 %y to half
+  %cmp = icmp sle i32 %a, %b
+  %cond = select i1 %cmp, half %x.half, half %y.half
+  %0 = bitcast half %cond to i16
+  %1 = zext i16 %0 to i32
+  ret i32 %1
+}
+
+define i32 @test_hi(i32 %x, i32 %y, i32 %a, i32 %b) {
+; CHECK-THUMB-LABEL: test_hi:
+; CHECK-THUMB:       @ %bb.0: @ %entry
+; CHECK-THUMB-NEXT:    vmov s2, r0
+; CHECK-THUMB-NEXT:    cmp r2, r3
+; CHECK-THUMB-NEXT:    vmov s0, r1
+; CHECK-THUMB-NEXT:    vcvt.f16.u32 s2, s2
+; CHECK-THUMB-NEXT:    vcvt.f16.u32 s0, s0
+; CHECK-THUMB-NEXT:    it hi
+; CHECK-THUMB-NEXT:    vmovhi.f32 s0, s2
+; CHECK-THUMB-NEXT:    vmov.f16 r0, s0
+; CHECK-THUMB-NEXT:    bx lr
+;
+; CHECK-ARM-LABEL: test_hi:
+; CHECK-ARM:       @ %bb.0: @ %entry
+; CHECK-ARM-NEXT:    vmov s2, r0
+; CHECK-ARM-NEXT:    cmp r2, r3
+; CHECK-ARM-NEXT:    vmov s0, r1
+; CHECK-ARM-NEXT:    vcvt.f16.u32 s2, s2
+; CHECK-ARM-NEXT:    vcvt.f16.u32 s0, s0
+; CHECK-ARM-NEXT:    vmovhi.f32 s0, s2
+; CHECK-ARM-NEXT:    vmov.f16 r0, s0
+; CHECK-ARM-NEXT:    bx lr
+entry:
+  %x.half = uitofp i32 %x to half
+  %y.half = uitofp i32 %y to half
+  %cmp = icmp ugt i32 %a, %b
+  %cond = select i1 %cmp, half %x.half, half %y.half
+  %0 = bitcast half %cond to i16
+  %1 = zext i16 %0 to i32
+  ret i32 %1
+}
+
+define i32 @test_hs(i32 %x, i32 %y, i32 %a, i32 %b) {
+; CHECK-THUMB-LABEL: test_hs:
+; CHECK-THUMB:       @ %bb.0: @ %entry
+; CHECK-THUMB-NEXT:    vmov s2, r0
+; CHECK-THUMB-NEXT:    cmp r2, r3
+; CHECK-THUMB-NEXT:    vmov s0, r1
+; CHECK-THUMB-NEXT:    vcvt.f16.u32 s2, s2
+; CHECK-THUMB-NEXT:    vcvt.f16.u32 s0, s0
+; CHECK-THUMB-NEXT:    it hs
+; CHECK-THUMB-NEXT:    vmovhs.f32 s0, s2
+; CHECK-THUMB-NEXT:    vmov.f16 r0, s0
+; CHECK-THUMB-NEXT:    bx lr
+;
+; CHECK-ARM-LABEL: test_hs:
+; CHECK-ARM:       @ %bb.0: @ %entry
+; CHECK-ARM-NEXT:    vmov s2, r0
+; CHECK-ARM-NEXT:    cmp r2, r3
+; CHECK-ARM-NEXT:    vmov s0, r1
+; CHECK-ARM-NEXT:    vcvt.f16.u32 s2, s2
+; CHECK-ARM-NEXT:    vcvt.f16.u32 s0, s0
+; CHECK-ARM-NEXT:    vmovhs.f32 s0, s2
+; CHECK-ARM-NEXT:    vmov.f16 r0, s0
+; CHECK-ARM-NEXT:    bx lr
+entry:
+  %x.half = uitofp i32 %x to half
+  %y.half = uitofp i32 %y to half
+  %cmp = icmp uge i32 %a, %b
+  %cond = select i1 %cmp, half %x.half, half %y.half
+  %0 = bitcast half %cond to i16
+  %1 = zext i16 %0 to i32
+  ret i32 %1
+}
+
+define i32 @test_lo(i32 %x, i32 %y, i32 %a, i32 %b) {
+; CHECK-THUMB-LABEL: test_lo:
+; CHECK-THUMB:       @ %bb.0: @ %entry
+; CHECK-THUMB-NEXT:    vmov s2, r0
+; CHECK-THUMB-NEXT:    cmp r2, r3
+; CHECK-THUMB-NEXT:    vmov s0, r1
+; CHECK-THUMB-NEXT:    vcvt.f16.u32 s2, s2
+; CHECK-THUMB-NEXT:    vcvt.f16.u32 s0, s0
+; CHECK-THUMB-NEXT:    it lo
+; CHECK-THUMB-NEXT:    vmovlo.f32 s0, s2
+; CHECK-THUMB-NEXT:    vmov.f16 r0, s0
+; CHECK-THUMB-NEXT:    bx lr
+;
+; CHECK-ARM-LABEL: test_lo:
+; CHECK-ARM:       @ %bb.0: @ %entry
+; CHECK-ARM-NEXT:    vmov s2, r0
+; CHECK-ARM-NEXT:    cmp r2, r3
+; CHECK-ARM-NEXT:    vmov s0, r1
+; CHECK-ARM-NEXT:    vcvt.f16.u32 s2, s2
+; CHECK-ARM-NEXT:    vcvt.f16.u32 s0, s0
+; CHECK-ARM-NEXT:    vmovlo.f32 s0, s2
+; CHECK-ARM-NEXT:    vmov.f16 r0, s0
+; CHECK-ARM-NEXT:    bx lr
+entry:
+  %x.half = uitofp i32 %x to half
+  %y.half = uitofp i32 %y to half
+  %cmp = icmp ult i32 %a, %b
+  %cond = select i1 %cmp, half %x.half, half %y.half
+  %0 = bitcast half %cond to i16
+  %1 = zext i16 %0 to i32
+  ret i32 %1
+}
+
+define i32 @test_ls(i32 %x, i32 %y, i32 %a, i32 %b) {
+; CHECK-THUMB-LABEL: test_ls:
+; CHECK-THUMB:       @ %bb.0: @ %entry
+; CHECK-THUMB-NEXT:    vmov s2, r0
+; CHECK-THUMB-NEXT:    cmp r2, r3
+; CHECK-THUMB-NEXT:    vmov s0, r1
+; CHECK-THUMB-NEXT:    vcvt.f16.u32 s2, s2
+; CHECK-THUMB-NEXT:    vcvt.f16.u32 s0, s0
+; CHECK-THUMB-NEXT:    it ls
+; CHECK-THUMB-NEXT:    vmovls.f32 s0, s2
+; CHECK-THUMB-NEXT:    vmov.f16 r0, s0
+; CHECK-THUMB-NEXT:    bx lr
+;
+; CHECK-ARM-LABEL: test_ls:
+; CHECK-ARM:       @ %bb.0: @ %entry
+; CHECK-ARM-NEXT:    vmov s2, r0
+; CHECK-ARM-NEXT:    cmp r2, r3
+; CHECK-ARM-NEXT:    vmov s0, r1
+; CHECK-ARM-NEXT:    vcvt.f16.u32 s2, s2
+; CHECK-ARM-NEXT:    vcvt.f16.u32 s0, s0
+; CHECK-ARM-NEXT:    vmovls.f32 s0, s2
+; CHECK-ARM-NEXT:    vmov.f16 r0, s0
+; CHECK-ARM-NEXT:    bx lr
+entry:
+  %x.half = uitofp i32 %x to half
+  %y.half = uitofp i32 %y to half
+  %cmp = icmp ule i32 %a, %b
+  %cond = select i1 %cmp, half %x.half, half %y.half
+  %0 = bitcast half %cond to i16
+  %1 = zext i16 %0 to i32
+  ret i32 %1
+}
+
diff --git a/llvm/test/CodeGen/ARM/coalesce-dbgvalue.ll b/llvm/test/CodeGen/ARM/coalesce-dbgvalue.ll
index f917278fa..74598fba8 100644
--- a/llvm/test/CodeGen/ARM/coalesce-dbgvalue.ll
+++ b/llvm/test/CodeGen/ARM/coalesce-dbgvalue.ll
@@ -72,8 +72,8 @@ declare i32 @fn3(...) #1
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.value(metadata, metadata, metadata) #2
 
-attributes #0 = { nounwind ssp "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind ssp "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind readnone }
 attributes #3 = { nounwind }
 
diff --git a/llvm/test/CodeGen/ARM/combine-vmovdrr.ll b/llvm/test/CodeGen/ARM/combine-vmovdrr.ll
index 358f7e3a9..01526b371 100644
--- a/llvm/test/CodeGen/ARM/combine-vmovdrr.ll
+++ b/llvm/test/CodeGen/ARM/combine-vmovdrr.ll
@@ -9,8 +9,8 @@ declare <8 x i8> @llvm.arm.neon.vtbl2(<8 x i8> %shuffle.i.i307, <8 x i8> %shuffl
 ; they are defined on VPRs and used on VPRs.
 ;
 ; CHECK-LABEL: motivatingExample:
-; CHECK: vldr [[ARG2_VAL:d[0-9]+]], [r1]
-; CHECK-NEXT: vld1.32 {[[ARG1_VALlo:d[0-9]+]], [[ARG1_VALhi:d[0-9]+]]}, [r0]
+; CHECK: vld1.32 {[[ARG1_VALlo:d[0-9]+]], [[ARG1_VALhi:d[0-9]+]]}, [r0]
+; CHECK-NEXT: vldr [[ARG2_VAL:d[0-9]+]], [r1]
 ; CHECK-NEXT: vtbl.8 [[RES:d[0-9]+]], {[[ARG1_VALlo]], [[ARG1_VALhi]]}, [[ARG2_VAL]]
 ; CHECK-NEXT: vstr [[RES]], [r1]
 ; CHECK-NEXT: bx lr
diff --git a/llvm/test/CodeGen/ARM/compare-call.ll b/llvm/test/CodeGen/ARM/compare-call.ll
index f45ed73ad..47f20a28b 100644
--- a/llvm/test/CodeGen/ARM/compare-call.ll
+++ b/llvm/test/CodeGen/ARM/compare-call.ll
@@ -18,5 +18,5 @@ UnifiedReturnBlock:             ; preds = %entry
 
 declare i32 @bar(...)
 
-; CHECK: vcmpe.f32
+; CHECK: vcmp.f32
 
diff --git a/llvm/test/CodeGen/ARM/constantpool-align.ll b/llvm/test/CodeGen/ARM/constantpool-align.ll
index 1815b8746..e8891ed31 100644
--- a/llvm/test/CodeGen/ARM/constantpool-align.ll
+++ b/llvm/test/CodeGen/ARM/constantpool-align.ll
@@ -17,3 +17,28 @@ define void @f_optsize(<4 x i32>* %p) optsize {
   store <4 x i32> <i32 -1, i32 0, i32 0, i32 -1>, <4 x i32>* %p, align 4
   ret void 
 }
+
+; CHECK-LABEL: f_pgso:
+; CHECK: vld1.64 {{.*}}, [r1]
+; CHECK: .p2align 3
+define void @f_pgso(<4 x i32>* %p) !prof !14 {
+  store <4 x i32> <i32 -1, i32 0, i32 0, i32 -1>, <4 x i32>* %p, align 4
+  ret void 
+}
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 1, !"ProfileSummary", !1}
+!1 = !{!2, !3, !4, !5, !6, !7, !8, !9}
+!2 = !{!"ProfileFormat", !"InstrProf"}
+!3 = !{!"TotalCount", i64 10000}
+!4 = !{!"MaxCount", i64 10}
+!5 = !{!"MaxInternalCount", i64 1}
+!6 = !{!"MaxFunctionCount", i64 1000}
+!7 = !{!"NumCounts", i64 3}
+!8 = !{!"NumFunctions", i64 3}
+!9 = !{!"DetailedSummary", !10}
+!10 = !{!11, !12, !13}
+!11 = !{i32 10000, i64 100, i32 1}
+!12 = !{i32 999000, i64 100, i32 1}
+!13 = !{i32 999999, i64 1, i32 2}
+!14 = !{!"function_entry_count", i64 0}
diff --git a/llvm/test/CodeGen/ARM/constantpool-promote-dbg.ll b/llvm/test/CodeGen/ARM/constantpool-promote-dbg.ll
index 261131032..0b707386f 100644
--- a/llvm/test/CodeGen/ARM/constantpool-promote-dbg.ll
+++ b/llvm/test/CodeGen/ARM/constantpool-promote-dbg.ll
@@ -19,7 +19,7 @@ entry:
   ret i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str, i32 0, i32 1), !dbg !16
 }
 
-attributes #0 = { minsize norecurse nounwind optsize readnone "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="cortex-m3" "target-features"="+hwdiv,+soft-float,-crypto,-neon" "unsafe-fp-math"="false" "use-soft-float"="true" }
+attributes #0 = { minsize norecurse nounwind optsize readnone "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="cortex-m3" "target-features"="+hwdiv,+soft-float,-crypto,-neon" "unsafe-fp-math"="false" "use-soft-float"="true" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!3, !4, !5, !6}
diff --git a/llvm/test/CodeGen/ARM/constantpool-promote.ll b/llvm/test/CodeGen/ARM/constantpool-promote.ll
index ac16e600c..43196561f 100644
--- a/llvm/test/CodeGen/ARM/constantpool-promote.ll
+++ b/llvm/test/CodeGen/ARM/constantpool-promote.ll
@@ -203,8 +203,8 @@ declare void @llvm.memcpy.p0i8.p0i8.i32(i8* nocapture writeonly, i8* nocapture r
 declare void @llvm.memmove.p0i8.p0i8.i32(i8*, i8*, i32, i1) local_unnamed_addr
 declare void @llvm.memmove.p0i16.p0i16.i32(i16*, i16*, i32, i1) local_unnamed_addr
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind }
 
 !llvm.module.flags = !{!0, !1}
diff --git a/llvm/test/CodeGen/ARM/cortex-a57-misched-ldm-wrback.ll b/llvm/test/CodeGen/ARM/cortex-a57-misched-ldm-wrback.ll
index 2c0aa98ea..be3df4aae 100644
--- a/llvm/test/CodeGen/ARM/cortex-a57-misched-ldm-wrback.ll
+++ b/llvm/test/CodeGen/ARM/cortex-a57-misched-ldm-wrback.ll
@@ -1,5 +1,5 @@
 ; REQUIRES: asserts
-; RUN: llc < %s -mtriple=armv8r-eabi -mcpu=cortex-a57 -misched-postra -enable-misched -verify-misched -debug-only=machine-scheduler -o - 2>&1 > /dev/null | FileCheck %s
+; RUN: llc < %s -mtriple=armv8r-eabi -mcpu=cortex-a57 -mattr=use-misched -verify-misched -debug-only=machine-scheduler -o - 2>&1 > /dev/null | FileCheck %s
 ; 
 
 @a = global i32 0, align 4
diff --git a/llvm/test/CodeGen/ARM/cortex-a57-misched-ldm.ll b/llvm/test/CodeGen/ARM/cortex-a57-misched-ldm.ll
index 02d1c2f55..183587085 100644
--- a/llvm/test/CodeGen/ARM/cortex-a57-misched-ldm.ll
+++ b/llvm/test/CodeGen/ARM/cortex-a57-misched-ldm.ll
@@ -1,5 +1,5 @@
 ; REQUIRES: asserts
-; RUN: llc < %s -mtriple=armv8r-eabi -mcpu=cortex-a57 -misched-postra -enable-misched -verify-misched -debug-only=machine-scheduler -o - 2>&1 > /dev/null | FileCheck %s
+; RUN: llc < %s -mtriple=armv8r-eabi -mcpu=cortex-a57 -mattr=use-misched -verify-misched -debug-only=machine-scheduler -o - 2>&1 > /dev/null | FileCheck %s
 
 ; CHECK:       ********** MI Scheduling **********
 ; We need second, post-ra scheduling to have LDM instruction combined from single-loads
diff --git a/llvm/test/CodeGen/ARM/cortex-a57-misched-stm-wrback.ll b/llvm/test/CodeGen/ARM/cortex-a57-misched-stm-wrback.ll
index 67cddc14d..2ee5f75be 100644
--- a/llvm/test/CodeGen/ARM/cortex-a57-misched-stm-wrback.ll
+++ b/llvm/test/CodeGen/ARM/cortex-a57-misched-stm-wrback.ll
@@ -1,5 +1,5 @@
 ; REQUIRES: asserts
-; RUN: llc < %s -mtriple=armv8r-eabi -mcpu=cortex-a57 -misched-postra -enable-misched -verify-misched -debug-only=machine-scheduler -o - 2>&1 > /dev/null | FileCheck %s
+; RUN: llc < %s -mtriple=armv8r-eabi -mcpu=cortex-a57 -mattr=use-misched -verify-misched -debug-only=machine-scheduler -o - 2>&1 > /dev/null | FileCheck %s
 ; N=3 STMIA_UPD should have latency 2cyc and writeback latency 1cyc
 
 ; CHECK:       ********** MI Scheduling **********
diff --git a/llvm/test/CodeGen/ARM/cortex-a57-misched-stm.ll b/llvm/test/CodeGen/ARM/cortex-a57-misched-stm.ll
index 474f39d84..026a62f35 100644
--- a/llvm/test/CodeGen/ARM/cortex-a57-misched-stm.ll
+++ b/llvm/test/CodeGen/ARM/cortex-a57-misched-stm.ll
@@ -1,5 +1,5 @@
 ; REQUIRES: asserts
-; RUN: llc < %s -mtriple=armv8r-eabi -mcpu=cortex-a57 -misched-postra -enable-misched -verify-misched -debug-only=machine-scheduler -o - 2>&1 > /dev/null | FileCheck %s
+; RUN: llc < %s -mtriple=armv8r-eabi -mcpu=cortex-a57 -mattr=use-misched -verify-misched -debug-only=machine-scheduler -o - 2>&1 > /dev/null | FileCheck %s
 ; N=3 STMIB should have latency 2cyc
 
 ; CHECK:       ********** MI Scheduling **********
diff --git a/llvm/test/CodeGen/ARM/cortex-a57-misched-vfma.ll b/llvm/test/CodeGen/ARM/cortex-a57-misched-vfma.ll
index a3e07ba17..c0318a6f6 100644
--- a/llvm/test/CodeGen/ARM/cortex-a57-misched-vfma.ll
+++ b/llvm/test/CodeGen/ARM/cortex-a57-misched-vfma.ll
@@ -3,7 +3,7 @@
 ; RUN: llc < %s -mtriple=armv8r-eabi -mcpu=cortex-a57 -enable-misched -verify-misched -debug-only=machine-scheduler -o - 2>&1 > /dev/null -fp-contract=fast | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-FAST
 ; Check latencies of vmul/vfma accumulate chains.
 
-define float @Test1(float %f1, float %f2, float %f3, float %f4, float %f5, float %f6) {
+define arm_aapcs_vfpcc float @Test1(float %f1, float %f2, float %f3, float %f4, float %f5, float %f6) {
 ; CHECK:       ********** MI Scheduling **********
 ; CHECK:       Test1:%bb.0
 
@@ -42,7 +42,7 @@ define float @Test1(float %f1, float %f2, float %f3, float %f4, float %f5, float
 }
 
 ; ASIMD form
-define <2 x float> @Test2(<2 x float> %f1, <2 x float> %f2, <2 x float> %f3, <2 x float> %f4, <2 x float> %f5, <2 x float> %f6) {
+define arm_aapcs_vfpcc <2 x float> @Test2(<2 x float> %f1, <2 x float> %f2, <2 x float> %f3, <2 x float> %f4, <2 x float> %f5, <2 x float> %f6) {
 ; CHECK:       ********** MI Scheduling **********
 ; CHECK:       Test2:%bb.0
 
@@ -80,7 +80,7 @@ define <2 x float> @Test2(<2 x float> %f1, <2 x float> %f2, <2 x float> %f3, <2
   ret <2 x float> %add2
 }
 
-define float @Test3(float %f1, float %f2, float %f3, float %f4, float %f5, float %f6) {
+define arm_aapcs_vfpcc float @Test3(float %f1, float %f2, float %f3, float %f4, float %f5, float %f6) {
 ; CHECK:       ********** MI Scheduling **********
 ; CHECK:       Test3:%bb.0
 
@@ -93,12 +93,12 @@ define float @Test3(float %f1, float %f2, float %f3, float %f4, float %f5, float
 ; CHECK-SAME:  Latency=0
 
 ; CHECK-DEFAULT: VMLSS
-; CHECK-FAST:    VFMSS
-; > VMLSS common latency = 9
+; CHECK-FAST:    VFNMSS
+; > VFNMSS common latency = 9
 ; CHECK:       Latency            : 9
 ; CHECK:       Successors:
 ; CHECK:       Data
-; > VMLSS read-advanced latency to the next VMLSS = 4
+; > VFNMSS read-advanced latency to the next VMLSS = 4
 ; CHECK-SAME:  Latency=4
 
 ; CHECK-DEFAULT: VMLSS
@@ -119,7 +119,7 @@ define float @Test3(float %f1, float %f2, float %f3, float %f4, float %f5, float
 }
 
 ; ASIMD form
-define <2 x float> @Test4(<2 x float> %f1, <2 x float> %f2, <2 x float> %f3, <2 x float> %f4, <2 x float> %f5, <2 x float> %f6) {
+define arm_aapcs_vfpcc <2 x float> @Test4(<2 x float> %f1, <2 x float> %f2, <2 x float> %f3, <2 x float> %f4, <2 x float> %f5, <2 x float> %f6) {
 ; CHECK:       ********** MI Scheduling **********
 ; CHECK:       Test4:%bb.0
 
@@ -157,7 +157,7 @@ define <2 x float> @Test4(<2 x float> %f1, <2 x float> %f2, <2 x float> %f3, <2
   ret <2 x float> %sub2
 }
 
-define float @Test5(float %f1, float %f2, float %f3) {
+define arm_aapcs_vfpcc float @Test5(float %f1, float %f2, float %f3) {
 ; CHECK:       ********** MI Scheduling **********
 ; CHECK:       Test5:%bb.0
 
@@ -176,7 +176,7 @@ define float @Test5(float %f1, float %f2, float %f3) {
 }
 
 
-define float @Test6(float %f1, float %f2, float %f3) {
+define arm_aapcs_vfpcc float @Test6(float %f1, float %f2, float %f3) {
 ; CHECK:       ********** MI Scheduling **********
 ; CHECK:       Test6:%bb.0
 
diff --git a/llvm/test/CodeGen/ARM/cortex-a57-misched-vldm-wrback.ll b/llvm/test/CodeGen/ARM/cortex-a57-misched-vldm-wrback.ll
index 1baf472ca..88b772cc2 100644
--- a/llvm/test/CodeGen/ARM/cortex-a57-misched-vldm-wrback.ll
+++ b/llvm/test/CodeGen/ARM/cortex-a57-misched-vldm-wrback.ll
@@ -1,5 +1,5 @@
 ; REQUIRES: asserts
-; RUN: llc < %s -mtriple=armv8r-eabi -mcpu=cortex-a57 -misched-postra -enable-misched -verify-misched -debug-only=machine-scheduler -o - 2>&1 > /dev/null | FileCheck %s
+; RUN: llc < %s -mtriple=armv8r-eabi -mcpu=cortex-a57 -mattr=use-misched -verify-misched -debug-only=machine-scheduler -o - 2>&1 > /dev/null | FileCheck %s
 ; 
 
 @a = global double 0.0, align 4
diff --git a/llvm/test/CodeGen/ARM/cortex-a57-misched-vldm.ll b/llvm/test/CodeGen/ARM/cortex-a57-misched-vldm.ll
index 8da133e80..ac208c65a 100644
--- a/llvm/test/CodeGen/ARM/cortex-a57-misched-vldm.ll
+++ b/llvm/test/CodeGen/ARM/cortex-a57-misched-vldm.ll
@@ -1,5 +1,5 @@
 ; REQUIRES: asserts
-; RUN: llc < %s -mtriple=armv8r-eabi -mcpu=cortex-a57 -misched-postra -enable-misched -verify-misched -debug-only=machine-scheduler -o - 2>&1 > /dev/null | FileCheck %s
+; RUN: llc < %s -mtriple=armv8r-eabi -mcpu=cortex-a57 -mattr=use-misched -verify-misched -debug-only=machine-scheduler -o - 2>&1 > /dev/null | FileCheck %s
 
 ; CHECK:       ********** MI Scheduling **********
 ; We need second, post-ra scheduling to have VLDM instruction combined from single-loads
diff --git a/llvm/test/CodeGen/ARM/cortex-a57-misched-vstm-wrback.ll b/llvm/test/CodeGen/ARM/cortex-a57-misched-vstm-wrback.ll
index 05c498eee..c517f46e5 100644
--- a/llvm/test/CodeGen/ARM/cortex-a57-misched-vstm-wrback.ll
+++ b/llvm/test/CodeGen/ARM/cortex-a57-misched-vstm-wrback.ll
@@ -1,5 +1,5 @@
 ; REQUIRES: asserts
-; RUN: llc < %s -mtriple=armv8r-eabi -mcpu=cortex-a57 -misched-postra -enable-misched -verify-misched -debug-only=machine-scheduler -o - 2>&1 > /dev/null | FileCheck %s
+; RUN: llc < %s -mtriple=armv8r-eabi -mcpu=cortex-a57 -mattr=use-misched -verify-misched -debug-only=machine-scheduler -o - 2>&1 > /dev/null | FileCheck %s
 
 ; CHECK:       ********** MI Scheduling **********
 ; We need second, post-ra scheduling to have VSTM instruction combined from single-stores
diff --git a/llvm/test/CodeGen/ARM/cortex-a57-misched-vstm.ll b/llvm/test/CodeGen/ARM/cortex-a57-misched-vstm.ll
index f31474f66..5e9041ce0 100644
--- a/llvm/test/CodeGen/ARM/cortex-a57-misched-vstm.ll
+++ b/llvm/test/CodeGen/ARM/cortex-a57-misched-vstm.ll
@@ -1,5 +1,5 @@
 ; REQUIRES: asserts
-; RUN: llc < %s -mtriple=armv8r-eabi -mcpu=cortex-a57 -misched-postra -enable-misched -verify-misched -debug-only=machine-scheduler -o - 2>&1 > /dev/null | FileCheck %s
+; RUN: llc < %s -mtriple=armv8r-eabi -mcpu=cortex-a57 -mattr=use-misched -verify-misched -debug-only=machine-scheduler -o - 2>&1 > /dev/null | FileCheck %s
 
 ; CHECK:       ********** MI Scheduling **********
 ; We need second, post-ra scheduling to have VSTM instruction combined from single-stores
diff --git a/llvm/test/CodeGen/ARM/csr-split.ll b/llvm/test/CodeGen/ARM/csr-split.ll
new file mode 100644
index 000000000..f9246cb6d
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/csr-split.ll
@@ -0,0 +1,137 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -verify-machineinstrs -mtriple=arm-unknown-linux < %s | FileCheck %s
+
+; Check CSR split can work properly for tests below.
+
+@a = common dso_local local_unnamed_addr global i32 0, align 4
+
+define dso_local signext i32 @test1(i32* %b) local_unnamed_addr  {
+; CHECK-LABEL: test1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    push {r4, lr}
+; CHECK-NEXT:    mov r4, r0
+; CHECK-NEXT:    ldr r0, .LCPI0_0
+; CHECK-NEXT:    ldr r0, [r0]
+; CHECK-NEXT:    cmp r0, r4
+; CHECK-NEXT:    popne {r4, lr}
+; CHECK-NEXT:    movne pc, lr
+; CHECK-NEXT:    bl callVoid
+; CHECK-NEXT:    mov r0, r4
+; CHECK-NEXT:    pop {r4, lr}
+; CHECK-NEXT:    b callNonVoid
+; CHECK-NEXT:    .p2align 2
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI0_0:
+; CHECK-NEXT:    .long a
+entry:
+  %0 = load i32, i32* @a, align 4, !tbaa !2
+  %conv = sext i32 %0 to i64
+  %1 = inttoptr i64 %conv to i32*
+  %cmp = icmp eq i32* %1, %b
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %call = tail call signext i32 bitcast (i32 (...)* @callVoid to i32 ()*)()
+  %call2 = tail call signext i32 @callNonVoid(i32* %b)
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %entry
+  %retval.0 = phi i32 [ %call2, %if.then ], [ undef, %entry ]
+  ret i32 %retval.0
+}
+
+declare signext i32 @callVoid(...) local_unnamed_addr
+
+declare signext i32 @callNonVoid(i32*) local_unnamed_addr
+
+define dso_local signext i32 @test2(i32* %p1) local_unnamed_addr  {
+; CHECK-LABEL: test2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    push {r4, lr}
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    beq .LBB1_2
+; CHECK-NEXT:  @ %bb.1: @ %if.end
+; CHECK-NEXT:    mov r4, r0
+; CHECK-NEXT:    ldr r0, .LCPI1_0
+; CHECK-NEXT:    ldr r0, [r0]
+; CHECK-NEXT:    cmp r0, r4
+; CHECK-NEXT:    beq .LBB1_3
+; CHECK-NEXT:  .LBB1_2: @ %return
+; CHECK-NEXT:    mov r0, #0
+; CHECK-NEXT:    pop {r4, lr}
+; CHECK-NEXT:    mov pc, lr
+; CHECK-NEXT:  .LBB1_3: @ %if.then2
+; CHECK-NEXT:    bl callVoid
+; CHECK-NEXT:    mov r0, r4
+; CHECK-NEXT:    pop {r4, lr}
+; CHECK-NEXT:    b callNonVoid
+; CHECK-NEXT:    .p2align 2
+; CHECK-NEXT:  @ %bb.4:
+; CHECK-NEXT:  .LCPI1_0:
+; CHECK-NEXT:    .long a
+entry:
+  %tobool = icmp eq i32* %p1, null
+  br i1 %tobool, label %return, label %if.end
+
+if.end:                                           ; preds = %entry
+  %0 = load i32, i32* @a, align 4, !tbaa !2
+  %conv = sext i32 %0 to i64
+  %1 = inttoptr i64 %conv to i32*
+  %cmp = icmp eq i32* %1, %p1
+  br i1 %cmp, label %if.then2, label %return
+
+if.then2:                                         ; preds = %if.end
+  %call = tail call signext i32 bitcast (i32 (...)* @callVoid to i32 ()*)()
+  %call3 = tail call signext i32 @callNonVoid(i32* nonnull %p1)
+  br label %return
+
+return:                                           ; preds = %if.end, %entry, %if.then2
+  %retval.0 = phi i32 [ %call3, %if.then2 ], [ 0, %entry ], [ 0, %if.end ]
+  ret i32 %retval.0
+}
+
+
+define dso_local i8* @test3(i8** nocapture %p1, i8 zeroext %p2) local_unnamed_addr  {
+; CHECK-LABEL: test3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    push {r4, r5, r11, lr}
+; CHECK-NEXT:    ldr r4, [r0]
+; CHECK-NEXT:    cmp r4, #0
+; CHECK-NEXT:    beq .LBB2_2
+; CHECK-NEXT:  @ %bb.1: @ %land.rhs
+; CHECK-NEXT:    mov r5, r0
+; CHECK-NEXT:    mov r0, r4
+; CHECK-NEXT:    bl bar
+; CHECK-NEXT:    str r0, [r5]
+; CHECK-NEXT:  .LBB2_2: @ %land.end
+; CHECK-NEXT:    mov r0, r4
+; CHECK-NEXT:    pop {r4, r5, r11, lr}
+; CHECK-NEXT:    mov pc, lr
+entry:
+  %0 = load i8*, i8** %p1, align 8, !tbaa !6
+  %tobool = icmp eq i8* %0, null
+  br i1 %tobool, label %land.end, label %land.rhs
+
+land.rhs:                                         ; preds = %entry
+  %call = tail call i8* @bar(i8* nonnull %0, i8 zeroext %p2)
+  store i8* %call, i8** %p1, align 8, !tbaa !6
+  br label %land.end
+
+land.end:                                         ; preds = %entry, %land.rhs
+  ret i8* %0
+}
+
+declare i8* @bar(i8*, i8 zeroext) local_unnamed_addr
+
+
+!llvm.module.flags = !{!0}
+!llvm.ident = !{!1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{!"clang version 10.0.0 (trunk 367381) (llvm/trunk 367388)"}
+!2 = !{!3, !3, i64 0}
+!3 = !{!"int", !4, i64 0}
+!4 = !{!"omnipotent char", !5, i64 0}
+!5 = !{!"Simple C/C++ TBAA"}
+!6 = !{!7, !7, i64 0}
+!7 = !{!"any pointer", !4, i64 0}
diff --git a/llvm/test/CodeGen/ARM/cxx-tlscc.ll b/llvm/test/CodeGen/ARM/cxx-tlscc.ll
index 6a66c5f19..649ed17db 100644
--- a/llvm/test/CodeGen/ARM/cxx-tlscc.ll
+++ b/llvm/test/CodeGen/ARM/cxx-tlscc.ll
@@ -33,7 +33,7 @@ declare i32 @_tlv_atexit(void (i8*)*, i8*, i8*)
 ; THUMB: blx
 ; THUMB: r4
 ; THUMB: pop {{.*}}r4
-define cxx_fast_tlscc nonnull %struct.S* @_ZTW2sg() nounwind "no-frame-pointer-elim"="true" {
+define cxx_fast_tlscc nonnull %struct.S* @_ZTW2sg() nounwind "frame-pointer"="all" {
   %.b.i = load i1, i1* @__tls_guard, align 1
   br i1 %.b.i, label %__tls_init.exit, label %init.i
 
@@ -95,7 +95,7 @@ __tls_init.exit:
 ; CHECK-O0-NOT: vpop
 ; CHECK-O0-NOT: vldr
 ; CHECK-O0: pop
-define cxx_fast_tlscc nonnull i32* @_ZTW4sum1() nounwind "no-frame-pointer-elim"="true" {
+define cxx_fast_tlscc nonnull i32* @_ZTW4sum1() nounwind "frame-pointer"="all" {
   ret i32* @sum1
 }
 
@@ -109,7 +109,7 @@ define cxx_fast_tlscc nonnull i32* @_ZTW4sum1() nounwind "no-frame-pointer-elim"
 ; CHECK-O0-NOT: vldr
 ; CHECK-O0: pop
 declare cxx_fast_tlscc void @tls_helper()
-define cxx_fast_tlscc %class.C* @tls_test2() #1 "no-frame-pointer-elim"="true" {
+define cxx_fast_tlscc %class.C* @tls_test2() #1 "frame-pointer"="all" {
   call cxx_fast_tlscc void @tls_helper()
   ret %class.C* @tC
 }
@@ -119,7 +119,7 @@ define cxx_fast_tlscc %class.C* @tls_test2() #1 "no-frame-pointer-elim"="true" {
 declare %class.C* @_ZN1CD1Ev(%class.C* readnone returned %this)
 ; CHECK-LABEL: tls_test
 ; CHECK: bl __tlv_atexit
-define cxx_fast_tlscc void @__tls_test() "no-frame-pointer-elim"="true" {
+define cxx_fast_tlscc void @__tls_test() "frame-pointer"="all" {
 entry:
   store i32 0, i32* getelementptr inbounds (%class.C, %class.C* @tC, i64 0, i32 0), align 4
   %0 = tail call i32 @_tlv_atexit(void (i8*)* bitcast (%class.C* (%class.C*)* @_ZN1CD1Ev to void (i8*)*), i8* bitcast (%class.C* @tC to i8*), i8* nonnull @__dso_handle) #1
@@ -127,7 +127,7 @@ entry:
 }
 
 declare void @somefunc()
-define cxx_fast_tlscc void @test_ccmismatch_notail() "no-frame-pointer-elim"="true" {
+define cxx_fast_tlscc void @test_ccmismatch_notail() "frame-pointer"="all" {
 ; A tail call is not possible here because somefunc does not preserve enough
 ; registers.
 ; CHECK-LABEL: test_ccmismatch_notail:
@@ -138,7 +138,7 @@ define cxx_fast_tlscc void @test_ccmismatch_notail() "no-frame-pointer-elim"="tr
 }
 
 declare cxx_fast_tlscc void @some_fast_tls_func()
-define void @test_ccmismatch_tail() "no-frame-pointer-elim"="true" {
+define void @test_ccmismatch_tail() "frame-pointer"="all" {
 ; We can perform a tail call here because some_fast_tls_func preserves all
 ; necessary registers (and more).
 ; CHECK-LABEL: test_ccmismatch_tail:
@@ -148,5 +148,5 @@ define void @test_ccmismatch_tail() "no-frame-pointer-elim"="true" {
   ret void
 }
 
-attributes #0 = { nounwind "no-frame-pointer-elim"="true" }
+attributes #0 = { nounwind "frame-pointer"="all" }
 attributes #1 = { nounwind }
diff --git a/llvm/test/CodeGen/ARM/dagcombine-anyexttozeroext.ll b/llvm/test/CodeGen/ARM/dagcombine-anyexttozeroext.ll
index de2468147..643468c6d 100644
--- a/llvm/test/CodeGen/ARM/dagcombine-anyexttozeroext.ll
+++ b/llvm/test/CodeGen/ARM/dagcombine-anyexttozeroext.ll
@@ -113,17 +113,14 @@ define float @k(<8 x i8>* nocapture %in) {
 }
 
 define float @KnownUpperZero(<4 x i16> %v) {
-; FIXME: uxtb are not required
 ; CHECK-LABEL: KnownUpperZero:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vmov.i16 d16, #0x3
-; CHECK-NEXT:    vmov d17, r0, r1
-; CHECK-NEXT:    vand d16, d17, d16
+; CHECK-NEXT:    vmov d16, r0, r1
 ; CHECK-NEXT:    vmov.u16 r0, d16[0]
 ; CHECK-NEXT:    vmov.u16 r1, d16[3]
-; CHECK-NEXT:    uxtb r0, r0
+; CHECK-NEXT:    and r0, r0, #3
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    uxtb r0, r1
+; CHECK-NEXT:    and r0, r1, #3
 ; CHECK-NEXT:    vmov s2, r0
 ; CHECK-NEXT:    vcvt.f32.s32 s0, s0
 ; CHECK-NEXT:    vcvt.f32.s32 s2, s2
diff --git a/llvm/test/CodeGen/ARM/debug-info-arg.ll b/llvm/test/CodeGen/ARM/debug-info-arg.ll
index 37fa2884b..579d04131 100644
--- a/llvm/test/CodeGen/ARM/debug-info-arg.ll
+++ b/llvm/test/CodeGen/ARM/debug-info-arg.ll
@@ -6,7 +6,7 @@ target triple = "thumbv7-apple-ios"
 
 %struct.tag_s = type { i32, i32, i32 }
 
-define void @foo(%struct.tag_s* nocapture %this, %struct.tag_s* %c, i64 %x, i64 %y, %struct.tag_s* nocapture %ptr1, %struct.tag_s* nocapture %ptr2) nounwind ssp "no-frame-pointer-elim"="true" !dbg !1 {
+define void @foo(%struct.tag_s* nocapture %this, %struct.tag_s* %c, i64 %x, i64 %y, %struct.tag_s* nocapture %ptr1, %struct.tag_s* nocapture %ptr2) nounwind ssp "frame-pointer"="all" !dbg !1 {
   tail call void @llvm.dbg.value(metadata %struct.tag_s* %this, metadata !5, metadata !DIExpression()), !dbg !20
   tail call void @llvm.dbg.value(metadata %struct.tag_s* %c, metadata !13, metadata !DIExpression()), !dbg !21
   tail call void @llvm.dbg.value(metadata i64 %x, metadata !14, metadata !DIExpression()), !dbg !22
diff --git a/llvm/test/CodeGen/ARM/debug-info-blocks.ll b/llvm/test/CodeGen/ARM/debug-info-blocks.ll
index cc1a45f23..8b31e7a51 100644
--- a/llvm/test/CodeGen/ARM/debug-info-blocks.ll
+++ b/llvm/test/CodeGen/ARM/debug-info-blocks.ll
@@ -155,7 +155,7 @@ define hidden void @foobar_func_block_invoke_0(i8* %.block_descriptor, %0* %load
 !47 = !DIDerivedType(tag: DW_TAG_member, name: "DestroyFuncPtr", line: 307, size: 32, align: 32, offset: 96, file: !153, scope: !40, baseType: !46)
 !48 = !DIDerivedType(tag: DW_TAG_member, name: "mydata", line: 609, size: 32, align: 32, offset: 160, file: !152, scope: !24, baseType: !49)
 !49 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 32, scope: !0, baseType: !50)
-!50 = !DICompositeType(tag: DW_TAG_structure_type, size: 224, flags: DIFlagBlockByrefStruct, file: !152, scope: !24, elements: !51)
+!50 = !DICompositeType(tag: DW_TAG_structure_type, size: 224, file: !152, scope: !24, elements: !51)
 !51 = !{!52, !53, !54, !55, !56, !57, !58}
 !52 = !DIDerivedType(tag: DW_TAG_member, name: "__isa", size: 32, align: 32, file: !152, scope: !24, baseType: !32)
 !53 = !DIDerivedType(tag: DW_TAG_member, name: "__forwarding", size: 32, align: 32, offset: 32, file: !152, scope: !24, baseType: !32)
diff --git a/llvm/test/CodeGen/ARM/debug-info-sreg2.ll b/llvm/test/CodeGen/ARM/debug-info-sreg2.ll
index 26af21171..aa5c423ca 100644
--- a/llvm/test/CodeGen/ARM/debug-info-sreg2.ll
+++ b/llvm/test/CodeGen/ARM/debug-info-sreg2.ll
@@ -7,7 +7,7 @@ target triple = "thumbv7-apple-macosx10.6.7"
 ; of the size of the location description.
 
 ; CHECK: 0x00000000:
-; CHECK-NEXT:        [0x{{[0-9]*[a-f]*}}, 0x{{[0-9]*[a-f]*}}): DW_OP_regx D8
+; CHECK-NEXT:        (0x{{[0-9]*[a-f]*}}, 0x{{[0-9]*[a-f]*}}): DW_OP_regx D8
 
 define void @_Z3foov() optsize ssp !dbg !1 {
 entry:
diff --git a/llvm/test/CodeGen/ARM/debuginfo-split-carryexpr.ll b/llvm/test/CodeGen/ARM/debuginfo-split-carryexpr.ll
new file mode 100644
index 000000000..62ebc4cd5
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/debuginfo-split-carryexpr.ll
@@ -0,0 +1,51 @@
+; RUN: llc < %s -mtriple=armv7-linux-gnueabihf -stop-before=finalize-isel | FileCheck %s
+
+; When splitting-up integers during CodeGen, if the debug info contains
+; an expression with carry operators (i.e. arithmetic and shift ops), the
+; debug information will be dropped as these operators cannot be correctly
+; expressed across different registers.
+
+; CHECK: [[HIGH:![0-9]+]] = !DILocalVariable(name: "high"
+; CHECK: [[LOW:![0-9]+]] = !DILocalVariable(name: "low"
+;
+; As the debug information for "high" contains an arithmetic shift, while for
+; "low" it does not, only the former should be undefined.
+; CHECK-LABEL: body:
+; CHECK: [[LOWR:%[0-9]+]]:gpr = COPY $r0
+; CHECK: DBG_VALUE [[LOWR]], $noreg, [[LOW]]
+; CHECK: DBG_VALUE $noreg, $noreg, [[HIGH]]
+; CHECK: DBG_VALUE $noreg, $noreg, [[HIGH]]
+
+define dso_local i64 @_Z2fnx(i64 returned %value) local_unnamed_addr !dbg !7 {
+entry:
+  call void @llvm.dbg.value(metadata i64 %value, metadata !13, metadata !DIExpression()), !dbg !17
+  call void @llvm.dbg.value(metadata i64 %value, metadata !14, metadata !DIExpression(DW_OP_constu, 32, DW_OP_shra, DW_OP_LLVM_convert, 64, DW_ATE_unsigned, DW_OP_LLVM_convert, 32, DW_ATE_unsigned, DW_OP_stack_value)), !dbg !17
+  call void @llvm.dbg.value(metadata i64 %value, metadata !16, metadata !DIExpression(DW_OP_LLVM_convert, 64, DW_ATE_unsigned, DW_OP_LLVM_convert, 32, DW_ATE_unsigned, DW_OP_stack_value)), !dbg !17
+  ret i64 %value, !dbg !18
+}
+
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 10.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "test.cpp", directory: "/tmp")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{!"clang version 10.0.0"}
+!7 = distinct !DISubprogram(name: "fn", linkageName: "_Z2fnx", scope: !8, file: !8, line: 2, type: !9, scopeLine: 2, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !12)
+!8 = !DIFile(filename: "test.cpp", directory: "/tmp")
+!9 = !DISubroutineType(types: !10)
+!10 = !{!11, !11}
+!11 = !DIBasicType(name: "long long int", size: 64, encoding: DW_ATE_signed)
+!12 = !{!13, !14, !16}
+!13 = !DILocalVariable(name: "value", arg: 1, scope: !7, file: !8, line: 2, type: !11)
+!14 = !DILocalVariable(name: "high", scope: !7, file: !8, line: 3, type: !15)
+!15 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!16 = !DILocalVariable(name: "low", scope: !7, file: !8, line: 4, type: !15)
+!17 = !DILocation(line: 0, scope: !7)
+!18 = !DILocation(line: 5, column: 3, scope: !7)
diff --git a/llvm/test/CodeGen/ARM/disable-fp-elim.ll b/llvm/test/CodeGen/ARM/disable-fp-elim.ll
index ddbe36597..82a5e149c 100644
--- a/llvm/test/CodeGen/ARM/disable-fp-elim.ll
+++ b/llvm/test/CodeGen/ARM/disable-fp-elim.ll
@@ -4,7 +4,7 @@
 ; RUN: llc < %s -mtriple armv7-none-linux-gnueabi -frame-pointer=none -O0 | FileCheck %s --check-prefix=DISABLE-FP-ELIM
 
 ; Check that command line option "-frame-pointer=all" overrides function attribute
-; "no-frame-pointer-elim". Also, check frame pointer elimination is disabled
+; "frame-pointer"="all". Also, check frame pointer elimination is disabled
 ; when fast-isel is used.
 
 ; ENABLE-FP-ELIM-NOT: .setfp
@@ -22,4 +22,4 @@ entry:
 
 declare i32 @foo2(i32)
 
-attributes #0 = { nounwind "no-frame-pointer-elim"="true" }
+attributes #0 = { nounwind "frame-pointer"="all" }
diff --git a/llvm/test/CodeGen/ARM/dsp-mlal.ll b/llvm/test/CodeGen/ARM/dsp-mlal.ll
index 04968e475..53593034b 100644
--- a/llvm/test/CodeGen/ARM/dsp-mlal.ll
+++ b/llvm/test/CodeGen/ARM/dsp-mlal.ll
@@ -1,16 +1,34 @@
-; RUN: llc -mtriple=thumbv7m -mattr=+dsp %s -o - | FileCheck %s
-; RUN: llc -mtriple=armv7a %s -o - | FileCheck %s
-; RUN: llc -mtriple=thumbv7m -mattr=-dsp %s -o - | FileCheck --check-prefix=NODSP %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv7m -mattr=+dsp %s -o - | FileCheck %s --check-prefix=DSP
+; RUN: llc -mtriple=armv7a %s -o - | FileCheck %s --check-prefix=ARM7
+; RUN: llc -mtriple=thumbv7m -mattr=-dsp %s -o - | FileCheck %s --check-prefix=NODSP
 
 define hidden i32 @SMMULR_SMMLAR(i32 %a, i32 %b0, i32 %b1, i32 %Xn, i32 %Xn1) local_unnamed_addr {
-entry:
-; CHECK-LABEL: SMMULR_SMMLAR:
-; CHECK: ldr r0, [sp]
-; CHECK-NEXT: smmulr r0, {{(r0, r2|r2, r0)}}
-; CHECK-NEXT: smmlar r0, {{(r1, r3|r3, r1)}}, r0
+; DSP-LABEL: SMMULR_SMMLAR:
+; DSP:       @ %bb.0: @ %entry
+; DSP-NEXT:    ldr r0, [sp]
+; DSP-NEXT:    smmulr r0, r0, r2
+; DSP-NEXT:    smmlar r0, r3, r1, r0
+; DSP-NEXT:    bx lr
+;
+; ARM7-LABEL: SMMULR_SMMLAR:
+; ARM7:       @ %bb.0: @ %entry
+; ARM7-NEXT:    ldr r0, [sp]
+; ARM7-NEXT:    smmulr r0, r0, r2
+; ARM7-NEXT:    smmlar r0, r3, r1, r0
+; ARM7-NEXT:    bx lr
+;
 ; NODSP-LABEL: SMMULR_SMMLAR:
-; NODSP-NOT: smmulr
-; NODSP-NOT: smmlar
+; NODSP:       @ %bb.0: @ %entry
+; NODSP-NEXT:    push {r4, lr}
+; NODSP-NEXT:    ldr.w lr, [sp, #8]
+; NODSP-NEXT:    movs r0, #0
+; NODSP-NEXT:    mov.w r4, #-2147483648
+; NODSP-NEXT:    mov.w r12, #-2147483648
+; NODSP-NEXT:    smlal r4, r0, lr, r2
+; NODSP-NEXT:    smlal r12, r0, r3, r1
+; NODSP-NEXT:    pop {r4, pc}
+entry:
   %conv = sext i32 %b1 to i64
   %conv1 = sext i32 %Xn1 to i64
   %mul = mul nsw i64 %conv1, %conv
@@ -27,11 +45,24 @@ entry:
 }
 
 define hidden i32 @SMMULR(i32 %a, i32 %b) local_unnamed_addr {
-entry:
-; CHECK-LABEL: SMMULR:
-; CHECK: smmulr r0, {{(r0, r1|r1, r0)}}
+; DSP-LABEL: SMMULR:
+; DSP:       @ %bb.0: @ %entry
+; DSP-NEXT:    smmulr r0, r1, r0
+; DSP-NEXT:    bx lr
+;
+; ARM7-LABEL: SMMULR:
+; ARM7:       @ %bb.0: @ %entry
+; ARM7-NEXT:    smmulr r0, r1, r0
+; ARM7-NEXT:    bx lr
+;
 ; NODSP-LABEL: SMMULR:
-; NODSP-NOT: smmulr
+; NODSP:       @ %bb.0: @ %entry
+; NODSP-NEXT:    movs r2, #0
+; NODSP-NEXT:    mov.w r3, #-2147483648
+; NODSP-NEXT:    smlal r3, r2, r1, r0
+; NODSP-NEXT:    mov r0, r2
+; NODSP-NEXT:    bx lr
+entry:
   %conv = sext i32 %a to i64
   %conv1 = sext i32 %b to i64
   %mul = mul nsw i64 %conv1, %conv
@@ -42,11 +73,21 @@ entry:
 }
 
 define hidden i32 @SMMUL(i32 %a, i32 %b) local_unnamed_addr {
-entry:
-; CHECK-LABEL: SMMUL:
-; CHECK: smmul r0, {{(r0, r1|r1, r0)}}
+; DSP-LABEL: SMMUL:
+; DSP:       @ %bb.0: @ %entry
+; DSP-NEXT:    smmul r0, r1, r0
+; DSP-NEXT:    bx lr
+;
+; ARM7-LABEL: SMMUL:
+; ARM7:       @ %bb.0: @ %entry
+; ARM7-NEXT:    smmul r0, r1, r0
+; ARM7-NEXT:    bx lr
+;
 ; NODSP-LABEL: SMMUL:
-; NODSP-NOT: smmul
+; NODSP:       @ %bb.0: @ %entry
+; NODSP-NEXT:    smull r1, r0, r1, r0
+; NODSP-NEXT:    bx lr
+entry:
   %conv = sext i32 %a to i64
   %conv1 = sext i32 %b to i64
   %mul = mul nsw i64 %conv1, %conv
@@ -56,11 +97,23 @@ entry:
 }
 
 define hidden i32 @SMMLSR(i32 %a, i32 %b, i32 %c) local_unnamed_addr {
-entry:
-; CHECK-LABEL: SMMLSR:
-; CHECK: smmlsr r0, {{(r1, r2|r2, r1)}}, r0
+; DSP-LABEL: SMMLSR:
+; DSP:       @ %bb.0: @ %entry
+; DSP-NEXT:    smmlsr r0, r2, r1, r0
+; DSP-NEXT:    bx lr
+;
+; ARM7-LABEL: SMMLSR:
+; ARM7:       @ %bb.0: @ %entry
+; ARM7-NEXT:    smmlsr r0, r2, r1, r0
+; ARM7-NEXT:    bx lr
+;
 ; NODSP-LABEL: SMMLSR:
-; NODSP-NOT: smmlsr
+; NODSP:       @ %bb.0: @ %entry
+; NODSP-NEXT:    smull r1, r2, r2, r1
+; NODSP-NEXT:    rsbs.w r1, r1, #-2147483648
+; NODSP-NEXT:    sbcs r0, r2
+; NODSP-NEXT:    bx lr
+entry:
   %conv6 = zext i32 %a to i64
   %shl = shl nuw i64 %conv6, 32
   %conv1 = sext i32 %b to i64
@@ -74,11 +127,26 @@ entry:
 }
 
 define hidden i32 @NOT_SMMLSR(i32 %a, i32 %b, i32 %c) local_unnamed_addr {
-entry:
-; CHECK-LABEL: NOT_SMMLSR:
-; CHECK-NOT: smmlsr
+; DSP-LABEL: NOT_SMMLSR:
+; DSP:       @ %bb.0: @ %entry
+; DSP-NEXT:    smmulr r1, r2, r1
+; DSP-NEXT:    subs r0, r0, r1
+; DSP-NEXT:    bx lr
+;
+; ARM7-LABEL: NOT_SMMLSR:
+; ARM7:       @ %bb.0: @ %entry
+; ARM7-NEXT:    smmulr r1, r2, r1
+; ARM7-NEXT:    sub r0, r0, r1
+; ARM7-NEXT:    bx lr
+;
 ; NODSP-LABEL: NOT_SMMLSR:
-; NODSP-NOT: smmlsr
+; NODSP:       @ %bb.0: @ %entry
+; NODSP-NEXT:    mov.w r12, #0
+; NODSP-NEXT:    mov.w r3, #-2147483648
+; NODSP-NEXT:    smlal r3, r12, r2, r1
+; NODSP-NEXT:    sub.w r0, r0, r12
+; NODSP-NEXT:    bx lr
+entry:
   %conv = sext i32 %b to i64
   %conv1 = sext i32 %c to i64
   %mul = mul nsw i64 %conv1, %conv
@@ -90,11 +158,23 @@ entry:
 }
 
 define hidden i32 @SMMLS(i32 %a, i32 %b, i32 %c) local_unnamed_addr {
-entry:
-; CHECK-LABEL: SMMLS:
-; CHECK: smmls r0, {{(r1, r2|r2, r1)}}, r0
+; DSP-LABEL: SMMLS:
+; DSP:       @ %bb.0: @ %entry
+; DSP-NEXT:    smmls r0, r2, r1, r0
+; DSP-NEXT:    bx lr
+;
+; ARM7-LABEL: SMMLS:
+; ARM7:       @ %bb.0: @ %entry
+; ARM7-NEXT:    smmls r0, r2, r1, r0
+; ARM7-NEXT:    bx lr
+;
 ; NODSP-LABEL: SMMLS:
-; NODSP-NOT: smmls
+; NODSP:       @ %bb.0: @ %entry
+; NODSP-NEXT:    smull r1, r2, r2, r1
+; NODSP-NEXT:    rsbs r1, r1, #0
+; NODSP-NEXT:    sbcs r0, r2
+; NODSP-NEXT:    bx lr
+entry:
   %conv5 = zext i32 %a to i64
   %shl = shl nuw i64 %conv5, 32
   %conv1 = sext i32 %b to i64
@@ -107,11 +187,24 @@ entry:
 }
 
 define hidden i32 @NOT_SMMLS(i32 %a, i32 %b, i32 %c) local_unnamed_addr {
-entry:
-; CHECK-LABEL: NOT_SMMLS:
-; CHECK-NOT: smmls
+; DSP-LABEL: NOT_SMMLS:
+; DSP:       @ %bb.0: @ %entry
+; DSP-NEXT:    smmul r1, r2, r1
+; DSP-NEXT:    subs r0, r0, r1
+; DSP-NEXT:    bx lr
+;
+; ARM7-LABEL: NOT_SMMLS:
+; ARM7:       @ %bb.0: @ %entry
+; ARM7-NEXT:    smmul r1, r2, r1
+; ARM7-NEXT:    sub r0, r0, r1
+; ARM7-NEXT:    bx lr
+;
 ; NODSP-LABEL: NOT_SMMLS:
-; NODSP-NOT: smmls
+; NODSP:       @ %bb.0: @ %entry
+; NODSP-NEXT:    smull r1, r2, r2, r1
+; NODSP-NEXT:    subs r0, r0, r2
+; NODSP-NEXT:    bx lr
+entry:
   %conv = sext i32 %b to i64
   %conv1 = sext i32 %c to i64
   %mul = mul nsw i64 %conv1, %conv
@@ -122,11 +215,22 @@ entry:
 }
 
 define hidden i32 @SMMLA(i32 %a, i32 %b, i32 %c) local_unnamed_addr {
-entry:
-; CHECK-LABEL: SMMLA:
-; CHECK: smmla r0, {{(r1, r2|r2, r1)}}, r0
+; DSP-LABEL: SMMLA:
+; DSP:       @ %bb.0: @ %entry
+; DSP-NEXT:    smmla r0, r1, r2, r0
+; DSP-NEXT:    bx lr
+;
+; ARM7-LABEL: SMMLA:
+; ARM7:       @ %bb.0: @ %entry
+; ARM7-NEXT:    smmla r0, r2, r1, r0
+; ARM7-NEXT:    bx lr
+;
 ; NODSP-LABEL: SMMLA:
-; NODSP-NOT: smmla
+; NODSP:       @ %bb.0: @ %entry
+; NODSP-NEXT:    smull r1, r2, r2, r1
+; NODSP-NEXT:    add r0, r2
+; NODSP-NEXT:    bx lr
+entry:
   %conv = sext i32 %b to i64
   %conv1 = sext i32 %c to i64
   %mul = mul nsw i64 %conv1, %conv
@@ -137,11 +241,22 @@ entry:
 }
 
 define hidden i32 @SMMLAR(i32 %a, i32 %b, i32 %c) local_unnamed_addr {
-entry:
-; CHECK-LABEL: SMMLAR:
-; CHECK: smmlar r0, {{(r1, r2|r2, r1)}}, r0
+; DSP-LABEL: SMMLAR:
+; DSP:       @ %bb.0: @ %entry
+; DSP-NEXT:    smmlar r0, r2, r1, r0
+; DSP-NEXT:    bx lr
+;
+; ARM7-LABEL: SMMLAR:
+; ARM7:       @ %bb.0: @ %entry
+; ARM7-NEXT:    smmlar r0, r2, r1, r0
+; ARM7-NEXT:    bx lr
+;
 ; NODSP-LABEL: SMMLAR:
-; NODSP-NOT: smmlar
+; NODSP:       @ %bb.0: @ %entry
+; NODSP-NEXT:    mov.w r3, #-2147483648
+; NODSP-NEXT:    smlal r3, r0, r2, r1
+; NODSP-NEXT:    bx lr
+entry:
   %conv7 = zext i32 %a to i64
   %shl = shl nuw i64 %conv7, 32
   %conv1 = sext i32 %b to i64
@@ -155,11 +270,27 @@ entry:
 }
 
 define hidden i32 @NOT_SMMLA(i32 %a, i32 %b, i32 %c) local_unnamed_addr {
-entry:
-; CHECK-LABEL: NOT_SMMLA:
-; CHECK-NOT: smmla
+; DSP-LABEL: NOT_SMMLA:
+; DSP:       @ %bb.0: @ %entry
+; DSP-NEXT:    smmul r1, r2, r1
+; DSP-NEXT:    eor r1, r1, #-2147483648
+; DSP-NEXT:    add r0, r1
+; DSP-NEXT:    bx lr
+;
+; ARM7-LABEL: NOT_SMMLA:
+; ARM7:       @ %bb.0: @ %entry
+; ARM7-NEXT:    smmul r1, r2, r1
+; ARM7-NEXT:    eor r1, r1, #-2147483648
+; ARM7-NEXT:    add r0, r1, r0
+; ARM7-NEXT:    bx lr
+;
 ; NODSP-LABEL: NOT_SMMLA:
-; NODSP-NOT: smmla
+; NODSP:       @ %bb.0: @ %entry
+; NODSP-NEXT:    smull r1, r2, r2, r1
+; NODSP-NEXT:    eor r1, r2, #-2147483648
+; NODSP-NEXT:    add r0, r1
+; NODSP-NEXT:    bx lr
+entry:
   %conv = sext i32 %b to i64
   %conv1 = sext i32 %c to i64
   %mul = mul nsw i64 %conv1, %conv
diff --git a/llvm/test/CodeGen/ARM/dwarf-frame.ll b/llvm/test/CodeGen/ARM/dwarf-frame.ll
new file mode 100644
index 000000000..a15c9c50f
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/dwarf-frame.ll
@@ -0,0 +1,38 @@
+; RUN: llc -mtriple armv7-unknown -frame-pointer=all -filetype=asm -o - %s | FileCheck %s --check-prefix=CHECK-NO-CFI
+; RUN: llc -mtriple armv7-unknown -frame-pointer=all -filetype=asm -force-dwarf-frame-section -o - %s | FileCheck %s --check-prefix=CHECK-ALWAYS-CFI
+
+declare void @dummy_use(i32*, i32)
+
+define void @test_basic() #0 {
+        %mem = alloca i32, i32 10
+        call void @dummy_use (i32* %mem, i32 10)
+  ret void
+}
+
+; CHECK-NO-CFI-LABEL: test_basic:
+; CHECK-NO-CFI:   .fnstart
+; CHECK-NO-CFI-NOT:   .cfi_sections .debug_frame
+; CHECK-NO-CFI-NOT:   .cfi_startproc
+; CHECK-NO-CFI:       @ %bb.0:
+; CHECK-NO-CFI:       push {r11, lr}
+; CHECK-NO-CFI-NOT:   .cfi_def_cfa_offset 8
+; CHECK-NO-CFI-NOT:   .cfi_offset lr, -4
+; CHECK-NO-CFI-NOT:   .cfi_offset r11, -8
+; CHECK-NO-CFI:       mov r11, sp
+; CHECK-NO-CFI-NOT:   .cfi_def_cfa_register r11
+; CHECK-NO-CFI-NOT:   .cfi_endproc
+; CHECK-NO-CFI:       .fnend
+
+; CHECK-ALWAYS-CFI-LABEL: test_basic:
+; CHECK-ALWAYS-CFI:   .fnstart
+; CHECK-ALWAYS-CFI:   .cfi_sections .debug_frame
+; CHECK-ALWAYS-CFI:   .cfi_startproc
+; CHECK-ALWAYS-CFI:   @ %bb.0:
+; CHECK-ALWAYS-CFI:   push {r11, lr}
+; CHECK-ALWAYS-CFI:   .cfi_def_cfa_offset 8
+; CHECK-ALWAYS-CFI:   .cfi_offset lr, -4
+; CHECK-ALWAYS-CFI:   .cfi_offset r11, -8
+; CHECK-ALWAYS-CFI:   mov r11, sp
+; CHECK-ALWAYS-CFI:   .cfi_def_cfa_register r11
+; CHECK-ALWAYS-CFI:   .cfi_endproc
+; CHECK-ALWAYS-CFI:   .fnend
diff --git a/llvm/test/CodeGen/ARM/dwarf-unwind.ll b/llvm/test/CodeGen/ARM/dwarf-unwind.ll
index 58a116bde..d537a9779 100644
--- a/llvm/test/CodeGen/ARM/dwarf-unwind.ll
+++ b/llvm/test/CodeGen/ARM/dwarf-unwind.ll
@@ -69,7 +69,7 @@ define void @test_nodpr_noalign(i8 %l, i8 %r) {
   ret void
 }
 
-define void @test_frame_pointer_offset() minsize "no-frame-pointer-elim"="true" {
+define void @test_frame_pointer_offset() minsize "frame-pointer"="all" {
 ; CHECK-LABEL: test_frame_pointer_offset:
 ; CHECK: push {r4, r5, r6, r7, lr}
 ; CHECK: .cfi_def_cfa_offset 20
diff --git a/llvm/test/CodeGen/ARM/early-cfi-sections.ll b/llvm/test/CodeGen/ARM/early-cfi-sections.ll
index 5b497fc35..72b87025a 100644
--- a/llvm/test/CodeGen/ARM/early-cfi-sections.ll
+++ b/llvm/test/CodeGen/ARM/early-cfi-sections.ll
@@ -13,7 +13,7 @@ entry:
   ret void, !dbg !10
 }
 
-attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="arm7tdmi" "target-features"="+soft-float,+strict-align,-crypto,-neon" "unsafe-fp-math"="false" "use-soft-float"="true" }
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="arm7tdmi" "target-features"="+soft-float,+strict-align,-crypto,-neon" "unsafe-fp-math"="false" "use-soft-float"="true" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!3, !4, !5, !6}
diff --git a/llvm/test/CodeGen/ARM/fcmp-xo.ll b/llvm/test/CodeGen/ARM/fcmp-xo.ll
index 8ff3b9017..3d5972f06 100644
--- a/llvm/test/CodeGen/ARM/fcmp-xo.ll
+++ b/llvm/test/CodeGen/ARM/fcmp-xo.ll
@@ -5,7 +5,7 @@
 define arm_aapcs_vfpcc float @foo0(float %a0) local_unnamed_addr {
 ; CHECK-LABEL: foo0:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vcmpe.f32 s0, #0
+; CHECK-NEXT:    vcmp.f32 s0, #0
 ; CHECK-NEXT:    vmov.f32 s2, #5.000000e-01
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    vmov.f32 s4, #-5.000000e-01
@@ -24,7 +24,7 @@ define arm_aapcs_vfpcc float @float1(float %a0) local_unnamed_addr {
 ; CHECK-NEXT:    vmov.f32 s2, #1.000000e+00
 ; CHECK-NEXT:    vmov.f32 s4, #5.000000e-01
 ; CHECK-NEXT:    vmov.f32 s6, #-5.000000e-01
-; CHECK-NEXT:    vcmpe.f32 s2, s0
+; CHECK-NEXT:    vcmp.f32 s2, s0
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    vselgt.f32 s0, s6, s4
 ; CHECK-NEXT:    bx lr
@@ -46,7 +46,7 @@ define arm_aapcs_vfpcc float @float128(float %a0) local_unnamed_addr {
 ; VMOVSR-NEXT:    vmov.f32 s4, #5.000000e-01
 ; VMOVSR-NEXT:    vmov s2, r0
 ; VMOVSR-NEXT:    vmov.f32 s6, #-5.000000e-01
-; VMOVSR-NEXT:    vcmpe.f32 s2, s0
+; VMOVSR-NEXT:    vcmp.f32 s2, s0
 ; VMOVSR-NEXT:    vmrs APSR_nzcv, fpscr
 ; VMOVSR-NEXT:    vselgt.f32 s0, s6, s4
 ; VMOVSR-NEXT:    bx lr
@@ -57,7 +57,7 @@ define arm_aapcs_vfpcc float @float128(float %a0) local_unnamed_addr {
 ; NEON-NEXT:    vmov.f32 s2, #5.000000e-01
 ; NEON-NEXT:    vmov d3, r0, r0
 ; NEON-NEXT:    vmov.f32 s4, #-5.000000e-01
-; NEON-NEXT:    vcmpe.f32 s6, s0
+; NEON-NEXT:    vcmp.f32 s6, s0
 ; NEON-NEXT:    vmrs APSR_nzcv, fpscr
 ; NEON-NEXT:    vselgt.f32 s0, s4, s2
 ; NEON-NEXT:    bx lr
@@ -70,7 +70,7 @@ define arm_aapcs_vfpcc double @double1(double %a0) local_unnamed_addr {
 ; CHECK-LABEL: double1:
 ; CHECK:       @ %bb.0:
 ; CHECK-NEXT:    vmov.f64 d18, #1.000000e+00
-; CHECK-NEXT:    vcmpe.f64 d18, d0
+; CHECK-NEXT:    vcmp.f64 d18, d0
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    vmov.f64 d16, #5.000000e-01
 ; CHECK-NEXT:    vmov.f64 d17, #-5.000000e-01
@@ -89,7 +89,7 @@ define arm_aapcs_vfpcc double @double128(double %a0) local_unnamed_addr {
 ; CHECK-NEXT:    movt r0, #16480
 ; CHECK-NEXT:    vmov.f64 d16, #5.000000e-01
 ; CHECK-NEXT:    vmov d18, r1, r0
-; CHECK-NEXT:    vcmpe.f64 d18, d0
+; CHECK-NEXT:    vcmp.f64 d18, d0
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    vmov.f64 d17, #-5.000000e-01
 ; CHECK-NEXT:    vselgt.f64 d0, d17, d16
diff --git a/llvm/test/CodeGen/ARM/fold-stack-adjust.ll b/llvm/test/CodeGen/ARM/fold-stack-adjust.ll
index b8ff910fd..6256138e9 100644
--- a/llvm/test/CodeGen/ARM/fold-stack-adjust.ll
+++ b/llvm/test/CodeGen/ARM/fold-stack-adjust.ll
@@ -42,6 +42,19 @@ define void @check_simple() minsize {
   ret void
 }
 
+define i32 @check_simple_ret() minsize {
+; CHECK-FNSTART-LABEL: check_simple_ret:
+; CHECK: push {r5, r6, r7, lr}
+; CHECK-NOT: sub sp,
+; ...
+; CHECK-NOT: add sp,
+; CHECK: pop {r2, r3, r7, pc}
+
+  %var = alloca i8, i32 8
+  call void @bar(i8* %var)
+  ret i32 0
+}
+
 define void @check_simple_too_big() minsize {
 ; CHECK-FNSTART-LABEL: check_simple_too_big:
 ; CHECK: push {r7, lr}
diff --git a/llvm/test/CodeGen/ARM/fp-intrinsics.ll b/llvm/test/CodeGen/ARM/fp-intrinsics.ll
new file mode 100644
index 000000000..5b81982f6
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/fp-intrinsics.ll
@@ -0,0 +1,1083 @@
+; RUN: llc -mtriple=armv8a-none-eabi %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SP,CHECK-DP,CHECK-SP-V8,CHECK-DP-V8
+; RUN: llc -mtriple=thumbv8m.main-none-eabi %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-NOSP,CHECK-NODP
+; RUN: llc -mtriple=thumbv8m.main-none-eabi %s -o - -mattr=fp-armv8 | FileCheck %s --check-prefixes=CHECK,CHECK-SP,CHECK-DP,CHECK-SP-V8,CHECK-DP-V8
+; RUN: llc -mtriple=thumbv8m.main-none-eabi %s -o - -mattr=fp-armv8sp | FileCheck %s --check-prefixes=CHECK,CHECK-SP,CHECK-NODP,CHECK-SP-V8
+; RUN: llc -mtriple=armv7a-none-eabi %s -o - -mattr=vfp4 | FileCheck %s --check-prefixes=CHECK,CHECK-SP,CHECK-DP,CHECK-SP-NOV8,CHECK-DP-NOV8
+; RUN: llc -mtriple=thumbv7m-none-eabi %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-NOSP,CHECK-NODP
+; RUN: llc -mtriple=thumbv7m-none-eabi %s -o - -mattr=vfp4 | FileCheck %s --check-prefixes=CHECK,CHECK-SP,CHECK-DP,CHECK-SP-NOV8,CHECK-DP-NOV8
+; RUN: llc -mtriple=thumbv7m-none-eabi %s -o - -mattr=vfp4sp | FileCheck %s --check-prefixes=CHECK,CHECK-SP,CHECK-NODP,CHECK-SP-NOV8
+
+; Check that constrained fp intrinsics are correctly lowered. In particular
+; check that the valid combinations of single-precision and double-precision
+; hardware being present or absent work as expected (i.e. we get an instruction
+; when one is available, otherwise a libcall).
+
+; FIXME: We're not generating the right instructions for some of these
+; operations (see further FIXMEs down below).
+
+; Single-precision intrinsics
+
+; CHECK-LABEL: add_f32:
+; CHECK-NOSP: bl __aeabi_fadd
+; CHECK-SP: vadd.f32
+define float @add_f32(float %x, float %y) #0 {
+  %val = call float @llvm.experimental.constrained.fadd.f32(float %x, float %y, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret float %val
+}
+
+; CHECK-LABEL: sub_f32:
+; CHECK-NOSP: bl __aeabi_fsub
+; CHECK-SP: vsub.f32
+define float @sub_f32(float %x, float %y) #0 {
+  %val = call float @llvm.experimental.constrained.fsub.f32(float %x, float %y, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret float %val
+}
+
+; CHECK-LABEL: mul_f32:
+; CHECK-NOSP: bl __aeabi_fmul
+; CHECK-SP: vmul.f32
+define float @mul_f32(float %x, float %y) #0 {
+  %val = call float @llvm.experimental.constrained.fmul.f32(float %x, float %y, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret float %val
+}
+
+; CHECK-LABEL: div_f32:
+; CHECK-NOSP: bl __aeabi_fdiv
+; CHECK-SP: vdiv.f32
+define float @div_f32(float %x, float %y) #0 {
+  %val = call float @llvm.experimental.constrained.fdiv.f32(float %x, float %y, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret float %val
+}
+
+; CHECK-LABEL: frem_f32:
+; CHECK: bl fmodf
+define float @frem_f32(float %x, float %y) #0 {
+  %val = call float @llvm.experimental.constrained.frem.f32(float %x, float %y, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret float %val
+}
+
+; CHECK-LABEL: fma_f32:
+; CHECK-NOSP: bl fmaf
+; CHECK-SP: vfma.f32
+define float @fma_f32(float %x, float %y, float %z) #0 {
+  %val = call float @llvm.experimental.constrained.fma.f32(float %x, float %y, float %z, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret float %val
+}
+
+; CHECK-LABEL: fptosi_f32:
+; CHECK-NOSP: bl __aeabi_f2iz
+; CHECK-SP: vcvt.s32.f32
+define i32 @fptosi_f32(float %x) #0 {
+  %val = call i32 @llvm.experimental.constrained.fptosi.f32(float %x, metadata !"fpexcept.strict") #0
+  ret i32 %val
+}
+
+; CHECK-LABEL: fptosi_f32_twice:
+; CHECK-NOSP: bl __aeabi_f2iz
+; CHECK-NOSP: bl __aeabi_f2iz
+; CHECK-SP: vcvt.s32.f32
+; FIXME-CHECK-SP: vcvt.s32.f32
+define void @fptosi_f32_twice(float %arg, i32* %ptr) #0 {
+entry:
+  %conv = call i32 @llvm.experimental.constrained.fptosi.f32(float %arg, metadata !"fpexcept.strict") #0
+  store i32 %conv, i32* %ptr, align 4
+  %conv1 = call i32 @llvm.experimental.constrained.fptosi.f32(float %arg, metadata !"fpexcept.strict") #0
+  %idx = getelementptr inbounds i32, i32* %ptr, i32 1
+  store i32 %conv1, i32* %idx, align 4
+  ret void
+}
+
+; CHECK-LABEL: fptoui_f32:
+; CHECK-NOSP: bl __aeabi_f2uiz
+; FIXME-CHECK-SP: vcvt.u32.f32
+define i32 @fptoui_f32(float %x) #0 {
+  %val = call i32 @llvm.experimental.constrained.fptoui.f32(float %x, metadata !"fpexcept.strict") #0
+  ret i32 %val
+}
+
+; CHECK-LABEL: fptoui_f32_twice:
+; CHECK-NOSP: bl __aeabi_f2uiz
+; CHECK-NOSP: bl __aeabi_f2uiz
+; FIXME-CHECK-SP: vcvt.u32.f32
+; FIXME-CHECK-SP: vcvt.u32.f32
+define void @fptoui_f32_twice(float %arg, i32* %ptr) #0 {
+entry:
+  %conv = call i32 @llvm.experimental.constrained.fptoui.f32(float %arg, metadata !"fpexcept.strict") #0
+  store i32 %conv, i32* %ptr, align 4
+  %conv1 = call i32 @llvm.experimental.constrained.fptoui.f32(float %arg, metadata !"fpexcept.strict") #0
+  %idx = getelementptr inbounds i32, i32* %ptr, i32 1
+  store i32 %conv1, i32* %idx, align 4
+  ret void
+}
+
+; CHECK-LABEL: sqrt_f32:
+; CHECK-NOSP: bl sqrtf
+; CHECK-SP: vsqrt.f32
+define float @sqrt_f32(float %x) #0 {
+  %val = call float @llvm.experimental.constrained.sqrt.f32(float %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret float %val
+}
+
+; CHECK-LABEL: powi_f32:
+; CHECK: bl __powisf2
+define float @powi_f32(float %x, i32 %y) #0 {
+  %val = call float @llvm.experimental.constrained.powi.f32(float %x, i32 %y, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret float %val
+}
+
+; CHECK-LABEL: sin_f32:
+; CHECK: bl sinf
+define float @sin_f32(float %x) #0 {
+  %val = call float @llvm.experimental.constrained.sin.f32(float %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret float %val
+}
+
+; CHECK-LABEL: cos_f32:
+; CHECK: bl cosf
+define float @cos_f32(float %x) #0 {
+  %val = call float @llvm.experimental.constrained.cos.f32(float %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret float %val
+}
+
+; CHECK-LABEL: pow_f32:
+; CHECK: bl powf
+define float @pow_f32(float %x, float %y) #0 {
+  %val = call float @llvm.experimental.constrained.pow.f32(float %x, float %y, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret float %val
+}
+
+; CHECK-LABEL: log_f32:
+; CHECK: bl logf
+define float @log_f32(float %x) #0 {
+  %val = call float @llvm.experimental.constrained.log.f32(float %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret float %val
+}
+
+; CHECK-LABEL: log10_f32:
+; CHECK: bl log10f
+define float @log10_f32(float %x) #0 {
+  %val = call float @llvm.experimental.constrained.log10.f32(float %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret float %val
+}
+
+; CHECK-LABEL: log2_f32:
+; CHECK: bl log2f
+define float @log2_f32(float %x) #0 {
+  %val = call float @llvm.experimental.constrained.log2.f32(float %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret float %val
+}
+
+; CHECK-LABEL: exp_f32:
+; CHECK: bl expf
+define float @exp_f32(float %x) #0 {
+  %val = call float @llvm.experimental.constrained.exp.f32(float %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret float %val
+}
+
+; CHECK-LABEL: exp2_f32:
+; CHECK: bl exp2f
+define float @exp2_f32(float %x) #0 {
+  %val = call float @llvm.experimental.constrained.exp2.f32(float %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret float %val
+}
+
+; CHECK-LABEL: rint_f32:
+; CHECK-NOSP: bl rintf
+; CHECK-SP-NOV8: bl rintf
+; CHECK-SP-V8: vrintx.f32
+define float @rint_f32(float %x) #0 {
+  %val = call float @llvm.experimental.constrained.rint.f32(float %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret float %val
+}
+
+; CHECK-LABEL: nearbyint_f32:
+; CHECK-NOSP: bl nearbyintf
+; CHECK-SP-NOV8: bl nearbyintf
+; CHECK-SP-V8: vrintr.f32
+define float @nearbyint_f32(float %x) #0 {
+  %val = call float @llvm.experimental.constrained.nearbyint.f32(float %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret float %val
+}
+
+; CHECK-LABEL: lrint_f32:
+; CHECK: bl lrintf
+define i32 @lrint_f32(float %x) #0 {
+  %val = call i32 @llvm.experimental.constrained.lrint.f32(float %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret i32 %val
+}
+
+; CHECK-LABEL: llrint_f32:
+; CHECK: bl llrintf
+define i32 @llrint_f32(float %x) #0 {
+  %val = call i32 @llvm.experimental.constrained.llrint.f32(float %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret i32 %val
+}
+
+; CHECK-LABEL: maxnum_f32:
+; CHECK-NOSP: bl fmaxf
+; CHECK-SP-NOV8: bl fmaxf
+; CHECK-SP-V8: vmaxnm.f32
+define float @maxnum_f32(float %x, float %y) #0 {
+  %val = call float @llvm.experimental.constrained.maxnum.f32(float %x, float %y, metadata !"fpexcept.strict") #0
+  ret float %val
+}
+
+; CHECK-LABEL: minnum_f32:
+; CHECK-NOSP: bl fminf
+; CHECK-SP-NOV8: bl fminf
+; CHECK-SP-V8: vminnm.f32
+define float @minnum_f32(float %x, float %y) #0 {
+  %val = call float @llvm.experimental.constrained.minnum.f32(float %x, float %y, metadata !"fpexcept.strict") #0
+  ret float %val
+}
+
+; CHECK-LABEL: ceil_f32:
+; CHECK-NOSP: bl ceilf
+; CHECK-SP-NOV8: bl ceilf
+; CHECK-SP-V8: vrintp.f32
+define float @ceil_f32(float %x) #0 {
+  %val = call float @llvm.experimental.constrained.ceil.f32(float %x, metadata !"fpexcept.strict") #0
+  ret float %val
+}
+
+; CHECK-LABEL: floor_f32:
+; CHECK-NOSP: bl floorf
+; CHECK-SP-NOV8: bl floorf
+; CHECK-SP-V8: vrintm.f32
+define float @floor_f32(float %x) #0 {
+  %val = call float @llvm.experimental.constrained.floor.f32(float %x, metadata !"fpexcept.strict") #0
+  ret float %val
+}
+
+; CHECK-LABEL: lround_f32:
+; CHECK: bl lroundf
+define i32 @lround_f32(float %x) #0 {
+  %val = call i32 @llvm.experimental.constrained.lround.f32(float %x, metadata !"fpexcept.strict") #0
+  ret i32 %val
+}
+
+; CHECK-LABEL: llround_f32:
+; CHECK: bl llroundf
+define i32 @llround_f32(float %x) #0 {
+  %val = call i32 @llvm.experimental.constrained.llround.f32(float %x, metadata !"fpexcept.strict") #0
+  ret i32 %val
+}
+
+; CHECK-LABEL: round_f32:
+; CHECK-NOSP: bl roundf
+; CHECK-SP-NOV8: bl roundf
+; CHECK-SP-V8: vrinta.f32
+define float @round_f32(float %x) #0 {
+  %val = call float @llvm.experimental.constrained.round.f32(float %x, metadata !"fpexcept.strict") #0
+  ret float %val
+}
+
+; CHECK-LABEL: trunc_f32:
+; CHECK-NOSP: bl truncf
+; CHECK-SP-NOV8: bl truncf
+; CHECK-SP-V8: vrintz.f32
+define float @trunc_f32(float %x) #0 {
+  %val = call float @llvm.experimental.constrained.trunc.f32(float %x, metadata !"fpexcept.strict") #0
+  ret float %val
+}
+
+; CHECK-LABEL: fcmp_olt_f32:
+; CHECK-NOSP: bl __aeabi_fcmplt
+; CHECK-SP: vcmp.f32
+define i32 @fcmp_olt_f32(float %a, float %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f32(float %a, float %b, metadata !"olt", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmp_ole_f32:
+; CHECK-NOSP: bl __aeabi_fcmple
+; CHECK-SP: vcmp.f32
+define i32 @fcmp_ole_f32(float %a, float %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f32(float %a, float %b, metadata !"ole", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmp_ogt_f32:
+; CHECK-NOSP: bl __aeabi_fcmpgt
+; CHECK-SP: vcmp.f32
+define i32 @fcmp_ogt_f32(float %a, float %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f32(float %a, float %b, metadata !"ogt", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmp_oge_f32:
+; CHECK-NOSP: bl __aeabi_fcmpge
+; CHECK-SP: vcmp.f32
+define i32 @fcmp_oge_f32(float %a, float %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f32(float %a, float %b, metadata !"oge", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmp_oeq_f32:
+; CHECK-NOSP: bl __aeabi_fcmpeq
+; CHECK-SP: vcmp.f32
+define i32 @fcmp_oeq_f32(float %a, float %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f32(float %a, float %b, metadata !"oeq", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmp_one_f32:
+; CHECK-NOSP: bl __aeabi_fcmpeq
+; CHECK-NOSP: bl __aeabi_fcmpun
+; CHECK-SP: vcmp.f32
+define i32 @fcmp_one_f32(float %a, float %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f32(float %a, float %b, metadata !"one", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmp_ult_f32:
+; CHECK-NOSP: bl __aeabi_fcmpge
+; CHECK-SP: vcmp.f32
+define i32 @fcmp_ult_f32(float %a, float %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f32(float %a, float %b, metadata !"ult", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmp_ule_f32:
+; CHECK-NOSP: bl __aeabi_fcmpgt
+; CHECK-SP: vcmp.f32
+define i32 @fcmp_ule_f32(float %a, float %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f32(float %a, float %b, metadata !"ule", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmp_ugt_f32:
+; CHECK-NOSP: bl __aeabi_fcmple
+; CHECK-SP: vcmp.f32
+define i32 @fcmp_ugt_f32(float %a, float %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f32(float %a, float %b, metadata !"ugt", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmp_uge_f32:
+; CHECK-NOSP: bl __aeabi_fcmplt
+; CHECK-SP: vcmp.f32
+define i32 @fcmp_uge_f32(float %a, float %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f32(float %a, float %b, metadata !"uge", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmp_ueq_f32:
+; CHECK-NOSP: bl __aeabi_fcmpeq
+; CHECK-NOSP: bl __aeabi_fcmpun
+; CHECK-SP: vcmp.f32
+define i32 @fcmp_ueq_f32(float %a, float %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f32(float %a, float %b, metadata !"ueq", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmp_une_f32:
+; CHECK-NOSP: bl __aeabi_fcmpeq
+; CHECK-SP: vcmp.f32
+define i32 @fcmp_une_f32(float %a, float %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f32(float %a, float %b, metadata !"une", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmps_olt_f32:
+; CHECK-NOSP: bl __aeabi_fcmplt
+; CHECK-SP: vcmpe.f32
+define i32 @fcmps_olt_f32(float %a, float %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmps.f32(float %a, float %b, metadata !"olt", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmps_ole_f32:
+; CHECK-NOSP: bl __aeabi_fcmple
+; CHECK-SP: vcmpe.f32
+define i32 @fcmps_ole_f32(float %a, float %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmps.f32(float %a, float %b, metadata !"ole", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmps_ogt_f32:
+; CHECK-NOSP: bl __aeabi_fcmpgt
+; CHECK-SP: vcmpe.f32
+define i32 @fcmps_ogt_f32(float %a, float %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmps.f32(float %a, float %b, metadata !"ogt", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmps_oge_f32:
+; CHECK-NOSP: bl __aeabi_fcmpge
+; CHECK-SP: vcmpe.f32
+define i32 @fcmps_oge_f32(float %a, float %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmps.f32(float %a, float %b, metadata !"oge", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmps_oeq_f32:
+; CHECK-NOSP: bl __aeabi_fcmpeq
+; CHECK-SP: vcmpe.f32
+define i32 @fcmps_oeq_f32(float %a, float %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmps.f32(float %a, float %b, metadata !"oeq", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmps_one_f32:
+; CHECK-NOSP: bl __aeabi_fcmpeq
+; CHECK-NOSP: bl __aeabi_fcmpun
+; CHECK-SP: vcmpe.f32
+define i32 @fcmps_one_f32(float %a, float %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmps.f32(float %a, float %b, metadata !"one", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmps_ult_f32:
+; CHECK-NOSP: bl __aeabi_fcmpge
+; CHECK-SP: vcmpe.f32
+define i32 @fcmps_ult_f32(float %a, float %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmps.f32(float %a, float %b, metadata !"ult", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmps_ule_f32:
+; CHECK-NOSP: bl __aeabi_fcmpgt
+; CHECK-SP: vcmpe.f32
+define i32 @fcmps_ule_f32(float %a, float %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmps.f32(float %a, float %b, metadata !"ule", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmps_ugt_f32:
+; CHECK-NOSP: bl __aeabi_fcmple
+; CHECK-SP: vcmpe.f32
+define i32 @fcmps_ugt_f32(float %a, float %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmps.f32(float %a, float %b, metadata !"ugt", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmps_uge_f32:
+; CHECK-NOSP: bl __aeabi_fcmplt
+; CHECK-SP: vcmpe.f32
+define i32 @fcmps_uge_f32(float %a, float %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmps.f32(float %a, float %b, metadata !"uge", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmps_ueq_f32:
+; CHECK-NOSP: bl __aeabi_fcmpeq
+; CHECK-NOSP: bl __aeabi_fcmpun
+; CHECK-SP: vcmpe.f32
+define i32 @fcmps_ueq_f32(float %a, float %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmps.f32(float %a, float %b, metadata !"ueq", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmps_une_f32:
+; CHECK-NOSP: bl __aeabi_fcmpeq
+; CHECK-SP: vcmpe.f32
+define i32 @fcmps_une_f32(float %a, float %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmps.f32(float %a, float %b, metadata !"une", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+
+; Double-precision intrinsics
+
+; CHECK-LABEL: add_f64:
+; CHECK-NODP: bl __aeabi_dadd
+; CHECK-DP: vadd.f64
+define double @add_f64(double %x, double %y) #0 {
+  %val = call double @llvm.experimental.constrained.fadd.f64(double %x, double %y, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret double %val
+}
+
+; CHECK-LABEL: sub_f64:
+; CHECK-NODP: bl __aeabi_dsub
+; CHECK-DP: vsub.f64
+define double @sub_f64(double %x, double %y) #0 {
+  %val = call double @llvm.experimental.constrained.fsub.f64(double %x, double %y, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret double %val
+}
+
+; CHECK-LABEL: mul_f64:
+; CHECK-NODP: bl __aeabi_dmul
+; CHECK-DP: vmul.f64
+define double @mul_f64(double %x, double %y) #0 {
+  %val = call double @llvm.experimental.constrained.fmul.f64(double %x, double %y, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret double %val
+}
+
+; CHECK-LABEL: div_f64:
+; CHECK-NODP: bl __aeabi_ddiv
+; CHECK-DP: vdiv.f64
+define double @div_f64(double %x, double %y) #0 {
+  %val = call double @llvm.experimental.constrained.fdiv.f64(double %x, double %y, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret double %val
+}
+
+; CHECK-LABEL: frem_f64:
+; CHECK: bl fmod
+define double @frem_f64(double %x, double %y) #0 {
+  %val = call double @llvm.experimental.constrained.frem.f64(double %x, double %y, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret double %val
+}
+
+; CHECK-LABEL: fma_f64:
+; CHECK-NODP: bl fma
+; CHECK-DP: vfma.f64
+define double @fma_f64(double %x, double %y, double %z) #0 {
+  %val = call double @llvm.experimental.constrained.fma.f64(double %x, double %y, double %z, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret double %val
+}
+
+; CHECK-LABEL: fptosi_f64:
+; CHECK-NODP: bl __aeabi_d2iz
+; CHECK-DP: vcvt.s32.f64
+define i32 @fptosi_f64(double %x) #0 {
+  %val = call i32 @llvm.experimental.constrained.fptosi.f64(double %x, metadata !"fpexcept.strict") #0
+  ret i32 %val
+}
+
+; CHECK-LABEL: fptoui_f64:
+; CHECK-NODP: bl __aeabi_d2uiz
+; FIXME-CHECK-DP: vcvt.u32.f64
+define i32 @fptoui_f64(double %x) #0 {
+  %val = call i32 @llvm.experimental.constrained.fptoui.f64(double %x, metadata !"fpexcept.strict") #0
+  ret i32 %val
+}
+
+; CHECK-LABEL: sqrt_f64:
+; CHECK-NODP: bl sqrt
+; CHECK-DP: vsqrt.f64
+define double @sqrt_f64(double %x) #0 {
+  %val = call double @llvm.experimental.constrained.sqrt.f64(double %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret double %val
+}
+
+; CHECK-LABEL: powi_f64:
+; CHECK: bl __powidf2
+define double @powi_f64(double %x, i32 %y) #0 {
+  %val = call double @llvm.experimental.constrained.powi.f64(double %x, i32 %y, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret double %val
+}
+
+; CHECK-LABEL: sin_f64:
+; CHECK: bl sin
+define double @sin_f64(double %x) #0 {
+  %val = call double @llvm.experimental.constrained.sin.f64(double %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret double %val
+}
+
+; CHECK-LABEL: cos_f64:
+; CHECK: bl cos
+define double @cos_f64(double %x) #0 {
+  %val = call double @llvm.experimental.constrained.cos.f64(double %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret double %val
+}
+
+; CHECK-LABEL: pow_f64:
+; CHECK: bl pow
+define double @pow_f64(double %x, double %y) #0 {
+  %val = call double @llvm.experimental.constrained.pow.f64(double %x, double %y, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret double %val
+}
+
+; CHECK-LABEL: log_f64:
+; CHECK: bl log
+define double @log_f64(double %x) #0 {
+  %val = call double @llvm.experimental.constrained.log.f64(double %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret double %val
+}
+
+; CHECK-LABEL: log10_f64:
+; CHECK: bl log10
+define double @log10_f64(double %x) #0 {
+  %val = call double @llvm.experimental.constrained.log10.f64(double %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret double %val
+}
+
+; CHECK-LABEL: log2_f64:
+; CHECK: bl log2
+define double @log2_f64(double %x) #0 {
+  %val = call double @llvm.experimental.constrained.log2.f64(double %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret double %val
+}
+
+; CHECK-LABEL: exp_f64:
+; CHECK: bl exp
+define double @exp_f64(double %x) #0 {
+  %val = call double @llvm.experimental.constrained.exp.f64(double %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret double %val
+}
+
+; CHECK-LABEL: exp2_f64:
+; CHECK: bl exp2
+define double @exp2_f64(double %x) #0 {
+  %val = call double @llvm.experimental.constrained.exp2.f64(double %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret double %val
+}
+
+; CHECK-LABEL: rint_f64:
+; CHECK-NODP: bl rint
+; CHECK-DP-NOV8: bl rint
+; CHECK-DP-V8: vrintx.f64
+define double @rint_f64(double %x) #0 {
+  %val = call double @llvm.experimental.constrained.rint.f64(double %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret double %val
+}
+
+; CHECK-LABEL: nearbyint_f64:
+; CHECK-NODP: bl nearbyint
+; CHECK-DP-NOV8: bl nearbyint
+; CHECK-DP-V8: vrintr.f64
+define double @nearbyint_f64(double %x) #0 {
+  %val = call double @llvm.experimental.constrained.nearbyint.f64(double %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret double %val
+}
+
+; CHECK-LABEL: lrint_f64:
+; CHECK: bl lrint
+define i32 @lrint_f64(double %x) #0 {
+  %val = call i32 @llvm.experimental.constrained.lrint.f64(double %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret i32 %val
+}
+
+; CHECK-LABEL: llrint_f64:
+; CHECK: bl llrint
+define i32 @llrint_f64(double %x) #0 {
+  %val = call i32 @llvm.experimental.constrained.llrint.f64(double %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret i32 %val
+}
+
+; CHECK-LABEL: maxnum_f64:
+; CHECK-NODP: bl fmax
+; CHECK-DP-NOV8: bl fmax
+; CHECK-DP-V8: vmaxnm.f64
+define double @maxnum_f64(double %x, double %y) #0 {
+  %val = call double @llvm.experimental.constrained.maxnum.f64(double %x, double %y, metadata !"fpexcept.strict") #0
+  ret double %val
+}
+
+; CHECK-LABEL: minnum_f64:
+; CHECK-NODP: bl fmin
+; CHECK-DP-NOV8: bl fmin
+; CHECK-DP-V8: vminnm.f64
+define double @minnum_f64(double %x, double %y) #0 {
+  %val = call double @llvm.experimental.constrained.minnum.f64(double %x, double %y, metadata !"fpexcept.strict") #0
+  ret double %val
+}
+
+; CHECK-LABEL: ceil_f64:
+; CHECK-NODP: bl ceil
+; CHECK-DP-NOV8: bl ceil
+; CHECK-DP-V8: vrintp.f64
+define double @ceil_f64(double %x) #0 {
+  %val = call double @llvm.experimental.constrained.ceil.f64(double %x, metadata !"fpexcept.strict") #0
+  ret double %val
+}
+
+; CHECK-LABEL: floor_f64:
+; CHECK-NODP: bl floor
+; CHECK-DP-NOV8: bl floor
+; CHECK-DP-V8: vrintm.f64
+define double @floor_f64(double %x) #0 {
+  %val = call double @llvm.experimental.constrained.floor.f64(double %x, metadata !"fpexcept.strict") #0
+  ret double %val
+}
+
+; CHECK-LABEL: lround_f64:
+; CHECK: bl lround
+define i32 @lround_f64(double %x) #0 {
+  %val = call i32 @llvm.experimental.constrained.lround.f64(double %x, metadata !"fpexcept.strict") #0
+  ret i32 %val
+}
+
+; CHECK-LABEL: llround_f64:
+; CHECK: bl llround
+define i32 @llround_f64(double %x) #0 {
+  %val = call i32 @llvm.experimental.constrained.llround.f64(double %x, metadata !"fpexcept.strict") #0
+  ret i32 %val
+}
+
+; CHECK-LABEL: round_f64:
+; CHECK-NODP: bl round
+; CHECK-DP-NOV8: bl round
+; CHECK-DP-V8: vrinta.f64
+define double @round_f64(double %x) #0 {
+  %val = call double @llvm.experimental.constrained.round.f64(double %x, metadata !"fpexcept.strict") #0
+  ret double %val
+}
+
+; CHECK-LABEL: trunc_f64:
+; CHECK-NODP: bl trunc
+; CHECK-DP-NOV8: bl trunc
+; CHECK-DP-V8: vrintz.f64
+define double @trunc_f64(double %x) #0 {
+  %val = call double @llvm.experimental.constrained.trunc.f64(double %x, metadata !"fpexcept.strict") #0
+  ret double %val
+}
+
+; CHECK-LABEL: fcmp_olt_f64:
+; CHECK-NODP: bl __aeabi_dcmplt
+; CHECK-DP: vcmp.f64
+define i32 @fcmp_olt_f64(double %a, double %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f64(double %a, double %b, metadata !"olt", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmp_ole_f64:
+; CHECK-NODP: bl __aeabi_dcmple
+; CHECK-DP: vcmp.f64
+define i32 @fcmp_ole_f64(double %a, double %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f64(double %a, double %b, metadata !"ole", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmp_ogt_f64:
+; CHECK-NODP: bl __aeabi_dcmpgt
+; CHECK-DP: vcmp.f64
+define i32 @fcmp_ogt_f64(double %a, double %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f64(double %a, double %b, metadata !"ogt", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmp_oge_f64:
+; CHECK-NODP: bl __aeabi_dcmpge
+; CHECK-DP: vcmp.f64
+define i32 @fcmp_oge_f64(double %a, double %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f64(double %a, double %b, metadata !"oge", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmp_oeq_f64:
+; CHECK-NODP: bl __aeabi_dcmpeq
+; CHECK-DP: vcmp.f64
+define i32 @fcmp_oeq_f64(double %a, double %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f64(double %a, double %b, metadata !"oeq", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmp_one_f64:
+; CHECK-NODP-DAG: bl __aeabi_dcmpeq
+; CHECK-NODP-DAG: bl __aeabi_dcmpun
+; CHECK-DP: vcmp.f64
+define i32 @fcmp_one_f64(double %a, double %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f64(double %a, double %b, metadata !"one", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmp_ult_f64:
+; CHECK-NODP: bl __aeabi_dcmpge
+; CHECK-DP: vcmp.f64
+define i32 @fcmp_ult_f64(double %a, double %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f64(double %a, double %b, metadata !"ult", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmp_ule_f64:
+; CHECK-NODP: bl __aeabi_dcmpgt
+; CHECK-DP: vcmp.f64
+define i32 @fcmp_ule_f64(double %a, double %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f64(double %a, double %b, metadata !"ule", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmp_ugt_f64:
+; CHECK-NODP: bl __aeabi_dcmple
+; CHECK-DP: vcmp.f64
+define i32 @fcmp_ugt_f64(double %a, double %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f64(double %a, double %b, metadata !"ugt", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmp_uge_f64:
+; CHECK-NODP: bl __aeabi_dcmplt
+; CHECK-DP: vcmp.f64
+define i32 @fcmp_uge_f64(double %a, double %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f64(double %a, double %b, metadata !"uge", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmp_ueq_f64:
+; CHECK-NODP-DAG: bl __aeabi_dcmpeq
+; CHECK-NODP-DAG: bl __aeabi_dcmpun
+; CHECK-DP: vcmp.f64
+define i32 @fcmp_ueq_f64(double %a, double %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f64(double %a, double %b, metadata !"ueq", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmp_une_f64:
+; CHECK-NODP: bl __aeabi_dcmpeq
+; CHECK-DP: vcmp.f64
+define i32 @fcmp_une_f64(double %a, double %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f64(double %a, double %b, metadata !"une", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmps_olt_f64:
+; CHECK-NODP: bl __aeabi_dcmplt
+; CHECK-DP: vcmpe.f64
+define i32 @fcmps_olt_f64(double %a, double %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmps.f64(double %a, double %b, metadata !"olt", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmps_ole_f64:
+; CHECK-NODP: bl __aeabi_dcmple
+; CHECK-DP: vcmpe.f64
+define i32 @fcmps_ole_f64(double %a, double %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmps.f64(double %a, double %b, metadata !"ole", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmps_ogt_f64:
+; CHECK-NODP: bl __aeabi_dcmpgt
+; CHECK-DP: vcmpe.f64
+define i32 @fcmps_ogt_f64(double %a, double %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmps.f64(double %a, double %b, metadata !"ogt", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmps_oge_f64:
+; CHECK-NODP: bl __aeabi_dcmpge
+; CHECK-DP: vcmpe.f64
+define i32 @fcmps_oge_f64(double %a, double %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmps.f64(double %a, double %b, metadata !"oge", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmps_oeq_f64:
+; CHECK-NODP: bl __aeabi_dcmpeq
+; CHECK-DP: vcmpe.f64
+define i32 @fcmps_oeq_f64(double %a, double %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmps.f64(double %a, double %b, metadata !"oeq", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmps_one_f64:
+; CHECK-NODP-DAG: bl __aeabi_dcmpeq
+; CHECK-NODP-DAG: bl __aeabi_dcmpun
+; CHECK-DP: vcmpe.f64
+define i32 @fcmps_one_f64(double %a, double %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmps.f64(double %a, double %b, metadata !"one", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmps_ult_f64:
+; CHECK-NODP: bl __aeabi_dcmpge
+; CHECK-DP: vcmpe.f64
+define i32 @fcmps_ult_f64(double %a, double %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmps.f64(double %a, double %b, metadata !"ult", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmps_ule_f64:
+; CHECK-NODP: bl __aeabi_dcmpgt
+; CHECK-DP: vcmpe.f64
+define i32 @fcmps_ule_f64(double %a, double %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmps.f64(double %a, double %b, metadata !"ule", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmps_ugt_f64:
+; CHECK-NODP: bl __aeabi_dcmple
+; CHECK-DP: vcmpe.f64
+define i32 @fcmps_ugt_f64(double %a, double %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmps.f64(double %a, double %b, metadata !"ugt", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmps_uge_f64:
+; CHECK-NODP: bl __aeabi_dcmplt
+; CHECK-DP: vcmpe.f64
+define i32 @fcmps_uge_f64(double %a, double %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmps.f64(double %a, double %b, metadata !"uge", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmps_ueq_f64:
+; CHECK-NODP-DAG: bl __aeabi_dcmpeq
+; CHECK-NODP-DAG: bl __aeabi_dcmpun
+; CHECK-DP: vcmpe.f64
+define i32 @fcmps_ueq_f64(double %a, double %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmps.f64(double %a, double %b, metadata !"ueq", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: fcmps_une_f64:
+; CHECK-NODP: bl __aeabi_dcmpeq
+; CHECK-DP: vcmpe.f64
+define i32 @fcmps_une_f64(double %a, double %b) #0 {
+  %cmp = call i1 @llvm.experimental.constrained.fcmps.f64(double %a, double %b, metadata !"une", metadata !"fpexcept.strict") #0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+
+; Single/Double conversion intrinsics
+
+; CHECK-LABEL: fptrunc_f32:
+; CHECK-NODP: bl __aeabi_d2f
+; CHECK-DP: vcvt.f32.f64
+define float @fptrunc_f32(double %x) #0 {
+  %val = call float @llvm.experimental.constrained.fptrunc.f32.f64(double %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret float %val
+}
+
+; CHECK-LABEL: fpext_f32:
+; CHECK-NODP: bl __aeabi_f2d
+; CHECK-DP: vcvt.f64.f32
+define double @fpext_f32(float %x) #0 {
+  %val = call double @llvm.experimental.constrained.fpext.f64.f32(float %x, metadata !"fpexcept.strict") #0
+  ret double %val
+}
+
+; CHECK-LABEL: fpext_f32_twice:
+; CHECK-NODP: bl __aeabi_f2d
+; CHECK-NODP: bl __aeabi_f2d
+; CHECK-DP: vcvt.f64.f32
+; FIXME-CHECK-DP: vcvt.f64.f32
+define void @fpext_f32_twice(float %arg, double* %ptr) #0 {
+entry:
+  %conv1 = call double @llvm.experimental.constrained.fpext.f64.f32(float %arg, metadata !"fpexcept.strict") #0
+  store double %conv1, double* %ptr, align 8
+  %conv2 = call double @llvm.experimental.constrained.fpext.f64.f32(float %arg, metadata !"fpexcept.strict") #0
+  %idx = getelementptr inbounds double, double* %ptr, i32 1
+  store double %conv2, double* %idx, align 8
+  ret void
+}
+
+; CHECK-LABEL: sitofp_f32_i32:
+; CHECK-NOSP: bl __aeabi_i2f
+; FIXME-CHECK-SP: vcvt.f32.s32
+define float @sitofp_f32_i32(i32 %x) #0 {
+  %val = call float @llvm.experimental.constrained.sitofp.f32.i32(i32 %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret float %val
+}
+
+; CHECK-LABEL: sitofp_f64_i32:
+; FIXME-CHECK-NODP: bl __aeabi_i2d
+; FIXME-CHECK-DP: vcvt.f64.s32
+define double @sitofp_f64_i32(i32 %x) #0 {
+  %val = call double @llvm.experimental.constrained.sitofp.f64.i32(i32 %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret double %val
+}
+
+
+attributes #0 = { strictfp }
+
+declare float @llvm.experimental.constrained.fadd.f32(float, float, metadata, metadata)
+declare float @llvm.experimental.constrained.fsub.f32(float, float, metadata, metadata)
+declare float @llvm.experimental.constrained.fmul.f32(float, float, metadata, metadata)
+declare float @llvm.experimental.constrained.fdiv.f32(float, float, metadata, metadata)
+declare float @llvm.experimental.constrained.frem.f32(float, float, metadata, metadata)
+declare float @llvm.experimental.constrained.fma.f32(float, float, float, metadata, metadata)
+declare i32 @llvm.experimental.constrained.fptosi.f32(float, metadata)
+declare i32 @llvm.experimental.constrained.fptoui.f32(float, metadata)
+declare float @llvm.experimental.constrained.sqrt.f32(float, metadata, metadata)
+declare float @llvm.experimental.constrained.powi.f32(float, i32, metadata, metadata)
+declare float @llvm.experimental.constrained.sin.f32(float, metadata, metadata)
+declare float @llvm.experimental.constrained.cos.f32(float, metadata, metadata)
+declare float @llvm.experimental.constrained.pow.f32(float, float, metadata, metadata)
+declare float @llvm.experimental.constrained.log.f32(float, metadata, metadata)
+declare float @llvm.experimental.constrained.log10.f32(float, metadata, metadata)
+declare float @llvm.experimental.constrained.log2.f32(float, metadata, metadata)
+declare float @llvm.experimental.constrained.exp.f32(float, metadata, metadata)
+declare float @llvm.experimental.constrained.exp2.f32(float, metadata, metadata)
+declare float @llvm.experimental.constrained.rint.f32(float, metadata, metadata)
+declare float @llvm.experimental.constrained.nearbyint.f32(float, metadata, metadata)
+declare i32 @llvm.experimental.constrained.lrint.f32(float, metadata, metadata)
+declare i32 @llvm.experimental.constrained.llrint.f32(float, metadata, metadata)
+declare float @llvm.experimental.constrained.maxnum.f32(float, float, metadata)
+declare float @llvm.experimental.constrained.minnum.f32(float, float, metadata)
+declare float @llvm.experimental.constrained.ceil.f32(float, metadata)
+declare float @llvm.experimental.constrained.floor.f32(float, metadata)
+declare i32 @llvm.experimental.constrained.lround.f32(float, metadata)
+declare i32 @llvm.experimental.constrained.llround.f32(float, metadata)
+declare float @llvm.experimental.constrained.round.f32(float, metadata)
+declare float @llvm.experimental.constrained.trunc.f32(float, metadata)
+declare i1 @llvm.experimental.constrained.fcmps.f32(float, float, metadata, metadata)
+declare i1 @llvm.experimental.constrained.fcmp.f32(float, float, metadata, metadata)
+
+declare double @llvm.experimental.constrained.fadd.f64(double, double, metadata, metadata)
+declare double @llvm.experimental.constrained.fsub.f64(double, double, metadata, metadata)
+declare double @llvm.experimental.constrained.fmul.f64(double, double, metadata, metadata)
+declare double @llvm.experimental.constrained.fdiv.f64(double, double, metadata, metadata)
+declare double @llvm.experimental.constrained.frem.f64(double, double, metadata, metadata)
+declare double @llvm.experimental.constrained.fma.f64(double, double, double, metadata, metadata)
+declare i32 @llvm.experimental.constrained.fptosi.f64(double, metadata)
+declare i32 @llvm.experimental.constrained.fptoui.f64(double, metadata)
+declare double @llvm.experimental.constrained.sqrt.f64(double, metadata, metadata)
+declare double @llvm.experimental.constrained.powi.f64(double, i32, metadata, metadata)
+declare double @llvm.experimental.constrained.sin.f64(double, metadata, metadata)
+declare double @llvm.experimental.constrained.cos.f64(double, metadata, metadata)
+declare double @llvm.experimental.constrained.pow.f64(double, double, metadata, metadata)
+declare double @llvm.experimental.constrained.log.f64(double, metadata, metadata)
+declare double @llvm.experimental.constrained.log10.f64(double, metadata, metadata)
+declare double @llvm.experimental.constrained.log2.f64(double, metadata, metadata)
+declare double @llvm.experimental.constrained.exp.f64(double, metadata, metadata)
+declare double @llvm.experimental.constrained.exp2.f64(double, metadata, metadata)
+declare double @llvm.experimental.constrained.rint.f64(double, metadata, metadata)
+declare double @llvm.experimental.constrained.nearbyint.f64(double, metadata, metadata)
+declare i32 @llvm.experimental.constrained.lrint.f64(double, metadata, metadata)
+declare i32 @llvm.experimental.constrained.llrint.f64(double, metadata, metadata)
+declare double @llvm.experimental.constrained.maxnum.f64(double, double, metadata)
+declare double @llvm.experimental.constrained.minnum.f64(double, double, metadata)
+declare double @llvm.experimental.constrained.ceil.f64(double, metadata)
+declare double @llvm.experimental.constrained.floor.f64(double, metadata)
+declare i32 @llvm.experimental.constrained.lround.f64(double, metadata)
+declare i32 @llvm.experimental.constrained.llround.f64(double, metadata)
+declare double @llvm.experimental.constrained.round.f64(double, metadata)
+declare double @llvm.experimental.constrained.trunc.f64(double, metadata)
+declare i1 @llvm.experimental.constrained.fcmps.f64(double, double, metadata, metadata)
+declare i1 @llvm.experimental.constrained.fcmp.f64(double, double, metadata, metadata)
+
+declare float @llvm.experimental.constrained.fptrunc.f32.f64(double, metadata, metadata)
+declare double @llvm.experimental.constrained.fpext.f64.f32(float, metadata)
+declare float @llvm.experimental.constrained.sitofp.f32.i32(i32, metadata, metadata)
+declare double @llvm.experimental.constrained.sitofp.f64.i32(i32, metadata, metadata)
diff --git a/llvm/test/CodeGen/ARM/fp16-fullfp16.ll b/llvm/test/CodeGen/ARM/fp16-fullfp16.ll
index a30b62acb..a0d41a2f5 100644
--- a/llvm/test/CodeGen/ARM/fp16-fullfp16.ll
+++ b/llvm/test/CodeGen/ARM/fp16-fullfp16.ll
@@ -571,7 +571,7 @@ define void @test_fmuladd(half* %p, half* %q, half* %r) {
 ; CHECK:         vldr.16 s0, [r1]
 ; CHECK-NEXT:    vldr.16 s2, [r0]
 ; CHECK-NEXT:    vldr.16 s4, [r2]
-; CHECK-NEXT:    vmla.f16 s4, s2, s0
+; CHECK-NEXT:    vfma.f16 s4, s2, s0
 ; CHECK-NEXT:    vstr.16 s4, [r0]
 ; CHECK-NEXT:    bx lr
   %a = load half, half* %p, align 2
diff --git a/llvm/test/CodeGen/ARM/fp16-fusedMAC.ll b/llvm/test/CodeGen/ARM/fp16-fusedMAC.ll
new file mode 100644
index 000000000..03909b800
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/fp16-fusedMAC.ll
@@ -0,0 +1,429 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=thumbv8.1-m-none-eabi -mattr=+fullfp16 -fp-contract=fast | FileCheck %s
+; RUN: llc < %s -mtriple=thumbv8.1-m-none-eabi -mattr=+fullfp16,+slowfpvfmx -fp-contract=fast | FileCheck %s -check-prefix=DONT-FUSE
+
+; Check generated fp16 fused MAC and MLS.
+
+define arm_aapcs_vfpcc void @fusedMACTest2(half *%a1, half *%a2, half *%a3) {
+; CHECK-LABEL: fusedMACTest2:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr.16 s0, [r1]
+; CHECK-NEXT:    vldr.16 s2, [r0]
+; CHECK-NEXT:    vldr.16 s4, [r2]
+; CHECK-NEXT:    vfma.f16 s4, s2, s0
+; CHECK-NEXT:    vstr.16 s4, [r0]
+; CHECK-NEXT:    bx lr
+;
+; DONT-FUSE-LABEL: fusedMACTest2:
+; DONT-FUSE:       @ %bb.0:
+; DONT-FUSE-NEXT:    vldr.16 s0, [r1]
+; DONT-FUSE-NEXT:    vldr.16 s2, [r0]
+; DONT-FUSE-NEXT:    vmul.f16 s0, s2, s0
+; DONT-FUSE-NEXT:    vldr.16 s2, [r2]
+; DONT-FUSE-NEXT:    vadd.f16 s0, s0, s2
+; DONT-FUSE-NEXT:    vstr.16 s0, [r0]
+; DONT-FUSE-NEXT:    bx lr
+
+  %f1 = load half, half *%a1, align 2
+  %f2 = load half, half *%a2, align 2
+  %f3 = load half, half *%a3, align 2
+  %1 = fmul half %f1, %f2
+  %2 = fadd half %1, %f3
+  store half %2, half *%a1, align 2
+  ret void
+}
+
+define arm_aapcs_vfpcc void @fusedMACTest4(half *%a1, half *%a2, half *%a3) {
+; CHECK-LABEL: fusedMACTest4:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr.16 s0, [r2]
+; CHECK-NEXT:    vldr.16 s2, [r1]
+; CHECK-NEXT:    vldr.16 s4, [r0]
+; CHECK-NEXT:    vfms.f16 s4, s2, s0
+; CHECK-NEXT:    vstr.16 s4, [r0]
+; CHECK-NEXT:    bx lr
+;
+; DONT-FUSE-LABEL: fusedMACTest4:
+; DONT-FUSE:       @ %bb.0:
+; DONT-FUSE-NEXT:    vldr.16 s0, [r2]
+; DONT-FUSE-NEXT:    vldr.16 s2, [r1]
+; DONT-FUSE-NEXT:    vmul.f16 s0, s2, s0
+; DONT-FUSE-NEXT:    vldr.16 s2, [r0]
+; DONT-FUSE-NEXT:    vsub.f16 s0, s2, s0
+; DONT-FUSE-NEXT:    vstr.16 s0, [r0]
+; DONT-FUSE-NEXT:    bx lr
+
+  %f1 = load half, half *%a1, align 2
+  %f2 = load half, half *%a2, align 2
+  %f3 = load half, half *%a3, align 2
+  %1 = fmul half %f2, %f3
+  %2 = fsub half %f1, %1
+  store half %2, half *%a1, align 2
+  ret void
+}
+
+define arm_aapcs_vfpcc void @fusedMACTest6(half *%a1, half *%a2, half *%a3) {
+; CHECK-LABEL: fusedMACTest6:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr.16 s0, [r1]
+; CHECK-NEXT:    vldr.16 s2, [r0]
+; CHECK-NEXT:    vldr.16 s4, [r2]
+; CHECK-NEXT:    vfnma.f16 s4, s2, s0
+; CHECK-NEXT:    vstr.16 s4, [r0]
+; CHECK-NEXT:    bx lr
+;
+; DONT-FUSE-LABEL: fusedMACTest6:
+; DONT-FUSE:       @ %bb.0:
+; DONT-FUSE-NEXT:    vldr.16 s0, [r1]
+; DONT-FUSE-NEXT:    vldr.16 s2, [r0]
+; DONT-FUSE-NEXT:    vnmul.f16 s0, s2, s0
+; DONT-FUSE-NEXT:    vldr.16 s2, [r2]
+; DONT-FUSE-NEXT:    vsub.f16 s0, s0, s2
+; DONT-FUSE-NEXT:    vstr.16 s0, [r0]
+; DONT-FUSE-NEXT:    bx lr
+
+  %f1 = load half, half *%a1, align 2
+  %f2 = load half, half *%a2, align 2
+  %f3 = load half, half *%a3, align 2
+  %1 = fmul half %f1, %f2
+  %2 = fsub half -0.0, %1
+  %3 = fsub half %2, %f3
+  store half %3, half *%a1, align 2
+  ret void
+}
+
+define arm_aapcs_vfpcc void @fusedMACTest8(half *%a1, half *%a2, half *%a3) {
+; CHECK-LABEL: fusedMACTest8:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr.16 s0, [r1]
+; CHECK-NEXT:    vldr.16 s2, [r0]
+; CHECK-NEXT:    vldr.16 s4, [r2]
+; CHECK-NEXT:    vfnms.f16 s4, s2, s0
+; CHECK-NEXT:    vstr.16 s4, [r0]
+; CHECK-NEXT:    bx lr
+;
+; DONT-FUSE-LABEL: fusedMACTest8:
+; DONT-FUSE:       @ %bb.0:
+; DONT-FUSE-NEXT:    vldr.16 s0, [r1]
+; DONT-FUSE-NEXT:    vldr.16 s2, [r0]
+; DONT-FUSE-NEXT:    vmul.f16 s0, s2, s0
+; DONT-FUSE-NEXT:    vldr.16 s2, [r2]
+; DONT-FUSE-NEXT:    vsub.f16 s0, s0, s2
+; DONT-FUSE-NEXT:    vstr.16 s0, [r0]
+; DONT-FUSE-NEXT:    bx lr
+
+  %f1 = load half, half *%a1, align 2
+  %f2 = load half, half *%a2, align 2
+  %f3 = load half, half *%a3, align 2
+  %1 = fmul half %f1, %f2
+  %2 = fsub half %1, %f3
+  store half %2, half *%a1, align 2
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_fma_f16(half *%aa, half *%bb, half *%cc) nounwind readnone ssp {
+; CHECK-LABEL: test_fma_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldr.16 s0, [r1]
+; CHECK-NEXT:    vldr.16 s2, [r0]
+; CHECK-NEXT:    vldr.16 s4, [r2]
+; CHECK-NEXT:    vfma.f16 s4, s2, s0
+; CHECK-NEXT:    vstr.16 s4, [r0]
+; CHECK-NEXT:    bx lr
+;
+; DONT-FUSE-LABEL: test_fma_f16:
+; DONT-FUSE:       @ %bb.0: @ %entry
+; DONT-FUSE-NEXT:    vldr.16 s0, [r1]
+; DONT-FUSE-NEXT:    vldr.16 s2, [r0]
+; DONT-FUSE-NEXT:    vldr.16 s4, [r2]
+; DONT-FUSE-NEXT:    vfma.f16 s4, s2, s0
+; DONT-FUSE-NEXT:    vstr.16 s4, [r0]
+; DONT-FUSE-NEXT:    bx lr
+entry:
+  %a = load half, half *%aa, align 2
+  %b = load half, half *%bb, align 2
+  %c = load half, half *%cc, align 2
+  %tmp1 = tail call half @llvm.fma.f16(half %a, half %b, half %c) nounwind readnone
+  store half %tmp1, half *%aa, align 2
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_fnms_f16(half *%aa, half *%bb, half *%cc) nounwind readnone ssp {
+; CHECK-LABEL: test_fnms_f16:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr.16 s0, [r1]
+; CHECK-NEXT:    vldr.16 s2, [r0]
+; CHECK-NEXT:    vldr.16 s4, [r2]
+; CHECK-NEXT:    vfma.f16 s4, s2, s0
+; CHECK-NEXT:    vstr.16 s4, [r0]
+; CHECK-NEXT:    bx lr
+;
+; DONT-FUSE-LABEL: test_fnms_f16:
+; DONT-FUSE:       @ %bb.0:
+; DONT-FUSE-NEXT:    vldr.16 s0, [r1]
+; DONT-FUSE-NEXT:    vldr.16 s2, [r0]
+; DONT-FUSE-NEXT:    vldr.16 s4, [r2]
+; DONT-FUSE-NEXT:    vfma.f16 s4, s2, s0
+; DONT-FUSE-NEXT:    vstr.16 s4, [r0]
+; DONT-FUSE-NEXT:    bx lr
+
+  %a = load half, half *%aa, align 2
+  %b = load half, half *%bb, align 2
+  %c = load half, half *%cc, align 2
+  %tmp2 = fsub half -0.0, %c
+  %tmp3 = tail call half @llvm.fma.f16(half %a, half %b, half %c) nounwind readnone
+  store half %tmp3, half *%aa, align 2
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_fma_const_fold(half *%aa, half *%bb) nounwind {
+; CHECK-LABEL: test_fma_const_fold:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr.16 s0, [r1]
+; CHECK-NEXT:    vldr.16 s2, [r0]
+; CHECK-NEXT:    vadd.f16 s0, s2, s0
+; CHECK-NEXT:    vstr.16 s0, [r0]
+; CHECK-NEXT:    bx lr
+;
+; DONT-FUSE-LABEL: test_fma_const_fold:
+; DONT-FUSE:       @ %bb.0:
+; DONT-FUSE-NEXT:    vldr.16 s0, [r1]
+; DONT-FUSE-NEXT:    vldr.16 s2, [r0]
+; DONT-FUSE-NEXT:    vadd.f16 s0, s2, s0
+; DONT-FUSE-NEXT:    vstr.16 s0, [r0]
+; DONT-FUSE-NEXT:    bx lr
+
+  %a = load half, half *%aa, align 2
+  %b = load half, half *%bb, align 2
+  %ret = call half @llvm.fma.f16(half %a, half 1.0, half %b)
+  store half %ret, half *%aa, align 2
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_fma_canonicalize(half *%aa, half *%bb) nounwind {
+; CHECK-LABEL: test_fma_canonicalize:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr.16 s0, [r0]
+; CHECK-NEXT:    vldr.16 s2, [r1]
+; CHECK-NEXT:    vmov.f16 s4, #2.000000e+00
+; CHECK-NEXT:    vfma.f16 s2, s0, s4
+; CHECK-NEXT:    vstr.16 s2, [r0]
+; CHECK-NEXT:    bx lr
+;
+; DONT-FUSE-LABEL: test_fma_canonicalize:
+; DONT-FUSE:       @ %bb.0:
+; DONT-FUSE-NEXT:    vldr.16 s0, [r0]
+; DONT-FUSE-NEXT:    vldr.16 s2, [r1]
+; DONT-FUSE-NEXT:    vmov.f16 s4, #2.000000e+00
+; DONT-FUSE-NEXT:    vfma.f16 s2, s0, s4
+; DONT-FUSE-NEXT:    vstr.16 s2, [r0]
+; DONT-FUSE-NEXT:    bx lr
+
+  %a = load half, half *%aa, align 2
+  %b = load half, half *%bb, align 2
+  %ret = call half @llvm.fma.f16(half 2.0, half %a, half %b)
+  store half %ret, half *%aa, align 2
+  ret void
+}
+
+define arm_aapcs_vfpcc void @fms1(half *%a1, half *%a2, half *%a3) {
+; CHECK-LABEL: fms1:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr.16 s0, [r1]
+; CHECK-NEXT:    vldr.16 s2, [r0]
+; CHECK-NEXT:    vldr.16 s4, [r2]
+; CHECK-NEXT:    vfms.f16 s4, s2, s0
+; CHECK-NEXT:    vstr.16 s4, [r0]
+; CHECK-NEXT:    bx lr
+;
+; DONT-FUSE-LABEL: fms1:
+; DONT-FUSE:       @ %bb.0:
+; DONT-FUSE-NEXT:    vldr.16 s0, [r1]
+; DONT-FUSE-NEXT:    vldr.16 s2, [r0]
+; DONT-FUSE-NEXT:    vldr.16 s4, [r2]
+; DONT-FUSE-NEXT:    vfms.f16 s4, s2, s0
+; DONT-FUSE-NEXT:    vstr.16 s4, [r0]
+; DONT-FUSE-NEXT:    bx lr
+
+  %f1 = load half, half *%a1, align 2
+  %f2 = load half, half *%a2, align 2
+  %f3 = load half, half *%a3, align 2
+  %s = fsub half -0.0, %f1
+  %ret = call half @llvm.fma.f16(half %s, half %f2, half %f3)
+  store half %ret, half *%a1, align 2
+  ret void
+}
+
+define arm_aapcs_vfpcc void @fms2(half *%a1, half *%a2, half *%a3) {
+; CHECK-LABEL: fms2:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr.16 s0, [r0]
+; CHECK-NEXT:    vldr.16 s2, [r1]
+; CHECK-NEXT:    vldr.16 s4, [r2]
+; CHECK-NEXT:    vfms.f16 s4, s2, s0
+; CHECK-NEXT:    vstr.16 s4, [r0]
+; CHECK-NEXT:    bx lr
+;
+; DONT-FUSE-LABEL: fms2:
+; DONT-FUSE:       @ %bb.0:
+; DONT-FUSE-NEXT:    vldr.16 s0, [r0]
+; DONT-FUSE-NEXT:    vldr.16 s2, [r1]
+; DONT-FUSE-NEXT:    vldr.16 s4, [r2]
+; DONT-FUSE-NEXT:    vfms.f16 s4, s2, s0
+; DONT-FUSE-NEXT:    vstr.16 s4, [r0]
+; DONT-FUSE-NEXT:    bx lr
+
+  %f1 = load half, half *%a1, align 2
+  %f2 = load half, half *%a2, align 2
+  %f3 = load half, half *%a3, align 2
+  %s = fsub half -0.0, %f1
+  %ret = call half @llvm.fma.f16(half %f2, half %s, half %f3)
+  store half %ret, half *%a1, align 2
+  ret void
+}
+
+define arm_aapcs_vfpcc void @fnma1(half *%a1, half *%a2, half *%a3) {
+; CHECK-LABEL: fnma1:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr.16 s0, [r1]
+; CHECK-NEXT:    vldr.16 s2, [r0]
+; CHECK-NEXT:    vldr.16 s4, [r2]
+; CHECK-NEXT:    vfnma.f16 s4, s2, s0
+; CHECK-NEXT:    vstr.16 s4, [r0]
+; CHECK-NEXT:    bx lr
+;
+; DONT-FUSE-LABEL: fnma1:
+; DONT-FUSE:       @ %bb.0:
+; DONT-FUSE-NEXT:    vldr.16 s0, [r1]
+; DONT-FUSE-NEXT:    vldr.16 s2, [r0]
+; DONT-FUSE-NEXT:    vldr.16 s4, [r2]
+; DONT-FUSE-NEXT:    vfnma.f16 s4, s2, s0
+; DONT-FUSE-NEXT:    vstr.16 s4, [r0]
+; DONT-FUSE-NEXT:    bx lr
+
+  %f1 = load half, half *%a1, align 2
+  %f2 = load half, half *%a2, align 2
+  %f3 = load half, half *%a3, align 2
+  %fma = call half @llvm.fma.f16(half %f1, half %f2, half %f3)
+  %n1 = fsub half -0.0, %fma
+  store half %n1, half *%a1, align 2
+  ret void
+}
+
+define arm_aapcs_vfpcc void @fnma2(half *%a1, half *%a2, half *%a3) {
+; CHECK-LABEL: fnma2:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr.16 s0, [r1]
+; CHECK-NEXT:    vldr.16 s2, [r0]
+; CHECK-NEXT:    vldr.16 s4, [r2]
+; CHECK-NEXT:    vfnma.f16 s4, s2, s0
+; CHECK-NEXT:    vstr.16 s4, [r0]
+; CHECK-NEXT:    bx lr
+;
+; DONT-FUSE-LABEL: fnma2:
+; DONT-FUSE:       @ %bb.0:
+; DONT-FUSE-NEXT:    vldr.16 s0, [r1]
+; DONT-FUSE-NEXT:    vldr.16 s2, [r0]
+; DONT-FUSE-NEXT:    vldr.16 s4, [r2]
+; DONT-FUSE-NEXT:    vfnma.f16 s4, s2, s0
+; DONT-FUSE-NEXT:    vstr.16 s4, [r0]
+; DONT-FUSE-NEXT:    bx lr
+
+  %f1 = load half, half *%a1, align 2
+  %f2 = load half, half *%a2, align 2
+  %f3 = load half, half *%a3, align 2
+  %n1 = fsub half -0.0, %f1
+  %n3 = fsub half -0.0, %f3
+  %ret = call half @llvm.fma.f16(half %n1, half %f2, half %n3)
+  store half %ret, half *%a1, align 2
+  ret void
+}
+
+define arm_aapcs_vfpcc void @fnms1(half *%a1, half *%a2, half *%a3) {
+; CHECK-LABEL: fnms1:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr.16 s0, [r1]
+; CHECK-NEXT:    vldr.16 s2, [r0]
+; CHECK-NEXT:    vldr.16 s4, [r2]
+; CHECK-NEXT:    vfnms.f16 s4, s2, s0
+; CHECK-NEXT:    vstr.16 s4, [r0]
+; CHECK-NEXT:    bx lr
+;
+; DONT-FUSE-LABEL: fnms1:
+; DONT-FUSE:       @ %bb.0:
+; DONT-FUSE-NEXT:    vldr.16 s0, [r1]
+; DONT-FUSE-NEXT:    vldr.16 s2, [r0]
+; DONT-FUSE-NEXT:    vldr.16 s4, [r2]
+; DONT-FUSE-NEXT:    vfnms.f16 s4, s2, s0
+; DONT-FUSE-NEXT:    vstr.16 s4, [r0]
+; DONT-FUSE-NEXT:    bx lr
+
+  %f1 = load half, half *%a1, align 2
+  %f2 = load half, half *%a2, align 2
+  %f3 = load half, half *%a3, align 2
+  %n3 = fsub half -0.0, %f3
+  %ret = call half @llvm.fma.f16(half %f1, half %f2, half %n3)
+  store half %ret, half *%a1, align 2
+  ret void
+}
+
+define arm_aapcs_vfpcc void @fnms2(half *%a1, half *%a2, half *%a3) {
+; CHECK-LABEL: fnms2:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr.16 s0, [r1]
+; CHECK-NEXT:    vldr.16 s2, [r0]
+; CHECK-NEXT:    vldr.16 s4, [r2]
+; CHECK-NEXT:    vfnms.f16 s4, s2, s0
+; CHECK-NEXT:    vstr.16 s4, [r0]
+; CHECK-NEXT:    bx lr
+;
+; DONT-FUSE-LABEL: fnms2:
+; DONT-FUSE:       @ %bb.0:
+; DONT-FUSE-NEXT:    vldr.16 s0, [r1]
+; DONT-FUSE-NEXT:    vldr.16 s2, [r0]
+; DONT-FUSE-NEXT:    vldr.16 s4, [r2]
+; DONT-FUSE-NEXT:    vfnms.f16 s4, s2, s0
+; DONT-FUSE-NEXT:    vstr.16 s4, [r0]
+; DONT-FUSE-NEXT:    bx lr
+
+  %f1 = load half, half *%a1, align 2
+  %f2 = load half, half *%a2, align 2
+  %f3 = load half, half *%a3, align 2
+  %n1 = fsub half -0.0, %f1
+  %fma = call half @llvm.fma.f16(half %n1, half %f2, half %f3)
+  %n = fsub half -0.0, %fma
+  store half %n, half *%a1, align 2
+  ret void
+}
+
+define arm_aapcs_vfpcc void @fnms3(half *%a1, half *%a2, half *%a3) {
+; CHECK-LABEL: fnms3:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr.16 s0, [r1]
+; CHECK-NEXT:    vldr.16 s2, [r0]
+; CHECK-NEXT:    vldr.16 s4, [r2]
+; CHECK-NEXT:    vfnms.f16 s4, s2, s0
+; CHECK-NEXT:    vstr.16 s4, [r0]
+; CHECK-NEXT:    bx lr
+;
+; DONT-FUSE-LABEL: fnms3:
+; DONT-FUSE:       @ %bb.0:
+; DONT-FUSE-NEXT:    vldr.16 s0, [r1]
+; DONT-FUSE-NEXT:    vldr.16 s2, [r0]
+; DONT-FUSE-NEXT:    vldr.16 s4, [r2]
+; DONT-FUSE-NEXT:    vfnms.f16 s4, s2, s0
+; DONT-FUSE-NEXT:    vstr.16 s4, [r0]
+; DONT-FUSE-NEXT:    bx lr
+
+  %f1 = load half, half *%a1, align 2
+  %f2 = load half, half *%a2, align 2
+  %f3 = load half, half *%a3, align 2
+  %n2 = fsub half -0.0, %f2
+  %fma = call half @llvm.fma.f16(half %f1, half %n2, half %f3)
+  %n1 = fsub half -0.0, %fma
+  store half %n1, half *%a1, align 2
+  ret void
+}
+
+
+declare half @llvm.fma.f16(half, half, half) nounwind readnone
diff --git a/llvm/test/CodeGen/ARM/fp16-instructions.ll b/llvm/test/CodeGen/ARM/fp16-instructions.ll
index a8fc53207..260dd12b3 100644
--- a/llvm/test/CodeGen/ARM/fp16-instructions.ll
+++ b/llvm/test/CodeGen/ARM/fp16-instructions.ll
@@ -164,9 +164,9 @@ entry:
 ; CHECK-LABEL:             VCMPE1:
 
 ; CHECK-SOFT:              bl  __aeabi_fcmplt
-; CHECK-SOFTFP-FP16:       vcmpe.f32 s0, #0
-; CHECK-SOFTFP-FULLFP16:   vcmpe.f16 s0, #0
-; CHECK-HARDFP-FULLFP16:   vcmpe.f16 s0, #0
+; CHECK-SOFTFP-FP16:       vcmp.f32 s0, #0
+; CHECK-SOFTFP-FULLFP16:   vcmp.f16 s0, #0
+; CHECK-HARDFP-FULLFP16:   vcmp.f16 s0, #0
 }
 
 define i32 @VCMPE2(float %F.coerce, float %G.coerce) {
@@ -184,9 +184,9 @@ entry:
 ; CHECK-LABEL:  VCMPE2:
 
 ; CHECK-SOFT:              bl  __aeabi_fcmplt
-; CHECK-SOFTFP-FP16:       vcmpe.f32 s{{.}}, s{{.}}
-; CHECK-SOFTFP-FULLFP16:   vcmpe.f16 s{{.}}, s{{.}}
-; CHECK-HARDFP-FULLFP16:   vcmpe.f16 s{{.}}, s{{.}}
+; CHECK-SOFTFP-FP16:       vcmp.f32 s{{.}}, s{{.}}
+; CHECK-SOFTFP-FULLFP16:   vcmp.f16 s{{.}}, s{{.}}
+; CHECK-HARDFP-FULLFP16:   vcmp.f16 s{{.}}, s{{.}}
 }
 
 ; Test lowering of BR_CC
@@ -212,10 +212,10 @@ for.end:
 ; CHECK-SOFT:             cmp r0, #{{0|1}}
 
 ; CHECK-SOFTFP-FP16:      vcvtb.f32.f16 [[S2:s[0-9]]], [[S2]]
-; CHECK-SOFTFP-FP16:      vcmpe.f32 [[S2]], s0
+; CHECK-SOFTFP-FP16:      vcmp.f32 [[S2]], s0
 ; CHECK-SOFTFP-FP16:      vmrs  APSR_nzcv, fpscr
 
-; CHECK-SOFTFP-FULLFP16:  vcmpe.f16 s{{.}}, s{{.}}
+; CHECK-SOFTFP-FULLFP16:  vcmp.f16 s{{.}}, s{{.}}
 ; CHECK-SOFTFP-FULLFP16:  vmrs  APSR_nzcv, fpscr
 }
 
@@ -727,15 +727,15 @@ define half @select_cc_ge1(half* %a0)  {
 
 ; CHECK-LABEL:                 select_cc_ge1:
 
-; CHECK-HARDFP-FULLFP16:       vcmpe.f16 s6, s0
+; CHECK-HARDFP-FULLFP16:       vcmp.f16 s6, s0
 ; CHECK-HARDFP-FULLFP16-NEXT:  vmrs APSR_nzcv, fpscr
 ; CHECK-HARDFP-FULLFP16-NEXT:  vselge.f16 s0, s{{.}}, s{{.}}
 
-; CHECK-SOFTFP-FP16-A32:       vcmpe.f32 s6, s0
+; CHECK-SOFTFP-FP16-A32:       vcmp.f32 s6, s0
 ; CHECK-SOFTFP-FP16-A32-NEXT:  vmrs APSR_nzcv, fpscr
 ; CHECK-SOFTFP-FP16-A32-NEXT:  vmovge.f32 s{{.}}, s{{.}}
 
-; CHECK-SOFTFP-FP16-T32:       vcmpe.f32 s6, s0
+; CHECK-SOFTFP-FP16-T32:       vcmp.f32 s6, s0
 ; CHECK-SOFTFP-FP16-T32-NEXT:  vmrs APSR_nzcv, fpscr
 ; CHECK-SOFTFP-FP16-T32-NEXT:  it ge
 ; CHECK-SOFTFP-FP16-T32-NEXT:  vmovge.f32 s{{.}}, s{{.}}
@@ -749,15 +749,15 @@ define half @select_cc_ge2(half* %a0)  {
 
 ; CHECK-LABEL:                 select_cc_ge2:
 
-; CHECK-HARDFP-FULLFP16:       vcmpe.f16 s0, s6
+; CHECK-HARDFP-FULLFP16:       vcmp.f16 s0, s6
 ; CHECK-HARDFP-FULLFP16-NEXT:  vmrs APSR_nzcv, fpscr
 ; CHECK-HARDFP-FULLFP16-NEXT:  vselge.f16 s0, s{{.}}, s{{.}}
 
-; CHECK-SOFTFP-FP16-A32:       vcmpe.f32 s6, s0
+; CHECK-SOFTFP-FP16-A32:       vcmp.f32 s6, s0
 ; CHECK-SOFTFP-FP16-A32-NEXT:  vmrs APSR_nzcv, fpscr
 ; CHECK-SOFTFP-FP16-A32-NEXT:  vmovls.f32 s{{.}}, s{{.}}
 
-; CHECK-SOFTFP-FP16-T32:       vcmpe.f32 s6, s0
+; CHECK-SOFTFP-FP16-T32:       vcmp.f32 s6, s0
 ; CHECK-SOFTFP-FP16-T32-NEXT:  vmrs APSR_nzcv, fpscr
 ; CHECK-SOFTFP-FP16-T32-NEXT:  it ls
 ; CHECK-SOFTFP-FP16-T32-NEXT:  vmovls.f32 s{{.}}, s{{.}}
@@ -771,15 +771,15 @@ define half @select_cc_ge3(half* %a0)  {
 
 ; CHECK-LABEL:                 select_cc_ge3:
 
-; CHECK-HARDFP-FULLFP16:       vcmpe.f16 s0, s6
+; CHECK-HARDFP-FULLFP16:       vcmp.f16 s0, s6
 ; CHECK-HARDFP-FULLFP16-NEXT:  vmrs APSR_nzcv, fpscr
 ; CHECK-HARDFP-FULLFP16-NEXT:  vselge.f16 s0, s{{.}}, s{{.}}
 
-; CHECK-SOFTFP-FP16-A32:       vcmpe.f32 s6, s0
+; CHECK-SOFTFP-FP16-A32:       vcmp.f32 s6, s0
 ; CHECK-SOFTFP-FP16-A32-NEXT:  vmrs APSR_nzcv, fpscr
 ; CHECK-SOFTFP-FP16-A32-NEXT:  vmovhi.f32 s{{.}}, s{{.}}
 
-; CHECK-SOFTFP-FP16-T32:       vcmpe.f32 s6, s0
+; CHECK-SOFTFP-FP16-T32:       vcmp.f32 s6, s0
 ; CHECK-SOFTFP-FP16-T32-NEXT:  vmrs APSR_nzcv, fpscr
 ; CHECK-SOFTFP-FP16-T32-NEXT:  it hi
 ; CHECK-SOFTFP-FP16-T32-NEXT:  vmovhi.f32 s{{.}}, s{{.}}
@@ -793,15 +793,15 @@ define half @select_cc_ge4(half* %a0)  {
 
 ; CHECK-LABEL:                 select_cc_ge4:
 
-; CHECK-HARDFP-FULLFP16:       vcmpe.f16 s6, s0
+; CHECK-HARDFP-FULLFP16:       vcmp.f16 s6, s0
 ; CHECK-HARDFP-FULLFP16-NEXT:  vmrs APSR_nzcv, fpscr
 ; CHECK-HARDFP-FULLFP16-NEXT:  vselge.f16 s0, s{{.}}, s{{.}}
 
-; CHECK-SOFTFP-FP16-A32:       vcmpe.f32 s6, s0
+; CHECK-SOFTFP-FP16-A32:       vcmp.f32 s6, s0
 ; CHECK-SOFTFP-FP16-A32-NEXT:  vmrs APSR_nzcv, fpscr
 ; CHECK-SOFTFP-FP16-A32-NEXT:  vmovlt.f32 s{{.}}, s{{.}}
 
-; CHECK-SOFTFP-FP16-T32:       vcmpe.f32 s6, s0
+; CHECK-SOFTFP-FP16-T32:       vcmp.f32 s6, s0
 ; CHECK-SOFTFP-FP16-T32-NEXT:  vmrs APSR_nzcv, fpscr
 ; CHECK-SOFTFP-FP16-T32-NEXT:  it lt
 ; CHECK-SOFTFP-FP16-T32-NEXT:  vmovlt.f32 s{{.}}, s{{.}}
@@ -816,15 +816,15 @@ define half @select_cc_gt1(half* %a0)  {
 
 ; CHECK-LABEL:                 select_cc_gt1:
 
-; CHECK-HARDFP-FULLFP16:       vcmpe.f16 s6, s0
+; CHECK-HARDFP-FULLFP16:       vcmp.f16 s6, s0
 ; CHECK-HARDFP-FULLFP16-NEXT:  vmrs APSR_nzcv, fpscr
 ; CHECK-HARDFP-FULLFP16-NEXT:  vselgt.f16  s0, s{{.}}, s{{.}}
 
-; CHECK-SOFTFP-FP16-A32:       vcmpe.f32 s6, s0
+; CHECK-SOFTFP-FP16-A32:       vcmp.f32 s6, s0
 ; CHECK-SOFTFP-FP16-A32-NEXT:  vmrs APSR_nzcv, fpscr
 ; CHECK-SOFTFP-FP16-A32-NEXT:  vmovgt.f32 s{{.}}, s{{.}}
 
-; CHECK-SOFTFP-FP16-T32:       vcmpe.f32 s6, s0
+; CHECK-SOFTFP-FP16-T32:       vcmp.f32 s6, s0
 ; CHECK-SOFTFP-FP16-T32-NEXT:  vmrs APSR_nzcv, fpscr
 ; CHECK-SOFTFP-FP16-T32-NEXT:  it gt
 ; CHECK-SOFTFP-FP16-T32-NEXT:  vmovgt.f32 s{{.}}, s{{.}}
@@ -838,15 +838,15 @@ define half @select_cc_gt2(half* %a0)  {
 
 ; CHECK-LABEL:                 select_cc_gt2:
 
-; CHECK-HARDFP-FULLFP16:       vcmpe.f16 s0, s6
+; CHECK-HARDFP-FULLFP16:       vcmp.f16 s0, s6
 ; CHECK-HARDFP-FULLFP16-NEXT:  vmrs  APSR_nzcv, fpscr
 ; CHECK-HARDFP-FULLFP16-NEXT:  vselgt.f16  s0, s{{.}}, s{{.}}
 
-; CHECK-SOFTFP-FP16-A32:       vcmpe.f32 s6, s0
+; CHECK-SOFTFP-FP16-A32:       vcmp.f32 s6, s0
 ; CHECK-SOFTFP-FP16-A32-NEXT:  vmrs APSR_nzcv, fpscr
 ; CHECK-SOFTFP-FP16-A32-NEXT:  vmovpl.f32 s{{.}}, s{{.}}
 
-; CHECK-SOFTFP-FP16-T32:       vcmpe.f32 s6, s0
+; CHECK-SOFTFP-FP16-T32:       vcmp.f32 s6, s0
 ; CHECK-SOFTFP-FP16-T32-NEXT:  vmrs APSR_nzcv, fpscr
 ; CHECK-SOFTFP-FP16-T32-NEXT:  it pl
 ; CHECK-SOFTFP-FP16-T32-NEXT:  vmovpl.f32 s{{.}}, s{{.}}
@@ -860,15 +860,15 @@ define half @select_cc_gt3(half* %a0)  {
 
 ; CHECK-LABEL:                 select_cc_gt3:
 
-; CHECK-HARDFP-FULLFP16:       vcmpe.f16 s6, s0
+; CHECK-HARDFP-FULLFP16:       vcmp.f16 s6, s0
 ; CHECK-HARDFP-FULLFP16-NEXT:  vmrs  APSR_nzcv, fpscr
 ; CHECK-HARDFP-FULLFP16-NEXT:  vselgt.f16  s0, s{{.}}, s{{.}}
 
-; CHECK-SOFTFP-FP16-A32:       vcmpe.f32 s6, s0
+; CHECK-SOFTFP-FP16-A32:       vcmp.f32 s6, s0
 ; CHECK-SOFTFP-FP16-A32-NEXT:  vmrs APSR_nzcv, fpscr
 ; CHECK-SOFTFP-FP16-A32-NEXT:  vmovle.f32 s{{.}}, s{{.}}
 
-; CHECK-SOFTFP-FP16-T32:       vcmpe.f32 s6, s0
+; CHECK-SOFTFP-FP16-T32:       vcmp.f32 s6, s0
 ; CHECK-SOFTFP-FP16-T32-NEXT:  vmrs APSR_nzcv, fpscr
 ; CHECK-SOFTFP-FP16-T32-NEXT:  it le
 ; CHECK-SOFTFP-FP16-T32-NEXT:  vmovle.f32 s{{.}}, s{{.}}
@@ -882,15 +882,15 @@ define half @select_cc_gt4(half* %a0)  {
 
 ; CHECK-LABEL:                 select_cc_gt4:
 
-; CHECK-HARDFP-FULLFP16:       vcmpe.f16 s0, s6
+; CHECK-HARDFP-FULLFP16:       vcmp.f16 s0, s6
 ; CHECK-HARDFP-FULLFP16-NEXT:  vmrs  APSR_nzcv, fpscr
 ; CHECK-HARDFP-FULLFP16-NEXT:  vselgt.f16  s0, s{{.}}, s{{.}}
 
-; CHECK-SOFTFP-FP16-A32:       vcmpe.f32 s6, s0
+; CHECK-SOFTFP-FP16-A32:       vcmp.f32 s6, s0
 ; CHECK-SOFTFP-FP16-A32-NEXT:  vmrs APSR_nzcv, fpscr
 ; CHECK-SOFTFP-FP16-A32-NEXT:  vmovmi.f32 s{{.}}, s{{.}}
 
-; CHECK-SOFTFP-FP16-T32:       vcmpe.f32 s6, s0
+; CHECK-SOFTFP-FP16-T32:       vcmp.f32 s6, s0
 ; CHECK-SOFTFP-FP16-T32-NEXT:  vmrs APSR_nzcv, fpscr
 ; CHECK-SOFTFP-FP16-T32-NEXT:  it mi
 ; CHECK-SOFTFP-FP16-T32-NEXT:  vmovmi.f32 s{{.}}, s{{.}}
diff --git a/llvm/test/CodeGen/ARM/fp16-promote.ll b/llvm/test/CodeGen/ARM/fp16-promote.ll
index f382144cf..5c4093502 100644
--- a/llvm/test/CodeGen/ARM/fp16-promote.ll
+++ b/llvm/test/CodeGen/ARM/fp16-promote.ll
@@ -202,7 +202,7 @@ define i1 @test_fcmp_ueq(half* %p, half* %q) #0 {
 ; CHECK-FP16: vcvtb.f32.f16
 ; CHECK-LIBCALL: bl __aeabi_h2f
 ; CHECK-LIBCALL: bl __aeabi_h2f
-; CHECK-VFP: vcmpe.f32
+; CHECK-VFP: vcmp.f32
 ; CHECK-NOVFP: bl __aeabi_fcmplt
 ; CHECK-FP16: vmrs APSR_nzcv, fpscr
 ; CHECK-VFP: strmi
@@ -497,6 +497,21 @@ define void @test_pow(half* %p, half* %q) #0 {
   ret void
 }
 
+; CHECK-FP16-LABEL: test_cbrt:
+; CHECK-FP16: vcvtb.f32.f16
+; CHECK-FP16: bl powf
+; CHECK-FP16: vcvtb.f16.f32
+; CHECK-LIBCALL-LABEL: test_cbrt:
+; CHECK-LIBCALL: bl __aeabi_h2f
+; CHECK-LIBCALL: bl powf
+; CHECK-LIBCALL: bl __aeabi_f2h
+define void @test_cbrt(half* %p) #0 {
+  %a = load half, half* %p, align 2
+  %r = call half @llvm.pow.f16(half %a, half 0x3FD5540000000000)
+  store half %r, half* %p
+  ret void
+}
+
 ; CHECK-FP16-LABEL: test_exp:
 ; CHECK-FP16: vcvtb.f32.f16
 ; CHECK-FP16: bl expf
diff --git a/llvm/test/CodeGen/ARM/fpcmp.ll b/llvm/test/CodeGen/ARM/fpcmp.ll
index 67326e000..b8fc21f81 100644
--- a/llvm/test/CodeGen/ARM/fpcmp.ll
+++ b/llvm/test/CodeGen/ARM/fpcmp.ll
@@ -2,7 +2,7 @@
 
 define i32 @f1(float %a) {
 ;CHECK-LABEL: f1:
-;CHECK: vcmpe.f32
+;CHECK: vcmp.f32
 ;CHECK: movmi
 entry:
         %tmp = fcmp olt float %a, 1.000000e+00          ; <i1> [#uses=1]
@@ -22,7 +22,7 @@ entry:
 
 define i32 @f3(float %a) {
 ;CHECK-LABEL: f3:
-;CHECK: vcmpe.f32
+;CHECK: vcmp.f32
 ;CHECK: movgt
 entry:
         %tmp = fcmp ogt float %a, 1.000000e+00          ; <i1> [#uses=1]
@@ -32,7 +32,7 @@ entry:
 
 define i32 @f4(float %a) {
 ;CHECK-LABEL: f4:
-;CHECK: vcmpe.f32
+;CHECK: vcmp.f32
 ;CHECK: movge
 entry:
         %tmp = fcmp oge float %a, 1.000000e+00          ; <i1> [#uses=1]
@@ -42,7 +42,7 @@ entry:
 
 define i32 @f5(float %a) {
 ;CHECK-LABEL: f5:
-;CHECK: vcmpe.f32
+;CHECK: vcmp.f32
 ;CHECK: movls
 entry:
         %tmp = fcmp ole float %a, 1.000000e+00          ; <i1> [#uses=1]
@@ -62,7 +62,7 @@ entry:
 
 define i32 @g1(double %a) {
 ;CHECK-LABEL: g1:
-;CHECK: vcmpe.f64
+;CHECK: vcmp.f64
 ;CHECK: movmi
 entry:
         %tmp = fcmp olt double %a, 1.000000e+00         ; <i1> [#uses=1]
diff --git a/llvm/test/CodeGen/ARM/fragmented-args-multiple-regs.ll b/llvm/test/CodeGen/ARM/fragmented-args-multiple-regs.ll
new file mode 100644
index 000000000..8c4822dfe
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/fragmented-args-multiple-regs.ll
@@ -0,0 +1,72 @@
+; RUN: llc < %s -mtriple=armv7-linux-gnueabihf -O1 -stop-after=finalize-isel | FileCheck %s
+
+define dso_local i32 @h(i64 %j) local_unnamed_addr !dbg !8 {
+entry:
+  call void @llvm.dbg.value(metadata i64 %j, metadata !14, metadata !DIExpression()), !dbg !29
+  call void @llvm.dbg.value(metadata i64 %j, metadata !15, metadata !DIExpression(DW_OP_LLVM_convert, 64, DW_ATE_unsigned, DW_OP_LLVM_convert, 32, DW_ATE_unsigned, DW_OP_stack_value, DW_OP_LLVM_fragment, 0, 32)), !dbg !29
+  call void @llvm.dbg.value(metadata i64 %j, metadata !15, metadata !DIExpression(DW_OP_constu, 32, DW_OP_shr, DW_OP_LLVM_convert, 64, DW_ATE_unsigned, DW_OP_LLVM_convert, 32, DW_ATE_unsigned, DW_OP_stack_value, DW_OP_LLVM_fragment, 32, 32)), !dbg !29
+  %tobool = icmp ult i64 %j, 4294967296, !dbg !30
+  br i1 %tobool, label %cleanup, label %if.then, !dbg !31
+
+if.then:                                          ; preds = %entry
+  call void @llvm.dbg.value(metadata i64 %j, metadata !15, metadata !DIExpression(DW_OP_LLVM_convert, 64, DW_ATE_unsigned, DW_OP_LLVM_convert, 32, DW_ATE_unsigned, DW_OP_stack_value, DW_OP_LLVM_fragment, 0, 32)), !dbg !29
+  %conv = sitofp i64 %j to double, !dbg !32
+  %add = fadd double %conv, 0x43F0000000000000, !dbg !33
+  call void @llvm.dbg.value(metadata double %add, metadata !25, metadata !DIExpression()), !dbg !34
+  %conv2 = fptosi double %add to i32, !dbg !35
+  br label %cleanup
+
+cleanup:                                          ; preds = %entry, %if.then
+  %retval.0 = phi i32 [ %conv2, %if.then ], [ undef, %entry ]
+  ret i32 %retval.0, !dbg !36
+}
+
+; CHECK-LABEL: bb.0.entry:
+; CHECK: DBG_VALUE [[REG1:%[0-9]+]], $noreg, !14, !DIExpression(DW_OP_LLVM_fragment, 32, 32
+; CHECK: DBG_VALUE [[REG2:%[0-9]+]], $noreg, !14, !DIExpression(DW_OP_LLVM_fragment, 0, 32
+; CHECK: DBG_VALUE [[REG2]], $noreg, !15, !DIExpression({{.+}}DW_OP_LLVM_fragment, 0, 32
+; CHECK: DBG_VALUE $noreg, $noreg, !15, !DIExpression({{.+}}DW_OP_LLVM_fragment, 32, 32
+
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5, !6}
+!llvm.ident = !{!7}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 10.0.0 ", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "tif_aux.c", directory: "/tmp")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{i32 1, !"min_enum_size", i32 4}
+!7 = !{!"clang version 10.0.0 "}
+!8 = distinct !DISubprogram(name: "h", scope: !1, file: !1, line: 10, type: !9, scopeLine: 10, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !13)
+!9 = !DISubroutineType(types: !10)
+!10 = !{!11, !12}
+!11 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!12 = !DIBasicType(name: "long long int", size: 64, encoding: DW_ATE_signed)
+!13 = !{!14, !15, !25}
+!14 = !DILocalVariable(name: "j", arg: 1, scope: !8, file: !1, line: 10, type: !12)
+!15 = !DILocalVariable(name: "i", scope: !8, file: !1, line: 11, type: !16)
+!16 = !DIDerivedType(tag: DW_TAG_typedef, name: "g", file: !1, line: 8, baseType: !17)
+!17 = distinct !DICompositeType(tag: DW_TAG_union_type, file: !1, line: 5, size: 64, elements: !18)
+!18 = !{!19, !24}
+!19 = !DIDerivedType(tag: DW_TAG_member, name: "e", scope: !17, file: !1, line: 6, baseType: !20, size: 64)
+!20 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "b", file: !1, line: 1, size: 64, elements: !21)
+!21 = !{!22, !23}
+!22 = !DIDerivedType(tag: DW_TAG_member, name: "c", scope: !20, file: !1, line: 2, baseType: !11, size: 32)
+!23 = !DIDerivedType(tag: DW_TAG_member, name: "d", scope: !20, file: !1, line: 2, baseType: !11, size: 32, offset: 32)
+!24 = !DIDerivedType(tag: DW_TAG_member, name: "f", scope: !17, file: !1, line: 7, baseType: !12, size: 64)
+!25 = !DILocalVariable(name: "a", scope: !26, file: !1, line: 14, type: !28)
+!26 = distinct !DILexicalBlock(scope: !27, file: !1, line: 13, column: 14)
+!27 = distinct !DILexicalBlock(scope: !8, file: !1, line: 13, column: 7)
+!28 = !DIBasicType(name: "double", size: 64, encoding: DW_ATE_float)
+!29 = !DILocation(line: 0, scope: !8)
+!30 = !DILocation(line: 13, column: 7, scope: !27)
+!31 = !DILocation(line: 13, column: 7, scope: !8)
+!32 = !DILocation(line: 14, column: 16, scope: !26)
+!33 = !DILocation(line: 14, column: 20, scope: !26)
+!34 = !DILocation(line: 0, scope: !26)
+!35 = !DILocation(line: 15, column: 12, scope: !26)
+!36 = !DILocation(line: 17, column: 1, scope: !8)
diff --git a/llvm/test/CodeGen/ARM/fusedMAC.ll b/llvm/test/CodeGen/ARM/fusedMAC.ll
index 6b922895b..493e71c9f 100644
--- a/llvm/test/CodeGen/ARM/fusedMAC.ll
+++ b/llvm/test/CodeGen/ARM/fusedMAC.ll
@@ -5,7 +5,7 @@
 
 ; Check generated fused MAC and MLS.
 
-define double @fusedMACTest1(double %d1, double %d2, double %d3) {
+define arm_aapcs_vfpcc double @fusedMACTest1(double %d1, double %d2, double %d3) {
 ;CHECK-LABEL: fusedMACTest1:
 ;CHECK: vfma.f64
   %1 = fmul double %d1, %d2
@@ -13,7 +13,7 @@ define double @fusedMACTest1(double %d1, double %d2, double %d3) {
   ret double %2
 }
 
-define float @fusedMACTest2(float %f1, float %f2, float %f3) {
+define arm_aapcs_vfpcc float @fusedMACTest2(float %f1, float %f2, float %f3) {
 ;CHECK-LABEL: fusedMACTest2:
 ;CHECK: vfma.f32
 
@@ -26,7 +26,7 @@ define float @fusedMACTest2(float %f1, float %f2, float %f3) {
   ret float %2
 }
 
-define double @fusedMACTest3(double %d1, double %d2, double %d3) {
+define arm_aapcs_vfpcc double @fusedMACTest3(double %d1, double %d2, double %d3) {
 ;CHECK-LABEL: fusedMACTest3:
 ;CHECK: vfms.f64
   %1 = fmul double %d2, %d3
@@ -34,7 +34,7 @@ define double @fusedMACTest3(double %d1, double %d2, double %d3) {
   ret double %2
 }
 
-define float @fusedMACTest4(float %f1, float %f2, float %f3) {
+define arm_aapcs_vfpcc float @fusedMACTest4(float %f1, float %f2, float %f3) {
 ;CHECK-LABEL: fusedMACTest4:
 ;CHECK: vfms.f32
   %1 = fmul float %f2, %f3
@@ -42,7 +42,7 @@ define float @fusedMACTest4(float %f1, float %f2, float %f3) {
   ret float %2
 }
 
-define double @fusedMACTest5(double %d1, double %d2, double %d3) {
+define arm_aapcs_vfpcc double @fusedMACTest5(double %d1, double %d2, double %d3) {
 ;CHECK-LABEL: fusedMACTest5:
 ;CHECK: vfnma.f64
   %1 = fmul double %d1, %d2
@@ -51,7 +51,7 @@ define double @fusedMACTest5(double %d1, double %d2, double %d3) {
   ret double %3
 }
 
-define float @fusedMACTest6(float %f1, float %f2, float %f3) {
+define arm_aapcs_vfpcc float @fusedMACTest6(float %f1, float %f2, float %f3) {
 ;CHECK-LABEL: fusedMACTest6:
 ;CHECK: vfnma.f32
   %1 = fmul float %f1, %f2
@@ -60,7 +60,7 @@ define float @fusedMACTest6(float %f1, float %f2, float %f3) {
   ret float %3
 }
 
-define double @fusedMACTest7(double %d1, double %d2, double %d3) {
+define arm_aapcs_vfpcc double @fusedMACTest7(double %d1, double %d2, double %d3) {
 ;CHECK-LABEL: fusedMACTest7:
 ;CHECK: vfnms.f64
   %1 = fmul double %d1, %d2
@@ -68,7 +68,7 @@ define double @fusedMACTest7(double %d1, double %d2, double %d3) {
   ret double %2
 }
 
-define float @fusedMACTest8(float %f1, float %f2, float %f3) {
+define arm_aapcs_vfpcc float @fusedMACTest8(float %f1, float %f2, float %f3) {
 ;CHECK-LABEL: fusedMACTest8:
 ;CHECK: vfnms.f32
   %1 = fmul float %f1, %f2
@@ -76,7 +76,7 @@ define float @fusedMACTest8(float %f1, float %f2, float %f3) {
   ret float %2
 }
 
-define <2 x float> @fusedMACTest9(<2 x float> %a, <2 x float> %b) {
+define arm_aapcs_vfpcc <2 x float> @fusedMACTest9(<2 x float> %a, <2 x float> %b) {
 ;CHECK-LABEL: fusedMACTest9:
 ;CHECK: vfma.f32
   %mul = fmul <2 x float> %a, %b
@@ -84,7 +84,7 @@ define <2 x float> @fusedMACTest9(<2 x float> %a, <2 x float> %b) {
   ret <2 x float> %add
 }
 
-define <2 x float> @fusedMACTest10(<2 x float> %a, <2 x float> %b) {
+define arm_aapcs_vfpcc <2 x float> @fusedMACTest10(<2 x float> %a, <2 x float> %b) {
 ;CHECK-LABEL: fusedMACTest10:
 ;CHECK: vfms.f32
   %mul = fmul <2 x float> %a, %b
@@ -92,7 +92,7 @@ define <2 x float> @fusedMACTest10(<2 x float> %a, <2 x float> %b) {
   ret <2 x float> %sub
 }
 
-define <4 x float> @fusedMACTest11(<4 x float> %a, <4 x float> %b) {
+define arm_aapcs_vfpcc <4 x float> @fusedMACTest11(<4 x float> %a, <4 x float> %b) {
 ;CHECK-LABEL: fusedMACTest11:
 ;CHECK: vfma.f32
   %mul = fmul <4 x float> %a, %b
@@ -100,7 +100,7 @@ define <4 x float> @fusedMACTest11(<4 x float> %a, <4 x float> %b) {
   ret <4 x float> %add
 }
 
-define <4 x float> @fusedMACTest12(<4 x float> %a, <4 x float> %b) {
+define arm_aapcs_vfpcc <4 x float> @fusedMACTest12(<4 x float> %a, <4 x float> %b) {
 ;CHECK-LABEL: fusedMACTest12:
 ;CHECK: vfms.f32
   %mul = fmul <4 x float> %a, %b
@@ -108,7 +108,7 @@ define <4 x float> @fusedMACTest12(<4 x float> %a, <4 x float> %b) {
   ret <4 x float> %sub
 }
 
-define float @test_fma_f32(float %a, float %b, float %c) nounwind readnone ssp {
+define arm_aapcs_vfpcc float @test_fma_f32(float %a, float %b, float %c) nounwind readnone ssp {
 entry:
 ; CHECK: test_fma_f32
 ; CHECK: vfma.f32
@@ -116,7 +116,7 @@ entry:
   ret float %tmp1
 }
 
-define double @test_fma_f64(double %a, double %b, double %c) nounwind readnone ssp {
+define arm_aapcs_vfpcc double @test_fma_f64(double %a, double %b, double %c) nounwind readnone ssp {
 entry:
 ; CHECK: test_fma_f64
 ; CHECK: vfma.f64
@@ -124,7 +124,7 @@ entry:
   ret double %tmp1
 }
 
-define <2 x float> @test_fma_v2f32(<2 x float> %a, <2 x float> %b, <2 x float> %c) nounwind readnone ssp {
+define arm_aapcs_vfpcc <2 x float> @test_fma_v2f32(<2 x float> %a, <2 x float> %b, <2 x float> %c) nounwind readnone ssp {
 entry:
 ; CHECK: test_fma_v2f32
 ; CHECK: vfma.f32
@@ -132,7 +132,7 @@ entry:
   ret <2 x float> %tmp1
 }
 
-define double @test_fms_f64(double %a, double %b, double %c) nounwind readnone ssp {
+define arm_aapcs_vfpcc double @test_fms_f64(double %a, double %b, double %c) nounwind readnone ssp {
 entry:
 ; CHECK: test_fms_f64
 ; CHECK: vfms.f64
@@ -141,7 +141,7 @@ entry:
   ret double %tmp2
 }
 
-define double @test_fms_f64_2(double %a, double %b, double %c) nounwind readnone ssp {
+define arm_aapcs_vfpcc double @test_fms_f64_2(double %a, double %b, double %c) nounwind readnone ssp {
 entry:
 ; CHECK: test_fms_f64_2
 ; CHECK: vfms.f64
@@ -150,7 +150,7 @@ entry:
   ret double %tmp2
 }
 
-define float @test_fnms_f32(float %a, float %b, float* %c) nounwind readnone ssp {
+define arm_aapcs_vfpcc float @test_fnms_f32(float %a, float %b, float* %c) nounwind readnone ssp {
 ; CHECK: test_fnms_f32
 ; CHECK: vfnms.f32
   %tmp1 = load float, float* %c, align 4
@@ -159,7 +159,7 @@ define float @test_fnms_f32(float %a, float %b, float* %c) nounwind readnone ssp
   ret float %tmp3 
 }
 
-define double @test_fnms_f64(double %a, double %b, double %c) nounwind readnone ssp {
+define arm_aapcs_vfpcc double @test_fnms_f64(double %a, double %b, double %c) nounwind readnone ssp {
 entry:
 ; CHECK: test_fnms_f64
 ; CHECK: vfnms.f64
@@ -169,7 +169,7 @@ entry:
   ret double %tmp3
 }
 
-define double @test_fnms_f64_2(double %a, double %b, double %c) nounwind readnone ssp {
+define arm_aapcs_vfpcc double @test_fnms_f64_2(double %a, double %b, double %c) nounwind readnone ssp {
 entry:
 ; CHECK: test_fnms_f64_2
 ; CHECK: vfnms.f64
@@ -179,7 +179,7 @@ entry:
   ret double %tmp3
 }
 
-define double @test_fnma_f64(double %a, double %b, double %c) nounwind readnone ssp {
+define arm_aapcs_vfpcc double @test_fnma_f64(double %a, double %b, double %c) nounwind readnone ssp {
 entry:
 ; CHECK: test_fnma_f64
 ; CHECK: vfnma.f64
@@ -188,7 +188,7 @@ entry:
   ret double %tmp2
 }
 
-define double @test_fnma_f64_2(double %a, double %b, double %c) nounwind readnone ssp {
+define arm_aapcs_vfpcc double @test_fnma_f64_2(double %a, double %b, double %c) nounwind readnone ssp {
 entry:
 ; CHECK: test_fnma_f64_2
 ; CHECK: vfnma.f64
@@ -198,7 +198,7 @@ entry:
   ret double %tmp3
 }
 
-define float @test_fma_const_fold(float %a, float %b) nounwind {
+define arm_aapcs_vfpcc float @test_fma_const_fold(float %a, float %b) nounwind {
 ; CHECK: test_fma_const_fold
 ; CHECK-NOT: vfma
 ; CHECK-NOT: vmul
@@ -207,7 +207,7 @@ define float @test_fma_const_fold(float %a, float %b) nounwind {
   ret float %ret
 }
 
-define float @test_fma_canonicalize(float %a, float %b) nounwind {
+define arm_aapcs_vfpcc float @test_fma_canonicalize(float %a, float %b) nounwind {
 ; CHECK: test_fma_canonicalize
 ; CHECK: vmov.f32 [[R1:s[0-9]+]], #2.000000e+00
 ; CHECK: vfma.f32 {{s[0-9]+}}, {{s[0-9]+}}, [[R1]]
@@ -216,7 +216,7 @@ define float @test_fma_canonicalize(float %a, float %b) nounwind {
 }
 
 ; Check that very wide vector fma's can be split into legal fma's.
-define void @test_fma_v8f32(<8 x float> %a, <8 x float> %b, <8 x float> %c, <8 x float>* %p) nounwind readnone ssp {
+define arm_aapcs_vfpcc void @test_fma_v8f32(<8 x float> %a, <8 x float> %b, <8 x float> %c, <8 x float>* %p) nounwind readnone ssp {
 ; CHECK: test_fma_v8f32
 ; CHECK: vfma.f32
 ; CHECK: vfma.f32
diff --git a/llvm/test/CodeGen/ARM/global-merge-1.ll b/llvm/test/CodeGen/ARM/global-merge-1.ll
index 67eec4b9c..68b346ec9 100644
--- a/llvm/test/CodeGen/ARM/global-merge-1.ll
+++ b/llvm/test/CodeGen/ARM/global-merge-1.ll
@@ -74,9 +74,9 @@ define internal i32* @returnFoo() #2 {
   ret i32* getelementptr inbounds ([5 x i32], [5 x i32]* @foo, i32 0, i32 0)
 }
 
-attributes #0 = { nounwind ssp "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #2 = { nounwind readnone ssp "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind ssp "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { nounwind readnone ssp "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #3 = { nounwind }
 
 !llvm.ident = !{!0}
diff --git a/llvm/test/CodeGen/ARM/gnu_mcount_nc.ll b/llvm/test/CodeGen/ARM/gnu_mcount_nc.ll
new file mode 100644
index 000000000..c36991f87
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/gnu_mcount_nc.ll
@@ -0,0 +1,41 @@
+; RUN: llc -mtriple=armv7a-linux-gnueabihf -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK-ARM
+; RUN: llc -mtriple=armv7a-linux-gnueabihf -verify-machineinstrs -fast-isel %s -o - | FileCheck %s --check-prefix=CHECK-ARM-FAST-ISEL
+; RUN: llc -mtriple=armv7a-linux-gnueabihf -verify-machineinstrs -global-isel -global-isel-abort=2 %s -o - | FileCheck %s --check-prefix=CHECK-ARM-GLOBAL-ISEL
+; RUN: llc -mtriple=thumbv7a-linux-gnueabihf -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK-THUMB
+; RUN: llc -mtriple=thumbv7a-linux-gnueabihf -verify-machineinstrs -fast-isel %s -o - | FileCheck %s --check-prefix=CHECK-THUMB-FAST-ISEL
+; RUN: llc -mtriple=thumbv7a-linux-gnueabihf -verify-machineinstrs -global-isel -global-isel-abort=2 %s -o - | FileCheck %s --check-prefix=CHECK-THUMB-GLOBAL-ISEL
+
+define dso_local void @callee() #0 {
+; CHECK-ARM:                    stmdb   sp!, {lr}
+; CHECK-ARM-NEXT:               bl      __gnu_mcount_nc
+; CHECK-ARM-FAST-ISEL:          stmdb   sp!, {lr}
+; CHECK-ARM-FAST-ISEL-NEXT:     bl      __gnu_mcount_nc
+; CHECK-ARM-GLOBAL-ISEL:        stmdb   sp!, {lr}
+; CHECK-ARM-GLOBAL-ISEL-NEXT:   bl      __gnu_mcount_nc
+; CHECK-THUMB:                  push    {lr}
+; CHECK-THUMB-NEXT:             bl      __gnu_mcount_nc
+; CHECK-THUMB-FAST-ISEL:        push    {lr}
+; CHECK-THUMB-FAST-ISEL-NEXT:   bl      __gnu_mcount_nc
+; CHECK-THUMB-GLOBAL-ISEL:      push    {lr}
+; CHECK-THUMB-GLOBAL-ISEL-NEXT: bl      __gnu_mcount_nc
+  ret void
+}
+
+define dso_local void @caller() #0 {
+; CHECK-ARM:                    stmdb   sp!, {lr}
+; CHECK-ARM-NEXT:               bl      __gnu_mcount_nc
+; CHECK-ARM-FAST-ISEL:          stmdb   sp!, {lr}
+; CHECK-ARM-FAST-ISEL-NEXT:     bl      __gnu_mcount_nc
+; CHECK-ARM-GLOBAL-ISEL:        stmdb   sp!, {lr}
+; CHECK-ARM-GLOBAL-ISEL-NEXT:   bl      __gnu_mcount_nc
+; CHECK-THUMB:                  push    {lr}
+; CHECK-THUMB-NEXT:             bl      __gnu_mcount_nc
+; CHECK-THUMB-FAST-ISEL:        push    {lr}
+; CHECK-THUMB-FAST-ISEL-NEXT:   bl      __gnu_mcount_nc
+; CHECK-THUMB-GLOBAL-ISEL:      push    {lr}
+; CHECK-THUMB-GLOBAL-ISEL-NEXT: bl      __gnu_mcount_nc
+  call void @callee()
+  ret void
+}
+
+attributes #0 = { nofree nounwind "instrument-function-entry-inlined"="llvm.arm.gnu.eabi.mcount" }
diff --git a/llvm/test/CodeGen/ARM/hello.ll b/llvm/test/CodeGen/ARM/hello.ll
index bdeb41dec..429a6b0ef 100644
--- a/llvm/test/CodeGen/ARM/hello.ll
+++ b/llvm/test/CodeGen/ARM/hello.ll
@@ -9,7 +9,7 @@
 
 @str = internal constant [12 x i8] c"Hello World\00"
 
-define i32 @main() "no-frame-pointer-elim"="true" {
+define i32 @main() "frame-pointer"="all" {
 	%tmp = call i32 @puts( i8* getelementptr ([12 x i8], [12 x i8]* @str, i32 0, i64 0) )		; <i32> [#uses=0]
 	ret i32 0
 }
diff --git a/llvm/test/CodeGen/ARM/hoist-and-by-const-from-lshr-in-eqcmp-zero.ll b/llvm/test/CodeGen/ARM/hoist-and-by-const-from-lshr-in-eqcmp-zero.ll
index 774149299..bf029ee86 100644
--- a/llvm/test/CodeGen/ARM/hoist-and-by-const-from-lshr-in-eqcmp-zero.ll
+++ b/llvm/test/CodeGen/ARM/hoist-and-by-const-from-lshr-in-eqcmp-zero.ll
@@ -21,20 +21,18 @@ define i1 @scalar_i8_signbit_eq(i8 %x, i8 %y) nounwind {
 ; ARM-LABEL: scalar_i8_signbit_eq:
 ; ARM:       @ %bb.0:
 ; ARM-NEXT:    uxtb r1, r1
-; ARM-NEXT:    mov r2, #128
-; ARM-NEXT:    and r0, r0, r2, lsr r1
+; ARM-NEXT:    lsl r0, r0, r1
+; ARM-NEXT:    mov r1, #1
 ; ARM-NEXT:    uxtb r0, r0
-; ARM-NEXT:    clz r0, r0
-; ARM-NEXT:    lsr r0, r0, #5
+; ARM-NEXT:    eor r0, r1, r0, lsr #7
 ; ARM-NEXT:    bx lr
 ;
 ; THUMB6-LABEL: scalar_i8_signbit_eq:
 ; THUMB6:       @ %bb.0:
 ; THUMB6-NEXT:    uxtb r1, r1
-; THUMB6-NEXT:    movs r2, #128
-; THUMB6-NEXT:    lsrs r2, r1
-; THUMB6-NEXT:    ands r2, r0
-; THUMB6-NEXT:    uxtb r1, r2
+; THUMB6-NEXT:    lsls r0, r1
+; THUMB6-NEXT:    movs r1, #128
+; THUMB6-NEXT:    ands r1, r0
 ; THUMB6-NEXT:    rsbs r0, r1, #0
 ; THUMB6-NEXT:    adcs r0, r1
 ; THUMB6-NEXT:    bx lr
@@ -42,12 +40,10 @@ define i1 @scalar_i8_signbit_eq(i8 %x, i8 %y) nounwind {
 ; THUMB78-LABEL: scalar_i8_signbit_eq:
 ; THUMB78:       @ %bb.0:
 ; THUMB78-NEXT:    uxtb r1, r1
-; THUMB78-NEXT:    movs r2, #128
-; THUMB78-NEXT:    lsr.w r1, r2, r1
-; THUMB78-NEXT:    ands r0, r1
+; THUMB78-NEXT:    lsls r0, r1
+; THUMB78-NEXT:    movs r1, #1
 ; THUMB78-NEXT:    uxtb r0, r0
-; THUMB78-NEXT:    clz r0, r0
-; THUMB78-NEXT:    lsrs r0, r0, #5
+; THUMB78-NEXT:    eor.w r0, r1, r0, lsr #7
 ; THUMB78-NEXT:    bx lr
   %t0 = lshr i8 128, %y
   %t1 = and i8 %t0, %x
@@ -60,19 +56,15 @@ define i1 @scalar_i8_lowestbit_eq(i8 %x, i8 %y) nounwind {
 ; ARM:       @ %bb.0:
 ; ARM-NEXT:    uxtb r1, r1
 ; ARM-NEXT:    mov r2, #1
-; ARM-NEXT:    and r0, r0, r2, lsr r1
-; ARM-NEXT:    uxtb r0, r0
-; ARM-NEXT:    clz r0, r0
-; ARM-NEXT:    lsr r0, r0, #5
+; ARM-NEXT:    bic r0, r2, r0, lsl r1
 ; ARM-NEXT:    bx lr
 ;
 ; THUMB6-LABEL: scalar_i8_lowestbit_eq:
 ; THUMB6:       @ %bb.0:
 ; THUMB6-NEXT:    uxtb r1, r1
-; THUMB6-NEXT:    movs r2, #1
-; THUMB6-NEXT:    lsrs r2, r1
-; THUMB6-NEXT:    ands r2, r0
-; THUMB6-NEXT:    uxtb r1, r2
+; THUMB6-NEXT:    lsls r0, r1
+; THUMB6-NEXT:    movs r1, #1
+; THUMB6-NEXT:    ands r1, r0
 ; THUMB6-NEXT:    rsbs r0, r1, #0
 ; THUMB6-NEXT:    adcs r0, r1
 ; THUMB6-NEXT:    bx lr
@@ -80,12 +72,9 @@ define i1 @scalar_i8_lowestbit_eq(i8 %x, i8 %y) nounwind {
 ; THUMB78-LABEL: scalar_i8_lowestbit_eq:
 ; THUMB78:       @ %bb.0:
 ; THUMB78-NEXT:    uxtb r1, r1
-; THUMB78-NEXT:    movs r2, #1
-; THUMB78-NEXT:    lsr.w r1, r2, r1
-; THUMB78-NEXT:    ands r0, r1
-; THUMB78-NEXT:    uxtb r0, r0
-; THUMB78-NEXT:    clz r0, r0
-; THUMB78-NEXT:    lsrs r0, r0, #5
+; THUMB78-NEXT:    lsls r0, r1
+; THUMB78-NEXT:    movs r1, #1
+; THUMB78-NEXT:    bic.w r0, r1, r0
 ; THUMB78-NEXT:    bx lr
   %t0 = lshr i8 1, %y
   %t1 = and i8 %t0, %x
@@ -98,8 +87,7 @@ define i1 @scalar_i8_bitsinmiddle_eq(i8 %x, i8 %y) nounwind {
 ; ARM:       @ %bb.0:
 ; ARM-NEXT:    uxtb r1, r1
 ; ARM-NEXT:    mov r2, #24
-; ARM-NEXT:    and r0, r0, r2, lsr r1
-; ARM-NEXT:    uxtb r0, r0
+; ARM-NEXT:    and r0, r2, r0, lsl r1
 ; ARM-NEXT:    clz r0, r0
 ; ARM-NEXT:    lsr r0, r0, #5
 ; ARM-NEXT:    bx lr
@@ -107,10 +95,9 @@ define i1 @scalar_i8_bitsinmiddle_eq(i8 %x, i8 %y) nounwind {
 ; THUMB6-LABEL: scalar_i8_bitsinmiddle_eq:
 ; THUMB6:       @ %bb.0:
 ; THUMB6-NEXT:    uxtb r1, r1
-; THUMB6-NEXT:    movs r2, #24
-; THUMB6-NEXT:    lsrs r2, r1
-; THUMB6-NEXT:    ands r2, r0
-; THUMB6-NEXT:    uxtb r1, r2
+; THUMB6-NEXT:    lsls r0, r1
+; THUMB6-NEXT:    movs r1, #24
+; THUMB6-NEXT:    ands r1, r0
 ; THUMB6-NEXT:    rsbs r0, r1, #0
 ; THUMB6-NEXT:    adcs r0, r1
 ; THUMB6-NEXT:    bx lr
@@ -118,10 +105,8 @@ define i1 @scalar_i8_bitsinmiddle_eq(i8 %x, i8 %y) nounwind {
 ; THUMB78-LABEL: scalar_i8_bitsinmiddle_eq:
 ; THUMB78:       @ %bb.0:
 ; THUMB78-NEXT:    uxtb r1, r1
-; THUMB78-NEXT:    movs r2, #24
-; THUMB78-NEXT:    lsr.w r1, r2, r1
-; THUMB78-NEXT:    ands r0, r1
-; THUMB78-NEXT:    uxtb r0, r0
+; THUMB78-NEXT:    lsls r0, r1
+; THUMB78-NEXT:    and r0, r0, #24
 ; THUMB78-NEXT:    clz r0, r0
 ; THUMB78-NEXT:    lsrs r0, r0, #5
 ; THUMB78-NEXT:    bx lr
@@ -137,21 +122,19 @@ define i1 @scalar_i16_signbit_eq(i16 %x, i16 %y) nounwind {
 ; ARM-LABEL: scalar_i16_signbit_eq:
 ; ARM:       @ %bb.0:
 ; ARM-NEXT:    uxth r1, r1
-; ARM-NEXT:    mov r2, #32768
-; ARM-NEXT:    and r0, r0, r2, lsr r1
+; ARM-NEXT:    lsl r0, r0, r1
+; ARM-NEXT:    mov r1, #1
 ; ARM-NEXT:    uxth r0, r0
-; ARM-NEXT:    clz r0, r0
-; ARM-NEXT:    lsr r0, r0, #5
+; ARM-NEXT:    eor r0, r1, r0, lsr #15
 ; ARM-NEXT:    bx lr
 ;
 ; THUMB6-LABEL: scalar_i16_signbit_eq:
 ; THUMB6:       @ %bb.0:
 ; THUMB6-NEXT:    uxth r1, r1
-; THUMB6-NEXT:    movs r2, #1
-; THUMB6-NEXT:    lsls r2, r2, #15
-; THUMB6-NEXT:    lsrs r2, r1
-; THUMB6-NEXT:    ands r2, r0
-; THUMB6-NEXT:    uxth r1, r2
+; THUMB6-NEXT:    lsls r0, r1
+; THUMB6-NEXT:    movs r1, #1
+; THUMB6-NEXT:    lsls r1, r1, #15
+; THUMB6-NEXT:    ands r1, r0
 ; THUMB6-NEXT:    rsbs r0, r1, #0
 ; THUMB6-NEXT:    adcs r0, r1
 ; THUMB6-NEXT:    bx lr
@@ -159,12 +142,10 @@ define i1 @scalar_i16_signbit_eq(i16 %x, i16 %y) nounwind {
 ; THUMB78-LABEL: scalar_i16_signbit_eq:
 ; THUMB78:       @ %bb.0:
 ; THUMB78-NEXT:    uxth r1, r1
-; THUMB78-NEXT:    mov.w r2, #32768
-; THUMB78-NEXT:    lsr.w r1, r2, r1
-; THUMB78-NEXT:    ands r0, r1
+; THUMB78-NEXT:    lsls r0, r1
+; THUMB78-NEXT:    movs r1, #1
 ; THUMB78-NEXT:    uxth r0, r0
-; THUMB78-NEXT:    clz r0, r0
-; THUMB78-NEXT:    lsrs r0, r0, #5
+; THUMB78-NEXT:    eor.w r0, r1, r0, lsr #15
 ; THUMB78-NEXT:    bx lr
   %t0 = lshr i16 32768, %y
   %t1 = and i16 %t0, %x
@@ -177,19 +158,15 @@ define i1 @scalar_i16_lowestbit_eq(i16 %x, i16 %y) nounwind {
 ; ARM:       @ %bb.0:
 ; ARM-NEXT:    uxth r1, r1
 ; ARM-NEXT:    mov r2, #1
-; ARM-NEXT:    and r0, r0, r2, lsr r1
-; ARM-NEXT:    uxth r0, r0
-; ARM-NEXT:    clz r0, r0
-; ARM-NEXT:    lsr r0, r0, #5
+; ARM-NEXT:    bic r0, r2, r0, lsl r1
 ; ARM-NEXT:    bx lr
 ;
 ; THUMB6-LABEL: scalar_i16_lowestbit_eq:
 ; THUMB6:       @ %bb.0:
 ; THUMB6-NEXT:    uxth r1, r1
-; THUMB6-NEXT:    movs r2, #1
-; THUMB6-NEXT:    lsrs r2, r1
-; THUMB6-NEXT:    ands r2, r0
-; THUMB6-NEXT:    uxth r1, r2
+; THUMB6-NEXT:    lsls r0, r1
+; THUMB6-NEXT:    movs r1, #1
+; THUMB6-NEXT:    ands r1, r0
 ; THUMB6-NEXT:    rsbs r0, r1, #0
 ; THUMB6-NEXT:    adcs r0, r1
 ; THUMB6-NEXT:    bx lr
@@ -197,12 +174,9 @@ define i1 @scalar_i16_lowestbit_eq(i16 %x, i16 %y) nounwind {
 ; THUMB78-LABEL: scalar_i16_lowestbit_eq:
 ; THUMB78:       @ %bb.0:
 ; THUMB78-NEXT:    uxth r1, r1
-; THUMB78-NEXT:    movs r2, #1
-; THUMB78-NEXT:    lsr.w r1, r2, r1
-; THUMB78-NEXT:    ands r0, r1
-; THUMB78-NEXT:    uxth r0, r0
-; THUMB78-NEXT:    clz r0, r0
-; THUMB78-NEXT:    lsrs r0, r0, #5
+; THUMB78-NEXT:    lsls r0, r1
+; THUMB78-NEXT:    movs r1, #1
+; THUMB78-NEXT:    bic.w r0, r1, r0
 ; THUMB78-NEXT:    bx lr
   %t0 = lshr i16 1, %y
   %t1 = and i16 %t0, %x
@@ -215,8 +189,7 @@ define i1 @scalar_i16_bitsinmiddle_eq(i16 %x, i16 %y) nounwind {
 ; ARM:       @ %bb.0:
 ; ARM-NEXT:    uxth r1, r1
 ; ARM-NEXT:    mov r2, #4080
-; ARM-NEXT:    and r0, r0, r2, lsr r1
-; ARM-NEXT:    uxth r0, r0
+; ARM-NEXT:    and r0, r2, r0, lsl r1
 ; ARM-NEXT:    clz r0, r0
 ; ARM-NEXT:    lsr r0, r0, #5
 ; ARM-NEXT:    bx lr
@@ -224,11 +197,10 @@ define i1 @scalar_i16_bitsinmiddle_eq(i16 %x, i16 %y) nounwind {
 ; THUMB6-LABEL: scalar_i16_bitsinmiddle_eq:
 ; THUMB6:       @ %bb.0:
 ; THUMB6-NEXT:    uxth r1, r1
-; THUMB6-NEXT:    movs r2, #255
-; THUMB6-NEXT:    lsls r2, r2, #4
-; THUMB6-NEXT:    lsrs r2, r1
-; THUMB6-NEXT:    ands r2, r0
-; THUMB6-NEXT:    uxth r1, r2
+; THUMB6-NEXT:    lsls r0, r1
+; THUMB6-NEXT:    movs r1, #255
+; THUMB6-NEXT:    lsls r1, r1, #4
+; THUMB6-NEXT:    ands r1, r0
 ; THUMB6-NEXT:    rsbs r0, r1, #0
 ; THUMB6-NEXT:    adcs r0, r1
 ; THUMB6-NEXT:    bx lr
@@ -236,10 +208,8 @@ define i1 @scalar_i16_bitsinmiddle_eq(i16 %x, i16 %y) nounwind {
 ; THUMB78-LABEL: scalar_i16_bitsinmiddle_eq:
 ; THUMB78:       @ %bb.0:
 ; THUMB78-NEXT:    uxth r1, r1
-; THUMB78-NEXT:    mov.w r2, #4080
-; THUMB78-NEXT:    lsr.w r1, r2, r1
-; THUMB78-NEXT:    ands r0, r1
-; THUMB78-NEXT:    uxth r0, r0
+; THUMB78-NEXT:    lsls r0, r1
+; THUMB78-NEXT:    and r0, r0, #4080
 ; THUMB78-NEXT:    clz r0, r0
 ; THUMB78-NEXT:    lsrs r0, r0, #5
 ; THUMB78-NEXT:    bx lr
@@ -254,29 +224,25 @@ define i1 @scalar_i16_bitsinmiddle_eq(i16 %x, i16 %y) nounwind {
 define i1 @scalar_i32_signbit_eq(i32 %x, i32 %y) nounwind {
 ; ARM-LABEL: scalar_i32_signbit_eq:
 ; ARM:       @ %bb.0:
-; ARM-NEXT:    mov r2, #-2147483648
-; ARM-NEXT:    and r0, r0, r2, lsr r1
-; ARM-NEXT:    clz r0, r0
-; ARM-NEXT:    lsr r0, r0, #5
+; ARM-NEXT:    mvn r0, r0, lsl r1
+; ARM-NEXT:    lsr r0, r0, #31
 ; ARM-NEXT:    bx lr
 ;
 ; THUMB6-LABEL: scalar_i32_signbit_eq:
 ; THUMB6:       @ %bb.0:
-; THUMB6-NEXT:    movs r2, #1
-; THUMB6-NEXT:    lsls r2, r2, #31
-; THUMB6-NEXT:    lsrs r2, r1
-; THUMB6-NEXT:    ands r2, r0
-; THUMB6-NEXT:    rsbs r0, r2, #0
-; THUMB6-NEXT:    adcs r0, r2
+; THUMB6-NEXT:    lsls r0, r1
+; THUMB6-NEXT:    movs r1, #1
+; THUMB6-NEXT:    lsls r1, r1, #31
+; THUMB6-NEXT:    ands r1, r0
+; THUMB6-NEXT:    rsbs r0, r1, #0
+; THUMB6-NEXT:    adcs r0, r1
 ; THUMB6-NEXT:    bx lr
 ;
 ; THUMB78-LABEL: scalar_i32_signbit_eq:
 ; THUMB78:       @ %bb.0:
-; THUMB78-NEXT:    mov.w r2, #-2147483648
-; THUMB78-NEXT:    lsr.w r1, r2, r1
-; THUMB78-NEXT:    ands r0, r1
-; THUMB78-NEXT:    clz r0, r0
-; THUMB78-NEXT:    lsrs r0, r0, #5
+; THUMB78-NEXT:    lsls r0, r1
+; THUMB78-NEXT:    mvns r0, r0
+; THUMB78-NEXT:    lsrs r0, r0, #31
 ; THUMB78-NEXT:    bx lr
   %t0 = lshr i32 2147483648, %y
   %t1 = and i32 %t0, %x
@@ -288,27 +254,23 @@ define i1 @scalar_i32_lowestbit_eq(i32 %x, i32 %y) nounwind {
 ; ARM-LABEL: scalar_i32_lowestbit_eq:
 ; ARM:       @ %bb.0:
 ; ARM-NEXT:    mov r2, #1
-; ARM-NEXT:    and r0, r0, r2, lsr r1
-; ARM-NEXT:    clz r0, r0
-; ARM-NEXT:    lsr r0, r0, #5
+; ARM-NEXT:    bic r0, r2, r0, lsl r1
 ; ARM-NEXT:    bx lr
 ;
 ; THUMB6-LABEL: scalar_i32_lowestbit_eq:
 ; THUMB6:       @ %bb.0:
-; THUMB6-NEXT:    movs r2, #1
-; THUMB6-NEXT:    lsrs r2, r1
-; THUMB6-NEXT:    ands r2, r0
-; THUMB6-NEXT:    rsbs r0, r2, #0
-; THUMB6-NEXT:    adcs r0, r2
+; THUMB6-NEXT:    lsls r0, r1
+; THUMB6-NEXT:    movs r1, #1
+; THUMB6-NEXT:    ands r1, r0
+; THUMB6-NEXT:    rsbs r0, r1, #0
+; THUMB6-NEXT:    adcs r0, r1
 ; THUMB6-NEXT:    bx lr
 ;
 ; THUMB78-LABEL: scalar_i32_lowestbit_eq:
 ; THUMB78:       @ %bb.0:
-; THUMB78-NEXT:    movs r2, #1
-; THUMB78-NEXT:    lsr.w r1, r2, r1
-; THUMB78-NEXT:    ands r0, r1
-; THUMB78-NEXT:    clz r0, r0
-; THUMB78-NEXT:    lsrs r0, r0, #5
+; THUMB78-NEXT:    lsls r0, r1
+; THUMB78-NEXT:    movs r1, #1
+; THUMB78-NEXT:    bic.w r0, r1, r0
 ; THUMB78-NEXT:    bx lr
   %t0 = lshr i32 1, %y
   %t1 = and i32 %t0, %x
@@ -321,7 +283,7 @@ define i1 @scalar_i32_bitsinmiddle_eq(i32 %x, i32 %y) nounwind {
 ; ARM6:       @ %bb.0:
 ; ARM6-NEXT:    mov r2, #65280
 ; ARM6-NEXT:    orr r2, r2, #16711680
-; ARM6-NEXT:    and r0, r0, r2, lsr r1
+; ARM6-NEXT:    and r0, r2, r0, lsl r1
 ; ARM6-NEXT:    clz r0, r0
 ; ARM6-NEXT:    lsr r0, r0, #5
 ; ARM6-NEXT:    bx lr
@@ -330,18 +292,18 @@ define i1 @scalar_i32_bitsinmiddle_eq(i32 %x, i32 %y) nounwind {
 ; ARM78:       @ %bb.0:
 ; ARM78-NEXT:    movw r2, #65280
 ; ARM78-NEXT:    movt r2, #255
-; ARM78-NEXT:    and r0, r0, r2, lsr r1
+; ARM78-NEXT:    and r0, r2, r0, lsl r1
 ; ARM78-NEXT:    clz r0, r0
 ; ARM78-NEXT:    lsr r0, r0, #5
 ; ARM78-NEXT:    bx lr
 ;
 ; THUMB6-LABEL: scalar_i32_bitsinmiddle_eq:
 ; THUMB6:       @ %bb.0:
-; THUMB6-NEXT:    ldr r2, .LCPI8_0
-; THUMB6-NEXT:    lsrs r2, r1
-; THUMB6-NEXT:    ands r2, r0
-; THUMB6-NEXT:    rsbs r0, r2, #0
-; THUMB6-NEXT:    adcs r0, r2
+; THUMB6-NEXT:    lsls r0, r1
+; THUMB6-NEXT:    ldr r1, .LCPI8_0
+; THUMB6-NEXT:    ands r1, r0
+; THUMB6-NEXT:    rsbs r0, r1, #0
+; THUMB6-NEXT:    adcs r0, r1
 ; THUMB6-NEXT:    bx lr
 ; THUMB6-NEXT:    .p2align 2
 ; THUMB6-NEXT:  @ %bb.1:
@@ -350,9 +312,9 @@ define i1 @scalar_i32_bitsinmiddle_eq(i32 %x, i32 %y) nounwind {
 ;
 ; THUMB78-LABEL: scalar_i32_bitsinmiddle_eq:
 ; THUMB78:       @ %bb.0:
-; THUMB78-NEXT:    movw r2, #65280
-; THUMB78-NEXT:    movt r2, #255
-; THUMB78-NEXT:    lsr.w r1, r2, r1
+; THUMB78-NEXT:    lsls r0, r1
+; THUMB78-NEXT:    movw r1, #65280
+; THUMB78-NEXT:    movt r1, #255
 ; THUMB78-NEXT:    ands r0, r1
 ; THUMB78-NEXT:    clz r0, r0
 ; THUMB78-NEXT:    lsrs r0, r0, #5
@@ -366,95 +328,54 @@ define i1 @scalar_i32_bitsinmiddle_eq(i32 %x, i32 %y) nounwind {
 ; i64 scalar
 
 define i1 @scalar_i64_signbit_eq(i64 %x, i64 %y) nounwind {
-; ARM6-LABEL: scalar_i64_signbit_eq:
-; ARM6:       @ %bb.0:
-; ARM6-NEXT:    push {r11, lr}
-; ARM6-NEXT:    mov r12, #-2147483648
-; ARM6-NEXT:    subs lr, r2, #32
-; ARM6-NEXT:    lsr r3, r12, r2
-; ARM6-NEXT:    rsb r2, r2, #32
-; ARM6-NEXT:    movpl r3, #0
-; ARM6-NEXT:    and r1, r3, r1
-; ARM6-NEXT:    lsl r2, r12, r2
-; ARM6-NEXT:    lsrpl r2, r12, lr
-; ARM6-NEXT:    and r0, r2, r0
-; ARM6-NEXT:    orr r0, r0, r1
-; ARM6-NEXT:    clz r0, r0
-; ARM6-NEXT:    lsr r0, r0, #5
-; ARM6-NEXT:    pop {r11, pc}
-;
-; ARM78-LABEL: scalar_i64_signbit_eq:
-; ARM78:       @ %bb.0:
-; ARM78-NEXT:    push {r11, lr}
-; ARM78-NEXT:    mov r12, #-2147483648
-; ARM78-NEXT:    subs lr, r2, #32
-; ARM78-NEXT:    lsr r3, r12, r2
-; ARM78-NEXT:    rsb r2, r2, #32
-; ARM78-NEXT:    movwpl r3, #0
-; ARM78-NEXT:    and r1, r3, r1
-; ARM78-NEXT:    lsl r2, r12, r2
-; ARM78-NEXT:    lsrpl r2, r12, lr
-; ARM78-NEXT:    and r0, r2, r0
-; ARM78-NEXT:    orr r0, r0, r1
-; ARM78-NEXT:    clz r0, r0
-; ARM78-NEXT:    lsr r0, r0, #5
-; ARM78-NEXT:    pop {r11, pc}
+; ARM-LABEL: scalar_i64_signbit_eq:
+; ARM:       @ %bb.0:
+; ARM-NEXT:    rsb r3, r2, #32
+; ARM-NEXT:    lsr r3, r0, r3
+; ARM-NEXT:    orr r1, r3, r1, lsl r2
+; ARM-NEXT:    subs r2, r2, #32
+; ARM-NEXT:    lslpl r1, r0, r2
+; ARM-NEXT:    mvn r0, r1
+; ARM-NEXT:    lsr r0, r0, #31
+; ARM-NEXT:    bx lr
 ;
 ; THUMB6-LABEL: scalar_i64_signbit_eq:
 ; THUMB6:       @ %bb.0:
-; THUMB6-NEXT:    push {r4, r5, r7, lr}
-; THUMB6-NEXT:    mov r4, r1
-; THUMB6-NEXT:    mov r5, r0
+; THUMB6-NEXT:    push {r7, lr}
+; THUMB6-NEXT:    bl __ashldi3
 ; THUMB6-NEXT:    movs r0, #1
-; THUMB6-NEXT:    lsls r1, r0, #31
-; THUMB6-NEXT:    movs r0, #0
-; THUMB6-NEXT:    bl __lshrdi3
-; THUMB6-NEXT:    ands r1, r4
-; THUMB6-NEXT:    ands r0, r5
-; THUMB6-NEXT:    orrs r0, r1
-; THUMB6-NEXT:    rsbs r1, r0, #0
-; THUMB6-NEXT:    adcs r0, r1
-; THUMB6-NEXT:    pop {r4, r5, r7, pc}
+; THUMB6-NEXT:    lsls r2, r0, #31
+; THUMB6-NEXT:    ands r2, r1
+; THUMB6-NEXT:    rsbs r0, r2, #0
+; THUMB6-NEXT:    adcs r0, r2
+; THUMB6-NEXT:    pop {r7, pc}
 ;
 ; THUMB7-LABEL: scalar_i64_signbit_eq:
 ; THUMB7:       @ %bb.0:
-; THUMB7-NEXT:    push {r7, lr}
 ; THUMB7-NEXT:    rsb.w r3, r2, #32
-; THUMB7-NEXT:    mov.w r12, #-2147483648
-; THUMB7-NEXT:    subs.w lr, r2, #32
-; THUMB7-NEXT:    lsr.w r2, r12, r2
-; THUMB7-NEXT:    lsl.w r3, r12, r3
-; THUMB7-NEXT:    it pl
-; THUMB7-NEXT:    lsrpl.w r3, r12, lr
+; THUMB7-NEXT:    lsls r1, r2
+; THUMB7-NEXT:    subs r2, #32
+; THUMB7-NEXT:    lsr.w r3, r0, r3
+; THUMB7-NEXT:    orr.w r1, r1, r3
 ; THUMB7-NEXT:    it pl
-; THUMB7-NEXT:    movpl r2, #0
-; THUMB7-NEXT:    ands r0, r3
-; THUMB7-NEXT:    ands r1, r2
-; THUMB7-NEXT:    orrs r0, r1
-; THUMB7-NEXT:    clz r0, r0
-; THUMB7-NEXT:    lsrs r0, r0, #5
-; THUMB7-NEXT:    pop {r7, pc}
+; THUMB7-NEXT:    lslpl.w r1, r0, r2
+; THUMB7-NEXT:    mvns r0, r1
+; THUMB7-NEXT:    lsrs r0, r0, #31
+; THUMB7-NEXT:    bx lr
 ;
 ; THUMB8-LABEL: scalar_i64_signbit_eq:
 ; THUMB8:       @ %bb.0:
-; THUMB8-NEXT:    .save {r7, lr}
-; THUMB8-NEXT:    push {r7, lr}
-; THUMB8-NEXT:    subs.w r3, r2, #32
-; THUMB8-NEXT:    mov.w r12, #-2147483648
-; THUMB8-NEXT:    lsr.w lr, r12, r3
 ; THUMB8-NEXT:    rsb.w r3, r2, #32
-; THUMB8-NEXT:    lsr.w r2, r12, r2
-; THUMB8-NEXT:    lsl.w r3, r12, r3
-; THUMB8-NEXT:    it pl
-; THUMB8-NEXT:    movpl r3, lr
-; THUMB8-NEXT:    it pl
-; THUMB8-NEXT:    movpl r2, #0
-; THUMB8-NEXT:    ands r0, r3
-; THUMB8-NEXT:    ands r1, r2
-; THUMB8-NEXT:    orrs r0, r1
-; THUMB8-NEXT:    clz r0, r0
-; THUMB8-NEXT:    lsrs r0, r0, #5
-; THUMB8-NEXT:    pop {r7, pc}
+; THUMB8-NEXT:    lsls r1, r2
+; THUMB8-NEXT:    lsr.w r3, r0, r3
+; THUMB8-NEXT:    orrs r1, r3
+; THUMB8-NEXT:    subs r2, #32
+; THUMB8-NEXT:    lsl.w r0, r0, r2
+; THUMB8-NEXT:    it mi
+; THUMB8-NEXT:    movmi r0, r1
+; THUMB8-NEXT:    mvns r0, r0
+; THUMB8-NEXT:    lsrs r0, r0, #31
+; THUMB8-NEXT:    bx lr
   %t0 = lshr i64 9223372036854775808, %y
   %t1 = and i64 %t0, %x
   %res = icmp eq i64 %t1, 0
@@ -464,51 +385,40 @@ define i1 @scalar_i64_signbit_eq(i64 %x, i64 %y) nounwind {
 define i1 @scalar_i64_lowestbit_eq(i64 %x, i64 %y) nounwind {
 ; ARM6-LABEL: scalar_i64_lowestbit_eq:
 ; ARM6:       @ %bb.0:
+; ARM6-NEXT:    subs r1, r2, #32
+; ARM6-NEXT:    lsl r0, r0, r2
+; ARM6-NEXT:    movpl r0, #0
 ; ARM6-NEXT:    mov r1, #1
-; ARM6-NEXT:    lsr r1, r1, r2
-; ARM6-NEXT:    subs r2, r2, #32
-; ARM6-NEXT:    movpl r1, #0
-; ARM6-NEXT:    and r0, r1, r0
-; ARM6-NEXT:    clz r0, r0
-; ARM6-NEXT:    lsr r0, r0, #5
+; ARM6-NEXT:    bic r0, r1, r0
 ; ARM6-NEXT:    bx lr
 ;
 ; ARM78-LABEL: scalar_i64_lowestbit_eq:
 ; ARM78:       @ %bb.0:
+; ARM78-NEXT:    subs r1, r2, #32
+; ARM78-NEXT:    lsl r0, r0, r2
+; ARM78-NEXT:    movwpl r0, #0
 ; ARM78-NEXT:    mov r1, #1
-; ARM78-NEXT:    lsr r1, r1, r2
-; ARM78-NEXT:    subs r2, r2, #32
-; ARM78-NEXT:    movwpl r1, #0
-; ARM78-NEXT:    and r0, r1, r0
-; ARM78-NEXT:    clz r0, r0
-; ARM78-NEXT:    lsr r0, r0, #5
+; ARM78-NEXT:    bic r0, r1, r0
 ; ARM78-NEXT:    bx lr
 ;
 ; THUMB6-LABEL: scalar_i64_lowestbit_eq:
 ; THUMB6:       @ %bb.0:
-; THUMB6-NEXT:    push {r4, r5, r7, lr}
-; THUMB6-NEXT:    mov r4, r1
-; THUMB6-NEXT:    mov r5, r0
-; THUMB6-NEXT:    movs r0, #1
-; THUMB6-NEXT:    movs r1, #0
-; THUMB6-NEXT:    bl __lshrdi3
-; THUMB6-NEXT:    ands r1, r4
-; THUMB6-NEXT:    ands r0, r5
-; THUMB6-NEXT:    orrs r0, r1
-; THUMB6-NEXT:    rsbs r1, r0, #0
+; THUMB6-NEXT:    push {r7, lr}
+; THUMB6-NEXT:    bl __ashldi3
+; THUMB6-NEXT:    movs r1, #1
+; THUMB6-NEXT:    ands r1, r0
+; THUMB6-NEXT:    rsbs r0, r1, #0
 ; THUMB6-NEXT:    adcs r0, r1
-; THUMB6-NEXT:    pop {r4, r5, r7, pc}
+; THUMB6-NEXT:    pop {r7, pc}
 ;
 ; THUMB78-LABEL: scalar_i64_lowestbit_eq:
 ; THUMB78:       @ %bb.0:
-; THUMB78-NEXT:    movs r1, #1
-; THUMB78-NEXT:    lsrs r1, r2
-; THUMB78-NEXT:    subs r2, #32
+; THUMB78-NEXT:    lsls r0, r2
+; THUMB78-NEXT:    subs.w r1, r2, #32
 ; THUMB78-NEXT:    it pl
-; THUMB78-NEXT:    movpl r1, #0
-; THUMB78-NEXT:    ands r0, r1
-; THUMB78-NEXT:    clz r0, r0
-; THUMB78-NEXT:    lsrs r0, r0, #5
+; THUMB78-NEXT:    movpl r0, #0
+; THUMB78-NEXT:    movs r1, #1
+; THUMB78-NEXT:    bic.w r0, r1, r0
 ; THUMB78-NEXT:    bx lr
   %t0 = lshr i64 1, %y
   %t1 = and i64 %t0, %x
@@ -519,115 +429,82 @@ define i1 @scalar_i64_lowestbit_eq(i64 %x, i64 %y) nounwind {
 define i1 @scalar_i64_bitsinmiddle_eq(i64 %x, i64 %y) nounwind {
 ; ARM6-LABEL: scalar_i64_bitsinmiddle_eq:
 ; ARM6:       @ %bb.0:
-; ARM6-NEXT:    push {r11, lr}
-; ARM6-NEXT:    mov r12, #255
-; ARM6-NEXT:    subs lr, r2, #32
-; ARM6-NEXT:    orr r12, r12, #65280
-; ARM6-NEXT:    lsr r3, r12, r2
-; ARM6-NEXT:    movpl r3, #0
-; ARM6-NEXT:    and r1, r3, r1
-; ARM6-NEXT:    mov r3, #16711680
-; ARM6-NEXT:    cmp lr, #0
-; ARM6-NEXT:    orr r3, r3, #-16777216
-; ARM6-NEXT:    lsr r3, r3, r2
-; ARM6-NEXT:    rsb r2, r2, #32
-; ARM6-NEXT:    orr r2, r3, r12, lsl r2
-; ARM6-NEXT:    lsrpl r2, r12, lr
-; ARM6-NEXT:    and r0, r2, r0
-; ARM6-NEXT:    orr r0, r0, r1
+; ARM6-NEXT:    rsb r3, r2, #32
+; ARM6-NEXT:    lsr r3, r0, r3
+; ARM6-NEXT:    orr r1, r3, r1, lsl r2
+; ARM6-NEXT:    subs r3, r2, #32
+; ARM6-NEXT:    lslpl r1, r0, r3
+; ARM6-NEXT:    lsl r0, r0, r2
+; ARM6-NEXT:    movpl r0, #0
+; ARM6-NEXT:    pkhbt r0, r1, r0
 ; ARM6-NEXT:    clz r0, r0
 ; ARM6-NEXT:    lsr r0, r0, #5
-; ARM6-NEXT:    pop {r11, pc}
+; ARM6-NEXT:    bx lr
 ;
 ; ARM78-LABEL: scalar_i64_bitsinmiddle_eq:
 ; ARM78:       @ %bb.0:
-; ARM78-NEXT:    push {r11, lr}
-; ARM78-NEXT:    movw r12, #65535
-; ARM78-NEXT:    subs lr, r2, #32
-; ARM78-NEXT:    lsr r3, r12, r2
-; ARM78-NEXT:    movwpl r3, #0
-; ARM78-NEXT:    and r1, r3, r1
-; ARM78-NEXT:    movw r3, #0
-; ARM78-NEXT:    cmp lr, #0
-; ARM78-NEXT:    movt r3, #65535
-; ARM78-NEXT:    lsr r3, r3, r2
-; ARM78-NEXT:    rsb r2, r2, #32
-; ARM78-NEXT:    orr r2, r3, r12, lsl r2
-; ARM78-NEXT:    lsrpl r2, r12, lr
-; ARM78-NEXT:    and r0, r2, r0
-; ARM78-NEXT:    orr r0, r0, r1
+; ARM78-NEXT:    rsb r3, r2, #32
+; ARM78-NEXT:    lsr r3, r0, r3
+; ARM78-NEXT:    orr r1, r3, r1, lsl r2
+; ARM78-NEXT:    subs r3, r2, #32
+; ARM78-NEXT:    lslpl r1, r0, r3
+; ARM78-NEXT:    lsl r0, r0, r2
+; ARM78-NEXT:    movwpl r0, #0
+; ARM78-NEXT:    pkhbt r0, r1, r0
 ; ARM78-NEXT:    clz r0, r0
 ; ARM78-NEXT:    lsr r0, r0, #5
-; ARM78-NEXT:    pop {r11, pc}
+; ARM78-NEXT:    bx lr
 ;
 ; THUMB6-LABEL: scalar_i64_bitsinmiddle_eq:
 ; THUMB6:       @ %bb.0:
-; THUMB6-NEXT:    push {r4, r5, r7, lr}
-; THUMB6-NEXT:    mov r4, r1
-; THUMB6-NEXT:    mov r5, r0
-; THUMB6-NEXT:    ldr r0, .LCPI11_0
-; THUMB6-NEXT:    ldr r1, .LCPI11_1
-; THUMB6-NEXT:    bl __lshrdi3
-; THUMB6-NEXT:    ands r1, r4
-; THUMB6-NEXT:    ands r0, r5
-; THUMB6-NEXT:    orrs r0, r1
-; THUMB6-NEXT:    rsbs r1, r0, #0
+; THUMB6-NEXT:    push {r7, lr}
+; THUMB6-NEXT:    bl __ashldi3
+; THUMB6-NEXT:    ldr r2, .LCPI11_0
+; THUMB6-NEXT:    ands r2, r0
+; THUMB6-NEXT:    uxth r0, r1
+; THUMB6-NEXT:    adds r1, r2, r0
+; THUMB6-NEXT:    rsbs r0, r1, #0
 ; THUMB6-NEXT:    adcs r0, r1
-; THUMB6-NEXT:    pop {r4, r5, r7, pc}
+; THUMB6-NEXT:    pop {r7, pc}
 ; THUMB6-NEXT:    .p2align 2
 ; THUMB6-NEXT:  @ %bb.1:
 ; THUMB6-NEXT:  .LCPI11_0:
 ; THUMB6-NEXT:    .long 4294901760 @ 0xffff0000
-; THUMB6-NEXT:  .LCPI11_1:
-; THUMB6-NEXT:    .long 65535 @ 0xffff
 ;
 ; THUMB7-LABEL: scalar_i64_bitsinmiddle_eq:
 ; THUMB7:       @ %bb.0:
-; THUMB7-NEXT:    push {r7, lr}
-; THUMB7-NEXT:    movs r3, #0
-; THUMB7-NEXT:    movw lr, #65535
-; THUMB7-NEXT:    movt r3, #65535
-; THUMB7-NEXT:    lsr.w r12, r3, r2
 ; THUMB7-NEXT:    rsb.w r3, r2, #32
-; THUMB7-NEXT:    lsl.w r3, lr, r3
-; THUMB7-NEXT:    orr.w r12, r12, r3
+; THUMB7-NEXT:    lsls r1, r2
+; THUMB7-NEXT:    lsr.w r3, r0, r3
+; THUMB7-NEXT:    orrs r1, r3
 ; THUMB7-NEXT:    subs.w r3, r2, #32
-; THUMB7-NEXT:    lsr.w r2, lr, r2
 ; THUMB7-NEXT:    it pl
-; THUMB7-NEXT:    lsrpl.w r12, lr, r3
+; THUMB7-NEXT:    lslpl.w r1, r0, r3
+; THUMB7-NEXT:    lsl.w r0, r0, r2
 ; THUMB7-NEXT:    it pl
-; THUMB7-NEXT:    movpl r2, #0
-; THUMB7-NEXT:    and.w r0, r0, r12
-; THUMB7-NEXT:    ands r1, r2
-; THUMB7-NEXT:    orrs r0, r1
+; THUMB7-NEXT:    movpl r0, #0
+; THUMB7-NEXT:    pkhbt r0, r1, r0
 ; THUMB7-NEXT:    clz r0, r0
 ; THUMB7-NEXT:    lsrs r0, r0, #5
-; THUMB7-NEXT:    pop {r7, pc}
+; THUMB7-NEXT:    bx lr
 ;
 ; THUMB8-LABEL: scalar_i64_bitsinmiddle_eq:
 ; THUMB8:       @ %bb.0:
-; THUMB8-NEXT:    .save {r7, lr}
-; THUMB8-NEXT:    push {r7, lr}
-; THUMB8-NEXT:    movs r3, #0
-; THUMB8-NEXT:    movw lr, #65535
-; THUMB8-NEXT:    movt r3, #65535
-; THUMB8-NEXT:    lsr.w r12, r3, r2
 ; THUMB8-NEXT:    rsb.w r3, r2, #32
-; THUMB8-NEXT:    lsl.w r3, lr, r3
-; THUMB8-NEXT:    orr.w r12, r12, r3
+; THUMB8-NEXT:    lsls r1, r2
+; THUMB8-NEXT:    lsr.w r3, r0, r3
+; THUMB8-NEXT:    orrs r1, r3
 ; THUMB8-NEXT:    subs.w r3, r2, #32
-; THUMB8-NEXT:    lsr.w r2, lr, r2
-; THUMB8-NEXT:    lsr.w r3, lr, r3
+; THUMB8-NEXT:    lsl.w r3, r0, r3
+; THUMB8-NEXT:    lsl.w r0, r0, r2
 ; THUMB8-NEXT:    it mi
-; THUMB8-NEXT:    movmi r3, r12
+; THUMB8-NEXT:    movmi r3, r1
 ; THUMB8-NEXT:    it pl
-; THUMB8-NEXT:    movpl r2, #0
-; THUMB8-NEXT:    ands r0, r3
-; THUMB8-NEXT:    ands r1, r2
-; THUMB8-NEXT:    orrs r0, r1
+; THUMB8-NEXT:    movpl r0, #0
+; THUMB8-NEXT:    pkhbt r0, r3, r0
 ; THUMB8-NEXT:    clz r0, r0
 ; THUMB8-NEXT:    lsrs r0, r0, #5
-; THUMB8-NEXT:    pop {r7, pc}
+; THUMB8-NEXT:    bx lr
   %t0 = lshr i64 281474976645120, %y
   %t1 = and i64 %t0, %x
   %res = icmp eq i64 %t1, 0
@@ -644,33 +521,24 @@ define <4 x i1> @vec_4xi32_splat_eq(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; ARM6-NEXT:    push {r11, lr}
 ; ARM6-NEXT:    ldr r12, [sp, #8]
 ; ARM6-NEXT:    mov lr, #1
-; ARM6-NEXT:    and r0, r0, lr, lsr r12
+; ARM6-NEXT:    bic r0, lr, r0, lsl r12
 ; ARM6-NEXT:    ldr r12, [sp, #12]
-; ARM6-NEXT:    clz r0, r0
-; ARM6-NEXT:    and r1, r1, lr, lsr r12
+; ARM6-NEXT:    bic r1, lr, r1, lsl r12
 ; ARM6-NEXT:    ldr r12, [sp, #16]
-; ARM6-NEXT:    clz r1, r1
-; ARM6-NEXT:    lsr r0, r0, #5
-; ARM6-NEXT:    and r2, r2, lr, lsr r12
+; ARM6-NEXT:    bic r2, lr, r2, lsl r12
 ; ARM6-NEXT:    ldr r12, [sp, #20]
-; ARM6-NEXT:    clz r2, r2
-; ARM6-NEXT:    lsr r1, r1, #5
-; ARM6-NEXT:    and r3, r3, lr, lsr r12
-; ARM6-NEXT:    lsr r2, r2, #5
-; ARM6-NEXT:    clz r3, r3
-; ARM6-NEXT:    lsr r3, r3, #5
+; ARM6-NEXT:    bic r3, lr, r3, lsl r12
 ; ARM6-NEXT:    pop {r11, pc}
 ;
 ; ARM78-LABEL: vec_4xi32_splat_eq:
 ; ARM78:       @ %bb.0:
+; ARM78-NEXT:    vmov d17, r2, r3
 ; ARM78-NEXT:    mov r12, sp
-; ARM78-NEXT:    vld1.64 {d16, d17}, [r12]
-; ARM78-NEXT:    vmov.i32 q9, #0x1
-; ARM78-NEXT:    vneg.s32 q8, q8
-; ARM78-NEXT:    vshl.u32 q8, q9, q8
-; ARM78-NEXT:    vmov d19, r2, r3
-; ARM78-NEXT:    vmov d18, r0, r1
-; ARM78-NEXT:    vtst.32 q8, q8, q9
+; ARM78-NEXT:    vld1.64 {d18, d19}, [r12]
+; ARM78-NEXT:    vmov d16, r0, r1
+; ARM78-NEXT:    vmov.i32 q10, #0x1
+; ARM78-NEXT:    vshl.u32 q8, q8, q9
+; ARM78-NEXT:    vtst.32 q8, q8, q10
 ; ARM78-NEXT:    vmvn q8, q8
 ; ARM78-NEXT:    vmovn.i32 d16, q8
 ; ARM78-NEXT:    vmov r0, r1, d16
@@ -678,43 +546,39 @@ define <4 x i1> @vec_4xi32_splat_eq(<4 x i32> %x, <4 x i32> %y) nounwind {
 ;
 ; THUMB6-LABEL: vec_4xi32_splat_eq:
 ; THUMB6:       @ %bb.0:
-; THUMB6-NEXT:    push {r4, r5, r6, lr}
-; THUMB6-NEXT:    ldr r5, [sp, #16]
+; THUMB6-NEXT:    push {r4, r5, r7, lr}
+; THUMB6-NEXT:    ldr r4, [sp, #16]
+; THUMB6-NEXT:    lsls r0, r4
 ; THUMB6-NEXT:    movs r4, #1
-; THUMB6-NEXT:    mov r6, r4
-; THUMB6-NEXT:    lsrs r6, r5
-; THUMB6-NEXT:    ands r6, r0
-; THUMB6-NEXT:    rsbs r0, r6, #0
-; THUMB6-NEXT:    adcs r0, r6
+; THUMB6-NEXT:    ands r0, r4
+; THUMB6-NEXT:    rsbs r5, r0, #0
+; THUMB6-NEXT:    adcs r0, r5
 ; THUMB6-NEXT:    ldr r5, [sp, #20]
-; THUMB6-NEXT:    mov r6, r4
-; THUMB6-NEXT:    lsrs r6, r5
-; THUMB6-NEXT:    ands r6, r1
-; THUMB6-NEXT:    rsbs r1, r6, #0
-; THUMB6-NEXT:    adcs r1, r6
+; THUMB6-NEXT:    lsls r1, r5
+; THUMB6-NEXT:    ands r1, r4
+; THUMB6-NEXT:    rsbs r5, r1, #0
+; THUMB6-NEXT:    adcs r1, r5
 ; THUMB6-NEXT:    ldr r5, [sp, #24]
-; THUMB6-NEXT:    mov r6, r4
-; THUMB6-NEXT:    lsrs r6, r5
-; THUMB6-NEXT:    ands r6, r2
-; THUMB6-NEXT:    rsbs r2, r6, #0
-; THUMB6-NEXT:    adcs r2, r6
+; THUMB6-NEXT:    lsls r2, r5
+; THUMB6-NEXT:    ands r2, r4
+; THUMB6-NEXT:    rsbs r5, r2, #0
+; THUMB6-NEXT:    adcs r2, r5
 ; THUMB6-NEXT:    ldr r5, [sp, #28]
-; THUMB6-NEXT:    lsrs r4, r5
-; THUMB6-NEXT:    ands r4, r3
-; THUMB6-NEXT:    rsbs r3, r4, #0
+; THUMB6-NEXT:    lsls r3, r5
+; THUMB6-NEXT:    ands r3, r4
+; THUMB6-NEXT:    rsbs r4, r3, #0
 ; THUMB6-NEXT:    adcs r3, r4
-; THUMB6-NEXT:    pop {r4, r5, r6, pc}
+; THUMB6-NEXT:    pop {r4, r5, r7, pc}
 ;
 ; THUMB78-LABEL: vec_4xi32_splat_eq:
 ; THUMB78:       @ %bb.0:
+; THUMB78-NEXT:    vmov d17, r2, r3
 ; THUMB78-NEXT:    mov r12, sp
-; THUMB78-NEXT:    vld1.64 {d16, d17}, [r12]
-; THUMB78-NEXT:    vmov.i32 q9, #0x1
-; THUMB78-NEXT:    vneg.s32 q8, q8
-; THUMB78-NEXT:    vshl.u32 q8, q9, q8
-; THUMB78-NEXT:    vmov d19, r2, r3
-; THUMB78-NEXT:    vmov d18, r0, r1
-; THUMB78-NEXT:    vtst.32 q8, q8, q9
+; THUMB78-NEXT:    vld1.64 {d18, d19}, [r12]
+; THUMB78-NEXT:    vmov d16, r0, r1
+; THUMB78-NEXT:    vmov.i32 q10, #0x1
+; THUMB78-NEXT:    vshl.u32 q8, q8, q9
+; THUMB78-NEXT:    vtst.32 q8, q8, q10
 ; THUMB78-NEXT:    vmvn q8, q8
 ; THUMB78-NEXT:    vmovn.i32 d16, q8
 ; THUMB78-NEXT:    vmov r0, r1, d16
@@ -730,20 +594,16 @@ define <4 x i1> @vec_4xi32_nonsplat_eq(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; ARM6:       @ %bb.0:
 ; ARM6-NEXT:    ldr r12, [sp, #4]
 ; ARM6-NEXT:    mov r0, #1
-; ARM6-NEXT:    and r0, r1, r0, lsr r12
+; ARM6-NEXT:    bic r1, r0, r1, lsl r12
 ; ARM6-NEXT:    ldr r12, [sp, #8]
-; ARM6-NEXT:    clz r0, r0
-; ARM6-NEXT:    lsr r1, r0, #5
 ; ARM6-NEXT:    mov r0, #65280
 ; ARM6-NEXT:    orr r0, r0, #16711680
-; ARM6-NEXT:    and r0, r2, r0, lsr r12
-; ARM6-NEXT:    ldr r12, [sp, #12]
+; ARM6-NEXT:    and r0, r0, r2, lsl r12
 ; ARM6-NEXT:    clz r0, r0
 ; ARM6-NEXT:    lsr r2, r0, #5
-; ARM6-NEXT:    mov r0, #-2147483648
-; ARM6-NEXT:    and r0, r3, r0, lsr r12
-; ARM6-NEXT:    clz r0, r0
-; ARM6-NEXT:    lsr r3, r0, #5
+; ARM6-NEXT:    ldr r0, [sp, #12]
+; ARM6-NEXT:    mvn r0, r3, lsl r0
+; ARM6-NEXT:    lsr r3, r0, #31
 ; ARM6-NEXT:    mov r0, #1
 ; ARM6-NEXT:    bx lr
 ;
@@ -772,27 +632,26 @@ define <4 x i1> @vec_4xi32_nonsplat_eq(<4 x i32> %x, <4 x i32> %y) nounwind {
 ;
 ; THUMB6-LABEL: vec_4xi32_nonsplat_eq:
 ; THUMB6:       @ %bb.0:
-; THUMB6-NEXT:    push {r4, r5, r7, lr}
-; THUMB6-NEXT:    ldr r4, [sp, #20]
+; THUMB6-NEXT:    push {r4, lr}
+; THUMB6-NEXT:    ldr r0, [sp, #12]
+; THUMB6-NEXT:    lsls r1, r0
 ; THUMB6-NEXT:    movs r0, #1
-; THUMB6-NEXT:    mov r5, r0
-; THUMB6-NEXT:    lsrs r5, r4
-; THUMB6-NEXT:    ands r5, r1
-; THUMB6-NEXT:    rsbs r1, r5, #0
-; THUMB6-NEXT:    adcs r1, r5
-; THUMB6-NEXT:    ldr r4, [sp, #24]
-; THUMB6-NEXT:    ldr r5, .LCPI13_0
-; THUMB6-NEXT:    lsrs r5, r4
-; THUMB6-NEXT:    ands r5, r2
-; THUMB6-NEXT:    rsbs r2, r5, #0
-; THUMB6-NEXT:    adcs r2, r5
+; THUMB6-NEXT:    ands r1, r0
+; THUMB6-NEXT:    rsbs r4, r1, #0
+; THUMB6-NEXT:    adcs r1, r4
+; THUMB6-NEXT:    ldr r4, [sp, #16]
+; THUMB6-NEXT:    lsls r2, r4
+; THUMB6-NEXT:    ldr r4, .LCPI13_0
+; THUMB6-NEXT:    ands r4, r2
+; THUMB6-NEXT:    rsbs r2, r4, #0
+; THUMB6-NEXT:    adcs r2, r4
+; THUMB6-NEXT:    ldr r4, [sp, #20]
+; THUMB6-NEXT:    lsls r3, r4
 ; THUMB6-NEXT:    lsls r4, r0, #31
-; THUMB6-NEXT:    ldr r5, [sp, #28]
-; THUMB6-NEXT:    lsrs r4, r5
 ; THUMB6-NEXT:    ands r4, r3
 ; THUMB6-NEXT:    rsbs r3, r4, #0
 ; THUMB6-NEXT:    adcs r3, r4
-; THUMB6-NEXT:    pop {r4, r5, r7, pc}
+; THUMB6-NEXT:    pop {r4, pc}
 ; THUMB6-NEXT:    .p2align 2
 ; THUMB6-NEXT:  @ %bb.1:
 ; THUMB6-NEXT:  .LCPI13_0:
@@ -833,29 +692,22 @@ define <4 x i1> @vec_4xi32_nonsplat_undef0_eq(<4 x i32> %x, <4 x i32> %y) nounwi
 ; ARM6-NEXT:    ldr r2, [sp, #12]
 ; ARM6-NEXT:    mov lr, #1
 ; ARM6-NEXT:    ldr r12, [sp, #8]
-; ARM6-NEXT:    and r1, r1, lr, lsr r2
+; ARM6-NEXT:    bic r1, lr, r1, lsl r2
 ; ARM6-NEXT:    ldr r2, [sp, #20]
-; ARM6-NEXT:    and r0, r0, lr, lsr r12
-; ARM6-NEXT:    clz r1, r1
-; ARM6-NEXT:    clz r0, r0
-; ARM6-NEXT:    and r2, r3, lr, lsr r2
-; ARM6-NEXT:    lsr r1, r1, #5
-; ARM6-NEXT:    clz r2, r2
-; ARM6-NEXT:    lsr r0, r0, #5
-; ARM6-NEXT:    lsr r3, r2, #5
+; ARM6-NEXT:    bic r0, lr, r0, lsl r12
+; ARM6-NEXT:    bic r3, lr, r3, lsl r2
 ; ARM6-NEXT:    mov r2, #1
 ; ARM6-NEXT:    pop {r11, pc}
 ;
 ; ARM78-LABEL: vec_4xi32_nonsplat_undef0_eq:
 ; ARM78:       @ %bb.0:
+; ARM78-NEXT:    vmov d17, r2, r3
 ; ARM78-NEXT:    mov r12, sp
-; ARM78-NEXT:    vld1.64 {d16, d17}, [r12]
-; ARM78-NEXT:    vmov.i32 q9, #0x1
-; ARM78-NEXT:    vneg.s32 q8, q8
-; ARM78-NEXT:    vshl.u32 q8, q9, q8
-; ARM78-NEXT:    vmov d19, r2, r3
-; ARM78-NEXT:    vmov d18, r0, r1
-; ARM78-NEXT:    vtst.32 q8, q8, q9
+; ARM78-NEXT:    vld1.64 {d18, d19}, [r12]
+; ARM78-NEXT:    vmov d16, r0, r1
+; ARM78-NEXT:    vmov.i32 q10, #0x1
+; ARM78-NEXT:    vshl.u32 q8, q8, q9
+; ARM78-NEXT:    vtst.32 q8, q8, q10
 ; ARM78-NEXT:    vmvn q8, q8
 ; ARM78-NEXT:    vmovn.i32 d16, q8
 ; ARM78-NEXT:    vmov r0, r1, d16
@@ -863,38 +715,34 @@ define <4 x i1> @vec_4xi32_nonsplat_undef0_eq(<4 x i32> %x, <4 x i32> %y) nounwi
 ;
 ; THUMB6-LABEL: vec_4xi32_nonsplat_undef0_eq:
 ; THUMB6:       @ %bb.0:
-; THUMB6-NEXT:    push {r4, r5, r7, lr}
-; THUMB6-NEXT:    ldr r4, [sp, #16]
+; THUMB6-NEXT:    push {r4, lr}
+; THUMB6-NEXT:    ldr r2, [sp, #8]
+; THUMB6-NEXT:    lsls r0, r2
 ; THUMB6-NEXT:    movs r2, #1
-; THUMB6-NEXT:    mov r5, r2
-; THUMB6-NEXT:    lsrs r5, r4
-; THUMB6-NEXT:    ands r5, r0
-; THUMB6-NEXT:    rsbs r0, r5, #0
-; THUMB6-NEXT:    adcs r0, r5
+; THUMB6-NEXT:    ands r0, r2
+; THUMB6-NEXT:    rsbs r4, r0, #0
+; THUMB6-NEXT:    adcs r0, r4
+; THUMB6-NEXT:    ldr r4, [sp, #12]
+; THUMB6-NEXT:    lsls r1, r4
+; THUMB6-NEXT:    ands r1, r2
+; THUMB6-NEXT:    rsbs r4, r1, #0
+; THUMB6-NEXT:    adcs r1, r4
 ; THUMB6-NEXT:    ldr r4, [sp, #20]
-; THUMB6-NEXT:    mov r5, r2
-; THUMB6-NEXT:    lsrs r5, r4
-; THUMB6-NEXT:    ands r5, r1
-; THUMB6-NEXT:    rsbs r1, r5, #0
-; THUMB6-NEXT:    adcs r1, r5
-; THUMB6-NEXT:    ldr r4, [sp, #28]
-; THUMB6-NEXT:    mov r5, r2
-; THUMB6-NEXT:    lsrs r5, r4
-; THUMB6-NEXT:    ands r5, r3
-; THUMB6-NEXT:    rsbs r3, r5, #0
-; THUMB6-NEXT:    adcs r3, r5
-; THUMB6-NEXT:    pop {r4, r5, r7, pc}
+; THUMB6-NEXT:    lsls r3, r4
+; THUMB6-NEXT:    ands r3, r2
+; THUMB6-NEXT:    rsbs r4, r3, #0
+; THUMB6-NEXT:    adcs r3, r4
+; THUMB6-NEXT:    pop {r4, pc}
 ;
 ; THUMB78-LABEL: vec_4xi32_nonsplat_undef0_eq:
 ; THUMB78:       @ %bb.0:
+; THUMB78-NEXT:    vmov d17, r2, r3
 ; THUMB78-NEXT:    mov r12, sp
-; THUMB78-NEXT:    vld1.64 {d16, d17}, [r12]
-; THUMB78-NEXT:    vmov.i32 q9, #0x1
-; THUMB78-NEXT:    vneg.s32 q8, q8
-; THUMB78-NEXT:    vshl.u32 q8, q9, q8
-; THUMB78-NEXT:    vmov d19, r2, r3
-; THUMB78-NEXT:    vmov d18, r0, r1
-; THUMB78-NEXT:    vtst.32 q8, q8, q9
+; THUMB78-NEXT:    vld1.64 {d18, d19}, [r12]
+; THUMB78-NEXT:    vmov d16, r0, r1
+; THUMB78-NEXT:    vmov.i32 q10, #0x1
+; THUMB78-NEXT:    vshl.u32 q8, q8, q9
+; THUMB78-NEXT:    vtst.32 q8, q8, q10
 ; THUMB78-NEXT:    vmvn q8, q8
 ; THUMB78-NEXT:    vmovn.i32 d16, q8
 ; THUMB78-NEXT:    vmov r0, r1, d16
@@ -911,16 +759,10 @@ define <4 x i1> @vec_4xi32_nonsplat_undef1_eq(<4 x i32> %x, <4 x i32> %y) nounwi
 ; ARM6-NEXT:    ldr r2, [sp, #12]
 ; ARM6-NEXT:    mov lr, #1
 ; ARM6-NEXT:    ldr r12, [sp, #8]
-; ARM6-NEXT:    and r1, r1, lr, lsr r2
+; ARM6-NEXT:    bic r1, lr, r1, lsl r2
 ; ARM6-NEXT:    ldr r2, [sp, #20]
-; ARM6-NEXT:    and r0, r0, lr, lsr r12
-; ARM6-NEXT:    clz r1, r1
-; ARM6-NEXT:    clz r0, r0
-; ARM6-NEXT:    and r2, r3, lr, lsr r2
-; ARM6-NEXT:    lsr r1, r1, #5
-; ARM6-NEXT:    clz r2, r2
-; ARM6-NEXT:    lsr r0, r0, #5
-; ARM6-NEXT:    lsr r3, r2, #5
+; ARM6-NEXT:    bic r0, lr, r0, lsl r12
+; ARM6-NEXT:    bic r3, lr, r3, lsl r2
 ; ARM6-NEXT:    pop {r11, pc}
 ;
 ; ARM78-LABEL: vec_4xi32_nonsplat_undef1_eq:
@@ -940,26 +782,24 @@ define <4 x i1> @vec_4xi32_nonsplat_undef1_eq(<4 x i32> %x, <4 x i32> %y) nounwi
 ;
 ; THUMB6-LABEL: vec_4xi32_nonsplat_undef1_eq:
 ; THUMB6:       @ %bb.0:
-; THUMB6-NEXT:    push {r4, r5, r7, lr}
-; THUMB6-NEXT:    ldr r4, [sp, #16]
+; THUMB6-NEXT:    push {r4, lr}
+; THUMB6-NEXT:    ldr r2, [sp, #8]
+; THUMB6-NEXT:    lsls r0, r2
 ; THUMB6-NEXT:    movs r2, #1
-; THUMB6-NEXT:    mov r5, r2
-; THUMB6-NEXT:    lsrs r5, r4
-; THUMB6-NEXT:    ands r5, r0
-; THUMB6-NEXT:    rsbs r0, r5, #0
-; THUMB6-NEXT:    adcs r0, r5
+; THUMB6-NEXT:    ands r0, r2
+; THUMB6-NEXT:    rsbs r4, r0, #0
+; THUMB6-NEXT:    adcs r0, r4
+; THUMB6-NEXT:    ldr r4, [sp, #12]
+; THUMB6-NEXT:    lsls r1, r4
+; THUMB6-NEXT:    ands r1, r2
+; THUMB6-NEXT:    rsbs r4, r1, #0
+; THUMB6-NEXT:    adcs r1, r4
 ; THUMB6-NEXT:    ldr r4, [sp, #20]
-; THUMB6-NEXT:    mov r5, r2
-; THUMB6-NEXT:    lsrs r5, r4
-; THUMB6-NEXT:    ands r5, r1
-; THUMB6-NEXT:    rsbs r1, r5, #0
-; THUMB6-NEXT:    adcs r1, r5
-; THUMB6-NEXT:    ldr r4, [sp, #28]
-; THUMB6-NEXT:    lsrs r2, r4
-; THUMB6-NEXT:    ands r2, r3
-; THUMB6-NEXT:    rsbs r3, r2, #0
+; THUMB6-NEXT:    lsls r3, r4
+; THUMB6-NEXT:    ands r3, r2
+; THUMB6-NEXT:    rsbs r2, r3, #0
 ; THUMB6-NEXT:    adcs r3, r2
-; THUMB6-NEXT:    pop {r4, r5, r7, pc}
+; THUMB6-NEXT:    pop {r4, pc}
 ;
 ; THUMB78-LABEL: vec_4xi32_nonsplat_undef1_eq:
 ; THUMB78:       @ %bb.0:
@@ -987,16 +827,10 @@ define <4 x i1> @vec_4xi32_nonsplat_undef2_eq(<4 x i32> %x, <4 x i32> %y) nounwi
 ; ARM6-NEXT:    ldr r2, [sp, #12]
 ; ARM6-NEXT:    mov lr, #1
 ; ARM6-NEXT:    ldr r12, [sp, #8]
-; ARM6-NEXT:    and r1, r1, lr, lsr r2
+; ARM6-NEXT:    bic r1, lr, r1, lsl r2
 ; ARM6-NEXT:    ldr r2, [sp, #20]
-; ARM6-NEXT:    and r0, r0, lr, lsr r12
-; ARM6-NEXT:    clz r1, r1
-; ARM6-NEXT:    clz r0, r0
-; ARM6-NEXT:    and r2, r3, lr, lsr r2
-; ARM6-NEXT:    lsr r1, r1, #5
-; ARM6-NEXT:    clz r2, r2
-; ARM6-NEXT:    lsr r0, r0, #5
-; ARM6-NEXT:    lsr r3, r2, #5
+; ARM6-NEXT:    bic r0, lr, r0, lsl r12
+; ARM6-NEXT:    bic r3, lr, r3, lsl r2
 ; ARM6-NEXT:    pop {r11, pc}
 ;
 ; ARM78-LABEL: vec_4xi32_nonsplat_undef2_eq:
@@ -1016,26 +850,24 @@ define <4 x i1> @vec_4xi32_nonsplat_undef2_eq(<4 x i32> %x, <4 x i32> %y) nounwi
 ;
 ; THUMB6-LABEL: vec_4xi32_nonsplat_undef2_eq:
 ; THUMB6:       @ %bb.0:
-; THUMB6-NEXT:    push {r4, r5, r7, lr}
-; THUMB6-NEXT:    ldr r4, [sp, #16]
+; THUMB6-NEXT:    push {r4, lr}
+; THUMB6-NEXT:    ldr r2, [sp, #8]
+; THUMB6-NEXT:    lsls r0, r2
 ; THUMB6-NEXT:    movs r2, #1
-; THUMB6-NEXT:    mov r5, r2
-; THUMB6-NEXT:    lsrs r5, r4
-; THUMB6-NEXT:    ands r5, r0
-; THUMB6-NEXT:    rsbs r0, r5, #0
-; THUMB6-NEXT:    adcs r0, r5
+; THUMB6-NEXT:    ands r0, r2
+; THUMB6-NEXT:    rsbs r4, r0, #0
+; THUMB6-NEXT:    adcs r0, r4
+; THUMB6-NEXT:    ldr r4, [sp, #12]
+; THUMB6-NEXT:    lsls r1, r4
+; THUMB6-NEXT:    ands r1, r2
+; THUMB6-NEXT:    rsbs r4, r1, #0
+; THUMB6-NEXT:    adcs r1, r4
 ; THUMB6-NEXT:    ldr r4, [sp, #20]
-; THUMB6-NEXT:    mov r5, r2
-; THUMB6-NEXT:    lsrs r5, r4
-; THUMB6-NEXT:    ands r5, r1
-; THUMB6-NEXT:    rsbs r1, r5, #0
-; THUMB6-NEXT:    adcs r1, r5
-; THUMB6-NEXT:    ldr r4, [sp, #28]
-; THUMB6-NEXT:    lsrs r2, r4
-; THUMB6-NEXT:    ands r2, r3
-; THUMB6-NEXT:    rsbs r3, r2, #0
+; THUMB6-NEXT:    lsls r3, r4
+; THUMB6-NEXT:    ands r3, r2
+; THUMB6-NEXT:    rsbs r2, r3, #0
 ; THUMB6-NEXT:    adcs r3, r2
-; THUMB6-NEXT:    pop {r4, r5, r7, pc}
+; THUMB6-NEXT:    pop {r4, pc}
 ;
 ; THUMB78-LABEL: vec_4xi32_nonsplat_undef2_eq:
 ; THUMB78:       @ %bb.0:
@@ -1062,48 +894,21 @@ define <4 x i1> @vec_4xi32_nonsplat_undef2_eq(<4 x i32> %x, <4 x i32> %y) nounwi
 ;------------------------------------------------------------------------------;
 
 define i1 @scalar_i8_signbit_ne(i8 %x, i8 %y) nounwind {
-; ARM6-LABEL: scalar_i8_signbit_ne:
-; ARM6:       @ %bb.0:
-; ARM6-NEXT:    uxtb r1, r1
-; ARM6-NEXT:    mov r2, #128
-; ARM6-NEXT:    and r0, r0, r2, lsr r1
-; ARM6-NEXT:    uxtb r0, r0
-; ARM6-NEXT:    cmp r0, #0
-; ARM6-NEXT:    movne r0, #1
-; ARM6-NEXT:    bx lr
-;
-; ARM78-LABEL: scalar_i8_signbit_ne:
-; ARM78:       @ %bb.0:
-; ARM78-NEXT:    uxtb r1, r1
-; ARM78-NEXT:    mov r2, #128
-; ARM78-NEXT:    and r0, r0, r2, lsr r1
-; ARM78-NEXT:    uxtb r0, r0
-; ARM78-NEXT:    cmp r0, #0
-; ARM78-NEXT:    movwne r0, #1
-; ARM78-NEXT:    bx lr
-;
-; THUMB6-LABEL: scalar_i8_signbit_ne:
-; THUMB6:       @ %bb.0:
-; THUMB6-NEXT:    uxtb r1, r1
-; THUMB6-NEXT:    movs r2, #128
-; THUMB6-NEXT:    lsrs r2, r1
-; THUMB6-NEXT:    ands r2, r0
-; THUMB6-NEXT:    uxtb r0, r2
-; THUMB6-NEXT:    subs r1, r0, #1
-; THUMB6-NEXT:    sbcs r0, r1
-; THUMB6-NEXT:    bx lr
+; ARM-LABEL: scalar_i8_signbit_ne:
+; ARM:       @ %bb.0:
+; ARM-NEXT:    uxtb r1, r1
+; ARM-NEXT:    lsl r0, r0, r1
+; ARM-NEXT:    uxtb r0, r0
+; ARM-NEXT:    lsr r0, r0, #7
+; ARM-NEXT:    bx lr
 ;
-; THUMB78-LABEL: scalar_i8_signbit_ne:
-; THUMB78:       @ %bb.0:
-; THUMB78-NEXT:    uxtb r1, r1
-; THUMB78-NEXT:    movs r2, #128
-; THUMB78-NEXT:    lsr.w r1, r2, r1
-; THUMB78-NEXT:    ands r0, r1
-; THUMB78-NEXT:    uxtb r0, r0
-; THUMB78-NEXT:    cmp r0, #0
-; THUMB78-NEXT:    it ne
-; THUMB78-NEXT:    movne r0, #1
-; THUMB78-NEXT:    bx lr
+; THUMB-LABEL: scalar_i8_signbit_ne:
+; THUMB:       @ %bb.0:
+; THUMB-NEXT:    uxtb r1, r1
+; THUMB-NEXT:    lsls r0, r1
+; THUMB-NEXT:    uxtb r0, r0
+; THUMB-NEXT:    lsrs r0, r0, #7
+; THUMB-NEXT:    bx lr
   %t0 = lshr i8 128, %y
   %t1 = and i8 %t0, %x
   %res = icmp ne i8 %t1, 0 ;  we are perfectly happy with 'ne' predicate
@@ -1162,52 +967,25 @@ define i1 @scalar_i32_x_is_const_eq(i32 %y) nounwind {
   ret i1 %res
 }
 define i1 @scalar_i32_x_is_const2_eq(i32 %y) nounwind {
-; ARM6-LABEL: scalar_i32_x_is_const2_eq:
-; ARM6:       @ %bb.0:
-; ARM6-NEXT:    ldr r2, .LCPI19_0
-; ARM6-NEXT:    mov r1, #1
-; ARM6-NEXT:    and r0, r2, r1, lsr r0
-; ARM6-NEXT:    clz r0, r0
-; ARM6-NEXT:    lsr r0, r0, #5
-; ARM6-NEXT:    bx lr
-; ARM6-NEXT:    .p2align 2
-; ARM6-NEXT:  @ %bb.1:
-; ARM6-NEXT:  .LCPI19_0:
-; ARM6-NEXT:    .long 2857740885 @ 0xaa55aa55
-;
-; ARM78-LABEL: scalar_i32_x_is_const2_eq:
-; ARM78:       @ %bb.0:
-; ARM78-NEXT:    movw r1, #43605
-; ARM78-NEXT:    mov r2, #1
-; ARM78-NEXT:    movt r1, #43605
-; ARM78-NEXT:    and r0, r1, r2, lsr r0
-; ARM78-NEXT:    clz r0, r0
-; ARM78-NEXT:    lsr r0, r0, #5
-; ARM78-NEXT:    bx lr
+; ARM-LABEL: scalar_i32_x_is_const2_eq:
+; ARM:       @ %bb.0:
+; ARM-NEXT:    mov r1, #1
+; ARM-NEXT:    eor r0, r1, r1, lsr r0
+; ARM-NEXT:    bx lr
 ;
 ; THUMB6-LABEL: scalar_i32_x_is_const2_eq:
 ; THUMB6:       @ %bb.0:
 ; THUMB6-NEXT:    movs r1, #1
 ; THUMB6-NEXT:    lsrs r1, r0
-; THUMB6-NEXT:    ldr r2, .LCPI19_0
-; THUMB6-NEXT:    ands r2, r1
-; THUMB6-NEXT:    rsbs r0, r2, #0
-; THUMB6-NEXT:    adcs r0, r2
+; THUMB6-NEXT:    rsbs r0, r1, #0
+; THUMB6-NEXT:    adcs r0, r1
 ; THUMB6-NEXT:    bx lr
-; THUMB6-NEXT:    .p2align 2
-; THUMB6-NEXT:  @ %bb.1:
-; THUMB6-NEXT:  .LCPI19_0:
-; THUMB6-NEXT:    .long 2857740885 @ 0xaa55aa55
 ;
 ; THUMB78-LABEL: scalar_i32_x_is_const2_eq:
 ; THUMB78:       @ %bb.0:
 ; THUMB78-NEXT:    movs r1, #1
 ; THUMB78-NEXT:    lsr.w r0, r1, r0
-; THUMB78-NEXT:    movw r1, #43605
-; THUMB78-NEXT:    movt r1, #43605
-; THUMB78-NEXT:    ands r0, r1
-; THUMB78-NEXT:    clz r0, r0
-; THUMB78-NEXT:    lsrs r0, r0, #5
+; THUMB78-NEXT:    eor r0, r0, #1
 ; THUMB78-NEXT:    bx lr
   %t0 = lshr i32 1, %y
   %t1 = and i32 %t0, 2857740885
@@ -1224,8 +1002,7 @@ define i1 @negative_scalar_i8_bitsinmiddle_slt(i8 %x, i8 %y) nounwind {
 ; ARM6:       @ %bb.0:
 ; ARM6-NEXT:    uxtb r1, r1
 ; ARM6-NEXT:    mov r2, #24
-; ARM6-NEXT:    and r0, r0, r2, lsr r1
-; ARM6-NEXT:    sxtb r1, r0
+; ARM6-NEXT:    and r1, r0, r2, lsr r1
 ; ARM6-NEXT:    mov r0, #0
 ; ARM6-NEXT:    cmp r1, #0
 ; ARM6-NEXT:    movmi r0, #1
@@ -1235,8 +1012,7 @@ define i1 @negative_scalar_i8_bitsinmiddle_slt(i8 %x, i8 %y) nounwind {
 ; ARM78:       @ %bb.0:
 ; ARM78-NEXT:    uxtb r1, r1
 ; ARM78-NEXT:    mov r2, #24
-; ARM78-NEXT:    and r0, r0, r2, lsr r1
-; ARM78-NEXT:    sxtb r1, r0
+; ARM78-NEXT:    and r1, r0, r2, lsr r1
 ; ARM78-NEXT:    mov r0, #0
 ; ARM78-NEXT:    cmp r1, #0
 ; ARM78-NEXT:    movwmi r0, #1
@@ -1248,8 +1024,6 @@ define i1 @negative_scalar_i8_bitsinmiddle_slt(i8 %x, i8 %y) nounwind {
 ; THUMB6-NEXT:    movs r2, #24
 ; THUMB6-NEXT:    lsrs r2, r1
 ; THUMB6-NEXT:    ands r2, r0
-; THUMB6-NEXT:    sxtb r0, r2
-; THUMB6-NEXT:    cmp r0, #0
 ; THUMB6-NEXT:    bmi .LBB20_2
 ; THUMB6-NEXT:  @ %bb.1:
 ; THUMB6-NEXT:    movs r0, #0
@@ -1264,9 +1038,7 @@ define i1 @negative_scalar_i8_bitsinmiddle_slt(i8 %x, i8 %y) nounwind {
 ; THUMB78-NEXT:    movs r2, #24
 ; THUMB78-NEXT:    lsr.w r1, r2, r1
 ; THUMB78-NEXT:    ands r0, r1
-; THUMB78-NEXT:    sxtb r1, r0
-; THUMB78-NEXT:    movs r0, #0
-; THUMB78-NEXT:    cmp r1, #0
+; THUMB78-NEXT:    mov.w r0, #0
 ; THUMB78-NEXT:    it mi
 ; THUMB78-NEXT:    movmi r0, #1
 ; THUMB78-NEXT:    bx lr
@@ -1282,8 +1054,7 @@ define i1 @scalar_i8_signbit_eq_with_nonzero(i8 %x, i8 %y) nounwind {
 ; ARM-NEXT:    uxtb r1, r1
 ; ARM-NEXT:    mov r2, #128
 ; ARM-NEXT:    and r0, r0, r2, lsr r1
-; ARM-NEXT:    mvn r1, #0
-; ARM-NEXT:    uxtab r0, r1, r0
+; ARM-NEXT:    sub r0, r0, #1
 ; ARM-NEXT:    clz r0, r0
 ; ARM-NEXT:    lsr r0, r0, #5
 ; ARM-NEXT:    bx lr
@@ -1294,8 +1065,7 @@ define i1 @scalar_i8_signbit_eq_with_nonzero(i8 %x, i8 %y) nounwind {
 ; THUMB6-NEXT:    movs r2, #128
 ; THUMB6-NEXT:    lsrs r2, r1
 ; THUMB6-NEXT:    ands r2, r0
-; THUMB6-NEXT:    uxtb r0, r2
-; THUMB6-NEXT:    subs r1, r0, #1
+; THUMB6-NEXT:    subs r1, r2, #1
 ; THUMB6-NEXT:    rsbs r0, r1, #0
 ; THUMB6-NEXT:    adcs r0, r1
 ; THUMB6-NEXT:    bx lr
@@ -1306,8 +1076,7 @@ define i1 @scalar_i8_signbit_eq_with_nonzero(i8 %x, i8 %y) nounwind {
 ; THUMB78-NEXT:    movs r2, #128
 ; THUMB78-NEXT:    lsr.w r1, r2, r1
 ; THUMB78-NEXT:    ands r0, r1
-; THUMB78-NEXT:    mov.w r1, #-1
-; THUMB78-NEXT:    uxtab r0, r1, r0
+; THUMB78-NEXT:    subs r0, #1
 ; THUMB78-NEXT:    clz r0, r0
 ; THUMB78-NEXT:    lsrs r0, r0, #5
 ; THUMB78-NEXT:    bx lr
diff --git a/llvm/test/CodeGen/ARM/hoist-and-by-const-from-shl-in-eqcmp-zero.ll b/llvm/test/CodeGen/ARM/hoist-and-by-const-from-shl-in-eqcmp-zero.ll
index 331085fe7..0de18e557 100644
--- a/llvm/test/CodeGen/ARM/hoist-and-by-const-from-shl-in-eqcmp-zero.ll
+++ b/llvm/test/CodeGen/ARM/hoist-and-by-const-from-shl-in-eqcmp-zero.ll
@@ -21,35 +21,40 @@ define i1 @scalar_i8_signbit_eq(i8 %x, i8 %y) nounwind {
 ; ARM-LABEL: scalar_i8_signbit_eq:
 ; ARM:       @ %bb.0:
 ; ARM-NEXT:    uxtb r1, r1
-; ARM-NEXT:    mvn r2, #127
-; ARM-NEXT:    and r0, r0, r2, lsl r1
 ; ARM-NEXT:    uxtb r0, r0
-; ARM-NEXT:    clz r0, r0
-; ARM-NEXT:    lsr r0, r0, #5
+; ARM-NEXT:    lsr r0, r0, r1
+; ARM-NEXT:    mov r1, #1
+; ARM-NEXT:    eor r0, r1, r0, lsr #7
 ; ARM-NEXT:    bx lr
 ;
 ; THUMB6-LABEL: scalar_i8_signbit_eq:
 ; THUMB6:       @ %bb.0:
 ; THUMB6-NEXT:    uxtb r1, r1
-; THUMB6-NEXT:    movs r2, #127
-; THUMB6-NEXT:    mvns r2, r2
-; THUMB6-NEXT:    lsls r2, r1
-; THUMB6-NEXT:    ands r2, r0
-; THUMB6-NEXT:    uxtb r1, r2
+; THUMB6-NEXT:    uxtb r0, r0
+; THUMB6-NEXT:    lsrs r0, r1
+; THUMB6-NEXT:    movs r1, #128
+; THUMB6-NEXT:    ands r1, r0
 ; THUMB6-NEXT:    rsbs r0, r1, #0
 ; THUMB6-NEXT:    adcs r0, r1
 ; THUMB6-NEXT:    bx lr
 ;
-; THUMB78-LABEL: scalar_i8_signbit_eq:
-; THUMB78:       @ %bb.0:
-; THUMB78-NEXT:    uxtb r1, r1
-; THUMB78-NEXT:    mvn r2, #127
-; THUMB78-NEXT:    lsl.w r1, r2, r1
-; THUMB78-NEXT:    ands r0, r1
-; THUMB78-NEXT:    uxtb r0, r0
-; THUMB78-NEXT:    clz r0, r0
-; THUMB78-NEXT:    lsrs r0, r0, #5
-; THUMB78-NEXT:    bx lr
+; THUMB7-LABEL: scalar_i8_signbit_eq:
+; THUMB7:       @ %bb.0:
+; THUMB7-NEXT:    uxtb r1, r1
+; THUMB7-NEXT:    uxtb r0, r0
+; THUMB7-NEXT:    lsrs r0, r1
+; THUMB7-NEXT:    movs r1, #1
+; THUMB7-NEXT:    eor.w r0, r1, r0, lsr #7
+; THUMB7-NEXT:    bx lr
+;
+; THUMB8-LABEL: scalar_i8_signbit_eq:
+; THUMB8:       @ %bb.0:
+; THUMB8-NEXT:    uxtb r0, r0
+; THUMB8-NEXT:    uxtb r1, r1
+; THUMB8-NEXT:    lsrs r0, r1
+; THUMB8-NEXT:    movs r1, #1
+; THUMB8-NEXT:    eor.w r0, r1, r0, lsr #7
+; THUMB8-NEXT:    bx lr
   %t0 = shl i8 128, %y
   %t1 = and i8 %t0, %x
   %res = icmp eq i8 %t1, 0
@@ -60,34 +65,39 @@ define i1 @scalar_i8_lowestbit_eq(i8 %x, i8 %y) nounwind {
 ; ARM-LABEL: scalar_i8_lowestbit_eq:
 ; ARM:       @ %bb.0:
 ; ARM-NEXT:    uxtb r1, r1
-; ARM-NEXT:    mov r2, #1
-; ARM-NEXT:    and r0, r0, r2, lsl r1
 ; ARM-NEXT:    uxtb r0, r0
-; ARM-NEXT:    clz r0, r0
-; ARM-NEXT:    lsr r0, r0, #5
+; ARM-NEXT:    mov r2, #1
+; ARM-NEXT:    bic r0, r2, r0, lsr r1
 ; ARM-NEXT:    bx lr
 ;
 ; THUMB6-LABEL: scalar_i8_lowestbit_eq:
 ; THUMB6:       @ %bb.0:
 ; THUMB6-NEXT:    uxtb r1, r1
-; THUMB6-NEXT:    movs r2, #1
-; THUMB6-NEXT:    lsls r2, r1
-; THUMB6-NEXT:    ands r2, r0
-; THUMB6-NEXT:    uxtb r1, r2
+; THUMB6-NEXT:    uxtb r0, r0
+; THUMB6-NEXT:    lsrs r0, r1
+; THUMB6-NEXT:    movs r1, #1
+; THUMB6-NEXT:    ands r1, r0
 ; THUMB6-NEXT:    rsbs r0, r1, #0
 ; THUMB6-NEXT:    adcs r0, r1
 ; THUMB6-NEXT:    bx lr
 ;
-; THUMB78-LABEL: scalar_i8_lowestbit_eq:
-; THUMB78:       @ %bb.0:
-; THUMB78-NEXT:    uxtb r1, r1
-; THUMB78-NEXT:    movs r2, #1
-; THUMB78-NEXT:    lsl.w r1, r2, r1
-; THUMB78-NEXT:    ands r0, r1
-; THUMB78-NEXT:    uxtb r0, r0
-; THUMB78-NEXT:    clz r0, r0
-; THUMB78-NEXT:    lsrs r0, r0, #5
-; THUMB78-NEXT:    bx lr
+; THUMB7-LABEL: scalar_i8_lowestbit_eq:
+; THUMB7:       @ %bb.0:
+; THUMB7-NEXT:    uxtb r1, r1
+; THUMB7-NEXT:    uxtb r0, r0
+; THUMB7-NEXT:    lsrs r0, r1
+; THUMB7-NEXT:    movs r1, #1
+; THUMB7-NEXT:    bic.w r0, r1, r0
+; THUMB7-NEXT:    bx lr
+;
+; THUMB8-LABEL: scalar_i8_lowestbit_eq:
+; THUMB8:       @ %bb.0:
+; THUMB8-NEXT:    uxtb r0, r0
+; THUMB8-NEXT:    uxtb r1, r1
+; THUMB8-NEXT:    lsrs r0, r1
+; THUMB8-NEXT:    movs r1, #1
+; THUMB8-NEXT:    bic.w r0, r1, r0
+; THUMB8-NEXT:    bx lr
   %t0 = shl i8 1, %y
   %t1 = and i8 %t0, %x
   %res = icmp eq i8 %t1, 0
@@ -98,9 +108,9 @@ define i1 @scalar_i8_bitsinmiddle_eq(i8 %x, i8 %y) nounwind {
 ; ARM-LABEL: scalar_i8_bitsinmiddle_eq:
 ; ARM:       @ %bb.0:
 ; ARM-NEXT:    uxtb r1, r1
-; ARM-NEXT:    mov r2, #24
-; ARM-NEXT:    and r0, r0, r2, lsl r1
 ; ARM-NEXT:    uxtb r0, r0
+; ARM-NEXT:    mov r2, #24
+; ARM-NEXT:    and r0, r2, r0, lsr r1
 ; ARM-NEXT:    clz r0, r0
 ; ARM-NEXT:    lsr r0, r0, #5
 ; ARM-NEXT:    bx lr
@@ -108,24 +118,33 @@ define i1 @scalar_i8_bitsinmiddle_eq(i8 %x, i8 %y) nounwind {
 ; THUMB6-LABEL: scalar_i8_bitsinmiddle_eq:
 ; THUMB6:       @ %bb.0:
 ; THUMB6-NEXT:    uxtb r1, r1
-; THUMB6-NEXT:    movs r2, #24
-; THUMB6-NEXT:    lsls r2, r1
-; THUMB6-NEXT:    ands r2, r0
-; THUMB6-NEXT:    uxtb r1, r2
+; THUMB6-NEXT:    uxtb r0, r0
+; THUMB6-NEXT:    lsrs r0, r1
+; THUMB6-NEXT:    movs r1, #24
+; THUMB6-NEXT:    ands r1, r0
 ; THUMB6-NEXT:    rsbs r0, r1, #0
 ; THUMB6-NEXT:    adcs r0, r1
 ; THUMB6-NEXT:    bx lr
 ;
-; THUMB78-LABEL: scalar_i8_bitsinmiddle_eq:
-; THUMB78:       @ %bb.0:
-; THUMB78-NEXT:    uxtb r1, r1
-; THUMB78-NEXT:    movs r2, #24
-; THUMB78-NEXT:    lsl.w r1, r2, r1
-; THUMB78-NEXT:    ands r0, r1
-; THUMB78-NEXT:    uxtb r0, r0
-; THUMB78-NEXT:    clz r0, r0
-; THUMB78-NEXT:    lsrs r0, r0, #5
-; THUMB78-NEXT:    bx lr
+; THUMB7-LABEL: scalar_i8_bitsinmiddle_eq:
+; THUMB7:       @ %bb.0:
+; THUMB7-NEXT:    uxtb r1, r1
+; THUMB7-NEXT:    uxtb r0, r0
+; THUMB7-NEXT:    lsrs r0, r1
+; THUMB7-NEXT:    and r0, r0, #24
+; THUMB7-NEXT:    clz r0, r0
+; THUMB7-NEXT:    lsrs r0, r0, #5
+; THUMB7-NEXT:    bx lr
+;
+; THUMB8-LABEL: scalar_i8_bitsinmiddle_eq:
+; THUMB8:       @ %bb.0:
+; THUMB8-NEXT:    uxtb r0, r0
+; THUMB8-NEXT:    uxtb r1, r1
+; THUMB8-NEXT:    lsrs r0, r1
+; THUMB8-NEXT:    and r0, r0, #24
+; THUMB8-NEXT:    clz r0, r0
+; THUMB8-NEXT:    lsrs r0, r0, #5
+; THUMB8-NEXT:    bx lr
   %t0 = shl i8 24, %y
   %t1 = and i8 %t0, %x
   %res = icmp eq i8 %t1, 0
@@ -135,57 +154,44 @@ define i1 @scalar_i8_bitsinmiddle_eq(i8 %x, i8 %y) nounwind {
 ; i16 scalar
 
 define i1 @scalar_i16_signbit_eq(i16 %x, i16 %y) nounwind {
-; ARM6-LABEL: scalar_i16_signbit_eq:
-; ARM6:       @ %bb.0:
-; ARM6-NEXT:    ldr r2, .LCPI3_0
-; ARM6-NEXT:    uxth r1, r1
-; ARM6-NEXT:    and r0, r0, r2, lsl r1
-; ARM6-NEXT:    uxth r0, r0
-; ARM6-NEXT:    clz r0, r0
-; ARM6-NEXT:    lsr r0, r0, #5
-; ARM6-NEXT:    bx lr
-; ARM6-NEXT:    .p2align 2
-; ARM6-NEXT:  @ %bb.1:
-; ARM6-NEXT:  .LCPI3_0:
-; ARM6-NEXT:    .long 4294934528 @ 0xffff8000
-;
-; ARM78-LABEL: scalar_i16_signbit_eq:
-; ARM78:       @ %bb.0:
-; ARM78-NEXT:    movw r2, #32768
-; ARM78-NEXT:    uxth r1, r1
-; ARM78-NEXT:    movt r2, #65535
-; ARM78-NEXT:    and r0, r0, r2, lsl r1
-; ARM78-NEXT:    uxth r0, r0
-; ARM78-NEXT:    clz r0, r0
-; ARM78-NEXT:    lsr r0, r0, #5
-; ARM78-NEXT:    bx lr
+; ARM-LABEL: scalar_i16_signbit_eq:
+; ARM:       @ %bb.0:
+; ARM-NEXT:    uxth r1, r1
+; ARM-NEXT:    uxth r0, r0
+; ARM-NEXT:    lsr r0, r0, r1
+; ARM-NEXT:    mov r1, #1
+; ARM-NEXT:    eor r0, r1, r0, lsr #15
+; ARM-NEXT:    bx lr
 ;
 ; THUMB6-LABEL: scalar_i16_signbit_eq:
 ; THUMB6:       @ %bb.0:
 ; THUMB6-NEXT:    uxth r1, r1
-; THUMB6-NEXT:    ldr r2, .LCPI3_0
-; THUMB6-NEXT:    lsls r2, r1
-; THUMB6-NEXT:    ands r2, r0
-; THUMB6-NEXT:    uxth r1, r2
+; THUMB6-NEXT:    uxth r0, r0
+; THUMB6-NEXT:    lsrs r0, r1
+; THUMB6-NEXT:    movs r1, #1
+; THUMB6-NEXT:    lsls r1, r1, #15
+; THUMB6-NEXT:    ands r1, r0
 ; THUMB6-NEXT:    rsbs r0, r1, #0
 ; THUMB6-NEXT:    adcs r0, r1
 ; THUMB6-NEXT:    bx lr
-; THUMB6-NEXT:    .p2align 2
-; THUMB6-NEXT:  @ %bb.1:
-; THUMB6-NEXT:  .LCPI3_0:
-; THUMB6-NEXT:    .long 4294934528 @ 0xffff8000
 ;
-; THUMB78-LABEL: scalar_i16_signbit_eq:
-; THUMB78:       @ %bb.0:
-; THUMB78-NEXT:    movw r2, #32768
-; THUMB78-NEXT:    uxth r1, r1
-; THUMB78-NEXT:    movt r2, #65535
-; THUMB78-NEXT:    lsl.w r1, r2, r1
-; THUMB78-NEXT:    ands r0, r1
-; THUMB78-NEXT:    uxth r0, r0
-; THUMB78-NEXT:    clz r0, r0
-; THUMB78-NEXT:    lsrs r0, r0, #5
-; THUMB78-NEXT:    bx lr
+; THUMB7-LABEL: scalar_i16_signbit_eq:
+; THUMB7:       @ %bb.0:
+; THUMB7-NEXT:    uxth r1, r1
+; THUMB7-NEXT:    uxth r0, r0
+; THUMB7-NEXT:    lsrs r0, r1
+; THUMB7-NEXT:    movs r1, #1
+; THUMB7-NEXT:    eor.w r0, r1, r0, lsr #15
+; THUMB7-NEXT:    bx lr
+;
+; THUMB8-LABEL: scalar_i16_signbit_eq:
+; THUMB8:       @ %bb.0:
+; THUMB8-NEXT:    uxth r0, r0
+; THUMB8-NEXT:    uxth r1, r1
+; THUMB8-NEXT:    lsrs r0, r1
+; THUMB8-NEXT:    movs r1, #1
+; THUMB8-NEXT:    eor.w r0, r1, r0, lsr #15
+; THUMB8-NEXT:    bx lr
   %t0 = shl i16 32768, %y
   %t1 = and i16 %t0, %x
   %res = icmp eq i16 %t1, 0
@@ -196,34 +202,39 @@ define i1 @scalar_i16_lowestbit_eq(i16 %x, i16 %y) nounwind {
 ; ARM-LABEL: scalar_i16_lowestbit_eq:
 ; ARM:       @ %bb.0:
 ; ARM-NEXT:    uxth r1, r1
-; ARM-NEXT:    mov r2, #1
-; ARM-NEXT:    and r0, r0, r2, lsl r1
 ; ARM-NEXT:    uxth r0, r0
-; ARM-NEXT:    clz r0, r0
-; ARM-NEXT:    lsr r0, r0, #5
+; ARM-NEXT:    mov r2, #1
+; ARM-NEXT:    bic r0, r2, r0, lsr r1
 ; ARM-NEXT:    bx lr
 ;
 ; THUMB6-LABEL: scalar_i16_lowestbit_eq:
 ; THUMB6:       @ %bb.0:
 ; THUMB6-NEXT:    uxth r1, r1
-; THUMB6-NEXT:    movs r2, #1
-; THUMB6-NEXT:    lsls r2, r1
-; THUMB6-NEXT:    ands r2, r0
-; THUMB6-NEXT:    uxth r1, r2
+; THUMB6-NEXT:    uxth r0, r0
+; THUMB6-NEXT:    lsrs r0, r1
+; THUMB6-NEXT:    movs r1, #1
+; THUMB6-NEXT:    ands r1, r0
 ; THUMB6-NEXT:    rsbs r0, r1, #0
 ; THUMB6-NEXT:    adcs r0, r1
 ; THUMB6-NEXT:    bx lr
 ;
-; THUMB78-LABEL: scalar_i16_lowestbit_eq:
-; THUMB78:       @ %bb.0:
-; THUMB78-NEXT:    uxth r1, r1
-; THUMB78-NEXT:    movs r2, #1
-; THUMB78-NEXT:    lsl.w r1, r2, r1
-; THUMB78-NEXT:    ands r0, r1
-; THUMB78-NEXT:    uxth r0, r0
-; THUMB78-NEXT:    clz r0, r0
-; THUMB78-NEXT:    lsrs r0, r0, #5
-; THUMB78-NEXT:    bx lr
+; THUMB7-LABEL: scalar_i16_lowestbit_eq:
+; THUMB7:       @ %bb.0:
+; THUMB7-NEXT:    uxth r1, r1
+; THUMB7-NEXT:    uxth r0, r0
+; THUMB7-NEXT:    lsrs r0, r1
+; THUMB7-NEXT:    movs r1, #1
+; THUMB7-NEXT:    bic.w r0, r1, r0
+; THUMB7-NEXT:    bx lr
+;
+; THUMB8-LABEL: scalar_i16_lowestbit_eq:
+; THUMB8:       @ %bb.0:
+; THUMB8-NEXT:    uxth r0, r0
+; THUMB8-NEXT:    uxth r1, r1
+; THUMB8-NEXT:    lsrs r0, r1
+; THUMB8-NEXT:    movs r1, #1
+; THUMB8-NEXT:    bic.w r0, r1, r0
+; THUMB8-NEXT:    bx lr
   %t0 = shl i16 1, %y
   %t1 = and i16 %t0, %x
   %res = icmp eq i16 %t1, 0
@@ -234,9 +245,9 @@ define i1 @scalar_i16_bitsinmiddle_eq(i16 %x, i16 %y) nounwind {
 ; ARM-LABEL: scalar_i16_bitsinmiddle_eq:
 ; ARM:       @ %bb.0:
 ; ARM-NEXT:    uxth r1, r1
-; ARM-NEXT:    mov r2, #4080
-; ARM-NEXT:    and r0, r0, r2, lsl r1
 ; ARM-NEXT:    uxth r0, r0
+; ARM-NEXT:    mov r2, #4080
+; ARM-NEXT:    and r0, r2, r0, lsr r1
 ; ARM-NEXT:    clz r0, r0
 ; ARM-NEXT:    lsr r0, r0, #5
 ; ARM-NEXT:    bx lr
@@ -244,25 +255,34 @@ define i1 @scalar_i16_bitsinmiddle_eq(i16 %x, i16 %y) nounwind {
 ; THUMB6-LABEL: scalar_i16_bitsinmiddle_eq:
 ; THUMB6:       @ %bb.0:
 ; THUMB6-NEXT:    uxth r1, r1
-; THUMB6-NEXT:    movs r2, #255
-; THUMB6-NEXT:    lsls r2, r2, #4
-; THUMB6-NEXT:    lsls r2, r1
-; THUMB6-NEXT:    ands r2, r0
-; THUMB6-NEXT:    uxth r1, r2
+; THUMB6-NEXT:    uxth r0, r0
+; THUMB6-NEXT:    lsrs r0, r1
+; THUMB6-NEXT:    movs r1, #255
+; THUMB6-NEXT:    lsls r1, r1, #4
+; THUMB6-NEXT:    ands r1, r0
 ; THUMB6-NEXT:    rsbs r0, r1, #0
 ; THUMB6-NEXT:    adcs r0, r1
 ; THUMB6-NEXT:    bx lr
 ;
-; THUMB78-LABEL: scalar_i16_bitsinmiddle_eq:
-; THUMB78:       @ %bb.0:
-; THUMB78-NEXT:    uxth r1, r1
-; THUMB78-NEXT:    mov.w r2, #4080
-; THUMB78-NEXT:    lsl.w r1, r2, r1
-; THUMB78-NEXT:    ands r0, r1
-; THUMB78-NEXT:    uxth r0, r0
-; THUMB78-NEXT:    clz r0, r0
-; THUMB78-NEXT:    lsrs r0, r0, #5
-; THUMB78-NEXT:    bx lr
+; THUMB7-LABEL: scalar_i16_bitsinmiddle_eq:
+; THUMB7:       @ %bb.0:
+; THUMB7-NEXT:    uxth r1, r1
+; THUMB7-NEXT:    uxth r0, r0
+; THUMB7-NEXT:    lsrs r0, r1
+; THUMB7-NEXT:    and r0, r0, #4080
+; THUMB7-NEXT:    clz r0, r0
+; THUMB7-NEXT:    lsrs r0, r0, #5
+; THUMB7-NEXT:    bx lr
+;
+; THUMB8-LABEL: scalar_i16_bitsinmiddle_eq:
+; THUMB8:       @ %bb.0:
+; THUMB8-NEXT:    uxth r0, r0
+; THUMB8-NEXT:    uxth r1, r1
+; THUMB8-NEXT:    lsrs r0, r1
+; THUMB8-NEXT:    and r0, r0, #4080
+; THUMB8-NEXT:    clz r0, r0
+; THUMB8-NEXT:    lsrs r0, r0, #5
+; THUMB8-NEXT:    bx lr
   %t0 = shl i16 4080, %y
   %t1 = and i16 %t0, %x
   %res = icmp eq i16 %t1, 0
@@ -274,29 +294,25 @@ define i1 @scalar_i16_bitsinmiddle_eq(i16 %x, i16 %y) nounwind {
 define i1 @scalar_i32_signbit_eq(i32 %x, i32 %y) nounwind {
 ; ARM-LABEL: scalar_i32_signbit_eq:
 ; ARM:       @ %bb.0:
-; ARM-NEXT:    mov r2, #-2147483648
-; ARM-NEXT:    and r0, r0, r2, lsl r1
-; ARM-NEXT:    clz r0, r0
-; ARM-NEXT:    lsr r0, r0, #5
+; ARM-NEXT:    mvn r0, r0, lsr r1
+; ARM-NEXT:    lsr r0, r0, #31
 ; ARM-NEXT:    bx lr
 ;
 ; THUMB6-LABEL: scalar_i32_signbit_eq:
 ; THUMB6:       @ %bb.0:
-; THUMB6-NEXT:    movs r2, #1
-; THUMB6-NEXT:    lsls r2, r2, #31
-; THUMB6-NEXT:    lsls r2, r1
-; THUMB6-NEXT:    ands r2, r0
-; THUMB6-NEXT:    rsbs r0, r2, #0
-; THUMB6-NEXT:    adcs r0, r2
+; THUMB6-NEXT:    lsrs r0, r1
+; THUMB6-NEXT:    movs r1, #1
+; THUMB6-NEXT:    lsls r1, r1, #31
+; THUMB6-NEXT:    ands r1, r0
+; THUMB6-NEXT:    rsbs r0, r1, #0
+; THUMB6-NEXT:    adcs r0, r1
 ; THUMB6-NEXT:    bx lr
 ;
 ; THUMB78-LABEL: scalar_i32_signbit_eq:
 ; THUMB78:       @ %bb.0:
-; THUMB78-NEXT:    mov.w r2, #-2147483648
-; THUMB78-NEXT:    lsl.w r1, r2, r1
-; THUMB78-NEXT:    ands r0, r1
-; THUMB78-NEXT:    clz r0, r0
-; THUMB78-NEXT:    lsrs r0, r0, #5
+; THUMB78-NEXT:    lsrs r0, r1
+; THUMB78-NEXT:    mvns r0, r0
+; THUMB78-NEXT:    lsrs r0, r0, #31
 ; THUMB78-NEXT:    bx lr
   %t0 = shl i32 2147483648, %y
   %t1 = and i32 %t0, %x
@@ -308,27 +324,23 @@ define i1 @scalar_i32_lowestbit_eq(i32 %x, i32 %y) nounwind {
 ; ARM-LABEL: scalar_i32_lowestbit_eq:
 ; ARM:       @ %bb.0:
 ; ARM-NEXT:    mov r2, #1
-; ARM-NEXT:    and r0, r0, r2, lsl r1
-; ARM-NEXT:    clz r0, r0
-; ARM-NEXT:    lsr r0, r0, #5
+; ARM-NEXT:    bic r0, r2, r0, lsr r1
 ; ARM-NEXT:    bx lr
 ;
 ; THUMB6-LABEL: scalar_i32_lowestbit_eq:
 ; THUMB6:       @ %bb.0:
-; THUMB6-NEXT:    movs r2, #1
-; THUMB6-NEXT:    lsls r2, r1
-; THUMB6-NEXT:    ands r2, r0
-; THUMB6-NEXT:    rsbs r0, r2, #0
-; THUMB6-NEXT:    adcs r0, r2
+; THUMB6-NEXT:    lsrs r0, r1
+; THUMB6-NEXT:    movs r1, #1
+; THUMB6-NEXT:    ands r1, r0
+; THUMB6-NEXT:    rsbs r0, r1, #0
+; THUMB6-NEXT:    adcs r0, r1
 ; THUMB6-NEXT:    bx lr
 ;
 ; THUMB78-LABEL: scalar_i32_lowestbit_eq:
 ; THUMB78:       @ %bb.0:
-; THUMB78-NEXT:    movs r2, #1
-; THUMB78-NEXT:    lsl.w r1, r2, r1
-; THUMB78-NEXT:    ands r0, r1
-; THUMB78-NEXT:    clz r0, r0
-; THUMB78-NEXT:    lsrs r0, r0, #5
+; THUMB78-NEXT:    lsrs r0, r1
+; THUMB78-NEXT:    movs r1, #1
+; THUMB78-NEXT:    bic.w r0, r1, r0
 ; THUMB78-NEXT:    bx lr
   %t0 = shl i32 1, %y
   %t1 = and i32 %t0, %x
@@ -341,7 +353,7 @@ define i1 @scalar_i32_bitsinmiddle_eq(i32 %x, i32 %y) nounwind {
 ; ARM6:       @ %bb.0:
 ; ARM6-NEXT:    mov r2, #65280
 ; ARM6-NEXT:    orr r2, r2, #16711680
-; ARM6-NEXT:    and r0, r0, r2, lsl r1
+; ARM6-NEXT:    and r0, r2, r0, lsr r1
 ; ARM6-NEXT:    clz r0, r0
 ; ARM6-NEXT:    lsr r0, r0, #5
 ; ARM6-NEXT:    bx lr
@@ -350,18 +362,18 @@ define i1 @scalar_i32_bitsinmiddle_eq(i32 %x, i32 %y) nounwind {
 ; ARM78:       @ %bb.0:
 ; ARM78-NEXT:    movw r2, #65280
 ; ARM78-NEXT:    movt r2, #255
-; ARM78-NEXT:    and r0, r0, r2, lsl r1
+; ARM78-NEXT:    and r0, r2, r0, lsr r1
 ; ARM78-NEXT:    clz r0, r0
 ; ARM78-NEXT:    lsr r0, r0, #5
 ; ARM78-NEXT:    bx lr
 ;
 ; THUMB6-LABEL: scalar_i32_bitsinmiddle_eq:
 ; THUMB6:       @ %bb.0:
-; THUMB6-NEXT:    ldr r2, .LCPI8_0
-; THUMB6-NEXT:    lsls r2, r1
-; THUMB6-NEXT:    ands r2, r0
-; THUMB6-NEXT:    rsbs r0, r2, #0
-; THUMB6-NEXT:    adcs r0, r2
+; THUMB6-NEXT:    lsrs r0, r1
+; THUMB6-NEXT:    ldr r1, .LCPI8_0
+; THUMB6-NEXT:    ands r1, r0
+; THUMB6-NEXT:    rsbs r0, r1, #0
+; THUMB6-NEXT:    adcs r0, r1
 ; THUMB6-NEXT:    bx lr
 ; THUMB6-NEXT:    .p2align 2
 ; THUMB6-NEXT:  @ %bb.1:
@@ -370,9 +382,9 @@ define i1 @scalar_i32_bitsinmiddle_eq(i32 %x, i32 %y) nounwind {
 ;
 ; THUMB78-LABEL: scalar_i32_bitsinmiddle_eq:
 ; THUMB78:       @ %bb.0:
-; THUMB78-NEXT:    movw r2, #65280
-; THUMB78-NEXT:    movt r2, #255
-; THUMB78-NEXT:    lsl.w r1, r2, r1
+; THUMB78-NEXT:    lsrs r0, r1
+; THUMB78-NEXT:    movw r1, #65280
+; THUMB78-NEXT:    movt r1, #255
 ; THUMB78-NEXT:    ands r0, r1
 ; THUMB78-NEXT:    clz r0, r0
 ; THUMB78-NEXT:    lsrs r0, r0, #5
@@ -388,52 +400,41 @@ define i1 @scalar_i32_bitsinmiddle_eq(i32 %x, i32 %y) nounwind {
 define i1 @scalar_i64_signbit_eq(i64 %x, i64 %y) nounwind {
 ; ARM6-LABEL: scalar_i64_signbit_eq:
 ; ARM6:       @ %bb.0:
-; ARM6-NEXT:    mov r0, #-2147483648
-; ARM6-NEXT:    lsl r0, r0, r2
-; ARM6-NEXT:    subs r2, r2, #32
+; ARM6-NEXT:    lsr r0, r1, r2
+; ARM6-NEXT:    subs r1, r2, #32
 ; ARM6-NEXT:    movpl r0, #0
-; ARM6-NEXT:    and r0, r0, r1
-; ARM6-NEXT:    clz r0, r0
-; ARM6-NEXT:    lsr r0, r0, #5
+; ARM6-NEXT:    mvn r0, r0
+; ARM6-NEXT:    lsr r0, r0, #31
 ; ARM6-NEXT:    bx lr
 ;
 ; ARM78-LABEL: scalar_i64_signbit_eq:
 ; ARM78:       @ %bb.0:
-; ARM78-NEXT:    mov r0, #-2147483648
-; ARM78-NEXT:    lsl r0, r0, r2
-; ARM78-NEXT:    subs r2, r2, #32
+; ARM78-NEXT:    lsr r0, r1, r2
+; ARM78-NEXT:    subs r1, r2, #32
 ; ARM78-NEXT:    movwpl r0, #0
-; ARM78-NEXT:    and r0, r0, r1
-; ARM78-NEXT:    clz r0, r0
-; ARM78-NEXT:    lsr r0, r0, #5
+; ARM78-NEXT:    mvn r0, r0
+; ARM78-NEXT:    lsr r0, r0, #31
 ; ARM78-NEXT:    bx lr
 ;
 ; THUMB6-LABEL: scalar_i64_signbit_eq:
 ; THUMB6:       @ %bb.0:
-; THUMB6-NEXT:    push {r4, r5, r7, lr}
-; THUMB6-NEXT:    mov r4, r1
-; THUMB6-NEXT:    mov r5, r0
+; THUMB6-NEXT:    push {r7, lr}
+; THUMB6-NEXT:    bl __lshrdi3
 ; THUMB6-NEXT:    movs r0, #1
-; THUMB6-NEXT:    lsls r1, r0, #31
-; THUMB6-NEXT:    movs r0, #0
-; THUMB6-NEXT:    bl __ashldi3
-; THUMB6-NEXT:    ands r1, r4
-; THUMB6-NEXT:    ands r0, r5
-; THUMB6-NEXT:    orrs r0, r1
-; THUMB6-NEXT:    rsbs r1, r0, #0
-; THUMB6-NEXT:    adcs r0, r1
-; THUMB6-NEXT:    pop {r4, r5, r7, pc}
+; THUMB6-NEXT:    lsls r2, r0, #31
+; THUMB6-NEXT:    ands r2, r1
+; THUMB6-NEXT:    rsbs r0, r2, #0
+; THUMB6-NEXT:    adcs r0, r2
+; THUMB6-NEXT:    pop {r7, pc}
 ;
 ; THUMB78-LABEL: scalar_i64_signbit_eq:
 ; THUMB78:       @ %bb.0:
-; THUMB78-NEXT:    mov.w r0, #-2147483648
-; THUMB78-NEXT:    lsls r0, r2
-; THUMB78-NEXT:    subs r2, #32
+; THUMB78-NEXT:    lsr.w r0, r1, r2
+; THUMB78-NEXT:    subs.w r1, r2, #32
 ; THUMB78-NEXT:    it pl
 ; THUMB78-NEXT:    movpl r0, #0
-; THUMB78-NEXT:    ands r0, r1
-; THUMB78-NEXT:    clz r0, r0
-; THUMB78-NEXT:    lsrs r0, r0, #5
+; THUMB78-NEXT:    mvns r0, r0
+; THUMB78-NEXT:    lsrs r0, r0, #31
 ; THUMB78-NEXT:    bx lr
   %t0 = shl i64 9223372036854775808, %y
   %t1 = and i64 %t0, %x
@@ -442,94 +443,53 @@ define i1 @scalar_i64_signbit_eq(i64 %x, i64 %y) nounwind {
 }
 
 define i1 @scalar_i64_lowestbit_eq(i64 %x, i64 %y) nounwind {
-; ARM6-LABEL: scalar_i64_lowestbit_eq:
-; ARM6:       @ %bb.0:
-; ARM6-NEXT:    push {r11, lr}
-; ARM6-NEXT:    mov r12, #1
-; ARM6-NEXT:    subs lr, r2, #32
-; ARM6-NEXT:    lsl r3, r12, r2
-; ARM6-NEXT:    rsb r2, r2, #32
-; ARM6-NEXT:    movpl r3, #0
-; ARM6-NEXT:    and r0, r3, r0
-; ARM6-NEXT:    lsr r2, r12, r2
-; ARM6-NEXT:    lslpl r2, r12, lr
-; ARM6-NEXT:    and r1, r2, r1
-; ARM6-NEXT:    orr r0, r0, r1
-; ARM6-NEXT:    clz r0, r0
-; ARM6-NEXT:    lsr r0, r0, #5
-; ARM6-NEXT:    pop {r11, pc}
-;
-; ARM78-LABEL: scalar_i64_lowestbit_eq:
-; ARM78:       @ %bb.0:
-; ARM78-NEXT:    push {r11, lr}
-; ARM78-NEXT:    mov r12, #1
-; ARM78-NEXT:    subs lr, r2, #32
-; ARM78-NEXT:    lsl r3, r12, r2
-; ARM78-NEXT:    rsb r2, r2, #32
-; ARM78-NEXT:    movwpl r3, #0
-; ARM78-NEXT:    and r0, r3, r0
-; ARM78-NEXT:    lsr r2, r12, r2
-; ARM78-NEXT:    lslpl r2, r12, lr
-; ARM78-NEXT:    and r1, r2, r1
-; ARM78-NEXT:    orr r0, r0, r1
-; ARM78-NEXT:    clz r0, r0
-; ARM78-NEXT:    lsr r0, r0, #5
-; ARM78-NEXT:    pop {r11, pc}
+; ARM-LABEL: scalar_i64_lowestbit_eq:
+; ARM:       @ %bb.0:
+; ARM-NEXT:    rsb r3, r2, #32
+; ARM-NEXT:    lsr r0, r0, r2
+; ARM-NEXT:    subs r2, r2, #32
+; ARM-NEXT:    orr r0, r0, r1, lsl r3
+; ARM-NEXT:    lsrpl r0, r1, r2
+; ARM-NEXT:    mov r1, #1
+; ARM-NEXT:    bic r0, r1, r0
+; ARM-NEXT:    bx lr
 ;
 ; THUMB6-LABEL: scalar_i64_lowestbit_eq:
 ; THUMB6:       @ %bb.0:
-; THUMB6-NEXT:    push {r4, r5, r7, lr}
-; THUMB6-NEXT:    mov r4, r1
-; THUMB6-NEXT:    mov r5, r0
-; THUMB6-NEXT:    movs r0, #1
-; THUMB6-NEXT:    movs r1, #0
-; THUMB6-NEXT:    bl __ashldi3
-; THUMB6-NEXT:    ands r1, r4
-; THUMB6-NEXT:    ands r0, r5
-; THUMB6-NEXT:    orrs r0, r1
-; THUMB6-NEXT:    rsbs r1, r0, #0
+; THUMB6-NEXT:    push {r7, lr}
+; THUMB6-NEXT:    bl __lshrdi3
+; THUMB6-NEXT:    movs r1, #1
+; THUMB6-NEXT:    ands r1, r0
+; THUMB6-NEXT:    rsbs r0, r1, #0
 ; THUMB6-NEXT:    adcs r0, r1
-; THUMB6-NEXT:    pop {r4, r5, r7, pc}
+; THUMB6-NEXT:    pop {r7, pc}
 ;
 ; THUMB7-LABEL: scalar_i64_lowestbit_eq:
 ; THUMB7:       @ %bb.0:
-; THUMB7-NEXT:    push {r7, lr}
 ; THUMB7-NEXT:    rsb.w r3, r2, #32
-; THUMB7-NEXT:    mov.w r12, #1
-; THUMB7-NEXT:    subs.w lr, r2, #32
-; THUMB7-NEXT:    lsl.w r2, r12, r2
-; THUMB7-NEXT:    lsr.w r3, r12, r3
+; THUMB7-NEXT:    lsrs r0, r2
+; THUMB7-NEXT:    subs r2, #32
+; THUMB7-NEXT:    lsl.w r3, r1, r3
+; THUMB7-NEXT:    orr.w r0, r0, r3
 ; THUMB7-NEXT:    it pl
-; THUMB7-NEXT:    lslpl.w r3, r12, lr
-; THUMB7-NEXT:    it pl
-; THUMB7-NEXT:    movpl r2, #0
-; THUMB7-NEXT:    ands r1, r3
-; THUMB7-NEXT:    ands r0, r2
-; THUMB7-NEXT:    orrs r0, r1
-; THUMB7-NEXT:    clz r0, r0
-; THUMB7-NEXT:    lsrs r0, r0, #5
-; THUMB7-NEXT:    pop {r7, pc}
+; THUMB7-NEXT:    lsrpl.w r0, r1, r2
+; THUMB7-NEXT:    movs r1, #1
+; THUMB7-NEXT:    bic.w r0, r1, r0
+; THUMB7-NEXT:    bx lr
 ;
 ; THUMB8-LABEL: scalar_i64_lowestbit_eq:
 ; THUMB8:       @ %bb.0:
-; THUMB8-NEXT:    .save {r7, lr}
-; THUMB8-NEXT:    push {r7, lr}
-; THUMB8-NEXT:    subs.w r3, r2, #32
-; THUMB8-NEXT:    mov.w r12, #1
-; THUMB8-NEXT:    lsl.w lr, r12, r3
 ; THUMB8-NEXT:    rsb.w r3, r2, #32
-; THUMB8-NEXT:    lsl.w r2, r12, r2
-; THUMB8-NEXT:    lsr.w r3, r12, r3
-; THUMB8-NEXT:    it pl
-; THUMB8-NEXT:    movpl r3, lr
-; THUMB8-NEXT:    it pl
-; THUMB8-NEXT:    movpl r2, #0
-; THUMB8-NEXT:    ands r1, r3
-; THUMB8-NEXT:    ands r0, r2
-; THUMB8-NEXT:    orrs r0, r1
-; THUMB8-NEXT:    clz r0, r0
-; THUMB8-NEXT:    lsrs r0, r0, #5
-; THUMB8-NEXT:    pop {r7, pc}
+; THUMB8-NEXT:    lsrs r0, r2
+; THUMB8-NEXT:    lsl.w r3, r1, r3
+; THUMB8-NEXT:    orrs r0, r3
+; THUMB8-NEXT:    subs r2, #32
+; THUMB8-NEXT:    lsr.w r1, r1, r2
+; THUMB8-NEXT:    it mi
+; THUMB8-NEXT:    movmi r1, r0
+; THUMB8-NEXT:    movs r0, #1
+; THUMB8-NEXT:    bics r0, r1
+; THUMB8-NEXT:    bx lr
   %t0 = shl i64 1, %y
   %t1 = and i64 %t0, %x
   %res = icmp eq i64 %t1, 0
@@ -539,115 +499,82 @@ define i1 @scalar_i64_lowestbit_eq(i64 %x, i64 %y) nounwind {
 define i1 @scalar_i64_bitsinmiddle_eq(i64 %x, i64 %y) nounwind {
 ; ARM6-LABEL: scalar_i64_bitsinmiddle_eq:
 ; ARM6:       @ %bb.0:
-; ARM6-NEXT:    push {r4, lr}
-; ARM6-NEXT:    mov r12, #16711680
-; ARM6-NEXT:    subs lr, r2, #32
-; ARM6-NEXT:    orr r12, r12, #-16777216
-; ARM6-NEXT:    mov r4, #255
-; ARM6-NEXT:    orr r4, r4, #65280
-; ARM6-NEXT:    lsl r3, r12, r2
-; ARM6-NEXT:    movpl r3, #0
-; ARM6-NEXT:    and r0, r3, r0
 ; ARM6-NEXT:    rsb r3, r2, #32
-; ARM6-NEXT:    cmp lr, #0
-; ARM6-NEXT:    lsr r3, r12, r3
-; ARM6-NEXT:    orr r2, r3, r4, lsl r2
-; ARM6-NEXT:    lslpl r2, r12, lr
-; ARM6-NEXT:    and r1, r2, r1
-; ARM6-NEXT:    orr r0, r0, r1
+; ARM6-NEXT:    lsr r0, r0, r2
+; ARM6-NEXT:    orr r0, r0, r1, lsl r3
+; ARM6-NEXT:    subs r3, r2, #32
+; ARM6-NEXT:    lsrpl r0, r1, r3
+; ARM6-NEXT:    lsr r1, r1, r2
+; ARM6-NEXT:    movpl r1, #0
+; ARM6-NEXT:    pkhbt r0, r1, r0
 ; ARM6-NEXT:    clz r0, r0
 ; ARM6-NEXT:    lsr r0, r0, #5
-; ARM6-NEXT:    pop {r4, pc}
+; ARM6-NEXT:    bx lr
 ;
 ; ARM78-LABEL: scalar_i64_bitsinmiddle_eq:
 ; ARM78:       @ %bb.0:
-; ARM78-NEXT:    push {r4, lr}
-; ARM78-NEXT:    movw r12, #0
-; ARM78-NEXT:    subs lr, r2, #32
-; ARM78-NEXT:    movt r12, #65535
-; ARM78-NEXT:    movw r4, #65535
-; ARM78-NEXT:    lsl r3, r12, r2
-; ARM78-NEXT:    movwpl r3, #0
-; ARM78-NEXT:    and r0, r3, r0
 ; ARM78-NEXT:    rsb r3, r2, #32
-; ARM78-NEXT:    cmp lr, #0
-; ARM78-NEXT:    lsr r3, r12, r3
-; ARM78-NEXT:    orr r2, r3, r4, lsl r2
-; ARM78-NEXT:    lslpl r2, r12, lr
-; ARM78-NEXT:    and r1, r2, r1
-; ARM78-NEXT:    orr r0, r0, r1
+; ARM78-NEXT:    lsr r0, r0, r2
+; ARM78-NEXT:    orr r0, r0, r1, lsl r3
+; ARM78-NEXT:    subs r3, r2, #32
+; ARM78-NEXT:    lsrpl r0, r1, r3
+; ARM78-NEXT:    lsr r1, r1, r2
+; ARM78-NEXT:    movwpl r1, #0
+; ARM78-NEXT:    pkhbt r0, r1, r0
 ; ARM78-NEXT:    clz r0, r0
 ; ARM78-NEXT:    lsr r0, r0, #5
-; ARM78-NEXT:    pop {r4, pc}
+; ARM78-NEXT:    bx lr
 ;
 ; THUMB6-LABEL: scalar_i64_bitsinmiddle_eq:
 ; THUMB6:       @ %bb.0:
-; THUMB6-NEXT:    push {r4, r5, r7, lr}
-; THUMB6-NEXT:    mov r4, r1
-; THUMB6-NEXT:    mov r5, r0
-; THUMB6-NEXT:    ldr r0, .LCPI11_0
-; THUMB6-NEXT:    ldr r1, .LCPI11_1
-; THUMB6-NEXT:    bl __ashldi3
-; THUMB6-NEXT:    ands r1, r4
-; THUMB6-NEXT:    ands r0, r5
-; THUMB6-NEXT:    orrs r0, r1
-; THUMB6-NEXT:    rsbs r1, r0, #0
+; THUMB6-NEXT:    push {r7, lr}
+; THUMB6-NEXT:    bl __lshrdi3
+; THUMB6-NEXT:    ldr r2, .LCPI11_0
+; THUMB6-NEXT:    ands r2, r0
+; THUMB6-NEXT:    uxth r0, r1
+; THUMB6-NEXT:    adds r1, r2, r0
+; THUMB6-NEXT:    rsbs r0, r1, #0
 ; THUMB6-NEXT:    adcs r0, r1
-; THUMB6-NEXT:    pop {r4, r5, r7, pc}
+; THUMB6-NEXT:    pop {r7, pc}
 ; THUMB6-NEXT:    .p2align 2
 ; THUMB6-NEXT:  @ %bb.1:
 ; THUMB6-NEXT:  .LCPI11_0:
 ; THUMB6-NEXT:    .long 4294901760 @ 0xffff0000
-; THUMB6-NEXT:  .LCPI11_1:
-; THUMB6-NEXT:    .long 65535 @ 0xffff
 ;
 ; THUMB7-LABEL: scalar_i64_bitsinmiddle_eq:
 ; THUMB7:       @ %bb.0:
-; THUMB7-NEXT:    push {r7, lr}
-; THUMB7-NEXT:    movw r3, #65535
-; THUMB7-NEXT:    movw lr, #0
-; THUMB7-NEXT:    lsl.w r12, r3, r2
 ; THUMB7-NEXT:    rsb.w r3, r2, #32
-; THUMB7-NEXT:    movt lr, #65535
-; THUMB7-NEXT:    lsr.w r3, lr, r3
-; THUMB7-NEXT:    orr.w r12, r12, r3
+; THUMB7-NEXT:    lsrs r0, r2
+; THUMB7-NEXT:    lsl.w r3, r1, r3
+; THUMB7-NEXT:    orrs r0, r3
 ; THUMB7-NEXT:    subs.w r3, r2, #32
-; THUMB7-NEXT:    lsl.w r2, lr, r2
 ; THUMB7-NEXT:    it pl
-; THUMB7-NEXT:    lslpl.w r12, lr, r3
+; THUMB7-NEXT:    lsrpl.w r0, r1, r3
+; THUMB7-NEXT:    lsr.w r1, r1, r2
 ; THUMB7-NEXT:    it pl
-; THUMB7-NEXT:    movpl r2, #0
-; THUMB7-NEXT:    and.w r1, r1, r12
-; THUMB7-NEXT:    ands r0, r2
-; THUMB7-NEXT:    orrs r0, r1
+; THUMB7-NEXT:    movpl r1, #0
+; THUMB7-NEXT:    pkhbt r0, r1, r0
 ; THUMB7-NEXT:    clz r0, r0
 ; THUMB7-NEXT:    lsrs r0, r0, #5
-; THUMB7-NEXT:    pop {r7, pc}
+; THUMB7-NEXT:    bx lr
 ;
 ; THUMB8-LABEL: scalar_i64_bitsinmiddle_eq:
 ; THUMB8:       @ %bb.0:
-; THUMB8-NEXT:    .save {r7, lr}
-; THUMB8-NEXT:    push {r7, lr}
-; THUMB8-NEXT:    movw r3, #65535
-; THUMB8-NEXT:    movw lr, #0
-; THUMB8-NEXT:    lsl.w r12, r3, r2
 ; THUMB8-NEXT:    rsb.w r3, r2, #32
-; THUMB8-NEXT:    movt lr, #65535
-; THUMB8-NEXT:    lsr.w r3, lr, r3
-; THUMB8-NEXT:    orr.w r12, r12, r3
+; THUMB8-NEXT:    lsrs r0, r2
+; THUMB8-NEXT:    lsl.w r3, r1, r3
+; THUMB8-NEXT:    orrs r0, r3
 ; THUMB8-NEXT:    subs.w r3, r2, #32
-; THUMB8-NEXT:    lsl.w r2, lr, r2
-; THUMB8-NEXT:    lsl.w r3, lr, r3
+; THUMB8-NEXT:    lsr.w r3, r1, r3
 ; THUMB8-NEXT:    it mi
-; THUMB8-NEXT:    movmi r3, r12
+; THUMB8-NEXT:    movmi r3, r0
+; THUMB8-NEXT:    lsr.w r0, r1, r2
 ; THUMB8-NEXT:    it pl
-; THUMB8-NEXT:    movpl r2, #0
-; THUMB8-NEXT:    ands r1, r3
-; THUMB8-NEXT:    ands r0, r2
-; THUMB8-NEXT:    orrs r0, r1
+; THUMB8-NEXT:    movpl r0, #0
+; THUMB8-NEXT:    pkhbt r0, r0, r3
 ; THUMB8-NEXT:    clz r0, r0
 ; THUMB8-NEXT:    lsrs r0, r0, #5
-; THUMB8-NEXT:    pop {r7, pc}
+; THUMB8-NEXT:    bx lr
   %t0 = shl i64 281474976645120, %y
   %t1 = and i64 %t0, %x
   %res = icmp eq i64 %t1, 0
@@ -664,32 +591,25 @@ define <4 x i1> @vec_4xi32_splat_eq(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; ARM6-NEXT:    push {r11, lr}
 ; ARM6-NEXT:    ldr r12, [sp, #8]
 ; ARM6-NEXT:    mov lr, #1
-; ARM6-NEXT:    and r0, r0, lr, lsl r12
+; ARM6-NEXT:    bic r0, lr, r0, lsr r12
 ; ARM6-NEXT:    ldr r12, [sp, #12]
-; ARM6-NEXT:    clz r0, r0
-; ARM6-NEXT:    and r1, r1, lr, lsl r12
+; ARM6-NEXT:    bic r1, lr, r1, lsr r12
 ; ARM6-NEXT:    ldr r12, [sp, #16]
-; ARM6-NEXT:    clz r1, r1
-; ARM6-NEXT:    lsr r0, r0, #5
-; ARM6-NEXT:    and r2, r2, lr, lsl r12
+; ARM6-NEXT:    bic r2, lr, r2, lsr r12
 ; ARM6-NEXT:    ldr r12, [sp, #20]
-; ARM6-NEXT:    clz r2, r2
-; ARM6-NEXT:    lsr r1, r1, #5
-; ARM6-NEXT:    and r3, r3, lr, lsl r12
-; ARM6-NEXT:    lsr r2, r2, #5
-; ARM6-NEXT:    clz r3, r3
-; ARM6-NEXT:    lsr r3, r3, #5
+; ARM6-NEXT:    bic r3, lr, r3, lsr r12
 ; ARM6-NEXT:    pop {r11, pc}
 ;
 ; ARM78-LABEL: vec_4xi32_splat_eq:
 ; ARM78:       @ %bb.0:
-; ARM78-NEXT:    vmov.i32 q8, #0x1
 ; ARM78-NEXT:    mov r12, sp
-; ARM78-NEXT:    vld1.64 {d18, d19}, [r12]
-; ARM78-NEXT:    vshl.u32 q8, q8, q9
+; ARM78-NEXT:    vld1.64 {d16, d17}, [r12]
 ; ARM78-NEXT:    vmov d19, r2, r3
+; ARM78-NEXT:    vneg.s32 q8, q8
 ; ARM78-NEXT:    vmov d18, r0, r1
-; ARM78-NEXT:    vtst.32 q8, q8, q9
+; ARM78-NEXT:    vmov.i32 q10, #0x1
+; ARM78-NEXT:    vshl.u32 q8, q9, q8
+; ARM78-NEXT:    vtst.32 q8, q8, q10
 ; ARM78-NEXT:    vmvn q8, q8
 ; ARM78-NEXT:    vmovn.i32 d16, q8
 ; ARM78-NEXT:    vmov r0, r1, d16
@@ -697,42 +617,40 @@ define <4 x i1> @vec_4xi32_splat_eq(<4 x i32> %x, <4 x i32> %y) nounwind {
 ;
 ; THUMB6-LABEL: vec_4xi32_splat_eq:
 ; THUMB6:       @ %bb.0:
-; THUMB6-NEXT:    push {r4, r5, r6, lr}
-; THUMB6-NEXT:    ldr r5, [sp, #16]
+; THUMB6-NEXT:    push {r4, r5, r7, lr}
+; THUMB6-NEXT:    ldr r4, [sp, #16]
+; THUMB6-NEXT:    lsrs r0, r4
 ; THUMB6-NEXT:    movs r4, #1
-; THUMB6-NEXT:    mov r6, r4
-; THUMB6-NEXT:    lsls r6, r5
-; THUMB6-NEXT:    ands r6, r0
-; THUMB6-NEXT:    rsbs r0, r6, #0
-; THUMB6-NEXT:    adcs r0, r6
+; THUMB6-NEXT:    ands r0, r4
+; THUMB6-NEXT:    rsbs r5, r0, #0
+; THUMB6-NEXT:    adcs r0, r5
 ; THUMB6-NEXT:    ldr r5, [sp, #20]
-; THUMB6-NEXT:    mov r6, r4
-; THUMB6-NEXT:    lsls r6, r5
-; THUMB6-NEXT:    ands r6, r1
-; THUMB6-NEXT:    rsbs r1, r6, #0
-; THUMB6-NEXT:    adcs r1, r6
+; THUMB6-NEXT:    lsrs r1, r5
+; THUMB6-NEXT:    ands r1, r4
+; THUMB6-NEXT:    rsbs r5, r1, #0
+; THUMB6-NEXT:    adcs r1, r5
 ; THUMB6-NEXT:    ldr r5, [sp, #24]
-; THUMB6-NEXT:    mov r6, r4
-; THUMB6-NEXT:    lsls r6, r5
-; THUMB6-NEXT:    ands r6, r2
-; THUMB6-NEXT:    rsbs r2, r6, #0
-; THUMB6-NEXT:    adcs r2, r6
+; THUMB6-NEXT:    lsrs r2, r5
+; THUMB6-NEXT:    ands r2, r4
+; THUMB6-NEXT:    rsbs r5, r2, #0
+; THUMB6-NEXT:    adcs r2, r5
 ; THUMB6-NEXT:    ldr r5, [sp, #28]
-; THUMB6-NEXT:    lsls r4, r5
-; THUMB6-NEXT:    ands r4, r3
-; THUMB6-NEXT:    rsbs r3, r4, #0
+; THUMB6-NEXT:    lsrs r3, r5
+; THUMB6-NEXT:    ands r3, r4
+; THUMB6-NEXT:    rsbs r4, r3, #0
 ; THUMB6-NEXT:    adcs r3, r4
-; THUMB6-NEXT:    pop {r4, r5, r6, pc}
+; THUMB6-NEXT:    pop {r4, r5, r7, pc}
 ;
 ; THUMB78-LABEL: vec_4xi32_splat_eq:
 ; THUMB78:       @ %bb.0:
-; THUMB78-NEXT:    vmov.i32 q8, #0x1
 ; THUMB78-NEXT:    mov r12, sp
-; THUMB78-NEXT:    vld1.64 {d18, d19}, [r12]
-; THUMB78-NEXT:    vshl.u32 q8, q8, q9
+; THUMB78-NEXT:    vld1.64 {d16, d17}, [r12]
 ; THUMB78-NEXT:    vmov d19, r2, r3
+; THUMB78-NEXT:    vneg.s32 q8, q8
 ; THUMB78-NEXT:    vmov d18, r0, r1
-; THUMB78-NEXT:    vtst.32 q8, q8, q9
+; THUMB78-NEXT:    vmov.i32 q10, #0x1
+; THUMB78-NEXT:    vshl.u32 q8, q9, q8
+; THUMB78-NEXT:    vtst.32 q8, q8, q10
 ; THUMB78-NEXT:    vmvn q8, q8
 ; THUMB78-NEXT:    vmovn.i32 d16, q8
 ; THUMB78-NEXT:    vmov r0, r1, d16
@@ -748,20 +666,16 @@ define <4 x i1> @vec_4xi32_nonsplat_eq(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; ARM6:       @ %bb.0:
 ; ARM6-NEXT:    ldr r12, [sp, #4]
 ; ARM6-NEXT:    mov r0, #1
-; ARM6-NEXT:    and r0, r1, r0, lsl r12
+; ARM6-NEXT:    bic r1, r0, r1, lsr r12
 ; ARM6-NEXT:    ldr r12, [sp, #8]
-; ARM6-NEXT:    clz r0, r0
-; ARM6-NEXT:    lsr r1, r0, #5
 ; ARM6-NEXT:    mov r0, #65280
 ; ARM6-NEXT:    orr r0, r0, #16711680
-; ARM6-NEXT:    and r0, r2, r0, lsl r12
-; ARM6-NEXT:    ldr r12, [sp, #12]
+; ARM6-NEXT:    and r0, r0, r2, lsr r12
 ; ARM6-NEXT:    clz r0, r0
 ; ARM6-NEXT:    lsr r2, r0, #5
-; ARM6-NEXT:    mov r0, #-2147483648
-; ARM6-NEXT:    and r0, r3, r0, lsl r12
-; ARM6-NEXT:    clz r0, r0
-; ARM6-NEXT:    lsr r3, r0, #5
+; ARM6-NEXT:    ldr r0, [sp, #12]
+; ARM6-NEXT:    mvn r0, r3, lsr r0
+; ARM6-NEXT:    lsr r3, r0, #31
 ; ARM6-NEXT:    mov r0, #1
 ; ARM6-NEXT:    bx lr
 ;
@@ -789,27 +703,26 @@ define <4 x i1> @vec_4xi32_nonsplat_eq(<4 x i32> %x, <4 x i32> %y) nounwind {
 ;
 ; THUMB6-LABEL: vec_4xi32_nonsplat_eq:
 ; THUMB6:       @ %bb.0:
-; THUMB6-NEXT:    push {r4, r5, r7, lr}
-; THUMB6-NEXT:    ldr r4, [sp, #20]
+; THUMB6-NEXT:    push {r4, lr}
+; THUMB6-NEXT:    ldr r0, [sp, #12]
+; THUMB6-NEXT:    lsrs r1, r0
 ; THUMB6-NEXT:    movs r0, #1
-; THUMB6-NEXT:    mov r5, r0
-; THUMB6-NEXT:    lsls r5, r4
-; THUMB6-NEXT:    ands r5, r1
-; THUMB6-NEXT:    rsbs r1, r5, #0
-; THUMB6-NEXT:    adcs r1, r5
-; THUMB6-NEXT:    ldr r4, [sp, #24]
-; THUMB6-NEXT:    ldr r5, .LCPI13_0
-; THUMB6-NEXT:    lsls r5, r4
-; THUMB6-NEXT:    ands r5, r2
-; THUMB6-NEXT:    rsbs r2, r5, #0
-; THUMB6-NEXT:    adcs r2, r5
+; THUMB6-NEXT:    ands r1, r0
+; THUMB6-NEXT:    rsbs r4, r1, #0
+; THUMB6-NEXT:    adcs r1, r4
+; THUMB6-NEXT:    ldr r4, [sp, #16]
+; THUMB6-NEXT:    lsrs r2, r4
+; THUMB6-NEXT:    ldr r4, .LCPI13_0
+; THUMB6-NEXT:    ands r4, r2
+; THUMB6-NEXT:    rsbs r2, r4, #0
+; THUMB6-NEXT:    adcs r2, r4
+; THUMB6-NEXT:    ldr r4, [sp, #20]
+; THUMB6-NEXT:    lsrs r3, r4
 ; THUMB6-NEXT:    lsls r4, r0, #31
-; THUMB6-NEXT:    ldr r5, [sp, #28]
-; THUMB6-NEXT:    lsls r4, r5
 ; THUMB6-NEXT:    ands r4, r3
 ; THUMB6-NEXT:    rsbs r3, r4, #0
 ; THUMB6-NEXT:    adcs r3, r4
-; THUMB6-NEXT:    pop {r4, r5, r7, pc}
+; THUMB6-NEXT:    pop {r4, pc}
 ; THUMB6-NEXT:    .p2align 2
 ; THUMB6-NEXT:  @ %bb.1:
 ; THUMB6-NEXT:  .LCPI13_0:
@@ -849,28 +762,23 @@ define <4 x i1> @vec_4xi32_nonsplat_undef0_eq(<4 x i32> %x, <4 x i32> %y) nounwi
 ; ARM6-NEXT:    ldr r2, [sp, #12]
 ; ARM6-NEXT:    mov lr, #1
 ; ARM6-NEXT:    ldr r12, [sp, #8]
-; ARM6-NEXT:    and r1, r1, lr, lsl r2
+; ARM6-NEXT:    bic r1, lr, r1, lsr r2
 ; ARM6-NEXT:    ldr r2, [sp, #20]
-; ARM6-NEXT:    and r0, r0, lr, lsl r12
-; ARM6-NEXT:    clz r1, r1
-; ARM6-NEXT:    clz r0, r0
-; ARM6-NEXT:    and r2, r3, lr, lsl r2
-; ARM6-NEXT:    lsr r1, r1, #5
-; ARM6-NEXT:    clz r2, r2
-; ARM6-NEXT:    lsr r0, r0, #5
-; ARM6-NEXT:    lsr r3, r2, #5
+; ARM6-NEXT:    bic r0, lr, r0, lsr r12
+; ARM6-NEXT:    bic r3, lr, r3, lsr r2
 ; ARM6-NEXT:    mov r2, #1
 ; ARM6-NEXT:    pop {r11, pc}
 ;
 ; ARM78-LABEL: vec_4xi32_nonsplat_undef0_eq:
 ; ARM78:       @ %bb.0:
-; ARM78-NEXT:    vmov.i32 q8, #0x1
 ; ARM78-NEXT:    mov r12, sp
-; ARM78-NEXT:    vld1.64 {d18, d19}, [r12]
-; ARM78-NEXT:    vshl.u32 q8, q8, q9
+; ARM78-NEXT:    vld1.64 {d16, d17}, [r12]
 ; ARM78-NEXT:    vmov d19, r2, r3
+; ARM78-NEXT:    vneg.s32 q8, q8
 ; ARM78-NEXT:    vmov d18, r0, r1
-; ARM78-NEXT:    vtst.32 q8, q8, q9
+; ARM78-NEXT:    vmov.i32 q10, #0x1
+; ARM78-NEXT:    vshl.u32 q8, q9, q8
+; ARM78-NEXT:    vtst.32 q8, q8, q10
 ; ARM78-NEXT:    vmvn q8, q8
 ; ARM78-NEXT:    vmovn.i32 d16, q8
 ; ARM78-NEXT:    vmov r0, r1, d16
@@ -878,37 +786,35 @@ define <4 x i1> @vec_4xi32_nonsplat_undef0_eq(<4 x i32> %x, <4 x i32> %y) nounwi
 ;
 ; THUMB6-LABEL: vec_4xi32_nonsplat_undef0_eq:
 ; THUMB6:       @ %bb.0:
-; THUMB6-NEXT:    push {r4, r5, r7, lr}
-; THUMB6-NEXT:    ldr r4, [sp, #16]
+; THUMB6-NEXT:    push {r4, lr}
+; THUMB6-NEXT:    ldr r2, [sp, #8]
+; THUMB6-NEXT:    lsrs r0, r2
 ; THUMB6-NEXT:    movs r2, #1
-; THUMB6-NEXT:    mov r5, r2
-; THUMB6-NEXT:    lsls r5, r4
-; THUMB6-NEXT:    ands r5, r0
-; THUMB6-NEXT:    rsbs r0, r5, #0
-; THUMB6-NEXT:    adcs r0, r5
+; THUMB6-NEXT:    ands r0, r2
+; THUMB6-NEXT:    rsbs r4, r0, #0
+; THUMB6-NEXT:    adcs r0, r4
+; THUMB6-NEXT:    ldr r4, [sp, #12]
+; THUMB6-NEXT:    lsrs r1, r4
+; THUMB6-NEXT:    ands r1, r2
+; THUMB6-NEXT:    rsbs r4, r1, #0
+; THUMB6-NEXT:    adcs r1, r4
 ; THUMB6-NEXT:    ldr r4, [sp, #20]
-; THUMB6-NEXT:    mov r5, r2
-; THUMB6-NEXT:    lsls r5, r4
-; THUMB6-NEXT:    ands r5, r1
-; THUMB6-NEXT:    rsbs r1, r5, #0
-; THUMB6-NEXT:    adcs r1, r5
-; THUMB6-NEXT:    ldr r4, [sp, #28]
-; THUMB6-NEXT:    mov r5, r2
-; THUMB6-NEXT:    lsls r5, r4
-; THUMB6-NEXT:    ands r5, r3
-; THUMB6-NEXT:    rsbs r3, r5, #0
-; THUMB6-NEXT:    adcs r3, r5
-; THUMB6-NEXT:    pop {r4, r5, r7, pc}
+; THUMB6-NEXT:    lsrs r3, r4
+; THUMB6-NEXT:    ands r3, r2
+; THUMB6-NEXT:    rsbs r4, r3, #0
+; THUMB6-NEXT:    adcs r3, r4
+; THUMB6-NEXT:    pop {r4, pc}
 ;
 ; THUMB78-LABEL: vec_4xi32_nonsplat_undef0_eq:
 ; THUMB78:       @ %bb.0:
-; THUMB78-NEXT:    vmov.i32 q8, #0x1
 ; THUMB78-NEXT:    mov r12, sp
-; THUMB78-NEXT:    vld1.64 {d18, d19}, [r12]
-; THUMB78-NEXT:    vshl.u32 q8, q8, q9
+; THUMB78-NEXT:    vld1.64 {d16, d17}, [r12]
 ; THUMB78-NEXT:    vmov d19, r2, r3
+; THUMB78-NEXT:    vneg.s32 q8, q8
 ; THUMB78-NEXT:    vmov d18, r0, r1
-; THUMB78-NEXT:    vtst.32 q8, q8, q9
+; THUMB78-NEXT:    vmov.i32 q10, #0x1
+; THUMB78-NEXT:    vshl.u32 q8, q9, q8
+; THUMB78-NEXT:    vtst.32 q8, q8, q10
 ; THUMB78-NEXT:    vmvn q8, q8
 ; THUMB78-NEXT:    vmovn.i32 d16, q8
 ; THUMB78-NEXT:    vmov r0, r1, d16
@@ -925,16 +831,10 @@ define <4 x i1> @vec_4xi32_nonsplat_undef1_eq(<4 x i32> %x, <4 x i32> %y) nounwi
 ; ARM6-NEXT:    ldr r2, [sp, #12]
 ; ARM6-NEXT:    mov lr, #1
 ; ARM6-NEXT:    ldr r12, [sp, #8]
-; ARM6-NEXT:    and r1, r1, lr, lsl r2
+; ARM6-NEXT:    bic r1, lr, r1, lsr r2
 ; ARM6-NEXT:    ldr r2, [sp, #20]
-; ARM6-NEXT:    and r0, r0, lr, lsl r12
-; ARM6-NEXT:    clz r1, r1
-; ARM6-NEXT:    clz r0, r0
-; ARM6-NEXT:    and r2, r3, lr, lsl r2
-; ARM6-NEXT:    lsr r1, r1, #5
-; ARM6-NEXT:    clz r2, r2
-; ARM6-NEXT:    lsr r0, r0, #5
-; ARM6-NEXT:    lsr r3, r2, #5
+; ARM6-NEXT:    bic r0, lr, r0, lsr r12
+; ARM6-NEXT:    bic r3, lr, r3, lsr r2
 ; ARM6-NEXT:    pop {r11, pc}
 ;
 ; ARM78-LABEL: vec_4xi32_nonsplat_undef1_eq:
@@ -953,26 +853,24 @@ define <4 x i1> @vec_4xi32_nonsplat_undef1_eq(<4 x i32> %x, <4 x i32> %y) nounwi
 ;
 ; THUMB6-LABEL: vec_4xi32_nonsplat_undef1_eq:
 ; THUMB6:       @ %bb.0:
-; THUMB6-NEXT:    push {r4, r5, r7, lr}
-; THUMB6-NEXT:    ldr r4, [sp, #16]
+; THUMB6-NEXT:    push {r4, lr}
+; THUMB6-NEXT:    ldr r2, [sp, #8]
+; THUMB6-NEXT:    lsrs r0, r2
 ; THUMB6-NEXT:    movs r2, #1
-; THUMB6-NEXT:    mov r5, r2
-; THUMB6-NEXT:    lsls r5, r4
-; THUMB6-NEXT:    ands r5, r0
-; THUMB6-NEXT:    rsbs r0, r5, #0
-; THUMB6-NEXT:    adcs r0, r5
+; THUMB6-NEXT:    ands r0, r2
+; THUMB6-NEXT:    rsbs r4, r0, #0
+; THUMB6-NEXT:    adcs r0, r4
+; THUMB6-NEXT:    ldr r4, [sp, #12]
+; THUMB6-NEXT:    lsrs r1, r4
+; THUMB6-NEXT:    ands r1, r2
+; THUMB6-NEXT:    rsbs r4, r1, #0
+; THUMB6-NEXT:    adcs r1, r4
 ; THUMB6-NEXT:    ldr r4, [sp, #20]
-; THUMB6-NEXT:    mov r5, r2
-; THUMB6-NEXT:    lsls r5, r4
-; THUMB6-NEXT:    ands r5, r1
-; THUMB6-NEXT:    rsbs r1, r5, #0
-; THUMB6-NEXT:    adcs r1, r5
-; THUMB6-NEXT:    ldr r4, [sp, #28]
-; THUMB6-NEXT:    lsls r2, r4
-; THUMB6-NEXT:    ands r2, r3
-; THUMB6-NEXT:    rsbs r3, r2, #0
+; THUMB6-NEXT:    lsrs r3, r4
+; THUMB6-NEXT:    ands r3, r2
+; THUMB6-NEXT:    rsbs r2, r3, #0
 ; THUMB6-NEXT:    adcs r3, r2
-; THUMB6-NEXT:    pop {r4, r5, r7, pc}
+; THUMB6-NEXT:    pop {r4, pc}
 ;
 ; THUMB78-LABEL: vec_4xi32_nonsplat_undef1_eq:
 ; THUMB78:       @ %bb.0:
@@ -999,16 +897,10 @@ define <4 x i1> @vec_4xi32_nonsplat_undef2_eq(<4 x i32> %x, <4 x i32> %y) nounwi
 ; ARM6-NEXT:    ldr r2, [sp, #12]
 ; ARM6-NEXT:    mov lr, #1
 ; ARM6-NEXT:    ldr r12, [sp, #8]
-; ARM6-NEXT:    and r1, r1, lr, lsl r2
+; ARM6-NEXT:    bic r1, lr, r1, lsr r2
 ; ARM6-NEXT:    ldr r2, [sp, #20]
-; ARM6-NEXT:    and r0, r0, lr, lsl r12
-; ARM6-NEXT:    clz r1, r1
-; ARM6-NEXT:    clz r0, r0
-; ARM6-NEXT:    and r2, r3, lr, lsl r2
-; ARM6-NEXT:    lsr r1, r1, #5
-; ARM6-NEXT:    clz r2, r2
-; ARM6-NEXT:    lsr r0, r0, #5
-; ARM6-NEXT:    lsr r3, r2, #5
+; ARM6-NEXT:    bic r0, lr, r0, lsr r12
+; ARM6-NEXT:    bic r3, lr, r3, lsr r2
 ; ARM6-NEXT:    pop {r11, pc}
 ;
 ; ARM78-LABEL: vec_4xi32_nonsplat_undef2_eq:
@@ -1027,26 +919,24 @@ define <4 x i1> @vec_4xi32_nonsplat_undef2_eq(<4 x i32> %x, <4 x i32> %y) nounwi
 ;
 ; THUMB6-LABEL: vec_4xi32_nonsplat_undef2_eq:
 ; THUMB6:       @ %bb.0:
-; THUMB6-NEXT:    push {r4, r5, r7, lr}
-; THUMB6-NEXT:    ldr r4, [sp, #16]
+; THUMB6-NEXT:    push {r4, lr}
+; THUMB6-NEXT:    ldr r2, [sp, #8]
+; THUMB6-NEXT:    lsrs r0, r2
 ; THUMB6-NEXT:    movs r2, #1
-; THUMB6-NEXT:    mov r5, r2
-; THUMB6-NEXT:    lsls r5, r4
-; THUMB6-NEXT:    ands r5, r0
-; THUMB6-NEXT:    rsbs r0, r5, #0
-; THUMB6-NEXT:    adcs r0, r5
+; THUMB6-NEXT:    ands r0, r2
+; THUMB6-NEXT:    rsbs r4, r0, #0
+; THUMB6-NEXT:    adcs r0, r4
+; THUMB6-NEXT:    ldr r4, [sp, #12]
+; THUMB6-NEXT:    lsrs r1, r4
+; THUMB6-NEXT:    ands r1, r2
+; THUMB6-NEXT:    rsbs r4, r1, #0
+; THUMB6-NEXT:    adcs r1, r4
 ; THUMB6-NEXT:    ldr r4, [sp, #20]
-; THUMB6-NEXT:    mov r5, r2
-; THUMB6-NEXT:    lsls r5, r4
-; THUMB6-NEXT:    ands r5, r1
-; THUMB6-NEXT:    rsbs r1, r5, #0
-; THUMB6-NEXT:    adcs r1, r5
-; THUMB6-NEXT:    ldr r4, [sp, #28]
-; THUMB6-NEXT:    lsls r2, r4
-; THUMB6-NEXT:    ands r2, r3
-; THUMB6-NEXT:    rsbs r3, r2, #0
+; THUMB6-NEXT:    lsrs r3, r4
+; THUMB6-NEXT:    ands r3, r2
+; THUMB6-NEXT:    rsbs r2, r3, #0
 ; THUMB6-NEXT:    adcs r3, r2
-; THUMB6-NEXT:    pop {r4, r5, r7, pc}
+; THUMB6-NEXT:    pop {r4, pc}
 ;
 ; THUMB78-LABEL: vec_4xi32_nonsplat_undef2_eq:
 ; THUMB78:       @ %bb.0:
@@ -1072,49 +962,37 @@ define <4 x i1> @vec_4xi32_nonsplat_undef2_eq(<4 x i32> %x, <4 x i32> %y) nounwi
 ;------------------------------------------------------------------------------;
 
 define i1 @scalar_i8_signbit_ne(i8 %x, i8 %y) nounwind {
-; ARM6-LABEL: scalar_i8_signbit_ne:
-; ARM6:       @ %bb.0:
-; ARM6-NEXT:    uxtb r1, r1
-; ARM6-NEXT:    mvn r2, #127
-; ARM6-NEXT:    and r0, r0, r2, lsl r1
-; ARM6-NEXT:    uxtb r0, r0
-; ARM6-NEXT:    cmp r0, #0
-; ARM6-NEXT:    movne r0, #1
-; ARM6-NEXT:    bx lr
-;
-; ARM78-LABEL: scalar_i8_signbit_ne:
-; ARM78:       @ %bb.0:
-; ARM78-NEXT:    uxtb r1, r1
-; ARM78-NEXT:    mvn r2, #127
-; ARM78-NEXT:    and r0, r0, r2, lsl r1
-; ARM78-NEXT:    uxtb r0, r0
-; ARM78-NEXT:    cmp r0, #0
-; ARM78-NEXT:    movwne r0, #1
-; ARM78-NEXT:    bx lr
+; ARM-LABEL: scalar_i8_signbit_ne:
+; ARM:       @ %bb.0:
+; ARM-NEXT:    uxtb r1, r1
+; ARM-NEXT:    uxtb r0, r0
+; ARM-NEXT:    lsr r0, r0, r1
+; ARM-NEXT:    lsr r0, r0, #7
+; ARM-NEXT:    bx lr
 ;
 ; THUMB6-LABEL: scalar_i8_signbit_ne:
 ; THUMB6:       @ %bb.0:
 ; THUMB6-NEXT:    uxtb r1, r1
-; THUMB6-NEXT:    movs r2, #127
-; THUMB6-NEXT:    mvns r2, r2
-; THUMB6-NEXT:    lsls r2, r1
-; THUMB6-NEXT:    ands r2, r0
-; THUMB6-NEXT:    uxtb r0, r2
-; THUMB6-NEXT:    subs r1, r0, #1
-; THUMB6-NEXT:    sbcs r0, r1
+; THUMB6-NEXT:    uxtb r0, r0
+; THUMB6-NEXT:    lsrs r0, r1
+; THUMB6-NEXT:    lsrs r0, r0, #7
 ; THUMB6-NEXT:    bx lr
 ;
-; THUMB78-LABEL: scalar_i8_signbit_ne:
-; THUMB78:       @ %bb.0:
-; THUMB78-NEXT:    uxtb r1, r1
-; THUMB78-NEXT:    mvn r2, #127
-; THUMB78-NEXT:    lsl.w r1, r2, r1
-; THUMB78-NEXT:    ands r0, r1
-; THUMB78-NEXT:    uxtb r0, r0
-; THUMB78-NEXT:    cmp r0, #0
-; THUMB78-NEXT:    it ne
-; THUMB78-NEXT:    movne r0, #1
-; THUMB78-NEXT:    bx lr
+; THUMB7-LABEL: scalar_i8_signbit_ne:
+; THUMB7:       @ %bb.0:
+; THUMB7-NEXT:    uxtb r1, r1
+; THUMB7-NEXT:    uxtb r0, r0
+; THUMB7-NEXT:    lsrs r0, r1
+; THUMB7-NEXT:    lsrs r0, r0, #7
+; THUMB7-NEXT:    bx lr
+;
+; THUMB8-LABEL: scalar_i8_signbit_ne:
+; THUMB8:       @ %bb.0:
+; THUMB8-NEXT:    uxtb r0, r0
+; THUMB8-NEXT:    uxtb r1, r1
+; THUMB8-NEXT:    lsrs r0, r1
+; THUMB8-NEXT:    lsrs r0, r0, #7
+; THUMB8-NEXT:    bx lr
   %t0 = shl i8 128, %y
   %t1 = and i8 %t0, %x
   %res = icmp ne i8 %t1, 0 ;  we are perfectly happy with 'ne' predicate
@@ -1290,13 +1168,7 @@ define i1 @negative_scalar_i8_bitsinmiddle_slt(i8 %x, i8 %y) nounwind {
 define i1 @scalar_i8_signbit_eq_with_nonzero(i8 %x, i8 %y) nounwind {
 ; ARM-LABEL: scalar_i8_signbit_eq_with_nonzero:
 ; ARM:       @ %bb.0:
-; ARM-NEXT:    uxtb r1, r1
-; ARM-NEXT:    mvn r2, #127
-; ARM-NEXT:    and r0, r0, r2, lsl r1
-; ARM-NEXT:    mvn r1, #0
-; ARM-NEXT:    uxtab r0, r1, r0
-; ARM-NEXT:    clz r0, r0
-; ARM-NEXT:    lsr r0, r0, #5
+; ARM-NEXT:    mov r0, #0
 ; ARM-NEXT:    bx lr
 ;
 ; THUMB6-LABEL: scalar_i8_signbit_eq_with_nonzero:
@@ -1314,14 +1186,7 @@ define i1 @scalar_i8_signbit_eq_with_nonzero(i8 %x, i8 %y) nounwind {
 ;
 ; THUMB78-LABEL: scalar_i8_signbit_eq_with_nonzero:
 ; THUMB78:       @ %bb.0:
-; THUMB78-NEXT:    uxtb r1, r1
-; THUMB78-NEXT:    mvn r2, #127
-; THUMB78-NEXT:    lsl.w r1, r2, r1
-; THUMB78-NEXT:    ands r0, r1
-; THUMB78-NEXT:    mov.w r1, #-1
-; THUMB78-NEXT:    uxtab r0, r1, r0
-; THUMB78-NEXT:    clz r0, r0
-; THUMB78-NEXT:    lsrs r0, r0, #5
+; THUMB78-NEXT:    movs r0, #0
 ; THUMB78-NEXT:    bx lr
   %t0 = shl i8 128, %y
   %t1 = and i8 %t0, %x
diff --git a/llvm/test/CodeGen/ARM/ifcvt-iter-indbr.ll b/llvm/test/CodeGen/ARM/ifcvt-iter-indbr.ll
index bfeae51a8..79b9900fa 100644
--- a/llvm/test/CodeGen/ARM/ifcvt-iter-indbr.ll
+++ b/llvm/test/CodeGen/ARM/ifcvt-iter-indbr.ll
@@ -35,7 +35,7 @@ declare i8* @bar(i32, i8*, i8*)
 ; CHECK-PROB: bb.2{{[0-9a-zA-Z.]*}}:
 ; CHECK-PROB: successors: %bb.3(0x40000000), %bb.5(0x40000000)
 
-define i32 @test(i32 %a, i32 %a2, i32* %p, i32* %p2) "no-frame-pointer-elim"="true" {
+define i32 @test(i32 %a, i32 %a2, i32* %p, i32* %p2) "frame-pointer"="all" {
 entry:
   %dst1 = call i8* @bar(i32 1, i8* blockaddress(@test, %bb1), i8* blockaddress(@test, %bb2))
   %dst2 = call i8* @bar(i32 2, i8* blockaddress(@test, %bb1), i8* blockaddress(@test, %bb2))
diff --git a/llvm/test/CodeGen/ARM/ifcvt10.ll b/llvm/test/CodeGen/ARM/ifcvt10.ll
index a63341274..9da135577 100644
--- a/llvm/test/CodeGen/ARM/ifcvt10.ll
+++ b/llvm/test/CodeGen/ARM/ifcvt10.ll
@@ -4,7 +4,7 @@
 ; micro-coded and would have long issue latency even if predicated on
 ; false predicate.
 
-define void @t(double %a, double %b, double %c, double %d, i32* nocapture %solutions, double* nocapture %x) nounwind "no-frame-pointer-elim"="true" {
+define void @t(double %a, double %b, double %c, double %d, i32* nocapture %solutions, double* nocapture %x) nounwind "frame-pointer"="all" {
 entry:
 ; CHECK-LABEL: t:
 ; CHECK: vpop {d8}
diff --git a/llvm/test/CodeGen/ARM/ifcvt11.ll b/llvm/test/CodeGen/ARM/ifcvt11.ll
index eae41e21c..7d577065a 100644
--- a/llvm/test/CodeGen/ARM/ifcvt11.ll
+++ b/llvm/test/CodeGen/ARM/ifcvt11.ll
@@ -17,7 +17,7 @@ bb.nph:                                           ; preds = %entry
   br label %bb
 
 bb:                                               ; preds = %bb4, %bb.nph
-; CHECK: vcmpe.f64
+; CHECK: vcmp.f64
 ; CHECK: vmrs APSR_nzcv, fpscr
   %r.19 = phi i32 [ 0, %bb.nph ], [ %r.0, %bb4 ]
   %n.08 = phi i32 [ 0, %bb.nph ], [ %10, %bb4 ]
@@ -30,9 +30,9 @@ bb:                                               ; preds = %bb4, %bb.nph
 
 bb1:                                              ; preds = %bb
 ; CHECK-NOT: it
-; CHECK-NOT: vcmpemi
+; CHECK-NOT: vcmpmi
 ; CHECK-NOT: vmrsmi
-; CHECK: vcmpe.f64
+; CHECK: vcmp.f64
 ; CHECK: vmrs APSR_nzcv, fpscr
   %scevgep12 = getelementptr %struct.xyz_t, %struct.xyz_t* %p, i32 %n.08, i32 2
   %6 = load double, double* %scevgep12, align 4
diff --git a/llvm/test/CodeGen/ARM/ifcvt5.ll b/llvm/test/CodeGen/ARM/ifcvt5.ll
index 3819bc218..20df7cef9 100644
--- a/llvm/test/CodeGen/ARM/ifcvt5.ll
+++ b/llvm/test/CodeGen/ARM/ifcvt5.ll
@@ -4,14 +4,14 @@
 
 @x = external global i32*		; <i32**> [#uses=1]
 
-define void @foo(i32 %a) "no-frame-pointer-elim"="true" {
+define void @foo(i32 %a) "frame-pointer"="all" {
 entry:
 	%tmp = load i32*, i32** @x		; <i32*> [#uses=1]
 	store i32 %a, i32* %tmp
 	ret void
 }
 
-define i32 @t1(i32 %a, i32 %b) "no-frame-pointer-elim"="true" {
+define i32 @t1(i32 %a, i32 %b) "frame-pointer"="all" {
 ; A8-LABEL: t1:
 ; A8: bxlt lr
 
diff --git a/llvm/test/CodeGen/ARM/inlineasm-X-allocation.ll b/llvm/test/CodeGen/ARM/inlineasm-X-allocation.ll
index ff8dba6e3..2b08e6d46 100644
--- a/llvm/test/CodeGen/ARM/inlineasm-X-allocation.ll
+++ b/llvm/test/CodeGen/ARM/inlineasm-X-allocation.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=armv7-none-eabi -mattr=-neon,-vfp2d16sp %s -o - | FileCheck %s  -check-prefixes=COMMON,NOVFP
+; RUN: llc -mtriple=armv7-none-eabi -mattr=-neon,-vfp2sp %s -o - | FileCheck %s  -check-prefixes=COMMON,NOVFP
 ; RUN: llc -mtriple=armv7-none-eabi -mattr=+neon %s -float-abi=hard -o - | FileCheck %s -check-prefixes=COMMON,VFP
 
 ; The intent here is to test "X", which says that any operand whatsoever is allowed.
diff --git a/llvm/test/CodeGen/ARM/insn-sched1.ll b/llvm/test/CodeGen/ARM/insn-sched1.ll
index 120252d96..ca1cf6469 100644
--- a/llvm/test/CodeGen/ARM/insn-sched1.ll
+++ b/llvm/test/CodeGen/ARM/insn-sched1.ll
@@ -1,7 +1,7 @@
 ; RUN: llc -mtriple=arm-eabi -mattr=+v6 %s -o /dev/null
 ; RUN: llc -mtriple=arm-apple-ios -mattr=+v6 %s -o - | FileCheck %s
 
-define i32 @test(i32 %x) "no-frame-pointer-elim"="true" {
+define i32 @test(i32 %x) "frame-pointer"="all" {
         %tmp = trunc i32 %x to i16              ; <i16> [#uses=1]
         %tmp2 = call i32 @f( i32 1, i16 %tmp )             ; <i32> [#uses=1]
         ret i32 %tmp2
diff --git a/llvm/test/CodeGen/ARM/intrinsics-cmse.ll b/llvm/test/CodeGen/ARM/intrinsics-cmse.ll
new file mode 100644
index 000000000..be4fed196
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/intrinsics-cmse.ll
@@ -0,0 +1,45 @@
+; RUN: llc < %s -mtriple=thumbv8m.base   | FileCheck %s
+; RUN: llc < %s -mtriple=thumbebv8m.base | FileCheck %s
+
+define i32 @test_tt(i8* readnone %p) #0 {
+entry:
+  %0 = tail call i32 @llvm.arm.cmse.tt(i8* %p)
+  ret i32 %0
+}
+; CHECK-LABEL: test_tt:
+; CHECK: tt r{{[0-9]+}}, r{{[0-9]+}}
+
+declare i32 @llvm.arm.cmse.tt(i8*) #1
+
+define i32 @test_ttt(i8* readnone %p) #0 {
+entry:
+  %0 = tail call i32 @llvm.arm.cmse.ttt(i8* %p)
+  ret i32 %0
+}
+; CHECK-LABEL: test_ttt:
+; CHECK: ttt r{{[0-9]+}}, r{{[0-9]+}}
+
+declare i32 @llvm.arm.cmse.ttt(i8*) #1
+
+define i32 @test_tta(i8* readnone %p) #0 {
+entry:
+  %0 = tail call i32 @llvm.arm.cmse.tta(i8* %p)
+  ret i32 %0
+}
+; CHECK-LABEL: test_tta:
+; CHECK: tta r{{[0-9]+}}, r{{[0-9]+}}
+
+declare i32 @llvm.arm.cmse.tta(i8*) #1
+
+define i32 @test_ttat(i8* readnone %p) #0 {
+entry:
+  %0 = tail call i32 @llvm.arm.cmse.ttat(i8* %p)
+  ret i32 %0
+}
+; CHECK-LABEL: test_ttat:
+; CHECK: ttat r{{[0-9]+}}, r{{[0-9]+}}
+
+declare i32 @llvm.arm.cmse.ttat(i8*) #1
+
+attributes #0 = { nounwind readnone "target-features"="+8msecext"}
+attributes #1 = { nounwind readnone }
diff --git a/llvm/test/CodeGen/ARM/ipra-exact-definition.ll b/llvm/test/CodeGen/ARM/ipra-exact-definition.ll
new file mode 100644
index 000000000..2f3b3c7b3
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/ipra-exact-definition.ll
@@ -0,0 +1,44 @@
+; RUN: llc -mtriple armv7a--none-eabi < %s -enable-ipra | FileCheck %s
+
+; A linkone_odr function (the same applies to available_externally, linkonce,
+; weak, common, extern_weak and weak_odr) could be replaced with a
+; differently-compiled version of the same source at link time, which might use
+; different registers, so we can't do IPRA on it.
+define linkonce_odr void @leaf_linkonce_odr() {
+entry:
+  ret void
+}
+define void @test_linkonce_odr() {
+; CHECK-LABEL: test_linkonce_odr:
+entry:
+; CHECK: ASM1: r3
+; CHECK: mov   [[TEMP:r[0-9]+]], r3
+; CHECK: bl    leaf_linkonce_odr
+; CHECK: mov   r3, [[TEMP]]
+; CHECK: ASM2: r3
+  %0 = tail call i32 asm sideeffect "// ASM1: $0", "={r3},0"(i32 undef)
+  tail call void @leaf_linkonce_odr()
+  %1 = tail call i32 asm sideeffect "// ASM2: $0", "={r3},0"(i32 %0)
+  ret void
+}
+
+; This function has external linkage (the same applies to private, internal and
+; appending), so the version we see here is guaranteed to be the version
+; selected by the linker, so we can do IPRA.
+define external void @leaf_external() {
+entry:
+  ret void
+}
+define void @test_external() {
+; CHECK-LABEL: test_external:
+entry:
+; CHECK: ASM1: r3
+; CHECK-NOT:   r3
+; CHECK: bl    leaf_external
+; CHECK-NOT:   r3
+; CHECK: ASM2: r3
+  %0 = tail call i32 asm sideeffect "// ASM1: $0", "={r3},0"(i32 undef)
+  tail call void @leaf_external()
+  %1 = tail call i32 asm sideeffect "// ASM2: $0", "={r3},0"(i32 %0)
+  ret void
+}
diff --git a/llvm/test/CodeGen/ARM/ipra-no-csr.ll b/llvm/test/CodeGen/ARM/ipra-no-csr.ll
new file mode 100644
index 000000000..8070573fb
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/ipra-no-csr.ll
@@ -0,0 +1,22 @@
+; RUN: llc -mtriple armv7a--none-eabi < %s              | FileCheck %s
+; RUN: llc -mtriple armv7a--none-eabi < %s -enable-ipra | FileCheck %s
+
+; Other targets disable callee-saved registers for internal functions when
+; using IPRA, but that isn't profitable for ARM because the PUSH/POP
+; instructions can more efficiently save registers than using individual
+; LDR/STRs in the caller.
+
+define internal void @callee() norecurse {
+; CHECK-LABEL: callee:
+entry:
+; CHECK: push {r4, lr}
+; CHECK: pop {r4, pc}
+  tail call void asm sideeffect "", "~{r4}"()
+  ret void
+}
+
+define void @caller() {
+entry:
+  call void @callee()
+  ret void
+}
diff --git a/llvm/test/CodeGen/ARM/ipra-r0-returned.ll b/llvm/test/CodeGen/ARM/ipra-r0-returned.ll
new file mode 100644
index 000000000..cd3069c0c
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/ipra-r0-returned.ll
@@ -0,0 +1,18 @@
+; RUN: llc -mtriple armv7a--none-eabi -enable-ipra=false < %s | FileCheck %s
+; RUN: llc -mtriple armv7a--none-eabi -enable-ipra=true  < %s | FileCheck %s
+
+define i32 @returns_r0(i32 returned %a)  {
+entry:
+  call void asm sideeffect "", "~{r0}"()
+  ret i32 %a
+}
+
+define i32 @test(i32 %a) {
+; CHECK-LABEL: test:
+entry:
+; CHECK-NOT: r0
+; CHECK: bl      returns_r0
+; CHECK-NOT: r0
+  %b = call i32 @returns_r0(i32 %a)
+  ret i32 %a
+}
diff --git a/llvm/test/CodeGen/ARM/ipra-reg-usage.ll b/llvm/test/CodeGen/ARM/ipra-reg-usage.ll
index 007d176e5..f74d63ada 100644
--- a/llvm/test/CodeGen/ARM/ipra-reg-usage.ll
+++ b/llvm/test/CodeGen/ARM/ipra-reg-usage.ll
@@ -6,7 +6,7 @@ target triple = "armv7-eabi"
 
 declare void @bar1()
 define void @foo()#0 {
-; CHECK: foo Clobbered Registers: $apsr $apsr_nzcv $cpsr $fpcxtns $fpcxts $fpexc $fpinst $fpscr $fpscr_nzcv $fpscr_nzcvqc $fpsid $itstate $pc $sp $spsr $vpr $zr $d0 $d1 $d2 $d3 $d4 $d5 $d6 $d7 $d16 $d17 $d18 $d19 $d20 $d21 $d22 $d23 $d24 $d25 $d26 $d27 $d28 $d29 $d30 $d31 $fpinst2 $mvfr0 $mvfr1 $mvfr2 $p0 $q0 $q1 $q2 $q3 $q8 $q9 $q10 $q11 $q12 $q13 $q14 $q15 $r0 $r1 $r2 $r3 $r12 $s0 $s1 $s2 $s3 $s4 $s5 $s6 $s7 $s8 $s9 $s10 $s11 $s12 $s13 $s14 $s15 $d0_d2 $d1_d3 $d2_d4 $d3_d5 $d4_d6 $d5_d7 $d6_d8 $d7_d9 $d14_d16 $d15_d17 $d16_d18 $d17_d19 $d18_d20 $d19_d21 $d20_d22 $d21_d23 $d22_d24 $d23_d25 $d24_d26 $d25_d27 $d26_d28 $d27_d29 $d28_d30 $d29_d31 $q0_q1 $q1_q2 $q2_q3 $q3_q4 $q7_q8 $q8_q9 $q9_q10 $q10_q11 $q11_q12 $q12_q13 $q13_q14 $q14_q15 $q0_q1_q2_q3 $q1_q2_q3_q4 $q2_q3_q4_q5 $q3_q4_q5_q6 $q5_q6_q7_q8 $q6_q7_q8_q9 $q7_q8_q9_q10 $q8_q9_q10_q11 $q9_q10_q11_q12 $q10_q11_q12_q13 $q11_q12_q13_q14 $q12_q13_q14_q15 $r12_sp $r0_r1 $r2_r3 $d0_d1_d2 $d1_d2_d3 $d2_d3_d4 $d3_d4_d5 $d4_d5_d6 $d5_d6_d7 $d6_d7_d8 $d7_d8_d9 $d14_d15_d16 $d15_d16_d17 $d16_d17_d18 $d17_d18_d19 $d18_d19_d20 $d19_d20_d21 $d20_d21_d22 $d21_d22_d23 $d22_d23_d24 $d23_d24_d25 $d24_d25_d26 $d25_d26_d27 $d26_d27_d28 $d27_d28_d29 $d28_d29_d30 $d29_d30_d31 $d0_d2_d4 $d1_d3_d5 $d2_d4_d6 $d3_d5_d7 $d4_d6_d8 $d5_d7_d9 $d6_d8_d10 $d7_d9_d11 $d12_d14_d16 $d13_d15_d17 $d14_d16_d18 $d15_d17_d19 $d16_d18_d20 $d17_d19_d21 $d18_d20_d22 $d19_d21_d23 $d20_d22_d24 $d21_d23_d25 $d22_d24_d26 $d23_d25_d27 $d24_d26_d28 $d25_d27_d29 $d26_d28_d30 $d27_d29_d31 $d0_d2_d4_d6 $d1_d3_d5_d7 $d2_d4_d6_d8 $d3_d5_d7_d9 $d4_d6_d8_d10 $d5_d7_d9_d11 $d6_d8_d10_d12 $d7_d9_d11_d13 $d10_d12_d14_d16 $d11_d13_d15_d17 $d12_d14_d16_d18 $d13_d15_d17_d19 $d14_d16_d18_d20 $d15_d17_d19_d21 $d16_d18_d20_d22 $d17_d19_d21_d23 $d18_d20_d22_d24 $d19_d21_d23_d25 $d20_d22_d24_d26 $d21_d23_d25_d27 $d22_d24_d26_d28 $d23_d25_d27_d29 $d24_d26_d28_d30 $d25_d27_d29_d31 $d1_d2 $d3_d4 $d5_d6 $d7_d8 $d15_d16 $d17_d18 $d19_d20 $d21_d22 $d23_d24 $d25_d26 $d27_d28 $d29_d30 $d1_d2_d3_d4 $d3_d4_d5_d6 $d5_d6_d7_d8 $d7_d8_d9_d10 $d13_d14_d15_d16 $d15_d16_d17_d18 $d17_d18_d19_d20 $d19_d20_d21_d22 $d21_d22_d23_d24 $d23_d24_d25_d26 $d25_d26_d27_d28 $d27_d28_d29_d30
+; CHECK: foo Clobbered Registers: $apsr $apsr_nzcv $cpsr $fpcxtns $fpcxts $fpexc $fpinst $fpscr $fpscr_nzcv $fpscr_nzcvqc $fpsid $itstate $pc $sp $spsr $vpr $zr $d0 $d1 $d2 $d3 $d4 $d5 $d6 $d7 $d16 $d17 $d18 $d19 $d20 $d21 $d22 $d23 $d24 $d25 $d26 $d27 $d28 $d29 $d30 $d31 $fpinst2 $mvfr0 $mvfr1 $mvfr2 $p0 $q0 $q1 $q2 $q3 $q8 $q9 $q10 $q11 $q12 $q13 $q14 $q15 $r0 $r1 $r2 $r3 $r12 $s0 $s1 $s2 $s3 $s4 $s5 $s6 $s7 $s8 $s9 $s10 $s11 $s12 $s13 $s14 $s15 $d0_d2 $d1_d3 $d2_d4 $d3_d5 $d4_d6 $d5_d7 $d6_d8 $d7_d9 $d14_d16 $d15_d17 $d16_d18 $d17_d19 $d18_d20 $d19_d21 $d20_d22 $d21_d23 $d22_d24 $d23_d25 $d24_d26 $d25_d27 $d26_d28 $d27_d29 $d28_d30 $d29_d31 $q0_q1 $q1_q2 $q2_q3 $q3_q4 $q7_q8 $q8_q9 $q9_q10 $q10_q11 $q11_q12 $q12_q13 $q13_q14 $q14_q15 $q0_q1_q2_q3 $q1_q2_q3_q4 $q2_q3_q4_q5 $q3_q4_q5_q6 $q5_q6_q7_q8 $q6_q7_q8_q9 $q7_q8_q9_q10 $q8_q9_q10_q11 $q9_q10_q11_q12 $q10_q11_q12_q13 $q11_q12_q13_q14 $q12_q13_q14_q15 $r0_r1 $r2_r3 $r12_sp $d0_d1_d2 $d1_d2_d3 $d2_d3_d4 $d3_d4_d5 $d4_d5_d6 $d5_d6_d7 $d6_d7_d8 $d7_d8_d9 $d14_d15_d16 $d15_d16_d17 $d16_d17_d18 $d17_d18_d19 $d18_d19_d20 $d19_d20_d21 $d20_d21_d22 $d21_d22_d23 $d22_d23_d24 $d23_d24_d25 $d24_d25_d26 $d25_d26_d27 $d26_d27_d28 $d27_d28_d29 $d28_d29_d30 $d29_d30_d31 $d0_d2_d4 $d1_d3_d5 $d2_d4_d6 $d3_d5_d7 $d4_d6_d8 $d5_d7_d9 $d6_d8_d10 $d7_d9_d11 $d12_d14_d16 $d13_d15_d17 $d14_d16_d18 $d15_d17_d19 $d16_d18_d20 $d17_d19_d21 $d18_d20_d22 $d19_d21_d23 $d20_d22_d24 $d21_d23_d25 $d22_d24_d26 $d23_d25_d27 $d24_d26_d28 $d25_d27_d29 $d26_d28_d30 $d27_d29_d31 $d0_d2_d4_d6 $d1_d3_d5_d7 $d2_d4_d6_d8 $d3_d5_d7_d9 $d4_d6_d8_d10 $d5_d7_d9_d11 $d6_d8_d10_d12 $d7_d9_d11_d13 $d10_d12_d14_d16 $d11_d13_d15_d17 $d12_d14_d16_d18 $d13_d15_d17_d19 $d14_d16_d18_d20 $d15_d17_d19_d21 $d16_d18_d20_d22 $d17_d19_d21_d23 $d18_d20_d22_d24 $d19_d21_d23_d25 $d20_d22_d24_d26 $d21_d23_d25_d27 $d22_d24_d26_d28 $d23_d25_d27_d29 $d24_d26_d28_d30 $d25_d27_d29_d31 $d1_d2 $d3_d4 $d5_d6 $d7_d8 $d15_d16 $d17_d18 $d19_d20 $d21_d22 $d23_d24 $d25_d26 $d27_d28 $d29_d30 $d1_d2_d3_d4 $d3_d4_d5_d6 $d5_d6_d7_d8 $d7_d8_d9_d10 $d13_d14_d15_d16 $d15_d16_d17_d18 $d17_d18_d19_d20 $d19_d20_d21_d22 $d21_d22_d23_d24 $d23_d24_d25_d26 $d25_d26_d27_d28 $d27_d28_d29_d30
   call void @bar1()
   call void @bar2()
   ret void
diff --git a/llvm/test/CodeGen/ARM/ipra.ll b/llvm/test/CodeGen/ARM/ipra.ll
new file mode 100644
index 000000000..3252c1325
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/ipra.ll
@@ -0,0 +1,202 @@
+; RUN: llc -mtriple armv7a--none-eabi < %s              | FileCheck %s --check-prefix=CHECK --check-prefix=DISABLED
+; RUN: llc -mtriple armv7a--none-eabi < %s -enable-ipra | FileCheck %s --check-prefix=CHECK --check-prefix=ENABLED
+
+define void @leaf()  {
+entry:
+  tail call void asm sideeffect "", ""()
+  ret void
+}
+
+define void @leaf_r3() {
+entry:
+  tail call void asm sideeffect "", "~{r3}"()
+  ret void
+}
+
+define void @leaf_r4() {
+entry:
+  tail call void asm sideeffect "", "~{r4}"()
+  ret void
+}
+
+define void @leaf_s0() {
+entry:
+  tail call void asm sideeffect "", "~{s0}"()
+  ret void
+}
+
+define void @leaf_d0() {
+entry:
+  tail call void asm sideeffect "", "~{d0}"()
+  ret void
+}
+
+; r3 is normally caller-saved, but with IPRA we can see that it isn't used in
+; the callee, so can leave a live value in it.
+define void @test_r3_presrved() {
+; CHECK-LABEL: test_r3_presrved:
+entry:
+; CHECK: ASM1: r3
+; DISABLED: mov [[TEMP:r[0-9]+]], r3
+; ENABLED-NOT: r3
+; CHECK: bl      leaf
+; DISABLED: mov r3, [[TEMP]]
+; ENABLED-NOT: r3
+; CHECK: ASM2: r3
+  %a = tail call i32 asm sideeffect "// ASM1: $0", "={r3},0"(i32 undef)
+  tail call void @leaf()
+  %b = tail call i32 asm sideeffect "// ASM2: $0", "={r3},0"(i32 %a)
+  ret void
+}
+
+; Same as above, but r3 is clobbered in the callee, so it is clobbered by the
+; call as normal.
+define void @test_r3_clobbered() {
+; CHECK-LABEL: test_r3_clobbered:
+entry:
+; CHECK: ASM1: r3
+; CHECK: mov [[TEMP:r[0-9]+]], r3
+; CHECK: bl      leaf
+; CHECK: mov r3, [[TEMP]]
+; CHECK: ASM2: r3
+  %a = tail call i32 asm sideeffect "// ASM1: $0", "={r3},0"(i32 undef)
+  tail call void @leaf_r3()
+  %b = tail call i32 asm sideeffect "// ASM2: $0", "={r3},0"(i32 %a)
+  ret void
+}
+
+; r4 is a callee-saved register, so IPRA has no effect.
+define void @test_r4_preserved() {
+; CHECK-LABEL: test_r4_preserved:
+entry:
+; CHECK: ASM1: r4
+; CHECK-NOT: r4
+; CHECK: bl      leaf
+; CHECK-NOT: r4
+; CHECK: ASM2: r4
+  %a = tail call i32 asm sideeffect "// ASM1: $0", "={r4},0"(i32 undef)
+  tail call void @leaf()
+  %b = tail call i32 asm sideeffect "// ASM2: $0", "={r4},0"(i32 %a)
+  ret void
+}
+define void @test_r4_clobbered() {
+; CHECK-LABEL: test_r4_clobbered:
+entry:
+; CHECK: ASM1: r4
+; CHECK-NOT: r4
+; CHECK: bl      leaf_r4
+; CHECK-NOT: r4
+; CHECK: ASM2: r4
+  %a = tail call i32 asm sideeffect "// ASM1: $0", "={r4},0"(i32 undef)
+  tail call void @leaf_r4()
+  %b = tail call i32 asm sideeffect "// ASM2: $0", "={r4},0"(i32 %a)
+  ret void
+}
+
+; r12 is the intra-call scratch register, so we have to assume it is clobbered
+; even if we can see that the callee does not touch it.
+define void @test_r12() {
+; CHECK-LABEL: test_r12:
+entry:
+; CHECK: ASM1: r12
+; CHECK: mov [[TEMP:r[0-9]+]], r12
+; CHECK: bl      leaf
+; CHECK: mov r12, [[TEMP]]
+; CHECK: ASM2: r12
+  %a = tail call i32 asm sideeffect "// ASM1: $0", "={r12},0"(i32 undef)
+  tail call void @leaf()
+  %b = tail call i32 asm sideeffect "// ASM2: $0", "={r12},0"(i32 %a)
+  ret void
+}
+
+; s0 and d0 are caller-saved, IPRA allows us to keep them live in the caller if
+; the callee doesn't modify them.
+define void @test_s0_preserved() {
+; CHECK-LABEL: test_s0_preserved:
+entry:
+; CHECK: ASM1: s0
+; DISABLED: vmov.f32 [[TEMP:s[0-9]+]], s0
+; ENABLED-NOT: s0
+; CHECK: bl      leaf
+; DISABLED: vmov.f32 s0, [[TEMP]]
+; ENABLED-NOT: s0
+; CHECK: ASM2: s0
+  %a = tail call float asm sideeffect "// ASM1: $0", "={s0},0"(float undef)
+  tail call void @leaf()
+  %b = tail call float asm sideeffect "// ASM2: $0", "={s0},0"(float %a)
+  ret void
+}
+
+define void @test_s0_clobbered() {
+; CHECK-LABEL: test_s0_clobbered:
+entry:
+; CHECK: ASM1: s0
+; CHECK: vmov.f32 [[TEMP:s[0-9]+]], s0
+; CHECK: bl      leaf_s0
+; CHECK: vmov.f32 s0, [[TEMP]]
+; CHECK: ASM2: s0
+  %a = tail call float asm sideeffect "// ASM1: $0", "={s0},0"(float undef)
+  tail call void @leaf_s0()
+  %b = tail call float asm sideeffect "// ASM2: $0", "={s0},0"(float %a)
+  ret void
+}
+
+define void @test_d0_preserved() {
+; CHECK-LABEL: test_d0_preserved:
+entry:
+; CHECK: ASM1: d0
+; DISABLED: vmov.f64 [[TEMP:d[0-9]+]], d0
+; ENABLED-NOT: d0
+; CHECK: bl      leaf
+; DISABLED: vmov.f64 d0, [[TEMP]]
+; ENABLED-NOT: d0
+; CHECK: ASM2: d0
+  %a = tail call double asm sideeffect "// ASM1: $0", "={d0},0"(double undef)
+  tail call void @leaf()
+  %b = tail call double asm sideeffect "// ASM2: $0", "={d0},0"(double %a)
+  ret void
+}
+
+define void @test_d0_clobbered() {
+; CHECK-LABEL: test_d0_clobbered:
+entry:
+; CHECK: ASM1: d0
+; CHECK: vmov.f64 [[TEMP:d[0-9]+]], d0
+; CHECK: bl      leaf_d0
+; CHECK: vmov.f64 d0, [[TEMP]]
+; CHECK: ASM2: d0
+  %a = tail call double asm sideeffect "// ASM1: $0", "={d0},0"(double undef)
+  tail call void @leaf_d0()
+  %b = tail call double asm sideeffect "// ASM2: $0", "={d0},0"(double %a)
+  ret void
+}
+
+; s0 and d0 overlap, so clobbering one in the callee prevents the other from
+; being kept live across the call.
+define void @test_s0_clobber_d0() {
+; CHECK-LABEL: test_s0_clobber_d0:
+entry:
+; CHECK: ASM1: s0
+; CHECK: vmov.f32 [[TEMP:s[0-9]+]], s0
+; CHECK: bl      leaf_d0
+; CHECK: vmov.f32 s0, [[TEMP]]
+; CHECK: ASM2: s0
+  %a = tail call float asm sideeffect "// ASM1: $0", "={s0},0"(float undef)
+  tail call void @leaf_d0()
+  %b = tail call float asm sideeffect "// ASM2: $0", "={s0},0"(float %a)
+  ret void
+}
+
+define void @test_d0_clobber_s0() {
+; CHECK-LABEL: test_d0_clobber_s0:
+entry:
+; CHECK: ASM1: d0
+; CHECK: vmov.f64 [[TEMP:d[0-9]+]], d0
+; CHECK: bl      leaf_s0
+; CHECK: vmov.f64 d0, [[TEMP]]
+; CHECK: ASM2: d0
+  %a = tail call double asm sideeffect "// ASM1: $0", "={d0},0"(double undef)
+  tail call void @leaf_s0()
+  %b = tail call double asm sideeffect "// ASM2: $0", "={d0},0"(double %a)
+  ret void
+}
diff --git a/llvm/test/CodeGen/ARM/isel-v8i32-crash.ll b/llvm/test/CodeGen/ARM/isel-v8i32-crash.ll
index f1745bd47..57279d0ba 100644
--- a/llvm/test/CodeGen/ARM/isel-v8i32-crash.ll
+++ b/llvm/test/CodeGen/ARM/isel-v8i32-crash.ll
@@ -23,4 +23,4 @@ entry:
   ret void
 }
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/ARM/ldrd.ll b/llvm/test/CodeGen/ARM/ldrd.ll
index 4cdafa72f..b4325c78d 100644
--- a/llvm/test/CodeGen/ARM/ldrd.ll
+++ b/llvm/test/CodeGen/ARM/ldrd.ll
@@ -15,7 +15,7 @@
 declare i64* @get_ptr()
 declare void @use_i64(i64 %v)
 
-define void @test_ldrd(i64 %a) nounwind readonly "no-frame-pointer-elim"="true" {
+define void @test_ldrd(i64 %a) nounwind readonly "frame-pointer"="all" {
 ; CHECK-LABEL: test_ldrd:
 ; NORMAL: bl{{x?}} _get_ptr
 ; A8: ldrd r0, r1, [r0]
@@ -49,7 +49,7 @@ define void @test_ldrd(i64 %a) nounwind readonly "no-frame-pointer-elim"="true"
 ; GREEDY: %bb
 ; GREEDY: ldrd
 ; GREEDY: str
-define void @f(i32* nocapture %a, i32* nocapture %b, i32 %n) nounwind "no-frame-pointer-elim"="true" {
+define void @f(i32* nocapture %a, i32* nocapture %b, i32 %n) nounwind "frame-pointer"="all" {
 entry:
   %0 = add nsw i32 %n, -1                         ; <i32> [#uses=2]
   %1 = icmp sgt i32 %0, 0                         ; <i1> [#uses=1]
@@ -79,7 +79,7 @@ return:                                           ; preds = %bb, %entry
 @TestVar = external global %struct.Test
 
 ; CHECK-LABEL: Func1:
-define void @Func1() nounwind ssp "no-frame-pointer-elim"="true" {
+define void @Func1() nounwind ssp "frame-pointer"="all" {
 entry:
 ; A8: movw [[BASE:r[0-9]+]], :lower16:{{.*}}TestVar{{.*}}
 ; A8: movt [[BASE]], :upper16:{{.*}}TestVar{{.*}}
@@ -104,7 +104,7 @@ declare void @extfunc(i32, i32, i32, i32)
 ; A8: ldrd
 ; CHECK: bl{{x?}} _extfunc
 ; A8: pop
-define void @Func2(i32* %p) "no-frame-pointer-elim"="true" {
+define void @Func2(i32* %p) "frame-pointer"="all" {
 entry:
   %addr0 = getelementptr i32, i32* %p, i32 0
   %addr1 = getelementptr i32, i32* %p, i32 1
@@ -129,7 +129,7 @@ entry:
 ; GREEDY: ldrd r1, r2, [sp]
 ; CONSERVATIVE: ldrd r1, r2, [sp]
 ; CHECK: bl{{x?}} _extfunc
-define void @strd_spill_ldrd_reload(i32 %v0, i32 %v1) "no-frame-pointer-elim"="true" {
+define void @strd_spill_ldrd_reload(i32 %v0, i32 %v1) "frame-pointer"="all" {
   ; force %v0 and %v1 to be spilled
   call void asm sideeffect "", "~{r0},~{r1},~{r2},~{r3},~{r4},~{r5},~{r6},~{r7},~{r8},~{r9},~{r10},~{r11},~{r12},~{lr}"()
   ; force the reloaded %v0, %v1 into different registers
@@ -143,7 +143,7 @@ declare void @extfunc2(i32*, i32, i32)
 ; NORMAL: ldrd r1, r2, [r0], #-8
 ; CONSERVATIVE-NOT: ldrd
 ; CHECK: bl{{x?}} _extfunc
-define void @ldrd_postupdate_dec(i32* %p0) "no-frame-pointer-elim"="true" {
+define void @ldrd_postupdate_dec(i32* %p0) "frame-pointer"="all" {
   %p0.1 = getelementptr i32, i32* %p0, i32 1
   %v0 = load i32, i32* %p0
   %v1 = load i32, i32* %p0.1
@@ -156,7 +156,7 @@ define void @ldrd_postupdate_dec(i32* %p0) "no-frame-pointer-elim"="true" {
 ; NORMAL: ldrd r1, r2, [r0], #8
 ; CONSERVATIVE-NOT: ldrd
 ; CHECK: bl{{x?}} _extfunc
-define void @ldrd_postupdate_inc(i32* %p0) "no-frame-pointer-elim"="true" {
+define void @ldrd_postupdate_inc(i32* %p0) "frame-pointer"="all" {
   %p0.1 = getelementptr i32, i32* %p0, i32 1
   %v0 = load i32, i32* %p0
   %v1 = load i32, i32* %p0.1
@@ -169,7 +169,7 @@ define void @ldrd_postupdate_inc(i32* %p0) "no-frame-pointer-elim"="true" {
 ; NORMAL: strd r1, r2, [r0], #-8
 ; CONSERVATIVE-NOT: strd
 ; CHECK: bx lr
-define i32* @strd_postupdate_dec(i32* %p0, i32 %v0, i32 %v1) "no-frame-pointer-elim"="true" {
+define i32* @strd_postupdate_dec(i32* %p0, i32 %v0, i32 %v1) "frame-pointer"="all" {
   %p0.1 = getelementptr i32, i32* %p0, i32 1
   store i32 %v0, i32* %p0
   store i32 %v1, i32* %p0.1
@@ -181,7 +181,7 @@ define i32* @strd_postupdate_dec(i32* %p0, i32 %v0, i32 %v1) "no-frame-pointer-e
 ; NORMAL: strd r1, r2, [r0], #8
 ; CONSERVATIVE-NOT: strd
 ; CHECK: bx lr
-define i32* @strd_postupdate_inc(i32* %p0, i32 %v0, i32 %v1) "no-frame-pointer-elim"="true" {
+define i32* @strd_postupdate_inc(i32* %p0, i32 %v0, i32 %v1) "frame-pointer"="all" {
   %p0.1 = getelementptr i32, i32* %p0, i32 1
   store i32 %v0, i32* %p0
   store i32 %v1, i32* %p0.1
diff --git a/llvm/test/CodeGen/ARM/legalize-bitcast.ll b/llvm/test/CodeGen/ARM/legalize-bitcast.ll
new file mode 100644
index 000000000..478ff985b
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/legalize-bitcast.ll
@@ -0,0 +1,59 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -O0 -mtriple=armebv7 -target-abi apcs -o - %s | FileCheck %s
+
+@vec6_p = external global <6 x i16>
+
+define i32 @vec_to_int() {
+; CHECK-LABEL: vec_to_int:
+; CHECK:       @ %bb.0: @ %bb.0
+; CHECK-NEXT:    push {r4}
+; CHECK-NEXT:    sub sp, sp, #28
+; CHECK-NEXT:    movw r0, :lower16:vec6_p
+; CHECK-NEXT:    movt r0, :upper16:vec6_p
+; CHECK-NEXT:    vld1.8 {d16}, [r0]!
+; CHECK-NEXT:    ldr r0, [r0]
+; CHECK-NEXT:    @ implicit-def: $d17
+; CHECK-NEXT:    vmov.32 d17[0], r0
+; CHECK-NEXT:    vrev32.16 d17, d17
+; CHECK-NEXT:    vrev16.8 d16, d16
+; CHECK-NEXT:    vmov.f64 d18, d16
+; CHECK-NEXT:    vmov.f64 d19, d17
+; CHECK-NEXT:    vstmia sp, {d18, d19} @ 16-byte Spill
+; CHECK-NEXT:    b .LBB0_1
+; CHECK-NEXT:  .LBB0_1: @ %bb.1
+; CHECK-NEXT:    vldmia sp, {d16, d17} @ 16-byte Reload
+; CHECK-NEXT:    vrev32.16 q9, q8
+; CHECK-NEXT:    @ kill: def $d19 killed $d19 killed $q9
+; CHECK-NEXT:    vmov.32 r0, d19[0]
+; CHECK-NEXT:    add sp, sp, #28
+; CHECK-NEXT:    pop {r4}
+; CHECK-NEXT:    bx lr
+bb.0:
+  %vec6 = load <6 x i16>, <6 x i16>* @vec6_p, align 1
+  br label %bb.1
+
+bb.1:
+  %0 = bitcast <6 x i16> %vec6 to i96
+  %1 = trunc i96 %0 to i32
+  ret i32 %1
+}
+
+define i16 @int_to_vec(i80 %in) {
+; CHECK-LABEL: int_to_vec:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    mov r3, r1
+; CHECK-NEXT:    mov r12, r0
+; CHECK-NEXT:    lsl r0, r0, #16
+; CHECK-NEXT:    orr r0, r0, r1, lsr #16
+; CHECK-NEXT:    @ implicit-def: $d16
+; CHECK-NEXT:    vmov.32 d16[0], r0
+; CHECK-NEXT:    @ implicit-def: $q9
+; CHECK-NEXT:    vmov.f64 d18, d16
+; CHECK-NEXT:    vrev32.16 q9, q9
+; CHECK-NEXT:    @ kill: def $d18 killed $d18 killed $q9
+; CHECK-NEXT:    vmov.u16 r0, d18[0]
+; CHECK-NEXT:    bx lr
+  %vec = bitcast i80 %in to <5 x i16>
+  %e0 = extractelement <5 x i16> %vec, i32 0
+  ret i16 %e0
+}
diff --git a/llvm/test/CodeGen/ARM/load-combine-big-endian.ll b/llvm/test/CodeGen/ARM/load-combine-big-endian.ll
index 4e6e6a719..0ed85501a 100644
--- a/llvm/test/CodeGen/ARM/load-combine-big-endian.ll
+++ b/llvm/test/CodeGen/ARM/load-combine-big-endian.ll
@@ -1,16 +1,32 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=armeb-unknown | FileCheck %s
 ; RUN: llc < %s -mtriple=armv6eb-unknown | FileCheck %s --check-prefix=CHECK-ARMv6
+; RUN: llc < %s -mtriple=thumbv6meb-none-eabi | FileCheck %s --check-prefix=CHECK-THUMBv6
+; RUN: llc < %s -mtriple=thumbv6meb-none-eabi | FileCheck %s --check-prefix=CHECK-THUMBv7
 
 ; i8* p; // p is 4 byte aligned
 ; ((i32) p[0] << 24) | ((i32) p[1] << 16) | ((i32) p[2] << 8) | (i32) p[3]
 define i32 @load_i32_by_i8_big_endian(i32* %arg) {
 ; CHECK-LABEL: load_i32_by_i8_big_endian:
-; CHECK: ldr r0, [r0]
-; CHECK-NEXT: mov pc, lr
-
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    ldr r0, [r0]
+; CHECK-NEXT:    mov pc, lr
+;
 ; CHECK-ARMv6-LABEL: load_i32_by_i8_big_endian:
-; CHECK-ARMv6: ldr r0, [r0]
-; CHECK-ARMv6-NEXT: bx  lr
+; CHECK-ARMv6:       @ %bb.0:
+; CHECK-ARMv6-NEXT:    ldr r0, [r0]
+; CHECK-ARMv6-NEXT:    bx lr
+;
+; CHECK-THUMBv6-LABEL: load_i32_by_i8_big_endian:
+; CHECK-THUMBv6:       @ %bb.0:
+; CHECK-THUMBv6-NEXT:    ldr r0, [r0]
+; CHECK-THUMBv6-NEXT:    bx lr
+;
+; CHECK-THUMBv7-LABEL: load_i32_by_i8_big_endian:
+; CHECK-THUMBv7:       @ %bb.0:
+; CHECK-THUMBv7-NEXT:    ldr r0, [r0]
+; CHECK-THUMBv7-NEXT:    bx lr
+
   %tmp = bitcast i32* %arg to i8*
   %tmp1 = load i8, i8* %tmp, align 4
   %tmp2 = zext i8 %tmp1 to i32
@@ -37,18 +53,35 @@ define i32 @load_i32_by_i8_big_endian(i32* %arg) {
 define i32 @load_i32_by_i8_bswap(i32* %arg) {
 ; BSWAP is not supported by 32 bit target
 ; CHECK-LABEL: load_i32_by_i8_bswap:
-; CHECK: ldr  r0, [r0]
-; CHECK: and
-; CHECK-NEXT: and
-; CHECK-NEXT: orr
-; CHECK-NEXT: orr
-; CHECK-NEXT: orr
-; CHECK-NEXT: mov pc, lr
-
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    ldr r0, [r0]
+; CHECK-NEXT:    mov r1, #65280
+; CHECK-NEXT:    mov r2, #16711680
+; CHECK-NEXT:    and r1, r1, r0, lsr #8
+; CHECK-NEXT:    and r2, r2, r0, lsl #8
+; CHECK-NEXT:    orr r1, r1, r0, lsr #24
+; CHECK-NEXT:    orr r0, r2, r0, lsl #24
+; CHECK-NEXT:    orr r0, r0, r1
+; CHECK-NEXT:    mov pc, lr
+;
 ; CHECK-ARMv6-LABEL: load_i32_by_i8_bswap:
-; CHECK-ARMv6: ldr  r0, [r0]
-; CHECK-ARMv6-NEXT: rev  r0, r0
-; CHECK-ARMv6-NEXT: bx lr
+; CHECK-ARMv6:       @ %bb.0:
+; CHECK-ARMv6-NEXT:    ldr r0, [r0]
+; CHECK-ARMv6-NEXT:    rev r0, r0
+; CHECK-ARMv6-NEXT:    bx lr
+;
+; CHECK-THUMBv6-LABEL: load_i32_by_i8_bswap:
+; CHECK-THUMBv6:       @ %bb.0:
+; CHECK-THUMBv6-NEXT:    ldr r0, [r0]
+; CHECK-THUMBv6-NEXT:    rev r0, r0
+; CHECK-THUMBv6-NEXT:    bx lr
+;
+; CHECK-THUMBv7-LABEL: load_i32_by_i8_bswap:
+; CHECK-THUMBv7:       @ %bb.0:
+; CHECK-THUMBv7-NEXT:    ldr r0, [r0]
+; CHECK-THUMBv7-NEXT:    rev r0, r0
+; CHECK-THUMBv7-NEXT:    bx lr
+
   %tmp = bitcast i32* %arg to i8*
   %tmp1 = getelementptr inbounds i8, i8* %tmp, i32 0
   %tmp2 = load i8, i8* %tmp1, align 4
@@ -75,12 +108,25 @@ define i32 @load_i32_by_i8_bswap(i32* %arg) {
 ; ((i32) (((i16) p[0] << 8) | (i16) p[1]) << 16) | (i32) (((i16) p[3] << 8) | (i16) p[4])
 define i32 @load_i32_by_i16_by_i8_big_endian(i32* %arg) {
 ; CHECK-LABEL: load_i32_by_i16_by_i8_big_endian:
-; CHECK: ldr r0, [r0]
-; CHECK-NEXT: mov pc, lr
-
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    ldr r0, [r0]
+; CHECK-NEXT:    mov pc, lr
+;
 ; CHECK-ARMv6-LABEL: load_i32_by_i16_by_i8_big_endian:
-; CHECK-ARMv6: ldr r0, [r0]
-; CHECK-ARMv6-NEXT: bx  lr
+; CHECK-ARMv6:       @ %bb.0:
+; CHECK-ARMv6-NEXT:    ldr r0, [r0]
+; CHECK-ARMv6-NEXT:    bx lr
+;
+; CHECK-THUMBv6-LABEL: load_i32_by_i16_by_i8_big_endian:
+; CHECK-THUMBv6:       @ %bb.0:
+; CHECK-THUMBv6-NEXT:    ldr r0, [r0]
+; CHECK-THUMBv6-NEXT:    bx lr
+;
+; CHECK-THUMBv7-LABEL: load_i32_by_i16_by_i8_big_endian:
+; CHECK-THUMBv7:       @ %bb.0:
+; CHECK-THUMBv7-NEXT:    ldr r0, [r0]
+; CHECK-THUMBv7-NEXT:    bx lr
+
   %tmp = bitcast i32* %arg to i8*
   %tmp1 = load i8, i8* %tmp, align 4
   %tmp2 = zext i8 %tmp1 to i16
@@ -108,12 +154,25 @@ define i32 @load_i32_by_i16_by_i8_big_endian(i32* %arg) {
 ; ((i32) p[0] << 16) | (i32) p[1]
 define i32 @load_i32_by_i16(i32* %arg) {
 ; CHECK-LABEL: load_i32_by_i16:
-; CHECK: ldr r0, [r0]
-; CHECK-NEXT: mov pc, lr
-
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    ldr r0, [r0]
+; CHECK-NEXT:    mov pc, lr
+;
 ; CHECK-ARMv6-LABEL: load_i32_by_i16:
-; CHECK-ARMv6: ldr r0, [r0]
-; CHECK-ARMv6-NEXT: bx  lr
+; CHECK-ARMv6:       @ %bb.0:
+; CHECK-ARMv6-NEXT:    ldr r0, [r0]
+; CHECK-ARMv6-NEXT:    bx lr
+;
+; CHECK-THUMBv6-LABEL: load_i32_by_i16:
+; CHECK-THUMBv6:       @ %bb.0:
+; CHECK-THUMBv6-NEXT:    ldr r0, [r0]
+; CHECK-THUMBv6-NEXT:    bx lr
+;
+; CHECK-THUMBv7-LABEL: load_i32_by_i16:
+; CHECK-THUMBv7:       @ %bb.0:
+; CHECK-THUMBv7-NEXT:    ldr r0, [r0]
+; CHECK-THUMBv7-NEXT:    bx lr
+
   %tmp = bitcast i32* %arg to i16*
   %tmp1 = load i16, i16* %tmp, align 4
   %tmp2 = zext i16 %tmp1 to i32
@@ -130,12 +189,25 @@ define i32 @load_i32_by_i16(i32* %arg) {
 ; (i32) (p_16[0] << 16) | ((i32) p[2] << 8) | (i32) p[3]
 define i32 @load_i32_by_i16_i8(i32* %arg) {
 ; CHECK-LABEL: load_i32_by_i16_i8:
-; CHECK: ldr r0, [r0]
-; CHECK-NEXT: mov pc, lr
-
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    ldr r0, [r0]
+; CHECK-NEXT:    mov pc, lr
+;
 ; CHECK-ARMv6-LABEL: load_i32_by_i16_i8:
-; CHECK-ARMv6: ldr r0, [r0]
-; CHECK-ARMv6-NEXT: bx  lr
+; CHECK-ARMv6:       @ %bb.0:
+; CHECK-ARMv6-NEXT:    ldr r0, [r0]
+; CHECK-ARMv6-NEXT:    bx lr
+;
+; CHECK-THUMBv6-LABEL: load_i32_by_i16_i8:
+; CHECK-THUMBv6:       @ %bb.0:
+; CHECK-THUMBv6-NEXT:    ldr r0, [r0]
+; CHECK-THUMBv6-NEXT:    bx lr
+;
+; CHECK-THUMBv7-LABEL: load_i32_by_i16_i8:
+; CHECK-THUMBv7:       @ %bb.0:
+; CHECK-THUMBv7-NEXT:    ldr r0, [r0]
+; CHECK-THUMBv7-NEXT:    bx lr
+
   %tmp = bitcast i32* %arg to i16*
   %tmp1 = bitcast i32* %arg to i8*
   %tmp2 = load i16, i16* %tmp, align 4
@@ -157,25 +229,48 @@ define i32 @load_i32_by_i16_i8(i32* %arg) {
 ; (i64) p[0] | ((i64) p[1] << 8) | ((i64) p[2] << 16) | ((i64) p[3] << 24) | ((i64) p[4] << 32) | ((i64) p[5] << 40) | ((i64) p[6] << 48) | ((i64) p[7] << 56)
 define i64 @load_i64_by_i8_bswap(i64* %arg) {
 ; CHECK-LABEL: load_i64_by_i8_bswap:
-; CHECK: ldr{{.*}}r0
-; CHECK: ldr{{.*}}r0
-; CHECK: and
-; CHECK-NEXT: and
-; CHECK-NEXT: orr
-; CHECK-NEXT: orr
-; CHECK-NEXT: and
-; CHECK-NEXT: orr
-; CHECK-NEXT: and
-; CHECK-NEXT: orr
-; CHECK-NEXT: orr
-; CHECK-NEXT: orr
-; CHECK: mov pc, lr
-
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    push {r11, lr}
+; CHECK-NEXT:    ldr r1, [r0]
+; CHECK-NEXT:    mov r12, #65280
+; CHECK-NEXT:    ldr r0, [r0, #4]
+; CHECK-NEXT:    mov lr, #16711680
+; CHECK-NEXT:    and r3, r12, r0, lsr #8
+; CHECK-NEXT:    and r2, lr, r0, lsl #8
+; CHECK-NEXT:    orr r3, r3, r0, lsr #24
+; CHECK-NEXT:    orr r0, r2, r0, lsl #24
+; CHECK-NEXT:    and r2, r12, r1, lsr #8
+; CHECK-NEXT:    orr r0, r0, r3
+; CHECK-NEXT:    and r3, lr, r1, lsl #8
+; CHECK-NEXT:    orr r2, r2, r1, lsr #24
+; CHECK-NEXT:    orr r1, r3, r1, lsl #24
+; CHECK-NEXT:    orr r1, r1, r2
+; CHECK-NEXT:    pop {r11, lr}
+; CHECK-NEXT:    mov pc, lr
+;
 ; CHECK-ARMv6-LABEL: load_i64_by_i8_bswap:
-; CHECK-ARMv6: ldrd  r2, r3, [r0]
-; CHECK-ARMv6: rev r0, r3
-; CHECK-ARMv6: rev r1, r2
-; CHECK-ARMv6: bx  lr
+; CHECK-ARMv6:       @ %bb.0:
+; CHECK-ARMv6-NEXT:    ldrd r2, r3, [r0]
+; CHECK-ARMv6-NEXT:    rev r0, r3
+; CHECK-ARMv6-NEXT:    rev r1, r2
+; CHECK-ARMv6-NEXT:    bx lr
+;
+; CHECK-THUMBv6-LABEL: load_i64_by_i8_bswap:
+; CHECK-THUMBv6:       @ %bb.0:
+; CHECK-THUMBv6-NEXT:    ldr r1, [r0]
+; CHECK-THUMBv6-NEXT:    ldr r0, [r0, #4]
+; CHECK-THUMBv6-NEXT:    rev r0, r0
+; CHECK-THUMBv6-NEXT:    rev r1, r1
+; CHECK-THUMBv6-NEXT:    bx lr
+;
+; CHECK-THUMBv7-LABEL: load_i64_by_i8_bswap:
+; CHECK-THUMBv7:       @ %bb.0:
+; CHECK-THUMBv7-NEXT:    ldr r1, [r0]
+; CHECK-THUMBv7-NEXT:    ldr r0, [r0, #4]
+; CHECK-THUMBv7-NEXT:    rev r0, r0
+; CHECK-THUMBv7-NEXT:    rev r1, r1
+; CHECK-THUMBv7-NEXT:    bx lr
+
   %tmp = bitcast i64* %arg to i8*
   %tmp1 = load i8, i8* %tmp, align 8
   %tmp2 = zext i8 %tmp1 to i64
@@ -221,14 +316,31 @@ define i64 @load_i64_by_i8_bswap(i64* %arg) {
 ; ((i64) p[0] << 56) | ((i64) p[1] << 48) | ((i64) p[2] << 40) | ((i64) p[3] << 32) | ((i64) p[4] << 24) | ((i64) p[5] << 16) | ((i64) p[6] << 8) | (i64) p[7]
 define i64 @load_i64_by_i8(i64* %arg) {
 ; CHECK-LABEL: load_i64_by_i8:
-; CHECK: ldr r2, [r0]
-; CHECK: ldr r1, [r0, #4]
-; CHECK: mov r0, r2
-; CHECK: mov pc, lr
-
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    ldr r2, [r0]
+; CHECK-NEXT:    ldr r1, [r0, #4]
+; CHECK-NEXT:    mov r0, r2
+; CHECK-NEXT:    mov pc, lr
+;
 ; CHECK-ARMv6-LABEL: load_i64_by_i8:
-; CHECK-ARMv6: ldrd  r0, r1, [r0]
-; CHECK-ARMv6: bx  lr
+; CHECK-ARMv6:       @ %bb.0:
+; CHECK-ARMv6-NEXT:    ldrd r0, r1, [r0]
+; CHECK-ARMv6-NEXT:    bx lr
+;
+; CHECK-THUMBv6-LABEL: load_i64_by_i8:
+; CHECK-THUMBv6:       @ %bb.0:
+; CHECK-THUMBv6-NEXT:    ldr r2, [r0]
+; CHECK-THUMBv6-NEXT:    ldr r1, [r0, #4]
+; CHECK-THUMBv6-NEXT:    mov r0, r2
+; CHECK-THUMBv6-NEXT:    bx lr
+;
+; CHECK-THUMBv7-LABEL: load_i64_by_i8:
+; CHECK-THUMBv7:       @ %bb.0:
+; CHECK-THUMBv7-NEXT:    ldr r2, [r0]
+; CHECK-THUMBv7-NEXT:    ldr r1, [r0, #4]
+; CHECK-THUMBv7-NEXT:    mov r0, r2
+; CHECK-THUMBv7-NEXT:    bx lr
+
   %tmp = bitcast i64* %arg to i8*
   %tmp1 = load i8, i8* %tmp, align 8
   %tmp2 = zext i8 %tmp1 to i64
@@ -274,20 +386,37 @@ define i64 @load_i64_by_i8(i64* %arg) {
 ; (i32) p[1] | ((i32) p[2] << 8) | ((i32) p[3] << 16) | ((i32) p[4] << 24)
 define i32 @load_i32_by_i8_nonzero_offset(i32* %arg) {
 ; CHECK-LABEL: load_i32_by_i8_nonzero_offset:
-; CHECK: ldr r0, [r0, #1]
-; CHECK-NEXT: mov r1, #65280
-; CHECK-NEXT: mov r2, #16711680
-; CHECK-NEXT: and r1, r1, r0, lsr #8
-; CHECK-NEXT: and r2, r2, r0, lsl #8
-; CHECK-NEXT: orr r1, r1, r0, lsr #24
-; CHECK-NEXT: orr r0, r2, r0, lsl #24
-; CHECK-NEXT: orr r0, r0, r1
-; CHECK-NEXT: mov pc, lr
-
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    ldr r0, [r0, #1]
+; CHECK-NEXT:    mov r1, #65280
+; CHECK-NEXT:    mov r2, #16711680
+; CHECK-NEXT:    and r1, r1, r0, lsr #8
+; CHECK-NEXT:    and r2, r2, r0, lsl #8
+; CHECK-NEXT:    orr r1, r1, r0, lsr #24
+; CHECK-NEXT:    orr r0, r2, r0, lsl #24
+; CHECK-NEXT:    orr r0, r0, r1
+; CHECK-NEXT:    mov pc, lr
+;
 ; CHECK-ARMv6-LABEL: load_i32_by_i8_nonzero_offset:
-; CHECK-ARMv6: ldr r0, [r0, #1]
-; CHECK-ARMv6-NEXT: rev r0, r0
-; CHECK-ARMv6-NEXT: bx  lr
+; CHECK-ARMv6:       @ %bb.0:
+; CHECK-ARMv6-NEXT:    ldr r0, [r0, #1]
+; CHECK-ARMv6-NEXT:    rev r0, r0
+; CHECK-ARMv6-NEXT:    bx lr
+;
+; CHECK-THUMBv6-LABEL: load_i32_by_i8_nonzero_offset:
+; CHECK-THUMBv6:       @ %bb.0:
+; CHECK-THUMBv6-NEXT:    movs r1, #1
+; CHECK-THUMBv6-NEXT:    ldr r0, [r0, r1]
+; CHECK-THUMBv6-NEXT:    rev r0, r0
+; CHECK-THUMBv6-NEXT:    bx lr
+;
+; CHECK-THUMBv7-LABEL: load_i32_by_i8_nonzero_offset:
+; CHECK-THUMBv7:       @ %bb.0:
+; CHECK-THUMBv7-NEXT:    movs r1, #1
+; CHECK-THUMBv7-NEXT:    ldr r0, [r0, r1]
+; CHECK-THUMBv7-NEXT:    rev r0, r0
+; CHECK-THUMBv7-NEXT:    bx lr
+
 
   %tmp = bitcast i32* %arg to i8*
   %tmp1 = getelementptr inbounds i8, i8* %tmp, i32 1
@@ -315,20 +444,37 @@ define i32 @load_i32_by_i8_nonzero_offset(i32* %arg) {
 ; (i32) p[-4] | ((i32) p[-3] << 8) | ((i32) p[-2] << 16) | ((i32) p[-1] << 24)
 define i32 @load_i32_by_i8_neg_offset(i32* %arg) {
 ; CHECK-LABEL: load_i32_by_i8_neg_offset:
-; CHECK: ldr r0, [r0, #-4]
-; CHECK-NEXT: mov r1, #65280
-; CHECK-NEXT: mov r2, #16711680
-; CHECK-NEXT: and r1, r1, r0, lsr #8
-; CHECK-NEXT: and r2, r2, r0, lsl #8
-; CHECK-NEXT: orr r1, r1, r0, lsr #24
-; CHECK-NEXT: orr r0, r2, r0, lsl #24
-; CHECK-NEXT: orr r0, r0, r1
-; CHECK-NEXT: mov pc, lr
-
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    ldr r0, [r0, #-4]
+; CHECK-NEXT:    mov r1, #65280
+; CHECK-NEXT:    mov r2, #16711680
+; CHECK-NEXT:    and r1, r1, r0, lsr #8
+; CHECK-NEXT:    and r2, r2, r0, lsl #8
+; CHECK-NEXT:    orr r1, r1, r0, lsr #24
+; CHECK-NEXT:    orr r0, r2, r0, lsl #24
+; CHECK-NEXT:    orr r0, r0, r1
+; CHECK-NEXT:    mov pc, lr
+;
 ; CHECK-ARMv6-LABEL: load_i32_by_i8_neg_offset:
-; CHECK-ARMv6: ldr r0, [r0, #-4]
-; CHECK-ARMv6-NEXT: rev r0, r0
-; CHECK-ARMv6-NEXT: bx  lr
+; CHECK-ARMv6:       @ %bb.0:
+; CHECK-ARMv6-NEXT:    ldr r0, [r0, #-4]
+; CHECK-ARMv6-NEXT:    rev r0, r0
+; CHECK-ARMv6-NEXT:    bx lr
+;
+; CHECK-THUMBv6-LABEL: load_i32_by_i8_neg_offset:
+; CHECK-THUMBv6:       @ %bb.0:
+; CHECK-THUMBv6-NEXT:    subs r0, r0, #4
+; CHECK-THUMBv6-NEXT:    ldr r0, [r0]
+; CHECK-THUMBv6-NEXT:    rev r0, r0
+; CHECK-THUMBv6-NEXT:    bx lr
+;
+; CHECK-THUMBv7-LABEL: load_i32_by_i8_neg_offset:
+; CHECK-THUMBv7:       @ %bb.0:
+; CHECK-THUMBv7-NEXT:    subs r0, r0, #4
+; CHECK-THUMBv7-NEXT:    ldr r0, [r0]
+; CHECK-THUMBv7-NEXT:    rev r0, r0
+; CHECK-THUMBv7-NEXT:    bx lr
+
 
   %tmp = bitcast i32* %arg to i8*
   %tmp1 = getelementptr inbounds i8, i8* %tmp, i32 -4
@@ -356,12 +502,27 @@ define i32 @load_i32_by_i8_neg_offset(i32* %arg) {
 ; (i32) p[4] | ((i32) p[3] << 8) | ((i32) p[2] << 16) | ((i32) p[1] << 24)
 define i32 @load_i32_by_i8_nonzero_offset_bswap(i32* %arg) {
 ; CHECK-LABEL: load_i32_by_i8_nonzero_offset_bswap:
-; CHECK: ldr r0, [r0, #1]
-; CHECK-NEXT: mov pc, lr
-
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    ldr r0, [r0, #1]
+; CHECK-NEXT:    mov pc, lr
+;
 ; CHECK-ARMv6-LABEL: load_i32_by_i8_nonzero_offset_bswap:
-; CHECK-ARMv6: ldr r0, [r0, #1]
-; CHECK-ARMv6-NEXT: bx  lr
+; CHECK-ARMv6:       @ %bb.0:
+; CHECK-ARMv6-NEXT:    ldr r0, [r0, #1]
+; CHECK-ARMv6-NEXT:    bx lr
+;
+; CHECK-THUMBv6-LABEL: load_i32_by_i8_nonzero_offset_bswap:
+; CHECK-THUMBv6:       @ %bb.0:
+; CHECK-THUMBv6-NEXT:    movs r1, #1
+; CHECK-THUMBv6-NEXT:    ldr r0, [r0, r1]
+; CHECK-THUMBv6-NEXT:    bx lr
+;
+; CHECK-THUMBv7-LABEL: load_i32_by_i8_nonzero_offset_bswap:
+; CHECK-THUMBv7:       @ %bb.0:
+; CHECK-THUMBv7-NEXT:    movs r1, #1
+; CHECK-THUMBv7-NEXT:    ldr r0, [r0, r1]
+; CHECK-THUMBv7-NEXT:    bx lr
+
 
   %tmp = bitcast i32* %arg to i8*
   %tmp1 = getelementptr inbounds i8, i8* %tmp, i32 4
@@ -389,12 +550,27 @@ define i32 @load_i32_by_i8_nonzero_offset_bswap(i32* %arg) {
 ; (i32) p[-1] | ((i32) p[-2] << 8) | ((i32) p[-3] << 16) | ((i32) p[-4] << 24)
 define i32 @load_i32_by_i8_neg_offset_bswap(i32* %arg) {
 ; CHECK-LABEL: load_i32_by_i8_neg_offset_bswap:
-; CHECK: ldr r0, [r0, #-4]
-; CHECK-NEXT: mov pc, lr
-
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    ldr r0, [r0, #-4]
+; CHECK-NEXT:    mov pc, lr
+;
 ; CHECK-ARMv6-LABEL: load_i32_by_i8_neg_offset_bswap:
-; CHECK-ARMv6: ldr r0, [r0, #-4]
-; CHECK-ARMv6-NEXT: bx  lr
+; CHECK-ARMv6:       @ %bb.0:
+; CHECK-ARMv6-NEXT:    ldr r0, [r0, #-4]
+; CHECK-ARMv6-NEXT:    bx lr
+;
+; CHECK-THUMBv6-LABEL: load_i32_by_i8_neg_offset_bswap:
+; CHECK-THUMBv6:       @ %bb.0:
+; CHECK-THUMBv6-NEXT:    subs r0, r0, #4
+; CHECK-THUMBv6-NEXT:    ldr r0, [r0]
+; CHECK-THUMBv6-NEXT:    bx lr
+;
+; CHECK-THUMBv7-LABEL: load_i32_by_i8_neg_offset_bswap:
+; CHECK-THUMBv7:       @ %bb.0:
+; CHECK-THUMBv7-NEXT:    subs r0, r0, #4
+; CHECK-THUMBv7-NEXT:    ldr r0, [r0]
+; CHECK-THUMBv7-NEXT:    bx lr
+
 
   %tmp = bitcast i32* %arg to i8*
   %tmp1 = getelementptr inbounds i8, i8* %tmp, i32 -1
@@ -424,20 +600,35 @@ declare i16 @llvm.bswap.i16(i16)
 ; (i32) bswap(p[0]) | (i32) bswap(p[1] << 16)
 define i32 @load_i32_by_bswap_i16(i32* %arg) {
 ; CHECK-LABEL: load_i32_by_bswap_i16:
-; CHECK: ldr r0, [r0]
-; CHECK-NEXT: mov r1, #65280
-; CHECK-NEXT: mov r2, #16711680
-; CHECK-NEXT: and r1, r1, r0, lsr #8
-; CHECK-NEXT: and r2, r2, r0, lsl #8
-; CHECK-NEXT: orr r1, r1, r0, lsr #24
-; CHECK-NEXT: orr r0, r2, r0, lsl #24
-; CHECK-NEXT: orr r0, r0, r1
-; CHECK-NEXT: mov pc, lr
-
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    ldr r0, [r0]
+; CHECK-NEXT:    mov r1, #65280
+; CHECK-NEXT:    mov r2, #16711680
+; CHECK-NEXT:    and r1, r1, r0, lsr #8
+; CHECK-NEXT:    and r2, r2, r0, lsl #8
+; CHECK-NEXT:    orr r1, r1, r0, lsr #24
+; CHECK-NEXT:    orr r0, r2, r0, lsl #24
+; CHECK-NEXT:    orr r0, r0, r1
+; CHECK-NEXT:    mov pc, lr
+;
 ; CHECK-ARMv6-LABEL: load_i32_by_bswap_i16:
-; CHECK-ARMv6: ldr  r0, [r0]
-; CHECK-ARMv6-NEXT: rev r0, r0
-; CHECK-ARMv6-NEXT: bx  lr
+; CHECK-ARMv6:       @ %bb.0:
+; CHECK-ARMv6-NEXT:    ldr r0, [r0]
+; CHECK-ARMv6-NEXT:    rev r0, r0
+; CHECK-ARMv6-NEXT:    bx lr
+;
+; CHECK-THUMBv6-LABEL: load_i32_by_bswap_i16:
+; CHECK-THUMBv6:       @ %bb.0:
+; CHECK-THUMBv6-NEXT:    ldr r0, [r0]
+; CHECK-THUMBv6-NEXT:    rev r0, r0
+; CHECK-THUMBv6-NEXT:    bx lr
+;
+; CHECK-THUMBv7-LABEL: load_i32_by_bswap_i16:
+; CHECK-THUMBv7:       @ %bb.0:
+; CHECK-THUMBv7-NEXT:    ldr r0, [r0]
+; CHECK-THUMBv7-NEXT:    rev r0, r0
+; CHECK-THUMBv7-NEXT:    bx lr
+
 
   %tmp = bitcast i32* %arg to i16*
   %tmp1 = load i16, i16* %tmp, align 4
@@ -456,12 +647,24 @@ define i32 @load_i32_by_bswap_i16(i32* %arg) {
 ; (i32) p[1] | (sext(p[0] << 16) to i32)
 define i32 @load_i32_by_sext_i16(i32* %arg) {
 ; CHECK-LABEL: load_i32_by_sext_i16:
-; CHECK: ldr  r0, [r0]
-; CHECK-NEXT: mov pc, lr
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    ldr r0, [r0]
+; CHECK-NEXT:    mov pc, lr
 ;
 ; CHECK-ARMv6-LABEL: load_i32_by_sext_i16:
-; CHECK-ARMv6: ldr r0, [r0]
-; CHECK-ARMv6-NEXT: bx  lr
+; CHECK-ARMv6:       @ %bb.0:
+; CHECK-ARMv6-NEXT:    ldr r0, [r0]
+; CHECK-ARMv6-NEXT:    bx lr
+;
+; CHECK-THUMBv6-LABEL: load_i32_by_sext_i16:
+; CHECK-THUMBv6:       @ %bb.0:
+; CHECK-THUMBv6-NEXT:    ldr r0, [r0]
+; CHECK-THUMBv6-NEXT:    bx lr
+;
+; CHECK-THUMBv7-LABEL: load_i32_by_sext_i16:
+; CHECK-THUMBv7:       @ %bb.0:
+; CHECK-THUMBv7-NEXT:    ldr r0, [r0]
+; CHECK-THUMBv7-NEXT:    bx lr
   %tmp = bitcast i32* %arg to i16*
   %tmp1 = load i16, i16* %tmp, align 4
   %tmp2 = sext i16 %tmp1 to i32
@@ -478,22 +681,38 @@ define i32 @load_i32_by_sext_i16(i32* %arg) {
 ; (i32) p[i] | ((i32) p[i + 1] << 8) | ((i32) p[i + 2] << 16) | ((i32) p[i + 3] << 24)
 define i32 @load_i32_by_i8_base_offset_index(i8* %arg, i32 %i) {
 ; CHECK-LABEL: load_i32_by_i8_base_offset_index:
-; CHECK: add r0, r0, r1
-; CHECK-NEXT: mov r1, #65280
-; CHECK-NEXT: mov r2, #16711680
-; CHECK-NEXT: ldr r0, [r0, #12]
-; CHECK-NEXT: and r1, r1, r0, lsr #8
-; CHECK-NEXT: and r2, r2, r0, lsl #8
-; CHECK-NEXT: orr r1, r1, r0, lsr #24
-; CHECK-NEXT: orr r0, r2, r0, lsl #24
-; CHECK-NEXT: orr r0, r0, r1
-; CHECK-NEXT: mov pc, lr
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    add r0, r0, r1
+; CHECK-NEXT:    mov r1, #65280
+; CHECK-NEXT:    mov r2, #16711680
+; CHECK-NEXT:    ldr r0, [r0, #12]
+; CHECK-NEXT:    and r1, r1, r0, lsr #8
+; CHECK-NEXT:    and r2, r2, r0, lsl #8
+; CHECK-NEXT:    orr r1, r1, r0, lsr #24
+; CHECK-NEXT:    orr r0, r2, r0, lsl #24
+; CHECK-NEXT:    orr r0, r0, r1
+; CHECK-NEXT:    mov pc, lr
 ;
 ; CHECK-ARMv6-LABEL: load_i32_by_i8_base_offset_index:
-; CHECK-ARMv6: add r0, r0, r1
-; CHECK-ARMv6-NEXT: ldr r0, [r0, #12]
-; CHECK-ARMv6-NEXT: rev r0, r0
-; CHECK-ARMv6-NEXT: bx  lr
+; CHECK-ARMv6:       @ %bb.0:
+; CHECK-ARMv6-NEXT:    add r0, r0, r1
+; CHECK-ARMv6-NEXT:    ldr r0, [r0, #12]
+; CHECK-ARMv6-NEXT:    rev r0, r0
+; CHECK-ARMv6-NEXT:    bx lr
+;
+; CHECK-THUMBv6-LABEL: load_i32_by_i8_base_offset_index:
+; CHECK-THUMBv6:       @ %bb.0:
+; CHECK-THUMBv6-NEXT:    adds r0, r0, r1
+; CHECK-THUMBv6-NEXT:    ldr r0, [r0, #12]
+; CHECK-THUMBv6-NEXT:    rev r0, r0
+; CHECK-THUMBv6-NEXT:    bx lr
+;
+; CHECK-THUMBv7-LABEL: load_i32_by_i8_base_offset_index:
+; CHECK-THUMBv7:       @ %bb.0:
+; CHECK-THUMBv7-NEXT:    adds r0, r0, r1
+; CHECK-THUMBv7-NEXT:    ldr r0, [r0, #12]
+; CHECK-THUMBv7-NEXT:    rev r0, r0
+; CHECK-THUMBv7-NEXT:    bx lr
   %tmp = add nuw nsw i32 %i, 3
   %tmp2 = add nuw nsw i32 %i, 2
   %tmp3 = add nuw nsw i32 %i, 1
@@ -528,22 +747,40 @@ define i32 @load_i32_by_i8_base_offset_index(i8* %arg, i32 %i) {
 ; (i32) p[i + 1] | ((i32) p[i + 2] << 8) | ((i32) p[i + 3] << 16) | ((i32) p[i + 4] << 24)
 define i32 @load_i32_by_i8_base_offset_index_2(i8* %arg, i32 %i) {
 ; CHECK-LABEL: load_i32_by_i8_base_offset_index_2:
-; CHECK: add r0, r1, r0
-; CHECK-NEXT: mov r1, #65280
-; CHECK-NEXT: mov r2, #16711680
-; CHECK-NEXT: ldr r0, [r0, #13]
-; CHECK-NEXT: and r1, r1, r0, lsr #8
-; CHECK-NEXT: and r2, r2, r0, lsl #8
-; CHECK-NEXT: orr r1, r1, r0, lsr #24
-; CHECK-NEXT: orr r0, r2, r0, lsl #24
-; CHECK-NEXT: orr r0, r0, r1
-; CHECK-NEXT: mov pc, lr
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    add r0, r1, r0
+; CHECK-NEXT:    mov r1, #65280
+; CHECK-NEXT:    mov r2, #16711680
+; CHECK-NEXT:    ldr r0, [r0, #13]
+; CHECK-NEXT:    and r1, r1, r0, lsr #8
+; CHECK-NEXT:    and r2, r2, r0, lsl #8
+; CHECK-NEXT:    orr r1, r1, r0, lsr #24
+; CHECK-NEXT:    orr r0, r2, r0, lsl #24
+; CHECK-NEXT:    orr r0, r0, r1
+; CHECK-NEXT:    mov pc, lr
 ;
 ; CHECK-ARMv6-LABEL: load_i32_by_i8_base_offset_index_2:
-; CHECK-ARMv6: add r0, r1, r0
-; CHECK-ARMv6-NEXT: ldr r0, [r0, #13]
-; CHECK-ARMv6-NEXT: rev r0, r0
-; CHECK-ARMv6-NEXT: bx  lr
+; CHECK-ARMv6:       @ %bb.0:
+; CHECK-ARMv6-NEXT:    add r0, r1, r0
+; CHECK-ARMv6-NEXT:    ldr r0, [r0, #13]
+; CHECK-ARMv6-NEXT:    rev r0, r0
+; CHECK-ARMv6-NEXT:    bx lr
+;
+; CHECK-THUMBv6-LABEL: load_i32_by_i8_base_offset_index_2:
+; CHECK-THUMBv6:       @ %bb.0:
+; CHECK-THUMBv6-NEXT:    adds r0, r1, r0
+; CHECK-THUMBv6-NEXT:    movs r1, #13
+; CHECK-THUMBv6-NEXT:    ldr r0, [r0, r1]
+; CHECK-THUMBv6-NEXT:    rev r0, r0
+; CHECK-THUMBv6-NEXT:    bx lr
+;
+; CHECK-THUMBv7-LABEL: load_i32_by_i8_base_offset_index_2:
+; CHECK-THUMBv7:       @ %bb.0:
+; CHECK-THUMBv7-NEXT:    adds r0, r1, r0
+; CHECK-THUMBv7-NEXT:    movs r1, #13
+; CHECK-THUMBv7-NEXT:    ldr r0, [r0, r1]
+; CHECK-THUMBv7-NEXT:    rev r0, r0
+; CHECK-THUMBv7-NEXT:    bx lr
 
   %tmp = add nuw nsw i32 %i, 4
   %tmp2 = add nuw nsw i32 %i, 3
@@ -579,16 +816,32 @@ define i32 @load_i32_by_i8_base_offset_index_2(i8* %arg, i32 %i) {
 ; (i32) p[0] | ((i32) p[1] << 8)
 define i32 @zext_load_i32_by_i8(i32* %arg) {
 ; CHECK-LABEL: zext_load_i32_by_i8:
-; CHECK: ldrb  r1, [r0]
-; CHECK-NEXT: ldrb  r0, [r0, #1]
-; CHECK-NEXT: orr r0, r1, r0, lsl #8
-; CHECK-NEXT: mov pc, lr
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    ldrb r1, [r0]
+; CHECK-NEXT:    ldrb r0, [r0, #1]
+; CHECK-NEXT:    orr r0, r1, r0, lsl #8
+; CHECK-NEXT:    mov pc, lr
 ;
 ; CHECK-ARMv6-LABEL: zext_load_i32_by_i8:
-; CHECK-ARMv6: ldrb  r1, [r0]
-; CHECK-ARMv6-NEXT: ldrb  r0, [r0, #1]
-; CHECK-ARMv6-NEXT: orr r0, r1, r0, lsl #8
-; CHECK-ARMv6-NEXT: bx  lr
+; CHECK-ARMv6:       @ %bb.0:
+; CHECK-ARMv6-NEXT:    ldrh r0, [r0]
+; CHECK-ARMv6-NEXT:    lsl r0, r0, #16
+; CHECK-ARMv6-NEXT:    rev r0, r0
+; CHECK-ARMv6-NEXT:    bx lr
+;
+; CHECK-THUMBv6-LABEL: zext_load_i32_by_i8:
+; CHECK-THUMBv6:       @ %bb.0:
+; CHECK-THUMBv6-NEXT:    ldrh r0, [r0]
+; CHECK-THUMBv6-NEXT:    lsls r0, r0, #16
+; CHECK-THUMBv6-NEXT:    rev r0, r0
+; CHECK-THUMBv6-NEXT:    bx lr
+;
+; CHECK-THUMBv7-LABEL: zext_load_i32_by_i8:
+; CHECK-THUMBv7:       @ %bb.0:
+; CHECK-THUMBv7-NEXT:    ldrh r0, [r0]
+; CHECK-THUMBv7-NEXT:    lsls r0, r0, #16
+; CHECK-THUMBv7-NEXT:    rev r0, r0
+; CHECK-THUMBv7-NEXT:    bx lr
 
   %tmp = bitcast i32* %arg to i8*
   %tmp1 = getelementptr inbounds i8, i8* %tmp, i32 0
@@ -606,18 +859,38 @@ define i32 @zext_load_i32_by_i8(i32* %arg) {
 ; ((i32) p[0] << 8) | ((i32) p[1] << 16)
 define i32 @zext_load_i32_by_i8_shl_8(i32* %arg) {
 ; CHECK-LABEL: zext_load_i32_by_i8_shl_8:
-; CHECK: ldrb  r1, [r0]
-; CHECK-NEXT: ldrb  r0, [r0, #1]
-; CHECK-NEXT: lsl r0, r0, #16
-; CHECK-NEXT: orr r0, r0, r1, lsl #8
-; CHECK-NEXT: mov pc, lr
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    ldrb r1, [r0]
+; CHECK-NEXT:    ldrb r0, [r0, #1]
+; CHECK-NEXT:    lsl r0, r0, #16
+; CHECK-NEXT:    orr r0, r0, r1, lsl #8
+; CHECK-NEXT:    mov pc, lr
 ;
 ; CHECK-ARMv6-LABEL: zext_load_i32_by_i8_shl_8:
-; CHECK-ARMv6: ldrb  r1, [r0]
-; CHECK-ARMv6-NEXT: ldrb  r0, [r0, #1]
-; CHECK-ARMv6-NEXT: lsl r0, r0, #16
-; CHECK-ARMv6-NEXT: orr r0, r0, r1, lsl #8
-; CHECK-ARMv6-NEXT: bx  lr
+; CHECK-ARMv6:       @ %bb.0:
+; CHECK-ARMv6-NEXT:    ldrb r1, [r0]
+; CHECK-ARMv6-NEXT:    ldrb r0, [r0, #1]
+; CHECK-ARMv6-NEXT:    lsl r0, r0, #16
+; CHECK-ARMv6-NEXT:    orr r0, r0, r1, lsl #8
+; CHECK-ARMv6-NEXT:    bx lr
+;
+; CHECK-THUMBv6-LABEL: zext_load_i32_by_i8_shl_8:
+; CHECK-THUMBv6:       @ %bb.0:
+; CHECK-THUMBv6-NEXT:    ldrb r1, [r0]
+; CHECK-THUMBv6-NEXT:    lsls r1, r1, #8
+; CHECK-THUMBv6-NEXT:    ldrb r0, [r0, #1]
+; CHECK-THUMBv6-NEXT:    lsls r0, r0, #16
+; CHECK-THUMBv6-NEXT:    adds r0, r0, r1
+; CHECK-THUMBv6-NEXT:    bx lr
+;
+; CHECK-THUMBv7-LABEL: zext_load_i32_by_i8_shl_8:
+; CHECK-THUMBv7:       @ %bb.0:
+; CHECK-THUMBv7-NEXT:    ldrb r1, [r0]
+; CHECK-THUMBv7-NEXT:    lsls r1, r1, #8
+; CHECK-THUMBv7-NEXT:    ldrb r0, [r0, #1]
+; CHECK-THUMBv7-NEXT:    lsls r0, r0, #16
+; CHECK-THUMBv7-NEXT:    adds r0, r0, r1
+; CHECK-THUMBv7-NEXT:    bx lr
 
   %tmp = bitcast i32* %arg to i8*
   %tmp1 = getelementptr inbounds i8, i8* %tmp, i32 0
@@ -636,18 +909,38 @@ define i32 @zext_load_i32_by_i8_shl_8(i32* %arg) {
 ; ((i32) p[0] << 16) | ((i32) p[1] << 24)
 define i32 @zext_load_i32_by_i8_shl_16(i32* %arg) {
 ; CHECK-LABEL: zext_load_i32_by_i8_shl_16:
-; CHECK: ldrb  r1, [r0]
-; CHECK-NEXT: ldrb  r0, [r0, #1]
-; CHECK-NEXT: lsl r0, r0, #24
-; CHECK-NEXT: orr r0, r0, r1, lsl #16
-; CHECK-NEXT: mov pc, lr
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    ldrb r1, [r0]
+; CHECK-NEXT:    ldrb r0, [r0, #1]
+; CHECK-NEXT:    lsl r0, r0, #24
+; CHECK-NEXT:    orr r0, r0, r1, lsl #16
+; CHECK-NEXT:    mov pc, lr
 ;
 ; CHECK-ARMv6-LABEL: zext_load_i32_by_i8_shl_16:
-; CHECK-ARMv6: ldrb  r1, [r0]
-; CHECK-ARMv6-NEXT: ldrb  r0, [r0, #1]
-; CHECK-ARMv6-NEXT: lsl r0, r0, #24
-; CHECK-ARMv6-NEXT: orr r0, r0, r1, lsl #16
-; CHECK-ARMv6-NEXT: bx  lr
+; CHECK-ARMv6:       @ %bb.0:
+; CHECK-ARMv6-NEXT:    ldrb r1, [r0]
+; CHECK-ARMv6-NEXT:    ldrb r0, [r0, #1]
+; CHECK-ARMv6-NEXT:    lsl r0, r0, #24
+; CHECK-ARMv6-NEXT:    orr r0, r0, r1, lsl #16
+; CHECK-ARMv6-NEXT:    bx lr
+;
+; CHECK-THUMBv6-LABEL: zext_load_i32_by_i8_shl_16:
+; CHECK-THUMBv6:       @ %bb.0:
+; CHECK-THUMBv6-NEXT:    ldrb r1, [r0]
+; CHECK-THUMBv6-NEXT:    lsls r1, r1, #16
+; CHECK-THUMBv6-NEXT:    ldrb r0, [r0, #1]
+; CHECK-THUMBv6-NEXT:    lsls r0, r0, #24
+; CHECK-THUMBv6-NEXT:    adds r0, r0, r1
+; CHECK-THUMBv6-NEXT:    bx lr
+;
+; CHECK-THUMBv7-LABEL: zext_load_i32_by_i8_shl_16:
+; CHECK-THUMBv7:       @ %bb.0:
+; CHECK-THUMBv7-NEXT:    ldrb r1, [r0]
+; CHECK-THUMBv7-NEXT:    lsls r1, r1, #16
+; CHECK-THUMBv7-NEXT:    ldrb r0, [r0, #1]
+; CHECK-THUMBv7-NEXT:    lsls r0, r0, #24
+; CHECK-THUMBv7-NEXT:    adds r0, r0, r1
+; CHECK-THUMBv7-NEXT:    bx lr
 
   %tmp = bitcast i32* %arg to i8*
   %tmp1 = getelementptr inbounds i8, i8* %tmp, i32 0
@@ -666,16 +959,24 @@ define i32 @zext_load_i32_by_i8_shl_16(i32* %arg) {
 ; (i32) p[1] | ((i32) p[0] << 8)
 define i32 @zext_load_i32_by_i8_bswap(i32* %arg) {
 ; CHECK-LABEL: zext_load_i32_by_i8_bswap:
-; CHECK: ldrb  r1, [r0]
-; CHECK-NEXT: ldrb  r0, [r0, #1]
-; CHECK-NEXT: orr r0, r0, r1, lsl #8
-; CHECK-NEXT: mov pc, lr
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    ldrh r0, [r0]
+; CHECK-NEXT:    mov pc, lr
 ;
 ; CHECK-ARMv6-LABEL: zext_load_i32_by_i8_bswap:
-; CHECK-ARMv6: ldrb  r1, [r0]
-; CHECK-ARMv6-NEXT: ldrb  r0, [r0, #1]
-; CHECK-ARMv6-NEXT: orr r0, r0, r1, lsl #8
-; CHECK-ARMv6-NEXT: bx  lr
+; CHECK-ARMv6:       @ %bb.0:
+; CHECK-ARMv6-NEXT:    ldrh r0, [r0]
+; CHECK-ARMv6-NEXT:    bx lr
+;
+; CHECK-THUMBv6-LABEL: zext_load_i32_by_i8_bswap:
+; CHECK-THUMBv6:       @ %bb.0:
+; CHECK-THUMBv6-NEXT:    ldrh r0, [r0]
+; CHECK-THUMBv6-NEXT:    bx lr
+;
+; CHECK-THUMBv7-LABEL: zext_load_i32_by_i8_bswap:
+; CHECK-THUMBv7:       @ %bb.0:
+; CHECK-THUMBv7-NEXT:    ldrh r0, [r0]
+; CHECK-THUMBv7-NEXT:    bx lr
 
   %tmp = bitcast i32* %arg to i8*
   %tmp1 = getelementptr inbounds i8, i8* %tmp, i32 1
@@ -693,18 +994,38 @@ define i32 @zext_load_i32_by_i8_bswap(i32* %arg) {
 ; ((i32) p[1] << 8) | ((i32) p[0] << 16)
 define i32 @zext_load_i32_by_i8_bswap_shl_8(i32* %arg) {
 ; CHECK-LABEL: zext_load_i32_by_i8_bswap_shl_8:
-; CHECK: ldrb  r1, [r0]
-; CHECK-NEXT: ldrb  r0, [r0, #1]
-; CHECK-NEXT: lsl r1, r1, #16
-; CHECK-NEXT: orr r0, r1, r0, lsl #8
-; CHECK-NEXT: mov pc, lr
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    ldrb r1, [r0]
+; CHECK-NEXT:    ldrb r0, [r0, #1]
+; CHECK-NEXT:    lsl r1, r1, #16
+; CHECK-NEXT:    orr r0, r1, r0, lsl #8
+; CHECK-NEXT:    mov pc, lr
 ;
 ; CHECK-ARMv6-LABEL: zext_load_i32_by_i8_bswap_shl_8:
-; CHECK-ARMv6: ldrb  r1, [r0]
-; CHECK-ARMv6-NEXT: ldrb  r0, [r0, #1]
-; CHECK-ARMv6-NEXT: lsl r1, r1, #16
-; CHECK-ARMv6-NEXT: orr r0, r1, r0, lsl #8
-; CHECK-ARMv6-NEXT: bx  lr
+; CHECK-ARMv6:       @ %bb.0:
+; CHECK-ARMv6-NEXT:    ldrb r1, [r0]
+; CHECK-ARMv6-NEXT:    ldrb r0, [r0, #1]
+; CHECK-ARMv6-NEXT:    lsl r1, r1, #16
+; CHECK-ARMv6-NEXT:    orr r0, r1, r0, lsl #8
+; CHECK-ARMv6-NEXT:    bx lr
+;
+; CHECK-THUMBv6-LABEL: zext_load_i32_by_i8_bswap_shl_8:
+; CHECK-THUMBv6:       @ %bb.0:
+; CHECK-THUMBv6-NEXT:    ldrb r1, [r0, #1]
+; CHECK-THUMBv6-NEXT:    lsls r1, r1, #8
+; CHECK-THUMBv6-NEXT:    ldrb r0, [r0]
+; CHECK-THUMBv6-NEXT:    lsls r0, r0, #16
+; CHECK-THUMBv6-NEXT:    adds r0, r0, r1
+; CHECK-THUMBv6-NEXT:    bx lr
+;
+; CHECK-THUMBv7-LABEL: zext_load_i32_by_i8_bswap_shl_8:
+; CHECK-THUMBv7:       @ %bb.0:
+; CHECK-THUMBv7-NEXT:    ldrb r1, [r0, #1]
+; CHECK-THUMBv7-NEXT:    lsls r1, r1, #8
+; CHECK-THUMBv7-NEXT:    ldrb r0, [r0]
+; CHECK-THUMBv7-NEXT:    lsls r0, r0, #16
+; CHECK-THUMBv7-NEXT:    adds r0, r0, r1
+; CHECK-THUMBv7-NEXT:    bx lr
 
   %tmp = bitcast i32* %arg to i8*
   %tmp1 = getelementptr inbounds i8, i8* %tmp, i32 1
@@ -723,18 +1044,38 @@ define i32 @zext_load_i32_by_i8_bswap_shl_8(i32* %arg) {
 ; ((i32) p[1] << 16) | ((i32) p[0] << 24)
 define i32 @zext_load_i32_by_i8_bswap_shl_16(i32* %arg) {
 ; CHECK-LABEL: zext_load_i32_by_i8_bswap_shl_16:
-; CHECK: ldrb  r1, [r0]
-; CHECK-NEXT: ldrb  r0, [r0, #1]
-; CHECK-NEXT: lsl r1, r1, #24
-; CHECK-NEXT: orr r0, r1, r0, lsl #16
-; CHECK-NEXT: mov pc, lr
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    ldrb r1, [r0]
+; CHECK-NEXT:    ldrb r0, [r0, #1]
+; CHECK-NEXT:    lsl r1, r1, #24
+; CHECK-NEXT:    orr r0, r1, r0, lsl #16
+; CHECK-NEXT:    mov pc, lr
 ;
 ; CHECK-ARMv6-LABEL: zext_load_i32_by_i8_bswap_shl_16:
-; CHECK-ARMv6: ldrb  r1, [r0]
-; CHECK-ARMv6-NEXT: ldrb  r0, [r0, #1]
-; CHECK-ARMv6-NEXT: lsl r1, r1, #24
-; CHECK-ARMv6-NEXT: orr r0, r1, r0, lsl #16
-; CHECK-ARMv6-NEXT: bx  lr
+; CHECK-ARMv6:       @ %bb.0:
+; CHECK-ARMv6-NEXT:    ldrb r1, [r0]
+; CHECK-ARMv6-NEXT:    ldrb r0, [r0, #1]
+; CHECK-ARMv6-NEXT:    lsl r1, r1, #24
+; CHECK-ARMv6-NEXT:    orr r0, r1, r0, lsl #16
+; CHECK-ARMv6-NEXT:    bx lr
+;
+; CHECK-THUMBv6-LABEL: zext_load_i32_by_i8_bswap_shl_16:
+; CHECK-THUMBv6:       @ %bb.0:
+; CHECK-THUMBv6-NEXT:    ldrb r1, [r0, #1]
+; CHECK-THUMBv6-NEXT:    lsls r1, r1, #16
+; CHECK-THUMBv6-NEXT:    ldrb r0, [r0]
+; CHECK-THUMBv6-NEXT:    lsls r0, r0, #24
+; CHECK-THUMBv6-NEXT:    adds r0, r0, r1
+; CHECK-THUMBv6-NEXT:    bx lr
+;
+; CHECK-THUMBv7-LABEL: zext_load_i32_by_i8_bswap_shl_16:
+; CHECK-THUMBv7:       @ %bb.0:
+; CHECK-THUMBv7-NEXT:    ldrb r1, [r0, #1]
+; CHECK-THUMBv7-NEXT:    lsls r1, r1, #16
+; CHECK-THUMBv7-NEXT:    ldrb r0, [r0]
+; CHECK-THUMBv7-NEXT:    lsls r0, r0, #24
+; CHECK-THUMBv7-NEXT:    adds r0, r0, r1
+; CHECK-THUMBv7-NEXT:    bx lr
 
   %tmp = bitcast i32* %arg to i8*
   %tmp1 = getelementptr inbounds i8, i8* %tmp, i32 1
@@ -757,16 +1098,34 @@ define i32 @zext_load_i32_by_i8_bswap_shl_16(i32* %arg) {
 ; because in the original DAG we don't have p[1] address available
 define i16 @load_i16_from_nonzero_offset(i8* %p) {
 ; CHECK-LABEL: load_i16_from_nonzero_offset:
-; CHECK: ldrh  r1, [r0]
-; CHECK-NEXT: ldrb  r0, [r0, #2]
-; CHECK-NEXT: orr r0, r0, r1, lsl #8
-; CHECK-NEXT: mov pc, lr
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    ldrh r1, [r0]
+; CHECK-NEXT:    ldrb r0, [r0, #2]
+; CHECK-NEXT:    orr r0, r0, r1, lsl #8
+; CHECK-NEXT:    mov pc, lr
 ;
 ; CHECK-ARMv6-LABEL: load_i16_from_nonzero_offset:
-; CHECK-ARMv6: ldrh  r1, [r0]
-; CHECK-ARMv6-NEXT: ldrb  r0, [r0, #2]
-; CHECK-ARMv6-NEXT: orr r0, r0, r1, lsl #8
-; CHECK-ARMv6-NEXT: bx  lr
+; CHECK-ARMv6:       @ %bb.0:
+; CHECK-ARMv6-NEXT:    ldrh r1, [r0]
+; CHECK-ARMv6-NEXT:    ldrb r0, [r0, #2]
+; CHECK-ARMv6-NEXT:    orr r0, r0, r1, lsl #8
+; CHECK-ARMv6-NEXT:    bx lr
+;
+; CHECK-THUMBv6-LABEL: load_i16_from_nonzero_offset:
+; CHECK-THUMBv6:       @ %bb.0:
+; CHECK-THUMBv6-NEXT:    ldrb r1, [r0, #2]
+; CHECK-THUMBv6-NEXT:    ldrh r0, [r0]
+; CHECK-THUMBv6-NEXT:    lsls r0, r0, #8
+; CHECK-THUMBv6-NEXT:    adds r0, r0, r1
+; CHECK-THUMBv6-NEXT:    bx lr
+;
+; CHECK-THUMBv7-LABEL: load_i16_from_nonzero_offset:
+; CHECK-THUMBv7:       @ %bb.0:
+; CHECK-THUMBv7-NEXT:    ldrb r1, [r0, #2]
+; CHECK-THUMBv7-NEXT:    ldrh r0, [r0]
+; CHECK-THUMBv7-NEXT:    lsls r0, r0, #8
+; CHECK-THUMBv7-NEXT:    adds r0, r0, r1
+; CHECK-THUMBv7-NEXT:    bx lr
 
   %p1.i16 = bitcast i8* %p to i16*
   %p2.i8 = getelementptr i8, i8* %p, i64 2
diff --git a/llvm/test/CodeGen/ARM/load-combine.ll b/llvm/test/CodeGen/ARM/load-combine.ll
index 4206aad1d..bf03898c8 100644
--- a/llvm/test/CodeGen/ARM/load-combine.ll
+++ b/llvm/test/CodeGen/ARM/load-combine.ll
@@ -1,24 +1,53 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=arm-unknown | FileCheck %s
 ; RUN: llc < %s -mtriple=armv6-unknown | FileCheck %s --check-prefix=CHECK-ARMv6
+; RUN: llc < %s -mtriple=thumbv6m-none-eabi | FileCheck %s --check-prefix=CHECK-THUMBv6
+; RUN: llc < %s -mtriple=thumbv7m-none-eabi | FileCheck %s --check-prefix=CHECK-THUMBv7
 
 ; i8* p; // p is 1 byte aligned
 ; (i32) p[0] | ((i32) p[1] << 8) | ((i32) p[2] << 16) | ((i32) p[3] << 24)
 define i32 @load_i32_by_i8_unaligned(i32* %arg) {
 ; CHECK-LABEL: load_i32_by_i8_unaligned:
-; CHECK: ldrb{{.*}}r0
-; CHECK: ldrb{{.*}}r0
-; CHECK: ldrb{{.*}}r0
-; CHECK: ldrb{{.*}}r0
-; CHECK: orr
-; CHECK: mov pc, lr
-
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    ldrb r2, [r0, #1]
+; CHECK-NEXT:    ldrb r1, [r0]
+; CHECK-NEXT:    ldrb r3, [r0, #2]
+; CHECK-NEXT:    ldrb r0, [r0, #3]
+; CHECK-NEXT:    orr r1, r1, r2, lsl #8
+; CHECK-NEXT:    orr r1, r1, r3, lsl #16
+; CHECK-NEXT:    orr r0, r1, r0, lsl #24
+; CHECK-NEXT:    mov pc, lr
+;
 ; CHECK-ARMv6-LABEL: load_i32_by_i8_unaligned:
-; CHECK-ARMv6: ldrb{{.*}}r0
-; CHECK-ARMv6: ldrb{{.*}}r0
-; CHECK-ARMv6: ldrb{{.*}}r0
-; CHECK-ARMv6: ldrb{{.*}}r0
-; CHECK-ARMv6: orr
-; CHECK-ARMv6: bx lr
+; CHECK-ARMv6:       @ %bb.0:
+; CHECK-ARMv6-NEXT:    ldrb r2, [r0, #1]
+; CHECK-ARMv6-NEXT:    ldrb r1, [r0]
+; CHECK-ARMv6-NEXT:    ldrb r3, [r0, #2]
+; CHECK-ARMv6-NEXT:    ldrb r0, [r0, #3]
+; CHECK-ARMv6-NEXT:    orr r1, r1, r2, lsl #8
+; CHECK-ARMv6-NEXT:    orr r1, r1, r3, lsl #16
+; CHECK-ARMv6-NEXT:    orr r0, r1, r0, lsl #24
+; CHECK-ARMv6-NEXT:    bx lr
+;
+; CHECK-THUMBv6-LABEL: load_i32_by_i8_unaligned:
+; CHECK-THUMBv6:       @ %bb.0:
+; CHECK-THUMBv6-NEXT:    ldrb r1, [r0]
+; CHECK-THUMBv6-NEXT:    ldrb r2, [r0, #1]
+; CHECK-THUMBv6-NEXT:    lsls r2, r2, #8
+; CHECK-THUMBv6-NEXT:    adds r1, r2, r1
+; CHECK-THUMBv6-NEXT:    ldrb r2, [r0, #2]
+; CHECK-THUMBv6-NEXT:    lsls r2, r2, #16
+; CHECK-THUMBv6-NEXT:    adds r1, r1, r2
+; CHECK-THUMBv6-NEXT:    ldrb r0, [r0, #3]
+; CHECK-THUMBv6-NEXT:    lsls r0, r0, #24
+; CHECK-THUMBv6-NEXT:    adds r0, r1, r0
+; CHECK-THUMBv6-NEXT:    bx lr
+;
+; CHECK-THUMBv7-LABEL: load_i32_by_i8_unaligned:
+; CHECK-THUMBv7:       @ %bb.0:
+; CHECK-THUMBv7-NEXT:    ldr r0, [r0]
+; CHECK-THUMBv7-NEXT:    bx lr
+
   %tmp = bitcast i32* %arg to i8*
   %tmp1 = getelementptr inbounds i8, i8* %tmp, i32 0
   %tmp2 = load i8, i8* %tmp1, align 1
@@ -45,12 +74,25 @@ define i32 @load_i32_by_i8_unaligned(i32* %arg) {
 ; (i32) p[0] | ((i32) p[1] << 8) | ((i32) p[2] << 16) | ((i32) p[3] << 24)
 define i32 @load_i32_by_i8_aligned(i32* %arg) {
 ; CHECK-LABEL: load_i32_by_i8_aligned:
-; CHECK: ldr  r0, [r0]
-; CHECK-NEXT: mov pc, lr
-
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    ldr r0, [r0]
+; CHECK-NEXT:    mov pc, lr
+;
 ; CHECK-ARMv6-LABEL: load_i32_by_i8_aligned:
-; CHECK-ARMv6: ldr  r0, [r0]
-; CHECK-ARMv6-NEXT: bx lr
+; CHECK-ARMv6:       @ %bb.0:
+; CHECK-ARMv6-NEXT:    ldr r0, [r0]
+; CHECK-ARMv6-NEXT:    bx lr
+;
+; CHECK-THUMBv6-LABEL: load_i32_by_i8_aligned:
+; CHECK-THUMBv6:       @ %bb.0:
+; CHECK-THUMBv6-NEXT:    ldr r0, [r0]
+; CHECK-THUMBv6-NEXT:    bx lr
+;
+; CHECK-THUMBv7-LABEL: load_i32_by_i8_aligned:
+; CHECK-THUMBv7:       @ %bb.0:
+; CHECK-THUMBv7-NEXT:    ldr r0, [r0]
+; CHECK-THUMBv7-NEXT:    bx lr
+
   %tmp = bitcast i32* %arg to i8*
   %tmp1 = getelementptr inbounds i8, i8* %tmp, i32 0
   %tmp2 = load i8, i8* %tmp1, align 4
@@ -78,18 +120,35 @@ define i32 @load_i32_by_i8_aligned(i32* %arg) {
 define i32 @load_i32_by_i8_bswap(i32* %arg) {
 ; BSWAP is not supported by 32 bit target
 ; CHECK-LABEL: load_i32_by_i8_bswap:
-; CHECK: ldr r0, [r0]
-; CHECK: and
-; CHECK-NEXT: and
-; CHECK-NEXT: orr
-; CHECK-NEXT: orr
-; CHECK-NEXT: orr
-; CHECK: mov pc, lr
-
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    ldr r0, [r0]
+; CHECK-NEXT:    mov r1, #65280
+; CHECK-NEXT:    mov r2, #16711680
+; CHECK-NEXT:    and r1, r1, r0, lsr #8
+; CHECK-NEXT:    and r2, r2, r0, lsl #8
+; CHECK-NEXT:    orr r1, r1, r0, lsr #24
+; CHECK-NEXT:    orr r0, r2, r0, lsl #24
+; CHECK-NEXT:    orr r0, r0, r1
+; CHECK-NEXT:    mov pc, lr
+;
 ; CHECK-ARMv6-LABEL: load_i32_by_i8_bswap:
-; CHECK-ARMv6: ldr  r0, [r0]
-; CHECK-ARMv6-NEXT: rev  r0, r0
-; CHECK-ARMv6-NEXT: bx lr
+; CHECK-ARMv6:       @ %bb.0:
+; CHECK-ARMv6-NEXT:    ldr r0, [r0]
+; CHECK-ARMv6-NEXT:    rev r0, r0
+; CHECK-ARMv6-NEXT:    bx lr
+;
+; CHECK-THUMBv6-LABEL: load_i32_by_i8_bswap:
+; CHECK-THUMBv6:       @ %bb.0:
+; CHECK-THUMBv6-NEXT:    ldr r0, [r0]
+; CHECK-THUMBv6-NEXT:    rev r0, r0
+; CHECK-THUMBv6-NEXT:    bx lr
+;
+; CHECK-THUMBv7-LABEL: load_i32_by_i8_bswap:
+; CHECK-THUMBv7:       @ %bb.0:
+; CHECK-THUMBv7-NEXT:    ldr r0, [r0]
+; CHECK-THUMBv7-NEXT:    rev r0, r0
+; CHECK-THUMBv7-NEXT:    bx lr
+
   %tmp = bitcast i32* %arg to i8*
   %tmp1 = load i8, i8* %tmp, align 4
   %tmp2 = zext i8 %tmp1 to i32
@@ -115,14 +174,29 @@ define i32 @load_i32_by_i8_bswap(i32* %arg) {
 ; (i64) p[0] | ((i64) p[1] << 8) | ((i64) p[2] << 16) | ((i64) p[3] << 24) | ((i64) p[4] << 32) | ((i64) p[5] << 40) | ((i64) p[6] << 48) | ((i64) p[7] << 56)
 define i64 @load_i64_by_i8(i64* %arg) {
 ; CHECK-LABEL: load_i64_by_i8:
-; CHECK: ldr r2, [r0]
-; CHECK-NEXT: ldr r1, [r0, #4]
-; CHECK-NEXT: mov r0, r2
-; CHECK-NEXT: mov pc, lr
-
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    ldr r2, [r0]
+; CHECK-NEXT:    ldr r1, [r0, #4]
+; CHECK-NEXT:    mov r0, r2
+; CHECK-NEXT:    mov pc, lr
+;
 ; CHECK-ARMv6-LABEL: load_i64_by_i8:
-; CHECK-ARMv6: ldrd  r0, r1, [r0]
-; CHECK-ARMv6: bx  lr
+; CHECK-ARMv6:       @ %bb.0:
+; CHECK-ARMv6-NEXT:    ldrd r0, r1, [r0]
+; CHECK-ARMv6-NEXT:    bx lr
+;
+; CHECK-THUMBv6-LABEL: load_i64_by_i8:
+; CHECK-THUMBv6:       @ %bb.0:
+; CHECK-THUMBv6-NEXT:    ldr r2, [r0]
+; CHECK-THUMBv6-NEXT:    ldr r1, [r0, #4]
+; CHECK-THUMBv6-NEXT:    mov r0, r2
+; CHECK-THUMBv6-NEXT:    bx lr
+;
+; CHECK-THUMBv7-LABEL: load_i64_by_i8:
+; CHECK-THUMBv7:       @ %bb.0:
+; CHECK-THUMBv7-NEXT:    ldrd r0, r1, [r0]
+; CHECK-THUMBv7-NEXT:    bx lr
+
   %tmp = bitcast i64* %arg to i8*
   %tmp1 = load i8, i8* %tmp, align 8
   %tmp2 = zext i8 %tmp1 to i64
@@ -168,25 +242,47 @@ define i64 @load_i64_by_i8(i64* %arg) {
 ; ((i64) p[0] << 56) | ((i64) p[1] << 48) | ((i64) p[2] << 40) | ((i64) p[3] << 32) | ((i64) p[4] << 24) | ((i64) p[5] << 16) | ((i64) p[6] << 8) | (i64) p[7]
 define i64 @load_i64_by_i8_bswap(i64* %arg) {
 ; CHECK-LABEL: load_i64_by_i8_bswap:
-; CHECK: ldr{{.*}}r0
-; CHECK: ldr{{.*}}r0
-; CHECK: and
-; CHECK-NEXT: and
-; CHECK-NEXT: orr
-; CHECK-NEXT: orr
-; CHECK-NEXT: and
-; CHECK-NEXT: orr
-; CHECK-NEXT: and
-; CHECK-NEXT: orr
-; CHECK-NEXT: orr
-; CHECK-NEXT: orr
-; CHECK: mov pc, lr
-
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    push {r11, lr}
+; CHECK-NEXT:    ldr r1, [r0]
+; CHECK-NEXT:    mov r12, #65280
+; CHECK-NEXT:    ldr r0, [r0, #4]
+; CHECK-NEXT:    mov lr, #16711680
+; CHECK-NEXT:    and r3, r12, r0, lsr #8
+; CHECK-NEXT:    and r2, lr, r0, lsl #8
+; CHECK-NEXT:    orr r3, r3, r0, lsr #24
+; CHECK-NEXT:    orr r0, r2, r0, lsl #24
+; CHECK-NEXT:    and r2, r12, r1, lsr #8
+; CHECK-NEXT:    orr r0, r0, r3
+; CHECK-NEXT:    and r3, lr, r1, lsl #8
+; CHECK-NEXT:    orr r2, r2, r1, lsr #24
+; CHECK-NEXT:    orr r1, r3, r1, lsl #24
+; CHECK-NEXT:    orr r1, r1, r2
+; CHECK-NEXT:    pop {r11, lr}
+; CHECK-NEXT:    mov pc, lr
+;
 ; CHECK-ARMv6-LABEL: load_i64_by_i8_bswap:
-; CHECK-ARMv6: ldrd  r2, r3, [r0]
-; CHECK-ARMv6: rev r0, r3
-; CHECK-ARMv6: rev r1, r2
-; CHECK-ARMv6: bx  lr
+; CHECK-ARMv6:       @ %bb.0:
+; CHECK-ARMv6-NEXT:    ldrd r2, r3, [r0]
+; CHECK-ARMv6-NEXT:    rev r0, r3
+; CHECK-ARMv6-NEXT:    rev r1, r2
+; CHECK-ARMv6-NEXT:    bx lr
+;
+; CHECK-THUMBv6-LABEL: load_i64_by_i8_bswap:
+; CHECK-THUMBv6:       @ %bb.0:
+; CHECK-THUMBv6-NEXT:    ldr r1, [r0]
+; CHECK-THUMBv6-NEXT:    ldr r0, [r0, #4]
+; CHECK-THUMBv6-NEXT:    rev r0, r0
+; CHECK-THUMBv6-NEXT:    rev r1, r1
+; CHECK-THUMBv6-NEXT:    bx lr
+;
+; CHECK-THUMBv7-LABEL: load_i64_by_i8_bswap:
+; CHECK-THUMBv7:       @ %bb.0:
+; CHECK-THUMBv7-NEXT:    ldrd r1, r0, [r0]
+; CHECK-THUMBv7-NEXT:    rev r0, r0
+; CHECK-THUMBv7-NEXT:    rev r1, r1
+; CHECK-THUMBv7-NEXT:    bx lr
+
   %tmp = bitcast i64* %arg to i8*
   %tmp1 = load i8, i8* %tmp, align 8
   %tmp2 = zext i8 %tmp1 to i64
@@ -232,12 +328,26 @@ define i64 @load_i64_by_i8_bswap(i64* %arg) {
 ; (i32) p[1] | ((i32) p[2] << 8) | ((i32) p[3] << 16) | ((i32) p[4] << 24)
 define i32 @load_i32_by_i8_nonzero_offset(i32* %arg) {
 ; CHECK-LABEL: load_i32_by_i8_nonzero_offset:
-; CHECK: ldr r0, [r0, #1]
-; CHECK-NEXT: mov pc, lr
-
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    ldr r0, [r0, #1]
+; CHECK-NEXT:    mov pc, lr
+;
 ; CHECK-ARMv6-LABEL: load_i32_by_i8_nonzero_offset:
-; CHECK-ARMv6: ldr r0, [r0, #1]
-; CHECK-ARMv6-NEXT: bx  lr
+; CHECK-ARMv6:       @ %bb.0:
+; CHECK-ARMv6-NEXT:    ldr r0, [r0, #1]
+; CHECK-ARMv6-NEXT:    bx lr
+;
+; CHECK-THUMBv6-LABEL: load_i32_by_i8_nonzero_offset:
+; CHECK-THUMBv6:       @ %bb.0:
+; CHECK-THUMBv6-NEXT:    movs r1, #1
+; CHECK-THUMBv6-NEXT:    ldr r0, [r0, r1]
+; CHECK-THUMBv6-NEXT:    bx lr
+;
+; CHECK-THUMBv7-LABEL: load_i32_by_i8_nonzero_offset:
+; CHECK-THUMBv7:       @ %bb.0:
+; CHECK-THUMBv7-NEXT:    ldr.w r0, [r0, #1]
+; CHECK-THUMBv7-NEXT:    bx lr
+
 
   %tmp = bitcast i32* %arg to i8*
   %tmp1 = getelementptr inbounds i8, i8* %tmp, i32 1
@@ -265,12 +375,26 @@ define i32 @load_i32_by_i8_nonzero_offset(i32* %arg) {
 ; (i32) p[-4] | ((i32) p[-3] << 8) | ((i32) p[-2] << 16) | ((i32) p[-1] << 24)
 define i32 @load_i32_by_i8_neg_offset(i32* %arg) {
 ; CHECK-LABEL: load_i32_by_i8_neg_offset:
-; CHECK: ldr r0, [r0, #-4]
-; CHECK-NEXT: mov pc, lr
-
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    ldr r0, [r0, #-4]
+; CHECK-NEXT:    mov pc, lr
+;
 ; CHECK-ARMv6-LABEL: load_i32_by_i8_neg_offset:
-; CHECK-ARMv6: ldr r0, [r0, #-4]
-; CHECK-ARMv6-NEXT: bx  lr
+; CHECK-ARMv6:       @ %bb.0:
+; CHECK-ARMv6-NEXT:    ldr r0, [r0, #-4]
+; CHECK-ARMv6-NEXT:    bx lr
+;
+; CHECK-THUMBv6-LABEL: load_i32_by_i8_neg_offset:
+; CHECK-THUMBv6:       @ %bb.0:
+; CHECK-THUMBv6-NEXT:    subs r0, r0, #4
+; CHECK-THUMBv6-NEXT:    ldr r0, [r0]
+; CHECK-THUMBv6-NEXT:    bx lr
+;
+; CHECK-THUMBv7-LABEL: load_i32_by_i8_neg_offset:
+; CHECK-THUMBv7:       @ %bb.0:
+; CHECK-THUMBv7-NEXT:    ldr r0, [r0, #-4]
+; CHECK-THUMBv7-NEXT:    bx lr
+
 
   %tmp = bitcast i32* %arg to i8*
   %tmp1 = getelementptr inbounds i8, i8* %tmp, i32 -4
@@ -298,20 +422,36 @@ define i32 @load_i32_by_i8_neg_offset(i32* %arg) {
 ; (i32) p[4] | ((i32) p[3] << 8) | ((i32) p[2] << 16) | ((i32) p[1] << 24)
 define i32 @load_i32_by_i8_nonzero_offset_bswap(i32* %arg) {
 ; CHECK-LABEL: load_i32_by_i8_nonzero_offset_bswap:
-; CHECK: ldr r0, [r0, #1]
-; CHECK-NEXT: mov r1, #65280
-; CHECK-NEXT: mov r2, #16711680
-; CHECK-NEXT: and r1, r1, r0, lsr #8
-; CHECK-NEXT: and r2, r2, r0, lsl #8
-; CHECK-NEXT: orr r1, r1, r0, lsr #24
-; CHECK-NEXT: orr r0, r2, r0, lsl #24
-; CHECK-NEXT: orr r0, r0, r1
-; CHECK-NEXT: mov pc, lr
-
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    ldr r0, [r0, #1]
+; CHECK-NEXT:    mov r1, #65280
+; CHECK-NEXT:    mov r2, #16711680
+; CHECK-NEXT:    and r1, r1, r0, lsr #8
+; CHECK-NEXT:    and r2, r2, r0, lsl #8
+; CHECK-NEXT:    orr r1, r1, r0, lsr #24
+; CHECK-NEXT:    orr r0, r2, r0, lsl #24
+; CHECK-NEXT:    orr r0, r0, r1
+; CHECK-NEXT:    mov pc, lr
+;
 ; CHECK-ARMv6-LABEL: load_i32_by_i8_nonzero_offset_bswap:
-; CHECK-ARMv6: ldr r0, [r0, #1]
-; CHECK-ARMv6-NEXT: rev r0, r0
-; CHECK-ARMv6-NEXT: bx  lr
+; CHECK-ARMv6:       @ %bb.0:
+; CHECK-ARMv6-NEXT:    ldr r0, [r0, #1]
+; CHECK-ARMv6-NEXT:    rev r0, r0
+; CHECK-ARMv6-NEXT:    bx lr
+;
+; CHECK-THUMBv6-LABEL: load_i32_by_i8_nonzero_offset_bswap:
+; CHECK-THUMBv6:       @ %bb.0:
+; CHECK-THUMBv6-NEXT:    movs r1, #1
+; CHECK-THUMBv6-NEXT:    ldr r0, [r0, r1]
+; CHECK-THUMBv6-NEXT:    rev r0, r0
+; CHECK-THUMBv6-NEXT:    bx lr
+;
+; CHECK-THUMBv7-LABEL: load_i32_by_i8_nonzero_offset_bswap:
+; CHECK-THUMBv7:       @ %bb.0:
+; CHECK-THUMBv7-NEXT:    ldr.w r0, [r0, #1]
+; CHECK-THUMBv7-NEXT:    rev r0, r0
+; CHECK-THUMBv7-NEXT:    bx lr
+
 
   %tmp = bitcast i32* %arg to i8*
   %tmp1 = getelementptr inbounds i8, i8* %tmp, i32 4
@@ -339,20 +479,36 @@ define i32 @load_i32_by_i8_nonzero_offset_bswap(i32* %arg) {
 ; (i32) p[-1] | ((i32) p[-2] << 8) | ((i32) p[-3] << 16) | ((i32) p[-4] << 24)
 define i32 @load_i32_by_i8_neg_offset_bswap(i32* %arg) {
 ; CHECK-LABEL: load_i32_by_i8_neg_offset_bswap:
-; CHECK: ldr r0, [r0, #-4]
-; CHECK-NEXT: mov r1, #65280
-; CHECK-NEXT: mov r2, #16711680
-; CHECK-NEXT: and r1, r1, r0, lsr #8
-; CHECK-NEXT: and r2, r2, r0, lsl #8
-; CHECK-NEXT: orr r1, r1, r0, lsr #24
-; CHECK-NEXT: orr r0, r2, r0, lsl #24
-; CHECK-NEXT: orr r0, r0, r1
-; CHECK-NEXT: mov pc, lr
-
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    ldr r0, [r0, #-4]
+; CHECK-NEXT:    mov r1, #65280
+; CHECK-NEXT:    mov r2, #16711680
+; CHECK-NEXT:    and r1, r1, r0, lsr #8
+; CHECK-NEXT:    and r2, r2, r0, lsl #8
+; CHECK-NEXT:    orr r1, r1, r0, lsr #24
+; CHECK-NEXT:    orr r0, r2, r0, lsl #24
+; CHECK-NEXT:    orr r0, r0, r1
+; CHECK-NEXT:    mov pc, lr
+;
 ; CHECK-ARMv6-LABEL: load_i32_by_i8_neg_offset_bswap:
-; CHECK-ARMv6: ldr r0, [r0, #-4]
-; CHECK-ARMv6-NEXT: rev r0, r0
-; CHECK-ARMv6-NEXT: bx  lr
+; CHECK-ARMv6:       @ %bb.0:
+; CHECK-ARMv6-NEXT:    ldr r0, [r0, #-4]
+; CHECK-ARMv6-NEXT:    rev r0, r0
+; CHECK-ARMv6-NEXT:    bx lr
+;
+; CHECK-THUMBv6-LABEL: load_i32_by_i8_neg_offset_bswap:
+; CHECK-THUMBv6:       @ %bb.0:
+; CHECK-THUMBv6-NEXT:    subs r0, r0, #4
+; CHECK-THUMBv6-NEXT:    ldr r0, [r0]
+; CHECK-THUMBv6-NEXT:    rev r0, r0
+; CHECK-THUMBv6-NEXT:    bx lr
+;
+; CHECK-THUMBv7-LABEL: load_i32_by_i8_neg_offset_bswap:
+; CHECK-THUMBv7:       @ %bb.0:
+; CHECK-THUMBv7-NEXT:    ldr r0, [r0, #-4]
+; CHECK-THUMBv7-NEXT:    rev r0, r0
+; CHECK-THUMBv7-NEXT:    bx lr
+
 
   %tmp = bitcast i32* %arg to i8*
   %tmp1 = getelementptr inbounds i8, i8* %tmp, i32 -1
@@ -382,20 +538,35 @@ declare i16 @llvm.bswap.i16(i16)
 ; (i32) bswap(p[1]) | (i32) bswap(p[0] << 16)
 define i32 @load_i32_by_bswap_i16(i32* %arg) {
 ; CHECK-LABEL: load_i32_by_bswap_i16:
-; CHECK: ldr  r0, [r0]
-; CHECK-NEXT: mov r1, #65280
-; CHECK-NEXT: mov r2, #16711680
-; CHECK-NEXT: and r1, r1, r0, lsr #8
-; CHECK-NEXT: and r2, r2, r0, lsl #8
-; CHECK-NEXT: orr r1, r1, r0, lsr #24
-; CHECK-NEXT: orr r0, r2, r0, lsl #24
-; CHECK-NEXT: orr r0, r0, r1
-; CHECK-NEXT: mov pc, lr
-
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    ldr r0, [r0]
+; CHECK-NEXT:    mov r1, #65280
+; CHECK-NEXT:    mov r2, #16711680
+; CHECK-NEXT:    and r1, r1, r0, lsr #8
+; CHECK-NEXT:    and r2, r2, r0, lsl #8
+; CHECK-NEXT:    orr r1, r1, r0, lsr #24
+; CHECK-NEXT:    orr r0, r2, r0, lsl #24
+; CHECK-NEXT:    orr r0, r0, r1
+; CHECK-NEXT:    mov pc, lr
+;
 ; CHECK-ARMv6-LABEL: load_i32_by_bswap_i16:
-; CHECK-ARMv6: ldr r0, [r0]
-; CHECK-ARMv6-NEXT: rev r0, r0
-; CHECK-ARMv6-NEXT: bx  lr
+; CHECK-ARMv6:       @ %bb.0:
+; CHECK-ARMv6-NEXT:    ldr r0, [r0]
+; CHECK-ARMv6-NEXT:    rev r0, r0
+; CHECK-ARMv6-NEXT:    bx lr
+;
+; CHECK-THUMBv6-LABEL: load_i32_by_bswap_i16:
+; CHECK-THUMBv6:       @ %bb.0:
+; CHECK-THUMBv6-NEXT:    ldr r0, [r0]
+; CHECK-THUMBv6-NEXT:    rev r0, r0
+; CHECK-THUMBv6-NEXT:    bx lr
+;
+; CHECK-THUMBv7-LABEL: load_i32_by_bswap_i16:
+; CHECK-THUMBv7:       @ %bb.0:
+; CHECK-THUMBv7-NEXT:    ldr r0, [r0]
+; CHECK-THUMBv7-NEXT:    rev r0, r0
+; CHECK-THUMBv7-NEXT:    bx lr
+
 
   %tmp = bitcast i32* %arg to i16*
   %tmp1 = load i16, i16* %tmp, align 4
@@ -414,12 +585,24 @@ define i32 @load_i32_by_bswap_i16(i32* %arg) {
 ; (i32) p[0] | (sext(p[1] << 16) to i32)
 define i32 @load_i32_by_sext_i16(i32* %arg) {
 ; CHECK-LABEL: load_i32_by_sext_i16:
-; CHECK: ldr  r0, [r0]
-; CHECK-NEXT: mov pc, lr
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    ldr r0, [r0]
+; CHECK-NEXT:    mov pc, lr
 ;
 ; CHECK-ARMv6-LABEL: load_i32_by_sext_i16:
-; CHECK-ARMv6: ldr  r0, [r0]
-; CHECK-ARMv6-NEXT: bx lr
+; CHECK-ARMv6:       @ %bb.0:
+; CHECK-ARMv6-NEXT:    ldr r0, [r0]
+; CHECK-ARMv6-NEXT:    bx lr
+;
+; CHECK-THUMBv6-LABEL: load_i32_by_sext_i16:
+; CHECK-THUMBv6:       @ %bb.0:
+; CHECK-THUMBv6-NEXT:    ldr r0, [r0]
+; CHECK-THUMBv6-NEXT:    bx lr
+;
+; CHECK-THUMBv7-LABEL: load_i32_by_sext_i16:
+; CHECK-THUMBv7:       @ %bb.0:
+; CHECK-THUMBv7-NEXT:    ldr r0, [r0]
+; CHECK-THUMBv7-NEXT:    bx lr
   %tmp = bitcast i32* %arg to i16*
   %tmp1 = load i16, i16* %tmp, align 4
   %tmp2 = zext i16 %tmp1 to i32
@@ -436,14 +619,28 @@ define i32 @load_i32_by_sext_i16(i32* %arg) {
 ; (i32) p[i] | ((i32) p[i + 1] << 8) | ((i32) p[i + 2] << 16) | ((i32) p[i + 3] << 24)
 define i32 @load_i32_by_i8_base_offset_index(i8* %arg, i32 %i) {
 ; CHECK-LABEL: load_i32_by_i8_base_offset_index:
-; CHECK: add r0, r0, r1
-; CHECK-NEXT: ldr r0, [r0, #12]
-; CHECK-NEXT: mov pc, lr
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    add r0, r0, r1
+; CHECK-NEXT:    ldr r0, [r0, #12]
+; CHECK-NEXT:    mov pc, lr
 ;
 ; CHECK-ARMv6-LABEL: load_i32_by_i8_base_offset_index:
-; CHECK-ARMv6: add r0, r0, r1
-; CHECK-ARMv6-NEXT: ldr r0, [r0, #12]
-; CHECK-ARMv6-NEXT: bx  lr
+; CHECK-ARMv6:       @ %bb.0:
+; CHECK-ARMv6-NEXT:    add r0, r0, r1
+; CHECK-ARMv6-NEXT:    ldr r0, [r0, #12]
+; CHECK-ARMv6-NEXT:    bx lr
+;
+; CHECK-THUMBv6-LABEL: load_i32_by_i8_base_offset_index:
+; CHECK-THUMBv6:       @ %bb.0:
+; CHECK-THUMBv6-NEXT:    adds r0, r0, r1
+; CHECK-THUMBv6-NEXT:    ldr r0, [r0, #12]
+; CHECK-THUMBv6-NEXT:    bx lr
+;
+; CHECK-THUMBv7-LABEL: load_i32_by_i8_base_offset_index:
+; CHECK-THUMBv7:       @ %bb.0:
+; CHECK-THUMBv7-NEXT:    add r0, r1
+; CHECK-THUMBv7-NEXT:    ldr r0, [r0, #12]
+; CHECK-THUMBv7-NEXT:    bx lr
 
   %tmp = add nuw nsw i32 %i, 3
   %tmp2 = add nuw nsw i32 %i, 2
@@ -479,14 +676,29 @@ define i32 @load_i32_by_i8_base_offset_index(i8* %arg, i32 %i) {
 ; (i32) p[i + 1] | ((i32) p[i + 2] << 8) | ((i32) p[i + 3] << 16) | ((i32) p[i + 4] << 24)
 define i32 @load_i32_by_i8_base_offset_index_2(i8* %arg, i32 %i) {
 ; CHECK-LABEL: load_i32_by_i8_base_offset_index_2:
-; CHECK: add r0, r1, r0
-; CHECK-NEXT: ldr r0, [r0, #13]
-; CHECK-NEXT: mov pc, lr
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    add r0, r1, r0
+; CHECK-NEXT:    ldr r0, [r0, #13]
+; CHECK-NEXT:    mov pc, lr
 ;
 ; CHECK-ARMv6-LABEL: load_i32_by_i8_base_offset_index_2:
-; CHECK-ARMv6: add r0, r1, r0
-; CHECK-ARMv6-NEXT: ldr r0, [r0, #13]
-; CHECK-ARMv6-NEXT: bx  lr
+; CHECK-ARMv6:       @ %bb.0:
+; CHECK-ARMv6-NEXT:    add r0, r1, r0
+; CHECK-ARMv6-NEXT:    ldr r0, [r0, #13]
+; CHECK-ARMv6-NEXT:    bx lr
+;
+; CHECK-THUMBv6-LABEL: load_i32_by_i8_base_offset_index_2:
+; CHECK-THUMBv6:       @ %bb.0:
+; CHECK-THUMBv6-NEXT:    adds r0, r1, r0
+; CHECK-THUMBv6-NEXT:    movs r1, #13
+; CHECK-THUMBv6-NEXT:    ldr r0, [r0, r1]
+; CHECK-THUMBv6-NEXT:    bx lr
+;
+; CHECK-THUMBv7-LABEL: load_i32_by_i8_base_offset_index_2:
+; CHECK-THUMBv7:       @ %bb.0:
+; CHECK-THUMBv7-NEXT:    add r0, r1
+; CHECK-THUMBv7-NEXT:    ldr.w r0, [r0, #13]
+; CHECK-THUMBv7-NEXT:    bx lr
   %tmp = add nuw nsw i32 %i, 4
   %tmp2 = add nuw nsw i32 %i, 3
   %tmp3 = add nuw nsw i32 %i, 2
@@ -521,16 +733,24 @@ define i32 @load_i32_by_i8_base_offset_index_2(i8* %arg, i32 %i) {
 ; (i32) p[0] | ((i32) p[1] << 8)
 define i32 @zext_load_i32_by_i8(i32* %arg) {
 ; CHECK-LABEL: zext_load_i32_by_i8:
-; CHECK: ldrb  r1, [r0]
-; CHECK-NEXT: ldrb  r0, [r0, #1]
-; CHECK-NEXT: orr r0, r1, r0, lsl #8
-; CHECK-NEXT: mov pc, lr
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    ldrh r0, [r0]
+; CHECK-NEXT:    mov pc, lr
 ;
 ; CHECK-ARMv6-LABEL: zext_load_i32_by_i8:
-; CHECK-ARMv6: ldrb  r1, [r0]
-; CHECK-ARMv6-NEXT: ldrb  r0, [r0, #1]
-; CHECK-ARMv6-NEXT: orr r0, r1, r0, lsl #8
-; CHECK-ARMv6-NEXT: bx  lr
+; CHECK-ARMv6:       @ %bb.0:
+; CHECK-ARMv6-NEXT:    ldrh r0, [r0]
+; CHECK-ARMv6-NEXT:    bx lr
+;
+; CHECK-THUMBv6-LABEL: zext_load_i32_by_i8:
+; CHECK-THUMBv6:       @ %bb.0:
+; CHECK-THUMBv6-NEXT:    ldrh r0, [r0]
+; CHECK-THUMBv6-NEXT:    bx lr
+;
+; CHECK-THUMBv7-LABEL: zext_load_i32_by_i8:
+; CHECK-THUMBv7:       @ %bb.0:
+; CHECK-THUMBv7-NEXT:    ldrh r0, [r0]
+; CHECK-THUMBv7-NEXT:    bx lr
 
   %tmp = bitcast i32* %arg to i8*
   %tmp1 = getelementptr inbounds i8, i8* %tmp, i32 0
@@ -548,18 +768,37 @@ define i32 @zext_load_i32_by_i8(i32* %arg) {
 ; ((i32) p[0] << 8) | ((i32) p[1] << 16)
 define i32 @zext_load_i32_by_i8_shl_8(i32* %arg) {
 ; CHECK-LABEL: zext_load_i32_by_i8_shl_8:
-; CHECK: ldrb  r1, [r0]
-; CHECK-NEXT: ldrb  r0, [r0, #1]
-; CHECK-NEXT: lsl r0, r0, #16
-; CHECK-NEXT: orr r0, r0, r1, lsl #8
-; CHECK-NEXT: mov pc, lr
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    ldrb r1, [r0]
+; CHECK-NEXT:    ldrb r0, [r0, #1]
+; CHECK-NEXT:    lsl r0, r0, #16
+; CHECK-NEXT:    orr r0, r0, r1, lsl #8
+; CHECK-NEXT:    mov pc, lr
 ;
 ; CHECK-ARMv6-LABEL: zext_load_i32_by_i8_shl_8:
-; CHECK-ARMv6: ldrb  r1, [r0]
-; CHECK-ARMv6-NEXT: ldrb  r0, [r0, #1]
-; CHECK-ARMv6-NEXT: lsl r0, r0, #16
-; CHECK-ARMv6-NEXT: orr r0, r0, r1, lsl #8
-; CHECK-ARMv6-NEXT: bx  lr
+; CHECK-ARMv6:       @ %bb.0:
+; CHECK-ARMv6-NEXT:    ldrb r1, [r0]
+; CHECK-ARMv6-NEXT:    ldrb r0, [r0, #1]
+; CHECK-ARMv6-NEXT:    lsl r0, r0, #16
+; CHECK-ARMv6-NEXT:    orr r0, r0, r1, lsl #8
+; CHECK-ARMv6-NEXT:    bx lr
+;
+; CHECK-THUMBv6-LABEL: zext_load_i32_by_i8_shl_8:
+; CHECK-THUMBv6:       @ %bb.0:
+; CHECK-THUMBv6-NEXT:    ldrb r1, [r0]
+; CHECK-THUMBv6-NEXT:    lsls r1, r1, #8
+; CHECK-THUMBv6-NEXT:    ldrb r0, [r0, #1]
+; CHECK-THUMBv6-NEXT:    lsls r0, r0, #16
+; CHECK-THUMBv6-NEXT:    adds r0, r0, r1
+; CHECK-THUMBv6-NEXT:    bx lr
+;
+; CHECK-THUMBv7-LABEL: zext_load_i32_by_i8_shl_8:
+; CHECK-THUMBv7:       @ %bb.0:
+; CHECK-THUMBv7-NEXT:    ldrb r1, [r0]
+; CHECK-THUMBv7-NEXT:    ldrb r0, [r0, #1]
+; CHECK-THUMBv7-NEXT:    lsls r0, r0, #16
+; CHECK-THUMBv7-NEXT:    orr.w r0, r0, r1, lsl #8
+; CHECK-THUMBv7-NEXT:    bx lr
 
   %tmp = bitcast i32* %arg to i8*
   %tmp1 = getelementptr inbounds i8, i8* %tmp, i32 0
@@ -578,18 +817,37 @@ define i32 @zext_load_i32_by_i8_shl_8(i32* %arg) {
 ; ((i32) p[0] << 16) | ((i32) p[1] << 24)
 define i32 @zext_load_i32_by_i8_shl_16(i32* %arg) {
 ; CHECK-LABEL: zext_load_i32_by_i8_shl_16:
-; CHECK: ldrb  r1, [r0]
-; CHECK-NEXT: ldrb  r0, [r0, #1]
-; CHECK-NEXT: lsl r0, r0, #24
-; CHECK-NEXT: orr r0, r0, r1, lsl #16
-; CHECK-NEXT: mov pc, lr
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    ldrb r1, [r0]
+; CHECK-NEXT:    ldrb r0, [r0, #1]
+; CHECK-NEXT:    lsl r0, r0, #24
+; CHECK-NEXT:    orr r0, r0, r1, lsl #16
+; CHECK-NEXT:    mov pc, lr
 ;
 ; CHECK-ARMv6-LABEL: zext_load_i32_by_i8_shl_16:
-; CHECK-ARMv6: ldrb  r1, [r0]
-; CHECK-ARMv6-NEXT: ldrb  r0, [r0, #1]
-; CHECK-ARMv6-NEXT: lsl r0, r0, #24
-; CHECK-ARMv6-NEXT: orr r0, r0, r1, lsl #16
-; CHECK-ARMv6-NEXT: bx  lr
+; CHECK-ARMv6:       @ %bb.0:
+; CHECK-ARMv6-NEXT:    ldrb r1, [r0]
+; CHECK-ARMv6-NEXT:    ldrb r0, [r0, #1]
+; CHECK-ARMv6-NEXT:    lsl r0, r0, #24
+; CHECK-ARMv6-NEXT:    orr r0, r0, r1, lsl #16
+; CHECK-ARMv6-NEXT:    bx lr
+;
+; CHECK-THUMBv6-LABEL: zext_load_i32_by_i8_shl_16:
+; CHECK-THUMBv6:       @ %bb.0:
+; CHECK-THUMBv6-NEXT:    ldrb r1, [r0]
+; CHECK-THUMBv6-NEXT:    lsls r1, r1, #16
+; CHECK-THUMBv6-NEXT:    ldrb r0, [r0, #1]
+; CHECK-THUMBv6-NEXT:    lsls r0, r0, #24
+; CHECK-THUMBv6-NEXT:    adds r0, r0, r1
+; CHECK-THUMBv6-NEXT:    bx lr
+;
+; CHECK-THUMBv7-LABEL: zext_load_i32_by_i8_shl_16:
+; CHECK-THUMBv7:       @ %bb.0:
+; CHECK-THUMBv7-NEXT:    ldrb r1, [r0]
+; CHECK-THUMBv7-NEXT:    ldrb r0, [r0, #1]
+; CHECK-THUMBv7-NEXT:    lsls r0, r0, #24
+; CHECK-THUMBv7-NEXT:    orr.w r0, r0, r1, lsl #16
+; CHECK-THUMBv7-NEXT:    bx lr
 
   %tmp = bitcast i32* %arg to i8*
   %tmp1 = getelementptr inbounds i8, i8* %tmp, i32 0
@@ -608,16 +866,32 @@ define i32 @zext_load_i32_by_i8_shl_16(i32* %arg) {
 ; (i32) p[1] | ((i32) p[0] << 8)
 define i32 @zext_load_i32_by_i8_bswap(i32* %arg) {
 ; CHECK-LABEL: zext_load_i32_by_i8_bswap:
-; CHECK: ldrb  r1, [r0]
-; CHECK-NEXT: ldrb  r0, [r0, #1]
-; CHECK-NEXT: orr r0, r0, r1, lsl #8
-; CHECK-NEXT: mov pc, lr
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    ldrb r1, [r0]
+; CHECK-NEXT:    ldrb r0, [r0, #1]
+; CHECK-NEXT:    orr r0, r0, r1, lsl #8
+; CHECK-NEXT:    mov pc, lr
 ;
 ; CHECK-ARMv6-LABEL: zext_load_i32_by_i8_bswap:
-; CHECK-ARMv6: ldrb  r1, [r0]
-; CHECK-ARMv6-NEXT: ldrb  r0, [r0, #1]
-; CHECK-ARMv6-NEXT: orr r0, r0, r1, lsl #8
-; CHECK-ARMv6-NEXT: bx  lr
+; CHECK-ARMv6:       @ %bb.0:
+; CHECK-ARMv6-NEXT:    ldrh r0, [r0]
+; CHECK-ARMv6-NEXT:    lsl r0, r0, #16
+; CHECK-ARMv6-NEXT:    rev r0, r0
+; CHECK-ARMv6-NEXT:    bx lr
+;
+; CHECK-THUMBv6-LABEL: zext_load_i32_by_i8_bswap:
+; CHECK-THUMBv6:       @ %bb.0:
+; CHECK-THUMBv6-NEXT:    ldrh r0, [r0]
+; CHECK-THUMBv6-NEXT:    lsls r0, r0, #16
+; CHECK-THUMBv6-NEXT:    rev r0, r0
+; CHECK-THUMBv6-NEXT:    bx lr
+;
+; CHECK-THUMBv7-LABEL: zext_load_i32_by_i8_bswap:
+; CHECK-THUMBv7:       @ %bb.0:
+; CHECK-THUMBv7-NEXT:    ldrh r0, [r0]
+; CHECK-THUMBv7-NEXT:    lsls r0, r0, #16
+; CHECK-THUMBv7-NEXT:    rev r0, r0
+; CHECK-THUMBv7-NEXT:    bx lr
 
   %tmp = bitcast i32* %arg to i8*
   %tmp1 = getelementptr inbounds i8, i8* %tmp, i32 1
@@ -635,18 +909,37 @@ define i32 @zext_load_i32_by_i8_bswap(i32* %arg) {
 ; ((i32) p[1] << 8) | ((i32) p[0] << 16)
 define i32 @zext_load_i32_by_i8_bswap_shl_8(i32* %arg) {
 ; CHECK-LABEL: zext_load_i32_by_i8_bswap_shl_8:
-; CHECK: ldrb  r1, [r0]
-; CHECK-NEXT: ldrb  r0, [r0, #1]
-; CHECK-NEXT: lsl r1, r1, #16
-; CHECK-NEXT: orr r0, r1, r0, lsl #8
-; CHECK-NEXT: mov pc, lr
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    ldrb r1, [r0]
+; CHECK-NEXT:    ldrb r0, [r0, #1]
+; CHECK-NEXT:    lsl r1, r1, #16
+; CHECK-NEXT:    orr r0, r1, r0, lsl #8
+; CHECK-NEXT:    mov pc, lr
 ;
 ; CHECK-ARMv6-LABEL: zext_load_i32_by_i8_bswap_shl_8:
-; CHECK-ARMv6: ldrb  r1, [r0]
-; CHECK-ARMv6-NEXT: ldrb  r0, [r0, #1]
-; CHECK-ARMv6-NEXT: lsl r1, r1, #16
-; CHECK-ARMv6-NEXT: orr r0, r1, r0, lsl #8
-; CHECK-ARMv6-NEXT: bx  lr
+; CHECK-ARMv6:       @ %bb.0:
+; CHECK-ARMv6-NEXT:    ldrb r1, [r0]
+; CHECK-ARMv6-NEXT:    ldrb r0, [r0, #1]
+; CHECK-ARMv6-NEXT:    lsl r1, r1, #16
+; CHECK-ARMv6-NEXT:    orr r0, r1, r0, lsl #8
+; CHECK-ARMv6-NEXT:    bx lr
+;
+; CHECK-THUMBv6-LABEL: zext_load_i32_by_i8_bswap_shl_8:
+; CHECK-THUMBv6:       @ %bb.0:
+; CHECK-THUMBv6-NEXT:    ldrb r1, [r0, #1]
+; CHECK-THUMBv6-NEXT:    lsls r1, r1, #8
+; CHECK-THUMBv6-NEXT:    ldrb r0, [r0]
+; CHECK-THUMBv6-NEXT:    lsls r0, r0, #16
+; CHECK-THUMBv6-NEXT:    adds r0, r0, r1
+; CHECK-THUMBv6-NEXT:    bx lr
+;
+; CHECK-THUMBv7-LABEL: zext_load_i32_by_i8_bswap_shl_8:
+; CHECK-THUMBv7:       @ %bb.0:
+; CHECK-THUMBv7-NEXT:    ldrb r1, [r0]
+; CHECK-THUMBv7-NEXT:    ldrb r0, [r0, #1]
+; CHECK-THUMBv7-NEXT:    lsls r1, r1, #16
+; CHECK-THUMBv7-NEXT:    orr.w r0, r1, r0, lsl #8
+; CHECK-THUMBv7-NEXT:    bx lr
 
   %tmp = bitcast i32* %arg to i8*
   %tmp1 = getelementptr inbounds i8, i8* %tmp, i32 1
@@ -665,18 +958,37 @@ define i32 @zext_load_i32_by_i8_bswap_shl_8(i32* %arg) {
 ; ((i32) p[1] << 16) | ((i32) p[0] << 24)
 define i32 @zext_load_i32_by_i8_bswap_shl_16(i32* %arg) {
 ; CHECK-LABEL: zext_load_i32_by_i8_bswap_shl_16:
-; CHECK: ldrb  r1, [r0]
-; CHECK-NEXT: ldrb  r0, [r0, #1]
-; CHECK-NEXT: lsl r1, r1, #24
-; CHECK-NEXT: orr r0, r1, r0, lsl #16
-; CHECK-NEXT: mov pc, lr
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    ldrb r1, [r0]
+; CHECK-NEXT:    ldrb r0, [r0, #1]
+; CHECK-NEXT:    lsl r1, r1, #24
+; CHECK-NEXT:    orr r0, r1, r0, lsl #16
+; CHECK-NEXT:    mov pc, lr
 ;
 ; CHECK-ARMv6-LABEL: zext_load_i32_by_i8_bswap_shl_16:
-; CHECK-ARMv6: ldrb  r1, [r0]
-; CHECK-ARMv6-NEXT: ldrb  r0, [r0, #1]
-; CHECK-ARMv6-NEXT: lsl r1, r1, #24
-; CHECK-ARMv6-NEXT: orr r0, r1, r0, lsl #16
-; CHECK-ARMv6-NEXT: bx  lr
+; CHECK-ARMv6:       @ %bb.0:
+; CHECK-ARMv6-NEXT:    ldrb r1, [r0]
+; CHECK-ARMv6-NEXT:    ldrb r0, [r0, #1]
+; CHECK-ARMv6-NEXT:    lsl r1, r1, #24
+; CHECK-ARMv6-NEXT:    orr r0, r1, r0, lsl #16
+; CHECK-ARMv6-NEXT:    bx lr
+;
+; CHECK-THUMBv6-LABEL: zext_load_i32_by_i8_bswap_shl_16:
+; CHECK-THUMBv6:       @ %bb.0:
+; CHECK-THUMBv6-NEXT:    ldrb r1, [r0, #1]
+; CHECK-THUMBv6-NEXT:    lsls r1, r1, #16
+; CHECK-THUMBv6-NEXT:    ldrb r0, [r0]
+; CHECK-THUMBv6-NEXT:    lsls r0, r0, #24
+; CHECK-THUMBv6-NEXT:    adds r0, r0, r1
+; CHECK-THUMBv6-NEXT:    bx lr
+;
+; CHECK-THUMBv7-LABEL: zext_load_i32_by_i8_bswap_shl_16:
+; CHECK-THUMBv7:       @ %bb.0:
+; CHECK-THUMBv7-NEXT:    ldrb r1, [r0]
+; CHECK-THUMBv7-NEXT:    ldrb r0, [r0, #1]
+; CHECK-THUMBv7-NEXT:    lsls r1, r1, #24
+; CHECK-THUMBv7-NEXT:    orr.w r0, r1, r0, lsl #16
+; CHECK-THUMBv7-NEXT:    bx lr
 
   %tmp = bitcast i32* %arg to i8*
   %tmp1 = getelementptr inbounds i8, i8* %tmp, i32 1
diff --git a/llvm/test/CodeGen/ARM/loopvectorize_pr33804.ll b/llvm/test/CodeGen/ARM/loopvectorize_pr33804.ll
index d197a3757..c318a3b18 100644
--- a/llvm/test/CodeGen/ARM/loopvectorize_pr33804.ll
+++ b/llvm/test/CodeGen/ARM/loopvectorize_pr33804.ll
@@ -110,5 +110,5 @@ for.end22.i.i:                                    ; preds = %for.body14.i.i
 
 declare i32 @__gxx_personality_v0(...)
 
-attributes #0 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="generic" "target-features"="+dsp,+neon,+vfp3,-thumb-mode" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="generic" "target-features"="+dsp,+neon,+vfp3,-thumb-mode" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
diff --git a/llvm/test/CodeGen/ARM/lsr-unfolded-offset.ll b/llvm/test/CodeGen/ARM/lsr-unfolded-offset.ll
index c4fe8dc64..a9a353cad 100644
--- a/llvm/test/CodeGen/ARM/lsr-unfolded-offset.ll
+++ b/llvm/test/CodeGen/ARM/lsr-unfolded-offset.ll
@@ -15,7 +15,7 @@ target triple = "thumbv7-apple-ios"
 
 %struct.partition_entry = type { i32, i32, i64, i64 }
 
-define i32 @partition_overlap_check(%struct.partition_entry* nocapture %part, i32 %num_entries) nounwind readonly optsize ssp "no-frame-pointer-elim"="true" {
+define i32 @partition_overlap_check(%struct.partition_entry* nocapture %part, i32 %num_entries) nounwind readonly optsize ssp "frame-pointer"="all" {
 entry:
   %cmp79 = icmp sgt i32 %num_entries, 0
   br i1 %cmp79, label %outer.loop, label %for.end72
diff --git a/llvm/test/CodeGen/ARM/memcpy-ldm-stm.ll b/llvm/test/CodeGen/ARM/memcpy-ldm-stm.ll
index 4009de5bd..0bda070f7 100644
--- a/llvm/test/CodeGen/ARM/memcpy-ldm-stm.ll
+++ b/llvm/test/CodeGen/ARM/memcpy-ldm-stm.ll
@@ -36,10 +36,10 @@ entry:
 ; CHECKV6-NEXT: ldr [[SB:r[0-7]]],
 ; CHECKV6-NEXT: ldm{{(\.w)?}} [[LB]]!,
 ; CHECKV6-NEXT: stm{{(\.w)?}} [[SB]]!,
-; CHECKV6-NEXT: ldrh{{(\.w)?}} {{.*}}, {{\[}}[[LB]]]
-; CHECKV6-NEXT: ldrb{{(\.w)?}} {{.*}}, {{\[}}[[LB]], #2]
-; CHECKV6-NEXT: strb{{(\.w)?}} {{.*}}, {{\[}}[[SB]], #2]
-; CHECKV6-NEXT: strh{{(\.w)?}} {{.*}}, {{\[}}[[SB]]]
+; CHECKV6-DAG: ldrh{{(\.w)?}} {{.*}}, {{\[}}[[LB]]]
+; CHECKV6-DAG: ldrb{{(\.w)?}} {{.*}}, {{\[}}[[LB]], #2]
+; CHECKV6-DAG: strb{{(\.w)?}} {{.*}}, {{\[}}[[SB]], #2]
+; CHECKV6-DAG: strh{{(\.w)?}} {{.*}}, {{\[}}[[SB]]]
 ; CHECKV7: movt [[LB:[rl0-9]+]], :upper16:d
 ; CHECKV7-NEXT: movt [[SB:[rl0-9]+]], :upper16:s
 ; CHECKV7: ldr{{(\.w)?}} {{.*}}, {{\[}}[[LB]], #11]
@@ -89,8 +89,8 @@ define void @test3(%struct.S* %d, %struct.S* %s) #0 {
 
 declare void @g(i32*)
 
-; Set "no-frame-pointer-elim" to increase register pressure
-attributes #0 = { "no-frame-pointer-elim"="true" }
+; Set "frame-pointer"="all" to increase register pressure
+attributes #0 = { "frame-pointer"="all" }
 
 ; Function Attrs: nounwind
 declare void @llvm.memcpy.p0i8.p0i8.i32(i8* nocapture, i8* nocapture readonly, i32, i1) #1
diff --git a/llvm/test/CodeGen/ARM/memfunc.ll b/llvm/test/CodeGen/ARM/memfunc.ll
index 6c0668a53..0fe1f630c 100644
--- a/llvm/test/CodeGen/ARM/memfunc.ll
+++ b/llvm/test/CodeGen/ARM/memfunc.ll
@@ -8,7 +8,7 @@
 ; RUN: llc < %s -mtriple=arm-none-musleabi -disable-post-ra -o - | FileCheck %s --check-prefix=CHECK-GNUEABI --check-prefix=CHECK
 ; RUN: llc < %s -mtriple=arm-none-musleabihf -disable-post-ra -o - | FileCheck %s --check-prefix=CHECK-GNUEABI --check-prefix=CHECK
 
-define void @f1(i8* %dest, i8* %src) "no-frame-pointer-elim"="true" {
+define void @f1(i8* %dest, i8* %src) "frame-pointer"="all" {
 entry:
   ; CHECK-LABEL: f1
 
@@ -98,7 +98,7 @@ entry:
 }
 
 ; Check that alloca arguments to memory intrinsics are automatically aligned if at least 8 bytes in size
-define void @f2(i8* %dest, i32 %n) "no-frame-pointer-elim"="true" {
+define void @f2(i8* %dest, i32 %n) "frame-pointer"="all" {
 entry:
   ; CHECK-LABEL: f2
 
@@ -144,7 +144,7 @@ entry:
 }
 
 ; Check that alloca arguments are not aligned if less than 8 bytes in size
-define void @f3(i8* %dest, i32 %n) "no-frame-pointer-elim"="true" {
+define void @f3(i8* %dest, i32 %n) "frame-pointer"="all" {
 entry:
   ; CHECK-LABEL: f3
 
@@ -183,7 +183,7 @@ entry:
 }
 
 ; Check that alloca arguments are not aligned if size+offset is less than 8 bytes
-define void @f4(i8* %dest, i32 %n) "no-frame-pointer-elim"="true" {
+define void @f4(i8* %dest, i32 %n) "frame-pointer"="all" {
 entry:
   ; CHECK-LABEL: f4
 
@@ -222,7 +222,7 @@ entry:
 }
 
 ; Check that alloca arguments are not aligned if the offset is not a multiple of 4
-define void @f5(i8* %dest, i32 %n) "no-frame-pointer-elim"="true" {
+define void @f5(i8* %dest, i32 %n) "frame-pointer"="all" {
 entry:
   ; CHECK-LABEL: f5
 
@@ -261,7 +261,7 @@ entry:
 }
 
 ; Check that alloca arguments are not aligned if the offset is unknown
-define void @f6(i8* %dest, i32 %n, i32 %i) "no-frame-pointer-elim"="true" {
+define void @f6(i8* %dest, i32 %n, i32 %i) "frame-pointer"="all" {
 entry:
   ; CHECK-LABEL: f6
 
@@ -300,7 +300,7 @@ entry:
 }
 
 ; Check that alloca arguments are not aligned if the GEP is not inbounds
-define void @f7(i8* %dest, i32 %n) "no-frame-pointer-elim"="true" {
+define void @f7(i8* %dest, i32 %n) "frame-pointer"="all" {
 entry:
   ; CHECK-LABEL: f7
 
@@ -339,7 +339,7 @@ entry:
 }
 
 ; Check that alloca arguments are not aligned when the offset is past the end of the allocation
-define void @f8(i8* %dest, i32 %n) "no-frame-pointer-elim"="true" {
+define void @f8(i8* %dest, i32 %n) "frame-pointer"="all" {
 entry:
   ; CHECK-LABEL: f8
 
@@ -389,7 +389,7 @@ entry:
 @arr8 = internal global [128 x i8] undef
 @arr9 = weak_odr global [128 x i8] undef
 @arr10 = dso_local global [8 x i8] c"\01\02\03\04\05\06\07\08", align 1
-define void @f9(i8* %dest, i32 %n) "no-frame-pointer-elim"="true" {
+define void @f9(i8* %dest, i32 %n) "frame-pointer"="all" {
 entry:
   call void @llvm.memcpy.p0i8.p0i8.i32(i8* %dest, i8* getelementptr inbounds ([7 x i8], [7 x i8]* @arr1, i32 0, i32 0), i32 %n, i1 false)
   call void @llvm.memcpy.p0i8.p0i8.i32(i8* %dest, i8* getelementptr inbounds ([8 x i8], [8 x i8]* @arr2, i32 0, i32 0), i32 %n, i1 false)
diff --git a/llvm/test/CodeGen/ARM/mul_const.ll b/llvm/test/CodeGen/ARM/mul_const.ll
index ada3d4e5b..adb615af2 100644
--- a/llvm/test/CodeGen/ARM/mul_const.ll
+++ b/llvm/test/CodeGen/ARM/mul_const.ll
@@ -1,73 +1,203 @@
-; RUN: llc -mtriple=arm-eabi %s -o - | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=arm-eabi %s -o - | FileCheck %s --check-prefix=ARM
+; RUN: llc -mtriple=thumbv6t2-eabi %s -o - | FileCheck %s --check-prefix=THUMB2
+; RUN: llc -mtriple=thumb-eabi %s -o - | FileCheck %s --check-prefix=THUMB
 
 define i32 @t9(i32 %v) nounwind readnone {
+; ARM-LABEL: t9:
+; ARM:       @ %bb.0: @ %entry
+; ARM-NEXT:    add r0, r0, r0, lsl #3
+; ARM-NEXT:    mov pc, lr
+;
+; THUMB2-LABEL: t9:
+; THUMB2:       @ %bb.0: @ %entry
+; THUMB2-NEXT:    add.w r0, r0, r0, lsl #3
+; THUMB2-NEXT:    bx lr
+;
+; THUMB-LABEL: t9:
+; THUMB:       @ %bb.0: @ %entry
+; THUMB-NEXT:    movs r1, #9
+; THUMB-NEXT:    muls r0, r1, r0
+; THUMB-NEXT:    bx lr
 entry:
-; CHECK-LABEL: t9:
-; CHECK: add r0, r0, r0, lsl #3
 	%0 = mul i32 %v, 9
 	ret i32 %0
 }
 
 define i32 @t7(i32 %v) nounwind readnone {
+; ARM-LABEL: t7:
+; ARM:       @ %bb.0: @ %entry
+; ARM-NEXT:    rsb r0, r0, r0, lsl #3
+; ARM-NEXT:    mov pc, lr
+;
+; THUMB2-LABEL: t7:
+; THUMB2:       @ %bb.0: @ %entry
+; THUMB2-NEXT:    rsb r0, r0, r0, lsl #3
+; THUMB2-NEXT:    bx lr
+;
+; THUMB-LABEL: t7:
+; THUMB:       @ %bb.0: @ %entry
+; THUMB-NEXT:    movs r1, #7
+; THUMB-NEXT:    muls r0, r1, r0
+; THUMB-NEXT:    bx lr
 entry:
-; CHECK-LABEL: t7:
-; CHECK: rsb r0, r0, r0, lsl #3
 	%0 = mul i32 %v, 7
 	ret i32 %0
 }
 
 define i32 @t5(i32 %v) nounwind readnone {
+; ARM-LABEL: t5:
+; ARM:       @ %bb.0: @ %entry
+; ARM-NEXT:    add r0, r0, r0, lsl #2
+; ARM-NEXT:    mov pc, lr
+;
+; THUMB2-LABEL: t5:
+; THUMB2:       @ %bb.0: @ %entry
+; THUMB2-NEXT:    add.w r0, r0, r0, lsl #2
+; THUMB2-NEXT:    bx lr
+;
+; THUMB-LABEL: t5:
+; THUMB:       @ %bb.0: @ %entry
+; THUMB-NEXT:    movs r1, #5
+; THUMB-NEXT:    muls r0, r1, r0
+; THUMB-NEXT:    bx lr
 entry:
-; CHECK-LABEL: t5:
-; CHECK: add r0, r0, r0, lsl #2
         %0 = mul i32 %v, 5
         ret i32 %0
 }
 
 define i32 @t3(i32 %v) nounwind readnone {
+; ARM-LABEL: t3:
+; ARM:       @ %bb.0: @ %entry
+; ARM-NEXT:    add r0, r0, r0, lsl #1
+; ARM-NEXT:    mov pc, lr
+;
+; THUMB2-LABEL: t3:
+; THUMB2:       @ %bb.0: @ %entry
+; THUMB2-NEXT:    add.w r0, r0, r0, lsl #1
+; THUMB2-NEXT:    bx lr
+;
+; THUMB-LABEL: t3:
+; THUMB:       @ %bb.0: @ %entry
+; THUMB-NEXT:    movs r1, #3
+; THUMB-NEXT:    muls r0, r1, r0
+; THUMB-NEXT:    bx lr
 entry:
-; CHECK-LABEL: t3:
-; CHECK: add r0, r0, r0, lsl #1
         %0 = mul i32 %v, 3
         ret i32 %0
 }
 
 define i32 @t12288(i32 %v) nounwind readnone {
+; ARM-LABEL: t12288:
+; ARM:       @ %bb.0: @ %entry
+; ARM-NEXT:    add r0, r0, r0, lsl #1
+; ARM-NEXT:    lsl r0, r0, #12
+; ARM-NEXT:    mov pc, lr
+;
+; THUMB2-LABEL: t12288:
+; THUMB2:       @ %bb.0: @ %entry
+; THUMB2-NEXT:    add.w r0, r0, r0, lsl #1
+; THUMB2-NEXT:    lsls r0, r0, #12
+; THUMB2-NEXT:    bx lr
+;
+; THUMB-LABEL: t12288:
+; THUMB:       @ %bb.0: @ %entry
+; THUMB-NEXT:    movs r1, #3
+; THUMB-NEXT:    lsls r1, r1, #12
+; THUMB-NEXT:    muls r0, r1, r0
+; THUMB-NEXT:    bx lr
 entry:
-; CHECK-LABEL: t12288:
-; CHECK: add r0, r0, r0, lsl #1
-; CHECK: lsl{{.*}}#12
         %0 = mul i32 %v, 12288
         ret i32 %0
 }
 
 define i32 @tn9(i32 %v) nounwind readnone {
+; ARM-LABEL: tn9:
+; ARM:       @ %bb.0: @ %entry
+; ARM-NEXT:    add r0, r0, r0, lsl #3
+; ARM-NEXT:    rsb r0, r0, #0
+; ARM-NEXT:    mov pc, lr
+;
+; THUMB2-LABEL: tn9:
+; THUMB2:       @ %bb.0: @ %entry
+; THUMB2-NEXT:    add.w r0, r0, r0, lsl #3
+; THUMB2-NEXT:    rsbs r0, r0, #0
+; THUMB2-NEXT:    bx lr
+;
+; THUMB-LABEL: tn9:
+; THUMB:       @ %bb.0: @ %entry
+; THUMB-NEXT:    movs r1, #8
+; THUMB-NEXT:    mvns r1, r1
+; THUMB-NEXT:    muls r0, r1, r0
+; THUMB-NEXT:    bx lr
 entry:
-; CHECK-LABEL: tn9:
-; CHECK: add	r0, r0, r0, lsl #3
-; CHECK: rsb	r0, r0, #0
         %0 = mul i32 %v, -9
         ret i32 %0
 }
 
 define i32 @tn7(i32 %v) nounwind readnone {
+; ARM-LABEL: tn7:
+; ARM:       @ %bb.0: @ %entry
+; ARM-NEXT:    sub r0, r0, r0, lsl #3
+; ARM-NEXT:    mov pc, lr
+;
+; THUMB2-LABEL: tn7:
+; THUMB2:       @ %bb.0: @ %entry
+; THUMB2-NEXT:    sub.w r0, r0, r0, lsl #3
+; THUMB2-NEXT:    bx lr
+;
+; THUMB-LABEL: tn7:
+; THUMB:       @ %bb.0: @ %entry
+; THUMB-NEXT:    movs r1, #6
+; THUMB-NEXT:    mvns r1, r1
+; THUMB-NEXT:    muls r0, r1, r0
+; THUMB-NEXT:    bx lr
 entry:
-; CHECK-LABEL: tn7:
-; CHECK: sub r0, r0, r0, lsl #3
 	%0 = mul i32 %v, -7
 	ret i32 %0
 }
 
 define i32 @tn5(i32 %v) nounwind readnone {
+; ARM-LABEL: tn5:
+; ARM:       @ %bb.0: @ %entry
+; ARM-NEXT:    add r0, r0, r0, lsl #2
+; ARM-NEXT:    rsb r0, r0, #0
+; ARM-NEXT:    mov pc, lr
+;
+; THUMB2-LABEL: tn5:
+; THUMB2:       @ %bb.0: @ %entry
+; THUMB2-NEXT:    add.w r0, r0, r0, lsl #2
+; THUMB2-NEXT:    rsbs r0, r0, #0
+; THUMB2-NEXT:    bx lr
+;
+; THUMB-LABEL: tn5:
+; THUMB:       @ %bb.0: @ %entry
+; THUMB-NEXT:    movs r1, #4
+; THUMB-NEXT:    mvns r1, r1
+; THUMB-NEXT:    muls r0, r1, r0
+; THUMB-NEXT:    bx lr
 entry:
-; CHECK-LABEL: tn5:
-; CHECK: add r0, r0, r0, lsl #2
-; CHECK: rsb r0, r0, #0
         %0 = mul i32 %v, -5
         ret i32 %0
 }
 
 define i32 @tn3(i32 %v) nounwind readnone {
+; ARM-LABEL: tn3:
+; ARM:       @ %bb.0: @ %entry
+; ARM-NEXT:    sub r0, r0, r0, lsl #2
+; ARM-NEXT:    mov pc, lr
+;
+; THUMB2-LABEL: tn3:
+; THUMB2:       @ %bb.0: @ %entry
+; THUMB2-NEXT:    sub.w r0, r0, r0, lsl #2
+; THUMB2-NEXT:    bx lr
+;
+; THUMB-LABEL: tn3:
+; THUMB:       @ %bb.0: @ %entry
+; THUMB-NEXT:    movs r1, #2
+; THUMB-NEXT:    mvns r1, r1
+; THUMB-NEXT:    muls r0, r1, r0
+; THUMB-NEXT:    bx lr
 entry:
 ; CHECK-LABEL: tn3:
 ; CHECK: sub r0, r0, r0, lsl #2
@@ -76,6 +206,27 @@ entry:
 }
 
 define i32 @tn12288(i32 %v) nounwind readnone {
+; ARM-LABEL: tn12288:
+; ARM:       @ %bb.0: @ %entry
+; ARM-NEXT:    sub r0, r0, r0, lsl #2
+; ARM-NEXT:    lsl r0, r0, #12
+; ARM-NEXT:    mov pc, lr
+;
+; THUMB2-LABEL: tn12288:
+; THUMB2:       @ %bb.0: @ %entry
+; THUMB2-NEXT:    sub.w r0, r0, r0, lsl #2
+; THUMB2-NEXT:    lsls r0, r0, #12
+; THUMB2-NEXT:    bx lr
+;
+; THUMB-LABEL: tn12288:
+; THUMB:       @ %bb.0: @ %entry
+; THUMB-NEXT:    ldr r1, .LCPI9_0
+; THUMB-NEXT:    muls r0, r1, r0
+; THUMB-NEXT:    bx lr
+; THUMB-NEXT:    .p2align 2
+; THUMB-NEXT:  @ %bb.1:
+; THUMB-NEXT:  .LCPI9_0:
+; THUMB-NEXT:    .long 4294955008 @ 0xffffd000
 entry:
 ; CHECK-LABEL: tn12288:
 ; CHECK: sub r0, r0, r0, lsl #2
diff --git a/llvm/test/CodeGen/ARM/neon-v8.1a.ll b/llvm/test/CodeGen/ARM/neon-v8.1a.ll
index 91259139d..95d208580 100644
--- a/llvm/test/CodeGen/ARM/neon-v8.1a.ll
+++ b/llvm/test/CodeGen/ARM/neon-v8.1a.ll
@@ -8,20 +8,20 @@ declare <8 x i16> @llvm.arm.neon.vqrdmulh.v8i16(<8 x i16>, <8 x i16>)
 declare <2 x i32> @llvm.arm.neon.vqrdmulh.v2i32(<2 x i32>, <2 x i32>)
 declare <4 x i32> @llvm.arm.neon.vqrdmulh.v4i32(<4 x i32>, <4 x i32>)
 
-declare <4 x i16> @llvm.arm.neon.vqadds.v4i16(<4 x i16>, <4 x i16>)
-declare <8 x i16> @llvm.arm.neon.vqadds.v8i16(<8 x i16>, <8 x i16>)
-declare <2 x i32> @llvm.arm.neon.vqadds.v2i32(<2 x i32>, <2 x i32>)
-declare <4 x i32> @llvm.arm.neon.vqadds.v4i32(<4 x i32>, <4 x i32>)
+declare <4 x i16> @llvm.sadd.sat.v4i16(<4 x i16>, <4 x i16>)
+declare <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16>, <8 x i16>)
+declare <2 x i32> @llvm.sadd.sat.v2i32(<2 x i32>, <2 x i32>)
+declare <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32>, <4 x i32>)
 
-declare <4 x i16> @llvm.arm.neon.vqsubs.v4i16(<4 x i16>, <4 x i16>)
-declare <8 x i16> @llvm.arm.neon.vqsubs.v8i16(<8 x i16>, <8 x i16>)
-declare <2 x i32> @llvm.arm.neon.vqsubs.v2i32(<2 x i32>, <2 x i32>)
-declare <4 x i32> @llvm.arm.neon.vqsubs.v4i32(<4 x i32>, <4 x i32>)
+declare <4 x i16> @llvm.ssub.sat.v4i16(<4 x i16>, <4 x i16>)
+declare <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16>, <8 x i16>)
+declare <2 x i32> @llvm.ssub.sat.v2i32(<2 x i32>, <2 x i32>)
+declare <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32>, <4 x i32>)
 
 define <4 x i16> @test_vqrdmlah_v4i16(<4 x i16> %acc, <4 x i16> %mhs, <4 x i16> %rhs) {
 ; CHECK-LABEL: test_vqrdmlah_v4i16:
    %prod = call <4 x i16> @llvm.arm.neon.vqrdmulh.v4i16(<4 x i16> %mhs,  <4 x i16> %rhs)
-   %retval =  call <4 x i16> @llvm.arm.neon.vqadds.v4i16(<4 x i16> %acc,  <4 x i16> %prod)
+   %retval =  call <4 x i16> @llvm.sadd.sat.v4i16(<4 x i16> %acc,  <4 x i16> %prod)
 ; CHECK: vqrdmlah.s16 {{d[0-9]+}}, {{d[0-9]+}}, {{d[0-9]+}}
    ret <4 x i16> %retval
 }
@@ -29,7 +29,7 @@ define <4 x i16> @test_vqrdmlah_v4i16(<4 x i16> %acc, <4 x i16> %mhs, <4 x i16>
 define <8 x i16> @test_vqrdmlah_v8i16(<8 x i16> %acc, <8 x i16> %mhs, <8 x i16> %rhs) {
 ; CHECK-LABEL: test_vqrdmlah_v8i16:
    %prod = call <8 x i16> @llvm.arm.neon.vqrdmulh.v8i16(<8 x i16> %mhs, <8 x i16> %rhs)
-   %retval =  call <8 x i16> @llvm.arm.neon.vqadds.v8i16(<8 x i16> %acc, <8 x i16> %prod)
+   %retval =  call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> %acc, <8 x i16> %prod)
 ; CHECK: vqrdmlah.s16 {{q[0-9]+}}, {{q[0-9]+}}, {{q[0-9]+}}
    ret <8 x i16> %retval
 }
@@ -37,7 +37,7 @@ define <8 x i16> @test_vqrdmlah_v8i16(<8 x i16> %acc, <8 x i16> %mhs, <8 x i16>
 define <2 x i32> @test_vqrdmlah_v2i32(<2 x i32> %acc, <2 x i32> %mhs, <2 x i32> %rhs) {
 ; CHECK-LABEL: test_vqrdmlah_v2i32:
    %prod = call <2 x i32> @llvm.arm.neon.vqrdmulh.v2i32(<2 x i32> %mhs, <2 x i32> %rhs)
-   %retval =  call <2 x i32> @llvm.arm.neon.vqadds.v2i32(<2 x i32> %acc, <2 x i32> %prod)
+   %retval =  call <2 x i32> @llvm.sadd.sat.v2i32(<2 x i32> %acc, <2 x i32> %prod)
 ; CHECK: vqrdmlah.s32 {{d[0-9]+}}, {{d[0-9]+}}, {{d[0-9]+}}
    ret <2 x i32> %retval
 }
@@ -45,7 +45,7 @@ define <2 x i32> @test_vqrdmlah_v2i32(<2 x i32> %acc, <2 x i32> %mhs, <2 x i32>
 define <4 x i32> @test_vqrdmlah_v4i32(<4 x i32> %acc, <4 x i32> %mhs, <4 x i32> %rhs) {
 ; CHECK-LABEL: test_vqrdmlah_v4i32:
    %prod = call <4 x i32> @llvm.arm.neon.vqrdmulh.v4i32(<4 x i32> %mhs, <4 x i32> %rhs)
-   %retval =  call <4 x i32> @llvm.arm.neon.vqadds.v4i32(<4 x i32> %acc, <4 x i32> %prod)
+   %retval =  call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> %acc, <4 x i32> %prod)
 ; CHECK: vqrdmlah.s32 {{q[0-9]+}}, {{q[0-9]+}}, {{q[0-9]+}}
    ret <4 x i32> %retval
 }
@@ -53,7 +53,7 @@ define <4 x i32> @test_vqrdmlah_v4i32(<4 x i32> %acc, <4 x i32> %mhs, <4 x i32>
 define <4 x i16> @test_vqrdmlsh_v4i16(<4 x i16> %acc, <4 x i16> %mhs, <4 x i16> %rhs) {
 ; CHECK-LABEL: test_vqrdmlsh_v4i16:
    %prod = call <4 x i16> @llvm.arm.neon.vqrdmulh.v4i16(<4 x i16> %mhs,  <4 x i16> %rhs)
-   %retval =  call <4 x i16> @llvm.arm.neon.vqsubs.v4i16(<4 x i16> %acc, <4 x i16> %prod)
+   %retval =  call <4 x i16> @llvm.ssub.sat.v4i16(<4 x i16> %acc, <4 x i16> %prod)
 ; CHECK: vqrdmlsh.s16 {{d[0-9]+}}, {{d[0-9]+}}, {{d[0-9]+}}
    ret <4 x i16> %retval
 }
@@ -61,7 +61,7 @@ define <4 x i16> @test_vqrdmlsh_v4i16(<4 x i16> %acc, <4 x i16> %mhs, <4 x i16>
 define <8 x i16> @test_vqrdmlsh_v8i16(<8 x i16> %acc, <8 x i16> %mhs, <8 x i16> %rhs) {
 ; CHECK-LABEL: test_vqrdmlsh_v8i16:
    %prod = call <8 x i16> @llvm.arm.neon.vqrdmulh.v8i16(<8 x i16> %mhs, <8 x i16> %rhs)
-   %retval =  call <8 x i16> @llvm.arm.neon.vqsubs.v8i16(<8 x i16> %acc, <8 x i16> %prod)
+   %retval =  call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> %acc, <8 x i16> %prod)
 ; CHECK: vqrdmlsh.s16 {{q[0-9]+}}, {{q[0-9]+}}, {{q[0-9]+}}
    ret <8 x i16> %retval
 }
@@ -69,7 +69,7 @@ define <8 x i16> @test_vqrdmlsh_v8i16(<8 x i16> %acc, <8 x i16> %mhs, <8 x i16>
 define <2 x i32> @test_vqrdmlsh_v2i32(<2 x i32> %acc, <2 x i32> %mhs, <2 x i32> %rhs) {
 ; CHECK-LABEL: test_vqrdmlsh_v2i32:
    %prod = call <2 x i32> @llvm.arm.neon.vqrdmulh.v2i32(<2 x i32> %mhs, <2 x i32> %rhs)
-   %retval =  call <2 x i32> @llvm.arm.neon.vqsubs.v2i32(<2 x i32> %acc, <2 x i32> %prod)
+   %retval =  call <2 x i32> @llvm.ssub.sat.v2i32(<2 x i32> %acc, <2 x i32> %prod)
 ; CHECK: vqrdmlsh.s32 {{d[0-9]+}}, {{d[0-9]+}}, {{d[0-9]+}}
    ret <2 x i32> %retval
 }
@@ -77,7 +77,7 @@ define <2 x i32> @test_vqrdmlsh_v2i32(<2 x i32> %acc, <2 x i32> %mhs, <2 x i32>
 define <4 x i32> @test_vqrdmlsh_v4i32(<4 x i32> %acc, <4 x i32> %mhs, <4 x i32> %rhs) {
 ; CHECK-LABEL: test_vqrdmlsh_v4i32:
    %prod = call <4 x i32> @llvm.arm.neon.vqrdmulh.v4i32(<4 x i32> %mhs, <4 x i32> %rhs)
-   %retval =  call <4 x i32> @llvm.arm.neon.vqsubs.v4i32(<4 x i32> %acc, <4 x i32> %prod)
+   %retval =  call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> %acc, <4 x i32> %prod)
 ; CHECK: vqrdmlsh.s32 {{q[0-9]+}}, {{q[0-9]+}}, {{q[0-9]+}}
    ret <4 x i32> %retval
 }
@@ -90,7 +90,7 @@ define <4 x i16> @test_vqrdmlah_lane_s16(<4 x i16> %acc, <4 x i16> %x, <4 x i16>
 entry:
   %shuffle = shufflevector <4 x i16> %v, <4 x i16> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
   %prod = call <4 x i16> @llvm.arm.neon.vqrdmulh.v4i16(<4 x i16> %x, <4 x i16> %shuffle)
-  %retval =  call <4 x i16> @llvm.arm.neon.vqadds.v4i16(<4 x i16> %acc, <4 x i16> %prod)
+  %retval =  call <4 x i16> @llvm.sadd.sat.v4i16(<4 x i16> %acc, <4 x i16> %prod)
 ; CHECK: vqrdmlah.s16 {{d[0-9]+}}, {{d[0-9]+}}, {{d[0-9]+}}[3]
   ret <4 x i16> %retval
 }
@@ -100,7 +100,7 @@ define <8 x i16> @test_vqrdmlahq_lane_s16(<8 x i16> %acc, <8 x i16> %x, <4 x i16
 entry:
   %shuffle = shufflevector <4 x i16> %v, <4 x i16> undef, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
   %prod = call <8 x i16> @llvm.arm.neon.vqrdmulh.v8i16(<8 x i16> %x, <8 x i16> %shuffle)
-  %retval =  call <8 x i16> @llvm.arm.neon.vqadds.v8i16(<8 x i16> %acc, <8 x i16> %prod)
+  %retval =  call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> %acc, <8 x i16> %prod)
 ; CHECK: vqrdmlah.s16 {{q[0-9]+}}, {{q[0-9]+}}, {{d[0-9]+}}[2]
   ret <8 x i16> %retval
 }
@@ -110,7 +110,7 @@ define <2 x i32> @test_vqrdmlah_lane_s32(<2 x i32> %acc, <2 x i32> %x, <2 x i32>
 entry:
   %shuffle = shufflevector <2 x i32> %v, <2 x i32> undef, <2 x i32> <i32 1, i32 1>
   %prod = tail call <2 x i32> @llvm.arm.neon.vqrdmulh.v2i32(<2 x i32> %x, <2 x i32> %shuffle)
-  %retval =  call <2 x i32> @llvm.arm.neon.vqadds.v2i32(<2 x i32> %acc, <2 x i32> %prod)
+  %retval =  call <2 x i32> @llvm.sadd.sat.v2i32(<2 x i32> %acc, <2 x i32> %prod)
 ; CHECK: vqrdmlah.s32 {{d[0-9]+}}, {{d[0-9]+}}, {{d[0-9]+}}[1]
   ret <2 x i32> %retval
 }
@@ -120,7 +120,7 @@ define <4 x i32> @test_vqrdmlahq_lane_s32(<4 x i32> %acc,<4 x i32> %x, <2 x i32>
 entry:
   %shuffle = shufflevector <2 x i32> %v, <2 x i32> undef, <4 x i32> zeroinitializer
   %prod = tail call <4 x i32> @llvm.arm.neon.vqrdmulh.v4i32(<4 x i32> %x, <4 x i32> %shuffle)
-  %retval =  call <4 x i32> @llvm.arm.neon.vqadds.v4i32(<4 x i32> %acc, <4 x i32> %prod)
+  %retval =  call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> %acc, <4 x i32> %prod)
 ; CHECK: vqrdmlah.s32 {{q[0-9]+}}, {{q[0-9]+}}, {{d[0-9]+}}[0]
   ret <4 x i32> %retval
 }
@@ -130,7 +130,7 @@ define <4 x i16> @test_vqrdmlsh_lane_s16(<4 x i16> %acc, <4 x i16> %x, <4 x i16>
 entry:
   %shuffle = shufflevector <4 x i16> %v, <4 x i16> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
   %prod = call <4 x i16> @llvm.arm.neon.vqrdmulh.v4i16(<4 x i16> %x, <4 x i16> %shuffle)
-  %retval =  call <4 x i16> @llvm.arm.neon.vqsubs.v4i16(<4 x i16> %acc, <4 x i16> %prod)
+  %retval =  call <4 x i16> @llvm.ssub.sat.v4i16(<4 x i16> %acc, <4 x i16> %prod)
 ; CHECK: vqrdmlsh.s16 {{d[0-9]+}}, {{d[0-9]+}}, {{d[0-9]+}}[3]
   ret <4 x i16> %retval
 }
@@ -140,7 +140,7 @@ define <8 x i16> @test_vqrdmlshq_lane_s16(<8 x i16> %acc, <8 x i16> %x, <4 x i16
 entry:
   %shuffle = shufflevector <4 x i16> %v, <4 x i16> undef, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
   %prod = call <8 x i16> @llvm.arm.neon.vqrdmulh.v8i16(<8 x i16> %x, <8 x i16> %shuffle)
-  %retval =  call <8 x i16> @llvm.arm.neon.vqsubs.v8i16(<8 x i16> %acc, <8 x i16> %prod)
+  %retval =  call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> %acc, <8 x i16> %prod)
 ; CHECK: vqrdmlsh.s16 {{q[0-9]+}}, {{q[0-9]+}}, {{d[0-9]+}}[2]
   ret <8 x i16> %retval
 }
@@ -150,7 +150,7 @@ define <2 x i32> @test_vqrdmlsh_lane_s32(<2 x i32> %acc, <2 x i32> %x, <2 x i32>
 entry:
   %shuffle = shufflevector <2 x i32> %v, <2 x i32> undef, <2 x i32> <i32 1, i32 1>
   %prod = tail call <2 x i32> @llvm.arm.neon.vqrdmulh.v2i32(<2 x i32> %x, <2 x i32> %shuffle)
-  %retval =  call <2 x i32> @llvm.arm.neon.vqsubs.v2i32(<2 x i32> %acc, <2 x i32> %prod)
+  %retval =  call <2 x i32> @llvm.ssub.sat.v2i32(<2 x i32> %acc, <2 x i32> %prod)
 ; CHECK: vqrdmlsh.s32  {{d[0-9]+}}, {{d[0-9]+}}, {{d[0-9]+}}[1]
   ret <2 x i32> %retval
 }
@@ -160,7 +160,7 @@ define <4 x i32> @test_vqrdmlshq_lane_s32(<4 x i32> %acc,<4 x i32> %x, <2 x i32>
 entry:
   %shuffle = shufflevector <2 x i32> %v, <2 x i32> undef, <4 x i32> zeroinitializer
   %prod = tail call <4 x i32> @llvm.arm.neon.vqrdmulh.v4i32(<4 x i32> %x, <4 x i32> %shuffle)
-  %retval =  call <4 x i32> @llvm.arm.neon.vqsubs.v4i32(<4 x i32> %acc, <4 x i32> %prod)
+  %retval =  call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> %acc, <4 x i32> %prod)
 ; CHECK: vqrdmlsh.s32 {{q[0-9]+}}, {{q[0-9]+}}, {{d[0-9]+}}[0]
   ret <4 x i32> %retval
 }
diff --git a/llvm/test/CodeGen/ARM/neon-vcadd.ll b/llvm/test/CodeGen/ARM/neon-vcadd.ll
new file mode 100644
index 000000000..93a85c8c7
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/neon-vcadd.ll
@@ -0,0 +1,54 @@
+; RUN: llc %s -mtriple=arm -mattr=+armv8.3-a,+fullfp16 -o - | FileCheck %s
+
+define <4 x half> @foo16x4_rot(<4 x half> %a, <4 x half> %b) {
+entry:
+; CHECK-LABEL: foo16x4_rot
+; CHECK-DAG: vcadd.f16 d{{[0-9]+}}, d{{[0-9]+}}, d{{[0-9]+}}, #90
+; CHECK-DAG: vcadd.f16 d{{[0-9]+}}, d{{[0-9]+}}, d{{[0-9]+}}, #270
+  %vcadd_rot90_v2.i = tail call <4 x half> @llvm.arm.neon.vcadd.rot90.v4f16(<4 x half> %a, <4 x half> %b)
+  %vcadd_rot270_v2.i = tail call <4 x half> @llvm.arm.neon.vcadd.rot270.v4f16(<4 x half> %a, <4 x half> %b)
+  %add = fadd <4 x half> %vcadd_rot90_v2.i, %vcadd_rot270_v2.i
+  ret <4 x half> %add
+}
+
+define <2 x float> @foo32x2_rot(<2 x float> %a, <2 x float> %b) {
+entry:
+; CHECK-LABEL: foo32x2_rot
+; CHECK-DAG: vcadd.f32 d{{[0-9]+}}, d{{[0-9]+}}, d{{[0-9]+}}, #90
+; CHECK-DAG: vcadd.f32 d{{[0-9]+}}, d{{[0-9]+}}, d{{[0-9]+}}, #270
+  %vcadd_rot90_v2.i = tail call <2 x float> @llvm.arm.neon.vcadd.rot90.v2f32(<2 x float> %a, <2 x float> %b)
+  %vcadd_rot270_v2.i = tail call <2 x float> @llvm.arm.neon.vcadd.rot270.v2f32(<2 x float> %a, <2 x float> %b)
+  %add = fadd <2 x float> %vcadd_rot90_v2.i, %vcadd_rot270_v2.i
+  ret <2 x float> %add
+}
+
+define <8 x half> @foo16x8_rot(<8 x half> %a, <8 x half> %b) {
+entry:
+; CHECK-LABEL: foo16x8_rot
+; CHECK-DAG: vcadd.f16 q{{[0-9]+}}, q{{[0-9]+}}, q{{[0-9]+}}, #90
+; CHECK-DAG: vcadd.f16 q{{[0-9]+}}, q{{[0-9]+}}, q{{[0-9]+}}, #270
+  %vcaddq_rot90_v2.i = tail call <8 x half> @llvm.arm.neon.vcadd.rot90.v8f16(<8 x half> %a, <8 x half> %b)
+  %vcaddq_rot270_v2.i = tail call <8 x half> @llvm.arm.neon.vcadd.rot270.v8f16(<8 x half> %a, <8 x half> %b)
+  %add = fadd <8 x half> %vcaddq_rot90_v2.i, %vcaddq_rot270_v2.i
+  ret <8 x half> %add
+}
+
+define <4 x float> @foo32x4_rot(<4 x float> %a, <4 x float> %b) {
+entry:
+; CHECK-LABEL: foo32x4_rot
+; CHECK-DAG: vcadd.f32 q{{[0-9]+}}, q{{[0-9]+}}, q{{[0-9]+}}, #90
+; CHECK-DAG: vcadd.f32 q{{[0-9]+}}, q{{[0-9]+}}, q{{[0-9]+}}, #270
+  %vcaddq_rot90_v2.i = tail call <4 x float> @llvm.arm.neon.vcadd.rot90.v4f32(<4 x float> %a, <4 x float> %b)
+  %vcaddq_rot270_v2.i = tail call <4 x float> @llvm.arm.neon.vcadd.rot270.v4f32(<4 x float> %a, <4 x float> %b)
+  %add = fadd <4 x float> %vcaddq_rot90_v2.i, %vcaddq_rot270_v2.i
+  ret <4 x float> %add
+}
+
+declare <4 x half> @llvm.arm.neon.vcadd.rot90.v4f16(<4 x half>, <4 x half>)
+declare <4 x half> @llvm.arm.neon.vcadd.rot270.v4f16(<4 x half>, <4 x half>)
+declare <2 x float> @llvm.arm.neon.vcadd.rot90.v2f32(<2 x float>, <2 x float>)
+declare <2 x float> @llvm.arm.neon.vcadd.rot270.v2f32(<2 x float>, <2 x float>)
+declare <8 x half> @llvm.arm.neon.vcadd.rot90.v8f16(<8 x half>, <8 x half>)
+declare <8 x half> @llvm.arm.neon.vcadd.rot270.v8f16(<8 x half>, <8 x half>)
+declare <4 x float> @llvm.arm.neon.vcadd.rot90.v4f32(<4 x float>, <4 x float>)
+declare <4 x float> @llvm.arm.neon.vcadd.rot270.v4f32(<4 x float>, <4 x float>)
diff --git a/llvm/test/CodeGen/ARM/neon-vqaddsub-upgrade.ll b/llvm/test/CodeGen/ARM/neon-vqaddsub-upgrade.ll
new file mode 100644
index 000000000..a13238101
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/neon-vqaddsub-upgrade.ll
@@ -0,0 +1,330 @@
+; RUN: llc -mtriple=arm-eabi -mattr=+neon %s -o - | FileCheck %s
+
+define <8 x i8> @vqadds8(<8 x i8>* %A, <8 x i8>* %B) nounwind {
+;CHECK-LABEL: vqadds8:
+;CHECK: vqadd.s8
+	%tmp1 = load <8 x i8>, <8 x i8>* %A
+	%tmp2 = load <8 x i8>, <8 x i8>* %B
+	%tmp3 = call <8 x i8> @llvm.arm.neon.vqadds.v8i8(<8 x i8> %tmp1, <8 x i8> %tmp2)
+	ret <8 x i8> %tmp3
+}
+
+define <4 x i16> @vqadds16(<4 x i16>* %A, <4 x i16>* %B) nounwind {
+;CHECK-LABEL: vqadds16:
+;CHECK: vqadd.s16
+	%tmp1 = load <4 x i16>, <4 x i16>* %A
+	%tmp2 = load <4 x i16>, <4 x i16>* %B
+	%tmp3 = call <4 x i16> @llvm.arm.neon.vqadds.v4i16(<4 x i16> %tmp1, <4 x i16> %tmp2)
+	ret <4 x i16> %tmp3
+}
+
+define <2 x i32> @vqadds32(<2 x i32>* %A, <2 x i32>* %B) nounwind {
+;CHECK-LABEL: vqadds32:
+;CHECK: vqadd.s32
+	%tmp1 = load <2 x i32>, <2 x i32>* %A
+	%tmp2 = load <2 x i32>, <2 x i32>* %B
+	%tmp3 = call <2 x i32> @llvm.arm.neon.vqadds.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2)
+	ret <2 x i32> %tmp3
+}
+
+define <1 x i64> @vqadds64(<1 x i64>* %A, <1 x i64>* %B) nounwind {
+;CHECK-LABEL: vqadds64:
+;CHECK: vqadd.s64
+	%tmp1 = load <1 x i64>, <1 x i64>* %A
+	%tmp2 = load <1 x i64>, <1 x i64>* %B
+	%tmp3 = call <1 x i64> @llvm.arm.neon.vqadds.v1i64(<1 x i64> %tmp1, <1 x i64> %tmp2)
+	ret <1 x i64> %tmp3
+}
+
+define <8 x i8> @vqaddu8(<8 x i8>* %A, <8 x i8>* %B) nounwind {
+;CHECK-LABEL: vqaddu8:
+;CHECK: vqadd.u8
+	%tmp1 = load <8 x i8>, <8 x i8>* %A
+	%tmp2 = load <8 x i8>, <8 x i8>* %B
+	%tmp3 = call <8 x i8> @llvm.arm.neon.vqaddu.v8i8(<8 x i8> %tmp1, <8 x i8> %tmp2)
+	ret <8 x i8> %tmp3
+}
+
+define <4 x i16> @vqaddu16(<4 x i16>* %A, <4 x i16>* %B) nounwind {
+;CHECK-LABEL: vqaddu16:
+;CHECK: vqadd.u16
+	%tmp1 = load <4 x i16>, <4 x i16>* %A
+	%tmp2 = load <4 x i16>, <4 x i16>* %B
+	%tmp3 = call <4 x i16> @llvm.arm.neon.vqaddu.v4i16(<4 x i16> %tmp1, <4 x i16> %tmp2)
+	ret <4 x i16> %tmp3
+}
+
+define <2 x i32> @vqaddu32(<2 x i32>* %A, <2 x i32>* %B) nounwind {
+;CHECK-LABEL: vqaddu32:
+;CHECK: vqadd.u32
+	%tmp1 = load <2 x i32>, <2 x i32>* %A
+	%tmp2 = load <2 x i32>, <2 x i32>* %B
+	%tmp3 = call <2 x i32> @llvm.arm.neon.vqaddu.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2)
+	ret <2 x i32> %tmp3
+}
+
+define <1 x i64> @vqaddu64(<1 x i64>* %A, <1 x i64>* %B) nounwind {
+;CHECK-LABEL: vqaddu64:
+;CHECK: vqadd.u64
+	%tmp1 = load <1 x i64>, <1 x i64>* %A
+	%tmp2 = load <1 x i64>, <1 x i64>* %B
+	%tmp3 = call <1 x i64> @llvm.arm.neon.vqaddu.v1i64(<1 x i64> %tmp1, <1 x i64> %tmp2)
+	ret <1 x i64> %tmp3
+}
+
+define <16 x i8> @vqaddQs8(<16 x i8>* %A, <16 x i8>* %B) nounwind {
+;CHECK-LABEL: vqaddQs8:
+;CHECK: vqadd.s8
+	%tmp1 = load <16 x i8>, <16 x i8>* %A
+	%tmp2 = load <16 x i8>, <16 x i8>* %B
+	%tmp3 = call <16 x i8> @llvm.arm.neon.vqadds.v16i8(<16 x i8> %tmp1, <16 x i8> %tmp2)
+	ret <16 x i8> %tmp3
+}
+
+define <8 x i16> @vqaddQs16(<8 x i16>* %A, <8 x i16>* %B) nounwind {
+;CHECK-LABEL: vqaddQs16:
+;CHECK: vqadd.s16
+	%tmp1 = load <8 x i16>, <8 x i16>* %A
+	%tmp2 = load <8 x i16>, <8 x i16>* %B
+	%tmp3 = call <8 x i16> @llvm.arm.neon.vqadds.v8i16(<8 x i16> %tmp1, <8 x i16> %tmp2)
+	ret <8 x i16> %tmp3
+}
+
+define <4 x i32> @vqaddQs32(<4 x i32>* %A, <4 x i32>* %B) nounwind {
+;CHECK-LABEL: vqaddQs32:
+;CHECK: vqadd.s32
+	%tmp1 = load <4 x i32>, <4 x i32>* %A
+	%tmp2 = load <4 x i32>, <4 x i32>* %B
+	%tmp3 = call <4 x i32> @llvm.arm.neon.vqadds.v4i32(<4 x i32> %tmp1, <4 x i32> %tmp2)
+	ret <4 x i32> %tmp3
+}
+
+define <2 x i64> @vqaddQs64(<2 x i64>* %A, <2 x i64>* %B) nounwind {
+;CHECK-LABEL: vqaddQs64:
+;CHECK: vqadd.s64
+	%tmp1 = load <2 x i64>, <2 x i64>* %A
+	%tmp2 = load <2 x i64>, <2 x i64>* %B
+	%tmp3 = call <2 x i64> @llvm.arm.neon.vqadds.v2i64(<2 x i64> %tmp1, <2 x i64> %tmp2)
+	ret <2 x i64> %tmp3
+}
+
+define <16 x i8> @vqaddQu8(<16 x i8>* %A, <16 x i8>* %B) nounwind {
+;CHECK-LABEL: vqaddQu8:
+;CHECK: vqadd.u8
+	%tmp1 = load <16 x i8>, <16 x i8>* %A
+	%tmp2 = load <16 x i8>, <16 x i8>* %B
+	%tmp3 = call <16 x i8> @llvm.arm.neon.vqaddu.v16i8(<16 x i8> %tmp1, <16 x i8> %tmp2)
+	ret <16 x i8> %tmp3
+}
+
+define <8 x i16> @vqaddQu16(<8 x i16>* %A, <8 x i16>* %B) nounwind {
+;CHECK-LABEL: vqaddQu16:
+;CHECK: vqadd.u16
+	%tmp1 = load <8 x i16>, <8 x i16>* %A
+	%tmp2 = load <8 x i16>, <8 x i16>* %B
+	%tmp3 = call <8 x i16> @llvm.arm.neon.vqaddu.v8i16(<8 x i16> %tmp1, <8 x i16> %tmp2)
+	ret <8 x i16> %tmp3
+}
+
+define <4 x i32> @vqaddQu32(<4 x i32>* %A, <4 x i32>* %B) nounwind {
+;CHECK-LABEL: vqaddQu32:
+;CHECK: vqadd.u32
+	%tmp1 = load <4 x i32>, <4 x i32>* %A
+	%tmp2 = load <4 x i32>, <4 x i32>* %B
+	%tmp3 = call <4 x i32> @llvm.arm.neon.vqaddu.v4i32(<4 x i32> %tmp1, <4 x i32> %tmp2)
+	ret <4 x i32> %tmp3
+}
+
+define <2 x i64> @vqaddQu64(<2 x i64>* %A, <2 x i64>* %B) nounwind {
+;CHECK-LABEL: vqaddQu64:
+;CHECK: vqadd.u64
+	%tmp1 = load <2 x i64>, <2 x i64>* %A
+	%tmp2 = load <2 x i64>, <2 x i64>* %B
+	%tmp3 = call <2 x i64> @llvm.arm.neon.vqaddu.v2i64(<2 x i64> %tmp1, <2 x i64> %tmp2)
+	ret <2 x i64> %tmp3
+}
+
+
+define <8 x i8> @vqsubs8(<8 x i8>* %A, <8 x i8>* %B) nounwind {
+;CHECK-LABEL: vqsubs8:
+;CHECK: vqsub.s8
+	%tmp1 = load <8 x i8>, <8 x i8>* %A
+	%tmp2 = load <8 x i8>, <8 x i8>* %B
+	%tmp3 = call <8 x i8> @llvm.arm.neon.vqsubs.v8i8(<8 x i8> %tmp1, <8 x i8> %tmp2)
+	ret <8 x i8> %tmp3
+}
+
+define <4 x i16> @vqsubs16(<4 x i16>* %A, <4 x i16>* %B) nounwind {
+;CHECK-LABEL: vqsubs16:
+;CHECK: vqsub.s16
+	%tmp1 = load <4 x i16>, <4 x i16>* %A
+	%tmp2 = load <4 x i16>, <4 x i16>* %B
+	%tmp3 = call <4 x i16> @llvm.arm.neon.vqsubs.v4i16(<4 x i16> %tmp1, <4 x i16> %tmp2)
+	ret <4 x i16> %tmp3
+}
+
+define <2 x i32> @vqsubs32(<2 x i32>* %A, <2 x i32>* %B) nounwind {
+;CHECK-LABEL: vqsubs32:
+;CHECK: vqsub.s32
+	%tmp1 = load <2 x i32>, <2 x i32>* %A
+	%tmp2 = load <2 x i32>, <2 x i32>* %B
+	%tmp3 = call <2 x i32> @llvm.arm.neon.vqsubs.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2)
+	ret <2 x i32> %tmp3
+}
+
+define <1 x i64> @vqsubs64(<1 x i64>* %A, <1 x i64>* %B) nounwind {
+;CHECK-LABEL: vqsubs64:
+;CHECK: vqsub.s64
+	%tmp1 = load <1 x i64>, <1 x i64>* %A
+	%tmp2 = load <1 x i64>, <1 x i64>* %B
+	%tmp3 = call <1 x i64> @llvm.arm.neon.vqsubs.v1i64(<1 x i64> %tmp1, <1 x i64> %tmp2)
+	ret <1 x i64> %tmp3
+}
+
+define <8 x i8> @vqsubu8(<8 x i8>* %A, <8 x i8>* %B) nounwind {
+;CHECK-LABEL: vqsubu8:
+;CHECK: vqsub.u8
+	%tmp1 = load <8 x i8>, <8 x i8>* %A
+	%tmp2 = load <8 x i8>, <8 x i8>* %B
+	%tmp3 = call <8 x i8> @llvm.arm.neon.vqsubu.v8i8(<8 x i8> %tmp1, <8 x i8> %tmp2)
+	ret <8 x i8> %tmp3
+}
+
+define <4 x i16> @vqsubu16(<4 x i16>* %A, <4 x i16>* %B) nounwind {
+;CHECK-LABEL: vqsubu16:
+;CHECK: vqsub.u16
+	%tmp1 = load <4 x i16>, <4 x i16>* %A
+	%tmp2 = load <4 x i16>, <4 x i16>* %B
+	%tmp3 = call <4 x i16> @llvm.arm.neon.vqsubu.v4i16(<4 x i16> %tmp1, <4 x i16> %tmp2)
+	ret <4 x i16> %tmp3
+}
+
+define <2 x i32> @vqsubu32(<2 x i32>* %A, <2 x i32>* %B) nounwind {
+;CHECK-LABEL: vqsubu32:
+;CHECK: vqsub.u32
+	%tmp1 = load <2 x i32>, <2 x i32>* %A
+	%tmp2 = load <2 x i32>, <2 x i32>* %B
+	%tmp3 = call <2 x i32> @llvm.arm.neon.vqsubu.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2)
+	ret <2 x i32> %tmp3
+}
+
+define <1 x i64> @vqsubu64(<1 x i64>* %A, <1 x i64>* %B) nounwind {
+;CHECK-LABEL: vqsubu64:
+;CHECK: vqsub.u64
+	%tmp1 = load <1 x i64>, <1 x i64>* %A
+	%tmp2 = load <1 x i64>, <1 x i64>* %B
+	%tmp3 = call <1 x i64> @llvm.arm.neon.vqsubu.v1i64(<1 x i64> %tmp1, <1 x i64> %tmp2)
+	ret <1 x i64> %tmp3
+}
+
+define <16 x i8> @vqsubQs8(<16 x i8>* %A, <16 x i8>* %B) nounwind {
+;CHECK-LABEL: vqsubQs8:
+;CHECK: vqsub.s8
+	%tmp1 = load <16 x i8>, <16 x i8>* %A
+	%tmp2 = load <16 x i8>, <16 x i8>* %B
+	%tmp3 = call <16 x i8> @llvm.arm.neon.vqsubs.v16i8(<16 x i8> %tmp1, <16 x i8> %tmp2)
+	ret <16 x i8> %tmp3
+}
+
+define <8 x i16> @vqsubQs16(<8 x i16>* %A, <8 x i16>* %B) nounwind {
+;CHECK-LABEL: vqsubQs16:
+;CHECK: vqsub.s16
+	%tmp1 = load <8 x i16>, <8 x i16>* %A
+	%tmp2 = load <8 x i16>, <8 x i16>* %B
+	%tmp3 = call <8 x i16> @llvm.arm.neon.vqsubs.v8i16(<8 x i16> %tmp1, <8 x i16> %tmp2)
+	ret <8 x i16> %tmp3
+}
+
+define <4 x i32> @vqsubQs32(<4 x i32>* %A, <4 x i32>* %B) nounwind {
+;CHECK-LABEL: vqsubQs32:
+;CHECK: vqsub.s32
+	%tmp1 = load <4 x i32>, <4 x i32>* %A
+	%tmp2 = load <4 x i32>, <4 x i32>* %B
+	%tmp3 = call <4 x i32> @llvm.arm.neon.vqsubs.v4i32(<4 x i32> %tmp1, <4 x i32> %tmp2)
+	ret <4 x i32> %tmp3
+}
+
+define <2 x i64> @vqsubQs64(<2 x i64>* %A, <2 x i64>* %B) nounwind {
+;CHECK-LABEL: vqsubQs64:
+;CHECK: vqsub.s64
+	%tmp1 = load <2 x i64>, <2 x i64>* %A
+	%tmp2 = load <2 x i64>, <2 x i64>* %B
+	%tmp3 = call <2 x i64> @llvm.arm.neon.vqsubs.v2i64(<2 x i64> %tmp1, <2 x i64> %tmp2)
+	ret <2 x i64> %tmp3
+}
+
+define <16 x i8> @vqsubQu8(<16 x i8>* %A, <16 x i8>* %B) nounwind {
+;CHECK-LABEL: vqsubQu8:
+;CHECK: vqsub.u8
+	%tmp1 = load <16 x i8>, <16 x i8>* %A
+	%tmp2 = load <16 x i8>, <16 x i8>* %B
+	%tmp3 = call <16 x i8> @llvm.arm.neon.vqsubu.v16i8(<16 x i8> %tmp1, <16 x i8> %tmp2)
+	ret <16 x i8> %tmp3
+}
+
+define <8 x i16> @vqsubQu16(<8 x i16>* %A, <8 x i16>* %B) nounwind {
+;CHECK-LABEL: vqsubQu16:
+;CHECK: vqsub.u16
+	%tmp1 = load <8 x i16>, <8 x i16>* %A
+	%tmp2 = load <8 x i16>, <8 x i16>* %B
+	%tmp3 = call <8 x i16> @llvm.arm.neon.vqsubu.v8i16(<8 x i16> %tmp1, <8 x i16> %tmp2)
+	ret <8 x i16> %tmp3
+}
+
+define <4 x i32> @vqsubQu32(<4 x i32>* %A, <4 x i32>* %B) nounwind {
+;CHECK-LABEL: vqsubQu32:
+;CHECK: vqsub.u32
+	%tmp1 = load <4 x i32>, <4 x i32>* %A
+	%tmp2 = load <4 x i32>, <4 x i32>* %B
+	%tmp3 = call <4 x i32> @llvm.arm.neon.vqsubu.v4i32(<4 x i32> %tmp1, <4 x i32> %tmp2)
+	ret <4 x i32> %tmp3
+}
+
+define <2 x i64> @vqsubQu64(<2 x i64>* %A, <2 x i64>* %B) nounwind {
+;CHECK-LABEL: vqsubQu64:
+;CHECK: vqsub.u64
+	%tmp1 = load <2 x i64>, <2 x i64>* %A
+	%tmp2 = load <2 x i64>, <2 x i64>* %B
+	%tmp3 = call <2 x i64> @llvm.arm.neon.vqsubu.v2i64(<2 x i64> %tmp1, <2 x i64> %tmp2)
+	ret <2 x i64> %tmp3
+}
+
+declare <8 x i8>  @llvm.arm.neon.vqadds.v8i8(<8 x i8>, <8 x i8>) nounwind readnone
+declare <4 x i16> @llvm.arm.neon.vqadds.v4i16(<4 x i16>, <4 x i16>) nounwind readnone
+declare <2 x i32> @llvm.arm.neon.vqadds.v2i32(<2 x i32>, <2 x i32>) nounwind readnone
+declare <1 x i64> @llvm.arm.neon.vqadds.v1i64(<1 x i64>, <1 x i64>) nounwind readnone
+
+declare <8 x i8>  @llvm.arm.neon.vqaddu.v8i8(<8 x i8>, <8 x i8>) nounwind readnone
+declare <4 x i16> @llvm.arm.neon.vqaddu.v4i16(<4 x i16>, <4 x i16>) nounwind readnone
+declare <2 x i32> @llvm.arm.neon.vqaddu.v2i32(<2 x i32>, <2 x i32>) nounwind readnone
+declare <1 x i64> @llvm.arm.neon.vqaddu.v1i64(<1 x i64>, <1 x i64>) nounwind readnone
+
+declare <16 x i8> @llvm.arm.neon.vqadds.v16i8(<16 x i8>, <16 x i8>) nounwind readnone
+declare <8 x i16> @llvm.arm.neon.vqadds.v8i16(<8 x i16>, <8 x i16>) nounwind readnone
+declare <4 x i32> @llvm.arm.neon.vqadds.v4i32(<4 x i32>, <4 x i32>) nounwind readnone
+declare <2 x i64> @llvm.arm.neon.vqadds.v2i64(<2 x i64>, <2 x i64>) nounwind readnone
+
+declare <16 x i8> @llvm.arm.neon.vqaddu.v16i8(<16 x i8>, <16 x i8>) nounwind readnone
+declare <8 x i16> @llvm.arm.neon.vqaddu.v8i16(<8 x i16>, <8 x i16>) nounwind readnone
+declare <4 x i32> @llvm.arm.neon.vqaddu.v4i32(<4 x i32>, <4 x i32>) nounwind readnone
+declare <2 x i64> @llvm.arm.neon.vqaddu.v2i64(<2 x i64>, <2 x i64>) nounwind readnone
+
+declare <8 x i8>  @llvm.arm.neon.vqsubs.v8i8(<8 x i8>, <8 x i8>) nounwind readnone
+declare <4 x i16> @llvm.arm.neon.vqsubs.v4i16(<4 x i16>, <4 x i16>) nounwind readnone
+declare <2 x i32> @llvm.arm.neon.vqsubs.v2i32(<2 x i32>, <2 x i32>) nounwind readnone
+declare <1 x i64> @llvm.arm.neon.vqsubs.v1i64(<1 x i64>, <1 x i64>) nounwind readnone
+
+declare <8 x i8>  @llvm.arm.neon.vqsubu.v8i8(<8 x i8>, <8 x i8>) nounwind readnone
+declare <4 x i16> @llvm.arm.neon.vqsubu.v4i16(<4 x i16>, <4 x i16>) nounwind readnone
+declare <2 x i32> @llvm.arm.neon.vqsubu.v2i32(<2 x i32>, <2 x i32>) nounwind readnone
+declare <1 x i64> @llvm.arm.neon.vqsubu.v1i64(<1 x i64>, <1 x i64>) nounwind readnone
+
+declare <16 x i8> @llvm.arm.neon.vqsubs.v16i8(<16 x i8>, <16 x i8>) nounwind readnone
+declare <8 x i16> @llvm.arm.neon.vqsubs.v8i16(<8 x i16>, <8 x i16>) nounwind readnone
+declare <4 x i32> @llvm.arm.neon.vqsubs.v4i32(<4 x i32>, <4 x i32>) nounwind readnone
+declare <2 x i64> @llvm.arm.neon.vqsubs.v2i64(<2 x i64>, <2 x i64>) nounwind readnone
+
+declare <16 x i8> @llvm.arm.neon.vqsubu.v16i8(<16 x i8>, <16 x i8>) nounwind readnone
+declare <8 x i16> @llvm.arm.neon.vqsubu.v8i16(<8 x i16>, <8 x i16>) nounwind readnone
+declare <4 x i32> @llvm.arm.neon.vqsubu.v4i32(<4 x i32>, <4 x i32>) nounwind readnone
+declare <2 x i64> @llvm.arm.neon.vqsubu.v2i64(<2 x i64>, <2 x i64>) nounwind readnone
diff --git a/llvm/test/CodeGen/ARM/noreturn.ll b/llvm/test/CodeGen/ARM/noreturn.ll
index f242afb99..97327082d 100644
--- a/llvm/test/CodeGen/ARM/noreturn.ll
+++ b/llvm/test/CodeGen/ARM/noreturn.ll
@@ -61,7 +61,7 @@ entry:
 }
 
 
-define i32 @test1_nofpelim() "no-frame-pointer-elim"="true" {
+define i32 @test1_nofpelim() "frame-pointer"="all" {
 ; CHECK-LABEL: @test1_nofpelim
 ; CHECK: push
 entry:
@@ -69,7 +69,7 @@ entry:
   unreachable
 }
 
-define i32 @test2_nofpelim(i32 %x, i32 %y) "no-frame-pointer-elim"="true" {
+define i32 @test2_nofpelim(i32 %x, i32 %y) "frame-pointer"="all" {
 ; CHECK-LABEL: @test2_nofpelim
 ; CHECK: push
 entry:
@@ -90,7 +90,7 @@ if.end:                                           ; preds = %entry
 }
 
 ; Test case for PR17825.
-define i32 @test3_nofpelim() "no-frame-pointer-elim"="true" {
+define i32 @test3_nofpelim() "frame-pointer"="all" {
 ; CHECK-LABEL: @test3_nofpelim
 ; CHECK: push
 entry:
@@ -99,7 +99,7 @@ entry:
 }
 
 ; Test case for uwtable
-define i32 @test4_nofpelim() uwtable "no-frame-pointer-elim"="true" {
+define i32 @test4_nofpelim() uwtable "frame-pointer"="all" {
 ; CHECK-LABEL: @test4_nofpelim
 ; CHECK: push
 entry:
@@ -107,7 +107,7 @@ entry:
   unreachable
 }
 
-define i32 @test5_nofpelim() uwtable "no-frame-pointer-elim"="true" {
+define i32 @test5_nofpelim() uwtable "frame-pointer"="all" {
 ; CHECK-LABEL: @test5_nofpelim
 ; CHECK: push
 entry:
diff --git a/llvm/test/CodeGen/ARM/out-of-registers.ll b/llvm/test/CodeGen/ARM/out-of-registers.ll
index 891319881..a6d9fe098 100644
--- a/llvm/test/CodeGen/ARM/out-of-registers.ll
+++ b/llvm/test/CodeGen/ARM/out-of-registers.ll
@@ -32,7 +32,7 @@ declare { <4 x float>, <4 x float>, <4 x float>, <4 x float> } @llvm.arm.neon.vl
 
 ; Function Attrs: nounwind
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="true" "no-nans-fp-math"="true" "stack-protector-buffer-size"="8" "unsafe-fp-math"="true" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="true" "no-nans-fp-math"="true" "stack-protector-buffer-size"="8" "unsafe-fp-math"="true" "use-soft-float"="false" }
 attributes #1 = { nounwind }
 attributes #2 = { nounwind readonly }
 
diff --git a/llvm/test/CodeGen/ARM/postrasched.ll b/llvm/test/CodeGen/ARM/postrasched.ll
new file mode 100644
index 000000000..85593d551
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/postrasched.ll
@@ -0,0 +1,30 @@
+; REQUIRES: asserts
+; RUN: llc < %s -mtriple=thumbv8m.main-none-eabi -debug-only=machine-scheduler,post-RA-sched -print-before=machine-scheduler -o - 2>&1 > /dev/null | FileCheck %s
+
+; CHECK-LABEL: test_misched
+; Pre and post ra machine scheduling
+; CHECK:  ********** MI Scheduling **********
+; CHECK:  t2LDRi12
+; CHECK:  Latency            : 2
+; CHECK:  ********** MI Scheduling **********
+; CHECK:  t2LDRi12
+; CHECK:  Latency            : 2
+
+define i32 @test_misched(i32* %ptr) "target-cpu"="cortex-m33" {
+entry:
+  %l = load i32, i32* %ptr
+  store i32 0, i32* %ptr
+  ret i32 %l
+}
+
+; CHECK-LABEL: test_rasched
+; CHECK: Subtarget disables post-MI-sched.
+; CHECK: ********** List Scheduling **********
+
+define i32 @test_rasched(i32* %ptr) {
+entry:
+  %l = load i32, i32* %ptr
+  store i32 0, i32* %ptr
+  ret i32 %l
+}
+
diff --git a/llvm/test/CodeGen/ARM/pow.ll b/llvm/test/CodeGen/ARM/pow.ll
index 2b3df92aa..8abc37de3 100644
--- a/llvm/test/CodeGen/ARM/pow.ll
+++ b/llvm/test/CodeGen/ARM/pow.ll
@@ -26,6 +26,22 @@ define double @pow_f64_one_fourth_fmf(double %x) nounwind {
   ret double %r
 }
 
+define float @pow_f32_one_third_fmf(float %x) nounwind {
+; ANY-LABEL: pow_f32_one_third_fmf:
+; SOFTFLOAT:    bl cbrtf
+; HARDFLOAT:    b cbrtf
+  %r = call fast float @llvm.pow.f32(float %x, float 0x3FD5555560000000)
+  ret float %r
+}
+
+define double @pow_f64_one_third_fmf(double %x) nounwind {
+; ANY-LABEL: pow_f64_one_third_fmf:
+; SOFTFLOAT:    bl cbrt
+; HARDFLOAT:    b cbrt
+  %r = call fast double @llvm.pow.f64(double %x, double 0x3FD5555555555555)
+  ret double %r
+}
+
 define <4 x float> @pow_v4f32_one_fourth_fmf(<4 x float> %x) nounwind {
 ; ANY-LABEL: pow_v4f32_one_fourth_fmf:
 ; SOFTFLOAT:    bl powf
diff --git a/llvm/test/CodeGen/ARM/prefetch.ll b/llvm/test/CodeGen/ARM/prefetch.ll
index 447411d51..f594be346 100644
--- a/llvm/test/CodeGen/ARM/prefetch.ll
+++ b/llvm/test/CodeGen/ARM/prefetch.ll
@@ -1,5 +1,6 @@
 ; RUN: llc -mtriple=thumb-eabi -mattr=-thumb2 %s -o - | FileCheck %s -check-prefix CHECK-T1
 ; RUN: llc -mtriple=thumb-eabi -mattr=+v7 %s -o - | FileCheck %s -check-prefix=THUMB2
+; RUN: llc -mtriple=thumb-eabi -mattr=+v7 -mattr=+mp %s -o - | FileCheck %s -check-prefix=THUMB2-MP
 ; RUN: llc -mtriple=arm-eabi -mattr=+v7 %s -o - | FileCheck %s -check-prefix=ARM
 ; RUN: llc -mtriple=arm-eabi -mcpu=cortex-a9 %s -o - | FileCheck %s -check-prefix=ARM-MP
 ; rdar://8601536
@@ -83,15 +84,63 @@ entry:
 ;ARM-LABEL: t6:
 ;ARM: pld [sp]
 ;ARM: pld [sp, #50]
+;ARM: pld [sp, #-50]
 
 ;THUMB2-LABEL: t6:
 ;THUMB2: pld [sp]
 ;THUMB2: pld [sp, #50]
+;THUMB2: pld [sp, #-50]
 
 %red = alloca [100 x i8], align 1
 %0 = getelementptr inbounds [100 x i8], [100 x i8]* %red, i32 0, i32 0
 %1 = getelementptr inbounds [100 x i8], [100 x i8]* %red, i32 0, i32 50
+%2 = getelementptr inbounds [100 x i8], [100 x i8]* %red, i32 0, i32 -50
 call void @llvm.prefetch(i8* %0, i32 0, i32 3, i32 1)
 call void @llvm.prefetch(i8* %1, i32 0, i32 3, i32 1)
+call void @llvm.prefetch(i8* %2, i32 0, i32 3, i32 1)
+ret void
+}
+
+define void @t7() {
+entry:
+;ARM-LABEL: t7:
+;ARM-MP: pldw [sp]
+;ARM-MP: pldw [sp, #50]
+;ARM-MP: pldw [sp, #-50]
+
+;THUMB2-MP-LABEL: t7:
+;THUMB2-MP: pldw [sp]
+;THUMB2-MP: pldw [sp, #50]
+;THUMB2-MP: pldw [sp, #-50]
+
+%red = alloca [100 x i8], align 1
+%0 = getelementptr inbounds [100 x i8], [100 x i8]* %red, i32 0, i32 0
+%1 = getelementptr inbounds [100 x i8], [100 x i8]* %red, i32 0, i32 50
+%2 = getelementptr inbounds [100 x i8], [100 x i8]* %red, i32 0, i32 -50
+call void @llvm.prefetch(i8* %0, i32 1, i32 3, i32 1)
+call void @llvm.prefetch(i8* %1, i32 1, i32 3, i32 1)
+call void @llvm.prefetch(i8* %2, i32 1, i32 3, i32 1)
+ret void
+}
+
+define void @t8() {
+entry:
+;ARM-LABEL: t8:
+;ARM: pli [sp]
+;ARM: pli [sp, #50]
+;ARM: pli [sp, #-50]
+
+;THUMB2-LABEL: t8:
+;THUMB2: pli [sp]
+;THUMB2: pli [sp, #50]
+;THUMB2: pli [sp, #-50]
+
+%red = alloca [100 x i8], align 1
+%0 = getelementptr inbounds [100 x i8], [100 x i8]* %red, i32 0, i32 0
+%1 = getelementptr inbounds [100 x i8], [100 x i8]* %red, i32 0, i32 50
+%2 = getelementptr inbounds [100 x i8], [100 x i8]* %red, i32 0, i32 -50
+call void @llvm.prefetch(i8* %0, i32 0, i32 3, i32 0)
+call void @llvm.prefetch(i8* %1, i32 0, i32 3, i32 0)
+call void @llvm.prefetch(i8* %2, i32 0, i32 3, i32 0)
 ret void
 }
diff --git a/llvm/test/CodeGen/ARM/qdadd.ll b/llvm/test/CodeGen/ARM/qdadd.ll
new file mode 100644
index 000000000..94442ca93
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/qdadd.ll
@@ -0,0 +1,186 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=thumbv7m-none-eabi | FileCheck %s --check-prefix=CHECK-T2 --check-prefix=CHECK-T2NODSP
+; RUN: llc < %s -mtriple=thumbv7em-none-eabi | FileCheck %s --check-prefix=CHECK-T2 --check-prefix=CHECK-T2DSP
+; RUN: llc < %s -mtriple=armv5te-none-none-eabi | FileCheck %s --check-prefix=CHECK-ARM --check-prefix=CHECK-ARM6
+; RUN: llc < %s -mtriple=armv8a-none-eabi | FileCheck %s --check-prefix=CHECK-ARM --check-prefix=CHECK-ARM8
+
+define i32 @qdadd(i32 %x, i32 %y) nounwind {
+; CHECK-T2NODSP-LABEL: qdadd:
+; CHECK-T2NODSP:       @ %bb.0:
+; CHECK-T2NODSP-NEXT:    .save {r7, lr}
+; CHECK-T2NODSP-NEXT:    push {r7, lr}
+; CHECK-T2NODSP-NEXT:    movs r3, #0
+; CHECK-T2NODSP-NEXT:    adds.w r12, r0, r0
+; CHECK-T2NODSP-NEXT:    it mi
+; CHECK-T2NODSP-NEXT:    movmi r3, #1
+; CHECK-T2NODSP-NEXT:    cmp r3, #0
+; CHECK-T2NODSP-NEXT:    mov.w r3, #-2147483648
+; CHECK-T2NODSP-NEXT:    mov.w lr, #0
+; CHECK-T2NODSP-NEXT:    it ne
+; CHECK-T2NODSP-NEXT:    mvnne r3, #-2147483648
+; CHECK-T2NODSP-NEXT:    cmp r12, r0
+; CHECK-T2NODSP-NEXT:    it vc
+; CHECK-T2NODSP-NEXT:    movvc r3, r12
+; CHECK-T2NODSP-NEXT:    adds r0, r3, r1
+; CHECK-T2NODSP-NEXT:    mov.w r2, #-2147483648
+; CHECK-T2NODSP-NEXT:    it mi
+; CHECK-T2NODSP-NEXT:    movmi.w lr, #1
+; CHECK-T2NODSP-NEXT:    cmp.w lr, #0
+; CHECK-T2NODSP-NEXT:    it ne
+; CHECK-T2NODSP-NEXT:    mvnne r2, #-2147483648
+; CHECK-T2NODSP-NEXT:    cmp r0, r3
+; CHECK-T2NODSP-NEXT:    it vc
+; CHECK-T2NODSP-NEXT:    movvc r2, r0
+; CHECK-T2NODSP-NEXT:    mov r0, r2
+; CHECK-T2NODSP-NEXT:    pop {r7, pc}
+;
+; CHECK-T2DSP-LABEL: qdadd:
+; CHECK-T2DSP:       @ %bb.0:
+; CHECK-T2DSP-NEXT:    qdadd r0, r0, r1
+; CHECK-T2DSP-NEXT:    bx lr
+;
+; CHECK-ARM-LABEL: qdadd:
+; CHECK-ARM:       @ %bb.0:
+; CHECK-ARM-NEXT:    qdadd r0, r0, r1
+; CHECK-ARM-NEXT:    bx lr
+  %z = call i32 @llvm.sadd.sat.i32(i32 %x, i32 %x)
+  %tmp = call i32 @llvm.sadd.sat.i32(i32 %z, i32 %y)
+  ret i32 %tmp
+}
+
+define i32 @qdadd_c(i32 %x, i32 %y) nounwind {
+; CHECK-T2NODSP-LABEL: qdadd_c:
+; CHECK-T2NODSP:       @ %bb.0:
+; CHECK-T2NODSP-NEXT:    .save {r7, lr}
+; CHECK-T2NODSP-NEXT:    push {r7, lr}
+; CHECK-T2NODSP-NEXT:    movs r3, #0
+; CHECK-T2NODSP-NEXT:    adds.w r12, r0, r0
+; CHECK-T2NODSP-NEXT:    it mi
+; CHECK-T2NODSP-NEXT:    movmi r3, #1
+; CHECK-T2NODSP-NEXT:    cmp r3, #0
+; CHECK-T2NODSP-NEXT:    mov.w r3, #-2147483648
+; CHECK-T2NODSP-NEXT:    mov.w lr, #0
+; CHECK-T2NODSP-NEXT:    it ne
+; CHECK-T2NODSP-NEXT:    mvnne r3, #-2147483648
+; CHECK-T2NODSP-NEXT:    cmp r12, r0
+; CHECK-T2NODSP-NEXT:    it vc
+; CHECK-T2NODSP-NEXT:    movvc r3, r12
+; CHECK-T2NODSP-NEXT:    adds r0, r1, r3
+; CHECK-T2NODSP-NEXT:    mov.w r2, #-2147483648
+; CHECK-T2NODSP-NEXT:    it mi
+; CHECK-T2NODSP-NEXT:    movmi.w lr, #1
+; CHECK-T2NODSP-NEXT:    cmp.w lr, #0
+; CHECK-T2NODSP-NEXT:    it ne
+; CHECK-T2NODSP-NEXT:    mvnne r2, #-2147483648
+; CHECK-T2NODSP-NEXT:    cmp r0, r1
+; CHECK-T2NODSP-NEXT:    it vc
+; CHECK-T2NODSP-NEXT:    movvc r2, r0
+; CHECK-T2NODSP-NEXT:    mov r0, r2
+; CHECK-T2NODSP-NEXT:    pop {r7, pc}
+;
+; CHECK-T2DSP-LABEL: qdadd_c:
+; CHECK-T2DSP:       @ %bb.0:
+; CHECK-T2DSP-NEXT:    qdadd r0, r0, r1
+; CHECK-T2DSP-NEXT:    bx lr
+;
+; CHECK-ARM-LABEL: qdadd_c:
+; CHECK-ARM:       @ %bb.0:
+; CHECK-ARM-NEXT:    qdadd r0, r0, r1
+; CHECK-ARM-NEXT:    bx lr
+  %z = call i32 @llvm.sadd.sat.i32(i32 %x, i32 %x)
+  %tmp = call i32 @llvm.sadd.sat.i32(i32 %y, i32 %z)
+  ret i32 %tmp
+}
+
+define i32 @qdsub(i32 %x, i32 %y) nounwind {
+; CHECK-T2NODSP-LABEL: qdsub:
+; CHECK-T2NODSP:       @ %bb.0:
+; CHECK-T2NODSP-NEXT:    .save {r7, lr}
+; CHECK-T2NODSP-NEXT:    push {r7, lr}
+; CHECK-T2NODSP-NEXT:    movs r3, #0
+; CHECK-T2NODSP-NEXT:    adds.w r12, r0, r0
+; CHECK-T2NODSP-NEXT:    it mi
+; CHECK-T2NODSP-NEXT:    movmi r3, #1
+; CHECK-T2NODSP-NEXT:    cmp r3, #0
+; CHECK-T2NODSP-NEXT:    mov.w r3, #-2147483648
+; CHECK-T2NODSP-NEXT:    mov.w lr, #0
+; CHECK-T2NODSP-NEXT:    it ne
+; CHECK-T2NODSP-NEXT:    mvnne r3, #-2147483648
+; CHECK-T2NODSP-NEXT:    cmp r12, r0
+; CHECK-T2NODSP-NEXT:    it vc
+; CHECK-T2NODSP-NEXT:    movvc r3, r12
+; CHECK-T2NODSP-NEXT:    subs r0, r1, r3
+; CHECK-T2NODSP-NEXT:    mov.w r2, #-2147483648
+; CHECK-T2NODSP-NEXT:    it mi
+; CHECK-T2NODSP-NEXT:    movmi.w lr, #1
+; CHECK-T2NODSP-NEXT:    cmp.w lr, #0
+; CHECK-T2NODSP-NEXT:    it ne
+; CHECK-T2NODSP-NEXT:    mvnne r2, #-2147483648
+; CHECK-T2NODSP-NEXT:    cmp r1, r3
+; CHECK-T2NODSP-NEXT:    it vc
+; CHECK-T2NODSP-NEXT:    movvc r2, r0
+; CHECK-T2NODSP-NEXT:    mov r0, r2
+; CHECK-T2NODSP-NEXT:    pop {r7, pc}
+;
+; CHECK-T2DSP-LABEL: qdsub:
+; CHECK-T2DSP:       @ %bb.0:
+; CHECK-T2DSP-NEXT:    qdsub r0, r1, r0
+; CHECK-T2DSP-NEXT:    bx lr
+;
+; CHECK-ARM-LABEL: qdsub:
+; CHECK-ARM:       @ %bb.0:
+; CHECK-ARM-NEXT:    qdsub r0, r1, r0
+; CHECK-ARM-NEXT:    bx lr
+  %z = call i32 @llvm.sadd.sat.i32(i32 %x, i32 %x)
+  %tmp = call i32 @llvm.ssub.sat.i32(i32 %y, i32 %z)
+  ret i32 %tmp
+}
+
+define i32 @qdsub_c(i32 %x, i32 %y) nounwind {
+; CHECK-T2NODSP-LABEL: qdsub_c:
+; CHECK-T2NODSP:       @ %bb.0:
+; CHECK-T2NODSP-NEXT:    .save {r7, lr}
+; CHECK-T2NODSP-NEXT:    push {r7, lr}
+; CHECK-T2NODSP-NEXT:    movs r3, #0
+; CHECK-T2NODSP-NEXT:    adds.w r12, r0, r0
+; CHECK-T2NODSP-NEXT:    it mi
+; CHECK-T2NODSP-NEXT:    movmi r3, #1
+; CHECK-T2NODSP-NEXT:    cmp r3, #0
+; CHECK-T2NODSP-NEXT:    mov.w r3, #-2147483648
+; CHECK-T2NODSP-NEXT:    mov.w lr, #0
+; CHECK-T2NODSP-NEXT:    it ne
+; CHECK-T2NODSP-NEXT:    mvnne r3, #-2147483648
+; CHECK-T2NODSP-NEXT:    cmp r12, r0
+; CHECK-T2NODSP-NEXT:    it vc
+; CHECK-T2NODSP-NEXT:    movvc r3, r12
+; CHECK-T2NODSP-NEXT:    subs r0, r3, r1
+; CHECK-T2NODSP-NEXT:    mov.w r2, #-2147483648
+; CHECK-T2NODSP-NEXT:    it mi
+; CHECK-T2NODSP-NEXT:    movmi.w lr, #1
+; CHECK-T2NODSP-NEXT:    cmp.w lr, #0
+; CHECK-T2NODSP-NEXT:    it ne
+; CHECK-T2NODSP-NEXT:    mvnne r2, #-2147483648
+; CHECK-T2NODSP-NEXT:    cmp r3, r1
+; CHECK-T2NODSP-NEXT:    it vc
+; CHECK-T2NODSP-NEXT:    movvc r2, r0
+; CHECK-T2NODSP-NEXT:    mov r0, r2
+; CHECK-T2NODSP-NEXT:    pop {r7, pc}
+;
+; CHECK-T2DSP-LABEL: qdsub_c:
+; CHECK-T2DSP:       @ %bb.0:
+; CHECK-T2DSP-NEXT:    qadd r0, r0, r0
+; CHECK-T2DSP-NEXT:    qsub r0, r0, r1
+; CHECK-T2DSP-NEXT:    bx lr
+;
+; CHECK-ARM-LABEL: qdsub_c:
+; CHECK-ARM:       @ %bb.0:
+; CHECK-ARM-NEXT:    qadd r0, r0, r0
+; CHECK-ARM-NEXT:    qsub r0, r0, r1
+; CHECK-ARM-NEXT:    bx lr
+  %z = call i32 @llvm.sadd.sat.i32(i32 %x, i32 %x)
+  %tmp = call i32 @llvm.ssub.sat.i32(i32 %z, i32 %y)
+  ret i32 %tmp
+}
+
+declare i32 @llvm.sadd.sat.i32(i32, i32)
+declare i32 @llvm.ssub.sat.i32(i32, i32)
diff --git a/llvm/test/CodeGen/ARM/rev.ll b/llvm/test/CodeGen/ARM/rev.ll
index a36526ff1..b97dbc844 100644
--- a/llvm/test/CodeGen/ARM/rev.ll
+++ b/llvm/test/CodeGen/ARM/rev.ll
@@ -1,8 +1,11 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=arm-eabi -mattr=+v6 %s -o - | FileCheck %s
 
 define i32 @test1(i32 %X) nounwind {
-; CHECK-LABEL: test1
-; CHECK: rev16 r0, r0
+; CHECK-LABEL: test1:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    rev16 r0, r0
+; CHECK-NEXT:    bx lr
         %tmp1 = lshr i32 %X, 8
         %X15 = bitcast i32 %X to i32
         %tmp4 = shl i32 %X15, 8
@@ -17,8 +20,10 @@ define i32 @test1(i32 %X) nounwind {
 }
 
 define i32 @test2(i32 %X) nounwind {
-; CHECK-LABEL: test2
-; CHECK: revsh r0, r0
+; CHECK-LABEL: test2:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    revsh r0, r0
+; CHECK-NEXT:    bx lr
         %tmp1 = lshr i32 %X, 8
         %tmp1.upgrd.1 = trunc i32 %tmp1 to i16
         %tmp3 = trunc i32 %X to i16
@@ -31,9 +36,11 @@ define i32 @test2(i32 %X) nounwind {
 
 ; rdar://9147637
 define i32 @test3(i16 zeroext %a) nounwind {
-entry:
 ; CHECK-LABEL: test3:
-; CHECK: revsh r0, r0
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    revsh r0, r0
+; CHECK-NEXT:    bx lr
+entry:
   %0 = tail call i16 @llvm.bswap.i16(i16 %a)
   %1 = sext i16 %0 to i32
   ret i32 %1
@@ -42,9 +49,11 @@ entry:
 declare i16 @llvm.bswap.i16(i16) nounwind readnone
 
 define i32 @test4(i16 zeroext %a) nounwind {
-entry:
 ; CHECK-LABEL: test4:
-; CHECK: revsh r0, r0
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    revsh r0, r0
+; CHECK-NEXT:    bx lr
+entry:
   %conv = zext i16 %a to i32
   %shr9 = lshr i16 %a, 8
   %conv2 = zext i16 %shr9 to i32
@@ -57,9 +66,11 @@ entry:
 
 ; rdar://9609059
 define i32 @test5(i32 %i) nounwind readnone {
+; CHECK-LABEL: test5:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    revsh r0, r0
+; CHECK-NEXT:    bx lr
 entry:
-; CHECK-LABEL: test5
-; CHECK: revsh r0, r0
   %shl = shl i32 %i, 24
   %shr = ashr exact i32 %shl, 16
   %shr23 = lshr i32 %i, 8
@@ -70,9 +81,11 @@ entry:
 
 ; rdar://9609108
 define i32 @test6(i32 %x) nounwind readnone {
+; CHECK-LABEL: test6:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    rev16 r0, r0
+; CHECK-NEXT:    bx lr
 entry:
-; CHECK-LABEL: test6
-; CHECK: rev16 r0, r0
   %and = shl i32 %x, 8
   %shl = and i32 %and, 65280
   %and2 = lshr i32 %x, 8
@@ -87,10 +100,12 @@ entry:
 
 ; rdar://9164521
 define i32 @test7(i32 %a) nounwind readnone {
+; CHECK-LABEL: test7:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    rev r0, r0
+; CHECK-NEXT:    lsr r0, r0, #16
+; CHECK-NEXT:    bx lr
 entry:
-; CHECK-LABEL: test7
-; CHECK: rev r0, r0
-; CHECK: lsr r0, r0, #16
   %and = lshr i32 %a, 8
   %shr3 = and i32 %and, 255
   %and2 = shl i32 %a, 8
@@ -100,9 +115,11 @@ entry:
 }
 
 define i32 @test8(i32 %a) nounwind readnone {
+; CHECK-LABEL: test8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    revsh r0, r0
+; CHECK-NEXT:    bx lr
 entry:
-; CHECK-LABEL: test8
-; CHECK: revsh r0, r0
   %and = lshr i32 %a, 8
   %shr4 = and i32 %and, 255
   %and2 = shl i32 %a, 8
@@ -114,9 +131,11 @@ entry:
 
 ; rdar://10750814
 define zeroext i16 @test9(i16 zeroext %v) nounwind readnone {
+; CHECK-LABEL: test9:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    rev16 r0, r0
+; CHECK-NEXT:    bx lr
 entry:
-; CHECK-LABEL: test9
-; CHECK: rev16 r0, r0
   %conv = zext i16 %v to i32
   %shr4 = lshr i32 %conv, 8
   %shl = shl nuw nsw i32 %conv, 8
diff --git a/llvm/test/CodeGen/ARM/ror.ll b/llvm/test/CodeGen/ARM/ror.ll
index 0f699a8dd..7a8c5fd1e 100644
--- a/llvm/test/CodeGen/ARM/ror.ll
+++ b/llvm/test/CodeGen/ARM/ror.ll
@@ -1,10 +1,12 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=arm-eabi -mattr=+v6 %s -o - | FileCheck %s
 
 ; rotr (rotr x, 4), 6 -> rotr x, 10 -> ror r0, r0, #10
 define i32 @test1(i32 %x) nounwind readnone {
 ; CHECK-LABEL: test1:
-; CHECK: ror  r0, r0, #10
-; CHECK: bx  lr
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    ror r0, r0, #10
+; CHECK-NEXT:    bx lr
 entry:
   %high_part.i = shl i32 %x, 28
   %low_part.i = lshr i32 %x, 4
@@ -18,9 +20,10 @@ entry:
 ; the same vector test
 define <2 x i32> @test2(<2 x i32> %x) nounwind readnone {
 ; CHECK-LABEL: test2:
-; CHECK: ror  r0, r0, #10
-; CHECK: ror  r1, r1, #10
-; CHECK: bx  lr
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    ror r0, r0, #10
+; CHECK-NEXT:    ror r1, r1, #10
+; CHECK-NEXT:    bx lr
 entry:
   %high_part.i = shl <2 x i32> %x, <i32 28, i32 28>
   %low_part.i = lshr <2 x i32> %x, <i32 4, i32 4>
diff --git a/llvm/test/CodeGen/ARM/sadd_sat.ll b/llvm/test/CodeGen/ARM/sadd_sat.ll
new file mode 100644
index 000000000..386e750dd
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/sadd_sat.ll
@@ -0,0 +1,428 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=thumbv6m-none-eabi | FileCheck %s --check-prefix=CHECK-T1
+; RUN: llc < %s -mtriple=thumbv7m-none-eabi | FileCheck %s --check-prefix=CHECK-T2 --check-prefix=CHECK-T2NODSP
+; RUN: llc < %s -mtriple=thumbv7em-none-eabi | FileCheck %s --check-prefix=CHECK-T2 --check-prefix=CHECK-T2DSP
+; RUN: llc < %s -mtriple=armv5t-none-eabi | FileCheck %s --check-prefix=CHECK-ARM --check-prefix=CHECK-ARMNODPS
+; RUN: llc < %s -mtriple=armv5te-none-eabi | FileCheck %s --check-prefix=CHECK-ARM --check-prefix=CHECK-ARMBASEDSP
+; RUN: llc < %s -mtriple=armv6-none-eabi | FileCheck %s --check-prefix=CHECK-ARM --check-prefix=CHECK-ARMDSP
+
+declare i4 @llvm.sadd.sat.i4(i4, i4)
+declare i8 @llvm.sadd.sat.i8(i8, i8)
+declare i16 @llvm.sadd.sat.i16(i16, i16)
+declare i32 @llvm.sadd.sat.i32(i32, i32)
+declare i64 @llvm.sadd.sat.i64(i64, i64)
+
+define i32 @func(i32 %x, i32 %y) nounwind {
+; CHECK-T1-LABEL: func:
+; CHECK-T1:       @ %bb.0:
+; CHECK-T1-NEXT:    mov r2, r0
+; CHECK-T1-NEXT:    movs r3, #1
+; CHECK-T1-NEXT:    adds r0, r0, r1
+; CHECK-T1-NEXT:    mov r1, r3
+; CHECK-T1-NEXT:    bmi .LBB0_2
+; CHECK-T1-NEXT:  @ %bb.1:
+; CHECK-T1-NEXT:    movs r1, #0
+; CHECK-T1-NEXT:  .LBB0_2:
+; CHECK-T1-NEXT:    cmp r1, #0
+; CHECK-T1-NEXT:    bne .LBB0_4
+; CHECK-T1-NEXT:  @ %bb.3:
+; CHECK-T1-NEXT:    lsls r1, r3, #31
+; CHECK-T1-NEXT:    cmp r0, r2
+; CHECK-T1-NEXT:    bvs .LBB0_5
+; CHECK-T1-NEXT:    b .LBB0_6
+; CHECK-T1-NEXT:  .LBB0_4:
+; CHECK-T1-NEXT:    ldr r1, .LCPI0_0
+; CHECK-T1-NEXT:    cmp r0, r2
+; CHECK-T1-NEXT:    bvc .LBB0_6
+; CHECK-T1-NEXT:  .LBB0_5:
+; CHECK-T1-NEXT:    mov r0, r1
+; CHECK-T1-NEXT:  .LBB0_6:
+; CHECK-T1-NEXT:    bx lr
+; CHECK-T1-NEXT:    .p2align 2
+; CHECK-T1-NEXT:  @ %bb.7:
+; CHECK-T1-NEXT:  .LCPI0_0:
+; CHECK-T1-NEXT:    .long 2147483647 @ 0x7fffffff
+;
+; CHECK-T2NODSP-LABEL: func:
+; CHECK-T2NODSP:       @ %bb.0:
+; CHECK-T2NODSP-NEXT:    adds r2, r0, r1
+; CHECK-T2NODSP-NEXT:    mov.w r3, #0
+; CHECK-T2NODSP-NEXT:    mov.w r1, #-2147483648
+; CHECK-T2NODSP-NEXT:    it mi
+; CHECK-T2NODSP-NEXT:    movmi r3, #1
+; CHECK-T2NODSP-NEXT:    cmp r3, #0
+; CHECK-T2NODSP-NEXT:    it ne
+; CHECK-T2NODSP-NEXT:    mvnne r1, #-2147483648
+; CHECK-T2NODSP-NEXT:    cmp r2, r0
+; CHECK-T2NODSP-NEXT:    it vc
+; CHECK-T2NODSP-NEXT:    movvc r1, r2
+; CHECK-T2NODSP-NEXT:    mov r0, r1
+; CHECK-T2NODSP-NEXT:    bx lr
+;
+; CHECK-T2DSP-LABEL: func:
+; CHECK-T2DSP:       @ %bb.0:
+; CHECK-T2DSP-NEXT:    qadd r0, r0, r1
+; CHECK-T2DSP-NEXT:    bx lr
+;
+; CHECK-ARMNODPS-LABEL: func:
+; CHECK-ARMNODPS:       @ %bb.0:
+; CHECK-ARMNODPS-NEXT:    adds r2, r0, r1
+; CHECK-ARMNODPS-NEXT:    mov r3, #0
+; CHECK-ARMNODPS-NEXT:    movmi r3, #1
+; CHECK-ARMNODPS-NEXT:    mov r1, #-2147483648
+; CHECK-ARMNODPS-NEXT:    cmp r3, #0
+; CHECK-ARMNODPS-NEXT:    mvnne r1, #-2147483648
+; CHECK-ARMNODPS-NEXT:    cmp r2, r0
+; CHECK-ARMNODPS-NEXT:    movvc r1, r2
+; CHECK-ARMNODPS-NEXT:    mov r0, r1
+; CHECK-ARMNODPS-NEXT:    bx lr
+;
+; CHECK-ARMBASEDSP-LABEL: func:
+; CHECK-ARMBASEDSP:       @ %bb.0:
+; CHECK-ARMBASEDSP-NEXT:    qadd r0, r0, r1
+; CHECK-ARMBASEDSP-NEXT:    bx lr
+;
+; CHECK-ARMDSP-LABEL: func:
+; CHECK-ARMDSP:       @ %bb.0:
+; CHECK-ARMDSP-NEXT:    qadd r0, r0, r1
+; CHECK-ARMDSP-NEXT:    bx lr
+  %tmp = call i32 @llvm.sadd.sat.i32(i32 %x, i32 %y)
+  ret i32 %tmp
+}
+
+define i64 @func2(i64 %x, i64 %y) nounwind {
+; CHECK-T1-LABEL: func2:
+; CHECK-T1:       @ %bb.0:
+; CHECK-T1-NEXT:    .save {r4, r5, r6, r7, lr}
+; CHECK-T1-NEXT:    push {r4, r5, r6, r7, lr}
+; CHECK-T1-NEXT:    .pad #4
+; CHECK-T1-NEXT:    sub sp, #4
+; CHECK-T1-NEXT:    str r2, [sp] @ 4-byte Spill
+; CHECK-T1-NEXT:    mov r2, r0
+; CHECK-T1-NEXT:    movs r4, #1
+; CHECK-T1-NEXT:    movs r0, #0
+; CHECK-T1-NEXT:    cmp r3, #0
+; CHECK-T1-NEXT:    mov r5, r4
+; CHECK-T1-NEXT:    bge .LBB1_2
+; CHECK-T1-NEXT:  @ %bb.1:
+; CHECK-T1-NEXT:    mov r5, r0
+; CHECK-T1-NEXT:  .LBB1_2:
+; CHECK-T1-NEXT:    cmp r1, #0
+; CHECK-T1-NEXT:    mov r7, r4
+; CHECK-T1-NEXT:    bge .LBB1_4
+; CHECK-T1-NEXT:  @ %bb.3:
+; CHECK-T1-NEXT:    mov r7, r0
+; CHECK-T1-NEXT:  .LBB1_4:
+; CHECK-T1-NEXT:    subs r6, r7, r5
+; CHECK-T1-NEXT:    rsbs r5, r6, #0
+; CHECK-T1-NEXT:    adcs r5, r6
+; CHECK-T1-NEXT:    ldr r6, [sp] @ 4-byte Reload
+; CHECK-T1-NEXT:    adds r6, r2, r6
+; CHECK-T1-NEXT:    adcs r1, r3
+; CHECK-T1-NEXT:    cmp r1, #0
+; CHECK-T1-NEXT:    mov r2, r4
+; CHECK-T1-NEXT:    bge .LBB1_6
+; CHECK-T1-NEXT:  @ %bb.5:
+; CHECK-T1-NEXT:    mov r2, r0
+; CHECK-T1-NEXT:  .LBB1_6:
+; CHECK-T1-NEXT:    subs r0, r7, r2
+; CHECK-T1-NEXT:    subs r2, r0, #1
+; CHECK-T1-NEXT:    sbcs r0, r2
+; CHECK-T1-NEXT:    ands r5, r0
+; CHECK-T1-NEXT:    beq .LBB1_8
+; CHECK-T1-NEXT:  @ %bb.7:
+; CHECK-T1-NEXT:    asrs r6, r1, #31
+; CHECK-T1-NEXT:  .LBB1_8:
+; CHECK-T1-NEXT:    cmp r1, #0
+; CHECK-T1-NEXT:    bmi .LBB1_10
+; CHECK-T1-NEXT:  @ %bb.9:
+; CHECK-T1-NEXT:    lsls r2, r4, #31
+; CHECK-T1-NEXT:    cmp r5, #0
+; CHECK-T1-NEXT:    beq .LBB1_11
+; CHECK-T1-NEXT:    b .LBB1_12
+; CHECK-T1-NEXT:  .LBB1_10:
+; CHECK-T1-NEXT:    ldr r2, .LCPI1_0
+; CHECK-T1-NEXT:    cmp r5, #0
+; CHECK-T1-NEXT:    bne .LBB1_12
+; CHECK-T1-NEXT:  .LBB1_11:
+; CHECK-T1-NEXT:    mov r2, r1
+; CHECK-T1-NEXT:  .LBB1_12:
+; CHECK-T1-NEXT:    mov r0, r6
+; CHECK-T1-NEXT:    mov r1, r2
+; CHECK-T1-NEXT:    add sp, #4
+; CHECK-T1-NEXT:    pop {r4, r5, r6, r7, pc}
+; CHECK-T1-NEXT:    .p2align 2
+; CHECK-T1-NEXT:  @ %bb.13:
+; CHECK-T1-NEXT:  .LCPI1_0:
+; CHECK-T1-NEXT:    .long 2147483647 @ 0x7fffffff
+;
+; CHECK-T2-LABEL: func2:
+; CHECK-T2:       @ %bb.0:
+; CHECK-T2-NEXT:    .save {r7, lr}
+; CHECK-T2-NEXT:    push {r7, lr}
+; CHECK-T2-NEXT:    cmp.w r1, #-1
+; CHECK-T2-NEXT:    mov.w lr, #0
+; CHECK-T2-NEXT:    it gt
+; CHECK-T2-NEXT:    movgt.w lr, #1
+; CHECK-T2-NEXT:    adds r0, r0, r2
+; CHECK-T2-NEXT:    adc.w r2, r1, r3
+; CHECK-T2-NEXT:    movs r1, #0
+; CHECK-T2-NEXT:    cmp.w r2, #-1
+; CHECK-T2-NEXT:    it gt
+; CHECK-T2-NEXT:    movgt r1, #1
+; CHECK-T2-NEXT:    subs.w r1, lr, r1
+; CHECK-T2-NEXT:    mov.w r12, #0
+; CHECK-T2-NEXT:    it ne
+; CHECK-T2-NEXT:    movne r1, #1
+; CHECK-T2-NEXT:    cmp.w r3, #-1
+; CHECK-T2-NEXT:    it gt
+; CHECK-T2-NEXT:    movgt.w r12, #1
+; CHECK-T2-NEXT:    sub.w r3, lr, r12
+; CHECK-T2-NEXT:    clz r3, r3
+; CHECK-T2-NEXT:    lsrs r3, r3, #5
+; CHECK-T2-NEXT:    ands r3, r1
+; CHECK-T2-NEXT:    mov.w r1, #-2147483648
+; CHECK-T2-NEXT:    it ne
+; CHECK-T2-NEXT:    asrne r0, r2, #31
+; CHECK-T2-NEXT:    cmp r2, #0
+; CHECK-T2-NEXT:    it mi
+; CHECK-T2-NEXT:    mvnmi r1, #-2147483648
+; CHECK-T2-NEXT:    cmp r3, #0
+; CHECK-T2-NEXT:    it eq
+; CHECK-T2-NEXT:    moveq r1, r2
+; CHECK-T2-NEXT:    pop {r7, pc}
+;
+; CHECK-ARM-LABEL: func2:
+; CHECK-ARM:       @ %bb.0:
+; CHECK-ARM-NEXT:    .save {r11, lr}
+; CHECK-ARM-NEXT:    push {r11, lr}
+; CHECK-ARM-NEXT:    cmn r1, #1
+; CHECK-ARM-NEXT:    mov lr, #0
+; CHECK-ARM-NEXT:    movgt lr, #1
+; CHECK-ARM-NEXT:    adds r0, r0, r2
+; CHECK-ARM-NEXT:    adc r2, r1, r3
+; CHECK-ARM-NEXT:    mov r1, #0
+; CHECK-ARM-NEXT:    cmn r2, #1
+; CHECK-ARM-NEXT:    mov r12, #0
+; CHECK-ARM-NEXT:    movgt r1, #1
+; CHECK-ARM-NEXT:    subs r1, lr, r1
+; CHECK-ARM-NEXT:    movne r1, #1
+; CHECK-ARM-NEXT:    cmn r3, #1
+; CHECK-ARM-NEXT:    movgt r12, #1
+; CHECK-ARM-NEXT:    sub r3, lr, r12
+; CHECK-ARM-NEXT:    clz r3, r3
+; CHECK-ARM-NEXT:    lsr r3, r3, #5
+; CHECK-ARM-NEXT:    ands r3, r3, r1
+; CHECK-ARM-NEXT:    mov r1, #-2147483648
+; CHECK-ARM-NEXT:    asrne r0, r2, #31
+; CHECK-ARM-NEXT:    cmp r2, #0
+; CHECK-ARM-NEXT:    mvnmi r1, #-2147483648
+; CHECK-ARM-NEXT:    cmp r3, #0
+; CHECK-ARM-NEXT:    moveq r1, r2
+; CHECK-ARM-NEXT:    pop {r11, pc}
+  %tmp = call i64 @llvm.sadd.sat.i64(i64 %x, i64 %y)
+  ret i64 %tmp
+}
+
+define signext i16 @func16(i16 signext %x, i16 signext %y) nounwind {
+; CHECK-T1-LABEL: func16:
+; CHECK-T1:       @ %bb.0:
+; CHECK-T1-NEXT:    adds r0, r0, r1
+; CHECK-T1-NEXT:    ldr r1, .LCPI2_0
+; CHECK-T1-NEXT:    cmp r0, r1
+; CHECK-T1-NEXT:    blt .LBB2_2
+; CHECK-T1-NEXT:  @ %bb.1:
+; CHECK-T1-NEXT:    mov r0, r1
+; CHECK-T1-NEXT:  .LBB2_2:
+; CHECK-T1-NEXT:    ldr r1, .LCPI2_1
+; CHECK-T1-NEXT:    cmp r0, r1
+; CHECK-T1-NEXT:    bgt .LBB2_4
+; CHECK-T1-NEXT:  @ %bb.3:
+; CHECK-T1-NEXT:    mov r0, r1
+; CHECK-T1-NEXT:  .LBB2_4:
+; CHECK-T1-NEXT:    bx lr
+; CHECK-T1-NEXT:    .p2align 2
+; CHECK-T1-NEXT:  @ %bb.5:
+; CHECK-T1-NEXT:  .LCPI2_0:
+; CHECK-T1-NEXT:    .long 32767 @ 0x7fff
+; CHECK-T1-NEXT:  .LCPI2_1:
+; CHECK-T1-NEXT:    .long 4294934528 @ 0xffff8000
+;
+; CHECK-T2NODSP-LABEL: func16:
+; CHECK-T2NODSP:       @ %bb.0:
+; CHECK-T2NODSP-NEXT:    add r0, r1
+; CHECK-T2NODSP-NEXT:    movw r1, #32767
+; CHECK-T2NODSP-NEXT:    cmp r0, r1
+; CHECK-T2NODSP-NEXT:    it lt
+; CHECK-T2NODSP-NEXT:    movlt r1, r0
+; CHECK-T2NODSP-NEXT:    movw r0, #32768
+; CHECK-T2NODSP-NEXT:    cmn.w r1, #32768
+; CHECK-T2NODSP-NEXT:    movt r0, #65535
+; CHECK-T2NODSP-NEXT:    it gt
+; CHECK-T2NODSP-NEXT:    movgt r0, r1
+; CHECK-T2NODSP-NEXT:    bx lr
+;
+; CHECK-T2DSP-LABEL: func16:
+; CHECK-T2DSP:       @ %bb.0:
+; CHECK-T2DSP-NEXT:    qadd16 r0, r0, r1
+; CHECK-T2DSP-NEXT:    sxth r0, r0
+; CHECK-T2DSP-NEXT:    bx lr
+;
+; CHECK-ARMNODPS-LABEL: func16:
+; CHECK-ARMNODPS:       @ %bb.0:
+; CHECK-ARMNODPS-NEXT:    add r0, r0, r1
+; CHECK-ARMNODPS-NEXT:    mov r1, #255
+; CHECK-ARMNODPS-NEXT:    orr r1, r1, #32512
+; CHECK-ARMNODPS-NEXT:    cmp r0, r1
+; CHECK-ARMNODPS-NEXT:    movlt r1, r0
+; CHECK-ARMNODPS-NEXT:    ldr r0, .LCPI2_0
+; CHECK-ARMNODPS-NEXT:    cmn r1, #32768
+; CHECK-ARMNODPS-NEXT:    movgt r0, r1
+; CHECK-ARMNODPS-NEXT:    bx lr
+; CHECK-ARMNODPS-NEXT:    .p2align 2
+; CHECK-ARMNODPS-NEXT:  @ %bb.1:
+; CHECK-ARMNODPS-NEXT:  .LCPI2_0:
+; CHECK-ARMNODPS-NEXT:    .long 4294934528 @ 0xffff8000
+;
+; CHECK-ARMBASEDSP-LABEL: func16:
+; CHECK-ARMBASEDSP:       @ %bb.0:
+; CHECK-ARMBASEDSP-NEXT:    lsl r0, r0, #16
+; CHECK-ARMBASEDSP-NEXT:    lsl r1, r1, #16
+; CHECK-ARMBASEDSP-NEXT:    qadd r0, r0, r1
+; CHECK-ARMBASEDSP-NEXT:    asr r0, r0, #16
+; CHECK-ARMBASEDSP-NEXT:    bx lr
+;
+; CHECK-ARMDSP-LABEL: func16:
+; CHECK-ARMDSP:       @ %bb.0:
+; CHECK-ARMDSP-NEXT:    qadd16 r0, r0, r1
+; CHECK-ARMDSP-NEXT:    sxth r0, r0
+; CHECK-ARMDSP-NEXT:    bx lr
+  %tmp = call i16 @llvm.sadd.sat.i16(i16 %x, i16 %y)
+  ret i16 %tmp
+}
+
+define signext i8 @func8(i8 signext %x, i8 signext %y) nounwind {
+; CHECK-T1-LABEL: func8:
+; CHECK-T1:       @ %bb.0:
+; CHECK-T1-NEXT:    adds r0, r0, r1
+; CHECK-T1-NEXT:    movs r1, #127
+; CHECK-T1-NEXT:    cmp r0, #127
+; CHECK-T1-NEXT:    blt .LBB3_2
+; CHECK-T1-NEXT:  @ %bb.1:
+; CHECK-T1-NEXT:    mov r0, r1
+; CHECK-T1-NEXT:  .LBB3_2:
+; CHECK-T1-NEXT:    mvns r1, r1
+; CHECK-T1-NEXT:    cmp r0, r1
+; CHECK-T1-NEXT:    bgt .LBB3_4
+; CHECK-T1-NEXT:  @ %bb.3:
+; CHECK-T1-NEXT:    mov r0, r1
+; CHECK-T1-NEXT:  .LBB3_4:
+; CHECK-T1-NEXT:    bx lr
+;
+; CHECK-T2NODSP-LABEL: func8:
+; CHECK-T2NODSP:       @ %bb.0:
+; CHECK-T2NODSP-NEXT:    add r0, r1
+; CHECK-T2NODSP-NEXT:    cmp r0, #127
+; CHECK-T2NODSP-NEXT:    it ge
+; CHECK-T2NODSP-NEXT:    movge r0, #127
+; CHECK-T2NODSP-NEXT:    cmn.w r0, #128
+; CHECK-T2NODSP-NEXT:    it le
+; CHECK-T2NODSP-NEXT:    mvnle r0, #127
+; CHECK-T2NODSP-NEXT:    bx lr
+;
+; CHECK-T2DSP-LABEL: func8:
+; CHECK-T2DSP:       @ %bb.0:
+; CHECK-T2DSP-NEXT:    qadd8 r0, r0, r1
+; CHECK-T2DSP-NEXT:    sxtb r0, r0
+; CHECK-T2DSP-NEXT:    bx lr
+;
+; CHECK-ARMNODPS-LABEL: func8:
+; CHECK-ARMNODPS:       @ %bb.0:
+; CHECK-ARMNODPS-NEXT:    add r0, r0, r1
+; CHECK-ARMNODPS-NEXT:    cmp r0, #127
+; CHECK-ARMNODPS-NEXT:    movge r0, #127
+; CHECK-ARMNODPS-NEXT:    cmn r0, #128
+; CHECK-ARMNODPS-NEXT:    mvnle r0, #127
+; CHECK-ARMNODPS-NEXT:    bx lr
+;
+; CHECK-ARMBASEDSP-LABEL: func8:
+; CHECK-ARMBASEDSP:       @ %bb.0:
+; CHECK-ARMBASEDSP-NEXT:    lsl r0, r0, #24
+; CHECK-ARMBASEDSP-NEXT:    lsl r1, r1, #24
+; CHECK-ARMBASEDSP-NEXT:    qadd r0, r0, r1
+; CHECK-ARMBASEDSP-NEXT:    asr r0, r0, #24
+; CHECK-ARMBASEDSP-NEXT:    bx lr
+;
+; CHECK-ARMDSP-LABEL: func8:
+; CHECK-ARMDSP:       @ %bb.0:
+; CHECK-ARMDSP-NEXT:    qadd8 r0, r0, r1
+; CHECK-ARMDSP-NEXT:    sxtb r0, r0
+; CHECK-ARMDSP-NEXT:    bx lr
+  %tmp = call i8 @llvm.sadd.sat.i8(i8 %x, i8 %y)
+  ret i8 %tmp
+}
+
+define signext i4 @func3(i4 signext %x, i4 signext %y) nounwind {
+; CHECK-T1-LABEL: func3:
+; CHECK-T1:       @ %bb.0:
+; CHECK-T1-NEXT:    adds r0, r0, r1
+; CHECK-T1-NEXT:    movs r1, #7
+; CHECK-T1-NEXT:    cmp r0, #7
+; CHECK-T1-NEXT:    blt .LBB4_2
+; CHECK-T1-NEXT:  @ %bb.1:
+; CHECK-T1-NEXT:    mov r0, r1
+; CHECK-T1-NEXT:  .LBB4_2:
+; CHECK-T1-NEXT:    mvns r1, r1
+; CHECK-T1-NEXT:    cmp r0, r1
+; CHECK-T1-NEXT:    bgt .LBB4_4
+; CHECK-T1-NEXT:  @ %bb.3:
+; CHECK-T1-NEXT:    mov r0, r1
+; CHECK-T1-NEXT:  .LBB4_4:
+; CHECK-T1-NEXT:    bx lr
+;
+; CHECK-T2NODSP-LABEL: func3:
+; CHECK-T2NODSP:       @ %bb.0:
+; CHECK-T2NODSP-NEXT:    add r0, r1
+; CHECK-T2NODSP-NEXT:    cmp r0, #7
+; CHECK-T2NODSP-NEXT:    it ge
+; CHECK-T2NODSP-NEXT:    movge r0, #7
+; CHECK-T2NODSP-NEXT:    cmn.w r0, #8
+; CHECK-T2NODSP-NEXT:    it le
+; CHECK-T2NODSP-NEXT:    mvnle r0, #7
+; CHECK-T2NODSP-NEXT:    bx lr
+;
+; CHECK-T2DSP-LABEL: func3:
+; CHECK-T2DSP:       @ %bb.0:
+; CHECK-T2DSP-NEXT:    lsls r1, r1, #28
+; CHECK-T2DSP-NEXT:    lsls r0, r0, #28
+; CHECK-T2DSP-NEXT:    qadd r0, r0, r1
+; CHECK-T2DSP-NEXT:    asrs r0, r0, #28
+; CHECK-T2DSP-NEXT:    bx lr
+;
+; CHECK-ARMNODPS-LABEL: func3:
+; CHECK-ARMNODPS:       @ %bb.0:
+; CHECK-ARMNODPS-NEXT:    add r0, r0, r1
+; CHECK-ARMNODPS-NEXT:    cmp r0, #7
+; CHECK-ARMNODPS-NEXT:    movge r0, #7
+; CHECK-ARMNODPS-NEXT:    cmn r0, #8
+; CHECK-ARMNODPS-NEXT:    mvnle r0, #7
+; CHECK-ARMNODPS-NEXT:    bx lr
+;
+; CHECK-ARMBASEDSP-LABEL: func3:
+; CHECK-ARMBASEDSP:       @ %bb.0:
+; CHECK-ARMBASEDSP-NEXT:    lsl r0, r0, #28
+; CHECK-ARMBASEDSP-NEXT:    lsl r1, r1, #28
+; CHECK-ARMBASEDSP-NEXT:    qadd r0, r0, r1
+; CHECK-ARMBASEDSP-NEXT:    asr r0, r0, #28
+; CHECK-ARMBASEDSP-NEXT:    bx lr
+;
+; CHECK-ARMDSP-LABEL: func3:
+; CHECK-ARMDSP:       @ %bb.0:
+; CHECK-ARMDSP-NEXT:    lsl r0, r0, #28
+; CHECK-ARMDSP-NEXT:    lsl r1, r1, #28
+; CHECK-ARMDSP-NEXT:    qadd r0, r0, r1
+; CHECK-ARMDSP-NEXT:    asr r0, r0, #28
+; CHECK-ARMDSP-NEXT:    bx lr
+  %tmp = call i4 @llvm.sadd.sat.i4(i4 %x, i4 %y)
+  ret i4 %tmp
+}
diff --git a/llvm/test/CodeGen/ARM/sadd_sat_plus.ll b/llvm/test/CodeGen/ARM/sadd_sat_plus.ll
new file mode 100644
index 000000000..041506816
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/sadd_sat_plus.ll
@@ -0,0 +1,379 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=thumbv6m-none-eabi | FileCheck %s --check-prefix=CHECK-T1
+; RUN: llc < %s -mtriple=thumbv7m-none-eabi | FileCheck %s --check-prefix=CHECK-T2 --check-prefix=CHECK-T2NODSP
+; RUN: llc < %s -mtriple=thumbv7em-none-eabi | FileCheck %s --check-prefix=CHECK-T2 --check-prefix=CHECK-T2DSP
+; RUN: llc < %s -mtriple=armv8a-none-eabi | FileCheck %s --check-prefix=CHECK-ARM
+
+declare i4 @llvm.sadd.sat.i4(i4, i4)
+declare i8 @llvm.sadd.sat.i8(i8, i8)
+declare i16 @llvm.sadd.sat.i16(i16, i16)
+declare i32 @llvm.sadd.sat.i32(i32, i32)
+declare i64 @llvm.sadd.sat.i64(i64, i64)
+
+define i32 @func32(i32 %x, i32 %y, i32 %z) nounwind {
+; CHECK-T1-LABEL: func32:
+; CHECK-T1:       @ %bb.0:
+; CHECK-T1-NEXT:    mov r3, r0
+; CHECK-T1-NEXT:    muls r1, r2, r1
+; CHECK-T1-NEXT:    movs r2, #1
+; CHECK-T1-NEXT:    adds r0, r0, r1
+; CHECK-T1-NEXT:    mov r1, r2
+; CHECK-T1-NEXT:    bmi .LBB0_2
+; CHECK-T1-NEXT:  @ %bb.1:
+; CHECK-T1-NEXT:    movs r1, #0
+; CHECK-T1-NEXT:  .LBB0_2:
+; CHECK-T1-NEXT:    cmp r1, #0
+; CHECK-T1-NEXT:    bne .LBB0_4
+; CHECK-T1-NEXT:  @ %bb.3:
+; CHECK-T1-NEXT:    lsls r1, r2, #31
+; CHECK-T1-NEXT:    cmp r0, r3
+; CHECK-T1-NEXT:    bvs .LBB0_5
+; CHECK-T1-NEXT:    b .LBB0_6
+; CHECK-T1-NEXT:  .LBB0_4:
+; CHECK-T1-NEXT:    ldr r1, .LCPI0_0
+; CHECK-T1-NEXT:    cmp r0, r3
+; CHECK-T1-NEXT:    bvc .LBB0_6
+; CHECK-T1-NEXT:  .LBB0_5:
+; CHECK-T1-NEXT:    mov r0, r1
+; CHECK-T1-NEXT:  .LBB0_6:
+; CHECK-T1-NEXT:    bx lr
+; CHECK-T1-NEXT:    .p2align 2
+; CHECK-T1-NEXT:  @ %bb.7:
+; CHECK-T1-NEXT:  .LCPI0_0:
+; CHECK-T1-NEXT:    .long 2147483647 @ 0x7fffffff
+;
+; CHECK-T2NODSP-LABEL: func32:
+; CHECK-T2NODSP:       @ %bb.0:
+; CHECK-T2NODSP-NEXT:    mla r2, r1, r2, r0
+; CHECK-T2NODSP-NEXT:    movs r3, #0
+; CHECK-T2NODSP-NEXT:    mov.w r1, #-2147483648
+; CHECK-T2NODSP-NEXT:    cmp r2, #0
+; CHECK-T2NODSP-NEXT:    it mi
+; CHECK-T2NODSP-NEXT:    movmi r3, #1
+; CHECK-T2NODSP-NEXT:    cmp r3, #0
+; CHECK-T2NODSP-NEXT:    it ne
+; CHECK-T2NODSP-NEXT:    mvnne r1, #-2147483648
+; CHECK-T2NODSP-NEXT:    cmp r2, r0
+; CHECK-T2NODSP-NEXT:    it vc
+; CHECK-T2NODSP-NEXT:    movvc r1, r2
+; CHECK-T2NODSP-NEXT:    mov r0, r1
+; CHECK-T2NODSP-NEXT:    bx lr
+;
+; CHECK-T2DSP-LABEL: func32:
+; CHECK-T2DSP:       @ %bb.0:
+; CHECK-T2DSP-NEXT:    muls r1, r2, r1
+; CHECK-T2DSP-NEXT:    qadd r0, r0, r1
+; CHECK-T2DSP-NEXT:    bx lr
+;
+; CHECK-ARM-LABEL: func32:
+; CHECK-ARM:       @ %bb.0:
+; CHECK-ARM-NEXT:    mul r1, r1, r2
+; CHECK-ARM-NEXT:    qadd r0, r0, r1
+; CHECK-ARM-NEXT:    bx lr
+  %a = mul i32 %y, %z
+  %tmp = call i32 @llvm.sadd.sat.i32(i32 %x, i32 %a)
+  ret i32 %tmp
+}
+
+define i64 @func64(i64 %x, i64 %y, i64 %z) nounwind {
+; CHECK-T1-LABEL: func64:
+; CHECK-T1:       @ %bb.0:
+; CHECK-T1-NEXT:    .save {r4, r5, r6, r7, lr}
+; CHECK-T1-NEXT:    push {r4, r5, r6, r7, lr}
+; CHECK-T1-NEXT:    .pad #4
+; CHECK-T1-NEXT:    sub sp, #4
+; CHECK-T1-NEXT:    ldr r5, [sp, #28]
+; CHECK-T1-NEXT:    movs r2, #1
+; CHECK-T1-NEXT:    movs r4, #0
+; CHECK-T1-NEXT:    cmp r5, #0
+; CHECK-T1-NEXT:    mov r3, r2
+; CHECK-T1-NEXT:    bge .LBB1_2
+; CHECK-T1-NEXT:  @ %bb.1:
+; CHECK-T1-NEXT:    mov r3, r4
+; CHECK-T1-NEXT:  .LBB1_2:
+; CHECK-T1-NEXT:    cmp r1, #0
+; CHECK-T1-NEXT:    mov r6, r2
+; CHECK-T1-NEXT:    bge .LBB1_4
+; CHECK-T1-NEXT:  @ %bb.3:
+; CHECK-T1-NEXT:    mov r6, r4
+; CHECK-T1-NEXT:  .LBB1_4:
+; CHECK-T1-NEXT:    subs r7, r6, r3
+; CHECK-T1-NEXT:    rsbs r3, r7, #0
+; CHECK-T1-NEXT:    adcs r3, r7
+; CHECK-T1-NEXT:    ldr r7, [sp, #24]
+; CHECK-T1-NEXT:    adds r0, r0, r7
+; CHECK-T1-NEXT:    adcs r1, r5
+; CHECK-T1-NEXT:    cmp r1, #0
+; CHECK-T1-NEXT:    mov r5, r2
+; CHECK-T1-NEXT:    bge .LBB1_6
+; CHECK-T1-NEXT:  @ %bb.5:
+; CHECK-T1-NEXT:    mov r5, r4
+; CHECK-T1-NEXT:  .LBB1_6:
+; CHECK-T1-NEXT:    subs r4, r6, r5
+; CHECK-T1-NEXT:    subs r5, r4, #1
+; CHECK-T1-NEXT:    sbcs r4, r5
+; CHECK-T1-NEXT:    ands r3, r4
+; CHECK-T1-NEXT:    beq .LBB1_8
+; CHECK-T1-NEXT:  @ %bb.7:
+; CHECK-T1-NEXT:    asrs r0, r1, #31
+; CHECK-T1-NEXT:  .LBB1_8:
+; CHECK-T1-NEXT:    cmp r1, #0
+; CHECK-T1-NEXT:    bmi .LBB1_10
+; CHECK-T1-NEXT:  @ %bb.9:
+; CHECK-T1-NEXT:    lsls r2, r2, #31
+; CHECK-T1-NEXT:    cmp r3, #0
+; CHECK-T1-NEXT:    beq .LBB1_11
+; CHECK-T1-NEXT:    b .LBB1_12
+; CHECK-T1-NEXT:  .LBB1_10:
+; CHECK-T1-NEXT:    ldr r2, .LCPI1_0
+; CHECK-T1-NEXT:    cmp r3, #0
+; CHECK-T1-NEXT:    bne .LBB1_12
+; CHECK-T1-NEXT:  .LBB1_11:
+; CHECK-T1-NEXT:    mov r2, r1
+; CHECK-T1-NEXT:  .LBB1_12:
+; CHECK-T1-NEXT:    mov r1, r2
+; CHECK-T1-NEXT:    add sp, #4
+; CHECK-T1-NEXT:    pop {r4, r5, r6, r7, pc}
+; CHECK-T1-NEXT:    .p2align 2
+; CHECK-T1-NEXT:  @ %bb.13:
+; CHECK-T1-NEXT:  .LCPI1_0:
+; CHECK-T1-NEXT:    .long 2147483647 @ 0x7fffffff
+;
+; CHECK-T2-LABEL: func64:
+; CHECK-T2:       @ %bb.0:
+; CHECK-T2-NEXT:    .save {r7, lr}
+; CHECK-T2-NEXT:    push {r7, lr}
+; CHECK-T2-NEXT:    ldrd r2, r12, [sp, #8]
+; CHECK-T2-NEXT:    cmp.w r1, #-1
+; CHECK-T2-NEXT:    mov.w r3, #0
+; CHECK-T2-NEXT:    mov.w lr, #0
+; CHECK-T2-NEXT:    it gt
+; CHECK-T2-NEXT:    movgt r3, #1
+; CHECK-T2-NEXT:    adds r0, r0, r2
+; CHECK-T2-NEXT:    adc.w r2, r1, r12
+; CHECK-T2-NEXT:    movs r1, #0
+; CHECK-T2-NEXT:    cmp.w r2, #-1
+; CHECK-T2-NEXT:    it gt
+; CHECK-T2-NEXT:    movgt r1, #1
+; CHECK-T2-NEXT:    subs r1, r3, r1
+; CHECK-T2-NEXT:    it ne
+; CHECK-T2-NEXT:    movne r1, #1
+; CHECK-T2-NEXT:    cmp.w r12, #-1
+; CHECK-T2-NEXT:    it gt
+; CHECK-T2-NEXT:    movgt.w lr, #1
+; CHECK-T2-NEXT:    sub.w r3, r3, lr
+; CHECK-T2-NEXT:    clz r3, r3
+; CHECK-T2-NEXT:    lsrs r3, r3, #5
+; CHECK-T2-NEXT:    ands r3, r1
+; CHECK-T2-NEXT:    mov.w r1, #-2147483648
+; CHECK-T2-NEXT:    it ne
+; CHECK-T2-NEXT:    asrne r0, r2, #31
+; CHECK-T2-NEXT:    cmp r2, #0
+; CHECK-T2-NEXT:    it mi
+; CHECK-T2-NEXT:    mvnmi r1, #-2147483648
+; CHECK-T2-NEXT:    cmp r3, #0
+; CHECK-T2-NEXT:    it eq
+; CHECK-T2-NEXT:    moveq r1, r2
+; CHECK-T2-NEXT:    pop {r7, pc}
+;
+; CHECK-ARM-LABEL: func64:
+; CHECK-ARM:       @ %bb.0:
+; CHECK-ARM-NEXT:    .save {r11, lr}
+; CHECK-ARM-NEXT:    push {r11, lr}
+; CHECK-ARM-NEXT:    ldr r2, [sp, #8]
+; CHECK-ARM-NEXT:    mov r3, #0
+; CHECK-ARM-NEXT:    ldr r12, [sp, #12]
+; CHECK-ARM-NEXT:    adds r0, r0, r2
+; CHECK-ARM-NEXT:    mov r2, #0
+; CHECK-ARM-NEXT:    adc lr, r1, r12
+; CHECK-ARM-NEXT:    cmn r1, #1
+; CHECK-ARM-NEXT:    mov r1, #0
+; CHECK-ARM-NEXT:    movwgt r1, #1
+; CHECK-ARM-NEXT:    cmn lr, #1
+; CHECK-ARM-NEXT:    movwgt r2, #1
+; CHECK-ARM-NEXT:    subs r2, r1, r2
+; CHECK-ARM-NEXT:    movwne r2, #1
+; CHECK-ARM-NEXT:    cmn r12, #1
+; CHECK-ARM-NEXT:    movwgt r3, #1
+; CHECK-ARM-NEXT:    sub r1, r1, r3
+; CHECK-ARM-NEXT:    clz r1, r1
+; CHECK-ARM-NEXT:    lsr r1, r1, #5
+; CHECK-ARM-NEXT:    ands r2, r1, r2
+; CHECK-ARM-NEXT:    asrne r0, lr, #31
+; CHECK-ARM-NEXT:    mov r1, #-2147483648
+; CHECK-ARM-NEXT:    cmp lr, #0
+; CHECK-ARM-NEXT:    mvnmi r1, #-2147483648
+; CHECK-ARM-NEXT:    cmp r2, #0
+; CHECK-ARM-NEXT:    moveq r1, lr
+; CHECK-ARM-NEXT:    pop {r11, pc}
+  %a = mul i64 %y, %z
+  %tmp = call i64 @llvm.sadd.sat.i64(i64 %x, i64 %z)
+  ret i64 %tmp
+}
+
+define signext i16 @func16(i16 signext %x, i16 signext %y, i16 signext %z) nounwind {
+; CHECK-T1-LABEL: func16:
+; CHECK-T1:       @ %bb.0:
+; CHECK-T1-NEXT:    muls r1, r2, r1
+; CHECK-T1-NEXT:    sxth r1, r1
+; CHECK-T1-NEXT:    adds r0, r0, r1
+; CHECK-T1-NEXT:    ldr r1, .LCPI2_0
+; CHECK-T1-NEXT:    cmp r0, r1
+; CHECK-T1-NEXT:    blt .LBB2_2
+; CHECK-T1-NEXT:  @ %bb.1:
+; CHECK-T1-NEXT:    mov r0, r1
+; CHECK-T1-NEXT:  .LBB2_2:
+; CHECK-T1-NEXT:    ldr r1, .LCPI2_1
+; CHECK-T1-NEXT:    cmp r0, r1
+; CHECK-T1-NEXT:    bgt .LBB2_4
+; CHECK-T1-NEXT:  @ %bb.3:
+; CHECK-T1-NEXT:    mov r0, r1
+; CHECK-T1-NEXT:  .LBB2_4:
+; CHECK-T1-NEXT:    bx lr
+; CHECK-T1-NEXT:    .p2align 2
+; CHECK-T1-NEXT:  @ %bb.5:
+; CHECK-T1-NEXT:  .LCPI2_0:
+; CHECK-T1-NEXT:    .long 32767 @ 0x7fff
+; CHECK-T1-NEXT:  .LCPI2_1:
+; CHECK-T1-NEXT:    .long 4294934528 @ 0xffff8000
+;
+; CHECK-T2NODSP-LABEL: func16:
+; CHECK-T2NODSP:       @ %bb.0:
+; CHECK-T2NODSP-NEXT:    muls r1, r2, r1
+; CHECK-T2NODSP-NEXT:    sxth r1, r1
+; CHECK-T2NODSP-NEXT:    add r0, r1
+; CHECK-T2NODSP-NEXT:    movw r1, #32767
+; CHECK-T2NODSP-NEXT:    cmp r0, r1
+; CHECK-T2NODSP-NEXT:    it lt
+; CHECK-T2NODSP-NEXT:    movlt r1, r0
+; CHECK-T2NODSP-NEXT:    movw r0, #32768
+; CHECK-T2NODSP-NEXT:    movt r0, #65535
+; CHECK-T2NODSP-NEXT:    cmn.w r1, #32768
+; CHECK-T2NODSP-NEXT:    it gt
+; CHECK-T2NODSP-NEXT:    movgt r0, r1
+; CHECK-T2NODSP-NEXT:    bx lr
+;
+; CHECK-T2DSP-LABEL: func16:
+; CHECK-T2DSP:       @ %bb.0:
+; CHECK-T2DSP-NEXT:    muls r1, r2, r1
+; CHECK-T2DSP-NEXT:    qadd16 r0, r0, r1
+; CHECK-T2DSP-NEXT:    sxth r0, r0
+; CHECK-T2DSP-NEXT:    bx lr
+;
+; CHECK-ARM-LABEL: func16:
+; CHECK-ARM:       @ %bb.0:
+; CHECK-ARM-NEXT:    smulbb r1, r1, r2
+; CHECK-ARM-NEXT:    qadd16 r0, r0, r1
+; CHECK-ARM-NEXT:    sxth r0, r0
+; CHECK-ARM-NEXT:    bx lr
+  %a = mul i16 %y, %z
+  %tmp = call i16 @llvm.sadd.sat.i16(i16 %x, i16 %a)
+  ret i16 %tmp
+}
+
+define signext i8 @func8(i8 signext %x, i8 signext %y, i8 signext %z) nounwind {
+; CHECK-T1-LABEL: func8:
+; CHECK-T1:       @ %bb.0:
+; CHECK-T1-NEXT:    muls r1, r2, r1
+; CHECK-T1-NEXT:    sxtb r1, r1
+; CHECK-T1-NEXT:    adds r0, r0, r1
+; CHECK-T1-NEXT:    movs r1, #127
+; CHECK-T1-NEXT:    cmp r0, #127
+; CHECK-T1-NEXT:    blt .LBB3_2
+; CHECK-T1-NEXT:  @ %bb.1:
+; CHECK-T1-NEXT:    mov r0, r1
+; CHECK-T1-NEXT:  .LBB3_2:
+; CHECK-T1-NEXT:    mvns r1, r1
+; CHECK-T1-NEXT:    cmp r0, r1
+; CHECK-T1-NEXT:    bgt .LBB3_4
+; CHECK-T1-NEXT:  @ %bb.3:
+; CHECK-T1-NEXT:    mov r0, r1
+; CHECK-T1-NEXT:  .LBB3_4:
+; CHECK-T1-NEXT:    bx lr
+;
+; CHECK-T2NODSP-LABEL: func8:
+; CHECK-T2NODSP:       @ %bb.0:
+; CHECK-T2NODSP-NEXT:    muls r1, r2, r1
+; CHECK-T2NODSP-NEXT:    sxtb r1, r1
+; CHECK-T2NODSP-NEXT:    add r0, r1
+; CHECK-T2NODSP-NEXT:    cmp r0, #127
+; CHECK-T2NODSP-NEXT:    it ge
+; CHECK-T2NODSP-NEXT:    movge r0, #127
+; CHECK-T2NODSP-NEXT:    cmn.w r0, #128
+; CHECK-T2NODSP-NEXT:    it le
+; CHECK-T2NODSP-NEXT:    mvnle r0, #127
+; CHECK-T2NODSP-NEXT:    bx lr
+;
+; CHECK-T2DSP-LABEL: func8:
+; CHECK-T2DSP:       @ %bb.0:
+; CHECK-T2DSP-NEXT:    muls r1, r2, r1
+; CHECK-T2DSP-NEXT:    qadd8 r0, r0, r1
+; CHECK-T2DSP-NEXT:    sxtb r0, r0
+; CHECK-T2DSP-NEXT:    bx lr
+;
+; CHECK-ARM-LABEL: func8:
+; CHECK-ARM:       @ %bb.0:
+; CHECK-ARM-NEXT:    smulbb r1, r1, r2
+; CHECK-ARM-NEXT:    qadd8 r0, r0, r1
+; CHECK-ARM-NEXT:    sxtb r0, r0
+; CHECK-ARM-NEXT:    bx lr
+  %a = mul i8 %y, %z
+  %tmp = call i8 @llvm.sadd.sat.i8(i8 %x, i8 %a)
+  ret i8 %tmp
+}
+
+define signext i4 @func4(i4 signext %x, i4 signext %y, i4 signext %z) nounwind {
+; CHECK-T1-LABEL: func4:
+; CHECK-T1:       @ %bb.0:
+; CHECK-T1-NEXT:    muls r1, r2, r1
+; CHECK-T1-NEXT:    lsls r1, r1, #28
+; CHECK-T1-NEXT:    asrs r1, r1, #28
+; CHECK-T1-NEXT:    adds r0, r0, r1
+; CHECK-T1-NEXT:    movs r1, #7
+; CHECK-T1-NEXT:    cmp r0, #7
+; CHECK-T1-NEXT:    blt .LBB4_2
+; CHECK-T1-NEXT:  @ %bb.1:
+; CHECK-T1-NEXT:    mov r0, r1
+; CHECK-T1-NEXT:  .LBB4_2:
+; CHECK-T1-NEXT:    mvns r1, r1
+; CHECK-T1-NEXT:    cmp r0, r1
+; CHECK-T1-NEXT:    bgt .LBB4_4
+; CHECK-T1-NEXT:  @ %bb.3:
+; CHECK-T1-NEXT:    mov r0, r1
+; CHECK-T1-NEXT:  .LBB4_4:
+; CHECK-T1-NEXT:    bx lr
+;
+; CHECK-T2NODSP-LABEL: func4:
+; CHECK-T2NODSP:       @ %bb.0:
+; CHECK-T2NODSP-NEXT:    muls r1, r2, r1
+; CHECK-T2NODSP-NEXT:    lsls r1, r1, #28
+; CHECK-T2NODSP-NEXT:    add.w r0, r0, r1, asr #28
+; CHECK-T2NODSP-NEXT:    cmp r0, #7
+; CHECK-T2NODSP-NEXT:    it ge
+; CHECK-T2NODSP-NEXT:    movge r0, #7
+; CHECK-T2NODSP-NEXT:    cmn.w r0, #8
+; CHECK-T2NODSP-NEXT:    it le
+; CHECK-T2NODSP-NEXT:    mvnle r0, #7
+; CHECK-T2NODSP-NEXT:    bx lr
+;
+; CHECK-T2DSP-LABEL: func4:
+; CHECK-T2DSP:       @ %bb.0:
+; CHECK-T2DSP-NEXT:    muls r1, r2, r1
+; CHECK-T2DSP-NEXT:    lsls r0, r0, #28
+; CHECK-T2DSP-NEXT:    lsls r1, r1, #28
+; CHECK-T2DSP-NEXT:    qadd r0, r0, r1
+; CHECK-T2DSP-NEXT:    asrs r0, r0, #28
+; CHECK-T2DSP-NEXT:    bx lr
+;
+; CHECK-ARM-LABEL: func4:
+; CHECK-ARM:       @ %bb.0:
+; CHECK-ARM-NEXT:    smulbb r1, r1, r2
+; CHECK-ARM-NEXT:    lsl r0, r0, #28
+; CHECK-ARM-NEXT:    lsl r1, r1, #28
+; CHECK-ARM-NEXT:    qadd r0, r0, r1
+; CHECK-ARM-NEXT:    asr r0, r0, #28
+; CHECK-ARM-NEXT:    bx lr
+  %a = mul i4 %y, %z
+  %tmp = call i4 @llvm.sadd.sat.i4(i4 %x, i4 %a)
+  ret i4 %tmp
+}
diff --git a/llvm/test/CodeGen/ARM/select.ll b/llvm/test/CodeGen/ARM/select.ll
index 45c32665e..dbec6dd57 100644
--- a/llvm/test/CodeGen/ARM/select.ll
+++ b/llvm/test/CodeGen/ARM/select.ll
@@ -143,11 +143,11 @@ define float @f12(i32 %a, i32 %b) nounwind uwtable readnone ssp {
 }
 
 ; CHECK-LABEL: test_overflow_recombine:
-define i1 @test_overflow_recombine(i32 %in) {
+define i1 @test_overflow_recombine(i32 %in1, i32 %in2) {
 ; CHECK: smull [[LO:r[0-9]+]], [[HI:r[0-9]+]]
 ; CHECK: subs [[ZERO:r[0-9]+]], [[HI]], [[LO]], asr #31
 ; CHECK: movne [[ZERO]], #1
-  %prod = call { i32, i1 } @llvm.smul.with.overflow.i32(i32 0, i32 %in)
+  %prod = call { i32, i1 } @llvm.smul.with.overflow.i32(i32 %in1, i32 %in2)
   %overflow = extractvalue { i32, i1 } %prod, 1
   ret i1 %overflow
 }
diff --git a/llvm/test/CodeGen/ARM/select_const.ll b/llvm/test/CodeGen/ARM/select_const.ll
index 81b0db3b3..500426074 100644
--- a/llvm/test/CodeGen/ARM/select_const.ll
+++ b/llvm/test/CodeGen/ARM/select_const.ll
@@ -1,5 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=arm-eabi-unknown-unknown | FileCheck %s
+; RUN: llc < %s -mtriple=arm-eabi-unknown-unknown | FileCheck %s --check-prefix=ARM
+; RUN: llc < %s -mtriple=thumbv6t2-eabi-unknown-unknown | FileCheck %s --check-prefix=THUMB2
+; RUN: llc < %s -mtriple=thumb-eabi-unknown-unknown | FileCheck %s --check-prefix=THUMB
 
 ; Select of constants: control flow / conditional moves can always be replaced by logic+math (but may not be worth it?).
 ; Test the zeroext/signext variants of each pattern to see if that makes a difference.
@@ -7,30 +9,67 @@
 ; select Cond, 0, 1 --> zext (!Cond)
 
 define i32 @select_0_or_1(i1 %cond) {
-; CHECK-LABEL: select_0_or_1:
-; CHECK:       @ %bb.0:
-; CHECK-NEXT:    mov r1, #1
-; CHECK-NEXT:    bic r0, r1, r0
-; CHECK-NEXT:    mov pc, lr
+; ARM-LABEL: select_0_or_1:
+; ARM:       @ %bb.0:
+; ARM-NEXT:    mov r1, #1
+; ARM-NEXT:    bic r0, r1, r0
+; ARM-NEXT:    mov pc, lr
+;
+; THUMB2-LABEL: select_0_or_1:
+; THUMB2:       @ %bb.0:
+; THUMB2-NEXT:    movs r1, #1
+; THUMB2-NEXT:    bic.w r0, r1, r0
+; THUMB2-NEXT:    bx lr
+;
+; THUMB-LABEL: select_0_or_1:
+; THUMB:       @ %bb.0:
+; THUMB-NEXT:    movs r1, #1
+; THUMB-NEXT:    bics r1, r0
+; THUMB-NEXT:    movs r0, r1
+; THUMB-NEXT:    bx lr
   %sel = select i1 %cond, i32 0, i32 1
   ret i32 %sel
 }
 
 define i32 @select_0_or_1_zeroext(i1 zeroext %cond) {
-; CHECK-LABEL: select_0_or_1_zeroext:
-; CHECK:       @ %bb.0:
-; CHECK-NEXT:    eor r0, r0, #1
-; CHECK-NEXT:    mov pc, lr
+; ARM-LABEL: select_0_or_1_zeroext:
+; ARM:       @ %bb.0:
+; ARM-NEXT:    eor r0, r0, #1
+; ARM-NEXT:    mov pc, lr
+;
+; THUMB2-LABEL: select_0_or_1_zeroext:
+; THUMB2:       @ %bb.0:
+; THUMB2-NEXT:    eor r0, r0, #1
+; THUMB2-NEXT:    bx lr
+;
+; THUMB-LABEL: select_0_or_1_zeroext:
+; THUMB:       @ %bb.0:
+; THUMB-NEXT:    movs r1, #1
+; THUMB-NEXT:    eors r0, r1
+; THUMB-NEXT:    bx lr
   %sel = select i1 %cond, i32 0, i32 1
   ret i32 %sel
 }
 
 define i32 @select_0_or_1_signext(i1 signext %cond) {
-; CHECK-LABEL: select_0_or_1_signext:
-; CHECK:       @ %bb.0:
-; CHECK-NEXT:    mov r1, #1
-; CHECK-NEXT:    bic r0, r1, r0
-; CHECK-NEXT:    mov pc, lr
+; ARM-LABEL: select_0_or_1_signext:
+; ARM:       @ %bb.0:
+; ARM-NEXT:    mov r1, #1
+; ARM-NEXT:    bic r0, r1, r0
+; ARM-NEXT:    mov pc, lr
+;
+; THUMB2-LABEL: select_0_or_1_signext:
+; THUMB2:       @ %bb.0:
+; THUMB2-NEXT:    movs r1, #1
+; THUMB2-NEXT:    bic.w r0, r1, r0
+; THUMB2-NEXT:    bx lr
+;
+; THUMB-LABEL: select_0_or_1_signext:
+; THUMB:       @ %bb.0:
+; THUMB-NEXT:    movs r1, #1
+; THUMB-NEXT:    bics r1, r0
+; THUMB-NEXT:    movs r0, r1
+; THUMB-NEXT:    bx lr
   %sel = select i1 %cond, i32 0, i32 1
   ret i32 %sel
 }
@@ -38,27 +77,57 @@ define i32 @select_0_or_1_signext(i1 signext %cond) {
 ; select Cond, 1, 0 --> zext (Cond)
 
 define i32 @select_1_or_0(i1 %cond) {
-; CHECK-LABEL: select_1_or_0:
-; CHECK:       @ %bb.0:
-; CHECK-NEXT:    and r0, r0, #1
-; CHECK-NEXT:    mov pc, lr
+; ARM-LABEL: select_1_or_0:
+; ARM:       @ %bb.0:
+; ARM-NEXT:    and r0, r0, #1
+; ARM-NEXT:    mov pc, lr
+;
+; THUMB2-LABEL: select_1_or_0:
+; THUMB2:       @ %bb.0:
+; THUMB2-NEXT:    and r0, r0, #1
+; THUMB2-NEXT:    bx lr
+;
+; THUMB-LABEL: select_1_or_0:
+; THUMB:       @ %bb.0:
+; THUMB-NEXT:    movs r1, #1
+; THUMB-NEXT:    ands r0, r1
+; THUMB-NEXT:    bx lr
   %sel = select i1 %cond, i32 1, i32 0
   ret i32 %sel
 }
 
 define i32 @select_1_or_0_zeroext(i1 zeroext %cond) {
-; CHECK-LABEL: select_1_or_0_zeroext:
-; CHECK:       @ %bb.0:
-; CHECK-NEXT:    mov pc, lr
+; ARM-LABEL: select_1_or_0_zeroext:
+; ARM:       @ %bb.0:
+; ARM-NEXT:    mov pc, lr
+;
+; THUMB2-LABEL: select_1_or_0_zeroext:
+; THUMB2:       @ %bb.0:
+; THUMB2-NEXT:    bx lr
+;
+; THUMB-LABEL: select_1_or_0_zeroext:
+; THUMB:       @ %bb.0:
+; THUMB-NEXT:    bx lr
   %sel = select i1 %cond, i32 1, i32 0
   ret i32 %sel
 }
 
 define i32 @select_1_or_0_signext(i1 signext %cond) {
-; CHECK-LABEL: select_1_or_0_signext:
-; CHECK:       @ %bb.0:
-; CHECK-NEXT:    and r0, r0, #1
-; CHECK-NEXT:    mov pc, lr
+; ARM-LABEL: select_1_or_0_signext:
+; ARM:       @ %bb.0:
+; ARM-NEXT:    and r0, r0, #1
+; ARM-NEXT:    mov pc, lr
+;
+; THUMB2-LABEL: select_1_or_0_signext:
+; THUMB2:       @ %bb.0:
+; THUMB2-NEXT:    and r0, r0, #1
+; THUMB2-NEXT:    bx lr
+;
+; THUMB-LABEL: select_1_or_0_signext:
+; THUMB:       @ %bb.0:
+; THUMB-NEXT:    movs r1, #1
+; THUMB-NEXT:    ands r0, r1
+; THUMB-NEXT:    bx lr
   %sel = select i1 %cond, i32 1, i32 0
   ret i32 %sel
 }
@@ -66,61 +135,131 @@ define i32 @select_1_or_0_signext(i1 signext %cond) {
 ; select Cond, 0, -1 --> sext (!Cond)
 
 define i32 @select_0_or_neg1(i1 %cond) {
-; CHECK-LABEL: select_0_or_neg1:
-; CHECK:       @ %bb.0:
-; CHECK-NEXT:    mov r1, #1
-; CHECK-NEXT:    bic r0, r1, r0
-; CHECK-NEXT:    rsb r0, r0, #0
-; CHECK-NEXT:    mov pc, lr
+; ARM-LABEL: select_0_or_neg1:
+; ARM:       @ %bb.0:
+; ARM-NEXT:    mov r1, #1
+; ARM-NEXT:    bic r0, r1, r0
+; ARM-NEXT:    rsb r0, r0, #0
+; ARM-NEXT:    mov pc, lr
+;
+; THUMB2-LABEL: select_0_or_neg1:
+; THUMB2:       @ %bb.0:
+; THUMB2-NEXT:    movs r1, #1
+; THUMB2-NEXT:    bic.w r0, r1, r0
+; THUMB2-NEXT:    rsbs r0, r0, #0
+; THUMB2-NEXT:    bx lr
+;
+; THUMB-LABEL: select_0_or_neg1:
+; THUMB:       @ %bb.0:
+; THUMB-NEXT:    movs r1, #1
+; THUMB-NEXT:    bics r1, r0
+; THUMB-NEXT:    rsbs r0, r1, #0
+; THUMB-NEXT:    bx lr
   %sel = select i1 %cond, i32 0, i32 -1
   ret i32 %sel
 }
 
 define i32 @select_0_or_neg1_zeroext(i1 zeroext %cond) {
-; CHECK-LABEL: select_0_or_neg1_zeroext:
-; CHECK:       @ %bb.0:
-; CHECK-NEXT:    eor r0, r0, #1
-; CHECK-NEXT:    rsb r0, r0, #0
-; CHECK-NEXT:    mov pc, lr
+; ARM-LABEL: select_0_or_neg1_zeroext:
+; ARM:       @ %bb.0:
+; ARM-NEXT:    eor r0, r0, #1
+; ARM-NEXT:    rsb r0, r0, #0
+; ARM-NEXT:    mov pc, lr
+;
+; THUMB2-LABEL: select_0_or_neg1_zeroext:
+; THUMB2:       @ %bb.0:
+; THUMB2-NEXT:    eor r0, r0, #1
+; THUMB2-NEXT:    rsbs r0, r0, #0
+; THUMB2-NEXT:    bx lr
+;
+; THUMB-LABEL: select_0_or_neg1_zeroext:
+; THUMB:       @ %bb.0:
+; THUMB-NEXT:    movs r1, #1
+; THUMB-NEXT:    eors r1, r0
+; THUMB-NEXT:    rsbs r0, r1, #0
+; THUMB-NEXT:    bx lr
   %sel = select i1 %cond, i32 0, i32 -1
   ret i32 %sel
 }
 
 define i32 @select_0_or_neg1_signext(i1 signext %cond) {
-; CHECK-LABEL: select_0_or_neg1_signext:
-; CHECK:       @ %bb.0:
-; CHECK-NEXT:    mvn r0, r0
-; CHECK-NEXT:    mov pc, lr
+; ARM-LABEL: select_0_or_neg1_signext:
+; ARM:       @ %bb.0:
+; ARM-NEXT:    mvn r0, r0
+; ARM-NEXT:    mov pc, lr
+;
+; THUMB2-LABEL: select_0_or_neg1_signext:
+; THUMB2:       @ %bb.0:
+; THUMB2-NEXT:    mvns r0, r0
+; THUMB2-NEXT:    bx lr
+;
+; THUMB-LABEL: select_0_or_neg1_signext:
+; THUMB:       @ %bb.0:
+; THUMB-NEXT:    mvns r0, r0
+; THUMB-NEXT:    bx lr
   %sel = select i1 %cond, i32 0, i32 -1
   ret i32 %sel
 }
 
 define i32 @select_0_or_neg1_alt(i1 %cond) {
-; CHECK-LABEL: select_0_or_neg1_alt:
-; CHECK:       @ %bb.0:
-; CHECK-NEXT:    and r0, r0, #1
-; CHECK-NEXT:    sub r0, r0, #1
-; CHECK-NEXT:    mov pc, lr
+; ARM-LABEL: select_0_or_neg1_alt:
+; ARM:       @ %bb.0:
+; ARM-NEXT:    and r0, r0, #1
+; ARM-NEXT:    sub r0, r0, #1
+; ARM-NEXT:    mov pc, lr
+;
+; THUMB2-LABEL: select_0_or_neg1_alt:
+; THUMB2:       @ %bb.0:
+; THUMB2-NEXT:    and r0, r0, #1
+; THUMB2-NEXT:    subs r0, #1
+; THUMB2-NEXT:    bx lr
+;
+; THUMB-LABEL: select_0_or_neg1_alt:
+; THUMB:       @ %bb.0:
+; THUMB-NEXT:    movs r1, #1
+; THUMB-NEXT:    ands r1, r0
+; THUMB-NEXT:    subs r0, r1, #1
+; THUMB-NEXT:    bx lr
   %z = zext i1 %cond to i32
   %add = add i32 %z, -1
   ret i32 %add
 }
 
 define i32 @select_0_or_neg1_alt_zeroext(i1 zeroext %cond) {
-; CHECK-LABEL: select_0_or_neg1_alt_zeroext:
-; CHECK:       @ %bb.0:
-; CHECK-NEXT:    sub r0, r0, #1
-; CHECK-NEXT:    mov pc, lr
+; ARM-LABEL: select_0_or_neg1_alt_zeroext:
+; ARM:       @ %bb.0:
+; ARM-NEXT:    sub r0, r0, #1
+; ARM-NEXT:    mov pc, lr
+;
+; THUMB2-LABEL: select_0_or_neg1_alt_zeroext:
+; THUMB2:       @ %bb.0:
+; THUMB2-NEXT:    subs r0, #1
+; THUMB2-NEXT:    bx lr
+;
+; THUMB-LABEL: select_0_or_neg1_alt_zeroext:
+; THUMB:       @ %bb.0:
+; THUMB-NEXT:    subs r0, r0, #1
+; THUMB-NEXT:    bx lr
   %z = zext i1 %cond to i32
   %add = add i32 %z, -1
   ret i32 %add
 }
 
 define i32 @select_0_or_neg1_alt_signext(i1 signext %cond) {
-; CHECK-LABEL: select_0_or_neg1_alt_signext:
-; CHECK:       @ %bb.0:
-; CHECK-NEXT:    mvn r0, r0
-; CHECK-NEXT:    mov pc, lr
+; ARM-LABEL: select_0_or_neg1_alt_signext:
+; ARM:       @ %bb.0:
+; ARM-NEXT:    mvn r0, r0
+; ARM-NEXT:    mov pc, lr
+;
+; THUMB2-LABEL: select_0_or_neg1_alt_signext:
+; THUMB2:       @ %bb.0:
+; THUMB2-NEXT:    mvns r0, r0
+; THUMB2-NEXT:    bx lr
+;
+; THUMB-LABEL: select_0_or_neg1_alt_signext:
+; THUMB:       @ %bb.0:
+; THUMB-NEXT:    mvns r0, r0
+; THUMB-NEXT:    bx lr
   %z = zext i1 %cond to i32
   %add = add i32 %z, -1
   ret i32 %add
@@ -129,28 +268,59 @@ define i32 @select_0_or_neg1_alt_signext(i1 signext %cond) {
 ; select Cond, -1, 0 --> sext (Cond)
 
 define i32 @select_neg1_or_0(i1 %cond) {
-; CHECK-LABEL: select_neg1_or_0:
-; CHECK:       @ %bb.0:
-; CHECK-NEXT:    and r0, r0, #1
-; CHECK-NEXT:    rsb r0, r0, #0
-; CHECK-NEXT:    mov pc, lr
+; ARM-LABEL: select_neg1_or_0:
+; ARM:       @ %bb.0:
+; ARM-NEXT:    and r0, r0, #1
+; ARM-NEXT:    rsb r0, r0, #0
+; ARM-NEXT:    mov pc, lr
+;
+; THUMB2-LABEL: select_neg1_or_0:
+; THUMB2:       @ %bb.0:
+; THUMB2-NEXT:    and r0, r0, #1
+; THUMB2-NEXT:    rsbs r0, r0, #0
+; THUMB2-NEXT:    bx lr
+;
+; THUMB-LABEL: select_neg1_or_0:
+; THUMB:       @ %bb.0:
+; THUMB-NEXT:    movs r1, #1
+; THUMB-NEXT:    ands r1, r0
+; THUMB-NEXT:    rsbs r0, r1, #0
+; THUMB-NEXT:    bx lr
   %sel = select i1 %cond, i32 -1, i32 0
   ret i32 %sel
 }
 
 define i32 @select_neg1_or_0_zeroext(i1 zeroext %cond) {
-; CHECK-LABEL: select_neg1_or_0_zeroext:
-; CHECK:       @ %bb.0:
-; CHECK-NEXT:    rsb r0, r0, #0
-; CHECK-NEXT:    mov pc, lr
+; ARM-LABEL: select_neg1_or_0_zeroext:
+; ARM:       @ %bb.0:
+; ARM-NEXT:    rsb r0, r0, #0
+; ARM-NEXT:    mov pc, lr
+;
+; THUMB2-LABEL: select_neg1_or_0_zeroext:
+; THUMB2:       @ %bb.0:
+; THUMB2-NEXT:    rsbs r0, r0, #0
+; THUMB2-NEXT:    bx lr
+;
+; THUMB-LABEL: select_neg1_or_0_zeroext:
+; THUMB:       @ %bb.0:
+; THUMB-NEXT:    rsbs r0, r0, #0
+; THUMB-NEXT:    bx lr
   %sel = select i1 %cond, i32 -1, i32 0
   ret i32 %sel
 }
 
 define i32 @select_neg1_or_0_signext(i1 signext %cond) {
-; CHECK-LABEL: select_neg1_or_0_signext:
-; CHECK:       @ %bb.0:
-; CHECK-NEXT:    mov pc, lr
+; ARM-LABEL: select_neg1_or_0_signext:
+; ARM:       @ %bb.0:
+; ARM-NEXT:    mov pc, lr
+;
+; THUMB2-LABEL: select_neg1_or_0_signext:
+; THUMB2:       @ %bb.0:
+; THUMB2-NEXT:    bx lr
+;
+; THUMB-LABEL: select_neg1_or_0_signext:
+; THUMB:       @ %bb.0:
+; THUMB-NEXT:    bx lr
   %sel = select i1 %cond, i32 -1, i32 0
   ret i32 %sel
 }
@@ -158,37 +328,95 @@ define i32 @select_neg1_or_0_signext(i1 signext %cond) {
 ; select Cond, C+1, C --> add (zext Cond), C
 
 define i32 @select_Cplus1_C(i1 %cond) {
-; CHECK-LABEL: select_Cplus1_C:
-; CHECK:       @ %bb.0:
-; CHECK-NEXT:    mov r1, #41
-; CHECK-NEXT:    tst r0, #1
-; CHECK-NEXT:    movne r1, #42
-; CHECK-NEXT:    mov r0, r1
-; CHECK-NEXT:    mov pc, lr
+; ARM-LABEL: select_Cplus1_C:
+; ARM:       @ %bb.0:
+; ARM-NEXT:    mov r1, #41
+; ARM-NEXT:    tst r0, #1
+; ARM-NEXT:    movne r1, #42
+; ARM-NEXT:    mov r0, r1
+; ARM-NEXT:    mov pc, lr
+;
+; THUMB2-LABEL: select_Cplus1_C:
+; THUMB2:       @ %bb.0:
+; THUMB2-NEXT:    lsls r0, r0, #31
+; THUMB2-NEXT:    mov.w r0, #41
+; THUMB2-NEXT:    it ne
+; THUMB2-NEXT:    movne r0, #42
+; THUMB2-NEXT:    bx lr
+;
+; THUMB-LABEL: select_Cplus1_C:
+; THUMB:       @ %bb.0:
+; THUMB-NEXT:    lsls r0, r0, #31
+; THUMB-NEXT:    bne .LBB15_2
+; THUMB-NEXT:  @ %bb.1:
+; THUMB-NEXT:    movs r0, #41
+; THUMB-NEXT:    bx lr
+; THUMB-NEXT:  .LBB15_2:
+; THUMB-NEXT:    movs r0, #42
+; THUMB-NEXT:    bx lr
   %sel = select i1 %cond, i32 42, i32 41
   ret i32 %sel
 }
 
 define i32 @select_Cplus1_C_zeroext(i1 zeroext %cond) {
-; CHECK-LABEL: select_Cplus1_C_zeroext:
-; CHECK:       @ %bb.0:
-; CHECK-NEXT:    mov r1, #41
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    movne r1, #42
-; CHECK-NEXT:    mov r0, r1
-; CHECK-NEXT:    mov pc, lr
+; ARM-LABEL: select_Cplus1_C_zeroext:
+; ARM:       @ %bb.0:
+; ARM-NEXT:    mov r1, #41
+; ARM-NEXT:    cmp r0, #0
+; ARM-NEXT:    movne r1, #42
+; ARM-NEXT:    mov r0, r1
+; ARM-NEXT:    mov pc, lr
+;
+; THUMB2-LABEL: select_Cplus1_C_zeroext:
+; THUMB2:       @ %bb.0:
+; THUMB2-NEXT:    movs r1, #41
+; THUMB2-NEXT:    cmp r0, #0
+; THUMB2-NEXT:    it ne
+; THUMB2-NEXT:    movne r1, #42
+; THUMB2-NEXT:    mov r0, r1
+; THUMB2-NEXT:    bx lr
+;
+; THUMB-LABEL: select_Cplus1_C_zeroext:
+; THUMB:       @ %bb.0:
+; THUMB-NEXT:    cmp r0, #0
+; THUMB-NEXT:    bne .LBB16_2
+; THUMB-NEXT:  @ %bb.1:
+; THUMB-NEXT:    movs r0, #41
+; THUMB-NEXT:    bx lr
+; THUMB-NEXT:  .LBB16_2:
+; THUMB-NEXT:    movs r0, #42
+; THUMB-NEXT:    bx lr
   %sel = select i1 %cond, i32 42, i32 41
   ret i32 %sel
 }
 
 define i32 @select_Cplus1_C_signext(i1 signext %cond) {
-; CHECK-LABEL: select_Cplus1_C_signext:
-; CHECK:       @ %bb.0:
-; CHECK-NEXT:    mov r1, #41
-; CHECK-NEXT:    tst r0, #1
-; CHECK-NEXT:    movne r1, #42
-; CHECK-NEXT:    mov r0, r1
-; CHECK-NEXT:    mov pc, lr
+; ARM-LABEL: select_Cplus1_C_signext:
+; ARM:       @ %bb.0:
+; ARM-NEXT:    mov r1, #41
+; ARM-NEXT:    tst r0, #1
+; ARM-NEXT:    movne r1, #42
+; ARM-NEXT:    mov r0, r1
+; ARM-NEXT:    mov pc, lr
+;
+; THUMB2-LABEL: select_Cplus1_C_signext:
+; THUMB2:       @ %bb.0:
+; THUMB2-NEXT:    lsls r0, r0, #31
+; THUMB2-NEXT:    mov.w r0, #41
+; THUMB2-NEXT:    it ne
+; THUMB2-NEXT:    movne r0, #42
+; THUMB2-NEXT:    bx lr
+;
+; THUMB-LABEL: select_Cplus1_C_signext:
+; THUMB:       @ %bb.0:
+; THUMB-NEXT:    lsls r0, r0, #31
+; THUMB-NEXT:    bne .LBB17_2
+; THUMB-NEXT:  @ %bb.1:
+; THUMB-NEXT:    movs r0, #41
+; THUMB-NEXT:    bx lr
+; THUMB-NEXT:  .LBB17_2:
+; THUMB-NEXT:    movs r0, #42
+; THUMB-NEXT:    bx lr
   %sel = select i1 %cond, i32 42, i32 41
   ret i32 %sel
 }
@@ -196,37 +424,95 @@ define i32 @select_Cplus1_C_signext(i1 signext %cond) {
 ; select Cond, C, C+1 --> add (sext Cond), C
 
 define i32 @select_C_Cplus1(i1 %cond) {
-; CHECK-LABEL: select_C_Cplus1:
-; CHECK:       @ %bb.0:
-; CHECK-NEXT:    mov r1, #42
-; CHECK-NEXT:    tst r0, #1
-; CHECK-NEXT:    movne r1, #41
-; CHECK-NEXT:    mov r0, r1
-; CHECK-NEXT:    mov pc, lr
+; ARM-LABEL: select_C_Cplus1:
+; ARM:       @ %bb.0:
+; ARM-NEXT:    mov r1, #42
+; ARM-NEXT:    tst r0, #1
+; ARM-NEXT:    movne r1, #41
+; ARM-NEXT:    mov r0, r1
+; ARM-NEXT:    mov pc, lr
+;
+; THUMB2-LABEL: select_C_Cplus1:
+; THUMB2:       @ %bb.0:
+; THUMB2-NEXT:    lsls r0, r0, #31
+; THUMB2-NEXT:    mov.w r0, #42
+; THUMB2-NEXT:    it ne
+; THUMB2-NEXT:    movne r0, #41
+; THUMB2-NEXT:    bx lr
+;
+; THUMB-LABEL: select_C_Cplus1:
+; THUMB:       @ %bb.0:
+; THUMB-NEXT:    lsls r0, r0, #31
+; THUMB-NEXT:    bne .LBB18_2
+; THUMB-NEXT:  @ %bb.1:
+; THUMB-NEXT:    movs r0, #42
+; THUMB-NEXT:    bx lr
+; THUMB-NEXT:  .LBB18_2:
+; THUMB-NEXT:    movs r0, #41
+; THUMB-NEXT:    bx lr
   %sel = select i1 %cond, i32 41, i32 42
   ret i32 %sel
 }
 
 define i32 @select_C_Cplus1_zeroext(i1 zeroext %cond) {
-; CHECK-LABEL: select_C_Cplus1_zeroext:
-; CHECK:       @ %bb.0:
-; CHECK-NEXT:    mov r1, #42
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    movne r1, #41
-; CHECK-NEXT:    mov r0, r1
-; CHECK-NEXT:    mov pc, lr
+; ARM-LABEL: select_C_Cplus1_zeroext:
+; ARM:       @ %bb.0:
+; ARM-NEXT:    mov r1, #42
+; ARM-NEXT:    cmp r0, #0
+; ARM-NEXT:    movne r1, #41
+; ARM-NEXT:    mov r0, r1
+; ARM-NEXT:    mov pc, lr
+;
+; THUMB2-LABEL: select_C_Cplus1_zeroext:
+; THUMB2:       @ %bb.0:
+; THUMB2-NEXT:    movs r1, #42
+; THUMB2-NEXT:    cmp r0, #0
+; THUMB2-NEXT:    it ne
+; THUMB2-NEXT:    movne r1, #41
+; THUMB2-NEXT:    mov r0, r1
+; THUMB2-NEXT:    bx lr
+;
+; THUMB-LABEL: select_C_Cplus1_zeroext:
+; THUMB:       @ %bb.0:
+; THUMB-NEXT:    cmp r0, #0
+; THUMB-NEXT:    bne .LBB19_2
+; THUMB-NEXT:  @ %bb.1:
+; THUMB-NEXT:    movs r0, #42
+; THUMB-NEXT:    bx lr
+; THUMB-NEXT:  .LBB19_2:
+; THUMB-NEXT:    movs r0, #41
+; THUMB-NEXT:    bx lr
   %sel = select i1 %cond, i32 41, i32 42
   ret i32 %sel
 }
 
 define i32 @select_C_Cplus1_signext(i1 signext %cond) {
-; CHECK-LABEL: select_C_Cplus1_signext:
-; CHECK:       @ %bb.0:
-; CHECK-NEXT:    mov r1, #42
-; CHECK-NEXT:    tst r0, #1
-; CHECK-NEXT:    movne r1, #41
-; CHECK-NEXT:    mov r0, r1
-; CHECK-NEXT:    mov pc, lr
+; ARM-LABEL: select_C_Cplus1_signext:
+; ARM:       @ %bb.0:
+; ARM-NEXT:    mov r1, #42
+; ARM-NEXT:    tst r0, #1
+; ARM-NEXT:    movne r1, #41
+; ARM-NEXT:    mov r0, r1
+; ARM-NEXT:    mov pc, lr
+;
+; THUMB2-LABEL: select_C_Cplus1_signext:
+; THUMB2:       @ %bb.0:
+; THUMB2-NEXT:    lsls r0, r0, #31
+; THUMB2-NEXT:    mov.w r0, #42
+; THUMB2-NEXT:    it ne
+; THUMB2-NEXT:    movne r0, #41
+; THUMB2-NEXT:    bx lr
+;
+; THUMB-LABEL: select_C_Cplus1_signext:
+; THUMB:       @ %bb.0:
+; THUMB-NEXT:    lsls r0, r0, #31
+; THUMB-NEXT:    bne .LBB20_2
+; THUMB-NEXT:  @ %bb.1:
+; THUMB-NEXT:    movs r0, #42
+; THUMB-NEXT:    bx lr
+; THUMB-NEXT:  .LBB20_2:
+; THUMB-NEXT:    movs r0, #41
+; THUMB-NEXT:    bx lr
   %sel = select i1 %cond, i32 41, i32 42
   ret i32 %sel
 }
@@ -235,40 +521,101 @@ define i32 @select_C_Cplus1_signext(i1 signext %cond) {
 ; select Cond, C1, C2 --> add (mul (zext Cond), C1-C2), C2 --> add (and (sext Cond), C1-C2), C2
 
 define i32 @select_C1_C2(i1 %cond) {
-; CHECK-LABEL: select_C1_C2:
-; CHECK:       @ %bb.0:
-; CHECK-NEXT:    mov r1, #165
-; CHECK-NEXT:    tst r0, #1
-; CHECK-NEXT:    orr r1, r1, #256
-; CHECK-NEXT:    moveq r1, #42
-; CHECK-NEXT:    mov r0, r1
-; CHECK-NEXT:    mov pc, lr
+; ARM-LABEL: select_C1_C2:
+; ARM:       @ %bb.0:
+; ARM-NEXT:    mov r1, #165
+; ARM-NEXT:    tst r0, #1
+; ARM-NEXT:    orr r1, r1, #256
+; ARM-NEXT:    moveq r1, #42
+; ARM-NEXT:    mov r0, r1
+; ARM-NEXT:    mov pc, lr
+;
+; THUMB2-LABEL: select_C1_C2:
+; THUMB2:       @ %bb.0:
+; THUMB2-NEXT:    lsls r0, r0, #31
+; THUMB2-NEXT:    mov.w r0, #42
+; THUMB2-NEXT:    it ne
+; THUMB2-NEXT:    movwne r0, #421
+; THUMB2-NEXT:    bx lr
+;
+; THUMB-LABEL: select_C1_C2:
+; THUMB:       @ %bb.0:
+; THUMB-NEXT:    lsls r0, r0, #31
+; THUMB-NEXT:    bne .LBB21_2
+; THUMB-NEXT:  @ %bb.1:
+; THUMB-NEXT:    movs r0, #42
+; THUMB-NEXT:    bx lr
+; THUMB-NEXT:  .LBB21_2:
+; THUMB-NEXT:    movs r0, #255
+; THUMB-NEXT:    adds r0, #166
+; THUMB-NEXT:    bx lr
   %sel = select i1 %cond, i32 421, i32 42
   ret i32 %sel
 }
 
 define i32 @select_C1_C2_zeroext(i1 zeroext %cond) {
-; CHECK-LABEL: select_C1_C2_zeroext:
-; CHECK:       @ %bb.0:
-; CHECK-NEXT:    mov r1, #165
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    orr r1, r1, #256
-; CHECK-NEXT:    moveq r1, #42
-; CHECK-NEXT:    mov r0, r1
-; CHECK-NEXT:    mov pc, lr
+; ARM-LABEL: select_C1_C2_zeroext:
+; ARM:       @ %bb.0:
+; ARM-NEXT:    mov r1, #165
+; ARM-NEXT:    cmp r0, #0
+; ARM-NEXT:    orr r1, r1, #256
+; ARM-NEXT:    moveq r1, #42
+; ARM-NEXT:    mov r0, r1
+; ARM-NEXT:    mov pc, lr
+;
+; THUMB2-LABEL: select_C1_C2_zeroext:
+; THUMB2:       @ %bb.0:
+; THUMB2-NEXT:    movs r1, #42
+; THUMB2-NEXT:    cmp r0, #0
+; THUMB2-NEXT:    it ne
+; THUMB2-NEXT:    movwne r1, #421
+; THUMB2-NEXT:    mov r0, r1
+; THUMB2-NEXT:    bx lr
+;
+; THUMB-LABEL: select_C1_C2_zeroext:
+; THUMB:       @ %bb.0:
+; THUMB-NEXT:    cmp r0, #0
+; THUMB-NEXT:    bne .LBB22_2
+; THUMB-NEXT:  @ %bb.1:
+; THUMB-NEXT:    movs r0, #42
+; THUMB-NEXT:    bx lr
+; THUMB-NEXT:  .LBB22_2:
+; THUMB-NEXT:    movs r0, #255
+; THUMB-NEXT:    adds r0, #166
+; THUMB-NEXT:    bx lr
   %sel = select i1 %cond, i32 421, i32 42
   ret i32 %sel
 }
 
 define i32 @select_C1_C2_signext(i1 signext %cond) {
-; CHECK-LABEL: select_C1_C2_signext:
-; CHECK:       @ %bb.0:
-; CHECK-NEXT:    mov r1, #165
-; CHECK-NEXT:    tst r0, #1
-; CHECK-NEXT:    orr r1, r1, #256
-; CHECK-NEXT:    moveq r1, #42
-; CHECK-NEXT:    mov r0, r1
-; CHECK-NEXT:    mov pc, lr
+; ARM-LABEL: select_C1_C2_signext:
+; ARM:       @ %bb.0:
+; ARM-NEXT:    mov r1, #165
+; ARM-NEXT:    tst r0, #1
+; ARM-NEXT:    orr r1, r1, #256
+; ARM-NEXT:    moveq r1, #42
+; ARM-NEXT:    mov r0, r1
+; ARM-NEXT:    mov pc, lr
+;
+; THUMB2-LABEL: select_C1_C2_signext:
+; THUMB2:       @ %bb.0:
+; THUMB2-NEXT:    lsls r0, r0, #31
+; THUMB2-NEXT:    mov.w r0, #42
+; THUMB2-NEXT:    it ne
+; THUMB2-NEXT:    movwne r0, #421
+; THUMB2-NEXT:    bx lr
+;
+; THUMB-LABEL: select_C1_C2_signext:
+; THUMB:       @ %bb.0:
+; THUMB-NEXT:    lsls r0, r0, #31
+; THUMB-NEXT:    bne .LBB23_2
+; THUMB-NEXT:  @ %bb.1:
+; THUMB-NEXT:    movs r0, #42
+; THUMB-NEXT:    bx lr
+; THUMB-NEXT:  .LBB23_2:
+; THUMB-NEXT:    movs r0, #255
+; THUMB-NEXT:    adds r0, #166
+; THUMB-NEXT:    bx lr
   %sel = select i1 %cond, i32 421, i32 42
   ret i32 %sel
 }
@@ -277,26 +624,92 @@ define i32 @select_C1_C2_signext(i1 signext %cond) {
 ; This becomes an opaque constant via ConstantHoisting, so we don't fold it into the select.
 
 define i64 @opaque_constant1(i1 %cond, i64 %x) {
-; CHECK-LABEL: opaque_constant1:
-; CHECK:       @ %bb.0:
-; CHECK-NEXT:    .save {r4, lr}
-; CHECK-NEXT:    push {r4, lr}
-; CHECK-NEXT:    mov lr, #1
-; CHECK-NEXT:    ands r12, r0, #1
-; CHECK-NEXT:    mov r0, #23
-; CHECK-NEXT:    orr lr, lr, #65536
-; CHECK-NEXT:    mvnne r0, #3
-; CHECK-NEXT:    and r4, r0, lr
-; CHECK-NEXT:    movne r12, #1
-; CHECK-NEXT:    subs r0, r4, #1
-; CHECK-NEXT:    eor r2, r2, lr
-; CHECK-NEXT:    eor r3, r3, #1
-; CHECK-NEXT:    sbc r1, r12, #0
-; CHECK-NEXT:    orrs r2, r2, r3
-; CHECK-NEXT:    movne r0, r4
-; CHECK-NEXT:    movne r1, r12
-; CHECK-NEXT:    pop {r4, lr}
-; CHECK-NEXT:    mov pc, lr
+; ARM-LABEL: opaque_constant1:
+; ARM:       @ %bb.0:
+; ARM-NEXT:    .save {r4, lr}
+; ARM-NEXT:    push {r4, lr}
+; ARM-NEXT:    mov lr, #1
+; ARM-NEXT:    ands r12, r0, #1
+; ARM-NEXT:    mov r0, #23
+; ARM-NEXT:    orr lr, lr, #65536
+; ARM-NEXT:    mvnne r0, #3
+; ARM-NEXT:    and r4, r0, lr
+; ARM-NEXT:    movne r12, #1
+; ARM-NEXT:    subs r0, r4, #1
+; ARM-NEXT:    eor r2, r2, lr
+; ARM-NEXT:    eor r3, r3, #1
+; ARM-NEXT:    sbc r1, r12, #0
+; ARM-NEXT:    orrs r2, r2, r3
+; ARM-NEXT:    movne r0, r4
+; ARM-NEXT:    movne r1, r12
+; ARM-NEXT:    pop {r4, lr}
+; ARM-NEXT:    mov pc, lr
+;
+; THUMB2-LABEL: opaque_constant1:
+; THUMB2:       @ %bb.0:
+; THUMB2-NEXT:    .save {r7, lr}
+; THUMB2-NEXT:    push {r7, lr}
+; THUMB2-NEXT:    ands r12, r0, #1
+; THUMB2-NEXT:    mov.w lr, #1
+; THUMB2-NEXT:    itt ne
+; THUMB2-NEXT:    movne.w lr, #65536
+; THUMB2-NEXT:    movne.w r12, #1
+; THUMB2-NEXT:    subs.w r0, lr, #1
+; THUMB2-NEXT:    sbc r1, r12, #0
+; THUMB2-NEXT:    eor r3, r3, #1
+; THUMB2-NEXT:    eor r2, r2, #65537
+; THUMB2-NEXT:    orrs r2, r3
+; THUMB2-NEXT:    itt ne
+; THUMB2-NEXT:    movne r0, lr
+; THUMB2-NEXT:    movne r1, r12
+; THUMB2-NEXT:    pop {r7, pc}
+;
+; THUMB-LABEL: opaque_constant1:
+; THUMB:       @ %bb.0:
+; THUMB-NEXT:    .save {r4, r5, r6, r7, lr}
+; THUMB-NEXT:    push {r4, r5, r6, r7, lr}
+; THUMB-NEXT:    movs r7, #1
+; THUMB-NEXT:    ands r0, r7
+; THUMB-NEXT:    subs r1, r0, #1
+; THUMB-NEXT:    push {r0}
+; THUMB-NEXT:    pop {r4}
+; THUMB-NEXT:    sbcs r4, r1
+; THUMB-NEXT:    cmp r0, #0
+; THUMB-NEXT:    bne .LBB24_2
+; THUMB-NEXT:  @ %bb.1:
+; THUMB-NEXT:    movs r5, #23
+; THUMB-NEXT:    b .LBB24_3
+; THUMB-NEXT:  .LBB24_2:
+; THUMB-NEXT:    movs r0, #3
+; THUMB-NEXT:    mvns r5, r0
+; THUMB-NEXT:  .LBB24_3:
+; THUMB-NEXT:    ldr r0, .LCPI24_0
+; THUMB-NEXT:    ands r5, r0
+; THUMB-NEXT:    movs r6, #0
+; THUMB-NEXT:    subs r0, r5, #1
+; THUMB-NEXT:    push {r4}
+; THUMB-NEXT:    pop {r1}
+; THUMB-NEXT:    sbcs r1, r6
+; THUMB-NEXT:    eors r3, r7
+; THUMB-NEXT:    ldr r6, .LCPI24_0
+; THUMB-NEXT:    eors r2, r6
+; THUMB-NEXT:    orrs r2, r3
+; THUMB-NEXT:    beq .LBB24_5
+; THUMB-NEXT:  @ %bb.4:
+; THUMB-NEXT:    movs r1, r4
+; THUMB-NEXT:  .LBB24_5:
+; THUMB-NEXT:    cmp r2, #0
+; THUMB-NEXT:    beq .LBB24_7
+; THUMB-NEXT:  @ %bb.6:
+; THUMB-NEXT:    movs r0, r5
+; THUMB-NEXT:  .LBB24_7:
+; THUMB-NEXT:    pop {r4, r5, r6, r7}
+; THUMB-NEXT:    pop {r2}
+; THUMB-NEXT:    bx r2
+; THUMB-NEXT:    .p2align 2
+; THUMB-NEXT:  @ %bb.8:
+; THUMB-NEXT:  .LCPI24_0:
+; THUMB-NEXT:    .long 65537 @ 0x10001
   %sel = select i1 %cond, i64 -4, i64 23
   %bo = and i64 %sel, 4295032833  ; 0x100010001
   %cmp = icmp eq i64 %x, 4295032833
@@ -309,15 +722,43 @@ define i64 @opaque_constant1(i1 %cond, i64 %x) {
 ; This becomes an opaque constant via ConstantHoisting, so we don't fold it into the select.
 
 define i64 @opaque_constant2(i1 %cond, i64 %x) {
-; CHECK-LABEL: opaque_constant2:
-; CHECK:       @ %bb.0:
-; CHECK-NEXT:    mov r1, #1
-; CHECK-NEXT:    tst r0, #1
-; CHECK-NEXT:    orr r1, r1, #65536
-; CHECK-NEXT:    moveq r1, #23
-; CHECK-NEXT:    bic r0, r1, #22
-; CHECK-NEXT:    mov r1, #0
-; CHECK-NEXT:    mov pc, lr
+; ARM-LABEL: opaque_constant2:
+; ARM:       @ %bb.0:
+; ARM-NEXT:    mov r1, #1
+; ARM-NEXT:    tst r0, #1
+; ARM-NEXT:    orr r1, r1, #65536
+; ARM-NEXT:    moveq r1, #23
+; ARM-NEXT:    bic r0, r1, #22
+; ARM-NEXT:    mov r1, #0
+; ARM-NEXT:    mov pc, lr
+;
+; THUMB2-LABEL: opaque_constant2:
+; THUMB2:       @ %bb.0:
+; THUMB2-NEXT:    lsls r0, r0, #31
+; THUMB2-NEXT:    mov.w r1, #0
+; THUMB2-NEXT:    mov.w r0, #1
+; THUMB2-NEXT:    it ne
+; THUMB2-NEXT:    movne.w r0, #65537
+; THUMB2-NEXT:    bx lr
+;
+; THUMB-LABEL: opaque_constant2:
+; THUMB:       @ %bb.0:
+; THUMB-NEXT:    lsls r0, r0, #31
+; THUMB-NEXT:    bne .LBB25_2
+; THUMB-NEXT:  @ %bb.1:
+; THUMB-NEXT:    movs r0, #23
+; THUMB-NEXT:    b .LBB25_3
+; THUMB-NEXT:  .LBB25_2:
+; THUMB-NEXT:    ldr r0, .LCPI25_0
+; THUMB-NEXT:  .LBB25_3:
+; THUMB-NEXT:    movs r1, #22
+; THUMB-NEXT:    bics r0, r1
+; THUMB-NEXT:    movs r1, #0
+; THUMB-NEXT:    bx lr
+; THUMB-NEXT:    .p2align 2
+; THUMB-NEXT:  @ %bb.4:
+; THUMB-NEXT:  .LCPI25_0:
+; THUMB-NEXT:    .long 65537 @ 0x10001
   %sel = select i1 %cond, i64 65537, i64 23
   %bo = and i64 %sel, 65537
   ret i64 %bo
diff --git a/llvm/test/CodeGen/ARM/shift_minsize.ll b/llvm/test/CodeGen/ARM/shift_minsize.ll
index 4d10c6439..de7327ed3 100644
--- a/llvm/test/CodeGen/ARM/shift_minsize.ll
+++ b/llvm/test/CodeGen/ARM/shift_minsize.ll
@@ -1,4 +1,10 @@
-; RUN: llc -mtriple=arm-eabi %s -o - | FileCheck %s
+; RUN: llc -mtriple=arm-eabi        %s -o - | FileCheck %s
+; RUN: llc -mtriple=thumbv7-windows %s -o - | FileCheck %s -check-prefix=CHECK-WIN
+
+; The Windows runtime doesn't have these.
+; CHECK-WIN-NOT: __ashldi3
+; CHECK-WIN-NOT: __ashrdi3
+; CHECK-WIN-NOT: __lshrdi3
 
 define i64 @f0(i64 %val, i64 %amt) minsize optsize {
 ; CHECK-LABEL:   f0:
diff --git a/llvm/test/CodeGen/ARM/shift_parts.ll b/llvm/test/CodeGen/ARM/shift_parts.ll
index a3a98e6d2..9bc77d585 100644
--- a/llvm/test/CodeGen/ARM/shift_parts.ll
+++ b/llvm/test/CodeGen/ARM/shift_parts.ll
@@ -52,7 +52,8 @@ entry:
 define i64 @shift_left_imm_big2(i64 %x) {
 ; CHECK-MVE-LABEL: shift_left_imm_big2:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    lsll r0, r1, #32
+; CHECK-MVE-NEXT:    mov r1, r0
+; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-NON-MVE-LABEL: shift_left_imm_big2:
@@ -128,7 +129,8 @@ entry:
 define i64 @shift_right_imm_big2(i64 %x) {
 ; CHECK-MVE-LABEL: shift_right_imm_big2:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    lsrl r0, r1, #32
+; CHECK-MVE-NEXT:    mov r0, r1
+; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-NON-MVE-LABEL: shift_right_imm_big2:
@@ -219,3 +221,62 @@ entry:
   store i40 %bf.clear, i40* %0, align 1
   ret void
 }
+
+%struct.a = type { i96 }
+
+define void @lsll_128bit_shift(%struct.a* nocapture %x) local_unnamed_addr #0 {
+; CHECK-MVE-LABEL: lsll_128bit_shift:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    strd r1, r1, [r0]
+; CHECK-MVE-NEXT:    str r1, [r0, #8]
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-NON-MVE-LABEL: lsll_128bit_shift:
+; CHECK-NON-MVE:       @ %bb.0: @ %entry
+; CHECK-NON-MVE-NEXT:    movs r1, #0
+; CHECK-NON-MVE-NEXT:    str r1, [r0]
+; CHECK-NON-MVE-NEXT:    str r1, [r0, #4]
+; CHECK-NON-MVE-NEXT:    str r1, [r0, #8]
+; CHECK-NON-MVE-NEXT:    bx lr
+entry:
+  %0 = bitcast %struct.a* %x to i128*
+  %bf.load = load i128, i128* %0, align 8
+  %bf.clear4 = and i128 %bf.load, -79228162514264337593543950336
+  store i128 %bf.clear4, i128* %0, align 8
+  ret void
+}
+
+%struct.b = type { i184 }
+
+define void @lsll_256bit_shift(%struct.b* nocapture %x) local_unnamed_addr #0 {
+; CHECK-MVE-LABEL: lsll_256bit_shift:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    str r1, [r0, #16]
+; CHECK-MVE-NEXT:    strd r1, r1, [r0, #8]
+; CHECK-MVE-NEXT:    strd r1, r1, [r0]
+; CHECK-MVE-NEXT:    ldrb r1, [r0, #23]
+; CHECK-MVE-NEXT:    lsls r1, r1, #24
+; CHECK-MVE-NEXT:    str r1, [r0, #20]
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-NON-MVE-LABEL: lsll_256bit_shift:
+; CHECK-NON-MVE:       @ %bb.0: @ %entry
+; CHECK-NON-MVE-NEXT:    movs r1, #0
+; CHECK-NON-MVE-NEXT:    str r1, [r0, #16]
+; CHECK-NON-MVE-NEXT:    str r1, [r0, #8]
+; CHECK-NON-MVE-NEXT:    str r1, [r0, #12]
+; CHECK-NON-MVE-NEXT:    str r1, [r0]
+; CHECK-NON-MVE-NEXT:    str r1, [r0, #4]
+; CHECK-NON-MVE-NEXT:    ldrb r1, [r0, #23]
+; CHECK-NON-MVE-NEXT:    lsls r1, r1, #24
+; CHECK-NON-MVE-NEXT:    str r1, [r0, #20]
+; CHECK-NON-MVE-NEXT:    bx lr
+entry:
+  %0 = bitcast %struct.b* %x to i192*
+  %bf.load = load i192, i192* %0, align 8
+  %bf.clear4 = and i192 %bf.load, -24519928653854221733733552434404946937899825954937634816
+  store i192 %bf.clear4, i192* %0, align 8
+  ret void
+}
diff --git a/llvm/test/CodeGen/ARM/signext-inreg.ll b/llvm/test/CodeGen/ARM/signext-inreg.ll
new file mode 100644
index 000000000..dd8b144bb
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/signext-inreg.ll
@@ -0,0 +1,17 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=armv8 | FileCheck %s
+define <4 x i32> @test(<4 x i32> %m) {
+; CHECK-LABEL: test:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov d17, r2, r3
+; CHECK-NEXT:    vmov d16, r0, r1
+; CHECK-NEXT:    vshl.i32 q8, q8, #24
+; CHECK-NEXT:    vshr.s32 q8, q8, #24
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    vmov r2, r3, d17
+; CHECK-NEXT:    bx lr
+entry:
+  %shl = shl <4 x i32> %m, <i32 24, i32 24, i32 24, i32 24>
+  %shr = ashr exact <4 x i32> %shl, <i32 24, i32 24, i32 24, i32 24>
+  ret <4 x i32> %shr
+}
diff --git a/llvm/test/CodeGen/ARM/smml.ll b/llvm/test/CodeGen/ARM/smml.ll
index af34000b2..712aaa439 100644
--- a/llvm/test/CodeGen/ARM/smml.ll
+++ b/llvm/test/CodeGen/ARM/smml.ll
@@ -8,6 +8,13 @@
 ; RUN: llc -mtriple=thumbv7m-eabi %s -o - | FileCheck %s -check-prefix=CHECK -check-prefix=CHECK-V4
 ; RUN: llc -mtriple=thumbv7em-eabi %s -o - | FileCheck %s -check-prefix=CHECK -check-prefix=CHECK-THUMBV6T2
 
+; Next test would previously trigger an assertion responsible for verification of
+; call site info state.
+; RUN: llc -stop-after=if-converter -debug-entry-values -mtriple=thumbv6t2-eabi %s -o -| FileCheck %s -check-prefix=CHECK-CALLSITE
+; CHECK-CALLSITE: name:  test_used_flags
+; CHECK-CALLSITE: callSites:
+
+
 define i32 @Test0(i32 %a, i32 %b, i32 %c) nounwind readnone ssp {
 entry:
 ; CHECK-LABEL: Test0
diff --git a/llvm/test/CodeGen/ARM/softfp-constant-comparison.ll b/llvm/test/CodeGen/ARM/softfp-constant-comparison.ll
new file mode 100644
index 000000000..e7e4d2bf2
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/softfp-constant-comparison.ll
@@ -0,0 +1,46 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv7em-arm-none-eabi < %s | FileCheck %s
+
+target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
+target triple = "thumbv7em-arm-none-eabi"
+
+@a = hidden global i32 0, align 4
+
+define hidden void @fn1() nounwind #0 {
+; CHECK-LABEL: fn1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    vldr d0, .LCPI0_0
+; CHECK-NEXT:    vmov r0, r1, d0
+; CHECK-NEXT:    mov r2, r0
+; CHECK-NEXT:    mov r3, r1
+; CHECK-NEXT:    bl __aeabi_dcmpeq
+; CHECK-NEXT:    cbnz r0, .LBB0_2
+; CHECK-NEXT:    b .LBB0_1
+; CHECK-NEXT:  .LBB0_1: @ %land.rhs
+; CHECK-NEXT:    b .LBB0_2
+; CHECK-NEXT:  .LBB0_2: @ %land.end
+; CHECK-NEXT:    pop {r7, pc}
+; CHECK-NEXT:    .p2align 3
+; CHECK-NEXT:  @ %bb.3:
+; CHECK-NEXT:  .LCPI0_0:
+; CHECK-NEXT:    .long 0 @ double 0
+; CHECK-NEXT:    .long 0
+entry:
+  %0 = load i32, i32* @a, align 4
+  %conv = sitofp i32 %0 to double
+  %mul = fmul nnan ninf nsz double 0.000000e+00, %conv
+  %tobool = fcmp nnan ninf nsz une double %mul, 0.000000e+00
+  br i1 %tobool, label %land.rhs, label %land.end
+
+land.rhs:                                         ; preds = %entry
+  br label %land.end
+
+land.end:                                         ; preds = %land.rhs, %entry
+  %1 = phi i1 [ false, %entry ], [ false, %land.rhs ]
+  %land.ext = zext i1 %1 to i32
+  ret void
+}
+
+attributes #0 = { noinline nounwind optnone "correctly-rounded-divide-sqrt-fp-math"="false" "denormal-fp-math"="preserve-sign" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="true" "no-jump-tables"="false" "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="true" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="cortex-m4" "target-features"="+armv7e-m,+dsp,+fp16,+hwdiv,+thumb-mode,+vfp2sp,+vfp3d16sp,+vfp4d16sp,-aes,-crc,-crypto,-dotprod,-fp16fml,-fullfp16,-hwdiv-arm,-lob,-mve,-mve.fp,-ras,-sb,-sha2" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/ARM/ssub_sat.ll b/llvm/test/CodeGen/ARM/ssub_sat.ll
new file mode 100644
index 000000000..9c3c5babc
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/ssub_sat.ll
@@ -0,0 +1,678 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=thumbv6m-none-eabi | FileCheck %s --check-prefix=CHECK-T1
+; RUN: llc < %s -mtriple=thumbv7m-none-eabi | FileCheck %s --check-prefix=CHECK-T2 --check-prefix=CHECK-T2NODSP
+; RUN: llc < %s -mtriple=thumbv7em-none-eabi | FileCheck %s --check-prefix=CHECK-T2 --check-prefix=CHECK-T2DSP
+; RUN: llc < %s -mtriple=armv5t-none-eabi | FileCheck %s --check-prefix=CHECK-ARM --check-prefix=CHECK-ARMNODPS
+; RUN: llc < %s -mtriple=armv5te-none-eabi | FileCheck %s --check-prefix=CHECK-ARM --check-prefix=CHECK-ARMBASEDSP
+; RUN: llc < %s -mtriple=armv6-none-eabi | FileCheck %s --check-prefix=CHECK-ARM --check-prefix=CHECK-ARMDSP
+
+declare i4 @llvm.ssub.sat.i4(i4, i4)
+declare i8 @llvm.ssub.sat.i8(i8, i8)
+declare i16 @llvm.ssub.sat.i16(i16, i16)
+declare i32 @llvm.ssub.sat.i32(i32, i32)
+declare i64 @llvm.ssub.sat.i64(i64, i64)
+declare <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32>, <4 x i32>)
+
+define i32 @func(i32 %x, i32 %y) nounwind {
+; CHECK-T1-LABEL: func:
+; CHECK-T1:       @ %bb.0:
+; CHECK-T1-NEXT:    .save {r4, lr}
+; CHECK-T1-NEXT:    push {r4, lr}
+; CHECK-T1-NEXT:    mov r2, r0
+; CHECK-T1-NEXT:    movs r3, #1
+; CHECK-T1-NEXT:    subs r0, r0, r1
+; CHECK-T1-NEXT:    mov r4, r3
+; CHECK-T1-NEXT:    bmi .LBB0_2
+; CHECK-T1-NEXT:  @ %bb.1:
+; CHECK-T1-NEXT:    movs r4, #0
+; CHECK-T1-NEXT:  .LBB0_2:
+; CHECK-T1-NEXT:    cmp r4, #0
+; CHECK-T1-NEXT:    bne .LBB0_4
+; CHECK-T1-NEXT:  @ %bb.3:
+; CHECK-T1-NEXT:    lsls r3, r3, #31
+; CHECK-T1-NEXT:    cmp r2, r1
+; CHECK-T1-NEXT:    bvs .LBB0_5
+; CHECK-T1-NEXT:    b .LBB0_6
+; CHECK-T1-NEXT:  .LBB0_4:
+; CHECK-T1-NEXT:    ldr r3, .LCPI0_0
+; CHECK-T1-NEXT:    cmp r2, r1
+; CHECK-T1-NEXT:    bvc .LBB0_6
+; CHECK-T1-NEXT:  .LBB0_5:
+; CHECK-T1-NEXT:    mov r0, r3
+; CHECK-T1-NEXT:  .LBB0_6:
+; CHECK-T1-NEXT:    pop {r4, pc}
+; CHECK-T1-NEXT:    .p2align 2
+; CHECK-T1-NEXT:  @ %bb.7:
+; CHECK-T1-NEXT:  .LCPI0_0:
+; CHECK-T1-NEXT:    .long 2147483647 @ 0x7fffffff
+;
+; CHECK-T2NODSP-LABEL: func:
+; CHECK-T2NODSP:       @ %bb.0:
+; CHECK-T2NODSP-NEXT:    subs.w r12, r0, r1
+; CHECK-T2NODSP-NEXT:    mov.w r3, #0
+; CHECK-T2NODSP-NEXT:    mov.w r2, #-2147483648
+; CHECK-T2NODSP-NEXT:    it mi
+; CHECK-T2NODSP-NEXT:    movmi r3, #1
+; CHECK-T2NODSP-NEXT:    cmp r3, #0
+; CHECK-T2NODSP-NEXT:    it ne
+; CHECK-T2NODSP-NEXT:    mvnne r2, #-2147483648
+; CHECK-T2NODSP-NEXT:    cmp r0, r1
+; CHECK-T2NODSP-NEXT:    it vc
+; CHECK-T2NODSP-NEXT:    movvc r2, r12
+; CHECK-T2NODSP-NEXT:    mov r0, r2
+; CHECK-T2NODSP-NEXT:    bx lr
+;
+; CHECK-T2DSP-LABEL: func:
+; CHECK-T2DSP:       @ %bb.0:
+; CHECK-T2DSP-NEXT:    qsub r0, r0, r1
+; CHECK-T2DSP-NEXT:    bx lr
+;
+; CHECK-ARMNODPS-LABEL: func:
+; CHECK-ARMNODPS:       @ %bb.0:
+; CHECK-ARMNODPS-NEXT:    subs r12, r0, r1
+; CHECK-ARMNODPS-NEXT:    mov r3, #0
+; CHECK-ARMNODPS-NEXT:    movmi r3, #1
+; CHECK-ARMNODPS-NEXT:    mov r2, #-2147483648
+; CHECK-ARMNODPS-NEXT:    cmp r3, #0
+; CHECK-ARMNODPS-NEXT:    mvnne r2, #-2147483648
+; CHECK-ARMNODPS-NEXT:    cmp r0, r1
+; CHECK-ARMNODPS-NEXT:    movvc r2, r12
+; CHECK-ARMNODPS-NEXT:    mov r0, r2
+; CHECK-ARMNODPS-NEXT:    bx lr
+;
+; CHECK-ARMBASEDSP-LABEL: func:
+; CHECK-ARMBASEDSP:       @ %bb.0:
+; CHECK-ARMBASEDSP-NEXT:    qsub r0, r0, r1
+; CHECK-ARMBASEDSP-NEXT:    bx lr
+;
+; CHECK-ARMDSP-LABEL: func:
+; CHECK-ARMDSP:       @ %bb.0:
+; CHECK-ARMDSP-NEXT:    qsub r0, r0, r1
+; CHECK-ARMDSP-NEXT:    bx lr
+  %tmp = call i32 @llvm.ssub.sat.i32(i32 %x, i32 %y)
+  ret i32 %tmp
+}
+
+define i64 @func2(i64 %x, i64 %y) nounwind {
+; CHECK-T1-LABEL: func2:
+; CHECK-T1:       @ %bb.0:
+; CHECK-T1-NEXT:    .save {r4, r5, r6, r7, lr}
+; CHECK-T1-NEXT:    push {r4, r5, r6, r7, lr}
+; CHECK-T1-NEXT:    .pad #4
+; CHECK-T1-NEXT:    sub sp, #4
+; CHECK-T1-NEXT:    str r2, [sp] @ 4-byte Spill
+; CHECK-T1-NEXT:    mov r2, r0
+; CHECK-T1-NEXT:    movs r4, #1
+; CHECK-T1-NEXT:    movs r0, #0
+; CHECK-T1-NEXT:    cmp r3, #0
+; CHECK-T1-NEXT:    mov r5, r4
+; CHECK-T1-NEXT:    bge .LBB1_2
+; CHECK-T1-NEXT:  @ %bb.1:
+; CHECK-T1-NEXT:    mov r5, r0
+; CHECK-T1-NEXT:  .LBB1_2:
+; CHECK-T1-NEXT:    cmp r1, #0
+; CHECK-T1-NEXT:    mov r7, r4
+; CHECK-T1-NEXT:    bge .LBB1_4
+; CHECK-T1-NEXT:  @ %bb.3:
+; CHECK-T1-NEXT:    mov r7, r0
+; CHECK-T1-NEXT:  .LBB1_4:
+; CHECK-T1-NEXT:    subs r5, r7, r5
+; CHECK-T1-NEXT:    subs r6, r5, #1
+; CHECK-T1-NEXT:    sbcs r5, r6
+; CHECK-T1-NEXT:    ldr r6, [sp] @ 4-byte Reload
+; CHECK-T1-NEXT:    subs r6, r2, r6
+; CHECK-T1-NEXT:    sbcs r1, r3
+; CHECK-T1-NEXT:    cmp r1, #0
+; CHECK-T1-NEXT:    mov r2, r4
+; CHECK-T1-NEXT:    bge .LBB1_6
+; CHECK-T1-NEXT:  @ %bb.5:
+; CHECK-T1-NEXT:    mov r2, r0
+; CHECK-T1-NEXT:  .LBB1_6:
+; CHECK-T1-NEXT:    subs r0, r7, r2
+; CHECK-T1-NEXT:    subs r2, r0, #1
+; CHECK-T1-NEXT:    sbcs r0, r2
+; CHECK-T1-NEXT:    ands r5, r0
+; CHECK-T1-NEXT:    beq .LBB1_8
+; CHECK-T1-NEXT:  @ %bb.7:
+; CHECK-T1-NEXT:    asrs r6, r1, #31
+; CHECK-T1-NEXT:  .LBB1_8:
+; CHECK-T1-NEXT:    cmp r1, #0
+; CHECK-T1-NEXT:    bmi .LBB1_10
+; CHECK-T1-NEXT:  @ %bb.9:
+; CHECK-T1-NEXT:    lsls r2, r4, #31
+; CHECK-T1-NEXT:    cmp r5, #0
+; CHECK-T1-NEXT:    beq .LBB1_11
+; CHECK-T1-NEXT:    b .LBB1_12
+; CHECK-T1-NEXT:  .LBB1_10:
+; CHECK-T1-NEXT:    ldr r2, .LCPI1_0
+; CHECK-T1-NEXT:    cmp r5, #0
+; CHECK-T1-NEXT:    bne .LBB1_12
+; CHECK-T1-NEXT:  .LBB1_11:
+; CHECK-T1-NEXT:    mov r2, r1
+; CHECK-T1-NEXT:  .LBB1_12:
+; CHECK-T1-NEXT:    mov r0, r6
+; CHECK-T1-NEXT:    mov r1, r2
+; CHECK-T1-NEXT:    add sp, #4
+; CHECK-T1-NEXT:    pop {r4, r5, r6, r7, pc}
+; CHECK-T1-NEXT:    .p2align 2
+; CHECK-T1-NEXT:  @ %bb.13:
+; CHECK-T1-NEXT:  .LCPI1_0:
+; CHECK-T1-NEXT:    .long 2147483647 @ 0x7fffffff
+;
+; CHECK-T2-LABEL: func2:
+; CHECK-T2:       @ %bb.0:
+; CHECK-T2-NEXT:    .save {r4, lr}
+; CHECK-T2-NEXT:    push {r4, lr}
+; CHECK-T2-NEXT:    cmp.w r3, #-1
+; CHECK-T2-NEXT:    mov.w lr, #0
+; CHECK-T2-NEXT:    it gt
+; CHECK-T2-NEXT:    movgt.w lr, #1
+; CHECK-T2-NEXT:    cmp.w r1, #-1
+; CHECK-T2-NEXT:    mov.w r4, #0
+; CHECK-T2-NEXT:    mov.w r12, #0
+; CHECK-T2-NEXT:    it gt
+; CHECK-T2-NEXT:    movgt r4, #1
+; CHECK-T2-NEXT:    subs.w lr, r4, lr
+; CHECK-T2-NEXT:    it ne
+; CHECK-T2-NEXT:    movne.w lr, #1
+; CHECK-T2-NEXT:    subs r0, r0, r2
+; CHECK-T2-NEXT:    sbc.w r2, r1, r3
+; CHECK-T2-NEXT:    cmp.w r2, #-1
+; CHECK-T2-NEXT:    it gt
+; CHECK-T2-NEXT:    movgt.w r12, #1
+; CHECK-T2-NEXT:    subs.w r1, r4, r12
+; CHECK-T2-NEXT:    it ne
+; CHECK-T2-NEXT:    movne r1, #1
+; CHECK-T2-NEXT:    ands.w r3, lr, r1
+; CHECK-T2-NEXT:    mov.w r1, #-2147483648
+; CHECK-T2-NEXT:    it ne
+; CHECK-T2-NEXT:    asrne r0, r2, #31
+; CHECK-T2-NEXT:    cmp r2, #0
+; CHECK-T2-NEXT:    it mi
+; CHECK-T2-NEXT:    mvnmi r1, #-2147483648
+; CHECK-T2-NEXT:    cmp r3, #0
+; CHECK-T2-NEXT:    it eq
+; CHECK-T2-NEXT:    moveq r1, r2
+; CHECK-T2-NEXT:    pop {r4, pc}
+;
+; CHECK-ARM-LABEL: func2:
+; CHECK-ARM:       @ %bb.0:
+; CHECK-ARM-NEXT:    .save {r4, lr}
+; CHECK-ARM-NEXT:    push {r4, lr}
+; CHECK-ARM-NEXT:    cmn r3, #1
+; CHECK-ARM-NEXT:    mov lr, #0
+; CHECK-ARM-NEXT:    movgt lr, #1
+; CHECK-ARM-NEXT:    cmn r1, #1
+; CHECK-ARM-NEXT:    mov r4, #0
+; CHECK-ARM-NEXT:    mov r12, #0
+; CHECK-ARM-NEXT:    movgt r4, #1
+; CHECK-ARM-NEXT:    subs lr, r4, lr
+; CHECK-ARM-NEXT:    movne lr, #1
+; CHECK-ARM-NEXT:    subs r0, r0, r2
+; CHECK-ARM-NEXT:    sbc r2, r1, r3
+; CHECK-ARM-NEXT:    cmn r2, #1
+; CHECK-ARM-NEXT:    movgt r12, #1
+; CHECK-ARM-NEXT:    subs r1, r4, r12
+; CHECK-ARM-NEXT:    movne r1, #1
+; CHECK-ARM-NEXT:    ands r3, lr, r1
+; CHECK-ARM-NEXT:    asrne r0, r2, #31
+; CHECK-ARM-NEXT:    mov r1, #-2147483648
+; CHECK-ARM-NEXT:    cmp r2, #0
+; CHECK-ARM-NEXT:    mvnmi r1, #-2147483648
+; CHECK-ARM-NEXT:    cmp r3, #0
+; CHECK-ARM-NEXT:    moveq r1, r2
+; CHECK-ARM-NEXT:    pop {r4, pc}
+  %tmp = call i64 @llvm.ssub.sat.i64(i64 %x, i64 %y)
+  ret i64 %tmp
+}
+
+define signext i16 @func16(i16 signext %x, i16 signext %y) nounwind {
+; CHECK-T1-LABEL: func16:
+; CHECK-T1:       @ %bb.0:
+; CHECK-T1-NEXT:    subs r0, r0, r1
+; CHECK-T1-NEXT:    ldr r1, .LCPI2_0
+; CHECK-T1-NEXT:    cmp r0, r1
+; CHECK-T1-NEXT:    blt .LBB2_2
+; CHECK-T1-NEXT:  @ %bb.1:
+; CHECK-T1-NEXT:    mov r0, r1
+; CHECK-T1-NEXT:  .LBB2_2:
+; CHECK-T1-NEXT:    ldr r1, .LCPI2_1
+; CHECK-T1-NEXT:    cmp r0, r1
+; CHECK-T1-NEXT:    bgt .LBB2_4
+; CHECK-T1-NEXT:  @ %bb.3:
+; CHECK-T1-NEXT:    mov r0, r1
+; CHECK-T1-NEXT:  .LBB2_4:
+; CHECK-T1-NEXT:    bx lr
+; CHECK-T1-NEXT:    .p2align 2
+; CHECK-T1-NEXT:  @ %bb.5:
+; CHECK-T1-NEXT:  .LCPI2_0:
+; CHECK-T1-NEXT:    .long 32767 @ 0x7fff
+; CHECK-T1-NEXT:  .LCPI2_1:
+; CHECK-T1-NEXT:    .long 4294934528 @ 0xffff8000
+;
+; CHECK-T2NODSP-LABEL: func16:
+; CHECK-T2NODSP:       @ %bb.0:
+; CHECK-T2NODSP-NEXT:    subs r0, r0, r1
+; CHECK-T2NODSP-NEXT:    movw r1, #32767
+; CHECK-T2NODSP-NEXT:    cmp r0, r1
+; CHECK-T2NODSP-NEXT:    it lt
+; CHECK-T2NODSP-NEXT:    movlt r1, r0
+; CHECK-T2NODSP-NEXT:    movw r0, #32768
+; CHECK-T2NODSP-NEXT:    cmn.w r1, #32768
+; CHECK-T2NODSP-NEXT:    movt r0, #65535
+; CHECK-T2NODSP-NEXT:    it gt
+; CHECK-T2NODSP-NEXT:    movgt r0, r1
+; CHECK-T2NODSP-NEXT:    bx lr
+;
+; CHECK-T2DSP-LABEL: func16:
+; CHECK-T2DSP:       @ %bb.0:
+; CHECK-T2DSP-NEXT:    qsub16 r0, r0, r1
+; CHECK-T2DSP-NEXT:    sxth r0, r0
+; CHECK-T2DSP-NEXT:    bx lr
+;
+; CHECK-ARMNODPS-LABEL: func16:
+; CHECK-ARMNODPS:       @ %bb.0:
+; CHECK-ARMNODPS-NEXT:    sub r0, r0, r1
+; CHECK-ARMNODPS-NEXT:    mov r1, #255
+; CHECK-ARMNODPS-NEXT:    orr r1, r1, #32512
+; CHECK-ARMNODPS-NEXT:    cmp r0, r1
+; CHECK-ARMNODPS-NEXT:    movlt r1, r0
+; CHECK-ARMNODPS-NEXT:    ldr r0, .LCPI2_0
+; CHECK-ARMNODPS-NEXT:    cmn r1, #32768
+; CHECK-ARMNODPS-NEXT:    movgt r0, r1
+; CHECK-ARMNODPS-NEXT:    bx lr
+; CHECK-ARMNODPS-NEXT:    .p2align 2
+; CHECK-ARMNODPS-NEXT:  @ %bb.1:
+; CHECK-ARMNODPS-NEXT:  .LCPI2_0:
+; CHECK-ARMNODPS-NEXT:    .long 4294934528 @ 0xffff8000
+;
+; CHECK-ARMBASEDSP-LABEL: func16:
+; CHECK-ARMBASEDSP:       @ %bb.0:
+; CHECK-ARMBASEDSP-NEXT:    lsl r0, r0, #16
+; CHECK-ARMBASEDSP-NEXT:    lsl r1, r1, #16
+; CHECK-ARMBASEDSP-NEXT:    qsub r0, r0, r1
+; CHECK-ARMBASEDSP-NEXT:    asr r0, r0, #16
+; CHECK-ARMBASEDSP-NEXT:    bx lr
+;
+; CHECK-ARMDSP-LABEL: func16:
+; CHECK-ARMDSP:       @ %bb.0:
+; CHECK-ARMDSP-NEXT:    qsub16 r0, r0, r1
+; CHECK-ARMDSP-NEXT:    sxth r0, r0
+; CHECK-ARMDSP-NEXT:    bx lr
+  %tmp = call i16 @llvm.ssub.sat.i16(i16 %x, i16 %y)
+  ret i16 %tmp
+}
+
+define signext i8 @func8(i8 signext %x, i8 signext %y) nounwind {
+; CHECK-T1-LABEL: func8:
+; CHECK-T1:       @ %bb.0:
+; CHECK-T1-NEXT:    subs r0, r0, r1
+; CHECK-T1-NEXT:    movs r1, #127
+; CHECK-T1-NEXT:    cmp r0, #127
+; CHECK-T1-NEXT:    blt .LBB3_2
+; CHECK-T1-NEXT:  @ %bb.1:
+; CHECK-T1-NEXT:    mov r0, r1
+; CHECK-T1-NEXT:  .LBB3_2:
+; CHECK-T1-NEXT:    mvns r1, r1
+; CHECK-T1-NEXT:    cmp r0, r1
+; CHECK-T1-NEXT:    bgt .LBB3_4
+; CHECK-T1-NEXT:  @ %bb.3:
+; CHECK-T1-NEXT:    mov r0, r1
+; CHECK-T1-NEXT:  .LBB3_4:
+; CHECK-T1-NEXT:    bx lr
+;
+; CHECK-T2NODSP-LABEL: func8:
+; CHECK-T2NODSP:       @ %bb.0:
+; CHECK-T2NODSP-NEXT:    subs r0, r0, r1
+; CHECK-T2NODSP-NEXT:    cmp r0, #127
+; CHECK-T2NODSP-NEXT:    it ge
+; CHECK-T2NODSP-NEXT:    movge r0, #127
+; CHECK-T2NODSP-NEXT:    cmn.w r0, #128
+; CHECK-T2NODSP-NEXT:    it le
+; CHECK-T2NODSP-NEXT:    mvnle r0, #127
+; CHECK-T2NODSP-NEXT:    bx lr
+;
+; CHECK-T2DSP-LABEL: func8:
+; CHECK-T2DSP:       @ %bb.0:
+; CHECK-T2DSP-NEXT:    qsub8 r0, r0, r1
+; CHECK-T2DSP-NEXT:    sxtb r0, r0
+; CHECK-T2DSP-NEXT:    bx lr
+;
+; CHECK-ARMNODPS-LABEL: func8:
+; CHECK-ARMNODPS:       @ %bb.0:
+; CHECK-ARMNODPS-NEXT:    sub r0, r0, r1
+; CHECK-ARMNODPS-NEXT:    cmp r0, #127
+; CHECK-ARMNODPS-NEXT:    movge r0, #127
+; CHECK-ARMNODPS-NEXT:    cmn r0, #128
+; CHECK-ARMNODPS-NEXT:    mvnle r0, #127
+; CHECK-ARMNODPS-NEXT:    bx lr
+;
+; CHECK-ARMBASEDSP-LABEL: func8:
+; CHECK-ARMBASEDSP:       @ %bb.0:
+; CHECK-ARMBASEDSP-NEXT:    lsl r0, r0, #24
+; CHECK-ARMBASEDSP-NEXT:    lsl r1, r1, #24
+; CHECK-ARMBASEDSP-NEXT:    qsub r0, r0, r1
+; CHECK-ARMBASEDSP-NEXT:    asr r0, r0, #24
+; CHECK-ARMBASEDSP-NEXT:    bx lr
+;
+; CHECK-ARMDSP-LABEL: func8:
+; CHECK-ARMDSP:       @ %bb.0:
+; CHECK-ARMDSP-NEXT:    qsub8 r0, r0, r1
+; CHECK-ARMDSP-NEXT:    sxtb r0, r0
+; CHECK-ARMDSP-NEXT:    bx lr
+  %tmp = call i8 @llvm.ssub.sat.i8(i8 %x, i8 %y)
+  ret i8 %tmp
+}
+
+define signext i4 @func3(i4 signext %x, i4 signext %y) nounwind {
+; CHECK-T1-LABEL: func3:
+; CHECK-T1:       @ %bb.0:
+; CHECK-T1-NEXT:    subs r0, r0, r1
+; CHECK-T1-NEXT:    movs r1, #7
+; CHECK-T1-NEXT:    cmp r0, #7
+; CHECK-T1-NEXT:    blt .LBB4_2
+; CHECK-T1-NEXT:  @ %bb.1:
+; CHECK-T1-NEXT:    mov r0, r1
+; CHECK-T1-NEXT:  .LBB4_2:
+; CHECK-T1-NEXT:    mvns r1, r1
+; CHECK-T1-NEXT:    cmp r0, r1
+; CHECK-T1-NEXT:    bgt .LBB4_4
+; CHECK-T1-NEXT:  @ %bb.3:
+; CHECK-T1-NEXT:    mov r0, r1
+; CHECK-T1-NEXT:  .LBB4_4:
+; CHECK-T1-NEXT:    bx lr
+;
+; CHECK-T2NODSP-LABEL: func3:
+; CHECK-T2NODSP:       @ %bb.0:
+; CHECK-T2NODSP-NEXT:    subs r0, r0, r1
+; CHECK-T2NODSP-NEXT:    cmp r0, #7
+; CHECK-T2NODSP-NEXT:    it ge
+; CHECK-T2NODSP-NEXT:    movge r0, #7
+; CHECK-T2NODSP-NEXT:    cmn.w r0, #8
+; CHECK-T2NODSP-NEXT:    it le
+; CHECK-T2NODSP-NEXT:    mvnle r0, #7
+; CHECK-T2NODSP-NEXT:    bx lr
+;
+; CHECK-T2DSP-LABEL: func3:
+; CHECK-T2DSP:       @ %bb.0:
+; CHECK-T2DSP-NEXT:    lsls r1, r1, #28
+; CHECK-T2DSP-NEXT:    lsls r0, r0, #28
+; CHECK-T2DSP-NEXT:    qsub r0, r0, r1
+; CHECK-T2DSP-NEXT:    asrs r0, r0, #28
+; CHECK-T2DSP-NEXT:    bx lr
+;
+; CHECK-ARMNODPS-LABEL: func3:
+; CHECK-ARMNODPS:       @ %bb.0:
+; CHECK-ARMNODPS-NEXT:    sub r0, r0, r1
+; CHECK-ARMNODPS-NEXT:    cmp r0, #7
+; CHECK-ARMNODPS-NEXT:    movge r0, #7
+; CHECK-ARMNODPS-NEXT:    cmn r0, #8
+; CHECK-ARMNODPS-NEXT:    mvnle r0, #7
+; CHECK-ARMNODPS-NEXT:    bx lr
+;
+; CHECK-ARMBASEDSP-LABEL: func3:
+; CHECK-ARMBASEDSP:       @ %bb.0:
+; CHECK-ARMBASEDSP-NEXT:    lsl r0, r0, #28
+; CHECK-ARMBASEDSP-NEXT:    lsl r1, r1, #28
+; CHECK-ARMBASEDSP-NEXT:    qsub r0, r0, r1
+; CHECK-ARMBASEDSP-NEXT:    asr r0, r0, #28
+; CHECK-ARMBASEDSP-NEXT:    bx lr
+;
+; CHECK-ARMDSP-LABEL: func3:
+; CHECK-ARMDSP:       @ %bb.0:
+; CHECK-ARMDSP-NEXT:    lsl r0, r0, #28
+; CHECK-ARMDSP-NEXT:    lsl r1, r1, #28
+; CHECK-ARMDSP-NEXT:    qsub r0, r0, r1
+; CHECK-ARMDSP-NEXT:    asr r0, r0, #28
+; CHECK-ARMDSP-NEXT:    bx lr
+  %tmp = call i4 @llvm.ssub.sat.i4(i4 %x, i4 %y)
+  ret i4 %tmp
+}
+
+define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
+; CHECK-T1-LABEL: vec:
+; CHECK-T1:       @ %bb.0:
+; CHECK-T1-NEXT:    .save {r4, r5, r6, r7, lr}
+; CHECK-T1-NEXT:    push {r4, r5, r6, r7, lr}
+; CHECK-T1-NEXT:    .pad #12
+; CHECK-T1-NEXT:    sub sp, #12
+; CHECK-T1-NEXT:    str r3, [sp] @ 4-byte Spill
+; CHECK-T1-NEXT:    mov r4, r1
+; CHECK-T1-NEXT:    mov r1, r0
+; CHECK-T1-NEXT:    ldr r5, [sp, #32]
+; CHECK-T1-NEXT:    movs r7, #1
+; CHECK-T1-NEXT:    movs r0, #0
+; CHECK-T1-NEXT:    str r0, [sp, #8] @ 4-byte Spill
+; CHECK-T1-NEXT:    subs r0, r1, r5
+; CHECK-T1-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-T1-NEXT:    mov r6, r7
+; CHECK-T1-NEXT:    bmi .LBB5_2
+; CHECK-T1-NEXT:  @ %bb.1:
+; CHECK-T1-NEXT:    ldr r6, [sp, #8] @ 4-byte Reload
+; CHECK-T1-NEXT:  .LBB5_2:
+; CHECK-T1-NEXT:    lsls r3, r7, #31
+; CHECK-T1-NEXT:    ldr r0, .LCPI5_0
+; CHECK-T1-NEXT:    cmp r6, #0
+; CHECK-T1-NEXT:    mov r6, r0
+; CHECK-T1-NEXT:    bne .LBB5_4
+; CHECK-T1-NEXT:  @ %bb.3:
+; CHECK-T1-NEXT:    mov r6, r3
+; CHECK-T1-NEXT:  .LBB5_4:
+; CHECK-T1-NEXT:    cmp r1, r5
+; CHECK-T1-NEXT:    bvc .LBB5_6
+; CHECK-T1-NEXT:  @ %bb.5:
+; CHECK-T1-NEXT:    str r6, [sp, #4] @ 4-byte Spill
+; CHECK-T1-NEXT:  .LBB5_6:
+; CHECK-T1-NEXT:    ldr r5, [sp, #36]
+; CHECK-T1-NEXT:    subs r1, r4, r5
+; CHECK-T1-NEXT:    mov r6, r7
+; CHECK-T1-NEXT:    bmi .LBB5_8
+; CHECK-T1-NEXT:  @ %bb.7:
+; CHECK-T1-NEXT:    ldr r6, [sp, #8] @ 4-byte Reload
+; CHECK-T1-NEXT:  .LBB5_8:
+; CHECK-T1-NEXT:    cmp r6, #0
+; CHECK-T1-NEXT:    mov r6, r0
+; CHECK-T1-NEXT:    bne .LBB5_10
+; CHECK-T1-NEXT:  @ %bb.9:
+; CHECK-T1-NEXT:    mov r6, r3
+; CHECK-T1-NEXT:  .LBB5_10:
+; CHECK-T1-NEXT:    cmp r4, r5
+; CHECK-T1-NEXT:    bvc .LBB5_12
+; CHECK-T1-NEXT:  @ %bb.11:
+; CHECK-T1-NEXT:    mov r1, r6
+; CHECK-T1-NEXT:  .LBB5_12:
+; CHECK-T1-NEXT:    ldr r5, [sp, #40]
+; CHECK-T1-NEXT:    subs r4, r2, r5
+; CHECK-T1-NEXT:    mov r6, r7
+; CHECK-T1-NEXT:    bmi .LBB5_14
+; CHECK-T1-NEXT:  @ %bb.13:
+; CHECK-T1-NEXT:    ldr r6, [sp, #8] @ 4-byte Reload
+; CHECK-T1-NEXT:  .LBB5_14:
+; CHECK-T1-NEXT:    cmp r6, #0
+; CHECK-T1-NEXT:    mov r6, r0
+; CHECK-T1-NEXT:    bne .LBB5_16
+; CHECK-T1-NEXT:  @ %bb.15:
+; CHECK-T1-NEXT:    mov r6, r3
+; CHECK-T1-NEXT:  .LBB5_16:
+; CHECK-T1-NEXT:    cmp r2, r5
+; CHECK-T1-NEXT:    bvc .LBB5_18
+; CHECK-T1-NEXT:  @ %bb.17:
+; CHECK-T1-NEXT:    mov r4, r6
+; CHECK-T1-NEXT:  .LBB5_18:
+; CHECK-T1-NEXT:    ldr r2, [sp, #44]
+; CHECK-T1-NEXT:    ldr r6, [sp] @ 4-byte Reload
+; CHECK-T1-NEXT:    subs r5, r6, r2
+; CHECK-T1-NEXT:    bpl .LBB5_23
+; CHECK-T1-NEXT:  @ %bb.19:
+; CHECK-T1-NEXT:    cmp r7, #0
+; CHECK-T1-NEXT:    beq .LBB5_24
+; CHECK-T1-NEXT:  .LBB5_20:
+; CHECK-T1-NEXT:    cmp r6, r2
+; CHECK-T1-NEXT:    bvc .LBB5_22
+; CHECK-T1-NEXT:  .LBB5_21:
+; CHECK-T1-NEXT:    mov r5, r0
+; CHECK-T1-NEXT:  .LBB5_22:
+; CHECK-T1-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
+; CHECK-T1-NEXT:    mov r2, r4
+; CHECK-T1-NEXT:    mov r3, r5
+; CHECK-T1-NEXT:    add sp, #12
+; CHECK-T1-NEXT:    pop {r4, r5, r6, r7, pc}
+; CHECK-T1-NEXT:  .LBB5_23:
+; CHECK-T1-NEXT:    ldr r7, [sp, #8] @ 4-byte Reload
+; CHECK-T1-NEXT:    cmp r7, #0
+; CHECK-T1-NEXT:    bne .LBB5_20
+; CHECK-T1-NEXT:  .LBB5_24:
+; CHECK-T1-NEXT:    mov r0, r3
+; CHECK-T1-NEXT:    cmp r6, r2
+; CHECK-T1-NEXT:    bvs .LBB5_21
+; CHECK-T1-NEXT:    b .LBB5_22
+; CHECK-T1-NEXT:    .p2align 2
+; CHECK-T1-NEXT:  @ %bb.25:
+; CHECK-T1-NEXT:  .LCPI5_0:
+; CHECK-T1-NEXT:    .long 2147483647 @ 0x7fffffff
+;
+; CHECK-T2NODSP-LABEL: vec:
+; CHECK-T2NODSP:       @ %bb.0:
+; CHECK-T2NODSP-NEXT:    .save {r4, r5, r6, r7, lr}
+; CHECK-T2NODSP-NEXT:    push {r4, r5, r6, r7, lr}
+; CHECK-T2NODSP-NEXT:    .pad #4
+; CHECK-T2NODSP-NEXT:    sub sp, #4
+; CHECK-T2NODSP-NEXT:    ldr r4, [sp, #24]
+; CHECK-T2NODSP-NEXT:    mov lr, r0
+; CHECK-T2NODSP-NEXT:    ldr r7, [sp, #28]
+; CHECK-T2NODSP-NEXT:    movs r5, #0
+; CHECK-T2NODSP-NEXT:    subs r6, r0, r4
+; CHECK-T2NODSP-NEXT:    mov.w r0, #0
+; CHECK-T2NODSP-NEXT:    it mi
+; CHECK-T2NODSP-NEXT:    movmi r0, #1
+; CHECK-T2NODSP-NEXT:    cmp r0, #0
+; CHECK-T2NODSP-NEXT:    mov.w r0, #-2147483648
+; CHECK-T2NODSP-NEXT:    mov.w r12, #-2147483648
+; CHECK-T2NODSP-NEXT:    it ne
+; CHECK-T2NODSP-NEXT:    mvnne r0, #-2147483648
+; CHECK-T2NODSP-NEXT:    cmp lr, r4
+; CHECK-T2NODSP-NEXT:    it vc
+; CHECK-T2NODSP-NEXT:    movvc r0, r6
+; CHECK-T2NODSP-NEXT:    subs r6, r1, r7
+; CHECK-T2NODSP-NEXT:    mov.w r4, #0
+; CHECK-T2NODSP-NEXT:    mov.w lr, #-2147483648
+; CHECK-T2NODSP-NEXT:    it mi
+; CHECK-T2NODSP-NEXT:    movmi r4, #1
+; CHECK-T2NODSP-NEXT:    cmp r4, #0
+; CHECK-T2NODSP-NEXT:    it ne
+; CHECK-T2NODSP-NEXT:    mvnne lr, #-2147483648
+; CHECK-T2NODSP-NEXT:    cmp r1, r7
+; CHECK-T2NODSP-NEXT:    ldr r1, [sp, #32]
+; CHECK-T2NODSP-NEXT:    mov.w r4, #0
+; CHECK-T2NODSP-NEXT:    it vc
+; CHECK-T2NODSP-NEXT:    movvc lr, r6
+; CHECK-T2NODSP-NEXT:    subs r6, r2, r1
+; CHECK-T2NODSP-NEXT:    it mi
+; CHECK-T2NODSP-NEXT:    movmi r4, #1
+; CHECK-T2NODSP-NEXT:    cmp r4, #0
+; CHECK-T2NODSP-NEXT:    mov.w r4, #-2147483648
+; CHECK-T2NODSP-NEXT:    it ne
+; CHECK-T2NODSP-NEXT:    mvnne r4, #-2147483648
+; CHECK-T2NODSP-NEXT:    cmp r2, r1
+; CHECK-T2NODSP-NEXT:    ldr r1, [sp, #36]
+; CHECK-T2NODSP-NEXT:    it vc
+; CHECK-T2NODSP-NEXT:    movvc r4, r6
+; CHECK-T2NODSP-NEXT:    subs r2, r3, r1
+; CHECK-T2NODSP-NEXT:    it mi
+; CHECK-T2NODSP-NEXT:    movmi r5, #1
+; CHECK-T2NODSP-NEXT:    cmp r5, #0
+; CHECK-T2NODSP-NEXT:    it ne
+; CHECK-T2NODSP-NEXT:    mvnne r12, #-2147483648
+; CHECK-T2NODSP-NEXT:    cmp r3, r1
+; CHECK-T2NODSP-NEXT:    it vc
+; CHECK-T2NODSP-NEXT:    movvc r12, r2
+; CHECK-T2NODSP-NEXT:    mov r1, lr
+; CHECK-T2NODSP-NEXT:    mov r2, r4
+; CHECK-T2NODSP-NEXT:    mov r3, r12
+; CHECK-T2NODSP-NEXT:    add sp, #4
+; CHECK-T2NODSP-NEXT:    pop {r4, r5, r6, r7, pc}
+;
+; CHECK-T2DSP-LABEL: vec:
+; CHECK-T2DSP:       @ %bb.0:
+; CHECK-T2DSP-NEXT:    ldr.w r12, [sp]
+; CHECK-T2DSP-NEXT:    qsub r0, r0, r12
+; CHECK-T2DSP-NEXT:    ldr.w r12, [sp, #4]
+; CHECK-T2DSP-NEXT:    qsub r1, r1, r12
+; CHECK-T2DSP-NEXT:    ldr.w r12, [sp, #8]
+; CHECK-T2DSP-NEXT:    qsub r2, r2, r12
+; CHECK-T2DSP-NEXT:    ldr.w r12, [sp, #12]
+; CHECK-T2DSP-NEXT:    qsub r3, r3, r12
+; CHECK-T2DSP-NEXT:    bx lr
+;
+; CHECK-ARMNODPS-LABEL: vec:
+; CHECK-ARMNODPS:       @ %bb.0:
+; CHECK-ARMNODPS-NEXT:    .save {r4, r5, r6, r7, r11, lr}
+; CHECK-ARMNODPS-NEXT:    push {r4, r5, r6, r7, r11, lr}
+; CHECK-ARMNODPS-NEXT:    ldr r4, [sp, #24]
+; CHECK-ARMNODPS-NEXT:    mov lr, r0
+; CHECK-ARMNODPS-NEXT:    ldr r7, [sp, #28]
+; CHECK-ARMNODPS-NEXT:    mov r5, #0
+; CHECK-ARMNODPS-NEXT:    subs r6, r0, r4
+; CHECK-ARMNODPS-NEXT:    mov r0, #0
+; CHECK-ARMNODPS-NEXT:    movmi r0, #1
+; CHECK-ARMNODPS-NEXT:    cmp r0, #0
+; CHECK-ARMNODPS-NEXT:    mov r0, #-2147483648
+; CHECK-ARMNODPS-NEXT:    mov r12, #-2147483648
+; CHECK-ARMNODPS-NEXT:    mvnne r0, #-2147483648
+; CHECK-ARMNODPS-NEXT:    cmp lr, r4
+; CHECK-ARMNODPS-NEXT:    movvc r0, r6
+; CHECK-ARMNODPS-NEXT:    subs r6, r1, r7
+; CHECK-ARMNODPS-NEXT:    mov r4, #0
+; CHECK-ARMNODPS-NEXT:    mov lr, #-2147483648
+; CHECK-ARMNODPS-NEXT:    movmi r4, #1
+; CHECK-ARMNODPS-NEXT:    cmp r4, #0
+; CHECK-ARMNODPS-NEXT:    mvnne lr, #-2147483648
+; CHECK-ARMNODPS-NEXT:    cmp r1, r7
+; CHECK-ARMNODPS-NEXT:    ldr r1, [sp, #32]
+; CHECK-ARMNODPS-NEXT:    movvc lr, r6
+; CHECK-ARMNODPS-NEXT:    mov r4, #0
+; CHECK-ARMNODPS-NEXT:    subs r6, r2, r1
+; CHECK-ARMNODPS-NEXT:    movmi r4, #1
+; CHECK-ARMNODPS-NEXT:    cmp r4, #0
+; CHECK-ARMNODPS-NEXT:    mov r4, #-2147483648
+; CHECK-ARMNODPS-NEXT:    mvnne r4, #-2147483648
+; CHECK-ARMNODPS-NEXT:    cmp r2, r1
+; CHECK-ARMNODPS-NEXT:    ldr r1, [sp, #36]
+; CHECK-ARMNODPS-NEXT:    movvc r4, r6
+; CHECK-ARMNODPS-NEXT:    subs r2, r3, r1
+; CHECK-ARMNODPS-NEXT:    movmi r5, #1
+; CHECK-ARMNODPS-NEXT:    cmp r5, #0
+; CHECK-ARMNODPS-NEXT:    mvnne r12, #-2147483648
+; CHECK-ARMNODPS-NEXT:    cmp r3, r1
+; CHECK-ARMNODPS-NEXT:    movvc r12, r2
+; CHECK-ARMNODPS-NEXT:    mov r1, lr
+; CHECK-ARMNODPS-NEXT:    mov r2, r4
+; CHECK-ARMNODPS-NEXT:    mov r3, r12
+; CHECK-ARMNODPS-NEXT:    pop {r4, r5, r6, r7, r11, pc}
+;
+; CHECK-ARMBASEDSP-LABEL: vec:
+; CHECK-ARMBASEDSP:       @ %bb.0:
+; CHECK-ARMBASEDSP-NEXT:    ldr r12, [sp]
+; CHECK-ARMBASEDSP-NEXT:    qsub r0, r0, r12
+; CHECK-ARMBASEDSP-NEXT:    ldr r12, [sp, #4]
+; CHECK-ARMBASEDSP-NEXT:    qsub r1, r1, r12
+; CHECK-ARMBASEDSP-NEXT:    ldr r12, [sp, #8]
+; CHECK-ARMBASEDSP-NEXT:    qsub r2, r2, r12
+; CHECK-ARMBASEDSP-NEXT:    ldr r12, [sp, #12]
+; CHECK-ARMBASEDSP-NEXT:    qsub r3, r3, r12
+; CHECK-ARMBASEDSP-NEXT:    bx lr
+;
+; CHECK-ARMDSP-LABEL: vec:
+; CHECK-ARMDSP:       @ %bb.0:
+; CHECK-ARMDSP-NEXT:    ldr r12, [sp]
+; CHECK-ARMDSP-NEXT:    qsub r0, r0, r12
+; CHECK-ARMDSP-NEXT:    ldr r12, [sp, #4]
+; CHECK-ARMDSP-NEXT:    qsub r1, r1, r12
+; CHECK-ARMDSP-NEXT:    ldr r12, [sp, #8]
+; CHECK-ARMDSP-NEXT:    qsub r2, r2, r12
+; CHECK-ARMDSP-NEXT:    ldr r12, [sp, #12]
+; CHECK-ARMDSP-NEXT:    qsub r3, r3, r12
+; CHECK-ARMDSP-NEXT:    bx lr
+  %tmp = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> %x, <4 x i32> %y)
+  ret <4 x i32> %tmp
+}
diff --git a/llvm/test/CodeGen/ARM/ssub_sat_plus.ll b/llvm/test/CodeGen/ARM/ssub_sat_plus.ll
new file mode 100644
index 000000000..147c0a0e5
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/ssub_sat_plus.ll
@@ -0,0 +1,384 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=thumbv6m-none-eabi | FileCheck %s --check-prefix=CHECK-T1
+; RUN: llc < %s -mtriple=thumbv7m-none-eabi | FileCheck %s --check-prefix=CHECK-T2 --check-prefix=CHECK-T2NODSP
+; RUN: llc < %s -mtriple=thumbv7em-none-eabi | FileCheck %s --check-prefix=CHECK-T2 --check-prefix=CHECK-T2DSP
+; RUN: llc < %s -mtriple=armv8a-none-eabi | FileCheck %s --check-prefix=CHECK-ARM
+
+declare i4 @llvm.ssub.sat.i4(i4, i4)
+declare i8 @llvm.ssub.sat.i8(i8, i8)
+declare i16 @llvm.ssub.sat.i16(i16, i16)
+declare i32 @llvm.ssub.sat.i32(i32, i32)
+declare i64 @llvm.ssub.sat.i64(i64, i64)
+
+define i32 @func32(i32 %x, i32 %y, i32 %z) nounwind {
+; CHECK-T1-LABEL: func32:
+; CHECK-T1:       @ %bb.0:
+; CHECK-T1-NEXT:    .save {r4, lr}
+; CHECK-T1-NEXT:    push {r4, lr}
+; CHECK-T1-NEXT:    mov r3, r0
+; CHECK-T1-NEXT:    muls r1, r2, r1
+; CHECK-T1-NEXT:    movs r2, #1
+; CHECK-T1-NEXT:    subs r0, r0, r1
+; CHECK-T1-NEXT:    mov r4, r2
+; CHECK-T1-NEXT:    bmi .LBB0_2
+; CHECK-T1-NEXT:  @ %bb.1:
+; CHECK-T1-NEXT:    movs r4, #0
+; CHECK-T1-NEXT:  .LBB0_2:
+; CHECK-T1-NEXT:    cmp r4, #0
+; CHECK-T1-NEXT:    bne .LBB0_4
+; CHECK-T1-NEXT:  @ %bb.3:
+; CHECK-T1-NEXT:    lsls r2, r2, #31
+; CHECK-T1-NEXT:    cmp r3, r1
+; CHECK-T1-NEXT:    bvs .LBB0_5
+; CHECK-T1-NEXT:    b .LBB0_6
+; CHECK-T1-NEXT:  .LBB0_4:
+; CHECK-T1-NEXT:    ldr r2, .LCPI0_0
+; CHECK-T1-NEXT:    cmp r3, r1
+; CHECK-T1-NEXT:    bvc .LBB0_6
+; CHECK-T1-NEXT:  .LBB0_5:
+; CHECK-T1-NEXT:    mov r0, r2
+; CHECK-T1-NEXT:  .LBB0_6:
+; CHECK-T1-NEXT:    pop {r4, pc}
+; CHECK-T1-NEXT:    .p2align 2
+; CHECK-T1-NEXT:  @ %bb.7:
+; CHECK-T1-NEXT:  .LCPI0_0:
+; CHECK-T1-NEXT:    .long 2147483647 @ 0x7fffffff
+;
+; CHECK-T2NODSP-LABEL: func32:
+; CHECK-T2NODSP:       @ %bb.0:
+; CHECK-T2NODSP-NEXT:    .save {r7, lr}
+; CHECK-T2NODSP-NEXT:    push {r7, lr}
+; CHECK-T2NODSP-NEXT:    mls r12, r1, r2, r0
+; CHECK-T2NODSP-NEXT:    mov.w lr, #0
+; CHECK-T2NODSP-NEXT:    mov.w r3, #-2147483648
+; CHECK-T2NODSP-NEXT:    muls r1, r2, r1
+; CHECK-T2NODSP-NEXT:    cmp.w r12, #0
+; CHECK-T2NODSP-NEXT:    it mi
+; CHECK-T2NODSP-NEXT:    movmi.w lr, #1
+; CHECK-T2NODSP-NEXT:    cmp.w lr, #0
+; CHECK-T2NODSP-NEXT:    it ne
+; CHECK-T2NODSP-NEXT:    mvnne r3, #-2147483648
+; CHECK-T2NODSP-NEXT:    cmp r0, r1
+; CHECK-T2NODSP-NEXT:    it vc
+; CHECK-T2NODSP-NEXT:    movvc r3, r12
+; CHECK-T2NODSP-NEXT:    mov r0, r3
+; CHECK-T2NODSP-NEXT:    pop {r7, pc}
+;
+; CHECK-T2DSP-LABEL: func32:
+; CHECK-T2DSP:       @ %bb.0:
+; CHECK-T2DSP-NEXT:    muls r1, r2, r1
+; CHECK-T2DSP-NEXT:    qsub r0, r0, r1
+; CHECK-T2DSP-NEXT:    bx lr
+;
+; CHECK-ARM-LABEL: func32:
+; CHECK-ARM:       @ %bb.0:
+; CHECK-ARM-NEXT:    mul r1, r1, r2
+; CHECK-ARM-NEXT:    qsub r0, r0, r1
+; CHECK-ARM-NEXT:    bx lr
+  %a = mul i32 %y, %z
+  %tmp = call i32 @llvm.ssub.sat.i32(i32 %x, i32 %a)
+  ret i32 %tmp
+}
+
+define i64 @func64(i64 %x, i64 %y, i64 %z) nounwind {
+; CHECK-T1-LABEL: func64:
+; CHECK-T1:       @ %bb.0:
+; CHECK-T1-NEXT:    .save {r4, r5, r6, r7, lr}
+; CHECK-T1-NEXT:    push {r4, r5, r6, r7, lr}
+; CHECK-T1-NEXT:    .pad #4
+; CHECK-T1-NEXT:    sub sp, #4
+; CHECK-T1-NEXT:    ldr r5, [sp, #28]
+; CHECK-T1-NEXT:    movs r2, #1
+; CHECK-T1-NEXT:    movs r4, #0
+; CHECK-T1-NEXT:    cmp r5, #0
+; CHECK-T1-NEXT:    mov r3, r2
+; CHECK-T1-NEXT:    bge .LBB1_2
+; CHECK-T1-NEXT:  @ %bb.1:
+; CHECK-T1-NEXT:    mov r3, r4
+; CHECK-T1-NEXT:  .LBB1_2:
+; CHECK-T1-NEXT:    cmp r1, #0
+; CHECK-T1-NEXT:    mov r6, r2
+; CHECK-T1-NEXT:    bge .LBB1_4
+; CHECK-T1-NEXT:  @ %bb.3:
+; CHECK-T1-NEXT:    mov r6, r4
+; CHECK-T1-NEXT:  .LBB1_4:
+; CHECK-T1-NEXT:    subs r3, r6, r3
+; CHECK-T1-NEXT:    subs r7, r3, #1
+; CHECK-T1-NEXT:    sbcs r3, r7
+; CHECK-T1-NEXT:    ldr r7, [sp, #24]
+; CHECK-T1-NEXT:    subs r0, r0, r7
+; CHECK-T1-NEXT:    sbcs r1, r5
+; CHECK-T1-NEXT:    cmp r1, #0
+; CHECK-T1-NEXT:    mov r5, r2
+; CHECK-T1-NEXT:    bge .LBB1_6
+; CHECK-T1-NEXT:  @ %bb.5:
+; CHECK-T1-NEXT:    mov r5, r4
+; CHECK-T1-NEXT:  .LBB1_6:
+; CHECK-T1-NEXT:    subs r4, r6, r5
+; CHECK-T1-NEXT:    subs r5, r4, #1
+; CHECK-T1-NEXT:    sbcs r4, r5
+; CHECK-T1-NEXT:    ands r3, r4
+; CHECK-T1-NEXT:    beq .LBB1_8
+; CHECK-T1-NEXT:  @ %bb.7:
+; CHECK-T1-NEXT:    asrs r0, r1, #31
+; CHECK-T1-NEXT:  .LBB1_8:
+; CHECK-T1-NEXT:    cmp r1, #0
+; CHECK-T1-NEXT:    bmi .LBB1_10
+; CHECK-T1-NEXT:  @ %bb.9:
+; CHECK-T1-NEXT:    lsls r2, r2, #31
+; CHECK-T1-NEXT:    cmp r3, #0
+; CHECK-T1-NEXT:    beq .LBB1_11
+; CHECK-T1-NEXT:    b .LBB1_12
+; CHECK-T1-NEXT:  .LBB1_10:
+; CHECK-T1-NEXT:    ldr r2, .LCPI1_0
+; CHECK-T1-NEXT:    cmp r3, #0
+; CHECK-T1-NEXT:    bne .LBB1_12
+; CHECK-T1-NEXT:  .LBB1_11:
+; CHECK-T1-NEXT:    mov r2, r1
+; CHECK-T1-NEXT:  .LBB1_12:
+; CHECK-T1-NEXT:    mov r1, r2
+; CHECK-T1-NEXT:    add sp, #4
+; CHECK-T1-NEXT:    pop {r4, r5, r6, r7, pc}
+; CHECK-T1-NEXT:    .p2align 2
+; CHECK-T1-NEXT:  @ %bb.13:
+; CHECK-T1-NEXT:  .LCPI1_0:
+; CHECK-T1-NEXT:    .long 2147483647 @ 0x7fffffff
+;
+; CHECK-T2-LABEL: func64:
+; CHECK-T2:       @ %bb.0:
+; CHECK-T2-NEXT:    .save {r4, lr}
+; CHECK-T2-NEXT:    push {r4, lr}
+; CHECK-T2-NEXT:    ldr.w r12, [sp, #12]
+; CHECK-T2-NEXT:    movs r2, #0
+; CHECK-T2-NEXT:    movs r3, #0
+; CHECK-T2-NEXT:    ldr r4, [sp, #8]
+; CHECK-T2-NEXT:    cmp.w r12, #-1
+; CHECK-T2-NEXT:    it gt
+; CHECK-T2-NEXT:    movgt r2, #1
+; CHECK-T2-NEXT:    cmp.w r1, #-1
+; CHECK-T2-NEXT:    it gt
+; CHECK-T2-NEXT:    movgt r3, #1
+; CHECK-T2-NEXT:    subs r2, r3, r2
+; CHECK-T2-NEXT:    mov.w lr, #0
+; CHECK-T2-NEXT:    it ne
+; CHECK-T2-NEXT:    movne r2, #1
+; CHECK-T2-NEXT:    subs r0, r0, r4
+; CHECK-T2-NEXT:    sbc.w r4, r1, r12
+; CHECK-T2-NEXT:    cmp.w r4, #-1
+; CHECK-T2-NEXT:    it gt
+; CHECK-T2-NEXT:    movgt.w lr, #1
+; CHECK-T2-NEXT:    subs.w r1, r3, lr
+; CHECK-T2-NEXT:    it ne
+; CHECK-T2-NEXT:    movne r1, #1
+; CHECK-T2-NEXT:    ands r2, r1
+; CHECK-T2-NEXT:    mov.w r1, #-2147483648
+; CHECK-T2-NEXT:    it ne
+; CHECK-T2-NEXT:    asrne r0, r4, #31
+; CHECK-T2-NEXT:    cmp r4, #0
+; CHECK-T2-NEXT:    it mi
+; CHECK-T2-NEXT:    mvnmi r1, #-2147483648
+; CHECK-T2-NEXT:    cmp r2, #0
+; CHECK-T2-NEXT:    it eq
+; CHECK-T2-NEXT:    moveq r1, r4
+; CHECK-T2-NEXT:    pop {r4, pc}
+;
+; CHECK-ARM-LABEL: func64:
+; CHECK-ARM:       @ %bb.0:
+; CHECK-ARM-NEXT:    .save {r4, r5, r11, lr}
+; CHECK-ARM-NEXT:    push {r4, r5, r11, lr}
+; CHECK-ARM-NEXT:    ldr lr, [sp, #20]
+; CHECK-ARM-NEXT:    cmn r1, #1
+; CHECK-ARM-NEXT:    mov r3, #0
+; CHECK-ARM-NEXT:    mov r4, #0
+; CHECK-ARM-NEXT:    movwgt r3, #1
+; CHECK-ARM-NEXT:    cmn lr, #1
+; CHECK-ARM-NEXT:    movwgt r4, #1
+; CHECK-ARM-NEXT:    ldr r12, [sp, #16]
+; CHECK-ARM-NEXT:    subs r4, r3, r4
+; CHECK-ARM-NEXT:    mov r5, #0
+; CHECK-ARM-NEXT:    movwne r4, #1
+; CHECK-ARM-NEXT:    subs r0, r0, r12
+; CHECK-ARM-NEXT:    sbc r2, r1, lr
+; CHECK-ARM-NEXT:    cmn r2, #1
+; CHECK-ARM-NEXT:    movwgt r5, #1
+; CHECK-ARM-NEXT:    subs r1, r3, r5
+; CHECK-ARM-NEXT:    movwne r1, #1
+; CHECK-ARM-NEXT:    ands r3, r4, r1
+; CHECK-ARM-NEXT:    asrne r0, r2, #31
+; CHECK-ARM-NEXT:    mov r1, #-2147483648
+; CHECK-ARM-NEXT:    cmp r2, #0
+; CHECK-ARM-NEXT:    mvnmi r1, #-2147483648
+; CHECK-ARM-NEXT:    cmp r3, #0
+; CHECK-ARM-NEXT:    moveq r1, r2
+; CHECK-ARM-NEXT:    pop {r4, r5, r11, pc}
+  %a = mul i64 %y, %z
+  %tmp = call i64 @llvm.ssub.sat.i64(i64 %x, i64 %z)
+  ret i64 %tmp
+}
+
+define signext i16 @func16(i16 signext %x, i16 signext %y, i16 signext %z) nounwind {
+; CHECK-T1-LABEL: func16:
+; CHECK-T1:       @ %bb.0:
+; CHECK-T1-NEXT:    muls r1, r2, r1
+; CHECK-T1-NEXT:    sxth r1, r1
+; CHECK-T1-NEXT:    subs r0, r0, r1
+; CHECK-T1-NEXT:    ldr r1, .LCPI2_0
+; CHECK-T1-NEXT:    cmp r0, r1
+; CHECK-T1-NEXT:    blt .LBB2_2
+; CHECK-T1-NEXT:  @ %bb.1:
+; CHECK-T1-NEXT:    mov r0, r1
+; CHECK-T1-NEXT:  .LBB2_2:
+; CHECK-T1-NEXT:    ldr r1, .LCPI2_1
+; CHECK-T1-NEXT:    cmp r0, r1
+; CHECK-T1-NEXT:    bgt .LBB2_4
+; CHECK-T1-NEXT:  @ %bb.3:
+; CHECK-T1-NEXT:    mov r0, r1
+; CHECK-T1-NEXT:  .LBB2_4:
+; CHECK-T1-NEXT:    bx lr
+; CHECK-T1-NEXT:    .p2align 2
+; CHECK-T1-NEXT:  @ %bb.5:
+; CHECK-T1-NEXT:  .LCPI2_0:
+; CHECK-T1-NEXT:    .long 32767 @ 0x7fff
+; CHECK-T1-NEXT:  .LCPI2_1:
+; CHECK-T1-NEXT:    .long 4294934528 @ 0xffff8000
+;
+; CHECK-T2NODSP-LABEL: func16:
+; CHECK-T2NODSP:       @ %bb.0:
+; CHECK-T2NODSP-NEXT:    muls r1, r2, r1
+; CHECK-T2NODSP-NEXT:    sxth r1, r1
+; CHECK-T2NODSP-NEXT:    subs r0, r0, r1
+; CHECK-T2NODSP-NEXT:    movw r1, #32767
+; CHECK-T2NODSP-NEXT:    cmp r0, r1
+; CHECK-T2NODSP-NEXT:    it lt
+; CHECK-T2NODSP-NEXT:    movlt r1, r0
+; CHECK-T2NODSP-NEXT:    movw r0, #32768
+; CHECK-T2NODSP-NEXT:    movt r0, #65535
+; CHECK-T2NODSP-NEXT:    cmn.w r1, #32768
+; CHECK-T2NODSP-NEXT:    it gt
+; CHECK-T2NODSP-NEXT:    movgt r0, r1
+; CHECK-T2NODSP-NEXT:    bx lr
+;
+; CHECK-T2DSP-LABEL: func16:
+; CHECK-T2DSP:       @ %bb.0:
+; CHECK-T2DSP-NEXT:    muls r1, r2, r1
+; CHECK-T2DSP-NEXT:    qsub16 r0, r0, r1
+; CHECK-T2DSP-NEXT:    sxth r0, r0
+; CHECK-T2DSP-NEXT:    bx lr
+;
+; CHECK-ARM-LABEL: func16:
+; CHECK-ARM:       @ %bb.0:
+; CHECK-ARM-NEXT:    smulbb r1, r1, r2
+; CHECK-ARM-NEXT:    qsub16 r0, r0, r1
+; CHECK-ARM-NEXT:    sxth r0, r0
+; CHECK-ARM-NEXT:    bx lr
+  %a = mul i16 %y, %z
+  %tmp = call i16 @llvm.ssub.sat.i16(i16 %x, i16 %a)
+  ret i16 %tmp
+}
+
+define signext i8 @func8(i8 signext %x, i8 signext %y, i8 signext %z) nounwind {
+; CHECK-T1-LABEL: func8:
+; CHECK-T1:       @ %bb.0:
+; CHECK-T1-NEXT:    muls r1, r2, r1
+; CHECK-T1-NEXT:    sxtb r1, r1
+; CHECK-T1-NEXT:    subs r0, r0, r1
+; CHECK-T1-NEXT:    movs r1, #127
+; CHECK-T1-NEXT:    cmp r0, #127
+; CHECK-T1-NEXT:    blt .LBB3_2
+; CHECK-T1-NEXT:  @ %bb.1:
+; CHECK-T1-NEXT:    mov r0, r1
+; CHECK-T1-NEXT:  .LBB3_2:
+; CHECK-T1-NEXT:    mvns r1, r1
+; CHECK-T1-NEXT:    cmp r0, r1
+; CHECK-T1-NEXT:    bgt .LBB3_4
+; CHECK-T1-NEXT:  @ %bb.3:
+; CHECK-T1-NEXT:    mov r0, r1
+; CHECK-T1-NEXT:  .LBB3_4:
+; CHECK-T1-NEXT:    bx lr
+;
+; CHECK-T2NODSP-LABEL: func8:
+; CHECK-T2NODSP:       @ %bb.0:
+; CHECK-T2NODSP-NEXT:    muls r1, r2, r1
+; CHECK-T2NODSP-NEXT:    sxtb r1, r1
+; CHECK-T2NODSP-NEXT:    subs r0, r0, r1
+; CHECK-T2NODSP-NEXT:    cmp r0, #127
+; CHECK-T2NODSP-NEXT:    it ge
+; CHECK-T2NODSP-NEXT:    movge r0, #127
+; CHECK-T2NODSP-NEXT:    cmn.w r0, #128
+; CHECK-T2NODSP-NEXT:    it le
+; CHECK-T2NODSP-NEXT:    mvnle r0, #127
+; CHECK-T2NODSP-NEXT:    bx lr
+;
+; CHECK-T2DSP-LABEL: func8:
+; CHECK-T2DSP:       @ %bb.0:
+; CHECK-T2DSP-NEXT:    muls r1, r2, r1
+; CHECK-T2DSP-NEXT:    qsub8 r0, r0, r1
+; CHECK-T2DSP-NEXT:    sxtb r0, r0
+; CHECK-T2DSP-NEXT:    bx lr
+;
+; CHECK-ARM-LABEL: func8:
+; CHECK-ARM:       @ %bb.0:
+; CHECK-ARM-NEXT:    smulbb r1, r1, r2
+; CHECK-ARM-NEXT:    qsub8 r0, r0, r1
+; CHECK-ARM-NEXT:    sxtb r0, r0
+; CHECK-ARM-NEXT:    bx lr
+  %a = mul i8 %y, %z
+  %tmp = call i8 @llvm.ssub.sat.i8(i8 %x, i8 %a)
+  ret i8 %tmp
+}
+
+define signext i4 @func4(i4 signext %x, i4 signext %y, i4 signext %z) nounwind {
+; CHECK-T1-LABEL: func4:
+; CHECK-T1:       @ %bb.0:
+; CHECK-T1-NEXT:    muls r1, r2, r1
+; CHECK-T1-NEXT:    lsls r1, r1, #28
+; CHECK-T1-NEXT:    asrs r1, r1, #28
+; CHECK-T1-NEXT:    subs r0, r0, r1
+; CHECK-T1-NEXT:    movs r1, #7
+; CHECK-T1-NEXT:    cmp r0, #7
+; CHECK-T1-NEXT:    blt .LBB4_2
+; CHECK-T1-NEXT:  @ %bb.1:
+; CHECK-T1-NEXT:    mov r0, r1
+; CHECK-T1-NEXT:  .LBB4_2:
+; CHECK-T1-NEXT:    mvns r1, r1
+; CHECK-T1-NEXT:    cmp r0, r1
+; CHECK-T1-NEXT:    bgt .LBB4_4
+; CHECK-T1-NEXT:  @ %bb.3:
+; CHECK-T1-NEXT:    mov r0, r1
+; CHECK-T1-NEXT:  .LBB4_4:
+; CHECK-T1-NEXT:    bx lr
+;
+; CHECK-T2NODSP-LABEL: func4:
+; CHECK-T2NODSP:       @ %bb.0:
+; CHECK-T2NODSP-NEXT:    muls r1, r2, r1
+; CHECK-T2NODSP-NEXT:    lsls r1, r1, #28
+; CHECK-T2NODSP-NEXT:    sub.w r0, r0, r1, asr #28
+; CHECK-T2NODSP-NEXT:    cmp r0, #7
+; CHECK-T2NODSP-NEXT:    it ge
+; CHECK-T2NODSP-NEXT:    movge r0, #7
+; CHECK-T2NODSP-NEXT:    cmn.w r0, #8
+; CHECK-T2NODSP-NEXT:    it le
+; CHECK-T2NODSP-NEXT:    mvnle r0, #7
+; CHECK-T2NODSP-NEXT:    bx lr
+;
+; CHECK-T2DSP-LABEL: func4:
+; CHECK-T2DSP:       @ %bb.0:
+; CHECK-T2DSP-NEXT:    muls r1, r2, r1
+; CHECK-T2DSP-NEXT:    lsls r0, r0, #28
+; CHECK-T2DSP-NEXT:    lsls r1, r1, #28
+; CHECK-T2DSP-NEXT:    qsub r0, r0, r1
+; CHECK-T2DSP-NEXT:    asrs r0, r0, #28
+; CHECK-T2DSP-NEXT:    bx lr
+;
+; CHECK-ARM-LABEL: func4:
+; CHECK-ARM:       @ %bb.0:
+; CHECK-ARM-NEXT:    smulbb r1, r1, r2
+; CHECK-ARM-NEXT:    lsl r0, r0, #28
+; CHECK-ARM-NEXT:    lsl r1, r1, #28
+; CHECK-ARM-NEXT:    qsub r0, r0, r1
+; CHECK-ARM-NEXT:    asr r0, r0, #28
+; CHECK-ARM-NEXT:    bx lr
+  %a = mul i4 %y, %z
+  %tmp = call i4 @llvm.ssub.sat.i4(i4 %x, i4 %a)
+  ret i4 %tmp
+}
diff --git a/llvm/test/CodeGen/ARM/stack-guard-reassign.ll b/llvm/test/CodeGen/ARM/stack-guard-reassign.ll
index 2ce1d1588..02ee9c067 100644
--- a/llvm/test/CodeGen/ARM/stack-guard-reassign.ll
+++ b/llvm/test/CodeGen/ARM/stack-guard-reassign.ll
@@ -5,10 +5,9 @@
 ; CHECK-LABEL: fn:
 ; CHECK:      sub sp, sp, #32
 ; CHECK-NEXT: sub sp, sp, #65536
+; CHECK-NEXT: ldr r1, .LCPI0_0
+; CHECK-NEXT: ldr r2, [r1]
 ; CHECK-NEXT: add lr, sp, #65536
-; CHECK-NEXT: add r1, lr, #28
-; CHECK-NEXT: ldr r2, .LCPI0_0
-; CHECK-NEXT: ldr r3, [r2]
-; CHECK-NEXT: str r3, [r1]
+; CHECK-NEXT: str r2, [lr, #28]
 ; CHECK: .LCPI0_0:
 ; CHECK-NEXT: .long __stack_chk_guard
diff --git a/llvm/test/CodeGen/ARM/stack-protector-bmovpcb_call.ll b/llvm/test/CodeGen/ARM/stack-protector-bmovpcb_call.ll
index 84bf7ac82..9f37203d8 100644
--- a/llvm/test/CodeGen/ARM/stack-protector-bmovpcb_call.ll
+++ b/llvm/test/CodeGen/ARM/stack-protector-bmovpcb_call.ll
@@ -26,7 +26,7 @@ declare void @llvm.memcpy.p0i8.p0i8.i32(i8* nocapture, i8* nocapture readonly, i
 ; Function Attrs: nounwind optsize
 declare i32 @printf(i8* nocapture readonly, ...) #2
 
-attributes #0 = { nounwind optsize ssp "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind optsize ssp "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind }
-attributes #2 = { nounwind optsize "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { nounwind optsize "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #3 = { nounwind optsize }
diff --git a/llvm/test/CodeGen/ARM/stack-size-section.ll b/llvm/test/CodeGen/ARM/stack-size-section.ll
index 10f156439..7f687bba9 100644
--- a/llvm/test/CodeGen/ARM/stack-size-section.ll
+++ b/llvm/test/CodeGen/ARM/stack-size-section.ll
@@ -29,4 +29,4 @@ define void @dynalloc(i32 %N) #0 {
   ret void
 }
 
-attributes #0 = { "no-frame-pointer-elim"="true" }
+attributes #0 = { "frame-pointer"="all" }
diff --git a/llvm/test/CodeGen/ARM/stack_guard_remat.ll b/llvm/test/CodeGen/ARM/stack_guard_remat.ll
index 9b5677608..eb6538603 100644
--- a/llvm/test/CodeGen/ARM/stack_guard_remat.ll
+++ b/llvm/test/CodeGen/ARM/stack_guard_remat.ll
@@ -67,4 +67,4 @@ declare void @foo3(i32*)
 ; Function Attrs: nounwind
 declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture)
 
-attributes #0 = { nounwind ssp "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind ssp "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/ARM/struct-byval-frame-index.ll b/llvm/test/CodeGen/ARM/struct-byval-frame-index.ll
index bea1ed97f..bffbb83ec 100644
--- a/llvm/test/CodeGen/ARM/struct-byval-frame-index.ll
+++ b/llvm/test/CodeGen/ARM/struct-byval-frame-index.ll
@@ -217,4 +217,4 @@ declare void @update_offset_params(i32, i32) #1
 declare void @RestoreMVBlock8x8(i32, i32, %structN* byval nocapture, i32) #1
 
 attributes #0 = { nounwind }
-attributes #1 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/ARM/swifterror.ll b/llvm/test/CodeGen/ARM/swifterror.ll
index 6424754a9..d96bc0249 100644
--- a/llvm/test/CodeGen/ARM/swifterror.ll
+++ b/llvm/test/CodeGen/ARM/swifterror.ll
@@ -194,7 +194,7 @@ define float @foo_loop(%swift_error** swifterror %error_ptr_ref, i32 %cc, float
 ; CHECK-O0: strb [[ID2]], [{{.*}}[[ID]], #8]
 ; spill r0
 ; CHECK-O0: str r0, [sp{{.*}}]
-; CHECK-O0: vcmpe
+; CHECK-O0: vcmp
 ; CHECK-O0: ble
 ; reload from stack
 ; CHECK-O0: ldr r8
diff --git a/llvm/test/CodeGen/ARM/swiftself.ll b/llvm/test/CodeGen/ARM/swiftself.ll
index 1e06b34c7..6db0ae811 100644
--- a/llvm/test/CodeGen/ARM/swiftself.ll
+++ b/llvm/test/CodeGen/ARM/swiftself.ll
@@ -7,7 +7,7 @@
 ; Parameter with swiftself should be allocated to r10.
 ; CHECK-LABEL: swiftself_param:
 ; CHECK: mov r0, r10
-define i8 *@swiftself_param(i8* swiftself %addr0) "no-frame-pointer-elim"="true" {
+define i8 *@swiftself_param(i8* swiftself %addr0) "frame-pointer"="all" {
     ret i8 *%addr0
 }
 
@@ -15,7 +15,7 @@ define i8 *@swiftself_param(i8* swiftself %addr0) "no-frame-pointer-elim"="true"
 ; CHECK-LABEL: call_swiftself:
 ; CHECK: mov r10, r0
 ; CHECK: bl {{_?}}swiftself_param
-define i8 *@call_swiftself(i8* %arg) "no-frame-pointer-elim"="true" {
+define i8 *@call_swiftself(i8* %arg) "frame-pointer"="all" {
   %res = call i8 *@swiftself_param(i8* swiftself %arg)
   ret i8 *%res
 }
@@ -25,7 +25,7 @@ define i8 *@call_swiftself(i8* %arg) "no-frame-pointer-elim"="true" {
 ; CHECK: push {r10}
 ; ...
 ; CHECK: pop {r10}
-define i8 *@swiftself_clobber(i8* swiftself %addr0) "no-frame-pointer-elim"="true" {
+define i8 *@swiftself_clobber(i8* swiftself %addr0) "frame-pointer"="all" {
   call void asm sideeffect "", "~{r10}"()
   ret i8 *%addr0
 }
@@ -37,7 +37,7 @@ define i8 *@swiftself_clobber(i8* swiftself %addr0) "no-frame-pointer-elim"="tru
 ; OPT: bl {{_?}}swiftself_param
 ; OPT-NOT: mov{{.*}}r10
 ; OPT-NEXT: bl {{_?}}swiftself_param
-define void @swiftself_passthrough(i8* swiftself %addr0) "no-frame-pointer-elim"="true" {
+define void @swiftself_passthrough(i8* swiftself %addr0) "frame-pointer"="all" {
   call i8 *@swiftself_param(i8* swiftself %addr0)
   call i8 *@swiftself_param(i8* swiftself %addr0)
   ret void
@@ -47,7 +47,7 @@ define void @swiftself_passthrough(i8* swiftself %addr0) "no-frame-pointer-elim"
 ; CHECK-LABEL: swiftself_tail:
 ; TAILCALL: b {{_?}}swiftself_param
 ; TAILCALL-NOT: pop
-define i8* @swiftself_tail(i8* swiftself %addr0) "no-frame-pointer-elim"="true" {
+define i8* @swiftself_tail(i8* swiftself %addr0) "frame-pointer"="all" {
   call void asm sideeffect "", "~{r10}"()
   %res = tail call i8* @swiftself_param(i8* swiftself %addr0)
   ret i8* %res
@@ -59,7 +59,7 @@ define i8* @swiftself_tail(i8* swiftself %addr0) "no-frame-pointer-elim"="true"
 ; CHECK: mov r10, r0
 ; CHECK: bl {{_?}}swiftself_param
 ; CHECK: pop
-define i8* @swiftself_notail(i8* swiftself %addr0, i8* %addr1) nounwind "no-frame-pointer-elim"="true" {
+define i8* @swiftself_notail(i8* swiftself %addr0, i8* %addr1) nounwind "frame-pointer"="all" {
   %res = tail call i8* @swiftself_param(i8* swiftself %addr1)
   ret i8* %res
 }
diff --git a/llvm/test/CodeGen/ARM/tail-call-weak.ll b/llvm/test/CodeGen/ARM/tail-call-weak.ll
index e0117dffe..04ba51042 100644
--- a/llvm/test/CodeGen/ARM/tail-call-weak.ll
+++ b/llvm/test/CodeGen/ARM/tail-call-weak.ll
@@ -5,14 +5,17 @@
 declare i8* @f()
 declare extern_weak i8* @g(i8*)
 
-; weak symbol resolution occurs statically in PE/COFF, ensure that we permit
-; tail calls on weak externals when targeting a COFF environment.
 define void @test() {
   %call = tail call i8* @f()
   %call1 = tail call i8* @g(i8* %call)
   ret void
 }
 
-; CHECK-COFF: b g
+; CHECK-COFF: movw r0, :lower16:.refptr.g
+; CHECK-COFF: movt r0, :upper16:.refptr.g
+; CHECK-COFF: ldr r4, [r0]
+; CHECK-COFF: mov r1, r4
+; CHECK-COFF: bx r1
+
 ; CHECK-OTHER: bl {{_?}}g
 
diff --git a/llvm/test/CodeGen/ARM/thumb1_return_sequence.ll b/llvm/test/CodeGen/ARM/thumb1_return_sequence.ll
index a7f78c74f..11e18f134 100644
--- a/llvm/test/CodeGen/ARM/thumb1_return_sequence.ll
+++ b/llvm/test/CodeGen/ARM/thumb1_return_sequence.ll
@@ -57,14 +57,14 @@ entry:
 
 ; Epilogue
 ; --------
-; CHECK-V4T:         ldr [[POP:r[4567]]], [sp, #16]
+; CHECK-V4T:         ldr [[POP:r[4567]]], [sp, #12]
 ; CHECK-V4T-NEXT:    mov lr, [[POP]]
 ; CHECK-V4T-NEXT:    pop {[[SAVED]]}
 ; CHECK-V4T-NEXT:    add sp, #16
 ; CHECK-V4T-NEXT:    bx  lr
 ; CHECK-V5T:         lsls r4
 ; CHECK-V5T-NEXT:    mov sp, r4
-; CHECK-V5T:         ldr [[POP:r[4567]]], [sp, #16]
+; CHECK-V5T:         ldr [[POP:r[4567]]], [sp, #12]
 ; CHECK-V5T-NEXT:    mov lr, [[POP]]
 ; CHECK-V5T-NEXT:    pop {[[SAVED]]}
 ; CHECK-V5T-NEXT:    add sp, #16
diff --git a/llvm/test/CodeGen/ARM/uadd_sat.ll b/llvm/test/CodeGen/ARM/uadd_sat.ll
new file mode 100644
index 000000000..d7c4e977c
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/uadd_sat.ll
@@ -0,0 +1,185 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=thumbv6m-none-eabi | FileCheck %s --check-prefix=CHECK-T1
+; RUN: llc < %s -mtriple=thumbv7m-none-eabi | FileCheck %s --check-prefix=CHECK-T2 --check-prefix=CHECK-T2NODSP
+; RUN: llc < %s -mtriple=thumbv7em-none-eabi | FileCheck %s --check-prefix=CHECK-T2 --check-prefix=CHECK-T2DSP
+; RUN: llc < %s -mtriple=armv8a-none-eabi | FileCheck %s --check-prefix=CHECK-ARM
+
+declare i4 @llvm.uadd.sat.i4(i4, i4)
+declare i8 @llvm.uadd.sat.i8(i8, i8)
+declare i16 @llvm.uadd.sat.i16(i16, i16)
+declare i32 @llvm.uadd.sat.i32(i32, i32)
+declare i64 @llvm.uadd.sat.i64(i64, i64)
+
+define i32 @func(i32 %x, i32 %y) nounwind {
+; CHECK-T1-LABEL: func:
+; CHECK-T1:       @ %bb.0:
+; CHECK-T1-NEXT:    adds r0, r0, r1
+; CHECK-T1-NEXT:    blo .LBB0_2
+; CHECK-T1-NEXT:  @ %bb.1:
+; CHECK-T1-NEXT:    movs r0, #0
+; CHECK-T1-NEXT:    mvns r0, r0
+; CHECK-T1-NEXT:  .LBB0_2:
+; CHECK-T1-NEXT:    bx lr
+;
+; CHECK-T2-LABEL: func:
+; CHECK-T2:       @ %bb.0:
+; CHECK-T2-NEXT:    adds r0, r0, r1
+; CHECK-T2-NEXT:    it hs
+; CHECK-T2-NEXT:    movhs.w r0, #-1
+; CHECK-T2-NEXT:    bx lr
+;
+; CHECK-ARM-LABEL: func:
+; CHECK-ARM:       @ %bb.0:
+; CHECK-ARM-NEXT:    adds r0, r0, r1
+; CHECK-ARM-NEXT:    mvnhs r0, #0
+; CHECK-ARM-NEXT:    bx lr
+  %tmp = call i32 @llvm.uadd.sat.i32(i32 %x, i32 %y)
+  ret i32 %tmp
+}
+
+define i64 @func2(i64 %x, i64 %y) nounwind {
+; CHECK-T1-LABEL: func2:
+; CHECK-T1:       @ %bb.0:
+; CHECK-T1-NEXT:    .save {r4, r5, r7, lr}
+; CHECK-T1-NEXT:    push {r4, r5, r7, lr}
+; CHECK-T1-NEXT:    movs r5, #0
+; CHECK-T1-NEXT:    adds r4, r0, r2
+; CHECK-T1-NEXT:    adcs r1, r3
+; CHECK-T1-NEXT:    mov r3, r5
+; CHECK-T1-NEXT:    adcs r3, r5
+; CHECK-T1-NEXT:    mvns r2, r5
+; CHECK-T1-NEXT:    cmp r3, #0
+; CHECK-T1-NEXT:    mov r0, r2
+; CHECK-T1-NEXT:    beq .LBB1_3
+; CHECK-T1-NEXT:  @ %bb.1:
+; CHECK-T1-NEXT:    cmp r3, #0
+; CHECK-T1-NEXT:    beq .LBB1_4
+; CHECK-T1-NEXT:  .LBB1_2:
+; CHECK-T1-NEXT:    mov r1, r2
+; CHECK-T1-NEXT:    pop {r4, r5, r7, pc}
+; CHECK-T1-NEXT:  .LBB1_3:
+; CHECK-T1-NEXT:    mov r0, r4
+; CHECK-T1-NEXT:    cmp r3, #0
+; CHECK-T1-NEXT:    bne .LBB1_2
+; CHECK-T1-NEXT:  .LBB1_4:
+; CHECK-T1-NEXT:    mov r2, r1
+; CHECK-T1-NEXT:    mov r1, r2
+; CHECK-T1-NEXT:    pop {r4, r5, r7, pc}
+;
+; CHECK-T2-LABEL: func2:
+; CHECK-T2:       @ %bb.0:
+; CHECK-T2-NEXT:    adds r0, r0, r2
+; CHECK-T2-NEXT:    mov.w r12, #0
+; CHECK-T2-NEXT:    adcs r1, r3
+; CHECK-T2-NEXT:    adcs r2, r12, #0
+; CHECK-T2-NEXT:    itt ne
+; CHECK-T2-NEXT:    movne.w r0, #-1
+; CHECK-T2-NEXT:    movne.w r1, #-1
+; CHECK-T2-NEXT:    bx lr
+;
+; CHECK-ARM-LABEL: func2:
+; CHECK-ARM:       @ %bb.0:
+; CHECK-ARM-NEXT:    adds r0, r0, r2
+; CHECK-ARM-NEXT:    mov r12, #0
+; CHECK-ARM-NEXT:    adcs r1, r1, r3
+; CHECK-ARM-NEXT:    adcs r2, r12, #0
+; CHECK-ARM-NEXT:    mvnne r0, #0
+; CHECK-ARM-NEXT:    mvnne r1, #0
+; CHECK-ARM-NEXT:    bx lr
+  %tmp = call i64 @llvm.uadd.sat.i64(i64 %x, i64 %y)
+  ret i64 %tmp
+}
+
+define zeroext i16 @func16(i16 zeroext %x, i16 zeroext %y) nounwind {
+; CHECK-T1-LABEL: func16:
+; CHECK-T1:       @ %bb.0:
+; CHECK-T1-NEXT:    adds r0, r0, r1
+; CHECK-T1-NEXT:    ldr r1, .LCPI2_0
+; CHECK-T1-NEXT:    cmp r0, r1
+; CHECK-T1-NEXT:    blo .LBB2_2
+; CHECK-T1-NEXT:  @ %bb.1:
+; CHECK-T1-NEXT:    mov r0, r1
+; CHECK-T1-NEXT:  .LBB2_2:
+; CHECK-T1-NEXT:    bx lr
+; CHECK-T1-NEXT:    .p2align 2
+; CHECK-T1-NEXT:  @ %bb.3:
+; CHECK-T1-NEXT:  .LCPI2_0:
+; CHECK-T1-NEXT:    .long 65535 @ 0xffff
+;
+; CHECK-T2-LABEL: func16:
+; CHECK-T2:       @ %bb.0:
+; CHECK-T2-NEXT:    add r1, r0
+; CHECK-T2-NEXT:    movw r0, #65535
+; CHECK-T2-NEXT:    cmp r1, r0
+; CHECK-T2-NEXT:    it lo
+; CHECK-T2-NEXT:    movlo r0, r1
+; CHECK-T2-NEXT:    bx lr
+;
+; CHECK-ARM-LABEL: func16:
+; CHECK-ARM:       @ %bb.0:
+; CHECK-ARM-NEXT:    add r1, r0, r1
+; CHECK-ARM-NEXT:    movw r0, #65535
+; CHECK-ARM-NEXT:    cmp r1, r0
+; CHECK-ARM-NEXT:    movlo r0, r1
+; CHECK-ARM-NEXT:    bx lr
+  %tmp = call i16 @llvm.uadd.sat.i16(i16 %x, i16 %y)
+  ret i16 %tmp
+}
+
+define zeroext i8 @func8(i8 zeroext %x, i8 zeroext %y) nounwind {
+; CHECK-T1-LABEL: func8:
+; CHECK-T1:       @ %bb.0:
+; CHECK-T1-NEXT:    adds r0, r0, r1
+; CHECK-T1-NEXT:    cmp r0, #255
+; CHECK-T1-NEXT:    blo .LBB3_2
+; CHECK-T1-NEXT:  @ %bb.1:
+; CHECK-T1-NEXT:    movs r0, #255
+; CHECK-T1-NEXT:  .LBB3_2:
+; CHECK-T1-NEXT:    bx lr
+;
+; CHECK-T2-LABEL: func8:
+; CHECK-T2:       @ %bb.0:
+; CHECK-T2-NEXT:    add r0, r1
+; CHECK-T2-NEXT:    cmp r0, #255
+; CHECK-T2-NEXT:    it hs
+; CHECK-T2-NEXT:    movhs r0, #255
+; CHECK-T2-NEXT:    bx lr
+;
+; CHECK-ARM-LABEL: func8:
+; CHECK-ARM:       @ %bb.0:
+; CHECK-ARM-NEXT:    add r0, r0, r1
+; CHECK-ARM-NEXT:    cmp r0, #255
+; CHECK-ARM-NEXT:    movhs r0, #255
+; CHECK-ARM-NEXT:    bx lr
+  %tmp = call i8 @llvm.uadd.sat.i8(i8 %x, i8 %y)
+  ret i8 %tmp
+}
+
+define zeroext i4 @func3(i4 zeroext %x, i4 zeroext %y) nounwind {
+; CHECK-T1-LABEL: func3:
+; CHECK-T1:       @ %bb.0:
+; CHECK-T1-NEXT:    adds r0, r0, r1
+; CHECK-T1-NEXT:    cmp r0, #15
+; CHECK-T1-NEXT:    blo .LBB4_2
+; CHECK-T1-NEXT:  @ %bb.1:
+; CHECK-T1-NEXT:    movs r0, #15
+; CHECK-T1-NEXT:  .LBB4_2:
+; CHECK-T1-NEXT:    bx lr
+;
+; CHECK-T2-LABEL: func3:
+; CHECK-T2:       @ %bb.0:
+; CHECK-T2-NEXT:    add r0, r1
+; CHECK-T2-NEXT:    cmp r0, #15
+; CHECK-T2-NEXT:    it hs
+; CHECK-T2-NEXT:    movhs r0, #15
+; CHECK-T2-NEXT:    bx lr
+;
+; CHECK-ARM-LABEL: func3:
+; CHECK-ARM:       @ %bb.0:
+; CHECK-ARM-NEXT:    add r0, r0, r1
+; CHECK-ARM-NEXT:    cmp r0, #15
+; CHECK-ARM-NEXT:    movhs r0, #15
+; CHECK-ARM-NEXT:    bx lr
+  %tmp = call i4 @llvm.uadd.sat.i4(i4 %x, i4 %y)
+  ret i4 %tmp
+}
diff --git a/llvm/test/CodeGen/ARM/uadd_sat_plus.ll b/llvm/test/CodeGen/ARM/uadd_sat_plus.ll
new file mode 100644
index 000000000..3ebdafa49
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/uadd_sat_plus.ll
@@ -0,0 +1,232 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=thumbv6m-none-eabi | FileCheck %s --check-prefix=CHECK-T1
+; RUN: llc < %s -mtriple=thumbv7m-none-eabi | FileCheck %s --check-prefix=CHECK-T2 --check-prefix=CHECK-T2NODSP
+; RUN: llc < %s -mtriple=thumbv7em-none-eabi | FileCheck %s --check-prefix=CHECK-T2 --check-prefix=CHECK-T2DSP
+; RUN: llc < %s -mtriple=armv8a-none-eabi | FileCheck %s --check-prefix=CHECK-ARM
+
+declare i4 @llvm.uadd.sat.i4(i4, i4)
+declare i8 @llvm.uadd.sat.i8(i8, i8)
+declare i16 @llvm.uadd.sat.i16(i16, i16)
+declare i32 @llvm.uadd.sat.i32(i32, i32)
+declare i64 @llvm.uadd.sat.i64(i64, i64)
+
+define i32 @func32(i32 %x, i32 %y, i32 %z) nounwind {
+; CHECK-T1-LABEL: func32:
+; CHECK-T1:       @ %bb.0:
+; CHECK-T1-NEXT:    muls r1, r2, r1
+; CHECK-T1-NEXT:    adds r0, r0, r1
+; CHECK-T1-NEXT:    blo .LBB0_2
+; CHECK-T1-NEXT:  @ %bb.1:
+; CHECK-T1-NEXT:    movs r0, #0
+; CHECK-T1-NEXT:    mvns r0, r0
+; CHECK-T1-NEXT:  .LBB0_2:
+; CHECK-T1-NEXT:    bx lr
+;
+; CHECK-T2-LABEL: func32:
+; CHECK-T2:       @ %bb.0:
+; CHECK-T2-NEXT:    muls r1, r2, r1
+; CHECK-T2-NEXT:    adds r0, r0, r1
+; CHECK-T2-NEXT:    it hs
+; CHECK-T2-NEXT:    movhs.w r0, #-1
+; CHECK-T2-NEXT:    bx lr
+;
+; CHECK-ARM-LABEL: func32:
+; CHECK-ARM:       @ %bb.0:
+; CHECK-ARM-NEXT:    mul r1, r1, r2
+; CHECK-ARM-NEXT:    adds r0, r0, r1
+; CHECK-ARM-NEXT:    mvnhs r0, #0
+; CHECK-ARM-NEXT:    bx lr
+  %a = mul i32 %y, %z
+  %tmp = call i32 @llvm.uadd.sat.i32(i32 %x, i32 %a)
+  ret i32 %tmp
+}
+
+define i64 @func64(i64 %x, i64 %y, i64 %z) nounwind {
+; CHECK-T1-LABEL: func64:
+; CHECK-T1:       @ %bb.0:
+; CHECK-T1-NEXT:    .save {r4, r5, r7, lr}
+; CHECK-T1-NEXT:    push {r4, r5, r7, lr}
+; CHECK-T1-NEXT:    movs r5, #0
+; CHECK-T1-NEXT:    ldr r2, [sp, #20]
+; CHECK-T1-NEXT:    ldr r3, [sp, #16]
+; CHECK-T1-NEXT:    adds r3, r0, r3
+; CHECK-T1-NEXT:    adcs r2, r1
+; CHECK-T1-NEXT:    mov r4, r5
+; CHECK-T1-NEXT:    adcs r4, r5
+; CHECK-T1-NEXT:    mvns r1, r5
+; CHECK-T1-NEXT:    cmp r4, #0
+; CHECK-T1-NEXT:    mov r0, r1
+; CHECK-T1-NEXT:    beq .LBB1_3
+; CHECK-T1-NEXT:  @ %bb.1:
+; CHECK-T1-NEXT:    cmp r4, #0
+; CHECK-T1-NEXT:    beq .LBB1_4
+; CHECK-T1-NEXT:  .LBB1_2:
+; CHECK-T1-NEXT:    pop {r4, r5, r7, pc}
+; CHECK-T1-NEXT:  .LBB1_3:
+; CHECK-T1-NEXT:    mov r0, r3
+; CHECK-T1-NEXT:    cmp r4, #0
+; CHECK-T1-NEXT:    bne .LBB1_2
+; CHECK-T1-NEXT:  .LBB1_4:
+; CHECK-T1-NEXT:    mov r1, r2
+; CHECK-T1-NEXT:    pop {r4, r5, r7, pc}
+;
+; CHECK-T2-LABEL: func64:
+; CHECK-T2:       @ %bb.0:
+; CHECK-T2-NEXT:    ldrd r2, r3, [sp]
+; CHECK-T2-NEXT:    mov.w r12, #0
+; CHECK-T2-NEXT:    adds r0, r0, r2
+; CHECK-T2-NEXT:    adcs r1, r3
+; CHECK-T2-NEXT:    adcs r2, r12, #0
+; CHECK-T2-NEXT:    itt ne
+; CHECK-T2-NEXT:    movne.w r0, #-1
+; CHECK-T2-NEXT:    movne.w r1, #-1
+; CHECK-T2-NEXT:    bx lr
+;
+; CHECK-ARM-LABEL: func64:
+; CHECK-ARM:       @ %bb.0:
+; CHECK-ARM-NEXT:    ldr r2, [sp]
+; CHECK-ARM-NEXT:    mov r12, #0
+; CHECK-ARM-NEXT:    ldr r3, [sp, #4]
+; CHECK-ARM-NEXT:    adds r0, r0, r2
+; CHECK-ARM-NEXT:    adcs r1, r1, r3
+; CHECK-ARM-NEXT:    adcs r2, r12, #0
+; CHECK-ARM-NEXT:    mvnne r0, #0
+; CHECK-ARM-NEXT:    mvnne r1, #0
+; CHECK-ARM-NEXT:    bx lr
+  %a = mul i64 %y, %z
+  %tmp = call i64 @llvm.uadd.sat.i64(i64 %x, i64 %z)
+  ret i64 %tmp
+}
+
+define zeroext i16 @func16(i16 zeroext %x, i16 zeroext %y, i16 zeroext %z) nounwind {
+; CHECK-T1-LABEL: func16:
+; CHECK-T1:       @ %bb.0:
+; CHECK-T1-NEXT:    muls r1, r2, r1
+; CHECK-T1-NEXT:    uxth r1, r1
+; CHECK-T1-NEXT:    adds r0, r0, r1
+; CHECK-T1-NEXT:    ldr r1, .LCPI2_0
+; CHECK-T1-NEXT:    cmp r0, r1
+; CHECK-T1-NEXT:    blo .LBB2_2
+; CHECK-T1-NEXT:  @ %bb.1:
+; CHECK-T1-NEXT:    mov r0, r1
+; CHECK-T1-NEXT:  .LBB2_2:
+; CHECK-T1-NEXT:    bx lr
+; CHECK-T1-NEXT:    .p2align 2
+; CHECK-T1-NEXT:  @ %bb.3:
+; CHECK-T1-NEXT:  .LCPI2_0:
+; CHECK-T1-NEXT:    .long 65535 @ 0xffff
+;
+; CHECK-T2NODSP-LABEL: func16:
+; CHECK-T2NODSP:       @ %bb.0:
+; CHECK-T2NODSP-NEXT:    muls r1, r2, r1
+; CHECK-T2NODSP-NEXT:    uxth r1, r1
+; CHECK-T2NODSP-NEXT:    add r1, r0
+; CHECK-T2NODSP-NEXT:    movw r0, #65535
+; CHECK-T2NODSP-NEXT:    cmp r1, r0
+; CHECK-T2NODSP-NEXT:    it lo
+; CHECK-T2NODSP-NEXT:    movlo r0, r1
+; CHECK-T2NODSP-NEXT:    bx lr
+;
+; CHECK-T2DSP-LABEL: func16:
+; CHECK-T2DSP:       @ %bb.0:
+; CHECK-T2DSP-NEXT:    muls r1, r2, r1
+; CHECK-T2DSP-NEXT:    uxtah r1, r0, r1
+; CHECK-T2DSP-NEXT:    movw r0, #65535
+; CHECK-T2DSP-NEXT:    cmp r1, r0
+; CHECK-T2DSP-NEXT:    it lo
+; CHECK-T2DSP-NEXT:    movlo r0, r1
+; CHECK-T2DSP-NEXT:    bx lr
+;
+; CHECK-ARM-LABEL: func16:
+; CHECK-ARM:       @ %bb.0:
+; CHECK-ARM-NEXT:    mul r1, r1, r2
+; CHECK-ARM-NEXT:    uxtah r1, r0, r1
+; CHECK-ARM-NEXT:    movw r0, #65535
+; CHECK-ARM-NEXT:    cmp r1, r0
+; CHECK-ARM-NEXT:    movlo r0, r1
+; CHECK-ARM-NEXT:    bx lr
+  %a = mul i16 %y, %z
+  %tmp = call i16 @llvm.uadd.sat.i16(i16 %x, i16 %a)
+  ret i16 %tmp
+}
+
+define zeroext i8 @func8(i8 zeroext %x, i8 zeroext %y, i8 zeroext %z) nounwind {
+; CHECK-T1-LABEL: func8:
+; CHECK-T1:       @ %bb.0:
+; CHECK-T1-NEXT:    muls r1, r2, r1
+; CHECK-T1-NEXT:    uxtb r1, r1
+; CHECK-T1-NEXT:    adds r0, r0, r1
+; CHECK-T1-NEXT:    cmp r0, #255
+; CHECK-T1-NEXT:    blo .LBB3_2
+; CHECK-T1-NEXT:  @ %bb.1:
+; CHECK-T1-NEXT:    movs r0, #255
+; CHECK-T1-NEXT:  .LBB3_2:
+; CHECK-T1-NEXT:    bx lr
+;
+; CHECK-T2NODSP-LABEL: func8:
+; CHECK-T2NODSP:       @ %bb.0:
+; CHECK-T2NODSP-NEXT:    muls r1, r2, r1
+; CHECK-T2NODSP-NEXT:    uxtb r1, r1
+; CHECK-T2NODSP-NEXT:    add r0, r1
+; CHECK-T2NODSP-NEXT:    cmp r0, #255
+; CHECK-T2NODSP-NEXT:    it hs
+; CHECK-T2NODSP-NEXT:    movhs r0, #255
+; CHECK-T2NODSP-NEXT:    bx lr
+;
+; CHECK-T2DSP-LABEL: func8:
+; CHECK-T2DSP:       @ %bb.0:
+; CHECK-T2DSP-NEXT:    muls r1, r2, r1
+; CHECK-T2DSP-NEXT:    uxtab r0, r0, r1
+; CHECK-T2DSP-NEXT:    cmp r0, #255
+; CHECK-T2DSP-NEXT:    it hs
+; CHECK-T2DSP-NEXT:    movhs r0, #255
+; CHECK-T2DSP-NEXT:    bx lr
+;
+; CHECK-ARM-LABEL: func8:
+; CHECK-ARM:       @ %bb.0:
+; CHECK-ARM-NEXT:    smulbb r1, r1, r2
+; CHECK-ARM-NEXT:    uxtab r0, r0, r1
+; CHECK-ARM-NEXT:    cmp r0, #255
+; CHECK-ARM-NEXT:    movhs r0, #255
+; CHECK-ARM-NEXT:    bx lr
+  %a = mul i8 %y, %z
+  %tmp = call i8 @llvm.uadd.sat.i8(i8 %x, i8 %a)
+  ret i8 %tmp
+}
+
+define zeroext i4 @func4(i4 zeroext %x, i4 zeroext %y, i4 zeroext %z) nounwind {
+; CHECK-T1-LABEL: func4:
+; CHECK-T1:       @ %bb.0:
+; CHECK-T1-NEXT:    muls r1, r2, r1
+; CHECK-T1-NEXT:    movs r2, #15
+; CHECK-T1-NEXT:    ands r1, r2
+; CHECK-T1-NEXT:    adds r0, r0, r1
+; CHECK-T1-NEXT:    cmp r0, #15
+; CHECK-T1-NEXT:    blo .LBB4_2
+; CHECK-T1-NEXT:  @ %bb.1:
+; CHECK-T1-NEXT:    mov r0, r2
+; CHECK-T1-NEXT:  .LBB4_2:
+; CHECK-T1-NEXT:    bx lr
+;
+; CHECK-T2-LABEL: func4:
+; CHECK-T2:       @ %bb.0:
+; CHECK-T2-NEXT:    muls r1, r2, r1
+; CHECK-T2-NEXT:    and r1, r1, #15
+; CHECK-T2-NEXT:    add r0, r1
+; CHECK-T2-NEXT:    cmp r0, #15
+; CHECK-T2-NEXT:    it hs
+; CHECK-T2-NEXT:    movhs r0, #15
+; CHECK-T2-NEXT:    bx lr
+;
+; CHECK-ARM-LABEL: func4:
+; CHECK-ARM:       @ %bb.0:
+; CHECK-ARM-NEXT:    smulbb r1, r1, r2
+; CHECK-ARM-NEXT:    and r1, r1, #15
+; CHECK-ARM-NEXT:    add r0, r0, r1
+; CHECK-ARM-NEXT:    cmp r0, #15
+; CHECK-ARM-NEXT:    movhs r0, #15
+; CHECK-ARM-NEXT:    bx lr
+  %a = mul i4 %y, %z
+  %tmp = call i4 @llvm.uadd.sat.i4(i4 %x, i4 %a)
+  ret i4 %tmp
+}
diff --git a/llvm/test/CodeGen/ARM/unschedule-first-call.ll b/llvm/test/CodeGen/ARM/unschedule-first-call.ll
index 6fb0b2281..7e88afb6a 100644
--- a/llvm/test/CodeGen/ARM/unschedule-first-call.ll
+++ b/llvm/test/CodeGen/ARM/unschedule-first-call.ll
@@ -128,7 +128,7 @@ declare { i64, i1 } @llvm.sadd.with.overflow.i64(i64, i64) #1
 ; Function Attrs: nounwind readnone
 declare { i64, i1 } @llvm.ssub.with.overflow.i64(i64, i64) #1
 
-attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "polly-optimized" "stack-protector-buffer-size"="8" "target-cpu"="arm1176jzf-s" "target-features"="+dsp,+strict-align,+vfp2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "polly-optimized" "stack-protector-buffer-size"="8" "target-cpu"="arm1176jzf-s" "target-features"="+dsp,+strict-align,+vfp2" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !llvm.ident = !{!0}
diff --git a/llvm/test/CodeGen/ARM/urem-opt-size.ll b/llvm/test/CodeGen/ARM/urem-opt-size.ll
index bcc536049..a37415d59 100644
--- a/llvm/test/CodeGen/ARM/urem-opt-size.ll
+++ b/llvm/test/CodeGen/ARM/urem-opt-size.ll
@@ -113,5 +113,5 @@ entry:
 declare i32 @GetValue(...) local_unnamed_addr
 
 attributes #0 = { minsize nounwind optsize }
-attributes #4 = { norecurse nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-jump-tables"="false" "stack-protector-buffer-size"="8" "target-cpu"="cortex-a15" "target-features"="+dsp,+hwdiv,+hwdiv-arm,+neon,+vfp4" "use-soft-float"="false" }
+attributes #4 = { norecurse nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-jump-tables"="false" "stack-protector-buffer-size"="8" "target-cpu"="cortex-a15" "target-features"="+dsp,+hwdiv,+hwdiv-arm,+neon,+vfp4" "use-soft-float"="false" }
 
diff --git a/llvm/test/CodeGen/ARM/useaa.ll b/llvm/test/CodeGen/ARM/useaa.ll
index 076466d4d..0cd4732e5 100644
--- a/llvm/test/CodeGen/ARM/useaa.ll
+++ b/llvm/test/CodeGen/ARM/useaa.ll
@@ -7,9 +7,9 @@
 
 ; CHECK-LABEL: test
 ; GENERIC: ldr
-; GENERIC: str
 ; GENERIC: ldr
 ; GENERIC: str
+; GENERIC: str
 ; USEAA: ldr
 ; USEAA: ldr
 ; USEAA: str
diff --git a/llvm/test/CodeGen/ARM/usub_sat.ll b/llvm/test/CodeGen/ARM/usub_sat.ll
new file mode 100644
index 000000000..c348801e5
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/usub_sat.ll
@@ -0,0 +1,188 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=thumbv6m-none-eabi | FileCheck %s --check-prefix=CHECK-T1
+; RUN: llc < %s -mtriple=thumbv7m-none-eabi | FileCheck %s --check-prefix=CHECK-T2 --check-prefix=CHECK-T2NODSP
+; RUN: llc < %s -mtriple=thumbv7em-none-eabi | FileCheck %s --check-prefix=CHECK-T2 --check-prefix=CHECK-T2DSP
+; RUN: llc < %s -mtriple=armv8a-none-eabi | FileCheck %s --check-prefix=CHECK-ARM
+
+declare i4 @llvm.usub.sat.i4(i4, i4)
+declare i8 @llvm.usub.sat.i8(i8, i8)
+declare i16 @llvm.usub.sat.i16(i16, i16)
+declare i32 @llvm.usub.sat.i32(i32, i32)
+declare i64 @llvm.usub.sat.i64(i64, i64)
+
+define i32 @func(i32 %x, i32 %y) nounwind {
+; CHECK-T1-LABEL: func:
+; CHECK-T1:       @ %bb.0:
+; CHECK-T1-NEXT:    subs r0, r0, r1
+; CHECK-T1-NEXT:    bhs .LBB0_2
+; CHECK-T1-NEXT:  @ %bb.1:
+; CHECK-T1-NEXT:    movs r0, #0
+; CHECK-T1-NEXT:  .LBB0_2:
+; CHECK-T1-NEXT:    bx lr
+;
+; CHECK-T2-LABEL: func:
+; CHECK-T2:       @ %bb.0:
+; CHECK-T2-NEXT:    subs r0, r0, r1
+; CHECK-T2-NEXT:    it lo
+; CHECK-T2-NEXT:    movlo r0, #0
+; CHECK-T2-NEXT:    bx lr
+;
+; CHECK-ARM-LABEL: func:
+; CHECK-ARM:       @ %bb.0:
+; CHECK-ARM-NEXT:    subs r0, r0, r1
+; CHECK-ARM-NEXT:    movlo r0, #0
+; CHECK-ARM-NEXT:    bx lr
+  %tmp = call i32 @llvm.usub.sat.i32(i32 %x, i32 %y)
+  ret i32 %tmp
+}
+
+define i64 @func2(i64 %x, i64 %y) nounwind {
+; CHECK-T1-LABEL: func2:
+; CHECK-T1:       @ %bb.0:
+; CHECK-T1-NEXT:    .save {r4, lr}
+; CHECK-T1-NEXT:    push {r4, lr}
+; CHECK-T1-NEXT:    mov r4, r1
+; CHECK-T1-NEXT:    movs r1, #0
+; CHECK-T1-NEXT:    subs r2, r0, r2
+; CHECK-T1-NEXT:    sbcs r4, r3
+; CHECK-T1-NEXT:    mov r0, r1
+; CHECK-T1-NEXT:    adcs r0, r1
+; CHECK-T1-NEXT:    movs r3, #1
+; CHECK-T1-NEXT:    subs r3, r3, r0
+; CHECK-T1-NEXT:    mov r0, r1
+; CHECK-T1-NEXT:    beq .LBB1_3
+; CHECK-T1-NEXT:  @ %bb.1:
+; CHECK-T1-NEXT:    cmp r3, #0
+; CHECK-T1-NEXT:    beq .LBB1_4
+; CHECK-T1-NEXT:  .LBB1_2:
+; CHECK-T1-NEXT:    pop {r4, pc}
+; CHECK-T1-NEXT:  .LBB1_3:
+; CHECK-T1-NEXT:    mov r0, r2
+; CHECK-T1-NEXT:    cmp r3, #0
+; CHECK-T1-NEXT:    bne .LBB1_2
+; CHECK-T1-NEXT:  .LBB1_4:
+; CHECK-T1-NEXT:    mov r1, r4
+; CHECK-T1-NEXT:    pop {r4, pc}
+;
+; CHECK-T2-LABEL: func2:
+; CHECK-T2:       @ %bb.0:
+; CHECK-T2-NEXT:    subs r0, r0, r2
+; CHECK-T2-NEXT:    mov.w r12, #0
+; CHECK-T2-NEXT:    sbcs r1, r3
+; CHECK-T2-NEXT:    adc r2, r12, #0
+; CHECK-T2-NEXT:    rsbs.w r2, r2, #1
+; CHECK-T2-NEXT:    itt ne
+; CHECK-T2-NEXT:    movne r0, #0
+; CHECK-T2-NEXT:    movne r1, #0
+; CHECK-T2-NEXT:    bx lr
+;
+; CHECK-ARM-LABEL: func2:
+; CHECK-ARM:       @ %bb.0:
+; CHECK-ARM-NEXT:    subs r0, r0, r2
+; CHECK-ARM-NEXT:    mov r12, #0
+; CHECK-ARM-NEXT:    sbcs r1, r1, r3
+; CHECK-ARM-NEXT:    adc r2, r12, #0
+; CHECK-ARM-NEXT:    rsbs r2, r2, #1
+; CHECK-ARM-NEXT:    movwne r0, #0
+; CHECK-ARM-NEXT:    movwne r1, #0
+; CHECK-ARM-NEXT:    bx lr
+  %tmp = call i64 @llvm.usub.sat.i64(i64 %x, i64 %y)
+  ret i64 %tmp
+}
+
+define zeroext i16 @func16(i16 zeroext %x, i16 zeroext %y) nounwind {
+; CHECK-T1-LABEL: func16:
+; CHECK-T1:       @ %bb.0:
+; CHECK-T1-NEXT:    cmp r0, r1
+; CHECK-T1-NEXT:    bhi .LBB2_2
+; CHECK-T1-NEXT:  @ %bb.1:
+; CHECK-T1-NEXT:    mov r0, r1
+; CHECK-T1-NEXT:  .LBB2_2:
+; CHECK-T1-NEXT:    subs r0, r0, r1
+; CHECK-T1-NEXT:    uxth r0, r0
+; CHECK-T1-NEXT:    bx lr
+;
+; CHECK-T2-LABEL: func16:
+; CHECK-T2:       @ %bb.0:
+; CHECK-T2-NEXT:    cmp r0, r1
+; CHECK-T2-NEXT:    it ls
+; CHECK-T2-NEXT:    movls r0, r1
+; CHECK-T2-NEXT:    subs r0, r0, r1
+; CHECK-T2-NEXT:    uxth r0, r0
+; CHECK-T2-NEXT:    bx lr
+;
+; CHECK-ARM-LABEL: func16:
+; CHECK-ARM:       @ %bb.0:
+; CHECK-ARM-NEXT:    cmp r0, r1
+; CHECK-ARM-NEXT:    movls r0, r1
+; CHECK-ARM-NEXT:    sub r0, r0, r1
+; CHECK-ARM-NEXT:    uxth r0, r0
+; CHECK-ARM-NEXT:    bx lr
+  %tmp = call i16 @llvm.usub.sat.i16(i16 %x, i16 %y)
+  ret i16 %tmp
+}
+
+define zeroext i8 @func8(i8 zeroext %x, i8 zeroext %y) nounwind {
+; CHECK-T1-LABEL: func8:
+; CHECK-T1:       @ %bb.0:
+; CHECK-T1-NEXT:    cmp r0, r1
+; CHECK-T1-NEXT:    bhi .LBB3_2
+; CHECK-T1-NEXT:  @ %bb.1:
+; CHECK-T1-NEXT:    mov r0, r1
+; CHECK-T1-NEXT:  .LBB3_2:
+; CHECK-T1-NEXT:    subs r0, r0, r1
+; CHECK-T1-NEXT:    uxtb r0, r0
+; CHECK-T1-NEXT:    bx lr
+;
+; CHECK-T2-LABEL: func8:
+; CHECK-T2:       @ %bb.0:
+; CHECK-T2-NEXT:    cmp r0, r1
+; CHECK-T2-NEXT:    it ls
+; CHECK-T2-NEXT:    movls r0, r1
+; CHECK-T2-NEXT:    subs r0, r0, r1
+; CHECK-T2-NEXT:    uxtb r0, r0
+; CHECK-T2-NEXT:    bx lr
+;
+; CHECK-ARM-LABEL: func8:
+; CHECK-ARM:       @ %bb.0:
+; CHECK-ARM-NEXT:    cmp r0, r1
+; CHECK-ARM-NEXT:    movls r0, r1
+; CHECK-ARM-NEXT:    sub r0, r0, r1
+; CHECK-ARM-NEXT:    uxtb r0, r0
+; CHECK-ARM-NEXT:    bx lr
+  %tmp = call i8 @llvm.usub.sat.i8(i8 %x, i8 %y)
+  ret i8 %tmp
+}
+
+define zeroext i4 @func3(i4 zeroext %x, i4 zeroext %y) nounwind {
+; CHECK-T1-LABEL: func3:
+; CHECK-T1:       @ %bb.0:
+; CHECK-T1-NEXT:    cmp r0, r1
+; CHECK-T1-NEXT:    bhi .LBB4_2
+; CHECK-T1-NEXT:  @ %bb.1:
+; CHECK-T1-NEXT:    mov r0, r1
+; CHECK-T1-NEXT:  .LBB4_2:
+; CHECK-T1-NEXT:    subs r1, r0, r1
+; CHECK-T1-NEXT:    movs r0, #15
+; CHECK-T1-NEXT:    ands r0, r1
+; CHECK-T1-NEXT:    bx lr
+;
+; CHECK-T2-LABEL: func3:
+; CHECK-T2:       @ %bb.0:
+; CHECK-T2-NEXT:    cmp r0, r1
+; CHECK-T2-NEXT:    it ls
+; CHECK-T2-NEXT:    movls r0, r1
+; CHECK-T2-NEXT:    subs r0, r0, r1
+; CHECK-T2-NEXT:    and r0, r0, #15
+; CHECK-T2-NEXT:    bx lr
+;
+; CHECK-ARM-LABEL: func3:
+; CHECK-ARM:       @ %bb.0:
+; CHECK-ARM-NEXT:    cmp r0, r1
+; CHECK-ARM-NEXT:    movls r0, r1
+; CHECK-ARM-NEXT:    sub r0, r0, r1
+; CHECK-ARM-NEXT:    and r0, r0, #15
+; CHECK-ARM-NEXT:    bx lr
+  %tmp = call i4 @llvm.usub.sat.i4(i4 %x, i4 %y)
+  ret i4 %tmp
+}
diff --git a/llvm/test/CodeGen/ARM/usub_sat_plus.ll b/llvm/test/CodeGen/ARM/usub_sat_plus.ll
new file mode 100644
index 000000000..0db6c7a82
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/usub_sat_plus.ll
@@ -0,0 +1,220 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=thumbv6m-none-eabi | FileCheck %s --check-prefix=CHECK-T1
+; RUN: llc < %s -mtriple=thumbv7m-none-eabi | FileCheck %s --check-prefix=CHECK-T2 --check-prefix=CHECK-T2NODSP
+; RUN: llc < %s -mtriple=thumbv7em-none-eabi | FileCheck %s --check-prefix=CHECK-T2 --check-prefix=CHECK-T2DSP
+; RUN: llc < %s -mtriple=armv8a-none-eabi | FileCheck %s --check-prefix=CHECK-ARM
+
+declare i4 @llvm.usub.sat.i4(i4, i4)
+declare i8 @llvm.usub.sat.i8(i8, i8)
+declare i16 @llvm.usub.sat.i16(i16, i16)
+declare i32 @llvm.usub.sat.i32(i32, i32)
+declare i64 @llvm.usub.sat.i64(i64, i64)
+
+define i32 @func32(i32 %x, i32 %y, i32 %z) nounwind {
+; CHECK-T1-LABEL: func32:
+; CHECK-T1:       @ %bb.0:
+; CHECK-T1-NEXT:    muls r1, r2, r1
+; CHECK-T1-NEXT:    subs r0, r0, r1
+; CHECK-T1-NEXT:    bhs .LBB0_2
+; CHECK-T1-NEXT:  @ %bb.1:
+; CHECK-T1-NEXT:    movs r0, #0
+; CHECK-T1-NEXT:  .LBB0_2:
+; CHECK-T1-NEXT:    bx lr
+;
+; CHECK-T2-LABEL: func32:
+; CHECK-T2:       @ %bb.0:
+; CHECK-T2-NEXT:    muls r1, r2, r1
+; CHECK-T2-NEXT:    subs r0, r0, r1
+; CHECK-T2-NEXT:    it lo
+; CHECK-T2-NEXT:    movlo r0, #0
+; CHECK-T2-NEXT:    bx lr
+;
+; CHECK-ARM-LABEL: func32:
+; CHECK-ARM:       @ %bb.0:
+; CHECK-ARM-NEXT:    mul r1, r1, r2
+; CHECK-ARM-NEXT:    subs r0, r0, r1
+; CHECK-ARM-NEXT:    movlo r0, #0
+; CHECK-ARM-NEXT:    bx lr
+  %a = mul i32 %y, %z
+  %tmp = call i32 @llvm.usub.sat.i32(i32 %x, i32 %a)
+  ret i32 %tmp
+}
+
+define i64 @func64(i64 %x, i64 %y, i64 %z) nounwind {
+; CHECK-T1-LABEL: func64:
+; CHECK-T1:       @ %bb.0:
+; CHECK-T1-NEXT:    .save {r4, lr}
+; CHECK-T1-NEXT:    push {r4, lr}
+; CHECK-T1-NEXT:    mov r2, r1
+; CHECK-T1-NEXT:    movs r1, #0
+; CHECK-T1-NEXT:    ldr r4, [sp, #12]
+; CHECK-T1-NEXT:    ldr r3, [sp, #8]
+; CHECK-T1-NEXT:    subs r3, r0, r3
+; CHECK-T1-NEXT:    sbcs r2, r4
+; CHECK-T1-NEXT:    mov r0, r1
+; CHECK-T1-NEXT:    adcs r0, r1
+; CHECK-T1-NEXT:    movs r4, #1
+; CHECK-T1-NEXT:    subs r4, r4, r0
+; CHECK-T1-NEXT:    mov r0, r1
+; CHECK-T1-NEXT:    beq .LBB1_3
+; CHECK-T1-NEXT:  @ %bb.1:
+; CHECK-T1-NEXT:    cmp r4, #0
+; CHECK-T1-NEXT:    beq .LBB1_4
+; CHECK-T1-NEXT:  .LBB1_2:
+; CHECK-T1-NEXT:    pop {r4, pc}
+; CHECK-T1-NEXT:  .LBB1_3:
+; CHECK-T1-NEXT:    mov r0, r3
+; CHECK-T1-NEXT:    cmp r4, #0
+; CHECK-T1-NEXT:    bne .LBB1_2
+; CHECK-T1-NEXT:  .LBB1_4:
+; CHECK-T1-NEXT:    mov r1, r2
+; CHECK-T1-NEXT:    pop {r4, pc}
+;
+; CHECK-T2-LABEL: func64:
+; CHECK-T2:       @ %bb.0:
+; CHECK-T2-NEXT:    ldrd r2, r3, [sp]
+; CHECK-T2-NEXT:    mov.w r12, #0
+; CHECK-T2-NEXT:    subs r0, r0, r2
+; CHECK-T2-NEXT:    sbcs r1, r3
+; CHECK-T2-NEXT:    adc r2, r12, #0
+; CHECK-T2-NEXT:    rsbs.w r2, r2, #1
+; CHECK-T2-NEXT:    itt ne
+; CHECK-T2-NEXT:    movne r0, #0
+; CHECK-T2-NEXT:    movne r1, #0
+; CHECK-T2-NEXT:    bx lr
+;
+; CHECK-ARM-LABEL: func64:
+; CHECK-ARM:       @ %bb.0:
+; CHECK-ARM-NEXT:    ldr r2, [sp]
+; CHECK-ARM-NEXT:    mov r12, #0
+; CHECK-ARM-NEXT:    ldr r3, [sp, #4]
+; CHECK-ARM-NEXT:    subs r0, r0, r2
+; CHECK-ARM-NEXT:    sbcs r1, r1, r3
+; CHECK-ARM-NEXT:    adc r2, r12, #0
+; CHECK-ARM-NEXT:    rsbs r2, r2, #1
+; CHECK-ARM-NEXT:    movwne r0, #0
+; CHECK-ARM-NEXT:    movwne r1, #0
+; CHECK-ARM-NEXT:    bx lr
+  %a = mul i64 %y, %z
+  %tmp = call i64 @llvm.usub.sat.i64(i64 %x, i64 %z)
+  ret i64 %tmp
+}
+
+define zeroext i16 @func16(i16 zeroext %x, i16 zeroext %y, i16 zeroext %z) nounwind {
+; CHECK-T1-LABEL: func16:
+; CHECK-T1:       @ %bb.0:
+; CHECK-T1-NEXT:    muls r1, r2, r1
+; CHECK-T1-NEXT:    uxth r2, r1
+; CHECK-T1-NEXT:    cmp r0, r2
+; CHECK-T1-NEXT:    bhi .LBB2_2
+; CHECK-T1-NEXT:  @ %bb.1:
+; CHECK-T1-NEXT:    mov r0, r2
+; CHECK-T1-NEXT:  .LBB2_2:
+; CHECK-T1-NEXT:    subs r0, r0, r1
+; CHECK-T1-NEXT:    uxth r0, r0
+; CHECK-T1-NEXT:    bx lr
+;
+; CHECK-T2-LABEL: func16:
+; CHECK-T2:       @ %bb.0:
+; CHECK-T2-NEXT:    mul r3, r1, r2
+; CHECK-T2-NEXT:    uxth r3, r3
+; CHECK-T2-NEXT:    cmp r0, r3
+; CHECK-T2-NEXT:    it hi
+; CHECK-T2-NEXT:    movhi r3, r0
+; CHECK-T2-NEXT:    mls r0, r1, r2, r3
+; CHECK-T2-NEXT:    uxth r0, r0
+; CHECK-T2-NEXT:    bx lr
+;
+; CHECK-ARM-LABEL: func16:
+; CHECK-ARM:       @ %bb.0:
+; CHECK-ARM-NEXT:    mul r3, r1, r2
+; CHECK-ARM-NEXT:    uxth r3, r3
+; CHECK-ARM-NEXT:    cmp r0, r3
+; CHECK-ARM-NEXT:    movhi r3, r0
+; CHECK-ARM-NEXT:    mls r0, r1, r2, r3
+; CHECK-ARM-NEXT:    uxth r0, r0
+; CHECK-ARM-NEXT:    bx lr
+  %a = mul i16 %y, %z
+  %tmp = call i16 @llvm.usub.sat.i16(i16 %x, i16 %a)
+  ret i16 %tmp
+}
+
+define zeroext i8 @func8(i8 zeroext %x, i8 zeroext %y, i8 zeroext %z) nounwind {
+; CHECK-T1-LABEL: func8:
+; CHECK-T1:       @ %bb.0:
+; CHECK-T1-NEXT:    muls r1, r2, r1
+; CHECK-T1-NEXT:    uxtb r2, r1
+; CHECK-T1-NEXT:    cmp r0, r2
+; CHECK-T1-NEXT:    bhi .LBB3_2
+; CHECK-T1-NEXT:  @ %bb.1:
+; CHECK-T1-NEXT:    mov r0, r2
+; CHECK-T1-NEXT:  .LBB3_2:
+; CHECK-T1-NEXT:    subs r0, r0, r1
+; CHECK-T1-NEXT:    uxtb r0, r0
+; CHECK-T1-NEXT:    bx lr
+;
+; CHECK-T2-LABEL: func8:
+; CHECK-T2:       @ %bb.0:
+; CHECK-T2-NEXT:    mul r3, r1, r2
+; CHECK-T2-NEXT:    uxtb r3, r3
+; CHECK-T2-NEXT:    cmp r0, r3
+; CHECK-T2-NEXT:    it hi
+; CHECK-T2-NEXT:    movhi r3, r0
+; CHECK-T2-NEXT:    mls r0, r1, r2, r3
+; CHECK-T2-NEXT:    uxtb r0, r0
+; CHECK-T2-NEXT:    bx lr
+;
+; CHECK-ARM-LABEL: func8:
+; CHECK-ARM:       @ %bb.0:
+; CHECK-ARM-NEXT:    smulbb r3, r1, r2
+; CHECK-ARM-NEXT:    uxtb r3, r3
+; CHECK-ARM-NEXT:    cmp r0, r3
+; CHECK-ARM-NEXT:    movhi r3, r0
+; CHECK-ARM-NEXT:    mls r0, r1, r2, r3
+; CHECK-ARM-NEXT:    uxtb r0, r0
+; CHECK-ARM-NEXT:    bx lr
+  %a = mul i8 %y, %z
+  %tmp = call i8 @llvm.usub.sat.i8(i8 %x, i8 %a)
+  ret i8 %tmp
+}
+
+define zeroext i4 @func4(i4 zeroext %x, i4 zeroext %y, i4 zeroext %z) nounwind {
+; CHECK-T1-LABEL: func4:
+; CHECK-T1:       @ %bb.0:
+; CHECK-T1-NEXT:    muls r1, r2, r1
+; CHECK-T1-NEXT:    movs r2, #15
+; CHECK-T1-NEXT:    mov r3, r1
+; CHECK-T1-NEXT:    ands r3, r2
+; CHECK-T1-NEXT:    cmp r0, r3
+; CHECK-T1-NEXT:    bhi .LBB4_2
+; CHECK-T1-NEXT:  @ %bb.1:
+; CHECK-T1-NEXT:    mov r0, r3
+; CHECK-T1-NEXT:  .LBB4_2:
+; CHECK-T1-NEXT:    subs r0, r0, r1
+; CHECK-T1-NEXT:    ands r0, r2
+; CHECK-T1-NEXT:    bx lr
+;
+; CHECK-T2-LABEL: func4:
+; CHECK-T2:       @ %bb.0:
+; CHECK-T2-NEXT:    mul r3, r1, r2
+; CHECK-T2-NEXT:    and r3, r3, #15
+; CHECK-T2-NEXT:    cmp r0, r3
+; CHECK-T2-NEXT:    it hi
+; CHECK-T2-NEXT:    movhi r3, r0
+; CHECK-T2-NEXT:    mls r0, r1, r2, r3
+; CHECK-T2-NEXT:    and r0, r0, #15
+; CHECK-T2-NEXT:    bx lr
+;
+; CHECK-ARM-LABEL: func4:
+; CHECK-ARM:       @ %bb.0:
+; CHECK-ARM-NEXT:    smulbb r3, r1, r2
+; CHECK-ARM-NEXT:    and r3, r3, #15
+; CHECK-ARM-NEXT:    cmp r0, r3
+; CHECK-ARM-NEXT:    movhi r3, r0
+; CHECK-ARM-NEXT:    mls r0, r1, r2, r3
+; CHECK-ARM-NEXT:    and r0, r0, #15
+; CHECK-ARM-NEXT:    bx lr
+  %a = mul i4 %y, %z
+  %tmp = call i4 @llvm.usub.sat.i4(i4 %x, i4 %a)
+  ret i4 %tmp
+}
diff --git a/llvm/test/CodeGen/ARM/uxtb.ll b/llvm/test/CodeGen/ARM/uxtb.ll
index 28d951c16..b9caa87e9 100644
--- a/llvm/test/CodeGen/ARM/uxtb.ll
+++ b/llvm/test/CodeGen/ARM/uxtb.ll
@@ -1,50 +1,59 @@
 ; RUN: llc -mtriple armv6-apple-darwin -filetype asm -o - %s | FileCheck %s
 
 define i32 @test1(i32 %x) {
+; CHECK-LABEL: test1:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    uxtb16 r0, r0
+; CHECK-NEXT:    bx lr
   %tmp1 = and i32 %x, 16711935
   ret i32 %tmp1
 }
 
-; CHECK-LABEL: test1:
-; CHECK: uxt
-
 define i32 @test2(i32 %x) {
+; CHECK-LABEL: test2:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    uxtb16 r0, r0, ror #8
+; CHECK-NEXT:    bx lr
   %tmp1 = lshr i32 %x, 8
   %tmp2 = and i32 %tmp1, 16711935
   ret i32 %tmp2
 }
 
-; CHECK-LABEL: test2:
-; CHECK: uxt
-
 define i32 @test3(i32 %x) {
+; CHECK-LABEL: test3:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    uxtb16 r0, r0, ror #8
+; CHECK-NEXT:    bx lr
   %tmp1 = lshr i32 %x, 8
   %tmp2 = and i32 %tmp1, 16711935
   ret i32 %tmp2
 }
 
-; CHECK-LABEL: test3:
-; CHECK: uxt
-
 define i32 @test4(i32 %x) {
+; CHECK-LABEL: test4:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    uxtb16 r0, r0, ror #8
+; CHECK-NEXT:    bx lr
   %tmp1 = lshr i32 %x, 8
   %tmp6 = and i32 %tmp1, 16711935
   ret i32 %tmp6
 }
 
-; CHECK-LABEL: test4:
-; CHECK: uxt
-
 define i32 @test5(i32 %x) {
+; CHECK-LABEL: test5:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    uxtb16 r0, r0, ror #8
+; CHECK-NEXT:    bx lr
   %tmp1 = lshr i32 %x, 8
   %tmp2 = and i32 %tmp1, 16711935
   ret i32 %tmp2
 }
 
-; CHECK-LABEL: test5:
-; CHECK: uxt
-
 define i32 @test6(i32 %x) {
+; CHECK-LABEL: test6:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    uxtb16 r0, r0, ror #16
+; CHECK-NEXT:    bx lr
   %tmp1 = lshr i32 %x, 16
   %tmp2 = and i32 %tmp1, 255
   %tmp4 = shl i32 %x, 16
@@ -53,10 +62,11 @@ define i32 @test6(i32 %x) {
   ret i32 %tmp6
 }
 
-; CHECK-LABEL: test6:
-; CHECK: uxt
-
 define i32 @test7(i32 %x) {
+; CHECK-LABEL: test7:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    uxtb16 r0, r0, ror #16
+; CHECK-NEXT:    bx lr
   %tmp1 = lshr i32 %x, 16
   %tmp2 = and i32 %tmp1, 255
   %tmp4 = shl i32 %x, 16
@@ -65,10 +75,11 @@ define i32 @test7(i32 %x) {
   ret i32 %tmp6
 }
 
-; CHECK-LABEL: test7:
-; CHECK: uxt
-
 define i32 @test8(i32 %x) {
+; CHECK-LABEL: test8:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    uxtb16 r0, r0, ror #24
+; CHECK-NEXT:    bx lr
   %tmp1 = shl i32 %x, 8
   %tmp2 = and i32 %tmp1, 16711680
   %tmp5 = lshr i32 %x, 24
@@ -76,10 +87,11 @@ define i32 @test8(i32 %x) {
   ret i32 %tmp6
 }
 
-; CHECK-LABEL: test8:
-; CHECK: uxt
-
 define i32 @test9(i32 %x) {
+; CHECK-LABEL: test9:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    uxtb16 r0, r0, ror #24
+; CHECK-NEXT:    bx lr
   %tmp1 = lshr i32 %x, 24
   %tmp4 = shl i32 %x, 8
   %tmp5 = and i32 %tmp4, 16711680
@@ -87,10 +99,16 @@ define i32 @test9(i32 %x) {
   ret i32 %tmp6
 }
 
-; CHECK-LABEL: test9:
-; CHECK: uxt
-
 define i32 @test10(i32 %p0) {
+; CHECK-LABEL: test10:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    mov r1, #248
+; CHECK-NEXT:    orr r1, r1, #16252928
+; CHECK-NEXT:    and r0, r1, r0, lsr #7
+; CHECK-NEXT:    lsr r1, r0, #5
+; CHECK-NEXT:    uxtb16 r1, r1
+; CHECK-NEXT:    orr r0, r1, r0
+; CHECK-NEXT:    bx lr
   %tmp1 = lshr i32 %p0, 7
   %tmp2 = and i32 %tmp1, 16253176
   %tmp4 = lshr i32 %tmp2, 5
@@ -99,6 +117,3 @@ define i32 @test10(i32 %p0) {
   ret i32 %tmp7
 }
 
-; CHECK-LABEL: test10:
-; CHECK: uxt
-
diff --git a/llvm/test/CodeGen/ARM/v7k-abi-align.ll b/llvm/test/CodeGen/ARM/v7k-abi-align.ll
index a5cdb8f89..d7a95c0fa 100644
--- a/llvm/test/CodeGen/ARM/v7k-abi-align.ll
+++ b/llvm/test/CodeGen/ARM/v7k-abi-align.ll
@@ -2,25 +2,25 @@
 
 %struct = type { i8, i64, i8, double, i8, <2 x float>, i8, <4 x float> }
 
-define i32 @test_i64_align() "no-frame-pointer-elim"="true" {
+define i32 @test_i64_align() "frame-pointer"="all" {
 ; CHECK-LABEL: test_i64_align:
 ; CHECL: movs r0, #8
   ret i32 ptrtoint(i64* getelementptr(%struct, %struct* null, i32 0, i32 1) to i32)
 }
 
-define i32 @test_f64_align() "no-frame-pointer-elim"="true" {
+define i32 @test_f64_align() "frame-pointer"="all" {
 ; CHECK-LABEL: test_f64_align:
 ; CHECL: movs r0, #24
   ret i32 ptrtoint(double* getelementptr(%struct, %struct* null, i32 0, i32 3) to i32)
 }
 
-define i32 @test_v2f32_align() "no-frame-pointer-elim"="true" {
+define i32 @test_v2f32_align() "frame-pointer"="all" {
 ; CHECK-LABEL: test_v2f32_align:
 ; CHECL: movs r0, #40
   ret i32 ptrtoint(<2 x float>* getelementptr(%struct, %struct* null, i32 0, i32 5) to i32)
 }
 
-define i32 @test_v4f32_align() "no-frame-pointer-elim"="true" {
+define i32 @test_v4f32_align() "frame-pointer"="all" {
 ; CHECK-LABEL: test_v4f32_align:
 ; CHECL: movs r0, #64
   ret i32 ptrtoint(<4 x float>* getelementptr(%struct, %struct* null, i32 0, i32 7) to i32)
@@ -28,7 +28,7 @@ define i32 @test_v4f32_align() "no-frame-pointer-elim"="true" {
 
 ; Key point here is than an extra register has to be saved so that the DPRs end
 ; up in an aligned location (as prologue/epilogue inserter had calculated).
-define void @test_dpr_unwind_align() "no-frame-pointer-elim"="true" {
+define void @test_dpr_unwind_align() "frame-pointer"="all" {
 ; CHECK-LABEL: test_dpr_unwind_align:
 ; CHECK: push {r5, r6, r7, lr}
 ; CHECK-NOT: sub sp
@@ -51,7 +51,7 @@ define void @test_dpr_unwind_align() "no-frame-pointer-elim"="true" {
 
 ; This time, there's no viable way to tack CS-registers onto the list: a real SP
 ; adjustment needs to be performed to put d8 and d9 where they should be.
-define void @test_dpr_unwind_align_manually() "no-frame-pointer-elim"="true" {
+define void @test_dpr_unwind_align_manually() "frame-pointer"="all" {
 ; CHECK-LABEL: test_dpr_unwind_align_manually:
 ; CHECK: push {r4, r5, r6, r7, lr}
 ; CHECK-NOT: sub sp
@@ -76,7 +76,7 @@ define void @test_dpr_unwind_align_manually() "no-frame-pointer-elim"="true" {
 }
 
 ; If there's only a CS1 area, the sub should be in the right place:
-define void @test_dpr_unwind_align_just_cs1() "no-frame-pointer-elim"="true" {
+define void @test_dpr_unwind_align_just_cs1() "frame-pointer"="all" {
 ; CHECK-LABEL: test_dpr_unwind_align_just_cs1:
 ; CHECK: push {r4, r5, r6, r7, lr}
 ; CHECK: sub sp, #4
@@ -99,7 +99,7 @@ define void @test_dpr_unwind_align_just_cs1() "no-frame-pointer-elim"="true" {
 }
 
 ; If there are no DPRs, we shouldn't try to align the stack in stages anyway
-define void @test_dpr_unwind_align_no_dprs() "no-frame-pointer-elim"="true" {
+define void @test_dpr_unwind_align_no_dprs() "frame-pointer"="all" {
 ; CHECK-LABEL: test_dpr_unwind_align_no_dprs:
 ; CHECK: push {r4, r5, r6, r7, lr}
 ; CHECK: sub sp, #12
@@ -117,7 +117,7 @@ define void @test_dpr_unwind_align_no_dprs() "no-frame-pointer-elim"="true" {
 
 ; 128-bit vectors should use 128-bit (i.e. correctly aligned) slots on
 ; the stack.
-define <4 x float> @test_v128_stack_pass([8 x double], float, <4 x float> %in) "no-frame-pointer-elim"="true" {
+define <4 x float> @test_v128_stack_pass([8 x double], float, <4 x float> %in) "frame-pointer"="all" {
 ; CHECK-LABEL: test_v128_stack_pass:
 ; CHECK: add r[[ADDR:[0-9]+]], sp, #16
 ; CHECK: vld1.64 {d0, d1}, [r[[ADDR]]:128]
@@ -129,7 +129,7 @@ declare void @varargs(i32, ...)
 
 ; When varargs are enabled, we go down a different route. Still want 128-bit
 ; alignment though.
-define void @test_v128_stack_pass_varargs(<4 x float> %in) "no-frame-pointer-elim"="true" {
+define void @test_v128_stack_pass_varargs(<4 x float> %in) "frame-pointer"="all" {
 ; CHECK-LABEL: test_v128_stack_pass_varargs:
 ; CHECK: add r[[ADDR:[0-9]+]], sp, #16
 ; CHECK: vst1.64 {d0, d1}, [r[[ADDR]]:128]
@@ -140,7 +140,7 @@ define void @test_v128_stack_pass_varargs(<4 x float> %in) "no-frame-pointer-eli
 
 ; To be compatible with AAPCS's va_start model (store r0-r3 at incoming SP, give
 ; a single pointer), 64-bit quantities must be pass
-define i64 @test_64bit_gpr_align(i32, i64 %r2_r3, i32 %sp) "no-frame-pointer-elim"="true" {
+define i64 @test_64bit_gpr_align(i32, i64 %r2_r3, i32 %sp) "frame-pointer"="all" {
 ; CHECK-LABEL: test_64bit_gpr_align:
 ; CHECK: ldr [[RHS:r[0-9]+]], [sp]
 ; CHECK: adds r0, [[RHS]], r2
diff --git a/llvm/test/CodeGen/ARM/va_arg.ll b/llvm/test/CodeGen/ARM/va_arg.ll
index 57470694b..87e771375 100644
--- a/llvm/test/CodeGen/ARM/va_arg.ll
+++ b/llvm/test/CodeGen/ARM/va_arg.ll
@@ -1,13 +1,27 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=armv7-none-linux-gnueabi -pre-RA-sched=source | FileCheck %s
 ; Test that we correctly align elements when using va_arg
 
-; CHECK-LABEL: test1:
-; CHECK-NOT: bfc
-; CHECK: add	[[REG:(r[0-9]+)|(lr)]], {{(r[0-9]+)|(lr)}}, #7
-; CHECK: bic	{{(r[0-9]+)|(lr)}}, [[REG]], #7
-; CHECK-NOT: bic
-
 define i64 @test1(i32 %i, ...) nounwind optsize {
+; CHECK-LABEL: test1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .pad #12
+; CHECK-NEXT:    sub sp, sp, #12
+; CHECK-NEXT:    .pad #4
+; CHECK-NEXT:    sub sp, sp, #4
+; CHECK-NEXT:    add r0, sp, #4
+; CHECK-NEXT:    stmib sp, {r1, r2, r3}
+; CHECK-NEXT:    add r0, r0, #7
+; CHECK-NEXT:    bic r1, r0, #7
+; CHECK-NEXT:    orr r2, r1, #4
+; CHECK-NEXT:    str r2, [sp]
+; CHECK-NEXT:    ldr r0, [r1]
+; CHECK-NEXT:    add r2, r2, #4
+; CHECK-NEXT:    str r2, [sp]
+; CHECK-NEXT:    ldr r1, [r1, #4]
+; CHECK-NEXT:    add sp, sp, #4
+; CHECK-NEXT:    add sp, sp, #12
+; CHECK-NEXT:    bx lr
 entry:
   %g = alloca i8*, align 4
   %g1 = bitcast i8** %g to i8*
@@ -17,14 +31,25 @@ entry:
   ret i64 %0
 }
 
-; CHECK-LABEL: test2:
-; CHECK-NOT: bfc
-; CHECK: add	[[REG:(r[0-9]+)|(lr)]], {{(r[0-9]+)|(lr)}}, #7
-; CHECK: bic	{{(r[0-9]+)|(lr)}}, [[REG]], #7
-; CHECK-NOT:	bic
-; CHECK: bx	lr
-
 define double @test2(i32 %a, i32* %b, ...) nounwind optsize {
+; CHECK-LABEL: test2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .pad #8
+; CHECK-NEXT:    sub sp, sp, #8
+; CHECK-NEXT:    .pad #4
+; CHECK-NEXT:    sub sp, sp, #4
+; CHECK-NEXT:    add r0, sp, #4
+; CHECK-NEXT:    stmib sp, {r2, r3}
+; CHECK-NEXT:    add r0, r0, #11
+; CHECK-NEXT:    bic r0, r0, #3
+; CHECK-NEXT:    str r2, [r1]
+; CHECK-NEXT:    add r1, r0, #8
+; CHECK-NEXT:    str r1, [sp]
+; CHECK-NEXT:    vldr d16, [r0]
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    add sp, sp, #4
+; CHECK-NEXT:    add sp, sp, #8
+; CHECK-NEXT:    bx lr
 entry:
   %ap = alloca i8*, align 4                       ; <i8**> [#uses=3]
   %ap1 = bitcast i8** %ap to i8*                  ; <i8*> [#uses=2]
diff --git a/llvm/test/CodeGen/ARM/vecreduce-fadd-legalization-soft-float.ll b/llvm/test/CodeGen/ARM/vecreduce-fadd-legalization-soft-float.ll
new file mode 100644
index 000000000..f3eeb11a1
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/vecreduce-fadd-legalization-soft-float.ll
@@ -0,0 +1,63 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=arm-none-eabi -mattr=-neon | FileCheck %s --check-prefix=CHECK
+
+declare float @llvm.experimental.vector.reduce.v2.fadd.f32.v4f32(float, <4 x float>)
+declare double @llvm.experimental.vector.reduce.v2.fadd.f64.v2f64(double, <2 x double>)
+declare fp128 @llvm.experimental.vector.reduce.v2.fadd.f128.v2f128(fp128, <2 x fp128>)
+
+define float @test_v4f32(<4 x float> %a) nounwind {
+; CHECK-LABEL: test_v4f32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    .save {r4, r5, r6, lr}
+; CHECK-NEXT:    push {r4, r5, r6, lr}
+; CHECK-NEXT:    mov r5, r1
+; CHECK-NEXT:    mov r1, r2
+; CHECK-NEXT:    mov r4, r3
+; CHECK-NEXT:    bl __aeabi_fadd
+; CHECK-NEXT:    mov r6, r0
+; CHECK-NEXT:    mov r0, r5
+; CHECK-NEXT:    mov r1, r4
+; CHECK-NEXT:    bl __aeabi_fadd
+; CHECK-NEXT:    mov r1, r0
+; CHECK-NEXT:    mov r0, r6
+; CHECK-NEXT:    bl __aeabi_fadd
+; CHECK-NEXT:    pop {r4, r5, r6, lr}
+; CHECK-NEXT:    mov pc, lr
+  %b = call fast float @llvm.experimental.vector.reduce.v2.fadd.f32.v4f32(float 0.0, <4 x float> %a)
+  ret float %b
+}
+
+define double @test_v2f64(<2 x double> %a) nounwind {
+; CHECK-LABEL: test_v2f64:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    .save {r11, lr}
+; CHECK-NEXT:    push {r11, lr}
+; CHECK-NEXT:    bl __aeabi_dadd
+; CHECK-NEXT:    pop {r11, lr}
+; CHECK-NEXT:    mov pc, lr
+  %b = call fast double @llvm.experimental.vector.reduce.v2.fadd.f64.v2f64(double zeroinitializer, <2 x double> %a)
+  ret double %b
+}
+
+define fp128 @test_v2f128(<2 x fp128> %a) nounwind {
+; CHECK-LABEL: test_v2f128:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    .save {r11, lr}
+; CHECK-NEXT:    push {r11, lr}
+; CHECK-NEXT:    .pad #16
+; CHECK-NEXT:    sub sp, sp, #16
+; CHECK-NEXT:    ldr r12, [sp, #36]
+; CHECK-NEXT:    str r12, [sp, #12]
+; CHECK-NEXT:    ldr r12, [sp, #32]
+; CHECK-NEXT:    str r12, [sp, #8]
+; CHECK-NEXT:    ldr r12, [sp, #28]
+; CHECK-NEXT:    str r12, [sp, #4]
+; CHECK-NEXT:    ldr r12, [sp, #24]
+; CHECK-NEXT:    str r12, [sp]
+; CHECK-NEXT:    bl __addtf3
+; CHECK-NEXT:    add sp, sp, #16
+; CHECK-NEXT:    pop {r11, lr}
+; CHECK-NEXT:    mov pc, lr
+  %b = call fast fp128 @llvm.experimental.vector.reduce.v2.fadd.f128.v2f128(fp128 zeroinitializer, <2 x fp128> %a)
+  ret fp128 %b
+}
diff --git a/llvm/test/CodeGen/ARM/vecreduce-fadd-legalization-strict.ll b/llvm/test/CodeGen/ARM/vecreduce-fadd-legalization-strict.ll
new file mode 100644
index 000000000..f2f4ee289
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/vecreduce-fadd-legalization-strict.ll
@@ -0,0 +1,166 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=arm-none-eabi -mattr=+neon | FileCheck %s --check-prefix=CHECK
+
+declare half @llvm.experimental.vector.reduce.v2.fadd.f16.v1f16(half, <1 x half>)
+declare float @llvm.experimental.vector.reduce.v2.fadd.f32.v1f32(float, <1 x float>)
+declare double @llvm.experimental.vector.reduce.v2.fadd.f64.v1f64(double, <1 x double>)
+declare fp128 @llvm.experimental.vector.reduce.v2.fadd.f128.v1f128(fp128, <1 x fp128>)
+
+declare float @llvm.experimental.vector.reduce.v2.fadd.f32.v3f32(float, <3 x float>)
+declare fp128 @llvm.experimental.vector.reduce.v2.fadd.f128.v2f128(fp128, <2 x fp128>)
+declare float @llvm.experimental.vector.reduce.v2.fadd.f32.v16f32(float, <16 x float>)
+
+define half @test_v1f16(<1 x half> %a) nounwind {
+; CHECK-LABEL: test_v1f16:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    .save {r11, lr}
+; CHECK-NEXT:    push {r11, lr}
+; CHECK-NEXT:    bl __aeabi_f2h
+; CHECK-NEXT:    bl __aeabi_h2f
+; CHECK-NEXT:    vldr s0, .LCPI0_0
+; CHECK-NEXT:    vmov s2, r0
+; CHECK-NEXT:    vadd.f32 s0, s2, s0
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    pop {r11, lr}
+; CHECK-NEXT:    mov pc, lr
+; CHECK-NEXT:    .p2align 2
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI0_0:
+; CHECK-NEXT:    .long 0 @ float 0
+  %b = call half @llvm.experimental.vector.reduce.v2.fadd.f16.v1f16(half 0.0, <1 x half> %a)
+  ret half %b
+}
+
+define float @test_v1f32(<1 x float> %a) nounwind {
+; CHECK-LABEL: test_v1f32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr s0, .LCPI1_0
+; CHECK-NEXT:    vmov s2, r0
+; CHECK-NEXT:    vadd.f32 s0, s2, s0
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    mov pc, lr
+; CHECK-NEXT:    .p2align 2
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI1_0:
+; CHECK-NEXT:    .long 0 @ float 0
+  %b = call float @llvm.experimental.vector.reduce.v2.fadd.f32.v1f32(float 0.0, <1 x float> %a)
+  ret float %b
+}
+
+define double @test_v1f64(<1 x double> %a) nounwind {
+; CHECK-LABEL: test_v1f64:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmov.i32 d16, #0x0
+; CHECK-NEXT:    vmov d17, r0, r1
+; CHECK-NEXT:    vadd.f64 d16, d17, d16
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    mov pc, lr
+  %b = call double @llvm.experimental.vector.reduce.v2.fadd.f64.v1f64(double 0.0, <1 x double> %a)
+  ret double %b
+}
+
+define fp128 @test_v1f128(<1 x fp128> %a) nounwind {
+; CHECK-LABEL: test_v1f128:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    .save {r11, lr}
+; CHECK-NEXT:    push {r11, lr}
+; CHECK-NEXT:    .pad #16
+; CHECK-NEXT:    sub sp, sp, #16
+; CHECK-NEXT:    mov r12, #0
+; CHECK-NEXT:    str r12, [sp]
+; CHECK-NEXT:    str r12, [sp, #4]
+; CHECK-NEXT:    str r12, [sp, #8]
+; CHECK-NEXT:    str r12, [sp, #12]
+; CHECK-NEXT:    bl __addtf3
+; CHECK-NEXT:    add sp, sp, #16
+; CHECK-NEXT:    pop {r11, lr}
+; CHECK-NEXT:    mov pc, lr
+  %b = call fp128 @llvm.experimental.vector.reduce.v2.fadd.f128.v1f128(fp128 zeroinitializer, <1 x fp128> %a)
+  ret fp128 %b
+}
+
+define float @test_v3f32(<3 x float> %a) nounwind {
+; CHECK-LABEL: test_v3f32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmov d3, r2, r3
+; CHECK-NEXT:    vldr s0, .LCPI4_0
+; CHECK-NEXT:    vmov d2, r0, r1
+; CHECK-NEXT:    vadd.f32 s0, s4, s0
+; CHECK-NEXT:    vadd.f32 s0, s0, s5
+; CHECK-NEXT:    vadd.f32 s0, s0, s6
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    mov pc, lr
+; CHECK-NEXT:    .p2align 2
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI4_0:
+; CHECK-NEXT:    .long 0 @ float 0
+  %b = call float @llvm.experimental.vector.reduce.v2.fadd.f32.v3f32(float 0.0, <3 x float> %a)
+  ret float %b
+}
+
+define fp128 @test_v2f128(<2 x fp128> %a) nounwind {
+; CHECK-LABEL: test_v2f128:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    .save {r4, r5, r11, lr}
+; CHECK-NEXT:    push {r4, r5, r11, lr}
+; CHECK-NEXT:    .pad #16
+; CHECK-NEXT:    sub sp, sp, #16
+; CHECK-NEXT:    mov r12, #0
+; CHECK-NEXT:    str r12, [sp]
+; CHECK-NEXT:    str r12, [sp, #4]
+; CHECK-NEXT:    str r12, [sp, #8]
+; CHECK-NEXT:    str r12, [sp, #12]
+; CHECK-NEXT:    bl __addtf3
+; CHECK-NEXT:    ldr r12, [sp, #36]
+; CHECK-NEXT:    ldr lr, [sp, #32]
+; CHECK-NEXT:    ldr r4, [sp, #40]
+; CHECK-NEXT:    ldr r5, [sp, #44]
+; CHECK-NEXT:    str lr, [sp]
+; CHECK-NEXT:    str r12, [sp, #4]
+; CHECK-NEXT:    str r4, [sp, #8]
+; CHECK-NEXT:    str r5, [sp, #12]
+; CHECK-NEXT:    bl __addtf3
+; CHECK-NEXT:    add sp, sp, #16
+; CHECK-NEXT:    pop {r4, r5, r11, lr}
+; CHECK-NEXT:    mov pc, lr
+  %b = call fp128 @llvm.experimental.vector.reduce.v2.fadd.f128.v2f128(fp128 zeroinitializer, <2 x fp128> %a)
+  ret fp128 %b
+}
+
+define float @test_v16f32(<16 x float> %a) nounwind {
+; CHECK-LABEL: test_v16f32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmov d3, r2, r3
+; CHECK-NEXT:    vldr s0, .LCPI6_0
+; CHECK-NEXT:    vmov d2, r0, r1
+; CHECK-NEXT:    mov r0, sp
+; CHECK-NEXT:    vadd.f32 s0, s4, s0
+; CHECK-NEXT:    vadd.f32 s0, s0, s5
+; CHECK-NEXT:    vadd.f32 s0, s0, s6
+; CHECK-NEXT:    vadd.f32 s0, s0, s7
+; CHECK-NEXT:    vld1.64 {d2, d3}, [r0]
+; CHECK-NEXT:    add r0, sp, #16
+; CHECK-NEXT:    vadd.f32 s0, s0, s4
+; CHECK-NEXT:    vadd.f32 s0, s0, s5
+; CHECK-NEXT:    vadd.f32 s0, s0, s6
+; CHECK-NEXT:    vadd.f32 s0, s0, s7
+; CHECK-NEXT:    vld1.64 {d2, d3}, [r0]
+; CHECK-NEXT:    add r0, sp, #32
+; CHECK-NEXT:    vadd.f32 s0, s0, s4
+; CHECK-NEXT:    vadd.f32 s0, s0, s5
+; CHECK-NEXT:    vadd.f32 s0, s0, s6
+; CHECK-NEXT:    vadd.f32 s0, s0, s7
+; CHECK-NEXT:    vld1.64 {d2, d3}, [r0]
+; CHECK-NEXT:    vadd.f32 s0, s0, s4
+; CHECK-NEXT:    vadd.f32 s0, s0, s5
+; CHECK-NEXT:    vadd.f32 s0, s0, s6
+; CHECK-NEXT:    vadd.f32 s0, s0, s7
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    mov pc, lr
+; CHECK-NEXT:    .p2align 2
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI6_0:
+; CHECK-NEXT:    .long 0 @ float 0
+  %b = call float @llvm.experimental.vector.reduce.v2.fadd.f32.v16f32(float 0.0, <16 x float> %a)
+  ret float %b
+}
diff --git a/llvm/test/CodeGen/ARM/vecreduce-fmul-legalization-strict.ll b/llvm/test/CodeGen/ARM/vecreduce-fmul-legalization-strict.ll
new file mode 100644
index 000000000..396cbc806
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/vecreduce-fmul-legalization-strict.ll
@@ -0,0 +1,166 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=arm-none-eabi -mattr=+neon | FileCheck %s --check-prefix=CHECK
+
+declare half @llvm.experimental.vector.reduce.v2.fmul.f16.v1f16(half, <1 x half>)
+declare float @llvm.experimental.vector.reduce.v2.fmul.f32.v1f32(float, <1 x float>)
+declare double @llvm.experimental.vector.reduce.v2.fmul.f64.v1f64(double, <1 x double>)
+declare fp128 @llvm.experimental.vector.reduce.v2.fmul.f128.v1f128(fp128, <1 x fp128>)
+
+declare float @llvm.experimental.vector.reduce.v2.fmul.f32.v3f32(float, <3 x float>)
+declare fp128 @llvm.experimental.vector.reduce.v2.fmul.f128.v2f128(fp128, <2 x fp128>)
+declare float @llvm.experimental.vector.reduce.v2.fmul.f32.v16f32(float, <16 x float>)
+
+define half @test_v1f16(<1 x half> %a) nounwind {
+; CHECK-LABEL: test_v1f16:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    .save {r11, lr}
+; CHECK-NEXT:    push {r11, lr}
+; CHECK-NEXT:    bl __aeabi_f2h
+; CHECK-NEXT:    bl __aeabi_h2f
+; CHECK-NEXT:    vldr s0, .LCPI0_0
+; CHECK-NEXT:    vmov s2, r0
+; CHECK-NEXT:    vmul.f32 s0, s2, s0
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    pop {r11, lr}
+; CHECK-NEXT:    mov pc, lr
+; CHECK-NEXT:    .p2align 2
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI0_0:
+; CHECK-NEXT:    .long 0 @ float 0
+  %b = call half @llvm.experimental.vector.reduce.v2.fmul.f16.v1f16(half 0.0, <1 x half> %a)
+  ret half %b
+}
+
+define float @test_v1f32(<1 x float> %a) nounwind {
+; CHECK-LABEL: test_v1f32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr s0, .LCPI1_0
+; CHECK-NEXT:    vmov s2, r0
+; CHECK-NEXT:    vmul.f32 s0, s2, s0
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    mov pc, lr
+; CHECK-NEXT:    .p2align 2
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI1_0:
+; CHECK-NEXT:    .long 0 @ float 0
+  %b = call float @llvm.experimental.vector.reduce.v2.fmul.f32.v1f32(float 0.0, <1 x float> %a)
+  ret float %b
+}
+
+define double @test_v1f64(<1 x double> %a) nounwind {
+; CHECK-LABEL: test_v1f64:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmov.i32 d16, #0x0
+; CHECK-NEXT:    vmov d17, r0, r1
+; CHECK-NEXT:    vmul.f64 d16, d17, d16
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    mov pc, lr
+  %b = call double @llvm.experimental.vector.reduce.v2.fmul.f64.v1f64(double 0.0, <1 x double> %a)
+  ret double %b
+}
+
+define fp128 @test_v1f128(<1 x fp128> %a) nounwind {
+; CHECK-LABEL: test_v1f128:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    .save {r11, lr}
+; CHECK-NEXT:    push {r11, lr}
+; CHECK-NEXT:    .pad #16
+; CHECK-NEXT:    sub sp, sp, #16
+; CHECK-NEXT:    mov r12, #0
+; CHECK-NEXT:    str r12, [sp]
+; CHECK-NEXT:    str r12, [sp, #4]
+; CHECK-NEXT:    str r12, [sp, #8]
+; CHECK-NEXT:    str r12, [sp, #12]
+; CHECK-NEXT:    bl __multf3
+; CHECK-NEXT:    add sp, sp, #16
+; CHECK-NEXT:    pop {r11, lr}
+; CHECK-NEXT:    mov pc, lr
+  %b = call fp128 @llvm.experimental.vector.reduce.v2.fmul.f128.v1f128(fp128 zeroinitializer, <1 x fp128> %a)
+  ret fp128 %b
+}
+
+define float @test_v3f32(<3 x float> %a) nounwind {
+; CHECK-LABEL: test_v3f32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmov d3, r2, r3
+; CHECK-NEXT:    vldr s0, .LCPI4_0
+; CHECK-NEXT:    vmov d2, r0, r1
+; CHECK-NEXT:    vmul.f32 s0, s4, s0
+; CHECK-NEXT:    vmul.f32 s0, s0, s5
+; CHECK-NEXT:    vmul.f32 s0, s0, s6
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    mov pc, lr
+; CHECK-NEXT:    .p2align 2
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI4_0:
+; CHECK-NEXT:    .long 0 @ float 0
+  %b = call float @llvm.experimental.vector.reduce.v2.fmul.f32.v3f32(float 0.0, <3 x float> %a)
+  ret float %b
+}
+
+define fp128 @test_v2f128(<2 x fp128> %a) nounwind {
+; CHECK-LABEL: test_v2f128:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    .save {r4, r5, r11, lr}
+; CHECK-NEXT:    push {r4, r5, r11, lr}
+; CHECK-NEXT:    .pad #16
+; CHECK-NEXT:    sub sp, sp, #16
+; CHECK-NEXT:    mov r12, #0
+; CHECK-NEXT:    str r12, [sp]
+; CHECK-NEXT:    str r12, [sp, #4]
+; CHECK-NEXT:    str r12, [sp, #8]
+; CHECK-NEXT:    str r12, [sp, #12]
+; CHECK-NEXT:    bl __multf3
+; CHECK-NEXT:    ldr r12, [sp, #36]
+; CHECK-NEXT:    ldr lr, [sp, #32]
+; CHECK-NEXT:    ldr r4, [sp, #40]
+; CHECK-NEXT:    ldr r5, [sp, #44]
+; CHECK-NEXT:    str lr, [sp]
+; CHECK-NEXT:    str r12, [sp, #4]
+; CHECK-NEXT:    str r4, [sp, #8]
+; CHECK-NEXT:    str r5, [sp, #12]
+; CHECK-NEXT:    bl __multf3
+; CHECK-NEXT:    add sp, sp, #16
+; CHECK-NEXT:    pop {r4, r5, r11, lr}
+; CHECK-NEXT:    mov pc, lr
+  %b = call fp128 @llvm.experimental.vector.reduce.v2.fmul.f128.v2f128(fp128 zeroinitializer, <2 x fp128> %a)
+  ret fp128 %b
+}
+
+define float @test_v16f32(<16 x float> %a) nounwind {
+; CHECK-LABEL: test_v16f32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmov d3, r2, r3
+; CHECK-NEXT:    vldr s0, .LCPI6_0
+; CHECK-NEXT:    vmov d2, r0, r1
+; CHECK-NEXT:    mov r0, sp
+; CHECK-NEXT:    vmul.f32 s0, s4, s0
+; CHECK-NEXT:    vmul.f32 s0, s0, s5
+; CHECK-NEXT:    vmul.f32 s0, s0, s6
+; CHECK-NEXT:    vmul.f32 s0, s0, s7
+; CHECK-NEXT:    vld1.64 {d2, d3}, [r0]
+; CHECK-NEXT:    add r0, sp, #16
+; CHECK-NEXT:    vmul.f32 s0, s0, s4
+; CHECK-NEXT:    vmul.f32 s0, s0, s5
+; CHECK-NEXT:    vmul.f32 s0, s0, s6
+; CHECK-NEXT:    vmul.f32 s0, s0, s7
+; CHECK-NEXT:    vld1.64 {d2, d3}, [r0]
+; CHECK-NEXT:    add r0, sp, #32
+; CHECK-NEXT:    vmul.f32 s0, s0, s4
+; CHECK-NEXT:    vmul.f32 s0, s0, s5
+; CHECK-NEXT:    vmul.f32 s0, s0, s6
+; CHECK-NEXT:    vmul.f32 s0, s0, s7
+; CHECK-NEXT:    vld1.64 {d2, d3}, [r0]
+; CHECK-NEXT:    vmul.f32 s0, s0, s4
+; CHECK-NEXT:    vmul.f32 s0, s0, s5
+; CHECK-NEXT:    vmul.f32 s0, s0, s6
+; CHECK-NEXT:    vmul.f32 s0, s0, s7
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    mov pc, lr
+; CHECK-NEXT:    .p2align 2
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI6_0:
+; CHECK-NEXT:    .long 0 @ float 0
+  %b = call float @llvm.experimental.vector.reduce.v2.fmul.f32.v16f32(float 0.0, <16 x float> %a)
+  ret float %b
+}
diff --git a/llvm/test/CodeGen/ARM/vector-spilling.ll b/llvm/test/CodeGen/ARM/vector-spilling.ll
index 914696e6d..2d77393d6 100644
--- a/llvm/test/CodeGen/ARM/vector-spilling.ll
+++ b/llvm/test/CodeGen/ARM/vector-spilling.ll
@@ -31,4 +31,4 @@ entry:
 
 declare void @foo(<8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>)
 
-attributes #0 = { noredzone "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noredzone "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/ARM/vfp.ll b/llvm/test/CodeGen/ARM/vfp.ll
index 8fa5113d8..c18855abd 100644
--- a/llvm/test/CodeGen/ARM/vfp.ll
+++ b/llvm/test/CodeGen/ARM/vfp.ll
@@ -142,7 +142,7 @@ define void @test_cmpfp0(float* %glob, i32 %X) {
 ;CHECK-LABEL: test_cmpfp0:
 entry:
 	%tmp = load float, float* %glob		; <float> [#uses=1]
-;CHECK: vcmpe.f32
+;CHECK: vcmp.f32
 	%tmp.upgrd.3 = fcmp ogt float %tmp, 0.000000e+00		; <i1> [#uses=1]
 	br i1 %tmp.upgrd.3, label %cond_true, label %cond_false
 
diff --git a/llvm/test/CodeGen/ARM/vldm-sched-a9.ll b/llvm/test/CodeGen/ARM/vldm-sched-a9.ll
index 4a4b880a5..1e4692bfc 100644
--- a/llvm/test/CodeGen/ARM/vldm-sched-a9.ll
+++ b/llvm/test/CodeGen/ARM/vldm-sched-a9.ll
@@ -134,4 +134,4 @@ entry:
 
 declare void @capture(i64*, i64*)
 
-attributes #0 = { noredzone "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noredzone "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/ARM/vmul.ll b/llvm/test/CodeGen/ARM/vmul.ll
index fcffe175e..e8cf8d9b2 100644
--- a/llvm/test/CodeGen/ARM/vmul.ll
+++ b/llvm/test/CodeGen/ARM/vmul.ll
@@ -574,7 +574,7 @@ for.body33:                                       ; preds = %for.body33, %for.bo
   %vmovl.i225 = zext <8 x i8> undef to <8 x i16>
   %mul.i223 = mul <8 x i16> %vmovl.i249, %vmovl.i249
   %vshl_n = shl <8 x i16> %mul.i223, <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
-  %vqsub2.i216 = tail call <8 x i16> @llvm.arm.neon.vqsubu.v8i16(<8 x i16> <i16 256, i16 256, i16 256, i16 256, i16 256, i16 256, i16 256, i16 256>, <8 x i16> %vshl_n) nounwind
+  %vqsub2.i216 = tail call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> <i16 256, i16 256, i16 256, i16 256, i16 256, i16 256, i16 256, i16 256>, <8 x i16> %vshl_n) nounwind
   %mul.i209 = mul <8 x i16> undef, <i16 80, i16 80, i16 80, i16 80, i16 80, i16 80, i16 80, i16 80>
   %vshr_n130 = lshr <8 x i16> undef, <i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8>
   %vshr_n134 = lshr <8 x i16> %mul.i209, <i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8>
@@ -608,7 +608,7 @@ for.end179:                                       ; preds = %for.cond.loopexit,
 }
 
 declare <8 x i16> @llvm.arm.neon.vrshiftu.v8i16(<8 x i16>, <8 x i16>) nounwind readnone
-declare <8 x i16> @llvm.arm.neon.vqsubu.v8i16(<8 x i16>, <8 x i16>) nounwind readnone
+declare <8 x i16> @llvm.usub.sat.v8i16(<8 x i16>, <8 x i16>) nounwind readnone
 declare <8 x i8> @llvm.arm.neon.vqmovnu.v8i8(<8 x i16>) nounwind readnone
 
 ; vmull lowering would create a zext(v4i8 load()) instead of a zextload(v4i8),
diff --git a/llvm/test/CodeGen/ARM/vqadd.ll b/llvm/test/CodeGen/ARM/vqadd.ll
index d1e90cb20..47432c7b7 100644
--- a/llvm/test/CodeGen/ARM/vqadd.ll
+++ b/llvm/test/CodeGen/ARM/vqadd.ll
@@ -5,7 +5,7 @@ define <8 x i8> @vqadds8(<8 x i8>* %A, <8 x i8>* %B) nounwind {
 ;CHECK: vqadd.s8
 	%tmp1 = load <8 x i8>, <8 x i8>* %A
 	%tmp2 = load <8 x i8>, <8 x i8>* %B
-	%tmp3 = call <8 x i8> @llvm.arm.neon.vqadds.v8i8(<8 x i8> %tmp1, <8 x i8> %tmp2)
+	%tmp3 = call <8 x i8> @llvm.sadd.sat.v8i8(<8 x i8> %tmp1, <8 x i8> %tmp2)
 	ret <8 x i8> %tmp3
 }
 
@@ -14,7 +14,7 @@ define <4 x i16> @vqadds16(<4 x i16>* %A, <4 x i16>* %B) nounwind {
 ;CHECK: vqadd.s16
 	%tmp1 = load <4 x i16>, <4 x i16>* %A
 	%tmp2 = load <4 x i16>, <4 x i16>* %B
-	%tmp3 = call <4 x i16> @llvm.arm.neon.vqadds.v4i16(<4 x i16> %tmp1, <4 x i16> %tmp2)
+	%tmp3 = call <4 x i16> @llvm.sadd.sat.v4i16(<4 x i16> %tmp1, <4 x i16> %tmp2)
 	ret <4 x i16> %tmp3
 }
 
@@ -23,7 +23,7 @@ define <2 x i32> @vqadds32(<2 x i32>* %A, <2 x i32>* %B) nounwind {
 ;CHECK: vqadd.s32
 	%tmp1 = load <2 x i32>, <2 x i32>* %A
 	%tmp2 = load <2 x i32>, <2 x i32>* %B
-	%tmp3 = call <2 x i32> @llvm.arm.neon.vqadds.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2)
+	%tmp3 = call <2 x i32> @llvm.sadd.sat.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2)
 	ret <2 x i32> %tmp3
 }
 
@@ -32,7 +32,7 @@ define <1 x i64> @vqadds64(<1 x i64>* %A, <1 x i64>* %B) nounwind {
 ;CHECK: vqadd.s64
 	%tmp1 = load <1 x i64>, <1 x i64>* %A
 	%tmp2 = load <1 x i64>, <1 x i64>* %B
-	%tmp3 = call <1 x i64> @llvm.arm.neon.vqadds.v1i64(<1 x i64> %tmp1, <1 x i64> %tmp2)
+	%tmp3 = call <1 x i64> @llvm.sadd.sat.v1i64(<1 x i64> %tmp1, <1 x i64> %tmp2)
 	ret <1 x i64> %tmp3
 }
 
@@ -41,7 +41,7 @@ define <8 x i8> @vqaddu8(<8 x i8>* %A, <8 x i8>* %B) nounwind {
 ;CHECK: vqadd.u8
 	%tmp1 = load <8 x i8>, <8 x i8>* %A
 	%tmp2 = load <8 x i8>, <8 x i8>* %B
-	%tmp3 = call <8 x i8> @llvm.arm.neon.vqaddu.v8i8(<8 x i8> %tmp1, <8 x i8> %tmp2)
+	%tmp3 = call <8 x i8> @llvm.uadd.sat.v8i8(<8 x i8> %tmp1, <8 x i8> %tmp2)
 	ret <8 x i8> %tmp3
 }
 
@@ -50,7 +50,7 @@ define <4 x i16> @vqaddu16(<4 x i16>* %A, <4 x i16>* %B) nounwind {
 ;CHECK: vqadd.u16
 	%tmp1 = load <4 x i16>, <4 x i16>* %A
 	%tmp2 = load <4 x i16>, <4 x i16>* %B
-	%tmp3 = call <4 x i16> @llvm.arm.neon.vqaddu.v4i16(<4 x i16> %tmp1, <4 x i16> %tmp2)
+	%tmp3 = call <4 x i16> @llvm.uadd.sat.v4i16(<4 x i16> %tmp1, <4 x i16> %tmp2)
 	ret <4 x i16> %tmp3
 }
 
@@ -59,7 +59,7 @@ define <2 x i32> @vqaddu32(<2 x i32>* %A, <2 x i32>* %B) nounwind {
 ;CHECK: vqadd.u32
 	%tmp1 = load <2 x i32>, <2 x i32>* %A
 	%tmp2 = load <2 x i32>, <2 x i32>* %B
-	%tmp3 = call <2 x i32> @llvm.arm.neon.vqaddu.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2)
+	%tmp3 = call <2 x i32> @llvm.uadd.sat.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2)
 	ret <2 x i32> %tmp3
 }
 
@@ -68,7 +68,7 @@ define <1 x i64> @vqaddu64(<1 x i64>* %A, <1 x i64>* %B) nounwind {
 ;CHECK: vqadd.u64
 	%tmp1 = load <1 x i64>, <1 x i64>* %A
 	%tmp2 = load <1 x i64>, <1 x i64>* %B
-	%tmp3 = call <1 x i64> @llvm.arm.neon.vqaddu.v1i64(<1 x i64> %tmp1, <1 x i64> %tmp2)
+	%tmp3 = call <1 x i64> @llvm.uadd.sat.v1i64(<1 x i64> %tmp1, <1 x i64> %tmp2)
 	ret <1 x i64> %tmp3
 }
 
@@ -77,7 +77,7 @@ define <16 x i8> @vqaddQs8(<16 x i8>* %A, <16 x i8>* %B) nounwind {
 ;CHECK: vqadd.s8
 	%tmp1 = load <16 x i8>, <16 x i8>* %A
 	%tmp2 = load <16 x i8>, <16 x i8>* %B
-	%tmp3 = call <16 x i8> @llvm.arm.neon.vqadds.v16i8(<16 x i8> %tmp1, <16 x i8> %tmp2)
+	%tmp3 = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> %tmp1, <16 x i8> %tmp2)
 	ret <16 x i8> %tmp3
 }
 
@@ -86,7 +86,7 @@ define <8 x i16> @vqaddQs16(<8 x i16>* %A, <8 x i16>* %B) nounwind {
 ;CHECK: vqadd.s16
 	%tmp1 = load <8 x i16>, <8 x i16>* %A
 	%tmp2 = load <8 x i16>, <8 x i16>* %B
-	%tmp3 = call <8 x i16> @llvm.arm.neon.vqadds.v8i16(<8 x i16> %tmp1, <8 x i16> %tmp2)
+	%tmp3 = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> %tmp1, <8 x i16> %tmp2)
 	ret <8 x i16> %tmp3
 }
 
@@ -95,7 +95,7 @@ define <4 x i32> @vqaddQs32(<4 x i32>* %A, <4 x i32>* %B) nounwind {
 ;CHECK: vqadd.s32
 	%tmp1 = load <4 x i32>, <4 x i32>* %A
 	%tmp2 = load <4 x i32>, <4 x i32>* %B
-	%tmp3 = call <4 x i32> @llvm.arm.neon.vqadds.v4i32(<4 x i32> %tmp1, <4 x i32> %tmp2)
+	%tmp3 = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> %tmp1, <4 x i32> %tmp2)
 	ret <4 x i32> %tmp3
 }
 
@@ -104,7 +104,7 @@ define <2 x i64> @vqaddQs64(<2 x i64>* %A, <2 x i64>* %B) nounwind {
 ;CHECK: vqadd.s64
 	%tmp1 = load <2 x i64>, <2 x i64>* %A
 	%tmp2 = load <2 x i64>, <2 x i64>* %B
-	%tmp3 = call <2 x i64> @llvm.arm.neon.vqadds.v2i64(<2 x i64> %tmp1, <2 x i64> %tmp2)
+	%tmp3 = call <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64> %tmp1, <2 x i64> %tmp2)
 	ret <2 x i64> %tmp3
 }
 
@@ -113,7 +113,7 @@ define <16 x i8> @vqaddQu8(<16 x i8>* %A, <16 x i8>* %B) nounwind {
 ;CHECK: vqadd.u8
 	%tmp1 = load <16 x i8>, <16 x i8>* %A
 	%tmp2 = load <16 x i8>, <16 x i8>* %B
-	%tmp3 = call <16 x i8> @llvm.arm.neon.vqaddu.v16i8(<16 x i8> %tmp1, <16 x i8> %tmp2)
+	%tmp3 = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> %tmp1, <16 x i8> %tmp2)
 	ret <16 x i8> %tmp3
 }
 
@@ -122,7 +122,7 @@ define <8 x i16> @vqaddQu16(<8 x i16>* %A, <8 x i16>* %B) nounwind {
 ;CHECK: vqadd.u16
 	%tmp1 = load <8 x i16>, <8 x i16>* %A
 	%tmp2 = load <8 x i16>, <8 x i16>* %B
-	%tmp3 = call <8 x i16> @llvm.arm.neon.vqaddu.v8i16(<8 x i16> %tmp1, <8 x i16> %tmp2)
+	%tmp3 = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> %tmp1, <8 x i16> %tmp2)
 	ret <8 x i16> %tmp3
 }
 
@@ -131,7 +131,7 @@ define <4 x i32> @vqaddQu32(<4 x i32>* %A, <4 x i32>* %B) nounwind {
 ;CHECK: vqadd.u32
 	%tmp1 = load <4 x i32>, <4 x i32>* %A
 	%tmp2 = load <4 x i32>, <4 x i32>* %B
-	%tmp3 = call <4 x i32> @llvm.arm.neon.vqaddu.v4i32(<4 x i32> %tmp1, <4 x i32> %tmp2)
+	%tmp3 = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> %tmp1, <4 x i32> %tmp2)
 	ret <4 x i32> %tmp3
 }
 
@@ -140,26 +140,26 @@ define <2 x i64> @vqaddQu64(<2 x i64>* %A, <2 x i64>* %B) nounwind {
 ;CHECK: vqadd.u64
 	%tmp1 = load <2 x i64>, <2 x i64>* %A
 	%tmp2 = load <2 x i64>, <2 x i64>* %B
-	%tmp3 = call <2 x i64> @llvm.arm.neon.vqaddu.v2i64(<2 x i64> %tmp1, <2 x i64> %tmp2)
+	%tmp3 = call <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64> %tmp1, <2 x i64> %tmp2)
 	ret <2 x i64> %tmp3
 }
 
-declare <8 x i8>  @llvm.arm.neon.vqadds.v8i8(<8 x i8>, <8 x i8>) nounwind readnone
-declare <4 x i16> @llvm.arm.neon.vqadds.v4i16(<4 x i16>, <4 x i16>) nounwind readnone
-declare <2 x i32> @llvm.arm.neon.vqadds.v2i32(<2 x i32>, <2 x i32>) nounwind readnone
-declare <1 x i64> @llvm.arm.neon.vqadds.v1i64(<1 x i64>, <1 x i64>) nounwind readnone
+declare <8 x i8>  @llvm.sadd.sat.v8i8(<8 x i8>, <8 x i8>) nounwind readnone
+declare <4 x i16> @llvm.sadd.sat.v4i16(<4 x i16>, <4 x i16>) nounwind readnone
+declare <2 x i32> @llvm.sadd.sat.v2i32(<2 x i32>, <2 x i32>) nounwind readnone
+declare <1 x i64> @llvm.sadd.sat.v1i64(<1 x i64>, <1 x i64>) nounwind readnone
 
-declare <8 x i8>  @llvm.arm.neon.vqaddu.v8i8(<8 x i8>, <8 x i8>) nounwind readnone
-declare <4 x i16> @llvm.arm.neon.vqaddu.v4i16(<4 x i16>, <4 x i16>) nounwind readnone
-declare <2 x i32> @llvm.arm.neon.vqaddu.v2i32(<2 x i32>, <2 x i32>) nounwind readnone
-declare <1 x i64> @llvm.arm.neon.vqaddu.v1i64(<1 x i64>, <1 x i64>) nounwind readnone
+declare <8 x i8>  @llvm.uadd.sat.v8i8(<8 x i8>, <8 x i8>) nounwind readnone
+declare <4 x i16> @llvm.uadd.sat.v4i16(<4 x i16>, <4 x i16>) nounwind readnone
+declare <2 x i32> @llvm.uadd.sat.v2i32(<2 x i32>, <2 x i32>) nounwind readnone
+declare <1 x i64> @llvm.uadd.sat.v1i64(<1 x i64>, <1 x i64>) nounwind readnone
 
-declare <16 x i8> @llvm.arm.neon.vqadds.v16i8(<16 x i8>, <16 x i8>) nounwind readnone
-declare <8 x i16> @llvm.arm.neon.vqadds.v8i16(<8 x i16>, <8 x i16>) nounwind readnone
-declare <4 x i32> @llvm.arm.neon.vqadds.v4i32(<4 x i32>, <4 x i32>) nounwind readnone
-declare <2 x i64> @llvm.arm.neon.vqadds.v2i64(<2 x i64>, <2 x i64>) nounwind readnone
+declare <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8>, <16 x i8>) nounwind readnone
+declare <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16>, <8 x i16>) nounwind readnone
+declare <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32>, <4 x i32>) nounwind readnone
+declare <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64>, <2 x i64>) nounwind readnone
 
-declare <16 x i8> @llvm.arm.neon.vqaddu.v16i8(<16 x i8>, <16 x i8>) nounwind readnone
-declare <8 x i16> @llvm.arm.neon.vqaddu.v8i16(<8 x i16>, <8 x i16>) nounwind readnone
-declare <4 x i32> @llvm.arm.neon.vqaddu.v4i32(<4 x i32>, <4 x i32>) nounwind readnone
-declare <2 x i64> @llvm.arm.neon.vqaddu.v2i64(<2 x i64>, <2 x i64>) nounwind readnone
+declare <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8>, <16 x i8>) nounwind readnone
+declare <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16>, <8 x i16>) nounwind readnone
+declare <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32>, <4 x i32>) nounwind readnone
+declare <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64>, <2 x i64>) nounwind readnone
diff --git a/llvm/test/CodeGen/ARM/vqdmul.ll b/llvm/test/CodeGen/ARM/vqdmul.ll
index 6da080012..fa938d45b 100644
--- a/llvm/test/CodeGen/ARM/vqdmul.ll
+++ b/llvm/test/CodeGen/ARM/vqdmul.ll
@@ -204,7 +204,7 @@ define <4 x i32> @vqdmlals16_natural(<4 x i32>* %A, <4 x i16>* %B, <4 x i16>* %C
         %tmp2 = load <4 x i16>, <4 x i16>* %B
         %tmp3 = load <4 x i16>, <4 x i16>* %C
         %tmp4 = call <4 x i32> @llvm.arm.neon.vqdmull.v4i32(<4 x i16> %tmp2, <4 x i16> %tmp3)
-        %tmp5 = call <4 x i32> @llvm.arm.neon.vqadds.v4i32(<4 x i32> %tmp1, <4 x i32> %tmp4)
+        %tmp5 = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> %tmp1, <4 x i32> %tmp4)
         ret <4 x i32> %tmp5
 }
 
@@ -215,7 +215,7 @@ define <2 x i64> @vqdmlals32_natural(<2 x i64>* %A, <2 x i32>* %B, <2 x i32>* %C
         %tmp2 = load <2 x i32>, <2 x i32>* %B
         %tmp3 = load <2 x i32>, <2 x i32>* %C
         %tmp4 = call <2 x i64> @llvm.arm.neon.vqdmull.v2i64(<2 x i32> %tmp2, <2 x i32> %tmp3)
-        %tmp5 = call <2 x i64> @llvm.arm.neon.vqadds.v2i64(<2 x i64> %tmp1, <2 x i64> %tmp4)
+        %tmp5 = call <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64> %tmp1, <2 x i64> %tmp4)
         ret <2 x i64> %tmp5
 }
 
@@ -225,7 +225,7 @@ entry:
 ; CHECK: vqdmlal.s16 q0, d2, d3[1]
   %0 = shufflevector <4 x i16> %arg2_int16x4_t, <4 x i16> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1> ; <<4 x i16>> [#uses=1]
   %1 = tail call <4 x i32> @llvm.arm.neon.vqdmull.v4i32(<4 x i16> %arg1_int16x4_t, <4 x i16> %0)
-  %2 = tail call <4 x i32> @llvm.arm.neon.vqadds.v4i32(<4 x i32> %arg0_int32x4_t, <4 x i32> %1)
+  %2 = tail call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> %arg0_int32x4_t, <4 x i32> %1)
   ret <4 x i32> %2
 }
 
@@ -235,12 +235,12 @@ entry:
 ; CHECK: vqdmlal.s32 q0, d2, d3[1]
   %0 = shufflevector <2 x i32> %arg2_int32x2_t, <2 x i32> undef, <2 x i32> <i32 1, i32 1> ; <<2 x i32>> [#uses=1]
   %1 = tail call <2 x i64> @llvm.arm.neon.vqdmull.v2i64(<2 x i32> %arg1_int32x2_t, <2 x i32> %0)
-  %2 = call <2 x i64> @llvm.arm.neon.vqadds.v2i64(<2 x i64> %arg0_int64x2_t, <2 x i64> %1)
+  %2 = call <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64> %arg0_int64x2_t, <2 x i64> %1)
   ret <2 x i64> %2
 }
 
-declare <4 x i32>  @llvm.arm.neon.vqadds.v4i32(<4 x i32>, <4 x i32>) nounwind readnone
-declare <2 x i64>  @llvm.arm.neon.vqadds.v2i64(<2 x i64>, <2 x i64>) nounwind readnone
+declare <4 x i32>  @llvm.sadd.sat.v4i32(<4 x i32>, <4 x i32>) nounwind readnone
+declare <2 x i64>  @llvm.sadd.sat.v2i64(<2 x i64>, <2 x i64>) nounwind readnone
 
 define <4 x i32> @vqdmlsls16_natural(<4 x i32>* %A, <4 x i16>* %B, <4 x i16>* %C) nounwind {
 ;CHECK-LABEL: vqdmlsls16_natural:
@@ -249,7 +249,7 @@ define <4 x i32> @vqdmlsls16_natural(<4 x i32>* %A, <4 x i16>* %B, <4 x i16>* %C
         %tmp2 = load <4 x i16>, <4 x i16>* %B
         %tmp3 = load <4 x i16>, <4 x i16>* %C
         %tmp4 = call <4 x i32> @llvm.arm.neon.vqdmull.v4i32(<4 x i16> %tmp2, <4 x i16> %tmp3)
-        %tmp5 = call <4 x i32> @llvm.arm.neon.vqsubs.v4i32(<4 x i32> %tmp1, <4 x i32> %tmp4)
+        %tmp5 = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> %tmp1, <4 x i32> %tmp4)
         ret <4 x i32> %tmp5
 }
 
@@ -260,7 +260,7 @@ define <2 x i64> @vqdmlsls32_natural(<2 x i64>* %A, <2 x i32>* %B, <2 x i32>* %C
         %tmp2 = load <2 x i32>, <2 x i32>* %B
         %tmp3 = load <2 x i32>, <2 x i32>* %C
         %tmp4 = call <2 x i64> @llvm.arm.neon.vqdmull.v2i64(<2 x i32> %tmp2, <2 x i32> %tmp3)
-        %tmp5 = call <2 x i64> @llvm.arm.neon.vqsubs.v2i64(<2 x i64> %tmp1, <2 x i64> %tmp4)
+        %tmp5 = call <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64> %tmp1, <2 x i64> %tmp4)
         ret <2 x i64> %tmp5
 }
 
@@ -270,7 +270,7 @@ entry:
 ; CHECK: vqdmlsl.s16 q0, d2, d3[1]
   %0 = shufflevector <4 x i16> %arg2_int16x4_t, <4 x i16> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1> ; <<4 x i16>> [#uses=1]
   %1 = tail call <4 x i32> @llvm.arm.neon.vqdmull.v4i32(<4 x i16> %arg1_int16x4_t, <4 x i16> %0)
-  %2 = tail call <4 x i32> @llvm.arm.neon.vqsubs.v4i32(<4 x i32> %arg0_int32x4_t, <4 x i32> %1)
+  %2 = tail call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> %arg0_int32x4_t, <4 x i32> %1)
   ret <4 x i32> %2
 }
 
@@ -280,9 +280,9 @@ entry:
 ; CHECK: vqdmlsl.s32 q0, d2, d3[1]
   %0 = shufflevector <2 x i32> %arg2_int32x2_t, <2 x i32> undef, <2 x i32> <i32 1, i32 1> ; <<2 x i32>> [#uses=1]
   %1 = tail call <2 x i64> @llvm.arm.neon.vqdmull.v2i64(<2 x i32> %arg1_int32x2_t, <2 x i32> %0)
-  %2 = call <2 x i64> @llvm.arm.neon.vqsubs.v2i64(<2 x i64> %arg0_int64x2_t, <2 x i64> %1)
+  %2 = call <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64> %arg0_int64x2_t, <2 x i64> %1)
   ret <2 x i64> %2
 }
 
-declare <4 x i32>  @llvm.arm.neon.vqsubs.v4i32(<4 x i32>, <4 x i32>) nounwind readnone
-declare <2 x i64>  @llvm.arm.neon.vqsubs.v2i64(<2 x i64>, <2 x i64>) nounwind readnone
+declare <4 x i32>  @llvm.ssub.sat.v4i32(<4 x i32>, <4 x i32>) nounwind readnone
+declare <2 x i64>  @llvm.ssub.sat.v2i64(<2 x i64>, <2 x i64>) nounwind readnone
diff --git a/llvm/test/CodeGen/ARM/vqsub.ll b/llvm/test/CodeGen/ARM/vqsub.ll
index 40963ce82..9864f6421 100644
--- a/llvm/test/CodeGen/ARM/vqsub.ll
+++ b/llvm/test/CodeGen/ARM/vqsub.ll
@@ -5,7 +5,7 @@ define <8 x i8> @vqsubs8(<8 x i8>* %A, <8 x i8>* %B) nounwind {
 ;CHECK: vqsub.s8
 	%tmp1 = load <8 x i8>, <8 x i8>* %A
 	%tmp2 = load <8 x i8>, <8 x i8>* %B
-	%tmp3 = call <8 x i8> @llvm.arm.neon.vqsubs.v8i8(<8 x i8> %tmp1, <8 x i8> %tmp2)
+	%tmp3 = call <8 x i8> @llvm.ssub.sat.v8i8(<8 x i8> %tmp1, <8 x i8> %tmp2)
 	ret <8 x i8> %tmp3
 }
 
@@ -14,7 +14,7 @@ define <4 x i16> @vqsubs16(<4 x i16>* %A, <4 x i16>* %B) nounwind {
 ;CHECK: vqsub.s16
 	%tmp1 = load <4 x i16>, <4 x i16>* %A
 	%tmp2 = load <4 x i16>, <4 x i16>* %B
-	%tmp3 = call <4 x i16> @llvm.arm.neon.vqsubs.v4i16(<4 x i16> %tmp1, <4 x i16> %tmp2)
+	%tmp3 = call <4 x i16> @llvm.ssub.sat.v4i16(<4 x i16> %tmp1, <4 x i16> %tmp2)
 	ret <4 x i16> %tmp3
 }
 
@@ -23,7 +23,7 @@ define <2 x i32> @vqsubs32(<2 x i32>* %A, <2 x i32>* %B) nounwind {
 ;CHECK: vqsub.s32
 	%tmp1 = load <2 x i32>, <2 x i32>* %A
 	%tmp2 = load <2 x i32>, <2 x i32>* %B
-	%tmp3 = call <2 x i32> @llvm.arm.neon.vqsubs.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2)
+	%tmp3 = call <2 x i32> @llvm.ssub.sat.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2)
 	ret <2 x i32> %tmp3
 }
 
@@ -32,7 +32,7 @@ define <1 x i64> @vqsubs64(<1 x i64>* %A, <1 x i64>* %B) nounwind {
 ;CHECK: vqsub.s64
 	%tmp1 = load <1 x i64>, <1 x i64>* %A
 	%tmp2 = load <1 x i64>, <1 x i64>* %B
-	%tmp3 = call <1 x i64> @llvm.arm.neon.vqsubs.v1i64(<1 x i64> %tmp1, <1 x i64> %tmp2)
+	%tmp3 = call <1 x i64> @llvm.ssub.sat.v1i64(<1 x i64> %tmp1, <1 x i64> %tmp2)
 	ret <1 x i64> %tmp3
 }
 
@@ -41,7 +41,7 @@ define <8 x i8> @vqsubu8(<8 x i8>* %A, <8 x i8>* %B) nounwind {
 ;CHECK: vqsub.u8
 	%tmp1 = load <8 x i8>, <8 x i8>* %A
 	%tmp2 = load <8 x i8>, <8 x i8>* %B
-	%tmp3 = call <8 x i8> @llvm.arm.neon.vqsubu.v8i8(<8 x i8> %tmp1, <8 x i8> %tmp2)
+	%tmp3 = call <8 x i8> @llvm.usub.sat.v8i8(<8 x i8> %tmp1, <8 x i8> %tmp2)
 	ret <8 x i8> %tmp3
 }
 
@@ -50,7 +50,7 @@ define <4 x i16> @vqsubu16(<4 x i16>* %A, <4 x i16>* %B) nounwind {
 ;CHECK: vqsub.u16
 	%tmp1 = load <4 x i16>, <4 x i16>* %A
 	%tmp2 = load <4 x i16>, <4 x i16>* %B
-	%tmp3 = call <4 x i16> @llvm.arm.neon.vqsubu.v4i16(<4 x i16> %tmp1, <4 x i16> %tmp2)
+	%tmp3 = call <4 x i16> @llvm.usub.sat.v4i16(<4 x i16> %tmp1, <4 x i16> %tmp2)
 	ret <4 x i16> %tmp3
 }
 
@@ -59,7 +59,7 @@ define <2 x i32> @vqsubu32(<2 x i32>* %A, <2 x i32>* %B) nounwind {
 ;CHECK: vqsub.u32
 	%tmp1 = load <2 x i32>, <2 x i32>* %A
 	%tmp2 = load <2 x i32>, <2 x i32>* %B
-	%tmp3 = call <2 x i32> @llvm.arm.neon.vqsubu.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2)
+	%tmp3 = call <2 x i32> @llvm.usub.sat.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2)
 	ret <2 x i32> %tmp3
 }
 
@@ -68,7 +68,7 @@ define <1 x i64> @vqsubu64(<1 x i64>* %A, <1 x i64>* %B) nounwind {
 ;CHECK: vqsub.u64
 	%tmp1 = load <1 x i64>, <1 x i64>* %A
 	%tmp2 = load <1 x i64>, <1 x i64>* %B
-	%tmp3 = call <1 x i64> @llvm.arm.neon.vqsubu.v1i64(<1 x i64> %tmp1, <1 x i64> %tmp2)
+	%tmp3 = call <1 x i64> @llvm.usub.sat.v1i64(<1 x i64> %tmp1, <1 x i64> %tmp2)
 	ret <1 x i64> %tmp3
 }
 
@@ -77,7 +77,7 @@ define <16 x i8> @vqsubQs8(<16 x i8>* %A, <16 x i8>* %B) nounwind {
 ;CHECK: vqsub.s8
 	%tmp1 = load <16 x i8>, <16 x i8>* %A
 	%tmp2 = load <16 x i8>, <16 x i8>* %B
-	%tmp3 = call <16 x i8> @llvm.arm.neon.vqsubs.v16i8(<16 x i8> %tmp1, <16 x i8> %tmp2)
+	%tmp3 = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> %tmp1, <16 x i8> %tmp2)
 	ret <16 x i8> %tmp3
 }
 
@@ -86,7 +86,7 @@ define <8 x i16> @vqsubQs16(<8 x i16>* %A, <8 x i16>* %B) nounwind {
 ;CHECK: vqsub.s16
 	%tmp1 = load <8 x i16>, <8 x i16>* %A
 	%tmp2 = load <8 x i16>, <8 x i16>* %B
-	%tmp3 = call <8 x i16> @llvm.arm.neon.vqsubs.v8i16(<8 x i16> %tmp1, <8 x i16> %tmp2)
+	%tmp3 = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> %tmp1, <8 x i16> %tmp2)
 	ret <8 x i16> %tmp3
 }
 
@@ -95,7 +95,7 @@ define <4 x i32> @vqsubQs32(<4 x i32>* %A, <4 x i32>* %B) nounwind {
 ;CHECK: vqsub.s32
 	%tmp1 = load <4 x i32>, <4 x i32>* %A
 	%tmp2 = load <4 x i32>, <4 x i32>* %B
-	%tmp3 = call <4 x i32> @llvm.arm.neon.vqsubs.v4i32(<4 x i32> %tmp1, <4 x i32> %tmp2)
+	%tmp3 = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> %tmp1, <4 x i32> %tmp2)
 	ret <4 x i32> %tmp3
 }
 
@@ -104,7 +104,7 @@ define <2 x i64> @vqsubQs64(<2 x i64>* %A, <2 x i64>* %B) nounwind {
 ;CHECK: vqsub.s64
 	%tmp1 = load <2 x i64>, <2 x i64>* %A
 	%tmp2 = load <2 x i64>, <2 x i64>* %B
-	%tmp3 = call <2 x i64> @llvm.arm.neon.vqsubs.v2i64(<2 x i64> %tmp1, <2 x i64> %tmp2)
+	%tmp3 = call <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64> %tmp1, <2 x i64> %tmp2)
 	ret <2 x i64> %tmp3
 }
 
@@ -113,7 +113,7 @@ define <16 x i8> @vqsubQu8(<16 x i8>* %A, <16 x i8>* %B) nounwind {
 ;CHECK: vqsub.u8
 	%tmp1 = load <16 x i8>, <16 x i8>* %A
 	%tmp2 = load <16 x i8>, <16 x i8>* %B
-	%tmp3 = call <16 x i8> @llvm.arm.neon.vqsubu.v16i8(<16 x i8> %tmp1, <16 x i8> %tmp2)
+	%tmp3 = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> %tmp1, <16 x i8> %tmp2)
 	ret <16 x i8> %tmp3
 }
 
@@ -122,7 +122,7 @@ define <8 x i16> @vqsubQu16(<8 x i16>* %A, <8 x i16>* %B) nounwind {
 ;CHECK: vqsub.u16
 	%tmp1 = load <8 x i16>, <8 x i16>* %A
 	%tmp2 = load <8 x i16>, <8 x i16>* %B
-	%tmp3 = call <8 x i16> @llvm.arm.neon.vqsubu.v8i16(<8 x i16> %tmp1, <8 x i16> %tmp2)
+	%tmp3 = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> %tmp1, <8 x i16> %tmp2)
 	ret <8 x i16> %tmp3
 }
 
@@ -131,7 +131,7 @@ define <4 x i32> @vqsubQu32(<4 x i32>* %A, <4 x i32>* %B) nounwind {
 ;CHECK: vqsub.u32
 	%tmp1 = load <4 x i32>, <4 x i32>* %A
 	%tmp2 = load <4 x i32>, <4 x i32>* %B
-	%tmp3 = call <4 x i32> @llvm.arm.neon.vqsubu.v4i32(<4 x i32> %tmp1, <4 x i32> %tmp2)
+	%tmp3 = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> %tmp1, <4 x i32> %tmp2)
 	ret <4 x i32> %tmp3
 }
 
@@ -140,26 +140,26 @@ define <2 x i64> @vqsubQu64(<2 x i64>* %A, <2 x i64>* %B) nounwind {
 ;CHECK: vqsub.u64
 	%tmp1 = load <2 x i64>, <2 x i64>* %A
 	%tmp2 = load <2 x i64>, <2 x i64>* %B
-	%tmp3 = call <2 x i64> @llvm.arm.neon.vqsubu.v2i64(<2 x i64> %tmp1, <2 x i64> %tmp2)
+	%tmp3 = call <2 x i64> @llvm.usub.sat.v2i64(<2 x i64> %tmp1, <2 x i64> %tmp2)
 	ret <2 x i64> %tmp3
 }
 
-declare <8 x i8>  @llvm.arm.neon.vqsubs.v8i8(<8 x i8>, <8 x i8>) nounwind readnone
-declare <4 x i16> @llvm.arm.neon.vqsubs.v4i16(<4 x i16>, <4 x i16>) nounwind readnone
-declare <2 x i32> @llvm.arm.neon.vqsubs.v2i32(<2 x i32>, <2 x i32>) nounwind readnone
-declare <1 x i64> @llvm.arm.neon.vqsubs.v1i64(<1 x i64>, <1 x i64>) nounwind readnone
+declare <8 x i8>  @llvm.ssub.sat.v8i8(<8 x i8>, <8 x i8>) nounwind readnone
+declare <4 x i16> @llvm.ssub.sat.v4i16(<4 x i16>, <4 x i16>) nounwind readnone
+declare <2 x i32> @llvm.ssub.sat.v2i32(<2 x i32>, <2 x i32>) nounwind readnone
+declare <1 x i64> @llvm.ssub.sat.v1i64(<1 x i64>, <1 x i64>) nounwind readnone
 
-declare <8 x i8>  @llvm.arm.neon.vqsubu.v8i8(<8 x i8>, <8 x i8>) nounwind readnone
-declare <4 x i16> @llvm.arm.neon.vqsubu.v4i16(<4 x i16>, <4 x i16>) nounwind readnone
-declare <2 x i32> @llvm.arm.neon.vqsubu.v2i32(<2 x i32>, <2 x i32>) nounwind readnone
-declare <1 x i64> @llvm.arm.neon.vqsubu.v1i64(<1 x i64>, <1 x i64>) nounwind readnone
+declare <8 x i8>  @llvm.usub.sat.v8i8(<8 x i8>, <8 x i8>) nounwind readnone
+declare <4 x i16> @llvm.usub.sat.v4i16(<4 x i16>, <4 x i16>) nounwind readnone
+declare <2 x i32> @llvm.usub.sat.v2i32(<2 x i32>, <2 x i32>) nounwind readnone
+declare <1 x i64> @llvm.usub.sat.v1i64(<1 x i64>, <1 x i64>) nounwind readnone
 
-declare <16 x i8> @llvm.arm.neon.vqsubs.v16i8(<16 x i8>, <16 x i8>) nounwind readnone
-declare <8 x i16> @llvm.arm.neon.vqsubs.v8i16(<8 x i16>, <8 x i16>) nounwind readnone
-declare <4 x i32> @llvm.arm.neon.vqsubs.v4i32(<4 x i32>, <4 x i32>) nounwind readnone
-declare <2 x i64> @llvm.arm.neon.vqsubs.v2i64(<2 x i64>, <2 x i64>) nounwind readnone
+declare <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8>, <16 x i8>) nounwind readnone
+declare <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16>, <8 x i16>) nounwind readnone
+declare <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32>, <4 x i32>) nounwind readnone
+declare <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64>, <2 x i64>) nounwind readnone
 
-declare <16 x i8> @llvm.arm.neon.vqsubu.v16i8(<16 x i8>, <16 x i8>) nounwind readnone
-declare <8 x i16> @llvm.arm.neon.vqsubu.v8i16(<8 x i16>, <8 x i16>) nounwind readnone
-declare <4 x i32> @llvm.arm.neon.vqsubu.v4i32(<4 x i32>, <4 x i32>) nounwind readnone
-declare <2 x i64> @llvm.arm.neon.vqsubu.v2i64(<2 x i64>, <2 x i64>) nounwind readnone
+declare <16 x i8> @llvm.usub.sat.v16i8(<16 x i8>, <16 x i8>) nounwind readnone
+declare <8 x i16> @llvm.usub.sat.v8i16(<8 x i16>, <8 x i16>) nounwind readnone
+declare <4 x i32> @llvm.usub.sat.v4i32(<4 x i32>, <4 x i32>) nounwind readnone
+declare <2 x i64> @llvm.usub.sat.v2i64(<2 x i64>, <2 x i64>) nounwind readnone
diff --git a/llvm/test/CodeGen/ARM/vrev.ll b/llvm/test/CodeGen/ARM/vrev.ll
index a20d4b6ba..4bea88021 100644
--- a/llvm/test/CodeGen/ARM/vrev.ll
+++ b/llvm/test/CodeGen/ARM/vrev.ll
@@ -1,112 +1,226 @@
-; RUN: llc -mtriple=arm-eabi -mattr=+neon %s -o - | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=arm-eabi -mattr=+neon,+fullfp16 %s -o - | FileCheck %s
 
 define <8 x i8> @test_vrev64D8(<8 x i8>* %A) nounwind {
-;CHECK-LABEL: test_vrev64D8:
-;CHECK: vrev64.8
+; CHECK-LABEL: test_vrev64D8:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr d16, [r0]
+; CHECK-NEXT:    vrev64.8 d16, d16
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    mov pc, lr
 	%tmp1 = load <8 x i8>, <8 x i8>* %A
 	%tmp2 = shufflevector <8 x i8> %tmp1, <8 x i8> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 	ret <8 x i8> %tmp2
 }
 
 define <4 x i16> @test_vrev64D16(<4 x i16>* %A) nounwind {
-;CHECK-LABEL: test_vrev64D16:
-;CHECK: vrev64.16
+; CHECK-LABEL: test_vrev64D16:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr d16, [r0]
+; CHECK-NEXT:    vrev64.16 d16, d16
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    mov pc, lr
 	%tmp1 = load <4 x i16>, <4 x i16>* %A
 	%tmp2 = shufflevector <4 x i16> %tmp1, <4 x i16> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
 	ret <4 x i16> %tmp2
 }
 
+define <4 x half> @test_vrev64Df16(<4 x half>* %A) nounwind {
+; CHECK-LABEL: test_vrev64Df16:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr d16, [r0]
+; CHECK-NEXT:    vrev64.16 d16, d16
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    mov pc, lr
+	%tmp1 = load <4 x half>, <4 x half>* %A
+	%tmp2 = shufflevector <4 x half> %tmp1, <4 x half> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+	ret <4 x half> %tmp2
+}
+
 define <2 x i32> @test_vrev64D32(<2 x i32>* %A) nounwind {
-;CHECK-LABEL: test_vrev64D32:
-;CHECK: vrev64.32
+; CHECK-LABEL: test_vrev64D32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr d16, [r0]
+; CHECK-NEXT:    vrev64.32 d16, d16
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    mov pc, lr
 	%tmp1 = load <2 x i32>, <2 x i32>* %A
 	%tmp2 = shufflevector <2 x i32> %tmp1, <2 x i32> undef, <2 x i32> <i32 1, i32 0>
 	ret <2 x i32> %tmp2
 }
 
 define <2 x float> @test_vrev64Df(<2 x float>* %A) nounwind {
-;CHECK-LABEL: test_vrev64Df:
-;CHECK: vrev64.32
+; CHECK-LABEL: test_vrev64Df:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr d16, [r0]
+; CHECK-NEXT:    vrev64.32 d16, d16
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    mov pc, lr
 	%tmp1 = load <2 x float>, <2 x float>* %A
 	%tmp2 = shufflevector <2 x float> %tmp1, <2 x float> undef, <2 x i32> <i32 1, i32 0>
 	ret <2 x float> %tmp2
 }
 
 define <16 x i8> @test_vrev64Q8(<16 x i8>* %A) nounwind {
-;CHECK-LABEL: test_vrev64Q8:
-;CHECK: vrev64.8
+; CHECK-LABEL: test_vrev64Q8:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r0]
+; CHECK-NEXT:    vrev64.8 q8, q8
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    vmov r2, r3, d17
+; CHECK-NEXT:    mov pc, lr
 	%tmp1 = load <16 x i8>, <16 x i8>* %A
 	%tmp2 = shufflevector <16 x i8> %tmp1, <16 x i8> undef, <16 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8>
 	ret <16 x i8> %tmp2
 }
 
 define <8 x i16> @test_vrev64Q16(<8 x i16>* %A) nounwind {
-;CHECK-LABEL: test_vrev64Q16:
-;CHECK: vrev64.16
+; CHECK-LABEL: test_vrev64Q16:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r0]
+; CHECK-NEXT:    vrev64.16 q8, q8
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    vmov r2, r3, d17
+; CHECK-NEXT:    mov pc, lr
 	%tmp1 = load <8 x i16>, <8 x i16>* %A
 	%tmp2 = shufflevector <8 x i16> %tmp1, <8 x i16> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
 	ret <8 x i16> %tmp2
 }
 
+define <8 x half> @test_vrev64Qf16(<8 x half>* %A) nounwind {
+; CHECK-LABEL: test_vrev64Qf16:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r0]
+; CHECK-NEXT:    vrev64.16 q8, q8
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    vmov r2, r3, d17
+; CHECK-NEXT:    mov pc, lr
+	%tmp1 = load <8 x half>, <8 x half>* %A
+	%tmp2 = shufflevector <8 x half> %tmp1, <8 x half> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
+	ret <8 x half> %tmp2
+}
+
 define <4 x i32> @test_vrev64Q32(<4 x i32>* %A) nounwind {
-;CHECK-LABEL: test_vrev64Q32:
-;CHECK: vrev64.32
+; CHECK-LABEL: test_vrev64Q32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r0]
+; CHECK-NEXT:    vrev64.32 q8, q8
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    vmov r2, r3, d17
+; CHECK-NEXT:    mov pc, lr
 	%tmp1 = load <4 x i32>, <4 x i32>* %A
 	%tmp2 = shufflevector <4 x i32> %tmp1, <4 x i32> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
 	ret <4 x i32> %tmp2
 }
 
 define <4 x float> @test_vrev64Qf(<4 x float>* %A) nounwind {
-;CHECK-LABEL: test_vrev64Qf:
-;CHECK: vrev64.32
+; CHECK-LABEL: test_vrev64Qf:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r0]
+; CHECK-NEXT:    vrev64.32 q8, q8
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    vmov r2, r3, d17
+; CHECK-NEXT:    mov pc, lr
 	%tmp1 = load <4 x float>, <4 x float>* %A
 	%tmp2 = shufflevector <4 x float> %tmp1, <4 x float> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
 	ret <4 x float> %tmp2
 }
 
 define <8 x i8> @test_vrev32D8(<8 x i8>* %A) nounwind {
-;CHECK-LABEL: test_vrev32D8:
-;CHECK: vrev32.8
+; CHECK-LABEL: test_vrev32D8:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr d16, [r0]
+; CHECK-NEXT:    vrev32.8 d16, d16
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    mov pc, lr
 	%tmp1 = load <8 x i8>, <8 x i8>* %A
 	%tmp2 = shufflevector <8 x i8> %tmp1, <8 x i8> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
 	ret <8 x i8> %tmp2
 }
 
 define <4 x i16> @test_vrev32D16(<4 x i16>* %A) nounwind {
-;CHECK-LABEL: test_vrev32D16:
-;CHECK: vrev32.16
+; CHECK-LABEL: test_vrev32D16:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr d16, [r0]
+; CHECK-NEXT:    vrev32.16 d16, d16
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    mov pc, lr
 	%tmp1 = load <4 x i16>, <4 x i16>* %A
 	%tmp2 = shufflevector <4 x i16> %tmp1, <4 x i16> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
 	ret <4 x i16> %tmp2
 }
 
+define <4 x half> @test_vrev32Df16(<4 x half>* %A) nounwind {
+; CHECK-LABEL: test_vrev32Df16:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr d16, [r0]
+; CHECK-NEXT:    vrev32.16 d16, d16
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    mov pc, lr
+	%tmp1 = load <4 x half>, <4 x half>* %A
+	%tmp2 = shufflevector <4 x half> %tmp1, <4 x half> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+	ret <4 x half> %tmp2
+}
+
 define <16 x i8> @test_vrev32Q8(<16 x i8>* %A) nounwind {
-;CHECK-LABEL: test_vrev32Q8:
-;CHECK: vrev32.8
+; CHECK-LABEL: test_vrev32Q8:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r0]
+; CHECK-NEXT:    vrev32.8 q8, q8
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    vmov r2, r3, d17
+; CHECK-NEXT:    mov pc, lr
 	%tmp1 = load <16 x i8>, <16 x i8>* %A
 	%tmp2 = shufflevector <16 x i8> %tmp1, <16 x i8> undef, <16 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 11, i32 10, i32 9, i32 8, i32 15, i32 14, i32 13, i32 12>
 	ret <16 x i8> %tmp2
 }
 
 define <8 x i16> @test_vrev32Q16(<8 x i16>* %A) nounwind {
-;CHECK-LABEL: test_vrev32Q16:
-;CHECK: vrev32.16
+; CHECK-LABEL: test_vrev32Q16:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r0]
+; CHECK-NEXT:    vrev32.16 q8, q8
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    vmov r2, r3, d17
+; CHECK-NEXT:    mov pc, lr
 	%tmp1 = load <8 x i16>, <8 x i16>* %A
 	%tmp2 = shufflevector <8 x i16> %tmp1, <8 x i16> undef, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
 	ret <8 x i16> %tmp2
 }
 
+define <8 x half> @test_vrev32Qf16(<8 x half>* %A) nounwind {
+; CHECK-LABEL: test_vrev32Qf16:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r0]
+; CHECK-NEXT:    vrev32.16 q8, q8
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    vmov r2, r3, d17
+; CHECK-NEXT:    mov pc, lr
+	%tmp1 = load <8 x half>, <8 x half>* %A
+	%tmp2 = shufflevector <8 x half> %tmp1, <8 x half> undef, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
+	ret <8 x half> %tmp2
+}
+
 define <8 x i8> @test_vrev16D8(<8 x i8>* %A) nounwind {
-;CHECK-LABEL: test_vrev16D8:
-;CHECK: vrev16.8
+; CHECK-LABEL: test_vrev16D8:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr d16, [r0]
+; CHECK-NEXT:    vrev16.8 d16, d16
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    mov pc, lr
 	%tmp1 = load <8 x i8>, <8 x i8>* %A
 	%tmp2 = shufflevector <8 x i8> %tmp1, <8 x i8> undef, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
 	ret <8 x i8> %tmp2
 }
 
 define <16 x i8> @test_vrev16Q8(<16 x i8>* %A) nounwind {
-;CHECK-LABEL: test_vrev16Q8:
-;CHECK: vrev16.8
+; CHECK-LABEL: test_vrev16Q8:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r0]
+; CHECK-NEXT:    vrev16.8 q8, q8
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    vmov r2, r3, d17
+; CHECK-NEXT:    mov pc, lr
 	%tmp1 = load <16 x i8>, <16 x i8>* %A
 	%tmp2 = shufflevector <16 x i8> %tmp1, <16 x i8> undef, <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14>
 	ret <16 x i8> %tmp2
@@ -115,27 +229,54 @@ define <16 x i8> @test_vrev16Q8(<16 x i8>* %A) nounwind {
 ; Undef shuffle indices should not prevent matching to VREV:
 
 define <8 x i8> @test_vrev64D8_undef(<8 x i8>* %A) nounwind {
-;CHECK-LABEL: test_vrev64D8_undef:
-;CHECK: vrev64.8
+; CHECK-LABEL: test_vrev64D8_undef:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr d16, [r0]
+; CHECK-NEXT:    vrev64.8 d16, d16
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    mov pc, lr
 	%tmp1 = load <8 x i8>, <8 x i8>* %A
 	%tmp2 = shufflevector <8 x i8> %tmp1, <8 x i8> undef, <8 x i32> <i32 7, i32 undef, i32 undef, i32 4, i32 3, i32 2, i32 1, i32 0>
 	ret <8 x i8> %tmp2
 }
 
 define <8 x i16> @test_vrev32Q16_undef(<8 x i16>* %A) nounwind {
-;CHECK-LABEL: test_vrev32Q16_undef:
-;CHECK: vrev32.16
+; CHECK-LABEL: test_vrev32Q16_undef:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r0]
+; CHECK-NEXT:    vrev32.16 q8, q8
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    vmov r2, r3, d17
+; CHECK-NEXT:    mov pc, lr
 	%tmp1 = load <8 x i16>, <8 x i16>* %A
 	%tmp2 = shufflevector <8 x i16> %tmp1, <8 x i16> undef, <8 x i32> <i32 undef, i32 0, i32 undef, i32 2, i32 5, i32 4, i32 7, i32 undef>
 	ret <8 x i16> %tmp2
 }
 
+define <8 x half> @test_vrev32Qf16_undef(<8 x half>* %A) nounwind {
+; CHECK-LABEL: test_vrev32Qf16_undef:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r0]
+; CHECK-NEXT:    vrev32.16 q8, q8
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    vmov r2, r3, d17
+; CHECK-NEXT:    mov pc, lr
+	%tmp1 = load <8 x half>, <8 x half>* %A
+	%tmp2 = shufflevector <8 x half> %tmp1, <8 x half> undef, <8 x i32> <i32 undef, i32 0, i32 undef, i32 2, i32 5, i32 4, i32 7, i32 undef>
+	ret <8 x half> %tmp2
+}
+
 ; A vcombine feeding a VREV should not obscure things.  Radar 8597007.
 
 define void @test_with_vcombine(<4 x float>* %v) nounwind {
-;CHECK-LABEL: test_with_vcombine:
-;CHECK-NOT: vext
-;CHECK: vrev64.32
+; CHECK-LABEL: test_with_vcombine:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r0:128]
+; CHECK-NEXT:    vadd.f32 d18, d17, d17
+; CHECK-NEXT:    vrev64.32 d16, d16
+; CHECK-NEXT:    vrev64.32 d17, d18
+; CHECK-NEXT:    vst1.64 {d16, d17}, [r0:128]
+; CHECK-NEXT:    mov pc, lr
   %tmp1 = load <4 x float>, <4 x float>* %v, align 16
   %tmp2 = bitcast <4 x float> %tmp1 to <2 x double>
   %tmp3 = extractelement <2 x double> %tmp2, i32 0
@@ -152,7 +293,15 @@ define void @test_with_vcombine(<4 x float>* %v) nounwind {
 ; to <2 x i16> when stored to memory.
 define void @test_vrev64(<4 x i16>* nocapture %source, <2 x i16>* nocapture %dst) nounwind ssp {
 ; CHECK-LABEL: test_vrev64:
-; CHECK: vst1.32
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vld1.32 {d16, d17}, [r0]
+; CHECK-NEXT:    vmov.u16 r0, d17[2]
+; CHECK-NEXT:    vmov.u16 r2, d17[1]
+; CHECK-NEXT:    vmov.32 d16[0], r0
+; CHECK-NEXT:    vmov.32 d16[1], r2
+; CHECK-NEXT:    vuzp.16 d16, d17
+; CHECK-NEXT:    vst1.32 {d16[0]}, [r1:32]
+; CHECK-NEXT:    mov pc, lr
 entry:
   %0 = bitcast <4 x i16>* %source to <8 x i16>*
   %tmp2 = load <8 x i16>, <8 x i16>* %0, align 4
@@ -166,9 +315,15 @@ entry:
 
 ; Test vrev of float4
 define void @float_vrev64(float* nocapture %source, <4 x float>* nocapture %dest) nounwind noinline ssp {
-; CHECK: float_vrev64
-; CHECK: vext.32
-; CHECK: vrev64.32
+; CHECK-LABEL: float_vrev64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.i32 q8, #0x0
+; CHECK-NEXT:    vld1.32 {d18, d19}, [r0]
+; CHECK-NEXT:    add r0, r1, #176
+; CHECK-NEXT:    vext.32 q8, q9, q8, #3
+; CHECK-NEXT:    vrev64.32 q8, q8
+; CHECK-NEXT:    vst1.32 {d16, d17}, [r0]
+; CHECK-NEXT:    mov pc, lr
 entry:
   %0 = bitcast float* %source to <4 x float>*
   %tmp2 = load <4 x float>, <4 x float>* %0, align 4
@@ -180,7 +335,13 @@ entry:
 
 define <4 x i32> @test_vrev32_bswap(<4 x i32> %source) nounwind {
 ; CHECK-LABEL: test_vrev32_bswap:
-; CHECK: vrev32.8
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmov d17, r2, r3
+; CHECK-NEXT:    vmov d16, r0, r1
+; CHECK-NEXT:    vrev32.8 q8, q8
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    vmov r2, r3, d17
+; CHECK-NEXT:    mov pc, lr
   %bswap = call <4 x i32> @llvm.bswap.v4i32(<4 x i32> %source)
   ret <4 x i32> %bswap
 }
diff --git a/llvm/test/CodeGen/ARM/vsel-fp16.ll b/llvm/test/CodeGen/ARM/vsel-fp16.ll
index 9ccc6f427..fda1fcb5f 100644
--- a/llvm/test/CodeGen/ARM/vsel-fp16.ll
+++ b/llvm/test/CodeGen/ARM/vsel-fp16.ll
@@ -106,7 +106,7 @@ define void @test_vsel32ogt(half* %lhs_ptr, half* %rhs_ptr, half* %a_ptr, half*
 ; CHECK-NEXT:    vldr.16 s4, [r0]
 ; CHECK-NEXT:    vldr.16 s6, [r1]
 ; CHECK-NEXT:    movw r0, :lower16:varhalf
-; CHECK-NEXT:    vcmpe.f16 s4, s6
+; CHECK-NEXT:    vcmp.f16 s4, s6
 ; CHECK-NEXT:    movt r0, :upper16:varhalf
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    vselgt.f16 s0, s0, s2
@@ -130,7 +130,7 @@ define void @test_vsel32oge(half* %lhs_ptr, half* %rhs_ptr, half* %a_ptr, half*
 ; CHECK-NEXT:    vldr.16 s4, [r0]
 ; CHECK-NEXT:    vldr.16 s6, [r1]
 ; CHECK-NEXT:    movw r0, :lower16:varhalf
-; CHECK-NEXT:    vcmpe.f16 s4, s6
+; CHECK-NEXT:    vcmp.f16 s4, s6
 ; CHECK-NEXT:    movt r0, :upper16:varhalf
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    vselge.f16 s0, s0, s2
@@ -178,7 +178,7 @@ define void @test_vsel32ugt(half* %lhs_ptr, half* %rhs_ptr, half* %a_ptr, half*
 ; CHECK-NEXT:    vldr.16 s4, [r0]
 ; CHECK-NEXT:    vldr.16 s6, [r1]
 ; CHECK-NEXT:    movw r0, :lower16:varhalf
-; CHECK-NEXT:    vcmpe.f16 s6, s4
+; CHECK-NEXT:    vcmp.f16 s6, s4
 ; CHECK-NEXT:    movt r0, :upper16:varhalf
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    vselge.f16 s0, s2, s0
@@ -202,7 +202,7 @@ define void @test_vsel32uge(half* %lhs_ptr, half* %rhs_ptr, half* %a_ptr, half*
 ; CHECK-NEXT:    vldr.16 s4, [r0]
 ; CHECK-NEXT:    vldr.16 s6, [r1]
 ; CHECK-NEXT:    movw r0, :lower16:varhalf
-; CHECK-NEXT:    vcmpe.f16 s6, s4
+; CHECK-NEXT:    vcmp.f16 s6, s4
 ; CHECK-NEXT:    movt r0, :upper16:varhalf
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    vselgt.f16 s0, s2, s0
@@ -226,7 +226,7 @@ define void @test_vsel32olt(half* %lhs_ptr, half* %rhs_ptr, half* %a_ptr, half*
 ; CHECK-NEXT:    vldr.16 s4, [r0]
 ; CHECK-NEXT:    vldr.16 s6, [r1]
 ; CHECK-NEXT:    movw r0, :lower16:varhalf
-; CHECK-NEXT:    vcmpe.f16 s6, s4
+; CHECK-NEXT:    vcmp.f16 s6, s4
 ; CHECK-NEXT:    movt r0, :upper16:varhalf
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    vselgt.f16 s0, s0, s2
@@ -250,7 +250,7 @@ define void @test_vsel32ult(half* %lhs_ptr, half* %rhs_ptr, half* %a_ptr, half*
 ; CHECK-NEXT:    vldr.16 s4, [r0]
 ; CHECK-NEXT:    vldr.16 s6, [r1]
 ; CHECK-NEXT:    movw r0, :lower16:varhalf
-; CHECK-NEXT:    vcmpe.f16 s4, s6
+; CHECK-NEXT:    vcmp.f16 s4, s6
 ; CHECK-NEXT:    movt r0, :upper16:varhalf
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    vselge.f16 s0, s2, s0
@@ -274,7 +274,7 @@ define void @test_vsel32ole(half* %lhs_ptr, half* %rhs_ptr, half* %a_ptr, half*
 ; CHECK-NEXT:    vldr.16 s4, [r0]
 ; CHECK-NEXT:    vldr.16 s6, [r1]
 ; CHECK-NEXT:    movw r0, :lower16:varhalf
-; CHECK-NEXT:    vcmpe.f16 s6, s4
+; CHECK-NEXT:    vcmp.f16 s6, s4
 ; CHECK-NEXT:    movt r0, :upper16:varhalf
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    vselge.f16 s0, s0, s2
@@ -298,7 +298,7 @@ define void @test_vsel32ule(half* %lhs_ptr, half* %rhs_ptr, half* %a_ptr, half*
 ; CHECK-NEXT:    vldr.16 s4, [r0]
 ; CHECK-NEXT:    vldr.16 s6, [r1]
 ; CHECK-NEXT:    movw r0, :lower16:varhalf
-; CHECK-NEXT:    vcmpe.f16 s4, s6
+; CHECK-NEXT:    vcmp.f16 s4, s6
 ; CHECK-NEXT:    movt r0, :upper16:varhalf
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    vselgt.f16 s0, s2, s0
@@ -322,7 +322,7 @@ define void @test_vsel32ord(half* %lhs_ptr, half* %rhs_ptr, half* %a_ptr, half*
 ; CHECK-NEXT:    vldr.16 s4, [r0]
 ; CHECK-NEXT:    vldr.16 s6, [r1]
 ; CHECK-NEXT:    movw r0, :lower16:varhalf
-; CHECK-NEXT:    vcmpe.f16 s4, s6
+; CHECK-NEXT:    vcmp.f16 s4, s6
 ; CHECK-NEXT:    movt r0, :upper16:varhalf
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    vselvs.f16 s0, s2, s0
@@ -370,7 +370,7 @@ define void @test_vsel32uno(half* %lhs_ptr, half* %rhs_ptr, half* %a_ptr, half*
 ; CHECK-NEXT:    vldr.16 s4, [r0]
 ; CHECK-NEXT:    vldr.16 s6, [r1]
 ; CHECK-NEXT:    movw r0, :lower16:varhalf
-; CHECK-NEXT:    vcmpe.f16 s4, s6
+; CHECK-NEXT:    vcmp.f16 s4, s6
 ; CHECK-NEXT:    movt r0, :upper16:varhalf
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    vselvs.f16 s0, s0, s2
@@ -395,7 +395,7 @@ define void @test_vsel32ogt_nnan(half* %lhs_ptr, half* %rhs_ptr, half* %a_ptr, h
 ; CHECK-NEXT:    vldr.16 s4, [r0]
 ; CHECK-NEXT:    vldr.16 s6, [r1]
 ; CHECK-NEXT:    movw r0, :lower16:varhalf
-; CHECK-NEXT:    vcmpe.f16 s4, s6
+; CHECK-NEXT:    vcmp.f16 s4, s6
 ; CHECK-NEXT:    movt r0, :upper16:varhalf
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    vselgt.f16 s0, s0, s2
@@ -419,7 +419,7 @@ define void @test_vsel32oge_nnan(half* %lhs_ptr, half* %rhs_ptr, half* %a_ptr, h
 ; CHECK-NEXT:    vldr.16 s4, [r0]
 ; CHECK-NEXT:    vldr.16 s6, [r1]
 ; CHECK-NEXT:    movw r0, :lower16:varhalf
-; CHECK-NEXT:    vcmpe.f16 s4, s6
+; CHECK-NEXT:    vcmp.f16 s4, s6
 ; CHECK-NEXT:    movt r0, :upper16:varhalf
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    vselge.f16 s0, s0, s2
@@ -467,7 +467,7 @@ define void @test_vsel32ugt_nnan(half* %lhs_ptr, half* %rhs_ptr, half* %a_ptr, h
 ; CHECK-NEXT:    vldr.16 s4, [r0]
 ; CHECK-NEXT:    vldr.16 s6, [r1]
 ; CHECK-NEXT:    movw r0, :lower16:varhalf
-; CHECK-NEXT:    vcmpe.f16 s4, s6
+; CHECK-NEXT:    vcmp.f16 s4, s6
 ; CHECK-NEXT:    movt r0, :upper16:varhalf
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    vselgt.f16 s0, s0, s2
@@ -491,7 +491,7 @@ define void @test_vsel32uge_nnan(half* %lhs_ptr, half* %rhs_ptr, half* %a_ptr, h
 ; CHECK-NEXT:    vldr.16 s4, [r0]
 ; CHECK-NEXT:    vldr.16 s6, [r1]
 ; CHECK-NEXT:    movw r0, :lower16:varhalf
-; CHECK-NEXT:    vcmpe.f16 s4, s6
+; CHECK-NEXT:    vcmp.f16 s4, s6
 ; CHECK-NEXT:    movt r0, :upper16:varhalf
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    vselge.f16 s0, s0, s2
@@ -515,7 +515,7 @@ define void @test_vsel32olt_nnan(half* %lhs_ptr, half* %rhs_ptr, half* %a_ptr, h
 ; CHECK-NEXT:    vldr.16 s4, [r0]
 ; CHECK-NEXT:    vldr.16 s6, [r1]
 ; CHECK-NEXT:    movw r0, :lower16:varhalf
-; CHECK-NEXT:    vcmpe.f16 s6, s4
+; CHECK-NEXT:    vcmp.f16 s6, s4
 ; CHECK-NEXT:    movt r0, :upper16:varhalf
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    vselgt.f16 s0, s0, s2
@@ -539,7 +539,7 @@ define void @test_vsel32ult_nnan(half* %lhs_ptr, half* %rhs_ptr, half* %a_ptr, h
 ; CHECK-NEXT:    vldr.16 s4, [r0]
 ; CHECK-NEXT:    vldr.16 s6, [r1]
 ; CHECK-NEXT:    movw r0, :lower16:varhalf
-; CHECK-NEXT:    vcmpe.f16 s6, s4
+; CHECK-NEXT:    vcmp.f16 s6, s4
 ; CHECK-NEXT:    movt r0, :upper16:varhalf
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    vselgt.f16 s0, s0, s2
@@ -563,7 +563,7 @@ define void @test_vsel32ole_nnan(half* %lhs_ptr, half* %rhs_ptr, half* %a_ptr, h
 ; CHECK-NEXT:    vldr.16 s4, [r0]
 ; CHECK-NEXT:    vldr.16 s6, [r1]
 ; CHECK-NEXT:    movw r0, :lower16:varhalf
-; CHECK-NEXT:    vcmpe.f16 s6, s4
+; CHECK-NEXT:    vcmp.f16 s6, s4
 ; CHECK-NEXT:    movt r0, :upper16:varhalf
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    vselge.f16 s0, s0, s2
@@ -587,7 +587,7 @@ define void @test_vsel32ule_nnan(half* %lhs_ptr, half* %rhs_ptr, half* %a_ptr, h
 ; CHECK-NEXT:    vldr.16 s4, [r0]
 ; CHECK-NEXT:    vldr.16 s6, [r1]
 ; CHECK-NEXT:    movw r0, :lower16:varhalf
-; CHECK-NEXT:    vcmpe.f16 s6, s4
+; CHECK-NEXT:    vcmp.f16 s6, s4
 ; CHECK-NEXT:    movt r0, :upper16:varhalf
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    vselge.f16 s0, s0, s2
@@ -611,7 +611,7 @@ define void @test_vsel32ord_nnan(half* %lhs_ptr, half* %rhs_ptr, half* %a_ptr, h
 ; CHECK-NEXT:    vldr.16 s4, [r0]
 ; CHECK-NEXT:    vldr.16 s6, [r1]
 ; CHECK-NEXT:    movw r0, :lower16:varhalf
-; CHECK-NEXT:    vcmpe.f16 s4, s6
+; CHECK-NEXT:    vcmp.f16 s4, s6
 ; CHECK-NEXT:    movt r0, :upper16:varhalf
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    vselvs.f16 s0, s2, s0
@@ -659,7 +659,7 @@ define void @test_vsel32uno_nnan(half* %lhs_ptr, half* %rhs_ptr, half* %a_ptr, h
 ; CHECK-NEXT:    vldr.16 s4, [r0]
 ; CHECK-NEXT:    vldr.16 s6, [r1]
 ; CHECK-NEXT:    movw r0, :lower16:varhalf
-; CHECK-NEXT:    vcmpe.f16 s4, s6
+; CHECK-NEXT:    vcmp.f16 s4, s6
 ; CHECK-NEXT:    movt r0, :upper16:varhalf
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    vselvs.f16 s0, s0, s2
diff --git a/llvm/test/CodeGen/ARM/vsel.ll b/llvm/test/CodeGen/ARM/vsel.ll
index 9408424e3..33d16ad45 100644
--- a/llvm/test/CodeGen/ARM/vsel.ll
+++ b/llvm/test/CodeGen/ARM/vsel.ll
@@ -96,7 +96,7 @@ define void @test_vsel32ogt(float %lhs32, float %rhs32, float %a, float %b) {
   %tst1 = fcmp ogt float %lhs32, %rhs32
   %val1 = select i1 %tst1, float %a, float %b
   store float %val1, float* @varfloat
-; CHECK: vcmpe.f32 s0, s1
+; CHECK: vcmp.f32 s0, s1
 ; CHECK: vselgt.f32 s0, s2, s3
   ret void
 }
@@ -105,7 +105,7 @@ define void @test_vsel64ogt(float %lhs32, float %rhs32, double %a, double %b) {
   %tst1 = fcmp ogt float %lhs32, %rhs32
   %val1 = select i1 %tst1, double %a, double %b
   store double %val1, double* @vardouble
-; CHECK: vcmpe.f32 s0, s1
+; CHECK: vcmp.f32 s0, s1
 ; CHECK: vselgt.f64 d16, d1, d2
   ret void
 }
@@ -114,7 +114,7 @@ define void @test_vsel32oge(float %lhs32, float %rhs32, float %a, float %b) {
   %tst1 = fcmp oge float %lhs32, %rhs32
   %val1 = select i1 %tst1, float %a, float %b
   store float %val1, float* @varfloat
-; CHECK: vcmpe.f32 s0, s1
+; CHECK: vcmp.f32 s0, s1
 ; CHECK: vselge.f32 s0, s2, s3
   ret void
 }
@@ -123,7 +123,7 @@ define void @test_vsel64oge(float %lhs32, float %rhs32, double %a, double %b) {
   %tst1 = fcmp oge float %lhs32, %rhs32
   %val1 = select i1 %tst1, double %a, double %b
   store double %val1, double* @vardouble
-; CHECK: vcmpe.f32 s0, s1
+; CHECK: vcmp.f32 s0, s1
 ; CHECK: vselge.f64 d16, d1, d2
   ret void
 }
@@ -150,7 +150,7 @@ define void @test_vsel32ugt(float %lhs32, float %rhs32, float %a, float %b) {
   %tst1 = fcmp ugt float %lhs32, %rhs32
   %val1 = select i1 %tst1, float %a, float %b
   store float %val1, float* @varfloat
-; CHECK: vcmpe.f32 s1, s0
+; CHECK: vcmp.f32 s1, s0
 ; CHECK: vselge.f32 s0, s3, s2
   ret void
 }
@@ -159,7 +159,7 @@ define void @test_vsel64ugt(float %lhs32, float %rhs32, double %a, double %b) {
   %tst1 = fcmp ugt float %lhs32, %rhs32
   %val1 = select i1 %tst1, double %a, double %b
   store double %val1, double* @vardouble
-; CHECK: vcmpe.f32 s1, s0
+; CHECK: vcmp.f32 s1, s0
 ; CHECK: vselge.f64 d16, d2, d1
   ret void
 }
@@ -168,7 +168,7 @@ define void @test_vsel32uge(float %lhs32, float %rhs32, float %a, float %b) {
   %tst1 = fcmp uge float %lhs32, %rhs32
   %val1 = select i1 %tst1, float %a, float %b
   store float %val1, float* @varfloat
-; CHECK: vcmpe.f32 s1, s0
+; CHECK: vcmp.f32 s1, s0
 ; CHECK: vselgt.f32 s0, s3, s2
   ret void
 }
@@ -177,7 +177,7 @@ define void @test_vsel64uge(float %lhs32, float %rhs32, double %a, double %b) {
   %tst1 = fcmp uge float %lhs32, %rhs32
   %val1 = select i1 %tst1, double %a, double %b
   store double %val1, double* @vardouble
-; CHECK: vcmpe.f32 s1, s0
+; CHECK: vcmp.f32 s1, s0
 ; CHECK: vselgt.f64 d16, d2, d1
   ret void
 }
@@ -186,7 +186,7 @@ define void @test_vsel32olt(float %lhs32, float %rhs32, float %a, float %b) {
   %tst1 = fcmp olt float %lhs32, %rhs32
   %val1 = select i1 %tst1, float %a, float %b
   store float %val1, float* @varfloat
-; CHECK: vcmpe.f32 s1, s0
+; CHECK: vcmp.f32 s1, s0
 ; CHECK: vselgt.f32 s0, s2, s3
   ret void
 }
@@ -195,7 +195,7 @@ define void @test_vsel64olt(float %lhs32, float %rhs32, double %a, double %b) {
   %tst1 = fcmp olt float %lhs32, %rhs32
   %val1 = select i1 %tst1, double %a, double %b
   store double %val1, double* @vardouble
-; CHECK: vcmpe.f32 s1, s0
+; CHECK: vcmp.f32 s1, s0
 ; CHECK: vselgt.f64 d16, d1, d2
   ret void
 }
@@ -204,7 +204,7 @@ define void @test_vsel32ult(float %lhs32, float %rhs32, float %a, float %b) {
   %tst1 = fcmp ult float %lhs32, %rhs32
   %val1 = select i1 %tst1, float %a, float %b
   store float %val1, float* @varfloat
-; CHECK: vcmpe.f32 s0, s1
+; CHECK: vcmp.f32 s0, s1
 ; CHECK: vselge.f32 s0, s3, s2
   ret void
 }
@@ -213,7 +213,7 @@ define void @test_vsel64ult(float %lhs32, float %rhs32, double %a, double %b) {
   %tst1 = fcmp ult float %lhs32, %rhs32
   %val1 = select i1 %tst1, double %a, double %b
   store double %val1, double* @vardouble
-; CHECK: vcmpe.f32 s0, s1
+; CHECK: vcmp.f32 s0, s1
 ; CHECK: vselge.f64 d16, d2, d1
   ret void
 }
@@ -222,7 +222,7 @@ define void @test_vsel32ole(float %lhs32, float %rhs32, float %a, float %b) {
   %tst1 = fcmp ole float %lhs32, %rhs32
   %val1 = select i1 %tst1, float %a, float %b
   store float %val1, float* @varfloat
-; CHECK: vcmpe.f32 s1, s0
+; CHECK: vcmp.f32 s1, s0
 ; CHECK: vselge.f32 s0, s2, s3
   ret void
 }
@@ -231,7 +231,7 @@ define void @test_vsel64ole(float %lhs32, float %rhs32, double %a, double %b) {
   %tst1 = fcmp ole float %lhs32, %rhs32
   %val1 = select i1 %tst1, double %a, double %b
   store double %val1, double* @vardouble
-; CHECK: vcmpe.f32 s1, s0
+; CHECK: vcmp.f32 s1, s0
 ; CHECK: vselge.f64 d16, d1, d2
   ret void
 }
@@ -240,7 +240,7 @@ define void @test_vsel32ule(float %lhs32, float %rhs32, float %a, float %b) {
   %tst1 = fcmp ule float %lhs32, %rhs32
   %val1 = select i1 %tst1, float %a, float %b
   store float %val1, float* @varfloat
-; CHECK: vcmpe.f32 s0, s1
+; CHECK: vcmp.f32 s0, s1
 ; CHECK: vselgt.f32 s0, s3, s2
   ret void
 }
@@ -249,7 +249,7 @@ define void @test_vsel64ule(float %lhs32, float %rhs32, double %a, double %b) {
   %tst1 = fcmp ule float %lhs32, %rhs32
   %val1 = select i1 %tst1, double %a, double %b
   store double %val1, double* @vardouble
-; CHECK: vcmpe.f32 s0, s1
+; CHECK: vcmp.f32 s0, s1
 ; CHECK: vselgt.f64 d16, d2, d1
   ret void
 }
@@ -258,7 +258,7 @@ define void @test_vsel32ord(float %lhs32, float %rhs32, float %a, float %b) {
   %tst1 = fcmp ord float %lhs32, %rhs32
   %val1 = select i1 %tst1, float %a, float %b
   store float %val1, float* @varfloat
-; CHECK: vcmpe.f32 s0, s1
+; CHECK: vcmp.f32 s0, s1
 ; CHECK: vselvs.f32 s0, s3, s2
   ret void
 }
@@ -267,7 +267,7 @@ define void @test_vsel64ord(float %lhs32, float %rhs32, double %a, double %b) {
   %tst1 = fcmp ord float %lhs32, %rhs32
   %val1 = select i1 %tst1, double %a, double %b
   store double %val1, double* @vardouble
-; CHECK: vcmpe.f32 s0, s1
+; CHECK: vcmp.f32 s0, s1
 ; CHECK: vselvs.f64 d16, d2, d1
   ret void
 }
@@ -294,7 +294,7 @@ define void @test_vsel32uno(float %lhs32, float %rhs32, float %a, float %b) {
   %tst1 = fcmp uno float %lhs32, %rhs32
   %val1 = select i1 %tst1, float %a, float %b
   store float %val1, float* @varfloat
-; CHECK: vcmpe.f32 s0, s1
+; CHECK: vcmp.f32 s0, s1
 ; CHECK: vselvs.f32 s0, s2, s3
   ret void
 }
@@ -303,7 +303,7 @@ define void @test_vsel64uno(float %lhs32, float %rhs32, double %a, double %b) {
   %tst1 = fcmp uno float %lhs32, %rhs32
   %val1 = select i1 %tst1, double %a, double %b
   store double %val1, double* @vardouble
-; CHECK: vcmpe.f32 s0, s1
+; CHECK: vcmp.f32 s0, s1
 ; CHECK: vselvs.f64 d16, d1, d2
   ret void
 }
@@ -313,7 +313,7 @@ define void @test_vsel32ogt_nnan(float %lhs32, float %rhs32, float %a, float %b)
   %tst1 = fcmp nnan ogt float %lhs32, %rhs32
   %val1 = select i1 %tst1, float %a, float %b
   store float %val1, float* @varfloat
-; CHECK: vcmpe.f32 s0, s1
+; CHECK: vcmp.f32 s0, s1
 ; CHECK: vselgt.f32 s0, s2, s3
   ret void
 }
@@ -322,7 +322,7 @@ define void @test_vsel64ogt_nnan(float %lhs32, float %rhs32, double %a, double %
   %tst1 = fcmp nnan ogt float %lhs32, %rhs32
   %val1 = select i1 %tst1, double %a, double %b
   store double %val1, double* @vardouble
-; CHECK: vcmpe.f32 s0, s1
+; CHECK: vcmp.f32 s0, s1
 ; CHECK: vselgt.f64 d16, d1, d2
   ret void
 }
@@ -331,7 +331,7 @@ define void @test_vsel32oge_nnan(float %lhs32, float %rhs32, float %a, float %b)
   %tst1 = fcmp nnan oge float %lhs32, %rhs32
   %val1 = select i1 %tst1, float %a, float %b
   store float %val1, float* @varfloat
-; CHECK: vcmpe.f32 s0, s1
+; CHECK: vcmp.f32 s0, s1
 ; CHECK: vselge.f32 s0, s2, s3
   ret void
 }
@@ -340,7 +340,7 @@ define void @test_vsel64oge_nnan(float %lhs32, float %rhs32, double %a, double %
   %tst1 = fcmp nnan oge float %lhs32, %rhs32
   %val1 = select i1 %tst1, double %a, double %b
   store double %val1, double* @vardouble
-; CHECK: vcmpe.f32 s0, s1
+; CHECK: vcmp.f32 s0, s1
 ; CHECK: vselge.f64 d16, d1, d2
   ret void
 }
@@ -367,7 +367,7 @@ define void @test_vsel32ugt_nnan(float %lhs32, float %rhs32, float %a, float %b)
   %tst1 = fcmp nnan ugt float %lhs32, %rhs32
   %val1 = select i1 %tst1, float %a, float %b
   store float %val1, float* @varfloat
-; CHECK: vcmpe.f32	s0, s1
+; CHECK: vcmp.f32	s0, s1
 ; CHECK: vselgt.f32	s0, s2, s3
   ret void
 }
@@ -376,7 +376,7 @@ define void @test_vsel64ugt_nnan(float %lhs32, float %rhs32, double %a, double %
   %tst1 = fcmp nnan ugt float %lhs32, %rhs32
   %val1 = select i1 %tst1, double %a, double %b
   store double %val1, double* @vardouble
-; CHECK: vcmpe.f32 s0, s1
+; CHECK: vcmp.f32 s0, s1
 ; CHECK: vselgt.f64 d16, d1, d2
   ret void
 }
@@ -385,7 +385,7 @@ define void @test_vsel32uge_nnan(float %lhs32, float %rhs32, float %a, float %b)
   %tst1 = fcmp nnan uge float %lhs32, %rhs32
   %val1 = select i1 %tst1, float %a, float %b
   store float %val1, float* @varfloat
-; CHECK: vcmpe.f32 s0, s1
+; CHECK: vcmp.f32 s0, s1
 ; CHECK: vselge.f32 s0, s2, s3
   ret void
 }
@@ -394,7 +394,7 @@ define void @test_vsel64uge_nnan(float %lhs32, float %rhs32, double %a, double %
   %tst1 = fcmp nnan uge float %lhs32, %rhs32
   %val1 = select i1 %tst1, double %a, double %b
   store double %val1, double* @vardouble
-; CHECK: vcmpe.f32 s0, s1
+; CHECK: vcmp.f32 s0, s1
 ; CHECK: vselge.f64 d16, d1, d2
   ret void
 }
@@ -403,7 +403,7 @@ define void @test_vsel32olt_nnan(float %lhs32, float %rhs32, float %a, float %b)
   %tst1 = fcmp nnan olt float %lhs32, %rhs32
   %val1 = select i1 %tst1, float %a, float %b
   store float %val1, float* @varfloat
-; CHECK: vcmpe.f32 s1, s0
+; CHECK: vcmp.f32 s1, s0
 ; CHECK: vselgt.f32 s0, s2, s3
   ret void
 }
@@ -412,7 +412,7 @@ define void @test_vsel64olt_nnan(float %lhs32, float %rhs32, double %a, double %
   %tst1 = fcmp nnan olt float %lhs32, %rhs32
   %val1 = select i1 %tst1, double %a, double %b
   store double %val1, double* @vardouble
-; CHECK: vcmpe.f32 s1, s0
+; CHECK: vcmp.f32 s1, s0
 ; CHECK: vselgt.f64 d16, d1, d2
   ret void
 }
@@ -421,7 +421,7 @@ define void @test_vsel32ult_nnan(float %lhs32, float %rhs32, float %a, float %b)
   %tst1 = fcmp nnan ult float %lhs32, %rhs32
   %val1 = select i1 %tst1, float %a, float %b
   store float %val1, float* @varfloat
-; CHECK: vcmpe.f32 s1, s0
+; CHECK: vcmp.f32 s1, s0
 ; CHECK: vselgt.f32 s0, s2, s3
   ret void
 }
@@ -430,7 +430,7 @@ define void @test_vsel64ult_nnan(float %lhs32, float %rhs32, double %a, double %
   %tst1 = fcmp nnan ult float %lhs32, %rhs32
   %val1 = select i1 %tst1, double %a, double %b
   store double %val1, double* @vardouble
-; CHECK: vcmpe.f32 s1, s0
+; CHECK: vcmp.f32 s1, s0
 ; CHECK: vselgt.f64 d16, d1, d2
   ret void
 }
@@ -439,7 +439,7 @@ define void @test_vsel32ole_nnan(float %lhs32, float %rhs32, float %a, float %b)
   %tst1 = fcmp nnan ole float %lhs32, %rhs32
   %val1 = select i1 %tst1, float %a, float %b
   store float %val1, float* @varfloat
-; CHECK: vcmpe.f32 s1, s0
+; CHECK: vcmp.f32 s1, s0
 ; CHECK: vselge.f32 s0, s2, s3
   ret void
 }
@@ -448,7 +448,7 @@ define void @test_vsel64ole_nnan(float %lhs32, float %rhs32, double %a, double %
   %tst1 = fcmp nnan ole float %lhs32, %rhs32
   %val1 = select i1 %tst1, double %a, double %b
   store double %val1, double* @vardouble
-; CHECK: vcmpe.f32 s1, s0
+; CHECK: vcmp.f32 s1, s0
 ; CHECK: vselge.f64 d16, d1, d2
   ret void
 }
@@ -457,7 +457,7 @@ define void @test_vsel32ule_nnan(float %lhs32, float %rhs32, float %a, float %b)
   %tst1 = fcmp nnan ule float %lhs32, %rhs32
   %val1 = select i1 %tst1, float %a, float %b
   store float %val1, float* @varfloat
-; CHECK: vcmpe.f32 s1, s0
+; CHECK: vcmp.f32 s1, s0
 ; CHECK: vselge.f32 s0, s2, s3
   ret void
 }
@@ -466,7 +466,7 @@ define void @test_vsel64ule_nnan(float %lhs32, float %rhs32, double %a, double %
   %tst1 = fcmp nnan ule float %lhs32, %rhs32
   %val1 = select i1 %tst1, double %a, double %b
   store double %val1, double* @vardouble
-; CHECK: vcmpe.f32 s1, s0
+; CHECK: vcmp.f32 s1, s0
 ; CHECK: vselge.f64 d16, d1, d2
   ret void
 }
@@ -475,7 +475,7 @@ define void @test_vsel32ord_nnan(float %lhs32, float %rhs32, float %a, float %b)
   %tst1 = fcmp nnan ord float %lhs32, %rhs32
   %val1 = select i1 %tst1, float %a, float %b
   store float %val1, float* @varfloat
-; CHECK: vcmpe.f32 s0, s1
+; CHECK: vcmp.f32 s0, s1
 ; CHECK: vselvs.f32 s0, s3, s2
   ret void
 }
@@ -484,7 +484,7 @@ define void @test_vsel64ord_nnan(float %lhs32, float %rhs32, double %a, double %
   %tst1 = fcmp nnan ord float %lhs32, %rhs32
   %val1 = select i1 %tst1, double %a, double %b
   store double %val1, double* @vardouble
-; CHECK: vcmpe.f32 s0, s1
+; CHECK: vcmp.f32 s0, s1
 ; CHECK: vselvs.f64 d16, d2, d1
   ret void
 }
@@ -511,7 +511,7 @@ define void @test_vsel32uno_nnan(float %lhs32, float %rhs32, float %a, float %b)
   %tst1 = fcmp nnan uno float %lhs32, %rhs32
   %val1 = select i1 %tst1, float %a, float %b
   store float %val1, float* @varfloat
-; CHECK: vcmpe.f32 s0, s1
+; CHECK: vcmp.f32 s0, s1
 ; CHECK: vselvs.f32 s0, s2, s3
   ret void
 }
@@ -520,7 +520,7 @@ define void @test_vsel64uno_nnan(float %lhs32, float %rhs32, double %a, double %
   %tst1 = fcmp nnan uno float %lhs32, %rhs32
   %val1 = select i1 %tst1, double %a, double %b
   store double %val1, double* @vardouble
-; CHECK: vcmpe.f32 s0, s1
+; CHECK: vcmp.f32 s0, s1
 ; CHECK: vselvs.f64 d16, d1, d2
   ret void
 }
diff --git a/llvm/test/CodeGen/ARM/vstlane.ll b/llvm/test/CodeGen/ARM/vstlane.ll
index 49af0be92..2515034fb 100644
--- a/llvm/test/CodeGen/ARM/vstlane.ll
+++ b/llvm/test/CodeGen/ARM/vstlane.ll
@@ -1,19 +1,28 @@
-; RUN: llc -mtriple=arm -mattr=+neon %s -o - | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=arm -mattr=+neon | FileCheck %s
 
-define void @vst1lanei8(i8* %A, <8 x i8>* %B) nounwind {
-;CHECK-LABEL: vst1lanei8:
 ;Check the (default) alignment.
-;CHECK: vst1.8 {d16[3]}, [r0]
+define void @vst1lanei8(i8* %A, <8 x i8>* %B) nounwind {
+; CHECK-LABEL: vst1lanei8:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr d16, [r1]
+; CHECK-NEXT:    vst1.8 {d16[3]}, [r0]
+; CHECK-NEXT:    mov pc, lr
 	%tmp1 = load <8 x i8>, <8 x i8>* %B
-        %tmp2 = extractelement <8 x i8> %tmp1, i32 3
-        store i8 %tmp2, i8* %A, align 8
+	%tmp2 = extractelement <8 x i8> %tmp1, i32 3
+	store i8 %tmp2, i8* %A, align 8
 	ret void
 }
 
 ;Check for a post-increment updating store.
 define void @vst1lanei8_update(i8** %ptr, <8 x i8>* %B) nounwind {
-;CHECK-LABEL: vst1lanei8_update:
-;CHECK: vst1.8 {d16[3]}, [{{r[0-9]}}]!
+; CHECK-LABEL: vst1lanei8_update:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    ldr r2, [r0]
+; CHECK-NEXT:    vldr d16, [r1]
+; CHECK-NEXT:    vst1.8 {d16[3]}, [r2]!
+; CHECK-NEXT:    str r2, [r0]
+; CHECK-NEXT:    mov pc, lr
 	%A = load i8*, i8** %ptr
 	%tmp1 = load <8 x i8>, <8 x i8>* %B
 	%tmp2 = extractelement <8 x i8> %tmp1, i32 3
@@ -23,69 +32,95 @@ define void @vst1lanei8_update(i8** %ptr, <8 x i8>* %B) nounwind {
 	ret void
 }
 
-define void @vst1lanei16(i16* %A, <4 x i16>* %B) nounwind {
-;CHECK-LABEL: vst1lanei16:
 ;Check the alignment value.  Max for this instruction is 16 bits:
-;CHECK: vst1.16 {d16[2]}, [r0:16]
+define void @vst1lanei16(i16* %A, <4 x i16>* %B) nounwind {
+; CHECK-LABEL: vst1lanei16:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr d16, [r1]
+; CHECK-NEXT:    vst1.16 {d16[2]}, [r0:16]
+; CHECK-NEXT:    mov pc, lr
 	%tmp1 = load <4 x i16>, <4 x i16>* %B
-        %tmp2 = extractelement <4 x i16> %tmp1, i32 2
-        store i16 %tmp2, i16* %A, align 8
+	%tmp2 = extractelement <4 x i16> %tmp1, i32 2
+	store i16 %tmp2, i16* %A, align 8
 	ret void
 }
 
-define void @vst1lanei32(i32* %A, <2 x i32>* %B) nounwind {
-;CHECK-LABEL: vst1lanei32:
 ;Check the alignment value.  Max for this instruction is 32 bits:
-;CHECK: vst1.32 {d16[1]}, [r0:32]
+define void @vst1lanei32(i32* %A, <2 x i32>* %B) nounwind {
+; CHECK-LABEL: vst1lanei32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr d16, [r1]
+; CHECK-NEXT:    vst1.32 {d16[1]}, [r0:32]
+; CHECK-NEXT:    mov pc, lr
 	%tmp1 = load <2 x i32>, <2 x i32>* %B
-        %tmp2 = extractelement <2 x i32> %tmp1, i32 1
-        store i32 %tmp2, i32* %A, align 8
+	%tmp2 = extractelement <2 x i32> %tmp1, i32 1
+	store i32 %tmp2, i32* %A, align 8
 	ret void
 }
 
 define void @vst1lanef(float* %A, <2 x float>* %B) nounwind {
-;CHECK-LABEL: vst1lanef:
-;CHECK: vst1.32 {d16[1]}, [r0:32]
+; CHECK-LABEL: vst1lanef:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr d16, [r1]
+; CHECK-NEXT:    vst1.32 {d16[1]}, [r0:32]
+; CHECK-NEXT:    mov pc, lr
 	%tmp1 = load <2 x float>, <2 x float>* %B
-        %tmp2 = extractelement <2 x float> %tmp1, i32 1
-        store float %tmp2, float* %A
+	%tmp2 = extractelement <2 x float> %tmp1, i32 1
+	store float %tmp2, float* %A
 	ret void
 }
 
-define void @vst1laneQi8(i8* %A, <16 x i8>* %B) nounwind {
-;CHECK-LABEL: vst1laneQi8:
 ; // Can use scalar load. No need to use vectors.
 ; // CHE-CK: vst1.8 {d17[1]}, [r0]
+define void @vst1laneQi8(i8* %A, <16 x i8>* %B) nounwind {
+; CHECK-LABEL: vst1laneQi8:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r1]
+; CHECK-NEXT:    vst1.8 {d17[1]}, [r0]
+; CHECK-NEXT:    mov pc, lr
 	%tmp1 = load <16 x i8>, <16 x i8>* %B
-        %tmp2 = extractelement <16 x i8> %tmp1, i32 9
-        store i8 %tmp2, i8* %A, align 8
+	%tmp2 = extractelement <16 x i8> %tmp1, i32 9
+	store i8 %tmp2, i8* %A, align 8
 	ret void
 }
 
 define void @vst1laneQi16(i16* %A, <8 x i16>* %B) nounwind {
-;CHECK-LABEL: vst1laneQi16:
-;CHECK: vst1.16 {d17[1]}, [r0:16]
+; CHECK-LABEL: vst1laneQi16:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r1]
+; CHECK-NEXT:    vst1.16 {d17[1]}, [r0:16]
+; CHECK-NEXT:    mov pc, lr
 	%tmp1 = load <8 x i16>, <8 x i16>* %B
-        %tmp2 = extractelement <8 x i16> %tmp1, i32 5
-        store i16 %tmp2, i16* %A, align 8
+	%tmp2 = extractelement <8 x i16> %tmp1, i32 5
+	store i16 %tmp2, i16* %A, align 8
 	ret void
 }
 
-define void @vst1laneQi32(i32* %A, <4 x i32>* %B) nounwind {
-;CHECK-LABEL: vst1laneQi32:
 ; // Can use scalar load. No need to use vectors.
 ; // CHE-CK: vst1.32 {d17[1]}, [r0:32]
+define void @vst1laneQi32(i32* %A, <4 x i32>* %B) nounwind {
+; CHECK-LABEL: vst1laneQi32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    ldr r1, [r1, #12]
+; CHECK-NEXT:    str r1, [r0]
+; CHECK-NEXT:    mov pc, lr
 	%tmp1 = load <4 x i32>, <4 x i32>* %B
-        %tmp2 = extractelement <4 x i32> %tmp1, i32 3
-        store i32 %tmp2, i32* %A, align 8
+	%tmp2 = extractelement <4 x i32> %tmp1, i32 3
+	store i32 %tmp2, i32* %A, align 8
 	ret void
 }
 
 ;Check for a post-increment updating store.
-define void @vst1laneQi32_update(i32** %ptr, <4 x i32>* %B) nounwind {
-;CHECK-LABEL: vst1laneQi32_update:
 ; // Can use scalar load. No need to use vectors.
 ; // CHE-CK: vst1.32 {d17[1]}, [r1:32]!
+define void @vst1laneQi32_update(i32** %ptr, <4 x i32>* %B) nounwind {
+; CHECK-LABEL: vst1laneQi32_update:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    ldr r2, [r0]
+; CHECK-NEXT:    ldr r1, [r1, #12]
+; CHECK-NEXT:    str r1, [r2], #4
+; CHECK-NEXT:    str r2, [r0]
+; CHECK-NEXT:    mov pc, lr
 	%A = load i32*, i32** %ptr
 	%tmp1 = load <4 x i32>, <4 x i32>* %B
 	%tmp2 = extractelement <4 x i32> %tmp1, i32 3
@@ -95,29 +130,41 @@ define void @vst1laneQi32_update(i32** %ptr, <4 x i32>* %B) nounwind {
 	ret void
 }
 
-define void @vst1laneQf(float* %A, <4 x float>* %B) nounwind {
-;CHECK-LABEL: vst1laneQf:
 ; // Can use scalar load. No need to use vectors.
 ; // CHE-CK: vst1.32 {d17[1]}, [r0]
+define void @vst1laneQf(float* %A, <4 x float>* %B) nounwind {
+; CHECK-LABEL: vst1laneQf:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    ldr r1, [r1, #12]
+; CHECK-NEXT:    str r1, [r0]
+; CHECK-NEXT:    mov pc, lr
 	%tmp1 = load <4 x float>, <4 x float>* %B
-        %tmp2 = extractelement <4 x float> %tmp1, i32 3
-        store float %tmp2, float* %A
+	%tmp2 = extractelement <4 x float> %tmp1, i32 3
+	store float %tmp2, float* %A
 	ret void
 }
 
-define void @vst2lanei8(i8* %A, <8 x i8>* %B) nounwind {
-;CHECK-LABEL: vst2lanei8:
 ;Check the alignment value.  Max for this instruction is 16 bits:
-;CHECK: vst2.8 {d16[1], d17[1]}, [r0:16]
+define void @vst2lanei8(i8* %A, <8 x i8>* %B) nounwind {
+; CHECK-LABEL: vst2lanei8:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr d16, [r1]
+; CHECK-NEXT:    vorr d17, d16, d16
+; CHECK-NEXT:    vst2.8 {d16[1], d17[1]}, [r0:16]
+; CHECK-NEXT:    mov pc, lr
 	%tmp1 = load <8 x i8>, <8 x i8>* %B
 	call void @llvm.arm.neon.vst2lane.p0i8.v8i8(i8* %A, <8 x i8> %tmp1, <8 x i8> %tmp1, i32 1, i32 4)
 	ret void
 }
 
-define void @vst2lanei16(i16* %A, <4 x i16>* %B) nounwind {
-;CHECK-LABEL: vst2lanei16:
 ;Check the alignment value.  Max for this instruction is 32 bits:
-;CHECK: vst2.16 {d16[1], d17[1]}, [r0:32]
+define void @vst2lanei16(i16* %A, <4 x i16>* %B) nounwind {
+; CHECK-LABEL: vst2lanei16:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr d16, [r1]
+; CHECK-NEXT:    vorr d17, d16, d16
+; CHECK-NEXT:    vst2.16 {d16[1], d17[1]}, [r0:32]
+; CHECK-NEXT:    mov pc, lr
 	%tmp0 = bitcast i16* %A to i8*
 	%tmp1 = load <4 x i16>, <4 x i16>* %B
 	call void @llvm.arm.neon.vst2lane.p0i8.v4i16(i8* %tmp0, <4 x i16> %tmp1, <4 x i16> %tmp1, i32 1, i32 8)
@@ -126,8 +173,15 @@ define void @vst2lanei16(i16* %A, <4 x i16>* %B) nounwind {
 
 ;Check for a post-increment updating store with register increment.
 define void @vst2lanei16_update(i16** %ptr, <4 x i16>* %B, i32 %inc) nounwind {
-;CHECK-LABEL: vst2lanei16_update:
-;CHECK: vst2.16 {d16[1], d17[1]}, [r{{[0-9]+}}], r{{[0-9]+}}
+; CHECK-LABEL: vst2lanei16_update:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr d16, [r1]
+; CHECK-NEXT:    lsl r1, r2, #1
+; CHECK-NEXT:    ldr r3, [r0]
+; CHECK-NEXT:    vorr d17, d16, d16
+; CHECK-NEXT:    vst2.16 {d16[1], d17[1]}, [r3], r1
+; CHECK-NEXT:    str r3, [r0]
+; CHECK-NEXT:    mov pc, lr
 	%A = load i16*, i16** %ptr
 	%tmp0 = bitcast i16* %A to i8*
 	%tmp1 = load <4 x i16>, <4 x i16>* %B
@@ -138,8 +192,12 @@ define void @vst2lanei16_update(i16** %ptr, <4 x i16>* %B, i32 %inc) nounwind {
 }
 
 define void @vst2lanei32(i32* %A, <2 x i32>* %B) nounwind {
-;CHECK-LABEL: vst2lanei32:
-;CHECK: vst2.32
+; CHECK-LABEL: vst2lanei32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr d16, [r1]
+; CHECK-NEXT:    vorr d17, d16, d16
+; CHECK-NEXT:    vst2.32 {d16[1], d17[1]}, [r0]
+; CHECK-NEXT:    mov pc, lr
 	%tmp0 = bitcast i32* %A to i8*
 	%tmp1 = load <2 x i32>, <2 x i32>* %B
 	call void @llvm.arm.neon.vst2lane.p0i8.v2i32(i8* %tmp0, <2 x i32> %tmp1, <2 x i32> %tmp1, i32 1, i32 1)
@@ -147,28 +205,40 @@ define void @vst2lanei32(i32* %A, <2 x i32>* %B) nounwind {
 }
 
 define void @vst2lanef(float* %A, <2 x float>* %B) nounwind {
-;CHECK-LABEL: vst2lanef:
-;CHECK: vst2.32
+; CHECK-LABEL: vst2lanef:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr d16, [r1]
+; CHECK-NEXT:    vorr d17, d16, d16
+; CHECK-NEXT:    vst2.32 {d16[1], d17[1]}, [r0]
+; CHECK-NEXT:    mov pc, lr
 	%tmp0 = bitcast float* %A to i8*
 	%tmp1 = load <2 x float>, <2 x float>* %B
 	call void @llvm.arm.neon.vst2lane.p0i8.v2f32(i8* %tmp0, <2 x float> %tmp1, <2 x float> %tmp1, i32 1, i32 1)
 	ret void
 }
 
-define void @vst2laneQi16(i16* %A, <8 x i16>* %B) nounwind {
-;CHECK-LABEL: vst2laneQi16:
 ;Check the (default) alignment.
-;CHECK: vst2.16 {d17[1], d19[1]}, [r0]
+define void @vst2laneQi16(i16* %A, <8 x i16>* %B) nounwind {
+; CHECK-LABEL: vst2laneQi16:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r1]
+; CHECK-NEXT:    vorr q9, q8, q8
+; CHECK-NEXT:    vst2.16 {d17[1], d19[1]}, [r0]
+; CHECK-NEXT:    mov pc, lr
 	%tmp0 = bitcast i16* %A to i8*
 	%tmp1 = load <8 x i16>, <8 x i16>* %B
 	call void @llvm.arm.neon.vst2lane.p0i8.v8i16(i8* %tmp0, <8 x i16> %tmp1, <8 x i16> %tmp1, i32 5, i32 1)
 	ret void
 }
 
-define void @vst2laneQi32(i32* %A, <4 x i32>* %B) nounwind {
-;CHECK-LABEL: vst2laneQi32:
 ;Check the alignment value.  Max for this instruction is 64 bits:
-;CHECK: vst2.32 {d17[0], d19[0]}, [r0:64]
+define void @vst2laneQi32(i32* %A, <4 x i32>* %B) nounwind {
+; CHECK-LABEL: vst2laneQi32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r1]
+; CHECK-NEXT:    vorr q9, q8, q8
+; CHECK-NEXT:    vst2.32 {d17[0], d19[0]}, [r0:64]
+; CHECK-NEXT:    mov pc, lr
 	%tmp0 = bitcast i32* %A to i8*
 	%tmp1 = load <4 x i32>, <4 x i32>* %B
 	call void @llvm.arm.neon.vst2lane.p0i8.v4i32(i8* %tmp0, <4 x i32> %tmp1, <4 x i32> %tmp1, i32 2, i32 16)
@@ -176,8 +246,12 @@ define void @vst2laneQi32(i32* %A, <4 x i32>* %B) nounwind {
 }
 
 define void @vst2laneQf(float* %A, <4 x float>* %B) nounwind {
-;CHECK-LABEL: vst2laneQf:
-;CHECK: vst2.32
+; CHECK-LABEL: vst2laneQf:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r1]
+; CHECK-NEXT:    vorr q9, q8, q8
+; CHECK-NEXT:    vst2.32 {d17[1], d19[1]}, [r0]
+; CHECK-NEXT:    mov pc, lr
 	%tmp0 = bitcast float* %A to i8*
 	%tmp1 = load <4 x float>, <4 x float>* %B
 	call void @llvm.arm.neon.vst2lane.p0i8.v4f32(i8* %tmp0, <4 x float> %tmp1, <4 x float> %tmp1, i32 3, i32 1)
@@ -194,17 +268,27 @@ declare void @llvm.arm.neon.vst2lane.p0i8.v4i32(i8*, <4 x i32>, <4 x i32>, i32,
 declare void @llvm.arm.neon.vst2lane.p0i8.v4f32(i8*, <4 x float>, <4 x float>, i32, i32) nounwind
 
 define void @vst3lanei8(i8* %A, <8 x i8>* %B) nounwind {
-;CHECK-LABEL: vst3lanei8:
-;CHECK: vst3.8
+; CHECK-LABEL: vst3lanei8:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr d16, [r1]
+; CHECK-NEXT:    vorr d17, d16, d16
+; CHECK-NEXT:    vorr d18, d16, d16
+; CHECK-NEXT:    vst3.8 {d16[1], d17[1], d18[1]}, [r0]
+; CHECK-NEXT:    mov pc, lr
 	%tmp1 = load <8 x i8>, <8 x i8>* %B
 	call void @llvm.arm.neon.vst3lane.p0i8.v8i8(i8* %A, <8 x i8> %tmp1, <8 x i8> %tmp1, <8 x i8> %tmp1, i32 1, i32 1)
 	ret void
 }
 
-define void @vst3lanei16(i16* %A, <4 x i16>* %B) nounwind {
-;CHECK-LABEL: vst3lanei16:
 ;Check the (default) alignment value.  VST3 does not support alignment.
-;CHECK: vst3.16 {d16[1], d17[1], d18[1]}, [r0]
+define void @vst3lanei16(i16* %A, <4 x i16>* %B) nounwind {
+; CHECK-LABEL: vst3lanei16:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr d16, [r1]
+; CHECK-NEXT:    vorr d17, d16, d16
+; CHECK-NEXT:    vorr d18, d16, d16
+; CHECK-NEXT:    vst3.16 {d16[1], d17[1], d18[1]}, [r0]
+; CHECK-NEXT:    mov pc, lr
 	%tmp0 = bitcast i16* %A to i8*
 	%tmp1 = load <4 x i16>, <4 x i16>* %B
 	call void @llvm.arm.neon.vst3lane.p0i8.v4i16(i8* %tmp0, <4 x i16> %tmp1, <4 x i16> %tmp1, <4 x i16> %tmp1, i32 1, i32 8)
@@ -212,8 +296,13 @@ define void @vst3lanei16(i16* %A, <4 x i16>* %B) nounwind {
 }
 
 define void @vst3lanei32(i32* %A, <2 x i32>* %B) nounwind {
-;CHECK-LABEL: vst3lanei32:
-;CHECK: vst3.32
+; CHECK-LABEL: vst3lanei32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr d16, [r1]
+; CHECK-NEXT:    vorr d17, d16, d16
+; CHECK-NEXT:    vorr d18, d16, d16
+; CHECK-NEXT:    vst3.32 {d16[1], d17[1], d18[1]}, [r0]
+; CHECK-NEXT:    mov pc, lr
 	%tmp0 = bitcast i32* %A to i8*
 	%tmp1 = load <2 x i32>, <2 x i32>* %B
 	call void @llvm.arm.neon.vst3lane.p0i8.v2i32(i8* %tmp0, <2 x i32> %tmp1, <2 x i32> %tmp1, <2 x i32> %tmp1, i32 1, i32 1)
@@ -221,8 +310,13 @@ define void @vst3lanei32(i32* %A, <2 x i32>* %B) nounwind {
 }
 
 define void @vst3lanef(float* %A, <2 x float>* %B) nounwind {
-;CHECK-LABEL: vst3lanef:
-;CHECK: vst3.32
+; CHECK-LABEL: vst3lanef:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr d16, [r1]
+; CHECK-NEXT:    vorr d17, d16, d16
+; CHECK-NEXT:    vorr d18, d16, d16
+; CHECK-NEXT:    vst3.32 {d16[1], d17[1], d18[1]}, [r0]
+; CHECK-NEXT:    mov pc, lr
 	%tmp0 = bitcast float* %A to i8*
 	%tmp1 = load <2 x float>, <2 x float>* %B
 	call void @llvm.arm.neon.vst3lane.p0i8.v2f32(i8* %tmp0, <2 x float> %tmp1, <2 x float> %tmp1, <2 x float> %tmp1, i32 1, i32 1)
@@ -230,9 +324,14 @@ define void @vst3lanef(float* %A, <2 x float>* %B) nounwind {
 }
 
 define void @vst3laneQi16(i16* %A, <8 x i16>* %B) nounwind {
-;CHECK-LABEL: vst3laneQi16:
+; CHECK-LABEL: vst3laneQi16:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r1]
+; CHECK-NEXT:    vorr q9, q8, q8
+; CHECK-NEXT:    vorr q10, q8, q8
+; CHECK-NEXT:    vst3.16 {d17[2], d19[2], d21[2]}, [r0]
+; CHECK-NEXT:    mov pc, lr
 ;Check the (default) alignment value.  VST3 does not support alignment.
-;CHECK: vst3.16 {d17[2], d19[2], d21[2]}, [r0]
 	%tmp0 = bitcast i16* %A to i8*
 	%tmp1 = load <8 x i16>, <8 x i16>* %B
 	call void @llvm.arm.neon.vst3lane.p0i8.v8i16(i8* %tmp0, <8 x i16> %tmp1, <8 x i16> %tmp1, <8 x i16> %tmp1, i32 6, i32 8)
@@ -240,8 +339,13 @@ define void @vst3laneQi16(i16* %A, <8 x i16>* %B) nounwind {
 }
 
 define void @vst3laneQi32(i32* %A, <4 x i32>* %B) nounwind {
-;CHECK-LABEL: vst3laneQi32:
-;CHECK: vst3.32
+; CHECK-LABEL: vst3laneQi32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r1]
+; CHECK-NEXT:    vorr q9, q8, q8
+; CHECK-NEXT:    vorr q10, q8, q8
+; CHECK-NEXT:    vst3.32 {d16[0], d18[0], d20[0]}, [r0]
+; CHECK-NEXT:    mov pc, lr
 	%tmp0 = bitcast i32* %A to i8*
 	%tmp1 = load <4 x i32>, <4 x i32>* %B
 	call void @llvm.arm.neon.vst3lane.p0i8.v4i32(i8* %tmp0, <4 x i32> %tmp1, <4 x i32> %tmp1, <4 x i32> %tmp1, i32 0, i32 1)
@@ -250,8 +354,15 @@ define void @vst3laneQi32(i32* %A, <4 x i32>* %B) nounwind {
 
 ;Check for a post-increment updating store.
 define void @vst3laneQi32_update(i32** %ptr, <4 x i32>* %B) nounwind {
-;CHECK-LABEL: vst3laneQi32_update:
-;CHECK: vst3.32 {d16[0], d18[0], d20[0]}, [r{{[0-9]+}}]!
+; CHECK-LABEL: vst3laneQi32_update:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r1]
+; CHECK-NEXT:    vorr q9, q8, q8
+; CHECK-NEXT:    ldr r2, [r0]
+; CHECK-NEXT:    vorr q10, q8, q8
+; CHECK-NEXT:    vst3.32 {d16[0], d18[0], d20[0]}, [r2]!
+; CHECK-NEXT:    str r2, [r0]
+; CHECK-NEXT:    mov pc, lr
 	%A = load i32*, i32** %ptr
 	%tmp0 = bitcast i32* %A to i8*
 	%tmp1 = load <4 x i32>, <4 x i32>* %B
@@ -262,8 +373,13 @@ define void @vst3laneQi32_update(i32** %ptr, <4 x i32>* %B) nounwind {
 }
 
 define void @vst3laneQf(float* %A, <4 x float>* %B) nounwind {
-;CHECK-LABEL: vst3laneQf:
-;CHECK: vst3.32
+; CHECK-LABEL: vst3laneQf:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r1]
+; CHECK-NEXT:    vorr q9, q8, q8
+; CHECK-NEXT:    vorr q10, q8, q8
+; CHECK-NEXT:    vst3.32 {d16[1], d18[1], d20[1]}, [r0]
+; CHECK-NEXT:    mov pc, lr
 	%tmp0 = bitcast float* %A to i8*
 	%tmp1 = load <4 x float>, <4 x float>* %B
 	call void @llvm.arm.neon.vst3lane.p0i8.v4f32(i8* %tmp0, <4 x float> %tmp1, <4 x float> %tmp1, <4 x float> %tmp1, i32 1, i32 1)
@@ -280,10 +396,16 @@ declare void @llvm.arm.neon.vst3lane.p0i8.v4i32(i8*, <4 x i32>, <4 x i32>, <4 x
 declare void @llvm.arm.neon.vst3lane.p0i8.v4f32(i8*, <4 x float>, <4 x float>, <4 x float>, i32, i32) nounwind
 
 
-define void @vst4lanei8(i8* %A, <8 x i8>* %B) nounwind {
-;CHECK-LABEL: vst4lanei8:
 ;Check the alignment value.  Max for this instruction is 32 bits:
-;CHECK: vst4.8 {d16[1], d17[1], d18[1], d19[1]}, [r0:32]
+define void @vst4lanei8(i8* %A, <8 x i8>* %B) nounwind {
+; CHECK-LABEL: vst4lanei8:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr d16, [r1]
+; CHECK-NEXT:    vorr d17, d16, d16
+; CHECK-NEXT:    vorr d18, d16, d16
+; CHECK-NEXT:    vorr d19, d16, d16
+; CHECK-NEXT:    vst4.8 {d16[1], d17[1], d18[1], d19[1]}, [r0:32]
+; CHECK-NEXT:    mov pc, lr
 	%tmp1 = load <8 x i8>, <8 x i8>* %B
 	call void @llvm.arm.neon.vst4lane.p0i8.v8i8(i8* %A, <8 x i8> %tmp1, <8 x i8> %tmp1, <8 x i8> %tmp1, <8 x i8> %tmp1, i32 1, i32 8)
 	ret void
@@ -291,8 +413,16 @@ define void @vst4lanei8(i8* %A, <8 x i8>* %B) nounwind {
 
 ;Check for a post-increment updating store.
 define void @vst4lanei8_update(i8** %ptr, <8 x i8>* %B) nounwind {
-;CHECK-LABEL: vst4lanei8_update:
-;CHECK: vst4.8 {d16[1], d17[1], d18[1], d19[1]}, [r{{[0-9]+}}:32]!
+; CHECK-LABEL: vst4lanei8_update:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr d16, [r1]
+; CHECK-NEXT:    vorr d17, d16, d16
+; CHECK-NEXT:    ldr r2, [r0]
+; CHECK-NEXT:    vorr d18, d16, d16
+; CHECK-NEXT:    vorr d19, d16, d16
+; CHECK-NEXT:    vst4.8 {d16[1], d17[1], d18[1], d19[1]}, [r2:32]!
+; CHECK-NEXT:    str r2, [r0]
+; CHECK-NEXT:    mov pc, lr
 	%A = load i8*, i8** %ptr
 	%tmp1 = load <8 x i8>, <8 x i8>* %B
 	call void @llvm.arm.neon.vst4lane.p0i8.v8i8(i8* %A, <8 x i8> %tmp1, <8 x i8> %tmp1, <8 x i8> %tmp1, <8 x i8> %tmp1, i32 1, i32 8)
@@ -302,18 +432,30 @@ define void @vst4lanei8_update(i8** %ptr, <8 x i8>* %B) nounwind {
 }
 
 define void @vst4lanei16(i16* %A, <4 x i16>* %B) nounwind {
-;CHECK-LABEL: vst4lanei16:
-;CHECK: vst4.16
+; CHECK-LABEL: vst4lanei16:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr d16, [r1]
+; CHECK-NEXT:    vorr d17, d16, d16
+; CHECK-NEXT:    vorr d18, d16, d16
+; CHECK-NEXT:    vorr d19, d16, d16
+; CHECK-NEXT:    vst4.16 {d16[1], d17[1], d18[1], d19[1]}, [r0]
+; CHECK-NEXT:    mov pc, lr
 	%tmp0 = bitcast i16* %A to i8*
 	%tmp1 = load <4 x i16>, <4 x i16>* %B
 	call void @llvm.arm.neon.vst4lane.p0i8.v4i16(i8* %tmp0, <4 x i16> %tmp1, <4 x i16> %tmp1, <4 x i16> %tmp1, <4 x i16> %tmp1, i32 1, i32 1)
 	ret void
 }
 
-define void @vst4lanei32(i32* %A, <2 x i32>* %B) nounwind {
-;CHECK-LABEL: vst4lanei32:
 ;Check the alignment value.  Max for this instruction is 128 bits:
-;CHECK: vst4.32 {d16[1], d17[1], d18[1], d19[1]}, [r0:128]
+define void @vst4lanei32(i32* %A, <2 x i32>* %B) nounwind {
+; CHECK-LABEL: vst4lanei32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr d16, [r1]
+; CHECK-NEXT:    vorr d17, d16, d16
+; CHECK-NEXT:    vorr d18, d16, d16
+; CHECK-NEXT:    vorr d19, d16, d16
+; CHECK-NEXT:    vst4.32 {d16[1], d17[1], d18[1], d19[1]}, [r0:128]
+; CHECK-NEXT:    mov pc, lr
 	%tmp0 = bitcast i32* %A to i8*
 	%tmp1 = load <2 x i32>, <2 x i32>* %B
 	call void @llvm.arm.neon.vst4lane.p0i8.v2i32(i8* %tmp0, <2 x i32> %tmp1, <2 x i32> %tmp1, <2 x i32> %tmp1, <2 x i32> %tmp1, i32 1, i32 16)
@@ -321,28 +463,46 @@ define void @vst4lanei32(i32* %A, <2 x i32>* %B) nounwind {
 }
 
 define void @vst4lanef(float* %A, <2 x float>* %B) nounwind {
-;CHECK-LABEL: vst4lanef:
-;CHECK: vst4.32
+; CHECK-LABEL: vst4lanef:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr d16, [r1]
+; CHECK-NEXT:    vorr d17, d16, d16
+; CHECK-NEXT:    vorr d18, d16, d16
+; CHECK-NEXT:    vorr d19, d16, d16
+; CHECK-NEXT:    vst4.32 {d16[1], d17[1], d18[1], d19[1]}, [r0]
+; CHECK-NEXT:    mov pc, lr
 	%tmp0 = bitcast float* %A to i8*
 	%tmp1 = load <2 x float>, <2 x float>* %B
 	call void @llvm.arm.neon.vst4lane.p0i8.v2f32(i8* %tmp0, <2 x float> %tmp1, <2 x float> %tmp1, <2 x float> %tmp1, <2 x float> %tmp1, i32 1, i32 1)
 	ret void
 }
 
-define void @vst4laneQi16(i16* %A, <8 x i16>* %B) nounwind {
-;CHECK-LABEL: vst4laneQi16:
 ;Check the alignment value.  Max for this instruction is 64 bits:
-;CHECK: vst4.16 {d17[3], d19[3], d21[3], d23[3]}, [r0:64]
+define void @vst4laneQi16(i16* %A, <8 x i16>* %B) nounwind {
+; CHECK-LABEL: vst4laneQi16:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r1]
+; CHECK-NEXT:    vorr q9, q8, q8
+; CHECK-NEXT:    vorr q10, q8, q8
+; CHECK-NEXT:    vorr q11, q8, q8
+; CHECK-NEXT:    vst4.16 {d17[3], d19[3], d21[3], d23[3]}, [r0:64]
+; CHECK-NEXT:    mov pc, lr
 	%tmp0 = bitcast i16* %A to i8*
 	%tmp1 = load <8 x i16>, <8 x i16>* %B
 	call void @llvm.arm.neon.vst4lane.p0i8.v8i16(i8* %tmp0, <8 x i16> %tmp1, <8 x i16> %tmp1, <8 x i16> %tmp1, <8 x i16> %tmp1, i32 7, i32 16)
 	ret void
 }
 
-define void @vst4laneQi32(i32* %A, <4 x i32>* %B) nounwind {
-;CHECK-LABEL: vst4laneQi32:
 ;Check the (default) alignment.
-;CHECK: vst4.32 {d17[0], d19[0], d21[0], d23[0]}, [r0]
+define void @vst4laneQi32(i32* %A, <4 x i32>* %B) nounwind {
+; CHECK-LABEL: vst4laneQi32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r1]
+; CHECK-NEXT:    vorr q9, q8, q8
+; CHECK-NEXT:    vorr q10, q8, q8
+; CHECK-NEXT:    vorr q11, q8, q8
+; CHECK-NEXT:    vst4.32 {d17[0], d19[0], d21[0], d23[0]}, [r0]
+; CHECK-NEXT:    mov pc, lr
 	%tmp0 = bitcast i32* %A to i8*
 	%tmp1 = load <4 x i32>, <4 x i32>* %B
 	call void @llvm.arm.neon.vst4lane.p0i8.v4i32(i8* %tmp0, <4 x i32> %tmp1, <4 x i32> %tmp1, <4 x i32> %tmp1, <4 x i32> %tmp1, i32 2, i32 1)
@@ -350,8 +510,14 @@ define void @vst4laneQi32(i32* %A, <4 x i32>* %B) nounwind {
 }
 
 define void @vst4laneQf(float* %A, <4 x float>* %B) nounwind {
-;CHECK-LABEL: vst4laneQf:
-;CHECK: vst4.32
+; CHECK-LABEL: vst4laneQf:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r1]
+; CHECK-NEXT:    vorr q9, q8, q8
+; CHECK-NEXT:    vorr q10, q8, q8
+; CHECK-NEXT:    vorr q11, q8, q8
+; CHECK-NEXT:    vst4.32 {d16[1], d18[1], d20[1], d22[1]}, [r0]
+; CHECK-NEXT:    mov pc, lr
 	%tmp0 = bitcast float* %A to i8*
 	%tmp1 = load <4 x float>, <4 x float>* %B
 	call void @llvm.arm.neon.vst4lane.p0i8.v4f32(i8* %tmp0, <4 x float> %tmp1, <4 x float> %tmp1, <4 x float> %tmp1, <4 x float> %tmp1, i32 1, i32 1)
@@ -360,7 +526,28 @@ define void @vst4laneQf(float* %A, <4 x float>* %B) nounwind {
 
 ; Make sure this doesn't crash; PR10258
 define <8 x i16> @variable_insertelement(<8 x i16> %a, i16 %b, i32 %c) nounwind readnone {
-;CHECK-LABEL: variable_insertelement:
+; CHECK-LABEL: variable_insertelement:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    push {r11, lr}
+; CHECK-NEXT:    mov r11, sp
+; CHECK-NEXT:    sub sp, sp, #24
+; CHECK-NEXT:    bic sp, sp, #15
+; CHECK-NEXT:    ldr lr, [r11, #12]
+; CHECK-NEXT:    vmov d17, r2, r3
+; CHECK-NEXT:    vmov d16, r0, r1
+; CHECK-NEXT:    mov r1, sp
+; CHECK-NEXT:    and r0, lr, #7
+; CHECK-NEXT:    mov r2, r1
+; CHECK-NEXT:    ldrh r12, [r11, #8]
+; CHECK-NEXT:    lsl r0, r0, #1
+; CHECK-NEXT:    vst1.64 {d16, d17}, [r2:128], r0
+; CHECK-NEXT:    strh r12, [r2]
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r1:128]
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    vmov r2, r3, d17
+; CHECK-NEXT:    mov sp, r11
+; CHECK-NEXT:    pop {r11, lr}
+; CHECK-NEXT:    mov pc, lr
     %r = insertelement <8 x i16> %a, i16 %b, i32 %c
     ret <8 x i16> %r
 }
diff --git a/llvm/test/CodeGen/ARM/warn-stack.ll b/llvm/test/CodeGen/ARM/warn-stack.ll
index f07cb64cd..6756a4b9e 100644
--- a/llvm/test/CodeGen/ARM/warn-stack.ll
+++ b/llvm/test/CodeGen/ARM/warn-stack.ll
@@ -4,7 +4,7 @@
 ; <rdar://13987214>
 
 ; CHECK-NOT: nowarn
-define void @nowarn() nounwind ssp "no-frame-pointer-elim"="true" {
+define void @nowarn() nounwind ssp "frame-pointer"="all" {
 entry:
   %buffer = alloca [12 x i8], align 1
   %arraydecay = getelementptr inbounds [12 x i8], [12 x i8]* %buffer, i64 0, i64 0
@@ -13,7 +13,7 @@ entry:
 }
 
 ; CHECK: warning: stack size limit exceeded (92) in warn
-define void @warn() nounwind ssp "no-frame-pointer-elim"="true" {
+define void @warn() nounwind ssp "frame-pointer"="all" {
 entry:
   %buffer = alloca [80 x i8], align 1
   %arraydecay = getelementptr inbounds [80 x i8], [80 x i8]* %buffer, i64 0, i64 0
diff --git a/llvm/test/CodeGen/AVR/rot.ll b/llvm/test/CodeGen/AVR/rot.ll
index a7b77d97b..49981aabe 100644
--- a/llvm/test/CodeGen/AVR/rot.ll
+++ b/llvm/test/CodeGen/AVR/rot.ll
@@ -10,7 +10,8 @@ define i8 @rol8(i8 %val, i8 %amt) {
   ; CHECK-NEXT: breq LBB0_2
 
 ; CHECK-NEXT: LBB0_1:
-  ; CHECK-NEXT: rol r24
+  ; CHECK-NEXT: lsl r24
+  ; CHECK-NEXT: adc r24, r1
   ; CHECK-NEXT: subi r22, 1
   ; CHECK-NEXT: brne LBB0_1
 
@@ -36,7 +37,10 @@ define i8 @ror8(i8 %val, i8 %amt) {
   ; CHECK-NEXT: breq LBB1_2
 
 ; CHECK-NEXT: LBB1_1:
-  ; CHECK-NEXT: ror r24
+  ; CHECK-NEXT: lsr r24
+  ; CHECK-NEXT: ldi r0, 0
+  ; CHECK-NEXT: ror r0
+  ; CHECK-NEXT: or r24, r0
   ; CHECK-NEXT: subi r22, 1
   ; CHECK-NEXT: brne LBB1_1
 
diff --git a/llvm/test/CodeGen/BPF/32-bit-subreg-alu.ll b/llvm/test/CodeGen/BPF/32-bit-subreg-alu.ll
index 2a4984945..2e4e92e01 100644
--- a/llvm/test/CodeGen/BPF/32-bit-subreg-alu.ll
+++ b/llvm/test/CodeGen/BPF/32-bit-subreg-alu.ll
@@ -1,4 +1,5 @@
 ; RUN: llc -O2 -march=bpfel -mattr=+alu32 < %s | FileCheck %s
+; RUN: llc -O2 -march=bpfel -mcpu=v3 < %s | FileCheck %s
 ;
 ; int mov(int a)
 ; {
diff --git a/llvm/test/CodeGen/BPF/32-bit-subreg-cond-select.ll b/llvm/test/CodeGen/BPF/32-bit-subreg-cond-select.ll
index a48141cf0..160be56c3 100644
--- a/llvm/test/CodeGen/BPF/32-bit-subreg-cond-select.ll
+++ b/llvm/test/CodeGen/BPF/32-bit-subreg-cond-select.ll
@@ -55,6 +55,9 @@ entry:
   %c.d = select i1 %cmp, i32 %c, i32 %d
   ret i32 %c.d
 }
+; CHECK-LABEL: select_cc_32
+; CHECK: r{{[0-9]+}} <<= 32
+; CHECK-NEXT: r{{[0-9]+}} >>= 32
 
 ; Function Attrs: norecurse nounwind readnone
 define dso_local i64 @select_cc_32_64(i32 %a, i32 %b, i64 %c, i64 %d) local_unnamed_addr #0 {
@@ -63,6 +66,9 @@ entry:
   %c.d = select i1 %cmp, i64 %c, i64 %d
   ret i64 %c.d
 }
+; CHECK-LABEL: select_cc_32_64
+; CHECK: r{{[0-9]+}} <<= 32
+; CHECK-NEXT: r{{[0-9]+}} >>= 32
 
 ; Function Attrs: norecurse nounwind readnone
 define dso_local i32 @select_cc_64_32(i64 %a, i64 %b, i32 %c, i32 %d) local_unnamed_addr #0 {
@@ -71,6 +77,8 @@ entry:
   %c.d = select i1 %cmp, i32 %c, i32 %d
   ret i32 %c.d
 }
+; CHECK-LABEL: select_cc_64_32
+; CHECK-NOT: r{{[0-9]+}} <<= 32
 
 ; Function Attrs: norecurse nounwind readnone
 define dso_local i32 @selecti_cc_32(i32 %a, i32 %c, i32 %d) local_unnamed_addr #0 {
@@ -79,6 +87,9 @@ entry:
   %c.d = select i1 %cmp, i32 %c, i32 %d
   ret i32 %c.d
 }
+; CHECK-LABEL: selecti_cc_32
+; CHECK: r{{[0-9]+}} <<= 32
+; CHECK-NEXT: r{{[0-9]+}} >>= 32
 
 ; Function Attrs: norecurse nounwind readnone
 define dso_local i64 @selecti_cc_32_64(i32 %a, i64 %c, i64 %d) local_unnamed_addr #0 {
@@ -87,6 +98,9 @@ entry:
   %c.d = select i1 %cmp, i64 %c, i64 %d
   ret i64 %c.d
 }
+; CHECK-LABEL: selecti_cc_32_64
+; CHECK: r{{[0-9]+}} <<= 32
+; CHECK-NEXT: r{{[0-9]+}} >>= 32
 
 ; Function Attrs: norecurse nounwind readnone
 define dso_local i32 @selecti_cc_64_32(i64 %a, i32 %c, i32 %d) local_unnamed_addr #0 {
@@ -95,6 +109,5 @@ entry:
   %c.d = select i1 %cmp, i32 %c, i32 %d
   ret i32 %c.d
 }
-; There shouldn't be any type promotion, all of them are expected to be
-; eliminated by peephole optimization.
+; CHECK-LABEL: selecti_cc_64_32
 ; CHECK-NOT: r{{[0-9]+}} <<= 32
diff --git a/llvm/test/CodeGen/BPF/32-bit-subreg-peephole-phi-1.ll b/llvm/test/CodeGen/BPF/32-bit-subreg-peephole-phi-1.ll
new file mode 100644
index 000000000..5a72f5959
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/32-bit-subreg-peephole-phi-1.ll
@@ -0,0 +1,34 @@
+; RUN: llc -O2 -march=bpfel -mcpu=v2 -mattr=+alu32 < %s | FileCheck %s
+;
+; For the below test case, 'b' in 'ret == b' needs SLL/SLR.
+; 'ret' in 'ret == b' does not need SLL/SLR as all 'ret' values
+; are assigned through 'w<reg> = <value>' alu32 operations.
+;
+; extern int helper(int);
+; int test(int a, int b, int c, int d) {
+;   int ret;
+;   if (a < b)
+;     ret = (c < d) ? -1 : 0;
+;   else
+;     ret = (c < a) ? 1 : 2;
+;   return helper(ret == b);
+; }
+
+define dso_local i32 @test(i32 %a, i32 %b, i32 %c, i32 %d) local_unnamed_addr {
+entry:
+  %cmp = icmp slt i32 %a, %b
+  %cmp1 = icmp slt i32 %c, %d
+  %cond = sext i1 %cmp1 to i32
+  %cmp2 = icmp slt i32 %c, %a
+  %cond3 = select i1 %cmp2, i32 1, i32 2
+  %ret.0 = select i1 %cmp, i32 %cond, i32 %cond3
+  %cmp4 = icmp eq i32 %ret.0, %b
+  %conv = zext i1 %cmp4 to i32
+  %call = tail call i32 @helper(i32 %conv)
+  ret i32 %call
+}
+; CHECK: r{{[0-9]+}} >>= 32
+; CHECK-NOT: r{{[0-9]+}} >>= 32
+; CHECK: if r{{[0-9]+}} == r{{[0-9]+}} goto
+
+declare dso_local i32 @helper(i32) local_unnamed_addr
diff --git a/llvm/test/CodeGen/BPF/32-bit-subreg-peephole-phi-2.ll b/llvm/test/CodeGen/BPF/32-bit-subreg-peephole-phi-2.ll
new file mode 100644
index 000000000..46a1b231c
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/32-bit-subreg-peephole-phi-2.ll
@@ -0,0 +1,34 @@
+; RUN: llc -O2 -march=bpfel -mcpu=v2 -mattr=+alu32 < %s | FileCheck %s
+;
+; For the below test case, both 'ret' and 'b' at 'ret == b'
+; need SLL/SLR. For 'ret', 'ret = a' may receive the value
+; from argument with high 32-bit invalid data.
+;
+; extern int helper(int);
+; int test(int a, int b, int c, int d) {
+;   int ret;
+;   if (a < b)
+;     ret = (c < d) ? a : 0;
+;   else
+;     ret = (c < a) ? 1 : 2;
+;   return helper(ret == b);
+; }
+
+define dso_local i32 @test(i32 %a, i32 %b, i32 %c, i32 %d) local_unnamed_addr {
+entry:
+  %cmp = icmp slt i32 %a, %b
+  %cmp1 = icmp slt i32 %c, %d
+  %cond = select i1 %cmp1, i32 %a, i32 0
+  %cmp2 = icmp slt i32 %c, %a
+  %cond3 = select i1 %cmp2, i32 1, i32 2
+  %ret.0 = select i1 %cmp, i32 %cond, i32 %cond3
+  %cmp4 = icmp eq i32 %ret.0, %b
+  %conv = zext i1 %cmp4 to i32
+  %call = tail call i32 @helper(i32 %conv)
+  ret i32 %call
+}
+; CHECK: r{{[0-9]+}} >>= 32
+; CHECK: r{{[0-9]+}} >>= 32
+; CHECK: if r{{[0-9]+}} == r{{[0-9]+}} goto
+
+declare dso_local i32 @helper(i32) local_unnamed_addr
diff --git a/llvm/test/CodeGen/BPF/32-bit-subreg-peephole-phi-3.ll b/llvm/test/CodeGen/BPF/32-bit-subreg-peephole-phi-3.ll
new file mode 100644
index 000000000..d46214032
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/32-bit-subreg-peephole-phi-3.ll
@@ -0,0 +1,52 @@
+; RUN: llc -O2 -march=bpfel -mcpu=v2 -mattr=+alu32 < %s | FileCheck %s
+;
+; For the below example, two phi node in the loop may depend on
+; each other. So implementation must handle recursion properly.
+;
+; int test(unsigned long a, unsigned long b, unsigned long c) {
+;   int val = 0;
+;
+;   #pragma clang loop unroll(disable)
+;   for (long i = 0; i < 100; i++) {
+;     if (a > b)
+;       val = 1;
+;     a += b;
+;     if (b > c)
+;       val = 1;
+;     b += c;
+;   }
+;
+;   return val == 0 ? 1 : 0;
+; }
+
+
+define dso_local i32 @test(i64 %a, i64 %b, i64 %c) local_unnamed_addr {
+entry:
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body
+  %cmp6 = icmp eq i32 %val.2, 0
+  %cond = zext i1 %cmp6 to i32
+  ret i32 %cond
+
+for.body:                                         ; preds = %for.body, %entry
+  %i.018 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
+  %val.017 = phi i32 [ 0, %entry ], [ %val.2, %for.body ]
+  %a.addr.016 = phi i64 [ %a, %entry ], [ %add, %for.body ]
+  %b.addr.015 = phi i64 [ %b, %entry ], [ %add5, %for.body ]
+  %cmp1 = icmp ugt i64 %a.addr.016, %b.addr.015
+  %add = add i64 %a.addr.016, %b.addr.015
+  %cmp2 = icmp ugt i64 %b.addr.015, %c
+  %0 = or i1 %cmp2, %cmp1
+  %val.2 = select i1 %0, i32 1, i32 %val.017
+  %add5 = add i64 %b.addr.015, %c
+  %inc = add nuw nsw i64 %i.018, 1
+  %exitcond = icmp eq i64 %inc, 100
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body, !llvm.loop !2
+}
+; CHECK: [[VAL:r[0-9]+]] <<= 32
+; CHECK: [[VAL]] >>= 32
+; CHECK: if [[VAL]] == 0 goto
+
+!2 = distinct !{!2, !3}
+!3 = !{!"llvm.loop.unroll.disable"}
diff --git a/llvm/test/CodeGen/BPF/32-bit-subreg-peephole.ll b/llvm/test/CodeGen/BPF/32-bit-subreg-peephole.ll
index cf68ec56c..63a7c25ed 100644
--- a/llvm/test/CodeGen/BPF/32-bit-subreg-peephole.ll
+++ b/llvm/test/CodeGen/BPF/32-bit-subreg-peephole.ll
@@ -47,8 +47,8 @@ define dso_local i64 @select_u(i32 %a, i32 %b, i64 %c, i64 %d) local_unnamed_add
 entry:
   %cmp = icmp ugt i32 %a, %b
   %c.d = select i1 %cmp, i64 %c, i64 %d
-; CHECK-NOT: r{{[0-9]+}} <<= 32
-; CHECK-NOT: r{{[0-9]+}} >>= 32
+; CHECK: r{{[0-9]+}} <<= 32
+; CHECK-NEXT: r{{[0-9]+}} >>= 32
 ; CHECK: if r{{[0-9]+}} {{<|>}} r{{[0-9]+}} goto
   ret i64 %c.d
 }
@@ -58,8 +58,8 @@ define dso_local i64 @select_u_2(i32 %a, i64 %b, i64 %c, i64 %d) local_unnamed_a
 ; CHECK-LABEL: select_u_2:
 entry:
   %conv = zext i32 %a to i64
-; CHECK-NOT: r{{[0-9]+}} <<= 32
-; CHECK-NOT: r{{[0-9]+}} >>= 32
+; CHECK: r{{[0-9]+}} <<= 32
+; CHECK-NEXT: r{{[0-9]+}} >>= 32
   %cmp = icmp ugt i64 %conv, %b
   %c.d = select i1 %cmp, i64 %c, i64 %d
   ret i64 %c.d
@@ -100,8 +100,23 @@ define dso_local i32* @inc_p(i32* readnone %p, i32 %a) local_unnamed_addr #0 {
 ; CHECK-LABEL: inc_p:
 entry:
   %idx.ext = zext i32 %a to i64
-; CHECK-NOT: r{{[0-9]+}} <<= 32
-; CHECK-NOT: r{{[0-9]+}} >>= 32
+; CHECK: r{{[0-9]+}} <<= 32
+; CHECK-NEXT: r{{[0-9]+}} >>= 32
   %add.ptr = getelementptr inbounds i32, i32* %p, i64 %idx.ext
   ret i32* %add.ptr
 }
+
+define dso_local i32 @test() local_unnamed_addr {
+; CHECK-LABEL: test:
+entry:
+  %call = tail call i32 bitcast (i32 (...)* @helper to i32 ()*)()
+  %cmp = icmp sgt i32 %call, 6
+; The shifts can't be optimized out because %call comes from function call
+; return i32 so the high bits might be invalid.
+; CHECK: r{{[0-9]+}} <<= 32
+; CHECK-NEXT: r{{[0-9]+}} s>>= 32
+  %cond = zext i1 %cmp to i32
+; CHECK: if r{{[0-9]+}} s{{<|>}} {{[0-9]+}} goto
+  ret i32 %cond
+}
+declare dso_local i32 @helper(...) local_unnamed_addr
diff --git a/llvm/test/CodeGen/BPF/BTF/array-size-0.ll b/llvm/test/CodeGen/BPF/BTF/array-size-0.ll
index d397c2c3a..65a084187 100644
--- a/llvm/test/CodeGen/BPF/BTF/array-size-0.ll
+++ b/llvm/test/CodeGen/BPF/BTF/array-size-0.ll
@@ -32,7 +32,7 @@
 ; CHECK-NEXT:        .long   0
 ; CHECK-NEXT:        .long   1
 ; CHECK-NEXT:        .long   3
-; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   10
 ; CHECK-NEXT:        .long   3                       # BTF_KIND_INT(id = 3)
 ; CHECK-NEXT:        .long   16777216                # 0x1000000
 ; CHECK-NEXT:        .long   4
diff --git a/llvm/test/CodeGen/BPF/BTF/binary-format.ll b/llvm/test/CodeGen/BPF/BTF/binary-format.ll
index bc561560c..ec4363cce 100644
--- a/llvm/test/CodeGen/BPF/BTF/binary-format.ll
+++ b/llvm/test/CodeGen/BPF/BTF/binary-format.ll
@@ -18,35 +18,33 @@ entry:
 ; CHECK-EL: 0x00000010 30000000 33000000 01000000 00000001
 ; CHECK-EL: 0x00000020 04000000 20000001 00000000 0100000d
 ; CHECK-EL: 0x00000030 01000000 05000000 01000000 07000000
-; CHECK-EL: 0x00000040 0000000c 02000000 00696e74 00610066
+; CHECK-EL: 0x00000040 0100000c 02000000 00696e74 00610066
 ; CHECK-EB: 0x00000000 eb9f0100 00000018 00000000 00000030
 ; CHECK-EB: 0x00000010 00000030 00000033 00000001 01000000
 ; CHECK-EB: 0x00000020 00000004 01000020 00000000 0d000001
 ; CHECK-EB: 0x00000030 00000001 00000005 00000001 00000007
-; CHECK-EB: 0x00000040 0c000000 00000002 00696e74 00610066
+; CHECK-EB: 0x00000040 0c000001 00000002 00696e74 00610066
 ; CHECK:    0x00000050 002e7465 7874002f 746d702f 742e6300
 ; CHECK:    0x00000060 696e7420 6628696e 74206129 207b2072
 ; CHECK:    0x00000070 65747572 6e20613b 207d00
 ; CHECK:    '.BTF.ext'
-; CHECK-EL: 0x00000000 9feb0100 28000000 00000000 14000000
+; CHECK-EL: 0x00000000 9feb0100 20000000 00000000 14000000
 ; CHECK-EL: 0x00000010 14000000 2c000000 40000000 00000000
-; CHECK-EL: 0x00000020 40000000 00000000 08000000 09000000
-; CHECK-EL: 0x00000030 01000000 00000000 03000000 10000000
-; CHECK-EL: 0x00000040 09000000 02000000 00000000 0f000000
-; CHECK-EL: 0x00000050 18000000 00040000 08000000 0f000000
-; CHECK-EL: 0x00000060 18000000 10040000
-; CHECK-EB: 0x00000000 eb9f0100 00000028 00000000 00000014
+; CHECK-EL: 0x00000020 08000000 09000000 01000000 00000000
+; CHECK-EL: 0x00000030 03000000 10000000 09000000 02000000
+; CHECK-EL: 0x00000040 00000000 0f000000 18000000 00040000
+; CHECK-EL: 0x00000050 08000000 0f000000 18000000 10040000
+; CHECK-EB: 0x00000000 eb9f0100 00000020 00000000 00000014
 ; CHECK-EB: 0x00000010 00000014 0000002c 00000040 00000000
-; CHECK-EB: 0x00000020 00000040 00000000 00000008 00000009
-; CHECK-EB: 0x00000030 00000001 00000000 00000003 00000010
-; CHECK-EB: 0x00000040 00000009 00000002 00000000 0000000f
-; CHECK-EB: 0x00000050 00000018 00000400 00000008 0000000f
-; CHECK-EB: 0x00000060 00000018 00000410
+; CHECK-EB: 0x00000020 00000008 00000009 00000001 00000000
+; CHECK-EB: 0x00000030 00000003 00000010 00000009 00000002
+; CHECK-EB: 0x00000040 00000000 0000000f 00000018 00000400
+; CHECK-EB: 0x00000050 00000008 0000000f 00000018 00000410
 
 ; Function Attrs: nounwind readnone speculatable
 declare void @llvm.dbg.value(metadata, metadata, metadata) #1
 
-attributes #0 = { nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone speculatable }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/CodeGen/BPF/BTF/char-no-debuginfo.ll b/llvm/test/CodeGen/BPF/BTF/char-no-debuginfo.ll
index 38d5e09a7..ee565ab23 100644
--- a/llvm/test/CodeGen/BPF/BTF/char-no-debuginfo.ll
+++ b/llvm/test/CodeGen/BPF/BTF/char-no-debuginfo.ll
@@ -18,7 +18,7 @@ define dso_local i32 @test() local_unnamed_addr #0 {
 ; CHECK-NOT:         .section        .BTF
 ; CHECK-NOT:         .section        .BTF.ext
 
-attributes #0 = { norecurse nounwind readonly "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { norecurse nounwind readonly "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.module.flags = !{!0}
 !llvm.ident = !{!1}
diff --git a/llvm/test/CodeGen/BPF/BTF/extern-builtin.ll b/llvm/test/CodeGen/BPF/BTF/extern-builtin.ll
new file mode 100644
index 000000000..e5527d763
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/BTF/extern-builtin.ll
@@ -0,0 +1,89 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; Source code:
+;   unsigned long long load_byte(void *skb,
+;       unsigned long long off) asm("llvm.bpf.load.byte");
+;   unsigned long long test(void *skb) {
+;     return load_byte(skb, 10);
+;   }
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm test.c
+
+; Function Attrs: nounwind readonly
+define dso_local i64 @test(i8* readonly %skb) local_unnamed_addr #0 !dbg !13 {
+entry:
+  call void @llvm.dbg.value(metadata i8* %skb, metadata !17, metadata !DIExpression()), !dbg !18
+  %call = tail call i64 @llvm.bpf.load.byte(i8* %skb, i64 10), !dbg !19
+  ret i64 %call, !dbg !20
+}
+
+; CHECK:             .section        .BTF,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   60
+; CHECK-NEXT:        .long   60
+; CHECK-NEXT:        .long   78
+; CHECK-NEXT:        .long   0                       # BTF_KIND_PTR(id = 1)
+; CHECK-NEXT:        .long   33554432                # 0x2000000
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   0                       # BTF_KIND_FUNC_PROTO(id = 2)
+; CHECK-NEXT:        .long   218103809               # 0xd000001
+; CHECK-NEXT:        .long   3
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   5                       # BTF_KIND_INT(id = 3)
+; CHECK-NEXT:        .long   16777216                # 0x1000000
+; CHECK-NEXT:        .long   8
+; CHECK-NEXT:        .long   64                      # 0x40
+; CHECK-NEXT:        .long   28                      # BTF_KIND_FUNC(id = 4)
+; CHECK-NEXT:        .long   201326593               # 0xc000001
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .byte   0                       # string offset=0
+; CHECK-NEXT:        .ascii  "skb"                   # string offset=1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "long long unsigned int" # string offset=5
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "test"                  # string offset=28
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  ".text"                 # string offset=33
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "/tmp/home/yhs/work/tests/extern/test.c" # string offset=39
+; CHECK-NEXT:        .byte   0
+
+; Function Attrs: nounwind readonly
+declare !dbg !4 i64 @llvm.bpf.load.byte(i8*, i64) #1
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.value(metadata, metadata, metadata) #2
+
+attributes #0 = { nounwind readonly "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readonly }
+attributes #2 = { nounwind readnone speculatable willreturn }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!9, !10, !11}
+!llvm.ident = !{!12}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 10.0.0 (https://github.com/llvm/llvm-project.git 907019d835895443b198afcd992c42c9d3478fdf)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !3, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "test.c", directory: "/tmp/home/yhs/work/tests/extern")
+!2 = !{}
+!3 = !{!4}
+!4 = !DISubprogram(name: "load_byte", linkageName: "llvm.bpf.load.byte", scope: !1, file: !1, line: 1, type: !5, flags: DIFlagPrototyped, spFlags: DISPFlagOptimized, retainedNodes: !2)
+!5 = !DISubroutineType(types: !6)
+!6 = !{!7, !8, !7}
+!7 = !DIBasicType(name: "long long unsigned int", size: 64, encoding: DW_ATE_unsigned)
+!8 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: null, size: 64)
+!9 = !{i32 7, !"Dwarf Version", i32 4}
+!10 = !{i32 2, !"Debug Info Version", i32 3}
+!11 = !{i32 1, !"wchar_size", i32 4}
+!12 = !{!"clang version 10.0.0 (https://github.com/llvm/llvm-project.git 907019d835895443b198afcd992c42c9d3478fdf)"}
+!13 = distinct !DISubprogram(name: "test", scope: !1, file: !1, line: 3, type: !14, scopeLine: 3, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !16)
+!14 = !DISubroutineType(types: !15)
+!15 = !{!7, !8}
+!16 = !{!17}
+!17 = !DILocalVariable(name: "skb", arg: 1, scope: !13, file: !1, line: 3, type: !8)
+!18 = !DILocation(line: 0, scope: !13)
+!19 = !DILocation(line: 4, column: 10, scope: !13)
+!20 = !DILocation(line: 4, column: 3, scope: !13)
diff --git a/llvm/test/CodeGen/BPF/BTF/extern-func-arg.ll b/llvm/test/CodeGen/BPF/BTF/extern-func-arg.ll
new file mode 100644
index 000000000..cda248359
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/BTF/extern-func-arg.ll
@@ -0,0 +1,79 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+;
+; Source code:
+;   extern int global_func(char arg);
+;   int test() { return global_func(0); }
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm test.c
+
+; Function Attrs: nounwind
+define dso_local i32 @test() local_unnamed_addr #0 !dbg !13 {
+entry:
+  %call = tail call i32 @global_func(i8 signext 0) #2, !dbg !16
+  ret i32 %call, !dbg !17
+}
+
+; CHECK:             .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   88
+; CHECK-NEXT:        .long   88
+; CHECK-NEXT:        .long   72
+; CHECK-NEXT:        .long   0                       # BTF_KIND_FUNC_PROTO(id = 1)
+; CHECK-NEXT:        .long   218103808               # 0xd000000
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   1                       # BTF_KIND_INT(id = 2)
+; CHECK-NEXT:        .long   16777216                # 0x1000000
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   16777248                # 0x1000020
+; CHECK-NEXT:        .long   5                       # BTF_KIND_FUNC(id = 3)
+; CHECK-NEXT:        .long   201326593               # 0xc000001
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   0                       # BTF_KIND_FUNC_PROTO(id = 4)
+; CHECK-NEXT:        .long   218103809               # 0xd000001
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   5
+; CHECK-NEXT:        .long   55                      # BTF_KIND_INT(id = 5)
+; CHECK-NEXT:        .long   16777216                # 0x1000000
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   16777224                # 0x1000008
+; CHECK-NEXT:        .long   60                      # BTF_KIND_FUNC(id = 6)
+; CHECK-NEXT:        .long   201326594               # 0xc000002
+; CHECK-NEXT:        .long   4
+; CHECK:             .ascii  "int"                   # string offset=1
+; CHECK:             .ascii  "test"                  # string offset=5
+; CHECK:             .ascii  "char"                  # string offset=55
+; CHECK:             .ascii  "global_func"           # string offset=60
+
+declare !dbg !4 dso_local i32 @global_func(i8 signext) local_unnamed_addr #1
+
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { nounwind }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!9, !10, !11}
+!llvm.ident = !{!12}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 10.0.0 (https://github.com/llvm/llvm-project.git 987c5665e81822b32c895fd0c97a9a084b0d3106)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !3, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "test.c", directory: "/tmp/home/yhs/work/tests/extern")
+!2 = !{}
+!3 = !{!4}
+!4 = !DISubprogram(name: "global_func", scope: !1, file: !1, line: 1, type: !5, flags: DIFlagPrototyped, spFlags: DISPFlagOptimized, retainedNodes: !2)
+!5 = !DISubroutineType(types: !6)
+!6 = !{!7, !8}
+!7 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!8 = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_signed_char)
+!9 = !{i32 7, !"Dwarf Version", i32 4}
+!10 = !{i32 2, !"Debug Info Version", i32 3}
+!11 = !{i32 1, !"wchar_size", i32 4}
+!12 = !{!"clang version 10.0.0 (https://github.com/llvm/llvm-project.git 987c5665e81822b32c895fd0c97a9a084b0d3106)"}
+!13 = distinct !DISubprogram(name: "test", scope: !1, file: !1, line: 2, type: !14, scopeLine: 2, flags: DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2)
+!14 = !DISubroutineType(types: !15)
+!15 = !{!7}
+!16 = !DILocation(line: 2, column: 21, scope: !13)
+!17 = !DILocation(line: 2, column: 14, scope: !13)
diff --git a/llvm/test/CodeGen/BPF/BTF/extern-global-var.ll b/llvm/test/CodeGen/BPF/BTF/extern-global-var.ll
index 235a0ed3f..2f5a50402 100644
--- a/llvm/test/CodeGen/BPF/BTF/extern-global-var.ll
+++ b/llvm/test/CodeGen/BPF/BTF/extern-global-var.ll
@@ -33,7 +33,7 @@ define dso_local i32 @foo() local_unnamed_addr #0 !dbg !7 {
 ; CHECK-NEXT:        .long   4
 ; CHECK-NEXT:        .long   16777248                # 0x1000020
 ; CHECK-NEXT:        .long   5                       # BTF_KIND_FUNC(id = 3)
-; CHECK-NEXT:        .long   201326592               # 0xc000000
+; CHECK-NEXT:        .long   201326593               # 0xc000001
 ; CHECK-NEXT:        .long   1
 ; CHECK-NEXT:        .byte   0                       # string offset=0
 ; CHECK-NEXT:        .ascii  "int"                   # string offset=1
@@ -45,7 +45,7 @@ define dso_local i32 @foo() local_unnamed_addr #0 !dbg !7 {
 ; CHECK-NEXT:        .ascii  "/home/yhs/work/tests/llvm/bug/test.c" # string offset=15
 ; CHECK-NEXT:        .byte   0
 
-attributes #0 = { norecurse nounwind readonly "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { norecurse nounwind readonly "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!3, !4, !5}
diff --git a/llvm/test/CodeGen/BPF/BTF/extern-var-func-weak-section.ll b/llvm/test/CodeGen/BPF/BTF/extern-var-func-weak-section.ll
new file mode 100644
index 000000000..23332c9d9
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/BTF/extern-var-func-weak-section.ll
@@ -0,0 +1,90 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+;
+; Source code:
+;   extern int global_func(char c) __attribute__((weak, section("abc")));
+;   int test() {
+;      return global_func(0);
+;   }
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm test.c
+
+; Function Attrs: nounwind
+define dso_local i32 @test() local_unnamed_addr #0 !dbg !13 {
+entry:
+  %call = tail call i32 @global_func(i8 signext 0) #2, !dbg !16
+  ret i32 %call, !dbg !17
+}
+declare !dbg !4 extern_weak dso_local i32 @global_func(i8 signext) local_unnamed_addr #1 section "abc"
+
+; CHECK:             .section        .BTF,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   88
+; CHECK-NEXT:        .long   88
+; CHECK-NEXT:        .long   72
+; CHECK-NEXT:        .long   0                       # BTF_KIND_FUNC_PROTO(id = 1)
+; CHECK-NEXT:        .long   218103808               # 0xd000000
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   1                       # BTF_KIND_INT(id = 2)
+; CHECK-NEXT:        .long   16777216                # 0x1000000
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   16777248                # 0x1000020
+; CHECK-NEXT:        .long   5                       # BTF_KIND_FUNC(id = 3)
+; CHECK-NEXT:        .long   201326593               # 0xc000001
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   0                       # BTF_KIND_FUNC_PROTO(id = 4)
+; CHECK-NEXT:        .long   218103809               # 0xd000001
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   5
+; CHECK-NEXT:        .long   55                      # BTF_KIND_INT(id = 5)
+; CHECK-NEXT:        .long   16777216                # 0x1000000
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   16777224                # 0x1000008
+; CHECK-NEXT:        .long   60                      # BTF_KIND_FUNC(id = 6)
+; CHECK-NEXT:        .long   201326594               # 0xc000002
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .byte   0                       # string offset=0
+; CHECK-NEXT:        .ascii  "int"                   # string offset=1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "test"                  # string offset=5
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  ".text"                 # string offset=10
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "/tmp/home/yhs/work/tests/extern/test.c" # string offset=16
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "char"                  # string offset=55
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "global_func"           # string offset=60
+; CHECK-NEXT:        .byte   0
+
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { nounwind }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!9, !10, !11}
+!llvm.ident = !{!12}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 10.0.0 (https://github.com/llvm/llvm-project.git 71a9518c93fe1dce9611c24bc707e5baf1f39f0d)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !3, nameTableKind: None)
+!1 = !DIFile(filename: "test.c", directory: "/tmp/home/yhs/work/tests/extern")
+!2 = !{}
+!3 = !{!4}
+!4 = !DISubprogram(name: "global_func", scope: !1, file: !1, line: 1, type: !5, flags: DIFlagPrototyped, spFlags: DISPFlagOptimized, retainedNodes: !2)
+!5 = !DISubroutineType(types: !6)
+!6 = !{!7, !8}
+!7 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!8 = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_signed_char)
+!9 = !{i32 7, !"Dwarf Version", i32 4}
+!10 = !{i32 2, !"Debug Info Version", i32 3}
+!11 = !{i32 1, !"wchar_size", i32 4}
+!12 = !{!"clang version 10.0.0 (https://github.com/llvm/llvm-project.git 71a9518c93fe1dce9611c24bc707e5baf1f39f0d)"}
+!13 = distinct !DISubprogram(name: "test", scope: !1, file: !1, line: 2, type: !14, scopeLine: 2, flags: DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2)
+!14 = !DISubroutineType(types: !15)
+!15 = !{!7}
+!16 = !DILocation(line: 3, column: 11, scope: !13)
+!17 = !DILocation(line: 3, column: 4, scope: !13)
diff --git a/llvm/test/CodeGen/BPF/BTF/extern-var-func-weak.ll b/llvm/test/CodeGen/BPF/BTF/extern-var-func-weak.ll
new file mode 100644
index 000000000..984a93655
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/BTF/extern-var-func-weak.ll
@@ -0,0 +1,90 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+;
+; Source code:
+;   extern int global_func(char c) __attribute__((weak));
+;   int test() {
+;     return global_func(0);
+;   }
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm test.c
+
+; Function Attrs: nounwind
+define dso_local i32 @test() local_unnamed_addr #0 !dbg !13 {
+entry:
+  %call = tail call i32 @global_func(i8 signext 0) #2, !dbg !16
+  ret i32 %call, !dbg !17
+}
+declare !dbg !4 extern_weak dso_local i32 @global_func(i8 signext) local_unnamed_addr #1
+
+; CHECK:             .section        .BTF,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   88
+; CHECK-NEXT:        .long   88
+; CHECK-NEXT:        .long   72
+; CHECK-NEXT:        .long   0                       # BTF_KIND_FUNC_PROTO(id = 1)
+; CHECK-NEXT:        .long   218103808               # 0xd000000
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   1                       # BTF_KIND_INT(id = 2)
+; CHECK-NEXT:        .long   16777216                # 0x1000000
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   16777248                # 0x1000020
+; CHECK-NEXT:        .long   5                       # BTF_KIND_FUNC(id = 3)
+; CHECK-NEXT:        .long   201326593               # 0xc000001
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   0                       # BTF_KIND_FUNC_PROTO(id = 4)
+; CHECK-NEXT:        .long   218103809               # 0xd000001
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   5
+; CHECK-NEXT:        .long   55                      # BTF_KIND_INT(id = 5)
+; CHECK-NEXT:        .long   16777216                # 0x1000000
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   16777224                # 0x1000008
+; CHECK-NEXT:        .long   60                      # BTF_KIND_FUNC(id = 6)
+; CHECK-NEXT:        .long   201326594               # 0xc000002
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .byte   0                       # string offset=0
+; CHECK-NEXT:        .ascii  "int"                   # string offset=1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "test"                  # string offset=5
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  ".text"                 # string offset=10
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "/tmp/home/yhs/work/tests/extern/test.c" # string offset=16
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "char"                  # string offset=55
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "global_func"           # string offset=60
+; CHECK-NEXT:        .byte   0
+
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { nounwind }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!9, !10, !11}
+!llvm.ident = !{!12}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 10.0.0 (https://github.com/llvm/llvm-project.git 71a9518c93fe1dce9611c24bc707e5baf1f39f0d)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !3, nameTableKind: None)
+!1 = !DIFile(filename: "test.c", directory: "/tmp/home/yhs/work/tests/extern")
+!2 = !{}
+!3 = !{!4}
+!4 = !DISubprogram(name: "global_func", scope: !1, file: !1, line: 1, type: !5, flags: DIFlagPrototyped, spFlags: DISPFlagOptimized, retainedNodes: !2)
+!5 = !DISubroutineType(types: !6)
+!6 = !{!7, !8}
+!7 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!8 = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_signed_char)
+!9 = !{i32 7, !"Dwarf Version", i32 4}
+!10 = !{i32 2, !"Debug Info Version", i32 3}
+!11 = !{i32 1, !"wchar_size", i32 4}
+!12 = !{!"clang version 10.0.0 (https://github.com/llvm/llvm-project.git 71a9518c93fe1dce9611c24bc707e5baf1f39f0d)"}
+!13 = distinct !DISubprogram(name: "test", scope: !1, file: !1, line: 2, type: !14, scopeLine: 2, flags: DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2)
+!14 = !DISubroutineType(types: !15)
+!15 = !{!7}
+!16 = !DILocation(line: 3, column: 11, scope: !13)
+!17 = !DILocation(line: 3, column: 4, scope: !13)
diff --git a/llvm/test/CodeGen/BPF/BTF/extern-var-func.ll b/llvm/test/CodeGen/BPF/BTF/extern-var-func.ll
new file mode 100644
index 000000000..61239befe
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/BTF/extern-var-func.ll
@@ -0,0 +1,91 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+;
+; Source code:
+;   extern int global_func(char c);
+;   int test() {
+;     return global_func(0);
+;   }
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm test.c
+
+; Function Attrs: nounwind
+define dso_local i32 @test() local_unnamed_addr #0 !dbg !13 {
+entry:
+  %call = tail call i32 @global_func(i8 signext 0) #2, !dbg !16
+  ret i32 %call, !dbg !17
+}
+
+; CHECK:             .section        .BTF,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   88
+; CHECK-NEXT:        .long   88
+; CHECK-NEXT:        .long   72
+; CHECK-NEXT:        .long   0                       # BTF_KIND_FUNC_PROTO(id = 1)
+; CHECK-NEXT:        .long   218103808               # 0xd000000
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   1                       # BTF_KIND_INT(id = 2)
+; CHECK-NEXT:        .long   16777216                # 0x1000000
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   16777248                # 0x1000020
+; CHECK-NEXT:        .long   5                       # BTF_KIND_FUNC(id = 3)
+; CHECK-NEXT:        .long   201326593               # 0xc000001
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   0                       # BTF_KIND_FUNC_PROTO(id = 4)
+; CHECK-NEXT:        .long   218103809               # 0xd000001
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   5
+; CHECK-NEXT:        .long   55                      # BTF_KIND_INT(id = 5)
+; CHECK-NEXT:        .long   16777216                # 0x1000000
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   16777224                # 0x1000008
+; CHECK-NEXT:        .long   60                      # BTF_KIND_FUNC(id = 6)
+; CHECK-NEXT:        .long   201326594               # 0xc000002
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .byte   0                       # string offset=0
+; CHECK-NEXT:        .ascii  "int"                   # string offset=1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "test"                  # string offset=5
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  ".text"                 # string offset=10
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "/tmp/home/yhs/work/tests/extern/test.c" # string offset=16
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "char"                  # string offset=55
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "global_func"           # string offset=60
+; CHECK-NEXT:        .byte   0
+
+declare !dbg !4 dso_local i32 @global_func(i8 signext) local_unnamed_addr #1
+
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { nounwind }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!9, !10, !11}
+!llvm.ident = !{!12}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 10.0.0 (https://github.com/llvm/llvm-project.git 77e5c60f04c4597ba5704d3cee61c6d359404ccd)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !3, nameTableKind: None)
+!1 = !DIFile(filename: "test.c", directory: "/tmp/home/yhs/work/tests/extern")
+!2 = !{}
+!3 = !{!4}
+!4 = !DISubprogram(name: "global_func", scope: !1, file: !1, line: 1, type: !5, flags: DIFlagPrototyped, spFlags: DISPFlagOptimized, retainedNodes: !2)
+!5 = !DISubroutineType(types: !6)
+!6 = !{!7, !8}
+!7 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!8 = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_signed_char)
+!9 = !{i32 2, !"Dwarf Version", i32 4}
+!10 = !{i32 2, !"Debug Info Version", i32 3}
+!11 = !{i32 1, !"wchar_size", i32 4}
+!12 = !{!"clang version 10.0.0 (https://github.com/llvm/llvm-project.git 77e5c60f04c4597ba5704d3cee61c6d359404ccd)"}
+!13 = distinct !DISubprogram(name: "test", scope: !1, file: !1, line: 2, type: !14, scopeLine: 2, flags: DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2)
+!14 = !DISubroutineType(types: !15)
+!15 = !{!7}
+!16 = !DILocation(line: 3, column: 10, scope: !13)
+!17 = !DILocation(line: 3, column: 3, scope: !13)
diff --git a/llvm/test/CodeGen/BPF/BTF/extern-var-section.ll b/llvm/test/CodeGen/BPF/BTF/extern-var-section.ll
new file mode 100644
index 000000000..e01da7e20
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/BTF/extern-var-section.ll
@@ -0,0 +1,119 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+;
+; Source code:
+;   extern int global_func(char c) __attribute__((section("abc")));
+;   extern char ch __attribute__((section("abc")));
+;   int test() {
+;     return global_func(0) + ch;
+;   }
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm test.c
+
+@ch = external dso_local local_unnamed_addr global i8, section "abc", align 1, !dbg !0
+
+; Function Attrs: nounwind
+define dso_local i32 @test() local_unnamed_addr #0 !dbg !16 {
+entry:
+  %call = tail call i32 @global_func(i8 signext 0) #2, !dbg !19
+  %0 = load i8, i8* @ch, align 1, !dbg !20, !tbaa !21
+  %conv = sext i8 %0 to i32, !dbg !20
+  %add = add nsw i32 %call, %conv, !dbg !24
+  ret i32 %add, !dbg !25
+}
+
+; CHECK:             .section        .BTF,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   128
+; CHECK-NEXT:        .long   128
+; CHECK-NEXT:        .long   79
+; CHECK-NEXT:        .long   0                       # BTF_KIND_FUNC_PROTO(id = 1)
+; CHECK-NEXT:        .long   218103808               # 0xd000000
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   1                       # BTF_KIND_INT(id = 2)
+; CHECK-NEXT:        .long   16777216                # 0x1000000
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   16777248                # 0x1000020
+; CHECK-NEXT:        .long   5                       # BTF_KIND_FUNC(id = 3)
+; CHECK-NEXT:        .long   201326593               # 0xc000001
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   0                       # BTF_KIND_FUNC_PROTO(id = 4)
+; CHECK-NEXT:        .long   218103809               # 0xd000001
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   5
+; CHECK-NEXT:        .long   55                      # BTF_KIND_INT(id = 5)
+; CHECK-NEXT:        .long   16777216                # 0x1000000
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   16777224                # 0x1000008
+; CHECK-NEXT:        .long   60                      # BTF_KIND_FUNC(id = 6)
+; CHECK-NEXT:        .long   201326594               # 0xc000002
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   72                      # BTF_KIND_VAR(id = 7)
+; CHECK-NEXT:        .long   234881024               # 0xe000000
+; CHECK-NEXT:        .long   5
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   75                      # BTF_KIND_DATASEC(id = 8)
+; CHECK-NEXT:        .long   251658241               # 0xf000001
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   7
+; CHECK-NEXT:        .long   ch
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .byte   0                       # string offset=0
+; CHECK-NEXT:        .ascii  "int"                   # string offset=1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "test"                  # string offset=5
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  ".text"                 # string offset=10
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "/tmp/home/yhs/work/tests/extern/test.c" # string offset=16
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "char"                  # string offset=55
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "global_func"           # string offset=60
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "ch"                    # string offset=72
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "abc"                   # string offset=75
+; CHECK-NEXT:        .byte   0
+
+declare !dbg !6 dso_local i32 @global_func(i8 signext) local_unnamed_addr #1 section "abc"
+
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { nounwind }
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!12, !13, !14}
+!llvm.ident = !{!15}
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "ch", scope: !2, file: !3, line: 2, type: !10, isLocal: false, isDefinition: false)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 10.0.0 (https://github.com/llvm/llvm-project.git 0ad024346185b3f0b5167438e126568982b1168d)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, retainedTypes: !5, globals: !11, nameTableKind: None)
+!3 = !DIFile(filename: "test.c", directory: "/tmp/home/yhs/work/tests/extern")
+!4 = !{}
+!5 = !{!6}
+!6 = !DISubprogram(name: "global_func", scope: !3, file: !3, line: 1, type: !7, flags: DIFlagPrototyped, spFlags: DISPFlagOptimized, retainedNodes: !4)
+!7 = !DISubroutineType(types: !8)
+!8 = !{!9, !10}
+!9 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!10 = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_signed_char)
+!11 = !{!0}
+!12 = !{i32 2, !"Dwarf Version", i32 4}
+!13 = !{i32 2, !"Debug Info Version", i32 3}
+!14 = !{i32 1, !"wchar_size", i32 4}
+!15 = !{!"clang version 10.0.0 (https://github.com/llvm/llvm-project.git 0ad024346185b3f0b5167438e126568982b1168d)"}
+!16 = distinct !DISubprogram(name: "test", scope: !3, file: !3, line: 3, type: !17, scopeLine: 3, flags: DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !2, retainedNodes: !4)
+!17 = !DISubroutineType(types: !18)
+!18 = !{!9}
+!19 = !DILocation(line: 4, column: 10, scope: !16)
+!20 = !DILocation(line: 4, column: 27, scope: !16)
+!21 = !{!22, !22, i64 0}
+!22 = !{!"omnipotent char", !23, i64 0}
+!23 = !{!"Simple C/C++ TBAA"}
+!24 = !DILocation(line: 4, column: 25, scope: !16)
+!25 = !DILocation(line: 4, column: 3, scope: !16)
diff --git a/llvm/test/CodeGen/BPF/BTF/extern-var-struct-weak.ll b/llvm/test/CodeGen/BPF/BTF/extern-var-struct-weak.ll
new file mode 100644
index 000000000..3422afe94
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/BTF/extern-var-struct-weak.ll
@@ -0,0 +1,109 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+;
+; Source code:
+;   typedef struct t1 { int f1; } __t1;
+;   extern __t1 global __attribute__((weak));
+;   int test() { return global.f1; }
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm test.c
+
+%struct.t1 = type { i32 }
+
+@global = extern_weak dso_local local_unnamed_addr global %struct.t1, align 4, !dbg !0
+; Function Attrs: norecurse nounwind readonly
+define dso_local i32 @test() local_unnamed_addr #0 !dbg !15 {
+entry:
+  %0 = load i32, i32* getelementptr (%struct.t1, %struct.t1* @global, i64 0, i32 0), align 4, !dbg !18, !tbaa !19
+  ret i32 %0, !dbg !24
+}
+
+; CHECK:             .section        .BTF,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   116
+; CHECK-NEXT:        .long   116
+; CHECK-NEXT:        .long   81
+; CHECK-NEXT:        .long   0                       # BTF_KIND_FUNC_PROTO(id = 1)
+; CHECK-NEXT:        .long   218103808               # 0xd000000
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   1                       # BTF_KIND_INT(id = 2)
+; CHECK-NEXT:        .long   16777216                # 0x1000000
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   16777248                # 0x1000020
+; CHECK-NEXT:        .long   5                       # BTF_KIND_FUNC(id = 3)
+; CHECK-NEXT:        .long   201326593               # 0xc000001
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   55                      # BTF_KIND_TYPEDEF(id = 4)
+; CHECK-NEXT:        .long   134217728               # 0x8000000
+; CHECK-NEXT:        .long   5
+; CHECK-NEXT:        .long   60                      # BTF_KIND_STRUCT(id = 5)
+; CHECK-NEXT:        .long   67108865                # 0x4000001
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   63
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   0                       # 0x0
+; CHECK-NEXT:        .long   66                      # BTF_KIND_VAR(id = 6)
+; CHECK-NEXT:        .long   234881024               # 0xe000000
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   73                      # BTF_KIND_DATASEC(id = 7)
+; CHECK-NEXT:        .long   251658241               # 0xf000001
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   6
+; CHECK-NEXT:        .long   global
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .byte   0                       # string offset=0
+; CHECK-NEXT:        .ascii  "int"                   # string offset=1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "test"                  # string offset=5
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  ".text"                 # string offset=10
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "/tmp/home/yhs/work/tests/extern/test.c" # string offset=16
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "__t1"                  # string offset=55
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "t1"                    # string offset=60
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "f1"                    # string offset=63
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "global"                # string offset=66
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  ".extern"               # string offset=73
+; CHECK-NEXT:        .byte   0
+
+attributes #0 = { norecurse nounwind readonly "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!11, !12, !13}
+!llvm.ident = !{!14}
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "global", scope: !2, file: !3, line: 2, type: !6, isLocal: false, isDefinition: false)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 10.0.0 (https://github.com/llvm/llvm-project.git 71a9518c93fe1dce9611c24bc707e5baf1f39f0d)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5, nameTableKind: None)
+!3 = !DIFile(filename: "test.c", directory: "/tmp/home/yhs/work/tests/extern")
+!4 = !{}
+!5 = !{!0}
+!6 = !DIDerivedType(tag: DW_TAG_typedef, name: "__t1", file: !3, line: 1, baseType: !7)
+!7 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t1", file: !3, line: 1, size: 32, elements: !8)
+!8 = !{!9}
+!9 = !DIDerivedType(tag: DW_TAG_member, name: "f1", scope: !7, file: !3, line: 1, baseType: !10, size: 32)
+!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!11 = !{i32 7, !"Dwarf Version", i32 4}
+!12 = !{i32 2, !"Debug Info Version", i32 3}
+!13 = !{i32 1, !"wchar_size", i32 4}
+!14 = !{!"clang version 10.0.0 (https://github.com/llvm/llvm-project.git 71a9518c93fe1dce9611c24bc707e5baf1f39f0d)"}
+!15 = distinct !DISubprogram(name: "test", scope: !3, file: !3, line: 3, type: !16, scopeLine: 3, flags: DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !2, retainedNodes: !4)
+!16 = !DISubroutineType(types: !17)
+!17 = !{!10}
+!18 = !DILocation(line: 3, column: 28, scope: !15)
+!19 = !{!20, !21, i64 0}
+!20 = !{!"t1", !21, i64 0}
+!21 = !{!"int", !22, i64 0}
+!22 = !{!"omnipotent char", !23, i64 0}
+!23 = !{!"Simple C/C++ TBAA"}
+!24 = !DILocation(line: 3, column: 14, scope: !15)
diff --git a/llvm/test/CodeGen/BPF/BTF/extern-var-struct.ll b/llvm/test/CodeGen/BPF/BTF/extern-var-struct.ll
new file mode 100644
index 000000000..5d8883b71
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/BTF/extern-var-struct.ll
@@ -0,0 +1,110 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+;
+; Source code:
+;   typedef struct t1 { int f1; } __t1;
+;   extern __t1 global;
+;   int test() { return global.f1; }
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm test.c
+
+%struct.t1 = type { i32 }
+
+@global = external dso_local local_unnamed_addr global %struct.t1, align 4, !dbg !0
+
+; Function Attrs: norecurse nounwind readonly
+define dso_local i32 @test() local_unnamed_addr #0 !dbg !15 {
+entry:
+  %0 = load i32, i32* getelementptr inbounds (%struct.t1, %struct.t1* @global, i64 0, i32 0), align 4, !dbg !18, !tbaa !19
+  ret i32 %0, !dbg !24
+}
+
+; CHECK:             .section        .BTF,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   116
+; CHECK-NEXT:        .long   116
+; CHECK-NEXT:        .long   81
+; CHECK-NEXT:        .long   0                       # BTF_KIND_FUNC_PROTO(id = 1)
+; CHECK-NEXT:        .long   218103808               # 0xd000000
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   1                       # BTF_KIND_INT(id = 2)
+; CHECK-NEXT:        .long   16777216                # 0x1000000
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   16777248                # 0x1000020
+; CHECK-NEXT:        .long   5                       # BTF_KIND_FUNC(id = 3)
+; CHECK-NEXT:        .long   201326593               # 0xc000001
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   55                      # BTF_KIND_TYPEDEF(id = 4)
+; CHECK-NEXT:        .long   134217728               # 0x8000000
+; CHECK-NEXT:        .long   5
+; CHECK-NEXT:        .long   60                      # BTF_KIND_STRUCT(id = 5)
+; CHECK-NEXT:        .long   67108865                # 0x4000001
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   63
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   0                       # 0x0
+; CHECK-NEXT:        .long   66                      # BTF_KIND_VAR(id = 6)
+; CHECK-NEXT:        .long   234881024               # 0xe000000
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   73                      # BTF_KIND_DATASEC(id = 7)
+; CHECK-NEXT:        .long   251658241               # 0xf000001
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   6
+; CHECK-NEXT:        .long   global
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .byte   0                       # string offset=0
+; CHECK-NEXT:        .ascii  "int"                   # string offset=1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "test"                  # string offset=5
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  ".text"                 # string offset=10
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "/tmp/home/yhs/work/tests/extern/test.c" # string offset=16
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "__t1"                  # string offset=55
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "t1"                    # string offset=60
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "f1"                    # string offset=63
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "global"                # string offset=66
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  ".extern"               # string offset=73
+; CHECK-NEXT:        .byte   0
+
+attributes #0 = { norecurse nounwind readonly "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!11, !12, !13}
+!llvm.ident = !{!14}
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "global", scope: !2, file: !3, line: 2, type: !6, isLocal: false, isDefinition: false)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 10.0.0 (https://github.com/llvm/llvm-project.git 2798d63180f4cc873bdaf689705fd4f9521ae89f)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5, nameTableKind: None)
+!3 = !DIFile(filename: "test.c", directory: "/tmp/home/yhs/work/tests/extern")
+!4 = !{}
+!5 = !{!0}
+!6 = !DIDerivedType(tag: DW_TAG_typedef, name: "__t1", file: !3, line: 1, baseType: !7)
+!7 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t1", file: !3, line: 1, size: 32, elements: !8)
+!8 = !{!9}
+!9 = !DIDerivedType(tag: DW_TAG_member, name: "f1", scope: !7, file: !3, line: 1, baseType: !10, size: 32)
+!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!11 = !{i32 2, !"Dwarf Version", i32 4}
+!12 = !{i32 2, !"Debug Info Version", i32 3}
+!13 = !{i32 1, !"wchar_size", i32 4}
+!14 = !{!"clang version 10.0.0 (https://github.com/llvm/llvm-project.git 2798d63180f4cc873bdaf689705fd4f9521ae89f)"}
+!15 = distinct !DISubprogram(name: "test", scope: !3, file: !3, line: 4, type: !16, scopeLine: 4, flags: DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !2, retainedNodes: !4)
+!16 = !DISubroutineType(types: !17)
+!17 = !{!10}
+!18 = !DILocation(line: 4, column: 28, scope: !15)
+!19 = !{!20, !21, i64 0}
+!20 = !{!"t1", !21, i64 0}
+!21 = !{!"int", !22, i64 0}
+!22 = !{!"omnipotent char", !23, i64 0}
+!23 = !{!"Simple C/C++ TBAA"}
+!24 = !DILocation(line: 4, column: 14, scope: !15)
diff --git a/llvm/test/CodeGen/BPF/BTF/extern-var-weak-section.ll b/llvm/test/CodeGen/BPF/BTF/extern-var-weak-section.ll
new file mode 100644
index 000000000..6e64d9b4e
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/BTF/extern-var-weak-section.ll
@@ -0,0 +1,117 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+;
+; Source code:
+;   extern int global_func(char c) __attribute__((weak, section("abc")));
+;   extern char ch __attribute__((weak, section("abc")));
+;   int test() {
+;     return global_func(0) + ch;
+;   }
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm test.c
+
+@ch = extern_weak dso_local local_unnamed_addr global i8, section "abc", align 1, !dbg !0
+; Function Attrs: nounwind
+define dso_local i32 @test() local_unnamed_addr #0 !dbg !16 {
+entry:
+  %call = tail call i32 @global_func(i8 signext 0) #2, !dbg !19
+  %0 = load i8, i8* @ch, align 1, !dbg !20, !tbaa !21
+  %conv = sext i8 %0 to i32, !dbg !20
+  %add = add nsw i32 %call, %conv, !dbg !24
+  ret i32 %add, !dbg !25
+}
+declare !dbg !6 extern_weak dso_local i32 @global_func(i8 signext) local_unnamed_addr #1 section "abc"
+
+; CHECK:             .section        .BTF,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   128
+; CHECK-NEXT:        .long   128
+; CHECK-NEXT:        .long   79
+; CHECK-NEXT:        .long   0                       # BTF_KIND_FUNC_PROTO(id = 1)
+; CHECK-NEXT:        .long   218103808               # 0xd000000
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   1                       # BTF_KIND_INT(id = 2)
+; CHECK-NEXT:        .long   16777216                # 0x1000000
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   16777248                # 0x1000020
+; CHECK-NEXT:        .long   5                       # BTF_KIND_FUNC(id = 3)
+; CHECK-NEXT:        .long   201326593               # 0xc000001
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   0                       # BTF_KIND_FUNC_PROTO(id = 4)
+; CHECK-NEXT:        .long   218103809               # 0xd000001
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   5
+; CHECK-NEXT:        .long   55                      # BTF_KIND_INT(id = 5)
+; CHECK-NEXT:        .long   16777216                # 0x1000000
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   16777224                # 0x1000008
+; CHECK-NEXT:        .long   60                      # BTF_KIND_FUNC(id = 6)
+; CHECK-NEXT:        .long   201326594               # 0xc000002
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   72                      # BTF_KIND_VAR(id = 7)
+; CHECK-NEXT:        .long   234881024               # 0xe000000
+; CHECK-NEXT:        .long   5
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   75                      # BTF_KIND_DATASEC(id = 8)
+; CHECK-NEXT:        .long   251658241               # 0xf000001
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   7
+; CHECK-NEXT:        .long   ch
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .byte   0                       # string offset=0
+; CHECK-NEXT:        .ascii  "int"                   # string offset=1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "test"                  # string offset=5
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  ".text"                 # string offset=10
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "/tmp/home/yhs/work/tests/extern/test.c" # string offset=16
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "char"                  # string offset=55
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "global_func"           # string offset=60
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "ch"                    # string offset=72
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "abc"                   # string offset=75
+; CHECK-NEXT:        .byte   0
+
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { nounwind }
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!12, !13, !14}
+!llvm.ident = !{!15}
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "ch", scope: !2, file: !3, line: 2, type: !10, isLocal: false, isDefinition: false)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 10.0.0 (https://github.com/llvm/llvm-project.git 71a9518c93fe1dce9611c24bc707e5baf1f39f0d)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, retainedTypes: !5, globals: !11, nameTableKind: None)
+!3 = !DIFile(filename: "test.c", directory: "/tmp/home/yhs/work/tests/extern")
+!4 = !{}
+!5 = !{!6}
+!6 = !DISubprogram(name: "global_func", scope: !3, file: !3, line: 1, type: !7, flags: DIFlagPrototyped, spFlags: DISPFlagOptimized, retainedNodes: !4)
+!7 = !DISubroutineType(types: !8)
+!8 = !{!9, !10}
+!9 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!10 = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_signed_char)
+!11 = !{!0}
+!12 = !{i32 7, !"Dwarf Version", i32 4}
+!13 = !{i32 2, !"Debug Info Version", i32 3}
+!14 = !{i32 1, !"wchar_size", i32 4}
+!15 = !{!"clang version 10.0.0 (https://github.com/llvm/llvm-project.git 71a9518c93fe1dce9611c24bc707e5baf1f39f0d)"}
+!16 = distinct !DISubprogram(name: "test", scope: !3, file: !3, line: 3, type: !17, scopeLine: 3, flags: DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !2, retainedNodes: !4)
+!17 = !DISubroutineType(types: !18)
+!18 = !{!9}
+!19 = !DILocation(line: 4, column: 10, scope: !16)
+!20 = !DILocation(line: 4, column: 27, scope: !16)
+!21 = !{!22, !22, i64 0}
+!22 = !{!"omnipotent char", !23, i64 0}
+!23 = !{!"Simple C/C++ TBAA"}
+!24 = !DILocation(line: 4, column: 25, scope: !16)
+!25 = !DILocation(line: 4, column: 3, scope: !16)
diff --git a/llvm/test/CodeGen/BPF/BTF/filename.ll b/llvm/test/CodeGen/BPF/BTF/filename.ll
index fd96720f0..2ae03a647 100644
--- a/llvm/test/CodeGen/BPF/BTF/filename.ll
+++ b/llvm/test/CodeGen/BPF/BTF/filename.ll
@@ -28,7 +28,7 @@ define dso_local i32 @test() local_unnamed_addr #0 !dbg !7 {
 ; CHECK-NEXT:        .long   4
 ; CHECK-NEXT:        .long   16777248                # 0x1000020
 ; CHECK-NEXT:        .long   5                       # BTF_KIND_FUNC(id = 3)
-; CHECK-NEXT:        .long   201326592               # 0xc000000
+; CHECK-NEXT:        .long   201326593               # 0xc000001
 ; CHECK-NEXT:        .long   1
 ; CHECK-NEXT:        .byte   0                       # string offset=0
 ; CHECK-NEXT:        .ascii  "int"                   # string offset=1
@@ -43,15 +43,13 @@ define dso_local i32 @test() local_unnamed_addr #0 !dbg !7 {
 ; CHECK-NEXT:        .short  60319                   # 0xeb9f
 ; CHECK-NEXT:        .byte   1
 ; CHECK-NEXT:        .byte   0
-; CHECK-NEXT:        .long   40
+; CHECK-NEXT:        .long   32
 ; CHECK-NEXT:        .long   0
 ; CHECK-NEXT:        .long   20
 ; CHECK-NEXT:        .long   20
 ; CHECK-NEXT:        .long   28
 ; CHECK-NEXT:        .long   48
 ; CHECK-NEXT:        .long   0
-; CHECK-NEXT:        .long   48
-; CHECK-NEXT:        .long   0
 ; CHECK-NEXT:        .long   8                       # FuncInfo
 ; CHECK-NEXT:        .long   10                      # FuncInfo section string offset=10
 ; CHECK-NEXT:        .long   1
@@ -65,7 +63,7 @@ define dso_local i32 @test() local_unnamed_addr #0 !dbg !7 {
 ; CHECK-NEXT:        .long   0
 ; CHECK-NEXT:        .long   1038                    # Line 1 Col 14
 
-attributes #0 = { norecurse nounwind readnone uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { norecurse nounwind readnone uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!3, !4, !5}
diff --git a/llvm/test/CodeGen/BPF/BTF/func-func-ptr.ll b/llvm/test/CodeGen/BPF/BTF/func-func-ptr.ll
index e61459125..786e29494 100644
--- a/llvm/test/CodeGen/BPF/BTF/func-func-ptr.ll
+++ b/llvm/test/CodeGen/BPF/BTF/func-func-ptr.ll
@@ -39,7 +39,7 @@ entry:
 ; CHECK-NEXT:        .long   5
 ; CHECK-NEXT:        .long   1
 ; CHECK-NEXT:        .long   8                       # BTF_KIND_FUNC(id = 3)
-; CHECK-NEXT:        .long   201326592               # 0xc000000
+; CHECK-NEXT:        .long   201326593               # 0xc000001
 ; CHECK-NEXT:        .long   2
 ; CHECK-NEXT:        .long   0                       # BTF_KIND_PTR(id = 4)
 ; CHECK-NEXT:        .long   33554432                # 0x2000000
@@ -74,15 +74,13 @@ entry:
 ; CHECK-NEXT:        .short  60319                   # 0xeb9f
 ; CHECK-NEXT:        .byte   1
 ; CHECK-NEXT:        .byte   0
-; CHECK-NEXT:        .long   40
+; CHECK-NEXT:        .long   32
 ; CHECK-NEXT:        .long   0
 ; CHECK-NEXT:        .long   20
 ; CHECK-NEXT:        .long   20
 ; CHECK-NEXT:        .long   28
 ; CHECK-NEXT:        .long   48
 ; CHECK-NEXT:        .long   0
-; CHECK-NEXT:        .long   48
-; CHECK-NEXT:        .long   0
 ; CHECK-NEXT:        .long   8                       # FuncInfo
 ; CHECK-NEXT:        .long   11                      # FuncInfo section string offset=11
 ; CHECK-NEXT:        .long   1
@@ -99,7 +97,7 @@ entry:
 ; Function Attrs: nounwind readnone speculatable
 declare void @llvm.dbg.value(metadata, metadata, metadata) #1
 
-attributes #0 = { nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone speculatable }
 
 !llvm.dbg.cu = !{!2}
diff --git a/llvm/test/CodeGen/BPF/BTF/func-non-void.ll b/llvm/test/CodeGen/BPF/BTF/func-non-void.ll
index 5593ea888..59e9b89a3 100644
--- a/llvm/test/CodeGen/BPF/BTF/func-non-void.ll
+++ b/llvm/test/CodeGen/BPF/BTF/func-non-void.ll
@@ -31,7 +31,7 @@ define dso_local i32 @f1(i32 returned) local_unnamed_addr #0 !dbg !7 {
 ; CHECK-NEXT:        .long   5
 ; CHECK-NEXT:        .long   1
 ; CHECK-NEXT:        .long   8                       # BTF_KIND_FUNC(id = 3)
-; CHECK-NEXT:        .long   201326592               # 0xc000000
+; CHECK-NEXT:        .long   201326593               # 0xc000001
 ; CHECK-NEXT:        .long   2
 ; CHECK-NEXT:        .byte   0                       # string offset=0
 ; CHECK-NEXT:        .ascii  "int"                   # string offset=1
@@ -48,15 +48,13 @@ define dso_local i32 @f1(i32 returned) local_unnamed_addr #0 !dbg !7 {
 ; CHECK-NEXT:        .short  60319                   # 0xeb9f
 ; CHECK-NEXT:        .byte   1
 ; CHECK-NEXT:        .byte   0
-; CHECK-NEXT:        .long   40
+; CHECK-NEXT:        .long   32
 ; CHECK-NEXT:        .long   0
 ; CHECK-NEXT:        .long   20
 ; CHECK-NEXT:        .long   20
 ; CHECK-NEXT:        .long   44
 ; CHECK-NEXT:        .long   64
 ; CHECK-NEXT:        .long   0
-; CHECK-NEXT:        .long   64
-; CHECK-NEXT:        .long   0
 ; CHECK-NEXT:        .long   8                       # FuncInfo
 ; CHECK-NEXT:        .long   11                      # FuncInfo section string offset=11
 ; CHECK-NEXT:        .long   1
@@ -77,7 +75,7 @@ define dso_local i32 @f1(i32 returned) local_unnamed_addr #0 !dbg !7 {
 ; Function Attrs: nounwind readnone speculatable
 declare void @llvm.dbg.value(metadata, metadata, metadata) #1
 
-attributes #0 = { nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone speculatable }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/CodeGen/BPF/BTF/func-source.ll b/llvm/test/CodeGen/BPF/BTF/func-source.ll
index 0d6e098b3..63814d766 100644
--- a/llvm/test/CodeGen/BPF/BTF/func-source.ll
+++ b/llvm/test/CodeGen/BPF/BTF/func-source.ll
@@ -28,7 +28,7 @@ entry:
 ; CHECK-NEXT:        .long   218103808               # 0xd000000
 ; CHECK-NEXT:        .long   0
 ; CHECK-NEXT:        .long   1                       # BTF_KIND_FUNC(id = 2)
-; CHECK-NEXT:        .long   201326592               # 0xc000000
+; CHECK-NEXT:        .long   201326593               # 0xc000001
 ; CHECK-NEXT:        .long   1
 ; CHECK-NEXT:        .byte   0                       # string offset=0
 ; CHECK-NEXT:        .byte   102                     # string offset=1
@@ -43,15 +43,13 @@ entry:
 ; CHECK-NEXT:        .short  60319                   # 0xeb9f
 ; CHECK-NEXT:        .byte   1
 ; CHECK-NEXT:        .byte   0
-; CHECK-NEXT:        .long   40
+; CHECK-NEXT:        .long   32
 ; CHECK-NEXT:        .long   0
 ; CHECK-NEXT:        .long   20
 ; CHECK-NEXT:        .long   20
 ; CHECK-NEXT:        .long   28
 ; CHECK-NEXT:        .long   48
 ; CHECK-NEXT:        .long   0
-; CHECK-NEXT:        .long   48
-; CHECK-NEXT:        .long   0
 ; CHECK-NEXT:        .long   8                       # FuncInfo
 ; CHECK-NEXT:        .long   3                       # FuncInfo section string offset=3
 ; CHECK-NEXT:        .long   1
@@ -65,7 +63,7 @@ entry:
 ; CHECK-NEXT:        .long   18
 ; CHECK-NEXT:        .long   1040                    # Line 1 Col 16
 
-attributes #0 = { norecurse nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { norecurse nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!3, !4, !5}
diff --git a/llvm/test/CodeGen/BPF/BTF/func-typedef.ll b/llvm/test/CodeGen/BPF/BTF/func-typedef.ll
index 48fcb3362..ac0813f8a 100644
--- a/llvm/test/CodeGen/BPF/BTF/func-typedef.ll
+++ b/llvm/test/CodeGen/BPF/BTF/func-typedef.ll
@@ -40,7 +40,7 @@ entry:
 ; CHECK-NEXT:        .long   16
 ; CHECK-NEXT:        .long   1
 ; CHECK-NEXT:        .long   18                      # BTF_KIND_FUNC(id = 5)
-; CHECK-NEXT:        .long   201326592               # 0xc000000
+; CHECK-NEXT:        .long   201326593               # 0xc000001
 ; CHECK-NEXT:        .long   4
 ; CHECK-NEXT:        .byte   0                       # string offset=0
 ; CHECK-NEXT:        .ascii  "__int"                 # string offset=1
@@ -61,15 +61,13 @@ entry:
 ; CHECK-NEXT:        .short  60319                   # 0xeb9f
 ; CHECK-NEXT:        .byte   1
 ; CHECK-NEXT:        .byte   0
-; CHECK-NEXT:        .long   40
+; CHECK-NEXT:        .long   32
 ; CHECK-NEXT:        .long   0
 ; CHECK-NEXT:        .long   20
 ; CHECK-NEXT:        .long   20
 ; CHECK-NEXT:        .long   44
 ; CHECK-NEXT:        .long   64
 ; CHECK-NEXT:        .long   0
-; CHECK-NEXT:        .long   64
-; CHECK-NEXT:        .long   0
 ; CHECK-NEXT:        .long   8                       # FuncInfo
 ; CHECK-NEXT:        .long   20                      # FuncInfo section string offset=20
 ; CHECK-NEXT:        .long   1
@@ -91,7 +89,7 @@ entry:
 ; Function Attrs: nounwind readnone speculatable
 declare void @llvm.dbg.value(metadata, metadata, metadata) #1
 
-attributes #0 = { nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone speculatable }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/CodeGen/BPF/BTF/func-unused-arg.ll b/llvm/test/CodeGen/BPF/BTF/func-unused-arg.ll
index ea94fb746..9c31c5423 100644
--- a/llvm/test/CodeGen/BPF/BTF/func-unused-arg.ll
+++ b/llvm/test/CodeGen/BPF/BTF/func-unused-arg.ll
@@ -31,7 +31,7 @@ define dso_local i32 @f1(i32) local_unnamed_addr #0 !dbg !7 {
 ; CHECK-NEXT:        .long   5
 ; CHECK-NEXT:        .long   1
 ; CHECK-NEXT:        .long   8                       # BTF_KIND_FUNC(id = 3)
-; CHECK-NEXT:        .long   201326592               # 0xc000000
+; CHECK-NEXT:        .long   201326593               # 0xc000001
 ; CHECK-NEXT:        .long   2
 ; CHECK-NEXT:        .byte   0                       # string offset=0
 ; CHECK-NEXT:        .ascii  "int"                   # string offset=1
@@ -48,15 +48,13 @@ define dso_local i32 @f1(i32) local_unnamed_addr #0 !dbg !7 {
 ; CHECK-NEXT:        .short  60319                   # 0xeb9f
 ; CHECK-NEXT:        .byte   1
 ; CHECK-NEXT:        .byte   0
-; CHECK-NEXT:        .long   40
+; CHECK-NEXT:        .long   32
 ; CHECK-NEXT:        .long   0
 ; CHECK-NEXT:        .long   20
 ; CHECK-NEXT:        .long   20
 ; CHECK-NEXT:        .long   28
 ; CHECK-NEXT:        .long   48
 ; CHECK-NEXT:        .long   0
-; CHECK-NEXT:        .long   48
-; CHECK-NEXT:        .long   0
 ; CHECK-NEXT:        .long   8                       # FuncInfo
 ; CHECK-NEXT:        .long   11                      # FuncInfo section string offset=11
 ; CHECK-NEXT:        .long   1
@@ -73,7 +71,7 @@ define dso_local i32 @f1(i32) local_unnamed_addr #0 !dbg !7 {
 ; Function Attrs: nounwind readnone speculatable
 declare void @llvm.dbg.value(metadata, metadata, metadata) #1
 
-attributes #0 = { nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone speculatable }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/CodeGen/BPF/BTF/func-void.ll b/llvm/test/CodeGen/BPF/BTF/func-void.ll
index 42a24d188..d7b46afb7 100644
--- a/llvm/test/CodeGen/BPF/BTF/func-void.ll
+++ b/llvm/test/CodeGen/BPF/BTF/func-void.ll
@@ -24,7 +24,7 @@ define dso_local void @f1() local_unnamed_addr #0 !dbg !7 {
 ; CHECK-NEXT:        .long   218103808               # 0xd000000
 ; CHECK-NEXT:        .long   0
 ; CHECK-NEXT:        .long   1                       # BTF_KIND_FUNC(id = 2)
-; CHECK-NEXT:        .long   201326592               # 0xc000000
+; CHECK-NEXT:        .long   201326593               # 0xc000001
 ; CHECK-NEXT:        .long   1
 ; CHECK-NEXT:        .byte   0                       # string offset=0
 ; CHECK-NEXT:        .ascii  "f1"                    # string offset=1
@@ -37,15 +37,13 @@ define dso_local void @f1() local_unnamed_addr #0 !dbg !7 {
 ; CHECK-NEXT:        .short  60319                   # 0xeb9f
 ; CHECK-NEXT:        .byte   1
 ; CHECK-NEXT:        .byte   0
-; CHECK-NEXT:        .long   40
+; CHECK-NEXT:        .long   32
 ; CHECK-NEXT:        .long   0
 ; CHECK-NEXT:        .long   20
 ; CHECK-NEXT:        .long   20
 ; CHECK-NEXT:        .long   28
 ; CHECK-NEXT:        .long   48
 ; CHECK-NEXT:        .long   0
-; CHECK-NEXT:        .long   48
-; CHECK-NEXT:        .long   0
 ; CHECK-NEXT:        .long   8                       # FuncInfo
 ; CHECK-NEXT:        .long   4                       # FuncInfo section string offset=4
 ; CHECK-NEXT:        .long   1
@@ -59,7 +57,7 @@ define dso_local void @f1() local_unnamed_addr #0 !dbg !7 {
 ; CHECK-NEXT:        .long   0
 ; CHECK-NEXT:        .long   1040                    # Line 1 Col 16
 
-attributes #0 = { norecurse nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { norecurse nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!3, !4, !5}
diff --git a/llvm/test/CodeGen/BPF/BTF/global-var-inited.ll b/llvm/test/CodeGen/BPF/BTF/global-var-inited.ll
index 18313b5a2..d5f745076 100644
--- a/llvm/test/CodeGen/BPF/BTF/global-var-inited.ll
+++ b/llvm/test/CodeGen/BPF/BTF/global-var-inited.ll
@@ -14,16 +14,30 @@
 ; CHECK-NEXT:        .byte   0
 ; CHECK-NEXT:        .long   24
 ; CHECK-NEXT:        .long   0
-; CHECK-NEXT:        .long   16
-; CHECK-NEXT:        .long   16
-; CHECK-NEXT:        .long   5
+; CHECK-NEXT:        .long   56
+; CHECK-NEXT:        .long   56
+; CHECK-NEXT:        .long   13
 ; CHECK-NEXT:        .long   1                       # BTF_KIND_INT(id = 1)
 ; CHECK-NEXT:        .long   16777216                # 0x1000000
 ; CHECK-NEXT:        .long   4
 ; CHECK-NEXT:        .long   16777248                # 0x1000020
+; CHECK-NEXT:        .long   5                       # BTF_KIND_VAR(id = 2)
+; CHECK-NEXT:        .long   234881024               # 0xe000000
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   7                       # BTF_KIND_DATASEC(id = 3)
+; CHECK-NEXT:        .long   251658241               # 0xf000001
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   a
+; CHECK-NEXT:        .long   4
 ; CHECK-NEXT:        .byte   0                       # string offset=0
 ; CHECK-NEXT:        .ascii  "int"                   # string offset=1
 ; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .byte   97                      # string offset=5
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  ".data"                 # string offset=7
+; CHECK-NEXT:        .byte   0
 
 !llvm.dbg.cu = !{!2}
 !llvm.module.flags = !{!7, !8, !9}
diff --git a/llvm/test/CodeGen/BPF/BTF/local-var.ll b/llvm/test/CodeGen/BPF/BTF/local-var.ll
index fa7f09f3f..88907b9fa 100644
--- a/llvm/test/CodeGen/BPF/BTF/local-var.ll
+++ b/llvm/test/CodeGen/BPF/BTF/local-var.ll
@@ -43,7 +43,7 @@ define dso_local i32 @foo(i8 signext) local_unnamed_addr #0 !dbg !7 {
 ; CHECK-NEXT:        .long   4
 ; CHECK-NEXT:        .long   16777248                # 0x1000020
 ; CHECK-NEXT:        .long   12                      # BTF_KIND_FUNC(id = 4)
-; CHECK-NEXT:        .long   201326592               # 0xc000000
+; CHECK-NEXT:        .long   201326593               # 0xc000001
 ; CHECK-NEXT:        .long   2
 ; CHECK-NEXT:        .byte   0                       # string offset=0
 ; CHECK-NEXT:        .ascii  "char"                  # string offset=1
@@ -71,7 +71,7 @@ declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture) #2
 ; Function Attrs: nounwind readnone speculatable
 declare void @llvm.dbg.value(metadata, metadata, metadata) #1
 
-attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone speculatable }
 attributes #2 = { argmemonly nounwind }
 
diff --git a/llvm/test/CodeGen/BPF/BTF/static-func.ll b/llvm/test/CodeGen/BPF/BTF/static-func.ll
new file mode 100644
index 000000000..9227cf066
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/BTF/static-func.ll
@@ -0,0 +1,96 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+;
+; Source code:
+;   extern int foo(void);
+;   static __attribute__((noinline)) int test1() { return foo(); }
+;   int test2() { return test1(); }
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm test.c
+
+; Function Attrs: nounwind
+define dso_local i32 @test2() local_unnamed_addr #0 !dbg !12 {
+entry:
+  %call = tail call fastcc i32 @test1(), !dbg !13
+  ret i32 %call, !dbg !14
+}
+; Function Attrs: noinline nounwind
+define internal fastcc i32 @test1() unnamed_addr #1 !dbg !15 {
+entry:
+  %call = tail call i32 @foo() #3, !dbg !16
+  ret i32 %call, !dbg !17
+}
+declare !dbg !4 dso_local i32 @foo() local_unnamed_addr #2
+
+; CHECK:             .section        .BTF,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   88
+; CHECK-NEXT:        .long   88
+; CHECK-NEXT:        .long   64
+; CHECK-NEXT:        .long   0                       # BTF_KIND_FUNC_PROTO(id = 1)
+; CHECK-NEXT:        .long   218103808               # 0xd000000
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   1                       # BTF_KIND_INT(id = 2)
+; CHECK-NEXT:        .long   16777216                # 0x1000000
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   16777248                # 0x1000020
+; CHECK-NEXT:        .long   5                       # BTF_KIND_FUNC(id = 3)
+; CHECK-NEXT:        .long   201326593               # 0xc000001
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   0                       # BTF_KIND_FUNC_PROTO(id = 4)
+; CHECK-NEXT:        .long   218103808               # 0xd000000
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   54                      # BTF_KIND_FUNC(id = 5)
+; CHECK-NEXT:        .long   201326592               # 0xc000000
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   0                       # BTF_KIND_FUNC_PROTO(id = 6)
+; CHECK-NEXT:        .long   218103808               # 0xd000000
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   60                      # BTF_KIND_FUNC(id = 7)
+; CHECK-NEXT:        .long   201326594               # 0xc000002
+; CHECK-NEXT:        .long   6
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "int"                   # string offset=1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "test2"                 # string offset=5
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  ".text"                 # string offset=11
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "/tmp/home/yhs/work/tests/bugs/test.c" # string offset=17
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "test1"                 # string offset=54
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "foo"                   # string offset=60
+; CHECK-NEXT:        .byte   0
+
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { noinline nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { nounwind }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!8, !9, !10}
+!llvm.ident = !{!11}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 10.0.0 (https://github.com/llvm/llvm-project.git 000f95c4157aee07bd4ffc3f59ffdb6c7ecae4af)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !3, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "test.c", directory: "/tmp/home/yhs/work/tests/bugs")
+!2 = !{}
+!3 = !{!4}
+!4 = !DISubprogram(name: "foo", scope: !1, file: !1, line: 1, type: !5, flags: DIFlagPrototyped, spFlags: DISPFlagOptimized, retainedNodes: !2)
+!5 = !DISubroutineType(types: !6)
+!6 = !{!7}
+!7 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!8 = !{i32 7, !"Dwarf Version", i32 4}
+!9 = !{i32 2, !"Debug Info Version", i32 3}
+!10 = !{i32 1, !"wchar_size", i32 4}
+!11 = !{!"clang version 10.0.0 (https://github.com/llvm/llvm-project.git 000f95c4157aee07bd4ffc3f59ffdb6c7ecae4af)"}
+!12 = distinct !DISubprogram(name: "test2", scope: !1, file: !1, line: 3, type: !5, scopeLine: 3, flags: DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2)
+!13 = !DILocation(line: 3, column: 22, scope: !12)
+!14 = !DILocation(line: 3, column: 15, scope: !12)
+!15 = distinct !DISubprogram(name: "test1", scope: !1, file: !1, line: 2, type: !5, scopeLine: 2, flags: DIFlagAllCallsDescribed, spFlags: DISPFlagLocalToUnit | DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2)
+!16 = !DILocation(line: 2, column: 55, scope: !15)
+!17 = !DILocation(line: 2, column: 48, scope: !15)
diff --git a/llvm/test/CodeGen/BPF/BTF/static-var-derived-type.ll b/llvm/test/CodeGen/BPF/BTF/static-var-derived-type.ll
index 680bcd636..1a1adc48c 100644
--- a/llvm/test/CodeGen/BPF/BTF/static-var-derived-type.ll
+++ b/llvm/test/CodeGen/BPF/BTF/static-var-derived-type.ll
@@ -51,7 +51,7 @@ define dso_local i64 @foo() local_unnamed_addr #0 !dbg !27 {
 ; CHECK-NEXT:        .long   8
 ; CHECK-NEXT:        .long   16777280                # 0x1000040
 ; CHECK-NEXT:        .long   10                      # BTF_KIND_FUNC(id = 3)
-; CHECK-NEXT:        .long   201326592               # 0xc000000
+; CHECK-NEXT:        .long   201326593               # 0xc000001
 ; CHECK-NEXT:        .long   1
 ; CHECK-NEXT:        .long   0                       # BTF_KIND_VOLATILE(id = 4)
 ; CHECK-NEXT:        .long   150994944               # 0x9000000
@@ -141,7 +141,7 @@ define dso_local i64 @foo() local_unnamed_addr #0 !dbg !27 {
 ; CHECK-NEXT:        .ascii  ".rodata"               # string offset=87
 ; CHECK-NEXT:        .byte   0
 
-attributes #0 = { norecurse nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { norecurse nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!2}
 !llvm.module.flags = !{!23, !24, !25}
diff --git a/llvm/test/CodeGen/BPF/BTF/static-var-inited-sec.ll b/llvm/test/CodeGen/BPF/BTF/static-var-inited-sec.ll
index 4533742b5..5ae2e99b9 100644
--- a/llvm/test/CodeGen/BPF/BTF/static-var-inited-sec.ll
+++ b/llvm/test/CodeGen/BPF/BTF/static-var-inited-sec.ll
@@ -40,7 +40,7 @@ define dso_local i32 @foo() local_unnamed_addr #0 !dbg !2 {
 ; CHECK-NEXT:        .long   4
 ; CHECK-NEXT:        .long   16777248                # 0x1000020
 ; CHECK-NEXT:        .long   5                       # BTF_KIND_FUNC(id = 3)
-; CHECK-NEXT:        .long   201326592               # 0xc000000
+; CHECK-NEXT:        .long   201326593               # 0xc000001
 ; CHECK-NEXT:        .long   1
 ; CHECK-NEXT:        .long   0                       # BTF_KIND_VOLATILE(id = 4)
 ; CHECK-NEXT:        .long   150994944               # 0x9000000
@@ -93,7 +93,7 @@ define dso_local i32 @foo() local_unnamed_addr #0 !dbg !2 {
 ; CHECK-NEXT:        .ascii  "maps"                  # string offset=71
 ; CHECK-NEXT:        .byte   0
 
-attributes #0 = { norecurse nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { norecurse nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!7}
 !llvm.module.flags = !{!16, !17, !18}
diff --git a/llvm/test/CodeGen/BPF/BTF/static-var-inited.ll b/llvm/test/CodeGen/BPF/BTF/static-var-inited.ll
index ed6e9eaf3..4cfcb0b81 100644
--- a/llvm/test/CodeGen/BPF/BTF/static-var-inited.ll
+++ b/llvm/test/CodeGen/BPF/BTF/static-var-inited.ll
@@ -40,7 +40,7 @@ define dso_local i32 @foo() local_unnamed_addr #0 !dbg !2 {
 ; CHECK-NEXT:        .long   4
 ; CHECK-NEXT:        .long   16777248                # 0x1000020
 ; CHECK-NEXT:        .long   5                       # BTF_KIND_FUNC(id = 3)
-; CHECK-NEXT:        .long   201326592               # 0xc000000
+; CHECK-NEXT:        .long   201326593               # 0xc000001
 ; CHECK-NEXT:        .long   1
 ; CHECK-NEXT:        .long   0                       # BTF_KIND_VOLATILE(id = 4)
 ; CHECK-NEXT:        .long   150994944               # 0x9000000
@@ -93,7 +93,7 @@ define dso_local i32 @foo() local_unnamed_addr #0 !dbg !2 {
 ; CHECK-NEXT:        .ascii  ".data"                 # string offset=71
 ; CHECK-NEXT:        .byte   0
 
-attributes #0 = { norecurse nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { norecurse nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!7}
 !llvm.module.flags = !{!16, !17, !18}
diff --git a/llvm/test/CodeGen/BPF/BTF/static-var-readonly-sec.ll b/llvm/test/CodeGen/BPF/BTF/static-var-readonly-sec.ll
index 5901bbb38..2ebc3723c 100644
--- a/llvm/test/CodeGen/BPF/BTF/static-var-readonly-sec.ll
+++ b/llvm/test/CodeGen/BPF/BTF/static-var-readonly-sec.ll
@@ -40,7 +40,7 @@ define dso_local i32 @foo() local_unnamed_addr #0 !dbg !2 {
 ; CHECK-NEXT:        .long   4
 ; CHECK-NEXT:        .long   16777248                # 0x1000020
 ; CHECK-NEXT:        .long   5                       # BTF_KIND_FUNC(id = 3)
-; CHECK-NEXT:        .long   201326592               # 0xc000000
+; CHECK-NEXT:        .long   201326593               # 0xc000001
 ; CHECK-NEXT:        .long   1
 ; CHECK-NEXT:        .long   0                       # BTF_KIND_CONST(id = 4)
 ; CHECK-NEXT:        .long   167772160               # 0xa000000
@@ -99,7 +99,7 @@ define dso_local i32 @foo() local_unnamed_addr #0 !dbg !2 {
 ; CHECK-NEXT:        .ascii  "maps"                  # string offset=71
 ; CHECK-NEXT:        .byte   0
 
-attributes #0 = { norecurse nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { norecurse nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!7}
 !llvm.module.flags = !{!18, !19, !20}
diff --git a/llvm/test/CodeGen/BPF/BTF/static-var-readonly.ll b/llvm/test/CodeGen/BPF/BTF/static-var-readonly.ll
index 04a3fbc5c..50812b1a4 100644
--- a/llvm/test/CodeGen/BPF/BTF/static-var-readonly.ll
+++ b/llvm/test/CodeGen/BPF/BTF/static-var-readonly.ll
@@ -40,7 +40,7 @@ define dso_local i32 @foo() local_unnamed_addr #0 !dbg !2 {
 ; CHECK-NEXT:        .long   4
 ; CHECK-NEXT:        .long   16777248                # 0x1000020
 ; CHECK-NEXT:        .long   5                       # BTF_KIND_FUNC(id = 3)
-; CHECK-NEXT:        .long   201326592               # 0xc000000
+; CHECK-NEXT:        .long   201326593               # 0xc000001
 ; CHECK-NEXT:        .long   1
 ; CHECK-NEXT:        .long   0                       # BTF_KIND_CONST(id = 4)
 ; CHECK-NEXT:        .long   167772160               # 0xa000000
@@ -99,7 +99,7 @@ define dso_local i32 @foo() local_unnamed_addr #0 !dbg !2 {
 ; CHECK-NEXT:        .ascii  ".rodata"               # string offset=71
 ; CHECK-NEXT:        .byte   0
 
-attributes #0 = { norecurse nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { norecurse nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!7}
 !llvm.module.flags = !{!18, !19, !20}
diff --git a/llvm/test/CodeGen/BPF/BTF/static-var-sec.ll b/llvm/test/CodeGen/BPF/BTF/static-var-sec.ll
index 6dcd4fefe..dc1318256 100644
--- a/llvm/test/CodeGen/BPF/BTF/static-var-sec.ll
+++ b/llvm/test/CodeGen/BPF/BTF/static-var-sec.ll
@@ -40,7 +40,7 @@ define dso_local i32 @foo() local_unnamed_addr #0 !dbg !2 {
 ; CHECK-NEXT:        .long   4
 ; CHECK-NEXT:        .long   16777248                # 0x1000020
 ; CHECK-NEXT:        .long   5                       # BTF_KIND_FUNC(id = 3)
-; CHECK-NEXT:        .long   201326592               # 0xc000000
+; CHECK-NEXT:        .long   201326593               # 0xc000001
 ; CHECK-NEXT:        .long   1
 ; CHECK-NEXT:        .long   0                       # BTF_KIND_VOLATILE(id = 4)
 ; CHECK-NEXT:        .long   150994944               # 0x9000000
@@ -93,7 +93,7 @@ define dso_local i32 @foo() local_unnamed_addr #0 !dbg !2 {
 ; CHECK-NEXT:        .ascii  "maps"                  # string offset=71
 ; CHECK-NEXT:        .byte   0
 
-attributes #0 = { norecurse nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { norecurse nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!7}
 !llvm.module.flags = !{!16, !17, !18}
diff --git a/llvm/test/CodeGen/BPF/BTF/static-var-zerolen-array.ll b/llvm/test/CodeGen/BPF/BTF/static-var-zerolen-array.ll
index 93f5da342..987a206e9 100644
--- a/llvm/test/CodeGen/BPF/BTF/static-var-zerolen-array.ll
+++ b/llvm/test/CodeGen/BPF/BTF/static-var-zerolen-array.ll
@@ -37,7 +37,7 @@ define dso_local i32 @test() local_unnamed_addr #0 !dbg !21 {
 ; CHECK-NEXT:        .long   4
 ; CHECK-NEXT:        .long   16777248                # 0x1000020
 ; CHECK-NEXT:        .long   5                       # BTF_KIND_FUNC(id = 3)
-; CHECK-NEXT:        .long   201326592               # 0xc000000
+; CHECK-NEXT:        .long   201326593               # 0xc000001
 ; CHECK-NEXT:        .long   1
 ; CHECK-NEXT:        .long   0                       # BTF_KIND_VOLATILE(id = 4)
 ; CHECK-NEXT:        .long   150994944               # 0x9000000
@@ -104,7 +104,7 @@ define dso_local i32 @test() local_unnamed_addr #0 !dbg !21 {
 ; CHECK-NEXT:        .ascii  ".data"                 # string offset=89
 ; CHECK-NEXT:        .byte   0
 
-attributes #0 = { norecurse nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { norecurse nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!2}
 !llvm.module.flags = !{!17, !18, !19}
diff --git a/llvm/test/CodeGen/BPF/BTF/static-var.ll b/llvm/test/CodeGen/BPF/BTF/static-var.ll
index 0eb05652d..6828a9d3f 100644
--- a/llvm/test/CodeGen/BPF/BTF/static-var.ll
+++ b/llvm/test/CodeGen/BPF/BTF/static-var.ll
@@ -40,7 +40,7 @@ define dso_local i32 @foo() local_unnamed_addr #0 !dbg !2 {
 ; CHECK-NEXT:        .long   4
 ; CHECK-NEXT:        .long   16777248                # 0x1000020
 ; CHECK-NEXT:        .long   5                       # BTF_KIND_FUNC(id = 3)
-; CHECK-NEXT:        .long   201326592               # 0xc000000
+; CHECK-NEXT:        .long   201326593               # 0xc000001
 ; CHECK-NEXT:        .long   1
 ; CHECK-NEXT:        .long   0                       # BTF_KIND_VOLATILE(id = 4)
 ; CHECK-NEXT:        .long   150994944               # 0x9000000
@@ -93,7 +93,7 @@ define dso_local i32 @foo() local_unnamed_addr #0 !dbg !2 {
 ; CHECK-NEXT:        .ascii  ".bss"                  # string offset=71
 ; CHECK-NEXT:        .byte   0
 
-attributes #0 = { norecurse nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { norecurse nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!7}
 !llvm.module.flags = !{!16, !17, !18}
diff --git a/llvm/test/CodeGen/BPF/BTF/weak-global-2.ll b/llvm/test/CodeGen/BPF/BTF/weak-global-2.ll
new file mode 100644
index 000000000..a3b187a7d
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/BTF/weak-global-2.ll
@@ -0,0 +1,66 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+;
+; Source code:
+;   char g __attribute__((weak)) = 2;
+;   int test() {
+;     return g;
+;   }
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm test.c
+
+@g = weak dso_local local_unnamed_addr global i8 2, align 1, !dbg !0
+; Function Attrs: norecurse nounwind readonly
+define dso_local i32 @test() local_unnamed_addr #0 !dbg !11 {
+entry:
+  %0 = load i8, i8* @g, align 1, !dbg !15, !tbaa !16
+  %conv = sext i8 %0 to i32, !dbg !15
+  ret i32 %conv, !dbg !19
+}
+
+; CHECK:             .long   55                      # BTF_KIND_INT(id = 4)
+; CHECK-NEXT:        .long   16777216                # 0x1000000
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   16777224                # 0x1000008
+; CHECK-NEXT:        .long   60                      # BTF_KIND_VAR(id = 5)
+; CHECK-NEXT:        .long   234881024               # 0xe000000
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   62                      # BTF_KIND_DATASEC(id = 6)
+; CHECK-NEXT:        .long   251658241               # 0xf000001
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   5
+; CHECK-NEXT:        .long   g
+; CHECK-NEXT:        .long   1
+
+; CHECK:             .ascii  "char"                  # string offset=55
+; CHECK:             .byte   103                     # string offset=60
+; CHECK:             .ascii  ".data"                 # string offset=62
+
+
+attributes #0 = { norecurse nounwind readonly "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!7, !8, !9}
+!llvm.ident = !{!10}
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "g", scope: !2, file: !3, line: 1, type: !6, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 10.0.0 (https://github.com/llvm/llvm-project.git edf6717d8d30034da932b95350898e03c90a5082)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5, nameTableKind: None)
+!3 = !DIFile(filename: "test.c", directory: "/tmp/home/yhs/work/tests/global")
+!4 = !{}
+!5 = !{!0}
+!6 = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_signed_char)
+!7 = !{i32 7, !"Dwarf Version", i32 4}
+!8 = !{i32 2, !"Debug Info Version", i32 3}
+!9 = !{i32 1, !"wchar_size", i32 4}
+!10 = !{!"clang version 10.0.0 (https://github.com/llvm/llvm-project.git edf6717d8d30034da932b95350898e03c90a5082)"}
+!11 = distinct !DISubprogram(name: "test", scope: !3, file: !3, line: 2, type: !12, scopeLine: 2, flags: DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !2, retainedNodes: !4)
+!12 = !DISubroutineType(types: !13)
+!13 = !{!14}
+!14 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!15 = !DILocation(line: 3, column: 10, scope: !11)
+!16 = !{!17, !17, i64 0}
+!17 = !{!"omnipotent char", !18, i64 0}
+!18 = !{!"Simple C/C++ TBAA"}
+!19 = !DILocation(line: 3, column: 3, scope: !11)
diff --git a/llvm/test/CodeGen/BPF/BTF/weak-global.ll b/llvm/test/CodeGen/BPF/BTF/weak-global.ll
new file mode 100644
index 000000000..79e62f8b9
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/BTF/weak-global.ll
@@ -0,0 +1,65 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+;
+; Source code:
+;   char g __attribute__((weak));
+;   int test() {
+;     return g;
+;   }
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm test.c
+
+@g = weak dso_local local_unnamed_addr global i8 0, align 1, !dbg !0
+; Function Attrs: norecurse nounwind readonly
+define dso_local i32 @test() local_unnamed_addr #0 !dbg !11 {
+entry:
+  %0 = load i8, i8* @g, align 1, !dbg !15, !tbaa !16
+  %conv = sext i8 %0 to i32, !dbg !15
+  ret i32 %conv, !dbg !19
+}
+
+; CHECK:             .long   55                      # BTF_KIND_INT(id = 4)
+; CHECK-NEXT:        .long   16777216                # 0x1000000
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   16777224                # 0x1000008
+; CHECK-NEXT:        .long   60                      # BTF_KIND_VAR(id = 5)
+; CHECK-NEXT:        .long   234881024               # 0xe000000
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   62                      # BTF_KIND_DATASEC(id = 6)
+; CHECK-NEXT:        .long   251658241               # 0xf000001
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   5
+; CHECK-NEXT:        .long   g
+; CHECK-NEXT:        .long   1
+
+; CHECK:             .ascii  "char"                  # string offset=55
+; CHECK:             .byte   103                     # string offset=60
+; CHECK:             .ascii  ".bss"                  # string offset=62
+
+attributes #0 = { norecurse nounwind readonly "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!7, !8, !9}
+!llvm.ident = !{!10}
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "g", scope: !2, file: !3, line: 1, type: !6, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 10.0.0 (https://github.com/llvm/llvm-project.git edf6717d8d30034da932b95350898e03c90a5082)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5, nameTableKind: None)
+!3 = !DIFile(filename: "test.c", directory: "/tmp/home/yhs/work/tests/global")
+!4 = !{}
+!5 = !{!0}
+!6 = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_signed_char)
+!7 = !{i32 7, !"Dwarf Version", i32 4}
+!8 = !{i32 2, !"Debug Info Version", i32 3}
+!9 = !{i32 1, !"wchar_size", i32 4}
+!10 = !{!"clang version 10.0.0 (https://github.com/llvm/llvm-project.git edf6717d8d30034da932b95350898e03c90a5082)"}
+!11 = distinct !DISubprogram(name: "test", scope: !3, file: !3, line: 2, type: !12, scopeLine: 2, flags: DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !2, retainedNodes: !4)
+!12 = !DISubroutineType(types: !13)
+!13 = !{!14}
+!14 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!15 = !DILocation(line: 3, column: 10, scope: !11)
+!16 = !{!17, !17, i64 0}
+!17 = !{!"omnipotent char", !18, i64 0}
+!18 = !{!"Simple C/C++ TBAA"}
+!19 = !DILocation(line: 3, column: 3, scope: !11)
diff --git a/llvm/test/CodeGen/BPF/CORE/field-reloc-alu32.ll b/llvm/test/CodeGen/BPF/CORE/field-reloc-alu32.ll
new file mode 100644
index 000000000..c915fef6b
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/CORE/field-reloc-alu32.ll
@@ -0,0 +1,73 @@
+; RUN: llc -march=bpfel -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; Source code:
+;   struct b { int d; int e; } c;
+;   int f() {
+;     return __builtin_preserve_field_info(c.e, 0);
+;   }
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm test.c
+
+%struct.b = type { i32, i32 }
+
+@c = common dso_local global %struct.b zeroinitializer, align 4, !dbg !0
+
+; Function Attrs: nounwind readnone
+define dso_local i32 @f() local_unnamed_addr #0 !dbg !15 {
+entry:
+  %0 = tail call i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.bs(%struct.b* nonnull @c, i32 1, i32 1), !dbg !18, !llvm.preserve.access.index !6
+  %1 = tail call i32 @llvm.bpf.preserve.field.info.p0i32(i32* %0, i64 0), !dbg !19
+  ret i32 %1, !dbg !20
+}
+
+; CHECK:             r0 = 4
+; CHECK:             exit
+
+; CHECK:             .long   13                      # BTF_KIND_STRUCT(id = 4)
+
+; CHECK:             .ascii  ".text"                 # string offset=7
+; CHECK:             .byte   98                      # string offset=13
+; CHECK:             .ascii  "0:1"                   # string offset=19
+
+; CHECK:             .long   16                      # FieldReloc
+; CHECK-NEXT:        .long   7                       # Field reloc section string offset=7
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   19
+; CHECK-NEXT:        .long   0
+
+; Function Attrs: nounwind readnone
+declare i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.bs(%struct.b*, i32, i32) #1
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.bpf.preserve.field.info.p0i32(i32*, i64) #1
+
+attributes #0 = { nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone }
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!11, !12, !13}
+!llvm.ident = !{!14}
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "c", scope: !2, file: !3, line: 1, type: !6, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 10.0.0 (https://github.com/llvm/llvm-project.git d5509db439a1bb3f0822c42398c8b5921a665478)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5, nameTableKind: None)
+!3 = !DIFile(filename: "test.c", directory: "/tmp/home/yhs/work/tests/core")
+!4 = !{}
+!5 = !{!0}
+!6 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "b", file: !3, line: 1, size: 64, elements: !7)
+!7 = !{!8, !10}
+!8 = !DIDerivedType(tag: DW_TAG_member, name: "d", scope: !6, file: !3, line: 1, baseType: !9, size: 32)
+!9 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!10 = !DIDerivedType(tag: DW_TAG_member, name: "e", scope: !6, file: !3, line: 1, baseType: !9, size: 32, offset: 32)
+!11 = !{i32 2, !"Dwarf Version", i32 4}
+!12 = !{i32 2, !"Debug Info Version", i32 3}
+!13 = !{i32 1, !"wchar_size", i32 4}
+!14 = !{!"clang version 10.0.0 (https://github.com/llvm/llvm-project.git d5509db439a1bb3f0822c42398c8b5921a665478)"}
+!15 = distinct !DISubprogram(name: "f", scope: !3, file: !3, line: 2, type: !16, scopeLine: 2, isDefinition: true, isOptimized: true, unit: !2, retainedNodes: !4)
+!16 = !DISubroutineType(types: !17)
+!17 = !{!9}
+!18 = !DILocation(line: 3, column: 42, scope: !15)
+!19 = !DILocation(line: 3, column: 10, scope: !15)
+!20 = !DILocation(line: 3, column: 3, scope: !15)
diff --git a/llvm/test/CodeGen/BPF/CORE/field-reloc-bitfield-1.ll b/llvm/test/CodeGen/BPF/CORE/field-reloc-bitfield-1.ll
new file mode 100644
index 000000000..4bec7f3b7
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/CORE/field-reloc-bitfield-1.ll
@@ -0,0 +1,126 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-EL,CHECK-ALU64 %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-EB,CHECK-ALU64 %s
+; RUN: llc -march=bpfel -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-EL,CHECK-ALU32 %s
+; RUN: llc -march=bpfeb -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-EB,CHECK-ALU32 %s
+; Source code:
+;   struct s {
+;     unsigned long long f1;
+;     unsigned f2;
+;     unsigned f3;
+;     unsigned f4;
+;     unsigned char f5;
+;     unsigned bf1:5,
+;              bf2:1;
+;   };
+;   enum {FIELD_TYPE_OFFSET = 0, FIELD_TYPE_SIZE = 1, FIELD_TYPE_LSHIFT_U64 = 4,};
+;   int test(struct s *arg) {
+;     return __builtin_preserve_field_info(arg->bf2, FIELD_TYPE_OFFSET) +
+;            __builtin_preserve_field_info(arg->bf2, FIELD_TYPE_SIZE) +
+;            __builtin_preserve_field_info(arg->bf2, FIELD_TYPE_LSHIFT_U64);
+;   }
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm test.c
+
+%struct.s = type { i64, i32, i32, i32, i8, i8 }
+
+; Function Attrs: nounwind readnone
+define dso_local i32 @test(%struct.s* %arg) local_unnamed_addr #0 !dbg !13 {
+entry:
+  call void @llvm.dbg.value(metadata %struct.s* %arg, metadata !30, metadata !DIExpression()), !dbg !31
+  %0 = tail call i8* @llvm.preserve.struct.access.index.p0i8.p0s_struct.ss(%struct.s* %arg, i32 5, i32 6), !dbg !32, !llvm.preserve.access.index !18
+  %1 = tail call i32 @llvm.bpf.preserve.field.info.p0i8(i8* %0, i64 0), !dbg !33
+  %2 = tail call i32 @llvm.bpf.preserve.field.info.p0i8(i8* %0, i64 1), !dbg !34
+  %add = add i32 %2, %1, !dbg !35
+  %3 = tail call i32 @llvm.bpf.preserve.field.info.p0i8(i8* %0, i64 4), !dbg !36
+  %add1 = add i32 %add, %3, !dbg !37
+  ret i32 %add1, !dbg !38
+}
+
+; CHECK:             r1 = 16
+; CHECK:             r0 = 8
+; CHECK-ALU64:       r0 += r1
+; CHECK-ALU32:       w0 += w1
+; CHECK-EL:          r1 = 18
+; CHECK-EB:          r1 = 45
+; CHECK-ALU64:       r0 += r1
+; CHECK-ALU32:       w0 += w1
+; CHECK:             exit
+
+; CHECK:             .long   1                       # BTF_KIND_STRUCT(id = 2)
+
+; CHECK:             .byte   115                     # string offset=1
+; CHECK:             .ascii  ".text"                 # string offset=89
+; CHECK:             .ascii  "0:6"                   # string offset=95
+
+; CHECK:             .long   16                      # FieldReloc
+; CHECK-NEXT:        .long   89                      # Field reloc section string offset=89
+; CHECK-NEXT:        .long   3
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   95
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   95
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   95
+; CHECK-NEXT:        .long   4
+
+; Function Attrs: nounwind readnone
+declare i8* @llvm.preserve.struct.access.index.p0i8.p0s_struct.ss(%struct.s*, i32, i32) #1
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.bpf.preserve.field.info.p0i8(i8*, i64) #1
+
+; Function Attrs: nounwind readnone speculatable
+declare void @llvm.dbg.value(metadata, metadata, metadata) #2
+
+attributes #0 = { nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone }
+attributes #2 = { nounwind readnone speculatable }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!9, !10, !11}
+!llvm.ident = !{!12}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 10.0.0 (https://github.com/llvm/llvm-project.git bc6913e314806882e2b537b5b03996800078d2ad)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "test.c", directory: "/tmp/home/yhs/work/tests/core")
+!2 = !{!3}
+!3 = !DICompositeType(tag: DW_TAG_enumeration_type, file: !1, line: 10, baseType: !4, size: 32, elements: !5)
+!4 = !DIBasicType(name: "unsigned int", size: 32, encoding: DW_ATE_unsigned)
+!5 = !{!6, !7, !8}
+!6 = !DIEnumerator(name: "FIELD_TYPE_OFFSET", value: 0, isUnsigned: true)
+!7 = !DIEnumerator(name: "FIELD_TYPE_SIZE", value: 1, isUnsigned: true)
+!8 = !DIEnumerator(name: "FIELD_TYPE_LSHIFT_U64", value: 4, isUnsigned: true)
+!9 = !{i32 2, !"Dwarf Version", i32 4}
+!10 = !{i32 2, !"Debug Info Version", i32 3}
+!11 = !{i32 1, !"wchar_size", i32 4}
+!12 = !{!"clang version 10.0.0 (https://github.com/llvm/llvm-project.git bc6913e314806882e2b537b5b03996800078d2ad)"}
+!13 = distinct !DISubprogram(name: "test", scope: !1, file: !1, line: 11, type: !14, scopeLine: 11, flags: DIFlagPrototyped, isDefinition: true, isOptimized: true, unit: !0, retainedNodes: !29)
+!14 = !DISubroutineType(types: !15)
+!15 = !{!16, !17}
+!16 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!17 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !18, size: 64)
+!18 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "s", file: !1, line: 1, size: 192, elements: !19)
+!19 = !{!20, !22, !23, !24, !25, !27, !28}
+!20 = !DIDerivedType(tag: DW_TAG_member, name: "f1", scope: !18, file: !1, line: 2, baseType: !21, size: 64)
+!21 = !DIBasicType(name: "long long unsigned int", size: 64, encoding: DW_ATE_unsigned)
+!22 = !DIDerivedType(tag: DW_TAG_member, name: "f2", scope: !18, file: !1, line: 3, baseType: !4, size: 32, offset: 64)
+!23 = !DIDerivedType(tag: DW_TAG_member, name: "f3", scope: !18, file: !1, line: 4, baseType: !4, size: 32, offset: 96)
+!24 = !DIDerivedType(tag: DW_TAG_member, name: "f4", scope: !18, file: !1, line: 5, baseType: !4, size: 32, offset: 128)
+!25 = !DIDerivedType(tag: DW_TAG_member, name: "f5", scope: !18, file: !1, line: 6, baseType: !26, size: 8, offset: 160)
+!26 = !DIBasicType(name: "unsigned char", size: 8, encoding: DW_ATE_unsigned_char)
+!27 = !DIDerivedType(tag: DW_TAG_member, name: "bf1", scope: !18, file: !1, line: 7, baseType: !4, size: 5, offset: 168, flags: DIFlagBitField, extraData: i64 168)
+!28 = !DIDerivedType(tag: DW_TAG_member, name: "bf2", scope: !18, file: !1, line: 8, baseType: !4, size: 1, offset: 173, flags: DIFlagBitField, extraData: i64 168)
+!29 = !{!30}
+!30 = !DILocalVariable(name: "arg", arg: 1, scope: !13, file: !1, line: 11, type: !17)
+!31 = !DILocation(line: 0, scope: !13)
+!32 = !DILocation(line: 12, column: 45, scope: !13)
+!33 = !DILocation(line: 12, column: 10, scope: !13)
+!34 = !DILocation(line: 13, column: 10, scope: !13)
+!35 = !DILocation(line: 12, column: 69, scope: !13)
+!36 = !DILocation(line: 14, column: 10, scope: !13)
+!37 = !DILocation(line: 13, column: 67, scope: !13)
+!38 = !DILocation(line: 12, column: 3, scope: !13)
diff --git a/llvm/test/CodeGen/BPF/CORE/field-reloc-bitfield-2.ll b/llvm/test/CodeGen/BPF/CORE/field-reloc-bitfield-2.ll
new file mode 100644
index 000000000..14ffa8c47
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/CORE/field-reloc-bitfield-2.ll
@@ -0,0 +1,124 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-EL,CHECK-ALU64 %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-EB,CHECK-ALU64 %s
+; RUN: llc -march=bpfel -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-EL,CHECK-ALU32 %s
+; RUN: llc -march=bpfeb -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-EB,CHECK-ALU32 %s
+; Source code:
+;   struct s {
+;     char f1;
+;     char bf1:6,
+;          bf2:2,
+;          bf3:5,
+;          bf4:3;
+;   };
+;   enum {FIELD_TYPE_OFFSET = 0, FIELD_TYPE_SIZE = 1, FIELD_TYPE_LSHIFT_U64 = 4,};
+;   int test(struct s *arg) {
+;     return __builtin_preserve_field_info(arg->bf4, FIELD_TYPE_OFFSET) +
+;            __builtin_preserve_field_info(arg->bf4, FIELD_TYPE_SIZE) +
+;            __builtin_preserve_field_info(arg->bf4, FIELD_TYPE_LSHIFT_U64);
+;   }
+; For this case, the IR type has the same starting storage offset for fields
+; bf1, bf1, bf3 and bf4 and the ABI alignment is 1 byte. So for bf4 access,
+; the starting offset has to be at the beginning of field bf3.
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm test.c
+
+%struct.s = type <{ i8, i16 }>
+
+; Function Attrs: nounwind readnone
+define dso_local i32 @test(%struct.s* %arg) local_unnamed_addr #0 !dbg !13 {
+entry:
+  call void @llvm.dbg.value(metadata %struct.s* %arg, metadata !27, metadata !DIExpression()), !dbg !28
+  %0 = tail call i16* @llvm.preserve.struct.access.index.p0i16.p0s_struct.ss(%struct.s* %arg, i32 1, i32 4), !dbg !29, !llvm.preserve.access.index !18
+  %1 = tail call i32 @llvm.bpf.preserve.field.info.p0i16(i16* %0, i64 0), !dbg !30
+  %2 = tail call i32 @llvm.bpf.preserve.field.info.p0i16(i16* %0, i64 1), !dbg !31
+  %add = add i32 %2, %1, !dbg !32
+  %3 = tail call i32 @llvm.bpf.preserve.field.info.p0i16(i16* %0, i64 4), !dbg !33
+  %add1 = add i32 %add, %3, !dbg !34
+  ret i32 %add1, !dbg !35
+}
+
+; CHECK:             r1 = 2
+; CHECK:             r0 = 1
+; CHECK-ALU64:       r0 += r1
+; CHECK-ALU32:       w0 += w1
+; CHECK-EL:          r1 = 56
+; CHECK-EB:          r1 = 61
+; CHECK-ALU64:       r0 += r1
+; CHECK-ALU32:       w0 += w1
+; CHECK:             exit
+
+; CHECK:             .long   1                       # BTF_KIND_STRUCT(id = 2)
+
+; CHECK:             .byte   115                     # string offset=1
+; CHECK:             .ascii  ".text"                 # string offset=40
+; CHECK:             .ascii  "0:4"                   # string offset=46
+
+; CHECK:             .long   16                      # FieldReloc
+; CHECK-NEXT:        .long   40                      # Field reloc section string offset=40
+; CHECK-NEXT:        .long   3
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   46
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   46
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   46
+; CHECK-NEXT:        .long   4
+
+; Function Attrs: nounwind readnone
+declare i16* @llvm.preserve.struct.access.index.p0i16.p0s_struct.ss(%struct.s*, i32, i32) #1
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.bpf.preserve.field.info.p0i16(i16*, i64) #1
+
+; Function Attrs: nounwind readnone speculatable
+declare void @llvm.dbg.value(metadata, metadata, metadata) #2
+
+attributes #0 = { nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone }
+attributes #2 = { nounwind readnone speculatable }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!9, !10, !11}
+!llvm.ident = !{!12}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 10.0.0 (https://github.com/llvm/llvm-project.git 630ca91834ecc06349cb3b4bd2982c1b85b5ad96)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "test.c", directory: "/tmp/home/yhs/work/tests/core")
+!2 = !{!3}
+!3 = !DICompositeType(tag: DW_TAG_enumeration_type, file: !1, line: 8, baseType: !4, size: 32, elements: !5)
+!4 = !DIBasicType(name: "unsigned int", size: 32, encoding: DW_ATE_unsigned)
+!5 = !{!6, !7, !8}
+!6 = !DIEnumerator(name: "FIELD_TYPE_OFFSET", value: 0, isUnsigned: true)
+!7 = !DIEnumerator(name: "FIELD_TYPE_SIZE", value: 1, isUnsigned: true)
+!8 = !DIEnumerator(name: "FIELD_TYPE_LSHIFT_U64", value: 4, isUnsigned: true)
+!9 = !{i32 2, !"Dwarf Version", i32 4}
+!10 = !{i32 2, !"Debug Info Version", i32 3}
+!11 = !{i32 1, !"wchar_size", i32 4}
+!12 = !{!"clang version 10.0.0 (https://github.com/llvm/llvm-project.git 630ca91834ecc06349cb3b4bd2982c1b85b5ad96)"}
+!13 = distinct !DISubprogram(name: "test", scope: !1, file: !1, line: 9, type: !14, scopeLine: 9, flags: DIFlagPrototyped, isDefinition: true, isOptimized: true, unit: !0, retainedNodes: !26)
+!14 = !DISubroutineType(types: !15)
+!15 = !{!16, !17}
+!16 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!17 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !18, size: 64)
+!18 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "s", file: !1, line: 1, size: 24, elements: !19)
+!19 = !{!20, !22, !23, !24, !25}
+!20 = !DIDerivedType(tag: DW_TAG_member, name: "f1", scope: !18, file: !1, line: 2, baseType: !21, size: 8)
+!21 = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_signed_char)
+!22 = !DIDerivedType(tag: DW_TAG_member, name: "bf1", scope: !18, file: !1, line: 3, baseType: !21, size: 6, offset: 8, flags: DIFlagBitField, extraData: i64 8)
+!23 = !DIDerivedType(tag: DW_TAG_member, name: "bf2", scope: !18, file: !1, line: 4, baseType: !21, size: 2, offset: 14, flags: DIFlagBitField, extraData: i64 8)
+!24 = !DIDerivedType(tag: DW_TAG_member, name: "bf3", scope: !18, file: !1, line: 5, baseType: !21, size: 5, offset: 16, flags: DIFlagBitField, extraData: i64 8)
+!25 = !DIDerivedType(tag: DW_TAG_member, name: "bf4", scope: !18, file: !1, line: 6, baseType: !21, size: 3, offset: 21, flags: DIFlagBitField, extraData: i64 8)
+!26 = !{!27}
+!27 = !DILocalVariable(name: "arg", arg: 1, scope: !13, file: !1, line: 9, type: !17)
+!28 = !DILocation(line: 0, scope: !13)
+!29 = !DILocation(line: 10, column: 45, scope: !13)
+!30 = !DILocation(line: 10, column: 10, scope: !13)
+!31 = !DILocation(line: 11, column: 10, scope: !13)
+!32 = !DILocation(line: 10, column: 69, scope: !13)
+!33 = !DILocation(line: 12, column: 10, scope: !13)
+!34 = !DILocation(line: 11, column: 67, scope: !13)
+!35 = !DILocation(line: 10, column: 3, scope: !13)
diff --git a/llvm/test/CodeGen/BPF/CORE/intrinsic-array.ll b/llvm/test/CodeGen/BPF/CORE/intrinsic-array.ll
index adbcb9fd7..77b340bd1 100644
--- a/llvm/test/CodeGen/BPF/CORE/intrinsic-array.ll
+++ b/llvm/test/CodeGen/BPF/CORE/intrinsic-array.ll
@@ -1,4 +1,5 @@
 ; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck %s
+; RUN: llc -march=bpfel -mattr=+alu32 -filetype=asm -o - %s | FileCheck %s
 ;
 ; Source code:
 ;   #define _(x) (__builtin_preserve_access_index(x))
@@ -14,7 +15,7 @@
 define dso_local i32 @test(%struct.s* %arg) local_unnamed_addr #0 !dbg !7 {
 entry:
   call void @llvm.dbg.value(metadata %struct.s* %arg, metadata !17, metadata !DIExpression()), !dbg !18
-  %0 = tail call %struct.s* @llvm.preserve.array.access.index.p0s_struct.ss.p0s_struct.ss(%struct.s* %arg, i32 0, i32 2), !dbg !19
+  %0 = tail call %struct.s* @llvm.preserve.array.access.index.p0s_struct.ss.p0s_struct.ss(%struct.s* %arg, i32 0, i32 2), !dbg !19, !llvm.preserve.access.index !11
   %1 = tail call i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.ss(%struct.s* %0, i32 1, i32 1), !dbg !19, !llvm.preserve.access.index !12
   %2 = bitcast i32* %1 to i8*, !dbg !19
   %call = tail call i32 @get_value(i8* %2) #4, !dbg !20
@@ -28,12 +29,13 @@ entry:
 ; CHECK:       exit
 ;
 ; CHECK:      .section        .BTF.ext,"",@progbits
-; CHECK:      .long   12                      # OffsetReloc
-; CHECK-NEXT: .long   20                      # Offset reloc section string offset=20
+; CHECK:      .long   16                      # FieldReloc
+; CHECK-NEXT: .long   20                      # Field reloc section string offset=20
 ; CHECK-NEXT: .long   1
 ; CHECK-NEXT: .long   [[RELOC]]
 ; CHECK-NEXT: .long   2
 ; CHECK-NEXT: .long   26
+; CHECK-NEXT: .long   0
 
 declare dso_local i32 @get_value(i8*) local_unnamed_addr #1
 
@@ -46,8 +48,8 @@ declare i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.ss(%struct.s*,
 ; Function Attrs: nounwind readnone speculatable
 declare void @llvm.dbg.value(metadata, metadata, metadata) #3
 
-attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind readnone }
 attributes #3 = { nounwind readnone speculatable }
 attributes #4 = { nounwind }
diff --git a/llvm/test/CodeGen/BPF/CORE/intrinsic-fieldinfo-byte-size-1.ll b/llvm/test/CodeGen/BPF/CORE/intrinsic-fieldinfo-byte-size-1.ll
new file mode 100644
index 000000000..b65bb0e5e
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/CORE/intrinsic-fieldinfo-byte-size-1.ll
@@ -0,0 +1,153 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-ALU64 %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-ALU64 %s
+; RUN: llc -march=bpfel -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-ALU32 %s
+; RUN: llc -march=bpfeb -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-ALU32 %s
+; Source code:
+;   typedef struct s1 { int a1:7; int a2:4; int a3:5; int a4:16;} __s1;
+;   union u1 { int b1; __s1 b2; };
+;   enum { FIELD_BYTE_SIZE = 1, };
+;   int test(union u1 *arg) {
+;     unsigned r1 = __builtin_preserve_field_info(arg->b2.a1, FIELD_BYTE_SIZE);
+;     unsigned r2 = __builtin_preserve_field_info(arg->b2.a2, FIELD_BYTE_SIZE);
+;     unsigned r3 = __builtin_preserve_field_info(arg->b2.a3, FIELD_BYTE_SIZE);
+;     unsigned r4 = __builtin_preserve_field_info(arg->b2.a4, FIELD_BYTE_SIZE);
+;     /* r1: 4, r2: 4, r3: 4, r4: 4 */
+;     return r1 + r2 + r3 + r4;
+;   }
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm test.c
+
+%union.u1 = type { i32 }
+%struct.s1 = type { i32 }
+
+; Function Attrs: nounwind readnone
+define dso_local i32 @test(%union.u1* %arg) local_unnamed_addr #0 !dbg !11 {
+entry:
+  call void @llvm.dbg.value(metadata %union.u1* %arg, metadata !28, metadata !DIExpression()), !dbg !33
+  %0 = tail call %union.u1* @llvm.preserve.union.access.index.p0s_union.u1s.p0s_union.u1s(%union.u1* %arg, i32 1), !dbg !34, !llvm.preserve.access.index !16
+  %b2 = bitcast %union.u1* %0 to %struct.s1*, !dbg !34
+  %1 = tail call i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.s1s(%struct.s1* %b2, i32 0, i32 0), !dbg !35, !llvm.preserve.access.index !21
+  %2 = tail call i32 @llvm.bpf.preserve.field.info.p0i32(i32* %1, i64 1), !dbg !36
+  call void @llvm.dbg.value(metadata i32 %2, metadata !29, metadata !DIExpression()), !dbg !33
+  %3 = tail call i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.s1s(%struct.s1* %b2, i32 0, i32 1), !dbg !37, !llvm.preserve.access.index !21
+  %4 = tail call i32 @llvm.bpf.preserve.field.info.p0i32(i32* %3, i64 1), !dbg !38
+  call void @llvm.dbg.value(metadata i32 %4, metadata !30, metadata !DIExpression()), !dbg !33
+  %5 = tail call i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.s1s(%struct.s1* %b2, i32 0, i32 2), !dbg !39, !llvm.preserve.access.index !21
+  %6 = tail call i32 @llvm.bpf.preserve.field.info.p0i32(i32* %5, i64 1), !dbg !40
+  call void @llvm.dbg.value(metadata i32 %6, metadata !31, metadata !DIExpression()), !dbg !33
+  %7 = tail call i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.s1s(%struct.s1* %b2, i32 0, i32 3), !dbg !41, !llvm.preserve.access.index !21
+  %8 = tail call i32 @llvm.bpf.preserve.field.info.p0i32(i32* %7, i64 1), !dbg !42
+  call void @llvm.dbg.value(metadata i32 %8, metadata !32, metadata !DIExpression()), !dbg !33
+  %add = add i32 %4, %2, !dbg !43
+  %add4 = add i32 %add, %6, !dbg !44
+  %add5 = add i32 %add4, %8, !dbg !45
+  ret i32 %add5, !dbg !46
+}
+
+; CHECK:             r1 = 4
+; CHECK:             r0 = 4
+; CHECK-ALU64:       r0 += r1
+; CHECK-ALU32:       w0 += w1
+; CHECK:             r1 = 4
+; CHECK-ALU64:       r0 += r1
+; CHECK-ALU32:       w0 += w1
+; CHECK:             r1 = 4
+; CHECK-ALU64:       r0 += r1
+; CHECK-ALU32:       w0 += w1
+; CHECK:             exit
+
+; CHECK:             .long   1                       # BTF_KIND_UNION(id = 2)
+; CHECK:             .ascii  "u1"                    # string offset=1
+; CHECK:             .ascii  ".text"                 # string offset=43
+; CHECK:             .ascii  "0:1:0"                 # string offset=49
+; CHECK:             .ascii  "0:1:1"                 # string offset=92
+; CHECK:             .ascii  "0:1:2"                 # string offset=98
+; CHECK:             .ascii  "0:1:3"                 # string offset=104
+
+; CHECK:             .long   16                      # FieldReloc
+; CHECK-NEXT:        .long   43                      # Field reloc section string offset=43
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   49
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   92
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   98
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   104
+; CHECK-NEXT:        .long   1
+
+; Function Attrs: nounwind readnone
+declare %union.u1* @llvm.preserve.union.access.index.p0s_union.u1s.p0s_union.u1s(%union.u1*, i32) #1
+
+; Function Attrs: nounwind readnone
+declare i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.s1s(%struct.s1*, i32, i32) #1
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.bpf.preserve.field.info.p0i32(i32*, i64) #1
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.value(metadata, metadata, metadata) #2
+
+attributes #0 = { nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone }
+attributes #2 = { nounwind readnone speculatable willreturn }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!7, !8, !9}
+!llvm.ident = !{!10}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 10.0.0 (https://github.com/llvm/llvm-project.git 4a60741b74384f14b21fdc0131ede326438840ab)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "test.c", directory: "/tmp/home/yhs/work/tests/core")
+!2 = !{!3}
+!3 = !DICompositeType(tag: DW_TAG_enumeration_type, file: !1, line: 3, baseType: !4, size: 32, elements: !5)
+!4 = !DIBasicType(name: "unsigned int", size: 32, encoding: DW_ATE_unsigned)
+!5 = !{!6}
+!6 = !DIEnumerator(name: "FIELD_BYTE_SIZE", value: 1, isUnsigned: true)
+!7 = !{i32 2, !"Dwarf Version", i32 4}
+!8 = !{i32 2, !"Debug Info Version", i32 3}
+!9 = !{i32 1, !"wchar_size", i32 4}
+!10 = !{!"clang version 10.0.0 (https://github.com/llvm/llvm-project.git 4a60741b74384f14b21fdc0131ede326438840ab)"}
+!11 = distinct !DISubprogram(name: "test", scope: !1, file: !1, line: 4, type: !12, scopeLine: 4, flags: DIFlagPrototyped, isDefinition: true, isOptimized: true, unit: !0, retainedNodes: !27)
+!12 = !DISubroutineType(types: !13)
+!13 = !{!14, !15}
+!14 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!15 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !16, size: 64)
+!16 = distinct !DICompositeType(tag: DW_TAG_union_type, name: "u1", file: !1, line: 2, size: 32, elements: !17)
+!17 = !{!18, !19}
+!18 = !DIDerivedType(tag: DW_TAG_member, name: "b1", scope: !16, file: !1, line: 2, baseType: !14, size: 32)
+!19 = !DIDerivedType(tag: DW_TAG_member, name: "b2", scope: !16, file: !1, line: 2, baseType: !20, size: 32)
+!20 = !DIDerivedType(tag: DW_TAG_typedef, name: "__s1", file: !1, line: 1, baseType: !21)
+!21 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "s1", file: !1, line: 1, size: 32, elements: !22)
+!22 = !{!23, !24, !25, !26}
+!23 = !DIDerivedType(tag: DW_TAG_member, name: "a1", scope: !21, file: !1, line: 1, baseType: !14, size: 7, flags: DIFlagBitField, extraData: i64 0)
+!24 = !DIDerivedType(tag: DW_TAG_member, name: "a2", scope: !21, file: !1, line: 1, baseType: !14, size: 4, offset: 7, flags: DIFlagBitField, extraData: i64 0)
+!25 = !DIDerivedType(tag: DW_TAG_member, name: "a3", scope: !21, file: !1, line: 1, baseType: !14, size: 5, offset: 11, flags: DIFlagBitField, extraData: i64 0)
+!26 = !DIDerivedType(tag: DW_TAG_member, name: "a4", scope: !21, file: !1, line: 1, baseType: !14, size: 16, offset: 16, flags: DIFlagBitField, extraData: i64 0)
+!27 = !{!28, !29, !30, !31, !32}
+!28 = !DILocalVariable(name: "arg", arg: 1, scope: !11, file: !1, line: 4, type: !15)
+!29 = !DILocalVariable(name: "r1", scope: !11, file: !1, line: 5, type: !4)
+!30 = !DILocalVariable(name: "r2", scope: !11, file: !1, line: 6, type: !4)
+!31 = !DILocalVariable(name: "r3", scope: !11, file: !1, line: 7, type: !4)
+!32 = !DILocalVariable(name: "r4", scope: !11, file: !1, line: 8, type: !4)
+!33 = !DILocation(line: 0, scope: !11)
+!34 = !DILocation(line: 5, column: 52, scope: !11)
+!35 = !DILocation(line: 5, column: 55, scope: !11)
+!36 = !DILocation(line: 5, column: 17, scope: !11)
+!37 = !DILocation(line: 6, column: 55, scope: !11)
+!38 = !DILocation(line: 6, column: 17, scope: !11)
+!39 = !DILocation(line: 7, column: 55, scope: !11)
+!40 = !DILocation(line: 7, column: 17, scope: !11)
+!41 = !DILocation(line: 8, column: 55, scope: !11)
+!42 = !DILocation(line: 8, column: 17, scope: !11)
+!43 = !DILocation(line: 10, column: 13, scope: !11)
+!44 = !DILocation(line: 10, column: 18, scope: !11)
+!45 = !DILocation(line: 10, column: 23, scope: !11)
+!46 = !DILocation(line: 10, column: 3, scope: !11)
diff --git a/llvm/test/CodeGen/BPF/CORE/intrinsic-fieldinfo-byte-size-2.ll b/llvm/test/CodeGen/BPF/CORE/intrinsic-fieldinfo-byte-size-2.ll
new file mode 100644
index 000000000..23352a3b9
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/CORE/intrinsic-fieldinfo-byte-size-2.ll
@@ -0,0 +1,142 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-ALU64 %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-ALU64 %s
+; RUN: llc -march=bpfel -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-ALU32 %s
+; RUN: llc -march=bpfeb -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-ALU32 %s
+; Source code:
+;   typedef struct s1 { int a1; char a2; } __s1;
+;   union u1 { int b1; __s1 b2; };
+;   enum { FIELD_BYTE_SIZE = 1, };
+;   int test(union u1 *arg) {
+;     unsigned r1 = __builtin_preserve_field_info(arg->b2, FIELD_BYTE_SIZE);
+;     unsigned r2 = __builtin_preserve_field_info(arg->b2.a1, FIELD_BYTE_SIZE);
+;     unsigned r3 = __builtin_preserve_field_info(arg->b2.a2, FIELD_BYTE_SIZE);
+;     /* r1: 8, r2: 4, r3: 1 */
+;     return r1 + r2 + r3;
+;   }
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm test.c
+
+%union.u1 = type { %struct.s1 }
+%struct.s1 = type { i32, i8 }
+
+; Function Attrs: nounwind readnone
+define dso_local i32 @test(%union.u1* %arg) local_unnamed_addr #0 !dbg !11 {
+entry:
+  call void @llvm.dbg.value(metadata %union.u1* %arg, metadata !27, metadata !DIExpression()), !dbg !31
+  %0 = tail call %union.u1* @llvm.preserve.union.access.index.p0s_union.u1s.p0s_union.u1s(%union.u1* %arg, i32 1), !dbg !32, !llvm.preserve.access.index !16
+  %b2 = getelementptr inbounds %union.u1, %union.u1* %0, i64 0, i32 0, !dbg !32
+  %1 = tail call i32 @llvm.bpf.preserve.field.info.p0s_struct.s1s(%struct.s1* %b2, i64 1), !dbg !33
+  call void @llvm.dbg.value(metadata i32 %1, metadata !28, metadata !DIExpression()), !dbg !31
+  %2 = tail call i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.s1s(%struct.s1* %b2, i32 0, i32 0), !dbg !34, !llvm.preserve.access.index !21
+  %3 = tail call i32 @llvm.bpf.preserve.field.info.p0i32(i32* %2, i64 1), !dbg !35
+  call void @llvm.dbg.value(metadata i32 %3, metadata !29, metadata !DIExpression()), !dbg !31
+  %4 = tail call i8* @llvm.preserve.struct.access.index.p0i8.p0s_struct.s1s(%struct.s1* %b2, i32 1, i32 1), !dbg !36, !llvm.preserve.access.index !21
+  %5 = tail call i32 @llvm.bpf.preserve.field.info.p0i8(i8* %4, i64 1), !dbg !37
+  call void @llvm.dbg.value(metadata i32 %5, metadata !30, metadata !DIExpression()), !dbg !31
+  %add = add i32 %3, %1, !dbg !38
+  %add3 = add i32 %add, %5, !dbg !39
+  ret i32 %add3, !dbg !40
+}
+
+; CHECK:             r1 = 8
+; CHECK:             r0 = 4
+; CHECK-ALU64:       r0 += r1
+; CHECK-ALU32:       w0 += w1
+; CHECK:             r1 = 1
+; CHECK-ALU64:       r0 += r1
+; CHECK-ALU32:       w0 += w1
+; CHECK:             exit
+
+; CHECK:             .long   1                       # BTF_KIND_UNION(id = 2)
+; CHECK:             .ascii  "u1"                    # string offset=1
+; CHECK:             .ascii  ".text"                 # string offset=42
+; CHECK:             .ascii  "0:1"                   # string offset=48
+; CHECK:             .ascii  "0:1:0"                 # string offset=89
+; CHECK:             .ascii  "0:1:1"                 # string offset=95
+
+; CHECK:             .long   16                      # FieldReloc
+; CHECK-NEXT:        .long   42                      # Field reloc section string offset=42
+; CHECK-NEXT:        .long   3
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   48
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   89
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   95
+; CHECK-NEXT:        .long   1
+
+; Function Attrs: nounwind readnone
+declare %union.u1* @llvm.preserve.union.access.index.p0s_union.u1s.p0s_union.u1s(%union.u1*, i32) #1
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.bpf.preserve.field.info.p0s_struct.s1s(%struct.s1*, i64) #1
+
+; Function Attrs: nounwind readnone
+declare i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.s1s(%struct.s1*, i32, i32) #1
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.bpf.preserve.field.info.p0i32(i32*, i64) #1
+
+; Function Attrs: nounwind readnone
+declare i8* @llvm.preserve.struct.access.index.p0i8.p0s_struct.s1s(%struct.s1*, i32, i32) #1
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.bpf.preserve.field.info.p0i8(i8*, i64) #1
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.value(metadata, metadata, metadata) #2
+
+attributes #0 = { nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone }
+attributes #2 = { nounwind readnone speculatable willreturn }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!7, !8, !9}
+!llvm.ident = !{!10}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 10.0.0 (https://github.com/llvm/llvm-project.git 4a60741b74384f14b21fdc0131ede326438840ab)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "test.c", directory: "/tmp/home/yhs/work/tests/core")
+!2 = !{!3}
+!3 = !DICompositeType(tag: DW_TAG_enumeration_type, file: !1, line: 3, baseType: !4, size: 32, elements: !5)
+!4 = !DIBasicType(name: "unsigned int", size: 32, encoding: DW_ATE_unsigned)
+!5 = !{!6}
+!6 = !DIEnumerator(name: "FIELD_BYTE_SIZE", value: 1, isUnsigned: true)
+!7 = !{i32 2, !"Dwarf Version", i32 4}
+!8 = !{i32 2, !"Debug Info Version", i32 3}
+!9 = !{i32 1, !"wchar_size", i32 4}
+!10 = !{!"clang version 10.0.0 (https://github.com/llvm/llvm-project.git 4a60741b74384f14b21fdc0131ede326438840ab)"}
+!11 = distinct !DISubprogram(name: "test", scope: !1, file: !1, line: 4, type: !12, scopeLine: 4, flags: DIFlagPrototyped, isDefinition: true, isOptimized: true, unit: !0, retainedNodes: !26)
+!12 = !DISubroutineType(types: !13)
+!13 = !{!14, !15}
+!14 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!15 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !16, size: 64)
+!16 = distinct !DICompositeType(tag: DW_TAG_union_type, name: "u1", file: !1, line: 2, size: 64, elements: !17)
+!17 = !{!18, !19}
+!18 = !DIDerivedType(tag: DW_TAG_member, name: "b1", scope: !16, file: !1, line: 2, baseType: !14, size: 32)
+!19 = !DIDerivedType(tag: DW_TAG_member, name: "b2", scope: !16, file: !1, line: 2, baseType: !20, size: 64)
+!20 = !DIDerivedType(tag: DW_TAG_typedef, name: "__s1", file: !1, line: 1, baseType: !21)
+!21 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "s1", file: !1, line: 1, size: 64, elements: !22)
+!22 = !{!23, !24}
+!23 = !DIDerivedType(tag: DW_TAG_member, name: "a1", scope: !21, file: !1, line: 1, baseType: !14, size: 32)
+!24 = !DIDerivedType(tag: DW_TAG_member, name: "a2", scope: !21, file: !1, line: 1, baseType: !25, size: 8, offset: 32)
+!25 = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_signed_char)
+!26 = !{!27, !28, !29, !30}
+!27 = !DILocalVariable(name: "arg", arg: 1, scope: !11, file: !1, line: 4, type: !15)
+!28 = !DILocalVariable(name: "r1", scope: !11, file: !1, line: 5, type: !4)
+!29 = !DILocalVariable(name: "r2", scope: !11, file: !1, line: 6, type: !4)
+!30 = !DILocalVariable(name: "r3", scope: !11, file: !1, line: 7, type: !4)
+!31 = !DILocation(line: 0, scope: !11)
+!32 = !DILocation(line: 5, column: 52, scope: !11)
+!33 = !DILocation(line: 5, column: 17, scope: !11)
+!34 = !DILocation(line: 6, column: 55, scope: !11)
+!35 = !DILocation(line: 6, column: 17, scope: !11)
+!36 = !DILocation(line: 7, column: 55, scope: !11)
+!37 = !DILocation(line: 7, column: 17, scope: !11)
+!38 = !DILocation(line: 9, column: 13, scope: !11)
+!39 = !DILocation(line: 9, column: 18, scope: !11)
+!40 = !DILocation(line: 9, column: 3, scope: !11)
diff --git a/llvm/test/CodeGen/BPF/CORE/intrinsic-fieldinfo-byte-size-3.ll b/llvm/test/CodeGen/BPF/CORE/intrinsic-fieldinfo-byte-size-3.ll
new file mode 100644
index 000000000..5e3eb0316
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/CORE/intrinsic-fieldinfo-byte-size-3.ll
@@ -0,0 +1,133 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-ALU64 %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-ALU64 %s
+; RUN: llc -march=bpfel -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-ALU32 %s
+; RUN: llc -march=bpfeb -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-ALU32 %s
+; Source code:
+;   typedef struct s1 { int a1[10][10]; } __s1;
+;   union u1 { int b1; __s1 b2; };
+;   enum { FIELD_BYTE_SIZE = 1, };
+;   int test(union u1 *arg) {
+;     unsigned r1 = __builtin_preserve_field_info(arg->b2.a1[5], FIELD_BYTE_SIZE);
+;     unsigned r2 = __builtin_preserve_field_info(arg->b2.a1[5][5], FIELD_BYTE_SIZE);
+;     /* r1: 40, r2: 4 */
+;     return r1 + r2;
+;   }
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm test.c
+
+%union.u1 = type { %struct.s1 }
+%struct.s1 = type { [10 x [10 x i32]] }
+
+; Function Attrs: nounwind readnone
+define dso_local i32 @test(%union.u1* %arg) local_unnamed_addr #0 !dbg !18 {
+entry:
+  call void @llvm.dbg.value(metadata %union.u1* %arg, metadata !31, metadata !DIExpression()), !dbg !34
+  %0 = tail call %union.u1* @llvm.preserve.union.access.index.p0s_union.u1s.p0s_union.u1s(%union.u1* %arg, i32 1), !dbg !35, !llvm.preserve.access.index !22
+  %b2 = getelementptr inbounds %union.u1, %union.u1* %0, i64 0, i32 0, !dbg !35
+  %1 = tail call [10 x [10 x i32]]* @llvm.preserve.struct.access.index.p0a10a10i32.p0s_struct.s1s(%struct.s1* %b2, i32 0, i32 0), !dbg !36, !llvm.preserve.access.index !27
+  %2 = tail call [10 x i32]* @llvm.preserve.array.access.index.p0a10i32.p0a10a10i32([10 x [10 x i32]]* %1, i32 1, i32 5), !dbg !37, !llvm.preserve.access.index !8
+  %3 = tail call i32 @llvm.bpf.preserve.field.info.p0a10i32([10 x i32]* %2, i64 1), !dbg !38
+  call void @llvm.dbg.value(metadata i32 %3, metadata !32, metadata !DIExpression()), !dbg !34
+  %4 = tail call i32* @llvm.preserve.array.access.index.p0i32.p0a10i32([10 x i32]* %2, i32 1, i32 5), !dbg !39, !llvm.preserve.access.index !12
+  %5 = tail call i32 @llvm.bpf.preserve.field.info.p0i32(i32* %4, i64 1), !dbg !40
+  call void @llvm.dbg.value(metadata i32 %5, metadata !33, metadata !DIExpression()), !dbg !34
+  %add = add i32 %5, %3, !dbg !41
+  ret i32 %add, !dbg !42
+}
+
+; CHECK:             r1 = 40
+; CHECK:             r0 = 4
+; CHECK-ALU64:       r0 += r1
+; CHECK-ALU32:       w0 += w1
+; CHECK:             exit
+
+; CHECK:             .long   1                       # BTF_KIND_UNION(id = 2)
+; CHECK:             .ascii  "u1"                    # string offset=1
+; CHECK:             .ascii  ".text"                 # string offset=54
+; CHECK:             .ascii  "0:1:0:5"               # string offset=60
+; CHECK:             .ascii  "0:1:0:5:5"             # string offset=105
+
+; CHECK:             .long   16                      # FieldReloc
+; CHECK-NEXT:        .long   54                      # Field reloc section string offset=54
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   60
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   105
+; CHECK-NEXT:        .long   1
+
+; Function Attrs: nounwind readnone
+declare %union.u1* @llvm.preserve.union.access.index.p0s_union.u1s.p0s_union.u1s(%union.u1*, i32) #1
+
+; Function Attrs: nounwind readnone
+declare [10 x [10 x i32]]* @llvm.preserve.struct.access.index.p0a10a10i32.p0s_struct.s1s(%struct.s1*, i32, i32) #1
+
+; Function Attrs: nounwind readnone
+declare [10 x i32]* @llvm.preserve.array.access.index.p0a10i32.p0a10a10i32([10 x [10 x i32]]*, i32, i32) #1
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.bpf.preserve.field.info.p0a10i32([10 x i32]*, i64) #1
+
+; Function Attrs: nounwind readnone
+declare i32* @llvm.preserve.array.access.index.p0i32.p0a10i32([10 x i32]*, i32, i32) #1
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.bpf.preserve.field.info.p0i32(i32*, i64) #1
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.value(metadata, metadata, metadata) #2
+
+attributes #0 = { nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone }
+attributes #2 = { nounwind readnone speculatable willreturn }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!14, !15, !16}
+!llvm.ident = !{!17}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 10.0.0 (https://github.com/llvm/llvm-project.git c1e02f16f1105ffaf1c35ee8bc38b7d6db5c6ea9)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !7, nameTableKind: None)
+!1 = !DIFile(filename: "test.c", directory: "/tmp/home/yhs/work/tests/core")
+!2 = !{!3}
+!3 = !DICompositeType(tag: DW_TAG_enumeration_type, file: !1, line: 3, baseType: !4, size: 32, elements: !5)
+!4 = !DIBasicType(name: "unsigned int", size: 32, encoding: DW_ATE_unsigned)
+!5 = !{!6}
+!6 = !DIEnumerator(name: "FIELD_BYTE_SIZE", value: 1, isUnsigned: true)
+!7 = !{!8, !12}
+!8 = !DICompositeType(tag: DW_TAG_array_type, baseType: !9, size: 3200, elements: !10)
+!9 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!10 = !{!11, !11}
+!11 = !DISubrange(count: 10)
+!12 = !DICompositeType(tag: DW_TAG_array_type, baseType: !9, size: 320, elements: !13)
+!13 = !{!11}
+!14 = !{i32 2, !"Dwarf Version", i32 4}
+!15 = !{i32 2, !"Debug Info Version", i32 3}
+!16 = !{i32 1, !"wchar_size", i32 4}
+!17 = !{!"clang version 10.0.0 (https://github.com/llvm/llvm-project.git c1e02f16f1105ffaf1c35ee8bc38b7d6db5c6ea9)"}
+!18 = distinct !DISubprogram(name: "test", scope: !1, file: !1, line: 4, type: !19, scopeLine: 4, flags: DIFlagPrototyped, isDefinition: true, isOptimized: true, unit: !0, retainedNodes: !30)
+!19 = !DISubroutineType(types: !20)
+!20 = !{!9, !21}
+!21 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !22, size: 64)
+!22 = distinct !DICompositeType(tag: DW_TAG_union_type, name: "u1", file: !1, line: 2, size: 3200, elements: !23)
+!23 = !{!24, !25}
+!24 = !DIDerivedType(tag: DW_TAG_member, name: "b1", scope: !22, file: !1, line: 2, baseType: !9, size: 32)
+!25 = !DIDerivedType(tag: DW_TAG_member, name: "b2", scope: !22, file: !1, line: 2, baseType: !26, size: 3200)
+!26 = !DIDerivedType(tag: DW_TAG_typedef, name: "__s1", file: !1, line: 1, baseType: !27)
+!27 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "s1", file: !1, line: 1, size: 3200, elements: !28)
+!28 = !{!29}
+!29 = !DIDerivedType(tag: DW_TAG_member, name: "a1", scope: !27, file: !1, line: 1, baseType: !8, size: 3200)
+!30 = !{!31, !32, !33}
+!31 = !DILocalVariable(name: "arg", arg: 1, scope: !18, file: !1, line: 4, type: !21)
+!32 = !DILocalVariable(name: "r1", scope: !18, file: !1, line: 5, type: !4)
+!33 = !DILocalVariable(name: "r2", scope: !18, file: !1, line: 6, type: !4)
+!34 = !DILocation(line: 0, scope: !18)
+!35 = !DILocation(line: 5, column: 52, scope: !18)
+!36 = !DILocation(line: 5, column: 55, scope: !18)
+!37 = !DILocation(line: 5, column: 47, scope: !18)
+!38 = !DILocation(line: 5, column: 17, scope: !18)
+!39 = !DILocation(line: 6, column: 47, scope: !18)
+!40 = !DILocation(line: 6, column: 17, scope: !18)
+!41 = !DILocation(line: 8, column: 13, scope: !18)
+!42 = !DILocation(line: 8, column: 3, scope: !18)
diff --git a/llvm/test/CodeGen/BPF/CORE/intrinsic-fieldinfo-byte-size-4.ll b/llvm/test/CodeGen/BPF/CORE/intrinsic-fieldinfo-byte-size-4.ll
new file mode 100644
index 000000000..02e0fd791
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/CORE/intrinsic-fieldinfo-byte-size-4.ll
@@ -0,0 +1,86 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfel -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; Source code:
+;   typedef struct s1 { int a1; int a2:4; int a3;} __s1;
+;   enum { FIELD_BYTE_SIZE = 1, };
+;   int test(__s1 *arg) {
+;     return __builtin_preserve_field_info(arg->a2, FIELD_BYTE_SIZE);
+;   }
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm test.c
+
+%struct.s1 = type { i32, i8, i32 }
+
+; Function Attrs: nounwind readnone
+define dso_local i32 @test(%struct.s1* readnone %arg) local_unnamed_addr #0 !dbg !11 {
+entry:
+  call void @llvm.dbg.value(metadata %struct.s1* %arg, metadata !23, metadata !DIExpression()), !dbg !24
+  %0 = tail call i8* @llvm.preserve.struct.access.index.p0i8.p0s_struct.s1s(%struct.s1* %arg, i32 1, i32 1), !dbg !25, !llvm.preserve.access.index !17
+  %1 = tail call i32 @llvm.bpf.preserve.field.info.p0i8(i8* %0, i64 1), !dbg !26
+  ret i32 %1, !dbg !27
+}
+
+; CHECK:             r0 = 4
+; CHECK:             exit
+
+; CHECK:             .long   6                       # BTF_KIND_STRUCT(id = 3)
+
+; CHECK:             .ascii  "s1"                    # string offset=6
+; CHECK:             .ascii  ".text"                 # string offset=31
+; CHECK:             .ascii  "0:1"                   # string offset=37
+
+; CHECK:             .long   16                      # FieldReloc
+; CHECK-NEXT:        .long   31                      # Field reloc section string offset=31
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   3
+; CHECK-NEXT:        .long   37
+; CHECK-NEXT:        .long   1
+
+; Function Attrs: nounwind readnone
+declare i8* @llvm.preserve.struct.access.index.p0i8.p0s_struct.s1s(%struct.s1*, i32, i32) #1
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.bpf.preserve.field.info.p0i8(i8*, i64) #1
+
+; Function Attrs: nounwind readnone speculatable
+declare void @llvm.dbg.value(metadata, metadata, metadata) #2
+
+attributes #0 = { nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone }
+attributes #2 = { nounwind readnone speculatable }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!7, !8, !9}
+!llvm.ident = !{!10}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 10.0.0 (https://github.com/llvm/llvm-project.git 6c63b5d6a1cb84a75807804337c855a41c976260)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "test.c", directory: "/tmp/home/yhs/work/tests/core")
+!2 = !{!3}
+!3 = !DICompositeType(tag: DW_TAG_enumeration_type, file: !1, line: 2, baseType: !4, size: 32, elements: !5)
+!4 = !DIBasicType(name: "unsigned int", size: 32, encoding: DW_ATE_unsigned)
+!5 = !{!6}
+!6 = !DIEnumerator(name: "FIELD_BYTE_SIZE", value: 1, isUnsigned: true)
+!7 = !{i32 2, !"Dwarf Version", i32 4}
+!8 = !{i32 2, !"Debug Info Version", i32 3}
+!9 = !{i32 1, !"wchar_size", i32 4}
+!10 = !{!"clang version 10.0.0 (https://github.com/llvm/llvm-project.git 6c63b5d6a1cb84a75807804337c855a41c976260)"}
+!11 = distinct !DISubprogram(name: "test", scope: !1, file: !1, line: 3, type: !12, scopeLine: 3, flags: DIFlagPrototyped, isDefinition: true, isOptimized: true, unit: !0, retainedNodes: !22)
+!12 = !DISubroutineType(types: !13)
+!13 = !{!14, !15}
+!14 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!15 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !16, size: 64)
+!16 = !DIDerivedType(tag: DW_TAG_typedef, name: "__s1", file: !1, line: 1, baseType: !17)
+!17 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "s1", file: !1, line: 1, size: 96, elements: !18)
+!18 = !{!19, !20, !21}
+!19 = !DIDerivedType(tag: DW_TAG_member, name: "a1", scope: !17, file: !1, line: 1, baseType: !14, size: 32)
+!20 = !DIDerivedType(tag: DW_TAG_member, name: "a2", scope: !17, file: !1, line: 1, baseType: !14, size: 4, offset: 32, flags: DIFlagBitField, extraData: i64 32)
+!21 = !DIDerivedType(tag: DW_TAG_member, name: "a3", scope: !17, file: !1, line: 1, baseType: !14, size: 32, offset: 64)
+!22 = !{!23}
+!23 = !DILocalVariable(name: "arg", arg: 1, scope: !11, file: !1, line: 3, type: !15)
+!24 = !DILocation(line: 0, scope: !11)
+!25 = !DILocation(line: 4, column: 45, scope: !11)
+!26 = !DILocation(line: 4, column: 10, scope: !11)
+!27 = !DILocation(line: 4, column: 3, scope: !11)
diff --git a/llvm/test/CodeGen/BPF/CORE/intrinsic-fieldinfo-existence-1.ll b/llvm/test/CodeGen/BPF/CORE/intrinsic-fieldinfo-existence-1.ll
new file mode 100644
index 000000000..ffe1cceb8
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/CORE/intrinsic-fieldinfo-existence-1.ll
@@ -0,0 +1,167 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-ALU64 %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-ALU64 %s
+; RUN: llc -march=bpfel -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-ALU32 %s
+; RUN: llc -march=bpfeb -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-ALU32 %s
+; Source code:
+;   typedef unsigned __uint;
+;   struct s1 { int a1; __uint a2:9; __uint a3:4; };
+;   union u1 { int b1; __uint b2:9; __uint b3:4; };
+;   enum { FIELD_EXISTENCE = 2, };
+;   int test(struct s1 *arg1, union u1 *arg2) {
+;     unsigned r1 = __builtin_preserve_field_info(arg1->a1, FIELD_EXISTENCE);
+;     unsigned r2 = __builtin_preserve_field_info(arg1->a3, FIELD_EXISTENCE);
+;     unsigned r3 = __builtin_preserve_field_info(arg2->b1, FIELD_EXISTENCE);
+;     unsigned r4 = __builtin_preserve_field_info(arg2->b3, FIELD_EXISTENCE);
+;     return r1 + r2 + r3 + r4;
+;   }
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm test.c
+
+%struct.s1 = type { i32, i16 }
+%union.u1 = type { i32 }
+
+; Function Attrs: nounwind readnone
+define dso_local i32 @test(%struct.s1* %arg1, %union.u1* %arg2) local_unnamed_addr #0 !dbg !11 {
+entry:
+  call void @llvm.dbg.value(metadata %struct.s1* %arg1, metadata !29, metadata !DIExpression()), !dbg !35
+  call void @llvm.dbg.value(metadata %union.u1* %arg2, metadata !30, metadata !DIExpression()), !dbg !35
+  %0 = tail call i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.s1s(%struct.s1* %arg1, i32 0, i32 0), !dbg !36, !llvm.preserve.access.index !16
+  %1 = tail call i32 @llvm.bpf.preserve.field.info.p0i32(i32* %0, i64 2), !dbg !37
+  call void @llvm.dbg.value(metadata i32 %1, metadata !31, metadata !DIExpression()), !dbg !35
+  %2 = tail call i16* @llvm.preserve.struct.access.index.p0i16.p0s_struct.s1s(%struct.s1* %arg1, i32 1, i32 2), !dbg !38, !llvm.preserve.access.index !16
+  %3 = tail call i32 @llvm.bpf.preserve.field.info.p0i16(i16* %2, i64 2), !dbg !39
+  call void @llvm.dbg.value(metadata i32 %3, metadata !32, metadata !DIExpression()), !dbg !35
+  %4 = tail call %union.u1* @llvm.preserve.union.access.index.p0s_union.u1s.p0s_union.u1s(%union.u1* %arg2, i32 0), !dbg !40, !llvm.preserve.access.index !23
+  %b1 = getelementptr inbounds %union.u1, %union.u1* %4, i64 0, i32 0, !dbg !40
+  %5 = tail call i32 @llvm.bpf.preserve.field.info.p0i32(i32* %b1, i64 2), !dbg !41
+  call void @llvm.dbg.value(metadata i32 %5, metadata !33, metadata !DIExpression()), !dbg !35
+  %6 = tail call i32* @llvm.preserve.struct.access.index.p0i32.p0s_union.u1s(%union.u1* %arg2, i32 0, i32 2), !dbg !42, !llvm.preserve.access.index !23
+  %7 = bitcast i32* %6 to i8*, !dbg !42
+  %8 = tail call i32 @llvm.bpf.preserve.field.info.p0i8(i8* %7, i64 2), !dbg !43
+  call void @llvm.dbg.value(metadata i32 %8, metadata !34, metadata !DIExpression()), !dbg !35
+  %add = add i32 %3, %1, !dbg !44
+  %add1 = add i32 %add, %5, !dbg !45
+  %add2 = add i32 %add1, %8, !dbg !46
+  ret i32 %add2, !dbg !47
+}
+
+; CHECK:             r1 = 1
+; CHECK:             r0 = 1
+; CHECK-ALU64:       r0 += r1
+; CHECK-ALU32:       w0 += w1
+; CHECK:             r1 = 1
+; CHECK-ALU64:       r0 += r1
+; CHECK-ALU32:       w0 += w1
+; CHECK:             r1 = 1
+; CHECK-ALU64:       r0 += r1
+; CHECK-ALU32:       w0 += w1
+; CHECK:             exit
+
+; CHECK:             .long   1                       # BTF_KIND_STRUCT(id = 2)
+; CHECK:             .long   37                      # BTF_KIND_UNION(id = 7)
+; CHECK:             .ascii  "s1"                    # string offset=1
+; CHECK:             .ascii  "u1"                    # string offset=37
+; CHECK:             .ascii  ".text"                 # string offset=64
+; CHECK:             .ascii  "0:0"                   # string offset=70
+; CHECK:             .ascii  "0:2"                   # string offset=111
+
+; CHECK:             .long   16                      # FieldReloc
+; CHECK-NEXT:        .long   64                      # Field reloc section string offset=64
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   70
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   111
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   7
+; CHECK-NEXT:        .long   70
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   7
+; CHECK-NEXT:        .long   111
+; CHECK-NEXT:        .long   2
+
+; Function Attrs: nounwind readnone
+declare i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.s1s(%struct.s1*, i32, i32) #1
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.bpf.preserve.field.info.p0i32(i32*, i64) #1
+
+; Function Attrs: nounwind readnone
+declare i16* @llvm.preserve.struct.access.index.p0i16.p0s_struct.s1s(%struct.s1*, i32, i32) #1
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.bpf.preserve.field.info.p0i16(i16*, i64) #1
+
+; Function Attrs: nounwind readnone
+declare %union.u1* @llvm.preserve.union.access.index.p0s_union.u1s.p0s_union.u1s(%union.u1*, i32) #1
+
+; Function Attrs: nounwind readnone
+declare i32* @llvm.preserve.struct.access.index.p0i32.p0s_union.u1s(%union.u1*, i32, i32) #1
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.bpf.preserve.field.info.p0i8(i8*, i64) #1
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.value(metadata, metadata, metadata) #2
+
+attributes #0 = { nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone }
+attributes #2 = { nounwind readnone speculatable willreturn }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!7, !8, !9}
+!llvm.ident = !{!10}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 10.0.0 (https://github.com/llvm/llvm-project.git 4a60741b74384f14b21fdc0131ede326438840ab)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "test.c", directory: "/tmp/home/yhs/work/tests/core")
+!2 = !{!3}
+!3 = !DICompositeType(tag: DW_TAG_enumeration_type, file: !1, line: 4, baseType: !4, size: 32, elements: !5)
+!4 = !DIBasicType(name: "unsigned int", size: 32, encoding: DW_ATE_unsigned)
+!5 = !{!6}
+!6 = !DIEnumerator(name: "FIELD_EXISTENCE", value: 2, isUnsigned: true)
+!7 = !{i32 2, !"Dwarf Version", i32 4}
+!8 = !{i32 2, !"Debug Info Version", i32 3}
+!9 = !{i32 1, !"wchar_size", i32 4}
+!10 = !{!"clang version 10.0.0 (https://github.com/llvm/llvm-project.git 4a60741b74384f14b21fdc0131ede326438840ab)"}
+!11 = distinct !DISubprogram(name: "test", scope: !1, file: !1, line: 5, type: !12, scopeLine: 5, flags: DIFlagPrototyped, isDefinition: true, isOptimized: true, unit: !0, retainedNodes: !28)
+!12 = !DISubroutineType(types: !13)
+!13 = !{!14, !15, !22}
+!14 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!15 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !16, size: 64)
+!16 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "s1", file: !1, line: 2, size: 64, elements: !17)
+!17 = !{!18, !19, !21}
+!18 = !DIDerivedType(tag: DW_TAG_member, name: "a1", scope: !16, file: !1, line: 2, baseType: !14, size: 32)
+!19 = !DIDerivedType(tag: DW_TAG_member, name: "a2", scope: !16, file: !1, line: 2, baseType: !20, size: 9, offset: 32, flags: DIFlagBitField, extraData: i64 32)
+!20 = !DIDerivedType(tag: DW_TAG_typedef, name: "__uint", file: !1, line: 1, baseType: !4)
+!21 = !DIDerivedType(tag: DW_TAG_member, name: "a3", scope: !16, file: !1, line: 2, baseType: !20, size: 4, offset: 41, flags: DIFlagBitField, extraData: i64 32)
+!22 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !23, size: 64)
+!23 = distinct !DICompositeType(tag: DW_TAG_union_type, name: "u1", file: !1, line: 3, size: 32, elements: !24)
+!24 = !{!25, !26, !27}
+!25 = !DIDerivedType(tag: DW_TAG_member, name: "b1", scope: !23, file: !1, line: 3, baseType: !14, size: 32)
+!26 = !DIDerivedType(tag: DW_TAG_member, name: "b2", scope: !23, file: !1, line: 3, baseType: !20, size: 9, flags: DIFlagBitField, extraData: i64 0)
+!27 = !DIDerivedType(tag: DW_TAG_member, name: "b3", scope: !23, file: !1, line: 3, baseType: !20, size: 4, flags: DIFlagBitField, extraData: i64 0)
+!28 = !{!29, !30, !31, !32, !33, !34}
+!29 = !DILocalVariable(name: "arg1", arg: 1, scope: !11, file: !1, line: 5, type: !15)
+!30 = !DILocalVariable(name: "arg2", arg: 2, scope: !11, file: !1, line: 5, type: !22)
+!31 = !DILocalVariable(name: "r1", scope: !11, file: !1, line: 6, type: !4)
+!32 = !DILocalVariable(name: "r2", scope: !11, file: !1, line: 7, type: !4)
+!33 = !DILocalVariable(name: "r3", scope: !11, file: !1, line: 8, type: !4)
+!34 = !DILocalVariable(name: "r4", scope: !11, file: !1, line: 9, type: !4)
+!35 = !DILocation(line: 0, scope: !11)
+!36 = !DILocation(line: 6, column: 53, scope: !11)
+!37 = !DILocation(line: 6, column: 17, scope: !11)
+!38 = !DILocation(line: 7, column: 53, scope: !11)
+!39 = !DILocation(line: 7, column: 17, scope: !11)
+!40 = !DILocation(line: 8, column: 53, scope: !11)
+!41 = !DILocation(line: 8, column: 17, scope: !11)
+!42 = !DILocation(line: 9, column: 53, scope: !11)
+!43 = !DILocation(line: 9, column: 17, scope: !11)
+!44 = !DILocation(line: 10, column: 13, scope: !11)
+!45 = !DILocation(line: 10, column: 18, scope: !11)
+!46 = !DILocation(line: 10, column: 23, scope: !11)
+!47 = !DILocation(line: 10, column: 3, scope: !11)
diff --git a/llvm/test/CodeGen/BPF/CORE/intrinsic-fieldinfo-existence-2.ll b/llvm/test/CodeGen/BPF/CORE/intrinsic-fieldinfo-existence-2.ll
new file mode 100644
index 000000000..68018b838
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/CORE/intrinsic-fieldinfo-existence-2.ll
@@ -0,0 +1,124 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-ALU64 %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-ALU64 %s
+; RUN: llc -march=bpfel -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-ALU32 %s
+; RUN: llc -march=bpfeb -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-ALU32 %s
+; Source code:
+;   typedef unsigned __uint;
+;   struct s1 { int a1; __uint a2:9; __uint a3:4; };
+;   union u1 { int b1; struct s1 b2; };
+;   enum { FIELD_EXISTENCE = 2, };
+;   int test(union u1 *arg) {
+;     unsigned r1 = __builtin_preserve_field_info(arg->b2.a1, FIELD_EXISTENCE);
+;     unsigned r2 = __builtin_preserve_field_info(arg->b2.a3, FIELD_EXISTENCE);
+;     return r1 + r2;
+;   }
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm test.c
+
+%union.u1 = type { %struct.s1 }
+%struct.s1 = type { i32, i16 }
+
+; Function Attrs: nounwind readnone
+define dso_local i32 @test(%union.u1* %arg) local_unnamed_addr #0 !dbg !11 {
+entry:
+  call void @llvm.dbg.value(metadata %union.u1* %arg, metadata !27, metadata !DIExpression()), !dbg !30
+  %0 = tail call %union.u1* @llvm.preserve.union.access.index.p0s_union.u1s.p0s_union.u1s(%union.u1* %arg, i32 1), !dbg !31, !llvm.preserve.access.index !16
+  %b2 = getelementptr inbounds %union.u1, %union.u1* %0, i64 0, i32 0, !dbg !31
+  %1 = tail call i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.s1s(%struct.s1* %b2, i32 0, i32 0), !dbg !32, !llvm.preserve.access.index !20
+  %2 = tail call i32 @llvm.bpf.preserve.field.info.p0i32(i32* %1, i64 2), !dbg !33
+  call void @llvm.dbg.value(metadata i32 %2, metadata !28, metadata !DIExpression()), !dbg !30
+  %3 = tail call i16* @llvm.preserve.struct.access.index.p0i16.p0s_struct.s1s(%struct.s1* %b2, i32 1, i32 2), !dbg !34, !llvm.preserve.access.index !20
+  %4 = tail call i32 @llvm.bpf.preserve.field.info.p0i16(i16* %3, i64 2), !dbg !35
+  call void @llvm.dbg.value(metadata i32 %4, metadata !29, metadata !DIExpression()), !dbg !30
+  %add = add i32 %4, %2, !dbg !36
+  ret i32 %add, !dbg !37
+}
+
+; CHECK:             r1 = 1
+; CHECK:             r0 = 1
+; CHECK-ALU64:       r0 += r1
+; CHECK-ALU32:       w0 += w1
+; CHECK:             exit
+
+; CHECK:             .long   1                       # BTF_KIND_UNION(id = 2)
+; CHECK:             .ascii  "u1"                    # string offset=1
+; CHECK:             .ascii  ".text"                 # string offset=55
+; CHECK:             .ascii  "0:1:0"                 # string offset=61
+; CHECK:             .ascii  "0:1:2"                 # string offset=104
+
+; CHECK:             .long   16                      # FieldReloc
+; CHECK-NEXT:        .long   55                      # Field reloc section string offset=55
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   61
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   104
+; CHECK-NEXT:        .long   2
+
+; Function Attrs: nounwind readnone
+declare %union.u1* @llvm.preserve.union.access.index.p0s_union.u1s.p0s_union.u1s(%union.u1*, i32) #1
+
+; Function Attrs: nounwind readnone
+declare i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.s1s(%struct.s1*, i32, i32) #1
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.bpf.preserve.field.info.p0i32(i32*, i64) #1
+
+; Function Attrs: nounwind readnone
+declare i16* @llvm.preserve.struct.access.index.p0i16.p0s_struct.s1s(%struct.s1*, i32, i32) #1
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.bpf.preserve.field.info.p0i16(i16*, i64) #1
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.value(metadata, metadata, metadata) #2
+
+attributes #0 = { nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone }
+attributes #2 = { nounwind readnone speculatable willreturn }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!7, !8, !9}
+!llvm.ident = !{!10}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 10.0.0 (https://github.com/llvm/llvm-project.git 4a60741b74384f14b21fdc0131ede326438840ab)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "test.c", directory: "/tmp/home/yhs/work/tests/core")
+!2 = !{!3}
+!3 = !DICompositeType(tag: DW_TAG_enumeration_type, file: !1, line: 4, baseType: !4, size: 32, elements: !5)
+!4 = !DIBasicType(name: "unsigned int", size: 32, encoding: DW_ATE_unsigned)
+!5 = !{!6}
+!6 = !DIEnumerator(name: "FIELD_EXISTENCE", value: 2, isUnsigned: true)
+!7 = !{i32 2, !"Dwarf Version", i32 4}
+!8 = !{i32 2, !"Debug Info Version", i32 3}
+!9 = !{i32 1, !"wchar_size", i32 4}
+!10 = !{!"clang version 10.0.0 (https://github.com/llvm/llvm-project.git 4a60741b74384f14b21fdc0131ede326438840ab)"}
+!11 = distinct !DISubprogram(name: "test", scope: !1, file: !1, line: 5, type: !12, scopeLine: 5, flags: DIFlagPrototyped, isDefinition: true, isOptimized: true, unit: !0, retainedNodes: !26)
+!12 = !DISubroutineType(types: !13)
+!13 = !{!14, !15}
+!14 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!15 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !16, size: 64)
+!16 = distinct !DICompositeType(tag: DW_TAG_union_type, name: "u1", file: !1, line: 3, size: 64, elements: !17)
+!17 = !{!18, !19}
+!18 = !DIDerivedType(tag: DW_TAG_member, name: "b1", scope: !16, file: !1, line: 3, baseType: !14, size: 32)
+!19 = !DIDerivedType(tag: DW_TAG_member, name: "b2", scope: !16, file: !1, line: 3, baseType: !20, size: 64)
+!20 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "s1", file: !1, line: 2, size: 64, elements: !21)
+!21 = !{!22, !23, !25}
+!22 = !DIDerivedType(tag: DW_TAG_member, name: "a1", scope: !20, file: !1, line: 2, baseType: !14, size: 32)
+!23 = !DIDerivedType(tag: DW_TAG_member, name: "a2", scope: !20, file: !1, line: 2, baseType: !24, size: 9, offset: 32, flags: DIFlagBitField, extraData: i64 32)
+!24 = !DIDerivedType(tag: DW_TAG_typedef, name: "__uint", file: !1, line: 1, baseType: !4)
+!25 = !DIDerivedType(tag: DW_TAG_member, name: "a3", scope: !20, file: !1, line: 2, baseType: !24, size: 4, offset: 41, flags: DIFlagBitField, extraData: i64 32)
+!26 = !{!27, !28, !29}
+!27 = !DILocalVariable(name: "arg", arg: 1, scope: !11, file: !1, line: 5, type: !15)
+!28 = !DILocalVariable(name: "r1", scope: !11, file: !1, line: 6, type: !4)
+!29 = !DILocalVariable(name: "r2", scope: !11, file: !1, line: 7, type: !4)
+!30 = !DILocation(line: 0, scope: !11)
+!31 = !DILocation(line: 6, column: 52, scope: !11)
+!32 = !DILocation(line: 6, column: 55, scope: !11)
+!33 = !DILocation(line: 6, column: 17, scope: !11)
+!34 = !DILocation(line: 7, column: 55, scope: !11)
+!35 = !DILocation(line: 7, column: 17, scope: !11)
+!36 = !DILocation(line: 8, column: 13, scope: !11)
+!37 = !DILocation(line: 8, column: 3, scope: !11)
diff --git a/llvm/test/CodeGen/BPF/CORE/intrinsic-fieldinfo-existence-3.ll b/llvm/test/CodeGen/BPF/CORE/intrinsic-fieldinfo-existence-3.ll
new file mode 100644
index 000000000..8fb227393
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/CORE/intrinsic-fieldinfo-existence-3.ll
@@ -0,0 +1,132 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-ALU64 %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-ALU64 %s
+; RUN: llc -march=bpfel -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-ALU32 %s
+; RUN: llc -march=bpfeb -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-ALU32 %s
+; Source code:
+;   typedef struct s1 { int a1[10][10]; } __s1;
+;   union u1 { int b1; __s1 b2; };
+;   enum { FIELD_EXISTENCE = 2, };
+;   int test(union u1 *arg) {
+;     unsigned r1 = __builtin_preserve_field_info(arg->b2.a1[5], FIELD_EXISTENCE);
+;     unsigned r2 = __builtin_preserve_field_info(arg->b2.a1[5][5], FIELD_EXISTENCE);
+;     return r1 + r2;
+;   }
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm test.c
+
+%union.u1 = type { %struct.s1 }
+%struct.s1 = type { [10 x [10 x i32]] }
+
+; Function Attrs: nounwind readnone
+define dso_local i32 @test(%union.u1* %arg) local_unnamed_addr #0 !dbg !18 {
+entry:
+  call void @llvm.dbg.value(metadata %union.u1* %arg, metadata !31, metadata !DIExpression()), !dbg !34
+  %0 = tail call %union.u1* @llvm.preserve.union.access.index.p0s_union.u1s.p0s_union.u1s(%union.u1* %arg, i32 1), !dbg !35, !llvm.preserve.access.index !22
+  %b2 = getelementptr inbounds %union.u1, %union.u1* %0, i64 0, i32 0, !dbg !35
+  %1 = tail call [10 x [10 x i32]]* @llvm.preserve.struct.access.index.p0a10a10i32.p0s_struct.s1s(%struct.s1* %b2, i32 0, i32 0), !dbg !36, !llvm.preserve.access.index !27
+  %2 = tail call [10 x i32]* @llvm.preserve.array.access.index.p0a10i32.p0a10a10i32([10 x [10 x i32]]* %1, i32 1, i32 5), !dbg !37, !llvm.preserve.access.index !8
+  %3 = tail call i32 @llvm.bpf.preserve.field.info.p0a10i32([10 x i32]* %2, i64 2), !dbg !38
+  call void @llvm.dbg.value(metadata i32 %3, metadata !32, metadata !DIExpression()), !dbg !34
+  %4 = tail call i32* @llvm.preserve.array.access.index.p0i32.p0a10i32([10 x i32]* %2, i32 1, i32 5), !dbg !39, !llvm.preserve.access.index !12
+  %5 = tail call i32 @llvm.bpf.preserve.field.info.p0i32(i32* %4, i64 2), !dbg !40
+  call void @llvm.dbg.value(metadata i32 %5, metadata !33, metadata !DIExpression()), !dbg !34
+  %add = add i32 %5, %3, !dbg !41
+  ret i32 %add, !dbg !42
+}
+
+; CHECK:             r1 = 1
+; CHECK:             r0 = 1
+; CHECK-ALU64:       r0 += r1
+; CHECK-ALU32:       w0 += w1
+; CHECK:             exit
+
+; CHECK:             .long   1                       # BTF_KIND_UNION(id = 2)
+; CHECK:             .ascii  "u1"                    # string offset=1
+; CHECK:             .ascii  ".text"                 # string offset=54
+; CHECK:             .ascii  "0:1:0:5"               # string offset=60
+; CHECK:             .ascii  "0:1:0:5:5"             # string offset=105
+
+; CHECK:             .long   16                      # FieldReloc
+; CHECK-NEXT:        .long   54                      # Field reloc section string offset=54
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   60
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   105
+; CHECK-NEXT:        .long   2
+
+; Function Attrs: nounwind readnone
+declare %union.u1* @llvm.preserve.union.access.index.p0s_union.u1s.p0s_union.u1s(%union.u1*, i32) #1
+
+; Function Attrs: nounwind readnone
+declare [10 x [10 x i32]]* @llvm.preserve.struct.access.index.p0a10a10i32.p0s_struct.s1s(%struct.s1*, i32, i32) #1
+
+; Function Attrs: nounwind readnone
+declare [10 x i32]* @llvm.preserve.array.access.index.p0a10i32.p0a10a10i32([10 x [10 x i32]]*, i32, i32) #1
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.bpf.preserve.field.info.p0a10i32([10 x i32]*, i64) #1
+
+; Function Attrs: nounwind readnone
+declare i32* @llvm.preserve.array.access.index.p0i32.p0a10i32([10 x i32]*, i32, i32) #1
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.bpf.preserve.field.info.p0i32(i32*, i64) #1
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.value(metadata, metadata, metadata) #2
+
+attributes #0 = { nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone }
+attributes #2 = { nounwind readnone speculatable willreturn }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!14, !15, !16}
+!llvm.ident = !{!17}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 10.0.0 (https://github.com/llvm/llvm-project.git c1e02f16f1105ffaf1c35ee8bc38b7d6db5c6ea9)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !7, nameTableKind: None)
+!1 = !DIFile(filename: "test.c", directory: "/tmp/home/yhs/work/tests/core")
+!2 = !{!3}
+!3 = !DICompositeType(tag: DW_TAG_enumeration_type, file: !1, line: 3, baseType: !4, size: 32, elements: !5)
+!4 = !DIBasicType(name: "unsigned int", size: 32, encoding: DW_ATE_unsigned)
+!5 = !{!6}
+!6 = !DIEnumerator(name: "FIELD_EXISTENCE", value: 2, isUnsigned: true)
+!7 = !{!8, !12}
+!8 = !DICompositeType(tag: DW_TAG_array_type, baseType: !9, size: 3200, elements: !10)
+!9 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!10 = !{!11, !11}
+!11 = !DISubrange(count: 10)
+!12 = !DICompositeType(tag: DW_TAG_array_type, baseType: !9, size: 320, elements: !13)
+!13 = !{!11}
+!14 = !{i32 2, !"Dwarf Version", i32 4}
+!15 = !{i32 2, !"Debug Info Version", i32 3}
+!16 = !{i32 1, !"wchar_size", i32 4}
+!17 = !{!"clang version 10.0.0 (https://github.com/llvm/llvm-project.git c1e02f16f1105ffaf1c35ee8bc38b7d6db5c6ea9)"}
+!18 = distinct !DISubprogram(name: "test", scope: !1, file: !1, line: 4, type: !19, scopeLine: 4, flags: DIFlagPrototyped, isDefinition: true, isOptimized: true, unit: !0, retainedNodes: !30)
+!19 = !DISubroutineType(types: !20)
+!20 = !{!9, !21}
+!21 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !22, size: 64)
+!22 = distinct !DICompositeType(tag: DW_TAG_union_type, name: "u1", file: !1, line: 2, size: 3200, elements: !23)
+!23 = !{!24, !25}
+!24 = !DIDerivedType(tag: DW_TAG_member, name: "b1", scope: !22, file: !1, line: 2, baseType: !9, size: 32)
+!25 = !DIDerivedType(tag: DW_TAG_member, name: "b2", scope: !22, file: !1, line: 2, baseType: !26, size: 3200)
+!26 = !DIDerivedType(tag: DW_TAG_typedef, name: "__s1", file: !1, line: 1, baseType: !27)
+!27 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "s1", file: !1, line: 1, size: 3200, elements: !28)
+!28 = !{!29}
+!29 = !DIDerivedType(tag: DW_TAG_member, name: "a1", scope: !27, file: !1, line: 1, baseType: !8, size: 3200)
+!30 = !{!31, !32, !33}
+!31 = !DILocalVariable(name: "arg", arg: 1, scope: !18, file: !1, line: 4, type: !21)
+!32 = !DILocalVariable(name: "r1", scope: !18, file: !1, line: 5, type: !4)
+!33 = !DILocalVariable(name: "r2", scope: !18, file: !1, line: 6, type: !4)
+!34 = !DILocation(line: 0, scope: !18)
+!35 = !DILocation(line: 5, column: 52, scope: !18)
+!36 = !DILocation(line: 5, column: 55, scope: !18)
+!37 = !DILocation(line: 5, column: 47, scope: !18)
+!38 = !DILocation(line: 5, column: 17, scope: !18)
+!39 = !DILocation(line: 6, column: 47, scope: !18)
+!40 = !DILocation(line: 6, column: 17, scope: !18)
+!41 = !DILocation(line: 7, column: 13, scope: !18)
+!42 = !DILocation(line: 7, column: 3, scope: !18)
diff --git a/llvm/test/CodeGen/BPF/CORE/intrinsic-fieldinfo-lshift-1.ll b/llvm/test/CodeGen/BPF/CORE/intrinsic-fieldinfo-lshift-1.ll
new file mode 100644
index 000000000..ca925d917
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/CORE/intrinsic-fieldinfo-lshift-1.ll
@@ -0,0 +1,158 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-EL,CHECK-ALU64 %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-EB,CHECK-ALU64 %s
+; RUN: llc -march=bpfel -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-EL,CHECK-ALU32 %s
+; RUN: llc -march=bpfeb -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-EB,CHECK-ALU32 %s
+; Source code:
+;   typedef struct s1 { int a1:7; int a2:4; int a3:5; int a4:16;} __s1;
+;   union u1 { int b1; __s1 b2; };
+;   enum { FIELD_LSHIFT_U64 = 4, };
+;   int test(union u1 *arg) {
+;     unsigned r1 = __builtin_preserve_field_info(arg->b2.a1, FIELD_LSHIFT_U64);
+;     unsigned r2 = __builtin_preserve_field_info(arg->b2.a2, FIELD_LSHIFT_U64);
+;     unsigned r3 = __builtin_preserve_field_info(arg->b2.a3, FIELD_LSHIFT_U64);
+;     unsigned r4 = __builtin_preserve_field_info(arg->b2.a4, FIELD_LSHIFT_U64);
+;     /* big endian:    r1: 32, r2: 39, r3: 43, r4: 48 */
+;     /* little endian: r1: 57, r2: 53, r3: 48, r4: 32 */
+;     return r1 + r2 + r3 + r4;
+;   }
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm test.c
+
+%union.u1 = type { i32 }
+%struct.s1 = type { i32 }
+
+; Function Attrs: nounwind readnone
+define dso_local i32 @test(%union.u1* %arg) local_unnamed_addr #0 !dbg !11 {
+entry:
+  call void @llvm.dbg.value(metadata %union.u1* %arg, metadata !28, metadata !DIExpression()), !dbg !33
+  %0 = tail call %union.u1* @llvm.preserve.union.access.index.p0s_union.u1s.p0s_union.u1s(%union.u1* %arg, i32 1), !dbg !34, !llvm.preserve.access.index !16
+  %b2 = bitcast %union.u1* %0 to %struct.s1*, !dbg !34
+  %1 = tail call i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.s1s(%struct.s1* %b2, i32 0, i32 0), !dbg !35, !llvm.preserve.access.index !21
+  %2 = tail call i32 @llvm.bpf.preserve.field.info.p0i32(i32* %1, i64 4), !dbg !36
+  call void @llvm.dbg.value(metadata i32 %2, metadata !29, metadata !DIExpression()), !dbg !33
+  %3 = tail call i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.s1s(%struct.s1* %b2, i32 0, i32 1), !dbg !37, !llvm.preserve.access.index !21
+  %4 = tail call i32 @llvm.bpf.preserve.field.info.p0i32(i32* %3, i64 4), !dbg !38
+  call void @llvm.dbg.value(metadata i32 %4, metadata !30, metadata !DIExpression()), !dbg !33
+  %5 = tail call i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.s1s(%struct.s1* %b2, i32 0, i32 2), !dbg !39, !llvm.preserve.access.index !21
+  %6 = tail call i32 @llvm.bpf.preserve.field.info.p0i32(i32* %5, i64 4), !dbg !40
+  call void @llvm.dbg.value(metadata i32 %6, metadata !31, metadata !DIExpression()), !dbg !33
+  %7 = tail call i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.s1s(%struct.s1* %b2, i32 0, i32 3), !dbg !41, !llvm.preserve.access.index !21
+  %8 = tail call i32 @llvm.bpf.preserve.field.info.p0i32(i32* %7, i64 4), !dbg !42
+  call void @llvm.dbg.value(metadata i32 %8, metadata !32, metadata !DIExpression()), !dbg !33
+  %add = add i32 %4, %2, !dbg !43
+  %add4 = add i32 %add, %6, !dbg !44
+  %add5 = add i32 %add4, %8, !dbg !45
+  ret i32 %add5, !dbg !46
+}
+
+; CHECK-EL:          r1 = 57
+; CHECK-EL:          r0 = 53
+; CHECK-EB:          r1 = 32
+; CHECK-EB:          r0 = 39
+; CHECK-ALU64:       r0 += r1
+; CHECK-ALU32:       w0 += w1
+; CHECK-EL:          r1 = 48
+; CHECK-EB:          r1 = 43
+; CHECK-ALU64:       r0 += r1
+; CHECK-ALU32:       w0 += w1
+; CHECK-EL:          r1 = 32
+; CHECK-EB:          r1 = 48
+; CHECK-ALU64:       r0 += r1
+; CHECK-ALU32:       w0 += w1
+; CHECK:             exit
+
+; CHECK:             .long   1                       # BTF_KIND_UNION(id = 2)
+; CHECK:             .ascii  "u1"                    # string offset=1
+; CHECK:             .ascii  ".text"                 # string offset=43
+; CHECK:             .ascii  "0:1:0"                 # string offset=49
+; CHECK:             .ascii  "0:1:1"                 # string offset=92
+; CHECK:             .ascii  "0:1:2"                 # string offset=98
+; CHECK:             .ascii  "0:1:3"                 # string offset=104
+
+; CHECK:             .long   16                      # FieldReloc
+; CHECK-NEXT:        .long   43                      # Field reloc section string offset=43
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   49
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   92
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   98
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   104
+; CHECK-NEXT:        .long   4
+
+; Function Attrs: nounwind readnone
+declare %union.u1* @llvm.preserve.union.access.index.p0s_union.u1s.p0s_union.u1s(%union.u1*, i32) #1
+
+; Function Attrs: nounwind readnone
+declare i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.s1s(%struct.s1*, i32, i32) #1
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.bpf.preserve.field.info.p0i32(i32*, i64) #1
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.value(metadata, metadata, metadata) #2
+
+attributes #0 = { nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone }
+attributes #2 = { nounwind readnone speculatable willreturn }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!7, !8, !9}
+!llvm.ident = !{!10}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 10.0.0 (https://github.com/llvm/llvm-project.git 5635073377f153f7f2ff9b34c77af3c79885ff4a)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "test.c", directory: "/tmp/home/yhs/work/tests/core")
+!2 = !{!3}
+!3 = !DICompositeType(tag: DW_TAG_enumeration_type, file: !1, line: 3, baseType: !4, size: 32, elements: !5)
+!4 = !DIBasicType(name: "unsigned int", size: 32, encoding: DW_ATE_unsigned)
+!5 = !{!6}
+!6 = !DIEnumerator(name: "FIELD_LSHIFT_U64", value: 4, isUnsigned: true)
+!7 = !{i32 2, !"Dwarf Version", i32 4}
+!8 = !{i32 2, !"Debug Info Version", i32 3}
+!9 = !{i32 1, !"wchar_size", i32 4}
+!10 = !{!"clang version 10.0.0 (https://github.com/llvm/llvm-project.git 5635073377f153f7f2ff9b34c77af3c79885ff4a)"}
+!11 = distinct !DISubprogram(name: "test", scope: !1, file: !1, line: 4, type: !12, scopeLine: 4, flags: DIFlagPrototyped, isDefinition: true, isOptimized: true, unit: !0, retainedNodes: !27)
+!12 = !DISubroutineType(types: !13)
+!13 = !{!14, !15}
+!14 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!15 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !16, size: 64)
+!16 = distinct !DICompositeType(tag: DW_TAG_union_type, name: "u1", file: !1, line: 2, size: 32, elements: !17)
+!17 = !{!18, !19}
+!18 = !DIDerivedType(tag: DW_TAG_member, name: "b1", scope: !16, file: !1, line: 2, baseType: !14, size: 32)
+!19 = !DIDerivedType(tag: DW_TAG_member, name: "b2", scope: !16, file: !1, line: 2, baseType: !20, size: 32)
+!20 = !DIDerivedType(tag: DW_TAG_typedef, name: "__s1", file: !1, line: 1, baseType: !21)
+!21 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "s1", file: !1, line: 1, size: 32, elements: !22)
+!22 = !{!23, !24, !25, !26}
+!23 = !DIDerivedType(tag: DW_TAG_member, name: "a1", scope: !21, file: !1, line: 1, baseType: !14, size: 7, flags: DIFlagBitField, extraData: i64 0)
+!24 = !DIDerivedType(tag: DW_TAG_member, name: "a2", scope: !21, file: !1, line: 1, baseType: !14, size: 4, offset: 7, flags: DIFlagBitField, extraData: i64 0)
+!25 = !DIDerivedType(tag: DW_TAG_member, name: "a3", scope: !21, file: !1, line: 1, baseType: !14, size: 5, offset: 11, flags: DIFlagBitField, extraData: i64 0)
+!26 = !DIDerivedType(tag: DW_TAG_member, name: "a4", scope: !21, file: !1, line: 1, baseType: !14, size: 16, offset: 16, flags: DIFlagBitField, extraData: i64 0)
+!27 = !{!28, !29, !30, !31, !32}
+!28 = !DILocalVariable(name: "arg", arg: 1, scope: !11, file: !1, line: 4, type: !15)
+!29 = !DILocalVariable(name: "r1", scope: !11, file: !1, line: 5, type: !4)
+!30 = !DILocalVariable(name: "r2", scope: !11, file: !1, line: 6, type: !4)
+!31 = !DILocalVariable(name: "r3", scope: !11, file: !1, line: 7, type: !4)
+!32 = !DILocalVariable(name: "r4", scope: !11, file: !1, line: 8, type: !4)
+!33 = !DILocation(line: 0, scope: !11)
+!34 = !DILocation(line: 5, column: 52, scope: !11)
+!35 = !DILocation(line: 5, column: 55, scope: !11)
+!36 = !DILocation(line: 5, column: 17, scope: !11)
+!37 = !DILocation(line: 6, column: 55, scope: !11)
+!38 = !DILocation(line: 6, column: 17, scope: !11)
+!39 = !DILocation(line: 7, column: 55, scope: !11)
+!40 = !DILocation(line: 7, column: 17, scope: !11)
+!41 = !DILocation(line: 8, column: 55, scope: !11)
+!42 = !DILocation(line: 8, column: 17, scope: !11)
+!43 = !DILocation(line: 11, column: 13, scope: !11)
+!44 = !DILocation(line: 11, column: 18, scope: !11)
+!45 = !DILocation(line: 11, column: 23, scope: !11)
+!46 = !DILocation(line: 11, column: 3, scope: !11)
diff --git a/llvm/test/CodeGen/BPF/CORE/intrinsic-fieldinfo-lshift-2.ll b/llvm/test/CodeGen/BPF/CORE/intrinsic-fieldinfo-lshift-2.ll
new file mode 100644
index 000000000..d4cb4ec5c
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/CORE/intrinsic-fieldinfo-lshift-2.ll
@@ -0,0 +1,125 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-ALU64 %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-ALU64 %s
+; RUN: llc -march=bpfel -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-ALU32 %s
+; RUN: llc -march=bpfeb -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-ALU32 %s
+; Source code:
+;   typedef struct s1 { int a1; short a2; } __s1;
+;   union u1 { int b1; __s1 b2; };
+;   enum { FIELD_LSHIFT_U64 = 4, };
+;   int test(union u1 *arg) {
+;     unsigned r1 = __builtin_preserve_field_info(arg->b2.a1, FIELD_LSHIFT_U64);
+;     unsigned r2 = __builtin_preserve_field_info(arg->b2.a2, FIELD_LSHIFT_U64);
+;     /* big endian:    r1: 32, r2: 48 */
+;     /* little endian: r1: 32, r2: 48 */
+;     return r1 + r2;
+;   }
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm test.c
+
+%union.u1 = type { %struct.s1 }
+%struct.s1 = type { i32, i16 }
+
+; Function Attrs: nounwind readnone
+define dso_local i32 @test(%union.u1* %arg) local_unnamed_addr #0 !dbg !11 {
+entry:
+  call void @llvm.dbg.value(metadata %union.u1* %arg, metadata !27, metadata !DIExpression()), !dbg !30
+  %0 = tail call %union.u1* @llvm.preserve.union.access.index.p0s_union.u1s.p0s_union.u1s(%union.u1* %arg, i32 1), !dbg !31, !llvm.preserve.access.index !16
+  %b2 = getelementptr %union.u1, %union.u1* %0, i64 0, i32 0, !dbg !31
+  %1 = tail call i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.s1s(%struct.s1* %b2, i32 0, i32 0), !dbg !32, !llvm.preserve.access.index !21
+  %2 = tail call i32 @llvm.bpf.preserve.field.info.p0i32(i32* %1, i64 4), !dbg !33
+  call void @llvm.dbg.value(metadata i32 %2, metadata !28, metadata !DIExpression()), !dbg !30
+  %3 = tail call i16* @llvm.preserve.struct.access.index.p0i16.p0s_struct.s1s(%struct.s1* %b2, i32 1, i32 1), !dbg !34, !llvm.preserve.access.index !21
+  %4 = tail call i32 @llvm.bpf.preserve.field.info.p0i16(i16* %3, i64 4), !dbg !35
+  call void @llvm.dbg.value(metadata i32 %4, metadata !29, metadata !DIExpression()), !dbg !30
+  %add = add i32 %4, %2, !dbg !36
+  ret i32 %add, !dbg !37
+}
+
+; CHECK:             r1 = 32
+; CHECK:             r0 = 48
+; CHECK-ALU64:       r0 += r1
+; CHECK-ALU32:       w0 += w1
+; CHECK:             exit
+
+; CHECK:             .long   1                       # BTF_KIND_UNION(id = 2)
+; CHECK:             .ascii  "u1"                    # string offset=1
+; CHECK:             .ascii  ".text"                 # string offset=43
+; CHECK:             .ascii  "0:1:0"                 # string offset=49
+; CHECK:             .ascii  "0:1:1"                 # string offset=92
+
+; CHECK:             .long   16                      # FieldReloc
+; CHECK-NEXT:        .long   43                      # Field reloc section string offset=43
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   49
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   92
+; CHECK-NEXT:        .long   4
+
+; Function Attrs: nounwind readnone
+declare %union.u1* @llvm.preserve.union.access.index.p0s_union.u1s.p0s_union.u1s(%union.u1*, i32) #1
+
+; Function Attrs: nounwind readnone
+declare i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.s1s(%struct.s1*, i32, i32) #1
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.bpf.preserve.field.info.p0i32(i32*, i64) #1
+
+; Function Attrs: nounwind readnone
+declare i16* @llvm.preserve.struct.access.index.p0i16.p0s_struct.s1s(%struct.s1*, i32, i32) #1
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.bpf.preserve.field.info.p0i16(i16*, i64) #1
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.value(metadata, metadata, metadata) #2
+
+attributes #0 = { nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone }
+attributes #2 = { nounwind readnone speculatable willreturn }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!7, !8, !9}
+!llvm.ident = !{!10}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 10.0.0 (https://github.com/llvm/llvm-project.git 5635073377f153f7f2ff9b34c77af3c79885ff4a)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "test.c", directory: "/tmp/home/yhs/work/tests/core")
+!2 = !{!3}
+!3 = !DICompositeType(tag: DW_TAG_enumeration_type, file: !1, line: 3, baseType: !4, size: 32, elements: !5)
+!4 = !DIBasicType(name: "unsigned int", size: 32, encoding: DW_ATE_unsigned)
+!5 = !{!6}
+!6 = !DIEnumerator(name: "FIELD_LSHIFT_U64", value: 4, isUnsigned: true)
+!7 = !{i32 2, !"Dwarf Version", i32 4}
+!8 = !{i32 2, !"Debug Info Version", i32 3}
+!9 = !{i32 1, !"wchar_size", i32 4}
+!10 = !{!"clang version 10.0.0 (https://github.com/llvm/llvm-project.git 5635073377f153f7f2ff9b34c77af3c79885ff4a)"}
+!11 = distinct !DISubprogram(name: "test", scope: !1, file: !1, line: 4, type: !12, scopeLine: 4, flags: DIFlagPrototyped, isDefinition: true, isOptimized: true, unit: !0, retainedNodes: !26)
+!12 = !DISubroutineType(types: !13)
+!13 = !{!14, !15}
+!14 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!15 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !16, size: 64)
+!16 = distinct !DICompositeType(tag: DW_TAG_union_type, name: "u1", file: !1, line: 2, size: 64, elements: !17)
+!17 = !{!18, !19}
+!18 = !DIDerivedType(tag: DW_TAG_member, name: "b1", scope: !16, file: !1, line: 2, baseType: !14, size: 32)
+!19 = !DIDerivedType(tag: DW_TAG_member, name: "b2", scope: !16, file: !1, line: 2, baseType: !20, size: 64)
+!20 = !DIDerivedType(tag: DW_TAG_typedef, name: "__s1", file: !1, line: 1, baseType: !21)
+!21 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "s1", file: !1, line: 1, size: 64, elements: !22)
+!22 = !{!23, !24}
+!23 = !DIDerivedType(tag: DW_TAG_member, name: "a1", scope: !21, file: !1, line: 1, baseType: !14, size: 32)
+!24 = !DIDerivedType(tag: DW_TAG_member, name: "a2", scope: !21, file: !1, line: 1, baseType: !25, size: 16, offset: 32)
+!25 = !DIBasicType(name: "short", size: 16, encoding: DW_ATE_signed)
+!26 = !{!27, !28, !29}
+!27 = !DILocalVariable(name: "arg", arg: 1, scope: !11, file: !1, line: 4, type: !15)
+!28 = !DILocalVariable(name: "r1", scope: !11, file: !1, line: 5, type: !4)
+!29 = !DILocalVariable(name: "r2", scope: !11, file: !1, line: 6, type: !4)
+!30 = !DILocation(line: 0, scope: !11)
+!31 = !DILocation(line: 5, column: 52, scope: !11)
+!32 = !DILocation(line: 5, column: 55, scope: !11)
+!33 = !DILocation(line: 5, column: 17, scope: !11)
+!34 = !DILocation(line: 6, column: 55, scope: !11)
+!35 = !DILocation(line: 6, column: 17, scope: !11)
+!36 = !DILocation(line: 9, column: 13, scope: !11)
+!37 = !DILocation(line: 9, column: 3, scope: !11)
diff --git a/llvm/test/CodeGen/BPF/CORE/intrinsic-fieldinfo-rshift-1.ll b/llvm/test/CodeGen/BPF/CORE/intrinsic-fieldinfo-rshift-1.ll
new file mode 100644
index 000000000..3e3be9626
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/CORE/intrinsic-fieldinfo-rshift-1.ll
@@ -0,0 +1,153 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-ALU64 %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-ALU64 %s
+; RUN: llc -march=bpfel -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-ALU32 %s
+; RUN: llc -march=bpfeb -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-ALU32 %s
+; Source code:
+;   typedef struct s1 { int a1:7; int a2:4; int a3:5; int a4:16;} __s1;
+;   union u1 { int b1; __s1 b2; };
+;   enum { FIELD_RSHIFT_U64 = 5, };
+;   int test(union u1 *arg) {
+;     unsigned r1 = __builtin_preserve_field_info(arg->b2.a1, FIELD_RSHIFT_U64);
+;     unsigned r2 = __builtin_preserve_field_info(arg->b2.a2, FIELD_RSHIFT_U64);
+;     unsigned r3 = __builtin_preserve_field_info(arg->b2.a3, FIELD_RSHIFT_U64);
+;     unsigned r4 = __builtin_preserve_field_info(arg->b2.a4, FIELD_RSHIFT_U64);
+;     /* r1: 57, r2: 60, r3: 59, r4: 48 */
+;     return r1 + r2 + r3 + r4;
+;   }
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm test.c
+
+%union.u1 = type { i32 }
+%struct.s1 = type { i32 }
+
+; Function Attrs: nounwind readnone
+define dso_local i32 @test(%union.u1* %arg) local_unnamed_addr #0 !dbg !11 {
+entry:
+  call void @llvm.dbg.value(metadata %union.u1* %arg, metadata !28, metadata !DIExpression()), !dbg !33
+  %0 = tail call %union.u1* @llvm.preserve.union.access.index.p0s_union.u1s.p0s_union.u1s(%union.u1* %arg, i32 1), !dbg !34, !llvm.preserve.access.index !16
+  %b2 = bitcast %union.u1* %0 to %struct.s1*, !dbg !34
+  %1 = tail call i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.s1s(%struct.s1* %b2, i32 0, i32 0), !dbg !35, !llvm.preserve.access.index !21
+  %2 = tail call i32 @llvm.bpf.preserve.field.info.p0i32(i32* %1, i64 5), !dbg !36
+  call void @llvm.dbg.value(metadata i32 %2, metadata !29, metadata !DIExpression()), !dbg !33
+  %3 = tail call i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.s1s(%struct.s1* %b2, i32 0, i32 1), !dbg !37, !llvm.preserve.access.index !21
+  %4 = tail call i32 @llvm.bpf.preserve.field.info.p0i32(i32* %3, i64 5), !dbg !38
+  call void @llvm.dbg.value(metadata i32 %4, metadata !30, metadata !DIExpression()), !dbg !33
+  %5 = tail call i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.s1s(%struct.s1* %b2, i32 0, i32 2), !dbg !39, !llvm.preserve.access.index !21
+  %6 = tail call i32 @llvm.bpf.preserve.field.info.p0i32(i32* %5, i64 5), !dbg !40
+  call void @llvm.dbg.value(metadata i32 %6, metadata !31, metadata !DIExpression()), !dbg !33
+  %7 = tail call i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.s1s(%struct.s1* %b2, i32 0, i32 3), !dbg !41, !llvm.preserve.access.index !21
+  %8 = tail call i32 @llvm.bpf.preserve.field.info.p0i32(i32* %7, i64 5), !dbg !42
+  call void @llvm.dbg.value(metadata i32 %8, metadata !32, metadata !DIExpression()), !dbg !33
+  %add = add i32 %4, %2, !dbg !43
+  %add4 = add i32 %add, %6, !dbg !44
+  %add5 = add i32 %add4, %8, !dbg !45
+  ret i32 %add5, !dbg !46
+}
+
+; CHECK:             r1 = 57
+; CHECK:             r0 = 60
+; CHECK-ALU64:       r0 += r1
+; CHECK-ALU32:       w0 += w1
+; CHECK:             r1 = 59
+; CHECK-ALU64:       r0 += r1
+; CHECK-ALU32:       w0 += w1
+; CHECK:             r1 = 48
+; CHECK-ALU64:       r0 += r1
+; CHECK-ALU32:       w0 += w1
+; CHECK:             exit
+
+; CHECK:             .long   1                       # BTF_KIND_UNION(id = 2)
+; CHECK:             .ascii  "u1"                    # string offset=1
+; CHECK:             .ascii  ".text"                 # string offset=43
+; CHECK:             .ascii  "0:1:0"                 # string offset=49
+; CHECK:             .ascii  "0:1:1"                 # string offset=92
+; CHECK:             .ascii  "0:1:2"                 # string offset=98
+; CHECK:             .ascii  "0:1:3"                 # string offset=104
+
+; CHECK:             .long   16                      # FieldReloc
+; CHECK-NEXT:        .long   43                      # Field reloc section string offset=43
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   49
+; CHECK-NEXT:        .long   5
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   92
+; CHECK-NEXT:        .long   5
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   98
+; CHECK-NEXT:        .long   5
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   104
+; CHECK-NEXT:        .long   5
+
+; Function Attrs: nounwind readnone
+declare %union.u1* @llvm.preserve.union.access.index.p0s_union.u1s.p0s_union.u1s(%union.u1*, i32) #1
+
+; Function Attrs: nounwind readnone
+declare i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.s1s(%struct.s1*, i32, i32) #1
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.bpf.preserve.field.info.p0i32(i32*, i64) #1
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.value(metadata, metadata, metadata) #2
+
+attributes #0 = { nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone }
+attributes #2 = { nounwind readnone speculatable willreturn }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!7, !8, !9}
+!llvm.ident = !{!10}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 10.0.0 (https://github.com/llvm/llvm-project.git 4a60741b74384f14b21fdc0131ede326438840ab)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "test.c", directory: "/tmp/home/yhs/work/tests/core")
+!2 = !{!3}
+!3 = !DICompositeType(tag: DW_TAG_enumeration_type, file: !1, line: 3, baseType: !4, size: 32, elements: !5)
+!4 = !DIBasicType(name: "unsigned int", size: 32, encoding: DW_ATE_unsigned)
+!5 = !{!6}
+!6 = !DIEnumerator(name: "FIELD_RSHIFT_U64", value: 5, isUnsigned: true)
+!7 = !{i32 2, !"Dwarf Version", i32 4}
+!8 = !{i32 2, !"Debug Info Version", i32 3}
+!9 = !{i32 1, !"wchar_size", i32 4}
+!10 = !{!"clang version 10.0.0 (https://github.com/llvm/llvm-project.git 4a60741b74384f14b21fdc0131ede326438840ab)"}
+!11 = distinct !DISubprogram(name: "test", scope: !1, file: !1, line: 4, type: !12, scopeLine: 4, flags: DIFlagPrototyped, isDefinition: true, isOptimized: true, unit: !0, retainedNodes: !27)
+!12 = !DISubroutineType(types: !13)
+!13 = !{!14, !15}
+!14 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!15 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !16, size: 64)
+!16 = distinct !DICompositeType(tag: DW_TAG_union_type, name: "u1", file: !1, line: 2, size: 32, elements: !17)
+!17 = !{!18, !19}
+!18 = !DIDerivedType(tag: DW_TAG_member, name: "b1", scope: !16, file: !1, line: 2, baseType: !14, size: 32)
+!19 = !DIDerivedType(tag: DW_TAG_member, name: "b2", scope: !16, file: !1, line: 2, baseType: !20, size: 32)
+!20 = !DIDerivedType(tag: DW_TAG_typedef, name: "__s1", file: !1, line: 1, baseType: !21)
+!21 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "s1", file: !1, line: 1, size: 32, elements: !22)
+!22 = !{!23, !24, !25, !26}
+!23 = !DIDerivedType(tag: DW_TAG_member, name: "a1", scope: !21, file: !1, line: 1, baseType: !14, size: 7, flags: DIFlagBitField, extraData: i64 0)
+!24 = !DIDerivedType(tag: DW_TAG_member, name: "a2", scope: !21, file: !1, line: 1, baseType: !14, size: 4, offset: 7, flags: DIFlagBitField, extraData: i64 0)
+!25 = !DIDerivedType(tag: DW_TAG_member, name: "a3", scope: !21, file: !1, line: 1, baseType: !14, size: 5, offset: 11, flags: DIFlagBitField, extraData: i64 0)
+!26 = !DIDerivedType(tag: DW_TAG_member, name: "a4", scope: !21, file: !1, line: 1, baseType: !14, size: 16, offset: 16, flags: DIFlagBitField, extraData: i64 0)
+!27 = !{!28, !29, !30, !31, !32}
+!28 = !DILocalVariable(name: "arg", arg: 1, scope: !11, file: !1, line: 4, type: !15)
+!29 = !DILocalVariable(name: "r1", scope: !11, file: !1, line: 5, type: !4)
+!30 = !DILocalVariable(name: "r2", scope: !11, file: !1, line: 6, type: !4)
+!31 = !DILocalVariable(name: "r3", scope: !11, file: !1, line: 7, type: !4)
+!32 = !DILocalVariable(name: "r4", scope: !11, file: !1, line: 8, type: !4)
+!33 = !DILocation(line: 0, scope: !11)
+!34 = !DILocation(line: 5, column: 52, scope: !11)
+!35 = !DILocation(line: 5, column: 55, scope: !11)
+!36 = !DILocation(line: 5, column: 17, scope: !11)
+!37 = !DILocation(line: 6, column: 55, scope: !11)
+!38 = !DILocation(line: 6, column: 17, scope: !11)
+!39 = !DILocation(line: 7, column: 55, scope: !11)
+!40 = !DILocation(line: 7, column: 17, scope: !11)
+!41 = !DILocation(line: 8, column: 55, scope: !11)
+!42 = !DILocation(line: 8, column: 17, scope: !11)
+!43 = !DILocation(line: 10, column: 13, scope: !11)
+!44 = !DILocation(line: 10, column: 18, scope: !11)
+!45 = !DILocation(line: 10, column: 23, scope: !11)
+!46 = !DILocation(line: 10, column: 3, scope: !11)
diff --git a/llvm/test/CodeGen/BPF/CORE/intrinsic-fieldinfo-rshift-2.ll b/llvm/test/CodeGen/BPF/CORE/intrinsic-fieldinfo-rshift-2.ll
new file mode 100644
index 000000000..638f37023
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/CORE/intrinsic-fieldinfo-rshift-2.ll
@@ -0,0 +1,124 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-ALU64 %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-ALU64 %s
+; RUN: llc -march=bpfel -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-ALU32 %s
+; RUN: llc -march=bpfeb -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-ALU32 %s
+; Source code:
+;   typedef struct s1 { int a1; char a2; } __s1;
+;   union u1 { int b1; __s1 b2; };
+;   enum { FIELD_RSHIFT_U64 = 5, };
+;   int test(union u1 *arg) {
+;     unsigned r1 = __builtin_preserve_field_info(arg->b2.a1, FIELD_RSHIFT_U64);
+;     unsigned r2 = __builtin_preserve_field_info(arg->b2.a2, FIELD_RSHIFT_U64);
+;     /* r1: 32, r2: 56 */
+;     return r1 + r2;
+;   }
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm test.c
+
+%union.u1 = type { %struct.s1 }
+%struct.s1 = type { i32, i8 }
+
+; Function Attrs: nounwind readnone
+define dso_local i32 @test(%union.u1* %arg) local_unnamed_addr #0 !dbg !11 {
+entry:
+  call void @llvm.dbg.value(metadata %union.u1* %arg, metadata !27, metadata !DIExpression()), !dbg !30
+  %0 = tail call %union.u1* @llvm.preserve.union.access.index.p0s_union.u1s.p0s_union.u1s(%union.u1* %arg, i32 1), !dbg !31, !llvm.preserve.access.index !16
+  %b2 = getelementptr inbounds %union.u1, %union.u1* %0, i64 0, i32 0, !dbg !31
+  %1 = tail call i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.s1s(%struct.s1* %b2, i32 0, i32 0), !dbg !32, !llvm.preserve.access.index !21
+  %2 = tail call i32 @llvm.bpf.preserve.field.info.p0i32(i32* %1, i64 5), !dbg !33
+  call void @llvm.dbg.value(metadata i32 %2, metadata !28, metadata !DIExpression()), !dbg !30
+  %3 = tail call i8* @llvm.preserve.struct.access.index.p0i8.p0s_struct.s1s(%struct.s1* %b2, i32 1, i32 1), !dbg !34, !llvm.preserve.access.index !21
+  %4 = tail call i32 @llvm.bpf.preserve.field.info.p0i8(i8* %3, i64 5), !dbg !35
+  call void @llvm.dbg.value(metadata i32 %4, metadata !29, metadata !DIExpression()), !dbg !30
+  %add = add i32 %4, %2, !dbg !36
+  ret i32 %add, !dbg !37
+}
+
+; CHECK:             r1 = 32
+; CHECK:             r0 = 56
+; CHECK-ALU64:       r0 += r1
+; CHECK-ALU32:       w0 += w1
+; CHECK:             exit
+
+; CHECK:             .long   1                       # BTF_KIND_UNION(id = 2)
+; CHECK:             .ascii  "u1"                    # string offset=1
+; CHECK:             .ascii  ".text"                 # string offset=42
+; CHECK:             .ascii  "0:1:0"                 # string offset=48
+; CHECK:             .ascii  "0:1:1"                 # string offset=91
+
+; CHECK:             .long   16                      # FieldReloc
+; CHECK-NEXT:        .long   42                      # Field reloc section string offset=42
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   48
+; CHECK-NEXT:        .long   5
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   91
+; CHECK-NEXT:        .long   5
+
+; Function Attrs: nounwind readnone
+declare %union.u1* @llvm.preserve.union.access.index.p0s_union.u1s.p0s_union.u1s(%union.u1*, i32) #1
+
+; Function Attrs: nounwind readnone
+declare i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.s1s(%struct.s1*, i32, i32) #1
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.bpf.preserve.field.info.p0i32(i32*, i64) #1
+
+; Function Attrs: nounwind readnone
+declare i8* @llvm.preserve.struct.access.index.p0i8.p0s_struct.s1s(%struct.s1*, i32, i32) #1
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.bpf.preserve.field.info.p0i8(i8*, i64) #1
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.value(metadata, metadata, metadata) #2
+
+attributes #0 = { nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone }
+attributes #2 = { nounwind readnone speculatable willreturn }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!7, !8, !9}
+!llvm.ident = !{!10}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 10.0.0 (https://github.com/llvm/llvm-project.git 4a60741b74384f14b21fdc0131ede326438840ab)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "test.c", directory: "/tmp/home/yhs/work/tests/core")
+!2 = !{!3}
+!3 = !DICompositeType(tag: DW_TAG_enumeration_type, file: !1, line: 3, baseType: !4, size: 32, elements: !5)
+!4 = !DIBasicType(name: "unsigned int", size: 32, encoding: DW_ATE_unsigned)
+!5 = !{!6}
+!6 = !DIEnumerator(name: "FIELD_RSHIFT_U64", value: 5, isUnsigned: true)
+!7 = !{i32 2, !"Dwarf Version", i32 4}
+!8 = !{i32 2, !"Debug Info Version", i32 3}
+!9 = !{i32 1, !"wchar_size", i32 4}
+!10 = !{!"clang version 10.0.0 (https://github.com/llvm/llvm-project.git 4a60741b74384f14b21fdc0131ede326438840ab)"}
+!11 = distinct !DISubprogram(name: "test", scope: !1, file: !1, line: 4, type: !12, scopeLine: 4, flags: DIFlagPrototyped, isDefinition: true, isOptimized: true, unit: !0, retainedNodes: !26)
+!12 = !DISubroutineType(types: !13)
+!13 = !{!14, !15}
+!14 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!15 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !16, size: 64)
+!16 = distinct !DICompositeType(tag: DW_TAG_union_type, name: "u1", file: !1, line: 2, size: 64, elements: !17)
+!17 = !{!18, !19}
+!18 = !DIDerivedType(tag: DW_TAG_member, name: "b1", scope: !16, file: !1, line: 2, baseType: !14, size: 32)
+!19 = !DIDerivedType(tag: DW_TAG_member, name: "b2", scope: !16, file: !1, line: 2, baseType: !20, size: 64)
+!20 = !DIDerivedType(tag: DW_TAG_typedef, name: "__s1", file: !1, line: 1, baseType: !21)
+!21 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "s1", file: !1, line: 1, size: 64, elements: !22)
+!22 = !{!23, !24}
+!23 = !DIDerivedType(tag: DW_TAG_member, name: "a1", scope: !21, file: !1, line: 1, baseType: !14, size: 32)
+!24 = !DIDerivedType(tag: DW_TAG_member, name: "a2", scope: !21, file: !1, line: 1, baseType: !25, size: 8, offset: 32)
+!25 = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_signed_char)
+!26 = !{!27, !28, !29}
+!27 = !DILocalVariable(name: "arg", arg: 1, scope: !11, file: !1, line: 4, type: !15)
+!28 = !DILocalVariable(name: "r1", scope: !11, file: !1, line: 5, type: !4)
+!29 = !DILocalVariable(name: "r2", scope: !11, file: !1, line: 6, type: !4)
+!30 = !DILocation(line: 0, scope: !11)
+!31 = !DILocation(line: 5, column: 52, scope: !11)
+!32 = !DILocation(line: 5, column: 55, scope: !11)
+!33 = !DILocation(line: 5, column: 17, scope: !11)
+!34 = !DILocation(line: 6, column: 55, scope: !11)
+!35 = !DILocation(line: 6, column: 17, scope: !11)
+!36 = !DILocation(line: 8, column: 13, scope: !11)
+!37 = !DILocation(line: 8, column: 3, scope: !11)
diff --git a/llvm/test/CodeGen/BPF/CORE/intrinsic-fieldinfo-rshift-3.ll b/llvm/test/CodeGen/BPF/CORE/intrinsic-fieldinfo-rshift-3.ll
new file mode 100644
index 000000000..ac821a95e
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/CORE/intrinsic-fieldinfo-rshift-3.ll
@@ -0,0 +1,134 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-ALU64 %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-ALU64 %s
+; RUN: llc -march=bpfel -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-ALU32 %s
+; RUN: llc -march=bpfeb -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-ALU32 %s
+; Source code:
+;   typedef struct s1 { char a1 [5][5]; } __s1;
+;   union u1 { int b1; __s1 b2; };
+;   enum { FIELD_RSHIFT_U64 = 5, };
+;   int test(union u1 *arg) {
+;     unsigned r1 = __builtin_preserve_field_info(arg->b2.a1[3], FIELD_RSHIFT_U64);
+;     unsigned r2 = __builtin_preserve_field_info(arg->b2.a1[3][3], FIELD_RSHIFT_U64);
+;     /* r1 : 24, r2 : 56 */
+;     return r1 + r2;
+;   }
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm test.c
+
+%union.u1 = type { i32, [24 x i8] }
+%struct.s1 = type { [5 x [5 x i8]] }
+
+; Function Attrs: nounwind readnone
+define dso_local i32 @test(%union.u1* %arg) local_unnamed_addr #0 !dbg !18 {
+entry:
+  call void @llvm.dbg.value(metadata %union.u1* %arg, metadata !32, metadata !DIExpression()), !dbg !35
+  %0 = tail call %union.u1* @llvm.preserve.union.access.index.p0s_union.u1s.p0s_union.u1s(%union.u1* %arg, i32 1), !dbg !36, !llvm.preserve.access.index !23
+  %b2 = bitcast %union.u1* %0 to %struct.s1*, !dbg !36
+  %1 = tail call [5 x [5 x i8]]* @llvm.preserve.struct.access.index.p0a5a5i8.p0s_struct.s1s(%struct.s1* %b2, i32 0, i32 0), !dbg !37, !llvm.preserve.access.index !28
+  %2 = tail call [5 x i8]* @llvm.preserve.array.access.index.p0a5i8.p0a5a5i8([5 x [5 x i8]]* %1, i32 1, i32 3), !dbg !38, !llvm.preserve.access.index !8
+  %3 = tail call i32 @llvm.bpf.preserve.field.info.p0a5i8([5 x i8]* %2, i64 5), !dbg !39
+  call void @llvm.dbg.value(metadata i32 %3, metadata !33, metadata !DIExpression()), !dbg !35
+  %4 = tail call i8* @llvm.preserve.array.access.index.p0i8.p0a5i8([5 x i8]* %2, i32 1, i32 3), !dbg !40, !llvm.preserve.access.index !12
+  %5 = tail call i32 @llvm.bpf.preserve.field.info.p0i8(i8* %4, i64 5), !dbg !41
+  call void @llvm.dbg.value(metadata i32 %5, metadata !34, metadata !DIExpression()), !dbg !35
+  %add = add i32 %5, %3, !dbg !42
+  ret i32 %add, !dbg !43
+}
+
+; CHECK:             r1 = 24
+; CHECK:             r0 = 56
+; CHECK-ALU64:       r0 += r1
+; CHECK-ALU32:       w0 += w1
+; CHECK:             exit
+
+; CHECK:             .long   1                       # BTF_KIND_UNION(id = 2)
+; CHECK:             .ascii  "u1"                    # string offset=1
+; CHECK:             .ascii  ".text"                 # string offset=59
+; CHECK:             .ascii  "0:1:0:3"               # string offset=65
+; CHECK:             .ascii  "0:1:0:3:3"             # string offset=110
+
+; CHECK:             .long   16                      # FieldReloc
+; CHECK-NEXT:        .long   59                      # Field reloc section string offset=59
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   65
+; CHECK-NEXT:        .long   5
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   110
+; CHECK-NEXT:        .long   5
+
+; Function Attrs: nounwind readnone
+declare %union.u1* @llvm.preserve.union.access.index.p0s_union.u1s.p0s_union.u1s(%union.u1*, i32) #1
+
+; Function Attrs: nounwind readnone
+declare [5 x [5 x i8]]* @llvm.preserve.struct.access.index.p0a5a5i8.p0s_struct.s1s(%struct.s1*, i32, i32) #1
+
+; Function Attrs: nounwind readnone
+declare [5 x i8]* @llvm.preserve.array.access.index.p0a5i8.p0a5a5i8([5 x [5 x i8]]*, i32, i32) #1
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.bpf.preserve.field.info.p0a5i8([5 x i8]*, i64) #1
+
+; Function Attrs: nounwind readnone
+declare i8* @llvm.preserve.array.access.index.p0i8.p0a5i8([5 x i8]*, i32, i32) #1
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.bpf.preserve.field.info.p0i8(i8*, i64) #1
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.value(metadata, metadata, metadata) #2
+
+attributes #0 = { nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone }
+attributes #2 = { nounwind readnone speculatable willreturn }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!14, !15, !16}
+!llvm.ident = !{!17}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 10.0.0 (https://github.com/llvm/llvm-project.git c1e02f16f1105ffaf1c35ee8bc38b7d6db5c6ea9)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !7, nameTableKind: None)
+!1 = !DIFile(filename: "test.c", directory: "/tmp/home/yhs/work/tests/core")
+!2 = !{!3}
+!3 = !DICompositeType(tag: DW_TAG_enumeration_type, file: !1, line: 3, baseType: !4, size: 32, elements: !5)
+!4 = !DIBasicType(name: "unsigned int", size: 32, encoding: DW_ATE_unsigned)
+!5 = !{!6}
+!6 = !DIEnumerator(name: "FIELD_RSHIFT_U64", value: 5, isUnsigned: true)
+!7 = !{!8, !12}
+!8 = !DICompositeType(tag: DW_TAG_array_type, baseType: !9, size: 200, elements: !10)
+!9 = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_signed_char)
+!10 = !{!11, !11}
+!11 = !DISubrange(count: 5)
+!12 = !DICompositeType(tag: DW_TAG_array_type, baseType: !9, size: 40, elements: !13)
+!13 = !{!11}
+!14 = !{i32 2, !"Dwarf Version", i32 4}
+!15 = !{i32 2, !"Debug Info Version", i32 3}
+!16 = !{i32 1, !"wchar_size", i32 4}
+!17 = !{!"clang version 10.0.0 (https://github.com/llvm/llvm-project.git c1e02f16f1105ffaf1c35ee8bc38b7d6db5c6ea9)"}
+!18 = distinct !DISubprogram(name: "test", scope: !1, file: !1, line: 4, type: !19, scopeLine: 4, flags: DIFlagPrototyped, isDefinition: true, isOptimized: true, unit: !0, retainedNodes: !31)
+!19 = !DISubroutineType(types: !20)
+!20 = !{!21, !22}
+!21 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!22 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !23, size: 64)
+!23 = distinct !DICompositeType(tag: DW_TAG_union_type, name: "u1", file: !1, line: 2, size: 224, elements: !24)
+!24 = !{!25, !26}
+!25 = !DIDerivedType(tag: DW_TAG_member, name: "b1", scope: !23, file: !1, line: 2, baseType: !21, size: 32)
+!26 = !DIDerivedType(tag: DW_TAG_member, name: "b2", scope: !23, file: !1, line: 2, baseType: !27, size: 200)
+!27 = !DIDerivedType(tag: DW_TAG_typedef, name: "__s1", file: !1, line: 1, baseType: !28)
+!28 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "s1", file: !1, line: 1, size: 200, elements: !29)
+!29 = !{!30}
+!30 = !DIDerivedType(tag: DW_TAG_member, name: "a1", scope: !28, file: !1, line: 1, baseType: !8, size: 200)
+!31 = !{!32, !33, !34}
+!32 = !DILocalVariable(name: "arg", arg: 1, scope: !18, file: !1, line: 4, type: !22)
+!33 = !DILocalVariable(name: "r1", scope: !18, file: !1, line: 5, type: !4)
+!34 = !DILocalVariable(name: "r2", scope: !18, file: !1, line: 6, type: !4)
+!35 = !DILocation(line: 0, scope: !18)
+!36 = !DILocation(line: 5, column: 52, scope: !18)
+!37 = !DILocation(line: 5, column: 55, scope: !18)
+!38 = !DILocation(line: 5, column: 47, scope: !18)
+!39 = !DILocation(line: 5, column: 17, scope: !18)
+!40 = !DILocation(line: 6, column: 47, scope: !18)
+!41 = !DILocation(line: 6, column: 17, scope: !18)
+!42 = !DILocation(line: 8, column: 13, scope: !18)
+!43 = !DILocation(line: 8, column: 3, scope: !18)
diff --git a/llvm/test/CodeGen/BPF/CORE/intrinsic-fieldinfo-signedness-1.ll b/llvm/test/CodeGen/BPF/CORE/intrinsic-fieldinfo-signedness-1.ll
new file mode 100644
index 000000000..e55146540
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/CORE/intrinsic-fieldinfo-signedness-1.ll
@@ -0,0 +1,167 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-ALU64 %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-ALU64 %s
+; RUN: llc -march=bpfel -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-ALU32 %s
+; RUN: llc -march=bpfeb -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-ALU32 %s
+; Source code:
+;   typedef unsigned __uint;
+;   struct s1 { int a1; __uint a2:9; __uint a3:4; };
+;   union u1 { int b1; __uint b2:9; __uint b3:4; };
+;   enum { FIELD_SIGNEDNESS = 3, };
+;   int test(struct s1 *arg1, union u1 *arg2) {
+;     unsigned r1 = __builtin_preserve_field_info(arg1->a1, FIELD_SIGNEDNESS);
+;     unsigned r2 = __builtin_preserve_field_info(arg1->a3, FIELD_SIGNEDNESS);
+;     unsigned r3 = __builtin_preserve_field_info(arg2->b1, FIELD_SIGNEDNESS);
+;     unsigned r4 = __builtin_preserve_field_info(arg2->b3, FIELD_SIGNEDNESS);
+;     return r1 + r2 + r3 + r4;
+;   }
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm test.c
+
+%struct.s1 = type { i32, i16 }
+%union.u1 = type { i32 }
+
+; Function Attrs: nounwind readnone
+define dso_local i32 @test(%struct.s1* %arg1, %union.u1* %arg2) local_unnamed_addr #0 !dbg !11 {
+entry:
+  call void @llvm.dbg.value(metadata %struct.s1* %arg1, metadata !29, metadata !DIExpression()), !dbg !35
+  call void @llvm.dbg.value(metadata %union.u1* %arg2, metadata !30, metadata !DIExpression()), !dbg !35
+  %0 = tail call i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.s1s(%struct.s1* %arg1, i32 0, i32 0), !dbg !36, !llvm.preserve.access.index !16
+  %1 = tail call i32 @llvm.bpf.preserve.field.info.p0i32(i32* %0, i64 3), !dbg !37
+  call void @llvm.dbg.value(metadata i32 %1, metadata !31, metadata !DIExpression()), !dbg !35
+  %2 = tail call i16* @llvm.preserve.struct.access.index.p0i16.p0s_struct.s1s(%struct.s1* %arg1, i32 1, i32 2), !dbg !38, !llvm.preserve.access.index !16
+  %3 = tail call i32 @llvm.bpf.preserve.field.info.p0i16(i16* %2, i64 3), !dbg !39
+  call void @llvm.dbg.value(metadata i32 %3, metadata !32, metadata !DIExpression()), !dbg !35
+  %4 = tail call %union.u1* @llvm.preserve.union.access.index.p0s_union.u1s.p0s_union.u1s(%union.u1* %arg2, i32 0), !dbg !40, !llvm.preserve.access.index !23
+  %b1 = getelementptr inbounds %union.u1, %union.u1* %4, i64 0, i32 0, !dbg !40
+  %5 = tail call i32 @llvm.bpf.preserve.field.info.p0i32(i32* %b1, i64 3), !dbg !41
+  call void @llvm.dbg.value(metadata i32 %5, metadata !33, metadata !DIExpression()), !dbg !35
+  %6 = tail call i32* @llvm.preserve.struct.access.index.p0i32.p0s_union.u1s(%union.u1* %arg2, i32 0, i32 2), !dbg !42, !llvm.preserve.access.index !23
+  %7 = bitcast i32* %6 to i8*, !dbg !42
+  %8 = tail call i32 @llvm.bpf.preserve.field.info.p0i8(i8* %7, i64 3), !dbg !43
+  call void @llvm.dbg.value(metadata i32 %8, metadata !34, metadata !DIExpression()), !dbg !35
+  %add = add i32 %3, %1, !dbg !44
+  %add1 = add i32 %add, %5, !dbg !45
+  %add2 = add i32 %add1, %8, !dbg !46
+  ret i32 %add2, !dbg !47
+}
+
+; CHECK:             r1 = 1
+; CHECK:             r0 = 0
+; CHECK-ALU64:       r0 += r1
+; CHECK-ALU32:       w0 += w1
+; CHECK:             r1 = 1
+; CHECK-ALU64:       r0 += r1
+; CHECK-ALU32:       w0 += w1
+; CHECK:             r1 = 0
+; CHECK-ALU64:       r0 += r1
+; CHECK-ALU32:       w0 += w1
+; CHECK:             exit
+
+; CHECK:             .long   1                       # BTF_KIND_STRUCT(id = 2)
+; CHECK:             .long   37                      # BTF_KIND_UNION(id = 7)
+; CHECK:             .ascii  "s1"                    # string offset=1
+; CHECK:             .ascii  "u1"                    # string offset=37
+; CHECK:             .ascii  ".text"                 # string offset=64
+; CHECK:             .ascii  "0:0"                   # string offset=70
+; CHECK:             .ascii  "0:2"                   # string offset=111
+
+; CHECK:             .long   16                      # FieldReloc
+; CHECK-NEXT:        .long   64                      # Field reloc section string offset=64
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   70
+; CHECK-NEXT:        .long   3
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   111
+; CHECK-NEXT:        .long   3
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   7
+; CHECK-NEXT:        .long   70
+; CHECK-NEXT:        .long   3
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   7
+; CHECK-NEXT:        .long   111
+; CHECK-NEXT:        .long   3
+
+; Function Attrs: nounwind readnone
+declare i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.s1s(%struct.s1*, i32, i32) #1
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.bpf.preserve.field.info.p0i32(i32*, i64) #1
+
+; Function Attrs: nounwind readnone
+declare i16* @llvm.preserve.struct.access.index.p0i16.p0s_struct.s1s(%struct.s1*, i32, i32) #1
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.bpf.preserve.field.info.p0i16(i16*, i64) #1
+
+; Function Attrs: nounwind readnone
+declare %union.u1* @llvm.preserve.union.access.index.p0s_union.u1s.p0s_union.u1s(%union.u1*, i32) #1
+
+; Function Attrs: nounwind readnone
+declare i32* @llvm.preserve.struct.access.index.p0i32.p0s_union.u1s(%union.u1*, i32, i32) #1
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.bpf.preserve.field.info.p0i8(i8*, i64) #1
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.value(metadata, metadata, metadata) #2
+
+attributes #0 = { nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone }
+attributes #2 = { nounwind readnone speculatable willreturn }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!7, !8, !9}
+!llvm.ident = !{!10}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 10.0.0 (https://github.com/llvm/llvm-project.git 4a60741b74384f14b21fdc0131ede326438840ab)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "test.c", directory: "/tmp/home/yhs/work/tests/core")
+!2 = !{!3}
+!3 = !DICompositeType(tag: DW_TAG_enumeration_type, file: !1, line: 4, baseType: !4, size: 32, elements: !5)
+!4 = !DIBasicType(name: "unsigned int", size: 32, encoding: DW_ATE_unsigned)
+!5 = !{!6}
+!6 = !DIEnumerator(name: "FIELD_SIGNEDNESS", value: 3, isUnsigned: true)
+!7 = !{i32 2, !"Dwarf Version", i32 4}
+!8 = !{i32 2, !"Debug Info Version", i32 3}
+!9 = !{i32 1, !"wchar_size", i32 4}
+!10 = !{!"clang version 10.0.0 (https://github.com/llvm/llvm-project.git 4a60741b74384f14b21fdc0131ede326438840ab)"}
+!11 = distinct !DISubprogram(name: "test", scope: !1, file: !1, line: 5, type: !12, scopeLine: 5, flags: DIFlagPrototyped, isDefinition: true, isOptimized: true, unit: !0, retainedNodes: !28)
+!12 = !DISubroutineType(types: !13)
+!13 = !{!14, !15, !22}
+!14 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!15 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !16, size: 64)
+!16 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "s1", file: !1, line: 2, size: 64, elements: !17)
+!17 = !{!18, !19, !21}
+!18 = !DIDerivedType(tag: DW_TAG_member, name: "a1", scope: !16, file: !1, line: 2, baseType: !14, size: 32)
+!19 = !DIDerivedType(tag: DW_TAG_member, name: "a2", scope: !16, file: !1, line: 2, baseType: !20, size: 9, offset: 32, flags: DIFlagBitField, extraData: i64 32)
+!20 = !DIDerivedType(tag: DW_TAG_typedef, name: "__uint", file: !1, line: 1, baseType: !4)
+!21 = !DIDerivedType(tag: DW_TAG_member, name: "a3", scope: !16, file: !1, line: 2, baseType: !20, size: 4, offset: 41, flags: DIFlagBitField, extraData: i64 32)
+!22 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !23, size: 64)
+!23 = distinct !DICompositeType(tag: DW_TAG_union_type, name: "u1", file: !1, line: 3, size: 32, elements: !24)
+!24 = !{!25, !26, !27}
+!25 = !DIDerivedType(tag: DW_TAG_member, name: "b1", scope: !23, file: !1, line: 3, baseType: !14, size: 32)
+!26 = !DIDerivedType(tag: DW_TAG_member, name: "b2", scope: !23, file: !1, line: 3, baseType: !20, size: 9, flags: DIFlagBitField, extraData: i64 0)
+!27 = !DIDerivedType(tag: DW_TAG_member, name: "b3", scope: !23, file: !1, line: 3, baseType: !20, size: 4, flags: DIFlagBitField, extraData: i64 0)
+!28 = !{!29, !30, !31, !32, !33, !34}
+!29 = !DILocalVariable(name: "arg1", arg: 1, scope: !11, file: !1, line: 5, type: !15)
+!30 = !DILocalVariable(name: "arg2", arg: 2, scope: !11, file: !1, line: 5, type: !22)
+!31 = !DILocalVariable(name: "r1", scope: !11, file: !1, line: 6, type: !4)
+!32 = !DILocalVariable(name: "r2", scope: !11, file: !1, line: 7, type: !4)
+!33 = !DILocalVariable(name: "r3", scope: !11, file: !1, line: 8, type: !4)
+!34 = !DILocalVariable(name: "r4", scope: !11, file: !1, line: 9, type: !4)
+!35 = !DILocation(line: 0, scope: !11)
+!36 = !DILocation(line: 6, column: 53, scope: !11)
+!37 = !DILocation(line: 6, column: 17, scope: !11)
+!38 = !DILocation(line: 7, column: 53, scope: !11)
+!39 = !DILocation(line: 7, column: 17, scope: !11)
+!40 = !DILocation(line: 8, column: 53, scope: !11)
+!41 = !DILocation(line: 8, column: 17, scope: !11)
+!42 = !DILocation(line: 9, column: 53, scope: !11)
+!43 = !DILocation(line: 9, column: 17, scope: !11)
+!44 = !DILocation(line: 10, column: 13, scope: !11)
+!45 = !DILocation(line: 10, column: 18, scope: !11)
+!46 = !DILocation(line: 10, column: 23, scope: !11)
+!47 = !DILocation(line: 10, column: 3, scope: !11)
diff --git a/llvm/test/CodeGen/BPF/CORE/intrinsic-fieldinfo-signedness-2.ll b/llvm/test/CodeGen/BPF/CORE/intrinsic-fieldinfo-signedness-2.ll
new file mode 100644
index 000000000..8c6675d97
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/CORE/intrinsic-fieldinfo-signedness-2.ll
@@ -0,0 +1,155 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-ALU64 %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-ALU64 %s
+; RUN: llc -march=bpfel -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-ALU32 %s
+; RUN: llc -march=bpfeb -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-ALU32 %s
+; Source code:
+;   enum A { AA = -1, AB = 0, }; /* signed */
+;   enum B { BA = 0, BB = 1, };  /* unsigned */
+;   typedef enum A __A;
+;   typedef enum B __B;
+;   typedef int __int;           /* signed */
+;   struct s1 { __A a1; __B a2:9; __int a3:4; };
+;   union u1 { int b1; struct s1 b2; };
+;   enum { FIELD_SIGNEDNESS = 3, };
+;   int test(union u1 *arg) {
+;     unsigned r1 = __builtin_preserve_field_info(arg->b2.a1, FIELD_SIGNEDNESS);
+;     unsigned r2 = __builtin_preserve_field_info(arg->b2.a2, FIELD_SIGNEDNESS);
+;     unsigned r3 = __builtin_preserve_field_info(arg->b2.a3, FIELD_SIGNEDNESS);
+;     return r1 + r2 + r3;
+;   }
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm test.c
+
+%union.u1 = type { %struct.s1 }
+%struct.s1 = type { i32, i16 }
+
+; Function Attrs: nounwind readnone
+define dso_local i32 @test(%union.u1* %arg) local_unnamed_addr #0 !dbg !20 {
+entry:
+  call void @llvm.dbg.value(metadata %union.u1* %arg, metadata !37, metadata !DIExpression()), !dbg !41
+  %0 = tail call %union.u1* @llvm.preserve.union.access.index.p0s_union.u1s.p0s_union.u1s(%union.u1* %arg, i32 1), !dbg !42, !llvm.preserve.access.index !24
+  %b2 = getelementptr inbounds %union.u1, %union.u1* %0, i64 0, i32 0, !dbg !42
+  %1 = tail call i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.s1s(%struct.s1* %b2, i32 0, i32 0), !dbg !43, !llvm.preserve.access.index !28
+  %2 = tail call i32 @llvm.bpf.preserve.field.info.p0i32(i32* %1, i64 3), !dbg !44
+  call void @llvm.dbg.value(metadata i32 %2, metadata !38, metadata !DIExpression()), !dbg !41
+  %3 = tail call i16* @llvm.preserve.struct.access.index.p0i16.p0s_struct.s1s(%struct.s1* %b2, i32 1, i32 1), !dbg !45, !llvm.preserve.access.index !28
+  %4 = tail call i32 @llvm.bpf.preserve.field.info.p0i16(i16* %3, i64 3), !dbg !46
+  call void @llvm.dbg.value(metadata i32 %4, metadata !39, metadata !DIExpression()), !dbg !41
+  %5 = tail call i16* @llvm.preserve.struct.access.index.p0i16.p0s_struct.s1s(%struct.s1* %b2, i32 1, i32 2), !dbg !47, !llvm.preserve.access.index !28
+  %6 = tail call i32 @llvm.bpf.preserve.field.info.p0i16(i16* %5, i64 3), !dbg !48
+  call void @llvm.dbg.value(metadata i32 %6, metadata !40, metadata !DIExpression()), !dbg !41
+  %add = add i32 %4, %2, !dbg !49
+  %add3 = add i32 %add, %6, !dbg !50
+  ret i32 %add3, !dbg !51
+}
+
+; CHECK:             r1 = 1
+; CHECK:             r0 = 0
+; CHECK-ALU64:       r0 += r1
+; CHECK-ALU32:       w0 += w1
+; CHECK:             r1 = 1
+; CHECK-ALU64:       r0 += r1
+; CHECK-ALU32:       w0 += w1
+; CHECK:             exit
+
+; CHECK:             .long   1                       # BTF_KIND_UNION(id = 2)
+; CHECK:             .ascii  "u1"                    # string offset=1
+; CHECK:             .ascii  ".text"                 # string offset=65
+; CHECK:             .ascii  "0:1:0"                 # string offset=71
+; CHECK:             .ascii  "0:1:1"                 # string offset=114
+; CHECK:             .ascii  "0:1:2"                 # string offset=120
+
+; CHECK:             .long   16                      # FieldReloc
+; CHECK-NEXT:        .long   65                      # Field reloc section string offset=65
+; CHECK-NEXT:        .long   3
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   71
+; CHECK-NEXT:        .long   3
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   114
+; CHECK-NEXT:        .long   3
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   120
+; CHECK-NEXT:        .long   3
+
+; Function Attrs: nounwind readnone
+declare %union.u1* @llvm.preserve.union.access.index.p0s_union.u1s.p0s_union.u1s(%union.u1*, i32) #1
+
+; Function Attrs: nounwind readnone
+declare i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.s1s(%struct.s1*, i32, i32) #1
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.bpf.preserve.field.info.p0i32(i32*, i64) #1
+
+; Function Attrs: nounwind readnone
+declare i16* @llvm.preserve.struct.access.index.p0i16.p0s_struct.s1s(%struct.s1*, i32, i32) #1
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.bpf.preserve.field.info.p0i16(i16*, i64) #1
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.value(metadata, metadata, metadata) #2
+
+attributes #0 = { nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone }
+attributes #2 = { nounwind readnone speculatable willreturn }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!16, !17, !18}
+!llvm.ident = !{!19}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 10.0.0 (https://github.com/llvm/llvm-project.git 4a60741b74384f14b21fdc0131ede326438840ab)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "test.c", directory: "/tmp/home/yhs/work/tests/core")
+!2 = !{!3, !8, !13}
+!3 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "A", file: !1, line: 1, baseType: !4, size: 32, elements: !5)
+!4 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!5 = !{!6, !7}
+!6 = !DIEnumerator(name: "AA", value: -1)
+!7 = !DIEnumerator(name: "AB", value: 0)
+!8 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "B", file: !1, line: 2, baseType: !9, size: 32, elements: !10)
+!9 = !DIBasicType(name: "unsigned int", size: 32, encoding: DW_ATE_unsigned)
+!10 = !{!11, !12}
+!11 = !DIEnumerator(name: "BA", value: 0, isUnsigned: true)
+!12 = !DIEnumerator(name: "BB", value: 1, isUnsigned: true)
+!13 = !DICompositeType(tag: DW_TAG_enumeration_type, file: !1, line: 8, baseType: !9, size: 32, elements: !14)
+!14 = !{!15}
+!15 = !DIEnumerator(name: "FIELD_SIGNEDNESS", value: 3, isUnsigned: true)
+!16 = !{i32 2, !"Dwarf Version", i32 4}
+!17 = !{i32 2, !"Debug Info Version", i32 3}
+!18 = !{i32 1, !"wchar_size", i32 4}
+!19 = !{!"clang version 10.0.0 (https://github.com/llvm/llvm-project.git 4a60741b74384f14b21fdc0131ede326438840ab)"}
+!20 = distinct !DISubprogram(name: "test", scope: !1, file: !1, line: 9, type: !21, scopeLine: 9, flags: DIFlagPrototyped, isDefinition: true, isOptimized: true, unit: !0, retainedNodes: !36)
+!21 = !DISubroutineType(types: !22)
+!22 = !{!4, !23}
+!23 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !24, size: 64)
+!24 = distinct !DICompositeType(tag: DW_TAG_union_type, name: "u1", file: !1, line: 7, size: 64, elements: !25)
+!25 = !{!26, !27}
+!26 = !DIDerivedType(tag: DW_TAG_member, name: "b1", scope: !24, file: !1, line: 7, baseType: !4, size: 32)
+!27 = !DIDerivedType(tag: DW_TAG_member, name: "b2", scope: !24, file: !1, line: 7, baseType: !28, size: 64)
+!28 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "s1", file: !1, line: 6, size: 64, elements: !29)
+!29 = !{!30, !32, !34}
+!30 = !DIDerivedType(tag: DW_TAG_member, name: "a1", scope: !28, file: !1, line: 6, baseType: !31, size: 32)
+!31 = !DIDerivedType(tag: DW_TAG_typedef, name: "__A", file: !1, line: 3, baseType: !3)
+!32 = !DIDerivedType(tag: DW_TAG_member, name: "a2", scope: !28, file: !1, line: 6, baseType: !33, size: 9, offset: 32, flags: DIFlagBitField, extraData: i64 32)
+!33 = !DIDerivedType(tag: DW_TAG_typedef, name: "__B", file: !1, line: 4, baseType: !8)
+!34 = !DIDerivedType(tag: DW_TAG_member, name: "a3", scope: !28, file: !1, line: 6, baseType: !35, size: 4, offset: 41, flags: DIFlagBitField, extraData: i64 32)
+!35 = !DIDerivedType(tag: DW_TAG_typedef, name: "__int", file: !1, line: 5, baseType: !4)
+!36 = !{!37, !38, !39, !40}
+!37 = !DILocalVariable(name: "arg", arg: 1, scope: !20, file: !1, line: 9, type: !23)
+!38 = !DILocalVariable(name: "r1", scope: !20, file: !1, line: 10, type: !9)
+!39 = !DILocalVariable(name: "r2", scope: !20, file: !1, line: 11, type: !9)
+!40 = !DILocalVariable(name: "r3", scope: !20, file: !1, line: 12, type: !9)
+!41 = !DILocation(line: 0, scope: !20)
+!42 = !DILocation(line: 10, column: 52, scope: !20)
+!43 = !DILocation(line: 10, column: 55, scope: !20)
+!44 = !DILocation(line: 10, column: 17, scope: !20)
+!45 = !DILocation(line: 11, column: 55, scope: !20)
+!46 = !DILocation(line: 11, column: 17, scope: !20)
+!47 = !DILocation(line: 12, column: 55, scope: !20)
+!48 = !DILocation(line: 12, column: 17, scope: !20)
+!49 = !DILocation(line: 13, column: 13, scope: !20)
+!50 = !DILocation(line: 13, column: 18, scope: !20)
+!51 = !DILocation(line: 13, column: 3, scope: !20)
diff --git a/llvm/test/CodeGen/BPF/CORE/intrinsic-fieldinfo-signedness-3.ll b/llvm/test/CodeGen/BPF/CORE/intrinsic-fieldinfo-signedness-3.ll
new file mode 100644
index 000000000..e9de678f7
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/CORE/intrinsic-fieldinfo-signedness-3.ll
@@ -0,0 +1,152 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-ALU64 %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-ALU64 %s
+; RUN: llc -march=bpfel -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-ALU32 %s
+; RUN: llc -march=bpfeb -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-ALU32 %s
+; Source code:
+;   enum A { AA = -1, AB = 0, };
+;   enum B { BA = 0, BB = 1, };
+;   typedef enum A __A;
+;   typedef enum B __B;
+;   typedef struct s1 { __A a1[10]; __B a2[10][10]; } __s1;
+;   union u1 { int b1; __s1 b2; };
+;   enum { FIELD_SIGNEDNESS = 3, };
+;   int test(union u1 *arg) {
+;     unsigned r1 = __builtin_preserve_field_info(arg->b2.a1[5], FIELD_SIGNEDNESS);
+;     unsigned r2 = __builtin_preserve_field_info(arg->b2.a2[5][5], FIELD_SIGNEDNESS);
+;     /* r1 : 1, r2 : 0 */
+;     return r1 + r2;
+;   }
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm test.c
+
+%union.u1 = type { %struct.s1 }
+%struct.s1 = type { [10 x i32], [10 x [10 x i32]] }
+
+; Function Attrs: nounwind readnone
+define dso_local i32 @test(%union.u1* %arg) local_unnamed_addr #0 !dbg !29 {
+entry:
+  call void @llvm.dbg.value(metadata %union.u1* %arg, metadata !43, metadata !DIExpression()), !dbg !46
+  %0 = tail call %union.u1* @llvm.preserve.union.access.index.p0s_union.u1s.p0s_union.u1s(%union.u1* %arg, i32 1), !dbg !47, !llvm.preserve.access.index !33
+  %b2 = getelementptr inbounds %union.u1, %union.u1* %0, i64 0, i32 0, !dbg !47
+  %1 = tail call [10 x i32]* @llvm.preserve.struct.access.index.p0a10i32.p0s_struct.s1s(%struct.s1* %b2, i32 0, i32 0), !dbg !48, !llvm.preserve.access.index !38
+  %2 = tail call i32* @llvm.preserve.array.access.index.p0i32.p0a10i32([10 x i32]* %1, i32 1, i32 5), !dbg !49, !llvm.preserve.access.index !17
+  %3 = tail call i32 @llvm.bpf.preserve.field.info.p0i32(i32* %2, i64 3), !dbg !50
+  call void @llvm.dbg.value(metadata i32 %3, metadata !44, metadata !DIExpression()), !dbg !46
+  %4 = tail call [10 x [10 x i32]]* @llvm.preserve.struct.access.index.p0a10a10i32.p0s_struct.s1s(%struct.s1* %b2, i32 1, i32 1), !dbg !51, !llvm.preserve.access.index !38
+  %5 = tail call [10 x i32]* @llvm.preserve.array.access.index.p0a10i32.p0a10a10i32([10 x [10 x i32]]* %4, i32 1, i32 5), !dbg !52, !llvm.preserve.access.index !21
+  %6 = tail call i32* @llvm.preserve.array.access.index.p0i32.p0a10i32([10 x i32]* %5, i32 1, i32 5), !dbg !52, !llvm.preserve.access.index !24
+  %7 = tail call i32 @llvm.bpf.preserve.field.info.p0i32(i32* %6, i64 3), !dbg !53
+  call void @llvm.dbg.value(metadata i32 %7, metadata !45, metadata !DIExpression()), !dbg !46
+  %add = add i32 %7, %3, !dbg !54
+  ret i32 %add, !dbg !55
+}
+
+; CHECK:             r1 = 1
+; CHECK:             r0 = 0
+; CHECK-ALU64:       r0 += r1
+; CHECK-ALU32:       w0 += w1
+; CHECK:             exit
+
+; CHECK:             .long   1                       # BTF_KIND_UNION(id = 2)
+; CHECK:             .ascii  "u1"                    # string offset=1
+; CHECK:             .ascii  ".text"                 # string offset=81
+; CHECK:             .ascii  "0:1:0:5"               # string offset=87
+; CHECK:             .ascii  "0:1:1:5:5"             # string offset=132
+
+; CHECK:             .long   16                      # FieldReloc
+; CHECK-NEXT:        .long   81                      # Field reloc section string offset=81
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   87
+; CHECK-NEXT:        .long   3
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   132
+; CHECK-NEXT:        .long   3
+
+; Function Attrs: nounwind readnone
+declare %union.u1* @llvm.preserve.union.access.index.p0s_union.u1s.p0s_union.u1s(%union.u1*, i32) #1
+
+; Function Attrs: nounwind readnone
+declare [10 x i32]* @llvm.preserve.struct.access.index.p0a10i32.p0s_struct.s1s(%struct.s1*, i32, i32) #1
+
+; Function Attrs: nounwind readnone
+declare i32* @llvm.preserve.array.access.index.p0i32.p0a10i32([10 x i32]*, i32, i32) #1
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.bpf.preserve.field.info.p0i32(i32*, i64) #1
+
+; Function Attrs: nounwind readnone
+declare [10 x [10 x i32]]* @llvm.preserve.struct.access.index.p0a10a10i32.p0s_struct.s1s(%struct.s1*, i32, i32) #1
+
+; Function Attrs: nounwind readnone
+declare [10 x i32]* @llvm.preserve.array.access.index.p0a10i32.p0a10a10i32([10 x [10 x i32]]*, i32, i32) #1
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.value(metadata, metadata, metadata) #2
+
+attributes #0 = { nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone }
+attributes #2 = { nounwind readnone speculatable willreturn }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!25, !26, !27}
+!llvm.ident = !{!28}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 10.0.0 (https://github.com/llvm/llvm-project.git c1e02f16f1105ffaf1c35ee8bc38b7d6db5c6ea9)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !16, nameTableKind: None)
+!1 = !DIFile(filename: "test.c", directory: "/tmp/home/yhs/work/tests/core")
+!2 = !{!3, !8, !13}
+!3 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "A", file: !1, line: 1, baseType: !4, size: 32, elements: !5)
+!4 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!5 = !{!6, !7}
+!6 = !DIEnumerator(name: "AA", value: -1)
+!7 = !DIEnumerator(name: "AB", value: 0)
+!8 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "B", file: !1, line: 2, baseType: !9, size: 32, elements: !10)
+!9 = !DIBasicType(name: "unsigned int", size: 32, encoding: DW_ATE_unsigned)
+!10 = !{!11, !12}
+!11 = !DIEnumerator(name: "BA", value: 0, isUnsigned: true)
+!12 = !DIEnumerator(name: "BB", value: 1, isUnsigned: true)
+!13 = !DICompositeType(tag: DW_TAG_enumeration_type, file: !1, line: 7, baseType: !9, size: 32, elements: !14)
+!14 = !{!15}
+!15 = !DIEnumerator(name: "FIELD_SIGNEDNESS", value: 3, isUnsigned: true)
+!16 = !{!17, !21, !24}
+!17 = !DICompositeType(tag: DW_TAG_array_type, baseType: !18, size: 320, elements: !19)
+!18 = !DIDerivedType(tag: DW_TAG_typedef, name: "__A", file: !1, line: 3, baseType: !3)
+!19 = !{!20}
+!20 = !DISubrange(count: 10)
+!21 = !DICompositeType(tag: DW_TAG_array_type, baseType: !22, size: 3200, elements: !23)
+!22 = !DIDerivedType(tag: DW_TAG_typedef, name: "__B", file: !1, line: 4, baseType: !8)
+!23 = !{!20, !20}
+!24 = !DICompositeType(tag: DW_TAG_array_type, baseType: !22, size: 320, elements: !19)
+!25 = !{i32 2, !"Dwarf Version", i32 4}
+!26 = !{i32 2, !"Debug Info Version", i32 3}
+!27 = !{i32 1, !"wchar_size", i32 4}
+!28 = !{!"clang version 10.0.0 (https://github.com/llvm/llvm-project.git c1e02f16f1105ffaf1c35ee8bc38b7d6db5c6ea9)"}
+!29 = distinct !DISubprogram(name: "test", scope: !1, file: !1, line: 8, type: !30, scopeLine: 8, flags: DIFlagPrototyped, isDefinition: true, isOptimized: true, unit: !0, retainedNodes: !42)
+!30 = !DISubroutineType(types: !31)
+!31 = !{!4, !32}
+!32 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !33, size: 64)
+!33 = distinct !DICompositeType(tag: DW_TAG_union_type, name: "u1", file: !1, line: 6, size: 3520, elements: !34)
+!34 = !{!35, !36}
+!35 = !DIDerivedType(tag: DW_TAG_member, name: "b1", scope: !33, file: !1, line: 6, baseType: !4, size: 32)
+!36 = !DIDerivedType(tag: DW_TAG_member, name: "b2", scope: !33, file: !1, line: 6, baseType: !37, size: 3520)
+!37 = !DIDerivedType(tag: DW_TAG_typedef, name: "__s1", file: !1, line: 5, baseType: !38)
+!38 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "s1", file: !1, line: 5, size: 3520, elements: !39)
+!39 = !{!40, !41}
+!40 = !DIDerivedType(tag: DW_TAG_member, name: "a1", scope: !38, file: !1, line: 5, baseType: !17, size: 320)
+!41 = !DIDerivedType(tag: DW_TAG_member, name: "a2", scope: !38, file: !1, line: 5, baseType: !21, size: 3200, offset: 320)
+!42 = !{!43, !44, !45}
+!43 = !DILocalVariable(name: "arg", arg: 1, scope: !29, file: !1, line: 8, type: !32)
+!44 = !DILocalVariable(name: "r1", scope: !29, file: !1, line: 9, type: !9)
+!45 = !DILocalVariable(name: "r2", scope: !29, file: !1, line: 10, type: !9)
+!46 = !DILocation(line: 0, scope: !29)
+!47 = !DILocation(line: 9, column: 52, scope: !29)
+!48 = !DILocation(line: 9, column: 55, scope: !29)
+!49 = !DILocation(line: 9, column: 47, scope: !29)
+!50 = !DILocation(line: 9, column: 17, scope: !29)
+!51 = !DILocation(line: 10, column: 55, scope: !29)
+!52 = !DILocation(line: 10, column: 47, scope: !29)
+!53 = !DILocation(line: 10, column: 17, scope: !29)
+!54 = !DILocation(line: 12, column: 13, scope: !29)
+!55 = !DILocation(line: 12, column: 3, scope: !29)
diff --git a/llvm/test/CodeGen/BPF/CORE/intrinsic-struct.ll b/llvm/test/CodeGen/BPF/CORE/intrinsic-struct.ll
index c07c16f52..ecbcc4fce 100644
--- a/llvm/test/CodeGen/BPF/CORE/intrinsic-struct.ll
+++ b/llvm/test/CodeGen/BPF/CORE/intrinsic-struct.ll
@@ -1,4 +1,5 @@
 ; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck %s
+; RUN: llc -march=bpfel -mattr=+alu32 -filetype=asm -o - %s | FileCheck %s
 ;
 ; Source code:
 ;   struct s { int a; int b; };
@@ -28,12 +29,13 @@ entry:
 ; CHECK:       exit
 ;
 ; CHECK:      .section        .BTF.ext,"",@progbits
-; CHECK:      .long   12                      # OffsetReloc
-; CHECK-NEXT: .long   20                      # Offset reloc section string offset=20
+; CHECK:      .long   16                      # FieldReloc
+; CHECK-NEXT: .long   20                      # Field reloc section string offset=20
 ; CHECK-NEXT: .long   1
 ; CHECK-NEXT: .long   [[RELOC]]
 ; CHECK-NEXT: .long   2
 ; CHECK-NEXT: .long   26
+; CHECK-NEXT: .long   0
 
 declare dso_local i32 @get_value(i8*) local_unnamed_addr #1
 
@@ -43,8 +45,8 @@ declare i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.ss(%struct.s*,
 ; Function Attrs: nounwind readnone speculatable
 declare void @llvm.dbg.value(metadata, metadata, metadata) #3
 
-attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind readnone }
 attributes #3 = { nounwind readnone speculatable }
 attributes #4 = { nounwind }
diff --git a/llvm/test/CodeGen/BPF/CORE/intrinsic-transforms.ll b/llvm/test/CodeGen/BPF/CORE/intrinsic-transforms.ll
new file mode 100644
index 000000000..db082233c
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/CORE/intrinsic-transforms.ll
@@ -0,0 +1,120 @@
+; RUN: opt -O2 -march=bpfeb < %s | llvm-dis | FileCheck %s
+; RUN: opt -O2 -march=bpfel < %s | llvm-dis | FileCheck %s
+;
+; Source code:
+;   #define _(x) (__builtin_preserve_access_index(x))
+;   int get_value(const int *arg);
+;   int test(int b, int *arg) {
+;     int v1 = b ? get_value(_(&arg[4])) : 0;
+;     int v2 = b ? get_value(_(&arg[4])) : 0;
+;     return v1 + v2;
+;   }
+; Compilation flag:
+;   clang -target bpf -O0 -g -S -emit-llvm -Xclang -disable-O0-optnone test.c
+
+; Function Attrs: noinline nounwind
+define dso_local i32 @test(i32 %b, i32* %arg) #0 !dbg !10 {
+entry:
+  %b.addr = alloca i32, align 4
+  %arg.addr = alloca i32*, align 8
+  %v1 = alloca i32, align 4
+  %v2 = alloca i32, align 4
+  store i32 %b, i32* %b.addr, align 4
+  call void @llvm.dbg.declare(metadata i32* %b.addr, metadata !13, metadata !DIExpression()), !dbg !14
+  store i32* %arg, i32** %arg.addr, align 8
+  call void @llvm.dbg.declare(metadata i32** %arg.addr, metadata !15, metadata !DIExpression()), !dbg !16
+  call void @llvm.dbg.declare(metadata i32* %v1, metadata !17, metadata !DIExpression()), !dbg !18
+  %0 = load i32, i32* %b.addr, align 4, !dbg !19
+  %tobool = icmp ne i32 %0, 0, !dbg !19
+  br i1 %tobool, label %cond.true, label %cond.false, !dbg !19
+
+cond.true:                                        ; preds = %entry
+  %1 = load i32*, i32** %arg.addr, align 8, !dbg !20
+  %2 = call i32* @llvm.preserve.array.access.index.p0i32.p0i32(i32* %1, i32 0, i32 4), !dbg !20, !llvm.preserve.access.index !4
+  %3 = bitcast i32* %2 to i8*, !dbg !20
+  %4 = bitcast i8* %3 to i32*, !dbg !20
+  %call = call i32 @get_value(i32* %4), !dbg !21
+  br label %cond.end, !dbg !19
+
+cond.false:                                       ; preds = %entry
+  br label %cond.end, !dbg !19
+
+cond.end:                                         ; preds = %cond.false, %cond.true
+  %cond = phi i32 [ %call, %cond.true ], [ 0, %cond.false ], !dbg !19
+  store i32 %cond, i32* %v1, align 4, !dbg !18
+  call void @llvm.dbg.declare(metadata i32* %v2, metadata !22, metadata !DIExpression()), !dbg !23
+  %5 = load i32, i32* %b.addr, align 4, !dbg !24
+  %tobool1 = icmp ne i32 %5, 0, !dbg !24
+  br i1 %tobool1, label %cond.true2, label %cond.false4, !dbg !24
+
+cond.true2:                                       ; preds = %cond.end
+  %6 = load i32*, i32** %arg.addr, align 8, !dbg !25
+  %7 = call i32* @llvm.preserve.array.access.index.p0i32.p0i32(i32* %6, i32 0, i32 4), !dbg !25, !llvm.preserve.access.index !4
+  %8 = bitcast i32* %7 to i8*, !dbg !25
+  %9 = bitcast i8* %8 to i32*, !dbg !25
+  %call3 = call i32 @get_value(i32* %9), !dbg !26
+  br label %cond.end5, !dbg !24
+
+; CHECK: tail call i32* @llvm.preserve.array.access.index.p0i32.p0i32(i32* %{{[0-9a-z]+}}, i32 0, i32 4), !dbg !{{[0-9]+}}, !llvm.preserve.access.index !{{[0-9]+}}
+; CHECK-NOT: tail call i32* @llvm.preserve.array.access.index
+
+cond.false4:                                      ; preds = %cond.end
+  br label %cond.end5, !dbg !24
+
+cond.end5:                                        ; preds = %cond.false4, %cond.true2
+  %cond6 = phi i32 [ %call3, %cond.true2 ], [ 0, %cond.false4 ], !dbg !24
+  store i32 %cond6, i32* %v2, align 4, !dbg !23
+  %10 = load i32, i32* %v1, align 4, !dbg !27
+  %11 = load i32, i32* %v2, align 4, !dbg !28
+  %add = add nsw i32 %10, %11, !dbg !29
+  ret i32 %add, !dbg !30
+}
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
+
+declare dso_local i32 @get_value(i32*) #2
+
+; Function Attrs: nounwind readnone
+declare i32* @llvm.preserve.array.access.index.p0i32.p0i32(i32*, i32 immarg, i32 immarg) #3
+
+attributes #0 = { noinline nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone speculatable willreturn }
+attributes #2 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { nounwind readnone }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!6, !7, !8}
+!llvm.ident = !{!9}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 10.0.0 (https://github.com/llvm/llvm-project.git 4618b07fe2cede1b73512d1c260cf4981661f47f)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !3, nameTableKind: None)
+!1 = !DIFile(filename: "test.c", directory: "/tmp/home/yhs/work/tests/llvm/cast")
+!2 = !{}
+!3 = !{!4}
+!4 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !5, size: 64)
+!5 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!6 = !{i32 2, !"Dwarf Version", i32 4}
+!7 = !{i32 2, !"Debug Info Version", i32 3}
+!8 = !{i32 1, !"wchar_size", i32 4}
+!9 = !{!"clang version 10.0.0 (https://github.com/llvm/llvm-project.git 4618b07fe2cede1b73512d1c260cf4981661f47f)"}
+!10 = distinct !DISubprogram(name: "test", scope: !1, file: !1, line: 3, type: !11, scopeLine: 3, flags: DIFlagPrototyped, isDefinition: true, unit: !0, retainedNodes: !2)
+!11 = !DISubroutineType(types: !12)
+!12 = !{!5, !5, !4}
+!13 = !DILocalVariable(name: "b", arg: 1, scope: !10, file: !1, line: 3, type: !5)
+!14 = !DILocation(line: 3, column: 14, scope: !10)
+!15 = !DILocalVariable(name: "arg", arg: 2, scope: !10, file: !1, line: 3, type: !4)
+!16 = !DILocation(line: 3, column: 22, scope: !10)
+!17 = !DILocalVariable(name: "v1", scope: !10, file: !1, line: 4, type: !5)
+!18 = !DILocation(line: 4, column: 7, scope: !10)
+!19 = !DILocation(line: 4, column: 12, scope: !10)
+!20 = !DILocation(line: 4, column: 26, scope: !10)
+!21 = !DILocation(line: 4, column: 16, scope: !10)
+!22 = !DILocalVariable(name: "v2", scope: !10, file: !1, line: 5, type: !5)
+!23 = !DILocation(line: 5, column: 7, scope: !10)
+!24 = !DILocation(line: 5, column: 12, scope: !10)
+!25 = !DILocation(line: 5, column: 26, scope: !10)
+!26 = !DILocation(line: 5, column: 16, scope: !10)
+!27 = !DILocation(line: 6, column: 10, scope: !10)
+!28 = !DILocation(line: 6, column: 15, scope: !10)
+!29 = !DILocation(line: 6, column: 13, scope: !10)
+!30 = !DILocation(line: 6, column: 3, scope: !10)
diff --git a/llvm/test/CodeGen/BPF/CORE/intrinsic-union.ll b/llvm/test/CodeGen/BPF/CORE/intrinsic-union.ll
index 19b459533..093440d8e 100644
--- a/llvm/test/CodeGen/BPF/CORE/intrinsic-union.ll
+++ b/llvm/test/CodeGen/BPF/CORE/intrinsic-union.ll
@@ -1,4 +1,5 @@
 ; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck %s
+; RUN: llc -march=bpfel -mattr=+alu32 -filetype=asm -o - %s | FileCheck %s
 ;
 ; Source code:
 ;   union u { int a; int b; };
@@ -27,12 +28,13 @@ entry:
 ; CHECK:       exit
 
 ; CHECK:      .section        .BTF.ext,"",@progbits
-; CHECK:      .long   12                      # OffsetReloc
-; CHECK-NEXT: .long   20                      # Offset reloc section string offset=20
+; CHECK:      .long   16                      # FieldReloc
+; CHECK-NEXT: .long   20                      # Field reloc section string offset=20
 ; CHECK-NEXT: .long   1
 ; CHECK-NEXT: .long   [[RELOC]]
 ; CHECK-NEXT: .long   2
 ; CHECK-NEXT: .long   26
+; CHECK-NEXT: .long   0
 
 declare dso_local i32 @get_value(i8*) local_unnamed_addr #1
 
@@ -42,8 +44,8 @@ declare %union.u* @llvm.preserve.union.access.index.p0s_union.us.p0s_union.us(%u
 ; Function Attrs: nounwind readnone speculatable
 declare void @llvm.dbg.value(metadata, metadata, metadata) #3
 
-attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind readnone }
 attributes #3 = { nounwind readnone speculatable }
 attributes #4 = { nounwind }
diff --git a/llvm/test/CodeGen/BPF/CORE/no-elf-ama-symbol.ll b/llvm/test/CodeGen/BPF/CORE/no-elf-ama-symbol.ll
new file mode 100644
index 000000000..8851c502b
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/CORE/no-elf-ama-symbol.ll
@@ -0,0 +1,65 @@
+; RUN: llc -march=bpfel -filetype=obj -o - %s | llvm-readelf -s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=obj -o - %s | llvm-readelf -s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfel -filetype=obj -addrsig -o - %s | llvm-readelf -s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=obj -addrsig -o - %s | llvm-readelf -s | FileCheck -check-prefixes=CHECK %s
+;
+; Source Code:
+;   struct tt { int a; } __attribute__((preserve_access_index));
+;   int test(struct tt *arg) {
+;     return arg->a;
+;   }
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm t.c
+
+%struct.tt = type { i32 }
+
+; Function Attrs: nounwind readonly
+define dso_local i32 @test(%struct.tt* readonly %arg) local_unnamed_addr #0 !dbg !7 {
+entry:
+  call void @llvm.dbg.value(metadata %struct.tt* %arg, metadata !16, metadata !DIExpression()), !dbg !17
+  %0 = tail call i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.tts(%struct.tt* %arg, i32 0, i32 0), !dbg !18, !llvm.preserve.access.index !12
+  %1 = load i32, i32* %0, align 4, !dbg !18, !tbaa !19
+  ret i32 %1, !dbg !24
+}
+
+; CHECK-NOT: llvm.tt:0:0$0:0
+
+; Function Attrs: nounwind readnone
+declare i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.tts(%struct.tt*, i32, i32) #1
+
+; Function Attrs: nounwind readnone speculatable
+declare void @llvm.dbg.value(metadata, metadata, metadata) #2
+
+attributes #0 = { nounwind readonly "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone }
+attributes #2 = { nounwind readnone speculatable}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 10.0.0 (https://github.com/llvm/llvm-project.git 947f9692440836dcb8d88b74b69dd379d85974ce)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "test.c", directory: "/tmp/home/yhs/work/tests/bug")
+!2 = !{}
+!3 = !{i32 7, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{!"clang version 10.0.0 (https://github.com/llvm/llvm-project.git 947f9692440836dcb8d88b74b69dd379d85974ce)"}
+!7 = distinct !DISubprogram(name: "test", scope: !1, file: !1, line: 2, type: !8, scopeLine: 2, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !15)
+!8 = !DISubroutineType(types: !9)
+!9 = !{!10, !11}
+!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!11 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !12, size: 64)
+!12 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "tt", file: !1, line: 1, size: 32, elements: !13)
+!13 = !{!14}
+!14 = !DIDerivedType(tag: DW_TAG_member, name: "a", scope: !12, file: !1, line: 1, baseType: !10, size: 32)
+!15 = !{!16}
+!16 = !DILocalVariable(name: "arg", arg: 1, scope: !7, file: !1, line: 2, type: !11)
+!17 = !DILocation(line: 0, scope: !7)
+!18 = !DILocation(line: 3, column: 15, scope: !7)
+!19 = !{!20, !21, i64 0}
+!20 = !{!"tt", !21, i64 0}
+!21 = !{!"int", !22, i64 0}
+!22 = !{!"omnipotent char", !23, i64 0}
+!23 = !{!"Simple C/C++ TBAA"}
+!24 = !DILocation(line: 3, column: 3, scope: !7)
diff --git a/llvm/test/CodeGen/BPF/CORE/no-narrow-load.ll b/llvm/test/CodeGen/BPF/CORE/no-narrow-load.ll
new file mode 100644
index 000000000..fd33f0388
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/CORE/no-narrow-load.ll
@@ -0,0 +1,156 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; Source code:
+;   struct data_t {
+;     int d1;
+;     int d2;
+;   };
+;   struct info_t {
+;     int pid;
+;     int flags;
+;   } __attribute__((preserve_access_index));
+;
+;   extern void output(void *);
+;   void test(struct info_t * args) {
+;     int is_mask2 = args->flags & 0x10000;
+;     struct data_t data = {};
+;
+;     data.d1 = is_mask2 ? 2 : args->pid;
+;     data.d2 = (is_mask2 || (args->flags & 0x8000)) ? 1 : 2;
+;     output(&data);
+;   }
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm test.c
+
+%struct.info_t = type { i32, i32 }
+%struct.data_t = type { i32, i32 }
+
+; Function Attrs: nounwind
+define dso_local void @test(%struct.info_t* readonly %args) local_unnamed_addr #0 !dbg !12 {
+entry:
+  %data = alloca i64, align 8
+  %tmpcast = bitcast i64* %data to %struct.data_t*
+  call void @llvm.dbg.value(metadata %struct.info_t* %args, metadata !22, metadata !DIExpression()), !dbg !29
+  %0 = tail call i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.info_ts(%struct.info_t* %args, i32 1, i32 1), !dbg !30, !llvm.preserve.access.index !16
+  %1 = load i32, i32* %0, align 4, !dbg !30, !tbaa !31
+  %and = and i32 %1, 65536, !dbg !36
+  call void @llvm.dbg.value(metadata i32 %and, metadata !23, metadata !DIExpression()), !dbg !29
+  %2 = bitcast i64* %data to i8*, !dbg !37
+  call void @llvm.lifetime.start.p0i8(i64 8, i8* nonnull %2) #5, !dbg !37
+  call void @llvm.dbg.declare(metadata %struct.data_t* %tmpcast, metadata !24, metadata !DIExpression()), !dbg !38
+  store i64 0, i64* %data, align 8, !dbg !38
+  %tobool = icmp eq i32 %and, 0, !dbg !39
+  br i1 %tobool, label %cond.false, label %lor.end.critedge, !dbg !39
+
+cond.false:                                       ; preds = %entry
+  %3 = tail call i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.info_ts(%struct.info_t* %args, i32 0, i32 0), !dbg !40, !llvm.preserve.access.index !16
+  %4 = load i32, i32* %3, align 4, !dbg !40, !tbaa !41
+  %d1 = bitcast i64* %data to i32*, !dbg !42
+  store i32 %4, i32* %d1, align 8, !dbg !43, !tbaa !44
+  %5 = load i32, i32* %0, align 4, !dbg !46, !tbaa !31
+  %and2 = and i32 %5, 32768, !dbg !47
+  %tobool3 = icmp eq i32 %and2, 0, !dbg !48
+  %phitmp = select i1 %tobool3, i32 2, i32 1, !dbg !48
+  br label %lor.end, !dbg !48
+
+lor.end.critedge:                                 ; preds = %entry
+  %d1.c = bitcast i64* %data to i32*, !dbg !42
+  store i32 2, i32* %d1.c, align 8, !dbg !43, !tbaa !44
+  br label %lor.end, !dbg !48
+
+lor.end:                                          ; preds = %lor.end.critedge, %cond.false
+  %6 = phi i32 [ %phitmp, %cond.false ], [ 1, %lor.end.critedge ]
+  %d2 = getelementptr inbounds %struct.data_t, %struct.data_t* %tmpcast, i64 0, i32 1, !dbg !49
+  store i32 %6, i32* %d2, align 4, !dbg !50, !tbaa !51
+  call void @output(i8* nonnull %2) #5, !dbg !52
+  call void @llvm.lifetime.end.p0i8(i64 8, i8* nonnull %2) #5, !dbg !53
+  ret void, !dbg !53
+}
+
+; CHECK: r[[LOAD1:[0-9]+]] = *(u32 *)(r{{[0-9]+}} + 4)
+; CHECK: r[[LOAD1]] &= 65536
+; CHECK: r[[LOAD2:[0-9]+]] = *(u32 *)(r{{[0-9]+}} + 4)
+; CHECK: r[[LOAD2]] &= 32768
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.lifetime.start.p0i8(i64 immarg, i8* nocapture) #2
+
+; Function Attrs: nounwind readnone
+declare i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.info_ts(%struct.info_t*, i32 immarg, i32 immarg) #3
+
+declare !dbg !4 dso_local void @output(i8*) local_unnamed_addr #4
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.lifetime.end.p0i8(i64 immarg, i8* nocapture) #2
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.value(metadata, metadata, metadata) #1
+
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone speculatable willreturn }
+attributes #2 = { argmemonly nounwind willreturn }
+attributes #3 = { nounwind readnone }
+attributes #4 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #5 = { nounwind }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!8, !9, !10}
+!llvm.ident = !{!11}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 11.0.0 (https://github.com/llvm/llvm-project.git 5884aae58f56786475bbc0f13ad8bd35f7f1ce69)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !3, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "test.c", directory: "/tmp/home/yhs/work/tests/core")
+!2 = !{}
+!3 = !{!4}
+!4 = !DISubprogram(name: "output", scope: !1, file: !1, line: 10, type: !5, flags: DIFlagPrototyped, spFlags: DISPFlagOptimized, retainedNodes: !2)
+!5 = !DISubroutineType(types: !6)
+!6 = !{null, !7}
+!7 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: null, size: 64)
+!8 = !{i32 7, !"Dwarf Version", i32 4}
+!9 = !{i32 2, !"Debug Info Version", i32 3}
+!10 = !{i32 1, !"wchar_size", i32 4}
+!11 = !{!"clang version 11.0.0 (https://github.com/llvm/llvm-project.git 5884aae58f56786475bbc0f13ad8bd35f7f1ce69)"}
+!12 = distinct !DISubprogram(name: "test", scope: !1, file: !1, line: 11, type: !13, scopeLine: 11, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !21)
+!13 = !DISubroutineType(types: !14)
+!14 = !{null, !15}
+!15 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !16, size: 64)
+!16 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "info_t", file: !1, line: 5, size: 64, elements: !17)
+!17 = !{!18, !20}
+!18 = !DIDerivedType(tag: DW_TAG_member, name: "pid", scope: !16, file: !1, line: 6, baseType: !19, size: 32)
+!19 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!20 = !DIDerivedType(tag: DW_TAG_member, name: "flags", scope: !16, file: !1, line: 7, baseType: !19, size: 32, offset: 32)
+!21 = !{!22, !23, !24}
+!22 = !DILocalVariable(name: "args", arg: 1, scope: !12, file: !1, line: 11, type: !15)
+!23 = !DILocalVariable(name: "is_mask2", scope: !12, file: !1, line: 12, type: !19)
+!24 = !DILocalVariable(name: "data", scope: !12, file: !1, line: 13, type: !25)
+!25 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "data_t", file: !1, line: 1, size: 64, elements: !26)
+!26 = !{!27, !28}
+!27 = !DIDerivedType(tag: DW_TAG_member, name: "d1", scope: !25, file: !1, line: 2, baseType: !19, size: 32)
+!28 = !DIDerivedType(tag: DW_TAG_member, name: "d2", scope: !25, file: !1, line: 3, baseType: !19, size: 32, offset: 32)
+!29 = !DILocation(line: 0, scope: !12)
+!30 = !DILocation(line: 12, column: 24, scope: !12)
+!31 = !{!32, !33, i64 4}
+!32 = !{!"info_t", !33, i64 0, !33, i64 4}
+!33 = !{!"int", !34, i64 0}
+!34 = !{!"omnipotent char", !35, i64 0}
+!35 = !{!"Simple C/C++ TBAA"}
+!36 = !DILocation(line: 12, column: 30, scope: !12)
+!37 = !DILocation(line: 13, column: 3, scope: !12)
+!38 = !DILocation(line: 13, column: 17, scope: !12)
+!39 = !DILocation(line: 15, column: 13, scope: !12)
+!40 = !DILocation(line: 15, column: 34, scope: !12)
+!41 = !{!32, !33, i64 0}
+!42 = !DILocation(line: 15, column: 8, scope: !12)
+!43 = !DILocation(line: 15, column: 11, scope: !12)
+!44 = !{!45, !33, i64 0}
+!45 = !{!"data_t", !33, i64 0, !33, i64 4}
+!46 = !DILocation(line: 16, column: 33, scope: !12)
+!47 = !DILocation(line: 16, column: 39, scope: !12)
+!48 = !DILocation(line: 16, column: 23, scope: !12)
+!49 = !DILocation(line: 16, column: 8, scope: !12)
+!50 = !DILocation(line: 16, column: 11, scope: !12)
+!51 = !{!45, !33, i64 4}
+!52 = !DILocation(line: 17, column: 3, scope: !12)
+!53 = !DILocation(line: 18, column: 1, scope: !12)
diff --git a/llvm/test/CodeGen/BPF/CORE/offset-reloc-access-str.ll b/llvm/test/CodeGen/BPF/CORE/offset-reloc-access-str.ll
index 5625bf99f..1906c9ba7 100644
--- a/llvm/test/CodeGen/BPF/CORE/offset-reloc-access-str.ll
+++ b/llvm/test/CodeGen/BPF/CORE/offset-reloc-access-str.ll
@@ -1,5 +1,7 @@
 ; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck %s
 ; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck %s
+; RUN: llc -march=bpfel -mattr=+alu32 -filetype=asm -o - %s | FileCheck %s
+; RUN: llc -march=bpfeb -mattr=+alu32 -filetype=asm -o - %s | FileCheck %s
 ;
 ; Source code:
 ;   struct s { int a; int b; };
@@ -33,15 +35,17 @@ entry:
 ; CHECK:             .ascii  "0:1"                   # string offset=[[ACCESS_STR:[0-9]+]]
 ; CHECK-NEXT:        .byte   0
 ; CHECK:             .section        .BTF.ext,"",@progbits
-; CHECK:             .long   12                      # OffsetReloc
-; CHECK-NEXT:        .long   [[SEC_INDEX]]           # Offset reloc section string offset=[[SEC_INDEX]]
+; CHECK:             .long   16                      # FieldReloc
+; CHECK-NEXT:        .long   [[SEC_INDEX]]           # Field reloc section string offset=[[SEC_INDEX]]
 ; CHECK-NEXT:        .long   2
 ; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
 ; CHECK-NEXT:        .long   {{[0-9]+}}
 ; CHECK-NEXT:        .long   [[ACCESS_STR]]
+; CHECK-NEXT:        .long   0
 ; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
 ; CHECK-NEXT:        .long   {{[0-9]+}}
 ; CHECK-NEXT:        .long   [[ACCESS_STR]]
+; CHECK-NEXT:        .long   0
 
 declare dso_local i32 @get_value(i8*, i8*) local_unnamed_addr #1
 
diff --git a/llvm/test/CodeGen/BPF/CORE/offset-reloc-basic.ll b/llvm/test/CodeGen/BPF/CORE/offset-reloc-basic.ll
index a6fb3a34d..1433f0c5c 100644
--- a/llvm/test/CodeGen/BPF/CORE/offset-reloc-basic.ll
+++ b/llvm/test/CodeGen/BPF/CORE/offset-reloc-basic.ll
@@ -1,5 +1,7 @@
 ; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
 ; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfel -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
 ; Source code:
 ;    struct sk_buff {
 ;      int i;
@@ -76,7 +78,7 @@ define dso_local i32 @bpf_prog(%struct.sk_buff*) local_unnamed_addr #0 !dbg !15
 ; CHECK-NEXT:        .long   30
 ; CHECK-NEXT:        .long   1
 ; CHECK-NEXT:        .long   34                      # BTF_KIND_FUNC(id = 7)
-; CHECK-NEXT:        .long   201326592               # 0xc000000
+; CHECK-NEXT:        .long   201326593               # 0xc000001
 ; CHECK-NEXT:        .long   6
 ; CHECK-NEXT:        .byte   0                       # string offset=0
 ; CHECK-NEXT:        .ascii  "sk_buff"               # string offset=1
@@ -103,23 +105,22 @@ define dso_local i32 @bpf_prog(%struct.sk_buff*) local_unnamed_addr #0 !dbg !15
 ; CHECK-NEXT:        .short  60319                   # 0xeb9f
 ; CHECK-NEXT:        .byte   1
 ; CHECK-NEXT:        .byte   0
-; CHECK-NEXT:        .long   40
+; CHECK-NEXT:        .long   32
 ; CHECK-NEXT:        .long   0
 ; CHECK-NEXT:        .long   20
 ; CHECK-NEXT:        .long   20
 ; CHECK-NEXT:        .long   124
 ; CHECK-NEXT:        .long   144
-; CHECK-NEXT:        .long   24
-; CHECK-NEXT:        .long   168
-; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   28
 ; CHECK-NEXT:        .long   8                       # FuncInfo
 
-; CHECK:             .long   12                      # OffsetReloc
-; CHECK-NEXT:        .long   43                      # Offset reloc section string offset=43
+; CHECK:             .long   16                      # FieldReloc
+; CHECK-NEXT:        .long   43                      # Field reloc section string offset=43
 ; CHECK-NEXT:        .long   1
-; CHECK-NEXT:        .long   .Ltmp2
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
 ; CHECK-NEXT:        .long   2
 ; CHECK-NEXT:        .long   86
+; CHECK-NEXT:        .long   0
 
 ; Function Attrs: argmemonly nounwind
 declare void @llvm.lifetime.start.p0i8(i64 immarg, i8* nocapture) #1
@@ -133,7 +134,7 @@ declare void @llvm.lifetime.end.p0i8(i64 immarg, i8* nocapture) #1
 ; Function Attrs: nounwind readnone speculatable
 declare void @llvm.dbg.value(metadata, metadata, metadata) #3
 
-attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { argmemonly nounwind }
 attributes #2 = { nounwind readnone }
 attributes #3 = { nounwind readnone speculatable }
diff --git a/llvm/test/CodeGen/BPF/CORE/offset-reloc-cast-array-1.ll b/llvm/test/CodeGen/BPF/CORE/offset-reloc-cast-array-1.ll
new file mode 100644
index 000000000..1c2ae63f6
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/CORE/offset-reloc-cast-array-1.ll
@@ -0,0 +1,128 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfel -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; Source code:
+;   struct v1 {int a; int b;};
+;   typedef struct v1 __v1;
+;   typedef __v1 arr[4];
+;   struct v3 { char c; int d[100]; };
+;   #define _(x) (__builtin_preserve_access_index(x))
+;   #define cast_to_arr(x) ((arr *)(x))
+;   int get_value(const int *arg);
+;   int test(struct v3 *arg) {
+;     return get_value(_(&cast_to_arr(&arg->d[0])[0][2].b));
+;   }
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm test.c
+
+%struct.v3 = type { i8, [100 x i32] }
+%struct.v1 = type { i32, i32 }
+
+; Function Attrs: nounwind
+define dso_local i32 @test(%struct.v3* %arg) local_unnamed_addr #0 !dbg !22 {
+entry:
+  call void @llvm.dbg.value(metadata %struct.v3* %arg, metadata !32, metadata !DIExpression()), !dbg !33
+  %0 = tail call [100 x i32]* @llvm.preserve.struct.access.index.p0a100i32.p0s_struct.v3s(%struct.v3* %arg, i32 1, i32 1), !dbg !34, !llvm.preserve.access.index !26
+  %1 = tail call i32* @llvm.preserve.array.access.index.p0i32.p0a100i32([100 x i32]* %0, i32 1, i32 0), !dbg !34, !llvm.preserve.access.index !15
+  %2 = bitcast i32* %1 to [4 x %struct.v1]*, !dbg !34
+  %3 = tail call [4 x %struct.v1]* @llvm.preserve.array.access.index.p0a4s_struct.v1s.p0a4s_struct.v1s([4 x %struct.v1]* %2, i32 0, i32 0), !dbg !34, !llvm.preserve.access.index !4
+  %4 = tail call %struct.v1* @llvm.preserve.array.access.index.p0s_struct.v1s.p0a4s_struct.v1s([4 x %struct.v1]* %3, i32 1, i32 2), !dbg !34, !llvm.preserve.access.index !5
+  %5 = tail call i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.v1s(%struct.v1* %4, i32 1, i32 1), !dbg !34, !llvm.preserve.access.index !8
+  %call = tail call i32 @get_value(i32* %5) #4, !dbg !35
+  ret i32 %call, !dbg !36
+}
+
+; CHECK:              r2 = 4
+; CHECK:              r1 += r2
+; CHECK:              r2 = 20
+; CHECK:              r1 += r2
+; CHECK:              call get_value
+
+; CHECK:              .long   1                       # BTF_KIND_STRUCT(id = [[TID1:[0-9]+]])
+; CHECK:              .long   100                     # BTF_KIND_STRUCT(id = [[TID2:[0-9]+]])
+
+; CHECK:              .ascii  "v3"                    # string offset=1
+; CHECK:              .ascii  ".text"                 # string offset=46
+; CHECK:              .ascii  "0:1:0"                 # string offset=52
+; CHECK:              .ascii  "2:1"                   # string offset=107
+
+; CHECK:              .long   16                      # FieldReloc
+; CHECK-NEXT:         .long   46                      # Field reloc section string offset=46
+; CHECK-NEXT:         .long   2
+; CHECK-NEXT:         .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:         .long   [[TID1]]
+; CHECK-NEXT:         .long   52
+; CHECK-NEXT:         .long   0
+; CHECK-NEXT:         .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:         .long   [[TID2]]
+; CHECK-NEXT:         .long   107
+; CHECK-NEXT:         .long   0
+
+declare dso_local i32 @get_value(i32*) local_unnamed_addr #1
+
+; Function Attrs: nounwind readnone
+declare [100 x i32]* @llvm.preserve.struct.access.index.p0a100i32.p0s_struct.v3s(%struct.v3*, i32, i32) #2
+
+; Function Attrs: nounwind readnone
+declare i32* @llvm.preserve.array.access.index.p0i32.p0a100i32([100 x i32]*, i32, i32) #2
+
+; Function Attrs: nounwind readnone
+declare [4 x %struct.v1]* @llvm.preserve.array.access.index.p0a4s_struct.v1s.p0a4s_struct.v1s([4 x %struct.v1]*, i32, i32) #2
+
+; Function Attrs: nounwind readnone
+declare %struct.v1* @llvm.preserve.array.access.index.p0s_struct.v1s.p0a4s_struct.v1s([4 x %struct.v1]*, i32, i32) #2
+
+; Function Attrs: nounwind readnone
+declare i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.v1s(%struct.v1*, i32, i32) #2
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.value(metadata, metadata, metadata) #3
+
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { nounwind readnone }
+attributes #3 = { nounwind readnone speculatable willreturn }
+attributes #4 = { nounwind }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!18, !19, !20}
+!llvm.ident = !{!21}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 10.0.0 (trunk 367256) (llvm/trunk 367266)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !3, nameTableKind: None)
+!1 = !DIFile(filename: "test.c", directory: "/tmp/home/yhs/work/tests/llvm/cast")
+!2 = !{}
+!3 = !{!4, !15, !5}
+!4 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !5, size: 64)
+!5 = !DIDerivedType(tag: DW_TAG_typedef, name: "arr", file: !1, line: 3, baseType: !6)
+!6 = !DICompositeType(tag: DW_TAG_array_type, baseType: !7, size: 256, elements: !13)
+!7 = !DIDerivedType(tag: DW_TAG_typedef, name: "__v1", file: !1, line: 2, baseType: !8)
+!8 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "v1", file: !1, line: 1, size: 64, elements: !9)
+!9 = !{!10, !12}
+!10 = !DIDerivedType(tag: DW_TAG_member, name: "a", scope: !8, file: !1, line: 1, baseType: !11, size: 32)
+!11 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!12 = !DIDerivedType(tag: DW_TAG_member, name: "b", scope: !8, file: !1, line: 1, baseType: !11, size: 32, offset: 32)
+!13 = !{!14}
+!14 = !DISubrange(count: 4)
+!15 = !DICompositeType(tag: DW_TAG_array_type, baseType: !11, size: 3200, elements: !16)
+!16 = !{!17}
+!17 = !DISubrange(count: 100)
+!18 = !{i32 2, !"Dwarf Version", i32 4}
+!19 = !{i32 2, !"Debug Info Version", i32 3}
+!20 = !{i32 1, !"wchar_size", i32 4}
+!21 = !{!"clang version 10.0.0 (trunk 367256) (llvm/trunk 367266)"}
+!22 = distinct !DISubprogram(name: "test", scope: !1, file: !1, line: 8, type: !23, scopeLine: 8, flags: DIFlagPrototyped, isDefinition: true, isOptimized: true, unit: !0, retainedNodes: !31)
+!23 = !DISubroutineType(types: !24)
+!24 = !{!11, !25}
+!25 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !26, size: 64)
+!26 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "v3", file: !1, line: 4, size: 3232, elements: !27)
+!27 = !{!28, !30}
+!28 = !DIDerivedType(tag: DW_TAG_member, name: "c", scope: !26, file: !1, line: 4, baseType: !29, size: 8)
+!29 = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_signed_char)
+!30 = !DIDerivedType(tag: DW_TAG_member, name: "d", scope: !26, file: !1, line: 4, baseType: !15, size: 3200, offset: 32)
+!31 = !{!32}
+!32 = !DILocalVariable(name: "arg", arg: 1, scope: !22, file: !1, line: 8, type: !25)
+!33 = !DILocation(line: 0, scope: !22)
+!34 = !DILocation(line: 9, column: 20, scope: !22)
+!35 = !DILocation(line: 9, column: 10, scope: !22)
+!36 = !DILocation(line: 9, column: 3, scope: !22)
diff --git a/llvm/test/CodeGen/BPF/CORE/offset-reloc-cast-array-2.ll b/llvm/test/CodeGen/BPF/CORE/offset-reloc-cast-array-2.ll
new file mode 100644
index 000000000..3633d16ce
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/CORE/offset-reloc-cast-array-2.ll
@@ -0,0 +1,135 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfel -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; Source code:
+;   struct v1 {int a; int b;};
+;   typedef struct v1 __v1;
+;   typedef __v1 arr[4][4];
+;   struct v3 { char c; int d[100]; };
+;   #define _(x) (__builtin_preserve_access_index(x))
+;   #define cast_to_arr(x) ((arr *)(x))
+;   int get_value(const int *arg);
+;   int test(struct v3 *arg) {
+;     return get_value(_(&cast_to_arr(&arg->d[0])[0][2][3].b));
+;   }
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm test.c
+
+%struct.v3 = type { i8, [100 x i32] }
+%struct.v1 = type { i32, i32 }
+
+; Function Attrs: nounwind
+define dso_local i32 @test(%struct.v3* %arg) local_unnamed_addr #0 !dbg !24 {
+entry:
+  call void @llvm.dbg.value(metadata %struct.v3* %arg, metadata !34, metadata !DIExpression()), !dbg !35
+  %0 = tail call [100 x i32]* @llvm.preserve.struct.access.index.p0a100i32.p0s_struct.v3s(%struct.v3* %arg, i32 1, i32 1), !dbg !36, !llvm.preserve.access.index !28
+  %1 = tail call i32* @llvm.preserve.array.access.index.p0i32.p0a100i32([100 x i32]* %0, i32 1, i32 0), !dbg !36, !llvm.preserve.access.index !15
+  %2 = bitcast i32* %1 to [4 x [4 x %struct.v1]]*, !dbg !36
+  %3 = tail call [4 x [4 x %struct.v1]]* @llvm.preserve.array.access.index.p0a4a4s_struct.v1s.p0a4a4s_struct.v1s([4 x [4 x %struct.v1]]* %2, i32 0, i32 0), !dbg !36, !llvm.preserve.access.index !4
+  %4 = tail call [4 x %struct.v1]* @llvm.preserve.array.access.index.p0a4s_struct.v1s.p0a4a4s_struct.v1s([4 x [4 x %struct.v1]]* %3, i32 1, i32 2), !dbg !36, !llvm.preserve.access.index !5
+  %5 = tail call %struct.v1* @llvm.preserve.array.access.index.p0s_struct.v1s.p0a4s_struct.v1s([4 x %struct.v1]* %4, i32 1, i32 3), !dbg !36, !llvm.preserve.access.index !18
+  %6 = tail call i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.v1s(%struct.v1* %5, i32 1, i32 1), !dbg !36, !llvm.preserve.access.index !8
+  %call = tail call i32 @get_value(i32* %6) #4, !dbg !37
+  ret i32 %call, !dbg !38
+}
+
+; CHECK:              r2 = 4
+; CHECK:              r1 += r2
+; CHECK:              r2 = 92
+; CHECK:              r1 += r2
+; CHECK:              call get_value
+
+; CHECK:              .long   1                       # BTF_KIND_STRUCT(id = [[TID1:[0-9]+]])
+; CHECK:              .long   100                     # BTF_KIND_STRUCT(id = [[TID2:[0-9]+]])
+
+; CHECK:              .ascii  "v3"                    # string offset=1
+; CHECK:              .ascii  ".text"                 # string offset=46
+; CHECK:              .ascii  "0:1:0"                 # string offset=52
+; CHECK:              .ascii  "v1"                    # string offset=100
+; CHECK:              .ascii  "11:1"                  # string offset=107
+
+; CHECK:              .long   16                      # FieldReloc
+; CHECK-NEXT:         .long   46                      # Field reloc section string offset=46
+; CHECK-NEXT:         .long   2
+; CHECK-NEXT:         .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:         .long   [[TID1]]
+; CHECK-NEXT:         .long   52
+; CHECK-NEXT:         .long   0
+; CHECK-NEXT:         .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:         .long   [[TID2]]
+; CHECK-NEXT:         .long   107
+; CHECK-NEXT:         .long   0
+
+declare dso_local i32 @get_value(i32*) local_unnamed_addr #1
+
+; Function Attrs: nounwind readnone
+declare [100 x i32]* @llvm.preserve.struct.access.index.p0a100i32.p0s_struct.v3s(%struct.v3*, i32, i32) #2
+
+; Function Attrs: nounwind readnone
+declare i32* @llvm.preserve.array.access.index.p0i32.p0a100i32([100 x i32]*, i32, i32) #2
+
+; Function Attrs: nounwind readnone
+declare [4 x [4 x %struct.v1]]* @llvm.preserve.array.access.index.p0a4a4s_struct.v1s.p0a4a4s_struct.v1s([4 x [4 x %struct.v1]]*, i32, i32) #2
+
+; Function Attrs: nounwind readnone
+declare [4 x %struct.v1]* @llvm.preserve.array.access.index.p0a4s_struct.v1s.p0a4a4s_struct.v1s([4 x [4 x %struct.v1]]*, i32, i32) #2
+
+; Function Attrs: nounwind readnone
+declare %struct.v1* @llvm.preserve.array.access.index.p0s_struct.v1s.p0a4s_struct.v1s([4 x %struct.v1]*, i32, i32) #2
+
+; Function Attrs: nounwind readnone
+declare i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.v1s(%struct.v1*, i32, i32) #2
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.value(metadata, metadata, metadata) #3
+
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { nounwind readnone }
+attributes #3 = { nounwind readnone speculatable willreturn }
+attributes #4 = { nounwind }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!20, !21, !22}
+!llvm.ident = !{!23}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 10.0.0 (trunk 367256) (llvm/trunk 367266)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !3, nameTableKind: None)
+!1 = !DIFile(filename: "test.c", directory: "/tmp/home/yhs/work/tests/llvm/cast")
+!2 = !{}
+!3 = !{!4, !15, !5, !18}
+!4 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !5, size: 64)
+!5 = !DIDerivedType(tag: DW_TAG_typedef, name: "arr", file: !1, line: 3, baseType: !6)
+!6 = !DICompositeType(tag: DW_TAG_array_type, baseType: !7, size: 1024, elements: !13)
+!7 = !DIDerivedType(tag: DW_TAG_typedef, name: "__v1", file: !1, line: 2, baseType: !8)
+!8 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "v1", file: !1, line: 1, size: 64, elements: !9)
+!9 = !{!10, !12}
+!10 = !DIDerivedType(tag: DW_TAG_member, name: "a", scope: !8, file: !1, line: 1, baseType: !11, size: 32)
+!11 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!12 = !DIDerivedType(tag: DW_TAG_member, name: "b", scope: !8, file: !1, line: 1, baseType: !11, size: 32, offset: 32)
+!13 = !{!14, !14}
+!14 = !DISubrange(count: 4)
+!15 = !DICompositeType(tag: DW_TAG_array_type, baseType: !11, size: 3200, elements: !16)
+!16 = !{!17}
+!17 = !DISubrange(count: 100)
+!18 = !DICompositeType(tag: DW_TAG_array_type, baseType: !7, size: 256, elements: !19)
+!19 = !{!14}
+!20 = !{i32 2, !"Dwarf Version", i32 4}
+!21 = !{i32 2, !"Debug Info Version", i32 3}
+!22 = !{i32 1, !"wchar_size", i32 4}
+!23 = !{!"clang version 10.0.0 (trunk 367256) (llvm/trunk 367266)"}
+!24 = distinct !DISubprogram(name: "test", scope: !1, file: !1, line: 8, type: !25, scopeLine: 8, flags: DIFlagPrototyped, isDefinition: true, isOptimized: true, unit: !0, retainedNodes: !33)
+!25 = !DISubroutineType(types: !26)
+!26 = !{!11, !27}
+!27 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !28, size: 64)
+!28 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "v3", file: !1, line: 4, size: 3232, elements: !29)
+!29 = !{!30, !32}
+!30 = !DIDerivedType(tag: DW_TAG_member, name: "c", scope: !28, file: !1, line: 4, baseType: !31, size: 8)
+!31 = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_signed_char)
+!32 = !DIDerivedType(tag: DW_TAG_member, name: "d", scope: !28, file: !1, line: 4, baseType: !15, size: 3200, offset: 32)
+!33 = !{!34}
+!34 = !DILocalVariable(name: "arg", arg: 1, scope: !24, file: !1, line: 8, type: !27)
+!35 = !DILocation(line: 0, scope: !24)
+!36 = !DILocation(line: 9, column: 20, scope: !24)
+!37 = !DILocation(line: 9, column: 10, scope: !24)
+!38 = !DILocation(line: 9, column: 3, scope: !24)
diff --git a/llvm/test/CodeGen/BPF/CORE/offset-reloc-cast-struct-1.ll b/llvm/test/CodeGen/BPF/CORE/offset-reloc-cast-struct-1.ll
new file mode 100644
index 000000000..b2b9455ca
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/CORE/offset-reloc-cast-struct-1.ll
@@ -0,0 +1,116 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfel -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; Source code:
+;   struct v1 { int a; int b; };
+;   struct v2 { int c; int d; };
+;   struct v3 { char c; struct v2 d; };
+;   #define _(x) (__builtin_preserve_access_index(x))
+;   #define cast_to_v1(x) ((struct v1 *)(x))
+;   int get_value(const int *arg);
+;   int test(struct v3 *arg) {
+;     return get_value(_(&cast_to_v1(&arg->d)->b));
+;   }
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm test.c
+
+%struct.v3 = type { i8, %struct.v2 }
+%struct.v2 = type { i32, i32 }
+%struct.v1 = type { i32, i32 }
+
+; Function Attrs: nounwind
+define dso_local i32 @test(%struct.v3* %arg) local_unnamed_addr #0 !dbg !14 {
+entry:
+  call void @llvm.dbg.value(metadata %struct.v3* %arg, metadata !28, metadata !DIExpression()), !dbg !29
+  %0 = tail call %struct.v2* @llvm.preserve.struct.access.index.p0s_struct.v2s.p0s_struct.v3s(%struct.v3* %arg, i32 1, i32 1), !dbg !30, !llvm.preserve.access.index !18
+  %1 = bitcast %struct.v2* %0 to %struct.v1*, !dbg !30
+  %2 = tail call i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.v1s(%struct.v1* %1, i32 1, i32 1), !dbg !30, !llvm.preserve.access.index !5
+  %call = tail call i32 @get_value(i32* %2) #4, !dbg !31
+  ret i32 %call, !dbg !32
+}
+
+; CHECK:              r2 = 4
+; CHECK:              r1 += r2
+; CHECK:              r2 = 4
+; CHECK:              r1 += r2
+; CHECK:              call get_value
+
+; CHECK:             .long   1                       # BTF_KIND_STRUCT(id = [[V3_TID:[0-9]+]])
+; CHECK:             .long   81                      # BTF_KIND_STRUCT(id = [[V1_TID:[0-9]+]])
+
+; CHECK:             .ascii  "v3"                    # string offset=1
+; CHECK-NEXT:        .byte   0
+; CHECK:             .ascii  ".text"                 # string offset=[[SEC_STR:[0-9]+]]
+; CHECK-NEXT:        .byte   0
+; CHECK:             .ascii  "0:1"                   # string offset=[[ACCESS_STR:[0-9]+]]
+; CHECK-NEXT:        .byte   0
+; CHECK:             .ascii  "v1"                    # string offset=81
+; CHECK-NEXT:        .byte   0
+
+; CHECK:             .long   16                      # FieldReloc
+; CHECK-NEXT:        .long   [[SEC_STR]]             # Field reloc section string offset=[[SEC_STR]]
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   [[V3_TID]]
+; CHECK-NEXT:        .long   [[ACCESS_STR]]
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   [[V1_TID]]
+; CHECK-NEXT:        .long   [[ACCESS_STR]]
+; CHECK-NEXT:        .long   0
+
+declare dso_local i32 @get_value(i32*) local_unnamed_addr #1
+
+; Function Attrs: nounwind readnone
+declare %struct.v2* @llvm.preserve.struct.access.index.p0s_struct.v2s.p0s_struct.v3s(%struct.v3*, i32, i32) #2
+
+; Function Attrs: nounwind readnone
+declare i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.v1s(%struct.v1*, i32, i32) #2
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.value(metadata, metadata, metadata) #3
+
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { nounwind readnone }
+attributes #3 = { nounwind readnone speculatable willreturn }
+attributes #4 = { nounwind }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!10, !11, !12}
+!llvm.ident = !{!13}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 10.0.0 (trunk 367256) (llvm/trunk 367266)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !3, nameTableKind: None)
+!1 = !DIFile(filename: "test.c", directory: "/tmp/home/yhs/work/tests/llvm/cast")
+!2 = !{}
+!3 = !{!4}
+!4 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !5, size: 64)
+!5 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "v1", file: !1, line: 1, size: 64, elements: !6)
+!6 = !{!7, !9}
+!7 = !DIDerivedType(tag: DW_TAG_member, name: "a", scope: !5, file: !1, line: 1, baseType: !8, size: 32)
+!8 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!9 = !DIDerivedType(tag: DW_TAG_member, name: "b", scope: !5, file: !1, line: 1, baseType: !8, size: 32, offset: 32)
+!10 = !{i32 2, !"Dwarf Version", i32 4}
+!11 = !{i32 2, !"Debug Info Version", i32 3}
+!12 = !{i32 1, !"wchar_size", i32 4}
+!13 = !{!"clang version 10.0.0 (trunk 367256) (llvm/trunk 367266)"}
+!14 = distinct !DISubprogram(name: "test", scope: !1, file: !1, line: 7, type: !15, scopeLine: 7, flags: DIFlagPrototyped, isDefinition: true, isOptimized: true, unit: !0, retainedNodes: !27)
+!15 = !DISubroutineType(types: !16)
+!16 = !{!8, !17}
+!17 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !18, size: 64)
+!18 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "v3", file: !1, line: 3, size: 96, elements: !19)
+!19 = !{!20, !22}
+!20 = !DIDerivedType(tag: DW_TAG_member, name: "c", scope: !18, file: !1, line: 3, baseType: !21, size: 8)
+!21 = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_signed_char)
+!22 = !DIDerivedType(tag: DW_TAG_member, name: "d", scope: !18, file: !1, line: 3, baseType: !23, size: 64, offset: 32)
+!23 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "v2", file: !1, line: 2, size: 64, elements: !24)
+!24 = !{!25, !26}
+!25 = !DIDerivedType(tag: DW_TAG_member, name: "c", scope: !23, file: !1, line: 2, baseType: !8, size: 32)
+!26 = !DIDerivedType(tag: DW_TAG_member, name: "d", scope: !23, file: !1, line: 2, baseType: !8, size: 32, offset: 32)
+!27 = !{!28}
+!28 = !DILocalVariable(name: "arg", arg: 1, scope: !14, file: !1, line: 7, type: !17)
+!29 = !DILocation(line: 0, scope: !14)
+!30 = !DILocation(line: 8, column: 20, scope: !14)
+!31 = !DILocation(line: 8, column: 10, scope: !14)
+!32 = !DILocation(line: 8, column: 3, scope: !14)
diff --git a/llvm/test/CodeGen/BPF/CORE/offset-reloc-cast-struct-2.ll b/llvm/test/CodeGen/BPF/CORE/offset-reloc-cast-struct-2.ll
new file mode 100644
index 000000000..6bfe034b9
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/CORE/offset-reloc-cast-struct-2.ll
@@ -0,0 +1,121 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfel -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; Source code:
+;   struct v1 { int a; int b; };
+;   typedef struct v1 __v1;
+;   struct v2 { int c; int d; };
+;   typedef struct v2 __v2;
+;   struct v3 { char c; volatile const __v2 d; };
+;   typedef struct v3 __v3;
+;   #define _(x) (__builtin_preserve_access_index(x))
+;   #define cast_to_v1(x) ((__v1 *)(x))
+;   int get_value(const int *arg);
+;   int test(__v3 *arg) {
+;     return get_value(_(&cast_to_v1(&arg->d)->b));
+;   }
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm test.c
+
+%struct.v3 = type { i8, %struct.v2 }
+%struct.v2 = type { i32, i32 }
+%struct.v1 = type { i32, i32 }
+
+; Function Attrs: nounwind
+define dso_local i32 @test(%struct.v3* %arg) local_unnamed_addr #0 !dbg !15 {
+entry:
+  call void @llvm.dbg.value(metadata %struct.v3* %arg, metadata !33, metadata !DIExpression()), !dbg !34
+  %0 = tail call %struct.v2* @llvm.preserve.struct.access.index.p0s_struct.v2s.p0s_struct.v3s(%struct.v3* %arg, i32 1, i32 1), !dbg !35, !llvm.preserve.access.index !20
+  %1 = bitcast %struct.v2* %0 to %struct.v1*, !dbg !35
+  %2 = tail call i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.v1s(%struct.v1* %1, i32 1, i32 1), !dbg !35, !llvm.preserve.access.index !6
+  %call = tail call i32 @get_value(i32* %2) #4, !dbg !36
+  ret i32 %call, !dbg !37
+}
+
+; CHECK:             r2 = 4
+; CHECK:             r1 += r2
+; CHECK:             r2 = 4
+; CHECK:             r1 += r2
+; CHECK:             call get_value
+
+; CHECK:             .long   6                       # BTF_KIND_STRUCT(id = [[TID1:[0-9]+]])
+; CHECK:             .long   91                      # BTF_KIND_STRUCT(id = [[TID2:[0-9]+]])
+
+; CHECK:             .ascii  "v3"                    # string offset=6
+; CHECK:             .ascii  ".text"                 # string offset=39
+; CHECK:             .ascii  "0:1"                   # string offset=45
+; CHECK:             .ascii  "v1"                    # string offset=91
+
+
+; CHECK:             .long   16                      # FieldReloc
+; CHECK-NEXT:        .long   39                      # Field reloc section string offset=39
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   [[TID1]]
+; CHECK-NEXT:        .long   45
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   [[TID2]]
+; CHECK-NEXT:        .long   45
+; CHECK-NEXT:        .long   0
+
+declare dso_local i32 @get_value(i32*) local_unnamed_addr #1
+
+; Function Attrs: nounwind readnone
+declare %struct.v2* @llvm.preserve.struct.access.index.p0s_struct.v2s.p0s_struct.v3s(%struct.v3*, i32, i32) #2
+
+; Function Attrs: nounwind readnone
+declare i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.v1s(%struct.v1*, i32, i32) #2
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.value(metadata, metadata, metadata) #3
+
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { nounwind readnone }
+attributes #3 = { nounwind readnone speculatable willreturn }
+attributes #4 = { nounwind }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!11, !12, !13}
+!llvm.ident = !{!14}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 10.0.0 (trunk 367256) (llvm/trunk 367266)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !3, nameTableKind: None)
+!1 = !DIFile(filename: "test.c", directory: "/tmp/home/yhs/work/tests/llvm/cast")
+!2 = !{}
+!3 = !{!4}
+!4 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !5, size: 64)
+!5 = !DIDerivedType(tag: DW_TAG_typedef, name: "__v1", file: !1, line: 2, baseType: !6)
+!6 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "v1", file: !1, line: 1, size: 64, elements: !7)
+!7 = !{!8, !10}
+!8 = !DIDerivedType(tag: DW_TAG_member, name: "a", scope: !6, file: !1, line: 1, baseType: !9, size: 32)
+!9 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!10 = !DIDerivedType(tag: DW_TAG_member, name: "b", scope: !6, file: !1, line: 1, baseType: !9, size: 32, offset: 32)
+!11 = !{i32 2, !"Dwarf Version", i32 4}
+!12 = !{i32 2, !"Debug Info Version", i32 3}
+!13 = !{i32 1, !"wchar_size", i32 4}
+!14 = !{!"clang version 10.0.0 (trunk 367256) (llvm/trunk 367266)"}
+!15 = distinct !DISubprogram(name: "test", scope: !1, file: !1, line: 10, type: !16, scopeLine: 10, flags: DIFlagPrototyped, isDefinition: true, isOptimized: true, unit: !0, retainedNodes: !32)
+!16 = !DISubroutineType(types: !17)
+!17 = !{!9, !18}
+!18 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !19, size: 64)
+!19 = !DIDerivedType(tag: DW_TAG_typedef, name: "__v3", file: !1, line: 6, baseType: !20)
+!20 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "v3", file: !1, line: 5, size: 96, elements: !21)
+!21 = !{!22, !24}
+!22 = !DIDerivedType(tag: DW_TAG_member, name: "c", scope: !20, file: !1, line: 5, baseType: !23, size: 8)
+!23 = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_signed_char)
+!24 = !DIDerivedType(tag: DW_TAG_member, name: "d", scope: !20, file: !1, line: 5, baseType: !25, size: 64, offset: 32)
+!25 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !26)
+!26 = !DIDerivedType(tag: DW_TAG_volatile_type, baseType: !27)
+!27 = !DIDerivedType(tag: DW_TAG_typedef, name: "__v2", file: !1, line: 4, baseType: !28)
+!28 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "v2", file: !1, line: 3, size: 64, elements: !29)
+!29 = !{!30, !31}
+!30 = !DIDerivedType(tag: DW_TAG_member, name: "c", scope: !28, file: !1, line: 3, baseType: !9, size: 32)
+!31 = !DIDerivedType(tag: DW_TAG_member, name: "d", scope: !28, file: !1, line: 3, baseType: !9, size: 32, offset: 32)
+!32 = !{!33}
+!33 = !DILocalVariable(name: "arg", arg: 1, scope: !15, file: !1, line: 10, type: !18)
+!34 = !DILocation(line: 0, scope: !15)
+!35 = !DILocation(line: 11, column: 20, scope: !15)
+!36 = !DILocation(line: 11, column: 10, scope: !15)
+!37 = !DILocation(line: 11, column: 3, scope: !15)
diff --git a/llvm/test/CodeGen/BPF/CORE/offset-reloc-cast-struct-3.ll b/llvm/test/CodeGen/BPF/CORE/offset-reloc-cast-struct-3.ll
new file mode 100644
index 000000000..90d194efe
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/CORE/offset-reloc-cast-struct-3.ll
@@ -0,0 +1,120 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfel -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; Source code:
+;   struct v1 { int a; int b; };
+;   typedef struct v1 __v1;
+;   typedef int __int;
+;   struct v3 { char c; __int d[40]; };
+;   typedef struct v3 __v3;
+;   #define _(x) (__builtin_preserve_access_index(x))
+;   #define cast_to_v1(x) ((__v1 *)(x))
+;   int get_value(const int *arg);
+;   int test(__v3 *arg) {
+;     return get_value(_(&cast_to_v1(&arg->d[4])->b));
+;   }
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm test.c
+
+%struct.v3 = type { i8, [40 x i32] }
+%struct.v1 = type { i32, i32 }
+
+; Function Attrs: nounwind
+define dso_local i32 @test(%struct.v3* %arg) local_unnamed_addr #0 !dbg !19 {
+entry:
+  call void @llvm.dbg.value(metadata %struct.v3* %arg, metadata !30, metadata !DIExpression()), !dbg !31
+  %0 = tail call [40 x i32]* @llvm.preserve.struct.access.index.p0a40i32.p0s_struct.v3s(%struct.v3* %arg, i32 1, i32 1), !dbg !32, !llvm.preserve.access.index !24
+  %1 = tail call i32* @llvm.preserve.array.access.index.p0i32.p0a40i32([40 x i32]* %0, i32 1, i32 4), !dbg !32, !llvm.preserve.access.index !11
+  %2 = bitcast i32* %1 to %struct.v1*, !dbg !32
+  %3 = tail call i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.v1s(%struct.v1* %2, i32 1, i32 1), !dbg !32, !llvm.preserve.access.index !6
+  %call = tail call i32 @get_value(i32* %3) #4, !dbg !33
+  ret i32 %call, !dbg !34
+}
+
+; CHECK:             r2 = 20
+; CHECK:             r1 += r2
+; CHECK:             r2 = 4
+; CHECK:             r1 += r2
+; CHECK:             call get_value
+
+; CHECK:             .long   6                       # BTF_KIND_STRUCT(id = [[TID1:[0-9]+]])
+; CHECK:             .long   111                     # BTF_KIND_STRUCT(id = [[TID2:[0-9]+]])
+
+; CHECK:             .ascii  "v3"                    # string offset=6
+; CHECK:             .ascii  ".text"                 # string offset=57
+; CHECK:             .ascii  "0:1:4"                 # string offset=63
+; CHECK:             .ascii  "v1"                    # string offset=111
+; CHECK:             .ascii  "0:1"                   # string offset=118
+
+; CHECK:             .long   16                      # FieldReloc
+; CHECK-NEXT:        .long   57                      # Field reloc section string offset=57
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   [[TID1]]
+; CHECK-NEXT:        .long   63
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   [[TID2]]
+; CHECK-NEXT:        .long   118
+; CHECK-NEXT:        .long   0
+
+declare dso_local i32 @get_value(i32*) local_unnamed_addr #1
+
+; Function Attrs: nounwind readnone
+declare [40 x i32]* @llvm.preserve.struct.access.index.p0a40i32.p0s_struct.v3s(%struct.v3*, i32, i32) #2
+
+; Function Attrs: nounwind readnone
+declare i32* @llvm.preserve.array.access.index.p0i32.p0a40i32([40 x i32]*, i32, i32) #2
+
+; Function Attrs: nounwind readnone
+declare i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.v1s(%struct.v1*, i32, i32) #2
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.value(metadata, metadata, metadata) #3
+
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { nounwind readnone }
+attributes #3 = { nounwind readnone speculatable willreturn }
+attributes #4 = { nounwind }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!15, !16, !17}
+!llvm.ident = !{!18}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 10.0.0 (trunk 367256) (llvm/trunk 367266)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !3, nameTableKind: None)
+!1 = !DIFile(filename: "test.c", directory: "/tmp/home/yhs/work/tests/llvm/cast")
+!2 = !{}
+!3 = !{!4, !11}
+!4 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !5, size: 64)
+!5 = !DIDerivedType(tag: DW_TAG_typedef, name: "__v1", file: !1, line: 2, baseType: !6)
+!6 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "v1", file: !1, line: 1, size: 64, elements: !7)
+!7 = !{!8, !10}
+!8 = !DIDerivedType(tag: DW_TAG_member, name: "a", scope: !6, file: !1, line: 1, baseType: !9, size: 32)
+!9 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!10 = !DIDerivedType(tag: DW_TAG_member, name: "b", scope: !6, file: !1, line: 1, baseType: !9, size: 32, offset: 32)
+!11 = !DICompositeType(tag: DW_TAG_array_type, baseType: !12, size: 1280, elements: !13)
+!12 = !DIDerivedType(tag: DW_TAG_typedef, name: "__int", file: !1, line: 3, baseType: !9)
+!13 = !{!14}
+!14 = !DISubrange(count: 40)
+!15 = !{i32 2, !"Dwarf Version", i32 4}
+!16 = !{i32 2, !"Debug Info Version", i32 3}
+!17 = !{i32 1, !"wchar_size", i32 4}
+!18 = !{!"clang version 10.0.0 (trunk 367256) (llvm/trunk 367266)"}
+!19 = distinct !DISubprogram(name: "test", scope: !1, file: !1, line: 9, type: !20, scopeLine: 9, flags: DIFlagPrototyped, isDefinition: true, isOptimized: true, unit: !0, retainedNodes: !29)
+!20 = !DISubroutineType(types: !21)
+!21 = !{!9, !22}
+!22 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !23, size: 64)
+!23 = !DIDerivedType(tag: DW_TAG_typedef, name: "__v3", file: !1, line: 5, baseType: !24)
+!24 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "v3", file: !1, line: 4, size: 1312, elements: !25)
+!25 = !{!26, !28}
+!26 = !DIDerivedType(tag: DW_TAG_member, name: "c", scope: !24, file: !1, line: 4, baseType: !27, size: 8)
+!27 = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_signed_char)
+!28 = !DIDerivedType(tag: DW_TAG_member, name: "d", scope: !24, file: !1, line: 4, baseType: !11, size: 1280, offset: 32)
+!29 = !{!30}
+!30 = !DILocalVariable(name: "arg", arg: 1, scope: !19, file: !1, line: 9, type: !22)
+!31 = !DILocation(line: 0, scope: !19)
+!32 = !DILocation(line: 10, column: 20, scope: !19)
+!33 = !DILocation(line: 10, column: 10, scope: !19)
+!34 = !DILocation(line: 10, column: 3, scope: !19)
diff --git a/llvm/test/CodeGen/BPF/CORE/offset-reloc-cast-union-1.ll b/llvm/test/CodeGen/BPF/CORE/offset-reloc-cast-union-1.ll
new file mode 100644
index 000000000..b0d0c2ad9
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/CORE/offset-reloc-cast-union-1.ll
@@ -0,0 +1,121 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfel -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; Source code:
+;   union v1 { int a; int b; };
+;   typedef union v1 __v1;
+;   union v2 { int c; int d; };
+;   typedef union v2 __v2;
+;   union v3 { char c; volatile const __v2 d; };
+;   typedef union v3 __v3;
+;   #define _(x) (__builtin_preserve_access_index(x))
+;   #define cast_to_v1(x) ((__v1 *)(x))
+;   int get_value(const int *arg);
+;   int test(__v3 *arg) {
+;     return get_value(_(&cast_to_v1(&arg->d)->b));
+;   }
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm test.c
+
+%union.v3 = type { %union.v2 }
+%union.v2 = type { i32 }
+%union.v1 = type { i32 }
+
+; Function Attrs: nounwind
+define dso_local i32 @test(%union.v3* %arg) local_unnamed_addr #0 !dbg !15 {
+entry:
+  call void @llvm.dbg.value(metadata %union.v3* %arg, metadata !33, metadata !DIExpression()), !dbg !34
+  %0 = tail call %union.v3* @llvm.preserve.union.access.index.p0s_union.v3s.p0s_union.v3s(%union.v3* %arg, i32 1), !dbg !35, !llvm.preserve.access.index !20
+  %1 = bitcast %union.v3* %0 to %union.v1*, !dbg !35
+  %2 = tail call %union.v1* @llvm.preserve.union.access.index.p0s_union.v1s.p0s_union.v1s(%union.v1* %1, i32 1), !dbg !35, !llvm.preserve.access.index !6
+  %b = getelementptr inbounds %union.v1, %union.v1* %2, i64 0, i32 0, !dbg !35
+  %call = tail call i32 @get_value(i32* %b) #4, !dbg !36
+  ret i32 %call, !dbg !37
+}
+
+; CHECK:             r2 = 0
+; CHECK:             r1 += r2
+; CHECK:             r2 = 0
+; CHECK:             r1 += r2
+; CHECK:             call get_value
+
+; CHECK:             .long   6                       # BTF_KIND_UNION(id = [[TID1:[0-9]+]])
+; CHECK:             .long   91                      # BTF_KIND_UNION(id = [[TID2:[0-9]+]])
+
+; CHECK:             .ascii  "v3"                    # string offset=6
+; CHECK:             .ascii  ".text"                 # string offset=39
+; CHECK:             .ascii  "0:1"                   # string offset=45
+; CHECK:             .ascii  "v1"                    # string offset=91
+
+; CHECK:             .long   16                      # FieldReloc
+; CHECK-NEXT:        .long   39                      # Field reloc section string offset=39
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   [[TID1]]
+; CHECK-NEXT:        .long   45
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   [[TID2]]
+; CHECK-NEXT:        .long   45
+; CHECK-NEXT:        .long   0
+
+declare dso_local i32 @get_value(i32*) local_unnamed_addr #1
+
+; Function Attrs: nounwind readnone
+declare %union.v3* @llvm.preserve.union.access.index.p0s_union.v3s.p0s_union.v3s(%union.v3*, i32) #2
+
+; Function Attrs: nounwind readnone
+declare %union.v1* @llvm.preserve.union.access.index.p0s_union.v1s.p0s_union.v1s(%union.v1*, i32) #2
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.value(metadata, metadata, metadata) #3
+
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { nounwind readnone }
+attributes #3 = { nounwind readnone speculatable willreturn }
+attributes #4 = { nounwind }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!11, !12, !13}
+!llvm.ident = !{!14}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 10.0.0 (trunk 367256) (llvm/trunk 367266)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !3, nameTableKind: None)
+!1 = !DIFile(filename: "test.c", directory: "/tmp/home/yhs/work/tests/llvm/cast")
+!2 = !{}
+!3 = !{!4}
+!4 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !5, size: 64)
+!5 = !DIDerivedType(tag: DW_TAG_typedef, name: "__v1", file: !1, line: 2, baseType: !6)
+!6 = distinct !DICompositeType(tag: DW_TAG_union_type, name: "v1", file: !1, line: 1, size: 32, elements: !7)
+!7 = !{!8, !10}
+!8 = !DIDerivedType(tag: DW_TAG_member, name: "a", scope: !6, file: !1, line: 1, baseType: !9, size: 32)
+!9 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!10 = !DIDerivedType(tag: DW_TAG_member, name: "b", scope: !6, file: !1, line: 1, baseType: !9, size: 32)
+!11 = !{i32 2, !"Dwarf Version", i32 4}
+!12 = !{i32 2, !"Debug Info Version", i32 3}
+!13 = !{i32 1, !"wchar_size", i32 4}
+!14 = !{!"clang version 10.0.0 (trunk 367256) (llvm/trunk 367266)"}
+!15 = distinct !DISubprogram(name: "test", scope: !1, file: !1, line: 10, type: !16, scopeLine: 10, flags: DIFlagPrototyped, isDefinition: true, isOptimized: true, unit: !0, retainedNodes: !32)
+!16 = !DISubroutineType(types: !17)
+!17 = !{!9, !18}
+!18 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !19, size: 64)
+!19 = !DIDerivedType(tag: DW_TAG_typedef, name: "__v3", file: !1, line: 6, baseType: !20)
+!20 = distinct !DICompositeType(tag: DW_TAG_union_type, name: "v3", file: !1, line: 5, size: 32, elements: !21)
+!21 = !{!22, !24}
+!22 = !DIDerivedType(tag: DW_TAG_member, name: "c", scope: !20, file: !1, line: 5, baseType: !23, size: 8)
+!23 = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_signed_char)
+!24 = !DIDerivedType(tag: DW_TAG_member, name: "d", scope: !20, file: !1, line: 5, baseType: !25, size: 32)
+!25 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !26)
+!26 = !DIDerivedType(tag: DW_TAG_volatile_type, baseType: !27)
+!27 = !DIDerivedType(tag: DW_TAG_typedef, name: "__v2", file: !1, line: 4, baseType: !28)
+!28 = distinct !DICompositeType(tag: DW_TAG_union_type, name: "v2", file: !1, line: 3, size: 32, elements: !29)
+!29 = !{!30, !31}
+!30 = !DIDerivedType(tag: DW_TAG_member, name: "c", scope: !28, file: !1, line: 3, baseType: !9, size: 32)
+!31 = !DIDerivedType(tag: DW_TAG_member, name: "d", scope: !28, file: !1, line: 3, baseType: !9, size: 32)
+!32 = !{!33}
+!33 = !DILocalVariable(name: "arg", arg: 1, scope: !15, file: !1, line: 10, type: !18)
+!34 = !DILocation(line: 0, scope: !15)
+!35 = !DILocation(line: 11, column: 20, scope: !15)
+!36 = !DILocation(line: 11, column: 10, scope: !15)
+!37 = !DILocation(line: 11, column: 3, scope: !15)
diff --git a/llvm/test/CodeGen/BPF/CORE/offset-reloc-cast-union-2.ll b/llvm/test/CodeGen/BPF/CORE/offset-reloc-cast-union-2.ll
new file mode 100644
index 000000000..832f561a1
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/CORE/offset-reloc-cast-union-2.ll
@@ -0,0 +1,122 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfel -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; Source code:
+;   union v1 { int a; int b; };
+;   typedef union v1 __v1;
+;   typedef int __int;
+;   union v3 { char c; __int d[40]; };
+;   typedef union v3 __v3;
+;   #define _(x) (__builtin_preserve_access_index(x))
+;   #define cast_to_v1(x) ((__v1 *)(x))
+;   int get_value(const int *arg);
+;   int test(__v3 *arg) {
+;     return get_value(_(&cast_to_v1(&arg->d[4])->b));
+;   }
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm test.c
+
+%union.v3 = type { [40 x i32] }
+%union.v1 = type { i32 }
+
+; Function Attrs: nounwind
+define dso_local i32 @test(%union.v3* %arg) local_unnamed_addr #0 !dbg !19 {
+entry:
+  call void @llvm.dbg.value(metadata %union.v3* %arg, metadata !30, metadata !DIExpression()), !dbg !31
+  %0 = tail call %union.v3* @llvm.preserve.union.access.index.p0s_union.v3s.p0s_union.v3s(%union.v3* %arg, i32 1), !dbg !32, !llvm.preserve.access.index !24
+  %d = getelementptr inbounds %union.v3, %union.v3* %0, i64 0, i32 0, !dbg !32
+  %1 = tail call i32* @llvm.preserve.array.access.index.p0i32.p0a40i32([40 x i32]* %d, i32 1, i32 4), !dbg !32, !llvm.preserve.access.index !11
+  %2 = bitcast i32* %1 to %union.v1*, !dbg !32
+  %3 = tail call %union.v1* @llvm.preserve.union.access.index.p0s_union.v1s.p0s_union.v1s(%union.v1* %2, i32 1), !dbg !32, !llvm.preserve.access.index !6
+  %b = getelementptr inbounds %union.v1, %union.v1* %3, i64 0, i32 0, !dbg !32
+  %call = tail call i32 @get_value(i32* %b) #4, !dbg !33
+  ret i32 %call, !dbg !34
+}
+
+; CHECK:             r2 = 16
+; CHECK:             r1 += r2
+; CHECK:             r2 = 0
+; CHECK:             r1 += r2
+; CHECK:             call get_value
+
+; CHECK:             .long   6                       # BTF_KIND_UNION(id = [[TID1:[0-9]+]])
+; CHECK:             .long   111                     # BTF_KIND_UNION(id = [[TID2:[0-9]+]])
+
+; CHECK:             .ascii  "v3"                    # string offset=6
+; CHECK:             .ascii  ".text"                 # string offset=57
+; CHECK:             .ascii  "0:1:4"                 # string offset=63
+; CHECK:             .ascii  "v1"                    # string offset=111
+; CHECK:             .ascii  "0:1"                   # string offset=118
+
+; CHECK:             .long   16                      # FieldReloc
+; CHECK-NEXT:        .long   57                      # Field reloc section string offset=57
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   [[TID1]]
+; CHECK-NEXT:        .long   63
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   [[TID2]]
+; CHECK-NEXT:        .long   118
+; CHECK-NEXT:        .long   0
+
+declare dso_local i32 @get_value(i32*) local_unnamed_addr #1
+
+; Function Attrs: nounwind readnone
+declare %union.v3* @llvm.preserve.union.access.index.p0s_union.v3s.p0s_union.v3s(%union.v3*, i32) #2
+
+; Function Attrs: nounwind readnone
+declare i32* @llvm.preserve.array.access.index.p0i32.p0a40i32([40 x i32]*, i32, i32) #2
+
+; Function Attrs: nounwind readnone
+declare %union.v1* @llvm.preserve.union.access.index.p0s_union.v1s.p0s_union.v1s(%union.v1*, i32) #2
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.value(metadata, metadata, metadata) #3
+
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { nounwind readnone }
+attributes #3 = { nounwind readnone speculatable willreturn }
+attributes #4 = { nounwind }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!15, !16, !17}
+!llvm.ident = !{!18}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 10.0.0 (trunk 367256) (llvm/trunk 367266)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !3, nameTableKind: None)
+!1 = !DIFile(filename: "test.c", directory: "/tmp/home/yhs/work/tests/llvm/cast")
+!2 = !{}
+!3 = !{!4, !11}
+!4 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !5, size: 64)
+!5 = !DIDerivedType(tag: DW_TAG_typedef, name: "__v1", file: !1, line: 2, baseType: !6)
+!6 = distinct !DICompositeType(tag: DW_TAG_union_type, name: "v1", file: !1, line: 1, size: 32, elements: !7)
+!7 = !{!8, !10}
+!8 = !DIDerivedType(tag: DW_TAG_member, name: "a", scope: !6, file: !1, line: 1, baseType: !9, size: 32)
+!9 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!10 = !DIDerivedType(tag: DW_TAG_member, name: "b", scope: !6, file: !1, line: 1, baseType: !9, size: 32)
+!11 = !DICompositeType(tag: DW_TAG_array_type, baseType: !12, size: 1280, elements: !13)
+!12 = !DIDerivedType(tag: DW_TAG_typedef, name: "__int", file: !1, line: 3, baseType: !9)
+!13 = !{!14}
+!14 = !DISubrange(count: 40)
+!15 = !{i32 2, !"Dwarf Version", i32 4}
+!16 = !{i32 2, !"Debug Info Version", i32 3}
+!17 = !{i32 1, !"wchar_size", i32 4}
+!18 = !{!"clang version 10.0.0 (trunk 367256) (llvm/trunk 367266)"}
+!19 = distinct !DISubprogram(name: "test", scope: !1, file: !1, line: 9, type: !20, scopeLine: 9, flags: DIFlagPrototyped, isDefinition: true, isOptimized: true, unit: !0, retainedNodes: !29)
+!20 = !DISubroutineType(types: !21)
+!21 = !{!9, !22}
+!22 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !23, size: 64)
+!23 = !DIDerivedType(tag: DW_TAG_typedef, name: "__v3", file: !1, line: 5, baseType: !24)
+!24 = distinct !DICompositeType(tag: DW_TAG_union_type, name: "v3", file: !1, line: 4, size: 1280, elements: !25)
+!25 = !{!26, !28}
+!26 = !DIDerivedType(tag: DW_TAG_member, name: "c", scope: !24, file: !1, line: 4, baseType: !27, size: 8)
+!27 = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_signed_char)
+!28 = !DIDerivedType(tag: DW_TAG_member, name: "d", scope: !24, file: !1, line: 4, baseType: !11, size: 1280)
+!29 = !{!30}
+!30 = !DILocalVariable(name: "arg", arg: 1, scope: !19, file: !1, line: 9, type: !22)
+!31 = !DILocation(line: 0, scope: !19)
+!32 = !DILocation(line: 10, column: 20, scope: !19)
+!33 = !DILocation(line: 10, column: 10, scope: !19)
+!34 = !DILocation(line: 10, column: 3, scope: !19)
diff --git a/llvm/test/CodeGen/BPF/CORE/offset-reloc-end-load.ll b/llvm/test/CodeGen/BPF/CORE/offset-reloc-end-load.ll
new file mode 100644
index 000000000..a0dd6c8da
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/CORE/offset-reloc-end-load.ll
@@ -0,0 +1,83 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-ALU64 %s
+; RUN: llc -march=bpfel -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-ALU32 %s
+;
+; Source Code:
+;   #define _(x) (__builtin_preserve_access_index(x))
+;   struct s {int a; int b;};
+;   int test(struct s *arg) { return *(const int *)_(&arg->b); }
+; Compiler flag to generate IR:
+;   clang -target bpf -S -O2 -g -emit-llvm test.c
+
+%struct.s = type { i32, i32 }
+
+; Function Attrs: nounwind readonly
+define dso_local i32 @test(%struct.s* readonly %arg) local_unnamed_addr #0 !dbg !11 {
+entry:
+  call void @llvm.dbg.value(metadata %struct.s* %arg, metadata !20, metadata !DIExpression()), !dbg !21
+  %0 = tail call i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.ss(%struct.s* %arg, i32 1, i32 1), !dbg !22, !llvm.preserve.access.index !15
+  %1 = load i32, i32* %0, align 4, !dbg !23, !tbaa !24
+  ret i32 %1, !dbg !28
+}
+
+; CHECK-LABEL: test
+; CHECK-ALU64: r0 = *(u32 *)(r1 + 4)
+; CHECK-ALU32: w0 = *(u32 *)(r1 + 4)
+; CHECK:       exit
+;
+; CHECK:       .long   1                       # BTF_KIND_STRUCT(id = 2)
+;
+; CHECK:       .byte   115                     # string offset=1
+; CHECK:       .ascii  ".text"                 # string offset=20
+; CHECK:       .ascii  "0:1"                   # string offset=26
+;
+; CHECK:       .long   16                      # FieldReloc
+; CHECK-NEXT:  .long   20                      # Field reloc section string offset=20
+; CHECK-NEXT:  .long   1
+; CHECK-NEXT:  .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:  .long   2
+; CHECK-NEXT:  .long   26
+; CHECK-NEXT:  .long   0
+
+; Function Attrs: nounwind readnone
+declare i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.ss(%struct.s*, i32, i32) #1
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.value(metadata, metadata, metadata) #2
+
+attributes #0 = { nounwind readonly "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone }
+attributes #2 = { nounwind readnone speculatable willreturn }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!7, !8, !9}
+!llvm.ident = !{!10}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 10.0.0 (https://github.com/llvm/llvm-project.git 6e353b4df3aa452ed4741a5e5caea02b1a876d8c)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !3, nameTableKind: None)
+!1 = !DIFile(filename: "test.c", directory: "/tmp/home/yhs/work/tests/core")
+!2 = !{}
+!3 = !{!4}
+!4 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !5, size: 64)
+!5 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !6)
+!6 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!7 = !{i32 2, !"Dwarf Version", i32 4}
+!8 = !{i32 2, !"Debug Info Version", i32 3}
+!9 = !{i32 1, !"wchar_size", i32 4}
+!10 = !{!"clang version 10.0.0 (https://github.com/llvm/llvm-project.git 6e353b4df3aa452ed4741a5e5caea02b1a876d8c)"}
+!11 = distinct !DISubprogram(name: "test", scope: !1, file: !1, line: 3, type: !12, scopeLine: 3, flags: DIFlagPrototyped, isDefinition: true, isOptimized: true, unit: !0, retainedNodes: !19)
+!12 = !DISubroutineType(types: !13)
+!13 = !{!6, !14}
+!14 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !15, size: 64)
+!15 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "s", file: !1, line: 2, size: 64, elements: !16)
+!16 = !{!17, !18}
+!17 = !DIDerivedType(tag: DW_TAG_member, name: "a", scope: !15, file: !1, line: 2, baseType: !6, size: 32)
+!18 = !DIDerivedType(tag: DW_TAG_member, name: "b", scope: !15, file: !1, line: 2, baseType: !6, size: 32, offset: 32)
+!19 = !{!20}
+!20 = !DILocalVariable(name: "arg", arg: 1, scope: !11, file: !1, line: 3, type: !14)
+!21 = !DILocation(line: 0, scope: !11)
+!22 = !DILocation(line: 3, column: 48, scope: !11)
+!23 = !DILocation(line: 3, column: 34, scope: !11)
+!24 = !{!25, !25, i64 0}
+!25 = !{!"int", !26, i64 0}
+!26 = !{!"omnipotent char", !27, i64 0}
+!27 = !{!"Simple C/C++ TBAA"}
+!28 = !DILocation(line: 3, column: 27, scope: !11)
diff --git a/llvm/test/CodeGen/BPF/CORE/offset-reloc-end-ret.ll b/llvm/test/CodeGen/BPF/CORE/offset-reloc-end-ret.ll
new file mode 100644
index 000000000..32276496f
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/CORE/offset-reloc-end-ret.ll
@@ -0,0 +1,78 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck %s
+; RUN: llc -march=bpfel -mattr=+alu32 -filetype=asm -o - %s | FileCheck %s
+;
+; Source Code:
+;   #define _(x) (__builtin_preserve_access_index(x))
+;   struct s {int a; int b;};
+;   const void *test(struct s *arg) { return _(&arg->b); }
+; Compiler flag to generate IR:
+;   clang -target bpf -S -O2 -g -emit-llvm test.c
+
+%struct.s = type { i32, i32 }
+
+; Function Attrs: nounwind readnone
+define dso_local i8* @test(%struct.s* readnone %arg) local_unnamed_addr #0 !dbg !7 {
+entry:
+  call void @llvm.dbg.value(metadata %struct.s* %arg, metadata !19, metadata !DIExpression()), !dbg !20
+  %0 = tail call i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.ss(%struct.s* %arg, i32 1, i32 1), !dbg !21, !llvm.preserve.access.index !13
+  %1 = bitcast i32* %0 to i8*, !dbg !21
+  ret i8* %1, !dbg !22
+}
+
+; CHECK-LABEL: test
+; CHECK:       r0 = r1
+; CHECK:       r1 = 4
+; CHECK:       r0 += r1
+; CHECK:       exit
+;
+; CHECK:       .long   1                       # BTF_KIND_STRUCT(id = 2)
+;
+; CHECK:       .byte   115                     # string offset=1
+; CHECK:       .ascii  ".text"                 # string offset=20
+; CHECK:       .ascii  "0:1"                   # string offset=63
+;
+; CHECK:       .long   16                      # FieldReloc
+; CHECK-NEXT:  .long   20                      # Field reloc section string offset=20
+; CHECK-NEXT:  .long   1
+; CHECK-NEXT:  .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:  .long   2
+; CHECK-NEXT:  .long   63
+; CHECK-NEXT:  .long   0
+
+; Function Attrs: nounwind readnone
+declare i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.ss(%struct.s*, i32, i32) #1
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.value(metadata, metadata, metadata) #2
+
+attributes #0 = { nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone }
+attributes #2 = { nounwind readnone speculatable willreturn }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 10.0.0 (https://github.com/llvm/llvm-project.git 6e353b4df3aa452ed4741a5e5caea02b1a876d8c)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "test.c", directory: "/tmp/home/yhs/work/tests/core")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{!"clang version 10.0.0 (https://github.com/llvm/llvm-project.git 6e353b4df3aa452ed4741a5e5caea02b1a876d8c)"}
+!7 = distinct !DISubprogram(name: "test", scope: !1, file: !1, line: 3, type: !8, scopeLine: 3, flags: DIFlagPrototyped, isDefinition: true, isOptimized: true, unit: !0, retainedNodes: !18)
+!8 = !DISubroutineType(types: !9)
+!9 = !{!10, !12}
+!10 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !11, size: 64)
+!11 = !DIDerivedType(tag: DW_TAG_const_type, baseType: null)
+!12 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !13, size: 64)
+!13 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "s", file: !1, line: 2, size: 64, elements: !14)
+!14 = !{!15, !17}
+!15 = !DIDerivedType(tag: DW_TAG_member, name: "a", scope: !13, file: !1, line: 2, baseType: !16, size: 32)
+!16 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!17 = !DIDerivedType(tag: DW_TAG_member, name: "b", scope: !13, file: !1, line: 2, baseType: !16, size: 32, offset: 32)
+!18 = !{!19}
+!19 = !DILocalVariable(name: "arg", arg: 1, scope: !7, file: !1, line: 3, type: !12)
+!20 = !DILocation(line: 0, scope: !7)
+!21 = !DILocation(line: 3, column: 42, scope: !7)
+!22 = !DILocation(line: 3, column: 35, scope: !7)
diff --git a/llvm/test/CodeGen/BPF/CORE/offset-reloc-fieldinfo-1.ll b/llvm/test/CodeGen/BPF/CORE/offset-reloc-fieldinfo-1.ll
new file mode 100644
index 000000000..d7e48d390
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/CORE/offset-reloc-fieldinfo-1.ll
@@ -0,0 +1,195 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfel -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; Source code:
+;   struct s {
+;     int a;
+;     int b1:9;
+;     int b2:4;
+;   };
+;   enum {
+;       FIELD_BYTE_OFFSET = 0,
+;       FIELD_BYTE_SIZE,
+;       FIELD_EXISTENCE,
+;       FIELD_SIGNEDNESS,
+;       FIELD_LSHIFT_U64,
+;       FIELD_RSHIFT_U64,
+;   };
+;   void bpf_probe_read(void *, unsigned, const void *);
+;   int field_read(struct s *arg) {
+;     unsigned long long ull;
+;     unsigned offset = __builtin_preserve_field_info(arg->b2, FIELD_BYTE_OFFSET);
+;     unsigned size = __builtin_preserve_field_info(arg->b2, FIELD_BYTE_SIZE);
+;     unsigned lshift;
+;
+;     bpf_probe_read(&ull, size, (const void *)arg + offset);
+;     lshift = __builtin_preserve_field_info(arg->b2, FIELD_LSHIFT_U64);
+;   #if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+;     lshift = lshift + (size << 3) - 64;
+;   #endif
+;     ull <<= lshift;
+;     if (__builtin_preserve_field_info(arg->b2, FIELD_SIGNEDNESS))
+;       return (long long)ull >> __builtin_preserve_field_info(arg->b2, FIELD_RSHIFT_U64);
+;     return ull >> __builtin_preserve_field_info(arg->b2, FIELD_RSHIFT_U64);
+;   }
+; Compilation flag:
+;   clang -target bpfel -O2 -g -S -emit-llvm test.c
+
+%struct.s = type { i32, i16 }
+
+; Function Attrs: nounwind
+define dso_local i32 @field_read(%struct.s* %arg) local_unnamed_addr #0 !dbg !20 {
+entry:
+  %ull = alloca i64, align 8
+  call void @llvm.dbg.value(metadata %struct.s* %arg, metadata !31, metadata !DIExpression()), !dbg !37
+  %0 = bitcast i64* %ull to i8*, !dbg !38
+  call void @llvm.lifetime.start.p0i8(i64 8, i8* nonnull %0) #5, !dbg !38
+  %1 = tail call i16* @llvm.preserve.struct.access.index.p0i16.p0s_struct.ss(%struct.s* %arg, i32 1, i32 2), !dbg !39, !llvm.preserve.access.index !25
+  %2 = tail call i32 @llvm.bpf.preserve.field.info.p0i16(i16* %1, i64 0), !dbg !40
+  call void @llvm.dbg.value(metadata i32 %2, metadata !34, metadata !DIExpression()), !dbg !37
+  %3 = tail call i32 @llvm.bpf.preserve.field.info.p0i16(i16* %1, i64 1), !dbg !41
+  call void @llvm.dbg.value(metadata i32 %3, metadata !35, metadata !DIExpression()), !dbg !37
+  %4 = bitcast %struct.s* %arg to i8*, !dbg !42
+  %idx.ext = zext i32 %2 to i64, !dbg !43
+  %add.ptr = getelementptr i8, i8* %4, i64 %idx.ext, !dbg !43
+  call void @bpf_probe_read(i8* nonnull %0, i32 %3, i8* %add.ptr) #5, !dbg !44
+  %5 = call i32 @llvm.bpf.preserve.field.info.p0i16(i16* %1, i64 4), !dbg !45
+  call void @llvm.dbg.value(metadata i32 %5, metadata !36, metadata !DIExpression()), !dbg !37
+  %6 = load i64, i64* %ull, align 8, !dbg !46, !tbaa !47
+  call void @llvm.dbg.value(metadata i64 %6, metadata !32, metadata !DIExpression()), !dbg !37
+  %sh_prom = zext i32 %5 to i64, !dbg !46
+  %shl = shl i64 %6, %sh_prom, !dbg !46
+  call void @llvm.dbg.value(metadata i64 %shl, metadata !32, metadata !DIExpression()), !dbg !37
+  %7 = call i32 @llvm.bpf.preserve.field.info.p0i16(i16* %1, i64 3), !dbg !51
+  %tobool = icmp eq i32 %7, 0, !dbg !51
+  %8 = call i32 @llvm.bpf.preserve.field.info.p0i16(i16* %1, i64 5), !dbg !37
+  %sh_prom1 = zext i32 %8 to i64, !dbg !37
+  %shr = ashr i64 %shl, %sh_prom1, !dbg !53
+  %shr3 = lshr i64 %shl, %sh_prom1, !dbg !53
+  %retval.0.in = select i1 %tobool, i64 %shr3, i64 %shr, !dbg !53
+  %retval.0 = trunc i64 %retval.0.in to i32, !dbg !37
+  call void @llvm.lifetime.end.p0i8(i64 8, i8* nonnull %0) #5, !dbg !54
+  ret i32 %retval.0, !dbg !54
+}
+
+; CHECK:             r{{[0-9]+}} = 4
+; CHECK:             r{{[0-9]+}} = 4
+; CHECK:             r{{[0-9]+}} <<= 51
+; CHECK:             r{{[0-9]+}} s>>= 60
+; CHECK:             r{{[0-9]+}} >>= 60
+; CHECK:             r{{[0-9]+}} = 1
+
+; CHECK:             .byte   115                     # string offset=1
+; CHECK:             .ascii  ".text"                 # string offset=30
+; CHECK:             .ascii  "0:2"                   # string offset=73
+
+; CHECK:             .long   16                      # FieldReloc
+; CHECK-NEXT:        .long   30                      # Field reloc section string offset=30
+; CHECK-NEXT:        .long   6
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   73
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   73
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   73
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   73
+; CHECK-NEXT:        .long   5
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   73
+; CHECK-NEXT:        .long   5
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   73
+; CHECK-NEXT:        .long   3
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture) #1
+
+; Function Attrs: nounwind readnone
+declare i16* @llvm.preserve.struct.access.index.p0i16.p0s_struct.ss(%struct.s*, i32, i32) #2
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.bpf.preserve.field.info.p0i16(i16*, i64) #2
+
+declare dso_local void @bpf_probe_read(i8*, i32, i8*) local_unnamed_addr #3
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture) #1
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.value(metadata, metadata, metadata) #4
+
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { argmemonly nounwind willreturn }
+attributes #2 = { nounwind readnone }
+attributes #3 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #4 = { nounwind readnone speculatable willreturn }
+attributes #5 = { nounwind }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!16, !17, !18}
+!llvm.ident = !{!19}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 10.0.0 (https://github.com/llvm/llvm-project.git 923aa0ce806f7739b754167239fee2c9a15e2f31)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !12, nameTableKind: None)
+!1 = !DIFile(filename: "test.c", directory: "/tmp/home/yhs/work/tests/core")
+!2 = !{!3}
+!3 = !DICompositeType(tag: DW_TAG_enumeration_type, file: !1, line: 6, baseType: !4, size: 32, elements: !5)
+!4 = !DIBasicType(name: "unsigned int", size: 32, encoding: DW_ATE_unsigned)
+!5 = !{!6, !7, !8, !9, !10, !11}
+!6 = !DIEnumerator(name: "FIELD_BYTE_OFFSET", value: 0, isUnsigned: true)
+!7 = !DIEnumerator(name: "FIELD_BYTE_SIZE", value: 1, isUnsigned: true)
+!8 = !DIEnumerator(name: "FIELD_EXISTENCE", value: 2, isUnsigned: true)
+!9 = !DIEnumerator(name: "FIELD_SIGNEDNESS", value: 3, isUnsigned: true)
+!10 = !DIEnumerator(name: "FIELD_LSHIFT_U64", value: 4, isUnsigned: true)
+!11 = !DIEnumerator(name: "FIELD_RSHIFT_U64", value: 5, isUnsigned: true)
+!12 = !{!13, !15}
+!13 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !14, size: 64)
+!14 = !DIDerivedType(tag: DW_TAG_const_type, baseType: null)
+!15 = !DIBasicType(name: "long long int", size: 64, encoding: DW_ATE_signed)
+!16 = !{i32 2, !"Dwarf Version", i32 4}
+!17 = !{i32 2, !"Debug Info Version", i32 3}
+!18 = !{i32 1, !"wchar_size", i32 4}
+!19 = !{!"clang version 10.0.0 (https://github.com/llvm/llvm-project.git 923aa0ce806f7739b754167239fee2c9a15e2f31)"}
+!20 = distinct !DISubprogram(name: "field_read", scope: !1, file: !1, line: 15, type: !21, scopeLine: 15, flags: DIFlagPrototyped, isDefinition: true, isOptimized: true, unit: !0, retainedNodes: !30)
+!21 = !DISubroutineType(types: !22)
+!22 = !{!23, !24}
+!23 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!24 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !25, size: 64)
+!25 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "s", file: !1, line: 1, size: 64, elements: !26)
+!26 = !{!27, !28, !29}
+!27 = !DIDerivedType(tag: DW_TAG_member, name: "a", scope: !25, file: !1, line: 2, baseType: !23, size: 32)
+!28 = !DIDerivedType(tag: DW_TAG_member, name: "b1", scope: !25, file: !1, line: 3, baseType: !23, size: 9, offset: 32, flags: DIFlagBitField, extraData: i64 32)
+!29 = !DIDerivedType(tag: DW_TAG_member, name: "b2", scope: !25, file: !1, line: 4, baseType: !23, size: 4, offset: 41, flags: DIFlagBitField, extraData: i64 32)
+!30 = !{!31, !32, !34, !35, !36}
+!31 = !DILocalVariable(name: "arg", arg: 1, scope: !20, file: !1, line: 15, type: !24)
+!32 = !DILocalVariable(name: "ull", scope: !20, file: !1, line: 16, type: !33)
+!33 = !DIBasicType(name: "long long unsigned int", size: 64, encoding: DW_ATE_unsigned)
+!34 = !DILocalVariable(name: "offset", scope: !20, file: !1, line: 17, type: !4)
+!35 = !DILocalVariable(name: "size", scope: !20, file: !1, line: 18, type: !4)
+!36 = !DILocalVariable(name: "lshift", scope: !20, file: !1, line: 19, type: !4)
+!37 = !DILocation(line: 0, scope: !20)
+!38 = !DILocation(line: 16, column: 3, scope: !20)
+!39 = !DILocation(line: 17, column: 56, scope: !20)
+!40 = !DILocation(line: 17, column: 21, scope: !20)
+!41 = !DILocation(line: 18, column: 19, scope: !20)
+!42 = !DILocation(line: 21, column: 30, scope: !20)
+!43 = !DILocation(line: 21, column: 48, scope: !20)
+!44 = !DILocation(line: 21, column: 3, scope: !20)
+!45 = !DILocation(line: 22, column: 12, scope: !20)
+!46 = !DILocation(line: 26, column: 7, scope: !20)
+!47 = !{!48, !48, i64 0}
+!48 = !{!"long long", !49, i64 0}
+!49 = !{!"omnipotent char", !50, i64 0}
+!50 = !{!"Simple C/C++ TBAA"}
+!51 = !DILocation(line: 27, column: 7, scope: !52)
+!52 = distinct !DILexicalBlock(scope: !20, file: !1, line: 27, column: 7)
+!53 = !DILocation(line: 27, column: 7, scope: !20)
+!54 = !DILocation(line: 30, column: 1, scope: !20)
diff --git a/llvm/test/CodeGen/BPF/CORE/offset-reloc-fieldinfo-2.ll b/llvm/test/CodeGen/BPF/CORE/offset-reloc-fieldinfo-2.ll
new file mode 100644
index 000000000..01af9d8a6
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/CORE/offset-reloc-fieldinfo-2.ll
@@ -0,0 +1,258 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-EL,CHECK64 %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-EB,CHECK64 %s
+; RUN: llc -march=bpfel -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-EL,CHECK32 %s
+; RUN: llc -march=bpfeb -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-EB,CHECK32 %s
+; Source code:
+;   struct s {
+;     int a;
+;     int b1:9;
+;     int b2:4;
+;   };
+;   enum {
+;     FIELD_BYTE_OFFSET = 0,
+;     FIELD_BYTE_SIZE,
+;     FIELD_EXISTENCE,
+;     FIELD_SIGNEDNESS,
+;     FIELD_LSHIFT_U64,
+;     FIELD_RSHIFT_U64,
+;   };
+;   int field_read(struct s *arg) {
+;     unsigned long long ull;
+;     unsigned offset = __builtin_preserve_field_info(arg->b2, FIELD_BYTE_OFFSET);
+;     unsigned size = __builtin_preserve_field_info(arg->b2, FIELD_BYTE_SIZE);
+;     switch(size) {
+;     case 1:
+;       ull = *(unsigned char *)((void *)arg + offset); break;
+;     case 2:
+;       ull = *(unsigned short *)((void *)arg + offset); break;
+;     case 4:
+;       ull = *(unsigned int *)((void *)arg + offset); break;
+;     case 8:
+;       ull = *(unsigned long long *)((void *)arg + offset); break;
+;     }
+;     ull <<= __builtin_preserve_field_info(arg->b2, FIELD_LSHIFT_U64);
+;     if (__builtin_preserve_field_info(arg->b2, FIELD_SIGNEDNESS))
+;       return ((long long)ull) >>__builtin_preserve_field_info(arg->b2, FIELD_RSHIFT_U64);
+;     return ull >> __builtin_preserve_field_info(arg->b2, FIELD_RSHIFT_U64);
+;   }
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm test.c
+
+%struct.s = type { i32, i16 }
+
+; Function Attrs: nounwind readonly
+define dso_local i32 @field_read(%struct.s* %arg) local_unnamed_addr #0 !dbg !26 {
+entry:
+  call void @llvm.dbg.value(metadata %struct.s* %arg, metadata !37, metadata !DIExpression()), !dbg !41
+  %0 = tail call i16* @llvm.preserve.struct.access.index.p0i16.p0s_struct.ss(%struct.s* %arg, i32 1, i32 2), !dbg !42, !llvm.preserve.access.index !31
+  %1 = tail call i32 @llvm.bpf.preserve.field.info.p0i16(i16* %0, i64 0), !dbg !43
+  call void @llvm.dbg.value(metadata i32 %1, metadata !39, metadata !DIExpression()), !dbg !41
+  %2 = tail call i32 @llvm.bpf.preserve.field.info.p0i16(i16* %0, i64 1), !dbg !44
+  call void @llvm.dbg.value(metadata i32 %2, metadata !40, metadata !DIExpression()), !dbg !41
+  switch i32 %2, label %sw.epilog [
+    i32 1, label %sw.bb
+    i32 2, label %sw.bb1
+    i32 4, label %sw.bb5
+    i32 8, label %sw.bb9
+  ], !dbg !45
+
+sw.bb:                                            ; preds = %entry
+  %3 = bitcast %struct.s* %arg to i8*, !dbg !46
+  %idx.ext = zext i32 %1 to i64, !dbg !48
+  %add.ptr = getelementptr i8, i8* %3, i64 %idx.ext, !dbg !48
+  %4 = load i8, i8* %add.ptr, align 1, !dbg !49, !tbaa !50
+  %conv = zext i8 %4 to i64, !dbg !49
+  call void @llvm.dbg.value(metadata i64 %conv, metadata !38, metadata !DIExpression()), !dbg !41
+  br label %sw.epilog, !dbg !53
+
+sw.bb1:                                           ; preds = %entry
+  %5 = bitcast %struct.s* %arg to i8*, !dbg !54
+  %idx.ext2 = zext i32 %1 to i64, !dbg !55
+  %add.ptr3 = getelementptr i8, i8* %5, i64 %idx.ext2, !dbg !55
+  %6 = bitcast i8* %add.ptr3 to i16*, !dbg !56
+  %7 = load i16, i16* %6, align 2, !dbg !57, !tbaa !58
+  %conv4 = zext i16 %7 to i64, !dbg !57
+  call void @llvm.dbg.value(metadata i64 %conv4, metadata !38, metadata !DIExpression()), !dbg !41
+  br label %sw.epilog, !dbg !60
+
+sw.bb5:                                           ; preds = %entry
+  %8 = bitcast %struct.s* %arg to i8*, !dbg !61
+  %idx.ext6 = zext i32 %1 to i64, !dbg !62
+  %add.ptr7 = getelementptr i8, i8* %8, i64 %idx.ext6, !dbg !62
+  %9 = bitcast i8* %add.ptr7 to i32*, !dbg !63
+  %10 = load i32, i32* %9, align 4, !dbg !64, !tbaa !65
+  %conv8 = zext i32 %10 to i64, !dbg !64
+  call void @llvm.dbg.value(metadata i64 %conv8, metadata !38, metadata !DIExpression()), !dbg !41
+  br label %sw.epilog, !dbg !67
+
+sw.bb9:                                           ; preds = %entry
+  %11 = bitcast %struct.s* %arg to i8*, !dbg !68
+  %idx.ext10 = zext i32 %1 to i64, !dbg !69
+  %add.ptr11 = getelementptr i8, i8* %11, i64 %idx.ext10, !dbg !69
+  %12 = bitcast i8* %add.ptr11 to i64*, !dbg !70
+  %13 = load i64, i64* %12, align 8, !dbg !71, !tbaa !72
+  call void @llvm.dbg.value(metadata i64 %13, metadata !38, metadata !DIExpression()), !dbg !41
+  br label %sw.epilog, !dbg !74
+
+sw.epilog:                                        ; preds = %entry, %sw.bb9, %sw.bb5, %sw.bb1, %sw.bb
+  %ull.0 = phi i64 [ undef, %entry ], [ %13, %sw.bb9 ], [ %conv8, %sw.bb5 ], [ %conv4, %sw.bb1 ], [ %conv, %sw.bb ]
+  call void @llvm.dbg.value(metadata i64 %ull.0, metadata !38, metadata !DIExpression()), !dbg !41
+  %14 = tail call i32 @llvm.bpf.preserve.field.info.p0i16(i16* %0, i64 4), !dbg !75
+  %sh_prom = zext i32 %14 to i64, !dbg !76
+  %shl = shl i64 %ull.0, %sh_prom, !dbg !76
+  call void @llvm.dbg.value(metadata i64 %shl, metadata !38, metadata !DIExpression()), !dbg !41
+  %15 = tail call i32 @llvm.bpf.preserve.field.info.p0i16(i16* %0, i64 3), !dbg !77
+  %tobool = icmp eq i32 %15, 0, !dbg !77
+  %16 = tail call i32 @llvm.bpf.preserve.field.info.p0i16(i16* %0, i64 5), !dbg !41
+  %sh_prom12 = zext i32 %16 to i64, !dbg !41
+  %shr = ashr i64 %shl, %sh_prom12, !dbg !79
+  %shr15 = lshr i64 %shl, %sh_prom12, !dbg !79
+  %retval.0.in = select i1 %tobool, i64 %shr15, i64 %shr, !dbg !79
+  %retval.0 = trunc i64 %retval.0.in to i32, !dbg !41
+  ret i32 %retval.0, !dbg !80
+}
+
+; CHECK:             r{{[0-9]+}} = 4
+; CHECK:             r{{[0-9]+}} = 4
+; CHECK-EL:          r{{[0-9]+}} <<= 51
+; CHECK-EB:          r{{[0-9]+}} <<= 41
+; CHECK:             r{{[0-9]+}} s>>= 60
+; CHECK:             r{{[0-9]+}} >>= 60
+; CHECK:             r{{[0-9]+}} = 1
+
+; CHECK:             .long   1                       # BTF_KIND_STRUCT(id = 2)
+; CHECK:             .byte   115                     # string offset=1
+; CHECK:             .ascii  ".text"                 # string offset=30
+; CHECK:             .ascii  "0:2"                   # string offset=36
+
+; CHECK:             .long   16                      # FieldReloc
+; CHECK-NEXT:        .long   30                      # Field reloc section string offset=30
+; CHECK32:           .long   6
+; CHECK64:           .long   7
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   36
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   36
+; CHECK-NEXT:        .long   1
+; CHECK64:           .long   .Ltmp{{[0-9]+}}
+; CHECK64:           .long   2
+; CHECK64:           .long   36
+; CHECK64:           .long   0
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   36
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   36
+; CHECK-NEXT:        .long   5
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   36
+; CHECK-NEXT:        .long   5
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   36
+; CHECK-NEXT:        .long   3
+
+; Function Attrs: nounwind readnone
+declare i16* @llvm.preserve.struct.access.index.p0i16.p0s_struct.ss(%struct.s*, i32, i32) #1
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.bpf.preserve.field.info.p0i16(i16*, i64) #1
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.value(metadata, metadata, metadata) #2
+
+attributes #0 = { nounwind readonly "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone }
+attributes #2 = { nounwind readnone speculatable willreturn }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!22, !23, !24}
+!llvm.ident = !{!25}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 10.0.0 (https://github.com/llvm/llvm-project.git 923aa0ce806f7739b754167239fee2c9a15e2f31)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !12, nameTableKind: None)
+!1 = !DIFile(filename: "test.c", directory: "/tmp/home/yhs/work/tests/core")
+!2 = !{!3}
+!3 = !DICompositeType(tag: DW_TAG_enumeration_type, file: !1, line: 6, baseType: !4, size: 32, elements: !5)
+!4 = !DIBasicType(name: "unsigned int", size: 32, encoding: DW_ATE_unsigned)
+!5 = !{!6, !7, !8, !9, !10, !11}
+!6 = !DIEnumerator(name: "FIELD_BYTE_OFFSET", value: 0, isUnsigned: true)
+!7 = !DIEnumerator(name: "FIELD_BYTE_SIZE", value: 1, isUnsigned: true)
+!8 = !DIEnumerator(name: "FIELD_EXISTENCE", value: 2, isUnsigned: true)
+!9 = !DIEnumerator(name: "FIELD_SIGNEDNESS", value: 3, isUnsigned: true)
+!10 = !DIEnumerator(name: "FIELD_LSHIFT_U64", value: 4, isUnsigned: true)
+!11 = !DIEnumerator(name: "FIELD_RSHIFT_U64", value: 5, isUnsigned: true)
+!12 = !{!13, !15, !16, !18, !19, !21}
+!13 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !14, size: 64)
+!14 = !DIBasicType(name: "unsigned char", size: 8, encoding: DW_ATE_unsigned_char)
+!15 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: null, size: 64)
+!16 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !17, size: 64)
+!17 = !DIBasicType(name: "unsigned short", size: 16, encoding: DW_ATE_unsigned)
+!18 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !4, size: 64)
+!19 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !20, size: 64)
+!20 = !DIBasicType(name: "long long unsigned int", size: 64, encoding: DW_ATE_unsigned)
+!21 = !DIBasicType(name: "long long int", size: 64, encoding: DW_ATE_signed)
+!22 = !{i32 2, !"Dwarf Version", i32 4}
+!23 = !{i32 2, !"Debug Info Version", i32 3}
+!24 = !{i32 1, !"wchar_size", i32 4}
+!25 = !{!"clang version 10.0.0 (https://github.com/llvm/llvm-project.git 923aa0ce806f7739b754167239fee2c9a15e2f31)"}
+!26 = distinct !DISubprogram(name: "field_read", scope: !1, file: !1, line: 14, type: !27, scopeLine: 14, flags: DIFlagPrototyped, isDefinition: true, isOptimized: true, unit: !0, retainedNodes: !36)
+!27 = !DISubroutineType(types: !28)
+!28 = !{!29, !30}
+!29 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!30 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !31, size: 64)
+!31 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "s", file: !1, line: 1, size: 64, elements: !32)
+!32 = !{!33, !34, !35}
+!33 = !DIDerivedType(tag: DW_TAG_member, name: "a", scope: !31, file: !1, line: 2, baseType: !29, size: 32)
+!34 = !DIDerivedType(tag: DW_TAG_member, name: "b1", scope: !31, file: !1, line: 3, baseType: !29, size: 9, offset: 32, flags: DIFlagBitField, extraData: i64 32)
+!35 = !DIDerivedType(tag: DW_TAG_member, name: "b2", scope: !31, file: !1, line: 4, baseType: !29, size: 4, offset: 41, flags: DIFlagBitField, extraData: i64 32)
+!36 = !{!37, !38, !39, !40}
+!37 = !DILocalVariable(name: "arg", arg: 1, scope: !26, file: !1, line: 14, type: !30)
+!38 = !DILocalVariable(name: "ull", scope: !26, file: !1, line: 15, type: !20)
+!39 = !DILocalVariable(name: "offset", scope: !26, file: !1, line: 16, type: !4)
+!40 = !DILocalVariable(name: "size", scope: !26, file: !1, line: 17, type: !4)
+!41 = !DILocation(line: 0, scope: !26)
+!42 = !DILocation(line: 16, column: 56, scope: !26)
+!43 = !DILocation(line: 16, column: 21, scope: !26)
+!44 = !DILocation(line: 17, column: 19, scope: !26)
+!45 = !DILocation(line: 18, column: 3, scope: !26)
+!46 = !DILocation(line: 20, column: 30, scope: !47)
+!47 = distinct !DILexicalBlock(scope: !26, file: !1, line: 18, column: 16)
+!48 = !DILocation(line: 20, column: 42, scope: !47)
+!49 = !DILocation(line: 20, column: 11, scope: !47)
+!50 = !{!51, !51, i64 0}
+!51 = !{!"omnipotent char", !52, i64 0}
+!52 = !{!"Simple C/C++ TBAA"}
+!53 = !DILocation(line: 20, column: 53, scope: !47)
+!54 = !DILocation(line: 22, column: 31, scope: !47)
+!55 = !DILocation(line: 22, column: 43, scope: !47)
+!56 = !DILocation(line: 22, column: 12, scope: !47)
+!57 = !DILocation(line: 22, column: 11, scope: !47)
+!58 = !{!59, !59, i64 0}
+!59 = !{!"short", !51, i64 0}
+!60 = !DILocation(line: 22, column: 54, scope: !47)
+!61 = !DILocation(line: 24, column: 29, scope: !47)
+!62 = !DILocation(line: 24, column: 41, scope: !47)
+!63 = !DILocation(line: 24, column: 12, scope: !47)
+!64 = !DILocation(line: 24, column: 11, scope: !47)
+!65 = !{!66, !66, i64 0}
+!66 = !{!"int", !51, i64 0}
+!67 = !DILocation(line: 24, column: 52, scope: !47)
+!68 = !DILocation(line: 26, column: 35, scope: !47)
+!69 = !DILocation(line: 26, column: 47, scope: !47)
+!70 = !DILocation(line: 26, column: 12, scope: !47)
+!71 = !DILocation(line: 26, column: 11, scope: !47)
+!72 = !{!73, !73, i64 0}
+!73 = !{!"long long", !51, i64 0}
+!74 = !DILocation(line: 26, column: 58, scope: !47)
+!75 = !DILocation(line: 28, column: 11, scope: !26)
+!76 = !DILocation(line: 28, column: 7, scope: !26)
+!77 = !DILocation(line: 29, column: 7, scope: !78)
+!78 = distinct !DILexicalBlock(scope: !26, file: !1, line: 29, column: 7)
+!79 = !DILocation(line: 29, column: 7, scope: !26)
+!80 = !DILocation(line: 32, column: 1, scope: !26)
diff --git a/llvm/test/CodeGen/BPF/CORE/offset-reloc-global-1.ll b/llvm/test/CodeGen/BPF/CORE/offset-reloc-global-1.ll
new file mode 100644
index 000000000..a4d4a2764
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/CORE/offset-reloc-global-1.ll
@@ -0,0 +1,82 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfel -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; Source code:
+;   typedef struct v3 { int a; int b; } __v3;
+;   #define _(x) (__builtin_preserve_access_index(x))
+;   int get_value(const int *arg);
+;   __v3 g __attribute__((section("stats")));
+;   int test() {
+;     return get_value(_(&g.b));
+;   }
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm test.c
+
+%struct.v3 = type { i32, i32 }
+
+@g = dso_local global %struct.v3 zeroinitializer, section "stats", align 4, !dbg !0
+
+; Function Attrs: nounwind
+define dso_local i32 @test() local_unnamed_addr #0 !dbg !16 {
+entry:
+  %0 = tail call i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.v3s(%struct.v3* nonnull @g, i32 1, i32 1), !dbg !19, !llvm.preserve.access.index !7
+  %call = tail call i32 @get_value(i32* %0) #3, !dbg !20
+  ret i32 %call, !dbg !21
+}
+
+; CHECK:              r2 = 4
+; CHECK:              r1 = g ll
+; CHECK:              r1 += r2
+; CHECK:              call get_value
+
+; CHECK:              .long   16                      # BTF_KIND_STRUCT(id = [[TID1:[0-9]+]])
+
+; CHECK:              .ascii  ".text"                 # string offset=10
+; CHECK:              .ascii  "v3"                    # string offset=16
+; CHECK:              .ascii  "0:1"                   # string offset=23
+
+; CHECK:              .long   16                      # FieldReloc
+; CHECK-NEXT:         .long   10                      # Field reloc section string offset=10
+; CHECK-NEXT:         .long   1
+; CHECK-NEXT:         .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:         .long   [[TID1]]
+; CHECK-NEXT:         .long   23
+; CHECK-NEXT:         .long   0
+
+declare dso_local i32 @get_value(i32*) local_unnamed_addr #1
+
+; Function Attrs: nounwind readnone
+declare i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.v3s(%struct.v3*, i32, i32) #2
+
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { nounwind readnone }
+attributes #3 = { nounwind }
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!12, !13, !14}
+!llvm.ident = !{!15}
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "g", scope: !2, file: !3, line: 4, type: !6, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 10.0.0 (trunk 367256) (llvm/trunk 367266)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5, nameTableKind: None)
+!3 = !DIFile(filename: "test.c", directory: "/tmp/home/yhs/work/tests/llvm/cast")
+!4 = !{}
+!5 = !{!0}
+!6 = !DIDerivedType(tag: DW_TAG_typedef, name: "__v3", file: !3, line: 1, baseType: !7)
+!7 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "v3", file: !3, line: 1, size: 64, elements: !8)
+!8 = !{!9, !11}
+!9 = !DIDerivedType(tag: DW_TAG_member, name: "a", scope: !7, file: !3, line: 1, baseType: !10, size: 32)
+!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!11 = !DIDerivedType(tag: DW_TAG_member, name: "b", scope: !7, file: !3, line: 1, baseType: !10, size: 32, offset: 32)
+!12 = !{i32 2, !"Dwarf Version", i32 4}
+!13 = !{i32 2, !"Debug Info Version", i32 3}
+!14 = !{i32 1, !"wchar_size", i32 4}
+!15 = !{!"clang version 10.0.0 (trunk 367256) (llvm/trunk 367266)"}
+!16 = distinct !DISubprogram(name: "test", scope: !3, file: !3, line: 5, type: !17, scopeLine: 5, isDefinition: true, isOptimized: true, unit: !2, retainedNodes: !4)
+!17 = !DISubroutineType(types: !18)
+!18 = !{!10}
+!19 = !DILocation(line: 6, column: 20, scope: !16)
+!20 = !DILocation(line: 6, column: 10, scope: !16)
+!21 = !DILocation(line: 6, column: 3, scope: !16)
diff --git a/llvm/test/CodeGen/BPF/CORE/offset-reloc-global-2.ll b/llvm/test/CodeGen/BPF/CORE/offset-reloc-global-2.ll
new file mode 100644
index 000000000..a336e22d4
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/CORE/offset-reloc-global-2.ll
@@ -0,0 +1,98 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfel -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; Source code:
+;   typedef struct v3 { int a; int b; } __v3;
+;   #define _(x) (__builtin_preserve_access_index(x))
+;   int get_value(const int *arg);
+;   __v3 g[4][5] __attribute__((section("stats")));
+;   int test() {
+;     return get_value(_(&g[1][2].b));
+;   }
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm test.c
+
+%struct.v3 = type { i32, i32 }
+
+@g = dso_local global [4 x [5 x %struct.v3]] zeroinitializer, section "stats", align 4, !dbg !0
+
+; Function Attrs: nounwind
+define dso_local i32 @test() local_unnamed_addr #0 !dbg !23 {
+entry:
+  %0 = tail call [5 x %struct.v3]* @llvm.preserve.array.access.index.p0a5s_struct.v3s.p0a4a5s_struct.v3s([4 x [5 x %struct.v3]]* nonnull @g, i32 1, i32 1), !dbg !26, !llvm.preserve.access.index !6
+  %1 = tail call %struct.v3* @llvm.preserve.array.access.index.p0s_struct.v3s.p0a5s_struct.v3s([5 x %struct.v3]* %0, i32 1, i32 2), !dbg !26, !llvm.preserve.access.index !16
+  %2 = tail call i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.v3s(%struct.v3* %1, i32 1, i32 1), !dbg !26, !llvm.preserve.access.index !8
+  %call = tail call i32 @get_value(i32* %2) #3, !dbg !27
+  ret i32 %call, !dbg !28
+}
+
+; CHECK:              r2 = 60
+; CHECK:              r1 = g ll
+; CHECK:              r1 += r2
+; CHECK:              call get_value
+
+; CHECK:              .long   16                      # BTF_KIND_STRUCT(id = [[TID1:[0-9]+]])
+
+; CHECK:              .ascii  ".text"                 # string offset=10
+; CHECK:              .ascii  "v3"                    # string offset=16
+; CHECK:              .ascii  "7:1"                   # string offset=23
+
+; CHECK:              .long   16                      # FieldReloc
+; CHECK-NEXT:         .long   10                      # Field reloc section string offset=10
+; CHECK-NEXT:         .long   1
+; CHECK-NEXT:         .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:         .long   [[TID1]]
+; CHECK-NEXT:         .long   23
+; CHECK-NEXT:         .long   0
+
+
+declare dso_local i32 @get_value(i32*) local_unnamed_addr #1
+
+; Function Attrs: nounwind readnone
+declare [5 x %struct.v3]* @llvm.preserve.array.access.index.p0a5s_struct.v3s.p0a4a5s_struct.v3s([4 x [5 x %struct.v3]]*, i32, i32) #2
+
+; Function Attrs: nounwind readnone
+declare %struct.v3* @llvm.preserve.array.access.index.p0s_struct.v3s.p0a5s_struct.v3s([5 x %struct.v3]*, i32, i32) #2
+
+; Function Attrs: nounwind readnone
+declare i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.v3s(%struct.v3*, i32, i32) #2
+
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { nounwind readnone }
+attributes #3 = { nounwind }
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!19, !20, !21}
+!llvm.ident = !{!22}
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "g", scope: !2, file: !3, line: 4, type: !6, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 10.0.0 (trunk 367256) (llvm/trunk 367266)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, retainedTypes: !5, globals: !18, nameTableKind: None)
+!3 = !DIFile(filename: "test.c", directory: "/tmp/home/yhs/work/tests/llvm/cast")
+!4 = !{}
+!5 = !{!6, !16}
+!6 = !DICompositeType(tag: DW_TAG_array_type, baseType: !7, size: 1280, elements: !13)
+!7 = !DIDerivedType(tag: DW_TAG_typedef, name: "__v3", file: !3, line: 1, baseType: !8)
+!8 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "v3", file: !3, line: 1, size: 64, elements: !9)
+!9 = !{!10, !12}
+!10 = !DIDerivedType(tag: DW_TAG_member, name: "a", scope: !8, file: !3, line: 1, baseType: !11, size: 32)
+!11 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!12 = !DIDerivedType(tag: DW_TAG_member, name: "b", scope: !8, file: !3, line: 1, baseType: !11, size: 32, offset: 32)
+!13 = !{!14, !15}
+!14 = !DISubrange(count: 4)
+!15 = !DISubrange(count: 5)
+!16 = !DICompositeType(tag: DW_TAG_array_type, baseType: !7, size: 320, elements: !17)
+!17 = !{!15}
+!18 = !{!0}
+!19 = !{i32 2, !"Dwarf Version", i32 4}
+!20 = !{i32 2, !"Debug Info Version", i32 3}
+!21 = !{i32 1, !"wchar_size", i32 4}
+!22 = !{!"clang version 10.0.0 (trunk 367256) (llvm/trunk 367266)"}
+!23 = distinct !DISubprogram(name: "test", scope: !3, file: !3, line: 5, type: !24, scopeLine: 5, isDefinition: true, isOptimized: true, unit: !2, retainedNodes: !4)
+!24 = !DISubroutineType(types: !25)
+!25 = !{!11}
+!26 = !DILocation(line: 6, column: 20, scope: !23)
+!27 = !DILocation(line: 6, column: 10, scope: !23)
+!28 = !DILocation(line: 6, column: 3, scope: !23)
diff --git a/llvm/test/CodeGen/BPF/CORE/offset-reloc-global-3.ll b/llvm/test/CodeGen/BPF/CORE/offset-reloc-global-3.ll
new file mode 100644
index 000000000..b2642563a
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/CORE/offset-reloc-global-3.ll
@@ -0,0 +1,87 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfel -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; Source code:
+;   typedef struct v3 { int a; int b; } __v3;
+;   #define _(x) (__builtin_preserve_access_index(x))
+;   int get_value(const int *arg);
+;   __v3 *g __attribute__((section("stats")));
+;   int test() {
+;     return get_value(_(&g->b));
+;   }
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm test.c
+
+%struct.v3 = type { i32, i32 }
+
+@g = dso_local local_unnamed_addr global %struct.v3* null, section "stats", align 8, !dbg !0
+
+; Function Attrs: nounwind
+define dso_local i32 @test() local_unnamed_addr #0 !dbg !17 {
+entry:
+  %0 = load %struct.v3*, %struct.v3** @g, align 8, !dbg !20, !tbaa !21
+  %1 = tail call i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.v3s(%struct.v3* %0, i32 1, i32 1), !dbg !20, !llvm.preserve.access.index !8
+  %call = tail call i32 @get_value(i32* %1) #3, !dbg !25
+  ret i32 %call, !dbg !26
+}
+
+; CHECK:              r2 = 4
+; CHECK:              r1 += r2
+; CHECK:              call get_value
+
+; CHECK:              .long   16                      # BTF_KIND_STRUCT(id = [[TID1:[0-9]+]])
+
+; CHECK:              .ascii  ".text"                 # string offset=10
+; CHECK:              .ascii  "v3"                    # string offset=16
+; CHECK:              .ascii  "0:1"                   # string offset=23
+
+; CHECK:              .long   16                      # FieldReloc
+; CHECK-NEXT:         .long   10                      # Field reloc section string offset=10
+; CHECK-NEXT:         .long   1
+; CHECK-NEXT:         .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:         .long   [[TID1]]
+; CHECK-NEXT:         .long   23
+; CHECK-NEXT:         .long   0
+
+declare dso_local i32 @get_value(i32*) local_unnamed_addr #1
+
+; Function Attrs: nounwind readnone
+declare i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.v3s(%struct.v3*, i32, i32) #2
+
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { nounwind readnone }
+attributes #3 = { nounwind }
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!13, !14, !15}
+!llvm.ident = !{!16}
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "g", scope: !2, file: !3, line: 4, type: !6, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 10.0.0 (trunk 367256) (llvm/trunk 367266)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5, nameTableKind: None)
+!3 = !DIFile(filename: "test.c", directory: "/tmp/home/yhs/work/tests/llvm/cast")
+!4 = !{}
+!5 = !{!0}
+!6 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !7, size: 64)
+!7 = !DIDerivedType(tag: DW_TAG_typedef, name: "__v3", file: !3, line: 1, baseType: !8)
+!8 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "v3", file: !3, line: 1, size: 64, elements: !9)
+!9 = !{!10, !12}
+!10 = !DIDerivedType(tag: DW_TAG_member, name: "a", scope: !8, file: !3, line: 1, baseType: !11, size: 32)
+!11 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!12 = !DIDerivedType(tag: DW_TAG_member, name: "b", scope: !8, file: !3, line: 1, baseType: !11, size: 32, offset: 32)
+!13 = !{i32 2, !"Dwarf Version", i32 4}
+!14 = !{i32 2, !"Debug Info Version", i32 3}
+!15 = !{i32 1, !"wchar_size", i32 4}
+!16 = !{!"clang version 10.0.0 (trunk 367256) (llvm/trunk 367266)"}
+!17 = distinct !DISubprogram(name: "test", scope: !3, file: !3, line: 5, type: !18, scopeLine: 5, isDefinition: true, isOptimized: true, unit: !2, retainedNodes: !4)
+!18 = !DISubroutineType(types: !19)
+!19 = !{!11}
+!20 = !DILocation(line: 6, column: 20, scope: !17)
+!21 = !{!22, !22, i64 0}
+!22 = !{!"any pointer", !23, i64 0}
+!23 = !{!"omnipotent char", !24, i64 0}
+!24 = !{!"Simple C/C++ TBAA"}
+!25 = !DILocation(line: 6, column: 10, scope: !17)
+!26 = !DILocation(line: 6, column: 3, scope: !17)
diff --git a/llvm/test/CodeGen/BPF/CORE/offset-reloc-ignore.ll b/llvm/test/CodeGen/BPF/CORE/offset-reloc-ignore.ll
new file mode 100644
index 000000000..e4609a647
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/CORE/offset-reloc-ignore.ll
@@ -0,0 +1,64 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfel -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; Source code:
+;   #define _(x) (__builtin_preserve_access_index(x))
+;   int get_value(const int *arg);
+;   int test(int *arg) {
+;     return get_value(_(&arg[4]));
+;   }
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm test.c
+
+; Function Attrs: nounwind
+define dso_local i32 @test(i32* %arg) local_unnamed_addr #0 !dbg !10 {
+entry:
+  call void @llvm.dbg.value(metadata i32* %arg, metadata !14, metadata !DIExpression()), !dbg !15
+  %0 = tail call i32* @llvm.preserve.array.access.index.p0i32.p0i32(i32* %arg, i32 0, i32 4), !dbg !16, !llvm.preserve.access.index !4
+  %call = tail call i32 @get_value(i32* %0) #4, !dbg !17
+  ret i32 %call, !dbg !18
+}
+
+; CHECK:             r1 += 16
+; CHECK:             call get_value
+; CHECK:             .section        .BTF.ext,"",@progbits
+; CHECK-NOT:         .long   16                      # FieldReloc
+
+declare dso_local i32 @get_value(i32*) local_unnamed_addr #1
+
+; Function Attrs: nounwind readnone
+declare i32* @llvm.preserve.array.access.index.p0i32.p0i32(i32*, i32, i32) #2
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.value(metadata, metadata, metadata) #3
+
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { nounwind readnone }
+attributes #3 = { nounwind readnone speculatable willreturn }
+attributes #4 = { nounwind }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!6, !7, !8}
+!llvm.ident = !{!9}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 10.0.0 (trunk 367256) (llvm/trunk 367266)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !3, nameTableKind: None)
+!1 = !DIFile(filename: "test.c", directory: "/tmp/home/yhs/work/tests/llvm/cast")
+!2 = !{}
+!3 = !{!4}
+!4 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !5, size: 64)
+!5 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!6 = !{i32 2, !"Dwarf Version", i32 4}
+!7 = !{i32 2, !"Debug Info Version", i32 3}
+!8 = !{i32 1, !"wchar_size", i32 4}
+!9 = !{!"clang version 10.0.0 (trunk 367256) (llvm/trunk 367266)"}
+!10 = distinct !DISubprogram(name: "test", scope: !1, file: !1, line: 3, type: !11, scopeLine: 3, flags: DIFlagPrototyped, isDefinition: true, isOptimized: true, unit: !0, retainedNodes: !13)
+!11 = !DISubroutineType(types: !12)
+!12 = !{!5, !4}
+!13 = !{!14}
+!14 = !DILocalVariable(name: "arg", arg: 1, scope: !10, file: !1, line: 3, type: !4)
+!15 = !DILocation(line: 0, scope: !10)
+!16 = !DILocation(line: 4, column: 20, scope: !10)
+!17 = !DILocation(line: 4, column: 10, scope: !10)
+!18 = !DILocation(line: 4, column: 3, scope: !10)
diff --git a/llvm/test/CodeGen/BPF/CORE/offset-reloc-middle-chain.ll b/llvm/test/CodeGen/BPF/CORE/offset-reloc-middle-chain.ll
new file mode 100644
index 000000000..241258527
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/CORE/offset-reloc-middle-chain.ll
@@ -0,0 +1,132 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfel -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; Source code:
+;   struct t1 {
+;     int c;
+;   };
+;   struct s1 {
+;     struct t1 b;
+;   };
+;   struct r1 {
+;     struct s1 a;
+;   };
+;   #define _(x) __builtin_preserve_access_index(x)
+;   void test1(void *p1, void *p2, void *p3);
+;   void test(struct r1 *arg) {
+;     struct s1 *ps = _(&arg->a);
+;     struct t1 *pt = _(&arg->a.b);
+;     int *pi = _(&arg->a.b.c);
+;     test1(ps, pt, pi);
+;   }
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm test.c
+
+%struct.r1 = type { %struct.s1 }
+%struct.s1 = type { %struct.t1 }
+%struct.t1 = type { i32 }
+
+; Function Attrs: nounwind
+define dso_local void @test(%struct.r1* %arg) local_unnamed_addr #0 !dbg !7 {
+entry:
+  call void @llvm.dbg.value(metadata %struct.r1* %arg, metadata !22, metadata !DIExpression()), !dbg !29
+  %0 = tail call %struct.s1* @llvm.preserve.struct.access.index.p0s_struct.s1s.p0s_struct.r1s(%struct.r1* %arg, i32 0, i32 0), !dbg !30, !llvm.preserve.access.index !11
+  call void @llvm.dbg.value(metadata %struct.s1* %0, metadata !23, metadata !DIExpression()), !dbg !29
+  %1 = tail call %struct.t1* @llvm.preserve.struct.access.index.p0s_struct.t1s.p0s_struct.s1s(%struct.s1* %0, i32 0, i32 0), !dbg !31, !llvm.preserve.access.index !14
+  call void @llvm.dbg.value(metadata %struct.t1* %1, metadata !25, metadata !DIExpression()), !dbg !29
+  %2 = tail call i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.t1s(%struct.t1* %1, i32 0, i32 0), !dbg !32, !llvm.preserve.access.index !17
+  call void @llvm.dbg.value(metadata i32* %2, metadata !27, metadata !DIExpression()), !dbg !29
+  %3 = bitcast %struct.s1* %0 to i8*, !dbg !33
+  %4 = bitcast %struct.t1* %1 to i8*, !dbg !34
+  %5 = bitcast i32* %2 to i8*, !dbg !35
+  tail call void @test1(i8* %3, i8* %4, i8* %5) #4, !dbg !36
+  ret void, !dbg !37
+}
+
+; CHECK:             .long   1                       # BTF_KIND_STRUCT(id = 2)
+
+; CHECK:             .ascii  "r1"                    # string offset=1
+; CHECK:             .ascii  ".text"                 # string offset=29
+; CHECK:             .ascii  "0:0"                   # string offset=72
+; CHECK:             .ascii  "0:0:0"                 # string offset=76
+; CHECK:             .ascii  "0:0:0:0"               # string offset=82
+
+; CHECK:             .long   16                      # FieldReloc
+; CHECK-NEXT:        .long   29                      # Field reloc section string offset=29
+; CHECK-NEXT:        .long   3
+; CHECK_NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK_NEXT:        .long   2
+; CHECK_NEXT:        .long   72
+; CHECK_NEXT:        .long   0
+; CHECK_NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK_NEXT:        .long   2
+; CHECK_NEXT:        .long   76
+; CHECK_NEXT:        .long   0
+; CHECK_NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK_NEXT:        .long   2
+; CHECK_NEXT:        .long   82
+; CHECK_NEXT:        .long   0
+
+; Function Attrs: nounwind readnone
+declare %struct.s1* @llvm.preserve.struct.access.index.p0s_struct.s1s.p0s_struct.r1s(%struct.r1*, i32, i32) #1
+
+; Function Attrs: nounwind readnone
+declare %struct.t1* @llvm.preserve.struct.access.index.p0s_struct.t1s.p0s_struct.s1s(%struct.s1*, i32, i32) #1
+
+; Function Attrs: nounwind readnone
+declare i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.t1s(%struct.t1*, i32, i32) #1
+
+declare dso_local void @test1(i8*, i8*, i8*) local_unnamed_addr #2
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.value(metadata, metadata, metadata) #3
+
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone }
+attributes #2 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { nounwind readnone speculatable willreturn }
+attributes #4 = { nounwind }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 10.0.0 (https://github.com/llvm/llvm-project.git 42b3328a2368b38fba6bdb0c616fe6c5520e3bc5)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "test.c", directory: "/tmp/home/yhs/work/tests/core")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{!"clang version 10.0.0 (https://github.com/llvm/llvm-project.git 42b3328a2368b38fba6bdb0c616fe6c5520e3bc5)"}
+!7 = distinct !DISubprogram(name: "test", scope: !1, file: !1, line: 12, type: !8, scopeLine: 12, flags: DIFlagPrototyped, isDefinition: true, isOptimized: true, unit: !0, retainedNodes: !21)
+!8 = !DISubroutineType(types: !9)
+!9 = !{null, !10}
+!10 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !11, size: 64)
+!11 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "r1", file: !1, line: 7, size: 32, elements: !12)
+!12 = !{!13}
+!13 = !DIDerivedType(tag: DW_TAG_member, name: "a", scope: !11, file: !1, line: 8, baseType: !14, size: 32)
+!14 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "s1", file: !1, line: 4, size: 32, elements: !15)
+!15 = !{!16}
+!16 = !DIDerivedType(tag: DW_TAG_member, name: "b", scope: !14, file: !1, line: 5, baseType: !17, size: 32)
+!17 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t1", file: !1, line: 1, size: 32, elements: !18)
+!18 = !{!19}
+!19 = !DIDerivedType(tag: DW_TAG_member, name: "c", scope: !17, file: !1, line: 2, baseType: !20, size: 32)
+!20 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!21 = !{!22, !23, !25, !27}
+!22 = !DILocalVariable(name: "arg", arg: 1, scope: !7, file: !1, line: 12, type: !10)
+!23 = !DILocalVariable(name: "ps", scope: !7, file: !1, line: 13, type: !24)
+!24 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !14, size: 64)
+!25 = !DILocalVariable(name: "pt", scope: !7, file: !1, line: 14, type: !26)
+!26 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !17, size: 64)
+!27 = !DILocalVariable(name: "pi", scope: !7, file: !1, line: 15, type: !28)
+!28 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !20, size: 64)
+!29 = !DILocation(line: 0, scope: !7)
+!30 = !DILocation(line: 13, column: 19, scope: !7)
+!31 = !DILocation(line: 14, column: 19, scope: !7)
+!32 = !DILocation(line: 15, column: 13, scope: !7)
+!33 = !DILocation(line: 16, column: 9, scope: !7)
+!34 = !DILocation(line: 16, column: 13, scope: !7)
+!35 = !DILocation(line: 16, column: 17, scope: !7)
+!36 = !DILocation(line: 16, column: 3, scope: !7)
+!37 = !DILocation(line: 17, column: 1, scope: !7)
diff --git a/llvm/test/CodeGen/BPF/CORE/offset-reloc-multi-array-1.ll b/llvm/test/CodeGen/BPF/CORE/offset-reloc-multi-array-1.ll
new file mode 100644
index 000000000..d61e41c3b
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/CORE/offset-reloc-multi-array-1.ll
@@ -0,0 +1,104 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfel -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; Source code:
+;   typedef int __int;
+;   typedef struct v3 { int a; __int b[4][4]; } __v3;
+;   #define _(x) (__builtin_preserve_access_index(x))
+;   int get_value(const int *arg);
+;   int test(__v3 *arg) {
+;     return get_value(_(&arg[1].b[2][3]));
+;   }
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm test.c
+
+%struct.v3 = type { i32, [4 x [4 x i32]] }
+
+; Function Attrs: nounwind
+define dso_local i32 @test(%struct.v3* %arg) local_unnamed_addr #0 !dbg !21 {
+entry:
+  call void @llvm.dbg.value(metadata %struct.v3* %arg, metadata !25, metadata !DIExpression()), !dbg !26
+  %0 = tail call %struct.v3* @llvm.preserve.array.access.index.p0s_struct.v3s.p0s_struct.v3s(%struct.v3* %arg, i32 0, i32 1), !dbg !27, !llvm.preserve.access.index !4
+  %1 = tail call [4 x [4 x i32]]* @llvm.preserve.struct.access.index.p0a4a4i32.p0s_struct.v3s(%struct.v3* %0, i32 1, i32 1), !dbg !27, !llvm.preserve.access.index !6
+  %2 = tail call [4 x i32]* @llvm.preserve.array.access.index.p0a4i32.p0a4a4i32([4 x [4 x i32]]* %1, i32 1, i32 2), !dbg !27, !llvm.preserve.access.index !11
+  %3 = tail call i32* @llvm.preserve.array.access.index.p0i32.p0a4i32([4 x i32]* %2, i32 1, i32 3), !dbg !27, !llvm.preserve.access.index !15
+  %call = tail call i32 @get_value(i32* %3) #4, !dbg !28
+  ret i32 %call, !dbg !29
+}
+
+; CHECK:              r2 = 116
+; CHECK:              r1 += r2
+; CHECK:              call get_value
+
+; CHECK:              .long   6                       # BTF_KIND_STRUCT(id = [[TID1:[0-9]+]])
+
+; CHECK:              .ascii  "v3"                    # string offset=6
+; CHECK:              .ascii  ".text"                 # string offset=52
+; CHECK:              .ascii  "1:1:2:3"               # string offset=58
+
+; CHECK:              .long   16                      # FieldReloc
+; CHECK-NEXT:         .long   52                      # Field reloc section string offset=52
+; CHECK-NEXT:         .long   1
+; CHECK-NEXT:         .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:         .long   [[TID1]]
+; CHECK-NEXT:         .long   58
+; CHECK-NEXT:         .long   0
+
+declare dso_local i32 @get_value(i32*) local_unnamed_addr #1
+
+; Function Attrs: nounwind readnone
+declare %struct.v3* @llvm.preserve.array.access.index.p0s_struct.v3s.p0s_struct.v3s(%struct.v3*, i32, i32) #2
+
+; Function Attrs: nounwind readnone
+declare [4 x [4 x i32]]* @llvm.preserve.struct.access.index.p0a4a4i32.p0s_struct.v3s(%struct.v3*, i32, i32) #2
+
+; Function Attrs: nounwind readnone
+declare [4 x i32]* @llvm.preserve.array.access.index.p0a4i32.p0a4a4i32([4 x [4 x i32]]*, i32, i32) #2
+
+; Function Attrs: nounwind readnone
+declare i32* @llvm.preserve.array.access.index.p0i32.p0a4i32([4 x i32]*, i32, i32) #2
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.value(metadata, metadata, metadata) #3
+
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { nounwind readnone }
+attributes #3 = { nounwind readnone speculatable willreturn }
+attributes #4 = { nounwind }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!17, !18, !19}
+!llvm.ident = !{!20}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 10.0.0 (trunk 367256) (llvm/trunk 367266)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !3, nameTableKind: None)
+!1 = !DIFile(filename: "test.c", directory: "/tmp/home/yhs/work/tests/llvm/cast")
+!2 = !{}
+!3 = !{!4, !11, !15}
+!4 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !5, size: 64)
+!5 = !DIDerivedType(tag: DW_TAG_typedef, name: "__v3", file: !1, line: 2, baseType: !6)
+!6 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "v3", file: !1, line: 2, size: 544, elements: !7)
+!7 = !{!8, !10}
+!8 = !DIDerivedType(tag: DW_TAG_member, name: "a", scope: !6, file: !1, line: 2, baseType: !9, size: 32)
+!9 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!10 = !DIDerivedType(tag: DW_TAG_member, name: "b", scope: !6, file: !1, line: 2, baseType: !11, size: 512, offset: 32)
+!11 = !DICompositeType(tag: DW_TAG_array_type, baseType: !12, size: 512, elements: !13)
+!12 = !DIDerivedType(tag: DW_TAG_typedef, name: "__int", file: !1, line: 1, baseType: !9)
+!13 = !{!14, !14}
+!14 = !DISubrange(count: 4)
+!15 = !DICompositeType(tag: DW_TAG_array_type, baseType: !12, size: 128, elements: !16)
+!16 = !{!14}
+!17 = !{i32 2, !"Dwarf Version", i32 4}
+!18 = !{i32 2, !"Debug Info Version", i32 3}
+!19 = !{i32 1, !"wchar_size", i32 4}
+!20 = !{!"clang version 10.0.0 (trunk 367256) (llvm/trunk 367266)"}
+!21 = distinct !DISubprogram(name: "test", scope: !1, file: !1, line: 5, type: !22, scopeLine: 5, flags: DIFlagPrototyped, isDefinition: true, isOptimized: true, unit: !0, retainedNodes: !24)
+!22 = !DISubroutineType(types: !23)
+!23 = !{!9, !4}
+!24 = !{!25}
+!25 = !DILocalVariable(name: "arg", arg: 1, scope: !21, file: !1, line: 5, type: !4)
+!26 = !DILocation(line: 0, scope: !21)
+!27 = !DILocation(line: 6, column: 20, scope: !21)
+!28 = !DILocation(line: 6, column: 10, scope: !21)
+!29 = !DILocation(line: 6, column: 3, scope: !21)
diff --git a/llvm/test/CodeGen/BPF/CORE/offset-reloc-multi-array-2.ll b/llvm/test/CodeGen/BPF/CORE/offset-reloc-multi-array-2.ll
new file mode 100644
index 000000000..d7d65e530
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/CORE/offset-reloc-multi-array-2.ll
@@ -0,0 +1,110 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfel -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; Source code:
+;   typedef int __int;
+;   typedef struct v3 { int a; __int b[4][4][4]; } __v3;
+;   #define _(x) (__builtin_preserve_access_index(x))
+;   int get_value(const int *arg);
+;   int test(__v3 *arg) {
+;     return get_value(_(&arg[1].b[2][3][2]));
+;   }
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm test.c
+
+%struct.v3 = type { i32, [4 x [4 x [4 x i32]]] }
+
+; Function Attrs: nounwind
+define dso_local i32 @test(%struct.v3* %arg) local_unnamed_addr #0 !dbg !23 {
+entry:
+  call void @llvm.dbg.value(metadata %struct.v3* %arg, metadata !27, metadata !DIExpression()), !dbg !28
+  %0 = tail call %struct.v3* @llvm.preserve.array.access.index.p0s_struct.v3s.p0s_struct.v3s(%struct.v3* %arg, i32 0, i32 1), !dbg !29, !llvm.preserve.access.index !4
+  %1 = tail call [4 x [4 x [4 x i32]]]* @llvm.preserve.struct.access.index.p0a4a4a4i32.p0s_struct.v3s(%struct.v3* %0, i32 1, i32 1), !dbg !29, !llvm.preserve.access.index !6
+  %2 = tail call [4 x [4 x i32]]* @llvm.preserve.array.access.index.p0a4a4i32.p0a4a4a4i32([4 x [4 x [4 x i32]]]* %1, i32 1, i32 2), !dbg !29, !llvm.preserve.access.index !11
+  %3 = tail call [4 x i32]* @llvm.preserve.array.access.index.p0a4i32.p0a4a4i32([4 x [4 x i32]]* %2, i32 1, i32 3), !dbg !29, !llvm.preserve.access.index !15
+  %4 = tail call i32* @llvm.preserve.array.access.index.p0i32.p0a4i32([4 x i32]* %3, i32 1, i32 2), !dbg !29, !llvm.preserve.access.index !17
+  %call = tail call i32 @get_value(i32* %4) #4, !dbg !30
+  ret i32 %call, !dbg !31
+}
+
+; CHECK:             r2 = 448
+; CHECK:             r1 += r2
+; CHECK:             call get_value
+
+; CHECK:             .long   6                       # BTF_KIND_STRUCT(id = [[TID1:[0-9]+]])
+
+; CHECK:             .ascii  "v3"                    # string offset=6
+; CHECK:             .ascii  ".text"                 # string offset=52
+; CHECK:             .ascii  "1:1:2:3:2"             # string offset=58
+
+; CHECK:             .long   16                      # FieldReloc
+; CHECK-NEXT:        .long   52                      # Field reloc section string offset=52
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   [[TID1]]
+; CHECK-NEXT:        .long   58
+; CHECK-NEXT:        .long   0
+
+declare dso_local i32 @get_value(i32*) local_unnamed_addr #1
+
+; Function Attrs: nounwind readnone
+declare %struct.v3* @llvm.preserve.array.access.index.p0s_struct.v3s.p0s_struct.v3s(%struct.v3*, i32, i32) #2
+
+; Function Attrs: nounwind readnone
+declare [4 x [4 x [4 x i32]]]* @llvm.preserve.struct.access.index.p0a4a4a4i32.p0s_struct.v3s(%struct.v3*, i32, i32) #2
+
+; Function Attrs: nounwind readnone
+declare [4 x [4 x i32]]* @llvm.preserve.array.access.index.p0a4a4i32.p0a4a4a4i32([4 x [4 x [4 x i32]]]*, i32, i32) #2
+
+; Function Attrs: nounwind readnone
+declare [4 x i32]* @llvm.preserve.array.access.index.p0a4i32.p0a4a4i32([4 x [4 x i32]]*, i32, i32) #2
+
+; Function Attrs: nounwind readnone
+declare i32* @llvm.preserve.array.access.index.p0i32.p0a4i32([4 x i32]*, i32, i32) #2
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.value(metadata, metadata, metadata) #3
+
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { nounwind readnone }
+attributes #3 = { nounwind readnone speculatable willreturn }
+attributes #4 = { nounwind }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!19, !20, !21}
+!llvm.ident = !{!22}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 10.0.0 (trunk 367256) (llvm/trunk 367266)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !3, nameTableKind: None)
+!1 = !DIFile(filename: "test.c", directory: "/tmp/home/yhs/work/tests/llvm/cast")
+!2 = !{}
+!3 = !{!4, !11, !15, !17}
+!4 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !5, size: 64)
+!5 = !DIDerivedType(tag: DW_TAG_typedef, name: "__v3", file: !1, line: 2, baseType: !6)
+!6 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "v3", file: !1, line: 2, size: 2080, elements: !7)
+!7 = !{!8, !10}
+!8 = !DIDerivedType(tag: DW_TAG_member, name: "a", scope: !6, file: !1, line: 2, baseType: !9, size: 32)
+!9 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!10 = !DIDerivedType(tag: DW_TAG_member, name: "b", scope: !6, file: !1, line: 2, baseType: !11, size: 2048, offset: 32)
+!11 = !DICompositeType(tag: DW_TAG_array_type, baseType: !12, size: 2048, elements: !13)
+!12 = !DIDerivedType(tag: DW_TAG_typedef, name: "__int", file: !1, line: 1, baseType: !9)
+!13 = !{!14, !14, !14}
+!14 = !DISubrange(count: 4)
+!15 = !DICompositeType(tag: DW_TAG_array_type, baseType: !12, size: 512, elements: !16)
+!16 = !{!14, !14}
+!17 = !DICompositeType(tag: DW_TAG_array_type, baseType: !12, size: 128, elements: !18)
+!18 = !{!14}
+!19 = !{i32 2, !"Dwarf Version", i32 4}
+!20 = !{i32 2, !"Debug Info Version", i32 3}
+!21 = !{i32 1, !"wchar_size", i32 4}
+!22 = !{!"clang version 10.0.0 (trunk 367256) (llvm/trunk 367266)"}
+!23 = distinct !DISubprogram(name: "test", scope: !1, file: !1, line: 5, type: !24, scopeLine: 5, flags: DIFlagPrototyped, isDefinition: true, isOptimized: true, unit: !0, retainedNodes: !26)
+!24 = !DISubroutineType(types: !25)
+!25 = !{!9, !4}
+!26 = !{!27}
+!27 = !DILocalVariable(name: "arg", arg: 1, scope: !23, file: !1, line: 5, type: !4)
+!28 = !DILocation(line: 0, scope: !23)
+!29 = !DILocation(line: 6, column: 20, scope: !23)
+!30 = !DILocation(line: 6, column: 10, scope: !23)
+!31 = !DILocation(line: 6, column: 3, scope: !23)
diff --git a/llvm/test/CodeGen/BPF/CORE/offset-reloc-multilevel.ll b/llvm/test/CodeGen/BPF/CORE/offset-reloc-multilevel.ll
index 779b39529..905a03d16 100644
--- a/llvm/test/CodeGen/BPF/CORE/offset-reloc-multilevel.ll
+++ b/llvm/test/CodeGen/BPF/CORE/offset-reloc-multilevel.ll
@@ -1,5 +1,7 @@
 ; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
 ; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfel -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
 ; Source code:
 ;   struct net_device {
 ;     int dev_id;
@@ -80,7 +82,7 @@ define dso_local i32 @bpf_prog(%struct.sk_buff*) local_unnamed_addr #0 !dbg !15
 ; CHECK-NEXT:        .long   44
 ; CHECK-NEXT:        .long   1
 ; CHECK-NEXT:        .long   48                      # BTF_KIND_FUNC(id = 6)
-; CHECK-NEXT:        .long   201326592               # 0xc000000
+; CHECK-NEXT:        .long   201326593               # 0xc000001
 ; CHECK-NEXT:        .long   5
 ; CHECK-NEXT:        .byte   0                       # string offset=0
 ; CHECK-NEXT:        .ascii  "sk_buff"               # string offset=1
@@ -111,23 +113,22 @@ define dso_local i32 @bpf_prog(%struct.sk_buff*) local_unnamed_addr #0 !dbg !15
 ; CHECK-NEXT:        .short  60319                   # 0xeb9f
 ; CHECK-NEXT:        .byte   1
 ; CHECK-NEXT:        .byte   0
-; CHECK-NEXT:        .long   40
+; CHECK-NEXT:        .long   32
 ; CHECK-NEXT:        .long   0
 ; CHECK-NEXT:        .long   20
 ; CHECK-NEXT:        .long   20
-; CHECK-NEXT:        .long   76
-; CHECK-NEXT:        .long   96
-; CHECK-NEXT:        .long   24
-; CHECK-NEXT:        .long   120
-; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   {{[0-9]+}}
+; CHECK-NEXT:        .long   {{[0-9]+}}
+; CHECK-NEXT:        .long   28
 ; CHECK-NEXT:        .long   8                       # FuncInfo
 
-; CHECK:             .long   12                      # OffsetReloc
-; CHECK-NEXT:        .long   57                      # Offset reloc section string offset=57
+; CHECK:             .long   16                      # FieldReloc
+; CHECK-NEXT:        .long   57                      # Field reloc section string offset=57
 ; CHECK-NEXT:        .long   1
-; CHECK-NEXT:        .long   .Ltmp2
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
 ; CHECK-NEXT:        .long   2
 ; CHECK-NEXT:        .long   100
+; CHECK-NEXT:        .long   0
 
 ; Function Attrs: argmemonly nounwind
 declare void @llvm.lifetime.start.p0i8(i64 immarg, i8* nocapture) #1
@@ -144,7 +145,7 @@ declare void @llvm.lifetime.end.p0i8(i64 immarg, i8* nocapture) #1
 ; Function Attrs: nounwind readnone speculatable
 declare void @llvm.dbg.value(metadata, metadata, metadata) #3
 
-attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { argmemonly nounwind }
 attributes #2 = { nounwind readnone }
 attributes #3 = { nounwind readnone speculatable }
diff --git a/llvm/test/CodeGen/BPF/CORE/offset-reloc-pointer-1.ll b/llvm/test/CodeGen/BPF/CORE/offset-reloc-pointer-1.ll
new file mode 100644
index 000000000..a988a7d12
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/CORE/offset-reloc-pointer-1.ll
@@ -0,0 +1,86 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfel -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; Source code:
+;   typedef struct v3 { int a; int b; } __v3;
+;   #define _(x) (__builtin_preserve_access_index(x))
+;   int get_value(const int *arg);
+;   int test(__v3 *arg) {
+;     return get_value(_(&arg[1]));
+;   }
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm test.c
+
+%struct.v3 = type { i32, i32 }
+
+; Function Attrs: nounwind
+define dso_local i32 @test(%struct.v3* %arg) local_unnamed_addr #0 !dbg !15 {
+entry:
+  call void @llvm.dbg.value(metadata %struct.v3* %arg, metadata !19, metadata !DIExpression()), !dbg !20
+  %0 = tail call %struct.v3* @llvm.preserve.array.access.index.p0s_struct.v3s.p0s_struct.v3s(%struct.v3* %arg, i32 0, i32 1), !dbg !21, !llvm.preserve.access.index !4
+  %1 = getelementptr inbounds %struct.v3, %struct.v3* %0, i64 0, i32 0, !dbg !21
+  %call = tail call i32 @get_value(i32* %1) #4, !dbg !22
+  ret i32 %call, !dbg !23
+}
+
+; CHECK:              r2 = 8
+; CHECK:              r1 += r2
+; CHECK:              call get_value
+
+; CHECK:              .long   6                       # BTF_KIND_STRUCT(id = [[TID1:[0-9]+]])
+
+; CHECK:              .ascii  "v3"                    # string offset=6
+; CHECK:              .ascii  ".text"                 # string offset=26
+; CHECK:              .byte   49                      # string offset=32
+
+; CHECK:              .long   16                      # FieldReloc
+; CHECK-NEXT:         .long   26                      # Field reloc section string offset=26
+; CHECK-NEXT:         .long   1
+; CHECK-NEXT:         .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:         .long   [[TID1]]
+; CHECK-NEXT:         .long   32
+; CHECK-NEXT:         .long   0
+
+declare dso_local i32 @get_value(i32*) local_unnamed_addr #1
+
+; Function Attrs: nounwind readnone
+declare %struct.v3* @llvm.preserve.array.access.index.p0s_struct.v3s.p0s_struct.v3s(%struct.v3*, i32, i32) #2
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.value(metadata, metadata, metadata) #3
+
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { nounwind readnone }
+attributes #3 = { nounwind readnone speculatable willreturn }
+attributes #4 = { nounwind }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!11, !12, !13}
+!llvm.ident = !{!14}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 10.0.0 (trunk 367256) (llvm/trunk 367266)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !3, nameTableKind: None)
+!1 = !DIFile(filename: "test.c", directory: "/tmp/home/yhs/work/tests/llvm/cast")
+!2 = !{}
+!3 = !{!4}
+!4 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !5, size: 64)
+!5 = !DIDerivedType(tag: DW_TAG_typedef, name: "__v3", file: !1, line: 1, baseType: !6)
+!6 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "v3", file: !1, line: 1, size: 64, elements: !7)
+!7 = !{!8, !10}
+!8 = !DIDerivedType(tag: DW_TAG_member, name: "a", scope: !6, file: !1, line: 1, baseType: !9, size: 32)
+!9 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!10 = !DIDerivedType(tag: DW_TAG_member, name: "b", scope: !6, file: !1, line: 1, baseType: !9, size: 32, offset: 32)
+!11 = !{i32 2, !"Dwarf Version", i32 4}
+!12 = !{i32 2, !"Debug Info Version", i32 3}
+!13 = !{i32 1, !"wchar_size", i32 4}
+!14 = !{!"clang version 10.0.0 (trunk 367256) (llvm/trunk 367266)"}
+!15 = distinct !DISubprogram(name: "test", scope: !1, file: !1, line: 4, type: !16, scopeLine: 4, flags: DIFlagPrototyped, isDefinition: true, isOptimized: true, unit: !0, retainedNodes: !18)
+!16 = !DISubroutineType(types: !17)
+!17 = !{!9, !4}
+!18 = !{!19}
+!19 = !DILocalVariable(name: "arg", arg: 1, scope: !15, file: !1, line: 4, type: !4)
+!20 = !DILocation(line: 0, scope: !15)
+!21 = !DILocation(line: 5, column: 20, scope: !15)
+!22 = !DILocation(line: 5, column: 10, scope: !15)
+!23 = !DILocation(line: 5, column: 3, scope: !15)
diff --git a/llvm/test/CodeGen/BPF/CORE/offset-reloc-pointer-2.ll b/llvm/test/CodeGen/BPF/CORE/offset-reloc-pointer-2.ll
new file mode 100644
index 000000000..b747e75b2
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/CORE/offset-reloc-pointer-2.ll
@@ -0,0 +1,88 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfel -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; Source code:
+;   typedef struct v3 { int a; int b; } __v3;
+;   #define _(x) (__builtin_preserve_access_index(x))
+;   int get_value(const int *arg);
+;   int test(__v3 *arg) {
+;     return get_value(_(&arg[1].b));
+;   }
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm test.c
+
+%struct.v3 = type { i32, i32 }
+
+; Function Attrs: nounwind
+define dso_local i32 @test(%struct.v3* %arg) local_unnamed_addr #0 !dbg !15 {
+entry:
+  call void @llvm.dbg.value(metadata %struct.v3* %arg, metadata !19, metadata !DIExpression()), !dbg !20
+  %0 = tail call %struct.v3* @llvm.preserve.array.access.index.p0s_struct.v3s.p0s_struct.v3s(%struct.v3* %arg, i32 0, i32 1), !dbg !21, !llvm.preserve.access.index !4
+  %1 = tail call i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.v3s(%struct.v3* %0, i32 1, i32 1), !dbg !21, !llvm.preserve.access.index !6
+  %call = tail call i32 @get_value(i32* %1) #4, !dbg !22
+  ret i32 %call, !dbg !23
+}
+
+; CHECK:              r2 = 12
+; CHECK-NEXT:         r1 += r2
+; CHECK:              call get_value
+
+; CHECK:              .long   6                       # BTF_KIND_STRUCT(id = [[TID1:[0-9]+]])
+; CHECK:              .ascii  "v3"                    # string offset=6
+; CHECK:              .ascii  ".text"                 # string offset=26
+; CHECK:              .ascii  "1:1"                   # string offset=32
+
+; CHECK:              .long   16                      # FieldReloc
+; CHECK-NEXT:         .long   26                      # Field reloc section string offset=26
+; CHECK-NEXT:         .long   1
+; CHECK-NEXT:         .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:         .long   [[TID1]]
+; CHECK-NEXT:         .long   32
+; CHECK-NEXT:         .long   0
+
+declare dso_local i32 @get_value(i32*) local_unnamed_addr #1
+
+; Function Attrs: nounwind readnone
+declare %struct.v3* @llvm.preserve.array.access.index.p0s_struct.v3s.p0s_struct.v3s(%struct.v3*, i32, i32) #2
+
+; Function Attrs: nounwind readnone
+declare i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.v3s(%struct.v3*, i32, i32) #2
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.value(metadata, metadata, metadata) #3
+
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { nounwind readnone }
+attributes #3 = { nounwind readnone speculatable willreturn }
+attributes #4 = { nounwind }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!11, !12, !13}
+!llvm.ident = !{!14}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 10.0.0 (trunk 367256) (llvm/trunk 367266)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !3, nameTableKind: None)
+!1 = !DIFile(filename: "test.c", directory: "/tmp/home/yhs/work/tests/llvm/cast")
+!2 = !{}
+!3 = !{!4}
+!4 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !5, size: 64)
+!5 = !DIDerivedType(tag: DW_TAG_typedef, name: "__v3", file: !1, line: 1, baseType: !6)
+!6 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "v3", file: !1, line: 1, size: 64, elements: !7)
+!7 = !{!8, !10}
+!8 = !DIDerivedType(tag: DW_TAG_member, name: "a", scope: !6, file: !1, line: 1, baseType: !9, size: 32)
+!9 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!10 = !DIDerivedType(tag: DW_TAG_member, name: "b", scope: !6, file: !1, line: 1, baseType: !9, size: 32, offset: 32)
+!11 = !{i32 2, !"Dwarf Version", i32 4}
+!12 = !{i32 2, !"Debug Info Version", i32 3}
+!13 = !{i32 1, !"wchar_size", i32 4}
+!14 = !{!"clang version 10.0.0 (trunk 367256) (llvm/trunk 367266)"}
+!15 = distinct !DISubprogram(name: "test", scope: !1, file: !1, line: 4, type: !16, scopeLine: 4, flags: DIFlagPrototyped, isDefinition: true, isOptimized: true, unit: !0, retainedNodes: !18)
+!16 = !DISubroutineType(types: !17)
+!17 = !{!9, !4}
+!18 = !{!19}
+!19 = !DILocalVariable(name: "arg", arg: 1, scope: !15, file: !1, line: 4, type: !4)
+!20 = !DILocation(line: 0, scope: !15)
+!21 = !DILocation(line: 5, column: 20, scope: !15)
+!22 = !DILocation(line: 5, column: 10, scope: !15)
+!23 = !DILocation(line: 5, column: 3, scope: !15)
diff --git a/llvm/test/CodeGen/BPF/CORE/offset-reloc-struct-anonymous.ll b/llvm/test/CodeGen/BPF/CORE/offset-reloc-struct-anonymous.ll
index 732187c01..540aecf46 100644
--- a/llvm/test/CodeGen/BPF/CORE/offset-reloc-struct-anonymous.ll
+++ b/llvm/test/CodeGen/BPF/CORE/offset-reloc-struct-anonymous.ll
@@ -1,5 +1,7 @@
 ; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
 ; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfel -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
 ; Source code:
 ;   struct sk_buff {
 ;     int i;
@@ -30,7 +32,7 @@ define dso_local i32 @bpf_prog(%struct.sk_buff*) local_unnamed_addr #0 !dbg !15
   %3 = bitcast i32* %2 to i8*, !dbg !34
   call void @llvm.lifetime.start.p0i8(i64 4, i8* nonnull %3) #4, !dbg !34
   %4 = tail call [10 x %struct.anon]* @llvm.preserve.struct.access.index.p0a10s_struct.anons.p0s_struct.sk_buffs(%struct.sk_buff* %0, i32 1, i32 1), !dbg !35, !llvm.preserve.access.index !19
-  %5 = tail call %struct.anon* @llvm.preserve.array.access.index.p0s_struct.anons.p0a10s_struct.anons([10 x %struct.anon]* %4, i32 1, i32 5), !dbg !35
+  %5 = tail call %struct.anon* @llvm.preserve.array.access.index.p0s_struct.anons.p0a10s_struct.anons([10 x %struct.anon]* %4, i32 1, i32 5), !dbg !35, !llvm.preserve.access.index !23
   %6 = tail call i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.anons(%struct.anon* %5, i32 0, i32 0), !dbg !35, !llvm.preserve.access.index !24
   %7 = bitcast i32* %6 to i8*, !dbg !35
   %8 = call i32 inttoptr (i64 4 to i32 (i8*, i32, i8*)*)(i8* nonnull %3, i32 4, i8* %7) #4, !dbg !36
@@ -90,7 +92,7 @@ define dso_local i32 @bpf_prog(%struct.sk_buff*) local_unnamed_addr #0 !dbg !15
 ; CHECK-NEXT:        .long   53
 ; CHECK-NEXT:        .long   1
 ; CHECK-NEXT:        .long   57                      # BTF_KIND_FUNC(id = 8)
-; CHECK-NEXT:        .long   201326592               # 0xc000000
+; CHECK-NEXT:        .long   201326593               # 0xc000001
 ; CHECK-NEXT:        .long   7
 ; CHECK-NEXT:        .byte   0                       # string offset=0
 ; CHECK-NEXT:        .ascii  "sk_buff"               # string offset=1
@@ -121,23 +123,22 @@ define dso_local i32 @bpf_prog(%struct.sk_buff*) local_unnamed_addr #0 !dbg !15
 ; CHECK-NEXT:        .short  60319                   # 0xeb9f
 ; CHECK-NEXT:        .byte   1
 ; CHECK-NEXT:        .byte   0
-; CHECK-NEXT:        .long   40
+; CHECK-NEXT:        .long   32
 ; CHECK-NEXT:        .long   0
 ; CHECK-NEXT:        .long   20
 ; CHECK-NEXT:        .long   20
-; CHECK-NEXT:        .long   76
-; CHECK-NEXT:        .long   96
-; CHECK-NEXT:        .long   24
-; CHECK-NEXT:        .long   120
-; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   {{[0-9]+}}
+; CHECK-NEXT:        .long   {{[0-9]+}}
+; CHECK-NEXT:        .long   28
 ; CHECK-NEXT:        .long   8                       # FuncInfo
 
-; CHECK:             .long   12                      # OffsetReloc
-; CHECK-NEXT:        .long   66                      # Offset reloc section string offset=66
+; CHECK:             .long   16                      # FieldReloc
+; CHECK-NEXT:        .long   66                      # Field reloc section string offset=66
 ; CHECK-NEXT:        .long   1
-; CHECK-NEXT:        .long   .Ltmp2
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
 ; CHECK-NEXT:        .long   2
 ; CHECK-NEXT:        .long   109
+; CHECK-NEXT:        .long   0
 
 ; Function Attrs: argmemonly nounwind
 declare void @llvm.lifetime.start.p0i8(i64 immarg, i8* nocapture) #1
@@ -157,7 +158,7 @@ declare void @llvm.lifetime.end.p0i8(i64 immarg, i8* nocapture) #1
 ; Function Attrs: nounwind readnone speculatable
 declare void @llvm.dbg.value(metadata, metadata, metadata) #3
 
-attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { argmemonly nounwind }
 attributes #2 = { nounwind readnone }
 attributes #3 = { nounwind readnone speculatable }
diff --git a/llvm/test/CodeGen/BPF/CORE/offset-reloc-struct-array.ll b/llvm/test/CodeGen/BPF/CORE/offset-reloc-struct-array.ll
index 77d23753d..3837e15dd 100644
--- a/llvm/test/CodeGen/BPF/CORE/offset-reloc-struct-array.ll
+++ b/llvm/test/CodeGen/BPF/CORE/offset-reloc-struct-array.ll
@@ -1,5 +1,7 @@
 ; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
 ; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfel -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
 ; Source code:
 ;   struct net_device {
 ;     int dev_id;
@@ -31,7 +33,7 @@ define dso_local i32 @bpf_prog(%struct.sk_buff*) local_unnamed_addr #0 !dbg !15
   %3 = bitcast i32* %2 to i8*, !dbg !34
   call void @llvm.lifetime.start.p0i8(i64 4, i8* nonnull %3) #4, !dbg !34
   %4 = tail call [10 x %struct.net_device]* @llvm.preserve.struct.access.index.p0a10s_struct.net_devices.p0s_struct.sk_buffs(%struct.sk_buff* %0, i32 1, i32 1), !dbg !35, !llvm.preserve.access.index !19
-  %5 = tail call %struct.net_device* @llvm.preserve.array.access.index.p0s_struct.net_devices.p0a10s_struct.net_devices([10 x %struct.net_device]* %4, i32 1, i32 5), !dbg !35
+  %5 = tail call %struct.net_device* @llvm.preserve.array.access.index.p0s_struct.net_devices.p0a10s_struct.net_devices([10 x %struct.net_device]* %4, i32 1, i32 5), !dbg !35, !llvm.preserve.access.index !23
   %6 = tail call i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.net_devices(%struct.net_device* %5, i32 0, i32 0), !dbg !35, !llvm.preserve.access.index !24
   %7 = bitcast i32* %6 to i8*, !dbg !35
   %8 = call i32 inttoptr (i64 4 to i32 (i8*, i32, i8*)*)(i8* nonnull %3, i32 4, i8* %7) #4, !dbg !36
@@ -91,7 +93,7 @@ define dso_local i32 @bpf_prog(%struct.sk_buff*) local_unnamed_addr #0 !dbg !15
 ; CHECK-NEXT:        .long   64
 ; CHECK-NEXT:        .long   1
 ; CHECK-NEXT:        .long   68                      # BTF_KIND_FUNC(id = 8)
-; CHECK-NEXT:        .long   201326592               # 0xc000000
+; CHECK-NEXT:        .long   201326593               # 0xc000001
 ; CHECK-NEXT:        .long   7
 ; CHECK-NEXT:        .byte   0                       # string offset=0
 ; CHECK-NEXT:        .ascii  "sk_buff"               # string offset=1
@@ -124,23 +126,22 @@ define dso_local i32 @bpf_prog(%struct.sk_buff*) local_unnamed_addr #0 !dbg !15
 ; CHECK-NEXT:        .short  60319                   # 0xeb9f
 ; CHECK-NEXT:        .byte   1
 ; CHECK-NEXT:        .byte   0
-; CHECK-NEXT:        .long   40
+; CHECK-NEXT:        .long   32
 ; CHECK-NEXT:        .long   0
 ; CHECK-NEXT:        .long   20
 ; CHECK-NEXT:        .long   20
-; CHECK-NEXT:        .long   76
-; CHECK-NEXT:        .long   96
-; CHECK-NEXT:        .long   24
-; CHECK-NEXT:        .long   120
-; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   {{[0-9]+}}
+; CHECK-NEXT:        .long   {{[0-9]+}}
+; CHECK-NEXT:        .long   28
 ; CHECK-NEXT:        .long   8                       # FuncInfo
 
-; CHECK:             .long   12                      # OffsetReloc
-; CHECK-NEXT:        .long   77                      # Offset reloc section string offset=77
+; CHECK:             .long   16                      # FieldReloc
+; CHECK-NEXT:        .long   77                      # Field reloc section string offset=77
 ; CHECK-NEXT:        .long   1
-; CHECK-NEXT:        .long   .Ltmp2
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
 ; CHECK-NEXT:        .long   2
 ; CHECK-NEXT:        .long   120
+; CHECK-NEXT:        .long   0
 
 ; Function Attrs: argmemonly nounwind
 declare void @llvm.lifetime.start.p0i8(i64 immarg, i8* nocapture) #1
@@ -160,7 +161,7 @@ declare void @llvm.lifetime.end.p0i8(i64 immarg, i8* nocapture) #1
 ; Function Attrs: nounwind readnone speculatable
 declare void @llvm.dbg.value(metadata, metadata, metadata) #3
 
-attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { argmemonly nounwind }
 attributes #2 = { nounwind readnone }
 attributes #3 = { nounwind readnone speculatable }
diff --git a/llvm/test/CodeGen/BPF/CORE/offset-reloc-typedef-array.ll b/llvm/test/CodeGen/BPF/CORE/offset-reloc-typedef-array.ll
index 7843c52c5..64931d9bd 100644
--- a/llvm/test/CodeGen/BPF/CORE/offset-reloc-typedef-array.ll
+++ b/llvm/test/CodeGen/BPF/CORE/offset-reloc-typedef-array.ll
@@ -1,5 +1,7 @@
 ; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck %s
 ; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck %s
+; RUN: llc -march=bpfel -mattr=+alu32 -filetype=asm -o - %s | FileCheck %s
+; RUN: llc -march=bpfeb -mattr=+alu32 -filetype=asm -o - %s | FileCheck %s
 ;
 ; Source code:
 ;   typedef const int arr_t[7];
@@ -21,7 +23,7 @@ define dso_local i32 @test(%struct.__s* %arg) local_unnamed_addr #0 !dbg !7 {
 entry:
   call void @llvm.dbg.value(metadata %struct.__s* %arg, metadata !24, metadata !DIExpression()), !dbg !25
   %0 = tail call [7 x i32]* @llvm.preserve.struct.access.index.p0a7i32.p0s_struct.__ss(%struct.__s* %arg, i32 0, i32 0), !dbg !26, !llvm.preserve.access.index !13
-  %1 = tail call i32* @llvm.preserve.array.access.index.p0i32.p0a7i32([7 x i32]* %0, i32 1, i32 1), !dbg !26
+  %1 = tail call i32* @llvm.preserve.array.access.index.p0i32.p0a7i32([7 x i32]* %0, i32 1, i32 1), !dbg !26, !llvm.preserve.access.index !19
   %2 = bitcast i32* %1 to i8*, !dbg !26
   %call = tail call i32 @get_value(i8* %2) #4, !dbg !27
   ret i32 %call, !dbg !28
@@ -38,12 +40,13 @@ entry:
 ; CHECK-NEXT:    .byte   0
 ; CHECK:         .ascii  "0:0:1"                 # string offset=[[ACCESS_STR:[0-9]+]]
 ; CHECK-NEXT:    .byte   0
-; CHECK:         .long   12                      # OffsetReloc
-; CHECK-NEXT:    .long   [[SEC_INDEX]]           # Offset reloc section string offset=[[SEC_INDEX]]
+; CHECK:         .long   16                      # FieldReloc
+; CHECK-NEXT:    .long   [[SEC_INDEX]]           # Field reloc section string offset=[[SEC_INDEX]]
 ; CHECK-NEXT:    .long   1
 ; CHECK-NEXT:    .long   [[RELOC]]
 ; CHECK-NEXT:    .long   [[TYPE_ID]]
 ; CHECK-NEXT:    .long   [[ACCESS_STR]]
+; CHECK-NEXT:    .long   0
 
 declare dso_local i32 @get_value(i8*) local_unnamed_addr #1
 
diff --git a/llvm/test/CodeGen/BPF/CORE/offset-reloc-typedef-struct.ll b/llvm/test/CodeGen/BPF/CORE/offset-reloc-typedef-struct.ll
index a0d7532bf..7f676d84d 100644
--- a/llvm/test/CodeGen/BPF/CORE/offset-reloc-typedef-struct.ll
+++ b/llvm/test/CodeGen/BPF/CORE/offset-reloc-typedef-struct.ll
@@ -1,5 +1,7 @@
 ; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck %s
 ; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck %s
+; RUN: llc -march=bpfel -mattr=+alu32 -filetype=asm -o - %s | FileCheck %s
+; RUN: llc -march=bpfeb -mattr=+alu32 -filetype=asm -o - %s | FileCheck %s
 ;
 ; Source code:
 ;   typedef int _int;
@@ -37,12 +39,13 @@ entry:
 ; CHECK-NEXT:   .byte   0
 ; CHECK:        .ascii  "0:1"                   # string offset=[[ACCESS_STR:[0-9]+]]
 ; CHECK-NEXT:   .byte   0
-; CHECK:        .long   12                      # OffsetReloc
-; CHECK-NEXT:   .long   [[SEC_STR]]             # Offset reloc section string offset={{[0-9]+}}
+; CHECK:        .long   16                      # FieldReloc
+; CHECK-NEXT:   .long   [[SEC_STR]]             # Field reloc section string offset={{[0-9]+}}
 ; CHECK-NEXT:   .long   1
 ; CHECK-NEXT:   .long   [[RELOC]]
 ; CHECK-NEXT:   .long   [[TYPE_ID]]
 ; CHECK-NEXT:   .long   [[ACCESS_STR]]
+; CHECK-NEXT:   .long   0
 
 declare dso_local i32 @get_value(i8*) local_unnamed_addr #1
 
diff --git a/llvm/test/CodeGen/BPF/CORE/offset-reloc-typedef-union.ll b/llvm/test/CodeGen/BPF/CORE/offset-reloc-typedef-union.ll
index caefc2b4b..265390712 100644
--- a/llvm/test/CodeGen/BPF/CORE/offset-reloc-typedef-union.ll
+++ b/llvm/test/CodeGen/BPF/CORE/offset-reloc-typedef-union.ll
@@ -1,5 +1,7 @@
 ; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck %s
 ; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck %s
+; RUN: llc -march=bpfel -mattr=+alu32 -filetype=asm -o - %s | FileCheck %s
+; RUN: llc -march=bpfeb -mattr=+alu32 -filetype=asm -o - %s | FileCheck %s
 ;
 ; Source code:
 ;   typedef int _int;
@@ -37,12 +39,13 @@ entry:
 ; CHECK-NEXT:    .byte   0
 ; CHECK:         .ascii  "0:1"                   # string offset=[[ACCESS_STR:[0-9]+]]
 ; CHECK-NEXT:    .byte   0
-; CHECK:         .long   12                      # OffsetReloc
-; CHECK-NEXT:    .long   [[SEC_INDEX]]           # Offset reloc section string offset=[[SEC_INDEX]]
+; CHECK:         .long   16                      # FieldReloc
+; CHECK-NEXT:    .long   [[SEC_INDEX]]           # Field reloc section string offset=[[SEC_INDEX]]
 ; CHECK-NEXT:    .long   1
 ; CHECK-NEXT:    .long   [[RELOC]]
 ; CHECK-NEXT:    .long   [[TYPE_ID]]
 ; CHECK-NEXT:    .long   [[ACCESS_STR]]
+; CHECK-NEXT:    .long   0
 
 declare dso_local i32 @get_value(i8*) local_unnamed_addr #1
 
diff --git a/llvm/test/CodeGen/BPF/CORE/offset-reloc-typedef.ll b/llvm/test/CodeGen/BPF/CORE/offset-reloc-typedef.ll
index c09d97991..6c581bd32 100644
--- a/llvm/test/CodeGen/BPF/CORE/offset-reloc-typedef.ll
+++ b/llvm/test/CodeGen/BPF/CORE/offset-reloc-typedef.ll
@@ -1,5 +1,7 @@
 ; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck %s
 ; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck %s
+; RUN: llc -march=bpfel -mattr=+alu32 -filetype=asm -o - %s | FileCheck %s
+; RUN: llc -march=bpfeb -mattr=+alu32 -filetype=asm -o - %s | FileCheck %s
 ;
 ; Source code:
 ;   struct s { int a; int b; };
@@ -24,7 +26,7 @@
 define dso_local i32 @test([7 x %union.u]* %arg) local_unnamed_addr #0 !dbg !7 {
 entry:
   call void @llvm.dbg.value(metadata [7 x %union.u]* %arg, metadata !28, metadata !DIExpression()), !dbg !29
-  %0 = tail call [7 x %union.u]* @llvm.preserve.array.access.index.p0a7s_union.us.p0a7s_union.us([7 x %union.u]* %arg, i32 0, i32 1), !dbg !30
+  %0 = tail call [7 x %union.u]* @llvm.preserve.array.access.index.p0a7s_union.us.p0a7s_union.us([7 x %union.u]* %arg, i32 0, i32 1), !dbg !30, !llvm.preserve.access.index !14
   %arraydecay = getelementptr inbounds [7 x %union.u], [7 x %union.u]* %0, i64 0, i64 0, !dbg !30
   %1 = tail call %union.u* @llvm.preserve.union.access.index.p0s_union.us.p0s_union.us(%union.u* %arraydecay, i32 1), !dbg !30, !llvm.preserve.access.index !16
   %d = getelementptr inbounds %union.u, %union.u* %1, i64 0, i32 0, !dbg !30
@@ -45,12 +47,13 @@ entry:
 ; CHECK-NEXT:    .byte   0
 ; CHECK:         .ascii  "1:1:1"                 # string offset=[[ACCESS_STR:[0-9]+]]
 ; CHECK-NEXT:    .byte   0
-; CHECK:         .long   12                      # OffsetReloc
-; CHECK-NEXT:    .long   [[SEC_STR:[0-9]+]]      # Offset reloc section string offset=[[SEC_STR:[0-9]+]]
+; CHECK:         .long   16                      # FieldReloc
+; CHECK-NEXT:    .long   [[SEC_STR:[0-9]+]]      # Field reloc section string offset=[[SEC_STR:[0-9]+]]
 ; CHECK-NEXT:    .long   1
 ; CHECK-NEXT:    .long   [[RELOC:.Ltmp[0-9]+]]
 ; CHECK-NEXT:    .long   [[TYPE_ID:[0-9]+]]
 ; CHECK-NEXT:    .long   [[ACCESS_STR:[0-9]+]]
+; CHECK-NEXT:    .long   0
 
 declare dso_local i32 @get_value(i8*) local_unnamed_addr #1
 
diff --git a/llvm/test/CodeGen/BPF/CORE/offset-reloc-union.ll b/llvm/test/CodeGen/BPF/CORE/offset-reloc-union.ll
index eefa1ebda..d0711f65a 100644
--- a/llvm/test/CodeGen/BPF/CORE/offset-reloc-union.ll
+++ b/llvm/test/CodeGen/BPF/CORE/offset-reloc-union.ll
@@ -1,5 +1,7 @@
 ; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
 ; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfel -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
 ; Source code:
 ;   union sk_buff {
 ;     int i;
@@ -94,7 +96,7 @@ define dso_local i32 @bpf_prog(%union.sk_buff*) local_unnamed_addr #0 !dbg !15 {
 ; CHECK-NEXT:        .long   41
 ; CHECK-NEXT:        .long   1
 ; CHECK-NEXT:        .long   45                      # BTF_KIND_FUNC(id = 7)
-; CHECK-NEXT:        .long   201326592               # 0xc000000
+; CHECK-NEXT:        .long   201326593               # 0xc000001
 ; CHECK-NEXT:        .long   6
 ; CHECK-NEXT:        .byte   0                       # string offset=0
 ; CHECK-NEXT:        .ascii  "sk_buff"               # string offset=1
@@ -127,23 +129,22 @@ define dso_local i32 @bpf_prog(%union.sk_buff*) local_unnamed_addr #0 !dbg !15 {
 ; CHECK-NEXT:        .short  60319                   # 0xeb9f
 ; CHECK-NEXT:        .byte   1
 ; CHECK-NEXT:        .byte   0
-; CHECK-NEXT:        .long   40
+; CHECK-NEXT:        .long   32
 ; CHECK-NEXT:        .long   0
 ; CHECK-NEXT:        .long   20
 ; CHECK-NEXT:        .long   20
-; CHECK-NEXT:        .long   76
-; CHECK-NEXT:        .long   96
-; CHECK-NEXT:        .long   24
-; CHECK-NEXT:        .long   120
-; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   {{[0-9]+}}
+; CHECK-NEXT:        .long   {{[0-9]+}}
+; CHECK-NEXT:        .long   28
 ; CHECK-NEXT:        .long   8                       # FuncInfo
 
-; CHECK:             .long   12                      # OffsetReloc
-; CHECK-NEXT:        .long   54                      # Offset reloc section string offset=54
+; CHECK:             .long   16                      # FieldReloc
+; CHECK-NEXT:        .long   54                      # Field reloc section string offset=54
 ; CHECK-NEXT:        .long   1
-; CHECK-NEXT:        .long   .Ltmp2
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
 ; CHECK-NEXT:        .long   2
 ; CHECK-NEXT:        .long   97
+; CHECK-NEXT:        .long   0
 
 ; Function Attrs: argmemonly nounwind
 declare void @llvm.lifetime.start.p0i8(i64 immarg, i8* nocapture) #1
@@ -163,7 +164,7 @@ declare void @llvm.lifetime.end.p0i8(i64 immarg, i8* nocapture) #1
 ; Function Attrs: nounwind readnone speculatable
 declare void @llvm.dbg.value(metadata, metadata, metadata) #3
 
-attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { argmemonly nounwind }
 attributes #2 = { nounwind readnone }
 attributes #3 = { nounwind readnone speculatable }
diff --git a/llvm/test/CodeGen/BPF/callx.ll b/llvm/test/CodeGen/BPF/callx.ll
new file mode 100644
index 000000000..bb31e1189
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/callx.ll
@@ -0,0 +1,20 @@
+; RUN: llc < %s -march=bpfel | FileCheck %s
+; source:
+;   int test(int (*f)(void)) { return f(); }
+
+; Function Attrs: nounwind
+define dso_local i32 @test(i32 ()* nocapture %f) local_unnamed_addr #0 {
+entry:
+  %call = tail call i32 %f() #1
+; CHECK: callx r{{[0-9]+}}
+  ret i32 %call
+}
+
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind }
+
+!llvm.module.flags = !{!0}
+!llvm.ident = !{!1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{!"clang version 10.0.0 (https://github.com/llvm/llvm-project.git 7015a5c54b53d8d2297a3aa38bc32aab167bdcfc)"}
diff --git a/llvm/test/CodeGen/BPF/dwarfdump.ll b/llvm/test/CodeGen/BPF/dwarfdump.ll
index 8b1186b08..c8215fdac 100644
--- a/llvm/test/CodeGen/BPF/dwarfdump.ll
+++ b/llvm/test/CodeGen/BPF/dwarfdump.ll
@@ -23,7 +23,7 @@ define i32 @testprog(i32, i32) local_unnamed_addr #0 !dbg !2 {
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.value(metadata, i64, metadata, metadata) #1
 
-attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!7}
diff --git a/llvm/test/CodeGen/BPF/objdump_two_funcs.ll b/llvm/test/CodeGen/BPF/objdump_two_funcs.ll
index 689cf6f70..d129b3fa1 100644
--- a/llvm/test/CodeGen/BPF/objdump_two_funcs.ll
+++ b/llvm/test/CodeGen/BPF/objdump_two_funcs.ll
@@ -37,7 +37,7 @@ entry:
 ; Function Attrs: nounwind readnone speculatable
 declare void @llvm.dbg.value(metadata, metadata, metadata) #1
 
-attributes #0 = { norecurse nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { norecurse nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone speculatable }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/CodeGen/BPF/optnone-1.ll b/llvm/test/CodeGen/BPF/optnone-1.ll
new file mode 100644
index 000000000..b38e2630f
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/optnone-1.ll
@@ -0,0 +1,52 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; Source Code:
+;   int test(int a, int b) { return a + b; }
+; Compilation flag:
+;   clang -target bpf -g -S -emit-llvm test.c
+
+; Function Attrs: noinline nounwind optnone
+define dso_local i32 @test(i32 %a, i32 %b) #0 !dbg !7 {
+entry:
+  %a.addr = alloca i32, align 4
+  %b.addr = alloca i32, align 4
+  store i32 %a, i32* %a.addr, align 4
+  call void @llvm.dbg.declare(metadata i32* %a.addr, metadata !11, metadata !DIExpression()), !dbg !12
+  store i32 %b, i32* %b.addr, align 4
+  call void @llvm.dbg.declare(metadata i32* %b.addr, metadata !13, metadata !DIExpression()), !dbg !14
+  %0 = load i32, i32* %a.addr, align 4, !dbg !15
+  %1 = load i32, i32* %b.addr, align 4, !dbg !16
+  %add = add nsw i32 %0, %1, !dbg !17
+  ret i32 %add, !dbg !18
+}
+
+; CHECK-LABEL: test
+
+; Function Attrs: nounwind readnone speculatable
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
+
+attributes #0 = { noinline nounwind optnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone speculatable}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 11.0.0 (https://github.com/llvm/llvm-project.git 7f49dc496682e38335b204bbae371a0ca5057719)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "test.c", directory: "/tmp/home/yhs/work/tests/llvm")
+!2 = !{}
+!3 = !{i32 7, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{!"clang version 11.0.0 (https://github.com/llvm/llvm-project.git 7f49dc496682e38335b204bbae371a0ca5057719)"}
+!7 = distinct !DISubprogram(name: "test", scope: !1, file: !1, line: 1, type: !8, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !2)
+!8 = !DISubroutineType(types: !9)
+!9 = !{!10, !10, !10}
+!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!11 = !DILocalVariable(name: "a", arg: 1, scope: !7, file: !1, line: 1, type: !10)
+!12 = !DILocation(line: 1, column: 14, scope: !7)
+!13 = !DILocalVariable(name: "b", arg: 2, scope: !7, file: !1, line: 1, type: !10)
+!14 = !DILocation(line: 1, column: 21, scope: !7)
+!15 = !DILocation(line: 1, column: 33, scope: !7)
+!16 = !DILocation(line: 1, column: 37, scope: !7)
+!17 = !DILocation(line: 1, column: 35, scope: !7)
+!18 = !DILocation(line: 1, column: 26, scope: !7)
diff --git a/llvm/test/CodeGen/BPF/reloc-btf-2.ll b/llvm/test/CodeGen/BPF/reloc-btf-2.ll
index 2871daa38..2afeb24ba 100644
--- a/llvm/test/CodeGen/BPF/reloc-btf-2.ll
+++ b/llvm/test/CodeGen/BPF/reloc-btf-2.ll
@@ -27,7 +27,7 @@ define dso_local i32 @test() local_unnamed_addr #0 !dbg !14 {
 ; CHECK-RELOC: R_BPF_NONE g
 ; CHECK-RELOC: RELOCATION RECORDS FOR [.BTF.ext]:
 
-attributes #0 = { norecurse nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { norecurse nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!2}
 !llvm.module.flags = !{!10, !11, !12}
diff --git a/llvm/test/CodeGen/BPF/reloc-btf.ll b/llvm/test/CodeGen/BPF/reloc-btf.ll
index 8dbd2eff7..484ef3336 100644
--- a/llvm/test/CodeGen/BPF/reloc-btf.ll
+++ b/llvm/test/CodeGen/BPF/reloc-btf.ll
@@ -13,7 +13,7 @@ entry:
 ; CHECK-RELOC: RELOCATION RECORDS FOR [.BTF.ext]:
 ; CHECK-RELOC: R_BPF_NONE .text
 
-attributes #0 = { norecurse nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { norecurse nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!3, !4, !5}
diff --git a/llvm/test/CodeGen/BPF/remove_truncate_6.ll b/llvm/test/CodeGen/BPF/remove_truncate_6.ll
new file mode 100644
index 000000000..6577afb13
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/remove_truncate_6.ll
@@ -0,0 +1,80 @@
+; RUN: llc < %s -march=bpf -verify-machineinstrs | FileCheck %s
+; RUN: llc < %s -march=bpf -mattr=+alu32 -verify-machineinstrs | FileCheck --check-prefix=CHECK-32 %s
+;
+; void cal1(unsigned short *a, unsigned long *b, unsigned int k)
+; {
+;   unsigned short e;
+;
+;   e = *a;
+;   for (unsigned int i = 0; i < k; i++) {
+;     b[i] = e;
+;     e = ~e;
+;   }
+; }
+;
+; void cal2(unsigned short *a, unsigned int *b, unsigned int k)
+; {
+;   unsigned short e;
+;
+;   e = *a;
+;   for (unsigned int i = 0; i < k; i++) {
+;     b[i] = e;
+;     e = ~e;
+;   }
+; }
+
+; Function Attrs: nofree norecurse nounwind optsize
+define dso_local void @cal1(i16* nocapture readonly %a, i64* nocapture %b, i32 %k) local_unnamed_addr #0 {
+entry:
+  %cmp8 = icmp eq i32 %k, 0
+  br i1 %cmp8, label %for.cond.cleanup, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  %0 = load i16, i16* %a, align 2
+  %wide.trip.count = zext i32 %k to i64
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body, %entry
+  ret void
+
+for.body:                                         ; preds = %for.body, %for.body.preheader
+  %indvars.iv = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
+  %e.09 = phi i16 [ %0, %for.body.preheader ], [ %neg, %for.body ]
+  %conv = zext i16 %e.09 to i64
+  %arrayidx = getelementptr inbounds i64, i64* %b, i64 %indvars.iv
+; CHECK: r{{[0-9]+}} &= 65535
+; CHECK-32: r{{[0-9]+}} &= 65535
+  store i64 %conv, i64* %arrayidx, align 8
+  %neg = xor i16 %e.09, -1
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+; Function Attrs: nofree norecurse nounwind optsize
+define dso_local void @cal2(i16* nocapture readonly %a, i32* nocapture %b, i32 %k) local_unnamed_addr #0 {
+entry:
+  %cmp8 = icmp eq i32 %k, 0
+  br i1 %cmp8, label %for.cond.cleanup, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  %0 = load i16, i16* %a, align 2
+  %wide.trip.count = zext i32 %k to i64
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body, %entry
+  ret void
+
+for.body:                                         ; preds = %for.body, %for.body.preheader
+  %indvars.iv = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
+  %e.09 = phi i16 [ %0, %for.body.preheader ], [ %neg, %for.body ]
+  %conv = zext i16 %e.09 to i32
+  %arrayidx = getelementptr inbounds i32, i32* %b, i64 %indvars.iv
+; CHECK: r{{[0-9]+}} &= 65535
+; CHECK-32: w{{[0-9]+}} &= 65535
+  store i32 %conv, i32* %arrayidx, align 4
+  %neg = xor i16 %e.09, -1
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
diff --git a/llvm/test/CodeGen/BPF/shifts.ll b/llvm/test/CodeGen/BPF/shifts.ll
index 2d73ffddf..2ffcb8891 100644
--- a/llvm/test/CodeGen/BPF/shifts.ll
+++ b/llvm/test/CodeGen/BPF/shifts.ll
@@ -52,7 +52,7 @@ define zeroext i32 @lshr32(i32 zeroext %a, i32 zeroext %cnt) nounwind readnone {
 entry:
 ; CHECK-LABEL: lshr32:
 ; CHECK: r0 >>= r2 # encoding: [0x7f,0x20,0x00,0x00,0x00,0x00,0x00,0x00]
-; CHECK: r0 <<= 32 # encoding: [0x67,0x00,0x00,0x00,0x20,0x00,0x00,0x00]
+; CHECK: exit # encoding: [0x95,0x00,0x00,0x00,0x00,0x00,0x00,0x00]
   %shr = lshr i32 %a, %cnt
   ret i32 %shr
 }
diff --git a/llvm/test/CodeGen/BPF/warn-stack.ll b/llvm/test/CodeGen/BPF/warn-stack.ll
index 217a7d50d..0bf72780d 100644
--- a/llvm/test/CodeGen/BPF/warn-stack.ll
+++ b/llvm/test/CodeGen/BPF/warn-stack.ll
@@ -34,10 +34,10 @@ define void @warn() local_unnamed_addr #0 !dbg !20 {
   ret void, !dbg !29
 }
 
-attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { argmemonly nounwind }
 attributes #2 = { nounwind readnone }
-attributes #3 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #4 = { nounwind }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/CodeGen/BPF/xadd.ll b/llvm/test/CodeGen/BPF/xadd.ll
index 49f0c10c3..0bf8576df 100644
--- a/llvm/test/CodeGen/BPF/xadd.ll
+++ b/llvm/test/CodeGen/BPF/xadd.ll
@@ -32,7 +32,7 @@ entry:
 ; Function Attrs: nounwind readnone speculatable
 declare void @llvm.dbg.value(metadata, metadata, metadata) #1
 
-attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone speculatable }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/CodeGen/Generic/DbgValueAggregate.ll b/llvm/test/CodeGen/Generic/DbgValueAggregate.ll
new file mode 100644
index 000000000..e68322558
--- /dev/null
+++ b/llvm/test/CodeGen/Generic/DbgValueAggregate.ll
@@ -0,0 +1,36 @@
+; RUN: llc -O0 -global-isel < %s | FileCheck %s
+; REQUIRES: aarch64
+target triple = "aarch64-unknown-linux-gnu"
+
+define void @MAIN_() #0 {
+L.entry:
+  %0 = load <{ float, float }>, <{ float, float }>* undef, align 1
+  ; CHECK: DEBUG_VALUE: localvar
+  ; CHECK: DEBUG_VALUE: localvar
+  call void @llvm.dbg.value(metadata <{ float, float }> %0, metadata !10, metadata !DIExpression()), !dbg !13
+  unreachable
+}
+
+; Function Attrs: nounwind readnone speculatable
+declare void @llvm.dbg.value(metadata, metadata, metadata) #1
+
+attributes #0 = { "frame-pointer"="non-leaf" }
+attributes #1 = { nounwind readnone speculatable }
+
+!llvm.module.flags = !{!0}
+!llvm.dbg.cu = !{!1}
+
+!0 = !{i32 2, !"Debug Info Version", i32 3}
+!1 = distinct !DICompileUnit(language: DW_LANG_Fortran90, file: !2, producer: "", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !3, retainedTypes: !3, globals: !3, imports: !4)
+!2 = !DIFile(filename: "input", directory: "/")
+!3 = !{}
+!4 = !{!5}
+!5 = !DIImportedEntity(tag: DW_TAG_imported_module, scope: !6, entity: !9, file: !2, line: 18)
+!6 = distinct !DISubprogram(name: "p", scope: !1, file: !2, line: 18, type: !7, isLocal: false, isDefinition: true, scopeLine: 18, isOptimized: false, unit: !1)
+!7 = !DISubroutineType(cc: DW_CC_program, types: !8)
+!8 = !{null}
+!9 = !DIModule(scope: !1, name: "mod")
+!10 = !DILocalVariable(name: "localvar", scope: !11, file: !2, type: !12)
+!11 = !DILexicalBlock(scope: !6, file: !2, line: 18, column: 1)
+!12 = !DIBasicType(name: "complex", size: 64, align: 32, encoding: DW_ATE_complex_float)
+!13 = !DILocation(line: 0, scope: !11)
diff --git a/llvm/test/CodeGen/Generic/cfi-sections.ll b/llvm/test/CodeGen/Generic/cfi-sections.ll
index bd3b175ad..eb21fb5b0 100644
--- a/llvm/test/CodeGen/Generic/cfi-sections.ll
+++ b/llvm/test/CodeGen/Generic/cfi-sections.ll
@@ -20,8 +20,8 @@ define i32 @foo() #0 !dbg !7 {
 
 declare i32 @bar() #1
 
-attributes #0 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="arm7tdmi" "target-features"="+soft-float,+strict-align,-crypto,-neon" "unsafe-fp-math"="false" "use-soft-float"="true" }
-attributes #1 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="arm7tdmi" "target-features"="+soft-float,+strict-align,-crypto,-neon" "unsafe-fp-math"="false" "use-soft-float"="true" }
+attributes #0 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="arm7tdmi" "target-features"="+soft-float,+strict-align,-crypto,-neon" "unsafe-fp-math"="false" "use-soft-float"="true" }
+attributes #1 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="arm7tdmi" "target-features"="+soft-float,+strict-align,-crypto,-neon" "unsafe-fp-math"="false" "use-soft-float"="true" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!3, !4, !5, !6}
diff --git a/llvm/test/CodeGen/Generic/expand-experimental-reductions.ll b/llvm/test/CodeGen/Generic/expand-experimental-reductions.ll
index a13c23d2e..11abf902e 100644
--- a/llvm/test/CodeGen/Generic/expand-experimental-reductions.ll
+++ b/llvm/test/CodeGen/Generic/expand-experimental-reductions.ll
@@ -18,6 +18,7 @@ declare i64 @llvm.experimental.vector.reduce.umin.v2i64(<2 x i64>)
 declare double @llvm.experimental.vector.reduce.fmax.v2f64(<2 x double>)
 declare double @llvm.experimental.vector.reduce.fmin.v2f64(<2 x double>)
 
+declare i8 @llvm.experimental.vector.reduce.and.i8.v3i8(<3 x i8>)
 
 define i64 @add_i64(<2 x i64> %vec) {
 ; CHECK-LABEL: @add_i64(
@@ -303,3 +304,15 @@ entry:
   %r = call double @llvm.experimental.vector.reduce.fmin.v2f64(<2 x double> %vec)
   ret double %r
 }
+
+; Test when the vector size is not power of two.
+define i8 @test_v3i8(<3 x i8> %a) nounwind {
+; CHECK-LABEL: @test_v3i8(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %b = call i8 @llvm.experimental.vector.reduce.and.v3i8(<3 x i8> %a)
+; CHECK-NEXT:    ret i8 %b
+;
+entry:
+  %b = call i8 @llvm.experimental.vector.reduce.and.i8.v3i8(<3 x i8> %a)
+  ret i8 %b
+}
diff --git a/llvm/test/CodeGen/Hexagon/64bit_tstbit.ll b/llvm/test/CodeGen/Hexagon/64bit_tstbit.ll
new file mode 100644
index 000000000..b265d1bef
--- /dev/null
+++ b/llvm/test/CodeGen/Hexagon/64bit_tstbit.ll
@@ -0,0 +1,34 @@
+; RUN: llc -march=hexagon  < %s | FileCheck %s
+
+; This test checks that S2_tstbit_i instruction is generated
+; and it does not assert.
+
+; CHECK: p{{[0-9]+}} = tstbit
+
+
+target triple = "hexagon-unknown-unknown-elf"
+
+%struct.hlist_node.45.966.3115.3729.4036.4650.4957.6492.6799.7413.7720.9562.10790.11097.11404.11711.14474.17192 = type { %struct.hlist_node.45.966.3115.3729.4036.4650.4957.6492.6799.7413.7720.9562.10790.11097.11404.11711.14474.17192*, %struct.hlist_node.45.966.3115.3729.4036.4650.4957.6492.6799.7413.7720.9562.10790.11097.11404.11711.14474.17192** }
+
+@.str.8 = external dso_local unnamed_addr constant [5 x i8], align 1
+
+declare dso_local void @panic(i8*, ...) local_unnamed_addr
+
+define dso_local fastcc void @elv_rqhash_find() unnamed_addr {
+entry:
+  %cmd_flags = getelementptr inbounds %struct.hlist_node.45.966.3115.3729.4036.4650.4957.6492.6799.7413.7720.9562.10790.11097.11404.11711.14474.17192, %struct.hlist_node.45.966.3115.3729.4036.4650.4957.6492.6799.7413.7720.9562.10790.11097.11404.11711.14474.17192* null, i32 -5
+  %0 = bitcast %struct.hlist_node.45.966.3115.3729.4036.4650.4957.6492.6799.7413.7720.9562.10790.11097.11404.11711.14474.17192* %cmd_flags to i64*
+  %1 = load i64, i64* %0, align 8
+  %2 = and i64 %1, 4294967296
+  %tobool10 = icmp eq i64 %2, 0
+  br i1 %tobool10, label %do.body11, label %do.end14
+
+do.body11:                                        ; preds = %entry
+  tail call void (i8*, ...) @panic(i8* getelementptr inbounds ([5 x i8], [5 x i8]* @.str.8, i32 0, i32 0)) #1
+  unreachable
+
+do.end14:                                         ; preds = %entry
+  %and.i = and i64 %1, -4294967297
+  store i64 %and.i, i64* %0, align 8
+  ret void
+}
diff --git a/llvm/test/CodeGen/Hexagon/NVJumpCmp.ll b/llvm/test/CodeGen/Hexagon/NVJumpCmp.ll
index 6b160d962..57d1824fa 100644
--- a/llvm/test/CodeGen/Hexagon/NVJumpCmp.ll
+++ b/llvm/test/CodeGen/Hexagon/NVJumpCmp.ll
@@ -75,7 +75,7 @@ if.else:                                          ; preds = %entry
   unreachable
 }
 
-attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.module.flags = !{!0}
 
diff --git a/llvm/test/CodeGen/Hexagon/addh-sext-trunc.ll b/llvm/test/CodeGen/Hexagon/addh-sext-trunc.ll
index ec5dc6111..821ed011f 100644
--- a/llvm/test/CodeGen/Hexagon/addh-sext-trunc.ll
+++ b/llvm/test/CodeGen/Hexagon/addh-sext-trunc.ll
@@ -15,7 +15,7 @@ define i32 @foo(i16 %a, i32 %b) #0 {
 }
 
 
-attributes #0 = { nounwind readonly "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf"="true" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind readonly "less-precise-fpmad"="false" "frame-pointer"="non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !0 = !{!"short", !1}
 !1 = !{!"omnipotent char", !2}
diff --git a/llvm/test/CodeGen/Hexagon/alu64.ll b/llvm/test/CodeGen/Hexagon/alu64.ll
index 453b40a6e..49e06d2da 100644
--- a/llvm/test/CodeGen/Hexagon/alu64.ll
+++ b/llvm/test/CodeGen/Hexagon/alu64.ll
@@ -595,5 +595,5 @@ declare i32 @llvm.hexagon.S4.ori.lsr.ri(i32, i32, i32) #1
 declare i32 @llvm.hexagon.S4.addi.lsr.ri(i32, i32, i32) #1
 declare i32 @llvm.hexagon.S4.subi.lsr.ri(i32, i32, i32) #1
 
-attributes #0 = { nounwind readnone "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind readnone "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
diff --git a/llvm/test/CodeGen/Hexagon/autohvx/isel-setcc-v256i1.ll b/llvm/test/CodeGen/Hexagon/autohvx/isel-setcc-v256i1.ll
new file mode 100644
index 000000000..b0fbb1cea
--- /dev/null
+++ b/llvm/test/CodeGen/Hexagon/autohvx/isel-setcc-v256i1.ll
@@ -0,0 +1,15 @@
+; RUN: llc -march=hexagon < %s | FileCheck %s
+
+; Check that this doesn't crash. The select should be broken up into two
+; vmux instructions.
+
+; CHECK-LABEL: foo:
+; CHECK: vmux
+; CHECK: vmux
+define <256 x i8> @foo(<256 x i8> %a0, <256 x i8> %a1) #0 {
+  %v0 = icmp slt <256 x i8> %a0, zeroinitializer
+  %v1 = select <256 x i1> %v0, <256 x i8> %a1, <256 x i8> %a0
+  ret <256 x i8> %v1
+}
+
+attributes #0 = { "target-cpu"="hexagonv62" "target-features"="+hvx,+hvx-length128b" }
diff --git a/llvm/test/CodeGen/Hexagon/autohvx/minmax-128b.ll b/llvm/test/CodeGen/Hexagon/autohvx/minmax-128b.ll
new file mode 100644
index 000000000..bd5ca331e
--- /dev/null
+++ b/llvm/test/CodeGen/Hexagon/autohvx/minmax-128b.ll
@@ -0,0 +1,344 @@
+; RUN: llc -march=hexagon < %s | FileCheck %s
+
+; minb
+
+; CHECK: test_00:
+; CHECK: v0.b = vmin(v1.b,v0.b)
+define <128 x i8> @test_00(<128 x i8> %v0, <128 x i8> %v1) #0 {
+  %t0 = icmp slt <128 x i8> %v0, %v1
+  %t1 = select <128 x i1> %t0, <128 x i8> %v0, <128 x i8> %v1
+  ret <128 x i8> %t1
+}
+
+; CHECK: test_01:
+; CHECK: v0.b = vmin(v0.b,v1.b)
+define <128 x i8> @test_01(<128 x i8> %v0, <128 x i8> %v1) #0 {
+  %t0 = icmp sle <128 x i8> %v0, %v1
+  %t1 = select <128 x i1> %t0, <128 x i8> %v0, <128 x i8> %v1
+  ret <128 x i8> %t1
+}
+
+; CHECK: test_02:
+; CHECK: v0.b = vmin(v0.b,v1.b)
+define <128 x i8> @test_02(<128 x i8> %v0, <128 x i8> %v1) #0 {
+  %t0 = icmp sgt <128 x i8> %v0, %v1
+  %t1 = select <128 x i1> %t0, <128 x i8> %v1, <128 x i8> %v0
+  ret <128 x i8> %t1
+}
+
+; CHECK: test_03:
+; CHECK: v0.b = vmin(v1.b,v0.b)
+define <128 x i8> @test_03(<128 x i8> %v0, <128 x i8> %v1) #0 {
+  %t0 = icmp sge <128 x i8> %v0, %v1
+  %t1 = select <128 x i1> %t0, <128 x i8> %v1, <128 x i8> %v0
+  ret <128 x i8> %t1
+}
+
+; maxb
+
+; CHECK: test_04:
+; CHECK: v0.b = vmax(v1.b,v0.b)
+define <128 x i8> @test_04(<128 x i8> %v0, <128 x i8> %v1) #0 {
+  %t0 = icmp slt <128 x i8> %v0, %v1
+  %t1 = select <128 x i1> %t0, <128 x i8> %v1, <128 x i8> %v0
+  ret <128 x i8> %t1
+}
+
+; CHECK: test_05:
+; CHECK: v0.b = vmax(v0.b,v1.b)
+define <128 x i8> @test_05(<128 x i8> %v0, <128 x i8> %v1) #0 {
+  %t0 = icmp sle <128 x i8> %v0, %v1
+  %t1 = select <128 x i1> %t0, <128 x i8> %v1, <128 x i8> %v0
+  ret <128 x i8> %t1
+}
+
+; CHECK: test_06:
+; CHECK: v0.b = vmax(v0.b,v1.b)
+define <128 x i8> @test_06(<128 x i8> %v0, <128 x i8> %v1) #0 {
+  %t0 = icmp sgt <128 x i8> %v0, %v1
+  %t1 = select <128 x i1> %t0, <128 x i8> %v0, <128 x i8> %v1
+  ret <128 x i8> %t1
+}
+
+; CHECK: test_07:
+; CHECK: v0.b = vmax(v1.b,v0.b)
+define <128 x i8> @test_07(<128 x i8> %v0, <128 x i8> %v1) #0 {
+  %t0 = icmp sge <128 x i8> %v0, %v1
+  %t1 = select <128 x i1> %t0, <128 x i8> %v0, <128 x i8> %v1
+  ret <128 x i8> %t1
+}
+
+; minub
+
+; CHECK: test_08:
+; CHECK: v0.ub = vmin(v1.ub,v0.ub)
+define <128 x i8> @test_08(<128 x i8> %v0, <128 x i8> %v1) #0 {
+  %t0 = icmp ult <128 x i8> %v0, %v1
+  %t1 = select <128 x i1> %t0, <128 x i8> %v0, <128 x i8> %v1
+  ret <128 x i8> %t1
+}
+
+; CHECK: test_09:
+; CHECK: v0.ub = vmin(v0.ub,v1.ub)
+define <128 x i8> @test_09(<128 x i8> %v0, <128 x i8> %v1) #0 {
+  %t0 = icmp ule <128 x i8> %v0, %v1
+  %t1 = select <128 x i1> %t0, <128 x i8> %v0, <128 x i8> %v1
+  ret <128 x i8> %t1
+}
+
+; CHECK: test_0a:
+; CHECK: v0.ub = vmin(v0.ub,v1.ub)
+define <128 x i8> @test_0a(<128 x i8> %v0, <128 x i8> %v1) #0 {
+  %t0 = icmp ugt <128 x i8> %v0, %v1
+  %t1 = select <128 x i1> %t0, <128 x i8> %v1, <128 x i8> %v0
+  ret <128 x i8> %t1
+}
+
+; CHECK: test_0b:
+; CHECK: v0.ub = vmin(v1.ub,v0.ub)
+define <128 x i8> @test_0b(<128 x i8> %v0, <128 x i8> %v1) #0 {
+  %t0 = icmp uge <128 x i8> %v0, %v1
+  %t1 = select <128 x i1> %t0, <128 x i8> %v1, <128 x i8> %v0
+  ret <128 x i8> %t1
+}
+
+; maxub
+
+; CHECK: test_0c:
+; CHECK: v0.ub = vmax(v1.ub,v0.ub)
+define <128 x i8> @test_0c(<128 x i8> %v0, <128 x i8> %v1) #0 {
+  %t0 = icmp ult <128 x i8> %v0, %v1
+  %t1 = select <128 x i1> %t0, <128 x i8> %v1, <128 x i8> %v0
+  ret <128 x i8> %t1
+}
+
+; CHECK: test_0d:
+; CHECK: v0.ub = vmax(v0.ub,v1.ub)
+define <128 x i8> @test_0d(<128 x i8> %v0, <128 x i8> %v1) #0 {
+  %t0 = icmp ule <128 x i8> %v0, %v1
+  %t1 = select <128 x i1> %t0, <128 x i8> %v1, <128 x i8> %v0
+  ret <128 x i8> %t1
+}
+
+; CHECK: test_0e:
+; CHECK: v0.ub = vmax(v0.ub,v1.ub)
+define <128 x i8> @test_0e(<128 x i8> %v0, <128 x i8> %v1) #0 {
+  %t0 = icmp ugt <128 x i8> %v0, %v1
+  %t1 = select <128 x i1> %t0, <128 x i8> %v0, <128 x i8> %v1
+  ret <128 x i8> %t1
+}
+
+; CHECK: test_0f:
+; CHECK: v0.ub = vmax(v1.ub,v0.ub)
+define <128 x i8> @test_0f(<128 x i8> %v0, <128 x i8> %v1) #0 {
+  %t0 = icmp uge <128 x i8> %v0, %v1
+  %t1 = select <128 x i1> %t0, <128 x i8> %v0, <128 x i8> %v1
+  ret <128 x i8> %t1
+}
+
+; minh
+
+; CHECK: test_10:
+; CHECK: v0.h = vmin(v1.h,v0.h)
+define <64 x i16> @test_10(<64 x i16> %v0, <64 x i16> %v1) #0 {
+  %t0 = icmp slt <64 x i16> %v0, %v1
+  %t1 = select <64 x i1> %t0, <64 x i16> %v0, <64 x i16> %v1
+  ret <64 x i16> %t1
+}
+
+; CHECK: test_11:
+; CHECK: v0.h = vmin(v0.h,v1.h)
+define <64 x i16> @test_11(<64 x i16> %v0, <64 x i16> %v1) #0 {
+  %t0 = icmp sle <64 x i16> %v0, %v1
+  %t1 = select <64 x i1> %t0, <64 x i16> %v0, <64 x i16> %v1
+  ret <64 x i16> %t1
+}
+
+; CHECK: test_12:
+; CHECK: v0.h = vmin(v0.h,v1.h)
+define <64 x i16> @test_12(<64 x i16> %v0, <64 x i16> %v1) #0 {
+  %t0 = icmp sgt <64 x i16> %v0, %v1
+  %t1 = select <64 x i1> %t0, <64 x i16> %v1, <64 x i16> %v0
+  ret <64 x i16> %t1
+}
+
+; CHECK: test_13:
+; CHECK: v0.h = vmin(v1.h,v0.h)
+define <64 x i16> @test_13(<64 x i16> %v0, <64 x i16> %v1) #0 {
+  %t0 = icmp sge <64 x i16> %v0, %v1
+  %t1 = select <64 x i1> %t0, <64 x i16> %v1, <64 x i16> %v0
+  ret <64 x i16> %t1
+}
+
+; maxh
+
+; CHECK: test_14:
+; CHECK: v0.h = vmax(v1.h,v0.h)
+define <64 x i16> @test_14(<64 x i16> %v0, <64 x i16> %v1) #0 {
+  %t0 = icmp slt <64 x i16> %v0, %v1
+  %t1 = select <64 x i1> %t0, <64 x i16> %v1, <64 x i16> %v0
+  ret <64 x i16> %t1
+}
+
+; CHECK: test_15:
+; CHECK: v0.h = vmax(v0.h,v1.h)
+define <64 x i16> @test_15(<64 x i16> %v0, <64 x i16> %v1) #0 {
+  %t0 = icmp sle <64 x i16> %v0, %v1
+  %t1 = select <64 x i1> %t0, <64 x i16> %v1, <64 x i16> %v0
+  ret <64 x i16> %t1
+}
+
+; CHECK: test_16:
+; CHECK: v0.h = vmax(v0.h,v1.h)
+define <64 x i16> @test_16(<64 x i16> %v0, <64 x i16> %v1) #0 {
+  %t0 = icmp sgt <64 x i16> %v0, %v1
+  %t1 = select <64 x i1> %t0, <64 x i16> %v0, <64 x i16> %v1
+  ret <64 x i16> %t1
+}
+
+; CHECK: test_17:
+; CHECK: v0.h = vmax(v1.h,v0.h)
+define <64 x i16> @test_17(<64 x i16> %v0, <64 x i16> %v1) #0 {
+  %t0 = icmp sge <64 x i16> %v0, %v1
+  %t1 = select <64 x i1> %t0, <64 x i16> %v0, <64 x i16> %v1
+  ret <64 x i16> %t1
+}
+
+; minuh
+
+; CHECK: test_18:
+; CHECK: v0.uh = vmin(v1.uh,v0.uh)
+define <64 x i16> @test_18(<64 x i16> %v0, <64 x i16> %v1) #0 {
+  %t0 = icmp ult <64 x i16> %v0, %v1
+  %t1 = select <64 x i1> %t0, <64 x i16> %v0, <64 x i16> %v1
+  ret <64 x i16> %t1
+}
+
+; CHECK: test_19:
+; CHECK: v0.uh = vmin(v0.uh,v1.uh)
+define <64 x i16> @test_19(<64 x i16> %v0, <64 x i16> %v1) #0 {
+  %t0 = icmp ule <64 x i16> %v0, %v1
+  %t1 = select <64 x i1> %t0, <64 x i16> %v0, <64 x i16> %v1
+  ret <64 x i16> %t1
+}
+
+; CHECK: test_1a:
+; CHECK: v0.uh = vmin(v0.uh,v1.uh)
+define <64 x i16> @test_1a(<64 x i16> %v0, <64 x i16> %v1) #0 {
+  %t0 = icmp ugt <64 x i16> %v0, %v1
+  %t1 = select <64 x i1> %t0, <64 x i16> %v1, <64 x i16> %v0
+  ret <64 x i16> %t1
+}
+
+; CHECK: test_1b:
+; CHECK: v0.uh = vmin(v1.uh,v0.uh)
+define <64 x i16> @test_1b(<64 x i16> %v0, <64 x i16> %v1) #0 {
+  %t0 = icmp uge <64 x i16> %v0, %v1
+  %t1 = select <64 x i1> %t0, <64 x i16> %v1, <64 x i16> %v0
+  ret <64 x i16> %t1
+}
+
+; maxuh
+
+; CHECK: test_1c:
+; CHECK: v0.uh = vmax(v1.uh,v0.uh)
+define <64 x i16> @test_1c(<64 x i16> %v0, <64 x i16> %v1) #0 {
+  %t0 = icmp ult <64 x i16> %v0, %v1
+  %t1 = select <64 x i1> %t0, <64 x i16> %v1, <64 x i16> %v0
+  ret <64 x i16> %t1
+}
+
+; CHECK: test_1d:
+; CHECK: v0.uh = vmax(v0.uh,v1.uh)
+define <64 x i16> @test_1d(<64 x i16> %v0, <64 x i16> %v1) #0 {
+  %t0 = icmp ule <64 x i16> %v0, %v1
+  %t1 = select <64 x i1> %t0, <64 x i16> %v1, <64 x i16> %v0
+  ret <64 x i16> %t1
+}
+
+; CHECK: test_1e:
+; CHECK: v0.uh = vmax(v0.uh,v1.uh)
+define <64 x i16> @test_1e(<64 x i16> %v0, <64 x i16> %v1) #0 {
+  %t0 = icmp ugt <64 x i16> %v0, %v1
+  %t1 = select <64 x i1> %t0, <64 x i16> %v0, <64 x i16> %v1
+  ret <64 x i16> %t1
+}
+
+; CHECK: test_1f:
+; CHECK: v0.uh = vmax(v1.uh,v0.uh)
+define <64 x i16> @test_1f(<64 x i16> %v0, <64 x i16> %v1) #0 {
+  %t0 = icmp uge <64 x i16> %v0, %v1
+  %t1 = select <64 x i1> %t0, <64 x i16> %v0, <64 x i16> %v1
+  ret <64 x i16> %t1
+}
+
+; minw
+
+; CHECK: test_20:
+; CHECK: v0.w = vmin(v1.w,v0.w)
+define <32 x i32> @test_20(<32 x i32> %v0, <32 x i32> %v1) #0 {
+  %t0 = icmp slt <32 x i32> %v0, %v1
+  %t1 = select <32 x i1> %t0, <32 x i32> %v0, <32 x i32> %v1
+  ret <32 x i32> %t1
+}
+
+; CHECK: test_21:
+; CHECK: v0.w = vmin(v0.w,v1.w)
+define <32 x i32> @test_21(<32 x i32> %v0, <32 x i32> %v1) #0 {
+  %t0 = icmp sle <32 x i32> %v0, %v1
+  %t1 = select <32 x i1> %t0, <32 x i32> %v0, <32 x i32> %v1
+  ret <32 x i32> %t1
+}
+
+; CHECK: test_22:
+; CHECK: v0.w = vmin(v0.w,v1.w)
+define <32 x i32> @test_22(<32 x i32> %v0, <32 x i32> %v1) #0 {
+  %t0 = icmp sgt <32 x i32> %v0, %v1
+  %t1 = select <32 x i1> %t0, <32 x i32> %v1, <32 x i32> %v0
+  ret <32 x i32> %t1
+}
+
+; CHECK: test_23:
+; CHECK: v0.w = vmin(v1.w,v0.w)
+define <32 x i32> @test_23(<32 x i32> %v0, <32 x i32> %v1) #0 {
+  %t0 = icmp sge <32 x i32> %v0, %v1
+  %t1 = select <32 x i1> %t0, <32 x i32> %v1, <32 x i32> %v0
+  ret <32 x i32> %t1
+}
+
+; maxw
+
+; CHECK: test_24:
+; CHECK: v0.w = vmax(v1.w,v0.w)
+define <32 x i32> @test_24(<32 x i32> %v0, <32 x i32> %v1) #0 {
+  %t0 = icmp slt <32 x i32> %v0, %v1
+  %t1 = select <32 x i1> %t0, <32 x i32> %v1, <32 x i32> %v0
+  ret <32 x i32> %t1
+}
+
+; CHECK: test_25:
+; CHECK: v0.w = vmax(v0.w,v1.w)
+define <32 x i32> @test_25(<32 x i32> %v0, <32 x i32> %v1) #0 {
+  %t0 = icmp sle <32 x i32> %v0, %v1
+  %t1 = select <32 x i1> %t0, <32 x i32> %v1, <32 x i32> %v0
+  ret <32 x i32> %t1
+}
+
+; CHECK: test_26:
+; CHECK: v0.w = vmax(v0.w,v1.w)
+define <32 x i32> @test_26(<32 x i32> %v0, <32 x i32> %v1) #0 {
+  %t0 = icmp sgt <32 x i32> %v0, %v1
+  %t1 = select <32 x i1> %t0, <32 x i32> %v0, <32 x i32> %v1
+  ret <32 x i32> %t1
+}
+
+; CHECK: test_27:
+; CHECK: v0.w = vmax(v1.w,v0.w)
+define <32 x i32> @test_27(<32 x i32> %v0, <32 x i32> %v1) #0 {
+  %t0 = icmp sge <32 x i32> %v0, %v1
+  %t1 = select <32 x i1> %t0, <32 x i32> %v0, <32 x i32> %v1
+  ret <32 x i32> %t1
+}
+
+attributes #0 = { readnone nounwind "target-cpu"="hexagonv62" "target-features"="+hvx,+hvx-length128b" }
+
diff --git a/llvm/test/CodeGen/Hexagon/autohvx/minmax-64b.ll b/llvm/test/CodeGen/Hexagon/autohvx/minmax-64b.ll
new file mode 100644
index 000000000..15e33e322
--- /dev/null
+++ b/llvm/test/CodeGen/Hexagon/autohvx/minmax-64b.ll
@@ -0,0 +1,344 @@
+; RUN: llc -march=hexagon < %s | FileCheck %s
+
+; minb
+
+; CHECK: test_00:
+; CHECK: v0.b = vmin(v1.b,v0.b)
+define <64 x i8> @test_00(<64 x i8> %v0, <64 x i8> %v1) #0 {
+  %t0 = icmp slt <64 x i8> %v0, %v1
+  %t1 = select <64 x i1> %t0, <64 x i8> %v0, <64 x i8> %v1
+  ret <64 x i8> %t1
+}
+
+; CHECK: test_01:
+; CHECK: v0.b = vmin(v0.b,v1.b)
+define <64 x i8> @test_01(<64 x i8> %v0, <64 x i8> %v1) #0 {
+  %t0 = icmp sle <64 x i8> %v0, %v1
+  %t1 = select <64 x i1> %t0, <64 x i8> %v0, <64 x i8> %v1
+  ret <64 x i8> %t1
+}
+
+; CHECK: test_02:
+; CHECK: v0.b = vmin(v0.b,v1.b)
+define <64 x i8> @test_02(<64 x i8> %v0, <64 x i8> %v1) #0 {
+  %t0 = icmp sgt <64 x i8> %v0, %v1
+  %t1 = select <64 x i1> %t0, <64 x i8> %v1, <64 x i8> %v0
+  ret <64 x i8> %t1
+}
+
+; CHECK: test_03:
+; CHECK: v0.b = vmin(v1.b,v0.b)
+define <64 x i8> @test_03(<64 x i8> %v0, <64 x i8> %v1) #0 {
+  %t0 = icmp sge <64 x i8> %v0, %v1
+  %t1 = select <64 x i1> %t0, <64 x i8> %v1, <64 x i8> %v0
+  ret <64 x i8> %t1
+}
+
+; maxb
+
+; CHECK: test_04:
+; CHECK: v0.b = vmax(v1.b,v0.b)
+define <64 x i8> @test_04(<64 x i8> %v0, <64 x i8> %v1) #0 {
+  %t0 = icmp slt <64 x i8> %v0, %v1
+  %t1 = select <64 x i1> %t0, <64 x i8> %v1, <64 x i8> %v0
+  ret <64 x i8> %t1
+}
+
+; CHECK: test_05:
+; CHECK: v0.b = vmax(v0.b,v1.b)
+define <64 x i8> @test_05(<64 x i8> %v0, <64 x i8> %v1) #0 {
+  %t0 = icmp sle <64 x i8> %v0, %v1
+  %t1 = select <64 x i1> %t0, <64 x i8> %v1, <64 x i8> %v0
+  ret <64 x i8> %t1
+}
+
+; CHECK: test_06:
+; CHECK: v0.b = vmax(v0.b,v1.b)
+define <64 x i8> @test_06(<64 x i8> %v0, <64 x i8> %v1) #0 {
+  %t0 = icmp sgt <64 x i8> %v0, %v1
+  %t1 = select <64 x i1> %t0, <64 x i8> %v0, <64 x i8> %v1
+  ret <64 x i8> %t1
+}
+
+; CHECK: test_07:
+; CHECK: v0.b = vmax(v1.b,v0.b)
+define <64 x i8> @test_07(<64 x i8> %v0, <64 x i8> %v1) #0 {
+  %t0 = icmp sge <64 x i8> %v0, %v1
+  %t1 = select <64 x i1> %t0, <64 x i8> %v0, <64 x i8> %v1
+  ret <64 x i8> %t1
+}
+
+; minub
+
+; CHECK: test_08:
+; CHECK: v0.ub = vmin(v1.ub,v0.ub)
+define <64 x i8> @test_08(<64 x i8> %v0, <64 x i8> %v1) #0 {
+  %t0 = icmp ult <64 x i8> %v0, %v1
+  %t1 = select <64 x i1> %t0, <64 x i8> %v0, <64 x i8> %v1
+  ret <64 x i8> %t1
+}
+
+; CHECK: test_09:
+; CHECK: v0.ub = vmin(v0.ub,v1.ub)
+define <64 x i8> @test_09(<64 x i8> %v0, <64 x i8> %v1) #0 {
+  %t0 = icmp ule <64 x i8> %v0, %v1
+  %t1 = select <64 x i1> %t0, <64 x i8> %v0, <64 x i8> %v1
+  ret <64 x i8> %t1
+}
+
+; CHECK: test_0a:
+; CHECK: v0.ub = vmin(v0.ub,v1.ub)
+define <64 x i8> @test_0a(<64 x i8> %v0, <64 x i8> %v1) #0 {
+  %t0 = icmp ugt <64 x i8> %v0, %v1
+  %t1 = select <64 x i1> %t0, <64 x i8> %v1, <64 x i8> %v0
+  ret <64 x i8> %t1
+}
+
+; CHECK: test_0b:
+; CHECK: v0.ub = vmin(v1.ub,v0.ub)
+define <64 x i8> @test_0b(<64 x i8> %v0, <64 x i8> %v1) #0 {
+  %t0 = icmp uge <64 x i8> %v0, %v1
+  %t1 = select <64 x i1> %t0, <64 x i8> %v1, <64 x i8> %v0
+  ret <64 x i8> %t1
+}
+
+; maxub
+
+; CHECK: test_0c:
+; CHECK: v0.ub = vmax(v1.ub,v0.ub)
+define <64 x i8> @test_0c(<64 x i8> %v0, <64 x i8> %v1) #0 {
+  %t0 = icmp ult <64 x i8> %v0, %v1
+  %t1 = select <64 x i1> %t0, <64 x i8> %v1, <64 x i8> %v0
+  ret <64 x i8> %t1
+}
+
+; CHECK: test_0d:
+; CHECK: v0.ub = vmax(v0.ub,v1.ub)
+define <64 x i8> @test_0d(<64 x i8> %v0, <64 x i8> %v1) #0 {
+  %t0 = icmp ule <64 x i8> %v0, %v1
+  %t1 = select <64 x i1> %t0, <64 x i8> %v1, <64 x i8> %v0
+  ret <64 x i8> %t1
+}
+
+; CHECK: test_0e:
+; CHECK: v0.ub = vmax(v0.ub,v1.ub)
+define <64 x i8> @test_0e(<64 x i8> %v0, <64 x i8> %v1) #0 {
+  %t0 = icmp ugt <64 x i8> %v0, %v1
+  %t1 = select <64 x i1> %t0, <64 x i8> %v0, <64 x i8> %v1
+  ret <64 x i8> %t1
+}
+
+; CHECK: test_0f:
+; CHECK: v0.ub = vmax(v1.ub,v0.ub)
+define <64 x i8> @test_0f(<64 x i8> %v0, <64 x i8> %v1) #0 {
+  %t0 = icmp uge <64 x i8> %v0, %v1
+  %t1 = select <64 x i1> %t0, <64 x i8> %v0, <64 x i8> %v1
+  ret <64 x i8> %t1
+}
+
+; minh
+
+; CHECK: test_10:
+; CHECK: v0.h = vmin(v1.h,v0.h)
+define <32 x i16> @test_10(<32 x i16> %v0, <32 x i16> %v1) #0 {
+  %t0 = icmp slt <32 x i16> %v0, %v1
+  %t1 = select <32 x i1> %t0, <32 x i16> %v0, <32 x i16> %v1
+  ret <32 x i16> %t1
+}
+
+; CHECK: test_11:
+; CHECK: v0.h = vmin(v0.h,v1.h)
+define <32 x i16> @test_11(<32 x i16> %v0, <32 x i16> %v1) #0 {
+  %t0 = icmp sle <32 x i16> %v0, %v1
+  %t1 = select <32 x i1> %t0, <32 x i16> %v0, <32 x i16> %v1
+  ret <32 x i16> %t1
+}
+
+; CHECK: test_12:
+; CHECK: v0.h = vmin(v0.h,v1.h)
+define <32 x i16> @test_12(<32 x i16> %v0, <32 x i16> %v1) #0 {
+  %t0 = icmp sgt <32 x i16> %v0, %v1
+  %t1 = select <32 x i1> %t0, <32 x i16> %v1, <32 x i16> %v0
+  ret <32 x i16> %t1
+}
+
+; CHECK: test_13:
+; CHECK: v0.h = vmin(v1.h,v0.h)
+define <32 x i16> @test_13(<32 x i16> %v0, <32 x i16> %v1) #0 {
+  %t0 = icmp sge <32 x i16> %v0, %v1
+  %t1 = select <32 x i1> %t0, <32 x i16> %v1, <32 x i16> %v0
+  ret <32 x i16> %t1
+}
+
+; maxh
+
+; CHECK: test_14:
+; CHECK: v0.h = vmax(v1.h,v0.h)
+define <32 x i16> @test_14(<32 x i16> %v0, <32 x i16> %v1) #0 {
+  %t0 = icmp slt <32 x i16> %v0, %v1
+  %t1 = select <32 x i1> %t0, <32 x i16> %v1, <32 x i16> %v0
+  ret <32 x i16> %t1
+}
+
+; CHECK: test_15:
+; CHECK: v0.h = vmax(v0.h,v1.h)
+define <32 x i16> @test_15(<32 x i16> %v0, <32 x i16> %v1) #0 {
+  %t0 = icmp sle <32 x i16> %v0, %v1
+  %t1 = select <32 x i1> %t0, <32 x i16> %v1, <32 x i16> %v0
+  ret <32 x i16> %t1
+}
+
+; CHECK: test_16:
+; CHECK: v0.h = vmax(v0.h,v1.h)
+define <32 x i16> @test_16(<32 x i16> %v0, <32 x i16> %v1) #0 {
+  %t0 = icmp sgt <32 x i16> %v0, %v1
+  %t1 = select <32 x i1> %t0, <32 x i16> %v0, <32 x i16> %v1
+  ret <32 x i16> %t1
+}
+
+; CHECK: test_17:
+; CHECK: v0.h = vmax(v1.h,v0.h)
+define <32 x i16> @test_17(<32 x i16> %v0, <32 x i16> %v1) #0 {
+  %t0 = icmp sge <32 x i16> %v0, %v1
+  %t1 = select <32 x i1> %t0, <32 x i16> %v0, <32 x i16> %v1
+  ret <32 x i16> %t1
+}
+
+; minuh
+
+; CHECK: test_18:
+; CHECK: v0.uh = vmin(v1.uh,v0.uh)
+define <32 x i16> @test_18(<32 x i16> %v0, <32 x i16> %v1) #0 {
+  %t0 = icmp ult <32 x i16> %v0, %v1
+  %t1 = select <32 x i1> %t0, <32 x i16> %v0, <32 x i16> %v1
+  ret <32 x i16> %t1
+}
+
+; CHECK: test_19:
+; CHECK: v0.uh = vmin(v0.uh,v1.uh)
+define <32 x i16> @test_19(<32 x i16> %v0, <32 x i16> %v1) #0 {
+  %t0 = icmp ule <32 x i16> %v0, %v1
+  %t1 = select <32 x i1> %t0, <32 x i16> %v0, <32 x i16> %v1
+  ret <32 x i16> %t1
+}
+
+; CHECK: test_1a:
+; CHECK: v0.uh = vmin(v0.uh,v1.uh)
+define <32 x i16> @test_1a(<32 x i16> %v0, <32 x i16> %v1) #0 {
+  %t0 = icmp ugt <32 x i16> %v0, %v1
+  %t1 = select <32 x i1> %t0, <32 x i16> %v1, <32 x i16> %v0
+  ret <32 x i16> %t1
+}
+
+; CHECK: test_1b:
+; CHECK: v0.uh = vmin(v1.uh,v0.uh)
+define <32 x i16> @test_1b(<32 x i16> %v0, <32 x i16> %v1) #0 {
+  %t0 = icmp uge <32 x i16> %v0, %v1
+  %t1 = select <32 x i1> %t0, <32 x i16> %v1, <32 x i16> %v0
+  ret <32 x i16> %t1
+}
+
+; maxuh
+
+; CHECK: test_1c:
+; CHECK: v0.uh = vmax(v1.uh,v0.uh)
+define <32 x i16> @test_1c(<32 x i16> %v0, <32 x i16> %v1) #0 {
+  %t0 = icmp ult <32 x i16> %v0, %v1
+  %t1 = select <32 x i1> %t0, <32 x i16> %v1, <32 x i16> %v0
+  ret <32 x i16> %t1
+}
+
+; CHECK: test_1d:
+; CHECK: v0.uh = vmax(v0.uh,v1.uh)
+define <32 x i16> @test_1d(<32 x i16> %v0, <32 x i16> %v1) #0 {
+  %t0 = icmp ule <32 x i16> %v0, %v1
+  %t1 = select <32 x i1> %t0, <32 x i16> %v1, <32 x i16> %v0
+  ret <32 x i16> %t1
+}
+
+; CHECK: test_1e:
+; CHECK: v0.uh = vmax(v0.uh,v1.uh)
+define <32 x i16> @test_1e(<32 x i16> %v0, <32 x i16> %v1) #0 {
+  %t0 = icmp ugt <32 x i16> %v0, %v1
+  %t1 = select <32 x i1> %t0, <32 x i16> %v0, <32 x i16> %v1
+  ret <32 x i16> %t1
+}
+
+; CHECK: test_1f:
+; CHECK: v0.uh = vmax(v1.uh,v0.uh)
+define <32 x i16> @test_1f(<32 x i16> %v0, <32 x i16> %v1) #0 {
+  %t0 = icmp uge <32 x i16> %v0, %v1
+  %t1 = select <32 x i1> %t0, <32 x i16> %v0, <32 x i16> %v1
+  ret <32 x i16> %t1
+}
+
+; minw
+
+; CHECK: test_20:
+; CHECK: v0.w = vmin(v1.w,v0.w)
+define <16 x i32> @test_20(<16 x i32> %v0, <16 x i32> %v1) #0 {
+  %t0 = icmp slt <16 x i32> %v0, %v1
+  %t1 = select <16 x i1> %t0, <16 x i32> %v0, <16 x i32> %v1
+  ret <16 x i32> %t1
+}
+
+; CHECK: test_21:
+; CHECK: v0.w = vmin(v0.w,v1.w)
+define <16 x i32> @test_21(<16 x i32> %v0, <16 x i32> %v1) #0 {
+  %t0 = icmp sle <16 x i32> %v0, %v1
+  %t1 = select <16 x i1> %t0, <16 x i32> %v0, <16 x i32> %v1
+  ret <16 x i32> %t1
+}
+
+; CHECK: test_22:
+; CHECK: v0.w = vmin(v0.w,v1.w)
+define <16 x i32> @test_22(<16 x i32> %v0, <16 x i32> %v1) #0 {
+  %t0 = icmp sgt <16 x i32> %v0, %v1
+  %t1 = select <16 x i1> %t0, <16 x i32> %v1, <16 x i32> %v0
+  ret <16 x i32> %t1
+}
+
+; CHECK: test_23:
+; CHECK: v0.w = vmin(v1.w,v0.w)
+define <16 x i32> @test_23(<16 x i32> %v0, <16 x i32> %v1) #0 {
+  %t0 = icmp sge <16 x i32> %v0, %v1
+  %t1 = select <16 x i1> %t0, <16 x i32> %v1, <16 x i32> %v0
+  ret <16 x i32> %t1
+}
+
+; maxw
+
+; CHECK: test_24:
+; CHECK: v0.w = vmax(v1.w,v0.w)
+define <16 x i32> @test_24(<16 x i32> %v0, <16 x i32> %v1) #0 {
+  %t0 = icmp slt <16 x i32> %v0, %v1
+  %t1 = select <16 x i1> %t0, <16 x i32> %v1, <16 x i32> %v0
+  ret <16 x i32> %t1
+}
+
+; CHECK: test_25:
+; CHECK: v0.w = vmax(v0.w,v1.w)
+define <16 x i32> @test_25(<16 x i32> %v0, <16 x i32> %v1) #0 {
+  %t0 = icmp sle <16 x i32> %v0, %v1
+  %t1 = select <16 x i1> %t0, <16 x i32> %v1, <16 x i32> %v0
+  ret <16 x i32> %t1
+}
+
+; CHECK: test_26:
+; CHECK: v0.w = vmax(v0.w,v1.w)
+define <16 x i32> @test_26(<16 x i32> %v0, <16 x i32> %v1) #0 {
+  %t0 = icmp sgt <16 x i32> %v0, %v1
+  %t1 = select <16 x i1> %t0, <16 x i32> %v0, <16 x i32> %v1
+  ret <16 x i32> %t1
+}
+
+; CHECK: test_27:
+; CHECK: v0.w = vmax(v1.w,v0.w)
+define <16 x i32> @test_27(<16 x i32> %v0, <16 x i32> %v1) #0 {
+  %t0 = icmp sge <16 x i32> %v0, %v1
+  %t1 = select <16 x i1> %t0, <16 x i32> %v0, <16 x i32> %v1
+  ret <16 x i32> %t1
+}
+
+attributes #0 = { readnone nounwind "target-cpu"="hexagonv62" "target-features"="+hvx,+hvx-length64b" }
+
diff --git a/llvm/test/CodeGen/Hexagon/autohvx/vector-compare-128b.ll b/llvm/test/CodeGen/Hexagon/autohvx/vector-compare-128b.ll
index 8205c06d9..2252d9597 100644
--- a/llvm/test/CodeGen/Hexagon/autohvx/vector-compare-128b.ll
+++ b/llvm/test/CodeGen/Hexagon/autohvx/vector-compare-128b.ll
@@ -4,91 +4,91 @@
 
 ; CHECK-LABEL: test_00:
 ; CHECK: q[[Q000:[0-3]]] = vcmp.eq(v0.b,v1.b)
-; CHECK: v0 = vmux(q[[Q000]],v0,v1)
-define <128 x i8> @test_00(<128 x i8> %v0, <128 x i8> %v1) #0 {
+; CHECK: v0 = vmux(q[[Q000]],v1,v2)
+define <128 x i8> @test_00(<128 x i8> %v0, <128 x i8> %v1, <128 x i8> %v2) #0 {
   %t0 = icmp eq <128 x i8> %v0, %v1
-  %t1 = select <128 x i1> %t0, <128 x i8> %v0, <128 x i8> %v1
+  %t1 = select <128 x i1> %t0, <128 x i8> %v1, <128 x i8> %v2
   ret <128 x i8> %t1
 }
 
 ; CHECK-LABEL: test_01:
 ; CHECK: q[[Q010:[0-3]]] = vcmp.eq(v0.b,v1.b)
-; CHECK: v0 = vmux(q[[Q010]],v1,v0)
-define <128 x i8> @test_01(<128 x i8> %v0, <128 x i8> %v1) #0 {
+; CHECK: v0 = vmux(q[[Q010]],v2,v1)
+define <128 x i8> @test_01(<128 x i8> %v0, <128 x i8> %v1, <128 x i8> %v2) #0 {
   %t0 = icmp ne <128 x i8> %v0, %v1
-  %t1 = select <128 x i1> %t0, <128 x i8> %v0, <128 x i8> %v1
+  %t1 = select <128 x i1> %t0, <128 x i8> %v1, <128 x i8> %v2
   ret <128 x i8> %t1
 }
 
 ; CHECK-LABEL: test_02:
 ; CHECK: q[[Q020:[0-3]]] = vcmp.gt(v1.b,v0.b)
-; CHECK: v0 = vmux(q[[Q020]],v0,v1)
-define <128 x i8> @test_02(<128 x i8> %v0, <128 x i8> %v1) #0 {
+; CHECK: v0 = vmux(q[[Q020]],v1,v2)
+define <128 x i8> @test_02(<128 x i8> %v0, <128 x i8> %v1, <128 x i8> %v2) #0 {
   %t0 = icmp slt <128 x i8> %v0, %v1
-  %t1 = select <128 x i1> %t0, <128 x i8> %v0, <128 x i8> %v1
+  %t1 = select <128 x i1> %t0, <128 x i8> %v1, <128 x i8> %v2
   ret <128 x i8> %t1
 }
 
 ; CHECK-LABEL: test_03:
 ; CHECK: q[[Q030:[0-3]]] = vcmp.gt(v0.b,v1.b)
-; CHECK: v0 = vmux(q[[Q030]],v1,v0)
-define <128 x i8> @test_03(<128 x i8> %v0, <128 x i8> %v1) #0 {
+; CHECK: v0 = vmux(q[[Q030]],v2,v1)
+define <128 x i8> @test_03(<128 x i8> %v0, <128 x i8> %v1, <128 x i8> %v2) #0 {
   %t0 = icmp sle <128 x i8> %v0, %v1
-  %t1 = select <128 x i1> %t0, <128 x i8> %v0, <128 x i8> %v1
+  %t1 = select <128 x i1> %t0, <128 x i8> %v1, <128 x i8> %v2
   ret <128 x i8> %t1
 }
 
 ; CHECK-LABEL: test_04:
 ; CHECK: q[[Q040:[0-3]]] = vcmp.gt(v0.b,v1.b)
-; CHECK: v0 = vmux(q[[Q040]],v0,v1)
-define <128 x i8> @test_04(<128 x i8> %v0, <128 x i8> %v1) #0 {
+; CHECK: v0 = vmux(q[[Q040]],v1,v2)
+define <128 x i8> @test_04(<128 x i8> %v0, <128 x i8> %v1, <128 x i8> %v2) #0 {
   %t0 = icmp sgt <128 x i8> %v0, %v1
-  %t1 = select <128 x i1> %t0, <128 x i8> %v0, <128 x i8> %v1
+  %t1 = select <128 x i1> %t0, <128 x i8> %v1, <128 x i8> %v2
   ret <128 x i8> %t1
 }
 
 ; CHECK-LABEL: test_05:
 ; CHECK: q[[Q050:[0-3]]] = vcmp.gt(v1.b,v0.b)
-; CHECK: v0 = vmux(q[[Q050]],v1,v0)
-define <128 x i8> @test_05(<128 x i8> %v0, <128 x i8> %v1) #0 {
+; CHECK: v0 = vmux(q[[Q050]],v2,v1)
+define <128 x i8> @test_05(<128 x i8> %v0, <128 x i8> %v1, <128 x i8> %v2) #0 {
   %t0 = icmp sge <128 x i8> %v0, %v1
-  %t1 = select <128 x i1> %t0, <128 x i8> %v0, <128 x i8> %v1
+  %t1 = select <128 x i1> %t0, <128 x i8> %v1, <128 x i8> %v2
   ret <128 x i8> %t1
 }
 
 ; CHECK-LABEL: test_06:
 ; CHECK: q[[Q060:[0-3]]] = vcmp.gt(v1.ub,v0.ub)
-; CHECK: v0 = vmux(q[[Q060]],v0,v1)
-define <128 x i8> @test_06(<128 x i8> %v0, <128 x i8> %v1) #0 {
+; CHECK: v0 = vmux(q[[Q060]],v1,v2)
+define <128 x i8> @test_06(<128 x i8> %v0, <128 x i8> %v1, <128 x i8> %v2) #0 {
   %t0 = icmp ult <128 x i8> %v0, %v1
-  %t1 = select <128 x i1> %t0, <128 x i8> %v0, <128 x i8> %v1
+  %t1 = select <128 x i1> %t0, <128 x i8> %v1, <128 x i8> %v2
   ret <128 x i8> %t1
 }
 
 ; CHECK-LABEL: test_07:
 ; CHECK: q[[Q070:[0-3]]] = vcmp.gt(v0.ub,v1.ub)
-; CHECK: v0 = vmux(q[[Q070]],v1,v0)
-define <128 x i8> @test_07(<128 x i8> %v0, <128 x i8> %v1) #0 {
+; CHECK: v0 = vmux(q[[Q070]],v2,v1)
+define <128 x i8> @test_07(<128 x i8> %v0, <128 x i8> %v1, <128 x i8> %v2) #0 {
   %t0 = icmp ule <128 x i8> %v0, %v1
-  %t1 = select <128 x i1> %t0, <128 x i8> %v0, <128 x i8> %v1
+  %t1 = select <128 x i1> %t0, <128 x i8> %v1, <128 x i8> %v2
   ret <128 x i8> %t1
 }
 
 ; CHECK-LABEL: test_08:
 ; CHECK: q[[Q080:[0-3]]] = vcmp.gt(v0.ub,v1.ub)
-; CHECK: v0 = vmux(q[[Q080]],v0,v1)
-define <128 x i8> @test_08(<128 x i8> %v0, <128 x i8> %v1) #0 {
+; CHECK: v0 = vmux(q[[Q080]],v1,v2)
+define <128 x i8> @test_08(<128 x i8> %v0, <128 x i8> %v1, <128 x i8> %v2) #0 {
   %t0 = icmp ugt <128 x i8> %v0, %v1
-  %t1 = select <128 x i1> %t0, <128 x i8> %v0, <128 x i8> %v1
+  %t1 = select <128 x i1> %t0, <128 x i8> %v1, <128 x i8> %v2
   ret <128 x i8> %t1
 }
 
 ; CHECK-LABEL: test_09:
 ; CHECK: q[[Q090:[0-3]]] = vcmp.gt(v1.ub,v0.ub)
-; CHECK: v0 = vmux(q[[Q090]],v1,v0)
-define <128 x i8> @test_09(<128 x i8> %v0, <128 x i8> %v1) #0 {
+; CHECK: v0 = vmux(q[[Q090]],v2,v1)
+define <128 x i8> @test_09(<128 x i8> %v0, <128 x i8> %v1, <128 x i8> %v2) #0 {
   %t0 = icmp uge <128 x i8> %v0, %v1
-  %t1 = select <128 x i1> %t0, <128 x i8> %v0, <128 x i8> %v1
+  %t1 = select <128 x i1> %t0, <128 x i8> %v1, <128 x i8> %v2
   ret <128 x i8> %t1
 }
 
@@ -195,91 +195,91 @@ define <128 x i8> @test_0i(<128 x i8> %v0, <128 x i8> %v1, <128 x i8> %v2) #0 {
 
 ; CHECK-LABEL: test_10:
 ; CHECK: q[[Q100:[0-3]]] = vcmp.eq(v0.h,v1.h)
-; CHECK: v0 = vmux(q[[Q100]],v0,v1)
-define <64 x i16> @test_10(<64 x i16> %v0, <64 x i16> %v1) #0 {
+; CHECK: v0 = vmux(q[[Q100]],v1,v2)
+define <64 x i16> @test_10(<64 x i16> %v0, <64 x i16> %v1, <64 x i16> %v2) #0 {
   %t0 = icmp eq <64 x i16> %v0, %v1
-  %t1 = select <64 x i1> %t0, <64 x i16> %v0, <64 x i16> %v1
+  %t1 = select <64 x i1> %t0, <64 x i16> %v1, <64 x i16> %v2
   ret <64 x i16> %t1
 }
 
 ; CHECK-LABEL: test_11:
 ; CHECK: q[[Q110:[0-3]]] = vcmp.eq(v0.h,v1.h)
-; CHECK: v0 = vmux(q[[Q110]],v1,v0)
-define <64 x i16> @test_11(<64 x i16> %v0, <64 x i16> %v1) #0 {
+; CHECK: v0 = vmux(q[[Q110]],v2,v1)
+define <64 x i16> @test_11(<64 x i16> %v0, <64 x i16> %v1, <64 x i16> %v2) #0 {
   %t0 = icmp ne <64 x i16> %v0, %v1
-  %t1 = select <64 x i1> %t0, <64 x i16> %v0, <64 x i16> %v1
+  %t1 = select <64 x i1> %t0, <64 x i16> %v1, <64 x i16> %v2
   ret <64 x i16> %t1
 }
 
 ; CHECK-LABEL: test_12:
 ; CHECK: q[[Q120:[0-3]]] = vcmp.gt(v1.h,v0.h)
-; CHECK: v0 = vmux(q[[Q120]],v0,v1)
-define <64 x i16> @test_12(<64 x i16> %v0, <64 x i16> %v1) #0 {
+; CHECK: v0 = vmux(q[[Q120]],v1,v2)
+define <64 x i16> @test_12(<64 x i16> %v0, <64 x i16> %v1, <64 x i16> %v2) #0 {
   %t0 = icmp slt <64 x i16> %v0, %v1
-  %t1 = select <64 x i1> %t0, <64 x i16> %v0, <64 x i16> %v1
+  %t1 = select <64 x i1> %t0, <64 x i16> %v1, <64 x i16> %v2
   ret <64 x i16> %t1
 }
 
 ; CHECK-LABEL: test_13:
 ; CHECK: q[[Q130:[0-3]]] = vcmp.gt(v0.h,v1.h)
-; CHECK: v0 = vmux(q[[Q130]],v1,v0)
-define <64 x i16> @test_13(<64 x i16> %v0, <64 x i16> %v1) #0 {
+; CHECK: v0 = vmux(q[[Q130]],v2,v1)
+define <64 x i16> @test_13(<64 x i16> %v0, <64 x i16> %v1, <64 x i16> %v2) #0 {
   %t0 = icmp sle <64 x i16> %v0, %v1
-  %t1 = select <64 x i1> %t0, <64 x i16> %v0, <64 x i16> %v1
+  %t1 = select <64 x i1> %t0, <64 x i16> %v1, <64 x i16> %v2
   ret <64 x i16> %t1
 }
 
 ; CHECK-LABEL: test_14:
 ; CHECK: q[[Q140:[0-3]]] = vcmp.gt(v0.h,v1.h)
-; CHECK: v0 = vmux(q[[Q140]],v0,v1)
-define <64 x i16> @test_14(<64 x i16> %v0, <64 x i16> %v1) #0 {
+; CHECK: v0 = vmux(q[[Q140]],v1,v2)
+define <64 x i16> @test_14(<64 x i16> %v0, <64 x i16> %v1, <64 x i16> %v2) #0 {
   %t0 = icmp sgt <64 x i16> %v0, %v1
-  %t1 = select <64 x i1> %t0, <64 x i16> %v0, <64 x i16> %v1
+  %t1 = select <64 x i1> %t0, <64 x i16> %v1, <64 x i16> %v2
   ret <64 x i16> %t1
 }
 
 ; CHECK-LABEL: test_15:
 ; CHECK: q[[Q150:[0-3]]] = vcmp.gt(v1.h,v0.h)
-; CHECK: v0 = vmux(q[[Q150]],v1,v0)
-define <64 x i16> @test_15(<64 x i16> %v0, <64 x i16> %v1) #0 {
+; CHECK: v0 = vmux(q[[Q150]],v2,v1)
+define <64 x i16> @test_15(<64 x i16> %v0, <64 x i16> %v1, <64 x i16> %v2) #0 {
   %t0 = icmp sge <64 x i16> %v0, %v1
-  %t1 = select <64 x i1> %t0, <64 x i16> %v0, <64 x i16> %v1
+  %t1 = select <64 x i1> %t0, <64 x i16> %v1, <64 x i16> %v2
   ret <64 x i16> %t1
 }
 
 ; CHECK-LABEL: test_16:
 ; CHECK: q[[Q160:[0-3]]] = vcmp.gt(v1.uh,v0.uh)
-; CHECK: v0 = vmux(q[[Q160]],v0,v1)
-define <64 x i16> @test_16(<64 x i16> %v0, <64 x i16> %v1) #0 {
+; CHECK: v0 = vmux(q[[Q160]],v1,v2)
+define <64 x i16> @test_16(<64 x i16> %v0, <64 x i16> %v1, <64 x i16> %v2) #0 {
   %t0 = icmp ult <64 x i16> %v0, %v1
-  %t1 = select <64 x i1> %t0, <64 x i16> %v0, <64 x i16> %v1
+  %t1 = select <64 x i1> %t0, <64 x i16> %v1, <64 x i16> %v2
   ret <64 x i16> %t1
 }
 
 ; CHECK-LABEL: test_17:
 ; CHECK: q[[Q170:[0-3]]] = vcmp.gt(v0.uh,v1.uh)
-; CHECK: v0 = vmux(q[[Q170]],v1,v0)
-define <64 x i16> @test_17(<64 x i16> %v0, <64 x i16> %v1) #0 {
+; CHECK: v0 = vmux(q[[Q170]],v2,v1)
+define <64 x i16> @test_17(<64 x i16> %v0, <64 x i16> %v1, <64 x i16> %v2) #0 {
   %t0 = icmp ule <64 x i16> %v0, %v1
-  %t1 = select <64 x i1> %t0, <64 x i16> %v0, <64 x i16> %v1
+  %t1 = select <64 x i1> %t0, <64 x i16> %v1, <64 x i16> %v2
   ret <64 x i16> %t1
 }
 
 ; CHECK-LABEL: test_18:
 ; CHECK: q[[Q180:[0-3]]] = vcmp.gt(v0.uh,v1.uh)
-; CHECK: v0 = vmux(q[[Q180]],v0,v1)
-define <64 x i16> @test_18(<64 x i16> %v0, <64 x i16> %v1) #0 {
+; CHECK: v0 = vmux(q[[Q180]],v1,v2)
+define <64 x i16> @test_18(<64 x i16> %v0, <64 x i16> %v1, <64 x i16> %v2) #0 {
   %t0 = icmp ugt <64 x i16> %v0, %v1
-  %t1 = select <64 x i1> %t0, <64 x i16> %v0, <64 x i16> %v1
+  %t1 = select <64 x i1> %t0, <64 x i16> %v1, <64 x i16> %v2
   ret <64 x i16> %t1
 }
 
 ; CHECK-LABEL: test_19:
 ; CHECK: q[[Q190:[0-3]]] = vcmp.gt(v1.uh,v0.uh)
-; CHECK: v0 = vmux(q[[Q190]],v1,v0)
-define <64 x i16> @test_19(<64 x i16> %v0, <64 x i16> %v1) #0 {
+; CHECK: v0 = vmux(q[[Q190]],v2,v1)
+define <64 x i16> @test_19(<64 x i16> %v0, <64 x i16> %v1, <64 x i16> %v2) #0 {
   %t0 = icmp uge <64 x i16> %v0, %v1
-  %t1 = select <64 x i1> %t0, <64 x i16> %v0, <64 x i16> %v1
+  %t1 = select <64 x i1> %t0, <64 x i16> %v1, <64 x i16> %v2
   ret <64 x i16> %t1
 }
 
@@ -386,91 +386,91 @@ define <64 x i16> @test_1i(<64 x i16> %v0, <64 x i16> %v1, <64 x i16> %v2) #0 {
 
 ; CHECK-LABEL: test_20:
 ; CHECK: q[[Q200:[0-3]]] = vcmp.eq(v0.w,v1.w)
-; CHECK: v0 = vmux(q[[Q200]],v0,v1)
-define <32 x i32> @test_20(<32 x i32> %v0, <32 x i32> %v1) #0 {
+; CHECK: v0 = vmux(q[[Q200]],v1,v2)
+define <32 x i32> @test_20(<32 x i32> %v0, <32 x i32> %v1, <32 x i32> %v2) #0 {
   %t0 = icmp eq <32 x i32> %v0, %v1
-  %t1 = select <32 x i1> %t0, <32 x i32> %v0, <32 x i32> %v1
+  %t1 = select <32 x i1> %t0, <32 x i32> %v1, <32 x i32> %v2
   ret <32 x i32> %t1
 }
 
 ; CHECK-LABEL: test_21:
 ; CHECK: q[[Q210:[0-3]]] = vcmp.eq(v0.w,v1.w)
-; CHECK: v0 = vmux(q[[Q210]],v1,v0)
-define <32 x i32> @test_21(<32 x i32> %v0, <32 x i32> %v1) #0 {
+; CHECK: v0 = vmux(q[[Q210]],v2,v1)
+define <32 x i32> @test_21(<32 x i32> %v0, <32 x i32> %v1, <32 x i32> %v2) #0 {
   %t0 = icmp ne <32 x i32> %v0, %v1
-  %t1 = select <32 x i1> %t0, <32 x i32> %v0, <32 x i32> %v1
+  %t1 = select <32 x i1> %t0, <32 x i32> %v1, <32 x i32> %v2
   ret <32 x i32> %t1
 }
 
 ; CHECK-LABEL: test_22:
 ; CHECK: q[[Q220:[0-3]]] = vcmp.gt(v1.w,v0.w)
-; CHECK: v0 = vmux(q[[Q220]],v0,v1)
-define <32 x i32> @test_22(<32 x i32> %v0, <32 x i32> %v1) #0 {
+; CHECK: v0 = vmux(q[[Q220]],v1,v2)
+define <32 x i32> @test_22(<32 x i32> %v0, <32 x i32> %v1, <32 x i32> %v2) #0 {
   %t0 = icmp slt <32 x i32> %v0, %v1
-  %t1 = select <32 x i1> %t0, <32 x i32> %v0, <32 x i32> %v1
+  %t1 = select <32 x i1> %t0, <32 x i32> %v1, <32 x i32> %v2
   ret <32 x i32> %t1
 }
 
 ; CHECK-LABEL: test_23:
 ; CHECK: q[[Q230:[0-3]]] = vcmp.gt(v0.w,v1.w)
-; CHECK: v0 = vmux(q[[Q230]],v1,v0)
-define <32 x i32> @test_23(<32 x i32> %v0, <32 x i32> %v1) #0 {
+; CHECK: v0 = vmux(q[[Q230]],v2,v1)
+define <32 x i32> @test_23(<32 x i32> %v0, <32 x i32> %v1, <32 x i32> %v2) #0 {
   %t0 = icmp sle <32 x i32> %v0, %v1
-  %t1 = select <32 x i1> %t0, <32 x i32> %v0, <32 x i32> %v1
+  %t1 = select <32 x i1> %t0, <32 x i32> %v1, <32 x i32> %v2
   ret <32 x i32> %t1
 }
 
 ; CHECK-LABEL: test_24:
 ; CHECK: q[[Q240:[0-3]]] = vcmp.gt(v0.w,v1.w)
-; CHECK: v0 = vmux(q[[Q240]],v0,v1)
-define <32 x i32> @test_24(<32 x i32> %v0, <32 x i32> %v1) #0 {
+; CHECK: v0 = vmux(q[[Q240]],v1,v2)
+define <32 x i32> @test_24(<32 x i32> %v0, <32 x i32> %v1, <32 x i32> %v2) #0 {
   %t0 = icmp sgt <32 x i32> %v0, %v1
-  %t1 = select <32 x i1> %t0, <32 x i32> %v0, <32 x i32> %v1
+  %t1 = select <32 x i1> %t0, <32 x i32> %v1, <32 x i32> %v2
   ret <32 x i32> %t1
 }
 
 ; CHECK-LABEL: test_25:
 ; CHECK: q[[Q250:[0-3]]] = vcmp.gt(v1.w,v0.w)
-; CHECK: v0 = vmux(q[[Q250]],v1,v0)
-define <32 x i32> @test_25(<32 x i32> %v0, <32 x i32> %v1) #0 {
+; CHECK: v0 = vmux(q[[Q250]],v2,v1)
+define <32 x i32> @test_25(<32 x i32> %v0, <32 x i32> %v1, <32 x i32> %v2) #0 {
   %t0 = icmp sge <32 x i32> %v0, %v1
-  %t1 = select <32 x i1> %t0, <32 x i32> %v0, <32 x i32> %v1
+  %t1 = select <32 x i1> %t0, <32 x i32> %v1, <32 x i32> %v2
   ret <32 x i32> %t1
 }
 
 ; CHECK-LABEL: test_26:
 ; CHECK: q[[Q260:[0-3]]] = vcmp.gt(v1.uw,v0.uw)
-; CHECK: v0 = vmux(q[[Q260]],v0,v1)
-define <32 x i32> @test_26(<32 x i32> %v0, <32 x i32> %v1) #0 {
+; CHECK: v0 = vmux(q[[Q260]],v1,v2)
+define <32 x i32> @test_26(<32 x i32> %v0, <32 x i32> %v1, <32 x i32> %v2) #0 {
   %t0 = icmp ult <32 x i32> %v0, %v1
-  %t1 = select <32 x i1> %t0, <32 x i32> %v0, <32 x i32> %v1
+  %t1 = select <32 x i1> %t0, <32 x i32> %v1, <32 x i32> %v2
   ret <32 x i32> %t1
 }
 
 ; CHECK-LABEL: test_27:
 ; CHECK: q[[Q270:[0-3]]] = vcmp.gt(v0.uw,v1.uw)
-; CHECK: v0 = vmux(q[[Q270]],v1,v0)
-define <32 x i32> @test_27(<32 x i32> %v0, <32 x i32> %v1) #0 {
+; CHECK: v0 = vmux(q[[Q270]],v2,v1)
+define <32 x i32> @test_27(<32 x i32> %v0, <32 x i32> %v1, <32 x i32> %v2) #0 {
   %t0 = icmp ule <32 x i32> %v0, %v1
-  %t1 = select <32 x i1> %t0, <32 x i32> %v0, <32 x i32> %v1
+  %t1 = select <32 x i1> %t0, <32 x i32> %v1, <32 x i32> %v2
   ret <32 x i32> %t1
 }
 
 ; CHECK-LABEL: test_28:
 ; CHECK: q[[Q280:[0-3]]] = vcmp.gt(v0.uw,v1.uw)
-; CHECK: v0 = vmux(q[[Q280]],v0,v1)
-define <32 x i32> @test_28(<32 x i32> %v0, <32 x i32> %v1) #0 {
+; CHECK: v0 = vmux(q[[Q280]],v1,v2)
+define <32 x i32> @test_28(<32 x i32> %v0, <32 x i32> %v1, <32 x i32> %v2) #0 {
   %t0 = icmp ugt <32 x i32> %v0, %v1
-  %t1 = select <32 x i1> %t0, <32 x i32> %v0, <32 x i32> %v1
+  %t1 = select <32 x i1> %t0, <32 x i32> %v1, <32 x i32> %v2
   ret <32 x i32> %t1
 }
 
 ; CHECK-LABEL: test_29:
 ; CHECK: q[[Q290:[0-3]]] = vcmp.gt(v1.uw,v0.uw)
-; CHECK: v0 = vmux(q[[Q290]],v1,v0)
-define <32 x i32> @test_29(<32 x i32> %v0, <32 x i32> %v1) #0 {
+; CHECK: v0 = vmux(q[[Q290]],v2,v1)
+define <32 x i32> @test_29(<32 x i32> %v0, <32 x i32> %v1, <32 x i32> %v2) #0 {
   %t0 = icmp uge <32 x i32> %v0, %v1
-  %t1 = select <32 x i1> %t0, <32 x i32> %v0, <32 x i32> %v1
+  %t1 = select <32 x i1> %t0, <32 x i32> %v1, <32 x i32> %v2
   ret <32 x i32> %t1
 }
 
diff --git a/llvm/test/CodeGen/Hexagon/autohvx/vector-compare-64b.ll b/llvm/test/CodeGen/Hexagon/autohvx/vector-compare-64b.ll
index 9475eb6a8..55ac8981a 100644
--- a/llvm/test/CodeGen/Hexagon/autohvx/vector-compare-64b.ll
+++ b/llvm/test/CodeGen/Hexagon/autohvx/vector-compare-64b.ll
@@ -4,91 +4,91 @@
 
 ; CHECK-LABEL: test_00:
 ; CHECK: q[[Q000:[0-3]]] = vcmp.eq(v0.b,v1.b)
-; CHECK: v0 = vmux(q[[Q000]],v0,v1)
-define <64 x i8> @test_00(<64 x i8> %v0, <64 x i8> %v1) #0 {
+; CHECK: v0 = vmux(q[[Q000]],v1,v2)
+define <64 x i8> @test_00(<64 x i8> %v0, <64 x i8> %v1, <64 x i8> %v2) #0 {
   %t0 = icmp eq <64 x i8> %v0, %v1
-  %t1 = select <64 x i1> %t0, <64 x i8> %v0, <64 x i8> %v1
+  %t1 = select <64 x i1> %t0, <64 x i8> %v1, <64 x i8> %v2
   ret <64 x i8> %t1
 }
 
 ; CHECK-LABEL: test_01:
 ; CHECK: q[[Q010:[0-3]]] = vcmp.eq(v0.b,v1.b)
-; CHECK: v0 = vmux(q[[Q010]],v1,v0)
-define <64 x i8> @test_01(<64 x i8> %v0, <64 x i8> %v1) #0 {
+; CHECK: v0 = vmux(q[[Q010]],v2,v1)
+define <64 x i8> @test_01(<64 x i8> %v0, <64 x i8> %v1, <64 x i8> %v2) #0 {
   %t0 = icmp ne <64 x i8> %v0, %v1
-  %t1 = select <64 x i1> %t0, <64 x i8> %v0, <64 x i8> %v1
+  %t1 = select <64 x i1> %t0, <64 x i8> %v1, <64 x i8> %v2
   ret <64 x i8> %t1
 }
 
 ; CHECK-LABEL: test_02:
 ; CHECK: q[[Q020:[0-3]]] = vcmp.gt(v1.b,v0.b)
-; CHECK: v0 = vmux(q[[Q020]],v0,v1)
-define <64 x i8> @test_02(<64 x i8> %v0, <64 x i8> %v1) #0 {
+; CHECK: v0 = vmux(q[[Q020]],v1,v2)
+define <64 x i8> @test_02(<64 x i8> %v0, <64 x i8> %v1, <64 x i8> %v2) #0 {
   %t0 = icmp slt <64 x i8> %v0, %v1
-  %t1 = select <64 x i1> %t0, <64 x i8> %v0, <64 x i8> %v1
+  %t1 = select <64 x i1> %t0, <64 x i8> %v1, <64 x i8> %v2
   ret <64 x i8> %t1
 }
 
 ; CHECK-LABEL: test_03:
 ; CHECK: q[[Q030:[0-3]]] = vcmp.gt(v0.b,v1.b)
-; CHECK: v0 = vmux(q[[Q030]],v1,v0)
-define <64 x i8> @test_03(<64 x i8> %v0, <64 x i8> %v1) #0 {
+; CHECK: v0 = vmux(q[[Q030]],v2,v1)
+define <64 x i8> @test_03(<64 x i8> %v0, <64 x i8> %v1, <64 x i8> %v2) #0 {
   %t0 = icmp sle <64 x i8> %v0, %v1
-  %t1 = select <64 x i1> %t0, <64 x i8> %v0, <64 x i8> %v1
+  %t1 = select <64 x i1> %t0, <64 x i8> %v1, <64 x i8> %v2
   ret <64 x i8> %t1
 }
 
 ; CHECK-LABEL: test_04:
 ; CHECK: q[[Q040:[0-3]]] = vcmp.gt(v0.b,v1.b)
-; CHECK: v0 = vmux(q[[Q040]],v0,v1)
-define <64 x i8> @test_04(<64 x i8> %v0, <64 x i8> %v1) #0 {
+; CHECK: v0 = vmux(q[[Q040]],v1,v2)
+define <64 x i8> @test_04(<64 x i8> %v0, <64 x i8> %v1, <64 x i8> %v2) #0 {
   %t0 = icmp sgt <64 x i8> %v0, %v1
-  %t1 = select <64 x i1> %t0, <64 x i8> %v0, <64 x i8> %v1
+  %t1 = select <64 x i1> %t0, <64 x i8> %v1, <64 x i8> %v2
   ret <64 x i8> %t1
 }
 
 ; CHECK-LABEL: test_05:
 ; CHECK: q[[Q050:[0-3]]] = vcmp.gt(v1.b,v0.b)
-; CHECK: v0 = vmux(q[[Q050]],v1,v0)
-define <64 x i8> @test_05(<64 x i8> %v0, <64 x i8> %v1) #0 {
+; CHECK: v0 = vmux(q[[Q050]],v2,v1)
+define <64 x i8> @test_05(<64 x i8> %v0, <64 x i8> %v1, <64 x i8> %v2) #0 {
   %t0 = icmp sge <64 x i8> %v0, %v1
-  %t1 = select <64 x i1> %t0, <64 x i8> %v0, <64 x i8> %v1
+  %t1 = select <64 x i1> %t0, <64 x i8> %v1, <64 x i8> %v2
   ret <64 x i8> %t1
 }
 
 ; CHECK-LABEL: test_06:
 ; CHECK: q[[Q060:[0-3]]] = vcmp.gt(v1.ub,v0.ub)
-; CHECK: v0 = vmux(q[[Q060]],v0,v1)
-define <64 x i8> @test_06(<64 x i8> %v0, <64 x i8> %v1) #0 {
+; CHECK: v0 = vmux(q[[Q060]],v1,v2)
+define <64 x i8> @test_06(<64 x i8> %v0, <64 x i8> %v1, <64 x i8> %v2) #0 {
   %t0 = icmp ult <64 x i8> %v0, %v1
-  %t1 = select <64 x i1> %t0, <64 x i8> %v0, <64 x i8> %v1
+  %t1 = select <64 x i1> %t0, <64 x i8> %v1, <64 x i8> %v2
   ret <64 x i8> %t1
 }
 
 ; CHECK-LABEL: test_07:
 ; CHECK: q[[Q070:[0-3]]] = vcmp.gt(v0.ub,v1.ub)
-; CHECK: v0 = vmux(q[[Q070]],v1,v0)
-define <64 x i8> @test_07(<64 x i8> %v0, <64 x i8> %v1) #0 {
+; CHECK: v0 = vmux(q[[Q070]],v2,v1)
+define <64 x i8> @test_07(<64 x i8> %v0, <64 x i8> %v1, <64 x i8> %v2) #0 {
   %t0 = icmp ule <64 x i8> %v0, %v1
-  %t1 = select <64 x i1> %t0, <64 x i8> %v0, <64 x i8> %v1
+  %t1 = select <64 x i1> %t0, <64 x i8> %v1, <64 x i8> %v2
   ret <64 x i8> %t1
 }
 
 ; CHECK-LABEL: test_08:
 ; CHECK: q[[Q080:[0-3]]] = vcmp.gt(v0.ub,v1.ub)
-; CHECK: v0 = vmux(q[[Q080]],v0,v1)
-define <64 x i8> @test_08(<64 x i8> %v0, <64 x i8> %v1) #0 {
+; CHECK: v0 = vmux(q[[Q080]],v1,v2)
+define <64 x i8> @test_08(<64 x i8> %v0, <64 x i8> %v1, <64 x i8> %v2) #0 {
   %t0 = icmp ugt <64 x i8> %v0, %v1
-  %t1 = select <64 x i1> %t0, <64 x i8> %v0, <64 x i8> %v1
+  %t1 = select <64 x i1> %t0, <64 x i8> %v1, <64 x i8> %v2
   ret <64 x i8> %t1
 }
 
 ; CHECK-LABEL: test_09:
 ; CHECK: q[[Q090:[0-3]]] = vcmp.gt(v1.ub,v0.ub)
-; CHECK: v0 = vmux(q[[Q090]],v1,v0)
-define <64 x i8> @test_09(<64 x i8> %v0, <64 x i8> %v1) #0 {
+; CHECK: v0 = vmux(q[[Q090]],v2,v1)
+define <64 x i8> @test_09(<64 x i8> %v0, <64 x i8> %v1, <64 x i8> %v2) #0 {
   %t0 = icmp uge <64 x i8> %v0, %v1
-  %t1 = select <64 x i1> %t0, <64 x i8> %v0, <64 x i8> %v1
+  %t1 = select <64 x i1> %t0, <64 x i8> %v1, <64 x i8> %v2
   ret <64 x i8> %t1
 }
 
@@ -196,91 +196,91 @@ define <64 x i8> @test_0i(<64 x i8> %v0, <64 x i8> %v1, <64 x i8> %v2) #0 {
 
 ; CHECK-LABEL: test_10:
 ; CHECK: q[[Q100:[0-3]]] = vcmp.eq(v0.h,v1.h)
-; CHECK: v0 = vmux(q[[Q100]],v0,v1)
-define <32 x i16> @test_10(<32 x i16> %v0, <32 x i16> %v1) #0 {
+; CHECK: v0 = vmux(q[[Q100]],v1,v2)
+define <32 x i16> @test_10(<32 x i16> %v0, <32 x i16> %v1, <32 x i16> %v2) #0 {
   %t0 = icmp eq <32 x i16> %v0, %v1
-  %t1 = select <32 x i1> %t0, <32 x i16> %v0, <32 x i16> %v1
+  %t1 = select <32 x i1> %t0, <32 x i16> %v1, <32 x i16> %v2
   ret <32 x i16> %t1
 }
 
 ; CHECK-LABEL: test_11:
 ; CHECK: q[[Q110:[0-3]]] = vcmp.eq(v0.h,v1.h)
-; CHECK: v0 = vmux(q[[Q110]],v1,v0)
-define <32 x i16> @test_11(<32 x i16> %v0, <32 x i16> %v1) #0 {
+; CHECK: v0 = vmux(q[[Q110]],v2,v1)
+define <32 x i16> @test_11(<32 x i16> %v0, <32 x i16> %v1, <32 x i16> %v2) #0 {
   %t0 = icmp ne <32 x i16> %v0, %v1
-  %t1 = select <32 x i1> %t0, <32 x i16> %v0, <32 x i16> %v1
+  %t1 = select <32 x i1> %t0, <32 x i16> %v1, <32 x i16> %v2
   ret <32 x i16> %t1
 }
 
 ; CHECK-LABEL: test_12:
 ; CHECK: q[[Q120:[0-3]]] = vcmp.gt(v1.h,v0.h)
-; CHECK: v0 = vmux(q[[Q120]],v0,v1)
-define <32 x i16> @test_12(<32 x i16> %v0, <32 x i16> %v1) #0 {
+; CHECK: v0 = vmux(q[[Q120]],v1,v2)
+define <32 x i16> @test_12(<32 x i16> %v0, <32 x i16> %v1, <32 x i16> %v2) #0 {
   %t0 = icmp slt <32 x i16> %v0, %v1
-  %t1 = select <32 x i1> %t0, <32 x i16> %v0, <32 x i16> %v1
+  %t1 = select <32 x i1> %t0, <32 x i16> %v1, <32 x i16> %v2
   ret <32 x i16> %t1
 }
 
 ; CHECK-LABEL: test_13:
 ; CHECK: q[[Q130:[0-3]]] = vcmp.gt(v0.h,v1.h)
-; CHECK: v0 = vmux(q[[Q130]],v1,v0)
-define <32 x i16> @test_13(<32 x i16> %v0, <32 x i16> %v1) #0 {
+; CHECK: v0 = vmux(q[[Q130]],v2,v1)
+define <32 x i16> @test_13(<32 x i16> %v0, <32 x i16> %v1, <32 x i16> %v2) #0 {
   %t0 = icmp sle <32 x i16> %v0, %v1
-  %t1 = select <32 x i1> %t0, <32 x i16> %v0, <32 x i16> %v1
+  %t1 = select <32 x i1> %t0, <32 x i16> %v1, <32 x i16> %v2
   ret <32 x i16> %t1
 }
 
 ; CHECK-LABEL: test_14:
 ; CHECK: q[[Q140:[0-3]]] = vcmp.gt(v0.h,v1.h)
-; CHECK: v0 = vmux(q[[Q140]],v0,v1)
-define <32 x i16> @test_14(<32 x i16> %v0, <32 x i16> %v1) #0 {
+; CHECK: v0 = vmux(q[[Q140]],v1,v2)
+define <32 x i16> @test_14(<32 x i16> %v0, <32 x i16> %v1, <32 x i16> %v2) #0 {
   %t0 = icmp sgt <32 x i16> %v0, %v1
-  %t1 = select <32 x i1> %t0, <32 x i16> %v0, <32 x i16> %v1
+  %t1 = select <32 x i1> %t0, <32 x i16> %v1, <32 x i16> %v2
   ret <32 x i16> %t1
 }
 
 ; CHECK-LABEL: test_15:
 ; CHECK: q[[Q150:[0-3]]] = vcmp.gt(v1.h,v0.h)
-; CHECK: v0 = vmux(q[[Q150]],v1,v0)
-define <32 x i16> @test_15(<32 x i16> %v0, <32 x i16> %v1) #0 {
+; CHECK: v0 = vmux(q[[Q150]],v2,v1)
+define <32 x i16> @test_15(<32 x i16> %v0, <32 x i16> %v1, <32 x i16> %v2) #0 {
   %t0 = icmp sge <32 x i16> %v0, %v1
-  %t1 = select <32 x i1> %t0, <32 x i16> %v0, <32 x i16> %v1
+  %t1 = select <32 x i1> %t0, <32 x i16> %v1, <32 x i16> %v2
   ret <32 x i16> %t1
 }
 
 ; CHECK-LABEL: test_16:
 ; CHECK: q[[Q160:[0-3]]] = vcmp.gt(v1.uh,v0.uh)
-; CHECK: v0 = vmux(q[[Q160]],v0,v1)
-define <32 x i16> @test_16(<32 x i16> %v0, <32 x i16> %v1) #0 {
+; CHECK: v0 = vmux(q[[Q160]],v1,v2)
+define <32 x i16> @test_16(<32 x i16> %v0, <32 x i16> %v1, <32 x i16> %v2) #0 {
   %t0 = icmp ult <32 x i16> %v0, %v1
-  %t1 = select <32 x i1> %t0, <32 x i16> %v0, <32 x i16> %v1
+  %t1 = select <32 x i1> %t0, <32 x i16> %v1, <32 x i16> %v2
   ret <32 x i16> %t1
 }
 
 ; CHECK-LABEL: test_17:
 ; CHECK: q[[Q170:[0-3]]] = vcmp.gt(v0.uh,v1.uh)
-; CHECK: v0 = vmux(q[[Q170]],v1,v0)
-define <32 x i16> @test_17(<32 x i16> %v0, <32 x i16> %v1) #0 {
+; CHECK: v0 = vmux(q[[Q170]],v2,v1)
+define <32 x i16> @test_17(<32 x i16> %v0, <32 x i16> %v1, <32 x i16> %v2) #0 {
   %t0 = icmp ule <32 x i16> %v0, %v1
-  %t1 = select <32 x i1> %t0, <32 x i16> %v0, <32 x i16> %v1
+  %t1 = select <32 x i1> %t0, <32 x i16> %v1, <32 x i16> %v2
   ret <32 x i16> %t1
 }
 
 ; CHECK-LABEL: test_18:
 ; CHECK: q[[Q180:[0-3]]] = vcmp.gt(v0.uh,v1.uh)
-; CHECK: v0 = vmux(q[[Q180]],v0,v1)
-define <32 x i16> @test_18(<32 x i16> %v0, <32 x i16> %v1) #0 {
+; CHECK: v0 = vmux(q[[Q180]],v1,v2)
+define <32 x i16> @test_18(<32 x i16> %v0, <32 x i16> %v1, <32 x i16> %v2) #0 {
   %t0 = icmp ugt <32 x i16> %v0, %v1
-  %t1 = select <32 x i1> %t0, <32 x i16> %v0, <32 x i16> %v1
+  %t1 = select <32 x i1> %t0, <32 x i16> %v1, <32 x i16> %v2
   ret <32 x i16> %t1
 }
 
 ; CHECK-LABEL: test_19:
 ; CHECK: q[[Q190:[0-3]]] = vcmp.gt(v1.uh,v0.uh)
-; CHECK: v0 = vmux(q[[Q190]],v1,v0)
-define <32 x i16> @test_19(<32 x i16> %v0, <32 x i16> %v1) #0 {
+; CHECK: v0 = vmux(q[[Q190]],v2,v1)
+define <32 x i16> @test_19(<32 x i16> %v0, <32 x i16> %v1, <32 x i16> %v2) #0 {
   %t0 = icmp uge <32 x i16> %v0, %v1
-  %t1 = select <32 x i1> %t0, <32 x i16> %v0, <32 x i16> %v1
+  %t1 = select <32 x i1> %t0, <32 x i16> %v1, <32 x i16> %v2
   ret <32 x i16> %t1
 }
 
@@ -387,91 +387,91 @@ define <32 x i16> @test_1i(<32 x i16> %v0, <32 x i16> %v1, <32 x i16> %v2) #0 {
 
 ; CHECK-LABEL: test_20:
 ; CHECK: q[[Q200:[0-3]]] = vcmp.eq(v0.w,v1.w)
-; CHECK: v0 = vmux(q[[Q200]],v0,v1)
-define <16 x i32> @test_20(<16 x i32> %v0, <16 x i32> %v1) #0 {
+; CHECK: v0 = vmux(q[[Q200]],v1,v2)
+define <16 x i32> @test_20(<16 x i32> %v0, <16 x i32> %v1, <16 x i32> %v2) #0 {
   %t0 = icmp eq <16 x i32> %v0, %v1
-  %t1 = select <16 x i1> %t0, <16 x i32> %v0, <16 x i32> %v1
+  %t1 = select <16 x i1> %t0, <16 x i32> %v1, <16 x i32> %v2
   ret <16 x i32> %t1
 }
 
 ; CHECK-LABEL: test_21:
 ; CHECK: q[[Q210:[0-3]]] = vcmp.eq(v0.w,v1.w)
-; CHECK: v0 = vmux(q[[Q210]],v1,v0)
-define <16 x i32> @test_21(<16 x i32> %v0, <16 x i32> %v1) #0 {
+; CHECK: v0 = vmux(q[[Q210]],v2,v1)
+define <16 x i32> @test_21(<16 x i32> %v0, <16 x i32> %v1, <16 x i32> %v2) #0 {
   %t0 = icmp ne <16 x i32> %v0, %v1
-  %t1 = select <16 x i1> %t0, <16 x i32> %v0, <16 x i32> %v1
+  %t1 = select <16 x i1> %t0, <16 x i32> %v1, <16 x i32> %v2
   ret <16 x i32> %t1
 }
 
 ; CHECK-LABEL: test_22:
 ; CHECK: q[[Q220:[0-3]]] = vcmp.gt(v1.w,v0.w)
-; CHECK: v0 = vmux(q[[Q220]],v0,v1)
-define <16 x i32> @test_22(<16 x i32> %v0, <16 x i32> %v1) #0 {
+; CHECK: v0 = vmux(q[[Q220]],v1,v2)
+define <16 x i32> @test_22(<16 x i32> %v0, <16 x i32> %v1, <16 x i32> %v2) #0 {
   %t0 = icmp slt <16 x i32> %v0, %v1
-  %t1 = select <16 x i1> %t0, <16 x i32> %v0, <16 x i32> %v1
+  %t1 = select <16 x i1> %t0, <16 x i32> %v1, <16 x i32> %v2
   ret <16 x i32> %t1
 }
 
 ; CHECK-LABEL: test_23:
 ; CHECK: q[[Q230:[0-3]]] = vcmp.gt(v0.w,v1.w)
-; CHECK: v0 = vmux(q[[Q230]],v1,v0)
-define <16 x i32> @test_23(<16 x i32> %v0, <16 x i32> %v1) #0 {
+; CHECK: v0 = vmux(q[[Q230]],v2,v1)
+define <16 x i32> @test_23(<16 x i32> %v0, <16 x i32> %v1, <16 x i32> %v2) #0 {
   %t0 = icmp sle <16 x i32> %v0, %v1
-  %t1 = select <16 x i1> %t0, <16 x i32> %v0, <16 x i32> %v1
+  %t1 = select <16 x i1> %t0, <16 x i32> %v1, <16 x i32> %v2
   ret <16 x i32> %t1
 }
 
 ; CHECK-LABEL: test_24:
 ; CHECK: q[[Q240:[0-3]]] = vcmp.gt(v0.w,v1.w)
-; CHECK: v0 = vmux(q[[Q240]],v0,v1)
-define <16 x i32> @test_24(<16 x i32> %v0, <16 x i32> %v1) #0 {
+; CHECK: v0 = vmux(q[[Q240]],v1,v2)
+define <16 x i32> @test_24(<16 x i32> %v0, <16 x i32> %v1, <16 x i32> %v2) #0 {
   %t0 = icmp sgt <16 x i32> %v0, %v1
-  %t1 = select <16 x i1> %t0, <16 x i32> %v0, <16 x i32> %v1
+  %t1 = select <16 x i1> %t0, <16 x i32> %v1, <16 x i32> %v2
   ret <16 x i32> %t1
 }
 
 ; CHECK-LABEL: test_25:
 ; CHECK: q[[Q250:[0-3]]] = vcmp.gt(v1.w,v0.w)
-; CHECK: v0 = vmux(q[[Q250]],v1,v0)
-define <16 x i32> @test_25(<16 x i32> %v0, <16 x i32> %v1) #0 {
+; CHECK: v0 = vmux(q[[Q250]],v2,v1)
+define <16 x i32> @test_25(<16 x i32> %v0, <16 x i32> %v1, <16 x i32> %v2) #0 {
   %t0 = icmp sge <16 x i32> %v0, %v1
-  %t1 = select <16 x i1> %t0, <16 x i32> %v0, <16 x i32> %v1
+  %t1 = select <16 x i1> %t0, <16 x i32> %v1, <16 x i32> %v2
   ret <16 x i32> %t1
 }
 
 ; CHECK-LABEL: test_26:
 ; CHECK: q[[Q260:[0-3]]] = vcmp.gt(v1.uw,v0.uw)
-; CHECK: v0 = vmux(q[[Q260]],v0,v1)
-define <16 x i32> @test_26(<16 x i32> %v0, <16 x i32> %v1) #0 {
+; CHECK: v0 = vmux(q[[Q260]],v1,v2)
+define <16 x i32> @test_26(<16 x i32> %v0, <16 x i32> %v1, <16 x i32> %v2) #0 {
   %t0 = icmp ult <16 x i32> %v0, %v1
-  %t1 = select <16 x i1> %t0, <16 x i32> %v0, <16 x i32> %v1
+  %t1 = select <16 x i1> %t0, <16 x i32> %v1, <16 x i32> %v2
   ret <16 x i32> %t1
 }
 
 ; CHECK-LABEL: test_27:
 ; CHECK: q[[Q270:[0-3]]] = vcmp.gt(v0.uw,v1.uw)
-; CHECK: v0 = vmux(q[[Q270]],v1,v0)
-define <16 x i32> @test_27(<16 x i32> %v0, <16 x i32> %v1) #0 {
+; CHECK: v0 = vmux(q[[Q270]],v2,v1)
+define <16 x i32> @test_27(<16 x i32> %v0, <16 x i32> %v1, <16 x i32> %v2) #0 {
   %t0 = icmp ule <16 x i32> %v0, %v1
-  %t1 = select <16 x i1> %t0, <16 x i32> %v0, <16 x i32> %v1
+  %t1 = select <16 x i1> %t0, <16 x i32> %v1, <16 x i32> %v2
   ret <16 x i32> %t1
 }
 
 ; CHECK-LABEL: test_28:
 ; CHECK: q[[Q280:[0-3]]] = vcmp.gt(v0.uw,v1.uw)
-; CHECK: v0 = vmux(q[[Q280]],v0,v1)
-define <16 x i32> @test_28(<16 x i32> %v0, <16 x i32> %v1) #0 {
+; CHECK: v0 = vmux(q[[Q280]],v1,v2)
+define <16 x i32> @test_28(<16 x i32> %v0, <16 x i32> %v1, <16 x i32> %v2) #0 {
   %t0 = icmp ugt <16 x i32> %v0, %v1
-  %t1 = select <16 x i1> %t0, <16 x i32> %v0, <16 x i32> %v1
+  %t1 = select <16 x i1> %t0, <16 x i32> %v1, <16 x i32> %v2
   ret <16 x i32> %t1
 }
 
 ; CHECK-LABEL: test_29:
 ; CHECK: q[[Q290:[0-3]]] = vcmp.gt(v1.uw,v0.uw)
-; CHECK: v0 = vmux(q[[Q290]],v1,v0)
-define <16 x i32> @test_29(<16 x i32> %v0, <16 x i32> %v1) #0 {
+; CHECK: v0 = vmux(q[[Q290]],v2,v1)
+define <16 x i32> @test_29(<16 x i32> %v0, <16 x i32> %v1, <16 x i32> %v2) #0 {
   %t0 = icmp uge <16 x i32> %v0, %v1
-  %t1 = select <16 x i1> %t0, <16 x i32> %v0, <16 x i32> %v1
+  %t1 = select <16 x i1> %t0, <16 x i32> %v1, <16 x i32> %v2
   ret <16 x i32> %t1
 }
 
diff --git a/llvm/test/CodeGen/Hexagon/base-offset-addr.ll b/llvm/test/CodeGen/Hexagon/base-offset-addr.ll
index 30410fe92..6d379a087 100644
--- a/llvm/test/CodeGen/Hexagon/base-offset-addr.ll
+++ b/llvm/test/CodeGen/Hexagon/base-offset-addr.ll
@@ -12,4 +12,4 @@ entry:
   ret void
 }
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf"="true" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/Hexagon/base-offset-post.ll b/llvm/test/CodeGen/Hexagon/base-offset-post.ll
index f5c7f6dc2..5af384c53 100644
--- a/llvm/test/CodeGen/Hexagon/base-offset-post.ll
+++ b/llvm/test/CodeGen/Hexagon/base-offset-post.ll
@@ -27,4 +27,4 @@ if.end:
   ret void
 }
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf"="true" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/Hexagon/bit-loop-rc-mismatch.ll b/llvm/test/CodeGen/Hexagon/bit-loop-rc-mismatch.ll
index e7dd87c1d..e2f044b8b 100644
--- a/llvm/test/CodeGen/Hexagon/bit-loop-rc-mismatch.ll
+++ b/llvm/test/CodeGen/Hexagon/bit-loop-rc-mismatch.ll
@@ -24,7 +24,7 @@ for.end:                                          ; preds = %for.body, %entry
 declare hidden i64 @danny(i32*, i32* nocapture readonly dereferenceable(4)) #1 align 2
 declare hidden i32 @sammy(i32* nocapture, i32) #0 align 2
 
-attributes #0 = { nounwind optsize "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="hexagonv60" "target-features"="+hvx,+hvx-length64b" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { nounwind optsize readonly "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="hexagonv60" "target-features"="+hvx,+hvx-length64b" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind optsize "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="hexagonv60" "target-features"="+hvx,+hvx-length64b" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind optsize readonly "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="hexagonv60" "target-features"="+hvx,+hvx-length64b" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { optsize }
 
diff --git a/llvm/test/CodeGen/Hexagon/builtin-prefetch.ll b/llvm/test/CodeGen/Hexagon/builtin-prefetch.ll
index ae236645b..8f7b44f65 100644
--- a/llvm/test/CodeGen/Hexagon/builtin-prefetch.ll
+++ b/llvm/test/CodeGen/Hexagon/builtin-prefetch.ll
@@ -25,5 +25,5 @@ entry:
 declare void @llvm.prefetch(i8* nocapture, i32, i32, i32) #1
 declare void @llvm.hexagon.prefetch(i8* nocapture) #1
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf"="true" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind }
diff --git a/llvm/test/CodeGen/Hexagon/cfi-offset.ll b/llvm/test/CodeGen/Hexagon/cfi-offset.ll
index c7d447d16..f3ee869fa 100644
--- a/llvm/test/CodeGen/Hexagon/cfi-offset.ll
+++ b/llvm/test/CodeGen/Hexagon/cfi-offset.ll
@@ -39,5 +39,5 @@ declare i8* @__cxa_begin_catch(i8*)
 
 declare void @__cxa_end_catch()
 
-attributes #0 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="hexagonv60" "target-features"="-hvx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="hexagonv60" "target-features"="-hvx" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind }
diff --git a/llvm/test/CodeGen/Hexagon/cmp-extend.ll b/llvm/test/CodeGen/Hexagon/cmp-extend.ll
index 312a5c087..4107e6235 100644
--- a/llvm/test/CodeGen/Hexagon/cmp-extend.ll
+++ b/llvm/test/CodeGen/Hexagon/cmp-extend.ll
@@ -36,5 +36,5 @@ while.body19.i160:
   br label %while.body19.i160
 }
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
diff --git a/llvm/test/CodeGen/Hexagon/cmp.ll b/llvm/test/CodeGen/Hexagon/cmp.ll
index a0bb90de1..acaac04e7 100644
--- a/llvm/test/CodeGen/Hexagon/cmp.ll
+++ b/llvm/test/CodeGen/Hexagon/cmp.ll
@@ -152,7 +152,7 @@ entry:
 ; CHECK: { p{{[0-3]}} = cmp.eq(r{{[0-9]}},r{{[0-9]}})
 
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !llvm.ident = !{!0}
diff --git a/llvm/test/CodeGen/Hexagon/constp-combine-neg.ll b/llvm/test/CodeGen/Hexagon/constp-combine-neg.ll
index 283bd4801..c0cf7a53d 100644
--- a/llvm/test/CodeGen/Hexagon/constp-combine-neg.ll
+++ b/llvm/test/CodeGen/Hexagon/constp-combine-neg.ll
@@ -26,5 +26,5 @@ entry:
 ; CHECK-TEST2: combine(#6,#-4)
 ; CHECK-TEST3: combine(#-10,#-8)
 
-attributes #0 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf"="true" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { "less-precise-fpmad"="false" "frame-pointer"="non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
diff --git a/llvm/test/CodeGen/Hexagon/constp-extract.ll b/llvm/test/CodeGen/Hexagon/constp-extract.ll
index 00f176317..e7b6f959d 100644
--- a/llvm/test/CodeGen/Hexagon/constp-extract.ll
+++ b/llvm/test/CodeGen/Hexagon/constp-extract.ll
@@ -27,5 +27,5 @@ declare i32 @llvm.hexagon.S2.extractu(i32, i32, i32) #1
 
 declare i32 @llvm.hexagon.S4.extract(i32, i32, i32) #1
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf"="true" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
diff --git a/llvm/test/CodeGen/Hexagon/constp-physreg.ll b/llvm/test/CodeGen/Hexagon/constp-physreg.ll
index 0473b96f6..42ea0b360 100644
--- a/llvm/test/CodeGen/Hexagon/constp-physreg.ll
+++ b/llvm/test/CodeGen/Hexagon/constp-physreg.ll
@@ -18,4 +18,4 @@ return:                                           ; preds = %if.end, %if.end15,
   ret i16 %retval.0.reg2mem.0
 }
 
-attributes #0 = { nounwind readnone "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf"="true" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind readnone "less-precise-fpmad"="false" "frame-pointer"="non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/Hexagon/dead-store-stack.ll b/llvm/test/CodeGen/Hexagon/dead-store-stack.ll
index 99f736ee6..a539be440 100644
--- a/llvm/test/CodeGen/Hexagon/dead-store-stack.ll
+++ b/llvm/test/CodeGen/Hexagon/dead-store-stack.ll
@@ -126,7 +126,7 @@ sw.epilog:
 ; Function Attrs: nounwind
 declare void @snprintf(i8* nocapture, i32, i8* nocapture readonly, ...) local_unnamed_addr #1
 
-attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="hexagonv62" "target-features"="+hvx,+hvx-length64b" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="hexagonv62" "target-features"="+hvx,+hvx-length64b" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="hexagonv62" "target-features"="+hvx,+hvx-length64b" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="hexagonv62" "target-features"="+hvx,+hvx-length64b" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind }
 
diff --git a/llvm/test/CodeGen/Hexagon/dealloc_return.ll b/llvm/test/CodeGen/Hexagon/dealloc_return.ll
index 4550b67c4..e750f37df 100644
--- a/llvm/test/CodeGen/Hexagon/dealloc_return.ll
+++ b/llvm/test/CodeGen/Hexagon/dealloc_return.ll
@@ -24,4 +24,4 @@ b0:
   ret i32 %v7
 }
 
-attributes #0 = { nounwind "no-frame-pointer-elim"="true" }
+attributes #0 = { nounwind "frame-pointer"="all" }
diff --git a/llvm/test/CodeGen/Hexagon/early-if-conversion-bug1.ll b/llvm/test/CodeGen/Hexagon/early-if-conversion-bug1.ll
index 804b083e8..4a67a98d1 100644
--- a/llvm/test/CodeGen/Hexagon/early-if-conversion-bug1.ll
+++ b/llvm/test/CodeGen/Hexagon/early-if-conversion-bug1.ll
@@ -404,8 +404,8 @@ unreachable:                                      ; preds = %cleanup
 declare void @_ZNSt3__112basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE6__initEjc(%"class.std::__1::basic_string"*, i32, i8 zeroext) #2
 
 attributes #0 = { nounwind }
-attributes #1 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #2 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.ident = !{!0}
 
diff --git a/llvm/test/CodeGen/Hexagon/ehabi.ll b/llvm/test/CodeGen/Hexagon/ehabi.ll
index 82aae6a42..6389c5cc5 100644
--- a/llvm/test/CodeGen/Hexagon/ehabi.ll
+++ b/llvm/test/CodeGen/Hexagon/ehabi.ll
@@ -75,7 +75,7 @@ declare i8* @f4(i8*)
 
 declare void @f5()
 
-attributes #0 = { "no-frame-pointer-elim"="true" }
+attributes #0 = { "frame-pointer"="all" }
 attributes #1 = { nounwind readnone }
 attributes #2 = { nounwind }
 attributes #3 = { noreturn }
diff --git a/llvm/test/CodeGen/Hexagon/fminmax.ll b/llvm/test/CodeGen/Hexagon/fminmax.ll
index cf1dc6cdf..c90b2a25b 100644
--- a/llvm/test/CodeGen/Hexagon/fminmax.ll
+++ b/llvm/test/CodeGen/Hexagon/fminmax.ll
@@ -22,6 +22,6 @@ entry:
 declare float @fminf(float, float) #0
 declare float @fmaxf(float, float) #0
 
-attributes #0 = { nounwind readnone "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="hexagonv60" "target-features"="+hvx,+hvx-length64b" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind readnone "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="hexagonv60" "target-features"="+hvx,+hvx-length64b" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
diff --git a/llvm/test/CodeGen/Hexagon/fpelim-basic.ll b/llvm/test/CodeGen/Hexagon/fpelim-basic.ll
index ffec07f7d..db926d694 100644
--- a/llvm/test/CodeGen/Hexagon/fpelim-basic.ll
+++ b/llvm/test/CodeGen/Hexagon/fpelim-basic.ll
@@ -85,7 +85,7 @@ b20:                                              ; preds = %b12
 declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture) #2
 declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture) #2
 
-attributes #0 = { nounwind readnone "no-frame-pointer-elim"="false" "target-cpu"="hexagonv60" }
-attributes #1 = { nounwind readnone "no-frame-pointer-elim"="true" "target-cpu"="hexagonv60" }
+attributes #0 = { nounwind readnone "frame-pointer"="none" "target-cpu"="hexagonv60" }
+attributes #1 = { nounwind readnone "frame-pointer"="all" "target-cpu"="hexagonv60" }
 attributes #2 = { argmemonly nounwind }
 attributes #3 = { nounwind }
diff --git a/llvm/test/CodeGen/Hexagon/hasfp-crash1.ll b/llvm/test/CodeGen/Hexagon/hasfp-crash1.ll
index 7afb4bdb2..161b7aba1 100644
--- a/llvm/test/CodeGen/Hexagon/hasfp-crash1.ll
+++ b/llvm/test/CodeGen/Hexagon/hasfp-crash1.ll
@@ -18,7 +18,7 @@ entry:
 ; Function Attrs: nounwind readnone speculatable
 declare void @llvm.dbg.value(metadata, i64, metadata, metadata) #1
 
-attributes #0 = { nounwind "disable-tail-calls"="true" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "stack-protector-buffer-size"="8" "target-cpu"="hexagonv5" "target-features"="-hvx,-long-calls" }
+attributes #0 = { nounwind "disable-tail-calls"="true" "frame-pointer"="all" "stack-protector-buffer-size"="8" "target-cpu"="hexagonv5" "target-features"="-hvx,-long-calls" }
 attributes #1 = { nounwind readnone speculatable }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/CodeGen/Hexagon/hasfp-crash2.ll b/llvm/test/CodeGen/Hexagon/hasfp-crash2.ll
index c6cc4815e..2f690d397 100644
--- a/llvm/test/CodeGen/Hexagon/hasfp-crash2.ll
+++ b/llvm/test/CodeGen/Hexagon/hasfp-crash2.ll
@@ -19,7 +19,7 @@ entry:
 ; Function Attrs: nounwind readnone speculatable
 declare void @llvm.dbg.value(metadata, i64, metadata, metadata) #1
 
-attributes #0 = { nounwind "disable-tail-calls"="true" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "stack-protector-buffer-size"="8" "target-cpu"="hexagonv5" "target-features"=",-hvx,-long-calls" }
+attributes #0 = { nounwind "disable-tail-calls"="true" "frame-pointer"="all" "stack-protector-buffer-size"="8" "target-cpu"="hexagonv5" "target-features"=",-hvx,-long-calls" }
 attributes #1 = { nounwind readnone speculatable }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse.ll b/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse.ll
index ca1ba2fe1..b86cef9ce 100644
--- a/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse.ll
+++ b/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse.ll
@@ -73,7 +73,7 @@ declare <32 x i32> @llvm.hexagon.V6.vmaxub.128B(<32 x i32>, <32 x i32>) #1
 ; Function Attrs: nounwind readnone
 declare <32 x i32> @llvm.hexagon.V6.valignbi.128B(<32 x i32>, <32 x i32>, i32) #1
 
-attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="hexagonv60" "target-features"="+hvx,+hvx-length128b,-long-calls" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="hexagonv60" "target-features"="+hvx,+hvx-length128b,-long-calls" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !llvm.ident = !{!0}
diff --git a/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse_constant.ll b/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse_constant.ll
index 8fb62b3fa..205b0786a 100644
--- a/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse_constant.ll
+++ b/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse_constant.ll
@@ -73,7 +73,7 @@ declare <32 x i32> @llvm.hexagon.V6.vmaxub.128B(<32 x i32>, <32 x i32>) #1
 ; Function Attrs: nounwind readnone
 declare <32 x i32> @llvm.hexagon.V6.valignbi.128B(<32 x i32>, <32 x i32>, i32) #1
 
-attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="hexagonv60" "target-features"="+hvx,+hvx-length128b,-long-calls" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="hexagonv60" "target-features"="+hvx,+hvx-length128b,-long-calls" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !llvm.ident = !{!0}
diff --git a/llvm/test/CodeGen/Hexagon/hwloop-crit-edge.ll b/llvm/test/CodeGen/Hexagon/hwloop-crit-edge.ll
index f6e08eaf2..af64a63c2 100644
--- a/llvm/test/CodeGen/Hexagon/hwloop-crit-edge.ll
+++ b/llvm/test/CodeGen/Hexagon/hwloop-crit-edge.ll
@@ -55,5 +55,5 @@ declare i64 @llvm.hexagon.S2.asl.r.p(i64, i32) #1
 
 declare i64 @llvm.hexagon.M2.dpmpyss.s0(i32, i32) #1
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf"="true" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "ssp-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "ssp-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
diff --git a/llvm/test/CodeGen/Hexagon/hwloop-preheader.ll b/llvm/test/CodeGen/Hexagon/hwloop-preheader.ll
index 66efd2089..19a8ae1a8 100644
--- a/llvm/test/CodeGen/Hexagon/hwloop-preheader.ll
+++ b/llvm/test/CodeGen/Hexagon/hwloop-preheader.ll
@@ -37,4 +37,4 @@ for.end22:
   ret void
 }
 
-attributes #0 = { nounwind readonly "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf"="true" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind readonly "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/Hexagon/insert4.ll b/llvm/test/CodeGen/Hexagon/insert4.ll
index 3bc8e9e57..3a0f9b999 100644
--- a/llvm/test/CodeGen/Hexagon/insert4.ll
+++ b/llvm/test/CodeGen/Hexagon/insert4.ll
@@ -104,7 +104,7 @@ declare i64 @llvm.hexagon.M2.dpmpyss.s0(i32, i32) #1
 
 declare i64 @llvm.hexagon.A2.vaddhs(i64, i64) #1
 
-attributes #0 = { nounwind "fp-contract-model"="standard" "no-frame-pointer-elim-non-leaf" "realign-stack" "relocation-model"="static" "ssp-buffers-size"="8" }
+attributes #0 = { nounwind "fp-contract-model"="standard" "frame-pointer"="non-leaf" "realign-stack" "relocation-model"="static" "ssp-buffers-size"="8" }
 attributes #1 = { nounwind readnone }
 
 !0 = !{!"short", !1}
diff --git a/llvm/test/CodeGen/Hexagon/isel-bitcast-v8i1-i8.ll b/llvm/test/CodeGen/Hexagon/isel-bitcast-v8i1-i8.ll
new file mode 100644
index 000000000..37b82433d
--- /dev/null
+++ b/llvm/test/CodeGen/Hexagon/isel-bitcast-v8i1-i8.ll
@@ -0,0 +1,18 @@
+; RUN: llc -march=hexagon < %s | FileCheck %s
+; REQUIRES: asserts
+
+; Check that this doesn't crash.
+; CHECK-LABEL: foo:
+; CHECK: p[[P:[0-3]]] = vcmpb.eq
+; CHECK: r[[R:[0-9]+]] = p[[P]]
+; CHECK: and(r[[R]],#32)
+
+define i32 @foo(<8 x i8> %a0, <8 x i8> %a1) #0 {
+  %v0 = icmp eq <8 x i8> %a0, %a1
+  %v1 = bitcast <8 x i1> %v0 to i8
+  %v2 = and i8 %v1, 32
+  %v3 = zext i8 %v2 to i32
+  ret i32 %v3
+}
+
+attributes #0 = { readnone nounwind }
diff --git a/llvm/test/CodeGen/Hexagon/isel-bitcast-v8i8-v4i16.ll b/llvm/test/CodeGen/Hexagon/isel-bitcast-v8i8-v4i16.ll
new file mode 100644
index 000000000..f45b1a77c
--- /dev/null
+++ b/llvm/test/CodeGen/Hexagon/isel-bitcast-v8i8-v4i16.ll
@@ -0,0 +1,13 @@
+; RUN: llc -march=hexagon < %s | FileCheck %s
+
+; Check that this doesn't fail to select instructions.
+; CHECK: vsplath
+
+define <8 x i8> @fred(i16 %a0) #0 {
+  %t0 = insertelement <4 x i16> undef, i16 %a0, i32 0
+  %t1 = shufflevector <4 x i16> %t0, <4 x i16> undef, <4 x i32> zeroinitializer
+  %t2 = bitcast <4 x i16> %t1 to <8 x i8>
+  ret <8 x i8> %t2
+}
+
+attributes #0 = { readnone nounwind "target-cpu"="hexagonv62" }
diff --git a/llvm/test/CodeGen/Hexagon/isel-minmax-v64bit.ll b/llvm/test/CodeGen/Hexagon/isel-minmax-v64bit.ll
new file mode 100644
index 000000000..53dec510a
--- /dev/null
+++ b/llvm/test/CodeGen/Hexagon/isel-minmax-v64bit.ll
@@ -0,0 +1,202 @@
+; RUN: llc -march=hexagon < %s | FileCheck %s
+
+; min
+
+; CHECK-LABEL: test_00:
+; CHECK: r1:0 = vminb(r3:2,r1:0)
+define <8 x i8> @test_00(<8 x i8> %a0, <8 x i8> %a1) #0 {
+  %v0 = icmp slt <8 x i8> %a0, %a1
+  %v1 = select <8 x i1> %v0, <8 x i8> %a0, <8 x i8> %a1
+  ret <8 x i8> %v1
+}
+
+; CHECK-LABEL: test_01:
+; CHECK: r1:0 = vminb(r1:0,r3:2)
+define <8 x i8> @test_01(<8 x i8> %a0, <8 x i8> %a1) #0 {
+  %v0 = icmp sle <8 x i8> %a0, %a1
+  %v1 = select <8 x i1> %v0, <8 x i8> %a0, <8 x i8> %a1
+  ret <8 x i8> %v1
+}
+
+; CHECK-LABEL: test_02:
+; CHECK: r1:0 = vminh(r3:2,r1:0)
+define <4 x i16> @test_02(<4 x i16> %a0, <4 x i16> %a1) #0 {
+  %v0 = icmp slt <4 x i16> %a0, %a1
+  %v1 = select <4 x i1> %v0, <4 x i16> %a0, <4 x i16> %a1
+  ret <4 x i16> %v1
+}
+
+; CHECK-LABEL: test_03:
+; CHECK: r1:0 = vminh(r1:0,r3:2)
+define <4 x i16> @test_03(<4 x i16> %a0, <4 x i16> %a1) #0 {
+  %v0 = icmp sle <4 x i16> %a0, %a1
+  %v1 = select <4 x i1> %v0, <4 x i16> %a0, <4 x i16> %a1
+  ret <4 x i16> %v1
+}
+
+; CHECK-LABEL: test_04:
+; CHECK: r1:0 = vminw(r3:2,r1:0)
+define <2 x i32> @test_04(<2 x i32> %a0, <2 x i32> %a1) #0 {
+  %v0 = icmp slt <2 x i32> %a0, %a1
+  %v1 = select <2 x i1> %v0, <2 x i32> %a0, <2 x i32> %a1
+  ret <2 x i32> %v1
+}
+
+; CHECK-LABEL: test_05:
+; CHECK: r1:0 = vminw(r1:0,r3:2)
+define <2 x i32> @test_05(<2 x i32> %a0, <2 x i32> %a1) #0 {
+  %v0 = icmp sle <2 x i32> %a0, %a1
+  %v1 = select <2 x i1> %v0, <2 x i32> %a0, <2 x i32> %a1
+  ret <2 x i32> %v1
+}
+
+; minu
+
+; CHECK-LABEL: test_06:
+; CHECK: r1:0 = vminub(r3:2,r1:0)
+define <8 x i8> @test_06(<8 x i8> %a0, <8 x i8> %a1) #0 {
+  %v0 = icmp ult <8 x i8> %a0, %a1
+  %v1 = select <8 x i1> %v0, <8 x i8> %a0, <8 x i8> %a1
+  ret <8 x i8> %v1
+}
+
+; CHECK-LABEL: test_07:
+; CHECK: r1:0 = vminub(r1:0,r3:2)
+define <8 x i8> @test_07(<8 x i8> %a0, <8 x i8> %a1) #0 {
+  %v0 = icmp ule <8 x i8> %a0, %a1
+  %v1 = select <8 x i1> %v0, <8 x i8> %a0, <8 x i8> %a1
+  ret <8 x i8> %v1
+}
+
+; CHECK-LABEL: test_08:
+; CHECK: r1:0 = vminuh(r3:2,r1:0)
+define <4 x i16> @test_08(<4 x i16> %a0, <4 x i16> %a1) #0 {
+  %v0 = icmp ult <4 x i16> %a0, %a1
+  %v1 = select <4 x i1> %v0, <4 x i16> %a0, <4 x i16> %a1
+  ret <4 x i16> %v1
+}
+
+; CHECK-LABEL: test_09:
+; CHECK: r1:0 = vminuh(r1:0,r3:2)
+define <4 x i16> @test_09(<4 x i16> %a0, <4 x i16> %a1) #0 {
+  %v0 = icmp ule <4 x i16> %a0, %a1
+  %v1 = select <4 x i1> %v0, <4 x i16> %a0, <4 x i16> %a1
+  ret <4 x i16> %v1
+}
+
+; CHECK-LABEL: test_0a:
+; CHECK: r1:0 = vminuw(r3:2,r1:0)
+define <2 x i32> @test_0a(<2 x i32> %a0, <2 x i32> %a1) #0 {
+  %v0 = icmp ult <2 x i32> %a0, %a1
+  %v1 = select <2 x i1> %v0, <2 x i32> %a0, <2 x i32> %a1
+  ret <2 x i32> %v1
+}
+
+; CHECK-LABEL: test_0b:
+; CHECK: r1:0 = vminuw(r1:0,r3:2)
+define <2 x i32> @test_0b(<2 x i32> %a0, <2 x i32> %a1) #0 {
+  %v0 = icmp ule <2 x i32> %a0, %a1
+  %v1 = select <2 x i1> %v0, <2 x i32> %a0, <2 x i32> %a1
+  ret <2 x i32> %v1
+}
+
+; max
+
+; CHECK-LABEL: test_0c:
+; CHECK: r1:0 = vmaxb(r1:0,r3:2)
+define <8 x i8> @test_0c(<8 x i8> %a0, <8 x i8> %a1) #0 {
+  %v0 = icmp sgt <8 x i8> %a0, %a1
+  %v1 = select <8 x i1> %v0, <8 x i8> %a0, <8 x i8> %a1
+  ret <8 x i8> %v1
+}
+
+; CHECK-LABEL: test_0d:
+; CHECK: r1:0 = vmaxb(r3:2,r1:0)
+define <8 x i8> @test_0d(<8 x i8> %a0, <8 x i8> %a1) #0 {
+  %v0 = icmp sge <8 x i8> %a0, %a1
+  %v1 = select <8 x i1> %v0, <8 x i8> %a0, <8 x i8> %a1
+  ret <8 x i8> %v1
+}
+
+; CHECK-LABEL: test_0e:
+; CHECK: r1:0 = vmaxh(r1:0,r3:2)
+define <4 x i16> @test_0e(<4 x i16> %a0, <4 x i16> %a1) #0 {
+  %v0 = icmp sgt <4 x i16> %a0, %a1
+  %v1 = select <4 x i1> %v0, <4 x i16> %a0, <4 x i16> %a1
+  ret <4 x i16> %v1
+}
+
+; CHECK-LABEL: test_0f:
+; CHECK: r1:0 = vmaxh(r3:2,r1:0)
+define <4 x i16> @test_0f(<4 x i16> %a0, <4 x i16> %a1) #0 {
+  %v0 = icmp sge <4 x i16> %a0, %a1
+  %v1 = select <4 x i1> %v0, <4 x i16> %a0, <4 x i16> %a1
+  ret <4 x i16> %v1
+}
+
+; CHECK-LABEL: test_10:
+; CHECK: r1:0 = vmaxw(r1:0,r3:2)
+define <2 x i32> @test_10(<2 x i32> %a0, <2 x i32> %a1) #0 {
+  %v0 = icmp sgt <2 x i32> %a0, %a1
+  %v1 = select <2 x i1> %v0, <2 x i32> %a0, <2 x i32> %a1
+  ret <2 x i32> %v1
+}
+
+; CHECK-LABEL: test_11:
+; CHECK: r1:0 = vmaxw(r3:2,r1:0)
+define <2 x i32> @test_11(<2 x i32> %a0, <2 x i32> %a1) #0 {
+  %v0 = icmp sge <2 x i32> %a0, %a1
+  %v1 = select <2 x i1> %v0, <2 x i32> %a0, <2 x i32> %a1
+  ret <2 x i32> %v1
+}
+
+; maxu
+
+; CHECK-LABEL: test_12:
+; CHECK: r1:0 = vmaxub(r1:0,r3:2)
+define <8 x i8> @test_12(<8 x i8> %a0, <8 x i8> %a1) #0 {
+  %v0 = icmp ugt <8 x i8> %a0, %a1
+  %v1 = select <8 x i1> %v0, <8 x i8> %a0, <8 x i8> %a1
+  ret <8 x i8> %v1
+}
+
+; CHECK-LABEL: test_13:
+; CHECK: r1:0 = vmaxub(r3:2,r1:0)
+define <8 x i8> @test_13(<8 x i8> %a0, <8 x i8> %a1) #0 {
+  %v0 = icmp uge <8 x i8> %a0, %a1
+  %v1 = select <8 x i1> %v0, <8 x i8> %a0, <8 x i8> %a1
+  ret <8 x i8> %v1
+}
+
+; CHECK-LABEL: test_14:
+; CHECK: r1:0 = vmaxuh(r1:0,r3:2)
+define <4 x i16> @test_14(<4 x i16> %a0, <4 x i16> %a1) #0 {
+  %v0 = icmp ugt <4 x i16> %a0, %a1
+  %v1 = select <4 x i1> %v0, <4 x i16> %a0, <4 x i16> %a1
+  ret <4 x i16> %v1
+}
+
+; CHECK-LABEL: test_15:
+; CHECK: r1:0 = vmaxuh(r3:2,r1:0)
+define <4 x i16> @test_15(<4 x i16> %a0, <4 x i16> %a1) #0 {
+  %v0 = icmp uge <4 x i16> %a0, %a1
+  %v1 = select <4 x i1> %v0, <4 x i16> %a0, <4 x i16> %a1
+  ret <4 x i16> %v1
+}
+
+; CHECK-LABEL: test_16:
+; CHECK: r1:0 = vmaxuw(r1:0,r3:2)
+define <2 x i32> @test_16(<2 x i32> %a0, <2 x i32> %a1) #0 {
+  %v0 = icmp ugt <2 x i32> %a0, %a1
+  %v1 = select <2 x i1> %v0, <2 x i32> %a0, <2 x i32> %a1
+  ret <2 x i32> %v1
+}
+
+; CHECK-LABEL: test_17:
+; CHECK: r1:0 = vmaxuw(r3:2,r1:0)
+define <2 x i32> @test_17(<2 x i32> %a0, <2 x i32> %a1) #0 {
+  %v0 = icmp uge <2 x i32> %a0, %a1
+  %v1 = select <2 x i1> %v0, <2 x i32> %a0, <2 x i32> %a1
+  ret <2 x i32> %v1
+}
+
diff --git a/llvm/test/CodeGen/Hexagon/isel-prefer.ll b/llvm/test/CodeGen/Hexagon/isel-prefer.ll
index 1b69f5691..130ea0406 100644
--- a/llvm/test/CodeGen/Hexagon/isel-prefer.ll
+++ b/llvm/test/CodeGen/Hexagon/isel-prefer.ll
@@ -80,8 +80,8 @@ b2:
   ret i32 %v6
 }
 
-define i32 @Prefer_S2_ntstbit_r(i32 %a0, i32 %a1) #0 {
-; CHECK-LABEL: Prefer_S2_ntstbit_r:
+define i32 @Prefer_S4_ntstbit_r(i32 %a0, i32 %a1) #0 {
+; CHECK-LABEL: Prefer_S4_ntstbit_r:
 ; CHECK:       // %bb.0: // %b2
 ; CHECK-NEXT:    {
 ; CHECK-NEXT:     p0 = !tstbit(r0,r1)
diff --git a/llvm/test/CodeGen/Hexagon/isel-vselect-v4i8.ll b/llvm/test/CodeGen/Hexagon/isel-vselect-v4i8.ll
new file mode 100644
index 000000000..d895f129e
--- /dev/null
+++ b/llvm/test/CodeGen/Hexagon/isel-vselect-v4i8.ll
@@ -0,0 +1,9 @@
+; RUN: llc -march=hexagon < %s | FileCheck %s
+; This used to crash with "cannot select (v4i8 vselect ...)"
+; CHECK: vtrunehb
+
+define <4 x i8> @f0(<4 x i8> %a0, <4 x i8> %a1) #0 {
+  %v0 = icmp slt <4 x i8> %a0, %a1
+  %v1 = select <4 x i1> %v0, <4 x i8> %a0, <4 x i8> %a1
+  ret <4 x i8> %v1
+}
diff --git a/llvm/test/CodeGen/Hexagon/jt-in-text.ll b/llvm/test/CodeGen/Hexagon/jt-in-text.ll
index 7389c960b..95c5d95c9 100644
--- a/llvm/test/CodeGen/Hexagon/jt-in-text.ll
+++ b/llvm/test/CodeGen/Hexagon/jt-in-text.ll
@@ -54,4 +54,4 @@ sw.epilog:                                        ; preds = %entry, %sw.bb4, %sw
   ret void
 }
 
-attributes #0 = { noinline nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="hexagonv60" "target-features"="-hvx,-long-calls" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noinline nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="hexagonv60" "target-features"="-hvx,-long-calls" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/Hexagon/mem-fi-add.ll b/llvm/test/CodeGen/Hexagon/mem-fi-add.ll
index 4ec62c581..6bda29947 100644
--- a/llvm/test/CodeGen/Hexagon/mem-fi-add.ll
+++ b/llvm/test/CodeGen/Hexagon/mem-fi-add.ll
@@ -24,6 +24,6 @@ declare void @llvm.memset.p0i8.i32(i8* nocapture, i8, i32, i1) #1
 
 declare void @bar([2 x i32]*) #2
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind }
-attributes #2 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/Hexagon/memops-stack.ll b/llvm/test/CodeGen/Hexagon/memops-stack.ll
index 9da319f44..d4bba476d 100644
--- a/llvm/test/CodeGen/Hexagon/memops-stack.ll
+++ b/llvm/test/CodeGen/Hexagon/memops-stack.ll
@@ -136,9 +136,9 @@ declare void @foo(i32*) #2
 declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture) #1
 declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture) #1
 
-attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="hexagonv60" "target-features"="+hvx,+hvx-length64b" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="hexagonv60" "target-features"="+hvx,+hvx-length64b" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { argmemonly nounwind }
-attributes #2 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="hexagonv60" "target-features"="+hvx,+hvx-length64b" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="hexagonv60" "target-features"="+hvx,+hvx-length64b" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #3 = { nounwind }
 
 !1 = !{!2, !2, i64 0}
diff --git a/llvm/test/CodeGen/Hexagon/muxii-bug.ll b/llvm/test/CodeGen/Hexagon/muxii-bug.ll
new file mode 100644
index 000000000..7267efe9e
--- /dev/null
+++ b/llvm/test/CodeGen/Hexagon/muxii-bug.ll
@@ -0,0 +1,30 @@
+; RUN: llc -march=hexagon < %s | FileCheck %s
+
+; Make sure "generate mux" pass does not optimize out the value "1908".
+; CHECK-LABEL: foo
+; CHECK: 1908
+define internal fastcc i32 @foo(i32) #0 {
+  %2 = icmp eq i32 %0, 1
+  %3 = select i1 %2, i32 1712, i32 0
+  %4 = icmp eq i32 %0, 1
+  %5 = select i1 %4, i32 1908, i32 %3
+  %6 = icmp eq i32 %0, 1
+  %7 = icmp ult i32 %5, 1740
+  %8 = and i1 %6, %7
+  %9 = select i1 %8, i32 1740, i32 %5
+  %10 = icmp eq i32 %0, 1
+  %11 = icmp ult i32 %9, 1732
+  %12 = and i1 %10, %11
+  %13 = select i1 %12, i32 1732, i32 %9
+  %14 = icmp eq i32 %0, 2
+  %15 = icmp ult i32 %13, 1936
+  %16 = and i1 %14, %15
+  %17 = select i1 %16, i32 1936, i32 %13
+  %18 = icmp eq i32 %0, 1
+  %19 = icmp ult i32 %17, 1580
+  %20 = and i1 %18, %19
+  %21 = select i1 %20, i32 1580, i32 %17
+  ret i32 %21
+}
+
+attributes #0 = { nounwind }
diff --git a/llvm/test/CodeGen/Hexagon/packetizer-resources.ll b/llvm/test/CodeGen/Hexagon/packetizer-resources.ll
new file mode 100644
index 000000000..6b705396b
--- /dev/null
+++ b/llvm/test/CodeGen/Hexagon/packetizer-resources.ll
@@ -0,0 +1,29 @@
+; RUN: llc -O2 -march=hexagon < %s -debug-only=packets 2>&1 | FileCheck %s
+; REQUIRES: asserts
+
+; CHECK: Finalizing packet:
+; CHECK-NEXT: * [res:0x4] renamable $r1 = S2_vsplatrb renamable $r0
+; CHECK-NEXT: * [res:0x8] renamable $d1 = S2_vsplatrh killed renamable $r0
+
+target triple = "hexagon"
+
+; Function Attrs: nounwind readnone
+define i64 @f0(i64 %a0) #0 {
+b0:
+  %v0 = trunc i64 %a0 to i32
+  %v1 = and i32 %v0, 65535
+  %v2 = tail call i64 @llvm.hexagon.S2.vsplatrh(i32 %v1)
+  %v3 = and i32 %v0, 255
+  %v4 = tail call i32 @llvm.hexagon.S2.vsplatrb(i32 %v3)
+  %v5 = sext i32 %v4 to i64
+  %v6 = add nsw i64 %v5, %v2
+  ret i64 %v6
+}
+
+; Function Attrs: nounwind readnone
+declare i64 @llvm.hexagon.S2.vsplatrh(i32) #0
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.hexagon.S2.vsplatrb(i32) #0
+
+attributes #0 = { nounwind readnone }
diff --git a/llvm/test/CodeGen/Hexagon/pic-regusage.ll b/llvm/test/CodeGen/Hexagon/pic-regusage.ll
index e6cf1cfa6..9d3b6cec3 100644
--- a/llvm/test/CodeGen/Hexagon/pic-regusage.ll
+++ b/llvm/test/CodeGen/Hexagon/pic-regusage.ll
@@ -64,6 +64,6 @@ entry:
   ret i32 0
 }
 
-attributes #0 = { nounwind optsize "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind optsize "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind }
 attributes #2 = { nounwind optsize }
diff --git a/llvm/test/CodeGen/Hexagon/postinc-offset.ll b/llvm/test/CodeGen/Hexagon/postinc-offset.ll
index cf2031b4e..4173de289 100644
--- a/llvm/test/CodeGen/Hexagon/postinc-offset.ll
+++ b/llvm/test/CodeGen/Hexagon/postinc-offset.ll
@@ -35,7 +35,7 @@ b4:                                               ; preds = %b4, %b0
   br i1 %v8, label %b4, label %b3
 }
 
-attributes #0 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf"="true" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !0 = !{!1, !1, i64 0}
 !1 = !{!"int", !2}
diff --git a/llvm/test/CodeGen/Hexagon/rdf-dead-loop.ll b/llvm/test/CodeGen/Hexagon/rdf-dead-loop.ll
index 3762c79d4..656c15a58 100644
--- a/llvm/test/CodeGen/Hexagon/rdf-dead-loop.ll
+++ b/llvm/test/CodeGen/Hexagon/rdf-dead-loop.ll
@@ -24,7 +24,7 @@ exit:
 
 }
 
-attributes #0 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf"="true" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !0 = !{!"int", !1}
 !1 = !{!"omnipotent char", !2}
diff --git a/llvm/test/CodeGen/Hexagon/rdf-inline-asm-fixed.ll b/llvm/test/CodeGen/Hexagon/rdf-inline-asm-fixed.ll
index d06da9346..bbd05ae0b 100644
--- a/llvm/test/CodeGen/Hexagon/rdf-inline-asm-fixed.ll
+++ b/llvm/test/CodeGen/Hexagon/rdf-inline-asm-fixed.ll
@@ -26,7 +26,7 @@ declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture) #1
 ; Function Attrs: argmemonly nounwind
 declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture) #1
 
-attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="hexagonv5" "target-features"="-hvx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="hexagonv5" "target-features"="-hvx" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { argmemonly nounwind }
 attributes #2 = { nounwind }
 
diff --git a/llvm/test/CodeGen/Hexagon/rdf-inline-asm.ll b/llvm/test/CodeGen/Hexagon/rdf-inline-asm.ll
index 2661f8c0d..fcba80d2c 100644
--- a/llvm/test/CodeGen/Hexagon/rdf-inline-asm.ll
+++ b/llvm/test/CodeGen/Hexagon/rdf-inline-asm.ll
@@ -24,7 +24,7 @@ if.end:                                           ; preds = %if.then, %entry
   ret i32 %retval1.0
 }
 
-attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="hexagonv60" "target-features"="+hvx,+hvx-length64b" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="hexagonv60" "target-features"="+hvx,+hvx-length64b" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind }
 
 !1 = !{i32 155}
diff --git a/llvm/test/CodeGen/Hexagon/reg-by-name.ll b/llvm/test/CodeGen/Hexagon/reg-by-name.ll
new file mode 100644
index 000000000..4abea83ba
--- /dev/null
+++ b/llvm/test/CodeGen/Hexagon/reg-by-name.ll
@@ -0,0 +1,780 @@
+; RUN: llc -march=hexagon < %s | FileCheck %s
+
+target triple = "hexagon"
+
+declare void @llvm.write_register.i32(metadata, i32) #1
+declare void @llvm.write_register.i64(metadata, i64) #1
+declare i32 @llvm.read_register.i32(metadata) #2
+declare i64 @llvm.read_register.i64(metadata) #2
+
+; CHECK-LABEL: reg_r0:
+; CHECK: r0 = #1
+define dso_local i32 @reg_r0() #0 {
+entry:
+  call void @llvm.write_register.i32(metadata !0, i32 1)
+  %0 = call i32 @llvm.read_register.i32(metadata !0)
+  ret i32 %0
+}
+
+; CHECK-LABEL: reg_r1:
+; CHECK: r1 = #1
+; CHECK: r0 = r1
+define dso_local i32 @reg_r1() #0 {
+entry:
+  call void @llvm.write_register.i32(metadata !1, i32 1)
+  %0 = call i32 @llvm.read_register.i32(metadata !1)
+  ret i32 %0
+}
+
+; CHECK-LABEL: reg_r2:
+; CHECK: r2 = #1
+; CHECK: r0 = r2
+define dso_local i32 @reg_r2() #0 {
+entry:
+  call void @llvm.write_register.i32(metadata !2, i32 1)
+  %0 = call i32 @llvm.read_register.i32(metadata !2)
+  ret i32 %0
+}
+
+; CHECK-LABEL: reg_r3:
+; CHECK: r3 = #1
+; CHECK: r0 = r3
+define dso_local i32 @reg_r3() #0 {
+entry:
+  call void @llvm.write_register.i32(metadata !3, i32 1)
+  %0 = call i32 @llvm.read_register.i32(metadata !3)
+  ret i32 %0
+}
+
+; CHECK-LABEL: reg_r4:
+; CHECK: r4 = #1
+; CHECK: r0 = r4
+define dso_local i32 @reg_r4() #0 {
+entry:
+  call void @llvm.write_register.i32(metadata !4, i32 1)
+  %0 = call i32 @llvm.read_register.i32(metadata !4)
+  ret i32 %0
+}
+
+; CHECK-LABEL: reg_r5:
+; CHECK: r5 = #1
+; CHECK: r0 = r5
+define dso_local i32 @reg_r5() #0 {
+entry:
+  call void @llvm.write_register.i32(metadata !5, i32 1)
+  %0 = call i32 @llvm.read_register.i32(metadata !5)
+  ret i32 %0
+}
+
+; CHECK-LABEL: reg_r6:
+; CHECK: r6 = #1
+; CHECK: r0 = r6
+define dso_local i32 @reg_r6() #0 {
+entry:
+  call void @llvm.write_register.i32(metadata !6, i32 1)
+  %0 = call i32 @llvm.read_register.i32(metadata !6)
+  ret i32 %0
+}
+
+; CHECK-LABEL: reg_r7:
+; CHECK: r7 = #1
+; CHECK: r0 = r7
+define dso_local i32 @reg_r7() #0 {
+entry:
+  call void @llvm.write_register.i32(metadata !7, i32 1)
+  %0 = call i32 @llvm.read_register.i32(metadata !7)
+  ret i32 %0
+}
+
+; CHECK-LABEL: reg_r8:
+; CHECK: r8 = #1
+; CHECK: r0 = r8
+define dso_local i32 @reg_r8() #0 {
+entry:
+  call void @llvm.write_register.i32(metadata !8, i32 1)
+  %0 = call i32 @llvm.read_register.i32(metadata !8)
+  ret i32 %0
+}
+
+; CHECK-LABEL: reg_r9:
+; CHECK: r9 = #1
+; CHECK: r0 = r9
+define dso_local i32 @reg_r9() #0 {
+entry:
+  call void @llvm.write_register.i32(metadata !9, i32 1)
+  %0 = call i32 @llvm.read_register.i32(metadata !9)
+  ret i32 %0
+}
+
+; CHECK-LABEL: reg_r10:
+; CHECK: r10 = #1
+; CHECK: r0 = r10
+define dso_local i32 @reg_r10() #0 {
+entry:
+  call void @llvm.write_register.i32(metadata !10, i32 1)
+  %0 = call i32 @llvm.read_register.i32(metadata !10)
+  ret i32 %0
+}
+
+; CHECK-LABEL: reg_r11:
+; CHECK: r11 = #1
+; CHECK: r0 = r11
+define dso_local i32 @reg_r11() #0 {
+entry:
+  call void @llvm.write_register.i32(metadata !11, i32 1)
+  %0 = call i32 @llvm.read_register.i32(metadata !11)
+  ret i32 %0
+}
+
+; CHECK-LABEL: reg_r12:
+; CHECK: r12 = #1
+; CHECK: r0 = r12
+define dso_local i32 @reg_r12() #0 {
+entry:
+  call void @llvm.write_register.i32(metadata !12, i32 1)
+  %0 = call i32 @llvm.read_register.i32(metadata !12)
+  ret i32 %0
+}
+
+; CHECK-LABEL: reg_r13:
+; CHECK: r13 = #1
+; CHECK: r0 = r13
+define dso_local i32 @reg_r13() #0 {
+entry:
+  call void @llvm.write_register.i32(metadata !13, i32 1)
+  %0 = call i32 @llvm.read_register.i32(metadata !13)
+  ret i32 %0
+}
+
+; CHECK-LABEL: reg_r14:
+; CHECK: r14 = #1
+; CHECK: r0 = r14
+define dso_local i32 @reg_r14() #0 {
+entry:
+  call void @llvm.write_register.i32(metadata !14, i32 1)
+  %0 = call i32 @llvm.read_register.i32(metadata !14)
+  ret i32 %0
+}
+
+; CHECK-LABEL: reg_r15:
+; CHECK: r15 = #1
+; CHECK: r0 = r15
+define dso_local i32 @reg_r15() #0 {
+entry:
+  call void @llvm.write_register.i32(metadata !15, i32 1)
+  %0 = call i32 @llvm.read_register.i32(metadata !15)
+  ret i32 %0
+}
+
+; CHECK-LABEL: reg_r16:
+; CHECK: r16 = #1
+; CHECK: r0 = r16
+define dso_local i32 @reg_r16() #0 {
+entry:
+  call void @llvm.write_register.i32(metadata !16, i32 1)
+  %0 = call i32 @llvm.read_register.i32(metadata !16)
+  ret i32 %0
+}
+
+; CHECK-LABEL: reg_r17:
+; CHECK: r17 = #1
+; CHECK: r0 = r17
+define dso_local i32 @reg_r17() #0 {
+entry:
+  call void @llvm.write_register.i32(metadata !17, i32 1)
+  %0 = call i32 @llvm.read_register.i32(metadata !17)
+  ret i32 %0
+}
+
+; CHECK-LABEL: reg_r18:
+; CHECK: r18 = #1
+; CHECK: r0 = r18
+define dso_local i32 @reg_r18() #0 {
+entry:
+  call void @llvm.write_register.i32(metadata !18, i32 1)
+  %0 = call i32 @llvm.read_register.i32(metadata !18)
+  ret i32 %0
+}
+
+; CHECK-LABEL: reg_r19:
+; CHECK: r19 = #1
+; CHECK: r0 = r19
+define dso_local i32 @reg_r19() #0 {
+entry:
+  call void @llvm.write_register.i32(metadata !19, i32 1)
+  %0 = call i32 @llvm.read_register.i32(metadata !19)
+  ret i32 %0
+}
+
+; CHECK-LABEL: reg_r20:
+; CHECK: r20 = #1
+; CHECK: r0 = r20
+define dso_local i32 @reg_r20() #0 {
+entry:
+  call void @llvm.write_register.i32(metadata !20, i32 1)
+  %0 = call i32 @llvm.read_register.i32(metadata !20)
+  ret i32 %0
+}
+
+; CHECK-LABEL: reg_r21:
+; CHECK: r21 = #1
+; CHECK: r0 = r21
+define dso_local i32 @reg_r21() #0 {
+entry:
+  call void @llvm.write_register.i32(metadata !21, i32 1)
+  %0 = call i32 @llvm.read_register.i32(metadata !21)
+  ret i32 %0
+}
+
+; CHECK-LABEL: reg_r22:
+; CHECK: r22 = #1
+; CHECK: r0 = r22
+define dso_local i32 @reg_r22() #0 {
+entry:
+  call void @llvm.write_register.i32(metadata !22, i32 1)
+  %0 = call i32 @llvm.read_register.i32(metadata !22)
+  ret i32 %0
+}
+
+; CHECK-LABEL: reg_r23:
+; CHECK: r23 = #1
+; CHECK: r0 = r23
+define dso_local i32 @reg_r23() #0 {
+entry:
+  call void @llvm.write_register.i32(metadata !23, i32 1)
+  %0 = call i32 @llvm.read_register.i32(metadata !23)
+  ret i32 %0
+}
+
+; CHECK-LABEL: reg_r24:
+; CHECK: r24 = #1
+; CHECK: r0 = r24
+define dso_local i32 @reg_r24() #0 {
+entry:
+  call void @llvm.write_register.i32(metadata !24, i32 1)
+  %0 = call i32 @llvm.read_register.i32(metadata !24)
+  ret i32 %0
+}
+
+; CHECK-LABEL: reg_r25:
+; CHECK: r25 = #1
+; CHECK: r0 = r25
+define dso_local i32 @reg_r25() #0 {
+entry:
+  call void @llvm.write_register.i32(metadata !25, i32 1)
+  %0 = call i32 @llvm.read_register.i32(metadata !25)
+  ret i32 %0
+}
+
+; CHECK-LABEL: reg_r26:
+; CHECK: r26 = #1
+; CHECK: r0 = r26
+define dso_local i32 @reg_r26() #0 {
+entry:
+  call void @llvm.write_register.i32(metadata !26, i32 1)
+  %0 = call i32 @llvm.read_register.i32(metadata !26)
+  ret i32 %0
+}
+
+; CHECK-LABEL: reg_r27:
+; CHECK: r27 = #1
+; CHECK: r0 = r27
+define dso_local i32 @reg_r27() #0 {
+entry:
+  call void @llvm.write_register.i32(metadata !27, i32 1)
+  %0 = call i32 @llvm.read_register.i32(metadata !27)
+  ret i32 %0
+}
+
+; CHECK-LABEL: reg_r28:
+; CHECK: r28 = #1
+; CHECK: r0 = r28
+define dso_local i32 @reg_r28() #0 {
+entry:
+  call void @llvm.write_register.i32(metadata !28, i32 1)
+  %0 = call i32 @llvm.read_register.i32(metadata !28)
+  ret i32 %0
+}
+
+; CHECK-LABEL: reg_r29:
+; CHECK: r29 = #1
+; CHECK: r0 = r29
+define dso_local i32 @reg_r29() #0 {
+entry:
+  call void @llvm.write_register.i32(metadata !29, i32 1)
+  %0 = call i32 @llvm.read_register.i32(metadata !29)
+  ret i32 %0
+}
+
+; CHECK-LABEL: reg_r30:
+; CHECK: r30 = #1
+; CHECK: r0 = r30
+define dso_local i32 @reg_r30() #0 {
+entry:
+  call void @llvm.write_register.i32(metadata !30, i32 1)
+  %0 = call i32 @llvm.read_register.i32(metadata !30)
+  ret i32 %0
+}
+
+; CHECK-LABEL: reg_r31:
+; CHECK: r31 = #1
+; CHECK: r0 = r31
+define dso_local i32 @reg_r31() #0 {
+entry:
+  call void @llvm.write_register.i32(metadata !31, i32 1)
+  %0 = call i32 @llvm.read_register.i32(metadata !31)
+  ret i32 %0
+}
+
+; CHECK-LABEL: reg_d0:
+; CHECK: r1:0 = combine(#0,#1)
+define dso_local i64 @reg_d0() #0 {
+entry:
+  call void @llvm.write_register.i64(metadata !32, i64 1)
+  %0 = call i64 @llvm.read_register.i64(metadata !32)
+  ret i64 %0
+}
+
+; CHECK-LABEL: reg_d1:
+; CHECK: r3:2 = combine(#0,#1)
+; CHECK: r1:0 = combine(r3,r2)
+define dso_local i64 @reg_d1() #0 {
+entry:
+  call void @llvm.write_register.i64(metadata !33, i64 1)
+  %0 = call i64 @llvm.read_register.i64(metadata !33)
+  ret i64 %0
+}
+
+; CHECK-LABEL: reg_d2:
+; CHECK: r5:4 = combine(#0,#1)
+; CHECK: r1:0 = combine(r5,r4)
+define dso_local i64 @reg_d2() #0 {
+entry:
+  call void @llvm.write_register.i64(metadata !34, i64 1)
+  %0 = call i64 @llvm.read_register.i64(metadata !34)
+  ret i64 %0
+}
+
+; CHECK-LABEL: reg_d3:
+; CHECK: r7:6 = combine(#0,#1)
+; CHECK: r1:0 = combine(r7,r6)
+define dso_local i64 @reg_d3() #0 {
+entry:
+  call void @llvm.write_register.i64(metadata !35, i64 1)
+  %0 = call i64 @llvm.read_register.i64(metadata !35)
+  ret i64 %0
+}
+
+; CHECK-LABEL: reg_d4:
+; CHECK: r9:8 = combine(#0,#1)
+; CHECK: r1:0 = combine(r9,r8)
+define dso_local i64 @reg_d4() #0 {
+entry:
+  call void @llvm.write_register.i64(metadata !36, i64 1)
+  %0 = call i64 @llvm.read_register.i64(metadata !36)
+  ret i64 %0
+}
+
+; CHECK-LABEL: reg_d5:
+; CHECK: r11:10 = combine(#0,#1)
+; CHECK: r1:0 = combine(r11,r10)
+define dso_local i64 @reg_d5() #0 {
+entry:
+  call void @llvm.write_register.i64(metadata !37, i64 1)
+  %0 = call i64 @llvm.read_register.i64(metadata !37)
+  ret i64 %0
+}
+
+; CHECK-LABEL: reg_d6:
+; CHECK: r13:12 = combine(#0,#1)
+; CHECK: r1:0 = combine(r13,r12)
+define dso_local i64 @reg_d6() #0 {
+entry:
+  call void @llvm.write_register.i64(metadata !38, i64 1)
+  %0 = call i64 @llvm.read_register.i64(metadata !38)
+  ret i64 %0
+}
+
+; CHECK-LABEL: reg_d7:
+; CHECK: r15:14 = combine(#0,#1)
+; CHECK: r1:0 = combine(r15,r14)
+define dso_local i64 @reg_d7() #0 {
+entry:
+  call void @llvm.write_register.i64(metadata !39, i64 1)
+  %0 = call i64 @llvm.read_register.i64(metadata !39)
+  ret i64 %0
+}
+
+; CHECK-LABEL: reg_d8:
+; CHECK: r17:16 = combine(#0,#1)
+; CHECK: r1:0 = combine(r17,r16)
+define dso_local i64 @reg_d8() #0 {
+entry:
+  call void @llvm.write_register.i64(metadata !40, i64 1)
+  %0 = call i64 @llvm.read_register.i64(metadata !40)
+  ret i64 %0
+}
+
+; CHECK-LABEL: reg_d9:
+; CHECK: r19:18 = combine(#0,#1)
+; CHECK: r1:0 = combine(r19,r18)
+define dso_local i64 @reg_d9() #0 {
+entry:
+  call void @llvm.write_register.i64(metadata !41, i64 1)
+  %0 = call i64 @llvm.read_register.i64(metadata !41)
+  ret i64 %0
+}
+
+; CHECK-LABEL: reg_d10:
+; CHECK: r21:20 = combine(#0,#1)
+; CHECK: r1:0 = combine(r21,r20)
+define dso_local i64 @reg_d10() #0 {
+entry:
+  call void @llvm.write_register.i64(metadata !42, i64 1)
+  %0 = call i64 @llvm.read_register.i64(metadata !42)
+  ret i64 %0
+}
+
+; CHECK-LABEL: reg_d11:
+; CHECK: r23:22 = combine(#0,#1)
+; CHECK: r1:0 = combine(r23,r22)
+define dso_local i64 @reg_d11() #0 {
+entry:
+  call void @llvm.write_register.i64(metadata !43, i64 1)
+  %0 = call i64 @llvm.read_register.i64(metadata !43)
+  ret i64 %0
+}
+
+; CHECK-LABEL: reg_d12:
+; CHECK: r25:24 = combine(#0,#1)
+; CHECK: r1:0 = combine(r25,r24)
+define dso_local i64 @reg_d12() #0 {
+entry:
+  call void @llvm.write_register.i64(metadata !44, i64 1)
+  %0 = call i64 @llvm.read_register.i64(metadata !44)
+  ret i64 %0
+}
+
+; CHECK-LABEL: reg_d13:
+; CHECK: r27:26 = combine(#0,#1)
+; CHECK: r1:0 = combine(r27,r26)
+define dso_local i64 @reg_d13() #0 {
+entry:
+  call void @llvm.write_register.i64(metadata !45, i64 1)
+  %0 = call i64 @llvm.read_register.i64(metadata !45)
+  ret i64 %0
+}
+
+; CHECK-LABEL: reg_d14:
+; CHECK: r29:28 = combine(#0,#1)
+; CHECK: r1:0 = combine(r29,r28)
+define dso_local i64 @reg_d14() #0 {
+entry:
+  call void @llvm.write_register.i64(metadata !46, i64 1)
+  %0 = call i64 @llvm.read_register.i64(metadata !46)
+  ret i64 %0
+}
+
+; CHECK-LABEL: reg_d15:
+; CHECK: r31:30 = combine(#0,#1)
+; CHECK: r1:0 = combine(r31,r30)
+define dso_local i64 @reg_d15() #0 {
+entry:
+  call void @llvm.write_register.i64(metadata !47, i64 1)
+  %0 = call i64 @llvm.read_register.i64(metadata !47)
+  ret i64 %0
+}
+
+; CHECK-LABEL: reg_sp:
+; CHECK: r29 = #1
+; CHECK: r0 = r29
+define dso_local i32 @reg_sp() #0 {
+entry:
+  call void @llvm.write_register.i32(metadata !48, i32 1)
+  %0 = call i32 @llvm.read_register.i32(metadata !48)
+  ret i32 %0
+}
+
+; CHECK-LABEL: reg_fp:
+; CHECK: r30 = #1
+; CHECK: r0 = r30
+define dso_local i32 @reg_fp() #0 {
+entry:
+  call void @llvm.write_register.i32(metadata !49, i32 1)
+  %0 = call i32 @llvm.read_register.i32(metadata !49)
+  ret i32 %0
+}
+
+; CHECK-LABEL: reg_lr:
+; CHECK: r31 = #1
+; CHECK: r0 = r31
+define dso_local i32 @reg_lr() #0 {
+entry:
+  call void @llvm.write_register.i32(metadata !50, i32 1)
+  %0 = call i32 @llvm.read_register.i32(metadata !50)
+  ret i32 %0
+}
+
+; CHECK-LABEL: reg_p0:
+; CHECK: p0 = r31
+; CHECK: r0 = p0
+define dso_local i32 @reg_p0() #0 {
+entry:
+  %0 = call i32 @llvm.read_register.i32(metadata !31)
+  call void @llvm.write_register.i32(metadata !51, i32 %0)
+  %1 = call i32 @llvm.read_register.i32(metadata !51)
+  ret i32 %1
+}
+
+; CHECK-LABEL: reg_p1:
+; CHECK: p1 = r31
+; CHECK: r0 = p1
+define dso_local i32 @reg_p1() #0 {
+entry:
+  %0 = call i32 @llvm.read_register.i32(metadata !31)
+  call void @llvm.write_register.i32(metadata !52, i32 %0)
+  %1 = call i32 @llvm.read_register.i32(metadata !52)
+  ret i32 %1
+}
+
+; CHECK-LABEL: reg_p2:
+; CHECK: p2 = r31
+; CHECK: r0 = p2
+define dso_local i32 @reg_p2() #0 {
+entry:
+  %0 = call i32 @llvm.read_register.i32(metadata !31)
+  call void @llvm.write_register.i32(metadata !53, i32 %0)
+  %1 = call i32 @llvm.read_register.i32(metadata !53)
+  ret i32 %1
+}
+
+; CHECK-LABEL: reg_p3:
+; CHECK: p3 = r31
+; CHECK: r0 = p3
+define dso_local i32 @reg_p3() #0 {
+entry:
+  %0 = call i32 @llvm.read_register.i32(metadata !31)
+  call void @llvm.write_register.i32(metadata !54, i32 %0)
+  %1 = call i32 @llvm.read_register.i32(metadata !54)
+  ret i32 %1
+}
+
+; CHECK-LABEL: reg_sa0:
+; CHECK: sa0 = r31
+; CHECK: r0 = sa0
+define dso_local i32 @reg_sa0() #0 {
+entry:
+  %0 = call i32 @llvm.read_register.i32(metadata !31)
+  call void @llvm.write_register.i32(metadata !55, i32 %0)
+  %1 = call i32 @llvm.read_register.i32(metadata !55)
+  ret i32 %1
+}
+
+; CHECK-LABEL: reg_lc0:
+; CHECK: lc0 = r31
+; CHECK: r0 = lc0
+define dso_local i32 @reg_lc0() #0 {
+entry:
+  %0 = call i32 @llvm.read_register.i32(metadata !31)
+  call void @llvm.write_register.i32(metadata !56, i32 %0)
+  %1 = call i32 @llvm.read_register.i32(metadata !56)
+  ret i32 %1
+}
+
+; CHECK-LABEL: reg_sa1:
+; CHECK: sa1 = r31
+; CHECK: r0 = sa1
+define dso_local i32 @reg_sa1() #0 {
+entry:
+  %0 = call i32 @llvm.read_register.i32(metadata !31)
+  call void @llvm.write_register.i32(metadata !57, i32 %0)
+  %1 = call i32 @llvm.read_register.i32(metadata !57)
+  ret i32 %1
+}
+
+; CHECK-LABEL: reg_lc1:
+; CHECK: lc1 = r31
+; CHECK: r0 = lc1
+define dso_local i32 @reg_lc1() #0 {
+entry:
+  %0 = call i32 @llvm.read_register.i32(metadata !31)
+  call void @llvm.write_register.i32(metadata !58, i32 %0)
+  %1 = call i32 @llvm.read_register.i32(metadata !58)
+  ret i32 %1
+}
+
+; CHECK-LABEL: reg_m0:
+; CHECK: m0 = r31
+; CHECK: r0 = m0
+define dso_local i32 @reg_m0() #0 {
+entry:
+  %0 = call i32 @llvm.read_register.i32(metadata !31)
+  call void @llvm.write_register.i32(metadata !59, i32 %0)
+  %1 = call i32 @llvm.read_register.i32(metadata !59)
+  ret i32 %1
+}
+
+; CHECK-LABEL: reg_m1:
+; CHECK: m1 = r31
+; CHECK: r0 = m1
+define dso_local i32 @reg_m1() #0 {
+entry:
+  %0 = call i32 @llvm.read_register.i32(metadata !31)
+  call void @llvm.write_register.i32(metadata !60, i32 %0)
+  %1 = call i32 @llvm.read_register.i32(metadata !60)
+  ret i32 %1
+}
+
+; CHECK-LABEL: reg_usr:
+; CHECK: usr = r31
+; CHECK: r0 = usr
+define dso_local i32 @reg_usr() #0 {
+entry:
+  %0 = call i32 @llvm.read_register.i32(metadata !31)
+  call void @llvm.write_register.i32(metadata !61, i32 %0)
+  %1 = call i32 @llvm.read_register.i32(metadata !61)
+  ret i32 %1
+}
+
+; CHECK-LABEL: reg_ugp:
+; CHECK: ugp = r31
+; CHECK: r0 = ugp
+define dso_local i32 @reg_ugp() #0 {
+entry:
+  %0 = call i32 @llvm.read_register.i32(metadata !31)
+  call void @llvm.write_register.i32(metadata !62, i32 %0)
+  %1 = call i32 @llvm.read_register.i32(metadata !62)
+  ret i32 %1
+}
+
+attributes #0 = { noinline nounwind optnone "target-cpu"="hexagonv62" }
+attributes #1 = { nounwind }
+attributes #2 = { nounwind readonly }
+
+!llvm.named.register.r0 = !{!0}
+!llvm.named.register.r1 = !{!1}
+!llvm.named.register.r2 = !{!2}
+!llvm.named.register.r3 = !{!3}
+!llvm.named.register.r4 = !{!4}
+!llvm.named.register.r5 = !{!5}
+!llvm.named.register.r6 = !{!6}
+!llvm.named.register.r7 = !{!7}
+!llvm.named.register.r8 = !{!8}
+!llvm.named.register.r9 = !{!9}
+!llvm.named.register.r10 = !{!10}
+!llvm.named.register.r11 = !{!11}
+!llvm.named.register.r12 = !{!12}
+!llvm.named.register.r13 = !{!13}
+!llvm.named.register.r14 = !{!14}
+!llvm.named.register.r15 = !{!15}
+!llvm.named.register.r16 = !{!16}
+!llvm.named.register.r17 = !{!17}
+!llvm.named.register.r18 = !{!18}
+!llvm.named.register.r19 = !{!19}
+!llvm.named.register.r20 = !{!20}
+!llvm.named.register.r21 = !{!21}
+!llvm.named.register.r22 = !{!22}
+!llvm.named.register.r23 = !{!23}
+!llvm.named.register.r24 = !{!24}
+!llvm.named.register.r25 = !{!25}
+!llvm.named.register.r26 = !{!26}
+!llvm.named.register.r27 = !{!27}
+!llvm.named.register.r28 = !{!28}
+!llvm.named.register.r29 = !{!29}
+!llvm.named.register.r30 = !{!30}
+!llvm.named.register.r31 = !{!31}
+!llvm.named.register.r1\3A0 = !{!32}
+!llvm.named.register.r3\3A2 = !{!33}
+!llvm.named.register.r5\3A4 = !{!34}
+!llvm.named.register.r7\3A6 = !{!35}
+!llvm.named.register.r9\3A8 = !{!36}
+!llvm.named.register.r11\3A10 = !{!37}
+!llvm.named.register.r13\3A12 = !{!38}
+!llvm.named.register.r15\3A14 = !{!39}
+!llvm.named.register.r17\3A16 = !{!40}
+!llvm.named.register.r19\3A18 = !{!41}
+!llvm.named.register.r21\3A20 = !{!42}
+!llvm.named.register.r23\3A22 = !{!43}
+!llvm.named.register.r25\3A24 = !{!44}
+!llvm.named.register.r27\3A26 = !{!45}
+!llvm.named.register.r29\3A28 = !{!46}
+!llvm.named.register.r31\3A30 = !{!47}
+!llvm.named.register.sp = !{!48}
+!llvm.named.register.fp = !{!49}
+!llvm.named.register.lr = !{!50}
+!llvm.named.register.p0 = !{!51}
+!llvm.named.register.p1 = !{!52}
+!llvm.named.register.p2 = !{!53}
+!llvm.named.register.p3 = !{!54}
+!llvm.named.register.sa0 = !{!55}
+!llvm.named.register.lc0 = !{!56}
+!llvm.named.register.sa1 = !{!57}
+!llvm.named.register.lc1 = !{!58}
+!llvm.named.register.m0 = !{!59}
+!llvm.named.register.m1 = !{!60}
+!llvm.named.register.usr = !{!61}
+!llvm.named.register.ugp = !{!62}
+
+!0 = !{!"r0"}
+!1 = !{!"r1"}
+!2 = !{!"r2"}
+!3 = !{!"r3"}
+!4 = !{!"r4"}
+!5 = !{!"r5"}
+!6 = !{!"r6"}
+!7 = !{!"r7"}
+!8 = !{!"r8"}
+!9 = !{!"r9"}
+!10 = !{!"r10"}
+!11 = !{!"r11"}
+!12 = !{!"r12"}
+!13 = !{!"r13"}
+!14 = !{!"r14"}
+!15 = !{!"r15"}
+!16 = !{!"r16"}
+!17 = !{!"r17"}
+!18 = !{!"r18"}
+!19 = !{!"r19"}
+!20 = !{!"r20"}
+!21 = !{!"r21"}
+!22 = !{!"r22"}
+!23 = !{!"r23"}
+!24 = !{!"r24"}
+!25 = !{!"r25"}
+!26 = !{!"r26"}
+!27 = !{!"r27"}
+!28 = !{!"r28"}
+!29 = !{!"r29"}
+!30 = !{!"r30"}
+!31 = !{!"r31"}
+!32 = !{!"r1:0"}
+!33 = !{!"r3:2"}
+!34 = !{!"r5:4"}
+!35 = !{!"r7:6"}
+!36 = !{!"r9:8"}
+!37 = !{!"r11:10"}
+!38 = !{!"r13:12"}
+!39 = !{!"r15:14"}
+!40 = !{!"r17:16"}
+!41 = !{!"r19:18"}
+!42 = !{!"r21:20"}
+!43 = !{!"r23:22"}
+!44 = !{!"r25:24"}
+!45 = !{!"r27:26"}
+!46 = !{!"r29:28"}
+!47 = !{!"r31:30"}
+!48 = !{!"sp"}
+!49 = !{!"fp"}
+!50 = !{!"lr"}
+!51 = !{!"p0"}
+!52 = !{!"p1"}
+!53 = !{!"p2"}
+!54 = !{!"p3"}
+!55 = !{!"sa0"}
+!56 = !{!"lc0"}
+!57 = !{!"sa1"}
+!58 = !{!"lc1"}
+!59 = !{!"m0"}
+!60 = !{!"m1"}
+!61 = !{!"usr"}
+!62 = !{!"ugp"}
diff --git a/llvm/test/CodeGen/Hexagon/reg-scavengebug-3.ll b/llvm/test/CodeGen/Hexagon/reg-scavengebug-3.ll
index c73d4c7bc..49e451d54 100644
--- a/llvm/test/CodeGen/Hexagon/reg-scavengebug-3.ll
+++ b/llvm/test/CodeGen/Hexagon/reg-scavengebug-3.ll
@@ -74,7 +74,7 @@ entry:
   ret i32 0
 }
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
-attributes #2 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #3 = { nounwind }
diff --git a/llvm/test/CodeGen/Hexagon/runtime-stkchk.ll b/llvm/test/CodeGen/Hexagon/runtime-stkchk.ll
index 38aa8726d..66e93d02e 100644
--- a/llvm/test/CodeGen/Hexagon/runtime-stkchk.ll
+++ b/llvm/test/CodeGen/Hexagon/runtime-stkchk.ll
@@ -37,8 +37,8 @@ declare void @baz_2(i32*, i32*) #2
 declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture) #1
 declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture) #1
 
-attributes #0 = { nounwind optsize "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind optsize "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind }
-attributes #2 = { optsize "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { optsize "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #3 = { optsize }
 
diff --git a/llvm/test/CodeGen/Hexagon/sdr-shr32.ll b/llvm/test/CodeGen/Hexagon/sdr-shr32.ll
index 67dc1c14d..2e548fc5b 100644
--- a/llvm/test/CodeGen/Hexagon/sdr-shr32.ll
+++ b/llvm/test/CodeGen/Hexagon/sdr-shr32.ll
@@ -14,7 +14,7 @@ entry:
 ; Function Attrs: nounwind readnone
 declare i64 @llvm.hexagon.S2.asr.i.p(i64, i32) #1
 
-attributes #0 = { nounwind readnone "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind readnone "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !llvm.ident = !{!0}
diff --git a/llvm/test/CodeGen/Hexagon/signext-inreg.ll b/llvm/test/CodeGen/Hexagon/signext-inreg.ll
new file mode 100644
index 000000000..cd9d78358
--- /dev/null
+++ b/llvm/test/CodeGen/Hexagon/signext-inreg.ll
@@ -0,0 +1,272 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -march=hexagon | FileCheck %s
+; RUN: llc < %s -march=hexagon -mattr=+hvx,hvx-length64b | FileCheck %s --check-prefix=CHECK-64B
+; RUN: llc < %s -march=hexagon -mattr=+hvx,hvx-length128b | FileCheck %s --check-prefix=CHECK-128B
+define <2 x i32> @test1(<2 x i32> %m) {
+; CHECK-LABEL: test1:
+; CHECK:         .cfi_startproc
+; CHECK-NEXT:  // %bb.0: // %entry
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r1 = extract(r1,#8,#0)
+; CHECK-NEXT:     r0 = sxtb(r0)
+; CHECK-NEXT:     jumpr r31
+; CHECK-NEXT:    }
+;
+; CHECK-64B-LABEL: test1:
+; CHECK-64B:         .cfi_startproc
+; CHECK-64B-NEXT:  // %bb.0: // %entry
+; CHECK-64B-NEXT:    {
+; CHECK-64B-NEXT:     r1 = extract(r1,#8,#0)
+; CHECK-64B-NEXT:     r0 = sxtb(r0)
+; CHECK-64B-NEXT:     jumpr r31
+; CHECK-64B-NEXT:    }
+;
+; CHECK-128B-LABEL: test1:
+; CHECK-128B:         .cfi_startproc
+; CHECK-128B-NEXT:  // %bb.0: // %entry
+; CHECK-128B-NEXT:    {
+; CHECK-128B-NEXT:     r1 = extract(r1,#8,#0)
+; CHECK-128B-NEXT:     r0 = sxtb(r0)
+; CHECK-128B-NEXT:     jumpr r31
+; CHECK-128B-NEXT:    }
+entry:
+  %shl = shl <2 x i32> %m, <i32 24, i32 24>
+  %shr = ashr exact <2 x i32> %shl, <i32 24, i32 24>
+  ret <2 x i32> %shr
+}
+
+define <16 x i32> @test2(<16 x i32> %m) {
+; CHECK-LABEL: test2:
+; CHECK:         .cfi_startproc
+; CHECK-NEXT:  // %bb.0: // %entry
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r3 = extract(r3,#8,#0)
+; CHECK-NEXT:     r29 = add(r29,#-8)
+; CHECK-NEXT:     r2 = sxtb(r2)
+; CHECK-NEXT:     r4 = sxtb(r4)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r5 = extract(r5,#8,#0)
+; CHECK-NEXT:     r13:12 = memd(r29+#48)
+; CHECK-NEXT:     memd(r29+#0) = r17:16
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r13 = extract(r13,#8,#0)
+; CHECK-NEXT:     r12 = sxtb(r12)
+; CHECK-NEXT:     r15:14 = memd(r29+#40)
+; CHECK-NEXT:     r9:8 = memd(r29+#32)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r9 = extract(r9,#8,#0)
+; CHECK-NEXT:     r8 = sxtb(r8)
+; CHECK-NEXT:     r11:10 = memd(r29+#24)
+; CHECK-NEXT:     r7:6 = memd(r29+#16)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r11 = extract(r11,#8,#0)
+; CHECK-NEXT:     r10 = sxtb(r10)
+; CHECK-NEXT:     r14 = sxtb(r14)
+; CHECK-NEXT:     r17:16 = memd(r29+#8)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r15 = extract(r15,#8,#0)
+; CHECK-NEXT:     r17 = extract(r17,#8,#0)
+; CHECK-NEXT:     r16 = sxtb(r16)
+; CHECK-NEXT:     r6 = sxtb(r6)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r7 = extract(r7,#8,#0)
+; CHECK-NEXT:     memd(r0+#56) = r13:12
+; CHECK-NEXT:     memd(r0+#48) = r15:14
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     memd(r0+#40) = r9:8
+; CHECK-NEXT:     memd(r0+#32) = r11:10
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     memd(r0+#24) = r7:6
+; CHECK-NEXT:     memd(r0+#16) = r17:16
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     memd(r0+#8) = r5:4
+; CHECK-NEXT:     memd(r0+#0) = r3:2
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r29 = add(r29,#8)
+; CHECK-NEXT:     r17:16 = memd(r29+#0)
+; CHECK-NEXT:     jumpr r31
+; CHECK-NEXT:    } // 8-byte Folded Reload
+;
+; CHECK-64B-LABEL: test2:
+; CHECK-64B:         .cfi_startproc
+; CHECK-64B-NEXT:  // %bb.0: // %entry
+; CHECK-64B-NEXT:    {
+; CHECK-64B-NEXT:     r0 = #24
+; CHECK-64B-NEXT:    }
+; CHECK-64B-NEXT:    {
+; CHECK-64B-NEXT:     v0.w = vasl(v0.w,r0)
+; CHECK-64B-NEXT:    }
+; CHECK-64B-NEXT:    {
+; CHECK-64B-NEXT:     v0.w = vasr(v0.w,r0)
+; CHECK-64B-NEXT:     jumpr r31
+; CHECK-64B-NEXT:    }
+;
+; CHECK-128B-LABEL: test2:
+; CHECK-128B:         .cfi_startproc
+; CHECK-128B-NEXT:  // %bb.0: // %entry
+; CHECK-128B-NEXT:    {
+; CHECK-128B-NEXT:     r0 = #24
+; CHECK-128B-NEXT:    }
+; CHECK-128B-NEXT:    {
+; CHECK-128B-NEXT:     v0.w = vasl(v0.w,r0)
+; CHECK-128B-NEXT:    }
+; CHECK-128B-NEXT:    {
+; CHECK-128B-NEXT:     v0.w = vasr(v0.w,r0)
+; CHECK-128B-NEXT:     jumpr r31
+; CHECK-128B-NEXT:    }
+entry:
+  %shl = shl <16 x i32> %m, <i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24>
+  %shr = ashr exact <16 x i32> %shl, <i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24>
+  ret <16 x i32> %shr
+}
+
+define <64 x i16> @test3(<64 x i16> %m) {
+; CHECK-LABEL: test3:
+; CHECK:         .cfi_startproc
+; CHECK-NEXT:  // %bb.0: // %entry
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r3:2 = vaslh(r3:2,#8)
+; CHECK-NEXT:     r5:4 = vaslh(r5:4,#8)
+; CHECK-NEXT:     r9:8 = memd(r29+#96)
+; CHECK-NEXT:     r11:10 = memd(r29+#88)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r13:12 = vaslh(r9:8,#8)
+; CHECK-NEXT:     r11:10 = vaslh(r11:10,#8)
+; CHECK-NEXT:     r9:8 = memd(r29+#80)
+; CHECK-NEXT:     r7:6 = memd(r29+#104)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r15:14 = vaslh(r7:6,#8)
+; CHECK-NEXT:     r9:8 = vaslh(r9:8,#8)
+; CHECK-NEXT:     r7:6 = memd(r29+#72)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r15:14 = vasrh(r15:14,#8)
+; CHECK-NEXT:     r13:12 = vasrh(r13:12,#8)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r11:10 = vasrh(r11:10,#8)
+; CHECK-NEXT:     r9:8 = vasrh(r9:8,#8)
+; CHECK-NEXT:     r15:14 = memd(r29+#64)
+; CHECK-NEXT:     memd(r0+#120) = r15:14
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r7:6 = vaslh(r7:6,#8)
+; CHECK-NEXT:     r15:14 = vaslh(r15:14,#8)
+; CHECK-NEXT:     r13:12 = memd(r29+#56)
+; CHECK-NEXT:     memd(r0+#112) = r13:12
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r13:12 = vaslh(r13:12,#8)
+; CHECK-NEXT:     r7:6 = vasrh(r7:6,#8)
+; CHECK-NEXT:     r11:10 = memd(r29+#48)
+; CHECK-NEXT:     memd(r0+#104) = r11:10
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r11:10 = vaslh(r11:10,#8)
+; CHECK-NEXT:     r15:14 = vasrh(r15:14,#8)
+; CHECK-NEXT:     r9:8 = memd(r29+#40)
+; CHECK-NEXT:     memd(r0+#96) = r9:8
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r9:8 = vaslh(r9:8,#8)
+; CHECK-NEXT:     r13:12 = vasrh(r13:12,#8)
+; CHECK-NEXT:     r7:6 = memd(r29+#32)
+; CHECK-NEXT:     memd(r0+#88) = r7:6
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r11:10 = vasrh(r11:10,#8)
+; CHECK-NEXT:     r9:8 = vasrh(r9:8,#8)
+; CHECK-NEXT:     r15:14 = memd(r29+#0)
+; CHECK-NEXT:     memd(r0+#80) = r15:14
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r7:6 = vaslh(r7:6,#8)
+; CHECK-NEXT:     r15:14 = vaslh(r15:14,#8)
+; CHECK-NEXT:     r13:12 = memd(r29+#16)
+; CHECK-NEXT:     memd(r0+#72) = r13:12
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r13:12 = vaslh(r13:12,#8)
+; CHECK-NEXT:     r7:6 = vasrh(r7:6,#8)
+; CHECK-NEXT:     r11:10 = memd(r29+#24)
+; CHECK-NEXT:     memd(r0+#64) = r11:10
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r11:10 = vaslh(r11:10,#8)
+; CHECK-NEXT:     r3:2 = vasrh(r3:2,#8)
+; CHECK-NEXT:     r9:8 = memd(r29+#8)
+; CHECK-NEXT:     memd(r0+#56) = r9:8
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r9:8 = vaslh(r9:8,#8)
+; CHECK-NEXT:     r13:12 = vasrh(r13:12,#8)
+; CHECK-NEXT:     memd(r0+#48) = r7:6
+; CHECK-NEXT:     memd(r0+#0) = r3:2
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r11:10 = vasrh(r11:10,#8)
+; CHECK-NEXT:     r7:6 = vasrh(r15:14,#8)
+; CHECK-NEXT:     memd(r0+#32) = r13:12
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r9:8 = vasrh(r9:8,#8)
+; CHECK-NEXT:     r5:4 = vasrh(r5:4,#8)
+; CHECK-NEXT:     memd(r0+#40) = r11:10
+; CHECK-NEXT:     memd(r0+#16) = r7:6
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     jumpr r31
+; CHECK-NEXT:     memd(r0+#24) = r9:8
+; CHECK-NEXT:     memd(r0+#8) = r5:4
+; CHECK-NEXT:    }
+;
+; CHECK-64B-LABEL: test3:
+; CHECK-64B:         .cfi_startproc
+; CHECK-64B-NEXT:  // %bb.0: // %entry
+; CHECK-64B-NEXT:    {
+; CHECK-64B-NEXT:     r0 = #8
+; CHECK-64B-NEXT:    }
+; CHECK-64B-NEXT:    {
+; CHECK-64B-NEXT:     v0.h = vasl(v0.h,r0)
+; CHECK-64B-NEXT:    }
+; CHECK-64B-NEXT:    {
+; CHECK-64B-NEXT:     v1.h = vasl(v1.h,r0)
+; CHECK-64B-NEXT:    }
+; CHECK-64B-NEXT:    {
+; CHECK-64B-NEXT:     v0.h = vasr(v0.h,r0)
+; CHECK-64B-NEXT:    }
+; CHECK-64B-NEXT:    {
+; CHECK-64B-NEXT:     v1.h = vasr(v1.h,r0)
+; CHECK-64B-NEXT:     jumpr r31
+; CHECK-64B-NEXT:    }
+;
+; CHECK-128B-LABEL: test3:
+; CHECK-128B:         .cfi_startproc
+; CHECK-128B-NEXT:  // %bb.0: // %entry
+; CHECK-128B-NEXT:    {
+; CHECK-128B-NEXT:     r0 = #8
+; CHECK-128B-NEXT:    }
+; CHECK-128B-NEXT:    {
+; CHECK-128B-NEXT:     v0.h = vasl(v0.h,r0)
+; CHECK-128B-NEXT:    }
+; CHECK-128B-NEXT:    {
+; CHECK-128B-NEXT:     v0.h = vasr(v0.h,r0)
+; CHECK-128B-NEXT:     jumpr r31
+; CHECK-128B-NEXT:    }
+entry:
+  %shl = shl <64 x i16> %m, <i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8>
+  %shr = ashr exact <64 x i16> %shl, <i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8>
+  ret <64 x i16> %shr
+}
diff --git a/llvm/test/CodeGen/Hexagon/switch-lut-explicit-section.ll b/llvm/test/CodeGen/Hexagon/switch-lut-explicit-section.ll
index b80e8e33b..b542c9c60 100644
--- a/llvm/test/CodeGen/Hexagon/switch-lut-explicit-section.ll
+++ b/llvm/test/CodeGen/Hexagon/switch-lut-explicit-section.ll
@@ -29,4 +29,4 @@ return:                                           ; preds = %entry
   ret i32 19
 }
 
-attributes #0 = { norecurse nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="hexagonv60" "target-features"="-hvx,-long-calls" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { norecurse nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="hexagonv60" "target-features"="-hvx,-long-calls" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/Hexagon/switch-lut-function-section.ll b/llvm/test/CodeGen/Hexagon/switch-lut-function-section.ll
index 542bfbb6d..42043530b 100644
--- a/llvm/test/CodeGen/Hexagon/switch-lut-function-section.ll
+++ b/llvm/test/CodeGen/Hexagon/switch-lut-function-section.ll
@@ -27,4 +27,4 @@ return:                                           ; preds = %entry
   ret i32 19
 }
 
-attributes #0 = { norecurse nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="hexagonv60" "target-features"="-hvx,-long-calls" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { norecurse nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="hexagonv60" "target-features"="-hvx,-long-calls" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/Hexagon/switch-lut-multiple-functions.ll b/llvm/test/CodeGen/Hexagon/switch-lut-multiple-functions.ll
index 22b61f0c9..be0b6817a 100644
--- a/llvm/test/CodeGen/Hexagon/switch-lut-multiple-functions.ll
+++ b/llvm/test/CodeGen/Hexagon/switch-lut-multiple-functions.ll
@@ -39,4 +39,4 @@ return:                                           ; preds = %entry
   ret i32 19
 }
 
-attributes #0 = { norecurse nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="hexagonv60" "target-features"="-hvx,-long-calls" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { norecurse nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="hexagonv60" "target-features"="-hvx,-long-calls" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/Hexagon/switch-lut-text-section.ll b/llvm/test/CodeGen/Hexagon/switch-lut-text-section.ll
index 203ea4abd..bca7036db 100644
--- a/llvm/test/CodeGen/Hexagon/switch-lut-text-section.ll
+++ b/llvm/test/CodeGen/Hexagon/switch-lut-text-section.ll
@@ -24,4 +24,4 @@ return:                                           ; preds = %entry
   ret i32 19
 }
 
-attributes #0 = { norecurse nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="hexagonv60" "target-features"="-hvx,-long-calls" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { norecurse nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="hexagonv60" "target-features"="-hvx,-long-calls" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/Hexagon/swp-art-deps-rec.ll b/llvm/test/CodeGen/Hexagon/swp-art-deps-rec.ll
index 5272faf8f..f89df6e55 100644
--- a/llvm/test/CodeGen/Hexagon/swp-art-deps-rec.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-art-deps-rec.ll
@@ -1,7 +1,7 @@
 ; REQUIRES: asserts
 
 ; RUN: llc -march=hexagon -mcpu=hexagonv65 -O3 -debug-only=pipeliner \
-; RUN: < %s 2>&1 | FileCheck %s
+; RUN: < %s 2>&1 -pipeliner-experimental-cg=true | FileCheck %s
 
 ; Test that the artificial dependences are ignored while computing the
 ; circuits.
diff --git a/llvm/test/CodeGen/Hexagon/swp-bad-sched.ll b/llvm/test/CodeGen/Hexagon/swp-bad-sched.ll
index 74f0647a7..ee93e8b34 100644
--- a/llvm/test/CodeGen/Hexagon/swp-bad-sched.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-bad-sched.ll
@@ -1,5 +1,5 @@
 ; REQUIRES: asserts
-; RUN: llc -march=hexagon -enable-pipeliner -enable-aa-sched-mi < %s | FileCheck %s
+; RUN: llc -march=hexagon -enable-pipeliner -enable-aa-sched-mi < %s -pipeliner-experimental-cg=true | FileCheck %s
 
 ; CHECK: loop0(
 ; CHECK: loop0(.LBB0_[[LOOP:.]],
diff --git a/llvm/test/CodeGen/Hexagon/swp-carried-1.ll b/llvm/test/CodeGen/Hexagon/swp-carried-1.ll
index b33cf5221..e5b5be4d4 100644
--- a/llvm/test/CodeGen/Hexagon/swp-carried-1.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-carried-1.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon -rdf-opt=0 -disable-hexagon-misched -hexagon-initial-cfg-cleanup=0 -lsr-setupcost-depth-limit=1 < %s | FileCheck %s
+; RUN: llc -march=hexagon -rdf-opt=0 -disable-hexagon-misched -hexagon-initial-cfg-cleanup=0 -lsr-setupcost-depth-limit=1 < %s -pipeliner-experimental-cg=true | FileCheck %s
 
 ; Test that we generate the correct code when a loop carried value
 ; is scheduled one stage earlier than it's use. The code in
diff --git a/llvm/test/CodeGen/Hexagon/swp-chain-refs.ll b/llvm/test/CodeGen/Hexagon/swp-chain-refs.ll
index d0e72be77..5695f3d61 100644
--- a/llvm/test/CodeGen/Hexagon/swp-chain-refs.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-chain-refs.ll
@@ -1,5 +1,5 @@
 ; RUN: llc -march=hexagon -enable-pipeliner=true -stats -o /dev/null < %s \
-; RUN:      2>&1 | FileCheck %s --check-prefix=STATS
+; RUN:      2>&1 -pipeliner-experimental-cg=true | FileCheck %s --check-prefix=STATS
 ; REQUIRES: asserts
 
 ; Test that we do not schedule chained references too far apart,
diff --git a/llvm/test/CodeGen/Hexagon/swp-change-dep1.ll b/llvm/test/CodeGen/Hexagon/swp-change-dep1.ll
index 855f43e50..157bdd069 100644
--- a/llvm/test/CodeGen/Hexagon/swp-change-dep1.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-change-dep1.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon -enable-pipeliner -pipeliner-max-stages=1 < %s | FileCheck %s
+; RUN: llc -march=hexagon -enable-pipeliner -pipeliner-max-stages=1 < %s -pipeliner-experimental-cg=true | FileCheck %s
 
 ; Test that we update the offset correctly for loads that are
 ; moved past stores. In these cases, we change the dependences
diff --git a/llvm/test/CodeGen/Hexagon/swp-change-deps.ll b/llvm/test/CodeGen/Hexagon/swp-change-deps.ll
index e2ca071f5..1b35c633c 100644
--- a/llvm/test/CodeGen/Hexagon/swp-change-deps.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-change-deps.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon -hexagon-initial-cfg-cleanup=0 < %s | FileCheck %s
+; RUN: llc -march=hexagon -hexagon-initial-cfg-cleanup=0 < %s -pipeliner-experimental-cg=true | FileCheck %s
 
 ; Test that we generate the correct offsets for loads in the prolog
 ; after removing dependences on a post-increment instructions of the
diff --git a/llvm/test/CodeGen/Hexagon/swp-check-offset.ll b/llvm/test/CodeGen/Hexagon/swp-check-offset.ll
index 220ebde0f..6a7211df1 100644
--- a/llvm/test/CodeGen/Hexagon/swp-check-offset.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-check-offset.ll
@@ -1,6 +1,6 @@
-; RUN: llc -march=hexagon -mcpu=hexagonv5 -enable-pipeliner < %s | FileCheck %s
-; RUN: llc -march=hexagon -mcpu=hexagonv62 -enable-pipeliner < %s | FileCheck --check-prefix=CHECK-V62 %s
-; RUN: llc -march=hexagon -mcpu=hexagonv65 -enable-pipeliner < %s | FileCheck --check-prefix=CHECK-V65 %s
+; RUN: llc -march=hexagon -mcpu=hexagonv5 -enable-pipeliner < %s -pipeliner-experimental-cg=true | FileCheck %s
+; RUN: llc -march=hexagon -mcpu=hexagonv62 -enable-pipeliner < %s -pipeliner-experimental-cg=true | FileCheck --check-prefix=CHECK-V62 %s
+; RUN: llc -march=hexagon -mcpu=hexagonv65 -enable-pipeliner < %s -pipeliner-experimental-cg=true | FileCheck --check-prefix=CHECK-V65 %s
 
 ;
 ; Make sure we pipeline the loop and that we generate the correct
diff --git a/llvm/test/CodeGen/Hexagon/swp-const-tc1.ll b/llvm/test/CodeGen/Hexagon/swp-const-tc1.ll
index 95dfc37e3..c785ee745 100644
--- a/llvm/test/CodeGen/Hexagon/swp-const-tc1.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-const-tc1.ll
@@ -1,7 +1,7 @@
 ; RUN: llc -march=hexagon -enable-pipeliner -enable-pipeliner-opt-size \
 ; RUN:     -verify-machineinstrs -hexagon-initial-cfg-cleanup=0 \
 ; RUN:     -enable-aa-sched-mi=false -hexagon-expand-condsets=0 \
-; RUN:     < %s | FileCheck %s
+; RUN:     < %s -pipeliner-experimental-cg=true | FileCheck %s
 
 ; Disable expand-condsets because it will assert on undefined registers.
 
diff --git a/llvm/test/CodeGen/Hexagon/swp-const-tc2.ll b/llvm/test/CodeGen/Hexagon/swp-const-tc2.ll
index 8b9f87f42..29d12bd14 100644
--- a/llvm/test/CodeGen/Hexagon/swp-const-tc2.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-const-tc2.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon -rdf-opt=0 < %s | FileCheck %s
+; RUN: llc -march=hexagon -rdf-opt=0 < %s -pipeliner-experimental-cg=true | FileCheck %s
 
 ; Test that we fixup a pipelined loop correctly when the number of
 ; stages is greater than the compile-time loop trip count. In this
diff --git a/llvm/test/CodeGen/Hexagon/swp-const-tc3.ll b/llvm/test/CodeGen/Hexagon/swp-const-tc3.ll
index a8caebd09..48a614285 100644
--- a/llvm/test/CodeGen/Hexagon/swp-const-tc3.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-const-tc3.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon < %s | FileCheck %s
+; RUN: llc -march=hexagon < %s -pipeliner-experimental-cg=true | FileCheck %s
 
 ; Test that the pipeliner correctly fixes up the pipelined CFG when the loop
 ; has a constant trip count, and the trip count is less than the number of
diff --git a/llvm/test/CodeGen/Hexagon/swp-conv3x3-nested.ll b/llvm/test/CodeGen/Hexagon/swp-conv3x3-nested.ll
index 48f33bd6d..2b69c3da8 100644
--- a/llvm/test/CodeGen/Hexagon/swp-conv3x3-nested.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-conv3x3-nested.ll
@@ -1,6 +1,4 @@
-; RUN: llc -march=hexagon < %s | FileCheck %s
-; XFAIL: *
-; LSR changes required.
+; RUN: llc -march=hexagon < %s -pipeliner-experimental-cg=true | FileCheck %s
 
 ; This version of the conv3x3 test has both loops. This test checks that the
 ; inner loop has 13 packets.
diff --git a/llvm/test/CodeGen/Hexagon/swp-copytophi-dag.ll b/llvm/test/CodeGen/Hexagon/swp-copytophi-dag.ll
index 69743407c..f511241a7 100644
--- a/llvm/test/CodeGen/Hexagon/swp-copytophi-dag.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-copytophi-dag.ll
@@ -1,7 +1,7 @@
 ; REQUIRES: asserts
 ;
 ; RUN: llc -march=hexagon -enable-pipeliner=true -debug-only=pipeliner < %s \
-; RUN: 2>&1 | FileCheck %s
+; RUN: 2>&1 -pipeliner-experimental-cg=true | FileCheck %s
 
 ; Test that the artificial dependence is created as a result of
 ; CopyToPhi DAG mutation.
diff --git a/llvm/test/CodeGen/Hexagon/swp-crash-iter.ll b/llvm/test/CodeGen/Hexagon/swp-crash-iter.ll
new file mode 100644
index 000000000..b2a8e40bd
--- /dev/null
+++ b/llvm/test/CodeGen/Hexagon/swp-crash-iter.ll
@@ -0,0 +1,32 @@
+; REQUIRES: asserts
+; RUN: llc -march=hexagon -enable-pipeliner -o /dev/null < %s
+; Test that we do not crash when running CopyToPhi DAG mutation due to
+; iterator invalidation.
+
+declare i64 @llvm.hexagon.M2.cmacsc.s0(i64, i32, i32) #0
+define dso_local void @foo() local_unnamed_addr #1 {
+entry:
+  br label %for.body
+for.body:                                         ; preds = %for.body, %entry
+  %loop_count.0420 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %tmp_rslt4.sroa.0.0415 = phi i64 [ 0, %entry ], [ %phitmp395, %for.body ]
+  %tmp_rslt4.sroa.14.0414 = phi i64 [ 0, %entry ], [ %phitmp394, %for.body ]
+  %tmp_rslt3.sroa.12.0412 = phi i64 [ 0, %entry ], [ %phitmp391, %for.body ]
+  %tmp_rslt3.sroa.0.0.insert.insert = or i64 0, %tmp_rslt3.sroa.12.0412
+  %0 = tail call i64 @llvm.hexagon.M2.cmacsc.s0(i64 %tmp_rslt3.sroa.0.0.insert.insert, i32 undef, i32 undef)
+  %tmp_rslt4.sroa.0.0.insert.insert = or i64 %tmp_rslt4.sroa.0.0415, %tmp_rslt4.sroa.14.0414
+  %1 = tail call i64 @llvm.hexagon.M2.cmacsc.s0(i64 %tmp_rslt4.sroa.0.0.insert.insert, i32 undef, i32 undef)
+  %inc = add nuw nsw i32 %loop_count.0420, 1
+  %phitmp391 = and i64 %0, -4294967296
+  %phitmp394 = and i64 %1, -4294967296
+  %phitmp395 = and i64 %1, 4294967295
+  %exitcond = icmp eq i32 %inc, 63
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  %2 = tail call i64 @llvm.hexagon.M2.cmacsc.s0(i64 %0, i32 undef, i32 undef)
+  ret void
+}
+
+attributes #0 = { nounwind readnone }
+attributes #1 = { "target-features"="-long-calls,-small-data" }
diff --git a/llvm/test/CodeGen/Hexagon/swp-dep-neg-offset.ll b/llvm/test/CodeGen/Hexagon/swp-dep-neg-offset.ll
index 7ba4286bf..cc19ce1ae 100644
--- a/llvm/test/CodeGen/Hexagon/swp-dep-neg-offset.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-dep-neg-offset.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon -enable-pipeliner -hexagon-initial-cfg-cleanup=0 < %s | FileCheck %s
+; RUN: llc -march=hexagon -enable-pipeliner -hexagon-initial-cfg-cleanup=0 < %s -pipeliner-experimental-cg=true | FileCheck %s
 
 ; Test that the code that changes the dependences does not allow
 ; a load with a negative offset to be overlapped with the post
diff --git a/llvm/test/CodeGen/Hexagon/swp-disable-Os.ll b/llvm/test/CodeGen/Hexagon/swp-disable-Os.ll
index cbdc3ba36..5698d37cb 100644
--- a/llvm/test/CodeGen/Hexagon/swp-disable-Os.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-disable-Os.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon < %s | FileCheck %s
+; RUN: llc -march=hexagon < %s -pipeliner-experimental-cg=true | FileCheck %s
 ; CHECK: loop0(.LBB0_{{[0-9]+}},#347)
 
 target triple = "hexagon"
diff --git a/llvm/test/CodeGen/Hexagon/swp-epilog-numphis.ll b/llvm/test/CodeGen/Hexagon/swp-epilog-numphis.ll
index a54ac5825..f57f94bf0 100644
--- a/llvm/test/CodeGen/Hexagon/swp-epilog-numphis.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-epilog-numphis.ll
@@ -1,6 +1,6 @@
 ; XFAIL: *
 ; Needs some fixed in the pipeliner.
-; RUN: llc -march=hexagon < %s | FileCheck %s
+; RUN: llc -march=hexagon < %s -pipeliner-experimental-cg=true | FileCheck %s
 
 ; CHECK: endloop0
 ; CHECK: vmem
diff --git a/llvm/test/CodeGen/Hexagon/swp-epilog-phi10.ll b/llvm/test/CodeGen/Hexagon/swp-epilog-phi10.ll
index ff35e0b30..dce1c406f 100644
--- a/llvm/test/CodeGen/Hexagon/swp-epilog-phi10.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-epilog-phi10.ll
@@ -84,5 +84,5 @@ for.end:
   unreachable
 }
 
-attributes #0 = { norecurse nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="hexagonv5" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { norecurse nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="hexagonv5" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
diff --git a/llvm/test/CodeGen/Hexagon/swp-epilog-phi12.ll b/llvm/test/CodeGen/Hexagon/swp-epilog-phi12.ll
new file mode 100644
index 000000000..194dfa8d0
--- /dev/null
+++ b/llvm/test/CodeGen/Hexagon/swp-epilog-phi12.ll
@@ -0,0 +1,54 @@
+; RUN: llc -march=hexagon -hexagon-initial-cfg-cleanup=0 -pipeliner-experimental-cg=true < %s | FileCheck %s
+
+; Test epilogue generation when reading loop-carried dependency from a previous
+; stage. The first epilogue should read value from iteration N-1 of the kernel.
+
+; CHECK: loop0
+; CHECK: r{{[0-9]+}} = add([[REG0:r([0-9]+)]],#8)
+; CHECK: [[REG0]] = [[REG1:r([0-9]+)]]
+; CHECK: endloop0
+; CHECK: = add([[REG1]],#8)
+
+; Function Attrs: nounwind
+define i32* @f0(i16* nocapture readonly %a0, i32 %a1) #0 {
+b0:
+  %v0 = alloca [129 x i32], align 8
+  br i1 undef, label %b1, label %b3
+
+b1:                                               ; preds = %b0
+  br label %b2
+
+b2:                                               ; preds = %b2, %b1
+  %v1 = phi i16* [ %a0, %b1 ], [ %v2, %b2 ]
+  %v2 = phi i16* [ undef, %b1 ], [ %v15, %b2 ]
+  %v3 = phi i32* [ null, %b1 ], [ %v4, %b2 ]
+  %v4 = phi i32* [ null, %b1 ], [ %v14, %b2 ]
+  %v5 = phi i32 [ 0, %b1 ], [ %v13, %b2 ]
+  %v6 = phi i16* [ undef, %b1 ], [ %v12, %b2 ]
+  %v7 = load i16, i16* %v2, align 2
+  %v8 = sext i16 %v7 to i32
+  %v9 = call i32 @llvm.hexagon.M2.mpy.ll.s0(i32 %v8, i32 %v8) #2
+  %v10 = load i16, i16* %v6, align 2
+  %v11 = call i32 @llvm.hexagon.M2.mpy.acc.sat.ll.s0(i32 %v9, i32 undef, i32 undef) #2
+  store i32 %v11, i32* %v4, align 4
+  %v12 = getelementptr inbounds i16, i16* %v6, i32 -1
+  %v13 = add i32 %v5, 1
+  %v14 = getelementptr inbounds i32, i32* %v3, i32 2
+  %v15 = getelementptr inbounds i16, i16* %v1, i32 2
+  %v16 = icmp slt i32 %v13, %a1
+  br i1 %v16, label %b2, label %b3
+
+b3:                                               ; preds = %b2, %b0
+  %out = phi i32* [ null, %b0 ], [ %v14, %b2 ]
+  ret i32* %out
+}
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.hexagon.M2.mpy.ll.s0(i32, i32) #1
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.hexagon.M2.mpy.acc.sat.ll.s0(i32, i32, i32) #1
+
+attributes #0 = { nounwind "target-cpu"="hexagonv60" }
+attributes #1 = { nounwind readnone }
+attributes #2 = { nounwind }
diff --git a/llvm/test/CodeGen/Hexagon/swp-epilog-phi2.ll b/llvm/test/CodeGen/Hexagon/swp-epilog-phi2.ll
index b2a7dada3..b32fed97f 100644
--- a/llvm/test/CodeGen/Hexagon/swp-epilog-phi2.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-epilog-phi2.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon -enable-pipeliner -pipeliner-max-stages=3 < %s | FileCheck %s
+; RUN: llc -march=hexagon -enable-pipeliner -pipeliner-max-stages=3 < %s -pipeliner-experimental-cg=true | FileCheck %s
 
 %s.0 = type { i16, i8, i8, i16, i8, i8, i16, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i32, i16, i8, i8, %s.1, [2 x [16 x %s.2]], i32 (i8*, i8*, i8*, i8*, i8*)*, %s.3*, %s.3*, [120 x i8], i8, i8, %s.4*, [2 x [120 x [8 x i8]]], [56 x i8], [2 x [121 x %s.5]], [2 x %s.5], %s.5*, %s.5*, i32, i32, i16, i8, i8, %s.7, %s.9, %s.11, %s.8*, %s.8* }
 %s.1 = type { i8, i8, i8, i8, i8, i8, i8, i8, i32, i8, [16 x i8], i8, [4 x i8], [32 x i16], [32 x i16], [2 x i8], [4 x i8], [2 x [4 x i8]], [2 x [4 x i8]], i32, i32, i16, i8 }
diff --git a/llvm/test/CodeGen/Hexagon/swp-epilog-phi4.ll b/llvm/test/CodeGen/Hexagon/swp-epilog-phi4.ll
index e85ea7654..8b611cfe0 100644
--- a/llvm/test/CodeGen/Hexagon/swp-epilog-phi4.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-epilog-phi4.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon -enable-pipeliner < %s | FileCheck %s
+; RUN: llc -march=hexagon -enable-pipeliner < %s -pipeliner-experimental-cg=true | FileCheck %s
 
 ; Test that we generate the correct value for a Phi in the epilog
 ; that is for a value defined two stages earlier. An extra copy in the
diff --git a/llvm/test/CodeGen/Hexagon/swp-epilog-phi5.ll b/llvm/test/CodeGen/Hexagon/swp-epilog-phi5.ll
index a524dc0d5..72c05284d 100644
--- a/llvm/test/CodeGen/Hexagon/swp-epilog-phi5.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-epilog-phi5.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon < %s | FileCheck %s
+; RUN: llc -march=hexagon < %s -pipeliner-experimental-cg=true | FileCheck %s
 
 ; Test that we use the correct name in an epilog phi for a phi value
 ; that is defined for the last time in the kernel. Previously, we
diff --git a/llvm/test/CodeGen/Hexagon/swp-epilog-phi7.ll b/llvm/test/CodeGen/Hexagon/swp-epilog-phi7.ll
index 8cad17493..89a04d9f7 100644
--- a/llvm/test/CodeGen/Hexagon/swp-epilog-phi7.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-epilog-phi7.ll
@@ -7,8 +7,8 @@
 
 ; CHECK: if ({{.*}}) jump
 ; CHECK: [[VREG:v([0-9]+)]]{{.*}} = {{.*}}vmem(r{{[0-9]+}}++#1)
-; CHECK: if ({{.*}}) {{jump|jump:nt}} [[EPLOG1:(.*)]]
-; CHECK: if ({{.*}}) {{jump|jump:nt}} [[EPLOG:(.*)]]
+; CHECK: if ({{.*}}) {{jump|jump:nt|jump:t}} [[EPLOG1:(.*)]]
+; CHECK: if ({{.*}}) {{jump|jump:nt|jump:t}} [[EPLOG:(.*)]]
 ; CHECK: [[EPLOG]]:
 ; CHECK: [[VREG1:v([0-9]+)]] = [[VREG]]
 ; CHECK: [[VREG]] = v{{[0-9]+}}
diff --git a/llvm/test/CodeGen/Hexagon/swp-epilog-phi8.ll b/llvm/test/CodeGen/Hexagon/swp-epilog-phi8.ll
index 370d31d92..214307e25 100644
--- a/llvm/test/CodeGen/Hexagon/swp-epilog-phi8.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-epilog-phi8.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon -mno-pairing -mno-compound -hexagon-initial-cfg-cleanup=0 < %s | FileCheck %s
+; RUN: llc -march=hexagon -mno-pairing -mno-compound -hexagon-initial-cfg-cleanup=0 < %s -pipeliner-experimental-cg=true | FileCheck %s
 ; XFAIL: *
 
 ; Test that we generate the correct phi names in the epilog when the pipeliner
diff --git a/llvm/test/CodeGen/Hexagon/swp-kernel-phi1.ll b/llvm/test/CodeGen/Hexagon/swp-kernel-phi1.ll
index 681e74923..c87479f6e 100644
--- a/llvm/test/CodeGen/Hexagon/swp-kernel-phi1.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-kernel-phi1.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon -enable-pipeliner-opt-size -hexagon-initial-cfg-cleanup=0 < %s | FileCheck %s
+; RUN: llc -march=hexagon -enable-pipeliner-opt-size -hexagon-initial-cfg-cleanup=0 < %s -pipeliner-experimental-cg=true | FileCheck %s
 
 ; Test that we generate the correct names for the phis in the kernel for the
 ; incoming values. In this case, the loop contains a phi and has another phi
diff --git a/llvm/test/CodeGen/Hexagon/swp-large-rec.ll b/llvm/test/CodeGen/Hexagon/swp-large-rec.ll
index ee88aaffd..45d40df4e 100644
--- a/llvm/test/CodeGen/Hexagon/swp-large-rec.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-large-rec.ll
@@ -1,6 +1,6 @@
 ; RUN: llc -march=hexagon -enable-pipeliner -stats \
 ; RUN:     -pipeliner-prune-loop-carried=false -fp-contract=fast \
-; RUN:     -o /dev/null < %s 2>&1 | FileCheck %s --check-prefix=STATS
+; RUN:     -o /dev/null < %s 2>&1 -pipeliner-experimental-cg=true | FileCheck %s --check-prefix=STATS
 ; REQUIRES: asserts
 
 ; That that we do not pipeline this loop. The recurrence is too large. If
diff --git a/llvm/test/CodeGen/Hexagon/swp-listen-loop3.ll b/llvm/test/CodeGen/Hexagon/swp-listen-loop3.ll
index 9b68cf956..d8e4f003d 100644
--- a/llvm/test/CodeGen/Hexagon/swp-listen-loop3.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-listen-loop3.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon -pipeliner-ignore-recmii -pipeliner-max-stages=2 -enable-pipeliner < %s | FileCheck %s
+; RUN: llc -march=hexagon -pipeliner-ignore-recmii -pipeliner-max-stages=2 -enable-pipeliner < %s -pipeliner-experimental-cg=true | FileCheck %s
 
 ; This is a loop we pipeline to three packets, though we could do bettter.
 
diff --git a/llvm/test/CodeGen/Hexagon/swp-loop-carried-unknown.ll b/llvm/test/CodeGen/Hexagon/swp-loop-carried-unknown.ll
index b95e62419..9f1451897 100644
--- a/llvm/test/CodeGen/Hexagon/swp-loop-carried-unknown.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-loop-carried-unknown.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon -hexagon-initial-cfg-cleanup=0 < %s | FileCheck %s
+; RUN: llc -march=hexagon -hexagon-initial-cfg-cleanup=0 < %s -pipeliner-experimental-cg=true | FileCheck %s
 
 ; Test that the pipeliner schedules a store before the load in which there is a
 ; loop carried dependence. Previously, the loop carried dependence wasn't added
diff --git a/llvm/test/CodeGen/Hexagon/swp-lots-deps.ll b/llvm/test/CodeGen/Hexagon/swp-lots-deps.ll
index a657f92c5..631d02649 100644
--- a/llvm/test/CodeGen/Hexagon/swp-lots-deps.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-lots-deps.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon -enable-pipeliner -stats -o /dev/null < %s 2>&1 | FileCheck %s --check-prefix=STATS
+; RUN: llc -march=hexagon -enable-pipeliner -stats -o /dev/null < %s 2>&1 -pipeliner-experimental-cg=true | FileCheck %s --check-prefix=STATS
 ; REQUIRES: asserts
 
 ; STATS: 1 pipeliner        - Number of loops software pipelined
diff --git a/llvm/test/CodeGen/Hexagon/swp-max.ll b/llvm/test/CodeGen/Hexagon/swp-max.ll
index 26238ea6f..32282204e 100644
--- a/llvm/test/CodeGen/Hexagon/swp-max.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-max.ll
@@ -1,5 +1,5 @@
 ; RUN: llc -march=hexagon -mcpu=hexagonv5 -enable-pipeliner \
-; RUN:     -pipeliner-max-stages=2 < %s | FileCheck %s
+; RUN:     -pipeliner-max-stages=2 < %s -pipeliner-experimental-cg=true | FileCheck %s
 
 @A = global [8 x i32] [i32 4, i32 -3, i32 5, i32 -2, i32 -1, i32 2, i32 6, i32 -2], align 8
 
diff --git a/llvm/test/CodeGen/Hexagon/swp-maxstart.ll b/llvm/test/CodeGen/Hexagon/swp-maxstart.ll
index 811c94062..8d65e7691 100644
--- a/llvm/test/CodeGen/Hexagon/swp-maxstart.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-maxstart.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon -O3 < %s | FileCheck %s
+; RUN: llc -march=hexagon -O3 < %s -pipeliner-experimental-cg=true | FileCheck %s
 
 ; Test that the MinStart computation, which is based upon the length
 ; of the chain edges, is computed correctly. A bug in the code allowed
diff --git a/llvm/test/CodeGen/Hexagon/swp-memrefs-epilog.ll b/llvm/test/CodeGen/Hexagon/swp-memrefs-epilog.ll
index 81f4d22cf..20e39dd08 100644
--- a/llvm/test/CodeGen/Hexagon/swp-memrefs-epilog.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-memrefs-epilog.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon -O2 -fp-contract=fast < %s | FileCheck %s
+; RUN: llc -march=hexagon -O2 -fp-contract=fast < %s -pipeliner-experimental-cg=true | FileCheck %s
 
 ; Test that the memoperands for instructions in the epilog are updated
 ; correctly. Previously, the pipeliner updated the offset for the memoperands
diff --git a/llvm/test/CodeGen/Hexagon/swp-multi-loops.ll b/llvm/test/CodeGen/Hexagon/swp-multi-loops.ll
index fc2576af8..5a2e7d4e1 100644
--- a/llvm/test/CodeGen/Hexagon/swp-multi-loops.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-multi-loops.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon -mcpu=hexagonv5 -enable-pipeliner < %s | FileCheck %s
+; RUN: llc -march=hexagon -mcpu=hexagonv5 -enable-pipeliner < %s -pipeliner-experimental-cg=true | FileCheck %s
 
 ; Make sure we attempt to pipeline all inner most loops.
 
diff --git a/llvm/test/CodeGen/Hexagon/swp-new-phi.ll b/llvm/test/CodeGen/Hexagon/swp-new-phi.ll
index 0ba3e3073..d3c1058fe 100644
--- a/llvm/test/CodeGen/Hexagon/swp-new-phi.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-new-phi.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon -enable-pipeliner -pipeliner-max-stages=2 < %s | FileCheck %s
+; RUN: llc -march=hexagon -enable-pipeliner -pipeliner-max-stages=2 < %s -pipeliner-experimental-cg=true | FileCheck %s
 
 ; Test that the generatePhi code doesn't rename a a Phi instruction that's defined
 ; in the same block.  The bug causes a Phi to incorrectly depend on another Phi.
diff --git a/llvm/test/CodeGen/Hexagon/swp-order-copies.ll b/llvm/test/CodeGen/Hexagon/swp-order-copies.ll
index 5de071765..0a017c4ab 100644
--- a/llvm/test/CodeGen/Hexagon/swp-order-copies.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-order-copies.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon < %s | FileCheck %s
+; RUN: llc -march=hexagon < %s -pipeliner-experimental-cg=true | FileCheck %s
 
 ; Test that the instruction ordering code in the pipeliner fixes up dependences
 ; between post-increment register definitions and uses so that the register
diff --git a/llvm/test/CodeGen/Hexagon/swp-order-deps7.ll b/llvm/test/CodeGen/Hexagon/swp-order-deps7.ll
index d1d852bca..4cd29a4a0 100644
--- a/llvm/test/CodeGen/Hexagon/swp-order-deps7.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-order-deps7.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon < %s | FileCheck %s
+; RUN: llc -march=hexagon < %s -pipeliner-experimental-cg=true | FileCheck %s
 
 ; Test that the pipeliner cause an assert and correctly pipelines the
 ; loop.
diff --git a/llvm/test/CodeGen/Hexagon/swp-order.ll b/llvm/test/CodeGen/Hexagon/swp-order.ll
index bc16b8835..14cc682eb 100644
--- a/llvm/test/CodeGen/Hexagon/swp-order.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-order.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon < %s | FileCheck %s
+; RUN: llc -march=hexagon < %s -pipeliner-experimental-cg=true | FileCheck %s
 
 ; Test that when we order instructions in a packet we check for
 ; order dependences so that the source of an order dependence
diff --git a/llvm/test/CodeGen/Hexagon/swp-phi-ch-offset.ll b/llvm/test/CodeGen/Hexagon/swp-phi-ch-offset.ll
index 68cb69ba2..31b98328a 100644
--- a/llvm/test/CodeGen/Hexagon/swp-phi-ch-offset.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-phi-ch-offset.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon -enable-pipeliner -pipeliner-max-stages=2 < %s | FileCheck %s
+; RUN: llc -march=hexagon -enable-pipeliner -pipeliner-max-stages=2 < %s -pipeliner-experimental-cg=true | FileCheck %s
 
 ; Test that we generate the correct offsets after we removed unneeded
 ; chain dependences between Phis and generated a better pipeline.
diff --git a/llvm/test/CodeGen/Hexagon/swp-phi-chains.ll b/llvm/test/CodeGen/Hexagon/swp-phi-chains.ll
index e3a6c9db6..3037dcc2d 100644
--- a/llvm/test/CodeGen/Hexagon/swp-phi-chains.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-phi-chains.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon -debug-only=pipeliner < %s -o - 2>&1 | FileCheck %s
+; RUN: llc -march=hexagon -debug-only=pipeliner < %s -o - 2>&1 -pipeliner-experimental-cg=true | FileCheck %s
 ; REQUIRES: asserts
 
 ; Test that there is a chain edge between two dependent Phis.
diff --git a/llvm/test/CodeGen/Hexagon/swp-phi-dep.ll b/llvm/test/CodeGen/Hexagon/swp-phi-dep.ll
index 38b56c112..ec7af41a3 100644
--- a/llvm/test/CodeGen/Hexagon/swp-phi-dep.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-phi-dep.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon -hexagon-initial-cfg-cleanup=0 -enable-pipeliner -pipeliner-max-stages=2 < %s | FileCheck %s
+; RUN: llc -march=hexagon -hexagon-initial-cfg-cleanup=0 -enable-pipeliner -pipeliner-max-stages=2 < %s -pipeliner-experimental-cg=true | FileCheck %s
 
 ; Check that the pipelined code uses the proper address in the
 ; prolog and the kernel. The bug occurs when the address computation
diff --git a/llvm/test/CodeGen/Hexagon/swp-phi-ref.ll b/llvm/test/CodeGen/Hexagon/swp-phi-ref.ll
index d39252141..be838e767 100644
--- a/llvm/test/CodeGen/Hexagon/swp-phi-ref.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-phi-ref.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon -enable-pipeliner -enable-bsb-sched=0 -join-liveintervals=false < %s | FileCheck %s
+; RUN: llc -march=hexagon -enable-pipeliner -enable-bsb-sched=0 -join-liveintervals=false < %s -pipeliner-experimental-cg=true | FileCheck %s
 
 ; XFAIL: *
 ; This test is failing after post-ra machine sinking.
diff --git a/llvm/test/CodeGen/Hexagon/swp-prolog-phi.ll b/llvm/test/CodeGen/Hexagon/swp-prolog-phi.ll
index 4a6fa5a6c..14b04a1cf 100644
--- a/llvm/test/CodeGen/Hexagon/swp-prolog-phi.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-prolog-phi.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon -rdf-opt=0 < %s | FileCheck %s
+; RUN: llc -march=hexagon -rdf-opt=0 < %s -pipeliner-experimental-cg=true | FileCheck %s
 
 ; Test that we generate the correct name for a value in a prolog block. The
 ; pipeliner was using an incorrect value for an instruction in the 2nd prolog
diff --git a/llvm/test/CodeGen/Hexagon/swp-prolog-phi4.ll b/llvm/test/CodeGen/Hexagon/swp-prolog-phi4.ll
index 5ed0514ef..8535ccb67 100644
--- a/llvm/test/CodeGen/Hexagon/swp-prolog-phi4.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-prolog-phi4.ll
@@ -62,4 +62,4 @@ for.end:
   br label %for.body
 }
 
-attributes #0 = { norecurse nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="hexagonv5" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { norecurse nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="hexagonv5" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/Hexagon/swp-rename.ll b/llvm/test/CodeGen/Hexagon/swp-rename.ll
index ab0cc11ec..eb60a0e38 100644
--- a/llvm/test/CodeGen/Hexagon/swp-rename.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-rename.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon -enable-pipeliner < %s | FileCheck %s
+; RUN: llc -march=hexagon -enable-pipeliner < %s -pipeliner-experimental-cg=true | FileCheck %s
 
 ; A test that the Phi rewrite logic is correct.
 
diff --git a/llvm/test/CodeGen/Hexagon/swp-resmii-1.ll b/llvm/test/CodeGen/Hexagon/swp-resmii-1.ll
index 1a9734860..1e2eefae6 100644
--- a/llvm/test/CodeGen/Hexagon/swp-resmii-1.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-resmii-1.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon -enable-pipeliner -debug-only=pipeliner < %s -o - 2>&1 > /dev/null | FileCheck %s
+; RUN: llc -march=hexagon -enable-pipeliner -debug-only=pipeliner < %s -o - 2>&1 > /dev/null -pipeliner-experimental-cg=true | FileCheck %s
 ; REQUIRES: asserts
 
 ; Test that checks that we compute the correct ResMII for haar.
diff --git a/llvm/test/CodeGen/Hexagon/swp-resmii.ll b/llvm/test/CodeGen/Hexagon/swp-resmii.ll
index 851d82ea5..99812af3b 100644
--- a/llvm/test/CodeGen/Hexagon/swp-resmii.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-resmii.ll
@@ -1,5 +1,5 @@
 ; RUN: llc -disable-lsr -march=hexagon -enable-pipeliner  \
-; RUN:     -debug-only=pipeliner < %s 2>&1 > /dev/null | FileCheck %s
+; RUN:     -debug-only=pipeliner < %s 2>&1 > /dev/null -pipeliner-experimental-cg=true | FileCheck %s
 ; REQUIRES: asserts
 ;
 ; Test that checks if the ResMII is 1.
diff --git a/llvm/test/CodeGen/Hexagon/swp-reuse-phi-6.ll b/llvm/test/CodeGen/Hexagon/swp-reuse-phi-6.ll
index 7371ed10a..6883e5150 100644
--- a/llvm/test/CodeGen/Hexagon/swp-reuse-phi-6.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-reuse-phi-6.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon < %s | FileCheck %s
+; RUN: llc -march=hexagon < %s -pipeliner-experimental-cg=true | FileCheck %s
 
 ; Test that the pipeliner generates correct code when attempting to reuse
 ; an existing phi. This test case contains a phi that references another
diff --git a/llvm/test/CodeGen/Hexagon/swp-sigma.ll b/llvm/test/CodeGen/Hexagon/swp-sigma.ll
index 56742ca83..968fafc44 100644
--- a/llvm/test/CodeGen/Hexagon/swp-sigma.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-sigma.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon -O2 < %s | FileCheck %s
+; RUN: llc -march=hexagon -O2 < %s -pipeliner-experimental-cg=true | FileCheck %s
 
 ; We do not pipeline sigma yet, but the non-pipelined version
 ; with good scheduling is pretty fast. The compiler generates
diff --git a/llvm/test/CodeGen/Hexagon/swp-stages4.ll b/llvm/test/CodeGen/Hexagon/swp-stages4.ll
index 2d88094cf..9d8fa5234 100644
--- a/llvm/test/CodeGen/Hexagon/swp-stages4.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-stages4.ll
@@ -1,11 +1,11 @@
-; RUN: llc -march=hexagon -mcpu=hexagonv5 -enable-pipeliner -pipeliner-max-stages=2 -disable-block-placement=0 -hexagon-bit=0 < %s | FileCheck %s
+; RUN: llc -march=hexagon -mcpu=hexagonv5 -enable-pipeliner -pipeliner-max-stages=2 -disable-block-placement=0 -hexagon-bit=0 < %s -pipeliner-experimental-cg=true | FileCheck %s
 
 ; Test that we rename registers correctly for multiple stages when there is a
 ; Phi and depends upon another Phi.
 
 ; CHECK: = and
 ; CHECK: = and
-; CHECK: = and
+; CHECK: r[[REGA:[0-9]+]] = memub(r{{[0-9]+}}+#1)
 ; CHECK: r[[REG0:[0-9]+]] = and(r[[REG1:[0-9]+]],#255)
 ; CHECK-NOT: r[[REG0]] = and(r[[REG1]],#255)
 ; CHECK: loop0(.LBB0_[[LOOP:.]],
@@ -87,7 +87,7 @@ for.end104:
   ret void
 }
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf"="true" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !0 = !{!"omnipotent char", !1}
 !1 = !{!"Simple C/C++ TBAA"}
diff --git a/llvm/test/CodeGen/Hexagon/swp-subreg.ll b/llvm/test/CodeGen/Hexagon/swp-subreg.ll
index d75b3afc7..b9754f4eb 100644
--- a/llvm/test/CodeGen/Hexagon/swp-subreg.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-subreg.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon -enable-pipeliner -stats -o /dev/null < %s 2>&1 | FileCheck %s --check-prefix=STATS
+; RUN: llc -march=hexagon -enable-pipeliner -stats -o /dev/null < %s 2>&1 -pipeliner-experimental-cg=true | FileCheck %s --check-prefix=STATS
 ; REQUIRES: asserts
 
 ; We're unable to pipeline a loop with a subreg as an operand of a Phi.
diff --git a/llvm/test/CodeGen/Hexagon/swp-swap.ll b/llvm/test/CodeGen/Hexagon/swp-swap.ll
index a8432cb7d..4cd073cb1 100644
--- a/llvm/test/CodeGen/Hexagon/swp-swap.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-swap.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon -enable-pipeliner -stats -o /dev/null < %s 2>&1 | FileCheck %s --check-prefix=STATS
+; RUN: llc -march=hexagon -enable-pipeliner -stats -o /dev/null < %s 2>&1 -pipeliner-experimental-cg=true | FileCheck %s --check-prefix=STATS
 ; REQUIRES: asserts
 
 ; Test that we don't pipeline, incorrectly, the swap operation.
diff --git a/llvm/test/CodeGen/Hexagon/swp-tfri.ll b/llvm/test/CodeGen/Hexagon/swp-tfri.ll
index 66b999e55..f0c260454 100644
--- a/llvm/test/CodeGen/Hexagon/swp-tfri.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-tfri.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon -enable-pipeliner -hexagon-initial-cfg-cleanup=0 -stats -o /dev/null < %s 2>&1 | FileCheck %s --check-prefix=STATS
+; RUN: llc -march=hexagon -enable-pipeliner -hexagon-initial-cfg-cleanup=0 -stats -o /dev/null < %s 2>&1 -pipeliner-experimental-cg=true | FileCheck %s --check-prefix=STATS
 ; REQUIRES: asserts
 
 ; Check that we handle the case when a value is first defined in the loop.
diff --git a/llvm/test/CodeGen/Hexagon/swp-vect-dotprod.ll b/llvm/test/CodeGen/Hexagon/swp-vect-dotprod.ll
index 3ff884524..4bd1a5134 100644
--- a/llvm/test/CodeGen/Hexagon/swp-vect-dotprod.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-vect-dotprod.ll
@@ -1,6 +1,6 @@
-; RUN: llc -march=hexagon -mcpu=hexagonv5 -enable-pipeliner < %s | FileCheck %s
-; RUN: llc -march=hexagon -mcpu=hexagonv5 -O2 < %s | FileCheck %s
-; RUN: llc -march=hexagon -mcpu=hexagonv5 -O3 < %s | FileCheck %s
+; RUN: llc -march=hexagon -mcpu=hexagonv5 -enable-pipeliner < %s -pipeliner-experimental-cg=true | FileCheck %s
+; RUN: llc -march=hexagon -mcpu=hexagonv5 -O2 < %s -pipeliner-experimental-cg=true | FileCheck %s
+; RUN: llc -march=hexagon -mcpu=hexagonv5 -O3 < %s -pipeliner-experimental-cg=true | FileCheck %s
 ;
 ; Check that we pipeline a vectorized dot product in a single packet.
 ;
diff --git a/llvm/test/CodeGen/Hexagon/swp-vmult.ll b/llvm/test/CodeGen/Hexagon/swp-vmult.ll
index dfc7dd913..fd9cdf9b3 100644
--- a/llvm/test/CodeGen/Hexagon/swp-vmult.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-vmult.ll
@@ -1,5 +1,5 @@
 ; REQUIRES: to-be-fixed
-; RUN: llc -march=hexagon -mcpu=hexagonv5 -enable-pipeliner < %s | FileCheck %s
+; RUN: llc -march=hexagon -mcpu=hexagonv5 -enable-pipeliner < %s -pipeliner-experimental-cg=true | FileCheck %s
 
 ; Multiply and accumulate
 ; CHECK: mpyi([[REG0:r([0-9]+)]],[[REG1:r([0-9]+)]])
diff --git a/llvm/test/CodeGen/Hexagon/swp-vsum.ll b/llvm/test/CodeGen/Hexagon/swp-vsum.ll
index 1c4d1c2ef..5dcd28245 100644
--- a/llvm/test/CodeGen/Hexagon/swp-vsum.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-vsum.ll
@@ -1,5 +1,5 @@
-; RUN: llc -march=hexagon -mcpu=hexagonv5 -enable-pipeliner < %s | FileCheck %s
-; RUN: llc -march=hexagon -mcpu=hexagonv60 -enable-pipeliner < %s | FileCheck %s --check-prefix=CHECKV60
+; RUN: llc -march=hexagon -mcpu=hexagonv5 -enable-pipeliner < %s -pipeliner-experimental-cg=true | FileCheck %s
+; RUN: llc -march=hexagon -mcpu=hexagonv60 -enable-pipeliner < %s -pipeliner-experimental-cg=true | FileCheck %s --check-prefix=CHECKV60
 
 ; Simple vector total.
 ; CHECK: loop0(.LBB0_[[LOOP:.]],
diff --git a/llvm/test/CodeGen/Hexagon/tstbit.ll b/llvm/test/CodeGen/Hexagon/tstbit.ll
index ac1799007..a5d618351 100644
--- a/llvm/test/CodeGen/Hexagon/tstbit.ll
+++ b/llvm/test/CodeGen/Hexagon/tstbit.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -march=hexagon < %s | FileCheck %s
+; RUN: llc -mtriple=hexagon < %s | FileCheck %s
 
 ; Function Attrs: nounwind readnone
 define i32 @f0(i32 %a0, i32 %a1) #0 {
@@ -20,4 +20,87 @@ b0:
   ret i32 %v3
 }
 
+define i64 @is_upper_bit_clear_i64(i64 %x) #0 {
+; CHECK-LABEL: is_upper_bit_clear_i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    {
+; CHECK-NEXT:     p0 = !tstbit(r1,#5)
+; CHECK-NEXT:     r1 = #0
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r0 = mux(p0,#1,#0)
+; CHECK-NEXT:     jumpr r31
+; CHECK-NEXT:    }
+  %sh = lshr i64 %x, 37
+  %m = and i64 %sh, 1
+  %r = xor i64 %m, 1
+  ret i64 %r
+}
+
+define i64 @is_lower_bit_clear_i64(i64 %x) #0 {
+; CHECK-LABEL: is_lower_bit_clear_i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    {
+; CHECK-NEXT:     p0 = !tstbit(r0,#27)
+; CHECK-NEXT:     r1 = #0
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r0 = mux(p0,#1,#0)
+; CHECK-NEXT:     jumpr r31
+; CHECK-NEXT:    }
+  %sh = lshr i64 %x, 27
+  %m = and i64 %sh, 1
+  %r = xor i64 %m, 1
+  ret i64 %r
+}
+
+define i32 @is_bit_clear_i32(i32 %x) #0 {
+; CHECK-LABEL: is_bit_clear_i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    {
+; CHECK-NEXT:     p0 = !tstbit(r0,#27)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r0 = mux(p0,#1,#0)
+; CHECK-NEXT:     jumpr r31
+; CHECK-NEXT:    }
+  %sh = lshr i32 %x, 27
+  %n = xor i32 %sh, -1
+  %r = and i32 %n, 1
+  ret i32 %r
+}
+
+define i16 @is_bit_clear_i16(i16 %x) #0 {
+; CHECK-LABEL: is_bit_clear_i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    {
+; CHECK-NEXT:     p0 = !tstbit(r0,#7)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r0 = mux(p0,#1,#0)
+; CHECK-NEXT:     jumpr r31
+; CHECK-NEXT:    }
+  %sh = lshr i16 %x, 7
+  %m = and i16 %sh, 1
+  %r = xor i16 %m, 1
+  ret i16 %r
+}
+
+define i8 @is_bit_clear_i8(i8 %x) #0 {
+; CHECK-LABEL: is_bit_clear_i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    {
+; CHECK-NEXT:     p0 = !tstbit(r0,#3)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r0 = mux(p0,#1,#0)
+; CHECK-NEXT:     jumpr r31
+; CHECK-NEXT:    }
+  %sh = lshr i8 %x, 3
+  %m = and i8 %sh, 1
+  %r = xor i8 %m, 1
+  ret i8 %r
+}
+
+
 attributes #0 = { nounwind readnone }
diff --git a/llvm/test/CodeGen/Hexagon/usr-ovf-dep.ll b/llvm/test/CodeGen/Hexagon/usr-ovf-dep.ll
index 1f06986f0..92c7be61b 100644
--- a/llvm/test/CodeGen/Hexagon/usr-ovf-dep.ll
+++ b/llvm/test/CodeGen/Hexagon/usr-ovf-dep.ll
@@ -23,6 +23,6 @@ entry:
 ; Function Attrs: nounwind readnone
 declare i32 @llvm.hexagon.S2.asr.r.r.sat(i32, i32) #1
 
-attributes #0 = { nounwind readnone "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind readnone "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
diff --git a/llvm/test/CodeGen/Hexagon/v6-unaligned-spill.ll b/llvm/test/CodeGen/Hexagon/v6-unaligned-spill.ll
index 2b7203937..be53694c4 100644
--- a/llvm/test/CodeGen/Hexagon/v6-unaligned-spill.ll
+++ b/llvm/test/CodeGen/Hexagon/v6-unaligned-spill.ll
@@ -4,7 +4,7 @@
 ; has an alloca. Also, make sure the addressing mode for unaligned store does
 ; is not a base+offset with a non-zero offset that is not a multiple of 128.
 
-; CHECK: vmemu(r{{[0-9]+}}+#0)
+; CHECK: vmemu({{.*}}) =
 
 %s.0 = type { [5 x [4 x i8]], i32, i32, i32, i32 }
 
diff --git a/llvm/test/CodeGen/Hexagon/v60Intrins.ll b/llvm/test/CodeGen/Hexagon/v60Intrins.ll
index 8c9804b54..45c122c1c 100644
--- a/llvm/test/CodeGen/Hexagon/v60Intrins.ll
+++ b/llvm/test/CodeGen/Hexagon/v60Intrins.ll
@@ -2554,5 +2554,5 @@ declare <32 x i32> @llvm.hexagon.V6.vunpackh(<16 x i32>) #1
 ; Function Attrs: nounwind readnone
 declare <32 x i32> @llvm.hexagon.V6.vunpackoh(<32 x i32>, <16 x i32>) #1
 
-attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="hexagonv60" "target-features"="+hvxv60,+hvx-length64b" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="hexagonv60" "target-features"="+hvxv60,+hvx-length64b" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
diff --git a/llvm/test/CodeGen/Hexagon/v60small.ll b/llvm/test/CodeGen/Hexagon/v60small.ll
index efa726e2c..746af018b 100644
--- a/llvm/test/CodeGen/Hexagon/v60small.ll
+++ b/llvm/test/CodeGen/Hexagon/v60small.ll
@@ -47,5 +47,5 @@ declare <512 x i1> @llvm.hexagon.V6.pred.and(<512 x i1>, <512 x i1>) #1
 ; Function Attrs: nounwind readnone
 declare <512 x i1> @llvm.hexagon.V6.pred.and.n(<512 x i1>, <512 x i1>) #1
 
-attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="hexagonv60" "target-features"="+hvxv60,+hvx-length64b" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="hexagonv60" "target-features"="+hvxv60,+hvx-length64b" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
diff --git a/llvm/test/CodeGen/Hexagon/v6vec-vprint.ll b/llvm/test/CodeGen/Hexagon/v6vec-vprint.ll
index 18c2cf65f..944d0891e 100644
--- a/llvm/test/CodeGen/Hexagon/v6vec-vprint.ll
+++ b/llvm/test/CodeGen/Hexagon/v6vec-vprint.ll
@@ -25,7 +25,7 @@ entry:
 ; Function Attrs: nounwind readnone
 declare <16 x i32> @llvm.hexagon.V6.vaddw(<16 x i32>, <16 x i32>) #1
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !llvm.ident = !{!0}
diff --git a/llvm/test/CodeGen/Hexagon/vec-pred-spill1.ll b/llvm/test/CodeGen/Hexagon/vec-pred-spill1.ll
index a0112a098..08f7e3357 100644
--- a/llvm/test/CodeGen/Hexagon/vec-pred-spill1.ll
+++ b/llvm/test/CodeGen/Hexagon/vec-pred-spill1.ll
@@ -70,8 +70,8 @@ declare void @print_vecpred(i32, i8*) #1
 ; Function Attrs: nounwind
 declare i32 @puts(i8* nocapture readonly) #3
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind readnone }
 attributes #3 = { nounwind }
 
diff --git a/llvm/test/CodeGen/Hexagon/vect/vect-bad-bitcast.ll b/llvm/test/CodeGen/Hexagon/vect/vect-bad-bitcast.ll
index 1672a789a..460e39e11 100644
--- a/llvm/test/CodeGen/Hexagon/vect/vect-bad-bitcast.ll
+++ b/llvm/test/CodeGen/Hexagon/vect/vect-bad-bitcast.ll
@@ -52,8 +52,8 @@ polly.stmt.for.body:                              ; preds = %entry, %polly.stmt.
 
 declare i32 @fxpBitAllocation(...) #1
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind }
 
 !llvm.ident = !{!0}
diff --git a/llvm/test/CodeGen/Hexagon/vect/vect-extract.ll b/llvm/test/CodeGen/Hexagon/vect/vect-extract.ll
index 75dc6850f..1a7eaf9b8 100644
--- a/llvm/test/CodeGen/Hexagon/vect/vect-extract.ll
+++ b/llvm/test/CodeGen/Hexagon/vect/vect-extract.ll
@@ -92,5 +92,5 @@ polly.stmt.for.body331:                           ; preds = %polly.stmt.for.body
 
 declare i64 @llvm.hexagon.A2.combinew(i32, i32) #1
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf"="true" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
diff --git a/llvm/test/CodeGen/Hexagon/vect/vect-shuffle.ll b/llvm/test/CodeGen/Hexagon/vect/vect-shuffle.ll
index 29559cbd2..da5016fb6 100644
--- a/llvm/test/CodeGen/Hexagon/vect/vect-shuffle.ll
+++ b/llvm/test/CodeGen/Hexagon/vect/vect-shuffle.ll
@@ -45,7 +45,7 @@ polly.loop_body:                                  ; preds = %entry, %polly.loop_
 
 declare i64 @llvm.hexagon.A2.combinew(i32, i32) #1
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf"="true" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 
diff --git a/llvm/test/CodeGen/Hexagon/vect/vect-v4i16.ll b/llvm/test/CodeGen/Hexagon/vect/vect-v4i16.ll
index 83d56ed9e..7b12c8f37 100644
--- a/llvm/test/CodeGen/Hexagon/vect/vect-v4i16.ll
+++ b/llvm/test/CodeGen/Hexagon/vect/vect-v4i16.ll
@@ -68,4 +68,4 @@ polly.stmt.for.body29:                            ; preds = %polly.stmt.for.body
   br i1 %exitcond, label %for.end.loopexit, label %polly.stmt.for.body29
 }
 
-attributes #0 = { nounwind "fp-contract-model"="standard" "no-frame-pointer-elim-non-leaf" "realign-stack" "relocation-model"="static" "ssp-buffers-size"="8" }
+attributes #0 = { nounwind "fp-contract-model"="standard" "frame-pointer"="non-leaf" "realign-stack" "relocation-model"="static" "ssp-buffers-size"="8" }
diff --git a/llvm/test/CodeGen/Hexagon/vect/vect-vshifts.ll b/llvm/test/CodeGen/Hexagon/vect/vect-vshifts.ll
index 9d3cbe6e1..229122223 100644
--- a/llvm/test/CodeGen/Hexagon/vect/vect-vshifts.ll
+++ b/llvm/test/CodeGen/Hexagon/vect/vect-vshifts.ll
@@ -271,7 +271,7 @@ declare i64 @llvm.hexagon.C2.vmux(i32, i64, i64) #1
 
 declare i64 @llvm.hexagon.S2.asl.r.vw(i64, i32) #1
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf"="true" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !0 = !{!"int", !1}
diff --git a/llvm/test/CodeGen/Hexagon/vect/vect-xor.ll b/llvm/test/CodeGen/Hexagon/vect/vect-xor.ll
index 8864ab5c5..e3c8c7783 100644
--- a/llvm/test/CodeGen/Hexagon/vect/vect-xor.ll
+++ b/llvm/test/CodeGen/Hexagon/vect/vect-xor.ll
@@ -35,4 +35,4 @@ polly.loop_body:                                  ; preds = %entry, %polly.loop_
   br i1 %0, label %polly.loop_body, label %polly.loop_after
 }
 
-attributes #0 = { nounwind "fp-contract-model"="standard" "no-frame-pointer-elim-non-leaf" "realign-stack" "relocation-model"="static" "ssp-buffers-size"="8" }
+attributes #0 = { nounwind "fp-contract-model"="standard" "frame-pointer"="non-leaf" "realign-stack" "relocation-model"="static" "ssp-buffers-size"="8" }
diff --git a/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info.ll b/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info.ll
index 79d3d82cc..f9de722c5 100644
--- a/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info.ll
+++ b/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info.ll
@@ -28,6 +28,9 @@
 ; CHECK-NEXT: mode:
 ; CHECK-NEXT: ieee: true
 ; CHECK-NEXT: dx10-clamp: true
+; CHECK-NEXT: fp32-denormals: false
+; CHECK-NEXT: fp64-fp16-denormals: true
+; CHECK-NEXT: highBitsOf32BitAddress: 0
 ; CHECK-NEXT: body:
 define amdgpu_kernel void @kernel(i32 %arg0, i64 %arg1, <16 x i32> %arg2) {
   %gep = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %arg0
@@ -38,7 +41,7 @@ define amdgpu_kernel void @kernel(i32 %arg0, i64 %arg1, <16 x i32> %arg2) {
 ; CHECK-LABEL: {{^}}name: ps_shader
 ; CHECK: machineFunctionInfo:
 ; CHECK-NEXT: explicitKernArgSize: 0
-; CHECK-NEXT: maxKernArgAlign: 0
+; CHECK-NEXT: maxKernArgAlign: 1
 ; CHECK-NEXT: ldsSize: 0
 ; CHECK-NEXT: isEntryFunction: true
 ; CHECK-NEXT: noSignedZerosFPMath: false
@@ -54,6 +57,9 @@ define amdgpu_kernel void @kernel(i32 %arg0, i64 %arg1, <16 x i32> %arg2) {
 ; CHECK-NEXT: mode:
 ; CHECK-NEXT: ieee: false
 ; CHECK-NEXT: dx10-clamp: true
+; CHECK-NEXT: fp32-denormals: false
+; CHECK-NEXT: fp64-fp16-denormals: true
+; CHECK-NEXT: highBitsOf32BitAddress: 0
 ; CHECK-NEXT: body:
 define amdgpu_ps void @ps_shader(i32 %arg0, i32 inreg %arg1) {
   ret void
@@ -62,7 +68,7 @@ define amdgpu_ps void @ps_shader(i32 %arg0, i32 inreg %arg1) {
 ; CHECK-LABEL: {{^}}name: function
 ; CHECK: machineFunctionInfo:
 ; CHECK-NEXT: explicitKernArgSize: 0
-; CHECK-NEXT: maxKernArgAlign: 0
+; CHECK-NEXT: maxKernArgAlign: 1
 ; CHECK-NEXT: ldsSize: 0
 ; CHECK-NEXT: isEntryFunction: false
 ; CHECK-NEXT: noSignedZerosFPMath: false
@@ -78,6 +84,9 @@ define amdgpu_ps void @ps_shader(i32 %arg0, i32 inreg %arg1) {
 ; CHECK-NEXT: mode:
 ; CHECK-NEXT: ieee: true
 ; CHECK-NEXT: dx10-clamp: true
+; CHECK-NEXT: fp32-denormals: false
+; CHECK-NEXT: fp64-fp16-denormals: true
+; CHECK-NEXT: highBitsOf32BitAddress: 0
 ; CHECK-NEXT: body:
 define void @function() {
   ret void
@@ -86,7 +95,7 @@ define void @function() {
 ; CHECK-LABEL: {{^}}name: function_nsz
 ; CHECK: machineFunctionInfo:
 ; CHECK-NEXT: explicitKernArgSize: 0
-; CHECK-NEXT: maxKernArgAlign: 0
+; CHECK-NEXT: maxKernArgAlign: 1
 ; CHECK-NEXT: ldsSize: 0
 ; CHECK-NEXT: isEntryFunction: false
 ; CHECK-NEXT: noSignedZerosFPMath: true
@@ -102,6 +111,9 @@ define void @function() {
 ; CHECK-NEXT: mode:
 ; CHECK-NEXT: ieee: true
 ; CHECK-NEXT: dx10-clamp: true
+; CHECK-NEXT: fp32-denormals: false
+; CHECK-NEXT: fp64-fp16-denormals: true
+; CHECK-NEXT: highBitsOf32BitAddress: 0
 ; CHECK-NEXT: body:
 define void @function_nsz() #0 {
   ret void
@@ -111,6 +123,8 @@ define void @function_nsz() #0 {
 ; CHECK: mode:
 ; CHECK-NEXT: ieee: true
 ; CHECK-NEXT: dx10-clamp: false
+; CHECK-NEXT: fp32-denormals: false
+; CHECK-NEXT: fp64-fp16-denormals: true
 define void @function_dx10_clamp_off() #1 {
   ret void
 }
@@ -119,6 +133,8 @@ define void @function_dx10_clamp_off() #1 {
 ; CHECK: mode:
 ; CHECK-NEXT: ieee: false
 ; CHECK-NEXT: dx10-clamp: true
+; CHECK-NEXT: fp32-denormals: false
+; CHECK-NEXT: fp64-fp16-denormals: true
 define void @function_ieee_off() #2 {
   ret void
 }
@@ -127,12 +143,21 @@ define void @function_ieee_off() #2 {
 ; CHECK: mode:
 ; CHECK-NEXT: ieee: false
 ; CHECK-NEXT: dx10-clamp: false
+; CHECK-NEXT: fp32-denormals: false
+; CHECK-NEXT: fp64-fp16-denormals: true
 define void @function_ieee_off_dx10_clamp_off() #3 {
   ret void
 }
 
-attributes #0 = { "no-signed-zeros-fp-math" = "true" }
+; CHECK-LABEL: {{^}}name: high_address_bits
+; CHECK: machineFunctionInfo:
+; CHECK: highBitsOf32BitAddress: 4294934528
+define amdgpu_ps void @high_address_bits() #4 {
+  ret void
+}
 
+attributes #0 = { "no-signed-zeros-fp-math" = "true" }
 attributes #1 = { "amdgpu-dx10-clamp" = "false" }
 attributes #2 = { "amdgpu-ieee" = "false" }
 attributes #3 = { "amdgpu-dx10-clamp" = "false" "amdgpu-ieee" = "false" }
+attributes #4 = { "amdgpu-32bit-address-high-bits"="0xffff8000" }
diff --git a/llvm/test/CodeGen/MIR/Mips/setRegClassOrRegBank.ll b/llvm/test/CodeGen/MIR/Mips/setRegClassOrRegBank.ll
new file mode 100644
index 000000000..71329fd45
--- /dev/null
+++ b/llvm/test/CodeGen/MIR/Mips/setRegClassOrRegBank.ll
@@ -0,0 +1,27 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; RUN: llc -O0 -mtriple=mipsel-linux-gnu -global-isel -mcpu=mips32r5 -mattr=+msa,+fp64,+nan2008 -stop-after=legalizer -verify-machineinstrs %s -o - | FileCheck %s -check-prefixes=P5600
+
+; Check there are no COPY instructions surrounding ADDVI_W instruction.
+; All virtual registers were created with createGenericVirtualRegister
+; which sets RegClassOrRegBank in VRegInfo.
+; Constraining register classes when G_INTRINSIC intrinsic(@llvm.mips.addvi.w)
+; gets selected into ADDVI_W works as expected.
+; Check that setRegClassOrRegBank.mir has same output.
+
+declare <4 x i32> @llvm.mips.addvi.w(<4 x i32>, i32 immarg)
+define void @add_v4i32_builtin_imm(<4 x i32>* %a, <4 x i32>* %c) {
+  ; P5600-LABEL: name: add_v4i32_builtin_imm
+  ; P5600: bb.1.entry:
+  ; P5600:   liveins: $a0, $a1
+  ; P5600:   [[COPY:%[0-9]+]]:_(p0) = COPY $a0
+  ; P5600:   [[COPY1:%[0-9]+]]:_(p0) = COPY $a1
+  ; P5600:   [[LOAD:%[0-9]+]]:msa128w(<4 x s32>) = G_LOAD [[COPY]](p0) :: (load 16 from %ir.a)
+  ; P5600:   [[ADDVI_W:%[0-9]+]]:msa128w(<4 x s32>) = ADDVI_W [[LOAD]](<4 x s32>), 25
+  ; P5600:   G_STORE [[ADDVI_W]](<4 x s32>), [[COPY1]](p0) :: (store 16 into %ir.c)
+  ; P5600:   RetRA
+entry:
+  %0 = load <4 x i32>, <4 x i32>* %a, align 16
+  %1 = tail call <4 x i32> @llvm.mips.addvi.w(<4 x i32> %0, i32 25)
+  store <4 x i32> %1, <4 x i32>* %c, align 16
+  ret void
+}
diff --git a/llvm/test/CodeGen/MSP430/cc_args.ll b/llvm/test/CodeGen/MSP430/cc_args.ll
index 6695a98b2..2bd29d727 100644
--- a/llvm/test/CodeGen/MSP430/cc_args.ll
+++ b/llvm/test/CodeGen/MSP430/cc_args.ll
@@ -215,4 +215,4 @@ define i64 @helper_call_i64() #0 {
   ret i64 %3
 }
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf"="true" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/MSP430/cc_ret.ll b/llvm/test/CodeGen/MSP430/cc_ret.ll
index b4bb05542..e5bd61941 100644
--- a/llvm/test/CodeGen/MSP430/cc_ret.ll
+++ b/llvm/test/CodeGen/MSP430/cc_ret.ll
@@ -58,4 +58,4 @@ define i64 @f_i64() #0 {
   ret i64 72623859790382856
 }
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf"="true" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/MSP430/interrupt.ll b/llvm/test/CodeGen/MSP430/interrupt.ll
index 94fb3bc45..dac3e1432 100644
--- a/llvm/test/CodeGen/MSP430/interrupt.ll
+++ b/llvm/test/CodeGen/MSP430/interrupt.ll
@@ -50,4 +50,13 @@ entry:
   ret void
 }
 
+; Functions without 'interrupt' attribute don't get a vector section.
+; CHECK-NOT: __interrupt_vector
+; CHECK-LABEL: NMI:
+; CHECK: reti
+define msp430_intrcc void @NMI() #1 {
+  ret void
+}
+
 attributes #0 = { noinline nounwind optnone "interrupt"="2" }
+attributes #1 = { noinline nounwind optnone }
diff --git a/llvm/test/CodeGen/MSP430/selectcc.ll b/llvm/test/CodeGen/MSP430/selectcc.ll
new file mode 100644
index 000000000..28b90f013
--- /dev/null
+++ b/llvm/test/CodeGen/MSP430/selectcc.ll
@@ -0,0 +1,46 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=msp430-- < %s | FileCheck %s
+
+define i16 @select_to_shifts_i16(i16 %a, i16 %b) {
+; CHECK-LABEL: select_to_shifts_i16:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    mov r12, r14
+; CHECK-NEXT:    clr r12
+; CHECK-NEXT:    bit #2, r14
+; CHECK-NEXT:    jeq .LBB0_2
+; CHECK-NEXT:  ; %bb.1:
+; CHECK-NEXT:    mov r13, r12
+; CHECK-NEXT:  .LBB0_2:
+; CHECK-NEXT:    ret
+  %and = and i16 %a, 2
+  %tobool = icmp eq i16 %and, 0
+  %select = select i1 %tobool, i16 0, i16 %b
+  ret i16 %select
+}
+
+define i32 @select_to_shifts_i32(i32 %a, i32 %b) {
+; CHECK-LABEL: select_to_shifts_i32:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    mov r12, r11
+; CHECK-NEXT:    and #2, r11
+; CHECK-NEXT:    clr r13
+; CHECK-NEXT:    tst r11
+; CHECK-NEXT:    clr r12
+; CHECK-NEXT:    jne .LBB1_3
+; CHECK-NEXT:  ; %bb.1:
+; CHECK-NEXT:    tst r11
+; CHECK-NEXT:    jne .LBB1_4
+; CHECK-NEXT:  .LBB1_2:
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB1_3:
+; CHECK-NEXT:    mov r14, r12
+; CHECK-NEXT:    tst r11
+; CHECK-NEXT:    jeq .LBB1_2
+; CHECK-NEXT:  .LBB1_4:
+; CHECK-NEXT:    mov r15, r13
+; CHECK-NEXT:    ret
+  %and = and i32 %a, 2
+  %tobool = icmp eq i32 %and, 0
+  %select = select i1 %tobool, i32 0, i32 %b
+  ret i32 %select
+}
diff --git a/llvm/test/CodeGen/MSP430/shift-amount-threshold-b.ll b/llvm/test/CodeGen/MSP430/shift-amount-threshold-b.ll
new file mode 100644
index 000000000..98efb30d8
--- /dev/null
+++ b/llvm/test/CodeGen/MSP430/shift-amount-threshold-b.ll
@@ -0,0 +1,50 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=msp430-- -msp430-no-legal-immediate=true < %s | FileCheck %s
+
+; Test case for the following transformation in TargetLowering::SimplifySetCC
+; (X & -256) == 256 -> (X >> 8) == 1
+define i16 @testSimplifySetCC_2(i16 %x) {
+; CHECK-LABEL: testSimplifySetCC_2:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    and #-64, r12
+; CHECK-NEXT:    cmp #64, r12
+; CHECK-NEXT:    mov r2, r12
+; CHECK-NEXT:    rra r12
+; CHECK-NEXT:    and #1, r12
+; CHECK-NEXT:    ret
+entry:
+  %and = and i16 %x, -64
+  %cmp = icmp eq i16 %and, 64
+  %conv = zext i1 %cmp to i16
+  ret i16 %conv
+}
+
+; Test case for the following transformation in TargetLowering::SimplifySetCC
+; X >  0x0ffffffff -> (X >> 32) >= 1
+define i16 @testSimplifySetCC_3(i16 %x) {
+; CHECK-LABEL: testSimplifySetCC_3:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    cmp #64, r12
+; CHECK-NEXT:    mov r2, r12
+; CHECK-NEXT:    and #1, r12
+; CHECK-NEXT:    ret
+entry:
+  %cmp = icmp ugt i16 %x, 63
+  %conv = zext i1 %cmp to i16
+  ret i16 %conv
+}
+
+; Test case for the following transformation in TargetLowering::SimplifySetCC
+; X <  0x100000000 -> (X >> 32) <  1
+define i16 @testSimplifySetCC_4(i16 %x) {
+; CHECK-LABEL: testSimplifySetCC_4:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    cmp #64, r12
+; CHECK-NEXT:    mov #1, r12
+; CHECK-NEXT:    bic r2, r12
+; CHECK-NEXT:    ret
+entry:
+  %cmp = icmp ult i16 %x, 64
+  %conv = zext i1 %cmp to i16
+  ret i16 %conv
+}
diff --git a/llvm/test/CodeGen/MSP430/shift-amount-threshold.ll b/llvm/test/CodeGen/MSP430/shift-amount-threshold.ll
new file mode 100644
index 000000000..8166c4688
--- /dev/null
+++ b/llvm/test/CodeGen/MSP430/shift-amount-threshold.ll
@@ -0,0 +1,210 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=msp430-- < %s | FileCheck %s
+
+; Check the following conversion in TargetLowering::SimplifySetCC
+; (X & 8) != 0  -->  (X & 8) >> 3
+define i16 @testSimplifySetCC_0(i16 %x) {
+; CHECK-LABEL: testSimplifySetCC_0:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    bit #32, r12
+; CHECK-NEXT:    mov r2, r12
+; CHECK-NEXT:    and #1, r12
+; CHECK-NEXT:    ret
+entry:
+  %and = and i16 %x, 32
+  %cmp = icmp ne i16 %and, 0
+  %conv = zext i1 %cmp to i16
+  ret i16 %conv
+}
+
+; Check the following conversion in TargetLowering::SimplifySetCC
+; (X & 8) == 8  -->  (X & 8) >> 3
+define i16 @testSimplifySetCC_1(i16 %x) {
+; CHECK-LABEL: testSimplifySetCC_1:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    bit #32, r12
+; CHECK-NEXT:    mov r2, r12
+; CHECK-NEXT:    and #1, r12
+; CHECK-NEXT:    ret
+entry:
+  %and = and i16 %x, 32
+  %cmp = icmp eq i16 %and, 32
+  %conv = zext i1 %cmp to i16
+  ret i16 %conv
+}
+
+; Check the following conversion in DAGCombiner::SimplifySelectCC
+; (select_cc seteq (and x, y), 0, 0, A) -> (and (shr (shl x)) A)
+define i16 @testSimplifySelectCC_0(i16 %x, i16 %a) {
+; CHECK-LABEL: testSimplifySelectCC_0:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    mov r12, r14
+; CHECK-NEXT:    clr r12
+; CHECK-NEXT:    bit #2048, r14
+; CHECK-NEXT:    jeq .LBB2_2
+; CHECK-NEXT:  ; %bb.1: ; %entry
+; CHECK-NEXT:    mov r13, r12
+; CHECK-NEXT:  .LBB2_2: ; %entry
+; CHECK-NEXT:    ret
+entry:
+  %and = and i16 %x, 2048
+  %cmp = icmp eq i16 %and, 0
+  %cond = select i1 %cmp, i16 0, i16 %a
+  ret i16 %cond
+}
+
+; Check the following conversion in DAGCombiner foldExtendedSignBitTest
+; sext i1 (setgt iN X, -1) --> sra (not X), (N - 1)
+define i16 @testExtendSignBit_0(i16 %x) {
+; CHECK-LABEL: testExtendSignBit_0:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    mov r12, r13
+; CHECK-NEXT:    mov #-1, r12
+; CHECK-NEXT:    tst r13
+; CHECK-NEXT:    jge .LBB3_2
+; CHECK-NEXT:  ; %bb.1: ; %entry
+; CHECK-NEXT:    clr r12
+; CHECK-NEXT:  .LBB3_2: ; %entry
+; CHECK-NEXT:    ret
+entry:
+  %cmp = icmp sgt i16 %x, -1
+  %cond = sext i1 %cmp to i16
+  ret i16 %cond
+}
+
+; Check the following conversion in DAGCombiner foldExtendedSignBitTest
+; zext i1 (setgt iN X, -1) --> srl (not X), (N - 1)
+define i16 @testExtendSignBit_1(i16 %x) {
+; CHECK-LABEL: testExtendSignBit_1:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    mov r12, r13
+; CHECK-NEXT:    mov #1, r12
+; CHECK-NEXT:    tst r13
+; CHECK-NEXT:    jge .LBB4_2
+; CHECK-NEXT:  ; %bb.1: ; %entry
+; CHECK-NEXT:    clr r12
+; CHECK-NEXT:  .LBB4_2: ; %entry
+; CHECK-NEXT:    ret
+entry:
+  %cmp = icmp sgt i16 %x, -1
+  %cond = zext i1 %cmp to i16
+  ret i16 %cond
+}
+
+; Check the following conversion in DAGCombiner::foldSelectCCToShiftAnd
+; select_cc setlt X, 0, A, 0 -> and (sra X, size(X)-1), A
+define i16 @testShiftAnd_0(i16 %x, i16 %a) {
+; CHECK-LABEL: testShiftAnd_0:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    tst r12
+; CHECK-NEXT:    jl .LBB5_2
+; CHECK-NEXT:  ; %bb.1: ; %entry
+; CHECK-NEXT:    clr r13
+; CHECK-NEXT:  .LBB5_2: ; %entry
+; CHECK-NEXT:    mov r13, r12
+; CHECK-NEXT:    ret
+entry:
+  %cmp = icmp slt i16 %x, 0
+  %cond = select i1 %cmp, i16 %a, i16 0
+  ret i16 %cond
+}
+
+; Check the following conversion in DAGCombiner::foldSelectCCToShiftAnd
+; select_cc setlt X, 0, A, 0 -> "and (srl X, C2), A" iff A is a single-bit
+define i16 @testShiftAnd_1(i16 %x) {
+; CHECK-LABEL: testShiftAnd_1:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    mov r12, r13
+; CHECK-NEXT:    mov #1, r12
+; CHECK-NEXT:    tst r13
+; CHECK-NEXT:    jl .LBB6_2
+; CHECK-NEXT:  ; %bb.1: ; %entry
+; CHECK-NEXT:    clr r12
+; CHECK-NEXT:  .LBB6_2: ; %entry
+; CHECK-NEXT:    add r12, r12
+; CHECK-NEXT:    ret
+entry:
+  %cmp = icmp slt i16 %x, 0
+  %cond = select i1 %cmp, i16 2, i16 0
+  ret i16 %cond
+}
+
+; Check the following conversion in DAGCombiner::SimplifySelectCC
+; select C, 16, 0 -> shl C, 4
+define i16 @testSimplifySelectCC_1(i16 %a, i16 %b) {
+; CHECK-LABEL: testSimplifySelectCC_1:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    mov r12, r14
+; CHECK-NEXT:    mov #32, r12
+; CHECK-NEXT:    cmp r14, r13
+; CHECK-NEXT:    jl .LBB7_2
+; CHECK-NEXT:  ; %bb.1: ; %entry
+; CHECK-NEXT:    clr r12
+; CHECK-NEXT:  .LBB7_2: ; %entry
+; CHECK-NEXT:    ret
+entry:
+  %cmp = icmp sgt i16 %a, %b
+  %cond = select i1 %cmp, i16 32, i16 0
+  ret i16 %cond
+}
+
+; Check the following conversion in TargetLowering::SimplifySetCC
+; (X & 8) != 0  -->  (X & 8) >> 3
+define i16 @testSimplifySetCC_0_sh8(i16 %x) {
+; CHECK-LABEL: testSimplifySetCC_0_sh8:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    and #256, r12
+; CHECK-NEXT:    swpb r12
+; CHECK-NEXT:    ret
+entry:
+  %and = and i16 %x, 256
+  %cmp = icmp ne i16 %and, 0
+  %conv = zext i1 %cmp to i16
+  ret i16 %conv
+}
+
+; Check the following conversion in TargetLowering::SimplifySetCC
+; (X & 8) == 8  -->  (X & 8) >> 3
+define i16 @testSimplifySetCC_1_sh8(i16 %x) {
+; CHECK-LABEL: testSimplifySetCC_1_sh8:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    and #256, r12
+; CHECK-NEXT:    swpb r12
+; CHECK-NEXT:    ret
+entry:
+  %and = and i16 %x, 256
+  %cmp = icmp eq i16 %and, 256
+  %conv = zext i1 %cmp to i16
+  ret i16 %conv
+}
+
+; Check the following conversion in DAGCombiner::foldSelectCCToShiftAnd
+; select_cc setlt X, 0, A, 0 -> "and (srl X, C2), A" iff A is a single-bit
+define i16 @testShiftAnd_1_sh8(i16 %x) {
+; CHECK-LABEL: testShiftAnd_1_sh8:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    swpb r12
+; CHECK-NEXT:    and #128, r12
+; CHECK-NEXT:    ret
+entry:
+  %cmp = icmp slt i16 %x, 0
+  %cond = select i1 %cmp, i16 128, i16 0
+  ret i16 %cond
+}
+
+; Check the following conversion in DAGCombiner::foldSelectCCToShiftAnd
+; select_cc setlt X, 0, A, 0 -> "and (srl X, C2), A" iff A is a single-bit
+define i16 @testShiftAnd_1_sh9(i16 %x) {
+; CHECK-LABEL: testShiftAnd_1_sh9:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    swpb r12
+; CHECK-NEXT:    mov.b r12, r12
+; CHECK-NEXT:    clrc
+; CHECK-NEXT:    rrc r12
+; CHECK-NEXT:    and #64, r12
+; CHECK-NEXT:    ret
+entry:
+  %cmp = icmp slt i16 %x, 0
+  %cond = select i1 %cmp, i16 64, i16 0
+  ret i16 %cond
+}
diff --git a/llvm/test/CodeGen/MSP430/transient-stack-alignment.ll b/llvm/test/CodeGen/MSP430/transient-stack-alignment.ll
index a2ddf8a0b..8b26ea9b5 100644
--- a/llvm/test/CodeGen/MSP430/transient-stack-alignment.ll
+++ b/llvm/test/CodeGen/MSP430/transient-stack-alignment.ll
@@ -14,4 +14,4 @@ define void @test() #0 {
   ret void
 }
 
-attributes #0 = { nounwind "no-frame-pointer-elim"="false" }
+attributes #0 = { nounwind "frame-pointer"="none" }
diff --git a/llvm/test/CodeGen/Mips/Fast-ISel/br1.ll b/llvm/test/CodeGen/Mips/Fast-ISel/br1.ll
index 2f2a33042..ceafb3229 100644
--- a/llvm/test/CodeGen/Mips/Fast-ISel/br1.ll
+++ b/llvm/test/CodeGen/Mips/Fast-ISel/br1.ll
@@ -31,4 +31,4 @@ if.end:                                           ; preds = %entry, %if.then
 
 }
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/Mips/Fast-ISel/loadstore2.ll b/llvm/test/CodeGen/Mips/Fast-ISel/loadstore2.ll
index dc7be744d..c059dbf41 100644
--- a/llvm/test/CodeGen/Mips/Fast-ISel/loadstore2.ll
+++ b/llvm/test/CodeGen/Mips/Fast-ISel/loadstore2.ll
@@ -80,6 +80,6 @@ entry:
   ret void
 }
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 
diff --git a/llvm/test/CodeGen/Mips/Fast-ISel/loadstrconst.ll b/llvm/test/CodeGen/Mips/Fast-ISel/loadstrconst.ll
index 78077f764..072d15f43 100644
--- a/llvm/test/CodeGen/Mips/Fast-ISel/loadstrconst.ll
+++ b/llvm/test/CodeGen/Mips/Fast-ISel/loadstrconst.ll
@@ -17,5 +17,5 @@ entry:
 
 }
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
diff --git a/llvm/test/CodeGen/Mips/Fast-ISel/logopm.ll b/llvm/test/CodeGen/Mips/Fast-ISel/logopm.ll
index ef6b5182a..bd35eeeda 100644
--- a/llvm/test/CodeGen/Mips/Fast-ISel/logopm.ll
+++ b/llvm/test/CodeGen/Mips/Fast-ISel/logopm.ll
@@ -590,8 +590,8 @@ entry:
   ret void
 }
 
-attributes #0 = { noinline nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noinline nounwind "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind }
 
 !llvm.module.flags = !{!0}
diff --git a/llvm/test/CodeGen/Mips/Fast-ISel/simplestorefp1.ll b/llvm/test/CodeGen/Mips/Fast-ISel/simplestorefp1.ll
index 4ee60bf5e..d1df4569c 100644
--- a/llvm/test/CodeGen/Mips/Fast-ISel/simplestorefp1.ll
+++ b/llvm/test/CodeGen/Mips/Fast-ISel/simplestorefp1.ll
@@ -51,4 +51,4 @@ entry:
   ret void
 }
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/Mips/Fast-ISel/simplestorei.ll b/llvm/test/CodeGen/Mips/Fast-ISel/simplestorei.ll
index 1f400b4b8..7fc0841e3 100644
--- a/llvm/test/CodeGen/Mips/Fast-ISel/simplestorei.ll
+++ b/llvm/test/CodeGen/Mips/Fast-ISel/simplestorei.ll
@@ -63,6 +63,6 @@ entry:
   ret void
 }
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/irtranslator/aggregate_struct_return.ll b/llvm/test/CodeGen/Mips/GlobalISel/irtranslator/aggregate_struct_return.ll
new file mode 100644
index 000000000..f6b034ea7
--- /dev/null
+++ b/llvm/test/CodeGen/Mips/GlobalISel/irtranslator/aggregate_struct_return.ll
@@ -0,0 +1,132 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; RUN: llc -O0 -mtriple=mipsel-linux-gnu -global-isel -stop-after=irtranslator -verify-machineinstrs %s -o - | FileCheck %s -check-prefixes=MIPS32
+
+define { float, float } @add_complex_float({ float, float }* %a, { float, float }* %b) {
+  ; MIPS32-LABEL: name: add_complex_float
+  ; MIPS32: bb.1.entry:
+  ; MIPS32:   liveins: $a0, $a1
+  ; MIPS32:   [[COPY:%[0-9]+]]:_(p0) = COPY $a0
+  ; MIPS32:   [[COPY1:%[0-9]+]]:_(p0) = COPY $a1
+  ; MIPS32:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+  ; MIPS32:   [[COPY2:%[0-9]+]]:_(p0) = COPY [[COPY]](p0)
+  ; MIPS32:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY2]](p0) :: (load 4 from %ir..realp)
+  ; MIPS32:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+  ; MIPS32:   [[GEP:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s32)
+  ; MIPS32:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[GEP]](p0) :: (load 4 from %ir..imagp)
+  ; MIPS32:   [[COPY3:%[0-9]+]]:_(p0) = COPY [[COPY1]](p0)
+  ; MIPS32:   [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[COPY3]](p0) :: (load 4 from %ir..realp1)
+  ; MIPS32:   [[GEP1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY1]], [[C]](s32)
+  ; MIPS32:   [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[GEP1]](p0) :: (load 4 from %ir..imagp3)
+  ; MIPS32:   [[FADD:%[0-9]+]]:_(s32) = G_FADD [[LOAD]], [[LOAD2]]
+  ; MIPS32:   [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[LOAD1]], [[LOAD3]]
+  ; MIPS32:   $f0 = COPY [[FADD]](s32)
+  ; MIPS32:   $f2 = COPY [[FADD1]](s32)
+  ; MIPS32:   RetRA implicit $f0, implicit $f2
+entry:
+  %.realp = getelementptr inbounds { float, float }, { float, float }* %a, i32 0, i32 0
+  %.real = load float, float* %.realp, align 4
+  %.imagp = getelementptr inbounds { float, float }, { float, float }* %a, i32 0, i32 1
+  %.imag = load float, float* %.imagp, align 4
+  %.realp1 = getelementptr inbounds { float, float }, { float, float }* %b, i32 0, i32 0
+  %.real2 = load float, float* %.realp1, align 4
+  %.imagp3 = getelementptr inbounds { float, float }, { float, float }* %b, i32 0, i32 1
+  %.imag4 = load float, float* %.imagp3, align 4
+  %add.r = fadd float %.real, %.real2
+  %add.i = fadd float %.imag, %.imag4
+  %.fca.0.insert = insertvalue { float, float } undef, float %add.r, 0
+  %.fca.1.insert = insertvalue { float, float } %.fca.0.insert, float %add.i, 1
+  ret { float, float } %.fca.1.insert
+}
+
+define { double, double } @add_complex_double({ double, double }* %a, { double, double }* %b) {
+  ; MIPS32-LABEL: name: add_complex_double
+  ; MIPS32: bb.1.entry:
+  ; MIPS32:   liveins: $a0, $a1
+  ; MIPS32:   [[COPY:%[0-9]+]]:_(p0) = COPY $a0
+  ; MIPS32:   [[COPY1:%[0-9]+]]:_(p0) = COPY $a1
+  ; MIPS32:   [[DEF:%[0-9]+]]:_(s64) = G_IMPLICIT_DEF
+  ; MIPS32:   [[COPY2:%[0-9]+]]:_(p0) = COPY [[COPY]](p0)
+  ; MIPS32:   [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[COPY2]](p0) :: (load 8 from %ir..realp)
+  ; MIPS32:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+  ; MIPS32:   [[GEP:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s32)
+  ; MIPS32:   [[LOAD1:%[0-9]+]]:_(s64) = G_LOAD [[GEP]](p0) :: (load 8 from %ir..imagp)
+  ; MIPS32:   [[COPY3:%[0-9]+]]:_(p0) = COPY [[COPY1]](p0)
+  ; MIPS32:   [[LOAD2:%[0-9]+]]:_(s64) = G_LOAD [[COPY3]](p0) :: (load 8 from %ir..realp1)
+  ; MIPS32:   [[GEP1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY1]], [[C]](s32)
+  ; MIPS32:   [[LOAD3:%[0-9]+]]:_(s64) = G_LOAD [[GEP1]](p0) :: (load 8 from %ir..imagp3)
+  ; MIPS32:   [[FADD:%[0-9]+]]:_(s64) = G_FADD [[LOAD]], [[LOAD2]]
+  ; MIPS32:   [[FADD1:%[0-9]+]]:_(s64) = G_FADD [[LOAD1]], [[LOAD3]]
+  ; MIPS32:   $d0 = COPY [[FADD]](s64)
+  ; MIPS32:   $d1 = COPY [[FADD1]](s64)
+  ; MIPS32:   RetRA implicit $d0, implicit $d1
+entry:
+  %.realp = getelementptr inbounds { double, double }, { double, double }* %a, i32 0, i32 0
+  %.real = load double, double* %.realp, align 8
+  %.imagp = getelementptr inbounds { double, double }, { double, double }* %a, i32 0, i32 1
+  %.imag = load double, double* %.imagp, align 8
+  %.realp1 = getelementptr inbounds { double, double }, { double, double }* %b, i32 0, i32 0
+  %.real2 = load double, double* %.realp1, align 8
+  %.imagp3 = getelementptr inbounds { double, double }, { double, double }* %b, i32 0, i32 1
+  %.imag4 = load double, double* %.imagp3, align 8
+  %add.r = fadd double %.real, %.real2
+  %add.i = fadd double %.imag, %.imag4
+  %.fca.0.insert = insertvalue { double, double } undef, double %add.r, 0
+  %.fca.1.insert = insertvalue { double, double } %.fca.0.insert, double %add.i, 1
+  ret { double, double } %.fca.1.insert
+}
+
+declare { float, float } @ret_complex_float()
+define void @call_ret_complex_float({ float, float }* %z) {
+  ; MIPS32-LABEL: name: call_ret_complex_float
+  ; MIPS32: bb.1.entry:
+  ; MIPS32:   liveins: $a0
+  ; MIPS32:   [[COPY:%[0-9]+]]:_(p0) = COPY $a0
+  ; MIPS32:   ADJCALLSTACKDOWN 16, 0, implicit-def $sp, implicit $sp
+  ; MIPS32:   JAL @ret_complex_float, csr_o32, implicit-def $ra, implicit-def $sp, implicit-def $f0, implicit-def $f2
+  ; MIPS32:   [[COPY1:%[0-9]+]]:_(s32) = COPY $f0
+  ; MIPS32:   [[COPY2:%[0-9]+]]:_(s32) = COPY $f2
+  ; MIPS32:   ADJCALLSTACKUP 16, 0, implicit-def $sp, implicit $sp
+  ; MIPS32:   [[COPY3:%[0-9]+]]:_(p0) = COPY [[COPY]](p0)
+  ; MIPS32:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+  ; MIPS32:   [[GEP:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s32)
+  ; MIPS32:   G_STORE [[COPY1]](s32), [[COPY3]](p0) :: (store 4 into %ir..realp)
+  ; MIPS32:   G_STORE [[COPY2]](s32), [[GEP]](p0) :: (store 4 into %ir..imagp)
+  ; MIPS32:   RetRA
+entry:
+  %call = call { float, float } @ret_complex_float()
+  %0 = extractvalue { float, float } %call, 0
+  %1 = extractvalue { float, float } %call, 1
+  %.realp = getelementptr inbounds { float, float }, { float, float }* %z, i32 0, i32 0
+  %.imagp = getelementptr inbounds { float, float }, { float, float }* %z, i32 0, i32 1
+  store float %0, float* %.realp, align 4
+  store float %1, float* %.imagp, align 4
+  ret void
+}
+
+declare { double, double } @ret_complex_double()
+define void @call_ret_complex_double({ double, double }* %z) {
+  ; MIPS32-LABEL: name: call_ret_complex_double
+  ; MIPS32: bb.1.entry:
+  ; MIPS32:   liveins: $a0
+  ; MIPS32:   [[COPY:%[0-9]+]]:_(p0) = COPY $a0
+  ; MIPS32:   ADJCALLSTACKDOWN 16, 0, implicit-def $sp, implicit $sp
+  ; MIPS32:   JAL @ret_complex_double, csr_o32, implicit-def $ra, implicit-def $sp, implicit-def $d0, implicit-def $d1
+  ; MIPS32:   [[COPY1:%[0-9]+]]:_(s64) = COPY $d0
+  ; MIPS32:   [[COPY2:%[0-9]+]]:_(s64) = COPY $d1
+  ; MIPS32:   ADJCALLSTACKUP 16, 0, implicit-def $sp, implicit $sp
+  ; MIPS32:   [[COPY3:%[0-9]+]]:_(p0) = COPY [[COPY]](p0)
+  ; MIPS32:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+  ; MIPS32:   [[GEP:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s32)
+  ; MIPS32:   G_STORE [[COPY1]](s64), [[COPY3]](p0) :: (store 8 into %ir..realp)
+  ; MIPS32:   G_STORE [[COPY2]](s64), [[GEP]](p0) :: (store 8 into %ir..imagp)
+  ; MIPS32:   RetRA
+entry:
+  %call = call { double, double } @ret_complex_double()
+  %0 = extractvalue { double, double } %call, 0
+  %1 = extractvalue { double, double } %call, 1
+  %.realp = getelementptr inbounds { double, double }, { double, double }* %z, i32 0, i32 0
+  %.imagp = getelementptr inbounds { double, double }, { double, double }* %z, i32 0, i32 1
+  store double %0, double* %.realp, align 8
+  store double %1, double* %.imagp, align 8
+  ret void
+}
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/irtranslator/call.ll b/llvm/test/CodeGen/Mips/GlobalISel/irtranslator/call.ll
index c76750208..4cc956c20 100644
--- a/llvm/test/CodeGen/Mips/GlobalISel/irtranslator/call.ll
+++ b/llvm/test/CodeGen/Mips/GlobalISel/irtranslator/call.ll
@@ -153,12 +153,7 @@ define void @call_symbol(i8* nocapture readonly %src, i8* nocapture %dest, i32 s
   ; MIPS32:   [[COPY:%[0-9]+]]:_(p0) = COPY $a0
   ; MIPS32:   [[COPY1:%[0-9]+]]:_(p0) = COPY $a1
   ; MIPS32:   [[COPY2:%[0-9]+]]:_(s32) = COPY $a2
-  ; MIPS32:   ADJCALLSTACKDOWN 16, 0, implicit-def $sp, implicit $sp
-  ; MIPS32:   $a0 = COPY [[COPY1]](p0)
-  ; MIPS32:   $a1 = COPY [[COPY]](p0)
-  ; MIPS32:   $a2 = COPY [[COPY2]](s32)
-  ; MIPS32:   JAL &memcpy, csr_o32, implicit-def $ra, implicit-def $sp, implicit $a0, implicit $a1, implicit $a2
-  ; MIPS32:   ADJCALLSTACKUP 16, 0, implicit-def $sp, implicit $sp
+  ; MIPS32:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[COPY1]](p0), [[COPY]](p0), [[COPY2]](s32), 0 :: (store 1 into %ir.dest), (load 1 from %ir.src)
   ; MIPS32:   RetRA
   ; MIPS32_PIC-LABEL: name: call_symbol
   ; MIPS32_PIC: bb.1.entry:
@@ -166,14 +161,33 @@ define void @call_symbol(i8* nocapture readonly %src, i8* nocapture %dest, i32 s
   ; MIPS32_PIC:   [[COPY:%[0-9]+]]:_(p0) = COPY $a0
   ; MIPS32_PIC:   [[COPY1:%[0-9]+]]:_(p0) = COPY $a1
   ; MIPS32_PIC:   [[COPY2:%[0-9]+]]:_(s32) = COPY $a2
+  ; MIPS32_PIC:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[COPY1]](p0), [[COPY]](p0), [[COPY2]](s32), 0 :: (store 1 into %ir.dest), (load 1 from %ir.src)
+  ; MIPS32_PIC:   RetRA
+entry:
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 %dest, i8* align 1 %src, i32 %length, i1 false)
+  ret void
+}
+
+declare void @f_with_void_ret();
+
+define void @call_f_with_void_ret() {
+  ; MIPS32-LABEL: name: call_f_with_void_ret
+  ; MIPS32: bb.1.entry:
+  ; MIPS32:   ADJCALLSTACKDOWN 16, 0, implicit-def $sp, implicit $sp
+  ; MIPS32:   JAL @f_with_void_ret, csr_o32, implicit-def $ra, implicit-def $sp
+  ; MIPS32:   ADJCALLSTACKUP 16, 0, implicit-def $sp, implicit $sp
+  ; MIPS32:   RetRA
+  ; MIPS32_PIC-LABEL: name: call_f_with_void_ret
+  ; MIPS32_PIC: bb.1.entry:
+  ; MIPS32_PIC:   liveins: $t9, $v0
+  ; MIPS32_PIC:   [[ADDu:%[0-9]+]]:gpr32 = ADDu $v0, $t9
   ; MIPS32_PIC:   ADJCALLSTACKDOWN 16, 0, implicit-def $sp, implicit $sp
-  ; MIPS32_PIC:   $a0 = COPY [[COPY1]](p0)
-  ; MIPS32_PIC:   $a1 = COPY [[COPY]](p0)
-  ; MIPS32_PIC:   $a2 = COPY [[COPY2]](s32)
-  ; MIPS32_PIC:   JAL &memcpy, csr_o32, implicit-def $ra, implicit-def $sp, implicit $a0, implicit $a1, implicit $a2
+  ; MIPS32_PIC:   [[GV:%[0-9]+]]:gpr32(p0) = G_GLOBAL_VALUE target-flags(mips-got-call) @f_with_void_ret
+  ; MIPS32_PIC:   $gp = COPY [[ADDu]]
+  ; MIPS32_PIC:   JALRPseudo [[GV]](p0), csr_o32, implicit-def $ra, implicit-def $sp, implicit-def $gp
   ; MIPS32_PIC:   ADJCALLSTACKUP 16, 0, implicit-def $sp, implicit $sp
   ; MIPS32_PIC:   RetRA
 entry:
-  call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 %dest, i8* align 1 %src, i32 %length, i1 false)
+  call void @f_with_void_ret()
   ret void
 }
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/irtranslator/extend_args.ll b/llvm/test/CodeGen/Mips/GlobalISel/irtranslator/extend_args.ll
index a1743c4c7..2d0a26eff 100644
--- a/llvm/test/CodeGen/Mips/GlobalISel/irtranslator/extend_args.ll
+++ b/llvm/test/CodeGen/Mips/GlobalISel/irtranslator/extend_args.ll
@@ -63,7 +63,7 @@ define signext i8 @call_sext_stack_arg_i8(i32 %x1, i32 %x2, i32 %x3, i32 %x4, i8
   ; MIPS32:   $a3 = COPY [[COPY3]](s32)
   ; MIPS32:   [[COPY4:%[0-9]+]]:_(p0) = COPY $sp
   ; MIPS32:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-  ; MIPS32:   [[GEP:%[0-9]+]]:_(p0) = G_GEP [[COPY4]], [[C]](s32)
+  ; MIPS32:   [[GEP:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY4]], [[C]](s32)
   ; MIPS32:   [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[TRUNC]](s8)
   ; MIPS32:   G_STORE [[SEXT]](s32), [[GEP]](p0) :: (store 4 into stack + 16, align 8)
   ; MIPS32:   JAL @sext_stack_arg_i8, csr_o32, implicit-def $ra, implicit-def $sp, implicit $a0, implicit $a1, implicit $a2, implicit $a3, implicit-def $v0
@@ -96,7 +96,7 @@ define zeroext i8 @call_zext_stack_arg_i8(i32 %x1, i32 %x2, i32 %x3, i32 %x4, i8
   ; MIPS32:   $a3 = COPY [[COPY3]](s32)
   ; MIPS32:   [[COPY4:%[0-9]+]]:_(p0) = COPY $sp
   ; MIPS32:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-  ; MIPS32:   [[GEP:%[0-9]+]]:_(p0) = G_GEP [[COPY4]], [[C]](s32)
+  ; MIPS32:   [[GEP:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY4]], [[C]](s32)
   ; MIPS32:   [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[TRUNC]](s8)
   ; MIPS32:   G_STORE [[ZEXT]](s32), [[GEP]](p0) :: (store 4 into stack + 16, align 8)
   ; MIPS32:   JAL @zext_stack_arg_i8, csr_o32, implicit-def $ra, implicit-def $sp, implicit $a0, implicit $a1, implicit $a2, implicit $a3, implicit-def $v0
@@ -129,7 +129,7 @@ define i8 @call_aext_stack_arg_i8(i32 %x1, i32 %x2, i32 %x3, i32 %x4, i8 %a) {
   ; MIPS32:   $a3 = COPY [[COPY3]](s32)
   ; MIPS32:   [[COPY4:%[0-9]+]]:_(p0) = COPY $sp
   ; MIPS32:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-  ; MIPS32:   [[GEP:%[0-9]+]]:_(p0) = G_GEP [[COPY4]], [[C]](s32)
+  ; MIPS32:   [[GEP:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY4]], [[C]](s32)
   ; MIPS32:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC]](s8)
   ; MIPS32:   G_STORE [[ANYEXT]](s32), [[GEP]](p0) :: (store 4 into stack + 16, align 8)
   ; MIPS32:   JAL @aext_stack_arg_i8, csr_o32, implicit-def $ra, implicit-def $sp, implicit $a0, implicit $a1, implicit $a2, implicit $a3, implicit-def $v0
@@ -206,7 +206,7 @@ define signext i16 @call_sext_stack_arg_i16(i32 %x1, i32 %x2, i32 %x3, i32 %x4,
   ; MIPS32:   $a3 = COPY [[COPY3]](s32)
   ; MIPS32:   [[COPY4:%[0-9]+]]:_(p0) = COPY $sp
   ; MIPS32:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-  ; MIPS32:   [[GEP:%[0-9]+]]:_(p0) = G_GEP [[COPY4]], [[C]](s32)
+  ; MIPS32:   [[GEP:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY4]], [[C]](s32)
   ; MIPS32:   [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[TRUNC]](s16)
   ; MIPS32:   G_STORE [[SEXT]](s32), [[GEP]](p0) :: (store 4 into stack + 16, align 8)
   ; MIPS32:   JAL @sext_stack_arg_i16, csr_o32, implicit-def $ra, implicit-def $sp, implicit $a0, implicit $a1, implicit $a2, implicit $a3, implicit-def $v0
@@ -239,7 +239,7 @@ define zeroext i16 @call_zext_stack_arg_i16(i32 %x1, i32 %x2, i32 %x3, i32 %x4,
   ; MIPS32:   $a3 = COPY [[COPY3]](s32)
   ; MIPS32:   [[COPY4:%[0-9]+]]:_(p0) = COPY $sp
   ; MIPS32:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-  ; MIPS32:   [[GEP:%[0-9]+]]:_(p0) = G_GEP [[COPY4]], [[C]](s32)
+  ; MIPS32:   [[GEP:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY4]], [[C]](s32)
   ; MIPS32:   [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[TRUNC]](s16)
   ; MIPS32:   G_STORE [[ZEXT]](s32), [[GEP]](p0) :: (store 4 into stack + 16, align 8)
   ; MIPS32:   JAL @zext_stack_arg_i16, csr_o32, implicit-def $ra, implicit-def $sp, implicit $a0, implicit $a1, implicit $a2, implicit $a3, implicit-def $v0
@@ -272,7 +272,7 @@ define i16 @call_aext_stack_arg_i16(i32 %x1, i32 %x2, i32 %x3, i32 %x4, i16 %a)
   ; MIPS32:   $a3 = COPY [[COPY3]](s32)
   ; MIPS32:   [[COPY4:%[0-9]+]]:_(p0) = COPY $sp
   ; MIPS32:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-  ; MIPS32:   [[GEP:%[0-9]+]]:_(p0) = G_GEP [[COPY4]], [[C]](s32)
+  ; MIPS32:   [[GEP:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY4]], [[C]](s32)
   ; MIPS32:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC]](s16)
   ; MIPS32:   G_STORE [[ANYEXT]](s32), [[GEP]](p0) :: (store 4 into stack + 16, align 8)
   ; MIPS32:   JAL @aext_stack_arg_i16, csr_o32, implicit-def $ra, implicit-def $sp, implicit $a0, implicit $a1, implicit $a2, implicit $a3, implicit-def $v0
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/irtranslator/sret_pointer.ll b/llvm/test/CodeGen/Mips/GlobalISel/irtranslator/sret_pointer.ll
new file mode 100644
index 000000000..7d37239c4
--- /dev/null
+++ b/llvm/test/CodeGen/Mips/GlobalISel/irtranslator/sret_pointer.ll
@@ -0,0 +1,39 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; RUN: llc -O0 -mtriple=mipsel-linux-gnu -global-isel -stop-after=irtranslator -verify-machineinstrs %s -o - | FileCheck %s -check-prefixes=MIPS32
+
+%struct.S = type { i32, i32 }
+
+define void @ZeroInit(%struct.S* noalias sret %agg.result) {
+  ; MIPS32-LABEL: name: ZeroInit
+  ; MIPS32: bb.1.entry:
+  ; MIPS32:   liveins: $a0
+  ; MIPS32:   [[COPY:%[0-9]+]]:_(p0) = COPY $a0
+  ; MIPS32:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; MIPS32:   [[COPY1:%[0-9]+]]:_(p0) = COPY [[COPY]](p0)
+  ; MIPS32:   G_STORE [[C]](s32), [[COPY1]](p0) :: (store 4 into %ir.x)
+  ; MIPS32:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+  ; MIPS32:   [[GEP:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C1]](s32)
+  ; MIPS32:   G_STORE [[C]](s32), [[GEP]](p0) :: (store 4 into %ir.y)
+  ; MIPS32:   RetRA
+entry:
+  %x = getelementptr inbounds %struct.S, %struct.S* %agg.result, i32 0, i32 0
+  store i32 0, i32* %x, align 4
+  %y = getelementptr inbounds %struct.S, %struct.S* %agg.result, i32 0, i32 1
+  store i32 0, i32* %y, align 4
+  ret void
+}
+
+define void @CallZeroInit(%struct.S* noalias sret %agg.result) {
+  ; MIPS32-LABEL: name: CallZeroInit
+  ; MIPS32: bb.1.entry:
+  ; MIPS32:   liveins: $a0
+  ; MIPS32:   [[COPY:%[0-9]+]]:_(p0) = COPY $a0
+  ; MIPS32:   ADJCALLSTACKDOWN 16, 0, implicit-def $sp, implicit $sp
+  ; MIPS32:   $a0 = COPY [[COPY]](p0)
+  ; MIPS32:   JAL @ZeroInit, csr_o32, implicit-def $ra, implicit-def $sp, implicit $a0
+  ; MIPS32:   ADJCALLSTACKUP 16, 0, implicit-def $sp, implicit $sp
+  ; MIPS32:   RetRA
+entry:
+  call void @ZeroInit(%struct.S* sret %agg.result)
+  ret void
+}
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/irtranslator/stack_args.ll b/llvm/test/CodeGen/Mips/GlobalISel/irtranslator/stack_args.ll
index 37d75bc7f..e33991c84 100644
--- a/llvm/test/CodeGen/Mips/GlobalISel/irtranslator/stack_args.ll
+++ b/llvm/test/CodeGen/Mips/GlobalISel/irtranslator/stack_args.ll
@@ -20,7 +20,7 @@ define  i32 @g(i32  %x1, i32 %x2, i32 %x3, i32 %x4, i32 %x5){
   ; MIPS32:   $a3 = COPY [[COPY3]](s32)
   ; MIPS32:   [[COPY4:%[0-9]+]]:_(p0) = COPY $sp
   ; MIPS32:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-  ; MIPS32:   [[GEP:%[0-9]+]]:_(p0) = G_GEP [[COPY4]], [[C]](s32)
+  ; MIPS32:   [[GEP:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY4]], [[C]](s32)
   ; MIPS32:   G_STORE [[LOAD]](s32), [[GEP]](p0) :: (store 4 into stack + 16, align 8)
   ; MIPS32:   JAL @f, csr_o32, implicit-def $ra, implicit-def $sp, implicit $a0, implicit $a1, implicit $a2, implicit $a3, implicit-def $v0
   ; MIPS32:   [[COPY5:%[0-9]+]]:_(s32) = COPY $v0
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/irtranslator/var_arg.ll b/llvm/test/CodeGen/Mips/GlobalISel/irtranslator/var_arg.ll
new file mode 100644
index 000000000..8938c8128
--- /dev/null
+++ b/llvm/test/CodeGen/Mips/GlobalISel/irtranslator/var_arg.ll
@@ -0,0 +1,66 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; RUN: llc -O0 -mtriple=mipsel-linux-gnu -global-isel -stop-after=irtranslator -verify-machineinstrs %s -o - | FileCheck %s -check-prefixes=MIPS32
+
+@.str = private unnamed_addr constant [11 x i8] c"string %s\0A\00", align 1
+declare void @llvm.va_start(i8*)
+declare void @llvm.va_copy(i8*, i8*)
+declare i32 @printf(i8*, ...)
+
+define void @testVaCopyArg(i8* %fmt, ...) {
+  ; MIPS32-LABEL: name: testVaCopyArg
+  ; MIPS32: bb.1.entry:
+  ; MIPS32:   liveins: $a0, $a1, $a2, $a3
+  ; MIPS32:   [[COPY:%[0-9]+]]:_(p0) = COPY $a0
+  ; MIPS32:   [[COPY1:%[0-9]+]]:_(s32) = COPY $a1
+  ; MIPS32:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.2
+  ; MIPS32:   G_STORE [[COPY1]](s32), [[FRAME_INDEX]](p0) :: (store 4 into %fixed-stack.2)
+  ; MIPS32:   [[COPY2:%[0-9]+]]:_(s32) = COPY $a2
+  ; MIPS32:   [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1
+  ; MIPS32:   G_STORE [[COPY2]](s32), [[FRAME_INDEX1]](p0) :: (store 4 into %fixed-stack.1)
+  ; MIPS32:   [[COPY3:%[0-9]+]]:_(s32) = COPY $a3
+  ; MIPS32:   [[FRAME_INDEX2:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0
+  ; MIPS32:   G_STORE [[COPY3]](s32), [[FRAME_INDEX2]](p0) :: (store 4 into %fixed-stack.0)
+  ; MIPS32:   [[GV:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @.str
+  ; MIPS32:   [[COPY4:%[0-9]+]]:_(p0) = COPY [[GV]](p0)
+  ; MIPS32:   [[FRAME_INDEX3:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.fmt.addr
+  ; MIPS32:   [[FRAME_INDEX4:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.1.ap
+  ; MIPS32:   [[FRAME_INDEX5:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.2.aq
+  ; MIPS32:   [[FRAME_INDEX6:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.3.s
+  ; MIPS32:   G_STORE [[COPY]](p0), [[FRAME_INDEX3]](p0) :: (store 4 into %ir.fmt.addr)
+  ; MIPS32:   G_VASTART [[FRAME_INDEX4]](p0) :: (store 4 into %ir.ap1, align 1)
+  ; MIPS32:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.va_copy), [[FRAME_INDEX5]](p0), [[FRAME_INDEX4]](p0)
+  ; MIPS32:   [[LOAD:%[0-9]+]]:_(p0) = G_LOAD [[FRAME_INDEX5]](p0) :: (load 4 from %ir.aq)
+  ; MIPS32:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+  ; MIPS32:   [[GEP:%[0-9]+]]:_(p0) = G_PTR_ADD [[LOAD]], [[C]](s32)
+  ; MIPS32:   G_STORE [[GEP]](p0), [[FRAME_INDEX5]](p0) :: (store 4 into %ir.aq)
+  ; MIPS32:   [[LOAD1:%[0-9]+]]:_(p0) = G_LOAD [[LOAD]](p0) :: (load 4 from %ir.2)
+  ; MIPS32:   G_STORE [[LOAD1]](p0), [[FRAME_INDEX6]](p0) :: (store 4 into %ir.s)
+  ; MIPS32:   [[LOAD2:%[0-9]+]]:_(p0) = G_LOAD [[FRAME_INDEX6]](p0) :: (load 4 from %ir.s)
+  ; MIPS32:   ADJCALLSTACKDOWN 16, 0, implicit-def $sp, implicit $sp
+  ; MIPS32:   $a0 = COPY [[COPY4]](p0)
+  ; MIPS32:   $a1 = COPY [[LOAD2]](p0)
+  ; MIPS32:   JAL @printf, csr_o32, implicit-def $ra, implicit-def $sp, implicit $a0, implicit $a1, implicit-def $v0
+  ; MIPS32:   [[COPY5:%[0-9]+]]:_(s32) = COPY $v0
+  ; MIPS32:   ADJCALLSTACKUP 16, 0, implicit-def $sp, implicit $sp
+  ; MIPS32:   RetRA
+entry:
+  %fmt.addr = alloca i8*, align 4
+  %ap = alloca i8*, align 4
+  %aq = alloca i8*, align 4
+  %s = alloca i8*, align 4
+  store i8* %fmt, i8** %fmt.addr, align 4
+  %ap1 = bitcast i8** %ap to i8*
+  call void @llvm.va_start(i8* %ap1)
+  %0 = bitcast i8** %aq to i8*
+  %1 = bitcast i8** %ap to i8*
+  call void @llvm.va_copy(i8* %0, i8* %1)
+  %argp.cur = load i8*, i8** %aq, align 4
+  %argp.next = getelementptr inbounds i8, i8* %argp.cur, i32 4
+  store i8* %argp.next, i8** %aq, align 4
+  %2 = bitcast i8* %argp.cur to i8**
+  %3 = load i8*, i8** %2, align 4
+  store i8* %3, i8** %s, align 4
+  %4 = load i8*, i8** %s, align 4
+  %call = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([11 x i8], [11 x i8]* @.str, i32 0, i32 0), i8* %4)
+  ret void
+}
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/add.ll b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/add.ll
index 8a6571108..0b217b837 100644
--- a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/add.ll
+++ b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/add.ll
@@ -28,8 +28,7 @@ define zeroext i8 @add_i8_zext(i8 zeroext %a, i8 zeroext %b) {
 ; MIPS32-LABEL: add_i8_zext:
 ; MIPS32:       # %bb.0: # %entry
 ; MIPS32-NEXT:    addu $1, $5, $4
-; MIPS32-NEXT:    ori $2, $zero, 255
-; MIPS32-NEXT:    and $2, $1, $2
+; MIPS32-NEXT:    andi $2, $1, 255
 ; MIPS32-NEXT:    jr $ra
 ; MIPS32-NEXT:    nop
 entry:
@@ -65,8 +64,7 @@ define zeroext i16 @add_i16_zext(i16 zeroext %a, i16 zeroext %b) {
 ; MIPS32-LABEL: add_i16_zext:
 ; MIPS32:       # %bb.0: # %entry
 ; MIPS32-NEXT:    addu $1, $5, $4
-; MIPS32-NEXT:    ori $2, $zero, 65535
-; MIPS32-NEXT:    and $2, $1, $2
+; MIPS32-NEXT:    andi $2, $1, 65535
 ; MIPS32-NEXT:    jr $ra
 ; MIPS32-NEXT:    nop
 entry:
@@ -88,15 +86,11 @@ entry:
 define i64 @add_i64(i64 %a, i64 %b) {
 ; MIPS32-LABEL: add_i64:
 ; MIPS32:       # %bb.0: # %entry
-; MIPS32-NEXT:    ori $1, $zero, 0
-; MIPS32-NEXT:    addu $2, $6, $4
-; MIPS32-NEXT:    ori $3, $zero, 1
-; MIPS32-NEXT:    and $1, $1, $3
-; MIPS32-NEXT:    addu $1, $2, $1
-; MIPS32-NEXT:    sltu $2, $1, $6
-; MIPS32-NEXT:    addu $4, $7, $5
-; MIPS32-NEXT:    and $2, $2, $3
-; MIPS32-NEXT:    addu $3, $4, $2
+; MIPS32-NEXT:    addu $1, $6, $4
+; MIPS32-NEXT:    sltu $2, $1, $4
+; MIPS32-NEXT:    addu $3, $7, $5
+; MIPS32-NEXT:    andi $2, $2, 1
+; MIPS32-NEXT:    addu $3, $3, $2
 ; MIPS32-NEXT:    move $2, $1
 ; MIPS32-NEXT:    jr $ra
 ; MIPS32-NEXT:    nop
@@ -118,26 +112,22 @@ define i128 @add_i128(i128 %a, i128 %b) {
 ; MIPS32-NEXT:    lw $3, 0($3)
 ; MIPS32-NEXT:    addiu $8, $sp, 36
 ; MIPS32-NEXT:    lw $8, 0($8)
-; MIPS32-NEXT:    ori $9, $zero, 0
-; MIPS32-NEXT:    addu $4, $1, $4
-; MIPS32-NEXT:    ori $10, $zero, 1
-; MIPS32-NEXT:    and $9, $9, $10
-; MIPS32-NEXT:    addu $4, $4, $9
-; MIPS32-NEXT:    sltu $1, $4, $1
+; MIPS32-NEXT:    addu $1, $1, $4
+; MIPS32-NEXT:    sltu $4, $1, $4
 ; MIPS32-NEXT:    addu $5, $2, $5
-; MIPS32-NEXT:    and $1, $1, $10
-; MIPS32-NEXT:    addu $1, $5, $1
-; MIPS32-NEXT:    sltu $2, $1, $2
+; MIPS32-NEXT:    andi $4, $4, 1
+; MIPS32-NEXT:    addu $4, $5, $4
+; MIPS32-NEXT:    sltu $2, $4, $2
 ; MIPS32-NEXT:    addu $5, $3, $6
-; MIPS32-NEXT:    and $2, $2, $10
+; MIPS32-NEXT:    andi $2, $2, 1
 ; MIPS32-NEXT:    addu $2, $5, $2
 ; MIPS32-NEXT:    sltu $3, $2, $3
 ; MIPS32-NEXT:    addu $5, $8, $7
-; MIPS32-NEXT:    and $3, $3, $10
+; MIPS32-NEXT:    andi $3, $3, 1
 ; MIPS32-NEXT:    addu $5, $5, $3
 ; MIPS32-NEXT:    sw $2, 4($sp) # 4-byte Folded Spill
-; MIPS32-NEXT:    move $2, $4
-; MIPS32-NEXT:    move $3, $1
+; MIPS32-NEXT:    move $2, $1
+; MIPS32-NEXT:    move $3, $4
 ; MIPS32-NEXT:    lw $4, 4($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    addiu $sp, $sp, 8
 ; MIPS32-NEXT:    jr $ra
@@ -147,14 +137,47 @@ entry:
   ret i128 %add
 }
 
+define i32 @add_imm(i32 %a) {
+; MIPS32-LABEL: add_imm:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    addiu $2, $4, 3
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %add = add i32 %a, 3
+  ret i32 %add
+}
+
+define i32 @add_negative_imm(i32 %a) {
+; MIPS32-LABEL: add_negative_imm:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    addiu $2, $4, -3
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %add = add i32 %a, -3
+  ret i32 %add
+}
+
+define i32 @add_not_imm32SExt16(i32 %a) {
+; MIPS32-LABEL: add_not_imm32SExt16:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    ori $1, $zero, 65535
+; MIPS32-NEXT:    addu $2, $4, $1
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %add = add i32 %a, 65535
+  ret i32 %add
+}
+
 declare { i32, i1 } @llvm.uadd.with.overflow.i32(i32, i32)
 define void @uadd_with_overflow(i32 %lhs, i32 %rhs, i32* %padd, i1* %pcarry_flag) {
 ; MIPS32-LABEL: uadd_with_overflow:
 ; MIPS32:       # %bb.0:
 ; MIPS32-NEXT:    addu $1, $4, $5
 ; MIPS32-NEXT:    sltu $2, $1, $5
-; MIPS32-NEXT:    ori $3, $zero, 1
-; MIPS32-NEXT:    and $2, $2, $3
+; MIPS32-NEXT:    andi $2, $2, 1
 ; MIPS32-NEXT:    sb $2, 0($7)
 ; MIPS32-NEXT:    sw $1, 0($6)
 ; MIPS32-NEXT:    jr $ra
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/add_vec.ll b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/add_vec.ll
new file mode 100644
index 000000000..5d8585173
--- /dev/null
+++ b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/add_vec.ll
@@ -0,0 +1,70 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -O0 -mtriple=mipsel-linux-gnu -global-isel -mcpu=mips32r5 -mattr=+msa,+fp64,+nan2008 -verify-machineinstrs %s -o -| FileCheck %s -check-prefixes=P5600
+
+define void @add_v16i8(<16 x i8>* %a, <16 x i8>* %b, <16 x i8>* %c) {
+; P5600-LABEL: add_v16i8:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.b $w0, 0($4)
+; P5600-NEXT:    ld.b $w1, 0($5)
+; P5600-NEXT:    addv.b $w0, $w1, $w0
+; P5600-NEXT:    st.b $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <16 x i8>, <16 x i8>* %a, align 16
+  %1 = load <16 x i8>, <16 x i8>* %b, align 16
+  %add = add <16 x i8> %1, %0
+  store <16 x i8> %add, <16 x i8>* %c, align 16
+  ret void
+}
+
+define void @add_v8i16(<8 x i16>* %a, <8 x i16>* %b, <8 x i16>* %c) {
+; P5600-LABEL: add_v8i16:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.h $w0, 0($4)
+; P5600-NEXT:    ld.h $w1, 0($5)
+; P5600-NEXT:    addv.h $w0, $w1, $w0
+; P5600-NEXT:    st.h $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <8 x i16>, <8 x i16>* %a, align 16
+  %1 = load <8 x i16>, <8 x i16>* %b, align 16
+  %add = add <8 x i16> %1, %0
+  store <8 x i16> %add, <8 x i16>* %c, align 16
+  ret void
+}
+
+define void @add_v4i32(<4 x i32>* %a, <4 x i32>* %b, <4 x i32>* %c) {
+; P5600-LABEL: add_v4i32:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.w $w0, 0($4)
+; P5600-NEXT:    ld.w $w1, 0($5)
+; P5600-NEXT:    addv.w $w0, $w1, $w0
+; P5600-NEXT:    st.w $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <4 x i32>, <4 x i32>* %a, align 16
+  %1 = load <4 x i32>, <4 x i32>* %b, align 16
+  %add = add <4 x i32> %1, %0
+  store <4 x i32> %add, <4 x i32>* %c, align 16
+  ret void
+}
+
+define void @add_v2i64(<2 x i64>* %a, <2 x i64>* %b, <2 x i64>* %c) {
+; P5600-LABEL: add_v2i64:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.d $w0, 0($4)
+; P5600-NEXT:    ld.d $w1, 0($5)
+; P5600-NEXT:    addv.d $w0, $w1, $w0
+; P5600-NEXT:    st.d $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <2 x i64>, <2 x i64>* %a, align 16
+  %1 = load <2 x i64>, <2 x i64>* %b, align 16
+  %add = add <2 x i64> %1, %0
+  store <2 x i64> %add, <2 x i64>* %c, align 16
+  ret void
+}
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/add_vec_builtin.ll b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/add_vec_builtin.ll
new file mode 100644
index 000000000..ea05479ce
--- /dev/null
+++ b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/add_vec_builtin.ll
@@ -0,0 +1,138 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -O0 -mtriple=mipsel-linux-gnu -global-isel -mcpu=mips32r5 -mattr=+msa,+fp64,+nan2008 -verify-machineinstrs %s -o -| FileCheck %s -check-prefixes=P5600
+
+declare <16 x i8> @llvm.mips.addv.b(<16 x i8>, <16 x i8>)
+define void @add_v16i8_builtin(<16 x i8>* %a, <16 x i8>* %b, <16 x i8>* %c) {
+; P5600-LABEL: add_v16i8_builtin:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.b $w0, 0($4)
+; P5600-NEXT:    ld.b $w1, 0($5)
+; P5600-NEXT:    addv.b $w0, $w0, $w1
+; P5600-NEXT:    st.b $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <16 x i8>, <16 x i8>* %a, align 16
+  %1 = load <16 x i8>, <16 x i8>* %b, align 16
+  %2 = tail call <16 x i8> @llvm.mips.addv.b(<16 x i8> %0, <16 x i8> %1)
+  store <16 x i8> %2, <16 x i8>* %c, align 16
+  ret void
+}
+
+declare <8 x i16> @llvm.mips.addv.h(<8 x i16>, <8 x i16>)
+define void @add_v8i16_builtin(<8 x i16>* %a, <8 x i16>* %b, <8 x i16>* %c) {
+; P5600-LABEL: add_v8i16_builtin:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.h $w0, 0($4)
+; P5600-NEXT:    ld.h $w1, 0($5)
+; P5600-NEXT:    addv.h $w0, $w0, $w1
+; P5600-NEXT:    st.h $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <8 x i16>, <8 x i16>* %a, align 16
+  %1 = load <8 x i16>, <8 x i16>* %b, align 16
+  %2 = tail call <8 x i16> @llvm.mips.addv.h(<8 x i16> %0, <8 x i16> %1)
+  store <8 x i16> %2, <8 x i16>* %c, align 16
+  ret void
+}
+
+declare <4 x i32> @llvm.mips.addv.w(<4 x i32>, <4 x i32>)
+define void @add_v4i32_builtin(<4 x i32>* %a, <4 x i32>* %b, <4 x i32>* %c) {
+; P5600-LABEL: add_v4i32_builtin:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.w $w0, 0($4)
+; P5600-NEXT:    ld.w $w1, 0($5)
+; P5600-NEXT:    addv.w $w0, $w0, $w1
+; P5600-NEXT:    st.w $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <4 x i32>, <4 x i32>* %a, align 16
+  %1 = load <4 x i32>, <4 x i32>* %b, align 16
+  %2 = tail call <4 x i32> @llvm.mips.addv.w(<4 x i32> %0, <4 x i32> %1)
+  store <4 x i32> %2, <4 x i32>* %c, align 16
+  ret void
+}
+
+declare <2 x i64> @llvm.mips.addv.d(<2 x i64>, <2 x i64>)
+define void @add_v2i64_builtin(<2 x i64>* %a, <2 x i64>* %b, <2 x i64>* %c) {
+; P5600-LABEL: add_v2i64_builtin:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.d $w0, 0($4)
+; P5600-NEXT:    ld.d $w1, 0($5)
+; P5600-NEXT:    addv.d $w0, $w0, $w1
+; P5600-NEXT:    st.d $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <2 x i64>, <2 x i64>* %a, align 16
+  %1 = load <2 x i64>, <2 x i64>* %b, align 16
+  %2 = tail call <2 x i64> @llvm.mips.addv.d(<2 x i64> %0, <2 x i64> %1)
+  store <2 x i64> %2, <2 x i64>* %c, align 16
+  ret void
+}
+
+declare <16 x i8> @llvm.mips.addvi.b(<16 x i8>, i32 immarg)
+define void @add_v16i8_builtin_imm(<16 x i8>* %a, <16 x i8>* %c) {
+; P5600-LABEL: add_v16i8_builtin_imm:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.b $w0, 0($4)
+; P5600-NEXT:    addvi.b $w0, $w0, 3
+; P5600-NEXT:    st.b $w0, 0($5)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <16 x i8>, <16 x i8>* %a, align 16
+  %1 = tail call <16 x i8> @llvm.mips.addvi.b(<16 x i8> %0, i32 3)
+  store <16 x i8> %1, <16 x i8>* %c, align 16
+  ret void
+}
+
+declare <8 x i16> @llvm.mips.addvi.h(<8 x i16>, i32 immarg)
+define void @add_v8i16_builtin_imm(<8 x i16>* %a, <8 x i16>* %c) {
+; P5600-LABEL: add_v8i16_builtin_imm:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.h $w0, 0($4)
+; P5600-NEXT:    addvi.h $w0, $w0, 18
+; P5600-NEXT:    st.h $w0, 0($5)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <8 x i16>, <8 x i16>* %a, align 16
+  %1 = tail call <8 x i16> @llvm.mips.addvi.h(<8 x i16> %0, i32 18)
+  store <8 x i16> %1, <8 x i16>* %c, align 16
+  ret void
+}
+
+declare <4 x i32> @llvm.mips.addvi.w(<4 x i32>, i32 immarg)
+define void @add_v4i32_builtin_imm(<4 x i32>* %a, <4 x i32>* %c) {
+; P5600-LABEL: add_v4i32_builtin_imm:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.w $w0, 0($4)
+; P5600-NEXT:    addvi.w $w0, $w0, 25
+; P5600-NEXT:    st.w $w0, 0($5)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <4 x i32>, <4 x i32>* %a, align 16
+  %1 = tail call <4 x i32> @llvm.mips.addvi.w(<4 x i32> %0, i32 25)
+  store <4 x i32> %1, <4 x i32>* %c, align 16
+  ret void
+}
+
+declare <2 x i64> @llvm.mips.addvi.d(<2 x i64>, i32 immarg)
+define void @add_v2i64_builtin_imm(<2 x i64>* %a, <2 x i64>* %c) {
+; P5600-LABEL: add_v2i64_builtin_imm:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.d $w0, 0($4)
+; P5600-NEXT:    addvi.d $w0, $w0, 31
+; P5600-NEXT:    st.d $w0, 0($5)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <2 x i64>, <2 x i64>* %a, align 16
+  %1 = tail call <2 x i64> @llvm.mips.addvi.d(<2 x i64> %0, i32 31)
+  store <2 x i64> %1, <2 x i64>* %c, align 16
+  ret void
+}
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/aggregate_struct_return.ll b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/aggregate_struct_return.ll
new file mode 100644
index 000000000..a9f49c025
--- /dev/null
+++ b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/aggregate_struct_return.ll
@@ -0,0 +1,114 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc  -O0 -mtriple=mipsel-linux-gnu -global-isel  -verify-machineinstrs %s -o -| FileCheck %s -check-prefixes=MIPS32
+
+define { float, float } @add_complex_float({ float, float }* %a, { float, float }* %b) {
+; MIPS32-LABEL: add_complex_float:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    lwc1 $f0, 0($4)
+; MIPS32-NEXT:    lwc1 $f1, 4($4)
+; MIPS32-NEXT:    lwc1 $f2, 0($5)
+; MIPS32-NEXT:    lwc1 $f3, 4($5)
+; MIPS32-NEXT:    add.s $f0, $f0, $f2
+; MIPS32-NEXT:    add.s $f2, $f1, $f3
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %.realp = getelementptr inbounds { float, float }, { float, float }* %a, i32 0, i32 0
+  %.real = load float, float* %.realp, align 4
+  %.imagp = getelementptr inbounds { float, float }, { float, float }* %a, i32 0, i32 1
+  %.imag = load float, float* %.imagp, align 4
+  %.realp1 = getelementptr inbounds { float, float }, { float, float }* %b, i32 0, i32 0
+  %.real2 = load float, float* %.realp1, align 4
+  %.imagp3 = getelementptr inbounds { float, float }, { float, float }* %b, i32 0, i32 1
+  %.imag4 = load float, float* %.imagp3, align 4
+  %add.r = fadd float %.real, %.real2
+  %add.i = fadd float %.imag, %.imag4
+  %.fca.0.insert = insertvalue { float, float } undef, float %add.r, 0
+  %.fca.1.insert = insertvalue { float, float } %.fca.0.insert, float %add.i, 1
+  ret { float, float } %.fca.1.insert
+}
+
+define { double, double } @add_complex_double({ double, double }* %a, { double, double }* %b) {
+; MIPS32-LABEL: add_complex_double:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    ldc1 $f0, 0($4)
+; MIPS32-NEXT:    ldc1 $f2, 8($4)
+; MIPS32-NEXT:    ldc1 $f4, 0($5)
+; MIPS32-NEXT:    ldc1 $f6, 8($5)
+; MIPS32-NEXT:    add.d $f0, $f0, $f4
+; MIPS32-NEXT:    add.d $f2, $f2, $f6
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %.realp = getelementptr inbounds { double, double }, { double, double }* %a, i32 0, i32 0
+  %.real = load double, double* %.realp, align 8
+  %.imagp = getelementptr inbounds { double, double }, { double, double }* %a, i32 0, i32 1
+  %.imag = load double, double* %.imagp, align 8
+  %.realp1 = getelementptr inbounds { double, double }, { double, double }* %b, i32 0, i32 0
+  %.real2 = load double, double* %.realp1, align 8
+  %.imagp3 = getelementptr inbounds { double, double }, { double, double }* %b, i32 0, i32 1
+  %.imag4 = load double, double* %.imagp3, align 8
+  %add.r = fadd double %.real, %.real2
+  %add.i = fadd double %.imag, %.imag4
+  %.fca.0.insert = insertvalue { double, double } undef, double %add.r, 0
+  %.fca.1.insert = insertvalue { double, double } %.fca.0.insert, double %add.i, 1
+  ret { double, double } %.fca.1.insert
+}
+
+declare { float, float } @ret_complex_float()
+define void @call_ret_complex_float({ float, float }* %z) {
+; MIPS32-LABEL: call_ret_complex_float:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    addiu $sp, $sp, -24
+; MIPS32-NEXT:    .cfi_def_cfa_offset 24
+; MIPS32-NEXT:    sw $ra, 20($sp) # 4-byte Folded Spill
+; MIPS32-NEXT:    .cfi_offset 31, -4
+; MIPS32-NEXT:    sw $4, 16($sp) # 4-byte Folded Spill
+; MIPS32-NEXT:    jal ret_complex_float
+; MIPS32-NEXT:    nop
+; MIPS32-NEXT:    lw $1, 16($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    swc1 $f0, 0($1)
+; MIPS32-NEXT:    swc1 $f2, 4($1)
+; MIPS32-NEXT:    lw $ra, 20($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    addiu $sp, $sp, 24
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %call = call { float, float } @ret_complex_float()
+  %0 = extractvalue { float, float } %call, 0
+  %1 = extractvalue { float, float } %call, 1
+  %.realp = getelementptr inbounds { float, float }, { float, float }* %z, i32 0, i32 0
+  %.imagp = getelementptr inbounds { float, float }, { float, float }* %z, i32 0, i32 1
+  store float %0, float* %.realp, align 4
+  store float %1, float* %.imagp, align 4
+  ret void
+}
+
+declare { double, double } @ret_complex_double()
+define void @call_ret_complex_double({ double, double }* %z) {
+; MIPS32-LABEL: call_ret_complex_double:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    addiu $sp, $sp, -24
+; MIPS32-NEXT:    .cfi_def_cfa_offset 24
+; MIPS32-NEXT:    sw $ra, 20($sp) # 4-byte Folded Spill
+; MIPS32-NEXT:    .cfi_offset 31, -4
+; MIPS32-NEXT:    sw $4, 16($sp) # 4-byte Folded Spill
+; MIPS32-NEXT:    jal ret_complex_double
+; MIPS32-NEXT:    nop
+; MIPS32-NEXT:    lw $1, 16($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    sdc1 $f0, 0($1)
+; MIPS32-NEXT:    sdc1 $f2, 8($1)
+; MIPS32-NEXT:    lw $ra, 20($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    addiu $sp, $sp, 24
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %call = call { double, double } @ret_complex_double()
+  %0 = extractvalue { double, double } %call, 0
+  %1 = extractvalue { double, double } %call, 1
+  %.realp = getelementptr inbounds { double, double }, { double, double }* %z, i32 0, i32 0
+  %.imagp = getelementptr inbounds { double, double }, { double, double }* %z, i32 0, i32 1
+  store double %0, double* %.realp, align 8
+  store double %1, double* %.imagp, align 8
+  ret void
+}
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/bitreverse.ll b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/bitreverse.ll
new file mode 100644
index 000000000..662bcdf75
--- /dev/null
+++ b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/bitreverse.ll
@@ -0,0 +1,184 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -O0 -mtriple=mipsel-linux-gnu -global-isel -verify-machineinstrs %s -o -| FileCheck %s -check-prefixes=MIPS32
+; RUN: llc -O0 -mtriple=mipsel-linux-gnu -global-isel -mattr=+mips32r2 -verify-machineinstrs %s -o -| FileCheck %s -check-prefixes=MIPS32R2
+
+declare i32 @llvm.bitreverse.i32(i32)
+define i32 @bitreverse_i32(i32 signext %a) {
+; MIPS32-LABEL: bitreverse_i32:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    sll $1, $4, 24
+; MIPS32-NEXT:    srl $2, $4, 24
+; MIPS32-NEXT:    or $1, $2, $1
+; MIPS32-NEXT:    andi $2, $4, 65280
+; MIPS32-NEXT:    sll $2, $2, 8
+; MIPS32-NEXT:    or $1, $1, $2
+; MIPS32-NEXT:    srl $2, $4, 8
+; MIPS32-NEXT:    andi $2, $2, 65280
+; MIPS32-NEXT:    or $1, $1, $2
+; MIPS32-NEXT:    lui $2, 61680
+; MIPS32-NEXT:    ori $2, $2, 61680
+; MIPS32-NEXT:    and $3, $1, $2
+; MIPS32-NEXT:    srl $3, $3, 4
+; MIPS32-NEXT:    sll $1, $1, 4
+; MIPS32-NEXT:    and $1, $1, $2
+; MIPS32-NEXT:    or $1, $3, $1
+; MIPS32-NEXT:    lui $2, 52428
+; MIPS32-NEXT:    ori $2, $2, 52428
+; MIPS32-NEXT:    and $3, $1, $2
+; MIPS32-NEXT:    srl $3, $3, 2
+; MIPS32-NEXT:    sll $1, $1, 2
+; MIPS32-NEXT:    and $1, $1, $2
+; MIPS32-NEXT:    or $1, $3, $1
+; MIPS32-NEXT:    lui $2, 43690
+; MIPS32-NEXT:    ori $2, $2, 43690
+; MIPS32-NEXT:    and $3, $1, $2
+; MIPS32-NEXT:    srl $3, $3, 1
+; MIPS32-NEXT:    sll $1, $1, 1
+; MIPS32-NEXT:    and $1, $1, $2
+; MIPS32-NEXT:    or $2, $3, $1
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+;
+; MIPS32R2-LABEL: bitreverse_i32:
+; MIPS32R2:       # %bb.0: # %entry
+; MIPS32R2-NEXT:    wsbh $1, $4
+; MIPS32R2-NEXT:    rotr $1, $1, 16
+; MIPS32R2-NEXT:    lui $2, 61680
+; MIPS32R2-NEXT:    ori $2, $2, 61680
+; MIPS32R2-NEXT:    and $3, $1, $2
+; MIPS32R2-NEXT:    srl $3, $3, 4
+; MIPS32R2-NEXT:    sll $1, $1, 4
+; MIPS32R2-NEXT:    and $1, $1, $2
+; MIPS32R2-NEXT:    or $1, $3, $1
+; MIPS32R2-NEXT:    lui $2, 52428
+; MIPS32R2-NEXT:    ori $2, $2, 52428
+; MIPS32R2-NEXT:    and $3, $1, $2
+; MIPS32R2-NEXT:    srl $3, $3, 2
+; MIPS32R2-NEXT:    sll $1, $1, 2
+; MIPS32R2-NEXT:    and $1, $1, $2
+; MIPS32R2-NEXT:    or $1, $3, $1
+; MIPS32R2-NEXT:    lui $2, 43690
+; MIPS32R2-NEXT:    ori $2, $2, 43690
+; MIPS32R2-NEXT:    and $3, $1, $2
+; MIPS32R2-NEXT:    srl $3, $3, 1
+; MIPS32R2-NEXT:    sll $1, $1, 1
+; MIPS32R2-NEXT:    and $1, $1, $2
+; MIPS32R2-NEXT:    or $2, $3, $1
+; MIPS32R2-NEXT:    jr $ra
+; MIPS32R2-NEXT:    nop
+entry:
+  %0 = call i32 @llvm.bitreverse.i32(i32 %a)
+  ret i32 %0
+}
+
+declare i64 @llvm.bitreverse.i64(i64)
+define i64 @bitreverse_i64(i64 signext %a) {
+; MIPS32-LABEL: bitreverse_i64:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    sll $1, $5, 24
+; MIPS32-NEXT:    srl $2, $5, 24
+; MIPS32-NEXT:    or $1, $2, $1
+; MIPS32-NEXT:    andi $2, $5, 65280
+; MIPS32-NEXT:    sll $2, $2, 8
+; MIPS32-NEXT:    or $1, $1, $2
+; MIPS32-NEXT:    srl $2, $5, 8
+; MIPS32-NEXT:    andi $2, $2, 65280
+; MIPS32-NEXT:    or $1, $1, $2
+; MIPS32-NEXT:    lui $2, 61680
+; MIPS32-NEXT:    ori $2, $2, 61680
+; MIPS32-NEXT:    and $3, $1, $2
+; MIPS32-NEXT:    srl $3, $3, 4
+; MIPS32-NEXT:    sll $1, $1, 4
+; MIPS32-NEXT:    and $1, $1, $2
+; MIPS32-NEXT:    or $1, $3, $1
+; MIPS32-NEXT:    lui $3, 52428
+; MIPS32-NEXT:    ori $3, $3, 52428
+; MIPS32-NEXT:    and $5, $1, $3
+; MIPS32-NEXT:    srl $5, $5, 2
+; MIPS32-NEXT:    sll $1, $1, 2
+; MIPS32-NEXT:    and $1, $1, $3
+; MIPS32-NEXT:    or $1, $5, $1
+; MIPS32-NEXT:    lui $5, 43690
+; MIPS32-NEXT:    ori $5, $5, 43690
+; MIPS32-NEXT:    and $6, $1, $5
+; MIPS32-NEXT:    srl $6, $6, 1
+; MIPS32-NEXT:    sll $1, $1, 1
+; MIPS32-NEXT:    and $1, $1, $5
+; MIPS32-NEXT:    or $1, $6, $1
+; MIPS32-NEXT:    sll $6, $4, 24
+; MIPS32-NEXT:    srl $7, $4, 24
+; MIPS32-NEXT:    or $6, $7, $6
+; MIPS32-NEXT:    andi $7, $4, 65280
+; MIPS32-NEXT:    sll $7, $7, 8
+; MIPS32-NEXT:    or $6, $6, $7
+; MIPS32-NEXT:    srl $4, $4, 8
+; MIPS32-NEXT:    andi $4, $4, 65280
+; MIPS32-NEXT:    or $4, $6, $4
+; MIPS32-NEXT:    and $6, $4, $2
+; MIPS32-NEXT:    srl $6, $6, 4
+; MIPS32-NEXT:    sll $4, $4, 4
+; MIPS32-NEXT:    and $2, $4, $2
+; MIPS32-NEXT:    or $2, $6, $2
+; MIPS32-NEXT:    and $4, $2, $3
+; MIPS32-NEXT:    srl $4, $4, 2
+; MIPS32-NEXT:    sll $2, $2, 2
+; MIPS32-NEXT:    and $2, $2, $3
+; MIPS32-NEXT:    or $2, $4, $2
+; MIPS32-NEXT:    and $3, $2, $5
+; MIPS32-NEXT:    srl $3, $3, 1
+; MIPS32-NEXT:    sll $2, $2, 1
+; MIPS32-NEXT:    and $2, $2, $5
+; MIPS32-NEXT:    or $3, $3, $2
+; MIPS32-NEXT:    move $2, $1
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+;
+; MIPS32R2-LABEL: bitreverse_i64:
+; MIPS32R2:       # %bb.0: # %entry
+; MIPS32R2-NEXT:    wsbh $1, $5
+; MIPS32R2-NEXT:    rotr $1, $1, 16
+; MIPS32R2-NEXT:    lui $2, 61680
+; MIPS32R2-NEXT:    ori $2, $2, 61680
+; MIPS32R2-NEXT:    and $3, $1, $2
+; MIPS32R2-NEXT:    srl $3, $3, 4
+; MIPS32R2-NEXT:    sll $1, $1, 4
+; MIPS32R2-NEXT:    and $1, $1, $2
+; MIPS32R2-NEXT:    or $1, $3, $1
+; MIPS32R2-NEXT:    lui $3, 52428
+; MIPS32R2-NEXT:    ori $3, $3, 52428
+; MIPS32R2-NEXT:    and $5, $1, $3
+; MIPS32R2-NEXT:    srl $5, $5, 2
+; MIPS32R2-NEXT:    sll $1, $1, 2
+; MIPS32R2-NEXT:    and $1, $1, $3
+; MIPS32R2-NEXT:    or $1, $5, $1
+; MIPS32R2-NEXT:    lui $5, 43690
+; MIPS32R2-NEXT:    ori $5, $5, 43690
+; MIPS32R2-NEXT:    and $6, $1, $5
+; MIPS32R2-NEXT:    srl $6, $6, 1
+; MIPS32R2-NEXT:    sll $1, $1, 1
+; MIPS32R2-NEXT:    and $1, $1, $5
+; MIPS32R2-NEXT:    or $1, $6, $1
+; MIPS32R2-NEXT:    wsbh $4, $4
+; MIPS32R2-NEXT:    rotr $4, $4, 16
+; MIPS32R2-NEXT:    and $6, $4, $2
+; MIPS32R2-NEXT:    srl $6, $6, 4
+; MIPS32R2-NEXT:    sll $4, $4, 4
+; MIPS32R2-NEXT:    and $2, $4, $2
+; MIPS32R2-NEXT:    or $2, $6, $2
+; MIPS32R2-NEXT:    and $4, $2, $3
+; MIPS32R2-NEXT:    srl $4, $4, 2
+; MIPS32R2-NEXT:    sll $2, $2, 2
+; MIPS32R2-NEXT:    and $2, $2, $3
+; MIPS32R2-NEXT:    or $2, $4, $2
+; MIPS32R2-NEXT:    and $3, $2, $5
+; MIPS32R2-NEXT:    srl $3, $3, 1
+; MIPS32R2-NEXT:    sll $2, $2, 1
+; MIPS32R2-NEXT:    and $2, $2, $5
+; MIPS32R2-NEXT:    or $3, $3, $2
+; MIPS32R2-NEXT:    move $2, $1
+; MIPS32R2-NEXT:    jr $ra
+; MIPS32R2-NEXT:    nop
+entry:
+  %0 = call i64 @llvm.bitreverse.i64(i64 %a)
+  ret i64 %0
+}
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/bitwise.ll b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/bitwise.ll
index 9d8671eed..4022efcaf 100644
--- a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/bitwise.ll
+++ b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/bitwise.ll
@@ -57,6 +57,29 @@ entry:
   ret i64 %and
 }
 
+define i32 @and_imm(i32 %a) {
+; MIPS32-LABEL: and_imm:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    andi $2, $4, 255
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %and = and i32 %a, 255
+  ret i32 %and
+}
+
+define i32 @and_not_imm32ZExt16(i32 %a) {
+; MIPS32-LABEL: and_not_imm32ZExt16:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    addiu $1, $zero, 65280
+; MIPS32-NEXT:    and $2, $4, $1
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %and = and i32 %a, -256
+  ret i32 %and
+}
+
 define i1 @or_i1(i1 %a, i1 %b) {
 ; MIPS32-LABEL: or_i1:
 ; MIPS32:       # %bb.0: # %entry
@@ -113,6 +136,29 @@ entry:
   ret i64 %or
 }
 
+define i32 @or_imm(i32 %a) {
+; MIPS32-LABEL: or_imm:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    ori $2, $4, 65535
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %or = or i32 %a, 65535
+  ret i32 %or
+}
+
+define i32 @or_not_imm32ZExt16(i32 %a) {
+; MIPS32-LABEL: or_not_imm32ZExt16:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    lui $1, 1
+; MIPS32-NEXT:    or $2, $4, $1
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %or = or i32 %a, 65536
+  ret i32 %or
+}
+
 define i1 @xor_i1(i1 %a, i1 %b) {
 ; MIPS32-LABEL: xor_i1:
 ; MIPS32:       # %bb.0: # %entry
@@ -169,6 +215,28 @@ entry:
   ret i64 %xor
 }
 
+define i32 @xor_imm(i32 %a) {
+; MIPS32-LABEL: xor_imm:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    xori $2, $4, 1
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %xor = xor i32 %a, 1
+  ret i32 %xor
+}
+
+define i32 @xor_not_imm32ZExt16(i32 %a) {
+; MIPS32-LABEL: xor_not_imm32ZExt16:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    not $2, $4
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %xor = xor i32 %a, -1
+  ret i32 %xor
+}
+
 define i32 @shl(i32 %a) {
 ; MIPS32-LABEL: shl:
 ; MIPS32:       # %bb.0: # %entry
@@ -235,3 +303,133 @@ entry:
   ret i32 %shr
 }
 
+define  i16 @shl_i16(i16 %a) {
+; MIPS32-LABEL: shl_i16:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    ori $1, $zero, 2
+; MIPS32-NEXT:    andi $1, $1, 65535
+; MIPS32-NEXT:    sllv $2, $4, $1
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %shl = shl i16 %a, 2
+  ret i16 %shl
+}
+
+define i8 @ashr_i8(i8 %a) {
+; MIPS32-LABEL: ashr_i8:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    ori $1, $zero, 2
+; MIPS32-NEXT:    andi $1, $1, 255
+; MIPS32-NEXT:    sll $2, $4, 24
+; MIPS32-NEXT:    sra $2, $2, 24
+; MIPS32-NEXT:    srav $2, $2, $1
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %0 = ashr i8 %a, 2
+  ret i8 %0
+}
+
+define i16 @lshr_i16(i16 %a) {
+; MIPS32-LABEL: lshr_i16:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    ori $1, $zero, 2
+; MIPS32-NEXT:    andi $1, $1, 65535
+; MIPS32-NEXT:    andi $2, $4, 65535
+; MIPS32-NEXT:    srlv $2, $2, $1
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %0 = lshr i16 %a, 2
+  ret i16 %0
+}
+
+define i64 @shl_i64(i64 %a, i64 %b) {
+; MIPS32-LABEL: shl_i64:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    addiu $sp, $sp, -8
+; MIPS32-NEXT:    .cfi_def_cfa_offset 8
+; MIPS32-NEXT:    ori $1, $zero, 32
+; MIPS32-NEXT:    subu $2, $6, $1
+; MIPS32-NEXT:    subu $3, $1, $6
+; MIPS32-NEXT:    ori $8, $zero, 0
+; MIPS32-NEXT:    sltu $1, $6, $1
+; MIPS32-NEXT:    sltiu $9, $6, 1
+; MIPS32-NEXT:    sllv $10, $4, $6
+; MIPS32-NEXT:    srlv $3, $4, $3
+; MIPS32-NEXT:    sllv $6, $5, $6
+; MIPS32-NEXT:    or $3, $3, $6
+; MIPS32-NEXT:    sllv $2, $4, $2
+; MIPS32-NEXT:    andi $4, $1, 1
+; MIPS32-NEXT:    movn $8, $10, $4
+; MIPS32-NEXT:    andi $1, $1, 1
+; MIPS32-NEXT:    movn $2, $3, $1
+; MIPS32-NEXT:    andi $1, $9, 1
+; MIPS32-NEXT:    movn $2, $5, $1
+; MIPS32-NEXT:    sw $2, 4($sp) # 4-byte Folded Spill
+; MIPS32-NEXT:    move $2, $8
+; MIPS32-NEXT:    lw $3, 4($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    addiu $sp, $sp, 8
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %shl = shl i64 %a, %b
+  ret i64 %shl
+}
+
+define i64 @ashl_i64(i64 %a, i64 %b) {
+; MIPS32-LABEL: ashl_i64:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    ori $1, $zero, 32
+; MIPS32-NEXT:    subu $2, $6, $1
+; MIPS32-NEXT:    subu $3, $1, $6
+; MIPS32-NEXT:    sltu $1, $6, $1
+; MIPS32-NEXT:    sltiu $8, $6, 1
+; MIPS32-NEXT:    srav $9, $5, $6
+; MIPS32-NEXT:    srlv $6, $4, $6
+; MIPS32-NEXT:    sllv $3, $5, $3
+; MIPS32-NEXT:    or $3, $6, $3
+; MIPS32-NEXT:    sra $6, $5, 31
+; MIPS32-NEXT:    srav $2, $5, $2
+; MIPS32-NEXT:    andi $5, $1, 1
+; MIPS32-NEXT:    movn $2, $3, $5
+; MIPS32-NEXT:    andi $3, $8, 1
+; MIPS32-NEXT:    movn $2, $4, $3
+; MIPS32-NEXT:    andi $1, $1, 1
+; MIPS32-NEXT:    movn $6, $9, $1
+; MIPS32-NEXT:    move $3, $6
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %shr = ashr i64 %a, %b
+  ret i64 %shr
+}
+
+define i64 @lshr_i64(i64 %a, i64 %b) {
+; MIPS32-LABEL: lshr_i64:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    ori $1, $zero, 32
+; MIPS32-NEXT:    subu $2, $6, $1
+; MIPS32-NEXT:    subu $3, $1, $6
+; MIPS32-NEXT:    ori $8, $zero, 0
+; MIPS32-NEXT:    sltu $1, $6, $1
+; MIPS32-NEXT:    sltiu $9, $6, 1
+; MIPS32-NEXT:    srlv $10, $5, $6
+; MIPS32-NEXT:    srlv $6, $4, $6
+; MIPS32-NEXT:    sllv $3, $5, $3
+; MIPS32-NEXT:    or $3, $6, $3
+; MIPS32-NEXT:    srlv $2, $5, $2
+; MIPS32-NEXT:    andi $5, $1, 1
+; MIPS32-NEXT:    movn $2, $3, $5
+; MIPS32-NEXT:    andi $3, $9, 1
+; MIPS32-NEXT:    movn $2, $4, $3
+; MIPS32-NEXT:    andi $1, $1, 1
+; MIPS32-NEXT:    movn $8, $10, $1
+; MIPS32-NEXT:    move $3, $8
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %shr = lshr i64 %a, %b
+  ret i64 %shr
+}
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/branch.ll b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/branch.ll
index 8e5cce8e7..4600142cb 100644
--- a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/branch.ll
+++ b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/branch.ll
@@ -30,8 +30,7 @@ define i32 @Conditional_branch(i1 %cond, i32 %a, i32 %b) {
 ; MIPS32:       # %bb.0:
 ; MIPS32-NEXT:    addiu $sp, $sp, -8
 ; MIPS32-NEXT:    .cfi_def_cfa_offset 8
-; MIPS32-NEXT:    ori $1, $zero, 1
-; MIPS32-NEXT:    and $1, $4, $1
+; MIPS32-NEXT:    andi $1, $4, 1
 ; MIPS32-NEXT:    sw $5, 4($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:    sw $6, 0($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:    bnez $1, $BB1_2
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/brindirect.ll b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/brindirect.ll
new file mode 100644
index 000000000..568558538
--- /dev/null
+++ b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/brindirect.ll
@@ -0,0 +1,33 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc  -O0 -mtriple=mipsel-linux-gnu -global-isel  -verify-machineinstrs %s -o -| FileCheck %s -check-prefixes=MIPS32
+
+define i32 @indirectbr(i8 *%addr) {
+; MIPS32-LABEL: indirectbr:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    addiu $sp, $sp, -8
+; MIPS32-NEXT:    .cfi_def_cfa_offset 8
+; MIPS32-NEXT:    ori $2, $zero, 1
+; MIPS32-NEXT:    ori $1, $zero, 0
+; MIPS32-NEXT:    sw $2, 4($sp) # 4-byte Folded Spill
+; MIPS32-NEXT:    sw $1, 0($sp) # 4-byte Folded Spill
+; MIPS32-NEXT:    jr $4
+; MIPS32-NEXT:    nop
+; MIPS32-NEXT:  $BB0_1: # %L1
+; MIPS32-NEXT:    lw $2, 0($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    addiu $sp, $sp, 8
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+; MIPS32-NEXT:  $BB0_2: # %L2
+; MIPS32-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    addiu $sp, $sp, 8
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  indirectbr i8* %addr, [label %L1, label %L2]
+
+L1:
+  ret i32 0
+
+L2:
+  ret i32 1
+}
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/bswap.ll b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/bswap.ll
new file mode 100644
index 000000000..3b371cca6
--- /dev/null
+++ b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/bswap.ll
@@ -0,0 +1,68 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -O0 -mtriple=mipsel-linux-gnu -global-isel -verify-machineinstrs %s -o -| FileCheck %s -check-prefixes=MIPS32
+; RUN: llc -O0 -mtriple=mipsel-linux-gnu -global-isel -mattr=+mips32r2 -verify-machineinstrs %s -o -| FileCheck %s -check-prefixes=MIPS32R2
+
+declare i32 @llvm.bswap.i32(i32)
+define i32 @bswap_i32(i32 %x) {
+; MIPS32-LABEL: bswap_i32:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    sll $1, $4, 24
+; MIPS32-NEXT:    srl $2, $4, 24
+; MIPS32-NEXT:    or $1, $2, $1
+; MIPS32-NEXT:    andi $2, $4, 65280
+; MIPS32-NEXT:    sll $2, $2, 8
+; MIPS32-NEXT:    or $1, $1, $2
+; MIPS32-NEXT:    srl $2, $4, 8
+; MIPS32-NEXT:    andi $2, $2, 65280
+; MIPS32-NEXT:    or $2, $1, $2
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+;
+; MIPS32R2-LABEL: bswap_i32:
+; MIPS32R2:       # %bb.0: # %entry
+; MIPS32R2-NEXT:    wsbh $1, $4
+; MIPS32R2-NEXT:    rotr $2, $1, 16
+; MIPS32R2-NEXT:    jr $ra
+; MIPS32R2-NEXT:    nop
+entry:
+  %0 = tail call i32 @llvm.bswap.i32(i32 %x)
+  ret i32 %0
+}
+
+declare i64 @llvm.bswap.i64(i64)
+define i64 @bswap_i64(i64 %x) {
+; MIPS32-LABEL: bswap_i64:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    sll $1, $5, 24
+; MIPS32-NEXT:    srl $2, $5, 24
+; MIPS32-NEXT:    or $1, $2, $1
+; MIPS32-NEXT:    andi $2, $5, 65280
+; MIPS32-NEXT:    sll $2, $2, 8
+; MIPS32-NEXT:    or $1, $1, $2
+; MIPS32-NEXT:    srl $2, $5, 8
+; MIPS32-NEXT:    andi $2, $2, 65280
+; MIPS32-NEXT:    or $2, $1, $2
+; MIPS32-NEXT:    sll $1, $4, 24
+; MIPS32-NEXT:    srl $3, $4, 24
+; MIPS32-NEXT:    or $1, $3, $1
+; MIPS32-NEXT:    andi $3, $4, 65280
+; MIPS32-NEXT:    sll $3, $3, 8
+; MIPS32-NEXT:    or $1, $1, $3
+; MIPS32-NEXT:    srl $3, $4, 8
+; MIPS32-NEXT:    andi $3, $3, 65280
+; MIPS32-NEXT:    or $3, $1, $3
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+;
+; MIPS32R2-LABEL: bswap_i64:
+; MIPS32R2:       # %bb.0: # %entry
+; MIPS32R2-NEXT:    wsbh $1, $5
+; MIPS32R2-NEXT:    rotr $2, $1, 16
+; MIPS32R2-NEXT:    wsbh $1, $4
+; MIPS32R2-NEXT:    rotr $3, $1, 16
+; MIPS32R2-NEXT:    jr $ra
+; MIPS32R2-NEXT:    nop
+entry:
+  %0 = tail call i64 @llvm.bswap.i64(i64 %x)
+  ret i64 %0
+}
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/constants.ll b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/constants.ll
index 579a04cb9..0187b7278 100644
--- a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/constants.ll
+++ b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/constants.ll
@@ -50,8 +50,7 @@ define zeroext i16 @unsigned_i16() {
 ; MIPS32-LABEL: unsigned_i16:
 ; MIPS32:       # %bb.0: # %entry
 ; MIPS32-NEXT:    addiu $1, $zero, 32768
-; MIPS32-NEXT:    ori $2, $zero, 65535
-; MIPS32-NEXT:    and $2, $1, $2
+; MIPS32-NEXT:    andi $2, $1, 65535
 ; MIPS32-NEXT:    jr $ra
 ; MIPS32-NEXT:    nop
 entry:
@@ -62,8 +61,7 @@ define zeroext i8 @unsigned_i8() {
 ; MIPS32-LABEL: unsigned_i8:
 ; MIPS32:       # %bb.0: # %entry
 ; MIPS32-NEXT:    addiu $1, $zero, 65408
-; MIPS32-NEXT:    ori $2, $zero, 255
-; MIPS32-NEXT:    and $2, $1, $2
+; MIPS32-NEXT:    andi $2, $1, 255
 ; MIPS32-NEXT:    jr $ra
 ; MIPS32-NEXT:    nop
 entry:
@@ -73,9 +71,8 @@ entry:
 define zeroext i1 @i1_true() {
 ; MIPS32-LABEL: i1_true:
 ; MIPS32:       # %bb.0: # %entry
-; MIPS32-NEXT:    addiu $1, $zero, 65535
-; MIPS32-NEXT:    ori $2, $zero, 1
-; MIPS32-NEXT:    and $2, $1, $2
+; MIPS32-NEXT:    ori $1, $zero, 1
+; MIPS32-NEXT:    andi $2, $1, 1
 ; MIPS32-NEXT:    jr $ra
 ; MIPS32-NEXT:    nop
 entry:
@@ -86,8 +83,7 @@ define zeroext i1 @i1_false() {
 ; MIPS32-LABEL: i1_false:
 ; MIPS32:       # %bb.0: # %entry
 ; MIPS32-NEXT:    ori $1, $zero, 0
-; MIPS32-NEXT:    ori $2, $zero, 1
-; MIPS32-NEXT:    and $2, $1, $2
+; MIPS32-NEXT:    andi $2, $1, 1
 ; MIPS32-NEXT:    jr $ra
 ; MIPS32-NEXT:    nop
 entry:
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/dyn_stackalloc.ll b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/dyn_stackalloc.ll
new file mode 100644
index 000000000..fcc2d6ef0
--- /dev/null
+++ b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/dyn_stackalloc.ll
@@ -0,0 +1,64 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc  -O0 -mtriple=mipsel-linux-gnu -global-isel  -verify-machineinstrs %s -o -| FileCheck %s -check-prefixes=MIPS32
+
+declare i32 @puts(i8*)
+declare void @llvm.memset.p0i8.i32(i8*, i8, i32, i1)
+
+define void @Print_c_N_times(i8 %c, i32 %N) {
+; MIPS32-LABEL: Print_c_N_times:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    addiu $sp, $sp, -32
+; MIPS32-NEXT:    .cfi_def_cfa_offset 32
+; MIPS32-NEXT:    sw $ra, 28($sp) # 4-byte Folded Spill
+; MIPS32-NEXT:    sw $fp, 24($sp) # 4-byte Folded Spill
+; MIPS32-NEXT:    .cfi_offset 31, -4
+; MIPS32-NEXT:    .cfi_offset 30, -8
+; MIPS32-NEXT:    move $fp, $sp
+; MIPS32-NEXT:    .cfi_def_cfa_register 30
+; MIPS32-NEXT:    ori $1, $zero, 1
+; MIPS32-NEXT:    ori $2, $zero, 0
+; MIPS32-NEXT:    addiu $3, $5, 1
+; MIPS32-NEXT:    mul $1, $3, $1
+; MIPS32-NEXT:    addiu $1, $1, 7
+; MIPS32-NEXT:    addiu $3, $zero, 65528
+; MIPS32-NEXT:    and $1, $1, $3
+; MIPS32-NEXT:    move $3, $sp
+; MIPS32-NEXT:    subu $1, $3, $1
+; MIPS32-NEXT:    move $sp, $1
+; MIPS32-NEXT:    addiu $sp, $sp, -16
+; MIPS32-NEXT:    sw $4, 20($fp) # 4-byte Folded Spill
+; MIPS32-NEXT:    move $4, $1
+; MIPS32-NEXT:    lw $3, 20($fp) # 4-byte Folded Reload
+; MIPS32-NEXT:    sw $5, 16($fp) # 4-byte Folded Spill
+; MIPS32-NEXT:    move $5, $3
+; MIPS32-NEXT:    lw $6, 16($fp) # 4-byte Folded Reload
+; MIPS32-NEXT:    sw $2, 12($fp) # 4-byte Folded Spill
+; MIPS32-NEXT:    sw $1, 8($fp) # 4-byte Folded Spill
+; MIPS32-NEXT:    jal memset
+; MIPS32-NEXT:    nop
+; MIPS32-NEXT:    addiu $sp, $sp, 16
+; MIPS32-NEXT:    lw $1, 8($fp) # 4-byte Folded Reload
+; MIPS32-NEXT:    lw $2, 16($fp) # 4-byte Folded Reload
+; MIPS32-NEXT:    addu $3, $1, $2
+; MIPS32-NEXT:    lw $4, 12($fp) # 4-byte Folded Reload
+; MIPS32-NEXT:    sb $4, 0($3)
+; MIPS32-NEXT:    addiu $sp, $sp, -16
+; MIPS32-NEXT:    move $4, $1
+; MIPS32-NEXT:    jal puts
+; MIPS32-NEXT:    nop
+; MIPS32-NEXT:    addiu $sp, $sp, 16
+; MIPS32-NEXT:    move $sp, $fp
+; MIPS32-NEXT:    lw $fp, 24($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    lw $ra, 28($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    addiu $sp, $sp, 32
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %add = add i32 %N, 1
+  %vla = alloca i8, i32 %add, align 1
+  call void @llvm.memset.p0i8.i32(i8* align 1 %vla, i8 %c, i32 %N, i1 false)
+  %arrayidx = getelementptr inbounds i8, i8* %vla, i32 %N
+  store i8 0, i8* %arrayidx, align 1
+  %call = call i32 @puts(i8* %vla)
+  ret void
+}
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/fabs_vec.ll b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/fabs_vec.ll
new file mode 100644
index 000000000..0413bf231
--- /dev/null
+++ b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/fabs_vec.ll
@@ -0,0 +1,34 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -O0 -mtriple=mipsel-linux-gnu -global-isel -mcpu=mips32r5 -mattr=+msa,+fp64,+nan2008 -verify-machineinstrs %s -o -| FileCheck %s -check-prefixes=P5600
+
+declare <4 x float>  @llvm.fabs.v4f32(<4 x float>  %Val)
+define void @fabs_v4f32(<4 x float>* %a, <4 x float>* %c) {
+; P5600-LABEL: fabs_v4f32:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.w $w0, 0($4)
+; P5600-NEXT:    fmax_a.w $w0, $w0, $w0
+; P5600-NEXT:    st.w $w0, 0($5)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <4 x float>, <4 x float>* %a, align 16
+  %fabs = call <4 x float> @llvm.fabs.v4f32 (<4 x float> %0)
+  store <4 x float> %fabs, <4 x float>* %c, align 16
+  ret void
+}
+
+declare <2 x double> @llvm.fabs.v2f64(<2 x double> %Val)
+define void @fabs_v2f64(<2 x double>* %a, <2 x double>* %c) {
+; P5600-LABEL: fabs_v2f64:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.d $w0, 0($4)
+; P5600-NEXT:    fmax_a.d $w0, $w0, $w0
+; P5600-NEXT:    st.d $w0, 0($5)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <2 x double>, <2 x double>* %a, align 16
+  %fabs = call <2 x double> @llvm.fabs.v2f64 (<2 x double> %0)
+  store <2 x double> %fabs, <2 x double>* %c, align 16
+  ret void
+}
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/fabs_vec_builtin.ll b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/fabs_vec_builtin.ll
new file mode 100644
index 000000000..b29d2f9a5
--- /dev/null
+++ b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/fabs_vec_builtin.ll
@@ -0,0 +1,35 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -O0 -mtriple=mipsel-linux-gnu -global-isel -mcpu=mips32r5 -mattr=+msa,+fp64,+nan2008 -verify-machineinstrs %s -o -| FileCheck %s -check-prefixes=P5600
+
+declare <4 x float> @llvm.mips.fmax.a.w(<4 x float>, <4 x float>)
+define void @fabs_v4f32_builtin(<4 x float>* %a, <4 x float>* %c) {
+; P5600-LABEL: fabs_v4f32_builtin:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.w $w0, 0($4)
+; P5600-NEXT:    fmax_a.w $w0, $w0, $w0
+; P5600-NEXT:    st.w $w0, 0($5)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <4 x float>, <4 x float>* %a, align 16
+  %1 = tail call <4 x float> @llvm.mips.fmax.a.w(<4 x float> %0, <4 x float> %0)
+  store <4 x float> %1, <4 x float>* %c, align 16
+  ret void
+}
+
+
+declare <2 x double> @llvm.mips.fmax.a.d(<2 x double>, <2 x double>)
+define void @fabs_v2f64_builtin(<2 x double>* %a, <2 x double>* %c) {
+; P5600-LABEL: fabs_v2f64_builtin:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.d $w0, 0($4)
+; P5600-NEXT:    fmax_a.d $w0, $w0, $w0
+; P5600-NEXT:    st.d $w0, 0($5)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <2 x double>, <2 x double>* %a, align 16
+  %1 = tail call <2 x double> @llvm.mips.fmax.a.d(<2 x double> %0, <2 x double> %0)
+  store <2 x double> %1, <2 x double>* %c, align 16
+  ret void
+}
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/fcmp.ll b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/fcmp.ll
index 8f559633c..58d5c8a16 100644
--- a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/fcmp.ll
+++ b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/fcmp.ll
@@ -15,7 +15,7 @@ entry:
 define i1 @true_s(float %x, float %y) {
 ; MIPS32-LABEL: true_s:
 ; MIPS32:       # %bb.0: # %entry
-; MIPS32-NEXT:    addiu $2, $zero, 65535
+; MIPS32-NEXT:    ori $2, $zero, 1
 ; MIPS32-NEXT:    jr $ra
 ; MIPS32-NEXT:    nop
 entry:
@@ -233,7 +233,7 @@ entry:
 define i1 @true_d(double %x, double %y) {
 ; MIPS32-LABEL: true_d:
 ; MIPS32:       # %bb.0: # %entry
-; MIPS32-NEXT:    addiu $2, $zero, 65535
+; MIPS32-NEXT:    ori $2, $zero, 1
 ; MIPS32-NEXT:    jr $ra
 ; MIPS32-NEXT:    nop
 entry:
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/fence.ll b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/fence.ll
new file mode 100644
index 000000000..ddf55a735
--- /dev/null
+++ b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/fence.ll
@@ -0,0 +1,13 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc  -O0 -mtriple=mipsel-linux-gnu -global-isel  -verify-machineinstrs %s -o -| FileCheck %s -check-prefixes=MIPS32
+
+define i32 @atomic_load_i32(i32* %ptr) {
+; MIPS32-LABEL: atomic_load_i32:
+; MIPS32:       # %bb.0:
+; MIPS32-NEXT:    lw $2, 0($4)
+; MIPS32-NEXT:    sync
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+  %val = load atomic i32, i32* %ptr acquire, align 4
+  ret i32 %val
+}
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/floating_point_vec_arithmetic_operations.ll b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/floating_point_vec_arithmetic_operations.ll
new file mode 100644
index 000000000..6a7486fca
--- /dev/null
+++ b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/floating_point_vec_arithmetic_operations.ll
@@ -0,0 +1,145 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -O0 -mtriple=mipsel-linux-gnu -global-isel -mcpu=mips32r5 -mattr=+msa,+fp64,+nan2008 -verify-machineinstrs %s -o -| FileCheck %s -check-prefixes=P5600
+
+define void @fadd_v4f32(<4 x float>* %a, <4 x float>* %b, <4 x float>* %c) {
+; P5600-LABEL: fadd_v4f32:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.w $w0, 0($4)
+; P5600-NEXT:    ld.w $w1, 0($5)
+; P5600-NEXT:    fadd.w $w0, $w0, $w1
+; P5600-NEXT:    st.w $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <4 x float>, <4 x float>* %a, align 16
+  %1 = load <4 x float>, <4 x float>* %b, align 16
+  %add = fadd <4 x float> %0, %1
+  store <4 x float> %add, <4 x float>* %c, align 16
+  ret void
+}
+
+
+define void @fadd_v2f64(<2 x double>* %a, <2 x double>* %b, <2 x double>* %c) {
+; P5600-LABEL: fadd_v2f64:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.d $w0, 0($4)
+; P5600-NEXT:    ld.d $w1, 0($5)
+; P5600-NEXT:    fadd.d $w0, $w0, $w1
+; P5600-NEXT:    st.d $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <2 x double>, <2 x double>* %a, align 16
+  %1 = load <2 x double>, <2 x double>* %b, align 16
+  %add = fadd <2 x double> %0, %1
+  store <2 x double> %add, <2 x double>* %c, align 16
+  ret void
+}
+
+
+define void @fsub_v4f32(<4 x float>* %a, <4 x float>* %b, <4 x float>* %c) {
+; P5600-LABEL: fsub_v4f32:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.w $w0, 0($4)
+; P5600-NEXT:    ld.w $w1, 0($5)
+; P5600-NEXT:    fsub.w $w0, $w0, $w1
+; P5600-NEXT:    st.w $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <4 x float>, <4 x float>* %a, align 16
+  %1 = load <4 x float>, <4 x float>* %b, align 16
+  %sub = fsub <4 x float> %0, %1
+  store <4 x float> %sub, <4 x float>* %c, align 16
+  ret void
+}
+
+
+define void @fsub_v2f64(<2 x double>* %a, <2 x double>* %b, <2 x double>* %c) {
+; P5600-LABEL: fsub_v2f64:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.d $w0, 0($4)
+; P5600-NEXT:    ld.d $w1, 0($5)
+; P5600-NEXT:    fsub.d $w0, $w0, $w1
+; P5600-NEXT:    st.d $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <2 x double>, <2 x double>* %a, align 16
+  %1 = load <2 x double>, <2 x double>* %b, align 16
+  %sub = fsub <2 x double> %0, %1
+  store <2 x double> %sub, <2 x double>* %c, align 16
+  ret void
+}
+
+
+define void @fmul_v4f32(<4 x float>* %a, <4 x float>* %b, <4 x float>* %c) {
+; P5600-LABEL: fmul_v4f32:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.w $w0, 0($4)
+; P5600-NEXT:    ld.w $w1, 0($5)
+; P5600-NEXT:    fmul.w $w0, $w0, $w1
+; P5600-NEXT:    st.w $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <4 x float>, <4 x float>* %a, align 16
+  %1 = load <4 x float>, <4 x float>* %b, align 16
+  %mul = fmul <4 x float> %0, %1
+  store <4 x float> %mul, <4 x float>* %c, align 16
+  ret void
+}
+
+
+define void @fmul_v2f64(<2 x double>* %a, <2 x double>* %b, <2 x double>* %c) {
+; P5600-LABEL: fmul_v2f64:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.d $w0, 0($4)
+; P5600-NEXT:    ld.d $w1, 0($5)
+; P5600-NEXT:    fmul.d $w0, $w0, $w1
+; P5600-NEXT:    st.d $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <2 x double>, <2 x double>* %a, align 16
+  %1 = load <2 x double>, <2 x double>* %b, align 16
+  %mul = fmul <2 x double> %0, %1
+  store <2 x double> %mul, <2 x double>* %c, align 16
+  ret void
+}
+
+
+define void @fdiv_v4f32(<4 x float>* %a, <4 x float>* %b, <4 x float>* %c) {
+; P5600-LABEL: fdiv_v4f32:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.w $w0, 0($4)
+; P5600-NEXT:    ld.w $w1, 0($5)
+; P5600-NEXT:    fdiv.w $w0, $w0, $w1
+; P5600-NEXT:    st.w $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <4 x float>, <4 x float>* %a, align 16
+  %1 = load <4 x float>, <4 x float>* %b, align 16
+  %div = fdiv <4 x float> %0, %1
+  store <4 x float> %div, <4 x float>* %c, align 16
+  ret void
+}
+
+
+define void @fdiv_v2f64(<2 x double>* %a, <2 x double>* %b, <2 x double>* %c) {
+; P5600-LABEL: fdiv_v2f64:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.d $w0, 0($4)
+; P5600-NEXT:    ld.d $w1, 0($5)
+; P5600-NEXT:    fdiv.d $w0, $w0, $w1
+; P5600-NEXT:    st.d $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <2 x double>, <2 x double>* %a, align 16
+  %1 = load <2 x double>, <2 x double>* %b, align 16
+  %div = fdiv <2 x double> %0, %1
+  store <2 x double> %div, <2 x double>* %c, align 16
+  ret void
+}
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/floating_point_vec_arithmetic_operations_builtin.ll b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/floating_point_vec_arithmetic_operations_builtin.ll
new file mode 100644
index 000000000..ad7a710be
--- /dev/null
+++ b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/floating_point_vec_arithmetic_operations_builtin.ll
@@ -0,0 +1,146 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -O0 -mtriple=mipsel-linux-gnu -global-isel -mcpu=mips32r5 -mattr=+msa,+fp64,+nan2008 -verify-machineinstrs %s -o -| FileCheck %s -check-prefixes=P5600
+
+declare <4 x float> @llvm.mips.fadd.w(<4 x float>, <4 x float>)
+define void @fadd_v4f32_builtin(<4 x float>* %a, <4 x float>* %b, <4 x float>* %c) {
+; P5600-LABEL: fadd_v4f32_builtin:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.w $w0, 0($4)
+; P5600-NEXT:    ld.w $w1, 0($5)
+; P5600-NEXT:    fadd.w $w0, $w0, $w1
+; P5600-NEXT:    st.w $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <4 x float>, <4 x float>* %a, align 16
+  %1 = load <4 x float>, <4 x float>* %b, align 16
+  %2 = tail call <4 x float> @llvm.mips.fadd.w(<4 x float> %0, <4 x float> %1)
+  store <4 x float> %2, <4 x float>* %c, align 16
+  ret void
+}
+
+declare <2 x double> @llvm.mips.fadd.d(<2 x double>, <2 x double>)
+define void @fadd_v2f64_builtin(<2 x double>* %a, <2 x double>* %b, <2 x double>* %c) {
+; P5600-LABEL: fadd_v2f64_builtin:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.d $w0, 0($4)
+; P5600-NEXT:    ld.d $w1, 0($5)
+; P5600-NEXT:    fadd.d $w0, $w0, $w1
+; P5600-NEXT:    st.d $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <2 x double>, <2 x double>* %a, align 16
+  %1 = load <2 x double>, <2 x double>* %b, align 16
+  %2 = tail call <2 x double> @llvm.mips.fadd.d(<2 x double> %0, <2 x double> %1)
+  store <2 x double> %2, <2 x double>* %c, align 16
+  ret void
+}
+
+declare <4 x float> @llvm.mips.fsub.w(<4 x float>, <4 x float>)
+define void @fsub_v4f32_builtin(<4 x float>* %a, <4 x float>* %b, <4 x float>* %c) {
+; P5600-LABEL: fsub_v4f32_builtin:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.w $w0, 0($4)
+; P5600-NEXT:    ld.w $w1, 0($5)
+; P5600-NEXT:    fsub.w $w0, $w0, $w1
+; P5600-NEXT:    st.w $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <4 x float>, <4 x float>* %a, align 16
+  %1 = load <4 x float>, <4 x float>* %b, align 16
+  %2 = tail call <4 x float> @llvm.mips.fsub.w(<4 x float> %0, <4 x float> %1)
+  store <4 x float> %2, <4 x float>* %c, align 16
+  ret void
+}
+
+declare <2 x double> @llvm.mips.fsub.d(<2 x double>, <2 x double>)
+define void @fsub_v2f64_builtin(<2 x double>* %a, <2 x double>* %b, <2 x double>* %c) {
+; P5600-LABEL: fsub_v2f64_builtin:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.d $w0, 0($4)
+; P5600-NEXT:    ld.d $w1, 0($5)
+; P5600-NEXT:    fsub.d $w0, $w0, $w1
+; P5600-NEXT:    st.d $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <2 x double>, <2 x double>* %a, align 16
+  %1 = load <2 x double>, <2 x double>* %b, align 16
+  %2 = tail call <2 x double> @llvm.mips.fsub.d(<2 x double> %0, <2 x double> %1)
+  store <2 x double> %2, <2 x double>* %c, align 16
+  ret void
+}
+
+declare <4 x float> @llvm.mips.fmul.w(<4 x float>, <4 x float>)
+define void @fmul_v4f32_builtin(<4 x float>* %a, <4 x float>* %b, <4 x float>* %c) {
+; P5600-LABEL: fmul_v4f32_builtin:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.w $w0, 0($4)
+; P5600-NEXT:    ld.w $w1, 0($5)
+; P5600-NEXT:    fmul.w $w0, $w0, $w1
+; P5600-NEXT:    st.w $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <4 x float>, <4 x float>* %a, align 16
+  %1 = load <4 x float>, <4 x float>* %b, align 16
+  %2 = tail call <4 x float> @llvm.mips.fmul.w(<4 x float> %0, <4 x float> %1)
+  store <4 x float> %2, <4 x float>* %c, align 16
+  ret void
+}
+
+declare <2 x double> @llvm.mips.fmul.d(<2 x double>, <2 x double>)
+define void @fmul_v2f64_builtin(<2 x double>* %a, <2 x double>* %b, <2 x double>* %c) {
+; P5600-LABEL: fmul_v2f64_builtin:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.d $w0, 0($4)
+; P5600-NEXT:    ld.d $w1, 0($5)
+; P5600-NEXT:    fmul.d $w0, $w0, $w1
+; P5600-NEXT:    st.d $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <2 x double>, <2 x double>* %a, align 16
+  %1 = load <2 x double>, <2 x double>* %b, align 16
+  %2 = tail call <2 x double> @llvm.mips.fmul.d(<2 x double> %0, <2 x double> %1)
+  store <2 x double> %2, <2 x double>* %c, align 16
+  ret void
+}
+
+declare <4 x float> @llvm.mips.fdiv.w(<4 x float>, <4 x float>)
+define void @fdiv_v4f32_builtin(<4 x float>* %a, <4 x float>* %b, <4 x float>* %c) {
+; P5600-LABEL: fdiv_v4f32_builtin:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.w $w0, 0($4)
+; P5600-NEXT:    ld.w $w1, 0($5)
+; P5600-NEXT:    fdiv.w $w0, $w0, $w1
+; P5600-NEXT:    st.w $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <4 x float>, <4 x float>* %a, align 16
+  %1 = load <4 x float>, <4 x float>* %b, align 16
+  %2 = tail call <4 x float> @llvm.mips.fdiv.w(<4 x float> %0, <4 x float> %1)
+  store <4 x float> %2, <4 x float>* %c, align 16
+  ret void
+}
+
+declare <2 x double> @llvm.mips.fdiv.d(<2 x double>, <2 x double>)
+define void @fdiv_v2f64_builtin(<2 x double>* %a, <2 x double>* %b, <2 x double>* %c) {
+; P5600-LABEL: fdiv_v2f64_builtin:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.d $w0, 0($4)
+; P5600-NEXT:    ld.d $w1, 0($5)
+; P5600-NEXT:    fdiv.d $w0, $w0, $w1
+; P5600-NEXT:    st.d $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <2 x double>, <2 x double>* %a, align 16
+  %1 = load <2 x double>, <2 x double>* %b, align 16
+  %2 = tail call <2 x double> @llvm.mips.fdiv.d(<2 x double> %0, <2 x double> %1)
+  store <2 x double> %2, <2 x double>* %c, align 16
+  ret void
+}
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/fptosi_and_fptoui.ll b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/fptosi_and_fptoui.ll
index f00e3c1b4..c63f24ea6 100644
--- a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/fptosi_and_fptoui.ll
+++ b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/fptosi_and_fptoui.ll
@@ -136,6 +136,80 @@ entry:
   ret i64 %conv
 }
 
+define i32 @f32tou32(float %a) {
+; MIPS32-LABEL: f32tou32:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    trunc.w.s $f0, $f12
+; MIPS32-NEXT:    mfc1 $1, $f0
+; MIPS32-NEXT:    lui $2, 20224
+; MIPS32-NEXT:    mtc1 $2, $f0
+; MIPS32-NEXT:    sub.s $f1, $f12, $f0
+; MIPS32-NEXT:    trunc.w.s $f1, $f1
+; MIPS32-NEXT:    mfc1 $2, $f1
+; MIPS32-NEXT:    lui $3, 32768
+; MIPS32-NEXT:    xor $2, $2, $3
+; MIPS32-NEXT:    addiu $3, $zero, 1
+; MIPS32-NEXT:    c.ult.s $f12, $f0
+; MIPS32-NEXT:    movf $3, $zero, $fcc0
+; MIPS32-NEXT:    andi $3, $3, 1
+; MIPS32-NEXT:    movn $2, $1, $3
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %conv = fptoui float %a to i32
+  ret i32 %conv
+}
+
+define zeroext i16 @f32tou16(float %a) {
+; MIPS32-LABEL: f32tou16:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    trunc.w.s $f0, $f12
+; MIPS32-NEXT:    mfc1 $1, $f0
+; MIPS32-NEXT:    lui $2, 20224
+; MIPS32-NEXT:    mtc1 $2, $f0
+; MIPS32-NEXT:    sub.s $f1, $f12, $f0
+; MIPS32-NEXT:    trunc.w.s $f1, $f1
+; MIPS32-NEXT:    mfc1 $2, $f1
+; MIPS32-NEXT:    lui $3, 32768
+; MIPS32-NEXT:    xor $2, $2, $3
+; MIPS32-NEXT:    addiu $3, $zero, 1
+; MIPS32-NEXT:    c.ult.s $f12, $f0
+; MIPS32-NEXT:    movf $3, $zero, $fcc0
+; MIPS32-NEXT:    andi $3, $3, 1
+; MIPS32-NEXT:    movn $2, $1, $3
+; MIPS32-NEXT:    andi $2, $2, 65535
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %conv = fptoui float %a to i16
+  ret i16 %conv
+}
+
+define zeroext i8 @f32tou8(float %a) {
+; MIPS32-LABEL: f32tou8:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    trunc.w.s $f0, $f12
+; MIPS32-NEXT:    mfc1 $1, $f0
+; MIPS32-NEXT:    lui $2, 20224
+; MIPS32-NEXT:    mtc1 $2, $f0
+; MIPS32-NEXT:    sub.s $f1, $f12, $f0
+; MIPS32-NEXT:    trunc.w.s $f1, $f1
+; MIPS32-NEXT:    mfc1 $2, $f1
+; MIPS32-NEXT:    lui $3, 32768
+; MIPS32-NEXT:    xor $2, $2, $3
+; MIPS32-NEXT:    addiu $3, $zero, 1
+; MIPS32-NEXT:    c.ult.s $f12, $f0
+; MIPS32-NEXT:    movf $3, $zero, $fcc0
+; MIPS32-NEXT:    andi $3, $3, 1
+; MIPS32-NEXT:    movn $2, $1, $3
+; MIPS32-NEXT:    andi $2, $2, 255
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %conv = fptoui float %a to i8
+  ret i8 %conv
+}
+
 define i64 @f64tou64(double %a) {
 ; MIPS32-LABEL: f64tou64:
 ; MIPS32:       # %bb.0: # %entry
@@ -153,3 +227,148 @@ entry:
   %conv = fptoui double %a to i64
   ret i64 %conv
 }
+
+define i32 @f64tou32(double %a) {
+; FP32-LABEL: f64tou32:
+; FP32:       # %bb.0: # %entry
+; FP32-NEXT:    trunc.w.d $f0, $f12
+; FP32-NEXT:    mfc1 $1, $f0
+; FP32-NEXT:    lui $2, 16864
+; FP32-NEXT:    ori $3, $zero, 0
+; FP32-NEXT:    mtc1 $3, $f2
+; FP32-NEXT:    mtc1 $2, $f3
+; FP32-NEXT:    sub.d $f4, $f12, $f2
+; FP32-NEXT:    trunc.w.d $f0, $f4
+; FP32-NEXT:    mfc1 $2, $f0
+; FP32-NEXT:    lui $3, 32768
+; FP32-NEXT:    xor $2, $2, $3
+; FP32-NEXT:    addiu $3, $zero, 1
+; FP32-NEXT:    c.ult.d $f12, $f2
+; FP32-NEXT:    movf $3, $zero, $fcc0
+; FP32-NEXT:    andi $3, $3, 1
+; FP32-NEXT:    movn $2, $1, $3
+; FP32-NEXT:    jr $ra
+; FP32-NEXT:    nop
+;
+; FP64-LABEL: f64tou32:
+; FP64:       # %bb.0: # %entry
+; FP64-NEXT:    trunc.w.d $f0, $f12
+; FP64-NEXT:    mfc1 $1, $f0
+; FP64-NEXT:    lui $2, 16864
+; FP64-NEXT:    ori $3, $zero, 0
+; FP64-NEXT:    mtc1 $3, $f1
+; FP64-NEXT:    mthc1 $2, $f1
+; FP64-NEXT:    sub.d $f2, $f12, $f1
+; FP64-NEXT:    trunc.w.d $f0, $f2
+; FP64-NEXT:    mfc1 $2, $f0
+; FP64-NEXT:    lui $3, 32768
+; FP64-NEXT:    xor $2, $2, $3
+; FP64-NEXT:    addiu $3, $zero, 1
+; FP64-NEXT:    c.ult.d $f12, $f1
+; FP64-NEXT:    movf $3, $zero, $fcc0
+; FP64-NEXT:    andi $3, $3, 1
+; FP64-NEXT:    movn $2, $1, $3
+; FP64-NEXT:    jr $ra
+; FP64-NEXT:    nop
+entry:
+  %conv = fptoui double %a to i32
+  ret i32 %conv
+}
+
+define zeroext i16 @f64tou16(double %a) {
+; FP32-LABEL: f64tou16:
+; FP32:       # %bb.0: # %entry
+; FP32-NEXT:    trunc.w.d $f0, $f12
+; FP32-NEXT:    mfc1 $1, $f0
+; FP32-NEXT:    lui $2, 16864
+; FP32-NEXT:    ori $3, $zero, 0
+; FP32-NEXT:    mtc1 $3, $f2
+; FP32-NEXT:    mtc1 $2, $f3
+; FP32-NEXT:    sub.d $f4, $f12, $f2
+; FP32-NEXT:    trunc.w.d $f0, $f4
+; FP32-NEXT:    mfc1 $2, $f0
+; FP32-NEXT:    lui $3, 32768
+; FP32-NEXT:    xor $2, $2, $3
+; FP32-NEXT:    addiu $3, $zero, 1
+; FP32-NEXT:    c.ult.d $f12, $f2
+; FP32-NEXT:    movf $3, $zero, $fcc0
+; FP32-NEXT:    andi $3, $3, 1
+; FP32-NEXT:    movn $2, $1, $3
+; FP32-NEXT:    andi $2, $2, 65535
+; FP32-NEXT:    jr $ra
+; FP32-NEXT:    nop
+;
+; FP64-LABEL: f64tou16:
+; FP64:       # %bb.0: # %entry
+; FP64-NEXT:    trunc.w.d $f0, $f12
+; FP64-NEXT:    mfc1 $1, $f0
+; FP64-NEXT:    lui $2, 16864
+; FP64-NEXT:    ori $3, $zero, 0
+; FP64-NEXT:    mtc1 $3, $f1
+; FP64-NEXT:    mthc1 $2, $f1
+; FP64-NEXT:    sub.d $f2, $f12, $f1
+; FP64-NEXT:    trunc.w.d $f0, $f2
+; FP64-NEXT:    mfc1 $2, $f0
+; FP64-NEXT:    lui $3, 32768
+; FP64-NEXT:    xor $2, $2, $3
+; FP64-NEXT:    addiu $3, $zero, 1
+; FP64-NEXT:    c.ult.d $f12, $f1
+; FP64-NEXT:    movf $3, $zero, $fcc0
+; FP64-NEXT:    andi $3, $3, 1
+; FP64-NEXT:    movn $2, $1, $3
+; FP64-NEXT:    andi $2, $2, 65535
+; FP64-NEXT:    jr $ra
+; FP64-NEXT:    nop
+entry:
+  %conv = fptoui double %a to i16
+  ret i16 %conv
+}
+
+define zeroext i8 @f64tou8(double %a) {
+; FP32-LABEL: f64tou8:
+; FP32:       # %bb.0: # %entry
+; FP32-NEXT:    trunc.w.d $f0, $f12
+; FP32-NEXT:    mfc1 $1, $f0
+; FP32-NEXT:    lui $2, 16864
+; FP32-NEXT:    ori $3, $zero, 0
+; FP32-NEXT:    mtc1 $3, $f2
+; FP32-NEXT:    mtc1 $2, $f3
+; FP32-NEXT:    sub.d $f4, $f12, $f2
+; FP32-NEXT:    trunc.w.d $f0, $f4
+; FP32-NEXT:    mfc1 $2, $f0
+; FP32-NEXT:    lui $3, 32768
+; FP32-NEXT:    xor $2, $2, $3
+; FP32-NEXT:    addiu $3, $zero, 1
+; FP32-NEXT:    c.ult.d $f12, $f2
+; FP32-NEXT:    movf $3, $zero, $fcc0
+; FP32-NEXT:    andi $3, $3, 1
+; FP32-NEXT:    movn $2, $1, $3
+; FP32-NEXT:    andi $2, $2, 255
+; FP32-NEXT:    jr $ra
+; FP32-NEXT:    nop
+;
+; FP64-LABEL: f64tou8:
+; FP64:       # %bb.0: # %entry
+; FP64-NEXT:    trunc.w.d $f0, $f12
+; FP64-NEXT:    mfc1 $1, $f0
+; FP64-NEXT:    lui $2, 16864
+; FP64-NEXT:    ori $3, $zero, 0
+; FP64-NEXT:    mtc1 $3, $f1
+; FP64-NEXT:    mthc1 $2, $f1
+; FP64-NEXT:    sub.d $f2, $f12, $f1
+; FP64-NEXT:    trunc.w.d $f0, $f2
+; FP64-NEXT:    mfc1 $2, $f0
+; FP64-NEXT:    lui $3, 32768
+; FP64-NEXT:    xor $2, $2, $3
+; FP64-NEXT:    addiu $3, $zero, 1
+; FP64-NEXT:    c.ult.d $f12, $f1
+; FP64-NEXT:    movf $3, $zero, $fcc0
+; FP64-NEXT:    andi $3, $3, 1
+; FP64-NEXT:    movn $2, $1, $3
+; FP64-NEXT:    andi $2, $2, 255
+; FP64-NEXT:    jr $ra
+; FP64-NEXT:    nop
+entry:
+  %conv = fptoui double %a to i8
+  ret i8 %conv
+}
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/fsqrt_vec.ll b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/fsqrt_vec.ll
new file mode 100644
index 000000000..27bbb9aa5
--- /dev/null
+++ b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/fsqrt_vec.ll
@@ -0,0 +1,34 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -O0 -mtriple=mipsel-linux-gnu -global-isel -mcpu=mips32r5 -mattr=+msa,+fp64,+nan2008 -verify-machineinstrs %s -o -| FileCheck %s -check-prefixes=P5600
+
+declare <4 x float>  @llvm.sqrt.v4f32(<4 x float>  %Val)
+define void @sqrt_v4f32(<4 x float>* %a, <4 x float>* %c) {
+; P5600-LABEL: sqrt_v4f32:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.w $w0, 0($4)
+; P5600-NEXT:    fsqrt.w $w0, $w0
+; P5600-NEXT:    st.w $w0, 0($5)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <4 x float>, <4 x float>* %a, align 16
+  %sqrt = call <4 x float> @llvm.sqrt.v4f32 (<4 x float> %0)
+  store <4 x float> %sqrt, <4 x float>* %c, align 16
+  ret void
+}
+
+declare <2 x double> @llvm.sqrt.v2f64(<2 x double> %Val)
+define void @sqrt_v2f64(<2 x double>* %a, <2 x double>* %c) {
+; P5600-LABEL: sqrt_v2f64:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.d $w0, 0($4)
+; P5600-NEXT:    fsqrt.d $w0, $w0
+; P5600-NEXT:    st.d $w0, 0($5)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <2 x double>, <2 x double>* %a, align 16
+  %sqrt = call <2 x double> @llvm.sqrt.v2f64 (<2 x double> %0)
+  store <2 x double> %sqrt, <2 x double>* %c, align 16
+  ret void
+}
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/fsqrt_vec_builtin.ll b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/fsqrt_vec_builtin.ll
new file mode 100644
index 000000000..a765591d4
--- /dev/null
+++ b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/fsqrt_vec_builtin.ll
@@ -0,0 +1,35 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -O0 -mtriple=mipsel-linux-gnu -global-isel -mcpu=mips32r5 -mattr=+msa,+fp64,+nan2008 -verify-machineinstrs %s -o -| FileCheck %s -check-prefixes=P5600
+
+declare <4 x float> @llvm.mips.fsqrt.w(<4 x float>)
+define void @fsqrt_v4f32_builtin(<4 x float>* %a, <4 x float>* %c) {
+; P5600-LABEL: fsqrt_v4f32_builtin:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.w $w0, 0($4)
+; P5600-NEXT:    fsqrt.w $w0, $w0
+; P5600-NEXT:    st.w $w0, 0($5)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <4 x float>, <4 x float>* %a, align 16
+  %1 = tail call <4 x float> @llvm.mips.fsqrt.w(<4 x float> %0)
+  store <4 x float> %1, <4 x float>* %c, align 16
+  ret void
+}
+
+declare <2 x double> @llvm.mips.fsqrt.d(<2 x double>)
+define void @fsqrt_v2f64_builtin(<2 x double>* %a, <2 x double>* %c) {
+; P5600-LABEL: fsqrt_v2f64_builtin:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.d $w0, 0($4)
+; P5600-NEXT:    fsqrt.d $w0, $w0
+; P5600-NEXT:    st.d $w0, 0($5)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <2 x double>, <2 x double>* %a, align 16
+  %1 = tail call <2 x double> @llvm.mips.fsqrt.d(<2 x double> %0)
+  store <2 x double> %1, <2 x double>* %c, align 16
+  ret void
+}
+
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/icmp.ll b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/icmp.ll
index bb098761f..7eb952b47 100644
--- a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/icmp.ll
+++ b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/icmp.ll
@@ -132,10 +132,9 @@ entry:
 define i1 @ult_i8(i8 %a, i8 %b) {
 ; MIPS32-LABEL: ult_i8:
 ; MIPS32:       # %bb.0: # %entry
-; MIPS32-NEXT:    ori $1, $zero, 255
-; MIPS32-NEXT:    and $2, $4, $1
-; MIPS32-NEXT:    and $1, $5, $1
-; MIPS32-NEXT:    sltu $2, $2, $1
+; MIPS32-NEXT:    andi $1, $4, 255
+; MIPS32-NEXT:    andi $2, $5, 255
+; MIPS32-NEXT:    sltu $2, $1, $2
 ; MIPS32-NEXT:    jr $ra
 ; MIPS32-NEXT:    nop
 entry:
@@ -164,8 +163,6 @@ define i1 @eq_i64(i64 %a, i64 %b){
 ; MIPS32-NEXT:    xor $1, $4, $6
 ; MIPS32-NEXT:    xor $2, $5, $7
 ; MIPS32-NEXT:    or $1, $1, $2
-; MIPS32-NEXT:    ori $2, $zero, 0
-; MIPS32-NEXT:    xor $1, $1, $2
 ; MIPS32-NEXT:    sltiu $2, $1, 1
 ; MIPS32-NEXT:    jr $ra
 ; MIPS32-NEXT:    nop
@@ -180,8 +177,6 @@ define i1 @ne_i64(i64 %a, i64 %b) {
 ; MIPS32-NEXT:    xor $1, $4, $6
 ; MIPS32-NEXT:    xor $2, $5, $7
 ; MIPS32-NEXT:    or $1, $1, $2
-; MIPS32-NEXT:    ori $2, $zero, 0
-; MIPS32-NEXT:    xor $1, $1, $2
 ; MIPS32-NEXT:    sltu $2, $zero, $1
 ; MIPS32-NEXT:    jr $ra
 ; MIPS32-NEXT:    nop
@@ -197,8 +192,7 @@ define i1 @sgt_i64(i64 %a, i64 %b) {
 ; MIPS32-NEXT:    xor $2, $5, $7
 ; MIPS32-NEXT:    sltiu $2, $2, 1
 ; MIPS32-NEXT:    sltu $3, $6, $4
-; MIPS32-NEXT:    ori $4, $zero, 1
-; MIPS32-NEXT:    and $2, $2, $4
+; MIPS32-NEXT:    andi $2, $2, 1
 ; MIPS32-NEXT:    movn $1, $3, $2
 ; MIPS32-NEXT:    move $2, $1
 ; MIPS32-NEXT:    jr $ra
@@ -217,8 +211,7 @@ define i1 @sge_i64(i64 %a, i64 %b) {
 ; MIPS32-NEXT:    sltiu $2, $2, 1
 ; MIPS32-NEXT:    sltu $3, $4, $6
 ; MIPS32-NEXT:    xori $3, $3, 1
-; MIPS32-NEXT:    ori $4, $zero, 1
-; MIPS32-NEXT:    and $2, $2, $4
+; MIPS32-NEXT:    andi $2, $2, 1
 ; MIPS32-NEXT:    movn $1, $3, $2
 ; MIPS32-NEXT:    move $2, $1
 ; MIPS32-NEXT:    jr $ra
@@ -235,8 +228,7 @@ define i1 @slt_i64(i64 %a, i64 %b) {
 ; MIPS32-NEXT:    xor $2, $5, $7
 ; MIPS32-NEXT:    sltiu $2, $2, 1
 ; MIPS32-NEXT:    sltu $3, $4, $6
-; MIPS32-NEXT:    ori $4, $zero, 1
-; MIPS32-NEXT:    and $2, $2, $4
+; MIPS32-NEXT:    andi $2, $2, 1
 ; MIPS32-NEXT:    movn $1, $3, $2
 ; MIPS32-NEXT:    move $2, $1
 ; MIPS32-NEXT:    jr $ra
@@ -255,8 +247,7 @@ define i1 @sle_i64(i64 %a, i64 %b) {
 ; MIPS32-NEXT:    sltiu $2, $2, 1
 ; MIPS32-NEXT:    sltu $3, $6, $4
 ; MIPS32-NEXT:    xori $3, $3, 1
-; MIPS32-NEXT:    ori $4, $zero, 1
-; MIPS32-NEXT:    and $2, $2, $4
+; MIPS32-NEXT:    andi $2, $2, 1
 ; MIPS32-NEXT:    movn $1, $3, $2
 ; MIPS32-NEXT:    move $2, $1
 ; MIPS32-NEXT:    jr $ra
@@ -273,8 +264,7 @@ define i1 @ugt_i64(i64 %a, i64 %b) {
 ; MIPS32-NEXT:    xor $2, $5, $7
 ; MIPS32-NEXT:    sltiu $2, $2, 1
 ; MIPS32-NEXT:    sltu $3, $6, $4
-; MIPS32-NEXT:    ori $4, $zero, 1
-; MIPS32-NEXT:    and $2, $2, $4
+; MIPS32-NEXT:    andi $2, $2, 1
 ; MIPS32-NEXT:    movn $1, $3, $2
 ; MIPS32-NEXT:    move $2, $1
 ; MIPS32-NEXT:    jr $ra
@@ -293,8 +283,7 @@ define i1 @uge_i64(i64 %a, i64 %b) {
 ; MIPS32-NEXT:    sltiu $2, $2, 1
 ; MIPS32-NEXT:    sltu $3, $4, $6
 ; MIPS32-NEXT:    xori $3, $3, 1
-; MIPS32-NEXT:    ori $4, $zero, 1
-; MIPS32-NEXT:    and $2, $2, $4
+; MIPS32-NEXT:    andi $2, $2, 1
 ; MIPS32-NEXT:    movn $1, $3, $2
 ; MIPS32-NEXT:    move $2, $1
 ; MIPS32-NEXT:    jr $ra
@@ -311,8 +300,7 @@ define i1 @ult_i64(i64 %a, i64 %b) {
 ; MIPS32-NEXT:    xor $2, $5, $7
 ; MIPS32-NEXT:    sltiu $2, $2, 1
 ; MIPS32-NEXT:    sltu $3, $4, $6
-; MIPS32-NEXT:    ori $4, $zero, 1
-; MIPS32-NEXT:    and $2, $2, $4
+; MIPS32-NEXT:    andi $2, $2, 1
 ; MIPS32-NEXT:    movn $1, $3, $2
 ; MIPS32-NEXT:    move $2, $1
 ; MIPS32-NEXT:    jr $ra
@@ -331,8 +319,7 @@ define i1 @ule_i64(i64 %a, i64 %b) {
 ; MIPS32-NEXT:    sltiu $2, $2, 1
 ; MIPS32-NEXT:    sltu $3, $6, $4
 ; MIPS32-NEXT:    xori $3, $3, 1
-; MIPS32-NEXT:    ori $4, $zero, 1
-; MIPS32-NEXT:    and $2, $2, $4
+; MIPS32-NEXT:    andi $2, $2, 1
 ; MIPS32-NEXT:    movn $1, $3, $2
 ; MIPS32-NEXT:    move $2, $1
 ; MIPS32-NEXT:    jr $ra
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/implicit_def.ll b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/implicit_def.ll
new file mode 100644
index 000000000..7c94a5b0b
--- /dev/null
+++ b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/implicit_def.ll
@@ -0,0 +1,83 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc  -O0 -mtriple=mipsel-linux-gnu -global-isel  -verify-machineinstrs %s -o -| FileCheck %s -check-prefixes=MIPS32
+
+declare void @f_i32(i32)
+define void @g_i32() {
+; MIPS32-LABEL: g_i32:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    addiu $sp, $sp, -24
+; MIPS32-NEXT:    .cfi_def_cfa_offset 24
+; MIPS32-NEXT:    sw $ra, 20($sp) # 4-byte Folded Spill
+; MIPS32-NEXT:    .cfi_offset 31, -4
+; MIPS32-NEXT:    # implicit-def: $a0
+; MIPS32-NEXT:    jal f_i32
+; MIPS32-NEXT:    nop
+; MIPS32-NEXT:    lw $ra, 20($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    addiu $sp, $sp, 24
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  call void @f_i32(i32 undef)
+  ret void
+}
+
+declare void @f_i64(i64)
+define void @g_i64() {
+; MIPS32-LABEL: g_i64:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    addiu $sp, $sp, -24
+; MIPS32-NEXT:    .cfi_def_cfa_offset 24
+; MIPS32-NEXT:    sw $ra, 20($sp) # 4-byte Folded Spill
+; MIPS32-NEXT:    .cfi_offset 31, -4
+; MIPS32-NEXT:    # implicit-def: $a0
+; MIPS32-NEXT:    # implicit-def: $a1
+; MIPS32-NEXT:    jal f_i64
+; MIPS32-NEXT:    nop
+; MIPS32-NEXT:    lw $ra, 20($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    addiu $sp, $sp, 24
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  call void @f_i64(i64 undef)
+  ret void
+}
+
+declare void @f_float(float)
+define void @g_float() {
+; MIPS32-LABEL: g_float:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    addiu $sp, $sp, -24
+; MIPS32-NEXT:    .cfi_def_cfa_offset 24
+; MIPS32-NEXT:    sw $ra, 20($sp) # 4-byte Folded Spill
+; MIPS32-NEXT:    .cfi_offset 31, -4
+; MIPS32-NEXT:    # implicit-def: $f12
+; MIPS32-NEXT:    jal f_float
+; MIPS32-NEXT:    nop
+; MIPS32-NEXT:    lw $ra, 20($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    addiu $sp, $sp, 24
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  call void @f_float(float undef)
+  ret void
+}
+
+declare void @f_double(double)
+define void @g_double() {
+; MIPS32-LABEL: g_double:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    addiu $sp, $sp, -24
+; MIPS32-NEXT:    .cfi_def_cfa_offset 24
+; MIPS32-NEXT:    sw $ra, 20($sp) # 4-byte Folded Spill
+; MIPS32-NEXT:    .cfi_offset 31, -4
+; MIPS32-NEXT:    # implicit-def: $d6
+; MIPS32-NEXT:    jal f_double
+; MIPS32-NEXT:    nop
+; MIPS32-NEXT:    lw $ra, 20($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    addiu $sp, $sp, 24
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  call void @f_double(double undef)
+  ret void
+}
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/inttoptr_and_ptrtoint.ll b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/inttoptr_and_ptrtoint.ll
new file mode 100644
index 000000000..c27b5a939
--- /dev/null
+++ b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/inttoptr_and_ptrtoint.ll
@@ -0,0 +1,24 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc  -O0 -mtriple=mipsel-linux-gnu -global-isel  -verify-machineinstrs %s -o -| FileCheck %s -check-prefixes=MIPS32
+
+define i32* @inttoptr(i32 %a) {
+; MIPS32-LABEL: inttoptr:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    move $2, $4
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %0 = inttoptr i32 %a to i32*
+  ret i32* %0
+}
+
+define i32 @ptrtoint(i32* %a) {
+; MIPS32-LABEL: ptrtoint:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    move $2, $4
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %0 = ptrtoint i32* %a to i32
+  ret i32 %0
+}
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/jump_table_and_brjt.ll b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/jump_table_and_brjt.ll
new file mode 100644
index 000000000..dcd6c76a8
--- /dev/null
+++ b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/jump_table_and_brjt.ll
@@ -0,0 +1,287 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc  -O0 -mtriple=mipsel-linux-gnu -global-isel -verify-machineinstrs %s -o -| FileCheck %s -check-prefixes=MIPS32
+; RUN: llc  -O0 -mtriple=mipsel-linux-gnu -global-isel -relocation-model=pic -verify-machineinstrs %s -o -| FileCheck %s -check-prefixes=MIPS32_PIC
+
+define i32 @mod4_0_to_11(i32 %a) {
+; MIPS32-LABEL: mod4_0_to_11:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    addiu $sp, $sp, -32
+; MIPS32-NEXT:    .cfi_def_cfa_offset 32
+; MIPS32-NEXT:    ori $1, $zero, 7
+; MIPS32-NEXT:    ori $2, $zero, 3
+; MIPS32-NEXT:    ori $3, $zero, 2
+; MIPS32-NEXT:    ori $5, $zero, 1
+; MIPS32-NEXT:    ori $6, $zero, 0
+; MIPS32-NEXT:    addiu $7, $zero, 65535
+; MIPS32-NEXT:    ori $8, $zero, 0
+; MIPS32-NEXT:    subu $8, $4, $8
+; MIPS32-NEXT:    sltu $1, $1, $8
+; MIPS32-NEXT:    andi $1, $1, 1
+; MIPS32-NEXT:    sw $4, 28($sp) # 4-byte Folded Spill
+; MIPS32-NEXT:    sw $2, 24($sp) # 4-byte Folded Spill
+; MIPS32-NEXT:    sw $3, 20($sp) # 4-byte Folded Spill
+; MIPS32-NEXT:    sw $5, 16($sp) # 4-byte Folded Spill
+; MIPS32-NEXT:    sw $6, 12($sp) # 4-byte Folded Spill
+; MIPS32-NEXT:    sw $7, 8($sp) # 4-byte Folded Spill
+; MIPS32-NEXT:    sw $8, 4($sp) # 4-byte Folded Spill
+; MIPS32-NEXT:    bnez $1, $BB0_6
+; MIPS32-NEXT:    nop
+; MIPS32-NEXT:  $BB0_1: # %entry
+; MIPS32-NEXT:    lui $1, %hi($JTI0_0)
+; MIPS32-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    sll $3, $2, 2
+; MIPS32-NEXT:    addu $1, $1, $3
+; MIPS32-NEXT:    lw $1, %lo($JTI0_0)($1)
+; MIPS32-NEXT:    jr $1
+; MIPS32-NEXT:    nop
+; MIPS32-NEXT:  $BB0_2: # %sw.bb
+; MIPS32-NEXT:    lw $2, 12($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    addiu $sp, $sp, 32
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+; MIPS32-NEXT:  $BB0_3: # %sw.bb1
+; MIPS32-NEXT:    lw $2, 16($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    addiu $sp, $sp, 32
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+; MIPS32-NEXT:  $BB0_4: # %sw.bb2
+; MIPS32-NEXT:    lw $2, 20($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    addiu $sp, $sp, 32
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+; MIPS32-NEXT:  $BB0_5: # %sw.bb3
+; MIPS32-NEXT:    lw $2, 24($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    addiu $sp, $sp, 32
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+; MIPS32-NEXT:  $BB0_6: # %sw.default
+; MIPS32-NEXT:    .insn
+; MIPS32-NEXT:  # %bb.7: # %sw.epilog
+; MIPS32-NEXT:    ori $1, $zero, 8
+; MIPS32-NEXT:    lw $2, 28($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    subu $1, $2, $1
+; MIPS32-NEXT:    lw $3, 24($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    sltu $4, $3, $1
+; MIPS32-NEXT:    andi $4, $4, 1
+; MIPS32-NEXT:    sw $1, 0($sp) # 4-byte Folded Spill
+; MIPS32-NEXT:    bnez $4, $BB0_13
+; MIPS32-NEXT:    nop
+; MIPS32-NEXT:  $BB0_8: # %sw.epilog
+; MIPS32-NEXT:    lui $1, %hi($JTI0_1)
+; MIPS32-NEXT:    lw $2, 0($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    sll $3, $2, 2
+; MIPS32-NEXT:    addu $1, $1, $3
+; MIPS32-NEXT:    lw $1, %lo($JTI0_1)($1)
+; MIPS32-NEXT:    jr $1
+; MIPS32-NEXT:    nop
+; MIPS32-NEXT:  $BB0_9: # %sw.bb4
+; MIPS32-NEXT:    lw $2, 12($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    addiu $sp, $sp, 32
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+; MIPS32-NEXT:  $BB0_10: # %sw.bb5
+; MIPS32-NEXT:    lw $2, 16($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    addiu $sp, $sp, 32
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+; MIPS32-NEXT:  $BB0_11: # %sw.bb6
+; MIPS32-NEXT:    lw $2, 20($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    addiu $sp, $sp, 32
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+; MIPS32-NEXT:  $BB0_12: # %sw.bb7
+; MIPS32-NEXT:    lw $2, 24($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    addiu $sp, $sp, 32
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+; MIPS32-NEXT:  $BB0_13: # %sw.default8
+; MIPS32-NEXT:    lw $2, 8($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    addiu $sp, $sp, 32
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+; MIPS32:       $JTI0_0:
+; MIPS32-NEXT:   .4byte ($BB0_2)
+; MIPS32-NEXT:   .4byte ($BB0_3)
+; MIPS32-NEXT:   .4byte ($BB0_4)
+; MIPS32-NEXT:   .4byte ($BB0_5)
+; MIPS32-NEXT:   .4byte ($BB0_2)
+; MIPS32-NEXT:   .4byte ($BB0_3)
+; MIPS32-NEXT:   .4byte ($BB0_4)
+; MIPS32-NEXT:   .4byte ($BB0_5)
+; MIPS32-NEXT:   $JTI0_1:
+; MIPS32-NEXT:   .4byte ($BB0_9)
+; MIPS32-NEXT:   .4byte ($BB0_10)
+; MIPS32-NEXT:   .4byte ($BB0_11)
+; MIPS32-NEXT:   .4byte ($BB0_12)
+
+;
+; MIPS32_PIC-LABEL: mod4_0_to_11:
+; MIPS32_PIC:       # %bb.0: # %entry
+; MIPS32_PIC-NEXT:    lui $2, %hi(_gp_disp)
+; MIPS32_PIC-NEXT:    addiu $2, $2, %lo(_gp_disp)
+; MIPS32_PIC-NEXT:    addiu $sp, $sp, -40
+; MIPS32_PIC-NEXT:    .cfi_def_cfa_offset 40
+; MIPS32_PIC-NEXT:    addu $1, $2, $25
+; MIPS32_PIC-NEXT:    ori $2, $zero, 7
+; MIPS32_PIC-NEXT:    ori $3, $zero, 3
+; MIPS32_PIC-NEXT:    ori $5, $zero, 2
+; MIPS32_PIC-NEXT:    ori $6, $zero, 1
+; MIPS32_PIC-NEXT:    ori $7, $zero, 0
+; MIPS32_PIC-NEXT:    addiu $8, $zero, 65535
+; MIPS32_PIC-NEXT:    ori $9, $zero, 0
+; MIPS32_PIC-NEXT:    subu $9, $4, $9
+; MIPS32_PIC-NEXT:    sltu $2, $2, $9
+; MIPS32_PIC-NEXT:    andi $2, $2, 1
+; MIPS32_PIC-NEXT:    sw $1, 36($sp) # 4-byte Folded Spill
+; MIPS32_PIC-NEXT:    sw $4, 32($sp) # 4-byte Folded Spill
+; MIPS32_PIC-NEXT:    sw $3, 28($sp) # 4-byte Folded Spill
+; MIPS32_PIC-NEXT:    sw $5, 24($sp) # 4-byte Folded Spill
+; MIPS32_PIC-NEXT:    sw $6, 20($sp) # 4-byte Folded Spill
+; MIPS32_PIC-NEXT:    sw $7, 16($sp) # 4-byte Folded Spill
+; MIPS32_PIC-NEXT:    sw $8, 12($sp) # 4-byte Folded Spill
+; MIPS32_PIC-NEXT:    sw $9, 8($sp) # 4-byte Folded Spill
+; MIPS32_PIC-NEXT:    bnez $2, $BB0_6
+; MIPS32_PIC-NEXT:    nop
+; MIPS32_PIC-NEXT:  $BB0_1: # %entry
+; MIPS32_PIC-NEXT:    lw $1, 36($sp) # 4-byte Folded Reload
+; MIPS32_PIC-NEXT:    lw $2, %got($JTI0_0)($1)
+; MIPS32_PIC-NEXT:    lw $3, 8($sp) # 4-byte Folded Reload
+; MIPS32_PIC-NEXT:    sll $4, $3, 2
+; MIPS32_PIC-NEXT:    addu $2, $2, $4
+; MIPS32_PIC-NEXT:    lw $2, %lo($JTI0_0)($2)
+; MIPS32_PIC-NEXT:    addu $2, $2, $1
+; MIPS32_PIC-NEXT:    jr $2
+; MIPS32_PIC-NEXT:    nop
+; MIPS32_PIC-NEXT:  $BB0_2: # %sw.bb
+; MIPS32_PIC-NEXT:    lw $2, 16($sp) # 4-byte Folded Reload
+; MIPS32_PIC-NEXT:    addiu $sp, $sp, 40
+; MIPS32_PIC-NEXT:    jr $ra
+; MIPS32_PIC-NEXT:    nop
+; MIPS32_PIC-NEXT:  $BB0_3: # %sw.bb1
+; MIPS32_PIC-NEXT:    lw $2, 20($sp) # 4-byte Folded Reload
+; MIPS32_PIC-NEXT:    addiu $sp, $sp, 40
+; MIPS32_PIC-NEXT:    jr $ra
+; MIPS32_PIC-NEXT:    nop
+; MIPS32_PIC-NEXT:  $BB0_4: # %sw.bb2
+; MIPS32_PIC-NEXT:    lw $2, 24($sp) # 4-byte Folded Reload
+; MIPS32_PIC-NEXT:    addiu $sp, $sp, 40
+; MIPS32_PIC-NEXT:    jr $ra
+; MIPS32_PIC-NEXT:    nop
+; MIPS32_PIC-NEXT:  $BB0_5: # %sw.bb3
+; MIPS32_PIC-NEXT:    lw $2, 28($sp) # 4-byte Folded Reload
+; MIPS32_PIC-NEXT:    addiu $sp, $sp, 40
+; MIPS32_PIC-NEXT:    jr $ra
+; MIPS32_PIC-NEXT:    nop
+; MIPS32_PIC-NEXT:  $BB0_6: # %sw.default
+; MIPS32_PIC-NEXT:    .insn
+; MIPS32_PIC-NEXT:  # %bb.7: # %sw.epilog
+; MIPS32_PIC-NEXT:    ori $1, $zero, 8
+; MIPS32_PIC-NEXT:    lw $2, 32($sp) # 4-byte Folded Reload
+; MIPS32_PIC-NEXT:    subu $1, $2, $1
+; MIPS32_PIC-NEXT:    lw $3, 28($sp) # 4-byte Folded Reload
+; MIPS32_PIC-NEXT:    sltu $4, $3, $1
+; MIPS32_PIC-NEXT:    andi $4, $4, 1
+; MIPS32_PIC-NEXT:    sw $1, 4($sp) # 4-byte Folded Spill
+; MIPS32_PIC-NEXT:    bnez $4, $BB0_13
+; MIPS32_PIC-NEXT:    nop
+; MIPS32_PIC-NEXT:  $BB0_8: # %sw.epilog
+; MIPS32_PIC-NEXT:    lw $1, 36($sp) # 4-byte Folded Reload
+; MIPS32_PIC-NEXT:    lw $2, %got($JTI0_1)($1)
+; MIPS32_PIC-NEXT:    lw $3, 4($sp) # 4-byte Folded Reload
+; MIPS32_PIC-NEXT:    sll $4, $3, 2
+; MIPS32_PIC-NEXT:    addu $2, $2, $4
+; MIPS32_PIC-NEXT:    lw $2, %lo($JTI0_1)($2)
+; MIPS32_PIC-NEXT:    addu $2, $2, $1
+; MIPS32_PIC-NEXT:    jr $2
+; MIPS32_PIC-NEXT:    nop
+; MIPS32_PIC-NEXT:  $BB0_9: # %sw.bb4
+; MIPS32_PIC-NEXT:    lw $2, 16($sp) # 4-byte Folded Reload
+; MIPS32_PIC-NEXT:    addiu $sp, $sp, 40
+; MIPS32_PIC-NEXT:    jr $ra
+; MIPS32_PIC-NEXT:    nop
+; MIPS32_PIC-NEXT:  $BB0_10: # %sw.bb5
+; MIPS32_PIC-NEXT:    lw $2, 20($sp) # 4-byte Folded Reload
+; MIPS32_PIC-NEXT:    addiu $sp, $sp, 40
+; MIPS32_PIC-NEXT:    jr $ra
+; MIPS32_PIC-NEXT:    nop
+; MIPS32_PIC-NEXT:  $BB0_11: # %sw.bb6
+; MIPS32_PIC-NEXT:    lw $2, 24($sp) # 4-byte Folded Reload
+; MIPS32_PIC-NEXT:    addiu $sp, $sp, 40
+; MIPS32_PIC-NEXT:    jr $ra
+; MIPS32_PIC-NEXT:    nop
+; MIPS32_PIC-NEXT:  $BB0_12: # %sw.bb7
+; MIPS32_PIC-NEXT:    lw $2, 28($sp) # 4-byte Folded Reload
+; MIPS32_PIC-NEXT:    addiu $sp, $sp, 40
+; MIPS32_PIC-NEXT:    jr $ra
+; MIPS32_PIC-NEXT:    nop
+; MIPS32_PIC-NEXT:  $BB0_13: # %sw.default8
+; MIPS32_PIC-NEXT:    lw $2, 12($sp) # 4-byte Folded Reload
+; MIPS32_PIC-NEXT:    addiu $sp, $sp, 40
+; MIPS32_PIC-NEXT:    jr $ra
+; MIPS32_PIC-NEXT:    nop
+; MIPS32_PIC:       $JTI0_0:
+; MIPS32_PIC-NEXT:   .gpword ($BB0_2)
+; MIPS32_PIC-NEXT:   .gpword ($BB0_3)
+; MIPS32_PIC-NEXT:   .gpword ($BB0_4)
+; MIPS32_PIC-NEXT:   .gpword ($BB0_5)
+; MIPS32_PIC-NEXT:   .gpword ($BB0_2)
+; MIPS32_PIC-NEXT:   .gpword ($BB0_3)
+; MIPS32_PIC-NEXT:   .gpword ($BB0_4)
+; MIPS32_PIC-NEXT:   .gpword ($BB0_5)
+; MIPS32_PIC-NEXT:   $JTI0_1:
+; MIPS32_PIC-NEXT:   .gpword ($BB0_9)
+; MIPS32_PIC-NEXT:   .gpword ($BB0_10)
+; MIPS32_PIC-NEXT:   .gpword ($BB0_11)
+; MIPS32_PIC-NEXT:   .gpword ($BB0_12)
+
+entry:
+  switch i32 %a, label %sw.default [
+    i32 0, label %sw.bb
+    i32 4, label %sw.bb
+    i32 1, label %sw.bb1
+    i32 5, label %sw.bb1
+    i32 2, label %sw.bb2
+    i32 6, label %sw.bb2
+    i32 3, label %sw.bb3
+    i32 7, label %sw.bb3
+  ]
+
+sw.bb:                                            ; preds = %entry, %entry
+  ret i32 0
+
+sw.bb1:                                           ; preds = %entry, %entry
+  ret i32 1
+
+sw.bb2:                                           ; preds = %entry, %entry
+  ret i32 2
+
+sw.bb3:                                           ; preds = %entry, %entry
+  ret i32 3
+
+sw.default:                                       ; preds = %entry
+  br label %sw.epilog
+
+sw.epilog:                                        ; preds = %sw.default
+  switch i32 %a, label %sw.default8 [
+    i32 8, label %sw.bb4
+    i32 9, label %sw.bb5
+    i32 10, label %sw.bb6
+    i32 11, label %sw.bb7
+  ]
+
+sw.bb4:                                           ; preds = %sw.epilog
+  ret i32 0
+
+sw.bb5:                                           ; preds = %sw.epilog
+  ret i32 1
+
+sw.bb6:                                           ; preds = %sw.epilog
+  ret i32 2
+
+sw.bb7:                                           ; preds = %sw.epilog
+  ret i32 3
+
+sw.default8:                                      ; preds = %sw.epilog
+  ret i32 -1
+
+}
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/load.ll b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/load.ll
index f970be23e..bf163ecae 100644
--- a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/load.ll
+++ b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/load.ll
@@ -16,9 +16,7 @@ define i64 @load_i64(i64* %ptr) {
 ; MIPS32-LABEL: load_i64:
 ; MIPS32:       # %bb.0: # %entry
 ; MIPS32-NEXT:    lw $2, 0($4)
-; MIPS32-NEXT:    ori $1, $zero, 4
-; MIPS32-NEXT:    addu $1, $4, $1
-; MIPS32-NEXT:    lw $3, 0($1)
+; MIPS32-NEXT:    lw $3, 4($4)
 ; MIPS32-NEXT:    jr $ra
 ; MIPS32-NEXT:    nop
 entry:
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/load_store_fold.ll b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/load_store_fold.ll
new file mode 100644
index 000000000..8eed2cc54
--- /dev/null
+++ b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/load_store_fold.ll
@@ -0,0 +1,105 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc  -O0 -mtriple=mipsel-linux-gnu -global-isel  -verify-machineinstrs %s -o -| FileCheck %s -check-prefixes=MIPS32
+
+define zeroext i8 @_16_bit_positive_offset(i8* %a) {
+; MIPS32-LABEL: _16_bit_positive_offset:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    lbu $2, 32767($4)
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %arrayidx = getelementptr inbounds i8, i8* %a, i32 32767
+  %0 = load i8, i8* %arrayidx
+  ret i8 %0
+}
+
+define void @_16_bit_negative_offset(i8 %val, i8* %a) {
+; MIPS32-LABEL: _16_bit_negative_offset:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    sb $4, -32768($5)
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %arrayidx = getelementptr inbounds i8, i8* %a, i32 -32768
+  store i8 %val, i8* %arrayidx
+  ret void
+}
+
+define void @_large_positive_offset(i8 %val, i8* %a) {
+; MIPS32-LABEL: _large_positive_offset:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    ori $1, $zero, 32768
+; MIPS32-NEXT:    addu $1, $5, $1
+; MIPS32-NEXT:    sb $4, 0($1)
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %arrayidx = getelementptr inbounds i8, i8* %a, i32 32768
+  store i8 %val, i8* %arrayidx
+  ret void
+}
+
+define signext i8 @_large_negative_offset(i8* %a) {
+; MIPS32-LABEL: _large_negative_offset:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    lui $1, 65535
+; MIPS32-NEXT:    ori $1, $1, 32767
+; MIPS32-NEXT:    addu $1, $4, $1
+; MIPS32-NEXT:    lb $2, 0($1)
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %arrayidx = getelementptr inbounds i8, i8* %a, i32 -32769
+  %0 = load i8, i8* %arrayidx
+  ret i8 %0
+}
+
+define float @fold_f32_load(float* %a) {
+; MIPS32-LABEL: fold_f32_load:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    lwc1 $f0, 40($4)
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %arrayidx = getelementptr inbounds float, float* %a, i32 10
+  %0 = load float, float* %arrayidx
+  ret float %0
+}
+
+define void @fold_f64_store(double %val, double* %a) {
+; MIPS32-LABEL: fold_f64_store:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    sdc1 $f12, -80($6)
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %arrayidx = getelementptr inbounds double, double* %a, i32 -10
+  store double %val, double* %arrayidx
+  ret void
+}
+
+define i16 @fold_i16_load(i16* %a) {
+; MIPS32-LABEL: fold_i16_load:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    lhu $2, -20($4)
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %arrayidx = getelementptr inbounds i16, i16* %a, i32 -10
+  %0 = load i16, i16* %arrayidx
+  ret i16 %0
+}
+
+define void @fold_i32_store(i32 %val, i32* %a) {
+; MIPS32-LABEL: fold_i32_store:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    sw $4, 40($5)
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %arrayidx = getelementptr inbounds i32, i32* %a, i32 10
+  store i32 %val, i32* %arrayidx
+  ret void
+}
+
+
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/load_store_vec.ll b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/load_store_vec.ll
new file mode 100644
index 000000000..6da35aa47
--- /dev/null
+++ b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/load_store_vec.ll
@@ -0,0 +1,80 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc  -O0 -mtriple=mipsel-linux-gnu -global-isel -mcpu=mips32r5 -mattr=msa,+fp64 -mattr=nan2008 -verify-machineinstrs %s -o -| FileCheck %s -check-prefixes=P5600
+
+define void @load_store_v16i8(<16 x i8>* %a, <16 x i8>* %b) {
+; P5600-LABEL: load_store_v16i8:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.b $w0, 0($5)
+; P5600-NEXT:    st.b $w0, 0($4)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <16 x i8>, <16 x i8>* %b, align 16
+  store <16 x i8> %0, <16 x i8>* %a, align 16
+  ret void
+}
+
+define void @load_store_v8i16(<8 x i16>* %a, <8 x i16>* %b) {
+; P5600-LABEL: load_store_v8i16:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.h $w0, 0($5)
+; P5600-NEXT:    st.h $w0, 0($4)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <8 x i16>, <8 x i16>* %b, align 16
+  store <8 x i16> %0, <8 x i16>* %a, align 16
+  ret void
+}
+
+define void @load_store_v4i32(<4 x i32>* %a, <4 x i32>* %b) {
+; P5600-LABEL: load_store_v4i32:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.w $w0, 0($5)
+; P5600-NEXT:    st.w $w0, 0($4)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <4 x i32>, <4 x i32>* %b, align 16
+  store <4 x i32> %0, <4 x i32>* %a, align 16
+  ret void
+}
+
+define void @load_store_v2i64(<2 x i64>* %a, <2 x i64>* %b) {
+; P5600-LABEL: load_store_v2i64:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.d $w0, 0($5)
+; P5600-NEXT:    st.d $w0, 0($4)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <2 x i64>, <2 x i64>* %b, align 16
+  store <2 x i64> %0, <2 x i64>* %a, align 16
+  ret void
+}
+
+define void @load_store_v4f32(<4 x float>* %a, <4 x float>* %b) {
+; P5600-LABEL: load_store_v4f32:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.w $w0, 0($5)
+; P5600-NEXT:    st.w $w0, 0($4)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <4 x float>, <4 x float>* %b, align 16
+  store <4 x float> %0, <4 x float>* %a, align 16
+  ret void
+}
+
+define void @load_store_v2f64(<2 x double>* %a, <2 x double>* %b) {
+; P5600-LABEL: load_store_v2f64:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.d $w0, 0($5)
+; P5600-NEXT:    st.d $w0, 0($4)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <2 x double>, <2 x double>* %b, align 16
+  store <2 x double> %0, <2 x double>* %a, align 16
+  ret void
+}
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/long_ambiguous_chain_s32.ll b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/long_ambiguous_chain_s32.ll
index 630aa1312..20e549b81 100644
--- a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/long_ambiguous_chain_s32.ll
+++ b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/long_ambiguous_chain_s32.ll
@@ -12,8 +12,7 @@ define void @long_chain_ambiguous_i32_in_gpr(i1 %cnd0, i1 %cnd1, i1 %cnd2, i32*
 ; MIPS32-NEXT:    lw $2, 0($2)
 ; MIPS32-NEXT:    addiu $3, $sp, 72
 ; MIPS32-NEXT:    lw $3, 0($3)
-; MIPS32-NEXT:    ori $8, $zero, 1
-; MIPS32-NEXT:    and $8, $4, $8
+; MIPS32-NEXT:    andi $8, $4, 1
 ; MIPS32-NEXT:    sw $1, 44($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:    sw $4, 40($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:    sw $5, 36($sp) # 4-byte Folded Spill
@@ -24,16 +23,14 @@ define void @long_chain_ambiguous_i32_in_gpr(i1 %cnd0, i1 %cnd1, i1 %cnd2, i32*
 ; MIPS32-NEXT:    bnez $8, $BB0_9
 ; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  # %bb.1: # %pre.PHI.1
-; MIPS32-NEXT:    ori $1, $zero, 1
-; MIPS32-NEXT:    lw $2, 36($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    and $1, $2, $1
-; MIPS32-NEXT:    bnez $1, $BB0_4
+; MIPS32-NEXT:    lw $1, 36($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    andi $2, $1, 1
+; MIPS32-NEXT:    bnez $2, $BB0_4
 ; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  # %bb.2: # %pre.PHI.1.0
-; MIPS32-NEXT:    ori $1, $zero, 1
-; MIPS32-NEXT:    lw $2, 32($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    and $1, $2, $1
-; MIPS32-NEXT:    bnez $1, $BB0_5
+; MIPS32-NEXT:    lw $1, 32($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    andi $2, $1, 1
+; MIPS32-NEXT:    bnez $2, $BB0_5
 ; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  # %bb.3: # %b.PHI.1.0
 ; MIPS32-NEXT:    lw $1, 28($sp) # 4-byte Folded Reload
@@ -53,13 +50,12 @@ define void @long_chain_ambiguous_i32_in_gpr(i1 %cnd0, i1 %cnd1, i1 %cnd2, i32*
 ; MIPS32-NEXT:    sw $2, 16($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:  $BB0_6: # %b.PHI.1
 ; MIPS32-NEXT:    lw $1, 16($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    ori $2, $zero, 1
-; MIPS32-NEXT:    lw $3, 32($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    and $2, $3, $2
+; MIPS32-NEXT:    lw $2, 32($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    andi $3, $2, 1
 ; MIPS32-NEXT:    move $4, $1
 ; MIPS32-NEXT:    sw $1, 12($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:    sw $4, 8($sp) # 4-byte Folded Spill
-; MIPS32-NEXT:    bnez $2, $BB0_8
+; MIPS32-NEXT:    bnez $3, $BB0_8
 ; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  # %bb.7: # %b.PHI.1
 ; MIPS32-NEXT:    j $BB0_15
@@ -72,10 +68,9 @@ define void @long_chain_ambiguous_i32_in_gpr(i1 %cnd0, i1 %cnd1, i1 %cnd2, i32*
 ; MIPS32-NEXT:    jr $ra
 ; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  $BB0_9: # %pre.PHI.2
-; MIPS32-NEXT:    ori $1, $zero, 1
-; MIPS32-NEXT:    lw $2, 40($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    and $1, $2, $1
-; MIPS32-NEXT:    bnez $1, $BB0_11
+; MIPS32-NEXT:    lw $1, 40($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    andi $2, $1, 1
+; MIPS32-NEXT:    bnez $2, $BB0_11
 ; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  # %bb.10: # %pre.PHI.2
 ; MIPS32-NEXT:    j $BB0_12
@@ -92,13 +87,12 @@ define void @long_chain_ambiguous_i32_in_gpr(i1 %cnd0, i1 %cnd1, i1 %cnd2, i32*
 ; MIPS32-NEXT:    sw $2, 4($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:  $BB0_13: # %b.PHI.2
 ; MIPS32-NEXT:    lw $1, 4($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    ori $2, $zero, 1
-; MIPS32-NEXT:    lw $3, 36($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    and $2, $3, $2
+; MIPS32-NEXT:    lw $2, 36($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    andi $3, $2, 1
 ; MIPS32-NEXT:    move $4, $1
 ; MIPS32-NEXT:    sw $1, 0($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:    sw $4, 8($sp) # 4-byte Folded Spill
-; MIPS32-NEXT:    bnez $2, $BB0_15
+; MIPS32-NEXT:    bnez $3, $BB0_15
 ; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  # %bb.14: # %b.PHI.2.end
 ; MIPS32-NEXT:    lw $1, 0($sp) # 4-byte Folded Reload
@@ -110,14 +104,13 @@ define void @long_chain_ambiguous_i32_in_gpr(i1 %cnd0, i1 %cnd1, i1 %cnd2, i32*
 ; MIPS32-NEXT:  $BB0_15: # %b.PHI.3
 ; MIPS32-NEXT:    lw $1, 8($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    lw $2, 8($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    ori $3, $zero, 1
-; MIPS32-NEXT:    lw $4, 32($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    and $5, $4, $3
-; MIPS32-NEXT:    movn $1, $2, $5
-; MIPS32-NEXT:    lw $5, 36($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    and $3, $5, $3
+; MIPS32-NEXT:    lw $3, 32($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    andi $4, $3, 1
+; MIPS32-NEXT:    movn $1, $2, $4
+; MIPS32-NEXT:    lw $4, 36($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    andi $5, $4, 1
 ; MIPS32-NEXT:    move $6, $2
-; MIPS32-NEXT:    movn $6, $1, $3
+; MIPS32-NEXT:    movn $6, $1, $5
 ; MIPS32-NEXT:    lw $1, 20($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    sw $6, 0($1)
 ; MIPS32-NEXT:    sw $2, 0($1)
@@ -195,8 +188,7 @@ define void @long_chain_i32_in_gpr(i1 %cnd0, i1 %cnd1, i1 %cnd2, i32* %a, i32* %
 ; MIPS32-NEXT:    addiu $3, $sp, 80
 ; MIPS32-NEXT:    lw $3, 0($3)
 ; MIPS32-NEXT:    ori $8, $zero, 0
-; MIPS32-NEXT:    ori $9, $zero, 1
-; MIPS32-NEXT:    and $9, $4, $9
+; MIPS32-NEXT:    andi $9, $4, 1
 ; MIPS32-NEXT:    sw $1, 52($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:    sw $4, 48($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:    sw $5, 44($sp) # 4-byte Folded Spill
@@ -208,16 +200,14 @@ define void @long_chain_i32_in_gpr(i1 %cnd0, i1 %cnd1, i1 %cnd2, i32* %a, i32* %
 ; MIPS32-NEXT:    bnez $9, $BB1_9
 ; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  # %bb.1: # %pre.PHI.1
-; MIPS32-NEXT:    ori $1, $zero, 1
-; MIPS32-NEXT:    lw $2, 44($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    and $1, $2, $1
-; MIPS32-NEXT:    bnez $1, $BB1_4
+; MIPS32-NEXT:    lw $1, 44($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    andi $2, $1, 1
+; MIPS32-NEXT:    bnez $2, $BB1_4
 ; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  # %bb.2: # %pre.PHI.1.0
-; MIPS32-NEXT:    ori $1, $zero, 1
-; MIPS32-NEXT:    lw $2, 40($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    and $1, $2, $1
-; MIPS32-NEXT:    bnez $1, $BB1_5
+; MIPS32-NEXT:    lw $1, 40($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    andi $2, $1, 1
+; MIPS32-NEXT:    bnez $2, $BB1_5
 ; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  # %bb.3: # %b.PHI.1.0
 ; MIPS32-NEXT:    lw $1, 36($sp) # 4-byte Folded Reload
@@ -237,15 +227,14 @@ define void @long_chain_i32_in_gpr(i1 %cnd0, i1 %cnd1, i1 %cnd2, i32* %a, i32* %
 ; MIPS32-NEXT:    sw $2, 20($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:  $BB1_6: # %b.PHI.1
 ; MIPS32-NEXT:    lw $1, 20($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    ori $2, $zero, 1
-; MIPS32-NEXT:    lw $3, 40($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    and $2, $3, $2
+; MIPS32-NEXT:    lw $2, 40($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    andi $3, $2, 1
 ; MIPS32-NEXT:    move $4, $1
 ; MIPS32-NEXT:    lw $5, 24($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    sw $1, 16($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:    sw $4, 12($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:    sw $5, 8($sp) # 4-byte Folded Spill
-; MIPS32-NEXT:    bnez $2, $BB1_8
+; MIPS32-NEXT:    bnez $3, $BB1_8
 ; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  # %bb.7: # %b.PHI.1
 ; MIPS32-NEXT:    j $BB1_15
@@ -258,10 +247,9 @@ define void @long_chain_i32_in_gpr(i1 %cnd0, i1 %cnd1, i1 %cnd2, i32* %a, i32* %
 ; MIPS32-NEXT:    jr $ra
 ; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  $BB1_9: # %pre.PHI.2
-; MIPS32-NEXT:    ori $1, $zero, 1
-; MIPS32-NEXT:    lw $2, 48($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    and $1, $2, $1
-; MIPS32-NEXT:    bnez $1, $BB1_11
+; MIPS32-NEXT:    lw $1, 48($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    andi $2, $1, 1
+; MIPS32-NEXT:    bnez $2, $BB1_11
 ; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  # %bb.10: # %pre.PHI.2
 ; MIPS32-NEXT:    j $BB1_12
@@ -278,15 +266,14 @@ define void @long_chain_i32_in_gpr(i1 %cnd0, i1 %cnd1, i1 %cnd2, i32* %a, i32* %
 ; MIPS32-NEXT:    sw $2, 4($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:  $BB1_13: # %b.PHI.2
 ; MIPS32-NEXT:    lw $1, 4($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    ori $2, $zero, 1
-; MIPS32-NEXT:    lw $3, 44($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    and $2, $3, $2
+; MIPS32-NEXT:    lw $2, 44($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    andi $3, $2, 1
 ; MIPS32-NEXT:    move $4, $1
 ; MIPS32-NEXT:    move $5, $1
 ; MIPS32-NEXT:    sw $1, 0($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:    sw $4, 12($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:    sw $5, 8($sp) # 4-byte Folded Spill
-; MIPS32-NEXT:    bnez $2, $BB1_15
+; MIPS32-NEXT:    bnez $3, $BB1_15
 ; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  # %bb.14: # %b.PHI.2.end
 ; MIPS32-NEXT:    lw $1, 0($sp) # 4-byte Folded Reload
@@ -298,14 +285,13 @@ define void @long_chain_i32_in_gpr(i1 %cnd0, i1 %cnd1, i1 %cnd2, i32* %a, i32* %
 ; MIPS32-NEXT:  $BB1_15: # %b.PHI.3
 ; MIPS32-NEXT:    lw $1, 8($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    lw $2, 12($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    ori $3, $zero, 1
-; MIPS32-NEXT:    lw $4, 40($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    and $5, $4, $3
-; MIPS32-NEXT:    movn $1, $2, $5
-; MIPS32-NEXT:    lw $5, 44($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    and $3, $5, $3
+; MIPS32-NEXT:    lw $3, 40($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    andi $4, $3, 1
+; MIPS32-NEXT:    movn $1, $2, $4
+; MIPS32-NEXT:    lw $4, 44($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    andi $5, $4, 1
 ; MIPS32-NEXT:    move $6, $2
-; MIPS32-NEXT:    movn $6, $1, $3
+; MIPS32-NEXT:    movn $6, $1, $5
 ; MIPS32-NEXT:    lw $1, 28($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    sw $6, 0($1)
 ; MIPS32-NEXT:    sw $2, 0($1)
@@ -381,8 +367,7 @@ define void @long_chain_ambiguous_float_in_fpr(i1 %cnd0, i1 %cnd1, i1 %cnd2, flo
 ; MIPS32-NEXT:    lw $2, 0($2)
 ; MIPS32-NEXT:    addiu $3, $sp, 72
 ; MIPS32-NEXT:    lw $3, 0($3)
-; MIPS32-NEXT:    ori $8, $zero, 1
-; MIPS32-NEXT:    and $8, $4, $8
+; MIPS32-NEXT:    andi $8, $4, 1
 ; MIPS32-NEXT:    sw $1, 44($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:    sw $4, 40($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:    sw $5, 36($sp) # 4-byte Folded Spill
@@ -393,16 +378,14 @@ define void @long_chain_ambiguous_float_in_fpr(i1 %cnd0, i1 %cnd1, i1 %cnd2, flo
 ; MIPS32-NEXT:    bnez $8, $BB2_9
 ; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  # %bb.1: # %pre.PHI.1
-; MIPS32-NEXT:    ori $1, $zero, 1
-; MIPS32-NEXT:    lw $2, 36($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    and $1, $2, $1
-; MIPS32-NEXT:    bnez $1, $BB2_4
+; MIPS32-NEXT:    lw $1, 36($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    andi $2, $1, 1
+; MIPS32-NEXT:    bnez $2, $BB2_4
 ; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  # %bb.2: # %pre.PHI.1.0
-; MIPS32-NEXT:    ori $1, $zero, 1
-; MIPS32-NEXT:    lw $2, 32($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    and $1, $2, $1
-; MIPS32-NEXT:    bnez $1, $BB2_5
+; MIPS32-NEXT:    lw $1, 32($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    andi $2, $1, 1
+; MIPS32-NEXT:    bnez $2, $BB2_5
 ; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  # %bb.3: # %b.PHI.1.0
 ; MIPS32-NEXT:    lw $1, 28($sp) # 4-byte Folded Reload
@@ -422,13 +405,12 @@ define void @long_chain_ambiguous_float_in_fpr(i1 %cnd0, i1 %cnd1, i1 %cnd2, flo
 ; MIPS32-NEXT:    sw $2, 16($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:  $BB2_6: # %b.PHI.1
 ; MIPS32-NEXT:    lw $1, 16($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    ori $2, $zero, 1
-; MIPS32-NEXT:    lw $3, 32($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    and $2, $3, $2
+; MIPS32-NEXT:    lw $2, 32($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    andi $3, $2, 1
 ; MIPS32-NEXT:    move $4, $1
 ; MIPS32-NEXT:    sw $1, 12($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:    sw $4, 8($sp) # 4-byte Folded Spill
-; MIPS32-NEXT:    bnez $2, $BB2_8
+; MIPS32-NEXT:    bnez $3, $BB2_8
 ; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  # %bb.7: # %b.PHI.1
 ; MIPS32-NEXT:    j $BB2_15
@@ -441,10 +423,9 @@ define void @long_chain_ambiguous_float_in_fpr(i1 %cnd0, i1 %cnd1, i1 %cnd2, flo
 ; MIPS32-NEXT:    jr $ra
 ; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  $BB2_9: # %pre.PHI.2
-; MIPS32-NEXT:    ori $1, $zero, 1
-; MIPS32-NEXT:    lw $2, 40($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    and $1, $2, $1
-; MIPS32-NEXT:    bnez $1, $BB2_11
+; MIPS32-NEXT:    lw $1, 40($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    andi $2, $1, 1
+; MIPS32-NEXT:    bnez $2, $BB2_11
 ; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  # %bb.10: # %pre.PHI.2
 ; MIPS32-NEXT:    j $BB2_12
@@ -461,13 +442,12 @@ define void @long_chain_ambiguous_float_in_fpr(i1 %cnd0, i1 %cnd1, i1 %cnd2, flo
 ; MIPS32-NEXT:    sw $2, 4($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:  $BB2_13: # %b.PHI.2
 ; MIPS32-NEXT:    lw $1, 4($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    ori $2, $zero, 1
-; MIPS32-NEXT:    lw $3, 36($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    and $2, $3, $2
+; MIPS32-NEXT:    lw $2, 36($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    andi $3, $2, 1
 ; MIPS32-NEXT:    move $4, $1
 ; MIPS32-NEXT:    sw $1, 0($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:    sw $4, 8($sp) # 4-byte Folded Spill
-; MIPS32-NEXT:    bnez $2, $BB2_15
+; MIPS32-NEXT:    bnez $3, $BB2_15
 ; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  # %bb.14: # %b.PHI.2.end
 ; MIPS32-NEXT:    lw $1, 0($sp) # 4-byte Folded Reload
@@ -479,14 +459,13 @@ define void @long_chain_ambiguous_float_in_fpr(i1 %cnd0, i1 %cnd1, i1 %cnd2, flo
 ; MIPS32-NEXT:  $BB2_15: # %b.PHI.3
 ; MIPS32-NEXT:    lw $1, 8($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    lw $2, 8($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    ori $3, $zero, 1
-; MIPS32-NEXT:    lw $4, 32($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    and $5, $4, $3
-; MIPS32-NEXT:    movn $1, $2, $5
-; MIPS32-NEXT:    lw $5, 36($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    and $3, $5, $3
+; MIPS32-NEXT:    lw $3, 32($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    andi $4, $3, 1
+; MIPS32-NEXT:    movn $1, $2, $4
+; MIPS32-NEXT:    lw $4, 36($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    andi $5, $4, 1
 ; MIPS32-NEXT:    move $6, $2
-; MIPS32-NEXT:    movn $6, $1, $3
+; MIPS32-NEXT:    movn $6, $1, $5
 ; MIPS32-NEXT:    lw $1, 20($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    sw $6, 0($1)
 ; MIPS32-NEXT:    sw $2, 0($1)
@@ -565,8 +544,7 @@ define void @long_chain_float_in_fpr(i1 %cnd0, i1 %cnd1, i1 %cnd2, float* %a, fl
 ; MIPS32-NEXT:    lw $3, 0($3)
 ; MIPS32-NEXT:    ori $8, $zero, 0
 ; MIPS32-NEXT:    mtc1 $8, $f0
-; MIPS32-NEXT:    ori $8, $zero, 1
-; MIPS32-NEXT:    and $8, $4, $8
+; MIPS32-NEXT:    andi $8, $4, 1
 ; MIPS32-NEXT:    sw $1, 52($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:    sw $4, 48($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:    sw $5, 44($sp) # 4-byte Folded Spill
@@ -578,16 +556,14 @@ define void @long_chain_float_in_fpr(i1 %cnd0, i1 %cnd1, i1 %cnd2, float* %a, fl
 ; MIPS32-NEXT:    bnez $8, $BB3_9
 ; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  # %bb.1: # %pre.PHI.1
-; MIPS32-NEXT:    ori $1, $zero, 1
-; MIPS32-NEXT:    lw $2, 44($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    and $1, $2, $1
-; MIPS32-NEXT:    bnez $1, $BB3_4
+; MIPS32-NEXT:    lw $1, 44($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    andi $2, $1, 1
+; MIPS32-NEXT:    bnez $2, $BB3_4
 ; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  # %bb.2: # %pre.PHI.1.0
-; MIPS32-NEXT:    ori $1, $zero, 1
-; MIPS32-NEXT:    lw $2, 40($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    and $1, $2, $1
-; MIPS32-NEXT:    bnez $1, $BB3_5
+; MIPS32-NEXT:    lw $1, 40($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    andi $2, $1, 1
+; MIPS32-NEXT:    bnez $2, $BB3_5
 ; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  # %bb.3: # %b.PHI.1.0
 ; MIPS32-NEXT:    lw $1, 36($sp) # 4-byte Folded Reload
@@ -607,15 +583,14 @@ define void @long_chain_float_in_fpr(i1 %cnd0, i1 %cnd1, i1 %cnd2, float* %a, fl
 ; MIPS32-NEXT:    swc1 $f0, 20($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:  $BB3_6: # %b.PHI.1
 ; MIPS32-NEXT:    lwc1 $f0, 20($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    ori $1, $zero, 1
-; MIPS32-NEXT:    lw $2, 40($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    and $1, $2, $1
+; MIPS32-NEXT:    lw $1, 40($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    andi $2, $1, 1
 ; MIPS32-NEXT:    mov.s $f1, $f0
 ; MIPS32-NEXT:    lwc1 $f2, 24($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    swc1 $f0, 16($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:    swc1 $f1, 12($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:    swc1 $f2, 8($sp) # 4-byte Folded Spill
-; MIPS32-NEXT:    bnez $1, $BB3_8
+; MIPS32-NEXT:    bnez $2, $BB3_8
 ; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  # %bb.7: # %b.PHI.1
 ; MIPS32-NEXT:    j $BB3_15
@@ -628,10 +603,9 @@ define void @long_chain_float_in_fpr(i1 %cnd0, i1 %cnd1, i1 %cnd2, float* %a, fl
 ; MIPS32-NEXT:    jr $ra
 ; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  $BB3_9: # %pre.PHI.2
-; MIPS32-NEXT:    ori $1, $zero, 1
-; MIPS32-NEXT:    lw $2, 48($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    and $1, $2, $1
-; MIPS32-NEXT:    bnez $1, $BB3_11
+; MIPS32-NEXT:    lw $1, 48($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    andi $2, $1, 1
+; MIPS32-NEXT:    bnez $2, $BB3_11
 ; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  # %bb.10: # %pre.PHI.2
 ; MIPS32-NEXT:    j $BB3_12
@@ -648,15 +622,14 @@ define void @long_chain_float_in_fpr(i1 %cnd0, i1 %cnd1, i1 %cnd2, float* %a, fl
 ; MIPS32-NEXT:    swc1 $f0, 4($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:  $BB3_13: # %b.PHI.2
 ; MIPS32-NEXT:    lwc1 $f0, 4($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    ori $1, $zero, 1
-; MIPS32-NEXT:    lw $2, 44($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    and $1, $2, $1
+; MIPS32-NEXT:    lw $1, 44($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    andi $2, $1, 1
 ; MIPS32-NEXT:    mov.s $f1, $f0
 ; MIPS32-NEXT:    mov.s $f2, $f0
 ; MIPS32-NEXT:    swc1 $f0, 0($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:    swc1 $f1, 12($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:    swc1 $f2, 8($sp) # 4-byte Folded Spill
-; MIPS32-NEXT:    bnez $1, $BB3_15
+; MIPS32-NEXT:    bnez $2, $BB3_15
 ; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  # %bb.14: # %b.PHI.2.end
 ; MIPS32-NEXT:    lwc1 $f0, 0($sp) # 4-byte Folded Reload
@@ -668,17 +641,16 @@ define void @long_chain_float_in_fpr(i1 %cnd0, i1 %cnd1, i1 %cnd2, float* %a, fl
 ; MIPS32-NEXT:  $BB3_15: # %b.PHI.3
 ; MIPS32-NEXT:    lwc1 $f0, 8($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    lwc1 $f1, 12($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    ori $1, $zero, 1
-; MIPS32-NEXT:    lw $2, 40($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    and $3, $2, $1
-; MIPS32-NEXT:    movn.s $f0, $f1, $3
-; MIPS32-NEXT:    lw $3, 44($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    and $1, $3, $1
+; MIPS32-NEXT:    lw $1, 40($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    andi $2, $1, 1
+; MIPS32-NEXT:    movn.s $f0, $f1, $2
+; MIPS32-NEXT:    lw $2, 44($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    andi $3, $2, 1
 ; MIPS32-NEXT:    mov.s $f2, $f1
-; MIPS32-NEXT:    movn.s $f2, $f0, $1
-; MIPS32-NEXT:    lw $1, 28($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    swc1 $f2, 0($1)
-; MIPS32-NEXT:    swc1 $f1, 0($1)
+; MIPS32-NEXT:    movn.s $f2, $f0, $3
+; MIPS32-NEXT:    lw $3, 28($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    swc1 $f2, 0($3)
+; MIPS32-NEXT:    swc1 $f1, 0($3)
 ; MIPS32-NEXT:    addiu $sp, $sp, 56
 ; MIPS32-NEXT:    jr $ra
 ; MIPS32-NEXT:    nop
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/long_ambiguous_chain_s64.ll b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/long_ambiguous_chain_s64.ll
index 71b8c7f9c..a237099eb 100644
--- a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/long_ambiguous_chain_s64.ll
+++ b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/long_ambiguous_chain_s64.ll
@@ -12,8 +12,7 @@ define void @long_chain_ambiguous_i64_in_fpr(i1 %cnd0, i1 %cnd1, i1 %cnd2, i64*
 ; MIPS32-NEXT:    lw $2, 0($2)
 ; MIPS32-NEXT:    addiu $3, $sp, 96
 ; MIPS32-NEXT:    lw $3, 0($3)
-; MIPS32-NEXT:    ori $8, $zero, 1
-; MIPS32-NEXT:    and $8, $4, $8
+; MIPS32-NEXT:    andi $8, $4, 1
 ; MIPS32-NEXT:    sw $1, 68($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:    sw $4, 64($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:    sw $5, 60($sp) # 4-byte Folded Spill
@@ -24,16 +23,14 @@ define void @long_chain_ambiguous_i64_in_fpr(i1 %cnd0, i1 %cnd1, i1 %cnd2, i64*
 ; MIPS32-NEXT:    bnez $8, $BB0_9
 ; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  # %bb.1: # %pre.PHI.1
-; MIPS32-NEXT:    ori $1, $zero, 1
-; MIPS32-NEXT:    lw $2, 60($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    and $1, $2, $1
-; MIPS32-NEXT:    bnez $1, $BB0_4
+; MIPS32-NEXT:    lw $1, 60($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    andi $2, $1, 1
+; MIPS32-NEXT:    bnez $2, $BB0_4
 ; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  # %bb.2: # %pre.PHI.1.0
-; MIPS32-NEXT:    ori $1, $zero, 1
-; MIPS32-NEXT:    lw $2, 56($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    and $1, $2, $1
-; MIPS32-NEXT:    bnez $1, $BB0_5
+; MIPS32-NEXT:    lw $1, 56($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    andi $2, $1, 1
+; MIPS32-NEXT:    bnez $2, $BB0_5
 ; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  # %bb.3: # %b.PHI.1.0
 ; MIPS32-NEXT:    lw $1, 52($sp) # 4-byte Folded Reload
@@ -53,13 +50,12 @@ define void @long_chain_ambiguous_i64_in_fpr(i1 %cnd0, i1 %cnd1, i1 %cnd2, i64*
 ; MIPS32-NEXT:    sdc1 $f0, 32($sp) # 8-byte Folded Spill
 ; MIPS32-NEXT:  $BB0_6: # %b.PHI.1
 ; MIPS32-NEXT:    ldc1 $f0, 32($sp) # 8-byte Folded Reload
-; MIPS32-NEXT:    ori $1, $zero, 1
-; MIPS32-NEXT:    lw $2, 56($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    and $1, $2, $1
+; MIPS32-NEXT:    lw $1, 56($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    andi $2, $1, 1
 ; MIPS32-NEXT:    mov.d $f2, $f0
 ; MIPS32-NEXT:    sdc1 $f0, 24($sp) # 8-byte Folded Spill
 ; MIPS32-NEXT:    sdc1 $f2, 16($sp) # 8-byte Folded Spill
-; MIPS32-NEXT:    bnez $1, $BB0_8
+; MIPS32-NEXT:    bnez $2, $BB0_8
 ; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  # %bb.7: # %b.PHI.1
 ; MIPS32-NEXT:    j $BB0_15
@@ -72,10 +68,9 @@ define void @long_chain_ambiguous_i64_in_fpr(i1 %cnd0, i1 %cnd1, i1 %cnd2, i64*
 ; MIPS32-NEXT:    jr $ra
 ; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  $BB0_9: # %pre.PHI.2
-; MIPS32-NEXT:    ori $1, $zero, 1
-; MIPS32-NEXT:    lw $2, 64($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    and $1, $2, $1
-; MIPS32-NEXT:    bnez $1, $BB0_11
+; MIPS32-NEXT:    lw $1, 64($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    andi $2, $1, 1
+; MIPS32-NEXT:    bnez $2, $BB0_11
 ; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  # %bb.10: # %pre.PHI.2
 ; MIPS32-NEXT:    j $BB0_12
@@ -92,13 +87,12 @@ define void @long_chain_ambiguous_i64_in_fpr(i1 %cnd0, i1 %cnd1, i1 %cnd2, i64*
 ; MIPS32-NEXT:    sdc1 $f0, 8($sp) # 8-byte Folded Spill
 ; MIPS32-NEXT:  $BB0_13: # %b.PHI.2
 ; MIPS32-NEXT:    ldc1 $f0, 8($sp) # 8-byte Folded Reload
-; MIPS32-NEXT:    ori $1, $zero, 1
-; MIPS32-NEXT:    lw $2, 60($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    and $1, $2, $1
+; MIPS32-NEXT:    lw $1, 60($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    andi $2, $1, 1
 ; MIPS32-NEXT:    mov.d $f2, $f0
 ; MIPS32-NEXT:    sdc1 $f0, 0($sp) # 8-byte Folded Spill
 ; MIPS32-NEXT:    sdc1 $f2, 16($sp) # 8-byte Folded Spill
-; MIPS32-NEXT:    bnez $1, $BB0_15
+; MIPS32-NEXT:    bnez $2, $BB0_15
 ; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  # %bb.14: # %b.PHI.2.end
 ; MIPS32-NEXT:    ldc1 $f0, 0($sp) # 8-byte Folded Reload
@@ -110,17 +104,16 @@ define void @long_chain_ambiguous_i64_in_fpr(i1 %cnd0, i1 %cnd1, i1 %cnd2, i64*
 ; MIPS32-NEXT:  $BB0_15: # %b.PHI.3
 ; MIPS32-NEXT:    ldc1 $f0, 16($sp) # 8-byte Folded Reload
 ; MIPS32-NEXT:    ldc1 $f2, 16($sp) # 8-byte Folded Reload
-; MIPS32-NEXT:    ori $1, $zero, 1
-; MIPS32-NEXT:    lw $2, 56($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    and $3, $2, $1
-; MIPS32-NEXT:    movn.d $f0, $f2, $3
-; MIPS32-NEXT:    lw $3, 60($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    and $1, $3, $1
+; MIPS32-NEXT:    lw $1, 56($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    andi $2, $1, 1
+; MIPS32-NEXT:    movn.d $f0, $f2, $2
+; MIPS32-NEXT:    lw $2, 60($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    andi $3, $2, 1
 ; MIPS32-NEXT:    mov.d $f4, $f2
-; MIPS32-NEXT:    movn.d $f4, $f0, $1
-; MIPS32-NEXT:    lw $1, 44($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    sdc1 $f4, 0($1)
-; MIPS32-NEXT:    sdc1 $f2, 0($1)
+; MIPS32-NEXT:    movn.d $f4, $f0, $3
+; MIPS32-NEXT:    lw $3, 44($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    sdc1 $f4, 0($3)
+; MIPS32-NEXT:    sdc1 $f2, 0($3)
 ; MIPS32-NEXT:    addiu $sp, $sp, 72
 ; MIPS32-NEXT:    jr $ra
 ; MIPS32-NEXT:    nop
@@ -195,8 +188,7 @@ define void @long_chain_i64_in_gpr(i1 %cnd0, i1 %cnd1, i1 %cnd2, i64* %a, i64* %
 ; MIPS32-NEXT:    addiu $3, $sp, 104
 ; MIPS32-NEXT:    lw $3, 0($3)
 ; MIPS32-NEXT:    ori $8, $zero, 0
-; MIPS32-NEXT:    ori $9, $zero, 1
-; MIPS32-NEXT:    and $9, $4, $9
+; MIPS32-NEXT:    andi $9, $4, 1
 ; MIPS32-NEXT:    sw $1, 76($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:    sw $4, 72($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:    sw $5, 68($sp) # 4-byte Folded Spill
@@ -208,23 +200,19 @@ define void @long_chain_i64_in_gpr(i1 %cnd0, i1 %cnd1, i1 %cnd2, i64* %a, i64* %
 ; MIPS32-NEXT:    bnez $9, $BB1_9
 ; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  # %bb.1: # %pre.PHI.1
-; MIPS32-NEXT:    ori $1, $zero, 1
-; MIPS32-NEXT:    lw $2, 68($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    and $1, $2, $1
-; MIPS32-NEXT:    bnez $1, $BB1_4
+; MIPS32-NEXT:    lw $1, 68($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    andi $2, $1, 1
+; MIPS32-NEXT:    bnez $2, $BB1_4
 ; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  # %bb.2: # %pre.PHI.1.0
-; MIPS32-NEXT:    ori $1, $zero, 1
-; MIPS32-NEXT:    lw $2, 64($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    and $1, $2, $1
-; MIPS32-NEXT:    bnez $1, $BB1_5
+; MIPS32-NEXT:    lw $1, 64($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    andi $2, $1, 1
+; MIPS32-NEXT:    bnez $2, $BB1_5
 ; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  # %bb.3: # %b.PHI.1.0
 ; MIPS32-NEXT:    lw $1, 60($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    lw $2, 0($1)
-; MIPS32-NEXT:    ori $3, $zero, 4
-; MIPS32-NEXT:    addu $3, $1, $3
-; MIPS32-NEXT:    lw $3, 0($3)
+; MIPS32-NEXT:    lw $3, 4($1)
 ; MIPS32-NEXT:    sw $2, 44($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:    sw $3, 40($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:    j $BB1_6
@@ -232,9 +220,7 @@ define void @long_chain_i64_in_gpr(i1 %cnd0, i1 %cnd1, i1 %cnd2, i64* %a, i64* %
 ; MIPS32-NEXT:  $BB1_4: # %b.PHI.1.1
 ; MIPS32-NEXT:    lw $1, 76($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    lw $2, 0($1)
-; MIPS32-NEXT:    ori $3, $zero, 4
-; MIPS32-NEXT:    addu $3, $1, $3
-; MIPS32-NEXT:    lw $3, 0($3)
+; MIPS32-NEXT:    lw $3, 4($1)
 ; MIPS32-NEXT:    sw $2, 44($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:    sw $3, 40($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:    j $BB1_6
@@ -242,17 +228,14 @@ define void @long_chain_i64_in_gpr(i1 %cnd0, i1 %cnd1, i1 %cnd2, i64* %a, i64* %
 ; MIPS32-NEXT:  $BB1_5: # %b.PHI.1.2
 ; MIPS32-NEXT:    lw $1, 56($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    lw $2, 0($1)
-; MIPS32-NEXT:    ori $3, $zero, 4
-; MIPS32-NEXT:    addu $3, $1, $3
-; MIPS32-NEXT:    lw $3, 0($3)
+; MIPS32-NEXT:    lw $3, 4($1)
 ; MIPS32-NEXT:    sw $2, 44($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:    sw $3, 40($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:  $BB1_6: # %b.PHI.1
 ; MIPS32-NEXT:    lw $1, 40($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    lw $2, 44($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    ori $3, $zero, 1
-; MIPS32-NEXT:    lw $4, 64($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    and $3, $4, $3
+; MIPS32-NEXT:    lw $3, 64($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    andi $4, $3, 1
 ; MIPS32-NEXT:    move $5, $2
 ; MIPS32-NEXT:    move $6, $1
 ; MIPS32-NEXT:    lw $7, 48($sp) # 4-byte Folded Reload
@@ -263,7 +246,7 @@ define void @long_chain_i64_in_gpr(i1 %cnd0, i1 %cnd1, i1 %cnd2, i64* %a, i64* %
 ; MIPS32-NEXT:    sw $6, 24($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:    sw $7, 20($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:    sw $8, 16($sp) # 4-byte Folded Spill
-; MIPS32-NEXT:    bnez $3, $BB1_8
+; MIPS32-NEXT:    bnez $4, $BB1_8
 ; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  # %bb.7: # %b.PHI.1
 ; MIPS32-NEXT:    j $BB1_15
@@ -272,18 +255,15 @@ define void @long_chain_i64_in_gpr(i1 %cnd0, i1 %cnd1, i1 %cnd2, i64* %a, i64* %
 ; MIPS32-NEXT:    lw $1, 32($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    lw $2, 52($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    sw $1, 0($2)
-; MIPS32-NEXT:    ori $3, $zero, 4
-; MIPS32-NEXT:    addu $3, $2, $3
-; MIPS32-NEXT:    lw $4, 36($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    sw $4, 0($3)
+; MIPS32-NEXT:    lw $3, 36($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    sw $3, 4($2)
 ; MIPS32-NEXT:    addiu $sp, $sp, 80
 ; MIPS32-NEXT:    jr $ra
 ; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  $BB1_9: # %pre.PHI.2
-; MIPS32-NEXT:    ori $1, $zero, 1
-; MIPS32-NEXT:    lw $2, 72($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    and $1, $2, $1
-; MIPS32-NEXT:    bnez $1, $BB1_11
+; MIPS32-NEXT:    lw $1, 72($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    andi $2, $1, 1
+; MIPS32-NEXT:    bnez $2, $BB1_11
 ; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  # %bb.10: # %pre.PHI.2
 ; MIPS32-NEXT:    j $BB1_12
@@ -291,9 +271,7 @@ define void @long_chain_i64_in_gpr(i1 %cnd0, i1 %cnd1, i1 %cnd2, i64* %a, i64* %
 ; MIPS32-NEXT:  $BB1_11: # %b.PHI.2.0
 ; MIPS32-NEXT:    lw $1, 60($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    lw $2, 0($1)
-; MIPS32-NEXT:    ori $3, $zero, 4
-; MIPS32-NEXT:    addu $3, $1, $3
-; MIPS32-NEXT:    lw $3, 0($3)
+; MIPS32-NEXT:    lw $3, 4($1)
 ; MIPS32-NEXT:    sw $2, 12($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:    sw $3, 8($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:    j $BB1_13
@@ -301,17 +279,14 @@ define void @long_chain_i64_in_gpr(i1 %cnd0, i1 %cnd1, i1 %cnd2, i64* %a, i64* %
 ; MIPS32-NEXT:  $BB1_12: # %b.PHI.2.1
 ; MIPS32-NEXT:    lw $1, 76($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    lw $2, 0($1)
-; MIPS32-NEXT:    ori $3, $zero, 4
-; MIPS32-NEXT:    addu $3, $1, $3
-; MIPS32-NEXT:    lw $3, 0($3)
+; MIPS32-NEXT:    lw $3, 4($1)
 ; MIPS32-NEXT:    sw $2, 12($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:    sw $3, 8($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:  $BB1_13: # %b.PHI.2
 ; MIPS32-NEXT:    lw $1, 8($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    lw $2, 12($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    ori $3, $zero, 1
-; MIPS32-NEXT:    lw $4, 68($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    and $3, $4, $3
+; MIPS32-NEXT:    lw $3, 68($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    andi $4, $3, 1
 ; MIPS32-NEXT:    move $5, $2
 ; MIPS32-NEXT:    move $6, $1
 ; MIPS32-NEXT:    move $7, $2
@@ -322,16 +297,14 @@ define void @long_chain_i64_in_gpr(i1 %cnd0, i1 %cnd1, i1 %cnd2, i64* %a, i64* %
 ; MIPS32-NEXT:    sw $6, 24($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:    sw $7, 20($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:    sw $8, 16($sp) # 4-byte Folded Spill
-; MIPS32-NEXT:    bnez $3, $BB1_15
+; MIPS32-NEXT:    bnez $4, $BB1_15
 ; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  # %bb.14: # %b.PHI.2.end
 ; MIPS32-NEXT:    lw $1, 0($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    lw $2, 52($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    sw $1, 0($2)
-; MIPS32-NEXT:    ori $3, $zero, 4
-; MIPS32-NEXT:    addu $3, $2, $3
-; MIPS32-NEXT:    lw $4, 4($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    sw $4, 0($3)
+; MIPS32-NEXT:    lw $3, 4($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    sw $3, 4($2)
 ; MIPS32-NEXT:    addiu $sp, $sp, 80
 ; MIPS32-NEXT:    jr $ra
 ; MIPS32-NEXT:    nop
@@ -340,26 +313,21 @@ define void @long_chain_i64_in_gpr(i1 %cnd0, i1 %cnd1, i1 %cnd2, i64* %a, i64* %
 ; MIPS32-NEXT:    lw $2, 20($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    lw $3, 24($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    lw $4, 28($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    ori $5, $zero, 1
-; MIPS32-NEXT:    lw $6, 64($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    and $7, $6, $5
-; MIPS32-NEXT:    movn $2, $4, $7
-; MIPS32-NEXT:    movn $1, $3, $7
-; MIPS32-NEXT:    lw $7, 68($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    and $5, $7, $5
+; MIPS32-NEXT:    lw $5, 64($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    andi $6, $5, 1
+; MIPS32-NEXT:    movn $2, $4, $6
+; MIPS32-NEXT:    movn $1, $3, $6
+; MIPS32-NEXT:    lw $6, 68($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    andi $7, $6, 1
 ; MIPS32-NEXT:    move $8, $4
-; MIPS32-NEXT:    movn $8, $2, $5
+; MIPS32-NEXT:    movn $8, $2, $7
 ; MIPS32-NEXT:    move $2, $3
-; MIPS32-NEXT:    movn $2, $1, $5
+; MIPS32-NEXT:    movn $2, $1, $7
 ; MIPS32-NEXT:    lw $1, 52($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    sw $8, 0($1)
-; MIPS32-NEXT:    ori $5, $zero, 4
-; MIPS32-NEXT:    addu $5, $1, $5
-; MIPS32-NEXT:    sw $2, 0($5)
+; MIPS32-NEXT:    sw $2, 4($1)
 ; MIPS32-NEXT:    sw $4, 0($1)
-; MIPS32-NEXT:    ori $2, $zero, 4
-; MIPS32-NEXT:    addu $2, $1, $2
-; MIPS32-NEXT:    sw $3, 0($2)
+; MIPS32-NEXT:    sw $3, 4($1)
 ; MIPS32-NEXT:    addiu $sp, $sp, 80
 ; MIPS32-NEXT:    jr $ra
 ; MIPS32-NEXT:    nop
@@ -432,8 +400,7 @@ define void @long_chain_ambiguous_double_in_fpr(i1 %cnd0, i1 %cnd1, i1 %cnd2, do
 ; MIPS32-NEXT:    lw $2, 0($2)
 ; MIPS32-NEXT:    addiu $3, $sp, 96
 ; MIPS32-NEXT:    lw $3, 0($3)
-; MIPS32-NEXT:    ori $8, $zero, 1
-; MIPS32-NEXT:    and $8, $4, $8
+; MIPS32-NEXT:    andi $8, $4, 1
 ; MIPS32-NEXT:    sw $1, 68($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:    sw $4, 64($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:    sw $5, 60($sp) # 4-byte Folded Spill
@@ -444,16 +411,14 @@ define void @long_chain_ambiguous_double_in_fpr(i1 %cnd0, i1 %cnd1, i1 %cnd2, do
 ; MIPS32-NEXT:    bnez $8, $BB2_9
 ; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  # %bb.1: # %pre.PHI.1
-; MIPS32-NEXT:    ori $1, $zero, 1
-; MIPS32-NEXT:    lw $2, 60($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    and $1, $2, $1
-; MIPS32-NEXT:    bnez $1, $BB2_4
+; MIPS32-NEXT:    lw $1, 60($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    andi $2, $1, 1
+; MIPS32-NEXT:    bnez $2, $BB2_4
 ; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  # %bb.2: # %pre.PHI.1.0
-; MIPS32-NEXT:    ori $1, $zero, 1
-; MIPS32-NEXT:    lw $2, 56($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    and $1, $2, $1
-; MIPS32-NEXT:    bnez $1, $BB2_5
+; MIPS32-NEXT:    lw $1, 56($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    andi $2, $1, 1
+; MIPS32-NEXT:    bnez $2, $BB2_5
 ; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  # %bb.3: # %b.PHI.1.0
 ; MIPS32-NEXT:    lw $1, 52($sp) # 4-byte Folded Reload
@@ -473,13 +438,12 @@ define void @long_chain_ambiguous_double_in_fpr(i1 %cnd0, i1 %cnd1, i1 %cnd2, do
 ; MIPS32-NEXT:    sdc1 $f0, 32($sp) # 8-byte Folded Spill
 ; MIPS32-NEXT:  $BB2_6: # %b.PHI.1
 ; MIPS32-NEXT:    ldc1 $f0, 32($sp) # 8-byte Folded Reload
-; MIPS32-NEXT:    ori $1, $zero, 1
-; MIPS32-NEXT:    lw $2, 56($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    and $1, $2, $1
+; MIPS32-NEXT:    lw $1, 56($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    andi $2, $1, 1
 ; MIPS32-NEXT:    mov.d $f2, $f0
 ; MIPS32-NEXT:    sdc1 $f0, 24($sp) # 8-byte Folded Spill
 ; MIPS32-NEXT:    sdc1 $f2, 16($sp) # 8-byte Folded Spill
-; MIPS32-NEXT:    bnez $1, $BB2_8
+; MIPS32-NEXT:    bnez $2, $BB2_8
 ; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  # %bb.7: # %b.PHI.1
 ; MIPS32-NEXT:    j $BB2_15
@@ -492,10 +456,9 @@ define void @long_chain_ambiguous_double_in_fpr(i1 %cnd0, i1 %cnd1, i1 %cnd2, do
 ; MIPS32-NEXT:    jr $ra
 ; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  $BB2_9: # %pre.PHI.2
-; MIPS32-NEXT:    ori $1, $zero, 1
-; MIPS32-NEXT:    lw $2, 64($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    and $1, $2, $1
-; MIPS32-NEXT:    bnez $1, $BB2_11
+; MIPS32-NEXT:    lw $1, 64($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    andi $2, $1, 1
+; MIPS32-NEXT:    bnez $2, $BB2_11
 ; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  # %bb.10: # %pre.PHI.2
 ; MIPS32-NEXT:    j $BB2_12
@@ -512,13 +475,12 @@ define void @long_chain_ambiguous_double_in_fpr(i1 %cnd0, i1 %cnd1, i1 %cnd2, do
 ; MIPS32-NEXT:    sdc1 $f0, 8($sp) # 8-byte Folded Spill
 ; MIPS32-NEXT:  $BB2_13: # %b.PHI.2
 ; MIPS32-NEXT:    ldc1 $f0, 8($sp) # 8-byte Folded Reload
-; MIPS32-NEXT:    ori $1, $zero, 1
-; MIPS32-NEXT:    lw $2, 60($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    and $1, $2, $1
+; MIPS32-NEXT:    lw $1, 60($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    andi $2, $1, 1
 ; MIPS32-NEXT:    mov.d $f2, $f0
 ; MIPS32-NEXT:    sdc1 $f0, 0($sp) # 8-byte Folded Spill
 ; MIPS32-NEXT:    sdc1 $f2, 16($sp) # 8-byte Folded Spill
-; MIPS32-NEXT:    bnez $1, $BB2_15
+; MIPS32-NEXT:    bnez $2, $BB2_15
 ; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  # %bb.14: # %b.PHI.2.end
 ; MIPS32-NEXT:    ldc1 $f0, 0($sp) # 8-byte Folded Reload
@@ -530,17 +492,16 @@ define void @long_chain_ambiguous_double_in_fpr(i1 %cnd0, i1 %cnd1, i1 %cnd2, do
 ; MIPS32-NEXT:  $BB2_15: # %b.PHI.3
 ; MIPS32-NEXT:    ldc1 $f0, 16($sp) # 8-byte Folded Reload
 ; MIPS32-NEXT:    ldc1 $f2, 16($sp) # 8-byte Folded Reload
-; MIPS32-NEXT:    ori $1, $zero, 1
-; MIPS32-NEXT:    lw $2, 56($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    and $3, $2, $1
-; MIPS32-NEXT:    movn.d $f0, $f2, $3
-; MIPS32-NEXT:    lw $3, 60($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    and $1, $3, $1
+; MIPS32-NEXT:    lw $1, 56($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    andi $2, $1, 1
+; MIPS32-NEXT:    movn.d $f0, $f2, $2
+; MIPS32-NEXT:    lw $2, 60($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    andi $3, $2, 1
 ; MIPS32-NEXT:    mov.d $f4, $f2
-; MIPS32-NEXT:    movn.d $f4, $f0, $1
-; MIPS32-NEXT:    lw $1, 44($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    sdc1 $f4, 0($1)
-; MIPS32-NEXT:    sdc1 $f2, 0($1)
+; MIPS32-NEXT:    movn.d $f4, $f0, $3
+; MIPS32-NEXT:    lw $3, 44($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    sdc1 $f4, 0($3)
+; MIPS32-NEXT:    sdc1 $f2, 0($3)
 ; MIPS32-NEXT:    addiu $sp, $sp, 72
 ; MIPS32-NEXT:    jr $ra
 ; MIPS32-NEXT:    nop
@@ -618,8 +579,7 @@ define void @long_chain_double_in_fpr(i1 %cnd0, i1 %cnd1, i1 %cnd2, double* %a,
 ; MIPS32-NEXT:    ori $9, $zero, 0
 ; MIPS32-NEXT:    mtc1 $9, $f0
 ; MIPS32-NEXT:    mtc1 $8, $f1
-; MIPS32-NEXT:    ori $8, $zero, 1
-; MIPS32-NEXT:    and $8, $4, $8
+; MIPS32-NEXT:    andi $8, $4, 1
 ; MIPS32-NEXT:    sw $1, 84($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:    sw $4, 80($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:    sw $5, 76($sp) # 4-byte Folded Spill
@@ -631,16 +591,14 @@ define void @long_chain_double_in_fpr(i1 %cnd0, i1 %cnd1, i1 %cnd2, double* %a,
 ; MIPS32-NEXT:    bnez $8, $BB3_9
 ; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  # %bb.1: # %pre.PHI.1
-; MIPS32-NEXT:    ori $1, $zero, 1
-; MIPS32-NEXT:    lw $2, 76($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    and $1, $2, $1
-; MIPS32-NEXT:    bnez $1, $BB3_4
+; MIPS32-NEXT:    lw $1, 76($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    andi $2, $1, 1
+; MIPS32-NEXT:    bnez $2, $BB3_4
 ; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  # %bb.2: # %pre.PHI.1.0
-; MIPS32-NEXT:    ori $1, $zero, 1
-; MIPS32-NEXT:    lw $2, 72($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    and $1, $2, $1
-; MIPS32-NEXT:    bnez $1, $BB3_5
+; MIPS32-NEXT:    lw $1, 72($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    andi $2, $1, 1
+; MIPS32-NEXT:    bnez $2, $BB3_5
 ; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  # %bb.3: # %b.PHI.1.0
 ; MIPS32-NEXT:    lw $1, 68($sp) # 4-byte Folded Reload
@@ -660,15 +618,14 @@ define void @long_chain_double_in_fpr(i1 %cnd0, i1 %cnd1, i1 %cnd2, double* %a,
 ; MIPS32-NEXT:    sdc1 $f0, 40($sp) # 8-byte Folded Spill
 ; MIPS32-NEXT:  $BB3_6: # %b.PHI.1
 ; MIPS32-NEXT:    ldc1 $f0, 40($sp) # 8-byte Folded Reload
-; MIPS32-NEXT:    ori $1, $zero, 1
-; MIPS32-NEXT:    lw $2, 72($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    and $1, $2, $1
+; MIPS32-NEXT:    lw $1, 72($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    andi $2, $1, 1
 ; MIPS32-NEXT:    mov.d $f2, $f0
 ; MIPS32-NEXT:    ldc1 $f4, 48($sp) # 8-byte Folded Reload
 ; MIPS32-NEXT:    sdc1 $f0, 32($sp) # 8-byte Folded Spill
 ; MIPS32-NEXT:    sdc1 $f2, 24($sp) # 8-byte Folded Spill
 ; MIPS32-NEXT:    sdc1 $f4, 16($sp) # 8-byte Folded Spill
-; MIPS32-NEXT:    bnez $1, $BB3_8
+; MIPS32-NEXT:    bnez $2, $BB3_8
 ; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  # %bb.7: # %b.PHI.1
 ; MIPS32-NEXT:    j $BB3_15
@@ -681,10 +638,9 @@ define void @long_chain_double_in_fpr(i1 %cnd0, i1 %cnd1, i1 %cnd2, double* %a,
 ; MIPS32-NEXT:    jr $ra
 ; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  $BB3_9: # %pre.PHI.2
-; MIPS32-NEXT:    ori $1, $zero, 1
-; MIPS32-NEXT:    lw $2, 80($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    and $1, $2, $1
-; MIPS32-NEXT:    bnez $1, $BB3_11
+; MIPS32-NEXT:    lw $1, 80($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    andi $2, $1, 1
+; MIPS32-NEXT:    bnez $2, $BB3_11
 ; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  # %bb.10: # %pre.PHI.2
 ; MIPS32-NEXT:    j $BB3_12
@@ -701,15 +657,14 @@ define void @long_chain_double_in_fpr(i1 %cnd0, i1 %cnd1, i1 %cnd2, double* %a,
 ; MIPS32-NEXT:    sdc1 $f0, 8($sp) # 8-byte Folded Spill
 ; MIPS32-NEXT:  $BB3_13: # %b.PHI.2
 ; MIPS32-NEXT:    ldc1 $f0, 8($sp) # 8-byte Folded Reload
-; MIPS32-NEXT:    ori $1, $zero, 1
-; MIPS32-NEXT:    lw $2, 76($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    and $1, $2, $1
+; MIPS32-NEXT:    lw $1, 76($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    andi $2, $1, 1
 ; MIPS32-NEXT:    mov.d $f2, $f0
 ; MIPS32-NEXT:    mov.d $f4, $f0
 ; MIPS32-NEXT:    sdc1 $f0, 0($sp) # 8-byte Folded Spill
 ; MIPS32-NEXT:    sdc1 $f2, 24($sp) # 8-byte Folded Spill
 ; MIPS32-NEXT:    sdc1 $f4, 16($sp) # 8-byte Folded Spill
-; MIPS32-NEXT:    bnez $1, $BB3_15
+; MIPS32-NEXT:    bnez $2, $BB3_15
 ; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  # %bb.14: # %b.PHI.2.end
 ; MIPS32-NEXT:    ldc1 $f0, 0($sp) # 8-byte Folded Reload
@@ -721,17 +676,16 @@ define void @long_chain_double_in_fpr(i1 %cnd0, i1 %cnd1, i1 %cnd2, double* %a,
 ; MIPS32-NEXT:  $BB3_15: # %b.PHI.3
 ; MIPS32-NEXT:    ldc1 $f0, 16($sp) # 8-byte Folded Reload
 ; MIPS32-NEXT:    ldc1 $f2, 24($sp) # 8-byte Folded Reload
-; MIPS32-NEXT:    ori $1, $zero, 1
-; MIPS32-NEXT:    lw $2, 72($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    and $3, $2, $1
-; MIPS32-NEXT:    movn.d $f0, $f2, $3
-; MIPS32-NEXT:    lw $3, 76($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    and $1, $3, $1
+; MIPS32-NEXT:    lw $1, 72($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    andi $2, $1, 1
+; MIPS32-NEXT:    movn.d $f0, $f2, $2
+; MIPS32-NEXT:    lw $2, 76($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    andi $3, $2, 1
 ; MIPS32-NEXT:    mov.d $f4, $f2
-; MIPS32-NEXT:    movn.d $f4, $f0, $1
-; MIPS32-NEXT:    lw $1, 60($sp) # 4-byte Folded Reload
-; MIPS32-NEXT:    sdc1 $f4, 0($1)
-; MIPS32-NEXT:    sdc1 $f2, 0($1)
+; MIPS32-NEXT:    movn.d $f4, $f0, $3
+; MIPS32-NEXT:    lw $3, 60($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    sdc1 $f4, 0($3)
+; MIPS32-NEXT:    sdc1 $f2, 0($3)
 ; MIPS32-NEXT:    addiu $sp, $sp, 88
 ; MIPS32-NEXT:    jr $ra
 ; MIPS32-NEXT:    nop
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/mul.ll b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/mul.ll
index 43b4b63f6..659eadf18 100644
--- a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/mul.ll
+++ b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/mul.ll
@@ -28,8 +28,7 @@ define zeroext i8 @mul_i8_zext(i8 zeroext %a, i8 zeroext %b) {
 ; MIPS32-LABEL: mul_i8_zext:
 ; MIPS32:       # %bb.0: # %entry
 ; MIPS32-NEXT:    mul $1, $5, $4
-; MIPS32-NEXT:    ori $2, $zero, 255
-; MIPS32-NEXT:    and $2, $1, $2
+; MIPS32-NEXT:    andi $2, $1, 255
 ; MIPS32-NEXT:    jr $ra
 ; MIPS32-NEXT:    nop
 entry:
@@ -65,8 +64,7 @@ define zeroext i16 @mul_i16_zext(i16 zeroext %a, i16 zeroext %b) {
 ; MIPS32-LABEL: mul_i16_zext:
 ; MIPS32:       # %bb.0: # %entry
 ; MIPS32-NEXT:    mul $1, $5, $4
-; MIPS32-NEXT:    ori $2, $zero, 65535
-; MIPS32-NEXT:    and $2, $1, $2
+; MIPS32-NEXT:    andi $2, $1, 65535
 ; MIPS32-NEXT:    jr $ra
 ; MIPS32-NEXT:    nop
 entry:
@@ -120,38 +118,37 @@ define i128 @mul_i128(i128 %a, i128 %b) {
 ; MIPS32-NEXT:    mfhi $12
 ; MIPS32-NEXT:    addu $10, $10, $11
 ; MIPS32-NEXT:    sltu $11, $10, $11
-; MIPS32-NEXT:    ori $13, $zero, 1
-; MIPS32-NEXT:    and $11, $11, $13
+; MIPS32-NEXT:    andi $11, $11, 1
 ; MIPS32-NEXT:    addu $10, $10, $12
 ; MIPS32-NEXT:    sltu $12, $10, $12
-; MIPS32-NEXT:    and $12, $12, $13
+; MIPS32-NEXT:    andi $12, $12, 1
 ; MIPS32-NEXT:    addu $11, $11, $12
 ; MIPS32-NEXT:    mul $12, $3, $4
-; MIPS32-NEXT:    mul $14, $2, $5
-; MIPS32-NEXT:    mul $15, $1, $6
+; MIPS32-NEXT:    mul $13, $2, $5
+; MIPS32-NEXT:    mul $14, $1, $6
 ; MIPS32-NEXT:    multu $2, $4
-; MIPS32-NEXT:    mfhi $24
+; MIPS32-NEXT:    mfhi $15
 ; MIPS32-NEXT:    multu $1, $5
-; MIPS32-NEXT:    mfhi $25
+; MIPS32-NEXT:    mfhi $24
+; MIPS32-NEXT:    addu $12, $12, $13
+; MIPS32-NEXT:    sltu $13, $12, $13
+; MIPS32-NEXT:    andi $13, $13, 1
 ; MIPS32-NEXT:    addu $12, $12, $14
 ; MIPS32-NEXT:    sltu $14, $12, $14
-; MIPS32-NEXT:    and $14, $14, $13
+; MIPS32-NEXT:    andi $14, $14, 1
+; MIPS32-NEXT:    addu $13, $13, $14
 ; MIPS32-NEXT:    addu $12, $12, $15
-; MIPS32-NEXT:    sltu $15, $12, $15
-; MIPS32-NEXT:    and $15, $15, $13
-; MIPS32-NEXT:    addu $14, $14, $15
+; MIPS32-NEXT:    sltu $14, $12, $15
+; MIPS32-NEXT:    andi $14, $14, 1
+; MIPS32-NEXT:    addu $13, $13, $14
 ; MIPS32-NEXT:    addu $12, $12, $24
-; MIPS32-NEXT:    sltu $15, $12, $24
-; MIPS32-NEXT:    and $15, $15, $13
-; MIPS32-NEXT:    addu $14, $14, $15
-; MIPS32-NEXT:    addu $12, $12, $25
-; MIPS32-NEXT:    sltu $15, $12, $25
-; MIPS32-NEXT:    and $15, $15, $13
-; MIPS32-NEXT:    addu $14, $14, $15
+; MIPS32-NEXT:    sltu $14, $12, $24
+; MIPS32-NEXT:    andi $14, $14, 1
+; MIPS32-NEXT:    addu $13, $13, $14
 ; MIPS32-NEXT:    addu $12, $12, $11
 ; MIPS32-NEXT:    sltu $11, $12, $11
-; MIPS32-NEXT:    and $11, $11, $13
-; MIPS32-NEXT:    addu $11, $14, $11
+; MIPS32-NEXT:    andi $11, $11, 1
+; MIPS32-NEXT:    addu $11, $13, $11
 ; MIPS32-NEXT:    mul $8, $8, $4
 ; MIPS32-NEXT:    mul $13, $3, $5
 ; MIPS32-NEXT:    mul $14, $2, $6
@@ -186,11 +183,8 @@ define void @umul_with_overflow(i32 %lhs, i32 %rhs, i32* %pmul, i1* %pcarry_flag
 ; MIPS32-NEXT:    mul $1, $4, $5
 ; MIPS32-NEXT:    multu $4, $5
 ; MIPS32-NEXT:    mfhi $2
-; MIPS32-NEXT:    ori $3, $zero, 0
-; MIPS32-NEXT:    xor $2, $2, $3
 ; MIPS32-NEXT:    sltu $2, $zero, $2
-; MIPS32-NEXT:    ori $3, $zero, 1
-; MIPS32-NEXT:    and $2, $2, $3
+; MIPS32-NEXT:    andi $2, $2, 1
 ; MIPS32-NEXT:    sb $2, 0($7)
 ; MIPS32-NEXT:    sw $1, 0($6)
 ; MIPS32-NEXT:    jr $ra
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/mul_vec.ll b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/mul_vec.ll
new file mode 100644
index 000000000..d3f085c23
--- /dev/null
+++ b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/mul_vec.ll
@@ -0,0 +1,70 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -O0 -mtriple=mipsel-linux-gnu -global-isel -mcpu=mips32r5 -mattr=+msa,+fp64,+nan2008 -verify-machineinstrs %s -o -| FileCheck %s -check-prefixes=P5600
+
+define void @mul_v16i8(<16 x i8>* %a, <16 x i8>* %b, <16 x i8>* %c) {
+; P5600-LABEL: mul_v16i8:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.b $w0, 0($4)
+; P5600-NEXT:    ld.b $w1, 0($5)
+; P5600-NEXT:    mulv.b $w0, $w1, $w0
+; P5600-NEXT:    st.b $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <16 x i8>, <16 x i8>* %a, align 16
+  %1 = load <16 x i8>, <16 x i8>* %b, align 16
+  %mul = mul <16 x i8> %1, %0
+  store <16 x i8> %mul, <16 x i8>* %c, align 16
+  ret void
+}
+
+define void @mul_v8i16(<8 x i16>* %a, <8 x i16>* %b, <8 x i16>* %c) {
+; P5600-LABEL: mul_v8i16:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.h $w0, 0($4)
+; P5600-NEXT:    ld.h $w1, 0($5)
+; P5600-NEXT:    mulv.h $w0, $w1, $w0
+; P5600-NEXT:    st.h $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <8 x i16>, <8 x i16>* %a, align 16
+  %1 = load <8 x i16>, <8 x i16>* %b, align 16
+  %mul = mul <8 x i16> %1, %0
+  store <8 x i16> %mul, <8 x i16>* %c, align 16
+  ret void
+}
+
+define void @mul_v4i32(<4 x i32>* %a, <4 x i32>* %b, <4 x i32>* %c) {
+; P5600-LABEL: mul_v4i32:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.w $w0, 0($4)
+; P5600-NEXT:    ld.w $w1, 0($5)
+; P5600-NEXT:    mulv.w $w0, $w1, $w0
+; P5600-NEXT:    st.w $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <4 x i32>, <4 x i32>* %a, align 16
+  %1 = load <4 x i32>, <4 x i32>* %b, align 16
+  %mul = mul <4 x i32> %1, %0
+  store <4 x i32> %mul, <4 x i32>* %c, align 16
+  ret void
+}
+
+define void @mul_v2i64(<2 x i64>* %a, <2 x i64>* %b, <2 x i64>* %c) {
+; P5600-LABEL: mul_v2i64:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.d $w0, 0($4)
+; P5600-NEXT:    ld.d $w1, 0($5)
+; P5600-NEXT:    mulv.d $w0, $w1, $w0
+; P5600-NEXT:    st.d $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <2 x i64>, <2 x i64>* %a, align 16
+  %1 = load <2 x i64>, <2 x i64>* %b, align 16
+  %mul = mul <2 x i64> %1, %0
+  store <2 x i64> %mul, <2 x i64>* %c, align 16
+  ret void
+}
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/mul_vec_builtin.ll b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/mul_vec_builtin.ll
new file mode 100644
index 000000000..0e5be5032
--- /dev/null
+++ b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/mul_vec_builtin.ll
@@ -0,0 +1,74 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -O0 -mtriple=mipsel-linux-gnu -global-isel -mcpu=mips32r5 -mattr=+msa,+fp64,+nan2008 -verify-machineinstrs %s -o -| FileCheck %s -check-prefixes=P5600
+
+declare <16 x i8> @llvm.mips.mulv.b(<16 x i8>, <16 x i8>)
+define void @mul_v16i8_builtin(<16 x i8>* %a, <16 x i8>* %b, <16 x i8>* %c) {
+; P5600-LABEL: mul_v16i8_builtin:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.b $w0, 0($4)
+; P5600-NEXT:    ld.b $w1, 0($5)
+; P5600-NEXT:    mulv.b $w0, $w0, $w1
+; P5600-NEXT:    st.b $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <16 x i8>, <16 x i8>* %a, align 16
+  %1 = load <16 x i8>, <16 x i8>* %b, align 16
+  %2 = tail call <16 x i8> @llvm.mips.mulv.b(<16 x i8> %0, <16 x i8> %1)
+  store <16 x i8> %2, <16 x i8>* %c, align 16
+  ret void
+}
+
+declare <8 x i16> @llvm.mips.mulv.h(<8 x i16>, <8 x i16>)
+define void @mul_v8i16_builtin(<8 x i16>* %a, <8 x i16>* %b, <8 x i16>* %c) {
+; P5600-LABEL: mul_v8i16_builtin:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.h $w0, 0($4)
+; P5600-NEXT:    ld.h $w1, 0($5)
+; P5600-NEXT:    mulv.h $w0, $w0, $w1
+; P5600-NEXT:    st.h $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <8 x i16>, <8 x i16>* %a, align 16
+  %1 = load <8 x i16>, <8 x i16>* %b, align 16
+  %2 = tail call <8 x i16> @llvm.mips.mulv.h(<8 x i16> %0, <8 x i16> %1)
+  store <8 x i16> %2, <8 x i16>* %c, align 16
+  ret void
+}
+
+declare <4 x i32> @llvm.mips.mulv.w(<4 x i32>, <4 x i32>)
+define void @mul_v4i32_builtin(<4 x i32>* %a, <4 x i32>* %b, <4 x i32>* %c) {
+; P5600-LABEL: mul_v4i32_builtin:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.w $w0, 0($4)
+; P5600-NEXT:    ld.w $w1, 0($5)
+; P5600-NEXT:    mulv.w $w0, $w0, $w1
+; P5600-NEXT:    st.w $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <4 x i32>, <4 x i32>* %a, align 16
+  %1 = load <4 x i32>, <4 x i32>* %b, align 16
+  %2 = tail call <4 x i32> @llvm.mips.mulv.w(<4 x i32> %0, <4 x i32> %1)
+  store <4 x i32> %2, <4 x i32>* %c, align 16
+  ret void
+}
+
+declare <2 x i64> @llvm.mips.mulv.d(<2 x i64>, <2 x i64>)
+define void @mul_v2i64_builtin(<2 x i64>* %a, <2 x i64>* %b, <2 x i64>* %c) {
+; P5600-LABEL: mul_v2i64_builtin:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.d $w0, 0($4)
+; P5600-NEXT:    ld.d $w1, 0($5)
+; P5600-NEXT:    mulv.d $w0, $w0, $w1
+; P5600-NEXT:    st.d $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <2 x i64>, <2 x i64>* %a, align 16
+  %1 = load <2 x i64>, <2 x i64>* %b, align 16
+  %2 = tail call <2 x i64> @llvm.mips.mulv.d(<2 x i64> %0, <2 x i64> %1)
+  store <2 x i64> %2, <2 x i64>* %c, align 16
+  ret void
+}
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/phi.ll b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/phi.ll
index 91c8e608e..4facc766f 100644
--- a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/phi.ll
+++ b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/phi.ll
@@ -6,8 +6,7 @@ define i1 @phi_i1(i1 %cnd, i1 %a, i1 %b) {
 ; MIPS32:       # %bb.0: # %entry
 ; MIPS32-NEXT:    addiu $sp, $sp, -16
 ; MIPS32-NEXT:    .cfi_def_cfa_offset 16
-; MIPS32-NEXT:    ori $1, $zero, 1
-; MIPS32-NEXT:    and $1, $4, $1
+; MIPS32-NEXT:    andi $1, $4, 1
 ; MIPS32-NEXT:    sw $5, 12($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:    sw $6, 8($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:    bnez $1, $BB0_2
@@ -48,8 +47,7 @@ define i8 @phi_i8(i1 %cnd, i8 %a, i8 %b) {
 ; MIPS32:       # %bb.0: # %entry
 ; MIPS32-NEXT:    addiu $sp, $sp, -16
 ; MIPS32-NEXT:    .cfi_def_cfa_offset 16
-; MIPS32-NEXT:    ori $1, $zero, 1
-; MIPS32-NEXT:    and $1, $4, $1
+; MIPS32-NEXT:    andi $1, $4, 1
 ; MIPS32-NEXT:    sw $5, 12($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:    sw $6, 8($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:    bnez $1, $BB1_2
@@ -90,8 +88,7 @@ define i16 @phi_i16(i1 %cnd, i16 %a, i16 %b) {
 ; MIPS32:       # %bb.0: # %entry
 ; MIPS32-NEXT:    addiu $sp, $sp, -16
 ; MIPS32-NEXT:    .cfi_def_cfa_offset 16
-; MIPS32-NEXT:    ori $1, $zero, 1
-; MIPS32-NEXT:    and $1, $4, $1
+; MIPS32-NEXT:    andi $1, $4, 1
 ; MIPS32-NEXT:    sw $5, 12($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:    sw $6, 8($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:    bnez $1, $BB2_2
@@ -132,8 +129,7 @@ define i32 @phi_i32(i1 %cnd, i32 %a, i32 %b) {
 ; MIPS32:       # %bb.0: # %entry
 ; MIPS32-NEXT:    addiu $sp, $sp, -16
 ; MIPS32-NEXT:    .cfi_def_cfa_offset 16
-; MIPS32-NEXT:    ori $1, $zero, 1
-; MIPS32-NEXT:    and $1, $4, $1
+; MIPS32-NEXT:    andi $1, $4, 1
 ; MIPS32-NEXT:    sw $5, 12($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:    sw $6, 8($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:    bnez $1, $BB3_2
@@ -178,8 +174,7 @@ define i64 @phi_i64(i1 %cnd, i64 %a, i64 %b) {
 ; MIPS32-NEXT:    lw $1, 0($1)
 ; MIPS32-NEXT:    addiu $2, $sp, 44
 ; MIPS32-NEXT:    lw $2, 0($2)
-; MIPS32-NEXT:    ori $3, $zero, 1
-; MIPS32-NEXT:    and $3, $4, $3
+; MIPS32-NEXT:    andi $3, $4, 1
 ; MIPS32-NEXT:    sw $1, 20($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:    sw $6, 16($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:    sw $7, 12($sp) # 4-byte Folded Spill
@@ -229,8 +224,7 @@ define void @phi_ambiguous_i64_in_fpr(i1 %cnd, i64* %i64_ptr_a, i64* %i64_ptr_b,
 ; MIPS32-NEXT:    .cfi_def_cfa_offset 32
 ; MIPS32-NEXT:    ldc1 $f0, 0($5)
 ; MIPS32-NEXT:    ldc1 $f2, 0($6)
-; MIPS32-NEXT:    ori $1, $zero, 1
-; MIPS32-NEXT:    and $1, $4, $1
+; MIPS32-NEXT:    andi $1, $4, 1
 ; MIPS32-NEXT:    sw $7, 28($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:    sdc1 $f0, 16($sp) # 8-byte Folded Spill
 ; MIPS32-NEXT:    sdc1 $f2, 8($sp) # 8-byte Folded Spill
@@ -278,8 +272,7 @@ define float @phi_float(i1 %cnd, float %a, float %b) {
 ; MIPS32-NEXT:    .cfi_def_cfa_offset 16
 ; MIPS32-NEXT:    mtc1 $5, $f0
 ; MIPS32-NEXT:    mtc1 $6, $f1
-; MIPS32-NEXT:    ori $1, $zero, 1
-; MIPS32-NEXT:    and $1, $4, $1
+; MIPS32-NEXT:    andi $1, $4, 1
 ; MIPS32-NEXT:    swc1 $f0, 12($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:    swc1 $f1, 8($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:    bnez $1, $BB6_2
@@ -321,8 +314,7 @@ define void @phi_ambiguous_float_in_gpr(i1 %cnd, float* %f32_ptr_a, float* %f32_
 ; MIPS32-NEXT:    .cfi_def_cfa_offset 16
 ; MIPS32-NEXT:    lw $1, 0($5)
 ; MIPS32-NEXT:    lw $2, 0($6)
-; MIPS32-NEXT:    ori $3, $zero, 1
-; MIPS32-NEXT:    and $3, $4, $3
+; MIPS32-NEXT:    andi $3, $4, 1
 ; MIPS32-NEXT:    sw $1, 12($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:    sw $7, 8($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:    sw $2, 4($sp) # 4-byte Folded Spill
@@ -370,8 +362,7 @@ define double @phi_double(double %a, double %b, i1 %cnd) {
 ; MIPS32-NEXT:    .cfi_def_cfa_offset 24
 ; MIPS32-NEXT:    addiu $1, $sp, 40
 ; MIPS32-NEXT:    lw $1, 0($1)
-; MIPS32-NEXT:    ori $2, $zero, 1
-; MIPS32-NEXT:    and $1, $1, $2
+; MIPS32-NEXT:    andi $1, $1, 1
 ; MIPS32-NEXT:    sdc1 $f12, 16($sp) # 8-byte Folded Spill
 ; MIPS32-NEXT:    sdc1 $f14, 8($sp) # 8-byte Folded Spill
 ; MIPS32-NEXT:    bnez $1, $BB8_2
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/rem_and_div.ll b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/rem_and_div.ll
index 7ce74bb68..d2520daf6 100644
--- a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/rem_and_div.ll
+++ b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/rem_and_div.ll
@@ -157,11 +157,10 @@ entry:
 define signext i8 @udiv_i8(i8 signext %a, i8 signext %b) {
 ; MIPS32-LABEL: udiv_i8:
 ; MIPS32:       # %bb.0: # %entry
-; MIPS32-NEXT:    ori $1, $zero, 255
-; MIPS32-NEXT:    and $2, $5, $1
-; MIPS32-NEXT:    and $1, $4, $1
-; MIPS32-NEXT:    divu $zero, $2, $1
-; MIPS32-NEXT:    teq $1, $zero, 7
+; MIPS32-NEXT:    andi $1, $5, 255
+; MIPS32-NEXT:    andi $2, $4, 255
+; MIPS32-NEXT:    divu $zero, $1, $2
+; MIPS32-NEXT:    teq $2, $zero, 7
 ; MIPS32-NEXT:    mflo $1
 ; MIPS32-NEXT:    sll $1, $1, 24
 ; MIPS32-NEXT:    sra $2, $1, 24
@@ -175,11 +174,10 @@ entry:
 define signext i16 @udiv_i16(i16 signext %a, i16 signext %b) {
 ; MIPS32-LABEL: udiv_i16:
 ; MIPS32:       # %bb.0: # %entry
-; MIPS32-NEXT:    ori $1, $zero, 65535
-; MIPS32-NEXT:    and $2, $5, $1
-; MIPS32-NEXT:    and $1, $4, $1
-; MIPS32-NEXT:    divu $zero, $2, $1
-; MIPS32-NEXT:    teq $1, $zero, 7
+; MIPS32-NEXT:    andi $1, $5, 65535
+; MIPS32-NEXT:    andi $2, $4, 65535
+; MIPS32-NEXT:    divu $zero, $1, $2
+; MIPS32-NEXT:    teq $2, $zero, 7
 ; MIPS32-NEXT:    mflo $1
 ; MIPS32-NEXT:    sll $1, $1, 16
 ; MIPS32-NEXT:    sra $2, $1, 16
@@ -231,11 +229,10 @@ entry:
 define signext i8 @urem_i8(i8 signext %a, i8 signext %b) {
 ; MIPS32-LABEL: urem_i8:
 ; MIPS32:       # %bb.0: # %entry
-; MIPS32-NEXT:    ori $1, $zero, 255
-; MIPS32-NEXT:    and $2, $5, $1
-; MIPS32-NEXT:    and $1, $4, $1
-; MIPS32-NEXT:    divu $zero, $2, $1
-; MIPS32-NEXT:    teq $1, $zero, 7
+; MIPS32-NEXT:    andi $1, $5, 255
+; MIPS32-NEXT:    andi $2, $4, 255
+; MIPS32-NEXT:    divu $zero, $1, $2
+; MIPS32-NEXT:    teq $2, $zero, 7
 ; MIPS32-NEXT:    mfhi $1
 ; MIPS32-NEXT:    sll $1, $1, 24
 ; MIPS32-NEXT:    sra $2, $1, 24
@@ -249,11 +246,10 @@ entry:
 define signext i16 @urem_i16(i16 signext %a, i16 signext %b) {
 ; MIPS32-LABEL: urem_i16:
 ; MIPS32:       # %bb.0: # %entry
-; MIPS32-NEXT:    ori $1, $zero, 65535
-; MIPS32-NEXT:    and $2, $5, $1
-; MIPS32-NEXT:    and $1, $4, $1
-; MIPS32-NEXT:    divu $zero, $2, $1
-; MIPS32-NEXT:    teq $1, $zero, 7
+; MIPS32-NEXT:    andi $1, $5, 65535
+; MIPS32-NEXT:    andi $2, $4, 65535
+; MIPS32-NEXT:    divu $zero, $1, $2
+; MIPS32-NEXT:    teq $2, $zero, 7
 ; MIPS32-NEXT:    mfhi $1
 ; MIPS32-NEXT:    sll $1, $1, 16
 ; MIPS32-NEXT:    sra $2, $1, 16
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/rem_and_div_vec.ll b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/rem_and_div_vec.ll
new file mode 100644
index 000000000..29b41b454
--- /dev/null
+++ b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/rem_and_div_vec.ll
@@ -0,0 +1,274 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -O0 -mtriple=mipsel-linux-gnu -global-isel -mcpu=mips32r5 -mattr=+msa,+fp64,+nan2008 -verify-machineinstrs %s -o -| FileCheck %s -check-prefixes=P5600
+
+define void @sdiv_v16i8(<16 x i8>* %a, <16 x i8>* %b, <16 x i8>* %c) {
+; P5600-LABEL: sdiv_v16i8:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.b $w0, 0($4)
+; P5600-NEXT:    ld.b $w1, 0($5)
+; P5600-NEXT:    div_s.b $w0, $w0, $w1
+; P5600-NEXT:    st.b $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <16 x i8>, <16 x i8>* %a, align 16
+  %1 = load <16 x i8>, <16 x i8>* %b, align 16
+  %div = sdiv <16 x i8> %0, %1
+  store <16 x i8> %div, <16 x i8>* %c, align 16
+  ret void
+}
+
+define void @sdiv_v8i16(<8 x i16>* %a, <8 x i16>* %b, <8 x i16>* %c) {
+; P5600-LABEL: sdiv_v8i16:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.h $w0, 0($4)
+; P5600-NEXT:    ld.h $w1, 0($5)
+; P5600-NEXT:    div_s.h $w0, $w0, $w1
+; P5600-NEXT:    st.h $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <8 x i16>, <8 x i16>* %a, align 16
+  %1 = load <8 x i16>, <8 x i16>* %b, align 16
+  %div = sdiv <8 x i16> %0, %1
+  store <8 x i16> %div, <8 x i16>* %c, align 16
+  ret void
+}
+
+define void @sdiv_v4i32(<4 x i32>* %a, <4 x i32>* %b, <4 x i32>* %c) {
+; P5600-LABEL: sdiv_v4i32:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.w $w0, 0($4)
+; P5600-NEXT:    ld.w $w1, 0($5)
+; P5600-NEXT:    div_s.w $w0, $w0, $w1
+; P5600-NEXT:    st.w $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <4 x i32>, <4 x i32>* %a, align 16
+  %1 = load <4 x i32>, <4 x i32>* %b, align 16
+  %div = sdiv <4 x i32> %0, %1
+  store <4 x i32> %div, <4 x i32>* %c, align 16
+  ret void
+}
+
+define void @sdiv_v2i64(<2 x i64>* %a, <2 x i64>* %b, <2 x i64>* %c) {
+; P5600-LABEL: sdiv_v2i64:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.d $w0, 0($4)
+; P5600-NEXT:    ld.d $w1, 0($5)
+; P5600-NEXT:    div_s.d $w0, $w0, $w1
+; P5600-NEXT:    st.d $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <2 x i64>, <2 x i64>* %a, align 16
+  %1 = load <2 x i64>, <2 x i64>* %b, align 16
+  %div = sdiv <2 x i64> %0, %1
+  store <2 x i64> %div, <2 x i64>* %c, align 16
+  ret void
+}
+
+define void @srem_v16i8(<16 x i8>* %a, <16 x i8>* %b, <16 x i8>* %c) {
+; P5600-LABEL: srem_v16i8:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.b $w0, 0($4)
+; P5600-NEXT:    ld.b $w1, 0($5)
+; P5600-NEXT:    mod_s.b $w0, $w0, $w1
+; P5600-NEXT:    st.b $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <16 x i8>, <16 x i8>* %a, align 16
+  %1 = load <16 x i8>, <16 x i8>* %b, align 16
+  %rem = srem <16 x i8> %0, %1
+  store <16 x i8> %rem, <16 x i8>* %c, align 16
+  ret void
+}
+
+define void @srem_v8i16(<8 x i16>* %a, <8 x i16>* %b, <8 x i16>* %c) {
+; P5600-LABEL: srem_v8i16:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.h $w0, 0($4)
+; P5600-NEXT:    ld.h $w1, 0($5)
+; P5600-NEXT:    mod_s.h $w0, $w0, $w1
+; P5600-NEXT:    st.h $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <8 x i16>, <8 x i16>* %a, align 16
+  %1 = load <8 x i16>, <8 x i16>* %b, align 16
+  %rem = srem <8 x i16> %0, %1
+  store <8 x i16> %rem, <8 x i16>* %c, align 16
+  ret void
+}
+
+define void @srem_v4i32(<4 x i32>* %a, <4 x i32>* %b, <4 x i32>* %c) {
+; P5600-LABEL: srem_v4i32:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.w $w0, 0($4)
+; P5600-NEXT:    ld.w $w1, 0($5)
+; P5600-NEXT:    mod_s.w $w0, $w0, $w1
+; P5600-NEXT:    st.w $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <4 x i32>, <4 x i32>* %a, align 16
+  %1 = load <4 x i32>, <4 x i32>* %b, align 16
+  %rem = srem <4 x i32> %0, %1
+  store <4 x i32> %rem, <4 x i32>* %c, align 16
+  ret void
+}
+
+define void @srem_v2i64(<2 x i64>* %a, <2 x i64>* %b, <2 x i64>* %c) {
+; P5600-LABEL: srem_v2i64:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.d $w0, 0($4)
+; P5600-NEXT:    ld.d $w1, 0($5)
+; P5600-NEXT:    mod_s.d $w0, $w0, $w1
+; P5600-NEXT:    st.d $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <2 x i64>, <2 x i64>* %a, align 16
+  %1 = load <2 x i64>, <2 x i64>* %b, align 16
+  %rem = srem <2 x i64> %0, %1
+  store <2 x i64> %rem, <2 x i64>* %c, align 16
+  ret void
+}
+
+define void @udiv_v16u8(<16 x i8>* %a, <16 x i8>* %b, <16 x i8>* %c) {
+; P5600-LABEL: udiv_v16u8:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.b $w0, 0($4)
+; P5600-NEXT:    ld.b $w1, 0($5)
+; P5600-NEXT:    div_u.b $w0, $w0, $w1
+; P5600-NEXT:    st.b $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <16 x i8>, <16 x i8>* %a, align 16
+  %1 = load <16 x i8>, <16 x i8>* %b, align 16
+  %div = udiv <16 x i8> %0, %1
+  store <16 x i8> %div, <16 x i8>* %c, align 16
+  ret void
+}
+
+define void @udiv_v8u16(<8 x i16>* %a, <8 x i16>* %b, <8 x i16>* %c) {
+; P5600-LABEL: udiv_v8u16:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.h $w0, 0($4)
+; P5600-NEXT:    ld.h $w1, 0($5)
+; P5600-NEXT:    div_u.h $w0, $w0, $w1
+; P5600-NEXT:    st.h $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <8 x i16>, <8 x i16>* %a, align 16
+  %1 = load <8 x i16>, <8 x i16>* %b, align 16
+  %div = udiv <8 x i16> %0, %1
+  store <8 x i16> %div, <8 x i16>* %c, align 16
+  ret void
+}
+
+define void @udiv_v4u32(<4 x i32>* %a, <4 x i32>* %b, <4 x i32>* %c) {
+; P5600-LABEL: udiv_v4u32:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.w $w0, 0($4)
+; P5600-NEXT:    ld.w $w1, 0($5)
+; P5600-NEXT:    div_u.w $w0, $w0, $w1
+; P5600-NEXT:    st.w $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <4 x i32>, <4 x i32>* %a, align 16
+  %1 = load <4 x i32>, <4 x i32>* %b, align 16
+  %div = udiv <4 x i32> %0, %1
+  store <4 x i32> %div, <4 x i32>* %c, align 16
+  ret void
+}
+
+define void @udiv_v2u64(<2 x i64>* %a, <2 x i64>* %b, <2 x i64>* %c) {
+; P5600-LABEL: udiv_v2u64:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.d $w0, 0($4)
+; P5600-NEXT:    ld.d $w1, 0($5)
+; P5600-NEXT:    div_u.d $w0, $w0, $w1
+; P5600-NEXT:    st.d $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <2 x i64>, <2 x i64>* %a, align 16
+  %1 = load <2 x i64>, <2 x i64>* %b, align 16
+  %div = udiv <2 x i64> %0, %1
+  store <2 x i64> %div, <2 x i64>* %c, align 16
+  ret void
+}
+
+define void @urem_v16u8(<16 x i8>* %a, <16 x i8>* %b, <16 x i8>* %c) {
+; P5600-LABEL: urem_v16u8:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.b $w0, 0($4)
+; P5600-NEXT:    ld.b $w1, 0($5)
+; P5600-NEXT:    mod_u.b $w0, $w0, $w1
+; P5600-NEXT:    st.b $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <16 x i8>, <16 x i8>* %a, align 16
+  %1 = load <16 x i8>, <16 x i8>* %b, align 16
+  %rem = urem <16 x i8> %0, %1
+  store <16 x i8> %rem, <16 x i8>* %c, align 16
+  ret void
+}
+
+define void @urem_v8u16(<8 x i16>* %a, <8 x i16>* %b, <8 x i16>* %c) {
+; P5600-LABEL: urem_v8u16:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.h $w0, 0($4)
+; P5600-NEXT:    ld.h $w1, 0($5)
+; P5600-NEXT:    mod_u.h $w0, $w0, $w1
+; P5600-NEXT:    st.h $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <8 x i16>, <8 x i16>* %a, align 16
+  %1 = load <8 x i16>, <8 x i16>* %b, align 16
+  %rem = urem <8 x i16> %0, %1
+  store <8 x i16> %rem, <8 x i16>* %c, align 16
+  ret void
+}
+
+define void @urem_v4u32(<4 x i32>* %a, <4 x i32>* %b, <4 x i32>* %c) {
+; P5600-LABEL: urem_v4u32:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.w $w0, 0($4)
+; P5600-NEXT:    ld.w $w1, 0($5)
+; P5600-NEXT:    mod_u.w $w0, $w0, $w1
+; P5600-NEXT:    st.w $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <4 x i32>, <4 x i32>* %a, align 16
+  %1 = load <4 x i32>, <4 x i32>* %b, align 16
+  %rem = urem <4 x i32> %0, %1
+  store <4 x i32> %rem, <4 x i32>* %c, align 16
+  ret void
+}
+
+define void @urem_v2u64(<2 x i64>* %a, <2 x i64>* %b, <2 x i64>* %c) {
+; P5600-LABEL: urem_v2u64:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.d $w0, 0($4)
+; P5600-NEXT:    ld.d $w1, 0($5)
+; P5600-NEXT:    mod_u.d $w0, $w0, $w1
+; P5600-NEXT:    st.d $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <2 x i64>, <2 x i64>* %a, align 16
+  %1 = load <2 x i64>, <2 x i64>* %b, align 16
+  %rem = urem <2 x i64> %0, %1
+  store <2 x i64> %rem, <2 x i64>* %c, align 16
+  ret void
+}
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/rem_and_div_vec_builtin.ll b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/rem_and_div_vec_builtin.ll
new file mode 100644
index 000000000..8246971fe
--- /dev/null
+++ b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/rem_and_div_vec_builtin.ll
@@ -0,0 +1,290 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -O0 -mtriple=mipsel-linux-gnu -global-isel -mcpu=mips32r5 -mattr=+msa,+fp64,+nan2008 -verify-machineinstrs %s -o -| FileCheck %s -check-prefixes=P5600
+
+declare <16 x i8> @llvm.mips.div.s.b(<16 x i8>, <16 x i8>)
+define void @sdiv_v16i8_builtin(<16 x i8>* %a, <16 x i8>* %b, <16 x i8>* %c) {
+; P5600-LABEL: sdiv_v16i8_builtin:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.b $w0, 0($4)
+; P5600-NEXT:    ld.b $w1, 0($5)
+; P5600-NEXT:    div_s.b $w0, $w0, $w1
+; P5600-NEXT:    st.b $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <16 x i8>, <16 x i8>* %a, align 16
+  %1 = load <16 x i8>, <16 x i8>* %b, align 16
+  %2 = tail call <16 x i8> @llvm.mips.div.s.b(<16 x i8> %0, <16 x i8> %1)
+  store <16 x i8> %2, <16 x i8>* %c, align 16
+  ret void
+}
+
+declare <8 x i16> @llvm.mips.div.s.h(<8 x i16>, <8 x i16>)
+define void @sdiv_v8i16_builtin(<8 x i16>* %a, <8 x i16>* %b, <8 x i16>* %c) {
+; P5600-LABEL: sdiv_v8i16_builtin:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.h $w0, 0($4)
+; P5600-NEXT:    ld.h $w1, 0($5)
+; P5600-NEXT:    div_s.h $w0, $w0, $w1
+; P5600-NEXT:    st.h $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <8 x i16>, <8 x i16>* %a, align 16
+  %1 = load <8 x i16>, <8 x i16>* %b, align 16
+  %2 = tail call <8 x i16> @llvm.mips.div.s.h(<8 x i16> %0, <8 x i16> %1)
+  store <8 x i16> %2, <8 x i16>* %c, align 16
+  ret void
+}
+
+declare <4 x i32> @llvm.mips.div.s.w(<4 x i32>, <4 x i32>)
+define void @sdiv_v4i32_builtin(<4 x i32>* %a, <4 x i32>* %b, <4 x i32>* %c) {
+; P5600-LABEL: sdiv_v4i32_builtin:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.w $w0, 0($4)
+; P5600-NEXT:    ld.w $w1, 0($5)
+; P5600-NEXT:    div_s.w $w0, $w0, $w1
+; P5600-NEXT:    st.w $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <4 x i32>, <4 x i32>* %a, align 16
+  %1 = load <4 x i32>, <4 x i32>* %b, align 16
+  %2 = tail call <4 x i32> @llvm.mips.div.s.w(<4 x i32> %0, <4 x i32> %1)
+  store <4 x i32> %2, <4 x i32>* %c, align 16
+  ret void
+}
+
+declare <2 x i64> @llvm.mips.div.s.d(<2 x i64>, <2 x i64>)
+define void @sdiv_v2i64_builtin(<2 x i64>* %a, <2 x i64>* %b, <2 x i64>* %c) {
+; P5600-LABEL: sdiv_v2i64_builtin:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.d $w0, 0($4)
+; P5600-NEXT:    ld.d $w1, 0($5)
+; P5600-NEXT:    div_s.d $w0, $w0, $w1
+; P5600-NEXT:    st.d $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <2 x i64>, <2 x i64>* %a, align 16
+  %1 = load <2 x i64>, <2 x i64>* %b, align 16
+  %2 = tail call <2 x i64> @llvm.mips.div.s.d(<2 x i64> %0, <2 x i64> %1)
+  store <2 x i64> %2, <2 x i64>* %c, align 16
+  ret void
+}
+
+declare <16 x i8> @llvm.mips.mod.s.b(<16 x i8>, <16 x i8>)
+define void @smod_v16i8_builtin(<16 x i8>* %a, <16 x i8>* %b, <16 x i8>* %c) {
+; P5600-LABEL: smod_v16i8_builtin:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.b $w0, 0($4)
+; P5600-NEXT:    ld.b $w1, 0($5)
+; P5600-NEXT:    mod_s.b $w0, $w0, $w1
+; P5600-NEXT:    st.b $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <16 x i8>, <16 x i8>* %a, align 16
+  %1 = load <16 x i8>, <16 x i8>* %b, align 16
+  %2 = tail call <16 x i8> @llvm.mips.mod.s.b(<16 x i8> %0, <16 x i8> %1)
+  store <16 x i8> %2, <16 x i8>* %c, align 16
+  ret void
+}
+
+declare <8 x i16> @llvm.mips.mod.s.h(<8 x i16>, <8 x i16>)
+define void @smod_v8i16_builtin(<8 x i16>* %a, <8 x i16>* %b, <8 x i16>* %c) {
+; P5600-LABEL: smod_v8i16_builtin:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.h $w0, 0($4)
+; P5600-NEXT:    ld.h $w1, 0($5)
+; P5600-NEXT:    mod_s.h $w0, $w0, $w1
+; P5600-NEXT:    st.h $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <8 x i16>, <8 x i16>* %a, align 16
+  %1 = load <8 x i16>, <8 x i16>* %b, align 16
+  %2 = tail call <8 x i16> @llvm.mips.mod.s.h(<8 x i16> %0, <8 x i16> %1)
+  store <8 x i16> %2, <8 x i16>* %c, align 16
+  ret void
+}
+
+declare <4 x i32> @llvm.mips.mod.s.w(<4 x i32>, <4 x i32>)
+define void @smod_v4i32_builtin(<4 x i32>* %a, <4 x i32>* %b, <4 x i32>* %c) {
+; P5600-LABEL: smod_v4i32_builtin:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.w $w0, 0($4)
+; P5600-NEXT:    ld.w $w1, 0($5)
+; P5600-NEXT:    mod_s.w $w0, $w0, $w1
+; P5600-NEXT:    st.w $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <4 x i32>, <4 x i32>* %a, align 16
+  %1 = load <4 x i32>, <4 x i32>* %b, align 16
+  %2 = tail call <4 x i32> @llvm.mips.mod.s.w(<4 x i32> %0, <4 x i32> %1)
+  store <4 x i32> %2, <4 x i32>* %c, align 16
+  ret void
+}
+
+declare <2 x i64> @llvm.mips.mod.s.d(<2 x i64>, <2 x i64>)
+define void @smod_v2i64_builtin(<2 x i64>* %a, <2 x i64>* %b, <2 x i64>* %c) {
+; P5600-LABEL: smod_v2i64_builtin:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.d $w0, 0($4)
+; P5600-NEXT:    ld.d $w1, 0($5)
+; P5600-NEXT:    mod_s.d $w0, $w0, $w1
+; P5600-NEXT:    st.d $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <2 x i64>, <2 x i64>* %a, align 16
+  %1 = load <2 x i64>, <2 x i64>* %b, align 16
+  %2 = tail call <2 x i64> @llvm.mips.mod.s.d(<2 x i64> %0, <2 x i64> %1)
+  store <2 x i64> %2, <2 x i64>* %c, align 16
+  ret void
+}
+
+declare <16 x i8> @llvm.mips.div.u.b(<16 x i8>, <16 x i8>)
+define void @udiv_v16u8_builtin(<16 x i8>* %a, <16 x i8>* %b, <16 x i8>* %c) {
+; P5600-LABEL: udiv_v16u8_builtin:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.b $w0, 0($4)
+; P5600-NEXT:    ld.b $w1, 0($5)
+; P5600-NEXT:    div_u.b $w0, $w0, $w1
+; P5600-NEXT:    st.b $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <16 x i8>, <16 x i8>* %a, align 16
+  %1 = load <16 x i8>, <16 x i8>* %b, align 16
+  %2 = tail call <16 x i8> @llvm.mips.div.u.b(<16 x i8> %0, <16 x i8> %1)
+  store <16 x i8> %2, <16 x i8>* %c, align 16
+  ret void
+}
+
+declare <8 x i16> @llvm.mips.div.u.h(<8 x i16>, <8 x i16>)
+define void @udiv_v8u16_builtin(<8 x i16>* %a, <8 x i16>* %b, <8 x i16>* %c) {
+; P5600-LABEL: udiv_v8u16_builtin:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.h $w0, 0($4)
+; P5600-NEXT:    ld.h $w1, 0($5)
+; P5600-NEXT:    div_u.h $w0, $w0, $w1
+; P5600-NEXT:    st.h $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <8 x i16>, <8 x i16>* %a, align 16
+  %1 = load <8 x i16>, <8 x i16>* %b, align 16
+  %2 = tail call <8 x i16> @llvm.mips.div.u.h(<8 x i16> %0, <8 x i16> %1)
+  store <8 x i16> %2, <8 x i16>* %c, align 16
+  ret void
+}
+
+declare <4 x i32> @llvm.mips.div.u.w(<4 x i32>, <4 x i32>)
+define void @udiv_v4u32_builtin(<4 x i32>* %a, <4 x i32>* %b, <4 x i32>* %c) {
+; P5600-LABEL: udiv_v4u32_builtin:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.w $w0, 0($4)
+; P5600-NEXT:    ld.w $w1, 0($5)
+; P5600-NEXT:    div_u.w $w0, $w0, $w1
+; P5600-NEXT:    st.w $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <4 x i32>, <4 x i32>* %a, align 16
+  %1 = load <4 x i32>, <4 x i32>* %b, align 16
+  %2 = tail call <4 x i32> @llvm.mips.div.u.w(<4 x i32> %0, <4 x i32> %1)
+  store <4 x i32> %2, <4 x i32>* %c, align 16
+  ret void
+}
+
+declare <2 x i64> @llvm.mips.div.u.d(<2 x i64>, <2 x i64>)
+define void @udiv_v2u64_builtin(<2 x i64>* %a, <2 x i64>* %b, <2 x i64>* %c) {
+; P5600-LABEL: udiv_v2u64_builtin:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.d $w0, 0($4)
+; P5600-NEXT:    ld.d $w1, 0($5)
+; P5600-NEXT:    div_u.d $w0, $w0, $w1
+; P5600-NEXT:    st.d $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <2 x i64>, <2 x i64>* %a, align 16
+  %1 = load <2 x i64>, <2 x i64>* %b, align 16
+  %2 = tail call <2 x i64> @llvm.mips.div.u.d(<2 x i64> %0, <2 x i64> %1)
+  store <2 x i64> %2, <2 x i64>* %c, align 16
+  ret void
+}
+
+declare <16 x i8> @llvm.mips.mod.u.b(<16 x i8>, <16 x i8>)
+define void @umod_v16u8_builtin(<16 x i8>* %a, <16 x i8>* %b, <16 x i8>* %c) {
+; P5600-LABEL: umod_v16u8_builtin:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.b $w0, 0($4)
+; P5600-NEXT:    ld.b $w1, 0($5)
+; P5600-NEXT:    mod_u.b $w0, $w0, $w1
+; P5600-NEXT:    st.b $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <16 x i8>, <16 x i8>* %a, align 16
+  %1 = load <16 x i8>, <16 x i8>* %b, align 16
+  %2 = tail call <16 x i8> @llvm.mips.mod.u.b(<16 x i8> %0, <16 x i8> %1)
+  store <16 x i8> %2, <16 x i8>* %c, align 16
+  ret void
+}
+
+declare <8 x i16> @llvm.mips.mod.u.h(<8 x i16>, <8 x i16>)
+define void @umod_v8u16_builtin(<8 x i16>* %a, <8 x i16>* %b, <8 x i16>* %c) {
+; P5600-LABEL: umod_v8u16_builtin:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.h $w0, 0($4)
+; P5600-NEXT:    ld.h $w1, 0($5)
+; P5600-NEXT:    mod_u.h $w0, $w0, $w1
+; P5600-NEXT:    st.h $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <8 x i16>, <8 x i16>* %a, align 16
+  %1 = load <8 x i16>, <8 x i16>* %b, align 16
+  %2 = tail call <8 x i16> @llvm.mips.mod.u.h(<8 x i16> %0, <8 x i16> %1)
+  store <8 x i16> %2, <8 x i16>* %c, align 16
+  ret void
+}
+
+declare <4 x i32> @llvm.mips.mod.u.w(<4 x i32>, <4 x i32>)
+define void @umod_v4u32_builtin(<4 x i32>* %a, <4 x i32>* %b, <4 x i32>* %c) {
+; P5600-LABEL: umod_v4u32_builtin:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.w $w0, 0($4)
+; P5600-NEXT:    ld.w $w1, 0($5)
+; P5600-NEXT:    mod_u.w $w0, $w0, $w1
+; P5600-NEXT:    st.w $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <4 x i32>, <4 x i32>* %a, align 16
+  %1 = load <4 x i32>, <4 x i32>* %b, align 16
+  %2 = tail call <4 x i32> @llvm.mips.mod.u.w(<4 x i32> %0, <4 x i32> %1)
+  store <4 x i32> %2, <4 x i32>* %c, align 16
+  ret void
+}
+
+declare <2 x i64> @llvm.mips.mod.u.d(<2 x i64>, <2 x i64>)
+define void @umod_v2u64_builtin(<2 x i64>* %a, <2 x i64>* %b, <2 x i64>* %c) {
+; P5600-LABEL: umod_v2u64_builtin:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.d $w0, 0($4)
+; P5600-NEXT:    ld.d $w1, 0($5)
+; P5600-NEXT:    mod_u.d $w0, $w0, $w1
+; P5600-NEXT:    st.d $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <2 x i64>, <2 x i64>* %a, align 16
+  %1 = load <2 x i64>, <2 x i64>* %b, align 16
+  %2 = tail call <2 x i64> @llvm.mips.mod.u.d(<2 x i64> %0, <2 x i64> %1)
+  store <2 x i64> %2, <2 x i64>* %c, align 16
+  ret void
+}
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/select.ll b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/select.ll
index e7b25219c..71c3023ca 100644
--- a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/select.ll
+++ b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/select.ll
@@ -4,8 +4,7 @@
 define i8 @select_i8(i1 %test, i8 %a, i8 %b) {
 ; MIPS32-LABEL: select_i8:
 ; MIPS32:       # %bb.0: # %entry
-; MIPS32-NEXT:    ori $1, $zero, 1
-; MIPS32-NEXT:    and $1, $4, $1
+; MIPS32-NEXT:    andi $1, $4, 1
 ; MIPS32-NEXT:    movn $6, $5, $1
 ; MIPS32-NEXT:    move $2, $6
 ; MIPS32-NEXT:    jr $ra
@@ -18,8 +17,7 @@ entry:
 define i16 @select_i16(i1 %test, i16 %a, i16 %b) {
 ; MIPS32-LABEL: select_i16:
 ; MIPS32:       # %bb.0: # %entry
-; MIPS32-NEXT:    ori $1, $zero, 1
-; MIPS32-NEXT:    and $1, $4, $1
+; MIPS32-NEXT:    andi $1, $4, 1
 ; MIPS32-NEXT:    movn $6, $5, $1
 ; MIPS32-NEXT:    move $2, $6
 ; MIPS32-NEXT:    jr $ra
@@ -32,8 +30,7 @@ entry:
 define i32 @select_i32(i1 %test, i32 %a, i32 %b) {
 ; MIPS32-LABEL: select_i32:
 ; MIPS32:       # %bb.0: # %entry
-; MIPS32-NEXT:    ori $1, $zero, 1
-; MIPS32-NEXT:    and $1, $4, $1
+; MIPS32-NEXT:    andi $1, $4, 1
 ; MIPS32-NEXT:    movn $6, $5, $1
 ; MIPS32-NEXT:    move $2, $6
 ; MIPS32-NEXT:    jr $ra
@@ -46,8 +43,7 @@ entry:
 define i32* @select_ptr(i1 %test, i32* %a, i32* %b) {
 ; MIPS32-LABEL: select_ptr:
 ; MIPS32:       # %bb.0: # %entry
-; MIPS32-NEXT:    ori $1, $zero, 1
-; MIPS32-NEXT:    and $1, $4, $1
+; MIPS32-NEXT:    andi $1, $4, 1
 ; MIPS32-NEXT:    movn $6, $5, $1
 ; MIPS32-NEXT:    move $2, $6
 ; MIPS32-NEXT:    jr $ra
@@ -60,10 +56,10 @@ entry:
 define i32 @select_with_negation(i32 %a, i32 %b, i32 %x, i32 %y) {
 ; MIPS32-LABEL: select_with_negation:
 ; MIPS32:       # %bb.0: # %entry
-; MIPS32-NEXT:    slt $1, $4, $5
-; MIPS32-NEXT:    not $1, $1
-; MIPS32-NEXT:    ori $2, $zero, 1
-; MIPS32-NEXT:    and $1, $1, $2
+; MIPS32-NEXT:    ori $1, $zero, 1
+; MIPS32-NEXT:    slt $2, $4, $5
+; MIPS32-NEXT:    xor $1, $2, $1
+; MIPS32-NEXT:    andi $1, $1, 1
 ; MIPS32-NEXT:    movn $7, $6, $1
 ; MIPS32-NEXT:    move $2, $7
 ; MIPS32-NEXT:    jr $ra
@@ -84,8 +80,7 @@ define i64 @select_i64(i1 %test, i64 %a, i64 %b) {
 ; MIPS32-NEXT:    lw $1, 0($1)
 ; MIPS32-NEXT:    addiu $2, $sp, 28
 ; MIPS32-NEXT:    lw $2, 0($2)
-; MIPS32-NEXT:    ori $3, $zero, 1
-; MIPS32-NEXT:    and $3, $4, $3
+; MIPS32-NEXT:    andi $3, $4, 1
 ; MIPS32-NEXT:    movn $1, $6, $3
 ; MIPS32-NEXT:    movn $2, $7, $3
 ; MIPS32-NEXT:    sw $2, 4($sp) # 4-byte Folded Spill
@@ -104,8 +99,7 @@ define void @select_ambiguous_i64_in_fpr(i1 %test, i64* %i64_ptr_a, i64* %i64_pt
 ; MIPS32:       # %bb.0: # %entry
 ; MIPS32-NEXT:    ldc1 $f0, 0($5)
 ; MIPS32-NEXT:    ldc1 $f2, 0($6)
-; MIPS32-NEXT:    ori $1, $zero, 1
-; MIPS32-NEXT:    and $1, $4, $1
+; MIPS32-NEXT:    andi $1, $4, 1
 ; MIPS32-NEXT:    movn.d $f2, $f0, $1
 ; MIPS32-NEXT:    sdc1 $f2, 0($7)
 ; MIPS32-NEXT:    jr $ra
@@ -123,8 +117,7 @@ define float @select_float(i1 %test, float %a, float %b) {
 ; MIPS32:       # %bb.0: # %entry
 ; MIPS32-NEXT:    mtc1 $5, $f0
 ; MIPS32-NEXT:    mtc1 $6, $f1
-; MIPS32-NEXT:    ori $1, $zero, 1
-; MIPS32-NEXT:    and $1, $4, $1
+; MIPS32-NEXT:    andi $1, $4, 1
 ; MIPS32-NEXT:    movn.s $f1, $f0, $1
 ; MIPS32-NEXT:    mov.s $f0, $f1
 ; MIPS32-NEXT:    jr $ra
@@ -139,8 +132,7 @@ define void @select_ambiguous_float_in_gpr(i1 %test, float* %f32_ptr_a, float* %
 ; MIPS32:       # %bb.0: # %entry
 ; MIPS32-NEXT:    lw $1, 0($5)
 ; MIPS32-NEXT:    lw $2, 0($6)
-; MIPS32-NEXT:    ori $3, $zero, 1
-; MIPS32-NEXT:    and $3, $4, $3
+; MIPS32-NEXT:    andi $3, $4, 1
 ; MIPS32-NEXT:    movn $2, $1, $3
 ; MIPS32-NEXT:    sw $2, 0($7)
 ; MIPS32-NEXT:    jr $ra
@@ -158,8 +150,7 @@ define double @select_double(double %a, double %b, i1 %test) {
 ; MIPS32:       # %bb.0: # %entry
 ; MIPS32-NEXT:    addiu $1, $sp, 16
 ; MIPS32-NEXT:    lw $1, 0($1)
-; MIPS32-NEXT:    ori $2, $zero, 1
-; MIPS32-NEXT:    and $1, $1, $2
+; MIPS32-NEXT:    andi $1, $1, 1
 ; MIPS32-NEXT:    movn.d $f14, $f12, $1
 ; MIPS32-NEXT:    mov.d $f0, $f14
 ; MIPS32-NEXT:    jr $ra
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/sitofp_and_uitofp.ll b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/sitofp_and_uitofp.ll
index 3803c661e..0017f0c0e 100644
--- a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/sitofp_and_uitofp.ll
+++ b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/sitofp_and_uitofp.ll
@@ -136,6 +136,110 @@ entry:
   ret float %conv
 }
 
+
+define float @u32tof32(i32 zeroext %a) {
+; FP32-LABEL: u32tof32:
+; FP32:       # %bb.0: # %entry
+; FP32-NEXT:    lui $1, 17200
+; FP32-NEXT:    mtc1 $4, $f0
+; FP32-NEXT:    mtc1 $1, $f1
+; FP32-NEXT:    lui $1, 17200
+; FP32-NEXT:    ori $2, $zero, 0
+; FP32-NEXT:    mtc1 $2, $f2
+; FP32-NEXT:    mtc1 $1, $f3
+; FP32-NEXT:    sub.d $f0, $f0, $f2
+; FP32-NEXT:    cvt.s.d $f0, $f0
+; FP32-NEXT:    jr $ra
+; FP32-NEXT:    nop
+;
+; FP64-LABEL: u32tof32:
+; FP64:       # %bb.0: # %entry
+; FP64-NEXT:    lui $1, 17200
+; FP64-NEXT:    mtc1 $4, $f0
+; FP64-NEXT:    mthc1 $1, $f0
+; FP64-NEXT:    lui $1, 17200
+; FP64-NEXT:    ori $2, $zero, 0
+; FP64-NEXT:    mtc1 $2, $f1
+; FP64-NEXT:    mthc1 $1, $f1
+; FP64-NEXT:    sub.d $f0, $f0, $f1
+; FP64-NEXT:    cvt.s.d $f0, $f0
+; FP64-NEXT:    jr $ra
+; FP64-NEXT:    nop
+entry:
+  %conv = uitofp i32 %a to float
+  ret float %conv
+}
+
+define float @u16tof32(i16 zeroext %a) {
+; FP32-LABEL: u16tof32:
+; FP32:       # %bb.0: # %entry
+; FP32-NEXT:    andi $1, $4, 65535
+; FP32-NEXT:    lui $2, 17200
+; FP32-NEXT:    mtc1 $1, $f0
+; FP32-NEXT:    mtc1 $2, $f1
+; FP32-NEXT:    lui $1, 17200
+; FP32-NEXT:    ori $2, $zero, 0
+; FP32-NEXT:    mtc1 $2, $f2
+; FP32-NEXT:    mtc1 $1, $f3
+; FP32-NEXT:    sub.d $f0, $f0, $f2
+; FP32-NEXT:    cvt.s.d $f0, $f0
+; FP32-NEXT:    jr $ra
+; FP32-NEXT:    nop
+;
+; FP64-LABEL: u16tof32:
+; FP64:       # %bb.0: # %entry
+; FP64-NEXT:    andi $1, $4, 65535
+; FP64-NEXT:    lui $2, 17200
+; FP64-NEXT:    mtc1 $1, $f0
+; FP64-NEXT:    mthc1 $2, $f0
+; FP64-NEXT:    lui $1, 17200
+; FP64-NEXT:    ori $2, $zero, 0
+; FP64-NEXT:    mtc1 $2, $f1
+; FP64-NEXT:    mthc1 $1, $f1
+; FP64-NEXT:    sub.d $f0, $f0, $f1
+; FP64-NEXT:    cvt.s.d $f0, $f0
+; FP64-NEXT:    jr $ra
+; FP64-NEXT:    nop
+entry:
+  %conv = uitofp i16 %a to float
+  ret float %conv
+}
+
+define float @u8tof32(i8 zeroext %a) {
+; FP32-LABEL: u8tof32:
+; FP32:       # %bb.0: # %entry
+; FP32-NEXT:    andi $1, $4, 255
+; FP32-NEXT:    lui $2, 17200
+; FP32-NEXT:    mtc1 $1, $f0
+; FP32-NEXT:    mtc1 $2, $f1
+; FP32-NEXT:    lui $1, 17200
+; FP32-NEXT:    ori $2, $zero, 0
+; FP32-NEXT:    mtc1 $2, $f2
+; FP32-NEXT:    mtc1 $1, $f3
+; FP32-NEXT:    sub.d $f0, $f0, $f2
+; FP32-NEXT:    cvt.s.d $f0, $f0
+; FP32-NEXT:    jr $ra
+; FP32-NEXT:    nop
+;
+; FP64-LABEL: u8tof32:
+; FP64:       # %bb.0: # %entry
+; FP64-NEXT:    andi $1, $4, 255
+; FP64-NEXT:    lui $2, 17200
+; FP64-NEXT:    mtc1 $1, $f0
+; FP64-NEXT:    mthc1 $2, $f0
+; FP64-NEXT:    lui $1, 17200
+; FP64-NEXT:    ori $2, $zero, 0
+; FP64-NEXT:    mtc1 $2, $f1
+; FP64-NEXT:    mthc1 $1, $f1
+; FP64-NEXT:    sub.d $f0, $f0, $f1
+; FP64-NEXT:    cvt.s.d $f0, $f0
+; FP64-NEXT:    jr $ra
+; FP64-NEXT:    nop
+entry:
+  %conv = uitofp i8 %a to float
+  ret float %conv
+}
+
 define double @u64tof64(i64 zeroext %a) {
 ; MIPS32-LABEL: u64tof64:
 ; MIPS32:       # %bb.0: # %entry
@@ -153,3 +257,100 @@ entry:
   %conv = uitofp i64 %a to double
   ret double %conv
 }
+
+define double @u32tof64(i32 zeroext %a) {
+; FP32-LABEL: u32tof64:
+; FP32:       # %bb.0: # %entry
+; FP32-NEXT:    lui $1, 17200
+; FP32-NEXT:    mtc1 $4, $f0
+; FP32-NEXT:    mtc1 $1, $f1
+; FP32-NEXT:    lui $1, 17200
+; FP32-NEXT:    ori $2, $zero, 0
+; FP32-NEXT:    mtc1 $2, $f2
+; FP32-NEXT:    mtc1 $1, $f3
+; FP32-NEXT:    sub.d $f0, $f0, $f2
+; FP32-NEXT:    jr $ra
+; FP32-NEXT:    nop
+;
+; FP64-LABEL: u32tof64:
+; FP64:       # %bb.0: # %entry
+; FP64-NEXT:    lui $1, 17200
+; FP64-NEXT:    mtc1 $4, $f0
+; FP64-NEXT:    mthc1 $1, $f0
+; FP64-NEXT:    lui $1, 17200
+; FP64-NEXT:    ori $2, $zero, 0
+; FP64-NEXT:    mtc1 $2, $f1
+; FP64-NEXT:    mthc1 $1, $f1
+; FP64-NEXT:    sub.d $f0, $f0, $f1
+; FP64-NEXT:    jr $ra
+; FP64-NEXT:    nop
+entry:
+  %conv = uitofp i32 %a to double
+  ret double %conv
+}
+
+define double @u16tof64(i16 zeroext %a) {
+; FP32-LABEL: u16tof64:
+; FP32:       # %bb.0: # %entry
+; FP32-NEXT:    andi $1, $4, 65535
+; FP32-NEXT:    lui $2, 17200
+; FP32-NEXT:    mtc1 $1, $f0
+; FP32-NEXT:    mtc1 $2, $f1
+; FP32-NEXT:    lui $1, 17200
+; FP32-NEXT:    ori $2, $zero, 0
+; FP32-NEXT:    mtc1 $2, $f2
+; FP32-NEXT:    mtc1 $1, $f3
+; FP32-NEXT:    sub.d $f0, $f0, $f2
+; FP32-NEXT:    jr $ra
+; FP32-NEXT:    nop
+;
+; FP64-LABEL: u16tof64:
+; FP64:       # %bb.0: # %entry
+; FP64-NEXT:    andi $1, $4, 65535
+; FP64-NEXT:    lui $2, 17200
+; FP64-NEXT:    mtc1 $1, $f0
+; FP64-NEXT:    mthc1 $2, $f0
+; FP64-NEXT:    lui $1, 17200
+; FP64-NEXT:    ori $2, $zero, 0
+; FP64-NEXT:    mtc1 $2, $f1
+; FP64-NEXT:    mthc1 $1, $f1
+; FP64-NEXT:    sub.d $f0, $f0, $f1
+; FP64-NEXT:    jr $ra
+; FP64-NEXT:    nop
+entry:
+  %conv = uitofp i16 %a to double
+  ret double %conv
+}
+
+define double @u8tof64(i8 zeroext %a) {
+; FP32-LABEL: u8tof64:
+; FP32:       # %bb.0: # %entry
+; FP32-NEXT:    andi $1, $4, 255
+; FP32-NEXT:    lui $2, 17200
+; FP32-NEXT:    mtc1 $1, $f0
+; FP32-NEXT:    mtc1 $2, $f1
+; FP32-NEXT:    lui $1, 17200
+; FP32-NEXT:    ori $2, $zero, 0
+; FP32-NEXT:    mtc1 $2, $f2
+; FP32-NEXT:    mtc1 $1, $f3
+; FP32-NEXT:    sub.d $f0, $f0, $f2
+; FP32-NEXT:    jr $ra
+; FP32-NEXT:    nop
+;
+; FP64-LABEL: u8tof64:
+; FP64:       # %bb.0: # %entry
+; FP64-NEXT:    andi $1, $4, 255
+; FP64-NEXT:    lui $2, 17200
+; FP64-NEXT:    mtc1 $1, $f0
+; FP64-NEXT:    mthc1 $2, $f0
+; FP64-NEXT:    lui $1, 17200
+; FP64-NEXT:    ori $2, $zero, 0
+; FP64-NEXT:    mtc1 $2, $f1
+; FP64-NEXT:    mthc1 $1, $f1
+; FP64-NEXT:    sub.d $f0, $f0, $f1
+; FP64-NEXT:    jr $ra
+; FP64-NEXT:    nop
+entry:
+  %conv = uitofp i8 %a to double
+  ret double %conv
+}
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/sret_pointer.ll b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/sret_pointer.ll
new file mode 100644
index 000000000..4e5cbb08f
--- /dev/null
+++ b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/sret_pointer.ll
@@ -0,0 +1,38 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc  -O0 -mtriple=mipsel-linux-gnu -global-isel  -verify-machineinstrs %s -o -| FileCheck %s -check-prefixes=MIPS32
+
+%struct.S = type { i32, i32 }
+
+define void @ZeroInit(%struct.S* noalias sret %agg.result) {
+; MIPS32-LABEL: ZeroInit:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    ori $1, $zero, 0
+; MIPS32-NEXT:    sw $1, 0($4)
+; MIPS32-NEXT:    sw $1, 4($4)
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %x = getelementptr inbounds %struct.S, %struct.S* %agg.result, i32 0, i32 0
+  store i32 0, i32* %x, align 4
+  %y = getelementptr inbounds %struct.S, %struct.S* %agg.result, i32 0, i32 1
+  store i32 0, i32* %y, align 4
+  ret void
+}
+
+define void @CallZeroInit(%struct.S* noalias sret %agg.result) {
+; MIPS32-LABEL: CallZeroInit:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    addiu $sp, $sp, -24
+; MIPS32-NEXT:    .cfi_def_cfa_offset 24
+; MIPS32-NEXT:    sw $ra, 20($sp) # 4-byte Folded Spill
+; MIPS32-NEXT:    .cfi_offset 31, -4
+; MIPS32-NEXT:    jal ZeroInit
+; MIPS32-NEXT:    nop
+; MIPS32-NEXT:    lw $ra, 20($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    addiu $sp, $sp, 24
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  call void @ZeroInit(%struct.S* sret %agg.result)
+  ret void
+}
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/stack_args.ll b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/stack_args.ll
index 0fe1b7c53..f44cfdc86 100644
--- a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/stack_args.ll
+++ b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/stack_args.ll
@@ -13,9 +13,7 @@ define  i32 @g(i32  %x1, i32 %x2, i32 %x3, i32 %x4, i32 %x5){
 ; MIPS32-NEXT:    addiu $1, $sp, 48
 ; MIPS32-NEXT:    lw $1, 0($1)
 ; MIPS32-NEXT:    move $2, $sp
-; MIPS32-NEXT:    ori $3, $zero, 16
-; MIPS32-NEXT:    addu $2, $2, $3
-; MIPS32-NEXT:    sw $1, 0($2)
+; MIPS32-NEXT:    sw $1, 16($2)
 ; MIPS32-NEXT:    jal f
 ; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:    lw $ra, 28($sp) # 4-byte Folded Reload
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/store.ll b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/store.ll
index 17ba6c099..be7974736 100644
--- a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/store.ll
+++ b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/store.ll
@@ -16,9 +16,7 @@ define void @store_i64(i64 %val, i64* %ptr)  {
 ; MIPS32-LABEL: store_i64:
 ; MIPS32:       # %bb.0: # %entry
 ; MIPS32-NEXT:    sw $4, 0($6)
-; MIPS32-NEXT:    ori $1, $zero, 4
-; MIPS32-NEXT:    addu $1, $6, $1
-; MIPS32-NEXT:    sw $5, 0($1)
+; MIPS32-NEXT:    sw $5, 4($6)
 ; MIPS32-NEXT:    jr $ra
 ; MIPS32-NEXT:    nop
 entry:
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/sub.ll b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/sub.ll
index 0a15f673b..66dc761c5 100644
--- a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/sub.ll
+++ b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/sub.ll
@@ -29,8 +29,7 @@ define zeroext i8 @sub_i8_zext(i8 zeroext %a, i8 zeroext %b) {
 ; MIPS32-LABEL: sub_i8_zext:
 ; MIPS32:       # %bb.0: # %entry
 ; MIPS32-NEXT:    subu $1, $5, $4
-; MIPS32-NEXT:    ori $2, $zero, 255
-; MIPS32-NEXT:    and $2, $1, $2
+; MIPS32-NEXT:    andi $2, $1, 255
 ; MIPS32-NEXT:    jr $ra
 ; MIPS32-NEXT:    nop
 entry:
@@ -66,8 +65,7 @@ define zeroext i16 @sub_i16_zext(i16 zeroext %a, i16 zeroext %b) {
 ; MIPS32-LABEL: sub_i16_zext:
 ; MIPS32:       # %bb.0: # %entry
 ; MIPS32-NEXT:    subu $1, $5, $4
-; MIPS32-NEXT:    ori $2, $zero, 65535
-; MIPS32-NEXT:    and $2, $1, $2
+; MIPS32-NEXT:    andi $2, $1, 65535
 ; MIPS32-NEXT:    jr $ra
 ; MIPS32-NEXT:    nop
 entry:
@@ -92,8 +90,7 @@ define i64 @sub_i64(i64 %a, i64 %b) {
 ; MIPS32-NEXT:    subu $2, $6, $4
 ; MIPS32-NEXT:    sltu $1, $6, $4
 ; MIPS32-NEXT:    subu $3, $7, $5
-; MIPS32-NEXT:    ori $4, $zero, 1
-; MIPS32-NEXT:    and $1, $1, $4
+; MIPS32-NEXT:    andi $1, $1, 1
 ; MIPS32-NEXT:    subu $3, $3, $1
 ; MIPS32-NEXT:    jr $ra
 ; MIPS32-NEXT:    nop
@@ -116,24 +113,23 @@ define i128 @sub_i128(i128 %a, i128 %b) {
 ; MIPS32-NEXT:    subu $9, $1, $4
 ; MIPS32-NEXT:    sltu $1, $1, $4
 ; MIPS32-NEXT:    subu $4, $2, $5
-; MIPS32-NEXT:    ori $10, $zero, 1
-; MIPS32-NEXT:    and $11, $1, $10
-; MIPS32-NEXT:    subu $4, $4, $11
-; MIPS32-NEXT:    xor $11, $2, $5
-; MIPS32-NEXT:    sltiu $11, $11, 1
+; MIPS32-NEXT:    andi $10, $1, 1
+; MIPS32-NEXT:    subu $4, $4, $10
+; MIPS32-NEXT:    xor $10, $2, $5
+; MIPS32-NEXT:    sltiu $10, $10, 1
 ; MIPS32-NEXT:    sltu $2, $2, $5
-; MIPS32-NEXT:    and $5, $11, $10
+; MIPS32-NEXT:    andi $5, $10, 1
 ; MIPS32-NEXT:    movn $2, $1, $5
 ; MIPS32-NEXT:    subu $1, $3, $6
-; MIPS32-NEXT:    and $5, $2, $10
+; MIPS32-NEXT:    andi $5, $2, 1
 ; MIPS32-NEXT:    subu $1, $1, $5
 ; MIPS32-NEXT:    xor $5, $3, $6
 ; MIPS32-NEXT:    sltiu $5, $5, 1
 ; MIPS32-NEXT:    sltu $3, $3, $6
-; MIPS32-NEXT:    and $5, $5, $10
+; MIPS32-NEXT:    andi $5, $5, 1
 ; MIPS32-NEXT:    movn $3, $2, $5
 ; MIPS32-NEXT:    subu $2, $8, $7
-; MIPS32-NEXT:    and $3, $3, $10
+; MIPS32-NEXT:    andi $3, $3, 1
 ; MIPS32-NEXT:    subu $5, $2, $3
 ; MIPS32-NEXT:    move $2, $9
 ; MIPS32-NEXT:    move $3, $4
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/sub_vec.ll b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/sub_vec.ll
new file mode 100644
index 000000000..6ad041d3a
--- /dev/null
+++ b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/sub_vec.ll
@@ -0,0 +1,70 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -O0 -mtriple=mipsel-linux-gnu -global-isel -mcpu=mips32r5 -mattr=+msa,+fp64,+nan2008 -verify-machineinstrs %s -o -| FileCheck %s -check-prefixes=P5600
+
+define void @sub_v16i8(<16 x i8>* %a, <16 x i8>* %b, <16 x i8>* %c) {
+; P5600-LABEL: sub_v16i8:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.b $w0, 0($4)
+; P5600-NEXT:    ld.b $w1, 0($5)
+; P5600-NEXT:    subv.b $w0, $w1, $w0
+; P5600-NEXT:    st.b $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <16 x i8>, <16 x i8>* %a, align 16
+  %1 = load <16 x i8>, <16 x i8>* %b, align 16
+  %sub = sub <16 x i8> %1, %0
+  store <16 x i8> %sub, <16 x i8>* %c, align 16
+  ret void
+}
+
+define void @sub_v8i16(<8 x i16>* %a, <8 x i16>* %b, <8 x i16>* %c) {
+; P5600-LABEL: sub_v8i16:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.h $w0, 0($4)
+; P5600-NEXT:    ld.h $w1, 0($5)
+; P5600-NEXT:    subv.h $w0, $w1, $w0
+; P5600-NEXT:    st.h $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <8 x i16>, <8 x i16>* %a, align 16
+  %1 = load <8 x i16>, <8 x i16>* %b, align 16
+  %sub = sub <8 x i16> %1, %0
+  store <8 x i16> %sub, <8 x i16>* %c, align 16
+  ret void
+}
+
+define void @sub_v4i32(<4 x i32>* %a, <4 x i32>* %b, <4 x i32>* %c) {
+; P5600-LABEL: sub_v4i32:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.w $w0, 0($4)
+; P5600-NEXT:    ld.w $w1, 0($5)
+; P5600-NEXT:    subv.w $w0, $w1, $w0
+; P5600-NEXT:    st.w $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <4 x i32>, <4 x i32>* %a, align 16
+  %1 = load <4 x i32>, <4 x i32>* %b, align 16
+  %sub = sub <4 x i32> %1, %0
+  store <4 x i32> %sub, <4 x i32>* %c, align 16
+  ret void
+}
+
+define void @sub_v2i64(<2 x i64>* %a, <2 x i64>* %b, <2 x i64>* %c) {
+; P5600-LABEL: sub_v2i64:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.d $w0, 0($4)
+; P5600-NEXT:    ld.d $w1, 0($5)
+; P5600-NEXT:    subv.d $w0, $w1, $w0
+; P5600-NEXT:    st.d $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <2 x i64>, <2 x i64>* %a, align 16
+  %1 = load <2 x i64>, <2 x i64>* %b, align 16
+  %sub = sub <2 x i64> %1, %0
+  store <2 x i64> %sub, <2 x i64>* %c, align 16
+  ret void
+}
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/sub_vec_builtin.ll b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/sub_vec_builtin.ll
new file mode 100644
index 000000000..5e7952146
--- /dev/null
+++ b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/sub_vec_builtin.ll
@@ -0,0 +1,138 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -O0 -mtriple=mipsel-linux-gnu -global-isel -mcpu=mips32r5 -mattr=+msa,+fp64,+nan2008 -verify-machineinstrs %s -o -| FileCheck %s -check-prefixes=P5600
+
+declare <16 x i8> @llvm.mips.subv.b(<16 x i8>, <16 x i8>)
+define void @sub_v16i8_builtin(<16 x i8>* %a, <16 x i8>* %b, <16 x i8>* %c) {
+; P5600-LABEL: sub_v16i8_builtin:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.b $w0, 0($4)
+; P5600-NEXT:    ld.b $w1, 0($5)
+; P5600-NEXT:    subv.b $w0, $w0, $w1
+; P5600-NEXT:    st.b $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <16 x i8>, <16 x i8>* %a, align 16
+  %1 = load <16 x i8>, <16 x i8>* %b, align 16
+  %2 = tail call <16 x i8> @llvm.mips.subv.b(<16 x i8> %0, <16 x i8> %1)
+  store <16 x i8> %2, <16 x i8>* %c, align 16
+  ret void
+}
+
+declare <8 x i16> @llvm.mips.subv.h(<8 x i16>, <8 x i16>)
+define void @sub_v8i16_builtin(<8 x i16>* %a, <8 x i16>* %b, <8 x i16>* %c) {
+; P5600-LABEL: sub_v8i16_builtin:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.h $w0, 0($4)
+; P5600-NEXT:    ld.h $w1, 0($5)
+; P5600-NEXT:    subv.h $w0, $w0, $w1
+; P5600-NEXT:    st.h $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <8 x i16>, <8 x i16>* %a, align 16
+  %1 = load <8 x i16>, <8 x i16>* %b, align 16
+  %2 = tail call <8 x i16> @llvm.mips.subv.h(<8 x i16> %0, <8 x i16> %1)
+  store <8 x i16> %2, <8 x i16>* %c, align 16
+  ret void
+}
+
+declare <4 x i32> @llvm.mips.subv.w(<4 x i32>, <4 x i32>)
+define void @sub_v4i32_builtin(<4 x i32>* %a, <4 x i32>* %b, <4 x i32>* %c) {
+; P5600-LABEL: sub_v4i32_builtin:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.w $w0, 0($4)
+; P5600-NEXT:    ld.w $w1, 0($5)
+; P5600-NEXT:    subv.w $w0, $w0, $w1
+; P5600-NEXT:    st.w $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <4 x i32>, <4 x i32>* %a, align 16
+  %1 = load <4 x i32>, <4 x i32>* %b, align 16
+  %2 = tail call <4 x i32> @llvm.mips.subv.w(<4 x i32> %0, <4 x i32> %1)
+  store <4 x i32> %2, <4 x i32>* %c, align 16
+  ret void
+}
+
+declare <2 x i64> @llvm.mips.subv.d(<2 x i64>, <2 x i64>)
+define void @sub_v2i64_builtin(<2 x i64>* %a, <2 x i64>* %b, <2 x i64>* %c) {
+; P5600-LABEL: sub_v2i64_builtin:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.d $w0, 0($4)
+; P5600-NEXT:    ld.d $w1, 0($5)
+; P5600-NEXT:    subv.d $w0, $w0, $w1
+; P5600-NEXT:    st.d $w0, 0($6)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <2 x i64>, <2 x i64>* %a, align 16
+  %1 = load <2 x i64>, <2 x i64>* %b, align 16
+  %2 = tail call <2 x i64> @llvm.mips.subv.d(<2 x i64> %0, <2 x i64> %1)
+  store <2 x i64> %2, <2 x i64>* %c, align 16
+  ret void
+}
+
+declare <16 x i8> @llvm.mips.subvi.b(<16 x i8>, i32 immarg)
+define void @sub_v16i8_builtin_imm(<16 x i8>* %a, <16 x i8>* %c) {
+; P5600-LABEL: sub_v16i8_builtin_imm:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.b $w0, 0($4)
+; P5600-NEXT:    subvi.b $w0, $w0, 3
+; P5600-NEXT:    st.b $w0, 0($5)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <16 x i8>, <16 x i8>* %a, align 16
+  %1 = tail call <16 x i8> @llvm.mips.subvi.b(<16 x i8> %0, i32 3)
+  store <16 x i8> %1, <16 x i8>* %c, align 16
+  ret void
+}
+
+declare <8 x i16> @llvm.mips.subvi.h(<8 x i16>, i32 immarg)
+define void @sub_v8i16_builtin_imm(<8 x i16>* %a, <8 x i16>* %c) {
+; P5600-LABEL: sub_v8i16_builtin_imm:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.h $w0, 0($4)
+; P5600-NEXT:    subvi.h $w0, $w0, 18
+; P5600-NEXT:    st.h $w0, 0($5)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <8 x i16>, <8 x i16>* %a, align 16
+  %1 = tail call <8 x i16> @llvm.mips.subvi.h(<8 x i16> %0, i32 18)
+  store <8 x i16> %1, <8 x i16>* %c, align 16
+  ret void
+}
+
+declare <4 x i32> @llvm.mips.subvi.w(<4 x i32>, i32 immarg)
+define void @sub_v4i32_builtin_imm(<4 x i32>* %a, <4 x i32>* %c) {
+; P5600-LABEL: sub_v4i32_builtin_imm:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.w $w0, 0($4)
+; P5600-NEXT:    subvi.w $w0, $w0, 25
+; P5600-NEXT:    st.w $w0, 0($5)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <4 x i32>, <4 x i32>* %a, align 16
+  %1 = tail call <4 x i32> @llvm.mips.subvi.w(<4 x i32> %0, i32 25)
+  store <4 x i32> %1, <4 x i32>* %c, align 16
+  ret void
+}
+
+declare <2 x i64> @llvm.mips.subvi.d(<2 x i64>, i32 immarg)
+define void @sub_v2i64_builtin_imm(<2 x i64>* %a, <2 x i64>* %c) {
+; P5600-LABEL: sub_v2i64_builtin_imm:
+; P5600:       # %bb.0: # %entry
+; P5600-NEXT:    ld.d $w0, 0($4)
+; P5600-NEXT:    subvi.d $w0, $w0, 31
+; P5600-NEXT:    st.d $w0, 0($5)
+; P5600-NEXT:    jr $ra
+; P5600-NEXT:    nop
+entry:
+  %0 = load <2 x i64>, <2 x i64>* %a, align 16
+  %1 = tail call <2 x i64> @llvm.mips.subvi.d(<2 x i64> %0, i32 31)
+  store <2 x i64> %1, <2 x i64>* %c, align 16
+  ret void
+}
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/test_TypeInfoforMF.ll b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/test_TypeInfoforMF.ll
index e06bf0609..25e87da5a 100644
--- a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/test_TypeInfoforMF.ll
+++ b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/test_TypeInfoforMF.ll
@@ -57,8 +57,7 @@ define i32 @incoming_gpr(i32 %incoming_phys_reg, i1 %test, i32* %a) {
 ; MIPS32-LABEL: incoming_gpr:
 ; MIPS32:       # %bb.0: # %entry
 ; MIPS32-NEXT:    lw $1, 0($6)
-; MIPS32-NEXT:    ori $2, $zero, 1
-; MIPS32-NEXT:    and $2, $5, $2
+; MIPS32-NEXT:    andi $2, $5, 1
 ; MIPS32-NEXT:    movn $4, $1, $2
 ; MIPS32-NEXT:    move $2, $4
 ; MIPS32-NEXT:    jr $ra
@@ -73,8 +72,7 @@ define float @incoming_fpr(float %incoming_phys_reg, i1 %test, float* %a) {
 ; MIPS32-LABEL: incoming_fpr:
 ; MIPS32:       # %bb.0: # %entry
 ; MIPS32-NEXT:    lwc1 $f0, 0($6)
-; MIPS32-NEXT:    ori $1, $zero, 1
-; MIPS32-NEXT:    and $1, $5, $1
+; MIPS32-NEXT:    andi $1, $5, 1
 ; MIPS32-NEXT:    movn.s $f12, $f0, $1
 ; MIPS32-NEXT:    mov.s $f0, $f12
 ; MIPS32-NEXT:    jr $ra
@@ -91,8 +89,7 @@ define i32 @incoming_i32_instr(i32 %val1, i32 %val2, i32* %i32_ptr, i1 %test) {
 ; MIPS32:       # %bb.0: # %entry
 ; MIPS32-NEXT:    lw $1, 0($6)
 ; MIPS32-NEXT:    addu $2, $5, $4
-; MIPS32-NEXT:    ori $3, $zero, 1
-; MIPS32-NEXT:    and $3, $7, $3
+; MIPS32-NEXT:    andi $3, $7, 1
 ; MIPS32-NEXT:    movn $2, $1, $3
 ; MIPS32-NEXT:    jr $ra
 ; MIPS32-NEXT:    nop
@@ -108,8 +105,7 @@ define float @incoming_float_instr(float %val1, float %val2, float* %float_ptr,
 ; MIPS32:       # %bb.0: # %entry
 ; MIPS32-NEXT:    lwc1 $f0, 0($6)
 ; MIPS32-NEXT:    add.s $f1, $f14, $f12
-; MIPS32-NEXT:    ori $1, $zero, 1
-; MIPS32-NEXT:    and $1, $7, $1
+; MIPS32-NEXT:    andi $1, $7, 1
 ; MIPS32-NEXT:    movn.s $f1, $f0, $1
 ; MIPS32-NEXT:    mov.s $f0, $f1
 ; MIPS32-NEXT:    jr $ra
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/trap.ll b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/trap.ll
new file mode 100644
index 000000000..cf71f5f56
--- /dev/null
+++ b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/trap.ll
@@ -0,0 +1,14 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc  -O0 -mtriple=mipsel-linux-gnu -global-isel  -verify-machineinstrs %s -o -| FileCheck %s -check-prefixes=MIPS32
+
+  declare void @llvm.trap()
+
+  define void @f() {
+; MIPS32-LABEL: f:
+; MIPS32:       # %bb.0:
+; MIPS32-NEXT:    break
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+    call void @llvm.trap()
+    ret void
+  }
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/trunc.ll b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/trunc.ll
new file mode 100644
index 000000000..7ea5ad8c3
--- /dev/null
+++ b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/trunc.ll
@@ -0,0 +1,13 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc  -O0 -mtriple=mipsel-linux-gnu -global-isel  -verify-machineinstrs %s -o -| FileCheck %s -check-prefixes=MIPS32
+
+define i32 @trunc(i64 %x) {
+; MIPS32-LABEL: trunc:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    move $2, $4
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %conv = trunc i64 %x to i32
+  ret i32 %conv
+}
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/truncStore_and_aExtLoad.ll b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/truncStore_and_aExtLoad.ll
index e051b3079..e4f93520f 100644
--- a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/truncStore_and_aExtLoad.ll
+++ b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/truncStore_and_aExtLoad.ll
@@ -27,8 +27,7 @@ define void @load_store_i1(i1* %px, i1* %py) {
 ; MIPS32-LABEL: load_store_i1:
 ; MIPS32:       # %bb.0: # %entry
 ; MIPS32-NEXT:    lbu $1, 0($5)
-; MIPS32-NEXT:    ori $2, $zero, 1
-; MIPS32-NEXT:    and $1, $1, $2
+; MIPS32-NEXT:    andi $1, $1, 1
 ; MIPS32-NEXT:    sb $1, 0($4)
 ; MIPS32-NEXT:    jr $ra
 ; MIPS32-NEXT:    nop
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/var_arg.ll b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/var_arg.ll
new file mode 100644
index 000000000..91a82f2a7
--- /dev/null
+++ b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/var_arg.ll
@@ -0,0 +1,67 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc  -O0 -mtriple=mipsel-linux-gnu -global-isel  -verify-machineinstrs %s -o -| FileCheck %s -check-prefixes=MIPS32
+
+@.str = private unnamed_addr constant [11 x i8] c"string %s\0A\00", align 1
+declare void @llvm.va_start(i8*)
+declare void @llvm.va_copy(i8*, i8*)
+declare i32 @printf(i8*, ...)
+
+define void @testVaCopyArg(i8* %fmt, ...) {
+; MIPS32-LABEL: testVaCopyArg:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    addiu $sp, $sp, -40
+; MIPS32-NEXT:    .cfi_def_cfa_offset 40
+; MIPS32-NEXT:    sw $ra, 36($sp) # 4-byte Folded Spill
+; MIPS32-NEXT:    .cfi_offset 31, -4
+; MIPS32-NEXT:    addiu $1, $sp, 44
+; MIPS32-NEXT:    sw $5, 0($1)
+; MIPS32-NEXT:    addiu $1, $sp, 48
+; MIPS32-NEXT:    sw $6, 0($1)
+; MIPS32-NEXT:    addiu $1, $sp, 52
+; MIPS32-NEXT:    sw $7, 0($1)
+; MIPS32-NEXT:    lui $1, %hi($.str)
+; MIPS32-NEXT:    addiu $1, $1, %lo($.str)
+; MIPS32-NEXT:    addiu $2, $sp, 32
+; MIPS32-NEXT:    addiu $3, $sp, 28
+; MIPS32-NEXT:    addiu $5, $sp, 24
+; MIPS32-NEXT:    addiu $6, $sp, 20
+; MIPS32-NEXT:    sw $4, 0($2)
+; MIPS32-NEXT:    addiu $2, $sp, 44
+; MIPS32-NEXT:    sw $2, 0($3)
+; MIPS32-NEXT:    lw $2, 0($3)
+; MIPS32-NEXT:    sw $2, 0($5)
+; MIPS32-NEXT:    lw $2, 0($5)
+; MIPS32-NEXT:    ori $3, $zero, 4
+; MIPS32-NEXT:    addu $3, $2, $3
+; MIPS32-NEXT:    sw $3, 0($5)
+; MIPS32-NEXT:    lw $2, 0($2)
+; MIPS32-NEXT:    sw $2, 0($6)
+; MIPS32-NEXT:    lw $5, 0($6)
+; MIPS32-NEXT:    move $4, $1
+; MIPS32-NEXT:    jal printf
+; MIPS32-NEXT:    nop
+; MIPS32-NEXT:    lw $ra, 36($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    addiu $sp, $sp, 40
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %fmt.addr = alloca i8*, align 4
+  %ap = alloca i8*, align 4
+  %aq = alloca i8*, align 4
+  %s = alloca i8*, align 4
+  store i8* %fmt, i8** %fmt.addr, align 4
+  %ap1 = bitcast i8** %ap to i8*
+  call void @llvm.va_start(i8* %ap1)
+  %0 = bitcast i8** %aq to i8*
+  %1 = bitcast i8** %ap to i8*
+  call void @llvm.va_copy(i8* %0, i8* %1)
+  %argp.cur = load i8*, i8** %aq, align 4
+  %argp.next = getelementptr inbounds i8, i8* %argp.cur, i32 4
+  store i8* %argp.next, i8** %aq, align 4
+  %2 = bitcast i8* %argp.cur to i8**
+  %3 = load i8*, i8** %2, align 4
+  store i8* %3, i8** %s, align 4
+  %4 = load i8*, i8** %s, align 4
+  %call = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([11 x i8], [11 x i8]* @.str, i32 0, i32 0), i8* %4)
+  ret void
+}
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/zextLoad_and_sextLoad.ll b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/zextLoad_and_sextLoad.ll
index 40bfc1dec..bc5986031 100644
--- a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/zextLoad_and_sextLoad.ll
+++ b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/zextLoad_and_sextLoad.ll
@@ -49,6 +49,19 @@ entry:
   ret i16 %conv
 }
 
+define i64 @load4_s32_to_zextLoad4_s64(i32* %px) {
+; MIPS32-LABEL: load4_s32_to_zextLoad4_s64:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    lw $2, 0($4)
+; MIPS32-NEXT:    ori $3, $zero, 0
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %0 = load i32, i32* %px
+  %conv = zext i32 %0 to i64
+  ret i64 %conv
+}
+
 define i32 @load1_s8_to_sextLoad1_s32(i8* %px) {
 ; MIPS32-LABEL: load1_s8_to_sextLoad1_s32:
 ; MIPS32:       # %bb.0: # %entry
@@ -96,3 +109,18 @@ entry:
   %conv = sext i8 %0 to i16
   ret i16 %conv
 }
+
+define i64 @load4_s32_to_sextLoad4_s64(i32* %px) {
+; MIPS32-LABEL: load4_s32_to_sextLoad4_s64:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    lw $1, 0($4)
+; MIPS32-NEXT:    ori $2, $zero, 31
+; MIPS32-NEXT:    srav $3, $1, $2
+; MIPS32-NEXT:    move $2, $1
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %0 = load i32, i32* %px
+  %conv = sext i32 %0 to i64
+  ret i64 %conv
+}
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/zext_and_sext.ll b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/zext_and_sext.ll
new file mode 100644
index 000000000..1249d5c95
--- /dev/null
+++ b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/zext_and_sext.ll
@@ -0,0 +1,27 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc  -O0 -mtriple=mipsel-linux-gnu -global-isel  -verify-machineinstrs %s -o -| FileCheck %s -check-prefixes=MIPS32
+
+define i64 @zext(i32 %x) {
+; MIPS32-LABEL: zext:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    ori $3, $zero, 0
+; MIPS32-NEXT:    move $2, $4
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %conv = zext i32 %x to i64
+  ret i64 %conv
+}
+
+define i64 @sext(i32 %x) {
+; MIPS32-LABEL: sext:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    ori $1, $zero, 31
+; MIPS32-NEXT:    srav $3, $4, $1
+; MIPS32-NEXT:    move $2, $4
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %conv = sext i32 %x to i64
+  ret i64 %conv
+}
diff --git a/llvm/test/CodeGen/Mips/address-selection.ll b/llvm/test/CodeGen/Mips/address-selection.ll
index 36a1d5117..1e0a9a42e 100644
--- a/llvm/test/CodeGen/Mips/address-selection.ll
+++ b/llvm/test/CodeGen/Mips/address-selection.ll
@@ -1,8 +1,10 @@
 ; RUN: llc -march=mips < %s -debug 2>&1 | FileCheck %s --check-prefix=MIPS
-; RUN: llc -march=mips -relocation-model=pic -mxgot < %s -debug 2>&1 | FileCheck %s --check-prefix=MIPS-XGOT
+; RUN: llc -march=mips -relocation-model=pic -mattr=+xgot < %s \
+; RUN:     -debug 2>&1 | FileCheck %s --check-prefix=MIPS-XGOT
 
 ; RUN: llc -march=mips -mattr=+micromips < %s -debug 2>&1 | FileCheck %s --check-prefix=MM
-; RUN: llc -march=mips -relocation-model=pic -mxgot -mattr=+micromips < %s -debug 2>&1 | FileCheck %s --check-prefix=MM-XGOT
+; RUN: llc -march=mips -relocation-model=pic -mattr=+xgot,+micromips < %s \
+; RUN:     -debug 2>&1 | FileCheck %s --check-prefix=MM-XGOT
 
 ; REQUIRES: asserts
 
diff --git a/llvm/test/CodeGen/Mips/atomic-min-max-64.ll b/llvm/test/CodeGen/Mips/atomic-min-max-64.ll
new file mode 100644
index 000000000..6b71fe6e1
--- /dev/null
+++ b/llvm/test/CodeGen/Mips/atomic-min-max-64.ll
@@ -0,0 +1,158 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -march=mips64 -O0 -mcpu=mips64r2 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=MIPS
+; RUN: llc -march=mips64el -O0 -mcpu=mips64r2 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=MIPS
+; RUN: llc -march=mips64 -O0 -mcpu=mips64r6 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=MIPSR6
+; RUN: llc -march=mips64el -O0 -mcpu=mips64r6 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=MIPSR6
+
+define i64 @test_max(i64* nocapture %ptr, i64 signext %val) {
+; MIPS-LABEL: test_max:
+; MIPS:       # %bb.0: # %entry
+; MIPS-NEXT:    sync
+; MIPS-NEXT:  .LBB0_1: # %entry
+; MIPS-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS-NEXT:    lld $2, 0($4)
+; MIPS-NEXT:    slt $3, $2, $5
+; MIPS-NEXT:    move $1, $2
+; MIPS-NEXT:    movn $1, $5, $3
+; MIPS-NEXT:    scd $1, 0($4)
+; MIPS-NEXT:    beqz $1, .LBB0_1
+; MIPS-NEXT:    nop
+; MIPS-NEXT:  # %bb.2: # %entry
+; MIPS-NEXT:    sync
+; MIPS-NEXT:    jr $ra
+; MIPS-NEXT:    nop
+;
+; MIPSR6-LABEL: test_max:
+; MIPSR6:       # %bb.0: # %entry
+; MIPSR6-NEXT:    sync
+; MIPSR6-NEXT:  .LBB0_1: # %entry
+; MIPSR6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPSR6-NEXT:    lld $2, 0($4)
+; MIPSR6-NEXT:    slt $3, $2, $5
+; MIPSR6-NEXT:    seleqz $1, $2, $3
+; MIPSR6-NEXT:    selnez $3, $5, $3
+; MIPSR6-NEXT:    or $1, $1, $3
+; MIPSR6-NEXT:    scd $1, 0($4)
+; MIPSR6-NEXT:    beqzc $1, .LBB0_1
+; MIPSR6-NEXT:  # %bb.2: # %entry
+; MIPSR6-NEXT:    sync
+; MIPSR6-NEXT:    jrc $ra
+entry:
+  %0 = atomicrmw max i64* %ptr, i64 %val seq_cst
+  ret i64 %0
+}
+
+define i64 @test_min(i64* nocapture %ptr, i64 signext %val) {
+; MIPS-LABEL: test_min:
+; MIPS:       # %bb.0: # %entry
+; MIPS-NEXT:    sync
+; MIPS-NEXT:  .LBB1_1: # %entry
+; MIPS-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS-NEXT:    lld $2, 0($4)
+; MIPS-NEXT:    slt $3, $2, $5
+; MIPS-NEXT:    move $1, $2
+; MIPS-NEXT:    movz $1, $5, $3
+; MIPS-NEXT:    scd $1, 0($4)
+; MIPS-NEXT:    beqz $1, .LBB1_1
+; MIPS-NEXT:    nop
+; MIPS-NEXT:  # %bb.2: # %entry
+; MIPS-NEXT:    sync
+; MIPS-NEXT:    jr $ra
+; MIPS-NEXT:    nop
+;
+; MIPSR6-LABEL: test_min:
+; MIPSR6:       # %bb.0: # %entry
+; MIPSR6-NEXT:    sync
+; MIPSR6-NEXT:  .LBB1_1: # %entry
+; MIPSR6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPSR6-NEXT:    lld $2, 0($4)
+; MIPSR6-NEXT:    slt $3, $2, $5
+; MIPSR6-NEXT:    selnez $1, $2, $3
+; MIPSR6-NEXT:    seleqz $3, $5, $3
+; MIPSR6-NEXT:    or $1, $1, $3
+; MIPSR6-NEXT:    scd $1, 0($4)
+; MIPSR6-NEXT:    beqzc $1, .LBB1_1
+; MIPSR6-NEXT:  # %bb.2: # %entry
+; MIPSR6-NEXT:    sync
+; MIPSR6-NEXT:    jrc $ra
+entry:
+  %0 = atomicrmw min i64* %ptr, i64 %val seq_cst
+  ret i64 %0
+}
+
+define i64 @test_umax(i64* nocapture %ptr, i64 zeroext %val) {
+; MIPS-LABEL: test_umax:
+; MIPS:       # %bb.0: # %entry
+; MIPS-NEXT:    sync
+; MIPS-NEXT:  .LBB2_1: # %entry
+; MIPS-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS-NEXT:    lld $2, 0($4)
+; MIPS-NEXT:    sltu $3, $2, $5
+; MIPS-NEXT:    move $1, $2
+; MIPS-NEXT:    movn $1, $5, $3
+; MIPS-NEXT:    scd $1, 0($4)
+; MIPS-NEXT:    beqz $1, .LBB2_1
+; MIPS-NEXT:    nop
+; MIPS-NEXT:  # %bb.2: # %entry
+; MIPS-NEXT:    sync
+; MIPS-NEXT:    jr $ra
+; MIPS-NEXT:    nop
+;
+; MIPSR6-LABEL: test_umax:
+; MIPSR6:       # %bb.0: # %entry
+; MIPSR6-NEXT:    sync
+; MIPSR6-NEXT:  .LBB2_1: # %entry
+; MIPSR6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPSR6-NEXT:    lld $2, 0($4)
+; MIPSR6-NEXT:    sltu $3, $2, $5
+; MIPSR6-NEXT:    seleqz $1, $2, $3
+; MIPSR6-NEXT:    selnez $3, $5, $3
+; MIPSR6-NEXT:    or $1, $1, $3
+; MIPSR6-NEXT:    scd $1, 0($4)
+; MIPSR6-NEXT:    beqzc $1, .LBB2_1
+; MIPSR6-NEXT:  # %bb.2: # %entry
+; MIPSR6-NEXT:    sync
+; MIPSR6-NEXT:    jrc $ra
+entry:
+  %0 = atomicrmw umax i64* %ptr, i64 %val seq_cst
+  ret i64 %0
+}
+
+define i64 @test_umin(i64* nocapture %ptr, i64 zeroext %val) {
+; MIPS-LABEL: test_umin:
+; MIPS:       # %bb.0: # %entry
+; MIPS-NEXT:    sync
+; MIPS-NEXT:  .LBB3_1: # %entry
+; MIPS-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS-NEXT:    lld $2, 0($4)
+; MIPS-NEXT:    sltu $3, $2, $5
+; MIPS-NEXT:    move $1, $2
+; MIPS-NEXT:    movz $1, $5, $3
+; MIPS-NEXT:    scd $1, 0($4)
+; MIPS-NEXT:    beqz $1, .LBB3_1
+; MIPS-NEXT:    nop
+; MIPS-NEXT:  # %bb.2: # %entry
+; MIPS-NEXT:    sync
+; MIPS-NEXT:    jr $ra
+; MIPS-NEXT:    nop
+;
+; MIPSR6-LABEL: test_umin:
+; MIPSR6:       # %bb.0: # %entry
+; MIPSR6-NEXT:    sync
+; MIPSR6-NEXT:  .LBB3_1: # %entry
+; MIPSR6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPSR6-NEXT:    lld $2, 0($4)
+; MIPSR6-NEXT:    sltu $3, $2, $5
+; MIPSR6-NEXT:    selnez $1, $2, $3
+; MIPSR6-NEXT:    seleqz $3, $5, $3
+; MIPSR6-NEXT:    or $1, $1, $3
+; MIPSR6-NEXT:    scd $1, 0($4)
+; MIPSR6-NEXT:    beqzc $1, .LBB3_1
+; MIPSR6-NEXT:  # %bb.2: # %entry
+; MIPSR6-NEXT:    sync
+; MIPSR6-NEXT:    jrc $ra
+entry:
+  %0 = atomicrmw umin i64* %ptr, i64 %val seq_cst
+  ret i64 %0
+}
+
diff --git a/llvm/test/CodeGen/Mips/atomic-min-max.ll b/llvm/test/CodeGen/Mips/atomic-min-max.ll
new file mode 100644
index 000000000..a62008519
--- /dev/null
+++ b/llvm/test/CodeGen/Mips/atomic-min-max.ll
@@ -0,0 +1,4674 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -march=mips -O0 -mcpu=mips32r2 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=MIPS
+; RUN: llc -march=mips -O0 -mcpu=mips32r6 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=MIPSR6
+; RUN: llc -march=mips -O0 -mcpu=mips32r2 -mattr=+micromips -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=MM
+; RUN: llc -march=mips -O0 -mcpu=mips32r6 -mattr=+micromips -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=MMR6
+; RUN: llc -march=mipsel -O0 -mcpu=mips32r2 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=MIPSEL
+; RUN: llc -march=mipsel -O0 -mcpu=mips32r6 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=MIPSELR6
+; RUN: llc -march=mipsel -O0 -mcpu=mips32r2 -mattr=+micromips -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=MMEL
+; RUN: llc -march=mipsel -O0 -mcpu=mips32r6 -mattr=+micromips -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=MMELR6
+; RUN: llc -march=mips64 -O0 -mcpu=mips64r2 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=MIPS64
+; RUN: llc -march=mips64 -O0 -mcpu=mips64r6 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=MIPS64R6
+; RUN: llc -march=mips64el -O0 -mcpu=mips64r2 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=MIPS64EL
+; RUN: llc -march=mips64el -O0 -mcpu=mips64r6 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=MIPS64ELR6
+
+define i32 @test_max_32(i32* nocapture %ptr, i32 signext %val) {
+; MIPS-LABEL: test_max_32:
+; MIPS:       # %bb.0: # %entry
+; MIPS-NEXT:    sync
+; MIPS-NEXT:  $BB0_1: # %entry
+; MIPS-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS-NEXT:    ll $2, 0($4)
+; MIPS-NEXT:    slt $3, $2, $5
+; MIPS-NEXT:    move $1, $2
+; MIPS-NEXT:    movn $1, $5, $3
+; MIPS-NEXT:    sc $1, 0($4)
+; MIPS-NEXT:    beqz $1, $BB0_1
+; MIPS-NEXT:    nop
+; MIPS-NEXT:  # %bb.2: # %entry
+; MIPS-NEXT:    sync
+; MIPS-NEXT:    jr $ra
+; MIPS-NEXT:    nop
+;
+; MIPSR6-LABEL: test_max_32:
+; MIPSR6:       # %bb.0: # %entry
+; MIPSR6-NEXT:    sync
+; MIPSR6-NEXT:  $BB0_1: # %entry
+; MIPSR6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPSR6-NEXT:    ll $2, 0($4)
+; MIPSR6-NEXT:    slt $3, $2, $5
+; MIPSR6-NEXT:    seleqz $1, $2, $3
+; MIPSR6-NEXT:    selnez $3, $5, $3
+; MIPSR6-NEXT:    or $1, $1, $3
+; MIPSR6-NEXT:    sc $1, 0($4)
+; MIPSR6-NEXT:    beqzc $1, $BB0_1
+; MIPSR6-NEXT:  # %bb.2: # %entry
+; MIPSR6-NEXT:    sync
+; MIPSR6-NEXT:    jrc $ra
+;
+; MM-LABEL: test_max_32:
+; MM:       # %bb.0: # %entry
+; MM-NEXT:    sync
+; MM-NEXT:  $BB0_1: # %entry
+; MM-NEXT:    # =>This Inner Loop Header: Depth=1
+; MM-NEXT:    ll $2, 0($4)
+; MM-NEXT:    slt $3, $2, $5
+; MM-NEXT:    or $1, $2, $zero
+; MM-NEXT:    movn $1, $5, $3
+; MM-NEXT:    sc $1, 0($4)
+; MM-NEXT:    beqzc $1, $BB0_1
+; MM-NEXT:  # %bb.2: # %entry
+; MM-NEXT:    sync
+; MM-NEXT:    jrc $ra
+;
+; MMR6-LABEL: test_max_32:
+; MMR6:       # %bb.0: # %entry
+; MMR6-NEXT:    sync
+; MMR6-NEXT:  $BB0_1: # %entry
+; MMR6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MMR6-NEXT:    ll $2, 0($4)
+; MMR6-NEXT:    slt $3, $2, $5
+; MMR6-NEXT:    seleqz $1, $2, $3
+; MMR6-NEXT:    selnez $3, $5, $3
+; MMR6-NEXT:    or $1, $1, $3
+; MMR6-NEXT:    sc $1, 0($4)
+; MMR6-NEXT:    beqc $1, $zero, $BB0_1
+; MMR6-NEXT:  # %bb.2: # %entry
+; MMR6-NEXT:    sync
+; MMR6-NEXT:    jrc $ra
+;
+; MIPSEL-LABEL: test_max_32:
+; MIPSEL:       # %bb.0: # %entry
+; MIPSEL-NEXT:    sync
+; MIPSEL-NEXT:  $BB0_1: # %entry
+; MIPSEL-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPSEL-NEXT:    ll $2, 0($4)
+; MIPSEL-NEXT:    slt $3, $2, $5
+; MIPSEL-NEXT:    move $1, $2
+; MIPSEL-NEXT:    movn $1, $5, $3
+; MIPSEL-NEXT:    sc $1, 0($4)
+; MIPSEL-NEXT:    beqz $1, $BB0_1
+; MIPSEL-NEXT:    nop
+; MIPSEL-NEXT:  # %bb.2: # %entry
+; MIPSEL-NEXT:    sync
+; MIPSEL-NEXT:    jr $ra
+; MIPSEL-NEXT:    nop
+;
+; MIPSELR6-LABEL: test_max_32:
+; MIPSELR6:       # %bb.0: # %entry
+; MIPSELR6-NEXT:    sync
+; MIPSELR6-NEXT:  $BB0_1: # %entry
+; MIPSELR6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPSELR6-NEXT:    ll $2, 0($4)
+; MIPSELR6-NEXT:    slt $3, $2, $5
+; MIPSELR6-NEXT:    seleqz $1, $2, $3
+; MIPSELR6-NEXT:    selnez $3, $5, $3
+; MIPSELR6-NEXT:    or $1, $1, $3
+; MIPSELR6-NEXT:    sc $1, 0($4)
+; MIPSELR6-NEXT:    beqzc $1, $BB0_1
+; MIPSELR6-NEXT:  # %bb.2: # %entry
+; MIPSELR6-NEXT:    sync
+; MIPSELR6-NEXT:    jrc $ra
+;
+; MMEL-LABEL: test_max_32:
+; MMEL:       # %bb.0: # %entry
+; MMEL-NEXT:    sync
+; MMEL-NEXT:  $BB0_1: # %entry
+; MMEL-NEXT:    # =>This Inner Loop Header: Depth=1
+; MMEL-NEXT:    ll $2, 0($4)
+; MMEL-NEXT:    slt $3, $2, $5
+; MMEL-NEXT:    or $1, $2, $zero
+; MMEL-NEXT:    movn $1, $5, $3
+; MMEL-NEXT:    sc $1, 0($4)
+; MMEL-NEXT:    beqzc $1, $BB0_1
+; MMEL-NEXT:  # %bb.2: # %entry
+; MMEL-NEXT:    sync
+; MMEL-NEXT:    jrc $ra
+;
+; MMELR6-LABEL: test_max_32:
+; MMELR6:       # %bb.0: # %entry
+; MMELR6-NEXT:    sync
+; MMELR6-NEXT:  $BB0_1: # %entry
+; MMELR6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MMELR6-NEXT:    ll $2, 0($4)
+; MMELR6-NEXT:    slt $3, $2, $5
+; MMELR6-NEXT:    seleqz $1, $2, $3
+; MMELR6-NEXT:    selnez $3, $5, $3
+; MMELR6-NEXT:    or $1, $1, $3
+; MMELR6-NEXT:    sc $1, 0($4)
+; MMELR6-NEXT:    beqc $1, $zero, $BB0_1
+; MMELR6-NEXT:  # %bb.2: # %entry
+; MMELR6-NEXT:    sync
+; MMELR6-NEXT:    jrc $ra
+;
+; MIPS64-LABEL: test_max_32:
+; MIPS64:       # %bb.0: # %entry
+; MIPS64-NEXT:    # kill: def $a1 killed $a1 killed $a1_64
+; MIPS64-NEXT:    sync
+; MIPS64-NEXT:  .LBB0_1: # %entry
+; MIPS64-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS64-NEXT:    ll $2, 0($4)
+; MIPS64-NEXT:    slt $3, $2, $5
+; MIPS64-NEXT:    move $1, $2
+; MIPS64-NEXT:    movn $1, $5, $3
+; MIPS64-NEXT:    sc $1, 0($4)
+; MIPS64-NEXT:    beqz $1, .LBB0_1
+; MIPS64-NEXT:    nop
+; MIPS64-NEXT:  # %bb.2: # %entry
+; MIPS64-NEXT:    sync
+; MIPS64-NEXT:    jr $ra
+; MIPS64-NEXT:    nop
+;
+; MIPS64R6-LABEL: test_max_32:
+; MIPS64R6:       # %bb.0: # %entry
+; MIPS64R6-NEXT:    # kill: def $a1 killed $a1 killed $a1_64
+; MIPS64R6-NEXT:    sync
+; MIPS64R6-NEXT:  .LBB0_1: # %entry
+; MIPS64R6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS64R6-NEXT:    ll $2, 0($4)
+; MIPS64R6-NEXT:    slt $3, $2, $5
+; MIPS64R6-NEXT:    seleqz $1, $2, $3
+; MIPS64R6-NEXT:    selnez $3, $5, $3
+; MIPS64R6-NEXT:    or $1, $1, $3
+; MIPS64R6-NEXT:    sc $1, 0($4)
+; MIPS64R6-NEXT:    beqzc $1, .LBB0_1
+; MIPS64R6-NEXT:  # %bb.2: # %entry
+; MIPS64R6-NEXT:    sync
+; MIPS64R6-NEXT:    jrc $ra
+;
+; MIPS64EL-LABEL: test_max_32:
+; MIPS64EL:       # %bb.0: # %entry
+; MIPS64EL-NEXT:    # kill: def $a1 killed $a1 killed $a1_64
+; MIPS64EL-NEXT:    sync
+; MIPS64EL-NEXT:  .LBB0_1: # %entry
+; MIPS64EL-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS64EL-NEXT:    ll $2, 0($4)
+; MIPS64EL-NEXT:    slt $3, $2, $5
+; MIPS64EL-NEXT:    move $1, $2
+; MIPS64EL-NEXT:    movn $1, $5, $3
+; MIPS64EL-NEXT:    sc $1, 0($4)
+; MIPS64EL-NEXT:    beqz $1, .LBB0_1
+; MIPS64EL-NEXT:    nop
+; MIPS64EL-NEXT:  # %bb.2: # %entry
+; MIPS64EL-NEXT:    sync
+; MIPS64EL-NEXT:    jr $ra
+; MIPS64EL-NEXT:    nop
+;
+; MIPS64ELR6-LABEL: test_max_32:
+; MIPS64ELR6:       # %bb.0: # %entry
+; MIPS64ELR6-NEXT:    # kill: def $a1 killed $a1 killed $a1_64
+; MIPS64ELR6-NEXT:    sync
+; MIPS64ELR6-NEXT:  .LBB0_1: # %entry
+; MIPS64ELR6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS64ELR6-NEXT:    ll $2, 0($4)
+; MIPS64ELR6-NEXT:    slt $3, $2, $5
+; MIPS64ELR6-NEXT:    seleqz $1, $2, $3
+; MIPS64ELR6-NEXT:    selnez $3, $5, $3
+; MIPS64ELR6-NEXT:    or $1, $1, $3
+; MIPS64ELR6-NEXT:    sc $1, 0($4)
+; MIPS64ELR6-NEXT:    beqzc $1, .LBB0_1
+; MIPS64ELR6-NEXT:  # %bb.2: # %entry
+; MIPS64ELR6-NEXT:    sync
+; MIPS64ELR6-NEXT:    jrc $ra
+entry:
+  %0 = atomicrmw max i32* %ptr, i32 %val seq_cst
+  ret i32 %0
+}
+
+define i32 @test_min_32(i32* nocapture %ptr, i32 signext %val) {
+; MIPS-LABEL: test_min_32:
+; MIPS:       # %bb.0: # %entry
+; MIPS-NEXT:    sync
+; MIPS-NEXT:  $BB1_1: # %entry
+; MIPS-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS-NEXT:    ll $2, 0($4)
+; MIPS-NEXT:    slt $3, $2, $5
+; MIPS-NEXT:    move $1, $2
+; MIPS-NEXT:    movz $1, $5, $3
+; MIPS-NEXT:    sc $1, 0($4)
+; MIPS-NEXT:    beqz $1, $BB1_1
+; MIPS-NEXT:    nop
+; MIPS-NEXT:  # %bb.2: # %entry
+; MIPS-NEXT:    sync
+; MIPS-NEXT:    jr $ra
+; MIPS-NEXT:    nop
+;
+; MIPSR6-LABEL: test_min_32:
+; MIPSR6:       # %bb.0: # %entry
+; MIPSR6-NEXT:    sync
+; MIPSR6-NEXT:  $BB1_1: # %entry
+; MIPSR6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPSR6-NEXT:    ll $2, 0($4)
+; MIPSR6-NEXT:    slt $3, $2, $5
+; MIPSR6-NEXT:    selnez $1, $2, $3
+; MIPSR6-NEXT:    seleqz $3, $5, $3
+; MIPSR6-NEXT:    or $1, $1, $3
+; MIPSR6-NEXT:    sc $1, 0($4)
+; MIPSR6-NEXT:    beqzc $1, $BB1_1
+; MIPSR6-NEXT:  # %bb.2: # %entry
+; MIPSR6-NEXT:    sync
+; MIPSR6-NEXT:    jrc $ra
+;
+; MM-LABEL: test_min_32:
+; MM:       # %bb.0: # %entry
+; MM-NEXT:    sync
+; MM-NEXT:  $BB1_1: # %entry
+; MM-NEXT:    # =>This Inner Loop Header: Depth=1
+; MM-NEXT:    ll $2, 0($4)
+; MM-NEXT:    slt $3, $2, $5
+; MM-NEXT:    or $1, $2, $zero
+; MM-NEXT:    movz $1, $5, $3
+; MM-NEXT:    sc $1, 0($4)
+; MM-NEXT:    beqzc $1, $BB1_1
+; MM-NEXT:  # %bb.2: # %entry
+; MM-NEXT:    sync
+; MM-NEXT:    jrc $ra
+;
+; MMR6-LABEL: test_min_32:
+; MMR6:       # %bb.0: # %entry
+; MMR6-NEXT:    sync
+; MMR6-NEXT:  $BB1_1: # %entry
+; MMR6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MMR6-NEXT:    ll $2, 0($4)
+; MMR6-NEXT:    slt $3, $2, $5
+; MMR6-NEXT:    selnez $1, $2, $3
+; MMR6-NEXT:    seleqz $3, $5, $3
+; MMR6-NEXT:    or $1, $1, $3
+; MMR6-NEXT:    sc $1, 0($4)
+; MMR6-NEXT:    beqc $1, $zero, $BB1_1
+; MMR6-NEXT:  # %bb.2: # %entry
+; MMR6-NEXT:    sync
+; MMR6-NEXT:    jrc $ra
+;
+; MIPSEL-LABEL: test_min_32:
+; MIPSEL:       # %bb.0: # %entry
+; MIPSEL-NEXT:    sync
+; MIPSEL-NEXT:  $BB1_1: # %entry
+; MIPSEL-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPSEL-NEXT:    ll $2, 0($4)
+; MIPSEL-NEXT:    slt $3, $2, $5
+; MIPSEL-NEXT:    move $1, $2
+; MIPSEL-NEXT:    movz $1, $5, $3
+; MIPSEL-NEXT:    sc $1, 0($4)
+; MIPSEL-NEXT:    beqz $1, $BB1_1
+; MIPSEL-NEXT:    nop
+; MIPSEL-NEXT:  # %bb.2: # %entry
+; MIPSEL-NEXT:    sync
+; MIPSEL-NEXT:    jr $ra
+; MIPSEL-NEXT:    nop
+;
+; MIPSELR6-LABEL: test_min_32:
+; MIPSELR6:       # %bb.0: # %entry
+; MIPSELR6-NEXT:    sync
+; MIPSELR6-NEXT:  $BB1_1: # %entry
+; MIPSELR6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPSELR6-NEXT:    ll $2, 0($4)
+; MIPSELR6-NEXT:    slt $3, $2, $5
+; MIPSELR6-NEXT:    selnez $1, $2, $3
+; MIPSELR6-NEXT:    seleqz $3, $5, $3
+; MIPSELR6-NEXT:    or $1, $1, $3
+; MIPSELR6-NEXT:    sc $1, 0($4)
+; MIPSELR6-NEXT:    beqzc $1, $BB1_1
+; MIPSELR6-NEXT:  # %bb.2: # %entry
+; MIPSELR6-NEXT:    sync
+; MIPSELR6-NEXT:    jrc $ra
+;
+; MMEL-LABEL: test_min_32:
+; MMEL:       # %bb.0: # %entry
+; MMEL-NEXT:    sync
+; MMEL-NEXT:  $BB1_1: # %entry
+; MMEL-NEXT:    # =>This Inner Loop Header: Depth=1
+; MMEL-NEXT:    ll $2, 0($4)
+; MMEL-NEXT:    slt $3, $2, $5
+; MMEL-NEXT:    or $1, $2, $zero
+; MMEL-NEXT:    movz $1, $5, $3
+; MMEL-NEXT:    sc $1, 0($4)
+; MMEL-NEXT:    beqzc $1, $BB1_1
+; MMEL-NEXT:  # %bb.2: # %entry
+; MMEL-NEXT:    sync
+; MMEL-NEXT:    jrc $ra
+;
+; MMELR6-LABEL: test_min_32:
+; MMELR6:       # %bb.0: # %entry
+; MMELR6-NEXT:    sync
+; MMELR6-NEXT:  $BB1_1: # %entry
+; MMELR6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MMELR6-NEXT:    ll $2, 0($4)
+; MMELR6-NEXT:    slt $3, $2, $5
+; MMELR6-NEXT:    selnez $1, $2, $3
+; MMELR6-NEXT:    seleqz $3, $5, $3
+; MMELR6-NEXT:    or $1, $1, $3
+; MMELR6-NEXT:    sc $1, 0($4)
+; MMELR6-NEXT:    beqc $1, $zero, $BB1_1
+; MMELR6-NEXT:  # %bb.2: # %entry
+; MMELR6-NEXT:    sync
+; MMELR6-NEXT:    jrc $ra
+;
+; MIPS64-LABEL: test_min_32:
+; MIPS64:       # %bb.0: # %entry
+; MIPS64-NEXT:    # kill: def $a1 killed $a1 killed $a1_64
+; MIPS64-NEXT:    sync
+; MIPS64-NEXT:  .LBB1_1: # %entry
+; MIPS64-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS64-NEXT:    ll $2, 0($4)
+; MIPS64-NEXT:    slt $3, $2, $5
+; MIPS64-NEXT:    move $1, $2
+; MIPS64-NEXT:    movz $1, $5, $3
+; MIPS64-NEXT:    sc $1, 0($4)
+; MIPS64-NEXT:    beqz $1, .LBB1_1
+; MIPS64-NEXT:    nop
+; MIPS64-NEXT:  # %bb.2: # %entry
+; MIPS64-NEXT:    sync
+; MIPS64-NEXT:    jr $ra
+; MIPS64-NEXT:    nop
+;
+; MIPS64R6-LABEL: test_min_32:
+; MIPS64R6:       # %bb.0: # %entry
+; MIPS64R6-NEXT:    # kill: def $a1 killed $a1 killed $a1_64
+; MIPS64R6-NEXT:    sync
+; MIPS64R6-NEXT:  .LBB1_1: # %entry
+; MIPS64R6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS64R6-NEXT:    ll $2, 0($4)
+; MIPS64R6-NEXT:    slt $3, $2, $5
+; MIPS64R6-NEXT:    selnez $1, $2, $3
+; MIPS64R6-NEXT:    seleqz $3, $5, $3
+; MIPS64R6-NEXT:    or $1, $1, $3
+; MIPS64R6-NEXT:    sc $1, 0($4)
+; MIPS64R6-NEXT:    beqzc $1, .LBB1_1
+; MIPS64R6-NEXT:  # %bb.2: # %entry
+; MIPS64R6-NEXT:    sync
+; MIPS64R6-NEXT:    jrc $ra
+;
+; MIPS64EL-LABEL: test_min_32:
+; MIPS64EL:       # %bb.0: # %entry
+; MIPS64EL-NEXT:    # kill: def $a1 killed $a1 killed $a1_64
+; MIPS64EL-NEXT:    sync
+; MIPS64EL-NEXT:  .LBB1_1: # %entry
+; MIPS64EL-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS64EL-NEXT:    ll $2, 0($4)
+; MIPS64EL-NEXT:    slt $3, $2, $5
+; MIPS64EL-NEXT:    move $1, $2
+; MIPS64EL-NEXT:    movz $1, $5, $3
+; MIPS64EL-NEXT:    sc $1, 0($4)
+; MIPS64EL-NEXT:    beqz $1, .LBB1_1
+; MIPS64EL-NEXT:    nop
+; MIPS64EL-NEXT:  # %bb.2: # %entry
+; MIPS64EL-NEXT:    sync
+; MIPS64EL-NEXT:    jr $ra
+; MIPS64EL-NEXT:    nop
+;
+; MIPS64ELR6-LABEL: test_min_32:
+; MIPS64ELR6:       # %bb.0: # %entry
+; MIPS64ELR6-NEXT:    # kill: def $a1 killed $a1 killed $a1_64
+; MIPS64ELR6-NEXT:    sync
+; MIPS64ELR6-NEXT:  .LBB1_1: # %entry
+; MIPS64ELR6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS64ELR6-NEXT:    ll $2, 0($4)
+; MIPS64ELR6-NEXT:    slt $3, $2, $5
+; MIPS64ELR6-NEXT:    selnez $1, $2, $3
+; MIPS64ELR6-NEXT:    seleqz $3, $5, $3
+; MIPS64ELR6-NEXT:    or $1, $1, $3
+; MIPS64ELR6-NEXT:    sc $1, 0($4)
+; MIPS64ELR6-NEXT:    beqzc $1, .LBB1_1
+; MIPS64ELR6-NEXT:  # %bb.2: # %entry
+; MIPS64ELR6-NEXT:    sync
+; MIPS64ELR6-NEXT:    jrc $ra
+entry:
+  %0 = atomicrmw min i32* %ptr, i32 %val seq_cst
+  ret i32 %0
+}
+
+define i32 @test_umax_32(i32* nocapture %ptr, i32 signext %val) {
+; MIPS-LABEL: test_umax_32:
+; MIPS:       # %bb.0: # %entry
+; MIPS-NEXT:    sync
+; MIPS-NEXT:  $BB2_1: # %entry
+; MIPS-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS-NEXT:    ll $2, 0($4)
+; MIPS-NEXT:    sltu $3, $2, $5
+; MIPS-NEXT:    move $1, $2
+; MIPS-NEXT:    movn $1, $5, $3
+; MIPS-NEXT:    sc $1, 0($4)
+; MIPS-NEXT:    beqz $1, $BB2_1
+; MIPS-NEXT:    nop
+; MIPS-NEXT:  # %bb.2: # %entry
+; MIPS-NEXT:    sync
+; MIPS-NEXT:    jr $ra
+; MIPS-NEXT:    nop
+;
+; MIPSR6-LABEL: test_umax_32:
+; MIPSR6:       # %bb.0: # %entry
+; MIPSR6-NEXT:    sync
+; MIPSR6-NEXT:  $BB2_1: # %entry
+; MIPSR6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPSR6-NEXT:    ll $2, 0($4)
+; MIPSR6-NEXT:    sltu $3, $2, $5
+; MIPSR6-NEXT:    seleqz $1, $2, $3
+; MIPSR6-NEXT:    selnez $3, $5, $3
+; MIPSR6-NEXT:    or $1, $1, $3
+; MIPSR6-NEXT:    sc $1, 0($4)
+; MIPSR6-NEXT:    beqzc $1, $BB2_1
+; MIPSR6-NEXT:  # %bb.2: # %entry
+; MIPSR6-NEXT:    sync
+; MIPSR6-NEXT:    jrc $ra
+;
+; MM-LABEL: test_umax_32:
+; MM:       # %bb.0: # %entry
+; MM-NEXT:    sync
+; MM-NEXT:  $BB2_1: # %entry
+; MM-NEXT:    # =>This Inner Loop Header: Depth=1
+; MM-NEXT:    ll $2, 0($4)
+; MM-NEXT:    sltu $3, $2, $5
+; MM-NEXT:    or $1, $2, $zero
+; MM-NEXT:    movn $1, $5, $3
+; MM-NEXT:    sc $1, 0($4)
+; MM-NEXT:    beqzc $1, $BB2_1
+; MM-NEXT:  # %bb.2: # %entry
+; MM-NEXT:    sync
+; MM-NEXT:    jrc $ra
+;
+; MMR6-LABEL: test_umax_32:
+; MMR6:       # %bb.0: # %entry
+; MMR6-NEXT:    sync
+; MMR6-NEXT:  $BB2_1: # %entry
+; MMR6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MMR6-NEXT:    ll $2, 0($4)
+; MMR6-NEXT:    sltu $3, $2, $5
+; MMR6-NEXT:    seleqz $1, $2, $3
+; MMR6-NEXT:    selnez $3, $5, $3
+; MMR6-NEXT:    or $1, $1, $3
+; MMR6-NEXT:    sc $1, 0($4)
+; MMR6-NEXT:    beqc $1, $zero, $BB2_1
+; MMR6-NEXT:  # %bb.2: # %entry
+; MMR6-NEXT:    sync
+; MMR6-NEXT:    jrc $ra
+;
+; MIPSEL-LABEL: test_umax_32:
+; MIPSEL:       # %bb.0: # %entry
+; MIPSEL-NEXT:    sync
+; MIPSEL-NEXT:  $BB2_1: # %entry
+; MIPSEL-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPSEL-NEXT:    ll $2, 0($4)
+; MIPSEL-NEXT:    sltu $3, $2, $5
+; MIPSEL-NEXT:    move $1, $2
+; MIPSEL-NEXT:    movn $1, $5, $3
+; MIPSEL-NEXT:    sc $1, 0($4)
+; MIPSEL-NEXT:    beqz $1, $BB2_1
+; MIPSEL-NEXT:    nop
+; MIPSEL-NEXT:  # %bb.2: # %entry
+; MIPSEL-NEXT:    sync
+; MIPSEL-NEXT:    jr $ra
+; MIPSEL-NEXT:    nop
+;
+; MIPSELR6-LABEL: test_umax_32:
+; MIPSELR6:       # %bb.0: # %entry
+; MIPSELR6-NEXT:    sync
+; MIPSELR6-NEXT:  $BB2_1: # %entry
+; MIPSELR6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPSELR6-NEXT:    ll $2, 0($4)
+; MIPSELR6-NEXT:    sltu $3, $2, $5
+; MIPSELR6-NEXT:    seleqz $1, $2, $3
+; MIPSELR6-NEXT:    selnez $3, $5, $3
+; MIPSELR6-NEXT:    or $1, $1, $3
+; MIPSELR6-NEXT:    sc $1, 0($4)
+; MIPSELR6-NEXT:    beqzc $1, $BB2_1
+; MIPSELR6-NEXT:  # %bb.2: # %entry
+; MIPSELR6-NEXT:    sync
+; MIPSELR6-NEXT:    jrc $ra
+;
+; MMEL-LABEL: test_umax_32:
+; MMEL:       # %bb.0: # %entry
+; MMEL-NEXT:    sync
+; MMEL-NEXT:  $BB2_1: # %entry
+; MMEL-NEXT:    # =>This Inner Loop Header: Depth=1
+; MMEL-NEXT:    ll $2, 0($4)
+; MMEL-NEXT:    sltu $3, $2, $5
+; MMEL-NEXT:    or $1, $2, $zero
+; MMEL-NEXT:    movn $1, $5, $3
+; MMEL-NEXT:    sc $1, 0($4)
+; MMEL-NEXT:    beqzc $1, $BB2_1
+; MMEL-NEXT:  # %bb.2: # %entry
+; MMEL-NEXT:    sync
+; MMEL-NEXT:    jrc $ra
+;
+; MMELR6-LABEL: test_umax_32:
+; MMELR6:       # %bb.0: # %entry
+; MMELR6-NEXT:    sync
+; MMELR6-NEXT:  $BB2_1: # %entry
+; MMELR6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MMELR6-NEXT:    ll $2, 0($4)
+; MMELR6-NEXT:    sltu $3, $2, $5
+; MMELR6-NEXT:    seleqz $1, $2, $3
+; MMELR6-NEXT:    selnez $3, $5, $3
+; MMELR6-NEXT:    or $1, $1, $3
+; MMELR6-NEXT:    sc $1, 0($4)
+; MMELR6-NEXT:    beqc $1, $zero, $BB2_1
+; MMELR6-NEXT:  # %bb.2: # %entry
+; MMELR6-NEXT:    sync
+; MMELR6-NEXT:    jrc $ra
+;
+; MIPS64-LABEL: test_umax_32:
+; MIPS64:       # %bb.0: # %entry
+; MIPS64-NEXT:    # kill: def $a1 killed $a1 killed $a1_64
+; MIPS64-NEXT:    sync
+; MIPS64-NEXT:  .LBB2_1: # %entry
+; MIPS64-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS64-NEXT:    ll $2, 0($4)
+; MIPS64-NEXT:    sltu $3, $2, $5
+; MIPS64-NEXT:    move $1, $2
+; MIPS64-NEXT:    movn $1, $5, $3
+; MIPS64-NEXT:    sc $1, 0($4)
+; MIPS64-NEXT:    beqz $1, .LBB2_1
+; MIPS64-NEXT:    nop
+; MIPS64-NEXT:  # %bb.2: # %entry
+; MIPS64-NEXT:    sync
+; MIPS64-NEXT:    jr $ra
+; MIPS64-NEXT:    nop
+;
+; MIPS64R6-LABEL: test_umax_32:
+; MIPS64R6:       # %bb.0: # %entry
+; MIPS64R6-NEXT:    # kill: def $a1 killed $a1 killed $a1_64
+; MIPS64R6-NEXT:    sync
+; MIPS64R6-NEXT:  .LBB2_1: # %entry
+; MIPS64R6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS64R6-NEXT:    ll $2, 0($4)
+; MIPS64R6-NEXT:    sltu $3, $2, $5
+; MIPS64R6-NEXT:    seleqz $1, $2, $3
+; MIPS64R6-NEXT:    selnez $3, $5, $3
+; MIPS64R6-NEXT:    or $1, $1, $3
+; MIPS64R6-NEXT:    sc $1, 0($4)
+; MIPS64R6-NEXT:    beqzc $1, .LBB2_1
+; MIPS64R6-NEXT:  # %bb.2: # %entry
+; MIPS64R6-NEXT:    sync
+; MIPS64R6-NEXT:    jrc $ra
+;
+; MIPS64EL-LABEL: test_umax_32:
+; MIPS64EL:       # %bb.0: # %entry
+; MIPS64EL-NEXT:    # kill: def $a1 killed $a1 killed $a1_64
+; MIPS64EL-NEXT:    sync
+; MIPS64EL-NEXT:  .LBB2_1: # %entry
+; MIPS64EL-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS64EL-NEXT:    ll $2, 0($4)
+; MIPS64EL-NEXT:    sltu $3, $2, $5
+; MIPS64EL-NEXT:    move $1, $2
+; MIPS64EL-NEXT:    movn $1, $5, $3
+; MIPS64EL-NEXT:    sc $1, 0($4)
+; MIPS64EL-NEXT:    beqz $1, .LBB2_1
+; MIPS64EL-NEXT:    nop
+; MIPS64EL-NEXT:  # %bb.2: # %entry
+; MIPS64EL-NEXT:    sync
+; MIPS64EL-NEXT:    jr $ra
+; MIPS64EL-NEXT:    nop
+;
+; MIPS64ELR6-LABEL: test_umax_32:
+; MIPS64ELR6:       # %bb.0: # %entry
+; MIPS64ELR6-NEXT:    # kill: def $a1 killed $a1 killed $a1_64
+; MIPS64ELR6-NEXT:    sync
+; MIPS64ELR6-NEXT:  .LBB2_1: # %entry
+; MIPS64ELR6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS64ELR6-NEXT:    ll $2, 0($4)
+; MIPS64ELR6-NEXT:    sltu $3, $2, $5
+; MIPS64ELR6-NEXT:    seleqz $1, $2, $3
+; MIPS64ELR6-NEXT:    selnez $3, $5, $3
+; MIPS64ELR6-NEXT:    or $1, $1, $3
+; MIPS64ELR6-NEXT:    sc $1, 0($4)
+; MIPS64ELR6-NEXT:    beqzc $1, .LBB2_1
+; MIPS64ELR6-NEXT:  # %bb.2: # %entry
+; MIPS64ELR6-NEXT:    sync
+; MIPS64ELR6-NEXT:    jrc $ra
+entry:
+  %0 = atomicrmw umax i32* %ptr, i32 %val seq_cst
+  ret i32 %0
+}
+
+define i32 @test_umin_32(i32* nocapture %ptr, i32 signext %val) {
+; MIPS-LABEL: test_umin_32:
+; MIPS:       # %bb.0: # %entry
+; MIPS-NEXT:    sync
+; MIPS-NEXT:  $BB3_1: # %entry
+; MIPS-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS-NEXT:    ll $2, 0($4)
+; MIPS-NEXT:    sltu $3, $2, $5
+; MIPS-NEXT:    move $1, $2
+; MIPS-NEXT:    movz $1, $5, $3
+; MIPS-NEXT:    sc $1, 0($4)
+; MIPS-NEXT:    beqz $1, $BB3_1
+; MIPS-NEXT:    nop
+; MIPS-NEXT:  # %bb.2: # %entry
+; MIPS-NEXT:    sync
+; MIPS-NEXT:    jr $ra
+; MIPS-NEXT:    nop
+;
+; MIPSR6-LABEL: test_umin_32:
+; MIPSR6:       # %bb.0: # %entry
+; MIPSR6-NEXT:    sync
+; MIPSR6-NEXT:  $BB3_1: # %entry
+; MIPSR6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPSR6-NEXT:    ll $2, 0($4)
+; MIPSR6-NEXT:    sltu $3, $2, $5
+; MIPSR6-NEXT:    selnez $1, $2, $3
+; MIPSR6-NEXT:    seleqz $3, $5, $3
+; MIPSR6-NEXT:    or $1, $1, $3
+; MIPSR6-NEXT:    sc $1, 0($4)
+; MIPSR6-NEXT:    beqzc $1, $BB3_1
+; MIPSR6-NEXT:  # %bb.2: # %entry
+; MIPSR6-NEXT:    sync
+; MIPSR6-NEXT:    jrc $ra
+;
+; MM-LABEL: test_umin_32:
+; MM:       # %bb.0: # %entry
+; MM-NEXT:    sync
+; MM-NEXT:  $BB3_1: # %entry
+; MM-NEXT:    # =>This Inner Loop Header: Depth=1
+; MM-NEXT:    ll $2, 0($4)
+; MM-NEXT:    sltu $3, $2, $5
+; MM-NEXT:    or $1, $2, $zero
+; MM-NEXT:    movz $1, $5, $3
+; MM-NEXT:    sc $1, 0($4)
+; MM-NEXT:    beqzc $1, $BB3_1
+; MM-NEXT:  # %bb.2: # %entry
+; MM-NEXT:    sync
+; MM-NEXT:    jrc $ra
+;
+; MMR6-LABEL: test_umin_32:
+; MMR6:       # %bb.0: # %entry
+; MMR6-NEXT:    sync
+; MMR6-NEXT:  $BB3_1: # %entry
+; MMR6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MMR6-NEXT:    ll $2, 0($4)
+; MMR6-NEXT:    sltu $3, $2, $5
+; MMR6-NEXT:    selnez $1, $2, $3
+; MMR6-NEXT:    seleqz $3, $5, $3
+; MMR6-NEXT:    or $1, $1, $3
+; MMR6-NEXT:    sc $1, 0($4)
+; MMR6-NEXT:    beqc $1, $zero, $BB3_1
+; MMR6-NEXT:  # %bb.2: # %entry
+; MMR6-NEXT:    sync
+; MMR6-NEXT:    jrc $ra
+;
+; MIPSEL-LABEL: test_umin_32:
+; MIPSEL:       # %bb.0: # %entry
+; MIPSEL-NEXT:    sync
+; MIPSEL-NEXT:  $BB3_1: # %entry
+; MIPSEL-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPSEL-NEXT:    ll $2, 0($4)
+; MIPSEL-NEXT:    sltu $3, $2, $5
+; MIPSEL-NEXT:    move $1, $2
+; MIPSEL-NEXT:    movz $1, $5, $3
+; MIPSEL-NEXT:    sc $1, 0($4)
+; MIPSEL-NEXT:    beqz $1, $BB3_1
+; MIPSEL-NEXT:    nop
+; MIPSEL-NEXT:  # %bb.2: # %entry
+; MIPSEL-NEXT:    sync
+; MIPSEL-NEXT:    jr $ra
+; MIPSEL-NEXT:    nop
+;
+; MIPSELR6-LABEL: test_umin_32:
+; MIPSELR6:       # %bb.0: # %entry
+; MIPSELR6-NEXT:    sync
+; MIPSELR6-NEXT:  $BB3_1: # %entry
+; MIPSELR6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPSELR6-NEXT:    ll $2, 0($4)
+; MIPSELR6-NEXT:    sltu $3, $2, $5
+; MIPSELR6-NEXT:    selnez $1, $2, $3
+; MIPSELR6-NEXT:    seleqz $3, $5, $3
+; MIPSELR6-NEXT:    or $1, $1, $3
+; MIPSELR6-NEXT:    sc $1, 0($4)
+; MIPSELR6-NEXT:    beqzc $1, $BB3_1
+; MIPSELR6-NEXT:  # %bb.2: # %entry
+; MIPSELR6-NEXT:    sync
+; MIPSELR6-NEXT:    jrc $ra
+;
+; MMEL-LABEL: test_umin_32:
+; MMEL:       # %bb.0: # %entry
+; MMEL-NEXT:    sync
+; MMEL-NEXT:  $BB3_1: # %entry
+; MMEL-NEXT:    # =>This Inner Loop Header: Depth=1
+; MMEL-NEXT:    ll $2, 0($4)
+; MMEL-NEXT:    sltu $3, $2, $5
+; MMEL-NEXT:    or $1, $2, $zero
+; MMEL-NEXT:    movz $1, $5, $3
+; MMEL-NEXT:    sc $1, 0($4)
+; MMEL-NEXT:    beqzc $1, $BB3_1
+; MMEL-NEXT:  # %bb.2: # %entry
+; MMEL-NEXT:    sync
+; MMEL-NEXT:    jrc $ra
+;
+; MMELR6-LABEL: test_umin_32:
+; MMELR6:       # %bb.0: # %entry
+; MMELR6-NEXT:    sync
+; MMELR6-NEXT:  $BB3_1: # %entry
+; MMELR6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MMELR6-NEXT:    ll $2, 0($4)
+; MMELR6-NEXT:    sltu $3, $2, $5
+; MMELR6-NEXT:    selnez $1, $2, $3
+; MMELR6-NEXT:    seleqz $3, $5, $3
+; MMELR6-NEXT:    or $1, $1, $3
+; MMELR6-NEXT:    sc $1, 0($4)
+; MMELR6-NEXT:    beqc $1, $zero, $BB3_1
+; MMELR6-NEXT:  # %bb.2: # %entry
+; MMELR6-NEXT:    sync
+; MMELR6-NEXT:    jrc $ra
+;
+; MIPS64-LABEL: test_umin_32:
+; MIPS64:       # %bb.0: # %entry
+; MIPS64-NEXT:    # kill: def $a1 killed $a1 killed $a1_64
+; MIPS64-NEXT:    sync
+; MIPS64-NEXT:  .LBB3_1: # %entry
+; MIPS64-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS64-NEXT:    ll $2, 0($4)
+; MIPS64-NEXT:    sltu $3, $2, $5
+; MIPS64-NEXT:    move $1, $2
+; MIPS64-NEXT:    movz $1, $5, $3
+; MIPS64-NEXT:    sc $1, 0($4)
+; MIPS64-NEXT:    beqz $1, .LBB3_1
+; MIPS64-NEXT:    nop
+; MIPS64-NEXT:  # %bb.2: # %entry
+; MIPS64-NEXT:    sync
+; MIPS64-NEXT:    jr $ra
+; MIPS64-NEXT:    nop
+;
+; MIPS64R6-LABEL: test_umin_32:
+; MIPS64R6:       # %bb.0: # %entry
+; MIPS64R6-NEXT:    # kill: def $a1 killed $a1 killed $a1_64
+; MIPS64R6-NEXT:    sync
+; MIPS64R6-NEXT:  .LBB3_1: # %entry
+; MIPS64R6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS64R6-NEXT:    ll $2, 0($4)
+; MIPS64R6-NEXT:    sltu $3, $2, $5
+; MIPS64R6-NEXT:    selnez $1, $2, $3
+; MIPS64R6-NEXT:    seleqz $3, $5, $3
+; MIPS64R6-NEXT:    or $1, $1, $3
+; MIPS64R6-NEXT:    sc $1, 0($4)
+; MIPS64R6-NEXT:    beqzc $1, .LBB3_1
+; MIPS64R6-NEXT:  # %bb.2: # %entry
+; MIPS64R6-NEXT:    sync
+; MIPS64R6-NEXT:    jrc $ra
+;
+; MIPS64EL-LABEL: test_umin_32:
+; MIPS64EL:       # %bb.0: # %entry
+; MIPS64EL-NEXT:    # kill: def $a1 killed $a1 killed $a1_64
+; MIPS64EL-NEXT:    sync
+; MIPS64EL-NEXT:  .LBB3_1: # %entry
+; MIPS64EL-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS64EL-NEXT:    ll $2, 0($4)
+; MIPS64EL-NEXT:    sltu $3, $2, $5
+; MIPS64EL-NEXT:    move $1, $2
+; MIPS64EL-NEXT:    movz $1, $5, $3
+; MIPS64EL-NEXT:    sc $1, 0($4)
+; MIPS64EL-NEXT:    beqz $1, .LBB3_1
+; MIPS64EL-NEXT:    nop
+; MIPS64EL-NEXT:  # %bb.2: # %entry
+; MIPS64EL-NEXT:    sync
+; MIPS64EL-NEXT:    jr $ra
+; MIPS64EL-NEXT:    nop
+;
+; MIPS64ELR6-LABEL: test_umin_32:
+; MIPS64ELR6:       # %bb.0: # %entry
+; MIPS64ELR6-NEXT:    # kill: def $a1 killed $a1 killed $a1_64
+; MIPS64ELR6-NEXT:    sync
+; MIPS64ELR6-NEXT:  .LBB3_1: # %entry
+; MIPS64ELR6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS64ELR6-NEXT:    ll $2, 0($4)
+; MIPS64ELR6-NEXT:    sltu $3, $2, $5
+; MIPS64ELR6-NEXT:    selnez $1, $2, $3
+; MIPS64ELR6-NEXT:    seleqz $3, $5, $3
+; MIPS64ELR6-NEXT:    or $1, $1, $3
+; MIPS64ELR6-NEXT:    sc $1, 0($4)
+; MIPS64ELR6-NEXT:    beqzc $1, .LBB3_1
+; MIPS64ELR6-NEXT:  # %bb.2: # %entry
+; MIPS64ELR6-NEXT:    sync
+; MIPS64ELR6-NEXT:    jrc $ra
+entry:
+  %0 = atomicrmw umin i32* %ptr, i32 %val seq_cst
+  ret i32 %0
+}
+
+define i16 @test_max_16(i16* nocapture %ptr, i16 signext %val) {
+; MIPS-LABEL: test_max_16:
+; MIPS:       # %bb.0: # %entry
+; MIPS-NEXT:    addiu $sp, $sp, -8
+; MIPS-NEXT:    .cfi_def_cfa_offset 8
+; MIPS-NEXT:    move $1, $5
+; MIPS-NEXT:    sync
+; MIPS-NEXT:    addiu $2, $zero, -4
+; MIPS-NEXT:    and $2, $4, $2
+; MIPS-NEXT:    andi $3, $4, 3
+; MIPS-NEXT:    xori $3, $3, 2
+; MIPS-NEXT:    sll $3, $3, 3
+; MIPS-NEXT:    ori $4, $zero, 65535
+; MIPS-NEXT:    sllv $4, $4, $3
+; MIPS-NEXT:    nor $6, $zero, $4
+; MIPS-NEXT:    sllv $5, $5, $3
+; MIPS-NEXT:  $BB4_1: # %entry
+; MIPS-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS-NEXT:    ll $8, 0($2)
+; MIPS-NEXT:    slt $11, $8, $5
+; MIPS-NEXT:    move $9, $8
+; MIPS-NEXT:    movn $9, $5, $11
+; MIPS-NEXT:    and $9, $9, $4
+; MIPS-NEXT:    and $10, $8, $6
+; MIPS-NEXT:    or $10, $10, $9
+; MIPS-NEXT:    sc $10, 0($2)
+; MIPS-NEXT:    beqz $10, $BB4_1
+; MIPS-NEXT:    nop
+; MIPS-NEXT:  # %bb.2: # %entry
+; MIPS-NEXT:    and $7, $8, $4
+; MIPS-NEXT:    srlv $7, $7, $3
+; MIPS-NEXT:    seh $7, $7
+; MIPS-NEXT:  # %bb.3: # %entry
+; MIPS-NEXT:    sw $7, 4($sp) # 4-byte Folded Spill
+; MIPS-NEXT:  # %bb.4: # %entry
+; MIPS-NEXT:    sync
+; MIPS-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MIPS-NEXT:    addiu $sp, $sp, 8
+; MIPS-NEXT:    jr $ra
+; MIPS-NEXT:    nop
+;
+; MIPSR6-LABEL: test_max_16:
+; MIPSR6:       # %bb.0: # %entry
+; MIPSR6-NEXT:    addiu $sp, $sp, -8
+; MIPSR6-NEXT:    .cfi_def_cfa_offset 8
+; MIPSR6-NEXT:    move $1, $5
+; MIPSR6-NEXT:    sync
+; MIPSR6-NEXT:    addiu $2, $zero, -4
+; MIPSR6-NEXT:    and $2, $4, $2
+; MIPSR6-NEXT:    andi $3, $4, 3
+; MIPSR6-NEXT:    xori $3, $3, 2
+; MIPSR6-NEXT:    sll $3, $3, 3
+; MIPSR6-NEXT:    ori $4, $zero, 65535
+; MIPSR6-NEXT:    sllv $4, $4, $3
+; MIPSR6-NEXT:    nor $6, $zero, $4
+; MIPSR6-NEXT:    sllv $5, $5, $3
+; MIPSR6-NEXT:  $BB4_1: # %entry
+; MIPSR6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPSR6-NEXT:    ll $8, 0($2)
+; MIPSR6-NEXT:    slt $11, $8, $5
+; MIPSR6-NEXT:    seleqz $9, $8, $11
+; MIPSR6-NEXT:    selnez $11, $5, $11
+; MIPSR6-NEXT:    or $9, $9, $11
+; MIPSR6-NEXT:    and $9, $9, $4
+; MIPSR6-NEXT:    and $10, $8, $6
+; MIPSR6-NEXT:    or $10, $10, $9
+; MIPSR6-NEXT:    sc $10, 0($2)
+; MIPSR6-NEXT:    beqzc $10, $BB4_1
+; MIPSR6-NEXT:  # %bb.2: # %entry
+; MIPSR6-NEXT:    and $7, $8, $4
+; MIPSR6-NEXT:    srlv $7, $7, $3
+; MIPSR6-NEXT:    seh $7, $7
+; MIPSR6-NEXT:  # %bb.3: # %entry
+; MIPSR6-NEXT:    sw $7, 4($sp) # 4-byte Folded Spill
+; MIPSR6-NEXT:  # %bb.4: # %entry
+; MIPSR6-NEXT:    sync
+; MIPSR6-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MIPSR6-NEXT:    addiu $sp, $sp, 8
+; MIPSR6-NEXT:    jrc $ra
+;
+; MM-LABEL: test_max_16:
+; MM:       # %bb.0: # %entry
+; MM-NEXT:    addiu $sp, $sp, -8
+; MM-NEXT:    .cfi_def_cfa_offset 8
+; MM-NEXT:    move $1, $5
+; MM-NEXT:    sync
+; MM-NEXT:    addiu $2, $zero, -4
+; MM-NEXT:    and $2, $4, $2
+; MM-NEXT:    andi $3, $4, 3
+; MM-NEXT:    xori $3, $3, 2
+; MM-NEXT:    sll $3, $3, 3
+; MM-NEXT:    ori $4, $zero, 65535
+; MM-NEXT:    sllv $4, $4, $3
+; MM-NEXT:    nor $6, $zero, $4
+; MM-NEXT:    sllv $5, $5, $3
+; MM-NEXT:  $BB4_1: # %entry
+; MM-NEXT:    # =>This Inner Loop Header: Depth=1
+; MM-NEXT:    ll $8, 0($2)
+; MM-NEXT:    slt $11, $8, $5
+; MM-NEXT:    or $9, $8, $zero
+; MM-NEXT:    movn $9, $5, $11
+; MM-NEXT:    and $9, $9, $4
+; MM-NEXT:    and $10, $8, $6
+; MM-NEXT:    or $10, $10, $9
+; MM-NEXT:    sc $10, 0($2)
+; MM-NEXT:    beqzc $10, $BB4_1
+; MM-NEXT:  # %bb.2: # %entry
+; MM-NEXT:    and $7, $8, $4
+; MM-NEXT:    srlv $7, $7, $3
+; MM-NEXT:    seh $7, $7
+; MM-NEXT:  # %bb.3: # %entry
+; MM-NEXT:    sw $7, 4($sp) # 4-byte Folded Spill
+; MM-NEXT:  # %bb.4: # %entry
+; MM-NEXT:    sync
+; MM-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MM-NEXT:    addiusp 8
+; MM-NEXT:    jrc $ra
+;
+; MMR6-LABEL: test_max_16:
+; MMR6:       # %bb.0: # %entry
+; MMR6-NEXT:    addiu $sp, $sp, -8
+; MMR6-NEXT:    .cfi_def_cfa_offset 8
+; MMR6-NEXT:    move $1, $5
+; MMR6-NEXT:    sync
+; MMR6-NEXT:    addiu $2, $zero, -4
+; MMR6-NEXT:    and $2, $4, $2
+; MMR6-NEXT:    andi $3, $4, 3
+; MMR6-NEXT:    xori $3, $3, 2
+; MMR6-NEXT:    sll $3, $3, 3
+; MMR6-NEXT:    ori $4, $zero, 65535
+; MMR6-NEXT:    sllv $4, $4, $3
+; MMR6-NEXT:    nor $6, $zero, $4
+; MMR6-NEXT:    sllv $5, $5, $3
+; MMR6-NEXT:  $BB4_1: # %entry
+; MMR6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MMR6-NEXT:    ll $8, 0($2)
+; MMR6-NEXT:    slt $11, $8, $5
+; MMR6-NEXT:    seleqz $9, $8, $11
+; MMR6-NEXT:    selnez $11, $5, $11
+; MMR6-NEXT:    or $9, $9, $11
+; MMR6-NEXT:    and $9, $9, $4
+; MMR6-NEXT:    and $10, $8, $6
+; MMR6-NEXT:    or $10, $10, $9
+; MMR6-NEXT:    sc $10, 0($2)
+; MMR6-NEXT:    beqc $10, $zero, $BB4_1
+; MMR6-NEXT:  # %bb.2: # %entry
+; MMR6-NEXT:    and $7, $8, $4
+; MMR6-NEXT:    srlv $7, $7, $3
+; MMR6-NEXT:    seh $7, $7
+; MMR6-NEXT:  # %bb.3: # %entry
+; MMR6-NEXT:    sw $7, 4($sp) # 4-byte Folded Spill
+; MMR6-NEXT:  # %bb.4: # %entry
+; MMR6-NEXT:    sync
+; MMR6-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MMR6-NEXT:    addiu $sp, $sp, 8
+; MMR6-NEXT:    jrc $ra
+;
+; MIPSEL-LABEL: test_max_16:
+; MIPSEL:       # %bb.0: # %entry
+; MIPSEL-NEXT:    addiu $sp, $sp, -8
+; MIPSEL-NEXT:    .cfi_def_cfa_offset 8
+; MIPSEL-NEXT:    move $1, $5
+; MIPSEL-NEXT:    sync
+; MIPSEL-NEXT:    addiu $2, $zero, -4
+; MIPSEL-NEXT:    and $2, $4, $2
+; MIPSEL-NEXT:    andi $3, $4, 3
+; MIPSEL-NEXT:    sll $3, $3, 3
+; MIPSEL-NEXT:    ori $4, $zero, 65535
+; MIPSEL-NEXT:    sllv $4, $4, $3
+; MIPSEL-NEXT:    nor $6, $zero, $4
+; MIPSEL-NEXT:    sllv $5, $5, $3
+; MIPSEL-NEXT:  $BB4_1: # %entry
+; MIPSEL-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPSEL-NEXT:    ll $8, 0($2)
+; MIPSEL-NEXT:    and $8, $8, $4
+; MIPSEL-NEXT:    and $5, $5, $4
+; MIPSEL-NEXT:    slt $11, $8, $5
+; MIPSEL-NEXT:    move $9, $8
+; MIPSEL-NEXT:    movn $9, $5, $11
+; MIPSEL-NEXT:    and $9, $9, $4
+; MIPSEL-NEXT:    and $10, $8, $6
+; MIPSEL-NEXT:    or $10, $10, $9
+; MIPSEL-NEXT:    sc $10, 0($2)
+; MIPSEL-NEXT:    beqz $10, $BB4_1
+; MIPSEL-NEXT:    nop
+; MIPSEL-NEXT:  # %bb.2: # %entry
+; MIPSEL-NEXT:    and $7, $8, $4
+; MIPSEL-NEXT:    srlv $7, $7, $3
+; MIPSEL-NEXT:    seh $7, $7
+; MIPSEL-NEXT:  # %bb.3: # %entry
+; MIPSEL-NEXT:    sw $7, 4($sp) # 4-byte Folded Spill
+; MIPSEL-NEXT:  # %bb.4: # %entry
+; MIPSEL-NEXT:    sync
+; MIPSEL-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MIPSEL-NEXT:    addiu $sp, $sp, 8
+; MIPSEL-NEXT:    jr $ra
+; MIPSEL-NEXT:    nop
+;
+; MIPSELR6-LABEL: test_max_16:
+; MIPSELR6:       # %bb.0: # %entry
+; MIPSELR6-NEXT:    addiu $sp, $sp, -8
+; MIPSELR6-NEXT:    .cfi_def_cfa_offset 8
+; MIPSELR6-NEXT:    move $1, $5
+; MIPSELR6-NEXT:    sync
+; MIPSELR6-NEXT:    addiu $2, $zero, -4
+; MIPSELR6-NEXT:    and $2, $4, $2
+; MIPSELR6-NEXT:    andi $3, $4, 3
+; MIPSELR6-NEXT:    sll $3, $3, 3
+; MIPSELR6-NEXT:    ori $4, $zero, 65535
+; MIPSELR6-NEXT:    sllv $4, $4, $3
+; MIPSELR6-NEXT:    nor $6, $zero, $4
+; MIPSELR6-NEXT:    sllv $5, $5, $3
+; MIPSELR6-NEXT:  $BB4_1: # %entry
+; MIPSELR6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPSELR6-NEXT:    ll $8, 0($2)
+; MIPSELR6-NEXT:    and $8, $8, $4
+; MIPSELR6-NEXT:    and $5, $5, $4
+; MIPSELR6-NEXT:    slt $11, $8, $5
+; MIPSELR6-NEXT:    seleqz $9, $8, $11
+; MIPSELR6-NEXT:    selnez $11, $5, $11
+; MIPSELR6-NEXT:    or $9, $9, $11
+; MIPSELR6-NEXT:    and $9, $9, $4
+; MIPSELR6-NEXT:    and $10, $8, $6
+; MIPSELR6-NEXT:    or $10, $10, $9
+; MIPSELR6-NEXT:    sc $10, 0($2)
+; MIPSELR6-NEXT:    beqzc $10, $BB4_1
+; MIPSELR6-NEXT:  # %bb.2: # %entry
+; MIPSELR6-NEXT:    and $7, $8, $4
+; MIPSELR6-NEXT:    srlv $7, $7, $3
+; MIPSELR6-NEXT:    seh $7, $7
+; MIPSELR6-NEXT:  # %bb.3: # %entry
+; MIPSELR6-NEXT:    sw $7, 4($sp) # 4-byte Folded Spill
+; MIPSELR6-NEXT:  # %bb.4: # %entry
+; MIPSELR6-NEXT:    sync
+; MIPSELR6-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MIPSELR6-NEXT:    addiu $sp, $sp, 8
+; MIPSELR6-NEXT:    jrc $ra
+;
+; MMEL-LABEL: test_max_16:
+; MMEL:       # %bb.0: # %entry
+; MMEL-NEXT:    addiu $sp, $sp, -8
+; MMEL-NEXT:    .cfi_def_cfa_offset 8
+; MMEL-NEXT:    move $1, $5
+; MMEL-NEXT:    sync
+; MMEL-NEXT:    addiu $2, $zero, -4
+; MMEL-NEXT:    and $2, $4, $2
+; MMEL-NEXT:    andi $3, $4, 3
+; MMEL-NEXT:    sll $3, $3, 3
+; MMEL-NEXT:    ori $4, $zero, 65535
+; MMEL-NEXT:    sllv $4, $4, $3
+; MMEL-NEXT:    nor $6, $zero, $4
+; MMEL-NEXT:    sllv $5, $5, $3
+; MMEL-NEXT:  $BB4_1: # %entry
+; MMEL-NEXT:    # =>This Inner Loop Header: Depth=1
+; MMEL-NEXT:    ll $8, 0($2)
+; MMEL-NEXT:    and $8, $8, $4
+; MMEL-NEXT:    and $5, $5, $4
+; MMEL-NEXT:    slt $11, $8, $5
+; MMEL-NEXT:    or $9, $8, $zero
+; MMEL-NEXT:    movn $9, $5, $11
+; MMEL-NEXT:    and $9, $9, $4
+; MMEL-NEXT:    and $10, $8, $6
+; MMEL-NEXT:    or $10, $10, $9
+; MMEL-NEXT:    sc $10, 0($2)
+; MMEL-NEXT:    beqzc $10, $BB4_1
+; MMEL-NEXT:  # %bb.2: # %entry
+; MMEL-NEXT:    and $7, $8, $4
+; MMEL-NEXT:    srlv $7, $7, $3
+; MMEL-NEXT:    seh $7, $7
+; MMEL-NEXT:  # %bb.3: # %entry
+; MMEL-NEXT:    sw $7, 4($sp) # 4-byte Folded Spill
+; MMEL-NEXT:  # %bb.4: # %entry
+; MMEL-NEXT:    sync
+; MMEL-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MMEL-NEXT:    addiusp 8
+; MMEL-NEXT:    jrc $ra
+;
+; MMELR6-LABEL: test_max_16:
+; MMELR6:       # %bb.0: # %entry
+; MMELR6-NEXT:    addiu $sp, $sp, -8
+; MMELR6-NEXT:    .cfi_def_cfa_offset 8
+; MMELR6-NEXT:    move $1, $5
+; MMELR6-NEXT:    sync
+; MMELR6-NEXT:    addiu $2, $zero, -4
+; MMELR6-NEXT:    and $2, $4, $2
+; MMELR6-NEXT:    andi $3, $4, 3
+; MMELR6-NEXT:    sll $3, $3, 3
+; MMELR6-NEXT:    ori $4, $zero, 65535
+; MMELR6-NEXT:    sllv $4, $4, $3
+; MMELR6-NEXT:    nor $6, $zero, $4
+; MMELR6-NEXT:    sllv $5, $5, $3
+; MMELR6-NEXT:  $BB4_1: # %entry
+; MMELR6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MMELR6-NEXT:    ll $8, 0($2)
+; MMELR6-NEXT:    and $8, $8, $4
+; MMELR6-NEXT:    and $5, $5, $4
+; MMELR6-NEXT:    slt $11, $8, $5
+; MMELR6-NEXT:    seleqz $9, $8, $11
+; MMELR6-NEXT:    selnez $11, $5, $11
+; MMELR6-NEXT:    or $9, $9, $11
+; MMELR6-NEXT:    and $9, $9, $4
+; MMELR6-NEXT:    and $10, $8, $6
+; MMELR6-NEXT:    or $10, $10, $9
+; MMELR6-NEXT:    sc $10, 0($2)
+; MMELR6-NEXT:    beqc $10, $zero, $BB4_1
+; MMELR6-NEXT:  # %bb.2: # %entry
+; MMELR6-NEXT:    and $7, $8, $4
+; MMELR6-NEXT:    srlv $7, $7, $3
+; MMELR6-NEXT:    seh $7, $7
+; MMELR6-NEXT:  # %bb.3: # %entry
+; MMELR6-NEXT:    sw $7, 4($sp) # 4-byte Folded Spill
+; MMELR6-NEXT:  # %bb.4: # %entry
+; MMELR6-NEXT:    sync
+; MMELR6-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MMELR6-NEXT:    addiu $sp, $sp, 8
+; MMELR6-NEXT:    jrc $ra
+;
+; MIPS64-LABEL: test_max_16:
+; MIPS64:       # %bb.0: # %entry
+; MIPS64-NEXT:    daddiu $sp, $sp, -16
+; MIPS64-NEXT:    .cfi_def_cfa_offset 16
+; MIPS64-NEXT:    # kill: def $a1 killed $a1 killed $a1_64
+; MIPS64-NEXT:    sync
+; MIPS64-NEXT:    daddiu $1, $zero, -4
+; MIPS64-NEXT:    and $1, $4, $1
+; MIPS64-NEXT:    andi $2, $4, 3
+; MIPS64-NEXT:    xori $2, $2, 2
+; MIPS64-NEXT:    sll $2, $2, 3
+; MIPS64-NEXT:    ori $3, $zero, 65535
+; MIPS64-NEXT:    sllv $3, $3, $2
+; MIPS64-NEXT:    nor $6, $zero, $3
+; MIPS64-NEXT:    sllv $5, $5, $2
+; MIPS64-NEXT:  .LBB4_1: # %entry
+; MIPS64-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS64-NEXT:    ll $8, 0($1)
+; MIPS64-NEXT:    slt $11, $8, $5
+; MIPS64-NEXT:    move $9, $8
+; MIPS64-NEXT:    movn $9, $5, $11
+; MIPS64-NEXT:    and $9, $9, $3
+; MIPS64-NEXT:    and $10, $8, $6
+; MIPS64-NEXT:    or $10, $10, $9
+; MIPS64-NEXT:    sc $10, 0($1)
+; MIPS64-NEXT:    beqz $10, .LBB4_1
+; MIPS64-NEXT:    nop
+; MIPS64-NEXT:  # %bb.2: # %entry
+; MIPS64-NEXT:    and $7, $8, $3
+; MIPS64-NEXT:    srlv $7, $7, $2
+; MIPS64-NEXT:    seh $7, $7
+; MIPS64-NEXT:  # %bb.3: # %entry
+; MIPS64-NEXT:    sw $7, 12($sp) # 4-byte Folded Spill
+; MIPS64-NEXT:  # %bb.4: # %entry
+; MIPS64-NEXT:    sync
+; MIPS64-NEXT:    lw $2, 12($sp) # 4-byte Folded Reload
+; MIPS64-NEXT:    daddiu $sp, $sp, 16
+; MIPS64-NEXT:    jr $ra
+; MIPS64-NEXT:    nop
+;
+; MIPS64R6-LABEL: test_max_16:
+; MIPS64R6:       # %bb.0: # %entry
+; MIPS64R6-NEXT:    daddiu $sp, $sp, -16
+; MIPS64R6-NEXT:    .cfi_def_cfa_offset 16
+; MIPS64R6-NEXT:    # kill: def $a1 killed $a1 killed $a1_64
+; MIPS64R6-NEXT:    sync
+; MIPS64R6-NEXT:    daddiu $1, $zero, -4
+; MIPS64R6-NEXT:    and $1, $4, $1
+; MIPS64R6-NEXT:    andi $2, $4, 3
+; MIPS64R6-NEXT:    xori $2, $2, 2
+; MIPS64R6-NEXT:    sll $2, $2, 3
+; MIPS64R6-NEXT:    ori $3, $zero, 65535
+; MIPS64R6-NEXT:    sllv $3, $3, $2
+; MIPS64R6-NEXT:    nor $6, $zero, $3
+; MIPS64R6-NEXT:    sllv $5, $5, $2
+; MIPS64R6-NEXT:  .LBB4_1: # %entry
+; MIPS64R6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS64R6-NEXT:    ll $8, 0($1)
+; MIPS64R6-NEXT:    slt $11, $8, $5
+; MIPS64R6-NEXT:    seleqz $9, $8, $11
+; MIPS64R6-NEXT:    selnez $11, $5, $11
+; MIPS64R6-NEXT:    or $9, $9, $11
+; MIPS64R6-NEXT:    and $9, $9, $3
+; MIPS64R6-NEXT:    and $10, $8, $6
+; MIPS64R6-NEXT:    or $10, $10, $9
+; MIPS64R6-NEXT:    sc $10, 0($1)
+; MIPS64R6-NEXT:    beqzc $10, .LBB4_1
+; MIPS64R6-NEXT:  # %bb.2: # %entry
+; MIPS64R6-NEXT:    and $7, $8, $3
+; MIPS64R6-NEXT:    srlv $7, $7, $2
+; MIPS64R6-NEXT:    seh $7, $7
+; MIPS64R6-NEXT:  # %bb.3: # %entry
+; MIPS64R6-NEXT:    sw $7, 12($sp) # 4-byte Folded Spill
+; MIPS64R6-NEXT:  # %bb.4: # %entry
+; MIPS64R6-NEXT:    sync
+; MIPS64R6-NEXT:    lw $2, 12($sp) # 4-byte Folded Reload
+; MIPS64R6-NEXT:    daddiu $sp, $sp, 16
+; MIPS64R6-NEXT:    jrc $ra
+;
+; MIPS64EL-LABEL: test_max_16:
+; MIPS64EL:       # %bb.0: # %entry
+; MIPS64EL-NEXT:    daddiu $sp, $sp, -16
+; MIPS64EL-NEXT:    .cfi_def_cfa_offset 16
+; MIPS64EL-NEXT:    # kill: def $a1 killed $a1 killed $a1_64
+; MIPS64EL-NEXT:    sync
+; MIPS64EL-NEXT:    daddiu $1, $zero, -4
+; MIPS64EL-NEXT:    and $1, $4, $1
+; MIPS64EL-NEXT:    andi $2, $4, 3
+; MIPS64EL-NEXT:    sll $2, $2, 3
+; MIPS64EL-NEXT:    ori $3, $zero, 65535
+; MIPS64EL-NEXT:    sllv $3, $3, $2
+; MIPS64EL-NEXT:    nor $6, $zero, $3
+; MIPS64EL-NEXT:    sllv $5, $5, $2
+; MIPS64EL-NEXT:  .LBB4_1: # %entry
+; MIPS64EL-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS64EL-NEXT:    ll $8, 0($1)
+; MIPS64EL-NEXT:    and $8, $8, $3
+; MIPS64EL-NEXT:    and $5, $5, $3
+; MIPS64EL-NEXT:    slt $11, $8, $5
+; MIPS64EL-NEXT:    move $9, $8
+; MIPS64EL-NEXT:    movn $9, $5, $11
+; MIPS64EL-NEXT:    and $9, $9, $3
+; MIPS64EL-NEXT:    and $10, $8, $6
+; MIPS64EL-NEXT:    or $10, $10, $9
+; MIPS64EL-NEXT:    sc $10, 0($1)
+; MIPS64EL-NEXT:    beqz $10, .LBB4_1
+; MIPS64EL-NEXT:    nop
+; MIPS64EL-NEXT:  # %bb.2: # %entry
+; MIPS64EL-NEXT:    and $7, $8, $3
+; MIPS64EL-NEXT:    srlv $7, $7, $2
+; MIPS64EL-NEXT:    seh $7, $7
+; MIPS64EL-NEXT:  # %bb.3: # %entry
+; MIPS64EL-NEXT:    sw $7, 12($sp) # 4-byte Folded Spill
+; MIPS64EL-NEXT:  # %bb.4: # %entry
+; MIPS64EL-NEXT:    sync
+; MIPS64EL-NEXT:    lw $2, 12($sp) # 4-byte Folded Reload
+; MIPS64EL-NEXT:    daddiu $sp, $sp, 16
+; MIPS64EL-NEXT:    jr $ra
+; MIPS64EL-NEXT:    nop
+;
+; MIPS64ELR6-LABEL: test_max_16:
+; MIPS64ELR6:       # %bb.0: # %entry
+; MIPS64ELR6-NEXT:    daddiu $sp, $sp, -16
+; MIPS64ELR6-NEXT:    .cfi_def_cfa_offset 16
+; MIPS64ELR6-NEXT:    # kill: def $a1 killed $a1 killed $a1_64
+; MIPS64ELR6-NEXT:    sync
+; MIPS64ELR6-NEXT:    daddiu $1, $zero, -4
+; MIPS64ELR6-NEXT:    and $1, $4, $1
+; MIPS64ELR6-NEXT:    andi $2, $4, 3
+; MIPS64ELR6-NEXT:    sll $2, $2, 3
+; MIPS64ELR6-NEXT:    ori $3, $zero, 65535
+; MIPS64ELR6-NEXT:    sllv $3, $3, $2
+; MIPS64ELR6-NEXT:    nor $6, $zero, $3
+; MIPS64ELR6-NEXT:    sllv $5, $5, $2
+; MIPS64ELR6-NEXT:  .LBB4_1: # %entry
+; MIPS64ELR6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS64ELR6-NEXT:    ll $8, 0($1)
+; MIPS64ELR6-NEXT:    and $8, $8, $3
+; MIPS64ELR6-NEXT:    and $5, $5, $3
+; MIPS64ELR6-NEXT:    slt $11, $8, $5
+; MIPS64ELR6-NEXT:    seleqz $9, $8, $11
+; MIPS64ELR6-NEXT:    selnez $11, $5, $11
+; MIPS64ELR6-NEXT:    or $9, $9, $11
+; MIPS64ELR6-NEXT:    and $9, $9, $3
+; MIPS64ELR6-NEXT:    and $10, $8, $6
+; MIPS64ELR6-NEXT:    or $10, $10, $9
+; MIPS64ELR6-NEXT:    sc $10, 0($1)
+; MIPS64ELR6-NEXT:    beqzc $10, .LBB4_1
+; MIPS64ELR6-NEXT:  # %bb.2: # %entry
+; MIPS64ELR6-NEXT:    and $7, $8, $3
+; MIPS64ELR6-NEXT:    srlv $7, $7, $2
+; MIPS64ELR6-NEXT:    seh $7, $7
+; MIPS64ELR6-NEXT:  # %bb.3: # %entry
+; MIPS64ELR6-NEXT:    sw $7, 12($sp) # 4-byte Folded Spill
+; MIPS64ELR6-NEXT:  # %bb.4: # %entry
+; MIPS64ELR6-NEXT:    sync
+; MIPS64ELR6-NEXT:    lw $2, 12($sp) # 4-byte Folded Reload
+; MIPS64ELR6-NEXT:    daddiu $sp, $sp, 16
+; MIPS64ELR6-NEXT:    jrc $ra
+entry:
+  %0 = atomicrmw max i16* %ptr, i16 %val seq_cst
+  ret i16 %0
+}
+
+define i16 @test_min_16(i16* nocapture %ptr, i16 signext %val) {
+; MIPS-LABEL: test_min_16:
+; MIPS:       # %bb.0: # %entry
+; MIPS-NEXT:    addiu $sp, $sp, -8
+; MIPS-NEXT:    .cfi_def_cfa_offset 8
+; MIPS-NEXT:    move $1, $5
+; MIPS-NEXT:    sync
+; MIPS-NEXT:    addiu $2, $zero, -4
+; MIPS-NEXT:    and $2, $4, $2
+; MIPS-NEXT:    andi $3, $4, 3
+; MIPS-NEXT:    xori $3, $3, 2
+; MIPS-NEXT:    sll $3, $3, 3
+; MIPS-NEXT:    ori $4, $zero, 65535
+; MIPS-NEXT:    sllv $4, $4, $3
+; MIPS-NEXT:    nor $6, $zero, $4
+; MIPS-NEXT:    sllv $5, $5, $3
+; MIPS-NEXT:  $BB5_1: # %entry
+; MIPS-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS-NEXT:    ll $8, 0($2)
+; MIPS-NEXT:    slt $11, $8, $5
+; MIPS-NEXT:    move $9, $8
+; MIPS-NEXT:    movz $9, $5, $11
+; MIPS-NEXT:    and $9, $9, $4
+; MIPS-NEXT:    and $10, $8, $6
+; MIPS-NEXT:    or $10, $10, $9
+; MIPS-NEXT:    sc $10, 0($2)
+; MIPS-NEXT:    beqz $10, $BB5_1
+; MIPS-NEXT:    nop
+; MIPS-NEXT:  # %bb.2: # %entry
+; MIPS-NEXT:    and $7, $8, $4
+; MIPS-NEXT:    srlv $7, $7, $3
+; MIPS-NEXT:    seh $7, $7
+; MIPS-NEXT:  # %bb.3: # %entry
+; MIPS-NEXT:    sw $7, 4($sp) # 4-byte Folded Spill
+; MIPS-NEXT:  # %bb.4: # %entry
+; MIPS-NEXT:    sync
+; MIPS-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MIPS-NEXT:    addiu $sp, $sp, 8
+; MIPS-NEXT:    jr $ra
+; MIPS-NEXT:    nop
+;
+; MIPSR6-LABEL: test_min_16:
+; MIPSR6:       # %bb.0: # %entry
+; MIPSR6-NEXT:    addiu $sp, $sp, -8
+; MIPSR6-NEXT:    .cfi_def_cfa_offset 8
+; MIPSR6-NEXT:    move $1, $5
+; MIPSR6-NEXT:    sync
+; MIPSR6-NEXT:    addiu $2, $zero, -4
+; MIPSR6-NEXT:    and $2, $4, $2
+; MIPSR6-NEXT:    andi $3, $4, 3
+; MIPSR6-NEXT:    xori $3, $3, 2
+; MIPSR6-NEXT:    sll $3, $3, 3
+; MIPSR6-NEXT:    ori $4, $zero, 65535
+; MIPSR6-NEXT:    sllv $4, $4, $3
+; MIPSR6-NEXT:    nor $6, $zero, $4
+; MIPSR6-NEXT:    sllv $5, $5, $3
+; MIPSR6-NEXT:  $BB5_1: # %entry
+; MIPSR6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPSR6-NEXT:    ll $8, 0($2)
+; MIPSR6-NEXT:    slt $11, $8, $5
+; MIPSR6-NEXT:    selnez $9, $8, $11
+; MIPSR6-NEXT:    seleqz $11, $5, $11
+; MIPSR6-NEXT:    or $9, $9, $11
+; MIPSR6-NEXT:    and $9, $9, $4
+; MIPSR6-NEXT:    and $10, $8, $6
+; MIPSR6-NEXT:    or $10, $10, $9
+; MIPSR6-NEXT:    sc $10, 0($2)
+; MIPSR6-NEXT:    beqzc $10, $BB5_1
+; MIPSR6-NEXT:  # %bb.2: # %entry
+; MIPSR6-NEXT:    and $7, $8, $4
+; MIPSR6-NEXT:    srlv $7, $7, $3
+; MIPSR6-NEXT:    seh $7, $7
+; MIPSR6-NEXT:  # %bb.3: # %entry
+; MIPSR6-NEXT:    sw $7, 4($sp) # 4-byte Folded Spill
+; MIPSR6-NEXT:  # %bb.4: # %entry
+; MIPSR6-NEXT:    sync
+; MIPSR6-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MIPSR6-NEXT:    addiu $sp, $sp, 8
+; MIPSR6-NEXT:    jrc $ra
+;
+; MM-LABEL: test_min_16:
+; MM:       # %bb.0: # %entry
+; MM-NEXT:    addiu $sp, $sp, -8
+; MM-NEXT:    .cfi_def_cfa_offset 8
+; MM-NEXT:    move $1, $5
+; MM-NEXT:    sync
+; MM-NEXT:    addiu $2, $zero, -4
+; MM-NEXT:    and $2, $4, $2
+; MM-NEXT:    andi $3, $4, 3
+; MM-NEXT:    xori $3, $3, 2
+; MM-NEXT:    sll $3, $3, 3
+; MM-NEXT:    ori $4, $zero, 65535
+; MM-NEXT:    sllv $4, $4, $3
+; MM-NEXT:    nor $6, $zero, $4
+; MM-NEXT:    sllv $5, $5, $3
+; MM-NEXT:  $BB5_1: # %entry
+; MM-NEXT:    # =>This Inner Loop Header: Depth=1
+; MM-NEXT:    ll $8, 0($2)
+; MM-NEXT:    slt $11, $8, $5
+; MM-NEXT:    or $9, $8, $zero
+; MM-NEXT:    movz $9, $5, $11
+; MM-NEXT:    and $9, $9, $4
+; MM-NEXT:    and $10, $8, $6
+; MM-NEXT:    or $10, $10, $9
+; MM-NEXT:    sc $10, 0($2)
+; MM-NEXT:    beqzc $10, $BB5_1
+; MM-NEXT:  # %bb.2: # %entry
+; MM-NEXT:    and $7, $8, $4
+; MM-NEXT:    srlv $7, $7, $3
+; MM-NEXT:    seh $7, $7
+; MM-NEXT:  # %bb.3: # %entry
+; MM-NEXT:    sw $7, 4($sp) # 4-byte Folded Spill
+; MM-NEXT:  # %bb.4: # %entry
+; MM-NEXT:    sync
+; MM-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MM-NEXT:    addiusp 8
+; MM-NEXT:    jrc $ra
+;
+; MMR6-LABEL: test_min_16:
+; MMR6:       # %bb.0: # %entry
+; MMR6-NEXT:    addiu $sp, $sp, -8
+; MMR6-NEXT:    .cfi_def_cfa_offset 8
+; MMR6-NEXT:    move $1, $5
+; MMR6-NEXT:    sync
+; MMR6-NEXT:    addiu $2, $zero, -4
+; MMR6-NEXT:    and $2, $4, $2
+; MMR6-NEXT:    andi $3, $4, 3
+; MMR6-NEXT:    xori $3, $3, 2
+; MMR6-NEXT:    sll $3, $3, 3
+; MMR6-NEXT:    ori $4, $zero, 65535
+; MMR6-NEXT:    sllv $4, $4, $3
+; MMR6-NEXT:    nor $6, $zero, $4
+; MMR6-NEXT:    sllv $5, $5, $3
+; MMR6-NEXT:  $BB5_1: # %entry
+; MMR6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MMR6-NEXT:    ll $8, 0($2)
+; MMR6-NEXT:    slt $11, $8, $5
+; MMR6-NEXT:    selnez $9, $8, $11
+; MMR6-NEXT:    seleqz $11, $5, $11
+; MMR6-NEXT:    or $9, $9, $11
+; MMR6-NEXT:    and $9, $9, $4
+; MMR6-NEXT:    and $10, $8, $6
+; MMR6-NEXT:    or $10, $10, $9
+; MMR6-NEXT:    sc $10, 0($2)
+; MMR6-NEXT:    beqc $10, $zero, $BB5_1
+; MMR6-NEXT:  # %bb.2: # %entry
+; MMR6-NEXT:    and $7, $8, $4
+; MMR6-NEXT:    srlv $7, $7, $3
+; MMR6-NEXT:    seh $7, $7
+; MMR6-NEXT:  # %bb.3: # %entry
+; MMR6-NEXT:    sw $7, 4($sp) # 4-byte Folded Spill
+; MMR6-NEXT:  # %bb.4: # %entry
+; MMR6-NEXT:    sync
+; MMR6-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MMR6-NEXT:    addiu $sp, $sp, 8
+; MMR6-NEXT:    jrc $ra
+;
+; MIPSEL-LABEL: test_min_16:
+; MIPSEL:       # %bb.0: # %entry
+; MIPSEL-NEXT:    addiu $sp, $sp, -8
+; MIPSEL-NEXT:    .cfi_def_cfa_offset 8
+; MIPSEL-NEXT:    move $1, $5
+; MIPSEL-NEXT:    sync
+; MIPSEL-NEXT:    addiu $2, $zero, -4
+; MIPSEL-NEXT:    and $2, $4, $2
+; MIPSEL-NEXT:    andi $3, $4, 3
+; MIPSEL-NEXT:    sll $3, $3, 3
+; MIPSEL-NEXT:    ori $4, $zero, 65535
+; MIPSEL-NEXT:    sllv $4, $4, $3
+; MIPSEL-NEXT:    nor $6, $zero, $4
+; MIPSEL-NEXT:    sllv $5, $5, $3
+; MIPSEL-NEXT:  $BB5_1: # %entry
+; MIPSEL-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPSEL-NEXT:    ll $8, 0($2)
+; MIPSEL-NEXT:    and $8, $8, $4
+; MIPSEL-NEXT:    and $5, $5, $4
+; MIPSEL-NEXT:    slt $11, $8, $5
+; MIPSEL-NEXT:    move $9, $8
+; MIPSEL-NEXT:    movz $9, $5, $11
+; MIPSEL-NEXT:    and $9, $9, $4
+; MIPSEL-NEXT:    and $10, $8, $6
+; MIPSEL-NEXT:    or $10, $10, $9
+; MIPSEL-NEXT:    sc $10, 0($2)
+; MIPSEL-NEXT:    beqz $10, $BB5_1
+; MIPSEL-NEXT:    nop
+; MIPSEL-NEXT:  # %bb.2: # %entry
+; MIPSEL-NEXT:    and $7, $8, $4
+; MIPSEL-NEXT:    srlv $7, $7, $3
+; MIPSEL-NEXT:    seh $7, $7
+; MIPSEL-NEXT:  # %bb.3: # %entry
+; MIPSEL-NEXT:    sw $7, 4($sp) # 4-byte Folded Spill
+; MIPSEL-NEXT:  # %bb.4: # %entry
+; MIPSEL-NEXT:    sync
+; MIPSEL-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MIPSEL-NEXT:    addiu $sp, $sp, 8
+; MIPSEL-NEXT:    jr $ra
+; MIPSEL-NEXT:    nop
+;
+; MIPSELR6-LABEL: test_min_16:
+; MIPSELR6:       # %bb.0: # %entry
+; MIPSELR6-NEXT:    addiu $sp, $sp, -8
+; MIPSELR6-NEXT:    .cfi_def_cfa_offset 8
+; MIPSELR6-NEXT:    move $1, $5
+; MIPSELR6-NEXT:    sync
+; MIPSELR6-NEXT:    addiu $2, $zero, -4
+; MIPSELR6-NEXT:    and $2, $4, $2
+; MIPSELR6-NEXT:    andi $3, $4, 3
+; MIPSELR6-NEXT:    sll $3, $3, 3
+; MIPSELR6-NEXT:    ori $4, $zero, 65535
+; MIPSELR6-NEXT:    sllv $4, $4, $3
+; MIPSELR6-NEXT:    nor $6, $zero, $4
+; MIPSELR6-NEXT:    sllv $5, $5, $3
+; MIPSELR6-NEXT:  $BB5_1: # %entry
+; MIPSELR6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPSELR6-NEXT:    ll $8, 0($2)
+; MIPSELR6-NEXT:    and $8, $8, $4
+; MIPSELR6-NEXT:    and $5, $5, $4
+; MIPSELR6-NEXT:    slt $11, $8, $5
+; MIPSELR6-NEXT:    selnez $9, $8, $11
+; MIPSELR6-NEXT:    seleqz $11, $5, $11
+; MIPSELR6-NEXT:    or $9, $9, $11
+; MIPSELR6-NEXT:    and $9, $9, $4
+; MIPSELR6-NEXT:    and $10, $8, $6
+; MIPSELR6-NEXT:    or $10, $10, $9
+; MIPSELR6-NEXT:    sc $10, 0($2)
+; MIPSELR6-NEXT:    beqzc $10, $BB5_1
+; MIPSELR6-NEXT:  # %bb.2: # %entry
+; MIPSELR6-NEXT:    and $7, $8, $4
+; MIPSELR6-NEXT:    srlv $7, $7, $3
+; MIPSELR6-NEXT:    seh $7, $7
+; MIPSELR6-NEXT:  # %bb.3: # %entry
+; MIPSELR6-NEXT:    sw $7, 4($sp) # 4-byte Folded Spill
+; MIPSELR6-NEXT:  # %bb.4: # %entry
+; MIPSELR6-NEXT:    sync
+; MIPSELR6-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MIPSELR6-NEXT:    addiu $sp, $sp, 8
+; MIPSELR6-NEXT:    jrc $ra
+;
+; MMEL-LABEL: test_min_16:
+; MMEL:       # %bb.0: # %entry
+; MMEL-NEXT:    addiu $sp, $sp, -8
+; MMEL-NEXT:    .cfi_def_cfa_offset 8
+; MMEL-NEXT:    move $1, $5
+; MMEL-NEXT:    sync
+; MMEL-NEXT:    addiu $2, $zero, -4
+; MMEL-NEXT:    and $2, $4, $2
+; MMEL-NEXT:    andi $3, $4, 3
+; MMEL-NEXT:    sll $3, $3, 3
+; MMEL-NEXT:    ori $4, $zero, 65535
+; MMEL-NEXT:    sllv $4, $4, $3
+; MMEL-NEXT:    nor $6, $zero, $4
+; MMEL-NEXT:    sllv $5, $5, $3
+; MMEL-NEXT:  $BB5_1: # %entry
+; MMEL-NEXT:    # =>This Inner Loop Header: Depth=1
+; MMEL-NEXT:    ll $8, 0($2)
+; MMEL-NEXT:    and $8, $8, $4
+; MMEL-NEXT:    and $5, $5, $4
+; MMEL-NEXT:    slt $11, $8, $5
+; MMEL-NEXT:    or $9, $8, $zero
+; MMEL-NEXT:    movz $9, $5, $11
+; MMEL-NEXT:    and $9, $9, $4
+; MMEL-NEXT:    and $10, $8, $6
+; MMEL-NEXT:    or $10, $10, $9
+; MMEL-NEXT:    sc $10, 0($2)
+; MMEL-NEXT:    beqzc $10, $BB5_1
+; MMEL-NEXT:  # %bb.2: # %entry
+; MMEL-NEXT:    and $7, $8, $4
+; MMEL-NEXT:    srlv $7, $7, $3
+; MMEL-NEXT:    seh $7, $7
+; MMEL-NEXT:  # %bb.3: # %entry
+; MMEL-NEXT:    sw $7, 4($sp) # 4-byte Folded Spill
+; MMEL-NEXT:  # %bb.4: # %entry
+; MMEL-NEXT:    sync
+; MMEL-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MMEL-NEXT:    addiusp 8
+; MMEL-NEXT:    jrc $ra
+;
+; MMELR6-LABEL: test_min_16:
+; MMELR6:       # %bb.0: # %entry
+; MMELR6-NEXT:    addiu $sp, $sp, -8
+; MMELR6-NEXT:    .cfi_def_cfa_offset 8
+; MMELR6-NEXT:    move $1, $5
+; MMELR6-NEXT:    sync
+; MMELR6-NEXT:    addiu $2, $zero, -4
+; MMELR6-NEXT:    and $2, $4, $2
+; MMELR6-NEXT:    andi $3, $4, 3
+; MMELR6-NEXT:    sll $3, $3, 3
+; MMELR6-NEXT:    ori $4, $zero, 65535
+; MMELR6-NEXT:    sllv $4, $4, $3
+; MMELR6-NEXT:    nor $6, $zero, $4
+; MMELR6-NEXT:    sllv $5, $5, $3
+; MMELR6-NEXT:  $BB5_1: # %entry
+; MMELR6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MMELR6-NEXT:    ll $8, 0($2)
+; MMELR6-NEXT:    and $8, $8, $4
+; MMELR6-NEXT:    and $5, $5, $4
+; MMELR6-NEXT:    slt $11, $8, $5
+; MMELR6-NEXT:    selnez $9, $8, $11
+; MMELR6-NEXT:    seleqz $11, $5, $11
+; MMELR6-NEXT:    or $9, $9, $11
+; MMELR6-NEXT:    and $9, $9, $4
+; MMELR6-NEXT:    and $10, $8, $6
+; MMELR6-NEXT:    or $10, $10, $9
+; MMELR6-NEXT:    sc $10, 0($2)
+; MMELR6-NEXT:    beqc $10, $zero, $BB5_1
+; MMELR6-NEXT:  # %bb.2: # %entry
+; MMELR6-NEXT:    and $7, $8, $4
+; MMELR6-NEXT:    srlv $7, $7, $3
+; MMELR6-NEXT:    seh $7, $7
+; MMELR6-NEXT:  # %bb.3: # %entry
+; MMELR6-NEXT:    sw $7, 4($sp) # 4-byte Folded Spill
+; MMELR6-NEXT:  # %bb.4: # %entry
+; MMELR6-NEXT:    sync
+; MMELR6-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MMELR6-NEXT:    addiu $sp, $sp, 8
+; MMELR6-NEXT:    jrc $ra
+;
+; MIPS64-LABEL: test_min_16:
+; MIPS64:       # %bb.0: # %entry
+; MIPS64-NEXT:    daddiu $sp, $sp, -16
+; MIPS64-NEXT:    .cfi_def_cfa_offset 16
+; MIPS64-NEXT:    # kill: def $a1 killed $a1 killed $a1_64
+; MIPS64-NEXT:    sync
+; MIPS64-NEXT:    daddiu $1, $zero, -4
+; MIPS64-NEXT:    and $1, $4, $1
+; MIPS64-NEXT:    andi $2, $4, 3
+; MIPS64-NEXT:    xori $2, $2, 2
+; MIPS64-NEXT:    sll $2, $2, 3
+; MIPS64-NEXT:    ori $3, $zero, 65535
+; MIPS64-NEXT:    sllv $3, $3, $2
+; MIPS64-NEXT:    nor $6, $zero, $3
+; MIPS64-NEXT:    sllv $5, $5, $2
+; MIPS64-NEXT:  .LBB5_1: # %entry
+; MIPS64-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS64-NEXT:    ll $8, 0($1)
+; MIPS64-NEXT:    slt $11, $8, $5
+; MIPS64-NEXT:    move $9, $8
+; MIPS64-NEXT:    movz $9, $5, $11
+; MIPS64-NEXT:    and $9, $9, $3
+; MIPS64-NEXT:    and $10, $8, $6
+; MIPS64-NEXT:    or $10, $10, $9
+; MIPS64-NEXT:    sc $10, 0($1)
+; MIPS64-NEXT:    beqz $10, .LBB5_1
+; MIPS64-NEXT:    nop
+; MIPS64-NEXT:  # %bb.2: # %entry
+; MIPS64-NEXT:    and $7, $8, $3
+; MIPS64-NEXT:    srlv $7, $7, $2
+; MIPS64-NEXT:    seh $7, $7
+; MIPS64-NEXT:  # %bb.3: # %entry
+; MIPS64-NEXT:    sw $7, 12($sp) # 4-byte Folded Spill
+; MIPS64-NEXT:  # %bb.4: # %entry
+; MIPS64-NEXT:    sync
+; MIPS64-NEXT:    lw $2, 12($sp) # 4-byte Folded Reload
+; MIPS64-NEXT:    daddiu $sp, $sp, 16
+; MIPS64-NEXT:    jr $ra
+; MIPS64-NEXT:    nop
+;
+; MIPS64R6-LABEL: test_min_16:
+; MIPS64R6:       # %bb.0: # %entry
+; MIPS64R6-NEXT:    daddiu $sp, $sp, -16
+; MIPS64R6-NEXT:    .cfi_def_cfa_offset 16
+; MIPS64R6-NEXT:    # kill: def $a1 killed $a1 killed $a1_64
+; MIPS64R6-NEXT:    sync
+; MIPS64R6-NEXT:    daddiu $1, $zero, -4
+; MIPS64R6-NEXT:    and $1, $4, $1
+; MIPS64R6-NEXT:    andi $2, $4, 3
+; MIPS64R6-NEXT:    xori $2, $2, 2
+; MIPS64R6-NEXT:    sll $2, $2, 3
+; MIPS64R6-NEXT:    ori $3, $zero, 65535
+; MIPS64R6-NEXT:    sllv $3, $3, $2
+; MIPS64R6-NEXT:    nor $6, $zero, $3
+; MIPS64R6-NEXT:    sllv $5, $5, $2
+; MIPS64R6-NEXT:  .LBB5_1: # %entry
+; MIPS64R6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS64R6-NEXT:    ll $8, 0($1)
+; MIPS64R6-NEXT:    slt $11, $8, $5
+; MIPS64R6-NEXT:    selnez $9, $8, $11
+; MIPS64R6-NEXT:    seleqz $11, $5, $11
+; MIPS64R6-NEXT:    or $9, $9, $11
+; MIPS64R6-NEXT:    and $9, $9, $3
+; MIPS64R6-NEXT:    and $10, $8, $6
+; MIPS64R6-NEXT:    or $10, $10, $9
+; MIPS64R6-NEXT:    sc $10, 0($1)
+; MIPS64R6-NEXT:    beqzc $10, .LBB5_1
+; MIPS64R6-NEXT:  # %bb.2: # %entry
+; MIPS64R6-NEXT:    and $7, $8, $3
+; MIPS64R6-NEXT:    srlv $7, $7, $2
+; MIPS64R6-NEXT:    seh $7, $7
+; MIPS64R6-NEXT:  # %bb.3: # %entry
+; MIPS64R6-NEXT:    sw $7, 12($sp) # 4-byte Folded Spill
+; MIPS64R6-NEXT:  # %bb.4: # %entry
+; MIPS64R6-NEXT:    sync
+; MIPS64R6-NEXT:    lw $2, 12($sp) # 4-byte Folded Reload
+; MIPS64R6-NEXT:    daddiu $sp, $sp, 16
+; MIPS64R6-NEXT:    jrc $ra
+;
+; MIPS64EL-LABEL: test_min_16:
+; MIPS64EL:       # %bb.0: # %entry
+; MIPS64EL-NEXT:    daddiu $sp, $sp, -16
+; MIPS64EL-NEXT:    .cfi_def_cfa_offset 16
+; MIPS64EL-NEXT:    # kill: def $a1 killed $a1 killed $a1_64
+; MIPS64EL-NEXT:    sync
+; MIPS64EL-NEXT:    daddiu $1, $zero, -4
+; MIPS64EL-NEXT:    and $1, $4, $1
+; MIPS64EL-NEXT:    andi $2, $4, 3
+; MIPS64EL-NEXT:    sll $2, $2, 3
+; MIPS64EL-NEXT:    ori $3, $zero, 65535
+; MIPS64EL-NEXT:    sllv $3, $3, $2
+; MIPS64EL-NEXT:    nor $6, $zero, $3
+; MIPS64EL-NEXT:    sllv $5, $5, $2
+; MIPS64EL-NEXT:  .LBB5_1: # %entry
+; MIPS64EL-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS64EL-NEXT:    ll $8, 0($1)
+; MIPS64EL-NEXT:    and $8, $8, $3
+; MIPS64EL-NEXT:    and $5, $5, $3
+; MIPS64EL-NEXT:    slt $11, $8, $5
+; MIPS64EL-NEXT:    move $9, $8
+; MIPS64EL-NEXT:    movz $9, $5, $11
+; MIPS64EL-NEXT:    and $9, $9, $3
+; MIPS64EL-NEXT:    and $10, $8, $6
+; MIPS64EL-NEXT:    or $10, $10, $9
+; MIPS64EL-NEXT:    sc $10, 0($1)
+; MIPS64EL-NEXT:    beqz $10, .LBB5_1
+; MIPS64EL-NEXT:    nop
+; MIPS64EL-NEXT:  # %bb.2: # %entry
+; MIPS64EL-NEXT:    and $7, $8, $3
+; MIPS64EL-NEXT:    srlv $7, $7, $2
+; MIPS64EL-NEXT:    seh $7, $7
+; MIPS64EL-NEXT:  # %bb.3: # %entry
+; MIPS64EL-NEXT:    sw $7, 12($sp) # 4-byte Folded Spill
+; MIPS64EL-NEXT:  # %bb.4: # %entry
+; MIPS64EL-NEXT:    sync
+; MIPS64EL-NEXT:    lw $2, 12($sp) # 4-byte Folded Reload
+; MIPS64EL-NEXT:    daddiu $sp, $sp, 16
+; MIPS64EL-NEXT:    jr $ra
+; MIPS64EL-NEXT:    nop
+;
+; MIPS64ELR6-LABEL: test_min_16:
+; MIPS64ELR6:       # %bb.0: # %entry
+; MIPS64ELR6-NEXT:    daddiu $sp, $sp, -16
+; MIPS64ELR6-NEXT:    .cfi_def_cfa_offset 16
+; MIPS64ELR6-NEXT:    # kill: def $a1 killed $a1 killed $a1_64
+; MIPS64ELR6-NEXT:    sync
+; MIPS64ELR6-NEXT:    daddiu $1, $zero, -4
+; MIPS64ELR6-NEXT:    and $1, $4, $1
+; MIPS64ELR6-NEXT:    andi $2, $4, 3
+; MIPS64ELR6-NEXT:    sll $2, $2, 3
+; MIPS64ELR6-NEXT:    ori $3, $zero, 65535
+; MIPS64ELR6-NEXT:    sllv $3, $3, $2
+; MIPS64ELR6-NEXT:    nor $6, $zero, $3
+; MIPS64ELR6-NEXT:    sllv $5, $5, $2
+; MIPS64ELR6-NEXT:  .LBB5_1: # %entry
+; MIPS64ELR6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS64ELR6-NEXT:    ll $8, 0($1)
+; MIPS64ELR6-NEXT:    and $8, $8, $3
+; MIPS64ELR6-NEXT:    and $5, $5, $3
+; MIPS64ELR6-NEXT:    slt $11, $8, $5
+; MIPS64ELR6-NEXT:    selnez $9, $8, $11
+; MIPS64ELR6-NEXT:    seleqz $11, $5, $11
+; MIPS64ELR6-NEXT:    or $9, $9, $11
+; MIPS64ELR6-NEXT:    and $9, $9, $3
+; MIPS64ELR6-NEXT:    and $10, $8, $6
+; MIPS64ELR6-NEXT:    or $10, $10, $9
+; MIPS64ELR6-NEXT:    sc $10, 0($1)
+; MIPS64ELR6-NEXT:    beqzc $10, .LBB5_1
+; MIPS64ELR6-NEXT:  # %bb.2: # %entry
+; MIPS64ELR6-NEXT:    and $7, $8, $3
+; MIPS64ELR6-NEXT:    srlv $7, $7, $2
+; MIPS64ELR6-NEXT:    seh $7, $7
+; MIPS64ELR6-NEXT:  # %bb.3: # %entry
+; MIPS64ELR6-NEXT:    sw $7, 12($sp) # 4-byte Folded Spill
+; MIPS64ELR6-NEXT:  # %bb.4: # %entry
+; MIPS64ELR6-NEXT:    sync
+; MIPS64ELR6-NEXT:    lw $2, 12($sp) # 4-byte Folded Reload
+; MIPS64ELR6-NEXT:    daddiu $sp, $sp, 16
+; MIPS64ELR6-NEXT:    jrc $ra
+entry:
+  %0 = atomicrmw min i16* %ptr, i16 %val seq_cst
+  ret i16 %0
+}
+
+define i16 @test_umax_16(i16* nocapture %ptr, i16 signext %val) {
+; MIPS-LABEL: test_umax_16:
+; MIPS:       # %bb.0: # %entry
+; MIPS-NEXT:    addiu $sp, $sp, -8
+; MIPS-NEXT:    .cfi_def_cfa_offset 8
+; MIPS-NEXT:    move $1, $5
+; MIPS-NEXT:    sync
+; MIPS-NEXT:    addiu $2, $zero, -4
+; MIPS-NEXT:    and $2, $4, $2
+; MIPS-NEXT:    andi $3, $4, 3
+; MIPS-NEXT:    xori $3, $3, 2
+; MIPS-NEXT:    sll $3, $3, 3
+; MIPS-NEXT:    ori $4, $zero, 65535
+; MIPS-NEXT:    sllv $4, $4, $3
+; MIPS-NEXT:    nor $6, $zero, $4
+; MIPS-NEXT:    sllv $5, $5, $3
+; MIPS-NEXT:  $BB6_1: # %entry
+; MIPS-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS-NEXT:    ll $8, 0($2)
+; MIPS-NEXT:    sltu $11, $8, $5
+; MIPS-NEXT:    move $9, $8
+; MIPS-NEXT:    movn $9, $5, $11
+; MIPS-NEXT:    and $9, $9, $4
+; MIPS-NEXT:    and $10, $8, $6
+; MIPS-NEXT:    or $10, $10, $9
+; MIPS-NEXT:    sc $10, 0($2)
+; MIPS-NEXT:    beqz $10, $BB6_1
+; MIPS-NEXT:    nop
+; MIPS-NEXT:  # %bb.2: # %entry
+; MIPS-NEXT:    and $7, $8, $4
+; MIPS-NEXT:    srlv $7, $7, $3
+; MIPS-NEXT:    seh $7, $7
+; MIPS-NEXT:  # %bb.3: # %entry
+; MIPS-NEXT:    sw $7, 4($sp) # 4-byte Folded Spill
+; MIPS-NEXT:  # %bb.4: # %entry
+; MIPS-NEXT:    sync
+; MIPS-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MIPS-NEXT:    addiu $sp, $sp, 8
+; MIPS-NEXT:    jr $ra
+; MIPS-NEXT:    nop
+;
+; MIPSR6-LABEL: test_umax_16:
+; MIPSR6:       # %bb.0: # %entry
+; MIPSR6-NEXT:    addiu $sp, $sp, -8
+; MIPSR6-NEXT:    .cfi_def_cfa_offset 8
+; MIPSR6-NEXT:    move $1, $5
+; MIPSR6-NEXT:    sync
+; MIPSR6-NEXT:    addiu $2, $zero, -4
+; MIPSR6-NEXT:    and $2, $4, $2
+; MIPSR6-NEXT:    andi $3, $4, 3
+; MIPSR6-NEXT:    xori $3, $3, 2
+; MIPSR6-NEXT:    sll $3, $3, 3
+; MIPSR6-NEXT:    ori $4, $zero, 65535
+; MIPSR6-NEXT:    sllv $4, $4, $3
+; MIPSR6-NEXT:    nor $6, $zero, $4
+; MIPSR6-NEXT:    sllv $5, $5, $3
+; MIPSR6-NEXT:  $BB6_1: # %entry
+; MIPSR6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPSR6-NEXT:    ll $8, 0($2)
+; MIPSR6-NEXT:    sltu $11, $8, $5
+; MIPSR6-NEXT:    seleqz $9, $8, $11
+; MIPSR6-NEXT:    selnez $11, $5, $11
+; MIPSR6-NEXT:    or $9, $9, $11
+; MIPSR6-NEXT:    and $9, $9, $4
+; MIPSR6-NEXT:    and $10, $8, $6
+; MIPSR6-NEXT:    or $10, $10, $9
+; MIPSR6-NEXT:    sc $10, 0($2)
+; MIPSR6-NEXT:    beqzc $10, $BB6_1
+; MIPSR6-NEXT:  # %bb.2: # %entry
+; MIPSR6-NEXT:    and $7, $8, $4
+; MIPSR6-NEXT:    srlv $7, $7, $3
+; MIPSR6-NEXT:    seh $7, $7
+; MIPSR6-NEXT:  # %bb.3: # %entry
+; MIPSR6-NEXT:    sw $7, 4($sp) # 4-byte Folded Spill
+; MIPSR6-NEXT:  # %bb.4: # %entry
+; MIPSR6-NEXT:    sync
+; MIPSR6-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MIPSR6-NEXT:    addiu $sp, $sp, 8
+; MIPSR6-NEXT:    jrc $ra
+;
+; MM-LABEL: test_umax_16:
+; MM:       # %bb.0: # %entry
+; MM-NEXT:    addiu $sp, $sp, -8
+; MM-NEXT:    .cfi_def_cfa_offset 8
+; MM-NEXT:    move $1, $5
+; MM-NEXT:    sync
+; MM-NEXT:    addiu $2, $zero, -4
+; MM-NEXT:    and $2, $4, $2
+; MM-NEXT:    andi $3, $4, 3
+; MM-NEXT:    xori $3, $3, 2
+; MM-NEXT:    sll $3, $3, 3
+; MM-NEXT:    ori $4, $zero, 65535
+; MM-NEXT:    sllv $4, $4, $3
+; MM-NEXT:    nor $6, $zero, $4
+; MM-NEXT:    sllv $5, $5, $3
+; MM-NEXT:  $BB6_1: # %entry
+; MM-NEXT:    # =>This Inner Loop Header: Depth=1
+; MM-NEXT:    ll $8, 0($2)
+; MM-NEXT:    sltu $11, $8, $5
+; MM-NEXT:    or $9, $8, $zero
+; MM-NEXT:    movn $9, $5, $11
+; MM-NEXT:    and $9, $9, $4
+; MM-NEXT:    and $10, $8, $6
+; MM-NEXT:    or $10, $10, $9
+; MM-NEXT:    sc $10, 0($2)
+; MM-NEXT:    beqzc $10, $BB6_1
+; MM-NEXT:  # %bb.2: # %entry
+; MM-NEXT:    and $7, $8, $4
+; MM-NEXT:    srlv $7, $7, $3
+; MM-NEXT:    seh $7, $7
+; MM-NEXT:  # %bb.3: # %entry
+; MM-NEXT:    sw $7, 4($sp) # 4-byte Folded Spill
+; MM-NEXT:  # %bb.4: # %entry
+; MM-NEXT:    sync
+; MM-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MM-NEXT:    addiusp 8
+; MM-NEXT:    jrc $ra
+;
+; MMR6-LABEL: test_umax_16:
+; MMR6:       # %bb.0: # %entry
+; MMR6-NEXT:    addiu $sp, $sp, -8
+; MMR6-NEXT:    .cfi_def_cfa_offset 8
+; MMR6-NEXT:    move $1, $5
+; MMR6-NEXT:    sync
+; MMR6-NEXT:    addiu $2, $zero, -4
+; MMR6-NEXT:    and $2, $4, $2
+; MMR6-NEXT:    andi $3, $4, 3
+; MMR6-NEXT:    xori $3, $3, 2
+; MMR6-NEXT:    sll $3, $3, 3
+; MMR6-NEXT:    ori $4, $zero, 65535
+; MMR6-NEXT:    sllv $4, $4, $3
+; MMR6-NEXT:    nor $6, $zero, $4
+; MMR6-NEXT:    sllv $5, $5, $3
+; MMR6-NEXT:  $BB6_1: # %entry
+; MMR6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MMR6-NEXT:    ll $8, 0($2)
+; MMR6-NEXT:    sltu $11, $8, $5
+; MMR6-NEXT:    seleqz $9, $8, $11
+; MMR6-NEXT:    selnez $11, $5, $11
+; MMR6-NEXT:    or $9, $9, $11
+; MMR6-NEXT:    and $9, $9, $4
+; MMR6-NEXT:    and $10, $8, $6
+; MMR6-NEXT:    or $10, $10, $9
+; MMR6-NEXT:    sc $10, 0($2)
+; MMR6-NEXT:    beqc $10, $zero, $BB6_1
+; MMR6-NEXT:  # %bb.2: # %entry
+; MMR6-NEXT:    and $7, $8, $4
+; MMR6-NEXT:    srlv $7, $7, $3
+; MMR6-NEXT:    seh $7, $7
+; MMR6-NEXT:  # %bb.3: # %entry
+; MMR6-NEXT:    sw $7, 4($sp) # 4-byte Folded Spill
+; MMR6-NEXT:  # %bb.4: # %entry
+; MMR6-NEXT:    sync
+; MMR6-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MMR6-NEXT:    addiu $sp, $sp, 8
+; MMR6-NEXT:    jrc $ra
+;
+; MIPSEL-LABEL: test_umax_16:
+; MIPSEL:       # %bb.0: # %entry
+; MIPSEL-NEXT:    addiu $sp, $sp, -8
+; MIPSEL-NEXT:    .cfi_def_cfa_offset 8
+; MIPSEL-NEXT:    move $1, $5
+; MIPSEL-NEXT:    sync
+; MIPSEL-NEXT:    addiu $2, $zero, -4
+; MIPSEL-NEXT:    and $2, $4, $2
+; MIPSEL-NEXT:    andi $3, $4, 3
+; MIPSEL-NEXT:    sll $3, $3, 3
+; MIPSEL-NEXT:    ori $4, $zero, 65535
+; MIPSEL-NEXT:    sllv $4, $4, $3
+; MIPSEL-NEXT:    nor $6, $zero, $4
+; MIPSEL-NEXT:    sllv $5, $5, $3
+; MIPSEL-NEXT:  $BB6_1: # %entry
+; MIPSEL-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPSEL-NEXT:    ll $8, 0($2)
+; MIPSEL-NEXT:    and $8, $8, $4
+; MIPSEL-NEXT:    and $5, $5, $4
+; MIPSEL-NEXT:    sltu $11, $8, $5
+; MIPSEL-NEXT:    move $9, $8
+; MIPSEL-NEXT:    movn $9, $5, $11
+; MIPSEL-NEXT:    and $9, $9, $4
+; MIPSEL-NEXT:    and $10, $8, $6
+; MIPSEL-NEXT:    or $10, $10, $9
+; MIPSEL-NEXT:    sc $10, 0($2)
+; MIPSEL-NEXT:    beqz $10, $BB6_1
+; MIPSEL-NEXT:    nop
+; MIPSEL-NEXT:  # %bb.2: # %entry
+; MIPSEL-NEXT:    and $7, $8, $4
+; MIPSEL-NEXT:    srlv $7, $7, $3
+; MIPSEL-NEXT:    seh $7, $7
+; MIPSEL-NEXT:  # %bb.3: # %entry
+; MIPSEL-NEXT:    sw $7, 4($sp) # 4-byte Folded Spill
+; MIPSEL-NEXT:  # %bb.4: # %entry
+; MIPSEL-NEXT:    sync
+; MIPSEL-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MIPSEL-NEXT:    addiu $sp, $sp, 8
+; MIPSEL-NEXT:    jr $ra
+; MIPSEL-NEXT:    nop
+;
+; MIPSELR6-LABEL: test_umax_16:
+; MIPSELR6:       # %bb.0: # %entry
+; MIPSELR6-NEXT:    addiu $sp, $sp, -8
+; MIPSELR6-NEXT:    .cfi_def_cfa_offset 8
+; MIPSELR6-NEXT:    move $1, $5
+; MIPSELR6-NEXT:    sync
+; MIPSELR6-NEXT:    addiu $2, $zero, -4
+; MIPSELR6-NEXT:    and $2, $4, $2
+; MIPSELR6-NEXT:    andi $3, $4, 3
+; MIPSELR6-NEXT:    sll $3, $3, 3
+; MIPSELR6-NEXT:    ori $4, $zero, 65535
+; MIPSELR6-NEXT:    sllv $4, $4, $3
+; MIPSELR6-NEXT:    nor $6, $zero, $4
+; MIPSELR6-NEXT:    sllv $5, $5, $3
+; MIPSELR6-NEXT:  $BB6_1: # %entry
+; MIPSELR6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPSELR6-NEXT:    ll $8, 0($2)
+; MIPSELR6-NEXT:    and $8, $8, $4
+; MIPSELR6-NEXT:    and $5, $5, $4
+; MIPSELR6-NEXT:    sltu $11, $8, $5
+; MIPSELR6-NEXT:    seleqz $9, $8, $11
+; MIPSELR6-NEXT:    selnez $11, $5, $11
+; MIPSELR6-NEXT:    or $9, $9, $11
+; MIPSELR6-NEXT:    and $9, $9, $4
+; MIPSELR6-NEXT:    and $10, $8, $6
+; MIPSELR6-NEXT:    or $10, $10, $9
+; MIPSELR6-NEXT:    sc $10, 0($2)
+; MIPSELR6-NEXT:    beqzc $10, $BB6_1
+; MIPSELR6-NEXT:  # %bb.2: # %entry
+; MIPSELR6-NEXT:    and $7, $8, $4
+; MIPSELR6-NEXT:    srlv $7, $7, $3
+; MIPSELR6-NEXT:    seh $7, $7
+; MIPSELR6-NEXT:  # %bb.3: # %entry
+; MIPSELR6-NEXT:    sw $7, 4($sp) # 4-byte Folded Spill
+; MIPSELR6-NEXT:  # %bb.4: # %entry
+; MIPSELR6-NEXT:    sync
+; MIPSELR6-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MIPSELR6-NEXT:    addiu $sp, $sp, 8
+; MIPSELR6-NEXT:    jrc $ra
+;
+; MMEL-LABEL: test_umax_16:
+; MMEL:       # %bb.0: # %entry
+; MMEL-NEXT:    addiu $sp, $sp, -8
+; MMEL-NEXT:    .cfi_def_cfa_offset 8
+; MMEL-NEXT:    move $1, $5
+; MMEL-NEXT:    sync
+; MMEL-NEXT:    addiu $2, $zero, -4
+; MMEL-NEXT:    and $2, $4, $2
+; MMEL-NEXT:    andi $3, $4, 3
+; MMEL-NEXT:    sll $3, $3, 3
+; MMEL-NEXT:    ori $4, $zero, 65535
+; MMEL-NEXT:    sllv $4, $4, $3
+; MMEL-NEXT:    nor $6, $zero, $4
+; MMEL-NEXT:    sllv $5, $5, $3
+; MMEL-NEXT:  $BB6_1: # %entry
+; MMEL-NEXT:    # =>This Inner Loop Header: Depth=1
+; MMEL-NEXT:    ll $8, 0($2)
+; MMEL-NEXT:    and $8, $8, $4
+; MMEL-NEXT:    and $5, $5, $4
+; MMEL-NEXT:    sltu $11, $8, $5
+; MMEL-NEXT:    or $9, $8, $zero
+; MMEL-NEXT:    movn $9, $5, $11
+; MMEL-NEXT:    and $9, $9, $4
+; MMEL-NEXT:    and $10, $8, $6
+; MMEL-NEXT:    or $10, $10, $9
+; MMEL-NEXT:    sc $10, 0($2)
+; MMEL-NEXT:    beqzc $10, $BB6_1
+; MMEL-NEXT:  # %bb.2: # %entry
+; MMEL-NEXT:    and $7, $8, $4
+; MMEL-NEXT:    srlv $7, $7, $3
+; MMEL-NEXT:    seh $7, $7
+; MMEL-NEXT:  # %bb.3: # %entry
+; MMEL-NEXT:    sw $7, 4($sp) # 4-byte Folded Spill
+; MMEL-NEXT:  # %bb.4: # %entry
+; MMEL-NEXT:    sync
+; MMEL-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MMEL-NEXT:    addiusp 8
+; MMEL-NEXT:    jrc $ra
+;
+; MMELR6-LABEL: test_umax_16:
+; MMELR6:       # %bb.0: # %entry
+; MMELR6-NEXT:    addiu $sp, $sp, -8
+; MMELR6-NEXT:    .cfi_def_cfa_offset 8
+; MMELR6-NEXT:    move $1, $5
+; MMELR6-NEXT:    sync
+; MMELR6-NEXT:    addiu $2, $zero, -4
+; MMELR6-NEXT:    and $2, $4, $2
+; MMELR6-NEXT:    andi $3, $4, 3
+; MMELR6-NEXT:    sll $3, $3, 3
+; MMELR6-NEXT:    ori $4, $zero, 65535
+; MMELR6-NEXT:    sllv $4, $4, $3
+; MMELR6-NEXT:    nor $6, $zero, $4
+; MMELR6-NEXT:    sllv $5, $5, $3
+; MMELR6-NEXT:  $BB6_1: # %entry
+; MMELR6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MMELR6-NEXT:    ll $8, 0($2)
+; MMELR6-NEXT:    and $8, $8, $4
+; MMELR6-NEXT:    and $5, $5, $4
+; MMELR6-NEXT:    sltu $11, $8, $5
+; MMELR6-NEXT:    seleqz $9, $8, $11
+; MMELR6-NEXT:    selnez $11, $5, $11
+; MMELR6-NEXT:    or $9, $9, $11
+; MMELR6-NEXT:    and $9, $9, $4
+; MMELR6-NEXT:    and $10, $8, $6
+; MMELR6-NEXT:    or $10, $10, $9
+; MMELR6-NEXT:    sc $10, 0($2)
+; MMELR6-NEXT:    beqc $10, $zero, $BB6_1
+; MMELR6-NEXT:  # %bb.2: # %entry
+; MMELR6-NEXT:    and $7, $8, $4
+; MMELR6-NEXT:    srlv $7, $7, $3
+; MMELR6-NEXT:    seh $7, $7
+; MMELR6-NEXT:  # %bb.3: # %entry
+; MMELR6-NEXT:    sw $7, 4($sp) # 4-byte Folded Spill
+; MMELR6-NEXT:  # %bb.4: # %entry
+; MMELR6-NEXT:    sync
+; MMELR6-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MMELR6-NEXT:    addiu $sp, $sp, 8
+; MMELR6-NEXT:    jrc $ra
+;
+; MIPS64-LABEL: test_umax_16:
+; MIPS64:       # %bb.0: # %entry
+; MIPS64-NEXT:    daddiu $sp, $sp, -16
+; MIPS64-NEXT:    .cfi_def_cfa_offset 16
+; MIPS64-NEXT:    # kill: def $a1 killed $a1 killed $a1_64
+; MIPS64-NEXT:    sync
+; MIPS64-NEXT:    daddiu $1, $zero, -4
+; MIPS64-NEXT:    and $1, $4, $1
+; MIPS64-NEXT:    andi $2, $4, 3
+; MIPS64-NEXT:    xori $2, $2, 2
+; MIPS64-NEXT:    sll $2, $2, 3
+; MIPS64-NEXT:    ori $3, $zero, 65535
+; MIPS64-NEXT:    sllv $3, $3, $2
+; MIPS64-NEXT:    nor $6, $zero, $3
+; MIPS64-NEXT:    sllv $5, $5, $2
+; MIPS64-NEXT:  .LBB6_1: # %entry
+; MIPS64-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS64-NEXT:    ll $8, 0($1)
+; MIPS64-NEXT:    sltu $11, $8, $5
+; MIPS64-NEXT:    move $9, $8
+; MIPS64-NEXT:    movn $9, $5, $11
+; MIPS64-NEXT:    and $9, $9, $3
+; MIPS64-NEXT:    and $10, $8, $6
+; MIPS64-NEXT:    or $10, $10, $9
+; MIPS64-NEXT:    sc $10, 0($1)
+; MIPS64-NEXT:    beqz $10, .LBB6_1
+; MIPS64-NEXT:    nop
+; MIPS64-NEXT:  # %bb.2: # %entry
+; MIPS64-NEXT:    and $7, $8, $3
+; MIPS64-NEXT:    srlv $7, $7, $2
+; MIPS64-NEXT:    seh $7, $7
+; MIPS64-NEXT:  # %bb.3: # %entry
+; MIPS64-NEXT:    sw $7, 12($sp) # 4-byte Folded Spill
+; MIPS64-NEXT:  # %bb.4: # %entry
+; MIPS64-NEXT:    sync
+; MIPS64-NEXT:    lw $2, 12($sp) # 4-byte Folded Reload
+; MIPS64-NEXT:    daddiu $sp, $sp, 16
+; MIPS64-NEXT:    jr $ra
+; MIPS64-NEXT:    nop
+;
+; MIPS64R6-LABEL: test_umax_16:
+; MIPS64R6:       # %bb.0: # %entry
+; MIPS64R6-NEXT:    daddiu $sp, $sp, -16
+; MIPS64R6-NEXT:    .cfi_def_cfa_offset 16
+; MIPS64R6-NEXT:    # kill: def $a1 killed $a1 killed $a1_64
+; MIPS64R6-NEXT:    sync
+; MIPS64R6-NEXT:    daddiu $1, $zero, -4
+; MIPS64R6-NEXT:    and $1, $4, $1
+; MIPS64R6-NEXT:    andi $2, $4, 3
+; MIPS64R6-NEXT:    xori $2, $2, 2
+; MIPS64R6-NEXT:    sll $2, $2, 3
+; MIPS64R6-NEXT:    ori $3, $zero, 65535
+; MIPS64R6-NEXT:    sllv $3, $3, $2
+; MIPS64R6-NEXT:    nor $6, $zero, $3
+; MIPS64R6-NEXT:    sllv $5, $5, $2
+; MIPS64R6-NEXT:  .LBB6_1: # %entry
+; MIPS64R6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS64R6-NEXT:    ll $8, 0($1)
+; MIPS64R6-NEXT:    sltu $11, $8, $5
+; MIPS64R6-NEXT:    seleqz $9, $8, $11
+; MIPS64R6-NEXT:    selnez $11, $5, $11
+; MIPS64R6-NEXT:    or $9, $9, $11
+; MIPS64R6-NEXT:    and $9, $9, $3
+; MIPS64R6-NEXT:    and $10, $8, $6
+; MIPS64R6-NEXT:    or $10, $10, $9
+; MIPS64R6-NEXT:    sc $10, 0($1)
+; MIPS64R6-NEXT:    beqzc $10, .LBB6_1
+; MIPS64R6-NEXT:  # %bb.2: # %entry
+; MIPS64R6-NEXT:    and $7, $8, $3
+; MIPS64R6-NEXT:    srlv $7, $7, $2
+; MIPS64R6-NEXT:    seh $7, $7
+; MIPS64R6-NEXT:  # %bb.3: # %entry
+; MIPS64R6-NEXT:    sw $7, 12($sp) # 4-byte Folded Spill
+; MIPS64R6-NEXT:  # %bb.4: # %entry
+; MIPS64R6-NEXT:    sync
+; MIPS64R6-NEXT:    lw $2, 12($sp) # 4-byte Folded Reload
+; MIPS64R6-NEXT:    daddiu $sp, $sp, 16
+; MIPS64R6-NEXT:    jrc $ra
+;
+; MIPS64EL-LABEL: test_umax_16:
+; MIPS64EL:       # %bb.0: # %entry
+; MIPS64EL-NEXT:    daddiu $sp, $sp, -16
+; MIPS64EL-NEXT:    .cfi_def_cfa_offset 16
+; MIPS64EL-NEXT:    # kill: def $a1 killed $a1 killed $a1_64
+; MIPS64EL-NEXT:    sync
+; MIPS64EL-NEXT:    daddiu $1, $zero, -4
+; MIPS64EL-NEXT:    and $1, $4, $1
+; MIPS64EL-NEXT:    andi $2, $4, 3
+; MIPS64EL-NEXT:    sll $2, $2, 3
+; MIPS64EL-NEXT:    ori $3, $zero, 65535
+; MIPS64EL-NEXT:    sllv $3, $3, $2
+; MIPS64EL-NEXT:    nor $6, $zero, $3
+; MIPS64EL-NEXT:    sllv $5, $5, $2
+; MIPS64EL-NEXT:  .LBB6_1: # %entry
+; MIPS64EL-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS64EL-NEXT:    ll $8, 0($1)
+; MIPS64EL-NEXT:    and $8, $8, $3
+; MIPS64EL-NEXT:    and $5, $5, $3
+; MIPS64EL-NEXT:    sltu $11, $8, $5
+; MIPS64EL-NEXT:    move $9, $8
+; MIPS64EL-NEXT:    movn $9, $5, $11
+; MIPS64EL-NEXT:    and $9, $9, $3
+; MIPS64EL-NEXT:    and $10, $8, $6
+; MIPS64EL-NEXT:    or $10, $10, $9
+; MIPS64EL-NEXT:    sc $10, 0($1)
+; MIPS64EL-NEXT:    beqz $10, .LBB6_1
+; MIPS64EL-NEXT:    nop
+; MIPS64EL-NEXT:  # %bb.2: # %entry
+; MIPS64EL-NEXT:    and $7, $8, $3
+; MIPS64EL-NEXT:    srlv $7, $7, $2
+; MIPS64EL-NEXT:    seh $7, $7
+; MIPS64EL-NEXT:  # %bb.3: # %entry
+; MIPS64EL-NEXT:    sw $7, 12($sp) # 4-byte Folded Spill
+; MIPS64EL-NEXT:  # %bb.4: # %entry
+; MIPS64EL-NEXT:    sync
+; MIPS64EL-NEXT:    lw $2, 12($sp) # 4-byte Folded Reload
+; MIPS64EL-NEXT:    daddiu $sp, $sp, 16
+; MIPS64EL-NEXT:    jr $ra
+; MIPS64EL-NEXT:    nop
+;
+; MIPS64ELR6-LABEL: test_umax_16:
+; MIPS64ELR6:       # %bb.0: # %entry
+; MIPS64ELR6-NEXT:    daddiu $sp, $sp, -16
+; MIPS64ELR6-NEXT:    .cfi_def_cfa_offset 16
+; MIPS64ELR6-NEXT:    # kill: def $a1 killed $a1 killed $a1_64
+; MIPS64ELR6-NEXT:    sync
+; MIPS64ELR6-NEXT:    daddiu $1, $zero, -4
+; MIPS64ELR6-NEXT:    and $1, $4, $1
+; MIPS64ELR6-NEXT:    andi $2, $4, 3
+; MIPS64ELR6-NEXT:    sll $2, $2, 3
+; MIPS64ELR6-NEXT:    ori $3, $zero, 65535
+; MIPS64ELR6-NEXT:    sllv $3, $3, $2
+; MIPS64ELR6-NEXT:    nor $6, $zero, $3
+; MIPS64ELR6-NEXT:    sllv $5, $5, $2
+; MIPS64ELR6-NEXT:  .LBB6_1: # %entry
+; MIPS64ELR6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS64ELR6-NEXT:    ll $8, 0($1)
+; MIPS64ELR6-NEXT:    and $8, $8, $3
+; MIPS64ELR6-NEXT:    and $5, $5, $3
+; MIPS64ELR6-NEXT:    sltu $11, $8, $5
+; MIPS64ELR6-NEXT:    seleqz $9, $8, $11
+; MIPS64ELR6-NEXT:    selnez $11, $5, $11
+; MIPS64ELR6-NEXT:    or $9, $9, $11
+; MIPS64ELR6-NEXT:    and $9, $9, $3
+; MIPS64ELR6-NEXT:    and $10, $8, $6
+; MIPS64ELR6-NEXT:    or $10, $10, $9
+; MIPS64ELR6-NEXT:    sc $10, 0($1)
+; MIPS64ELR6-NEXT:    beqzc $10, .LBB6_1
+; MIPS64ELR6-NEXT:  # %bb.2: # %entry
+; MIPS64ELR6-NEXT:    and $7, $8, $3
+; MIPS64ELR6-NEXT:    srlv $7, $7, $2
+; MIPS64ELR6-NEXT:    seh $7, $7
+; MIPS64ELR6-NEXT:  # %bb.3: # %entry
+; MIPS64ELR6-NEXT:    sw $7, 12($sp) # 4-byte Folded Spill
+; MIPS64ELR6-NEXT:  # %bb.4: # %entry
+; MIPS64ELR6-NEXT:    sync
+; MIPS64ELR6-NEXT:    lw $2, 12($sp) # 4-byte Folded Reload
+; MIPS64ELR6-NEXT:    daddiu $sp, $sp, 16
+; MIPS64ELR6-NEXT:    jrc $ra
+entry:
+  %0 = atomicrmw umax i16* %ptr, i16 %val seq_cst
+  ret i16 %0
+}
+
+define i16 @test_umin_16(i16* nocapture %ptr, i16 signext %val) {
+; MIPS-LABEL: test_umin_16:
+; MIPS:       # %bb.0: # %entry
+; MIPS-NEXT:    addiu $sp, $sp, -8
+; MIPS-NEXT:    .cfi_def_cfa_offset 8
+; MIPS-NEXT:    move $1, $5
+; MIPS-NEXT:    sync
+; MIPS-NEXT:    addiu $2, $zero, -4
+; MIPS-NEXT:    and $2, $4, $2
+; MIPS-NEXT:    andi $3, $4, 3
+; MIPS-NEXT:    xori $3, $3, 2
+; MIPS-NEXT:    sll $3, $3, 3
+; MIPS-NEXT:    ori $4, $zero, 65535
+; MIPS-NEXT:    sllv $4, $4, $3
+; MIPS-NEXT:    nor $6, $zero, $4
+; MIPS-NEXT:    sllv $5, $5, $3
+; MIPS-NEXT:  $BB7_1: # %entry
+; MIPS-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS-NEXT:    ll $8, 0($2)
+; MIPS-NEXT:    sltu $11, $8, $5
+; MIPS-NEXT:    move $9, $8
+; MIPS-NEXT:    movz $9, $5, $11
+; MIPS-NEXT:    and $9, $9, $4
+; MIPS-NEXT:    and $10, $8, $6
+; MIPS-NEXT:    or $10, $10, $9
+; MIPS-NEXT:    sc $10, 0($2)
+; MIPS-NEXT:    beqz $10, $BB7_1
+; MIPS-NEXT:    nop
+; MIPS-NEXT:  # %bb.2: # %entry
+; MIPS-NEXT:    and $7, $8, $4
+; MIPS-NEXT:    srlv $7, $7, $3
+; MIPS-NEXT:    seh $7, $7
+; MIPS-NEXT:  # %bb.3: # %entry
+; MIPS-NEXT:    sw $7, 4($sp) # 4-byte Folded Spill
+; MIPS-NEXT:  # %bb.4: # %entry
+; MIPS-NEXT:    sync
+; MIPS-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MIPS-NEXT:    addiu $sp, $sp, 8
+; MIPS-NEXT:    jr $ra
+; MIPS-NEXT:    nop
+;
+; MIPSR6-LABEL: test_umin_16:
+; MIPSR6:       # %bb.0: # %entry
+; MIPSR6-NEXT:    addiu $sp, $sp, -8
+; MIPSR6-NEXT:    .cfi_def_cfa_offset 8
+; MIPSR6-NEXT:    move $1, $5
+; MIPSR6-NEXT:    sync
+; MIPSR6-NEXT:    addiu $2, $zero, -4
+; MIPSR6-NEXT:    and $2, $4, $2
+; MIPSR6-NEXT:    andi $3, $4, 3
+; MIPSR6-NEXT:    xori $3, $3, 2
+; MIPSR6-NEXT:    sll $3, $3, 3
+; MIPSR6-NEXT:    ori $4, $zero, 65535
+; MIPSR6-NEXT:    sllv $4, $4, $3
+; MIPSR6-NEXT:    nor $6, $zero, $4
+; MIPSR6-NEXT:    sllv $5, $5, $3
+; MIPSR6-NEXT:  $BB7_1: # %entry
+; MIPSR6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPSR6-NEXT:    ll $8, 0($2)
+; MIPSR6-NEXT:    sltu $11, $8, $5
+; MIPSR6-NEXT:    selnez $9, $8, $11
+; MIPSR6-NEXT:    seleqz $11, $5, $11
+; MIPSR6-NEXT:    or $9, $9, $11
+; MIPSR6-NEXT:    and $9, $9, $4
+; MIPSR6-NEXT:    and $10, $8, $6
+; MIPSR6-NEXT:    or $10, $10, $9
+; MIPSR6-NEXT:    sc $10, 0($2)
+; MIPSR6-NEXT:    beqzc $10, $BB7_1
+; MIPSR6-NEXT:  # %bb.2: # %entry
+; MIPSR6-NEXT:    and $7, $8, $4
+; MIPSR6-NEXT:    srlv $7, $7, $3
+; MIPSR6-NEXT:    seh $7, $7
+; MIPSR6-NEXT:  # %bb.3: # %entry
+; MIPSR6-NEXT:    sw $7, 4($sp) # 4-byte Folded Spill
+; MIPSR6-NEXT:  # %bb.4: # %entry
+; MIPSR6-NEXT:    sync
+; MIPSR6-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MIPSR6-NEXT:    addiu $sp, $sp, 8
+; MIPSR6-NEXT:    jrc $ra
+;
+; MM-LABEL: test_umin_16:
+; MM:       # %bb.0: # %entry
+; MM-NEXT:    addiu $sp, $sp, -8
+; MM-NEXT:    .cfi_def_cfa_offset 8
+; MM-NEXT:    move $1, $5
+; MM-NEXT:    sync
+; MM-NEXT:    addiu $2, $zero, -4
+; MM-NEXT:    and $2, $4, $2
+; MM-NEXT:    andi $3, $4, 3
+; MM-NEXT:    xori $3, $3, 2
+; MM-NEXT:    sll $3, $3, 3
+; MM-NEXT:    ori $4, $zero, 65535
+; MM-NEXT:    sllv $4, $4, $3
+; MM-NEXT:    nor $6, $zero, $4
+; MM-NEXT:    sllv $5, $5, $3
+; MM-NEXT:  $BB7_1: # %entry
+; MM-NEXT:    # =>This Inner Loop Header: Depth=1
+; MM-NEXT:    ll $8, 0($2)
+; MM-NEXT:    sltu $11, $8, $5
+; MM-NEXT:    or $9, $8, $zero
+; MM-NEXT:    movz $9, $5, $11
+; MM-NEXT:    and $9, $9, $4
+; MM-NEXT:    and $10, $8, $6
+; MM-NEXT:    or $10, $10, $9
+; MM-NEXT:    sc $10, 0($2)
+; MM-NEXT:    beqzc $10, $BB7_1
+; MM-NEXT:  # %bb.2: # %entry
+; MM-NEXT:    and $7, $8, $4
+; MM-NEXT:    srlv $7, $7, $3
+; MM-NEXT:    seh $7, $7
+; MM-NEXT:  # %bb.3: # %entry
+; MM-NEXT:    sw $7, 4($sp) # 4-byte Folded Spill
+; MM-NEXT:  # %bb.4: # %entry
+; MM-NEXT:    sync
+; MM-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MM-NEXT:    addiusp 8
+; MM-NEXT:    jrc $ra
+;
+; MMR6-LABEL: test_umin_16:
+; MMR6:       # %bb.0: # %entry
+; MMR6-NEXT:    addiu $sp, $sp, -8
+; MMR6-NEXT:    .cfi_def_cfa_offset 8
+; MMR6-NEXT:    move $1, $5
+; MMR6-NEXT:    sync
+; MMR6-NEXT:    addiu $2, $zero, -4
+; MMR6-NEXT:    and $2, $4, $2
+; MMR6-NEXT:    andi $3, $4, 3
+; MMR6-NEXT:    xori $3, $3, 2
+; MMR6-NEXT:    sll $3, $3, 3
+; MMR6-NEXT:    ori $4, $zero, 65535
+; MMR6-NEXT:    sllv $4, $4, $3
+; MMR6-NEXT:    nor $6, $zero, $4
+; MMR6-NEXT:    sllv $5, $5, $3
+; MMR6-NEXT:  $BB7_1: # %entry
+; MMR6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MMR6-NEXT:    ll $8, 0($2)
+; MMR6-NEXT:    sltu $11, $8, $5
+; MMR6-NEXT:    selnez $9, $8, $11
+; MMR6-NEXT:    seleqz $11, $5, $11
+; MMR6-NEXT:    or $9, $9, $11
+; MMR6-NEXT:    and $9, $9, $4
+; MMR6-NEXT:    and $10, $8, $6
+; MMR6-NEXT:    or $10, $10, $9
+; MMR6-NEXT:    sc $10, 0($2)
+; MMR6-NEXT:    beqc $10, $zero, $BB7_1
+; MMR6-NEXT:  # %bb.2: # %entry
+; MMR6-NEXT:    and $7, $8, $4
+; MMR6-NEXT:    srlv $7, $7, $3
+; MMR6-NEXT:    seh $7, $7
+; MMR6-NEXT:  # %bb.3: # %entry
+; MMR6-NEXT:    sw $7, 4($sp) # 4-byte Folded Spill
+; MMR6-NEXT:  # %bb.4: # %entry
+; MMR6-NEXT:    sync
+; MMR6-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MMR6-NEXT:    addiu $sp, $sp, 8
+; MMR6-NEXT:    jrc $ra
+;
+; MIPSEL-LABEL: test_umin_16:
+; MIPSEL:       # %bb.0: # %entry
+; MIPSEL-NEXT:    addiu $sp, $sp, -8
+; MIPSEL-NEXT:    .cfi_def_cfa_offset 8
+; MIPSEL-NEXT:    move $1, $5
+; MIPSEL-NEXT:    sync
+; MIPSEL-NEXT:    addiu $2, $zero, -4
+; MIPSEL-NEXT:    and $2, $4, $2
+; MIPSEL-NEXT:    andi $3, $4, 3
+; MIPSEL-NEXT:    sll $3, $3, 3
+; MIPSEL-NEXT:    ori $4, $zero, 65535
+; MIPSEL-NEXT:    sllv $4, $4, $3
+; MIPSEL-NEXT:    nor $6, $zero, $4
+; MIPSEL-NEXT:    sllv $5, $5, $3
+; MIPSEL-NEXT:  $BB7_1: # %entry
+; MIPSEL-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPSEL-NEXT:    ll $8, 0($2)
+; MIPSEL-NEXT:    and $8, $8, $4
+; MIPSEL-NEXT:    and $5, $5, $4
+; MIPSEL-NEXT:    sltu $11, $8, $5
+; MIPSEL-NEXT:    move $9, $8
+; MIPSEL-NEXT:    movz $9, $5, $11
+; MIPSEL-NEXT:    and $9, $9, $4
+; MIPSEL-NEXT:    and $10, $8, $6
+; MIPSEL-NEXT:    or $10, $10, $9
+; MIPSEL-NEXT:    sc $10, 0($2)
+; MIPSEL-NEXT:    beqz $10, $BB7_1
+; MIPSEL-NEXT:    nop
+; MIPSEL-NEXT:  # %bb.2: # %entry
+; MIPSEL-NEXT:    and $7, $8, $4
+; MIPSEL-NEXT:    srlv $7, $7, $3
+; MIPSEL-NEXT:    seh $7, $7
+; MIPSEL-NEXT:  # %bb.3: # %entry
+; MIPSEL-NEXT:    sw $7, 4($sp) # 4-byte Folded Spill
+; MIPSEL-NEXT:  # %bb.4: # %entry
+; MIPSEL-NEXT:    sync
+; MIPSEL-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MIPSEL-NEXT:    addiu $sp, $sp, 8
+; MIPSEL-NEXT:    jr $ra
+; MIPSEL-NEXT:    nop
+;
+; MIPSELR6-LABEL: test_umin_16:
+; MIPSELR6:       # %bb.0: # %entry
+; MIPSELR6-NEXT:    addiu $sp, $sp, -8
+; MIPSELR6-NEXT:    .cfi_def_cfa_offset 8
+; MIPSELR6-NEXT:    move $1, $5
+; MIPSELR6-NEXT:    sync
+; MIPSELR6-NEXT:    addiu $2, $zero, -4
+; MIPSELR6-NEXT:    and $2, $4, $2
+; MIPSELR6-NEXT:    andi $3, $4, 3
+; MIPSELR6-NEXT:    sll $3, $3, 3
+; MIPSELR6-NEXT:    ori $4, $zero, 65535
+; MIPSELR6-NEXT:    sllv $4, $4, $3
+; MIPSELR6-NEXT:    nor $6, $zero, $4
+; MIPSELR6-NEXT:    sllv $5, $5, $3
+; MIPSELR6-NEXT:  $BB7_1: # %entry
+; MIPSELR6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPSELR6-NEXT:    ll $8, 0($2)
+; MIPSELR6-NEXT:    and $8, $8, $4
+; MIPSELR6-NEXT:    and $5, $5, $4
+; MIPSELR6-NEXT:    sltu $11, $8, $5
+; MIPSELR6-NEXT:    selnez $9, $8, $11
+; MIPSELR6-NEXT:    seleqz $11, $5, $11
+; MIPSELR6-NEXT:    or $9, $9, $11
+; MIPSELR6-NEXT:    and $9, $9, $4
+; MIPSELR6-NEXT:    and $10, $8, $6
+; MIPSELR6-NEXT:    or $10, $10, $9
+; MIPSELR6-NEXT:    sc $10, 0($2)
+; MIPSELR6-NEXT:    beqzc $10, $BB7_1
+; MIPSELR6-NEXT:  # %bb.2: # %entry
+; MIPSELR6-NEXT:    and $7, $8, $4
+; MIPSELR6-NEXT:    srlv $7, $7, $3
+; MIPSELR6-NEXT:    seh $7, $7
+; MIPSELR6-NEXT:  # %bb.3: # %entry
+; MIPSELR6-NEXT:    sw $7, 4($sp) # 4-byte Folded Spill
+; MIPSELR6-NEXT:  # %bb.4: # %entry
+; MIPSELR6-NEXT:    sync
+; MIPSELR6-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MIPSELR6-NEXT:    addiu $sp, $sp, 8
+; MIPSELR6-NEXT:    jrc $ra
+;
+; MMEL-LABEL: test_umin_16:
+; MMEL:       # %bb.0: # %entry
+; MMEL-NEXT:    addiu $sp, $sp, -8
+; MMEL-NEXT:    .cfi_def_cfa_offset 8
+; MMEL-NEXT:    move $1, $5
+; MMEL-NEXT:    sync
+; MMEL-NEXT:    addiu $2, $zero, -4
+; MMEL-NEXT:    and $2, $4, $2
+; MMEL-NEXT:    andi $3, $4, 3
+; MMEL-NEXT:    sll $3, $3, 3
+; MMEL-NEXT:    ori $4, $zero, 65535
+; MMEL-NEXT:    sllv $4, $4, $3
+; MMEL-NEXT:    nor $6, $zero, $4
+; MMEL-NEXT:    sllv $5, $5, $3
+; MMEL-NEXT:  $BB7_1: # %entry
+; MMEL-NEXT:    # =>This Inner Loop Header: Depth=1
+; MMEL-NEXT:    ll $8, 0($2)
+; MMEL-NEXT:    and $8, $8, $4
+; MMEL-NEXT:    and $5, $5, $4
+; MMEL-NEXT:    sltu $11, $8, $5
+; MMEL-NEXT:    or $9, $8, $zero
+; MMEL-NEXT:    movz $9, $5, $11
+; MMEL-NEXT:    and $9, $9, $4
+; MMEL-NEXT:    and $10, $8, $6
+; MMEL-NEXT:    or $10, $10, $9
+; MMEL-NEXT:    sc $10, 0($2)
+; MMEL-NEXT:    beqzc $10, $BB7_1
+; MMEL-NEXT:  # %bb.2: # %entry
+; MMEL-NEXT:    and $7, $8, $4
+; MMEL-NEXT:    srlv $7, $7, $3
+; MMEL-NEXT:    seh $7, $7
+; MMEL-NEXT:  # %bb.3: # %entry
+; MMEL-NEXT:    sw $7, 4($sp) # 4-byte Folded Spill
+; MMEL-NEXT:  # %bb.4: # %entry
+; MMEL-NEXT:    sync
+; MMEL-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MMEL-NEXT:    addiusp 8
+; MMEL-NEXT:    jrc $ra
+;
+; MMELR6-LABEL: test_umin_16:
+; MMELR6:       # %bb.0: # %entry
+; MMELR6-NEXT:    addiu $sp, $sp, -8
+; MMELR6-NEXT:    .cfi_def_cfa_offset 8
+; MMELR6-NEXT:    move $1, $5
+; MMELR6-NEXT:    sync
+; MMELR6-NEXT:    addiu $2, $zero, -4
+; MMELR6-NEXT:    and $2, $4, $2
+; MMELR6-NEXT:    andi $3, $4, 3
+; MMELR6-NEXT:    sll $3, $3, 3
+; MMELR6-NEXT:    ori $4, $zero, 65535
+; MMELR6-NEXT:    sllv $4, $4, $3
+; MMELR6-NEXT:    nor $6, $zero, $4
+; MMELR6-NEXT:    sllv $5, $5, $3
+; MMELR6-NEXT:  $BB7_1: # %entry
+; MMELR6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MMELR6-NEXT:    ll $8, 0($2)
+; MMELR6-NEXT:    and $8, $8, $4
+; MMELR6-NEXT:    and $5, $5, $4
+; MMELR6-NEXT:    sltu $11, $8, $5
+; MMELR6-NEXT:    selnez $9, $8, $11
+; MMELR6-NEXT:    seleqz $11, $5, $11
+; MMELR6-NEXT:    or $9, $9, $11
+; MMELR6-NEXT:    and $9, $9, $4
+; MMELR6-NEXT:    and $10, $8, $6
+; MMELR6-NEXT:    or $10, $10, $9
+; MMELR6-NEXT:    sc $10, 0($2)
+; MMELR6-NEXT:    beqc $10, $zero, $BB7_1
+; MMELR6-NEXT:  # %bb.2: # %entry
+; MMELR6-NEXT:    and $7, $8, $4
+; MMELR6-NEXT:    srlv $7, $7, $3
+; MMELR6-NEXT:    seh $7, $7
+; MMELR6-NEXT:  # %bb.3: # %entry
+; MMELR6-NEXT:    sw $7, 4($sp) # 4-byte Folded Spill
+; MMELR6-NEXT:  # %bb.4: # %entry
+; MMELR6-NEXT:    sync
+; MMELR6-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MMELR6-NEXT:    addiu $sp, $sp, 8
+; MMELR6-NEXT:    jrc $ra
+;
+; MIPS64-LABEL: test_umin_16:
+; MIPS64:       # %bb.0: # %entry
+; MIPS64-NEXT:    daddiu $sp, $sp, -16
+; MIPS64-NEXT:    .cfi_def_cfa_offset 16
+; MIPS64-NEXT:    # kill: def $a1 killed $a1 killed $a1_64
+; MIPS64-NEXT:    sync
+; MIPS64-NEXT:    daddiu $1, $zero, -4
+; MIPS64-NEXT:    and $1, $4, $1
+; MIPS64-NEXT:    andi $2, $4, 3
+; MIPS64-NEXT:    xori $2, $2, 2
+; MIPS64-NEXT:    sll $2, $2, 3
+; MIPS64-NEXT:    ori $3, $zero, 65535
+; MIPS64-NEXT:    sllv $3, $3, $2
+; MIPS64-NEXT:    nor $6, $zero, $3
+; MIPS64-NEXT:    sllv $5, $5, $2
+; MIPS64-NEXT:  .LBB7_1: # %entry
+; MIPS64-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS64-NEXT:    ll $8, 0($1)
+; MIPS64-NEXT:    sltu $11, $8, $5
+; MIPS64-NEXT:    move $9, $8
+; MIPS64-NEXT:    movz $9, $5, $11
+; MIPS64-NEXT:    and $9, $9, $3
+; MIPS64-NEXT:    and $10, $8, $6
+; MIPS64-NEXT:    or $10, $10, $9
+; MIPS64-NEXT:    sc $10, 0($1)
+; MIPS64-NEXT:    beqz $10, .LBB7_1
+; MIPS64-NEXT:    nop
+; MIPS64-NEXT:  # %bb.2: # %entry
+; MIPS64-NEXT:    and $7, $8, $3
+; MIPS64-NEXT:    srlv $7, $7, $2
+; MIPS64-NEXT:    seh $7, $7
+; MIPS64-NEXT:  # %bb.3: # %entry
+; MIPS64-NEXT:    sw $7, 12($sp) # 4-byte Folded Spill
+; MIPS64-NEXT:  # %bb.4: # %entry
+; MIPS64-NEXT:    sync
+; MIPS64-NEXT:    lw $2, 12($sp) # 4-byte Folded Reload
+; MIPS64-NEXT:    daddiu $sp, $sp, 16
+; MIPS64-NEXT:    jr $ra
+; MIPS64-NEXT:    nop
+;
+; MIPS64R6-LABEL: test_umin_16:
+; MIPS64R6:       # %bb.0: # %entry
+; MIPS64R6-NEXT:    daddiu $sp, $sp, -16
+; MIPS64R6-NEXT:    .cfi_def_cfa_offset 16
+; MIPS64R6-NEXT:    # kill: def $a1 killed $a1 killed $a1_64
+; MIPS64R6-NEXT:    sync
+; MIPS64R6-NEXT:    daddiu $1, $zero, -4
+; MIPS64R6-NEXT:    and $1, $4, $1
+; MIPS64R6-NEXT:    andi $2, $4, 3
+; MIPS64R6-NEXT:    xori $2, $2, 2
+; MIPS64R6-NEXT:    sll $2, $2, 3
+; MIPS64R6-NEXT:    ori $3, $zero, 65535
+; MIPS64R6-NEXT:    sllv $3, $3, $2
+; MIPS64R6-NEXT:    nor $6, $zero, $3
+; MIPS64R6-NEXT:    sllv $5, $5, $2
+; MIPS64R6-NEXT:  .LBB7_1: # %entry
+; MIPS64R6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS64R6-NEXT:    ll $8, 0($1)
+; MIPS64R6-NEXT:    sltu $11, $8, $5
+; MIPS64R6-NEXT:    selnez $9, $8, $11
+; MIPS64R6-NEXT:    seleqz $11, $5, $11
+; MIPS64R6-NEXT:    or $9, $9, $11
+; MIPS64R6-NEXT:    and $9, $9, $3
+; MIPS64R6-NEXT:    and $10, $8, $6
+; MIPS64R6-NEXT:    or $10, $10, $9
+; MIPS64R6-NEXT:    sc $10, 0($1)
+; MIPS64R6-NEXT:    beqzc $10, .LBB7_1
+; MIPS64R6-NEXT:  # %bb.2: # %entry
+; MIPS64R6-NEXT:    and $7, $8, $3
+; MIPS64R6-NEXT:    srlv $7, $7, $2
+; MIPS64R6-NEXT:    seh $7, $7
+; MIPS64R6-NEXT:  # %bb.3: # %entry
+; MIPS64R6-NEXT:    sw $7, 12($sp) # 4-byte Folded Spill
+; MIPS64R6-NEXT:  # %bb.4: # %entry
+; MIPS64R6-NEXT:    sync
+; MIPS64R6-NEXT:    lw $2, 12($sp) # 4-byte Folded Reload
+; MIPS64R6-NEXT:    daddiu $sp, $sp, 16
+; MIPS64R6-NEXT:    jrc $ra
+;
+; MIPS64EL-LABEL: test_umin_16:
+; MIPS64EL:       # %bb.0: # %entry
+; MIPS64EL-NEXT:    daddiu $sp, $sp, -16
+; MIPS64EL-NEXT:    .cfi_def_cfa_offset 16
+; MIPS64EL-NEXT:    # kill: def $a1 killed $a1 killed $a1_64
+; MIPS64EL-NEXT:    sync
+; MIPS64EL-NEXT:    daddiu $1, $zero, -4
+; MIPS64EL-NEXT:    and $1, $4, $1
+; MIPS64EL-NEXT:    andi $2, $4, 3
+; MIPS64EL-NEXT:    sll $2, $2, 3
+; MIPS64EL-NEXT:    ori $3, $zero, 65535
+; MIPS64EL-NEXT:    sllv $3, $3, $2
+; MIPS64EL-NEXT:    nor $6, $zero, $3
+; MIPS64EL-NEXT:    sllv $5, $5, $2
+; MIPS64EL-NEXT:  .LBB7_1: # %entry
+; MIPS64EL-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS64EL-NEXT:    ll $8, 0($1)
+; MIPS64EL-NEXT:    and $8, $8, $3
+; MIPS64EL-NEXT:    and $5, $5, $3
+; MIPS64EL-NEXT:    sltu $11, $8, $5
+; MIPS64EL-NEXT:    move $9, $8
+; MIPS64EL-NEXT:    movz $9, $5, $11
+; MIPS64EL-NEXT:    and $9, $9, $3
+; MIPS64EL-NEXT:    and $10, $8, $6
+; MIPS64EL-NEXT:    or $10, $10, $9
+; MIPS64EL-NEXT:    sc $10, 0($1)
+; MIPS64EL-NEXT:    beqz $10, .LBB7_1
+; MIPS64EL-NEXT:    nop
+; MIPS64EL-NEXT:  # %bb.2: # %entry
+; MIPS64EL-NEXT:    and $7, $8, $3
+; MIPS64EL-NEXT:    srlv $7, $7, $2
+; MIPS64EL-NEXT:    seh $7, $7
+; MIPS64EL-NEXT:  # %bb.3: # %entry
+; MIPS64EL-NEXT:    sw $7, 12($sp) # 4-byte Folded Spill
+; MIPS64EL-NEXT:  # %bb.4: # %entry
+; MIPS64EL-NEXT:    sync
+; MIPS64EL-NEXT:    lw $2, 12($sp) # 4-byte Folded Reload
+; MIPS64EL-NEXT:    daddiu $sp, $sp, 16
+; MIPS64EL-NEXT:    jr $ra
+; MIPS64EL-NEXT:    nop
+;
+; MIPS64ELR6-LABEL: test_umin_16:
+; MIPS64ELR6:       # %bb.0: # %entry
+; MIPS64ELR6-NEXT:    daddiu $sp, $sp, -16
+; MIPS64ELR6-NEXT:    .cfi_def_cfa_offset 16
+; MIPS64ELR6-NEXT:    # kill: def $a1 killed $a1 killed $a1_64
+; MIPS64ELR6-NEXT:    sync
+; MIPS64ELR6-NEXT:    daddiu $1, $zero, -4
+; MIPS64ELR6-NEXT:    and $1, $4, $1
+; MIPS64ELR6-NEXT:    andi $2, $4, 3
+; MIPS64ELR6-NEXT:    sll $2, $2, 3
+; MIPS64ELR6-NEXT:    ori $3, $zero, 65535
+; MIPS64ELR6-NEXT:    sllv $3, $3, $2
+; MIPS64ELR6-NEXT:    nor $6, $zero, $3
+; MIPS64ELR6-NEXT:    sllv $5, $5, $2
+; MIPS64ELR6-NEXT:  .LBB7_1: # %entry
+; MIPS64ELR6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS64ELR6-NEXT:    ll $8, 0($1)
+; MIPS64ELR6-NEXT:    and $8, $8, $3
+; MIPS64ELR6-NEXT:    and $5, $5, $3
+; MIPS64ELR6-NEXT:    sltu $11, $8, $5
+; MIPS64ELR6-NEXT:    selnez $9, $8, $11
+; MIPS64ELR6-NEXT:    seleqz $11, $5, $11
+; MIPS64ELR6-NEXT:    or $9, $9, $11
+; MIPS64ELR6-NEXT:    and $9, $9, $3
+; MIPS64ELR6-NEXT:    and $10, $8, $6
+; MIPS64ELR6-NEXT:    or $10, $10, $9
+; MIPS64ELR6-NEXT:    sc $10, 0($1)
+; MIPS64ELR6-NEXT:    beqzc $10, .LBB7_1
+; MIPS64ELR6-NEXT:  # %bb.2: # %entry
+; MIPS64ELR6-NEXT:    and $7, $8, $3
+; MIPS64ELR6-NEXT:    srlv $7, $7, $2
+; MIPS64ELR6-NEXT:    seh $7, $7
+; MIPS64ELR6-NEXT:  # %bb.3: # %entry
+; MIPS64ELR6-NEXT:    sw $7, 12($sp) # 4-byte Folded Spill
+; MIPS64ELR6-NEXT:  # %bb.4: # %entry
+; MIPS64ELR6-NEXT:    sync
+; MIPS64ELR6-NEXT:    lw $2, 12($sp) # 4-byte Folded Reload
+; MIPS64ELR6-NEXT:    daddiu $sp, $sp, 16
+; MIPS64ELR6-NEXT:    jrc $ra
+entry:
+  %0 = atomicrmw umin i16* %ptr, i16 %val seq_cst
+  ret i16 %0
+}
+
+
+define i8 @test_max_8(i8* nocapture %ptr, i8 signext %val) {
+; MIPS-LABEL: test_max_8:
+; MIPS:       # %bb.0: # %entry
+; MIPS-NEXT:    addiu $sp, $sp, -8
+; MIPS-NEXT:    .cfi_def_cfa_offset 8
+; MIPS-NEXT:    move $1, $5
+; MIPS-NEXT:    sync
+; MIPS-NEXT:    addiu $2, $zero, -4
+; MIPS-NEXT:    and $2, $4, $2
+; MIPS-NEXT:    andi $3, $4, 3
+; MIPS-NEXT:    xori $3, $3, 3
+; MIPS-NEXT:    sll $3, $3, 3
+; MIPS-NEXT:    ori $4, $zero, 255
+; MIPS-NEXT:    sllv $4, $4, $3
+; MIPS-NEXT:    nor $6, $zero, $4
+; MIPS-NEXT:    sllv $5, $5, $3
+; MIPS-NEXT:  $BB8_1: # %entry
+; MIPS-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS-NEXT:    ll $8, 0($2)
+; MIPS-NEXT:    slt $11, $8, $5
+; MIPS-NEXT:    move $9, $8
+; MIPS-NEXT:    movn $9, $5, $11
+; MIPS-NEXT:    and $9, $9, $4
+; MIPS-NEXT:    and $10, $8, $6
+; MIPS-NEXT:    or $10, $10, $9
+; MIPS-NEXT:    sc $10, 0($2)
+; MIPS-NEXT:    beqz $10, $BB8_1
+; MIPS-NEXT:    nop
+; MIPS-NEXT:  # %bb.2: # %entry
+; MIPS-NEXT:    and $7, $8, $4
+; MIPS-NEXT:    srlv $7, $7, $3
+; MIPS-NEXT:    seh $7, $7
+; MIPS-NEXT:  # %bb.3: # %entry
+; MIPS-NEXT:    sw $7, 4($sp) # 4-byte Folded Spill
+; MIPS-NEXT:  # %bb.4: # %entry
+; MIPS-NEXT:    sync
+; MIPS-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MIPS-NEXT:    addiu $sp, $sp, 8
+; MIPS-NEXT:    jr $ra
+; MIPS-NEXT:    nop
+;
+; MIPSR6-LABEL: test_max_8:
+; MIPSR6:       # %bb.0: # %entry
+; MIPSR6-NEXT:    addiu $sp, $sp, -8
+; MIPSR6-NEXT:    .cfi_def_cfa_offset 8
+; MIPSR6-NEXT:    move $1, $5
+; MIPSR6-NEXT:    sync
+; MIPSR6-NEXT:    addiu $2, $zero, -4
+; MIPSR6-NEXT:    and $2, $4, $2
+; MIPSR6-NEXT:    andi $3, $4, 3
+; MIPSR6-NEXT:    xori $3, $3, 3
+; MIPSR6-NEXT:    sll $3, $3, 3
+; MIPSR6-NEXT:    ori $4, $zero, 255
+; MIPSR6-NEXT:    sllv $4, $4, $3
+; MIPSR6-NEXT:    nor $6, $zero, $4
+; MIPSR6-NEXT:    sllv $5, $5, $3
+; MIPSR6-NEXT:  $BB8_1: # %entry
+; MIPSR6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPSR6-NEXT:    ll $8, 0($2)
+; MIPSR6-NEXT:    slt $11, $8, $5
+; MIPSR6-NEXT:    seleqz $9, $8, $11
+; MIPSR6-NEXT:    selnez $11, $5, $11
+; MIPSR6-NEXT:    or $9, $9, $11
+; MIPSR6-NEXT:    and $9, $9, $4
+; MIPSR6-NEXT:    and $10, $8, $6
+; MIPSR6-NEXT:    or $10, $10, $9
+; MIPSR6-NEXT:    sc $10, 0($2)
+; MIPSR6-NEXT:    beqzc $10, $BB8_1
+; MIPSR6-NEXT:  # %bb.2: # %entry
+; MIPSR6-NEXT:    and $7, $8, $4
+; MIPSR6-NEXT:    srlv $7, $7, $3
+; MIPSR6-NEXT:    seh $7, $7
+; MIPSR6-NEXT:  # %bb.3: # %entry
+; MIPSR6-NEXT:    sw $7, 4($sp) # 4-byte Folded Spill
+; MIPSR6-NEXT:  # %bb.4: # %entry
+; MIPSR6-NEXT:    sync
+; MIPSR6-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MIPSR6-NEXT:    addiu $sp, $sp, 8
+; MIPSR6-NEXT:    jrc $ra
+;
+; MM-LABEL: test_max_8:
+; MM:       # %bb.0: # %entry
+; MM-NEXT:    addiu $sp, $sp, -8
+; MM-NEXT:    .cfi_def_cfa_offset 8
+; MM-NEXT:    move $1, $5
+; MM-NEXT:    sync
+; MM-NEXT:    addiu $2, $zero, -4
+; MM-NEXT:    and $2, $4, $2
+; MM-NEXT:    andi $3, $4, 3
+; MM-NEXT:    xori $3, $3, 3
+; MM-NEXT:    sll $3, $3, 3
+; MM-NEXT:    ori $4, $zero, 255
+; MM-NEXT:    sllv $4, $4, $3
+; MM-NEXT:    nor $6, $zero, $4
+; MM-NEXT:    sllv $5, $5, $3
+; MM-NEXT:  $BB8_1: # %entry
+; MM-NEXT:    # =>This Inner Loop Header: Depth=1
+; MM-NEXT:    ll $8, 0($2)
+; MM-NEXT:    slt $11, $8, $5
+; MM-NEXT:    or $9, $8, $zero
+; MM-NEXT:    movn $9, $5, $11
+; MM-NEXT:    and $9, $9, $4
+; MM-NEXT:    and $10, $8, $6
+; MM-NEXT:    or $10, $10, $9
+; MM-NEXT:    sc $10, 0($2)
+; MM-NEXT:    beqzc $10, $BB8_1
+; MM-NEXT:  # %bb.2: # %entry
+; MM-NEXT:    and $7, $8, $4
+; MM-NEXT:    srlv $7, $7, $3
+; MM-NEXT:    seh $7, $7
+; MM-NEXT:  # %bb.3: # %entry
+; MM-NEXT:    sw $7, 4($sp) # 4-byte Folded Spill
+; MM-NEXT:  # %bb.4: # %entry
+; MM-NEXT:    sync
+; MM-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MM-NEXT:    addiusp 8
+; MM-NEXT:    jrc $ra
+;
+; MMR6-LABEL: test_max_8:
+; MMR6:       # %bb.0: # %entry
+; MMR6-NEXT:    addiu $sp, $sp, -8
+; MMR6-NEXT:    .cfi_def_cfa_offset 8
+; MMR6-NEXT:    move $1, $5
+; MMR6-NEXT:    sync
+; MMR6-NEXT:    addiu $2, $zero, -4
+; MMR6-NEXT:    and $2, $4, $2
+; MMR6-NEXT:    andi $3, $4, 3
+; MMR6-NEXT:    xori $3, $3, 3
+; MMR6-NEXT:    sll $3, $3, 3
+; MMR6-NEXT:    ori $4, $zero, 255
+; MMR6-NEXT:    sllv $4, $4, $3
+; MMR6-NEXT:    nor $6, $zero, $4
+; MMR6-NEXT:    sllv $5, $5, $3
+; MMR6-NEXT:  $BB8_1: # %entry
+; MMR6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MMR6-NEXT:    ll $8, 0($2)
+; MMR6-NEXT:    slt $11, $8, $5
+; MMR6-NEXT:    seleqz $9, $8, $11
+; MMR6-NEXT:    selnez $11, $5, $11
+; MMR6-NEXT:    or $9, $9, $11
+; MMR6-NEXT:    and $9, $9, $4
+; MMR6-NEXT:    and $10, $8, $6
+; MMR6-NEXT:    or $10, $10, $9
+; MMR6-NEXT:    sc $10, 0($2)
+; MMR6-NEXT:    beqc $10, $zero, $BB8_1
+; MMR6-NEXT:  # %bb.2: # %entry
+; MMR6-NEXT:    and $7, $8, $4
+; MMR6-NEXT:    srlv $7, $7, $3
+; MMR6-NEXT:    seh $7, $7
+; MMR6-NEXT:  # %bb.3: # %entry
+; MMR6-NEXT:    sw $7, 4($sp) # 4-byte Folded Spill
+; MMR6-NEXT:  # %bb.4: # %entry
+; MMR6-NEXT:    sync
+; MMR6-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MMR6-NEXT:    addiu $sp, $sp, 8
+; MMR6-NEXT:    jrc $ra
+;
+; MIPSEL-LABEL: test_max_8:
+; MIPSEL:       # %bb.0: # %entry
+; MIPSEL-NEXT:    addiu $sp, $sp, -8
+; MIPSEL-NEXT:    .cfi_def_cfa_offset 8
+; MIPSEL-NEXT:    move $1, $5
+; MIPSEL-NEXT:    sync
+; MIPSEL-NEXT:    addiu $2, $zero, -4
+; MIPSEL-NEXT:    and $2, $4, $2
+; MIPSEL-NEXT:    andi $3, $4, 3
+; MIPSEL-NEXT:    sll $3, $3, 3
+; MIPSEL-NEXT:    ori $4, $zero, 255
+; MIPSEL-NEXT:    sllv $4, $4, $3
+; MIPSEL-NEXT:    nor $6, $zero, $4
+; MIPSEL-NEXT:    sllv $5, $5, $3
+; MIPSEL-NEXT:  $BB8_1: # %entry
+; MIPSEL-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPSEL-NEXT:    ll $8, 0($2)
+; MIPSEL-NEXT:    and $8, $8, $4
+; MIPSEL-NEXT:    and $5, $5, $4
+; MIPSEL-NEXT:    slt $11, $8, $5
+; MIPSEL-NEXT:    move $9, $8
+; MIPSEL-NEXT:    movn $9, $5, $11
+; MIPSEL-NEXT:    and $9, $9, $4
+; MIPSEL-NEXT:    and $10, $8, $6
+; MIPSEL-NEXT:    or $10, $10, $9
+; MIPSEL-NEXT:    sc $10, 0($2)
+; MIPSEL-NEXT:    beqz $10, $BB8_1
+; MIPSEL-NEXT:    nop
+; MIPSEL-NEXT:  # %bb.2: # %entry
+; MIPSEL-NEXT:    and $7, $8, $4
+; MIPSEL-NEXT:    srlv $7, $7, $3
+; MIPSEL-NEXT:    seh $7, $7
+; MIPSEL-NEXT:  # %bb.3: # %entry
+; MIPSEL-NEXT:    sw $7, 4($sp) # 4-byte Folded Spill
+; MIPSEL-NEXT:  # %bb.4: # %entry
+; MIPSEL-NEXT:    sync
+; MIPSEL-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MIPSEL-NEXT:    addiu $sp, $sp, 8
+; MIPSEL-NEXT:    jr $ra
+; MIPSEL-NEXT:    nop
+;
+; MIPSELR6-LABEL: test_max_8:
+; MIPSELR6:       # %bb.0: # %entry
+; MIPSELR6-NEXT:    addiu $sp, $sp, -8
+; MIPSELR6-NEXT:    .cfi_def_cfa_offset 8
+; MIPSELR6-NEXT:    move $1, $5
+; MIPSELR6-NEXT:    sync
+; MIPSELR6-NEXT:    addiu $2, $zero, -4
+; MIPSELR6-NEXT:    and $2, $4, $2
+; MIPSELR6-NEXT:    andi $3, $4, 3
+; MIPSELR6-NEXT:    sll $3, $3, 3
+; MIPSELR6-NEXT:    ori $4, $zero, 255
+; MIPSELR6-NEXT:    sllv $4, $4, $3
+; MIPSELR6-NEXT:    nor $6, $zero, $4
+; MIPSELR6-NEXT:    sllv $5, $5, $3
+; MIPSELR6-NEXT:  $BB8_1: # %entry
+; MIPSELR6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPSELR6-NEXT:    ll $8, 0($2)
+; MIPSELR6-NEXT:    and $8, $8, $4
+; MIPSELR6-NEXT:    and $5, $5, $4
+; MIPSELR6-NEXT:    slt $11, $8, $5
+; MIPSELR6-NEXT:    seleqz $9, $8, $11
+; MIPSELR6-NEXT:    selnez $11, $5, $11
+; MIPSELR6-NEXT:    or $9, $9, $11
+; MIPSELR6-NEXT:    and $9, $9, $4
+; MIPSELR6-NEXT:    and $10, $8, $6
+; MIPSELR6-NEXT:    or $10, $10, $9
+; MIPSELR6-NEXT:    sc $10, 0($2)
+; MIPSELR6-NEXT:    beqzc $10, $BB8_1
+; MIPSELR6-NEXT:  # %bb.2: # %entry
+; MIPSELR6-NEXT:    and $7, $8, $4
+; MIPSELR6-NEXT:    srlv $7, $7, $3
+; MIPSELR6-NEXT:    seh $7, $7
+; MIPSELR6-NEXT:  # %bb.3: # %entry
+; MIPSELR6-NEXT:    sw $7, 4($sp) # 4-byte Folded Spill
+; MIPSELR6-NEXT:  # %bb.4: # %entry
+; MIPSELR6-NEXT:    sync
+; MIPSELR6-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MIPSELR6-NEXT:    addiu $sp, $sp, 8
+; MIPSELR6-NEXT:    jrc $ra
+;
+; MMEL-LABEL: test_max_8:
+; MMEL:       # %bb.0: # %entry
+; MMEL-NEXT:    addiu $sp, $sp, -8
+; MMEL-NEXT:    .cfi_def_cfa_offset 8
+; MMEL-NEXT:    move $1, $5
+; MMEL-NEXT:    sync
+; MMEL-NEXT:    addiu $2, $zero, -4
+; MMEL-NEXT:    and $2, $4, $2
+; MMEL-NEXT:    andi $3, $4, 3
+; MMEL-NEXT:    sll $3, $3, 3
+; MMEL-NEXT:    ori $4, $zero, 255
+; MMEL-NEXT:    sllv $4, $4, $3
+; MMEL-NEXT:    nor $6, $zero, $4
+; MMEL-NEXT:    sllv $5, $5, $3
+; MMEL-NEXT:  $BB8_1: # %entry
+; MMEL-NEXT:    # =>This Inner Loop Header: Depth=1
+; MMEL-NEXT:    ll $8, 0($2)
+; MMEL-NEXT:    and $8, $8, $4
+; MMEL-NEXT:    and $5, $5, $4
+; MMEL-NEXT:    slt $11, $8, $5
+; MMEL-NEXT:    or $9, $8, $zero
+; MMEL-NEXT:    movn $9, $5, $11
+; MMEL-NEXT:    and $9, $9, $4
+; MMEL-NEXT:    and $10, $8, $6
+; MMEL-NEXT:    or $10, $10, $9
+; MMEL-NEXT:    sc $10, 0($2)
+; MMEL-NEXT:    beqzc $10, $BB8_1
+; MMEL-NEXT:  # %bb.2: # %entry
+; MMEL-NEXT:    and $7, $8, $4
+; MMEL-NEXT:    srlv $7, $7, $3
+; MMEL-NEXT:    seh $7, $7
+; MMEL-NEXT:  # %bb.3: # %entry
+; MMEL-NEXT:    sw $7, 4($sp) # 4-byte Folded Spill
+; MMEL-NEXT:  # %bb.4: # %entry
+; MMEL-NEXT:    sync
+; MMEL-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MMEL-NEXT:    addiusp 8
+; MMEL-NEXT:    jrc $ra
+;
+; MMELR6-LABEL: test_max_8:
+; MMELR6:       # %bb.0: # %entry
+; MMELR6-NEXT:    addiu $sp, $sp, -8
+; MMELR6-NEXT:    .cfi_def_cfa_offset 8
+; MMELR6-NEXT:    move $1, $5
+; MMELR6-NEXT:    sync
+; MMELR6-NEXT:    addiu $2, $zero, -4
+; MMELR6-NEXT:    and $2, $4, $2
+; MMELR6-NEXT:    andi $3, $4, 3
+; MMELR6-NEXT:    sll $3, $3, 3
+; MMELR6-NEXT:    ori $4, $zero, 255
+; MMELR6-NEXT:    sllv $4, $4, $3
+; MMELR6-NEXT:    nor $6, $zero, $4
+; MMELR6-NEXT:    sllv $5, $5, $3
+; MMELR6-NEXT:  $BB8_1: # %entry
+; MMELR6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MMELR6-NEXT:    ll $8, 0($2)
+; MMELR6-NEXT:    and $8, $8, $4
+; MMELR6-NEXT:    and $5, $5, $4
+; MMELR6-NEXT:    slt $11, $8, $5
+; MMELR6-NEXT:    seleqz $9, $8, $11
+; MMELR6-NEXT:    selnez $11, $5, $11
+; MMELR6-NEXT:    or $9, $9, $11
+; MMELR6-NEXT:    and $9, $9, $4
+; MMELR6-NEXT:    and $10, $8, $6
+; MMELR6-NEXT:    or $10, $10, $9
+; MMELR6-NEXT:    sc $10, 0($2)
+; MMELR6-NEXT:    beqc $10, $zero, $BB8_1
+; MMELR6-NEXT:  # %bb.2: # %entry
+; MMELR6-NEXT:    and $7, $8, $4
+; MMELR6-NEXT:    srlv $7, $7, $3
+; MMELR6-NEXT:    seh $7, $7
+; MMELR6-NEXT:  # %bb.3: # %entry
+; MMELR6-NEXT:    sw $7, 4($sp) # 4-byte Folded Spill
+; MMELR6-NEXT:  # %bb.4: # %entry
+; MMELR6-NEXT:    sync
+; MMELR6-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MMELR6-NEXT:    addiu $sp, $sp, 8
+; MMELR6-NEXT:    jrc $ra
+;
+; MIPS64-LABEL: test_max_8:
+; MIPS64:       # %bb.0: # %entry
+; MIPS64-NEXT:    daddiu $sp, $sp, -16
+; MIPS64-NEXT:    .cfi_def_cfa_offset 16
+; MIPS64-NEXT:    # kill: def $a1 killed $a1 killed $a1_64
+; MIPS64-NEXT:    sync
+; MIPS64-NEXT:    daddiu $1, $zero, -4
+; MIPS64-NEXT:    and $1, $4, $1
+; MIPS64-NEXT:    andi $2, $4, 3
+; MIPS64-NEXT:    xori $2, $2, 3
+; MIPS64-NEXT:    sll $2, $2, 3
+; MIPS64-NEXT:    ori $3, $zero, 255
+; MIPS64-NEXT:    sllv $3, $3, $2
+; MIPS64-NEXT:    nor $6, $zero, $3
+; MIPS64-NEXT:    sllv $5, $5, $2
+; MIPS64-NEXT:  .LBB8_1: # %entry
+; MIPS64-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS64-NEXT:    ll $8, 0($1)
+; MIPS64-NEXT:    slt $11, $8, $5
+; MIPS64-NEXT:    move $9, $8
+; MIPS64-NEXT:    movn $9, $5, $11
+; MIPS64-NEXT:    and $9, $9, $3
+; MIPS64-NEXT:    and $10, $8, $6
+; MIPS64-NEXT:    or $10, $10, $9
+; MIPS64-NEXT:    sc $10, 0($1)
+; MIPS64-NEXT:    beqz $10, .LBB8_1
+; MIPS64-NEXT:    nop
+; MIPS64-NEXT:  # %bb.2: # %entry
+; MIPS64-NEXT:    and $7, $8, $3
+; MIPS64-NEXT:    srlv $7, $7, $2
+; MIPS64-NEXT:    seh $7, $7
+; MIPS64-NEXT:  # %bb.3: # %entry
+; MIPS64-NEXT:    sw $7, 12($sp) # 4-byte Folded Spill
+; MIPS64-NEXT:  # %bb.4: # %entry
+; MIPS64-NEXT:    sync
+; MIPS64-NEXT:    lw $2, 12($sp) # 4-byte Folded Reload
+; MIPS64-NEXT:    daddiu $sp, $sp, 16
+; MIPS64-NEXT:    jr $ra
+; MIPS64-NEXT:    nop
+;
+; MIPS64R6-LABEL: test_max_8:
+; MIPS64R6:       # %bb.0: # %entry
+; MIPS64R6-NEXT:    daddiu $sp, $sp, -16
+; MIPS64R6-NEXT:    .cfi_def_cfa_offset 16
+; MIPS64R6-NEXT:    # kill: def $a1 killed $a1 killed $a1_64
+; MIPS64R6-NEXT:    sync
+; MIPS64R6-NEXT:    daddiu $1, $zero, -4
+; MIPS64R6-NEXT:    and $1, $4, $1
+; MIPS64R6-NEXT:    andi $2, $4, 3
+; MIPS64R6-NEXT:    xori $2, $2, 3
+; MIPS64R6-NEXT:    sll $2, $2, 3
+; MIPS64R6-NEXT:    ori $3, $zero, 255
+; MIPS64R6-NEXT:    sllv $3, $3, $2
+; MIPS64R6-NEXT:    nor $6, $zero, $3
+; MIPS64R6-NEXT:    sllv $5, $5, $2
+; MIPS64R6-NEXT:  .LBB8_1: # %entry
+; MIPS64R6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS64R6-NEXT:    ll $8, 0($1)
+; MIPS64R6-NEXT:    slt $11, $8, $5
+; MIPS64R6-NEXT:    seleqz $9, $8, $11
+; MIPS64R6-NEXT:    selnez $11, $5, $11
+; MIPS64R6-NEXT:    or $9, $9, $11
+; MIPS64R6-NEXT:    and $9, $9, $3
+; MIPS64R6-NEXT:    and $10, $8, $6
+; MIPS64R6-NEXT:    or $10, $10, $9
+; MIPS64R6-NEXT:    sc $10, 0($1)
+; MIPS64R6-NEXT:    beqzc $10, .LBB8_1
+; MIPS64R6-NEXT:  # %bb.2: # %entry
+; MIPS64R6-NEXT:    and $7, $8, $3
+; MIPS64R6-NEXT:    srlv $7, $7, $2
+; MIPS64R6-NEXT:    seh $7, $7
+; MIPS64R6-NEXT:  # %bb.3: # %entry
+; MIPS64R6-NEXT:    sw $7, 12($sp) # 4-byte Folded Spill
+; MIPS64R6-NEXT:  # %bb.4: # %entry
+; MIPS64R6-NEXT:    sync
+; MIPS64R6-NEXT:    lw $2, 12($sp) # 4-byte Folded Reload
+; MIPS64R6-NEXT:    daddiu $sp, $sp, 16
+; MIPS64R6-NEXT:    jrc $ra
+;
+; MIPS64EL-LABEL: test_max_8:
+; MIPS64EL:       # %bb.0: # %entry
+; MIPS64EL-NEXT:    daddiu $sp, $sp, -16
+; MIPS64EL-NEXT:    .cfi_def_cfa_offset 16
+; MIPS64EL-NEXT:    # kill: def $a1 killed $a1 killed $a1_64
+; MIPS64EL-NEXT:    sync
+; MIPS64EL-NEXT:    daddiu $1, $zero, -4
+; MIPS64EL-NEXT:    and $1, $4, $1
+; MIPS64EL-NEXT:    andi $2, $4, 3
+; MIPS64EL-NEXT:    sll $2, $2, 3
+; MIPS64EL-NEXT:    ori $3, $zero, 255
+; MIPS64EL-NEXT:    sllv $3, $3, $2
+; MIPS64EL-NEXT:    nor $6, $zero, $3
+; MIPS64EL-NEXT:    sllv $5, $5, $2
+; MIPS64EL-NEXT:  .LBB8_1: # %entry
+; MIPS64EL-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS64EL-NEXT:    ll $8, 0($1)
+; MIPS64EL-NEXT:    and $8, $8, $3
+; MIPS64EL-NEXT:    and $5, $5, $3
+; MIPS64EL-NEXT:    slt $11, $8, $5
+; MIPS64EL-NEXT:    move $9, $8
+; MIPS64EL-NEXT:    movn $9, $5, $11
+; MIPS64EL-NEXT:    and $9, $9, $3
+; MIPS64EL-NEXT:    and $10, $8, $6
+; MIPS64EL-NEXT:    or $10, $10, $9
+; MIPS64EL-NEXT:    sc $10, 0($1)
+; MIPS64EL-NEXT:    beqz $10, .LBB8_1
+; MIPS64EL-NEXT:    nop
+; MIPS64EL-NEXT:  # %bb.2: # %entry
+; MIPS64EL-NEXT:    and $7, $8, $3
+; MIPS64EL-NEXT:    srlv $7, $7, $2
+; MIPS64EL-NEXT:    seh $7, $7
+; MIPS64EL-NEXT:  # %bb.3: # %entry
+; MIPS64EL-NEXT:    sw $7, 12($sp) # 4-byte Folded Spill
+; MIPS64EL-NEXT:  # %bb.4: # %entry
+; MIPS64EL-NEXT:    sync
+; MIPS64EL-NEXT:    lw $2, 12($sp) # 4-byte Folded Reload
+; MIPS64EL-NEXT:    daddiu $sp, $sp, 16
+; MIPS64EL-NEXT:    jr $ra
+; MIPS64EL-NEXT:    nop
+;
+; MIPS64ELR6-LABEL: test_max_8:
+; MIPS64ELR6:       # %bb.0: # %entry
+; MIPS64ELR6-NEXT:    daddiu $sp, $sp, -16
+; MIPS64ELR6-NEXT:    .cfi_def_cfa_offset 16
+; MIPS64ELR6-NEXT:    # kill: def $a1 killed $a1 killed $a1_64
+; MIPS64ELR6-NEXT:    sync
+; MIPS64ELR6-NEXT:    daddiu $1, $zero, -4
+; MIPS64ELR6-NEXT:    and $1, $4, $1
+; MIPS64ELR6-NEXT:    andi $2, $4, 3
+; MIPS64ELR6-NEXT:    sll $2, $2, 3
+; MIPS64ELR6-NEXT:    ori $3, $zero, 255
+; MIPS64ELR6-NEXT:    sllv $3, $3, $2
+; MIPS64ELR6-NEXT:    nor $6, $zero, $3
+; MIPS64ELR6-NEXT:    sllv $5, $5, $2
+; MIPS64ELR6-NEXT:  .LBB8_1: # %entry
+; MIPS64ELR6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS64ELR6-NEXT:    ll $8, 0($1)
+; MIPS64ELR6-NEXT:    and $8, $8, $3
+; MIPS64ELR6-NEXT:    and $5, $5, $3
+; MIPS64ELR6-NEXT:    slt $11, $8, $5
+; MIPS64ELR6-NEXT:    seleqz $9, $8, $11
+; MIPS64ELR6-NEXT:    selnez $11, $5, $11
+; MIPS64ELR6-NEXT:    or $9, $9, $11
+; MIPS64ELR6-NEXT:    and $9, $9, $3
+; MIPS64ELR6-NEXT:    and $10, $8, $6
+; MIPS64ELR6-NEXT:    or $10, $10, $9
+; MIPS64ELR6-NEXT:    sc $10, 0($1)
+; MIPS64ELR6-NEXT:    beqzc $10, .LBB8_1
+; MIPS64ELR6-NEXT:  # %bb.2: # %entry
+; MIPS64ELR6-NEXT:    and $7, $8, $3
+; MIPS64ELR6-NEXT:    srlv $7, $7, $2
+; MIPS64ELR6-NEXT:    seh $7, $7
+; MIPS64ELR6-NEXT:  # %bb.3: # %entry
+; MIPS64ELR6-NEXT:    sw $7, 12($sp) # 4-byte Folded Spill
+; MIPS64ELR6-NEXT:  # %bb.4: # %entry
+; MIPS64ELR6-NEXT:    sync
+; MIPS64ELR6-NEXT:    lw $2, 12($sp) # 4-byte Folded Reload
+; MIPS64ELR6-NEXT:    daddiu $sp, $sp, 16
+; MIPS64ELR6-NEXT:    jrc $ra
+entry:
+  %0 = atomicrmw max i8* %ptr, i8 %val seq_cst
+  ret i8 %0
+}
+
+define i8 @test_min_8(i8* nocapture %ptr, i8 signext %val) {
+; MIPS-LABEL: test_min_8:
+; MIPS:       # %bb.0: # %entry
+; MIPS-NEXT:    addiu $sp, $sp, -8
+; MIPS-NEXT:    .cfi_def_cfa_offset 8
+; MIPS-NEXT:    move $1, $5
+; MIPS-NEXT:    sync
+; MIPS-NEXT:    addiu $2, $zero, -4
+; MIPS-NEXT:    and $2, $4, $2
+; MIPS-NEXT:    andi $3, $4, 3
+; MIPS-NEXT:    xori $3, $3, 3
+; MIPS-NEXT:    sll $3, $3, 3
+; MIPS-NEXT:    ori $4, $zero, 255
+; MIPS-NEXT:    sllv $4, $4, $3
+; MIPS-NEXT:    nor $6, $zero, $4
+; MIPS-NEXT:    sllv $5, $5, $3
+; MIPS-NEXT:  $BB9_1: # %entry
+; MIPS-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS-NEXT:    ll $8, 0($2)
+; MIPS-NEXT:    slt $11, $8, $5
+; MIPS-NEXT:    move $9, $8
+; MIPS-NEXT:    movz $9, $5, $11
+; MIPS-NEXT:    and $9, $9, $4
+; MIPS-NEXT:    and $10, $8, $6
+; MIPS-NEXT:    or $10, $10, $9
+; MIPS-NEXT:    sc $10, 0($2)
+; MIPS-NEXT:    beqz $10, $BB9_1
+; MIPS-NEXT:    nop
+; MIPS-NEXT:  # %bb.2: # %entry
+; MIPS-NEXT:    and $7, $8, $4
+; MIPS-NEXT:    srlv $7, $7, $3
+; MIPS-NEXT:    seh $7, $7
+; MIPS-NEXT:  # %bb.3: # %entry
+; MIPS-NEXT:    sw $7, 4($sp) # 4-byte Folded Spill
+; MIPS-NEXT:  # %bb.4: # %entry
+; MIPS-NEXT:    sync
+; MIPS-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MIPS-NEXT:    addiu $sp, $sp, 8
+; MIPS-NEXT:    jr $ra
+; MIPS-NEXT:    nop
+;
+; MIPSR6-LABEL: test_min_8:
+; MIPSR6:       # %bb.0: # %entry
+; MIPSR6-NEXT:    addiu $sp, $sp, -8
+; MIPSR6-NEXT:    .cfi_def_cfa_offset 8
+; MIPSR6-NEXT:    move $1, $5
+; MIPSR6-NEXT:    sync
+; MIPSR6-NEXT:    addiu $2, $zero, -4
+; MIPSR6-NEXT:    and $2, $4, $2
+; MIPSR6-NEXT:    andi $3, $4, 3
+; MIPSR6-NEXT:    xori $3, $3, 3
+; MIPSR6-NEXT:    sll $3, $3, 3
+; MIPSR6-NEXT:    ori $4, $zero, 255
+; MIPSR6-NEXT:    sllv $4, $4, $3
+; MIPSR6-NEXT:    nor $6, $zero, $4
+; MIPSR6-NEXT:    sllv $5, $5, $3
+; MIPSR6-NEXT:  $BB9_1: # %entry
+; MIPSR6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPSR6-NEXT:    ll $8, 0($2)
+; MIPSR6-NEXT:    slt $11, $8, $5
+; MIPSR6-NEXT:    selnez $9, $8, $11
+; MIPSR6-NEXT:    seleqz $11, $5, $11
+; MIPSR6-NEXT:    or $9, $9, $11
+; MIPSR6-NEXT:    and $9, $9, $4
+; MIPSR6-NEXT:    and $10, $8, $6
+; MIPSR6-NEXT:    or $10, $10, $9
+; MIPSR6-NEXT:    sc $10, 0($2)
+; MIPSR6-NEXT:    beqzc $10, $BB9_1
+; MIPSR6-NEXT:  # %bb.2: # %entry
+; MIPSR6-NEXT:    and $7, $8, $4
+; MIPSR6-NEXT:    srlv $7, $7, $3
+; MIPSR6-NEXT:    seh $7, $7
+; MIPSR6-NEXT:  # %bb.3: # %entry
+; MIPSR6-NEXT:    sw $7, 4($sp) # 4-byte Folded Spill
+; MIPSR6-NEXT:  # %bb.4: # %entry
+; MIPSR6-NEXT:    sync
+; MIPSR6-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MIPSR6-NEXT:    addiu $sp, $sp, 8
+; MIPSR6-NEXT:    jrc $ra
+;
+; MM-LABEL: test_min_8:
+; MM:       # %bb.0: # %entry
+; MM-NEXT:    addiu $sp, $sp, -8
+; MM-NEXT:    .cfi_def_cfa_offset 8
+; MM-NEXT:    move $1, $5
+; MM-NEXT:    sync
+; MM-NEXT:    addiu $2, $zero, -4
+; MM-NEXT:    and $2, $4, $2
+; MM-NEXT:    andi $3, $4, 3
+; MM-NEXT:    xori $3, $3, 3
+; MM-NEXT:    sll $3, $3, 3
+; MM-NEXT:    ori $4, $zero, 255
+; MM-NEXT:    sllv $4, $4, $3
+; MM-NEXT:    nor $6, $zero, $4
+; MM-NEXT:    sllv $5, $5, $3
+; MM-NEXT:  $BB9_1: # %entry
+; MM-NEXT:    # =>This Inner Loop Header: Depth=1
+; MM-NEXT:    ll $8, 0($2)
+; MM-NEXT:    slt $11, $8, $5
+; MM-NEXT:    or $9, $8, $zero
+; MM-NEXT:    movz $9, $5, $11
+; MM-NEXT:    and $9, $9, $4
+; MM-NEXT:    and $10, $8, $6
+; MM-NEXT:    or $10, $10, $9
+; MM-NEXT:    sc $10, 0($2)
+; MM-NEXT:    beqzc $10, $BB9_1
+; MM-NEXT:  # %bb.2: # %entry
+; MM-NEXT:    and $7, $8, $4
+; MM-NEXT:    srlv $7, $7, $3
+; MM-NEXT:    seh $7, $7
+; MM-NEXT:  # %bb.3: # %entry
+; MM-NEXT:    sw $7, 4($sp) # 4-byte Folded Spill
+; MM-NEXT:  # %bb.4: # %entry
+; MM-NEXT:    sync
+; MM-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MM-NEXT:    addiusp 8
+; MM-NEXT:    jrc $ra
+;
+; MMR6-LABEL: test_min_8:
+; MMR6:       # %bb.0: # %entry
+; MMR6-NEXT:    addiu $sp, $sp, -8
+; MMR6-NEXT:    .cfi_def_cfa_offset 8
+; MMR6-NEXT:    move $1, $5
+; MMR6-NEXT:    sync
+; MMR6-NEXT:    addiu $2, $zero, -4
+; MMR6-NEXT:    and $2, $4, $2
+; MMR6-NEXT:    andi $3, $4, 3
+; MMR6-NEXT:    xori $3, $3, 3
+; MMR6-NEXT:    sll $3, $3, 3
+; MMR6-NEXT:    ori $4, $zero, 255
+; MMR6-NEXT:    sllv $4, $4, $3
+; MMR6-NEXT:    nor $6, $zero, $4
+; MMR6-NEXT:    sllv $5, $5, $3
+; MMR6-NEXT:  $BB9_1: # %entry
+; MMR6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MMR6-NEXT:    ll $8, 0($2)
+; MMR6-NEXT:    slt $11, $8, $5
+; MMR6-NEXT:    selnez $9, $8, $11
+; MMR6-NEXT:    seleqz $11, $5, $11
+; MMR6-NEXT:    or $9, $9, $11
+; MMR6-NEXT:    and $9, $9, $4
+; MMR6-NEXT:    and $10, $8, $6
+; MMR6-NEXT:    or $10, $10, $9
+; MMR6-NEXT:    sc $10, 0($2)
+; MMR6-NEXT:    beqc $10, $zero, $BB9_1
+; MMR6-NEXT:  # %bb.2: # %entry
+; MMR6-NEXT:    and $7, $8, $4
+; MMR6-NEXT:    srlv $7, $7, $3
+; MMR6-NEXT:    seh $7, $7
+; MMR6-NEXT:  # %bb.3: # %entry
+; MMR6-NEXT:    sw $7, 4($sp) # 4-byte Folded Spill
+; MMR6-NEXT:  # %bb.4: # %entry
+; MMR6-NEXT:    sync
+; MMR6-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MMR6-NEXT:    addiu $sp, $sp, 8
+; MMR6-NEXT:    jrc $ra
+;
+; MIPSEL-LABEL: test_min_8:
+; MIPSEL:       # %bb.0: # %entry
+; MIPSEL-NEXT:    addiu $sp, $sp, -8
+; MIPSEL-NEXT:    .cfi_def_cfa_offset 8
+; MIPSEL-NEXT:    move $1, $5
+; MIPSEL-NEXT:    sync
+; MIPSEL-NEXT:    addiu $2, $zero, -4
+; MIPSEL-NEXT:    and $2, $4, $2
+; MIPSEL-NEXT:    andi $3, $4, 3
+; MIPSEL-NEXT:    sll $3, $3, 3
+; MIPSEL-NEXT:    ori $4, $zero, 255
+; MIPSEL-NEXT:    sllv $4, $4, $3
+; MIPSEL-NEXT:    nor $6, $zero, $4
+; MIPSEL-NEXT:    sllv $5, $5, $3
+; MIPSEL-NEXT:  $BB9_1: # %entry
+; MIPSEL-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPSEL-NEXT:    ll $8, 0($2)
+; MIPSEL-NEXT:    and $8, $8, $4
+; MIPSEL-NEXT:    and $5, $5, $4
+; MIPSEL-NEXT:    slt $11, $8, $5
+; MIPSEL-NEXT:    move $9, $8
+; MIPSEL-NEXT:    movz $9, $5, $11
+; MIPSEL-NEXT:    and $9, $9, $4
+; MIPSEL-NEXT:    and $10, $8, $6
+; MIPSEL-NEXT:    or $10, $10, $9
+; MIPSEL-NEXT:    sc $10, 0($2)
+; MIPSEL-NEXT:    beqz $10, $BB9_1
+; MIPSEL-NEXT:    nop
+; MIPSEL-NEXT:  # %bb.2: # %entry
+; MIPSEL-NEXT:    and $7, $8, $4
+; MIPSEL-NEXT:    srlv $7, $7, $3
+; MIPSEL-NEXT:    seh $7, $7
+; MIPSEL-NEXT:  # %bb.3: # %entry
+; MIPSEL-NEXT:    sw $7, 4($sp) # 4-byte Folded Spill
+; MIPSEL-NEXT:  # %bb.4: # %entry
+; MIPSEL-NEXT:    sync
+; MIPSEL-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MIPSEL-NEXT:    addiu $sp, $sp, 8
+; MIPSEL-NEXT:    jr $ra
+; MIPSEL-NEXT:    nop
+;
+; MIPSELR6-LABEL: test_min_8:
+; MIPSELR6:       # %bb.0: # %entry
+; MIPSELR6-NEXT:    addiu $sp, $sp, -8
+; MIPSELR6-NEXT:    .cfi_def_cfa_offset 8
+; MIPSELR6-NEXT:    move $1, $5
+; MIPSELR6-NEXT:    sync
+; MIPSELR6-NEXT:    addiu $2, $zero, -4
+; MIPSELR6-NEXT:    and $2, $4, $2
+; MIPSELR6-NEXT:    andi $3, $4, 3
+; MIPSELR6-NEXT:    sll $3, $3, 3
+; MIPSELR6-NEXT:    ori $4, $zero, 255
+; MIPSELR6-NEXT:    sllv $4, $4, $3
+; MIPSELR6-NEXT:    nor $6, $zero, $4
+; MIPSELR6-NEXT:    sllv $5, $5, $3
+; MIPSELR6-NEXT:  $BB9_1: # %entry
+; MIPSELR6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPSELR6-NEXT:    ll $8, 0($2)
+; MIPSELR6-NEXT:    and $8, $8, $4
+; MIPSELR6-NEXT:    and $5, $5, $4
+; MIPSELR6-NEXT:    slt $11, $8, $5
+; MIPSELR6-NEXT:    selnez $9, $8, $11
+; MIPSELR6-NEXT:    seleqz $11, $5, $11
+; MIPSELR6-NEXT:    or $9, $9, $11
+; MIPSELR6-NEXT:    and $9, $9, $4
+; MIPSELR6-NEXT:    and $10, $8, $6
+; MIPSELR6-NEXT:    or $10, $10, $9
+; MIPSELR6-NEXT:    sc $10, 0($2)
+; MIPSELR6-NEXT:    beqzc $10, $BB9_1
+; MIPSELR6-NEXT:  # %bb.2: # %entry
+; MIPSELR6-NEXT:    and $7, $8, $4
+; MIPSELR6-NEXT:    srlv $7, $7, $3
+; MIPSELR6-NEXT:    seh $7, $7
+; MIPSELR6-NEXT:  # %bb.3: # %entry
+; MIPSELR6-NEXT:    sw $7, 4($sp) # 4-byte Folded Spill
+; MIPSELR6-NEXT:  # %bb.4: # %entry
+; MIPSELR6-NEXT:    sync
+; MIPSELR6-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MIPSELR6-NEXT:    addiu $sp, $sp, 8
+; MIPSELR6-NEXT:    jrc $ra
+;
+; MMEL-LABEL: test_min_8:
+; MMEL:       # %bb.0: # %entry
+; MMEL-NEXT:    addiu $sp, $sp, -8
+; MMEL-NEXT:    .cfi_def_cfa_offset 8
+; MMEL-NEXT:    move $1, $5
+; MMEL-NEXT:    sync
+; MMEL-NEXT:    addiu $2, $zero, -4
+; MMEL-NEXT:    and $2, $4, $2
+; MMEL-NEXT:    andi $3, $4, 3
+; MMEL-NEXT:    sll $3, $3, 3
+; MMEL-NEXT:    ori $4, $zero, 255
+; MMEL-NEXT:    sllv $4, $4, $3
+; MMEL-NEXT:    nor $6, $zero, $4
+; MMEL-NEXT:    sllv $5, $5, $3
+; MMEL-NEXT:  $BB9_1: # %entry
+; MMEL-NEXT:    # =>This Inner Loop Header: Depth=1
+; MMEL-NEXT:    ll $8, 0($2)
+; MMEL-NEXT:    and $8, $8, $4
+; MMEL-NEXT:    and $5, $5, $4
+; MMEL-NEXT:    slt $11, $8, $5
+; MMEL-NEXT:    or $9, $8, $zero
+; MMEL-NEXT:    movz $9, $5, $11
+; MMEL-NEXT:    and $9, $9, $4
+; MMEL-NEXT:    and $10, $8, $6
+; MMEL-NEXT:    or $10, $10, $9
+; MMEL-NEXT:    sc $10, 0($2)
+; MMEL-NEXT:    beqzc $10, $BB9_1
+; MMEL-NEXT:  # %bb.2: # %entry
+; MMEL-NEXT:    and $7, $8, $4
+; MMEL-NEXT:    srlv $7, $7, $3
+; MMEL-NEXT:    seh $7, $7
+; MMEL-NEXT:  # %bb.3: # %entry
+; MMEL-NEXT:    sw $7, 4($sp) # 4-byte Folded Spill
+; MMEL-NEXT:  # %bb.4: # %entry
+; MMEL-NEXT:    sync
+; MMEL-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MMEL-NEXT:    addiusp 8
+; MMEL-NEXT:    jrc $ra
+;
+; MMELR6-LABEL: test_min_8:
+; MMELR6:       # %bb.0: # %entry
+; MMELR6-NEXT:    addiu $sp, $sp, -8
+; MMELR6-NEXT:    .cfi_def_cfa_offset 8
+; MMELR6-NEXT:    move $1, $5
+; MMELR6-NEXT:    sync
+; MMELR6-NEXT:    addiu $2, $zero, -4
+; MMELR6-NEXT:    and $2, $4, $2
+; MMELR6-NEXT:    andi $3, $4, 3
+; MMELR6-NEXT:    sll $3, $3, 3
+; MMELR6-NEXT:    ori $4, $zero, 255
+; MMELR6-NEXT:    sllv $4, $4, $3
+; MMELR6-NEXT:    nor $6, $zero, $4
+; MMELR6-NEXT:    sllv $5, $5, $3
+; MMELR6-NEXT:  $BB9_1: # %entry
+; MMELR6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MMELR6-NEXT:    ll $8, 0($2)
+; MMELR6-NEXT:    and $8, $8, $4
+; MMELR6-NEXT:    and $5, $5, $4
+; MMELR6-NEXT:    slt $11, $8, $5
+; MMELR6-NEXT:    selnez $9, $8, $11
+; MMELR6-NEXT:    seleqz $11, $5, $11
+; MMELR6-NEXT:    or $9, $9, $11
+; MMELR6-NEXT:    and $9, $9, $4
+; MMELR6-NEXT:    and $10, $8, $6
+; MMELR6-NEXT:    or $10, $10, $9
+; MMELR6-NEXT:    sc $10, 0($2)
+; MMELR6-NEXT:    beqc $10, $zero, $BB9_1
+; MMELR6-NEXT:  # %bb.2: # %entry
+; MMELR6-NEXT:    and $7, $8, $4
+; MMELR6-NEXT:    srlv $7, $7, $3
+; MMELR6-NEXT:    seh $7, $7
+; MMELR6-NEXT:  # %bb.3: # %entry
+; MMELR6-NEXT:    sw $7, 4($sp) # 4-byte Folded Spill
+; MMELR6-NEXT:  # %bb.4: # %entry
+; MMELR6-NEXT:    sync
+; MMELR6-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MMELR6-NEXT:    addiu $sp, $sp, 8
+; MMELR6-NEXT:    jrc $ra
+;
+; MIPS64-LABEL: test_min_8:
+; MIPS64:       # %bb.0: # %entry
+; MIPS64-NEXT:    daddiu $sp, $sp, -16
+; MIPS64-NEXT:    .cfi_def_cfa_offset 16
+; MIPS64-NEXT:    # kill: def $a1 killed $a1 killed $a1_64
+; MIPS64-NEXT:    sync
+; MIPS64-NEXT:    daddiu $1, $zero, -4
+; MIPS64-NEXT:    and $1, $4, $1
+; MIPS64-NEXT:    andi $2, $4, 3
+; MIPS64-NEXT:    xori $2, $2, 3
+; MIPS64-NEXT:    sll $2, $2, 3
+; MIPS64-NEXT:    ori $3, $zero, 255
+; MIPS64-NEXT:    sllv $3, $3, $2
+; MIPS64-NEXT:    nor $6, $zero, $3
+; MIPS64-NEXT:    sllv $5, $5, $2
+; MIPS64-NEXT:  .LBB9_1: # %entry
+; MIPS64-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS64-NEXT:    ll $8, 0($1)
+; MIPS64-NEXT:    slt $11, $8, $5
+; MIPS64-NEXT:    move $9, $8
+; MIPS64-NEXT:    movz $9, $5, $11
+; MIPS64-NEXT:    and $9, $9, $3
+; MIPS64-NEXT:    and $10, $8, $6
+; MIPS64-NEXT:    or $10, $10, $9
+; MIPS64-NEXT:    sc $10, 0($1)
+; MIPS64-NEXT:    beqz $10, .LBB9_1
+; MIPS64-NEXT:    nop
+; MIPS64-NEXT:  # %bb.2: # %entry
+; MIPS64-NEXT:    and $7, $8, $3
+; MIPS64-NEXT:    srlv $7, $7, $2
+; MIPS64-NEXT:    seh $7, $7
+; MIPS64-NEXT:  # %bb.3: # %entry
+; MIPS64-NEXT:    sw $7, 12($sp) # 4-byte Folded Spill
+; MIPS64-NEXT:  # %bb.4: # %entry
+; MIPS64-NEXT:    sync
+; MIPS64-NEXT:    lw $2, 12($sp) # 4-byte Folded Reload
+; MIPS64-NEXT:    daddiu $sp, $sp, 16
+; MIPS64-NEXT:    jr $ra
+; MIPS64-NEXT:    nop
+;
+; MIPS64R6-LABEL: test_min_8:
+; MIPS64R6:       # %bb.0: # %entry
+; MIPS64R6-NEXT:    daddiu $sp, $sp, -16
+; MIPS64R6-NEXT:    .cfi_def_cfa_offset 16
+; MIPS64R6-NEXT:    # kill: def $a1 killed $a1 killed $a1_64
+; MIPS64R6-NEXT:    sync
+; MIPS64R6-NEXT:    daddiu $1, $zero, -4
+; MIPS64R6-NEXT:    and $1, $4, $1
+; MIPS64R6-NEXT:    andi $2, $4, 3
+; MIPS64R6-NEXT:    xori $2, $2, 3
+; MIPS64R6-NEXT:    sll $2, $2, 3
+; MIPS64R6-NEXT:    ori $3, $zero, 255
+; MIPS64R6-NEXT:    sllv $3, $3, $2
+; MIPS64R6-NEXT:    nor $6, $zero, $3
+; MIPS64R6-NEXT:    sllv $5, $5, $2
+; MIPS64R6-NEXT:  .LBB9_1: # %entry
+; MIPS64R6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS64R6-NEXT:    ll $8, 0($1)
+; MIPS64R6-NEXT:    slt $11, $8, $5
+; MIPS64R6-NEXT:    selnez $9, $8, $11
+; MIPS64R6-NEXT:    seleqz $11, $5, $11
+; MIPS64R6-NEXT:    or $9, $9, $11
+; MIPS64R6-NEXT:    and $9, $9, $3
+; MIPS64R6-NEXT:    and $10, $8, $6
+; MIPS64R6-NEXT:    or $10, $10, $9
+; MIPS64R6-NEXT:    sc $10, 0($1)
+; MIPS64R6-NEXT:    beqzc $10, .LBB9_1
+; MIPS64R6-NEXT:  # %bb.2: # %entry
+; MIPS64R6-NEXT:    and $7, $8, $3
+; MIPS64R6-NEXT:    srlv $7, $7, $2
+; MIPS64R6-NEXT:    seh $7, $7
+; MIPS64R6-NEXT:  # %bb.3: # %entry
+; MIPS64R6-NEXT:    sw $7, 12($sp) # 4-byte Folded Spill
+; MIPS64R6-NEXT:  # %bb.4: # %entry
+; MIPS64R6-NEXT:    sync
+; MIPS64R6-NEXT:    lw $2, 12($sp) # 4-byte Folded Reload
+; MIPS64R6-NEXT:    daddiu $sp, $sp, 16
+; MIPS64R6-NEXT:    jrc $ra
+;
+; MIPS64EL-LABEL: test_min_8:
+; MIPS64EL:       # %bb.0: # %entry
+; MIPS64EL-NEXT:    daddiu $sp, $sp, -16
+; MIPS64EL-NEXT:    .cfi_def_cfa_offset 16
+; MIPS64EL-NEXT:    # kill: def $a1 killed $a1 killed $a1_64
+; MIPS64EL-NEXT:    sync
+; MIPS64EL-NEXT:    daddiu $1, $zero, -4
+; MIPS64EL-NEXT:    and $1, $4, $1
+; MIPS64EL-NEXT:    andi $2, $4, 3
+; MIPS64EL-NEXT:    sll $2, $2, 3
+; MIPS64EL-NEXT:    ori $3, $zero, 255
+; MIPS64EL-NEXT:    sllv $3, $3, $2
+; MIPS64EL-NEXT:    nor $6, $zero, $3
+; MIPS64EL-NEXT:    sllv $5, $5, $2
+; MIPS64EL-NEXT:  .LBB9_1: # %entry
+; MIPS64EL-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS64EL-NEXT:    ll $8, 0($1)
+; MIPS64EL-NEXT:    and $8, $8, $3
+; MIPS64EL-NEXT:    and $5, $5, $3
+; MIPS64EL-NEXT:    slt $11, $8, $5
+; MIPS64EL-NEXT:    move $9, $8
+; MIPS64EL-NEXT:    movz $9, $5, $11
+; MIPS64EL-NEXT:    and $9, $9, $3
+; MIPS64EL-NEXT:    and $10, $8, $6
+; MIPS64EL-NEXT:    or $10, $10, $9
+; MIPS64EL-NEXT:    sc $10, 0($1)
+; MIPS64EL-NEXT:    beqz $10, .LBB9_1
+; MIPS64EL-NEXT:    nop
+; MIPS64EL-NEXT:  # %bb.2: # %entry
+; MIPS64EL-NEXT:    and $7, $8, $3
+; MIPS64EL-NEXT:    srlv $7, $7, $2
+; MIPS64EL-NEXT:    seh $7, $7
+; MIPS64EL-NEXT:  # %bb.3: # %entry
+; MIPS64EL-NEXT:    sw $7, 12($sp) # 4-byte Folded Spill
+; MIPS64EL-NEXT:  # %bb.4: # %entry
+; MIPS64EL-NEXT:    sync
+; MIPS64EL-NEXT:    lw $2, 12($sp) # 4-byte Folded Reload
+; MIPS64EL-NEXT:    daddiu $sp, $sp, 16
+; MIPS64EL-NEXT:    jr $ra
+; MIPS64EL-NEXT:    nop
+;
+; MIPS64ELR6-LABEL: test_min_8:
+; MIPS64ELR6:       # %bb.0: # %entry
+; MIPS64ELR6-NEXT:    daddiu $sp, $sp, -16
+; MIPS64ELR6-NEXT:    .cfi_def_cfa_offset 16
+; MIPS64ELR6-NEXT:    # kill: def $a1 killed $a1 killed $a1_64
+; MIPS64ELR6-NEXT:    sync
+; MIPS64ELR6-NEXT:    daddiu $1, $zero, -4
+; MIPS64ELR6-NEXT:    and $1, $4, $1
+; MIPS64ELR6-NEXT:    andi $2, $4, 3
+; MIPS64ELR6-NEXT:    sll $2, $2, 3
+; MIPS64ELR6-NEXT:    ori $3, $zero, 255
+; MIPS64ELR6-NEXT:    sllv $3, $3, $2
+; MIPS64ELR6-NEXT:    nor $6, $zero, $3
+; MIPS64ELR6-NEXT:    sllv $5, $5, $2
+; MIPS64ELR6-NEXT:  .LBB9_1: # %entry
+; MIPS64ELR6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS64ELR6-NEXT:    ll $8, 0($1)
+; MIPS64ELR6-NEXT:    and $8, $8, $3
+; MIPS64ELR6-NEXT:    and $5, $5, $3
+; MIPS64ELR6-NEXT:    slt $11, $8, $5
+; MIPS64ELR6-NEXT:    selnez $9, $8, $11
+; MIPS64ELR6-NEXT:    seleqz $11, $5, $11
+; MIPS64ELR6-NEXT:    or $9, $9, $11
+; MIPS64ELR6-NEXT:    and $9, $9, $3
+; MIPS64ELR6-NEXT:    and $10, $8, $6
+; MIPS64ELR6-NEXT:    or $10, $10, $9
+; MIPS64ELR6-NEXT:    sc $10, 0($1)
+; MIPS64ELR6-NEXT:    beqzc $10, .LBB9_1
+; MIPS64ELR6-NEXT:  # %bb.2: # %entry
+; MIPS64ELR6-NEXT:    and $7, $8, $3
+; MIPS64ELR6-NEXT:    srlv $7, $7, $2
+; MIPS64ELR6-NEXT:    seh $7, $7
+; MIPS64ELR6-NEXT:  # %bb.3: # %entry
+; MIPS64ELR6-NEXT:    sw $7, 12($sp) # 4-byte Folded Spill
+; MIPS64ELR6-NEXT:  # %bb.4: # %entry
+; MIPS64ELR6-NEXT:    sync
+; MIPS64ELR6-NEXT:    lw $2, 12($sp) # 4-byte Folded Reload
+; MIPS64ELR6-NEXT:    daddiu $sp, $sp, 16
+; MIPS64ELR6-NEXT:    jrc $ra
+entry:
+  %0 = atomicrmw min i8* %ptr, i8 %val seq_cst
+  ret i8 %0
+}
+
+define i8 @test_umax_8(i8* nocapture %ptr, i8 signext %val) {
+; MIPS-LABEL: test_umax_8:
+; MIPS:       # %bb.0: # %entry
+; MIPS-NEXT:    addiu $sp, $sp, -8
+; MIPS-NEXT:    .cfi_def_cfa_offset 8
+; MIPS-NEXT:    move $1, $5
+; MIPS-NEXT:    sync
+; MIPS-NEXT:    addiu $2, $zero, -4
+; MIPS-NEXT:    and $2, $4, $2
+; MIPS-NEXT:    andi $3, $4, 3
+; MIPS-NEXT:    xori $3, $3, 3
+; MIPS-NEXT:    sll $3, $3, 3
+; MIPS-NEXT:    ori $4, $zero, 255
+; MIPS-NEXT:    sllv $4, $4, $3
+; MIPS-NEXT:    nor $6, $zero, $4
+; MIPS-NEXT:    sllv $5, $5, $3
+; MIPS-NEXT:  $BB10_1: # %entry
+; MIPS-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS-NEXT:    ll $8, 0($2)
+; MIPS-NEXT:    sltu $11, $8, $5
+; MIPS-NEXT:    move $9, $8
+; MIPS-NEXT:    movn $9, $5, $11
+; MIPS-NEXT:    and $9, $9, $4
+; MIPS-NEXT:    and $10, $8, $6
+; MIPS-NEXT:    or $10, $10, $9
+; MIPS-NEXT:    sc $10, 0($2)
+; MIPS-NEXT:    beqz $10, $BB10_1
+; MIPS-NEXT:    nop
+; MIPS-NEXT:  # %bb.2: # %entry
+; MIPS-NEXT:    and $7, $8, $4
+; MIPS-NEXT:    srlv $7, $7, $3
+; MIPS-NEXT:    seh $7, $7
+; MIPS-NEXT:  # %bb.3: # %entry
+; MIPS-NEXT:    sw $7, 4($sp) # 4-byte Folded Spill
+; MIPS-NEXT:  # %bb.4: # %entry
+; MIPS-NEXT:    sync
+; MIPS-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MIPS-NEXT:    addiu $sp, $sp, 8
+; MIPS-NEXT:    jr $ra
+; MIPS-NEXT:    nop
+;
+; MIPSR6-LABEL: test_umax_8:
+; MIPSR6:       # %bb.0: # %entry
+; MIPSR6-NEXT:    addiu $sp, $sp, -8
+; MIPSR6-NEXT:    .cfi_def_cfa_offset 8
+; MIPSR6-NEXT:    move $1, $5
+; MIPSR6-NEXT:    sync
+; MIPSR6-NEXT:    addiu $2, $zero, -4
+; MIPSR6-NEXT:    and $2, $4, $2
+; MIPSR6-NEXT:    andi $3, $4, 3
+; MIPSR6-NEXT:    xori $3, $3, 3
+; MIPSR6-NEXT:    sll $3, $3, 3
+; MIPSR6-NEXT:    ori $4, $zero, 255
+; MIPSR6-NEXT:    sllv $4, $4, $3
+; MIPSR6-NEXT:    nor $6, $zero, $4
+; MIPSR6-NEXT:    sllv $5, $5, $3
+; MIPSR6-NEXT:  $BB10_1: # %entry
+; MIPSR6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPSR6-NEXT:    ll $8, 0($2)
+; MIPSR6-NEXT:    sltu $11, $8, $5
+; MIPSR6-NEXT:    seleqz $9, $8, $11
+; MIPSR6-NEXT:    selnez $11, $5, $11
+; MIPSR6-NEXT:    or $9, $9, $11
+; MIPSR6-NEXT:    and $9, $9, $4
+; MIPSR6-NEXT:    and $10, $8, $6
+; MIPSR6-NEXT:    or $10, $10, $9
+; MIPSR6-NEXT:    sc $10, 0($2)
+; MIPSR6-NEXT:    beqzc $10, $BB10_1
+; MIPSR6-NEXT:  # %bb.2: # %entry
+; MIPSR6-NEXT:    and $7, $8, $4
+; MIPSR6-NEXT:    srlv $7, $7, $3
+; MIPSR6-NEXT:    seh $7, $7
+; MIPSR6-NEXT:  # %bb.3: # %entry
+; MIPSR6-NEXT:    sw $7, 4($sp) # 4-byte Folded Spill
+; MIPSR6-NEXT:  # %bb.4: # %entry
+; MIPSR6-NEXT:    sync
+; MIPSR6-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MIPSR6-NEXT:    addiu $sp, $sp, 8
+; MIPSR6-NEXT:    jrc $ra
+;
+; MM-LABEL: test_umax_8:
+; MM:       # %bb.0: # %entry
+; MM-NEXT:    addiu $sp, $sp, -8
+; MM-NEXT:    .cfi_def_cfa_offset 8
+; MM-NEXT:    move $1, $5
+; MM-NEXT:    sync
+; MM-NEXT:    addiu $2, $zero, -4
+; MM-NEXT:    and $2, $4, $2
+; MM-NEXT:    andi $3, $4, 3
+; MM-NEXT:    xori $3, $3, 3
+; MM-NEXT:    sll $3, $3, 3
+; MM-NEXT:    ori $4, $zero, 255
+; MM-NEXT:    sllv $4, $4, $3
+; MM-NEXT:    nor $6, $zero, $4
+; MM-NEXT:    sllv $5, $5, $3
+; MM-NEXT:  $BB10_1: # %entry
+; MM-NEXT:    # =>This Inner Loop Header: Depth=1
+; MM-NEXT:    ll $8, 0($2)
+; MM-NEXT:    sltu $11, $8, $5
+; MM-NEXT:    or $9, $8, $zero
+; MM-NEXT:    movn $9, $5, $11
+; MM-NEXT:    and $9, $9, $4
+; MM-NEXT:    and $10, $8, $6
+; MM-NEXT:    or $10, $10, $9
+; MM-NEXT:    sc $10, 0($2)
+; MM-NEXT:    beqzc $10, $BB10_1
+; MM-NEXT:  # %bb.2: # %entry
+; MM-NEXT:    and $7, $8, $4
+; MM-NEXT:    srlv $7, $7, $3
+; MM-NEXT:    seh $7, $7
+; MM-NEXT:  # %bb.3: # %entry
+; MM-NEXT:    sw $7, 4($sp) # 4-byte Folded Spill
+; MM-NEXT:  # %bb.4: # %entry
+; MM-NEXT:    sync
+; MM-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MM-NEXT:    addiusp 8
+; MM-NEXT:    jrc $ra
+;
+; MMR6-LABEL: test_umax_8:
+; MMR6:       # %bb.0: # %entry
+; MMR6-NEXT:    addiu $sp, $sp, -8
+; MMR6-NEXT:    .cfi_def_cfa_offset 8
+; MMR6-NEXT:    move $1, $5
+; MMR6-NEXT:    sync
+; MMR6-NEXT:    addiu $2, $zero, -4
+; MMR6-NEXT:    and $2, $4, $2
+; MMR6-NEXT:    andi $3, $4, 3
+; MMR6-NEXT:    xori $3, $3, 3
+; MMR6-NEXT:    sll $3, $3, 3
+; MMR6-NEXT:    ori $4, $zero, 255
+; MMR6-NEXT:    sllv $4, $4, $3
+; MMR6-NEXT:    nor $6, $zero, $4
+; MMR6-NEXT:    sllv $5, $5, $3
+; MMR6-NEXT:  $BB10_1: # %entry
+; MMR6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MMR6-NEXT:    ll $8, 0($2)
+; MMR6-NEXT:    sltu $11, $8, $5
+; MMR6-NEXT:    seleqz $9, $8, $11
+; MMR6-NEXT:    selnez $11, $5, $11
+; MMR6-NEXT:    or $9, $9, $11
+; MMR6-NEXT:    and $9, $9, $4
+; MMR6-NEXT:    and $10, $8, $6
+; MMR6-NEXT:    or $10, $10, $9
+; MMR6-NEXT:    sc $10, 0($2)
+; MMR6-NEXT:    beqc $10, $zero, $BB10_1
+; MMR6-NEXT:  # %bb.2: # %entry
+; MMR6-NEXT:    and $7, $8, $4
+; MMR6-NEXT:    srlv $7, $7, $3
+; MMR6-NEXT:    seh $7, $7
+; MMR6-NEXT:  # %bb.3: # %entry
+; MMR6-NEXT:    sw $7, 4($sp) # 4-byte Folded Spill
+; MMR6-NEXT:  # %bb.4: # %entry
+; MMR6-NEXT:    sync
+; MMR6-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MMR6-NEXT:    addiu $sp, $sp, 8
+; MMR6-NEXT:    jrc $ra
+;
+; MIPSEL-LABEL: test_umax_8:
+; MIPSEL:       # %bb.0: # %entry
+; MIPSEL-NEXT:    addiu $sp, $sp, -8
+; MIPSEL-NEXT:    .cfi_def_cfa_offset 8
+; MIPSEL-NEXT:    move $1, $5
+; MIPSEL-NEXT:    sync
+; MIPSEL-NEXT:    addiu $2, $zero, -4
+; MIPSEL-NEXT:    and $2, $4, $2
+; MIPSEL-NEXT:    andi $3, $4, 3
+; MIPSEL-NEXT:    sll $3, $3, 3
+; MIPSEL-NEXT:    ori $4, $zero, 255
+; MIPSEL-NEXT:    sllv $4, $4, $3
+; MIPSEL-NEXT:    nor $6, $zero, $4
+; MIPSEL-NEXT:    sllv $5, $5, $3
+; MIPSEL-NEXT:  $BB10_1: # %entry
+; MIPSEL-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPSEL-NEXT:    ll $8, 0($2)
+; MIPSEL-NEXT:    and $8, $8, $4
+; MIPSEL-NEXT:    and $5, $5, $4
+; MIPSEL-NEXT:    sltu $11, $8, $5
+; MIPSEL-NEXT:    move $9, $8
+; MIPSEL-NEXT:    movn $9, $5, $11
+; MIPSEL-NEXT:    and $9, $9, $4
+; MIPSEL-NEXT:    and $10, $8, $6
+; MIPSEL-NEXT:    or $10, $10, $9
+; MIPSEL-NEXT:    sc $10, 0($2)
+; MIPSEL-NEXT:    beqz $10, $BB10_1
+; MIPSEL-NEXT:    nop
+; MIPSEL-NEXT:  # %bb.2: # %entry
+; MIPSEL-NEXT:    and $7, $8, $4
+; MIPSEL-NEXT:    srlv $7, $7, $3
+; MIPSEL-NEXT:    seh $7, $7
+; MIPSEL-NEXT:  # %bb.3: # %entry
+; MIPSEL-NEXT:    sw $7, 4($sp) # 4-byte Folded Spill
+; MIPSEL-NEXT:  # %bb.4: # %entry
+; MIPSEL-NEXT:    sync
+; MIPSEL-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MIPSEL-NEXT:    addiu $sp, $sp, 8
+; MIPSEL-NEXT:    jr $ra
+; MIPSEL-NEXT:    nop
+;
+; MIPSELR6-LABEL: test_umax_8:
+; MIPSELR6:       # %bb.0: # %entry
+; MIPSELR6-NEXT:    addiu $sp, $sp, -8
+; MIPSELR6-NEXT:    .cfi_def_cfa_offset 8
+; MIPSELR6-NEXT:    move $1, $5
+; MIPSELR6-NEXT:    sync
+; MIPSELR6-NEXT:    addiu $2, $zero, -4
+; MIPSELR6-NEXT:    and $2, $4, $2
+; MIPSELR6-NEXT:    andi $3, $4, 3
+; MIPSELR6-NEXT:    sll $3, $3, 3
+; MIPSELR6-NEXT:    ori $4, $zero, 255
+; MIPSELR6-NEXT:    sllv $4, $4, $3
+; MIPSELR6-NEXT:    nor $6, $zero, $4
+; MIPSELR6-NEXT:    sllv $5, $5, $3
+; MIPSELR6-NEXT:  $BB10_1: # %entry
+; MIPSELR6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPSELR6-NEXT:    ll $8, 0($2)
+; MIPSELR6-NEXT:    and $8, $8, $4
+; MIPSELR6-NEXT:    and $5, $5, $4
+; MIPSELR6-NEXT:    sltu $11, $8, $5
+; MIPSELR6-NEXT:    seleqz $9, $8, $11
+; MIPSELR6-NEXT:    selnez $11, $5, $11
+; MIPSELR6-NEXT:    or $9, $9, $11
+; MIPSELR6-NEXT:    and $9, $9, $4
+; MIPSELR6-NEXT:    and $10, $8, $6
+; MIPSELR6-NEXT:    or $10, $10, $9
+; MIPSELR6-NEXT:    sc $10, 0($2)
+; MIPSELR6-NEXT:    beqzc $10, $BB10_1
+; MIPSELR6-NEXT:  # %bb.2: # %entry
+; MIPSELR6-NEXT:    and $7, $8, $4
+; MIPSELR6-NEXT:    srlv $7, $7, $3
+; MIPSELR6-NEXT:    seh $7, $7
+; MIPSELR6-NEXT:  # %bb.3: # %entry
+; MIPSELR6-NEXT:    sw $7, 4($sp) # 4-byte Folded Spill
+; MIPSELR6-NEXT:  # %bb.4: # %entry
+; MIPSELR6-NEXT:    sync
+; MIPSELR6-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MIPSELR6-NEXT:    addiu $sp, $sp, 8
+; MIPSELR6-NEXT:    jrc $ra
+;
+; MMEL-LABEL: test_umax_8:
+; MMEL:       # %bb.0: # %entry
+; MMEL-NEXT:    addiu $sp, $sp, -8
+; MMEL-NEXT:    .cfi_def_cfa_offset 8
+; MMEL-NEXT:    move $1, $5
+; MMEL-NEXT:    sync
+; MMEL-NEXT:    addiu $2, $zero, -4
+; MMEL-NEXT:    and $2, $4, $2
+; MMEL-NEXT:    andi $3, $4, 3
+; MMEL-NEXT:    sll $3, $3, 3
+; MMEL-NEXT:    ori $4, $zero, 255
+; MMEL-NEXT:    sllv $4, $4, $3
+; MMEL-NEXT:    nor $6, $zero, $4
+; MMEL-NEXT:    sllv $5, $5, $3
+; MMEL-NEXT:  $BB10_1: # %entry
+; MMEL-NEXT:    # =>This Inner Loop Header: Depth=1
+; MMEL-NEXT:    ll $8, 0($2)
+; MMEL-NEXT:    and $8, $8, $4
+; MMEL-NEXT:    and $5, $5, $4
+; MMEL-NEXT:    sltu $11, $8, $5
+; MMEL-NEXT:    or $9, $8, $zero
+; MMEL-NEXT:    movn $9, $5, $11
+; MMEL-NEXT:    and $9, $9, $4
+; MMEL-NEXT:    and $10, $8, $6
+; MMEL-NEXT:    or $10, $10, $9
+; MMEL-NEXT:    sc $10, 0($2)
+; MMEL-NEXT:    beqzc $10, $BB10_1
+; MMEL-NEXT:  # %bb.2: # %entry
+; MMEL-NEXT:    and $7, $8, $4
+; MMEL-NEXT:    srlv $7, $7, $3
+; MMEL-NEXT:    seh $7, $7
+; MMEL-NEXT:  # %bb.3: # %entry
+; MMEL-NEXT:    sw $7, 4($sp) # 4-byte Folded Spill
+; MMEL-NEXT:  # %bb.4: # %entry
+; MMEL-NEXT:    sync
+; MMEL-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MMEL-NEXT:    addiusp 8
+; MMEL-NEXT:    jrc $ra
+;
+; MMELR6-LABEL: test_umax_8:
+; MMELR6:       # %bb.0: # %entry
+; MMELR6-NEXT:    addiu $sp, $sp, -8
+; MMELR6-NEXT:    .cfi_def_cfa_offset 8
+; MMELR6-NEXT:    move $1, $5
+; MMELR6-NEXT:    sync
+; MMELR6-NEXT:    addiu $2, $zero, -4
+; MMELR6-NEXT:    and $2, $4, $2
+; MMELR6-NEXT:    andi $3, $4, 3
+; MMELR6-NEXT:    sll $3, $3, 3
+; MMELR6-NEXT:    ori $4, $zero, 255
+; MMELR6-NEXT:    sllv $4, $4, $3
+; MMELR6-NEXT:    nor $6, $zero, $4
+; MMELR6-NEXT:    sllv $5, $5, $3
+; MMELR6-NEXT:  $BB10_1: # %entry
+; MMELR6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MMELR6-NEXT:    ll $8, 0($2)
+; MMELR6-NEXT:    and $8, $8, $4
+; MMELR6-NEXT:    and $5, $5, $4
+; MMELR6-NEXT:    sltu $11, $8, $5
+; MMELR6-NEXT:    seleqz $9, $8, $11
+; MMELR6-NEXT:    selnez $11, $5, $11
+; MMELR6-NEXT:    or $9, $9, $11
+; MMELR6-NEXT:    and $9, $9, $4
+; MMELR6-NEXT:    and $10, $8, $6
+; MMELR6-NEXT:    or $10, $10, $9
+; MMELR6-NEXT:    sc $10, 0($2)
+; MMELR6-NEXT:    beqc $10, $zero, $BB10_1
+; MMELR6-NEXT:  # %bb.2: # %entry
+; MMELR6-NEXT:    and $7, $8, $4
+; MMELR6-NEXT:    srlv $7, $7, $3
+; MMELR6-NEXT:    seh $7, $7
+; MMELR6-NEXT:  # %bb.3: # %entry
+; MMELR6-NEXT:    sw $7, 4($sp) # 4-byte Folded Spill
+; MMELR6-NEXT:  # %bb.4: # %entry
+; MMELR6-NEXT:    sync
+; MMELR6-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MMELR6-NEXT:    addiu $sp, $sp, 8
+; MMELR6-NEXT:    jrc $ra
+;
+; MIPS64-LABEL: test_umax_8:
+; MIPS64:       # %bb.0: # %entry
+; MIPS64-NEXT:    daddiu $sp, $sp, -16
+; MIPS64-NEXT:    .cfi_def_cfa_offset 16
+; MIPS64-NEXT:    # kill: def $a1 killed $a1 killed $a1_64
+; MIPS64-NEXT:    sync
+; MIPS64-NEXT:    daddiu $1, $zero, -4
+; MIPS64-NEXT:    and $1, $4, $1
+; MIPS64-NEXT:    andi $2, $4, 3
+; MIPS64-NEXT:    xori $2, $2, 3
+; MIPS64-NEXT:    sll $2, $2, 3
+; MIPS64-NEXT:    ori $3, $zero, 255
+; MIPS64-NEXT:    sllv $3, $3, $2
+; MIPS64-NEXT:    nor $6, $zero, $3
+; MIPS64-NEXT:    sllv $5, $5, $2
+; MIPS64-NEXT:  .LBB10_1: # %entry
+; MIPS64-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS64-NEXT:    ll $8, 0($1)
+; MIPS64-NEXT:    sltu $11, $8, $5
+; MIPS64-NEXT:    move $9, $8
+; MIPS64-NEXT:    movn $9, $5, $11
+; MIPS64-NEXT:    and $9, $9, $3
+; MIPS64-NEXT:    and $10, $8, $6
+; MIPS64-NEXT:    or $10, $10, $9
+; MIPS64-NEXT:    sc $10, 0($1)
+; MIPS64-NEXT:    beqz $10, .LBB10_1
+; MIPS64-NEXT:    nop
+; MIPS64-NEXT:  # %bb.2: # %entry
+; MIPS64-NEXT:    and $7, $8, $3
+; MIPS64-NEXT:    srlv $7, $7, $2
+; MIPS64-NEXT:    seh $7, $7
+; MIPS64-NEXT:  # %bb.3: # %entry
+; MIPS64-NEXT:    sw $7, 12($sp) # 4-byte Folded Spill
+; MIPS64-NEXT:  # %bb.4: # %entry
+; MIPS64-NEXT:    sync
+; MIPS64-NEXT:    lw $2, 12($sp) # 4-byte Folded Reload
+; MIPS64-NEXT:    daddiu $sp, $sp, 16
+; MIPS64-NEXT:    jr $ra
+; MIPS64-NEXT:    nop
+;
+; MIPS64R6-LABEL: test_umax_8:
+; MIPS64R6:       # %bb.0: # %entry
+; MIPS64R6-NEXT:    daddiu $sp, $sp, -16
+; MIPS64R6-NEXT:    .cfi_def_cfa_offset 16
+; MIPS64R6-NEXT:    # kill: def $a1 killed $a1 killed $a1_64
+; MIPS64R6-NEXT:    sync
+; MIPS64R6-NEXT:    daddiu $1, $zero, -4
+; MIPS64R6-NEXT:    and $1, $4, $1
+; MIPS64R6-NEXT:    andi $2, $4, 3
+; MIPS64R6-NEXT:    xori $2, $2, 3
+; MIPS64R6-NEXT:    sll $2, $2, 3
+; MIPS64R6-NEXT:    ori $3, $zero, 255
+; MIPS64R6-NEXT:    sllv $3, $3, $2
+; MIPS64R6-NEXT:    nor $6, $zero, $3
+; MIPS64R6-NEXT:    sllv $5, $5, $2
+; MIPS64R6-NEXT:  .LBB10_1: # %entry
+; MIPS64R6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS64R6-NEXT:    ll $8, 0($1)
+; MIPS64R6-NEXT:    sltu $11, $8, $5
+; MIPS64R6-NEXT:    seleqz $9, $8, $11
+; MIPS64R6-NEXT:    selnez $11, $5, $11
+; MIPS64R6-NEXT:    or $9, $9, $11
+; MIPS64R6-NEXT:    and $9, $9, $3
+; MIPS64R6-NEXT:    and $10, $8, $6
+; MIPS64R6-NEXT:    or $10, $10, $9
+; MIPS64R6-NEXT:    sc $10, 0($1)
+; MIPS64R6-NEXT:    beqzc $10, .LBB10_1
+; MIPS64R6-NEXT:  # %bb.2: # %entry
+; MIPS64R6-NEXT:    and $7, $8, $3
+; MIPS64R6-NEXT:    srlv $7, $7, $2
+; MIPS64R6-NEXT:    seh $7, $7
+; MIPS64R6-NEXT:  # %bb.3: # %entry
+; MIPS64R6-NEXT:    sw $7, 12($sp) # 4-byte Folded Spill
+; MIPS64R6-NEXT:  # %bb.4: # %entry
+; MIPS64R6-NEXT:    sync
+; MIPS64R6-NEXT:    lw $2, 12($sp) # 4-byte Folded Reload
+; MIPS64R6-NEXT:    daddiu $sp, $sp, 16
+; MIPS64R6-NEXT:    jrc $ra
+;
+; MIPS64EL-LABEL: test_umax_8:
+; MIPS64EL:       # %bb.0: # %entry
+; MIPS64EL-NEXT:    daddiu $sp, $sp, -16
+; MIPS64EL-NEXT:    .cfi_def_cfa_offset 16
+; MIPS64EL-NEXT:    # kill: def $a1 killed $a1 killed $a1_64
+; MIPS64EL-NEXT:    sync
+; MIPS64EL-NEXT:    daddiu $1, $zero, -4
+; MIPS64EL-NEXT:    and $1, $4, $1
+; MIPS64EL-NEXT:    andi $2, $4, 3
+; MIPS64EL-NEXT:    sll $2, $2, 3
+; MIPS64EL-NEXT:    ori $3, $zero, 255
+; MIPS64EL-NEXT:    sllv $3, $3, $2
+; MIPS64EL-NEXT:    nor $6, $zero, $3
+; MIPS64EL-NEXT:    sllv $5, $5, $2
+; MIPS64EL-NEXT:  .LBB10_1: # %entry
+; MIPS64EL-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS64EL-NEXT:    ll $8, 0($1)
+; MIPS64EL-NEXT:    and $8, $8, $3
+; MIPS64EL-NEXT:    and $5, $5, $3
+; MIPS64EL-NEXT:    sltu $11, $8, $5
+; MIPS64EL-NEXT:    move $9, $8
+; MIPS64EL-NEXT:    movn $9, $5, $11
+; MIPS64EL-NEXT:    and $9, $9, $3
+; MIPS64EL-NEXT:    and $10, $8, $6
+; MIPS64EL-NEXT:    or $10, $10, $9
+; MIPS64EL-NEXT:    sc $10, 0($1)
+; MIPS64EL-NEXT:    beqz $10, .LBB10_1
+; MIPS64EL-NEXT:    nop
+; MIPS64EL-NEXT:  # %bb.2: # %entry
+; MIPS64EL-NEXT:    and $7, $8, $3
+; MIPS64EL-NEXT:    srlv $7, $7, $2
+; MIPS64EL-NEXT:    seh $7, $7
+; MIPS64EL-NEXT:  # %bb.3: # %entry
+; MIPS64EL-NEXT:    sw $7, 12($sp) # 4-byte Folded Spill
+; MIPS64EL-NEXT:  # %bb.4: # %entry
+; MIPS64EL-NEXT:    sync
+; MIPS64EL-NEXT:    lw $2, 12($sp) # 4-byte Folded Reload
+; MIPS64EL-NEXT:    daddiu $sp, $sp, 16
+; MIPS64EL-NEXT:    jr $ra
+; MIPS64EL-NEXT:    nop
+;
+; MIPS64ELR6-LABEL: test_umax_8:
+; MIPS64ELR6:       # %bb.0: # %entry
+; MIPS64ELR6-NEXT:    daddiu $sp, $sp, -16
+; MIPS64ELR6-NEXT:    .cfi_def_cfa_offset 16
+; MIPS64ELR6-NEXT:    # kill: def $a1 killed $a1 killed $a1_64
+; MIPS64ELR6-NEXT:    sync
+; MIPS64ELR6-NEXT:    daddiu $1, $zero, -4
+; MIPS64ELR6-NEXT:    and $1, $4, $1
+; MIPS64ELR6-NEXT:    andi $2, $4, 3
+; MIPS64ELR6-NEXT:    sll $2, $2, 3
+; MIPS64ELR6-NEXT:    ori $3, $zero, 255
+; MIPS64ELR6-NEXT:    sllv $3, $3, $2
+; MIPS64ELR6-NEXT:    nor $6, $zero, $3
+; MIPS64ELR6-NEXT:    sllv $5, $5, $2
+; MIPS64ELR6-NEXT:  .LBB10_1: # %entry
+; MIPS64ELR6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS64ELR6-NEXT:    ll $8, 0($1)
+; MIPS64ELR6-NEXT:    and $8, $8, $3
+; MIPS64ELR6-NEXT:    and $5, $5, $3
+; MIPS64ELR6-NEXT:    sltu $11, $8, $5
+; MIPS64ELR6-NEXT:    seleqz $9, $8, $11
+; MIPS64ELR6-NEXT:    selnez $11, $5, $11
+; MIPS64ELR6-NEXT:    or $9, $9, $11
+; MIPS64ELR6-NEXT:    and $9, $9, $3
+; MIPS64ELR6-NEXT:    and $10, $8, $6
+; MIPS64ELR6-NEXT:    or $10, $10, $9
+; MIPS64ELR6-NEXT:    sc $10, 0($1)
+; MIPS64ELR6-NEXT:    beqzc $10, .LBB10_1
+; MIPS64ELR6-NEXT:  # %bb.2: # %entry
+; MIPS64ELR6-NEXT:    and $7, $8, $3
+; MIPS64ELR6-NEXT:    srlv $7, $7, $2
+; MIPS64ELR6-NEXT:    seh $7, $7
+; MIPS64ELR6-NEXT:  # %bb.3: # %entry
+; MIPS64ELR6-NEXT:    sw $7, 12($sp) # 4-byte Folded Spill
+; MIPS64ELR6-NEXT:  # %bb.4: # %entry
+; MIPS64ELR6-NEXT:    sync
+; MIPS64ELR6-NEXT:    lw $2, 12($sp) # 4-byte Folded Reload
+; MIPS64ELR6-NEXT:    daddiu $sp, $sp, 16
+; MIPS64ELR6-NEXT:    jrc $ra
+entry:
+  %0 = atomicrmw umax i8* %ptr, i8 %val seq_cst
+  ret i8 %0
+}
+
+define i8 @test_umin_8(i8* nocapture %ptr, i8 signext %val) {
+; MIPS-LABEL: test_umin_8:
+; MIPS:       # %bb.0: # %entry
+; MIPS-NEXT:    addiu $sp, $sp, -8
+; MIPS-NEXT:    .cfi_def_cfa_offset 8
+; MIPS-NEXT:    move $1, $5
+; MIPS-NEXT:    sync
+; MIPS-NEXT:    addiu $2, $zero, -4
+; MIPS-NEXT:    and $2, $4, $2
+; MIPS-NEXT:    andi $3, $4, 3
+; MIPS-NEXT:    xori $3, $3, 3
+; MIPS-NEXT:    sll $3, $3, 3
+; MIPS-NEXT:    ori $4, $zero, 255
+; MIPS-NEXT:    sllv $4, $4, $3
+; MIPS-NEXT:    nor $6, $zero, $4
+; MIPS-NEXT:    sllv $5, $5, $3
+; MIPS-NEXT:  $BB11_1: # %entry
+; MIPS-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS-NEXT:    ll $8, 0($2)
+; MIPS-NEXT:    sltu $11, $8, $5
+; MIPS-NEXT:    move $9, $8
+; MIPS-NEXT:    movz $9, $5, $11
+; MIPS-NEXT:    and $9, $9, $4
+; MIPS-NEXT:    and $10, $8, $6
+; MIPS-NEXT:    or $10, $10, $9
+; MIPS-NEXT:    sc $10, 0($2)
+; MIPS-NEXT:    beqz $10, $BB11_1
+; MIPS-NEXT:    nop
+; MIPS-NEXT:  # %bb.2: # %entry
+; MIPS-NEXT:    and $7, $8, $4
+; MIPS-NEXT:    srlv $7, $7, $3
+; MIPS-NEXT:    seh $7, $7
+; MIPS-NEXT:  # %bb.3: # %entry
+; MIPS-NEXT:    sw $7, 4($sp) # 4-byte Folded Spill
+; MIPS-NEXT:  # %bb.4: # %entry
+; MIPS-NEXT:    sync
+; MIPS-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MIPS-NEXT:    addiu $sp, $sp, 8
+; MIPS-NEXT:    jr $ra
+; MIPS-NEXT:    nop
+;
+; MIPSR6-LABEL: test_umin_8:
+; MIPSR6:       # %bb.0: # %entry
+; MIPSR6-NEXT:    addiu $sp, $sp, -8
+; MIPSR6-NEXT:    .cfi_def_cfa_offset 8
+; MIPSR6-NEXT:    move $1, $5
+; MIPSR6-NEXT:    sync
+; MIPSR6-NEXT:    addiu $2, $zero, -4
+; MIPSR6-NEXT:    and $2, $4, $2
+; MIPSR6-NEXT:    andi $3, $4, 3
+; MIPSR6-NEXT:    xori $3, $3, 3
+; MIPSR6-NEXT:    sll $3, $3, 3
+; MIPSR6-NEXT:    ori $4, $zero, 255
+; MIPSR6-NEXT:    sllv $4, $4, $3
+; MIPSR6-NEXT:    nor $6, $zero, $4
+; MIPSR6-NEXT:    sllv $5, $5, $3
+; MIPSR6-NEXT:  $BB11_1: # %entry
+; MIPSR6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPSR6-NEXT:    ll $8, 0($2)
+; MIPSR6-NEXT:    sltu $11, $8, $5
+; MIPSR6-NEXT:    selnez $9, $8, $11
+; MIPSR6-NEXT:    seleqz $11, $5, $11
+; MIPSR6-NEXT:    or $9, $9, $11
+; MIPSR6-NEXT:    and $9, $9, $4
+; MIPSR6-NEXT:    and $10, $8, $6
+; MIPSR6-NEXT:    or $10, $10, $9
+; MIPSR6-NEXT:    sc $10, 0($2)
+; MIPSR6-NEXT:    beqzc $10, $BB11_1
+; MIPSR6-NEXT:  # %bb.2: # %entry
+; MIPSR6-NEXT:    and $7, $8, $4
+; MIPSR6-NEXT:    srlv $7, $7, $3
+; MIPSR6-NEXT:    seh $7, $7
+; MIPSR6-NEXT:  # %bb.3: # %entry
+; MIPSR6-NEXT:    sw $7, 4($sp) # 4-byte Folded Spill
+; MIPSR6-NEXT:  # %bb.4: # %entry
+; MIPSR6-NEXT:    sync
+; MIPSR6-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MIPSR6-NEXT:    addiu $sp, $sp, 8
+; MIPSR6-NEXT:    jrc $ra
+;
+; MM-LABEL: test_umin_8:
+; MM:       # %bb.0: # %entry
+; MM-NEXT:    addiu $sp, $sp, -8
+; MM-NEXT:    .cfi_def_cfa_offset 8
+; MM-NEXT:    move $1, $5
+; MM-NEXT:    sync
+; MM-NEXT:    addiu $2, $zero, -4
+; MM-NEXT:    and $2, $4, $2
+; MM-NEXT:    andi $3, $4, 3
+; MM-NEXT:    xori $3, $3, 3
+; MM-NEXT:    sll $3, $3, 3
+; MM-NEXT:    ori $4, $zero, 255
+; MM-NEXT:    sllv $4, $4, $3
+; MM-NEXT:    nor $6, $zero, $4
+; MM-NEXT:    sllv $5, $5, $3
+; MM-NEXT:  $BB11_1: # %entry
+; MM-NEXT:    # =>This Inner Loop Header: Depth=1
+; MM-NEXT:    ll $8, 0($2)
+; MM-NEXT:    sltu $11, $8, $5
+; MM-NEXT:    or $9, $8, $zero
+; MM-NEXT:    movz $9, $5, $11
+; MM-NEXT:    and $9, $9, $4
+; MM-NEXT:    and $10, $8, $6
+; MM-NEXT:    or $10, $10, $9
+; MM-NEXT:    sc $10, 0($2)
+; MM-NEXT:    beqzc $10, $BB11_1
+; MM-NEXT:  # %bb.2: # %entry
+; MM-NEXT:    and $7, $8, $4
+; MM-NEXT:    srlv $7, $7, $3
+; MM-NEXT:    seh $7, $7
+; MM-NEXT:  # %bb.3: # %entry
+; MM-NEXT:    sw $7, 4($sp) # 4-byte Folded Spill
+; MM-NEXT:  # %bb.4: # %entry
+; MM-NEXT:    sync
+; MM-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MM-NEXT:    addiusp 8
+; MM-NEXT:    jrc $ra
+;
+; MMR6-LABEL: test_umin_8:
+; MMR6:       # %bb.0: # %entry
+; MMR6-NEXT:    addiu $sp, $sp, -8
+; MMR6-NEXT:    .cfi_def_cfa_offset 8
+; MMR6-NEXT:    move $1, $5
+; MMR6-NEXT:    sync
+; MMR6-NEXT:    addiu $2, $zero, -4
+; MMR6-NEXT:    and $2, $4, $2
+; MMR6-NEXT:    andi $3, $4, 3
+; MMR6-NEXT:    xori $3, $3, 3
+; MMR6-NEXT:    sll $3, $3, 3
+; MMR6-NEXT:    ori $4, $zero, 255
+; MMR6-NEXT:    sllv $4, $4, $3
+; MMR6-NEXT:    nor $6, $zero, $4
+; MMR6-NEXT:    sllv $5, $5, $3
+; MMR6-NEXT:  $BB11_1: # %entry
+; MMR6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MMR6-NEXT:    ll $8, 0($2)
+; MMR6-NEXT:    sltu $11, $8, $5
+; MMR6-NEXT:    selnez $9, $8, $11
+; MMR6-NEXT:    seleqz $11, $5, $11
+; MMR6-NEXT:    or $9, $9, $11
+; MMR6-NEXT:    and $9, $9, $4
+; MMR6-NEXT:    and $10, $8, $6
+; MMR6-NEXT:    or $10, $10, $9
+; MMR6-NEXT:    sc $10, 0($2)
+; MMR6-NEXT:    beqc $10, $zero, $BB11_1
+; MMR6-NEXT:  # %bb.2: # %entry
+; MMR6-NEXT:    and $7, $8, $4
+; MMR6-NEXT:    srlv $7, $7, $3
+; MMR6-NEXT:    seh $7, $7
+; MMR6-NEXT:  # %bb.3: # %entry
+; MMR6-NEXT:    sw $7, 4($sp) # 4-byte Folded Spill
+; MMR6-NEXT:  # %bb.4: # %entry
+; MMR6-NEXT:    sync
+; MMR6-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MMR6-NEXT:    addiu $sp, $sp, 8
+; MMR6-NEXT:    jrc $ra
+;
+; MIPSEL-LABEL: test_umin_8:
+; MIPSEL:       # %bb.0: # %entry
+; MIPSEL-NEXT:    addiu $sp, $sp, -8
+; MIPSEL-NEXT:    .cfi_def_cfa_offset 8
+; MIPSEL-NEXT:    move $1, $5
+; MIPSEL-NEXT:    sync
+; MIPSEL-NEXT:    addiu $2, $zero, -4
+; MIPSEL-NEXT:    and $2, $4, $2
+; MIPSEL-NEXT:    andi $3, $4, 3
+; MIPSEL-NEXT:    sll $3, $3, 3
+; MIPSEL-NEXT:    ori $4, $zero, 255
+; MIPSEL-NEXT:    sllv $4, $4, $3
+; MIPSEL-NEXT:    nor $6, $zero, $4
+; MIPSEL-NEXT:    sllv $5, $5, $3
+; MIPSEL-NEXT:  $BB11_1: # %entry
+; MIPSEL-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPSEL-NEXT:    ll $8, 0($2)
+; MIPSEL-NEXT:    and $8, $8, $4
+; MIPSEL-NEXT:    and $5, $5, $4
+; MIPSEL-NEXT:    sltu $11, $8, $5
+; MIPSEL-NEXT:    move $9, $8
+; MIPSEL-NEXT:    movz $9, $5, $11
+; MIPSEL-NEXT:    and $9, $9, $4
+; MIPSEL-NEXT:    and $10, $8, $6
+; MIPSEL-NEXT:    or $10, $10, $9
+; MIPSEL-NEXT:    sc $10, 0($2)
+; MIPSEL-NEXT:    beqz $10, $BB11_1
+; MIPSEL-NEXT:    nop
+; MIPSEL-NEXT:  # %bb.2: # %entry
+; MIPSEL-NEXT:    and $7, $8, $4
+; MIPSEL-NEXT:    srlv $7, $7, $3
+; MIPSEL-NEXT:    seh $7, $7
+; MIPSEL-NEXT:  # %bb.3: # %entry
+; MIPSEL-NEXT:    sw $7, 4($sp) # 4-byte Folded Spill
+; MIPSEL-NEXT:  # %bb.4: # %entry
+; MIPSEL-NEXT:    sync
+; MIPSEL-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MIPSEL-NEXT:    addiu $sp, $sp, 8
+; MIPSEL-NEXT:    jr $ra
+; MIPSEL-NEXT:    nop
+;
+; MIPSELR6-LABEL: test_umin_8:
+; MIPSELR6:       # %bb.0: # %entry
+; MIPSELR6-NEXT:    addiu $sp, $sp, -8
+; MIPSELR6-NEXT:    .cfi_def_cfa_offset 8
+; MIPSELR6-NEXT:    move $1, $5
+; MIPSELR6-NEXT:    sync
+; MIPSELR6-NEXT:    addiu $2, $zero, -4
+; MIPSELR6-NEXT:    and $2, $4, $2
+; MIPSELR6-NEXT:    andi $3, $4, 3
+; MIPSELR6-NEXT:    sll $3, $3, 3
+; MIPSELR6-NEXT:    ori $4, $zero, 255
+; MIPSELR6-NEXT:    sllv $4, $4, $3
+; MIPSELR6-NEXT:    nor $6, $zero, $4
+; MIPSELR6-NEXT:    sllv $5, $5, $3
+; MIPSELR6-NEXT:  $BB11_1: # %entry
+; MIPSELR6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPSELR6-NEXT:    ll $8, 0($2)
+; MIPSELR6-NEXT:    and $8, $8, $4
+; MIPSELR6-NEXT:    and $5, $5, $4
+; MIPSELR6-NEXT:    sltu $11, $8, $5
+; MIPSELR6-NEXT:    selnez $9, $8, $11
+; MIPSELR6-NEXT:    seleqz $11, $5, $11
+; MIPSELR6-NEXT:    or $9, $9, $11
+; MIPSELR6-NEXT:    and $9, $9, $4
+; MIPSELR6-NEXT:    and $10, $8, $6
+; MIPSELR6-NEXT:    or $10, $10, $9
+; MIPSELR6-NEXT:    sc $10, 0($2)
+; MIPSELR6-NEXT:    beqzc $10, $BB11_1
+; MIPSELR6-NEXT:  # %bb.2: # %entry
+; MIPSELR6-NEXT:    and $7, $8, $4
+; MIPSELR6-NEXT:    srlv $7, $7, $3
+; MIPSELR6-NEXT:    seh $7, $7
+; MIPSELR6-NEXT:  # %bb.3: # %entry
+; MIPSELR6-NEXT:    sw $7, 4($sp) # 4-byte Folded Spill
+; MIPSELR6-NEXT:  # %bb.4: # %entry
+; MIPSELR6-NEXT:    sync
+; MIPSELR6-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MIPSELR6-NEXT:    addiu $sp, $sp, 8
+; MIPSELR6-NEXT:    jrc $ra
+;
+; MMEL-LABEL: test_umin_8:
+; MMEL:       # %bb.0: # %entry
+; MMEL-NEXT:    addiu $sp, $sp, -8
+; MMEL-NEXT:    .cfi_def_cfa_offset 8
+; MMEL-NEXT:    move $1, $5
+; MMEL-NEXT:    sync
+; MMEL-NEXT:    addiu $2, $zero, -4
+; MMEL-NEXT:    and $2, $4, $2
+; MMEL-NEXT:    andi $3, $4, 3
+; MMEL-NEXT:    sll $3, $3, 3
+; MMEL-NEXT:    ori $4, $zero, 255
+; MMEL-NEXT:    sllv $4, $4, $3
+; MMEL-NEXT:    nor $6, $zero, $4
+; MMEL-NEXT:    sllv $5, $5, $3
+; MMEL-NEXT:  $BB11_1: # %entry
+; MMEL-NEXT:    # =>This Inner Loop Header: Depth=1
+; MMEL-NEXT:    ll $8, 0($2)
+; MMEL-NEXT:    and $8, $8, $4
+; MMEL-NEXT:    and $5, $5, $4
+; MMEL-NEXT:    sltu $11, $8, $5
+; MMEL-NEXT:    or $9, $8, $zero
+; MMEL-NEXT:    movz $9, $5, $11
+; MMEL-NEXT:    and $9, $9, $4
+; MMEL-NEXT:    and $10, $8, $6
+; MMEL-NEXT:    or $10, $10, $9
+; MMEL-NEXT:    sc $10, 0($2)
+; MMEL-NEXT:    beqzc $10, $BB11_1
+; MMEL-NEXT:  # %bb.2: # %entry
+; MMEL-NEXT:    and $7, $8, $4
+; MMEL-NEXT:    srlv $7, $7, $3
+; MMEL-NEXT:    seh $7, $7
+; MMEL-NEXT:  # %bb.3: # %entry
+; MMEL-NEXT:    sw $7, 4($sp) # 4-byte Folded Spill
+; MMEL-NEXT:  # %bb.4: # %entry
+; MMEL-NEXT:    sync
+; MMEL-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MMEL-NEXT:    addiusp 8
+; MMEL-NEXT:    jrc $ra
+;
+; MMELR6-LABEL: test_umin_8:
+; MMELR6:       # %bb.0: # %entry
+; MMELR6-NEXT:    addiu $sp, $sp, -8
+; MMELR6-NEXT:    .cfi_def_cfa_offset 8
+; MMELR6-NEXT:    move $1, $5
+; MMELR6-NEXT:    sync
+; MMELR6-NEXT:    addiu $2, $zero, -4
+; MMELR6-NEXT:    and $2, $4, $2
+; MMELR6-NEXT:    andi $3, $4, 3
+; MMELR6-NEXT:    sll $3, $3, 3
+; MMELR6-NEXT:    ori $4, $zero, 255
+; MMELR6-NEXT:    sllv $4, $4, $3
+; MMELR6-NEXT:    nor $6, $zero, $4
+; MMELR6-NEXT:    sllv $5, $5, $3
+; MMELR6-NEXT:  $BB11_1: # %entry
+; MMELR6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MMELR6-NEXT:    ll $8, 0($2)
+; MMELR6-NEXT:    and $8, $8, $4
+; MMELR6-NEXT:    and $5, $5, $4
+; MMELR6-NEXT:    sltu $11, $8, $5
+; MMELR6-NEXT:    selnez $9, $8, $11
+; MMELR6-NEXT:    seleqz $11, $5, $11
+; MMELR6-NEXT:    or $9, $9, $11
+; MMELR6-NEXT:    and $9, $9, $4
+; MMELR6-NEXT:    and $10, $8, $6
+; MMELR6-NEXT:    or $10, $10, $9
+; MMELR6-NEXT:    sc $10, 0($2)
+; MMELR6-NEXT:    beqc $10, $zero, $BB11_1
+; MMELR6-NEXT:  # %bb.2: # %entry
+; MMELR6-NEXT:    and $7, $8, $4
+; MMELR6-NEXT:    srlv $7, $7, $3
+; MMELR6-NEXT:    seh $7, $7
+; MMELR6-NEXT:  # %bb.3: # %entry
+; MMELR6-NEXT:    sw $7, 4($sp) # 4-byte Folded Spill
+; MMELR6-NEXT:  # %bb.4: # %entry
+; MMELR6-NEXT:    sync
+; MMELR6-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MMELR6-NEXT:    addiu $sp, $sp, 8
+; MMELR6-NEXT:    jrc $ra
+;
+; MIPS64-LABEL: test_umin_8:
+; MIPS64:       # %bb.0: # %entry
+; MIPS64-NEXT:    daddiu $sp, $sp, -16
+; MIPS64-NEXT:    .cfi_def_cfa_offset 16
+; MIPS64-NEXT:    # kill: def $a1 killed $a1 killed $a1_64
+; MIPS64-NEXT:    sync
+; MIPS64-NEXT:    daddiu $1, $zero, -4
+; MIPS64-NEXT:    and $1, $4, $1
+; MIPS64-NEXT:    andi $2, $4, 3
+; MIPS64-NEXT:    xori $2, $2, 3
+; MIPS64-NEXT:    sll $2, $2, 3
+; MIPS64-NEXT:    ori $3, $zero, 255
+; MIPS64-NEXT:    sllv $3, $3, $2
+; MIPS64-NEXT:    nor $6, $zero, $3
+; MIPS64-NEXT:    sllv $5, $5, $2
+; MIPS64-NEXT:  .LBB11_1: # %entry
+; MIPS64-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS64-NEXT:    ll $8, 0($1)
+; MIPS64-NEXT:    sltu $11, $8, $5
+; MIPS64-NEXT:    move $9, $8
+; MIPS64-NEXT:    movz $9, $5, $11
+; MIPS64-NEXT:    and $9, $9, $3
+; MIPS64-NEXT:    and $10, $8, $6
+; MIPS64-NEXT:    or $10, $10, $9
+; MIPS64-NEXT:    sc $10, 0($1)
+; MIPS64-NEXT:    beqz $10, .LBB11_1
+; MIPS64-NEXT:    nop
+; MIPS64-NEXT:  # %bb.2: # %entry
+; MIPS64-NEXT:    and $7, $8, $3
+; MIPS64-NEXT:    srlv $7, $7, $2
+; MIPS64-NEXT:    seh $7, $7
+; MIPS64-NEXT:  # %bb.3: # %entry
+; MIPS64-NEXT:    sw $7, 12($sp) # 4-byte Folded Spill
+; MIPS64-NEXT:  # %bb.4: # %entry
+; MIPS64-NEXT:    sync
+; MIPS64-NEXT:    lw $2, 12($sp) # 4-byte Folded Reload
+; MIPS64-NEXT:    daddiu $sp, $sp, 16
+; MIPS64-NEXT:    jr $ra
+; MIPS64-NEXT:    nop
+;
+; MIPS64R6-LABEL: test_umin_8:
+; MIPS64R6:       # %bb.0: # %entry
+; MIPS64R6-NEXT:    daddiu $sp, $sp, -16
+; MIPS64R6-NEXT:    .cfi_def_cfa_offset 16
+; MIPS64R6-NEXT:    # kill: def $a1 killed $a1 killed $a1_64
+; MIPS64R6-NEXT:    sync
+; MIPS64R6-NEXT:    daddiu $1, $zero, -4
+; MIPS64R6-NEXT:    and $1, $4, $1
+; MIPS64R6-NEXT:    andi $2, $4, 3
+; MIPS64R6-NEXT:    xori $2, $2, 3
+; MIPS64R6-NEXT:    sll $2, $2, 3
+; MIPS64R6-NEXT:    ori $3, $zero, 255
+; MIPS64R6-NEXT:    sllv $3, $3, $2
+; MIPS64R6-NEXT:    nor $6, $zero, $3
+; MIPS64R6-NEXT:    sllv $5, $5, $2
+; MIPS64R6-NEXT:  .LBB11_1: # %entry
+; MIPS64R6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS64R6-NEXT:    ll $8, 0($1)
+; MIPS64R6-NEXT:    sltu $11, $8, $5
+; MIPS64R6-NEXT:    selnez $9, $8, $11
+; MIPS64R6-NEXT:    seleqz $11, $5, $11
+; MIPS64R6-NEXT:    or $9, $9, $11
+; MIPS64R6-NEXT:    and $9, $9, $3
+; MIPS64R6-NEXT:    and $10, $8, $6
+; MIPS64R6-NEXT:    or $10, $10, $9
+; MIPS64R6-NEXT:    sc $10, 0($1)
+; MIPS64R6-NEXT:    beqzc $10, .LBB11_1
+; MIPS64R6-NEXT:  # %bb.2: # %entry
+; MIPS64R6-NEXT:    and $7, $8, $3
+; MIPS64R6-NEXT:    srlv $7, $7, $2
+; MIPS64R6-NEXT:    seh $7, $7
+; MIPS64R6-NEXT:  # %bb.3: # %entry
+; MIPS64R6-NEXT:    sw $7, 12($sp) # 4-byte Folded Spill
+; MIPS64R6-NEXT:  # %bb.4: # %entry
+; MIPS64R6-NEXT:    sync
+; MIPS64R6-NEXT:    lw $2, 12($sp) # 4-byte Folded Reload
+; MIPS64R6-NEXT:    daddiu $sp, $sp, 16
+; MIPS64R6-NEXT:    jrc $ra
+;
+; MIPS64EL-LABEL: test_umin_8:
+; MIPS64EL:       # %bb.0: # %entry
+; MIPS64EL-NEXT:    daddiu $sp, $sp, -16
+; MIPS64EL-NEXT:    .cfi_def_cfa_offset 16
+; MIPS64EL-NEXT:    # kill: def $a1 killed $a1 killed $a1_64
+; MIPS64EL-NEXT:    sync
+; MIPS64EL-NEXT:    daddiu $1, $zero, -4
+; MIPS64EL-NEXT:    and $1, $4, $1
+; MIPS64EL-NEXT:    andi $2, $4, 3
+; MIPS64EL-NEXT:    sll $2, $2, 3
+; MIPS64EL-NEXT:    ori $3, $zero, 255
+; MIPS64EL-NEXT:    sllv $3, $3, $2
+; MIPS64EL-NEXT:    nor $6, $zero, $3
+; MIPS64EL-NEXT:    sllv $5, $5, $2
+; MIPS64EL-NEXT:  .LBB11_1: # %entry
+; MIPS64EL-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS64EL-NEXT:    ll $8, 0($1)
+; MIPS64EL-NEXT:    and $8, $8, $3
+; MIPS64EL-NEXT:    and $5, $5, $3
+; MIPS64EL-NEXT:    sltu $11, $8, $5
+; MIPS64EL-NEXT:    move $9, $8
+; MIPS64EL-NEXT:    movz $9, $5, $11
+; MIPS64EL-NEXT:    and $9, $9, $3
+; MIPS64EL-NEXT:    and $10, $8, $6
+; MIPS64EL-NEXT:    or $10, $10, $9
+; MIPS64EL-NEXT:    sc $10, 0($1)
+; MIPS64EL-NEXT:    beqz $10, .LBB11_1
+; MIPS64EL-NEXT:    nop
+; MIPS64EL-NEXT:  # %bb.2: # %entry
+; MIPS64EL-NEXT:    and $7, $8, $3
+; MIPS64EL-NEXT:    srlv $7, $7, $2
+; MIPS64EL-NEXT:    seh $7, $7
+; MIPS64EL-NEXT:  # %bb.3: # %entry
+; MIPS64EL-NEXT:    sw $7, 12($sp) # 4-byte Folded Spill
+; MIPS64EL-NEXT:  # %bb.4: # %entry
+; MIPS64EL-NEXT:    sync
+; MIPS64EL-NEXT:    lw $2, 12($sp) # 4-byte Folded Reload
+; MIPS64EL-NEXT:    daddiu $sp, $sp, 16
+; MIPS64EL-NEXT:    jr $ra
+; MIPS64EL-NEXT:    nop
+;
+; MIPS64ELR6-LABEL: test_umin_8:
+; MIPS64ELR6:       # %bb.0: # %entry
+; MIPS64ELR6-NEXT:    daddiu $sp, $sp, -16
+; MIPS64ELR6-NEXT:    .cfi_def_cfa_offset 16
+; MIPS64ELR6-NEXT:    # kill: def $a1 killed $a1 killed $a1_64
+; MIPS64ELR6-NEXT:    sync
+; MIPS64ELR6-NEXT:    daddiu $1, $zero, -4
+; MIPS64ELR6-NEXT:    and $1, $4, $1
+; MIPS64ELR6-NEXT:    andi $2, $4, 3
+; MIPS64ELR6-NEXT:    sll $2, $2, 3
+; MIPS64ELR6-NEXT:    ori $3, $zero, 255
+; MIPS64ELR6-NEXT:    sllv $3, $3, $2
+; MIPS64ELR6-NEXT:    nor $6, $zero, $3
+; MIPS64ELR6-NEXT:    sllv $5, $5, $2
+; MIPS64ELR6-NEXT:  .LBB11_1: # %entry
+; MIPS64ELR6-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS64ELR6-NEXT:    ll $8, 0($1)
+; MIPS64ELR6-NEXT:    and $8, $8, $3
+; MIPS64ELR6-NEXT:    and $5, $5, $3
+; MIPS64ELR6-NEXT:    sltu $11, $8, $5
+; MIPS64ELR6-NEXT:    selnez $9, $8, $11
+; MIPS64ELR6-NEXT:    seleqz $11, $5, $11
+; MIPS64ELR6-NEXT:    or $9, $9, $11
+; MIPS64ELR6-NEXT:    and $9, $9, $3
+; MIPS64ELR6-NEXT:    and $10, $8, $6
+; MIPS64ELR6-NEXT:    or $10, $10, $9
+; MIPS64ELR6-NEXT:    sc $10, 0($1)
+; MIPS64ELR6-NEXT:    beqzc $10, .LBB11_1
+; MIPS64ELR6-NEXT:  # %bb.2: # %entry
+; MIPS64ELR6-NEXT:    and $7, $8, $3
+; MIPS64ELR6-NEXT:    srlv $7, $7, $2
+; MIPS64ELR6-NEXT:    seh $7, $7
+; MIPS64ELR6-NEXT:  # %bb.3: # %entry
+; MIPS64ELR6-NEXT:    sw $7, 12($sp) # 4-byte Folded Spill
+; MIPS64ELR6-NEXT:  # %bb.4: # %entry
+; MIPS64ELR6-NEXT:    sync
+; MIPS64ELR6-NEXT:    lw $2, 12($sp) # 4-byte Folded Reload
+; MIPS64ELR6-NEXT:    daddiu $sp, $sp, 16
+; MIPS64ELR6-NEXT:    jrc $ra
+entry:
+  %0 = atomicrmw umin i8* %ptr, i8 %val seq_cst
+  ret i8 %0
+}
diff --git a/llvm/test/CodeGen/Mips/atomic.ll b/llvm/test/CodeGen/Mips/atomic.ll
index 85254c46c..3846fda47 100644
--- a/llvm/test/CodeGen/Mips/atomic.ll
+++ b/llvm/test/CodeGen/Mips/atomic.ll
@@ -1790,7 +1790,7 @@ define i32 @AtomicSwap32(i32 signext %newval) nounwind {
 ; MM32-NEXT:  $BB6_1: # %entry
 ; MM32-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MM32-NEXT:    ll $2, 0($1)
-; MM32-NEXT:    move $3, $4
+; MM32-NEXT:    or $3, $4, $zero
 ; MM32-NEXT:    sc $3, 0($1)
 ; MM32-NEXT:    beqzc $3, $BB6_1
 ; MM32-NEXT:  # %bb.2: # %entry
diff --git a/llvm/test/CodeGen/Mips/beqzc.ll b/llvm/test/CodeGen/Mips/beqzc.ll
index c0845f718..ebd16cb82 100644
--- a/llvm/test/CodeGen/Mips/beqzc.ll
+++ b/llvm/test/CodeGen/Mips/beqzc.ll
@@ -14,7 +14,7 @@ entry:
   ret i32 0
 }
 
-attributes #0 = { nounwind optsize "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="true" }
+attributes #0 = { nounwind optsize "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="true" }
 
 
 
diff --git a/llvm/test/CodeGen/Mips/beqzc1.ll b/llvm/test/CodeGen/Mips/beqzc1.ll
index 144983513..488ba9288 100644
--- a/llvm/test/CodeGen/Mips/beqzc1.ll
+++ b/llvm/test/CodeGen/Mips/beqzc1.ll
@@ -19,6 +19,6 @@ if.end:                                           ; preds = %if.then, %entry
   ret i32 0
 }
 
-attributes #0 = { nounwind optsize "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="true" }
+attributes #0 = { nounwind optsize "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="true" }
 
 
diff --git a/llvm/test/CodeGen/Mips/biggot.ll b/llvm/test/CodeGen/Mips/biggot.ll
index 305dcf855..aed1c487e 100644
--- a/llvm/test/CodeGen/Mips/biggot.ll
+++ b/llvm/test/CodeGen/Mips/biggot.ll
@@ -1,9 +1,11 @@
-; RUN: llc -march=mipsel -mxgot -relocation-model=pic < %s | FileCheck %s -check-prefix=O32
-; RUN: llc -march=mips64el -mcpu=mips64r2 -mxgot -relocation-model=pic < %s | \
-; RUN: FileCheck %s -check-prefix=N64
-; RUN: llc -march=mipsel -mxgot -relocation-model=pic -fast-isel < %s | FileCheck %s -check-prefix=O32
-; RUN: llc -march=mips64el -mcpu=mips64r2 -mxgot -relocation-model=pic -fast-isel < %s | \
-; RUN: FileCheck %s -check-prefix=N64
+; RUN: llc -march=mipsel -mattr=+xgot \
+; RUN:     -relocation-model=pic < %s | FileCheck %s -check-prefix=O32
+; RUN: llc -march=mips64el -mcpu=mips64r2 -mattr=+xgot \
+; RUN:     -relocation-model=pic < %s | FileCheck %s -check-prefix=N64
+; RUN: llc -march=mipsel -mattr=+xgot -fast-isel \
+; RUN:     -relocation-model=pic < %s | FileCheck %s -check-prefix=O32
+; RUN: llc -march=mips64el -mcpu=mips64r2 -mattr=+xgot -fast-isel \
+; RUN:     -relocation-model=pic < %s | FileCheck %s -check-prefix=N64
 
 @v0 = external global i32
 
diff --git a/llvm/test/CodeGen/Mips/blockaddr.ll b/llvm/test/CodeGen/Mips/blockaddr.ll
index 80b24bcde..0e924b110 100644
--- a/llvm/test/CodeGen/Mips/blockaddr.ll
+++ b/llvm/test/CodeGen/Mips/blockaddr.ll
@@ -1,11 +1,17 @@
-; RUN: llc -march=mipsel -relocation-model=pic < %s | FileCheck %s -check-prefix=PIC-O32
-; RUN: llc -march=mipsel -relocation-model=static < %s | FileCheck %s -check-prefix=STATIC-O32
-; RUN: llc -march=mips64el -mcpu=mips64r2 -target-abi n32 -relocation-model=pic < %s | FileCheck %s -check-prefix=PIC-N32
-; RUN: llc -march=mips64el -mcpu=mips64r2 -target-abi n32 -relocation-model=static < %s | FileCheck %s -check-prefix=STATIC-N32
-; RUN: llc -march=mips64el -mcpu=mips64r2 -target-abi n64 -relocation-model=pic < %s | FileCheck %s -check-prefix=PIC-N64
-; RUN: llc -march=mips64el -mcpu=mips64r2 -target-abi n64 -relocation-model=static < %s | FileCheck %s -check-prefix=STATIC-N64
-; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips32 -mattr=+mips16 -relocation-model=static < %s | FileCheck %s -check-prefix=STATIC-MIPS16-1
-; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips32 -mattr=+mips16 -relocation-model=static < %s | FileCheck %s -check-prefix=STATIC-MIPS16-2
+; RUN: llc -march=mipsel -relocation-model=pic < %s \
+; RUN:   | FileCheck %s -check-prefix=PIC-O32
+; RUN: llc -march=mipsel -relocation-model=static < %s \
+; RUN:   | FileCheck %s -check-prefix=STATIC-O32
+; RUN: llc -march=mips64el -mcpu=mips64r2 -target-abi n32 \
+; RUN:     -relocation-model=pic < %s | FileCheck %s -check-prefix=PIC-N32
+; RUN: llc -march=mips64el -mcpu=mips64r2 -target-abi n32 \
+; RUN:      -relocation-model=static < %s | FileCheck %s -check-prefix=STATIC-N32
+; RUN: llc -march=mips64el -mcpu=mips64r2 -target-abi n64 \
+; RUN:     -relocation-model=pic < %s | FileCheck %s -check-prefix=PIC-N64
+; RUN: llc -march=mips64el -mcpu=mips64r2 -target-abi n64 \
+; RUN:     -relocation-model=static < %s | FileCheck %s -check-prefix=STATIC-N64
+; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips32 -mattr=+mips16 \
+; RUN:     -relocation-model=static < %s | FileCheck %s -check-prefix=STATIC-MIPS16
 
 @reg = common global i8* null, align 4
 
@@ -18,18 +24,22 @@ entry:
 ; PIC-O32: addiu ${{[0-9]+}}, $[[R0]], %lo($tmp[[T0]])
 ; PIC-O32: lw  $[[R1:[0-9]+]], %got($tmp[[T1:[0-9]+]])
 ; PIC-O32: addiu ${{[0-9]+}}, $[[R1]], %lo($tmp[[T1]])
+
 ; STATIC-O32: lui  $[[R2:[0-9]+]], %hi($tmp[[T2:[0-9]+]])
 ; STATIC-O32: addiu ${{[0-9]+}}, $[[R2]], %lo($tmp[[T2]])
 ; STATIC-O32: lui   $[[R3:[0-9]+]], %hi($tmp[[T3:[0-9]+]])
 ; STATIC-O32: addiu ${{[0-9]+}}, $[[R3]], %lo($tmp[[T3]])
+
 ; PIC-N32: lw  $[[R0:[0-9]+]], %got_page(.Ltmp[[T0:[0-9]+]])
 ; PIC-N32: addiu ${{[0-9]+}}, $[[R0]], %got_ofst(.Ltmp[[T0]])
 ; PIC-N32: lw  $[[R1:[0-9]+]], %got_page(.Ltmp[[T1:[0-9]+]])
 ; PIC-N32: addiu ${{[0-9]+}}, $[[R1]], %got_ofst(.Ltmp[[T1]])
+
 ; STATIC-N32: lui  $[[R2:[0-9]+]], %hi(.Ltmp[[T2:[0-9]+]])
 ; STATIC-N32: addiu ${{[0-9]+}}, $[[R2]], %lo(.Ltmp[[T2]])
 ; STATIC-N32: lui   $[[R3:[0-9]+]], %hi(.Ltmp[[T3:[0-9]+]])
 ; STATIC-N32: addiu ${{[0-9]+}}, $[[R3]], %lo(.Ltmp[[T3]])
+
 ; PIC-N64: ld  $[[R0:[0-9]+]], %got_page(.Ltmp[[T0:[0-9]+]])
 ; PIC-N64: daddiu ${{[0-9]+}}, $[[R0]], %got_ofst(.Ltmp[[T0]])
 ; PIC-N64: ld  $[[R1:[0-9]+]], %got_page(.Ltmp[[T1:[0-9]+]])
@@ -42,13 +52,12 @@ entry:
 ; STATIC-N64: dsll $[[R4:[0-9]]], $[[R3]], 16
 ; STATIC-N64: daddiu $[[R5:[0-9]]], $[[R4]], %lo(.Ltmp[[L0]])
 
-; STATIC-MIPS16-1: .ent	f
-; STATIC-MIPS16-2: .ent	f
-; STATIC-MIPS16-1: li  $[[R1_16:[0-9]+]], %hi($tmp[[TI_16:[0-9]+]])
-; STATIC-MIPS16-1: sll ${{[0-9]+}},  $[[R1_16]], 16
-; STATIC-MIPS16-2: li  ${{[0-9]+}}, %lo($tmp{{[0-9]+}})
-; STATIC-MIPS16-1: jal	dummy
-; STATIC-MIPS16-2: jal	dummy
+; STATIC-MIPS16: .ent	f
+; STATIC-MIPS16: li   $[[R0:[0-9]+]], %hi($tmp[[L0:[0-9]+]])
+; STATIC-MIPS16: sll  $[[R1:[0-9]+]], $[[R0]], 16
+; STATIC-MIPS16: li   $[[R2:[0-9]+]], %lo($tmp[[L0]])
+; STATIC-MIPS16: addu $[[R3:[0-9]+]], $[[R1]], $[[R2]]
+; STATIC-MIPS16: jal	dummy
 
 define void @f() nounwind {
 entry:
diff --git a/llvm/test/CodeGen/Mips/branch-relaxation-with-hazard.ll b/llvm/test/CodeGen/Mips/branch-relaxation-with-hazard.ll
index eb9ee7e9d..530c3a7d2 100644
--- a/llvm/test/CodeGen/Mips/branch-relaxation-with-hazard.ll
+++ b/llvm/test/CodeGen/Mips/branch-relaxation-with-hazard.ll
@@ -38,11 +38,13 @@ entry:
   br i1 %cmp, label %if.then, label %if.end4
 
 if.then:
+  call void asm sideeffect ".space 10", "~{$1}"()
   %1 = load i32, i32* %argc.addr, align 4
   %cmp1 = icmp sgt i32 %1, 3
   br i1 %cmp1, label %if.then2, label %if.end
 
 if.then2:
+  call void asm sideeffect ".space 10", "~{$1}"()
   %call = call i32 bitcast (i32 (...)* @boo to i32 ()*)()
   store i32 %call, i32* %retval, align 4
   br label %return
diff --git a/llvm/test/CodeGen/Mips/brsize3.ll b/llvm/test/CodeGen/Mips/brsize3.ll
index ce8b8f6e3..a93de3269 100644
--- a/llvm/test/CodeGen/Mips/brsize3.ll
+++ b/llvm/test/CodeGen/Mips/brsize3.ll
@@ -33,7 +33,7 @@ x:                                                ; preds = %x, %entry
 
 }
 
-attributes #0 = { noreturn nounwind optsize "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="true" }
+attributes #0 = { noreturn nounwind optsize "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="true" }
 attributes #1 = { nounwind }
 
 !1 = !{i32 45}
diff --git a/llvm/test/CodeGen/Mips/brsize3a.ll b/llvm/test/CodeGen/Mips/brsize3a.ll
index 24516018b..87f962030 100644
--- a/llvm/test/CodeGen/Mips/brsize3a.ll
+++ b/llvm/test/CodeGen/Mips/brsize3a.ll
@@ -20,7 +20,7 @@ x:                                                ; preds = %x, %entry
 
 }
 
-attributes #0 = { noreturn nounwind optsize "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="true" }
+attributes #0 = { noreturn nounwind optsize "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="true" }
 attributes #1 = { nounwind }
 
 !1 = !{i32 45}
diff --git a/llvm/test/CodeGen/Mips/cconv/vector.ll b/llvm/test/CodeGen/Mips/cconv/vector.ll
index 5c7c3f424..4a1f9cb6a 100644
--- a/llvm/test/CodeGen/Mips/cconv/vector.ll
+++ b/llvm/test/CodeGen/Mips/cconv/vector.ll
@@ -50,23 +50,25 @@ define <2 x i8> @i8_2(<2 x i8> %a, <2 x i8> %b) {
 ;
 ; MIPS32R5EB-LABEL: i8_2:
 ; MIPS32R5EB:       # %bb.0:
-; MIPS32R5EB-NEXT:    addiu $sp, $sp, -48
-; MIPS32R5EB-NEXT:    .cfi_def_cfa_offset 48
-; MIPS32R5EB-NEXT:    sw $fp, 44($sp) # 4-byte Folded Spill
-; MIPS32R5EB-NEXT:    .cfi_offset 30, -4
+; MIPS32R5EB-NEXT:    addiu $sp, $sp, -64
+; MIPS32R5EB-NEXT:    .cfi_def_cfa_offset 64
+; MIPS32R5EB-NEXT:    sw $ra, 60($sp) # 4-byte Folded Spill
+; MIPS32R5EB-NEXT:    sw $fp, 56($sp) # 4-byte Folded Spill
+; MIPS32R5EB-NEXT:    .cfi_offset 31, -4
+; MIPS32R5EB-NEXT:    .cfi_offset 30, -8
 ; MIPS32R5EB-NEXT:    move $fp, $sp
 ; MIPS32R5EB-NEXT:    .cfi_def_cfa_register 30
 ; MIPS32R5EB-NEXT:    addiu $1, $zero, -16
 ; MIPS32R5EB-NEXT:    and $sp, $sp, $1
-; MIPS32R5EB-NEXT:    sw $5, 36($sp)
-; MIPS32R5EB-NEXT:    sw $4, 40($sp)
-; MIPS32R5EB-NEXT:    lbu $1, 37($sp)
+; MIPS32R5EB-NEXT:    sw $5, 48($sp)
+; MIPS32R5EB-NEXT:    sw $4, 52($sp)
+; MIPS32R5EB-NEXT:    lbu $1, 49($sp)
 ; MIPS32R5EB-NEXT:    sw $1, 28($sp)
-; MIPS32R5EB-NEXT:    lbu $1, 36($sp)
+; MIPS32R5EB-NEXT:    lbu $1, 48($sp)
 ; MIPS32R5EB-NEXT:    sw $1, 20($sp)
-; MIPS32R5EB-NEXT:    lbu $1, 41($sp)
+; MIPS32R5EB-NEXT:    lbu $1, 53($sp)
 ; MIPS32R5EB-NEXT:    sw $1, 12($sp)
-; MIPS32R5EB-NEXT:    lbu $1, 40($sp)
+; MIPS32R5EB-NEXT:    lbu $1, 52($sp)
 ; MIPS32R5EB-NEXT:    sw $1, 4($sp)
 ; MIPS32R5EB-NEXT:    ld.d $w0, 16($sp)
 ; MIPS32R5EB-NEXT:    ld.d $w1, 0($sp)
@@ -74,12 +76,13 @@ define <2 x i8> @i8_2(<2 x i8> %a, <2 x i8> %b) {
 ; MIPS32R5EB-NEXT:    shf.w $w0, $w0, 177
 ; MIPS32R5EB-NEXT:    copy_s.w $1, $w0[1]
 ; MIPS32R5EB-NEXT:    copy_s.w $2, $w0[3]
-; MIPS32R5EB-NEXT:    sb $2, 33($sp)
-; MIPS32R5EB-NEXT:    sb $1, 32($sp)
-; MIPS32R5EB-NEXT:    lhu $2, 32($sp)
+; MIPS32R5EB-NEXT:    sb $2, 45($sp)
+; MIPS32R5EB-NEXT:    sb $1, 44($sp)
+; MIPS32R5EB-NEXT:    lhu $2, 44($sp)
 ; MIPS32R5EB-NEXT:    move $sp, $fp
-; MIPS32R5EB-NEXT:    lw $fp, 44($sp) # 4-byte Folded Reload
-; MIPS32R5EB-NEXT:    addiu $sp, $sp, 48
+; MIPS32R5EB-NEXT:    lw $fp, 56($sp) # 4-byte Folded Reload
+; MIPS32R5EB-NEXT:    lw $ra, 60($sp) # 4-byte Folded Reload
+; MIPS32R5EB-NEXT:    addiu $sp, $sp, 64
 ; MIPS32R5EB-NEXT:    jr $ra
 ; MIPS32R5EB-NEXT:    nop
 ;
@@ -151,35 +154,38 @@ define <2 x i8> @i8_2(<2 x i8> %a, <2 x i8> %b) {
 ;
 ; MIPS32R5EL-LABEL: i8_2:
 ; MIPS32R5EL:       # %bb.0:
-; MIPS32R5EL-NEXT:    addiu $sp, $sp, -48
-; MIPS32R5EL-NEXT:    .cfi_def_cfa_offset 48
-; MIPS32R5EL-NEXT:    sw $fp, 44($sp) # 4-byte Folded Spill
-; MIPS32R5EL-NEXT:    .cfi_offset 30, -4
+; MIPS32R5EL-NEXT:    addiu $sp, $sp, -64
+; MIPS32R5EL-NEXT:    .cfi_def_cfa_offset 64
+; MIPS32R5EL-NEXT:    sw $ra, 60($sp) # 4-byte Folded Spill
+; MIPS32R5EL-NEXT:    sw $fp, 56($sp) # 4-byte Folded Spill
+; MIPS32R5EL-NEXT:    .cfi_offset 31, -4
+; MIPS32R5EL-NEXT:    .cfi_offset 30, -8
 ; MIPS32R5EL-NEXT:    move $fp, $sp
 ; MIPS32R5EL-NEXT:    .cfi_def_cfa_register 30
 ; MIPS32R5EL-NEXT:    addiu $1, $zero, -16
 ; MIPS32R5EL-NEXT:    and $sp, $sp, $1
-; MIPS32R5EL-NEXT:    sw $5, 36($sp)
-; MIPS32R5EL-NEXT:    sw $4, 40($sp)
-; MIPS32R5EL-NEXT:    lbu $1, 37($sp)
+; MIPS32R5EL-NEXT:    sw $5, 48($sp)
+; MIPS32R5EL-NEXT:    sw $4, 52($sp)
+; MIPS32R5EL-NEXT:    lbu $1, 49($sp)
 ; MIPS32R5EL-NEXT:    sw $1, 24($sp)
-; MIPS32R5EL-NEXT:    lbu $1, 36($sp)
+; MIPS32R5EL-NEXT:    lbu $1, 48($sp)
 ; MIPS32R5EL-NEXT:    sw $1, 16($sp)
-; MIPS32R5EL-NEXT:    lbu $1, 41($sp)
+; MIPS32R5EL-NEXT:    lbu $1, 53($sp)
 ; MIPS32R5EL-NEXT:    sw $1, 8($sp)
-; MIPS32R5EL-NEXT:    lbu $1, 40($sp)
+; MIPS32R5EL-NEXT:    lbu $1, 52($sp)
 ; MIPS32R5EL-NEXT:    sw $1, 0($sp)
 ; MIPS32R5EL-NEXT:    ld.d $w0, 16($sp)
 ; MIPS32R5EL-NEXT:    ld.d $w1, 0($sp)
 ; MIPS32R5EL-NEXT:    addv.d $w0, $w1, $w0
 ; MIPS32R5EL-NEXT:    copy_s.w $1, $w0[0]
 ; MIPS32R5EL-NEXT:    copy_s.w $2, $w0[2]
-; MIPS32R5EL-NEXT:    sb $2, 33($sp)
-; MIPS32R5EL-NEXT:    sb $1, 32($sp)
-; MIPS32R5EL-NEXT:    lhu $2, 32($sp)
+; MIPS32R5EL-NEXT:    sb $2, 45($sp)
+; MIPS32R5EL-NEXT:    sb $1, 44($sp)
+; MIPS32R5EL-NEXT:    lhu $2, 44($sp)
 ; MIPS32R5EL-NEXT:    move $sp, $fp
-; MIPS32R5EL-NEXT:    lw $fp, 44($sp) # 4-byte Folded Reload
-; MIPS32R5EL-NEXT:    addiu $sp, $sp, 48
+; MIPS32R5EL-NEXT:    lw $fp, 56($sp) # 4-byte Folded Reload
+; MIPS32R5EL-NEXT:    lw $ra, 60($sp) # 4-byte Folded Reload
+; MIPS32R5EL-NEXT:    addiu $sp, $sp, 64
 ; MIPS32R5EL-NEXT:    jr $ra
 ; MIPS32R5EL-NEXT:    nop
 ;
@@ -312,36 +318,38 @@ define <2 x i8> @i8x2_7(<2 x i8> %a, <2 x i8> %b, <2 x i8> %c, <2 x i8> %d, <2 x
 ; MIPS32R5EB:       # %bb.0: # %entry
 ; MIPS32R5EB-NEXT:    addiu $sp, $sp, -144
 ; MIPS32R5EB-NEXT:    .cfi_def_cfa_offset 144
-; MIPS32R5EB-NEXT:    sw $fp, 140($sp) # 4-byte Folded Spill
-; MIPS32R5EB-NEXT:    .cfi_offset 30, -4
+; MIPS32R5EB-NEXT:    sw $ra, 140($sp) # 4-byte Folded Spill
+; MIPS32R5EB-NEXT:    sw $fp, 136($sp) # 4-byte Folded Spill
+; MIPS32R5EB-NEXT:    .cfi_offset 31, -4
+; MIPS32R5EB-NEXT:    .cfi_offset 30, -8
 ; MIPS32R5EB-NEXT:    move $fp, $sp
 ; MIPS32R5EB-NEXT:    .cfi_def_cfa_register 30
 ; MIPS32R5EB-NEXT:    addiu $1, $zero, -16
 ; MIPS32R5EB-NEXT:    and $sp, $sp, $1
-; MIPS32R5EB-NEXT:    sw $5, 132($sp)
-; MIPS32R5EB-NEXT:    sw $4, 136($sp)
-; MIPS32R5EB-NEXT:    lbu $1, 133($sp)
+; MIPS32R5EB-NEXT:    sw $5, 128($sp)
+; MIPS32R5EB-NEXT:    sw $4, 132($sp)
+; MIPS32R5EB-NEXT:    lbu $1, 129($sp)
 ; MIPS32R5EB-NEXT:    sw $1, 76($sp)
-; MIPS32R5EB-NEXT:    lbu $1, 132($sp)
+; MIPS32R5EB-NEXT:    lbu $1, 128($sp)
 ; MIPS32R5EB-NEXT:    sw $1, 68($sp)
-; MIPS32R5EB-NEXT:    lbu $1, 137($sp)
+; MIPS32R5EB-NEXT:    lbu $1, 133($sp)
 ; MIPS32R5EB-NEXT:    sw $1, 60($sp)
-; MIPS32R5EB-NEXT:    lbu $1, 136($sp)
+; MIPS32R5EB-NEXT:    lbu $1, 132($sp)
 ; MIPS32R5EB-NEXT:    sw $1, 52($sp)
 ; MIPS32R5EB-NEXT:    ld.d $w0, 64($sp)
 ; MIPS32R5EB-NEXT:    ld.d $w1, 48($sp)
 ; MIPS32R5EB-NEXT:    addv.d $w0, $w1, $w0
-; MIPS32R5EB-NEXT:    sw $6, 128($sp)
-; MIPS32R5EB-NEXT:    lbu $1, 129($sp)
+; MIPS32R5EB-NEXT:    sw $6, 124($sp)
+; MIPS32R5EB-NEXT:    lbu $1, 125($sp)
 ; MIPS32R5EB-NEXT:    sw $1, 92($sp)
-; MIPS32R5EB-NEXT:    lbu $1, 128($sp)
+; MIPS32R5EB-NEXT:    lbu $1, 124($sp)
 ; MIPS32R5EB-NEXT:    sw $1, 84($sp)
 ; MIPS32R5EB-NEXT:    ld.d $w1, 80($sp)
 ; MIPS32R5EB-NEXT:    addv.d $w0, $w0, $w1
-; MIPS32R5EB-NEXT:    sw $7, 124($sp)
-; MIPS32R5EB-NEXT:    lbu $1, 125($sp)
+; MIPS32R5EB-NEXT:    sw $7, 120($sp)
+; MIPS32R5EB-NEXT:    lbu $1, 121($sp)
 ; MIPS32R5EB-NEXT:    sw $1, 108($sp)
-; MIPS32R5EB-NEXT:    lbu $1, 124($sp)
+; MIPS32R5EB-NEXT:    lbu $1, 120($sp)
 ; MIPS32R5EB-NEXT:    sw $1, 100($sp)
 ; MIPS32R5EB-NEXT:    ld.d $w1, 96($sp)
 ; MIPS32R5EB-NEXT:    addv.d $w0, $w0, $w1
@@ -366,11 +374,12 @@ define <2 x i8> @i8x2_7(<2 x i8> %a, <2 x i8> %b, <2 x i8> %c, <2 x i8> %d, <2 x
 ; MIPS32R5EB-NEXT:    shf.w $w0, $w0, 177
 ; MIPS32R5EB-NEXT:    copy_s.w $1, $w0[1]
 ; MIPS32R5EB-NEXT:    copy_s.w $2, $w0[3]
-; MIPS32R5EB-NEXT:    sb $2, 121($sp)
-; MIPS32R5EB-NEXT:    sb $1, 120($sp)
-; MIPS32R5EB-NEXT:    lhu $2, 120($sp)
+; MIPS32R5EB-NEXT:    sb $2, 117($sp)
+; MIPS32R5EB-NEXT:    sb $1, 116($sp)
+; MIPS32R5EB-NEXT:    lhu $2, 116($sp)
 ; MIPS32R5EB-NEXT:    move $sp, $fp
-; MIPS32R5EB-NEXT:    lw $fp, 140($sp) # 4-byte Folded Reload
+; MIPS32R5EB-NEXT:    lw $fp, 136($sp) # 4-byte Folded Reload
+; MIPS32R5EB-NEXT:    lw $ra, 140($sp) # 4-byte Folded Reload
 ; MIPS32R5EB-NEXT:    addiu $sp, $sp, 144
 ; MIPS32R5EB-NEXT:    jr $ra
 ; MIPS32R5EB-NEXT:    nop
@@ -550,36 +559,38 @@ define <2 x i8> @i8x2_7(<2 x i8> %a, <2 x i8> %b, <2 x i8> %c, <2 x i8> %d, <2 x
 ; MIPS32R5EL:       # %bb.0: # %entry
 ; MIPS32R5EL-NEXT:    addiu $sp, $sp, -144
 ; MIPS32R5EL-NEXT:    .cfi_def_cfa_offset 144
-; MIPS32R5EL-NEXT:    sw $fp, 140($sp) # 4-byte Folded Spill
-; MIPS32R5EL-NEXT:    .cfi_offset 30, -4
+; MIPS32R5EL-NEXT:    sw $ra, 140($sp) # 4-byte Folded Spill
+; MIPS32R5EL-NEXT:    sw $fp, 136($sp) # 4-byte Folded Spill
+; MIPS32R5EL-NEXT:    .cfi_offset 31, -4
+; MIPS32R5EL-NEXT:    .cfi_offset 30, -8
 ; MIPS32R5EL-NEXT:    move $fp, $sp
 ; MIPS32R5EL-NEXT:    .cfi_def_cfa_register 30
 ; MIPS32R5EL-NEXT:    addiu $1, $zero, -16
 ; MIPS32R5EL-NEXT:    and $sp, $sp, $1
-; MIPS32R5EL-NEXT:    sw $5, 132($sp)
-; MIPS32R5EL-NEXT:    sw $4, 136($sp)
-; MIPS32R5EL-NEXT:    lbu $1, 133($sp)
+; MIPS32R5EL-NEXT:    sw $5, 128($sp)
+; MIPS32R5EL-NEXT:    sw $4, 132($sp)
+; MIPS32R5EL-NEXT:    lbu $1, 129($sp)
 ; MIPS32R5EL-NEXT:    sw $1, 72($sp)
-; MIPS32R5EL-NEXT:    lbu $1, 132($sp)
+; MIPS32R5EL-NEXT:    lbu $1, 128($sp)
 ; MIPS32R5EL-NEXT:    sw $1, 64($sp)
-; MIPS32R5EL-NEXT:    lbu $1, 137($sp)
+; MIPS32R5EL-NEXT:    lbu $1, 133($sp)
 ; MIPS32R5EL-NEXT:    sw $1, 56($sp)
-; MIPS32R5EL-NEXT:    lbu $1, 136($sp)
+; MIPS32R5EL-NEXT:    lbu $1, 132($sp)
 ; MIPS32R5EL-NEXT:    sw $1, 48($sp)
 ; MIPS32R5EL-NEXT:    ld.d $w0, 64($sp)
 ; MIPS32R5EL-NEXT:    ld.d $w1, 48($sp)
 ; MIPS32R5EL-NEXT:    addv.d $w0, $w1, $w0
-; MIPS32R5EL-NEXT:    sw $6, 128($sp)
-; MIPS32R5EL-NEXT:    lbu $1, 129($sp)
+; MIPS32R5EL-NEXT:    sw $6, 124($sp)
+; MIPS32R5EL-NEXT:    lbu $1, 125($sp)
 ; MIPS32R5EL-NEXT:    sw $1, 88($sp)
-; MIPS32R5EL-NEXT:    lbu $1, 128($sp)
+; MIPS32R5EL-NEXT:    lbu $1, 124($sp)
 ; MIPS32R5EL-NEXT:    sw $1, 80($sp)
 ; MIPS32R5EL-NEXT:    ld.d $w1, 80($sp)
 ; MIPS32R5EL-NEXT:    addv.d $w0, $w0, $w1
-; MIPS32R5EL-NEXT:    sw $7, 124($sp)
-; MIPS32R5EL-NEXT:    lbu $1, 125($sp)
+; MIPS32R5EL-NEXT:    sw $7, 120($sp)
+; MIPS32R5EL-NEXT:    lbu $1, 121($sp)
 ; MIPS32R5EL-NEXT:    sw $1, 104($sp)
-; MIPS32R5EL-NEXT:    lbu $1, 124($sp)
+; MIPS32R5EL-NEXT:    lbu $1, 120($sp)
 ; MIPS32R5EL-NEXT:    sw $1, 96($sp)
 ; MIPS32R5EL-NEXT:    ld.d $w1, 96($sp)
 ; MIPS32R5EL-NEXT:    addv.d $w0, $w0, $w1
@@ -603,11 +614,12 @@ define <2 x i8> @i8x2_7(<2 x i8> %a, <2 x i8> %b, <2 x i8> %c, <2 x i8> %d, <2 x
 ; MIPS32R5EL-NEXT:    addv.d $w0, $w0, $w1
 ; MIPS32R5EL-NEXT:    copy_s.w $1, $w0[0]
 ; MIPS32R5EL-NEXT:    copy_s.w $2, $w0[2]
-; MIPS32R5EL-NEXT:    sb $2, 121($sp)
-; MIPS32R5EL-NEXT:    sb $1, 120($sp)
-; MIPS32R5EL-NEXT:    lhu $2, 120($sp)
+; MIPS32R5EL-NEXT:    sb $2, 117($sp)
+; MIPS32R5EL-NEXT:    sb $1, 116($sp)
+; MIPS32R5EL-NEXT:    lhu $2, 116($sp)
 ; MIPS32R5EL-NEXT:    move $sp, $fp
-; MIPS32R5EL-NEXT:    lw $fp, 140($sp) # 4-byte Folded Reload
+; MIPS32R5EL-NEXT:    lw $fp, 136($sp) # 4-byte Folded Reload
+; MIPS32R5EL-NEXT:    lw $ra, 140($sp) # 4-byte Folded Reload
 ; MIPS32R5EL-NEXT:    addiu $sp, $sp, 144
 ; MIPS32R5EL-NEXT:    jr $ra
 ; MIPS32R5EL-NEXT:    nop
@@ -774,25 +786,23 @@ define <4 x i8> @i8_4(<4 x i8> %a, <4 x i8> %b) {
 ; MIPS32R5-NEXT:    .cfi_def_cfa_offset 16
 ; MIPS32R5-NEXT:    sw $5, 8($sp)
 ; MIPS32R5-NEXT:    sw $4, 12($sp)
-; MIPS32R5-NEXT:    ldi.b $w0, 0
 ; MIPS32R5-NEXT:    lbu $1, 9($sp)
 ; MIPS32R5-NEXT:    lbu $2, 8($sp)
-; MIPS32R5-NEXT:    move.v $w1, $w0
-; MIPS32R5-NEXT:    insert.w $w1[0], $2
-; MIPS32R5-NEXT:    insert.w $w1[1], $1
+; MIPS32R5-NEXT:    insert.w $w0[0], $2
+; MIPS32R5-NEXT:    insert.w $w0[1], $1
 ; MIPS32R5-NEXT:    lbu $1, 10($sp)
-; MIPS32R5-NEXT:    insert.w $w1[2], $1
-; MIPS32R5-NEXT:    lbu $1, 12($sp)
-; MIPS32R5-NEXT:    lbu $2, 11($sp)
-; MIPS32R5-NEXT:    insert.w $w1[3], $2
-; MIPS32R5-NEXT:    insert.w $w0[0], $1
+; MIPS32R5-NEXT:    insert.w $w0[2], $1
+; MIPS32R5-NEXT:    lbu $1, 11($sp)
+; MIPS32R5-NEXT:    insert.w $w0[3], $1
 ; MIPS32R5-NEXT:    lbu $1, 13($sp)
-; MIPS32R5-NEXT:    insert.w $w0[1], $1
+; MIPS32R5-NEXT:    lbu $2, 12($sp)
+; MIPS32R5-NEXT:    insert.w $w1[0], $2
+; MIPS32R5-NEXT:    insert.w $w1[1], $1
 ; MIPS32R5-NEXT:    lbu $1, 14($sp)
-; MIPS32R5-NEXT:    insert.w $w0[2], $1
+; MIPS32R5-NEXT:    insert.w $w1[2], $1
 ; MIPS32R5-NEXT:    lbu $1, 15($sp)
-; MIPS32R5-NEXT:    insert.w $w0[3], $1
-; MIPS32R5-NEXT:    addv.w $w0, $w0, $w1
+; MIPS32R5-NEXT:    insert.w $w1[3], $1
+; MIPS32R5-NEXT:    addv.w $w0, $w1, $w0
 ; MIPS32R5-NEXT:    copy_s.w $1, $w0[0]
 ; MIPS32R5-NEXT:    copy_s.w $2, $w0[1]
 ; MIPS32R5-NEXT:    copy_s.w $3, $w0[2]
@@ -814,25 +824,23 @@ define <4 x i8> @i8_4(<4 x i8> %a, <4 x i8> %b) {
 ; MIPS64R5-NEXT:    sw $1, 8($sp)
 ; MIPS64R5-NEXT:    sll $1, $4, 0
 ; MIPS64R5-NEXT:    sw $1, 12($sp)
-; MIPS64R5-NEXT:    ldi.b $w0, 0
 ; MIPS64R5-NEXT:    lbu $1, 9($sp)
 ; MIPS64R5-NEXT:    lbu $2, 8($sp)
-; MIPS64R5-NEXT:    move.v $w1, $w0
-; MIPS64R5-NEXT:    insert.w $w1[0], $2
-; MIPS64R5-NEXT:    insert.w $w1[1], $1
+; MIPS64R5-NEXT:    insert.w $w0[0], $2
+; MIPS64R5-NEXT:    insert.w $w0[1], $1
 ; MIPS64R5-NEXT:    lbu $1, 10($sp)
-; MIPS64R5-NEXT:    insert.w $w1[2], $1
-; MIPS64R5-NEXT:    lbu $1, 12($sp)
-; MIPS64R5-NEXT:    lbu $2, 11($sp)
-; MIPS64R5-NEXT:    insert.w $w1[3], $2
-; MIPS64R5-NEXT:    insert.w $w0[0], $1
+; MIPS64R5-NEXT:    insert.w $w0[2], $1
+; MIPS64R5-NEXT:    lbu $1, 11($sp)
+; MIPS64R5-NEXT:    insert.w $w0[3], $1
 ; MIPS64R5-NEXT:    lbu $1, 13($sp)
-; MIPS64R5-NEXT:    insert.w $w0[1], $1
+; MIPS64R5-NEXT:    lbu $2, 12($sp)
+; MIPS64R5-NEXT:    insert.w $w1[0], $2
+; MIPS64R5-NEXT:    insert.w $w1[1], $1
 ; MIPS64R5-NEXT:    lbu $1, 14($sp)
-; MIPS64R5-NEXT:    insert.w $w0[2], $1
+; MIPS64R5-NEXT:    insert.w $w1[2], $1
 ; MIPS64R5-NEXT:    lbu $1, 15($sp)
-; MIPS64R5-NEXT:    insert.w $w0[3], $1
-; MIPS64R5-NEXT:    addv.w $w0, $w0, $w1
+; MIPS64R5-NEXT:    insert.w $w1[3], $1
+; MIPS64R5-NEXT:    addv.w $w0, $w1, $w0
 ; MIPS64R5-NEXT:    copy_s.w $1, $w0[0]
 ; MIPS64R5-NEXT:    copy_s.w $2, $w0[1]
 ; MIPS64R5-NEXT:    copy_s.w $3, $w0[2]
@@ -956,51 +964,51 @@ define <8 x i8> @i8_8(<8 x i8> %a, <8 x i8> %b) {
 ; MIPS32R5EB:       # %bb.0:
 ; MIPS32R5EB-NEXT:    addiu $sp, $sp, -48
 ; MIPS32R5EB-NEXT:    .cfi_def_cfa_offset 48
-; MIPS32R5EB-NEXT:    sw $fp, 44($sp) # 4-byte Folded Spill
-; MIPS32R5EB-NEXT:    .cfi_offset 30, -4
+; MIPS32R5EB-NEXT:    sw $ra, 44($sp) # 4-byte Folded Spill
+; MIPS32R5EB-NEXT:    sw $fp, 40($sp) # 4-byte Folded Spill
+; MIPS32R5EB-NEXT:    .cfi_offset 31, -4
+; MIPS32R5EB-NEXT:    .cfi_offset 30, -8
 ; MIPS32R5EB-NEXT:    move $fp, $sp
 ; MIPS32R5EB-NEXT:    .cfi_def_cfa_register 30
 ; MIPS32R5EB-NEXT:    addiu $1, $zero, -16
 ; MIPS32R5EB-NEXT:    and $sp, $sp, $1
 ; MIPS32R5EB-NEXT:    sw $6, 24($sp)
-; MIPS32R5EB-NEXT:    ldi.b $w0, 0
 ; MIPS32R5EB-NEXT:    lbu $1, 25($sp)
 ; MIPS32R5EB-NEXT:    lbu $2, 24($sp)
 ; MIPS32R5EB-NEXT:    sw $7, 28($sp)
-; MIPS32R5EB-NEXT:    move.v $w1, $w0
-; MIPS32R5EB-NEXT:    insert.h $w1[0], $2
-; MIPS32R5EB-NEXT:    insert.h $w1[1], $1
+; MIPS32R5EB-NEXT:    insert.h $w0[0], $2
+; MIPS32R5EB-NEXT:    insert.h $w0[1], $1
 ; MIPS32R5EB-NEXT:    lbu $1, 26($sp)
 ; MIPS32R5EB-NEXT:    sw $4, 32($sp)
-; MIPS32R5EB-NEXT:    insert.h $w1[2], $1
+; MIPS32R5EB-NEXT:    insert.h $w0[2], $1
 ; MIPS32R5EB-NEXT:    lbu $1, 27($sp)
-; MIPS32R5EB-NEXT:    insert.h $w1[3], $1
+; MIPS32R5EB-NEXT:    insert.h $w0[3], $1
 ; MIPS32R5EB-NEXT:    lbu $1, 28($sp)
 ; MIPS32R5EB-NEXT:    sw $5, 36($sp)
-; MIPS32R5EB-NEXT:    insert.h $w1[4], $1
-; MIPS32R5EB-NEXT:    lbu $1, 32($sp)
-; MIPS32R5EB-NEXT:    insert.h $w0[0], $1
+; MIPS32R5EB-NEXT:    insert.h $w0[4], $1
 ; MIPS32R5EB-NEXT:    lbu $1, 33($sp)
-; MIPS32R5EB-NEXT:    insert.h $w0[1], $1
+; MIPS32R5EB-NEXT:    lbu $2, 32($sp)
+; MIPS32R5EB-NEXT:    insert.h $w1[0], $2
+; MIPS32R5EB-NEXT:    insert.h $w1[1], $1
 ; MIPS32R5EB-NEXT:    lbu $1, 29($sp)
 ; MIPS32R5EB-NEXT:    lbu $2, 34($sp)
-; MIPS32R5EB-NEXT:    insert.h $w0[2], $2
-; MIPS32R5EB-NEXT:    insert.h $w1[5], $1
+; MIPS32R5EB-NEXT:    insert.h $w1[2], $2
+; MIPS32R5EB-NEXT:    insert.h $w0[5], $1
 ; MIPS32R5EB-NEXT:    lbu $1, 35($sp)
 ; MIPS32R5EB-NEXT:    lbu $2, 31($sp)
 ; MIPS32R5EB-NEXT:    lbu $3, 30($sp)
 ; MIPS32R5EB-NEXT:    lbu $4, 39($sp)
-; MIPS32R5EB-NEXT:    insert.h $w1[6], $3
-; MIPS32R5EB-NEXT:    insert.h $w1[7], $2
-; MIPS32R5EB-NEXT:    insert.h $w0[3], $1
+; MIPS32R5EB-NEXT:    insert.h $w0[6], $3
+; MIPS32R5EB-NEXT:    insert.h $w0[7], $2
+; MIPS32R5EB-NEXT:    insert.h $w1[3], $1
 ; MIPS32R5EB-NEXT:    lbu $1, 36($sp)
-; MIPS32R5EB-NEXT:    insert.h $w0[4], $1
+; MIPS32R5EB-NEXT:    insert.h $w1[4], $1
 ; MIPS32R5EB-NEXT:    lbu $1, 37($sp)
-; MIPS32R5EB-NEXT:    insert.h $w0[5], $1
+; MIPS32R5EB-NEXT:    insert.h $w1[5], $1
 ; MIPS32R5EB-NEXT:    lbu $1, 38($sp)
-; MIPS32R5EB-NEXT:    insert.h $w0[6], $1
-; MIPS32R5EB-NEXT:    insert.h $w0[7], $4
-; MIPS32R5EB-NEXT:    addv.h $w0, $w0, $w1
+; MIPS32R5EB-NEXT:    insert.h $w1[6], $1
+; MIPS32R5EB-NEXT:    insert.h $w1[7], $4
+; MIPS32R5EB-NEXT:    addv.h $w0, $w1, $w0
 ; MIPS32R5EB-NEXT:    copy_s.h $1, $w0[0]
 ; MIPS32R5EB-NEXT:    copy_s.h $2, $w0[1]
 ; MIPS32R5EB-NEXT:    copy_s.h $3, $w0[2]
@@ -1025,7 +1033,8 @@ define <8 x i8> @i8_8(<8 x i8> %a, <8 x i8> %b) {
 ; MIPS32R5EB-NEXT:    copy_s.w $2, $w0[1]
 ; MIPS32R5EB-NEXT:    copy_s.w $3, $w0[3]
 ; MIPS32R5EB-NEXT:    move $sp, $fp
-; MIPS32R5EB-NEXT:    lw $fp, 44($sp) # 4-byte Folded Reload
+; MIPS32R5EB-NEXT:    lw $fp, 40($sp) # 4-byte Folded Reload
+; MIPS32R5EB-NEXT:    lw $ra, 44($sp) # 4-byte Folded Reload
 ; MIPS32R5EB-NEXT:    addiu $sp, $sp, 48
 ; MIPS32R5EB-NEXT:    jr $ra
 ; MIPS32R5EB-NEXT:    nop
@@ -1035,42 +1044,40 @@ define <8 x i8> @i8_8(<8 x i8> %a, <8 x i8> %b) {
 ; MIPS64R5-NEXT:    daddiu $sp, $sp, -32
 ; MIPS64R5-NEXT:    .cfi_def_cfa_offset 32
 ; MIPS64R5-NEXT:    sd $5, 16($sp)
-; MIPS64R5-NEXT:    ldi.b $w0, 0
 ; MIPS64R5-NEXT:    lbu $1, 17($sp)
 ; MIPS64R5-NEXT:    lbu $2, 16($sp)
 ; MIPS64R5-NEXT:    sd $4, 24($sp)
-; MIPS64R5-NEXT:    move.v $w1, $w0
-; MIPS64R5-NEXT:    insert.h $w1[0], $2
-; MIPS64R5-NEXT:    insert.h $w1[1], $1
+; MIPS64R5-NEXT:    insert.h $w0[0], $2
+; MIPS64R5-NEXT:    insert.h $w0[1], $1
 ; MIPS64R5-NEXT:    lbu $1, 18($sp)
-; MIPS64R5-NEXT:    insert.h $w1[2], $1
+; MIPS64R5-NEXT:    insert.h $w0[2], $1
 ; MIPS64R5-NEXT:    lbu $1, 19($sp)
-; MIPS64R5-NEXT:    insert.h $w1[3], $1
+; MIPS64R5-NEXT:    insert.h $w0[3], $1
 ; MIPS64R5-NEXT:    lbu $1, 20($sp)
-; MIPS64R5-NEXT:    insert.h $w1[4], $1
-; MIPS64R5-NEXT:    lbu $1, 24($sp)
-; MIPS64R5-NEXT:    insert.h $w0[0], $1
+; MIPS64R5-NEXT:    insert.h $w0[4], $1
 ; MIPS64R5-NEXT:    lbu $1, 25($sp)
-; MIPS64R5-NEXT:    insert.h $w0[1], $1
+; MIPS64R5-NEXT:    lbu $2, 24($sp)
+; MIPS64R5-NEXT:    insert.h $w1[0], $2
+; MIPS64R5-NEXT:    insert.h $w1[1], $1
 ; MIPS64R5-NEXT:    lbu $1, 21($sp)
 ; MIPS64R5-NEXT:    lbu $2, 26($sp)
-; MIPS64R5-NEXT:    insert.h $w0[2], $2
-; MIPS64R5-NEXT:    insert.h $w1[5], $1
+; MIPS64R5-NEXT:    insert.h $w1[2], $2
+; MIPS64R5-NEXT:    insert.h $w0[5], $1
 ; MIPS64R5-NEXT:    lbu $1, 27($sp)
 ; MIPS64R5-NEXT:    lbu $2, 23($sp)
 ; MIPS64R5-NEXT:    lbu $3, 22($sp)
 ; MIPS64R5-NEXT:    lbu $4, 31($sp)
-; MIPS64R5-NEXT:    insert.h $w1[6], $3
-; MIPS64R5-NEXT:    insert.h $w1[7], $2
-; MIPS64R5-NEXT:    insert.h $w0[3], $1
+; MIPS64R5-NEXT:    insert.h $w0[6], $3
+; MIPS64R5-NEXT:    insert.h $w0[7], $2
+; MIPS64R5-NEXT:    insert.h $w1[3], $1
 ; MIPS64R5-NEXT:    lbu $1, 28($sp)
-; MIPS64R5-NEXT:    insert.h $w0[4], $1
+; MIPS64R5-NEXT:    insert.h $w1[4], $1
 ; MIPS64R5-NEXT:    lbu $1, 29($sp)
-; MIPS64R5-NEXT:    insert.h $w0[5], $1
+; MIPS64R5-NEXT:    insert.h $w1[5], $1
 ; MIPS64R5-NEXT:    lbu $1, 30($sp)
-; MIPS64R5-NEXT:    insert.h $w0[6], $1
-; MIPS64R5-NEXT:    insert.h $w0[7], $4
-; MIPS64R5-NEXT:    addv.h $w0, $w0, $w1
+; MIPS64R5-NEXT:    insert.h $w1[6], $1
+; MIPS64R5-NEXT:    insert.h $w1[7], $4
+; MIPS64R5-NEXT:    addv.h $w0, $w1, $w0
 ; MIPS64R5-NEXT:    copy_s.h $1, $w0[0]
 ; MIPS64R5-NEXT:    copy_s.h $2, $w0[1]
 ; MIPS64R5-NEXT:    copy_s.h $3, $w0[2]
@@ -1096,51 +1103,51 @@ define <8 x i8> @i8_8(<8 x i8> %a, <8 x i8> %b) {
 ; MIPS32R5EL:       # %bb.0:
 ; MIPS32R5EL-NEXT:    addiu $sp, $sp, -48
 ; MIPS32R5EL-NEXT:    .cfi_def_cfa_offset 48
-; MIPS32R5EL-NEXT:    sw $fp, 44($sp) # 4-byte Folded Spill
-; MIPS32R5EL-NEXT:    .cfi_offset 30, -4
+; MIPS32R5EL-NEXT:    sw $ra, 44($sp) # 4-byte Folded Spill
+; MIPS32R5EL-NEXT:    sw $fp, 40($sp) # 4-byte Folded Spill
+; MIPS32R5EL-NEXT:    .cfi_offset 31, -4
+; MIPS32R5EL-NEXT:    .cfi_offset 30, -8
 ; MIPS32R5EL-NEXT:    move $fp, $sp
 ; MIPS32R5EL-NEXT:    .cfi_def_cfa_register 30
 ; MIPS32R5EL-NEXT:    addiu $1, $zero, -16
 ; MIPS32R5EL-NEXT:    and $sp, $sp, $1
 ; MIPS32R5EL-NEXT:    sw $6, 24($sp)
-; MIPS32R5EL-NEXT:    ldi.b $w0, 0
 ; MIPS32R5EL-NEXT:    lbu $1, 25($sp)
 ; MIPS32R5EL-NEXT:    lbu $2, 24($sp)
 ; MIPS32R5EL-NEXT:    sw $7, 28($sp)
-; MIPS32R5EL-NEXT:    move.v $w1, $w0
-; MIPS32R5EL-NEXT:    insert.h $w1[0], $2
-; MIPS32R5EL-NEXT:    insert.h $w1[1], $1
+; MIPS32R5EL-NEXT:    insert.h $w0[0], $2
+; MIPS32R5EL-NEXT:    insert.h $w0[1], $1
 ; MIPS32R5EL-NEXT:    lbu $1, 26($sp)
 ; MIPS32R5EL-NEXT:    sw $4, 32($sp)
-; MIPS32R5EL-NEXT:    insert.h $w1[2], $1
+; MIPS32R5EL-NEXT:    insert.h $w0[2], $1
 ; MIPS32R5EL-NEXT:    lbu $1, 27($sp)
-; MIPS32R5EL-NEXT:    insert.h $w1[3], $1
+; MIPS32R5EL-NEXT:    insert.h $w0[3], $1
 ; MIPS32R5EL-NEXT:    lbu $1, 28($sp)
 ; MIPS32R5EL-NEXT:    sw $5, 36($sp)
-; MIPS32R5EL-NEXT:    insert.h $w1[4], $1
-; MIPS32R5EL-NEXT:    lbu $1, 32($sp)
-; MIPS32R5EL-NEXT:    insert.h $w0[0], $1
+; MIPS32R5EL-NEXT:    insert.h $w0[4], $1
 ; MIPS32R5EL-NEXT:    lbu $1, 33($sp)
-; MIPS32R5EL-NEXT:    insert.h $w0[1], $1
+; MIPS32R5EL-NEXT:    lbu $2, 32($sp)
+; MIPS32R5EL-NEXT:    insert.h $w1[0], $2
+; MIPS32R5EL-NEXT:    insert.h $w1[1], $1
 ; MIPS32R5EL-NEXT:    lbu $1, 29($sp)
 ; MIPS32R5EL-NEXT:    lbu $2, 34($sp)
-; MIPS32R5EL-NEXT:    insert.h $w0[2], $2
-; MIPS32R5EL-NEXT:    insert.h $w1[5], $1
+; MIPS32R5EL-NEXT:    insert.h $w1[2], $2
+; MIPS32R5EL-NEXT:    insert.h $w0[5], $1
 ; MIPS32R5EL-NEXT:    lbu $1, 35($sp)
 ; MIPS32R5EL-NEXT:    lbu $2, 31($sp)
 ; MIPS32R5EL-NEXT:    lbu $3, 30($sp)
 ; MIPS32R5EL-NEXT:    lbu $4, 39($sp)
-; MIPS32R5EL-NEXT:    insert.h $w1[6], $3
-; MIPS32R5EL-NEXT:    insert.h $w1[7], $2
-; MIPS32R5EL-NEXT:    insert.h $w0[3], $1
+; MIPS32R5EL-NEXT:    insert.h $w0[6], $3
+; MIPS32R5EL-NEXT:    insert.h $w0[7], $2
+; MIPS32R5EL-NEXT:    insert.h $w1[3], $1
 ; MIPS32R5EL-NEXT:    lbu $1, 36($sp)
-; MIPS32R5EL-NEXT:    insert.h $w0[4], $1
+; MIPS32R5EL-NEXT:    insert.h $w1[4], $1
 ; MIPS32R5EL-NEXT:    lbu $1, 37($sp)
-; MIPS32R5EL-NEXT:    insert.h $w0[5], $1
+; MIPS32R5EL-NEXT:    insert.h $w1[5], $1
 ; MIPS32R5EL-NEXT:    lbu $1, 38($sp)
-; MIPS32R5EL-NEXT:    insert.h $w0[6], $1
-; MIPS32R5EL-NEXT:    insert.h $w0[7], $4
-; MIPS32R5EL-NEXT:    addv.h $w0, $w0, $w1
+; MIPS32R5EL-NEXT:    insert.h $w1[6], $1
+; MIPS32R5EL-NEXT:    insert.h $w1[7], $4
+; MIPS32R5EL-NEXT:    addv.h $w0, $w1, $w0
 ; MIPS32R5EL-NEXT:    copy_s.h $1, $w0[0]
 ; MIPS32R5EL-NEXT:    copy_s.h $2, $w0[1]
 ; MIPS32R5EL-NEXT:    copy_s.h $3, $w0[2]
@@ -1165,7 +1172,8 @@ define <8 x i8> @i8_8(<8 x i8> %a, <8 x i8> %b) {
 ; MIPS32R5EL-NEXT:    copy_s.w $2, $w0[0]
 ; MIPS32R5EL-NEXT:    copy_s.w $3, $w0[2]
 ; MIPS32R5EL-NEXT:    move $sp, $fp
-; MIPS32R5EL-NEXT:    lw $fp, 44($sp) # 4-byte Folded Reload
+; MIPS32R5EL-NEXT:    lw $fp, 40($sp) # 4-byte Folded Reload
+; MIPS32R5EL-NEXT:    lw $ra, 44($sp) # 4-byte Folded Reload
 ; MIPS32R5EL-NEXT:    addiu $sp, $sp, 48
 ; MIPS32R5EL-NEXT:    jr $ra
 ; MIPS32R5EL-NEXT:    nop
@@ -1374,23 +1382,21 @@ define <16 x i8> @i8_16(<16 x i8> %a, <16 x i8> %b) {
 ;
 ; MIPS32R5EB-LABEL: i8_16:
 ; MIPS32R5EB:       # %bb.0:
-; MIPS32R5EB-NEXT:    ldi.b $w0, 0
 ; MIPS32R5EB-NEXT:    lw $1, 20($sp)
 ; MIPS32R5EB-NEXT:    lw $2, 16($sp)
-; MIPS32R5EB-NEXT:    move.v $w1, $w0
-; MIPS32R5EB-NEXT:    insert.w $w1[0], $2
-; MIPS32R5EB-NEXT:    insert.w $w1[1], $1
+; MIPS32R5EB-NEXT:    insert.w $w0[0], $2
+; MIPS32R5EB-NEXT:    insert.w $w0[1], $1
 ; MIPS32R5EB-NEXT:    lw $1, 24($sp)
-; MIPS32R5EB-NEXT:    insert.w $w0[0], $4
-; MIPS32R5EB-NEXT:    insert.w $w1[2], $1
+; MIPS32R5EB-NEXT:    insert.w $w0[2], $1
 ; MIPS32R5EB-NEXT:    lw $1, 28($sp)
-; MIPS32R5EB-NEXT:    insert.w $w1[3], $1
-; MIPS32R5EB-NEXT:    shf.b $w1, $w1, 27
-; MIPS32R5EB-NEXT:    insert.w $w0[1], $5
-; MIPS32R5EB-NEXT:    insert.w $w0[2], $6
-; MIPS32R5EB-NEXT:    insert.w $w0[3], $7
+; MIPS32R5EB-NEXT:    insert.w $w0[3], $1
 ; MIPS32R5EB-NEXT:    shf.b $w0, $w0, 27
-; MIPS32R5EB-NEXT:    addv.b $w0, $w0, $w1
+; MIPS32R5EB-NEXT:    insert.w $w1[0], $4
+; MIPS32R5EB-NEXT:    insert.w $w1[1], $5
+; MIPS32R5EB-NEXT:    insert.w $w1[2], $6
+; MIPS32R5EB-NEXT:    insert.w $w1[3], $7
+; MIPS32R5EB-NEXT:    shf.b $w1, $w1, 27
+; MIPS32R5EB-NEXT:    addv.b $w0, $w1, $w0
 ; MIPS32R5EB-NEXT:    shf.b $w0, $w0, 27
 ; MIPS32R5EB-NEXT:    copy_s.w $2, $w0[0]
 ; MIPS32R5EB-NEXT:    copy_s.w $3, $w0[1]
@@ -1401,17 +1407,15 @@ define <16 x i8> @i8_16(<16 x i8> %a, <16 x i8> %b) {
 ;
 ; MIPS64R5EB-LABEL: i8_16:
 ; MIPS64R5EB:       # %bb.0:
-; MIPS64R5EB-NEXT:    ldi.b $w0, 0
-; MIPS64R5EB-NEXT:    move.v $w1, $w0
-; MIPS64R5EB-NEXT:    insert.d $w1[0], $6
-; MIPS64R5EB-NEXT:    insert.d $w1[1], $7
-; MIPS64R5EB-NEXT:    shf.b $w1, $w1, 27
-; MIPS64R5EB-NEXT:    shf.w $w1, $w1, 177
-; MIPS64R5EB-NEXT:    insert.d $w0[0], $4
-; MIPS64R5EB-NEXT:    insert.d $w0[1], $5
+; MIPS64R5EB-NEXT:    insert.d $w0[0], $6
+; MIPS64R5EB-NEXT:    insert.d $w0[1], $7
 ; MIPS64R5EB-NEXT:    shf.b $w0, $w0, 27
 ; MIPS64R5EB-NEXT:    shf.w $w0, $w0, 177
-; MIPS64R5EB-NEXT:    addv.b $w0, $w0, $w1
+; MIPS64R5EB-NEXT:    insert.d $w1[0], $4
+; MIPS64R5EB-NEXT:    insert.d $w1[1], $5
+; MIPS64R5EB-NEXT:    shf.b $w1, $w1, 27
+; MIPS64R5EB-NEXT:    shf.w $w1, $w1, 177
+; MIPS64R5EB-NEXT:    addv.b $w0, $w1, $w0
 ; MIPS64R5EB-NEXT:    shf.b $w0, $w0, 27
 ; MIPS64R5EB-NEXT:    shf.w $w0, $w0, 177
 ; MIPS64R5EB-NEXT:    copy_s.d $2, $w0[0]
@@ -1421,21 +1425,19 @@ define <16 x i8> @i8_16(<16 x i8> %a, <16 x i8> %b) {
 ;
 ; MIPS32R5EL-LABEL: i8_16:
 ; MIPS32R5EL:       # %bb.0:
-; MIPS32R5EL-NEXT:    ldi.b $w0, 0
 ; MIPS32R5EL-NEXT:    lw $1, 20($sp)
 ; MIPS32R5EL-NEXT:    lw $2, 16($sp)
-; MIPS32R5EL-NEXT:    move.v $w1, $w0
-; MIPS32R5EL-NEXT:    insert.w $w1[0], $2
-; MIPS32R5EL-NEXT:    insert.w $w1[1], $1
+; MIPS32R5EL-NEXT:    insert.w $w0[0], $2
+; MIPS32R5EL-NEXT:    insert.w $w0[1], $1
 ; MIPS32R5EL-NEXT:    lw $1, 24($sp)
-; MIPS32R5EL-NEXT:    insert.w $w1[2], $1
+; MIPS32R5EL-NEXT:    insert.w $w0[2], $1
 ; MIPS32R5EL-NEXT:    lw $1, 28($sp)
-; MIPS32R5EL-NEXT:    insert.w $w1[3], $1
-; MIPS32R5EL-NEXT:    insert.w $w0[0], $4
-; MIPS32R5EL-NEXT:    insert.w $w0[1], $5
-; MIPS32R5EL-NEXT:    insert.w $w0[2], $6
-; MIPS32R5EL-NEXT:    insert.w $w0[3], $7
-; MIPS32R5EL-NEXT:    addv.b $w0, $w0, $w1
+; MIPS32R5EL-NEXT:    insert.w $w0[3], $1
+; MIPS32R5EL-NEXT:    insert.w $w1[0], $4
+; MIPS32R5EL-NEXT:    insert.w $w1[1], $5
+; MIPS32R5EL-NEXT:    insert.w $w1[2], $6
+; MIPS32R5EL-NEXT:    insert.w $w1[3], $7
+; MIPS32R5EL-NEXT:    addv.b $w0, $w1, $w0
 ; MIPS32R5EL-NEXT:    copy_s.w $2, $w0[0]
 ; MIPS32R5EL-NEXT:    copy_s.w $3, $w0[1]
 ; MIPS32R5EL-NEXT:    copy_s.w $4, $w0[2]
@@ -1445,13 +1447,11 @@ define <16 x i8> @i8_16(<16 x i8> %a, <16 x i8> %b) {
 ;
 ; MIPS64R5EL-LABEL: i8_16:
 ; MIPS64R5EL:       # %bb.0:
-; MIPS64R5EL-NEXT:    ldi.b $w0, 0
-; MIPS64R5EL-NEXT:    move.v $w1, $w0
-; MIPS64R5EL-NEXT:    insert.d $w1[0], $6
-; MIPS64R5EL-NEXT:    insert.d $w1[1], $7
-; MIPS64R5EL-NEXT:    insert.d $w0[0], $4
-; MIPS64R5EL-NEXT:    insert.d $w0[1], $5
-; MIPS64R5EL-NEXT:    addv.b $w0, $w0, $w1
+; MIPS64R5EL-NEXT:    insert.d $w0[0], $6
+; MIPS64R5EL-NEXT:    insert.d $w0[1], $7
+; MIPS64R5EL-NEXT:    insert.d $w1[0], $4
+; MIPS64R5EL-NEXT:    insert.d $w1[1], $5
+; MIPS64R5EL-NEXT:    addv.b $w0, $w1, $w0
 ; MIPS64R5EL-NEXT:    copy_s.d $2, $w0[0]
 ; MIPS64R5EL-NEXT:    copy_s.d $3, $w0[1]
 ; MIPS64R5EL-NEXT:    jr $ra
@@ -1489,23 +1489,25 @@ define <2 x i16> @i16_2(<2 x i16> %a, <2 x i16> %b) {
 ;
 ; MIPS32R5EB-LABEL: i16_2:
 ; MIPS32R5EB:       # %bb.0:
-; MIPS32R5EB-NEXT:    addiu $sp, $sp, -48
-; MIPS32R5EB-NEXT:    .cfi_def_cfa_offset 48
-; MIPS32R5EB-NEXT:    sw $fp, 44($sp) # 4-byte Folded Spill
-; MIPS32R5EB-NEXT:    .cfi_offset 30, -4
+; MIPS32R5EB-NEXT:    addiu $sp, $sp, -64
+; MIPS32R5EB-NEXT:    .cfi_def_cfa_offset 64
+; MIPS32R5EB-NEXT:    sw $ra, 60($sp) # 4-byte Folded Spill
+; MIPS32R5EB-NEXT:    sw $fp, 56($sp) # 4-byte Folded Spill
+; MIPS32R5EB-NEXT:    .cfi_offset 31, -4
+; MIPS32R5EB-NEXT:    .cfi_offset 30, -8
 ; MIPS32R5EB-NEXT:    move $fp, $sp
 ; MIPS32R5EB-NEXT:    .cfi_def_cfa_register 30
 ; MIPS32R5EB-NEXT:    addiu $1, $zero, -16
 ; MIPS32R5EB-NEXT:    and $sp, $sp, $1
-; MIPS32R5EB-NEXT:    sw $5, 36($sp)
-; MIPS32R5EB-NEXT:    sw $4, 40($sp)
-; MIPS32R5EB-NEXT:    lhu $1, 38($sp)
+; MIPS32R5EB-NEXT:    sw $5, 48($sp)
+; MIPS32R5EB-NEXT:    sw $4, 52($sp)
+; MIPS32R5EB-NEXT:    lhu $1, 50($sp)
 ; MIPS32R5EB-NEXT:    sw $1, 28($sp)
-; MIPS32R5EB-NEXT:    lhu $1, 36($sp)
+; MIPS32R5EB-NEXT:    lhu $1, 48($sp)
 ; MIPS32R5EB-NEXT:    sw $1, 20($sp)
-; MIPS32R5EB-NEXT:    lhu $1, 42($sp)
+; MIPS32R5EB-NEXT:    lhu $1, 54($sp)
 ; MIPS32R5EB-NEXT:    sw $1, 12($sp)
-; MIPS32R5EB-NEXT:    lhu $1, 40($sp)
+; MIPS32R5EB-NEXT:    lhu $1, 52($sp)
 ; MIPS32R5EB-NEXT:    sw $1, 4($sp)
 ; MIPS32R5EB-NEXT:    ld.d $w0, 16($sp)
 ; MIPS32R5EB-NEXT:    ld.d $w1, 0($sp)
@@ -1513,12 +1515,13 @@ define <2 x i16> @i16_2(<2 x i16> %a, <2 x i16> %b) {
 ; MIPS32R5EB-NEXT:    shf.w $w0, $w0, 177
 ; MIPS32R5EB-NEXT:    copy_s.w $1, $w0[1]
 ; MIPS32R5EB-NEXT:    copy_s.w $2, $w0[3]
-; MIPS32R5EB-NEXT:    sh $2, 34($sp)
-; MIPS32R5EB-NEXT:    sh $1, 32($sp)
-; MIPS32R5EB-NEXT:    lw $2, 32($sp)
+; MIPS32R5EB-NEXT:    sh $2, 46($sp)
+; MIPS32R5EB-NEXT:    sh $1, 44($sp)
+; MIPS32R5EB-NEXT:    lw $2, 44($sp)
 ; MIPS32R5EB-NEXT:    move $sp, $fp
-; MIPS32R5EB-NEXT:    lw $fp, 44($sp) # 4-byte Folded Reload
-; MIPS32R5EB-NEXT:    addiu $sp, $sp, 48
+; MIPS32R5EB-NEXT:    lw $fp, 56($sp) # 4-byte Folded Reload
+; MIPS32R5EB-NEXT:    lw $ra, 60($sp) # 4-byte Folded Reload
+; MIPS32R5EB-NEXT:    addiu $sp, $sp, 64
 ; MIPS32R5EB-NEXT:    jr $ra
 ; MIPS32R5EB-NEXT:    nop
 ;
@@ -1530,17 +1533,15 @@ define <2 x i16> @i16_2(<2 x i16> %a, <2 x i16> %b) {
 ; MIPS64R5-NEXT:    sw $1, 8($sp)
 ; MIPS64R5-NEXT:    sll $1, $4, 0
 ; MIPS64R5-NEXT:    sw $1, 12($sp)
-; MIPS64R5-NEXT:    ldi.b $w0, 0
 ; MIPS64R5-NEXT:    lh $1, 10($sp)
 ; MIPS64R5-NEXT:    lh $2, 8($sp)
-; MIPS64R5-NEXT:    move.v $w1, $w0
+; MIPS64R5-NEXT:    insert.d $w0[0], $2
+; MIPS64R5-NEXT:    insert.d $w0[1], $1
+; MIPS64R5-NEXT:    lh $1, 14($sp)
+; MIPS64R5-NEXT:    lh $2, 12($sp)
 ; MIPS64R5-NEXT:    insert.d $w1[0], $2
 ; MIPS64R5-NEXT:    insert.d $w1[1], $1
-; MIPS64R5-NEXT:    lh $1, 12($sp)
-; MIPS64R5-NEXT:    insert.d $w0[0], $1
-; MIPS64R5-NEXT:    lh $1, 14($sp)
-; MIPS64R5-NEXT:    insert.d $w0[1], $1
-; MIPS64R5-NEXT:    addv.d $w0, $w0, $w1
+; MIPS64R5-NEXT:    addv.d $w0, $w1, $w0
 ; MIPS64R5-NEXT:    copy_s.d $1, $w0[0]
 ; MIPS64R5-NEXT:    copy_s.d $2, $w0[1]
 ; MIPS64R5-NEXT:    sh $2, 6($sp)
@@ -1552,35 +1553,38 @@ define <2 x i16> @i16_2(<2 x i16> %a, <2 x i16> %b) {
 ;
 ; MIPS32R5EL-LABEL: i16_2:
 ; MIPS32R5EL:       # %bb.0:
-; MIPS32R5EL-NEXT:    addiu $sp, $sp, -48
-; MIPS32R5EL-NEXT:    .cfi_def_cfa_offset 48
-; MIPS32R5EL-NEXT:    sw $fp, 44($sp) # 4-byte Folded Spill
-; MIPS32R5EL-NEXT:    .cfi_offset 30, -4
+; MIPS32R5EL-NEXT:    addiu $sp, $sp, -64
+; MIPS32R5EL-NEXT:    .cfi_def_cfa_offset 64
+; MIPS32R5EL-NEXT:    sw $ra, 60($sp) # 4-byte Folded Spill
+; MIPS32R5EL-NEXT:    sw $fp, 56($sp) # 4-byte Folded Spill
+; MIPS32R5EL-NEXT:    .cfi_offset 31, -4
+; MIPS32R5EL-NEXT:    .cfi_offset 30, -8
 ; MIPS32R5EL-NEXT:    move $fp, $sp
 ; MIPS32R5EL-NEXT:    .cfi_def_cfa_register 30
 ; MIPS32R5EL-NEXT:    addiu $1, $zero, -16
 ; MIPS32R5EL-NEXT:    and $sp, $sp, $1
-; MIPS32R5EL-NEXT:    sw $5, 36($sp)
-; MIPS32R5EL-NEXT:    sw $4, 40($sp)
-; MIPS32R5EL-NEXT:    lhu $1, 38($sp)
+; MIPS32R5EL-NEXT:    sw $5, 48($sp)
+; MIPS32R5EL-NEXT:    sw $4, 52($sp)
+; MIPS32R5EL-NEXT:    lhu $1, 50($sp)
 ; MIPS32R5EL-NEXT:    sw $1, 24($sp)
-; MIPS32R5EL-NEXT:    lhu $1, 36($sp)
+; MIPS32R5EL-NEXT:    lhu $1, 48($sp)
 ; MIPS32R5EL-NEXT:    sw $1, 16($sp)
-; MIPS32R5EL-NEXT:    lhu $1, 42($sp)
+; MIPS32R5EL-NEXT:    lhu $1, 54($sp)
 ; MIPS32R5EL-NEXT:    sw $1, 8($sp)
-; MIPS32R5EL-NEXT:    lhu $1, 40($sp)
+; MIPS32R5EL-NEXT:    lhu $1, 52($sp)
 ; MIPS32R5EL-NEXT:    sw $1, 0($sp)
 ; MIPS32R5EL-NEXT:    ld.d $w0, 16($sp)
 ; MIPS32R5EL-NEXT:    ld.d $w1, 0($sp)
 ; MIPS32R5EL-NEXT:    addv.d $w0, $w1, $w0
 ; MIPS32R5EL-NEXT:    copy_s.w $1, $w0[0]
 ; MIPS32R5EL-NEXT:    copy_s.w $2, $w0[2]
-; MIPS32R5EL-NEXT:    sh $2, 34($sp)
-; MIPS32R5EL-NEXT:    sh $1, 32($sp)
-; MIPS32R5EL-NEXT:    lw $2, 32($sp)
+; MIPS32R5EL-NEXT:    sh $2, 46($sp)
+; MIPS32R5EL-NEXT:    sh $1, 44($sp)
+; MIPS32R5EL-NEXT:    lw $2, 44($sp)
 ; MIPS32R5EL-NEXT:    move $sp, $fp
-; MIPS32R5EL-NEXT:    lw $fp, 44($sp) # 4-byte Folded Reload
-; MIPS32R5EL-NEXT:    addiu $sp, $sp, 48
+; MIPS32R5EL-NEXT:    lw $fp, 56($sp) # 4-byte Folded Reload
+; MIPS32R5EL-NEXT:    lw $ra, 60($sp) # 4-byte Folded Reload
+; MIPS32R5EL-NEXT:    addiu $sp, $sp, 64
 ; MIPS32R5EL-NEXT:    jr $ra
 ; MIPS32R5EL-NEXT:    nop
   %1 = add <2 x i16> %a, %b
@@ -1642,35 +1646,35 @@ define <4 x i16> @i16_4(<4 x i16> %a, <4 x i16> %b) {
 ; MIPS32R5EB:       # %bb.0:
 ; MIPS32R5EB-NEXT:    addiu $sp, $sp, -48
 ; MIPS32R5EB-NEXT:    .cfi_def_cfa_offset 48
-; MIPS32R5EB-NEXT:    sw $fp, 44($sp) # 4-byte Folded Spill
-; MIPS32R5EB-NEXT:    .cfi_offset 30, -4
+; MIPS32R5EB-NEXT:    sw $ra, 44($sp) # 4-byte Folded Spill
+; MIPS32R5EB-NEXT:    sw $fp, 40($sp) # 4-byte Folded Spill
+; MIPS32R5EB-NEXT:    .cfi_offset 31, -4
+; MIPS32R5EB-NEXT:    .cfi_offset 30, -8
 ; MIPS32R5EB-NEXT:    move $fp, $sp
 ; MIPS32R5EB-NEXT:    .cfi_def_cfa_register 30
 ; MIPS32R5EB-NEXT:    addiu $1, $zero, -16
 ; MIPS32R5EB-NEXT:    and $sp, $sp, $1
 ; MIPS32R5EB-NEXT:    sw $6, 24($sp)
 ; MIPS32R5EB-NEXT:    sw $7, 28($sp)
-; MIPS32R5EB-NEXT:    sw $4, 32($sp)
-; MIPS32R5EB-NEXT:    ldi.b $w0, 0
 ; MIPS32R5EB-NEXT:    lhu $1, 26($sp)
 ; MIPS32R5EB-NEXT:    lhu $2, 24($sp)
-; MIPS32R5EB-NEXT:    move.v $w1, $w0
-; MIPS32R5EB-NEXT:    insert.w $w1[0], $2
-; MIPS32R5EB-NEXT:    insert.w $w1[1], $1
+; MIPS32R5EB-NEXT:    sw $4, 32($sp)
+; MIPS32R5EB-NEXT:    insert.w $w0[0], $2
+; MIPS32R5EB-NEXT:    insert.w $w0[1], $1
 ; MIPS32R5EB-NEXT:    lhu $1, 28($sp)
 ; MIPS32R5EB-NEXT:    sw $5, 36($sp)
-; MIPS32R5EB-NEXT:    insert.w $w1[2], $1
-; MIPS32R5EB-NEXT:    lhu $1, 32($sp)
-; MIPS32R5EB-NEXT:    lhu $2, 30($sp)
-; MIPS32R5EB-NEXT:    insert.w $w1[3], $2
-; MIPS32R5EB-NEXT:    insert.w $w0[0], $1
+; MIPS32R5EB-NEXT:    insert.w $w0[2], $1
+; MIPS32R5EB-NEXT:    lhu $1, 30($sp)
+; MIPS32R5EB-NEXT:    insert.w $w0[3], $1
 ; MIPS32R5EB-NEXT:    lhu $1, 34($sp)
-; MIPS32R5EB-NEXT:    insert.w $w0[1], $1
+; MIPS32R5EB-NEXT:    lhu $2, 32($sp)
+; MIPS32R5EB-NEXT:    insert.w $w1[0], $2
+; MIPS32R5EB-NEXT:    insert.w $w1[1], $1
 ; MIPS32R5EB-NEXT:    lhu $1, 36($sp)
-; MIPS32R5EB-NEXT:    insert.w $w0[2], $1
+; MIPS32R5EB-NEXT:    insert.w $w1[2], $1
 ; MIPS32R5EB-NEXT:    lhu $1, 38($sp)
-; MIPS32R5EB-NEXT:    insert.w $w0[3], $1
-; MIPS32R5EB-NEXT:    addv.w $w0, $w0, $w1
+; MIPS32R5EB-NEXT:    insert.w $w1[3], $1
+; MIPS32R5EB-NEXT:    addv.w $w0, $w1, $w0
 ; MIPS32R5EB-NEXT:    copy_s.w $1, $w0[0]
 ; MIPS32R5EB-NEXT:    copy_s.w $2, $w0[1]
 ; MIPS32R5EB-NEXT:    copy_s.w $3, $w0[2]
@@ -1687,7 +1691,8 @@ define <4 x i16> @i16_4(<4 x i16> %a, <4 x i16> %b) {
 ; MIPS32R5EB-NEXT:    copy_s.w $2, $w0[1]
 ; MIPS32R5EB-NEXT:    copy_s.w $3, $w0[3]
 ; MIPS32R5EB-NEXT:    move $sp, $fp
-; MIPS32R5EB-NEXT:    lw $fp, 44($sp) # 4-byte Folded Reload
+; MIPS32R5EB-NEXT:    lw $fp, 40($sp) # 4-byte Folded Reload
+; MIPS32R5EB-NEXT:    lw $ra, 44($sp) # 4-byte Folded Reload
 ; MIPS32R5EB-NEXT:    addiu $sp, $sp, 48
 ; MIPS32R5EB-NEXT:    jr $ra
 ; MIPS32R5EB-NEXT:    nop
@@ -1698,25 +1703,23 @@ define <4 x i16> @i16_4(<4 x i16> %a, <4 x i16> %b) {
 ; MIPS64R5-NEXT:    .cfi_def_cfa_offset 32
 ; MIPS64R5-NEXT:    sd $5, 16($sp)
 ; MIPS64R5-NEXT:    sd $4, 24($sp)
-; MIPS64R5-NEXT:    ldi.b $w0, 0
 ; MIPS64R5-NEXT:    lhu $1, 18($sp)
 ; MIPS64R5-NEXT:    lhu $2, 16($sp)
-; MIPS64R5-NEXT:    move.v $w1, $w0
-; MIPS64R5-NEXT:    insert.w $w1[0], $2
-; MIPS64R5-NEXT:    insert.w $w1[1], $1
+; MIPS64R5-NEXT:    insert.w $w0[0], $2
+; MIPS64R5-NEXT:    insert.w $w0[1], $1
 ; MIPS64R5-NEXT:    lhu $1, 20($sp)
-; MIPS64R5-NEXT:    insert.w $w1[2], $1
-; MIPS64R5-NEXT:    lhu $1, 24($sp)
-; MIPS64R5-NEXT:    lhu $2, 22($sp)
-; MIPS64R5-NEXT:    insert.w $w1[3], $2
-; MIPS64R5-NEXT:    insert.w $w0[0], $1
+; MIPS64R5-NEXT:    insert.w $w0[2], $1
+; MIPS64R5-NEXT:    lhu $1, 22($sp)
+; MIPS64R5-NEXT:    insert.w $w0[3], $1
 ; MIPS64R5-NEXT:    lhu $1, 26($sp)
-; MIPS64R5-NEXT:    insert.w $w0[1], $1
+; MIPS64R5-NEXT:    lhu $2, 24($sp)
+; MIPS64R5-NEXT:    insert.w $w1[0], $2
+; MIPS64R5-NEXT:    insert.w $w1[1], $1
 ; MIPS64R5-NEXT:    lhu $1, 28($sp)
-; MIPS64R5-NEXT:    insert.w $w0[2], $1
+; MIPS64R5-NEXT:    insert.w $w1[2], $1
 ; MIPS64R5-NEXT:    lhu $1, 30($sp)
-; MIPS64R5-NEXT:    insert.w $w0[3], $1
-; MIPS64R5-NEXT:    addv.w $w0, $w0, $w1
+; MIPS64R5-NEXT:    insert.w $w1[3], $1
+; MIPS64R5-NEXT:    addv.w $w0, $w1, $w0
 ; MIPS64R5-NEXT:    copy_s.w $1, $w0[0]
 ; MIPS64R5-NEXT:    copy_s.w $2, $w0[1]
 ; MIPS64R5-NEXT:    copy_s.w $3, $w0[2]
@@ -1734,35 +1737,35 @@ define <4 x i16> @i16_4(<4 x i16> %a, <4 x i16> %b) {
 ; MIPS32R5EL:       # %bb.0:
 ; MIPS32R5EL-NEXT:    addiu $sp, $sp, -48
 ; MIPS32R5EL-NEXT:    .cfi_def_cfa_offset 48
-; MIPS32R5EL-NEXT:    sw $fp, 44($sp) # 4-byte Folded Spill
-; MIPS32R5EL-NEXT:    .cfi_offset 30, -4
+; MIPS32R5EL-NEXT:    sw $ra, 44($sp) # 4-byte Folded Spill
+; MIPS32R5EL-NEXT:    sw $fp, 40($sp) # 4-byte Folded Spill
+; MIPS32R5EL-NEXT:    .cfi_offset 31, -4
+; MIPS32R5EL-NEXT:    .cfi_offset 30, -8
 ; MIPS32R5EL-NEXT:    move $fp, $sp
 ; MIPS32R5EL-NEXT:    .cfi_def_cfa_register 30
 ; MIPS32R5EL-NEXT:    addiu $1, $zero, -16
 ; MIPS32R5EL-NEXT:    and $sp, $sp, $1
 ; MIPS32R5EL-NEXT:    sw $6, 24($sp)
 ; MIPS32R5EL-NEXT:    sw $7, 28($sp)
-; MIPS32R5EL-NEXT:    sw $4, 32($sp)
-; MIPS32R5EL-NEXT:    ldi.b $w0, 0
 ; MIPS32R5EL-NEXT:    lhu $1, 26($sp)
 ; MIPS32R5EL-NEXT:    lhu $2, 24($sp)
-; MIPS32R5EL-NEXT:    move.v $w1, $w0
-; MIPS32R5EL-NEXT:    insert.w $w1[0], $2
-; MIPS32R5EL-NEXT:    insert.w $w1[1], $1
+; MIPS32R5EL-NEXT:    sw $4, 32($sp)
+; MIPS32R5EL-NEXT:    insert.w $w0[0], $2
+; MIPS32R5EL-NEXT:    insert.w $w0[1], $1
 ; MIPS32R5EL-NEXT:    lhu $1, 28($sp)
 ; MIPS32R5EL-NEXT:    sw $5, 36($sp)
-; MIPS32R5EL-NEXT:    insert.w $w1[2], $1
-; MIPS32R5EL-NEXT:    lhu $1, 32($sp)
-; MIPS32R5EL-NEXT:    lhu $2, 30($sp)
-; MIPS32R5EL-NEXT:    insert.w $w1[3], $2
-; MIPS32R5EL-NEXT:    insert.w $w0[0], $1
+; MIPS32R5EL-NEXT:    insert.w $w0[2], $1
+; MIPS32R5EL-NEXT:    lhu $1, 30($sp)
+; MIPS32R5EL-NEXT:    insert.w $w0[3], $1
 ; MIPS32R5EL-NEXT:    lhu $1, 34($sp)
-; MIPS32R5EL-NEXT:    insert.w $w0[1], $1
+; MIPS32R5EL-NEXT:    lhu $2, 32($sp)
+; MIPS32R5EL-NEXT:    insert.w $w1[0], $2
+; MIPS32R5EL-NEXT:    insert.w $w1[1], $1
 ; MIPS32R5EL-NEXT:    lhu $1, 36($sp)
-; MIPS32R5EL-NEXT:    insert.w $w0[2], $1
+; MIPS32R5EL-NEXT:    insert.w $w1[2], $1
 ; MIPS32R5EL-NEXT:    lhu $1, 38($sp)
-; MIPS32R5EL-NEXT:    insert.w $w0[3], $1
-; MIPS32R5EL-NEXT:    addv.w $w0, $w0, $w1
+; MIPS32R5EL-NEXT:    insert.w $w1[3], $1
+; MIPS32R5EL-NEXT:    addv.w $w0, $w1, $w0
 ; MIPS32R5EL-NEXT:    copy_s.w $1, $w0[0]
 ; MIPS32R5EL-NEXT:    copy_s.w $2, $w0[1]
 ; MIPS32R5EL-NEXT:    copy_s.w $3, $w0[2]
@@ -1779,7 +1782,8 @@ define <4 x i16> @i16_4(<4 x i16> %a, <4 x i16> %b) {
 ; MIPS32R5EL-NEXT:    copy_s.w $2, $w0[0]
 ; MIPS32R5EL-NEXT:    copy_s.w $3, $w0[2]
 ; MIPS32R5EL-NEXT:    move $sp, $fp
-; MIPS32R5EL-NEXT:    lw $fp, 44($sp) # 4-byte Folded Reload
+; MIPS32R5EL-NEXT:    lw $fp, 40($sp) # 4-byte Folded Reload
+; MIPS32R5EL-NEXT:    lw $ra, 44($sp) # 4-byte Folded Reload
 ; MIPS32R5EL-NEXT:    addiu $sp, $sp, 48
 ; MIPS32R5EL-NEXT:    jr $ra
 ; MIPS32R5EL-NEXT:    nop
@@ -1884,23 +1888,21 @@ define <8 x i16> @i16_8(<8 x i16> %a, <8 x i16> %b) {
 ;
 ; MIPS32R5EB-LABEL: i16_8:
 ; MIPS32R5EB:       # %bb.0:
-; MIPS32R5EB-NEXT:    ldi.b $w0, 0
 ; MIPS32R5EB-NEXT:    lw $1, 20($sp)
 ; MIPS32R5EB-NEXT:    lw $2, 16($sp)
-; MIPS32R5EB-NEXT:    move.v $w1, $w0
-; MIPS32R5EB-NEXT:    insert.w $w1[0], $2
-; MIPS32R5EB-NEXT:    insert.w $w1[1], $1
+; MIPS32R5EB-NEXT:    insert.w $w0[0], $2
+; MIPS32R5EB-NEXT:    insert.w $w0[1], $1
 ; MIPS32R5EB-NEXT:    lw $1, 24($sp)
-; MIPS32R5EB-NEXT:    insert.w $w0[0], $4
-; MIPS32R5EB-NEXT:    insert.w $w1[2], $1
+; MIPS32R5EB-NEXT:    insert.w $w0[2], $1
 ; MIPS32R5EB-NEXT:    lw $1, 28($sp)
-; MIPS32R5EB-NEXT:    insert.w $w1[3], $1
-; MIPS32R5EB-NEXT:    shf.h $w1, $w1, 177
-; MIPS32R5EB-NEXT:    insert.w $w0[1], $5
-; MIPS32R5EB-NEXT:    insert.w $w0[2], $6
-; MIPS32R5EB-NEXT:    insert.w $w0[3], $7
+; MIPS32R5EB-NEXT:    insert.w $w0[3], $1
 ; MIPS32R5EB-NEXT:    shf.h $w0, $w0, 177
-; MIPS32R5EB-NEXT:    addv.h $w0, $w0, $w1
+; MIPS32R5EB-NEXT:    insert.w $w1[0], $4
+; MIPS32R5EB-NEXT:    insert.w $w1[1], $5
+; MIPS32R5EB-NEXT:    insert.w $w1[2], $6
+; MIPS32R5EB-NEXT:    insert.w $w1[3], $7
+; MIPS32R5EB-NEXT:    shf.h $w1, $w1, 177
+; MIPS32R5EB-NEXT:    addv.h $w0, $w1, $w0
 ; MIPS32R5EB-NEXT:    shf.h $w0, $w0, 177
 ; MIPS32R5EB-NEXT:    copy_s.w $2, $w0[0]
 ; MIPS32R5EB-NEXT:    copy_s.w $3, $w0[1]
@@ -1911,15 +1913,13 @@ define <8 x i16> @i16_8(<8 x i16> %a, <8 x i16> %b) {
 ;
 ; MIPS64R5EB-LABEL: i16_8:
 ; MIPS64R5EB:       # %bb.0:
-; MIPS64R5EB-NEXT:    ldi.b $w0, 0
-; MIPS64R5EB-NEXT:    move.v $w1, $w0
-; MIPS64R5EB-NEXT:    insert.d $w1[0], $6
-; MIPS64R5EB-NEXT:    insert.d $w1[1], $7
-; MIPS64R5EB-NEXT:    shf.h $w1, $w1, 27
-; MIPS64R5EB-NEXT:    insert.d $w0[0], $4
-; MIPS64R5EB-NEXT:    insert.d $w0[1], $5
+; MIPS64R5EB-NEXT:    insert.d $w0[0], $6
+; MIPS64R5EB-NEXT:    insert.d $w0[1], $7
 ; MIPS64R5EB-NEXT:    shf.h $w0, $w0, 27
-; MIPS64R5EB-NEXT:    addv.h $w0, $w0, $w1
+; MIPS64R5EB-NEXT:    insert.d $w1[0], $4
+; MIPS64R5EB-NEXT:    insert.d $w1[1], $5
+; MIPS64R5EB-NEXT:    shf.h $w1, $w1, 27
+; MIPS64R5EB-NEXT:    addv.h $w0, $w1, $w0
 ; MIPS64R5EB-NEXT:    shf.h $w0, $w0, 27
 ; MIPS64R5EB-NEXT:    copy_s.d $2, $w0[0]
 ; MIPS64R5EB-NEXT:    copy_s.d $3, $w0[1]
@@ -1928,21 +1928,19 @@ define <8 x i16> @i16_8(<8 x i16> %a, <8 x i16> %b) {
 ;
 ; MIPS32R5EL-LABEL: i16_8:
 ; MIPS32R5EL:       # %bb.0:
-; MIPS32R5EL-NEXT:    ldi.b $w0, 0
 ; MIPS32R5EL-NEXT:    lw $1, 20($sp)
 ; MIPS32R5EL-NEXT:    lw $2, 16($sp)
-; MIPS32R5EL-NEXT:    move.v $w1, $w0
-; MIPS32R5EL-NEXT:    insert.w $w1[0], $2
-; MIPS32R5EL-NEXT:    insert.w $w1[1], $1
+; MIPS32R5EL-NEXT:    insert.w $w0[0], $2
+; MIPS32R5EL-NEXT:    insert.w $w0[1], $1
 ; MIPS32R5EL-NEXT:    lw $1, 24($sp)
-; MIPS32R5EL-NEXT:    insert.w $w1[2], $1
+; MIPS32R5EL-NEXT:    insert.w $w0[2], $1
 ; MIPS32R5EL-NEXT:    lw $1, 28($sp)
-; MIPS32R5EL-NEXT:    insert.w $w1[3], $1
-; MIPS32R5EL-NEXT:    insert.w $w0[0], $4
-; MIPS32R5EL-NEXT:    insert.w $w0[1], $5
-; MIPS32R5EL-NEXT:    insert.w $w0[2], $6
-; MIPS32R5EL-NEXT:    insert.w $w0[3], $7
-; MIPS32R5EL-NEXT:    addv.h $w0, $w0, $w1
+; MIPS32R5EL-NEXT:    insert.w $w0[3], $1
+; MIPS32R5EL-NEXT:    insert.w $w1[0], $4
+; MIPS32R5EL-NEXT:    insert.w $w1[1], $5
+; MIPS32R5EL-NEXT:    insert.w $w1[2], $6
+; MIPS32R5EL-NEXT:    insert.w $w1[3], $7
+; MIPS32R5EL-NEXT:    addv.h $w0, $w1, $w0
 ; MIPS32R5EL-NEXT:    copy_s.w $2, $w0[0]
 ; MIPS32R5EL-NEXT:    copy_s.w $3, $w0[1]
 ; MIPS32R5EL-NEXT:    copy_s.w $4, $w0[2]
@@ -1952,13 +1950,11 @@ define <8 x i16> @i16_8(<8 x i16> %a, <8 x i16> %b) {
 ;
 ; MIPS64R5EL-LABEL: i16_8:
 ; MIPS64R5EL:       # %bb.0:
-; MIPS64R5EL-NEXT:    ldi.b $w0, 0
-; MIPS64R5EL-NEXT:    move.v $w1, $w0
-; MIPS64R5EL-NEXT:    insert.d $w1[0], $6
-; MIPS64R5EL-NEXT:    insert.d $w1[1], $7
-; MIPS64R5EL-NEXT:    insert.d $w0[0], $4
-; MIPS64R5EL-NEXT:    insert.d $w0[1], $5
-; MIPS64R5EL-NEXT:    addv.h $w0, $w0, $w1
+; MIPS64R5EL-NEXT:    insert.d $w0[0], $6
+; MIPS64R5EL-NEXT:    insert.d $w0[1], $7
+; MIPS64R5EL-NEXT:    insert.d $w1[0], $4
+; MIPS64R5EL-NEXT:    insert.d $w1[1], $5
+; MIPS64R5EL-NEXT:    addv.h $w0, $w1, $w0
 ; MIPS64R5EL-NEXT:    copy_s.d $2, $w0[0]
 ; MIPS64R5EL-NEXT:    copy_s.d $3, $w0[1]
 ; MIPS64R5EL-NEXT:    jr $ra
@@ -1996,8 +1992,10 @@ define <2 x i32> @i32_2(<2 x i32> %a, <2 x i32> %b) {
 ; MIPS32R5EB:       # %bb.0:
 ; MIPS32R5EB-NEXT:    addiu $sp, $sp, -48
 ; MIPS32R5EB-NEXT:    .cfi_def_cfa_offset 48
-; MIPS32R5EB-NEXT:    sw $fp, 44($sp) # 4-byte Folded Spill
-; MIPS32R5EB-NEXT:    .cfi_offset 30, -4
+; MIPS32R5EB-NEXT:    sw $ra, 44($sp) # 4-byte Folded Spill
+; MIPS32R5EB-NEXT:    sw $fp, 40($sp) # 4-byte Folded Spill
+; MIPS32R5EB-NEXT:    .cfi_offset 31, -4
+; MIPS32R5EB-NEXT:    .cfi_offset 30, -8
 ; MIPS32R5EB-NEXT:    move $fp, $sp
 ; MIPS32R5EB-NEXT:    .cfi_def_cfa_register 30
 ; MIPS32R5EB-NEXT:    addiu $1, $zero, -16
@@ -2013,7 +2011,8 @@ define <2 x i32> @i32_2(<2 x i32> %a, <2 x i32> %b) {
 ; MIPS32R5EB-NEXT:    copy_s.w $2, $w0[1]
 ; MIPS32R5EB-NEXT:    copy_s.w $3, $w0[3]
 ; MIPS32R5EB-NEXT:    move $sp, $fp
-; MIPS32R5EB-NEXT:    lw $fp, 44($sp) # 4-byte Folded Reload
+; MIPS32R5EB-NEXT:    lw $fp, 40($sp) # 4-byte Folded Reload
+; MIPS32R5EB-NEXT:    lw $ra, 44($sp) # 4-byte Folded Reload
 ; MIPS32R5EB-NEXT:    addiu $sp, $sp, 48
 ; MIPS32R5EB-NEXT:    jr $ra
 ; MIPS32R5EB-NEXT:    nop
@@ -2024,15 +2023,13 @@ define <2 x i32> @i32_2(<2 x i32> %a, <2 x i32> %b) {
 ; MIPS64R5EB-NEXT:    .cfi_def_cfa_offset 32
 ; MIPS64R5EB-NEXT:    sd $5, 16($sp)
 ; MIPS64R5EB-NEXT:    sd $4, 24($sp)
-; MIPS64R5EB-NEXT:    ldi.b $w0, 0
 ; MIPS64R5EB-NEXT:    lw $1, 16($sp)
-; MIPS64R5EB-NEXT:    move.v $w1, $w0
-; MIPS64R5EB-NEXT:    insert.d $w1[0], $1
-; MIPS64R5EB-NEXT:    insert.d $w1[1], $5
-; MIPS64R5EB-NEXT:    lw $1, 24($sp)
 ; MIPS64R5EB-NEXT:    insert.d $w0[0], $1
-; MIPS64R5EB-NEXT:    insert.d $w0[1], $4
-; MIPS64R5EB-NEXT:    addv.d $w0, $w0, $w1
+; MIPS64R5EB-NEXT:    insert.d $w0[1], $5
+; MIPS64R5EB-NEXT:    lw $1, 24($sp)
+; MIPS64R5EB-NEXT:    insert.d $w1[0], $1
+; MIPS64R5EB-NEXT:    insert.d $w1[1], $4
+; MIPS64R5EB-NEXT:    addv.d $w0, $w1, $w0
 ; MIPS64R5EB-NEXT:    copy_s.d $1, $w0[0]
 ; MIPS64R5EB-NEXT:    copy_s.d $2, $w0[1]
 ; MIPS64R5EB-NEXT:    sw $2, 12($sp)
@@ -2046,8 +2043,10 @@ define <2 x i32> @i32_2(<2 x i32> %a, <2 x i32> %b) {
 ; MIPS32R5EL:       # %bb.0:
 ; MIPS32R5EL-NEXT:    addiu $sp, $sp, -48
 ; MIPS32R5EL-NEXT:    .cfi_def_cfa_offset 48
-; MIPS32R5EL-NEXT:    sw $fp, 44($sp) # 4-byte Folded Spill
-; MIPS32R5EL-NEXT:    .cfi_offset 30, -4
+; MIPS32R5EL-NEXT:    sw $ra, 44($sp) # 4-byte Folded Spill
+; MIPS32R5EL-NEXT:    sw $fp, 40($sp) # 4-byte Folded Spill
+; MIPS32R5EL-NEXT:    .cfi_offset 31, -4
+; MIPS32R5EL-NEXT:    .cfi_offset 30, -8
 ; MIPS32R5EL-NEXT:    move $fp, $sp
 ; MIPS32R5EL-NEXT:    .cfi_def_cfa_register 30
 ; MIPS32R5EL-NEXT:    addiu $1, $zero, -16
@@ -2062,7 +2061,8 @@ define <2 x i32> @i32_2(<2 x i32> %a, <2 x i32> %b) {
 ; MIPS32R5EL-NEXT:    copy_s.w $2, $w0[0]
 ; MIPS32R5EL-NEXT:    copy_s.w $3, $w0[2]
 ; MIPS32R5EL-NEXT:    move $sp, $fp
-; MIPS32R5EL-NEXT:    lw $fp, 44($sp) # 4-byte Folded Reload
+; MIPS32R5EL-NEXT:    lw $fp, 40($sp) # 4-byte Folded Reload
+; MIPS32R5EL-NEXT:    lw $ra, 44($sp) # 4-byte Folded Reload
 ; MIPS32R5EL-NEXT:    addiu $sp, $sp, 48
 ; MIPS32R5EL-NEXT:    jr $ra
 ; MIPS32R5EL-NEXT:    nop
@@ -2073,15 +2073,13 @@ define <2 x i32> @i32_2(<2 x i32> %a, <2 x i32> %b) {
 ; MIPS64R5EL-NEXT:    .cfi_def_cfa_offset 32
 ; MIPS64R5EL-NEXT:    sd $5, 16($sp)
 ; MIPS64R5EL-NEXT:    sd $4, 24($sp)
-; MIPS64R5EL-NEXT:    ldi.b $w0, 0
 ; MIPS64R5EL-NEXT:    lw $1, 20($sp)
-; MIPS64R5EL-NEXT:    move.v $w1, $w0
-; MIPS64R5EL-NEXT:    insert.d $w1[0], $5
-; MIPS64R5EL-NEXT:    insert.d $w1[1], $1
-; MIPS64R5EL-NEXT:    insert.d $w0[0], $4
-; MIPS64R5EL-NEXT:    lw $1, 28($sp)
+; MIPS64R5EL-NEXT:    insert.d $w0[0], $5
 ; MIPS64R5EL-NEXT:    insert.d $w0[1], $1
-; MIPS64R5EL-NEXT:    addv.d $w0, $w0, $w1
+; MIPS64R5EL-NEXT:    lw $1, 28($sp)
+; MIPS64R5EL-NEXT:    insert.d $w1[0], $4
+; MIPS64R5EL-NEXT:    insert.d $w1[1], $1
+; MIPS64R5EL-NEXT:    addv.d $w0, $w1, $w0
 ; MIPS64R5EL-NEXT:    copy_s.d $1, $w0[0]
 ; MIPS64R5EL-NEXT:    copy_s.d $2, $w0[1]
 ; MIPS64R5EL-NEXT:    sw $2, 12($sp)
@@ -2139,21 +2137,19 @@ define <4 x i32> @i32_4(<4 x i32> %a, <4 x i32> %b) {
 ;
 ; MIPS32R5-LABEL: i32_4:
 ; MIPS32R5:       # %bb.0:
-; MIPS32R5-NEXT:    ldi.b $w0, 0
 ; MIPS32R5-NEXT:    lw $1, 20($sp)
 ; MIPS32R5-NEXT:    lw $2, 16($sp)
-; MIPS32R5-NEXT:    move.v $w1, $w0
-; MIPS32R5-NEXT:    insert.w $w1[0], $2
-; MIPS32R5-NEXT:    insert.w $w1[1], $1
+; MIPS32R5-NEXT:    insert.w $w0[0], $2
+; MIPS32R5-NEXT:    insert.w $w0[1], $1
 ; MIPS32R5-NEXT:    lw $1, 24($sp)
-; MIPS32R5-NEXT:    insert.w $w1[2], $1
+; MIPS32R5-NEXT:    insert.w $w0[2], $1
 ; MIPS32R5-NEXT:    lw $1, 28($sp)
-; MIPS32R5-NEXT:    insert.w $w1[3], $1
-; MIPS32R5-NEXT:    insert.w $w0[0], $4
-; MIPS32R5-NEXT:    insert.w $w0[1], $5
-; MIPS32R5-NEXT:    insert.w $w0[2], $6
-; MIPS32R5-NEXT:    insert.w $w0[3], $7
-; MIPS32R5-NEXT:    addv.w $w0, $w0, $w1
+; MIPS32R5-NEXT:    insert.w $w0[3], $1
+; MIPS32R5-NEXT:    insert.w $w1[0], $4
+; MIPS32R5-NEXT:    insert.w $w1[1], $5
+; MIPS32R5-NEXT:    insert.w $w1[2], $6
+; MIPS32R5-NEXT:    insert.w $w1[3], $7
+; MIPS32R5-NEXT:    addv.w $w0, $w1, $w0
 ; MIPS32R5-NEXT:    copy_s.w $2, $w0[0]
 ; MIPS32R5-NEXT:    copy_s.w $3, $w0[1]
 ; MIPS32R5-NEXT:    copy_s.w $4, $w0[2]
@@ -2163,15 +2159,13 @@ define <4 x i32> @i32_4(<4 x i32> %a, <4 x i32> %b) {
 ;
 ; MIPS64R5EB-LABEL: i32_4:
 ; MIPS64R5EB:       # %bb.0:
-; MIPS64R5EB-NEXT:    ldi.b $w0, 0
-; MIPS64R5EB-NEXT:    move.v $w1, $w0
-; MIPS64R5EB-NEXT:    insert.d $w1[0], $6
-; MIPS64R5EB-NEXT:    insert.d $w1[1], $7
-; MIPS64R5EB-NEXT:    shf.w $w1, $w1, 177
-; MIPS64R5EB-NEXT:    insert.d $w0[0], $4
-; MIPS64R5EB-NEXT:    insert.d $w0[1], $5
+; MIPS64R5EB-NEXT:    insert.d $w0[0], $6
+; MIPS64R5EB-NEXT:    insert.d $w0[1], $7
 ; MIPS64R5EB-NEXT:    shf.w $w0, $w0, 177
-; MIPS64R5EB-NEXT:    addv.w $w0, $w0, $w1
+; MIPS64R5EB-NEXT:    insert.d $w1[0], $4
+; MIPS64R5EB-NEXT:    insert.d $w1[1], $5
+; MIPS64R5EB-NEXT:    shf.w $w1, $w1, 177
+; MIPS64R5EB-NEXT:    addv.w $w0, $w1, $w0
 ; MIPS64R5EB-NEXT:    shf.w $w0, $w0, 177
 ; MIPS64R5EB-NEXT:    copy_s.d $2, $w0[0]
 ; MIPS64R5EB-NEXT:    copy_s.d $3, $w0[1]
@@ -2180,13 +2174,11 @@ define <4 x i32> @i32_4(<4 x i32> %a, <4 x i32> %b) {
 ;
 ; MIPS64R5EL-LABEL: i32_4:
 ; MIPS64R5EL:       # %bb.0:
-; MIPS64R5EL-NEXT:    ldi.b $w0, 0
-; MIPS64R5EL-NEXT:    move.v $w1, $w0
-; MIPS64R5EL-NEXT:    insert.d $w1[0], $6
-; MIPS64R5EL-NEXT:    insert.d $w1[1], $7
-; MIPS64R5EL-NEXT:    insert.d $w0[0], $4
-; MIPS64R5EL-NEXT:    insert.d $w0[1], $5
-; MIPS64R5EL-NEXT:    addv.w $w0, $w0, $w1
+; MIPS64R5EL-NEXT:    insert.d $w0[0], $6
+; MIPS64R5EL-NEXT:    insert.d $w0[1], $7
+; MIPS64R5EL-NEXT:    insert.d $w1[0], $4
+; MIPS64R5EL-NEXT:    insert.d $w1[1], $5
+; MIPS64R5EL-NEXT:    addv.w $w0, $w1, $w0
 ; MIPS64R5EL-NEXT:    copy_s.d $2, $w0[0]
 ; MIPS64R5EL-NEXT:    copy_s.d $3, $w0[1]
 ; MIPS64R5EL-NEXT:    jr $ra
@@ -2222,23 +2214,21 @@ define <2 x i64> @i64_2(<2 x i64> %a, <2 x i64> %b) {
 ;
 ; MIPS32R5EB-LABEL: i64_2:
 ; MIPS32R5EB:       # %bb.0:
-; MIPS32R5EB-NEXT:    ldi.b $w0, 0
 ; MIPS32R5EB-NEXT:    lw $1, 20($sp)
 ; MIPS32R5EB-NEXT:    lw $2, 16($sp)
-; MIPS32R5EB-NEXT:    move.v $w1, $w0
-; MIPS32R5EB-NEXT:    insert.w $w1[0], $2
-; MIPS32R5EB-NEXT:    insert.w $w1[1], $1
+; MIPS32R5EB-NEXT:    insert.w $w0[0], $2
+; MIPS32R5EB-NEXT:    insert.w $w0[1], $1
 ; MIPS32R5EB-NEXT:    lw $1, 24($sp)
-; MIPS32R5EB-NEXT:    insert.w $w0[0], $4
-; MIPS32R5EB-NEXT:    insert.w $w1[2], $1
+; MIPS32R5EB-NEXT:    insert.w $w0[2], $1
 ; MIPS32R5EB-NEXT:    lw $1, 28($sp)
-; MIPS32R5EB-NEXT:    insert.w $w1[3], $1
-; MIPS32R5EB-NEXT:    shf.w $w1, $w1, 177
-; MIPS32R5EB-NEXT:    insert.w $w0[1], $5
-; MIPS32R5EB-NEXT:    insert.w $w0[2], $6
-; MIPS32R5EB-NEXT:    insert.w $w0[3], $7
+; MIPS32R5EB-NEXT:    insert.w $w0[3], $1
 ; MIPS32R5EB-NEXT:    shf.w $w0, $w0, 177
-; MIPS32R5EB-NEXT:    addv.d $w0, $w0, $w1
+; MIPS32R5EB-NEXT:    insert.w $w1[0], $4
+; MIPS32R5EB-NEXT:    insert.w $w1[1], $5
+; MIPS32R5EB-NEXT:    insert.w $w1[2], $6
+; MIPS32R5EB-NEXT:    insert.w $w1[3], $7
+; MIPS32R5EB-NEXT:    shf.w $w1, $w1, 177
+; MIPS32R5EB-NEXT:    addv.d $w0, $w1, $w0
 ; MIPS32R5EB-NEXT:    shf.w $w0, $w0, 177
 ; MIPS32R5EB-NEXT:    copy_s.w $2, $w0[0]
 ; MIPS32R5EB-NEXT:    copy_s.w $3, $w0[1]
@@ -2249,13 +2239,11 @@ define <2 x i64> @i64_2(<2 x i64> %a, <2 x i64> %b) {
 ;
 ; MIPS64R5-LABEL: i64_2:
 ; MIPS64R5:       # %bb.0:
-; MIPS64R5-NEXT:    ldi.b $w0, 0
-; MIPS64R5-NEXT:    move.v $w1, $w0
-; MIPS64R5-NEXT:    insert.d $w1[0], $6
-; MIPS64R5-NEXT:    insert.d $w1[1], $7
-; MIPS64R5-NEXT:    insert.d $w0[0], $4
-; MIPS64R5-NEXT:    insert.d $w0[1], $5
-; MIPS64R5-NEXT:    addv.d $w0, $w0, $w1
+; MIPS64R5-NEXT:    insert.d $w0[0], $6
+; MIPS64R5-NEXT:    insert.d $w0[1], $7
+; MIPS64R5-NEXT:    insert.d $w1[0], $4
+; MIPS64R5-NEXT:    insert.d $w1[1], $5
+; MIPS64R5-NEXT:    addv.d $w0, $w1, $w0
 ; MIPS64R5-NEXT:    copy_s.d $2, $w0[0]
 ; MIPS64R5-NEXT:    copy_s.d $3, $w0[1]
 ; MIPS64R5-NEXT:    jr $ra
@@ -2280,21 +2268,19 @@ define <2 x i64> @i64_2(<2 x i64> %a, <2 x i64> %b) {
 ;
 ; MIPS32R5EL-LABEL: i64_2:
 ; MIPS32R5EL:       # %bb.0:
-; MIPS32R5EL-NEXT:    ldi.b $w0, 0
 ; MIPS32R5EL-NEXT:    lw $1, 20($sp)
 ; MIPS32R5EL-NEXT:    lw $2, 16($sp)
-; MIPS32R5EL-NEXT:    move.v $w1, $w0
-; MIPS32R5EL-NEXT:    insert.w $w1[0], $2
-; MIPS32R5EL-NEXT:    insert.w $w1[1], $1
+; MIPS32R5EL-NEXT:    insert.w $w0[0], $2
+; MIPS32R5EL-NEXT:    insert.w $w0[1], $1
 ; MIPS32R5EL-NEXT:    lw $1, 24($sp)
-; MIPS32R5EL-NEXT:    insert.w $w1[2], $1
+; MIPS32R5EL-NEXT:    insert.w $w0[2], $1
 ; MIPS32R5EL-NEXT:    lw $1, 28($sp)
-; MIPS32R5EL-NEXT:    insert.w $w1[3], $1
-; MIPS32R5EL-NEXT:    insert.w $w0[0], $4
-; MIPS32R5EL-NEXT:    insert.w $w0[1], $5
-; MIPS32R5EL-NEXT:    insert.w $w0[2], $6
-; MIPS32R5EL-NEXT:    insert.w $w0[3], $7
-; MIPS32R5EL-NEXT:    addv.d $w0, $w0, $w1
+; MIPS32R5EL-NEXT:    insert.w $w0[3], $1
+; MIPS32R5EL-NEXT:    insert.w $w1[0], $4
+; MIPS32R5EL-NEXT:    insert.w $w1[1], $5
+; MIPS32R5EL-NEXT:    insert.w $w1[2], $6
+; MIPS32R5EL-NEXT:    insert.w $w1[3], $7
+; MIPS32R5EL-NEXT:    addv.d $w0, $w1, $w0
 ; MIPS32R5EL-NEXT:    copy_s.w $2, $w0[0]
 ; MIPS32R5EL-NEXT:    copy_s.w $3, $w0[1]
 ; MIPS32R5EL-NEXT:    copy_s.w $4, $w0[2]
@@ -2362,8 +2348,10 @@ define void @float_2(<2 x float> %a, <2 x float> %b) {
 ; MIPS32R5:       # %bb.0:
 ; MIPS32R5-NEXT:    addiu $sp, $sp, -48
 ; MIPS32R5-NEXT:    .cfi_def_cfa_offset 48
-; MIPS32R5-NEXT:    sw $fp, 44($sp) # 4-byte Folded Spill
-; MIPS32R5-NEXT:    .cfi_offset 30, -4
+; MIPS32R5-NEXT:    sw $ra, 44($sp) # 4-byte Folded Spill
+; MIPS32R5-NEXT:    sw $fp, 40($sp) # 4-byte Folded Spill
+; MIPS32R5-NEXT:    .cfi_offset 31, -4
+; MIPS32R5-NEXT:    .cfi_offset 30, -8
 ; MIPS32R5-NEXT:    move $fp, $sp
 ; MIPS32R5-NEXT:    .cfi_def_cfa_register 30
 ; MIPS32R5-NEXT:    addiu $1, $zero, -16
@@ -2381,7 +2369,8 @@ define void @float_2(<2 x float> %a, <2 x float> %b) {
 ; MIPS32R5-NEXT:    swc1 $f1, 4($2)
 ; MIPS32R5-NEXT:    swc1 $f0, %lo(float_res_v2f32)($1)
 ; MIPS32R5-NEXT:    move $sp, $fp
-; MIPS32R5-NEXT:    lw $fp, 44($sp) # 4-byte Folded Reload
+; MIPS32R5-NEXT:    lw $fp, 40($sp) # 4-byte Folded Reload
+; MIPS32R5-NEXT:    lw $ra, 44($sp) # 4-byte Folded Reload
 ; MIPS32R5-NEXT:    addiu $sp, $sp, 48
 ; MIPS32R5-NEXT:    jr $ra
 ; MIPS32R5-NEXT:    nop
@@ -2520,21 +2509,19 @@ define void @float_4(<4 x float> %a, <4 x float> %b) {
 ;
 ; MIPS32R5-LABEL: float_4:
 ; MIPS32R5:       # %bb.0:
-; MIPS32R5-NEXT:    ldi.b $w0, 0
 ; MIPS32R5-NEXT:    lw $1, 20($sp)
 ; MIPS32R5-NEXT:    lw $2, 16($sp)
-; MIPS32R5-NEXT:    move.v $w1, $w0
-; MIPS32R5-NEXT:    insert.w $w1[0], $2
-; MIPS32R5-NEXT:    insert.w $w1[1], $1
+; MIPS32R5-NEXT:    insert.w $w0[0], $2
+; MIPS32R5-NEXT:    insert.w $w0[1], $1
 ; MIPS32R5-NEXT:    lw $1, 24($sp)
-; MIPS32R5-NEXT:    insert.w $w1[2], $1
+; MIPS32R5-NEXT:    insert.w $w0[2], $1
 ; MIPS32R5-NEXT:    lw $1, 28($sp)
-; MIPS32R5-NEXT:    insert.w $w1[3], $1
-; MIPS32R5-NEXT:    insert.w $w0[0], $4
-; MIPS32R5-NEXT:    insert.w $w0[1], $5
-; MIPS32R5-NEXT:    insert.w $w0[2], $6
-; MIPS32R5-NEXT:    insert.w $w0[3], $7
-; MIPS32R5-NEXT:    fadd.w $w0, $w0, $w1
+; MIPS32R5-NEXT:    insert.w $w0[3], $1
+; MIPS32R5-NEXT:    insert.w $w1[0], $4
+; MIPS32R5-NEXT:    insert.w $w1[1], $5
+; MIPS32R5-NEXT:    insert.w $w1[2], $6
+; MIPS32R5-NEXT:    insert.w $w1[3], $7
+; MIPS32R5-NEXT:    fadd.w $w0, $w1, $w0
 ; MIPS32R5-NEXT:    lui $1, %hi(float_res_v4f32)
 ; MIPS32R5-NEXT:    addiu $1, $1, %lo(float_res_v4f32)
 ; MIPS32R5-NEXT:    st.w $w0, 0($1)
@@ -2546,15 +2533,13 @@ define void @float_4(<4 x float> %a, <4 x float> %b) {
 ; MIPS64R5EB-NEXT:    lui $1, %hi(%neg(%gp_rel(float_4)))
 ; MIPS64R5EB-NEXT:    daddu $1, $1, $25
 ; MIPS64R5EB-NEXT:    daddiu $1, $1, %lo(%neg(%gp_rel(float_4)))
-; MIPS64R5EB-NEXT:    ldi.b $w0, 0
-; MIPS64R5EB-NEXT:    move.v $w1, $w0
-; MIPS64R5EB-NEXT:    insert.d $w1[0], $6
-; MIPS64R5EB-NEXT:    insert.d $w1[1], $7
-; MIPS64R5EB-NEXT:    shf.w $w1, $w1, 177
-; MIPS64R5EB-NEXT:    insert.d $w0[0], $4
-; MIPS64R5EB-NEXT:    insert.d $w0[1], $5
+; MIPS64R5EB-NEXT:    insert.d $w0[0], $6
+; MIPS64R5EB-NEXT:    insert.d $w0[1], $7
 ; MIPS64R5EB-NEXT:    shf.w $w0, $w0, 177
-; MIPS64R5EB-NEXT:    fadd.w $w0, $w0, $w1
+; MIPS64R5EB-NEXT:    insert.d $w1[0], $4
+; MIPS64R5EB-NEXT:    insert.d $w1[1], $5
+; MIPS64R5EB-NEXT:    shf.w $w1, $w1, 177
+; MIPS64R5EB-NEXT:    fadd.w $w0, $w1, $w0
 ; MIPS64R5EB-NEXT:    ld $1, %got_disp(float_res_v4f32)($1)
 ; MIPS64R5EB-NEXT:    st.w $w0, 0($1)
 ; MIPS64R5EB-NEXT:    jr $ra
@@ -2602,13 +2587,11 @@ define void @float_4(<4 x float> %a, <4 x float> %b) {
 ; MIPS64R5EL-NEXT:    lui $1, %hi(%neg(%gp_rel(float_4)))
 ; MIPS64R5EL-NEXT:    daddu $1, $1, $25
 ; MIPS64R5EL-NEXT:    daddiu $1, $1, %lo(%neg(%gp_rel(float_4)))
-; MIPS64R5EL-NEXT:    ldi.b $w0, 0
-; MIPS64R5EL-NEXT:    move.v $w1, $w0
-; MIPS64R5EL-NEXT:    insert.d $w1[0], $6
-; MIPS64R5EL-NEXT:    insert.d $w1[1], $7
-; MIPS64R5EL-NEXT:    insert.d $w0[0], $4
-; MIPS64R5EL-NEXT:    insert.d $w0[1], $5
-; MIPS64R5EL-NEXT:    fadd.w $w0, $w0, $w1
+; MIPS64R5EL-NEXT:    insert.d $w0[0], $6
+; MIPS64R5EL-NEXT:    insert.d $w0[1], $7
+; MIPS64R5EL-NEXT:    insert.d $w1[0], $4
+; MIPS64R5EL-NEXT:    insert.d $w1[1], $5
+; MIPS64R5EL-NEXT:    fadd.w $w0, $w1, $w0
 ; MIPS64R5EL-NEXT:    ld $1, %got_disp(float_res_v4f32)($1)
 ; MIPS64R5EL-NEXT:    st.w $w0, 0($1)
 ; MIPS64R5EL-NEXT:    jr $ra
@@ -2670,23 +2653,21 @@ define void @double_2(<2 x double> %a, <2 x double> %b) {
 ;
 ; MIPS32R5EB-LABEL: double_2:
 ; MIPS32R5EB:       # %bb.0:
-; MIPS32R5EB-NEXT:    ldi.b $w0, 0
 ; MIPS32R5EB-NEXT:    lw $1, 20($sp)
 ; MIPS32R5EB-NEXT:    lw $2, 16($sp)
-; MIPS32R5EB-NEXT:    move.v $w1, $w0
-; MIPS32R5EB-NEXT:    insert.w $w1[0], $2
-; MIPS32R5EB-NEXT:    insert.w $w1[1], $1
+; MIPS32R5EB-NEXT:    insert.w $w0[0], $2
+; MIPS32R5EB-NEXT:    insert.w $w0[1], $1
 ; MIPS32R5EB-NEXT:    lw $1, 24($sp)
-; MIPS32R5EB-NEXT:    insert.w $w0[0], $4
-; MIPS32R5EB-NEXT:    insert.w $w1[2], $1
+; MIPS32R5EB-NEXT:    insert.w $w0[2], $1
 ; MIPS32R5EB-NEXT:    lw $1, 28($sp)
-; MIPS32R5EB-NEXT:    insert.w $w1[3], $1
-; MIPS32R5EB-NEXT:    shf.w $w1, $w1, 177
-; MIPS32R5EB-NEXT:    insert.w $w0[1], $5
-; MIPS32R5EB-NEXT:    insert.w $w0[2], $6
-; MIPS32R5EB-NEXT:    insert.w $w0[3], $7
+; MIPS32R5EB-NEXT:    insert.w $w0[3], $1
 ; MIPS32R5EB-NEXT:    shf.w $w0, $w0, 177
-; MIPS32R5EB-NEXT:    fadd.d $w0, $w0, $w1
+; MIPS32R5EB-NEXT:    insert.w $w1[0], $4
+; MIPS32R5EB-NEXT:    insert.w $w1[1], $5
+; MIPS32R5EB-NEXT:    insert.w $w1[2], $6
+; MIPS32R5EB-NEXT:    insert.w $w1[3], $7
+; MIPS32R5EB-NEXT:    shf.w $w1, $w1, 177
+; MIPS32R5EB-NEXT:    fadd.d $w0, $w1, $w0
 ; MIPS32R5EB-NEXT:    lui $1, %hi(double_v2f64)
 ; MIPS32R5EB-NEXT:    addiu $1, $1, %lo(double_v2f64)
 ; MIPS32R5EB-NEXT:    st.d $w0, 0($1)
@@ -2698,13 +2679,11 @@ define void @double_2(<2 x double> %a, <2 x double> %b) {
 ; MIPS64R5-NEXT:    lui $1, %hi(%neg(%gp_rel(double_2)))
 ; MIPS64R5-NEXT:    daddu $1, $1, $25
 ; MIPS64R5-NEXT:    daddiu $1, $1, %lo(%neg(%gp_rel(double_2)))
-; MIPS64R5-NEXT:    ldi.b $w0, 0
-; MIPS64R5-NEXT:    move.v $w1, $w0
-; MIPS64R5-NEXT:    insert.d $w1[0], $6
-; MIPS64R5-NEXT:    insert.d $w1[1], $7
-; MIPS64R5-NEXT:    insert.d $w0[0], $4
-; MIPS64R5-NEXT:    insert.d $w0[1], $5
-; MIPS64R5-NEXT:    fadd.d $w0, $w0, $w1
+; MIPS64R5-NEXT:    insert.d $w0[0], $6
+; MIPS64R5-NEXT:    insert.d $w0[1], $7
+; MIPS64R5-NEXT:    insert.d $w1[0], $4
+; MIPS64R5-NEXT:    insert.d $w1[1], $5
+; MIPS64R5-NEXT:    fadd.d $w0, $w1, $w0
 ; MIPS64R5-NEXT:    ld $1, %got_disp(double_v2f64)($1)
 ; MIPS64R5-NEXT:    st.d $w0, 0($1)
 ; MIPS64R5-NEXT:    jr $ra
@@ -2712,21 +2691,19 @@ define void @double_2(<2 x double> %a, <2 x double> %b) {
 ;
 ; MIPS32R5EL-LABEL: double_2:
 ; MIPS32R5EL:       # %bb.0:
-; MIPS32R5EL-NEXT:    ldi.b $w0, 0
 ; MIPS32R5EL-NEXT:    lw $1, 20($sp)
 ; MIPS32R5EL-NEXT:    lw $2, 16($sp)
-; MIPS32R5EL-NEXT:    move.v $w1, $w0
-; MIPS32R5EL-NEXT:    insert.w $w1[0], $2
-; MIPS32R5EL-NEXT:    insert.w $w1[1], $1
+; MIPS32R5EL-NEXT:    insert.w $w0[0], $2
+; MIPS32R5EL-NEXT:    insert.w $w0[1], $1
 ; MIPS32R5EL-NEXT:    lw $1, 24($sp)
-; MIPS32R5EL-NEXT:    insert.w $w1[2], $1
+; MIPS32R5EL-NEXT:    insert.w $w0[2], $1
 ; MIPS32R5EL-NEXT:    lw $1, 28($sp)
-; MIPS32R5EL-NEXT:    insert.w $w1[3], $1
-; MIPS32R5EL-NEXT:    insert.w $w0[0], $4
-; MIPS32R5EL-NEXT:    insert.w $w0[1], $5
-; MIPS32R5EL-NEXT:    insert.w $w0[2], $6
-; MIPS32R5EL-NEXT:    insert.w $w0[3], $7
-; MIPS32R5EL-NEXT:    fadd.d $w0, $w0, $w1
+; MIPS32R5EL-NEXT:    insert.w $w0[3], $1
+; MIPS32R5EL-NEXT:    insert.w $w1[0], $4
+; MIPS32R5EL-NEXT:    insert.w $w1[1], $5
+; MIPS32R5EL-NEXT:    insert.w $w1[2], $6
+; MIPS32R5EL-NEXT:    insert.w $w1[3], $7
+; MIPS32R5EL-NEXT:    fadd.d $w0, $w1, $w0
 ; MIPS32R5EL-NEXT:    lui $1, %hi(double_v2f64)
 ; MIPS32R5EL-NEXT:    addiu $1, $1, %lo(double_v2f64)
 ; MIPS32R5EL-NEXT:    st.d $w0, 0($1)
@@ -2856,8 +2833,10 @@ define <8 x i8> @ret_8_i8() {
 ; MIPS32R5EB:       # %bb.0:
 ; MIPS32R5EB-NEXT:    addiu $sp, $sp, -32
 ; MIPS32R5EB-NEXT:    .cfi_def_cfa_offset 32
-; MIPS32R5EB-NEXT:    sw $fp, 28($sp) # 4-byte Folded Spill
-; MIPS32R5EB-NEXT:    .cfi_offset 30, -4
+; MIPS32R5EB-NEXT:    sw $ra, 28($sp) # 4-byte Folded Spill
+; MIPS32R5EB-NEXT:    sw $fp, 24($sp) # 4-byte Folded Spill
+; MIPS32R5EB-NEXT:    .cfi_offset 31, -4
+; MIPS32R5EB-NEXT:    .cfi_offset 30, -8
 ; MIPS32R5EB-NEXT:    move $fp, $sp
 ; MIPS32R5EB-NEXT:    .cfi_def_cfa_register 30
 ; MIPS32R5EB-NEXT:    addiu $1, $zero, -16
@@ -2872,7 +2851,8 @@ define <8 x i8> @ret_8_i8() {
 ; MIPS32R5EB-NEXT:    copy_s.w $2, $w0[1]
 ; MIPS32R5EB-NEXT:    copy_s.w $3, $w0[3]
 ; MIPS32R5EB-NEXT:    move $sp, $fp
-; MIPS32R5EB-NEXT:    lw $fp, 28($sp) # 4-byte Folded Reload
+; MIPS32R5EB-NEXT:    lw $fp, 24($sp) # 4-byte Folded Reload
+; MIPS32R5EB-NEXT:    lw $ra, 28($sp) # 4-byte Folded Reload
 ; MIPS32R5EB-NEXT:    addiu $sp, $sp, 32
 ; MIPS32R5EB-NEXT:    jr $ra
 ; MIPS32R5EB-NEXT:    nop
@@ -2891,8 +2871,10 @@ define <8 x i8> @ret_8_i8() {
 ; MIPS32R5EL:       # %bb.0:
 ; MIPS32R5EL-NEXT:    addiu $sp, $sp, -32
 ; MIPS32R5EL-NEXT:    .cfi_def_cfa_offset 32
-; MIPS32R5EL-NEXT:    sw $fp, 28($sp) # 4-byte Folded Spill
-; MIPS32R5EL-NEXT:    .cfi_offset 30, -4
+; MIPS32R5EL-NEXT:    sw $ra, 28($sp) # 4-byte Folded Spill
+; MIPS32R5EL-NEXT:    sw $fp, 24($sp) # 4-byte Folded Spill
+; MIPS32R5EL-NEXT:    .cfi_offset 31, -4
+; MIPS32R5EL-NEXT:    .cfi_offset 30, -8
 ; MIPS32R5EL-NEXT:    move $fp, $sp
 ; MIPS32R5EL-NEXT:    .cfi_def_cfa_register 30
 ; MIPS32R5EL-NEXT:    addiu $1, $zero, -16
@@ -2907,7 +2889,8 @@ define <8 x i8> @ret_8_i8() {
 ; MIPS32R5EL-NEXT:    copy_s.w $2, $w0[0]
 ; MIPS32R5EL-NEXT:    copy_s.w $3, $w0[2]
 ; MIPS32R5EL-NEXT:    move $sp, $fp
-; MIPS32R5EL-NEXT:    lw $fp, 28($sp) # 4-byte Folded Reload
+; MIPS32R5EL-NEXT:    lw $fp, 24($sp) # 4-byte Folded Reload
+; MIPS32R5EL-NEXT:    lw $ra, 28($sp) # 4-byte Folded Reload
 ; MIPS32R5EL-NEXT:    addiu $sp, $sp, 32
 ; MIPS32R5EL-NEXT:    jr $ra
 ; MIPS32R5EL-NEXT:    nop
@@ -3027,8 +3010,10 @@ define <4 x i16> @ret_4_i16() {
 ; MIPS32R5EB:       # %bb.0:
 ; MIPS32R5EB-NEXT:    addiu $sp, $sp, -32
 ; MIPS32R5EB-NEXT:    .cfi_def_cfa_offset 32
-; MIPS32R5EB-NEXT:    sw $fp, 28($sp) # 4-byte Folded Spill
-; MIPS32R5EB-NEXT:    .cfi_offset 30, -4
+; MIPS32R5EB-NEXT:    sw $ra, 28($sp) # 4-byte Folded Spill
+; MIPS32R5EB-NEXT:    sw $fp, 24($sp) # 4-byte Folded Spill
+; MIPS32R5EB-NEXT:    .cfi_offset 31, -4
+; MIPS32R5EB-NEXT:    .cfi_offset 30, -8
 ; MIPS32R5EB-NEXT:    move $fp, $sp
 ; MIPS32R5EB-NEXT:    .cfi_def_cfa_register 30
 ; MIPS32R5EB-NEXT:    addiu $1, $zero, -16
@@ -3043,7 +3028,8 @@ define <4 x i16> @ret_4_i16() {
 ; MIPS32R5EB-NEXT:    copy_s.w $2, $w0[1]
 ; MIPS32R5EB-NEXT:    copy_s.w $3, $w0[3]
 ; MIPS32R5EB-NEXT:    move $sp, $fp
-; MIPS32R5EB-NEXT:    lw $fp, 28($sp) # 4-byte Folded Reload
+; MIPS32R5EB-NEXT:    lw $fp, 24($sp) # 4-byte Folded Reload
+; MIPS32R5EB-NEXT:    lw $ra, 28($sp) # 4-byte Folded Reload
 ; MIPS32R5EB-NEXT:    addiu $sp, $sp, 32
 ; MIPS32R5EB-NEXT:    jr $ra
 ; MIPS32R5EB-NEXT:    nop
@@ -3062,8 +3048,10 @@ define <4 x i16> @ret_4_i16() {
 ; MIPS32R5EL:       # %bb.0:
 ; MIPS32R5EL-NEXT:    addiu $sp, $sp, -32
 ; MIPS32R5EL-NEXT:    .cfi_def_cfa_offset 32
-; MIPS32R5EL-NEXT:    sw $fp, 28($sp) # 4-byte Folded Spill
-; MIPS32R5EL-NEXT:    .cfi_offset 30, -4
+; MIPS32R5EL-NEXT:    sw $ra, 28($sp) # 4-byte Folded Spill
+; MIPS32R5EL-NEXT:    sw $fp, 24($sp) # 4-byte Folded Spill
+; MIPS32R5EL-NEXT:    .cfi_offset 31, -4
+; MIPS32R5EL-NEXT:    .cfi_offset 30, -8
 ; MIPS32R5EL-NEXT:    move $fp, $sp
 ; MIPS32R5EL-NEXT:    .cfi_def_cfa_register 30
 ; MIPS32R5EL-NEXT:    addiu $1, $zero, -16
@@ -3078,7 +3066,8 @@ define <4 x i16> @ret_4_i16() {
 ; MIPS32R5EL-NEXT:    copy_s.w $2, $w0[0]
 ; MIPS32R5EL-NEXT:    copy_s.w $3, $w0[2]
 ; MIPS32R5EL-NEXT:    move $sp, $fp
-; MIPS32R5EL-NEXT:    lw $fp, 28($sp) # 4-byte Folded Reload
+; MIPS32R5EL-NEXT:    lw $fp, 24($sp) # 4-byte Folded Reload
+; MIPS32R5EL-NEXT:    lw $ra, 28($sp) # 4-byte Folded Reload
 ; MIPS32R5EL-NEXT:    addiu $sp, $sp, 32
 ; MIPS32R5EL-NEXT:    jr $ra
 ; MIPS32R5EL-NEXT:    nop
@@ -3160,8 +3149,10 @@ define <2 x i32> @ret_2_i32() {
 ; MIPS32R5EB:       # %bb.0:
 ; MIPS32R5EB-NEXT:    addiu $sp, $sp, -32
 ; MIPS32R5EB-NEXT:    .cfi_def_cfa_offset 32
-; MIPS32R5EB-NEXT:    sw $fp, 28($sp) # 4-byte Folded Spill
-; MIPS32R5EB-NEXT:    .cfi_offset 30, -4
+; MIPS32R5EB-NEXT:    sw $ra, 28($sp) # 4-byte Folded Spill
+; MIPS32R5EB-NEXT:    sw $fp, 24($sp) # 4-byte Folded Spill
+; MIPS32R5EB-NEXT:    .cfi_offset 31, -4
+; MIPS32R5EB-NEXT:    .cfi_offset 30, -8
 ; MIPS32R5EB-NEXT:    move $fp, $sp
 ; MIPS32R5EB-NEXT:    .cfi_def_cfa_register 30
 ; MIPS32R5EB-NEXT:    addiu $1, $zero, -16
@@ -3176,7 +3167,8 @@ define <2 x i32> @ret_2_i32() {
 ; MIPS32R5EB-NEXT:    copy_s.w $2, $w0[1]
 ; MIPS32R5EB-NEXT:    copy_s.w $3, $w0[3]
 ; MIPS32R5EB-NEXT:    move $sp, $fp
-; MIPS32R5EB-NEXT:    lw $fp, 28($sp) # 4-byte Folded Reload
+; MIPS32R5EB-NEXT:    lw $fp, 24($sp) # 4-byte Folded Reload
+; MIPS32R5EB-NEXT:    lw $ra, 28($sp) # 4-byte Folded Reload
 ; MIPS32R5EB-NEXT:    addiu $sp, $sp, 32
 ; MIPS32R5EB-NEXT:    jr $ra
 ; MIPS32R5EB-NEXT:    nop
@@ -3195,8 +3187,10 @@ define <2 x i32> @ret_2_i32() {
 ; MIPS32R5EL:       # %bb.0:
 ; MIPS32R5EL-NEXT:    addiu $sp, $sp, -32
 ; MIPS32R5EL-NEXT:    .cfi_def_cfa_offset 32
-; MIPS32R5EL-NEXT:    sw $fp, 28($sp) # 4-byte Folded Spill
-; MIPS32R5EL-NEXT:    .cfi_offset 30, -4
+; MIPS32R5EL-NEXT:    sw $ra, 28($sp) # 4-byte Folded Spill
+; MIPS32R5EL-NEXT:    sw $fp, 24($sp) # 4-byte Folded Spill
+; MIPS32R5EL-NEXT:    .cfi_offset 31, -4
+; MIPS32R5EL-NEXT:    .cfi_offset 30, -8
 ; MIPS32R5EL-NEXT:    move $fp, $sp
 ; MIPS32R5EL-NEXT:    .cfi_def_cfa_register 30
 ; MIPS32R5EL-NEXT:    addiu $1, $zero, -16
@@ -3211,7 +3205,8 @@ define <2 x i32> @ret_2_i32() {
 ; MIPS32R5EL-NEXT:    copy_s.w $2, $w0[0]
 ; MIPS32R5EL-NEXT:    copy_s.w $3, $w0[2]
 ; MIPS32R5EL-NEXT:    move $sp, $fp
-; MIPS32R5EL-NEXT:    lw $fp, 28($sp) # 4-byte Folded Reload
+; MIPS32R5EL-NEXT:    lw $fp, 24($sp) # 4-byte Folded Reload
+; MIPS32R5EL-NEXT:    lw $ra, 28($sp) # 4-byte Folded Reload
 ; MIPS32R5EL-NEXT:    addiu $sp, $sp, 32
 ; MIPS32R5EL-NEXT:    jr $ra
 ; MIPS32R5EL-NEXT:    nop
@@ -4201,9 +4196,8 @@ define void @calli8_16() {
 ; MIPS32R5-NEXT:    sw $1, 16($sp)
 ; MIPS32R5-NEXT:    jal i8_16
 ; MIPS32R5-NEXT:    nop
-; MIPS32R5-NEXT:    ldi.b $w0, 0
-; MIPS32R5-NEXT:    insert.w $w0[0], $2
 ; MIPS32R5-NEXT:    lui $1, %hi(gv16i8)
+; MIPS32R5-NEXT:    insert.w $w0[0], $2
 ; MIPS32R5-NEXT:    insert.w $w0[1], $3
 ; MIPS32R5-NEXT:    addiu $1, $1, %lo(gv16i8)
 ; MIPS32R5-NEXT:    insert.w $w0[2], $4
@@ -4238,7 +4232,6 @@ define void @calli8_16() {
 ; MIPS64R5-NEXT:    ld $25, %call16(i8_16)($gp)
 ; MIPS64R5-NEXT:    jalr $25
 ; MIPS64R5-NEXT:    nop
-; MIPS64R5-NEXT:    ldi.b $w0, 0
 ; MIPS64R5-NEXT:    insert.d $w0[0], $2
 ; MIPS64R5-NEXT:    insert.d $w0[1], $3
 ; MIPS64R5-NEXT:    ld $1, %got_disp(gv16i8)($gp)
@@ -4858,7 +4851,6 @@ define void @calli16_8() {
 ; MIPS32R5EB-NEXT:    nop
 ; MIPS32R5EB-NEXT:    lui $1, %hi(gv8i16)
 ; MIPS32R5EB-NEXT:    addiu $1, $1, %lo(gv8i16)
-; MIPS32R5EB-NEXT:    ldi.b $w0, 0
 ; MIPS32R5EB-NEXT:    insert.w $w0[0], $2
 ; MIPS32R5EB-NEXT:    insert.w $w0[1], $3
 ; MIPS32R5EB-NEXT:    insert.w $w0[2], $4
@@ -4896,7 +4888,6 @@ define void @calli16_8() {
 ; MIPS64R5EB-NEXT:    ld $25, %call16(i16_8)($gp)
 ; MIPS64R5EB-NEXT:    jalr $25
 ; MIPS64R5EB-NEXT:    nop
-; MIPS64R5EB-NEXT:    ldi.b $w0, 0
 ; MIPS64R5EB-NEXT:    ld $1, %got_disp(gv8i16)($gp)
 ; MIPS64R5EB-NEXT:    insert.d $w0[0], $2
 ; MIPS64R5EB-NEXT:    insert.d $w0[1], $3
@@ -5007,7 +4998,6 @@ define void @calli16_8() {
 ; MIPS32R5EL-NEXT:    nop
 ; MIPS32R5EL-NEXT:    lui $1, %hi(gv8i16)
 ; MIPS32R5EL-NEXT:    addiu $1, $1, %lo(gv8i16)
-; MIPS32R5EL-NEXT:    ldi.b $w0, 0
 ; MIPS32R5EL-NEXT:    insert.w $w0[0], $2
 ; MIPS32R5EL-NEXT:    insert.w $w0[1], $3
 ; MIPS32R5EL-NEXT:    insert.w $w0[2], $4
@@ -5045,7 +5035,6 @@ define void @calli16_8() {
 ; MIPS64R5EL-NEXT:    ld $25, %call16(i16_8)($gp)
 ; MIPS64R5EL-NEXT:    jalr $25
 ; MIPS64R5EL-NEXT:    nop
-; MIPS64R5EL-NEXT:    ldi.b $w0, 0
 ; MIPS64R5EL-NEXT:    ld $1, %got_disp(gv8i16)($gp)
 ; MIPS64R5EL-NEXT:    insert.d $w0[0], $2
 ; MIPS64R5EL-NEXT:    insert.d $w0[1], $3
@@ -5303,7 +5292,6 @@ define void @calli32_4() {
 ; MIPS32R5-NEXT:    addiu $7, $zero, 10
 ; MIPS32R5-NEXT:    jal i32_4
 ; MIPS32R5-NEXT:    nop
-; MIPS32R5-NEXT:    ldi.b $w0, 0
 ; MIPS32R5-NEXT:    insert.w $w0[0], $2
 ; MIPS32R5-NEXT:    insert.w $w0[1], $3
 ; MIPS32R5-NEXT:    insert.w $w0[2], $4
@@ -5340,7 +5328,6 @@ define void @calli32_4() {
 ; MIPS64R5-NEXT:    ld $25, %call16(i32_4)($gp)
 ; MIPS64R5-NEXT:    jalr $25
 ; MIPS64R5-NEXT:    nop
-; MIPS64R5-NEXT:    ldi.b $w0, 0
 ; MIPS64R5-NEXT:    insert.d $w0[0], $2
 ; MIPS64R5-NEXT:    insert.d $w0[1], $3
 ; MIPS64R5-NEXT:    ld $1, %got_disp(gv4i32)($gp)
@@ -5472,9 +5459,8 @@ define void @calli64_2() {
 ; MIPS32R5-NEXT:    sw $1, 16($sp)
 ; MIPS32R5-NEXT:    jal i64_2
 ; MIPS32R5-NEXT:    nop
-; MIPS32R5-NEXT:    ldi.b $w0, 0
-; MIPS32R5-NEXT:    insert.w $w0[0], $2
 ; MIPS32R5-NEXT:    lui $1, %hi(gv2i64)
+; MIPS32R5-NEXT:    insert.w $w0[0], $2
 ; MIPS32R5-NEXT:    insert.w $w0[1], $3
 ; MIPS32R5-NEXT:    addiu $1, $1, %lo(gv2i64)
 ; MIPS32R5-NEXT:    insert.w $w0[2], $4
@@ -5503,7 +5489,6 @@ define void @calli64_2() {
 ; MIPS64R5-NEXT:    daddiu $7, $zero, 8
 ; MIPS64R5-NEXT:    jalr $25
 ; MIPS64R5-NEXT:    nop
-; MIPS64R5-NEXT:    ldi.b $w0, 0
 ; MIPS64R5-NEXT:    insert.d $w0[0], $2
 ; MIPS64R5-NEXT:    insert.d $w0[1], $3
 ; MIPS64R5-NEXT:    ld $1, %got_disp(gv2i64)($gp)
@@ -5850,7 +5835,6 @@ define void @callfloat_4() {
 ; MIPS64R5-NEXT:    ld $25, %call16(float4_extern)($gp)
 ; MIPS64R5-NEXT:    jalr $25
 ; MIPS64R5-NEXT:    nop
-; MIPS64R5-NEXT:    ldi.b $w0, 0
 ; MIPS64R5-NEXT:    insert.d $w0[0], $2
 ; MIPS64R5-NEXT:    insert.d $w0[1], $3
 ; MIPS64R5-NEXT:    ld $1, %got_disp(gv4f32)($gp)
@@ -6043,7 +6027,6 @@ define void @calldouble_2() {
 ; MIPS64R5-NEXT:    ld $25, %call16(double2_extern)($gp)
 ; MIPS64R5-NEXT:    jalr $25
 ; MIPS64R5-NEXT:    nop
-; MIPS64R5-NEXT:    ldi.b $w0, 0
 ; MIPS64R5-NEXT:    insert.d $w0[0], $2
 ; MIPS64R5-NEXT:    insert.d $w0[1], $3
 ; MIPS64R5-NEXT:    ld $1, %got_disp(gv2f64)($gp)
@@ -6147,8 +6130,10 @@ define float @mixed_i8(<2 x float> %a, i8 %b, <2 x float> %c) {
 ; MIPS32R5:       # %bb.0: # %entry
 ; MIPS32R5-NEXT:    addiu $sp, $sp, -64
 ; MIPS32R5-NEXT:    .cfi_def_cfa_offset 64
-; MIPS32R5-NEXT:    sw $fp, 60($sp) # 4-byte Folded Spill
-; MIPS32R5-NEXT:    .cfi_offset 30, -4
+; MIPS32R5-NEXT:    sw $ra, 60($sp) # 4-byte Folded Spill
+; MIPS32R5-NEXT:    sw $fp, 56($sp) # 4-byte Folded Spill
+; MIPS32R5-NEXT:    .cfi_offset 31, -4
+; MIPS32R5-NEXT:    .cfi_offset 30, -8
 ; MIPS32R5-NEXT:    move $fp, $sp
 ; MIPS32R5-NEXT:    .cfi_def_cfa_register 30
 ; MIPS32R5-NEXT:    addiu $1, $zero, -16
@@ -6172,7 +6157,8 @@ define float @mixed_i8(<2 x float> %a, i8 %b, <2 x float> %c) {
 ; MIPS32R5-NEXT:    splati.w $w1, $w0[1]
 ; MIPS32R5-NEXT:    add.s $f0, $f0, $f1
 ; MIPS32R5-NEXT:    move $sp, $fp
-; MIPS32R5-NEXT:    lw $fp, 60($sp) # 4-byte Folded Reload
+; MIPS32R5-NEXT:    lw $fp, 56($sp) # 4-byte Folded Reload
+; MIPS32R5-NEXT:    lw $ra, 60($sp) # 4-byte Folded Reload
 ; MIPS32R5-NEXT:    addiu $sp, $sp, 64
 ; MIPS32R5-NEXT:    jr $ra
 ; MIPS32R5-NEXT:    nop
@@ -6324,15 +6310,14 @@ define <4 x float> @mixed_32(<4 x float> %a, i32 %b) {
 ; MIPS32R5EB-NEXT:    ldc1 $f0, %lo($CPI41_0)($1)
 ; MIPS32R5EB-NEXT:    ldc1 $f1, 0($sp)
 ; MIPS32R5EB-NEXT:    sub.d $f0, $f1, $f0
-; MIPS32R5EB-NEXT:    cvt.s.d $f0, $f0
-; MIPS32R5EB-NEXT:    ldi.b $w1, 0
-; MIPS32R5EB-NEXT:    splati.w $w0, $w0[0]
 ; MIPS32R5EB-NEXT:    insert.w $w1[0], $6
 ; MIPS32R5EB-NEXT:    insert.w $w1[1], $7
 ; MIPS32R5EB-NEXT:    lw $1, 24($sp)
 ; MIPS32R5EB-NEXT:    insert.w $w1[2], $1
 ; MIPS32R5EB-NEXT:    lw $1, 28($sp)
 ; MIPS32R5EB-NEXT:    insert.w $w1[3], $1
+; MIPS32R5EB-NEXT:    cvt.s.d $f0, $f0
+; MIPS32R5EB-NEXT:    splati.w $w0, $w0[0]
 ; MIPS32R5EB-NEXT:    fadd.w $w0, $w0, $w1
 ; MIPS32R5EB-NEXT:    st.w $w0, 0($4)
 ; MIPS32R5EB-NEXT:    addiu $sp, $sp, 8
@@ -6354,7 +6339,6 @@ define <4 x float> @mixed_32(<4 x float> %a, i32 %b) {
 ; MIPS64R5EB-NEXT:    ldc1 $f0, %got_ofst(.LCPI41_0)($1)
 ; MIPS64R5EB-NEXT:    ldc1 $f1, 8($sp)
 ; MIPS64R5EB-NEXT:    sub.d $f0, $f1, $f0
-; MIPS64R5EB-NEXT:    ldi.b $w1, 0
 ; MIPS64R5EB-NEXT:    insert.d $w1[0], $4
 ; MIPS64R5EB-NEXT:    insert.d $w1[1], $5
 ; MIPS64R5EB-NEXT:    shf.w $w1, $w1, 177
@@ -6455,15 +6439,14 @@ define <4 x float> @mixed_32(<4 x float> %a, i32 %b) {
 ; MIPS32R5EL-NEXT:    ldc1 $f0, %lo($CPI41_0)($1)
 ; MIPS32R5EL-NEXT:    ldc1 $f1, 0($sp)
 ; MIPS32R5EL-NEXT:    sub.d $f0, $f1, $f0
-; MIPS32R5EL-NEXT:    cvt.s.d $f0, $f0
-; MIPS32R5EL-NEXT:    ldi.b $w1, 0
-; MIPS32R5EL-NEXT:    splati.w $w0, $w0[0]
 ; MIPS32R5EL-NEXT:    insert.w $w1[0], $6
 ; MIPS32R5EL-NEXT:    insert.w $w1[1], $7
 ; MIPS32R5EL-NEXT:    lw $1, 24($sp)
 ; MIPS32R5EL-NEXT:    insert.w $w1[2], $1
 ; MIPS32R5EL-NEXT:    lw $1, 28($sp)
 ; MIPS32R5EL-NEXT:    insert.w $w1[3], $1
+; MIPS32R5EL-NEXT:    cvt.s.d $f0, $f0
+; MIPS32R5EL-NEXT:    splati.w $w0, $w0[0]
 ; MIPS32R5EL-NEXT:    fadd.w $w0, $w0, $w1
 ; MIPS32R5EL-NEXT:    st.w $w0, 0($4)
 ; MIPS32R5EL-NEXT:    addiu $sp, $sp, 8
@@ -6485,7 +6468,6 @@ define <4 x float> @mixed_32(<4 x float> %a, i32 %b) {
 ; MIPS64R5EL-NEXT:    ldc1 $f0, %got_ofst(.LCPI41_0)($1)
 ; MIPS64R5EL-NEXT:    ldc1 $f1, 8($sp)
 ; MIPS64R5EL-NEXT:    sub.d $f0, $f1, $f0
-; MIPS64R5EL-NEXT:    ldi.b $w1, 0
 ; MIPS64R5EL-NEXT:    insert.d $w1[0], $4
 ; MIPS64R5EL-NEXT:    insert.d $w1[1], $5
 ; MIPS64R5EL-NEXT:    cvt.s.d $f0, $f0
@@ -6603,7 +6585,6 @@ define <4 x float> @cast(<4 x i32> %a) {
 ;
 ; MIPS32R5-LABEL: cast:
 ; MIPS32R5:       # %bb.0: # %entry
-; MIPS32R5-NEXT:    ldi.b $w0, 0
 ; MIPS32R5-NEXT:    insert.w $w0[0], $6
 ; MIPS32R5-NEXT:    insert.w $w0[1], $7
 ; MIPS32R5-NEXT:    lw $1, 16($sp)
@@ -6617,7 +6598,6 @@ define <4 x float> @cast(<4 x i32> %a) {
 ;
 ; MIPS64R5EB-LABEL: cast:
 ; MIPS64R5EB:       # %bb.0: # %entry
-; MIPS64R5EB-NEXT:    ldi.b $w0, 0
 ; MIPS64R5EB-NEXT:    insert.d $w0[0], $4
 ; MIPS64R5EB-NEXT:    insert.d $w0[1], $5
 ; MIPS64R5EB-NEXT:    shf.w $w0, $w0, 177
@@ -6719,7 +6699,6 @@ define <4 x float> @cast(<4 x i32> %a) {
 ;
 ; MIPS64R5EL-LABEL: cast:
 ; MIPS64R5EL:       # %bb.0: # %entry
-; MIPS64R5EL-NEXT:    ldi.b $w0, 0
 ; MIPS64R5EL-NEXT:    insert.d $w0[0], $4
 ; MIPS64R5EL-NEXT:    insert.d $w0[1], $5
 ; MIPS64R5EL-NEXT:    ffint_u.w $w0, $w0
@@ -6817,56 +6796,50 @@ define <4 x float> @select(<4 x i32> %cond, <4 x float> %arg1, <4 x float> %arg2
 ;
 ; MIPS32R5-LABEL: select:
 ; MIPS32R5:       # %bb.0: # %entry
-; MIPS32R5-NEXT:    ldi.b $w0, 0
 ; MIPS32R5-NEXT:    lw $1, 44($sp)
 ; MIPS32R5-NEXT:    lw $2, 40($sp)
-; MIPS32R5-NEXT:    move.v $w1, $w0
-; MIPS32R5-NEXT:    insert.w $w1[0], $2
-; MIPS32R5-NEXT:    insert.w $w1[1], $1
+; MIPS32R5-NEXT:    insert.w $w0[0], $2
+; MIPS32R5-NEXT:    insert.w $w0[1], $1
 ; MIPS32R5-NEXT:    lw $1, 48($sp)
-; MIPS32R5-NEXT:    insert.w $w1[2], $1
+; MIPS32R5-NEXT:    insert.w $w0[2], $1
 ; MIPS32R5-NEXT:    lw $1, 28($sp)
-; MIPS32R5-NEXT:    lw $2, 52($sp)
-; MIPS32R5-NEXT:    lw $3, 24($sp)
-; MIPS32R5-NEXT:    move.v $w2, $w0
-; MIPS32R5-NEXT:    insert.w $w2[0], $3
-; MIPS32R5-NEXT:    insert.w $w0[0], $6
-; MIPS32R5-NEXT:    insert.w $w1[3], $2
-; MIPS32R5-NEXT:    insert.w $w2[1], $1
+; MIPS32R5-NEXT:    lw $2, 24($sp)
+; MIPS32R5-NEXT:    lw $3, 52($sp)
+; MIPS32R5-NEXT:    insert.w $w0[3], $3
+; MIPS32R5-NEXT:    insert.w $w1[0], $2
+; MIPS32R5-NEXT:    insert.w $w1[1], $1
 ; MIPS32R5-NEXT:    lw $1, 32($sp)
-; MIPS32R5-NEXT:    insert.w $w2[2], $1
+; MIPS32R5-NEXT:    insert.w $w1[2], $1
 ; MIPS32R5-NEXT:    lw $1, 36($sp)
-; MIPS32R5-NEXT:    insert.w $w2[3], $1
-; MIPS32R5-NEXT:    insert.w $w0[1], $7
+; MIPS32R5-NEXT:    insert.w $w1[3], $1
+; MIPS32R5-NEXT:    insert.w $w2[0], $6
+; MIPS32R5-NEXT:    insert.w $w2[1], $7
 ; MIPS32R5-NEXT:    lw $1, 16($sp)
-; MIPS32R5-NEXT:    insert.w $w0[2], $1
+; MIPS32R5-NEXT:    insert.w $w2[2], $1
 ; MIPS32R5-NEXT:    lw $1, 20($sp)
-; MIPS32R5-NEXT:    insert.w $w0[3], $1
-; MIPS32R5-NEXT:    slli.w $w0, $w0, 31
-; MIPS32R5-NEXT:    srai.w $w0, $w0, 31
-; MIPS32R5-NEXT:    bsel.v $w0, $w1, $w2
-; MIPS32R5-NEXT:    st.w $w0, 0($4)
+; MIPS32R5-NEXT:    insert.w $w2[3], $1
+; MIPS32R5-NEXT:    slli.w $w2, $w2, 31
+; MIPS32R5-NEXT:    srai.w $w2, $w2, 31
+; MIPS32R5-NEXT:    bsel.v $w2, $w0, $w1
+; MIPS32R5-NEXT:    st.w $w2, 0($4)
 ; MIPS32R5-NEXT:    jr $ra
 ; MIPS32R5-NEXT:    nop
 ;
 ; MIPS64R5EB-LABEL: select:
 ; MIPS64R5EB:       # %bb.0: # %entry
-; MIPS64R5EB-NEXT:    ldi.b $w0, 0
-; MIPS64R5EB-NEXT:    move.v $w1, $w0
-; MIPS64R5EB-NEXT:    insert.d $w1[0], $8
-; MIPS64R5EB-NEXT:    insert.d $w1[1], $9
+; MIPS64R5EB-NEXT:    insert.d $w0[0], $8
+; MIPS64R5EB-NEXT:    insert.d $w0[1], $9
+; MIPS64R5EB-NEXT:    shf.w $w0, $w0, 177
+; MIPS64R5EB-NEXT:    insert.d $w1[0], $6
+; MIPS64R5EB-NEXT:    insert.d $w1[1], $7
 ; MIPS64R5EB-NEXT:    shf.w $w1, $w1, 177
-; MIPS64R5EB-NEXT:    move.v $w2, $w0
-; MIPS64R5EB-NEXT:    insert.d $w2[0], $6
-; MIPS64R5EB-NEXT:    insert.d $w2[1], $7
+; MIPS64R5EB-NEXT:    insert.d $w2[0], $4
+; MIPS64R5EB-NEXT:    insert.d $w2[1], $5
 ; MIPS64R5EB-NEXT:    shf.w $w2, $w2, 177
-; MIPS64R5EB-NEXT:    insert.d $w0[0], $4
-; MIPS64R5EB-NEXT:    insert.d $w0[1], $5
-; MIPS64R5EB-NEXT:    shf.w $w0, $w0, 177
-; MIPS64R5EB-NEXT:    slli.w $w0, $w0, 31
-; MIPS64R5EB-NEXT:    srai.w $w0, $w0, 31
-; MIPS64R5EB-NEXT:    bsel.v $w0, $w1, $w2
-; MIPS64R5EB-NEXT:    shf.w $w0, $w0, 177
+; MIPS64R5EB-NEXT:    slli.w $w2, $w2, 31
+; MIPS64R5EB-NEXT:    srai.w $w2, $w2, 31
+; MIPS64R5EB-NEXT:    bsel.v $w2, $w0, $w1
+; MIPS64R5EB-NEXT:    shf.w $w0, $w2, 177
 ; MIPS64R5EB-NEXT:    copy_s.d $2, $w0[0]
 ; MIPS64R5EB-NEXT:    copy_s.d $3, $w0[1]
 ; MIPS64R5EB-NEXT:    jr $ra
@@ -6874,20 +6847,17 @@ define <4 x float> @select(<4 x i32> %cond, <4 x float> %arg1, <4 x float> %arg2
 ;
 ; MIPS64R5EL-LABEL: select:
 ; MIPS64R5EL:       # %bb.0: # %entry
-; MIPS64R5EL-NEXT:    ldi.b $w0, 0
-; MIPS64R5EL-NEXT:    move.v $w1, $w0
-; MIPS64R5EL-NEXT:    insert.d $w1[0], $8
-; MIPS64R5EL-NEXT:    insert.d $w1[1], $9
-; MIPS64R5EL-NEXT:    move.v $w2, $w0
-; MIPS64R5EL-NEXT:    insert.d $w2[0], $6
-; MIPS64R5EL-NEXT:    insert.d $w2[1], $7
-; MIPS64R5EL-NEXT:    insert.d $w0[0], $4
-; MIPS64R5EL-NEXT:    insert.d $w0[1], $5
-; MIPS64R5EL-NEXT:    slli.w $w0, $w0, 31
-; MIPS64R5EL-NEXT:    srai.w $w0, $w0, 31
-; MIPS64R5EL-NEXT:    bsel.v $w0, $w1, $w2
-; MIPS64R5EL-NEXT:    copy_s.d $2, $w0[0]
-; MIPS64R5EL-NEXT:    copy_s.d $3, $w0[1]
+; MIPS64R5EL-NEXT:    insert.d $w0[0], $8
+; MIPS64R5EL-NEXT:    insert.d $w0[1], $9
+; MIPS64R5EL-NEXT:    insert.d $w1[0], $6
+; MIPS64R5EL-NEXT:    insert.d $w1[1], $7
+; MIPS64R5EL-NEXT:    insert.d $w2[0], $4
+; MIPS64R5EL-NEXT:    insert.d $w2[1], $5
+; MIPS64R5EL-NEXT:    slli.w $w2, $w2, 31
+; MIPS64R5EL-NEXT:    srai.w $w2, $w2, 31
+; MIPS64R5EL-NEXT:    bsel.v $w2, $w0, $w1
+; MIPS64R5EL-NEXT:    copy_s.d $2, $w2[0]
+; MIPS64R5EL-NEXT:    copy_s.d $3, $w2[1]
 ; MIPS64R5EL-NEXT:    jr $ra
 ; MIPS64R5EL-NEXT:    nop
 entry:
diff --git a/llvm/test/CodeGen/Mips/ci2.ll b/llvm/test/CodeGen/Mips/ci2.ll
index bb16fa83f..5e6b7814d 100644
--- a/llvm/test/CodeGen/Mips/ci2.ll
+++ b/llvm/test/CodeGen/Mips/ci2.ll
@@ -33,7 +33,7 @@ if.end:                                           ; preds = %if.else, %if.then
 ; constisle	.4byte	305419896               # 0x12345678
 }
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind }
 
 !1 = !{i32 103}
diff --git a/llvm/test/CodeGen/Mips/cmplarge.ll b/llvm/test/CodeGen/Mips/cmplarge.ll
index 1ca5b921e..aa8832336 100644
--- a/llvm/test/CodeGen/Mips/cmplarge.ll
+++ b/llvm/test/CodeGen/Mips/cmplarge.ll
@@ -33,6 +33,6 @@ for.end:                                          ; preds = %for.body, %entry
 ; cmp16:	.end	getSubImagesLuma
 declare i32 @iClip3(...) #1
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "less-precise-fpmad"="false" "frame-pointer"="non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind }
diff --git a/llvm/test/CodeGen/Mips/compactbranches/no-beqzc-bnezc.ll b/llvm/test/CodeGen/Mips/compactbranches/no-beqzc-bnezc.ll
index 7a9f287c1..2bda921d5 100644
--- a/llvm/test/CodeGen/Mips/compactbranches/no-beqzc-bnezc.ll
+++ b/llvm/test/CodeGen/Mips/compactbranches/no-beqzc-bnezc.ll
@@ -103,7 +103,7 @@ define i64 @f5(i64 %a, i64 %b) {
 
 define i32 @f6(i32 %a) {
 ; CHECK-LABEL: f6:
-; CHECK: beqzc ${{[0-9]+}}, $BB
+; CHECK: beqzc ${{[0-9]+}}, {{((\$)|(\.L))}}BB
 
   %cmp = icmp eq i32 %a, 0
   br i1 %cmp, label %if.then, label %if.end
@@ -117,7 +117,7 @@ define i32 @f6(i32 %a) {
 
 define i32 @f7(i32 %a) {
 ; CHECK-LABEL: f7:
-; CHECK: bnezc ${{[0-9]+}}, $BB
+; CHECK: bnezc ${{[0-9]+}}, {{((\$)|(\.L))}}BB
 
   %cmp = icmp eq i32 0, %a
   br i1 %cmp, label %if.then, label %if.end
diff --git a/llvm/test/CodeGen/Mips/const1.ll b/llvm/test/CodeGen/Mips/const1.ll
index 2bcd40517..8c6fe8fa7 100644
--- a/llvm/test/CodeGen/Mips/const1.ll
+++ b/llvm/test/CodeGen/Mips/const1.ll
@@ -28,7 +28,7 @@ entry:
 ; CHECK:	.4byte	262991277
 }
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="true" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="true" }
 
 !llvm.ident = !{!0}
 
diff --git a/llvm/test/CodeGen/Mips/const4a.ll b/llvm/test/CodeGen/Mips/const4a.ll
index df4a1d9ff..79cea7c18 100644
--- a/llvm/test/CodeGen/Mips/const4a.ll
+++ b/llvm/test/CodeGen/Mips/const4a.ll
@@ -172,8 +172,8 @@ declare void @goo(...) #1
 
 declare void @hoo(...) #1
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="true" }
-attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="true" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="true" }
+attributes #1 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="true" }
 
 !llvm.ident = !{!0}
 
diff --git a/llvm/test/CodeGen/Mips/const6.ll b/llvm/test/CodeGen/Mips/const6.ll
index 9085e38b3..e8c3f8c41 100644
--- a/llvm/test/CodeGen/Mips/const6.ll
+++ b/llvm/test/CodeGen/Mips/const6.ll
@@ -154,8 +154,8 @@ entry:
 
 declare void @hoo(...) #1
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="true" }
-attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="true" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="true" }
+attributes #1 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="true" }
 
 !llvm.ident = !{!0}
 
diff --git a/llvm/test/CodeGen/Mips/const6a.ll b/llvm/test/CodeGen/Mips/const6a.ll
index 80eedb4c8..ed0f52fb9 100644
--- a/llvm/test/CodeGen/Mips/const6a.ll
+++ b/llvm/test/CodeGen/Mips/const6a.ll
@@ -23,7 +23,7 @@ entry:
   ret void
 }
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="true" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="true" }
 attributes #1 = { nounwind }
 
 !1 = !{i32 121}
diff --git a/llvm/test/CodeGen/Mips/constraint-empty.ll b/llvm/test/CodeGen/Mips/constraint-empty.ll
new file mode 100644
index 000000000..65b5d4364
--- /dev/null
+++ b/llvm/test/CodeGen/Mips/constraint-empty.ll
@@ -0,0 +1,14 @@
+; Check that `getRegForInlineAsmConstraint` does not crash on empty Constraint.
+; RUN: llc -march=mips64 < %s | FileCheck %s
+
+define void @foo() {
+entry:
+  %s = alloca i32, align 4
+  %x = alloca i32, align 4
+  call void asm "", "=*imr,=*m,0,*m,~{$1}"(i32* %x, i32* %s, i32* %x, i32* %s)
+
+; CHECK: #APP
+; CHECK: #NO_APP
+
+  ret void
+}
diff --git a/llvm/test/CodeGen/Mips/copy-fp64.ll b/llvm/test/CodeGen/Mips/copy-fp64.ll
new file mode 100644
index 000000000..439c788eb
--- /dev/null
+++ b/llvm/test/CodeGen/Mips/copy-fp64.ll
@@ -0,0 +1,29 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+
+; RUN: llc -march=mips -mcpu=mips32r2 -O0 -relocation-model=pic -mattr=+fp64 \
+; RUN:   -stop-before=prologepilog %s -o - | FileCheck %s
+
+declare double @bar(double)
+
+define  double @foo(double %self) {
+  ; CHECK-LABEL: name: foo
+  ; CHECK: bb.0.start:
+  ; CHECK:   successors: %bb.1(0x80000000)
+  ; CHECK:   liveins: $d12_64, $t9, $v0
+  ; CHECK:   renamable $at = ADDu killed $v0, killed $t9
+  ; CHECK:   ADJCALLSTACKDOWN 16, 0, implicit-def $sp, implicit $sp
+  ; CHECK:   $d6_64 = COPY killed renamable $d12_64
+  ; CHECK:   renamable $t9 = LW killed renamable $at, target-flags(mips-got) @bar
+  ; CHECK:   dead $ra = JALR killed $t9, csr_o32_fp64, target-flags(mips-jalr) <mcsymbol bar>, implicit-def dead $ra, implicit killed $d6_64, implicit-def $d0_64
+  ; CHECK:   ADJCALLSTACKUP 16, 0, implicit-def $sp, implicit $sp
+  ; CHECK:   SDC164 killed $d0_64, %stack.0, 0 :: (store 8 into %stack.0)
+  ; CHECK: bb.1.bb1:
+  ; CHECK:   $d0_64 = LDC164 %stack.0, 0 :: (load 8 from %stack.0)
+  ; CHECK:   RetRA implicit killed $d0_64
+start:
+  %0 = call double @bar(double %self)
+  br label %bb1
+
+bb1:
+  ret double %0
+}
diff --git a/llvm/test/CodeGen/Mips/cpus-no-mips64.ll b/llvm/test/CodeGen/Mips/cpus-no-mips64.ll
new file mode 100644
index 000000000..301f6c215
--- /dev/null
+++ b/llvm/test/CodeGen/Mips/cpus-no-mips64.ll
@@ -0,0 +1,16 @@
+; Check that we reject 64-bit mode on 32-bit only CPUs.
+
+; RUN: not llc < %s -o /dev/null -mtriple=mips64 -mcpu=mips1 2>&1 | FileCheck %s
+; RUN: not llc < %s -o /dev/null -mtriple=mips64 -mcpu=mips2 2>&1 | FileCheck %s
+
+; RUN: not llc < %s -o /dev/null -mtriple=mips64 -mcpu=mips32 2>&1 | FileCheck %s
+; RUN: not llc < %s -o /dev/null -mtriple=mips64 -mcpu=mips32r2 2>&1 | FileCheck %s
+; RUN: not llc < %s -o /dev/null -mtriple=mips64 -mcpu=mips32r3 2>&1 | FileCheck %s
+; RUN: not llc < %s -o /dev/null -mtriple=mips64 -mcpu=mips32r5 2>&1 | FileCheck %s
+; RUN: not llc < %s -o /dev/null -mtriple=mips64 -mcpu=mips32r6 2>&1 | FileCheck %s
+
+; CHECK: LLVM ERROR: 64-bit code requested on a subtarget that doesn't support it!
+
+define void @foo() {
+  ret void
+}
diff --git a/llvm/test/CodeGen/Mips/cpus.ll b/llvm/test/CodeGen/Mips/cpus.ll
new file mode 100644
index 000000000..d9377044d
--- /dev/null
+++ b/llvm/test/CodeGen/Mips/cpus.ll
@@ -0,0 +1,69 @@
+; Check that the CPU names work.
+
+; RUN: llc -mtriple=mips -mcpu=generic -filetype=obj < %s \
+; RUN:   | llvm-readelf -A | FileCheck %s --check-prefix=GENERIC
+; GENERIC: ISA: MIPS32
+
+; RUN: llc -mtriple=mips -mcpu=mips2 -filetype=obj < %s \
+; RUN:   | llvm-readelf -A | FileCheck %s --check-prefix=MIPS2
+; MIPS2: ISA: MIPS2
+; RUN: llc -mtriple=mips64 -mcpu=mips3 -filetype=obj < %s \
+; RUN:   | llvm-readelf -A | FileCheck %s --check-prefix=MIPS3
+; MIPS3: ISA: MIPS3
+; RUN: llc -mtriple=mips64 -mcpu=mips4 -filetype=obj < %s \
+; RUN:   | llvm-readelf -A | FileCheck %s --check-prefix=MIPS4
+; MIPS4: ISA: MIPS4
+
+; RUN: llc -mtriple=mips -mcpu=mips32 -filetype=obj < %s \
+; RUN:   | llvm-readelf -A | FileCheck %s --check-prefix=MIPS32
+; MIPS32: ISA: MIPS32
+; RUN: llc -mtriple=mips -mcpu=mips32r2 -filetype=obj < %s \
+; RUN:   | llvm-readelf -A | FileCheck %s --check-prefix=MIPS32R2
+; MIPS32R2: ISA: MIPS32r2
+; RUN: llc -mtriple=mips -mcpu=mips32r3 -filetype=obj < %s \
+; RUN:   | llvm-readelf -A | FileCheck %s --check-prefix=MIPS32R3
+; MIPS32R3: ISA: MIPS32r3
+; RUN: llc -mtriple=mips -mcpu=mips32r5 -filetype=obj < %s \
+; RUN:   | llvm-readelf -A | FileCheck %s --check-prefix=MIPS32R5
+; MIPS32R5: ISA: MIPS32r5
+; RUN: llc -mtriple=mips -mcpu=mips32r6 -filetype=obj < %s \
+; RUN:   | llvm-readelf -A | FileCheck %s --check-prefix=MIPS32R6
+; MIPS32R6: ISA: MIPS32r6
+
+; RUN: llc -mtriple=mips64 -mcpu=mips64 -filetype=obj < %s \
+; RUN:   | llvm-readelf -A | FileCheck %s --check-prefix=MIPS64
+; MIPS64: ISA: MIPS64
+; RUN: llc -mtriple=mips64 -mcpu=mips64r2 -filetype=obj < %s \
+; RUN:   | llvm-readelf -A | FileCheck %s --check-prefix=MIPS64R2
+; MIPS64R2: ISA: MIPS64r2
+; RUN: llc -mtriple=mips64 -mcpu=mips64r3 -filetype=obj < %s \
+; RUN:   | llvm-readelf -A | FileCheck %s --check-prefix=MIPS64R3
+; MIPS64R3: ISA: MIPS64r3
+; RUN: llc -mtriple=mips64 -mcpu=mips64r5 -filetype=obj < %s \
+; RUN:   | llvm-readelf -A | FileCheck %s --check-prefix=MIPS64R5
+; MIPS64R5: ISA: MIPS64r5
+; RUN: llc -mtriple=mips64 -mcpu=mips64r6 -filetype=obj < %s \
+; RUN:   | llvm-readelf -A | FileCheck %s --check-prefix=MIPS64R6
+; MIPS64R6: ISA: MIPS64r6
+
+; RUN: llc -mtriple=mips64 -mcpu=octeon -filetype=obj < %s \
+; RUN:   | llvm-readelf -A | FileCheck %s --check-prefix=OCTEON
+; OCTEON: ISA: MIPS64r2
+; OCTEON: ISA Extension: Cavium Networks Octeon
+; RUN: llc -mtriple=mips64 -mcpu=octeon+ -filetype=obj < %s \
+; RUN:   | llvm-readelf -A | FileCheck %s --check-prefix=OCTEONP
+; OCTEONP: ISA: MIPS64r2
+; OCTEONP: ISA Extension: Cavium Networks OcteonP
+
+; Check that we reject CPUs that are not implemented.
+
+; RUN: not llc < %s -o /dev/null -mtriple=mips -mcpu=mips1 2>&1 \
+; RUN:   | FileCheck %s --check-prefix=ERROR
+; RUN: not llc < %s -o /dev/null -mtriple=mips64 -mcpu=mips5 2>&1 \
+; RUN:   | FileCheck %s --check-prefix=ERROR
+
+; ERROR: LLVM ERROR: Code generation for MIPS-{{.}} is not implemented
+
+define void @foo() {
+  ret void
+}
diff --git a/llvm/test/CodeGen/Mips/ctlz.ll b/llvm/test/CodeGen/Mips/ctlz.ll
index 96af1973b..c502b0d4c 100644
--- a/llvm/test/CodeGen/Mips/ctlz.ll
+++ b/llvm/test/CodeGen/Mips/ctlz.ll
@@ -22,6 +22,6 @@ declare i32 @llvm.ctlz.i32(i32, i1) #1
 
 
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="true" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="true" }
 attributes #1 = { nounwind readnone }
 
diff --git a/llvm/test/CodeGen/Mips/delay-slot-fill-forward.ll b/llvm/test/CodeGen/Mips/delay-slot-fill-forward.ll
index 7fc011da9..33591e15a 100644
--- a/llvm/test/CodeGen/Mips/delay-slot-fill-forward.ll
+++ b/llvm/test/CodeGen/Mips/delay-slot-fill-forward.ll
@@ -161,7 +161,7 @@ if.end461:                                        ; preds = %if.end436, %for.bod
   ret void
 }
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="mips32r2" "target-features"="+mips32r2,+nooddspreg,+fpxx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="mips32r2" "target-features"="+mips32r2,+nooddspreg,+fpxx" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind }
 
 !llvm.ident = !{!0}
diff --git a/llvm/test/CodeGen/Mips/dynamic-stack-realignment.ll b/llvm/test/CodeGen/Mips/dynamic-stack-realignment.ll
index 5054b9cd0..50acd8ffb 100644
--- a/llvm/test/CodeGen/Mips/dynamic-stack-realignment.ll
+++ b/llvm/test/CodeGen/Mips/dynamic-stack-realignment.ll
@@ -163,8 +163,9 @@ entry:
   ; GP32-M:     addiu   $sp, $sp, -1024
   ; GP32-MMR2:  addiusp -1024
   ; GP32-MMR6:  addiu   $sp, $sp, -1024
-  ; GP32:       sw      $fp, 1020($sp)
-  ; GP32:       sw      $23, 1016($sp)
+  ; GP32:       sw      $ra, 1020($sp)
+  ; GP32:       sw      $fp, 1016($sp)
+  ; GP32:       sw      $23, 1012($sp)
   ;
   ; GP32:       move    $fp, $sp
   ; GP32:       addiu   $[[T0:[0-9]+|gp]], $zero, -512
@@ -177,8 +178,9 @@ entry:
 
   ; epilogue
   ; GP32:       move    $sp, $fp
-  ; GP32:       lw      $23, 1016($sp)
-  ; GP32:       lw      $fp, 1020($sp)
+  ; GP32:       lw      $23, 1012($sp)
+  ; GP32:       lw      $fp, 1016($sp)
+  ; GP32:       lw      $ra, 1020($sp)
   ; GP32-M:     addiu   $sp, $sp, 1024
   ; GP32-MMR2:  addiusp 1024
   ; GP32-MMR6:  addiu   $sp, $sp, 1024
@@ -201,8 +203,9 @@ entry:
   ; FIXME: We are currently over-allocating stack space.
   ; N32:        addiu   $sp, $sp, -1024
   ; N64:        daddiu  $sp, $sp, -1024
-  ; GP64:       sd      $fp, 1016($sp)
-  ; GP64:       sd      $23, 1008($sp)
+  ; GP64:       sd      $ra, 1016($sp)
+  ; GP64:       sd      $fp, 1008($sp)
+  ; GP64:       sd      $23, 1000($sp)
   ;
   ; GP64:       move    $fp, $sp
   ; GP64:       addiu   $[[T0:[0-9]+|gp]], $zero, -512
@@ -215,8 +218,9 @@ entry:
 
   ; epilogue
   ; GP64:       move    $sp, $fp
-  ; GP64:       ld      $23, 1008($sp)
-  ; GP64:       ld      $fp, 1016($sp)
+  ; GP64:       ld      $23, 1000($sp)
+  ; GP64:       ld      $fp, 1008($sp)
+  ; GP64:       ld      $ra, 1016($sp)
   ; N32:        addiu   $sp, $sp, 1024
   ; N64:        daddiu  $sp, $sp, 1024
 
diff --git a/llvm/test/CodeGen/Mips/f16abs.ll b/llvm/test/CodeGen/Mips/f16abs.ll
index 6c33e0117..847ef627b 100644
--- a/llvm/test/CodeGen/Mips/f16abs.ll
+++ b/llvm/test/CodeGen/Mips/f16abs.ll
@@ -29,8 +29,8 @@ declare double @fabs(double) #1
 
 declare float @fabsf(float) #1
 
-attributes #0 = { nounwind optsize "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="true" }
-attributes #1 = { nounwind optsize readnone "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="true" }
+attributes #0 = { nounwind optsize "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="true" }
+attributes #1 = { nounwind optsize readnone "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="true" }
 attributes #2 = { nounwind optsize readnone }
 
 
diff --git a/llvm/test/CodeGen/Mips/fp16instrinsmc.ll b/llvm/test/CodeGen/Mips/fp16instrinsmc.ll
index 258b1bf97..b3d36ba29 100644
--- a/llvm/test/CodeGen/Mips/fp16instrinsmc.ll
+++ b/llvm/test/CodeGen/Mips/fp16instrinsmc.ll
@@ -385,7 +385,7 @@ entry:
 ; Function Attrs: nounwind
 declare double @exp2(double) #0
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { nounwind readnone "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="true" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="true" }
 attributes #2 = { nounwind readnone }
 attributes #3 = { nounwind }
diff --git a/llvm/test/CodeGen/Mips/fpneeded.ll b/llvm/test/CodeGen/Mips/fpneeded.ll
index a89e2a593..d5fa3d7f2 100644
--- a/llvm/test/CodeGen/Mips/fpneeded.ll
+++ b/llvm/test/CodeGen/Mips/fpneeded.ll
@@ -131,7 +131,7 @@ entry:
 ; 32:	.set	reorder
 ; 32:	.end	foo3
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 define void @vv() #0 {
 entry:
diff --git a/llvm/test/CodeGen/Mips/fpnotneeded.ll b/llvm/test/CodeGen/Mips/fpnotneeded.ll
index 02b8e8a34..59f01e25f 100644
--- a/llvm/test/CodeGen/Mips/fpnotneeded.ll
+++ b/llvm/test/CodeGen/Mips/fpnotneeded.ll
@@ -61,7 +61,7 @@ entry:
 
 ; cisle:	.end	foo
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 
 define float @fv() #0 {
diff --git a/llvm/test/CodeGen/Mips/frame-address.ll b/llvm/test/CodeGen/Mips/frame-address.ll
index 0ab7da30e..f7ceb575c 100644
--- a/llvm/test/CodeGen/Mips/frame-address.ll
+++ b/llvm/test/CodeGen/Mips/frame-address.ll
@@ -1,17 +1,26 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -march=mipsel < %s | FileCheck %s
 
 declare i8* @llvm.frameaddress(i32) nounwind readnone
 
 define i8* @f() nounwind uwtable {
+; CHECK-LABEL: f:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addiu $sp, $sp, -8
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    sw $ra, 4($sp) # 4-byte Folded Spill
+; CHECK-NEXT:    sw $fp, 0($sp) # 4-byte Folded Spill
+; CHECK-NEXT:    .cfi_offset 31, -4
+; CHECK-NEXT:    .cfi_offset 30, -8
+; CHECK-NEXT:    move $fp, $sp
+; CHECK-NEXT:    .cfi_def_cfa_register 30
+; CHECK-NEXT:    move $2, $fp
+; CHECK-NEXT:    move $sp, $fp
+; CHECK-NEXT:    lw $fp, 0($sp) # 4-byte Folded Reload
+; CHECK-NEXT:    lw $ra, 4($sp) # 4-byte Folded Reload
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    addiu $sp, $sp, 8
 entry:
   %0 = call i8* @llvm.frameaddress(i32 0)
   ret i8* %0
-
-; CHECK: .cfi_startproc
-; CHECK: .cfi_def_cfa_offset 8
-; CHECK: .cfi_offset 30, -4
-; CHECK:   move    $fp, $sp
-; CHECK: .cfi_def_cfa_register 30
-; CHECK:   move    $2, $fp
-; CHECK: .cfi_endproc
 }
diff --git a/llvm/test/CodeGen/Mips/global-address.ll b/llvm/test/CodeGen/Mips/global-address.ll
index ed79de920..21cc667e9 100644
--- a/llvm/test/CodeGen/Mips/global-address.ll
+++ b/llvm/test/CodeGen/Mips/global-address.ll
@@ -1,9 +1,15 @@
-; RUN: llc -march=mipsel -relocation-model=pic < %s | FileCheck %s -check-prefix=PIC-O32
-; RUN: llc -march=mipsel -relocation-model=static -mtriple=mipsel-linux-gnu < %s | FileCheck %s -check-prefix=STATIC-O32
-; RUN: llc -march=mips64el -mcpu=mips64r2 -target-abi n32 -relocation-model=pic < %s | FileCheck %s -check-prefix=PIC-N32
-; RUN: llc -march=mips64el -mcpu=mips64r2 -target-abi n32 -relocation-model=static  -mtriple=mipsel-linux-gnu < %s | FileCheck %s -check-prefix=STATIC-N32
-; RUN: llc -march=mips64el -mcpu=mips64r2 -target-abi n64 -relocation-model=pic < %s | FileCheck %s -check-prefix=PIC-N64
-; RUN: llc -march=mips64el -mcpu=mips64r2 -target-abi n64 -relocation-model=static < %s | FileCheck %s -check-prefix=STATIC-N64
+; RUN: llc -march=mipsel \
+; RUN:     -relocation-model=pic < %s | FileCheck %s -check-prefix=PIC-O32
+; RUN: llc -march=mipsel -mtriple=mipsel-linux-gnu \
+; RUN:     -relocation-model=static < %s | FileCheck %s -check-prefix=STATIC-O32
+; RUN: llc -march=mips64el -mcpu=mips64r2 -target-abi n32 \
+; RUN:     -relocation-model=pic < %s | FileCheck %s -check-prefix=PIC-N32
+; RUN: llc -march=mips64el -mcpu=mips64r2 -target-abi n32 \
+; RUN:     -relocation-model=static < %s | FileCheck %s -check-prefix=STATIC-N32
+; RUN: llc -march=mips64el -mcpu=mips64r2 -target-abi n64 \
+; RUN:     -relocation-model=pic < %s | FileCheck %s -check-prefix=PIC-N64
+; RUN: llc -march=mips64el -mcpu=mips64r2 -target-abi n64 \
+; RUN:     -relocation-model=static < %s | FileCheck %s -check-prefix=STATIC-N64
 
 @s1 = internal unnamed_addr global i32 8, align 4
 @g1 = external global i32
@@ -49,4 +55,3 @@ entry:
 }
 
 declare void @foo1(i32)
-
diff --git a/llvm/test/CodeGen/Mips/hf16call32.ll b/llvm/test/CodeGen/Mips/hf16call32.ll
index 5159477f2..616b9dce9 100644
--- a/llvm/test/CodeGen/Mips/hf16call32.ll
+++ b/llvm/test/CodeGen/Mips/hf16call32.ll
@@ -1026,5 +1026,5 @@ declare { double, double } @dc_sf(float) #1
 ; stel: jr $18
 ; stel: .end __call_stub_fp_dc_sf
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/Mips/hf16call32_body.ll b/llvm/test/CodeGen/Mips/hf16call32_body.ll
index 84be154de..420148728 100644
--- a/llvm/test/CodeGen/Mips/hf16call32_body.ll
+++ b/llvm/test/CodeGen/Mips/hf16call32_body.ll
@@ -303,4 +303,4 @@ entry:
 ; stel: .set $__fn_local_sf_df_df, sf_df_df
 ; stel: .end __fn_stub_sf_df_df
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/Mips/hfptrcall.ll b/llvm/test/CodeGen/Mips/hfptrcall.ll
index f6ce243c3..50e8de24d 100644
--- a/llvm/test/CodeGen/Mips/hfptrcall.ll
+++ b/llvm/test/CodeGen/Mips/hfptrcall.ll
@@ -118,8 +118,8 @@ entry:
 
 declare i32 @printf(i8*, ...) #1
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 
 
diff --git a/llvm/test/CodeGen/Mips/indirect-jump-hazard/long-calls.ll b/llvm/test/CodeGen/Mips/indirect-jump-hazard/long-calls.ll
index 50d8fd6dd..59a2c3eae 100644
--- a/llvm/test/CodeGen/Mips/indirect-jump-hazard/long-calls.ll
+++ b/llvm/test/CodeGen/Mips/indirect-jump-hazard/long-calls.ll
@@ -28,11 +28,10 @@ define void @caller() {
 ; O32-NEXT:    addiu $25, $1, %lo(callee)
 ; O32-NEXT:    jalr.hb $25
 ; O32-NEXT:    nop
-; O32-NEXT:    addiu $1, $zero, %lo(memset)
-; O32-NEXT:    lui $2, %hi(memset)
-; O32-NEXT:    addu $25, $2, $1
 ; O32-NEXT:    lui $1, %hi(val)
 ; O32-NEXT:    addiu $4, $1, %lo(val)
+; O32-NEXT:    lui $1, %hi(memset)
+; O32-NEXT:    addiu $25, $1, %lo(memset)
 ; O32-NEXT:    addiu $5, $zero, 0
 ; O32-NEXT:    jalr.hb $25
 ; O32-NEXT:    addiu $6, $zero, 80
@@ -50,11 +49,10 @@ define void @caller() {
 ; N32-NEXT:    addiu $25, $1, %lo(callee)
 ; N32-NEXT:    jalr.hb $25
 ; N32-NEXT:    nop
-; N32-NEXT:    addiu $1, $zero, %lo(memset)
-; N32-NEXT:    lui $2, %hi(memset)
-; N32-NEXT:    addu $25, $2, $1
 ; N32-NEXT:    lui $1, %hi(val)
 ; N32-NEXT:    addiu $4, $1, %lo(val)
+; N32-NEXT:    lui $1, %hi(memset)
+; N32-NEXT:    addiu $25, $1, %lo(memset)
 ; N32-NEXT:    daddiu $5, $zero, 0
 ; N32-NEXT:    jalr.hb $25
 ; N32-NEXT:    daddiu $6, $zero, 80
@@ -76,21 +74,18 @@ define void @caller() {
 ; N64-NEXT:    daddiu $25, $1, %lo(callee)
 ; N64-NEXT:    jalr.hb $25
 ; N64-NEXT:    nop
-; N64-NEXT:    daddiu $1, $zero, %higher(memset)
-; N64-NEXT:    lui $2, %highest(memset)
-; N64-NEXT:    daddu $1, $2, $1
-; N64-NEXT:    dsll $1, $1, 16
-; N64-NEXT:    lui $2, %hi(memset)
-; N64-NEXT:    daddu $1, $1, $2
-; N64-NEXT:    dsll $1, $1, 16
-; N64-NEXT:    daddiu $2, $zero, %lo(memset)
-; N64-NEXT:    daddu $25, $1, $2
 ; N64-NEXT:    lui $1, %highest(val)
 ; N64-NEXT:    daddiu $1, $1, %higher(val)
 ; N64-NEXT:    dsll $1, $1, 16
 ; N64-NEXT:    daddiu $1, $1, %hi(val)
 ; N64-NEXT:    dsll $1, $1, 16
+; N64-NEXT:    lui $2, %highest(memset)
 ; N64-NEXT:    daddiu $4, $1, %lo(val)
+; N64-NEXT:    daddiu $1, $2, %higher(memset)
+; N64-NEXT:    dsll $1, $1, 16
+; N64-NEXT:    daddiu $1, $1, %hi(memset)
+; N64-NEXT:    dsll $1, $1, 16
+; N64-NEXT:    daddiu $25, $1, %lo(memset)
 ; N64-NEXT:    daddiu $5, $zero, 0
 ; N64-NEXT:    jalr.hb $25
 ; N64-NEXT:    daddiu $6, $zero, 80
diff --git a/llvm/test/CodeGen/Mips/inlineasm-constraint-reg64.ll b/llvm/test/CodeGen/Mips/inlineasm-constraint-reg64.ll
index 3b078d6f7..ae0504980 100644
--- a/llvm/test/CodeGen/Mips/inlineasm-constraint-reg64.ll
+++ b/llvm/test/CodeGen/Mips/inlineasm-constraint-reg64.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ;
 ; Register constraint "r" shouldn't take long long unless
 ; The target is 64 bit.
@@ -7,13 +8,25 @@
 
 
 define i32 @main() nounwind {
+; CHECK-LABEL: main:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    daddiu $1, $zero, 7
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    .set push
+; CHECK-NEXT:    .set at
+; CHECK-NEXT:    .set macro
+; CHECK-NEXT:    .set reorder
+; CHECK-EMPTY:
+; CHECK-NEXT:    addiu $1, $1, 3
+; CHECK-EMPTY:
+; CHECK-NEXT:    .set pop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    addiu $2, $zero, 0
 entry:
 
 
 ; r with long long
-;CHECK:	#APP
-;CHECK:	addiu ${{[0-9]+}}, ${{[0-9]+}}, 3
-;CHECK:	#NO_APP
   tail call i64 asm sideeffect "addiu $0, $1, $2", "=r,r,i"(i64 7, i64 3) nounwind
   ret i32 0
 }
diff --git a/llvm/test/CodeGen/Mips/l3mc.ll b/llvm/test/CodeGen/Mips/l3mc.ll
index c6855fc03..ed829cb39 100644
--- a/llvm/test/CodeGen/Mips/l3mc.ll
+++ b/llvm/test/CodeGen/Mips/l3mc.ll
@@ -99,7 +99,7 @@ entry:
   ret void
 }
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 ; __call_stub_fp___fixunsdfsi:  __call_stub_fp___fixunsdfsi:
 ; __call_stub_fp___floatdidf:  __call_stub_fp___floatdidf:
diff --git a/llvm/test/CodeGen/Mips/lcb2.ll b/llvm/test/CodeGen/Mips/lcb2.ll
index b15a5085f..4987c606e 100644
--- a/llvm/test/CodeGen/Mips/lcb2.ll
+++ b/llvm/test/CodeGen/Mips/lcb2.ll
@@ -115,7 +115,7 @@ if.end:                                           ; preds = %if.then, %entry
 ; lcb: 	.end	btz
 ; lcbn:	.end	btz
 
-attributes #0 = { nounwind optsize "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind optsize "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind }
 
 !llvm.ident = !{!0}
diff --git a/llvm/test/CodeGen/Mips/lcb3c.ll b/llvm/test/CodeGen/Mips/lcb3c.ll
index b953229a1..a41c73819 100644
--- a/llvm/test/CodeGen/Mips/lcb3c.ll
+++ b/llvm/test/CodeGen/Mips/lcb3c.ll
@@ -51,7 +51,7 @@ if.end:                                           ; preds = %if.else, %if.then
 ; lcb:	jal	$BB1_2	# branch
 ; lcb: $BB1_1:                                 # %if.then
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind }
 
 
diff --git a/llvm/test/CodeGen/Mips/lcb4a.ll b/llvm/test/CodeGen/Mips/lcb4a.ll
index 016e895d1..c4fcbc240 100644
--- a/llvm/test/CodeGen/Mips/lcb4a.ll
+++ b/llvm/test/CodeGen/Mips/lcb4a.ll
@@ -55,7 +55,7 @@ if.end:                                           ; preds = %if.else, %if.then
 ; ci:	nop
 ; ci: $BB1_1:                                 # %if.else
 
-attributes #0 = { nounwind optsize "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind optsize "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind }
 
 
diff --git a/llvm/test/CodeGen/Mips/lcb5.ll b/llvm/test/CodeGen/Mips/lcb5.ll
index 83f5fa0fb..96e924a44 100644
--- a/llvm/test/CodeGen/Mips/lcb5.ll
+++ b/llvm/test/CodeGen/Mips/lcb5.ll
@@ -216,7 +216,7 @@ if.end:                                           ; preds = %if.then, %entry
 ; ci: $BB7_1:
 ; ci:	.end	z4
 
-attributes #0 = { nounwind optsize "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind optsize "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind }
 
 
diff --git a/llvm/test/CodeGen/Mips/llvm-ir/lshr.ll b/llvm/test/CodeGen/Mips/llvm-ir/lshr.ll
index 57b1b819c..ed2bfc9fc 100644
--- a/llvm/test/CodeGen/Mips/llvm-ir/lshr.ll
+++ b/llvm/test/CodeGen/Mips/llvm-ir/lshr.ll
@@ -94,68 +94,57 @@ entry:
 define zeroext i8 @lshr_i8(i8 zeroext %a, i8 zeroext %b) {
 ; MIPS2-LABEL: lshr_i8:
 ; MIPS2:       # %bb.0: # %entry
-; MIPS2-NEXT:    srlv $1, $4, $5
 ; MIPS2-NEXT:    jr $ra
-; MIPS2-NEXT:    andi $2, $1, 255
+; MIPS2-NEXT:    srlv $2, $4, $5
 ;
 ; MIPS32-LABEL: lshr_i8:
 ; MIPS32:       # %bb.0: # %entry
-; MIPS32-NEXT:    srlv $1, $4, $5
 ; MIPS32-NEXT:    jr $ra
-; MIPS32-NEXT:    andi $2, $1, 255
+; MIPS32-NEXT:    srlv $2, $4, $5
 ;
 ; MIPS32R2-LABEL: lshr_i8:
 ; MIPS32R2:       # %bb.0: # %entry
-; MIPS32R2-NEXT:    srlv $1, $4, $5
 ; MIPS32R2-NEXT:    jr $ra
-; MIPS32R2-NEXT:    andi $2, $1, 255
+; MIPS32R2-NEXT:    srlv $2, $4, $5
 ;
 ; MIPS32R6-LABEL: lshr_i8:
 ; MIPS32R6:       # %bb.0: # %entry
-; MIPS32R6-NEXT:    srlv $1, $4, $5
 ; MIPS32R6-NEXT:    jr $ra
-; MIPS32R6-NEXT:    andi $2, $1, 255
+; MIPS32R6-NEXT:    srlv $2, $4, $5
 ;
 ; MIPS3-LABEL: lshr_i8:
 ; MIPS3:       # %bb.0: # %entry
-; MIPS3-NEXT:    srlv $1, $4, $5
 ; MIPS3-NEXT:    jr $ra
-; MIPS3-NEXT:    andi $2, $1, 255
+; MIPS3-NEXT:    srlv $2, $4, $5
 ;
 ; MIPS4-LABEL: lshr_i8:
 ; MIPS4:       # %bb.0: # %entry
-; MIPS4-NEXT:    srlv $1, $4, $5
 ; MIPS4-NEXT:    jr $ra
-; MIPS4-NEXT:    andi $2, $1, 255
+; MIPS4-NEXT:    srlv $2, $4, $5
 ;
 ; MIPS64-LABEL: lshr_i8:
 ; MIPS64:       # %bb.0: # %entry
-; MIPS64-NEXT:    srlv $1, $4, $5
 ; MIPS64-NEXT:    jr $ra
-; MIPS64-NEXT:    andi $2, $1, 255
+; MIPS64-NEXT:    srlv $2, $4, $5
 ;
 ; MIPS64R2-LABEL: lshr_i8:
 ; MIPS64R2:       # %bb.0: # %entry
-; MIPS64R2-NEXT:    srlv $1, $4, $5
 ; MIPS64R2-NEXT:    jr $ra
-; MIPS64R2-NEXT:    andi $2, $1, 255
+; MIPS64R2-NEXT:    srlv $2, $4, $5
 ;
 ; MIPS64R6-LABEL: lshr_i8:
 ; MIPS64R6:       # %bb.0: # %entry
-; MIPS64R6-NEXT:    srlv $1, $4, $5
 ; MIPS64R6-NEXT:    jr $ra
-; MIPS64R6-NEXT:    andi $2, $1, 255
+; MIPS64R6-NEXT:    srlv $2, $4, $5
 ;
 ; MMR3-LABEL: lshr_i8:
 ; MMR3:       # %bb.0: # %entry
+; MMR3-NEXT:    jr $ra
 ; MMR3-NEXT:    srlv $2, $4, $5
-; MMR3-NEXT:    andi16 $2, $2, 255
-; MMR3-NEXT:    jrc $ra
 ;
 ; MMR6-LABEL: lshr_i8:
 ; MMR6:       # %bb.0: # %entry
 ; MMR6-NEXT:    srlv $2, $4, $5
-; MMR6-NEXT:    andi16 $2, $2, 255
 ; MMR6-NEXT:    jrc $ra
 entry:
 
@@ -166,68 +155,57 @@ entry:
 define zeroext i16 @lshr_i16(i16 zeroext %a, i16 zeroext %b) {
 ; MIPS2-LABEL: lshr_i16:
 ; MIPS2:       # %bb.0: # %entry
-; MIPS2-NEXT:    srlv $1, $4, $5
 ; MIPS2-NEXT:    jr $ra
-; MIPS2-NEXT:    andi $2, $1, 65535
+; MIPS2-NEXT:    srlv $2, $4, $5
 ;
 ; MIPS32-LABEL: lshr_i16:
 ; MIPS32:       # %bb.0: # %entry
-; MIPS32-NEXT:    srlv $1, $4, $5
 ; MIPS32-NEXT:    jr $ra
-; MIPS32-NEXT:    andi $2, $1, 65535
+; MIPS32-NEXT:    srlv $2, $4, $5
 ;
 ; MIPS32R2-LABEL: lshr_i16:
 ; MIPS32R2:       # %bb.0: # %entry
-; MIPS32R2-NEXT:    srlv $1, $4, $5
 ; MIPS32R2-NEXT:    jr $ra
-; MIPS32R2-NEXT:    andi $2, $1, 65535
+; MIPS32R2-NEXT:    srlv $2, $4, $5
 ;
 ; MIPS32R6-LABEL: lshr_i16:
 ; MIPS32R6:       # %bb.0: # %entry
-; MIPS32R6-NEXT:    srlv $1, $4, $5
 ; MIPS32R6-NEXT:    jr $ra
-; MIPS32R6-NEXT:    andi $2, $1, 65535
+; MIPS32R6-NEXT:    srlv $2, $4, $5
 ;
 ; MIPS3-LABEL: lshr_i16:
 ; MIPS3:       # %bb.0: # %entry
-; MIPS3-NEXT:    srlv $1, $4, $5
 ; MIPS3-NEXT:    jr $ra
-; MIPS3-NEXT:    andi $2, $1, 65535
+; MIPS3-NEXT:    srlv $2, $4, $5
 ;
 ; MIPS4-LABEL: lshr_i16:
 ; MIPS4:       # %bb.0: # %entry
-; MIPS4-NEXT:    srlv $1, $4, $5
 ; MIPS4-NEXT:    jr $ra
-; MIPS4-NEXT:    andi $2, $1, 65535
+; MIPS4-NEXT:    srlv $2, $4, $5
 ;
 ; MIPS64-LABEL: lshr_i16:
 ; MIPS64:       # %bb.0: # %entry
-; MIPS64-NEXT:    srlv $1, $4, $5
 ; MIPS64-NEXT:    jr $ra
-; MIPS64-NEXT:    andi $2, $1, 65535
+; MIPS64-NEXT:    srlv $2, $4, $5
 ;
 ; MIPS64R2-LABEL: lshr_i16:
 ; MIPS64R2:       # %bb.0: # %entry
-; MIPS64R2-NEXT:    srlv $1, $4, $5
 ; MIPS64R2-NEXT:    jr $ra
-; MIPS64R2-NEXT:    andi $2, $1, 65535
+; MIPS64R2-NEXT:    srlv $2, $4, $5
 ;
 ; MIPS64R6-LABEL: lshr_i16:
 ; MIPS64R6:       # %bb.0: # %entry
-; MIPS64R6-NEXT:    srlv $1, $4, $5
 ; MIPS64R6-NEXT:    jr $ra
-; MIPS64R6-NEXT:    andi $2, $1, 65535
+; MIPS64R6-NEXT:    srlv $2, $4, $5
 ;
 ; MMR3-LABEL: lshr_i16:
 ; MMR3:       # %bb.0: # %entry
+; MMR3-NEXT:    jr $ra
 ; MMR3-NEXT:    srlv $2, $4, $5
-; MMR3-NEXT:    andi16 $2, $2, 65535
-; MMR3-NEXT:    jrc $ra
 ;
 ; MMR6-LABEL: lshr_i16:
 ; MMR6:       # %bb.0: # %entry
 ; MMR6-NEXT:    srlv $2, $4, $5
-; MMR6-NEXT:    andi16 $2, $2, 65535
 ; MMR6-NEXT:    jrc $ra
 entry:
 
@@ -298,22 +276,22 @@ entry:
 
 define signext i64 @lshr_i64(i64 signext %a, i64 signext %b) {
 ; MIPS2-LABEL: lshr_i64:
-; MIPS2:       # %bb.0:
-; MIPS2-NEXT:    srlv  $6, $4, $7
-; MIPS2-NEXT:    andi  $1, $7, 32
-; MIPS2-NEXT:    bnez  $1, $BB4_2
+; MIPS2:       # %bb.0: # %entry
+; MIPS2-NEXT:    srlv $6, $4, $7
+; MIPS2-NEXT:    andi $1, $7, 32
+; MIPS2-NEXT:    bnez $1, $BB4_2
 ; MIPS2-NEXT:    addiu $2, $zero, 0
-; MIPS2-NEXT:  # %bb.1:
-; MIPS2-NEXT:    srlv  $1, $5, $7
+; MIPS2-NEXT:  # %bb.1: # %entry
+; MIPS2-NEXT:    srlv $1, $5, $7
 ; MIPS2-NEXT:    not $2, $7
 ; MIPS2-NEXT:    sll $3, $4, 1
-; MIPS2-NEXT:    sllv  $2, $3, $2
-; MIPS2-NEXT:    or  $3, $2, $1
-; MIPS2-NEXT:    jr  $ra
-; MIPS2-NEXT:    move  $2, $6
+; MIPS2-NEXT:    sllv $2, $3, $2
+; MIPS2-NEXT:    or $3, $2, $1
+; MIPS2-NEXT:    jr $ra
+; MIPS2-NEXT:    move $2, $6
 ; MIPS2-NEXT:  $BB4_2:
-; MIPS2-NEXT:    jr  $ra
-; MIPS2-NEXT:    move  $3, $6
+; MIPS2-NEXT:    jr $ra
+; MIPS2-NEXT:    move $3, $6
 ;
 ; MIPS32-LABEL: lshr_i64:
 ; MIPS32:       # %bb.0: # %entry
@@ -417,119 +395,119 @@ entry:
 
 define signext i128 @lshr_i128(i128 signext %a, i128 signext %b) {
 ; MIPS2-LABEL: lshr_i128:
-; MIPS2:       # %bb.0:
-; MIPS2-NEXT:    lw  $2, 28($sp)
+; MIPS2:       # %bb.0: # %entry
+; MIPS2-NEXT:    lw $2, 28($sp)
 ; MIPS2-NEXT:    addiu $1, $zero, 64
-; MIPS2-NEXT:    subu  $12, $1, $2
-; MIPS2-NEXT:    sllv  $10, $5, $12
-; MIPS2-NEXT:    andi  $15, $12, 32
-; MIPS2-NEXT:    andi  $8, $2, 32
+; MIPS2-NEXT:    subu $12, $1, $2
+; MIPS2-NEXT:    sllv $10, $5, $12
+; MIPS2-NEXT:    andi $15, $12, 32
+; MIPS2-NEXT:    andi $8, $2, 32
 ; MIPS2-NEXT:    addiu $3, $zero, 0
-; MIPS2-NEXT:    bnez  $15, $BB5_2
+; MIPS2-NEXT:    bnez $15, $BB5_2
 ; MIPS2-NEXT:    addiu $13, $zero, 0
-; MIPS2-NEXT:  # %bb.1:
-; MIPS2-NEXT:    move  $13, $10
-; MIPS2-NEXT:  $BB5_2:
+; MIPS2-NEXT:  # %bb.1: # %entry
+; MIPS2-NEXT:    move $13, $10
+; MIPS2-NEXT:  $BB5_2: # %entry
 ; MIPS2-NEXT:    not $9, $2
-; MIPS2-NEXT:    bnez  $8, $BB5_5
-; MIPS2-NEXT:    srlv  $24, $6, $2
-; MIPS2-NEXT:  # %bb.3:
+; MIPS2-NEXT:    bnez $8, $BB5_5
+; MIPS2-NEXT:    srlv $24, $6, $2
+; MIPS2-NEXT:  # %bb.3: # %entry
 ; MIPS2-NEXT:    sll $1, $6, 1
-; MIPS2-NEXT:    srlv  $11, $7, $2
-; MIPS2-NEXT:    sllv  $1, $1, $9
-; MIPS2-NEXT:    or  $14, $1, $11
-; MIPS2-NEXT:    bnez  $15, $BB5_7
-; MIPS2-NEXT:    move  $11, $24
-; MIPS2-NEXT:  # %bb.4:
+; MIPS2-NEXT:    srlv $11, $7, $2
+; MIPS2-NEXT:    sllv $1, $1, $9
+; MIPS2-NEXT:    or $14, $1, $11
+; MIPS2-NEXT:    bnez $15, $BB5_7
+; MIPS2-NEXT:    move $11, $24
+; MIPS2-NEXT:  # %bb.4: # %entry
 ; MIPS2-NEXT:    b $BB5_6
 ; MIPS2-NEXT:    nop
 ; MIPS2-NEXT:  $BB5_5:
 ; MIPS2-NEXT:    addiu $11, $zero, 0
-; MIPS2-NEXT:    bnez  $15, $BB5_7
-; MIPS2-NEXT:    move  $14, $24
-; MIPS2-NEXT:  $BB5_6:
-; MIPS2-NEXT:    sllv  $1, $4, $12
+; MIPS2-NEXT:    bnez $15, $BB5_7
+; MIPS2-NEXT:    move $14, $24
+; MIPS2-NEXT:  $BB5_6: # %entry
+; MIPS2-NEXT:    sllv $1, $4, $12
 ; MIPS2-NEXT:    not $10, $12
 ; MIPS2-NEXT:    srl $12, $5, 1
-; MIPS2-NEXT:    srlv  $10, $12, $10
-; MIPS2-NEXT:    or  $10, $1, $10
-; MIPS2-NEXT:  $BB5_7:
+; MIPS2-NEXT:    srlv $10, $12, $10
+; MIPS2-NEXT:    or $10, $1, $10
+; MIPS2-NEXT:  $BB5_7: # %entry
 ; MIPS2-NEXT:    addiu $15, $2, -64
 ; MIPS2-NEXT:    sll $12, $4, 1
-; MIPS2-NEXT:    andi  $1, $15, 32
-; MIPS2-NEXT:    bnez  $1, $BB5_10
-; MIPS2-NEXT:    srlv  $25, $4, $15
-; MIPS2-NEXT:  # %bb.8:
-; MIPS2-NEXT:    srlv  $1, $5, $15
+; MIPS2-NEXT:    andi $1, $15, 32
+; MIPS2-NEXT:    bnez $1, $BB5_10
+; MIPS2-NEXT:    srlv $25, $4, $15
+; MIPS2-NEXT:  # %bb.8: # %entry
+; MIPS2-NEXT:    srlv $1, $5, $15
 ; MIPS2-NEXT:    not $15, $15
-; MIPS2-NEXT:    sllv  $15, $12, $15
-; MIPS2-NEXT:    or  $24, $15, $1
-; MIPS2-NEXT:    move  $15, $25
+; MIPS2-NEXT:    sllv $15, $12, $15
+; MIPS2-NEXT:    or $24, $15, $1
+; MIPS2-NEXT:    move $15, $25
 ; MIPS2-NEXT:    sltiu $25, $2, 64
-; MIPS2-NEXT:    beqz  $25, $BB5_12
+; MIPS2-NEXT:    beqz $25, $BB5_12
 ; MIPS2-NEXT:    nop
-; MIPS2-NEXT:  # %bb.9:
+; MIPS2-NEXT:  # %bb.9: # %entry
 ; MIPS2-NEXT:    b $BB5_11
 ; MIPS2-NEXT:    nop
 ; MIPS2-NEXT:  $BB5_10:
-; MIPS2-NEXT:    move  $24, $25
+; MIPS2-NEXT:    move $24, $25
 ; MIPS2-NEXT:    sltiu $25, $2, 64
-; MIPS2-NEXT:    beqz  $25, $BB5_12
+; MIPS2-NEXT:    beqz $25, $BB5_12
 ; MIPS2-NEXT:    addiu $15, $zero, 0
 ; MIPS2-NEXT:  $BB5_11:
-; MIPS2-NEXT:    or  $24, $14, $13
-; MIPS2-NEXT:  $BB5_12:
+; MIPS2-NEXT:    or $24, $14, $13
+; MIPS2-NEXT:  $BB5_12: # %entry
 ; MIPS2-NEXT:    sltiu $13, $2, 1
-; MIPS2-NEXT:    beqz  $13, $BB5_19
+; MIPS2-NEXT:    beqz $13, $BB5_19
 ; MIPS2-NEXT:    nop
-; MIPS2-NEXT:  # %bb.13:
-; MIPS2-NEXT:    bnez  $25, $BB5_20
+; MIPS2-NEXT:  # %bb.13: # %entry
+; MIPS2-NEXT:    bnez $25, $BB5_20
 ; MIPS2-NEXT:    nop
-; MIPS2-NEXT:  $BB5_14:
-; MIPS2-NEXT:    bnez  $13, $BB5_16
+; MIPS2-NEXT:  $BB5_14: # %entry
+; MIPS2-NEXT:    bnez $13, $BB5_16
 ; MIPS2-NEXT:    addiu $10, $zero, 63
-; MIPS2-NEXT:  $BB5_15:
-; MIPS2-NEXT:    move  $6, $15
-; MIPS2-NEXT:  $BB5_16:
-; MIPS2-NEXT:    sltu  $10, $10, $2
-; MIPS2-NEXT:    bnez  $8, $BB5_22
-; MIPS2-NEXT:    srlv  $11, $4, $2
-; MIPS2-NEXT:  # %bb.17:
-; MIPS2-NEXT:    srlv  $1, $5, $2
-; MIPS2-NEXT:    sllv  $2, $12, $9
-; MIPS2-NEXT:    or  $4, $2, $1
-; MIPS2-NEXT:    move  $5, $11
-; MIPS2-NEXT:    bnez  $10, $BB5_24
+; MIPS2-NEXT:  $BB5_15: # %entry
+; MIPS2-NEXT:    move $6, $15
+; MIPS2-NEXT:  $BB5_16: # %entry
+; MIPS2-NEXT:    sltu $10, $10, $2
+; MIPS2-NEXT:    bnez $8, $BB5_22
+; MIPS2-NEXT:    srlv $11, $4, $2
+; MIPS2-NEXT:  # %bb.17: # %entry
+; MIPS2-NEXT:    srlv $1, $5, $2
+; MIPS2-NEXT:    sllv $2, $12, $9
+; MIPS2-NEXT:    or $4, $2, $1
+; MIPS2-NEXT:    move $5, $11
+; MIPS2-NEXT:    bnez $10, $BB5_24
 ; MIPS2-NEXT:    addiu $2, $zero, 0
-; MIPS2-NEXT:  # %bb.18:
+; MIPS2-NEXT:  # %bb.18: # %entry
 ; MIPS2-NEXT:    b $BB5_23
 ; MIPS2-NEXT:    nop
-; MIPS2-NEXT:  $BB5_19:
-; MIPS2-NEXT:    beqz  $25, $BB5_14
-; MIPS2-NEXT:    move  $7, $24
+; MIPS2-NEXT:  $BB5_19: # %entry
+; MIPS2-NEXT:    beqz $25, $BB5_14
+; MIPS2-NEXT:    move $7, $24
 ; MIPS2-NEXT:  $BB5_20:
-; MIPS2-NEXT:    or  $15, $11, $10
-; MIPS2-NEXT:    bnez  $13, $BB5_16
+; MIPS2-NEXT:    or $15, $11, $10
+; MIPS2-NEXT:    bnez $13, $BB5_16
 ; MIPS2-NEXT:    addiu $10, $zero, 63
 ; MIPS2-NEXT:  # %bb.21:
 ; MIPS2-NEXT:    b $BB5_15
 ; MIPS2-NEXT:    nop
 ; MIPS2-NEXT:  $BB5_22:
 ; MIPS2-NEXT:    addiu $5, $zero, 0
-; MIPS2-NEXT:    move  $4, $11
-; MIPS2-NEXT:    bnez  $10, $BB5_24
+; MIPS2-NEXT:    move $4, $11
+; MIPS2-NEXT:    bnez $10, $BB5_24
 ; MIPS2-NEXT:    addiu $2, $zero, 0
-; MIPS2-NEXT:  $BB5_23:
-; MIPS2-NEXT:    move  $2, $5
-; MIPS2-NEXT:  $BB5_24:
-; MIPS2-NEXT:    bnez  $10, $BB5_26
+; MIPS2-NEXT:  $BB5_23: # %entry
+; MIPS2-NEXT:    move $2, $5
+; MIPS2-NEXT:  $BB5_24: # %entry
+; MIPS2-NEXT:    bnez $10, $BB5_26
 ; MIPS2-NEXT:    nop
-; MIPS2-NEXT:  # %bb.25:
-; MIPS2-NEXT:    move  $3, $4
-; MIPS2-NEXT:  $BB5_26:
-; MIPS2-NEXT:    move  $4, $6
-; MIPS2-NEXT:    jr  $ra
-; MIPS2-NEXT:    move  $5, $7
+; MIPS2-NEXT:  # %bb.25: # %entry
+; MIPS2-NEXT:    move $3, $4
+; MIPS2-NEXT:  $BB5_26: # %entry
+; MIPS2-NEXT:    move $4, $6
+; MIPS2-NEXT:    jr $ra
+; MIPS2-NEXT:    move $5, $7
 ;
 ; MIPS32-LABEL: lshr_i128:
 ; MIPS32:       # %bb.0: # %entry
@@ -713,23 +691,23 @@ define signext i128 @lshr_i128(i128 signext %a, i128 signext %b) {
 ; MIPS32R6-NEXT:    addiu $sp, $sp, 8
 ;
 ; MIPS3-LABEL: lshr_i128:
-; MIPS3:       # %bb.0:
+; MIPS3:       # %bb.0: # %entry
 ; MIPS3-NEXT:    sll $3, $7, 0
 ; MIPS3-NEXT:    dsrlv $6, $4, $7
-; MIPS3-NEXT:    andi  $1, $3, 64
-; MIPS3-NEXT:    bnez  $1, .LBB5_2
-; MIPS3-NEXT:    daddiu  $2, $zero, 0
-; MIPS3-NEXT:  # %bb.1:
+; MIPS3-NEXT:    andi $1, $3, 64
+; MIPS3-NEXT:    bnez $1, .LBB5_2
+; MIPS3-NEXT:    daddiu $2, $zero, 0
+; MIPS3-NEXT:  # %bb.1: # %entry
 ; MIPS3-NEXT:    dsrlv $1, $5, $7
-; MIPS3-NEXT:    dsll  $2, $4, 1
+; MIPS3-NEXT:    dsll $2, $4, 1
 ; MIPS3-NEXT:    not $3, $3
 ; MIPS3-NEXT:    dsllv $2, $2, $3
-; MIPS3-NEXT:    or  $3, $2, $1
-; MIPS3-NEXT:    jr  $ra
-; MIPS3-NEXT:    move  $2, $6
+; MIPS3-NEXT:    or $3, $2, $1
+; MIPS3-NEXT:    jr $ra
+; MIPS3-NEXT:    move $2, $6
 ; MIPS3-NEXT:  .LBB5_2:
-; MIPS3-NEXT:    jr  $ra
-; MIPS3-NEXT:    move  $3, $6
+; MIPS3-NEXT:    jr $ra
+; MIPS3-NEXT:    move $3, $6
 ;
 ; MIPS4-LABEL: lshr_i128:
 ; MIPS4:       # %bb.0: # %entry
diff --git a/llvm/test/CodeGen/Mips/llvm-ir/shl.ll b/llvm/test/CodeGen/Mips/llvm-ir/shl.ll
index bd69a3ff0..a8d829bef 100644
--- a/llvm/test/CodeGen/Mips/llvm-ir/shl.ll
+++ b/llvm/test/CodeGen/Mips/llvm-ir/shl.ll
@@ -330,28 +330,28 @@ entry:
 
 define signext i64 @shl_i64(i64 signext %a, i64 signext %b) {
 ; MIPS2-LABEL: shl_i64:
-; MIPS2:       # %bb.0:
-; MIPS2-NEXT:    sllv  $6, $5, $7
-; MIPS2-NEXT:    andi  $8, $7, 32
-; MIPS2-NEXT:    beqz  $8, $BB4_3
-; MIPS2-NEXT:    move  $2, $6
-; MIPS2-NEXT:  # %bb.1:
-; MIPS2-NEXT:    beqz  $8, $BB4_4
+; MIPS2:       # %bb.0: # %entry
+; MIPS2-NEXT:    sllv $6, $5, $7
+; MIPS2-NEXT:    andi $8, $7, 32
+; MIPS2-NEXT:    beqz $8, $BB4_3
+; MIPS2-NEXT:    move $2, $6
+; MIPS2-NEXT:  # %bb.1: # %entry
+; MIPS2-NEXT:    beqz $8, $BB4_4
 ; MIPS2-NEXT:    addiu $3, $zero, 0
-; MIPS2-NEXT:  $BB4_2:
-; MIPS2-NEXT:    jr  $ra
+; MIPS2-NEXT:  $BB4_2: # %entry
+; MIPS2-NEXT:    jr $ra
 ; MIPS2-NEXT:    nop
-; MIPS2-NEXT:  $BB4_3:
-; MIPS2-NEXT:    sllv  $1, $4, $7
+; MIPS2-NEXT:  $BB4_3: # %entry
+; MIPS2-NEXT:    sllv $1, $4, $7
 ; MIPS2-NEXT:    not $2, $7
 ; MIPS2-NEXT:    srl $3, $5, 1
-; MIPS2-NEXT:    srlv  $2, $3, $2
-; MIPS2-NEXT:    or  $2, $1, $2
-; MIPS2-NEXT:    bnez  $8, $BB4_2
+; MIPS2-NEXT:    srlv $2, $3, $2
+; MIPS2-NEXT:    or $2, $1, $2
+; MIPS2-NEXT:    bnez $8, $BB4_2
 ; MIPS2-NEXT:    addiu $3, $zero, 0
-; MIPS2-NEXT:  $BB4_4:
-; MIPS2-NEXT:    jr  $ra
-; MIPS2-NEXT:    move  $3, $6
+; MIPS2-NEXT:  $BB4_4: # %entry
+; MIPS2-NEXT:    jr $ra
+; MIPS2-NEXT:    move $3, $6
 ;
 ; MIPS32-LABEL: shl_i64:
 ; MIPS32:       # %bb.0: # %entry
@@ -455,129 +455,129 @@ entry:
 
 define signext i128 @shl_i128(i128 signext %a, i128 signext %b) {
 ; MIPS2-LABEL: shl_i128:
-; MIPS2:       # %bb.0:
+; MIPS2:       # %bb.0: # %entry
 ; MIPS2-NEXT:    addiu $sp, $sp, -8
 ; MIPS2-NEXT:    .cfi_def_cfa_offset 8
-; MIPS2-NEXT:    sw  $17, 4($sp)
-; MIPS2-NEXT:    sw  $16, 0($sp)
+; MIPS2-NEXT:    sw $17, 4($sp) # 4-byte Folded Spill
+; MIPS2-NEXT:    sw $16, 0($sp) # 4-byte Folded Spill
 ; MIPS2-NEXT:    .cfi_offset 17, -4
 ; MIPS2-NEXT:    .cfi_offset 16, -8
-; MIPS2-NEXT:    lw  $8, 36($sp)
+; MIPS2-NEXT:    lw $8, 36($sp)
 ; MIPS2-NEXT:    addiu $1, $zero, 64
-; MIPS2-NEXT:    subu  $3, $1, $8
-; MIPS2-NEXT:    srlv  $9, $6, $3
-; MIPS2-NEXT:    andi  $1, $3, 32
-; MIPS2-NEXT:    bnez  $1, $BB5_2
+; MIPS2-NEXT:    subu $3, $1, $8
+; MIPS2-NEXT:    srlv $9, $6, $3
+; MIPS2-NEXT:    andi $1, $3, 32
+; MIPS2-NEXT:    bnez $1, $BB5_2
 ; MIPS2-NEXT:    addiu $2, $zero, 0
-; MIPS2-NEXT:  # %bb.1:
-; MIPS2-NEXT:    srlv  $1, $7, $3
+; MIPS2-NEXT:  # %bb.1: # %entry
+; MIPS2-NEXT:    srlv $1, $7, $3
 ; MIPS2-NEXT:    not $3, $3
 ; MIPS2-NEXT:    sll $10, $6, 1
-; MIPS2-NEXT:    sllv  $3, $10, $3
-; MIPS2-NEXT:    or  $3, $3, $1
+; MIPS2-NEXT:    sllv $3, $10, $3
+; MIPS2-NEXT:    or $3, $3, $1
 ; MIPS2-NEXT:    b $BB5_3
-; MIPS2-NEXT:    move  $15, $9
+; MIPS2-NEXT:    move $15, $9
 ; MIPS2-NEXT:  $BB5_2:
 ; MIPS2-NEXT:    addiu $15, $zero, 0
-; MIPS2-NEXT:    move  $3, $9
-; MIPS2-NEXT:  $BB5_3:
+; MIPS2-NEXT:    move $3, $9
+; MIPS2-NEXT:  $BB5_3: # %entry
 ; MIPS2-NEXT:    not $13, $8
-; MIPS2-NEXT:    sllv  $9, $5, $8
-; MIPS2-NEXT:    andi  $10, $8, 32
-; MIPS2-NEXT:    bnez  $10, $BB5_5
-; MIPS2-NEXT:    move  $25, $9
-; MIPS2-NEXT:  # %bb.4:
-; MIPS2-NEXT:    sllv  $1, $4, $8
+; MIPS2-NEXT:    sllv $9, $5, $8
+; MIPS2-NEXT:    andi $10, $8, 32
+; MIPS2-NEXT:    bnez $10, $BB5_5
+; MIPS2-NEXT:    move $25, $9
+; MIPS2-NEXT:  # %bb.4: # %entry
+; MIPS2-NEXT:    sllv $1, $4, $8
 ; MIPS2-NEXT:    srl $11, $5, 1
-; MIPS2-NEXT:    srlv  $11, $11, $13
-; MIPS2-NEXT:    or  $25, $1, $11
-; MIPS2-NEXT:  $BB5_5:
+; MIPS2-NEXT:    srlv $11, $11, $13
+; MIPS2-NEXT:    or $25, $1, $11
+; MIPS2-NEXT:  $BB5_5: # %entry
 ; MIPS2-NEXT:    addiu $14, $8, -64
 ; MIPS2-NEXT:    srl $24, $7, 1
-; MIPS2-NEXT:    sllv  $11, $7, $14
-; MIPS2-NEXT:    andi  $12, $14, 32
-; MIPS2-NEXT:    bnez  $12, $BB5_7
-; MIPS2-NEXT:    move  $gp, $11
-; MIPS2-NEXT:  # %bb.6:
-; MIPS2-NEXT:    sllv  $1, $6, $14
+; MIPS2-NEXT:    sllv $11, $7, $14
+; MIPS2-NEXT:    andi $12, $14, 32
+; MIPS2-NEXT:    bnez $12, $BB5_7
+; MIPS2-NEXT:    move $gp, $11
+; MIPS2-NEXT:  # %bb.6: # %entry
+; MIPS2-NEXT:    sllv $1, $6, $14
 ; MIPS2-NEXT:    not $14, $14
-; MIPS2-NEXT:    srlv  $14, $24, $14
-; MIPS2-NEXT:    or  $gp, $1, $14
-; MIPS2-NEXT:  $BB5_7:
+; MIPS2-NEXT:    srlv $14, $24, $14
+; MIPS2-NEXT:    or $gp, $1, $14
+; MIPS2-NEXT:  $BB5_7: # %entry
 ; MIPS2-NEXT:    sltiu $14, $8, 64
-; MIPS2-NEXT:    beqz  $14, $BB5_9
+; MIPS2-NEXT:    beqz $14, $BB5_9
 ; MIPS2-NEXT:    nop
 ; MIPS2-NEXT:  # %bb.8:
-; MIPS2-NEXT:    or  $gp, $25, $15
-; MIPS2-NEXT:  $BB5_9:
-; MIPS2-NEXT:    sllv  $25, $7, $8
-; MIPS2-NEXT:    bnez  $10, $BB5_11
+; MIPS2-NEXT:    or $gp, $25, $15
+; MIPS2-NEXT:  $BB5_9: # %entry
+; MIPS2-NEXT:    sllv $25, $7, $8
+; MIPS2-NEXT:    bnez $10, $BB5_11
 ; MIPS2-NEXT:    addiu $17, $zero, 0
-; MIPS2-NEXT:  # %bb.10:
-; MIPS2-NEXT:    move  $17, $25
-; MIPS2-NEXT:  $BB5_11:
+; MIPS2-NEXT:  # %bb.10: # %entry
+; MIPS2-NEXT:    move $17, $25
+; MIPS2-NEXT:  $BB5_11: # %entry
 ; MIPS2-NEXT:    addiu $1, $zero, 63
 ; MIPS2-NEXT:    sltiu $15, $8, 1
-; MIPS2-NEXT:    beqz  $15, $BB5_21
-; MIPS2-NEXT:    sltu  $16, $1, $8
-; MIPS2-NEXT:  # %bb.12:
-; MIPS2-NEXT:    beqz  $16, $BB5_22
+; MIPS2-NEXT:    beqz $15, $BB5_21
+; MIPS2-NEXT:    sltu $16, $1, $8
+; MIPS2-NEXT:  # %bb.12: # %entry
+; MIPS2-NEXT:    beqz $16, $BB5_22
 ; MIPS2-NEXT:    addiu $7, $zero, 0
-; MIPS2-NEXT:  $BB5_13:
-; MIPS2-NEXT:    beqz  $10, $BB5_23
+; MIPS2-NEXT:  $BB5_13: # %entry
+; MIPS2-NEXT:    beqz $10, $BB5_23
 ; MIPS2-NEXT:    nop
-; MIPS2-NEXT:  $BB5_14:
-; MIPS2-NEXT:    beqz  $16, $BB5_24
+; MIPS2-NEXT:  $BB5_14: # %entry
+; MIPS2-NEXT:    beqz $16, $BB5_24
 ; MIPS2-NEXT:    addiu $6, $zero, 0
-; MIPS2-NEXT:  $BB5_15:
-; MIPS2-NEXT:    beqz  $10, $BB5_25
+; MIPS2-NEXT:  $BB5_15: # %entry
+; MIPS2-NEXT:    beqz $10, $BB5_25
 ; MIPS2-NEXT:    addiu $8, $zero, 0
-; MIPS2-NEXT:  $BB5_16:
-; MIPS2-NEXT:    beqz  $12, $BB5_26
+; MIPS2-NEXT:  $BB5_16: # %entry
+; MIPS2-NEXT:    beqz $12, $BB5_26
 ; MIPS2-NEXT:    nop
-; MIPS2-NEXT:  $BB5_17:
-; MIPS2-NEXT:    bnez  $14, $BB5_27
+; MIPS2-NEXT:  $BB5_17: # %entry
+; MIPS2-NEXT:    bnez $14, $BB5_27
 ; MIPS2-NEXT:    nop
-; MIPS2-NEXT:  $BB5_18:
-; MIPS2-NEXT:    bnez  $15, $BB5_20
+; MIPS2-NEXT:  $BB5_18: # %entry
+; MIPS2-NEXT:    bnez $15, $BB5_20
 ; MIPS2-NEXT:    nop
-; MIPS2-NEXT:  $BB5_19:
-; MIPS2-NEXT:    move  $5, $2
-; MIPS2-NEXT:  $BB5_20:
-; MIPS2-NEXT:    move  $2, $4
-; MIPS2-NEXT:    move  $3, $5
-; MIPS2-NEXT:    move  $4, $6
-; MIPS2-NEXT:    move  $5, $7
-; MIPS2-NEXT:    lw  $16, 0($sp)
-; MIPS2-NEXT:    lw  $17, 4($sp)
-; MIPS2-NEXT:    jr  $ra
+; MIPS2-NEXT:  $BB5_19: # %entry
+; MIPS2-NEXT:    move $5, $2
+; MIPS2-NEXT:  $BB5_20: # %entry
+; MIPS2-NEXT:    move $2, $4
+; MIPS2-NEXT:    move $3, $5
+; MIPS2-NEXT:    move $4, $6
+; MIPS2-NEXT:    move $5, $7
+; MIPS2-NEXT:    lw $16, 0($sp) # 4-byte Folded Reload
+; MIPS2-NEXT:    lw $17, 4($sp) # 4-byte Folded Reload
+; MIPS2-NEXT:    jr $ra
 ; MIPS2-NEXT:    addiu $sp, $sp, 8
-; MIPS2-NEXT:  $BB5_21:
-; MIPS2-NEXT:    move  $4, $gp
-; MIPS2-NEXT:    bnez  $16, $BB5_13
+; MIPS2-NEXT:  $BB5_21: # %entry
+; MIPS2-NEXT:    move $4, $gp
+; MIPS2-NEXT:    bnez $16, $BB5_13
 ; MIPS2-NEXT:    addiu $7, $zero, 0
-; MIPS2-NEXT:  $BB5_22:
-; MIPS2-NEXT:    bnez  $10, $BB5_14
-; MIPS2-NEXT:    move  $7, $17
-; MIPS2-NEXT:  $BB5_23:
-; MIPS2-NEXT:    sllv  $1, $6, $8
-; MIPS2-NEXT:    srlv  $6, $24, $13
-; MIPS2-NEXT:    or  $25, $1, $6
-; MIPS2-NEXT:    bnez  $16, $BB5_15
+; MIPS2-NEXT:  $BB5_22: # %entry
+; MIPS2-NEXT:    bnez $10, $BB5_14
+; MIPS2-NEXT:    move $7, $17
+; MIPS2-NEXT:  $BB5_23: # %entry
+; MIPS2-NEXT:    sllv $1, $6, $8
+; MIPS2-NEXT:    srlv $6, $24, $13
+; MIPS2-NEXT:    or $25, $1, $6
+; MIPS2-NEXT:    bnez $16, $BB5_15
 ; MIPS2-NEXT:    addiu $6, $zero, 0
-; MIPS2-NEXT:  $BB5_24:
-; MIPS2-NEXT:    move  $6, $25
-; MIPS2-NEXT:    bnez  $10, $BB5_16
+; MIPS2-NEXT:  $BB5_24: # %entry
+; MIPS2-NEXT:    move $6, $25
+; MIPS2-NEXT:    bnez $10, $BB5_16
 ; MIPS2-NEXT:    addiu $8, $zero, 0
-; MIPS2-NEXT:  $BB5_25:
-; MIPS2-NEXT:    bnez  $12, $BB5_17
-; MIPS2-NEXT:    move  $8, $9
-; MIPS2-NEXT:  $BB5_26:
-; MIPS2-NEXT:    beqz  $14, $BB5_18
-; MIPS2-NEXT:    move  $2, $11
+; MIPS2-NEXT:  $BB5_25: # %entry
+; MIPS2-NEXT:    bnez $12, $BB5_17
+; MIPS2-NEXT:    move $8, $9
+; MIPS2-NEXT:  $BB5_26: # %entry
+; MIPS2-NEXT:    beqz $14, $BB5_18
+; MIPS2-NEXT:    move $2, $11
 ; MIPS2-NEXT:  $BB5_27:
-; MIPS2-NEXT:    bnez  $15, $BB5_20
-; MIPS2-NEXT:    or  $2, $8, $3
+; MIPS2-NEXT:    bnez $15, $BB5_20
+; MIPS2-NEXT:    or $2, $8, $3
 ; MIPS2-NEXT:  # %bb.28:
 ; MIPS2-NEXT:    b $BB5_19
 ; MIPS2-NEXT:    nop
@@ -759,29 +759,29 @@ define signext i128 @shl_i128(i128 signext %a, i128 signext %b) {
 ; MIPS32R6-NEXT:    move $5, $1
 ;
 ; MIPS3-LABEL: shl_i128:
-; MIPS3:       # %bb.0:
+; MIPS3:       # %bb.0: # %entry
 ; MIPS3-NEXT:    sll $3, $7, 0
 ; MIPS3-NEXT:    dsllv $6, $5, $7
-; MIPS3-NEXT:    andi  $8, $3, 64
-; MIPS3-NEXT:    beqz  $8, .LBB5_3
-; MIPS3-NEXT:    move  $2, $6
-; MIPS3-NEXT:  # %bb.1:
-; MIPS3-NEXT:    beqz  $8, .LBB5_4
-; MIPS3-NEXT:    daddiu  $3, $zero, 0
-; MIPS3-NEXT:  .LBB5_2:
-; MIPS3-NEXT:    jr  $ra
+; MIPS3-NEXT:    andi $8, $3, 64
+; MIPS3-NEXT:    beqz $8, .LBB5_3
+; MIPS3-NEXT:    move $2, $6
+; MIPS3-NEXT:  # %bb.1: # %entry
+; MIPS3-NEXT:    beqz $8, .LBB5_4
+; MIPS3-NEXT:    daddiu $3, $zero, 0
+; MIPS3-NEXT:  .LBB5_2: # %entry
+; MIPS3-NEXT:    jr $ra
 ; MIPS3-NEXT:    nop
-; MIPS3-NEXT:  .LBB5_3:
+; MIPS3-NEXT:  .LBB5_3: # %entry
 ; MIPS3-NEXT:    dsllv $1, $4, $7
-; MIPS3-NEXT:    dsrl  $2, $5, 1
+; MIPS3-NEXT:    dsrl $2, $5, 1
 ; MIPS3-NEXT:    not $3, $3
 ; MIPS3-NEXT:    dsrlv $2, $2, $3
-; MIPS3-NEXT:    or  $2, $1, $2
-; MIPS3-NEXT:    bnez  $8, .LBB5_2
-; MIPS3-NEXT:    daddiu  $3, $zero, 0
-; MIPS3-NEXT:  .LBB5_4:
-; MIPS3-NEXT:    jr  $ra
-; MIPS3-NEXT:    move  $3, $6
+; MIPS3-NEXT:    or $2, $1, $2
+; MIPS3-NEXT:    bnez $8, .LBB5_2
+; MIPS3-NEXT:    daddiu $3, $zero, 0
+; MIPS3-NEXT:  .LBB5_4: # %entry
+; MIPS3-NEXT:    jr $ra
+; MIPS3-NEXT:    move $3, $6
 ;
 ; MIPS4-LABEL: shl_i128:
 ; MIPS4:       # %bb.0: # %entry
diff --git a/llvm/test/CodeGen/Mips/long-calls.ll b/llvm/test/CodeGen/Mips/long-calls.ll
index d4652a546..0b1a76d56 100644
--- a/llvm/test/CodeGen/Mips/long-calls.ll
+++ b/llvm/test/CodeGen/Mips/long-calls.ll
@@ -33,9 +33,8 @@ define void @caller() {
 ; ON32: addiu  $25, $1, %lo(callee)
 ; ON32: jalr   $25
 
-; ON32: addiu  $1, $zero, %lo(memset)
-; ON32: lui    $2, %hi(memset)
-; ON32: addu   $25, $2, $1
+; ON32: lui    $1, %hi(memset)
+; ON32: addiu  $25, $1, %lo(memset)
 ; ON32: jalr   $25
 
 ; ON64: lui     $1, %highest(callee)
@@ -44,11 +43,12 @@ define void @caller() {
 ; ON64: daddiu  $25, $1, %lo(callee)
 ; ON64: jalr    $25
 
-; ON64: daddiu  $1, $zero, %higher(memset)
 ; ON64: lui     $2, %highest(memset)
-; ON64: lui     $2, %hi(memset)
-; ON64: daddiu  $2, $zero, %lo(memset)
-; ON64: daddu   $25, $1, $2
+; ON64: daddiu  $1, $2, %higher(memset)
+; ON64: dsll    $1, $1, 16
+; ON64: daddiu  $1, $1, %hi(memset)
+; ON64: dsll    $1, $1, 16
+; ON64: daddiu  $25, $1, %lo(memset)
 ; ON64: jalr    $25
 
   call void @callee()
diff --git a/llvm/test/CodeGen/Mips/mbrsize4a.ll b/llvm/test/CodeGen/Mips/mbrsize4a.ll
index 9562e55dc..53da433fb 100644
--- a/llvm/test/CodeGen/Mips/mbrsize4a.ll
+++ b/llvm/test/CodeGen/Mips/mbrsize4a.ll
@@ -30,8 +30,8 @@ declare i32 @foo(...) #1
 
 declare i32 @printf(i8*, ...) #1
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind }
 
 !1 = !{i32 68}
diff --git a/llvm/test/CodeGen/Mips/mcount.ll b/llvm/test/CodeGen/Mips/mcount.ll
new file mode 100644
index 000000000..fe8cee9d7
--- /dev/null
+++ b/llvm/test/CodeGen/Mips/mcount.ll
@@ -0,0 +1,123 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+
+; RUN: llc -march=mips -verify-machineinstrs \
+; RUN:   < %s | FileCheck %s -check-prefix=MIPS32
+; RUN: llc -march=mips -verify-machineinstrs -relocation-model=pic \
+; RUN:   < %s | FileCheck %s -check-prefix=MIPS32-PIC
+; RUN: llc -march=mips64 -verify-machineinstrs \
+; RUN:   < %s | FileCheck %s -check-prefix=MIPS64
+; RUN: llc -march=mips64 -verify-machineinstrs -relocation-model=pic \
+; RUN:   < %s | FileCheck %s -check-prefix=MIPS64-PIC
+; RUN: llc -march=mips -verify-machineinstrs -mattr=+micromips \
+; RUN:   < %s | FileCheck %s -check-prefix=MIPS32-MM
+; RUN: llc -march=mips -verify-machineinstrs -relocation-model=pic -mattr=+micromips \
+; RUN:   < %s | FileCheck %s -check-prefix=MIPS32-MM-PIC
+
+; Test that checks ABI for _mcount calls.
+
+; Function Attrs: noinline nounwind optnone
+define void @foo() #0 {
+; MIPS32-LABEL: foo:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    addiu $sp, $sp, -24
+; MIPS32-NEXT:    .cfi_def_cfa_offset 24
+; MIPS32-NEXT:    sw $ra, 20($sp) # 4-byte Folded Spill
+; MIPS32-NEXT:    .cfi_offset 31, -4
+; MIPS32-NEXT:    move $1, $ra
+; MIPS32-NEXT:    jal _mcount
+; MIPS32-NEXT:    addiu $sp, $sp, -8
+; MIPS32-NEXT:    lw $ra, 20($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    addiu $sp, $sp, 24
+;
+; MIPS32-PIC-LABEL: foo:
+; MIPS32-PIC:       # %bb.0: # %entry
+; MIPS32-PIC-NEXT:    lui $2, %hi(_gp_disp)
+; MIPS32-PIC-NEXT:    addiu $2, $2, %lo(_gp_disp)
+; MIPS32-PIC-NEXT:    addiu $sp, $sp, -24
+; MIPS32-PIC-NEXT:    .cfi_def_cfa_offset 24
+; MIPS32-PIC-NEXT:    sw $ra, 20($sp) # 4-byte Folded Spill
+; MIPS32-PIC-NEXT:    .cfi_offset 31, -4
+; MIPS32-PIC-NEXT:    addu $gp, $2, $25
+; MIPS32-PIC-NEXT:    lw $25, %call16(_mcount)($gp)
+; MIPS32-PIC-NEXT:    move $1, $ra
+; MIPS32-PIC-NEXT:    .reloc ($tmp0), R_MIPS_JALR, _mcount
+; MIPS32-PIC-NEXT:  $tmp0:
+; MIPS32-PIC-NEXT:    jalr $25
+; MIPS32-PIC-NEXT:    addiu $sp, $sp, -8
+; MIPS32-PIC-NEXT:    lw $ra, 20($sp) # 4-byte Folded Reload
+; MIPS32-PIC-NEXT:    jr $ra
+; MIPS32-PIC-NEXT:    addiu $sp, $sp, 24
+;
+; MIPS64-LABEL: foo:
+; MIPS64:       # %bb.0: # %entry
+; MIPS64-NEXT:    daddiu $sp, $sp, -16
+; MIPS64-NEXT:    .cfi_def_cfa_offset 16
+; MIPS64-NEXT:    sd $ra, 8($sp) # 8-byte Folded Spill
+; MIPS64-NEXT:    .cfi_offset 31, -8
+; MIPS64-NEXT:    move $1, $ra
+; MIPS64-NEXT:    jal _mcount
+; MIPS64-NEXT:    nop
+; MIPS64-NEXT:    ld $ra, 8($sp) # 8-byte Folded Reload
+; MIPS64-NEXT:    jr $ra
+; MIPS64-NEXT:    daddiu $sp, $sp, 16
+;
+; MIPS64-PIC-LABEL: foo:
+; MIPS64-PIC:       # %bb.0: # %entry
+; MIPS64-PIC-NEXT:    daddiu $sp, $sp, -16
+; MIPS64-PIC-NEXT:    .cfi_def_cfa_offset 16
+; MIPS64-PIC-NEXT:    sd $ra, 8($sp) # 8-byte Folded Spill
+; MIPS64-PIC-NEXT:    sd $gp, 0($sp) # 8-byte Folded Spill
+; MIPS64-PIC-NEXT:    .cfi_offset 31, -8
+; MIPS64-PIC-NEXT:    .cfi_offset 28, -16
+; MIPS64-PIC-NEXT:    lui $1, %hi(%neg(%gp_rel(foo)))
+; MIPS64-PIC-NEXT:    daddu $1, $1, $25
+; MIPS64-PIC-NEXT:    daddiu $gp, $1, %lo(%neg(%gp_rel(foo)))
+; MIPS64-PIC-NEXT:    ld $25, %call16(_mcount)($gp)
+; MIPS64-PIC-NEXT:    move $1, $ra
+; MIPS64-PIC-NEXT:    .reloc .Ltmp0, R_MIPS_JALR, _mcount
+; MIPS64-PIC-NEXT:  .Ltmp0:
+; MIPS64-PIC-NEXT:    jalr $25
+; MIPS64-PIC-NEXT:    nop
+; MIPS64-PIC-NEXT:    ld $gp, 0($sp) # 8-byte Folded Reload
+; MIPS64-PIC-NEXT:    ld $ra, 8($sp) # 8-byte Folded Reload
+; MIPS64-PIC-NEXT:    jr $ra
+; MIPS64-PIC-NEXT:    daddiu $sp, $sp, 16
+;
+; MIPS32-MM-LABEL: foo:
+; MIPS32-MM:       # %bb.0: # %entry
+; MIPS32-MM-NEXT:    addiu $sp, $sp, -24
+; MIPS32-MM-NEXT:    .cfi_def_cfa_offset 24
+; MIPS32-MM-NEXT:    sw $ra, 20($sp) # 4-byte Folded Spill
+; MIPS32-MM-NEXT:    .cfi_offset 31, -4
+; MIPS32-MM-NEXT:    move $1, $ra
+; MIPS32-MM-NEXT:    jal _mcount
+; MIPS32-MM-NEXT:    addiu $sp, $sp, -8
+; MIPS32-MM-NEXT:    lw $ra, 20($sp) # 4-byte Folded Reload
+; MIPS32-MM-NEXT:    jr $ra
+; MIPS32-MM-NEXT:    addiu $sp, $sp, 24
+;
+; MIPS32-MM-PIC-LABEL: foo:
+; MIPS32-MM-PIC:       # %bb.0: # %entry
+; MIPS32-MM-PIC-NEXT:    lui $2, %hi(_gp_disp)
+; MIPS32-MM-PIC-NEXT:    addiu $2, $2, %lo(_gp_disp)
+; MIPS32-MM-PIC-NEXT:    addiu $sp, $sp, -24
+; MIPS32-MM-PIC-NEXT:    .cfi_def_cfa_offset 24
+; MIPS32-MM-PIC-NEXT:    sw $ra, 20($sp) # 4-byte Folded Spill
+; MIPS32-MM-PIC-NEXT:    .cfi_offset 31, -4
+; MIPS32-MM-PIC-NEXT:    addu $2, $2, $25
+; MIPS32-MM-PIC-NEXT:    lw $25, %call16(_mcount)($2)
+; MIPS32-MM-PIC-NEXT:    move $gp, $2
+; MIPS32-MM-PIC-NEXT:    move $1, $ra
+; MIPS32-MM-PIC-NEXT:    .reloc ($tmp0), R_MICROMIPS_JALR, _mcount
+; MIPS32-MM-PIC-NEXT:  $tmp0:
+; MIPS32-MM-PIC-NEXT:    jalr $25
+; MIPS32-MM-PIC-NEXT:    addiu $sp, $sp, -8
+; MIPS32-MM-PIC-NEXT:    lw $ra, 20($sp) # 4-byte Folded Reload
+; MIPS32-MM-PIC-NEXT:    jr $ra
+; MIPS32-MM-PIC-NEXT:    addiu $sp, $sp, 24
+entry:
+  ret void
+}
+
+attributes #0 = { "instrument-function-entry-inlined"="_mcount" }
diff --git a/llvm/test/CodeGen/Mips/micromips-ase-function-attribute.ll b/llvm/test/CodeGen/Mips/micromips-ase-function-attribute.ll
index fe82b7c5b..cd78166d3 100644
--- a/llvm/test/CodeGen/Mips/micromips-ase-function-attribute.ll
+++ b/llvm/test/CodeGen/Mips/micromips-ase-function-attribute.ll
@@ -1,5 +1,5 @@
 ; RUN: llc -mtriple=mips-unknown-linux -filetype=obj %s -o - | \
-; RUN:   llvm-readobj --mips-abi-flags | \
+; RUN:   llvm-readobj -A | \
 ; RUN:   FileCheck --check-prefix=ASE-MICROMIPS %s
 
 define void @_Z3foov() #0 {
diff --git a/llvm/test/CodeGen/Mips/micromips-attr.ll b/llvm/test/CodeGen/Mips/micromips-attr.ll
index 78bcc04a9..1ea9b136e 100644
--- a/llvm/test/CodeGen/Mips/micromips-attr.ll
+++ b/llvm/test/CodeGen/Mips/micromips-attr.ll
@@ -21,7 +21,7 @@ entry:
 attributes #0 = {
   nounwind "micromips"
   "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false"
-  "less-precise-fpmad"="false" "no-frame-pointer-elim"="false"
+  "less-precise-fpmad"="false" "frame-pointer"="none"
   "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false"
   "no-signed-zeros-fp-math"="false" "no-trapping-math"="false"
   "stack-protector-buffer-size"="8" "unsafe-fp-math"="false"
@@ -31,7 +31,7 @@ attributes #0 = {
 attributes #1 = {
   nounwind
   "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false"
-  "less-precise-fpmad"="false" "no-frame-pointer-elim"="false"
+  "less-precise-fpmad"="false" "frame-pointer"="none"
   "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false"
   "no-signed-zeros-fp-math"="false" "no-trapping-math"="false"
   "stack-protector-buffer-size"="8" "unsafe-fp-math"="false"
diff --git a/llvm/test/CodeGen/Mips/mips16-hf-attr-2.ll b/llvm/test/CodeGen/Mips/mips16-hf-attr-2.ll
index 82c0989c4..46cba348c 100644
--- a/llvm/test/CodeGen/Mips/mips16-hf-attr-2.ll
+++ b/llvm/test/CodeGen/Mips/mips16-hf-attr-2.ll
@@ -25,21 +25,21 @@ declare float @foo(float) #2
 
 attributes #0 = {
   nounwind
-  "less-precise-fpmad"="false" "no-frame-pointer-elim"="true"
-  "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false"
+  "less-precise-fpmad"="false" "frame-pointer"="all"
+ "frame-pointer"="non-leaf" "no-infs-fp-math"="false"
   "no-nans-fp-math"="false" "stack-protector-buffer-size"="8"
   "unsafe-fp-math"="false" "use-soft-float"="false"
 }
 attributes #1 = {
   nounwind
-  "less-precise-fpmad"="false" "no-frame-pointer-elim"="true"
-  "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false"
+  "less-precise-fpmad"="false" "frame-pointer"="all"
+ "frame-pointer"="non-leaf" "no-infs-fp-math"="false"
   "no-nans-fp-math"="false" "stack-protector-buffer-size"="8"
   "unsafe-fp-math"="false" "use-soft-float"="true"
 }
 attributes #2 = {
-  "less-precise-fpmad"="false" "no-frame-pointer-elim"="true"
-  "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false"
+  "less-precise-fpmad"="false" "frame-pointer"="all"
+ "frame-pointer"="non-leaf" "no-infs-fp-math"="false"
   "no-nans-fp-math"="false" "stack-protector-buffer-size"="8"
   "unsafe-fp-math"="false" "use-soft-float"="true"
 }
diff --git a/llvm/test/CodeGen/Mips/mips16-hf-attr.ll b/llvm/test/CodeGen/Mips/mips16-hf-attr.ll
index bcae1e92f..dabe61842 100644
--- a/llvm/test/CodeGen/Mips/mips16-hf-attr.ll
+++ b/llvm/test/CodeGen/Mips/mips16-hf-attr.ll
@@ -25,21 +25,21 @@ declare float @foo(float) #2
 
 attributes #0 = {
   nounwind
-  "less-precise-fpmad"="false" "no-frame-pointer-elim"="true"
-  "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false"
+  "less-precise-fpmad"="false" "frame-pointer"="all"
+ "frame-pointer"="non-leaf" "no-infs-fp-math"="false"
   "no-nans-fp-math"="false" "stack-protector-buffer-size"="8"
   "unsafe-fp-math"="false" "use-soft-float"="false"
 }
 attributes #1 = {
   nounwind
-  "less-precise-fpmad"="false" "no-frame-pointer-elim"="true"
-  "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false"
+  "less-precise-fpmad"="false" "frame-pointer"="all"
+ "frame-pointer"="non-leaf" "no-infs-fp-math"="false"
   "no-nans-fp-math"="false" "stack-protector-buffer-size"="8"
   "unsafe-fp-math"="false" "use-soft-float"="true"
 }
 attributes #2 = {
-  "less-precise-fpmad"="false" "no-frame-pointer-elim"="true"
-  "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false"
+  "less-precise-fpmad"="false" "frame-pointer"="all"
+ "frame-pointer"="non-leaf" "no-infs-fp-math"="false"
   "no-nans-fp-math"="false" "stack-protector-buffer-size"="8"
   "unsafe-fp-math"="false" "use-soft-float"="true"
 }
diff --git a/llvm/test/CodeGen/Mips/mips16_32_1.ll b/llvm/test/CodeGen/Mips/mips16_32_1.ll
index 211aa2a0f..bf95c06a4 100644
--- a/llvm/test/CodeGen/Mips/mips16_32_1.ll
+++ b/llvm/test/CodeGen/Mips/mips16_32_1.ll
@@ -10,4 +10,4 @@ entry:
 ; CHECK:	.ent	foo
 ; CHECK:	jrc $ra
 ; CHECK:	.end	foo
-attributes #0 = { nounwind "less-precise-fpmad"="false" "mips16" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "mips16" "frame-pointer"="non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/Mips/mips16_32_10.ll b/llvm/test/CodeGen/Mips/mips16_32_10.ll
index b256912d7..83e82f6c6 100644
--- a/llvm/test/CodeGen/Mips/mips16_32_10.ll
+++ b/llvm/test/CodeGen/Mips/mips16_32_10.ll
@@ -53,6 +53,6 @@ entry:
 
 
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "nomips16" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false"  "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #2 = { nounwind "less-precise-fpmad"="false" "nomips16" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "nomips16" "frame-pointer"="non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false"  "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { nounwind "less-precise-fpmad"="false" "nomips16" "frame-pointer"="non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/Mips/mips16_32_3.ll b/llvm/test/CodeGen/Mips/mips16_32_3.ll
index 8891eba8c..a28af8b5e 100644
--- a/llvm/test/CodeGen/Mips/mips16_32_3.ll
+++ b/llvm/test/CodeGen/Mips/mips16_32_3.ll
@@ -62,6 +62,6 @@ entry:
 ; 32:	.set	reorder
 ; 32:	.end	main
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "mips16" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "nomips16" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #2 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "mips16" "frame-pointer"="non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "nomips16" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/Mips/mips16_32_4.ll b/llvm/test/CodeGen/Mips/mips16_32_4.ll
index 2d50881c6..118d1e390 100644
--- a/llvm/test/CodeGen/Mips/mips16_32_4.ll
+++ b/llvm/test/CodeGen/Mips/mips16_32_4.ll
@@ -56,6 +56,6 @@ entry:
 ; 32:	.end	main
 
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "mips16" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "nomips16" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #2 = { nounwind "less-precise-fpmad"="false" "mips16" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "mips16" "frame-pointer"="non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "nomips16" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { nounwind "less-precise-fpmad"="false" "mips16" "frame-pointer"="non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/Mips/mips16_32_5.ll b/llvm/test/CodeGen/Mips/mips16_32_5.ll
index 644ba4c98..55175534d 100644
--- a/llvm/test/CodeGen/Mips/mips16_32_5.ll
+++ b/llvm/test/CodeGen/Mips/mips16_32_5.ll
@@ -73,6 +73,6 @@ entry:
 
 
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "mips16" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "nomips16" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #2 = { nounwind "less-precise-fpmad"="false" "nomips16" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "mips16" "frame-pointer"="non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "nomips16" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { nounwind "less-precise-fpmad"="false" "nomips16" "frame-pointer"="non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/Mips/mips16_32_6.ll b/llvm/test/CodeGen/Mips/mips16_32_6.ll
index 1afce84b0..33a2bf27b 100644
--- a/llvm/test/CodeGen/Mips/mips16_32_6.ll
+++ b/llvm/test/CodeGen/Mips/mips16_32_6.ll
@@ -80,6 +80,6 @@ entry:
 
 
 
-attributes #0 = { nounwind "less-precise-fpmad"="false"  "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "nomips16" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #2 = { nounwind "less-precise-fpmad"="false" "nomips16" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "nomips16" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { nounwind "less-precise-fpmad"="false" "nomips16" "frame-pointer"="non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/Mips/mips16_32_7.ll b/llvm/test/CodeGen/Mips/mips16_32_7.ll
index cc247c054..680420b62 100644
--- a/llvm/test/CodeGen/Mips/mips16_32_7.ll
+++ b/llvm/test/CodeGen/Mips/mips16_32_7.ll
@@ -68,6 +68,6 @@ entry:
 
 
 
-attributes #0 = { nounwind "less-precise-fpmad"="false"  "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "nomips16" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #2 = { nounwind "less-precise-fpmad"="false" "mips16" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "nomips16" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { nounwind "less-precise-fpmad"="false" "mips16" "frame-pointer"="non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/Mips/mips16_32_8.ll b/llvm/test/CodeGen/Mips/mips16_32_8.ll
index 5e03928a1..0cde5f128 100644
--- a/llvm/test/CodeGen/Mips/mips16_32_8.ll
+++ b/llvm/test/CodeGen/Mips/mips16_32_8.ll
@@ -67,7 +67,7 @@ entry:
 ; 32:	.set	reorder
 ; 32:	.end	main
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "mips16" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "nomips16" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #2 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #3 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "mips16" "frame-pointer"="non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "nomips16" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { "less-precise-fpmad"="false" "frame-pointer"="non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/Mips/mips16_32_9.ll b/llvm/test/CodeGen/Mips/mips16_32_9.ll
index 8543147be..e2d329a0f 100644
--- a/llvm/test/CodeGen/Mips/mips16_32_9.ll
+++ b/llvm/test/CodeGen/Mips/mips16_32_9.ll
@@ -44,6 +44,6 @@ entry:
 
 
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "mips16" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false"  "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #2 = { nounwind "less-precise-fpmad"="false" "mips16" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "mips16" "frame-pointer"="non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false"  "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { nounwind "less-precise-fpmad"="false" "mips16" "frame-pointer"="non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/Mips/msa/2r_vector_scalar.ll b/llvm/test/CodeGen/Mips/msa/2r_vector_scalar.ll
index 065b72891..549363310 100644
--- a/llvm/test/CodeGen/Mips/msa/2r_vector_scalar.ll
+++ b/llvm/test/CodeGen/Mips/msa/2r_vector_scalar.ll
@@ -86,9 +86,8 @@ declare <2 x i64> @llvm.mips.fill.d(i64) nounwind
 ; MIPS-ANY: llvm_mips_fill_d_test:
 ; MIPS32-DAG: lw [[R1:\$[0-9]+]], 0(
 ; MIPS32-DAG: lw [[R2:\$[0-9]+]], 4(
-; MIPS64-DAG: ld [[R1:\$[0-9]+]], %got_disp(llvm_mips_fill_d_ARG1)
-; MIPS32-DAG: ldi.b [[R3:\$w[0-9]+]], 0
-; MIPS32-DAG: insert.w [[R3]][0], [[R1]]
+; MIPS64-DAG: ld [[R1]], %got_disp(llvm_mips_fill_d_ARG1)
+; MIPS32-DAG: insert.w [[R3:\$w[0-9]+]][0], [[R1]]
 ; MIPS32-DAG: insert.w [[R3]][1], [[R2]]
 ; MIPS32-DAG: insert.w [[R3]][2], [[R1]]
 ; MIPS32-DAG: insert.w [[R3]][3], [[R2]]
diff --git a/llvm/test/CodeGen/Mips/msa/arithmetic.ll b/llvm/test/CodeGen/Mips/msa/arithmetic.ll
index 413f04566..0caafb6cc 100644
--- a/llvm/test/CodeGen/Mips/msa/arithmetic.ll
+++ b/llvm/test/CodeGen/Mips/msa/arithmetic.ll
@@ -190,6 +190,20 @@ define void @sub_v16i8_i(<16 x i8>* %c, <16 x i8>* %a) nounwind {
   ret void
 }
 
+define void @sub_v16i8_i_negated(<16 x i8>* %c, <16 x i8>* %a) nounwind {
+; ALL-LABEL: sub_v16i8_i_negated:
+; ALL:       # %bb.0:
+; ALL-NEXT:    ld.b $w0, 0($5)
+; ALL-NEXT:    subvi.b $w0, $w0, 1
+; ALL-NEXT:    jr $ra
+; ALL-NEXT:    st.b $w0, 0($4)
+  %1 = load <16 x i8>, <16 x i8>* %a
+  %2 = add <16 x i8> %1, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1,
+              i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
+  store <16 x i8> %2, <16 x i8>* %c
+  ret void
+}
+
 define void @sub_v8i16_i(<8 x i16>* %c, <8 x i16>* %a) nounwind {
 ; ALL-LABEL: sub_v8i16_i:
 ; ALL:       # %bb.0:
@@ -204,6 +218,20 @@ define void @sub_v8i16_i(<8 x i16>* %c, <8 x i16>* %a) nounwind {
   ret void
 }
 
+define void @sub_v8i16_i_negated(<8 x i16>* %c, <8 x i16>* %a) nounwind {
+; ALL-LABEL: sub_v8i16_i_negated:
+; ALL:       # %bb.0:
+; ALL-NEXT:    ld.h $w0, 0($5)
+; ALL-NEXT:    subvi.h $w0, $w0, 1
+; ALL-NEXT:    jr $ra
+; ALL-NEXT:    st.h $w0, 0($4)
+  %1 = load <8 x i16>, <8 x i16>* %a
+  %2 = add <8 x i16> %1, <i16 -1, i16 -1, i16 -1, i16 -1,
+              i16 -1, i16 -1, i16 -1, i16 -1>
+  store <8 x i16> %2, <8 x i16>* %c
+  ret void
+}
+
 define void @sub_v4i32_i(<4 x i32>* %c, <4 x i32>* %a) nounwind {
 ; ALL-LABEL: sub_v4i32_i:
 ; ALL:       # %bb.0:
@@ -217,6 +245,19 @@ define void @sub_v4i32_i(<4 x i32>* %c, <4 x i32>* %a) nounwind {
   ret void
 }
 
+define void @sub_v4i32_i_negated(<4 x i32>* %c, <4 x i32>* %a) nounwind {
+; ALL-LABEL: sub_v4i32_i_negated:
+; ALL:       # %bb.0:
+; ALL-NEXT:    ld.w $w0, 0($5)
+; ALL-NEXT:    subvi.w $w0, $w0, 1
+; ALL-NEXT:    jr $ra
+; ALL-NEXT:    st.w $w0, 0($4)
+  %1 = load <4 x i32>, <4 x i32>* %a
+  %2 = add <4 x i32> %1, <i32 -1, i32 -1, i32 -1, i32 -1>
+  store <4 x i32> %2, <4 x i32>* %c
+  ret void
+}
+
 define void @sub_v2i64_i(<2 x i64>* %c, <2 x i64>* %a) nounwind {
 ; ALL-LABEL: sub_v2i64_i:
 ; ALL:       # %bb.0:
@@ -230,6 +271,29 @@ define void @sub_v2i64_i(<2 x i64>* %c, <2 x i64>* %a) nounwind {
   ret void
 }
 
+define void @sub_v2i64_i_negated(<2 x i64>* %c, <2 x i64>* %a) nounwind {
+; MIPS-LABEL: sub_v2i64_i_negated:
+; MIPS:       # %bb.0:
+; MIPS-NEXT:    ldi.b $w0, -1
+; MIPS-NEXT:    shf.w $w0, $w0, 177
+; MIPS-NEXT:    ld.d $w1, 0($5)
+; MIPS-NEXT:    addv.d $w0, $w1, $w0
+; MIPS-NEXT:    jr $ra
+; MIPS-NEXT:    st.d $w0, 0($4)
+;
+; MIPSEL-LABEL: sub_v2i64_i_negated:
+; MIPSEL:       # %bb.0:
+; MIPSEL-NEXT:    ldi.b $w0, -1
+; MIPSEL-NEXT:    ld.d $w1, 0($5)
+; MIPSEL-NEXT:    addv.d $w0, $w1, $w0
+; MIPSEL-NEXT:    jr $ra
+; MIPSEL-NEXT:    st.d $w0, 0($4)
+  %1 = load <2 x i64>, <2 x i64>* %a
+  %2 = add <2 x i64> %1, <i64 -1, i64 -1>
+  store <2 x i64> %2, <2 x i64>* %c
+  ret void
+}
+
 define void @mul_v16i8(<16 x i8>* %c, <16 x i8>* %a, <16 x i8>* %b) nounwind {
 ; ALL-LABEL: mul_v16i8:
 ; ALL:       # %bb.0:
diff --git a/llvm/test/CodeGen/Mips/msa/nori.b.ll b/llvm/test/CodeGen/Mips/msa/nori.b.ll
new file mode 100644
index 000000000..ea0d2a958
--- /dev/null
+++ b/llvm/test/CodeGen/Mips/msa/nori.b.ll
@@ -0,0 +1,26 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -march=mips64el -mcpu=mips64r6 -mattr=+msa,+fp64 < %s | FileCheck %s
+
+; Test that simply checks if it will finish when value 255 (-1) appears as
+; immediate in 'nori.b' instruction.
+
+; mips.nori.b %dst, %a, imm
+; mips.nori.b %dst, %a, -1
+; %dst = not (or %a, -1)
+; %dst = xor (or %a, -1), -1
+; %dst = xor -1, -1
+; %dst = 0
+
+define <16 x i8> @foo(<16 x i8> %a) {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0: # %bb2
+; CHECK-NEXT:    ldi.b $w0, 0
+; CHECK-NEXT:    copy_s.d $2, $w0[0]
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    copy_s.d $3, $w0[1]
+bb2:
+  %0 = tail call <16 x i8> @llvm.mips.nori.b(<16 x i8> %a, i32 255)
+  ret <16 x i8> %0
+}
+
+declare <16 x i8> @llvm.mips.nori.b(<16 x i8>, i32)
diff --git a/llvm/test/CodeGen/Mips/msa/remat-ldi.ll b/llvm/test/CodeGen/Mips/msa/remat-ldi.ll
new file mode 100644
index 000000000..64f976c97
--- /dev/null
+++ b/llvm/test/CodeGen/Mips/msa/remat-ldi.ll
@@ -0,0 +1,107 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -O3 -march=mipsel -mcpu=mips32r6 -mattr=+fp64,+msa %s -o - | FileCheck %s
+
+; Test that checks if spill for ldi can be avoided and instruction will be
+; rematerialized.
+
+declare dso_local void @foo()
+
+define dso_local void @test_ldi_b() {
+; CHECK-LABEL: test_ldi_b:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addiu $sp, $sp, -24
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    sw $ra, 20($sp) # 4-byte Folded Spill
+; CHECK-NEXT:    .cfi_offset 31, -4
+; CHECK-NEXT:    ldi.b $w0, 1
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    jal foo
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    ldi.b $w0, 1
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    lw $ra, 20($sp) # 4-byte Folded Reload
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    addiu $sp, $sp, 24
+entry:
+  tail call void asm sideeffect "", "f,~{memory},~{$1}"(<16 x i8> <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>)
+  tail call void @foo()
+  tail call void asm sideeffect "", "f,~{memory},~{$1}"(<16 x i8> <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>)
+  ret void
+}
+
+define dso_local void @test_ldi_h() {
+; CHECK-LABEL: test_ldi_h:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addiu $sp, $sp, -24
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    sw $ra, 20($sp) # 4-byte Folded Spill
+; CHECK-NEXT:    .cfi_offset 31, -4
+; CHECK-NEXT:    ldi.h $w0, 2
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    jal foo
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    ldi.h $w0, 2
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    lw $ra, 20($sp) # 4-byte Folded Reload
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    addiu $sp, $sp, 24
+entry:
+  tail call void asm sideeffect "", "f,~{memory},~{$1}"(<8 x i16> <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>)
+  tail call void @foo()
+  tail call void asm sideeffect "", "f,~{memory},~{$1}"(<8 x i16> <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>)
+  ret void
+}
+
+define dso_local void @test_ldi_w() {
+; CHECK-LABEL: test_ldi_w:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addiu $sp, $sp, -24
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    sw $ra, 20($sp) # 4-byte Folded Spill
+; CHECK-NEXT:    .cfi_offset 31, -4
+; CHECK-NEXT:    ldi.w $w0, 3
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    jal foo
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    ldi.w $w0, 3
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    lw $ra, 20($sp) # 4-byte Folded Reload
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    addiu $sp, $sp, 24
+entry:
+  tail call void asm sideeffect "", "f,~{memory},~{$1}"(<4 x i32> <i32 3, i32 3, i32 3, i32 3>)
+  tail call void @foo()
+  tail call void asm sideeffect "", "f,~{memory},~{$1}"(<4 x i32> <i32 3, i32 3, i32 3, i32 3>)
+  ret void
+}
+
+define dso_local void @test_ldi_d() {
+; CHECK-LABEL: test_ldi_d:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addiu $sp, $sp, -24
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    sw $ra, 20($sp) # 4-byte Folded Spill
+; CHECK-NEXT:    .cfi_offset 31, -4
+; CHECK-NEXT:    ldi.d $w0, 4
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    jal foo
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    ldi.d $w0, 4
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    lw $ra, 20($sp) # 4-byte Folded Reload
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    addiu $sp, $sp, 24
+entry:
+  tail call void asm sideeffect "", "f,~{memory},~{$1}"(<2 x i64> <i64 4, i64 4>)
+  tail call void @foo()
+  tail call void asm sideeffect "", "f,~{memory},~{$1}"(<2 x i64> <i64 4, i64 4>)
+  ret void
+}
diff --git a/llvm/test/CodeGen/Mips/no-frame-pointer-elim.ll b/llvm/test/CodeGen/Mips/no-frame-pointer-elim.ll
new file mode 100644
index 000000000..5c5752e72
--- /dev/null
+++ b/llvm/test/CodeGen/Mips/no-frame-pointer-elim.ll
@@ -0,0 +1,37 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -march=mips64 -relocation-model=static < %s \
+; RUN:   | FileCheck %s --check-prefix STATIC
+; RUN: llc -march=mips64 -relocation-model=pic < %s \
+; RUN:   | FileCheck %s --check-prefix PIC
+
+declare dso_local void @callee() noreturn nounwind
+
+define dso_local void @caller() nounwind "frame-pointer"="non-leaf" {
+; STATIC-LABEL: caller:
+; STATIC:       # %bb.0: # %entry
+; STATIC-NEXT:    daddiu $sp, $sp, -16
+; STATIC-NEXT:    sd $ra, 8($sp) # 8-byte Folded Spill
+; STATIC-NEXT:    sd $fp, 0($sp) # 8-byte Folded Spill
+; STATIC-NEXT:    move $fp, $sp
+; STATIC-NEXT:    jal callee
+; STATIC-NEXT:    nop
+;
+; PIC-LABEL: caller:
+; PIC:       # %bb.0: # %entry
+; PIC-NEXT:    daddiu $sp, $sp, -32
+; PIC-NEXT:    sd $ra, 24($sp) # 8-byte Folded Spill
+; PIC-NEXT:    sd $fp, 16($sp) # 8-byte Folded Spill
+; PIC-NEXT:    sd $gp, 8($sp) # 8-byte Folded Spill
+; PIC-NEXT:    move $fp, $sp
+; PIC-NEXT:    lui $1, %hi(%neg(%gp_rel(caller)))
+; PIC-NEXT:    daddu $1, $1, $25
+; PIC-NEXT:    daddiu $gp, $1, %lo(%neg(%gp_rel(caller)))
+; PIC-NEXT:    ld $25, %call16(callee)($gp)
+; PIC-NEXT:    .reloc .Ltmp0, R_MIPS_JALR, callee
+; PIC-NEXT:  .Ltmp0:
+; PIC-NEXT:    jalr $25
+; PIC-NEXT:    nop
+entry:
+  tail call void @callee()
+  unreachable
+}
diff --git a/llvm/test/CodeGen/Mips/nomips16.ll b/llvm/test/CodeGen/Mips/nomips16.ll
index 749272181..de668d663 100644
--- a/llvm/test/CodeGen/Mips/nomips16.ll
+++ b/llvm/test/CodeGen/Mips/nomips16.ll
@@ -33,6 +33,6 @@ entry:
 ; CHECK: 	.end	nofoo
 
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "mips16" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "nomips16" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "mips16" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "nomips16" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
diff --git a/llvm/test/CodeGen/Mips/pbqp-reserved-physreg.ll b/llvm/test/CodeGen/Mips/pbqp-reserved-physreg.ll
index eedc51bd1..730e01d8f 100644
--- a/llvm/test/CodeGen/Mips/pbqp-reserved-physreg.ll
+++ b/llvm/test/CodeGen/Mips/pbqp-reserved-physreg.ll
@@ -31,5 +31,5 @@ bb35:                                             ; preds = %bb
   unreachable
 }
 
-attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="generic" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="generic" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
diff --git a/llvm/test/CodeGen/Mips/powif64_16.ll b/llvm/test/CodeGen/Mips/powif64_16.ll
index 9be1a368b..d6dbc52d9 100644
--- a/llvm/test/CodeGen/Mips/powif64_16.ll
+++ b/llvm/test/CodeGen/Mips/powif64_16.ll
@@ -17,7 +17,7 @@ define double @foo_pow_f64(double %y, i32 %p)  {
   ret double %1
 } 
 
-attributes #0 = { nounwind optsize "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf"="true" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="true" }
+attributes #0 = { nounwind optsize "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="true" }
 attributes #1 = { nounwind readonly }
 
 !0 = !{!"double", !1}
diff --git a/llvm/test/CodeGen/Mips/pr42736.ll b/llvm/test/CodeGen/Mips/pr42736.ll
new file mode 100644
index 000000000..c0dbadb50
--- /dev/null
+++ b/llvm/test/CodeGen/Mips/pr42736.ll
@@ -0,0 +1,28 @@
+; RUN: llc -mtriple=mips64-linux-gnuabi64 \
+; RUN:     -relocation-model=pic < %s | FileCheck %s -check-prefix=PIC
+; RUN: llc -mtriple=mips64-linux-gnuabi64 \
+; RUN:     -relocation-model=static < %s | FileCheck %s -check-prefix=STATIC
+
+define void @bar1() nounwind {
+entry:
+; PIC:      lui  $[[R0:[0-9]+]], 4095
+; PIC-NEXT: ori  $[[R0]], $[[R0]], 65535
+; PIC-NEXT: ld   $[[R1:[0-9]+]], %got_disp(foo)(${{[0-9]+}})
+; PIC-NEXT: and  $[[R1]], $[[R1]], $[[R0]]
+; PIC-NEXT: sd   $[[R1]]
+
+; STATIC:      lui     $[[R0:[0-9]+]], 4095
+; STATIC-NEXT: ori     $[[R0]], $[[R0]], 65535
+; STATIC-NEXT: daddiu  $[[R1:[0-9]+]], $zero, %hi(foo)
+; STATIC-NEXT: dsll    $[[R1]], $[[R1]], 16
+; STATIC-NEXT: daddiu  $[[R1]], $[[R1]], %lo(foo)
+; STATIC-NEXT: and     $[[R0]], $[[R1]], $[[R0]]
+; STATIC-NEXT: sd      $[[R0]]
+
+  %val = alloca i64, align 8
+  store i64 and (i64 ptrtoint (void ()* @foo to i64), i64 268435455), i64* %val, align 8
+  %0 = load i64, i64* %val, align 8
+  ret void
+}
+
+declare void @foo()
diff --git a/llvm/test/CodeGen/Mips/reloc-jalr.ll b/llvm/test/CodeGen/Mips/reloc-jalr.ll
index f8fd90311..ef360266a 100644
--- a/llvm/test/CodeGen/Mips/reloc-jalr.ll
+++ b/llvm/test/CodeGen/Mips/reloc-jalr.ll
@@ -1,64 +1,64 @@
 ; RUN: llc -mtriple=mips-linux-gnu -relocation-model=pic -mips-tail-calls=1 \
 ; RUN:     -O2 < %s | \
-; RUN:     FileCheck %s -check-prefixes=ALL,JALR-32R2,TAILCALL-32R2
+; RUN:     FileCheck %s -check-prefixes=ALL,JALR-ALL,JALR-32,JALR-32R2,TAILCALL-32R2
 
 ; RUN: llc -mtriple=mips64-linux-gnu -relocation-model=pic -mips-tail-calls=1 \
 ; RUN:     -O2 < %s | \
-; RUN:     FileCheck %s -check-prefixes=ALL,JALR-64R2,TAILCALL-64R2
+; RUN:     FileCheck %s -check-prefixes=ALL,JALR-ALL,JALR-64,JALR-64R2,TAILCALL-64R2
 
 ; RUN: llc -mtriple=mips-linux-gnu -relocation-model=pic -mips-tail-calls=1 \
 ; RUN:     -O2 -mcpu=mips32r6 -mips-compact-branches=always < %s | \
-; RUN:     FileCheck %s -check-prefixes=ALL,JALR-32R6,TAILCALL-32R6
+; RUN:     FileCheck %s -check-prefixes=ALL,JALR-ALL,JALR-32,JALR-32R6,TAILCALL-32R6
 
 ; RUN: llc -mtriple=mips64-linux-gnu -relocation-model=pic -mips-tail-calls=1 \
 ; RUN:     -O2 -mcpu=mips64r6 -mips-compact-branches=always < %s | \
-; RUN:     FileCheck %s -check-prefixes=ALL,JALR-64R6,TAILCALL-64R6
+; RUN:     FileCheck %s -check-prefixes=ALL,JALR-ALL,JALR-64,JALR-64R6,TAILCALL-64R6
 
 ; RUN: llc -mtriple=mips-linux-gnu -relocation-model=pic -mips-tail-calls=1 \
 ; RUN:     -O2 -mcpu=mips32r6 -mips-compact-branches=never < %s | \
-; RUN:     FileCheck %s -check-prefixes=ALL,JALR-32R2,TAILCALL-32R2
+; RUN:     FileCheck %s -check-prefixes=ALL,JALR-ALL,JALR-32,JALR-32R2,TAILCALL-32R2
 
 ; RUN: llc -mtriple=mips64-linux-gnu -relocation-model=pic -mips-tail-calls=1 \
 ; RUN:     -O2 -mcpu=mips64r6 -mips-compact-branches=never < %s | \
-; RUN:     FileCheck %s -check-prefixes=ALL,JALR-64R2,TAILCALL-64R2
+; RUN:     FileCheck %s -check-prefixes=ALL,JALR-ALL,JALR-64,JALR-64R2,TAILCALL-64R2
 
 ; RUN: llc -mtriple=mips-linux-gnu -relocation-model=pic -mips-tail-calls=1 \
 ; RUN:     -O2 -mattr=+micromips -mcpu=mips32r2 < %s | \
-; RUN:     FileCheck %s -check-prefixes=ALL,JALR-MM,TAILCALL-MM
+; RUN:     FileCheck %s -check-prefixes=ALL,JALR-ALL,JALR-MM,TAILCALL-MM
 
 ; RUN: llc -mtriple=mips-linux-gnu -relocation-model=pic -mips-tail-calls=1 \
 ; RUN:     -O2 -mattr=+micromips -mcpu=mips32r6 < %s | \
-; RUN:     FileCheck %s -check-prefixes=ALL,JALR-MM
+; RUN:     FileCheck %s -check-prefixes=ALL,JALR-ALL,JALR-MM,TAILCALL-MM
 
 ; RUN: llc -mtriple=mips-linux-gnu -relocation-model=pic \
-; RUN:     -O0 < %s | FileCheck %s -check-prefixes=ALL,JALR-32R2
+; RUN:     -O0 < %s | FileCheck %s -check-prefixes=ALL,JALR-ALL,JALR-32,JALR-32R2,PIC-NOTAILCALL-R2
 
 ; RUN: llc -mtriple=mips64-linux-gnu -relocation-model=pic \
-; RUN:     -O0 < %s | FileCheck %s -check-prefixes=ALL,JALR-64R2
+; RUN:     -O0 < %s | FileCheck %s -check-prefixes=ALL,JALR-ALL,JALR-64,JALR-64R2,PIC-NOTAILCALL-R2
 
 ; RUN: llc -mtriple=mips-linux-gnu -relocation-model=pic \
 ; RUN:     -O0 -mcpu=mips32r6 -mips-compact-branches=always < %s | \
-; RUN:     FileCheck %s -check-prefixes=ALL,JALR-32R6
+; RUN:     FileCheck %s -check-prefixes=ALL,JALR-ALL,JALR-32,JALR-32R6,PIC-NOTAILCALL-R6
 
 ; RUN: llc -mtriple=mips64-linux-gnu -relocation-model=pic \
 ; RUN:     -O0 -mcpu=mips64r6 -mips-compact-branches=always < %s | \
-; RUN:     FileCheck %s -check-prefixes=ALL,JALR-64R6
+; RUN:     FileCheck %s -check-prefixes=ALL,JALR-ALL,JALR-64,JALR-64R6,PIC-NOTAILCALL-R6
 
 ; RUN: llc -mtriple=mips-linux-gnu -relocation-model=pic \
 ; RUN:     -O0 -mcpu=mips32r6 -mips-compact-branches=never < %s | \
-; RUN:     FileCheck %s -check-prefixes=ALL,JALR-32R2
+; RUN:     FileCheck %s -check-prefixes=ALL,JALR-ALL,JALR-32,JALR-32R2,PIC-NOTAILCALL-R2
 
 ; RUN: llc -mtriple=mips64-linux-gnu -relocation-model=pic \
 ; RUN:     -O0 -mcpu=mips64r6 -mips-compact-branches=never < %s | \
-; RUN:     FileCheck %s -check-prefixes=ALL,JALR-64R2
+; RUN:     FileCheck %s -check-prefixes=ALL,JALR-ALL,JALR-64,JALR-64R2,PIC-NOTAILCALL-R2
 
 ; RUN: llc -mtriple=mips-linux-gnu -relocation-model=pic \
 ; RUN:     -O0 -mattr=+micromips -mcpu=mips32r2 < %s | \
-; RUN:     FileCheck %s -check-prefixes=ALL,JALR-MM
+; RUN:     FileCheck %s -check-prefixes=ALL,JALR-ALL,JALR-MM,PIC-NOTAILCALL-MM
 
 ; RUN: llc -mtriple=mips-linux-gnu -relocation-model=pic \
 ; RUN:     -O0 -mattr=+micromips -mcpu=mips32r6 < %s | \
-; RUN:     FileCheck %s -check-prefixes=ALL,JALR-MM
+; RUN:     FileCheck %s -check-prefixes=ALL,JALR-ALL,JALR-MM,PIC-NOTAILCALL-MM
 
 ; RUN: llc -mtriple=mips-linux-gnu -relocation-model=pic -mips-tail-calls=1 \
 ; RUN:     -O2 -mips-jalr-reloc=false < %s | \
@@ -100,55 +100,92 @@ entry:
 define void @checkCall() {
 entry:
 ; ALL-LABEL: checkCall:
+; ALL-NOT: MIPS_JALR
   call void @foo()
-;	JALR-32R2: 	.reloc ([[TMPLABEL:.*]]), R_MIPS_JALR, foo
-; JALR-32R2-NEXT: [[TMPLABEL]]:
-;	JALR-32R2-NEXT: 	jalr	$25
-
-;	JALR-64R2: 	.reloc [[TMPLABEL:.*]], R_MIPS_JALR, foo
-; JALR-64R2-NEXT: [[TMPLABEL]]:
-;	JALR-64R2-NEXT: 	jalr	$25
-
-;	JALR-MM: 	.reloc ([[TMPLABEL:.*]]), R_MICROMIPS_JALR, foo
-; JALR-MM-NEXT: [[TMPLABEL]]:
-;	JALR-MM-NEXT: 	jalr	$25
-
-;	JALR-32R6: 	.reloc ([[TMPLABEL:.*]]), R_MIPS_JALR, foo
-; JALR-32R6-NEXT: [[TMPLABEL]]:
-;	JALR-32R6-NEXT: 	jalrc	$25
-
-;	JALR-64R6: 	.reloc [[TMPLABEL:.*]], R_MIPS_JALR, foo
-; JALR-64R6-NEXT: [[TMPLABEL]]:
-;	JALR-64R6-NEXT: 	jalrc	$25
-
-; NORELOC-NOT: R_MIPS_JALR
+; JALR-32:       .reloc ([[TMPLABEL:\$.+]]), R_MIPS_JALR, foo
+; JALR-64:       .reloc [[TMPLABEL:\..+]], R_MIPS_JALR, foo
+; JALR-MM:       .reloc ([[TMPLABEL:\$.+]]), R_MICROMIPS_JALR, foo
+; JALR-ALL-NEXT: [[TMPLABEL]]:
+; JALR-32R2-NEXT: 	jalr	$25
+; JALR-64R2-NEXT: 	jalr	$25
+; JALR-32R6-NEXT: 	jalrc	$25
+; JALR-64R6-NEXT: 	jalrc	$25
+; JALR-MM-NEXT: 	jalr	$25
+; ALL-NOT: MIPS_JALR
  ret void
 }
 
 define void @checkTailCall() {
 entry:
 ; ALL-LABEL: checkTailCall:
+; ALL-NOT: MIPS_JALR
   tail call void @foo()
-;	TAILCALL-32R2: 	.reloc ([[TMPLABEL:.*]]), R_MIPS_JALR, foo
-; TAILCALL-32R2-NEXT: [[TMPLABEL]]:
-;	TAILCALL-32R2-NEXT: 	jr	$25
+; JALR-32:       .reloc ([[TMPLABEL:\$.+]]), R_MIPS_JALR, foo
+; JALR-64:       .reloc [[TMPLABEL:\..+]], R_MIPS_JALR, foo
+; JALR-MM:       .reloc ([[TMPLABEL:\$.+]]), R_MICROMIPS_JALR, foo
+; JALR-ALL-NEXT: [[TMPLABEL]]:
+; TAILCALL-32R2-NEXT: 	jr	$25
+; TAILCALL-64R2-NEXT: 	jr	$25
+; TAILCALL-MM-NEXT: 	jrc	$25
+; TAILCALL-32R6-NEXT: 	jrc	$25
+; TAILCALL-64R6-NEXT: 	jrc	$25
+; PIC-NOTAILCALL-R2-NEXT: 	jalr	$25
+; PIC-NOTAILCALL-R6-NEXT: 	jalrc	$25
+; PIC-NOTAILCALL-MM-NEXT: 	jalr	$25
+; ALL-NOT: MIPS_JALR
+  ret void
+}
 
-;	TAILCALL-64R2: 	.reloc [[TMPLABEL:.*]], R_MIPS_JALR, foo
-; TAILCALL-64R2-NEXT: [[TMPLABEL]]:
-;	TAILCALL-64R2-NEXT: 	jr	$25
+; Check that we don't emit R_MIPS_JALR relocations against function pointers.
+; This resulted in run-time crashes until lld was modified to ignore
+; R_MIPS_JALR relocations against data symbols (commit 5bab291b7b).
+; However, the better approach is to not emit these relocations in the first
+; place so check that we no longer emit them.
+; Previously we were adding them for local dynamic TLS function pointers and
+; function pointers with internal linkage.
 
-;	TAILCALL-MM: 	.reloc ([[TMPLABEL:.*]]), R_MICROMIPS_JALR, foo
-; TAILCALL-MM-NEXT: [[TMPLABEL]]:
-;	TAILCALL-MM-NEXT: 	jrc	$25
+@fnptr_internal = internal global void()* @checkFunctionPointerCall
+@fnptr_internal_const = internal constant void()* @checkFunctionPointerCall
+@fnptr_const = constant void()* @checkFunctionPointerCall
+@fnptr_global = global void()* @checkFunctionPointerCall
 
-;	TAILCALL-32R6: 	.reloc ([[TMPLABEL:.*]]), R_MIPS_JALR, foo
-; TAILCALL-32R6-NEXT: [[TMPLABEL]]:
-;	TAILCALL-32R6-NEXT: 	jrc	$25
+define void @checkFunctionPointerCall() {
+entry:
+; ALL-LABEL: checkFunctionPointerCall:
+; ALL-NOT: MIPS_JALR
+  %func_internal = load void()*, void()** @fnptr_internal
+  call void %func_internal()
+  %func_internal_const = load void()*, void()** @fnptr_internal_const
+  call void %func_internal_const()
+  %func_const = load void()*, void()** @fnptr_const
+  call void %func_const()
+  %func_global = load void()*, void()** @fnptr_global
+  call void %func_global()
+  ret void
+}
 
-;	TAILCALL-64R6: 	.reloc [[TMPLABEL:.*]], R_MIPS_JALR, foo
-; TAILCALL-64R6-NEXT: [[TMPLABEL]]:
-;	TAILCALL-64R6-NEXT: 	jrc	$25
+@tls_fnptr_gd = thread_local global void()* @checkTlsFunctionPointerCall
+@tls_fnptr_ld = thread_local(localdynamic) global void()* @checkTlsFunctionPointerCall
+@tls_fnptr_ie = thread_local(initialexec) global void()* @checkTlsFunctionPointerCall
+@tls_fnptr_le = thread_local(localexec) global void()* @checkTlsFunctionPointerCall
 
-; NORELOC-NOT: R_MIPS_JALR
+define void @checkTlsFunctionPointerCall() {
+entry:
+; There should not be any *JALR relocations in this function other than the
+; calls to __tls_get_addr:
+; ALL-LABEL: checkTlsFunctionPointerCall:
+; ALL-NOT: MIPS_JALR
+; JALR-ALL: .reloc {{.+}}MIPS_JALR, __tls_get_addr
+; ALL-NOT: MIPS_JALR
+; JALR-ALL: .reloc {{.+}}MIPS_JALR, __tls_get_addr
+; ALL-NOT: _MIPS_JALR
+  %func_gd = load void()*, void()** @tls_fnptr_gd
+  call void %func_gd()
+  %func_ld = load void()*, void()** @tls_fnptr_ld
+  call void %func_ld()
+  %func_ie = load void()*, void()** @tls_fnptr_ie
+  call void %func_ie()
+  %func_le = load void()*, void()** @tls_fnptr_le
+  call void %func_le()
   ret void
 }
diff --git a/llvm/test/CodeGen/Mips/s2rem.ll b/llvm/test/CodeGen/Mips/s2rem.ll
index 715abc072..912563255 100644
--- a/llvm/test/CodeGen/Mips/s2rem.ll
+++ b/llvm/test/CodeGen/Mips/s2rem.ll
@@ -86,7 +86,7 @@ entry:
 
 declare void @vf(float) #1
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 
diff --git a/llvm/test/CodeGen/Mips/sel1c.ll b/llvm/test/CodeGen/Mips/sel1c.ll
index b09be8d41..7013f2813 100644
--- a/llvm/test/CodeGen/Mips/sel1c.ll
+++ b/llvm/test/CodeGen/Mips/sel1c.ll
@@ -16,6 +16,6 @@ entry:
 ; cond-b-short:	bteqz	$BB0_{{[0-9]+}}  # 16 bit inst
 }
 
-attributes #0 = { nounwind optsize "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="true" }
+attributes #0 = { nounwind optsize "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="true" }
 
 
diff --git a/llvm/test/CodeGen/Mips/sel2c.ll b/llvm/test/CodeGen/Mips/sel2c.ll
index 0650147be..73f9e1e91 100644
--- a/llvm/test/CodeGen/Mips/sel2c.ll
+++ b/llvm/test/CodeGen/Mips/sel2c.ll
@@ -16,6 +16,6 @@ entry:
   ret void
 }
 
-attributes #0 = { nounwind optsize "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="true" }
+attributes #0 = { nounwind optsize "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="true" }
 
 
diff --git a/llvm/test/CodeGen/Mips/simplebr.ll b/llvm/test/CodeGen/Mips/simplebr.ll
index 96dfce915..670ab7a0a 100644
--- a/llvm/test/CodeGen/Mips/simplebr.ll
+++ b/llvm/test/CodeGen/Mips/simplebr.ll
@@ -31,7 +31,7 @@ declare void @goo(...) #1
 
 declare void @hoo(...) #1
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="true" }
-attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="true" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="true" }
+attributes #1 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="true" }
 
 
diff --git a/llvm/test/CodeGen/Mips/sr1.ll b/llvm/test/CodeGen/Mips/sr1.ll
index b3fdef0a1..250552835 100644
--- a/llvm/test/CodeGen/Mips/sr1.ll
+++ b/llvm/test/CodeGen/Mips/sr1.ll
@@ -54,7 +54,7 @@ entry:
 
 declare float @xf() #1
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 
diff --git a/llvm/test/CodeGen/Mips/stack-alignment.ll b/llvm/test/CodeGen/Mips/stack-alignment.ll
index b25596f9b..4a8768ce6 100644
--- a/llvm/test/CodeGen/Mips/stack-alignment.ll
+++ b/llvm/test/CodeGen/Mips/stack-alignment.ll
@@ -16,4 +16,4 @@ entry:
   ret i32 14
 }
 
-attributes #0 = { "no-frame-pointer-elim"="true" }
+attributes #0 = { "frame-pointer"="all" }
diff --git a/llvm/test/CodeGen/Mips/start-asm-file.ll b/llvm/test/CodeGen/Mips/start-asm-file.ll
index bcea5da59..b8270574b 100644
--- a/llvm/test/CodeGen/Mips/start-asm-file.ll
+++ b/llvm/test/CodeGen/Mips/start-asm-file.ll
@@ -3,86 +3,73 @@
 ; ### O32 ABI ###
 ; RUN: llc -filetype=asm -mtriple mips-unknown-linux -mcpu=mips32 \
 ; RUN: -relocation-model=static %s -o - | \
-; RUN:   FileCheck -check-prefixes=CHECK-STATIC-O32,CHECK-STATIC-O32-NLEGACY %s
+; RUN:   FileCheck -DNAN=legacy -DABI=32 -check-prefixes=CHECK,STATIC-O32 %s
 
 ; RUN: llc -filetype=asm -mtriple mips-unknown-linux -mcpu=mips32 \
 ; RUN: -relocation-model=pic %s -o - | \
-; RUN:   FileCheck -check-prefixes=CHECK-PIC-O32,CHECK-PIC-O32-NLEGACY %s
+; RUN:   FileCheck -DNAN=legacy -DABI=32 -check-prefixes=CHECK,PIC-O32 %s
 
 ; RUN: llc -filetype=asm -mtriple mips-unknown-linux -mcpu=mips32 \
 ; RUN: -relocation-model=static -mattr=+nan2008 %s -o - | \
-; RUN:   FileCheck -check-prefixes=CHECK-STATIC-O32,CHECK-STATIC-O32-N2008 %s
+; RUN:   FileCheck -DNAN=2008 -DABI=32 -check-prefixes=CHECK,STATIC-O32 %s
 
 ; RUN: llc -filetype=asm -mtriple mips-unknown-linux -mcpu=mips32 \
 ; RUN: -relocation-model=pic -mattr=+nan2008 %s -o - | \
-; RUN:   FileCheck -check-prefixes=CHECK-PIC-O32,CHECK-PIC-O32-N2008 %s
+; RUN:   FileCheck -DNAN=2008 -DABI=32 -check-prefixes=CHECK,PIC-O32 %s
 
 ; ### N32 ABI ###
 ; RUN: llc -filetype=asm -mtriple mips64-unknown-linux -mcpu=mips64 \
 ; RUN: -relocation-model=static -target-abi n32 %s -o - | \
-; RUN:   FileCheck -check-prefixes=CHECK-STATIC-N32,CHECK-STATIC-N32-NLEGACY %s
+; RUN:   FileCheck -DNAN=legacy -DABI=N32 -check-prefixes=CHECK,STATIC-N32 %s
 
 ; RUN: llc -filetype=asm -mtriple mips64-unknown-linux -mcpu=mips64 \
 ; RUN: -relocation-model=pic -target-abi n32 %s -o - | \
-; RUN:   FileCheck -check-prefixes=CHECK-PIC-N32,CHECK-PIC-N32-NLEGACY %s
+; RUN:   FileCheck -DNAN=legacy -DABI=N32 -check-prefixes=CHECK,PIC-N32 %s
 
 ; RUN: llc -filetype=asm -mtriple mips64-unknown-linux -mcpu=mips64 \
 ; RUN: -relocation-model=static -target-abi n32 -mattr=+nan2008 %s -o - | \
-; RUN:   FileCheck -check-prefixes=CHECK-STATIC-N32,CHECK-STATIC-N32-N2008 %s
+; RUN:   FileCheck -DNAN=2008 -DABI=N32 -check-prefixes=CHECK,STATIC-N32 %s
 
 ; RUN: llc -filetype=asm -mtriple mips64-unknown-linux -mcpu=mips64 \
 ; RUN: -relocation-model=pic -target-abi n32 -mattr=+nan2008 %s -o - | \
-; RUN:   FileCheck -check-prefixes=CHECK-PIC-N32,CHECK-PIC-N32-N2008 %s
+; RUN:   FileCheck -DNAN=2008 -DABI=N32 -check-prefixes=CHECK,PIC-N32 %s
 
 ; ### N64 ABI ###
 ; RUN: llc -filetype=asm -mtriple mips64-unknown-linux -mcpu=mips64 \
 ; RUN: -relocation-model=static -target-abi n64 %s -o - | \
-; RUN:   FileCheck -check-prefixes=CHECK-STATIC-N64,CHECK-STATIC-N64-NLEGACY %s
+; RUN:   FileCheck -DNAN=legacy -DABI=64 -check-prefixes=CHECK,STATIC-N64 %s
 
 ; RUN: llc -filetype=asm -mtriple mips64-unknown-linux -mcpu=mips64 \
 ; RUN: -relocation-model=pic -target-abi n64 %s -o - | \
-; RUN:   FileCheck -check-prefixes=CHECK-PIC-N64,CHECK-PIC-N64-NLEGACY %s
+; RUN:   FileCheck -DNAN=legacy -DABI=64 -check-prefixes=CHECK,PIC-N64 %s
 
 ; RUN: llc -filetype=asm -mtriple mips64-unknown-linux -mcpu=mips64 \
 ; RUN: -relocation-model=static -target-abi n64 -mattr=+nan2008 %s -o - | \
-; RUN:   FileCheck -check-prefixes=CHECK-STATIC-N64,CHECK-STATIC-N64-N2008 %s
+; RUN:   FileCheck -DNAN=2008 -DABI=64 -check-prefixes=CHECK,STATIC-N64 %s
 
 ; RUN: llc -filetype=asm -mtriple mips64-unknown-linux -mcpu=mips64 \
 ; RUN: -relocation-model=pic -target-abi n64 -mattr=+nan2008 %s -o - | \
-; RUN:   FileCheck -check-prefixes=CHECK-PIC-N64,CHECK-PIC-N64-N2008 %s
-
-; CHECK-STATIC-O32: .abicalls
-; CHECK-STATIC-O32: .option pic0
-; CHECK-STATIC-O32: .section .mdebug.abi32
-; CHECK-STATIC-O32-NLEGACY: .nan legacy
-; CHECK-STATIC-O32-N2008: .nan 2008
-
-; CHECK-PIC-O32: .abicalls
-; CHECK-PIC-O32-NOT: .option pic0
-; CHECK-PIC-O32: .section .mdebug.abi32
-; CHECK-PIC-O32-NLEGACY: .nan legacy
-; CHECK-PIC-O32-N2008: .nan 2008
-
-; CHECK-STATIC-N32: .abicalls
-; CHECK-STATIC-N32: .option pic0
-; CHECK-STATIC-N32: .section .mdebug.abiN32
-; CHECK-STATIC-N32-NLEGACY: .nan legacy
-; CHECK-STATIC-N32-N2008: .nan 2008
-
-; CHECK-PIC-N32: .abicalls
-; CHECK-PIC-N32-NOT: .option pic0
-; CHECK-PIC-N32: .section .mdebug.abiN32
-; CHECK-PIC-N32-NLEGACY: .nan legacy
-; CHECK-PIC-N32-N2008: .nan 2008
-
-; CHECK-STATIC-N64-NOT: .abicalls
-; CHECK-STATIC-N64-NOT: .option pic0
-; CHECK-STATIC-N64: .section .mdebug.abi64
-; CHECK-STATIC-N64-NLEGACY: .nan legacy
-; CHECK-STATIC-N64-N2008: .nan 2008
-
-; CHECK-PIC-N64: .abicalls
-; CHECK-PIC-N64-NOT: .option pic0
-; CHECK-PIC-N64: .section .mdebug.abi64
-; CHECK-PIC-N64-NLEGACY: .nan legacy
-; CHECK-PIC-N64-N2008: .nan 2008
+; RUN:   FileCheck -DNAN=2008 -DABI=64 -check-prefixes=CHECK,PIC-N64 %s
+
+; STATIC-O32: .abicalls
+; STATIC-O32: .option pic0
+
+; PIC-O32: .abicalls
+; PIC-O32-NOT: .option pic0
+
+; STATIC-N32: .abicalls
+; STATIC-N32: .option pic0
+
+; PIC-N32: .abicalls
+; PIC-N32-NOT: .option pic0
+
+; STATIC-N64-NOT: .abicalls
+; STATIC-N64-NOT: .option pic0
+
+; PIC-N64: .abicalls
+; PIC-N64-NOT: .option pic0
+
+; CHECK: .section .mdebug.abi[[ABI]]
+; CHECK: .nan [[NAN]]
+; CHECK: .text
+; CHECK: .file
diff --git a/llvm/test/CodeGen/Mips/tnaked.ll b/llvm/test/CodeGen/Mips/tnaked.ll
index 7dff19c5d..803fef965 100644
--- a/llvm/test/CodeGen/Mips/tnaked.ll
+++ b/llvm/test/CodeGen/Mips/tnaked.ll
@@ -21,9 +21,9 @@ entry:
 ; CHECK: 	.ent	tnonaked
 ; CHECK-LABEL:         tnonaked:
 ; CHECK:	.frame	$fp,8,$ra
-; CHECK:        .mask 	0x40000000,-4
+; CHECK:        .mask 	0xc0000000,-4
 ; CHECK:	.fmask	0x00000000,0
 ; CHECK: 	addiu	$sp, $sp, -8
 
-attributes #0 = { naked noinline nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { naked noinline nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/Mips/v2i16tof32.ll b/llvm/test/CodeGen/Mips/v2i16tof32.ll
index 334413b03..ab9fa9eef 100644
--- a/llvm/test/CodeGen/Mips/v2i16tof32.ll
+++ b/llvm/test/CodeGen/Mips/v2i16tof32.ll
@@ -9,8 +9,10 @@ define float @f(<8 x i16>* %a) {
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    addiu $sp, $sp, -32
 ; CHECK-NEXT:    .cfi_def_cfa_offset 32
-; CHECK-NEXT:    sw $fp, 28($sp) # 4-byte Folded Spill
-; CHECK-NEXT:    .cfi_offset 30, -4
+; CHECK-NEXT:    sw $ra, 28($sp) # 4-byte Folded Spill
+; CHECK-NEXT:    sw $fp, 24($sp) # 4-byte Folded Spill
+; CHECK-NEXT:    .cfi_offset 31, -4
+; CHECK-NEXT:    .cfi_offset 30, -8
 ; CHECK-NEXT:    move $fp, $sp
 ; CHECK-NEXT:    .cfi_def_cfa_register 30
 ; CHECK-NEXT:    addiu $1, $zero, -16
@@ -25,7 +27,8 @@ define float @f(<8 x i16>* %a) {
 ; CHECK-NEXT:    sw $1, 4($sp)
 ; CHECK-NEXT:    mtc1 $2, $f0
 ; CHECK-NEXT:    move $sp, $fp
-; CHECK-NEXT:    lw $fp, 28($sp) # 4-byte Folded Reload
+; CHECK-NEXT:    lw $fp, 24($sp) # 4-byte Folded Reload
+; CHECK-NEXT:    lw $ra, 28($sp) # 4-byte Folded Reload
 ; CHECK-NEXT:    jr $ra
 ; CHECK-NEXT:    addiu $sp, $sp, 32
 entry:
diff --git a/llvm/test/CodeGen/NVPTX/bug21465.ll b/llvm/test/CodeGen/NVPTX/bug21465.ll
index 7a0909225..9a221988d 100644
--- a/llvm/test/CodeGen/NVPTX/bug21465.ll
+++ b/llvm/test/CodeGen/NVPTX/bug21465.ll
@@ -21,7 +21,7 @@ entry:
   ret void
 }
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !nvvm.annotations = !{!0}
 
diff --git a/llvm/test/CodeGen/NVPTX/bug22322.ll b/llvm/test/CodeGen/NVPTX/bug22322.ll
index d59699ace..b200fd794 100644
--- a/llvm/test/CodeGen/NVPTX/bug22322.ll
+++ b/llvm/test/CodeGen/NVPTX/bug22322.ll
@@ -51,7 +51,7 @@ declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture) #2
 ; Function Attrs: nounwind
 declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture) #2
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="true" "no-nans-fp-math"="true" "stack-protector-buffer-size"="8" "no-signed-zeros-fp-math"="true" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="true" "no-nans-fp-math"="true" "stack-protector-buffer-size"="8" "no-signed-zeros-fp-math"="true" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 attributes #2 = { nounwind }
 
diff --git a/llvm/test/CodeGen/NVPTX/bug41651.ll b/llvm/test/CodeGen/NVPTX/bug41651.ll
new file mode 100644
index 000000000..6039d3940
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/bug41651.ll
@@ -0,0 +1,13 @@
+; RUN: llc -filetype=asm -o - %s | FileCheck %s
+target datalayout = "e-i64:64-i128:128-v16:16-v32:32-n16:32:64"
+target triple = "nvptx64-nvidia-cuda"
+
+%func = type { i32 (i32, i32)** }
+
+; CHECK: foo
+; CHECK: call
+; CHECK: ret
+define void @foo() {
+  %call = call %func undef(i32 0, i32 1)
+  ret void
+}
diff --git a/llvm/test/CodeGen/NVPTX/pr17529.ll b/llvm/test/CodeGen/NVPTX/pr17529.ll
index a7519776f..d605c29fe 100644
--- a/llvm/test/CodeGen/NVPTX/pr17529.ll
+++ b/llvm/test/CodeGen/NVPTX/pr17529.ll
@@ -35,4 +35,4 @@ for.end:                                          ; preds = %middle.block, %entr
   ret void
 }
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/NVPTX/shfl-p.ll b/llvm/test/CodeGen/NVPTX/shfl-p.ll
new file mode 100644
index 000000000..27951f214
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/shfl-p.ll
@@ -0,0 +1,172 @@
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_30 | FileCheck %s
+
+declare {i32, i1} @llvm.nvvm.shfl.down.i32p(i32, i32, i32)
+declare {float, i1} @llvm.nvvm.shfl.down.f32p(float, i32, i32)
+declare {i32, i1} @llvm.nvvm.shfl.up.i32p(i32, i32, i32)
+declare {float, i1} @llvm.nvvm.shfl.up.f32p(float, i32, i32)
+declare {i32, i1} @llvm.nvvm.shfl.bfly.i32p(i32, i32, i32)
+declare {float, i1} @llvm.nvvm.shfl.bfly.f32p(float, i32, i32)
+declare {i32, i1} @llvm.nvvm.shfl.idx.i32p(i32, i32, i32)
+declare {float, i1} @llvm.nvvm.shfl.idx.f32p(float, i32, i32)
+
+; CHECK-LABEL: .func{{.*}}shfl.i32.rrr
+define {i32, i1} @shfl.i32.rrr(i32 %a, i32 %b, i32 %c) {
+  ; CHECK: ld.param.u32 [[A:%r[0-9]+]]
+  ; CHECK: ld.param.u32 [[B:%r[0-9]+]]
+  ; CHECK: ld.param.u32 [[C:%r[0-9]+]]
+  ; CHECK: shfl.down.b32 [[OUT:%r[0-9]+]]|[[OUTP:%p[0-9]+]], [[A]], [[B]], [[C]];
+  ; CHECK: st.param.{{.}}32 {{.*}}, [[OUT]]
+  %val = call {i32, i1} @llvm.nvvm.shfl.down.i32p(i32 %a, i32 %b, i32 %c)
+  ret {i32, i1} %val
+}
+
+; CHECK-LABEL: .func{{.*}}shfl.i32.irr
+define {i32, i1} @shfl.i32.irr(i32 %a, i32 %b, i32 %c) {
+  ; CHECK: ld.param.u32 [[A:%r[0-9]+]]
+  ; CHECK: ld.param.u32 [[B:%r[0-9]+]]
+  ; CHECK: ld.param.u32 [[C:%r[0-9]+]]
+  ; CHECK: shfl.down.b32 [[OUT:%r[0-9]+]]|[[OUTP:%p[0-9]+]], [[A]], [[B]], [[C]];
+  ; CHECK: st.param.{{.}}32 {{.*}}, [[OUT]]
+  %val = call {i32, i1} @llvm.nvvm.shfl.down.i32p(i32 %a, i32 %b, i32 %c)
+  ret {i32, i1} %val
+}
+
+; CHECK-LABEL: .func{{.*}}shfl.i32.rri
+define {i32, i1} @shfl.i32.rri(i32 %a, i32 %b) {
+  ; CHECK: ld.param.u32 [[A:%r[0-9]+]]
+  ; CHECK: ld.param.u32 [[B:%r[0-9]+]]
+  ; CHECK: shfl.down.b32 [[OUT:%r[0-9]+]]|[[OUTP:%p[0-9]+]], [[A]], [[B]], 1;
+  ; CHECK: st.param.{{.}}32 {{.*}}, [[OUT]]
+  %val = call {i32, i1} @llvm.nvvm.shfl.down.i32p(i32 %a, i32 %b, i32 1)
+  ret {i32, i1} %val
+}
+
+; CHECK-LABEL: .func{{.*}}shfl.i32.iri
+define {i32, i1} @shfl.i32.iri(i32 %a, i32 %b) {
+  ; CHECK: ld.param.u32 [[A:%r[0-9]+]]
+  ; CHECK: ld.param.u32 [[B:%r[0-9]+]]
+  ; CHECK: shfl.down.b32 [[OUT:%r[0-9]+]]|[[OUTP:%p[0-9]+]], [[A]], [[B]], 2;
+  ; CHECK: st.param.{{.}}32 {{.*}}, [[OUT]]
+  %val = call {i32, i1} @llvm.nvvm.shfl.down.i32p(i32 %a, i32 %b, i32 2)
+  ret {i32, i1} %val
+}
+
+; CHECK-LABEL: .func{{.*}}shfl.i32.rir
+define {i32, i1} @shfl.i32.rir(i32 %a, i32 %c) {
+  ; CHECK: ld.param.u32 [[A:%r[0-9]+]]
+  ; CHECK: ld.param.u32 [[C:%r[0-9]+]]
+  ; CHECK: shfl.down.b32 [[OUT:%r[0-9]+]]|[[OUTP:%p[0-9]+]], [[A]], 1, [[C]];
+  ; CHECK: st.param.{{.}}32 {{.*}}, [[OUT]]
+  %val = call {i32, i1} @llvm.nvvm.shfl.down.i32p(i32 %a, i32 1, i32 %c)
+  ret {i32, i1} %val
+}
+
+; CHECK-LABEL: .func{{.*}}shfl.i32.iir
+define {i32, i1} @shfl.i32.iir(i32 %a, i32 %c) {
+  ; CHECK: ld.param.u32 [[A:%r[0-9]+]]
+  ; CHECK: ld.param.u32 [[C:%r[0-9]+]]
+  ; CHECK: shfl.down.b32 [[OUT:%r[0-9]+]]|[[OUTP:%p[0-9]+]], [[A]], 2, [[C]];
+  ; CHECK: st.param.{{.}}32 {{.*}}, [[OUT]]
+  %val = call {i32, i1} @llvm.nvvm.shfl.down.i32p(i32 %a, i32 2, i32 %c)
+  ret {i32, i1} %val
+}
+
+; CHECK-LABEL: .func{{.*}}shfl.i32.rii
+define {i32, i1} @shfl.i32.rii(i32 %a) {
+  ; CHECK: ld.param.u32 [[A:%r[0-9]+]]
+  ; CHECK: shfl.down.b32 [[OUT:%r[0-9]+]]|[[OUTP:%p[0-9]+]], [[A]], 1, 2;
+  ; CHECK: st.param.{{.}}32 {{.*}}, [[OUT]]
+  %val = call {i32, i1} @llvm.nvvm.shfl.down.i32p(i32 %a, i32 1, i32 2)
+  ret {i32, i1} %val
+}
+
+; CHECK-LABEL: .func{{.*}}shfl.i32.iii
+define {i32, i1} @shfl.i32.iii(i32 %a, i32 %b) {
+  ; CHECK: ld.param.u32 [[A:%r[0-9]+]]
+  ; CHECK: shfl.down.b32 [[OUT:%r[0-9]+]]|[[OUTP:%p[0-9]+]], [[A]], 2, 3;
+  ; CHECK: st.param.{{.}}32 {{.*}}, [[OUT]]
+  %val = call {i32, i1} @llvm.nvvm.shfl.down.i32p(i32 %a, i32 2, i32 3)
+  ret {i32, i1} %val
+}
+
+;; Same intrinsics, but for float
+
+; CHECK-LABEL: .func{{.*}}shfl.f32.rrr
+define {float, i1} @shfl.f32.rrr(float %a, i32 %b, i32 %c) {
+  ; CHECK: ld.param.f32 [[A:%f[0-9]+]]
+  ; CHECK: ld.param.u32 [[B:%r[0-9]+]]
+  ; CHECK: ld.param.u32 [[C:%r[0-9]+]]
+  ; CHECK: shfl.down.b32 [[OUT:%f[0-9]+]]|[[OUTP:%p[0-9]+]], [[A]], [[B]], [[C]];
+  ; CHECK: st.param.{{.}}32 {{.*}}, [[OUT]]
+  %val = call {float, i1} @llvm.nvvm.shfl.down.f32p(float %a, i32 %b, i32 %c)
+  ret {float, i1} %val
+}
+
+; CHECK-LABEL: .func{{.*}}shfl.f32.irr
+define {float, i1} @shfl.f32.irr(float %a, i32 %b, i32 %c) {
+  ; CHECK: ld.param.f32 [[A:%f[0-9]+]]
+  ; CHECK: ld.param.u32 [[B:%r[0-9]+]]
+  ; CHECK: ld.param.u32 [[C:%r[0-9]+]]
+  ; CHECK: shfl.down.b32 [[OUT:%f[0-9]+]]|[[OUTP:%p[0-9]+]], [[A]], [[B]], [[C]];
+  ; CHECK: st.param.{{.}}32 {{.*}}, [[OUT]]
+  %val = call {float, i1} @llvm.nvvm.shfl.down.f32p(float %a, i32 %b, i32 %c)
+  ret {float, i1} %val
+}
+
+; CHECK-LABEL: .func{{.*}}shfl.f32.rri
+define {float, i1} @shfl.f32.rri(float %a, i32 %b) {
+  ; CHECK: ld.param.f32 [[A:%f[0-9]+]]
+  ; CHECK: ld.param.u32 [[B:%r[0-9]+]]
+  ; CHECK: shfl.down.b32 [[OUT:%f[0-9]+]]|[[OUTP:%p[0-9]+]], [[A]], [[B]], 1;
+  ; CHECK: st.param.{{.}}32 {{.*}}, [[OUT]]
+  %val = call {float, i1} @llvm.nvvm.shfl.down.f32p(float %a, i32 %b, i32 1)
+  ret {float, i1} %val
+}
+
+; CHECK-LABEL: .func{{.*}}shfl.f32.iri
+define {float, i1} @shfl.f32.iri(float %a, i32 %b) {
+  ; CHECK: ld.param.f32 [[A:%f[0-9]+]]
+  ; CHECK: ld.param.u32 [[B:%r[0-9]+]]
+  ; CHECK: shfl.down.b32 [[OUT:%f[0-9]+]]|[[OUTP:%p[0-9]+]], [[A]], [[B]], 2;
+  ; CHECK: st.param.{{.}}32 {{.*}}, [[OUT]]
+  %val = call {float, i1} @llvm.nvvm.shfl.down.f32p(float %a, i32 %b, i32 2)
+  ret {float, i1} %val
+}
+
+; CHECK-LABEL: .func{{.*}}shfl.f32.rir
+define {float, i1} @shfl.f32.rir(float %a, i32 %c) {
+  ; CHECK: ld.param.f32 [[A:%f[0-9]+]]
+  ; CHECK: ld.param.u32 [[C:%r[0-9]+]]
+  ; CHECK: shfl.down.b32 [[OUT:%f[0-9]+]]|[[OUTP:%p[0-9]+]], [[A]], 1, [[C]];
+  ; CHECK: st.param.{{.}}32 {{.*}}, [[OUT]]
+  %val = call {float, i1} @llvm.nvvm.shfl.down.f32p(float %a, i32 1, i32 %c)
+  ret {float, i1} %val
+}
+
+; CHECK-LABEL: .func{{.*}}shfl.f32.iir
+define {float, i1} @shfl.f32.iir(float %a, i32 %c) {
+  ; CHECK: ld.param.f32 [[A:%f[0-9]+]]
+  ; CHECK: ld.param.u32 [[C:%r[0-9]+]]
+  ; CHECK: shfl.down.b32 [[OUT:%f[0-9]+]]|[[OUTP:%p[0-9]+]], [[A]], 2, [[C]];
+  ; CHECK: st.param.{{.}}32 {{.*}}, [[OUT]]
+  %val = call {float, i1} @llvm.nvvm.shfl.down.f32p(float %a, i32 2, i32 %c)
+  ret {float, i1} %val
+}
+
+; CHECK-LABEL: .func{{.*}}shfl.f32.rii
+define {float, i1} @shfl.f32.rii(float %a) {
+  ; CHECK: ld.param.f32 [[A:%f[0-9]+]]
+  ; CHECK: shfl.down.b32 [[OUT:%f[0-9]+]]|[[OUTP:%p[0-9]+]], [[A]], 1, 2;
+  ; CHECK: st.param.{{.}}32 {{.*}}, [[OUT]]
+  %val = call {float, i1} @llvm.nvvm.shfl.down.f32p(float %a, i32 1, i32 2)
+  ret {float, i1} %val
+}
+
+; CHECK-LABEL: .func{{.*}}shfl.f32.iii
+define {float, i1} @shfl.f32.iii(float %a, i32 %b) {
+  ; CHECK: ld.param.f32 [[A:%f[0-9]+]]
+  ; CHECK: shfl.down.b32 [[OUT:%f[0-9]+]]|[[OUTP:%p[0-9]+]], [[A]], 2, 3;
+  ; CHECK: st.param.{{.}}32 {{.*}}, [[OUT]]
+  %val = call {float, i1} @llvm.nvvm.shfl.down.f32p(float %a, i32 2, i32 3)
+  ret {float, i1} %val
+}
diff --git a/llvm/test/CodeGen/NVPTX/shfl-sync-p.ll b/llvm/test/CodeGen/NVPTX/shfl-sync-p.ll
new file mode 100644
index 000000000..44189810c
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/shfl-sync-p.ll
@@ -0,0 +1,180 @@
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_30 | FileCheck %s
+
+declare {i32, i1} @llvm.nvvm.shfl.sync.down.i32p(i32, i32, i32, i32)
+declare {float, i1} @llvm.nvvm.shfl.sync.down.f32p(i32, float, i32, i32)
+declare {i32, i1} @llvm.nvvm.shfl.sync.up.i32p(i32, i32, i32, i32)
+declare {float, i1} @llvm.nvvm.shfl.sync.up.f32p(i32, float, i32, i32)
+declare {i32, i1} @llvm.nvvm.shfl.sync.bfly.i32p(i32, i32, i32, i32)
+declare {float, i1} @llvm.nvvm.shfl.sync.bfly.f32p(i32, float, i32, i32)
+declare {i32, i1} @llvm.nvvm.shfl.sync.idx.i32p(i32, i32, i32, i32)
+declare {float, i1} @llvm.nvvm.shfl.sync.idx.f32p(i32, float, i32, i32)
+
+; CHECK-LABEL: .func{{.*}}shfl.sync.i32.rrr
+define {i32, i1} @shfl.sync.i32.rrr(i32 %mask, i32 %a, i32 %b, i32 %c) {
+  ; CHECK: ld.param.u32 [[MASK:%r[0-9]+]]
+  ; CHECK: ld.param.u32 [[A:%r[0-9]+]]
+  ; CHECK: ld.param.u32 [[B:%r[0-9]+]]
+  ; CHECK: ld.param.u32 [[C:%r[0-9]+]]
+  ; CHECK: shfl.sync.down.b32 [[OUT:%r[0-9]+]]|[[OUTP:%p[0-9]+]], [[A]], [[B]], [[C]], [[MASK]];
+  ; CHECK: st.param.{{.}}32 {{.*}}, [[OUT]]
+  %val = call {i32, i1} @llvm.nvvm.shfl.sync.down.i32p(i32 %mask, i32 %a, i32 %b, i32 %c)
+  ret {i32, i1} %val
+}
+
+; CHECK-LABEL: .func{{.*}}shfl.sync.i32.irr
+define {i32, i1} @shfl.sync.i32.irr(i32 %a, i32 %b, i32 %c) {
+  ; CHECK: ld.param.u32 [[A:%r[0-9]+]]
+  ; CHECK: ld.param.u32 [[B:%r[0-9]+]]
+  ; CHECK: ld.param.u32 [[C:%r[0-9]+]]
+  ; CHECK: shfl.sync.down.b32 [[OUT:%r[0-9]+]]|[[OUTP:%p[0-9]+]], [[A]], [[B]], [[C]], 1;
+  ; CHECK: st.param.{{.}}32 {{.*}}, [[OUT]]
+  %val = call {i32, i1} @llvm.nvvm.shfl.sync.down.i32p(i32 1, i32 %a, i32 %b, i32 %c)
+  ret {i32, i1} %val
+}
+
+; CHECK-LABEL: .func{{.*}}shfl.sync.i32.rri
+define {i32, i1} @shfl.sync.i32.rri(i32 %mask, i32 %a, i32 %b) {
+  ; CHECK: ld.param.u32 [[MASK:%r[0-9]+]]
+  ; CHECK: ld.param.u32 [[A:%r[0-9]+]]
+  ; CHECK: ld.param.u32 [[B:%r[0-9]+]]
+  ; CHECK: shfl.sync.down.b32 [[OUT:%r[0-9]+]]|[[OUTP:%p[0-9]+]], [[A]], [[B]], 1, [[MASK]];
+  ; CHECK: st.param.{{.}}32 {{.*}}, [[OUT]]
+  %val = call {i32, i1} @llvm.nvvm.shfl.sync.down.i32p(i32 %mask, i32 %a, i32 %b, i32 1)
+  ret {i32, i1} %val
+}
+
+; CHECK-LABEL: .func{{.*}}shfl.sync.i32.iri
+define {i32, i1} @shfl.sync.i32.iri(i32 %a, i32 %b) {
+  ; CHECK: ld.param.u32 [[A:%r[0-9]+]]
+  ; CHECK: ld.param.u32 [[B:%r[0-9]+]]
+  ; CHECK: shfl.sync.down.b32 [[OUT:%r[0-9]+]]|[[OUTP:%p[0-9]+]], [[A]], [[B]], 2, 1;
+  ; CHECK: st.param.{{.}}32 {{.*}}, [[OUT]]
+  %val = call {i32, i1} @llvm.nvvm.shfl.sync.down.i32p(i32 1, i32 %a, i32 %b, i32 2)
+  ret {i32, i1} %val
+}
+
+; CHECK-LABEL: .func{{.*}}shfl.sync.i32.rir
+define {i32, i1} @shfl.sync.i32.rir(i32 %mask, i32 %a, i32 %c) {
+  ; CHECK: ld.param.u32 [[MASK:%r[0-9]+]]
+  ; CHECK: ld.param.u32 [[A:%r[0-9]+]]
+  ; CHECK: ld.param.u32 [[C:%r[0-9]+]]
+  ; CHECK: shfl.sync.down.b32 [[OUT:%r[0-9]+]]|[[OUTP:%p[0-9]+]], [[A]], 1, [[C]], [[MASK]];
+  ; CHECK: st.param.{{.}}32 {{.*}}, [[OUT]]
+  %val = call {i32, i1} @llvm.nvvm.shfl.sync.down.i32p(i32 %mask, i32 %a, i32 1, i32 %c)
+  ret {i32, i1} %val
+}
+
+; CHECK-LABEL: .func{{.*}}shfl.sync.i32.iir
+define {i32, i1} @shfl.sync.i32.iir(i32 %a, i32 %c) {
+  ; CHECK: ld.param.u32 [[A:%r[0-9]+]]
+  ; CHECK: ld.param.u32 [[C:%r[0-9]+]]
+  ; CHECK: shfl.sync.down.b32 [[OUT:%r[0-9]+]]|[[OUTP:%p[0-9]+]], [[A]], 2, [[C]], 1;
+  ; CHECK: st.param.{{.}}32 {{.*}}, [[OUT]]
+  %val = call {i32, i1} @llvm.nvvm.shfl.sync.down.i32p(i32 1, i32 %a, i32 2, i32 %c)
+  ret {i32, i1} %val
+}
+
+; CHECK-LABEL: .func{{.*}}shfl.sync.i32.rii
+define {i32, i1} @shfl.sync.i32.rii(i32 %mask, i32 %a) {
+  ; CHECK: ld.param.u32 [[MASK:%r[0-9]+]]
+  ; CHECK: ld.param.u32 [[A:%r[0-9]+]]
+  ; CHECK: shfl.sync.down.b32 [[OUT:%r[0-9]+]]|[[OUTP:%p[0-9]+]], [[A]], 1, 2, [[MASK]];
+  ; CHECK: st.param.{{.}}32 {{.*}}, [[OUT]]
+  %val = call {i32, i1} @llvm.nvvm.shfl.sync.down.i32p(i32 %mask, i32 %a, i32 1, i32 2)
+  ret {i32, i1} %val
+}
+
+; CHECK-LABEL: .func{{.*}}shfl.sync.i32.iii
+define {i32, i1} @shfl.sync.i32.iii(i32 %a, i32 %b) {
+  ; CHECK: ld.param.u32 [[A:%r[0-9]+]]
+  ; CHECK: shfl.sync.down.b32 [[OUT:%r[0-9]+]]|[[OUTP:%p[0-9]+]], [[A]], 2, 3, 1;
+  ; CHECK: st.param.{{.}}32 {{.*}}, [[OUT]]
+  %val = call {i32, i1} @llvm.nvvm.shfl.sync.down.i32p(i32 1, i32 %a, i32 2, i32 3)
+  ret {i32, i1} %val
+}
+
+;; Same intrinsics, but for float
+
+; CHECK-LABEL: .func{{.*}}shfl.sync.f32.rrr
+define {float, i1} @shfl.sync.f32.rrr(i32 %mask, float %a, i32 %b, i32 %c) {
+  ; CHECK: ld.param.u32 [[MASK:%r[0-9]+]]
+  ; CHECK: ld.param.f32 [[A:%f[0-9]+]]
+  ; CHECK: ld.param.u32 [[B:%r[0-9]+]]
+  ; CHECK: ld.param.u32 [[C:%r[0-9]+]]
+  ; CHECK: shfl.sync.down.b32 [[OUT:%f[0-9]+]]|[[OUTP:%p[0-9]+]], [[A]], [[B]], [[C]], [[MASK]];
+  ; CHECK: st.param.{{.}}32 {{.*}}, [[OUT]]
+  %val = call {float, i1} @llvm.nvvm.shfl.sync.down.f32p(i32 %mask, float %a, i32 %b, i32 %c)
+  ret {float, i1} %val
+}
+
+; CHECK-LABEL: .func{{.*}}shfl.sync.f32.irr
+define {float, i1} @shfl.sync.f32.irr(float %a, i32 %b, i32 %c) {
+  ; CHECK: ld.param.f32 [[A:%f[0-9]+]]
+  ; CHECK: ld.param.u32 [[B:%r[0-9]+]]
+  ; CHECK: ld.param.u32 [[C:%r[0-9]+]]
+  ; CHECK: shfl.sync.down.b32 [[OUT:%f[0-9]+]]|[[OUTP:%p[0-9]+]], [[A]], [[B]], [[C]], 1;
+  ; CHECK: st.param.{{.}}32 {{.*}}, [[OUT]]
+  %val = call {float, i1} @llvm.nvvm.shfl.sync.down.f32p(i32 1, float %a, i32 %b, i32 %c)
+  ret {float, i1} %val
+}
+
+; CHECK-LABEL: .func{{.*}}shfl.sync.f32.rri
+define {float, i1} @shfl.sync.f32.rri(i32 %mask, float %a, i32 %b) {
+  ; CHECK: ld.param.u32 [[MASK:%r[0-9]+]]
+  ; CHECK: ld.param.f32 [[A:%f[0-9]+]]
+  ; CHECK: ld.param.u32 [[B:%r[0-9]+]]
+  ; CHECK: shfl.sync.down.b32 [[OUT:%f[0-9]+]]|[[OUTP:%p[0-9]+]], [[A]], [[B]], 1, [[MASK]];
+  ; CHECK: st.param.{{.}}32 {{.*}}, [[OUT]]
+  %val = call {float, i1} @llvm.nvvm.shfl.sync.down.f32p(i32 %mask, float %a, i32 %b, i32 1)
+  ret {float, i1} %val
+}
+
+; CHECK-LABEL: .func{{.*}}shfl.sync.f32.iri
+define {float, i1} @shfl.sync.f32.iri(float %a, i32 %b) {
+  ; CHECK: ld.param.f32 [[A:%f[0-9]+]]
+  ; CHECK: ld.param.u32 [[B:%r[0-9]+]]
+  ; CHECK: shfl.sync.down.b32 [[OUT:%f[0-9]+]]|[[OUTP:%p[0-9]+]], [[A]], [[B]], 2, 1;
+  ; CHECK: st.param.{{.}}32 {{.*}}, [[OUT]]
+  %val = call {float, i1} @llvm.nvvm.shfl.sync.down.f32p(i32 1, float %a, i32 %b, i32 2)
+  ret {float, i1} %val
+}
+
+; CHECK-LABEL: .func{{.*}}shfl.sync.f32.rir
+define {float, i1} @shfl.sync.f32.rir(i32 %mask, float %a, i32 %c) {
+  ; CHECK: ld.param.u32 [[MASK:%r[0-9]+]]
+  ; CHECK: ld.param.f32 [[A:%f[0-9]+]]
+  ; CHECK: ld.param.u32 [[C:%r[0-9]+]]
+  ; CHECK: shfl.sync.down.b32 [[OUT:%f[0-9]+]]|[[OUTP:%p[0-9]+]], [[A]], 1, [[C]], [[MASK]];
+  ; CHECK: st.param.{{.}}32 {{.*}}, [[OUT]]
+  %val = call {float, i1} @llvm.nvvm.shfl.sync.down.f32p(i32 %mask, float %a, i32 1, i32 %c)
+  ret {float, i1} %val
+}
+
+; CHECK-LABEL: .func{{.*}}shfl.sync.f32.iir
+define {float, i1} @shfl.sync.f32.iir(float %a, i32 %c) {
+  ; CHECK: ld.param.f32 [[A:%f[0-9]+]]
+  ; CHECK: ld.param.u32 [[C:%r[0-9]+]]
+  ; CHECK: shfl.sync.down.b32 [[OUT:%f[0-9]+]]|[[OUTP:%p[0-9]+]], [[A]], 2, [[C]], 1;
+  ; CHECK: st.param.{{.}}32 {{.*}}, [[OUT]]
+  %val = call {float, i1} @llvm.nvvm.shfl.sync.down.f32p(i32 1, float %a, i32 2, i32 %c)
+  ret {float, i1} %val
+}
+
+; CHECK-LABEL: .func{{.*}}shfl.sync.f32.rii
+define {float, i1} @shfl.sync.f32.rii(i32 %mask, float %a) {
+  ; CHECK: ld.param.u32 [[MASK:%r[0-9]+]]
+  ; CHECK: ld.param.f32 [[A:%f[0-9]+]]
+  ; CHECK: shfl.sync.down.b32 [[OUT:%f[0-9]+]]|[[OUTP:%p[0-9]+]], [[A]], 1, 2, [[MASK]];
+  ; CHECK: st.param.{{.}}32 {{.*}}, [[OUT]]
+  %val = call {float, i1} @llvm.nvvm.shfl.sync.down.f32p(i32 %mask, float %a, i32 1, i32 2)
+  ret {float, i1} %val
+}
+
+; CHECK-LABEL: .func{{.*}}shfl.sync.f32.iii
+define {float, i1} @shfl.sync.f32.iii(float %a, i32 %b) {
+  ; CHECK: ld.param.f32 [[A:%f[0-9]+]]
+  ; CHECK: shfl.sync.down.b32 [[OUT:%f[0-9]+]]|[[OUTP:%p[0-9]+]], [[A]], 2, 3, 1;
+  ; CHECK: st.param.{{.}}32 {{.*}}, [[OUT]]
+  %val = call {float, i1} @llvm.nvvm.shfl.sync.down.f32p(i32 1, float %a, i32 2, i32 3)
+  ret {float, i1} %val
+}
diff --git a/llvm/test/CodeGen/PowerPC/2007-09-08-unaligned.ll b/llvm/test/CodeGen/PowerPC/2007-09-08-unaligned.ll
index 86efa0217..41ef5ccd0 100644
--- a/llvm/test/CodeGen/PowerPC/2007-09-08-unaligned.ll
+++ b/llvm/test/CodeGen/PowerPC/2007-09-08-unaligned.ll
@@ -1,4 +1,5 @@
-; RUN: llc -verify-machineinstrs -mattr=-vsx < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -mattr=-vsx \
+; RUN:   -mattr=+allow-unaligned-fp-access < %s | FileCheck %s
 ; ModuleID = 'foo.c'
 
 target triple = "powerpc-unknown-linux-gnu"
diff --git a/llvm/test/CodeGen/PowerPC/2008-10-28-f128-i32.ll b/llvm/test/CodeGen/PowerPC/2008-10-28-f128-i32.ll
index cc495b321..7897d1c6b 100644
--- a/llvm/test/CodeGen/PowerPC/2008-10-28-f128-i32.ll
+++ b/llvm/test/CodeGen/PowerPC/2008-10-28-f128-i32.ll
@@ -36,7 +36,7 @@ define i64 @__fixunstfdi(ppc_fp128 %a) nounwind readnone {
 ; CHECK-NEXT:  # %bb.1: # %bb5
 ; CHECK-NEXT:    li 3, 0
 ; CHECK-NEXT:    li 4, 0
-; CHECK-NEXT:    b .LBB0_16
+; CHECK-NEXT:    b .LBB0_17
 ; CHECK-NEXT:  .LBB0_2: # %bb1
 ; CHECK-NEXT:    lfd 0, 400(1)
 ; CHECK-NEXT:    lis 3, 15856
@@ -62,7 +62,7 @@ define i64 @__fixunstfdi(ppc_fp128 %a) nounwind readnone {
 ; CHECK-NEXT:    stw 3, 312(1)
 ; CHECK-NEXT:    lfd 30, 312(1)
 ; CHECK-NEXT:    fmr 2, 30
-; CHECK-NEXT:    bl __gcc_qmul@PLT
+; CHECK-NEXT:    bl __gcc_qmul
 ; CHECK-NEXT:    lis 3, 16864
 ; CHECK-NEXT:    stfd 1, 280(1)
 ; CHECK-NEXT:    stw 3, 368(1)
@@ -84,7 +84,7 @@ define i64 @__fixunstfdi(ppc_fp128 %a) nounwind readnone {
 ; CHECK-NEXT:    lwz 3, 288(1)
 ; CHECK-NEXT:    stw 3, 344(1)
 ; CHECK-NEXT:    lfd 2, 344(1)
-; CHECK-NEXT:    bl __gcc_qsub@PLT
+; CHECK-NEXT:    bl __gcc_qsub
 ; CHECK-NEXT:    mffs 0
 ; CHECK-NEXT:    mtfsb1 31
 ; CHECK-NEXT:    lis 3, .LCPI0_1@ha
@@ -117,7 +117,7 @@ define i64 @__fixunstfdi(ppc_fp128 %a) nounwind readnone {
 ; CHECK-NEXT:  .LBB0_5: # %bb1
 ; CHECK-NEXT:    li 4, 0
 ; CHECK-NEXT:    mr 3, 30
-; CHECK-NEXT:    bl __floatditf@PLT
+; CHECK-NEXT:    bl __floatditf
 ; CHECK-NEXT:    lis 3, 17392
 ; CHECK-NEXT:    stfd 1, 208(1)
 ; CHECK-NEXT:    stw 3, 240(1)
@@ -140,7 +140,7 @@ define i64 @__fixunstfdi(ppc_fp128 %a) nounwind readnone {
 ; CHECK-NEXT:    lwz 3, 200(1)
 ; CHECK-NEXT:    stw 3, 216(1)
 ; CHECK-NEXT:    lfd 2, 216(1)
-; CHECK-NEXT:    bl __gcc_qadd@PLT
+; CHECK-NEXT:    bl __gcc_qadd
 ; CHECK-NEXT:    blt 2, .LBB0_7
 ; CHECK-NEXT:  # %bb.6: # %bb1
 ; CHECK-NEXT:    fmr 2, 28
@@ -163,16 +163,14 @@ define i64 @__fixunstfdi(ppc_fp128 %a) nounwind readnone {
 ; CHECK-NEXT:    lwz 3, 192(1)
 ; CHECK-NEXT:    stw 3, 248(1)
 ; CHECK-NEXT:    lfd 4, 248(1)
-; CHECK-NEXT:    bl __gcc_qsub@PLT
+; CHECK-NEXT:    bl __gcc_qsub
 ; CHECK-NEXT:    stfd 2, 176(1)
 ; CHECK-NEXT:    stfd 1, 168(1)
-; CHECK-NEXT:    fcmpu 0, 2, 27
+; CHECK-NEXT:    fcmpu 1, 2, 27
 ; CHECK-NEXT:    lwz 3, 180(1)
-; CHECK-NEXT:    fcmpu 1, 1, 27
-; CHECK-NEXT:    crandc 20, 6, 0
-; CHECK-NEXT:    cror 21, 5, 7
+; CHECK-NEXT:    fcmpu 0, 1, 27
+; CHECK-NEXT:    crandc 20, 2, 4
 ; CHECK-NEXT:    stw 3, 268(1)
-; CHECK-NEXT:    cror 20, 21, 20
 ; CHECK-NEXT:    lwz 3, 176(1)
 ; CHECK-NEXT:    stw 3, 264(1)
 ; CHECK-NEXT:    lwz 3, 172(1)
@@ -181,8 +179,11 @@ define i64 @__fixunstfdi(ppc_fp128 %a) nounwind readnone {
 ; CHECK-NEXT:    lwz 3, 168(1)
 ; CHECK-NEXT:    stw 3, 272(1)
 ; CHECK-NEXT:    lfd 31, 272(1)
-; CHECK-NEXT:    bc 12, 20, .LBB0_13
-; CHECK-NEXT:  # %bb.10: # %bb2
+; CHECK-NEXT:    bc 12, 20, .LBB0_14
+; CHECK-NEXT:  # %bb.10: # %bb1
+; CHECK-NEXT:    cror 20, 1, 3
+; CHECK-NEXT:    bc 12, 20, .LBB0_14
+; CHECK-NEXT:  # %bb.11: # %bb2
 ; CHECK-NEXT:    fneg 28, 31
 ; CHECK-NEXT:    stfd 28, 48(1)
 ; CHECK-NEXT:    lis 3, 16864
@@ -205,7 +206,7 @@ define i64 @__fixunstfdi(ppc_fp128 %a) nounwind readnone {
 ; CHECK-NEXT:    lwz 3, 40(1)
 ; CHECK-NEXT:    stw 3, 56(1)
 ; CHECK-NEXT:    lfd 2, 56(1)
-; CHECK-NEXT:    bl __gcc_qsub@PLT
+; CHECK-NEXT:    bl __gcc_qsub
 ; CHECK-NEXT:    mffs 0
 ; CHECK-NEXT:    mtfsb1 31
 ; CHECK-NEXT:    lis 3, .LCPI0_2@ha
@@ -231,15 +232,15 @@ define i64 @__fixunstfdi(ppc_fp128 %a) nounwind readnone {
 ; CHECK-NEXT:    crandc 20, 6, 1
 ; CHECK-NEXT:    cror 20, 4, 20
 ; CHECK-NEXT:    addis 3, 3, -32768
-; CHECK-NEXT:    bc 12, 20, .LBB0_12
-; CHECK-NEXT:  # %bb.11: # %bb2
+; CHECK-NEXT:    bc 12, 20, .LBB0_13
+; CHECK-NEXT:  # %bb.12: # %bb2
 ; CHECK-NEXT:    ori 3, 4, 0
-; CHECK-NEXT:    b .LBB0_12
-; CHECK-NEXT:  .LBB0_12: # %bb2
+; CHECK-NEXT:    b .LBB0_13
+; CHECK-NEXT:  .LBB0_13: # %bb2
 ; CHECK-NEXT:    subfic 4, 3, 0
 ; CHECK-NEXT:    subfe 3, 29, 30
-; CHECK-NEXT:    b .LBB0_16
-; CHECK-NEXT:  .LBB0_13: # %bb3
+; CHECK-NEXT:    b .LBB0_17
+; CHECK-NEXT:  .LBB0_14: # %bb3
 ; CHECK-NEXT:    stfd 31, 112(1)
 ; CHECK-NEXT:    li 3, 0
 ; CHECK-NEXT:    stw 3, 148(1)
@@ -260,7 +261,7 @@ define i64 @__fixunstfdi(ppc_fp128 %a) nounwind readnone {
 ; CHECK-NEXT:    lwz 3, 104(1)
 ; CHECK-NEXT:    stw 3, 120(1)
 ; CHECK-NEXT:    lfd 2, 120(1)
-; CHECK-NEXT:    bl __gcc_qsub@PLT
+; CHECK-NEXT:    bl __gcc_qsub
 ; CHECK-NEXT:    mffs 0
 ; CHECK-NEXT:    mtfsb1 31
 ; CHECK-NEXT:    lis 3, .LCPI0_0@ha
@@ -286,13 +287,13 @@ define i64 @__fixunstfdi(ppc_fp128 %a) nounwind readnone {
 ; CHECK-NEXT:    crandc 20, 6, 0
 ; CHECK-NEXT:    cror 20, 5, 20
 ; CHECK-NEXT:    addis 3, 3, -32768
-; CHECK-NEXT:    bc 12, 20, .LBB0_14
-; CHECK-NEXT:    b .LBB0_15
-; CHECK-NEXT:  .LBB0_14: # %bb3
-; CHECK-NEXT:    addi 4, 3, 0
+; CHECK-NEXT:    bc 12, 20, .LBB0_15
+; CHECK-NEXT:    b .LBB0_16
 ; CHECK-NEXT:  .LBB0_15: # %bb3
+; CHECK-NEXT:    addi 4, 3, 0
+; CHECK-NEXT:  .LBB0_16: # %bb3
 ; CHECK-NEXT:    mr 3, 30
-; CHECK-NEXT:  .LBB0_16: # %bb5
+; CHECK-NEXT:  .LBB0_17: # %bb5
 ; CHECK-NEXT:    lfd 31, 456(1) # 8-byte Folded Reload
 ; CHECK-NEXT:    lfd 30, 448(1) # 8-byte Folded Reload
 ; CHECK-NEXT:    lfd 29, 440(1) # 8-byte Folded Reload
diff --git a/llvm/test/CodeGen/PowerPC/2010-02-12-saveCR.ll b/llvm/test/CodeGen/PowerPC/2010-02-12-saveCR.ll
index 995f6bd23..238af20a2 100644
--- a/llvm/test/CodeGen/PowerPC/2010-02-12-saveCR.ll
+++ b/llvm/test/CodeGen/PowerPC/2010-02-12-saveCR.ll
@@ -11,7 +11,7 @@ entry:
 ; CHECK-DAG: ori [[T2:[0-9]+]], [[T2]], 34492
 ; CHECK-DAG: stwx [[T1]], 1, [[T2]]
 ; CHECK-DAG: addi 3, 1, 28
-; CHECK: bl bar@PLT
+; CHECK: bl bar
   %x = alloca [100000 x i8]                       ; <[100000 x i8]*> [#uses=1]
   %"alloca point" = bitcast i32 0 to i32          ; <i32> [#uses=0]
   %x1 = bitcast [100000 x i8]* %x to i8*          ; <i8*> [#uses=1]
diff --git a/llvm/test/CodeGen/PowerPC/CSR-fit.ll b/llvm/test/CodeGen/PowerPC/CSR-fit.ll
index 20438a45a..ebaffdb05 100644
--- a/llvm/test/CodeGen/PowerPC/CSR-fit.ll
+++ b/llvm/test/CodeGen/PowerPC/CSR-fit.ll
@@ -27,9 +27,9 @@ define dso_local signext i32 @caller1(i32 signext %a, i32 signext %b) local_unna
 ; CHECK-PWR8-NEXT:    nop
 ; CHECK-PWR8-NEXT:    addi r1, r1, 176
 ; CHECK-PWR8-NEXT:    ld r0, 16(r1)
-; CHECK-PWR8-NEXT:    mtlr r0
 ; CHECK-PWR8-NEXT:    ld r15, -136(r1) # 8-byte Folded Reload
 ; CHECK-PWR8-NEXT:    ld r14, -144(r1) # 8-byte Folded Reload
+; CHECK-PWR8-NEXT:    mtlr r0
 ; CHECK-PWR8-NEXT:    blr
 ;
 ; CHECK-PWR9-LABEL: caller1:
@@ -51,9 +51,9 @@ define dso_local signext i32 @caller1(i32 signext %a, i32 signext %b) local_unna
 ; CHECK-PWR9-NEXT:    nop
 ; CHECK-PWR9-NEXT:    addi r1, r1, 176
 ; CHECK-PWR9-NEXT:    ld r0, 16(r1)
-; CHECK-PWR9-NEXT:    mtlr r0
 ; CHECK-PWR9-NEXT:    ld r15, -136(r1) # 8-byte Folded Reload
 ; CHECK-PWR9-NEXT:    ld r14, -144(r1) # 8-byte Folded Reload
+; CHECK-PWR9-NEXT:    mtlr r0
 ; CHECK-PWR9-NEXT:    blr
 entry:
   %0 = tail call i32 asm "add $0, $1, $2", "=r,r,r,~{r14},~{r15}"(i32 %a, i32 %b)
@@ -81,9 +81,9 @@ define dso_local signext i32 @caller2(i32 signext %a, i32 signext %b) local_unna
 ; CHECK-PWR8-NEXT:    nop
 ; CHECK-PWR8-NEXT:    addi r1, r1, 176
 ; CHECK-PWR8-NEXT:    ld r0, 16(r1)
-; CHECK-PWR8-NEXT:    mtlr r0
 ; CHECK-PWR8-NEXT:    lfd f15, -136(r1) # 8-byte Folded Reload
 ; CHECK-PWR8-NEXT:    lfd f14, -144(r1) # 8-byte Folded Reload
+; CHECK-PWR8-NEXT:    mtlr r0
 ; CHECK-PWR8-NEXT:    blr
 ;
 ; CHECK-PWR9-LABEL: caller2:
@@ -105,9 +105,9 @@ define dso_local signext i32 @caller2(i32 signext %a, i32 signext %b) local_unna
 ; CHECK-PWR9-NEXT:    nop
 ; CHECK-PWR9-NEXT:    addi r1, r1, 176
 ; CHECK-PWR9-NEXT:    ld r0, 16(r1)
-; CHECK-PWR9-NEXT:    mtlr r0
 ; CHECK-PWR9-NEXT:    lfd f15, -136(r1) # 8-byte Folded Reload
 ; CHECK-PWR9-NEXT:    lfd f14, -144(r1) # 8-byte Folded Reload
+; CHECK-PWR9-NEXT:    mtlr r0
 ; CHECK-PWR9-NEXT:    blr
 entry:
   %0 = tail call i32 asm "add $0, $1, $2", "=r,r,r,~{f14},~{f15}"(i32 %a, i32 %b)
@@ -178,16 +178,26 @@ define dso_local signext i32 @caller4(i32 signext %a, i32 signext %b) local_unna
 ; CHECK-PWR8:       # %bb.0: # %entry
 ; CHECK-PWR8-NEXT:    mflr r0
 ; CHECK-PWR8-NEXT:    std r0, 16(r1)
-; CHECK-PWR8-NEXT:    stdu r1, -32(r1)
-; CHECK-PWR8-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-PWR8-NEXT:    stdu r1, -240(r1)
+; CHECK-PWR8-NEXT:    .cfi_def_cfa_offset 240
 ; CHECK-PWR8-NEXT:    .cfi_offset lr, 16
+; CHECK-PWR8-NEXT:    .cfi_offset v20, -192
+; CHECK-PWR8-NEXT:    .cfi_offset v21, -176
+; CHECK-PWR8-NEXT:    li r5, 48
+; CHECK-PWR8-NEXT:    stxvd2x v20, r1, r5 # 16-byte Folded Spill
+; CHECK-PWR8-NEXT:    li r5, 64
+; CHECK-PWR8-NEXT:    stxvd2x v21, r1, r5 # 16-byte Folded Spill
 ; CHECK-PWR8-NEXT:    #APP
 ; CHECK-PWR8-NEXT:    add r3, r3, r4
 ; CHECK-PWR8-NEXT:    #NO_APP
 ; CHECK-PWR8-NEXT:    extsw r3, r3
 ; CHECK-PWR8-NEXT:    bl callee
 ; CHECK-PWR8-NEXT:    nop
-; CHECK-PWR8-NEXT:    addi r1, r1, 32
+; CHECK-PWR8-NEXT:    li r4, 64
+; CHECK-PWR8-NEXT:    lxvd2x v21, r1, r4 # 16-byte Folded Reload
+; CHECK-PWR8-NEXT:    li r4, 48
+; CHECK-PWR8-NEXT:    lxvd2x v20, r1, r4 # 16-byte Folded Reload
+; CHECK-PWR8-NEXT:    addi r1, r1, 240
 ; CHECK-PWR8-NEXT:    ld r0, 16(r1)
 ; CHECK-PWR8-NEXT:    mtlr r0
 ; CHECK-PWR8-NEXT:    blr
@@ -196,16 +206,22 @@ define dso_local signext i32 @caller4(i32 signext %a, i32 signext %b) local_unna
 ; CHECK-PWR9:       # %bb.0: # %entry
 ; CHECK-PWR9-NEXT:    mflr r0
 ; CHECK-PWR9-NEXT:    std r0, 16(r1)
-; CHECK-PWR9-NEXT:    stdu r1, -32(r1)
-; CHECK-PWR9-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-PWR9-NEXT:    stdu r1, -224(r1)
+; CHECK-PWR9-NEXT:    .cfi_def_cfa_offset 224
 ; CHECK-PWR9-NEXT:    .cfi_offset lr, 16
+; CHECK-PWR9-NEXT:    .cfi_offset v20, -192
+; CHECK-PWR9-NEXT:    .cfi_offset v21, -176
+; CHECK-PWR9-NEXT:    stxv v20, 32(r1) # 16-byte Folded Spill
+; CHECK-PWR9-NEXT:    stxv v21, 48(r1) # 16-byte Folded Spill
 ; CHECK-PWR9-NEXT:    #APP
 ; CHECK-PWR9-NEXT:    add r3, r3, r4
 ; CHECK-PWR9-NEXT:    #NO_APP
 ; CHECK-PWR9-NEXT:    extsw r3, r3
 ; CHECK-PWR9-NEXT:    bl callee
 ; CHECK-PWR9-NEXT:    nop
-; CHECK-PWR9-NEXT:    addi r1, r1, 32
+; CHECK-PWR9-NEXT:    lxv v21, 48(r1) # 16-byte Folded Reload
+; CHECK-PWR9-NEXT:    lxv v20, 32(r1) # 16-byte Folded Reload
+; CHECK-PWR9-NEXT:    addi r1, r1, 224
 ; CHECK-PWR9-NEXT:    ld r0, 16(r1)
 ; CHECK-PWR9-NEXT:    mtlr r0
 ; CHECK-PWR9-NEXT:    blr
@@ -226,19 +242,24 @@ define dso_local signext i32 @caller_mixed(i32 signext %a, i32 signext %b) local
 ; CHECK-PWR8-NEXT:    .cfi_offset r14, -288
 ; CHECK-PWR8-NEXT:    .cfi_offset f14, -144
 ; CHECK-PWR8-NEXT:    .cfi_offset v20, -480
+; CHECK-PWR8-NEXT:    .cfi_offset v21, -464
 ; CHECK-PWR8-NEXT:    li r5, 48
 ; CHECK-PWR8-NEXT:    std r14, 240(r1) # 8-byte Folded Spill
 ; CHECK-PWR8-NEXT:    stfd f14, 384(r1) # 8-byte Folded Spill
 ; CHECK-PWR8-NEXT:    stxvd2x v20, r1, r5 # 16-byte Folded Spill
+; CHECK-PWR8-NEXT:    li r5, 64
+; CHECK-PWR8-NEXT:    stxvd2x v21, r1, r5 # 16-byte Folded Spill
 ; CHECK-PWR8-NEXT:    #APP
 ; CHECK-PWR8-NEXT:    add r3, r3, r4
 ; CHECK-PWR8-NEXT:    #NO_APP
 ; CHECK-PWR8-NEXT:    extsw r3, r3
 ; CHECK-PWR8-NEXT:    bl callee
 ; CHECK-PWR8-NEXT:    nop
-; CHECK-PWR8-NEXT:    li r4, 48
+; CHECK-PWR8-NEXT:    li r4, 64
 ; CHECK-PWR8-NEXT:    lfd f14, 384(r1) # 8-byte Folded Reload
 ; CHECK-PWR8-NEXT:    ld r14, 240(r1) # 8-byte Folded Reload
+; CHECK-PWR8-NEXT:    lxvd2x v21, r1, r4 # 16-byte Folded Reload
+; CHECK-PWR8-NEXT:    li r4, 48
 ; CHECK-PWR8-NEXT:    lxvd2x v20, r1, r4 # 16-byte Folded Reload
 ; CHECK-PWR8-NEXT:    addi r1, r1, 528
 ; CHECK-PWR8-NEXT:    ld r0, 16(r1)
@@ -255,15 +276,18 @@ define dso_local signext i32 @caller_mixed(i32 signext %a, i32 signext %b) local
 ; CHECK-PWR9-NEXT:    .cfi_offset r14, -288
 ; CHECK-PWR9-NEXT:    .cfi_offset f14, -144
 ; CHECK-PWR9-NEXT:    .cfi_offset v20, -480
+; CHECK-PWR9-NEXT:    .cfi_offset v21, -464
 ; CHECK-PWR9-NEXT:    std r14, 224(r1) # 8-byte Folded Spill
 ; CHECK-PWR9-NEXT:    stfd f14, 368(r1) # 8-byte Folded Spill
 ; CHECK-PWR9-NEXT:    stxv v20, 32(r1) # 16-byte Folded Spill
+; CHECK-PWR9-NEXT:    stxv v21, 48(r1) # 16-byte Folded Spill
 ; CHECK-PWR9-NEXT:    #APP
 ; CHECK-PWR9-NEXT:    add r3, r3, r4
 ; CHECK-PWR9-NEXT:    #NO_APP
 ; CHECK-PWR9-NEXT:    extsw r3, r3
 ; CHECK-PWR9-NEXT:    bl callee
 ; CHECK-PWR9-NEXT:    nop
+; CHECK-PWR9-NEXT:    lxv v21, 48(r1) # 16-byte Folded Reload
 ; CHECK-PWR9-NEXT:    lxv v20, 32(r1) # 16-byte Folded Reload
 ; CHECK-PWR9-NEXT:    lfd f14, 368(r1) # 8-byte Folded Reload
 ; CHECK-PWR9-NEXT:    ld r14, 224(r1) # 8-byte Folded Reload
diff --git a/llvm/test/CodeGen/PowerPC/CompareEliminationSpillIssue.ll b/llvm/test/CodeGen/PowerPC/CompareEliminationSpillIssue.ll
index 75cac9030..10a5e0d43 100644
--- a/llvm/test/CodeGen/PowerPC/CompareEliminationSpillIssue.ll
+++ b/llvm/test/CodeGen/PowerPC/CompareEliminationSpillIssue.ll
@@ -9,7 +9,7 @@
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
-@glob = common local_unnamed_addr global i64 0, align 8
+@glob = local_unnamed_addr global i64 0, align 8
 @.str = private unnamed_addr constant [12 x i8] c"Value = %d\0A\00", align 1
 
 ; Function Attrs: noinline nounwind
diff --git a/llvm/test/CodeGen/PowerPC/MCSE-caller-preserved-reg.ll b/llvm/test/CodeGen/PowerPC/MCSE-caller-preserved-reg.ll
index 05e6d7b56..c2a3d1752 100644
--- a/llvm/test/CodeGen/PowerPC/MCSE-caller-preserved-reg.ll
+++ b/llvm/test/CodeGen/PowerPC/MCSE-caller-preserved-reg.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -relocation-model=pic -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu < %s | FileCheck %s
 ; The instructions addis,addi, bl are used to calculate the address of TLS
 ; thread local variables. These TLS access code sequences are generated
@@ -12,13 +13,11 @@
 
 @_ZN2CC2ccE = external thread_local global %"struct.CC::TT", align 8
 
-define noalias i8* @_ZN2CC3funEv(%class.CC* %this) {
+define noalias i8* @_ZN2CC3funEv(%class.CC* %this) nounwind {
 ; CHECK-LABEL: _ZN2CC3funEv:
-; CHECK:    mflr 0
-; CHECK-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-NEXT:    .cfi_offset lr, 16
-; CHECK-NEXT:    .cfi_offset r30, -16
-; CHECK-NEXT:    std 30, -16(1)
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    mflr 0
+; CHECK-NEXT:    std 30, -16(1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std 0, 16(1)
 ; CHECK-NEXT:    stdu 1, -48(1)
 ; CHECK-NEXT:    std 2, 24(1)
@@ -34,15 +33,17 @@ define noalias i8* @_ZN2CC3funEv(%class.CC* %this) {
 ; CHECK-NEXT:    ld 4, 0(3)
 ; CHECK-NEXT:    cmpldi 4, 0
 ; CHECK-NEXT:    beq 0, .LBB0_2
-; CHECK:    addi 4, 3, 8
+; CHECK-NEXT:  # %bb.1: # %if.then
+; CHECK-NEXT:    addi 4, 3, 8
 ; CHECK-NEXT:    mr 3, 30
 ; CHECK-NEXT:    bl _ZN2CC3barEPi
 ; CHECK-NEXT:    nop
-; CHECK:    li 3, 0
+; CHECK-NEXT:  .LBB0_2: # %if.end
+; CHECK-NEXT:    li 3, 0
 ; CHECK-NEXT:    addi 1, 1, 48
 ; CHECK-NEXT:    ld 0, 16(1)
+; CHECK-NEXT:    ld 30, -16(1) # 8-byte Folded Reload
 ; CHECK-NEXT:    mtlr 0
-; CHECK:    ld 30, -16(1)
 ; CHECK-NEXT:    blr
 entry:
   %foo = getelementptr inbounds %class.CC, %class.CC* %this, i64 0, i32 0, i32 0
diff --git a/llvm/test/CodeGen/PowerPC/PR35812-neg-cmpxchg.ll b/llvm/test/CodeGen/PowerPC/PR35812-neg-cmpxchg.ll
index 5ecce5ed4..9db7ec632 100644
--- a/llvm/test/CodeGen/PowerPC/PR35812-neg-cmpxchg.ll
+++ b/llvm/test/CodeGen/PowerPC/PR35812-neg-cmpxchg.ll
@@ -8,14 +8,12 @@
 @str.1 = private unnamed_addr constant [59 x i8] c"FAILED: __atomic_compare_exchange_n() set the wrong value.\00"
 @str.2 = private unnamed_addr constant [7 x i8] c"PASSED\00"
 
-define signext i32 @main() {
+define signext i32 @main() nounwind {
 ; CHECK-LABEL: main:
 ; CHECK:       # %bb.0: # %L.entry
 ; CHECK-NEXT:    mflr 0
 ; CHECK-NEXT:    std 0, 16(1)
 ; CHECK-NEXT:    stdu 1, -48(1)
-; CHECK-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-NEXT:    .cfi_offset lr, 16
 ; CHECK-NEXT:    li 3, -32477
 ; CHECK-NEXT:    li 6, 234
 ; CHECK-NEXT:    addi 5, 1, 46
@@ -72,8 +70,6 @@ define signext i32 @main() {
 ; CHECK-P7-NEXT:    mflr 0
 ; CHECK-P7-NEXT:    std 0, 16(1)
 ; CHECK-P7-NEXT:    stdu 1, -48(1)
-; CHECK-P7-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-P7-NEXT:    .cfi_offset lr, 16
 ; CHECK-P7-NEXT:    li 3, -32477
 ; CHECK-P7-NEXT:    lis 5, 0
 ; CHECK-P7-NEXT:    addi 4, 1, 46
diff --git a/llvm/test/CodeGen/PowerPC/VSX-XForm-Scalars.ll b/llvm/test/CodeGen/PowerPC/VSX-XForm-Scalars.ll
index 937264f98..06b93420a 100644
--- a/llvm/test/CodeGen/PowerPC/VSX-XForm-Scalars.ll
+++ b/llvm/test/CodeGen/PowerPC/VSX-XForm-Scalars.ll
@@ -27,18 +27,16 @@ define void @testExpandPostRAPseudo(i32* nocapture readonly %ptr) {
 ;
 ; CHECK-P9-LABEL: testExpandPostRAPseudo:
 ; CHECK-P9:  # %bb.0: # %entry
-; CHECK-P9:    lfiwzx f0, 0, r3
 ; CHECK-P9:    addis r4, r2, .LC0@toc@ha
+; CHECK-P9:    lxvwsx vs0, 0, r3
 ; CHECK-P9:    ld r4, .LC0@toc@l(r4)
-; CHECK-P9:    xxpermdi vs0, f0, f0, 2
-; CHECK-P9:    xxspltw vs0, vs0, 3
 ; CHECK-P9:    stxvx vs0, 0, r4
+; CHECK-P9:    lis r4, 1024
 ; CHECK-P9:    lfiwax f0, 0, r3
 ; CHECK-P9:    addis r3, r2, .LC1@toc@ha
 ; CHECK-P9:    ld r3, .LC1@toc@l(r3)
 ; CHECK-P9:    xscvsxdsp f0, f0
 ; CHECK-P9:    ld r3, 0(r3)
-; CHECK-P9:    lis r4, 1024
 ; CHECK-P9:    stfsx f0, r3, r4
 ; CHECK-P9:    blr
 entry:
diff --git a/llvm/test/CodeGen/PowerPC/aantidep-inline-asm-use.ll b/llvm/test/CodeGen/PowerPC/aantidep-inline-asm-use.ll
index 2e1a4cec2..1b0ea26f1 100644
--- a/llvm/test/CodeGen/PowerPC/aantidep-inline-asm-use.ll
+++ b/llvm/test/CodeGen/PowerPC/aantidep-inline-asm-use.ll
@@ -298,7 +298,7 @@ _ZN10SubProcess12SafeSyscalls5fcntlEiil.exit:     ; preds = %_ZN10SubProcess12Sa
 ; Function Attrs: nounwind argmemonly
 declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i1) #1
 
-attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pwr8" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+vsx,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pwr8" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+vsx,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind argmemonly }
 attributes #2 = { nounwind }
 
diff --git a/llvm/test/CodeGen/PowerPC/absol-jump-table-enabled.ll b/llvm/test/CodeGen/PowerPC/absol-jump-table-enabled.ll
new file mode 100644
index 000000000..4a9492d8e
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/absol-jump-table-enabled.ll
@@ -0,0 +1,100 @@
+; NOTE: This test case generates a jump table on PowerPC big and little endian
+; NOTE: then verifies that the command line option to enable absolute jump
+; NOTE: table works correctly.
+; RUN:  llc -mtriple=powerpc64le-unknown-linux-gnu -o - \
+; RUN:      -ppc-use-absolute-jumptables -ppc-asm-full-reg-names \
+; RUN:      -verify-machineinstrs %s | FileCheck %s -check-prefix=CHECK-LE
+; RUN:  llc -mtriple=powerpc64-unknown-linux-gnu -o - \
+; RUN:      -ppc-use-absolute-jumptables -ppc-asm-full-reg-names \
+; RUN:      -verify-machineinstrs %s | FileCheck %s -check-prefix=CHECK-BE
+
+%struct.node = type { i8, %struct.node* }
+
+; Function Attrs: norecurse nounwind readonly
+define zeroext i32 @jumpTableTest(%struct.node* readonly %list) {
+; CHECK-LE-LABEL: jumpTableTest:
+; CHECK-LE:       # %bb.0: # %entry
+; CHECK-LE:       rldic r[[REG:[0-9]+]], r[[REG]], 3, 29
+; CHECK-LE:       ldx r[[REG]], r[[REG]], r[[REG1:[0-9]+]]
+; CHECK-LE:       mtctr r[[REG]]
+; CHECK-LE:       bctr
+; CHECK-LE:       blr
+;
+; CHECK-BE-LABEL: jumpTableTest:
+; CHECK-BE:       # %bb.0: # %entry
+; CHECK-BE:       rldic r[[REG:[0-9]+]], r[[REG]], 2, 30
+; CHECK-BE:       lwax r[[REG]], r[[REG]], r[[REG1:[0-9]+]]
+; CHECK-BE:       mtctr r[[REG]]
+; CHECK-BE:       bctr
+; CHECK-BE:       blr
+entry:
+  %cmp36 = icmp eq %struct.node* %list, null
+  br i1 %cmp36, label %while.end, label %while.body
+
+while.body:                                       ; preds = %entry, %sw.epilog
+  %result.038 = phi i32 [ %result.1, %sw.epilog ], [ 0, %entry ]
+  %current.037 = phi %struct.node* [ %spec.store.select, %sw.epilog ], [ %list, %entry ]
+  %next1 = getelementptr inbounds %struct.node, %struct.node* %current.037, i64 0, i32 1
+  %0 = load %struct.node*, %struct.node** %next1, align 8
+  %cmp2 = icmp eq %struct.node* %0, %current.037
+  %spec.store.select = select i1 %cmp2, %struct.node* null, %struct.node* %0
+  %type = getelementptr inbounds %struct.node, %struct.node* %current.037, i64 0, i32 0
+  %1 = load i8, i8* %type, align 8
+  switch i8 %1, label %sw.epilog [
+    i8 1, label %sw.bb
+    i8 2, label %sw.bb3
+    i8 3, label %sw.bb5
+    i8 4, label %sw.bb7
+    i8 5, label %sw.bb9
+    i8 6, label %sw.bb11
+    i8 7, label %sw.bb13
+    i8 8, label %sw.bb15
+    i8 9, label %sw.bb17
+  ]
+
+sw.bb:                                            ; preds = %while.body
+  %add = add nsw i32 %result.038, 13
+  br label %sw.epilog
+
+sw.bb3:                                           ; preds = %while.body
+  %add4 = add nsw i32 %result.038, 5
+  br label %sw.epilog
+
+sw.bb5:                                           ; preds = %while.body
+  %add6 = add nsw i32 %result.038, 2
+  br label %sw.epilog
+
+sw.bb7:                                           ; preds = %while.body
+  %add8 = add nsw i32 %result.038, 7
+  br label %sw.epilog
+
+sw.bb9:                                           ; preds = %while.body
+  %add10 = add nsw i32 %result.038, 11
+  br label %sw.epilog
+
+sw.bb11:                                          ; preds = %while.body
+  %add12 = add nsw i32 %result.038, 17
+  br label %sw.epilog
+
+sw.bb13:                                          ; preds = %while.body
+  %add14 = add nsw i32 %result.038, 16
+  br label %sw.epilog
+
+sw.bb15:                                          ; preds = %while.body
+  %add16 = add nsw i32 %result.038, 81
+  br label %sw.epilog
+
+sw.bb17:                                          ; preds = %while.body
+  %add18 = add nsw i32 %result.038, 72
+  br label %sw.epilog
+
+sw.epilog:                                        ; preds = %while.body, %sw.bb17, %sw.bb15, %sw.bb13, %sw.bb11, %sw.bb9, %sw.bb7, %sw.bb5, %sw.bb3, %sw.bb
+  %result.1 = phi i32 [ %result.038, %while.body ], [ %add18, %sw.bb17 ], [ %add16, %sw.bb15 ], [ %add14, %sw.bb13 ], [ %add12, %sw.bb11 ], [ %add10, %sw.bb9 ], [ %add8, %sw.bb7 ], [ %add6, %sw.bb5 ], [ %add4, %sw.bb3 ], [ %add, %sw.bb ]
+  %cmp = icmp eq %struct.node* %spec.store.select, null
+  br i1 %cmp, label %while.end, label %while.body
+
+while.end:                                        ; preds = %sw.epilog, %entry
+  %result.0.lcssa = phi i32 [ 0, %entry ], [ %result.1, %sw.epilog ]
+  ret i32 %result.0.lcssa
+}
+
diff --git a/llvm/test/CodeGen/PowerPC/add_cmp.ll b/llvm/test/CodeGen/PowerPC/add_cmp.ll
index d23eeba59..cbe16a498 100644
--- a/llvm/test/CodeGen/PowerPC/add_cmp.ll
+++ b/llvm/test/CodeGen/PowerPC/add_cmp.ll
@@ -2,75 +2,55 @@
 ; RUN: llc -mcpu=pwr9 -mtriple=powerpc64le-unknown-linux-gnu < %s \
 ; RUN:   -verify-machineinstrs -debug 2>&1 | FileCheck %s
 
-define zeroext i1 @addiCmpiUnsign(i32 zeroext %x) {
+define zeroext i1 @addiCmpiUnsigned(i32 zeroext %x) {
 entry:
   %add = add nuw i32 10, %x 
   %cmp = icmp ugt i32 %add, 100
   ret i1 %cmp
 
-; CHECK: === addiCmpiUnsign
-; CHECK: Optimized lowered selection DAG: %bb.0 'addiCmpiUnsign:entry'
+; CHECK: === addiCmpiUnsigned
+; CHECK: Optimized lowered selection DAG: %bb.0 'addiCmpiUnsigned:entry'
 ; CHECK:   [[REG1:t[0-9]+]]: i32 = truncate {{t[0-9]+}}
 ; CHECK:   [[REG2:t[0-9]+]]: i32 = add nuw [[REG1]], Constant:i32<10>
 ; CHECK:   {{t[0-9]+}}: i1 = setcc [[REG2]], Constant:i32<100>, setugt:ch
 }
 
-define zeroext i1 @addiCmpiSign(i32 signext %x) {
+define zeroext i1 @addiCmpiSigned(i32 signext %x) {
 entry:
   %add = add nsw i32 16, %x 
   %cmp = icmp sgt i32 %add, 30
   ret i1 %cmp
 
-; CHECK: === addiCmpiSign
-; CHECK: Optimized lowered selection DAG: %bb.0 'addiCmpiSign:entry'
+; CHECK: === addiCmpiSigned
+; CHECK: Optimized lowered selection DAG: %bb.0 'addiCmpiSigned:entry'
 ; CHECK:   [[REG1:t[0-9]+]]: i32 = truncate {{t[0-9]+}}
 ; CHECK:   [[REG2:t[0-9]+]]: i32 = add nsw [[REG1]], Constant:i32<16>
 ; CHECK:   {{t[0-9]+}}: i1 = setcc [[REG2]], Constant:i32<30>, setgt:ch
 }
 
-define zeroext i1 @addiCmpiUnsignOverflow(i32 zeroext %x) {
+define zeroext i1 @addiCmpiUnsignedOverflow(i32 zeroext %x) {
 entry:
   %add = add nuw i32 110, %x 
   %cmp = icmp ugt i32 %add, 100
   ret i1 %cmp
 
-; CHECK: === addiCmpiUnsignOverflow
-; CHECK: Optimized lowered selection DAG: %bb.0 'addiCmpiUnsignOverflow:entry'
+; CHECK: === addiCmpiUnsignedOverflow
+; CHECK: Optimized lowered selection DAG: %bb.0 'addiCmpiUnsignedOverflow:entry'
 ; CHECK:   [[REG1:t[0-9]+]]: i32 = truncate {{t[0-9]+}}
 ; CHECK:   [[REG2:t[0-9]+]]: i32 = add nuw [[REG1]], Constant:i32<110>
 ; CHECK:   {{t[0-9]+}}: i1 = setcc [[REG2]], Constant:i32<100>, setugt:ch
 }
 
-define zeroext i1 @addiCmpiSignOverflow(i16 signext %x) {
+define zeroext i1 @addiCmpiSignedOverflow(i16 signext %x) {
 entry:
   %add = add nsw i16 16, %x 
   %cmp = icmp sgt i16 %add, -32767
   ret i1 %cmp
 
-; CHECK: === addiCmpiSignOverflow
-; CHECK: Optimized lowered selection DAG: %bb.0 'addiCmpiSignOverflow:entry'
+; CHECK: === addiCmpiSignedOverflow
+; CHECK: Optimized lowered selection DAG: %bb.0 'addiCmpiSignedOverflow:entry'
 ; CHECK:   [[REG1:t[0-9]+]]: i16 = truncate {{t[0-9]+}}
 ; CHECK:   [[REG2:t[0-9]+]]: i16 = add nsw [[REG1]], Constant:i16<16>
 ; CHECK:   {{t[0-9]+}}: i1 = setcc [[REG2]], Constant:i16<-32767>, setgt:ch
 }
 
-define zeroext i1 @addiCmpiNE(i16* %d) {
-entry:
-  %0 = load i16, i16* %d, align 2
-  %dec = add i16 %0, 100
-  store i16 %dec, i16* %d, align 2
-  %tobool = icmp eq i16 %dec, 40
-  br i1 %tobool, label %land.end, label %land.rhs
-
-land.rhs:       
-  ret i1 true
-
-land.end:      
-  ret i1 false
-
-; CHECK: === addiCmpiNE
-; CHECK: Optimized lowered selection DAG: %bb.0 'addiCmpiNE:entry'
-; CHECK:   [[REG1:t[0-9]+]]: i16,ch = load
-; CHECK:   [[REG2:t[0-9]+]]: i16 = add [[REG1]], Constant:i16<100>
-; CHECK:   {{t[0-9]+}}: i1 = setcc [[REG2]], Constant:i16<40>, seteq:ch
-}
diff --git a/llvm/test/CodeGen/PowerPC/addi-licm.ll b/llvm/test/CodeGen/PowerPC/addi-licm.ll
index e0314d19b..24c9805f1 100644
--- a/llvm/test/CodeGen/PowerPC/addi-licm.ll
+++ b/llvm/test/CodeGen/PowerPC/addi-licm.ll
@@ -1,4 +1,4 @@
-; RUN: llc -verify-machineinstrs -mcpu=pwr7 -disable-ppc-preinc-prep < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -mcpu=pwr7 -disable-ppc-instr-form-prep < %s | FileCheck %s
 ; RUN: llc -verify-machineinstrs -mcpu=pwr7 < %s | FileCheck %s -check-prefix=PIP
 target datalayout = "E-m:e-i64:64-n32:64"
 target triple = "powerpc64-unknown-linux-gnu"
diff --git a/llvm/test/CodeGen/PowerPC/addi-offset-fold.ll b/llvm/test/CodeGen/PowerPC/addi-offset-fold.ll
index db2fb0eee..22db32669 100644
--- a/llvm/test/CodeGen/PowerPC/addi-offset-fold.ll
+++ b/llvm/test/CodeGen/PowerPC/addi-offset-fold.ll
@@ -33,5 +33,5 @@ entry:
 ; CHECK: blr
 }
 
-attributes #0 = { nounwind "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "target-cpu"="ppc64le" }
+attributes #0 = { nounwind "frame-pointer"="all" "target-cpu"="ppc64le" }
 
diff --git a/llvm/test/CodeGen/PowerPC/addrfuncstr.ll b/llvm/test/CodeGen/PowerPC/addrfuncstr.ll
index 991964349..b9ac5f907 100644
--- a/llvm/test/CodeGen/PowerPC/addrfuncstr.ll
+++ b/llvm/test/CodeGen/PowerPC/addrfuncstr.ll
@@ -23,5 +23,5 @@ declare i64 @fread(i8*, i64, i64, %struct._IO_FILE*) #1
 ; CHECK: .section .data.rel.ro
 ; CHECK: .quad fread
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/PowerPC/aix-byval-param.ll b/llvm/test/CodeGen/PowerPC/aix-byval-param.ll
new file mode 100644
index 000000000..951475438
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/aix-byval-param.ll
@@ -0,0 +1,16 @@
+; RUN: not llc -mtriple powerpc-ibm-aix-xcoff < %s 2>&1 | FileCheck %s
+; RUN: not llc -mtriple powerpc64-ibm-aix-xcoff < %s 2>&1 | FileCheck %s
+
+%struct.S = type { i32, i32 }
+
+define void @bar() {
+entry:
+  %s1 = alloca %struct.S, align 4
+  %agg.tmp = alloca %struct.S, align 4
+  call void @foo(%struct.S* byval(%struct.S) align 4 %agg.tmp)
+  ret void
+}
+
+declare void @foo(%struct.S* byval(%struct.S) align 4)
+
+; CHECK: LLVM ERROR: Passing structure by value is unimplemented.
diff --git a/llvm/test/CodeGen/PowerPC/aix-cc-abi.ll b/llvm/test/CodeGen/PowerPC/aix-cc-abi.ll
new file mode 100644
index 000000000..7ef16d77a
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/aix-cc-abi.ll
@@ -0,0 +1,880 @@
+; RUN: llc -mtriple powerpc-ibm-aix-xcoff -stop-after=machine-cp -verify-machineinstrs < %s | \
+; RUN: FileCheck --check-prefixes=CHECK,32BIT %s
+
+; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mattr=-altivec \
+; RUN:  -mtriple powerpc-ibm-aix-xcoff < %s | \
+; RUN: FileCheck --check-prefixes=CHECKASM,ASM32PWR4 %s
+
+; RUN: llc -mtriple powerpc64-ibm-aix-xcoff -stop-after=machine-cp -verify-machineinstrs < %s | \
+; RUN: FileCheck --check-prefixes=CHECK,64BIT %s
+
+; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mattr=-altivec \
+; RUN:  -mtriple powerpc64-ibm-aix-xcoff < %s | \
+; RUN: FileCheck --check-prefixes=CHECKASM,ASM64PWR4 %s
+
+define void @call_test_chars() {
+entry:
+  call i8 @test_chars(i8 signext 97, i8 signext 97, i8 signext 97, i8 signext 97)
+  ret void
+}
+
+; CHECK-LABEL: name: call_test_chars
+
+; 32BIT: ADJCALLSTACKDOWN 56, 0, implicit-def dead $r1, implicit $r1
+; 32BIT: $r3 = LI 97
+; 32BIT: $r4 = LI 97
+; 32BIT: $r5 = LI 97
+; 32BIT: $r6 = LI 97
+; 32BIT: BL_NOP <mcsymbol .test_chars>, csr_aix32, implicit-def dead $lr, implicit $rm, implicit killed $r3, implicit killed $r4, implicit killed $r5, implicit killed $r6, implicit $r2, implicit-def $r1
+; 32BIT: ADJCALLSTACKUP 56, 0, implicit-def dead $r1, implicit $r1
+
+; 64BIT: ADJCALLSTACKDOWN 112, 0, implicit-def dead $r1, implicit $r1
+; 64BIT: $x3 = LI8 97
+; 64BIT: $x4 = LI8 97
+; 64BIT: $x5 = LI8 97
+; 64BIT: $x6 = LI8 97
+; 64BIT: BL8_NOP <mcsymbol .test_chars>, csr_aix64, implicit-def dead $lr8, implicit $rm, implicit killed $x3, implicit killed $x4, implicit killed $x5, implicit killed $x6, implicit $x2, implicit-def $r1
+; 64BIT: ADJCALLSTACKUP 112, 0, implicit-def dead $r1, implicit $r1
+
+define signext i8 @test_chars(i8 signext %c1, i8 signext %c2, i8 signext %c3, i8 signext %c4) {
+entry:
+  %conv = sext i8 %c1 to i32
+  %conv1 = sext i8 %c2 to i32
+  %add = add nsw i32 %conv, %conv1
+  %conv2 = sext i8 %c3 to i32
+  %add3 = add nsw i32 %add, %conv2
+  %conv4 = sext i8 %c4 to i32
+  %add5 = add nsw i32 %add3, %conv4
+  %conv6 = trunc i32 %add5 to i8
+  ret i8 %conv6
+}
+
+; CHECK-LABEL: name: test_chars
+
+; 32BIT:       liveins:
+; 32BIT-NEXT:  - { reg: '$r3', virtual-reg: '' }
+; 32BIT-NEXT:  - { reg: '$r4', virtual-reg: '' }
+; 32BIT-NEXT:  - { reg: '$r5', virtual-reg: '' }
+; 32BIT-NEXT:  - { reg: '$r6', virtual-reg: '' }
+; 32BIT:       body:
+; 32BIT-NEXT:    bb.0.entry:
+; 32BIT-NEXT:      liveins: $r3, $r4, $r5, $r6
+ 
+; 64BIT:       liveins:
+; 64BIT-NEXT:  - { reg: '$x3', virtual-reg: '' }
+; 64BIT-NEXT:  - { reg: '$x4', virtual-reg: '' }
+; 64BIT-NEXT:  - { reg: '$x5', virtual-reg: '' }
+; 64BIT-NEXT:  - { reg: '$x6', virtual-reg: '' }
+; 64BIT:       body:
+; 64BIT-NEXT:    bb.0.entry:
+; 64BIT-NEXT:      liveins: $x3, $x4, $x5, $x6
+
+define void @call_test_chars_mix() {
+entry:
+  call i8 @test_chars_mix(i8 signext 97, i8 zeroext -31, i8 zeroext 97, i8 signext -31)
+  ret void
+}
+
+; CHECK-LABEL: name: call_test_chars_mix
+
+; 32BIT: ADJCALLSTACKDOWN 56, 0, implicit-def dead $r1, implicit $r1
+; 32BIT: $r3 = LI 97
+; 32BIT: $r4 = LI 225
+; 32BIT: $r5 = LI 97
+; 32BIT: $r6 = LI -31
+; 32BIT: BL_NOP <mcsymbol .test_chars_mix>, csr_aix32, implicit-def dead $lr, implicit $rm, implicit killed $r3, implicit killed $r4, implicit killed $r5, implicit killed $r6, implicit $r2, implicit-def $r1
+; 32BIT: ADJCALLSTACKUP 56, 0, implicit-def dead $r1, implicit $r1
+
+; 64BIT: ADJCALLSTACKDOWN 112, 0, implicit-def dead $r1, implicit $r1
+; 64BIT: $x3 = LI8 97
+; 64BIT: $x4 = LI8 225
+; 64BIT: $x5 = LI8 97
+; 64BIT: $x6 = LI8 -31
+; 64BIT: BL8_NOP <mcsymbol .test_chars_mix>, csr_aix64, implicit-def dead $lr8, implicit $rm, implicit killed $x3, implicit killed $x4, implicit killed $x5, implicit killed $x6, implicit $x2, implicit-def $r1
+; 64BIT: ADJCALLSTACKUP 112, 0, implicit-def dead $r1, implicit $r1
+
+define signext i8 @test_chars_mix(i8 signext %c1, i8 zeroext %c2, i8 zeroext %c3, i8 signext %c4) {
+entry:
+  %conv = sext i8 %c1 to i32
+  %conv1 = zext i8 %c2 to i32
+  %add = add nsw i32 %conv, %conv1
+  %conv2 = zext i8 %c3 to i32
+  %add3 = add nsw i32 %add, %conv2
+  %conv4 = sext i8 %c4 to i32
+  %add5 = add nsw i32 %add3, %conv4
+  %conv6 = trunc i32 %add5 to i8
+  ret i8 %conv6
+}
+
+; CHECK-LABEL: name: test_chars_mix
+
+; 32BIT:       liveins:
+; 32BIT-NEXT:  - { reg: '$r3', virtual-reg: '' }
+; 32BIT-NEXT:  - { reg: '$r4', virtual-reg: '' }
+; 32BIT-NEXT:  - { reg: '$r5', virtual-reg: '' }
+; 32BIT-NEXT:  - { reg: '$r6', virtual-reg: '' }
+; 32BIT:       body:
+; 32BIT-NEXT:    bb.0.entry:
+; 32BIT-NEXT:      liveins: $r3, $r4, $r5, $r6
+ 
+; 64BIT:       liveins:
+; 64BIT-NEXT:  - { reg: '$x3', virtual-reg: '' }
+; 64BIT-NEXT:  - { reg: '$x4', virtual-reg: '' }
+; 64BIT-NEXT:  - { reg: '$x5', virtual-reg: '' }
+; 64BIT-NEXT:  - { reg: '$x6', virtual-reg: '' }
+; 64BIT:       body:
+; 64BIT-NEXT:    bb.0.entry:
+; 64BIT-NEXT:      liveins: $x3, $x4, $x5, $x6
+
+@global_i1 = global i8 0, align 1
+
+define  void @test_i1(i1 %b)  {
+  entry:
+   %frombool = zext i1 %b to i8
+   store i8 %frombool, i8* @global_i1, align 1
+   ret void
+}
+
+; 32BIT:       liveins:
+; 32BIT-NEXT:  - { reg: '$r3', virtual-reg: '' }
+; 32BIT:       body:             |
+; 32BIT-NEXT:    bb.0.entry:
+; 32BIT-NEXT:      liveins: $r3
+; 32BIT:           renamable $r3 = RLWINM killed renamable $r3, 0, 31, 31
+; 32BIT-NEXT:      STB killed renamable $r3, 0, killed renamable $r4 :: (store 1 into @global_i1)
+
+; 64BIT:       liveins:
+; 64BIT-NEXT:  - { reg: '$x3', virtual-reg: '' }
+; 64BIT:       body:             |
+; 64BIT-NEXT:    bb.0.entry:
+; 64BIT-NEXT:      liveins: $x3
+; 64BIT:           renamable $r[[REG1:[0-9]+]] = RLWINM renamable $r[[REG1]], 0, 31, 31, implicit killed $x3
+; 64BIT-NEXT:      STB killed renamable $r[[REG1]], 0, killed renamable $x4 :: (store 1 into @global_i1)
+
+define void @call_test_i1() {
+entry:
+  call void @test_i1(i1 1)
+  ret void
+}
+; CHECK-LABEL: name: call_test_i1
+
+; 32BIT: ADJCALLSTACKDOWN 56, 0, implicit-def dead $r1, implicit $r1
+; 32BIT: $r3 = LI 1
+; 32BIT: BL_NOP <mcsymbol .test_i1>, csr_aix32, implicit-def dead $lr, implicit $rm, implicit killed $r3, implicit $r2, implicit-def $r1
+; 32BIT: ADJCALLSTACKUP 56, 0, implicit-def dead $r1, implicit $r1
+
+; 64BIT: ADJCALLSTACKDOWN 112, 0, implicit-def dead $r1, implicit $r1
+; 64BIT: $x3 = LI8 1
+; 64BIT: BL8_NOP <mcsymbol .test_i1>, csr_aix64, implicit-def dead $lr8, implicit $rm, implicit killed $x3, implicit $x2, implicit-def $r1
+; 64BIT: ADJCALLSTACKUP 112, 0, implicit-def dead $r1, implicit $r1
+
+define void @test_i1zext(i1 zeroext %b) {
+  entry:
+    %frombool = zext i1 %b to i8 
+    store i8 %frombool, i8 * @global_i1, align 1 
+    ret void 
+  }
+
+; 32BIT:       liveins:
+; 32BIT-NEXT:  - { reg: '$r3', virtual-reg: '' }
+; 32BIT:       body:             |
+; 32BIT-NEXT:    bb.0.entry:
+; 32BIT-NEXT:      liveins: $r3
+; CHECK-NOT:       RLWINM 
+; 32BIT:           STB killed renamable $r3, 0, killed renamable $r4 :: (store 1 into @global_i1)
+
+; 64BIT:       liveins:
+; 64BIT-NEXT:  - { reg: '$x3', virtual-reg: '' }
+; 64BIT:       body:             |
+; 64BIT-NEXT:    bb.0.entry:
+; 64BIT-NEXT:      liveins: $x3
+; CHECK-NOT:       RLWINM
+; 64BIT:           STB8 killed renamable $x3, 0, killed renamable $x4 :: (store 1 into @global_i1)
+
+define i32 @test_ints(i32 signext %a, i32 zeroext %b, i32 zeroext %c, i32 signext %d, i32 signext %e, i32 signext %f, i32 signext %g, i32 signext %h) {
+entry:
+    %add = add i32 %a, %b
+    %add1 = add i32 %add, %c
+    %add2 = add i32 %add1, %d
+    %add3 = add i32 %add2, %e
+    %add4 = add i32 %add3, %f
+    %add5 = add i32 %add4, %g
+    %add6 = add i32 %add5, %h
+    ret i32 %add6
+}
+
+; CHECK-LABEL: name: test_ints
+
+; 32BIT:       liveins:
+; 32BIT-NEXT:  - { reg: '$r3', virtual-reg: '' }
+; 32BIT-NEXT:  - { reg: '$r4', virtual-reg: '' }
+; 32BIT-NEXT:  - { reg: '$r5', virtual-reg: '' }
+; 32BIT-NEXT:  - { reg: '$r6', virtual-reg: '' }
+; 32BIT-NEXT:  - { reg: '$r7', virtual-reg: '' }
+; 32BIT-NEXT:  - { reg: '$r8', virtual-reg: '' }
+; 32BIT-NEXT:  - { reg: '$r9', virtual-reg: '' }
+; 32BIT-NEXT:  - { reg: '$r10', virtual-reg: '' }
+; 32BIT:       body:             |
+; 32BIT-NEXT:    bb.0.entry:
+; 32BIT-NEXT:      liveins: $r3, $r4, $r5, $r6, $r7, $r8, $r9, $r10
+
+; 64BIT:       liveins:
+; 64BIT-NEXT:  - { reg: '$x3', virtual-reg: '' }
+; 64BIT-NEXT:  - { reg: '$x4', virtual-reg: '' }
+; 64BIT-NEXT:  - { reg: '$x5', virtual-reg: '' }
+; 64BIT-NEXT:  - { reg: '$x6', virtual-reg: '' }
+; 64BIT-NEXT:  - { reg: '$x7', virtual-reg: '' }
+; 64BIT-NEXT:  - { reg: '$x8', virtual-reg: '' }
+; 64BIT-NEXT:  - { reg: '$x9', virtual-reg: '' }
+; 64BIT-NEXT:  - { reg: '$x10', virtual-reg: '' }
+; 64BIT:       body:             |
+; 64BIT-NEXT:    bb.0.entry:
+; 64BIT-NEXT:      liveins: $x3, $x4, $x5, $x6, $x7, $x8, $x9, $x10
+
+define void @call_test_ints() {
+entry:
+  call i32 @test_ints(i32 signext 1, i32 zeroext 1, i32 zeroext 2147483648, i32 signext -2147483648, i32 signext 1, i32 signext 1, i32 signext 1, i32 signext 1)
+  ret void
+}
+
+; CHECK-LABEL: name: call_test_ints
+
+; 64BIT: ADJCALLSTACKDOWN 112, 0, implicit-def dead $r1, implicit $r1
+; 64BIT: renamable $x3 = LI8 1
+; 64BIT: renamable $x5 = RLDICR killed renamable $x3, 31, 32
+; 64BIT: $x3 = LI8 1
+; 64BIT: $x4 = LI8 1
+; 64BIT: $x6 = LIS8 32768
+; 64BIT: $x7 = LI8 1
+; 64BIT: $x8 = LI8 1
+; 64BIT: $x9 = LI8 1
+; 64BIT: $x10 = LI8 1
+; 64BIT:  BL8_NOP <mcsymbol .test_ints>, csr_aix64, implicit-def dead $lr8, implicit $rm, implicit $x3, implicit killed $x4, implicit $x5, implicit killed $x6, implicit killed $x7, implicit killed $x8, implicit killed $x9, implicit killed $x10, implicit $x2, implicit-def $r1, implicit-def dead $x3
+; 64BIT: ADJCALLSTACKUP 112, 0, implicit-def dead $r1, implicit $r1
+
+define void @call_test_i64() {
+entry:
+  call i64 @test_i64(i64 1, i64 2, i64 3, i64 4)
+  ret void
+}
+
+; CHECK-LABEL: name: call_test_i64
+
+; 32BIT: ADJCALLSTACKDOWN 56, 0, implicit-def dead $r1, implicit $r1
+; 32BIT: $r3 = LI 0
+; 32BIT: $r4 = LI 1
+; 32BIT: $r5 = LI 0
+; 32BIT: $r6 = LI 2
+; 32BIT: $r7 = LI 0
+; 32BIT: $r8 = LI 3
+; 32BIT: $r9 = LI 0
+; 32BIT: $r10 = LI 4
+; 32BIT: BL_NOP <mcsymbol .test_i64>, csr_aix32, implicit-def dead $lr, implicit $rm, implicit killed $r3, implicit killed $r4, implicit killed $r5, implicit killed $r6, implicit killed $r7, implicit killed $r8, implicit killed $r9, implicit killed $r10, implicit $r2, implicit-def $r1
+; 32BIT: ADJCALLSTACKUP 56, 0, implicit-def dead $r1, implicit $r1
+
+; 64BIT: ADJCALLSTACKDOWN 112, 0, implicit-def dead $r1, implicit $r1
+; 64BIT: $x3 = LI8 1
+; 64BIT: $x4 = LI8 2
+; 64BIT: $x5 = LI8 3
+; 64BIT: $x6 = LI8 4
+; 64BIT: BL8_NOP <mcsymbol .test_i64>, csr_aix64, implicit-def dead $lr8, implicit $rm, implicit killed $x3, implicit killed $x4, implicit killed $x5, implicit killed $x6, implicit $x2, implicit-def $r1
+; 64BIT: ADJCALLSTACKUP 112, 0, implicit-def dead $r1, implicit $r1
+
+define i64 @test_i64(i64 %a, i64 %b, i64 %c, i64 %d) {
+entry:
+  %add = add nsw i64 %a, %b
+  %add1 = add nsw i64 %add, %c
+  %add2 = add nsw i64 %add1, %d
+  ret i64 %add2
+}
+
+; CHECK-LABEL: name: test_i64
+
+; 32BIT:       liveins:
+; 32BIT-NEXT:  - { reg: '$r3', virtual-reg: '' }
+; 32BIT-NEXT:  - { reg: '$r4', virtual-reg: '' }
+; 32BIT-NEXT:  - { reg: '$r5', virtual-reg: '' }
+; 32BIT-NEXT:  - { reg: '$r6', virtual-reg: '' }
+; 32BIT-NEXT:  - { reg: '$r7', virtual-reg: '' }
+; 32BIT-NEXT:  - { reg: '$r8', virtual-reg: '' }
+; 32BIT-NEXT:  - { reg: '$r9', virtual-reg: '' }
+; 32BIT-NEXT:  - { reg: '$r10', virtual-reg: '' }
+; 32BIT:       body:             |
+; 32BIT-NEXT:    bb.0.entry:
+; 32BIT-NEXT:      liveins: $r3, $r4, $r5, $r6, $r7, $r8, $r9, $r10
+
+; 64BIT:       liveins:
+; 64BIT-NEXT:  - { reg: '$x3', virtual-reg: '' }
+; 64BIT-NEXT:  - { reg: '$x4', virtual-reg: '' }
+; 64BIT-NEXT:  - { reg: '$x5', virtual-reg: '' }
+; 64BIT-NEXT:  - { reg: '$x6', virtual-reg: '' }
+; 64BIT:       body:             |
+; 64BIT-NEXT:    bb.0.entry:
+; 64BIT-NEXT:      liveins: $x3, $x4, $x5, $x6
+
+define void @call_test_int_ptr() {
+entry:
+  %b = alloca i32, align 4
+  store i32 0, i32* %b, align 4
+  call void @test_int_ptr(i32* %b)
+  ret void
+}
+
+; CHECK-LABEL: name: call_test_int_ptr
+
+; 32BIT: ADJCALLSTACKDOWN 56, 0, implicit-def dead $r1, implicit $r1
+; 32BIT: renamable $r3 = ADDI %stack.0.b, 0
+; 32BIT: BL_NOP <mcsymbol .test_int_ptr>, csr_aix32, implicit-def dead $lr, implicit $rm, implicit $r3, implicit $r2, implicit-def $r1
+; 32BIT: ADJCALLSTACKUP 56, 0, implicit-def dead $r1, implicit $r1
+
+; 64BIT: ADJCALLSTACKDOWN 112, 0, implicit-def dead $r1, implicit $r1
+; 64BIT: renamable $x3 = ADDI8 %stack.0.b, 0
+; 64BIT: BL8_NOP <mcsymbol .test_int_ptr>, csr_aix64, implicit-def dead $lr8, implicit $rm, implicit $x3, implicit $x2, implicit-def $r1
+; 64BIT: ADJCALLSTACKUP 112, 0, implicit-def dead $r1, implicit $r1
+
+define void @test_int_ptr(i32* %a) {
+entry:
+  %a.addr = alloca i32*, align 8
+  store i32* %a, i32** %a.addr, align 8
+  ret void
+}
+
+; CHECK-LABEL: name: test_int_ptr
+
+; 32BIT:       liveins:
+; 32BIT-NEXT:  - { reg: '$r3', virtual-reg: '' }
+; 32BIT:       body:             |
+; 32BIT-NEXT:    bb.0.entry:
+; 32BIT-NEXT:      liveins: $r3
+; 32BIT:           STW killed renamable $r3, 0, %stack.0.a.addr :: (store 4 into %ir.a.addr, align 8)
+
+; 64BIT:       liveins:
+; 64BIT-NEXT:  - { reg: '$x3', virtual-reg: '' }
+; 64BIT:       body:             |
+; 64BIT-NEXT:    bb.0.entry:
+; 64BIT-NEXT:      liveins: $x3
+; 64BIT:           STD killed renamable $x3, 0, %stack.0.a.addr :: (store 8 into %ir.a.addr)
+
+
+define i32 @caller(i32 %i)  {
+entry:
+  %i.addr = alloca i32, align 4
+  %b = alloca i8, align 1
+  store i32 %i, i32* %i.addr, align 4
+  %0 = load i32, i32* %i.addr, align 4
+  %cmp = icmp ne i32 %0, 0
+  %frombool = zext i1 %cmp to i8
+  store i8 %frombool, i8* %b, align 1
+  %1 = load i8, i8* %b, align 1
+  %tobool = trunc i8 %1 to i1
+  %call = call i32 @call_test_bool(i1 zeroext %tobool)
+  ret i32 %call
+}
+
+declare i32 @call_test_bool(i1 zeroext)
+  
+; CHECK-LABEL: name:            caller
+
+; 32BIT:        liveins:
+; 32BIT-NEXT:   - { reg: '$r3', virtual-reg: '' }
+; 32BIT:        body:             |
+; 32BIT-NEXT:   bb.0.entry:
+; 32BIT:         liveins: $r3
+; 32BIT:          ADJCALLSTACKDOWN 56, 0, implicit-def dead $r1, implicit $r1
+; 32BIT:          BL_NOP <mcsymbol .call_test_bool>, csr_aix32, implicit-def dead $lr, implicit $rm, implicit $r3, implicit $r2, implicit-def $r1, implicit-def $r3
+; 32BIT:          ADJCALLSTACKUP 56, 0, implicit-def dead $r1, implicit $r1
+
+; 64BIT:        liveins:
+; 64BIT-NEXT:   - { reg: '$x3', virtual-reg: '' }
+; 64BIT:        body:             |
+; 64BIT-NEXT:    bb.0.entry:
+; 64BIT-NEXT:     liveins: $x3
+; 64BIT:          ADJCALLSTACKDOWN 112, 0, implicit-def dead $r1, implicit $r1
+; 64BIT:          BL8_NOP <mcsymbol .call_test_bool>, csr_aix64, implicit-def dead $lr8, implicit $rm, implicit $x3, implicit $x2, implicit-def $r1, implicit-def $x3
+; 64BIT:          ADJCALLSTACKUP 112, 0, implicit-def dead $r1, implicit $r1
+
+@f1 = global float 0.000000e+00, align 4
+@d1 = global double 0.000000e+00, align 8
+
+define void @call_test_floats() {
+entry:
+  %0 = load float, float* @f1, align 4
+  call float @test_floats(float %0, float %0, float %0)
+  ret void
+}
+
+; CHECK-LABEL: name: call_test_floats{{.*}}
+
+; 32BIT:      renamable $r3 = LWZtoc @f1, $r2 :: (load 4 from got)
+; 32BIT-NEXT: renamable $f1 = LFS 0, killed renamable $r3 :: (dereferenceable load 4 from @f1)
+; 32BIT-NEXT: ADJCALLSTACKDOWN 56, 0, implicit-def dead $r1, implicit $r1
+; 32BIT-NEXT: $f2 = COPY renamable $f1
+; 32BIT-NEXT: $f3 = COPY renamable $f1
+; 32BIT-NEXT: BL_NOP <mcsymbol .test_floats>, csr_aix32, implicit-def dead $lr, implicit $rm, implicit $f1, implicit killed $f2, implicit killed $f3, implicit $r2, implicit-def $r1
+; 32BIT-NEXT: ADJCALLSTACKUP 56, 0, implicit-def dead $r1, implicit $r1
+
+; 64BIT:      renamable $x3 = LDtoc @f1, $x2 :: (load 8 from got)
+; 64BIT-NEXT: renamable $f1 = LFS 0, killed renamable $x3 :: (dereferenceable load 4 from @f1)
+; 64BIT-NEXT: ADJCALLSTACKDOWN 112, 0, implicit-def dead $r1, implicit $r1
+; 64BIT-NEXT: $f2 = COPY renamable $f1
+; 64BIT-NEXT: $f3 = COPY renamable $f1
+; 64BIT-NEXT: BL8_NOP <mcsymbol .test_floats>, csr_aix64, implicit-def dead $lr8, implicit $rm, implicit $f1, implicit killed $f2, implicit killed $f3, implicit $x2, implicit-def $r1
+; 64BIT-NEXT: ADJCALLSTACKUP 112, 0, implicit-def dead $r1, implicit $r1
+
+define float @test_floats(float %f1, float %f2, float %f3) {
+entry:
+  %add = fadd float %f1, %f2
+  %add1 = fadd float %add, %f3
+  ret float %add1
+}
+
+; CHECK-LABEL: name: test_floats{{.*}}
+
+; CHECK:      liveins:
+; CHECK-NEXT: - { reg: '$f1', virtual-reg: '' }
+; CHECK-NEXT: - { reg: '$f2', virtual-reg: '' }
+; CHECK-NEXT: - { reg: '$f3', virtual-reg: '' }
+; CHECK:      body:             |
+; CHECK-NEXT:   bb.0.entry:
+; CHECK-NEXT:     liveins: $f1, $f2, $f3
+
+define void @call_test_fpr_max() {
+entry:
+  %0 = load double, double* @d1, align 8
+  call double @test_fpr_max(double %0, double %0, double %0, double %0, double %0, double %0, double %0, double %0, double %0, double %0, double %0, double %0, double %0)
+  ret void
+}
+
+; CHECK-LABEL: name: call_test_fpr_max{{.*}}
+
+; 32BIT:      renamable $r3 = LWZtoc @d1, $r2 :: (load 4 from got)
+; 32BIT-NEXT: renamable $f1 = LFD 0, killed renamable $r3 :: (dereferenceable load 8 from @d1)
+; 32BIT-NEXT: ADJCALLSTACKDOWN 56, 0, implicit-def dead $r1, implicit $r1
+; 32BIT-NEXT: $f2 = COPY renamable $f1
+; 32BIT-NEXT: $f3 = COPY renamable $f1
+; 32BIT-NEXT: $f4 = COPY renamable $f1
+; 32BIT-NEXT: $f5 = COPY renamable $f1
+; 32BIT-NEXT: $f6 = COPY renamable $f1
+; 32BIT-NEXT: $f7 = COPY renamable $f1
+; 32BIT-NEXT: $f8 = COPY renamable $f1
+; 32BIT-NEXT: $f9 = COPY renamable $f1
+; 32BIT-NEXT: $f10 = COPY renamable $f1
+; 32BIT-NEXT: $f11 = COPY renamable $f1
+; 32BIT-NEXT: $f12 = COPY renamable $f1
+; 32BIT-NEXT: $f13 = COPY renamable $f1
+; 32BIT-NEXT: BL_NOP <mcsymbol .test_fpr_max>, csr_aix32, implicit-def dead $lr, implicit $rm, implicit $f1, implicit killed $f2, implicit killed $f3, implicit killed $f4, implicit killed $f5, implicit killed $f6, implicit killed $f7, implicit killed $f8, implicit killed $f9, implicit killed $f10, implicit killed $f11, implicit killed $f12, implicit killed $f13, implicit $r2, implicit-def $r1
+; 32BIT-NEXT: ADJCALLSTACKUP 56, 0, implicit-def dead $r1, implicit $r1
+
+; 64BIT:      renamable $x3 = LDtoc @d1, $x2 :: (load 8 from got)
+; 64BIT-NEXT: renamable $f1 = LFD 0, killed renamable $x3 :: (dereferenceable load 8 from @d1)
+; 64BIT-NEXT: ADJCALLSTACKDOWN 112, 0, implicit-def dead $r1, implicit $r1
+; 64BIT-NEXT: $f2 = COPY renamable $f1
+; 64BIT-NEXT: $f3 = COPY renamable $f1
+; 64BIT-NEXT: $f4 = COPY renamable $f1
+; 64BIT-NEXT: $f5 = COPY renamable $f1
+; 64BIT-NEXT: $f6 = COPY renamable $f1
+; 64BIT-NEXT: $f7 = COPY renamable $f1
+; 64BIT-NEXT: $f8 = COPY renamable $f1
+; 64BIT-NEXT: $f9 = COPY renamable $f1
+; 64BIT-NEXT: $f10 = COPY renamable $f1
+; 64BIT-NEXT: $f11 = COPY renamable $f1
+; 64BIT-NEXT: $f12 = COPY renamable $f1
+; 64BIT-NEXT: $f13 = COPY renamable $f1
+; 64BIT-NEXT: BL8_NOP <mcsymbol .test_fpr_max>, csr_aix64, implicit-def dead $lr8, implicit $rm, implicit $f1, implicit killed $f2, implicit killed $f3, implicit killed $f4, implicit killed $f5, implicit killed $f6, implicit killed $f7, implicit killed $f8, implicit killed $f9, implicit killed $f10, implicit killed $f11, implicit killed $f12, implicit killed $f13, implicit $x2, implicit-def $r1
+; 64BIT-NEXT: ADJCALLSTACKUP 112, 0, implicit-def dead $r1, implicit $r1
+
+define double @test_fpr_max(double %d1, double %d2, double %d3, double %d4, double %d5, double %d6, double %d7, double %d8, double %d9, double %d10, double %d11, double %d12, double %d13) {
+entry:
+  %add = fadd double %d1, %d2
+  %add1 = fadd double %add, %d3
+  %add2 = fadd double %add1, %d4
+  %add3 = fadd double %add2, %d5
+  %add4 = fadd double %add3, %d6
+  %add5 = fadd double %add4, %d7
+  %add6 = fadd double %add5, %d8
+  %add7 = fadd double %add6, %d9
+  %add8 = fadd double %add7, %d10
+  %add9 = fadd double %add8, %d11
+  %add10 = fadd double %add9, %d12
+  %add11 = fadd double %add10, %d13
+  ret double %add11
+}
+
+; CHECK-LABEL: name: test_fpr_max{{.*}}
+
+; CHECK:      liveins:
+; CHECK-NEXT: - { reg: '$f1', virtual-reg: '' }
+; CHECK-NEXT: - { reg: '$f2', virtual-reg: '' }
+; CHECK-NEXT: - { reg: '$f3', virtual-reg: '' }
+; CHECK-NEXT: - { reg: '$f4', virtual-reg: '' }
+; CHECK-NEXT: - { reg: '$f5', virtual-reg: '' }
+; CHECK-NEXT: - { reg: '$f6', virtual-reg: '' }
+; CHECK-NEXT: - { reg: '$f7', virtual-reg: '' }
+; CHECK-NEXT: - { reg: '$f8', virtual-reg: '' }
+; CHECK-NEXT: - { reg: '$f9', virtual-reg: '' }
+; CHECK-NEXT: - { reg: '$f10', virtual-reg: '' }
+; CHECK-NEXT: - { reg: '$f11', virtual-reg: '' }
+; CHECK-NEXT: - { reg: '$f12', virtual-reg: '' }
+; CHECK-NEXT: - { reg: '$f13', virtual-reg: '' }
+; CHECK:      body:             |
+; CHECK-NEXT:   bb.0.entry:
+; CHECK-NEXT:     liveins: $f1, $f2, $f3, $f4, $f5, $f6, $f7, $f8, $f9, $f10, $f11, $f12, $f13
+
+define void @call_test_mix() {
+entry:
+  %0 = load float, float* @f1, align 4
+  %1 = load double, double* @d1, align 8
+  call i32 @test_mix(float %0, i32 1, double %1, i8 signext 97)
+  ret void
+}
+
+; CHECK-LABEL: name: call_test_mix{{.*}}
+
+; 32BIT:      renamable $r[[REG1:[0-9]+]] = LWZtoc @f1, $r2 :: (load 4 from got)
+; 32BIT-NEXT: renamable $r[[REG2:[0-9]+]] = LWZtoc @d1, $r2 :: (load 4 from got)
+; 32BIT-NEXT: renamable $f1 = LFS 0, killed renamable $r[[REG1]] :: (dereferenceable load 4 from @f1)
+; 32BIT-NEXT: renamable $f2 = LFD 0, killed renamable $r[[REG2]] :: (dereferenceable load 8 from @d1)
+; 32BIT-NEXT: ADJCALLSTACKDOWN 56, 0, implicit-def dead $r1, implicit $r1
+; 32BIT-NEXT: $r4 = LI 1
+; 32BIT-NEXT: $r7 = LI 97
+; 32BIT-NEXT: BL_NOP <mcsymbol .test_mix>, csr_aix32, implicit-def dead $lr, implicit $rm, implicit $f1, implicit $r4, implicit $f2, implicit killed $r7, implicit $r2, implicit-def $r1
+; 32BIT-NEXT: ADJCALLSTACKUP 56, 0, implicit-def dead $r1, implicit $r1
+
+; 64BIT:      renamable $x[[REG1:[0-9]+]] = LDtoc @f1, $x2 :: (load 8 from got)
+; 64BIT-NEXT: renamable $x[[REG2:[0-9]+]] = LDtoc @d1, $x2 :: (load 8 from got)
+; 64BIT-NEXT: renamable $f1 = LFS 0, killed renamable $x[[REG1]] :: (dereferenceable load 4 from @f1)
+; 64BIT-NEXT: renamable $f2 = LFD 0, killed renamable $x[[REG2]] :: (dereferenceable load 8 from @d1)
+; 64BIT-NEXT: ADJCALLSTACKDOWN 112, 0, implicit-def dead $r1, implicit $r1
+; 64BIT-NEXT: $x4 = LI8 1
+; 64BIT-NEXT: $x6 = LI8 97
+; 64BIT-NEXT: BL8_NOP <mcsymbol .test_mix>, csr_aix64, implicit-def dead $lr8, implicit $rm, implicit $f1, implicit $x4, implicit $f2, implicit killed $x6, implicit $x2, implicit-def $r1
+; 64BIT-NEXT: ADJCALLSTACKUP 112, 0, implicit-def dead $r1, implicit $r1
+
+define i32 @test_mix(float %f, i32 signext %i, double %d, i8 signext %c) {
+entry:
+  %conv = fpext float %f to double
+  %add = fadd double %conv, %d
+  %conv1 = fptrunc double %add to float
+  %conv2 = zext i8 %c to i32
+  %add3 = add nsw i32 %i, %conv2
+  %conv4 = sitofp i32 %add3 to float
+  %add5 = fadd float %conv4, %conv1
+  %conv6 = fptosi float %add5 to i32
+  ret i32 %conv6
+}
+
+; CHECK-LABEL: name: test_mix{{.*}}
+
+; 32BIT:      liveins:
+; 32BIT-NEXT: - { reg: '$f1', virtual-reg: '' }
+; 32BIT-NEXT: - { reg: '$r4', virtual-reg: '' }
+; 32BIT-NEXT: - { reg: '$f2', virtual-reg: '' }
+; 32BIT-NEXT: - { reg: '$r7', virtual-reg: '' }
+; 32BIT:      body:             |
+; 32BIT-NEXT:   bb.0.entry:
+; 32BIT-NEXT:     liveins: $f1, $f2, $r4, $r7
+
+; 64BIT:      liveins:
+; 64BIT-NEXT: - { reg: '$f1', virtual-reg: '' }
+; 64BIT-NEXT: - { reg: '$x4', virtual-reg: '' }
+; 64BIT-NEXT: - { reg: '$f2', virtual-reg: '' }
+; 64BIT-NEXT: - { reg: '$x6', virtual-reg: '' }
+; 64BIT:      body:             |
+; 64BIT-NEXT:   bb.0.entry:
+; 64BIT-NEXT:     liveins: $f1, $f2, $x4, $x6
+
+
+define i64 @callee_mixed_ints(i32 %a, i8 signext %b, i32 %c, i16 signext %d, i64 %e) {
+entry:
+  %conv = zext i8 %b to i32
+  %add = add nsw i32 %a, %conv
+  %add1 = add nsw i32 %add, %c
+  %conv2 = sext i16 %d to i32
+  %add3 = add nsw i32 %add1, %conv2
+  %conv4 = sext i32 %add3 to i64
+  %add5 = add nsw i64 %conv4, %e
+  ret i64 %add5
+  }
+
+; CHECK-LABEL: name:  callee_mixed_ints
+
+; 32BIT:      liveins:
+; 32BIT-NEXT: - { reg: '$r3', virtual-reg: '' }
+; 32BIT-NEXT: - { reg: '$r4', virtual-reg: '' }
+; 32BIT-NEXT: - { reg: '$r5', virtual-reg: '' }
+; 32BIT-NEXT: - { reg: '$r6', virtual-reg: '' }
+; 32BIT-NEXT: - { reg: '$r7', virtual-reg: '' }
+; 32BIT-NEXT: - { reg: '$r8', virtual-reg: '' }
+; 32BIT:      body:             |
+; 32BIT-NEXT:  bb.0.entry:
+; 32BIT-NEXT:   liveins: $r3, $r4, $r5, $r6, $r7, $r8
+  
+; 64BIT:        liveins:
+; 64BIT-NEXT:   - { reg: '$x3', virtual-reg: '' }
+; 64BIT-NEXT:   - { reg: '$x4', virtual-reg: '' }
+; 64BIT-NEXT:   - { reg: '$x5', virtual-reg: '' }
+; 64BIT-NEXT:   - { reg: '$x6', virtual-reg: '' }
+; 64BIT-NEXT:   - { reg: '$x7', virtual-reg: '' }
+; 64BIT:        body:             |
+; 64BIT-NEXT:    bb.0.entry:
+; 64BIT-NEXT:     liveins: $x3, $x4, $x5, $x6, $x7
+
+define void @call_test_vararg() {
+entry:
+  %0 = load float, float* @f1, align 4
+  %conv = fpext float %0 to double
+  %1 = load double, double* @d1, align 8
+  call void (i32, ...) @test_vararg(i32 42, double %conv, double %1)
+  ret void
+}
+
+declare void @test_vararg(i32, ...)
+
+; CHECK-LABEL:     name: call_test_vararg
+
+; 32BIT:      renamable $r[[REG:[0-9]+]] = LWZtoc @f1, $r2 :: (load 4 from got)
+; 32BIT-NEXT: renamable $f1 = LFS 0, killed renamable $r[[REG]] :: (dereferenceable load 4 from @f1)
+; 32BIT-NEXT: renamable $r[[REG:[0-9]+]] = LWZtoc @d1, $r2 :: (load 4 from got)
+; 32BIT-NEXT: STFD renamable $f1, 0, %stack.[[SLOT1:[0-9]+]] :: (store 8 into %stack.[[SLOT1]])
+; 32BIT-NEXT: renamable $f2 = LFD 0, killed renamable $r[[REG]] :: (dereferenceable load 8 from @d1)
+; 32BIT-NEXT: renamable $r4 = LWZ 0, %stack.[[SLOT1]] :: (load 4 from %stack.[[SLOT1]], align 8)
+; 32BIT-NEXT: renamable $r5 = LWZ 4, %stack.[[SLOT1]] :: (load 4 from %stack.[[SLOT1]] + 4)
+; 32BIT-NEXT: STFD renamable $f2, 0, %stack.[[SLOT2:[0-9]+]] :: (store 8 into %stack.[[SLOT2]])
+; 32BIT-NEXT: renamable $r6 = LWZ 0, %stack.[[SLOT2]] :: (load 4 from %stack.[[SLOT2]], align 8)
+; 32BIT-NEXT: renamable $r7 = LWZ 4, %stack.[[SLOT2]] :: (load 4 from %stack.[[SLOT2]] + 4)
+; 32BIT-NEXT: ADJCALLSTACKDOWN 56, 0, implicit-def dead $r1, implicit $r1
+; 32BIT-NEXT: $r3 = LI 42
+; 32BIT-NEXT: BL_NOP <mcsymbol .test_vararg>, csr_aix32, implicit-def dead $lr, implicit $rm, implicit $r3, implicit $f1, implicit $r4, implicit $r5, implicit $f2, implicit $r6, implicit $r7, implicit $r2, implicit-def $r1
+; 32BIT-NEXT: ADJCALLSTACKUP 56, 0, implicit-def dead $r1, implicit $r1
+
+; CHECKASM-LABEL: .call_test_vararg:
+
+; ASM32PWR4:      stwu 1, -80(1)
+; ASM32PWR4-NEXT: lwz [[REG:[0-9]+]], LC1(2)
+; ASM32PWR4-NEXT: lfs 1, 0([[REG]])
+; ASM32PWR4-NEXT: lwz [[REG:[0-9]+]], LC2(2)
+; ASM32PWR4-NEXT: stfd 1, 64(1)
+; ASM32PWR4-NEXT: lfd 2, 0([[REG]])
+; ASM32PWR4-NEXT: li 3, 42
+; ASM32PWR4-NEXT: stfd 2, 72(1)
+; ASM32PWR4-DAG:  lwz 4, 64(1)
+; ASM32PWR4-DAG:  lwz 5, 68(1)
+; ASM32PWR4-DAG:  lwz 6, 72(1)
+; ASM32PWR4-DAG:  lwz 7, 76(1)
+; ASM32PWR4-NEXT: bl .test_vararg
+; ASM32PWR4-NEXT: nop
+
+; 64BIT:      renamable $x[[REG:[0-9]+]] = LDtoc @f1, $x2 :: (load 8 from got)
+; 64BIT-NEXT: renamable $f1 = LFS 0, killed renamable $x[[REG]] :: (dereferenceable load 4 from @f1)
+; 64BIT-NEXT: renamable $x[[REG:[0-9]+]] = LDtoc @d1, $x2 :: (load 8 from got)
+; 64BIT-NEXT: STFD renamable $f1, 0, %stack.[[SLOT1:[0-9]+]] :: (store 8 into %stack.[[SLOT1]])
+; 64BIT-NEXT: renamable $f2 = LFD 0, killed renamable $x[[REG]] :: (dereferenceable load 8 from @d1)
+; 64BIT-NEXT: renamable $x4 = LD 0, %stack.[[SLOT1]] :: (load 8 from %stack.[[SLOT1]])
+; 64BIT-NEXT: STFD renamable $f2, 0, %stack.[[SLOT2:[0-9]+]] :: (store 8 into %stack.[[SLOT2]])
+; 64BIT-NEXT: renamable $x5 = LD 0, %stack.[[SLOT2]] :: (load 8 from %stack.[[SLOT2]])
+; 64BIT-NEXT: ADJCALLSTACKDOWN 112, 0, implicit-def dead $r1, implicit $r1
+; 64BIT-NEXT: $x3 = LI8 42
+; 64BIT-NEXT: BL8_NOP <mcsymbol .test_vararg>, csr_aix64, implicit-def dead $lr8, implicit $rm, implicit $x3, implicit $f1, implicit $x4, implicit $f2, implicit $x5, implicit $x2, implicit-def $r1
+; 64BIT-NEXT: ADJCALLSTACKUP 112, 0, implicit-def dead $r1, implicit $r1
+
+; ASM64PWR4:      stdu 1, -128(1)
+; ASM64PWR4-NEXT: ld [[REG:[0-9]+]], LC1(2)
+; ASM64PWR4-NEXT: lfs 1, 0([[REG]])
+; ASM64PWR4-NEXT: ld [[REG:[0-9]+]], LC2(2)
+; ASM64PWR4-NEXT: stfd 1, 112(1)
+; ASM64PWR4-NEXT: lfd 2, 0([[REG]])
+; ASM64PWR4-NEXT: li 3, 42
+; ASM64PWR4-NEXT: stfd 2, 120(1)
+; ASM64PWR4-NEXT: ld 4, 112(1)
+; ASM64PWR4-NEXT: ld 5, 120(1)
+; ASM64PWR4-NEXT: bl .test_vararg
+; ASM64PWR4-NEXT: nop
+
+define void @call_test_vararg2() {
+entry:
+  %0 = load float, float* @f1, align 4
+  %conv = fpext float %0 to double
+  %1 = load double, double* @d1, align 8
+  call void (i32, ...) @test_vararg(i32 42, double %conv, i32 42, double %1)
+  ret void
+}
+
+; CHECK-LABEL:     name: call_test_vararg2
+
+; 32BIT:      renamable $r[[REG:[0-9]+]] = LWZtoc @f1, $r2 :: (load 4 from got)
+; 32BIT-NEXT: renamable $f1 = LFS 0, killed renamable $r[[REG]] :: (dereferenceable load 4 from @f1)
+; 32BIT-NEXT: renamable $r[[REG:[0-9]+]] = LWZtoc @d1, $r2 :: (load 4 from got)
+; 32BIT-NEXT: STFD renamable $f1, 0, %stack.[[SLOT1:[0-9]+]] :: (store 8 into %stack.[[SLOT1]])
+; 32BIT-NEXT: renamable $f2 = LFD 0, killed renamable $r[[REG]] :: (dereferenceable load 8 from @d1)
+; 32BIT-NEXT: renamable $r4 = LWZ 0, %stack.[[SLOT1]] :: (load 4 from %stack.[[SLOT1]], align 8)
+; 32BIT-NEXT: renamable $r5 = LWZ 4, %stack.[[SLOT1]] :: (load 4 from %stack.[[SLOT1]] + 4)
+; 32BIT-NEXT: STFD renamable $f2, 0, %stack.[[SLOT2:[0-9]+]] :: (store 8 into %stack.[[SLOT2]])
+; 32BIT-NEXT: renamable $r7 = LWZ 0, %stack.[[SLOT2]] :: (load 4 from %stack.[[SLOT2]], align 8)
+; 32BIT-NEXT: renamable $r8 = LWZ 4, %stack.[[SLOT2]] :: (load 4 from %stack.[[SLOT2]] + 4)
+; 32BIT-NEXT: ADJCALLSTACKDOWN 56, 0, implicit-def dead $r1, implicit $r1
+; 32BIT-NEXT: $r3 = LI 42
+; 32BIT-NEXT: $r6 = LI 42
+; 32BIT-NEXT: BL_NOP <mcsymbol .test_vararg>, csr_aix32, implicit-def dead $lr, implicit $rm, implicit $r3, implicit $f1, implicit $r4, implicit $r5, implicit killed $r6, implicit $f2, implicit $r7, implicit $r8, implicit $r2, implicit-def $r1
+; 32BIT-NEXT: ADJCALLSTACKUP 56, 0, implicit-def dead $r1, implicit $r1
+
+; ASM32PWR4:      stwu 1, -80(1)
+; ASM32PWR4-NEXT: lwz [[REG:[0-9]+]], LC1(2)
+; ASM32PWR4-NEXT: li 6, 42
+; ASM32PWR4-NEXT: lfs 1, 0([[REG]])
+; ASM32PWR4-NEXT: lwz [[REG:[0-9]+]], LC2(2)
+; ASM32PWR4-NEXT: stfd 1, 64(1)
+; ASM32PWR4-NEXT: lfd 2, 0([[REG]])
+; ASM32PWR4-NEXT: li 3, 42
+; ASM32PWR4-NEXT: stfd 2, 72(1)
+; ASM32PWR4-DAG: lwz 4, 64(1)
+; ASM32PWR4-DAG: lwz 5, 68(1)
+; ASM32PWR4-DAG: lwz 7, 72(1)
+; ASM32PWR4-DAG: lwz 8, 76(1)
+; ASM32PWR4-NEXT: bl .test_vararg
+; ASM32PWR4-NEXT: nop
+
+; 64BIT:      renamable $x[[REG:[0-9]+]] = LDtoc @f1, $x2 :: (load 8 from got)
+; 64BIT-NEXT: renamable $f1 = LFS 0, killed renamable $x[[REG]] :: (dereferenceable load 4 from @f1)
+; 64BIT-NEXT: renamable $x[[REG:[0-9]+]] = LDtoc @d1, $x2 :: (load 8 from got)
+; 64BIT-NEXT: STFD renamable $f1, 0, %stack.[[SLOT1:[0-9]+]] :: (store 8 into %stack.[[SLOT1]])
+; 64BIT-NEXT: renamable $f2 = LFD 0, killed renamable $x[[REG]] :: (dereferenceable load 8 from @d1)
+; 64BIT-NEXT: renamable $x4 = LD 0, %stack.[[SLOT1]] :: (load 8 from %stack.[[SLOT1]])
+; 64BIT-NEXT: STFD renamable $f2, 0, %stack.[[SLOT2:[0-9]+]] :: (store 8 into %stack.[[SLOT2]])
+; 64BIT-NEXT: renamable $x6 = LD 0, %stack.[[SLOT2]] :: (load 8 from %stack.[[SLOT2]])
+; 64BIT-NEXT: ADJCALLSTACKDOWN 112, 0, implicit-def dead $r1, implicit $r1
+; 64BIT-NEXT: $x3 = LI8 42
+; 64BIT-NEXT: $x5 = LI8 42
+; 64BIT-NEXT: BL8_NOP <mcsymbol .test_vararg>, csr_aix64, implicit-def dead $lr8, implicit $rm, implicit $x3, implicit $f1, implicit $x4, implicit killed $x5, implicit $f2, implicit $x6, implicit $x2, implicit-def $r1
+; 64BIT-NEXT: ADJCALLSTACKUP 112, 0, implicit-def dead $r1, implicit $r1
+
+; ASM64PWR4:      stdu 1, -128(1)
+; ASM64PWR4-NEXT: ld [[REG:[0-9]+]], LC1(2)
+; ASM64PWR4-NEXT: li 5, 42
+; ASM64PWR4-NEXT: lfs 1, 0([[REG]])
+; ASM64PWR4-NEXT: ld [[REG:[0-9]+]], LC2(2)
+; ASM64PWR4-NEXT: stfd 1, 112(1)
+; ASM64PWR4-NEXT: lfd 2, 0([[REG]])
+; ASM64PWR4-NEXT: li 3, 42
+; ASM64PWR4-NEXT: stfd 2, 120(1)
+; ASM64PWR4-NEXT: ld 4, 112(1)
+; ASM64PWR4-NEXT: ld 6, 120(1)
+; ASM64PWR4-NEXT: bl .test_vararg
+; ASM64PWR4-NEXT: nop
+
+define void @call_test_vararg3() {
+entry:
+  %0 = load float, float* @f1, align 4
+  %conv = fpext float %0 to double
+  %1 = load double, double* @d1, align 8
+  call void (i32, ...) @test_vararg(i32 42, double %conv, i64 42, double %1)
+  ret void
+}
+
+; CHECK-LABEL:     name: call_test_vararg3
+
+; 32BIT:      renamable $r[[REG:[0-9]+]] = LWZtoc @f1, $r2 :: (load 4 from got)
+; 32BIT-NEXT: renamable $f1 = LFS 0, killed renamable $r[[REG]] :: (dereferenceable load 4 from @f1)
+; 32BIT-NEXT: renamable $r[[REG:[0-9]+]] = LWZtoc @d1, $r2 :: (load 4 from got)
+; 32BIT-NEXT: STFD renamable $f1, 0, %stack.[[SLOT1:[0-9]+]] :: (store 8 into %stack.[[SLOT1]])
+; 32BIT-NEXT: renamable $f2 = LFD 0, killed renamable $r[[REG]] :: (dereferenceable load 8 from @d1)
+; 32BIT-NEXT: renamable $r4 = LWZ 0, %stack.[[SLOT1]] :: (load 4 from %stack.[[SLOT1]], align 8)
+; 32BIT-NEXT: renamable $r5 = LWZ 4, %stack.[[SLOT1]] :: (load 4 from %stack.[[SLOT1]] + 4)
+; 32BIT-NEXT: STFD renamable $f2, 0, %stack.[[SLOT2:[0-9]+]] :: (store 8 into %stack.[[SLOT2]])
+; 32BIT-NEXT: renamable $r8 = LWZ 0, %stack.[[SLOT2]] :: (load 4 from %stack.[[SLOT2]], align 8)
+; 32BIT-NEXT: renamable $r9 = LWZ 4, %stack.[[SLOT2]] :: (load 4 from %stack.[[SLOT2]] + 4)
+; 32BIT-NEXT: ADJCALLSTACKDOWN 56, 0, implicit-def dead $r1, implicit $r1
+; 32BIT-NEXT: $r3 = LI 42
+; 32BIT-NEXT: $r6 = LI 0
+; 32BIT-NEXT: $r7 = LI 42
+; 32BIT-NEXT: BL_NOP <mcsymbol .test_vararg>, csr_aix32, implicit-def dead $lr, implicit $rm, implicit $r3, implicit $f1, implicit $r4, implicit $r5, implicit killed $r6, implicit killed $r7, implicit $f2, implicit $r8, implicit $r9, implicit $r2, implicit-def $r1
+; 32BIT-NEXT: ADJCALLSTACKUP 56, 0, implicit-def dead $r1, implicit $r1
+
+; ASM32PWR4:      stwu 1, -80(1)
+; ASM32PWR4-NEXT: lwz [[REG:[0-9]+]], LC1(2)
+; ASM32PWR4-DAG:  li 6, 0
+; ASM32PWR4-DAG:  li 7, 42
+; ASM32PWR4-NEXT: lfs 1, 0([[REG]])
+; ASM32PWR4-NEXT: lwz [[REG:[0-9]+]], LC2(2)
+; ASM32PWR4-NEXT: stfd 1, 64(1)
+; ASM32PWR4-NEXT: lfd 2, 0([[REG]])
+; ASM32PWR4-NEXT: li 3, 42
+; ASM32PWR4-NEXT: stfd 2, 72(1)
+; ASM32PWR4-DAG:  lwz 4, 64(1)
+; ASM32PWR4-DAG:  lwz 5, 68(1)
+; ASM32PWR4-DAG:  lwz 8, 72(1)
+; ASM32PWR4-DAG:  lwz 9, 76(1)
+; ASM32PWR4-NEXT: bl .test_vararg
+; ASM32PWR4-NEXT: nop
+
+; 64BIT:      renamable $x[[REG:[0-9]+]] = LDtoc @f1, $x2 :: (load 8 from got)
+; 64BIT-NEXT: renamable $f1 = LFS 0, killed renamable $x[[REG]] :: (dereferenceable load 4 from @f1)
+; 64BIT-NEXT: renamable $x[[REG:[0-9]+]] = LDtoc @d1, $x2 :: (load 8 from got)
+; 64BIT-NEXT: STFD renamable $f1, 0, %stack.[[SLOT1:[0-9]+]] :: (store 8 into %stack.[[SLOT1]])
+; 64BIT-NEXT: renamable $f2 = LFD 0, killed renamable $x[[REG]] :: (dereferenceable load 8 from @d1)
+; 64BIT-NEXT: renamable $x4 = LD 0, %stack.[[SLOT1]] :: (load 8 from %stack.[[SLOT1]])
+; 64BIT-NEXT: STFD renamable $f2, 0, %stack.[[SLOT2:[0-9]+]] :: (store 8 into %stack.[[SLOT2]])
+; 64BIT-NEXT: renamable $x6 = LD 0, %stack.[[SLOT2]] :: (load 8 from %stack.[[SLOT2]])
+; 64BIT-NEXT: ADJCALLSTACKDOWN 112, 0, implicit-def dead $r1, implicit $r1
+; 64BIT-NEXT: $x3 = LI8 42
+; 64BIT-NEXT: $x5 = LI8 42
+; 64BIT-NEXT: BL8_NOP <mcsymbol .test_vararg>, csr_aix64, implicit-def dead $lr8, implicit $rm, implicit $x3, implicit $f1, implicit $x4, implicit killed $x5, implicit $f2, implicit $x6, implicit $x2, implicit-def $r1
+; 64BIT-NEXT: ADJCALLSTACKUP 112, 0, implicit-def dead $r1, implicit $r1
+
+; ASM64PWR4:      stdu 1, -128(1)
+; ASM64PWR4-NEXT: ld [[REG:[0-9]+]], LC1(2)
+; ASM64PWR4-NEXT: li 5, 42
+; ASM64PWR4-NEXT: lfs 1, 0([[REG]])
+; ASM64PWR4-NEXT: ld [[REG:[0-9]+]], LC2(2)
+; ASM64PWR4-NEXT: stfd 1, 112(1)
+; ASM64PWR4-NEXT: lfd 2, 0([[REG]])
+; ASM64PWR4-NEXT: li 3, 42
+; ASM64PWR4-NEXT: stfd 2, 120(1)
+; ASM64PWR4-DAG: ld 4, 112(1)
+; ASM64PWR4-DAG: ld 6, 120(1)
+; ASM64PWR4-NEXT: bl .test_vararg
+; ASM64PWR4-NEXT: nop
+
+define void @call_test_vararg4() {
+entry:
+  %0 = load float, float* @f1, align 4
+  call void (i32, ...) @test_vararg(i32 42, float %0)
+  ret void
+}
+
+; CHECK-LABEL:     name: call_test_vararg4
+
+; 32BIT:      renamable $r[[REG:[0-9]+]] = LWZtoc @f1, $r2 :: (load 4 from got)
+; 32BIT-NEXT: renamable $f1 = LFS 0, killed renamable $r[[REG]] :: (dereferenceable load 4 from @f1)
+; 32BIT-NEXT: STFS renamable $f1, 0, %stack.[[SLOT:[0-9]+]] :: (store 4 into %stack.[[SLOT]])
+; 32BIT-NEXT: renamable $r4 = LWZ 0, %stack.[[SLOT]] :: (load 4 from %stack.[[SLOT]])
+; 32BIT-NEXT: ADJCALLSTACKDOWN 56, 0, implicit-def dead $r1, implicit $r1
+; 32BIT-NEXT: $r3 = LI 42
+; 32BIT-NEXT: BL_NOP <mcsymbol .test_vararg>, csr_aix32, implicit-def dead $lr, implicit $rm, implicit $r3, implicit $f1, implicit $r4, implicit $r2, implicit-def $r1
+; 32BIT-NEXT: ADJCALLSTACKUP 56, 0, implicit-def dead $r1, implicit $r1
+
+; ASM32PWR4:      stwu 1, -64(1)
+; ASM32PWR4-NEXT: lwz [[REG:[0-9]+]], LC1(2)
+; ASM32PWR4-NEXT: lfs 1, 0([[REG]])
+; ASM32PWR4-NEXT: li 3, 42
+; ASM32PWR4-NEXT: stfs 1, 60(1)
+; ASM32PWR4-NEXT: lwz 4, 60(1)
+; ASM32PWR4-NEXT: bl .test_vararg
+; ASM32PWR4-NEXT: nop
+
+; 64BIT:      renamable $x[[REG:[0-9]+]] = LDtoc @f1, $x2 :: (load 8 from got)
+; 64BIT-NEXT: renamable $f1 = LFS 0, killed renamable $x[[REG]] :: (dereferenceable load 4 from @f1)
+; 64BIT-NEXT: STFS renamable $f1, 0, %stack.[[SLOT:[0-9]+]] :: (store 4 into %stack.[[SLOT]])
+; 64BIT-NEXT: renamable $x4 = LWZ8 0, %stack.[[SLOT]] :: (load 4 from %stack.[[SLOT]])
+; 64BIT-NEXT: ADJCALLSTACKDOWN 112, 0, implicit-def dead $r1, implicit $r1
+; 64BIT-NEXT: $x3 = LI8 42
+; 64BIT-NEXT: BL8_NOP <mcsymbol .test_vararg>, csr_aix64, implicit-def dead $lr8, implicit $rm, implicit $x3, implicit $f1, implicit $x4, implicit $x2, implicit-def $r1
+; 64BIT-NEXT: ADJCALLSTACKUP 112, 0, implicit-def dead $r1, implicit $r1
+
+; ASM64PWR4:      stdu 1, -128(1)
+; ASM64PWR4-NEXT: ld [[REG:[0-9]+]], LC1(2)
+; ASM64PWR4-NEXT: lfs 1, 0([[REG]])
+; ASM64PWR4-NEXT: li 3, 42
+; ASM64PWR4-NEXT: stfs 1, 124(1)
+; ASM64PWR4-NEXT: lwz 4, 124(1)
+; ASM64PWR4-NEXT: bl .test_vararg
+; ASM64PWR4-NEXT: nop
diff --git a/llvm/test/CodeGen/PowerPC/aix-cc-altivec.ll b/llvm/test/CodeGen/PowerPC/aix-cc-altivec.ll
new file mode 100644
index 000000000..59ad57e1b
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/aix-cc-altivec.ll
@@ -0,0 +1,23 @@
+; RUN: not llc < %s -mtriple powerpc64-ibm-aix-xcoff -mcpu=pwr8 2>&1 | FileCheck %s
+; RUN: not llc < %s -mtriple powerpc-ibm-aix-xcoff -mcpu=pwr8 2>&1 | FileCheck %s
+
+; This test expects a compiler diagnostic for an AIX limitation on Altivec
+; support.  When the Altivec limitation diagnostic is removed, this test
+; should compile clean and fail in order to alert the author to validate the
+; instructions emitted to initialize the GPR for the double vararg.
+; The mfvsrwz and mfvsrd instructions should be used to initialize the GPR for
+; the double vararg without going through memory.
+
+@f1 = global float 0.000000e+00, align 4
+
+define void @call_test_vararg() {
+entry:
+  %0 = load float, float* @f1, align 4
+  %conv = fpext float %0 to double
+  call void (i32, ...) @test_vararg(i32 42, double %conv, float %0)
+  ret void
+}
+
+declare void @test_vararg(i32, ...)
+
+; CHECK: LLVM ERROR: Altivec support is unimplemented on AIX.
diff --git a/llvm/test/CodeGen/PowerPC/aix-external-sym-sdnode-lowering.ll b/llvm/test/CodeGen/PowerPC/aix-external-sym-sdnode-lowering.ll
new file mode 100644
index 000000000..f5864df4f
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/aix-external-sym-sdnode-lowering.ll
@@ -0,0 +1,128 @@
+; RUN: llc -mcpu=pwr4 -mattr=-altivec -verify-machineinstrs -mtriple powerpc-ibm-aix-xcoff \
+; RUN: -stop-after=machine-cp < %s | FileCheck \
+; RUN: --check-prefix=32BIT %s
+
+; RUN: llc -mcpu=pwr4 -mattr=-altivec -verify-machineinstrs -mtriple powerpc64-ibm-aix-xcoff \
+; RUN: -stop-after=machine-cp < %s | FileCheck \
+; RUN: --check-prefix=64BIT %s
+
+define i64 @call_divdi3(i64 %p, i64 %num) {
+entry:
+  %div = sdiv i64 %p, %num
+  ret i64 %div
+}
+
+; 32BIT: BL_NOP <mcsymbol .__divdi3>
+
+define i64 @call_fixunsdfdi(double %p) {
+entry:
+  %conv = fptoui double %p to i64
+  ret i64 %conv
+}
+
+; 32BIT: BL_NOP <mcsymbol .__fixunsdfdi>
+
+define double @call_floatundidf(i64 %p) {
+entry:
+  %conv = uitofp i64 %p to double
+  ret double %conv
+}
+
+; 32BIT: BL_NOP <mcsymbol .__floatundidf>
+
+define float @call_floatundisf(i64 %p) {
+entry:
+  %conv = uitofp i64 %p to float
+  ret float %conv
+}
+
+; 32BIT: BL_NOP <mcsymbol .__floatundisf>
+
+define i64 @call_moddi3(i64 %p, i64 %num) {
+entry:
+  %rem = srem i64 %p, %num
+  ret i64 %rem
+}
+
+; 32BIT: BL_NOP <mcsymbol .__moddi3>
+
+define i64 @call_udivdi3(i64 %p, i64 %q) {
+  %1 = udiv i64 %p, %q
+  ret i64 %1
+}
+
+; 32BIT: BL_NOP <mcsymbol .__udivdi3>
+
+define i64 @call_umoddi3(i64 %p, i64 %num) {
+entry:
+  %rem = urem i64 %p, %num
+  ret i64 %rem
+}
+
+; 32BIT: BL_NOP <mcsymbol .__umoddi3>
+
+define double @call_ceil(double %n) {
+entry:
+  %0 = call double @llvm.ceil.f64(double %n)
+  ret double %0
+}
+
+declare double @llvm.ceil.f64(double)
+
+; 32BIT: BL_NOP <mcsymbol .ceil>
+; 64BIT: BL8_NOP <mcsymbol .ceil>
+
+define double @call_floor(double %n) {
+entry:
+  %0 = call double @llvm.floor.f64(double %n)
+  ret double %0
+}
+
+declare double @llvm.floor.f64(double)
+
+; 32BIT: BL_NOP <mcsymbol .floor>
+; 64BIT: BL8_NOP <mcsymbol .floor>
+
+define void @call_memcpy(i8* %p, i8* %q, i32 %n) {
+entry:
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %p, i8* %q, i32 %n, i1 false)
+  ret void
+}
+
+declare void @llvm.memcpy.p0i8.p0i8.i32(i8* nocapture writeonly, i8* nocapture readonly, i32, i1)
+
+; 32BIT: BL_NOP <mcsymbol .memcpy>
+; 64BIT: BL8_NOP <mcsymbol .memcpy>
+
+define void @call_memmove(i8* %p, i8* %q, i32 %n) {
+entry:
+  call void @llvm.memmove.p0i8.p0i8.i32(i8* %p, i8* %q, i32 %n, i1 false)
+  ret void
+}
+
+declare void @llvm.memmove.p0i8.p0i8.i32(i8* nocapture, i8* nocapture readonly, i32, i1)
+
+; 32BIT: BL_NOP <mcsymbol .memmove>
+; 64BIT: BL8_NOP <mcsymbol .memmove>
+
+define void @call_memset(i8* %p, i8 %q, i32 %n) #0 {
+entry:
+  call void @llvm.memset.p0i8.i32(i8* %p, i8 %q, i32 %n, i1 false)
+  ret void
+}
+
+declare void @llvm.memset.p0i8.i32(i8* nocapture, i8, i32, i1)
+
+; 32BIT: BL_NOP <mcsymbol .memset>
+; 64BIT: BL8_NOP <mcsymbol .memset>
+
+define double @call_round(double %n) {
+entry:
+  %0 = call double @llvm.round.f64(double %n)
+  ret double %0
+}
+
+declare double @llvm.round.f64(double)
+
+; 32BIT: BL_NOP <mcsymbol .round>
+; 64BIT: BL8_NOP <mcsymbol .round>
diff --git a/llvm/test/CodeGen/PowerPC/aix-func-dsc-gen.ll b/llvm/test/CodeGen/PowerPC/aix-func-dsc-gen.ll
new file mode 100644
index 000000000..29b2d4c45
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/aix-func-dsc-gen.ll
@@ -0,0 +1,112 @@
+; RUN: llc -verify-machineinstrs -mcpu=pwr7 -mtriple powerpc-ibm-aix-xcoff -filetype=obj -o %t.o < %s
+; RUN: llvm-readobj  --symbols %t.o | FileCheck %s
+
+define void @foo() {
+entry:
+  ret void
+}
+
+; CHECK:      File: {{.*}}aix-func-dsc-gen.ll.tmp.o
+; CHECK-NEXT: Format: aixcoff-rs6000
+; CHECK-NEXT: Arch: powerpc
+; CHECK-NEXT: AddressSize: 32bit
+; CHECK:        Symbol {
+; CHECK-NEXT:     Index: [[#Index:]]
+; CHECK-NEXT:     Name: .text
+; CHECK-NEXT:     Value (RelocatableAddress): 0x0
+; CHECK-NEXT:     Section: .text
+; CHECK-NEXT:     Type: 0x0
+; CHECK-NEXT:     StorageClass: C_HIDEXT (0x6B)
+; CHECK-NEXT:     NumberOfAuxEntries: 1
+; CHECK-NEXT:     CSECT Auxiliary Entry {
+; CHECK-NEXT:       Index: [[#Index+1]]
+; CHECK-NEXT:       SectionLen: 4
+; CHECK-NEXT:       ParameterHashIndex: 0x0
+; CHECK-NEXT:       TypeChkSectNum: 0x0
+; CHECK-NEXT:       SymbolAlignmentLog2: 4
+; CHECK-NEXT:       SymbolType: XTY_SD (0x1)
+; CHECK-NEXT:       StorageMappingClass: XMC_PR (0x0)
+; CHECK-NEXT:       StabInfoIndex: 0x0
+; CHECK-NEXT:       StabSectNum: 0x0
+; CHECK-NEXT:     }
+; CHECK-NEXT:   }
+; CHECK-NEXT:   Symbol {
+; CHECK-NEXT:     Index: [[#Index+2]]
+; CHECK-NEXT:     Name: .foo
+; CHECK-NEXT:     Value (RelocatableAddress): 0x0
+; CHECK-NEXT:     Section: .text
+; CHECK-NEXT:     Type: 0x0
+; CHECK-NEXT:     StorageClass: C_EXT (0x2)
+; CHECK-NEXT:     NumberOfAuxEntries: 1
+; CHECK-NEXT:     CSECT Auxiliary Entry {
+; CHECK-NEXT:       Index: [[#Index+3]]
+; CHECK-NEXT:       ContainingCsectSymbolIndex: [[#Index]]
+; CHECK-NEXT:       ParameterHashIndex: 0x0
+; CHECK-NEXT:       TypeChkSectNum: 0x0
+; CHECK-NEXT:       SymbolAlignmentLog2: 0
+; CHECK-NEXT:       SymbolType: XTY_LD (0x2)
+; CHECK-NEXT:       StorageMappingClass: XMC_PR (0x0)
+; CHECK-NEXT:       StabInfoIndex: 0x0
+; CHECK-NEXT:       StabSectNum: 0x0
+; CHECK-NEXT:     }
+; CHECK-NEXT:   }
+; CHECK-NEXT:   Symbol {
+; CHECK-NEXT:     Index: [[#Index+4]]
+; CHECK-NEXT:     Name: foo
+; CHECK-NEXT:     Value (RelocatableAddress): 0x4
+; CHECK-NEXT:     Section: .data
+; CHECK-NEXT:     Type: 0x0
+; CHECK-NEXT:     StorageClass: C_HIDEXT (0x6B)
+; CHECK-NEXT:     NumberOfAuxEntries: 1
+; CHECK-NEXT:     CSECT Auxiliary Entry {
+; CHECK-NEXT:       Index: [[#Index+5]]
+; CHECK-NEXT:       SectionLen: 12
+; CHECK-NEXT:       ParameterHashIndex: 0x0
+; CHECK-NEXT:       TypeChkSectNum: 0x0
+; CHECK-NEXT:       SymbolAlignmentLog2: 0
+; CHECK-NEXT:       SymbolType: XTY_SD (0x1)
+; CHECK-NEXT:       StorageMappingClass: XMC_DS (0xA)
+; CHECK-NEXT:       StabInfoIndex: 0x0
+; CHECK-NEXT:       StabSectNum: 0x0
+; CHECK-NEXT:     }
+; CHECK-NEXT:   }
+; CHECK-NEXT:   Symbol {
+; CHECK-NEXT:     Index: [[#Index+6]]
+; CHECK-NEXT:     Name: foo
+; CHECK-NEXT:     Value (RelocatableAddress): 0x4
+; CHECK-NEXT:     Section: .data
+; CHECK-NEXT:     Type: 0x0
+; CHECK-NEXT:     StorageClass: C_EXT (0x2)
+; CHECK-NEXT:     NumberOfAuxEntries: 1
+; CHECK-NEXT:     CSECT Auxiliary Entry {
+; CHECK-NEXT:       Index: [[#Index+7]]
+; CHECK-NEXT:       ContainingCsectSymbolIndex: [[#Index+4]]
+; CHECK-NEXT:       ParameterHashIndex: 0x0
+; CHECK-NEXT:       TypeChkSectNum: 0x0
+; CHECK-NEXT:       SymbolAlignmentLog2: 0
+; CHECK-NEXT:       SymbolType: XTY_LD (0x2)
+; CHECK-NEXT:       StorageMappingClass: XMC_DS (0xA)
+; CHECK-NEXT:       StabInfoIndex: 0x0
+; CHECK-NEXT:       StabSectNum: 0x0
+; CHECK-NEXT:     }
+; CHECK-NEXT:   }
+; CHECK-NEXT:   Symbol {
+; CHECK-NEXT:     Index: [[#Index+8]]
+; CHECK-NEXT:     Name: TOC
+; CHECK-NEXT:     Value (RelocatableAddress): 0x10
+; CHECK-NEXT:     Section: .data
+; CHECK-NEXT:     Type: 0x0
+; CHECK-NEXT:     StorageClass: C_HIDEXT (0x6B)
+; CHECK-NEXT:     NumberOfAuxEntries: 1
+; CHECK-NEXT:     CSECT Auxiliary Entry {
+; CHECK-NEXT:       Index: [[#Index+9]]
+; CHECK-NEXT:       SectionLen: 0
+; CHECK-NEXT:       ParameterHashIndex: 0x0
+; CHECK-NEXT:       TypeChkSectNum: 0x0
+; CHECK-NEXT:       SymbolAlignmentLog2: 2
+; CHECK-NEXT:       SymbolType: XTY_SD (0x1)
+; CHECK-NEXT:       StorageMappingClass: XMC_TC0 (0xF)
+; CHECK-NEXT:       StabInfoIndex: 0x0
+; CHECK-NEXT:       StabSectNum: 0x0
+; CHECK-NEXT:     }
+; CHECK-NEXT:   }
diff --git a/llvm/test/CodeGen/PowerPC/aix-indirect-call.ll b/llvm/test/CodeGen/PowerPC/aix-indirect-call.ll
new file mode 100644
index 000000000..a80a137ee
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/aix-indirect-call.ll
@@ -0,0 +1,144 @@
+; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mattr=-altivec \
+; RUN:  -mtriple powerpc-ibm-aix-xcoff -stop-after=machine-cp < %s | \
+; RUN: FileCheck --check-prefixes=CHECKMIR,MIR32 %s
+
+; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mattr=-altivec \
+; RUN:  -mtriple powerpc-ibm-aix-xcoff < %s | \
+; RUN: FileCheck --check-prefixes=CHECKASM,ASMOBJ32,ASM32 %s
+
+; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mattr=-altivec \
+; RUN:  -mtriple powerpc64-ibm-aix-xcoff -stop-after=machine-cp < %s | \
+; RUN: FileCheck --check-prefixes=CHECKMIR,MIR64 %s
+
+; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mattr=-altivec \
+; RUN:  -mtriple powerpc64-ibm-aix-xcoff < %s | \
+; RUN: FileCheck --check-prefixes=CHECKASM,ASM64 %s
+
+; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mattr=-altivec \
+; RUN: -mtriple powerpc-ibm-aix-xcoff  -filetype=obj < %s -o %t
+; RUN: llvm-objdump -d %t | FileCheck \
+; RUN: --check-prefixes=CHECKOBJ,ASMOBJ32,OBJ32 %s
+
+define signext i32 @callThroughPtr(i32 ()* nocapture) {
+  %2 = tail call signext i32 %0()
+  ret i32 %2
+}
+
+; CHECKMIR:   name:            callThroughPtr
+
+; MIR32:      liveins: $r3
+; MIR32:      ADJCALLSTACKDOWN 56, 0, implicit-def dead $r1, implicit $r1
+; MIR32-DAG:  STW $r2, 20, $r1
+; MIR32-DAG:  renamable $r11 = LWZ 8, renamable $r3 :: (dereferenceable invariant load 4 from %ir.0 + 8)
+; MIR32-DAG:  renamable $[[REG:r[0-9]+]] = LWZ 0, renamable $r3 :: (dereferenceable invariant load 4 from %ir.0)
+; MIR32-DAG:  $r2 = LWZ 4, killed renamable $r3 :: (dereferenceable invariant load 4 from %ir.0 + 4)
+; MIR32-DAG:  MTCTR killed renamable $[[REG]], implicit-def $ctr
+; MIR32-NEXT: BCTRL_LWZinto_toc 20, $r1, csr_aix32, implicit-def dead $lr, implicit-def dead $r2, implicit $ctr, implicit $rm, implicit $r11, implicit $r2, implicit-def $r1, implicit-def $r3
+; MIR32-NEXT: ADJCALLSTACKUP 56, 0, implicit-def dead $r1, implicit $r1
+
+; MIR64:      liveins: $x3
+; MIR64:      ADJCALLSTACKDOWN 112, 0, implicit-def dead $r1, implicit $r1
+; MIR64-DAG:  STD $x2, 40, $x1 :: (store 8 into stack + 40)
+; MIR64-DAG:  renamable $x11 = LD 16, renamable $x3 :: (dereferenceable invariant load 8 from %ir.0 + 16)
+; MIR64-DAG:  renamable $[[REG:x[0-9]+]] = LD 0, renamable $x3 :: (dereferenceable invariant load 8 from %ir.0)
+; MIR64-DAG:  $x2 = LD 8, killed renamable $x3 :: (dereferenceable invariant load 8 from %ir.0 + 8)
+; MIR64-DAG:  MTCTR8 killed renamable $[[REG]], implicit-def $ctr8
+; MIR64-NEXT: BCTRL8_LDinto_toc 40, $x1, csr_aix64, implicit-def dead $lr8, implicit-def dead $x2, implicit $ctr8, implicit $rm, implicit $x11, implicit $x2, implicit-def $r1, implicit-def $x3
+; MIR64-NEXT: ADJCALLSTACKUP 112, 0, implicit-def dead $r1, implicit $r1
+
+; CHECKASM-LABEL: .callThroughPtr:
+
+; ASM32:         stwu 1, -64(1)
+; ASM32-DAG:     lwz [[REG:[0-9]+]], 0(3)
+; ASM32-DAG:     stw 2, 20(1)
+; ASM32-DAG:     mtctr [[REG]]
+; ASM32-DAG:     lwz 11, 8(3)
+; ASM32-DAG:     lwz 2, 4(3)
+; ASM32-NEXT:    bctrl
+; ASM32-NEXT:    lwz 2, 20(1)
+; ASM32-NEXT:    addi 1, 1, 64
+
+; ASM64:            stdu 1, -112(1)
+; ASM64-DAG:        ld [[REG:[0-9]+]], 0(3)
+; ASM64-DAG:        std 2, 40(1)
+; ASM64-DAG:        mtctr [[REG]]
+; ASM64-DAG:        ld 11, 16(3)
+; ASM64-DAG:        ld 2, 8(3)
+; ASM64-NEXT:       bctrl
+; ASM64-NEXT:       ld 2, 40(1)
+; ASM64-NEXT:       addi 1, 1, 112
+
+; OBJ32-LABEL: .text:
+; OBJ32:                      stwu 1, -64(1)
+; OBJ32-DAG:                  lwz [[REG:[0-9]+]], 0(3)
+; OBJ32-DAG:                  stw 2, 20(1)
+; OBJ32-DAG:                  mtctr [[REG]]
+; OBJ32-DAG:                  lwz 11, 8(3)
+; OBJ32-DAG:                  lwz 2, 4(3)
+; OBJ32-NEXT:    4e 80 04 21  bctrl
+; OBJ32-NEXT:    80 41 00 14  lwz 2, 20(1)
+; OBJ32-NEXT:                 addi 1, 1, 64
+
+define void @callThroughPtrWithArgs(void (i32, i16, i64)* nocapture) {
+  tail call void %0(i32 signext 1, i16 zeroext 2, i64 3)
+  ret void
+}
+
+; CHECKMIR:   name:            callThroughPtrWithArgs
+
+; MIR32:      liveins: $r3
+; MIR32:      ADJCALLSTACKDOWN 56, 0, implicit-def dead $r1, implicit $r1
+; MIR32-DAG:  renamable $[[REG:r[0-9]+]] = LWZ 0, renamable $r3 :: (dereferenceable invariant load 4 from %ir.0)
+; MIR32-DAG:  MTCTR killed renamable $[[REG]], implicit-def $ctr
+; MIR32-DAG:  STW $r2, 20, $r1 :: (store 4 into stack + 20)
+; MIR32-DAG:  renamable $r11 = LWZ 8, renamable $r3 :: (dereferenceable invariant load 4 from %ir.0 + 8)
+; MIR32-DAG:  $r2 = LWZ 4, killed renamable $r3 :: (dereferenceable invariant load 4 from %ir.0 + 4)
+; MIR32-DAG:  $r3 = LI 1
+; MIR32-DAG:  $r4 = LI 2
+; MIR32-DAG:  $r5 = LI 0
+; MIR32-DAG:  $r6 = LI 3
+; MIR32-NEXT: BCTRL_LWZinto_toc 20, $r1, csr_aix32, implicit-def dead $lr, implicit-def dead $r2, implicit $ctr, implicit $rm, implicit $r11, implicit $r3, implicit $r4, implicit $r5, implicit $r6, implicit $r2, implicit-def $r1
+; MIR32-NEXT: ADJCALLSTACKUP 56, 0, implicit-def dead $r1, implicit $r1
+
+; MIR64:      liveins: $x3
+; MIR64:      ADJCALLSTACKDOWN 112, 0, implicit-def dead $r1, implicit $r1
+; MIR64-DAG:  renamable $[[REG:x[0-9]+]] = LD 0, renamable $x3 :: (dereferenceable invariant load 8 from %ir.0)
+; MIR64-DAG:  MTCTR8 killed renamable $[[REG]], implicit-def $ctr8
+; MIR64-DAG:  STD $x2, 40, $x1 :: (store 8 into stack + 40)
+; MIR64-DAG:  renamable $x11 = LD 16, renamable $x3 :: (dereferenceable invariant load 8 from %ir.0 + 16)
+; MIR64-DAG:  $x2 = LD 8, killed renamable $x3 :: (dereferenceable invariant load 8 from %ir.0 + 8)
+; MIR64-DAG:  $x3 = LI8 1
+; MIR64-DAG:  $x4 = LI8 2
+; MIR64-DAG:  $x5 = LI8 3
+; MIR64-NEXT: BCTRL8_LDinto_toc 40, $x1, csr_aix64, implicit-def dead $lr8, implicit-def dead $x2, implicit $ctr8, implicit $rm, implicit $x11, implicit $x3, implicit $x4, implicit $x5, implicit $x2, implicit-def $r1
+; MIR64-NEXT: ADJCALLSTACKUP 112, 0, implicit-def dead $r1, implicit $r1
+
+; CHECKASM-LABEL: .callThroughPtrWithArgs:
+; CHECKOBJ-LABEL: .callThroughPtrWithArgs:
+
+; ASMOBJ32:      stwu 1, -64(1)
+; ASMOBJ32-DAG:  lwz [[REG:[0-9]+]], 0(3)
+; ASMOBJ32-DAG:  li 5, 0
+; ASMOBJ32-DAG:  li 6, 3
+; ASMOBJ32-DAG:  stw 2, 20(1)
+; ASMOBJ32-DAG:  mtctr [[REG]]
+; ASMOBJ32-DAG:  li 4, 2
+; ASMOBJ32-DAG:  lwz 11, 8(3)
+; ASMOBJ32-DAG:  lwz 2, 4(3)
+; ASMOBJ32-DAG:  li 3, 1
+; ASMOBJ32-NEXT: bctrl
+; ASMOBJ32-NEXT: lwz 2, 20(1)
+; ASMOBJ32-NEXT: addi 1, 1, 64
+
+; ASM64:            stdu 1, -112(1)
+; ASM64-DAG:        ld [[REG:[0-9]+]], 0(3)
+; ASM64-DAG:        li 5, 3
+; ASM64-DAG:        std 2, 40(1)
+; ASM64-DAG:        mtctr [[REG]]
+; ASM64-DAG:        li 4, 2
+; ASM64-DAG:        ld 11, 16(3)
+; ASM64-DAG:        ld 2, 8(3)
+; ASM64-DAG:        li 3, 1
+; ASM64-NEXT:       bctrl
+; ASM64-NEXT:       ld 2, 40(1)
+; ASM64-NEXT:       addi 1, 1, 112
diff --git a/llvm/test/CodeGen/PowerPC/aix-lower-block-address.ll b/llvm/test/CodeGen/PowerPC/aix-lower-block-address.ll
new file mode 100644
index 000000000..b4b9f029e
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/aix-lower-block-address.ll
@@ -0,0 +1,72 @@
+; RUN: llc -verify-machineinstrs -mcpu=pwr7 -mtriple powerpc-ibm-aix-xcoff \
+; RUN: -code-model=small -stop-after=machine-cp < %s | FileCheck \
+; RUN: --check-prefix=32SMALL-MIR %s
+
+; RUN: llc -verify-machineinstrs -mcpu=pwr7 -mtriple powerpc-ibm-aix-xcoff \
+; RUN: -code-model=large -stop-after=machine-cp < %s | FileCheck \
+; RUN: --check-prefix=32LARGE-MIR %s
+
+; RUN: llc -verify-machineinstrs -mcpu=pwr7 -mtriple powerpc64-ibm-aix-xcoff \
+; RUN: -code-model=small -stop-after=machine-cp < %s | FileCheck \
+; RUN: --check-prefix=64SMALL-MIR %s
+
+; RUN: llc -verify-machineinstrs -mcpu=pwr7 -mtriple powerpc64-ibm-aix-xcoff \
+; RUN: -code-model=large -stop-after=machine-cp < %s | FileCheck \
+; RUN: --check-prefix=64LARGE-MIR %s
+
+; RUN: llc -verify-machineinstrs -mcpu=pwr7 -mtriple powerpc-ibm-aix-xcoff \
+; RUN: -code-model=small < %s | FileCheck --check-prefixes=32SMALL-ASM,CHECK %s
+
+; RUN: llc -verify-machineinstrs -mcpu=pwr7 -mtriple powerpc-ibm-aix-xcoff \
+; RUN: -code-model=large < %s | FileCheck --check-prefixes=32LARGE-ASM,CHECK %s
+
+; RUN: llc -verify-machineinstrs -mcpu=pwr7 -mtriple powerpc64-ibm-aix-xcoff \
+; RUN: -code-model=small < %s | FileCheck --check-prefixes=64SMALL-ASM,CHECK %s
+
+; RUN: llc -verify-machineinstrs -mcpu=pwr7 -mtriple powerpc64-ibm-aix-xcoff \
+; RUN: -code-model=large < %s | FileCheck --check-prefixes=64LARGE-ASM,CHECK %s
+
+define void @foo() {
+entry:
+  %tmp = alloca i64
+  br label %__here
+
+__here:
+  store i64 ptrtoint (i8* blockaddress(@foo, %__here) to i64), i64* %tmp
+  ret void
+}
+
+; 32SMALL-MIR: renamable $r[[REG1:[0-9]+]] = LWZtoc blockaddress(@foo, %ir-block.__here), $r2 :: (load 4 from got)
+
+; 32LARGE-MIR: renamable $r[[REG1:[0-9]+]] = ADDIStocHA $r2, blockaddress(@foo, %ir-block.__here)
+; 32LARGE-MIR: renamable $r[[REG2:[0-9]+]] = LWZtocL blockaddress(@foo, %ir-block.__here), killed renamable $r[[REG1]], implicit $r2 :: (load 4 from got)
+
+; 64SMALL-MIR: renamable $x[[REG1:[0-9]+]] = LDtocBA blockaddress(@foo, %ir-block.__here), $x2 :: (load 8 from got)
+
+; 64LARGE-MIR: renamable $x[[REG1:[0-9]+]] = ADDIStocHA8 $x2, blockaddress(@foo, %ir-block.__here)
+; 64LARGE-MIR: renamable $x[[REG2:[0-9]+]] = LDtocL blockaddress(@foo, %ir-block.__here), killed renamable $x[[REG1]], implicit $x2 :: (load 8 from got)
+
+; 32SMALL-ASM-LABEL: foo
+; 32SMALL-ASM: .foo:
+; 32SMALL-ASM: Ltmp0:
+; 32SMALL-ASM: 	       lwz [[REG1:[0-9]+]], LC0(2)
+
+; 32LARGE-ASM-LABEL: foo
+; 32LARGE-ASM: .foo:
+; 32LARGE-ASM: Ltmp0:
+; 32LARGE-ASM:         addis [[REG1:[0-9]+]], LC0@u(2)
+; 32LARGE-ASM:         lwz [[REG2:[0-9]+]], LC0@l([[REG1]])
+
+; 64SMALL-ASM-LABEL: foo
+; 64SMALL-ASM: .foo:
+; 64SMALL-ASM: Ltmp0:
+; 64SMALL-ASM:         ld [[REG1:[0-9]+]], LC0(2)
+
+; 64LARGE-ASM-LABEL: foo
+; 64LARGE-ASM: .foo:
+; 64LARGE-ASM: Ltmp0:
+; 64LARGE-ASM:         addis [[REG1:[0-9]+]], LC0@u(2)
+; 64LARGE-ASM:         ld [[REG2:[0-9]+]], LC0@l([[REG1]])
+
+; CHECK: .toc
+; CHECK: .tc Ltmp0[TC],Ltmp0
diff --git a/llvm/test/CodeGen/PowerPC/aix-lower-constant-pool-index.ll b/llvm/test/CodeGen/PowerPC/aix-lower-constant-pool-index.ll
new file mode 100644
index 000000000..1db8a55fb
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/aix-lower-constant-pool-index.ll
@@ -0,0 +1,87 @@
+; RUN: llc -verify-machineinstrs -mcpu=pwr7 -mtriple powerpc-ibm-aix-xcoff \
+; RUN: -code-model=small -stop-after=machine-cp < %s | FileCheck \
+; RUN: --check-prefix=32SMALL-MIR %s
+
+; RUN: llc -verify-machineinstrs -mcpu=pwr7 -mtriple powerpc-ibm-aix-xcoff \
+; RUN: -code-model=large -stop-after=machine-cp < %s | FileCheck \
+; RUN: --check-prefix=32LARGE-MIR %s
+
+; RUN: llc -verify-machineinstrs -mcpu=pwr7 -mtriple powerpc64-ibm-aix-xcoff \
+; RUN: -code-model=small -stop-after=machine-cp < %s | FileCheck \
+; RUN: --check-prefix=64SMALL-MIR %s
+
+; RUN: llc -verify-machineinstrs -mcpu=pwr7 -mtriple powerpc64-ibm-aix-xcoff \
+; RUN: -code-model=large -stop-after=machine-cp < %s | FileCheck \
+; RUN: --check-prefix=64LARGE-MIR %s
+
+; RUN: llc -verify-machineinstrs -mcpu=pwr7 -mtriple powerpc-ibm-aix-xcoff \
+; RUN: -code-model=small < %s | FileCheck --check-prefixes=32SMALL-ASM,CHECK %s
+
+; RUN: llc -verify-machineinstrs -mcpu=pwr7 -mtriple powerpc-ibm-aix-xcoff \
+; RUN: -code-model=large < %s | FileCheck --check-prefixes=32LARGE-ASM,CHECK %s
+
+; RUN: llc -verify-machineinstrs -mcpu=pwr7 -mtriple powerpc64-ibm-aix-xcoff \
+; RUN: -code-model=small < %s | FileCheck --check-prefixes=64SMALL-ASM,CHECK %s
+
+; RUN: llc -verify-machineinstrs -mcpu=pwr7 -mtriple powerpc64-ibm-aix-xcoff \
+; RUN: -code-model=large < %s | FileCheck --check-prefixes=64LARGE-ASM,CHECK %s
+
+define float @test_float() {
+entry:
+  ret float 5.500000e+00
+}
+
+; 32SMALL-MIR: renamable $r[[REG1:[0-9]+]] = LWZtoc %const.0, $r2 :: (load 4 from got)
+; 32SMALL-MIR: renamable $f[[REG2:[0-9]+]] = LFS 0, killed renamable $r[[REG1]] :: (load 4 from constant-pool)
+
+; 32LARGE-MIR: renamable $r[[REG1:[0-9]+]] = ADDIStocHA $r2, %const.0
+; 32LARGE-MIR: renamable $r[[REG2:[0-9]+]] = LWZtocL %const.0, killed renamable $r[[REG1]], implicit $r2 :: (load 4 from got)
+; 32LARGE-MIR: renamable $f[[REG3:[0-9]+]] = LFS 0, killed renamable $r[[REG2]] :: (load 4 from constant-pool)
+
+; 64SMALL-MIR: renamable $x[[REG1:[0-9]+]] = LDtocCPT %const.0, $x2 :: (load 8 from got)
+; 64SMALL-MIR: renamable $f[[REG2:[0-9]+]] = LFS 0, killed renamable $x[[REG1]] :: (load 4 from constant-pool)
+
+; 64LARGE-MIR: renamable $x[[REG1:[0-9]+]] = ADDIStocHA8 $x2, %const.0
+; 64LARGE-MIR: renamable $x[[REG2:[0-9]+]] = LDtocL %const.0, killed renamable $x[[REG1]], implicit $x2 :: (load 8 from got)
+; 64LARGE-MIR: renamable $f[[REG3:[0-9]+]] = LFS 0, killed renamable $x[[REG2]] :: (load 4 from constant-pool)
+
+; 32SMALL-ASM:         .csect .rodata[RO]
+; 32SMALL-ASM:         .align  2
+; 32SMALL-ASM: .LCPI0_0:
+; 32SMALL-ASM:         .long  1085276160
+; 32SMALL-ASM: .test_float:
+; 32SMALL-ASM:         lwz [[REG1:[0-9]+]], LC0(2)
+; 32SMALL-ASM:         lfs 1, 0([[REG1]])
+; 32SMALL-ASM:         blr
+
+; 32LARGE-ASM:         .csect .rodata[RO]
+; 32LARGE-ASM:         .align  2
+; 32LARGE-ASM: .LCPI0_0:
+; 32LARGE-ASM:         .long   1085276160
+; 32LARGE-ASM: .test_float:
+; 32LARGE-ASM:         addis [[REG1:[0-9]+]], LC0@u(2)
+; 32LARGE-ASM:         lwz [[REG2:[0-9]+]], LC0@l([[REG1]])
+; 32LARGE-ASM:         lfs 1, 0([[REG2]])
+; 32LARGE-ASM:         blr
+
+; 64SMALL-ASM:         .csect .rodata[RO]
+; 64SMALL-ASM:         .align  2
+; 64SMALL-ASM: .LCPI0_0:
+; 64SMALL-ASM:         .long   1085276160
+; 64SMALL-ASM: .test_float:
+; 64SMALL-ASM:         ld [[REG1:[0-9]+]], LC0(2)
+; 64SMALL-ASM:         lfs 1, 0([[REG1]])
+; 64SMALL-ASM:         blr
+
+; 64LARGE-ASM:         .csect .rodata[RO]
+; 64LARGE-ASM:         .align  2
+; 64LARGE-ASM: .LCPI0_0:
+; 64LARGE-ASM:         .long   1085276160
+; 64LARGE-ASM: .test_float:
+; 64LARGE-ASM:         addis [[REG1:[0-9]+]], LC0@u(2)
+; 64LARGE-ASM:         ld [[REG2:[0-9]+]], LC0@l([[REG1]])
+; 64LARGE-ASM:         lfs 1, 0([[REG2]])
+; 64LARGE-ASM:         blr
+
+; CHECK: .toc
+; CHECK: .tc .LCPI0_0[TC],.LCPI0_0
diff --git a/llvm/test/CodeGen/PowerPC/aix-lower-jump-table.ll b/llvm/test/CodeGen/PowerPC/aix-lower-jump-table.ll
new file mode 100644
index 000000000..a5ec1942a
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/aix-lower-jump-table.ll
@@ -0,0 +1,188 @@
+; RUN: llc -verify-machineinstrs -mcpu=pwr7 -mtriple powerpc-ibm-aix-xcoff \
+; RUN: -code-model=small -stop-after=machine-cp < %s | FileCheck \
+; RUN: --check-prefix=32SMALL-MIR %s
+
+; RUN: llc -verify-machineinstrs -mcpu=pwr7 -mtriple powerpc-ibm-aix-xcoff \
+; RUN: -code-model=large -stop-after=machine-cp < %s | FileCheck \
+; RUN: --check-prefix=32LARGE-MIR %s
+
+; RUN: llc -verify-machineinstrs -mcpu=pwr7 -mtriple powerpc64-ibm-aix-xcoff \
+; RUN: -code-model=small -stop-after=machine-cp < %s | FileCheck \
+; RUN: --check-prefix=64SMALL-MIR %s
+
+; RUN: llc -verify-machineinstrs -mcpu=pwr7 -mtriple powerpc64-ibm-aix-xcoff \
+; RUN: -code-model=large -stop-after=machine-cp < %s | FileCheck \
+; RUN: --check-prefix=64LARGE-MIR %s
+
+; RUN: llc -mtriple powerpc-ibm-aix-xcoff -code-model=small < %s | FileCheck \
+; RUN: --check-prefixes=32SMALL-ASM,CHECK %s
+
+; RUN: llc -mtriple powerpc-ibm-aix-xcoff -code-model=large < %s | FileCheck \
+; RUN: --check-prefixes=32LARGE-ASM,CHECK %s
+
+; RUN: llc -mtriple powerpc64-ibm-aix-xcoff -code-model=small < %s | FileCheck \
+; RUN: --check-prefixes=64SMALL-ASM,CHECK %s
+
+; RUN: llc -mtriple powerpc64-ibm-aix-xcoff -code-model=large < %s | FileCheck \
+; RUN: --check-prefixes=64LARGE-ASM,CHECK %s
+
+  define i32 @jump_table(i32 %a) {
+  entry:
+    switch i32 %a, label %sw.epilog [
+      i32 1, label %sw.bb
+      i32 2, label %sw.bb1
+      i32 3, label %sw.bb2
+      i32 4, label %sw.bb3
+    ]
+
+  sw.bb:
+    tail call void asm sideeffect "", ""()
+    br label %sw.epilog
+
+  sw.bb1:
+    tail call void asm sideeffect "", ""()
+    br label %sw.epilog
+
+  sw.bb2:
+    tail call void asm sideeffect "", ""()
+    br label %sw.epilog
+
+  sw.bb3:
+    tail call void asm sideeffect "", ""()
+    br label %sw.epilog
+
+  sw.epilog:
+    ret i32 0
+  }
+
+
+; 32SMALL-MIR: renamable $r[[REG1:[0-9]+]] = LWZtoc %jump-table.0, $r2 :: (load 4 from got)
+; 32SMALL-MIR: renamable $r[[REG3:[0-9]+]] = RLWINM killed renamable $r[[REG2:[0-9]+]], 2, 0, 29
+; 32SMALL-MIR: renamable $r[[REG4:[0-9]+]] = LWZX killed renamable $r[[REG3]], renamable $r[[REG1]] :: (load 4 from jump-table)
+; 32SMALL-MIR: renamable $r[[REG5:[0-9]+]] = ADD4 killed renamable $r[[REG4]], killed renamable $r[[REG1]]
+
+; 32LARGE-MIR: renamable $r[[REG1:[0-9]+]] = ADDIStocHA $r2, %jump-table.0
+; 32LARGE-MIR: renamable $r[[REG2:[0-9]+]] = LWZtocL %jump-table.0, killed renamable $r[[REG1]], implicit $r2 :: (load 4 from got)
+; 32LARGE-MIR: renamable $r[[REG4:[0-9]+]] = RLWINM killed renamable $r[[REG3:[0-9]+]], 2, 0, 29
+; 32LARGE-MIR: renamable $r[[REG5:[0-9]+]] = LWZX killed renamable $r[[REG4]], renamable $r[[REG2]] :: (load 4 from jump-table)
+; 32LARGE-MIR: renamable $r[[REG6:[0-9]+]] = ADD4 killed renamable $r[[REG5]], killed renamable $r[[REG2]]
+
+; 64SMALL-MIR: renamable $x[[REG1:[0-9]+]] = LDtocJTI %jump-table.0, $x2 :: (load 8 from got)
+; 64SMALL-MIR: renamable $x[[REG3:[0-9]+]] = RLDIC killed renamable $x[[REG2:[0-9]+]], 2, 30
+; 64SMALL-MIR: renamable $x[[REG4:[0-9]+]] = LWAX killed renamable $x[[REG3]], renamable $x[[REG1]] :: (load 4 from jump-table)
+; 64SMALL-MIR: renamable $x[[REG6:[0-9]+]] = ADD8 killed renamable $x[[REG4]], killed renamable $x[[REG1]]
+
+; 64LARGE-MIR: renamable $x[[REG1:[0-9]+]] = ADDIStocHA8 $x2, %jump-table.0
+; 64LARGE-MIR: renamable $x[[REG2:[0-9]+]] = LDtocL %jump-table.0, killed renamable $x[[REG1]], implicit $x2 :: (load 8 from got)
+; 64LARGE-MIR: renamable $x[[REG4:[0-9]+]] = RLDIC killed renamable $x[[REG3:[0-9]+]], 2, 30
+; 64LARGE-MIR: renamable $x[[REG5:[0-9]+]] = LWAX killed renamable $x[[REG4]], renamable $x[[REG2]] :: (load 4 from jump-table)
+; 64LARGE-MIR: renamable $x[[REG6:[0-9]+]] = ADD8 killed renamable $x[[REG5]], killed renamable $x[[REG2]]
+
+; 32SMALL-ASM-LABEL: jump_table
+; 32SMALL-ASM: .jump_table:
+; 32SMALL-ASM:      addi 3, 3, -1
+; 32SMALL-ASM: 	    cmplwi 3, 3
+; 32SMALL-ASM: 	    bgt	0, LBB0_6
+; 32SMALL-ASM: 	    lwz 4, LC0(2)
+; 32SMALL-ASM: 	    slwi 3, 3, 2
+; 32SMALL-ASM: 	    lwzx 3, 3, 4
+; 32SMALL-ASM: 	    add 3, 3, 4
+; 32SMALL-ASM: 	    mtctr 3
+; 32SMALL-ASM: 	    bctr
+; 32SMALL-ASM: LBB0_2:
+; 32SMALL-ASM: LBB0_3:
+; 32SMALL-ASM: LBB0_4:
+; 32SMALL-ASM: LBB0_5:
+; 32SMALL-ASM: LBB0_6:
+; 32SMALL-ASM: 	    li 3, 0
+; 32SMALL-ASM: 	    blr
+; 32SMALL-ASM: 	    .csect .rodata[RO]
+; 32SMALL-ASM: 	    .align  2
+; 32SMALL-ASM: .LJTI0_0:
+; 32SMALL-ASM: 	    .long   LBB0_2-.LJTI0_0
+; 32SMALL-ASM: 	    .long   LBB0_3-.LJTI0_0
+; 32SMALL-ASM: 	    .long   LBB0_4-.LJTI0_0
+; 32SMALL-ASM: 	    .long   LBB0_5-.LJTI0_0
+
+; 32LARGE-ASM-LABEL: jump_table
+; 32LARGE-ASM: .jump_table:
+; 32LARGE-ASM:      addi 3, 3, -1
+; 32LARGE-ASM:      cmplwi  3, 3
+; 32LARGE-ASM:      bgt     0, LBB0_6
+; 32LARGE-ASM: 	    addis 4, LC0@u(2)
+; 32LARGE-ASM: 	    slwi 3, 3, 2
+; 32LARGE-ASM:      lwz 4, LC0@l(4)
+; 32LARGE-ASM:      lwzx 3, 3, 4
+; 32LARGE-ASM:      add 3, 3, 4
+; 32LARGE-ASM:      mtctr 3
+; 32LARGE-ASM:      bctr
+; 32LARGE-ASM: LBB0_2:
+; 32LARGE-ASM: LBB0_3:
+; 32LARGE-ASM: LBB0_4:
+; 32LARGE-ASM: LBB0_5:
+; 32LARGE-ASM: LBB0_6:
+; 32LARGE-ASM:      li 3, 0
+; 32LARGE-ASM:      blr
+; 32LARGE-ASM:      .csect .rodata[RO]
+; 32LARGE-ASM:      .align  2
+; 32LARGE-ASM: .LJTI0_0:
+; 32LARGE-ASM:      .long   LBB0_2-.LJTI0_0
+; 32LARGE-ASM:      .long   LBB0_3-.LJTI0_0
+; 32LARGE-ASM:      .long   LBB0_4-.LJTI0_0
+; 32LARGE-ASM:      .long   LBB0_5-.LJTI0_0
+
+; 64SMALL-ASM-LABEL: jump_table
+; 64SMALL-ASM: .jump_table:
+; 64SMALL-ASM:      addi 3, 3, -1
+; 64SMALL-ASM:      cmplwi  3, 3
+; 64SMALL-ASM:      bgt     0, LBB0_6
+; 64SMALL-ASM:      ld 4, LC0(2)
+; 64SMALL-ASM:      rldic 3, 3, 2, 30
+; 64SMALL-ASM:      lwax 3, 3, 4
+; 64SMALL-ASM:      add 3, 3, 4
+; 64SMALL-ASM:      mtctr 3
+; 64SMALL-ASM:      bctr
+; 64SMALL-ASM: LBB0_2:
+; 64SMALL-ASM: LBB0_3:
+; 64SMALL-ASM: LBB0_4:
+; 64SMALL-ASM: LBB0_5:
+; 64SMALL-ASM: LBB0_6:
+; 64SMALL-ASM:      li 3, 0
+; 64SMALL-ASM:      blr
+; 64SMALL-ASM:      .csect .rodata[RO]
+; 64SMALL-ASM:      .align  2
+; 64SMALL-ASM: .LJTI0_0:
+; 64SMALL-ASM:      .long   LBB0_2-.LJTI0_0
+; 64SMALL-ASM:      .long   LBB0_3-.LJTI0_0
+; 64SMALL-ASM:      .long   LBB0_4-.LJTI0_0
+; 64SMALL-ASM:      .long   LBB0_5-.LJTI0_0
+
+; 64LARGE-ASM-LABEL: jump_table
+; 64LARGE-ASM: .jump_table:
+; 64LARGE-ASM:      addi 3, 3, -1
+; 64LARGE-ASM:      cmplwi  3, 3
+; 64LARGE-ASM:      bgt     0, LBB0_6
+; 64LARGE-ASM:      addis 4, LC0@u(2)
+; 64LARGE-ASM:      rldic 3, 3, 2, 30
+; 64LARGE-ASM:      ld 4, LC0@l(4)
+; 64LARGE-ASM:      lwax 3, 3, 4
+; 64LARGE-ASM:      add 3, 3, 4
+; 64LARGE-ASM:      mtctr 3
+; 64LARGE-ASM:      bctr
+; 64LARGE-ASM: LBB0_2:
+; 64LARGE-ASM: LBB0_3:
+; 64LARGE-ASM: LBB0_4:
+; 64LARGE-ASM: LBB0_5:
+; 64LARGE-ASM: LBB0_6:
+; 64LARGE-ASM:      li 3, 0
+; 64LARGE-ASM:      blr
+; 64LARGE-ASM:      .csect .rodata[RO]
+; 64LARGE-ASM:      .align  2
+; 64LARGE-ASM: .LJTI0_0:
+; 64LARGE-ASM:      .long   LBB0_2-.LJTI0_0
+; 64LARGE-ASM:      .long   LBB0_3-.LJTI0_0
+; 64LARGE-ASM:      .long   LBB0_4-.LJTI0_0
+; 64LARGE-ASM:      .long   LBB0_5-.LJTI0_0
+
+; CHECK: .toc
+; CHECK: .tc .LJTI0_0[TC],.LJTI0_0
diff --git a/llvm/test/CodeGen/PowerPC/aix-lr.ll b/llvm/test/CodeGen/PowerPC/aix-lr.ll
new file mode 100644
index 000000000..ea92daf04
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/aix-lr.ll
@@ -0,0 +1,32 @@
+; RUN: llc -verify-machineinstrs -mtriple powerpc-ibm-aix-xcoff < %s | \
+; RUN: FileCheck --check-prefix=32BIT %s
+
+; RUN: llc -verify-machineinstrs -mtriple powerpc64-ibm-aix-xcoff < %s | \
+; RUN: FileCheck --check-prefix=64BIT %s
+
+define void @bar() {
+entry:
+
+; 32BIT: mflr 0
+; 32BIT: stw 0, 8(1)
+; 32BIT: stwu 1, -64(1)
+; 32BIT: bl .foo
+; 32BIT: nop
+; 32BIT: addi 1, 1, 64
+; 32BIT: lwz 0, 8(1)
+; 32BIT: mtlr 0
+
+; 64BIT: mflr 0
+; 64BIT: std 0, 16(1)
+; 64BIT: stdu 1, -112(1)
+; 64BIT: bl .foo
+; 64BIT: nop
+; 64BIT: addi 1, 1, 112
+; 64BIT: ld 0, 16(1)
+; 64BIT: mtlr 0
+
+  call void bitcast (void (...)* @foo to void ()*)()
+  ret void
+}
+
+declare void @foo(...)
diff --git a/llvm/test/CodeGen/PowerPC/aix-nest-param.ll b/llvm/test/CodeGen/PowerPC/aix-nest-param.ll
new file mode 100644
index 000000000..8534c80ca
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/aix-nest-param.ll
@@ -0,0 +1,13 @@
+; RUN: not llc -mtriple powerpc-ibm-aix-xcoff < %s 2>&1 | FileCheck %s
+; RUN: not llc -mtriple powerpc64-ibm-aix-xcoff < %s 2>&1 | FileCheck %s
+
+define i8* @nest_receiver(i8* nest %arg) nounwind {
+  ret i8* %arg
+}
+
+define i8* @nest_caller(i8* %arg) nounwind {
+  %result = call i8* @nest_receiver(i8* nest %arg)
+  ret i8* %result
+}
+
+; CHECK: LLVM ERROR: Nest arguments are unimplemented.
diff --git a/llvm/test/CodeGen/PowerPC/aix-readonly-with-relocation.ll b/llvm/test/CodeGen/PowerPC/aix-readonly-with-relocation.ll
new file mode 100644
index 000000000..97b39f522
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/aix-readonly-with-relocation.ll
@@ -0,0 +1,19 @@
+; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mtriple powerpc-ibm-aix-xcoff --relocation-model=pic < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mtriple powerpc64-ibm-aix-xcoff --relocation-model=pic < %s | FileCheck --check-prefix=CHECK64 %s
+
+@a = common global i32 0
+@b = constant i32* @a
+
+;CHECK:         .comm   a[RW],4,2
+;CHECK-NEXT:    .csect .data[RW]
+;CHECK-NEXT:    .globl  b
+;CHECK-NEXT:    .align  2
+;CHECK-NEXT: b:
+;CHECK-NEXT:    .long   a
+
+;CHECK64:       .comm   a[RW],4,2
+;CHECK64-NEXT:  .csect .data[RW]
+;CHECK64-NEXT:  .globl  b
+;CHECK64-NEXT:  .align  3
+;CHECK64-NEXT: b:
+;CHECK64-NEXT:  .llong  a
diff --git a/llvm/test/CodeGen/PowerPC/aix-reference-func-addr-const.ll b/llvm/test/CodeGen/PowerPC/aix-reference-func-addr-const.ll
new file mode 100644
index 000000000..88d5f3d14
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/aix-reference-func-addr-const.ll
@@ -0,0 +1,32 @@
+; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mtriple powerpc-ibm-aix-xcoff < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mtriple powerpc64-ibm-aix-xcoff < %s | FileCheck --check-prefix=CHECK64 %s
+
+@foo_ptr = global void (...)* @foo
+declare void @foo(...)
+
+@bar_ptr1 = global void (...)* bitcast (void ()* @bar to void (...)*)
+define void @bar() {
+entry:
+  ret void
+}
+
+
+;CHECK:          .csect .data[RW]
+;CHECK-NEXT:     .globl  foo_ptr
+;CHECK-NEXT:     .align  2
+;CHECK-NEXT:     foo_ptr:
+;CHECK-NEXT:     .long   foo[DS]
+;CHECK-NEXT:     .globl  bar_ptr1
+;CHECK-NEXT:     .align  2
+;CHECK-NEXT:     bar_ptr1:
+;CHECK-NEXT:     .long   bar[DS]
+
+;CHECK64:         .csect .data[RW]
+;CHECK64-NEXT:         .globl  foo_ptr
+;CHECK64-NEXT:         .align  3
+;CHECK64-NEXT:    foo_ptr:
+;CHECK64-NEXT:         .llong  foo[DS]
+;CHECK64-NEXT:         .globl  bar_ptr1
+;CHECK64-NEXT:         .align  3
+;CHECK64-NEXT:    bar_ptr1:
+;CHECK64-NEXT:         .llong  bar[DS]
diff --git a/llvm/test/CodeGen/PowerPC/aix-return55.ll b/llvm/test/CodeGen/PowerPC/aix-return55.ll
new file mode 100644
index 000000000..b16f2fa66
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/aix-return55.ll
@@ -0,0 +1,11 @@
+; RUN: llc -mcpu=pwr9 -mtriple=powerpc-ibm-aix-xcoff -verify-machineinstrs < %s | FileCheck %s
+
+
+define dso_local signext i32 @foo() {
+entry:
+  ret i32 55
+; CHECK-LABEL: .foo:
+; CHECK: li 3, 55
+; CHECK: blr
+}
+
diff --git a/llvm/test/CodeGen/PowerPC/aix-space.ll b/llvm/test/CodeGen/PowerPC/aix-space.ll
new file mode 100644
index 000000000..d681d636e
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/aix-space.ll
@@ -0,0 +1,17 @@
+; RUN: llc -verify-machineinstrs -O0 -mcpu=pwr7 -mtriple powerpc-ibm-aix-xcoff < %s | FileCheck %s
+
+@a = common global double 0.000000e+00, align 8
+
+; Get some constants into the constant pool that need spacing for alignment
+define void @e() {
+entry:
+  %0 = load double, double* @a, align 8
+  %mul = fmul double 1.500000e+00, %0
+  store double %mul, double* @a, align 8
+  %mul1 = fmul double 0x3F9C71C71C71C71C, %0
+  store double %mul1, double* @a, align 8
+  ret void
+}
+
+; CHECK:      .space 4
+; CHECK-NOT:  .zero
diff --git a/llvm/test/CodeGen/PowerPC/aix-sret-param.ll b/llvm/test/CodeGen/PowerPC/aix-sret-param.ll
new file mode 100644
index 000000000..0a2463ffd
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/aix-sret-param.ll
@@ -0,0 +1,98 @@
+; RUN: llc -mtriple powerpc-ibm-aix-xcoff -mcpu=pwr4 -mattr=-altivec \
+; RUN: -stop-after=machine-cp --verify-machineinstrs < %s | FileCheck \
+; RUN: --check-prefixes=MIR,MIR32 %s
+
+; RUN: llc -mtriple powerpc64-ibm-aix-xcoff -mcpu=pwr4 -mattr=-altivec \
+; RUN: -stop-after=machine-cp --verify-machineinstrs < %s | FileCheck \
+; RUN: --check-prefixes=MIR,MIR64 %s
+
+; RUN: llc -mtriple powerpc-ibm-aix-xcoff -mcpu=pwr4 -mattr=-altivec \
+; RUN: --verify-machineinstrs < %s | FileCheck --check-prefixes=ASM,ASM32 %s
+
+; RUN: llc -mtriple powerpc64-ibm-aix-xcoff -mcpu=pwr4 -mattr=-altivec \
+; RUN: --verify-machineinstrs < %s | FileCheck --check-prefixes=ASM,ASM64 %s
+
+%struct.S = type { i8 }
+%struct.T = type { double, i32, i32, i32, float }
+
+define void @test1() {
+entry:
+  %s = alloca %struct.S, align 4
+  call void @foo(%struct.S* sret %s)
+  ret void
+}
+
+define void @test2() {
+entry:
+  %t = alloca %struct.T, align 8
+  call void @bar(%struct.T* sret %t)
+  ret void
+}
+
+declare void @foo(%struct.S* sret)
+declare void @bar(%struct.T* sret)
+
+; MIR:      name:            test1
+; MIR:      stack:
+; MIR-NEXT:  - { id: 0, name: s, type: default, offset: 0, size: 1, alignment: 8,
+
+; MIR32:      bb.0.entry:
+; MIR32-NEXT:   ADJCALLSTACKDOWN 56, 0, implicit-def dead $r1, implicit $r1
+; MIR32-NEXT:   renamable $r3 = ADDI %stack.0.s, 0
+; MIR32-NEXT:   BL_NOP <mcsymbol .foo>, csr_aix32, implicit-def dead $lr, implicit $rm, implicit $r3, implicit $r2, implicit-def $r1
+; MIR32-NEXT:   ADJCALLSTACKUP 56, 0, implicit-def dead $r1, implicit $r1
+
+; MIR64:      bb.0.entry:
+; MIR64-NEXT:   ADJCALLSTACKDOWN 112, 0, implicit-def dead $r1, implicit $r1
+; MIR64-NEXT:   renamable $x3 = ADDI8 %stack.0.s, 0
+; MIR64-NEXT:   BL8_NOP <mcsymbol .foo>, csr_aix64, implicit-def dead $lr8, implicit $rm, implicit $x3, implicit $x2, implicit-def $r1
+; MIR64-NEXT:   ADJCALLSTACKUP 112, 0, implicit-def dead $r1, implicit $r1
+
+
+; ASM-LABEL: .test1:
+
+; ASM32:       stwu 1, -64(1)
+; ASM32-NEXT:  addi 3, 1, 56
+; ASM32-NEXT:  bl .foo
+; ASM32-NEXT:  nop
+; ASM32-NEXT:  addi 1, 1, 64
+
+; ASM64:       stdu 1, -128(1)
+; ASM64-NEXT:  addi 3, 1, 120
+; ASM64-NEXT:  bl .foo
+; ASM64-NEXT:  nop
+; ASM64-NEXT:  addi 1, 1, 128
+
+
+
+; MIR:      name:            test2
+; MIR:      stack:
+; MIR-NEXT:   - { id: 0, name: t, type: default, offset: 0, size: 24, alignment: 8,
+
+; MIR32:       bb.0.entry:
+; MIR32-NEXT:    ADJCALLSTACKDOWN 56, 0, implicit-def dead $r1, implicit $r1
+; MIR32-NEXT:    renamable $r3 = ADDI %stack.0.t, 0
+; MIR32-NEXT:    BL_NOP <mcsymbol .bar>, csr_aix32, implicit-def dead $lr, implicit $rm, implicit $r3, implicit $r2, implicit-def $r1
+; MIR32-NEXT:    ADJCALLSTACKUP 56, 0, implicit-def dead $r1, implicit $r1
+
+; MIR64:      bb.0.entry:
+; MIR64-NEXT:   ADJCALLSTACKDOWN 112, 0, implicit-def dead $r1, implicit $r1
+; MIR64-NEXT:   renamable $x3 = ADDI8 %stack.0.t, 0
+; MIR64-NEXT:   BL8_NOP <mcsymbol .bar>, csr_aix64, implicit-def dead $lr8, implicit $rm, implicit $x3, implicit $x2, implicit-def $r1
+; MIR64-NEXT:   ADJCALLSTACKUP 112, 0, implicit-def dead $r1, implicit $r1
+
+
+; ASM-LABEL: .test2:
+
+; ASM32:        stwu 1, -80(1)
+; ASM32-NEXT:   addi 3, 1, 56
+; ASM32-NEXT:   bl .bar
+; ASM32-NEXT:   nop
+; ASM32-NEXT:   addi 1, 1, 80
+
+
+; ASM64:        stdu 1, -144(1)
+; ASM64-NEXT:   addi 3, 1, 120
+; ASM64-NEXT:   bl .bar
+; ASM64-NEXT:   nop
+; ASM64-NEXT:   addi 1, 1, 144
diff --git a/llvm/test/CodeGen/PowerPC/aix-stackargs.ll b/llvm/test/CodeGen/PowerPC/aix-stackargs.ll
new file mode 100644
index 000000000..6cac691e0
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/aix-stackargs.ll
@@ -0,0 +1,12 @@
+; RUN: not llc -mtriple powerpc-ibm-aix-xcoff < %s 2>&1 | FileCheck %s
+; RUN: not llc -mtriple powerpc64-ibm-aix-xcoff < %s 2>&1 | FileCheck %s
+
+define void @bar() {
+entry:
+  call void @foo(i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9)
+  ret void
+}
+
+declare void @foo(i32, i32, i32, i32, i32, i32, i32, i32, i32)
+
+; CHECK: LLVM ERROR: Handling of placing parameters on the stack is unimplemented!
diff --git a/llvm/test/CodeGen/PowerPC/aix-trampoline.ll b/llvm/test/CodeGen/PowerPC/aix-trampoline.ll
new file mode 100644
index 000000000..5c45dc588
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/aix-trampoline.ll
@@ -0,0 +1,14 @@
+; RUN: not llc -mtriple powerpc-ibm-aix-xcoff < %s 2>&1 | FileCheck %s
+; RUN: not llc -mtriple powerpc64-ibm-aix-xcoff < %s 2>&1 | FileCheck %s
+
+; CHECK: LLVM ERROR: INIT_TRAMPOLINE operation is not supported on AIX.
+
+define void @create_trampoline(i8* %buffer, i8* %nval) nounwind {
+entry:
+  call void @llvm.init.trampoline(i8* %buffer, i8* bitcast (i32 (i32)* @nested to i8*) , i8* %nval)
+  ret void
+}
+
+declare i32 @nested(i32);
+
+declare void @llvm.init.trampoline(i8*, i8*, i8*) nounwind
diff --git a/llvm/test/CodeGen/PowerPC/aix-undef-func-call.ll b/llvm/test/CodeGen/PowerPC/aix-undef-func-call.ll
new file mode 100644
index 000000000..31d707c2a
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/aix-undef-func-call.ll
@@ -0,0 +1,29 @@
+; RUN: llc -mtriple powerpc-ibm-aix-xcoff -filetype=obj -o %t.o < %s
+; RUN: llvm-readobj  --symbols %t.o | FileCheck %s
+
+define void @bar() {
+entry:
+  call void bitcast (void (...)* @foo to void ()*)()
+  ret void
+}
+
+declare void @foo(...)
+
+;CHECK: Symbol {
+;CHECK:   Name: .foo
+;CHECK-NEXT:   Value (RelocatableAddress): 0x0
+;CHECK-NEXT:   Section: N_UNDEF
+;CHECK-NEXT:   Type: 0x0
+;CHECK-NEXT:   StorageClass: C_EXT (0x2)
+;CHECK-NEXT:   NumberOfAuxEntries: 1
+;CHECK-NEXT:   CSECT Auxiliary Entry {
+;CHECK:          SectionLen: 0
+;CHECK-NEXT:     ParameterHashIndex: 0x0
+;CHECK-NEXT:     TypeChkSectNum: 0x0
+;CHECK-NEXT:     SymbolAlignmentLog2: 0
+;CHECK-NEXT:     SymbolType: XTY_ER (0x0)
+;CHECK-NEXT:     StorageMappingClass: XMC_PR (0x0)
+;CHECK-NEXT:     StabInfoIndex: 0x0
+;CHECK-NEXT:     StabSectNum: 0x0
+;CHECK-NEXT:   }
+;CHECK-NEXT: }
diff --git a/llvm/test/CodeGen/PowerPC/aix-user-defined-memcpy.ll b/llvm/test/CodeGen/PowerPC/aix-user-defined-memcpy.ll
new file mode 100644
index 000000000..0b892a3de
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/aix-user-defined-memcpy.ll
@@ -0,0 +1,58 @@
+; RUN: llc -verify-machineinstrs -mtriple powerpc-ibm-aix-xcoff -mcpu=pwr4 \
+; RUN: -mattr=-altivec -filetype=obj -o %t.o < %s
+
+; RUN: llvm-readobj --syms %t.o | FileCheck --check-prefix=32-SYM %s
+
+; RUN: llvm-readobj --relocs --expand-relocs %t.o | FileCheck \
+; RUN: --check-prefix=32-REL %s
+
+; RUN: not llc -verify-machineinstrs -mtriple powerpc64-ibm-aix-xcoff \
+; RUN: -mcpu=pwr4 -mattr=-altivec -filetype=obj < %s 2>&1 | FileCheck \
+; RUN: --check-prefix=64-CHECK %s
+
+; Test verifies:
+; If there exists a user-defined function whose name is the same as the
+; "memcpy" ExternalSymbol's, we pick up the user-defined version, even if this
+; may lead to some undefined behavior.
+
+define dso_local signext i32 @memcpy(i8* %destination, i32 signext %num) {
+entry:
+  ret i32 3
+}
+
+define void @call_memcpy(i8* %p, i8* %q, i32 %n) {
+entry:
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %p, i8* %q, i32 %n, i1 false)
+  ret void
+}
+
+declare void @llvm.memcpy.p0i8.p0i8.i32(i8* nocapture writeonly, i8* nocapture readonly, i32, i1)
+
+; TODO: This test should preferably check the symbol table for .o file and
+;       the relocation associated with the call.
+
+; 32-SYM:      Symbol {{[{][[:space:]] *}}Index: [[#Index:]]{{[[:space:]] *}}Name: .memcpy 
+; 32-SYM-NEXT:    Value (RelocatableAddress): 0x0
+; 32-SYM-NEXT:    Section: .text
+; 32-SYM-NEXT:    Type: 0x0
+; 32-SYM-NEXT:    StorageClass: C_EXT (0x2)
+; 32-SYM-NEXT:    NumberOfAuxEntries: 1
+; 32-SYM-NEXT:    CSECT Auxiliary Entry {
+; 32-SYM-NEXT:      Index: 3
+; 32-SYM-NEXT:      ContainingCsectSymbolIndex: 0
+; 32-SYM-NEXT:      ParameterHashIndex: 0x0
+; 32-SYM-NEXT:      TypeChkSectNum: 0x0
+; 32-SYM-NEXT:      SymbolAlignmentLog2: 0
+; 32-SYM-NEXT:      SymbolType: XTY_LD (0x2)
+; 32-SYM-NEXT:      StorageMappingClass: XMC_PR (0x0)
+; 32-SYM-NEXT:      StabInfoIndex: 0x0
+; 32-SYM-NEXT:      StabSectNum: 0x0
+; 32-SYM-NEXT:    }
+; 32-SYM-NEXT:  }
+
+; 32-SYM-NOT: .memcpy
+
+; We are expecting to have the test fail when the support for relocations land.
+; 32-REL-NOT: Relocation{{[[:space:]]}}
+
+; 64-CHECK: LLVM ERROR: 64-bit XCOFF object files are not supported yet.
diff --git a/llvm/test/CodeGen/PowerPC/aix-weak-undef-func-call.ll b/llvm/test/CodeGen/PowerPC/aix-weak-undef-func-call.ll
new file mode 100644
index 000000000..9fb3dec19
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/aix-weak-undef-func-call.ll
@@ -0,0 +1,30 @@
+; RUN: llc -mtriple powerpc-ibm-aix-xcoff -filetype=obj -o %t.o < %s
+; RUN: llvm-readobj  --symbols %t.o | FileCheck %s
+
+define void @bar() {
+entry:
+  call void bitcast (void (...)* @foo to void ()*)()
+  ret void
+}
+
+declare extern_weak void @foo(...) 
+
+;CHECK: Symbol {
+;CHECK:   Name: .foo
+;CHECK-NEXT:   Value (RelocatableAddress): 0x0
+;CHECK-NEXT:   Section: N_UNDEF
+;CHECK-NEXT:   Type: 0x0
+;CHECK-NEXT:   StorageClass: C_WEAKEXT (0x6F)
+;CHECK-NEXT:   NumberOfAuxEntries: 1
+;CHECK-NEXT:   CSECT Auxiliary Entry {
+;CHECK:          SectionLen: 0
+;CHECK-NEXT:     ParameterHashIndex: 0x0
+;CHECK-NEXT:     TypeChkSectNum: 0x0
+;CHECK-NEXT:     SymbolAlignmentLog2: 0
+;CHECK-NEXT:     SymbolType: XTY_ER (0x0)
+;CHECK-NEXT:     StorageMappingClass: XMC_PR (0x0)
+;CHECK-NEXT:     StabInfoIndex: 0x0
+;CHECK-NEXT:     StabSectNum: 0x0
+;CHECK-NEXT:   }
+;CHECK-NEXT: }
+
diff --git a/llvm/test/CodeGen/PowerPC/aix-xcoff-data-only-notoc.ll b/llvm/test/CodeGen/PowerPC/aix-xcoff-data-only-notoc.ll
new file mode 100644
index 000000000..d6e772ffc
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/aix-xcoff-data-only-notoc.ll
@@ -0,0 +1,19 @@
+; RUN: llc -mtriple powerpc-ibm-aix-xcoff < %s | FileCheck %s
+; RUN: llc -mtriple powerpc64-ibm-aix-xcoff < %s 2>&1 | FileCheck %s
+
+; RUN: llc -mtriple powerpc-ibm-aix-xcoff -filetype=obj -o %t.o < %s
+; RUN: llvm-readobj --syms %t.o | FileCheck --check-prefix=SYMS %s
+
+; RUN: not llc -mtriple powerpc64-ibm-aix-xcoff -filetype=obj < %s 2>&1 | \
+; RUN: FileCheck --check-prefix=OBJ64 %s
+; OBJ64: LLVM ERROR: 64-bit XCOFF object files are not supported yet.
+
+@a = external global i32, align 4
+@b = external global i64, align 8
+@c = external global i16, align 2
+@globa = common global i32 0, align 4
+
+@ptr = internal global void (...)* null, align 4
+
+; CHECK-NOT: .toc
+; SYMS-NOT: Name: TOC
diff --git a/llvm/test/CodeGen/PowerPC/aix-xcoff-data.ll b/llvm/test/CodeGen/PowerPC/aix-xcoff-data.ll
new file mode 100644
index 000000000..c6a31ae80
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/aix-xcoff-data.ll
@@ -0,0 +1,595 @@
+; This file tests the codegen of initialized and common variables in AIX
+; assembly and XCOFF object files.
+
+; RUN: llc -mtriple powerpc-ibm-aix-xcoff < %s | FileCheck %s
+; RUN: llc -mtriple powerpc64-ibm-aix-xcoff < %s | FileCheck %s
+
+; RUN: llc -mtriple powerpc-ibm-aix-xcoff -filetype=obj -o %t.o < %s
+; RUN: llvm-readobj --section-headers --file-header %t.o | \
+; RUN: FileCheck --check-prefix=OBJ %s
+; RUN: llvm-readobj --syms %t.o | FileCheck --check-prefix=SYMS %s
+
+; RUN: not llc -mtriple powerpc64-ibm-aix-xcoff -filetype=obj < %s 2>&1 | \
+; RUN: FileCheck --check-prefix=XCOFF64 %s
+; XCOFF64: LLVM ERROR: 64-bit XCOFF object files are not supported yet.
+
+@ivar = local_unnamed_addr global i32 35, align 4
+@llvar = local_unnamed_addr global i64 36, align 8
+@svar = local_unnamed_addr global i16 37, align 2
+@fvar = local_unnamed_addr global float 8.000000e+02, align 4
+@dvar = local_unnamed_addr global double 9.000000e+02, align 8
+@over_aligned = local_unnamed_addr global double 9.000000e+02, align 32
+@chrarray = local_unnamed_addr global [4 x i8] c"abcd", align 1
+@dblarr = local_unnamed_addr global [4 x double] [double 1.000000e+00, double 2.000000e+00, double 3.000000e+00, double 4.000000e+00], align 8
+@d_0 = global double 0.000000e+00, align 8
+@s_0 = global i16 0, align 2
+@f_0 = global float 0.000000e+00, align 4
+
+@a = common global i32 0, align 4
+@b = common global i64 0, align 8
+@c = common global i16 0, align 2
+
+@d = common local_unnamed_addr global double 0.000000e+00, align 8
+@f = common local_unnamed_addr global float 0.000000e+00, align 4
+
+@over_aligned_comm = common local_unnamed_addr global double 0.000000e+00, align 32
+
+@array = common local_unnamed_addr global [33 x i8] zeroinitializer, align 1
+
+; CHECK-NOT: .toc
+
+; CHECK:      .csect .text[PR]
+; CHECK-NEXT:  .file
+
+; CHECK:      .csect .data[RW]
+; CHECK-NEXT: .globl  ivar
+; CHECK-NEXT: .align  2
+; CHECK-NEXT: ivar:
+; CHECK-NEXT: .long   35
+
+; CHECK:      .globl  llvar
+; CHECK-NEXT: .align  3
+; CHECK-NEXT: llvar:
+; CHECK-NEXT: .llong  36
+
+; CHECK:      .globl  svar
+; CHECK-NEXT: .align  1
+; CHECK-NEXT: svar:
+; CHECK-NEXT: .short  37
+
+; CHECK:      .globl  fvar
+; CHECK-NEXT: .align  2
+; CHECK-NEXT: fvar:
+; CHECK-NEXT: .long   1145569280
+
+; CHECK:      .globl  dvar
+; CHECK-NEXT: .align  3
+; CHECK-NEXT: dvar:
+; CHECK-NEXT: .llong  4651127699538968576
+
+; CHECK:      .globl  over_aligned
+; CHECK-NEXT: .align  5
+; CHECK-NEXT: over_aligned:
+; CHECK-NEXT: .llong  4651127699538968576
+
+; CHECK:      .globl  chrarray
+; CHECK-NEXT: chrarray:
+; CHECK-NEXT: .byte   97
+; CHECK-NEXT: .byte   98
+; CHECK-NEXT: .byte   99
+; CHECK-NEXT: .byte   100
+
+; CHECK:      .globl  dblarr
+; CHECK-NEXT: .align  3
+; CHECK-NEXT: dblarr:
+; CHECK-NEXT: .llong  4607182418800017408
+; CHECK-NEXT: .llong  4611686018427387904
+; CHECK-NEXT: .llong  4613937818241073152
+; CHECK-NEXT: .llong  4616189618054758400
+
+; CHECK:      .globl  d_0
+; CHECK-NEXT: .align 3
+; CHECK-NEXT: d_0:
+; CHECK-NEXT: .llong 0
+
+; CHECK:      .globl  s_0
+; CHECK-NEXT: .align  1
+; CHECK-NEXT: s_0:
+; CHECK-NEXT: .short 0
+
+; CHECK:      .globl f_0
+; CHECK-NEXT: .align 2
+; CHECK-NEXT: f_0:
+; CHECK-NEXT: .long 0
+
+; CHECK-NEXT: .comm   a[RW],4,2
+; CHECK-NEXT: .comm   b[RW],8,3
+; CHECK-NEXT: .comm   c[RW],2,1
+; CHECK-NEXT: .comm   d[RW],8,3
+; CHECK-NEXT: .comm   f[RW],4,2
+; CHECK-NEXT: .comm   over_aligned_comm[RW],8,5
+; CHECK-NEXT: .comm   array[RW],33,0
+
+; OBJ:      File: {{.*}}aix-xcoff-data.ll.tmp.o
+; OBJ-NEXT: Format: aixcoff-rs6000
+; OBJ-NEXT: Arch: powerpc
+; OBJ-NEXT: AddressSize: 32bit
+; OBJ-NEXT: FileHeader {
+; OBJ-NEXT:   Magic: 0x1DF
+; OBJ-NEXT:   NumberOfSections: 3
+; OBJ-NEXT:   TimeStamp:
+; OBJ-NEXT:   SymbolTableOffset: 0xEC
+; OBJ-NEXT:   SymbolTableEntries: 40
+; OBJ-NEXT:   OptionalHeaderSize: 0x0
+; OBJ-NEXT:   Flags: 0x0
+; OBJ-NEXT: }
+
+; OBJ:      Sections [
+; OBJ:        Section {
+; OBJ-NEXT:     Index: 1
+; OBJ-NEXT:     Name: .text
+; OBJ-NEXT:     PhysicalAddress: 0x0
+; OBJ-NEXT:     VirtualAddress: 0x0
+; OBJ-NEXT:     Size: 0x0
+; OBJ-NEXT:     RawDataOffset: 0x8C
+; OBJ-NEXT:     RelocationPointer: 0x0
+; OBJ-NEXT:     LineNumberPointer: 0x0
+; OBJ-NEXT:     NumberOfRelocations: 0
+; OBJ-NEXT:     NumberOfLineNumbers: 0
+; OBJ-NEXT:     Type: STYP_TEXT (0x20)
+; OBJ-NEXT:   }
+
+; OBJ:        Section {
+; OBJ-NEXT:     Index: 2
+; OBJ-NEXT:     Name: .data
+; OBJ-NEXT:     PhysicalAddress: 0x0
+; OBJ-NEXT:     VirtualAddress: 0x0
+; OBJ-NEXT:     Size: 0x60
+; OBJ-NEXT:     RawDataOffset: 0x8C
+; OBJ-NEXT:     RelocationPointer: 0x0
+; OBJ-NEXT:     LineNumberPointer: 0x0
+; OBJ-NEXT:     NumberOfRelocations: 0
+; OBJ-NEXT:     NumberOfLineNumbers: 0
+; OBJ-NEXT:     Type: STYP_DATA (0x40)
+; OBJ-NEXT:   }
+
+; OBJ:        Section {
+; OBJ-NEXT:     Index: 3
+; OBJ-NEXT:     Name: .bss
+; OBJ-NEXT:     PhysicalAddress: 0x60
+; OBJ-NEXT:     VirtualAddress: 0x60
+; OBJ-NEXT:     Size: 0x6C
+; OBJ-NEXT:     RawDataOffset: 0x0
+; OBJ-NEXT:     RelocationPointer: 0x0
+; OBJ-NEXT:     LineNumberPointer: 0x0
+; OBJ-NEXT:     NumberOfRelocations: 0
+; OBJ-NEXT:     NumberOfLineNumbers: 0
+; OBJ-NEXT:     Type: STYP_BSS (0x80)
+; OBJ-NEXT:   }
+; OBJ:      ]
+
+; SYMS:      File: {{.*}}aix-xcoff-data.ll.tmp.o
+; SYMS-NEXT: Format: aixcoff-rs6000
+; SYMS-NEXT: Arch: powerpc
+; SYMS-NEXT: AddressSize: 32bit
+; SYMS:      Symbols [
+; SYMS:        Symbol {
+; SYMS-NEXT:     Index: [[#INDX:]]
+; SYMS-NEXT:     Name: .text
+; SYMS-NEXT:     Value (RelocatableAddress): 0x0
+; SYMS-NEXT:     Section: .text
+; SYMS-NEXT:     Type: 0x0
+; SYMS-NEXT:     StorageClass: C_HIDEXT (0x6B)
+; SYMS-NEXT:     NumberOfAuxEntries: 1
+; SYMS-NEXT:     CSECT Auxiliary Entry {
+; SYMS-NEXT:       Index: [[#INDX+1]]
+; SYMS-NEXT:       SectionLen: 0
+; SYMS-NEXT:       ParameterHashIndex: 0x0
+; SYMS-NEXT:       TypeChkSectNum: 0x0
+; SYMS-NEXT:       SymbolAlignmentLog2: 0
+; SYMS-NEXT:       SymbolType: XTY_SD (0x1)
+; SYMS-NEXT:       StorageMappingClass: XMC_PR (0x0)
+; SYMS-NEXT:       StabInfoIndex: 0x0
+; SYMS-NEXT:       StabSectNum: 0x0
+; SYMS-NEXT:     }
+; SYMS-NEXT:   }
+
+; SYMS:        Symbol {
+; SYMS-NEXT:     Index: [[#INDX+2]]
+; SYMS-NEXT:     Name: .data
+; SYMS-NEXT:     Value (RelocatableAddress): 0x0
+; SYMS-NEXT:     Section: .data
+; SYMS-NEXT:     Type: 0x0
+; SYMS-NEXT:     StorageClass: C_HIDEXT (0x6B)
+; SYMS-NEXT:     NumberOfAuxEntries: 1
+; SYMS-NEXT:     CSECT Auxiliary Entry {
+; SYMS-NEXT:       Index: [[#INDX+3]]
+; SYMS-NEXT:       SectionLen: 96
+; SYMS-NEXT:       ParameterHashIndex: 0x0
+; SYMS-NEXT:       TypeChkSectNum: 0x0
+; SYMS-NEXT:       SymbolAlignmentLog2: 5
+; SYMS-NEXT:       SymbolType: XTY_SD (0x1)
+; SYMS-NEXT:       StorageMappingClass: XMC_RW (0x5)
+; SYMS-NEXT:       StabInfoIndex: 0x0
+; SYMS-NEXT:       StabSectNum: 0x0
+; SYMS-NEXT:     }
+; SYMS-NEXT:   }
+
+; SYMS:        Symbol {
+; SYMS-NEXT:     Index: [[#INDX+4]]
+; SYMS-NEXT:     Name: ivar
+; SYMS-NEXT:     Value (RelocatableAddress): 0x0
+; SYMS-NEXT:     Section: .data
+; SYMS-NEXT:     Type: 0x0
+; SYMS-NEXT:     StorageClass: C_EXT (0x2)
+; SYMS-NEXT:     NumberOfAuxEntries: 1
+; SYMS-NEXT:     CSECT Auxiliary Entry {
+; SYMS-NEXT:       Index: [[#INDX+5]]
+; SYMS-NEXT:       ContainingCsectSymbolIndex: [[#INDX+2]]
+; SYMS-NEXT:       ParameterHashIndex: 0x0
+; SYMS-NEXT:       TypeChkSectNum: 0x0
+; SYMS-NEXT:       SymbolAlignmentLog2: 0
+; SYMS-NEXT:       SymbolType: XTY_LD (0x2)
+; SYMS-NEXT:       StorageMappingClass: XMC_RW (0x5)
+; SYMS-NEXT:       StabInfoIndex: 0x0
+; SYMS-NEXT:       StabSectNum: 0x0
+; SYMS-NEXT:     }
+; SYMS-NEXT:   }
+
+; SYMS:        Symbol {
+; SYMS-NEXT:     Index: [[#INDX+6]]
+; SYMS-NEXT:     Name: llvar
+; SYMS-NEXT:     Value (RelocatableAddress): 0x8
+; SYMS-NEXT:     Section: .data
+; SYMS-NEXT:     Type: 0x0
+; SYMS-NEXT:     StorageClass: C_EXT (0x2)
+; SYMS-NEXT:     NumberOfAuxEntries: 1
+; SYMS-NEXT:     CSECT Auxiliary Entry {
+; SYMS-NEXT:       Index: [[#INDX+7]]
+; SYMS-NEXT:       ContainingCsectSymbolIndex: [[#INDX+2]]
+; SYMS-NEXT:       ParameterHashIndex: 0x0
+; SYMS-NEXT:       TypeChkSectNum: 0x0
+; SYMS-NEXT:       SymbolAlignmentLog2: 0
+; SYMS-NEXT:       SymbolType: XTY_LD (0x2)
+; SYMS-NEXT:       StorageMappingClass: XMC_RW (0x5)
+; SYMS-NEXT:       StabInfoIndex: 0x0
+; SYMS-NEXT:       StabSectNum: 0x0
+; SYMS-NEXT:     }
+; SYMS-NEXT:   }
+
+; SYMS:        Symbol {
+; SYMS-NEXT:     Index: [[#INDX+8]]
+; SYMS-NEXT:     Name: svar
+; SYMS-NEXT:     Value (RelocatableAddress): 0x10
+; SYMS-NEXT:     Section: .data
+; SYMS-NEXT:     Type: 0x0
+; SYMS-NEXT:     StorageClass: C_EXT (0x2)
+; SYMS-NEXT:     NumberOfAuxEntries: 1
+; SYMS-NEXT:     CSECT Auxiliary Entry {
+; SYMS-NEXT:       Index: [[#INDX+9]]
+; SYMS-NEXT:       ContainingCsectSymbolIndex: [[#INDX+2]]
+; SYMS-NEXT:       ParameterHashIndex: 0x0
+; SYMS-NEXT:       TypeChkSectNum: 0x0
+; SYMS-NEXT:       SymbolAlignmentLog2: 0
+; SYMS-NEXT:       SymbolType: XTY_LD (0x2)
+; SYMS-NEXT:       StorageMappingClass: XMC_RW (0x5)
+; SYMS-NEXT:       StabInfoIndex: 0x0
+; SYMS-NEXT:       StabSectNum: 0x0
+; SYMS-NEXT:     }
+; SYMS-NEXT:   }
+
+; SYMS:        Symbol {
+; SYMS-NEXT:     Index: [[#INDX+10]]
+; SYMS-NEXT:     Name: fvar
+; SYMS-NEXT:     Value (RelocatableAddress): 0x14
+; SYMS-NEXT:     Section: .data
+; SYMS-NEXT:     Type: 0x0
+; SYMS-NEXT:     StorageClass: C_EXT (0x2)
+; SYMS-NEXT:     NumberOfAuxEntries: 1
+; SYMS-NEXT:     CSECT Auxiliary Entry {
+; SYMS-NEXT:       Index: [[#INDX+11]]
+; SYMS-NEXT:       ContainingCsectSymbolIndex: [[#INDX+2]]
+; SYMS-NEXT:       ParameterHashIndex: 0x0
+; SYMS-NEXT:       TypeChkSectNum: 0x0
+; SYMS-NEXT:       SymbolAlignmentLog2: 0
+; SYMS-NEXT:       SymbolType: XTY_LD (0x2)
+; SYMS-NEXT:       StorageMappingClass: XMC_RW (0x5)
+; SYMS-NEXT:       StabInfoIndex: 0x0
+; SYMS-NEXT:       StabSectNum: 0x0
+; SYMS-NEXT:     }
+; SYMS-NEXT:   }
+
+; SYMS:        Symbol {
+; SYMS-NEXT:     Index: [[#INDX+12]]
+; SYMS-NEXT:     Name: dvar
+; SYMS-NEXT:     Value (RelocatableAddress): 0x18
+; SYMS-NEXT:     Section: .data
+; SYMS-NEXT:     Type: 0x0
+; SYMS-NEXT:     StorageClass: C_EXT (0x2)
+; SYMS-NEXT:     NumberOfAuxEntries: 1
+; SYMS-NEXT:     CSECT Auxiliary Entry {
+; SYMS-NEXT:       Index: [[#INDX+13]]
+; SYMS-NEXT:       ContainingCsectSymbolIndex: [[#INDX+2]]
+; SYMS-NEXT:       ParameterHashIndex: 0x0
+; SYMS-NEXT:       TypeChkSectNum: 0x0
+; SYMS-NEXT:       SymbolAlignmentLog2: 0
+; SYMS-NEXT:       SymbolType: XTY_LD (0x2)
+; SYMS-NEXT:       StorageMappingClass: XMC_RW (0x5)
+; SYMS-NEXT:       StabInfoIndex: 0x0
+; SYMS-NEXT:       StabSectNum: 0x0
+; SYMS-NEXT:     }
+; SYMS-NEXT:   }
+
+; SYMS:        Symbol {
+; SYMS-NEXT:     Index: [[#INDX+14]]
+; SYMS-NEXT:     Name: over_aligned
+; SYMS-NEXT:     Value (RelocatableAddress): 0x20
+; SYMS-NEXT:     Section: .data
+; SYMS-NEXT:     Type: 0x0
+; SYMS-NEXT:     StorageClass: C_EXT (0x2)
+; SYMS-NEXT:     NumberOfAuxEntries: 1
+; SYMS-NEXT:     CSECT Auxiliary Entry {
+; SYMS-NEXT:       Index: [[#INDX+15]]
+; SYMS-NEXT:       ContainingCsectSymbolIndex: [[#INDX+2]]
+; SYMS-NEXT:       ParameterHashIndex: 0x0
+; SYMS-NEXT:       TypeChkSectNum: 0x0
+; SYMS-NEXT:       SymbolAlignmentLog2: 0
+; SYMS-NEXT:       SymbolType: XTY_LD (0x2)
+; SYMS-NEXT:       StorageMappingClass: XMC_RW (0x5)
+; SYMS-NEXT:       StabInfoIndex: 0x0
+; SYMS-NEXT:       StabSectNum: 0x0
+; SYMS-NEXT:     }
+; SYMS-NEXT:   }
+
+; SYMS:        Symbol {
+; SYMS-NEXT:     Index: [[#INDX+16]]
+; SYMS-NEXT:     Name: chrarray
+; SYMS-NEXT:     Value (RelocatableAddress): 0x28
+; SYMS-NEXT:     Section: .data
+; SYMS-NEXT:     Type: 0x0
+; SYMS-NEXT:     StorageClass: C_EXT (0x2)
+; SYMS-NEXT:     NumberOfAuxEntries: 1
+; SYMS-NEXT:     CSECT Auxiliary Entry {
+; SYMS-NEXT:       Index: [[#INDX+17]]
+; SYMS-NEXT:       ContainingCsectSymbolIndex: [[#INDX+2]]
+; SYMS-NEXT:       ParameterHashIndex: 0x0
+; SYMS-NEXT:       TypeChkSectNum: 0x0
+; SYMS-NEXT:       SymbolAlignmentLog2: 0
+; SYMS-NEXT:       SymbolType: XTY_LD (0x2)
+; SYMS-NEXT:       StorageMappingClass: XMC_RW (0x5)
+; SYMS-NEXT:       StabInfoIndex: 0x0
+; SYMS-NEXT:       StabSectNum: 0x0
+; SYMS-NEXT:     }
+; SYMS-NEXT:   }
+
+; SYMS:        Symbol {
+; SYMS-NEXT:     Index: [[#INDX+18]]
+; SYMS-NEXT:     Name: dblarr
+; SYMS-NEXT:     Value (RelocatableAddress): 0x30
+; SYMS-NEXT:     Section: .data
+; SYMS-NEXT:     Type: 0x0
+; SYMS-NEXT:     StorageClass: C_EXT (0x2)
+; SYMS-NEXT:     NumberOfAuxEntries: 1
+; SYMS-NEXT:     CSECT Auxiliary Entry {
+; SYMS-NEXT:       Index: [[#INDX+19]]
+; SYMS-NEXT:       ContainingCsectSymbolIndex: [[#INDX+2]]
+; SYMS-NEXT:       ParameterHashIndex: 0x0
+; SYMS-NEXT:       TypeChkSectNum: 0x0
+; SYMS-NEXT:       SymbolAlignmentLog2: 0
+; SYMS-NEXT:       SymbolType: XTY_LD (0x2)
+; SYMS-NEXT:       StorageMappingClass: XMC_RW (0x5)
+; SYMS-NEXT:       StabInfoIndex: 0x0
+; SYMS-NEXT:       StabSectNum: 0x0
+; SYMS-NEXT:     }
+; SYMS-NEXT:   }
+
+; SYMS:        Symbol {
+; SYMS-NEXT:     Index: [[#INDX+20]]
+; SYMS-NEXT:     Name: d_0
+; SYMS-NEXT:     Value (RelocatableAddress): 0x50
+; SYMS-NEXT:     Section: .data
+; SYMS-NEXT:     Type: 0x0
+; SYMS-NEXT:     StorageClass: C_EXT (0x2)
+; SYMS-NEXT:     NumberOfAuxEntries: 1
+; SYMS-NEXT:     CSECT Auxiliary Entry {
+; SYMS-NEXT:       Index: [[#INDX+21]]
+; SYMS-NEXT:       ContainingCsectSymbolIndex: [[#INDX+2]]
+; SYMS-NEXT:       ParameterHashIndex: 0x0
+; SYMS-NEXT:       TypeChkSectNum: 0x0
+; SYMS-NEXT:       SymbolAlignmentLog2: 0
+; SYMS-NEXT:       SymbolType: XTY_LD (0x2)
+; SYMS-NEXT:       StorageMappingClass: XMC_RW (0x5)
+; SYMS-NEXT:       StabInfoIndex: 0x0
+; SYMS-NEXT:       StabSectNum: 0x0
+; SYMS-NEXT:     }
+; SYMS-NEXT:   }
+
+; SYMS:        Symbol {
+; SYMS-NEXT:     Index: [[#INDX+22]]
+; SYMS-NEXT:     Name: s_0
+; SYMS-NEXT:     Value (RelocatableAddress): 0x58
+; SYMS-NEXT:     Section: .data
+; SYMS-NEXT:     Type: 0x0
+; SYMS-NEXT:     StorageClass: C_EXT (0x2)
+; SYMS-NEXT:     NumberOfAuxEntries: 1
+; SYMS-NEXT:     CSECT Auxiliary Entry {
+; SYMS-NEXT:       Index: [[#INDX+23]]
+; SYMS-NEXT:       ContainingCsectSymbolIndex: [[#INDX+2]]
+; SYMS-NEXT:       ParameterHashIndex: 0x0
+; SYMS-NEXT:       TypeChkSectNum: 0x0
+; SYMS-NEXT:       SymbolAlignmentLog2: 0
+; SYMS-NEXT:       SymbolType: XTY_LD (0x2)
+; SYMS-NEXT:       StorageMappingClass: XMC_RW (0x5)
+; SYMS-NEXT:       StabInfoIndex: 0x0
+; SYMS-NEXT:       StabSectNum: 0x0
+; SYMS-NEXT:     }
+; SYMS-NEXT:   }
+
+; SYMS:        Symbol {
+; SYMS-NEXT:     Index: [[#INDX+24]]
+; SYMS-NEXT:     Name: f_0
+; SYMS-NEXT:     Value (RelocatableAddress): 0x5C
+; SYMS-NEXT:     Section: .data
+; SYMS-NEXT:     Type: 0x0
+; SYMS-NEXT:     StorageClass: C_EXT (0x2)
+; SYMS-NEXT:     NumberOfAuxEntries: 1
+; SYMS-NEXT:     CSECT Auxiliary Entry {
+; SYMS-NEXT:       Index: [[#INDX+25]]
+; SYMS-NEXT:       ContainingCsectSymbolIndex: [[#INDX+2]]
+; SYMS-NEXT:       ParameterHashIndex: 0x0
+; SYMS-NEXT:       TypeChkSectNum: 0x0
+; SYMS-NEXT:       SymbolAlignmentLog2: 0
+; SYMS-NEXT:       SymbolType: XTY_LD (0x2)
+; SYMS-NEXT:       StorageMappingClass: XMC_RW (0x5)
+; SYMS-NEXT:       StabInfoIndex: 0x0
+; SYMS-NEXT:       StabSectNum: 0x0
+; SYMS-NEXT:     }
+; SYMS-NEXT:   }
+
+; SYMS:        Symbol {
+; SYMS-NEXT:     Index: [[#INDX+26]]
+; SYMS-NEXT:     Name: a
+; SYMS-NEXT:     Value (RelocatableAddress): 0x60
+; SYMS-NEXT:     Section: .bss
+; SYMS-NEXT:     Type: 0x0
+; SYMS-NEXT:     StorageClass: C_EXT (0x2)
+; SYMS-NEXT:     NumberOfAuxEntries: 1
+; SYMS-NEXT:     CSECT Auxiliary Entry {
+; SYMS-NEXT:       Index: [[#INDX+27]]
+; SYMS-NEXT:       SectionLen: 4
+; SYMS-NEXT:       ParameterHashIndex: 0x0
+; SYMS-NEXT:       TypeChkSectNum: 0x0
+; SYMS-NEXT:       SymbolAlignmentLog2: 2
+; SYMS-NEXT:       SymbolType: XTY_CM (0x3)
+; SYMS-NEXT:       StorageMappingClass: XMC_RW (0x5)
+; SYMS-NEXT:       StabInfoIndex: 0x0
+; SYMS-NEXT:       StabSectNum: 0x0
+; SYMS-NEXT:     }
+; SYMS-NEXT:   }
+
+; SYMS:        Symbol {
+; SYMS-NEXT:     Index: [[#INDX+28]]
+; SYMS-NEXT:     Name: b
+; SYMS-NEXT:     Value (RelocatableAddress): 0x68
+; SYMS-NEXT:     Section: .bss
+; SYMS-NEXT:     Type: 0x0
+; SYMS-NEXT:     StorageClass: C_EXT (0x2)
+; SYMS-NEXT:     NumberOfAuxEntries: 1
+; SYMS-NEXT:     CSECT Auxiliary Entry {
+; SYMS-NEXT:       Index: [[#INDX+29]]
+; SYMS-NEXT:       SectionLen: 8
+; SYMS-NEXT:       ParameterHashIndex: 0x0
+; SYMS-NEXT:       TypeChkSectNum: 0x0
+; SYMS-NEXT:       SymbolAlignmentLog2: 3
+; SYMS-NEXT:       SymbolType: XTY_CM (0x3)
+; SYMS-NEXT:       StorageMappingClass: XMC_RW (0x5)
+; SYMS-NEXT:       StabInfoIndex: 0x0
+; SYMS-NEXT:       StabSectNum: 0x0
+; SYMS-NEXT:     }
+; SYMS-NEXT:   }
+
+; SYMS:        Symbol {
+; SYMS-NEXT:     Index: [[#INDX+30]]
+; SYMS-NEXT:     Name: c
+; SYMS-NEXT:     Value (RelocatableAddress): 0x70
+; SYMS-NEXT:     Section: .bss
+; SYMS-NEXT:     Type: 0x0
+; SYMS-NEXT:     StorageClass: C_EXT (0x2)
+; SYMS-NEXT:     NumberOfAuxEntries: 1
+; SYMS-NEXT:     CSECT Auxiliary Entry {
+; SYMS-NEXT:       Index: [[#INDX+31]]
+; SYMS-NEXT:       SectionLen: 2
+; SYMS-NEXT:       ParameterHashIndex: 0x0
+; SYMS-NEXT:       TypeChkSectNum: 0x0
+; SYMS-NEXT:       SymbolAlignmentLog2: 1
+; SYMS-NEXT:       SymbolType: XTY_CM (0x3)
+; SYMS-NEXT:       StorageMappingClass: XMC_RW (0x5)
+; SYMS-NEXT:       StabInfoIndex: 0x0
+; SYMS-NEXT:       StabSectNum: 0x0
+; SYMS-NEXT:     }
+; SYMS-NEXT:   }
+
+; SYMS:        Symbol {
+; SYMS-NEXT:     Index: [[#INDX+32]]
+; SYMS-NEXT:     Name: d
+; SYMS-NEXT:     Value (RelocatableAddress): 0x78
+; SYMS-NEXT:     Section: .bss
+; SYMS-NEXT:     Type: 0x0
+; SYMS-NEXT:     StorageClass: C_EXT (0x2)
+; SYMS-NEXT:     NumberOfAuxEntries: 1
+; SYMS-NEXT:     CSECT Auxiliary Entry {
+; SYMS-NEXT:       Index: [[#INDX+33]]
+; SYMS-NEXT:       SectionLen: 8
+; SYMS-NEXT:       ParameterHashIndex: 0x0
+; SYMS-NEXT:       TypeChkSectNum: 0x0
+; SYMS-NEXT:       SymbolAlignmentLog2: 3
+; SYMS-NEXT:       SymbolType: XTY_CM (0x3)
+; SYMS-NEXT:       StorageMappingClass: XMC_RW (0x5)
+; SYMS-NEXT:       StabInfoIndex: 0x0
+; SYMS-NEXT:       StabSectNum: 0x0
+; SYMS-NEXT:     }
+; SYMS-NEXT:   }
+
+; SYMS:        Symbol {
+; SYMS-NEXT:     Index: [[#INDX+34]]
+; SYMS-NEXT:     Name: f
+; SYMS-NEXT:     Value (RelocatableAddress): 0x80
+; SYMS-NEXT:     Section: .bss
+; SYMS-NEXT:     Type: 0x0
+; SYMS-NEXT:     StorageClass: C_EXT (0x2)
+; SYMS-NEXT:     NumberOfAuxEntries: 1
+; SYMS-NEXT:     CSECT Auxiliary Entry {
+; SYMS-NEXT:       Index: [[#INDX+35]]
+; SYMS-NEXT:       SectionLen: 4
+; SYMS-NEXT:       ParameterHashIndex: 0x0
+; SYMS-NEXT:       TypeChkSectNum: 0x0
+; SYMS-NEXT:       SymbolAlignmentLog2: 2
+; SYMS-NEXT:       SymbolType: XTY_CM (0x3)
+; SYMS-NEXT:       StorageMappingClass: XMC_RW (0x5)
+; SYMS-NEXT:       StabInfoIndex: 0x0
+; SYMS-NEXT:       StabSectNum: 0x0
+; SYMS-NEXT:     }
+; SYMS-NEXT:   }
+
+; SYMS:        Symbol {
+; SYMS-NEXT:     Index: [[#INDX+36]]
+; SYMS-NEXT:     Name: over_aligned_comm
+; SYMS-NEXT:     Value (RelocatableAddress): 0xA0
+; SYMS-NEXT:     Section: .bss
+; SYMS-NEXT:     Type: 0x0
+; SYMS-NEXT:     StorageClass: C_EXT (0x2)
+; SYMS-NEXT:     NumberOfAuxEntries: 1
+; SYMS-NEXT:     CSECT Auxiliary Entry {
+; SYMS-NEXT:       Index: [[#INDX+37]]
+; SYMS-NEXT:       SectionLen: 8
+; SYMS-NEXT:       ParameterHashIndex: 0x0
+; SYMS-NEXT:       TypeChkSectNum: 0x0
+; SYMS-NEXT:       SymbolAlignmentLog2: 5
+; SYMS-NEXT:       SymbolType: XTY_CM (0x3)
+; SYMS-NEXT:       StorageMappingClass: XMC_RW (0x5)
+; SYMS-NEXT:       StabInfoIndex: 0x0
+; SYMS-NEXT:       StabSectNum: 0x0
+; SYMS-NEXT:     }
+; SYMS-NEXT:   }
+
+; SYMS:        Symbol {
+; SYMS-NEXT:     Index: [[#INDX+38]]
+; SYMS-NEXT:     Name: array
+; SYMS-NEXT:     Value (RelocatableAddress): 0xA8
+; SYMS-NEXT:     Section: .bss
+; SYMS-NEXT:     Type: 0x0
+; SYMS-NEXT:     StorageClass: C_EXT (0x2)
+; SYMS-NEXT:     NumberOfAuxEntries: 1
+; SYMS-NEXT:     CSECT Auxiliary Entry {
+; SYMS-NEXT:       Index: [[#INDX+39]]
+; SYMS-NEXT:       SectionLen: 33
+; SYMS-NEXT:       ParameterHashIndex: 0x0
+; SYMS-NEXT:       TypeChkSectNum: 0x0
+; SYMS-NEXT:       SymbolAlignmentLog2: 0
+; SYMS-NEXT:       SymbolType: XTY_CM (0x3)
+; SYMS-NEXT:       StorageMappingClass: XMC_RW (0x5)
+; SYMS-NEXT:       StabInfoIndex: 0x0
+; SYMS-NEXT:       StabSectNum: 0x0
+; SYMS-NEXT:     }
+; SYMS-NEXT:   }
+; SYMS:      ]
diff --git a/llvm/test/CodeGen/PowerPC/aix-xcoff-lcomm.ll b/llvm/test/CodeGen/PowerPC/aix-xcoff-lcomm.ll
new file mode 100644
index 000000000..536229fc8
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/aix-xcoff-lcomm.ll
@@ -0,0 +1,112 @@
+; RUN: llc -mtriple powerpc-ibm-aix-xcoff < %s | FileCheck %s
+; RUN: llc -mtriple powerpc64-ibm-aix-xcoff < %s | FileCheck %s
+
+; RUN: llc -mtriple powerpc-ibm-aix-xcoff -filetype=obj -o %t.o < %s
+; RUN: llvm-readobj --section-headers --file-header %t.o | \
+; RUN: FileCheck --check-prefix=OBJ %s
+; RUN: llvm-readobj --syms %t.o | FileCheck --check-prefix=SYMS %s
+
+; RUN: not llc -mtriple powerpc64-ibm-aix-xcoff -filetype=obj < %s 2>&1 | \
+; RUN: FileCheck --check-prefix=OBJ64 %s
+; OBJ64: LLVM ERROR: 64-bit XCOFF object files are not supported yet.
+
+@a = internal global i32 0, align 4
+@b = internal global i64 0, align 8
+@c = internal global i16 0, align 2
+
+; CHECK:      .lcomm a,4,a[BS],2
+; CHECK-NEXT: .lcomm b,8,b[BS],3
+; CHECK-NEXT: .lcomm c,2,c[BS],1
+
+; OBJ:      File: {{.*}}aix-xcoff-lcomm.ll.tmp.o
+; OBJ-NEXT: Format: aixcoff-rs6000
+; OBJ-NEXT: Arch: powerpc
+; OBJ-NEXT: AddressSize: 32bit
+; OBJ-NEXT: FileHeader {
+; OBJ-NEXT:   Magic: 0x1DF
+; OBJ-NEXT:   NumberOfSections: 2
+; OBJ-NEXT:   TimeStamp:
+; OBJ-NEXT:   SymbolTableOffset: 0x64
+; OBJ-NEXT:   SymbolTableEntries: 8
+; OBJ-NEXT:   OptionalHeaderSize: 0x0
+; OBJ-NEXT:   Flags: 0x0
+; OBJ-NEXT: }
+; OBJ-NEXT: Sections [
+; OBJ:        Section {{[{][[:space:]] *}}Index: 2
+; OBJ-NEXT:     Name: .bss
+; OBJ-NEXT:     PhysicalAddress: 0x0
+; OBJ-NEXT:     VirtualAddress: 0x0
+; OBJ-NEXT:     Size: 0x14
+; OBJ-NEXT:     RawDataOffset: 0x0
+; OBJ-NEXT:     RelocationPointer: 0x0
+; OBJ-NEXT:     LineNumberPointer: 0x0
+; OBJ-NEXT:     NumberOfRelocations: 0
+; OBJ-NEXT:     NumberOfLineNumbers: 0
+; OBJ-NEXT:     Type: STYP_BSS (0x80)
+; OBJ-NEXT:   }
+; OBJ-NEXT: ]
+
+; SYMS:      File: {{.*}}aix-xcoff-lcomm.ll.tmp.o
+; SYMS-NEXT: Format: aixcoff-rs6000
+; SYMS-NEXT: Arch: powerpc
+; SYMS-NEXT: AddressSize: 32bit
+; SYMS-NEXT: Symbols [
+; SYMS:        Symbol {{[{][[:space:]] *}}Index: [[#Index:]]{{[[:space:]] *}}Name: a
+; SYMS-NEXT:     Value (RelocatableAddress): 0x0
+; SYMS-NEXT:     Section: .bss
+; SYMS-NEXT:     Type: 0x0
+; SYMS-NEXT:     StorageClass: C_HIDEXT (0x6B)
+; SYMS-NEXT:     NumberOfAuxEntries: 1
+; SYMS-NEXT:     CSECT Auxiliary Entry {
+; SYMS-NEXT:       Index: [[#Index + 1]]
+; SYMS-NEXT:       SectionLen: 4
+; SYMS-NEXT:       ParameterHashIndex: 0x0
+; SYMS-NEXT:       TypeChkSectNum: 0x0
+; SYMS-NEXT:       SymbolAlignmentLog2: 2
+; SYMS-NEXT:       SymbolType: XTY_CM (0x3)
+; SYMS-NEXT:       StorageMappingClass: XMC_BS (0x9)
+; SYMS-NEXT:       StabInfoIndex: 0x0
+; SYMS-NEXT:       StabSectNum: 0x0
+; SYMS-NEXT:     }
+; SYMS-NEXT:   }
+; SYMS-NEXT:   Symbol {
+; SYMS-NEXT:     Index: [[#Index + 2]]
+; SYMS-NEXT:     Name: b
+; SYMS-NEXT:     Value (RelocatableAddress): 0x8
+; SYMS-NEXT:     Section: .bss
+; SYMS-NEXT:     Type: 0x0
+; SYMS-NEXT:     StorageClass: C_HIDEXT (0x6B)
+; SYMS-NEXT:     NumberOfAuxEntries: 1
+; SYMS-NEXT:     CSECT Auxiliary Entry {
+; SYMS-NEXT:       Index: [[#Index + 3]]
+; SYMS-NEXT:       SectionLen: 8
+; SYMS-NEXT:       ParameterHashIndex: 0x0
+; SYMS-NEXT:       TypeChkSectNum: 0x0
+; SYMS-NEXT:       SymbolAlignmentLog2: 3
+; SYMS-NEXT:       SymbolType: XTY_CM (0x3)
+; SYMS-NEXT:       StorageMappingClass: XMC_BS (0x9)
+; SYMS-NEXT:       StabInfoIndex: 0x0
+; SYMS-NEXT:       StabSectNum: 0x0
+; SYMS-NEXT:     }
+; SYMS-NEXT:   }
+; SYMS-NEXT:   Symbol {
+; SYMS-NEXT:     Index: [[#Index + 4]]
+; SYMS-NEXT:     Name: c
+; SYMS-NEXT:     Value (RelocatableAddress): 0x10
+; SYMS-NEXT:     Section: .bss
+; SYMS-NEXT:     Type: 0x0
+; SYMS-NEXT:     StorageClass: C_HIDEXT (0x6B)
+; SYMS-NEXT:     NumberOfAuxEntries: 1
+; SYMS-NEXT:     CSECT Auxiliary Entry {
+; SYMS-NEXT:       Index: [[#Index + 5]]
+; SYMS-NEXT:       SectionLen: 2
+; SYMS-NEXT:       ParameterHashIndex: 0x0
+; SYMS-NEXT:       TypeChkSectNum: 0x0
+; SYMS-NEXT:       SymbolAlignmentLog2: 1
+; SYMS-NEXT:       SymbolType: XTY_CM (0x3)
+; SYMS-NEXT:       StorageMappingClass: XMC_BS (0x9)
+; SYMS-NEXT:       StabInfoIndex: 0x0
+; SYMS-NEXT:       StabSectNum: 0x0
+; SYMS-NEXT:     }
+; SYMS-NEXT:   }
+; SYMS-NEXT: ]
diff --git a/llvm/test/CodeGen/PowerPC/aix-xcoff-mergeable-const.ll b/llvm/test/CodeGen/PowerPC/aix-xcoff-mergeable-const.ll
new file mode 100644
index 000000000..09b1a4a33
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/aix-xcoff-mergeable-const.ll
@@ -0,0 +1,166 @@
+; This file tests the codegen of mergeable const in AIX assembly.
+; This file also tests mergeable const in XCOFF object file generation.
+; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mtriple powerpc-ibm-aix-xcoff < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mtriple powerpc64-ibm-aix-xcoff < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mtriple powerpc-ibm-aix-xcoff -filetype=obj -o %t.o < %s
+; RUN: llvm-objdump -D %t.o | FileCheck --check-prefix=CHECKOBJ %s
+; RUN: llvm-readobj -syms %t.o | FileCheck --check-prefix=CHECKSYM %s
+
+%struct.Merge_cnst32 = type { i64, i32, i64, i32 }
+%struct.Merge_cnst16 = type { i64, i32 }
+%struct.Merge_cnst8 = type { i32, i32 }
+%struct.Merge_cnst4 = type { i16, i8 }
+
+@__const.main.cnst32 = private unnamed_addr constant %struct.Merge_cnst32 { i64 4611686018427387954, i32 0, i64 0, i32 0 }
+@__const.main.cnst16 = private unnamed_addr constant %struct.Merge_cnst16 { i64 4611686018427387926, i32 0 }
+@__const.main.cnst8 = private unnamed_addr constant %struct.Merge_cnst8 { i32 1073741832, i32 0 }
+@__const.main.cnst4 = private unnamed_addr constant %struct.Merge_cnst4 { i16 16392, i8 0 }
+
+; Function Attrs: noinline nounwind optnone
+define i32 @main() #0 {
+entry:
+  ret i32 0
+}
+
+;CHECK:         .csect .rodata[RO]
+;CHECK-NEXT:         .align  4
+;CHECK-NEXT: .L__const.main.cnst32:
+;CHECK-NEXT:         .llong  4611686018427387954     # 0x4000000000000032
+;CHECK-NEXT:         .long   0                       # 0x0
+;CHECK-NEXT:         .space  4
+;CHECK-NEXT:         .llong  0                       # 0x0
+;CHECK-NEXT:         .long   0                       # 0x0
+;CHECK-NEXT:         .space  4
+;CHECK-NEXT:         .align  3
+;CHECK-NEXT: .L__const.main.cnst16:
+;CHECK-NEXT:         .llong  4611686018427387926     # 0x4000000000000016
+;CHECK-NEXT:         .long   0                       # 0x0
+;CHECK-NEXT:         .space  4
+;CHECK-NEXT:         .align  3
+;CHECK-NEXT: .L__const.main.cnst8:
+;CHECK-NEXT:         .long   1073741832              # 0x40000008
+;CHECK-NEXT:         .long   0                       # 0x0
+;CHECK-NEXT:         .align  3
+;CHECK-NEXT: .L__const.main.cnst4:
+;CHECK-NEXT:         .short  16392                   # 0x4008
+;CHECK-NEXT:         .byte   0                       # 0x0
+;CHECK-NEXT:         .space  1
+
+
+;CHECKOBJ:      00000000 .text:
+;CHECKOBJ-NEXT:        0: 38 60 00 00                    li 3, 0
+;CHECKOBJ-NEXT:        4: 4e 80 00 20                    blr
+;CHECKOBJ-NEXT:          ...{{[[:space:]] *}}
+;CHECKOBJ-NEXT: 00000010 .rodata:
+;CHECKOBJ-NEXT:        10: 40 00 00 00                    bdnzf   0, .+0
+;CHECKOBJ-NEXT:        14: 00 00 00 32                    <unknown>
+;CHECKOBJ-NEXT:          ...{{[[:space:]] *}}
+;CHECKOBJ-NEXT: 00000030 .L__const.main.cnst16:
+;CHECKOBJ-NEXT:       30: 40 00 00 00                    bdnzf   0, .+0
+;CHECKOBJ-NEXT:       34: 00 00 00 16                    <unknown>
+;CHECKOBJ-NEXT:          ...{{[[:space:]] *}}
+;CHECKOBJ-NEXT: 00000040 .L__const.main.cnst8:
+;CHECKOBJ-NEXT:       40: 40 00 00 08                    bdnzf   0, .+8
+;CHECKOBJ-NEXT:       44: 00 00 00 00                    <unknown>{{[[:space:]] *}}
+;CHECKOBJ-NEXT: 00000048 .L__const.main.cnst4:
+;CHECKOBJ-NEXT:       48: 40 08 00 00                    bdnzf   8, .+0
+
+
+;CHECKSYM:        Symbol {{[{][[:space:]] *}}Index: [[#Index:]]{{[[:space:]] *}}Name: .rodata
+;CHECKSYM-NEXT:     Value (RelocatableAddress): 0x10
+;CHECKSYM-NEXT:     Section: .text
+;CHECKSYM-NEXT:     Type: 0x0
+;CHECKSYM-NEXT:     StorageClass: C_HIDEXT (0x6B)
+;CHECKSYM-NEXT:     NumberOfAuxEntries: 1
+;CHECKSYM-NEXT:     CSECT Auxiliary Entry {
+;CHECKSYM-NEXT:       Index: [[#Index+1]]
+;CHECKSYM-NEXT:       SectionLen: 60
+;CHECKSYM-NEXT:       ParameterHashIndex: 0x0
+;CHECKSYM-NEXT:       TypeChkSectNum: 0x0
+;CHECKSYM-NEXT:       SymbolAlignmentLog2: 4
+;CHECKSYM-NEXT:       SymbolType: XTY_SD (0x1)
+;CHECKSYM-NEXT:       StorageMappingClass: XMC_RO (0x1)
+;CHECKSYM-NEXT:       StabInfoIndex: 0x0
+;CHECKSYM-NEXT:       StabSectNum: 0x0
+;CHECKSYM-NEXT:     }
+;CHECKSYM-NEXT:   }
+;CHECKSYM-NEXT:   Symbol {
+;CHECKSYM-NEXT:     Index: [[#Index+2]]
+;CHECKSYM-NEXT:     Name: .L__const.main.cnst32
+;CHECKSYM-NEXT:     Value (RelocatableAddress): 0x10
+;CHECKSYM-NEXT:     Section: .text
+;CHECKSYM-NEXT:     Type: 0x0
+;CHECKSYM-NEXT:     StorageClass: C_HIDEXT (0x6B)
+;CHECKSYM-NEXT:     NumberOfAuxEntries: 1
+;CHECKSYM-NEXT:     CSECT Auxiliary Entry {
+;CHECKSYM-NEXT:       Index: [[#Index+3]]
+;CHECKSYM-NEXT:       ContainingCsectSymbolIndex: [[#Index]]
+;CHECKSYM-NEXT:       ParameterHashIndex: 0x0
+;CHECKSYM-NEXT:       TypeChkSectNum: 0x0
+;CHECKSYM-NEXT:       SymbolAlignmentLog2: 0
+;CHECKSYM-NEXT:       SymbolType: XTY_LD (0x2)
+;CHECKSYM-NEXT:       StorageMappingClass: XMC_RO (0x1)
+;CHECKSYM-NEXT:       StabInfoIndex: 0x0
+;CHECKSYM-NEXT:       StabSectNum: 0x0
+;CHECKSYM-NEXT:     }
+;CHECKSYM-NEXT:   }
+;CHECKSYM-NEXT:   Symbol {
+;CHECKSYM-NEXT:     Index: [[#Index+4]]
+;CHECKSYM-NEXT:     Name: .L__const.main.cnst16
+;CHECKSYM-NEXT:     Value (RelocatableAddress): 0x30
+;CHECKSYM-NEXT:     Section: .text
+;CHECKSYM-NEXT:     Type: 0x0
+;CHECKSYM-NEXT:     StorageClass: C_HIDEXT (0x6B)
+;CHECKSYM-NEXT:     NumberOfAuxEntries: 1
+;CHECKSYM-NEXT:     CSECT Auxiliary Entry {
+;CHECKSYM-NEXT:       Index: [[#Index+5]]
+;CHECKSYM-NEXT:       ContainingCsectSymbolIndex: [[#Index]]
+;CHECKSYM-NEXT:       ParameterHashIndex: 0x0
+;CHECKSYM-NEXT:       TypeChkSectNum: 0x0
+;CHECKSYM-NEXT:       SymbolAlignmentLog2: 0
+;CHECKSYM-NEXT:       SymbolType: XTY_LD (0x2)
+;CHECKSYM-NEXT:       StorageMappingClass: XMC_RO (0x1)
+;CHECKSYM-NEXT:       StabInfoIndex: 0x0
+;CHECKSYM-NEXT:       StabSectNum: 0x0
+;CHECKSYM-NEXT:     }
+;CHECKSYM-NEXT:   }
+;CHECKSYM-NEXT:   Symbol {
+;CHECKSYM-NEXT:     Index: [[#Index+6]]
+;CHECKSYM-NEXT:     Name: .L__const.main.cnst8
+;CHECKSYM-NEXT:     Value (RelocatableAddress): 0x40
+;CHECKSYM-NEXT:     Section: .text
+;CHECKSYM-NEXT:     Type: 0x0
+;CHECKSYM-NEXT:     StorageClass: C_HIDEXT (0x6B)
+;CHECKSYM-NEXT:     NumberOfAuxEntries: 1
+;CHECKSYM-NEXT:     CSECT Auxiliary Entry {
+;CHECKSYM-NEXT:       Index: [[#Index+7]]
+;CHECKSYM-NEXT:       ContainingCsectSymbolIndex: [[#Index]]
+;CHECKSYM-NEXT:       ParameterHashIndex: 0x0
+;CHECKSYM-NEXT:       TypeChkSectNum: 0x0
+;CHECKSYM-NEXT:       SymbolAlignmentLog2: 0
+;CHECKSYM-NEXT:       SymbolType: XTY_LD (0x2)
+;CHECKSYM-NEXT:       StorageMappingClass: XMC_RO (0x1)
+;CHECKSYM-NEXT:       StabInfoIndex: 0x0
+;CHECKSYM-NEXT:       StabSectNum: 0x0
+;CHECKSYM-NEXT:     }
+;CHECKSYM-NEXT:   }
+;CHECKSYM-NEXT:   Symbol {
+;CHECKSYM-NEXT:     Index: [[#Index+8]]
+;CHECKSYM-NEXT:     Name: .L__const.main.cnst4
+;CHECKSYM-NEXT:     Value (RelocatableAddress): 0x48
+;CHECKSYM-NEXT:     Section: .text
+;CHECKSYM-NEXT:     Type: 0x0
+;CHECKSYM-NEXT:     StorageClass: C_HIDEXT (0x6B)
+;CHECKSYM-NEXT:     NumberOfAuxEntries: 1
+;CHECKSYM-NEXT:     CSECT Auxiliary Entry {
+;CHECKSYM-NEXT:       Index: [[#Index+9]]
+;CHECKSYM-NEXT:       ContainingCsectSymbolIndex: [[#Index]]
+;CHECKSYM-NEXT:       ParameterHashIndex: 0x0
+;CHECKSYM-NEXT:       TypeChkSectNum: 0x0
+;CHECKSYM-NEXT:       SymbolAlignmentLog2: 0
+;CHECKSYM-NEXT:       SymbolType: XTY_LD (0x2)
+;CHECKSYM-NEXT:       StorageMappingClass: XMC_RO (0x1)
+;CHECKSYM-NEXT:       StabInfoIndex: 0x0
+;CHECKSYM-NEXT:       StabSectNum: 0x0
+;CHECKSYM-NEXT:     }
+;CHECKSYM-NEXT:   }
diff --git a/llvm/test/CodeGen/PowerPC/aix-xcoff-mergeable-str.ll b/llvm/test/CodeGen/PowerPC/aix-xcoff-mergeable-str.ll
new file mode 100644
index 000000000..3df01a08f
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/aix-xcoff-mergeable-str.ll
@@ -0,0 +1,62 @@
+; This file tests the codegen of mergeable strings in AIX assembly only.
+; Once the codegen of mergeable strings for XCOFF object files is supported
+; the test in this file should be merged into aix-xcoff-data.ll with additional
+; tests for XCOFF object files.
+
+; RUN: llc -verify-machineinstrs -mcpu=pwr7 \
+; RUN:     -mtriple powerpc-ibm-aix-xcoff < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -mcpu=pwr7 \
+; RUN:     -mtriple powerpc64-ibm-aix-xcoff < %s | FileCheck %s
+
+; RUN: llc -verify-machineinstrs -mcpu=pwr7 -mtriple powerpc-ibm-aix-xcoff -filetype=obj -o %t.o < %s
+; RUN: llvm-objdump -D %t.o | FileCheck --check-prefix=CHECKOBJ %s
+
+@strA = private unnamed_addr constant [14 x i8] c"hello world!\0A\00", align 1
+@.str = private unnamed_addr constant [9 x i8] c"abcdefgh\00", align 1
+@p = global i8* getelementptr inbounds ([9 x i8], [9 x i8]* @.str, i32 0, i32 0), align 4
+
+; Function Attrs: noinline nounwind optnone
+define i8 @foo() #0 {
+entry:
+  %0 = load i8*, i8** @p, align 4
+  %1 = load i8, i8* %0, align 1
+  ret i8 %1
+}
+
+; CHECK:      .csect .rodata.str1.1[RO]
+; CHECK-NEXT: .LstrA:
+; CHECK-NEXT: .byte   104
+; CHECK-NEXT: .byte   101
+; CHECK-NEXT: .byte   108
+; CHECK-NEXT: .byte   108
+; CHECK-NEXT: .byte   111
+; CHECK-NEXT: .byte   32
+; CHECK-NEXT: .byte   119
+; CHECK-NEXT: .byte   111
+; CHECK-NEXT: .byte   114
+; CHECK-NEXT: .byte   108
+; CHECK-NEXT: .byte   100
+; CHECK-NEXT: .byte   33
+; CHECK-NEXT: .byte   10
+; CHECK-NEXT: .byte   0
+; CHECK-NEXT: .L.str:
+; CHECK-NEXT: .byte   97
+; CHECK-NEXT: .byte   98
+; CHECK-NEXT: .byte   99
+; CHECK-NEXT: .byte   100
+; CHECK-NEXT: .byte   101
+; CHECK-NEXT: .byte   102
+; CHECK-NEXT: .byte   103
+; CHECK-NEXT: .byte   104
+; CHECK-NEXT: .byte   0
+
+; CHECKOBJ: 00000010 .rodata.str1.1:
+; CHECKOBJ-NEXT:       10: 68 65 6c 6c                   xori 5, 3, 27756
+; CHECKOBJ-NEXT:       14: 6f 20 77 6f                   xoris 0, 25, 30575
+; CHECKOBJ-NEXT:       18: 72 6c 64 21                   andi. 12, 19, 25633
+; CHECKOBJ-NEXT:       1c: 0a 00 61 62                   tdlti   0, 24930{{[[:space:]] *}}
+; CHECKOBJ-NEXT: 0000001e .L.str:
+; CHECKOBJ-NEXT:       1e: 61 62 63 64                   ori 2, 11, 25444
+; CHECKOBJ-NEXT:       22: 65 66 67 68                   oris 6, 11, 26472
+; CHECKOBJ-NEXT:       26: 00                            <unknown>
+; CHECKOBJ-NEXT:       27: 00                            <unknown>
diff --git a/llvm/test/CodeGen/PowerPC/aix-xcoff-rodata.ll b/llvm/test/CodeGen/PowerPC/aix-xcoff-rodata.ll
new file mode 100644
index 000000000..769a9c2cc
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/aix-xcoff-rodata.ll
@@ -0,0 +1,322 @@
+; RUN: llc -verify-machineinstrs -mcpu=pwr7 -mtriple powerpc-ibm-aix-xcoff  < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -mcpu=pwr7 -mtriple powerpc64-ibm-aix-xcoff < %s | FileCheck %s
+
+; RUN: llc -verify-machineinstrs -mcpu=pwr7 -mtriple powerpc-ibm-aix-xcoff -filetype=obj -o %t.o < %s
+; RUN: llvm-readobj --section-headers --file-header %t.o | \
+; RUN: FileCheck --check-prefix=OBJ %s
+; RUN: llvm-readobj --syms %t.o | FileCheck --check-prefix=SYMS %s
+; RUN: llvm-objdump -D %t.o | FileCheck --check-prefix=DIS %s
+
+; RUN: not llc -verify-machineinstrs -mcpu=pwr7 -mtriple powerpc64-ibm-aix-xcoff -filetype=obj < %s 2>&1 | \
+; RUN: FileCheck --check-prefix=XCOFF64 %s
+; XCOFF64: LLVM ERROR: 64-bit XCOFF object files are not supported yet.
+
+@const_ivar = constant i32 35, align 4
+@const_llvar = constant i64 36, align 8
+@const_svar = constant i16 37, align 2
+@const_fvar = constant float 8.000000e+02, align 4
+@const_dvar = constant double 9.000000e+02, align 8
+@const_over_aligned = constant double 9.000000e+02, align 32
+@const_chrarray = constant [4 x i8] c"abcd", align 1
+@const_dblarr = constant [4 x double] [double 1.000000e+00, double 2.000000e+00, double 3.000000e+00, double 4.000000e+00], align 8
+
+; CHECK:               .csect .rodata[RO]
+; CHECK-NEXT:          .globl  const_ivar
+; CHECK-NEXT:          .align  2
+; CHECK-NEXT:  const_ivar:
+; CHECK-NEXT:          .long   35
+; CHECK-NEXT:          .globl  const_llvar
+; CHECK-NEXT:          .align  3
+; CHECK-NEXT:  const_llvar:
+; CHECK-NEXT:          .llong  36
+; CHECK-NEXT:          .globl  const_svar
+; CHECK-NEXT:          .align  1
+; CHECK-NEXT:  const_svar:
+; CHECK-NEXT:          .short  37
+; CHECK-NEXT:          .globl  const_fvar
+; CHECK-NEXT:          .align  2
+; CHECK-NEXT:  const_fvar:
+; CHECK-NEXT:          .long   1145569280
+; CHECK-NEXT:          .globl  const_dvar
+; CHECK-NEXT:          .align  3
+; CHECK-NEXT:  const_dvar:
+; CHECK-NEXT:          .llong  4651127699538968576
+; CHECK-NEXT:          .globl  const_over_aligned
+; CHECK-NEXT:          .align  5
+; CHECK-NEXT:  const_over_aligned:
+; CHECK-NEXT:          .llong  4651127699538968576
+; CHECK-NEXT:          .globl  const_chrarray
+; CHECK-NEXT:  const_chrarray:
+; CHECK-NEXT:          .byte   97
+; CHECK-NEXT:          .byte   98
+; CHECK-NEXT:          .byte   99
+; CHECK-NEXT:          .byte   100
+; CHECK-NEXT:          .globl  const_dblarr
+; CHECK-NEXT:          .align  3
+; CHECK-NEXT:  const_dblarr:
+; CHECK-NEXT:          .llong  4607182418800017408
+; CHECK-NEXT:          .llong  4611686018427387904
+; CHECK-NEXT:          .llong  4613937818241073152
+; CHECK-NEXT:          .llong  4616189618054758400
+
+
+; OBJ:      File: {{.*}}aix-xcoff-rodata.ll.tmp.o
+; OBJ-NEXT: Format: aixcoff-rs6000
+; OBJ-NEXT: Arch: powerpc
+; OBJ-NEXT: AddressSize: 32bit
+; OBJ-NEXT: FileHeader {
+; OBJ-NEXT:   Magic: 0x1DF
+; OBJ-NEXT:   NumberOfSections: 1
+; OBJ-NEXT:   TimeStamp: None (0x0)
+; OBJ-NEXT:   SymbolTableOffset: 0x8C
+; OBJ-NEXT:   SymbolTableEntries: 20
+; OBJ-NEXT:   OptionalHeaderSize: 0x0
+; OBJ-NEXT:   Flags: 0x0
+; OBJ-NEXT: }
+
+; OBJ:      Sections [
+; OBJ:        Section {
+; OBJ-NEXT:     Index: 1
+; OBJ-NEXT:     Name: .text
+; OBJ-NEXT:     PhysicalAddress: 0x0
+; OBJ-NEXT:     VirtualAddress: 0x0
+; OBJ-NEXT:     Size: 0x50
+; OBJ-NEXT:     RawDataOffset: 0x3C
+; OBJ-NEXT:     RelocationPointer: 0x0
+; OBJ-NEXT:     LineNumberPointer: 0x0
+; OBJ-NEXT:     NumberOfRelocations: 0
+; OBJ-NEXT:     NumberOfLineNumbers: 0
+; OBJ-NEXT:     Type: STYP_TEXT (0x20)
+; OBJ-NEXT:   }
+; OBJ-NEXT: ]
+
+
+; SYMS:       File: {{.*}}aix-xcoff-rodata.ll.tmp.o
+; SYMS-NEXT:  Format: aixcoff-rs6000
+; SYMS-NEXT:  Arch: powerpc
+; SYMS-NEXT:  AddressSize: 32bit
+; SYMS:       Symbols [
+; SYMS:        Symbol {{[{][[:space:]] *}}Index: [[#INDX:]]{{[[:space:]] *}}Name: .rodata
+; SYMS-NEXT:     Value (RelocatableAddress): 0x0
+; SYMS-NEXT:     Section: .text
+; SYMS-NEXT:     Type: 0x0
+; SYMS-NEXT:     StorageClass: C_HIDEXT (0x6B)
+; SYMS-NEXT:     NumberOfAuxEntries: 1
+; SYMS-NEXT:     CSECT Auxiliary Entry {
+; SYMS-NEXT:       Index: [[#INDX+1]]
+; SYMS-NEXT:       SectionLen: 80
+; SYMS-NEXT:       ParameterHashIndex: 0x0
+; SYMS-NEXT:       TypeChkSectNum: 0x0
+; SYMS-NEXT:       SymbolAlignmentLog2: 5
+; SYMS-NEXT:       SymbolType: XTY_SD (0x1)
+; SYMS-NEXT:       StorageMappingClass: XMC_RO (0x1)
+; SYMS-NEXT:       StabInfoIndex: 0x0
+; SYMS-NEXT:       StabSectNum: 0x0
+; SYMS-NEXT:     }
+; SYMS-NEXT:   }
+
+; SYMS:        Symbol {
+; SYMS-NEXT:     Index: [[#INDX+2]]
+; SYMS-NEXT:     Name: const_ivar
+; SYMS-NEXT:     Value (RelocatableAddress): 0x0
+; SYMS-NEXT:     Section: .text
+; SYMS-NEXT:     Type: 0x0
+; SYMS-NEXT:     StorageClass: C_EXT (0x2)
+; SYMS-NEXT:     NumberOfAuxEntries: 1
+; SYMS-NEXT:     CSECT Auxiliary Entry {
+; SYMS-NEXT:       Index: [[#INDX+3]]
+; SYMS-NEXT:       ContainingCsectSymbolIndex: 2
+; SYMS-NEXT:       ParameterHashIndex: 0x0
+; SYMS-NEXT:       TypeChkSectNum: 0x0
+; SYMS-NEXT:       SymbolAlignmentLog2: 0
+; SYMS-NEXT:       SymbolType: XTY_LD (0x2)
+; SYMS-NEXT:       StorageMappingClass: XMC_RO (0x1)
+; SYMS-NEXT:       StabInfoIndex: 0x0
+; SYMS-NEXT:       StabSectNum: 0x0
+; SYMS-NEXT:     }
+; SYMS-NEXT:   }
+
+; SYMS:        Symbol {
+; SYMS-NEXT:     Index: [[#INDX+4]]
+; SYMS-NEXT:     Name: const_llvar
+; SYMS-NEXT:     Value (RelocatableAddress): 0x8
+; SYMS-NEXT:     Section: .text
+; SYMS-NEXT:     Type: 0x0
+; SYMS-NEXT:     StorageClass: C_EXT (0x2)
+; SYMS-NEXT:     NumberOfAuxEntries: 1
+; SYMS-NEXT:     CSECT Auxiliary Entry {
+; SYMS-NEXT:       Index: [[#INDX+5]]
+; SYMS-NEXT:       ContainingCsectSymbolIndex: 2
+; SYMS-NEXT:       ParameterHashIndex: 0x0
+; SYMS-NEXT:       TypeChkSectNum: 0x0
+; SYMS-NEXT:       SymbolAlignmentLog2: 0
+; SYMS-NEXT:       SymbolType: XTY_LD (0x2)
+; SYMS-NEXT:       StorageMappingClass: XMC_RO (0x1)
+; SYMS-NEXT:       StabInfoIndex: 0x0
+; SYMS-NEXT:       StabSectNum: 0x0
+; SYMS-NEXT:     }
+; SYMS-NEXT:   }
+
+; SYMS:        Symbol {
+; SYMS-NEXT:     Index: [[#INDX+6]]
+; SYMS-NEXT:     Name: const_svar
+; SYMS-NEXT:     Value (RelocatableAddress): 0x10
+; SYMS-NEXT:     Section: .text
+; SYMS-NEXT:     Type: 0x0
+; SYMS-NEXT:     StorageClass: C_EXT (0x2)
+; SYMS-NEXT:     NumberOfAuxEntries: 1
+; SYMS-NEXT:     CSECT Auxiliary Entry {
+; SYMS-NEXT:       Index: [[#INDX+7]]
+; SYMS-NEXT:       ContainingCsectSymbolIndex: 2
+; SYMS-NEXT:       ParameterHashIndex: 0x0
+; SYMS-NEXT:       TypeChkSectNum: 0x0
+; SYMS-NEXT:       SymbolAlignmentLog2: 0
+; SYMS-NEXT:       SymbolType: XTY_LD (0x2)
+; SYMS-NEXT:       StorageMappingClass: XMC_RO (0x1)
+; SYMS-NEXT:       StabInfoIndex: 0x0
+; SYMS-NEXT:       StabSectNum: 0x0
+; SYMS-NEXT:     }
+; SYMS-NEXT:   }
+
+; SYMS:        Symbol {
+; SYMS-NEXT:     Index: [[#INDX+8]]
+; SYMS-NEXT:     Name: const_fvar
+; SYMS-NEXT:     Value (RelocatableAddress): 0x14
+; SYMS-NEXT:     Section: .text
+; SYMS-NEXT:     Type: 0x0
+; SYMS-NEXT:     StorageClass: C_EXT (0x2)
+; SYMS-NEXT:     NumberOfAuxEntries: 1
+; SYMS-NEXT:     CSECT Auxiliary Entry {
+; SYMS-NEXT:       Index: [[#INDX+9]]
+; SYMS-NEXT:       ContainingCsectSymbolIndex: 2
+; SYMS-NEXT:       ParameterHashIndex: 0x0
+; SYMS-NEXT:       TypeChkSectNum: 0x0
+; SYMS-NEXT:       SymbolAlignmentLog2: 0
+; SYMS-NEXT:       SymbolType: XTY_LD (0x2)
+; SYMS-NEXT:       StorageMappingClass: XMC_RO (0x1)
+; SYMS-NEXT:       StabInfoIndex: 0x0
+; SYMS-NEXT:       StabSectNum: 0x0
+; SYMS-NEXT:     }
+; SYMS-NEXT:   }
+
+; SYMS:        Symbol {
+; SYMS-NEXT:     Index: [[#INDX+10]]
+; SYMS-NEXT:     Name: const_dvar
+; SYMS-NEXT:     Value (RelocatableAddress): 0x18
+; SYMS-NEXT:     Section: .text
+; SYMS-NEXT:     Type: 0x0
+; SYMS-NEXT:     StorageClass: C_EXT (0x2)
+; SYMS-NEXT:     NumberOfAuxEntries: 1
+; SYMS-NEXT:     CSECT Auxiliary Entry {
+; SYMS-NEXT:       Index: [[#INDX+11]]
+; SYMS-NEXT:       ContainingCsectSymbolIndex: 2
+; SYMS-NEXT:       ParameterHashIndex: 0x0
+; SYMS-NEXT:       TypeChkSectNum: 0x0
+; SYMS-NEXT:       SymbolAlignmentLog2: 0
+; SYMS-NEXT:       SymbolType: XTY_LD (0x2)
+; SYMS-NEXT:       StorageMappingClass: XMC_RO (0x1)
+; SYMS-NEXT:       StabInfoIndex: 0x0
+; SYMS-NEXT:       StabSectNum: 0x0
+; SYMS-NEXT:     }
+; SYMS-NEXT:   }
+
+; SYMS:        Symbol {
+; SYMS-NEXT:     Index: [[#INDX+12]]
+; SYMS-NEXT:     Name: const_over_aligned
+; SYMS-NEXT:     Value (RelocatableAddress): 0x20
+; SYMS-NEXT:     Section: .text
+; SYMS-NEXT:     Type: 0x0
+; SYMS-NEXT:     StorageClass: C_EXT (0x2)
+; SYMS-NEXT:     NumberOfAuxEntries: 1
+; SYMS-NEXT:     CSECT Auxiliary Entry {
+; SYMS-NEXT:       Index: [[#INDX+13]]
+; SYMS-NEXT:       ContainingCsectSymbolIndex: 2
+; SYMS-NEXT:       ParameterHashIndex: 0x0
+; SYMS-NEXT:       TypeChkSectNum: 0x0
+; SYMS-NEXT:       SymbolAlignmentLog2: 0
+; SYMS-NEXT:       SymbolType: XTY_LD (0x2)
+; SYMS-NEXT:       StorageMappingClass: XMC_RO (0x1)
+; SYMS-NEXT:       StabInfoIndex: 0x0
+; SYMS-NEXT:       StabSectNum: 0x0
+; SYMS-NEXT:     }
+; SYMS-NEXT:   }
+
+; SYMS:        Symbol {
+; SYMS-NEXT:     Index: [[#INDX+14]]
+; SYMS-NEXT:     Name: const_chrarray
+; SYMS-NEXT:     Value (RelocatableAddress): 0x28
+; SYMS-NEXT:     Section: .text
+; SYMS-NEXT:     Type: 0x0
+; SYMS-NEXT:     StorageClass: C_EXT (0x2)
+; SYMS-NEXT:     NumberOfAuxEntries: 1
+; SYMS-NEXT:     CSECT Auxiliary Entry {
+; SYMS-NEXT:       Index: [[#INDX+15]]
+; SYMS-NEXT:       ContainingCsectSymbolIndex: 2
+; SYMS-NEXT:       ParameterHashIndex: 0x0
+; SYMS-NEXT:       TypeChkSectNum: 0x0
+; SYMS-NEXT:       SymbolAlignmentLog2: 0
+; SYMS-NEXT:       SymbolType: XTY_LD (0x2)
+; SYMS-NEXT:       StorageMappingClass: XMC_RO (0x1)
+; SYMS-NEXT:       StabInfoIndex: 0x0
+; SYMS-NEXT:       StabSectNum: 0x0
+; SYMS-NEXT:     }
+; SYMS-NEXT:   }
+
+; SYMS:        Symbol {
+; SYMS-NEXT:     Index: [[#INDX+16]]
+; SYMS-NEXT:     Name: const_dblarr
+; SYMS-NEXT:     Value (RelocatableAddress): 0x30
+; SYMS-NEXT:     Section: .text
+; SYMS-NEXT:     Type: 0x0
+; SYMS-NEXT:     StorageClass: C_EXT (0x2)
+; SYMS-NEXT:     NumberOfAuxEntries: 1
+; SYMS-NEXT:     CSECT Auxiliary Entry {
+; SYMS-NEXT:       Index: [[#INDX+17]]
+; SYMS-NEXT:       ContainingCsectSymbolIndex: 2
+; SYMS-NEXT:       ParameterHashIndex: 0x0
+; SYMS-NEXT:       TypeChkSectNum: 0x0
+; SYMS-NEXT:       SymbolAlignmentLog2: 0
+; SYMS-NEXT:       SymbolType: XTY_LD (0x2)
+; SYMS-NEXT:       StorageMappingClass: XMC_RO (0x1)
+; SYMS-NEXT:       StabInfoIndex: 0x0
+; SYMS-NEXT:       StabSectNum: 0x0
+; SYMS-NEXT:     }
+; SYMS-NEXT:   }
+; SYMS:      ]
+
+; DIS: Disassembly of section .text:
+; DIS: 00000000 const_ivar:
+; DIS-NEXT:        0: 00 00 00 23
+; DIS-NEXT:        4: 00 00 00 00
+
+; DIS: 00000008 const_llvar:
+; DIS-NEXT:        8: 00 00 00 00
+; DIS-NEXT:        c: 00 00 00 24
+
+; DIS: 00000010 const_svar:
+; DIS-NEXT:       10: 00 25 00 00
+
+; DIS: 00000014 const_fvar:
+; DIS-NEXT:       14: 44 48 00 00
+
+; DIS: 00000018 const_dvar:
+; DIS-NEXT:       18: 40 8c 20 00
+; DIS-NEXT:       1c: 00 00 00 00
+
+; DIS: 00000020 const_over_aligned:
+; DIS-NEXT:       20: 40 8c 20 00
+; DIS-NEXT:       24: 00 00 00 00
+
+; DIS: 00000028 const_chrarray:
+; DIS-NEXT:       28: 61 62 63 64
+; DIS-NEXT:       2c: 00 00 00 00
+
+; DIS: 00000030 const_dblarr:
+; DIS-NEXT:       30: 3f f0 00 00
+; DIS-NEXT:       34: 00 00 00 00
+; DIS-NEXT:       38: 40 00 00 00
+; DIS-NEXT:       3c: 00 00 00 00
+; DIS-NEXT:       40: 40 08 00 00
+; DIS-NEXT:       44: 00 00 00 00
+; DIS-NEXT:       48: 40 10 00 00
+; DIS-NEXT:       4c: 00 00 00 00
diff --git a/llvm/test/CodeGen/PowerPC/aix-xcoff-textdisassembly.ll b/llvm/test/CodeGen/PowerPC/aix-xcoff-textdisassembly.ll
new file mode 100644
index 000000000..d80a59ebd
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/aix-xcoff-textdisassembly.ll
@@ -0,0 +1,22 @@
+; RUN: llc -verify-machineinstrs -mcpu=pwr9 -mtriple powerpc-ibm-aix-xcoff -filetype=obj -o %t.o  < %s
+; RUN: llvm-objdump -D %t.o | FileCheck --check-prefix=CHECK %s
+
+define i32 @foo() #0 {
+entry:
+  ret i32 0
+}
+
+define i32 @foo1() #0 {
+entry:
+  ret i32 1
+}
+
+; CHECK:     Disassembly of section .text:{{[[:space:]] *}}
+; CHECK-NEXT:     00000000 .text:
+; CHECK-NEXT:        0: 38 60 00 00                   li 3, 0
+; CHECK-NEXT:        4: 4e 80 00 20                   blr
+; CHECK-NEXT:        8: 60 00 00 00                   nop
+; CHECK-NEXT:        c: 60 00 00 00                   nop
+; CHECK:     00000010 .foo1:
+; CHECK-NEXT:       10: 38 60 00 01                   li 3, 1
+; CHECK-NEXT:       14: 4e 80 00 20                   blr
diff --git a/llvm/test/CodeGen/PowerPC/aix-xcoff-toc.ll b/llvm/test/CodeGen/PowerPC/aix-xcoff-toc.ll
new file mode 100644
index 000000000..543cca720
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/aix-xcoff-toc.ll
@@ -0,0 +1,338 @@
+; This file tests TOC entry generation and undefined symbol generation.
+
+; RUN: llc  -verify-machineinstrs -mcpu=pwr4 -mtriple powerpc-ibm-aix-xcoff < %s | FileCheck --check-prefixes CHECK,CHECK32 %s
+; RUN: llc  -verify-machineinstrs -mcpu=pwr4 -mtriple powerpc64-ibm-aix-xcoff < %s 2>&1 | FileCheck --check-prefixes CHECK,CHECK64  %s
+
+; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mtriple powerpc-ibm-aix-xcoff -filetype=obj -o %t.o < %s
+; RUN: llvm-readobj --syms %t.o | FileCheck --check-prefix=SYM %s
+
+; RUN: not llc -verify-machineinstrs -mcpu=pwr4 -mtriple powerpc64-ibm-aix-xcoff -filetype=obj -o %t.o 2>&1 \
+; RUN: < %s | FileCheck --check-prefix=XCOFF64 %s
+; XCOFF64: LLVM ERROR: 64-bit XCOFF object files are not supported yet.
+
+@a = external global i32, align 4
+@b = external global i64, align 8
+@c = external global i16, align 2
+@globa = common global i32 0, align 4
+
+@ptr = internal global void (...)* null, align 4
+
+declare void @foo()
+
+define void @bar() {
+  %1 = alloca i8*, align 8
+  %2 = alloca i8*, align 8
+  store i32 0, i32* @a, align 4
+  store i64 0, i64* @b, align 8
+  store i16 0, i16* @c, align 2
+  store i32 0, i32* @globa, align 4
+  store void (...)* bitcast (void ()* @bar to void (...)*), void (...)** @ptr, align 4
+  store i8* bitcast (void ()* @foo to i8*), i8** %1, align 8
+  store i8* bitcast (void ()* @foobar to i8*), i8** %2, align 8
+  ret void
+}
+
+; We initialize a csect when we first reference an external global, so make sure we don't run into problems when we see it again.
+define void @bar2() {
+  store i32 0, i32* @a, align 4
+  store i64 0, i64* @b, align 8
+  store i16 0, i16* @c, align 2
+  ret void
+}
+
+define void @foobar() {
+  ret void
+}
+
+; Test tc entry assembly generation.
+
+; CHECK-NOT: .comm a
+; CHECK-NOT: .lcomm a
+; CHECK-NOT: .comm b
+; CHECK-NOT: .lcomm b
+; CHECK-NOT: .comm c
+; CHECK-NOT: .lcomm c
+; CHECK: .comm globa[RW],4,2
+; CHECK32: .lcomm ptr,4,ptr[BS],2
+; CHECK64: .lcomm ptr,8,ptr[BS],2
+; CHECK:      .toc
+; CHECK-NEXT: LC0:
+; CHECK-NEXT: .tc   a[TC],a[UA]
+; CHECK-NEXT: LC1:
+; CHECK-NEXT: .tc   b[TC],b[UA]
+; CHECK-NEXT: LC2:
+; CHECK-NEXT: .tc   c[TC],c[UA]
+; CHECK-NEXT: LC3:
+; CHECK-NEXT: .tc   globa[TC],globa[RW]
+; CHECK-NEXT: LC4:
+; CHECK-NEXT: .tc   ptr[TC],ptr[BS]
+; CHECK-NEXT: LC5:
+; CHECK-NEXT: .tc   bar[TC],bar[DS]
+; CHECK-NEXT: LC6:
+; CHECK-NEXT: .tc   foo[TC],foo[DS]
+; CHECK-NEXT: LC7:
+; CHECK-NEXT: .tc   foobar[TC],foobar[DS]
+
+
+; Test undefined symbol generation.
+
+; SYM:       File: {{.*}}aix-xcoff-toc.ll.tmp.o
+; SYM:       Symbol {{[{][[:space:]] *}}Index: [[#UNDEF_INDX:]]{{[[:space:]] *}}Name: a
+; SYM-NEXT:   Value (RelocatableAddress): 0x0
+; SYM-NEXT:   Section: N_UNDEF
+; SYM-NEXT:   Type: 0x0
+; SYM-NEXT:   StorageClass: C_EXT (0x2)
+; SYM-NEXT:   NumberOfAuxEntries: 1
+; SYM-NEXT:   CSECT Auxiliary Entry {
+; SYM-NEXT:     Index: [[#UNDEF_INDX+1]]
+; SYM-NEXT:     SectionLen: 0
+; SYM-NEXT:     ParameterHashIndex: 0x0
+; SYM-NEXT:     TypeChkSectNum: 0x0
+; SYM-NEXT:     SymbolAlignmentLog2: 0
+; SYM-NEXT:     SymbolType: XTY_ER (0x0)
+; SYM-NEXT:     StorageMappingClass: XMC_UA (0x4)
+; SYM-NEXT:     StabInfoIndex: 0x0
+; SYM-NEXT:     StabSectNum: 0x0
+; SYM-NEXT:   }
+; SYM-NEXT: }
+; SYM-NEXT: Symbol {
+; SYM-NEXT:   Index: [[#UNDEF_INDX+2]]
+; SYM-NEXT:   Name: b
+; SYM-NEXT:   Value (RelocatableAddress): 0x0
+; SYM-NEXT:   Section: N_UNDEF
+; SYM-NEXT:   Type: 0x0
+; SYM-NEXT:   StorageClass: C_EXT (0x2)
+; SYM-NEXT:   NumberOfAuxEntries: 1
+; SYM-NEXT:   CSECT Auxiliary Entry {
+; SYM-NEXT:     Index: [[#UNDEF_INDX+3]]
+; SYM-NEXT:     SectionLen: 0
+; SYM-NEXT:     ParameterHashIndex: 0x0
+; SYM-NEXT:     TypeChkSectNum: 0x0
+; SYM-NEXT:     SymbolAlignmentLog2: 0
+; SYM-NEXT:     SymbolType: XTY_ER (0x0)
+; SYM-NEXT:     StorageMappingClass: XMC_UA (0x4)
+; SYM-NEXT:     StabInfoIndex: 0x0
+; SYM-NEXT:     StabSectNum: 0x0
+; SYM-NEXT:   }
+; SYM-NEXT: }
+; SYM-NEXT: Symbol {
+; SYM-NEXT:   Index: [[#UNDEF_INDX+4]]
+; SYM-NEXT:   Name: c
+; SYM-NEXT:   Value (RelocatableAddress): 0x0
+; SYM-NEXT:   Section: N_UNDEF
+; SYM-NEXT:   Type: 0x0
+; SYM-NEXT:   StorageClass: C_EXT (0x2)
+; SYM-NEXT:   NumberOfAuxEntries: 1
+; SYM-NEXT:   CSECT Auxiliary Entry {
+; SYM-NEXT:     Index: [[#UNDEF_INDX+5]]
+; SYM-NEXT:     SectionLen: 0
+; SYM-NEXT:     ParameterHashIndex: 0x0
+; SYM-NEXT:     TypeChkSectNum: 0x0
+; SYM-NEXT:     SymbolAlignmentLog2: 0
+; SYM-NEXT:     SymbolType: XTY_ER (0x0)
+; SYM-NEXT:     StorageMappingClass: XMC_UA (0x4)
+; SYM-NEXT:     StabInfoIndex: 0x0
+; SYM-NEXT:     StabSectNum: 0x0
+; SYM-NEXT:   }
+; SYM-NEXT: }
+; SYM-NEXT: Symbol {
+; SYM-NEXT:   Index: [[#UNDEF_INDX+6]]
+; SYM-NEXT:   Name: foo
+; SYM-NEXT:   Value (RelocatableAddress): 0x0
+; SYM-NEXT:   Section: N_UNDEF
+; SYM-NEXT:   Type: 0x0
+; SYM-NEXT:   StorageClass: C_EXT (0x2)
+; SYM-NEXT:   NumberOfAuxEntries: 1
+; SYM-NEXT:   CSECT Auxiliary Entry {
+; SYM-NEXT:     Index: [[#UNDEF_INDX+7]]
+; SYM-NEXT:     SectionLen: 0
+; SYM-NEXT:     ParameterHashIndex: 0x0
+; SYM-NEXT:     TypeChkSectNum: 0x0
+; SYM-NEXT:     SymbolAlignmentLog2: 0
+; SYM-NEXT:     SymbolType: XTY_ER (0x0)
+; SYM-NEXT:     StorageMappingClass: XMC_DS (0xA)
+; SYM-NEXT:     StabInfoIndex: 0x0
+; SYM-NEXT:     StabSectNum: 0x0
+; SYM-NEXT:   }
+; SYM-NEXT: }
+
+; Test TOC entry symbol generation.
+
+; SYM:       Symbol {{[{][[:space:]] *}}Index: [[#TOC_INDX:]]{{[[:space:]] *}}Name: TOC
+; SYM-NEXT:    Value (RelocatableAddress): 0xA8
+; SYM-NEXT:    Section: .data
+; SYM-NEXT:    Type: 0x0
+; SYM-NEXT:    StorageClass: C_HIDEXT (0x6B)
+; SYM-NEXT:    NumberOfAuxEntries: 1
+; SYM-NEXT:    CSECT Auxiliary Entry {
+; SYM-NEXT:      Index: [[#TOC_INDX+1]]
+; SYM-NEXT:      SectionLen: 0
+; SYM-NEXT:      ParameterHashIndex: 0x0
+; SYM-NEXT:      TypeChkSectNum: 0x0
+; SYM-NEXT:      SymbolAlignmentLog2: 2
+; SYM-NEXT:      SymbolType: XTY_SD (0x1)
+; SYM-NEXT:      StorageMappingClass: XMC_TC0 (0xF)
+; SYM-NEXT:      StabInfoIndex: 0x0
+; SYM-NEXT:      StabSectNum: 0x0
+; SYM-NEXT:    }
+; SYM-NEXT:  }
+; SYM-NEXT:  Symbol {
+; SYM-NEXT:    Index: [[#TOC_INDX+2]]
+; SYM-NEXT:    Name: a
+; SYM-NEXT:    Value (RelocatableAddress): 0xA8
+; SYM-NEXT:    Section: .data
+; SYM-NEXT:    Type: 0x0
+; SYM-NEXT:    StorageClass: C_HIDEXT (0x6B)
+; SYM-NEXT:    NumberOfAuxEntries: 1
+; SYM-NEXT:    CSECT Auxiliary Entry {
+; SYM-NEXT:      Index: [[#TOC_INDX+3]]
+; SYM-NEXT:      SectionLen: 4
+; SYM-NEXT:      ParameterHashIndex: 0x0
+; SYM-NEXT:      TypeChkSectNum: 0x0
+; SYM-NEXT:      SymbolAlignmentLog2: 2
+; SYM-NEXT:      SymbolType: XTY_SD (0x1)
+; SYM-NEXT:      StorageMappingClass: XMC_TC (0x3)
+; SYM-NEXT:      StabInfoIndex: 0x0
+; SYM-NEXT:      StabSectNum: 0x0
+; SYM-NEXT:    }
+; SYM-NEXT:  }
+; SYM-NEXT:  Symbol {
+; SYM-NEXT:    Index: [[#TOC_INDX+4]]
+; SYM-NEXT:    Name: b
+; SYM-NEXT:    Value (RelocatableAddress): 0xAC
+; SYM-NEXT:    Section: .data
+; SYM-NEXT:    Type: 0x0
+; SYM-NEXT:    StorageClass: C_HIDEXT (0x6B)
+; SYM-NEXT:    NumberOfAuxEntries: 1
+; SYM-NEXT:    CSECT Auxiliary Entry {
+; SYM-NEXT:      Index: [[#TOC_INDX+5]]
+; SYM-NEXT:      SectionLen: 4
+; SYM-NEXT:      ParameterHashIndex: 0x0
+; SYM-NEXT:      TypeChkSectNum: 0x0
+; SYM-NEXT:      SymbolAlignmentLog2: 2
+; SYM-NEXT:      SymbolType: XTY_SD (0x1)
+; SYM-NEXT:      StorageMappingClass: XMC_TC (0x3)
+; SYM-NEXT:      StabInfoIndex: 0x0
+; SYM-NEXT:      StabSectNum: 0x0
+; SYM-NEXT:    }
+; SYM-NEXT:  }
+; SYM-NEXT:  Symbol {
+; SYM-NEXT:    Index: [[#TOC_INDX+6]]
+; SYM-NEXT:    Name: c
+; SYM-NEXT:    Value (RelocatableAddress): 0xB0
+; SYM-NEXT:    Section: .data
+; SYM-NEXT:    Type: 0x0
+; SYM-NEXT:    StorageClass: C_HIDEXT (0x6B)
+; SYM-NEXT:    NumberOfAuxEntries: 1
+; SYM-NEXT:    CSECT Auxiliary Entry {
+; SYM-NEXT:      Index: [[#TOC_INDX+7]]
+; SYM-NEXT:      SectionLen: 4
+; SYM-NEXT:      ParameterHashIndex: 0x0
+; SYM-NEXT:      TypeChkSectNum: 0x0
+; SYM-NEXT:      SymbolAlignmentLog2: 2
+; SYM-NEXT:      SymbolType: XTY_SD (0x1)
+; SYM-NEXT:      StorageMappingClass: XMC_TC (0x3)
+; SYM-NEXT:      StabInfoIndex: 0x0
+; SYM-NEXT:      StabSectNum: 0x0
+; SYM-NEXT:    }
+; SYM-NEXT:  }
+; SYM-NEXT:  Symbol {
+; SYM-NEXT:    Index: [[#TOC_INDX+8]]
+; SYM-NEXT:    Name: globa
+; SYM-NEXT:    Value (RelocatableAddress): 0xB4
+; SYM-NEXT:    Section: .data
+; SYM-NEXT:    Type: 0x0
+; SYM-NEXT:    StorageClass: C_HIDEXT (0x6B)
+; SYM-NEXT:    NumberOfAuxEntries: 1
+; SYM-NEXT:    CSECT Auxiliary Entry {
+; SYM-NEXT:      Index: [[#TOC_INDX+9]]
+; SYM-NEXT:      SectionLen: 4
+; SYM-NEXT:      ParameterHashIndex: 0x0
+; SYM-NEXT:      TypeChkSectNum: 0x0
+; SYM-NEXT:      SymbolAlignmentLog2: 2
+; SYM-NEXT:      SymbolType: XTY_SD (0x1)
+; SYM-NEXT:      StorageMappingClass: XMC_TC (0x3)
+; SYM-NEXT:      StabInfoIndex: 0x0
+; SYM-NEXT:      StabSectNum: 0x0
+; SYM-NEXT:    }
+; SYM-NEXT:  }
+; SYM-NEXT:  Symbol {
+; SYM-NEXT:    Index: [[#TOC_INDX+10]]
+; SYM-NEXT:    Name: ptr
+; SYM-NEXT:    Value (RelocatableAddress): 0xB8
+; SYM-NEXT:    Section: .data
+; SYM-NEXT:    Type: 0x0
+; SYM-NEXT:    StorageClass: C_HIDEXT (0x6B)
+; SYM-NEXT:    NumberOfAuxEntries: 1
+; SYM-NEXT:    CSECT Auxiliary Entry {
+; SYM-NEXT:      Index: [[#TOC_INDX+11]]
+; SYM-NEXT:      SectionLen: 4
+; SYM-NEXT:      ParameterHashIndex: 0x0
+; SYM-NEXT:      TypeChkSectNum: 0x0
+; SYM-NEXT:      SymbolAlignmentLog2: 2
+; SYM-NEXT:      SymbolType: XTY_SD (0x1)
+; SYM-NEXT:      StorageMappingClass: XMC_TC (0x3)
+; SYM-NEXT:      StabInfoIndex: 0x0
+; SYM-NEXT:      StabSectNum: 0x0
+; SYM-NEXT:    }
+; SYM-NEXT:  }
+; SYM-NEXT:  Symbol {
+; SYM-NEXT:    Index: [[#TOC_INDX+12]]
+; SYM-NEXT:    Name: bar
+; SYM-NEXT:    Value (RelocatableAddress): 0xBC
+; SYM-NEXT:    Section: .data
+; SYM-NEXT:    Type: 0x0
+; SYM-NEXT:    StorageClass: C_HIDEXT (0x6B)
+; SYM-NEXT:    NumberOfAuxEntries: 1
+; SYM-NEXT:    CSECT Auxiliary Entry {
+; SYM-NEXT:      Index: [[#TOC_INDX+13]]
+; SYM-NEXT:      SectionLen: 4
+; SYM-NEXT:      ParameterHashIndex: 0x0
+; SYM-NEXT:      TypeChkSectNum: 0x0
+; SYM-NEXT:      SymbolAlignmentLog2: 2
+; SYM-NEXT:      SymbolType: XTY_SD (0x1)
+; SYM-NEXT:      StorageMappingClass: XMC_TC (0x3)
+; SYM-NEXT:      StabInfoIndex: 0x0
+; SYM-NEXT:      StabSectNum: 0x0
+; SYM-NEXT:    }
+; SYM-NEXT:  }
+; SYM-NEXT:  Symbol {
+; SYM-NEXT:    Index: [[#TOC_INDX+14]]
+; SYM-NEXT:    Name: foo
+; SYM-NEXT:    Value (RelocatableAddress): 0xC0
+; SYM-NEXT:    Section: .data
+; SYM-NEXT:    Type: 0x0
+; SYM-NEXT:    StorageClass: C_HIDEXT (0x6B)
+; SYM-NEXT:    NumberOfAuxEntries: 1
+; SYM-NEXT:    CSECT Auxiliary Entry {
+; SYM-NEXT:      Index: [[#TOC_INDX+15]]
+; SYM-NEXT:      SectionLen: 4
+; SYM-NEXT:      ParameterHashIndex: 0x0
+; SYM-NEXT:      TypeChkSectNum: 0x0
+; SYM-NEXT:      SymbolAlignmentLog2: 2
+; SYM-NEXT:      SymbolType: XTY_SD (0x1)
+; SYM-NEXT:      StorageMappingClass: XMC_TC (0x3)
+; SYM-NEXT:      StabInfoIndex: 0x0
+; SYM-NEXT:      StabSectNum: 0x0
+; SYM-NEXT:    }
+; SYM-NEXT:  }
+; SYM-NEXT:  Symbol {
+; SYM-NEXT:    Index: [[#TOC_INDX+16]]
+; SYM-NEXT:    Name: foobar
+; SYM-NEXT:    Value (RelocatableAddress): 0xC4
+; SYM-NEXT:    Section: .data
+; SYM-NEXT:    Type: 0x0
+; SYM-NEXT:    StorageClass: C_HIDEXT (0x6B)
+; SYM-NEXT:    NumberOfAuxEntries: 1
+; SYM-NEXT:    CSECT Auxiliary Entry {
+; SYM-NEXT:      Index: [[#TOC_INDX+17]]
+; SYM-NEXT:      SectionLen: 4
+; SYM-NEXT:      ParameterHashIndex: 0x0
+; SYM-NEXT:      TypeChkSectNum: 0x0
+; SYM-NEXT:      SymbolAlignmentLog2: 2
+; SYM-NEXT:      SymbolType: XTY_SD (0x1)
+; SYM-NEXT:      StorageMappingClass: XMC_TC (0x3)
+; SYM-NEXT:      StabInfoIndex: 0x0
+; SYM-NEXT:      StabSectNum: 0x0
+; SYM-NEXT:    }
+; SYM-NEXT:  }
diff --git a/llvm/test/CodeGen/PowerPC/and-mask.ll b/llvm/test/CodeGen/PowerPC/and-mask.ll
new file mode 100644
index 000000000..89f568196
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/and-mask.ll
@@ -0,0 +1,88 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=powerpc64le-unknown-linux-gnu < %s | FileCheck %s
+
+; mask 0xFFFFFFFE
+define i32 @test1(i32 %a) {
+; CHECK-LABEL: test1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    rlwinm 3, 3, 0, 0, 30
+; CHECK-NEXT:    blr
+  %and = and i32 %a, -2
+  ret i32 %and
+}
+
+; mask 0xFFFFFFFFFFFFFFF9
+define i64 @test2(i64 %a) {
+; CHECK-LABEL: test2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li 4, -7
+; CHECK-NEXT:    and 3, 3, 4
+; CHECK-NEXT:    blr
+  %and = and i64 %a, -7
+  ret i64 %and
+}
+
+; mask: 0xFFFFFFC00000
+define i64 @test3(i64 %a) {
+; CHECK-LABEL: test3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lis 4, 1023
+; CHECK-NEXT:    ori 4, 4, 65535
+; CHECK-NEXT:    sldi 4, 4, 22
+; CHECK-NEXT:    and 3, 3, 4
+; CHECK-NEXT:    blr
+  %and = and i64 %a, 281474972516352
+  ret i64 %and
+}
+
+; mask: 0xC000000FF
+define i64 @test4(i64 %a) {
+; CHECK-LABEL: test4:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li 4, 12
+; CHECK-NEXT:    sldi 4, 4, 32
+; CHECK-NEXT:    ori 4, 4, 255
+; CHECK-NEXT:    and 3, 3, 4
+; CHECK-NEXT:    blr
+  %and = and i64 %a, 51539607807
+  ret i64 %and
+}
+
+; mask: 0xFFC0FFFF
+define i64 @test5(i64 %a) {
+; CHECK-LABEL: test5:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li 4, 0
+; CHECK-NEXT:    oris 4, 4, 65472
+; CHECK-NEXT:    ori 4, 4, 65535
+; CHECK-NEXT:    and 3, 3, 4
+; CHECK-NEXT:    blr
+  %and = and i64 %a, 4290838527
+  ret i64 %and
+}
+
+; mask: 0x3FC0FFE0
+define i64 @test6(i64 %a) {
+; CHECK-LABEL: test6:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lis 4, 16320
+; CHECK-NEXT:    ori 4, 4, 65504
+; CHECK-NEXT:    and 3, 3, 4
+; CHECK-NEXT:    blr
+  %and = and i64 %a, 1069613024
+  ret i64 %and
+}
+
+; mask: 0x3FC000001FFFF
+define i64 @test7(i64 %a) {
+; CHECK-LABEL: test7:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li 4, -32767
+; CHECK-NEXT:    sldi 4, 4, 32
+; CHECK-NEXT:    oris 4, 4, 65024
+; CHECK-NEXT:    rldicr 4, 4, 17, 63
+; CHECK-NEXT:    and 3, 3, 4
+; CHECK-NEXT:    blr
+  %and = and i64 %a, 1121501860462591
+  ret i64 %and
+}
diff --git a/llvm/test/CodeGen/PowerPC/asm-constraints.ll b/llvm/test/CodeGen/PowerPC/asm-constraints.ll
index 878ada372..a3e573d89 100644
--- a/llvm/test/CodeGen/PowerPC/asm-constraints.ll
+++ b/llvm/test/CodeGen/PowerPC/asm-constraints.ll
@@ -65,7 +65,7 @@ entry:
 }
 
 
-attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+vsx,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+vsx,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 attributes #1 = { nounwind }
 
diff --git a/llvm/test/CodeGen/PowerPC/asym-regclass-copy.ll b/llvm/test/CodeGen/PowerPC/asym-regclass-copy.ll
index 0f46fe1b6..f04d55976 100644
--- a/llvm/test/CodeGen/PowerPC/asym-regclass-copy.ll
+++ b/llvm/test/CodeGen/PowerPC/asym-regclass-copy.ll
@@ -52,5 +52,5 @@ declare void @free(i8* nocapture) #0
 
 declare i64 @strtol(i8*, i8** nocapture, i32 signext) #0
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind }
diff --git a/llvm/test/CodeGen/PowerPC/available-externally.ll b/llvm/test/CodeGen/PowerPC/available-externally.ll
index 51b972bd3..ec83c6799 100644
--- a/llvm/test/CodeGen/PowerPC/available-externally.ll
+++ b/llvm/test/CodeGen/PowerPC/available-externally.ll
@@ -1,3 +1,4 @@
+; RUN: llc -verify-machineinstrs < %s | FileCheck %s -check-prefix=STATIC
 ; RUN: llc -verify-machineinstrs < %s -relocation-model=static | FileCheck %s -check-prefix=STATIC
 ; RUN: llc -verify-machineinstrs < %s -relocation-model=pic -mtriple=powerpc-unknown-linux-gnu | FileCheck %s -check-prefix=PIC
 ; RUN: llc -verify-machineinstrs < %s -relocation-model=pic -mtriple=powerpc-unknown-linux | FileCheck %s -check-prefix=PICELF
@@ -14,7 +15,7 @@ target triple = "powerpc-unknown-linux-gnu"
 define i32 @foo(i64 %x) nounwind {
 entry:
 ; STATIC: foo:
-; STATIC: bl exact_log2@PLT
+; STATIC: bl exact_log2
 ; STATIC: blr
 
 ; PIC: foo:
diff --git a/llvm/test/CodeGen/PowerPC/bitcasts-direct-move.ll b/llvm/test/CodeGen/PowerPC/bitcasts-direct-move.ll
index 54e378e8d..0eafd86bb 100644
--- a/llvm/test/CodeGen/PowerPC/bitcasts-direct-move.ll
+++ b/llvm/test/CodeGen/PowerPC/bitcasts-direct-move.ll
@@ -40,7 +40,7 @@ entry:
   ret double %0
 ; CHECK-P7: std 3,
 ; CHECK-P7: lfd 1,
-; CHECK: mtvsrd 1, 3
+; CHECK: mtfprd 1, 3
 }
 
 define zeroext i32 @f32toi32u(float %a) {
@@ -80,5 +80,5 @@ entry:
   ret double %0
 ; CHECK-P7: std 3,
 ; CHECK-P7: lfd 1,
-; CHECK: mtvsrd 1, 3
+; CHECK: mtfprd 1, 3
 }
diff --git a/llvm/test/CodeGen/PowerPC/branch-opt.ll b/llvm/test/CodeGen/PowerPC/branch-opt.ll
index 5e3127084..af75e761d 100644
--- a/llvm/test/CodeGen/PowerPC/branch-opt.ll
+++ b/llvm/test/CodeGen/PowerPC/branch-opt.ll
@@ -8,13 +8,11 @@ target triple = "powerpc-unknown-linux-gnu"
 ; The last (whichever it is) should have a fallthrough exit, and the other three
 ; need an unconditional branch. No other block should have an unconditional
 ; branch to cond_next48
-; One of the blocks ends up with a loop exit block that gets a tail-duplicated copy
-; of %cond_next48, so there should only be two unconditional branches.
 
-;CHECK: b .LBB0_13
-;CHECK: b .LBB0_13
-;CHECK-NOT: b .LBB0_13
-;CHECK: .LBB0_13: # %cond_next48
+;CHECK: .LBB0_7: # %cond_next48
+;CHECK: b .LBB0_7
+;CHECK: b .LBB0_7
+;CHECK: b .LBB0_7
 
 define void @foo(i32 %W, i32 %X, i32 %Y, i32 %Z) {
 entry:
diff --git a/llvm/test/CodeGen/PowerPC/brcond.ll b/llvm/test/CodeGen/PowerPC/brcond.ll
index 5bf8c6c1f..b8c98427f 100644
--- a/llvm/test/CodeGen/PowerPC/brcond.ll
+++ b/llvm/test/CodeGen/PowerPC/brcond.ll
@@ -1,5 +1,7 @@
-; RUN: llc -verify-machineinstrs -mtriple=powerpc64-unknown-unknown < %s | FileCheck %s
-; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-unknown < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64-unknown-unknown \
+; RUN:   -ppc-reduce-cr-logicals=false < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-unknown \
+; RUN:   -ppc-reduce-cr-logicals=false < %s | FileCheck %s
 
 define signext i32 @testi32slt(i32 signext %c1, i32 signext %c2, i32 signext %c3, i32 signext %c4, i32 signext %a1, i32 signext %a2) #0 {
 ; CHECK-LABEL: testi32slt
diff --git a/llvm/test/CodeGen/PowerPC/build-vector-allones.ll b/llvm/test/CodeGen/PowerPC/build-vector-allones.ll
new file mode 100644
index 000000000..fcec36703
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/build-vector-allones.ll
@@ -0,0 +1,109 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mcpu=pwr7 -ppc-asm-full-reg-names -verify-machineinstrs \
+; RUN:   -mtriple=powerpc64-unknown-unknown < %s | FileCheck %s \
+; RUN:   -check-prefix=P7BE
+; RUN: llc -mcpu=pwr8 -ppc-asm-full-reg-names -verify-machineinstrs \
+; RUN:   -mtriple=powerpc64le-unknown-unknown < %s | FileCheck %s \
+; RUN:   -check-prefix=P8LE
+; RUN: llc -mcpu=pwr9 -ppc-asm-full-reg-names -verify-machineinstrs \
+; RUN:   -mtriple=powerpc64le-unknown-unknown < %s | FileCheck %s \
+; RUN:   -check-prefix=P9LE
+
+; FIXME: P7BE for i128 looks wrong.
+define <1 x i128> @One1i128() {
+; P7BE-LABEL: One1i128:
+; P7BE:       # %bb.0: # %entry
+; P7BE-NEXT:    li r3, -1
+; P7BE-NEXT:    li r4, -1
+; P7BE-NEXT:    blr
+;
+; P8LE-LABEL: One1i128:
+; P8LE:       # %bb.0: # %entry
+; P8LE-NEXT:    xxleqv vs34, vs34, vs34
+; P8LE-NEXT:    blr
+;
+; P9LE-LABEL: One1i128:
+; P9LE:       # %bb.0: # %entry
+; P9LE-NEXT:    xxleqv vs34, vs34, vs34
+; P9LE-NEXT:    blr
+entry:
+  ret <1 x i128> <i128 -1>
+}
+
+define <2 x i64> @One2i64() {
+; P7BE-LABEL: One2i64:
+; P7BE:       # %bb.0: # %entry
+; P7BE-NEXT:    addis r3, r2, .LCPI1_0@toc@ha
+; P7BE-NEXT:    addi r3, r3, .LCPI1_0@toc@l
+; P7BE-NEXT:    lxvd2x vs34, 0, r3
+; P7BE-NEXT:    blr
+;
+; P8LE-LABEL: One2i64:
+; P8LE:       # %bb.0: # %entry
+; P8LE-NEXT:    xxleqv vs34, vs34, vs34
+; P8LE-NEXT:    blr
+;
+; P9LE-LABEL: One2i64:
+; P9LE:       # %bb.0: # %entry
+; P9LE-NEXT:    xxleqv vs34, vs34, vs34
+; P9LE-NEXT:    blr
+entry:
+  ret <2 x i64> <i64 -1, i64 -1>
+}
+
+define <4 x i32> @One4i32() {
+; P7BE-LABEL: One4i32:
+; P7BE:       # %bb.0: # %entry
+; P7BE-NEXT:    vspltisb v2, -1
+; P7BE-NEXT:    blr
+;
+; P8LE-LABEL: One4i32:
+; P8LE:       # %bb.0: # %entry
+; P8LE-NEXT:    xxleqv vs34, vs34, vs34
+; P8LE-NEXT:    blr
+;
+; P9LE-LABEL: One4i32:
+; P9LE:       # %bb.0: # %entry
+; P9LE-NEXT:    xxleqv vs34, vs34, vs34
+; P9LE-NEXT:    blr
+entry:
+  ret <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>
+}
+
+define <8 x i16> @One8i16() {
+; P7BE-LABEL: One8i16:
+; P7BE:       # %bb.0: # %entry
+; P7BE-NEXT:    vspltisb v2, -1
+; P7BE-NEXT:    blr
+;
+; P8LE-LABEL: One8i16:
+; P8LE:       # %bb.0: # %entry
+; P8LE-NEXT:    xxleqv vs34, vs34, vs34
+; P8LE-NEXT:    blr
+;
+; P9LE-LABEL: One8i16:
+; P9LE:       # %bb.0: # %entry
+; P9LE-NEXT:    xxleqv vs34, vs34, vs34
+; P9LE-NEXT:    blr
+entry:
+  ret <8 x i16> <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>
+}
+
+define <16 x i8> @One16i8() {
+; P7BE-LABEL: One16i8:
+; P7BE:       # %bb.0: # %entry
+; P7BE-NEXT:    vspltisb v2, -1
+; P7BE-NEXT:    blr
+;
+; P8LE-LABEL: One16i8:
+; P8LE:       # %bb.0: # %entry
+; P8LE-NEXT:    xxleqv vs34, vs34, vs34
+; P8LE-NEXT:    blr
+;
+; P9LE-LABEL: One16i8:
+; P9LE:       # %bb.0: # %entry
+; P9LE-NEXT:    xxleqv vs34, vs34, vs34
+; P9LE-NEXT:    blr
+entry:
+  ret <16 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
+}
diff --git a/llvm/test/CodeGen/PowerPC/build-vector-tests.ll b/llvm/test/CodeGen/PowerPC/build-vector-tests.ll
index adbd7622a..4e096b1c5 100644
--- a/llvm/test/CodeGen/PowerPC/build-vector-tests.ll
+++ b/llvm/test/CodeGen/PowerPC/build-vector-tests.ll
@@ -1,15 +1,15 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mcpu=pwr9 -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr \
-; RUN:   -mtriple=powerpc64-unknown-unknown < %s | FileCheck -allow-deprecated-dag-overlap %s \
+; RUN:   -mtriple=powerpc64-unknown-unknown < %s | FileCheck  %s \
 ; RUN:   -check-prefix=P9BE -implicit-check-not frsp
 ; RUN: llc -mcpu=pwr9 -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr \
-; RUN:   -mtriple=powerpc64le-unknown-unknown < %s | FileCheck -allow-deprecated-dag-overlap %s \
+; RUN:   -mtriple=powerpc64le-unknown-unknown < %s | FileCheck  %s \
 ; RUN:   -check-prefix=P9LE -implicit-check-not frsp
 ; RUN: llc -mcpu=pwr8 -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr \
-; RUN:   -mtriple=powerpc64-unknown-unknown < %s | FileCheck -allow-deprecated-dag-overlap %s \
+; RUN:   -mtriple=powerpc64-unknown-unknown < %s | FileCheck  %s \
 ; RUN:   -check-prefix=P8BE -implicit-check-not frsp
 ; RUN: llc -mcpu=pwr8 -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr \
-; RUN:   -mtriple=powerpc64le-unknown-unknown < %s | FileCheck -allow-deprecated-dag-overlap %s \
+; RUN:   -mtriple=powerpc64le-unknown-unknown < %s | FileCheck  %s \
 ; RUN:   -check-prefix=P8LE -implicit-check-not frsp
 
 ; This test case comes from the following C test case (included as it may be
@@ -18,7 +18,7 @@
 ;/*  This test case provides various ways of building vectors to ensure we
 ;    produce optimal code for all cases. The cases are (for each type):
 ;    - All zeros
-;    - All ones
+;    - All ones - split to build-vector-allones.ll
 ;    - Splat of a constant
 ;    - From different values already in registers
 ;    - From different constants
@@ -47,11 +47,6 @@
 ;}                                                                            //
 ;// P8: vspltisb -1                                                           //
 ;// P9: xxspltisb 255                                                         //
-;vector int allOnei() {                                                       //
-;  return (vector int)-1;                                                     //
-;}                                                                            //
-;// P8: vspltisw 1                                                            //
-;// P9: vspltisw 1                                                            //
 ;vector int spltConst1i() {                                                   //
 ;  return (vector int)1;                                                      //
 ;}                                                                            //
@@ -218,11 +213,6 @@
 ;}                                                                            //
 ;// P8: vspltisb -1                                                           //
 ;// P9: xxspltisb 255                                                         //
-;vector unsigned int allOneui() {                                             //
-;  return (vector unsigned int)-1;                                            //
-;}                                                                            //
-;// P8: vspltisw 1                                                            //
-;// P9: vspltisw 1                                                            //
 ;vector unsigned int spltConst1ui() {                                         //
 ;  return (vector unsigned int)1;                                             //
 ;}                                                                            //
@@ -398,11 +388,6 @@
 ;}                                                                            //
 ;// P8: vspltisb -1                                                           //
 ;// P9: xxspltisb 255                                                         //
-;vector long long allOnell() {                                                //
-;  return (vector long long)-1;                                               //
-;}                                                                            //
-;// P8: constant pool load (possible: vmrgew (xxlxor), (vspltisw))            //
-;// P9: constant pool load (possible: vmrgew (xxlxor), (vspltisw))            //
 ;vector long long spltConst1ll() {                                            //
 ;  return (vector long long)1;                                                //
 ;}                                                                            //
@@ -565,11 +550,6 @@
 ;}                                                                            //
 ;// P8: vspltisb -1                                                           //
 ;// P9: xxspltisb 255                                                         //
-;vector unsigned long long allOneull() {                                      //
-;  return (vector unsigned long long)-1;                                      //
-;}                                                                            //
-;// P8: constant pool load (possible: vmrgew (xxlxor), (vspltisw))            //
-;// P9: constant pool load (possible: vmrgew (xxlxor), (vspltisw))            //
 ;vector unsigned long long spltConst1ull() {                                  //
 ;  return (vector unsigned long long)1;                                       //
 ;}                                                                            //
@@ -729,7 +709,6 @@
 ;}                                                                            //
 ;/*========================== unsigned long long ==============================*/
 
-; Function Attrs: norecurse nounwind readnone
 define <4 x i32> @allZeroi() {
 ; P9BE-LABEL: allZeroi:
 ; P9BE:       # %bb.0: # %entry
@@ -754,32 +733,6 @@ entry:
   ret <4 x i32> zeroinitializer
 }
 
-; Function Attrs: norecurse nounwind readnone
-define <4 x i32> @allOnei() {
-; P9BE-LABEL: allOnei:
-; P9BE:       # %bb.0: # %entry
-; P9BE-NEXT:    xxspltib v2, 255
-; P9BE-NEXT:    blr
-;
-; P9LE-LABEL: allOnei:
-; P9LE:       # %bb.0: # %entry
-; P9LE-NEXT:    xxspltib v2, 255
-; P9LE-NEXT:    blr
-;
-; P8BE-LABEL: allOnei:
-; P8BE:       # %bb.0: # %entry
-; P8BE-NEXT:    vspltisb v2, -1
-; P8BE-NEXT:    blr
-;
-; P8LE-LABEL: allOnei:
-; P8LE:       # %bb.0: # %entry
-; P8LE-NEXT:    vspltisb v2, -1
-; P8LE-NEXT:    blr
-entry:
-  ret <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>
-}
-
-; Function Attrs: norecurse nounwind readnone
 define <4 x i32> @spltConst1i() {
 ; P9BE-LABEL: spltConst1i:
 ; P9BE:       # %bb.0: # %entry
@@ -804,7 +757,6 @@ entry:
   ret <4 x i32> <i32 1, i32 1, i32 1, i32 1>
 }
 
-; Function Attrs: norecurse nounwind readnone
 define <4 x i32> @spltConst16ki() {
 ; P9BE-LABEL: spltConst16ki:
 ; P9BE:       # %bb.0: # %entry
@@ -833,7 +785,6 @@ entry:
   ret <4 x i32> <i32 32767, i32 32767, i32 32767, i32 32767>
 }
 
-; Function Attrs: norecurse nounwind readnone
 define <4 x i32> @spltConst32ki() {
 ; P9BE-LABEL: spltConst32ki:
 ; P9BE:       # %bb.0: # %entry
@@ -862,7 +813,6 @@ entry:
   ret <4 x i32> <i32 65535, i32 65535, i32 65535, i32 65535>
 }
 
-; Function Attrs: norecurse nounwind readnone
 define <4 x i32> @fromRegsi(i32 signext %a, i32 signext %b, i32 signext %c, i32 signext %d) {
 ; P9BE-LABEL: fromRegsi:
 ; P9BE:       # %bb.0: # %entry
@@ -903,40 +853,38 @@ entry:
   ret <4 x i32> %vecinit3
 }
 
-; Function Attrs: norecurse nounwind readnone
 define <4 x i32> @fromDiffConstsi() {
 ; P9BE-LABEL: fromDiffConstsi:
 ; P9BE:       # %bb.0: # %entry
-; P9BE-NEXT:    addis r3, r2, .LCPI6_0@toc@ha
-; P9BE-NEXT:    addi r3, r3, .LCPI6_0@toc@l
+; P9BE-NEXT:    addis r3, r2, .LCPI5_0@toc@ha
+; P9BE-NEXT:    addi r3, r3, .LCPI5_0@toc@l
 ; P9BE-NEXT:    lxvx v2, 0, r3
 ; P9BE-NEXT:    blr
 ;
 ; P9LE-LABEL: fromDiffConstsi:
 ; P9LE:       # %bb.0: # %entry
-; P9LE-NEXT:    addis r3, r2, .LCPI6_0@toc@ha
-; P9LE-NEXT:    addi r3, r3, .LCPI6_0@toc@l
+; P9LE-NEXT:    addis r3, r2, .LCPI5_0@toc@ha
+; P9LE-NEXT:    addi r3, r3, .LCPI5_0@toc@l
 ; P9LE-NEXT:    lxvx v2, 0, r3
 ; P9LE-NEXT:    blr
 ;
 ; P8BE-LABEL: fromDiffConstsi:
 ; P8BE:       # %bb.0: # %entry
-; P8BE-NEXT:    addis r3, r2, .LCPI6_0@toc@ha
-; P8BE-NEXT:    addi r3, r3, .LCPI6_0@toc@l
+; P8BE-NEXT:    addis r3, r2, .LCPI5_0@toc@ha
+; P8BE-NEXT:    addi r3, r3, .LCPI5_0@toc@l
 ; P8BE-NEXT:    lxvw4x v2, 0, r3
 ; P8BE-NEXT:    blr
 ;
 ; P8LE-LABEL: fromDiffConstsi:
 ; P8LE:       # %bb.0: # %entry
-; P8LE-NEXT:    addis r3, r2, .LCPI6_0@toc@ha
-; P8LE-NEXT:    addi r3, r3, .LCPI6_0@toc@l
+; P8LE-NEXT:    addis r3, r2, .LCPI5_0@toc@ha
+; P8LE-NEXT:    addi r3, r3, .LCPI5_0@toc@l
 ; P8LE-NEXT:    lvx v2, 0, r3
 ; P8LE-NEXT:    blr
 entry:
   ret <4 x i32> <i32 242, i32 -113, i32 889, i32 19>
 }
 
-; Function Attrs: norecurse nounwind readonly
 define <4 x i32> @fromDiffMemConsAi(i32* nocapture readonly %arr) {
 ; P9BE-LABEL: fromDiffMemConsAi:
 ; P9BE:       # %bb.0: # %entry
@@ -973,31 +921,26 @@ entry:
   ret <4 x i32> %vecinit6
 }
 
-; Function Attrs: norecurse nounwind readonly
 define <4 x i32> @fromDiffMemConsDi(i32* nocapture readonly %arr) {
 ; P9BE-LABEL: fromDiffMemConsDi:
 ; P9BE:       # %bb.0: # %entry
 ; P9BE-NEXT:    lxv v2, 0(r3)
-; P9BE-NEXT:    addis r3, r2, .LCPI8_0@toc@ha
-; P9BE-NEXT:    addi r3, r3, .LCPI8_0@toc@l
+; P9BE-NEXT:    addis r3, r2, .LCPI7_0@toc@ha
+; P9BE-NEXT:    addi r3, r3, .LCPI7_0@toc@l
 ; P9BE-NEXT:    lxvx v3, 0, r3
 ; P9BE-NEXT:    vperm v2, v2, v2, v3
 ; P9BE-NEXT:    blr
 ;
 ; P9LE-LABEL: fromDiffMemConsDi:
 ; P9LE:       # %bb.0: # %entry
-; P9LE-NEXT:    lxv v2, 0(r3)
-; P9LE-NEXT:    addis r3, r2, .LCPI8_0@toc@ha
-; P9LE-NEXT:    addi r3, r3, .LCPI8_0@toc@l
-; P9LE-NEXT:    lxvx v3, 0, r3
-; P9LE-NEXT:    vperm v2, v2, v2, v3
+; P9LE-NEXT:    lxvw4x v2, 0, r3
 ; P9LE-NEXT:    blr
 ;
 ; P8BE-LABEL: fromDiffMemConsDi:
 ; P8BE:       # %bb.0: # %entry
-; P8BE-NEXT:    addis r4, r2, .LCPI8_0@toc@ha
+; P8BE-NEXT:    addis r4, r2, .LCPI7_0@toc@ha
 ; P8BE-NEXT:    lxvw4x v2, 0, r3
-; P8BE-NEXT:    addi r4, r4, .LCPI8_0@toc@l
+; P8BE-NEXT:    addi r4, r4, .LCPI7_0@toc@l
 ; P8BE-NEXT:    lxvw4x v3, 0, r4
 ; P8BE-NEXT:    vperm v2, v2, v2, v3
 ; P8BE-NEXT:    blr
@@ -1005,8 +948,8 @@ define <4 x i32> @fromDiffMemConsDi(i32* nocapture readonly %arr) {
 ; P8LE-LABEL: fromDiffMemConsDi:
 ; P8LE:       # %bb.0: # %entry
 ; P8LE-NEXT:    lxvd2x vs0, 0, r3
-; P8LE-NEXT:    addis r4, r2, .LCPI8_0@toc@ha
-; P8LE-NEXT:    addi r3, r4, .LCPI8_0@toc@l
+; P8LE-NEXT:    addis r4, r2, .LCPI7_0@toc@ha
+; P8LE-NEXT:    addi r3, r4, .LCPI7_0@toc@l
 ; P8LE-NEXT:    lvx v2, 0, r3
 ; P8LE-NEXT:    xxswapd v3, vs0
 ; P8LE-NEXT:    vperm v2, v3, v3, v2
@@ -1026,7 +969,6 @@ entry:
   ret <4 x i32> %vecinit6
 }
 
-; Function Attrs: norecurse nounwind readonly
 define <4 x i32> @fromDiffMemVarAi(i32* nocapture readonly %arr, i32 signext %elem) {
 ; P9BE-LABEL: fromDiffMemVarAi:
 ; P9BE:       # %bb.0: # %entry
@@ -1075,7 +1017,6 @@ entry:
   ret <4 x i32> %vecinit11
 }
 
-; Function Attrs: norecurse nounwind readonly
 define <4 x i32> @fromDiffMemVarDi(i32* nocapture readonly %arr, i32 signext %elem) {
 ; P9BE-LABEL: fromDiffMemVarDi:
 ; P9BE:       # %bb.0: # %entry
@@ -1083,8 +1024,8 @@ define <4 x i32> @fromDiffMemVarDi(i32* nocapture readonly %arr, i32 signext %el
 ; P9BE-NEXT:    add r3, r3, r4
 ; P9BE-NEXT:    addi r3, r3, -12
 ; P9BE-NEXT:    lxvx v2, 0, r3
-; P9BE-NEXT:    addis r3, r2, .LCPI10_0@toc@ha
-; P9BE-NEXT:    addi r3, r3, .LCPI10_0@toc@l
+; P9BE-NEXT:    addis r3, r2, .LCPI9_0@toc@ha
+; P9BE-NEXT:    addi r3, r3, .LCPI9_0@toc@l
 ; P9BE-NEXT:    lxvx v3, 0, r3
 ; P9BE-NEXT:    vperm v2, v2, v2, v3
 ; P9BE-NEXT:    blr
@@ -1095,8 +1036,8 @@ define <4 x i32> @fromDiffMemVarDi(i32* nocapture readonly %arr, i32 signext %el
 ; P9LE-NEXT:    add r3, r3, r4
 ; P9LE-NEXT:    addi r3, r3, -12
 ; P9LE-NEXT:    lxvx v2, 0, r3
-; P9LE-NEXT:    addis r3, r2, .LCPI10_0@toc@ha
-; P9LE-NEXT:    addi r3, r3, .LCPI10_0@toc@l
+; P9LE-NEXT:    addis r3, r2, .LCPI9_0@toc@ha
+; P9LE-NEXT:    addi r3, r3, .LCPI9_0@toc@l
 ; P9LE-NEXT:    lxvx v3, 0, r3
 ; P9LE-NEXT:    vperm v2, v2, v2, v3
 ; P9LE-NEXT:    blr
@@ -1104,9 +1045,9 @@ define <4 x i32> @fromDiffMemVarDi(i32* nocapture readonly %arr, i32 signext %el
 ; P8BE-LABEL: fromDiffMemVarDi:
 ; P8BE:       # %bb.0: # %entry
 ; P8BE-NEXT:    sldi r4, r4, 2
-; P8BE-NEXT:    addis r5, r2, .LCPI10_0@toc@ha
+; P8BE-NEXT:    addis r5, r2, .LCPI9_0@toc@ha
 ; P8BE-NEXT:    add r3, r3, r4
-; P8BE-NEXT:    addi r4, r5, .LCPI10_0@toc@l
+; P8BE-NEXT:    addi r4, r5, .LCPI9_0@toc@l
 ; P8BE-NEXT:    addi r3, r3, -12
 ; P8BE-NEXT:    lxvw4x v3, 0, r4
 ; P8BE-NEXT:    lxvw4x v2, 0, r3
@@ -1116,11 +1057,11 @@ define <4 x i32> @fromDiffMemVarDi(i32* nocapture readonly %arr, i32 signext %el
 ; P8LE-LABEL: fromDiffMemVarDi:
 ; P8LE:       # %bb.0: # %entry
 ; P8LE-NEXT:    sldi r4, r4, 2
-; P8LE-NEXT:    addis r5, r2, .LCPI10_0@toc@ha
+; P8LE-NEXT:    addis r5, r2, .LCPI9_0@toc@ha
 ; P8LE-NEXT:    add r3, r3, r4
 ; P8LE-NEXT:    addi r3, r3, -12
 ; P8LE-NEXT:    lxvd2x vs0, 0, r3
-; P8LE-NEXT:    addi r3, r5, .LCPI10_0@toc@l
+; P8LE-NEXT:    addi r3, r5, .LCPI9_0@toc@l
 ; P8LE-NEXT:    lvx v3, 0, r3
 ; P8LE-NEXT:    xxswapd v2, vs0
 ; P8LE-NEXT:    vperm v2, v2, v2, v3
@@ -1148,7 +1089,6 @@ entry:
   ret <4 x i32> %vecinit11
 }
 
-; Function Attrs: norecurse nounwind readonly
 define <4 x i32> @fromRandMemConsi(i32* nocapture readonly %arr) {
 ; P9BE-LABEL: fromRandMemConsi:
 ; P9BE:       # %bb.0: # %entry
@@ -1213,7 +1153,6 @@ entry:
   ret <4 x i32> %vecinit6
 }
 
-; Function Attrs: norecurse nounwind readonly
 define <4 x i32> @fromRandMemVari(i32* nocapture readonly %arr, i32 signext %elem) {
 ; P9BE-LABEL: fromRandMemVari:
 ; P9BE:       # %bb.0: # %entry
@@ -1294,7 +1233,6 @@ entry:
   ret <4 x i32> %vecinit12
 }
 
-; Function Attrs: norecurse nounwind readnone
 define <4 x i32> @spltRegVali(i32 signext %val) {
 ; P9BE-LABEL: spltRegVali:
 ; P9BE:       # %bb.0: # %entry
@@ -1323,20 +1261,15 @@ entry:
   ret <4 x i32> %splat.splat
 }
 
-; Function Attrs: norecurse nounwind readonly
 define <4 x i32> @spltMemVali(i32* nocapture readonly %ptr) {
 ; P9BE-LABEL: spltMemVali:
 ; P9BE:       # %bb.0: # %entry
-; P9BE-NEXT:    lfiwzx f0, 0, r3
-; P9BE-NEXT:    xxsldwi vs0, f0, f0, 1
-; P9BE-NEXT:    xxspltw v2, vs0, 0
+; P9BE-NEXT:    lxvwsx v2, 0, r3
 ; P9BE-NEXT:    blr
 ;
 ; P9LE-LABEL: spltMemVali:
 ; P9LE:       # %bb.0: # %entry
-; P9LE-NEXT:    lfiwzx f0, 0, r3
-; P9LE-NEXT:    xxpermdi vs0, f0, f0, 2
-; P9LE-NEXT:    xxspltw v2, vs0, 3
+; P9LE-NEXT:    lxvwsx v2, 0, r3
 ; P9LE-NEXT:    blr
 ;
 ; P8BE-LABEL: spltMemVali:
@@ -1359,7 +1292,6 @@ entry:
   ret <4 x i32> %splat.splat
 }
 
-; Function Attrs: norecurse nounwind readnone
 define <4 x i32> @spltCnstConvftoi() {
 ; P9BE-LABEL: spltCnstConvftoi:
 ; P9BE:       # %bb.0: # %entry
@@ -1384,7 +1316,6 @@ entry:
   ret <4 x i32> <i32 4, i32 4, i32 4, i32 4>
 }
 
-; Function Attrs: norecurse nounwind readnone
 define <4 x i32> @fromRegsConvftoi(float %a, float %b, float %c, float %d) {
 ; P9BE-LABEL: fromRegsConvftoi:
 ; P9BE:       # %bb.0: # %entry
@@ -1449,40 +1380,38 @@ entry:
   ret <4 x i32> %vecinit6
 }
 
-; Function Attrs: norecurse nounwind readnone
 define <4 x i32> @fromDiffConstsConvftoi() {
 ; P9BE-LABEL: fromDiffConstsConvftoi:
 ; P9BE:       # %bb.0: # %entry
-; P9BE-NEXT:    addis r3, r2, .LCPI17_0@toc@ha
-; P9BE-NEXT:    addi r3, r3, .LCPI17_0@toc@l
+; P9BE-NEXT:    addis r3, r2, .LCPI16_0@toc@ha
+; P9BE-NEXT:    addi r3, r3, .LCPI16_0@toc@l
 ; P9BE-NEXT:    lxvx v2, 0, r3
 ; P9BE-NEXT:    blr
 ;
 ; P9LE-LABEL: fromDiffConstsConvftoi:
 ; P9LE:       # %bb.0: # %entry
-; P9LE-NEXT:    addis r3, r2, .LCPI17_0@toc@ha
-; P9LE-NEXT:    addi r3, r3, .LCPI17_0@toc@l
+; P9LE-NEXT:    addis r3, r2, .LCPI16_0@toc@ha
+; P9LE-NEXT:    addi r3, r3, .LCPI16_0@toc@l
 ; P9LE-NEXT:    lxvx v2, 0, r3
 ; P9LE-NEXT:    blr
 ;
 ; P8BE-LABEL: fromDiffConstsConvftoi:
 ; P8BE:       # %bb.0: # %entry
-; P8BE-NEXT:    addis r3, r2, .LCPI17_0@toc@ha
-; P8BE-NEXT:    addi r3, r3, .LCPI17_0@toc@l
+; P8BE-NEXT:    addis r3, r2, .LCPI16_0@toc@ha
+; P8BE-NEXT:    addi r3, r3, .LCPI16_0@toc@l
 ; P8BE-NEXT:    lxvw4x v2, 0, r3
 ; P8BE-NEXT:    blr
 ;
 ; P8LE-LABEL: fromDiffConstsConvftoi:
 ; P8LE:       # %bb.0: # %entry
-; P8LE-NEXT:    addis r3, r2, .LCPI17_0@toc@ha
-; P8LE-NEXT:    addi r3, r3, .LCPI17_0@toc@l
+; P8LE-NEXT:    addis r3, r2, .LCPI16_0@toc@ha
+; P8LE-NEXT:    addi r3, r3, .LCPI16_0@toc@l
 ; P8LE-NEXT:    lvx v2, 0, r3
 ; P8LE-NEXT:    blr
 entry:
   ret <4 x i32> <i32 24, i32 234, i32 988, i32 422>
 }
 
-; Function Attrs: norecurse nounwind readonly
 define <4 x i32> @fromDiffMemConsAConvftoi(float* nocapture readonly %ptr) {
 ; P9BE-LABEL: fromDiffMemConsAConvftoi:
 ; P9BE:       # %bb.0: # %entry
@@ -1515,13 +1444,12 @@ entry:
   ret <4 x i32> %2
 }
 
-; Function Attrs: norecurse nounwind readonly
 define <4 x i32> @fromDiffMemConsDConvftoi(float* nocapture readonly %ptr) {
 ; P9BE-LABEL: fromDiffMemConsDConvftoi:
 ; P9BE:       # %bb.0: # %entry
 ; P9BE-NEXT:    lxv v2, 0(r3)
-; P9BE-NEXT:    addis r3, r2, .LCPI19_0@toc@ha
-; P9BE-NEXT:    addi r3, r3, .LCPI19_0@toc@l
+; P9BE-NEXT:    addis r3, r2, .LCPI18_0@toc@ha
+; P9BE-NEXT:    addi r3, r3, .LCPI18_0@toc@l
 ; P9BE-NEXT:    lxvx v3, 0, r3
 ; P9BE-NEXT:    vperm v2, v2, v2, v3
 ; P9BE-NEXT:    xvcvspsxws v2, v2
@@ -1530,8 +1458,8 @@ define <4 x i32> @fromDiffMemConsDConvftoi(float* nocapture readonly %ptr) {
 ; P9LE-LABEL: fromDiffMemConsDConvftoi:
 ; P9LE:       # %bb.0: # %entry
 ; P9LE-NEXT:    lxv v2, 0(r3)
-; P9LE-NEXT:    addis r3, r2, .LCPI19_0@toc@ha
-; P9LE-NEXT:    addi r3, r3, .LCPI19_0@toc@l
+; P9LE-NEXT:    addis r3, r2, .LCPI18_0@toc@ha
+; P9LE-NEXT:    addi r3, r3, .LCPI18_0@toc@l
 ; P9LE-NEXT:    lxvx v3, 0, r3
 ; P9LE-NEXT:    vperm v2, v2, v2, v3
 ; P9LE-NEXT:    xvcvspsxws v2, v2
@@ -1539,9 +1467,9 @@ define <4 x i32> @fromDiffMemConsDConvftoi(float* nocapture readonly %ptr) {
 ;
 ; P8BE-LABEL: fromDiffMemConsDConvftoi:
 ; P8BE:       # %bb.0: # %entry
-; P8BE-NEXT:    addis r4, r2, .LCPI19_0@toc@ha
+; P8BE-NEXT:    addis r4, r2, .LCPI18_0@toc@ha
 ; P8BE-NEXT:    lxvw4x v2, 0, r3
-; P8BE-NEXT:    addi r4, r4, .LCPI19_0@toc@l
+; P8BE-NEXT:    addi r4, r4, .LCPI18_0@toc@l
 ; P8BE-NEXT:    lxvw4x v3, 0, r4
 ; P8BE-NEXT:    vperm v2, v2, v2, v3
 ; P8BE-NEXT:    xvcvspsxws v2, v2
@@ -1550,8 +1478,8 @@ define <4 x i32> @fromDiffMemConsDConvftoi(float* nocapture readonly %ptr) {
 ; P8LE-LABEL: fromDiffMemConsDConvftoi:
 ; P8LE:       # %bb.0: # %entry
 ; P8LE-NEXT:    lxvd2x vs0, 0, r3
-; P8LE-NEXT:    addis r4, r2, .LCPI19_0@toc@ha
-; P8LE-NEXT:    addi r3, r4, .LCPI19_0@toc@l
+; P8LE-NEXT:    addis r4, r2, .LCPI18_0@toc@ha
+; P8LE-NEXT:    addi r3, r4, .LCPI18_0@toc@l
 ; P8LE-NEXT:    lvx v2, 0, r3
 ; P8LE-NEXT:    xxswapd v3, vs0
 ; P8LE-NEXT:    vperm v2, v3, v3, v2
@@ -1576,7 +1504,6 @@ entry:
   ret <4 x i32> %vecinit9
 }
 
-; Function Attrs: norecurse nounwind readonly
 define <4 x i32> @fromDiffMemVarAConvftoi(float* nocapture readonly %arr, i32 signext %elem) {
 ; P9BE-LABEL: fromDiffMemVarAConvftoi:
 ; P9BE:       # %bb.0: # %entry
@@ -1662,10 +1589,8 @@ entry:
   %conv13 = fptosi float %3 to i32
   %vecinit14 = insertelement <4 x i32> %vecinit9, i32 %conv13, i32 3
   ret <4 x i32> %vecinit14
-; FIXME: implement finding consecutive loads with pre-inc
 }
 
-; Function Attrs: norecurse nounwind readonly
 define <4 x i32> @fromDiffMemVarDConvftoi(float* nocapture readonly %arr, i32 signext %elem) {
 ; P9BE-LABEL: fromDiffMemVarDConvftoi:
 ; P9BE:       # %bb.0: # %entry
@@ -1754,7 +1679,6 @@ entry:
 ; FIXME: implement finding consecutive loads with pre-inc
 }
 
-; Function Attrs: norecurse nounwind readnone
 define <4 x i32> @spltRegValConvftoi(float %val) {
 ; P9BE-LABEL: spltRegValConvftoi:
 ; P9BE:       # %bb.0: # %entry
@@ -1786,7 +1710,6 @@ entry:
   ret <4 x i32> %splat.splat
 }
 
-; Function Attrs: norecurse nounwind readonly
 define <4 x i32> @spltMemValConvftoi(float* nocapture readonly %ptr) {
 ; P9BE-LABEL: spltMemValConvftoi:
 ; P9BE:       # %bb.0: # %entry
@@ -1821,7 +1744,6 @@ entry:
   ret <4 x i32> %splat.splat
 }
 
-; Function Attrs: norecurse nounwind readnone
 define <4 x i32> @spltCnstConvdtoi() {
 ; P9BE-LABEL: spltCnstConvdtoi:
 ; P9BE:       # %bb.0: # %entry
@@ -1846,7 +1768,6 @@ entry:
   ret <4 x i32> <i32 4, i32 4, i32 4, i32 4>
 }
 
-; Function Attrs: norecurse nounwind readnone
 define <4 x i32> @fromRegsConvdtoi(double %a, double %b, double %c, double %d) {
 ; P9BE-LABEL: fromRegsConvdtoi:
 ; P9BE:       # %bb.0: # %entry
@@ -1911,40 +1832,38 @@ entry:
   ret <4 x i32> %vecinit6
 }
 
-; Function Attrs: norecurse nounwind readnone
 define <4 x i32> @fromDiffConstsConvdtoi() {
 ; P9BE-LABEL: fromDiffConstsConvdtoi:
 ; P9BE:       # %bb.0: # %entry
-; P9BE-NEXT:    addis r3, r2, .LCPI26_0@toc@ha
-; P9BE-NEXT:    addi r3, r3, .LCPI26_0@toc@l
+; P9BE-NEXT:    addis r3, r2, .LCPI25_0@toc@ha
+; P9BE-NEXT:    addi r3, r3, .LCPI25_0@toc@l
 ; P9BE-NEXT:    lxvx v2, 0, r3
 ; P9BE-NEXT:    blr
 ;
 ; P9LE-LABEL: fromDiffConstsConvdtoi:
 ; P9LE:       # %bb.0: # %entry
-; P9LE-NEXT:    addis r3, r2, .LCPI26_0@toc@ha
-; P9LE-NEXT:    addi r3, r3, .LCPI26_0@toc@l
+; P9LE-NEXT:    addis r3, r2, .LCPI25_0@toc@ha
+; P9LE-NEXT:    addi r3, r3, .LCPI25_0@toc@l
 ; P9LE-NEXT:    lxvx v2, 0, r3
 ; P9LE-NEXT:    blr
 ;
 ; P8BE-LABEL: fromDiffConstsConvdtoi:
 ; P8BE:       # %bb.0: # %entry
-; P8BE-NEXT:    addis r3, r2, .LCPI26_0@toc@ha
-; P8BE-NEXT:    addi r3, r3, .LCPI26_0@toc@l
+; P8BE-NEXT:    addis r3, r2, .LCPI25_0@toc@ha
+; P8BE-NEXT:    addi r3, r3, .LCPI25_0@toc@l
 ; P8BE-NEXT:    lxvw4x v2, 0, r3
 ; P8BE-NEXT:    blr
 ;
 ; P8LE-LABEL: fromDiffConstsConvdtoi:
 ; P8LE:       # %bb.0: # %entry
-; P8LE-NEXT:    addis r3, r2, .LCPI26_0@toc@ha
-; P8LE-NEXT:    addi r3, r3, .LCPI26_0@toc@l
+; P8LE-NEXT:    addis r3, r2, .LCPI25_0@toc@ha
+; P8LE-NEXT:    addi r3, r3, .LCPI25_0@toc@l
 ; P8LE-NEXT:    lvx v2, 0, r3
 ; P8LE-NEXT:    blr
 entry:
   ret <4 x i32> <i32 24, i32 234, i32 988, i32 422>
 }
 
-; Function Attrs: norecurse nounwind readonly
 define <4 x i32> @fromDiffMemConsAConvdtoi(double* nocapture readonly %ptr) {
 ; P9BE-LABEL: fromDiffMemConsAConvdtoi:
 ; P9BE:       # %bb.0: # %entry
@@ -2005,7 +1924,6 @@ entry:
   ret <4 x i32> %vecinit9
 }
 
-; Function Attrs: norecurse nounwind readonly
 define <4 x i32> @fromDiffMemConsDConvdtoi(double* nocapture readonly %ptr) {
 ; P9BE-LABEL: fromDiffMemConsDConvdtoi:
 ; P9BE:       # %bb.0: # %entry
@@ -2077,7 +1995,6 @@ entry:
   ret <4 x i32> %vecinit9
 }
 
-; Function Attrs: norecurse nounwind readonly
 define <4 x i32> @fromDiffMemVarAConvdtoi(double* nocapture readonly %arr, i32 signext %elem) {
 ; P9BE-LABEL: fromDiffMemVarAConvdtoi:
 ; P9BE:       # %bb.0: # %entry
@@ -2161,7 +2078,6 @@ entry:
   ret <4 x i32> %vecinit14
 }
 
-; Function Attrs: norecurse nounwind readonly
 define <4 x i32> @fromDiffMemVarDConvdtoi(double* nocapture readonly %arr, i32 signext %elem) {
 ; P9BE-LABEL: fromDiffMemVarDConvdtoi:
 ; P9BE:       # %bb.0: # %entry
@@ -2245,7 +2161,6 @@ entry:
   ret <4 x i32> %vecinit14
 }
 
-; Function Attrs: norecurse nounwind readnone
 define <4 x i32> @spltRegValConvdtoi(double %val) {
 ; P9BE-LABEL: spltRegValConvdtoi:
 ; P9BE:       # %bb.0: # %entry
@@ -2277,7 +2192,6 @@ entry:
   ret <4 x i32> %splat.splat
 }
 
-; Function Attrs: norecurse nounwind readonly
 define <4 x i32> @spltMemValConvdtoi(double* nocapture readonly %ptr) {
 ; P9BE-LABEL: spltMemValConvdtoi:
 ; P9BE:       # %bb.0: # %entry
@@ -2313,7 +2227,7 @@ entry:
   %splat.splat = shufflevector <4 x i32> %splat.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
   ret <4 x i32> %splat.splat
 }
-; Function Attrs: norecurse nounwind readnone
+
 define <4 x i32> @allZeroui() {
 ; P9BE-LABEL: allZeroui:
 ; P9BE:       # %bb.0: # %entry
@@ -2338,32 +2252,6 @@ entry:
   ret <4 x i32> zeroinitializer
 }
 
-; Function Attrs: norecurse nounwind readnone
-define <4 x i32> @allOneui() {
-; P9BE-LABEL: allOneui:
-; P9BE:       # %bb.0: # %entry
-; P9BE-NEXT:    xxspltib v2, 255
-; P9BE-NEXT:    blr
-;
-; P9LE-LABEL: allOneui:
-; P9LE:       # %bb.0: # %entry
-; P9LE-NEXT:    xxspltib v2, 255
-; P9LE-NEXT:    blr
-;
-; P8BE-LABEL: allOneui:
-; P8BE:       # %bb.0: # %entry
-; P8BE-NEXT:    vspltisb v2, -1
-; P8BE-NEXT:    blr
-;
-; P8LE-LABEL: allOneui:
-; P8LE:       # %bb.0: # %entry
-; P8LE-NEXT:    vspltisb v2, -1
-; P8LE-NEXT:    blr
-entry:
-  ret <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>
-}
-
-; Function Attrs: norecurse nounwind readnone
 define <4 x i32> @spltConst1ui() {
 ; P9BE-LABEL: spltConst1ui:
 ; P9BE:       # %bb.0: # %entry
@@ -2388,7 +2276,6 @@ entry:
   ret <4 x i32> <i32 1, i32 1, i32 1, i32 1>
 }
 
-; Function Attrs: norecurse nounwind readnone
 define <4 x i32> @spltConst16kui() {
 ; P9BE-LABEL: spltConst16kui:
 ; P9BE:       # %bb.0: # %entry
@@ -2417,7 +2304,6 @@ entry:
   ret <4 x i32> <i32 32767, i32 32767, i32 32767, i32 32767>
 }
 
-; Function Attrs: norecurse nounwind readnone
 define <4 x i32> @spltConst32kui() {
 ; P9BE-LABEL: spltConst32kui:
 ; P9BE:       # %bb.0: # %entry
@@ -2446,7 +2332,6 @@ entry:
   ret <4 x i32> <i32 65535, i32 65535, i32 65535, i32 65535>
 }
 
-; Function Attrs: norecurse nounwind readnone
 define <4 x i32> @fromRegsui(i32 zeroext %a, i32 zeroext %b, i32 zeroext %c, i32 zeroext %d) {
 ; P9BE-LABEL: fromRegsui:
 ; P9BE:       # %bb.0: # %entry
@@ -2487,40 +2372,38 @@ entry:
   ret <4 x i32> %vecinit3
 }
 
-; Function Attrs: norecurse nounwind readnone
 define <4 x i32> @fromDiffConstsui() {
 ; P9BE-LABEL: fromDiffConstsui:
 ; P9BE:       # %bb.0: # %entry
-; P9BE-NEXT:    addis r3, r2, .LCPI39_0@toc@ha
-; P9BE-NEXT:    addi r3, r3, .LCPI39_0@toc@l
+; P9BE-NEXT:    addis r3, r2, .LCPI37_0@toc@ha
+; P9BE-NEXT:    addi r3, r3, .LCPI37_0@toc@l
 ; P9BE-NEXT:    lxvx v2, 0, r3
 ; P9BE-NEXT:    blr
 ;
 ; P9LE-LABEL: fromDiffConstsui:
 ; P9LE:       # %bb.0: # %entry
-; P9LE-NEXT:    addis r3, r2, .LCPI39_0@toc@ha
-; P9LE-NEXT:    addi r3, r3, .LCPI39_0@toc@l
+; P9LE-NEXT:    addis r3, r2, .LCPI37_0@toc@ha
+; P9LE-NEXT:    addi r3, r3, .LCPI37_0@toc@l
 ; P9LE-NEXT:    lxvx v2, 0, r3
 ; P9LE-NEXT:    blr
 ;
 ; P8BE-LABEL: fromDiffConstsui:
 ; P8BE:       # %bb.0: # %entry
-; P8BE-NEXT:    addis r3, r2, .LCPI39_0@toc@ha
-; P8BE-NEXT:    addi r3, r3, .LCPI39_0@toc@l
+; P8BE-NEXT:    addis r3, r2, .LCPI37_0@toc@ha
+; P8BE-NEXT:    addi r3, r3, .LCPI37_0@toc@l
 ; P8BE-NEXT:    lxvw4x v2, 0, r3
 ; P8BE-NEXT:    blr
 ;
 ; P8LE-LABEL: fromDiffConstsui:
 ; P8LE:       # %bb.0: # %entry
-; P8LE-NEXT:    addis r3, r2, .LCPI39_0@toc@ha
-; P8LE-NEXT:    addi r3, r3, .LCPI39_0@toc@l
+; P8LE-NEXT:    addis r3, r2, .LCPI37_0@toc@ha
+; P8LE-NEXT:    addi r3, r3, .LCPI37_0@toc@l
 ; P8LE-NEXT:    lvx v2, 0, r3
 ; P8LE-NEXT:    blr
 entry:
   ret <4 x i32> <i32 242, i32 -113, i32 889, i32 19>
 }
 
-; Function Attrs: norecurse nounwind readonly
 define <4 x i32> @fromDiffMemConsAui(i32* nocapture readonly %arr) {
 ; P9BE-LABEL: fromDiffMemConsAui:
 ; P9BE:       # %bb.0: # %entry
@@ -2557,31 +2440,26 @@ entry:
   ret <4 x i32> %vecinit6
 }
 
-; Function Attrs: norecurse nounwind readonly
 define <4 x i32> @fromDiffMemConsDui(i32* nocapture readonly %arr) {
 ; P9BE-LABEL: fromDiffMemConsDui:
 ; P9BE:       # %bb.0: # %entry
 ; P9BE-NEXT:    lxv v2, 0(r3)
-; P9BE-NEXT:    addis r3, r2, .LCPI41_0@toc@ha
-; P9BE-NEXT:    addi r3, r3, .LCPI41_0@toc@l
+; P9BE-NEXT:    addis r3, r2, .LCPI39_0@toc@ha
+; P9BE-NEXT:    addi r3, r3, .LCPI39_0@toc@l
 ; P9BE-NEXT:    lxvx v3, 0, r3
 ; P9BE-NEXT:    vperm v2, v2, v2, v3
 ; P9BE-NEXT:    blr
 ;
 ; P9LE-LABEL: fromDiffMemConsDui:
 ; P9LE:       # %bb.0: # %entry
-; P9LE-NEXT:    lxv v2, 0(r3)
-; P9LE-NEXT:    addis r3, r2, .LCPI41_0@toc@ha
-; P9LE-NEXT:    addi r3, r3, .LCPI41_0@toc@l
-; P9LE-NEXT:    lxvx v3, 0, r3
-; P9LE-NEXT:    vperm v2, v2, v2, v3
+; P9LE-NEXT:    lxvw4x v2, 0, r3
 ; P9LE-NEXT:    blr
 ;
 ; P8BE-LABEL: fromDiffMemConsDui:
 ; P8BE:       # %bb.0: # %entry
-; P8BE-NEXT:    addis r4, r2, .LCPI41_0@toc@ha
+; P8BE-NEXT:    addis r4, r2, .LCPI39_0@toc@ha
 ; P8BE-NEXT:    lxvw4x v2, 0, r3
-; P8BE-NEXT:    addi r4, r4, .LCPI41_0@toc@l
+; P8BE-NEXT:    addi r4, r4, .LCPI39_0@toc@l
 ; P8BE-NEXT:    lxvw4x v3, 0, r4
 ; P8BE-NEXT:    vperm v2, v2, v2, v3
 ; P8BE-NEXT:    blr
@@ -2589,8 +2467,8 @@ define <4 x i32> @fromDiffMemConsDui(i32* nocapture readonly %arr) {
 ; P8LE-LABEL: fromDiffMemConsDui:
 ; P8LE:       # %bb.0: # %entry
 ; P8LE-NEXT:    lxvd2x vs0, 0, r3
-; P8LE-NEXT:    addis r4, r2, .LCPI41_0@toc@ha
-; P8LE-NEXT:    addi r3, r4, .LCPI41_0@toc@l
+; P8LE-NEXT:    addis r4, r2, .LCPI39_0@toc@ha
+; P8LE-NEXT:    addi r3, r4, .LCPI39_0@toc@l
 ; P8LE-NEXT:    lvx v2, 0, r3
 ; P8LE-NEXT:    xxswapd v3, vs0
 ; P8LE-NEXT:    vperm v2, v3, v3, v2
@@ -2610,7 +2488,6 @@ entry:
   ret <4 x i32> %vecinit6
 }
 
-; Function Attrs: norecurse nounwind readonly
 define <4 x i32> @fromDiffMemVarAui(i32* nocapture readonly %arr, i32 signext %elem) {
 ; P9BE-LABEL: fromDiffMemVarAui:
 ; P9BE:       # %bb.0: # %entry
@@ -2659,7 +2536,6 @@ entry:
   ret <4 x i32> %vecinit11
 }
 
-; Function Attrs: norecurse nounwind readonly
 define <4 x i32> @fromDiffMemVarDui(i32* nocapture readonly %arr, i32 signext %elem) {
 ; P9BE-LABEL: fromDiffMemVarDui:
 ; P9BE:       # %bb.0: # %entry
@@ -2667,8 +2543,8 @@ define <4 x i32> @fromDiffMemVarDui(i32* nocapture readonly %arr, i32 signext %e
 ; P9BE-NEXT:    add r3, r3, r4
 ; P9BE-NEXT:    addi r3, r3, -12
 ; P9BE-NEXT:    lxvx v2, 0, r3
-; P9BE-NEXT:    addis r3, r2, .LCPI43_0@toc@ha
-; P9BE-NEXT:    addi r3, r3, .LCPI43_0@toc@l
+; P9BE-NEXT:    addis r3, r2, .LCPI41_0@toc@ha
+; P9BE-NEXT:    addi r3, r3, .LCPI41_0@toc@l
 ; P9BE-NEXT:    lxvx v3, 0, r3
 ; P9BE-NEXT:    vperm v2, v2, v2, v3
 ; P9BE-NEXT:    blr
@@ -2679,8 +2555,8 @@ define <4 x i32> @fromDiffMemVarDui(i32* nocapture readonly %arr, i32 signext %e
 ; P9LE-NEXT:    add r3, r3, r4
 ; P9LE-NEXT:    addi r3, r3, -12
 ; P9LE-NEXT:    lxvx v2, 0, r3
-; P9LE-NEXT:    addis r3, r2, .LCPI43_0@toc@ha
-; P9LE-NEXT:    addi r3, r3, .LCPI43_0@toc@l
+; P9LE-NEXT:    addis r3, r2, .LCPI41_0@toc@ha
+; P9LE-NEXT:    addi r3, r3, .LCPI41_0@toc@l
 ; P9LE-NEXT:    lxvx v3, 0, r3
 ; P9LE-NEXT:    vperm v2, v2, v2, v3
 ; P9LE-NEXT:    blr
@@ -2688,9 +2564,9 @@ define <4 x i32> @fromDiffMemVarDui(i32* nocapture readonly %arr, i32 signext %e
 ; P8BE-LABEL: fromDiffMemVarDui:
 ; P8BE:       # %bb.0: # %entry
 ; P8BE-NEXT:    sldi r4, r4, 2
-; P8BE-NEXT:    addis r5, r2, .LCPI43_0@toc@ha
+; P8BE-NEXT:    addis r5, r2, .LCPI41_0@toc@ha
 ; P8BE-NEXT:    add r3, r3, r4
-; P8BE-NEXT:    addi r4, r5, .LCPI43_0@toc@l
+; P8BE-NEXT:    addi r4, r5, .LCPI41_0@toc@l
 ; P8BE-NEXT:    addi r3, r3, -12
 ; P8BE-NEXT:    lxvw4x v3, 0, r4
 ; P8BE-NEXT:    lxvw4x v2, 0, r3
@@ -2700,11 +2576,11 @@ define <4 x i32> @fromDiffMemVarDui(i32* nocapture readonly %arr, i32 signext %e
 ; P8LE-LABEL: fromDiffMemVarDui:
 ; P8LE:       # %bb.0: # %entry
 ; P8LE-NEXT:    sldi r4, r4, 2
-; P8LE-NEXT:    addis r5, r2, .LCPI43_0@toc@ha
+; P8LE-NEXT:    addis r5, r2, .LCPI41_0@toc@ha
 ; P8LE-NEXT:    add r3, r3, r4
 ; P8LE-NEXT:    addi r3, r3, -12
 ; P8LE-NEXT:    lxvd2x vs0, 0, r3
-; P8LE-NEXT:    addi r3, r5, .LCPI43_0@toc@l
+; P8LE-NEXT:    addi r3, r5, .LCPI41_0@toc@l
 ; P8LE-NEXT:    lvx v3, 0, r3
 ; P8LE-NEXT:    xxswapd v2, vs0
 ; P8LE-NEXT:    vperm v2, v2, v2, v3
@@ -2732,7 +2608,6 @@ entry:
   ret <4 x i32> %vecinit11
 }
 
-; Function Attrs: norecurse nounwind readonly
 define <4 x i32> @fromRandMemConsui(i32* nocapture readonly %arr) {
 ; P9BE-LABEL: fromRandMemConsui:
 ; P9BE:       # %bb.0: # %entry
@@ -2797,7 +2672,6 @@ entry:
   ret <4 x i32> %vecinit6
 }
 
-; Function Attrs: norecurse nounwind readonly
 define <4 x i32> @fromRandMemVarui(i32* nocapture readonly %arr, i32 signext %elem) {
 ; P9BE-LABEL: fromRandMemVarui:
 ; P9BE:       # %bb.0: # %entry
@@ -2878,7 +2752,6 @@ entry:
   ret <4 x i32> %vecinit12
 }
 
-; Function Attrs: norecurse nounwind readnone
 define <4 x i32> @spltRegValui(i32 zeroext %val) {
 ; P9BE-LABEL: spltRegValui:
 ; P9BE:       # %bb.0: # %entry
@@ -2907,20 +2780,15 @@ entry:
   ret <4 x i32> %splat.splat
 }
 
-; Function Attrs: norecurse nounwind readonly
 define <4 x i32> @spltMemValui(i32* nocapture readonly %ptr) {
 ; P9BE-LABEL: spltMemValui:
 ; P9BE:       # %bb.0: # %entry
-; P9BE-NEXT:    lfiwzx f0, 0, r3
-; P9BE-NEXT:    xxsldwi vs0, f0, f0, 1
-; P9BE-NEXT:    xxspltw v2, vs0, 0
+; P9BE-NEXT:    lxvwsx v2, 0, r3
 ; P9BE-NEXT:    blr
 ;
 ; P9LE-LABEL: spltMemValui:
 ; P9LE:       # %bb.0: # %entry
-; P9LE-NEXT:    lfiwzx f0, 0, r3
-; P9LE-NEXT:    xxpermdi vs0, f0, f0, 2
-; P9LE-NEXT:    xxspltw v2, vs0, 3
+; P9LE-NEXT:    lxvwsx v2, 0, r3
 ; P9LE-NEXT:    blr
 ;
 ; P8BE-LABEL: spltMemValui:
@@ -2943,7 +2811,6 @@ entry:
   ret <4 x i32> %splat.splat
 }
 
-; Function Attrs: norecurse nounwind readnone
 define <4 x i32> @spltCnstConvftoui() {
 ; P9BE-LABEL: spltCnstConvftoui:
 ; P9BE:       # %bb.0: # %entry
@@ -2968,7 +2835,6 @@ entry:
   ret <4 x i32> <i32 4, i32 4, i32 4, i32 4>
 }
 
-; Function Attrs: norecurse nounwind readnone
 define <4 x i32> @fromRegsConvftoui(float %a, float %b, float %c, float %d) {
 ; P9BE-LABEL: fromRegsConvftoui:
 ; P9BE:       # %bb.0: # %entry
@@ -3033,40 +2899,38 @@ entry:
   ret <4 x i32> %vecinit6
 }
 
-; Function Attrs: norecurse nounwind readnone
 define <4 x i32> @fromDiffConstsConvftoui() {
 ; P9BE-LABEL: fromDiffConstsConvftoui:
 ; P9BE:       # %bb.0: # %entry
-; P9BE-NEXT:    addis r3, r2, .LCPI50_0@toc@ha
-; P9BE-NEXT:    addi r3, r3, .LCPI50_0@toc@l
+; P9BE-NEXT:    addis r3, r2, .LCPI48_0@toc@ha
+; P9BE-NEXT:    addi r3, r3, .LCPI48_0@toc@l
 ; P9BE-NEXT:    lxvx v2, 0, r3
 ; P9BE-NEXT:    blr
 ;
 ; P9LE-LABEL: fromDiffConstsConvftoui:
 ; P9LE:       # %bb.0: # %entry
-; P9LE-NEXT:    addis r3, r2, .LCPI50_0@toc@ha
-; P9LE-NEXT:    addi r3, r3, .LCPI50_0@toc@l
+; P9LE-NEXT:    addis r3, r2, .LCPI48_0@toc@ha
+; P9LE-NEXT:    addi r3, r3, .LCPI48_0@toc@l
 ; P9LE-NEXT:    lxvx v2, 0, r3
 ; P9LE-NEXT:    blr
 ;
 ; P8BE-LABEL: fromDiffConstsConvftoui:
 ; P8BE:       # %bb.0: # %entry
-; P8BE-NEXT:    addis r3, r2, .LCPI50_0@toc@ha
-; P8BE-NEXT:    addi r3, r3, .LCPI50_0@toc@l
+; P8BE-NEXT:    addis r3, r2, .LCPI48_0@toc@ha
+; P8BE-NEXT:    addi r3, r3, .LCPI48_0@toc@l
 ; P8BE-NEXT:    lxvw4x v2, 0, r3
 ; P8BE-NEXT:    blr
 ;
 ; P8LE-LABEL: fromDiffConstsConvftoui:
 ; P8LE:       # %bb.0: # %entry
-; P8LE-NEXT:    addis r3, r2, .LCPI50_0@toc@ha
-; P8LE-NEXT:    addi r3, r3, .LCPI50_0@toc@l
+; P8LE-NEXT:    addis r3, r2, .LCPI48_0@toc@ha
+; P8LE-NEXT:    addi r3, r3, .LCPI48_0@toc@l
 ; P8LE-NEXT:    lvx v2, 0, r3
 ; P8LE-NEXT:    blr
 entry:
   ret <4 x i32> <i32 24, i32 234, i32 988, i32 422>
 }
 
-; Function Attrs: norecurse nounwind readonly
 define <4 x i32> @fromDiffMemConsAConvftoui(float* nocapture readonly %ptr) {
 ; P9BE-LABEL: fromDiffMemConsAConvftoui:
 ; P9BE:       # %bb.0: # %entry
@@ -3099,13 +2963,12 @@ entry:
   ret <4 x i32> %2
 }
 
-; Function Attrs: norecurse nounwind readonly
 define <4 x i32> @fromDiffMemConsDConvftoui(float* nocapture readonly %ptr) {
 ; P9BE-LABEL: fromDiffMemConsDConvftoui:
 ; P9BE:       # %bb.0: # %entry
 ; P9BE-NEXT:    lxv v2, 0(r3)
-; P9BE-NEXT:    addis r3, r2, .LCPI52_0@toc@ha
-; P9BE-NEXT:    addi r3, r3, .LCPI52_0@toc@l
+; P9BE-NEXT:    addis r3, r2, .LCPI50_0@toc@ha
+; P9BE-NEXT:    addi r3, r3, .LCPI50_0@toc@l
 ; P9BE-NEXT:    lxvx v3, 0, r3
 ; P9BE-NEXT:    vperm v2, v2, v2, v3
 ; P9BE-NEXT:    xvcvspuxws v2, v2
@@ -3114,8 +2977,8 @@ define <4 x i32> @fromDiffMemConsDConvftoui(float* nocapture readonly %ptr) {
 ; P9LE-LABEL: fromDiffMemConsDConvftoui:
 ; P9LE:       # %bb.0: # %entry
 ; P9LE-NEXT:    lxv v2, 0(r3)
-; P9LE-NEXT:    addis r3, r2, .LCPI52_0@toc@ha
-; P9LE-NEXT:    addi r3, r3, .LCPI52_0@toc@l
+; P9LE-NEXT:    addis r3, r2, .LCPI50_0@toc@ha
+; P9LE-NEXT:    addi r3, r3, .LCPI50_0@toc@l
 ; P9LE-NEXT:    lxvx v3, 0, r3
 ; P9LE-NEXT:    vperm v2, v2, v2, v3
 ; P9LE-NEXT:    xvcvspuxws v2, v2
@@ -3123,9 +2986,9 @@ define <4 x i32> @fromDiffMemConsDConvftoui(float* nocapture readonly %ptr) {
 ;
 ; P8BE-LABEL: fromDiffMemConsDConvftoui:
 ; P8BE:       # %bb.0: # %entry
-; P8BE-NEXT:    addis r4, r2, .LCPI52_0@toc@ha
+; P8BE-NEXT:    addis r4, r2, .LCPI50_0@toc@ha
 ; P8BE-NEXT:    lxvw4x v2, 0, r3
-; P8BE-NEXT:    addi r4, r4, .LCPI52_0@toc@l
+; P8BE-NEXT:    addi r4, r4, .LCPI50_0@toc@l
 ; P8BE-NEXT:    lxvw4x v3, 0, r4
 ; P8BE-NEXT:    vperm v2, v2, v2, v3
 ; P8BE-NEXT:    xvcvspuxws v2, v2
@@ -3134,8 +2997,8 @@ define <4 x i32> @fromDiffMemConsDConvftoui(float* nocapture readonly %ptr) {
 ; P8LE-LABEL: fromDiffMemConsDConvftoui:
 ; P8LE:       # %bb.0: # %entry
 ; P8LE-NEXT:    lxvd2x vs0, 0, r3
-; P8LE-NEXT:    addis r4, r2, .LCPI52_0@toc@ha
-; P8LE-NEXT:    addi r3, r4, .LCPI52_0@toc@l
+; P8LE-NEXT:    addis r4, r2, .LCPI50_0@toc@ha
+; P8LE-NEXT:    addi r3, r4, .LCPI50_0@toc@l
 ; P8LE-NEXT:    lvx v2, 0, r3
 ; P8LE-NEXT:    xxswapd v3, vs0
 ; P8LE-NEXT:    vperm v2, v3, v3, v2
@@ -3160,7 +3023,6 @@ entry:
   ret <4 x i32> %vecinit9
 }
 
-; Function Attrs: norecurse nounwind readonly
 define <4 x i32> @fromDiffMemVarAConvftoui(float* nocapture readonly %arr, i32 signext %elem) {
 ; P9BE-LABEL: fromDiffMemVarAConvftoui:
 ; P9BE:       # %bb.0: # %entry
@@ -3249,7 +3111,6 @@ entry:
 ; FIXME: implement finding consecutive loads with pre-inc
 }
 
-; Function Attrs: norecurse nounwind readonly
 define <4 x i32> @fromDiffMemVarDConvftoui(float* nocapture readonly %arr, i32 signext %elem) {
 ; P9BE-LABEL: fromDiffMemVarDConvftoui:
 ; P9BE:       # %bb.0: # %entry
@@ -3338,7 +3199,6 @@ entry:
 ; FIXME: implement finding consecutive loads with pre-inc
 }
 
-; Function Attrs: norecurse nounwind readnone
 define <4 x i32> @spltRegValConvftoui(float %val) {
 ; P9BE-LABEL: spltRegValConvftoui:
 ; P9BE:       # %bb.0: # %entry
@@ -3370,7 +3230,6 @@ entry:
   ret <4 x i32> %splat.splat
 }
 
-; Function Attrs: norecurse nounwind readonly
 define <4 x i32> @spltMemValConvftoui(float* nocapture readonly %ptr) {
 ; P9BE-LABEL: spltMemValConvftoui:
 ; P9BE:       # %bb.0: # %entry
@@ -3405,7 +3264,6 @@ entry:
   ret <4 x i32> %splat.splat
 }
 
-; Function Attrs: norecurse nounwind readnone
 define <4 x i32> @spltCnstConvdtoui() {
 ; P9BE-LABEL: spltCnstConvdtoui:
 ; P9BE:       # %bb.0: # %entry
@@ -3430,7 +3288,6 @@ entry:
   ret <4 x i32> <i32 4, i32 4, i32 4, i32 4>
 }
 
-; Function Attrs: norecurse nounwind readnone
 define <4 x i32> @fromRegsConvdtoui(double %a, double %b, double %c, double %d) {
 ; P9BE-LABEL: fromRegsConvdtoui:
 ; P9BE:       # %bb.0: # %entry
@@ -3495,40 +3352,38 @@ entry:
   ret <4 x i32> %vecinit6
 }
 
-; Function Attrs: norecurse nounwind readnone
 define <4 x i32> @fromDiffConstsConvdtoui() {
 ; P9BE-LABEL: fromDiffConstsConvdtoui:
 ; P9BE:       # %bb.0: # %entry
-; P9BE-NEXT:    addis r3, r2, .LCPI59_0@toc@ha
-; P9BE-NEXT:    addi r3, r3, .LCPI59_0@toc@l
+; P9BE-NEXT:    addis r3, r2, .LCPI57_0@toc@ha
+; P9BE-NEXT:    addi r3, r3, .LCPI57_0@toc@l
 ; P9BE-NEXT:    lxvx v2, 0, r3
 ; P9BE-NEXT:    blr
 ;
 ; P9LE-LABEL: fromDiffConstsConvdtoui:
 ; P9LE:       # %bb.0: # %entry
-; P9LE-NEXT:    addis r3, r2, .LCPI59_0@toc@ha
-; P9LE-NEXT:    addi r3, r3, .LCPI59_0@toc@l
+; P9LE-NEXT:    addis r3, r2, .LCPI57_0@toc@ha
+; P9LE-NEXT:    addi r3, r3, .LCPI57_0@toc@l
 ; P9LE-NEXT:    lxvx v2, 0, r3
 ; P9LE-NEXT:    blr
 ;
 ; P8BE-LABEL: fromDiffConstsConvdtoui:
 ; P8BE:       # %bb.0: # %entry
-; P8BE-NEXT:    addis r3, r2, .LCPI59_0@toc@ha
-; P8BE-NEXT:    addi r3, r3, .LCPI59_0@toc@l
+; P8BE-NEXT:    addis r3, r2, .LCPI57_0@toc@ha
+; P8BE-NEXT:    addi r3, r3, .LCPI57_0@toc@l
 ; P8BE-NEXT:    lxvw4x v2, 0, r3
 ; P8BE-NEXT:    blr
 ;
 ; P8LE-LABEL: fromDiffConstsConvdtoui:
 ; P8LE:       # %bb.0: # %entry
-; P8LE-NEXT:    addis r3, r2, .LCPI59_0@toc@ha
-; P8LE-NEXT:    addi r3, r3, .LCPI59_0@toc@l
+; P8LE-NEXT:    addis r3, r2, .LCPI57_0@toc@ha
+; P8LE-NEXT:    addi r3, r3, .LCPI57_0@toc@l
 ; P8LE-NEXT:    lvx v2, 0, r3
 ; P8LE-NEXT:    blr
 entry:
   ret <4 x i32> <i32 24, i32 234, i32 988, i32 422>
 }
 
-; Function Attrs: norecurse nounwind readonly
 define <4 x i32> @fromDiffMemConsAConvdtoui(double* nocapture readonly %ptr) {
 ; P9BE-LABEL: fromDiffMemConsAConvdtoui:
 ; P9BE:       # %bb.0: # %entry
@@ -3589,7 +3444,6 @@ entry:
   ret <4 x i32> %vecinit9
 }
 
-; Function Attrs: norecurse nounwind readonly
 define <4 x i32> @fromDiffMemConsDConvdtoui(double* nocapture readonly %ptr) {
 ; P9BE-LABEL: fromDiffMemConsDConvdtoui:
 ; P9BE:       # %bb.0: # %entry
@@ -3661,7 +3515,6 @@ entry:
   ret <4 x i32> %vecinit9
 }
 
-; Function Attrs: norecurse nounwind readonly
 define <4 x i32> @fromDiffMemVarAConvdtoui(double* nocapture readonly %arr, i32 signext %elem) {
 ; P9BE-LABEL: fromDiffMemVarAConvdtoui:
 ; P9BE:       # %bb.0: # %entry
@@ -3745,7 +3598,6 @@ entry:
   ret <4 x i32> %vecinit14
 }
 
-; Function Attrs: norecurse nounwind readonly
 define <4 x i32> @fromDiffMemVarDConvdtoui(double* nocapture readonly %arr, i32 signext %elem) {
 ; P9BE-LABEL: fromDiffMemVarDConvdtoui:
 ; P9BE:       # %bb.0: # %entry
@@ -3829,7 +3681,6 @@ entry:
   ret <4 x i32> %vecinit14
 }
 
-; Function Attrs: norecurse nounwind readnone
 define <4 x i32> @spltRegValConvdtoui(double %val) {
 ; P9BE-LABEL: spltRegValConvdtoui:
 ; P9BE:       # %bb.0: # %entry
@@ -3861,7 +3712,6 @@ entry:
   ret <4 x i32> %splat.splat
 }
 
-; Function Attrs: norecurse nounwind readonly
 define <4 x i32> @spltMemValConvdtoui(double* nocapture readonly %ptr) {
 ; P9BE-LABEL: spltMemValConvdtoui:
 ; P9BE:       # %bb.0: # %entry
@@ -3897,7 +3747,7 @@ entry:
   %splat.splat = shufflevector <4 x i32> %splat.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
   ret <4 x i32> %splat.splat
 }
-; Function Attrs: norecurse nounwind readnone
+
 define <2 x i64> @allZeroll() {
 ; P9BE-LABEL: allZeroll:
 ; P9BE:       # %bb.0: # %entry
@@ -3922,58 +3772,32 @@ entry:
   ret <2 x i64> zeroinitializer
 }
 
-; Function Attrs: norecurse nounwind readnone
-define <2 x i64> @allOnell() {
-; P9BE-LABEL: allOnell:
-; P9BE:       # %bb.0: # %entry
-; P9BE-NEXT:    xxspltib v2, 255
-; P9BE-NEXT:    blr
-;
-; P9LE-LABEL: allOnell:
-; P9LE:       # %bb.0: # %entry
-; P9LE-NEXT:    xxspltib v2, 255
-; P9LE-NEXT:    blr
-;
-; P8BE-LABEL: allOnell:
-; P8BE:       # %bb.0: # %entry
-; P8BE-NEXT:    vspltisb v2, -1
-; P8BE-NEXT:    blr
-;
-; P8LE-LABEL: allOnell:
-; P8LE:       # %bb.0: # %entry
-; P8LE-NEXT:    vspltisb v2, -1
-; P8LE-NEXT:    blr
-entry:
-  ret <2 x i64> <i64 -1, i64 -1>
-}
-
-; Function Attrs: norecurse nounwind readnone
 define <2 x i64> @spltConst1ll() {
 ; P9BE-LABEL: spltConst1ll:
 ; P9BE:       # %bb.0: # %entry
-; P9BE-NEXT:    addis r3, r2, .LCPI68_0@toc@ha
-; P9BE-NEXT:    addi r3, r3, .LCPI68_0@toc@l
+; P9BE-NEXT:    addis r3, r2, .LCPI65_0@toc@ha
+; P9BE-NEXT:    addi r3, r3, .LCPI65_0@toc@l
 ; P9BE-NEXT:    lxvx v2, 0, r3
 ; P9BE-NEXT:    blr
 ;
 ; P9LE-LABEL: spltConst1ll:
 ; P9LE:       # %bb.0: # %entry
-; P9LE-NEXT:    addis r3, r2, .LCPI68_0@toc@ha
-; P9LE-NEXT:    addi r3, r3, .LCPI68_0@toc@l
+; P9LE-NEXT:    addis r3, r2, .LCPI65_0@toc@ha
+; P9LE-NEXT:    addi r3, r3, .LCPI65_0@toc@l
 ; P9LE-NEXT:    lxvx v2, 0, r3
 ; P9LE-NEXT:    blr
 ;
 ; P8BE-LABEL: spltConst1ll:
 ; P8BE:       # %bb.0: # %entry
-; P8BE-NEXT:    addis r3, r2, .LCPI68_0@toc@ha
-; P8BE-NEXT:    addi r3, r3, .LCPI68_0@toc@l
+; P8BE-NEXT:    addis r3, r2, .LCPI65_0@toc@ha
+; P8BE-NEXT:    addi r3, r3, .LCPI65_0@toc@l
 ; P8BE-NEXT:    lxvd2x v2, 0, r3
 ; P8BE-NEXT:    blr
 ;
 ; P8LE-LABEL: spltConst1ll:
 ; P8LE:       # %bb.0: # %entry
-; P8LE-NEXT:    addis r3, r2, .LCPI68_0@toc@ha
-; P8LE-NEXT:    addi r3, r3, .LCPI68_0@toc@l
+; P8LE-NEXT:    addis r3, r2, .LCPI65_0@toc@ha
+; P8LE-NEXT:    addi r3, r3, .LCPI65_0@toc@l
 ; P8LE-NEXT:    lxvd2x vs0, 0, r3
 ; P8LE-NEXT:    xxswapd v2, vs0
 ; P8LE-NEXT:    blr
@@ -3981,33 +3805,32 @@ entry:
   ret <2 x i64> <i64 1, i64 1>
 }
 
-; Function Attrs: norecurse nounwind readnone
 define <2 x i64> @spltConst16kll() {
 ; P9BE-LABEL: spltConst16kll:
 ; P9BE:       # %bb.0: # %entry
-; P9BE-NEXT:    addis r3, r2, .LCPI69_0@toc@ha
-; P9BE-NEXT:    addi r3, r3, .LCPI69_0@toc@l
+; P9BE-NEXT:    addis r3, r2, .LCPI66_0@toc@ha
+; P9BE-NEXT:    addi r3, r3, .LCPI66_0@toc@l
 ; P9BE-NEXT:    lxvx v2, 0, r3
 ; P9BE-NEXT:    blr
 ;
 ; P9LE-LABEL: spltConst16kll:
 ; P9LE:       # %bb.0: # %entry
-; P9LE-NEXT:    addis r3, r2, .LCPI69_0@toc@ha
-; P9LE-NEXT:    addi r3, r3, .LCPI69_0@toc@l
+; P9LE-NEXT:    addis r3, r2, .LCPI66_0@toc@ha
+; P9LE-NEXT:    addi r3, r3, .LCPI66_0@toc@l
 ; P9LE-NEXT:    lxvx v2, 0, r3
 ; P9LE-NEXT:    blr
 ;
 ; P8BE-LABEL: spltConst16kll:
 ; P8BE:       # %bb.0: # %entry
-; P8BE-NEXT:    addis r3, r2, .LCPI69_0@toc@ha
-; P8BE-NEXT:    addi r3, r3, .LCPI69_0@toc@l
+; P8BE-NEXT:    addis r3, r2, .LCPI66_0@toc@ha
+; P8BE-NEXT:    addi r3, r3, .LCPI66_0@toc@l
 ; P8BE-NEXT:    lxvd2x v2, 0, r3
 ; P8BE-NEXT:    blr
 ;
 ; P8LE-LABEL: spltConst16kll:
 ; P8LE:       # %bb.0: # %entry
-; P8LE-NEXT:    addis r3, r2, .LCPI69_0@toc@ha
-; P8LE-NEXT:    addi r3, r3, .LCPI69_0@toc@l
+; P8LE-NEXT:    addis r3, r2, .LCPI66_0@toc@ha
+; P8LE-NEXT:    addi r3, r3, .LCPI66_0@toc@l
 ; P8LE-NEXT:    lxvd2x vs0, 0, r3
 ; P8LE-NEXT:    xxswapd v2, vs0
 ; P8LE-NEXT:    blr
@@ -4015,33 +3838,32 @@ entry:
   ret <2 x i64> <i64 32767, i64 32767>
 }
 
-; Function Attrs: norecurse nounwind readnone
 define <2 x i64> @spltConst32kll() {
 ; P9BE-LABEL: spltConst32kll:
 ; P9BE:       # %bb.0: # %entry
-; P9BE-NEXT:    addis r3, r2, .LCPI70_0@toc@ha
-; P9BE-NEXT:    addi r3, r3, .LCPI70_0@toc@l
+; P9BE-NEXT:    addis r3, r2, .LCPI67_0@toc@ha
+; P9BE-NEXT:    addi r3, r3, .LCPI67_0@toc@l
 ; P9BE-NEXT:    lxvx v2, 0, r3
 ; P9BE-NEXT:    blr
 ;
 ; P9LE-LABEL: spltConst32kll:
 ; P9LE:       # %bb.0: # %entry
-; P9LE-NEXT:    addis r3, r2, .LCPI70_0@toc@ha
-; P9LE-NEXT:    addi r3, r3, .LCPI70_0@toc@l
+; P9LE-NEXT:    addis r3, r2, .LCPI67_0@toc@ha
+; P9LE-NEXT:    addi r3, r3, .LCPI67_0@toc@l
 ; P9LE-NEXT:    lxvx v2, 0, r3
 ; P9LE-NEXT:    blr
 ;
 ; P8BE-LABEL: spltConst32kll:
 ; P8BE:       # %bb.0: # %entry
-; P8BE-NEXT:    addis r3, r2, .LCPI70_0@toc@ha
-; P8BE-NEXT:    addi r3, r3, .LCPI70_0@toc@l
+; P8BE-NEXT:    addis r3, r2, .LCPI67_0@toc@ha
+; P8BE-NEXT:    addi r3, r3, .LCPI67_0@toc@l
 ; P8BE-NEXT:    lxvd2x v2, 0, r3
 ; P8BE-NEXT:    blr
 ;
 ; P8LE-LABEL: spltConst32kll:
 ; P8LE:       # %bb.0: # %entry
-; P8LE-NEXT:    addis r3, r2, .LCPI70_0@toc@ha
-; P8LE-NEXT:    addi r3, r3, .LCPI70_0@toc@l
+; P8LE-NEXT:    addis r3, r2, .LCPI67_0@toc@ha
+; P8LE-NEXT:    addi r3, r3, .LCPI67_0@toc@l
 ; P8LE-NEXT:    lxvd2x vs0, 0, r3
 ; P8LE-NEXT:    xxswapd v2, vs0
 ; P8LE-NEXT:    blr
@@ -4049,7 +3871,6 @@ entry:
   ret <2 x i64> <i64 65535, i64 65535>
 }
 
-; Function Attrs: norecurse nounwind readnone
 define <2 x i64> @fromRegsll(i64 %a, i64 %b) {
 ; P9BE-LABEL: fromRegsll:
 ; P9BE:       # %bb.0: # %entry
@@ -4080,33 +3901,32 @@ entry:
   ret <2 x i64> %vecinit1
 }
 
-; Function Attrs: norecurse nounwind readnone
 define <2 x i64> @fromDiffConstsll() {
 ; P9BE-LABEL: fromDiffConstsll:
 ; P9BE:       # %bb.0: # %entry
-; P9BE-NEXT:    addis r3, r2, .LCPI72_0@toc@ha
-; P9BE-NEXT:    addi r3, r3, .LCPI72_0@toc@l
+; P9BE-NEXT:    addis r3, r2, .LCPI69_0@toc@ha
+; P9BE-NEXT:    addi r3, r3, .LCPI69_0@toc@l
 ; P9BE-NEXT:    lxvx v2, 0, r3
 ; P9BE-NEXT:    blr
 ;
 ; P9LE-LABEL: fromDiffConstsll:
 ; P9LE:       # %bb.0: # %entry
-; P9LE-NEXT:    addis r3, r2, .LCPI72_0@toc@ha
-; P9LE-NEXT:    addi r3, r3, .LCPI72_0@toc@l
+; P9LE-NEXT:    addis r3, r2, .LCPI69_0@toc@ha
+; P9LE-NEXT:    addi r3, r3, .LCPI69_0@toc@l
 ; P9LE-NEXT:    lxvx v2, 0, r3
 ; P9LE-NEXT:    blr
 ;
 ; P8BE-LABEL: fromDiffConstsll:
 ; P8BE:       # %bb.0: # %entry
-; P8BE-NEXT:    addis r3, r2, .LCPI72_0@toc@ha
-; P8BE-NEXT:    addi r3, r3, .LCPI72_0@toc@l
+; P8BE-NEXT:    addis r3, r2, .LCPI69_0@toc@ha
+; P8BE-NEXT:    addi r3, r3, .LCPI69_0@toc@l
 ; P8BE-NEXT:    lxvd2x v2, 0, r3
 ; P8BE-NEXT:    blr
 ;
 ; P8LE-LABEL: fromDiffConstsll:
 ; P8LE:       # %bb.0: # %entry
-; P8LE-NEXT:    addis r3, r2, .LCPI72_0@toc@ha
-; P8LE-NEXT:    addi r3, r3, .LCPI72_0@toc@l
+; P8LE-NEXT:    addis r3, r2, .LCPI69_0@toc@ha
+; P8LE-NEXT:    addi r3, r3, .LCPI69_0@toc@l
 ; P8LE-NEXT:    lxvd2x vs0, 0, r3
 ; P8LE-NEXT:    xxswapd v2, vs0
 ; P8LE-NEXT:    blr
@@ -4114,7 +3934,6 @@ entry:
   ret <2 x i64> <i64 242, i64 -113>
 }
 
-; Function Attrs: norecurse nounwind readonly
 define <2 x i64> @fromDiffMemConsAll(i64* nocapture readonly %arr) {
 ; P9BE-LABEL: fromDiffMemConsAll:
 ; P9BE:       # %bb.0: # %entry
@@ -4145,7 +3964,6 @@ entry:
   ret <2 x i64> %vecinit2
 }
 
-; Function Attrs: norecurse nounwind readonly
 define <2 x i64> @fromDiffMemConsDll(i64* nocapture readonly %arr) {
 ; P9BE-LABEL: fromDiffMemConsDll:
 ; P9BE:       # %bb.0: # %entry
@@ -4155,8 +3973,8 @@ define <2 x i64> @fromDiffMemConsDll(i64* nocapture readonly %arr) {
 ;
 ; P9LE-LABEL: fromDiffMemConsDll:
 ; P9LE:       # %bb.0: # %entry
-; P9LE-NEXT:    lxv v2, 16(r3)
-; P9LE-NEXT:    xxswapd v2, v2
+; P9LE-NEXT:    addi r3, r3, 16
+; P9LE-NEXT:    lxvd2x v2, 0, r3
 ; P9LE-NEXT:    blr
 ;
 ; P8BE-LABEL: fromDiffMemConsDll:
@@ -4181,7 +3999,6 @@ entry:
   ret <2 x i64> %vecinit2
 }
 
-; Function Attrs: norecurse nounwind readonly
 define <2 x i64> @fromDiffMemVarAll(i64* nocapture readonly %arr, i32 signext %elem) {
 ; P9BE-LABEL: fromDiffMemVarAll:
 ; P9BE:       # %bb.0: # %entry
@@ -4220,7 +4037,6 @@ entry:
   ret <2 x i64> %vecinit3
 }
 
-; Function Attrs: norecurse nounwind readonly
 define <2 x i64> @fromDiffMemVarDll(i64* nocapture readonly %arr, i32 signext %elem) {
 ; P9BE-LABEL: fromDiffMemVarDll:
 ; P9BE:       # %bb.0: # %entry
@@ -4235,9 +4051,8 @@ define <2 x i64> @fromDiffMemVarDll(i64* nocapture readonly %arr, i32 signext %e
 ; P9LE:       # %bb.0: # %entry
 ; P9LE-NEXT:    sldi r4, r4, 3
 ; P9LE-NEXT:    add r3, r3, r4
-; P9LE-NEXT:    li r4, -8
-; P9LE-NEXT:    lxvx v2, r3, r4
-; P9LE-NEXT:    xxswapd v2, v2
+; P9LE-NEXT:    addi r3, r3, -8
+; P9LE-NEXT:    lxvd2x v2, 0, r3
 ; P9LE-NEXT:    blr
 ;
 ; P8BE-LABEL: fromDiffMemVarDll:
@@ -4269,7 +4084,6 @@ entry:
   ret <2 x i64> %vecinit3
 }
 
-; Function Attrs: norecurse nounwind readonly
 define <2 x i64> @fromRandMemConsll(i64* nocapture readonly %arr) {
 ; P9BE-LABEL: fromRandMemConsll:
 ; P9BE:       # %bb.0: # %entry
@@ -4312,7 +4126,6 @@ entry:
   ret <2 x i64> %vecinit2
 }
 
-; Function Attrs: norecurse nounwind readonly
 define <2 x i64> @fromRandMemVarll(i64* nocapture readonly %arr, i32 signext %elem) {
 ; P9BE-LABEL: fromRandMemVarll:
 ; P9BE:       # %bb.0: # %entry
@@ -4367,7 +4180,6 @@ entry:
   ret <2 x i64> %vecinit4
 }
 
-; Function Attrs: norecurse nounwind readnone
 define <2 x i64> @spltRegValll(i64 %val) {
 ; P9BE-LABEL: spltRegValll:
 ; P9BE:       # %bb.0: # %entry
@@ -4396,7 +4208,6 @@ entry:
   ret <2 x i64> %splat.splat
 }
 
-; Function Attrs: norecurse nounwind readonly
 define <2 x i64> @spltMemValll(i64* nocapture readonly %ptr) {
 ; P9BE-LABEL: spltMemValll:
 ; P9BE:       # %bb.0: # %entry
@@ -4424,33 +4235,32 @@ entry:
   ret <2 x i64> %splat.splat
 }
 
-; Function Attrs: norecurse nounwind readnone
 define <2 x i64> @spltCnstConvftoll() {
 ; P9BE-LABEL: spltCnstConvftoll:
 ; P9BE:       # %bb.0: # %entry
-; P9BE-NEXT:    addis r3, r2, .LCPI81_0@toc@ha
-; P9BE-NEXT:    addi r3, r3, .LCPI81_0@toc@l
+; P9BE-NEXT:    addis r3, r2, .LCPI78_0@toc@ha
+; P9BE-NEXT:    addi r3, r3, .LCPI78_0@toc@l
 ; P9BE-NEXT:    lxvx v2, 0, r3
 ; P9BE-NEXT:    blr
 ;
 ; P9LE-LABEL: spltCnstConvftoll:
 ; P9LE:       # %bb.0: # %entry
-; P9LE-NEXT:    addis r3, r2, .LCPI81_0@toc@ha
-; P9LE-NEXT:    addi r3, r3, .LCPI81_0@toc@l
+; P9LE-NEXT:    addis r3, r2, .LCPI78_0@toc@ha
+; P9LE-NEXT:    addi r3, r3, .LCPI78_0@toc@l
 ; P9LE-NEXT:    lxvx v2, 0, r3
 ; P9LE-NEXT:    blr
 ;
 ; P8BE-LABEL: spltCnstConvftoll:
 ; P8BE:       # %bb.0: # %entry
-; P8BE-NEXT:    addis r3, r2, .LCPI81_0@toc@ha
-; P8BE-NEXT:    addi r3, r3, .LCPI81_0@toc@l
+; P8BE-NEXT:    addis r3, r2, .LCPI78_0@toc@ha
+; P8BE-NEXT:    addi r3, r3, .LCPI78_0@toc@l
 ; P8BE-NEXT:    lxvd2x v2, 0, r3
 ; P8BE-NEXT:    blr
 ;
 ; P8LE-LABEL: spltCnstConvftoll:
 ; P8LE:       # %bb.0: # %entry
-; P8LE-NEXT:    addis r3, r2, .LCPI81_0@toc@ha
-; P8LE-NEXT:    addi r3, r3, .LCPI81_0@toc@l
+; P8LE-NEXT:    addis r3, r2, .LCPI78_0@toc@ha
+; P8LE-NEXT:    addi r3, r3, .LCPI78_0@toc@l
 ; P8LE-NEXT:    lxvd2x vs0, 0, r3
 ; P8LE-NEXT:    xxswapd v2, vs0
 ; P8LE-NEXT:    blr
@@ -4458,7 +4268,6 @@ entry:
   ret <2 x i64> <i64 4, i64 4>
 }
 
-; Function Attrs: norecurse nounwind readnone
 define <2 x i64> @fromRegsConvftoll(float %a, float %b) {
 ; P9BE-LABEL: fromRegsConvftoll:
 ; P9BE:       # %bb.0: # %entry
@@ -4499,33 +4308,32 @@ entry:
   ret <2 x i64> %vecinit2
 }
 
-; Function Attrs: norecurse nounwind readnone
 define <2 x i64> @fromDiffConstsConvftoll() {
 ; P9BE-LABEL: fromDiffConstsConvftoll:
 ; P9BE:       # %bb.0: # %entry
-; P9BE-NEXT:    addis r3, r2, .LCPI83_0@toc@ha
-; P9BE-NEXT:    addi r3, r3, .LCPI83_0@toc@l
+; P9BE-NEXT:    addis r3, r2, .LCPI80_0@toc@ha
+; P9BE-NEXT:    addi r3, r3, .LCPI80_0@toc@l
 ; P9BE-NEXT:    lxvx v2, 0, r3
 ; P9BE-NEXT:    blr
 ;
 ; P9LE-LABEL: fromDiffConstsConvftoll:
 ; P9LE:       # %bb.0: # %entry
-; P9LE-NEXT:    addis r3, r2, .LCPI83_0@toc@ha
-; P9LE-NEXT:    addi r3, r3, .LCPI83_0@toc@l
+; P9LE-NEXT:    addis r3, r2, .LCPI80_0@toc@ha
+; P9LE-NEXT:    addi r3, r3, .LCPI80_0@toc@l
 ; P9LE-NEXT:    lxvx v2, 0, r3
 ; P9LE-NEXT:    blr
 ;
 ; P8BE-LABEL: fromDiffConstsConvftoll:
 ; P8BE:       # %bb.0: # %entry
-; P8BE-NEXT:    addis r3, r2, .LCPI83_0@toc@ha
-; P8BE-NEXT:    addi r3, r3, .LCPI83_0@toc@l
+; P8BE-NEXT:    addis r3, r2, .LCPI80_0@toc@ha
+; P8BE-NEXT:    addi r3, r3, .LCPI80_0@toc@l
 ; P8BE-NEXT:    lxvd2x v2, 0, r3
 ; P8BE-NEXT:    blr
 ;
 ; P8LE-LABEL: fromDiffConstsConvftoll:
 ; P8LE:       # %bb.0: # %entry
-; P8LE-NEXT:    addis r3, r2, .LCPI83_0@toc@ha
-; P8LE-NEXT:    addi r3, r3, .LCPI83_0@toc@l
+; P8LE-NEXT:    addis r3, r2, .LCPI80_0@toc@ha
+; P8LE-NEXT:    addi r3, r3, .LCPI80_0@toc@l
 ; P8LE-NEXT:    lxvd2x vs0, 0, r3
 ; P8LE-NEXT:    xxswapd v2, vs0
 ; P8LE-NEXT:    blr
@@ -4533,7 +4341,6 @@ entry:
   ret <2 x i64> <i64 24, i64 234>
 }
 
-; Function Attrs: norecurse nounwind readonly
 define <2 x i64> @fromDiffMemConsAConvftoll(float* nocapture readonly %ptr) {
 ; P9BE-LABEL: fromDiffMemConsAConvftoll:
 ; P9BE:       # %bb.0: # %entry
@@ -4577,7 +4384,6 @@ entry:
   ret <2 x i64> %vecinit3
 }
 
-; Function Attrs: norecurse nounwind readonly
 define <2 x i64> @fromDiffMemConsDConvftoll(float* nocapture readonly %ptr) {
 ; P9BE-LABEL: fromDiffMemConsDConvftoll:
 ; P9BE:       # %bb.0: # %entry
@@ -4622,7 +4428,6 @@ entry:
   ret <2 x i64> %vecinit3
 }
 
-; Function Attrs: norecurse nounwind readonly
 define <2 x i64> @fromDiffMemVarAConvftoll(float* nocapture readonly %arr, i32 signext %elem) {
 ; P9BE-LABEL: fromDiffMemVarAConvftoll:
 ; P9BE:       # %bb.0: # %entry
@@ -4674,7 +4479,6 @@ entry:
   ret <2 x i64> %vecinit4
 }
 
-; Function Attrs: norecurse nounwind readonly
 define <2 x i64> @fromDiffMemVarDConvftoll(float* nocapture readonly %arr, i32 signext %elem) {
 ; P9BE-LABEL: fromDiffMemVarDConvftoll:
 ; P9BE:       # %bb.0: # %entry
@@ -4726,7 +4530,6 @@ entry:
   ret <2 x i64> %vecinit4
 }
 
-; Function Attrs: norecurse nounwind readnone
 define <2 x i64> @spltRegValConvftoll(float %val) {
 ; P9BE-LABEL: spltRegValConvftoll:
 ; P9BE:       # %bb.0: # %entry
@@ -4758,7 +4561,6 @@ entry:
   ret <2 x i64> %splat.splat
 }
 
-; Function Attrs: norecurse nounwind readonly
 define <2 x i64> @spltMemValConvftoll(float* nocapture readonly %ptr) {
 ; P9BE-LABEL: spltMemValConvftoll:
 ; P9BE:       # %bb.0: # %entry
@@ -4795,33 +4597,32 @@ entry:
   ret <2 x i64> %splat.splat
 }
 
-; Function Attrs: norecurse nounwind readnone
 define <2 x i64> @spltCnstConvdtoll() {
 ; P9BE-LABEL: spltCnstConvdtoll:
 ; P9BE:       # %bb.0: # %entry
-; P9BE-NEXT:    addis r3, r2, .LCPI90_0@toc@ha
-; P9BE-NEXT:    addi r3, r3, .LCPI90_0@toc@l
+; P9BE-NEXT:    addis r3, r2, .LCPI87_0@toc@ha
+; P9BE-NEXT:    addi r3, r3, .LCPI87_0@toc@l
 ; P9BE-NEXT:    lxvx v2, 0, r3
 ; P9BE-NEXT:    blr
 ;
 ; P9LE-LABEL: spltCnstConvdtoll:
 ; P9LE:       # %bb.0: # %entry
-; P9LE-NEXT:    addis r3, r2, .LCPI90_0@toc@ha
-; P9LE-NEXT:    addi r3, r3, .LCPI90_0@toc@l
+; P9LE-NEXT:    addis r3, r2, .LCPI87_0@toc@ha
+; P9LE-NEXT:    addi r3, r3, .LCPI87_0@toc@l
 ; P9LE-NEXT:    lxvx v2, 0, r3
 ; P9LE-NEXT:    blr
 ;
 ; P8BE-LABEL: spltCnstConvdtoll:
 ; P8BE:       # %bb.0: # %entry
-; P8BE-NEXT:    addis r3, r2, .LCPI90_0@toc@ha
-; P8BE-NEXT:    addi r3, r3, .LCPI90_0@toc@l
+; P8BE-NEXT:    addis r3, r2, .LCPI87_0@toc@ha
+; P8BE-NEXT:    addi r3, r3, .LCPI87_0@toc@l
 ; P8BE-NEXT:    lxvd2x v2, 0, r3
 ; P8BE-NEXT:    blr
 ;
 ; P8LE-LABEL: spltCnstConvdtoll:
 ; P8LE:       # %bb.0: # %entry
-; P8LE-NEXT:    addis r3, r2, .LCPI90_0@toc@ha
-; P8LE-NEXT:    addi r3, r3, .LCPI90_0@toc@l
+; P8LE-NEXT:    addis r3, r2, .LCPI87_0@toc@ha
+; P8LE-NEXT:    addi r3, r3, .LCPI87_0@toc@l
 ; P8LE-NEXT:    lxvd2x vs0, 0, r3
 ; P8LE-NEXT:    xxswapd v2, vs0
 ; P8LE-NEXT:    blr
@@ -4829,7 +4630,6 @@ entry:
   ret <2 x i64> <i64 4, i64 4>
 }
 
-; Function Attrs: norecurse nounwind readnone
 define <2 x i64> @fromRegsConvdtoll(double %a, double %b) {
 ; P9BE-LABEL: fromRegsConvdtoll:
 ; P9BE:       # %bb.0: # %entry
@@ -4870,33 +4670,32 @@ entry:
   ret <2 x i64> %vecinit2
 }
 
-; Function Attrs: norecurse nounwind readnone
 define <2 x i64> @fromDiffConstsConvdtoll() {
 ; P9BE-LABEL: fromDiffConstsConvdtoll:
 ; P9BE:       # %bb.0: # %entry
-; P9BE-NEXT:    addis r3, r2, .LCPI92_0@toc@ha
-; P9BE-NEXT:    addi r3, r3, .LCPI92_0@toc@l
+; P9BE-NEXT:    addis r3, r2, .LCPI89_0@toc@ha
+; P9BE-NEXT:    addi r3, r3, .LCPI89_0@toc@l
 ; P9BE-NEXT:    lxvx v2, 0, r3
 ; P9BE-NEXT:    blr
 ;
 ; P9LE-LABEL: fromDiffConstsConvdtoll:
 ; P9LE:       # %bb.0: # %entry
-; P9LE-NEXT:    addis r3, r2, .LCPI92_0@toc@ha
-; P9LE-NEXT:    addi r3, r3, .LCPI92_0@toc@l
+; P9LE-NEXT:    addis r3, r2, .LCPI89_0@toc@ha
+; P9LE-NEXT:    addi r3, r3, .LCPI89_0@toc@l
 ; P9LE-NEXT:    lxvx v2, 0, r3
 ; P9LE-NEXT:    blr
 ;
 ; P8BE-LABEL: fromDiffConstsConvdtoll:
 ; P8BE:       # %bb.0: # %entry
-; P8BE-NEXT:    addis r3, r2, .LCPI92_0@toc@ha
-; P8BE-NEXT:    addi r3, r3, .LCPI92_0@toc@l
+; P8BE-NEXT:    addis r3, r2, .LCPI89_0@toc@ha
+; P8BE-NEXT:    addi r3, r3, .LCPI89_0@toc@l
 ; P8BE-NEXT:    lxvd2x v2, 0, r3
 ; P8BE-NEXT:    blr
 ;
 ; P8LE-LABEL: fromDiffConstsConvdtoll:
 ; P8LE:       # %bb.0: # %entry
-; P8LE-NEXT:    addis r3, r2, .LCPI92_0@toc@ha
-; P8LE-NEXT:    addi r3, r3, .LCPI92_0@toc@l
+; P8LE-NEXT:    addis r3, r2, .LCPI89_0@toc@ha
+; P8LE-NEXT:    addi r3, r3, .LCPI89_0@toc@l
 ; P8LE-NEXT:    lxvd2x vs0, 0, r3
 ; P8LE-NEXT:    xxswapd v2, vs0
 ; P8LE-NEXT:    blr
@@ -4904,7 +4703,6 @@ entry:
   ret <2 x i64> <i64 24, i64 234>
 }
 
-; Function Attrs: norecurse nounwind readonly
 define <2 x i64> @fromDiffMemConsAConvdtoll(double* nocapture readonly %ptr) {
 ; P9BE-LABEL: fromDiffMemConsAConvdtoll:
 ; P9BE:       # %bb.0: # %entry
@@ -4937,7 +4735,6 @@ entry:
   ret <2 x i64> %2
 }
 
-; Function Attrs: norecurse nounwind readonly
 define <2 x i64> @fromDiffMemConsDConvdtoll(double* nocapture readonly %ptr) {
 ; P9BE-LABEL: fromDiffMemConsDConvdtoll:
 ; P9BE:       # %bb.0: # %entry
@@ -4948,8 +4745,8 @@ define <2 x i64> @fromDiffMemConsDConvdtoll(double* nocapture readonly %ptr) {
 ;
 ; P9LE-LABEL: fromDiffMemConsDConvdtoll:
 ; P9LE:       # %bb.0: # %entry
-; P9LE-NEXT:    lxv vs0, 16(r3)
-; P9LE-NEXT:    xxswapd vs0, vs0
+; P9LE-NEXT:    addi r3, r3, 16
+; P9LE-NEXT:    lxvd2x vs0, 0, r3
 ; P9LE-NEXT:    xvcvdpsxds v2, vs0
 ; P9LE-NEXT:    blr
 ;
@@ -4979,7 +4776,6 @@ entry:
   ret <2 x i64> %vecinit3
 }
 
-; Function Attrs: norecurse nounwind readonly
 define <2 x i64> @fromDiffMemVarAConvdtoll(double* nocapture readonly %arr, i32 signext %elem) {
 ; P9BE-LABEL: fromDiffMemVarAConvdtoll:
 ; P9BE:       # %bb.0: # %entry
@@ -5024,7 +4820,6 @@ entry:
   ret <2 x i64> %vecinit4
 }
 
-; Function Attrs: norecurse nounwind readonly
 define <2 x i64> @fromDiffMemVarDConvdtoll(double* nocapture readonly %arr, i32 signext %elem) {
 ; P9BE-LABEL: fromDiffMemVarDConvdtoll:
 ; P9BE:       # %bb.0: # %entry
@@ -5040,9 +4835,8 @@ define <2 x i64> @fromDiffMemVarDConvdtoll(double* nocapture readonly %arr, i32
 ; P9LE:       # %bb.0: # %entry
 ; P9LE-NEXT:    sldi r4, r4, 3
 ; P9LE-NEXT:    add r3, r3, r4
-; P9LE-NEXT:    li r4, -8
-; P9LE-NEXT:    lxvx vs0, r3, r4
-; P9LE-NEXT:    xxswapd vs0, vs0
+; P9LE-NEXT:    addi r3, r3, -8
+; P9LE-NEXT:    lxvd2x vs0, 0, r3
 ; P9LE-NEXT:    xvcvdpsxds v2, vs0
 ; P9LE-NEXT:    blr
 ;
@@ -5079,7 +4873,6 @@ entry:
   ret <2 x i64> %vecinit4
 }
 
-; Function Attrs: norecurse nounwind readnone
 define <2 x i64> @spltRegValConvdtoll(double %val) {
 ; P9BE-LABEL: spltRegValConvdtoll:
 ; P9BE:       # %bb.0: # %entry
@@ -5111,7 +4904,6 @@ entry:
   ret <2 x i64> %splat.splat
 }
 
-; Function Attrs: norecurse nounwind readonly
 define <2 x i64> @spltMemValConvdtoll(double* nocapture readonly %ptr) {
 ; P9BE-LABEL: spltMemValConvdtoll:
 ; P9BE:       # %bb.0: # %entry
@@ -5144,7 +4936,6 @@ entry:
   ret <2 x i64> %splat.splat
 }
 
-; Function Attrs: norecurse nounwind readnone
 define <2 x i64> @allZeroull() {
 ; P9BE-LABEL: allZeroull:
 ; P9BE:       # %bb.0: # %entry
@@ -5169,58 +4960,32 @@ entry:
   ret <2 x i64> zeroinitializer
 }
 
-; Function Attrs: norecurse nounwind readnone
-define <2 x i64> @allOneull() {
-; P9BE-LABEL: allOneull:
-; P9BE:       # %bb.0: # %entry
-; P9BE-NEXT:    xxspltib v2, 255
-; P9BE-NEXT:    blr
-;
-; P9LE-LABEL: allOneull:
-; P9LE:       # %bb.0: # %entry
-; P9LE-NEXT:    xxspltib v2, 255
-; P9LE-NEXT:    blr
-;
-; P8BE-LABEL: allOneull:
-; P8BE:       # %bb.0: # %entry
-; P8BE-NEXT:    vspltisb v2, -1
-; P8BE-NEXT:    blr
-;
-; P8LE-LABEL: allOneull:
-; P8LE:       # %bb.0: # %entry
-; P8LE-NEXT:    vspltisb v2, -1
-; P8LE-NEXT:    blr
-entry:
-  ret <2 x i64> <i64 -1, i64 -1>
-}
-
-; Function Attrs: norecurse nounwind readnone
 define <2 x i64> @spltConst1ull() {
 ; P9BE-LABEL: spltConst1ull:
 ; P9BE:       # %bb.0: # %entry
-; P9BE-NEXT:    addis r3, r2, .LCPI101_0@toc@ha
-; P9BE-NEXT:    addi r3, r3, .LCPI101_0@toc@l
+; P9BE-NEXT:    addis r3, r2, .LCPI97_0@toc@ha
+; P9BE-NEXT:    addi r3, r3, .LCPI97_0@toc@l
 ; P9BE-NEXT:    lxvx v2, 0, r3
 ; P9BE-NEXT:    blr
 ;
 ; P9LE-LABEL: spltConst1ull:
 ; P9LE:       # %bb.0: # %entry
-; P9LE-NEXT:    addis r3, r2, .LCPI101_0@toc@ha
-; P9LE-NEXT:    addi r3, r3, .LCPI101_0@toc@l
+; P9LE-NEXT:    addis r3, r2, .LCPI97_0@toc@ha
+; P9LE-NEXT:    addi r3, r3, .LCPI97_0@toc@l
 ; P9LE-NEXT:    lxvx v2, 0, r3
 ; P9LE-NEXT:    blr
 ;
 ; P8BE-LABEL: spltConst1ull:
 ; P8BE:       # %bb.0: # %entry
-; P8BE-NEXT:    addis r3, r2, .LCPI101_0@toc@ha
-; P8BE-NEXT:    addi r3, r3, .LCPI101_0@toc@l
+; P8BE-NEXT:    addis r3, r2, .LCPI97_0@toc@ha
+; P8BE-NEXT:    addi r3, r3, .LCPI97_0@toc@l
 ; P8BE-NEXT:    lxvd2x v2, 0, r3
 ; P8BE-NEXT:    blr
 ;
 ; P8LE-LABEL: spltConst1ull:
 ; P8LE:       # %bb.0: # %entry
-; P8LE-NEXT:    addis r3, r2, .LCPI101_0@toc@ha
-; P8LE-NEXT:    addi r3, r3, .LCPI101_0@toc@l
+; P8LE-NEXT:    addis r3, r2, .LCPI97_0@toc@ha
+; P8LE-NEXT:    addi r3, r3, .LCPI97_0@toc@l
 ; P8LE-NEXT:    lxvd2x vs0, 0, r3
 ; P8LE-NEXT:    xxswapd v2, vs0
 ; P8LE-NEXT:    blr
@@ -5228,33 +4993,32 @@ entry:
   ret <2 x i64> <i64 1, i64 1>
 }
 
-; Function Attrs: norecurse nounwind readnone
 define <2 x i64> @spltConst16kull() {
 ; P9BE-LABEL: spltConst16kull:
 ; P9BE:       # %bb.0: # %entry
-; P9BE-NEXT:    addis r3, r2, .LCPI102_0@toc@ha
-; P9BE-NEXT:    addi r3, r3, .LCPI102_0@toc@l
+; P9BE-NEXT:    addis r3, r2, .LCPI98_0@toc@ha
+; P9BE-NEXT:    addi r3, r3, .LCPI98_0@toc@l
 ; P9BE-NEXT:    lxvx v2, 0, r3
 ; P9BE-NEXT:    blr
 ;
 ; P9LE-LABEL: spltConst16kull:
 ; P9LE:       # %bb.0: # %entry
-; P9LE-NEXT:    addis r3, r2, .LCPI102_0@toc@ha
-; P9LE-NEXT:    addi r3, r3, .LCPI102_0@toc@l
+; P9LE-NEXT:    addis r3, r2, .LCPI98_0@toc@ha
+; P9LE-NEXT:    addi r3, r3, .LCPI98_0@toc@l
 ; P9LE-NEXT:    lxvx v2, 0, r3
 ; P9LE-NEXT:    blr
 ;
 ; P8BE-LABEL: spltConst16kull:
 ; P8BE:       # %bb.0: # %entry
-; P8BE-NEXT:    addis r3, r2, .LCPI102_0@toc@ha
-; P8BE-NEXT:    addi r3, r3, .LCPI102_0@toc@l
+; P8BE-NEXT:    addis r3, r2, .LCPI98_0@toc@ha
+; P8BE-NEXT:    addi r3, r3, .LCPI98_0@toc@l
 ; P8BE-NEXT:    lxvd2x v2, 0, r3
 ; P8BE-NEXT:    blr
 ;
 ; P8LE-LABEL: spltConst16kull:
 ; P8LE:       # %bb.0: # %entry
-; P8LE-NEXT:    addis r3, r2, .LCPI102_0@toc@ha
-; P8LE-NEXT:    addi r3, r3, .LCPI102_0@toc@l
+; P8LE-NEXT:    addis r3, r2, .LCPI98_0@toc@ha
+; P8LE-NEXT:    addi r3, r3, .LCPI98_0@toc@l
 ; P8LE-NEXT:    lxvd2x vs0, 0, r3
 ; P8LE-NEXT:    xxswapd v2, vs0
 ; P8LE-NEXT:    blr
@@ -5262,33 +5026,32 @@ entry:
   ret <2 x i64> <i64 32767, i64 32767>
 }
 
-; Function Attrs: norecurse nounwind readnone
 define <2 x i64> @spltConst32kull() {
 ; P9BE-LABEL: spltConst32kull:
 ; P9BE:       # %bb.0: # %entry
-; P9BE-NEXT:    addis r3, r2, .LCPI103_0@toc@ha
-; P9BE-NEXT:    addi r3, r3, .LCPI103_0@toc@l
+; P9BE-NEXT:    addis r3, r2, .LCPI99_0@toc@ha
+; P9BE-NEXT:    addi r3, r3, .LCPI99_0@toc@l
 ; P9BE-NEXT:    lxvx v2, 0, r3
 ; P9BE-NEXT:    blr
 ;
 ; P9LE-LABEL: spltConst32kull:
 ; P9LE:       # %bb.0: # %entry
-; P9LE-NEXT:    addis r3, r2, .LCPI103_0@toc@ha
-; P9LE-NEXT:    addi r3, r3, .LCPI103_0@toc@l
+; P9LE-NEXT:    addis r3, r2, .LCPI99_0@toc@ha
+; P9LE-NEXT:    addi r3, r3, .LCPI99_0@toc@l
 ; P9LE-NEXT:    lxvx v2, 0, r3
 ; P9LE-NEXT:    blr
 ;
 ; P8BE-LABEL: spltConst32kull:
 ; P8BE:       # %bb.0: # %entry
-; P8BE-NEXT:    addis r3, r2, .LCPI103_0@toc@ha
-; P8BE-NEXT:    addi r3, r3, .LCPI103_0@toc@l
+; P8BE-NEXT:    addis r3, r2, .LCPI99_0@toc@ha
+; P8BE-NEXT:    addi r3, r3, .LCPI99_0@toc@l
 ; P8BE-NEXT:    lxvd2x v2, 0, r3
 ; P8BE-NEXT:    blr
 ;
 ; P8LE-LABEL: spltConst32kull:
 ; P8LE:       # %bb.0: # %entry
-; P8LE-NEXT:    addis r3, r2, .LCPI103_0@toc@ha
-; P8LE-NEXT:    addi r3, r3, .LCPI103_0@toc@l
+; P8LE-NEXT:    addis r3, r2, .LCPI99_0@toc@ha
+; P8LE-NEXT:    addi r3, r3, .LCPI99_0@toc@l
 ; P8LE-NEXT:    lxvd2x vs0, 0, r3
 ; P8LE-NEXT:    xxswapd v2, vs0
 ; P8LE-NEXT:    blr
@@ -5296,7 +5059,6 @@ entry:
   ret <2 x i64> <i64 65535, i64 65535>
 }
 
-; Function Attrs: norecurse nounwind readnone
 define <2 x i64> @fromRegsull(i64 %a, i64 %b) {
 ; P9BE-LABEL: fromRegsull:
 ; P9BE:       # %bb.0: # %entry
@@ -5327,33 +5089,32 @@ entry:
   ret <2 x i64> %vecinit1
 }
 
-; Function Attrs: norecurse nounwind readnone
 define <2 x i64> @fromDiffConstsull() {
 ; P9BE-LABEL: fromDiffConstsull:
 ; P9BE:       # %bb.0: # %entry
-; P9BE-NEXT:    addis r3, r2, .LCPI105_0@toc@ha
-; P9BE-NEXT:    addi r3, r3, .LCPI105_0@toc@l
+; P9BE-NEXT:    addis r3, r2, .LCPI101_0@toc@ha
+; P9BE-NEXT:    addi r3, r3, .LCPI101_0@toc@l
 ; P9BE-NEXT:    lxvx v2, 0, r3
 ; P9BE-NEXT:    blr
 ;
 ; P9LE-LABEL: fromDiffConstsull:
 ; P9LE:       # %bb.0: # %entry
-; P9LE-NEXT:    addis r3, r2, .LCPI105_0@toc@ha
-; P9LE-NEXT:    addi r3, r3, .LCPI105_0@toc@l
+; P9LE-NEXT:    addis r3, r2, .LCPI101_0@toc@ha
+; P9LE-NEXT:    addi r3, r3, .LCPI101_0@toc@l
 ; P9LE-NEXT:    lxvx v2, 0, r3
 ; P9LE-NEXT:    blr
 ;
 ; P8BE-LABEL: fromDiffConstsull:
 ; P8BE:       # %bb.0: # %entry
-; P8BE-NEXT:    addis r3, r2, .LCPI105_0@toc@ha
-; P8BE-NEXT:    addi r3, r3, .LCPI105_0@toc@l
+; P8BE-NEXT:    addis r3, r2, .LCPI101_0@toc@ha
+; P8BE-NEXT:    addi r3, r3, .LCPI101_0@toc@l
 ; P8BE-NEXT:    lxvd2x v2, 0, r3
 ; P8BE-NEXT:    blr
 ;
 ; P8LE-LABEL: fromDiffConstsull:
 ; P8LE:       # %bb.0: # %entry
-; P8LE-NEXT:    addis r3, r2, .LCPI105_0@toc@ha
-; P8LE-NEXT:    addi r3, r3, .LCPI105_0@toc@l
+; P8LE-NEXT:    addis r3, r2, .LCPI101_0@toc@ha
+; P8LE-NEXT:    addi r3, r3, .LCPI101_0@toc@l
 ; P8LE-NEXT:    lxvd2x vs0, 0, r3
 ; P8LE-NEXT:    xxswapd v2, vs0
 ; P8LE-NEXT:    blr
@@ -5361,7 +5122,6 @@ entry:
   ret <2 x i64> <i64 242, i64 -113>
 }
 
-; Function Attrs: norecurse nounwind readonly
 define <2 x i64> @fromDiffMemConsAull(i64* nocapture readonly %arr) {
 ; P9BE-LABEL: fromDiffMemConsAull:
 ; P9BE:       # %bb.0: # %entry
@@ -5392,7 +5152,6 @@ entry:
   ret <2 x i64> %vecinit2
 }
 
-; Function Attrs: norecurse nounwind readonly
 define <2 x i64> @fromDiffMemConsDull(i64* nocapture readonly %arr) {
 ; P9BE-LABEL: fromDiffMemConsDull:
 ; P9BE:       # %bb.0: # %entry
@@ -5402,8 +5161,8 @@ define <2 x i64> @fromDiffMemConsDull(i64* nocapture readonly %arr) {
 ;
 ; P9LE-LABEL: fromDiffMemConsDull:
 ; P9LE:       # %bb.0: # %entry
-; P9LE-NEXT:    lxv v2, 16(r3)
-; P9LE-NEXT:    xxswapd v2, v2
+; P9LE-NEXT:    addi r3, r3, 16
+; P9LE-NEXT:    lxvd2x v2, 0, r3
 ; P9LE-NEXT:    blr
 ;
 ; P8BE-LABEL: fromDiffMemConsDull:
@@ -5428,7 +5187,6 @@ entry:
   ret <2 x i64> %vecinit2
 }
 
-; Function Attrs: norecurse nounwind readonly
 define <2 x i64> @fromDiffMemVarAull(i64* nocapture readonly %arr, i32 signext %elem) {
 ; P9BE-LABEL: fromDiffMemVarAull:
 ; P9BE:       # %bb.0: # %entry
@@ -5467,7 +5225,6 @@ entry:
   ret <2 x i64> %vecinit3
 }
 
-; Function Attrs: norecurse nounwind readonly
 define <2 x i64> @fromDiffMemVarDull(i64* nocapture readonly %arr, i32 signext %elem) {
 ; P9BE-LABEL: fromDiffMemVarDull:
 ; P9BE:       # %bb.0: # %entry
@@ -5482,9 +5239,8 @@ define <2 x i64> @fromDiffMemVarDull(i64* nocapture readonly %arr, i32 signext %
 ; P9LE:       # %bb.0: # %entry
 ; P9LE-NEXT:    sldi r4, r4, 3
 ; P9LE-NEXT:    add r3, r3, r4
-; P9LE-NEXT:    li r4, -8
-; P9LE-NEXT:    lxvx v2, r3, r4
-; P9LE-NEXT:    xxswapd v2, v2
+; P9LE-NEXT:    addi r3, r3, -8
+; P9LE-NEXT:    lxvd2x v2, 0, r3
 ; P9LE-NEXT:    blr
 ;
 ; P8BE-LABEL: fromDiffMemVarDull:
@@ -5516,7 +5272,6 @@ entry:
   ret <2 x i64> %vecinit3
 }
 
-; Function Attrs: norecurse nounwind readonly
 define <2 x i64> @fromRandMemConsull(i64* nocapture readonly %arr) {
 ; P9BE-LABEL: fromRandMemConsull:
 ; P9BE:       # %bb.0: # %entry
@@ -5559,7 +5314,6 @@ entry:
   ret <2 x i64> %vecinit2
 }
 
-; Function Attrs: norecurse nounwind readonly
 define <2 x i64> @fromRandMemVarull(i64* nocapture readonly %arr, i32 signext %elem) {
 ; P9BE-LABEL: fromRandMemVarull:
 ; P9BE:       # %bb.0: # %entry
@@ -5614,7 +5368,6 @@ entry:
   ret <2 x i64> %vecinit4
 }
 
-; Function Attrs: norecurse nounwind readnone
 define <2 x i64> @spltRegValull(i64 %val) {
 ; P9BE-LABEL: spltRegValull:
 ; P9BE:       # %bb.0: # %entry
@@ -5643,7 +5396,6 @@ entry:
   ret <2 x i64> %splat.splat
 }
 
-; Function Attrs: norecurse nounwind readonly
 define <2 x i64> @spltMemValull(i64* nocapture readonly %ptr) {
 ; P9BE-LABEL: spltMemValull:
 ; P9BE:       # %bb.0: # %entry
@@ -5671,33 +5423,32 @@ entry:
   ret <2 x i64> %splat.splat
 }
 
-; Function Attrs: norecurse nounwind readnone
 define <2 x i64> @spltCnstConvftoull() {
 ; P9BE-LABEL: spltCnstConvftoull:
 ; P9BE:       # %bb.0: # %entry
-; P9BE-NEXT:    addis r3, r2, .LCPI114_0@toc@ha
-; P9BE-NEXT:    addi r3, r3, .LCPI114_0@toc@l
+; P9BE-NEXT:    addis r3, r2, .LCPI110_0@toc@ha
+; P9BE-NEXT:    addi r3, r3, .LCPI110_0@toc@l
 ; P9BE-NEXT:    lxvx v2, 0, r3
 ; P9BE-NEXT:    blr
 ;
 ; P9LE-LABEL: spltCnstConvftoull:
 ; P9LE:       # %bb.0: # %entry
-; P9LE-NEXT:    addis r3, r2, .LCPI114_0@toc@ha
-; P9LE-NEXT:    addi r3, r3, .LCPI114_0@toc@l
+; P9LE-NEXT:    addis r3, r2, .LCPI110_0@toc@ha
+; P9LE-NEXT:    addi r3, r3, .LCPI110_0@toc@l
 ; P9LE-NEXT:    lxvx v2, 0, r3
 ; P9LE-NEXT:    blr
 ;
 ; P8BE-LABEL: spltCnstConvftoull:
 ; P8BE:       # %bb.0: # %entry
-; P8BE-NEXT:    addis r3, r2, .LCPI114_0@toc@ha
-; P8BE-NEXT:    addi r3, r3, .LCPI114_0@toc@l
+; P8BE-NEXT:    addis r3, r2, .LCPI110_0@toc@ha
+; P8BE-NEXT:    addi r3, r3, .LCPI110_0@toc@l
 ; P8BE-NEXT:    lxvd2x v2, 0, r3
 ; P8BE-NEXT:    blr
 ;
 ; P8LE-LABEL: spltCnstConvftoull:
 ; P8LE:       # %bb.0: # %entry
-; P8LE-NEXT:    addis r3, r2, .LCPI114_0@toc@ha
-; P8LE-NEXT:    addi r3, r3, .LCPI114_0@toc@l
+; P8LE-NEXT:    addis r3, r2, .LCPI110_0@toc@ha
+; P8LE-NEXT:    addi r3, r3, .LCPI110_0@toc@l
 ; P8LE-NEXT:    lxvd2x vs0, 0, r3
 ; P8LE-NEXT:    xxswapd v2, vs0
 ; P8LE-NEXT:    blr
@@ -5705,7 +5456,6 @@ entry:
   ret <2 x i64> <i64 4, i64 4>
 }
 
-; Function Attrs: norecurse nounwind readnone
 define <2 x i64> @fromRegsConvftoull(float %a, float %b) {
 ; P9BE-LABEL: fromRegsConvftoull:
 ; P9BE:       # %bb.0: # %entry
@@ -5746,33 +5496,32 @@ entry:
   ret <2 x i64> %vecinit2
 }
 
-; Function Attrs: norecurse nounwind readnone
 define <2 x i64> @fromDiffConstsConvftoull() {
 ; P9BE-LABEL: fromDiffConstsConvftoull:
 ; P9BE:       # %bb.0: # %entry
-; P9BE-NEXT:    addis r3, r2, .LCPI116_0@toc@ha
-; P9BE-NEXT:    addi r3, r3, .LCPI116_0@toc@l
+; P9BE-NEXT:    addis r3, r2, .LCPI112_0@toc@ha
+; P9BE-NEXT:    addi r3, r3, .LCPI112_0@toc@l
 ; P9BE-NEXT:    lxvx v2, 0, r3
 ; P9BE-NEXT:    blr
 ;
 ; P9LE-LABEL: fromDiffConstsConvftoull:
 ; P9LE:       # %bb.0: # %entry
-; P9LE-NEXT:    addis r3, r2, .LCPI116_0@toc@ha
-; P9LE-NEXT:    addi r3, r3, .LCPI116_0@toc@l
+; P9LE-NEXT:    addis r3, r2, .LCPI112_0@toc@ha
+; P9LE-NEXT:    addi r3, r3, .LCPI112_0@toc@l
 ; P9LE-NEXT:    lxvx v2, 0, r3
 ; P9LE-NEXT:    blr
 ;
 ; P8BE-LABEL: fromDiffConstsConvftoull:
 ; P8BE:       # %bb.0: # %entry
-; P8BE-NEXT:    addis r3, r2, .LCPI116_0@toc@ha
-; P8BE-NEXT:    addi r3, r3, .LCPI116_0@toc@l
+; P8BE-NEXT:    addis r3, r2, .LCPI112_0@toc@ha
+; P8BE-NEXT:    addi r3, r3, .LCPI112_0@toc@l
 ; P8BE-NEXT:    lxvd2x v2, 0, r3
 ; P8BE-NEXT:    blr
 ;
 ; P8LE-LABEL: fromDiffConstsConvftoull:
 ; P8LE:       # %bb.0: # %entry
-; P8LE-NEXT:    addis r3, r2, .LCPI116_0@toc@ha
-; P8LE-NEXT:    addi r3, r3, .LCPI116_0@toc@l
+; P8LE-NEXT:    addis r3, r2, .LCPI112_0@toc@ha
+; P8LE-NEXT:    addi r3, r3, .LCPI112_0@toc@l
 ; P8LE-NEXT:    lxvd2x vs0, 0, r3
 ; P8LE-NEXT:    xxswapd v2, vs0
 ; P8LE-NEXT:    blr
@@ -5780,7 +5529,6 @@ entry:
   ret <2 x i64> <i64 24, i64 234>
 }
 
-; Function Attrs: norecurse nounwind readonly
 define <2 x i64> @fromDiffMemConsAConvftoull(float* nocapture readonly %ptr) {
 ; P9BE-LABEL: fromDiffMemConsAConvftoull:
 ; P9BE:       # %bb.0: # %entry
@@ -5824,7 +5572,6 @@ entry:
   ret <2 x i64> %vecinit3
 }
 
-; Function Attrs: norecurse nounwind readonly
 define <2 x i64> @fromDiffMemConsDConvftoull(float* nocapture readonly %ptr) {
 ; P9BE-LABEL: fromDiffMemConsDConvftoull:
 ; P9BE:       # %bb.0: # %entry
@@ -5869,7 +5616,6 @@ entry:
   ret <2 x i64> %vecinit3
 }
 
-; Function Attrs: norecurse nounwind readonly
 define <2 x i64> @fromDiffMemVarAConvftoull(float* nocapture readonly %arr, i32 signext %elem) {
 ; P9BE-LABEL: fromDiffMemVarAConvftoull:
 ; P9BE:       # %bb.0: # %entry
@@ -5921,7 +5667,6 @@ entry:
   ret <2 x i64> %vecinit4
 }
 
-; Function Attrs: norecurse nounwind readonly
 define <2 x i64> @fromDiffMemVarDConvftoull(float* nocapture readonly %arr, i32 signext %elem) {
 ; P9BE-LABEL: fromDiffMemVarDConvftoull:
 ; P9BE:       # %bb.0: # %entry
@@ -5973,7 +5718,6 @@ entry:
   ret <2 x i64> %vecinit4
 }
 
-; Function Attrs: norecurse nounwind readnone
 define <2 x i64> @spltRegValConvftoull(float %val) {
 ; P9BE-LABEL: spltRegValConvftoull:
 ; P9BE:       # %bb.0: # %entry
@@ -6005,7 +5749,6 @@ entry:
   ret <2 x i64> %splat.splat
 }
 
-; Function Attrs: norecurse nounwind readonly
 define <2 x i64> @spltMemValConvftoull(float* nocapture readonly %ptr) {
 ; P9BE-LABEL: spltMemValConvftoull:
 ; P9BE:       # %bb.0: # %entry
@@ -6042,33 +5785,32 @@ entry:
   ret <2 x i64> %splat.splat
 }
 
-; Function Attrs: norecurse nounwind readnone
 define <2 x i64> @spltCnstConvdtoull() {
 ; P9BE-LABEL: spltCnstConvdtoull:
 ; P9BE:       # %bb.0: # %entry
-; P9BE-NEXT:    addis r3, r2, .LCPI123_0@toc@ha
-; P9BE-NEXT:    addi r3, r3, .LCPI123_0@toc@l
+; P9BE-NEXT:    addis r3, r2, .LCPI119_0@toc@ha
+; P9BE-NEXT:    addi r3, r3, .LCPI119_0@toc@l
 ; P9BE-NEXT:    lxvx v2, 0, r3
 ; P9BE-NEXT:    blr
 ;
 ; P9LE-LABEL: spltCnstConvdtoull:
 ; P9LE:       # %bb.0: # %entry
-; P9LE-NEXT:    addis r3, r2, .LCPI123_0@toc@ha
-; P9LE-NEXT:    addi r3, r3, .LCPI123_0@toc@l
+; P9LE-NEXT:    addis r3, r2, .LCPI119_0@toc@ha
+; P9LE-NEXT:    addi r3, r3, .LCPI119_0@toc@l
 ; P9LE-NEXT:    lxvx v2, 0, r3
 ; P9LE-NEXT:    blr
 ;
 ; P8BE-LABEL: spltCnstConvdtoull:
 ; P8BE:       # %bb.0: # %entry
-; P8BE-NEXT:    addis r3, r2, .LCPI123_0@toc@ha
-; P8BE-NEXT:    addi r3, r3, .LCPI123_0@toc@l
+; P8BE-NEXT:    addis r3, r2, .LCPI119_0@toc@ha
+; P8BE-NEXT:    addi r3, r3, .LCPI119_0@toc@l
 ; P8BE-NEXT:    lxvd2x v2, 0, r3
 ; P8BE-NEXT:    blr
 ;
 ; P8LE-LABEL: spltCnstConvdtoull:
 ; P8LE:       # %bb.0: # %entry
-; P8LE-NEXT:    addis r3, r2, .LCPI123_0@toc@ha
-; P8LE-NEXT:    addi r3, r3, .LCPI123_0@toc@l
+; P8LE-NEXT:    addis r3, r2, .LCPI119_0@toc@ha
+; P8LE-NEXT:    addi r3, r3, .LCPI119_0@toc@l
 ; P8LE-NEXT:    lxvd2x vs0, 0, r3
 ; P8LE-NEXT:    xxswapd v2, vs0
 ; P8LE-NEXT:    blr
@@ -6076,7 +5818,6 @@ entry:
   ret <2 x i64> <i64 4, i64 4>
 }
 
-; Function Attrs: norecurse nounwind readnone
 define <2 x i64> @fromRegsConvdtoull(double %a, double %b) {
 ; P9BE-LABEL: fromRegsConvdtoull:
 ; P9BE:       # %bb.0: # %entry
@@ -6117,33 +5858,32 @@ entry:
   ret <2 x i64> %vecinit2
 }
 
-; Function Attrs: norecurse nounwind readnone
 define <2 x i64> @fromDiffConstsConvdtoull() {
 ; P9BE-LABEL: fromDiffConstsConvdtoull:
 ; P9BE:       # %bb.0: # %entry
-; P9BE-NEXT:    addis r3, r2, .LCPI125_0@toc@ha
-; P9BE-NEXT:    addi r3, r3, .LCPI125_0@toc@l
+; P9BE-NEXT:    addis r3, r2, .LCPI121_0@toc@ha
+; P9BE-NEXT:    addi r3, r3, .LCPI121_0@toc@l
 ; P9BE-NEXT:    lxvx v2, 0, r3
 ; P9BE-NEXT:    blr
 ;
 ; P9LE-LABEL: fromDiffConstsConvdtoull:
 ; P9LE:       # %bb.0: # %entry
-; P9LE-NEXT:    addis r3, r2, .LCPI125_0@toc@ha
-; P9LE-NEXT:    addi r3, r3, .LCPI125_0@toc@l
+; P9LE-NEXT:    addis r3, r2, .LCPI121_0@toc@ha
+; P9LE-NEXT:    addi r3, r3, .LCPI121_0@toc@l
 ; P9LE-NEXT:    lxvx v2, 0, r3
 ; P9LE-NEXT:    blr
 ;
 ; P8BE-LABEL: fromDiffConstsConvdtoull:
 ; P8BE:       # %bb.0: # %entry
-; P8BE-NEXT:    addis r3, r2, .LCPI125_0@toc@ha
-; P8BE-NEXT:    addi r3, r3, .LCPI125_0@toc@l
+; P8BE-NEXT:    addis r3, r2, .LCPI121_0@toc@ha
+; P8BE-NEXT:    addi r3, r3, .LCPI121_0@toc@l
 ; P8BE-NEXT:    lxvd2x v2, 0, r3
 ; P8BE-NEXT:    blr
 ;
 ; P8LE-LABEL: fromDiffConstsConvdtoull:
 ; P8LE:       # %bb.0: # %entry
-; P8LE-NEXT:    addis r3, r2, .LCPI125_0@toc@ha
-; P8LE-NEXT:    addi r3, r3, .LCPI125_0@toc@l
+; P8LE-NEXT:    addis r3, r2, .LCPI121_0@toc@ha
+; P8LE-NEXT:    addi r3, r3, .LCPI121_0@toc@l
 ; P8LE-NEXT:    lxvd2x vs0, 0, r3
 ; P8LE-NEXT:    xxswapd v2, vs0
 ; P8LE-NEXT:    blr
@@ -6151,7 +5891,6 @@ entry:
   ret <2 x i64> <i64 24, i64 234>
 }
 
-; Function Attrs: norecurse nounwind readonly
 define <2 x i64> @fromDiffMemConsAConvdtoull(double* nocapture readonly %ptr) {
 ; P9BE-LABEL: fromDiffMemConsAConvdtoull:
 ; P9BE:       # %bb.0: # %entry
@@ -6184,7 +5923,6 @@ entry:
   ret <2 x i64> %2
 }
 
-; Function Attrs: norecurse nounwind readonly
 define <2 x i64> @fromDiffMemConsDConvdtoull(double* nocapture readonly %ptr) {
 ; P9BE-LABEL: fromDiffMemConsDConvdtoull:
 ; P9BE:       # %bb.0: # %entry
@@ -6195,8 +5933,8 @@ define <2 x i64> @fromDiffMemConsDConvdtoull(double* nocapture readonly %ptr) {
 ;
 ; P9LE-LABEL: fromDiffMemConsDConvdtoull:
 ; P9LE:       # %bb.0: # %entry
-; P9LE-NEXT:    lxv vs0, 16(r3)
-; P9LE-NEXT:    xxswapd vs0, vs0
+; P9LE-NEXT:    addi r3, r3, 16
+; P9LE-NEXT:    lxvd2x vs0, 0, r3
 ; P9LE-NEXT:    xvcvdpuxds v2, vs0
 ; P9LE-NEXT:    blr
 ;
@@ -6226,7 +5964,6 @@ entry:
   ret <2 x i64> %vecinit3
 }
 
-; Function Attrs: norecurse nounwind readonly
 define <2 x i64> @fromDiffMemVarAConvdtoull(double* nocapture readonly %arr, i32 signext %elem) {
 ; P9BE-LABEL: fromDiffMemVarAConvdtoull:
 ; P9BE:       # %bb.0: # %entry
@@ -6271,7 +6008,6 @@ entry:
   ret <2 x i64> %vecinit4
 }
 
-; Function Attrs: norecurse nounwind readonly
 define <2 x i64> @fromDiffMemVarDConvdtoull(double* nocapture readonly %arr, i32 signext %elem) {
 ; P9BE-LABEL: fromDiffMemVarDConvdtoull:
 ; P9BE:       # %bb.0: # %entry
@@ -6287,9 +6023,8 @@ define <2 x i64> @fromDiffMemVarDConvdtoull(double* nocapture readonly %arr, i32
 ; P9LE:       # %bb.0: # %entry
 ; P9LE-NEXT:    sldi r4, r4, 3
 ; P9LE-NEXT:    add r3, r3, r4
-; P9LE-NEXT:    li r4, -8
-; P9LE-NEXT:    lxvx vs0, r3, r4
-; P9LE-NEXT:    xxswapd vs0, vs0
+; P9LE-NEXT:    addi r3, r3, -8
+; P9LE-NEXT:    lxvd2x vs0, 0, r3
 ; P9LE-NEXT:    xvcvdpuxds v2, vs0
 ; P9LE-NEXT:    blr
 ;
@@ -6326,7 +6061,6 @@ entry:
   ret <2 x i64> %vecinit4
 }
 
-; Function Attrs: norecurse nounwind readnone
 define <2 x i64> @spltRegValConvdtoull(double %val) {
 ; P9BE-LABEL: spltRegValConvdtoull:
 ; P9BE:       # %bb.0: # %entry
@@ -6358,7 +6092,6 @@ entry:
   ret <2 x i64> %splat.splat
 }
 
-; Function Attrs: norecurse nounwind readonly
 define <2 x i64> @spltMemValConvdtoull(double* nocapture readonly %ptr) {
 ; P9BE-LABEL: spltMemValConvdtoull:
 ; P9BE:       # %bb.0: # %entry
diff --git a/llvm/test/CodeGen/PowerPC/builtins-ppc-p9-f128.ll b/llvm/test/CodeGen/PowerPC/builtins-ppc-p9-f128.ll
index e3ffc0f7e..366493ae7 100644
--- a/llvm/test/CodeGen/PowerPC/builtins-ppc-p9-f128.ll
+++ b/llvm/test/CodeGen/PowerPC/builtins-ppc-p9-f128.ll
@@ -112,7 +112,7 @@ entry:
   %2 = call fp128 @llvm.ppc.scalar.insert.exp.qp(fp128 %0, i64 %1)
   ret fp128 %2
 ; CHECK-LABEL: insert_exp_qp
-; CHECK-DAG: mtvsrd [[FPREG:f[0-9]+]], r3
+; CHECK-DAG: mtfprd [[FPREG:f[0-9]+]], r3
 ; CHECK-DAG: lxvx [[VECREG:v[0-9]+]]
 ; CHECK: xsiexpqp v2, [[VECREG]], [[FPREG]]
 ; CHECK: blr
diff --git a/llvm/test/CodeGen/PowerPC/check-cpu.ll b/llvm/test/CodeGen/PowerPC/check-cpu.ll
new file mode 100644
index 000000000..baa39024e
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/check-cpu.ll
@@ -0,0 +1,11 @@
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu \
+; RUN:     -mcpu=future < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu \
+; RUN:     -mcpu=future < %s | FileCheck %s
+
+
+; Test mcpu=future that should be recognized on PowerPC.
+
+; CHECK-NOT: is not a recognized processor for this target
+; CHECK:     .text
+
diff --git a/llvm/test/CodeGen/PowerPC/combine-fneg.ll b/llvm/test/CodeGen/PowerPC/combine-fneg.ll
index 864f72357..14bace2f9 100644
--- a/llvm/test/CodeGen/PowerPC/combine-fneg.ll
+++ b/llvm/test/CodeGen/PowerPC/combine-fneg.ll
@@ -5,19 +5,16 @@
 define <4 x double> @fneg_fdiv_splat(double %a0, <4 x double> %a1) {
 ; CHECK-LABEL: fneg_fdiv_splat:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addis 3, 2, .LCPI0_0@toc@ha
 ; CHECK-NEXT:    # kill: def $f1 killed $f1 def $vsl1
 ; CHECK-NEXT:    xxspltd 0, 1, 0
-; CHECK-NEXT:    addis 3, 2, .LCPI0_0@toc@ha
 ; CHECK-NEXT:    addi 3, 3, .LCPI0_0@toc@l
 ; CHECK-NEXT:    lxvd2x 1, 0, 3
-; CHECK-NEXT:    addis 3, 2, .LCPI0_1@toc@ha
 ; CHECK-NEXT:    xvredp 2, 0
-; CHECK-NEXT:    addi 3, 3, .LCPI0_1@toc@l
-; CHECK-NEXT:    xxswapd 1, 1
-; CHECK-NEXT:    xvnmsubadp 1, 2, 0
-; CHECK-NEXT:    xvmaddadp 2, 2, 1
-; CHECK-NEXT:    lxvd2x 1, 0, 3
 ; CHECK-NEXT:    xxswapd 1, 1
+; CHECK-NEXT:    xxlor 3, 1, 1
+; CHECK-NEXT:    xvmaddadp 3, 0, 2
+; CHECK-NEXT:    xvnmsubadp 2, 2, 3
 ; CHECK-NEXT:    xvmaddadp 1, 0, 2
 ; CHECK-NEXT:    xvmsubadp 2, 2, 1
 ; CHECK-NEXT:    xvmuldp 34, 34, 2
diff --git a/llvm/test/CodeGen/PowerPC/constant-combines.ll b/llvm/test/CodeGen/PowerPC/constant-combines.ll
index dd40b75d5..05f23051d 100644
--- a/llvm/test/CodeGen/PowerPC/constant-combines.ll
+++ b/llvm/test/CodeGen/PowerPC/constant-combines.ll
@@ -13,8 +13,10 @@ define void @fold_constant_stores_loaddr(i8* %i8_ptr) {
 ;
 ; LE-LABEL: fold_constant_stores_loaddr:
 ; LE:       # %bb.0: # %entry
-; LE-NEXT:    li 4, 170
+; LE-NEXT:    li 4, 0
+; LE-NEXT:    li 5, -86
 ; LE-NEXT:    std 4, 0(3)
+; LE-NEXT:    stb 5, 0(3)
 ; LE-NEXT:    blr
 entry:
   %i64_ptr = bitcast i8* %i8_ptr to i64*
@@ -35,8 +37,10 @@ define void @fold_constant_stores_hiaddr(i8* %i8_ptr) {
 ;
 ; LE-LABEL: fold_constant_stores_hiaddr:
 ; LE:       # %bb.0: # %entry
-; LE-NEXT:    li 4, 170
+; LE-NEXT:    li 4, 0
+; LE-NEXT:    li 5, -86
 ; LE-NEXT:    std 4, 0(3)
+; LE-NEXT:    stb 5, 0(3)
 ; LE-NEXT:    blr
 entry:
   %i64_ptr = bitcast i8* %i8_ptr to i64*
diff --git a/llvm/test/CodeGen/PowerPC/cr-spills.ll b/llvm/test/CodeGen/PowerPC/cr-spills.ll
index 170744679..68c31552e 100644
--- a/llvm/test/CodeGen/PowerPC/cr-spills.ll
+++ b/llvm/test/CodeGen/PowerPC/cr-spills.ll
@@ -400,5 +400,5 @@ for.end999:                                       ; preds = %for.inc997
   ret void
 }
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind }
diff --git a/llvm/test/CodeGen/PowerPC/crypto_bifs.ll b/llvm/test/CodeGen/PowerPC/crypto_bifs.ll
index ba282ec30..b34482abf 100644
--- a/llvm/test/CodeGen/PowerPC/crypto_bifs.ll
+++ b/llvm/test/CodeGen/PowerPC/crypto_bifs.ll
@@ -268,7 +268,7 @@ entry:
 ; Function Attrs: nounwind readnone
 declare <2 x i64> @llvm.ppc.altivec.crypto.vshasigmad(<2 x i64>, i32, i32) #1
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !llvm.ident = !{!0}
diff --git a/llvm/test/CodeGen/PowerPC/csr-split.ll b/llvm/test/CodeGen/PowerPC/csr-split.ll
new file mode 100644
index 000000000..0e2cf306e
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/csr-split.ll
@@ -0,0 +1,263 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -verify-machineinstrs  -ppc-vsr-nums-as-vr -ppc-asm-full-reg-names \
+; RUN:     -mtriple=powerpc64le-unknown-linux-gnu -mcpu=pwr9 < %s | FileCheck %s --check-prefix=CHECK-PWR9
+; RUN: llc -verify-machineinstrs  -ppc-vsr-nums-as-vr -ppc-asm-full-reg-names \
+; RUN:     -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr8 < %s | FileCheck %s
+
+; Check CSR split can work properly for tests below.
+
+@a = dso_local local_unnamed_addr global i32 0, align 4
+
+define dso_local signext i32 @test1(i32* %b) local_unnamed_addr  {
+; CHECK-PWR9-LABEL: test1:
+; CHECK-PWR9:       # %bb.0: # %entry
+; CHECK-PWR9-NEXT:    mflr r0
+; CHECK-PWR9-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-PWR9-NEXT:    .cfi_offset lr, 16
+; CHECK-PWR9-NEXT:    .cfi_offset r30, -16
+; CHECK-PWR9-NEXT:    std r30, -16(r1) # 8-byte Folded Spill
+; CHECK-PWR9-NEXT:    std r0, 16(r1)
+; CHECK-PWR9-NEXT:    stdu r1, -48(r1)
+; CHECK-PWR9-NEXT:    mr r30, r3
+; CHECK-PWR9-NEXT:    addis r3, r2, a@toc@ha
+; CHECK-PWR9-NEXT:    lwa r3, a@toc@l(r3)
+; CHECK-PWR9-NEXT:    cmpld r3, r30
+; CHECK-PWR9-NEXT:    # implicit-def: $r3
+; CHECK-PWR9-NEXT:    bne cr0, .LBB0_2
+; CHECK-PWR9-NEXT:  # %bb.1: # %if.then
+; CHECK-PWR9-NEXT:    bl callVoid
+; CHECK-PWR9-NEXT:    nop
+; CHECK-PWR9-NEXT:    mr r3, r30
+; CHECK-PWR9-NEXT:    bl callNonVoid
+; CHECK-PWR9-NEXT:    nop
+; CHECK-PWR9-NEXT:  .LBB0_2: # %if.end
+; CHECK-PWR9-NEXT:    extsw r3, r3
+; CHECK-PWR9-NEXT:    addi r1, r1, 48
+; CHECK-PWR9-NEXT:    ld r0, 16(r1)
+; CHECK-PWR9-NEXT:    ld r30, -16(r1) # 8-byte Folded Reload
+; CHECK-PWR9-NEXT:    mtlr r0
+; CHECK-PWR9-NEXT:    blr
+;
+; CHECK-LABEL: test1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    std r0, 16(r1)
+; CHECK-NEXT:    stdu r1, -128(r1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 128
+; CHECK-NEXT:    .cfi_offset lr, 16
+; CHECK-NEXT:    .cfi_offset r30, -16
+; CHECK-NEXT:    addis r4, r2, a@toc@ha
+; CHECK-NEXT:    std r30, 112(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    mr r30, r3
+; CHECK-NEXT:    lwa r4, a@toc@l(r4)
+; CHECK-NEXT:    cmpld r4, r3
+; CHECK-NEXT:    # implicit-def: $r3
+; CHECK-NEXT:    bne cr0, .LBB0_2
+; CHECK-NEXT:  # %bb.1: # %if.then
+; CHECK-NEXT:    bl callVoid
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mr r3, r30
+; CHECK-NEXT:    bl callNonVoid
+; CHECK-NEXT:    nop
+; CHECK-NEXT:  .LBB0_2: # %if.end
+; CHECK-NEXT:    ld r30, 112(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    extsw r3, r3
+; CHECK-NEXT:    addi r1, r1, 128
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+entry:
+  %0 = load i32, i32* @a, align 4, !tbaa !2
+  %conv = sext i32 %0 to i64
+  %1 = inttoptr i64 %conv to i32*
+  %cmp = icmp eq i32* %1, %b
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %call = tail call signext i32 bitcast (i32 (...)* @callVoid to i32 ()*)()
+  %call2 = tail call signext i32 @callNonVoid(i32* %b)
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %entry
+  %retval.0 = phi i32 [ %call2, %if.then ], [ undef, %entry ]
+  ret i32 %retval.0
+}
+
+declare signext i32 @callVoid(...) local_unnamed_addr
+
+declare signext i32 @callNonVoid(i32*) local_unnamed_addr
+
+define dso_local signext i32 @test2(i32* %p1) local_unnamed_addr  {
+; CHECK-PWR9-LABEL: test2:
+; CHECK-PWR9:       # %bb.0: # %entry
+; CHECK-PWR9-NEXT:    mflr r0
+; CHECK-PWR9-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-PWR9-NEXT:    .cfi_offset lr, 16
+; CHECK-PWR9-NEXT:    .cfi_offset r30, -16
+; CHECK-PWR9-NEXT:    std r30, -16(r1) # 8-byte Folded Spill
+; CHECK-PWR9-NEXT:    std r0, 16(r1)
+; CHECK-PWR9-NEXT:    stdu r1, -48(r1)
+; CHECK-PWR9-NEXT:    mr r30, r3
+; CHECK-PWR9-NEXT:    li r3, 0
+; CHECK-PWR9-NEXT:    cmpldi r30, 0
+; CHECK-PWR9-NEXT:    beq cr0, .LBB1_3
+; CHECK-PWR9-NEXT:  # %bb.1: # %if.end
+; CHECK-PWR9-NEXT:    addis r4, r2, a@toc@ha
+; CHECK-PWR9-NEXT:    lwa r4, a@toc@l(r4)
+; CHECK-PWR9-NEXT:    cmpld r4, r30
+; CHECK-PWR9-NEXT:    bne cr0, .LBB1_3
+; CHECK-PWR9-NEXT:  # %bb.2: # %if.then2
+; CHECK-PWR9-NEXT:    bl callVoid
+; CHECK-PWR9-NEXT:    nop
+; CHECK-PWR9-NEXT:    mr r3, r30
+; CHECK-PWR9-NEXT:    bl callNonVoid
+; CHECK-PWR9-NEXT:    nop
+; CHECK-PWR9-NEXT:  .LBB1_3: # %return
+; CHECK-PWR9-NEXT:    extsw r3, r3
+; CHECK-PWR9-NEXT:    addi r1, r1, 48
+; CHECK-PWR9-NEXT:    ld r0, 16(r1)
+; CHECK-PWR9-NEXT:    ld r30, -16(r1) # 8-byte Folded Reload
+; CHECK-PWR9-NEXT:    mtlr r0
+; CHECK-PWR9-NEXT:    blr
+;
+; CHECK-LABEL: test2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    std r0, 16(r1)
+; CHECK-NEXT:    stdu r1, -128(r1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 128
+; CHECK-NEXT:    .cfi_offset lr, 16
+; CHECK-NEXT:    .cfi_offset r30, -16
+; CHECK-NEXT:    std r30, 112(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    mr r30, r3
+; CHECK-NEXT:    cmpldi r3, 0
+; CHECK-NEXT:    li r3, 0
+; CHECK-NEXT:    beq cr0, .LBB1_3
+; CHECK-NEXT:  # %bb.1: # %if.end
+; CHECK-NEXT:    addis r4, r2, a@toc@ha
+; CHECK-NEXT:    lwa r4, a@toc@l(r4)
+; CHECK-NEXT:    cmpld r4, r30
+; CHECK-NEXT:    bne cr0, .LBB1_3
+; CHECK-NEXT:  # %bb.2: # %if.then2
+; CHECK-NEXT:    bl callVoid
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mr r3, r30
+; CHECK-NEXT:    bl callNonVoid
+; CHECK-NEXT:    nop
+; CHECK-NEXT:  .LBB1_3: # %return
+; CHECK-NEXT:    ld r30, 112(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    extsw r3, r3
+; CHECK-NEXT:    addi r1, r1, 128
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+entry:
+  %tobool = icmp eq i32* %p1, null
+  br i1 %tobool, label %return, label %if.end
+
+if.end:                                           ; preds = %entry
+  %0 = load i32, i32* @a, align 4, !tbaa !2
+  %conv = sext i32 %0 to i64
+  %1 = inttoptr i64 %conv to i32*
+  %cmp = icmp eq i32* %1, %p1
+  br i1 %cmp, label %if.then2, label %return
+
+if.then2:                                         ; preds = %if.end
+  %call = tail call signext i32 bitcast (i32 (...)* @callVoid to i32 ()*)()
+  %call3 = tail call signext i32 @callNonVoid(i32* nonnull %p1)
+  br label %return
+
+return:                                           ; preds = %if.end, %entry, %if.then2
+  %retval.0 = phi i32 [ %call3, %if.then2 ], [ 0, %entry ], [ 0, %if.end ]
+  ret i32 %retval.0
+}
+
+
+define dso_local i8* @test3(i8** nocapture %p1, i8 zeroext %p2) local_unnamed_addr  {
+; CHECK-PWR9-LABEL: test3:
+; CHECK-PWR9:       # %bb.0: # %entry
+; CHECK-PWR9-NEXT:    mflr r0
+; CHECK-PWR9-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-PWR9-NEXT:    .cfi_offset lr, 16
+; CHECK-PWR9-NEXT:    .cfi_offset r29, -24
+; CHECK-PWR9-NEXT:    .cfi_offset r30, -16
+; CHECK-PWR9-NEXT:    std r29, -24(r1) # 8-byte Folded Spill
+; CHECK-PWR9-NEXT:    std r30, -16(r1) # 8-byte Folded Spill
+; CHECK-PWR9-NEXT:    std r0, 16(r1)
+; CHECK-PWR9-NEXT:    stdu r1, -64(r1)
+; CHECK-PWR9-NEXT:    ld r30, 0(r3)
+; CHECK-PWR9-NEXT:    cmpldi r30, 0
+; CHECK-PWR9-NEXT:    beq cr0, .LBB2_2
+; CHECK-PWR9-NEXT:  # %bb.1: # %land.rhs
+; CHECK-PWR9-NEXT:    mr r29, r3
+; CHECK-PWR9-NEXT:    clrldi r4, r4, 32
+; CHECK-PWR9-NEXT:    mr r3, r30
+; CHECK-PWR9-NEXT:    bl bar
+; CHECK-PWR9-NEXT:    nop
+; CHECK-PWR9-NEXT:    std r3, 0(r29)
+; CHECK-PWR9-NEXT:  .LBB2_2: # %land.end
+; CHECK-PWR9-NEXT:    mr r3, r30
+; CHECK-PWR9-NEXT:    addi r1, r1, 64
+; CHECK-PWR9-NEXT:    ld r0, 16(r1)
+; CHECK-PWR9-NEXT:    ld r30, -16(r1) # 8-byte Folded Reload
+; CHECK-PWR9-NEXT:    ld r29, -24(r1) # 8-byte Folded Reload
+; CHECK-PWR9-NEXT:    mtlr r0
+; CHECK-PWR9-NEXT:    blr
+;
+; CHECK-LABEL: test3:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    std r0, 16(r1)
+; CHECK-NEXT:    stdu r1, -144(r1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 144
+; CHECK-NEXT:    .cfi_offset lr, 16
+; CHECK-NEXT:    .cfi_offset r29, -24
+; CHECK-NEXT:    .cfi_offset r30, -16
+; CHECK-NEXT:    std r30, 128(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    ld r30, 0(r3)
+; CHECK-NEXT:    std r29, 120(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    cmpldi r30, 0
+; CHECK-NEXT:    beq cr0, .LBB2_2
+; CHECK-NEXT:  # %bb.1: # %land.rhs
+; CHECK-NEXT:    mr r29, r3
+; CHECK-NEXT:    clrldi r4, r4, 32
+; CHECK-NEXT:    mr r3, r30
+; CHECK-NEXT:    bl bar
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    std r3, 0(r29)
+; CHECK-NEXT:  .LBB2_2: # %land.end
+; CHECK-NEXT:    mr r3, r30
+; CHECK-NEXT:    ld r30, 128(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r29, 120(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    addi r1, r1, 144
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+entry:
+  %0 = load i8*, i8** %p1, align 8, !tbaa !6
+  %tobool = icmp eq i8* %0, null
+  br i1 %tobool, label %land.end, label %land.rhs
+
+land.rhs:                                         ; preds = %entry
+  %call = tail call i8* @bar(i8* nonnull %0, i8 zeroext %p2)
+  store i8* %call, i8** %p1, align 8, !tbaa !6
+  br label %land.end
+
+land.end:                                         ; preds = %entry, %land.rhs
+  ret i8* %0
+}
+
+declare i8* @bar(i8*, i8 zeroext) local_unnamed_addr
+
+
+!llvm.module.flags = !{!0}
+!llvm.ident = !{!1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{!"clang version 10.0.0 (trunk 367381) (llvm/trunk 367388)"}
+!2 = !{!3, !3, i64 0}
+!3 = !{!"int", !4, i64 0}
+!4 = !{!"omnipotent char", !5, i64 0}
+!5 = !{!"Simple C/C++ TBAA"}
+!6 = !{!7, !7, i64 0}
+!7 = !{!"any pointer", !4, i64 0}
diff --git a/llvm/test/CodeGen/PowerPC/ctr-cleanup.ll b/llvm/test/CodeGen/PowerPC/ctr-cleanup.ll
index 0824bb276..9e13b6f75 100644
--- a/llvm/test/CodeGen/PowerPC/ctr-cleanup.ll
+++ b/llvm/test/CodeGen/PowerPC/ctr-cleanup.ll
@@ -22,4 +22,4 @@ for.end:                                          ; preds = %for.body, %entry
   ret void
 }
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/PowerPC/ctr-minmaxnum.ll b/llvm/test/CodeGen/PowerPC/ctr-minmaxnum.ll
index a160074ba..ed3c9f07c 100644
--- a/llvm/test/CodeGen/PowerPC/ctr-minmaxnum.ll
+++ b/llvm/test/CodeGen/PowerPC/ctr-minmaxnum.ll
@@ -36,8 +36,8 @@ loop_exit:
 
 ; CHECK-LABEL: test1:
 ; CHECK-NOT: mtctr
-; CHECK: bl fminf
-; CHECK-NOT: bl fminf
+; CHECK: xsmindp
+; CHECK-NOT: xsmindp
 ; CHECK-NOT: mtctr
 ; CHECK: blr
 
@@ -59,9 +59,9 @@ loop_exit:
 
 ; CHECK-LABEL: test1v:
 ; CHECK: xvminsp
-; CHECK-NOT: bl fminf
+; CHECK-NOT: xsmindp
 ; CHECK: mtctr
-; CHECK-NOT: bl fminf
+; CHECK-NOT: xsmindp
 ; CHECK: blr
 
 ; QPX-LABEL: test1v:
@@ -87,8 +87,8 @@ loop_exit:
 
 ; CHECK-LABEL: test1a:
 ; CHECK-NOT: mtctr
-; CHECK: bl fminf
-; CHECK-NOT: bl fminf
+; CHECK: xsmindp
+; CHECK-NOT: xsmindp
 ; CHECK-NOT: mtctr
 ; CHECK: blr
 
@@ -110,8 +110,8 @@ loop_exit:
 
 ; CHECK-LABEL: test2:
 ; CHECK-NOT: mtctr
-; CHECK: bl fmaxf
-; CHECK-NOT: bl fmaxf
+; CHECK: xsmaxdp
+; CHECK-NOT: xsmaxdp
 ; CHECK-NOT: mtctr
 ; CHECK: blr
 
@@ -134,9 +134,9 @@ loop_exit:
 ; CHECK-LABEL: test2v:
 ; CHECK: xvmaxdp
 ; CHECK: xvmaxdp
-; CHECK-NOT: bl fmax
+; CHECK-NOT: xsmaxdp
 ; CHECK: mtctr
-; CHECK-NOT: bl fmax
+; CHECK-NOT: xsmaxdp
 ; CHECK: blr
 
 ; QPX-LABEL: test2v:
@@ -162,8 +162,8 @@ loop_exit:
 
 ; CHECK-LABEL: test2a:
 ; CHECK-NOT: mtctr
-; CHECK: bl fmaxf
-; CHECK-NOT: bl fmaxf
+; CHECK: xsmaxdp
+; CHECK-NOT: xsmaxdp
 ; CHECK-NOT: mtctr
 ; CHECK: blr
 
@@ -185,8 +185,8 @@ loop_exit:
 
 ; CHECK-LABEL: test3:
 ; CHECK-NOT: mtctr
-; CHECK: bl fmin
-; CHECK-NOT: bl fmin
+; CHECK: xsmindp
+; CHECK-NOT: xsmindp
 ; CHECK-NOT: mtctr
 ; CHECK: blr
 
@@ -208,8 +208,8 @@ loop_exit:
 
 ; CHECK-LABEL: test3a:
 ; CHECK-NOT: mtctr
-; CHECK: bl fmin
-; CHECK-NOT: bl fmin
+; CHECK: xsmindp
+; CHECK-NOT: xsmindp
 ; CHECK-NOT: mtctr
 ; CHECK: blr
 
@@ -231,8 +231,8 @@ loop_exit:
 
 ; CHECK-LABEL: test4:
 ; CHECK-NOT: mtctr
-; CHECK: bl fmax
-; CHECK-NOT: bl fmax
+; CHECK: xsmaxdp
+; CHECK-NOT: xsmaxdp
 ; CHECK-NOT: mtctr
 ; CHECK: blr
 
@@ -254,8 +254,8 @@ loop_exit:
 
 ; CHECK-LABEL: test4a:
 ; CHECK-NOT: mtctr
-; CHECK: bl fmax
-; CHECK-NOT: bl fmax
+; CHECK: xsmaxdp
+; CHECK-NOT: xsmaxdp
 ; CHECK-NOT: mtctr
 ; CHECK: blr
 
diff --git a/llvm/test/CodeGen/PowerPC/ctrloop-intrin.ll b/llvm/test/CodeGen/PowerPC/ctrloop-intrin.ll
index 9e18280c5..876b96bd6 100644
--- a/llvm/test/CodeGen/PowerPC/ctrloop-intrin.ll
+++ b/llvm/test/CodeGen/PowerPC/ctrloop-intrin.ll
@@ -330,7 +330,7 @@ declare void @llvm.set.loop.iterations.i64(i64) #0
 declare i64 @llvm.loop.dec(i64, i64) #0
 
 attributes #0 = { nounwind }
-attributes #1 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind }
 attributes #3 = { nounwind }
 
diff --git a/llvm/test/CodeGen/PowerPC/ctrloop-le.ll b/llvm/test/CodeGen/PowerPC/ctrloop-le.ll
index 9fad3cb15..43105a57c 100644
--- a/llvm/test/CodeGen/PowerPC/ctrloop-le.ll
+++ b/llvm/test/CodeGen/PowerPC/ctrloop-le.ll
@@ -2,9 +2,6 @@
 target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v128:128:128-n32:64"
 target triple = "powerpc64-unknown-linux-gnu"
 
-; XFAIL: *
-; SE needs improvement
-
 ; CHECK: test_pos1_ir_sle
 ; CHECK: bdnz
 ; a < b
diff --git a/llvm/test/CodeGen/PowerPC/ctrloop-lt.ll b/llvm/test/CodeGen/PowerPC/ctrloop-lt.ll
index b1ef85860..260b6cb60 100644
--- a/llvm/test/CodeGen/PowerPC/ctrloop-lt.ll
+++ b/llvm/test/CodeGen/PowerPC/ctrloop-lt.ll
@@ -2,9 +2,6 @@
 target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v128:128:128-n32:64"
 target triple = "powerpc64-unknown-linux-gnu"
 
-; XFAIL: *
-; SE needs improvement
-
 ; CHECK: test_pos1_ir_slt
 ; CHECK: bdnz
 ; a < b
@@ -35,7 +32,6 @@ for.end:                                          ; preds = %for.body, %entry
 
 
 ; CHECK: test_pos2_ir_slt
-; FIXME: Support this loop!
 ; CHECK: bdnz
 ; a < b
 define void @test_pos2_ir_slt(i8* nocapture %p, i32 %a, i32 %b) nounwind {
@@ -65,7 +61,6 @@ for.end:                                          ; preds = %for.body, %entry
 
 
 ; CHECK: test_pos4_ir_slt
-; FIXME: Support this loop!
 ; CHECK: bdnz
 ; a < b
 define void @test_pos4_ir_slt(i8* nocapture %p, i32 %a, i32 %b) nounwind {
diff --git a/llvm/test/CodeGen/PowerPC/ctrloop-shortLoops.ll b/llvm/test/CodeGen/PowerPC/ctrloop-shortLoops.ll
index 4f904d6fe..44acfcdd6 100644
--- a/llvm/test/CodeGen/PowerPC/ctrloop-shortLoops.ll
+++ b/llvm/test/CodeGen/PowerPC/ctrloop-shortLoops.ll
@@ -4,13 +4,13 @@
 ; Verify that we do NOT generate the mtctr instruction for loop trip counts < 4
 ; The latency of the mtctr is only justified if there are more than 4 comparisons that are removed as a result.
 
-@a = common local_unnamed_addr global i32 0, align 4
-@b = common local_unnamed_addr global i32 0, align 4
-@c = common local_unnamed_addr global i32 0, align 4
-@d = common local_unnamed_addr global i32 0, align 4
-@e = common local_unnamed_addr global i32 0, align 4
-@f = common local_unnamed_addr global i32 0, align 4
-@arr = common local_unnamed_addr global [5 x i32] zeroinitializer, align 4
+@a = local_unnamed_addr global i32 0, align 4
+@b = local_unnamed_addr global i32 0, align 4
+@c = local_unnamed_addr global i32 0, align 4
+@d = local_unnamed_addr global i32 0, align 4
+@e = local_unnamed_addr global i32 0, align 4
+@f = local_unnamed_addr global i32 0, align 4
+@arr = local_unnamed_addr global [5 x i32] zeroinitializer, align 4
 
 ; Function Attrs: norecurse nounwind readonly
 define signext i32 @testTripCount2(i32 signext %a) {
diff --git a/llvm/test/CodeGen/PowerPC/cxx_tlscc64.ll b/llvm/test/CodeGen/PowerPC/cxx_tlscc64.ll
index cfa243f20..d17f4c952 100644
--- a/llvm/test/CodeGen/PowerPC/cxx_tlscc64.ll
+++ b/llvm/test/CodeGen/PowerPC/cxx_tlscc64.ll
@@ -40,4 +40,4 @@ define cxx_fast_tlscc i32* @_ZTW4sum2() #0 {
   ret i32* @sum1
 }
 
-attributes #0 = { nounwind "no-frame-pointer-elim"="true" }
+attributes #0 = { nounwind "frame-pointer"="all" }
diff --git a/llvm/test/CodeGen/PowerPC/dform-adjust.ll b/llvm/test/CodeGen/PowerPC/dform-adjust.ll
new file mode 100644
index 000000000..e34c4b81e
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/dform-adjust.ll
@@ -0,0 +1,125 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu \
+; RUN:   -mcpu=pwr9 < %s | FileCheck %s
+define dso_local i64 @test1(i8* nocapture readonly %p, i32 signext %count) local_unnamed_addr #0 {
+; CHECK-LABEL: test1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    li 5, -13
+; CHECK-NEXT:    lxvx 0, 3, 5
+; CHECK-NEXT:    li 5, 19
+; CHECK-NEXT:    lxvx 1, 3, 5
+; CHECK-NEXT:    li 5, 3
+; CHECK-NEXT:    li 6, 7
+; CHECK-NEXT:    li 7, 11
+; CHECK-NEXT:    li 8, 15
+; CHECK-NEXT:    mfvsrld 9, 0
+; CHECK-NEXT:    ldx 5, 3, 5
+; CHECK-NEXT:    ldx 6, 3, 6
+; CHECK-NEXT:    ldx 7, 3, 7
+; CHECK-NEXT:    ldx 3, 3, 8
+; CHECK-NEXT:    mffprd 8, 0
+; CHECK-NEXT:    mfvsrld 10, 1
+; CHECK-NEXT:    mfvsrd 11, 1
+; CHECK-NEXT:    mulld 8, 9, 8
+; CHECK-NEXT:    mulld 5, 8, 5
+; CHECK-NEXT:    mulld 5, 5, 10
+; CHECK-NEXT:    mulld 5, 5, 11
+; CHECK-NEXT:    mulld 5, 5, 6
+; CHECK-NEXT:    mulld 5, 5, 7
+; CHECK-NEXT:    maddld 3, 5, 3, 4
+; CHECK-NEXT:    blr
+entry:
+  %add.ptr = getelementptr inbounds i8, i8* %p, i64 -13
+  %0 = bitcast i8* %add.ptr to <2 x i64>*
+  %1 = load <2 x i64>, <2 x i64>* %0, align 16
+  %add.ptr1 = getelementptr inbounds i8, i8* %p, i64 19
+  %2 = bitcast i8* %add.ptr1 to <2 x i64>*
+  %3 = load <2 x i64>, <2 x i64>* %2, align 16
+  %add.ptr3 = getelementptr inbounds i8, i8* %p, i64 3
+  %4 = bitcast i8* %add.ptr3 to i64*
+  %5 = load i64, i64* %4, align 8
+  %add.ptr5 = getelementptr inbounds i8, i8* %p, i64 7
+  %6 = bitcast i8* %add.ptr5 to i64*
+  %7 = load i64, i64* %6, align 8
+  %add.ptr7 = getelementptr inbounds i8, i8* %p, i64 11
+  %8 = bitcast i8* %add.ptr7 to i64*
+  %9 = load i64, i64* %8, align 8
+  %add.ptr9 = getelementptr inbounds i8, i8* %p, i64 15
+  %10 = bitcast i8* %add.ptr9 to i64*
+  %11 = load i64, i64* %10, align 8
+  %vecext = extractelement <2 x i64> %1, i32 1
+  %vecext13 = extractelement <2 x i64> %1, i32 0
+  %vecext15 = extractelement <2 x i64> %3, i32 0
+  %vecext17 = extractelement <2 x i64> %3, i32 1
+  %mul = mul i64 %vecext13, %vecext
+  %mul10 = mul i64 %mul, %5
+  %mul11 = mul i64 %mul10, %vecext15
+  %mul12 = mul i64 %mul11, %vecext17
+  %mul14 = mul i64 %mul12, %7
+  %mul16 = mul i64 %mul14, %9
+  %mul18 = mul i64 %mul16, %11
+  %conv = sext i32 %count to i64
+  %add19 = add i64 %mul18, %conv
+  ret i64 %add19
+}
+
+define dso_local i64 @test2(i8* nocapture readonly %p, i32 signext %count) local_unnamed_addr #0 {
+; CHECK-LABEL: test2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    li 5, 0
+; CHECK-NEXT:    ori 6, 5, 40009
+; CHECK-NEXT:    ori 7, 5, 40001
+; CHECK-NEXT:    ori 5, 5, 40005
+; CHECK-NEXT:    ldx 6, 3, 6
+; CHECK-NEXT:    ldx 7, 3, 7
+; CHECK-NEXT:    ldx 3, 3, 5
+; CHECK-NEXT:    mulld 5, 7, 6
+; CHECK-NEXT:    maddld 3, 5, 3, 4
+; CHECK-NEXT:    blr
+entry:
+  %add.ptr = getelementptr inbounds i8, i8* %p, i64 40009
+  %0 = bitcast i8* %add.ptr to i64*
+  %1 = load i64, i64* %0, align 8
+  %add.ptr2 = getelementptr inbounds i8, i8* %p, i64 40001
+  %2 = bitcast i8* %add.ptr2 to i64*
+  %3 = load i64, i64* %2, align 8
+  %add.ptr4 = getelementptr inbounds i8, i8* %p, i64 40005
+  %4 = bitcast i8* %add.ptr4 to i64*
+  %5 = load i64, i64* %4, align 8
+  %mul = mul i64 %3, %1
+  %mul5 = mul i64 %mul, %5
+  %conv = sext i32 %count to i64
+  %add6 = add i64 %mul5, %conv
+  ret i64 %add6
+}
+
+define dso_local i64 @test3(i8* nocapture readonly %p, i32 signext %count) local_unnamed_addr {
+; CHECK-LABEL: test3:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lis 5, 1
+; CHECK-NEXT:    ori 6, 5, 14497
+; CHECK-NEXT:    ori 7, 5, 14465
+; CHECK-NEXT:    ori 5, 5, 14481
+; CHECK-NEXT:    ldx 6, 3, 6
+; CHECK-NEXT:    ldx 7, 3, 7
+; CHECK-NEXT:    ldx 3, 3, 5
+; CHECK-NEXT:    mulld 5, 7, 6
+; CHECK-NEXT:    maddld 3, 5, 3, 4
+; CHECK-NEXT:    blr
+entry:
+  %add.ptr = getelementptr inbounds i8, i8* %p, i64 80033
+  %0 = bitcast i8* %add.ptr to i64*
+  %1 = load i64, i64* %0, align 8
+  %add.ptr2 = getelementptr inbounds i8, i8* %p, i64 80001
+  %2 = bitcast i8* %add.ptr2 to i64*
+  %3 = load i64, i64* %2, align 8
+  %add.ptr4 = getelementptr inbounds i8, i8* %p, i64 80017
+  %4 = bitcast i8* %add.ptr4 to i64*
+  %5 = load i64, i64* %4, align 8
+  %mul = mul i64 %3, %1
+  %mul5 = mul i64 %mul, %5
+  %conv = sext i32 %count to i64
+  %add6 = add i64 %mul5, %conv
+  ret i64 %add6
+}
+
diff --git a/llvm/test/CodeGen/PowerPC/direct-move-profit.ll b/llvm/test/CodeGen/PowerPC/direct-move-profit.ll
index 2e2da7ce8..7fb05a351 100644
--- a/llvm/test/CodeGen/PowerPC/direct-move-profit.ll
+++ b/llvm/test/CodeGen/PowerPC/direct-move-profit.ll
@@ -69,7 +69,7 @@ entry:
   store i32 %add, i32* %arrayidx6, align 4, !tbaa !1
   ret void
 
-; CHECK: mtvsrwa
+; CHECK: mtfprwa
 ; CHECK: blr
 
 }
diff --git a/llvm/test/CodeGen/PowerPC/div-e-32.ll b/llvm/test/CodeGen/PowerPC/div-e-32.ll
index d5d6c24aa..fe64abbe6 100644
--- a/llvm/test/CodeGen/PowerPC/div-e-32.ll
+++ b/llvm/test/CodeGen/PowerPC/div-e-32.ll
@@ -23,7 +23,7 @@ entry:
 ; Function Attrs: nounwind readnone
 declare i32 @llvm.ppc.divweu(i32, i32) #1
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !llvm.ident = !{!0}
diff --git a/llvm/test/CodeGen/PowerPC/div-e-all.ll b/llvm/test/CodeGen/PowerPC/div-e-all.ll
index a62997aca..462086bd2 100644
--- a/llvm/test/CodeGen/PowerPC/div-e-all.ll
+++ b/llvm/test/CodeGen/PowerPC/div-e-all.ll
@@ -46,7 +46,7 @@ entry:
 ; Function Attrs: nounwind readnone
 declare i64 @llvm.ppc.divdeu(i64, i64) #1
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !llvm.ident = !{!0}
diff --git a/llvm/test/CodeGen/PowerPC/ec-input.ll b/llvm/test/CodeGen/PowerPC/ec-input.ll
index 6ef93d268..9a1c12169 100644
--- a/llvm/test/CodeGen/PowerPC/ec-input.ll
+++ b/llvm/test/CodeGen/PowerPC/ec-input.ll
@@ -148,8 +148,8 @@ if.end23.3:                                       ; preds = %if.end14.3
   br i1 %cmp, label %for.cond2.preheader, label %for.end38
 }
 
-attributes #0 = { nounwind "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "target-cpu"="a2q" }
-attributes #1 = { inlinehint nounwind "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "target-cpu"="a2q" }
+attributes #0 = { nounwind "frame-pointer"="all" "target-cpu"="a2q" }
+attributes #1 = { inlinehint nounwind "frame-pointer"="all" "target-cpu"="a2q" }
 attributes #2 = { nounwind }
 attributes #3 = { cold nounwind }
 
diff --git a/llvm/test/CodeGen/PowerPC/eh-dwarf-cfa.ll b/llvm/test/CodeGen/PowerPC/eh-dwarf-cfa.ll
index b8f6a5968..3d492000b 100644
--- a/llvm/test/CodeGen/PowerPC/eh-dwarf-cfa.ll
+++ b/llvm/test/CodeGen/PowerPC/eh-dwarf-cfa.ll
@@ -23,6 +23,6 @@ declare void @_Z1gPv(i8*)
 ; Function Attrs: nounwind
 declare i8* @llvm.eh.dwarf.cfa(i32) #1
 
-attributes #0 = { "no-frame-pointer-elim"="true" "target-cpu"="ppc64le" }
+attributes #0 = { "frame-pointer"="all" "target-cpu"="ppc64le" }
 attributes #1 = { nounwind }
 
diff --git a/llvm/test/CodeGen/PowerPC/elf-common.ll b/llvm/test/CodeGen/PowerPC/elf-common.ll
new file mode 100644
index 000000000..cc73d9b58
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/elf-common.ll
@@ -0,0 +1,45 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -mcpu=pwr8 -O0 < %s \
+; RUN:    | FileCheck -check-prefix=NOOPT %s
+; RUN: llc -relocation-model=static -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu \
+; RUN:     -mcpu=pwr8 < %s | FileCheck -check-prefix=STATIC %s
+; RUN: llc -relocation-model=pic -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu \
+; RUN:     -mcpu=pwr8 < %s | FileCheck -check-prefix=PIC %s
+
+; Test correct code generation for static and pic for loading and storing a common symbol 
+
+@comm_glob = common global i32 0, align 4
+
+define signext i32 @test_comm() nounwind {
+; NOOPT-LABEL: test_comm:
+; NOOPT:       # %bb.0: # %entry
+; NOOPT-NEXT:    addis 3, 2, comm_glob@toc@ha
+; NOOPT-NEXT:    addi 3, 3, comm_glob@toc@l
+; NOOPT-NEXT:    lwz 4, 0(3)
+; NOOPT-NEXT:    addi 5, 4, 1
+; NOOPT-NEXT:    stw 5, 0(3)
+; NOOPT-NEXT:    extsw 3, 4
+; NOOPT-NEXT:    blr
+;
+; STATIC-LABEL: test_comm:
+; STATIC:       # %bb.0: # %entry
+; STATIC-NEXT:    addis 4, 2, comm_glob@toc@ha
+; STATIC-NEXT:    lwa 3, comm_glob@toc@l(4)
+; STATIC-NEXT:    addi 5, 3, 1
+; STATIC-NEXT:    stw 5, comm_glob@toc@l(4)
+; STATIC-NEXT:    blr
+;
+; PIC-LABEL: test_comm:
+; PIC:       # %bb.0: # %entry
+; PIC-NEXT:    addis 3, 2, .LC0@toc@ha
+; PIC-NEXT:    ld 4, .LC0@toc@l(3)
+; PIC-NEXT:    lwa 3, 0(4)
+; PIC-NEXT:    addi 5, 3, 1
+; PIC-NEXT:    stw 5, 0(4)
+; PIC-NEXT:    blr
+entry:
+  %0 = load i32, i32* @comm_glob, align 4
+  %inc = add nsw i32 %0, 1
+  store i32 %inc, i32* @comm_glob, align 4
+  ret i32 %0
+}
diff --git a/llvm/test/CodeGen/PowerPC/expand-contiguous-isel.ll b/llvm/test/CodeGen/PowerPC/expand-contiguous-isel.ll
index 86caedcc0..1acb3e17a 100644
--- a/llvm/test/CodeGen/PowerPC/expand-contiguous-isel.ll
+++ b/llvm/test/CodeGen/PowerPC/expand-contiguous-isel.ll
@@ -137,6 +137,7 @@ _ZNK4llvm9StringRef6substrEmm.exit:
 ; CHECK: bc 12, eq, [[TRUE:.LBB[0-9]+]]
 ; CHECK-NEXT: b [[SUCCESSOR:.LBB[0-9]+]]
 ; CHECK-NEXT: [[TRUE]]
+; CHECK-NEXT: # in Loop: Header
 ; CHECK-NEXT: addi {{r[0-9]+}}, {{r[0-9]+}}, 0
 ; CHECK-NEXT: [[SUCCESSOR]]
 }
diff --git a/llvm/test/CodeGen/PowerPC/expand-isel.ll b/llvm/test/CodeGen/PowerPC/expand-isel.ll
index d9b571eda..1d1c00e5f 100644
--- a/llvm/test/CodeGen/PowerPC/expand-isel.ll
+++ b/llvm/test/CodeGen/PowerPC/expand-isel.ll
@@ -169,8 +169,8 @@ entry:
 }
 
 
-@b = common local_unnamed_addr global i32 0, align 4
-@a = common local_unnamed_addr global i32 0, align 4
+@b = local_unnamed_addr global i32 0, align 4
+@a = local_unnamed_addr global i32 0, align 4
 ; Function Attrs: norecurse nounwind readonly
 define signext i32 @testComplexISEL() #0 {
 entry:
diff --git a/llvm/test/CodeGen/PowerPC/extra-toc-reg-deps.ll b/llvm/test/CodeGen/PowerPC/extra-toc-reg-deps.ll
index 10ad8dd88..e066b45d3 100644
--- a/llvm/test/CodeGen/PowerPC/extra-toc-reg-deps.ll
+++ b/llvm/test/CodeGen/PowerPC/extra-toc-reg-deps.ll
@@ -419,8 +419,8 @@ declare void @_ZN4Foam11regIOobjectD2Ev() #0
 
 declare void @_ZN4Foam6reduceIiNS_5sumOpIiEEEEvRKNS_4ListINS_8UPstream11commsStructEEERT_RKT0_ii() #0
 
-attributes #0 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="a2q" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { inlinehint "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="a2q" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="a2q" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { inlinehint "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="a2q" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.module.flags = !{!0}
 
diff --git a/llvm/test/CodeGen/PowerPC/extract-and-store.ll b/llvm/test/CodeGen/PowerPC/extract-and-store.ll
index 9668230e8..2731ffd07 100644
--- a/llvm/test/CodeGen/PowerPC/extract-and-store.ll
+++ b/llvm/test/CodeGen/PowerPC/extract-and-store.ll
@@ -484,8 +484,8 @@ define dso_local void @test_consecutive_i32(<4 x i32> %a, i32* nocapture %b) loc
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    xxsldwi vs0, vs34, vs34, 2
 ; CHECK-NEXT:    li r3, 4
-; CHECK-NEXT:    stfiwx f0, 0, r5
 ; CHECK-NEXT:    stxsiwx vs34, r5, r3
+; CHECK-NEXT:    stfiwx f0, 0, r5
 ; CHECK-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test_consecutive_i32:
@@ -501,8 +501,8 @@ define dso_local void @test_consecutive_i32(<4 x i32> %a, i32* nocapture %b) loc
 ; CHECK-P9:       # %bb.0: # %entry
 ; CHECK-P9-NEXT:    xxsldwi vs0, vs34, vs34, 2
 ; CHECK-P9-NEXT:    li r3, 4
-; CHECK-P9-NEXT:    stfiwx f0, 0, r5
 ; CHECK-P9-NEXT:    stxsiwx vs34, r5, r3
+; CHECK-P9-NEXT:    stfiwx f0, 0, r5
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-P9-BE-LABEL: test_consecutive_i32:
@@ -590,8 +590,8 @@ define dso_local void @test_stores_exceed_vec_size(<4 x i32> %a, i32* nocapture
 ; CHECK-BE-NEXT:    li r4, 20
 ; CHECK-BE-NEXT:    stxsiwx vs34, r5, r3
 ; CHECK-BE-NEXT:    xxsldwi vs0, vs34, vs0, 2
-; CHECK-BE-NEXT:    stxvw4x vs0, 0, r5
 ; CHECK-BE-NEXT:    stfiwx f1, r5, r4
+; CHECK-BE-NEXT:    stxvw4x vs0, 0, r5
 ; CHECK-BE-NEXT:    blr
 ;
 ; CHECK-P9-LABEL: test_stores_exceed_vec_size:
@@ -599,13 +599,13 @@ define dso_local void @test_stores_exceed_vec_size(<4 x i32> %a, i32* nocapture
 ; CHECK-P9-NEXT:    addis r3, r2, .LCPI16_0@toc@ha
 ; CHECK-P9-NEXT:    addi r3, r3, .LCPI16_0@toc@l
 ; CHECK-P9-NEXT:    lxvx vs35, 0, r3
-; CHECK-P9-NEXT:    li r3, 16
-; CHECK-P9-NEXT:    vperm v3, v2, v2, v3
 ; CHECK-P9-NEXT:    xxsldwi vs0, vs34, vs34, 1
-; CHECK-P9-NEXT:    stxv vs35, 0(r5)
+; CHECK-P9-NEXT:    li r3, 16
 ; CHECK-P9-NEXT:    stfiwx f0, r5, r3
 ; CHECK-P9-NEXT:    li r3, 20
 ; CHECK-P9-NEXT:    stxsiwx vs34, r5, r3
+; CHECK-P9-NEXT:    vperm v3, v2, v2, v3
+; CHECK-P9-NEXT:    stxv vs35, 0(r5)
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-P9-BE-LABEL: test_stores_exceed_vec_size:
@@ -613,10 +613,10 @@ define dso_local void @test_stores_exceed_vec_size(<4 x i32> %a, i32* nocapture
 ; CHECK-P9-BE-NEXT:    xxspltw vs0, vs34, 0
 ; CHECK-P9-BE-NEXT:    xxsldwi vs0, vs34, vs0, 2
 ; CHECK-P9-BE-NEXT:    li r3, 16
-; CHECK-P9-BE-NEXT:    stxv vs0, 0(r5)
-; CHECK-P9-BE-NEXT:    xxsldwi vs0, vs34, vs34, 1
 ; CHECK-P9-BE-NEXT:    stxsiwx vs34, r5, r3
 ; CHECK-P9-BE-NEXT:    li r3, 20
+; CHECK-P9-BE-NEXT:    stxv vs0, 0(r5)
+; CHECK-P9-BE-NEXT:    xxsldwi vs0, vs34, vs34, 1
 ; CHECK-P9-BE-NEXT:    stfiwx f0, r5, r3
 ; CHECK-P9-BE-NEXT:    blr
 entry:
@@ -794,8 +794,6 @@ define void @test_13_consecutive_stores_of_bytes(<16 x i8> %a, i8* nocapture %b)
 ;
 ; CHECK-P9-LABEL: test_13_consecutive_stores_of_bytes:
 ; CHECK-P9:       # %bb.0: # %entry
-; CHECK-P9-NEXT:    vsldoi v3, v2, v2, 4
-; CHECK-P9-NEXT:    stxsibx vs35, 0, r5
 ; CHECK-P9-NEXT:    vsldoi v3, v2, v2, 12
 ; CHECK-P9-NEXT:    li r3, 1
 ; CHECK-P9-NEXT:    stxsibx vs35, r5, r3
@@ -808,6 +806,8 @@ define void @test_13_consecutive_stores_of_bytes(<16 x i8> %a, i8* nocapture %b)
 ; CHECK-P9-NEXT:    vsldoi v3, v2, v2, 2
 ; CHECK-P9-NEXT:    li r3, 4
 ; CHECK-P9-NEXT:    stxsibx vs35, r5, r3
+; CHECK-P9-NEXT:    vsldoi v3, v2, v2, 4
+; CHECK-P9-NEXT:    stxsibx vs35, 0, r5
 ; CHECK-P9-NEXT:    vsldoi v3, v2, v2, 8
 ; CHECK-P9-NEXT:    li r3, 5
 ; CHECK-P9-NEXT:    stxsibx vs35, r5, r3
@@ -836,19 +836,19 @@ define void @test_13_consecutive_stores_of_bytes(<16 x i8> %a, i8* nocapture %b)
 ;
 ; CHECK-P9-BE-LABEL: test_13_consecutive_stores_of_bytes:
 ; CHECK-P9-BE:       # %bb.0: # %entry
-; CHECK-P9-BE-NEXT:    vsldoi v3, v2, v2, 13
-; CHECK-P9-BE-NEXT:    stxsibx vs35, 0, r5
+; CHECK-P9-BE-NEXT:    li r3, 3
 ; CHECK-P9-BE-NEXT:    vsldoi v3, v2, v2, 5
+; CHECK-P9-BE-NEXT:    stxsibx vs34, r5, r3
 ; CHECK-P9-BE-NEXT:    li r3, 1
 ; CHECK-P9-BE-NEXT:    stxsibx vs35, r5, r3
 ; CHECK-P9-BE-NEXT:    vsldoi v3, v2, v2, 2
 ; CHECK-P9-BE-NEXT:    li r3, 2
 ; CHECK-P9-BE-NEXT:    stxsibx vs35, r5, r3
-; CHECK-P9-BE-NEXT:    li r3, 3
 ; CHECK-P9-BE-NEXT:    vsldoi v3, v2, v2, 15
-; CHECK-P9-BE-NEXT:    stxsibx vs34, r5, r3
 ; CHECK-P9-BE-NEXT:    li r3, 4
 ; CHECK-P9-BE-NEXT:    stxsibx vs35, r5, r3
+; CHECK-P9-BE-NEXT:    vsldoi v3, v2, v2, 13
+; CHECK-P9-BE-NEXT:    stxsibx vs35, 0, r5
 ; CHECK-P9-BE-NEXT:    vsldoi v3, v2, v2, 9
 ; CHECK-P9-BE-NEXT:    li r3, 5
 ; CHECK-P9-BE-NEXT:    stxsibx vs35, r5, r3
@@ -930,8 +930,8 @@ define void @test_elements_from_two_vec(<4 x i32> %a, <4 x i32> %b, i32* nocaptu
 ; CHECK-BE:       # %bb.0: # %entry
 ; CHECK-BE-NEXT:    xxsldwi vs0, vs34, vs34, 3
 ; CHECK-BE-NEXT:    li r3, 4
-; CHECK-BE-NEXT:    stfiwx f0, r7, r3
 ; CHECK-BE-NEXT:    stxsiwx vs35, 0, r7
+; CHECK-BE-NEXT:    stfiwx f0, r7, r3
 ; CHECK-BE-NEXT:    blr
 ;
 ; CHECK-P9-LABEL: test_elements_from_two_vec:
@@ -977,19 +977,19 @@ define dso_local void @test_elements_from_three_vec(<4 x float> %a, <4 x float>
 ; CHECK-BE-NEXT:    xxsldwi vs1, vs35, vs35, 1
 ; CHECK-BE-NEXT:    li r3, 4
 ; CHECK-BE-NEXT:    li r4, 8
+; CHECK-BE-NEXT:    stxsiwx vs36, r9, r4
 ; CHECK-BE-NEXT:    stfiwx f1, r9, r3
 ; CHECK-BE-NEXT:    stfiwx f0, 0, r9
-; CHECK-BE-NEXT:    stxsiwx vs36, r9, r4
 ; CHECK-BE-NEXT:    blr
 ;
 ; CHECK-P9-LABEL: test_elements_from_three_vec:
 ; CHECK-P9:       # %bb.0: # %entry
 ; CHECK-P9-NEXT:    xxsldwi vs0, vs34, vs34, 3
 ; CHECK-P9-NEXT:    li r3, 4
-; CHECK-P9-NEXT:    stfiwx f0, 0, r9
-; CHECK-P9-NEXT:    xxsldwi vs0, vs36, vs36, 1
 ; CHECK-P9-NEXT:    stxsiwx vs35, r9, r3
 ; CHECK-P9-NEXT:    li r3, 8
+; CHECK-P9-NEXT:    stfiwx f0, 0, r9
+; CHECK-P9-NEXT:    xxsldwi vs0, vs36, vs36, 1
 ; CHECK-P9-NEXT:    stfiwx f0, r9, r3
 ; CHECK-P9-NEXT:    blr
 ;
diff --git a/llvm/test/CodeGen/PowerPC/f128-aggregates.ll b/llvm/test/CodeGen/PowerPC/f128-aggregates.ll
index 30476508d..6e782c2b0 100644
--- a/llvm/test/CodeGen/PowerPC/f128-aggregates.ll
+++ b/llvm/test/CodeGen/PowerPC/f128-aggregates.ll
@@ -11,7 +11,7 @@
 %struct.With9fp128params = type { fp128, fp128, fp128, fp128, fp128, fp128,
                                   fp128, fp128, fp128 }
 
-@a1 = common local_unnamed_addr global [3 x fp128] zeroinitializer, align 16
+@a1 = local_unnamed_addr global [3 x fp128] zeroinitializer, align 16
 
 ; Function Attrs: norecurse nounwind readonly
 define fp128 @testArray_01(fp128* nocapture readonly %sa) {
@@ -353,10 +353,10 @@ define fp128 @sum_float128(i32 signext %count, ...) {
 ; CHECK-NEXT:    addi r3, r1, 40
 ; CHECK-NEXT:    lxvx v3, 0, r3
 ; CHECK-NEXT:    xsaddqp v2, v3, v2
-; CHECK-NEXT:    addi [[REG2:r[0-9]+]], r1, 72
-; CHECK-NEXT:    std [[REG2]], -8(r1)
 ; CHECK-NEXT:    lxv v3, 16(r3)
 ; CHECK-NEXT:    xsaddqp v2, v2, v3
+; CHECK-NEXT:    addi [[REG2:r[0-9]+]], r1, 72
+; CHECK-NEXT:    std [[REG2]], -8(r1)
 ; CHECK-NEXT:    blr
 entry:
   %ap = alloca i8*, align 8
diff --git a/llvm/test/CodeGen/PowerPC/f128-fma.ll b/llvm/test/CodeGen/PowerPC/f128-fma.ll
index 9c3c062e4..8bd2ffe03 100644
--- a/llvm/test/CodeGen/PowerPC/f128-fma.ll
+++ b/llvm/test/CodeGen/PowerPC/f128-fma.ll
@@ -129,7 +129,7 @@ entry:
 ; CHECK-DAG: lxv v[[REG3:[0-9]+]], 0(r3)
 ; CHECK-DAG: lxv v[[REG4:[0-9]+]], 0(r4)
 ; CHECK-DAG: lxv v[[REG5:[0-9]+]], 0(r5)
-; CHECK: xsnmsubqp v[[REG3]], v[[REG5]], v[[REG4]]
+; CHECK: xsnmsubqp v[[REG3]], v[[REG4]], v[[REG5]]
 ; CHECK-NEXT: stxv v[[REG3]], 0(r6)
 ; CHECK-NEXT: blr
 }
diff --git a/llvm/test/CodeGen/PowerPC/fast-isel-call.ll b/llvm/test/CodeGen/PowerPC/fast-isel-call.ll
index 8823beb96..d3a1bea87 100644
--- a/llvm/test/CodeGen/PowerPC/fast-isel-call.ll
+++ b/llvm/test/CodeGen/PowerPC/fast-isel-call.ll
@@ -1,8 +1,4 @@
-; FIXME: FastISel currently returns false if it hits code that uses VSX
-; registers and with -fast-isel-abort=1 turned on the test case will then fail.
-; When fastisel better supports VSX fix up this test case.
-;
-; RUN: llc < %s -O0 -relocation-model=pic -verify-machineinstrs -mattr=-vsx -fast-isel-abort=1 -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr7 -ppc-late-peephole=true | FileCheck %s --check-prefix=ELF64
+; RUN: llc < %s -O0 -relocation-model=pic -verify-machineinstrs -fast-isel-abort=1 -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr7 -ppc-late-peephole=true | FileCheck %s --check-prefix=ELF64
 
 define i32 @t1(i8 signext %a) nounwind {
   %1 = sext i8 %a to i32
diff --git a/llvm/test/CodeGen/PowerPC/fast-isel-const.ll b/llvm/test/CodeGen/PowerPC/fast-isel-const.ll
index 3987e54a8..c0b9fe439 100644
--- a/llvm/test/CodeGen/PowerPC/fast-isel-const.ll
+++ b/llvm/test/CodeGen/PowerPC/fast-isel-const.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -O0 -verify-machineinstrs -fast-isel-abort=1 -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr7 -mattr=-vsx | FileCheck %s --check-prefix=ELF64
+; RUN: llc < %s -O0 -verify-machineinstrs -fast-isel-abort=1 -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr7 | FileCheck %s --check-prefix=ELF64
 
 define zeroext i1 @testi1(i8 %in) nounwind {
 entry:
diff --git a/llvm/test/CodeGen/PowerPC/fast-isel-load-store.ll b/llvm/test/CodeGen/PowerPC/fast-isel-load-store.ll
index 80a733c5d..cb9e1c925 100644
--- a/llvm/test/CodeGen/PowerPC/fast-isel-load-store.ll
+++ b/llvm/test/CodeGen/PowerPC/fast-isel-load-store.ll
@@ -1,8 +1,5 @@
-; FIXME: FastISel currently returns false if it hits code that uses VSX
-; registers and with -fast-isel-abort=1 turned on the test case will then fail.
-; When fastisel better supports VSX fix up this test case.
-;
 ; RUN: llc -relocation-model=static < %s -O0 -verify-machineinstrs -fast-isel -fast-isel-abort=1 -mattr=-vsx -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr7 | FileCheck %s --check-prefix=ELF64
+; RUN: llc -relocation-model=static < %s -O0 -verify-machineinstrs -fast-isel -fast-isel-abort=1 -mattr=+vsx -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr7 | FileCheck %s --check-prefix=VSX
 ; RUN: llc -relocation-model=static < %s -O0 -verify-machineinstrs -fast-isel -fast-isel-abort=1 -mattr=spe  -mtriple=powerpc-unknown-linux-gnu -mcpu=e500 | FileCheck %s --check-prefix=SPE
 
 ; This test verifies that load/store instructions are properly generated,
@@ -78,9 +75,11 @@ define double @t6() nounwind {
 ; SPE: t6
   %1 = load double, double* @f, align 8
 ; ELF64: lfd
+; VSX: lxsdx
 ; SPE: evldd
   %2 = fadd double %1, 1.0
 ; ELF64: fadd
+; VSX: xsadddp
 ; SPE: efdadd
   ret double %2
 }
@@ -150,6 +149,8 @@ define void @t12(double %v) nounwind {
   store double %1, double* @f, align 8
 ; ELF64: fadd
 ; ELF64: stfd
+; VSX: xsadddp
+; VSX: stxsdx
 ; SPE: efdadd
 ; SPE: evstdd
   ret void
diff --git a/llvm/test/CodeGen/PowerPC/fast-isel-ret.ll b/llvm/test/CodeGen/PowerPC/fast-isel-ret.ll
index 4c499e44e..5fe786129 100644
--- a/llvm/test/CodeGen/PowerPC/fast-isel-ret.ll
+++ b/llvm/test/CodeGen/PowerPC/fast-isel-ret.ll
@@ -1,8 +1,4 @@
-; FIXME: FastISel currently returns false if it hits code that uses VSX
-; registers and with -fast-isel-abort=1 turned on the test case will then fail.
-; When fastisel better supports VSX fix up this test case.
-;
-; RUN: llc < %s -O0 -verify-machineinstrs -fast-isel-abort=1 -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr7 -mattr=-vsx | FileCheck %s --check-prefix=ELF64
+; RUN: llc < %s -O0 -verify-machineinstrs -fast-isel-abort=1 -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr7 | FileCheck %s --check-prefix=ELF64
 
 define zeroext i1 @rettrue() nounwind {
 entry:
diff --git a/llvm/test/CodeGen/PowerPC/fdiv.ll b/llvm/test/CodeGen/PowerPC/fdiv.ll
new file mode 100644
index 000000000..183e514d9
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/fdiv.ll
@@ -0,0 +1,15 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -verify-machineinstrs < %s -mtriple=powerpc64le-unknown-linux-gnu -mcpu=pwr8 | FileCheck %s
+
+define dso_local float @foo(float %0, float %1) local_unnamed_addr {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xsresp 3, 2
+; CHECK-NEXT:    xsmulsp 0, 1, 3
+; CHECK-NEXT:    xsnmsubasp 1, 2, 0
+; CHECK-NEXT:    xsmaddasp 0, 3, 1
+; CHECK-NEXT:    fmr 1, 0
+; CHECK-NEXT:    blr
+  %3 = fdiv fast float %0, %1
+  ret float %3
+}
diff --git a/llvm/test/CodeGen/PowerPC/float-load-store-pair.ll b/llvm/test/CodeGen/PowerPC/float-load-store-pair.ll
index 6a8bd8e7a..a8ed39ee9 100644
--- a/llvm/test/CodeGen/PowerPC/float-load-store-pair.ll
+++ b/llvm/test/CodeGen/PowerPC/float-load-store-pair.ll
@@ -25,14 +25,12 @@
 
 ; Because this test function is trying to pass float argument by stack,
 ; so the fpr is only used to load/store float argument
-define signext i32 @test() {
+define signext i32 @test() nounwind {
 ; CHECK-LABEL: test:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    mflr 0
 ; CHECK-NEXT:    std 0, 16(1)
 ; CHECK-NEXT:    stdu 1, -192(1)
-; CHECK-NEXT:    .cfi_def_cfa_offset 192
-; CHECK-NEXT:    .cfi_offset lr, 16
 ; CHECK-NEXT:    addis 3, 2, a1@toc@ha
 ; CHECK-NEXT:    lfd 1, a1@toc@l(3)
 ; CHECK-NEXT:    addis 3, 2, a2@toc@ha
@@ -54,27 +52,27 @@ define signext i32 @test() {
 ; CHECK-NEXT:    addis 3, 2, a10@toc@ha
 ; CHECK-NEXT:    lfd 10, a10@toc@l(3)
 ; CHECK-NEXT:    addis 3, 2, a11@toc@ha
+; CHECK-NEXT:    addis 6, 2, a17@toc@ha
+; CHECK-NEXT:    addis 5, 2, a16@toc@ha
+; CHECK-NEXT:    addi 6, 6, a17@toc@l
+; CHECK-NEXT:    addi 5, 5, a16@toc@l
+; CHECK-NEXT:    lxvx 34, 0, 6
+; CHECK-NEXT:    addis 4, 2, a15@toc@ha
+; CHECK-NEXT:    lxvx 0, 0, 5
+; CHECK-NEXT:    ld 4, a15@toc@l(4)
+; CHECK-NEXT:    li 5, 168
 ; CHECK-NEXT:    lfd 11, a11@toc@l(3)
 ; CHECK-NEXT:    addis 3, 2, a12@toc@ha
 ; CHECK-NEXT:    lfd 12, a12@toc@l(3)
 ; CHECK-NEXT:    addis 3, 2, a13@toc@ha
 ; CHECK-NEXT:    lfd 13, a13@toc@l(3)
 ; CHECK-NEXT:    addis 3, 2, a14@toc@ha
-; CHECK-NEXT:    lfd 0, a14@toc@l(3)
-; CHECK-NEXT:    addis 3, 2, a15@toc@ha
-; CHECK-NEXT:    addis 4, 2, a17@toc@ha
-; CHECK-NEXT:    addi 4, 4, a17@toc@l
-; CHECK-NEXT:    lxsd 2, a15@toc@l(3)
-; CHECK-NEXT:    addis 3, 2, a16@toc@ha
-; CHECK-NEXT:    addi 3, 3, a16@toc@l
-; CHECK-NEXT:    lxvx 36, 0, 4
-; CHECK-NEXT:    lxvx 35, 0, 3
-; CHECK-NEXT:    li 3, 168
-; CHECK-NEXT:    stxvx 36, 1, 3
-; CHECK-NEXT:    li 3, 152
-; CHECK-NEXT:    stxvx 35, 1, 3
-; CHECK-NEXT:    stxsd 2, 144(1)
-; CHECK-NEXT:    stfd 0, 136(1)
+; CHECK-NEXT:    ld 3, a14@toc@l(3)
+; CHECK-NEXT:    stxvx 34, 1, 5
+; CHECK-NEXT:    li 5, 152
+; CHECK-NEXT:    stxvx 0, 1, 5
+; CHECK-NEXT:    std 4, 144(1)
+; CHECK-NEXT:    std 3, 136(1)
 ; CHECK-NEXT:    bl _Z3fooddddddddddddddd
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    li 3, 0
diff --git a/llvm/test/CodeGen/PowerPC/float-vector-gather.ll b/llvm/test/CodeGen/PowerPC/float-vector-gather.ll
new file mode 100644
index 000000000..b7bb622a1
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/float-vector-gather.ll
@@ -0,0 +1,48 @@
+; NOTE: This test ensures that for both Big and Little Endian cases a set of
+; NOTE: 4 floats is gathered into a v4f32 register using xxmrghw and xxmrgld
+; RUN: llc -verify-machineinstrs -mcpu=pwr9 -ppc-vsr-nums-as-vr \
+; RUN: -ppc-asm-full-reg-names -mtriple=powerpc64le-unknown-linux-gnu < %s \
+; RUN: | FileCheck %s -check-prefix=CHECK-LE
+; RUN: llc -verify-machineinstrs -mcpu=pwr9 -ppc-vsr-nums-as-vr \
+; RUN: -ppc-asm-full-reg-names -mtriple=powerpc64-unknown-linux-gnu < %s \
+; RUN: | FileCheck %s -check-prefix=CHECK-BE
+define dso_local <4 x float> @vector_gatherf(float* nocapture readonly %a,
+float* nocapture readonly %b, float* nocapture readonly %c,
+float* nocapture readonly %d) {
+; C code from which this IR test case was generated:
+; vector float test(float *a, float *b, float *c, float *d) {
+;  return (vector float) { *a, *b, *c, *d };
+; }
+; CHECK-LE-LABEL: vector_gatherf:
+; CHECK-LE:       # %bb.0: # %entry
+; CHECK-LE-DAG:    lfiwzx f[[REG0:[0-9]+]], 0, r6
+; CHECK-LE-DAG:    lfiwzx f[[REG1:[0-9]+]], 0, r5
+; CHECK-LE-DAG:    lfiwzx f[[REG2:[0-9]+]], 0, r4
+; CHECK-LE-DAG:    lfiwzx f[[REG3:[0-9]+]], 0, r3
+; CHECK-LE-DAG:    xxmrghw vs[[REG0]], vs[[REG0]], vs[[REG1]]
+; CHECK-LE-DAG:    xxmrghw vs[[REG4:[0-9]+]], vs[[REG2]], vs[[REG3]]
+; CHECK-LE-NEXT:   xxmrgld v[[REG:[0-9]+]], vs[[REG0]], vs[[REG4]]
+; CHECK-LE-NEXT:   blr
+
+; CHECK-BE-LABEL: vector_gatherf:
+; CHECK-BE:       # %bb.0: # %entry
+; CHECK-BE-DAG:    lfiwzx f[[REG0:[0-9]+]], 0, r3
+; CHECK-BE-DAG:    lfiwzx f[[REG1:[0-9]+]], 0, r4
+; CHECK-BE-DAG:    lfiwzx f[[REG2:[0-9]+]], 0, r5
+; CHECK-BE-DAG:    lfiwzx f[[REG3:[0-9]+]], 0, r6
+; CHECK-BE-DAG:    xxmrghw vs[[REG0]], vs[[REG0]], vs[[REG1]]
+; CHECK-BE-DAG:    xxmrghw vs[[REG4:[0-9]+]], vs[[REG2]], vs[[REG3]]
+; CHECK-BE-NEXT:   xxmrgld v[[REG:[0-9]+]], vs[[REG0]], vs[[REG4]]
+; CHECK-BE-NEXT:   blr
+entry:
+  %0 = load float, float* %a, align 4
+  %vecinit = insertelement <4 x float> undef, float %0, i32 0
+  %1 = load float, float* %b, align 4
+  %vecinit1 = insertelement <4 x float> %vecinit, float %1, i32 1
+  %2 = load float, float* %c, align 4
+  %vecinit2 = insertelement <4 x float> %vecinit1, float %2, i32 2
+  %3 = load float, float* %d, align 4
+  %vecinit3 = insertelement <4 x float> %vecinit2, float %3, i32 3
+  ret <4 x float> %vecinit3
+}
+
diff --git a/llvm/test/CodeGen/PowerPC/fma-assoc.ll b/llvm/test/CodeGen/PowerPC/fma-assoc.ll
index ffa3c22fd..9bca28001 100644
--- a/llvm/test/CodeGen/PowerPC/fma-assoc.ll
+++ b/llvm/test/CodeGen/PowerPC/fma-assoc.ll
@@ -260,7 +260,6 @@ define double @test_FMSUB_ASSOC_EXT3(float %A, float %B, double %C,
 ; CHECK-VSX-LABEL: test_FMSUB_ASSOC_EXT3:
 ; CHECK-VSX: xsnmsubmdp
 ; CHECK-VSX-NEXT: xsnmsubadp
-; CHECK-VSX-NEXT: fmr
 ; CHECK-VSX-NEXT: blr
 }
 
diff --git a/llvm/test/CodeGen/PowerPC/fma-combine.ll b/llvm/test/CodeGen/PowerPC/fma-combine.ll
new file mode 100644
index 000000000..88da29520
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/fma-combine.ll
@@ -0,0 +1,139 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=powerpc64le-unknown-linux-gnu -enable-no-signed-zeros-fp-math \
+; RUN:     -enable-unsafe-fp-math  < %s | FileCheck -check-prefix=CHECK-FAST %s
+; RUN: llc -mtriple=powerpc64le-unknown-linux-gnu -enable-no-signed-zeros-fp-math \
+; RUN:     -enable-unsafe-fp-math -mattr=-vsx < %s | FileCheck -check-prefix=CHECK-FAST-NOVSX %s
+; RUN: llc -mtriple=powerpc64le-unknown-linux-gnu < %s | FileCheck %s
+
+define double @fma_combine1(double %a, double %b, double %c) {
+; CHECK-FAST-LABEL: fma_combine1:
+; CHECK-FAST:       # %bb.0: # %entry
+; CHECK-FAST-NEXT:    xsnmaddadp 1, 3, 2
+; CHECK-FAST-NEXT:    blr
+;
+; CHECK-FAST-NOVSX-LABEL: fma_combine1:
+; CHECK-FAST-NOVSX:       # %bb.0: # %entry
+; CHECK-FAST-NOVSX-NEXT:    fnmadd 1, 3, 2, 1
+; CHECK-FAST-NOVSX-NEXT:    blr
+;
+; CHECK-LABEL: fma_combine1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xsnegdp 0, 3
+; CHECK-NEXT:    xsmuldp 0, 0, 2
+; CHECK-NEXT:    xssubdp 1, 0, 1
+; CHECK-NEXT:    blr
+entry:
+  %fneg1 = fneg double %c
+  %mul = fmul double %fneg1, %b
+  %add = fsub double %mul, %a
+  ret double %add
+}
+
+define double @fma_combine2(double %a, double %b, double %c) {
+; CHECK-FAST-LABEL: fma_combine2:
+; CHECK-FAST:       # %bb.0: # %entry
+; CHECK-FAST-NEXT:    xsnmaddadp 1, 2, 3
+; CHECK-FAST-NEXT:    blr
+;
+; CHECK-FAST-NOVSX-LABEL: fma_combine2:
+; CHECK-FAST-NOVSX:       # %bb.0: # %entry
+; CHECK-FAST-NOVSX-NEXT:    fnmadd 1, 2, 3, 1
+; CHECK-FAST-NOVSX-NEXT:    blr
+;
+; CHECK-LABEL: fma_combine2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xsnegdp 0, 3
+; CHECK-NEXT:    xsmuldp 0, 2, 0
+; CHECK-NEXT:    xssubdp 1, 0, 1
+; CHECK-NEXT:    blr
+entry:
+  %fneg1 = fneg double %c
+  %mul = fmul double %b, %fneg1
+  %add = fsub double %mul, %a
+  ret double %add
+}
+
+@v = common local_unnamed_addr global double 0.000000e+00, align 8
+@z = common local_unnamed_addr global double 0.000000e+00, align 8
+define double @fma_combine_two_uses(double %a, double %b, double %c) {
+; CHECK-FAST-LABEL: fma_combine_two_uses:
+; CHECK-FAST:       # %bb.0: # %entry
+; CHECK-FAST-NEXT:    xsnegdp 0, 1
+; CHECK-FAST-NEXT:    addis 3, 2, v@toc@ha
+; CHECK-FAST-NEXT:    addis 4, 2, z@toc@ha
+; CHECK-FAST-NEXT:    xsnmaddadp 1, 3, 2
+; CHECK-FAST-NEXT:    xsnegdp 2, 3
+; CHECK-FAST-NEXT:    stfd 0, v@toc@l(3)
+; CHECK-FAST-NEXT:    stfd 2, z@toc@l(4)
+; CHECK-FAST-NEXT:    blr
+;
+; CHECK-FAST-NOVSX-LABEL: fma_combine_two_uses:
+; CHECK-FAST-NOVSX:       # %bb.0: # %entry
+; CHECK-FAST-NOVSX-NEXT:    fnmadd 0, 3, 2, 1
+; CHECK-FAST-NOVSX-NEXT:    fneg 2, 1
+; CHECK-FAST-NOVSX-NEXT:    addis 3, 2, v@toc@ha
+; CHECK-FAST-NOVSX-NEXT:    addis 4, 2, z@toc@ha
+; CHECK-FAST-NOVSX-NEXT:    fneg 3, 3
+; CHECK-FAST-NOVSX-NEXT:    fmr 1, 0
+; CHECK-FAST-NOVSX-NEXT:    stfd 2, v@toc@l(3)
+; CHECK-FAST-NOVSX-NEXT:    stfd 3, z@toc@l(4)
+; CHECK-FAST-NOVSX-NEXT:    blr
+;
+; CHECK-LABEL: fma_combine_two_uses:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xsnegdp 3, 3
+; CHECK-NEXT:    addis 3, 2, v@toc@ha
+; CHECK-NEXT:    addis 4, 2, z@toc@ha
+; CHECK-NEXT:    xsmuldp 0, 3, 2
+; CHECK-NEXT:    stfd 3, z@toc@l(4)
+; CHECK-NEXT:    xsnegdp 2, 1
+; CHECK-NEXT:    xssubdp 0, 0, 1
+; CHECK-NEXT:    stfd 2, v@toc@l(3)
+; CHECK-NEXT:    fmr 1, 0
+; CHECK-NEXT:    blr
+entry:
+  %fneg = fneg double %a
+  store double %fneg, double* @v, align 8
+  %fneg1 = fneg double %c
+  store double %fneg1, double* @z, align 8
+  %mul = fmul double %fneg1, %b
+  %add = fsub double %mul, %a
+  ret double %add
+}
+
+define double @fma_combine_one_use(double %a, double %b, double %c) {
+; CHECK-FAST-LABEL: fma_combine_one_use:
+; CHECK-FAST:       # %bb.0: # %entry
+; CHECK-FAST-NEXT:    xsnegdp 0, 1
+; CHECK-FAST-NEXT:    addis 3, 2, v@toc@ha
+; CHECK-FAST-NEXT:    xsnmaddadp 1, 3, 2
+; CHECK-FAST-NEXT:    stfd 0, v@toc@l(3)
+; CHECK-FAST-NEXT:    blr
+;
+; CHECK-FAST-NOVSX-LABEL: fma_combine_one_use:
+; CHECK-FAST-NOVSX:       # %bb.0: # %entry
+; CHECK-FAST-NOVSX-NEXT:    fnmadd 0, 3, 2, 1
+; CHECK-FAST-NOVSX-NEXT:    fneg 2, 1
+; CHECK-FAST-NOVSX-NEXT:    addis 3, 2, v@toc@ha
+; CHECK-FAST-NOVSX-NEXT:    fmr 1, 0
+; CHECK-FAST-NOVSX-NEXT:    stfd 2, v@toc@l(3)
+; CHECK-FAST-NOVSX-NEXT:    blr
+;
+; CHECK-LABEL: fma_combine_one_use:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xsnegdp 0, 3
+; CHECK-NEXT:    addis 3, 2, v@toc@ha
+; CHECK-NEXT:    xsmuldp 0, 0, 2
+; CHECK-NEXT:    xsnegdp 2, 1
+; CHECK-NEXT:    xssubdp 0, 0, 1
+; CHECK-NEXT:    stfd 2, v@toc@l(3)
+; CHECK-NEXT:    fmr 1, 0
+; CHECK-NEXT:    blr
+entry:
+  %fneg = fneg double %a
+  store double %fneg, double* @v, align 8
+  %fneg1 = fneg double %c
+  %mul = fmul double %fneg1, %b
+  %add = fsub double %mul, %a
+  ret double %add
+}
diff --git a/llvm/test/CodeGen/PowerPC/fma-ext.ll b/llvm/test/CodeGen/PowerPC/fma-ext.ll
index 57771d3a4..f8ed49f9c 100644
--- a/llvm/test/CodeGen/PowerPC/fma-ext.ll
+++ b/llvm/test/CodeGen/PowerPC/fma-ext.ll
@@ -54,7 +54,6 @@ define double @test_FMSUB_EXT2(float %A, float %B, double %C) {
                                 
 ; CHECK-VSX-LABEL: test_FMSUB_EXT2:
 ; CHECK-VSX: xsnmsubmdp
-; CHECK-VSX-NEXT: fmr
 ; CHECK-VSX-NEXT: blr
 }
 
diff --git a/llvm/test/CodeGen/PowerPC/fma-mutate-duplicate-vreg.ll b/llvm/test/CodeGen/PowerPC/fma-mutate-duplicate-vreg.ll
index 1a9921713..eef6e0cca 100644
--- a/llvm/test/CodeGen/PowerPC/fma-mutate-duplicate-vreg.ll
+++ b/llvm/test/CodeGen/PowerPC/fma-mutate-duplicate-vreg.ll
@@ -33,4 +33,4 @@ define float @f(float %xf) #0 {
   ret float %25
 }
 
-attributes #0 = { norecurse nounwind readnone "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pwr8" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+vsx,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { norecurse nounwind readnone "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pwr8" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+vsx,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/PowerPC/fma-mutate.ll b/llvm/test/CodeGen/PowerPC/fma-mutate.ll
index e03bb2261..86fad0af2 100644
--- a/llvm/test/CodeGen/PowerPC/fma-mutate.ll
+++ b/llvm/test/CodeGen/PowerPC/fma-mutate.ll
@@ -3,19 +3,26 @@
 ; same as the FMA target register.  The second one is legal.  The third
 ; one doesn't fit the feeding-copy pattern.
 
-; RUN: llc -verify-machineinstrs < %s -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr7 -enable-unsafe-fp-math -mattr=+vsx -disable-ppc-vsx-fma-mutation=false | FileCheck %s
+; RUN: llc -verify-machineinstrs < %s -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr7 -mattr=+vsx -disable-ppc-vsx-fma-mutation=false | FileCheck %s
 target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
 target triple = "powerpc64-unknown-linux-gnu"
 
 declare double @llvm.sqrt.f64(double)
 
-define double @foo3(double %a) nounwind {
-  %r = call double @llvm.sqrt.f64(double %a)
-  ret double %r
-
-; CHECK: @foo3
+define double @foo3_fmf(double %a) nounwind {
+; CHECK: @foo3_fmf
 ; CHECK-NOT: fmr
 ; CHECK: xsmaddmdp
 ; CHECK: xsmaddadp
+  %r = call fast double @llvm.sqrt.f64(double %a)
+  ret double %r
+}
+
+define double @foo3_safe(double %a) nounwind {
+; CHECK: @foo3_safe
+; CHECK-NOT: fmr
+; CHECK: xssqrtdp
+  %r = call double @llvm.sqrt.f64(double %a)
+  ret double %r
 }
 
diff --git a/llvm/test/CodeGen/PowerPC/fmf-propagation.ll b/llvm/test/CodeGen/PowerPC/fmf-propagation.ll
index a62ef79f5..68d53f126 100644
--- a/llvm/test/CodeGen/PowerPC/fmf-propagation.ll
+++ b/llvm/test/CodeGen/PowerPC/fmf-propagation.ll
@@ -3,7 +3,7 @@
 ; RUN: llc < %s -mtriple=powerpc64le -debug-only=isel -o /dev/null 2>&1                        | FileCheck %s --check-prefix=FMFDEBUG
 ; RUN: llc < %s -mtriple=powerpc64le                                                           | FileCheck %s --check-prefix=FMF
 ; RUN: llc < %s -mtriple=powerpc64le -debug-only=isel -o /dev/null 2>&1 -enable-unsafe-fp-math -enable-no-nans-fp-math | FileCheck %s --check-prefix=GLOBALDEBUG
-; RUN: llc < %s -mtriple=powerpc64le -enable-unsafe-fp-math -enable-no-nans-fp-math                                    | FileCheck %s --check-prefix=GLOBAL
+; RUN: llc < %s -mtriple=powerpc64le -enable-unsafe-fp-math -enable-no-nans-fp-math -enable-no-signed-zeros-fp-math | FileCheck %s --check-prefix=GLOBAL
 
 ; Test FP transforms using instruction/node-level fast-math-flags.
 ; We're also checking debug output to verify that FMF is propagated to the newly created nodes.
diff --git a/llvm/test/CodeGen/PowerPC/fneg.ll b/llvm/test/CodeGen/PowerPC/fneg.ll
index 1a4f48c11..328ffecd1 100644
--- a/llvm/test/CodeGen/PowerPC/fneg.ll
+++ b/llvm/test/CodeGen/PowerPC/fneg.ll
@@ -1,12 +1,41 @@
-; RUN: llc -verify-machineinstrs < %s -mtriple=ppc32-- | not grep fneg
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -verify-machineinstrs < %s -mtriple=ppc32-- -ppc-asm-full-reg-names | FileCheck %s
 
 define double @test1(double %a, double %b, double %c, double %d) {
-entry:
-        %tmp2 = fsub double -0.000000e+00, %c            ; <double> [#uses=1]
-        %tmp4 = fmul double %tmp2, %d            ; <double> [#uses=1]
-        %tmp7 = fmul double %a, %b               ; <double> [#uses=1]
-        %tmp9 = fsub double %tmp7, %tmp4         ; <double> [#uses=1]
-        ret double %tmp9
+; CHECK-LABEL: test1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fmul f0, f3, f4
+; CHECK-NEXT:    fmul f1, f1, f2
+; CHECK-NEXT:    fadd f1, f1, f0
+; CHECK-NEXT:    blr
+  %tmp2 = fsub double -0.000000e+00, %c
+  %tmp4 = fmul double %tmp2, %d
+  %tmp7 = fmul double %a, %b
+  %tmp9 = fsub double %tmp7, %tmp4
+  ret double %tmp9
 }
 
+declare float @llvm.fmuladd.f32(float, float, float) #4
 
+define float @fma_fneg_fneg(float %x, float %y, float %z) {
+; CHECK-LABEL: fma_fneg_fneg:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fmadds f1, f1, f2, f3
+; CHECK-NEXT:    blr
+  %negx = fneg float %x
+  %negy = fneg float %y
+  %r = call float @llvm.fmuladd.f32(float %negx, float %negy, float %z)
+  ret float %r
+}
+
+define float @fma_fneg_fsub(float %x, float %y0, float %y1, float %z) {
+; CHECK-LABEL: fma_fneg_fsub:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fsubs f0, f3, f2
+; CHECK-NEXT:    fmadds f1, f1, f0, f4
+; CHECK-NEXT:    blr
+  %negx = fneg float %x
+  %negy = fsub nsz float %y0, %y1
+  %r = call float @llvm.fmuladd.f32(float %negx, float %negy, float %z)
+  ret float %r
+}
diff --git a/llvm/test/CodeGen/PowerPC/fold-rlwinm-1.ll b/llvm/test/CodeGen/PowerPC/fold-rlwinm-1.ll
new file mode 100644
index 000000000..12887d892
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/fold-rlwinm-1.ll
@@ -0,0 +1,44 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -verify-machineinstrs -ppc-asm-full-reg-names < %s \
+; RUN:   -mtriple=powerpc64le-unknown-linux-gnu | FileCheck %s
+
+@res = dso_local local_unnamed_addr global i32 0, align 4
+@res2 = dso_local local_unnamed_addr global i32 0, align 4
+
+define void @foo(i32 signext %var1) {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xori r3, r3, 1
+; CHECK-NEXT:    addis r4, r2, res@toc@ha
+; CHECK-NEXT:    cntlzw r3, r3
+; CHECK-NEXT:    rlwinm r3, r3, 14, 0, 12
+; CHECK-NEXT:    stw r3, res@toc@l(r4)
+; CHECK-NEXT:    blr
+entry:
+  %cmp = icmp eq i32 %var1, 1
+  %conv = zext i1 %cmp to i32
+  %shl = shl nuw nsw i32 %conv, 19
+  store i32 %shl, i32* @res, align 4
+  ret void
+}
+
+define void @foo_multiple_use(i32 signext %var1) {
+; CHECK-LABEL: foo_multiple_use:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xori r3, r3, 1
+; CHECK-NEXT:    addis r4, r2, res2@toc@ha
+; CHECK-NEXT:    addis r6, r2, res@toc@ha
+; CHECK-NEXT:    cntlzw r3, r3
+; CHECK-NEXT:    srwi r5, r3, 5
+; CHECK-NEXT:    rlwinm r3, r3, 14, 0, 12
+; CHECK-NEXT:    stw r5, res2@toc@l(r4)
+; CHECK-NEXT:    stw r3, res@toc@l(r6)
+; CHECK-NEXT:    blr
+entry:
+  %cmp = icmp eq i32 %var1, 1
+  %conv = zext i1 %cmp to i32
+  store i32 %conv, i32* @res2, align 4
+  %shl = shl nuw nsw i32 %conv, 19
+  store i32 %shl, i32* @res, align 4
+  ret void
+}
diff --git a/llvm/test/CodeGen/PowerPC/fp-int-conversions-direct-moves.ll b/llvm/test/CodeGen/PowerPC/fp-int-conversions-direct-moves.ll
index 955b1f27c..3a6ed21f5 100644
--- a/llvm/test/CodeGen/PowerPC/fp-int-conversions-direct-moves.ll
+++ b/llvm/test/CodeGen/PowerPC/fp-int-conversions-direct-moves.ll
@@ -1,418 +1,529 @@
-; RUN: llc -verify-machineinstrs -mcpu=pwr8 -mtriple=powerpc64-unknown-unknown < %s | FileCheck %s
-; RUN: llc -verify-machineinstrs -mcpu=pwr8 -mtriple=powerpc64le-unknown-unknown < %s | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -verify-machineinstrs -mcpu=pwr8 -ppc-asm-full-reg-names \
+; RUN:     -ppc-vsr-nums-as-vr -mtriple=powerpc64-unknown-unknown < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -mcpu=pwr8 -ppc-asm-full-reg-names \
+; RUN:     -ppc-vsr-nums-as-vr -mtriple=powerpc64le-unknown-unknown < %s | FileCheck %s
 
 ; Function Attrs: nounwind
 define zeroext i8 @_Z6testcff(float %arg) {
+; CHECK-LABEL: _Z6testcff:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xscvdpsxws f0, f1
+; CHECK-NEXT:    stfs f1, -4(r1)
+; CHECK-NEXT:    mfvsrwz r3, f0
+; CHECK-NEXT:    clrldi r3, r3, 32
+; CHECK-NEXT:    blr
 entry:
   %arg.addr = alloca float, align 4
   store float %arg, float* %arg.addr, align 4
   %0 = load float, float* %arg.addr, align 4
   %conv = fptoui float %0 to i8
   ret i8 %conv
-; CHECK-LABEL: @_Z6testcff
-; CHECK: xscvdpsxws [[CONVREG01:[0-9]+]], 1
-; CHECK: mfvsrwz 3, [[CONVREG01]]
 }
 
 ; Function Attrs: nounwind
 define float @_Z6testfcc(i8 zeroext %arg) {
+; CHECK-LABEL: _Z6testfcc:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    mtfprwz f0, r3
+; CHECK-NEXT:    stb r3, -1(r1)
+; CHECK-NEXT:    xscvuxdsp f1, f0
+; CHECK-NEXT:    blr
 entry:
   %arg.addr = alloca i8, align 1
   store i8 %arg, i8* %arg.addr, align 1
   %0 = load i8, i8* %arg.addr, align 1
   %conv = uitofp i8 %0 to float
   ret float %conv
-; CHECK-LABEL: @_Z6testfcc
-; CHECK: mtvsrwz [[MOVEREG01:[0-9]+]], 3
-; CHECK: xscvuxdsp 1, [[MOVEREG01]]
 }
 
 ; Function Attrs: nounwind
 define zeroext i8 @_Z6testcdd(double %arg) {
+; CHECK-LABEL: _Z6testcdd:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xscvdpsxws f0, f1
+; CHECK-NEXT:    stfd f1, -8(r1)
+; CHECK-NEXT:    mfvsrwz r3, f0
+; CHECK-NEXT:    clrldi r3, r3, 32
+; CHECK-NEXT:    blr
 entry:
   %arg.addr = alloca double, align 8
   store double %arg, double* %arg.addr, align 8
   %0 = load double, double* %arg.addr, align 8
   %conv = fptoui double %0 to i8
   ret i8 %conv
-; CHECK-LABEL: @_Z6testcdd
-; CHECK: xscvdpsxws [[CONVREG02:[0-9]+]], 1
-; CHECK: mfvsrwz 3, [[CONVREG02]]
 }
 
 ; Function Attrs: nounwind
 define double @_Z6testdcc(i8 zeroext %arg) {
+; CHECK-LABEL: _Z6testdcc:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    mtfprwz f0, r3
+; CHECK-NEXT:    stb r3, -1(r1)
+; CHECK-NEXT:    xscvuxddp f1, f0
+; CHECK-NEXT:    blr
 entry:
   %arg.addr = alloca i8, align 1
   store i8 %arg, i8* %arg.addr, align 1
   %0 = load i8, i8* %arg.addr, align 1
   %conv = uitofp i8 %0 to double
   ret double %conv
-; CHECK-LABEL: @_Z6testdcc
-; CHECK: mtvsrwz [[MOVEREG02:[0-9]+]], 3
-; CHECK: xscvuxddp 1, [[MOVEREG02]]
 }
 
 ; Function Attrs: nounwind
 define zeroext i8 @_Z7testucff(float %arg) {
+; CHECK-LABEL: _Z7testucff:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xscvdpsxws f0, f1
+; CHECK-NEXT:    stfs f1, -4(r1)
+; CHECK-NEXT:    mfvsrwz r3, f0
+; CHECK-NEXT:    clrldi r3, r3, 32
+; CHECK-NEXT:    blr
 entry:
   %arg.addr = alloca float, align 4
   store float %arg, float* %arg.addr, align 4
   %0 = load float, float* %arg.addr, align 4
   %conv = fptoui float %0 to i8
   ret i8 %conv
-; CHECK-LABEL: @_Z7testucff
-; CHECK: xscvdpsxws [[CONVREG03:[0-9]+]], 1
-; CHECK: mfvsrwz 3, [[CONVREG03]]
 }
 
 ; Function Attrs: nounwind
 define float @_Z7testfuch(i8 zeroext %arg) {
+; CHECK-LABEL: _Z7testfuch:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    mtfprwz f0, r3
+; CHECK-NEXT:    stb r3, -1(r1)
+; CHECK-NEXT:    xscvuxdsp f1, f0
+; CHECK-NEXT:    blr
 entry:
   %arg.addr = alloca i8, align 1
   store i8 %arg, i8* %arg.addr, align 1
   %0 = load i8, i8* %arg.addr, align 1
   %conv = uitofp i8 %0 to float
   ret float %conv
-; CHECK-LABEL: @_Z7testfuch
-; CHECK: mtvsrwz [[MOVEREG03:[0-9]+]], 3
-; CHECK: xscvuxdsp 1, [[MOVEREG03]]
 }
 
 ; Function Attrs: nounwind
 define zeroext i8 @_Z7testucdd(double %arg) {
+; CHECK-LABEL: _Z7testucdd:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xscvdpsxws f0, f1
+; CHECK-NEXT:    stfd f1, -8(r1)
+; CHECK-NEXT:    mfvsrwz r3, f0
+; CHECK-NEXT:    clrldi r3, r3, 32
+; CHECK-NEXT:    blr
 entry:
   %arg.addr = alloca double, align 8
   store double %arg, double* %arg.addr, align 8
   %0 = load double, double* %arg.addr, align 8
   %conv = fptoui double %0 to i8
   ret i8 %conv
-; CHECK-LABEL: @_Z7testucdd
-; CHECK: xscvdpsxws [[CONVREG04:[0-9]+]], 1
-; CHECK: mfvsrwz 3, [[CONVREG04]]
 }
 
 ; Function Attrs: nounwind
 define double @_Z7testduch(i8 zeroext %arg) {
+; CHECK-LABEL: _Z7testduch:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    mtfprwz f0, r3
+; CHECK-NEXT:    stb r3, -1(r1)
+; CHECK-NEXT:    xscvuxddp f1, f0
+; CHECK-NEXT:    blr
 entry:
   %arg.addr = alloca i8, align 1
   store i8 %arg, i8* %arg.addr, align 1
   %0 = load i8, i8* %arg.addr, align 1
   %conv = uitofp i8 %0 to double
   ret double %conv
-; CHECK-LABEL: @_Z7testduch
-; CHECK: mtvsrwz [[MOVEREG04:[0-9]+]], 3
-; CHECK: xscvuxddp 1, [[MOVEREG04]]
 }
 
 ; Function Attrs: nounwind
 define signext i16 @_Z6testsff(float %arg) {
+; CHECK-LABEL: _Z6testsff:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xscvdpsxws f0, f1
+; CHECK-NEXT:    stfs f1, -4(r1)
+; CHECK-NEXT:    mffprwz r3, f0
+; CHECK-NEXT:    extsw r3, r3
+; CHECK-NEXT:    blr
 entry:
   %arg.addr = alloca float, align 4
   store float %arg, float* %arg.addr, align 4
   %0 = load float, float* %arg.addr, align 4
   %conv = fptosi float %0 to i16
   ret i16 %conv
-; CHECK-LABEL: @_Z6testsff
-; CHECK: xscvdpsxws [[CONVREG05:[0-9]+]], 1
-; CHECK: mfvsrwz 3, [[CONVREG05]]
 }
 
 ; Function Attrs: nounwind
 define float @_Z6testfss(i16 signext %arg) {
+; CHECK-LABEL: _Z6testfss:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    mtfprwa f0, r3
+; CHECK-NEXT:    sth r3, -2(r1)
+; CHECK-NEXT:    xscvsxdsp f1, f0
+; CHECK-NEXT:    blr
 entry:
   %arg.addr = alloca i16, align 2
   store i16 %arg, i16* %arg.addr, align 2
   %0 = load i16, i16* %arg.addr, align 2
   %conv = sitofp i16 %0 to float
   ret float %conv
-; CHECK-LABEL: @_Z6testfss
-; CHECK: mtvsrwa [[MOVEREG05:[0-9]+]], 3
-; CHECK: xscvsxdsp 1, [[MOVEREG05]]
 }
 
 ; Function Attrs: nounwind
 define signext i16 @_Z6testsdd(double %arg) {
+; CHECK-LABEL: _Z6testsdd:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xscvdpsxws f0, f1
+; CHECK-NEXT:    stfd f1, -8(r1)
+; CHECK-NEXT:    mffprwz r3, f0
+; CHECK-NEXT:    extsw r3, r3
+; CHECK-NEXT:    blr
 entry:
   %arg.addr = alloca double, align 8
   store double %arg, double* %arg.addr, align 8
   %0 = load double, double* %arg.addr, align 8
   %conv = fptosi double %0 to i16
   ret i16 %conv
-; CHECK-LABEL: @_Z6testsdd
-; CHECK: xscvdpsxws [[CONVREG06:[0-9]+]], 1
-; CHECK: mfvsrwz 3, [[CONVREG06]]
 }
 
 ; Function Attrs: nounwind
 define double @_Z6testdss(i16 signext %arg) {
+; CHECK-LABEL: _Z6testdss:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    mtfprwa f0, r3
+; CHECK-NEXT:    sth r3, -2(r1)
+; CHECK-NEXT:    xscvsxddp f1, f0
+; CHECK-NEXT:    blr
 entry:
   %arg.addr = alloca i16, align 2
   store i16 %arg, i16* %arg.addr, align 2
   %0 = load i16, i16* %arg.addr, align 2
   %conv = sitofp i16 %0 to double
   ret double %conv
-; CHECK-LABEL: @_Z6testdss
-; CHECK: mtvsrwa [[MOVEREG06:[0-9]+]], 3
-; CHECK: xscvsxddp 1, [[MOVEREG06]]
 }
 
 ; Function Attrs: nounwind
 define zeroext i16 @_Z7testusff(float %arg) {
+; CHECK-LABEL: _Z7testusff:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xscvdpsxws f0, f1
+; CHECK-NEXT:    stfs f1, -4(r1)
+; CHECK-NEXT:    mfvsrwz r3, f0
+; CHECK-NEXT:    clrldi r3, r3, 32
+; CHECK-NEXT:    blr
 entry:
   %arg.addr = alloca float, align 4
   store float %arg, float* %arg.addr, align 4
   %0 = load float, float* %arg.addr, align 4
   %conv = fptoui float %0 to i16
   ret i16 %conv
-; CHECK-LABEL: @_Z7testusff
-; CHECK: xscvdpsxws [[CONVREG07:[0-9]+]], 1
-; CHECK: mfvsrwz 3, [[CONVREG07]]
 }
 
 ; Function Attrs: nounwind
 define float @_Z7testfust(i16 zeroext %arg) {
+; CHECK-LABEL: _Z7testfust:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    mtfprwz f0, r3
+; CHECK-NEXT:    sth r3, -2(r1)
+; CHECK-NEXT:    xscvuxdsp f1, f0
+; CHECK-NEXT:    blr
 entry:
   %arg.addr = alloca i16, align 2
   store i16 %arg, i16* %arg.addr, align 2
   %0 = load i16, i16* %arg.addr, align 2
   %conv = uitofp i16 %0 to float
   ret float %conv
-; CHECK-LABEL: @_Z7testfust
-; CHECK: mtvsrwz [[MOVEREG07:[0-9]+]], 3
-; CHECK: xscvuxdsp 1, [[MOVEREG07]]
 }
 
 ; Function Attrs: nounwind
 define zeroext i16 @_Z7testusdd(double %arg) {
+; CHECK-LABEL: _Z7testusdd:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xscvdpsxws f0, f1
+; CHECK-NEXT:    stfd f1, -8(r1)
+; CHECK-NEXT:    mfvsrwz r3, f0
+; CHECK-NEXT:    clrldi r3, r3, 32
+; CHECK-NEXT:    blr
 entry:
   %arg.addr = alloca double, align 8
   store double %arg, double* %arg.addr, align 8
   %0 = load double, double* %arg.addr, align 8
   %conv = fptoui double %0 to i16
   ret i16 %conv
-; CHECK-LABEL: @_Z7testusdd
-; CHECK: xscvdpsxws [[CONVREG08:[0-9]+]], 1
-; CHECK: mfvsrwz 3, [[CONVREG08]]
 }
 
 ; Function Attrs: nounwind
 define double @_Z7testdust(i16 zeroext %arg) {
+; CHECK-LABEL: _Z7testdust:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    mtfprwz f0, r3
+; CHECK-NEXT:    sth r3, -2(r1)
+; CHECK-NEXT:    xscvuxddp f1, f0
+; CHECK-NEXT:    blr
 entry:
   %arg.addr = alloca i16, align 2
   store i16 %arg, i16* %arg.addr, align 2
   %0 = load i16, i16* %arg.addr, align 2
   %conv = uitofp i16 %0 to double
   ret double %conv
-; CHECK-LABEL: @_Z7testdust
-; CHECK: mtvsrwz [[MOVEREG08:[0-9]+]], 3
-; CHECK: xscvuxddp 1, [[MOVEREG08]]
 }
 
 ; Function Attrs: nounwind
 define signext i32 @_Z6testiff(float %arg) {
+; CHECK-LABEL: _Z6testiff:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xscvdpsxws f0, f1
+; CHECK-NEXT:    stfs f1, -4(r1)
+; CHECK-NEXT:    mffprwz r3, f0
+; CHECK-NEXT:    extsw r3, r3
+; CHECK-NEXT:    blr
 entry:
   %arg.addr = alloca float, align 4
   store float %arg, float* %arg.addr, align 4
   %0 = load float, float* %arg.addr, align 4
   %conv = fptosi float %0 to i32
   ret i32 %conv
-; CHECK-LABEL: @_Z6testiff
-; CHECK: xscvdpsxws [[CONVREG09:[0-9]+]], 1
-; CHECK: mfvsrwz 3, [[CONVREG09]]
 }
 
 ; Function Attrs: nounwind
 define float @_Z6testfii(i32 signext %arg) {
+; CHECK-LABEL: _Z6testfii:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    mtfprwa f0, r3
+; CHECK-NEXT:    stw r3, -4(r1)
+; CHECK-NEXT:    xscvsxdsp f1, f0
+; CHECK-NEXT:    blr
 entry:
   %arg.addr = alloca i32, align 4
   store i32 %arg, i32* %arg.addr, align 4
   %0 = load i32, i32* %arg.addr, align 4
   %conv = sitofp i32 %0 to float
   ret float %conv
-; CHECK-LABEL: @_Z6testfii
-; CHECK: mtvsrwa [[MOVEREG09:[0-9]+]], 3
-; CHECK: xscvsxdsp 1, [[MOVEREG09]]
 }
 
 ; Function Attrs: nounwind
 define signext i32 @_Z6testidd(double %arg) {
+; CHECK-LABEL: _Z6testidd:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xscvdpsxws f0, f1
+; CHECK-NEXT:    stfd f1, -8(r1)
+; CHECK-NEXT:    mffprwz r3, f0
+; CHECK-NEXT:    extsw r3, r3
+; CHECK-NEXT:    blr
 entry:
   %arg.addr = alloca double, align 8
   store double %arg, double* %arg.addr, align 8
   %0 = load double, double* %arg.addr, align 8
   %conv = fptosi double %0 to i32
   ret i32 %conv
-; CHECK-LABEL: @_Z6testidd
-; CHECK: xscvdpsxws [[CONVREG10:[0-9]+]], 1
-; CHECK: mfvsrwz 3, [[CONVREG10]]
 }
 
 ; Function Attrs: nounwind
 define double @_Z6testdii(i32 signext %arg) {
+; CHECK-LABEL: _Z6testdii:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    mtfprwa f0, r3
+; CHECK-NEXT:    stw r3, -4(r1)
+; CHECK-NEXT:    xscvsxddp f1, f0
+; CHECK-NEXT:    blr
 entry:
   %arg.addr = alloca i32, align 4
   store i32 %arg, i32* %arg.addr, align 4
   %0 = load i32, i32* %arg.addr, align 4
   %conv = sitofp i32 %0 to double
   ret double %conv
-; CHECK-LABEL: @_Z6testdii
-; CHECK: mtvsrwa [[MOVEREG10:[0-9]+]], 3
-; CHECK: xscvsxddp 1, [[MOVEREG10]]
 }
 
 ; Function Attrs: nounwind
 define zeroext i32 @_Z7testuiff(float %arg) {
+; CHECK-LABEL: _Z7testuiff:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xscvdpuxws f0, f1
+; CHECK-NEXT:    stfs f1, -4(r1)
+; CHECK-NEXT:    mfvsrwz r3, f0
+; CHECK-NEXT:    clrldi r3, r3, 32
+; CHECK-NEXT:    blr
 entry:
   %arg.addr = alloca float, align 4
   store float %arg, float* %arg.addr, align 4
   %0 = load float, float* %arg.addr, align 4
   %conv = fptoui float %0 to i32
   ret i32 %conv
-; CHECK-LABEL: @_Z7testuiff
-; CHECK: xscvdpuxws [[CONVREG11:[0-9]+]], 1
-; CHECK: mfvsrwz 3, [[CONVREG11]]
 }
 
 ; Function Attrs: nounwind
 define float @_Z7testfuij(i32 zeroext %arg) {
+; CHECK-LABEL: _Z7testfuij:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    mtfprwz f0, r3
+; CHECK-NEXT:    stw r3, -4(r1)
+; CHECK-NEXT:    xscvuxdsp f1, f0
+; CHECK-NEXT:    blr
 entry:
   %arg.addr = alloca i32, align 4
   store i32 %arg, i32* %arg.addr, align 4
   %0 = load i32, i32* %arg.addr, align 4
   %conv = uitofp i32 %0 to float
   ret float %conv
-; CHECK-LABEL: @_Z7testfuij
-; CHECK: mtvsrwz [[MOVEREG11:[0-9]+]], 3
-; CHECK: xscvuxdsp 1, [[MOVEREG11]]
 }
 
 ; Function Attrs: nounwind
 define zeroext i32 @_Z7testuidd(double %arg) {
+; CHECK-LABEL: _Z7testuidd:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xscvdpuxws f0, f1
+; CHECK-NEXT:    stfd f1, -8(r1)
+; CHECK-NEXT:    mfvsrwz r3, f0
+; CHECK-NEXT:    clrldi r3, r3, 32
+; CHECK-NEXT:    blr
 entry:
   %arg.addr = alloca double, align 8
   store double %arg, double* %arg.addr, align 8
   %0 = load double, double* %arg.addr, align 8
   %conv = fptoui double %0 to i32
   ret i32 %conv
-; CHECK-LABEL: @_Z7testuidd
-; CHECK: xscvdpuxws [[CONVREG12:[0-9]+]], 1
-; CHECK: mfvsrwz 3, [[CONVREG12]]
 }
 
 ; Function Attrs: nounwind
 define double @_Z7testduij(i32 zeroext %arg) {
+; CHECK-LABEL: _Z7testduij:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    mtfprwz f0, r3
+; CHECK-NEXT:    stw r3, -4(r1)
+; CHECK-NEXT:    xscvuxddp f1, f0
+; CHECK-NEXT:    blr
 entry:
   %arg.addr = alloca i32, align 4
   store i32 %arg, i32* %arg.addr, align 4
   %0 = load i32, i32* %arg.addr, align 4
   %conv = uitofp i32 %0 to double
   ret double %conv
-; CHECK-LABEL: @_Z7testduij
-; CHECK: mtvsrwz [[MOVEREG12:[0-9]+]], 3
-; CHECK: xscvuxddp 1, [[MOVEREG12]]
 }
 
 ; Function Attrs: nounwind
 define i64 @_Z7testllff(float %arg) {
+; CHECK-LABEL: _Z7testllff:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xscvdpsxds f0, f1
+; CHECK-NEXT:    stfs f1, -4(r1)
+; CHECK-NEXT:    mffprd r3, f0
+; CHECK-NEXT:    blr
 entry:
   %arg.addr = alloca float, align 4
   store float %arg, float* %arg.addr, align 4
   %0 = load float, float* %arg.addr, align 4
   %conv = fptosi float %0 to i64
   ret i64 %conv
-; CHECK-LABEL: @_Z7testllff
-; CHECK: xscvdpsxds [[CONVREG13:[0-9]+]], 1
-; CHECK: mffprd 3, [[CONVREG13]]
 }
 
 ; Function Attrs: nounwind
 define float @_Z7testfllx(i64 %arg) {
+; CHECK-LABEL: _Z7testfllx:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    std r3, -8(r1)
+; CHECK-NEXT:    xscvsxdsp f1, f0
+; CHECK-NEXT:    blr
 entry:
   %arg.addr = alloca i64, align 8
   store i64 %arg, i64* %arg.addr, align 8
   %0 = load i64, i64* %arg.addr, align 8
   %conv = sitofp i64 %0 to float
   ret float %conv
-; CHECK-LABEL:@_Z7testfllx
-; CHECK: mtvsrd [[MOVEREG13:[0-9]+]], 3
-; CHECK: xscvsxdsp 1, [[MOVEREG13]]
 }
 
 ; Function Attrs: nounwind
 define i64 @_Z7testlldd(double %arg) {
+; CHECK-LABEL: _Z7testlldd:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xscvdpsxds f0, f1
+; CHECK-NEXT:    stfd f1, -8(r1)
+; CHECK-NEXT:    mffprd r3, f0
+; CHECK-NEXT:    blr
 entry:
   %arg.addr = alloca double, align 8
   store double %arg, double* %arg.addr, align 8
   %0 = load double, double* %arg.addr, align 8
   %conv = fptosi double %0 to i64
   ret i64 %conv
-; CHECK-LABEL: @_Z7testlldd
-; CHECK: xscvdpsxds [[CONVREG14:[0-9]+]], 1
-; CHECK: mffprd 3, [[CONVREG14]]
 }
 
 ; Function Attrs: nounwind
 define double @_Z7testdllx(i64 %arg) {
+; CHECK-LABEL: _Z7testdllx:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    std r3, -8(r1)
+; CHECK-NEXT:    xscvsxddp f1, f0
+; CHECK-NEXT:    blr
 entry:
   %arg.addr = alloca i64, align 8
   store i64 %arg, i64* %arg.addr, align 8
   %0 = load i64, i64* %arg.addr, align 8
   %conv = sitofp i64 %0 to double
   ret double %conv
-; CHECK-LABEL: @_Z7testdllx
-; CHECK: mtvsrd [[MOVEREG14:[0-9]+]], 3
-; CHECK: xscvsxddp 1, [[MOVEREG14]]
 }
 
 ; Function Attrs: nounwind
 define i64 @_Z8testullff(float %arg) {
+; CHECK-LABEL: _Z8testullff:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xscvdpuxds f0, f1
+; CHECK-NEXT:    stfs f1, -4(r1)
+; CHECK-NEXT:    mffprd r3, f0
+; CHECK-NEXT:    blr
 entry:
   %arg.addr = alloca float, align 4
   store float %arg, float* %arg.addr, align 4
   %0 = load float, float* %arg.addr, align 4
   %conv = fptoui float %0 to i64
   ret i64 %conv
-; CHECK-LABEL: @_Z8testullff
-; CHECK: xscvdpuxds [[CONVREG15:[0-9]+]], 1
-; CHECK: mffprd 3, [[CONVREG15]]
 }
 
 ; Function Attrs: nounwind
 define float @_Z8testfully(i64 %arg) {
+; CHECK-LABEL: _Z8testfully:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    std r3, -8(r1)
+; CHECK-NEXT:    xscvuxdsp f1, f0
+; CHECK-NEXT:    blr
 entry:
   %arg.addr = alloca i64, align 8
   store i64 %arg, i64* %arg.addr, align 8
   %0 = load i64, i64* %arg.addr, align 8
   %conv = uitofp i64 %0 to float
   ret float %conv
-; CHECK-LABEL: @_Z8testfully
-; CHECK: mtvsrd [[MOVEREG15:[0-9]+]], 3
-; CHECK: xscvuxdsp 1, [[MOVEREG15]]
 }
 
 ; Function Attrs: nounwind
 define i64 @_Z8testulldd(double %arg) {
+; CHECK-LABEL: _Z8testulldd:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xscvdpuxds f0, f1
+; CHECK-NEXT:    stfd f1, -8(r1)
+; CHECK-NEXT:    mffprd r3, f0
+; CHECK-NEXT:    blr
 entry:
   %arg.addr = alloca double, align 8
   store double %arg, double* %arg.addr, align 8
   %0 = load double, double* %arg.addr, align 8
   %conv = fptoui double %0 to i64
   ret i64 %conv
-; CHECK-LABEL: @_Z8testulldd
-; CHECK: xscvdpuxds [[CONVREG16:[0-9]+]], 1
-; CHECK: mffprd 3, [[CONVREG16]]
 }
 
 ; Function Attrs: nounwind
 define double @_Z8testdully(i64 %arg) {
+; CHECK-LABEL: _Z8testdully:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    std r3, -8(r1)
+; CHECK-NEXT:    xscvuxddp f1, f0
+; CHECK-NEXT:    blr
 entry:
   %arg.addr = alloca i64, align 8
   store i64 %arg, i64* %arg.addr, align 8
   %0 = load i64, i64* %arg.addr, align 8
   %conv = uitofp i64 %0 to double
   ret double %conv
-; CHECK-LABEL: @_Z8testdully
-; CHECK: mtvsrd [[MOVEREG16:[0-9]+]], 3
-; CHECK: xscvuxddp 1, [[MOVEREG16]]
 }
diff --git a/llvm/test/CodeGen/PowerPC/fp-int128-fp-combine.ll b/llvm/test/CodeGen/PowerPC/fp-int128-fp-combine.ll
index 6c9690527..47c05b56c 100644
--- a/llvm/test/CodeGen/PowerPC/fp-int128-fp-combine.ll
+++ b/llvm/test/CodeGen/PowerPC/fp-int128-fp-combine.ll
@@ -4,14 +4,12 @@
 ; xscvdpsxds should NOT be emitted, since it saturates the result down to i64.
 ; We can't use friz here because it may return -0.0 where the original code doesn't.
 
-define float @f_i128_f(float %v) {
+define float @f_i128_f(float %v) nounwind {
 ; CHECK-LABEL: f_i128_f:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    mflr 0
 ; CHECK-NEXT:    std 0, 16(1)
 ; CHECK-NEXT:    stdu 1, -32(1)
-; CHECK-NEXT:    .cfi_def_cfa_offset 32
-; CHECK-NEXT:    .cfi_offset lr, 16
 ; CHECK-NEXT:    bl __fixsfti
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    bl __floattisf
@@ -31,7 +29,7 @@ entry:
 define float @f_i128_fi_nsz(float %v) #0 {
 ; CHECK-LABEL: f_i128_fi_nsz:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    friz 0, 1
+; CHECK-NEXT:    xsrdpiz 0, 1
 ; CHECK-NEXT:    fmr 1, 0
 ; CHECK-NEXT:    blr
 entry:
diff --git a/llvm/test/CodeGen/PowerPC/fp-intrinsics-fptosi-legal.ll b/llvm/test/CodeGen/PowerPC/fp-intrinsics-fptosi-legal.ll
new file mode 100644
index 000000000..dff47786e
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/fp-intrinsics-fptosi-legal.ll
@@ -0,0 +1,20 @@
+; RUN: llc -O3 -mtriple=powerpc-unknown-linux-gnu -mcpu=e500 -mattr=spe < %s | FileCheck %s
+
+; PowerPC SPE is a rare in-tree target that has the FP_TO_SINT node marked
+; as Legal.
+
+; Verify that fptosi(42.1) isn't simplified when the rounding mode is
+; unknown.
+; Verify that no gross errors happen.
+; CHECK-LABEL: @f20
+; COMMON: cfdctsiz
+define i32 @f20(double %a) strictfp {
+entry:
+  %result = call i32 @llvm.experimental.constrained.fptosi.i32.f64(double 42.1,
+                                               metadata !"fpexcept.strict")
+                                               strictfp
+  ret i32 %result
+}
+
+@llvm.fp.env = thread_local global i8 zeroinitializer, section "llvm.metadata"
+declare i32 @llvm.experimental.constrained.fptosi.i32.f64(double, metadata)
diff --git a/llvm/test/CodeGen/PowerPC/fp64-to-int16.ll b/llvm/test/CodeGen/PowerPC/fp64-to-int16.ll
index 0048f916c..27c6e71ba 100644
--- a/llvm/test/CodeGen/PowerPC/fp64-to-int16.ll
+++ b/llvm/test/CodeGen/PowerPC/fp64-to-int16.ll
@@ -6,7 +6,7 @@ define i1 @Test(double %a) {
 ; CHECK-LABEL: Test:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    xscvdpsxws 0, 1
-; CHECK-NEXT:    mfvsrwz 3, 0
+; CHECK-NEXT:    mffprwz 3, 0
 ; CHECK-NEXT:    xori 3, 3, 65534
 ; CHECK-NEXT:    cntlzw 3, 3
 ; CHECK-NEXT:    srwi 3, 3, 5
diff --git a/llvm/test/CodeGen/PowerPC/frameaddr.ll b/llvm/test/CodeGen/PowerPC/frameaddr.ll
index 0b1733d79..2baf5e2d5 100644
--- a/llvm/test/CodeGen/PowerPC/frameaddr.ll
+++ b/llvm/test/CodeGen/PowerPC/frameaddr.ll
@@ -40,8 +40,8 @@ declare void @use(i8*)
 
 declare i8* @llvm.frameaddress(i32) #2
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { noreturn nounwind }
 attributes #2 = { nounwind readnone }
-attributes #3 = { nounwind naked "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { nounwind naked "less-precise-fpmad"="false" "frame-pointer"="non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
diff --git a/llvm/test/CodeGen/PowerPC/funnel-shift-rot.ll b/llvm/test/CodeGen/PowerPC/funnel-shift-rot.ll
index fd5f51c9e..fbbd01faa 100644
--- a/llvm/test/CodeGen/PowerPC/funnel-shift-rot.ll
+++ b/llvm/test/CodeGen/PowerPC/funnel-shift-rot.ll
@@ -75,11 +75,7 @@ define i64 @rotl_i64(i64 %x, i64 %z) {
 define <4 x i32> @rotl_v4i32(<4 x i32> %x, <4 x i32> %z) {
 ; CHECK-LABEL: rotl_v4i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    xxlxor 36, 36, 36
-; CHECK-NEXT:    vslw 5, 2, 3
-; CHECK-NEXT:    vsubuwm 3, 4, 3
-; CHECK-NEXT:    vsrw 2, 2, 3
-; CHECK-NEXT:    xxlor 34, 37, 34
+; CHECK-NEXT:    vrlw 2, 2, 3
 ; CHECK-NEXT:    blr
   %f = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %x, <4 x i32> %x, <4 x i32> %z)
   ret <4 x i32> %f
@@ -90,13 +86,8 @@ define <4 x i32> @rotl_v4i32(<4 x i32> %x, <4 x i32> %z) {
 define <4 x i32> @rotl_v4i32_const_shift(<4 x i32> %x) {
 ; CHECK-LABEL: rotl_v4i32_const_shift:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vspltisw 3, -16
-; CHECK-NEXT:    vspltisw 4, 13
-; CHECK-NEXT:    vspltisw 5, 3
-; CHECK-NEXT:    vsubuwm 3, 4, 3
-; CHECK-NEXT:    vslw 4, 2, 5
-; CHECK-NEXT:    vsrw 2, 2, 3
-; CHECK-NEXT:    xxlor 34, 36, 34
+; CHECK-NEXT:    vspltisw 3, 3
+; CHECK-NEXT:    vrlw 2, 2, 3
 ; CHECK-NEXT:    blr
   %f = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %x, <4 x i32> %x, <4 x i32> <i32 3, i32 3, i32 3, i32 3>)
   ret <4 x i32> %f
@@ -167,10 +158,8 @@ define <4 x i32> @rotr_v4i32(<4 x i32> %x, <4 x i32> %z) {
 ; CHECK-LABEL: rotr_v4i32:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    xxlxor 36, 36, 36
-; CHECK-NEXT:    vsrw 5, 2, 3
 ; CHECK-NEXT:    vsubuwm 3, 4, 3
-; CHECK-NEXT:    vslw 2, 2, 3
-; CHECK-NEXT:    xxlor 34, 34, 37
+; CHECK-NEXT:    vrlw 2, 2, 3
 ; CHECK-NEXT:    blr
   %f = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %x, <4 x i32> %x, <4 x i32> %z)
   ret <4 x i32> %f
@@ -183,11 +172,8 @@ define <4 x i32> @rotr_v4i32_const_shift(<4 x i32> %x) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vspltisw 3, -16
 ; CHECK-NEXT:    vspltisw 4, 13
-; CHECK-NEXT:    vspltisw 5, 3
 ; CHECK-NEXT:    vsubuwm 3, 4, 3
-; CHECK-NEXT:    vsrw 4, 2, 5
-; CHECK-NEXT:    vslw 2, 2, 3
-; CHECK-NEXT:    xxlor 34, 34, 36
+; CHECK-NEXT:    vrlw 2, 2, 3
 ; CHECK-NEXT:    blr
   %f = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %x, <4 x i32> %x, <4 x i32> <i32 3, i32 3, i32 3, i32 3>)
   ret <4 x i32> %f
diff --git a/llvm/test/CodeGen/PowerPC/glob-comp-aa-crash.ll b/llvm/test/CodeGen/PowerPC/glob-comp-aa-crash.ll
index 9f0e706da..54c3e1152 100644
--- a/llvm/test/CodeGen/PowerPC/glob-comp-aa-crash.ll
+++ b/llvm/test/CodeGen/PowerPC/glob-comp-aa-crash.ll
@@ -122,9 +122,9 @@ declare void @_ZNSt3__15mutex4lockEv(%"class.std::__1::mutex"*) #0
 ; Function Attrs: nounwind
 declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i1) #3
 
-attributes #0 = { optsize "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { nounwind optsize "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #2 = { noreturn optsize "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { optsize "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind optsize "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { noreturn optsize "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #3 = { nounwind }
 attributes #4 = { optsize }
 attributes #5 = { nounwind optsize }
diff --git a/llvm/test/CodeGen/PowerPC/gpr-vsr-spill.ll b/llvm/test/CodeGen/PowerPC/gpr-vsr-spill.ll
index be9df368d..a4f172322 100644
--- a/llvm/test/CodeGen/PowerPC/gpr-vsr-spill.ll
+++ b/llvm/test/CodeGen/PowerPC/gpr-vsr-spill.ll
@@ -17,7 +17,7 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK: @foo
 ; CHECK: mr [[NEWREG:[0-9]+]], 3
 ; CHECK: mr [[REG1:[0-9]+]], 4
-; CHECK: mtvsrd [[NEWREG2:[0-9]+]], 4
+; CHECK: mtfprd [[NEWREG2:[0-9]+]], 4
 ; CHECK: add {{[0-9]+}}, [[NEWREG]], [[REG1]]
 ; CHECK: mffprd [[REG2:[0-9]+]], [[NEWREG2]]
 ; CHECK: add {{[0-9]+}}, [[REG2]], [[NEWREG]]
diff --git a/llvm/test/CodeGen/PowerPC/ifcvt-forked-bug-2016-08-08.ll b/llvm/test/CodeGen/PowerPC/ifcvt-forked-bug-2016-08-08.ll
index 474be7a2a..74bfa75e5 100644
--- a/llvm/test/CodeGen/PowerPC/ifcvt-forked-bug-2016-08-08.ll
+++ b/llvm/test/CodeGen/PowerPC/ifcvt-forked-bug-2016-08-08.ll
@@ -33,5 +33,5 @@ declare i8* @_ZN11__sanitizer21internal_start_threadEPFvPvES0_(void (i8*)*, i8*)
 
 declare hidden void @_ZN11__sanitizer16BackgroundThreadEPv(i8* nocapture readnone) #5
 
-attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+vsx,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+vsx,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #7 = { nobuiltin nounwind }
diff --git a/llvm/test/CodeGen/PowerPC/ifunc.ll b/llvm/test/CodeGen/PowerPC/ifunc.ll
new file mode 100644
index 000000000..a964a2bac
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/ifunc.ll
@@ -0,0 +1,25 @@
+; RUN: llc %s -o - -mtriple=powerpc | FileCheck --check-prefix=REL %s
+; RUN: llc %s -o - -mtriple=powerpc -relocation-model=pic | FileCheck --check-prefix=PLTREL %s
+; RUN: llc %s -o - -mtriple=powerpc64 | FileCheck --check-prefix=REL %s
+; RUN: llc %s -o - -mtriple=powerpc64 -relocation-model=pic | FileCheck --check-prefix=REL %s
+
+@ifunc1 = dso_local ifunc void(), i8*()* @resolver
+@ifunc2 = ifunc void(), i8*()* @resolver
+
+define i8* @resolver() { ret i8* null }
+
+define void @foo() #0 {
+  ; REL: bl ifunc1{{$}}
+  ; REL: bl ifunc2{{$}}
+  ; PLTREL: bl ifunc1@PLT+32768
+  ; PLTREL: bl ifunc2@PLT+32768
+  call void @ifunc1()
+  call void @ifunc2()
+  ret void
+}
+
+;; Use Secure PLT ABI for PPC32.
+attributes #0 = { "target-features"="+secure-plt" }
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 1, !"PIC Level", i32 2}
diff --git a/llvm/test/CodeGen/PowerPC/inc-of-add.ll b/llvm/test/CodeGen/PowerPC/inc-of-add.ll
index 6ddf95530..11f8a0cfb 100644
--- a/llvm/test/CodeGen/PowerPC/inc-of-add.ll
+++ b/llvm/test/CodeGen/PowerPC/inc-of-add.ll
@@ -170,7 +170,6 @@ define <16 x i8> @vector_i128_i8(<16 x i8> %x, <16 x i8> %y) nounwind {
 ; PPC64BE-NEXT:    std 23, -72(1) # 8-byte Folded Spill
 ; PPC64BE-NEXT:    std 22, -80(1) # 8-byte Folded Spill
 ; PPC64BE-NEXT:    std 26, -48(1) # 8-byte Folded Spill
-; PPC64BE-NEXT:    std 2, -96(1) # 8-byte Folded Spill
 ; PPC64BE-NEXT:    std 25, -56(1) # 8-byte Folded Spill
 ; PPC64BE-NEXT:    std 24, -64(1) # 8-byte Folded Spill
 ; PPC64BE-NEXT:    std 29, -24(1) # 8-byte Folded Spill
@@ -191,8 +190,8 @@ define <16 x i8> @vector_i128_i8(<16 x i8> %x, <16 x i8> %y) nounwind {
 ; PPC64BE-NEXT:    add 7, 12, 7
 ; PPC64BE-NEXT:    lbz 12, 239(1)
 ; PPC64BE-NEXT:    lbz 26, 151(1)
-; PPC64BE-NEXT:    add 2, 21, 23
-; PPC64BE-NEXT:    lbz 23, 279(1)
+; PPC64BE-NEXT:    add 23, 21, 23
+; PPC64BE-NEXT:    lbz 21, 279(1)
 ; PPC64BE-NEXT:    lbz 25, 143(1)
 ; PPC64BE-NEXT:    add 11, 11, 22
 ; PPC64BE-NEXT:    lbz 22, 271(1)
@@ -201,8 +200,8 @@ define <16 x i8> @vector_i128_i8(<16 x i8> %x, <16 x i8> %y) nounwind {
 ; PPC64BE-NEXT:    lbz 12, 263(1)
 ; PPC64BE-NEXT:    lbz 30, 175(1)
 ; PPC64BE-NEXT:    lbz 29, 303(1)
-; PPC64BE-NEXT:    add 26, 23, 26
-; PPC64BE-NEXT:    lbz 23, 311(1)
+; PPC64BE-NEXT:    add 26, 21, 26
+; PPC64BE-NEXT:    lbz 21, 311(1)
 ; PPC64BE-NEXT:    std 28, -32(1) # 8-byte Folded Spill
 ; PPC64BE-NEXT:    add 25, 22, 25
 ; PPC64BE-NEXT:    lbz 28, 167(1)
@@ -212,7 +211,7 @@ define <16 x i8> @vector_i128_i8(<16 x i8> %x, <16 x i8> %y) nounwind {
 ; PPC64BE-NEXT:    lbz 27, 159(1)
 ; PPC64BE-NEXT:    lbz 24, 287(1)
 ; PPC64BE-NEXT:    add 30, 29, 30
-; PPC64BE-NEXT:    add 29, 23, 0
+; PPC64BE-NEXT:    add 29, 21, 0
 ; PPC64BE-NEXT:    addi 0, 29, 1
 ; PPC64BE-NEXT:    add 28, 22, 28
 ; PPC64BE-NEXT:    stb 0, 15(3)
@@ -228,7 +227,7 @@ define <16 x i8> @vector_i128_i8(<16 x i8> %x, <16 x i8> %y) nounwind {
 ; PPC64BE-NEXT:    stb 0, 11(3)
 ; PPC64BE-NEXT:    addi 0, 25, 1
 ; PPC64BE-NEXT:    stb 12, 9(3)
-; PPC64BE-NEXT:    addi 12, 2, 1
+; PPC64BE-NEXT:    addi 12, 23, 1
 ; PPC64BE-NEXT:    addi 11, 11, 1
 ; PPC64BE-NEXT:    addi 10, 10, 1
 ; PPC64BE-NEXT:    addi 9, 9, 1
@@ -247,7 +246,6 @@ define <16 x i8> @vector_i128_i8(<16 x i8> %x, <16 x i8> %y) nounwind {
 ; PPC64BE-NEXT:    stb 6, 2(3)
 ; PPC64BE-NEXT:    stb 5, 1(3)
 ; PPC64BE-NEXT:    stb 4, 0(3)
-; PPC64BE-NEXT:    ld 2, -96(1) # 8-byte Folded Reload
 ; PPC64BE-NEXT:    ld 30, -16(1) # 8-byte Folded Reload
 ; PPC64BE-NEXT:    ld 29, -24(1) # 8-byte Folded Reload
 ; PPC64BE-NEXT:    ld 28, -32(1) # 8-byte Folded Reload
diff --git a/llvm/test/CodeGen/PowerPC/inline-asm-vsx-clobbers.ll b/llvm/test/CodeGen/PowerPC/inline-asm-vsx-clobbers.ll
new file mode 100644
index 000000000..3cf0ce781
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/inline-asm-vsx-clobbers.ll
@@ -0,0 +1,32 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mcpu=pwr9 -mtriple=powerpc64le-unknown-unknown \
+; RUN:   -verify-machineinstrs -ppc-vsr-nums-as-vr \
+; RUN:   -ppc-asm-full-reg-names < %s | FileCheck %s
+
+define dso_local void @clobberVR(<4 x i32> %a, <4 x i32> %b) local_unnamed_addr {
+; CHECK-LABEL: clobberVR:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    stxv v22, -160(r1) # 16-byte Folded Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    lxv v22, -160(r1) # 16-byte Folded Reload
+; CHECK-NEXT:    blr
+entry:
+  tail call void asm sideeffect "nop", "~{vs54}"()
+  ret void
+}
+
+define dso_local void @clobberFPR(<4 x i32> %a, <4 x i32> %b) local_unnamed_addr {
+; CHECK-LABEL: clobberFPR:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    stfd f14, -144(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    lfd f14, -144(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    blr
+entry:
+  tail call void asm sideeffect "nop", "~{vs14}"()
+  ret void
+}
diff --git a/llvm/test/CodeGen/PowerPC/inlineasm-extendedmne.ll b/llvm/test/CodeGen/PowerPC/inlineasm-extendedmne.ll
new file mode 100644
index 000000000..81eba8238
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/inlineasm-extendedmne.ll
@@ -0,0 +1,72 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -verify-machineinstrs -ppc-asm-full-reg-names  \
+; RUN:     -mcpu=pwr9 -mtriple=powerpc64le-unknown-unknown < %s | FileCheck %s
+
+define dso_local void @foo()  {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    mfvsrd r0, vs33
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    mffprd r0, f3
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    mfvsrd r0, vs34
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    mfvsrwz r0, vs33
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    mffprwz r0, f3
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    mfvsrwz r0, vs34
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    mtvsrd vs33, r0
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    mtfprd f3, r0
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    mtvsrd vs34, r0
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    mtvsrwa vs33, r0
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    mtfprwa f3, r0
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    mtvsrwa vs34, r0
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    mtvsrwz vs33, r0
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    mtfprwz f3, r0
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    mtvsrwz vs34, r0
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    blr
+entry:
+  call void asm sideeffect "mfvsrd 0,33", ""()
+  call void asm sideeffect "mffprd 0,3", ""()
+  call void asm sideeffect "mfvrd  0,2", ""()
+  call void asm sideeffect "mfvsrwz 0,33", ""()
+  call void asm sideeffect "mffprwz 0,3", ""()
+  call void asm sideeffect "mfvrwz  0,2", ""()
+  call void asm sideeffect "mtvsrd 33,0", ""()
+  call void asm sideeffect "mtfprd 3,0", ""()
+  call void asm sideeffect "mtvrd  2,0", ""()
+  call void asm sideeffect "mtvsrwa 33,0", ""()
+  call void asm sideeffect "mtfprwa 3,0", ""()
+  call void asm sideeffect "mtvrwa  2,0", ""()
+  call void asm sideeffect "mtvsrwz 33,0", ""()
+  call void asm sideeffect "mtfprwz 3,0", ""()
+  call void asm sideeffect "mtvrwz  2,0", ""()
+  ret void
+}
+
diff --git a/llvm/test/CodeGen/PowerPC/instr-properties.ll b/llvm/test/CodeGen/PowerPC/instr-properties.ll
new file mode 100644
index 000000000..8d7115288
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/instr-properties.ll
@@ -0,0 +1,9 @@
+; RUN: llc < %s -mtriple=powerpc64le-unknown-linux-gnu -mcpu=pwr8 -verify-misched -stop-after=machine-scheduler -o - | FileCheck %s --check-prefix=CHECK-P8
+
+; Verify XFLOADf64 didn't implict def 'rm'.
+define double @rm() {
+; CHECK-P8-LABEL: bb.0.entry
+; CHECK-P8: %{{[0-9]+}}:vsfrc = XFLOADf64 $zero8, %{{[0-9]+}} ::
+entry:
+  ret double 2.300000e+00
+}
diff --git a/llvm/test/CodeGen/PowerPC/isel-rc-nox0.ll b/llvm/test/CodeGen/PowerPC/isel-rc-nox0.ll
index 582778f01..ed5fea958 100644
--- a/llvm/test/CodeGen/PowerPC/isel-rc-nox0.ll
+++ b/llvm/test/CodeGen/PowerPC/isel-rc-nox0.ll
@@ -42,5 +42,5 @@ func_80.exit2978.i:                               ; preds = %for.inc44.i2977.i
 ; Function Attrs: nounwind
 declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i1) #1
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "ssp-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "ssp-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind }
diff --git a/llvm/test/CodeGen/PowerPC/jump-tables-collapse-rotate.ll b/llvm/test/CodeGen/PowerPC/jump-tables-collapse-rotate.ll
index 1fa49fdbf..381e295d4 100644
--- a/llvm/test/CodeGen/PowerPC/jump-tables-collapse-rotate.ll
+++ b/llvm/test/CodeGen/PowerPC/jump-tables-collapse-rotate.ll
@@ -3,14 +3,12 @@
 ; RUN:   -ppc-asm-full-reg-names -verify-machineinstrs %s | FileCheck %s
 
 ; Function Attrs: nounwind
-define dso_local zeroext i32 @test(i32 signext %l) {
+define dso_local zeroext i32 @test(i32 signext %l) nounwind {
 ; CHECK-LABEL: test:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    mflr r0
 ; CHECK-NEXT:    std r0, 16(r1)
 ; CHECK-NEXT:    stdu r1, -32(r1)
-; CHECK-NEXT:    .cfi_def_cfa_offset 32
-; CHECK-NEXT:    .cfi_offset lr, 16
 ; CHECK-NEXT:    addi r3, r3, -1
 ; CHECK-NEXT:    cmplwi r3, 5
 ; CHECK-NEXT:    bgt cr0, .LBB0_3
diff --git a/llvm/test/CodeGen/PowerPC/knowCRBitSpill.ll b/llvm/test/CodeGen/PowerPC/knowCRBitSpill.ll
index e3a0e0ba9..7f8807d2f 100644
--- a/llvm/test/CodeGen/PowerPC/knowCRBitSpill.ll
+++ b/llvm/test/CodeGen/PowerPC/knowCRBitSpill.ll
@@ -2,6 +2,9 @@
 ; RUN:     -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr < %s | FileCheck %s
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr8 \
 ; RUN:     -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -ppc-late-peephole=false \
+; RUN:     -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr < %s | FileCheck %s \
+; RUN:     --implicit-check-not creqv --implicit-check-not crxor
 
 
 ; For known CRBit spills, CRSET/CRUNSET, it is more efficient to just load and
@@ -21,7 +24,7 @@ define dso_local signext i32 @spillCRSET(i32 signext %p1, i32 signext %p2) {
 ; CHECK-DAG:    mfocrf [[REG2:.*]], [[CREG]]
 ; CHECK-DAG:    rlwinm [[REG2]], [[REG2]]
 ; CHECK:        .LBB0_3:
-; CHECK-DAG:    creqv [[CREG:.*]]*cr5+lt, [[CREG]]*cr5+lt, [[CREG]]*cr5+lt
+; CHECK-NOT:    #UNENCODED_NOP
 ; CHECK:        lis [[REG1:.*]], -32768
 ; CHECK:        .LBB0_4:
 ; CHECK-NOT:    mfocrf [[REG2:.*]], [[CREG]]
@@ -81,8 +84,8 @@ if.end13:                                         ; preds = %if.then6, %for.end,
 define dso_local signext i32 @spillCRUNSET(%struct.p5rx* readonly %p1, i32 signext %p2, i32 signext %p3) {
 ; CHECK-LABEL: spillCRUNSET:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-DAG:    crxor [[CREG:.*]]*cr5+lt, [[CREG]]*cr5+lt, [[CREG]]*cr5+lt
 ; CHECK-DAG:    li [[REG1:.*]], 0
+; CHECK-NOT:    #UNENCODED_NOP
 ; CHECK-NOT:    mfocrf [[REG2:.*]], [[CREG]]
 ; CHECK-NOT:    rlwinm [[REG2]], [[REG2]]
 ; CHECK:        stw [[REG1]]
diff --git a/llvm/test/CodeGen/PowerPC/licm-tocReg.ll b/llvm/test/CodeGen/PowerPC/licm-tocReg.ll
index 82e44ef7b..246bf7fbc 100644
--- a/llvm/test/CodeGen/PowerPC/licm-tocReg.ll
+++ b/llvm/test/CodeGen/PowerPC/licm-tocReg.ll
@@ -1,6 +1,6 @@
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu < %s | FileCheck %s
 
-; The instructions ADDIStocHA/LDtocL are used to calculate the address of
+; The instructions ADDIStocHA8/LDtocL are used to calculate the address of
 ; globals. The ones that are in bb.3.if.end could not be hoisted by Machine
 ; LICM due to BCTRL_LDinto_toc in bb2.if.then.  This call causes the compiler
 ; to insert a save TOC to stack before the call and load into X2 to restore TOC
@@ -16,10 +16,10 @@
 ;    liveins: %x3
 ;
 ;    %4 = COPY %x3
-;    %5 = ADDIStocHA %x2, @ga
+;    %5 = ADDIStocHA8 %x2, @ga
 ;    %6 = LDtocL @ga, killed %5 :: (load 8 from got)
 ;    %7 = LWZ 0, %6 :: (volatile dereferenceable load 4 from @ga)
-;    %8 = ADDIStocHA %x2, @gb
+;    %8 = ADDIStocHA8 %x2, @gb
 ;    %9 = LDtocL @gb, killed %8 :: (load 8 from got)
 ;    %10 = LWZ 0, killed %9 :: (volatile dereferenceable load 4 from @gb)
 ;    %0 = LWZ 0, %6 :: (volatile dereferenceable load 4 from @ga)
@@ -47,11 +47,11 @@
 ;
 ;    %2 = PHI %0, %bb.0.entry, %3, %bb.3.if.end
 ;    %12 = ADDI %2, 1
-;    %13 = ADDIStocHA %x2, @ga
+;    %13 = ADDIStocHA8 %x2, @ga
 ;    %14 = LDtocL @ga, killed %13 :: (load 8 from got)
 ;    STW killed %12, 0, %14 :: (volatile store 4 into @ga)
 ;    %15 = LWZ 0, %14 :: (volatile dereferenceable load 4 from @ga)
-;    %16 = ADDIStocHA %x2, @gb
+;    %16 = ADDIStocHA8 %x2, @gb
 ;    %17 = LDtocL @gb, killed %16 :: (load 8 from got)
 ;    %18 = LWZ 0, killed %17 :: (volatile dereferenceable load 4 from @gb)
 ;    %3 = LWZ 0, %14 :: (volatile dereferenceable load 4 from @ga)
diff --git a/llvm/test/CodeGen/PowerPC/load-and-splat.ll b/llvm/test/CodeGen/PowerPC/load-and-splat.ll
new file mode 100644
index 000000000..800b79e61
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/load-and-splat.ll
@@ -0,0 +1,264 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mcpu=pwr9 -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr \
+; RUN:   -mtriple=powerpc64-unknown-unknown < %s | FileCheck %s \
+; RUN:   -check-prefix=P9
+; RUN: llc -mcpu=pwr8 -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr \
+; RUN:   -mtriple=powerpc64le-unknown-unknown < %s | FileCheck %s \
+; RUN:   -check-prefix=P8
+define dso_local void @test(<2 x double>* nocapture %c, double* nocapture readonly %a) local_unnamed_addr {
+; P9-LABEL: test:
+; P9:       # %bb.0: # %entry
+; P9-NEXT:    addi r4, r4, 24
+; P9-NEXT:    lxvdsx vs0, 0, r4
+; P9-NEXT:    stxv vs0, 0(r3)
+; P9-NEXT:    blr
+;
+; P8-LABEL: test:
+; P8:       # %bb.0: # %entry
+; P8-NEXT:    addi r4, r4, 24
+; P8-NEXT:    lxvdsx vs0, 0, r4
+; P8-NEXT:    stxvd2x vs0, 0, r3
+; P8-NEXT:    blr
+entry:
+  %arrayidx = getelementptr inbounds double, double* %a, i64 3
+  %0 = load double, double* %arrayidx, align 8
+  %splat.splatinsert.i = insertelement <2 x double> undef, double %0, i32 0
+  %splat.splat.i = shufflevector <2 x double> %splat.splatinsert.i, <2 x double> undef, <2 x i32> zeroinitializer
+  store <2 x double> %splat.splat.i, <2 x double>* %c, align 16
+  ret void
+}
+
+define dso_local void @test2(<4 x float>* nocapture %c, float* nocapture readonly %a) local_unnamed_addr {
+; P9-LABEL: test2:
+; P9:       # %bb.0: # %entry
+; P9-NEXT:    addi r4, r4, 12
+; P9-NEXT:    lxvwsx vs0, 0, r4
+; P9-NEXT:    stxv vs0, 0(r3)
+; P9-NEXT:    blr
+;
+; P8-LABEL: test2:
+; P8:       # %bb.0: # %entry
+; P8-NEXT:    addi r4, r4, 12
+; P8-NEXT:    lfiwzx f0, 0, r4
+; P8-NEXT:    xxpermdi vs0, f0, f0, 2
+; P8-NEXT:    xxspltw v2, vs0, 3
+; P8-NEXT:    stvx v2, 0, r3
+; P8-NEXT:    blr
+entry:
+  %arrayidx = getelementptr inbounds float, float* %a, i64 3
+  %0 = load float, float* %arrayidx, align 4
+  %splat.splatinsert.i = insertelement <4 x float> undef, float %0, i32 0
+  %splat.splat.i = shufflevector <4 x float> %splat.splatinsert.i, <4 x float> undef, <4 x i32> zeroinitializer
+  store <4 x float> %splat.splat.i, <4 x float>* %c, align 16
+  ret void
+}
+
+define dso_local void @test3(<4 x i32>* nocapture %c, i32* nocapture readonly %a) local_unnamed_addr {
+; P9-LABEL: test3:
+; P9:       # %bb.0: # %entry
+; P9-NEXT:    addi r4, r4, 12
+; P9-NEXT:    lxvwsx vs0, 0, r4
+; P9-NEXT:    stxv vs0, 0(r3)
+; P9-NEXT:    blr
+;
+; P8-LABEL: test3:
+; P8:       # %bb.0: # %entry
+; P8-NEXT:    addi r4, r4, 12
+; P8-NEXT:    lfiwzx f0, 0, r4
+; P8-NEXT:    xxpermdi vs0, f0, f0, 2
+; P8-NEXT:    xxspltw v2, vs0, 3
+; P8-NEXT:    stvx v2, 0, r3
+; P8-NEXT:    blr
+entry:
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 3
+  %0 = load i32, i32* %arrayidx, align 4
+  %splat.splatinsert.i = insertelement <4 x i32> undef, i32 %0, i32 0
+  %splat.splat.i = shufflevector <4 x i32> %splat.splatinsert.i, <4 x i32> undef, <4 x i32> zeroinitializer
+  store <4 x i32> %splat.splat.i, <4 x i32>* %c, align 16
+  ret void
+}
+
+define dso_local void @test4(<2 x i64>* nocapture %c, i64* nocapture readonly %a) local_unnamed_addr {
+; P9-LABEL: test4:
+; P9:       # %bb.0: # %entry
+; P9-NEXT:    addi r4, r4, 24
+; P9-NEXT:    lxvdsx vs0, 0, r4
+; P9-NEXT:    stxv vs0, 0(r3)
+; P9-NEXT:    blr
+;
+; P8-LABEL: test4:
+; P8:       # %bb.0: # %entry
+; P8-NEXT:    addi r4, r4, 24
+; P8-NEXT:    lxvdsx vs0, 0, r4
+; P8-NEXT:    stxvd2x vs0, 0, r3
+; P8-NEXT:    blr
+entry:
+  %arrayidx = getelementptr inbounds i64, i64* %a, i64 3
+  %0 = load i64, i64* %arrayidx, align 8
+  %splat.splatinsert.i = insertelement <2 x i64> undef, i64 %0, i32 0
+  %splat.splat.i = shufflevector <2 x i64> %splat.splatinsert.i, <2 x i64> undef, <2 x i32> zeroinitializer
+  store <2 x i64> %splat.splat.i, <2 x i64>* %c, align 16
+  ret void
+}
+
+define <16 x i8> @unadjusted_lxvwsx(i32* %s, i32* %t) {
+; P9-LABEL: unadjusted_lxvwsx:
+; P9:       # %bb.0: # %entry
+; P9-NEXT:    lxvwsx v2, 0, r3
+; P9-NEXT:    blr
+;
+; P8-LABEL: unadjusted_lxvwsx:
+; P8:       # %bb.0: # %entry
+; P8-NEXT:    lfiwzx f0, 0, r3
+; P8-NEXT:    xxpermdi vs0, f0, f0, 2
+; P8-NEXT:    xxspltw v2, vs0, 3
+; P8-NEXT:    blr
+  entry:
+    %0 = bitcast i32* %s to <4 x i8>*
+    %1 = load <4 x i8>, <4 x i8>* %0, align 4
+    %2 = shufflevector <4 x i8> %1, <4 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+    ret <16 x i8> %2
+}
+
+define <16 x i8> @adjusted_lxvwsx(i64* %s, i64* %t) {
+; P9-LABEL: adjusted_lxvwsx:
+; P9:       # %bb.0: # %entry
+; P9-NEXT:    addi r3, r3, 4
+; P9-NEXT:    lxvwsx v2, 0, r3
+; P9-NEXT:    blr
+;
+; P8-LABEL: adjusted_lxvwsx:
+; P8:       # %bb.0: # %entry
+; P8-NEXT:    ld r3, 0(r3)
+; P8-NEXT:    mtvsrd f0, r3
+; P8-NEXT:    xxswapd v2, vs0
+; P8-NEXT:    xxspltw v2, v2, 2
+; P8-NEXT:    blr
+  entry:
+    %0 = bitcast i64* %s to <8 x i8>*
+    %1 = load <8 x i8>, <8 x i8>* %0, align 8
+    %2 = shufflevector <8 x i8> %1, <8 x i8> undef, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7>
+    ret <16 x i8> %2
+}
+
+define <16 x i8> @unadjusted_lxvwsx_v16i8(<16 x i8> *%s, <16 x i8> %t) {
+; P9-LABEL: unadjusted_lxvwsx_v16i8:
+; P9:       # %bb.0: # %entry
+; P9-NEXT:    lxvwsx v2, 0, r3
+; P9-NEXT:    blr
+;
+; P8-LABEL: unadjusted_lxvwsx_v16i8:
+; P8:       # %bb.0: # %entry
+; P8-NEXT:    lvx v2, 0, r3
+; P8-NEXT:    xxspltw v2, v2, 3
+; P8-NEXT:    blr
+  entry:
+    %0 = load <16 x i8>, <16 x i8>* %s, align 16
+    %1 = shufflevector <16 x i8> %0, <16 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+    ret <16 x i8> %1
+}
+
+define <16 x i8> @adjusted_lxvwsx_v16i8(<16 x i8> *%s, <16 x i8> %t) {
+; P9-LABEL: adjusted_lxvwsx_v16i8:
+; P9:       # %bb.0: # %entry
+; P9-NEXT:    addi r3, r3, 4
+; P9-NEXT:    lxvwsx v2, 0, r3
+; P9-NEXT:    blr
+;
+; P8-LABEL: adjusted_lxvwsx_v16i8:
+; P8:       # %bb.0: # %entry
+; P8-NEXT:    lvx v2, 0, r3
+; P8-NEXT:    xxspltw v2, v2, 2
+; P8-NEXT:    blr
+  entry:
+    %0 = load <16 x i8>, <16 x i8>* %s, align 16
+    %1 = shufflevector <16 x i8> %0, <16 x i8> undef, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7>
+    ret <16 x i8> %1
+}
+
+define <16 x i8> @adjusted_lxvwsx_v16i8_2(<16 x i8> *%s, <16 x i8> %t) {
+; P9-LABEL: adjusted_lxvwsx_v16i8_2:
+; P9:       # %bb.0: # %entry
+; P9-NEXT:    addi r3, r3, 8
+; P9-NEXT:    lxvwsx v2, 0, r3
+; P9-NEXT:    blr
+;
+; P8-LABEL: adjusted_lxvwsx_v16i8_2:
+; P8:       # %bb.0: # %entry
+; P8-NEXT:    lvx v2, 0, r3
+; P8-NEXT:    xxspltw v2, v2, 1
+; P8-NEXT:    blr
+  entry:
+    %0 = load <16 x i8>, <16 x i8>* %s, align 16
+    %1 = shufflevector <16 x i8> %0, <16 x i8> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 8, i32 9, i32 10, i32 11, i32 8, i32 9, i32 10, i32 11, i32 8, i32 9, i32 10, i32 11>
+    ret <16 x i8> %1
+}
+
+define <16 x i8> @adjusted_lxvwsx_v16i8_3(<16 x i8> *%s, <16 x i8> %t) {
+; P9-LABEL: adjusted_lxvwsx_v16i8_3:
+; P9:       # %bb.0: # %entry
+; P9-NEXT:    addi r3, r3, 12
+; P9-NEXT:    lxvwsx v2, 0, r3
+; P9-NEXT:    blr
+;
+; P8-LABEL: adjusted_lxvwsx_v16i8_3:
+; P8:       # %bb.0: # %entry
+; P8-NEXT:    lvx v2, 0, r3
+; P8-NEXT:    xxspltw v2, v2, 0
+; P8-NEXT:    blr
+  entry:
+    %0 = load <16 x i8>, <16 x i8>* %s, align 16
+    %1 = shufflevector <16 x i8> %0, <16 x i8> undef, <16 x i32> <i32 12, i32 13, i32 14, i32 15, i32 12, i32 13, i32 14, i32 15, i32 12, i32 13, i32 14, i32 15, i32 12, i32 13, i32 14, i32 15>
+    ret <16 x i8> %1
+}
+
+define <16 x i8> @unadjusted_lxvdsx(i64* %s, i64* %t) {
+; P9-LABEL: unadjusted_lxvdsx:
+; P9:       # %bb.0: # %entry
+; P9-NEXT:    lxvdsx v2, 0, r3
+; P9-NEXT:    blr
+;
+; P8-LABEL: unadjusted_lxvdsx:
+; P8:       # %bb.0: # %entry
+; P8-NEXT:    lxvdsx v2, 0, r3
+; P8-NEXT:    blr
+  entry:
+    %0 = bitcast i64* %s to <8 x i8>*
+    %1 = load <8 x i8>, <8 x i8>* %0, align 8
+    %2 = shufflevector <8 x i8> %1, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+    ret <16 x i8> %2
+}
+
+define <16 x i8> @unadjusted_lxvdsx_v16i8(<16 x i8> *%s, <16 x i8> %t) {
+; P9-LABEL: unadjusted_lxvdsx_v16i8:
+; P9:       # %bb.0: # %entry
+; P9-NEXT:    lxvdsx v2, 0, r3
+; P9-NEXT:    blr
+;
+; P8-LABEL: unadjusted_lxvdsx_v16i8:
+; P8:       # %bb.0: # %entry
+; P8-NEXT:    lxvdsx v2, 0, r3
+; P8-NEXT:    blr
+  entry:
+    %0 = load <16 x i8>, <16 x i8>* %s, align 16
+    %1 = shufflevector <16 x i8> %0, <16 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+    ret <16 x i8> %1
+}
+
+define <16 x i8> @adjusted_lxvdsx_v16i8(<16 x i8> *%s, <16 x i8> %t) {
+; P9-LABEL: adjusted_lxvdsx_v16i8:
+; P9:       # %bb.0: # %entry
+; P9-NEXT:    addi r3, r3, 8
+; P9-NEXT:    lxvdsx v2, 0, r3
+; P9-NEXT:    blr
+;
+; P8-LABEL: adjusted_lxvdsx_v16i8:
+; P8:       # %bb.0: # %entry
+; P8-NEXT:    addi r3, r3, 8
+; P8-NEXT:    lxvdsx v2, 0, r3
+; P8-NEXT:    blr
+  entry:
+    %0 = load <16 x i8>, <16 x i8>* %s, align 16
+    %1 = shufflevector <16 x i8> %0, <16 x i8> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+    ret <16 x i8> %1
+}
diff --git a/llvm/test/CodeGen/PowerPC/load-shuffle-and-shuffle-store.ll b/llvm/test/CodeGen/PowerPC/load-shuffle-and-shuffle-store.ll
new file mode 100644
index 000000000..985dee83a
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/load-shuffle-and-shuffle-store.ll
@@ -0,0 +1,792 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr -verify-machineinstrs -mcpu=pwr8 -mattr=+vsx \
+; RUN:   -mtriple=powerpc64le-unknown-linux-gnu < %s | FileCheck %s --check-prefix=CHECK-P8
+
+; RUN: llc -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr -verify-machineinstrs -mcpu=pwr9 -mattr=+vsx \
+; RUN:   -mtriple=powerpc64le-unknown-linux-gnu < %s | FileCheck %s --check-prefix=CHECK-P9
+
+; RUN: llc -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr -verify-machineinstrs -mcpu=pwr8 -mattr=+vsx \
+; RUN:   -mtriple=powerpc64-unknown-linux-gnu < %s | FileCheck %s --check-prefix=CHECK-P8-BE
+
+; RUN: llc -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr -verify-machineinstrs -mcpu=pwr9 -mattr=+vsx \
+; RUN:   -mtriple=powerpc64-unknown-linux-gnu < %s | FileCheck %s --check-prefix=CHECK-P9-BE
+
+define <2 x i64> @load_swap00(<2 x i64>* %vp1, <2 x i64>* %vp2) {
+; CHECK-P8-LABEL: load_swap00:
+; CHECK-P8:       # %bb.0:
+; CHECK-P8-NEXT:    lxvd2x v2, 0, r3
+; CHECK-P8-NEXT:    blr
+;
+; CHECK-P9-LABEL: load_swap00:
+; CHECK-P9:       # %bb.0:
+; CHECK-P9-NEXT:    lxvd2x v2, 0, r3
+; CHECK-P9-NEXT:    blr
+;
+; CHECK-P8-BE-LABEL: load_swap00:
+; CHECK-P8-BE:       # %bb.0:
+; CHECK-P8-BE-NEXT:    lxvd2x v2, 0, r3
+; CHECK-P8-BE-NEXT:    xxswapd v2, v2
+; CHECK-P8-BE-NEXT:    blr
+;
+; CHECK-P9-BE-LABEL: load_swap00:
+; CHECK-P9-BE:       # %bb.0:
+; CHECK-P9-BE-NEXT:    lxv v2, 0(r3)
+; CHECK-P9-BE-NEXT:    xxswapd v2, v2
+; CHECK-P9-BE-NEXT:    blr
+  %v1 = load <2 x i64>, <2 x i64>* %vp1
+  %v2 = load <2 x i64>, <2 x i64>* %vp2
+  %v3 = shufflevector <2 x i64> %v1, <2 x i64> %v2, <2 x i32> <i32 1, i32 0>
+  ret <2 x i64> %v3
+}
+
+define <2 x i64> @load_swap01(<2 x i64>* %vp1, <2 x i64>* %vp2) {
+; CHECK-P8-LABEL: load_swap01:
+; CHECK-P8:       # %bb.0:
+; CHECK-P8-NEXT:    lxvd2x v2, 0, r4
+; CHECK-P8-NEXT:    blr
+;
+; CHECK-P9-LABEL: load_swap01:
+; CHECK-P9:       # %bb.0:
+; CHECK-P9-NEXT:    lxvd2x v2, 0, r4
+; CHECK-P9-NEXT:    blr
+;
+; CHECK-P8-BE-LABEL: load_swap01:
+; CHECK-P8-BE:       # %bb.0:
+; CHECK-P8-BE-NEXT:    lxvd2x v2, 0, r4
+; CHECK-P8-BE-NEXT:    xxswapd v2, v2
+; CHECK-P8-BE-NEXT:    blr
+;
+; CHECK-P9-BE-LABEL: load_swap01:
+; CHECK-P9-BE:       # %bb.0:
+; CHECK-P9-BE-NEXT:    lxv v2, 0(r4)
+; CHECK-P9-BE-NEXT:    xxswapd v2, v2
+; CHECK-P9-BE-NEXT:    blr
+  %v1 = load <2 x i64>, <2 x i64>* %vp1
+  %v2 = load <2 x i64>, <2 x i64>* %vp2
+  %v3 = shufflevector <2 x i64> %v1, <2 x i64> %v2, <2 x i32> <i32 3, i32 2>
+  ret <2 x i64> %v3
+}
+
+define <4 x i32> @load_swap10(<4 x i32>* %vp1, <4 x i32>* %vp2) {
+; CHECK-P8-LABEL: load_swap10:
+; CHECK-P8:       # %bb.0:
+; CHECK-P8-NEXT:    addis r4, r2, .LCPI2_0@toc@ha
+; CHECK-P8-NEXT:    lvx v3, 0, r3
+; CHECK-P8-NEXT:    addi r4, r4, .LCPI2_0@toc@l
+; CHECK-P8-NEXT:    lvx v2, 0, r4
+; CHECK-P8-NEXT:    vperm v2, v3, v3, v2
+; CHECK-P8-NEXT:    blr
+;
+; CHECK-P9-LABEL: load_swap10:
+; CHECK-P9:       # %bb.0:
+; CHECK-P9-NEXT:    lxvw4x v2, 0, r3
+; CHECK-P9-NEXT:    blr
+;
+; CHECK-P8-BE-LABEL: load_swap10:
+; CHECK-P8-BE:       # %bb.0:
+; CHECK-P8-BE-NEXT:    addis r4, r2, .LCPI2_0@toc@ha
+; CHECK-P8-BE-NEXT:    lxvw4x v2, 0, r3
+; CHECK-P8-BE-NEXT:    addi r4, r4, .LCPI2_0@toc@l
+; CHECK-P8-BE-NEXT:    lxvw4x v3, 0, r4
+; CHECK-P8-BE-NEXT:    vperm v2, v2, v2, v3
+; CHECK-P8-BE-NEXT:    blr
+;
+; CHECK-P9-BE-LABEL: load_swap10:
+; CHECK-P9-BE:       # %bb.0:
+; CHECK-P9-BE-NEXT:    lxv v2, 0(r3)
+; CHECK-P9-BE-NEXT:    addis r3, r2, .LCPI2_0@toc@ha
+; CHECK-P9-BE-NEXT:    addi r3, r3, .LCPI2_0@toc@l
+; CHECK-P9-BE-NEXT:    lxvx v3, 0, r3
+; CHECK-P9-BE-NEXT:    vperm v2, v2, v2, v3
+; CHECK-P9-BE-NEXT:    blr
+  %v1 = load <4 x i32>, <4 x i32>* %vp1
+  %v2 = load <4 x i32>, <4 x i32>* %vp2
+  %v3 = shufflevector <4 x i32> %v1, <4 x i32> %v2, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  ret <4 x i32> %v3
+}
+
+define <4 x i32> @load_swap11(<4 x i32>* %vp1, <4 x i32>* %vp2) {
+; CHECK-P8-LABEL: load_swap11:
+; CHECK-P8:       # %bb.0:
+; CHECK-P8-NEXT:    addis r3, r2, .LCPI3_0@toc@ha
+; CHECK-P8-NEXT:    lvx v3, 0, r4
+; CHECK-P8-NEXT:    addi r3, r3, .LCPI3_0@toc@l
+; CHECK-P8-NEXT:    lvx v2, 0, r3
+; CHECK-P8-NEXT:    vperm v2, v3, v3, v2
+; CHECK-P8-NEXT:    blr
+;
+; CHECK-P9-LABEL: load_swap11:
+; CHECK-P9:       # %bb.0:
+; CHECK-P9-NEXT:    lxvw4x v2, 0, r4
+; CHECK-P9-NEXT:    blr
+;
+; CHECK-P8-BE-LABEL: load_swap11:
+; CHECK-P8-BE:       # %bb.0:
+; CHECK-P8-BE-NEXT:    addis r3, r2, .LCPI3_0@toc@ha
+; CHECK-P8-BE-NEXT:    lxvw4x v2, 0, r4
+; CHECK-P8-BE-NEXT:    addi r3, r3, .LCPI3_0@toc@l
+; CHECK-P8-BE-NEXT:    lxvw4x v3, 0, r3
+; CHECK-P8-BE-NEXT:    vperm v2, v2, v2, v3
+; CHECK-P8-BE-NEXT:    blr
+;
+; CHECK-P9-BE-LABEL: load_swap11:
+; CHECK-P9-BE:       # %bb.0:
+; CHECK-P9-BE-NEXT:    addis r3, r2, .LCPI3_0@toc@ha
+; CHECK-P9-BE-NEXT:    addi r3, r3, .LCPI3_0@toc@l
+; CHECK-P9-BE-NEXT:    lxv v2, 0(r4)
+; CHECK-P9-BE-NEXT:    lxvx v3, 0, r3
+; CHECK-P9-BE-NEXT:    vperm v2, v2, v2, v3
+; CHECK-P9-BE-NEXT:    blr
+  %v1 = load <4 x i32>, <4 x i32>* %vp1
+  %v2 = load <4 x i32>, <4 x i32>* %vp2
+  %v3 = shufflevector <4 x i32> %v1, <4 x i32> %v2, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+  ret <4 x i32> %v3
+}
+
+define <8 x i16> @load_swap20(<8 x i16>* %vp1, <8 x i16>* %vp2){
+; CHECK-P8-LABEL: load_swap20:
+; CHECK-P8:       # %bb.0:
+; CHECK-P8-NEXT:    addis r4, r2, .LCPI4_0@toc@ha
+; CHECK-P8-NEXT:    lvx v3, 0, r3
+; CHECK-P8-NEXT:    addi r4, r4, .LCPI4_0@toc@l
+; CHECK-P8-NEXT:    lvx v2, 0, r4
+; CHECK-P8-NEXT:    vperm v2, v3, v3, v2
+; CHECK-P8-NEXT:    blr
+;
+; CHECK-P9-LABEL: load_swap20:
+; CHECK-P9:       # %bb.0:
+; CHECK-P9-NEXT:    lxvh8x v2, 0, r3
+; CHECK-P9-NEXT:    blr
+;
+; CHECK-P8-BE-LABEL: load_swap20:
+; CHECK-P8-BE:       # %bb.0:
+; CHECK-P8-BE-NEXT:    addis r4, r2, .LCPI4_0@toc@ha
+; CHECK-P8-BE-NEXT:    lxvw4x v2, 0, r3
+; CHECK-P8-BE-NEXT:    addi r4, r4, .LCPI4_0@toc@l
+; CHECK-P8-BE-NEXT:    lxvw4x v3, 0, r4
+; CHECK-P8-BE-NEXT:    vperm v2, v2, v2, v3
+; CHECK-P8-BE-NEXT:    blr
+;
+; CHECK-P9-BE-LABEL: load_swap20:
+; CHECK-P9-BE:       # %bb.0:
+; CHECK-P9-BE-NEXT:    lxv v2, 0(r3)
+; CHECK-P9-BE-NEXT:    addis r3, r2, .LCPI4_0@toc@ha
+; CHECK-P9-BE-NEXT:    addi r3, r3, .LCPI4_0@toc@l
+; CHECK-P9-BE-NEXT:    lxvx v3, 0, r3
+; CHECK-P9-BE-NEXT:    vperm v2, v2, v2, v3
+; CHECK-P9-BE-NEXT:    blr
+  %v1 = load <8 x i16>, <8 x i16>* %vp1
+  %v2 = load <8 x i16>, <8 x i16>* %vp2
+  %v3 = shufflevector <8 x i16> %v1, <8 x i16> %v2, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <8 x i16> %v3
+}
+
+define <8 x i16> @load_swap21(<8 x i16>* %vp1, <8 x i16>* %vp2){
+; CHECK-P8-LABEL: load_swap21:
+; CHECK-P8:       # %bb.0:
+; CHECK-P8-NEXT:    addis r3, r2, .LCPI5_0@toc@ha
+; CHECK-P8-NEXT:    lvx v3, 0, r4
+; CHECK-P8-NEXT:    addi r3, r3, .LCPI5_0@toc@l
+; CHECK-P8-NEXT:    lvx v2, 0, r3
+; CHECK-P8-NEXT:    vperm v2, v3, v3, v2
+; CHECK-P8-NEXT:    blr
+;
+; CHECK-P9-LABEL: load_swap21:
+; CHECK-P9:       # %bb.0:
+; CHECK-P9-NEXT:    lxvh8x v2, 0, r4
+; CHECK-P9-NEXT:    blr
+;
+; CHECK-P8-BE-LABEL: load_swap21:
+; CHECK-P8-BE:       # %bb.0:
+; CHECK-P8-BE-NEXT:    addis r3, r2, .LCPI5_0@toc@ha
+; CHECK-P8-BE-NEXT:    lxvw4x v2, 0, r4
+; CHECK-P8-BE-NEXT:    addi r3, r3, .LCPI5_0@toc@l
+; CHECK-P8-BE-NEXT:    lxvw4x v3, 0, r3
+; CHECK-P8-BE-NEXT:    vperm v2, v2, v2, v3
+; CHECK-P8-BE-NEXT:    blr
+;
+; CHECK-P9-BE-LABEL: load_swap21:
+; CHECK-P9-BE:       # %bb.0:
+; CHECK-P9-BE-NEXT:    addis r3, r2, .LCPI5_0@toc@ha
+; CHECK-P9-BE-NEXT:    addi r3, r3, .LCPI5_0@toc@l
+; CHECK-P9-BE-NEXT:    lxv v2, 0(r4)
+; CHECK-P9-BE-NEXT:    lxvx v3, 0, r3
+; CHECK-P9-BE-NEXT:    vperm v2, v2, v2, v3
+; CHECK-P9-BE-NEXT:    blr
+  %v1 = load <8 x i16>, <8 x i16>* %vp1
+  %v2 = load <8 x i16>, <8 x i16>* %vp2
+  %v3 = shufflevector <8 x i16> %v1, <8 x i16> %v2, <8 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8>
+  ret <8 x i16> %v3
+}
+
+define <16 x i8> @load_swap30(<16 x i8>* %vp1, <16 x i8>* %vp2){
+; CHECK-P8-LABEL: load_swap30:
+; CHECK-P8:       # %bb.0:
+; CHECK-P8-NEXT:    addis r4, r2, .LCPI6_0@toc@ha
+; CHECK-P8-NEXT:    lvx v3, 0, r3
+; CHECK-P8-NEXT:    addi r4, r4, .LCPI6_0@toc@l
+; CHECK-P8-NEXT:    lvx v2, 0, r4
+; CHECK-P8-NEXT:    vperm v2, v3, v3, v2
+; CHECK-P8-NEXT:    blr
+;
+; CHECK-P9-LABEL: load_swap30:
+; CHECK-P9:       # %bb.0:
+; CHECK-P9-NEXT:    lxvb16x v2, 0, r3
+; CHECK-P9-NEXT:    blr
+;
+; CHECK-P8-BE-LABEL: load_swap30:
+; CHECK-P8-BE:       # %bb.0:
+; CHECK-P8-BE-NEXT:    addis r4, r2, .LCPI6_0@toc@ha
+; CHECK-P8-BE-NEXT:    lxvw4x v2, 0, r3
+; CHECK-P8-BE-NEXT:    addi r4, r4, .LCPI6_0@toc@l
+; CHECK-P8-BE-NEXT:    lxvw4x v3, 0, r4
+; CHECK-P8-BE-NEXT:    vperm v2, v2, v2, v3
+; CHECK-P8-BE-NEXT:    blr
+;
+; CHECK-P9-BE-LABEL: load_swap30:
+; CHECK-P9-BE:       # %bb.0:
+; CHECK-P9-BE-NEXT:    lxv vs0, 0(r3)
+; CHECK-P9-BE-NEXT:    xxbrq v2, vs0
+; CHECK-P9-BE-NEXT:    blr
+  %v1 = load <16 x i8>, <16 x i8>* %vp1
+  %v2 = load <16 x i8>, <16 x i8>* %vp2
+  %v3 = shufflevector <16 x i8> %v1, <16 x i8> %v2, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <16 x i8> %v3
+}
+
+define <16 x i8> @load_swap31(<16 x i8>* %vp1, <16 x i8>* %vp2){
+; CHECK-P8-LABEL: load_swap31:
+; CHECK-P8:       # %bb.0:
+; CHECK-P8-NEXT:    addis r3, r2, .LCPI7_0@toc@ha
+; CHECK-P8-NEXT:    lvx v3, 0, r4
+; CHECK-P8-NEXT:    addi r3, r3, .LCPI7_0@toc@l
+; CHECK-P8-NEXT:    lvx v2, 0, r3
+; CHECK-P8-NEXT:    vperm v2, v3, v3, v2
+; CHECK-P8-NEXT:    blr
+;
+; CHECK-P9-LABEL: load_swap31:
+; CHECK-P9:       # %bb.0:
+; CHECK-P9-NEXT:    lxvb16x v2, 0, r4
+; CHECK-P9-NEXT:    blr
+;
+; CHECK-P8-BE-LABEL: load_swap31:
+; CHECK-P8-BE:       # %bb.0:
+; CHECK-P8-BE-NEXT:    addis r3, r2, .LCPI7_0@toc@ha
+; CHECK-P8-BE-NEXT:    lxvw4x v2, 0, r4
+; CHECK-P8-BE-NEXT:    addi r3, r3, .LCPI7_0@toc@l
+; CHECK-P8-BE-NEXT:    lxvw4x v3, 0, r3
+; CHECK-P8-BE-NEXT:    vperm v2, v2, v2, v3
+; CHECK-P8-BE-NEXT:    blr
+;
+; CHECK-P9-BE-LABEL: load_swap31:
+; CHECK-P9-BE:       # %bb.0:
+; CHECK-P9-BE-NEXT:    lxv vs0, 0(r4)
+; CHECK-P9-BE-NEXT:    xxbrq v2, vs0
+; CHECK-P9-BE-NEXT:    blr
+  %v1 = load <16 x i8>, <16 x i8>* %vp1
+  %v2 = load <16 x i8>, <16 x i8>* %vp2
+  %v3 = shufflevector <16 x i8> %v1, <16 x i8> %v2, <16 x i32> <i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16>
+  ret <16 x i8> %v3
+}
+
+define <2 x double> @load_swap40(<2 x double>* %vp1, <2 x double>* %vp2) {
+; CHECK-P8-LABEL: load_swap40:
+; CHECK-P8:       # %bb.0:
+; CHECK-P8-NEXT:    lxvd2x v2, 0, r4
+; CHECK-P8-NEXT:    blr
+;
+; CHECK-P9-LABEL: load_swap40:
+; CHECK-P9:       # %bb.0:
+; CHECK-P9-NEXT:    lxvd2x v2, 0, r4
+; CHECK-P9-NEXT:    blr
+;
+; CHECK-P8-BE-LABEL: load_swap40:
+; CHECK-P8-BE:       # %bb.0:
+; CHECK-P8-BE-NEXT:    lxvd2x vs0, 0, r4
+; CHECK-P8-BE-NEXT:    xxswapd v2, vs0
+; CHECK-P8-BE-NEXT:    blr
+;
+; CHECK-P9-BE-LABEL: load_swap40:
+; CHECK-P9-BE:       # %bb.0:
+; CHECK-P9-BE-NEXT:    lxv vs0, 0(r4)
+; CHECK-P9-BE-NEXT:    xxswapd v2, vs0
+; CHECK-P9-BE-NEXT:    blr
+  %v1 = load <2 x double>, <2 x double>* %vp1
+  %v2 = load <2 x double>, <2 x double>* %vp2
+  %v3 = shufflevector <2 x double> %v1, <2 x double> %v2, <2 x i32> <i32 3, i32 2>
+  ret <2 x double> %v3
+}
+
+define <4 x float> @load_swap50(<4 x float>* %vp1, <4 x float>* %vp2) {
+; CHECK-P8-LABEL: load_swap50:
+; CHECK-P8:       # %bb.0:
+; CHECK-P8-NEXT:    addis r4, r2, .LCPI9_0@toc@ha
+; CHECK-P8-NEXT:    lvx v3, 0, r3
+; CHECK-P8-NEXT:    addi r4, r4, .LCPI9_0@toc@l
+; CHECK-P8-NEXT:    lvx v2, 0, r4
+; CHECK-P8-NEXT:    vperm v2, v3, v3, v2
+; CHECK-P8-NEXT:    blr
+;
+; CHECK-P9-LABEL: load_swap50:
+; CHECK-P9:       # %bb.0:
+; CHECK-P9-NEXT:    lxvw4x v2, 0, r3
+; CHECK-P9-NEXT:    blr
+;
+; CHECK-P8-BE-LABEL: load_swap50:
+; CHECK-P8-BE:       # %bb.0:
+; CHECK-P8-BE-NEXT:    addis r4, r2, .LCPI9_0@toc@ha
+; CHECK-P8-BE-NEXT:    lxvw4x v2, 0, r3
+; CHECK-P8-BE-NEXT:    addi r4, r4, .LCPI9_0@toc@l
+; CHECK-P8-BE-NEXT:    lxvw4x v3, 0, r4
+; CHECK-P8-BE-NEXT:    vperm v2, v2, v2, v3
+; CHECK-P8-BE-NEXT:    blr
+;
+; CHECK-P9-BE-LABEL: load_swap50:
+; CHECK-P9-BE:       # %bb.0:
+; CHECK-P9-BE-NEXT:    lxv v2, 0(r3)
+; CHECK-P9-BE-NEXT:    addis r3, r2, .LCPI9_0@toc@ha
+; CHECK-P9-BE-NEXT:    addi r3, r3, .LCPI9_0@toc@l
+; CHECK-P9-BE-NEXT:    lxvx v3, 0, r3
+; CHECK-P9-BE-NEXT:    vperm v2, v2, v2, v3
+; CHECK-P9-BE-NEXT:    blr
+  %v1 = load <4 x float>, <4 x float>* %vp1
+  %v2 = load <4 x float>, <4 x float>* %vp2
+  %v3 = shufflevector <4 x float> %v1, <4 x float> %v2, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  ret <4 x float> %v3
+}
+
+define <4 x float> @load_swap51(<4 x float>* %vp1, <4 x float>* %vp2) {
+; CHECK-P8-LABEL: load_swap51:
+; CHECK-P8:       # %bb.0:
+; CHECK-P8-NEXT:    addis r3, r2, .LCPI10_0@toc@ha
+; CHECK-P8-NEXT:    lvx v3, 0, r4
+; CHECK-P8-NEXT:    addi r3, r3, .LCPI10_0@toc@l
+; CHECK-P8-NEXT:    lvx v2, 0, r3
+; CHECK-P8-NEXT:    vperm v2, v3, v3, v2
+; CHECK-P8-NEXT:    blr
+;
+; CHECK-P9-LABEL: load_swap51:
+; CHECK-P9:       # %bb.0:
+; CHECK-P9-NEXT:    lxvw4x v2, 0, r4
+; CHECK-P9-NEXT:    blr
+;
+; CHECK-P8-BE-LABEL: load_swap51:
+; CHECK-P8-BE:       # %bb.0:
+; CHECK-P8-BE-NEXT:    addis r3, r2, .LCPI10_0@toc@ha
+; CHECK-P8-BE-NEXT:    lxvw4x v2, 0, r4
+; CHECK-P8-BE-NEXT:    addi r3, r3, .LCPI10_0@toc@l
+; CHECK-P8-BE-NEXT:    lxvw4x v3, 0, r3
+; CHECK-P8-BE-NEXT:    vperm v2, v2, v2, v3
+; CHECK-P8-BE-NEXT:    blr
+;
+; CHECK-P9-BE-LABEL: load_swap51:
+; CHECK-P9-BE:       # %bb.0:
+; CHECK-P9-BE-NEXT:    addis r3, r2, .LCPI10_0@toc@ha
+; CHECK-P9-BE-NEXT:    addi r3, r3, .LCPI10_0@toc@l
+; CHECK-P9-BE-NEXT:    lxv v2, 0(r4)
+; CHECK-P9-BE-NEXT:    lxvx v3, 0, r3
+; CHECK-P9-BE-NEXT:    vperm v2, v2, v2, v3
+; CHECK-P9-BE-NEXT:    blr
+  %v1 = load <4 x float>, <4 x float>* %vp1
+  %v2 = load <4 x float>, <4 x float>* %vp2
+  %v3 = shufflevector <4 x float> %v1, <4 x float> %v2, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+  ret <4 x float> %v3
+}
+
+define void @swap_store00(<2 x i64> %v1, <2 x i64> %v2, <2 x i64>* %vp) {
+; CHECK-P8-LABEL: swap_store00:
+; CHECK-P8:       # %bb.0:
+; CHECK-P8-NEXT:    stxvd2x v2, 0, r7
+; CHECK-P8-NEXT:    blr
+;
+; CHECK-P9-LABEL: swap_store00:
+; CHECK-P9:       # %bb.0:
+; CHECK-P9-NEXT:    stxvd2x v2, 0, r7
+; CHECK-P9-NEXT:    blr
+;
+; CHECK-P8-BE-LABEL: swap_store00:
+; CHECK-P8-BE:       # %bb.0:
+; CHECK-P8-BE-NEXT:    xxswapd vs0, v2
+; CHECK-P8-BE-NEXT:    stxvd2x vs0, 0, r7
+; CHECK-P8-BE-NEXT:    blr
+;
+; CHECK-P9-BE-LABEL: swap_store00:
+; CHECK-P9-BE:       # %bb.0:
+; CHECK-P9-BE-NEXT:    xxswapd vs0, v2
+; CHECK-P9-BE-NEXT:    stxv vs0, 0(r7)
+; CHECK-P9-BE-NEXT:    blr
+  %v3 = shufflevector <2 x i64> %v1, <2 x i64> %v2, <2 x i32> <i32 1, i32 0>
+  store <2 x i64> %v3, <2 x i64>* %vp
+  ret void
+}
+
+define void @swap_store01(<2 x i64> %v1, <2 x i64> %v2, <2 x i64>* %vp) {
+; CHECK-P8-LABEL: swap_store01:
+; CHECK-P8:       # %bb.0:
+; CHECK-P8-NEXT:    stxvd2x v3, 0, r7
+; CHECK-P8-NEXT:    blr
+;
+; CHECK-P9-LABEL: swap_store01:
+; CHECK-P9:       # %bb.0:
+; CHECK-P9-NEXT:    stxvd2x v3, 0, r7
+; CHECK-P9-NEXT:    blr
+;
+; CHECK-P8-BE-LABEL: swap_store01:
+; CHECK-P8-BE:       # %bb.0:
+; CHECK-P8-BE-NEXT:    xxswapd vs0, v3
+; CHECK-P8-BE-NEXT:    stxvd2x vs0, 0, r7
+; CHECK-P8-BE-NEXT:    blr
+;
+; CHECK-P9-BE-LABEL: swap_store01:
+; CHECK-P9-BE:       # %bb.0:
+; CHECK-P9-BE-NEXT:    xxswapd vs0, v3
+; CHECK-P9-BE-NEXT:    stxv vs0, 0(r7)
+; CHECK-P9-BE-NEXT:    blr
+  %v3 = shufflevector <2 x i64> %v1, <2 x i64> %v2, <2 x i32> <i32 3, i32 2>
+  store <2 x i64> %v3, <2 x i64>* %vp
+  ret void
+}
+
+define void @swap_store10(<4 x i32> %v1, <4 x i32> %v2, <4 x i32>* %vp) {
+; CHECK-P8-LABEL: swap_store10:
+; CHECK-P8:       # %bb.0:
+; CHECK-P8-NEXT:    addis r3, r2, .LCPI13_0@toc@ha
+; CHECK-P8-NEXT:    addi r3, r3, .LCPI13_0@toc@l
+; CHECK-P8-NEXT:    lvx v3, 0, r3
+; CHECK-P8-NEXT:    vperm v2, v2, v2, v3
+; CHECK-P8-NEXT:    stvx v2, 0, r7
+; CHECK-P8-NEXT:    blr
+;
+; CHECK-P9-LABEL: swap_store10:
+; CHECK-P9:       # %bb.0:
+; CHECK-P9-NEXT:    stxvw4x v2, 0, r7
+; CHECK-P9-NEXT:    blr
+;
+; CHECK-P8-BE-LABEL: swap_store10:
+; CHECK-P8-BE:       # %bb.0:
+; CHECK-P8-BE-NEXT:    addis r3, r2, .LCPI13_0@toc@ha
+; CHECK-P8-BE-NEXT:    addi r3, r3, .LCPI13_0@toc@l
+; CHECK-P8-BE-NEXT:    lxvw4x v3, 0, r3
+; CHECK-P8-BE-NEXT:    vperm v2, v2, v2, v3
+; CHECK-P8-BE-NEXT:    stxvw4x v2, 0, r7
+; CHECK-P8-BE-NEXT:    blr
+;
+; CHECK-P9-BE-LABEL: swap_store10:
+; CHECK-P9-BE:       # %bb.0:
+; CHECK-P9-BE-NEXT:    addis r3, r2, .LCPI13_0@toc@ha
+; CHECK-P9-BE-NEXT:    addi r3, r3, .LCPI13_0@toc@l
+; CHECK-P9-BE-NEXT:    lxvx v3, 0, r3
+; CHECK-P9-BE-NEXT:    vperm v2, v2, v2, v3
+; CHECK-P9-BE-NEXT:    stxv v2, 0(r7)
+; CHECK-P9-BE-NEXT:    blr
+  %v3 = shufflevector <4 x i32> %v1, <4 x i32> %v2, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  store <4 x i32> %v3, <4 x i32>* %vp
+  ret void
+}
+
+define void @swap_store11(<4 x i32> %v1, <4 x i32> %v2, <4 x i32>* %vp) {
+; CHECK-P8-LABEL: swap_store11:
+; CHECK-P8:       # %bb.0:
+; CHECK-P8-NEXT:    addis r3, r2, .LCPI14_0@toc@ha
+; CHECK-P8-NEXT:    addi r3, r3, .LCPI14_0@toc@l
+; CHECK-P8-NEXT:    lvx v2, 0, r3
+; CHECK-P8-NEXT:    vperm v2, v3, v3, v2
+; CHECK-P8-NEXT:    stvx v2, 0, r7
+; CHECK-P8-NEXT:    blr
+;
+; CHECK-P9-LABEL: swap_store11:
+; CHECK-P9:       # %bb.0:
+; CHECK-P9-NEXT:    stxvw4x v3, 0, r7
+; CHECK-P9-NEXT:    blr
+;
+; CHECK-P8-BE-LABEL: swap_store11:
+; CHECK-P8-BE:       # %bb.0:
+; CHECK-P8-BE-NEXT:    addis r3, r2, .LCPI14_0@toc@ha
+; CHECK-P8-BE-NEXT:    addi r3, r3, .LCPI14_0@toc@l
+; CHECK-P8-BE-NEXT:    lxvw4x v2, 0, r3
+; CHECK-P8-BE-NEXT:    vperm v2, v3, v3, v2
+; CHECK-P8-BE-NEXT:    stxvw4x v2, 0, r7
+; CHECK-P8-BE-NEXT:    blr
+;
+; CHECK-P9-BE-LABEL: swap_store11:
+; CHECK-P9-BE:       # %bb.0:
+; CHECK-P9-BE-NEXT:    addis r3, r2, .LCPI14_0@toc@ha
+; CHECK-P9-BE-NEXT:    addi r3, r3, .LCPI14_0@toc@l
+; CHECK-P9-BE-NEXT:    lxvx v2, 0, r3
+; CHECK-P9-BE-NEXT:    vperm v2, v3, v3, v2
+; CHECK-P9-BE-NEXT:    stxv v2, 0(r7)
+; CHECK-P9-BE-NEXT:    blr
+  %v3 = shufflevector <4 x i32> %v1, <4 x i32> %v2, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+  store <4 x i32> %v3, <4 x i32>* %vp
+  ret void
+}
+
+define void @swap_store20(<8 x i16> %v1, <8 x i16> %v2, <8 x i16>* %vp) {
+; CHECK-P8-LABEL: swap_store20:
+; CHECK-P8:       # %bb.0:
+; CHECK-P8-NEXT:    addis r3, r2, .LCPI15_0@toc@ha
+; CHECK-P8-NEXT:    addi r3, r3, .LCPI15_0@toc@l
+; CHECK-P8-NEXT:    lvx v3, 0, r3
+; CHECK-P8-NEXT:    vperm v2, v2, v2, v3
+; CHECK-P8-NEXT:    stvx v2, 0, r7
+; CHECK-P8-NEXT:    blr
+;
+; CHECK-P9-LABEL: swap_store20:
+; CHECK-P9:       # %bb.0:
+; CHECK-P9-NEXT:    stxvh8x v2, 0, r7
+; CHECK-P9-NEXT:    blr
+;
+; CHECK-P8-BE-LABEL: swap_store20:
+; CHECK-P8-BE:       # %bb.0:
+; CHECK-P8-BE-NEXT:    addis r3, r2, .LCPI15_0@toc@ha
+; CHECK-P8-BE-NEXT:    addi r3, r3, .LCPI15_0@toc@l
+; CHECK-P8-BE-NEXT:    lxvw4x v3, 0, r3
+; CHECK-P8-BE-NEXT:    vperm v2, v2, v2, v3
+; CHECK-P8-BE-NEXT:    stxvw4x v2, 0, r7
+; CHECK-P8-BE-NEXT:    blr
+;
+; CHECK-P9-BE-LABEL: swap_store20:
+; CHECK-P9-BE:       # %bb.0:
+; CHECK-P9-BE-NEXT:    addis r3, r2, .LCPI15_0@toc@ha
+; CHECK-P9-BE-NEXT:    addi r3, r3, .LCPI15_0@toc@l
+; CHECK-P9-BE-NEXT:    lxvx v3, 0, r3
+; CHECK-P9-BE-NEXT:    vperm v2, v2, v2, v3
+; CHECK-P9-BE-NEXT:    stxv v2, 0(r7)
+; CHECK-P9-BE-NEXT:    blr
+  %v3 = shufflevector <8 x i16> %v1, <8 x i16> %v2, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  store <8 x i16> %v3, <8 x i16>* %vp
+  ret void
+}
+
+define void @swap_store21(<8 x i16> %v1, <8 x i16> %v2, <8 x i16>* %vp) {
+; CHECK-P8-LABEL: swap_store21:
+; CHECK-P8:       # %bb.0:
+; CHECK-P8-NEXT:    addis r3, r2, .LCPI16_0@toc@ha
+; CHECK-P8-NEXT:    addi r3, r3, .LCPI16_0@toc@l
+; CHECK-P8-NEXT:    lvx v2, 0, r3
+; CHECK-P8-NEXT:    vperm v2, v3, v3, v2
+; CHECK-P8-NEXT:    stvx v2, 0, r7
+; CHECK-P8-NEXT:    blr
+;
+; CHECK-P9-LABEL: swap_store21:
+; CHECK-P9:       # %bb.0:
+; CHECK-P9-NEXT:    stxvh8x v3, 0, r7
+; CHECK-P9-NEXT:    blr
+;
+; CHECK-P8-BE-LABEL: swap_store21:
+; CHECK-P8-BE:       # %bb.0:
+; CHECK-P8-BE-NEXT:    addis r3, r2, .LCPI16_0@toc@ha
+; CHECK-P8-BE-NEXT:    addi r3, r3, .LCPI16_0@toc@l
+; CHECK-P8-BE-NEXT:    lxvw4x v2, 0, r3
+; CHECK-P8-BE-NEXT:    vperm v2, v3, v3, v2
+; CHECK-P8-BE-NEXT:    stxvw4x v2, 0, r7
+; CHECK-P8-BE-NEXT:    blr
+;
+; CHECK-P9-BE-LABEL: swap_store21:
+; CHECK-P9-BE:       # %bb.0:
+; CHECK-P9-BE-NEXT:    addis r3, r2, .LCPI16_0@toc@ha
+; CHECK-P9-BE-NEXT:    addi r3, r3, .LCPI16_0@toc@l
+; CHECK-P9-BE-NEXT:    lxvx v2, 0, r3
+; CHECK-P9-BE-NEXT:    vperm v2, v3, v3, v2
+; CHECK-P9-BE-NEXT:    stxv v2, 0(r7)
+; CHECK-P9-BE-NEXT:    blr
+  %v3 = shufflevector <8 x i16> %v1, <8 x i16> %v2, <8 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8>
+  store <8 x i16> %v3, <8 x i16>* %vp
+  ret void
+}
+
+define void @swap_store30(<16 x i8> %v1, <16 x i8> %v2, <16 x i8>* %vp) {
+; CHECK-P8-LABEL: swap_store30:
+; CHECK-P8:       # %bb.0:
+; CHECK-P8-NEXT:    addis r3, r2, .LCPI17_0@toc@ha
+; CHECK-P8-NEXT:    addi r3, r3, .LCPI17_0@toc@l
+; CHECK-P8-NEXT:    lvx v3, 0, r3
+; CHECK-P8-NEXT:    vperm v2, v2, v2, v3
+; CHECK-P8-NEXT:    stvx v2, 0, r7
+; CHECK-P8-NEXT:    blr
+;
+; CHECK-P9-LABEL: swap_store30:
+; CHECK-P9:       # %bb.0:
+; CHECK-P9-NEXT:    stxvb16x v2, 0, r7
+; CHECK-P9-NEXT:    blr
+;
+; CHECK-P8-BE-LABEL: swap_store30:
+; CHECK-P8-BE:       # %bb.0:
+; CHECK-P8-BE-NEXT:    addis r3, r2, .LCPI17_0@toc@ha
+; CHECK-P8-BE-NEXT:    addi r3, r3, .LCPI17_0@toc@l
+; CHECK-P8-BE-NEXT:    lxvw4x v3, 0, r3
+; CHECK-P8-BE-NEXT:    vperm v2, v2, v2, v3
+; CHECK-P8-BE-NEXT:    stxvw4x v2, 0, r7
+; CHECK-P8-BE-NEXT:    blr
+;
+; CHECK-P9-BE-LABEL: swap_store30:
+; CHECK-P9-BE:       # %bb.0:
+; CHECK-P9-BE-NEXT:    xxbrq vs0, v2
+; CHECK-P9-BE-NEXT:    stxv vs0, 0(r7)
+; CHECK-P9-BE-NEXT:    blr
+  %v3 = shufflevector <16 x i8> %v1, <16 x i8> %v2, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  store <16 x i8> %v3, <16 x i8>* %vp
+  ret void
+}
+
+define void @swap_store31(<16 x i8> %v1, <16 x i8> %v2, <16 x i8>* %vp) {
+; CHECK-P8-LABEL: swap_store31:
+; CHECK-P8:       # %bb.0:
+; CHECK-P8-NEXT:    addis r3, r2, .LCPI18_0@toc@ha
+; CHECK-P8-NEXT:    addi r3, r3, .LCPI18_0@toc@l
+; CHECK-P8-NEXT:    lvx v2, 0, r3
+; CHECK-P8-NEXT:    vperm v2, v3, v3, v2
+; CHECK-P8-NEXT:    stvx v2, 0, r7
+; CHECK-P8-NEXT:    blr
+;
+; CHECK-P9-LABEL: swap_store31:
+; CHECK-P9:       # %bb.0:
+; CHECK-P9-NEXT:    stxvb16x v3, 0, r7
+; CHECK-P9-NEXT:    blr
+;
+; CHECK-P8-BE-LABEL: swap_store31:
+; CHECK-P8-BE:       # %bb.0:
+; CHECK-P8-BE-NEXT:    addis r3, r2, .LCPI18_0@toc@ha
+; CHECK-P8-BE-NEXT:    addi r3, r3, .LCPI18_0@toc@l
+; CHECK-P8-BE-NEXT:    lxvw4x v2, 0, r3
+; CHECK-P8-BE-NEXT:    vperm v2, v3, v3, v2
+; CHECK-P8-BE-NEXT:    stxvw4x v2, 0, r7
+; CHECK-P8-BE-NEXT:    blr
+;
+; CHECK-P9-BE-LABEL: swap_store31:
+; CHECK-P9-BE:       # %bb.0:
+; CHECK-P9-BE-NEXT:    xxbrq vs0, v3
+; CHECK-P9-BE-NEXT:    stxv vs0, 0(r7)
+; CHECK-P9-BE-NEXT:    blr
+  %v3 = shufflevector <16 x i8> %v1, <16 x i8> %v2, <16 x i32> <i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16>
+  store <16 x i8> %v3, <16 x i8>* %vp
+  ret void
+}
+
+define void @swap_store40(<2 x double> %v1, <2 x double> %v2, <2 x double>* %vp) {
+; CHECK-P8-LABEL: swap_store40:
+; CHECK-P8:       # %bb.0:
+; CHECK-P8-NEXT:    stxvd2x v2, 0, r7
+; CHECK-P8-NEXT:    blr
+;
+; CHECK-P9-LABEL: swap_store40:
+; CHECK-P9:       # %bb.0:
+; CHECK-P9-NEXT:    stxvd2x v2, 0, r7
+; CHECK-P9-NEXT:    blr
+;
+; CHECK-P8-BE-LABEL: swap_store40:
+; CHECK-P8-BE:       # %bb.0:
+; CHECK-P8-BE-NEXT:    xxswapd vs0, v2
+; CHECK-P8-BE-NEXT:    stxvd2x vs0, 0, r7
+; CHECK-P8-BE-NEXT:    blr
+;
+; CHECK-P9-BE-LABEL: swap_store40:
+; CHECK-P9-BE:       # %bb.0:
+; CHECK-P9-BE-NEXT:    xxswapd vs0, v2
+; CHECK-P9-BE-NEXT:    stxv vs0, 0(r7)
+; CHECK-P9-BE-NEXT:    blr
+  %v3 = shufflevector <2 x double> %v1, <2 x double> %v2, <2 x i32> <i32 1, i32 0>
+  store <2 x double> %v3, <2 x double>* %vp
+  ret void
+}
+
+define void @swap_store41(<2 x double> %v1, <2 x double> %v2, <2 x double>* %vp) {
+; CHECK-P8-LABEL: swap_store41:
+; CHECK-P8:       # %bb.0:
+; CHECK-P8-NEXT:    stxvd2x v3, 0, r7
+; CHECK-P8-NEXT:    blr
+;
+; CHECK-P9-LABEL: swap_store41:
+; CHECK-P9:       # %bb.0:
+; CHECK-P9-NEXT:    stxvd2x v3, 0, r7
+; CHECK-P9-NEXT:    blr
+;
+; CHECK-P8-BE-LABEL: swap_store41:
+; CHECK-P8-BE:       # %bb.0:
+; CHECK-P8-BE-NEXT:    xxswapd vs0, v3
+; CHECK-P8-BE-NEXT:    stxvd2x vs0, 0, r7
+; CHECK-P8-BE-NEXT:    blr
+;
+; CHECK-P9-BE-LABEL: swap_store41:
+; CHECK-P9-BE:       # %bb.0:
+; CHECK-P9-BE-NEXT:    xxswapd vs0, v3
+; CHECK-P9-BE-NEXT:    stxv vs0, 0(r7)
+; CHECK-P9-BE-NEXT:    blr
+  %v3 = shufflevector <2 x double> %v1, <2 x double> %v2, <2 x i32> <i32 3, i32 2>
+  store <2 x double> %v3, <2 x double>* %vp
+  ret void
+}
+
+define void @swap_store50(<4 x float> %v1, <4 x float> %v2, <4 x float>* %vp) {
+; CHECK-P8-LABEL: swap_store50:
+; CHECK-P8:       # %bb.0:
+; CHECK-P8-NEXT:    addis r3, r2, .LCPI21_0@toc@ha
+; CHECK-P8-NEXT:    addi r3, r3, .LCPI21_0@toc@l
+; CHECK-P8-NEXT:    lvx v3, 0, r3
+; CHECK-P8-NEXT:    vperm v2, v2, v2, v3
+; CHECK-P8-NEXT:    stvx v2, 0, r7
+; CHECK-P8-NEXT:    blr
+;
+; CHECK-P9-LABEL: swap_store50:
+; CHECK-P9:       # %bb.0:
+; CHECK-P9-NEXT:    stxvw4x v2, 0, r7
+; CHECK-P9-NEXT:    blr
+;
+; CHECK-P8-BE-LABEL: swap_store50:
+; CHECK-P8-BE:       # %bb.0:
+; CHECK-P8-BE-NEXT:    addis r3, r2, .LCPI21_0@toc@ha
+; CHECK-P8-BE-NEXT:    addi r3, r3, .LCPI21_0@toc@l
+; CHECK-P8-BE-NEXT:    lxvw4x v3, 0, r3
+; CHECK-P8-BE-NEXT:    vperm v2, v2, v2, v3
+; CHECK-P8-BE-NEXT:    stxvw4x v2, 0, r7
+; CHECK-P8-BE-NEXT:    blr
+;
+; CHECK-P9-BE-LABEL: swap_store50:
+; CHECK-P9-BE:       # %bb.0:
+; CHECK-P9-BE-NEXT:    addis r3, r2, .LCPI21_0@toc@ha
+; CHECK-P9-BE-NEXT:    addi r3, r3, .LCPI21_0@toc@l
+; CHECK-P9-BE-NEXT:    lxvx v3, 0, r3
+; CHECK-P9-BE-NEXT:    vperm v2, v2, v2, v3
+; CHECK-P9-BE-NEXT:    stxv v2, 0(r7)
+; CHECK-P9-BE-NEXT:    blr
+  %v3 = shufflevector <4 x float> %v1, <4 x float> %v2, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  store <4 x float> %v3, <4 x float>* %vp
+  ret void
+}
+
+define void @swap_store51(<4 x float> %v1, <4 x float> %v2, <4 x float>* %vp) {
+; CHECK-P8-LABEL: swap_store51:
+; CHECK-P8:       # %bb.0:
+; CHECK-P8-NEXT:    addis r3, r2, .LCPI22_0@toc@ha
+; CHECK-P8-NEXT:    addi r3, r3, .LCPI22_0@toc@l
+; CHECK-P8-NEXT:    lvx v2, 0, r3
+; CHECK-P8-NEXT:    vperm v2, v3, v3, v2
+; CHECK-P8-NEXT:    stvx v2, 0, r7
+; CHECK-P8-NEXT:    blr
+;
+; CHECK-P9-LABEL: swap_store51:
+; CHECK-P9:       # %bb.0:
+; CHECK-P9-NEXT:    stxvw4x v3, 0, r7
+; CHECK-P9-NEXT:    blr
+;
+; CHECK-P8-BE-LABEL: swap_store51:
+; CHECK-P8-BE:       # %bb.0:
+; CHECK-P8-BE-NEXT:    addis r3, r2, .LCPI22_0@toc@ha
+; CHECK-P8-BE-NEXT:    addi r3, r3, .LCPI22_0@toc@l
+; CHECK-P8-BE-NEXT:    lxvw4x v2, 0, r3
+; CHECK-P8-BE-NEXT:    vperm v2, v3, v3, v2
+; CHECK-P8-BE-NEXT:    stxvw4x v2, 0, r7
+; CHECK-P8-BE-NEXT:    blr
+;
+; CHECK-P9-BE-LABEL: swap_store51:
+; CHECK-P9-BE:       # %bb.0:
+; CHECK-P9-BE-NEXT:    addis r3, r2, .LCPI22_0@toc@ha
+; CHECK-P9-BE-NEXT:    addi r3, r3, .LCPI22_0@toc@l
+; CHECK-P9-BE-NEXT:    lxvx v2, 0, r3
+; CHECK-P9-BE-NEXT:    vperm v2, v3, v3, v2
+; CHECK-P9-BE-NEXT:    stxv v2, 0(r7)
+; CHECK-P9-BE-NEXT:    blr
+  %v3 = shufflevector <4 x float> %v1, <4 x float> %v2, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+  store <4 x float> %v3, <4 x float>* %vp
+  ret void
+}
diff --git a/llvm/test/CodeGen/PowerPC/loop-instr-form-prepare.ll b/llvm/test/CodeGen/PowerPC/loop-instr-form-prepare.ll
new file mode 100644
index 000000000..42a2c7828
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/loop-instr-form-prepare.ll
@@ -0,0 +1,827 @@
+; RUN: llc -ppc-asm-full-reg-names -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -mcpu=pwr9 < %s | FileCheck %s
+
+; test_no_prep:
+; unsigned long test_no_prep(char *p, int count) {
+;   unsigned long i=0, res=0;
+;   int DISP1 = 4001;
+;   int DISP2 = 4002;
+;   int DISP3 = 4003;
+;   int DISP4 = 4004;
+;   for (; i < count ; i++) {
+;     unsigned long x1 = *(unsigned long *)(p + i + DISP1);
+;     unsigned long x2 = *(unsigned long *)(p + i + DISP2);
+;     unsigned long x3 = *(unsigned long *)(p + i + DISP3);
+;     unsigned long x4 = *(unsigned long *)(p + i + DISP4);
+;     res += x1*x2*x3*x4;
+;   }
+;   return res + count;
+; }
+
+define i64 @test_no_prep(i8* %0, i32 signext %1) {
+; CHECK-LABEL: test_no_prep:
+; CHECK:         addi r3, r3, 4004
+; CHECK:       .LBB0_2: #
+; CHECK-NEXT:    ldx r9, r3, r6
+; CHECK-NEXT:    ldx r10, r3, r7
+; CHECK-NEXT:    mulld r9, r10, r9
+; CHECK-NEXT:    ldx r11, r3, r8
+; CHECK-NEXT:    mulld r9, r9, r11
+; CHECK-NEXT:    ld r12, 0(r3)
+; CHECK-NEXT:    addi r3, r3, 1
+; CHECK-NEXT:    maddld r5, r9, r12, r5
+; CHECK-NEXT:    bdnz .LBB0_2
+  %3 = sext i32 %1 to i64
+  %4 = icmp eq i32 %1, 0
+  br i1 %4, label %27, label %5
+
+5:                                                ; preds = %2, %5
+  %6 = phi i64 [ %25, %5 ], [ 0, %2 ]
+  %7 = phi i64 [ %24, %5 ], [ 0, %2 ]
+  %8 = getelementptr inbounds i8, i8* %0, i64 %6
+  %9 = getelementptr inbounds i8, i8* %8, i64 4001
+  %10 = bitcast i8* %9 to i64*
+  %11 = load i64, i64* %10, align 8
+  %12 = getelementptr inbounds i8, i8* %8, i64 4002
+  %13 = bitcast i8* %12 to i64*
+  %14 = load i64, i64* %13, align 8
+  %15 = getelementptr inbounds i8, i8* %8, i64 4003
+  %16 = bitcast i8* %15 to i64*
+  %17 = load i64, i64* %16, align 8
+  %18 = getelementptr inbounds i8, i8* %8, i64 4004
+  %19 = bitcast i8* %18 to i64*
+  %20 = load i64, i64* %19, align 8
+  %21 = mul i64 %14, %11
+  %22 = mul i64 %21, %17
+  %23 = mul i64 %22, %20
+  %24 = add i64 %23, %7
+  %25 = add nuw i64 %6, 1
+  %26 = icmp ult i64 %25, %3
+  br i1 %26, label %5, label %27
+
+27:                                               ; preds = %5, %2
+  %28 = phi i64 [ 0, %2 ], [ %24, %5 ]
+  %29 = add i64 %28, %3
+  ret i64 %29
+}
+
+; test_ds_prep:
+; unsigned long test_ds_prep(char *p, int count) {
+;   unsigned long i=0, res=0;
+;   int DISP1 = 4001;
+;   int DISP2 = 4002;
+;   int DISP3 = 4003;
+;   int DISP4 = 4006;
+;   for (; i < count ; i++) {
+;     unsigned long x1 = *(unsigned long *)(p + i + DISP1);
+;     unsigned long x2 = *(unsigned long *)(p + i + DISP2);
+;     unsigned long x3 = *(unsigned long *)(p + i + DISP3);
+;     unsigned long x4 = *(unsigned long *)(p + i + DISP4);
+;     res += x1*x2*x3*x4;
+;   }
+;   return res + count;
+; }
+
+define i64 @test_ds_prep(i8* %0, i32 signext %1) {
+; CHECK-LABEL: test_ds_prep:
+; CHECK:         addi r6, r3, 4002
+; CHECK:       .LBB1_2: #
+; CHECK-NEXT:    ldx r9, r6, r7
+; CHECK-NEXT:    ld r10, 0(r6)
+; CHECK-NEXT:    mulld r9, r10, r9
+; CHECK-NEXT:    ldx r11, r6, r5
+; CHECK-NEXT:    mulld r9, r9, r11
+; CHECK-NEXT:    addi r8, r6, 1
+; CHECK-NEXT:    ld r6, 4(r6)
+; CHECK-NEXT:    maddld r3, r9, r6, r3
+; CHECK-NEXT:    mr r6, r8
+; CHECK-NEXT:    bdnz .LBB1_2
+  %3 = sext i32 %1 to i64
+  %4 = icmp eq i32 %1, 0
+  br i1 %4, label %27, label %5
+
+5:                                                ; preds = %2, %5
+  %6 = phi i64 [ %25, %5 ], [ 0, %2 ]
+  %7 = phi i64 [ %24, %5 ], [ 0, %2 ]
+  %8 = getelementptr inbounds i8, i8* %0, i64 %6
+  %9 = getelementptr inbounds i8, i8* %8, i64 4001
+  %10 = bitcast i8* %9 to i64*
+  %11 = load i64, i64* %10, align 8
+  %12 = getelementptr inbounds i8, i8* %8, i64 4002
+  %13 = bitcast i8* %12 to i64*
+  %14 = load i64, i64* %13, align 8
+  %15 = getelementptr inbounds i8, i8* %8, i64 4003
+  %16 = bitcast i8* %15 to i64*
+  %17 = load i64, i64* %16, align 8
+  %18 = getelementptr inbounds i8, i8* %8, i64 4006
+  %19 = bitcast i8* %18 to i64*
+  %20 = load i64, i64* %19, align 8
+  %21 = mul i64 %14, %11
+  %22 = mul i64 %21, %17
+  %23 = mul i64 %22, %20
+  %24 = add i64 %23, %7
+  %25 = add nuw i64 %6, 1
+  %26 = icmp ult i64 %25, %3
+  br i1 %26, label %5, label %27
+
+27:                                               ; preds = %5, %2
+  %28 = phi i64 [ 0, %2 ], [ %24, %5 ]
+  %29 = add i64 %28, %3
+  ret i64 %29
+}
+
+; test_max_number_reminder:
+; unsigned long test_max_number_reminder(char *p, int count) {
+;  unsigned long i=0, res=0;
+;  int DISP1 = 4001;
+;  int DISP2 = 4002;
+;  int DISP3 = 4003;
+;  int DISP4 = 4005;
+;  int DISP5 = 4006;
+;  int DISP6 = 4007;
+;  int DISP7 = 4014;
+;  int DISP8 = 4010;
+;  int DISP9 = 4011;
+;  for (; i < count ; i++) {
+;    unsigned long x1 = *(unsigned long *)(p + i + DISP1);
+;    unsigned long x2 = *(unsigned long *)(p + i + DISP2);
+;    unsigned long x3 = *(unsigned long *)(p + i + DISP3);
+;    unsigned long x4 = *(unsigned long *)(p + i + DISP4);
+;    unsigned long x5 = *(unsigned long *)(p + i + DISP5);
+;    unsigned long x6 = *(unsigned long *)(p + i + DISP6);
+;    unsigned long x7 = *(unsigned long *)(p + i + DISP7);
+;    unsigned long x8 = *(unsigned long *)(p + i + DISP8);
+;    unsigned long x9 = *(unsigned long *)(p + i + DISP9);
+;    res += x1*x2*x3*x4*x5*x6*x7*x8*x9;
+;  }
+;  return res + count;
+;}
+
+define i64 @test_max_number_reminder(i8* %0, i32 signext %1) {
+; CHECK-LABEL: test_max_number_reminder:
+; CHECK:         addi r9, r3, 4002
+; CHECK:       .LBB2_2: #
+; CHECK-NEXT:    ldx r12, r9, r6
+; CHECK-NEXT:    ld r0, 0(r9)
+; CHECK-NEXT:    mulld r12, r0, r12
+; CHECK-NEXT:    addi r11, r9, 1
+; CHECK-NEXT:    ldx r30, r9, r7
+; CHECK-NEXT:    ld r29, 4(r9)
+; CHECK-NEXT:    ldx r28, r9, r8
+; CHECK-NEXT:    ld r27, 12(r9)
+; CHECK-NEXT:    ld r26, 8(r9)
+; CHECK-NEXT:    ldx r25, r9, r10
+; CHECK-NEXT:    ldx r9, r9, r5
+; CHECK-NEXT:    mulld r9, r12, r9
+; CHECK-NEXT:    mulld r9, r9, r30
+; CHECK-NEXT:    mulld r9, r9, r29
+; CHECK-NEXT:    mulld r9, r9, r28
+; CHECK-NEXT:    mulld r9, r9, r27
+; CHECK-NEXT:    mulld r9, r9, r26
+; CHECK-NEXT:    maddld r3, r9, r25, r3
+; CHECK-NEXT:    mr r9, r11
+; CHECK-NEXT:    bdnz .LBB2_2
+  %3 = sext i32 %1 to i64
+  %4 = icmp eq i32 %1, 0
+  br i1 %4, label %47, label %5
+
+5:                                                ; preds = %2, %5
+  %6 = phi i64 [ %45, %5 ], [ 0, %2 ]
+  %7 = phi i64 [ %44, %5 ], [ 0, %2 ]
+  %8 = getelementptr inbounds i8, i8* %0, i64 %6
+  %9 = getelementptr inbounds i8, i8* %8, i64 4001
+  %10 = bitcast i8* %9 to i64*
+  %11 = load i64, i64* %10, align 8
+  %12 = getelementptr inbounds i8, i8* %8, i64 4002
+  %13 = bitcast i8* %12 to i64*
+  %14 = load i64, i64* %13, align 8
+  %15 = getelementptr inbounds i8, i8* %8, i64 4003
+  %16 = bitcast i8* %15 to i64*
+  %17 = load i64, i64* %16, align 8
+  %18 = getelementptr inbounds i8, i8* %8, i64 4005
+  %19 = bitcast i8* %18 to i64*
+  %20 = load i64, i64* %19, align 8
+  %21 = getelementptr inbounds i8, i8* %8, i64 4006
+  %22 = bitcast i8* %21 to i64*
+  %23 = load i64, i64* %22, align 8
+  %24 = getelementptr inbounds i8, i8* %8, i64 4007
+  %25 = bitcast i8* %24 to i64*
+  %26 = load i64, i64* %25, align 8
+  %27 = getelementptr inbounds i8, i8* %8, i64 4014
+  %28 = bitcast i8* %27 to i64*
+  %29 = load i64, i64* %28, align 8
+  %30 = getelementptr inbounds i8, i8* %8, i64 4010
+  %31 = bitcast i8* %30 to i64*
+  %32 = load i64, i64* %31, align 8
+  %33 = getelementptr inbounds i8, i8* %8, i64 4011
+  %34 = bitcast i8* %33 to i64*
+  %35 = load i64, i64* %34, align 8
+  %36 = mul i64 %14, %11
+  %37 = mul i64 %36, %17
+  %38 = mul i64 %37, %20
+  %39 = mul i64 %38, %23
+  %40 = mul i64 %39, %26
+  %41 = mul i64 %40, %29
+  %42 = mul i64 %41, %32
+  %43 = mul i64 %42, %35
+  %44 = add i64 %43, %7
+  %45 = add nuw i64 %6, 1
+  %46 = icmp ult i64 %45, %3
+  br i1 %46, label %5, label %47
+
+47:                                               ; preds = %5, %2
+  %48 = phi i64 [ 0, %2 ], [ %44, %5 ]
+  %49 = add i64 %48, %3
+  ret i64 %49
+}
+
+; test_update_ds_prep_interact:
+; unsigned long test_update_ds_prep_interact(char *p, int count) {
+;   unsigned long i=0, res=0;
+;   int DISP1 = 4001;
+;   int DISP2 = 4002;
+;   int DISP3 = 4003;
+;   int DISP4 = 4006;
+;   for (; i < count ; i++) {
+;     unsigned long x1 = *(unsigned long *)(p + 4 * i + DISP1);
+;     unsigned long x2 = *(unsigned long *)(p + 4 * i + DISP2);
+;     unsigned long x3 = *(unsigned long *)(p + 4 * i + DISP3);
+;     unsigned long x4 = *(unsigned long *)(p + 4 * i + DISP4);
+;     res += x1*x2*x3*x4;
+;   }
+;   return res + count;
+; }
+
+define dso_local i64 @test_update_ds_prep_interact(i8* %0, i32 signext %1) {
+; CHECK-LABEL: test_update_ds_prep_interact:
+; CHECK:         addi r3, r3, 3998
+; CHECK:       .LBB3_2: #
+; CHECK-NEXT:    ldu r8, 4(r3)
+; CHECK-NEXT:    ldx r9, r3, r7
+; CHECK-NEXT:    mulld r8, r8, r9
+; CHECK-NEXT:    ldx r10, r3, r6
+; CHECK-NEXT:    mulld r8, r8, r10
+; CHECK-NEXT:    ld r11, 4(r3)
+; CHECK-NEXT:    maddld r5, r8, r11, r5
+; CHECK-NEXT:    bdnz .LBB3_2
+  %3 = sext i32 %1 to i64
+  %4 = icmp eq i32 %1, 0
+  br i1 %4, label %28, label %5
+
+5:                                                ; preds = %2, %5
+  %6 = phi i64 [ %26, %5 ], [ 0, %2 ]
+  %7 = phi i64 [ %25, %5 ], [ 0, %2 ]
+  %8 = shl i64 %6, 2
+  %9 = getelementptr inbounds i8, i8* %0, i64 %8
+  %10 = getelementptr inbounds i8, i8* %9, i64 4001
+  %11 = bitcast i8* %10 to i64*
+  %12 = load i64, i64* %11, align 8
+  %13 = getelementptr inbounds i8, i8* %9, i64 4002
+  %14 = bitcast i8* %13 to i64*
+  %15 = load i64, i64* %14, align 8
+  %16 = getelementptr inbounds i8, i8* %9, i64 4003
+  %17 = bitcast i8* %16 to i64*
+  %18 = load i64, i64* %17, align 8
+  %19 = getelementptr inbounds i8, i8* %9, i64 4006
+  %20 = bitcast i8* %19 to i64*
+  %21 = load i64, i64* %20, align 8
+  %22 = mul i64 %15, %12
+  %23 = mul i64 %22, %18
+  %24 = mul i64 %23, %21
+  %25 = add i64 %24, %7
+  %26 = add nuw i64 %6, 1
+  %27 = icmp ult i64 %26, %3
+  br i1 %27, label %5, label %28
+
+28:                                               ; preds = %5, %2
+  %29 = phi i64 [ 0, %2 ], [ %25, %5 ]
+  %30 = add i64 %29, %3
+  ret i64 %30
+}
+
+; test_update_ds_prep_nointeract:
+; unsigned long test_update_ds_prep_nointeract(char *p, int count) {
+;   unsigned long i=0, res=0;
+;   int DISP1 = 4001;
+;   int DISP2 = 4002;
+;   int DISP3 = 4003;
+;   int DISP4 = 4007;
+;   for (; i < count ; i++) {
+;     char x1 = *(p + i + DISP1);
+;     unsigned long x2 = *(unsigned long *)(p + i + DISP2);
+;     unsigned long x3 = *(unsigned long *)(p + i + DISP3);
+;     unsigned long x4 = *(unsigned long *)(p + i + DISP4);
+;     res += (unsigned long)x1*x2*x3*x4;
+;   }
+;   return res + count;
+; }
+
+define i64 @test_update_ds_prep_nointeract(i8* %0, i32 signext %1) {
+; CHECK-LABEL: test_update_ds_prep_nointeract:
+; CHECK:         addi r5, r3, 4000
+; CHECK:         addi r3, r3, 4003
+; CHECK:       .LBB4_2: #
+; CHECK-NEXT:    lbzu r8, 1(r5)
+; CHECK-NEXT:    ldx r9, r3, r7
+; CHECK-NEXT:    ld r10, 0(r3)
+; CHECK-NEXT:    ld r11, 4(r3)
+; CHECK-NEXT:    addi r3, r3, 1
+; CHECK-NEXT:    mulld r8, r9, r8
+; CHECK-NEXT:    mulld r8, r8, r10
+; CHECK-NEXT:    maddld r6, r8, r11, r6
+; CHECK-NEXT:    bdnz .LBB4_2
+  %3 = sext i32 %1 to i64
+  %4 = icmp eq i32 %1, 0
+  br i1 %4, label %27, label %5
+
+5:                                                ; preds = %2, %5
+  %6 = phi i64 [ %25, %5 ], [ 0, %2 ]
+  %7 = phi i64 [ %24, %5 ], [ 0, %2 ]
+  %8 = getelementptr inbounds i8, i8* %0, i64 %6
+  %9 = getelementptr inbounds i8, i8* %8, i64 4001
+  %10 = load i8, i8* %9, align 1
+  %11 = getelementptr inbounds i8, i8* %8, i64 4002
+  %12 = bitcast i8* %11 to i64*
+  %13 = load i64, i64* %12, align 8
+  %14 = getelementptr inbounds i8, i8* %8, i64 4003
+  %15 = bitcast i8* %14 to i64*
+  %16 = load i64, i64* %15, align 8
+  %17 = getelementptr inbounds i8, i8* %8, i64 4007
+  %18 = bitcast i8* %17 to i64*
+  %19 = load i64, i64* %18, align 8
+  %20 = zext i8 %10 to i64
+  %21 = mul i64 %13, %20
+  %22 = mul i64 %21, %16
+  %23 = mul i64 %22, %19
+  %24 = add i64 %23, %7
+  %25 = add nuw i64 %6, 1
+  %26 = icmp ult i64 %25, %3
+  br i1 %26, label %5, label %27
+
+27:                                               ; preds = %5, %2
+  %28 = phi i64 [ 0, %2 ], [ %24, %5 ]
+  %29 = add i64 %28, %3
+  ret i64 %29
+}
+
+; test_ds_multiple_chains:
+; unsigned long test_ds_multiple_chains(char *p, char *q, int count) {
+;   unsigned long i=0, res=0;
+;   int DISP1 = 4001;
+;   int DISP2 = 4010;
+;   int DISP3 = 4005;
+;   int DISP4 = 4009;
+;   for (; i < count ; i++) {
+;     unsigned long x1 = *(unsigned long *)(p + i + DISP1);
+;     unsigned long x2 = *(unsigned long *)(p + i + DISP2);
+;     unsigned long x3 = *(unsigned long *)(p + i + DISP3);
+;     unsigned long x4 = *(unsigned long *)(p + i + DISP4);
+;     unsigned long x5 = *(unsigned long *)(q + i + DISP1);
+;     unsigned long x6 = *(unsigned long *)(q + i + DISP2);
+;     unsigned long x7 = *(unsigned long *)(q + i + DISP3);
+;     unsigned long x8 = *(unsigned long *)(q + i + DISP4);
+;     res += x1*x2*x3*x4*x5*x6*x7*x8;
+;   }
+;   return res + count;
+; }
+
+define dso_local i64 @test_ds_multiple_chains(i8* %0, i8* %1, i32 signext %2) {
+; CHECK-LABEL: test_ds_multiple_chains:
+; CHECK:         addi r3, r3, 4001
+; CHECK:         addi r4, r4, 4001
+; CHECK:       .LBB5_2: #
+; CHECK-NEXT:    ld r8, 0(r3)
+; CHECK-NEXT:    ldx r9, r3, r7
+; CHECK-NEXT:    mulld r8, r9, r8
+; CHECK-NEXT:    ld r9, 4(r3)
+; CHECK-NEXT:    mulld r8, r8, r9
+; CHECK-NEXT:    ld r10, 8(r3)
+; CHECK-NEXT:    addi r3, r3, 1
+; CHECK-NEXT:    mulld r8, r8, r10
+; CHECK-NEXT:    ld r11, 0(r4)
+; CHECK-NEXT:    mulld r8, r8, r11
+; CHECK-NEXT:    ldx r12, r4, r7
+; CHECK-NEXT:    mulld r8, r8, r12
+; CHECK-NEXT:    ld r0, 4(r4)
+; CHECK-NEXT:    mulld r8, r8, r0
+; CHECK-NEXT:    ld r30, 8(r4)
+; CHECK-NEXT:    addi r4, r4, 1
+; CHECK-NEXT:    maddld r6, r8, r30, r6
+; CHECK-NEXT:    bdnz .LBB5_2
+  %4 = sext i32 %2 to i64
+  %5 = icmp eq i32 %2, 0
+  br i1 %5, label %45, label %6
+
+6:                                                ; preds = %3, %6
+  %7 = phi i64 [ %43, %6 ], [ 0, %3 ]
+  %8 = phi i64 [ %42, %6 ], [ 0, %3 ]
+  %9 = getelementptr inbounds i8, i8* %0, i64 %7
+  %10 = getelementptr inbounds i8, i8* %9, i64 4001
+  %11 = bitcast i8* %10 to i64*
+  %12 = load i64, i64* %11, align 8
+  %13 = getelementptr inbounds i8, i8* %9, i64 4010
+  %14 = bitcast i8* %13 to i64*
+  %15 = load i64, i64* %14, align 8
+  %16 = getelementptr inbounds i8, i8* %9, i64 4005
+  %17 = bitcast i8* %16 to i64*
+  %18 = load i64, i64* %17, align 8
+  %19 = getelementptr inbounds i8, i8* %9, i64 4009
+  %20 = bitcast i8* %19 to i64*
+  %21 = load i64, i64* %20, align 8
+  %22 = getelementptr inbounds i8, i8* %1, i64 %7
+  %23 = getelementptr inbounds i8, i8* %22, i64 4001
+  %24 = bitcast i8* %23 to i64*
+  %25 = load i64, i64* %24, align 8
+  %26 = getelementptr inbounds i8, i8* %22, i64 4010
+  %27 = bitcast i8* %26 to i64*
+  %28 = load i64, i64* %27, align 8
+  %29 = getelementptr inbounds i8, i8* %22, i64 4005
+  %30 = bitcast i8* %29 to i64*
+  %31 = load i64, i64* %30, align 8
+  %32 = getelementptr inbounds i8, i8* %22, i64 4009
+  %33 = bitcast i8* %32 to i64*
+  %34 = load i64, i64* %33, align 8
+  %35 = mul i64 %15, %12
+  %36 = mul i64 %35, %18
+  %37 = mul i64 %36, %21
+  %38 = mul i64 %37, %25
+  %39 = mul i64 %38, %28
+  %40 = mul i64 %39, %31
+  %41 = mul i64 %40, %34
+  %42 = add i64 %41, %8
+  %43 = add nuw i64 %7, 1
+  %44 = icmp ult i64 %43, %4
+  br i1 %44, label %6, label %45
+
+45:                                               ; preds = %6, %3
+  %46 = phi i64 [ 0, %3 ], [ %42, %6 ]
+  %47 = add i64 %46, %4
+  ret i64 %47
+}
+
+; test_ds_cross_basic_blocks:
+;extern char *arr;
+;unsigned long foo(char *p, int count)
+;{
+;  unsigned long i=0, res=0;
+;  int DISP1 = 4000;
+;  int DISP2 = 4001;
+;  int DISP3 = 4002;
+;  int DISP4 = 4003;
+;  int DISP5 = 4005;
+;  int DISP6 = 4009;
+;  unsigned long x1, x2, x3, x4, x5, x6;
+;  x1=x2=x3=x4=x5=x6=1;
+;  for (; i < count ; i++) {
+;    if (arr[i] % 3 == 1) {
+;      x1 += *(unsigned long *)(p + i + DISP1);
+;      x2 += *(unsigned long *)(p + i + DISP2);
+;    }
+;    else if (arr[i] % 3 == 2) {
+;      x3 += *(unsigned long *)(p + i + DISP3);
+;      x4 += *(unsigned long *)(p + i + DISP5);
+;    }
+;    else {
+;      x5 += *(unsigned long *)(p + i + DISP4);
+;      x6 += *(unsigned long *)(p + i + DISP6);
+;    }
+;    res += x1*x2*x3*x4*x5*x6;
+;  }
+;  return res;
+;}
+
+@arr = external local_unnamed_addr global i8*, align 8
+
+define i64 @test_ds_cross_basic_blocks(i8* %0, i32 signext %1) {
+; CHECK-LABEL: test_ds_cross_basic_blocks:
+; CHECK:         addi r6, r3, 4009
+; CHECK:       .LBB6_2: #
+; CHECK-NEXT:    ldx r0, r6, r8
+; CHECK-NEXT:    add r28, r0, r28
+; CHECK-NEXT:    ld r0, -8(r6)
+; CHECK-NEXT:    add r29, r0, r29
+; CHECK-NEXT:  .LBB6_3: #
+; CHECK-NEXT:    mulld r0, r29, r28
+; CHECK-NEXT:    mulld r0, r0, r30
+; CHECK-NEXT:    mulld r0, r0, r12
+; CHECK-NEXT:    mulld r0, r0, r11
+; CHECK-NEXT:    maddld r3, r0, r7, r3
+; CHECK-NEXT:    addi r6, r6, 1
+; CHECK-NEXT:    bdz .LBB6_9
+; CHECK-NEXT:  .LBB6_4: #
+; CHECK-NEXT:    lbzu r0, 1(r5)
+; CHECK-NEXT:    clrldi r27, r0, 32
+; CHECK-NEXT:    mulld r27, r27, r4
+; CHECK-NEXT:    rldicl r27, r27, 31, 33
+; CHECK-NEXT:    slwi r26, r27, 1
+; CHECK-NEXT:    add r27, r27, r26
+; CHECK-NEXT:    subf r0, r27, r0
+; CHECK-NEXT:    cmplwi r0, 1
+; CHECK-NEXT:    beq cr0, .LBB6_2
+; CHECK-NEXT:  # %bb.5: #
+; CHECK-NEXT:    clrlwi r0, r0, 24
+; CHECK-NEXT:    cmplwi r0, 2
+; CHECK-NEXT:    bne cr0, .LBB6_7
+; CHECK-NEXT:  # %bb.6: #
+; CHECK-NEXT:    ldx r0, r6, r9
+; CHECK-NEXT:    add r30, r0, r30
+; CHECK-NEXT:    ld r0, -4(r6)
+; CHECK-NEXT:    add r12, r0, r12
+; CHECK-NEXT:    b .LBB6_3
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  .LBB6_7: #
+; CHECK-NEXT:    ldx r0, r6, r10
+; CHECK-NEXT:    add r11, r0, r11
+; CHECK-NEXT:    ld r0, 0(r6)
+; CHECK-NEXT:    add r7, r0, r7
+  %3 = sext i32 %1 to i64
+  %4 = icmp eq i32 %1, 0
+  br i1 %4, label %66, label %5
+
+5:                                                ; preds = %2
+  %6 = load i8*, i8** @arr, align 8
+  br label %7
+
+7:                                                ; preds = %5, %51
+  %8 = phi i64 [ 1, %5 ], [ %57, %51 ]
+  %9 = phi i64 [ 1, %5 ], [ %56, %51 ]
+  %10 = phi i64 [ 1, %5 ], [ %55, %51 ]
+  %11 = phi i64 [ 1, %5 ], [ %54, %51 ]
+  %12 = phi i64 [ 1, %5 ], [ %53, %51 ]
+  %13 = phi i64 [ 1, %5 ], [ %52, %51 ]
+  %14 = phi i64 [ 0, %5 ], [ %64, %51 ]
+  %15 = phi i64 [ 0, %5 ], [ %63, %51 ]
+  %16 = getelementptr inbounds i8, i8* %6, i64 %14
+  %17 = load i8, i8* %16, align 1
+  %18 = urem i8 %17, 3
+  %19 = icmp eq i8 %18, 1
+  br i1 %19, label %20, label %30
+
+20:                                               ; preds = %7
+  %21 = getelementptr inbounds i8, i8* %0, i64 %14
+  %22 = getelementptr inbounds i8, i8* %21, i64 4000
+  %23 = bitcast i8* %22 to i64*
+  %24 = load i64, i64* %23, align 8
+  %25 = add i64 %24, %13
+  %26 = getelementptr inbounds i8, i8* %21, i64 4001
+  %27 = bitcast i8* %26 to i64*
+  %28 = load i64, i64* %27, align 8
+  %29 = add i64 %28, %12
+  br label %51
+
+30:                                               ; preds = %7
+  %31 = icmp eq i8 %18, 2
+  %32 = getelementptr inbounds i8, i8* %0, i64 %14
+  br i1 %31, label %33, label %42
+
+33:                                               ; preds = %30
+  %34 = getelementptr inbounds i8, i8* %32, i64 4002
+  %35 = bitcast i8* %34 to i64*
+  %36 = load i64, i64* %35, align 8
+  %37 = add i64 %36, %11
+  %38 = getelementptr inbounds i8, i8* %32, i64 4005
+  %39 = bitcast i8* %38 to i64*
+  %40 = load i64, i64* %39, align 8
+  %41 = add i64 %40, %10
+  br label %51
+
+42:                                               ; preds = %30
+  %43 = getelementptr inbounds i8, i8* %32, i64 4003
+  %44 = bitcast i8* %43 to i64*
+  %45 = load i64, i64* %44, align 8
+  %46 = add i64 %45, %9
+  %47 = getelementptr inbounds i8, i8* %32, i64 4009
+  %48 = bitcast i8* %47 to i64*
+  %49 = load i64, i64* %48, align 8
+  %50 = add i64 %49, %8
+  br label %51
+
+51:                                               ; preds = %33, %42, %20
+  %52 = phi i64 [ %25, %20 ], [ %13, %33 ], [ %13, %42 ]
+  %53 = phi i64 [ %29, %20 ], [ %12, %33 ], [ %12, %42 ]
+  %54 = phi i64 [ %11, %20 ], [ %37, %33 ], [ %11, %42 ]
+  %55 = phi i64 [ %10, %20 ], [ %41, %33 ], [ %10, %42 ]
+  %56 = phi i64 [ %9, %20 ], [ %9, %33 ], [ %46, %42 ]
+  %57 = phi i64 [ %8, %20 ], [ %8, %33 ], [ %50, %42 ]
+  %58 = mul i64 %53, %52
+  %59 = mul i64 %58, %54
+  %60 = mul i64 %59, %55
+  %61 = mul i64 %60, %56
+  %62 = mul i64 %61, %57
+  %63 = add i64 %62, %15
+  %64 = add nuw i64 %14, 1
+  %65 = icmp ult i64 %64, %3
+  br i1 %65, label %7, label %66
+
+66:                                               ; preds = %51, %2
+  %67 = phi i64 [ 0, %2 ], [ %63, %51 ]
+  ret i64 %67
+}
+
+; test_ds_float:
+;float test_ds_float(char *p, int count) {
+;  int i=0 ;
+;  float res=0;
+;  int DISP1 = 4001;
+;  int DISP2 = 4002;
+;  int DISP3 = 4022;
+;  int DISP4 = 4062;
+;  for (; i < count ; i++) {
+;    float x1 = *(float *)(p + i + DISP1);
+;    float x2 = *(float *)(p + i + DISP2);
+;    float x3 = *(float *)(p + i + DISP3);
+;    float x4 = *(float *)(p + i + DISP4);
+;    res += x1*x2*x3*x4;
+;  }
+;  return res;
+;}
+
+define float @test_ds_float(i8* %0, i32 signext %1) {
+; CHECK-LABEL: test_ds_float:
+; CHECK:         addi r3, r3, 4002
+; CHECK:       .LBB7_2: #
+; CHECK-NEXT:    lfsx f0, r3, r4
+; CHECK-NEXT:    lfs f2, 0(r3)
+; CHECK-NEXT:    xsmulsp f0, f0, f2
+; CHECK-NEXT:    lfs f3, 20(r3)
+; CHECK-NEXT:    xsmulsp f0, f0, f3
+; CHECK-NEXT:    lfs f4, 60(r3)
+; CHECK-NEXT:    addi r3, r3, 1
+; CHECK-NEXT:    xsmulsp f0, f0, f4
+; CHECK-NEXT:    xsaddsp f1, f1, f0
+; CHECK-NEXT:    bdnz .LBB7_2
+  %3 = icmp sgt i32 %1, 0
+  br i1 %3, label %4, label %28
+
+4:                                                ; preds = %2
+  %5 = zext i32 %1 to i64
+  br label %6
+
+6:                                                ; preds = %6, %4
+  %7 = phi i64 [ 0, %4 ], [ %26, %6 ]
+  %8 = phi float [ 0.000000e+00, %4 ], [ %25, %6 ]
+  %9 = getelementptr inbounds i8, i8* %0, i64 %7
+  %10 = getelementptr inbounds i8, i8* %9, i64 4001
+  %11 = bitcast i8* %10 to float*
+  %12 = load float, float* %11, align 4
+  %13 = getelementptr inbounds i8, i8* %9, i64 4002
+  %14 = bitcast i8* %13 to float*
+  %15 = load float, float* %14, align 4
+  %16 = getelementptr inbounds i8, i8* %9, i64 4022
+  %17 = bitcast i8* %16 to float*
+  %18 = load float, float* %17, align 4
+  %19 = getelementptr inbounds i8, i8* %9, i64 4062
+  %20 = bitcast i8* %19 to float*
+  %21 = load float, float* %20, align 4
+  %22 = fmul float %12, %15
+  %23 = fmul float %22, %18
+  %24 = fmul float %23, %21
+  %25 = fadd float %8, %24
+  %26 = add nuw nsw i64 %7, 1
+  %27 = icmp eq i64 %26, %5
+  br i1 %27, label %28, label %6
+
+28:                                               ; preds = %6, %2
+  %29 = phi float [ 0.000000e+00, %2 ], [ %25, %6 ]
+  ret float %29
+}
+
+; test_ds_combine_float_int:
+;float test_ds_combine_float_int(char *p, int count) {
+;  int i=0 ;
+;  float res=0;
+;  int DISP1 = 4001;
+;  int DISP2 = 4002;
+;  int DISP3 = 4022;
+;  int DISP4 = 4062;
+;  for (; i < count ; i++) {
+;    float x1 = *(float *)(p + i + DISP1);
+;    unsigned long x2 = *(unsigned long*)(p + i + DISP2);
+;    float x3 = *(float *)(p + i + DISP3);
+;    float x4 = *(float *)(p + i + DISP4);
+;    res += x1*x2*x3*x4;
+;  }
+;  return res;
+;}
+
+define float @test_ds_combine_float_int(i8* %0, i32 signext %1) {
+; CHECK-LABEL: test_ds_combine_float_int:
+; CHECK:         addi r3, r3, 4002
+; CHECK:       .LBB8_2: #
+; CHECK-NEXT:    lfd f4, 0(r3)
+; CHECK-NEXT:    lfsx f0, r3, r4
+; CHECK-NEXT:    xscvuxdsp f4, f4
+; CHECK-NEXT:    lfs f2, 20(r3)
+; CHECK-NEXT:    xsmulsp f0, f0, f4
+; CHECK-NEXT:    xsmulsp f0, f2, f0
+; CHECK-NEXT:    lfs f3, 60(r3)
+; CHECK-NEXT:    addi r3, r3, 1
+; CHECK-NEXT:    xsmulsp f0, f3, f0
+; CHECK-NEXT:    xsaddsp f1, f1, f0
+; CHECK-NEXT:    bdnz .LBB8_2
+  %3 = icmp sgt i32 %1, 0
+  br i1 %3, label %4, label %29
+
+4:                                                ; preds = %2
+  %5 = zext i32 %1 to i64
+  br label %6
+
+6:                                                ; preds = %6, %4
+  %7 = phi i64 [ 0, %4 ], [ %27, %6 ]
+  %8 = phi float [ 0.000000e+00, %4 ], [ %26, %6 ]
+  %9 = getelementptr inbounds i8, i8* %0, i64 %7
+  %10 = getelementptr inbounds i8, i8* %9, i64 4001
+  %11 = bitcast i8* %10 to float*
+  %12 = load float, float* %11, align 4
+  %13 = getelementptr inbounds i8, i8* %9, i64 4002
+  %14 = bitcast i8* %13 to i64*
+  %15 = load i64, i64* %14, align 8
+  %16 = getelementptr inbounds i8, i8* %9, i64 4022
+  %17 = bitcast i8* %16 to float*
+  %18 = load float, float* %17, align 4
+  %19 = getelementptr inbounds i8, i8* %9, i64 4062
+  %20 = bitcast i8* %19 to float*
+  %21 = load float, float* %20, align 4
+  %22 = uitofp i64 %15 to float
+  %23 = fmul float %12, %22
+  %24 = fmul float %18, %23
+  %25 = fmul float %21, %24
+  %26 = fadd float %8, %25
+  %27 = add nuw nsw i64 %7, 1
+  %28 = icmp eq i64 %27, %5
+  br i1 %28, label %29, label %6
+
+29:                                               ; preds = %6, %2
+  %30 = phi float [ 0.000000e+00, %2 ], [ %26, %6 ]
+  ret float %30
+}
+
+; test_ds_lwa_prep:
+; long long test_ds_lwa_prep(char *p, int count) {
+;   long long i=0, res=0;
+;   int DISP1 = 4001;
+;   int DISP2 = 4002;
+;   int DISP3 = 4006;
+;   int DISP4 = 4010;
+;   for (; i < count ; i++) {
+;     long long x1 = *(int *)(p + i + DISP1);
+;     long long x2 = *(int *)(p + i + DISP2);
+;     long long x3 = *(int *)(p + i + DISP3);
+;     long long x4 = *(int *)(p + i + DISP4);
+;     res += x1*x2*x3*x4;
+;   }
+;   return res + count;
+; }
+
+define i64 @test_ds_lwa_prep(i8* %0, i32 signext %1) {
+; CHECK-LABEL: test_ds_lwa_prep:
+; CHECK:         addi r5, r3, 2
+; CHECK:         li r6, -1
+; CHECK:       .LBB9_2: #
+; CHECK-NEXT:    lwax r7, r5, r6
+; CHECK-NEXT:    lwa r8, 0(r5)
+; CHECK-NEXT:    lwa r9, 4(r5)
+; CHECK-NEXT:    lwa r10, 8(r5)
+; CHECK-NEXT:    addi r5, r5, 1
+; CHECK-NEXT:    mulld r7, r8, r7
+; CHECK-NEXT:    mulld r7, r7, r9
+; CHECK-NEXT:    maddld r3, r7, r10, r3
+; CHECK-NEXT:    bdnz .LBB9_2
+
+  %3 = sext i32 %1 to i64
+  %4 = icmp sgt i32 %1, 0
+  br i1 %4, label %5, label %31
+
+5:                                                ; preds = %2, %5
+  %6 = phi i64 [ %29, %5 ], [ 0, %2 ]
+  %7 = phi i64 [ %28, %5 ], [ 0, %2 ]
+  %8 = getelementptr inbounds i8, i8* %0, i64 %6
+  %9 = getelementptr inbounds i8, i8* %8, i64 1
+  %10 = bitcast i8* %9 to i32*
+  %11 = load i32, i32* %10, align 4
+  %12 = sext i32 %11 to i64
+  %13 = getelementptr inbounds i8, i8* %8, i64 2
+  %14 = bitcast i8* %13 to i32*
+  %15 = load i32, i32* %14, align 4
+  %16 = sext i32 %15 to i64
+  %17 = getelementptr inbounds i8, i8* %8, i64 6
+  %18 = bitcast i8* %17 to i32*
+  %19 = load i32, i32* %18, align 4
+  %20 = sext i32 %19 to i64
+  %21 = getelementptr inbounds i8, i8* %8, i64 10
+  %22 = bitcast i8* %21 to i32*
+  %23 = load i32, i32* %22, align 4
+  %24 = sext i32 %23 to i64
+  %25 = mul nsw i64 %16, %12
+  %26 = mul nsw i64 %25, %20
+  %27 = mul nsw i64 %26, %24
+  %28 = add nsw i64 %27, %7
+  %29 = add nuw nsw i64 %6, 1
+  %30 = icmp eq i64 %29, %3
+  br i1 %30, label %31, label %5
+
+31:                                               ; preds = %5, %2
+  %32 = phi i64 [ 0, %2 ], [ %28, %5 ]
+  %33 = add nsw i64 %32, %3
+  ret i64 %33
+}
+
diff --git a/llvm/test/CodeGen/PowerPC/lower-globaladdr32-aix-asm.ll b/llvm/test/CodeGen/PowerPC/lower-globaladdr32-aix-asm.ll
new file mode 100644
index 000000000..e3254175d
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/lower-globaladdr32-aix-asm.ll
@@ -0,0 +1,45 @@
+; RUN: llc -verify-machineinstrs -mcpu=pwr7 -mtriple powerpc-ibm-aix-xcoff \
+; RUN: -code-model=small < %s | FileCheck %s --check-prefixes=CHECK,SMALL
+
+; RUN: llc -verify-machineinstrs -mcpu=pwr7 -mtriple powerpc-ibm-aix-xcoff \
+; RUN: -code-model=large < %s | FileCheck %s --check-prefixes=CHECK,LARGE
+
+@a = common global i32 0
+
+define i32 @test_load() {
+entry:
+  %0 = load i32, i32* @a
+  ret i32 %0
+}
+
+; SMALL-LABEL: .test_load:{{$}}
+; SMALL: lwz [[REG1:[0-9]+]], LC0(2)
+; SMALL: lwz [[REG2:[0-9]+]], 0([[REG1]])
+; SMALL: blr
+
+; LARGE-LABEL: .test_load:{{$}}
+; LARGE: addis [[REG1:[0-9]+]], LC0@u(2)
+; LARGE: lwz [[REG2:[0-9]+]], LC0@l([[REG1]])
+; LARGE: lwz [[REG3:[0-9]+]], 0([[REG2]])
+; LARGE: blr
+
+@b = common global i32 0
+
+define void @test_store(i32 %0) {
+  store i32 %0, i32* @b
+  ret void
+}
+
+; SMALL-LABEL: .test_store:{{$}}
+; SMALL: lwz [[REG1:[0-9]+]], LC1(2)
+; SMALL: stw [[REG2:[0-9]+]], 0([[REG1]])
+; SMALL: blr
+
+; LARGE-LABEL: .test_store:{{$}}
+; LARGE: addis [[REG1:[0-9]+]], LC1@u(2)
+; LARGE: lwz [[REG2:[0-9]+]], LC1@l([[REG1]])
+; LARGE: stw [[REG3:[0-9]+]], 0([[REG2]])
+; LARGE: blr
+
+; CHECK: .tc a[TC],a
+; CHECK: .tc b[TC],b
diff --git a/llvm/test/CodeGen/PowerPC/lower-globaladdr32-aix.ll b/llvm/test/CodeGen/PowerPC/lower-globaladdr32-aix.ll
new file mode 100644
index 000000000..3f02d4f6c
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/lower-globaladdr32-aix.ll
@@ -0,0 +1,38 @@
+; RUN: llc -mtriple powerpc-ibm-aix-xcoff -code-model=small \
+; RUN: -stop-after=machine-cp -print-before=simple-register-coalescing 2>&1 < \
+; RUN: %s | FileCheck --check-prefix=SMALL %s
+
+; RUN: not llc -mtriple powerpc-ibm-aix-xcoff -code-model=medium \
+; RUN: -stop-after=machine-cp 2>&1 < %s | FileCheck --check-prefix=MEDIUM %s
+
+; RUN: llc -mtriple powerpc-ibm-aix-xcoff -code-model=large \
+; RUN: -stop-after=machine-cp -print-before=simple-register-coalescing 2>&1 < \
+; RUN: %s | FileCheck --check-prefix=LARGE %s
+
+; RUN: llc -mtriple powerpc-ibm-aix-xcoff -stop-after=machine-cp \
+; RUN: -print-before=simple-register-coalescing 2>&1 < %s | FileCheck \
+; RUN: --check-prefix=SMALL %s
+
+@msg = common global i8* null, align 4
+@ptr = common global i8* null, align 4
+
+define void @foo() {
+entry:
+; SMALL: %0:gprc_and_gprc_nor0 = LWZtoc @msg, $r2 :: (load 4 from got)
+; SMALL: %1:gprc = LWZ 0, %0:gprc_and_gprc_nor0 :: (dereferenceable load 4 from @msg)
+; SMALL: %2:gprc_and_gprc_nor0 = LWZtoc @ptr, $r2 :: (load 4 from got)
+; SMALL: STW %1:gprc, 0, %2:gprc_and_gprc_nor0 :: (store 4 into @ptr)
+
+; MEDIUM: Medium code model is not supported on AIX.
+
+; LARGE: %0:gprc_and_gprc_nor0 = ADDIStocHA $r2, @msg
+; LARGE: %1:gprc_and_gprc_nor0 = LWZtocL @msg, %0:gprc_and_gprc_nor0, implicit $r2 :: (load 4 from got)
+; LARGE: %2:gprc = LWZ 0, %1:gprc_and_gprc_nor0 :: (dereferenceable load 4 from @msg)
+; LARGE: %3:gprc_and_gprc_nor0 = ADDIStocHA $r2, @ptr
+; LARGE: %4:gprc_and_gprc_nor0 = LWZtocL @ptr, %3:gprc_and_gprc_nor0, implicit $r2 :: (load 4 from got)
+; LARGE: STW %2:gprc, 0, %4:gprc_and_gprc_nor0 :: (store 4 into @ptr)
+
+  %0 = load i8*, i8** @msg, align 4
+  store i8* %0, i8** @ptr, align 4
+  ret void
+}
diff --git a/llvm/test/CodeGen/PowerPC/lower-globaladdr64-aix-asm.ll b/llvm/test/CodeGen/PowerPC/lower-globaladdr64-aix-asm.ll
new file mode 100644
index 000000000..6d1863bc9
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/lower-globaladdr64-aix-asm.ll
@@ -0,0 +1,45 @@
+; RUN: llc -verify-machineinstrs -mcpu=pwr7 -mtriple powerpc64-ibm-aix-xcoff \
+; RUN: -code-model=small < %s | FileCheck %s --check-prefixes=CHECK,SMALL
+
+; RUN: llc -verify-machineinstrs -mcpu=pwr7 -mtriple powerpc64-ibm-aix-xcoff \
+; RUN: -code-model=large < %s | FileCheck %s --check-prefixes=CHECK,LARGE
+
+@a = common global i32 0
+
+define zeroext i32 @test_load() {
+entry:
+  %0 = load i32, i32* @a
+  ret i32 %0
+}
+
+; SMALL-LABEL: .test_load:{{$}}
+; SMALL: ld [[REG1:[0-9]+]], LC0(2)
+; SMALL: lwz [[REG2:[0-9]+]], 0([[REG1]])
+; SMALL: blr
+
+; LARGE-LABEL: .test_load:{{$}}
+; LARGE: addis [[REG1:[0-9]+]], LC0@u(2)
+; LARGE: ld [[REG2:[0-9]+]], LC0@l([[REG1]])
+; LARGE: lwz [[REG3:[0-9]+]], 0([[REG2]])
+; LARGE: blr
+
+@b = common global i32 0
+
+define void @test_store(i32 zeroext %0) {
+  store i32 %0, i32* @b
+  ret void
+}
+
+; SMALL-LABEL: .test_store:{{$}}
+; SMALL: ld [[REG1:[0-9]+]], LC1(2)
+; SMALL: stw [[REG2:[0-9]+]], 0([[REG1]])
+; SMALL: blr
+
+; LARGE-LABEL: .test_store:{{$}}
+; LARGE: addis [[REG1:[0-9]+]], LC1@u(2)
+; LARGE: ld [[REG2:[0-9]+]], LC1@l([[REG1]])
+; LARGE: stw [[REG3:[0-9]+]], 0([[REG2]])
+; LARGE: blr
+
+; CHECK: .tc a[TC],a
+; CHECK: .tc b[TC],b
diff --git a/llvm/test/CodeGen/PowerPC/lower-globaladdr64-aix.ll b/llvm/test/CodeGen/PowerPC/lower-globaladdr64-aix.ll
new file mode 100644
index 000000000..7e00a595c
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/lower-globaladdr64-aix.ll
@@ -0,0 +1,38 @@
+; RUN: llc -mtriple powerpc64-ibm-aix-xcoff -code-model=small \
+; RUN: -stop-after=machine-cp -print-before=simple-register-coalescing 2>&1 < \
+; RUN: %s | FileCheck --check-prefix=SMALL %s
+
+; RUN: not llc -mtriple powerpc64-ibm-aix-xcoff -code-model=medium \
+; RUN: -stop-after=machine-cp 2>&1 < %s | FileCheck --check-prefix=MEDIUM %s
+
+; RUN: llc -mtriple powerpc64-ibm-aix-xcoff -code-model=large \
+; RUN: -stop-after=machine-cp -print-before=simple-register-coalescing 2>&1 < \
+; RUN: %s | FileCheck --check-prefix=LARGE %s
+
+; RUN: llc -mtriple powerpc64-ibm-aix-xcoff -stop-after=machine-cp \
+; RUN: -print-before=simple-register-coalescing 2>&1 < %s | FileCheck \
+; RUN: --check-prefix=SMALL %s
+
+@msg = common global i8* null, align 8
+@ptr = common global i8* null, align 8
+
+define void @foo() {
+entry:
+; SMALL: %0:g8rc_and_g8rc_nox0 = LDtoc @msg, $x2 :: (load 8 from got)
+; SMALL: %1:g8rc = LD 0, %0:g8rc_and_g8rc_nox0 :: (dereferenceable load 8 from @msg)
+; SMALL: %2:g8rc_and_g8rc_nox0 = LDtoc @ptr, $x2 :: (load 8 from got)
+; SMALL: STD %1:g8rc, 0, %2:g8rc_and_g8rc_nox0 :: (store 8 into @ptr)
+
+; MEDIUM: Medium code model is not supported on AIX.
+
+; LARGE: %0:g8rc_and_g8rc_nox0 = ADDIStocHA8 $x2, @msg
+; LARGE: %1:g8rc_and_g8rc_nox0 = LDtocL @msg, %0:g8rc_and_g8rc_nox0, implicit $x2 :: (load 8 from got)
+; LARGE: %2:g8rc = LD 0, %1:g8rc_and_g8rc_nox0 :: (dereferenceable load 8 from @msg)
+; LARGE: %3:g8rc_and_g8rc_nox0 = ADDIStocHA8 $x2, @ptr
+; LARGE: %4:g8rc_and_g8rc_nox0 = LDtocL @ptr, %3:g8rc_and_g8rc_nox0, implicit $x2 :: (load 8 from got)
+; LARGE: STD %2:g8rc, 0, %4:g8rc_and_g8rc_nox0 :: (store 8 into @ptr)
+
+  %0 = load i8*, i8** @msg, align 8
+  store i8* %0, i8** @ptr, align 8
+  ret void
+}
diff --git a/llvm/test/CodeGen/PowerPC/lower-massv-attr.ll b/llvm/test/CodeGen/PowerPC/lower-massv-attr.ll
new file mode 100644
index 000000000..65e178431
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/lower-massv-attr.ll
@@ -0,0 +1,29 @@
+; RUN: llc -verify-machineinstrs -mcpu=pwr9  < %s -mtriple=powerpc64le-unknown-linux-gnu -mattr=-power9-vector | FileCheck -check-prefixes=CHECK-PWR8,CHECK-ALL %s 
+; RUN: llc -verify-machineinstrs -mcpu=pwr8  < %s -mtriple=powerpc64le-unknown-linux-gnu -mattr=+power9-vector | FileCheck -check-prefixes=CHECK-PWR9,CHECK-ALL %s 
+
+declare <2 x double> @__cbrtd2_massv(<2 x double>)
+declare <4 x float> @__cbrtf4_massv(<4 x float>)
+
+; cbrt without the power9-vector attribute on the caller
+; check massv calls are correctly targeted for Power8
+define <2 x double>  @cbrt_f64_massv_nopwr9(<2 x double> %opnd) #0 {
+; CHECK-ALL-LABEL: @cbrt_f64_massv_nopwr9
+; CHECK-PWR8: bl __cbrtd2_P8
+; CHECK-NOT: bl __cbrtd2_massv
+; CHECK-ALL: blr
+;
+  %1 = call <2 x double> @__cbrtd2_massv(<2 x double> %opnd)
+  ret <2 x double> %1 
+}
+
+; cbrt with the power9-vector attribute on the caller
+; check massv calls are correctly targeted for Power9
+define <2 x double>  @cbrt_f64_massv_pwr9(<2 x double> %opnd) #1 {
+; CHECK-ALL-LABEL: @cbrt_f64_massv_pwr9
+; CHECK-PWR9: bl __cbrtd2_P9
+; CHECK-NOT: bl __cbrtd2_massv
+; CHECK-ALL: blr
+;
+  %1 = call <2 x double> @__cbrtd2_massv(<2 x double> %opnd)
+  ret <2 x double> %1 
+}
diff --git a/llvm/test/CodeGen/PowerPC/lower-massv.ll b/llvm/test/CodeGen/PowerPC/lower-massv.ll
new file mode 100644
index 000000000..89ee6e25e
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/lower-massv.ll
@@ -0,0 +1,603 @@
+; RUN: llc -verify-machineinstrs -mcpu=pwr9  < %s -mtriple=powerpc64le-unknown-linux-gnu | FileCheck -check-prefixes=CHECK-PWR9,CHECK-ALL %s 
+; RUN: llc -verify-machineinstrs -mcpu=pwr8  < %s -mtriple=powerpc64le-unknown-linux-gnu | FileCheck -check-prefixes=CHECK-PWR8,CHECK-ALL %s 
+; RUN: llc -verify-machineinstrs -mcpu=pwr8  < %s -mtriple=powerpc64le-unknown-linux-gnu | FileCheck -check-prefixes=CHECK-DFLT,CHECK-ALL %s 
+
+declare <2 x double> @__cbrtd2_massv(<2 x double>)
+declare <4 x float> @__cbrtf4_massv(<4 x float>)
+
+declare <2 x double> @__powd2_massv(<2 x double>, <2 x double>)
+declare <4 x float> @__powf4_massv(<4 x float>, <4 x float>)
+
+declare <2 x double> @__sqrtd2_massv(<2 x double>)
+declare <4 x float> @__sqrtf4_massv(<4 x float>)
+
+declare <2 x double> @__expd2_massv(<2 x double>)
+declare <4 x float> @__expf4_massv(<4 x float>)
+
+declare <2 x double> @__exp2d2_massv(<2 x double>)
+declare <4 x float> @__exp2f4_massv(<4 x float>)
+
+declare <2 x double> @__expm1d2_massv(<2 x double>)
+declare <4 x float> @__expm1f4_massv(<4 x float>)
+
+declare <2 x double> @__logd2_massv(<2 x double>)
+declare <4 x float> @__logf4_massv(<4 x float>)
+
+declare <2 x double> @__log1pd2_massv(<2 x double>)
+declare <4 x float> @__log1pf4_massv(<4 x float>)
+
+declare <2 x double> @__log10d2_massv(<2 x double>)
+declare <4 x float> @__log10f4_massv(<4 x float>)
+
+declare <2 x double> @__log2d2_massv(<2 x double>)
+declare <4 x float> @__log2f4_massv(<4 x float>)
+
+declare <2 x double> @__sind2_massv(<2 x double>)
+declare <4 x float> @__sinf4_massv(<4 x float>)
+
+declare <2 x double> @__cosd2_massv(<2 x double>)
+declare <4 x float> @__cosf4_massv(<4 x float>)
+
+declare <2 x double> @__tand2_massv(<2 x double>)
+declare <4 x float> @__tanf4_massv(<4 x float>)
+
+declare <2 x double> @__asind2_massv(<2 x double>)
+declare <4 x float> @__asinf4_massv(<4 x float>)
+
+declare <2 x double> @__acosd2_massv(<2 x double>)
+declare <4 x float> @__acosf4_massv(<4 x float>)
+
+declare <2 x double> @__atand2_massv(<2 x double>)
+declare <4 x float> @__atanf4_massv(<4 x float>)
+
+declare <2 x double> @__atan2d2_massv(<2 x double>)
+declare <4 x float> @__atan2f4_massv(<4 x float>)
+
+declare <2 x double> @__sinhd2_massv(<2 x double>)
+declare <4 x float> @__sinhf4_massv(<4 x float>)
+
+declare <2 x double> @__coshd2_massv(<2 x double>)
+declare <4 x float> @__coshf4_massv(<4 x float>)
+
+declare <2 x double> @__tanhd2_massv(<2 x double>)
+declare <4 x float> @__tanhf4_massv(<4 x float>)
+
+declare <2 x double> @__asinhd2_massv(<2 x double>)
+declare <4 x float> @__asinhf4_massv(<4 x float>)
+
+declare <2 x double> @__acoshd2_massv(<2 x double>)
+declare <4 x float> @__acoshf4_massv(<4 x float>)
+
+declare <2 x double> @__atanhd2_massv(<2 x double>)
+declare <4 x float> @__atanhf4_massv(<4 x float>)
+
+; following tests check generation of subtarget-specific calls
+; cbrt
+define <2 x double>  @cbrt_f64_massv(<2 x double> %opnd) {
+; CHECK-ALL-LABEL: @cbrt_f64_massv
+; CHECK-PWR9: bl __cbrtd2_P9
+; CHECK-PWR8: bl __cbrtd2_P8
+; CHECK-NOT: bl __cbrtd2_massv
+; CHECK-ALL: blr
+;
+  %1 = call <2 x double> @__cbrtd2_massv(<2 x double> %opnd)
+  ret <2 x double> %1 
+}
+
+define <4 x float>  @cbrt_f32_massv(<4 x float> %opnd) {
+; CHECK-ALL-LABEL: @cbrt_f32_massv
+; CHECK-PWR9: bl __cbrtf4_P9
+; CHECK-PWR8: bl __cbrtf4_P8
+; CHECK-NOT: bl __cbrtf4_massv
+; CHECK-ALL: blr
+;
+  %1 = call <4 x float> @__cbrtf4_massv(<4 x float> %opnd)
+  ret <4 x float> %1 
+}
+
+; pow
+define <2 x double>  @pow_f64_massv(<2 x double> %opnd1, <2 x double> %opnd2) {
+; CHECK-ALL-LABEL: @pow_f64_massv
+; CHECK-PWR9: bl __powd2_P9
+; CHECK-PWR8: bl __powd2_P8
+; CHECK-NOT: bl __powd2_massv
+; CHECK-ALL: blr
+;
+ %1 = call <2 x double> @__powd2_massv(<2 x double> %opnd1, <2 x double> %opnd2)
+  ret <2 x double> %1 
+}
+
+define <4 x float>  @pow_f32_massv(<4 x float> %opnd1, <4 x float> %opnd2) {
+; CHECK-ALL-LABEL: @pow_f32_massv
+; CHECK-PWR9: bl __powf4_P9
+; CHECK-PWR8: bl __powf4_P8
+; CHECK-NOT: bl __powf4_massv
+; CHECK-ALL: blr
+;
+ %1 = call <4 x float> @__powf4_massv(<4 x float> %opnd1, <4 x float> %opnd2)
+  ret <4 x float> %1 
+}
+
+; sqrt
+define <2 x double>  @sqrt_f64_massv(<2 x double> %opnd) {
+; CHECK-ALL-LABEL: @sqrt_f64_massv
+; CHECK-PWR9: bl __sqrtd2_P9
+; CHECK-PWR8: bl __sqrtd2_P8
+; CHECK-NOT: bl __sqrtd2_massv
+; CHECK-ALL: blr
+;
+  %1 = call <2 x double> @__sqrtd2_massv(<2 x double> %opnd)
+  ret <2 x double> %1 
+}
+
+define <4 x float>  @sqrt_f32_massv(<4 x float> %opnd) {
+; CHECK-ALL-LABEL: @sqrt_f32_massv
+; CHECK-PWR9: bl __sqrtf4_P9
+; CHECK-PWR8: bl __sqrtf4_P8
+; CHECK-NOT: bl __sqrtf4_massv
+; CHECK-ALL: blr
+;
+  %1 = call <4 x float> @__sqrtf4_massv(<4 x float> %opnd)
+  ret <4 x float> %1 
+}
+
+; exp
+define <2 x double>  @exp_f64_massv(<2 x double> %opnd) {
+; CHECK-ALL-LABEL: @exp_f64_massv
+; CHECK-PWR9: bl __expd2_P9
+; CHECK-PWR8: bl __expd2_P8
+; CHECK-NOT: bl __expd2_massv
+; CHECK-ALL: blr
+;
+  %1 = call <2 x double> @__expd2_massv(<2 x double> %opnd)
+  ret <2 x double> %1 
+}
+
+define <4 x float>  @exp_f32_massv(<4 x float> %opnd) {
+; CHECK-ALL-LABEL: @exp_f32_massv
+; CHECK-PWR9: bl __expf4_P9
+; CHECK-PWR8: bl __expf4_P8
+; CHECK-NOT: bl __expf4_massv
+; CHECK-ALL: blr
+;
+  %1 = call <4 x float> @__expf4_massv(<4 x float> %opnd)
+  ret <4 x float> %1 
+}
+
+; exp2
+define <2 x double>  @exp2_f64_massv(<2 x double> %opnd) {
+; CHECK-ALL-LABEL: @exp2_f64_massv
+; CHECK-PWR9: bl __exp2d2_P9
+; CHECK-PWR8: bl __exp2d2_P8
+; CHECK-NOT: bl __exp2d2_massv
+; CHECK-ALL: blr
+;
+  %1 = call <2 x double> @__exp2d2_massv(<2 x double> %opnd)
+  ret <2 x double> %1 
+}
+
+define <4 x float>  @exp2_f32_massv(<4 x float> %opnd) {
+; CHECK-ALL-LABEL: @exp2_f32_massv
+; CHECK-PWR9: bl __exp2f4_P9
+; CHECK-PWR8: bl __exp2f4_P8
+; CHECK-NOT: bl __exp2f4_massv
+; CHECK-ALL: blr
+;
+  %1 = call <4 x float> @__exp2f4_massv(<4 x float> %opnd)
+  ret <4 x float> %1 
+}
+
+; expm1
+define <2 x double>  @expm1_f64_massv(<2 x double> %opnd) {
+; CHECK-ALL-LABEL: @expm1_f64_massv
+; CHECK-PWR9: bl __expm1d2_P9
+; CHECK-PWR8: bl __expm1d2_P8
+; CHECK-NOT: bl __expm1d2_massv
+; CHECK-ALL: blr
+;
+  %1 = call <2 x double> @__expm1d2_massv(<2 x double> %opnd)
+  ret <2 x double> %1 
+}
+
+define <4 x float>  @expm1_f32_massv(<4 x float> %opnd) {
+; CHECK-ALL-LABEL: @expm1_f32_massv
+; CHECK-PWR9: bl __expm1f4_P9
+; CHECK-PWR8: bl __expm1f4_P8
+; CHECK-NOT: bl __expm1f4_massv
+; CHECK-ALL: blr
+;
+  %1 = call <4 x float> @__expm1f4_massv(<4 x float> %opnd)
+  ret <4 x float> %1 
+}
+
+; log
+define <2 x double>  @log_f64_massv(<2 x double> %opnd) {
+; CHECK-ALL-LABEL: @log_f64_massv
+; CHECK-PWR9: bl __logd2_P9
+; CHECK-PWR8: bl __logd2_P8
+; CHECK-NOT: bl __logd2_massv
+; CHECK-ALL: blr
+;
+  %1 = call <2 x double> @__logd2_massv(<2 x double> %opnd)
+  ret <2 x double> %1 
+}
+
+define <4 x float>  @log_f32_massv(<4 x float> %opnd) {
+; CHECK-ALL-LABEL: @log_f32_massv
+; CHECK-PWR9: bl __logf4_P9
+; CHECK-PWR8: bl __logf4_P8
+; CHECK-NOT: bl __logf4_massv
+; CHECK-ALL: blr
+;
+  %1 = call <4 x float> @__logf4_massv(<4 x float> %opnd)
+  ret <4 x float> %1 
+}
+
+; log1p
+define <2 x double>  @log1p_f64_massv(<2 x double> %opnd) {
+; CHECK-ALL-LABEL: @log1p_f64_massv
+; CHECK-PWR9: bl __log1pd2_P9
+; CHECK-PWR8: bl __log1pd2_P8
+; CHECK-NOT: bl __log1pd2_massv
+; CHECK-ALL: blr
+;
+  %1 = call <2 x double> @__log1pd2_massv(<2 x double> %opnd)
+  ret <2 x double> %1 
+}
+
+define <4 x float>  @log1p_f32_massv(<4 x float> %opnd) {
+; CHECK-ALL-LABEL: @log1p_f32_massv
+; CHECK-PWR9: bl __log1pf4_P9
+; CHECK-PWR8: bl __log1pf4_P8
+; CHECK-NOT: bl __log1pf4_massv
+; CHECK-ALL: blr
+;
+  %1 = call <4 x float> @__log1pf4_massv(<4 x float> %opnd)
+  ret <4 x float> %1 
+}
+
+; log10
+define <2 x double>  @log10_f64_massv(<2 x double> %opnd) {
+; CHECK-ALL-LABEL: @log10_f64_massv
+; CHECK-PWR9: bl __log10d2_P9
+; CHECK-PWR8: bl __log10d2_P8
+; CHECK-NOT: bl __log10d2_massv
+; CHECK-ALL: blr
+;
+  %1 = call <2 x double> @__log10d2_massv(<2 x double> %opnd)
+  ret <2 x double> %1 
+}
+
+define <4 x float>  @log10_f32_massv(<4 x float> %opnd) {
+; CHECK-ALL-LABEL: @log10_f32_massv
+; CHECK-PWR9: bl __log10f4_P9
+; CHECK-PWR8: bl __log10f4_P8
+; CHECK-NOT: bl __log10f4_massv
+; CHECK-ALL: blr
+;
+  %1 = call <4 x float> @__log10f4_massv(<4 x float> %opnd)
+  ret <4 x float> %1 
+}
+
+; log2
+define <2 x double>  @log2_f64_massv(<2 x double> %opnd) {
+; CHECK-ALL-LABEL: @log2_f64_massv
+; CHECK-PWR9: bl __log2d2_P9
+; CHECK-PWR8: bl __log2d2_P8
+; CHECK-NOT: bl __log2d2_massv
+; CHECK-ALL: blr
+;
+  %1 = call <2 x double> @__log2d2_massv(<2 x double> %opnd)
+  ret <2 x double> %1 
+}
+
+define <4 x float>  @log2_f32_massv(<4 x float> %opnd) {
+; CHECK-ALL-LABEL: @log2_f32_massv
+; CHECK-PWR9: bl __log2f4_P9
+; CHECK-PWR8: bl __log2f4_P8
+; CHECK-NOT: bl __log2f4_massv
+; CHECK-ALL: blr
+;
+  %1 = call <4 x float> @__log2f4_massv(<4 x float> %opnd)
+  ret <4 x float> %1 
+}
+
+; sin
+define <2 x double>  @sin_f64_massv(<2 x double> %opnd) {
+; CHECK-ALL-LABEL: @sin_f64_massv
+; CHECK-PWR9: bl __sind2_P9
+; CHECK-PWR8: bl __sind2_P8
+; CHECK-NOT: bl __sind2_massv
+; CHECK-ALL: blr
+;
+  %1 = call <2 x double> @__sind2_massv(<2 x double> %opnd)
+  ret <2 x double> %1 
+}
+
+define <4 x float>  @sin_f32_massv(<4 x float> %opnd) {
+; CHECK-ALL-LABEL: @sin_f32_massv
+; CHECK-PWR9: bl __sinf4_P9
+; CHECK-PWR8: bl __sinf4_P8
+; CHECK-NOT: bl __sinf4_massv
+; CHECK-ALL: blr
+;
+  %1 = call <4 x float> @__sinf4_massv(<4 x float> %opnd)
+  ret <4 x float> %1 
+}
+
+; cos
+define <2 x double>  @cos_f64_massv(<2 x double> %opnd) {
+; CHECK-ALL-LABEL: @cos_f64_massv
+; CHECK-PWR9: bl __cosd2_P9
+; CHECK-PWR8: bl __cosd2_P8
+; CHECK-NOT: bl __cosd2_massv
+; CHECK-ALL: blr
+;
+  %1 = call <2 x double> @__cosd2_massv(<2 x double> %opnd)
+  ret <2 x double> %1 
+}
+
+define <4 x float>  @cos_f32_massv(<4 x float> %opnd) {
+; CHECK-ALL-LABEL: @cos_f32_massv
+; CHECK-PWR9: bl __cosf4_P9
+; CHECK-PWR8: bl __cosf4_P8
+; CHECK-NOT: bl __cosf4_massv
+; CHECK-ALL: blr
+;
+  %1 = call <4 x float> @__cosf4_massv(<4 x float> %opnd)
+  ret <4 x float> %1 
+}
+
+; tan
+define <2 x double>  @tan_f64_massv(<2 x double> %opnd) {
+; CHECK-ALL-LABEL: @tan_f64_massv
+; CHECK-PWR9: bl __tand2_P9
+; CHECK-PWR8: bl __tand2_P8
+; CHECK-NOT: bl __tand2_massv
+; CHECK-ALL: blr
+;
+  %1 = call <2 x double> @__tand2_massv(<2 x double> %opnd)
+  ret <2 x double> %1 
+}
+
+define <4 x float>  @tan_f32_massv(<4 x float> %opnd) {
+; CHECK-ALL-LABEL: @tan_f32_massv
+; CHECK-PWR9: bl __tanf4_P9
+; CHECK-PWR8: bl __tanf4_P8
+; CHECK-NOT: bl __tanf4_massv
+; CHECK-ALL: blr
+;
+  %1 = call <4 x float> @__tanf4_massv(<4 x float> %opnd)
+  ret <4 x float> %1 
+}
+
+; asin
+define <2 x double>  @asin_f64_massv(<2 x double> %opnd) {
+; CHECK-ALL-LABEL: @asin_f64_massv
+; CHECK-PWR9: bl __asind2_P9
+; CHECK-PWR8: bl __asind2_P8
+; CHECK-NOT: bl __asind2_massv
+; CHECK-ALL: blr
+;
+  %1 = call <2 x double> @__asind2_massv(<2 x double> %opnd)
+  ret <2 x double> %1 
+}
+
+define <4 x float>  @asin_f32_massv(<4 x float> %opnd) {
+; CHECK-ALL-LABEL: @asin_f32_massv
+; CHECK-PWR9: bl __asinf4_P9
+; CHECK-PWR8: bl __asinf4_P8
+; CHECK-NOT: bl __asinf4_massv
+; CHECK-ALL: blr
+;
+  %1 = call <4 x float> @__asinf4_massv(<4 x float> %opnd)
+  ret <4 x float> %1 
+}
+
+; acos
+define <2 x double>  @acos_f64_massv(<2 x double> %opnd) {
+; CHECK-ALL-LABEL: @acos_f64_massv
+; CHECK-PWR9: bl __acosd2_P9
+; CHECK-PWR8: bl __acosd2_P8
+; CHECK-NOT: bl __acosd2_massv
+; CHECK-ALL: blr
+;
+  %1 = call <2 x double> @__acosd2_massv(<2 x double> %opnd)
+  ret <2 x double> %1 
+}
+
+define <4 x float>  @acos_f32_massv(<4 x float> %opnd) {
+; CHECK-ALL-LABEL: @acos_f32_massv
+; CHECK-PWR9: bl __acosf4_P9
+; CHECK-PWR8: bl __acosf4_P8
+; CHECK-NOT: bl __acosf4_massv
+; CHECK-ALL: blr
+;
+  %1 = call <4 x float> @__acosf4_massv(<4 x float> %opnd)
+  ret <4 x float> %1 
+}
+
+; atan
+define <2 x double>  @atan_f64_massv(<2 x double> %opnd) {
+; CHECK-ALL-LABEL: @atan_f64_massv
+; CHECK-PWR9: bl __atand2_P9
+; CHECK-PWR8: bl __atand2_P8
+; CHECK-NOT: bl __atand2_massv
+; CHECK-ALL: blr
+;
+  %1 = call <2 x double> @__atand2_massv(<2 x double> %opnd)
+  ret <2 x double> %1 
+}
+
+define <4 x float>  @atan_f32_massv(<4 x float> %opnd) {
+; CHECK-ALL-LABEL: @atan_f32_massv
+; CHECK-PWR9: bl __atanf4_P9
+; CHECK-PWR8: bl __atanf4_P8
+; CHECK-NOT: bl __atanf4_massv
+; CHECK-ALL: blr
+;
+  %1 = call <4 x float> @__atanf4_massv(<4 x float> %opnd)
+  ret <4 x float> %1 
+}
+
+; atan2
+define <2 x double>  @atan2_f64_massv(<2 x double> %opnd) {
+; CHECK-ALL-LABEL: @atan2_f64_massv
+; CHECK-PWR9: bl __atan2d2_P9
+; CHECK-PWR8: bl __atan2d2_P8
+; CHECK-NOT: bl __atan2d2_massv
+; CHECK-ALL: blr
+;
+  %1 = call <2 x double> @__atan2d2_massv(<2 x double> %opnd)
+  ret <2 x double> %1 
+}
+
+define <4 x float>  @atan2_f32_massv(<4 x float> %opnd) {
+; CHECK-ALL-LABEL: @atan2_f32_massv
+; CHECK-PWR9: bl __atan2f4_P9
+; CHECK-PWR8: bl __atan2f4_P8
+; CHECK-NOT: bl __atan2f4_massv
+; CHECK-ALL: blr
+;
+  %1 = call <4 x float> @__atan2f4_massv(<4 x float> %opnd)
+  ret <4 x float> %1 
+}
+
+; sinh
+define <2 x double>  @sinh_f64_massv(<2 x double> %opnd) {
+; CHECK-ALL-LABEL: @sinh_f64_massv
+; CHECK-PWR9: bl __sinhd2_P9
+; CHECK-PWR8: bl __sinhd2_P8
+; CHECK-NOT: bl __sinhd2_massv
+; CHECK-ALL: blr
+;
+  %1 = call <2 x double> @__sinhd2_massv(<2 x double> %opnd)
+  ret <2 x double> %1 
+}
+
+define <4 x float>  @sinh_f32_massv(<4 x float> %opnd) {
+; CHECK-ALL-LABEL: @sinh_f32_massv
+; CHECK-PWR9: bl __sinhf4_P9
+; CHECK-PWR8: bl __sinhf4_P8
+; CHECK-NOT: bl __sinhf4_massv
+; CHECK-ALL: blr
+;
+  %1 = call <4 x float> @__sinhf4_massv(<4 x float> %opnd)
+  ret <4 x float> %1 
+}
+
+; cosh
+define <2 x double>  @cosh_f64_massv(<2 x double> %opnd) {
+; CHECK-ALL-LABEL: @cosh_f64_massv
+; CHECK-PWR9: bl __coshd2_P9
+; CHECK-PWR8: bl __coshd2_P8
+; CHECK-NOT: bl __coshd2_massv
+; CHECK-ALL: blr
+;
+  %1 = call <2 x double> @__coshd2_massv(<2 x double> %opnd)
+  ret <2 x double> %1 
+}
+
+define <4 x float>  @cosh_f32_massv(<4 x float> %opnd) {
+; CHECK-ALL-LABEL: @cosh_f32_massv
+; CHECK-PWR9: bl __coshf4_P9
+; CHECK-PWR8: bl __coshf4_P8
+; CHECK-NOT: bl __coshf4_massv
+; CHECK-ALL: blr
+;
+  %1 = call <4 x float> @__coshf4_massv(<4 x float> %opnd)
+  ret <4 x float> %1 
+}
+
+; tanh
+define <2 x double>  @tanh_f64_massv(<2 x double> %opnd) {
+; CHECK-ALL-LABEL: @tanh_f64_massv
+; CHECK-PWR9: bl __tanhd2_P9
+; CHECK-PWR8: bl __tanhd2_P8
+; CHECK-NOT: bl __tanhd2_massv
+; CHECK-ALL: blr
+;
+  %1 = call <2 x double> @__tanhd2_massv(<2 x double> %opnd)
+  ret <2 x double> %1 
+}
+
+define <4 x float>  @tanh_f32_massv(<4 x float> %opnd) {
+; CHECK-ALL-LABEL: @tanh_f32_massv
+; CHECK-PWR9: bl __tanhf4_P9
+; CHECK-PWR8: bl __tanhf4_P8
+; CHECK-NOT: bl __tanhf4_massv
+; CHECK-ALL: blr
+;
+  %1 = call <4 x float> @__tanhf4_massv(<4 x float> %opnd)
+  ret <4 x float> %1 
+}
+
+; asinh
+define <2 x double>  @asinh_f64_massv(<2 x double> %opnd) {
+; CHECK-ALL-LABEL: @asinh_f64_massv
+; CHECK-PWR9: bl __asinhd2_P9
+; CHECK-PWR8: bl __asinhd2_P8
+; CHECK-NOT: bl __asinhd2_massv
+; CHECK-ALL: blr
+;
+  %1 = call <2 x double> @__asinhd2_massv(<2 x double> %opnd)
+  ret <2 x double> %1 
+}
+
+define <4 x float>  @asinh_f32_massv(<4 x float> %opnd) {
+; CHECK-ALL-LABEL: @asinh_f32_massv
+; CHECK-PWR9: bl __asinhf4_P9
+; CHECK-PWR8: bl __asinhf4_P8
+; CHECK-NOT: bl __asinhf4_massv
+; CHECK-ALL: blr
+;
+  %1 = call <4 x float> @__asinhf4_massv(<4 x float> %opnd)
+  ret <4 x float> %1 
+}
+
+; acosh
+define <2 x double>  @acosh_f64_massv(<2 x double> %opnd) {
+; CHECK-ALL-LABEL: @acosh_f64_massv
+; CHECK-PWR9: bl __acoshd2_P9
+; CHECK-PWR8: bl __acoshd2_P8
+; CHECK-NOT: bl __acoshd2_massv
+; CHECK-ALL: blr
+;
+  %1 = call <2 x double> @__acoshd2_massv(<2 x double> %opnd)
+  ret <2 x double> %1 
+}
+
+define <4 x float>  @acosh_f32_massv(<4 x float> %opnd) {
+; CHECK-ALL-LABEL: @acosh_f32_massv
+; CHECK-PWR9: bl __acoshf4_P9
+; CHECK-PWR8: bl __acoshf4_P8
+; CHECK-NOT: bl __acoshf4_massv
+; CHECK-ALL: blr
+;
+  %1 = call <4 x float> @__acoshf4_massv(<4 x float> %opnd)
+  ret <4 x float> %1 
+}
+
+; atanh
+define <2 x double>  @atanh_f64_massv(<2 x double> %opnd) {
+; CHECK-ALL-LABEL: @atanh_f64_massv
+; CHECK-PWR9: bl __atanhd2_P9
+; CHECK-PWR8: bl __atanhd2_P8
+; CHECK-NOT: bl __atanhd2_massv
+; CHECK-ALL: blr
+;
+  %1 = call <2 x double> @__atanhd2_massv(<2 x double> %opnd)
+  ret <2 x double> %1 
+}
+
+define <4 x float>  @atanh_f32_massv(<4 x float> %opnd) {
+; CHECK-ALL-LABEL: @atanh_f32_massv
+; CHECK-PWR9: bl __atanhf4_P9
+; CHECK-PWR8: bl __atanhf4_P8
+; CHECK-NOT: bl __atanhf4_massv
+; CHECK-ALL: blr
+;
+  %1 = call <4 x float> @__atanhf4_massv(<4 x float> %opnd)
+  ret <4 x float> %1 
+}
+
diff --git a/llvm/test/CodeGen/PowerPC/lsr-insns-cost.ll b/llvm/test/CodeGen/PowerPC/lsr-insns-cost.ll
new file mode 100644
index 000000000..4e81811a5
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/lsr-insns-cost.ll
@@ -0,0 +1,66 @@
+; RUN: llc -ppc-asm-full-reg-names  -verify-machineinstrs < %s \
+; RUN:   -mtriple=powerpc64le-unknown-linux-gnu | FileCheck %s
+
+; void test(unsigned *a, unsigned *b, unsigned *c)
+; {
+;   for (unsigned long i = 0; i < 1024; i++)
+;       c[i] = a[i] + b[i];
+; }
+;
+; compile with -fno-unroll-loops
+
+define void @lsr-insts-cost(i32* %0, i32* %1, i32* %2) {
+; CHECK-LABEL: lsr-insts-cost
+; CHECK:       .LBB0_4: # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    lxvd2x vs34, 0, r3
+; CHECK-NEXT:    lxvd2x vs35, 0, r4
+; CHECK-NEXT:    addi r4, r4, 16
+; CHECK-NEXT:    addi r3, r3, 16
+; CHECK-NEXT:    vadduwm v2, v3, v2
+; CHECK-NEXT:    stxvd2x vs34, 0, r5
+; CHECK-NEXT:    addi r5, r5, 16
+; CHECK-NEXT:    bdnz .LBB0_4
+  %4 = getelementptr i32, i32* %2, i64 1024
+  %5 = getelementptr i32, i32* %0, i64 1024
+  %6 = getelementptr i32, i32* %1, i64 1024
+  %7 = icmp ugt i32* %5, %2
+  %8 = icmp ugt i32* %4, %0
+  %9 = and i1 %7, %8
+  %10 = icmp ugt i32* %6, %2
+  %11 = icmp ugt i32* %4, %1
+  %12 = and i1 %10, %11
+  %13 = or i1 %9, %12
+  br i1 %13, label %28, label %14
+
+14:                                               ; preds = %3, %14
+  %15 = phi i64 [ %25, %14 ], [ 0, %3 ]
+  %16 = getelementptr inbounds i32, i32* %0, i64 %15
+  %17 = bitcast i32* %16 to <4 x i32>*
+  %18 = load <4 x i32>, <4 x i32>* %17, align 4
+  %19 = getelementptr inbounds i32, i32* %1, i64 %15
+  %20 = bitcast i32* %19 to <4 x i32>*
+  %21 = load <4 x i32>, <4 x i32>* %20, align 4
+  %22 = add <4 x i32> %21, %18
+  %23 = getelementptr inbounds i32, i32* %2, i64 %15
+  %24 = bitcast i32* %23 to <4 x i32>*
+  store <4 x i32> %22, <4 x i32>* %24, align 4
+  %25 = add i64 %15, 4
+  %26 = icmp eq i64 %25, 1024
+  br i1 %26, label %27, label %14
+
+27:                                               ; preds = %14, %28
+  ret void
+
+28:                                               ; preds = %3, %28
+  %29 = phi i64 [ %36, %28 ], [ 0, %3 ]
+  %30 = getelementptr inbounds i32, i32* %0, i64 %29
+  %31 = load i32, i32* %30, align 4
+  %32 = getelementptr inbounds i32, i32* %1, i64 %29
+  %33 = load i32, i32* %32, align 4
+  %34 = add i32 %33, %31
+  %35 = getelementptr inbounds i32, i32* %2, i64 %29
+  store i32 %34, i32* %35, align 4
+  %36 = add nuw nsw i64 %29, 1
+  %37 = icmp eq i64 %36, 1024
+  br i1 %37, label %27, label %28
+}
diff --git a/llvm/test/CodeGen/PowerPC/machine-combiner.ll b/llvm/test/CodeGen/PowerPC/machine-combiner.ll
index 86439bb5e..47df8921f 100644
--- a/llvm/test/CodeGen/PowerPC/machine-combiner.ll
+++ b/llvm/test/CodeGen/PowerPC/machine-combiner.ll
@@ -1,5 +1,5 @@
-; RUN: llc -verify-machineinstrs -O3 -mcpu=pwr7 -enable-unsafe-fp-math < %s | FileCheck -allow-deprecated-dag-overlap %s -check-prefix=CHECK -check-prefix=CHECK-PWR
-; RUN: llc -verify-machineinstrs -O3 -mcpu=a2q -enable-unsafe-fp-math < %s | FileCheck -allow-deprecated-dag-overlap %s -check-prefix=CHECK -check-prefix=CHECK-QPX
+; RUN: llc -verify-machineinstrs -O3 -mcpu=pwr7 -enable-unsafe-fp-math < %s | FileCheck  %s -check-prefix=CHECK -check-prefix=CHECK-PWR
+; RUN: llc -verify-machineinstrs -O3 -mcpu=a2q -enable-unsafe-fp-math < %s | FileCheck  %s -check-prefix=CHECK -check-prefix=CHECK-QPX
 target datalayout = "E-m:e-i64:64-n32:64"
 target triple = "powerpc64-unknown-linux-gnu"
 
@@ -71,7 +71,7 @@ define float @reassociate_adds5(float %x0, float %x1, float %x2, float %x3, floa
 ; CHECK-DAG:   fadds [[REG12:[0-9]+]], 5, 6
 ; CHECK-DAG:   fadds [[REG0:[0-9]+]], 1, 2
 ; CHECK-DAG:   fadds [[REG11:[0-9]+]], 3, 4
-; CHECK:       fadds [[REG13:[0-9]+]], [[REG12]], 7
+; CHECK-DAG:   fadds [[REG13:[0-9]+]], [[REG12]], 7
 ; CHECK-DAG:   fadds [[REG1:[0-9]+]], [[REG0]], [[REG11]]
 ; CHECK-DAG:   fadds [[REG2:[0-9]+]], [[REG1]], [[REG13]]
 ; CHECK:       fadds 1, [[REG2]], 8
diff --git a/llvm/test/CodeGen/PowerPC/machine-pre.ll b/llvm/test/CodeGen/PowerPC/machine-pre.ll
index 596c0a462..0a7949725 100644
--- a/llvm/test/CodeGen/PowerPC/machine-pre.ll
+++ b/llvm/test/CodeGen/PowerPC/machine-pre.ll
@@ -54,16 +54,10 @@ return:
   ret i32 %ret
 }
 
-define dso_local signext i32 @foo(i32 signext %x, i32 signext %y) local_unnamed_addr #0 {
+define dso_local signext i32 @foo(i32 signext %x, i32 signext %y) nounwind {
 ; CHECK-P9-LABEL: foo:
 ; CHECK-P9:       # %bb.0: # %entry
 ; CHECK-P9-NEXT:    mflr r0
-; CHECK-P9-NEXT:    .cfi_def_cfa_offset 80
-; CHECK-P9-NEXT:    .cfi_offset lr, 16
-; CHECK-P9-NEXT:    .cfi_offset r27, -40
-; CHECK-P9-NEXT:    .cfi_offset r28, -32
-; CHECK-P9-NEXT:    .cfi_offset r29, -24
-; CHECK-P9-NEXT:    .cfi_offset r30, -16
 ; CHECK-P9-NEXT:    std r27, -40(r1) # 8-byte Folded Spill
 ; CHECK-P9-NEXT:    std r28, -32(r1) # 8-byte Folded Spill
 ; CHECK-P9-NEXT:    std r29, -24(r1) # 8-byte Folded Spill
@@ -140,10 +134,10 @@ define dso_local signext i32 @foo(i32 signext %x, i32 signext %y) local_unnamed_
 ; CHECK-P9-NEXT:  .LBB1_10: # %cleanup20
 ; CHECK-P9-NEXT:    addi r1, r1, 80
 ; CHECK-P9-NEXT:    ld r0, 16(r1)
-; CHECK-P9-NEXT:    mtlr r0
 ; CHECK-P9-NEXT:    ld r30, -16(r1) # 8-byte Folded Reload
 ; CHECK-P9-NEXT:    ld r29, -24(r1) # 8-byte Folded Reload
 ; CHECK-P9-NEXT:    ld r28, -32(r1) # 8-byte Folded Reload
+; CHECK-P9-NEXT:    mtlr r0
 ; CHECK-P9-NEXT:    ld r27, -40(r1) # 8-byte Folded Reload
 ; CHECK-P9-NEXT:    blr
 entry:
diff --git a/llvm/test/CodeGen/PowerPC/memCmpUsedInZeroEqualityComparison.ll b/llvm/test/CodeGen/PowerPC/memCmpUsedInZeroEqualityComparison.ll
index 30aff0e32..334dac9fc 100644
--- a/llvm/test/CodeGen/PowerPC/memCmpUsedInZeroEqualityComparison.ll
+++ b/llvm/test/CodeGen/PowerPC/memCmpUsedInZeroEqualityComparison.ll
@@ -196,14 +196,12 @@ define signext i32 @equalityFoldOneConstant(i8* %X) {
   ret i32 %cond
 }
 
-define i1 @length2_eq_nobuiltin_attr(i8* %X, i8* %Y) {
+define i1 @length2_eq_nobuiltin_attr(i8* %X, i8* %Y) nounwind {
 ; CHECK-LABEL: length2_eq_nobuiltin_attr:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    mflr 0
 ; CHECK-NEXT:    std 0, 16(1)
 ; CHECK-NEXT:    stdu 1, -32(1)
-; CHECK-NEXT:    .cfi_def_cfa_offset 32
-; CHECK-NEXT:    .cfi_offset lr, 16
 ; CHECK-NEXT:    li 5, 2
 ; CHECK-NEXT:    bl memcmp
 ; CHECK-NEXT:    nop
diff --git a/llvm/test/CodeGen/PowerPC/memcmpIR.ll b/llvm/test/CodeGen/PowerPC/memcmpIR.ll
index 4c0de720f..ad4dc4b9e 100644
--- a/llvm/test/CodeGen/PowerPC/memcmpIR.ll
+++ b/llvm/test/CodeGen/PowerPC/memcmpIR.ll
@@ -19,9 +19,9 @@ entry:
   ; CHECK-LABEL: loadbb1:{{.*}}
   ; CHECK: [[BCC1:%[0-9]+]] = bitcast i32* {{.*}} to i8*
   ; CHECK-NEXT: [[BCC2:%[0-9]+]] = bitcast i32* {{.*}} to i8*
-  ; CHECK-NEXT: [[GEP1:%[0-9]+]] = getelementptr i8, i8* [[BCC2]], i8 8
+  ; CHECK-NEXT: [[GEP1:%[0-9]+]] = getelementptr i8, i8* [[BCC2]], i64 8
   ; CHECK-NEXT: [[BCL1:%[0-9]+]] = bitcast i8* [[GEP1]] to i64*
-  ; CHECK-NEXT: [[GEP2:%[0-9]+]] = getelementptr i8, i8* [[BCC1]], i8 8
+  ; CHECK-NEXT: [[GEP2:%[0-9]+]] = getelementptr i8, i8* [[BCC1]], i64 8
   ; CHECK-NEXT: [[BCL2:%[0-9]+]] = bitcast i8* [[GEP2]] to i64*
   ; CHECK-NEXT: [[LOAD1:%[0-9]+]] = load i64, i64* [[BCL1]]
   ; CHECK-NEXT: [[LOAD2:%[0-9]+]] = load i64, i64* [[BCL2]]
@@ -44,9 +44,9 @@ entry:
   ; CHECK-BE-LABEL: loadbb1:{{.*}}
   ; CHECK-BE: [[BCC1:%[0-9]+]] = bitcast i32* {{.*}} to i8*
   ; CHECK-BE-NEXT: [[BCC2:%[0-9]+]] = bitcast i32* {{.*}} to i8*
-  ; CHECK-BE-NEXT: [[GEP1:%[0-9]+]] = getelementptr i8, i8* [[BCC2]], i8 8
+  ; CHECK-BE-NEXT: [[GEP1:%[0-9]+]] = getelementptr i8, i8* [[BCC2]], i64 8
   ; CHECK-BE-NEXT: [[BCL1:%[0-9]+]] = bitcast i8* [[GEP1]] to i64*
-  ; CHECK-BE-NEXT: [[GEP2:%[0-9]+]] = getelementptr i8, i8* [[BCC1]], i8 8
+  ; CHECK-BE-NEXT: [[GEP2:%[0-9]+]] = getelementptr i8, i8* [[BCC1]], i64 8
   ; CHECK-BE-NEXT: [[BCL2:%[0-9]+]] = bitcast i8* [[GEP2]] to i64*
   ; CHECK-BE-NEXT: [[LOAD1:%[0-9]+]] = load i64, i64* [[BCL1]]
   ; CHECK-BE-NEXT: [[LOAD2:%[0-9]+]] = load i64, i64* [[BCL2]]
diff --git a/llvm/test/CodeGen/PowerPC/memcpy-vec.ll b/llvm/test/CodeGen/PowerPC/memcpy-vec.ll
index 70c1c466f..cd0abd614 100644
--- a/llvm/test/CodeGen/PowerPC/memcpy-vec.ll
+++ b/llvm/test/CodeGen/PowerPC/memcpy-vec.ll
@@ -1,6 +1,6 @@
-; RUN: llc -verify-machineinstrs -mcpu=pwr7 < %s | FileCheck -allow-deprecated-dag-overlap %s -check-prefix=PWR7
-; RUN: llc -verify-machineinstrs -mcpu=pwr8 < %s | FileCheck -allow-deprecated-dag-overlap %s -check-prefix=PWR8
-; RUN: llc -verify-machineinstrs -mcpu=a2q < %s | FileCheck -allow-deprecated-dag-overlap %s -check-prefix=A2Q
+; RUN: llc -verify-machineinstrs -mcpu=pwr7 < %s | FileCheck  %s -check-prefix=PWR7
+; RUN: llc -verify-machineinstrs -mcpu=pwr8 < %s | FileCheck  %s -check-prefix=PWR8
+; RUN: llc -verify-machineinstrs -mcpu=a2q < %s | FileCheck  %s -check-prefix=A2Q
 target datalayout = "E-m:e-i64:64-n32:64"
 target triple = "powerpc64-unknown-linux-gnu"
 
@@ -16,7 +16,7 @@ entry:
 ; PWR7-NOT: bl memcpy
 ; PWR7-DAG: li [[OFFSET:[0-9]+]], 16
 ; PWR7-DAG: lxvd2x [[TMP0:[0-9]+]], 4, [[OFFSET]]
-; PWR7-DAG: stxvd2x [[TMP0]], 0, 3
+; PWR7-DAG: stxvd2x [[TMP0]], 3, [[OFFSET]]
 ; PWR7-DAG: lxvd2x [[TMP1:[0-9]+]], 0, 4
 ; PWR7-DAG: stxvd2x [[TMP1]], 0, 3
 ; PWR7: blr
diff --git a/llvm/test/CodeGen/PowerPC/mi-peephole-splat.ll b/llvm/test/CodeGen/PowerPC/mi-peephole-splat.ll
new file mode 100644
index 000000000..b31e864b6
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/mi-peephole-splat.ll
@@ -0,0 +1,136 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-linux-gnu < %s \
+; RUN: | FileCheck --check-prefix=CHECK-LE %s
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64-linux-gnu -mattr=+vsx < %s \
+; RUN: | FileCheck --check-prefix=CHECK-BE %s
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-linux-gnu -mcpu=pwr9 < %s \
+; RUN: | FileCheck --check-prefix=CHECK-P9LE %s
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64-linux-gnu -mcpu=pwr9 < %s \
+; RUN: | FileCheck --check-prefix=CHECK-P9BE %s
+
+define double @splat_swap(<2 x double> %x, <2 x double> %y) nounwind  {
+; CHECK-LE-LABEL: splat_swap:
+; CHECK-LE:       # %bb.0:
+; CHECK-LE-NEXT:    mflr 0
+; CHECK-LE-NEXT:    std 0, 16(1)
+; CHECK-LE-NEXT:    stdu 1, -80(1)
+; CHECK-LE-NEXT:    li 3, 64
+; CHECK-LE-NEXT:    stxvd2x 63, 1, 3 # 16-byte Folded Spill
+; CHECK-LE-NEXT:    xvadddp 63, 34, 35
+; CHECK-LE-NEXT:    xxlor 1, 63, 63
+; CHECK-LE-NEXT:    bl rint
+; CHECK-LE-NEXT:    nop
+; CHECK-LE-NEXT:    xxswapd 0, 63
+; CHECK-LE-NEXT:    # kill: def $f1 killed $f1 def $vsl1
+; CHECK-LE-NEXT:    li 3, 48
+; CHECK-LE-NEXT:    stxvd2x 1, 1, 3 # 16-byte Folded Spill
+; CHECK-LE-NEXT:    fmr 1, 0
+; CHECK-LE-NEXT:    bl rint
+; CHECK-LE-NEXT:    nop
+; CHECK-LE-NEXT:    li 3, 48
+; CHECK-LE-NEXT:    # kill: def $f1 killed $f1 def $vsl1
+; CHECK-LE-NEXT:    lxvd2x 0, 1, 3 # 16-byte Folded Reload
+; CHECK-LE-NEXT:    li 3, 64
+; CHECK-LE-NEXT:    lxvd2x 63, 1, 3 # 16-byte Folded Reload
+; CHECK-LE-NEXT:    xxmrghd 0, 0, 1
+; CHECK-LE-NEXT:    xxswapd 1, 0
+; CHECK-LE-NEXT:    xssubdp 1, 1, 0
+; CHECK-LE-NEXT:    addi 1, 1, 80
+; CHECK-LE-NEXT:    ld 0, 16(1)
+; CHECK-LE-NEXT:    mtlr 0
+; CHECK-LE-NEXT:    blr
+;
+; CHECK-BE-LABEL: splat_swap:
+; CHECK-BE:       # %bb.0:
+; CHECK-BE-NEXT:    mflr 0
+; CHECK-BE-NEXT:    std 0, 16(1)
+; CHECK-BE-NEXT:    stdu 1, -160(1)
+; CHECK-BE-NEXT:    li 3, 144
+; CHECK-BE-NEXT:    stxvd2x 63, 1, 3 # 16-byte Folded Spill
+; CHECK-BE-NEXT:    xvadddp 63, 34, 35
+; CHECK-BE-NEXT:    xxlor 1, 63, 63
+; CHECK-BE-NEXT:    bl rint
+; CHECK-BE-NEXT:    nop
+; CHECK-BE-NEXT:    # kill: def $f1 killed $f1 def $vsl1
+; CHECK-BE-NEXT:    li 3, 128
+; CHECK-BE-NEXT:    stxvd2x 1, 1, 3 # 16-byte Folded Spill
+; CHECK-BE-NEXT:    xxswapd 1, 63
+; CHECK-BE-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
+; CHECK-BE-NEXT:    bl rint
+; CHECK-BE-NEXT:    nop
+; CHECK-BE-NEXT:    li 3, 128
+; CHECK-BE-NEXT:    lxvd2x 0, 1, 3 # 16-byte Folded Reload
+; CHECK-BE-NEXT:    # kill: def $f1 killed $f1 def $vsl1
+; CHECK-BE-NEXT:    li 3, 144
+; CHECK-BE-NEXT:    lxvd2x 63, 1, 3 # 16-byte Folded Reload
+; CHECK-BE-NEXT:    xxmrghd 0, 0, 1
+; CHECK-BE-NEXT:    xxswapd 1, 0
+; CHECK-BE-NEXT:    xssubdp 1, 0, 1
+; CHECK-BE-NEXT:    addi 1, 1, 160
+; CHECK-BE-NEXT:    ld 0, 16(1)
+; CHECK-BE-NEXT:    mtlr 0
+; CHECK-BE-NEXT:    blr
+;
+; CHECK-P9LE-LABEL: splat_swap:
+; CHECK-P9LE:       # %bb.0:
+; CHECK-P9LE-NEXT:    mflr 0
+; CHECK-P9LE-NEXT:    std 0, 16(1)
+; CHECK-P9LE-NEXT:    stdu 1, -64(1)
+; CHECK-P9LE-NEXT:    stxv 63, 48(1) # 16-byte Folded Spill
+; CHECK-P9LE-NEXT:    xvadddp 63, 34, 35
+; CHECK-P9LE-NEXT:    xscpsgndp 1, 63, 63
+; CHECK-P9LE-NEXT:    bl rint
+; CHECK-P9LE-NEXT:    nop
+; CHECK-P9LE-NEXT:    # kill: def $f1 killed $f1 def $vsl1
+; CHECK-P9LE-NEXT:    stxv 1, 32(1) # 16-byte Folded Spill
+; CHECK-P9LE-NEXT:    xxswapd 1, 63
+; CHECK-P9LE-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
+; CHECK-P9LE-NEXT:    bl rint
+; CHECK-P9LE-NEXT:    nop
+; CHECK-P9LE-NEXT:    lxv 0, 32(1) # 16-byte Folded Reload
+; CHECK-P9LE-NEXT:    # kill: def $f1 killed $f1 def $vsl1
+; CHECK-P9LE-NEXT:    xxmrghd 0, 0, 1
+; CHECK-P9LE-NEXT:    lxv 63, 48(1) # 16-byte Folded Reload
+; CHECK-P9LE-NEXT:    xxswapd 1, 0
+; CHECK-P9LE-NEXT:    xssubdp 1, 1, 0
+; CHECK-P9LE-NEXT:    addi 1, 1, 64
+; CHECK-P9LE-NEXT:    ld 0, 16(1)
+; CHECK-P9LE-NEXT:    mtlr 0
+; CHECK-P9LE-NEXT:    blr
+;
+; CHECK-P9BE-LABEL: splat_swap:
+; CHECK-P9BE:       # %bb.0:
+; CHECK-P9BE-NEXT:    mflr 0
+; CHECK-P9BE-NEXT:    std 0, 16(1)
+; CHECK-P9BE-NEXT:    stdu 1, -144(1)
+; CHECK-P9BE-NEXT:    stxv 63, 128(1) # 16-byte Folded Spill
+; CHECK-P9BE-NEXT:    xvadddp 63, 34, 35
+; CHECK-P9BE-NEXT:    xscpsgndp 1, 63, 63
+; CHECK-P9BE-NEXT:    bl rint
+; CHECK-P9BE-NEXT:    nop
+; CHECK-P9BE-NEXT:    # kill: def $f1 killed $f1 def $vsl1
+; CHECK-P9BE-NEXT:    stxv 1, 112(1) # 16-byte Folded Spill
+; CHECK-P9BE-NEXT:    xxswapd 1, 63
+; CHECK-P9BE-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
+; CHECK-P9BE-NEXT:    bl rint
+; CHECK-P9BE-NEXT:    nop
+; CHECK-P9BE-NEXT:    lxv 0, 112(1) # 16-byte Folded Reload
+; CHECK-P9BE-NEXT:    # kill: def $f1 killed $f1 def $vsl1
+; CHECK-P9BE-NEXT:    xxmrghd 0, 0, 1
+; CHECK-P9BE-NEXT:    lxv 63, 128(1) # 16-byte Folded Reload
+; CHECK-P9BE-NEXT:    xxswapd 1, 0
+; CHECK-P9BE-NEXT:    xssubdp 1, 0, 1
+; CHECK-P9BE-NEXT:    addi 1, 1, 144
+; CHECK-P9BE-NEXT:    ld 0, 16(1)
+; CHECK-P9BE-NEXT:    mtlr 0
+; CHECK-P9BE-NEXT:    blr
+  %added = fadd <2 x double> %x, %y
+  %call = tail call <2 x double> @llvm.rint.v2f64(<2 x double> %added) nounwind readnone
+  %res1 = extractelement <2 x double> %call, i32 0
+  %res2 = extractelement <2 x double> %call, i32 1
+  %ret = fsub double %res1, %res2
+  ret double %ret
+}
+
+declare <2 x double> @llvm.rint.v2f64(<2 x double>)
+
diff --git a/llvm/test/CodeGen/PowerPC/negctr.ll b/llvm/test/CodeGen/PowerPC/negctr.ll
index 5aa5bd1fa..4e6200106 100644
--- a/llvm/test/CodeGen/PowerPC/negctr.ll
+++ b/llvm/test/CodeGen/PowerPC/negctr.ll
@@ -87,4 +87,4 @@ for.end:                                          ; preds = %for.body, %entry
   ret void
 }
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/PowerPC/no-dup-spill-fp.ll b/llvm/test/CodeGen/PowerPC/no-dup-spill-fp.ll
index 95e069b54..1d9018cca 100644
--- a/llvm/test/CodeGen/PowerPC/no-dup-spill-fp.ll
+++ b/llvm/test/CodeGen/PowerPC/no-dup-spill-fp.ll
@@ -20,7 +20,7 @@ entry:
 
 declare void @func()
 
-attributes #0 = { nounwind "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "target-cpu"="ppc64" }
+attributes #0 = { nounwind "frame-pointer"="all" "target-cpu"="ppc64" }
 attributes #1 = { nounwind }
 
 !0 = !{i32 57}
diff --git a/llvm/test/CodeGen/PowerPC/no-duplicate.ll b/llvm/test/CodeGen/PowerPC/no-duplicate.ll
new file mode 100644
index 000000000..932ef1aa1
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/no-duplicate.ll
@@ -0,0 +1,91 @@
+; RUN: llc -O2 < %s | FileCheck %s
+
+target triple = "powerpc64le-grtev4-linux-gnu"
+
+; No duplication of loop header into entry block.
+define void @no_duplicate1(i64 %a) {
+; CHECK-LABEL: no_duplicate1
+; CHECK:        mr 30, 3
+; CHECK-NEXT:   b .LBB0_2
+
+; CHECK:      .LBB0_2:
+; CHECK-NEXT:   # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:   cmpldi 30, 100
+; CHECK-NEXT:   bne 0, .LBB0_1
+entry:
+  br label %header
+
+header:
+  %ind = phi i64 [%a, %entry], [%val3, %latch]
+  %cond1 = icmp eq i64 %ind, 100
+  br i1 %cond1, label %middle, label %latch
+
+middle:
+  %condx = call i1 @foo()
+  %val1 = xor i64 %ind, 2
+  br label %latch
+
+latch:
+  %val2 = phi i64 [%ind, %header], [%val1, %middle]
+  %val3 = add i64 %val2, 1
+  %cond2 = call i1 @foo()
+  br i1 %cond2, label %end, label %header
+
+end:
+  ret void
+}
+
+; No duplication of loop header into latches.
+define void @no_duplicate2(i64 %a) {
+; CHECK-LABEL: no_duplicate2
+; CHECK:        mr 30, 3
+; CHECK-NEXT:   b .LBB1_2
+
+; CHECK:      .LBB1_2:
+; CHECK-NEXT:   # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:   cmpldi 30, 100
+; CHECK-NEXT:   bne 0, .LBB1_1
+
+; CHECK:      %latch2
+; CHECK:        b .LBB1_2
+
+; CHECK:      %latch3
+; CHECK:        b .LBB1_2
+entry:
+  br label %header
+
+header:
+  %ind = phi i64 [%a, %entry], [%val1, %latch1], [%val2, %latch2], [%val2, %latch3]
+  %cond1 = icmp eq i64 %ind, 100
+  br i1 %cond1, label %middle1, label %latch1
+
+latch1:
+  %cond2 = call i1 @foo()
+  %val1 = xor i64 %ind, 2
+  br i1 %cond2, label %end, label %header
+
+middle1:
+  %cond3 = call i1 @foo()
+  br i1 %cond3, label %latch1, label %middle2
+
+middle2:
+  %cond4 = call i1 @foo()
+  %val2 = add i64 %ind, 1
+  br i1 %cond4, label %latch2, label %latch3
+
+latch2:
+  call void @a()
+  br label %header
+
+latch3:
+  call void @b()
+  br label %header
+
+end:
+  ret void
+}
+
+
+declare i1 @foo()
+declare void @a()
+declare void @b()
diff --git a/llvm/test/CodeGen/PowerPC/not-fixed-frame-object.ll b/llvm/test/CodeGen/PowerPC/not-fixed-frame-object.ll
index b45b74fc5..ffb54edb6 100644
--- a/llvm/test/CodeGen/PowerPC/not-fixed-frame-object.ll
+++ b/llvm/test/CodeGen/PowerPC/not-fixed-frame-object.ll
@@ -61,10 +61,10 @@ define dso_local signext i32 @caller(i32 signext %a, i32 signext %b, i32 signext
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    addi r1, r1, 192
 ; CHECK-NEXT:    ld r0, 16(r1)
-; CHECK-NEXT:    mtlr r0
 ; CHECK-NEXT:    ld r31, -8(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld r30, -16(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld r29, -24(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    mtlr r0
 ; CHECK-NEXT:    ld r28, -32(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld r27, -40(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld r26, -48(r1) # 8-byte Folded Reload
diff --git a/llvm/test/CodeGen/PowerPC/opt-cmp-inst-cr0-live.ll b/llvm/test/CodeGen/PowerPC/opt-cmp-inst-cr0-live.ll
index a247be785..406a3035a 100644
--- a/llvm/test/CodeGen/PowerPC/opt-cmp-inst-cr0-live.ll
+++ b/llvm/test/CodeGen/PowerPC/opt-cmp-inst-cr0-live.ll
@@ -7,12 +7,12 @@ define signext i32 @fn1(i32 %baz) {
   %2 = zext i32 %1 to i64
   %3 = shl i64 %2, 48
   %4 = ashr exact i64 %3, 48
-; CHECK: ANDIo8 killed {{[^,]+}}, 65520, implicit-def dead $cr0
+; CHECK: RLWINM8 killed {{[^,]+}}, 0, 16, 27
 ; CHECK: CMPLDI
 ; CHECK: BCC
 
-; CHECK: ANDIo8 {{[^,]+}}, 65520, implicit-def $cr0
-; CHECK: COPY $cr0
+; CHECK: ANDI8_rec {{[^,]+}}, 65520, implicit-def $cr0
+; CHECK: COPY killed $cr0
 ; CHECK: BCC
   %5 = icmp eq i64 %4, 0
   br i1 %5, label %foo, label %bar
@@ -26,7 +26,7 @@ bar:
 
 ; CHECK-LABEL: fn2
 define signext i32 @fn2(i64 %a, i64 %b) {
-; CHECK: OR8o {{[^, ]+}}, {{[^, ]+}}, implicit-def $cr0
+; CHECK: OR8_rec {{[^, ]+}}, {{[^, ]+}}, implicit-def $cr0
 ; CHECK: [[CREG:[^, ]+]]:crrc = COPY killed $cr
 ; CHECK: BCC 12, killed [[CREG]]
   %1 = or i64 %b, %a
@@ -42,7 +42,7 @@ bar:
 
 ; CHECK-LABEL: fn3
 define signext i32 @fn3(i32 %a) {
-; CHECK: ANDIo killed {{[%0-9]+}}{{[^,]*}}, 10, implicit-def $cr0
+; CHECK: ANDI_rec killed {{[%0-9]+}}{{[^,]*}}, 10, implicit-def $cr0
 ; CHECK: [[CREG:[^, ]+]]:crrc = COPY $cr0
 ; CHECK: BCC 76, killed [[CREG]]
   %1 = and i32 %a, 10
@@ -61,7 +61,7 @@ bar:
 
 ; CHECK-LABEL: fn4
 define i64 @fn4(i64 %a, i64 %b) {
-; CHECK: ADD8o
+; CHECK: ADD8_rec
 ; CHECK-NOT: CMP
 ; CHECK: BCC 71
 
@@ -81,11 +81,11 @@ if.end:
 declare void @exit(i32 signext)
 
 ; Since %v1 and %v2 are zero-extended 32-bit values, %1 is also zero-extended.
-; In this case, we want to use ORo instead of OR + CMPLWI.
+; In this case, we want to use OR_rec instead of OR + CMPLWI.
 
 ; CHECK-LABEL: fn5
 define zeroext i32 @fn5(i32* %p1, i32* %p2) {
-; CHECK: ORo
+; CHECK: OR_rec
 ; CHECK-NOT: CMP
 ; CHECK: BCC
   %v1 = load i32, i32* %p1
@@ -107,11 +107,11 @@ bar:
 ; CHECK-LABEL: fn6
 define i8* @fn6(i8* readonly %p) {
 ; CHECK: LBZU
-; CHECK: EXTSBo
+; CHECK: EXTSB_rec
 ; CHECK-NOT: CMP
 ; CHECK: BCC
 ; CHECK: LBZU
-; CHECK: EXTSBo
+; CHECK: EXTSB_rec
 ; CHECK-NOT: CMP
 ; CHECK: BCC
 
diff --git a/llvm/test/CodeGen/PowerPC/popcnt-zext.ll b/llvm/test/CodeGen/PowerPC/popcnt-zext.ll
new file mode 100644
index 000000000..eab90bf2f
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/popcnt-zext.ll
@@ -0,0 +1,339 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64-- -mattr=+popcntd < %s      | FileCheck %s --check-prefixes=ANY,FAST
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64-- -mattr=+slow-popcntd < %s | FileCheck %s --check-prefixes=ANY,SLOW
+
+define i16 @zpop_i8_i16(i8 %x) {
+; FAST-LABEL: zpop_i8_i16:
+; FAST:       # %bb.0:
+; FAST-NEXT:    clrldi 3, 3, 56
+; FAST-NEXT:    popcntd 3, 3
+; FAST-NEXT:    blr
+;
+; SLOW-LABEL: zpop_i8_i16:
+; SLOW:       # %bb.0:
+; SLOW-NEXT:    clrlwi 5, 3, 24
+; SLOW-NEXT:    rlwinm 3, 3, 31, 0, 31
+; SLOW-NEXT:    andi. 3, 3, 85
+; SLOW-NEXT:    lis 4, 13107
+; SLOW-NEXT:    subf 3, 3, 5
+; SLOW-NEXT:    ori 4, 4, 13107
+; SLOW-NEXT:    rotlwi 5, 3, 30
+; SLOW-NEXT:    and 3, 3, 4
+; SLOW-NEXT:    andis. 4, 5, 13107
+; SLOW-NEXT:    andi. 5, 5, 13107
+; SLOW-NEXT:    or 4, 5, 4
+; SLOW-NEXT:    add 3, 3, 4
+; SLOW-NEXT:    lis 5, 3855
+; SLOW-NEXT:    srwi 4, 3, 4
+; SLOW-NEXT:    add 3, 3, 4
+; SLOW-NEXT:    lis 4, 257
+; SLOW-NEXT:    ori 5, 5, 3855
+; SLOW-NEXT:    and 3, 3, 5
+; SLOW-NEXT:    ori 4, 4, 257
+; SLOW-NEXT:    mullw 3, 3, 4
+; SLOW-NEXT:    srwi 3, 3, 24
+; SLOW-NEXT:    blr
+  %z = zext i8 %x to i16
+  %pop = tail call i16 @llvm.ctpop.i16(i16 %z)
+  ret i16 %pop
+}
+
+define i16 @popz_i8_i16(i8 %x) {
+; FAST-LABEL: popz_i8_i16:
+; FAST:       # %bb.0:
+; FAST-NEXT:    clrldi 3, 3, 56
+; FAST-NEXT:    popcntd 3, 3
+; FAST-NEXT:    blr
+;
+; SLOW-LABEL: popz_i8_i16:
+; SLOW:       # %bb.0:
+; SLOW-NEXT:    clrlwi 5, 3, 24
+; SLOW-NEXT:    rlwinm 3, 3, 31, 0, 31
+; SLOW-NEXT:    andi. 3, 3, 85
+; SLOW-NEXT:    lis 4, 13107
+; SLOW-NEXT:    subf 3, 3, 5
+; SLOW-NEXT:    ori 4, 4, 13107
+; SLOW-NEXT:    rotlwi 5, 3, 30
+; SLOW-NEXT:    and 3, 3, 4
+; SLOW-NEXT:    andis. 4, 5, 13107
+; SLOW-NEXT:    andi. 5, 5, 13107
+; SLOW-NEXT:    or 4, 5, 4
+; SLOW-NEXT:    add 3, 3, 4
+; SLOW-NEXT:    lis 5, 3855
+; SLOW-NEXT:    srwi 4, 3, 4
+; SLOW-NEXT:    add 3, 3, 4
+; SLOW-NEXT:    lis 4, 257
+; SLOW-NEXT:    ori 5, 5, 3855
+; SLOW-NEXT:    and 3, 3, 5
+; SLOW-NEXT:    ori 4, 4, 257
+; SLOW-NEXT:    mullw 3, 3, 4
+; SLOW-NEXT:    rlwinm 3, 3, 8, 24, 31
+; SLOW-NEXT:    blr
+  %pop = tail call i8 @llvm.ctpop.i8(i8 %x)
+  %z = zext i8 %pop to i16
+  ret i16 %z
+}
+
+define i32 @zpop_i8_i32(i8 %x) {
+; FAST-LABEL: zpop_i8_i32:
+; FAST:       # %bb.0:
+; FAST-NEXT:    rlwinm 3, 3, 0, 24, 31
+; FAST-NEXT:    popcntw 3, 3
+; FAST-NEXT:    blr
+;
+; SLOW-LABEL: zpop_i8_i32:
+; SLOW:       # %bb.0:
+; SLOW-NEXT:    clrlwi 5, 3, 24
+; SLOW-NEXT:    rlwinm 3, 3, 31, 0, 31
+; SLOW-NEXT:    andi. 3, 3, 85
+; SLOW-NEXT:    lis 4, 13107
+; SLOW-NEXT:    subf 3, 3, 5
+; SLOW-NEXT:    ori 4, 4, 13107
+; SLOW-NEXT:    rotlwi 5, 3, 30
+; SLOW-NEXT:    and 3, 3, 4
+; SLOW-NEXT:    andis. 4, 5, 13107
+; SLOW-NEXT:    andi. 5, 5, 13107
+; SLOW-NEXT:    or 4, 5, 4
+; SLOW-NEXT:    add 3, 3, 4
+; SLOW-NEXT:    lis 5, 3855
+; SLOW-NEXT:    srwi 4, 3, 4
+; SLOW-NEXT:    add 3, 3, 4
+; SLOW-NEXT:    lis 4, 257
+; SLOW-NEXT:    ori 5, 5, 3855
+; SLOW-NEXT:    and 3, 3, 5
+; SLOW-NEXT:    ori 4, 4, 257
+; SLOW-NEXT:    mullw 3, 3, 4
+; SLOW-NEXT:    srwi 3, 3, 24
+; SLOW-NEXT:    blr
+  %z = zext i8 %x to i32
+  %pop = tail call i32 @llvm.ctpop.i32(i32 %z)
+  ret i32 %pop
+}
+
+define i32 @popz_i8_32(i8 %x) {
+; FAST-LABEL: popz_i8_32:
+; FAST:       # %bb.0:
+; FAST-NEXT:    clrldi 3, 3, 56
+; FAST-NEXT:    popcntd 3, 3
+; FAST-NEXT:    blr
+;
+; SLOW-LABEL: popz_i8_32:
+; SLOW:       # %bb.0:
+; SLOW-NEXT:    clrlwi 5, 3, 24
+; SLOW-NEXT:    rlwinm 3, 3, 31, 0, 31
+; SLOW-NEXT:    andi. 3, 3, 85
+; SLOW-NEXT:    lis 4, 13107
+; SLOW-NEXT:    subf 3, 3, 5
+; SLOW-NEXT:    ori 4, 4, 13107
+; SLOW-NEXT:    rotlwi 5, 3, 30
+; SLOW-NEXT:    and 3, 3, 4
+; SLOW-NEXT:    andis. 4, 5, 13107
+; SLOW-NEXT:    andi. 5, 5, 13107
+; SLOW-NEXT:    or 4, 5, 4
+; SLOW-NEXT:    add 3, 3, 4
+; SLOW-NEXT:    lis 5, 3855
+; SLOW-NEXT:    srwi 4, 3, 4
+; SLOW-NEXT:    add 3, 3, 4
+; SLOW-NEXT:    lis 4, 257
+; SLOW-NEXT:    ori 5, 5, 3855
+; SLOW-NEXT:    and 3, 3, 5
+; SLOW-NEXT:    ori 4, 4, 257
+; SLOW-NEXT:    mullw 3, 3, 4
+; SLOW-NEXT:    rlwinm 3, 3, 8, 24, 31
+; SLOW-NEXT:    blr
+  %pop = tail call i8 @llvm.ctpop.i8(i8 %x)
+  %z = zext i8 %pop to i32
+  ret i32 %z
+}
+
+define i32 @zpop_i16_i32(i16 %x) {
+; FAST-LABEL: zpop_i16_i32:
+; FAST:       # %bb.0:
+; FAST-NEXT:    rlwinm 3, 3, 0, 16, 31
+; FAST-NEXT:    popcntw 3, 3
+; FAST-NEXT:    blr
+;
+; SLOW-LABEL: zpop_i16_i32:
+; SLOW:       # %bb.0:
+; SLOW-NEXT:    clrlwi 5, 3, 16
+; SLOW-NEXT:    rlwinm 3, 3, 31, 0, 31
+; SLOW-NEXT:    andi. 3, 3, 21845
+; SLOW-NEXT:    lis 4, 13107
+; SLOW-NEXT:    subf 3, 3, 5
+; SLOW-NEXT:    ori 4, 4, 13107
+; SLOW-NEXT:    rotlwi 5, 3, 30
+; SLOW-NEXT:    and 3, 3, 4
+; SLOW-NEXT:    andis. 4, 5, 13107
+; SLOW-NEXT:    andi. 5, 5, 13107
+; SLOW-NEXT:    or 4, 5, 4
+; SLOW-NEXT:    add 3, 3, 4
+; SLOW-NEXT:    lis 5, 3855
+; SLOW-NEXT:    srwi 4, 3, 4
+; SLOW-NEXT:    add 3, 3, 4
+; SLOW-NEXT:    lis 4, 257
+; SLOW-NEXT:    ori 5, 5, 3855
+; SLOW-NEXT:    and 3, 3, 5
+; SLOW-NEXT:    ori 4, 4, 257
+; SLOW-NEXT:    mullw 3, 3, 4
+; SLOW-NEXT:    srwi 3, 3, 24
+; SLOW-NEXT:    blr
+  %z = zext i16 %x to i32
+  %pop = tail call i32 @llvm.ctpop.i32(i32 %z)
+  ret i32 %pop
+}
+
+define i32 @popz_i16_32(i16 %x) {
+; FAST-LABEL: popz_i16_32:
+; FAST:       # %bb.0:
+; FAST-NEXT:    clrldi 3, 3, 48
+; FAST-NEXT:    popcntd 3, 3
+; FAST-NEXT:    blr
+;
+; SLOW-LABEL: popz_i16_32:
+; SLOW:       # %bb.0:
+; SLOW-NEXT:    clrlwi 5, 3, 16
+; SLOW-NEXT:    rlwinm 3, 3, 31, 0, 31
+; SLOW-NEXT:    andi. 3, 3, 21845
+; SLOW-NEXT:    lis 4, 13107
+; SLOW-NEXT:    subf 3, 3, 5
+; SLOW-NEXT:    ori 4, 4, 13107
+; SLOW-NEXT:    rotlwi 5, 3, 30
+; SLOW-NEXT:    and 3, 3, 4
+; SLOW-NEXT:    andis. 4, 5, 13107
+; SLOW-NEXT:    andi. 5, 5, 13107
+; SLOW-NEXT:    or 4, 5, 4
+; SLOW-NEXT:    add 3, 3, 4
+; SLOW-NEXT:    lis 5, 3855
+; SLOW-NEXT:    srwi 4, 3, 4
+; SLOW-NEXT:    add 3, 3, 4
+; SLOW-NEXT:    lis 4, 257
+; SLOW-NEXT:    ori 5, 5, 3855
+; SLOW-NEXT:    and 3, 3, 5
+; SLOW-NEXT:    ori 4, 4, 257
+; SLOW-NEXT:    mullw 3, 3, 4
+; SLOW-NEXT:    rlwinm 3, 3, 8, 24, 31
+; SLOW-NEXT:    blr
+  %pop = tail call i16 @llvm.ctpop.i16(i16 %x)
+  %z = zext i16 %pop to i32
+  ret i32 %z
+}
+
+define i64 @zpop_i32_i64(i32 %x) {
+; FAST-LABEL: zpop_i32_i64:
+; FAST:       # %bb.0:
+; FAST-NEXT:    clrldi 3, 3, 32
+; FAST-NEXT:    popcntd 3, 3
+; FAST-NEXT:    blr
+;
+; SLOW-LABEL: zpop_i32_i64:
+; SLOW:       # %bb.0:
+; SLOW-NEXT:    rlwinm 5, 3, 31, 1, 0
+; SLOW-NEXT:    lis 4, 13107
+; SLOW-NEXT:    andis. 6, 5, 21845
+; SLOW-NEXT:    andi. 5, 5, 21845
+; SLOW-NEXT:    ori 4, 4, 13107
+; SLOW-NEXT:    or 5, 5, 6
+; SLOW-NEXT:    clrldi 3, 3, 32
+; SLOW-NEXT:    rldimi 4, 4, 32, 0
+; SLOW-NEXT:    sub 3, 3, 5
+; SLOW-NEXT:    and 5, 3, 4
+; SLOW-NEXT:    rotldi 3, 3, 62
+; SLOW-NEXT:    and 3, 3, 4
+; SLOW-NEXT:    add 3, 5, 3
+; SLOW-NEXT:    lis 4, 3855
+; SLOW-NEXT:    rldicl 5, 3, 60, 4
+; SLOW-NEXT:    ori 4, 4, 3855
+; SLOW-NEXT:    add 3, 3, 5
+; SLOW-NEXT:    lis 5, 257
+; SLOW-NEXT:    rldimi 4, 4, 32, 0
+; SLOW-NEXT:    ori 5, 5, 257
+; SLOW-NEXT:    and 3, 3, 4
+; SLOW-NEXT:    rldimi 5, 5, 32, 0
+; SLOW-NEXT:    mulld 3, 3, 5
+; SLOW-NEXT:    rldicl 3, 3, 8, 56
+; SLOW-NEXT:    blr
+  %z = zext i32 %x to i64
+  %pop = tail call i64 @llvm.ctpop.i64(i64 %z)
+  ret i64 %pop
+}
+
+define i64 @popz_i32_i64(i32 %x) {
+; FAST-LABEL: popz_i32_i64:
+; FAST:       # %bb.0:
+; FAST-NEXT:    popcntw 3, 3
+; FAST-NEXT:    clrldi 3, 3, 32
+; FAST-NEXT:    blr
+;
+; SLOW-LABEL: popz_i32_i64:
+; SLOW:       # %bb.0:
+; SLOW-NEXT:    rotlwi 5, 3, 31
+; SLOW-NEXT:    andis. 6, 5, 21845
+; SLOW-NEXT:    andi. 5, 5, 21845
+; SLOW-NEXT:    or 5, 5, 6
+; SLOW-NEXT:    lis 4, 13107
+; SLOW-NEXT:    subf 3, 5, 3
+; SLOW-NEXT:    ori 4, 4, 13107
+; SLOW-NEXT:    rotlwi 5, 3, 30
+; SLOW-NEXT:    and 3, 3, 4
+; SLOW-NEXT:    andis. 4, 5, 13107
+; SLOW-NEXT:    andi. 5, 5, 13107
+; SLOW-NEXT:    or 4, 5, 4
+; SLOW-NEXT:    add 3, 3, 4
+; SLOW-NEXT:    lis 5, 3855
+; SLOW-NEXT:    srwi 4, 3, 4
+; SLOW-NEXT:    add 3, 3, 4
+; SLOW-NEXT:    lis 4, 257
+; SLOW-NEXT:    ori 5, 5, 3855
+; SLOW-NEXT:    and 3, 3, 5
+; SLOW-NEXT:    ori 4, 4, 257
+; SLOW-NEXT:    mullw 3, 3, 4
+; SLOW-NEXT:    rlwinm 3, 3, 8, 24, 31
+; SLOW-NEXT:    blr
+  %pop = tail call i32 @llvm.ctpop.i32(i32 %x)
+  %z = zext i32 %pop to i64
+  ret i64 %z
+}
+
+define i64 @popa_i16_i64(i16 %x) {
+; FAST-LABEL: popa_i16_i64:
+; FAST:       # %bb.0:
+; FAST-NEXT:    clrldi 3, 3, 48
+; FAST-NEXT:    popcntd 3, 3
+; FAST-NEXT:    rlwinm 3, 3, 0, 27, 27
+; FAST-NEXT:    blr
+;
+; SLOW-LABEL: popa_i16_i64:
+; SLOW:       # %bb.0:
+; SLOW-NEXT:    clrlwi 5, 3, 16
+; SLOW-NEXT:    rlwinm 3, 3, 31, 0, 31
+; SLOW-NEXT:    andi. 3, 3, 21845
+; SLOW-NEXT:    lis 4, 13107
+; SLOW-NEXT:    subf 3, 3, 5
+; SLOW-NEXT:    ori 4, 4, 13107
+; SLOW-NEXT:    rotlwi 5, 3, 30
+; SLOW-NEXT:    and 3, 3, 4
+; SLOW-NEXT:    andis. 4, 5, 13107
+; SLOW-NEXT:    andi. 5, 5, 13107
+; SLOW-NEXT:    or 4, 5, 4
+; SLOW-NEXT:    add 3, 3, 4
+; SLOW-NEXT:    lis 5, 3855
+; SLOW-NEXT:    srwi 4, 3, 4
+; SLOW-NEXT:    add 3, 3, 4
+; SLOW-NEXT:    lis 4, 257
+; SLOW-NEXT:    ori 5, 5, 3855
+; SLOW-NEXT:    and 3, 3, 5
+; SLOW-NEXT:    ori 4, 4, 257
+; SLOW-NEXT:    mullw 3, 3, 4
+; SLOW-NEXT:    srwi 3, 3, 24
+; SLOW-NEXT:    rlwinm 3, 3, 0, 27, 27
+; SLOW-NEXT:    blr
+  %pop = call i16 @llvm.ctpop.i16(i16 %x)
+  %z = zext i16 %pop to i64 ; SimplifyDemandedBits may turn zext (or sext) into aext
+  %a = and i64 %z, 16
+  ret i64 %a
+}
+
+declare i8 @llvm.ctpop.i8(i8) nounwind readnone
+declare i16 @llvm.ctpop.i16(i16) nounwind readnone
+declare i32 @llvm.ctpop.i32(i32) nounwind readnone
+declare i64 @llvm.ctpop.i64(i64) nounwind readnone
diff --git a/llvm/test/CodeGen/PowerPC/popcnt.ll b/llvm/test/CodeGen/PowerPC/popcnt.ll
index 9c71c94f5..11e1ec3ff 100644
--- a/llvm/test/CodeGen/PowerPC/popcnt.ll
+++ b/llvm/test/CodeGen/PowerPC/popcnt.ll
@@ -8,8 +8,8 @@ define i8 @cnt8(i8 %x) nounwind readnone {
   %cnt = tail call i8 @llvm.ctpop.i8(i8 %x)
   ret i8 %cnt
 ; CHECK-LABEL: @cnt8
-; CHECK: rlwinm
-; CHECK: popcntw
+; CHECK: clrldi
+; CHECK: popcntd
 ; CHECK: blr
 
 ; SLOWPC-LABEL: @cnt8
@@ -20,8 +20,8 @@ define i16 @cnt16(i16 %x) nounwind readnone {
   %cnt = tail call i16 @llvm.ctpop.i16(i16 %x)
   ret i16 %cnt
 ; CHECK-LABEL: @cnt16
-; CHECK: rlwinm
-; CHECK: popcntw
+; CHECK: clrldi
+; CHECK: popcntd
 ; CHECK: blr
 
 ; SLOWPC-LABEL: @cnt16
diff --git a/llvm/test/CodeGen/PowerPC/power9-moves-and-splats.ll b/llvm/test/CodeGen/PowerPC/power9-moves-and-splats.ll
index d5468902b..0b2910d6b 100644
--- a/llvm/test/CodeGen/PowerPC/power9-moves-and-splats.ll
+++ b/llvm/test/CodeGen/PowerPC/power9-moves-and-splats.ll
@@ -61,16 +61,12 @@ entry:
 define <4 x i32> @test4(i32* nocapture readonly %in) {
 ; CHECK-LABEL: test4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    lfiwzx f0, 0, r3
-; CHECK-NEXT:    xxpermdi vs0, f0, f0, 2
-; CHECK-NEXT:    xxspltw v2, vs0, 3
+; CHECK-NEXT:    lxvwsx v2, 0, r3
 ; CHECK-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test4:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    lfiwzx f0, 0, r3
-; CHECK-BE-NEXT:    xxsldwi vs0, f0, f0, 1
-; CHECK-BE-NEXT:    xxspltw v2, vs0, 0
+; CHECK-BE-NEXT:    lxvwsx v2, 0, r3
 ; CHECK-BE-NEXT:    blr
 
 entry:
@@ -83,16 +79,12 @@ entry:
 define <4 x float> @test5(float* nocapture readonly %in) {
 ; CHECK-LABEL: test5:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    lfiwzx f0, 0, r3
-; CHECK-NEXT:    xxpermdi vs0, f0, f0, 2
-; CHECK-NEXT:    xxspltw v2, vs0, 3
+; CHECK-NEXT:    lxvwsx v2, 0, r3
 ; CHECK-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test5:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    lfiwzx f0, 0, r3
-; CHECK-BE-NEXT:    xxsldwi vs0, f0, f0, 1
-; CHECK-BE-NEXT:    xxspltw v2, vs0, 0
+; CHECK-BE-NEXT:    lxvwsx v2, 0, r3
 ; CHECK-BE-NEXT:    blr
 
 entry:
@@ -107,18 +99,14 @@ define <4 x i32> @test6() {
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    addis r3, r2, .LC0@toc@ha
 ; CHECK-NEXT:    ld r3, .LC0@toc@l(r3)
-; CHECK-NEXT:    lfiwzx f0, 0, r3
-; CHECK-NEXT:    xxpermdi vs0, f0, f0, 2
-; CHECK-NEXT:    xxspltw v2, vs0, 3
+; CHECK-NEXT:    lxvwsx v2, 0, r3
 ; CHECK-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test6:
 ; CHECK-BE:       # %bb.0: # %entry
 ; CHECK-BE-NEXT:    addis r3, r2, .LC0@toc@ha
 ; CHECK-BE-NEXT:    ld r3, .LC0@toc@l(r3)
-; CHECK-BE-NEXT:    lfiwzx f0, 0, r3
-; CHECK-BE-NEXT:    xxsldwi vs0, f0, f0, 1
-; CHECK-BE-NEXT:    xxspltw v2, vs0, 0
+; CHECK-BE-NEXT:    lxvwsx v2, 0, r3
 ; CHECK-BE-NEXT:    blr
 
 entry:
@@ -133,18 +121,14 @@ define <4 x float> @test7() {
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    addis r3, r2, .LC1@toc@ha
 ; CHECK-NEXT:    ld r3, .LC1@toc@l(r3)
-; CHECK-NEXT:    lfiwzx f0, 0, r3
-; CHECK-NEXT:    xxpermdi vs0, f0, f0, 2
-; CHECK-NEXT:    xxspltw v2, vs0, 3
+; CHECK-NEXT:    lxvwsx v2, 0, r3
 ; CHECK-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test7:
 ; CHECK-BE:       # %bb.0: # %entry
 ; CHECK-BE-NEXT:    addis r3, r2, .LC1@toc@ha
 ; CHECK-BE-NEXT:    ld r3, .LC1@toc@l(r3)
-; CHECK-BE-NEXT:    lfiwzx f0, 0, r3
-; CHECK-BE-NEXT:    xxsldwi vs0, f0, f0, 1
-; CHECK-BE-NEXT:    xxspltw v2, vs0, 0
+; CHECK-BE-NEXT:    lxvwsx v2, 0, r3
 ; CHECK-BE-NEXT:    blr
 
 entry:
@@ -217,12 +201,12 @@ entry:
 define <16 x i8> @test12() {
 ; CHECK-LABEL: test12:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    xxspltib v2, 255
+; CHECK-NEXT:    xxleqv v2, v2, v2
 ; CHECK-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test12:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    xxspltib v2, 255
+; CHECK-BE-NEXT:    xxleqv v2, v2, v2
 ; CHECK-BE-NEXT:    blr
 
 entry:
diff --git a/llvm/test/CodeGen/PowerPC/ppc-empty-fs.ll b/llvm/test/CodeGen/PowerPC/ppc-empty-fs.ll
index 8b6008932..71fd18213 100644
--- a/llvm/test/CodeGen/PowerPC/ppc-empty-fs.ll
+++ b/llvm/test/CodeGen/PowerPC/ppc-empty-fs.ll
@@ -24,7 +24,7 @@ entry:
 ; Function Attrs: nounwind
 declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture readonly, i64, i1) #1
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "target-features"="" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "target-features"="" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind }
 
 !llvm.ident = !{!0}
diff --git a/llvm/test/CodeGen/PowerPC/ppc-passname.ll b/llvm/test/CodeGen/PowerPC/ppc-passname.ll
index 005f0a25c..98343bdb5 100644
--- a/llvm/test/CodeGen/PowerPC/ppc-passname.ll
+++ b/llvm/test/CodeGen/PowerPC/ppc-passname.ll
@@ -1,13 +1,13 @@
-; Test pass name: ppc-loop-preinc-prep.
-; RUN: llc -mtriple=powerpc64le-unknown-unknown < %s -debug-pass=Structure -stop-before=ppc-loop-preinc-prep -o /dev/null 2>&1 | FileCheck %s -check-prefix=STOP-BEFORE-LOOP-PREINC-PREP
-; STOP-BEFORE-LOOP-PREINC-PREP-NOT: -ppc-loop-preinc-prep
-; STOP-BEFORE-LOOP-PREINC-PREP-NOT: "ppc-loop-preinc-prep" pass is not registered.
-; STOP-BEFORE-LOOP-PREINC-PREP-NOT: Prepare loop for pre-inc. addressing modes
-
-; RUN: llc -mtriple=powerpc64le-unknown-unknown < %s -debug-pass=Structure -stop-after=ppc-loop-preinc-prep -o /dev/null 2>&1 | FileCheck %s -check-prefix=STOP-AFTER-LOOP-PREINC-PREP
-; STOP-AFTER-LOOP-PREINC-PREP: -ppc-loop-preinc-prep
-; STOP-AFTER-LOOP-PREINC-PREP-NOT: "ppc-loop-preinc-prep" pass is not registered.
-; STOP-AFTER-LOOP-PREINC-PREP: Prepare loop for pre-inc. addressing modes
+; Test pass name: ppc-loop-instr-form-prep.
+; RUN: llc -mtriple=powerpc64le-unknown-unknown < %s -debug-pass=Structure -stop-before=ppc-loop-instr-form-prep -o /dev/null 2>&1 | FileCheck %s -check-prefix=STOP-BEFORE-LOOP-INSTR-FORM-PREP
+; STOP-BEFORE-LOOP-INSTR-FORM-PREP-NOT: -ppc-loop-instr-form-prep
+; STOP-BEFORE-LOOP-INSTR-FORM-PREP-NOT: "ppc-loop-instr-form-prep" pass is not registered.
+; STOP-BEFORE-LOOP-INSTR-FORM-PREP-NOT: Prepare loop for ppc preferred instruction forms
+
+; RUN: llc -mtriple=powerpc64le-unknown-unknown < %s -debug-pass=Structure -stop-after=ppc-loop-instr-form-prep -o /dev/null 2>&1 | FileCheck %s -check-prefix=STOP-AFTER-LOOP-INSTR-FORM-PREP
+; STOP-AFTER-LOOP-INSTR-FORM-PREP: -ppc-loop-instr-form-prep
+; STOP-AFTER-LOOP-INSTR-FORM-PREP-NOT: "ppc-loop-instr-form-prep" pass is not registered.
+; STOP-AFTER-LOOP-INSTR-FORM-PREP: Prepare loop for ppc preferred instruction forms
 
 
 ; Test pass name: ppc-toc-reg-deps.
diff --git a/llvm/test/CodeGen/PowerPC/ppc32-lshrti3.ll b/llvm/test/CodeGen/PowerPC/ppc32-lshrti3.ll
index 5fc6afa5a..971ee3f3e 100644
--- a/llvm/test/CodeGen/PowerPC/ppc32-lshrti3.ll
+++ b/llvm/test/CodeGen/PowerPC/ppc32-lshrti3.ll
@@ -32,7 +32,7 @@ while.end:                                        ; preds = %while.cond
   ret i32 undef
 }
 
-attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.ident = !{!0}
 
diff --git a/llvm/test/CodeGen/PowerPC/ppc64-P9-mod.ll b/llvm/test/CodeGen/PowerPC/ppc64-P9-mod.ll
index 46e347bec..547f9273f 100644
--- a/llvm/test/CodeGen/PowerPC/ppc64-P9-mod.ll
+++ b/llvm/test/CodeGen/PowerPC/ppc64-P9-mod.ll
@@ -2,14 +2,14 @@
 ; RUN: llc < %s -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr9 -verify-machineinstrs | FileCheck %s
 ; RUN: llc < %s -mtriple=powerpc64le-unknown-linux-gnu -mcpu=pwr8 | FileCheck %s -check-prefix=CHECK-PWR8 -implicit-check-not mod[us][wd]
 
-@mod_resultsw = common local_unnamed_addr global i32 0, align 4
-@mod_resultud = common local_unnamed_addr global i64 0, align 8
-@div_resultsw = common local_unnamed_addr global i32 0, align 4
-@mod_resultuw = common local_unnamed_addr global i32 0, align 4
-@div_resultuw = common local_unnamed_addr global i32 0, align 4
-@div_resultsd = common local_unnamed_addr global i64 0, align 8
-@mod_resultsd = common local_unnamed_addr global i64 0, align 8
-@div_resultud = common local_unnamed_addr global i64 0, align 8
+@mod_resultsw = local_unnamed_addr global i32 0, align 4
+@mod_resultud = local_unnamed_addr global i64 0, align 8
+@div_resultsw = local_unnamed_addr global i32 0, align 4
+@mod_resultuw = local_unnamed_addr global i32 0, align 4
+@div_resultuw = local_unnamed_addr global i32 0, align 4
+@div_resultsd = local_unnamed_addr global i64 0, align 8
+@mod_resultsd = local_unnamed_addr global i64 0, align 8
+@div_resultud = local_unnamed_addr global i64 0, align 8
 
 ; Function Attrs: norecurse nounwind
 define void @modulo_sw(i32 signext %a, i32 signext %b) local_unnamed_addr {
diff --git a/llvm/test/CodeGen/PowerPC/ppc64-align-long-double.ll b/llvm/test/CodeGen/PowerPC/ppc64-align-long-double.ll
index 8a5c57318..883d892fc 100644
--- a/llvm/test/CodeGen/PowerPC/ppc64-align-long-double.ll
+++ b/llvm/test/CodeGen/PowerPC/ppc64-align-long-double.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -verify-machineinstrs -mcpu=pwr7 -O2 -fast-isel=false -mattr=-vsx < %s | FileCheck %s
 ; RUN: llc -verify-machineinstrs -mcpu=pwr7 -O2 -fast-isel=false -mattr=+vsx < %s | FileCheck -check-prefix=CHECK-VSX %s
 ; RUN: llc -verify-machineinstrs -mcpu=pwr9 -O2 -fast-isel=false -mattr=+vsx < %s | FileCheck -check-prefix=CHECK-P9 %s
@@ -12,49 +13,48 @@ target triple = "powerpc64-unknown-linux-gnu"
 
 %struct.S = type { double, ppc_fp128 }
 
+; The additional stores are caused because we forward the value in the
+; store->load->bitcast path to make a store and bitcast of the same
+; value. Since the target does bitcast through memory and we no longer
+; remember the address we need to do the store in a fresh local
+; address.
 define ppc_fp128 @test(%struct.S* byval %x) nounwind {
+; CHECK-LABEL: test:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    std 5, -16(1)
+; CHECK-NEXT:    std 6, -8(1)
+; CHECK-NEXT:    lfd 1, -16(1)
+; CHECK-NEXT:    lfd 2, -8(1)
+; CHECK-NEXT:    std 6, 72(1)
+; CHECK-NEXT:    std 5, 64(1)
+; CHECK-NEXT:    std 3, 48(1)
+; CHECK-NEXT:    std 4, 56(1)
+; CHECK-NEXT:    blr
+;
+; CHECK-VSX-LABEL: test:
+; CHECK-VSX:       # %bb.0: # %entry
+; CHECK-VSX-NEXT:    std 3, 48(1)
+; CHECK-VSX-NEXT:    std 6, 72(1)
+; CHECK-VSX-NEXT:    std 5, 64(1)
+; CHECK-VSX-NEXT:    std 4, 56(1)
+; CHECK-VSX-NEXT:    std 5, -16(1)
+; CHECK-VSX-NEXT:    std 6, -8(1)
+; CHECK-VSX-NEXT:    lfd 1, -16(1)
+; CHECK-VSX-NEXT:    lfd 2, -8(1)
+; CHECK-VSX-NEXT:    blr
+;
+; CHECK-P9-LABEL: test:
+; CHECK-P9:       # %bb.0: # %entry
+; CHECK-P9-NEXT:    mtfprd 1, 5
+; CHECK-P9-NEXT:    mtfprd 2, 6
+; CHECK-P9-NEXT:    std 6, 72(1)
+; CHECK-P9-NEXT:    std 5, 64(1)
+; CHECK-P9-NEXT:    std 3, 48(1)
+; CHECK-P9-NEXT:    std 4, 56(1)
+; CHECK-P9-NEXT:    blr
 entry:
   %b = getelementptr inbounds %struct.S, %struct.S* %x, i32 0, i32 1
   %0 = load ppc_fp128, ppc_fp128* %b, align 16
   ret ppc_fp128 %0
 }
 
-; The additional stores are caused because we forward the value in the
-; store->load->bitcast path to make a store and bitcast of the same
-; value. Since the target does bitcast through memory and we no longer
-; remember the address we need to do the store in a fresh local
-; address. 
-
-; CHECK-DAG: std 6, 72(1)
-; CHECK-DAG: std 5, 64(1)
-; CHECK-DAG: std 4, 56(1)
-; CHECK-DAG: std 3, 48(1)
-
-; CHECK-DAG: std 5, -16(1)
-; CHECK-DAG: std 6, -8(1)
-; CHECK-DAG: lfd 1, -16(1)
-; CHECK-DAG: lfd 2, -8(1)
-
-; FIXMECHECK: lfd 1, 64(1)
-; FIXMECHECK: lfd 2, 72(1)
-
-; CHECK-VSX-DAG: std 6, 72(1)
-; CHECK-VSX-DAG: std 5, 64(1)
-; CHECK-VSX-DAG: std 4, 56(1)
-; CHECK-VSX-DAG: std 3, 48(1)
-; CHECK-VSX-DAG: std 5, -16(1)
-; CHECK-VSX-DAG: std 6, -8(1)
-; CHECK-VSX: lfd 1, -16(1)
-; CHECK-VSX: lfd 2, -8(1)
-
-; FIXME-VSX: addi 4, 1, 48
-; FIXME-VSX: lxsdx 1, 4, 3
-; FIXME-VSX: li 3, 24
-; FIXME-VSX: lxsdx 2, 4, 3
-
-; CHECK-P9-DAG: std 6, 72(1)
-; CHECK-P9-DAG: std 5, 64(1)
-; CHECK-P9-DAG: std 4, 56(1)
-; CHECK-P9-DAG: std 3, 48(1)
-; CHECK-P9-DAG: mtvsrd 1, 5
-; CHECK-P9-DAG: mtvsrd 2, 6
diff --git a/llvm/test/CodeGen/PowerPC/ppc64-elf-abi.ll b/llvm/test/CodeGen/PowerPC/ppc64-elf-abi.ll
index 0842f357d..8b1cf6b58 100644
--- a/llvm/test/CodeGen/PowerPC/ppc64-elf-abi.ll
+++ b/llvm/test/CodeGen/PowerPC/ppc64-elf-abi.ll
@@ -8,8 +8,6 @@
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64-unknown-freebsd < %s | FileCheck %s -check-prefix=CHECK-ELFv1
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64-unknown-freebsd -target-abi elfv1 < %s | FileCheck %s -check-prefix=CHECK-ELFv1
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64-unknown-freebsd -target-abi elfv2 < %s | FileCheck %s -check-prefix=CHECK-ELFv2
-; RUN: llc -verify-machineinstrs -mtriple=powerpc64-unknown-freebsd-elfv1  < %s | FileCheck %s -check-prefix=CHECK-ELFv1
-; RUN: llc -verify-machineinstrs -mtriple=powerpc64-unknown-freebsd-elfv2  < %s | FileCheck %s -check-prefix=CHECK-ELFv2
 
 ; CHECK-ELFv2: .abiversion 2
 ; CHECK-ELFv1-NOT: .abiversion 2
diff --git a/llvm/test/CodeGen/PowerPC/ppc64-pre-inc-no-extra-phi.ll b/llvm/test/CodeGen/PowerPC/ppc64-pre-inc-no-extra-phi.ll
index 032bad6f6..ab84707ce 100644
--- a/llvm/test/CodeGen/PowerPC/ppc64-pre-inc-no-extra-phi.ll
+++ b/llvm/test/CodeGen/PowerPC/ppc64-pre-inc-no-extra-phi.ll
@@ -3,7 +3,7 @@
 ; RUN: llc < %s -mtriple=powerpc64le-unknown-linux-gnu -mcpu=pwr8 -verify-machineinstrs | FileCheck %s
 ; RUN: llc < %s -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr8 -verify-machineinstrs | FileCheck %s
 
-@perm = common local_unnamed_addr global [100 x i64] zeroinitializer, align 8
+@perm = local_unnamed_addr global [100 x i64] zeroinitializer, align 8
 
 define void @sort_basket() local_unnamed_addr {
 entry:
diff --git a/llvm/test/CodeGen/PowerPC/ppc64-smallarg.ll b/llvm/test/CodeGen/PowerPC/ppc64-smallarg.ll
index 710077d6e..a71ea80b6 100644
--- a/llvm/test/CodeGen/PowerPC/ppc64-smallarg.ll
+++ b/llvm/test/CodeGen/PowerPC/ppc64-smallarg.ll
@@ -52,7 +52,7 @@ entry:
   ret void
 }
 ; CHECK: @caller2
-; CHECK: stfs {{[0-9]+}}, 156(1)
+; CHECK: std {{[0-9]+}}, 16(1)
 ; CHECK: bl test2
 
 declare float @test2(float, float, float, float, float, float, float, float, float, float, float, float, float, float)
diff --git a/llvm/test/CodeGen/PowerPC/ppc64le-smallarg.ll b/llvm/test/CodeGen/PowerPC/ppc64le-smallarg.ll
index 8e853cd7d..9eac7d0c4 100644
--- a/llvm/test/CodeGen/PowerPC/ppc64le-smallarg.ll
+++ b/llvm/test/CodeGen/PowerPC/ppc64le-smallarg.ll
@@ -52,7 +52,7 @@ entry:
   ret void
 }
 ; CHECK: @caller2
-; CHECK: stfs {{[0-9]+}}, 136({{[0-9]+}})
+; CHECK: std {{[0-9]+}}, 16({{[0-9]+}})
 ; CHECK: bl test2
 
 declare float @test2(float, float, float, float, float, float, float, float, float, float, float, float, float, float)
diff --git a/llvm/test/CodeGen/PowerPC/ppcf128-constrained-fp-intrinsics.ll b/llvm/test/CodeGen/PowerPC/ppcf128-constrained-fp-intrinsics.ll
new file mode 100644
index 000000000..e87620a37
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/ppcf128-constrained-fp-intrinsics.ll
@@ -0,0 +1,1563 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -O3 -mtriple=powerpc64le-linux-gnu < %s | FileCheck --check-prefix=PC64LE %s
+; RUN: llc -O3 -mtriple=powerpc64le-linux-gnu -mcpu=pwr9 < %s | FileCheck --check-prefix=PC64LE9 %s
+; RUN: llc -O3 -mtriple=powerpc64-linux-gnu < %s | FileCheck --check-prefix=PC64 %s
+
+define ppc_fp128 @test_fadd_ppc_fp128(ppc_fp128 %first, ppc_fp128 %second) nounwind {
+; PC64LE-LABEL: test_fadd_ppc_fp128:
+; PC64LE:       # %bb.0: # %entry
+; PC64LE-NEXT:    mflr 0
+; PC64LE-NEXT:    std 0, 16(1)
+; PC64LE-NEXT:    stdu 1, -32(1)
+; PC64LE-NEXT:    bl __gcc_qadd
+; PC64LE-NEXT:    nop
+; PC64LE-NEXT:    addi 1, 1, 32
+; PC64LE-NEXT:    ld 0, 16(1)
+; PC64LE-NEXT:    mtlr 0
+; PC64LE-NEXT:    blr
+;
+; PC64LE9-LABEL: test_fadd_ppc_fp128:
+; PC64LE9:       # %bb.0: # %entry
+; PC64LE9-NEXT:    mflr 0
+; PC64LE9-NEXT:    std 0, 16(1)
+; PC64LE9-NEXT:    stdu 1, -32(1)
+; PC64LE9-NEXT:    bl __gcc_qadd
+; PC64LE9-NEXT:    nop
+; PC64LE9-NEXT:    addi 1, 1, 32
+; PC64LE9-NEXT:    ld 0, 16(1)
+; PC64LE9-NEXT:    mtlr 0
+; PC64LE9-NEXT:    blr
+;
+; PC64-LABEL: test_fadd_ppc_fp128:
+; PC64:       # %bb.0: # %entry
+; PC64-NEXT:    mflr 0
+; PC64-NEXT:    std 0, 16(1)
+; PC64-NEXT:    stdu 1, -112(1)
+; PC64-NEXT:    bl __gcc_qadd
+; PC64-NEXT:    nop
+; PC64-NEXT:    addi 1, 1, 112
+; PC64-NEXT:    ld 0, 16(1)
+; PC64-NEXT:    mtlr 0
+; PC64-NEXT:    blr
+entry:
+  %add = call ppc_fp128 @llvm.experimental.constrained.fadd.ppcf128(
+                    ppc_fp128 %first,
+                    ppc_fp128 %second,
+                    metadata !"round.dynamic",
+                    metadata !"fpexcept.strict")
+  ret ppc_fp128 %add
+}
+
+define ppc_fp128 @test_fsub_ppc_fp128(ppc_fp128 %first, ppc_fp128 %second) nounwind {
+; PC64LE-LABEL: test_fsub_ppc_fp128:
+; PC64LE:       # %bb.0: # %entry
+; PC64LE-NEXT:    mflr 0
+; PC64LE-NEXT:    std 0, 16(1)
+; PC64LE-NEXT:    stdu 1, -32(1)
+; PC64LE-NEXT:    bl __gcc_qsub
+; PC64LE-NEXT:    nop
+; PC64LE-NEXT:    addi 1, 1, 32
+; PC64LE-NEXT:    ld 0, 16(1)
+; PC64LE-NEXT:    mtlr 0
+; PC64LE-NEXT:    blr
+;
+; PC64LE9-LABEL: test_fsub_ppc_fp128:
+; PC64LE9:       # %bb.0: # %entry
+; PC64LE9-NEXT:    mflr 0
+; PC64LE9-NEXT:    std 0, 16(1)
+; PC64LE9-NEXT:    stdu 1, -32(1)
+; PC64LE9-NEXT:    bl __gcc_qsub
+; PC64LE9-NEXT:    nop
+; PC64LE9-NEXT:    addi 1, 1, 32
+; PC64LE9-NEXT:    ld 0, 16(1)
+; PC64LE9-NEXT:    mtlr 0
+; PC64LE9-NEXT:    blr
+;
+; PC64-LABEL: test_fsub_ppc_fp128:
+; PC64:       # %bb.0: # %entry
+; PC64-NEXT:    mflr 0
+; PC64-NEXT:    std 0, 16(1)
+; PC64-NEXT:    stdu 1, -112(1)
+; PC64-NEXT:    bl __gcc_qsub
+; PC64-NEXT:    nop
+; PC64-NEXT:    addi 1, 1, 112
+; PC64-NEXT:    ld 0, 16(1)
+; PC64-NEXT:    mtlr 0
+; PC64-NEXT:    blr
+entry:
+  %sub = call ppc_fp128 @llvm.experimental.constrained.fsub.ppcf128(
+                    ppc_fp128 %first,
+                    ppc_fp128 %second,
+                    metadata !"round.dynamic",
+                    metadata !"fpexcept.strict")
+  ret ppc_fp128 %sub
+}
+
+define ppc_fp128 @test_fmul_ppc_fp128(ppc_fp128 %first, ppc_fp128 %second) nounwind {
+; PC64LE-LABEL: test_fmul_ppc_fp128:
+; PC64LE:       # %bb.0: # %entry
+; PC64LE-NEXT:    mflr 0
+; PC64LE-NEXT:    std 0, 16(1)
+; PC64LE-NEXT:    stdu 1, -32(1)
+; PC64LE-NEXT:    bl __gcc_qmul
+; PC64LE-NEXT:    nop
+; PC64LE-NEXT:    addi 1, 1, 32
+; PC64LE-NEXT:    ld 0, 16(1)
+; PC64LE-NEXT:    mtlr 0
+; PC64LE-NEXT:    blr
+;
+; PC64LE9-LABEL: test_fmul_ppc_fp128:
+; PC64LE9:       # %bb.0: # %entry
+; PC64LE9-NEXT:    mflr 0
+; PC64LE9-NEXT:    std 0, 16(1)
+; PC64LE9-NEXT:    stdu 1, -32(1)
+; PC64LE9-NEXT:    bl __gcc_qmul
+; PC64LE9-NEXT:    nop
+; PC64LE9-NEXT:    addi 1, 1, 32
+; PC64LE9-NEXT:    ld 0, 16(1)
+; PC64LE9-NEXT:    mtlr 0
+; PC64LE9-NEXT:    blr
+;
+; PC64-LABEL: test_fmul_ppc_fp128:
+; PC64:       # %bb.0: # %entry
+; PC64-NEXT:    mflr 0
+; PC64-NEXT:    std 0, 16(1)
+; PC64-NEXT:    stdu 1, -112(1)
+; PC64-NEXT:    bl __gcc_qmul
+; PC64-NEXT:    nop
+; PC64-NEXT:    addi 1, 1, 112
+; PC64-NEXT:    ld 0, 16(1)
+; PC64-NEXT:    mtlr 0
+; PC64-NEXT:    blr
+entry:
+  %mul = call ppc_fp128 @llvm.experimental.constrained.fmul.ppcf128(
+                    ppc_fp128 %first,
+                    ppc_fp128 %second,
+                    metadata !"round.dynamic",
+                    metadata !"fpexcept.strict")
+  ret ppc_fp128 %mul
+}
+
+define ppc_fp128 @test_fdiv_ppc_fp128(ppc_fp128 %first, ppc_fp128 %second) nounwind {
+; PC64LE-LABEL: test_fdiv_ppc_fp128:
+; PC64LE:       # %bb.0: # %entry
+; PC64LE-NEXT:    mflr 0
+; PC64LE-NEXT:    std 0, 16(1)
+; PC64LE-NEXT:    stdu 1, -32(1)
+; PC64LE-NEXT:    bl __gcc_qdiv
+; PC64LE-NEXT:    nop
+; PC64LE-NEXT:    addi 1, 1, 32
+; PC64LE-NEXT:    ld 0, 16(1)
+; PC64LE-NEXT:    mtlr 0
+; PC64LE-NEXT:    blr
+;
+; PC64LE9-LABEL: test_fdiv_ppc_fp128:
+; PC64LE9:       # %bb.0: # %entry
+; PC64LE9-NEXT:    mflr 0
+; PC64LE9-NEXT:    std 0, 16(1)
+; PC64LE9-NEXT:    stdu 1, -32(1)
+; PC64LE9-NEXT:    bl __gcc_qdiv
+; PC64LE9-NEXT:    nop
+; PC64LE9-NEXT:    addi 1, 1, 32
+; PC64LE9-NEXT:    ld 0, 16(1)
+; PC64LE9-NEXT:    mtlr 0
+; PC64LE9-NEXT:    blr
+;
+; PC64-LABEL: test_fdiv_ppc_fp128:
+; PC64:       # %bb.0: # %entry
+; PC64-NEXT:    mflr 0
+; PC64-NEXT:    std 0, 16(1)
+; PC64-NEXT:    stdu 1, -112(1)
+; PC64-NEXT:    bl __gcc_qdiv
+; PC64-NEXT:    nop
+; PC64-NEXT:    addi 1, 1, 112
+; PC64-NEXT:    ld 0, 16(1)
+; PC64-NEXT:    mtlr 0
+; PC64-NEXT:    blr
+entry:
+  %div = call ppc_fp128 @llvm.experimental.constrained.fdiv.ppcf128(
+                    ppc_fp128 %first,
+                    ppc_fp128 %second,
+                    metadata !"round.dynamic",
+                    metadata !"fpexcept.strict")
+  ret ppc_fp128 %div
+}
+
+define ppc_fp128 @test_frem_ppc_fp128(ppc_fp128 %first, ppc_fp128 %second) nounwind {
+; PC64LE-LABEL: test_frem_ppc_fp128:
+; PC64LE:       # %bb.0: # %entry
+; PC64LE-NEXT:    mflr 0
+; PC64LE-NEXT:    std 0, 16(1)
+; PC64LE-NEXT:    stdu 1, -32(1)
+; PC64LE-NEXT:    bl fmodl
+; PC64LE-NEXT:    nop
+; PC64LE-NEXT:    addi 1, 1, 32
+; PC64LE-NEXT:    ld 0, 16(1)
+; PC64LE-NEXT:    mtlr 0
+; PC64LE-NEXT:    blr
+;
+; PC64LE9-LABEL: test_frem_ppc_fp128:
+; PC64LE9:       # %bb.0: # %entry
+; PC64LE9-NEXT:    mflr 0
+; PC64LE9-NEXT:    std 0, 16(1)
+; PC64LE9-NEXT:    stdu 1, -32(1)
+; PC64LE9-NEXT:    bl fmodl
+; PC64LE9-NEXT:    nop
+; PC64LE9-NEXT:    addi 1, 1, 32
+; PC64LE9-NEXT:    ld 0, 16(1)
+; PC64LE9-NEXT:    mtlr 0
+; PC64LE9-NEXT:    blr
+;
+; PC64-LABEL: test_frem_ppc_fp128:
+; PC64:       # %bb.0: # %entry
+; PC64-NEXT:    mflr 0
+; PC64-NEXT:    std 0, 16(1)
+; PC64-NEXT:    stdu 1, -112(1)
+; PC64-NEXT:    bl fmodl
+; PC64-NEXT:    nop
+; PC64-NEXT:    addi 1, 1, 112
+; PC64-NEXT:    ld 0, 16(1)
+; PC64-NEXT:    mtlr 0
+; PC64-NEXT:    blr
+entry:
+  %rem = call ppc_fp128 @llvm.experimental.constrained.frem.ppcf128(
+                    ppc_fp128 %first,
+                    ppc_fp128 %second,
+                    metadata !"round.dynamic",
+                    metadata !"fpexcept.strict")
+  ret ppc_fp128 %rem
+}
+
+define ppc_fp128 @test_fma_ppc_fp128(ppc_fp128 %first, ppc_fp128 %second, ppc_fp128 %third) nounwind {
+; PC64LE-LABEL: test_fma_ppc_fp128:
+; PC64LE:       # %bb.0: # %entry
+; PC64LE-NEXT:    mflr 0
+; PC64LE-NEXT:    std 0, 16(1)
+; PC64LE-NEXT:    stdu 1, -32(1)
+; PC64LE-NEXT:    bl fmal
+; PC64LE-NEXT:    nop
+; PC64LE-NEXT:    addi 1, 1, 32
+; PC64LE-NEXT:    ld 0, 16(1)
+; PC64LE-NEXT:    mtlr 0
+; PC64LE-NEXT:    blr
+;
+; PC64LE9-LABEL: test_fma_ppc_fp128:
+; PC64LE9:       # %bb.0: # %entry
+; PC64LE9-NEXT:    mflr 0
+; PC64LE9-NEXT:    std 0, 16(1)
+; PC64LE9-NEXT:    stdu 1, -32(1)
+; PC64LE9-NEXT:    bl fmal
+; PC64LE9-NEXT:    nop
+; PC64LE9-NEXT:    addi 1, 1, 32
+; PC64LE9-NEXT:    ld 0, 16(1)
+; PC64LE9-NEXT:    mtlr 0
+; PC64LE9-NEXT:    blr
+;
+; PC64-LABEL: test_fma_ppc_fp128:
+; PC64:       # %bb.0: # %entry
+; PC64-NEXT:    mflr 0
+; PC64-NEXT:    std 0, 16(1)
+; PC64-NEXT:    stdu 1, -112(1)
+; PC64-NEXT:    bl fmal
+; PC64-NEXT:    nop
+; PC64-NEXT:    addi 1, 1, 112
+; PC64-NEXT:    ld 0, 16(1)
+; PC64-NEXT:    mtlr 0
+; PC64-NEXT:    blr
+entry:
+  %add = call ppc_fp128 @llvm.experimental.constrained.fma.ppcf128(
+                    ppc_fp128 %first,
+                    ppc_fp128 %second,
+                    ppc_fp128 %third,
+                    metadata !"round.dynamic",
+                    metadata !"fpexcept.strict")
+  ret ppc_fp128 %add
+}
+
+define ppc_fp128 @test_sqrt_ppc_fp128(ppc_fp128 %first) nounwind {
+; PC64LE-LABEL: test_sqrt_ppc_fp128:
+; PC64LE:       # %bb.0: # %entry
+; PC64LE-NEXT:    mflr 0
+; PC64LE-NEXT:    std 0, 16(1)
+; PC64LE-NEXT:    stdu 1, -32(1)
+; PC64LE-NEXT:    bl sqrtl
+; PC64LE-NEXT:    nop
+; PC64LE-NEXT:    addi 1, 1, 32
+; PC64LE-NEXT:    ld 0, 16(1)
+; PC64LE-NEXT:    mtlr 0
+; PC64LE-NEXT:    blr
+;
+; PC64LE9-LABEL: test_sqrt_ppc_fp128:
+; PC64LE9:       # %bb.0: # %entry
+; PC64LE9-NEXT:    mflr 0
+; PC64LE9-NEXT:    std 0, 16(1)
+; PC64LE9-NEXT:    stdu 1, -32(1)
+; PC64LE9-NEXT:    bl sqrtl
+; PC64LE9-NEXT:    nop
+; PC64LE9-NEXT:    addi 1, 1, 32
+; PC64LE9-NEXT:    ld 0, 16(1)
+; PC64LE9-NEXT:    mtlr 0
+; PC64LE9-NEXT:    blr
+;
+; PC64-LABEL: test_sqrt_ppc_fp128:
+; PC64:       # %bb.0: # %entry
+; PC64-NEXT:    mflr 0
+; PC64-NEXT:    std 0, 16(1)
+; PC64-NEXT:    stdu 1, -112(1)
+; PC64-NEXT:    bl sqrtl
+; PC64-NEXT:    nop
+; PC64-NEXT:    addi 1, 1, 112
+; PC64-NEXT:    ld 0, 16(1)
+; PC64-NEXT:    mtlr 0
+; PC64-NEXT:    blr
+entry:
+  %sqrt = call ppc_fp128 @llvm.experimental.constrained.sqrt.ppcf128(
+                    ppc_fp128 %first,
+                    metadata !"round.dynamic",
+                    metadata !"fpexcept.strict")
+  ret ppc_fp128 %sqrt
+}
+
+define ppc_fp128 @test_pow_ppc_fp128(ppc_fp128 %first, ppc_fp128 %second) nounwind {
+; PC64LE-LABEL: test_pow_ppc_fp128:
+; PC64LE:       # %bb.0: # %entry
+; PC64LE-NEXT:    mflr 0
+; PC64LE-NEXT:    std 0, 16(1)
+; PC64LE-NEXT:    stdu 1, -32(1)
+; PC64LE-NEXT:    bl powl
+; PC64LE-NEXT:    nop
+; PC64LE-NEXT:    addi 1, 1, 32
+; PC64LE-NEXT:    ld 0, 16(1)
+; PC64LE-NEXT:    mtlr 0
+; PC64LE-NEXT:    blr
+;
+; PC64LE9-LABEL: test_pow_ppc_fp128:
+; PC64LE9:       # %bb.0: # %entry
+; PC64LE9-NEXT:    mflr 0
+; PC64LE9-NEXT:    std 0, 16(1)
+; PC64LE9-NEXT:    stdu 1, -32(1)
+; PC64LE9-NEXT:    bl powl
+; PC64LE9-NEXT:    nop
+; PC64LE9-NEXT:    addi 1, 1, 32
+; PC64LE9-NEXT:    ld 0, 16(1)
+; PC64LE9-NEXT:    mtlr 0
+; PC64LE9-NEXT:    blr
+;
+; PC64-LABEL: test_pow_ppc_fp128:
+; PC64:       # %bb.0: # %entry
+; PC64-NEXT:    mflr 0
+; PC64-NEXT:    std 0, 16(1)
+; PC64-NEXT:    stdu 1, -112(1)
+; PC64-NEXT:    bl powl
+; PC64-NEXT:    nop
+; PC64-NEXT:    addi 1, 1, 112
+; PC64-NEXT:    ld 0, 16(1)
+; PC64-NEXT:    mtlr 0
+; PC64-NEXT:    blr
+entry:
+  %pow = call ppc_fp128 @llvm.experimental.constrained.pow.ppcf128(
+                    ppc_fp128 %first,
+                    ppc_fp128 %second,
+                    metadata !"round.dynamic",
+                    metadata !"fpexcept.strict")
+  ret ppc_fp128 %pow
+}
+
+define ppc_fp128 @test_powi_ppc_fp128(ppc_fp128 %first, i32 %second) nounwind {
+; PC64LE-LABEL: test_powi_ppc_fp128:
+; PC64LE:       # %bb.0: # %entry
+; PC64LE-NEXT:    mflr 0
+; PC64LE-NEXT:    std 0, 16(1)
+; PC64LE-NEXT:    stdu 1, -32(1)
+; PC64LE-NEXT:    clrldi 5, 5, 32
+; PC64LE-NEXT:    bl __powitf2
+; PC64LE-NEXT:    nop
+; PC64LE-NEXT:    addi 1, 1, 32
+; PC64LE-NEXT:    ld 0, 16(1)
+; PC64LE-NEXT:    mtlr 0
+; PC64LE-NEXT:    blr
+;
+; PC64LE9-LABEL: test_powi_ppc_fp128:
+; PC64LE9:       # %bb.0: # %entry
+; PC64LE9-NEXT:    mflr 0
+; PC64LE9-NEXT:    std 0, 16(1)
+; PC64LE9-NEXT:    stdu 1, -32(1)
+; PC64LE9-NEXT:    clrldi 5, 5, 32
+; PC64LE9-NEXT:    bl __powitf2
+; PC64LE9-NEXT:    nop
+; PC64LE9-NEXT:    addi 1, 1, 32
+; PC64LE9-NEXT:    ld 0, 16(1)
+; PC64LE9-NEXT:    mtlr 0
+; PC64LE9-NEXT:    blr
+;
+; PC64-LABEL: test_powi_ppc_fp128:
+; PC64:       # %bb.0: # %entry
+; PC64-NEXT:    mflr 0
+; PC64-NEXT:    std 0, 16(1)
+; PC64-NEXT:    stdu 1, -112(1)
+; PC64-NEXT:    clrldi 5, 5, 32
+; PC64-NEXT:    bl __powitf2
+; PC64-NEXT:    nop
+; PC64-NEXT:    addi 1, 1, 112
+; PC64-NEXT:    ld 0, 16(1)
+; PC64-NEXT:    mtlr 0
+; PC64-NEXT:    blr
+; PC64LE9     :    clrldi 5, 5, 32
+entry:
+  %powi = call ppc_fp128 @llvm.experimental.constrained.powi.ppcf128(
+                    ppc_fp128 %first,
+                    i32 %second,
+                    metadata !"round.dynamic",
+                    metadata !"fpexcept.strict")
+  ret ppc_fp128 %powi
+}
+
+define ppc_fp128 @test_sin_ppc_fp128(ppc_fp128 %first) nounwind {
+; PC64LE-LABEL: test_sin_ppc_fp128:
+; PC64LE:       # %bb.0: # %entry
+; PC64LE-NEXT:    mflr 0
+; PC64LE-NEXT:    std 0, 16(1)
+; PC64LE-NEXT:    stdu 1, -32(1)
+; PC64LE-NEXT:    bl sinl
+; PC64LE-NEXT:    nop
+; PC64LE-NEXT:    addi 1, 1, 32
+; PC64LE-NEXT:    ld 0, 16(1)
+; PC64LE-NEXT:    mtlr 0
+; PC64LE-NEXT:    blr
+;
+; PC64LE9-LABEL: test_sin_ppc_fp128:
+; PC64LE9:       # %bb.0: # %entry
+; PC64LE9-NEXT:    mflr 0
+; PC64LE9-NEXT:    std 0, 16(1)
+; PC64LE9-NEXT:    stdu 1, -32(1)
+; PC64LE9-NEXT:    bl sinl
+; PC64LE9-NEXT:    nop
+; PC64LE9-NEXT:    addi 1, 1, 32
+; PC64LE9-NEXT:    ld 0, 16(1)
+; PC64LE9-NEXT:    mtlr 0
+; PC64LE9-NEXT:    blr
+;
+; PC64-LABEL: test_sin_ppc_fp128:
+; PC64:       # %bb.0: # %entry
+; PC64-NEXT:    mflr 0
+; PC64-NEXT:    std 0, 16(1)
+; PC64-NEXT:    stdu 1, -112(1)
+; PC64-NEXT:    bl sinl
+; PC64-NEXT:    nop
+; PC64-NEXT:    addi 1, 1, 112
+; PC64-NEXT:    ld 0, 16(1)
+; PC64-NEXT:    mtlr 0
+; PC64-NEXT:    blr
+entry:
+  %sin = call ppc_fp128 @llvm.experimental.constrained.sin.ppcf128(
+                    ppc_fp128 %first,
+                    metadata !"round.dynamic",
+                    metadata !"fpexcept.strict")
+  ret ppc_fp128 %sin
+}
+
+define ppc_fp128 @test_cos_ppc_fp128(ppc_fp128 %first) nounwind {
+; PC64LE-LABEL: test_cos_ppc_fp128:
+; PC64LE:       # %bb.0: # %entry
+; PC64LE-NEXT:    mflr 0
+; PC64LE-NEXT:    std 0, 16(1)
+; PC64LE-NEXT:    stdu 1, -32(1)
+; PC64LE-NEXT:    bl cosl
+; PC64LE-NEXT:    nop
+; PC64LE-NEXT:    addi 1, 1, 32
+; PC64LE-NEXT:    ld 0, 16(1)
+; PC64LE-NEXT:    mtlr 0
+; PC64LE-NEXT:    blr
+;
+; PC64LE9-LABEL: test_cos_ppc_fp128:
+; PC64LE9:       # %bb.0: # %entry
+; PC64LE9-NEXT:    mflr 0
+; PC64LE9-NEXT:    std 0, 16(1)
+; PC64LE9-NEXT:    stdu 1, -32(1)
+; PC64LE9-NEXT:    bl cosl
+; PC64LE9-NEXT:    nop
+; PC64LE9-NEXT:    addi 1, 1, 32
+; PC64LE9-NEXT:    ld 0, 16(1)
+; PC64LE9-NEXT:    mtlr 0
+; PC64LE9-NEXT:    blr
+;
+; PC64-LABEL: test_cos_ppc_fp128:
+; PC64:       # %bb.0: # %entry
+; PC64-NEXT:    mflr 0
+; PC64-NEXT:    std 0, 16(1)
+; PC64-NEXT:    stdu 1, -112(1)
+; PC64-NEXT:    bl cosl
+; PC64-NEXT:    nop
+; PC64-NEXT:    addi 1, 1, 112
+; PC64-NEXT:    ld 0, 16(1)
+; PC64-NEXT:    mtlr 0
+; PC64-NEXT:    blr
+entry:
+  %cos = call ppc_fp128 @llvm.experimental.constrained.cos.ppcf128(
+                    ppc_fp128 %first,
+                    metadata !"round.dynamic",
+                    metadata !"fpexcept.strict")
+  ret ppc_fp128 %cos
+}
+
+define ppc_fp128 @test_exp_ppc_fp128(ppc_fp128 %first) nounwind {
+; PC64LE-LABEL: test_exp_ppc_fp128:
+; PC64LE:       # %bb.0: # %entry
+; PC64LE-NEXT:    mflr 0
+; PC64LE-NEXT:    std 0, 16(1)
+; PC64LE-NEXT:    stdu 1, -32(1)
+; PC64LE-NEXT:    bl expl
+; PC64LE-NEXT:    nop
+; PC64LE-NEXT:    addi 1, 1, 32
+; PC64LE-NEXT:    ld 0, 16(1)
+; PC64LE-NEXT:    mtlr 0
+; PC64LE-NEXT:    blr
+;
+; PC64LE9-LABEL: test_exp_ppc_fp128:
+; PC64LE9:       # %bb.0: # %entry
+; PC64LE9-NEXT:    mflr 0
+; PC64LE9-NEXT:    std 0, 16(1)
+; PC64LE9-NEXT:    stdu 1, -32(1)
+; PC64LE9-NEXT:    bl expl
+; PC64LE9-NEXT:    nop
+; PC64LE9-NEXT:    addi 1, 1, 32
+; PC64LE9-NEXT:    ld 0, 16(1)
+; PC64LE9-NEXT:    mtlr 0
+; PC64LE9-NEXT:    blr
+;
+; PC64-LABEL: test_exp_ppc_fp128:
+; PC64:       # %bb.0: # %entry
+; PC64-NEXT:    mflr 0
+; PC64-NEXT:    std 0, 16(1)
+; PC64-NEXT:    stdu 1, -112(1)
+; PC64-NEXT:    bl expl
+; PC64-NEXT:    nop
+; PC64-NEXT:    addi 1, 1, 112
+; PC64-NEXT:    ld 0, 16(1)
+; PC64-NEXT:    mtlr 0
+; PC64-NEXT:    blr
+entry:
+  %exp = call ppc_fp128 @llvm.experimental.constrained.exp.ppcf128(
+                    ppc_fp128 %first,
+                    metadata !"round.dynamic",
+                    metadata !"fpexcept.strict")
+  ret ppc_fp128 %exp
+}
+
+define ppc_fp128 @test_exp2_ppc_fp128(ppc_fp128 %first) nounwind {
+; PC64LE-LABEL: test_exp2_ppc_fp128:
+; PC64LE:       # %bb.0: # %entry
+; PC64LE-NEXT:    mflr 0
+; PC64LE-NEXT:    std 0, 16(1)
+; PC64LE-NEXT:    stdu 1, -32(1)
+; PC64LE-NEXT:    bl exp2l
+; PC64LE-NEXT:    nop
+; PC64LE-NEXT:    addi 1, 1, 32
+; PC64LE-NEXT:    ld 0, 16(1)
+; PC64LE-NEXT:    mtlr 0
+; PC64LE-NEXT:    blr
+;
+; PC64LE9-LABEL: test_exp2_ppc_fp128:
+; PC64LE9:       # %bb.0: # %entry
+; PC64LE9-NEXT:    mflr 0
+; PC64LE9-NEXT:    std 0, 16(1)
+; PC64LE9-NEXT:    stdu 1, -32(1)
+; PC64LE9-NEXT:    bl exp2l
+; PC64LE9-NEXT:    nop
+; PC64LE9-NEXT:    addi 1, 1, 32
+; PC64LE9-NEXT:    ld 0, 16(1)
+; PC64LE9-NEXT:    mtlr 0
+; PC64LE9-NEXT:    blr
+;
+; PC64-LABEL: test_exp2_ppc_fp128:
+; PC64:       # %bb.0: # %entry
+; PC64-NEXT:    mflr 0
+; PC64-NEXT:    std 0, 16(1)
+; PC64-NEXT:    stdu 1, -112(1)
+; PC64-NEXT:    bl exp2l
+; PC64-NEXT:    nop
+; PC64-NEXT:    addi 1, 1, 112
+; PC64-NEXT:    ld 0, 16(1)
+; PC64-NEXT:    mtlr 0
+; PC64-NEXT:    blr
+entry:
+  %exp2 = call ppc_fp128 @llvm.experimental.constrained.exp2.ppcf128(
+                    ppc_fp128 %first,
+                    metadata !"round.dynamic",
+                    metadata !"fpexcept.strict")
+  ret ppc_fp128 %exp2
+}
+
+define ppc_fp128 @test_log_ppc_fp128(ppc_fp128 %first) nounwind {
+; PC64LE-LABEL: test_log_ppc_fp128:
+; PC64LE:       # %bb.0: # %entry
+; PC64LE-NEXT:    mflr 0
+; PC64LE-NEXT:    std 0, 16(1)
+; PC64LE-NEXT:    stdu 1, -32(1)
+; PC64LE-NEXT:    bl logl
+; PC64LE-NEXT:    nop
+; PC64LE-NEXT:    addi 1, 1, 32
+; PC64LE-NEXT:    ld 0, 16(1)
+; PC64LE-NEXT:    mtlr 0
+; PC64LE-NEXT:    blr
+;
+; PC64LE9-LABEL: test_log_ppc_fp128:
+; PC64LE9:       # %bb.0: # %entry
+; PC64LE9-NEXT:    mflr 0
+; PC64LE9-NEXT:    std 0, 16(1)
+; PC64LE9-NEXT:    stdu 1, -32(1)
+; PC64LE9-NEXT:    bl logl
+; PC64LE9-NEXT:    nop
+; PC64LE9-NEXT:    addi 1, 1, 32
+; PC64LE9-NEXT:    ld 0, 16(1)
+; PC64LE9-NEXT:    mtlr 0
+; PC64LE9-NEXT:    blr
+;
+; PC64-LABEL: test_log_ppc_fp128:
+; PC64:       # %bb.0: # %entry
+; PC64-NEXT:    mflr 0
+; PC64-NEXT:    std 0, 16(1)
+; PC64-NEXT:    stdu 1, -112(1)
+; PC64-NEXT:    bl logl
+; PC64-NEXT:    nop
+; PC64-NEXT:    addi 1, 1, 112
+; PC64-NEXT:    ld 0, 16(1)
+; PC64-NEXT:    mtlr 0
+; PC64-NEXT:    blr
+entry:
+  %log = call ppc_fp128 @llvm.experimental.constrained.log.ppcf128(
+                    ppc_fp128 %first,
+                    metadata !"round.dynamic",
+                    metadata !"fpexcept.strict")
+  ret ppc_fp128 %log
+}
+
+define ppc_fp128 @test_log2_ppc_fp128(ppc_fp128 %first) nounwind {
+; PC64LE-LABEL: test_log2_ppc_fp128:
+; PC64LE:       # %bb.0: # %entry
+; PC64LE-NEXT:    mflr 0
+; PC64LE-NEXT:    std 0, 16(1)
+; PC64LE-NEXT:    stdu 1, -32(1)
+; PC64LE-NEXT:    bl log2l
+; PC64LE-NEXT:    nop
+; PC64LE-NEXT:    addi 1, 1, 32
+; PC64LE-NEXT:    ld 0, 16(1)
+; PC64LE-NEXT:    mtlr 0
+; PC64LE-NEXT:    blr
+;
+; PC64LE9-LABEL: test_log2_ppc_fp128:
+; PC64LE9:       # %bb.0: # %entry
+; PC64LE9-NEXT:    mflr 0
+; PC64LE9-NEXT:    std 0, 16(1)
+; PC64LE9-NEXT:    stdu 1, -32(1)
+; PC64LE9-NEXT:    bl log2l
+; PC64LE9-NEXT:    nop
+; PC64LE9-NEXT:    addi 1, 1, 32
+; PC64LE9-NEXT:    ld 0, 16(1)
+; PC64LE9-NEXT:    mtlr 0
+; PC64LE9-NEXT:    blr
+;
+; PC64-LABEL: test_log2_ppc_fp128:
+; PC64:       # %bb.0: # %entry
+; PC64-NEXT:    mflr 0
+; PC64-NEXT:    std 0, 16(1)
+; PC64-NEXT:    stdu 1, -112(1)
+; PC64-NEXT:    bl log2l
+; PC64-NEXT:    nop
+; PC64-NEXT:    addi 1, 1, 112
+; PC64-NEXT:    ld 0, 16(1)
+; PC64-NEXT:    mtlr 0
+; PC64-NEXT:    blr
+entry:
+  %log2 = call ppc_fp128 @llvm.experimental.constrained.log2.ppcf128(
+                    ppc_fp128 %first,
+                    metadata !"round.dynamic",
+                    metadata !"fpexcept.strict")
+  ret ppc_fp128 %log2
+}
+
+define ppc_fp128 @test_log10_ppc_fp128(ppc_fp128 %first) nounwind {
+; PC64LE-LABEL: test_log10_ppc_fp128:
+; PC64LE:       # %bb.0: # %entry
+; PC64LE-NEXT:    mflr 0
+; PC64LE-NEXT:    std 0, 16(1)
+; PC64LE-NEXT:    stdu 1, -32(1)
+; PC64LE-NEXT:    bl log10l
+; PC64LE-NEXT:    nop
+; PC64LE-NEXT:    addi 1, 1, 32
+; PC64LE-NEXT:    ld 0, 16(1)
+; PC64LE-NEXT:    mtlr 0
+; PC64LE-NEXT:    blr
+;
+; PC64LE9-LABEL: test_log10_ppc_fp128:
+; PC64LE9:       # %bb.0: # %entry
+; PC64LE9-NEXT:    mflr 0
+; PC64LE9-NEXT:    std 0, 16(1)
+; PC64LE9-NEXT:    stdu 1, -32(1)
+; PC64LE9-NEXT:    bl log10l
+; PC64LE9-NEXT:    nop
+; PC64LE9-NEXT:    addi 1, 1, 32
+; PC64LE9-NEXT:    ld 0, 16(1)
+; PC64LE9-NEXT:    mtlr 0
+; PC64LE9-NEXT:    blr
+;
+; PC64-LABEL: test_log10_ppc_fp128:
+; PC64:       # %bb.0: # %entry
+; PC64-NEXT:    mflr 0
+; PC64-NEXT:    std 0, 16(1)
+; PC64-NEXT:    stdu 1, -112(1)
+; PC64-NEXT:    bl log10l
+; PC64-NEXT:    nop
+; PC64-NEXT:    addi 1, 1, 112
+; PC64-NEXT:    ld 0, 16(1)
+; PC64-NEXT:    mtlr 0
+; PC64-NEXT:    blr
+entry:
+  %log10 = call ppc_fp128 @llvm.experimental.constrained.log10.ppcf128(
+                    ppc_fp128 %first,
+                    metadata !"round.dynamic",
+                    metadata !"fpexcept.strict")
+  ret ppc_fp128 %log10
+}
+
+define ppc_fp128 @test_rint_ppc_fp128(ppc_fp128 %first) nounwind {
+; PC64LE-LABEL: test_rint_ppc_fp128:
+; PC64LE:       # %bb.0: # %entry
+; PC64LE-NEXT:    mflr 0
+; PC64LE-NEXT:    std 0, 16(1)
+; PC64LE-NEXT:    stdu 1, -32(1)
+; PC64LE-NEXT:    bl rintl
+; PC64LE-NEXT:    nop
+; PC64LE-NEXT:    addi 1, 1, 32
+; PC64LE-NEXT:    ld 0, 16(1)
+; PC64LE-NEXT:    mtlr 0
+; PC64LE-NEXT:    blr
+;
+; PC64LE9-LABEL: test_rint_ppc_fp128:
+; PC64LE9:       # %bb.0: # %entry
+; PC64LE9-NEXT:    mflr 0
+; PC64LE9-NEXT:    std 0, 16(1)
+; PC64LE9-NEXT:    stdu 1, -32(1)
+; PC64LE9-NEXT:    bl rintl
+; PC64LE9-NEXT:    nop
+; PC64LE9-NEXT:    addi 1, 1, 32
+; PC64LE9-NEXT:    ld 0, 16(1)
+; PC64LE9-NEXT:    mtlr 0
+; PC64LE9-NEXT:    blr
+;
+; PC64-LABEL: test_rint_ppc_fp128:
+; PC64:       # %bb.0: # %entry
+; PC64-NEXT:    mflr 0
+; PC64-NEXT:    std 0, 16(1)
+; PC64-NEXT:    stdu 1, -112(1)
+; PC64-NEXT:    bl rintl
+; PC64-NEXT:    nop
+; PC64-NEXT:    addi 1, 1, 112
+; PC64-NEXT:    ld 0, 16(1)
+; PC64-NEXT:    mtlr 0
+; PC64-NEXT:    blr
+entry:
+  %rint = call ppc_fp128 @llvm.experimental.constrained.rint.ppcf128(
+                    ppc_fp128 %first,
+                    metadata !"round.dynamic",
+                    metadata !"fpexcept.strict")
+  ret ppc_fp128 %rint
+}
+
+define ppc_fp128 @test_nearbyint_ppc_fp128(ppc_fp128 %first) nounwind {
+; PC64LE-LABEL: test_nearbyint_ppc_fp128:
+; PC64LE:       # %bb.0: # %entry
+; PC64LE-NEXT:    mflr 0
+; PC64LE-NEXT:    std 0, 16(1)
+; PC64LE-NEXT:    stdu 1, -32(1)
+; PC64LE-NEXT:    bl nearbyintl
+; PC64LE-NEXT:    nop
+; PC64LE-NEXT:    addi 1, 1, 32
+; PC64LE-NEXT:    ld 0, 16(1)
+; PC64LE-NEXT:    mtlr 0
+; PC64LE-NEXT:    blr
+;
+; PC64LE9-LABEL: test_nearbyint_ppc_fp128:
+; PC64LE9:       # %bb.0: # %entry
+; PC64LE9-NEXT:    mflr 0
+; PC64LE9-NEXT:    std 0, 16(1)
+; PC64LE9-NEXT:    stdu 1, -32(1)
+; PC64LE9-NEXT:    bl nearbyintl
+; PC64LE9-NEXT:    nop
+; PC64LE9-NEXT:    addi 1, 1, 32
+; PC64LE9-NEXT:    ld 0, 16(1)
+; PC64LE9-NEXT:    mtlr 0
+; PC64LE9-NEXT:    blr
+;
+; PC64-LABEL: test_nearbyint_ppc_fp128:
+; PC64:       # %bb.0: # %entry
+; PC64-NEXT:    mflr 0
+; PC64-NEXT:    std 0, 16(1)
+; PC64-NEXT:    stdu 1, -112(1)
+; PC64-NEXT:    bl nearbyintl
+; PC64-NEXT:    nop
+; PC64-NEXT:    addi 1, 1, 112
+; PC64-NEXT:    ld 0, 16(1)
+; PC64-NEXT:    mtlr 0
+; PC64-NEXT:    blr
+entry:
+  %nearbyint = call ppc_fp128 @llvm.experimental.constrained.nearbyint.ppcf128(
+                    ppc_fp128 %first,
+                    metadata !"round.dynamic",
+                    metadata !"fpexcept.strict")
+  ret ppc_fp128 %nearbyint
+}
+
+define ppc_fp128 @test_maxnum_ppc_fp128(ppc_fp128 %first, ppc_fp128 %second) nounwind {
+; PC64LE-LABEL: test_maxnum_ppc_fp128:
+; PC64LE:       # %bb.0: # %entry
+; PC64LE-NEXT:    mflr 0
+; PC64LE-NEXT:    std 0, 16(1)
+; PC64LE-NEXT:    stdu 1, -32(1)
+; PC64LE-NEXT:    bl fmaxl
+; PC64LE-NEXT:    nop
+; PC64LE-NEXT:    addi 1, 1, 32
+; PC64LE-NEXT:    ld 0, 16(1)
+; PC64LE-NEXT:    mtlr 0
+; PC64LE-NEXT:    blr
+;
+; PC64LE9-LABEL: test_maxnum_ppc_fp128:
+; PC64LE9:       # %bb.0: # %entry
+; PC64LE9-NEXT:    mflr 0
+; PC64LE9-NEXT:    std 0, 16(1)
+; PC64LE9-NEXT:    stdu 1, -32(1)
+; PC64LE9-NEXT:    bl fmaxl
+; PC64LE9-NEXT:    nop
+; PC64LE9-NEXT:    addi 1, 1, 32
+; PC64LE9-NEXT:    ld 0, 16(1)
+; PC64LE9-NEXT:    mtlr 0
+; PC64LE9-NEXT:    blr
+;
+; PC64-LABEL: test_maxnum_ppc_fp128:
+; PC64:       # %bb.0: # %entry
+; PC64-NEXT:    mflr 0
+; PC64-NEXT:    std 0, 16(1)
+; PC64-NEXT:    stdu 1, -112(1)
+; PC64-NEXT:    bl fmaxl
+; PC64-NEXT:    nop
+; PC64-NEXT:    addi 1, 1, 112
+; PC64-NEXT:    ld 0, 16(1)
+; PC64-NEXT:    mtlr 0
+; PC64-NEXT:    blr
+entry:
+  %maxnum = call ppc_fp128 @llvm.experimental.constrained.maxnum.ppcf128(
+                    ppc_fp128 %first,
+                    ppc_fp128 %second,
+                    metadata !"fpexcept.strict")
+  ret ppc_fp128 %maxnum
+}
+
+define ppc_fp128 @test_minnum_ppc_fp128(ppc_fp128 %first, ppc_fp128 %second) nounwind {
+; PC64LE-LABEL: test_minnum_ppc_fp128:
+; PC64LE:       # %bb.0: # %entry
+; PC64LE-NEXT:    mflr 0
+; PC64LE-NEXT:    std 0, 16(1)
+; PC64LE-NEXT:    stdu 1, -32(1)
+; PC64LE-NEXT:    bl fminl
+; PC64LE-NEXT:    nop
+; PC64LE-NEXT:    addi 1, 1, 32
+; PC64LE-NEXT:    ld 0, 16(1)
+; PC64LE-NEXT:    mtlr 0
+; PC64LE-NEXT:    blr
+;
+; PC64LE9-LABEL: test_minnum_ppc_fp128:
+; PC64LE9:       # %bb.0: # %entry
+; PC64LE9-NEXT:    mflr 0
+; PC64LE9-NEXT:    std 0, 16(1)
+; PC64LE9-NEXT:    stdu 1, -32(1)
+; PC64LE9-NEXT:    bl fminl
+; PC64LE9-NEXT:    nop
+; PC64LE9-NEXT:    addi 1, 1, 32
+; PC64LE9-NEXT:    ld 0, 16(1)
+; PC64LE9-NEXT:    mtlr 0
+; PC64LE9-NEXT:    blr
+;
+; PC64-LABEL: test_minnum_ppc_fp128:
+; PC64:       # %bb.0: # %entry
+; PC64-NEXT:    mflr 0
+; PC64-NEXT:    std 0, 16(1)
+; PC64-NEXT:    stdu 1, -112(1)
+; PC64-NEXT:    bl fminl
+; PC64-NEXT:    nop
+; PC64-NEXT:    addi 1, 1, 112
+; PC64-NEXT:    ld 0, 16(1)
+; PC64-NEXT:    mtlr 0
+; PC64-NEXT:    blr
+entry:
+  %minnum = call ppc_fp128 @llvm.experimental.constrained.minnum.ppcf128(
+                    ppc_fp128 %first,
+                    ppc_fp128 %second,
+                    metadata !"fpexcept.strict")
+  ret ppc_fp128 %minnum
+}
+
+define ppc_fp128 @test_ceil_ppc_fp128(ppc_fp128 %first) nounwind {
+; PC64LE-LABEL: test_ceil_ppc_fp128:
+; PC64LE:       # %bb.0: # %entry
+; PC64LE-NEXT:    mflr 0
+; PC64LE-NEXT:    std 0, 16(1)
+; PC64LE-NEXT:    stdu 1, -32(1)
+; PC64LE-NEXT:    bl ceill
+; PC64LE-NEXT:    nop
+; PC64LE-NEXT:    addi 1, 1, 32
+; PC64LE-NEXT:    ld 0, 16(1)
+; PC64LE-NEXT:    mtlr 0
+; PC64LE-NEXT:    blr
+;
+; PC64LE9-LABEL: test_ceil_ppc_fp128:
+; PC64LE9:       # %bb.0: # %entry
+; PC64LE9-NEXT:    mflr 0
+; PC64LE9-NEXT:    std 0, 16(1)
+; PC64LE9-NEXT:    stdu 1, -32(1)
+; PC64LE9-NEXT:    bl ceill
+; PC64LE9-NEXT:    nop
+; PC64LE9-NEXT:    addi 1, 1, 32
+; PC64LE9-NEXT:    ld 0, 16(1)
+; PC64LE9-NEXT:    mtlr 0
+; PC64LE9-NEXT:    blr
+;
+; PC64-LABEL: test_ceil_ppc_fp128:
+; PC64:       # %bb.0: # %entry
+; PC64-NEXT:    mflr 0
+; PC64-NEXT:    std 0, 16(1)
+; PC64-NEXT:    stdu 1, -112(1)
+; PC64-NEXT:    bl ceill
+; PC64-NEXT:    nop
+; PC64-NEXT:    addi 1, 1, 112
+; PC64-NEXT:    ld 0, 16(1)
+; PC64-NEXT:    mtlr 0
+; PC64-NEXT:    blr
+entry:
+  %ceil = call ppc_fp128 @llvm.experimental.constrained.ceil.ppcf128(
+                    ppc_fp128 %first,
+                    metadata !"fpexcept.strict")
+  ret ppc_fp128 %ceil
+}
+
+define ppc_fp128 @test_floor_ppc_fp128(ppc_fp128 %first) nounwind {
+; PC64LE-LABEL: test_floor_ppc_fp128:
+; PC64LE:       # %bb.0: # %entry
+; PC64LE-NEXT:    mflr 0
+; PC64LE-NEXT:    std 0, 16(1)
+; PC64LE-NEXT:    stdu 1, -32(1)
+; PC64LE-NEXT:    bl floorl
+; PC64LE-NEXT:    nop
+; PC64LE-NEXT:    addi 1, 1, 32
+; PC64LE-NEXT:    ld 0, 16(1)
+; PC64LE-NEXT:    mtlr 0
+; PC64LE-NEXT:    blr
+;
+; PC64LE9-LABEL: test_floor_ppc_fp128:
+; PC64LE9:       # %bb.0: # %entry
+; PC64LE9-NEXT:    mflr 0
+; PC64LE9-NEXT:    std 0, 16(1)
+; PC64LE9-NEXT:    stdu 1, -32(1)
+; PC64LE9-NEXT:    bl floorl
+; PC64LE9-NEXT:    nop
+; PC64LE9-NEXT:    addi 1, 1, 32
+; PC64LE9-NEXT:    ld 0, 16(1)
+; PC64LE9-NEXT:    mtlr 0
+; PC64LE9-NEXT:    blr
+;
+; PC64-LABEL: test_floor_ppc_fp128:
+; PC64:       # %bb.0: # %entry
+; PC64-NEXT:    mflr 0
+; PC64-NEXT:    std 0, 16(1)
+; PC64-NEXT:    stdu 1, -112(1)
+; PC64-NEXT:    bl floorl
+; PC64-NEXT:    nop
+; PC64-NEXT:    addi 1, 1, 112
+; PC64-NEXT:    ld 0, 16(1)
+; PC64-NEXT:    mtlr 0
+; PC64-NEXT:    blr
+entry:
+  %floor = call ppc_fp128 @llvm.experimental.constrained.floor.ppcf128(
+                    ppc_fp128 %first,
+                    metadata !"fpexcept.strict")
+  ret ppc_fp128 %floor
+}
+
+define ppc_fp128 @test_round_ppc_fp128(ppc_fp128 %first) nounwind {
+; PC64LE-LABEL: test_round_ppc_fp128:
+; PC64LE:       # %bb.0: # %entry
+; PC64LE-NEXT:    mflr 0
+; PC64LE-NEXT:    std 0, 16(1)
+; PC64LE-NEXT:    stdu 1, -32(1)
+; PC64LE-NEXT:    bl roundl
+; PC64LE-NEXT:    nop
+; PC64LE-NEXT:    addi 1, 1, 32
+; PC64LE-NEXT:    ld 0, 16(1)
+; PC64LE-NEXT:    mtlr 0
+; PC64LE-NEXT:    blr
+;
+; PC64LE9-LABEL: test_round_ppc_fp128:
+; PC64LE9:       # %bb.0: # %entry
+; PC64LE9-NEXT:    mflr 0
+; PC64LE9-NEXT:    std 0, 16(1)
+; PC64LE9-NEXT:    stdu 1, -32(1)
+; PC64LE9-NEXT:    bl roundl
+; PC64LE9-NEXT:    nop
+; PC64LE9-NEXT:    addi 1, 1, 32
+; PC64LE9-NEXT:    ld 0, 16(1)
+; PC64LE9-NEXT:    mtlr 0
+; PC64LE9-NEXT:    blr
+;
+; PC64-LABEL: test_round_ppc_fp128:
+; PC64:       # %bb.0: # %entry
+; PC64-NEXT:    mflr 0
+; PC64-NEXT:    std 0, 16(1)
+; PC64-NEXT:    stdu 1, -112(1)
+; PC64-NEXT:    bl roundl
+; PC64-NEXT:    nop
+; PC64-NEXT:    addi 1, 1, 112
+; PC64-NEXT:    ld 0, 16(1)
+; PC64-NEXT:    mtlr 0
+; PC64-NEXT:    blr
+entry:
+  %round = call ppc_fp128 @llvm.experimental.constrained.round.ppcf128(
+                    ppc_fp128 %first,
+                    metadata !"fpexcept.strict")
+  ret ppc_fp128 %round
+}
+
+define ppc_fp128 @test_trunc_ppc_fp128(ppc_fp128 %first) nounwind {
+; PC64LE-LABEL: test_trunc_ppc_fp128:
+; PC64LE:       # %bb.0: # %entry
+; PC64LE-NEXT:    mflr 0
+; PC64LE-NEXT:    std 0, 16(1)
+; PC64LE-NEXT:    stdu 1, -32(1)
+; PC64LE-NEXT:    bl truncl
+; PC64LE-NEXT:    nop
+; PC64LE-NEXT:    addi 1, 1, 32
+; PC64LE-NEXT:    ld 0, 16(1)
+; PC64LE-NEXT:    mtlr 0
+; PC64LE-NEXT:    blr
+;
+; PC64LE9-LABEL: test_trunc_ppc_fp128:
+; PC64LE9:       # %bb.0: # %entry
+; PC64LE9-NEXT:    mflr 0
+; PC64LE9-NEXT:    std 0, 16(1)
+; PC64LE9-NEXT:    stdu 1, -32(1)
+; PC64LE9-NEXT:    bl truncl
+; PC64LE9-NEXT:    nop
+; PC64LE9-NEXT:    addi 1, 1, 32
+; PC64LE9-NEXT:    ld 0, 16(1)
+; PC64LE9-NEXT:    mtlr 0
+; PC64LE9-NEXT:    blr
+;
+; PC64-LABEL: test_trunc_ppc_fp128:
+; PC64:       # %bb.0: # %entry
+; PC64-NEXT:    mflr 0
+; PC64-NEXT:    std 0, 16(1)
+; PC64-NEXT:    stdu 1, -112(1)
+; PC64-NEXT:    bl truncl
+; PC64-NEXT:    nop
+; PC64-NEXT:    addi 1, 1, 112
+; PC64-NEXT:    ld 0, 16(1)
+; PC64-NEXT:    mtlr 0
+; PC64-NEXT:    blr
+entry:
+  %trunc = call ppc_fp128 @llvm.experimental.constrained.trunc.ppcf128(
+                    ppc_fp128 %first,
+                    metadata !"fpexcept.strict")
+  ret ppc_fp128 %trunc
+}
+
+define float @test_fptrunc_ppc_fp128_f32(ppc_fp128 %first) nounwind {
+; PC64LE-LABEL: test_fptrunc_ppc_fp128_f32:
+; PC64LE:       # %bb.0: # %entry
+; PC64LE-NEXT:    frsp 1, 1
+; PC64LE-NEXT:    blr
+;
+; PC64LE9-LABEL: test_fptrunc_ppc_fp128_f32:
+; PC64LE9:       # %bb.0: # %entry
+; PC64LE9-NEXT:    frsp 1, 1
+; PC64LE9-NEXT:    blr
+;
+; PC64-LABEL: test_fptrunc_ppc_fp128_f32:
+; PC64:       # %bb.0: # %entry
+; PC64-NEXT:    frsp 1, 1
+; PC64-NEXT:    blr
+entry:
+  %fptrunc = call float @llvm.experimental.constrained.fptrunc.ppcf128.f32(
+                    ppc_fp128 %first,
+                    metadata !"round.dynamic",
+                    metadata !"fpexcept.strict")
+  ret float %fptrunc
+}
+
+define double @test_fptrunc_ppc_fp128_f64(ppc_fp128 %first) nounwind {
+; PC64LE-LABEL: test_fptrunc_ppc_fp128_f64:
+; PC64LE:       # %bb.0: # %entry
+; PC64LE-NEXT:    blr
+;
+; PC64LE9-LABEL: test_fptrunc_ppc_fp128_f64:
+; PC64LE9:       # %bb.0: # %entry
+; PC64LE9-NEXT:    blr
+;
+; PC64-LABEL: test_fptrunc_ppc_fp128_f64:
+; PC64:       # %bb.0: # %entry
+; PC64-NEXT:    blr
+entry:
+  %fptrunc = call double @llvm.experimental.constrained.fptrunc.ppcf128.f64(
+                    ppc_fp128 %first,
+                    metadata !"round.dynamic",
+                    metadata !"fpexcept.strict")
+  ret double %fptrunc
+}
+
+define ppc_fp128 @test_fpext_ppc_fp128_f32(float %first) nounwind {
+; PC64LE-LABEL: test_fpext_ppc_fp128_f32:
+; PC64LE:       # %bb.0: # %entry
+; PC64LE-NEXT:    xxlxor 2, 2, 2
+; PC64LE-NEXT:    blr
+;
+; PC64LE9-LABEL: test_fpext_ppc_fp128_f32:
+; PC64LE9:       # %bb.0: # %entry
+; PC64LE9-NEXT:    xxlxor 2, 2, 2
+; PC64LE9-NEXT:    blr
+;
+; PC64-LABEL: test_fpext_ppc_fp128_f32:
+; PC64:       # %bb.0: # %entry
+; PC64-NEXT:    addis 3, 2, .LCPI26_0@toc@ha
+; PC64-NEXT:    lfs 2, .LCPI26_0@toc@l(3)
+; PC64-NEXT:    blr
+entry:
+  %fpext = call ppc_fp128 @llvm.experimental.constrained.fpext.f32.ppcf128(
+                    float %first,
+                    metadata !"fpexcept.strict")
+  ret ppc_fp128 %fpext
+}
+
+define ppc_fp128 @test_fpext_ppc_fp128_f64(double %first) nounwind {
+; PC64LE-LABEL: test_fpext_ppc_fp128_f64:
+; PC64LE:       # %bb.0: # %entry
+; PC64LE-NEXT:    xxlxor 2, 2, 2
+; PC64LE-NEXT:    blr
+;
+; PC64LE9-LABEL: test_fpext_ppc_fp128_f64:
+; PC64LE9:       # %bb.0: # %entry
+; PC64LE9-NEXT:    xxlxor 2, 2, 2
+; PC64LE9-NEXT:    blr
+;
+; PC64-LABEL: test_fpext_ppc_fp128_f64:
+; PC64:       # %bb.0: # %entry
+; PC64-NEXT:    addis 3, 2, .LCPI27_0@toc@ha
+; PC64-NEXT:    lfs 2, .LCPI27_0@toc@l(3)
+; PC64-NEXT:    blr
+entry:
+  %fpext = call ppc_fp128 @llvm.experimental.constrained.fpext.f64.ppcf128(
+                    double %first,
+                    metadata !"fpexcept.strict")
+  ret ppc_fp128 %fpext
+}
+
+define i64 @test_fptosi_ppc_i64_ppc_fp128(ppc_fp128 %first) nounwind {
+; PC64LE-LABEL: test_fptosi_ppc_i64_ppc_fp128:
+; PC64LE:       # %bb.0: # %entry
+; PC64LE-NEXT:    mflr 0
+; PC64LE-NEXT:    std 0, 16(1)
+; PC64LE-NEXT:    stdu 1, -32(1)
+; PC64LE-NEXT:    bl __fixtfdi
+; PC64LE-NEXT:    nop
+; PC64LE-NEXT:    addi 1, 1, 32
+; PC64LE-NEXT:    ld 0, 16(1)
+; PC64LE-NEXT:    mtlr 0
+; PC64LE-NEXT:    blr
+;
+; PC64LE9-LABEL: test_fptosi_ppc_i64_ppc_fp128:
+; PC64LE9:       # %bb.0: # %entry
+; PC64LE9-NEXT:    mflr 0
+; PC64LE9-NEXT:    std 0, 16(1)
+; PC64LE9-NEXT:    stdu 1, -32(1)
+; PC64LE9-NEXT:    bl __fixtfdi
+; PC64LE9-NEXT:    nop
+; PC64LE9-NEXT:    addi 1, 1, 32
+; PC64LE9-NEXT:    ld 0, 16(1)
+; PC64LE9-NEXT:    mtlr 0
+; PC64LE9-NEXT:    blr
+;
+; PC64-LABEL: test_fptosi_ppc_i64_ppc_fp128:
+; PC64:       # %bb.0: # %entry
+; PC64-NEXT:    mflr 0
+; PC64-NEXT:    std 0, 16(1)
+; PC64-NEXT:    stdu 1, -112(1)
+; PC64-NEXT:    bl __fixtfdi
+; PC64-NEXT:    nop
+; PC64-NEXT:    addi 1, 1, 112
+; PC64-NEXT:    ld 0, 16(1)
+; PC64-NEXT:    mtlr 0
+; PC64-NEXT:    blr
+entry:
+  %fpext = call i64 @llvm.experimental.constrained.fptosi.i64.ppcf128(
+                    ppc_fp128 %first,
+                    metadata !"fpexcept.strict")
+  ret i64 %fpext
+}
+
+define i32 @test_fptosi_ppc_i32_ppc_fp128(ppc_fp128 %first) nounwind {
+; PC64LE-LABEL: test_fptosi_ppc_i32_ppc_fp128:
+; PC64LE:       # %bb.0: # %entry
+; PC64LE-NEXT:    mflr 0
+; PC64LE-NEXT:    std 0, 16(1)
+; PC64LE-NEXT:    stdu 1, -32(1)
+; PC64LE-NEXT:    bl __gcc_qtou
+; PC64LE-NEXT:    nop
+; PC64LE-NEXT:    addi 1, 1, 32
+; PC64LE-NEXT:    ld 0, 16(1)
+; PC64LE-NEXT:    mtlr 0
+; PC64LE-NEXT:    blr
+;
+; PC64LE9-LABEL: test_fptosi_ppc_i32_ppc_fp128:
+; PC64LE9:       # %bb.0: # %entry
+; PC64LE9-NEXT:    mflr 0
+; PC64LE9-NEXT:    std 0, 16(1)
+; PC64LE9-NEXT:    stdu 1, -32(1)
+; PC64LE9-NEXT:    bl __gcc_qtou
+; PC64LE9-NEXT:    nop
+; PC64LE9-NEXT:    addi 1, 1, 32
+; PC64LE9-NEXT:    ld 0, 16(1)
+; PC64LE9-NEXT:    mtlr 0
+; PC64LE9-NEXT:    blr
+;
+; PC64-LABEL: test_fptosi_ppc_i32_ppc_fp128:
+; PC64:       # %bb.0: # %entry
+; PC64-NEXT:    mflr 0
+; PC64-NEXT:    std 0, 16(1)
+; PC64-NEXT:    stdu 1, -112(1)
+; PC64-NEXT:    bl __gcc_qtou
+; PC64-NEXT:    nop
+; PC64-NEXT:    addi 1, 1, 112
+; PC64-NEXT:    ld 0, 16(1)
+; PC64-NEXT:    mtlr 0
+; PC64-NEXT:    blr
+entry:
+  %fpext = call i32 @llvm.experimental.constrained.fptosi.i32.ppcf128(
+                    ppc_fp128  %first,
+                    metadata !"fpexcept.strict")
+  ret i32 %fpext
+}
+
+define i64 @test_fptoui_ppc_i64_ppc_fp128(ppc_fp128 %first) nounwind {
+; PC64LE-LABEL: test_fptoui_ppc_i64_ppc_fp128:
+; PC64LE:       # %bb.0: # %entry
+; PC64LE-NEXT:    mflr 0
+; PC64LE-NEXT:    std 0, 16(1)
+; PC64LE-NEXT:    stdu 1, -32(1)
+; PC64LE-NEXT:    bl __fixunstfdi
+; PC64LE-NEXT:    nop
+; PC64LE-NEXT:    addi 1, 1, 32
+; PC64LE-NEXT:    ld 0, 16(1)
+; PC64LE-NEXT:    mtlr 0
+; PC64LE-NEXT:    blr
+;
+; PC64LE9-LABEL: test_fptoui_ppc_i64_ppc_fp128:
+; PC64LE9:       # %bb.0: # %entry
+; PC64LE9-NEXT:    mflr 0
+; PC64LE9-NEXT:    std 0, 16(1)
+; PC64LE9-NEXT:    stdu 1, -32(1)
+; PC64LE9-NEXT:    bl __fixunstfdi
+; PC64LE9-NEXT:    nop
+; PC64LE9-NEXT:    addi 1, 1, 32
+; PC64LE9-NEXT:    ld 0, 16(1)
+; PC64LE9-NEXT:    mtlr 0
+; PC64LE9-NEXT:    blr
+;
+; PC64-LABEL: test_fptoui_ppc_i64_ppc_fp128:
+; PC64:       # %bb.0: # %entry
+; PC64-NEXT:    mflr 0
+; PC64-NEXT:    std 0, 16(1)
+; PC64-NEXT:    stdu 1, -112(1)
+; PC64-NEXT:    bl __fixunstfdi
+; PC64-NEXT:    nop
+; PC64-NEXT:    addi 1, 1, 112
+; PC64-NEXT:    ld 0, 16(1)
+; PC64-NEXT:    mtlr 0
+; PC64-NEXT:    blr
+entry:
+  %fpext = call i64 @llvm.experimental.constrained.fptoui.i64.ppcf128(
+                    ppc_fp128   %first,
+                    metadata !"fpexcept.strict")
+  ret i64 %fpext
+}
+
+define i32 @test_fptoui_ppc_i32_ppc_fp128(ppc_fp128 %first) nounwind {
+; PC64LE-LABEL: test_fptoui_ppc_i32_ppc_fp128:
+; PC64LE:       # %bb.0: # %entry
+; PC64LE-NEXT:    mflr 0
+; PC64LE-NEXT:    std 0, 16(1)
+; PC64LE-NEXT:    stdu 1, -32(1)
+; PC64LE-NEXT:    bl __fixunstfsi
+; PC64LE-NEXT:    nop
+; PC64LE-NEXT:    addi 1, 1, 32
+; PC64LE-NEXT:    ld 0, 16(1)
+; PC64LE-NEXT:    mtlr 0
+; PC64LE-NEXT:    blr
+;
+; PC64LE9-LABEL: test_fptoui_ppc_i32_ppc_fp128:
+; PC64LE9:       # %bb.0: # %entry
+; PC64LE9-NEXT:    mflr 0
+; PC64LE9-NEXT:    std 0, 16(1)
+; PC64LE9-NEXT:    stdu 1, -32(1)
+; PC64LE9-NEXT:    bl __fixunstfsi
+; PC64LE9-NEXT:    nop
+; PC64LE9-NEXT:    addi 1, 1, 32
+; PC64LE9-NEXT:    ld 0, 16(1)
+; PC64LE9-NEXT:    mtlr 0
+; PC64LE9-NEXT:    blr
+;
+; PC64-LABEL: test_fptoui_ppc_i32_ppc_fp128:
+; PC64:       # %bb.0: # %entry
+; PC64-NEXT:    mflr 0
+; PC64-NEXT:    std 0, 16(1)
+; PC64-NEXT:    stdu 1, -112(1)
+; PC64-NEXT:    bl __fixunstfsi
+; PC64-NEXT:    nop
+; PC64-NEXT:    addi 1, 1, 112
+; PC64-NEXT:    ld 0, 16(1)
+; PC64-NEXT:    mtlr 0
+; PC64-NEXT:    blr
+entry:
+  %fpext = call i32 @llvm.experimental.constrained.fptoui.i32.ppcf128(
+                    ppc_fp128   %first,
+                    metadata !"fpexcept.strict")
+  ret i32 %fpext
+}
+
+; Test that resultant libcalls retain order even when their non-strict FLOP form could be
+; trivially optimized into differing sequences.
+define void @test_constrained_libcall_multichain(float* %firstptr, ppc_fp128* %result) nounwind {
+; PC64LE-LABEL: test_constrained_libcall_multichain:
+; PC64LE:       # %bb.0:
+; PC64LE-NEXT:    mflr 0
+; PC64LE-NEXT:    std 29, -48(1) # 8-byte Folded Spill
+; PC64LE-NEXT:    std 30, -40(1) # 8-byte Folded Spill
+; PC64LE-NEXT:    stfd 29, -24(1) # 8-byte Folded Spill
+; PC64LE-NEXT:    stfd 30, -16(1) # 8-byte Folded Spill
+; PC64LE-NEXT:    stfd 31, -8(1) # 8-byte Folded Spill
+; PC64LE-NEXT:    std 0, 16(1)
+; PC64LE-NEXT:    stdu 1, -80(1)
+; PC64LE-NEXT:    mr 29, 3
+; PC64LE-NEXT:    xxlxor 2, 2, 2
+; PC64LE-NEXT:    li 3, 0
+; PC64LE-NEXT:    mr 30, 4
+; PC64LE-NEXT:    lfsx 31, 0, 29
+; PC64LE-NEXT:    xxlxor 4, 4, 4
+; PC64LE-NEXT:    std 3, 8(4)
+; PC64LE-NEXT:    fmr 1, 31
+; PC64LE-NEXT:    fmr 3, 31
+; PC64LE-NEXT:    stfdx 31, 0, 4
+; PC64LE-NEXT:    bl __gcc_qadd
+; PC64LE-NEXT:    nop
+; PC64LE-NEXT:    fmr 3, 1
+; PC64LE-NEXT:    fmr 4, 2
+; PC64LE-NEXT:    fmr 30, 1
+; PC64LE-NEXT:    fmr 29, 2
+; PC64LE-NEXT:    stfd 2, 24(30)
+; PC64LE-NEXT:    stfd 1, 16(30)
+; PC64LE-NEXT:    bl __gcc_qmul
+; PC64LE-NEXT:    nop
+; PC64LE-NEXT:    fmr 1, 31
+; PC64LE-NEXT:    xxlxor 2, 2, 2
+; PC64LE-NEXT:    li 5, 2
+; PC64LE-NEXT:    stfd 29, 40(30)
+; PC64LE-NEXT:    stfd 30, 32(30)
+; PC64LE-NEXT:    bl __powitf2
+; PC64LE-NEXT:    nop
+; PC64LE-NEXT:    frsp 0, 1
+; PC64LE-NEXT:    stfsx 0, 0, 29
+; PC64LE-NEXT:    stfd 1, -16(30)
+; PC64LE-NEXT:    stfd 2, -8(30)
+; PC64LE-NEXT:    addi 1, 1, 80
+; PC64LE-NEXT:    ld 0, 16(1)
+; PC64LE-NEXT:    lfd 31, -8(1) # 8-byte Folded Reload
+; PC64LE-NEXT:    lfd 30, -16(1) # 8-byte Folded Reload
+; PC64LE-NEXT:    ld 30, -40(1) # 8-byte Folded Reload
+; PC64LE-NEXT:    lfd 29, -24(1) # 8-byte Folded Reload
+; PC64LE-NEXT:    ld 29, -48(1) # 8-byte Folded Reload
+; PC64LE-NEXT:    mtlr 0
+; PC64LE-NEXT:    blr
+;
+; PC64LE9-LABEL: test_constrained_libcall_multichain:
+; PC64LE9:       # %bb.0:
+; PC64LE9-NEXT:    mflr 0
+; PC64LE9-NEXT:    std 29, -48(1) # 8-byte Folded Spill
+; PC64LE9-NEXT:    std 30, -40(1) # 8-byte Folded Spill
+; PC64LE9-NEXT:    stfd 29, -24(1) # 8-byte Folded Spill
+; PC64LE9-NEXT:    stfd 30, -16(1) # 8-byte Folded Spill
+; PC64LE9-NEXT:    stfd 31, -8(1) # 8-byte Folded Spill
+; PC64LE9-NEXT:    std 0, 16(1)
+; PC64LE9-NEXT:    stdu 1, -80(1)
+; PC64LE9-NEXT:    lfs 31, 0(3)
+; PC64LE9-NEXT:    mr 29, 3
+; PC64LE9-NEXT:    li 3, 0
+; PC64LE9-NEXT:    xxlxor 2, 2, 2
+; PC64LE9-NEXT:    xxlxor 4, 4, 4
+; PC64LE9-NEXT:    std 3, 8(4)
+; PC64LE9-NEXT:    fmr 1, 31
+; PC64LE9-NEXT:    fmr 3, 31
+; PC64LE9-NEXT:    mr 30, 4
+; PC64LE9-NEXT:    stfd 31, 0(4)
+; PC64LE9-NEXT:    bl __gcc_qadd
+; PC64LE9-NEXT:    nop
+; PC64LE9-NEXT:    fmr 3, 1
+; PC64LE9-NEXT:    fmr 4, 2
+; PC64LE9-NEXT:    fmr 30, 2
+; PC64LE9-NEXT:    fmr 29, 1
+; PC64LE9-NEXT:    stfd 2, 24(30)
+; PC64LE9-NEXT:    stfd 1, 16(30)
+; PC64LE9-NEXT:    bl __gcc_qmul
+; PC64LE9-NEXT:    nop
+; PC64LE9-NEXT:    fmr 1, 31
+; PC64LE9-NEXT:    xxlxor 2, 2, 2
+; PC64LE9-NEXT:    li 5, 2
+; PC64LE9-NEXT:    stfd 30, 40(30)
+; PC64LE9-NEXT:    stfd 29, 32(30)
+; PC64LE9-NEXT:    bl __powitf2
+; PC64LE9-NEXT:    nop
+; PC64LE9-NEXT:    frsp 0, 1
+; PC64LE9-NEXT:    stfs 0, 0(29)
+; PC64LE9-NEXT:    stfd 1, -16(30)
+; PC64LE9-NEXT:    stfd 2, -8(30)
+; PC64LE9-NEXT:    addi 1, 1, 80
+; PC64LE9-NEXT:    ld 0, 16(1)
+; PC64LE9-NEXT:    lfd 31, -8(1) # 8-byte Folded Reload
+; PC64LE9-NEXT:    lfd 30, -16(1) # 8-byte Folded Reload
+; PC64LE9-NEXT:    ld 30, -40(1) # 8-byte Folded Reload
+; PC64LE9-NEXT:    ld 29, -48(1) # 8-byte Folded Reload
+; PC64LE9-NEXT:    mtlr 0
+; PC64LE9-NEXT:    lfd 29, -24(1) # 8-byte Folded Reload
+; PC64LE9-NEXT:    blr
+;
+; PC64-LABEL: test_constrained_libcall_multichain:
+; PC64:       # %bb.0:
+; PC64-NEXT:    mflr 0
+; PC64-NEXT:    std 0, 16(1)
+; PC64-NEXT:    stdu 1, -176(1)
+; PC64-NEXT:    std 29, 120(1) # 8-byte Folded Spill
+; PC64-NEXT:    mr 29, 3
+; PC64-NEXT:    li 3, 0
+; PC64-NEXT:    stfd 31, 168(1) # 8-byte Folded Spill
+; PC64-NEXT:    stfd 30, 160(1) # 8-byte Folded Spill
+; PC64-NEXT:    std 30, 128(1) # 8-byte Folded Spill
+; PC64-NEXT:    stfd 28, 144(1) # 8-byte Folded Spill
+; PC64-NEXT:    stfd 29, 152(1) # 8-byte Folded Spill
+; PC64-NEXT:    mr 30, 4
+; PC64-NEXT:    lfs 31, 0(29)
+; PC64-NEXT:    std 3, 8(4)
+; PC64-NEXT:    addis 3, 2, .LCPI32_0@toc@ha
+; PC64-NEXT:    lfs 30, .LCPI32_0@toc@l(3)
+; PC64-NEXT:    fmr 1, 31
+; PC64-NEXT:    fmr 3, 31
+; PC64-NEXT:    fmr 2, 30
+; PC64-NEXT:    fmr 4, 30
+; PC64-NEXT:    stfd 31, 0(4)
+; PC64-NEXT:    bl __gcc_qadd
+; PC64-NEXT:    nop
+; PC64-NEXT:    fmr 3, 1
+; PC64-NEXT:    fmr 4, 2
+; PC64-NEXT:    fmr 29, 1
+; PC64-NEXT:    fmr 28, 2
+; PC64-NEXT:    stfd 2, 24(30)
+; PC64-NEXT:    stfd 1, 16(30)
+; PC64-NEXT:    bl __gcc_qmul
+; PC64-NEXT:    nop
+; PC64-NEXT:    fmr 1, 31
+; PC64-NEXT:    fmr 2, 30
+; PC64-NEXT:    li 5, 2
+; PC64-NEXT:    stfd 28, 40(30)
+; PC64-NEXT:    stfd 29, 32(30)
+; PC64-NEXT:    bl __powitf2
+; PC64-NEXT:    nop
+; PC64-NEXT:    frsp 0, 1
+; PC64-NEXT:    stfs 0, 0(29)
+; PC64-NEXT:    lfd 31, 168(1) # 8-byte Folded Reload
+; PC64-NEXT:    lfd 30, 160(1) # 8-byte Folded Reload
+; PC64-NEXT:    lfd 29, 152(1) # 8-byte Folded Reload
+; PC64-NEXT:    lfd 28, 144(1) # 8-byte Folded Reload
+; PC64-NEXT:    ld 29, 120(1) # 8-byte Folded Reload
+; PC64-NEXT:    stfd 1, -16(30)
+; PC64-NEXT:    stfd 2, -8(30)
+; PC64-NEXT:    ld 30, 128(1) # 8-byte Folded Reload
+; PC64-NEXT:    addi 1, 1, 176
+; PC64-NEXT:    ld 0, 16(1)
+; PC64-NEXT:    mtlr 0
+; PC64-NEXT:    blr
+  %load = load float, float* %firstptr
+  %first = call ppc_fp128 @llvm.experimental.constrained.fpext.f32.ppcf128(
+                    float %load,
+                    metadata !"fpexcept.strict")
+  store ppc_fp128 %first, ppc_fp128* %result
+
+  ; For unconstrained FLOPs, these next two FP instructions would necessarily
+  ; be executed in series with one another.
+  %fadd = call ppc_fp128 @llvm.experimental.constrained.fadd.ppcf128(
+                    ppc_fp128 %first,
+                    ppc_fp128 %first,
+                    metadata !"round.dynamic",
+                    metadata !"fpexcept.strict")
+  %stridx1 = getelementptr ppc_fp128, ppc_fp128* %result, i32 1
+  store ppc_fp128 %fadd, ppc_fp128* %stridx1
+  %fmul = call ppc_fp128 @llvm.experimental.constrained.fmul.ppcf128(
+                    ppc_fp128 %fadd,
+                    ppc_fp128 %fadd,
+                    metadata !"round.dynamic",
+                    metadata !"fpexcept.strict")
+  %stridx2 = getelementptr ppc_fp128, ppc_fp128* %stridx1, i32 1
+  store ppc_fp128 %fadd, ppc_fp128* %stridx2
+
+  ; For unconstrained FLOPs, these next two FP instructions could be reordered
+  ; or even executed in parallel with respect to the previous two instructions.
+  ; However, strict floating point rules would not allow this.
+  %powi = call ppc_fp128 @llvm.experimental.constrained.powi.ppcf128(
+                    ppc_fp128 %first,
+                    i32 2,
+                    metadata !"round.dynamic",
+                    metadata !"fpexcept.strict")
+  %tinypow = call float @llvm.experimental.constrained.fptrunc.ppcf128.f32(
+                    ppc_fp128 %powi,
+                    metadata !"round.dynamic",
+                    metadata !"fpexcept.strict")
+  store float %tinypow, float* %firstptr
+  %stridxn1 = getelementptr ppc_fp128, ppc_fp128* %result, i32 -1
+  store ppc_fp128 %powi, ppc_fp128* %stridxn1
+  ret void
+}
+
+declare ppc_fp128 @llvm.experimental.constrained.fadd.ppcf128(ppc_fp128, ppc_fp128, metadata, metadata)
+declare ppc_fp128 @llvm.experimental.constrained.ceil.ppcf128(ppc_fp128, metadata)
+declare ppc_fp128 @llvm.experimental.constrained.cos.ppcf128(ppc_fp128, metadata, metadata)
+declare ppc_fp128 @llvm.experimental.constrained.fdiv.ppcf128(ppc_fp128, ppc_fp128, metadata, metadata)
+declare ppc_fp128 @llvm.experimental.constrained.exp.ppcf128(ppc_fp128, metadata, metadata)
+declare ppc_fp128 @llvm.experimental.constrained.exp2.ppcf128(ppc_fp128, metadata, metadata)
+declare ppc_fp128 @llvm.experimental.constrained.floor.ppcf128(ppc_fp128, metadata)
+declare ppc_fp128 @llvm.experimental.constrained.fma.ppcf128(ppc_fp128, ppc_fp128, ppc_fp128, metadata, metadata)
+declare ppc_fp128 @llvm.experimental.constrained.fpext.f32.ppcf128(float, metadata)
+declare ppc_fp128 @llvm.experimental.constrained.fpext.f64.ppcf128(double, metadata)
+declare float @llvm.experimental.constrained.fptrunc.ppcf128.f32(ppc_fp128, metadata, metadata)
+declare double @llvm.experimental.constrained.fptrunc.ppcf128.f64(ppc_fp128, metadata, metadata)
+declare ppc_fp128 @llvm.experimental.constrained.log.ppcf128(ppc_fp128, metadata, metadata)
+declare ppc_fp128 @llvm.experimental.constrained.log10.ppcf128(ppc_fp128, metadata, metadata)
+declare ppc_fp128 @llvm.experimental.constrained.log2.ppcf128(ppc_fp128, metadata, metadata)
+declare ppc_fp128 @llvm.experimental.constrained.maxnum.ppcf128(ppc_fp128, ppc_fp128, metadata)
+declare ppc_fp128 @llvm.experimental.constrained.minnum.ppcf128(ppc_fp128, ppc_fp128, metadata)
+declare ppc_fp128 @llvm.experimental.constrained.fmul.ppcf128(ppc_fp128, ppc_fp128, metadata, metadata)
+declare ppc_fp128 @llvm.experimental.constrained.nearbyint.ppcf128(ppc_fp128, metadata, metadata)
+declare ppc_fp128 @llvm.experimental.constrained.pow.ppcf128(ppc_fp128, ppc_fp128, metadata, metadata)
+declare ppc_fp128 @llvm.experimental.constrained.powi.ppcf128(ppc_fp128, i32, metadata, metadata)
+declare ppc_fp128 @llvm.experimental.constrained.frem.ppcf128(ppc_fp128, ppc_fp128, metadata, metadata)
+declare ppc_fp128 @llvm.experimental.constrained.rint.ppcf128(ppc_fp128, metadata, metadata)
+declare ppc_fp128 @llvm.experimental.constrained.round.ppcf128(ppc_fp128, metadata)
+declare ppc_fp128 @llvm.experimental.constrained.sin.ppcf128(ppc_fp128, metadata, metadata)
+declare ppc_fp128 @llvm.experimental.constrained.sqrt.ppcf128(ppc_fp128, metadata, metadata)
+declare ppc_fp128 @llvm.experimental.constrained.fsub.ppcf128(ppc_fp128, ppc_fp128, metadata, metadata)
+declare ppc_fp128 @llvm.experimental.constrained.trunc.ppcf128(ppc_fp128, metadata)
+declare i64 @llvm.experimental.constrained.fptosi.i64.ppcf128(ppc_fp128, metadata)
+declare i32 @llvm.experimental.constrained.fptosi.i32.ppcf128(ppc_fp128, metadata)
+declare i64 @llvm.experimental.constrained.fptoui.i64.ppcf128(ppc_fp128, metadata)
+declare i32 @llvm.experimental.constrained.fptoui.i32.ppcf128(ppc_fp128, metadata)
diff --git a/llvm/test/CodeGen/PowerPC/pr17168.ll b/llvm/test/CodeGen/PowerPC/pr17168.ll
index 84c8d5656..c77424959 100644
--- a/llvm/test/CodeGen/PowerPC/pr17168.ll
+++ b/llvm/test/CodeGen/PowerPC/pr17168.ll
@@ -51,7 +51,7 @@ for.cond1816.preheader.for.inc1898_crit_edge:     ; preds = %for.cond1816.prehea
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.value(metadata, i64, metadata, metadata) #1
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!7}
diff --git a/llvm/test/CodeGen/PowerPC/pr17354.ll b/llvm/test/CodeGen/PowerPC/pr17354.ll
index 9b5f52aee..0f77e5936 100644
--- a/llvm/test/CodeGen/PowerPC/pr17354.ll
+++ b/llvm/test/CodeGen/PowerPC/pr17354.ll
@@ -36,4 +36,4 @@ entry:
   ret void
 }
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/PowerPC/pr18663-2.ll b/llvm/test/CodeGen/PowerPC/pr18663-2.ll
index 40d6036a9..7725c90a6 100644
--- a/llvm/test/CodeGen/PowerPC/pr18663-2.ll
+++ b/llvm/test/CodeGen/PowerPC/pr18663-2.ll
@@ -147,7 +147,7 @@ ehcleanup142:                                     ; preds = %lpad19, %lpad.i.i.i
   resume { i8*, i32 } undef
 }
 
-attributes #0 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { noreturn "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #2 = { inlinehint "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { noreturn "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { inlinehint "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
diff --git a/llvm/test/CodeGen/PowerPC/pr24546.ll b/llvm/test/CodeGen/PowerPC/pr24546.ll
index 172c9fd8f..28c032936 100644
--- a/llvm/test/CodeGen/PowerPC/pr24546.ll
+++ b/llvm/test/CodeGen/PowerPC/pr24546.ll
@@ -47,8 +47,8 @@ declare double @pow(double, double) #0
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.value(metadata, i64, metadata, metadata) #2
 
-attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+vsx,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+vsx,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+vsx,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+vsx,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind readnone }
 attributes #3 = { nounwind }
 
diff --git a/llvm/test/CodeGen/PowerPC/pr25080.ll b/llvm/test/CodeGen/PowerPC/pr25080.ll
new file mode 100644
index 000000000..5edaff2c5
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/pr25080.ll
@@ -0,0 +1,123 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-- -mcpu=pwr8 < %s | FileCheck %s --check-prefix=LE
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64-- -mcpu=pwr8 < %s | FileCheck %s --check-prefix=BE
+
+define <8 x i16> @pr25080(<8 x i32> %a) {
+; LE-LABEL: pr25080:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    addis 3, 2, .LCPI0_0@toc@ha
+; LE-NEXT:    xxlxor 37, 37, 37
+; LE-NEXT:    addi 3, 3, .LCPI0_0@toc@l
+; LE-NEXT:    lvx 4, 0, 3
+; LE-NEXT:    xxland 34, 34, 36
+; LE-NEXT:    xxland 35, 35, 36
+; LE-NEXT:    vcmpequw 2, 2, 5
+; LE-NEXT:    vcmpequw 3, 3, 5
+; LE-NEXT:    xxswapd 0, 34
+; LE-NEXT:    mfvsrwz 3, 34
+; LE-NEXT:    xxsldwi 1, 34, 34, 1
+; LE-NEXT:    mfvsrwz 4, 35
+; LE-NEXT:    xxsldwi 4, 34, 34, 3
+; LE-NEXT:    mtvsrd 2, 3
+; LE-NEXT:    mfvsrwz 3, 0
+; LE-NEXT:    xxswapd 0, 35
+; LE-NEXT:    mtvsrd 3, 4
+; LE-NEXT:    xxsldwi 5, 35, 35, 1
+; LE-NEXT:    mfvsrwz 4, 1
+; LE-NEXT:    xxsldwi 7, 35, 35, 3
+; LE-NEXT:    mtvsrd 1, 3
+; LE-NEXT:    xxswapd 33, 3
+; LE-NEXT:    mfvsrwz 3, 4
+; LE-NEXT:    mtvsrd 4, 4
+; LE-NEXT:    xxswapd 34, 1
+; LE-NEXT:    mfvsrwz 4, 0
+; LE-NEXT:    mtvsrd 0, 3
+; LE-NEXT:    xxswapd 35, 4
+; LE-NEXT:    mfvsrwz 3, 5
+; LE-NEXT:    mtvsrd 6, 4
+; LE-NEXT:    xxswapd 36, 0
+; LE-NEXT:    mtvsrd 1, 3
+; LE-NEXT:    mfvsrwz 3, 7
+; LE-NEXT:    xxswapd 37, 6
+; LE-NEXT:    vmrglh 2, 3, 2
+; LE-NEXT:    xxswapd 35, 2
+; LE-NEXT:    mtvsrd 2, 3
+; LE-NEXT:    xxswapd 32, 1
+; LE-NEXT:    addis 3, 2, .LCPI0_1@toc@ha
+; LE-NEXT:    addi 3, 3, .LCPI0_1@toc@l
+; LE-NEXT:    xxswapd 38, 2
+; LE-NEXT:    vmrglh 3, 4, 3
+; LE-NEXT:    vmrglh 4, 0, 5
+; LE-NEXT:    vmrglh 5, 6, 1
+; LE-NEXT:    vmrglw 2, 3, 2
+; LE-NEXT:    vmrglw 3, 5, 4
+; LE-NEXT:    vspltish 4, 15
+; LE-NEXT:    xxmrgld 34, 35, 34
+; LE-NEXT:    lvx 3, 0, 3
+; LE-NEXT:    xxlor 34, 34, 35
+; LE-NEXT:    vslh 2, 2, 4
+; LE-NEXT:    vsrah 2, 2, 4
+; LE-NEXT:    blr
+;
+; BE-LABEL: pr25080:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis 3, 2, .LCPI0_0@toc@ha
+; BE-NEXT:    xxlxor 36, 36, 36
+; BE-NEXT:    addi 3, 3, .LCPI0_0@toc@l
+; BE-NEXT:    lxvw4x 0, 0, 3
+; BE-NEXT:    xxland 35, 35, 0
+; BE-NEXT:    xxland 34, 34, 0
+; BE-NEXT:    vcmpequw 3, 3, 4
+; BE-NEXT:    vcmpequw 2, 2, 4
+; BE-NEXT:    xxswapd 0, 35
+; BE-NEXT:    mfvsrwz 3, 35
+; BE-NEXT:    xxsldwi 1, 35, 35, 1
+; BE-NEXT:    sldi 3, 3, 48
+; BE-NEXT:    mfvsrwz 4, 0
+; BE-NEXT:    xxsldwi 0, 35, 35, 3
+; BE-NEXT:    mtvsrd 36, 3
+; BE-NEXT:    mfvsrwz 3, 1
+; BE-NEXT:    sldi 4, 4, 48
+; BE-NEXT:    xxswapd 1, 34
+; BE-NEXT:    mtvsrd 35, 4
+; BE-NEXT:    mfvsrwz 4, 34
+; BE-NEXT:    sldi 3, 3, 48
+; BE-NEXT:    mtvsrd 37, 3
+; BE-NEXT:    mfvsrwz 3, 0
+; BE-NEXT:    sldi 4, 4, 48
+; BE-NEXT:    xxsldwi 0, 34, 34, 1
+; BE-NEXT:    vmrghh 3, 5, 3
+; BE-NEXT:    mtvsrd 37, 4
+; BE-NEXT:    sldi 3, 3, 48
+; BE-NEXT:    mfvsrwz 4, 1
+; BE-NEXT:    xxsldwi 1, 34, 34, 3
+; BE-NEXT:    mtvsrd 34, 3
+; BE-NEXT:    mfvsrwz 3, 0
+; BE-NEXT:    sldi 4, 4, 48
+; BE-NEXT:    mtvsrd 32, 4
+; BE-NEXT:    mfvsrwz 4, 1
+; BE-NEXT:    sldi 3, 3, 48
+; BE-NEXT:    mtvsrd 33, 3
+; BE-NEXT:    sldi 3, 4, 48
+; BE-NEXT:    vmrghh 2, 2, 4
+; BE-NEXT:    mtvsrd 36, 3
+; BE-NEXT:    addis 3, 2, .LCPI0_1@toc@ha
+; BE-NEXT:    vmrghh 0, 1, 0
+; BE-NEXT:    addi 3, 3, .LCPI0_1@toc@l
+; BE-NEXT:    vmrghh 4, 4, 5
+; BE-NEXT:    lxvw4x 0, 0, 3
+; BE-NEXT:    vmrghw 2, 2, 3
+; BE-NEXT:    vmrghw 3, 4, 0
+; BE-NEXT:    xxmrghd 34, 35, 34
+; BE-NEXT:    vspltish 3, 15
+; BE-NEXT:    xxlor 34, 34, 0
+; BE-NEXT:    vslh 2, 2, 3
+; BE-NEXT:    vsrah 2, 2, 3
+; BE-NEXT:    blr
+entry:
+  %0 = trunc <8 x i32> %a to <8 x i23>
+  %1 = icmp eq <8 x i23> %0, zeroinitializer
+  %2 = or <8 x i1> %1, <i1 true, i1 true, i1 true, i1 true, i1 false, i1 false, i1 false, i1 false>
+  %3 = sext <8 x i1> %2 to <8 x i16>
+  ret <8 x i16> %3
+}
diff --git a/llvm/test/CodeGen/PowerPC/pr26180.ll b/llvm/test/CodeGen/PowerPC/pr26180.ll
index e8c6c6006..b8b01c047 100644
--- a/llvm/test/CodeGen/PowerPC/pr26180.ll
+++ b/llvm/test/CodeGen/PowerPC/pr26180.ll
@@ -11,7 +11,7 @@ define i32 @bad(double %x) {
 ; CHECK: stfd [[REG0]], [[OFF:.*]](1)
 ; CHECK: lwz {{[0-9]*}}, [[OFF]](1)
 ; GENERIC: xscvdpuxws [[REG0:[0-9]+]], 1
-; GENERIC: mfvsrwz  {{[0-9]*}}, [[REG0]]
+; GENERIC: mffprwz  {{[0-9]*}}, [[REG0]]
 }
 
 define i32 @bad1(float %x) {
@@ -23,5 +23,5 @@ entry:
 ; CHECK: stfd [[REG0]], [[OFF:.*]](1)
 ; CHECK: lwa {{[0-9]*}}, [[OFF]](1)
 ; GENERIC: xscvdpsxws [[REG0:[0-9]+]], 1
-; GENERIC: mfvsrwz  {{[0-9]*}}, [[REG0]]
+; GENERIC: mffprwz  {{[0-9]*}}, [[REG0]]
 }
diff --git a/llvm/test/CodeGen/PowerPC/pr27350.ll b/llvm/test/CodeGen/PowerPC/pr27350.ll
index 7dbd51107..982023a1f 100644
--- a/llvm/test/CodeGen/PowerPC/pr27350.ll
+++ b/llvm/test/CodeGen/PowerPC/pr27350.ll
@@ -18,7 +18,7 @@ entry:
 declare fastcc void @bar([2 x i64], [2 x i64]) unnamed_addr #1 align 2
 
 attributes #0 = { argmemonly nounwind }
-attributes #1 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+vsx,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+vsx,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind }
 
 !llvm.ident = !{!0}
diff --git a/llvm/test/CodeGen/PowerPC/pr28130.ll b/llvm/test/CodeGen/PowerPC/pr28130.ll
index 73eb1683b..cb703dfda 100644
--- a/llvm/test/CodeGen/PowerPC/pr28130.ll
+++ b/llvm/test/CodeGen/PowerPC/pr28130.ll
@@ -67,4 +67,4 @@ bb:
   ret void
 }
 
-attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pwr8" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+vsx,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pwr8" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+vsx,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/PowerPC/pr31144.ll b/llvm/test/CodeGen/PowerPC/pr31144.ll
index dc48d7cdf..b968b4b80 100644
--- a/llvm/test/CodeGen/PowerPC/pr31144.ll
+++ b/llvm/test/CodeGen/PowerPC/pr31144.ll
@@ -10,7 +10,7 @@ entry:
   ret void
 
 ; CHECK-LABEL: @foo1
-; CHECK:     mtvsrwz
+; CHECK:     mtfprwz
 }
 
 define void @foo2(i16* %p) {
@@ -21,6 +21,6 @@ entry:
   ret void
 
 ; CHECK-LABEL: @foo2
-; CHECK:       mtvsrwz
+; CHECK:       mtfprwz
 }
 
diff --git a/llvm/test/CodeGen/PowerPC/pr32140.ll b/llvm/test/CodeGen/PowerPC/pr32140.ll
index ddcf27232..1a0054b0f 100644
--- a/llvm/test/CodeGen/PowerPC/pr32140.ll
+++ b/llvm/test/CodeGen/PowerPC/pr32140.ll
@@ -2,10 +2,10 @@
 ; RUN: llc -mtriple=powerpc64le-linux-gnu -mcpu=pwr8 < %s | FileCheck %s --check-prefix CHECK-LE
 ; RUN: llc -mtriple=powerpc64-linux-gnu -mcpu=pwr8 < %s | FileCheck %s --check-prefix CHECK-BE
 
-@as = common local_unnamed_addr global i16 0, align 2
-@bs = common local_unnamed_addr global i16 0, align 2
-@ai = common local_unnamed_addr global i32 0, align 4
-@bi = common local_unnamed_addr global i32 0, align 4
+@as = local_unnamed_addr global i16 0, align 2
+@bs = local_unnamed_addr global i16 0, align 2
+@ai = local_unnamed_addr global i32 0, align 4
+@bi = local_unnamed_addr global i32 0, align 4
 
 define void @bswapStorei64Toi32() {
 ; CHECK-LABEL: bswapStorei64Toi32:
diff --git a/llvm/test/CodeGen/PowerPC/pr36068.ll b/llvm/test/CodeGen/PowerPC/pr36068.ll
index aac659bfb..ee56d020e 100644
--- a/llvm/test/CodeGen/PowerPC/pr36068.ll
+++ b/llvm/test/CodeGen/PowerPC/pr36068.ll
@@ -1,7 +1,7 @@
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-unknown < %s  | \
 ; RUN:   FileCheck %s
 
-@glob = common local_unnamed_addr global <4 x float> zeroinitializer, align 4
+@glob = local_unnamed_addr global <4 x float> zeroinitializer, align 4
 
 ; Function Attrs: norecurse nounwind
 define void @test(float %a, <4 x float>* nocapture readonly %b) {
diff --git a/llvm/test/CodeGen/PowerPC/pr36292.ll b/llvm/test/CodeGen/PowerPC/pr36292.ll
index 00d99a703..a859121bb 100644
--- a/llvm/test/CodeGen/PowerPC/pr36292.ll
+++ b/llvm/test/CodeGen/PowerPC/pr36292.ll
@@ -4,14 +4,10 @@
 $test = comdat any
 
 ; No CTR loop due to frem (since it is always a call).
-define void @test() #0 comdat {
+define void @test() nounwind comdat {
 ; CHECK-LABEL: test:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    mflr 0
-; CHECK-NEXT:    .cfi_def_cfa_offset 64
-; CHECK-NEXT:    .cfi_offset lr, 16
-; CHECK-NEXT:    .cfi_offset r29, -24
-; CHECK-NEXT:    .cfi_offset r30, -16
 ; CHECK-NEXT:    std 29, -24(1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std 30, -16(1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std 0, 16(1)
@@ -19,17 +15,17 @@ define void @test() #0 comdat {
 ; CHECK-NEXT:    ld 29, 0(3)
 ; CHECK-NEXT:    ld 30, 32(1)
 ; CHECK-NEXT:    cmpld 30, 29
-; CHECK-NEXT:    bge- 0, .LBB0_2
-; CHECK-NEXT:    .p2align 5
+; CHECK-NEXT:    bge 0, .LBB0_2
 ; CHECK-NEXT:  .LBB0_1: # %bounds.ok
-; CHECK:         lfsx 2, 0, 3
+; CHECK-NEXT:    #
+; CHECK-NEXT:    lfsx 2, 0, 3
 ; CHECK-NEXT:    xxlxor 1, 1, 1
 ; CHECK-NEXT:    bl fmodf
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    addi 30, 30, 1
 ; CHECK-NEXT:    stfsx 1, 0, 3
 ; CHECK-NEXT:    cmpld 30, 29
-; CHECK-NEXT:    blt+ 0, .LBB0_1
+; CHECK-NEXT:    blt 0, .LBB0_1
 ; CHECK-NEXT:  .LBB0_2: # %bounds.fail
 ; CHECK-NEXT:    std 30, 32(1)
   %pos = alloca i64, align 8
diff --git a/llvm/test/CodeGen/PowerPC/pr38087.ll b/llvm/test/CodeGen/PowerPC/pr38087.ll
index 2736ffa72..2ef4c689b 100644
--- a/llvm/test/CodeGen/PowerPC/pr38087.ll
+++ b/llvm/test/CodeGen/PowerPC/pr38087.ll
@@ -8,16 +8,16 @@ declare <4 x float> @llvm.fmuladd.v4f32(<4 x float>, <4 x float>, <4 x float>) #
 ; Function Attrs: nounwind readnone speculatable
 declare { i32, i1 } @llvm.usub.with.overflow.i32(i32, i32) #0
 
-define void @draw_llvm_vs_variant0() {
+define void @draw_llvm_vs_variant0(<4 x float> %x) {
 ; CHECK-LABEL: draw_llvm_vs_variant0:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    lfd f0, 0(r3)
-; CHECK-NEXT:    xxpermdi v2, f0, f0, 2
-; CHECK-NEXT:    vmrglh v2, v2, v2
-; CHECK-NEXT:    vextsh2w v2, v2
-; CHECK-NEXT:    xvcvsxwsp vs0, v2
+; CHECK-NEXT:    xxpermdi v3, f0, f0, 2
+; CHECK-NEXT:    vmrglh v3, v3, v3
+; CHECK-NEXT:    vextsh2w v3, v3
+; CHECK-NEXT:    xvcvsxwsp vs0, v3
 ; CHECK-NEXT:    xxspltw vs0, vs0, 2
-; CHECK-NEXT:    xvmaddasp vs0, vs0, vs0
+; CHECK-NEXT:    xvmaddasp vs0, v2, v2
 ; CHECK-NEXT:    stxvx vs0, 0, r3
 ; CHECK-NEXT:    blr
 entry:
@@ -49,7 +49,7 @@ entry:
   %24 = and <4 x i32> %23, %22
   %25 = bitcast <4 x i32> %24 to <4 x float>
   %26 = shufflevector <4 x float> %25, <4 x float> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
-  %27 = call <4 x float> @llvm.fmuladd.v4f32(<4 x float> undef, <4 x float> undef, <4 x float> %26)
+  %27 = call <4 x float> @llvm.fmuladd.v4f32(<4 x float> %x, <4 x float> %x, <4 x float> %26)
   store <4 x float> %27, <4 x float>* undef
   ret void
 }
diff --git a/llvm/test/CodeGen/PowerPC/pr39815.ll b/llvm/test/CodeGen/PowerPC/pr39815.ll
index 062e05516..a0cd0644f 100644
--- a/llvm/test/CodeGen/PowerPC/pr39815.ll
+++ b/llvm/test/CodeGen/PowerPC/pr39815.ll
@@ -1,8 +1,8 @@
 ; RUN: llc -mcpu=pwr9 -mtriple=powerpc64le-unknown-linux-gnu < %s \
 ; RUN:   -verify-machineinstrs | FileCheck %s
 
-@b = common dso_local local_unnamed_addr global i64* null, align 8
-@a = common dso_local local_unnamed_addr global i8 0, align 1
+@b = dso_local local_unnamed_addr global i64* null, align 8
+@a = dso_local local_unnamed_addr global i8 0, align 1
 
 define void @testADDEPromoteResult() {
 entry:
@@ -20,11 +20,10 @@ entry:
 ; CHECK:      # %bb.0:
 ; CHECK-DAG:   addis [[REG1:[0-9]+]], [[REG2:[0-9]+]], [[VAR1:[a-z0-9A-Z_.]+]]@toc@ha
 ; CHECK-DAG:   ld [[REG3:[0-9]+]], [[VAR1]]@toc@l([[REG1]])
-; CHECK-DAG:   lwz [[REG4:[0-9]+]], 0([[REG3]])
-; CHECK-DAG:   addic [[REG5:[0-9]+]], [[REG3]], -1
-; CHECK-DAG:   addze [[REG7:[0-9]+]], [[REG4]]
-; CHECK-DAG:   addis [[REG8:[0-9]+]], [[REG2]], [[VAR2:[a-z0-9A-Z_.]+]]@toc@ha
+; CHECK-DAG:   lbz [[REG4:[0-9]+]], 0([[REG3]])
+; CHECK-DAG:   addi [[REG7:[0-9]+]], [[REG4]]
 ; CHECK-DAG:   andi. [[REG9:[0-9]+]], [[REG7]], 5
+; CHECK-DAG:   addis [[REG8:[0-9]+]], [[REG2]], [[VAR2:[a-z0-9A-Z_.]+]]@toc@ha
 ; CHECK-DAG:   stb [[REG9]], [[VAR2]]@toc@l([[REG8]])
 ; CHECK:       blr
 }
diff --git a/llvm/test/CodeGen/PowerPC/pr41088.ll b/llvm/test/CodeGen/PowerPC/pr41088.ll
new file mode 100644
index 000000000..26094358c
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/pr41088.ll
@@ -0,0 +1,141 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mcpu=pwr8 -ppc-asm-full-reg-names \
+; RUN:   -mtriple=powerpc64le-unknown-unknown -verify-machineinstrs < %s | \
+; RUN:   FileCheck %s
+
+%0 = type { [0 x i64], %1, [0 x i64], { i64, i8* }, [0 x i64] }
+%1 = type { [0 x i64], %2, [0 x i64], i64*, [0 x i64] }
+%2 = type { [0 x i64], %3, [0 x i64], %4, [0 x i8], i8, [7 x i8] }
+%3 = type { [0 x i64], { i64*, i64* }, [0 x i64], i64*, [0 x i8], i8, [7 x i8] }
+%4 = type { [0 x i64], { i64*, i64* }, [0 x i64], %5, [0 x i64] }
+%5 = type { [0 x i64], { i64*, i64* }, [0 x i64], i64*, [0 x i64] }
+%6 = type { [0 x i64], i64, [2 x i64] }
+%7 = type { [0 x i64], { i64*, i64* }, [0 x i64], %8, [0 x i64] }
+%8 = type { [0 x i64], %9*, [0 x i32], { i32, i32 }, [0 x i8], i8, [7 x i8] }
+%9 = type { [0 x i64], i64, [0 x i64], [0 x %10], [0 x i8], %11 }
+%10 = type { [0 x i8], i8, [31 x i8] }
+%11 = type {}
+%12 = type { [0 x i64], %13, [0 x i32], i32, [0 x i32], i32, [0 x i32] }
+%13 = type { [0 x i8], i8, [23 x i8] }
+%14 = type { [0 x i64], i64, [0 x i64], %15, [0 x i32], i32, [0 x i8], i8, [0 x i8], { i8, i8 }, [0 x i8], { i8, i8 }, [0 x i8], { i8, i8 }, [0 x i8], { i8, i8 }, [0 x i8], { i8, i8 }, [0 x i8], { i8, i8 }, [0 x i8], { i8, i8 }, [0 x i8], { i8, i8 }, [0 x i8], { i8, i8 }, [0 x i8], { i8, i8 }, [7 x i8] }
+%15 = type { [0 x i64], { i64*, i64 }, [0 x i64], i64, [0 x i64] }
+%16 = type { [0 x i64], %17, [0 x i64], %18, [0 x i64], %19, [0 x i64], i64, [0 x i8], { i8, i8 }, [6 x i8] }
+%17 = type { [0 x i32], i32, [27 x i32] }
+%18 = type { [0 x i64], i64, [6 x i64] }
+%19 = type { [0 x i8], i8, [103 x i8] }
+%20 = type { [0 x i64], { i64*, i64* }*, [0 x i64], %7**, [0 x i64], i64**, [0 x i64] }
+%21 = type { [0 x i64], i64, [0 x i64], void (i32, %21*)*, [0 x i64], [2 x i64], [0 x i64] }
+%22 = type { [0 x i8] }
+
+@var = external dso_local unnamed_addr constant <{ i8*, [8 x i8], i8*, [16 x i8] }>, align 8
+
+declare dso_local fastcc { i64*, i8* } @test2(%0**) unnamed_addr
+
+define void @test(%6* %arg, %7* %arg1, %12* %arg2) unnamed_addr personality i32 (i32, i32, i64, %21*, %22*)* @personality {
+; CHECK-LABEL: test:
+; CHECK:         .cfi_personality 148, DW.ref.personality
+; CHECK-NEXT:    .cfi_lsda 20, .Lexception0
+; CHECK-NEXT:  .Lfunc_gep0:
+; CHECK-NEXT:    addis r2, r12, .TOC.-.Lfunc_gep0@ha
+; CHECK-NEXT:    addi r2, r2, .TOC.-.Lfunc_gep0@l
+; CHECK-NEXT:  .Lfunc_lep0:
+; CHECK-NEXT:    .localentry test, .Lfunc_lep0-.Lfunc_gep0
+; CHECK-NEXT:  # %bb.0: # %bb
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    std r0, 16(r1)
+; CHECK-NEXT:    stdu r1, -32(r1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    .cfi_offset lr, 16
+; CHECK-NEXT:    li r3, 0
+; CHECK-NEXT:  # %bb.1: # %bb9
+; CHECK-NEXT:    bl test5
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    rlwinm r3, r3, 8, 16, 23
+; CHECK-NEXT:  # %bb.2: # %bb12
+; CHECK-NEXT:    clrldi r4, r3, 32
+; CHECK-NEXT:    bl test3
+; CHECK-NEXT:    addi r1, r1, 32
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+bb:
+  switch i64 undef, label %bb21 [
+    i64 3, label %bb3
+  ]
+
+bb3:                                              ; preds = %bb
+  switch i3 undef, label %bb4 [
+    i3 0, label %bb9
+    i3 1, label %bb12
+  ]
+
+bb4:                                              ; preds = %bb3
+  unreachable
+
+bb5:                                              ; No predecessors!
+  br label %bb12
+
+bb6:                                              ; No predecessors!
+  br label %bb12
+
+bb7:                                              ; No predecessors!
+  br label %bb12
+
+bb8:                                              ; No predecessors!
+  br label %bb12
+
+bb9:                                              ; preds = %bb3
+  %tmp = call i8 @test5(%14* noalias nonnull readonly align 8 dereferenceable(64) undef), !range !0
+  %tmp10 = zext i8 %tmp to i24
+  %tmp11 = shl nuw nsw i24 %tmp10, 8
+  br label %bb12
+
+bb12:                                             ; preds = %bb9, %bb8, %bb7, %bb6, %bb5, %bb3
+  %tmp13 = phi i24 [ 1024, %bb8 ], [ 768, %bb7 ], [ 512, %bb6 ], [ 256, %bb5 ], [ %tmp11, %bb9 ], [ 0, %bb3 ]
+  %tmp14 = call fastcc align 8 dereferenceable(288) %16* @test3(%20* noalias nonnull readonly align 8 dereferenceable(24) undef, i24 %tmp13)
+  br label %bb22
+
+bb15:                                             ; No predecessors!
+  %tmp16 = invoke fastcc { i64*, i8* } @test2(%0** nonnull align 8 dereferenceable(8) undef)
+          to label %bb17 unwind label %bb18
+
+bb17:                                             ; preds = %bb15
+  invoke void @test4({ [0 x i64], { [0 x i8]*, i64 }, [0 x i64], { [0 x i8]*, i64 }, [0 x i32], i32, [0 x i32], i32, [0 x i32] }* noalias readonly align 8 dereferenceable(40) bitcast (<{ i8*, [8 x i8], i8*, [16 x i8] }>* @var to { [0 x i64], { [0 x i8]*, i64 }, [0 x i64], { [0 x i8]*, i64 }, [0 x i32], i32, [0 x i32], i32, [0 x i32] }*))
+          to label %bb23 unwind label %bb25
+
+bb18:                                             ; preds = %bb15
+  %tmp19 = landingpad { i8*, i32 }
+          cleanup
+  resume { i8*, i32 } undef
+
+bb20:                                             ; No predecessors!
+  invoke void @test4({ [0 x i64], { [0 x i8]*, i64 }, [0 x i64], { [0 x i8]*, i64 }, [0 x i32], i32, [0 x i32], i32, [0 x i32] }* noalias readonly align 8 dereferenceable(40) bitcast (<{ i8*, [8 x i8], i8*, [16 x i8] }>* @var to { [0 x i64], { [0 x i8]*, i64 }, [0 x i64], { [0 x i8]*, i64 }, [0 x i32], i32, [0 x i32], i32, [0 x i32] }*))
+          to label %bb24 unwind label %bb25
+
+bb21:                                             ; preds = %bb
+  unreachable
+
+bb22:                                             ; preds = %bb12
+  ret void
+
+bb23:                                             ; preds = %bb17
+  unreachable
+
+bb24:                                             ; preds = %bb20
+  unreachable
+
+bb25:                                             ; preds = %bb20, %bb17
+  %tmp26 = landingpad { i8*, i32 }
+          cleanup
+  resume { i8*, i32 } undef
+}
+
+declare dso_local fastcc %16* @test3(%20*, i24) unnamed_addr
+
+declare i32 @personality(i32, i32, i64, %21*, %22*) unnamed_addr
+
+declare void @test4({ [0 x i64], { [0 x i8]*, i64 }, [0 x i64], { [0 x i8]*, i64 }, [0 x i32], i32, [0 x i32], i32, [0 x i32] }*) unnamed_addr
+
+declare i8 @test5(%14*) unnamed_addr
+
+!0 = !{i8 0, i8 5}
diff --git a/llvm/test/CodeGen/PowerPC/pr42492.ll b/llvm/test/CodeGen/PowerPC/pr42492.ll
index 8dface68b..e404d5d72 100644
--- a/llvm/test/CodeGen/PowerPC/pr42492.ll
+++ b/llvm/test/CodeGen/PowerPC/pr42492.ll
@@ -4,13 +4,27 @@
 define void @f(i8*, i8*, i64*) {
 ; Check we don't assert and this is not a Hardware Loop
 ; CHECK-LABEL: f:
-; CHECK:  .LBB0_2: #
-; CHECK-NEXT:    cmplwi
-; CHECK-NEXT:    cmpd
-; CHECK-NEXT:    sldi
-; CHECK-NEXT:    cror
-; CHECK-NEXT:    addi
-; CHECK-NEXT:    bc
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    cmpld 3, 4
+; CHECK-NEXT:    beqlr 0
+; CHECK-NEXT:  # %bb.1:
+; CHECK-NEXT:    ld 6, 8(5)
+; CHECK-NEXT:    not 3, 3
+; CHECK-NEXT:    add 3, 3, 4
+; CHECK-NEXT:    li 4, 0
+; CHECK-NEXT:    .p2align 5
+; CHECK-NEXT:  .LBB0_2: #
+; CHECK-NEXT:    sldi 6, 6, 4
+; CHECK-NEXT:    cmplwi 4, 14
+; CHECK-NEXT:    addi 7, 4, 1
+; CHECK-NEXT:    bc 12, 1, .LBB0_4
+; CHECK-NEXT:  # %bb.3: #
+; CHECK-NEXT:    cmpd 3, 4
+; CHECK-NEXT:    mr 4, 7
+; CHECK-NEXT:    bc 4, 2, .LBB0_2
+; CHECK-NEXT:  .LBB0_4:
+; CHECK-NEXT:    std 6, 8(5)
+; CHECK-NEXT:    blr
 
   %4 = icmp eq i8* %0, %1
   br i1 %4, label %9, label %5
diff --git a/llvm/test/CodeGen/PowerPC/pr43527.ll b/llvm/test/CodeGen/PowerPC/pr43527.ll
new file mode 100644
index 000000000..023898a46
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/pr43527.ll
@@ -0,0 +1,75 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -ppc-asm-full-reg-names -verify-machineinstrs \
+; RUN:   -mtriple=powerpc64le-unknown-linux-gnu -mcpu=pwr8 < %s | FileCheck %s
+; We don't want to produce a CTR loop due to the call to lrint in the body.
+define dso_local void @test(i64 %arg, i64 %arg1) {
+; CHECK-LABEL: test:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    bc 4, 4*cr5+lt, .LBB0_5
+; CHECK-NEXT:  # %bb.1: # %bb3
+; CHECK-NEXT:    bc 12, 4*cr5+lt, .LBB0_6
+; CHECK-NEXT:  # %bb.2: # %bb4
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-NEXT:    .cfi_offset lr, 16
+; CHECK-NEXT:    .cfi_offset r29, -24
+; CHECK-NEXT:    .cfi_offset r30, -16
+; CHECK-NEXT:    std r29, -24(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r30, -16(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r0, 16(r1)
+; CHECK-NEXT:    stdu r1, -64(r1)
+; CHECK-NEXT:    sub r30, r4, r3
+; CHECK-NEXT:    li r29, 0
+; CHECK-NEXT:    .p2align 5
+; CHECK-NEXT:  .LBB0_3: # %bb5
+; CHECK-NEXT:    #
+; CHECK-NEXT:    lfsx f1, 0, r29
+; CHECK-NEXT:    bl lrint
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    addi r30, r30, -1
+; CHECK-NEXT:    addi r29, r29, 4
+; CHECK-NEXT:    cmpldi r30, 0
+; CHECK-NEXT:    bne cr0, .LBB0_3
+; CHECK-NEXT:  # %bb.4: # %bb15
+; CHECK-NEXT:    stb r3, 0(r3)
+; CHECK-NEXT:    addi r1, r1, 64
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    ld r30, -16(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r29, -24(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+; CHECK-NEXT:  .LBB0_5: # %bb2
+; CHECK-NEXT:  .LBB0_6: # %bb14
+bb:
+  br i1 undef, label %bb3, label %bb2
+
+bb2:                                              ; preds = %bb
+  unreachable
+
+bb3:                                              ; preds = %bb
+  %tmp = sub i64 %arg1, %arg
+  br i1 undef, label %bb4, label %bb14
+
+bb4:                                              ; preds = %bb3
+  br label %bb5
+
+bb5:                                              ; preds = %bb5, %bb4
+  %tmp6 = phi i64 [ %tmp12, %bb5 ], [ 0, %bb4 ]
+  %tmp7 = getelementptr inbounds float, float* null, i64 %tmp6
+  %tmp8 = load float, float* %tmp7, align 4
+  %tmp9 = fpext float %tmp8 to double
+  %tmp10 = tail call i64 @llvm.lrint.i64.f64(double %tmp9) #2
+  %tmp11 = trunc i64 %tmp10 to i8
+  store i8 %tmp11, i8* undef, align 1
+  %tmp12 = add nuw i64 %tmp6, 1
+  %tmp13 = icmp eq i64 %tmp12, %tmp
+  br i1 %tmp13, label %bb15, label %bb5
+
+bb14:                                             ; preds = %bb3
+  unreachable
+
+bb15:                                             ; preds = %bb5
+  ret void
+}
+
+declare i64 @llvm.lrint.i64.f64(double)
diff --git a/llvm/test/CodeGen/PowerPC/pr44183.ll b/llvm/test/CodeGen/PowerPC/pr44183.ll
new file mode 100644
index 000000000..483f84c88
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/pr44183.ll
@@ -0,0 +1,56 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu \
+; RUN:     -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s
+%struct.m.2.5.8.11 = type { %struct.l.0.3.6.9, [7 x i8], %struct.a.1.4.7.10 }
+%struct.l.0.3.6.9 = type { i8 }
+%struct.a.1.4.7.10 = type { [27 x i8], [0 x i32], [4 x i8] }
+define void @_ZN1m1nEv(%struct.m.2.5.8.11* %this) local_unnamed_addr nounwind align 2 {
+; CHECK-LABEL: _ZN1m1nEv:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    std r30, -16(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r0, 16(r1)
+; CHECK-NEXT:    stdu r1, -48(r1)
+; CHECK-NEXT:    mr r30, r3
+; CHECK-NEXT:    ld r4, 8(r30)
+; CHECK-NEXT:    lwz r5, 36(r30)
+; CHECK-NEXT:    rldicl r4, r4, 60, 4
+; CHECK-NEXT:    rlwinm r3, r4, 31, 0, 0
+; CHECK-NEXT:    rlwinm r4, r5, 0, 31, 31
+; CHECK-NEXT:    or r4, r4, r3
+; CHECK-NEXT:    bl _ZN1llsE1d
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    ld r3, 16(r30)
+; CHECK-NEXT:    ld r4, 8(r30)
+; CHECK-NEXT:    rldicl r4, r4, 60, 4
+; CHECK-NEXT:    sldi r3, r3, 60
+; CHECK-NEXT:    or r3, r4, r3
+; CHECK-NEXT:    sldi r3, r3, 31
+; CHECK-NEXT:    clrldi r4, r3, 32
+; CHECK-NEXT:    bl _ZN1llsE1d
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    addi r1, r1, 48
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    ld r30, -16(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+entry:
+  %bc = getelementptr inbounds %struct.m.2.5.8.11, %struct.m.2.5.8.11* %this, i64 0, i32 2
+  %0 = bitcast %struct.a.1.4.7.10* %bc to i216*
+  %bf.load = load i216, i216* %0, align 8
+  %bf.lshr = lshr i216 %bf.load, 4
+  %shl.i23 = shl i216 %bf.lshr, 31
+  %shl.i = trunc i216 %shl.i23 to i32
+  %arrayidx = getelementptr inbounds %struct.m.2.5.8.11, %struct.m.2.5.8.11* %this, i64 0, i32 2, i32 1, i64 0
+  %1 = load i32, i32* %arrayidx, align 4
+  %and.i = and i32 %1, 1
+  %or.i = or i32 %and.i, %shl.i
+  tail call void @_ZN1llsE1d(%struct.l.0.3.6.9* undef, i32 %or.i) #1
+  %bf.load10 = load i216, i216* %0, align 8
+  %bf.lshr11 = lshr i216 %bf.load10, 4
+  %shl.i1524 = shl i216 %bf.lshr11, 31
+  %shl.i15 = trunc i216 %shl.i1524 to i32
+  tail call void @_ZN1llsE1d(%struct.l.0.3.6.9* undef, i32 %shl.i15) #1
+  ret void
+}
+declare void @_ZN1llsE1d(%struct.l.0.3.6.9*, i32) local_unnamed_addr #0
diff --git a/llvm/test/CodeGen/PowerPC/pr44239.ll b/llvm/test/CodeGen/PowerPC/pr44239.ll
new file mode 100644
index 000000000..5e71a31c6
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/pr44239.ll
@@ -0,0 +1,40 @@
+; RUN: llc -mtriple=powerpc64le-unknown-linux-gnu \
+; RUN:   -verify-machineinstrs < %s | FileCheck %s
+
+define <4 x float> @check_vcfsx(<4 x i32> %a) {
+entry:
+  %0 = tail call <4 x float> @llvm.ppc.altivec.vcfsx(<4 x i32> %a, i32 1)
+  ret  <4 x float> %0
+; CHECK-LABEL: check_vcfsx
+; CHECK: vcfsx {{[0-9]+}}, {{[0-9]+}}, 1
+}
+
+define <4 x float> @check_vcfux(<4 x i32> %a) {
+entry:
+  %0 = tail call <4 x float> @llvm.ppc.altivec.vcfux(<4 x i32> %a, i32 1)
+  ret  <4 x float> %0
+; CHECK-LABEL: check_vcfux
+; CHECK: vcfux {{[0-9]+}}, {{[0-9]+}}, 1
+}
+
+define <4 x i32> @check_vctsxs(<4 x float> %a) {
+entry:
+  %0 = tail call <4 x i32> @llvm.ppc.altivec.vctsxs(<4 x float> %a, i32 1)
+  ret  <4 x i32> %0
+; CHECK-LABEL: check_vctsxs
+; CHECK: vctsxs {{[0-9]+}}, {{[0-9]+}}, 1
+}
+
+define <4 x i32> @check_vctuxs(<4 x float> %a) {
+entry:
+  %0 = tail call <4 x i32> @llvm.ppc.altivec.vctuxs(<4 x float> %a, i32 1)
+  ret  <4 x i32> %0
+; CHECK-LABEL: check_vctuxs
+; CHECK: vctuxs {{[0-9]+}}, {{[0-9]+}}, 1
+}
+
+declare <4 x float> @llvm.ppc.altivec.vcfsx(<4 x i32>, i32 immarg)
+declare <4 x float> @llvm.ppc.altivec.vcfux(<4 x i32>, i32 immarg)
+declare <4 x i32> @llvm.ppc.altivec.vctsxs(<4 x float>, i32 immarg)
+declare <4 x i32> @llvm.ppc.altivec.vctuxs(<4 x float>, i32 immarg)
+
diff --git a/llvm/test/CodeGen/PowerPC/pre-inc-disable.ll b/llvm/test/CodeGen/PowerPC/pre-inc-disable.ll
index a19e7e224..60df77fc0 100644
--- a/llvm/test/CodeGen/PowerPC/pre-inc-disable.ll
+++ b/llvm/test/CodeGen/PowerPC/pre-inc-disable.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mcpu=pwr9 -O3 -verify-machineinstrs -ppc-vsr-nums-as-vr \
 ; RUN:     -ppc-asm-full-reg-names -mtriple=powerpc64le-unknown-linux-gnu \
 ; RUN:     < %s | FileCheck %s
@@ -29,11 +30,14 @@ define signext i32 @test_pre_inc_disable_1(i8* nocapture readonly %pix1, i32 sig
 ; CHECK-NEXT:    xvnegsp v0, v1
 ; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_1: # %for.cond1.preheader
-; CHECK:    lfd f0, 0(r3)
+; CHECK-NEXT:    #
+; CHECK-NEXT:    lfd f0, 0(r3)
 ; CHECK-NEXT:    xxpermdi v1, f0, f0, 2
+; CHECK-NEXT:    lfdx f0, r3, r4
 ; CHECK-NEXT:    vperm v6, v1, v3, v4
 ; CHECK-NEXT:    vperm v1, v3, v1, v2
 ; CHECK-NEXT:    xvnegsp v1, v1
+; CHECK-NEXT:    add r7, r3, r4
 ; CHECK-NEXT:    xvnegsp v6, v6
 ; CHECK-NEXT:    vabsduw v1, v1, v5
 ; CHECK-NEXT:    vabsduw v6, v6, v0
@@ -42,16 +46,14 @@ define signext i32 @test_pre_inc_disable_1(i8* nocapture readonly %pix1, i32 sig
 ; CHECK-NEXT:    vadduwm v1, v1, v6
 ; CHECK-NEXT:    xxspltw v6, v1, 2
 ; CHECK-NEXT:    vadduwm v1, v1, v6
-; CHECK-NEXT:    vextuwrx r7, r5, v1
-; CHECK-NEXT:    lfdx f0, r3, r4
-; CHECK-NEXT:    add r6, r7, r6
-; CHECK-NEXT:    add r7, r3, r4
-; CHECK-NEXT:    xxpermdi v1, f0, f0, 2
+; CHECK-NEXT:    xxpermdi v6, f0, f0, 2
+; CHECK-NEXT:    vextuwrx r3, r5, v1
+; CHECK-NEXT:    vperm v7, v6, v3, v4
+; CHECK-NEXT:    vperm v6, v3, v6, v2
+; CHECK-NEXT:    add r6, r3, r6
 ; CHECK-NEXT:    add r3, r7, r4
-; CHECK-NEXT:    vperm v6, v3, v1, v2
-; CHECK-NEXT:    vperm v1, v1, v3, v4
 ; CHECK-NEXT:    xvnegsp v6, v6
-; CHECK-NEXT:    xvnegsp v1, v1
+; CHECK-NEXT:    xvnegsp v1, v7
 ; CHECK-NEXT:    vabsduw v6, v6, v5
 ; CHECK-NEXT:    vabsduw v1, v1, v0
 ; CHECK-NEXT:    vadduwm v1, v1, v6
@@ -87,8 +89,10 @@ define signext i32 @test_pre_inc_disable_1(i8* nocapture readonly %pix1, i32 sig
 ; P9BE-NEXT:    xvnegsp v0, v1
 ; P9BE-NEXT:    .p2align 4
 ; P9BE-NEXT:  .LBB0_1: # %for.cond1.preheader
-; P9BE:    lfd f0, 0(r3)
+; P9BE-NEXT:    #
+; P9BE-NEXT:    lfd f0, 0(r3)
 ; P9BE-NEXT:    xxlor v1, vs0, vs0
+; P9BE-NEXT:    lfdx f0, r3, r4
 ; P9BE-NEXT:    vperm v6, v3, v1, v4
 ; P9BE-NEXT:    vperm v1, v3, v1, v2
 ; P9BE-NEXT:    xvnegsp v1, v1
@@ -100,24 +104,23 @@ define signext i32 @test_pre_inc_disable_1(i8* nocapture readonly %pix1, i32 sig
 ; P9BE-NEXT:    vadduwm v1, v1, v6
 ; P9BE-NEXT:    xxspltw v6, v1, 1
 ; P9BE-NEXT:    vadduwm v1, v1, v6
-; P9BE-NEXT:    vextuwlx r7, r5, v1
-; P9BE-NEXT:    lfdx f0, r3, r4
-; P9BE-NEXT:    add r6, r7, r6
+; P9BE-NEXT:    xxlor v6, vs0, vs0
+; P9BE-NEXT:    vperm v7, v3, v6, v4
+; P9BE-NEXT:    vperm v6, v3, v6, v2
 ; P9BE-NEXT:    add r7, r3, r4
-; P9BE-NEXT:    xxlor v1, vs0, vs0
-; P9BE-NEXT:    add r3, r7, r4
-; P9BE-NEXT:    vperm v6, v3, v1, v2
-; P9BE-NEXT:    vperm v1, v3, v1, v4
+; P9BE-NEXT:    vextuwlx r3, r5, v1
 ; P9BE-NEXT:    xvnegsp v6, v6
-; P9BE-NEXT:    xvnegsp v1, v1
-; P9BE-NEXT:    vabsduw v6, v6, v5
+; P9BE-NEXT:    xvnegsp v1, v7
 ; P9BE-NEXT:    vabsduw v1, v1, v0
+; P9BE-NEXT:    vabsduw v6, v6, v5
 ; P9BE-NEXT:    vadduwm v1, v1, v6
 ; P9BE-NEXT:    xxswapd v6, v1
+; P9BE-NEXT:    add r6, r3, r6
 ; P9BE-NEXT:    vadduwm v1, v1, v6
 ; P9BE-NEXT:    xxspltw v6, v1, 1
 ; P9BE-NEXT:    vadduwm v1, v1, v6
 ; P9BE-NEXT:    vextuwlx r8, r5, v1
+; P9BE-NEXT:    add r3, r7, r4
 ; P9BE-NEXT:    add r6, r8, r6
 ; P9BE-NEXT:    bdnz .LBB0_1
 ; P9BE-NEXT:  # %bb.2: # %for.cond.cleanup
@@ -280,6 +283,51 @@ entry:
 ;}
 
 define void @test32(i8* nocapture readonly %pix2, i32 signext %i_pix2) {
+; CHECK-LABEL: test32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    add r5, r3, r4
+; CHECK-NEXT:    lfiwzx f0, r3, r4
+; CHECK-NEXT:    addis r3, r2, .LCPI2_0@toc@ha
+; CHECK-NEXT:    addi r3, r3, .LCPI2_0@toc@l
+; CHECK-NEXT:    lxvx v4, 0, r3
+; CHECK-NEXT:    li r3, 4
+; CHECK-NEXT:    xxpermdi v2, f0, f0, 2
+; CHECK-NEXT:    lfiwzx f0, r5, r3
+; CHECK-NEXT:    xxlxor v3, v3, v3
+; CHECK-NEXT:    vperm v2, v2, v3, v4
+; CHECK-NEXT:    xxpermdi v5, f0, f0, 2
+; CHECK-NEXT:    vperm v3, v5, v3, v4
+; CHECK-NEXT:    vspltisw v4, 8
+; CHECK-NEXT:    vnegw v3, v3
+; CHECK-NEXT:    vadduwm v4, v4, v4
+; CHECK-NEXT:    vslw v3, v3, v4
+; CHECK-NEXT:    vsubuwm v2, v3, v2
+; CHECK-NEXT:    xxswapd vs0, v2
+; CHECK-NEXT:    stxvx vs0, 0, r3
+; CHECK-NEXT:    blr
+;
+; P9BE-LABEL: test32:
+; P9BE:       # %bb.0: # %entry
+; P9BE-NEXT:    add r5, r3, r4
+; P9BE-NEXT:    lfiwzx f0, r3, r4
+; P9BE-NEXT:    addis r3, r2, .LCPI2_0@toc@ha
+; P9BE-NEXT:    addi r3, r3, .LCPI2_0@toc@l
+; P9BE-NEXT:    lxvx v4, 0, r3
+; P9BE-NEXT:    li r3, 4
+; P9BE-NEXT:    xxsldwi v2, f0, f0, 1
+; P9BE-NEXT:    lfiwzx f0, r5, r3
+; P9BE-NEXT:    xxlxor v3, v3, v3
+; P9BE-NEXT:    vperm v2, v3, v2, v4
+; P9BE-NEXT:    xxsldwi v5, f0, f0, 1
+; P9BE-NEXT:    vperm v3, v3, v5, v4
+; P9BE-NEXT:    vspltisw v4, 8
+; P9BE-NEXT:    vnegw v3, v3
+; P9BE-NEXT:    vadduwm v4, v4, v4
+; P9BE-NEXT:    vslw v3, v3, v4
+; P9BE-NEXT:    vsubuwm v2, v3, v2
+; P9BE-NEXT:    xxswapd vs0, v2
+; P9BE-NEXT:    stxvx vs0, 0, r3
+; P9BE-NEXT:    blr
 entry:
   %idx.ext63 = sext i32 %i_pix2 to i64
   %add.ptr64 = getelementptr inbounds i8, i8* %pix2, i64 %idx.ext63
@@ -302,19 +350,62 @@ entry:
   %13 = shufflevector <4 x i32> %12, <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
   store <4 x i32> %13, <4 x i32>* undef, align 16
   ret void
-; CHECK-LABEL: test32:
-; CHECK-NOT: lwzux
-; CHECK-NOT: mtvsrws
-; CHECK: lfiwzx
-; CHECK: lfiwzx
-; P9BE-CHECK-LABEL: test32:
-; P9BE-CHECK-NOT: lwzux
-; P9BE-CHECK-NOT: mtvsrws
-; P9BE-CHECK: lfiwzx
-; P9BE-CHECK: lfiwzx
 }
 
 define void @test16(i16* nocapture readonly %sums, i32 signext %delta, i32 signext %thresh) {
+; CHECK-LABEL: test16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    sldi r4, r4, 1
+; CHECK-NEXT:    lxsihzx v2, r3, r4
+; CHECK-NEXT:    vsplth v2, v2, 3
+; CHECK-NEXT:    xxlxor v3, v3, v3
+; CHECK-NEXT:    vmrglh v2, v3, v2
+; CHECK-NEXT:    vsplth v4, v3, 7
+; CHECK-NEXT:    add r6, r3, r4
+; CHECK-NEXT:    li r3, 16
+; CHECK-NEXT:    vmrglw v2, v2, v4
+; CHECK-NEXT:    lxsihzx v4, r6, r3
+; CHECK-NEXT:    addis r3, r2, .LCPI3_0@toc@ha
+; CHECK-NEXT:    addi r3, r3, .LCPI3_0@toc@l
+; CHECK-NEXT:    vsplth v4, v4, 3
+; CHECK-NEXT:    vmrglh v3, v3, v4
+; CHECK-NEXT:    lxvx v4, 0, r3
+; CHECK-NEXT:    li r3, 0
+; CHECK-NEXT:    vperm v2, v3, v2, v4
+; CHECK-NEXT:    xxspltw v3, v2, 2
+; CHECK-NEXT:    vadduwm v2, v2, v3
+; CHECK-NEXT:    vextuwrx r3, r3, v2
+; CHECK-NEXT:    cmpw cr0, r3, r5
+; CHECK-NEXT:    bgelr+ cr0
+; CHECK-NEXT:  # %bb.1: # %if.then
+;
+; P9BE-LABEL: test16:
+; P9BE:       # %bb.0: # %entry
+; P9BE-NEXT:    sldi r4, r4, 1
+; P9BE-NEXT:    add r6, r3, r4
+; P9BE-NEXT:    li r7, 16
+; P9BE-NEXT:    lxsihzx v2, r6, r7
+; P9BE-NEXT:    lxsihzx v4, r3, r4
+; P9BE-NEXT:    li r6, 0
+; P9BE-NEXT:    sldi r6, r6, 48
+; P9BE-NEXT:    vsplth v4, v4, 3
+; P9BE-NEXT:    mtvsrd v3, r6
+; P9BE-NEXT:    vsplth v2, v2, 3
+; P9BE-NEXT:    addis r3, r2, .LCPI3_0@toc@ha
+; P9BE-NEXT:    addi r3, r3, .LCPI3_0@toc@l
+; P9BE-NEXT:    vmrghh v4, v3, v4
+; P9BE-NEXT:    vmrghh v2, v3, v2
+; P9BE-NEXT:    vsplth v3, v3, 0
+; P9BE-NEXT:    vmrghw v3, v3, v4
+; P9BE-NEXT:    lxvx v4, 0, r3
+; P9BE-NEXT:    li r3, 0
+; P9BE-NEXT:    vperm v2, v3, v2, v4
+; P9BE-NEXT:    xxspltw v3, v2, 1
+; P9BE-NEXT:    vadduwm v2, v2, v3
+; P9BE-NEXT:    vextuwlx r3, r3, v2
+; P9BE-NEXT:    cmpw cr0, r3, r5
+; P9BE-NEXT:    bgelr+ cr0
+; P9BE-NEXT:  # %bb.1: # %if.then
 entry:
   %idxprom = sext i32 %delta to i64
   %add14 = add nsw i32 %delta, 8
@@ -345,17 +436,67 @@ if.then:                                          ; preds = %for.body
 
 if.end:                                           ; preds = %for.body
   ret void
-; CHECK-LABEL: test16:
-; CHECK-NOT: lhzux
-; CHECK: lxsihzx
-; CHECK: lxsihzx
-; P9BE-CHECK-LABEL: test16:
-; P9BE-CHECK-NOT: lhzux
-; P9BE-CHECK: lxsihzx
-; P9BE-CHECK: lxsihzx
 }
 
 define void @test8(i8* nocapture readonly %sums, i32 signext %delta, i32 signext %thresh) {
+; CHECK-LABEL: test8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    add r6, r3, r4
+; CHECK-NEXT:    lxsibzx v2, r3, r4
+; CHECK-NEXT:    li r3, 0
+; CHECK-NEXT:    mtvsrd f0, r3
+; CHECK-NEXT:    li r3, 8
+; CHECK-NEXT:    lxsibzx v5, r6, r3
+; CHECK-NEXT:    xxswapd v3, vs0
+; CHECK-NEXT:    vspltb v4, v3, 15
+; CHECK-NEXT:    vspltb v2, v2, 7
+; CHECK-NEXT:    vmrglb v2, v3, v2
+; CHECK-NEXT:    addis r3, r2, .LCPI4_0@toc@ha
+; CHECK-NEXT:    addi r3, r3, .LCPI4_0@toc@l
+; CHECK-NEXT:    vspltb v5, v5, 7
+; CHECK-NEXT:    vmrglh v2, v2, v4
+; CHECK-NEXT:    vmrglb v3, v3, v5
+; CHECK-NEXT:    vmrglw v2, v2, v4
+; CHECK-NEXT:    vmrglh v3, v3, v4
+; CHECK-NEXT:    vmrglw v3, v4, v3
+; CHECK-NEXT:    lxvx v4, 0, r3
+; CHECK-NEXT:    li r3, 0
+; CHECK-NEXT:    vperm v2, v3, v2, v4
+; CHECK-NEXT:    xxspltw v3, v2, 2
+; CHECK-NEXT:    vadduwm v2, v2, v3
+; CHECK-NEXT:    vextuwrx r3, r3, v2
+; CHECK-NEXT:    cmpw cr0, r3, r5
+; CHECK-NEXT:    bgelr+ cr0
+; CHECK-NEXT:  # %bb.1: # %if.then
+;
+; P9BE-LABEL: test8:
+; P9BE:       # %bb.0: # %entry
+; P9BE-NEXT:    add r6, r3, r4
+; P9BE-NEXT:    li r7, 8
+; P9BE-NEXT:    lxsibzx v2, r6, r7
+; P9BE-NEXT:    lxsibzx v4, r3, r4
+; P9BE-NEXT:    li r6, 0
+; P9BE-NEXT:    sldi r6, r6, 56
+; P9BE-NEXT:    vspltb v4, v4, 7
+; P9BE-NEXT:    mtvsrd v3, r6
+; P9BE-NEXT:    vspltb v2, v2, 7
+; P9BE-NEXT:    vmrghb v4, v3, v4
+; P9BE-NEXT:    vmrghb v2, v3, v2
+; P9BE-NEXT:    vspltb v3, v3, 0
+; P9BE-NEXT:    addis r3, r2, .LCPI4_0@toc@ha
+; P9BE-NEXT:    addi r3, r3, .LCPI4_0@toc@l
+; P9BE-NEXT:    vmrghh v4, v4, v3
+; P9BE-NEXT:    xxspltw v3, v3, 0
+; P9BE-NEXT:    vmrghw v2, v4, v2
+; P9BE-NEXT:    lxvx v4, 0, r3
+; P9BE-NEXT:    li r3, 0
+; P9BE-NEXT:    vperm v2, v3, v2, v4
+; P9BE-NEXT:    xxspltw v3, v2, 1
+; P9BE-NEXT:    vadduwm v2, v2, v3
+; P9BE-NEXT:    vextuwlx r3, r3, v2
+; P9BE-NEXT:    cmpw cr0, r3, r5
+; P9BE-NEXT:    bgelr+ cr0
+; P9BE-NEXT:  # %bb.1: # %if.then
 entry:
   %idxprom = sext i32 %delta to i64
   %add14 = add nsw i32 %delta, 8
@@ -386,12 +527,4 @@ if.then:                                          ; preds = %for.body
 
 if.end:                                           ; preds = %for.body
   ret void
-; CHECK-LABEL: test8:
-; CHECK-NOT: lbzux
-; CHECK: lxsibzx
-; CHECK: lxsibzx
-; P9BE-CHECK-LABEL: test8:
-; P9BE-CHECK-NOT: lbzux
-; P9BE-CHECK: lxsibzx
-; P9BE-CHECK: lxsibzx
 }
diff --git a/llvm/test/CodeGen/PowerPC/pwr7-gt-nop.ll b/llvm/test/CodeGen/PowerPC/pwr7-gt-nop.ll
index be8578c1b..b7c899c0f 100644
--- a/llvm/test/CodeGen/PowerPC/pwr7-gt-nop.ll
+++ b/llvm/test/CodeGen/PowerPC/pwr7-gt-nop.ll
@@ -16,12 +16,12 @@ entry:
   store float %2, float* %d, align 4
   ret void
 
-; CHECK: lfs [[REG1:[0-9]+]], 0(4)
-; CHECK: stfs [[REG1]], 0(3)
-; CHECK: lfs [[REG2:[0-9]+]], 0(5)
-; CHECK: stfs [[REG2]], 0(4)
-; CHECK: lfs [[REG3:[0-9]+]], 0(3)
-; CHECK: stfs [[REG3]], 0(6)
+; CHECK: lwz [[REG1:[0-9]+]], 0(4)
+; CHECK: stw [[REG1]], 0(3)
+; CHECK: lwz [[REG2:[0-9]+]], 0(5)
+; CHECK: stw [[REG2]], 0(4)
+; CHECK: lwz [[REG3:[0-9]+]], 0(3)
+; CHECK: stw [[REG3]], 0(6)
 ; CHECK: blr
 }
 
diff --git a/llvm/test/CodeGen/PowerPC/qpx-load-splat.ll b/llvm/test/CodeGen/PowerPC/qpx-load-splat.ll
index 1afd27262..818738fa9 100644
--- a/llvm/test/CodeGen/PowerPC/qpx-load-splat.ll
+++ b/llvm/test/CodeGen/PowerPC/qpx-load-splat.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=powerpc64le-unknown-linux-gnu -ppc-vsr-nums-as-vr \
 ; RUN:   -ppc-asm-full-reg-names -verify-machineinstrs < %s | FileCheck %s
 
@@ -34,9 +35,9 @@ define <4 x double> @fooxu(double* nocapture readonly %a, i64 %idx, double** %pp
 ; CHECK-LABEL: fooxu:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    sldi r4, r4, 3
-; CHECK-NEXT:    lfdux f0, r3, r4
-; CHECK-NEXT:    xxspltd v2, vs0, 0
-; CHECK-NEXT:    std r3, 0(r5)
+; CHECK-NEXT:    add r6, r3, r4
+; CHECK-NEXT:    std r6, 0(r5)
+; CHECK-NEXT:    lxvdsx v2, r3, r4
 ; CHECK-NEXT:    vmr v3, v2
 ; CHECK-NEXT:    blr
 entry:
diff --git a/llvm/test/CodeGen/PowerPC/qpx-recipest.ll b/llvm/test/CodeGen/PowerPC/qpx-recipest.ll
index 72fbe0004..246bec191 100644
--- a/llvm/test/CodeGen/PowerPC/qpx-recipest.ll
+++ b/llvm/test/CodeGen/PowerPC/qpx-recipest.ll
@@ -1,194 +1,393 @@
-; RUN: llc -verify-machineinstrs < %s -mtriple=powerpc64-unknown-linux-gnu -mcpu=a2q -enable-unsafe-fp-math | FileCheck %s
-; RUN: llc -verify-machineinstrs < %s -mtriple=powerpc64-unknown-linux-gnu -mcpu=a2q | FileCheck -check-prefix=CHECK-SAFE %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -verify-machineinstrs < %s -mtriple=powerpc64-unknown-linux-gnu -mcpu=a2q | FileCheck %s
 target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
 target triple = "powerpc64-unknown-linux-gnu"
 
 declare <4 x double> @llvm.sqrt.v4f64(<4 x double>)
 declare <4 x float> @llvm.sqrt.v4f32(<4 x float>)
 
-define <4 x double> @foo(<4 x double> %a, <4 x double> %b) nounwind {
+define <4 x double> @foo_fmf(<4 x double> %a, <4 x double> %b) nounwind {
+; CHECK-LABEL: foo_fmf:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addis 3, 2, .LCPI0_0@toc@ha
+; CHECK-NEXT:    qvfrsqrte 3, 2
+; CHECK-NEXT:    addi 3, 3, .LCPI0_0@toc@l
+; CHECK-NEXT:    qvlfdx 0, 0, 3
+; CHECK-NEXT:    qvfmul 4, 3, 3
+; CHECK-NEXT:    qvfmsub 2, 2, 0, 2
+; CHECK-NEXT:    qvfnmsub 4, 2, 4, 0
+; CHECK-NEXT:    qvfmul 3, 3, 4
+; CHECK-NEXT:    qvfmul 4, 3, 3
+; CHECK-NEXT:    qvfnmsub 0, 2, 4, 0
+; CHECK-NEXT:    qvfmul 0, 3, 0
+; CHECK-NEXT:    qvfmul 1, 1, 0
+; CHECK-NEXT:    blr
+entry:
+  %x = call fast <4 x double> @llvm.sqrt.v4f64(<4 x double> %b)
+  %r = fdiv fast <4 x double> %a, %x
+  ret <4 x double> %r
+}
+
+define <4 x double> @foo_safe(<4 x double> %a, <4 x double> %b) nounwind {
+; CHECK-LABEL: foo_safe:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    qvesplati 5, 2, 3
+; CHECK-NEXT:    qvesplati 3, 2, 1
+; CHECK-NEXT:    qvesplati 4, 2, 2
+; CHECK-NEXT:    fsqrt 2, 2
+; CHECK-NEXT:    fsqrt 5, 5
+; CHECK-NEXT:    fsqrt 4, 4
+; CHECK-NEXT:    fsqrt 3, 3
+; CHECK-NEXT:    qvesplati 6, 1, 3
+; CHECK-NEXT:    qvgpci 0, 275
+; CHECK-NEXT:    fdiv 2, 1, 2
+; CHECK-NEXT:    fdiv 5, 6, 5
+; CHECK-NEXT:    qvesplati 6, 1, 2
+; CHECK-NEXT:    qvesplati 1, 1, 1
+; CHECK-NEXT:    fdiv 4, 6, 4
+; CHECK-NEXT:    fdiv 1, 1, 3
+; CHECK-NEXT:    qvfperm 3, 4, 5, 0
+; CHECK-NEXT:    qvfperm 0, 2, 1, 0
+; CHECK-NEXT:    qvgpci 1, 101
+; CHECK-NEXT:    qvfperm 1, 0, 3, 1
+; CHECK-NEXT:    blr
 entry:
   %x = call <4 x double> @llvm.sqrt.v4f64(<4 x double> %b)
   %r = fdiv <4 x double> %a, %x
   ret <4 x double> %r
+}
+
+define <4 x double> @foof_fmf(<4 x double> %a, <4 x float> %b) nounwind {
+; CHECK-LABEL: foof_fmf:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addis 3, 2, .LCPI2_0@toc@ha
+; CHECK-NEXT:    qvfrsqrtes 3, 2
+; CHECK-NEXT:    addi 3, 3, .LCPI2_0@toc@l
+; CHECK-NEXT:    qvlfsx 0, 0, 3
+; CHECK-NEXT:    qvfmuls 4, 3, 3
+; CHECK-NEXT:    qvfnmsubs 2, 2, 0, 2
+; CHECK-NEXT:    qvfmadds 0, 2, 4, 0
+; CHECK-NEXT:    qvfmuls 0, 3, 0
+; CHECK-NEXT:    qvfmul 1, 1, 0
+; CHECK-NEXT:    blr
+entry:
+  %x = call fast <4 x float> @llvm.sqrt.v4f32(<4 x float> %b)
+  %y = fpext <4 x float> %x to <4 x double>
+  %r = fdiv fast <4 x double> %a, %y
+  ret <4 x double> %r
+}
 
-; CHECK-LABEL: @foo
-; CHECK: qvfrsqrte
-; CHECK-DAG: qvfmul
-; FIXME: We're currently loading two constants here (1.5 and -1.5), and using
-;        an qvfmadd instead of a qvfnmsub
-; CHECK-DAG: qvfmadd
-; CHECK-DAG: qvfmadd
-; CHECK: qvfmul
-; CHECK: qvfmul
-; CHECK: qvfmadd
-; CHECK: qvfmul
-; CHECK: qvfmul
-; CHECK: blr
-
-; CHECK-SAFE-LABEL: @foo
-; CHECK-SAFE: fsqrt
-; CHECK-SAFE: fdiv
-; CHECK-SAFE: blr
-}
-
-define <4 x double> @foof(<4 x double> %a, <4 x float> %b) nounwind {
+define <4 x double> @foof_safe(<4 x double> %a, <4 x float> %b) nounwind {
+; CHECK-LABEL: foof_safe:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    qvesplati 0, 2, 3
+; CHECK-NEXT:    qvesplati 3, 2, 2
+; CHECK-NEXT:    fsqrts 4, 2
+; CHECK-NEXT:    qvesplati 2, 2, 1
+; CHECK-NEXT:    fsqrts 0, 0
+; CHECK-NEXT:    fsqrts 3, 3
+; CHECK-NEXT:    fsqrts 2, 2
+; CHECK-NEXT:    qvgpci 5, 275
+; CHECK-NEXT:    qvgpci 6, 101
+; CHECK-NEXT:    qvfperm 0, 3, 0, 5
+; CHECK-NEXT:    qvesplati 3, 1, 2
+; CHECK-NEXT:    qvfperm 2, 4, 2, 5
+; CHECK-NEXT:    qvfperm 0, 2, 0, 6
+; CHECK-NEXT:    qvesplati 2, 1, 3
+; CHECK-NEXT:    qvesplati 4, 0, 3
+; CHECK-NEXT:    fdiv 2, 2, 4
+; CHECK-NEXT:    qvesplati 4, 0, 2
+; CHECK-NEXT:    fdiv 3, 3, 4
+; CHECK-NEXT:    qvesplati 4, 1, 1
+; CHECK-NEXT:    fdiv 1, 1, 0
+; CHECK-NEXT:    qvesplati 0, 0, 1
+; CHECK-NEXT:    fdiv 0, 4, 0
+; CHECK-NEXT:    qvfperm 2, 3, 2, 5
+; CHECK-NEXT:    qvfperm 0, 1, 0, 5
+; CHECK-NEXT:    qvfperm 1, 0, 2, 6
+; CHECK-NEXT:    blr
 entry:
   %x = call <4 x float> @llvm.sqrt.v4f32(<4 x float> %b)
   %y = fpext <4 x float> %x to <4 x double>
   %r = fdiv <4 x double> %a, %y
   ret <4 x double> %r
+}
 
-; CHECK-LABEL: @foof
-; CHECK: qvfrsqrtes
-; CHECK-DAG: qvfmuls
-; FIXME: We're currently loading two constants here (1.5 and -1.5), and using
-;        an qvfmadd instead of a qvfnmsubs
-; CHECK-DAG: qvfmadds
-; CHECK-DAG: qvfmadds
-; CHECK: qvfmuls
-; CHECK: qvfmul
-; CHECK: blr
-
-; CHECK-SAFE-LABEL: @foof
-; CHECK-SAFE: fsqrts
-; CHECK-SAFE: fdiv
-; CHECK-SAFE: blr
+define <4 x float> @food_fmf(<4 x float> %a, <4 x double> %b) nounwind {
+; CHECK-LABEL: food_fmf:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addis 3, 2, .LCPI4_0@toc@ha
+; CHECK-NEXT:    qvfrsqrte 3, 2
+; CHECK-NEXT:    addi 3, 3, .LCPI4_0@toc@l
+; CHECK-NEXT:    qvlfdx 0, 0, 3
+; CHECK-NEXT:    qvfmul 4, 3, 3
+; CHECK-NEXT:    qvfmsub 2, 2, 0, 2
+; CHECK-NEXT:    qvfnmsub 4, 2, 4, 0
+; CHECK-NEXT:    qvfmul 3, 3, 4
+; CHECK-NEXT:    qvfmul 4, 3, 3
+; CHECK-NEXT:    qvfnmsub 0, 2, 4, 0
+; CHECK-NEXT:    qvfmul 0, 3, 0
+; CHECK-NEXT:    qvfrsp 0, 0
+; CHECK-NEXT:    qvfmuls 1, 1, 0
+; CHECK-NEXT:    blr
+entry:
+  %x = call fast <4 x double> @llvm.sqrt.v4f64(<4 x double> %b)
+  %y = fptrunc <4 x double> %x to <4 x float>
+  %r = fdiv fast <4 x float> %a, %y
+  ret <4 x float> %r
 }
 
-define <4 x float> @food(<4 x float> %a, <4 x double> %b) nounwind {
+define <4 x float> @food_safe(<4 x float> %a, <4 x double> %b) nounwind {
+; CHECK-LABEL: food_safe:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    qvesplati 0, 2, 3
+; CHECK-NEXT:    qvesplati 3, 2, 2
+; CHECK-NEXT:    fsqrt 4, 2
+; CHECK-NEXT:    qvesplati 2, 2, 1
+; CHECK-NEXT:    fsqrt 0, 0
+; CHECK-NEXT:    fsqrt 3, 3
+; CHECK-NEXT:    fsqrt 2, 2
+; CHECK-NEXT:    qvgpci 5, 275
+; CHECK-NEXT:    qvgpci 6, 101
+; CHECK-NEXT:    qvfperm 0, 3, 0, 5
+; CHECK-NEXT:    qvesplati 3, 1, 2
+; CHECK-NEXT:    qvfperm 2, 4, 2, 5
+; CHECK-NEXT:    qvfperm 0, 2, 0, 6
+; CHECK-NEXT:    qvesplati 2, 1, 3
+; CHECK-NEXT:    qvfrsp 0, 0
+; CHECK-NEXT:    qvesplati 4, 0, 3
+; CHECK-NEXT:    fdivs 2, 2, 4
+; CHECK-NEXT:    qvesplati 4, 0, 2
+; CHECK-NEXT:    fdivs 3, 3, 4
+; CHECK-NEXT:    qvesplati 4, 1, 1
+; CHECK-NEXT:    fdivs 1, 1, 0
+; CHECK-NEXT:    qvesplati 0, 0, 1
+; CHECK-NEXT:    fdivs 0, 4, 0
+; CHECK-NEXT:    qvfperm 2, 3, 2, 5
+; CHECK-NEXT:    qvfperm 0, 1, 0, 5
+; CHECK-NEXT:    qvfperm 1, 0, 2, 6
+; CHECK-NEXT:    blr
 entry:
   %x = call <4 x double> @llvm.sqrt.v4f64(<4 x double> %b)
   %y = fptrunc <4 x double> %x to <4 x float>
   %r = fdiv <4 x float> %a, %y
   ret <4 x float> %r
+}
+
+define <4 x float> @goo_fmf(<4 x float> %a, <4 x float> %b) nounwind {
+; CHECK-LABEL: goo_fmf:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addis 3, 2, .LCPI6_0@toc@ha
+; CHECK-NEXT:    qvfrsqrtes 3, 2
+; CHECK-NEXT:    addi 3, 3, .LCPI6_0@toc@l
+; CHECK-NEXT:    qvlfsx 0, 0, 3
+; CHECK-NEXT:    qvfmuls 4, 3, 3
+; CHECK-NEXT:    qvfnmsubs 2, 2, 0, 2
+; CHECK-NEXT:    qvfmadds 0, 2, 4, 0
+; CHECK-NEXT:    qvfmuls 0, 3, 0
+; CHECK-NEXT:    qvfmuls 1, 1, 0
+; CHECK-NEXT:    blr
+entry:
+  %x = call fast <4 x float> @llvm.sqrt.v4f32(<4 x float> %b)
+  %r = fdiv fast <4 x float> %a, %x
+  ret <4 x float> %r
+}
 
-; CHECK-LABEL: @food
-; CHECK: qvfrsqrte
-; CHECK-DAG: qvfmul
-; FIXME: We're currently loading two constants here (1.5 and -1.5), and using
-;        an qvfmadd instead of a qvfnmsub
-; CHECK-DAG: qvfmadd
-; CHECK-DAG: qvfmadd
-; CHECK: qvfmul
-; CHECK: qvfmul
-; CHECK: qvfmadd
-; CHECK: qvfmul
-; CHECK: qvfrsp
-; CHECK: qvfmuls
-; CHECK: blr
-
-; CHECK-SAFE-LABEL: @food
-; CHECK-SAFE: fsqrt
-; CHECK-SAFE: fdivs
-; CHECK-SAFE: blr
-}
-
-define <4 x float> @goo(<4 x float> %a, <4 x float> %b) nounwind {
+define <4 x float> @goo_safe(<4 x float> %a, <4 x float> %b) nounwind {
+; CHECK-LABEL: goo_safe:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    qvesplati 5, 2, 3
+; CHECK-NEXT:    qvesplati 3, 2, 1
+; CHECK-NEXT:    qvesplati 4, 2, 2
+; CHECK-NEXT:    fsqrts 2, 2
+; CHECK-NEXT:    fsqrts 5, 5
+; CHECK-NEXT:    fsqrts 4, 4
+; CHECK-NEXT:    fsqrts 3, 3
+; CHECK-NEXT:    qvesplati 6, 1, 3
+; CHECK-NEXT:    qvgpci 0, 275
+; CHECK-NEXT:    fdivs 2, 1, 2
+; CHECK-NEXT:    fdivs 5, 6, 5
+; CHECK-NEXT:    qvesplati 6, 1, 2
+; CHECK-NEXT:    qvesplati 1, 1, 1
+; CHECK-NEXT:    fdivs 4, 6, 4
+; CHECK-NEXT:    fdivs 1, 1, 3
+; CHECK-NEXT:    qvfperm 3, 4, 5, 0
+; CHECK-NEXT:    qvfperm 0, 2, 1, 0
+; CHECK-NEXT:    qvgpci 1, 101
+; CHECK-NEXT:    qvfperm 1, 0, 3, 1
+; CHECK-NEXT:    blr
 entry:
   %x = call <4 x float> @llvm.sqrt.v4f32(<4 x float> %b)
   %r = fdiv <4 x float> %a, %x
   ret <4 x float> %r
-
-; CHECK-LABEL: @goo
-; CHECK: qvfrsqrtes
-; CHECK-DAG: qvfmuls
-; FIXME: We're currently loading two constants here (1.5 and -1.5), and using
-;        an qvfmadd instead of a qvfnmsubs
-; CHECK-DAG: qvfmadds
-; CHECK-DAG: qvfmadds
-; CHECK: qvfmuls
-; CHECK: qvfmuls
-; CHECK: blr
-
-; CHECK-SAFE-LABEL: @goo
-; CHECK-SAFE: fsqrts
-; CHECK-SAFE: fdivs
-; CHECK-SAFE: blr
 }
 
-define <4 x double> @foo2(<4 x double> %a, <4 x double> %b) nounwind {
+define <4 x double> @foo2_fmf(<4 x double> %a, <4 x double> %b) nounwind {
+; CHECK-LABEL: foo2_fmf:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addis 3, 2, .LCPI8_0@toc@ha
+; CHECK-NEXT:    qvfre 3, 2
+; CHECK-NEXT:    addi 3, 3, .LCPI8_0@toc@l
+; CHECK-NEXT:    qvlfdx 0, 0, 3
+; CHECK-NEXT:    qvfmadd 0, 2, 3, 0
+; CHECK-NEXT:    qvfnmsub 0, 3, 0, 3
+; CHECK-NEXT:    qvfmul 3, 1, 0
+; CHECK-NEXT:    qvfnmsub 1, 2, 3, 1
+; CHECK-NEXT:    qvfmadd 1, 0, 1, 3
+; CHECK-NEXT:    blr
 entry:
-  %r = fdiv <4 x double> %a, %b
+  %r = fdiv fast <4 x double> %a, %b
   ret <4 x double> %r
+}
 
-; CHECK-LABEL: @foo2
-; CHECK: qvfre
-; CHECK: qvfnmsub
-; CHECK: qvfmadd
-; CHECK: qvfnmsub
-; CHECK: qvfmadd
-; CHECK: qvfmul
-; CHECK: blr
+define <4 x double> @foo2_safe(<4 x double> %a, <4 x double> %b) nounwind {
+; CHECK-LABEL: foo2_safe:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    qvesplati 3, 2, 3
+; CHECK-NEXT:    qvesplati 4, 1, 3
+; CHECK-NEXT:    qvesplati 5, 2, 2
+; CHECK-NEXT:    qvgpci 0, 275
+; CHECK-NEXT:    fdiv 3, 4, 3
+; CHECK-NEXT:    qvesplati 4, 1, 2
+; CHECK-NEXT:    fdiv 4, 4, 5
+; CHECK-NEXT:    fdiv 5, 1, 2
+; CHECK-NEXT:    qvesplati 2, 2, 1
+; CHECK-NEXT:    qvesplati 1, 1, 1
+; CHECK-NEXT:    fdiv 1, 1, 2
+; CHECK-NEXT:    qvfperm 2, 4, 3, 0
+; CHECK-NEXT:    qvfperm 0, 5, 1, 0
+; CHECK-NEXT:    qvgpci 1, 101
+; CHECK-NEXT:    qvfperm 1, 0, 2, 1
+; CHECK-NEXT:    blr
+  %r = fdiv <4 x double> %a, %b
+  ret <4 x double> %r
+}
 
-; CHECK-SAFE-LABEL: @foo2
-; CHECK-SAFE: fdiv
-; CHECK-SAFE: blr
+define <4 x float> @goo2_fmf(<4 x float> %a, <4 x float> %b) nounwind {
+; CHECK-LABEL: goo2_fmf:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    qvfres 0, 2
+; CHECK-NEXT:    qvfmuls 3, 1, 0
+; CHECK-NEXT:    qvfnmsubs 1, 2, 3, 1
+; CHECK-NEXT:    qvfmadds 1, 0, 1, 3
+; CHECK-NEXT:    blr
+entry:
+  %r = fdiv fast <4 x float> %a, %b
+  ret <4 x float> %r
 }
 
-define <4 x float> @goo2(<4 x float> %a, <4 x float> %b) nounwind {
+define <4 x float> @goo2_safe(<4 x float> %a, <4 x float> %b) nounwind {
+; CHECK-LABEL: goo2_safe:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    qvesplati 3, 2, 3
+; CHECK-NEXT:    qvesplati 4, 1, 3
+; CHECK-NEXT:    qvesplati 5, 2, 2
+; CHECK-NEXT:    qvgpci 0, 275
+; CHECK-NEXT:    fdivs 3, 4, 3
+; CHECK-NEXT:    qvesplati 4, 1, 2
+; CHECK-NEXT:    fdivs 4, 4, 5
+; CHECK-NEXT:    fdivs 5, 1, 2
+; CHECK-NEXT:    qvesplati 2, 2, 1
+; CHECK-NEXT:    qvesplati 1, 1, 1
+; CHECK-NEXT:    fdivs 1, 1, 2
+; CHECK-NEXT:    qvfperm 2, 4, 3, 0
+; CHECK-NEXT:    qvfperm 0, 5, 1, 0
+; CHECK-NEXT:    qvgpci 1, 101
+; CHECK-NEXT:    qvfperm 1, 0, 2, 1
+; CHECK-NEXT:    blr
 entry:
   %r = fdiv <4 x float> %a, %b
   ret <4 x float> %r
+}
 
-; CHECK-LABEL: @goo2
-; CHECK: qvfres
-; CHECK: qvfnmsubs
-; CHECK: qvfmadds
-; CHECK: qvfmuls
-; CHECK: blr
-
-; CHECK-SAFE-LABEL: @goo2
-; CHECK-SAFE: fdivs
-; CHECK-SAFE: blr
+define <4 x double> @foo3_fmf(<4 x double> %a) nounwind {
+; CHECK-LABEL: foo3_fmf:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addis 3, 2, .LCPI12_0@toc@ha
+; CHECK-NEXT:    qvfrsqrte 0, 1
+; CHECK-NEXT:    addi 3, 3, .LCPI12_0@toc@l
+; CHECK-NEXT:    qvlfdx 2, 0, 3
+; CHECK-NEXT:    addis 3, 2, .LCPI12_1@toc@ha
+; CHECK-NEXT:    addi 3, 3, .LCPI12_1@toc@l
+; CHECK-NEXT:    qvfmul 3, 0, 0
+; CHECK-NEXT:    qvfmsub 4, 1, 2, 1
+; CHECK-NEXT:    qvfnmsub 3, 4, 3, 2
+; CHECK-NEXT:    qvfmul 0, 0, 3
+; CHECK-NEXT:    qvfmul 3, 0, 0
+; CHECK-NEXT:    qvfnmsub 2, 4, 3, 2
+; CHECK-NEXT:    qvfmul 0, 0, 2
+; CHECK-NEXT:    qvlfdx 2, 0, 3
+; CHECK-NEXT:    qvfmul 0, 0, 1
+; CHECK-NEXT:    qvfcmpeq 1, 1, 2
+; CHECK-NEXT:    qvfsel 1, 1, 2, 0
+; CHECK-NEXT:    blr
+entry:
+  %r = call fast <4 x double> @llvm.sqrt.v4f64(<4 x double> %a)
+  ret <4 x double> %r
 }
 
-define <4 x double> @foo3(<4 x double> %a) nounwind {
+define <4 x double> @foo3_safe(<4 x double> %a) nounwind {
+; CHECK-LABEL: foo3_safe:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    qvesplati 2, 1, 3
+; CHECK-NEXT:    qvesplati 3, 1, 2
+; CHECK-NEXT:    fsqrt 4, 1
+; CHECK-NEXT:    qvesplati 1, 1, 1
+; CHECK-NEXT:    fsqrt 2, 2
+; CHECK-NEXT:    fsqrt 3, 3
+; CHECK-NEXT:    fsqrt 1, 1
+; CHECK-NEXT:    qvgpci 0, 275
+; CHECK-NEXT:    qvfperm 2, 3, 2, 0
+; CHECK-NEXT:    qvfperm 0, 4, 1, 0
+; CHECK-NEXT:    qvgpci 1, 101
+; CHECK-NEXT:    qvfperm 1, 0, 2, 1
+; CHECK-NEXT:    blr
 entry:
   %r = call <4 x double> @llvm.sqrt.v4f64(<4 x double> %a)
   ret <4 x double> %r
+}
 
-; CHECK-LABEL: @foo3
-; CHECK: qvfrsqrte
-; CHECK: qvfmul
-; FIXME: We're currently loading two constants here (1.5 and -1.5), and using
-;        an qvfmadd instead of a qvfnmsub
-; CHECK-DAG: qvfmadd
-; CHECK-DAG: qvfcmpeq
-; CHECK-DAG: qvfmadd
-; CHECK-DAG: qvfmul
-; CHECK-DAG: qvfmul
-; CHECK-DAG: qvfmadd
-; CHECK-DAG: qvfmul
-; CHECK-DAG: qvfmul
-; CHECK: qvfsel
-; CHECK: blr
-
-; CHECK-SAFE-LABEL: @foo3
-; CHECK-SAFE: fsqrt
-; CHECK-SAFE: blr
-}
-
-define <4 x float> @goo3(<4 x float> %a) nounwind {
+define <4 x float> @goo3_fmf(<4 x float> %a) nounwind {
+; CHECK-LABEL: goo3_fmf:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addis 3, 2, .LCPI14_1@toc@ha
+; CHECK-NEXT:    qvfrsqrtes 2, 1
+; CHECK-NEXT:    addi 3, 3, .LCPI14_1@toc@l
+; CHECK-NEXT:    qvlfsx 0, 0, 3
+; CHECK-NEXT:    addis 3, 2, .LCPI14_0@toc@ha
+; CHECK-NEXT:    addi 3, 3, .LCPI14_0@toc@l
+; CHECK-NEXT:    qvfmuls 4, 2, 2
+; CHECK-NEXT:    qvfnmsubs 3, 1, 0, 1
+; CHECK-NEXT:    qvfmadds 0, 3, 4, 0
+; CHECK-NEXT:    qvlfsx 3, 0, 3
+; CHECK-NEXT:    qvfmuls 0, 2, 0
+; CHECK-NEXT:    qvfmuls 0, 0, 1
+; CHECK-NEXT:    qvfcmpeq 1, 1, 3
+; CHECK-NEXT:    qvfsel 1, 1, 3, 0
+; CHECK-NEXT:    blr
 entry:
-  %r = call <4 x float> @llvm.sqrt.v4f32(<4 x float> %a)
+  %r = call fast <4 x float> @llvm.sqrt.v4f32(<4 x float> %a)
   ret <4 x float> %r
+}
 
-; CHECK-LABEL: @goo3
-; CHECK: qvfrsqrtes
-; CHECK: qvfmuls
-; FIXME: We're currently loading two constants here (1.5 and -1.5), and using
-;        an qvfmadds instead of a qvfnmsubs
-; CHECK-DAG: qvfmadds
-; CHECK-DAG: qvfcmpeq
-; CHECK-DAG: qvfmadds
-; CHECK-DAG: qvfmuls
-; CHECK-DAG: qvfmuls
-; CHECK: qvfsel
-; CHECK: blr
-
-; CHECK-SAFE-LABEL: @goo3
-; CHECK-SAFE: fsqrts
-; CHECK-SAFE: blr
+define <4 x float> @goo3_safe(<4 x float> %a) nounwind {
+; CHECK-LABEL: goo3_safe:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    qvesplati 2, 1, 3
+; CHECK-NEXT:    qvesplati 3, 1, 2
+; CHECK-NEXT:    fsqrts 4, 1
+; CHECK-NEXT:    qvesplati 1, 1, 1
+; CHECK-NEXT:    fsqrts 2, 2
+; CHECK-NEXT:    fsqrts 3, 3
+; CHECK-NEXT:    fsqrts 1, 1
+; CHECK-NEXT:    qvgpci 0, 275
+; CHECK-NEXT:    qvfperm 2, 3, 2, 0
+; CHECK-NEXT:    qvfperm 0, 4, 1, 0
+; CHECK-NEXT:    qvgpci 1, 101
+; CHECK-NEXT:    qvfperm 1, 0, 2, 1
+; CHECK-NEXT:    blr
+entry:
+  %r = call <4 x float> @llvm.sqrt.v4f32(<4 x float> %a)
+  ret <4 x float> %r
 }
 
diff --git a/llvm/test/CodeGen/PowerPC/recipest.ll b/llvm/test/CodeGen/PowerPC/recipest.ll
index b71918751..a0afb4b6e 100644
--- a/llvm/test/CodeGen/PowerPC/recipest.ll
+++ b/llvm/test/CodeGen/PowerPC/recipest.ll
@@ -1,5 +1,4 @@
-; RUN: llc -verify-machineinstrs < %s -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr7 -enable-unsafe-fp-math -mattr=-vsx | FileCheck %s
-; RUN: llc -verify-machineinstrs < %s -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr7 -mattr=-vsx | FileCheck -check-prefix=CHECK-SAFE %s
+; RUN: llc -verify-machineinstrs < %s -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr7 -mattr=-vsx | FileCheck %s
 
 target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
 target triple = "powerpc64-unknown-linux-gnu"
@@ -8,12 +7,8 @@ declare double @llvm.sqrt.f64(double)
 declare float @llvm.sqrt.f32(float)
 declare <4 x float> @llvm.sqrt.v4f32(<4 x float>)
 
-define double @foo(double %a, double %b) nounwind {
-  %x = call double @llvm.sqrt.f64(double %b)
-  %r = fdiv double %a, %x
-  ret double %r
-
-; CHECK: @foo
+define double @foo_fmf(double %a, double %b) nounwind {
+; CHECK: @foo_fmf
 ; CHECK: frsqrte
 ; CHECK: fmul
 ; CHECK-NEXT: fmadd
@@ -25,34 +20,35 @@ define double @foo(double %a, double %b) nounwind {
 ; CHECK-NEXT: fmul
 ; CHECK-NEXT: fmul
 ; CHECK: blr
-
-; CHECK-SAFE: @foo
-; CHECK-SAFE: fsqrt
-; CHECK-SAFE: fdiv
-; CHECK-SAFE: blr
+  %x = call fast double @llvm.sqrt.f64(double %b)
+  %r = fdiv fast double %a, %x
+  ret double %r
 }
 
-define double @no_estimate_refinement_f64(double %a, double %b) #0 {
+define double @foo_safe(double %a, double %b) nounwind {
+; CHECK: @foo_safe
+; CHECK: fsqrt
+; CHECK: fdiv
+; CHECK: blr
   %x = call double @llvm.sqrt.f64(double %b)
   %r = fdiv double %a, %x
   ret double %r
+}
 
+define double @no_estimate_refinement_f64(double %a, double %b) #0 {
 ; CHECK-LABEL: @no_estimate_refinement_f64
 ; CHECK: frsqrte
 ; CHECK-NOT: fmadd
 ; CHECK: fmul
 ; CHECK-NOT: fmadd
 ; CHECK: blr
-}
-
-
-define double @foof(double %a, float %b) nounwind {
-  %x = call float @llvm.sqrt.f32(float %b)
-  %y = fpext float %x to double
-  %r = fdiv double %a, %y
+  %x = call fast double @llvm.sqrt.f64(double %b)
+  %r = fdiv fast double %a, %x
   ret double %r
+}
 
-; CHECK: @foof
+define double @foof_fmf(double %a, float %b) nounwind {
+; CHECK: @foof_fmf
 ; CHECK-DAG: frsqrtes
 ; CHECK: fmuls
 ; CHECK-NEXT: fmadds
@@ -60,20 +56,25 @@ define double @foof(double %a, float %b) nounwind {
 ; CHECK-NEXT: fmuls
 ; CHECK-NEXT: fmul
 ; CHECK-NEXT: blr
-
-; CHECK-SAFE: @foof
-; CHECK-SAFE: fsqrts
-; CHECK-SAFE: fdiv
-; CHECK-SAFE: blr
+  %x = call fast float @llvm.sqrt.f32(float %b)
+  %y = fpext float %x to double
+  %r = fdiv fast double %a, %y
+  ret double %r
 }
 
-define float @food(float %a, double %b) nounwind {
-  %x = call double @llvm.sqrt.f64(double %b)
-  %y = fptrunc double %x to float
-  %r = fdiv float %a, %y
-  ret float %r
+define double @foof_safe(double %a, float %b) nounwind {
+; CHECK: @foof_safe
+; CHECK: fsqrts
+; CHECK: fdiv
+; CHECK: blr
+  %x = call float @llvm.sqrt.f32(float %b)
+  %y = fpext float %x to double
+  %r = fdiv double %a, %y
+  ret double %r
+}
 
-; CHECK: @foo
+define float @food_fmf(float %a, double %b) nounwind {
+; CHECK: @food_fmf
 ; CHECK-DAG: frsqrte
 ; CHECK: fmul
 ; CHECK-NEXT: fmadd
@@ -86,19 +87,25 @@ define float @food(float %a, double %b) nounwind {
 ; CHECK-NEXT: frsp
 ; CHECK-NEXT: fmuls
 ; CHECK-NEXT: blr
-
-; CHECK-SAFE: @foo
-; CHECK-SAFE: fsqrt
-; CHECK-SAFE: fdivs
-; CHECK-SAFE: blr
+  %x = call fast double @llvm.sqrt.f64(double %b)
+  %y = fptrunc double %x to float
+  %r = fdiv fast float %a, %y
+  ret float %r
 }
 
-define float @goo(float %a, float %b) nounwind {
-  %x = call float @llvm.sqrt.f32(float %b)
-  %r = fdiv float %a, %x
+define float @food_safe(float %a, double %b) nounwind {
+; CHECK: @food_safe
+; CHECK: fsqrt
+; CHECK: fdivs
+; CHECK: blr
+  %x = call double @llvm.sqrt.f64(double %b)
+  %y = fptrunc double %x to float
+  %r = fdiv float %a, %y
   ret float %r
+}
 
-; CHECK: @goo
+define float @goo_fmf(float %a, float %b) nounwind {
+; CHECK: @goo_fmf
 ; CHECK-DAG: frsqrtes
 ; CHECK: fmuls
 ; CHECK-NEXT: fmadds
@@ -106,118 +113,141 @@ define float @goo(float %a, float %b) nounwind {
 ; CHECK-NEXT: fmuls
 ; CHECK-NEXT: fmuls
 ; CHECK-NEXT: blr
-
-; CHECK-SAFE: @goo
-; CHECK-SAFE: fsqrts
-; CHECK-SAFE: fdivs
-; CHECK-SAFE: blr
+  %x = call fast float @llvm.sqrt.f32(float %b)
+  %r = fdiv fast float %a, %x
+  ret float %r
 }
 
-
-define float @no_estimate_refinement_f32(float %a, float %b) #0 {
+define float @goo_safe(float %a, float %b) nounwind {
+; CHECK: @goo_safe
+; CHECK: fsqrts
+; CHECK: fdivs
+; CHECK: blr
   %x = call float @llvm.sqrt.f32(float %b)
   %r = fdiv float %a, %x
   ret float %r
+}
 
+define float @no_estimate_refinement_f32(float %a, float %b) #0 {
 ; CHECK-LABEL: @no_estimate_refinement_f32
 ; CHECK: frsqrtes
 ; CHECK-NOT: fmadds
 ; CHECK: fmuls
 ; CHECK-NOT: fmadds
 ; CHECK: blr
+  %x = call fast float @llvm.sqrt.f32(float %b)
+  %r = fdiv fast float %a, %x
+  ret float %r
 }
 
 ; Recognize that this is rsqrt(a) * rcp(b) * c, 
 ; not 1 / ( 1 / sqrt(a)) * rcp(b) * c.
-define float @rsqrt_fmul(float %a, float %b, float %c) {
-  %x = call float @llvm.sqrt.f32(float %a)
-  %y = fmul float %x, %b 
-  %z = fdiv float %c, %y
-  ret float %z
-
-; CHECK: @rsqrt_fmul
+define float @rsqrt_fmul_fmf(float %a, float %b, float %c) {
+; CHECK: @rsqrt_fmul_fmf
 ; CHECK-DAG: frsqrtes
-; CHECK-DAG: fres
-; CHECK-DAG: fnmsubs
-; CHECK-DAG: fmuls
-; CHECK-DAG: fmadds
-; CHECK-DAG: fmadds
 ; CHECK: fmuls
+; CHECK-NEXT: fmadds
 ; CHECK-NEXT: fmuls
+; CHECK-DAG: fres
+; CHECK-COUNT-3: fmuls
+; CHECK-NEXT: fmsubs
+; CHECK-NEXT: fmadds
 ; CHECK-NEXT: fmuls
 ; CHECK-NEXT: blr
+  %x = call fast float @llvm.sqrt.f32(float %a)
+  %y = fmul fast float %x, %b 
+  %z = fdiv fast float %c, %y
+  ret float %z
+}
 
-; CHECK-SAFE: @rsqrt_fmul
-; CHECK-SAFE: fsqrts
-; CHECK-SAFE: fmuls
-; CHECK-SAFE: fdivs
-; CHECK-SAFE: blr
+; Recognize that this is rsqrt(a) * rcp(b) * c, 
+; not 1 / ( 1 / sqrt(a)) * rcp(b) * c.
+define float @rsqrt_fmul_safe(float %a, float %b, float %c) {
+; CHECK: @rsqrt_fmul_safe
+; CHECK: fsqrts
+; CHECK: fmuls
+; CHECK: fdivs
+; CHECK: blr
+  %x = call float @llvm.sqrt.f32(float %a)
+  %y = fmul float %x, %b 
+  %z = fdiv float %c, %y
+  ret float %z
+}
+
+define <4 x float> @hoo_fmf(<4 x float> %a, <4 x float> %b) nounwind {
+; CHECK: @hoo_fmf
+; CHECK: vrsqrtefp
+  %x = call fast <4 x float> @llvm.sqrt.v4f32(<4 x float> %b)
+  %r = fdiv fast <4 x float> %a, %x
+  ret <4 x float> %r
 }
 
-define <4 x float> @hoo(<4 x float> %a, <4 x float> %b) nounwind {
+define <4 x float> @hoo_safe(<4 x float> %a, <4 x float> %b) nounwind {
+; CHECK: @hoo_safe
+; CHECK-NOT: vrsqrtefp
+; CHECK: blr
   %x = call <4 x float> @llvm.sqrt.v4f32(<4 x float> %b)
   %r = fdiv <4 x float> %a, %x
   ret <4 x float> %r
-
-; CHECK: @hoo
-; CHECK: vrsqrtefp
-
-; CHECK-SAFE: @hoo
-; CHECK-SAFE-NOT: vrsqrtefp
-; CHECK-SAFE: blr
 }
 
-define double @foo2(double %a, double %b) nounwind {
-  %r = fdiv double %a, %b
-  ret double %r
-
-; CHECK: @foo2
+define double @foo2_fmf(double %a, double %b) nounwind {
+; CHECK: @foo2_fmf
 ; CHECK-DAG: fre
-; CHECK-DAG: fnmsub
-; CHECK: fmadd
+; CHECK-DAG: fmadd
+; CHECK: fnmsub
+; CHECK-NEXT: fmul
 ; CHECK-NEXT: fnmsub
 ; CHECK-NEXT: fmadd
-; CHECK-NEXT: fmul
 ; CHECK-NEXT: blr
-
-; CHECK-SAFE: @foo2
-; CHECK-SAFE: fdiv
-; CHECK-SAFE: blr
+  %r = fdiv fast double %a, %b
+  ret double %r
 }
 
-define float @goo2(float %a, float %b) nounwind {
-  %r = fdiv float %a, %b
-  ret float %r
+define double @foo2_safe(double %a, double %b) nounwind {
+; CHECK: @foo2_safe
+; CHECK: fdiv
+; CHECK: blr
+  %r = fdiv double %a, %b
+  ret double %r
+}
 
-; CHECK: @goo2
+define float @goo2_fmf(float %a, float %b) nounwind {
+; CHECK: @goo2_fmf
 ; CHECK-DAG: fres
+; CHECK-NEXT: fmuls
 ; CHECK-DAG: fnmsubs
 ; CHECK: fmadds
-; CHECK-NEXT: fmuls
 ; CHECK-NEXT: blr
-
-; CHECK-SAFE: @goo2
-; CHECK-SAFE: fdivs
-; CHECK-SAFE: blr
+  %r = fdiv fast float %a, %b
+  ret float %r
 }
 
-define <4 x float> @hoo2(<4 x float> %a, <4 x float> %b) nounwind {
-  %r = fdiv <4 x float> %a, %b
-  ret <4 x float> %r
+define float @goo2_safe(float %a, float %b) nounwind {
+; CHECK: @goo2_safe
+; CHECK: fdivs
+; CHECK: blr
+  %r = fdiv float %a, %b
+  ret float %r
+}
 
-; CHECK: @hoo2
+define <4 x float> @hoo2_fmf(<4 x float> %a, <4 x float> %b) nounwind {
+; CHECK: @hoo2_fmf
 ; CHECK: vrefp
-
-; CHECK-SAFE: @hoo2
-; CHECK-SAFE-NOT: vrefp
-; CHECK-SAFE: blr
+  %r = fdiv fast <4 x float> %a, %b
+  ret <4 x float> %r
 }
 
-define double @foo3(double %a) nounwind {
-  %r = call double @llvm.sqrt.f64(double %a)
-  ret double %r
+define <4 x float> @hoo2_safe(<4 x float> %a, <4 x float> %b) nounwind {
+; CHECK: @hoo2_safe
+; CHECK-NOT: vrefp
+; CHECK: blr
+  %r = fdiv <4 x float> %a, %b
+  ret <4 x float> %r
+}
 
-; CHECK: @foo3
+define double @foo3_fmf(double %a) nounwind {
+; CHECK: @foo3_fmf
 ; CHECK: fcmpu
 ; CHECK-DAG: frsqrte
 ; CHECK: fmul
@@ -229,17 +259,20 @@ define double @foo3(double %a) nounwind {
 ; CHECK-NEXT: fmul
 ; CHECK-NEXT: fmul
 ; CHECK: blr
-
-; CHECK-SAFE: @foo3
-; CHECK-SAFE: fsqrt
-; CHECK-SAFE: blr
+  %r = call fast double @llvm.sqrt.f64(double %a)
+  ret double %r
 }
 
-define float @goo3(float %a) nounwind {
-  %r = call float @llvm.sqrt.f32(float %a)
-  ret float %r
+define double @foo3_safe(double %a) nounwind {
+; CHECK: @foo3_safe
+; CHECK: fsqrt
+; CHECK: blr
+  %r = call double @llvm.sqrt.f64(double %a)
+  ret double %r
+}
 
-; CHECK: @goo3
+define float @goo3_fmf(float %a) nounwind {
+; CHECK: @goo3_fmf
 ; CHECK: fcmpu
 ; CHECK-DAG: frsqrtes
 ; CHECK: fmuls
@@ -247,24 +280,32 @@ define float @goo3(float %a) nounwind {
 ; CHECK-NEXT: fmuls
 ; CHECK-NEXT: fmuls
 ; CHECK: blr
-
-; CHECK-SAFE: @goo3
-; CHECK-SAFE: fsqrts
-; CHECK-SAFE: blr
+  %r = call fast float @llvm.sqrt.f32(float %a)
+  ret float %r
 }
 
-define <4 x float> @hoo3(<4 x float> %a) nounwind {
-  %r = call <4 x float> @llvm.sqrt.v4f32(<4 x float> %a)
-  ret <4 x float> %r
+define float @goo3_safe(float %a) nounwind {
+; CHECK: @goo3_safe
+; CHECK: fsqrts
+; CHECK: blr
+  %r = call float @llvm.sqrt.f32(float %a)
+  ret float %r
+}
 
-; CHECK: @hoo3
+define <4 x float> @hoo3_fmf(<4 x float> %a) nounwind {
+; CHECK: @hoo3_fmf
 ; CHECK: vrsqrtefp
 ; CHECK-DAG: vcmpeqfp
+  %r = call fast <4 x float> @llvm.sqrt.v4f32(<4 x float> %a)
+  ret <4 x float> %r
+}
 
-; CHECK-SAFE: @hoo3
-; CHECK-SAFE-NOT: vrsqrtefp
-; CHECK-SAFE: blr
+define <4 x float> @hoo3_safe(<4 x float> %a) nounwind {
+; CHECK: @hoo3_safe
+; CHECK-NOT: vrsqrtefp
+; CHECK: blr
+  %r = call <4 x float> @llvm.sqrt.v4f32(<4 x float> %a)
+  ret <4 x float> %r
 }
 
 attributes #0 = { nounwind "reciprocal-estimates"="sqrtf:0,sqrtd:0" }
-
diff --git a/llvm/test/CodeGen/PowerPC/reduce_scalarization02.ll b/llvm/test/CodeGen/PowerPC/reduce_scalarization02.ll
new file mode 100644
index 000000000..f7727d6f4
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/reduce_scalarization02.ll
@@ -0,0 +1,87 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-unknown \
+; RUN:     -mcpu=pwr9 -ppc-asm-full-reg-names \
+; RUN:     -ppc-vsr-nums-as-vr < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64-unknown-unknown \
+; RUN:     -mcpu=pwr9 -ppc-asm-full-reg-names \
+; RUN:     -ppc-vsr-nums-as-vr < %s | FileCheck %s --check-prefix=CHECK-BE
+
+; Test reduce scalarization in fpext v2f32 to v2f64 from the extract_subvector v4f32 node.
+
+define dso_local void @test(<4 x float>* nocapture readonly %a, <2 x double>* nocapture %b, <2 x double>* nocapture %c) {
+; CHECK-LABEL: test:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lxv vs0, 0(r3)
+; CHECK-NEXT:    xxmrglw vs1, vs0, vs0
+; CHECK-NEXT:    xxmrghw vs0, vs0, vs0
+; CHECK-NEXT:    xvcvspdp vs1, vs1
+; CHECK-NEXT:    xvcvspdp vs0, vs0
+; CHECK-NEXT:    stxv vs1, 0(r4)
+; CHECK-NEXT:    stxv vs0, 0(r5)
+; CHECK-NEXT:    blr
+;
+; CHECK-BE-LABEL: test:
+; CHECK-BE:       # %bb.0: # %entry
+; CHECK-BE-NEXT:    lxv vs0, 0(r3)
+; CHECK-BE-NEXT:    xxmrghw vs1, vs0, vs0
+; CHECK-BE-NEXT:    xxmrglw vs0, vs0, vs0
+; CHECK-BE-NEXT:    xvcvspdp vs1, vs1
+; CHECK-BE-NEXT:    xvcvspdp vs0, vs0
+; CHECK-BE-NEXT:    stxv vs1, 0(r4)
+; CHECK-BE-NEXT:    stxv vs0, 0(r5)
+; CHECK-BE-NEXT:    blr
+entry:
+  %0 = load <4 x float>, <4 x float>* %a, align 16
+  %shuffle = shufflevector <4 x float> %0, <4 x float> undef, <2 x i32> <i32 0, i32 1>
+  %shuffle1 = shufflevector <4 x float> %0, <4 x float> undef, <2 x i32> <i32 2, i32 3>
+  %vecinit4 = fpext <2 x float> %shuffle to <2 x double>
+  %vecinit11 = fpext <2 x float> %shuffle1 to <2 x double>
+  store <2 x double> %vecinit4, <2 x double>* %b, align 16
+  store <2 x double> %vecinit11, <2 x double>* %c, align 16
+  ret void
+}
+
+; Ensure we don't crash for wider types
+
+define dso_local void @test2(<16 x float>* nocapture readonly %a, <2 x double>* nocapture %b, <2 x double>* nocapture %c) {
+; CHECK-LABEL: test2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lxv vs0, 0(r3)
+; CHECK-NEXT:    xxsldwi vs1, vs0, vs0, 1
+; CHECK-NEXT:    xscvspdpn f2, vs0
+; CHECK-NEXT:    xxsldwi vs3, vs0, vs0, 3
+; CHECK-NEXT:    xxswapd vs0, vs0
+; CHECK-NEXT:    xscvspdpn f1, vs1
+; CHECK-NEXT:    xscvspdpn f3, vs3
+; CHECK-NEXT:    xscvspdpn f0, vs0
+; CHECK-NEXT:    xxmrghd vs0, vs0, vs3
+; CHECK-NEXT:    xxmrghd vs1, vs2, vs1
+; CHECK-NEXT:    stxv vs0, 0(r4)
+; CHECK-NEXT:    stxv vs1, 0(r5)
+; CHECK-NEXT:    blr
+;
+; CHECK-BE-LABEL: test2:
+; CHECK-BE:       # %bb.0: # %entry
+; CHECK-BE-NEXT:    lxv vs0, 0(r3)
+; CHECK-BE-NEXT:    xxswapd vs1, vs0
+; CHECK-BE-NEXT:    xxsldwi vs2, vs0, vs0, 3
+; CHECK-BE-NEXT:    xscvspdpn f3, vs0
+; CHECK-BE-NEXT:    xxsldwi vs0, vs0, vs0, 1
+; CHECK-BE-NEXT:    xscvspdpn f1, vs1
+; CHECK-BE-NEXT:    xscvspdpn f2, vs2
+; CHECK-BE-NEXT:    xscvspdpn f0, vs0
+; CHECK-BE-NEXT:    xxmrghd vs0, vs3, vs0
+; CHECK-BE-NEXT:    xxmrghd vs1, vs1, vs2
+; CHECK-BE-NEXT:    stxv vs0, 0(r4)
+; CHECK-BE-NEXT:    stxv vs1, 0(r5)
+; CHECK-BE-NEXT:    blr
+entry:
+  %0 = load <16 x float>, <16 x float>* %a, align 16
+  %shuffle = shufflevector <16 x float> %0, <16 x float> undef, <2 x i32> <i32 0, i32 1>
+  %shuffle1 = shufflevector <16 x float> %0, <16 x float> undef, <2 x i32> <i32 2, i32 3>
+  %vecinit4 = fpext <2 x float> %shuffle to <2 x double>
+  %vecinit11 = fpext <2 x float> %shuffle1 to <2 x double>
+  store <2 x double> %vecinit4, <2 x double>* %b, align 16
+  store <2 x double> %vecinit11, <2 x double>* %c, align 16
+  ret void
+}
diff --git a/llvm/test/CodeGen/PowerPC/redundant-copy-after-tail-dup.ll b/llvm/test/CodeGen/PowerPC/redundant-copy-after-tail-dup.ll
index 6aaf169da..dd41abd09 100644
--- a/llvm/test/CodeGen/PowerPC/redundant-copy-after-tail-dup.ll
+++ b/llvm/test/CodeGen/PowerPC/redundant-copy-after-tail-dup.ll
@@ -26,8 +26,7 @@ define dso_local i1 @t(%class.A* %this, i32 %color, i32 %vertex) local_unnamed_a
 ; CHECK-P9-NEXT:    cmplwi r3, 2
 ; CHECK-P9-NEXT:    bge- cr0, .LBB0_6
 ; CHECK-P9-NEXT:  # %bb.3: # %land.lhs.true.1
-; CHECK-P9-NEXT:    li r5, 0
-; CHECK-P9-NEXT:    mr r3, r5
+; CHECK-P9-NEXT:    li r3, 0
 ; CHECK-P9-NEXT:    blr
 ; CHECK-P9-NEXT:  .LBB0_4: # %lor.lhs.false
 ; CHECK-P9-NEXT:    cmplwi cr0, r4, 0
diff --git a/llvm/test/CodeGen/PowerPC/reloc-align.ll b/llvm/test/CodeGen/PowerPC/reloc-align.ll
index 090d072f0..59f7137fd 100644
--- a/llvm/test/CodeGen/PowerPC/reloc-align.ll
+++ b/llvm/test/CodeGen/PowerPC/reloc-align.ll
@@ -31,4 +31,4 @@ entry:
   ret i32 %bf.cast
 }
 
-attributes #0 = { nounwind readonly "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind readonly "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/PowerPC/remove-redundant-load-imm.ll b/llvm/test/CodeGen/PowerPC/remove-redundant-load-imm.ll
new file mode 100644
index 000000000..122f9f805
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/remove-redundant-load-imm.ll
@@ -0,0 +1,51 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mcpu=pwr9 -O3 < %s | FileCheck %s -check-prefix=PPC64LE
+
+target datalayout = "e-m:e-i64:64-n32:64"
+target triple = "powerpc64le-unknown-linux-gnu"
+
+@global.6 = external global i32*
+
+declare void @barney.88(i1, i32*)
+declare void @barney.94(i8*, i32)
+
+define void @redundancy_on_ppc_only(i1 %arg7) nounwind {
+; PPC64LE-LABEL: redundancy_on_ppc_only:
+; PPC64LE:       # %bb.0: # %bb
+; PPC64LE-NEXT:    mflr 0
+; PPC64LE-NEXT:    andi. 3, 3, 1
+; PPC64LE-NEXT:    std 0, 16(1)
+; PPC64LE-NEXT:    stdu 1, -32(1)
+; PPC64LE-NEXT:    li 3, 1
+; PPC64LE-NEXT:    li 4, 0
+; PPC64LE-NEXT:    isel 3, 3, 4, 1
+; PPC64LE-NEXT:    bl barney.88
+; PPC64LE-NEXT:    nop
+; PPC64LE-NEXT:    addi 1, 1, 32
+; PPC64LE-NEXT:    ld 0, 16(1)
+; PPC64LE-NEXT:    mtlr 0
+; PPC64LE-NEXT:    blr
+bb:
+  br label %bb10
+
+bb10:                                             ; preds = %bb
+  call void @barney.88(i1 %arg7, i32* null)
+  ret void
+}
+
+define void @redundancy_on_ppc_and_other_targets() nounwind {
+; PPC64LE-LABEL: redundancy_on_ppc_and_other_targets:
+; PPC64LE:       # %bb.0:
+; PPC64LE-NEXT:    mflr 0
+; PPC64LE-NEXT:    std 0, 16(1)
+; PPC64LE-NEXT:    stdu 1, -32(1)
+; PPC64LE-NEXT:    addis 3, 2, .LC0@toc@ha
+; PPC64LE-NEXT:    ld 3, .LC0@toc@l(3)
+; PPC64LE-NEXT:    li 4, 0
+; PPC64LE-NEXT:    std 4, 0(3)
+; PPC64LE-NEXT:    bl barney.94
+; PPC64LE-NEXT:    nop
+  store i32* null, i32** @global.6
+  call void @barney.94(i8* undef, i32 0)
+  unreachable
+}
diff --git a/llvm/test/CodeGen/PowerPC/repeated-fp-divisors.ll b/llvm/test/CodeGen/PowerPC/repeated-fp-divisors.ll
index bb30a7737..2cdf83283 100644
--- a/llvm/test/CodeGen/PowerPC/repeated-fp-divisors.ll
+++ b/llvm/test/CodeGen/PowerPC/repeated-fp-divisors.ll
@@ -13,9 +13,9 @@ define <4 x float> @repeated_fp_divisor(float %a, <4 x float> %b) {
 ; CHECK-NEXT:    lvx 4, 0, 3
 ; CHECK-NEXT:    xxspltw 0, 0, 0
 ; CHECK-NEXT:    xvresp 1, 0
-; CHECK-NEXT:    xvnmsubasp 35, 1, 0
+; CHECK-NEXT:    xvmaddasp 35, 0, 1
 ; CHECK-NEXT:    xvmulsp 0, 34, 36
-; CHECK-NEXT:    xvmaddasp 1, 1, 35
+; CHECK-NEXT:    xvnmsubasp 1, 1, 35
 ; CHECK-NEXT:    xvmulsp 34, 0, 1
 ; CHECK-NEXT:    blr
   %ins = insertelement <4 x float> undef, float %a, i32 0
diff --git a/llvm/test/CodeGen/PowerPC/rounding-ops.ll b/llvm/test/CodeGen/PowerPC/rounding-ops.ll
index 206a99a36..c4ad110eb 100644
--- a/llvm/test/CodeGen/PowerPC/rounding-ops.ll
+++ b/llvm/test/CodeGen/PowerPC/rounding-ops.ll
@@ -10,7 +10,7 @@ define float @test1(float %x) nounwind  {
 ; CHECK-LABEL: test1:
 ; CHECK: frim 1, 1
 ; CHECK-VSX-LABEL: test1:
-; CHECK-VSX: frim 1, 1
+; CHECK-VSX: xsrdpim 1, 1
 }
 
 declare float @floorf(float) nounwind readnone
@@ -34,7 +34,7 @@ define float @test3(float %x) nounwind  {
 ; CHECK-LABEL: test3:
 ; CHECK: frin 1, 1
 ; CHECK-VSX-LABEL: test3:
-; CHECK-VSX: frin 1, 1
+; CHECK-VSX: xsrdpi 1, 1
 }
 
 declare float @roundf(float) nounwind readnone
@@ -58,7 +58,7 @@ define float @test5(float %x) nounwind  {
 ; CHECK-LABEL: test5:
 ; CHECK: frip 1, 1
 ; CHECK-VSX-LABEL: test5:
-; CHECK-VSX: frip 1, 1
+; CHECK-VSX: xsrdpip 1, 1
 }
 
 declare float @ceilf(float) nounwind readnone
@@ -82,7 +82,7 @@ define float @test9(float %x) nounwind  {
 ; CHECK-LABEL: test9:
 ; CHECK: friz 1, 1
 ; CHECK-VSX-LABEL: test9:
-; CHECK-VSX: friz 1, 1
+; CHECK-VSX: xsrdpiz 1, 1
 }
 
 declare float @truncf(float) nounwind readnone
diff --git a/llvm/test/CodeGen/PowerPC/sat-add.ll b/llvm/test/CodeGen/PowerPC/sat-add.ll
index 2ebbc6246..932a3f786 100644
--- a/llvm/test/CodeGen/PowerPC/sat-add.ll
+++ b/llvm/test/CodeGen/PowerPC/sat-add.ll
@@ -396,12 +396,12 @@ define <16 x i8> @unsigned_sat_constant_v16i8_using_cmp_sum(<16 x i8> %x) {
 ; CHECK-LABEL: unsigned_sat_constant_v16i8_using_cmp_sum:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addis 3, 2, .LCPI25_0@toc@ha
-; CHECK-NEXT:    vspltisb 4, -1
+; CHECK-NEXT:    xxleqv 0, 0, 0
 ; CHECK-NEXT:    addi 3, 3, .LCPI25_0@toc@l
 ; CHECK-NEXT:    lvx 3, 0, 3
 ; CHECK-NEXT:    vaddubm 3, 2, 3
 ; CHECK-NEXT:    vcmpgtub 2, 2, 3
-; CHECK-NEXT:    xxsel 34, 35, 36, 34
+; CHECK-NEXT:    xxsel 34, 35, 0, 34
 ; CHECK-NEXT:    blr
   %a = add <16 x i8> %x, <i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42>
   %c = icmp ugt <16 x i8> %x, %a
@@ -413,7 +413,7 @@ define <16 x i8> @unsigned_sat_constant_v16i8_using_cmp_notval(<16 x i8> %x) {
 ; CHECK-LABEL: unsigned_sat_constant_v16i8_using_cmp_notval:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addis 3, 2, .LCPI26_1@toc@ha
-; CHECK-NEXT:    vspltisb 5, -1
+; CHECK-NEXT:    xxleqv 0, 0, 0
 ; CHECK-NEXT:    addi 3, 3, .LCPI26_1@toc@l
 ; CHECK-NEXT:    lvx 3, 0, 3
 ; CHECK-NEXT:    addis 3, 2, .LCPI26_0@toc@ha
@@ -421,7 +421,7 @@ define <16 x i8> @unsigned_sat_constant_v16i8_using_cmp_notval(<16 x i8> %x) {
 ; CHECK-NEXT:    vcmpgtub 3, 2, 3
 ; CHECK-NEXT:    lvx 4, 0, 3
 ; CHECK-NEXT:    vaddubm 2, 2, 4
-; CHECK-NEXT:    xxsel 34, 34, 37, 35
+; CHECK-NEXT:    xxsel 34, 34, 0, 35
 ; CHECK-NEXT:    blr
   %a = add <16 x i8> %x, <i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42>
   %c = icmp ugt <16 x i8> %x, <i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43>
@@ -451,12 +451,12 @@ define <8 x i16> @unsigned_sat_constant_v8i16_using_cmp_sum(<8 x i16> %x) {
 ; CHECK-LABEL: unsigned_sat_constant_v8i16_using_cmp_sum:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addis 3, 2, .LCPI28_0@toc@ha
-; CHECK-NEXT:    vspltisb 4, -1
+; CHECK-NEXT:    xxleqv 0, 0, 0
 ; CHECK-NEXT:    addi 3, 3, .LCPI28_0@toc@l
 ; CHECK-NEXT:    lvx 3, 0, 3
 ; CHECK-NEXT:    vadduhm 3, 2, 3
 ; CHECK-NEXT:    vcmpgtuh 2, 2, 3
-; CHECK-NEXT:    xxsel 34, 35, 36, 34
+; CHECK-NEXT:    xxsel 34, 35, 0, 34
 ; CHECK-NEXT:    blr
   %a = add <8 x i16> %x, <i16 42, i16 42, i16 42, i16 42, i16 42, i16 42, i16 42, i16 42>
   %c = icmp ugt <8 x i16> %x, %a
@@ -468,7 +468,7 @@ define <8 x i16> @unsigned_sat_constant_v8i16_using_cmp_notval(<8 x i16> %x) {
 ; CHECK-LABEL: unsigned_sat_constant_v8i16_using_cmp_notval:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addis 3, 2, .LCPI29_1@toc@ha
-; CHECK-NEXT:    vspltisb 5, -1
+; CHECK-NEXT:    xxleqv 0, 0, 0
 ; CHECK-NEXT:    addi 3, 3, .LCPI29_1@toc@l
 ; CHECK-NEXT:    lvx 3, 0, 3
 ; CHECK-NEXT:    addis 3, 2, .LCPI29_0@toc@ha
@@ -476,7 +476,7 @@ define <8 x i16> @unsigned_sat_constant_v8i16_using_cmp_notval(<8 x i16> %x) {
 ; CHECK-NEXT:    vcmpgtuh 3, 2, 3
 ; CHECK-NEXT:    lvx 4, 0, 3
 ; CHECK-NEXT:    vadduhm 2, 2, 4
-; CHECK-NEXT:    xxsel 34, 34, 37, 35
+; CHECK-NEXT:    xxsel 34, 34, 0, 35
 ; CHECK-NEXT:    blr
   %a = add <8 x i16> %x, <i16 42, i16 42, i16 42, i16 42, i16 42, i16 42, i16 42, i16 42>
   %c = icmp ugt <8 x i16> %x, <i16 -43, i16 -43, i16 -43, i16 -43, i16 -43, i16 -43, i16 -43, i16 -43>
@@ -506,12 +506,12 @@ define <4 x i32> @unsigned_sat_constant_v4i32_using_cmp_sum(<4 x i32> %x) {
 ; CHECK-LABEL: unsigned_sat_constant_v4i32_using_cmp_sum:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addis 3, 2, .LCPI31_0@toc@ha
-; CHECK-NEXT:    vspltisb 4, -1
+; CHECK-NEXT:    xxleqv 0, 0, 0
 ; CHECK-NEXT:    addi 3, 3, .LCPI31_0@toc@l
 ; CHECK-NEXT:    lvx 3, 0, 3
 ; CHECK-NEXT:    vadduwm 3, 2, 3
 ; CHECK-NEXT:    vcmpgtuw 2, 2, 3
-; CHECK-NEXT:    xxsel 34, 35, 36, 34
+; CHECK-NEXT:    xxsel 34, 35, 0, 34
 ; CHECK-NEXT:    blr
   %a = add <4 x i32> %x, <i32 42, i32 42, i32 42, i32 42>
   %c = icmp ugt <4 x i32> %x, %a
@@ -523,7 +523,7 @@ define <4 x i32> @unsigned_sat_constant_v4i32_using_cmp_notval(<4 x i32> %x) {
 ; CHECK-LABEL: unsigned_sat_constant_v4i32_using_cmp_notval:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addis 3, 2, .LCPI32_1@toc@ha
-; CHECK-NEXT:    vspltisb 5, -1
+; CHECK-NEXT:    xxleqv 0, 0, 0
 ; CHECK-NEXT:    addi 3, 3, .LCPI32_1@toc@l
 ; CHECK-NEXT:    lvx 3, 0, 3
 ; CHECK-NEXT:    addis 3, 2, .LCPI32_0@toc@ha
@@ -531,7 +531,7 @@ define <4 x i32> @unsigned_sat_constant_v4i32_using_cmp_notval(<4 x i32> %x) {
 ; CHECK-NEXT:    vcmpgtuw 3, 2, 3
 ; CHECK-NEXT:    lvx 4, 0, 3
 ; CHECK-NEXT:    vadduwm 2, 2, 4
-; CHECK-NEXT:    xxsel 34, 34, 37, 35
+; CHECK-NEXT:    xxsel 34, 34, 0, 35
 ; CHECK-NEXT:    blr
   %a = add <4 x i32> %x, <i32 42, i32 42, i32 42, i32 42>
   %c = icmp ugt <4 x i32> %x, <i32 -43, i32 -43, i32 -43, i32 -43>
@@ -563,13 +563,13 @@ define <2 x i64> @unsigned_sat_constant_v2i64_using_cmp_sum(<2 x i64> %x) {
 ; CHECK-LABEL: unsigned_sat_constant_v2i64_using_cmp_sum:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addis 3, 2, .LCPI34_0@toc@ha
-; CHECK-NEXT:    vspltisb 4, -1
 ; CHECK-NEXT:    addi 3, 3, .LCPI34_0@toc@l
 ; CHECK-NEXT:    lxvd2x 0, 0, 3
 ; CHECK-NEXT:    xxswapd 35, 0
+; CHECK-NEXT:    xxleqv 0, 0, 0
 ; CHECK-NEXT:    vaddudm 3, 2, 3
 ; CHECK-NEXT:    vcmpgtud 2, 2, 3
-; CHECK-NEXT:    xxsel 34, 35, 36, 34
+; CHECK-NEXT:    xxsel 34, 35, 0, 34
 ; CHECK-NEXT:    blr
   %a = add <2 x i64> %x, <i64 42, i64 42>
   %c = icmp ugt <2 x i64> %x, %a
@@ -581,17 +581,17 @@ define <2 x i64> @unsigned_sat_constant_v2i64_using_cmp_notval(<2 x i64> %x) {
 ; CHECK-LABEL: unsigned_sat_constant_v2i64_using_cmp_notval:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addis 3, 2, .LCPI35_1@toc@ha
-; CHECK-NEXT:    vspltisb 5, -1
 ; CHECK-NEXT:    addi 3, 3, .LCPI35_1@toc@l
 ; CHECK-NEXT:    lxvd2x 0, 0, 3
 ; CHECK-NEXT:    addis 3, 2, .LCPI35_0@toc@ha
 ; CHECK-NEXT:    addi 3, 3, .LCPI35_0@toc@l
+; CHECK-NEXT:    lxvd2x 1, 0, 3
 ; CHECK-NEXT:    xxswapd 35, 0
-; CHECK-NEXT:    lxvd2x 0, 0, 3
+; CHECK-NEXT:    xxleqv 0, 0, 0
+; CHECK-NEXT:    xxswapd 36, 1
 ; CHECK-NEXT:    vcmpgtud 3, 2, 3
-; CHECK-NEXT:    xxswapd 36, 0
 ; CHECK-NEXT:    vaddudm 2, 2, 4
-; CHECK-NEXT:    xxsel 34, 34, 37, 35
+; CHECK-NEXT:    xxsel 34, 34, 0, 35
 ; CHECK-NEXT:    blr
   %a = add <2 x i64> %x, <i64 42, i64 42>
   %c = icmp ugt <2 x i64> %x, <i64 -43, i64 -43>
@@ -617,9 +617,9 @@ define <16 x i8> @unsigned_sat_variable_v16i8_using_cmp_sum(<16 x i8> %x, <16 x
 ; CHECK-LABEL: unsigned_sat_variable_v16i8_using_cmp_sum:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vaddubm 3, 2, 3
-; CHECK-NEXT:    vspltisb 4, -1
+; CHECK-NEXT:    xxleqv 0, 0, 0
 ; CHECK-NEXT:    vcmpgtub 2, 2, 3
-; CHECK-NEXT:    xxsel 34, 35, 36, 34
+; CHECK-NEXT:    xxsel 34, 35, 0, 34
 ; CHECK-NEXT:    blr
   %a = add <16 x i8> %x, %y
   %c = icmp ugt <16 x i8> %x, %a
@@ -631,10 +631,10 @@ define <16 x i8> @unsigned_sat_variable_v16i8_using_cmp_notval(<16 x i8> %x, <16
 ; CHECK-LABEL: unsigned_sat_variable_v16i8_using_cmp_notval:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    xxlnor 36, 35, 35
-; CHECK-NEXT:    vspltisb 5, -1
+; CHECK-NEXT:    xxleqv 0, 0, 0
 ; CHECK-NEXT:    vcmpgtub 4, 2, 4
 ; CHECK-NEXT:    vaddubm 2, 2, 3
-; CHECK-NEXT:    xxsel 34, 34, 37, 36
+; CHECK-NEXT:    xxsel 34, 34, 0, 36
 ; CHECK-NEXT:    blr
   %noty = xor <16 x i8> %y, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
   %a = add <16 x i8> %x, %y
@@ -661,9 +661,9 @@ define <8 x i16> @unsigned_sat_variable_v8i16_using_cmp_sum(<8 x i16> %x, <8 x i
 ; CHECK-LABEL: unsigned_sat_variable_v8i16_using_cmp_sum:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vadduhm 3, 2, 3
-; CHECK-NEXT:    vspltisb 4, -1
+; CHECK-NEXT:    xxleqv 0, 0, 0
 ; CHECK-NEXT:    vcmpgtuh 2, 2, 3
-; CHECK-NEXT:    xxsel 34, 35, 36, 34
+; CHECK-NEXT:    xxsel 34, 35, 0, 34
 ; CHECK-NEXT:    blr
   %a = add <8 x i16> %x, %y
   %c = icmp ugt <8 x i16> %x, %a
@@ -675,10 +675,10 @@ define <8 x i16> @unsigned_sat_variable_v8i16_using_cmp_notval(<8 x i16> %x, <8
 ; CHECK-LABEL: unsigned_sat_variable_v8i16_using_cmp_notval:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    xxlnor 36, 35, 35
-; CHECK-NEXT:    vspltisb 5, -1
+; CHECK-NEXT:    xxleqv 0, 0, 0
 ; CHECK-NEXT:    vcmpgtuh 4, 2, 4
 ; CHECK-NEXT:    vadduhm 2, 2, 3
-; CHECK-NEXT:    xxsel 34, 34, 37, 36
+; CHECK-NEXT:    xxsel 34, 34, 0, 36
 ; CHECK-NEXT:    blr
   %noty = xor <8 x i16> %y, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>
   %a = add <8 x i16> %x, %y
@@ -705,9 +705,9 @@ define <4 x i32> @unsigned_sat_variable_v4i32_using_cmp_sum(<4 x i32> %x, <4 x i
 ; CHECK-LABEL: unsigned_sat_variable_v4i32_using_cmp_sum:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vadduwm 3, 2, 3
-; CHECK-NEXT:    vspltisb 4, -1
+; CHECK-NEXT:    xxleqv 0, 0, 0
 ; CHECK-NEXT:    vcmpgtuw 2, 2, 3
-; CHECK-NEXT:    xxsel 34, 35, 36, 34
+; CHECK-NEXT:    xxsel 34, 35, 0, 34
 ; CHECK-NEXT:    blr
   %a = add <4 x i32> %x, %y
   %c = icmp ugt <4 x i32> %x, %a
@@ -719,10 +719,10 @@ define <4 x i32> @unsigned_sat_variable_v4i32_using_cmp_notval(<4 x i32> %x, <4
 ; CHECK-LABEL: unsigned_sat_variable_v4i32_using_cmp_notval:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    xxlnor 36, 35, 35
-; CHECK-NEXT:    vspltisb 5, -1
+; CHECK-NEXT:    xxleqv 0, 0, 0
 ; CHECK-NEXT:    vcmpgtuw 4, 2, 4
 ; CHECK-NEXT:    vadduwm 2, 2, 3
-; CHECK-NEXT:    xxsel 34, 34, 37, 36
+; CHECK-NEXT:    xxsel 34, 34, 0, 36
 ; CHECK-NEXT:    blr
   %noty = xor <4 x i32> %y, <i32 -1, i32 -1, i32 -1, i32 -1>
   %a = add <4 x i32> %x, %y
@@ -749,9 +749,9 @@ define <2 x i64> @unsigned_sat_variable_v2i64_using_cmp_sum(<2 x i64> %x, <2 x i
 ; CHECK-LABEL: unsigned_sat_variable_v2i64_using_cmp_sum:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vaddudm 3, 2, 3
-; CHECK-NEXT:    vspltisb 4, -1
+; CHECK-NEXT:    xxleqv 0, 0, 0
 ; CHECK-NEXT:    vcmpgtud 2, 2, 3
-; CHECK-NEXT:    xxsel 34, 35, 36, 34
+; CHECK-NEXT:    xxsel 34, 35, 0, 34
 ; CHECK-NEXT:    blr
   %a = add <2 x i64> %x, %y
   %c = icmp ugt <2 x i64> %x, %a
@@ -763,10 +763,10 @@ define <2 x i64> @unsigned_sat_variable_v2i64_using_cmp_notval(<2 x i64> %x, <2
 ; CHECK-LABEL: unsigned_sat_variable_v2i64_using_cmp_notval:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    xxlnor 36, 35, 35
-; CHECK-NEXT:    vspltisb 5, -1
+; CHECK-NEXT:    xxleqv 0, 0, 0
 ; CHECK-NEXT:    vcmpgtud 4, 2, 4
 ; CHECK-NEXT:    vaddudm 2, 2, 3
-; CHECK-NEXT:    xxsel 34, 34, 37, 36
+; CHECK-NEXT:    xxsel 34, 34, 0, 36
 ; CHECK-NEXT:    blr
   %noty = xor <2 x i64> %y, <i64 -1, i64 -1>
   %a = add <2 x i64> %x, %y
diff --git a/llvm/test/CodeGen/PowerPC/save-bp.ll b/llvm/test/CodeGen/PowerPC/save-bp.ll
index 8a7cef692..f137daff8 100644
--- a/llvm/test/CodeGen/PowerPC/save-bp.ll
+++ b/llvm/test/CodeGen/PowerPC/save-bp.ll
@@ -51,4 +51,4 @@ entry:
   ret i64 0
 }
 
-attributes #0 = { norecurse readnone nounwind sspstrong "no-frame-pointer-elim"="true" "target-cpu"="ppc" }
+attributes #0 = { norecurse readnone nounwind sspstrong "frame-pointer"="all" "target-cpu"="ppc" }
diff --git a/llvm/test/CodeGen/PowerPC/save-cr-ppc32svr4.ll b/llvm/test/CodeGen/PowerPC/save-cr-ppc32svr4.ll
index a739a0210..286dfb7ec 100644
--- a/llvm/test/CodeGen/PowerPC/save-cr-ppc32svr4.ll
+++ b/llvm/test/CodeGen/PowerPC/save-cr-ppc32svr4.ll
@@ -44,4 +44,4 @@ b14:                                              ; preds = %b11, %b9, %b3
   ret i64 %v15
 }
 
-attributes #0 = { norecurse nounwind readnone sspstrong "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "stack-protector-buffer-size"="8" "target-cpu"="ppc" }
+attributes #0 = { norecurse nounwind readnone sspstrong "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "stack-protector-buffer-size"="8" "target-cpu"="ppc" }
diff --git a/llvm/test/CodeGen/PowerPC/save-crbp-ppc32svr4.ll b/llvm/test/CodeGen/PowerPC/save-crbp-ppc32svr4.ll
index 930d53d84..e47a5c7c3 100644
--- a/llvm/test/CodeGen/PowerPC/save-crbp-ppc32svr4.ll
+++ b/llvm/test/CodeGen/PowerPC/save-crbp-ppc32svr4.ll
@@ -55,4 +55,4 @@ b15:                                              ; preds = %b12, %b10, %b4
   ret i64 %v18
 }
 
-attributes #0 = { norecurse readnone sspstrong "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "stack-protector-buffer-size"="8" "target-cpu"="ppc" }
+attributes #0 = { norecurse readnone sspstrong "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "stack-protector-buffer-size"="8" "target-cpu"="ppc" }
diff --git a/llvm/test/CodeGen/PowerPC/scalar-min-max.ll b/llvm/test/CodeGen/PowerPC/scalar-min-max.ll
new file mode 100644
index 000000000..a7e95228b
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/scalar-min-max.ll
@@ -0,0 +1,203 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mcpu=pwr8 -ppc-asm-full-reg-names --enable-unsafe-fp-math \
+; RUN:   -verify-machineinstrs --enable-no-signed-zeros-fp-math \
+; RUN:   --enable-no-nans-fp-math \
+; RUN:   -mtriple=powerpc64le-unknown-unknown < %s | FileCheck %s
+; RUN: llc -mcpu=pwr9 -ppc-asm-full-reg-names --enable-unsafe-fp-math \
+; RUN:   -verify-machineinstrs --enable-no-signed-zeros-fp-math \
+; RUN:   --enable-no-nans-fp-math \
+; RUN:   -mtriple=powerpc64le-unknown-unknown < %s | FileCheck %s
+; RUN: llc -mcpu=pwr9 -ppc-asm-full-reg-names -verify-machineinstrs \
+; RUN:   -mtriple=powerpc64le-unknown-unknown < %s | FileCheck %s \
+; RUN:   --check-prefix=NO-FAST-P9
+; RUN: llc -mcpu=pwr8 -ppc-asm-full-reg-names -verify-machineinstrs \
+; RUN:   -mtriple=powerpc64le-unknown-unknown < %s | FileCheck %s \
+; RUN:   --check-prefix=NO-FAST-P8
+define dso_local float @testfmax(float %a, float %b) local_unnamed_addr {
+; CHECK-LABEL: testfmax:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xsmaxdp f1, f1, f2
+; CHECK-NEXT:    blr
+;
+; NO-FAST-P9-LABEL: testfmax:
+; NO-FAST-P9:       # %bb.0: # %entry
+; NO-FAST-P9-NEXT:    xsmaxcdp f1, f1, f2
+; NO-FAST-P9-NEXT:    blr
+;
+; NO-FAST-P8-LABEL: testfmax:
+; NO-FAST-P8:       # %bb.0: # %entry
+; NO-FAST-P8-NEXT:    fcmpu cr0, f1, f2
+; NO-FAST-P8-NEXT:    bgtlr cr0
+; NO-FAST-P8-NEXT:  # %bb.1: # %entry
+; NO-FAST-P8-NEXT:    fmr f1, f2
+; NO-FAST-P8-NEXT:    blr
+entry:
+  %cmp = fcmp ogt float %a, %b
+  %cond = select i1 %cmp, float %a, float %b
+  ret float %cond
+}
+
+define dso_local double @testdmax(double %a, double %b) local_unnamed_addr {
+; CHECK-LABEL: testdmax:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xsmaxdp f1, f1, f2
+; CHECK-NEXT:    blr
+;
+; NO-FAST-P9-LABEL: testdmax:
+; NO-FAST-P9:       # %bb.0: # %entry
+; NO-FAST-P9-NEXT:    xsmaxcdp f1, f1, f2
+; NO-FAST-P9-NEXT:    blr
+;
+; NO-FAST-P8-LABEL: testdmax:
+; NO-FAST-P8:       # %bb.0: # %entry
+; NO-FAST-P8-NEXT:    xscmpudp cr0, f1, f2
+; NO-FAST-P8-NEXT:    bgtlr cr0
+; NO-FAST-P8-NEXT:  # %bb.1: # %entry
+; NO-FAST-P8-NEXT:    fmr f1, f2
+; NO-FAST-P8-NEXT:    blr
+entry:
+  %cmp = fcmp ogt double %a, %b
+  %cond = select i1 %cmp, double %a, double %b
+  ret double %cond
+}
+
+define dso_local float @testfmin(float %a, float %b) local_unnamed_addr {
+; CHECK-LABEL: testfmin:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xsmindp f1, f1, f2
+; CHECK-NEXT:    blr
+;
+; NO-FAST-P9-LABEL: testfmin:
+; NO-FAST-P9:       # %bb.0: # %entry
+; NO-FAST-P9-NEXT:    xsmincdp f1, f1, f2
+; NO-FAST-P9-NEXT:    blr
+;
+; NO-FAST-P8-LABEL: testfmin:
+; NO-FAST-P8:       # %bb.0: # %entry
+; NO-FAST-P8-NEXT:    fcmpu cr0, f1, f2
+; NO-FAST-P8-NEXT:    bltlr cr0
+; NO-FAST-P8-NEXT:  # %bb.1: # %entry
+; NO-FAST-P8-NEXT:    fmr f1, f2
+; NO-FAST-P8-NEXT:    blr
+entry:
+  %cmp = fcmp olt float %a, %b
+  %cond = select i1 %cmp, float %a, float %b
+  ret float %cond
+}
+
+define dso_local double @testdmin(double %a, double %b) local_unnamed_addr {
+; CHECK-LABEL: testdmin:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xsmindp f1, f1, f2
+; CHECK-NEXT:    blr
+;
+; NO-FAST-P9-LABEL: testdmin:
+; NO-FAST-P9:       # %bb.0: # %entry
+; NO-FAST-P9-NEXT:    xsmincdp f1, f1, f2
+; NO-FAST-P9-NEXT:    blr
+;
+; NO-FAST-P8-LABEL: testdmin:
+; NO-FAST-P8:       # %bb.0: # %entry
+; NO-FAST-P8-NEXT:    xscmpudp cr0, f1, f2
+; NO-FAST-P8-NEXT:    bltlr cr0
+; NO-FAST-P8-NEXT:  # %bb.1: # %entry
+; NO-FAST-P8-NEXT:    fmr f1, f2
+; NO-FAST-P8-NEXT:    blr
+entry:
+  %cmp = fcmp olt double %a, %b
+  %cond = select i1 %cmp, double %a, double %b
+  ret double %cond
+}
+
+define dso_local float @testfmax_fast(float %a, float %b) local_unnamed_addr {
+; CHECK-LABEL: testfmax_fast:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xsmaxdp f1, f1, f2
+; CHECK-NEXT:    blr
+;
+; NO-FAST-P9-LABEL: testfmax_fast:
+; NO-FAST-P9:       # %bb.0: # %entry
+; NO-FAST-P9-NEXT:    xsmaxcdp f1, f1, f2
+; NO-FAST-P9-NEXT:    blr
+;
+; NO-FAST-P8-LABEL: testfmax_fast:
+; NO-FAST-P8:       # %bb.0: # %entry
+; NO-FAST-P8-NEXT:    fcmpu cr0, f1, f2
+; NO-FAST-P8-NEXT:    bgtlr cr0
+; NO-FAST-P8-NEXT:  # %bb.1: # %entry
+; NO-FAST-P8-NEXT:    fmr f1, f2
+; NO-FAST-P8-NEXT:    blr
+entry:
+  %cmp = fcmp fast ogt float %a, %b
+  %cond = select i1 %cmp, float %a, float %b
+  ret float %cond
+}
+define dso_local double @testdmax_fast(double %a, double %b) local_unnamed_addr {
+; CHECK-LABEL: testdmax_fast:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xsmaxdp f1, f1, f2
+; CHECK-NEXT:    blr
+;
+; NO-FAST-P9-LABEL: testdmax_fast:
+; NO-FAST-P9:       # %bb.0: # %entry
+; NO-FAST-P9-NEXT:    xsmaxcdp f1, f1, f2
+; NO-FAST-P9-NEXT:    blr
+;
+; NO-FAST-P8-LABEL: testdmax_fast:
+; NO-FAST-P8:       # %bb.0: # %entry
+; NO-FAST-P8-NEXT:    xscmpudp cr0, f1, f2
+; NO-FAST-P8-NEXT:    bgtlr cr0
+; NO-FAST-P8-NEXT:  # %bb.1: # %entry
+; NO-FAST-P8-NEXT:    fmr f1, f2
+; NO-FAST-P8-NEXT:    blr
+entry:
+  %cmp = fcmp fast ogt double %a, %b
+  %cond = select i1 %cmp, double %a, double %b
+  ret double %cond
+}
+define dso_local float @testfmin_fast(float %a, float %b) local_unnamed_addr {
+; CHECK-LABEL: testfmin_fast:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xsmindp f1, f1, f2
+; CHECK-NEXT:    blr
+;
+; NO-FAST-P9-LABEL: testfmin_fast:
+; NO-FAST-P9:       # %bb.0: # %entry
+; NO-FAST-P9-NEXT:    xsmincdp f1, f1, f2
+; NO-FAST-P9-NEXT:    blr
+;
+; NO-FAST-P8-LABEL: testfmin_fast:
+; NO-FAST-P8:       # %bb.0: # %entry
+; NO-FAST-P8-NEXT:    fcmpu cr0, f1, f2
+; NO-FAST-P8-NEXT:    bltlr cr0
+; NO-FAST-P8-NEXT:  # %bb.1: # %entry
+; NO-FAST-P8-NEXT:    fmr f1, f2
+; NO-FAST-P8-NEXT:    blr
+entry:
+  %cmp = fcmp fast olt float %a, %b
+  %cond = select i1 %cmp, float %a, float %b
+  ret float %cond
+}
+define dso_local double @testdmin_fast(double %a, double %b) local_unnamed_addr {
+; CHECK-LABEL: testdmin_fast:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xsmindp f1, f1, f2
+; CHECK-NEXT:    blr
+;
+; NO-FAST-P9-LABEL: testdmin_fast:
+; NO-FAST-P9:       # %bb.0: # %entry
+; NO-FAST-P9-NEXT:    xsmincdp f1, f1, f2
+; NO-FAST-P9-NEXT:    blr
+;
+; NO-FAST-P8-LABEL: testdmin_fast:
+; NO-FAST-P8:       # %bb.0: # %entry
+; NO-FAST-P8-NEXT:    xscmpudp cr0, f1, f2
+; NO-FAST-P8-NEXT:    bltlr cr0
+; NO-FAST-P8-NEXT:  # %bb.1: # %entry
+; NO-FAST-P8-NEXT:    fmr f1, f2
+; NO-FAST-P8-NEXT:    blr
+entry:
+  %cmp = fcmp fast olt double %a, %b
+  %cond = select i1 %cmp, double %a, double %b
+  ret double %cond
+}
diff --git a/llvm/test/CodeGen/PowerPC/scalar-rounding-ops.ll b/llvm/test/CodeGen/PowerPC/scalar-rounding-ops.ll
new file mode 100644
index 000000000..7f18853b8
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/scalar-rounding-ops.ll
@@ -0,0 +1,561 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mcpu=pwr7 -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr \
+; RUN:   -mtriple=powerpc64-unknown-unknown -verify-machineinstrs < %s | \
+; RUN:   FileCheck %s --check-prefix=BE
+; RUN: llc -mcpu=pwr8 -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr \
+; RUN:   -mtriple=powerpc64le-unknown-unknown -verify-machineinstrs < %s | \
+; RUN:   FileCheck %s
+; RUN: llc -mcpu=pwr8 -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr \
+; RUN:   -mtriple=powerpc64le-unknown-unknown -verify-machineinstrs < %s \
+; RUN:   --enable-unsafe-fp-math | FileCheck %s --check-prefix=FAST
+define dso_local i64 @test_lrint(double %d) local_unnamed_addr {
+; BE-LABEL: test_lrint:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    mflr r0
+; BE-NEXT:    std r0, 16(r1)
+; BE-NEXT:    stdu r1, -112(r1)
+; BE-NEXT:    .cfi_def_cfa_offset 112
+; BE-NEXT:    .cfi_offset lr, 16
+; BE-NEXT:    bl lrint
+; BE-NEXT:    nop
+; BE-NEXT:    addi r1, r1, 112
+; BE-NEXT:    ld r0, 16(r1)
+; BE-NEXT:    mtlr r0
+; BE-NEXT:    blr
+;
+; CHECK-LABEL: test_lrint:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    std r0, 16(r1)
+; CHECK-NEXT:    stdu r1, -32(r1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    .cfi_offset lr, 16
+; CHECK-NEXT:    bl lrint
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    addi r1, r1, 32
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+;
+; FAST-LABEL: test_lrint:
+; FAST:       # %bb.0: # %entry
+; FAST-NEXT:    fctid f0, f1
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    blr
+entry:
+  %0 = tail call i64 @llvm.lrint.i64.f64(double %d)
+  ret i64 %0
+}
+
+declare i64 @llvm.lrint.i64.f64(double)
+
+define dso_local i64 @test_lrintf(float %f) local_unnamed_addr {
+; BE-LABEL: test_lrintf:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    mflr r0
+; BE-NEXT:    std r0, 16(r1)
+; BE-NEXT:    stdu r1, -112(r1)
+; BE-NEXT:    .cfi_def_cfa_offset 112
+; BE-NEXT:    .cfi_offset lr, 16
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    addi r1, r1, 112
+; BE-NEXT:    ld r0, 16(r1)
+; BE-NEXT:    mtlr r0
+; BE-NEXT:    blr
+;
+; CHECK-LABEL: test_lrintf:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    std r0, 16(r1)
+; CHECK-NEXT:    stdu r1, -32(r1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    .cfi_offset lr, 16
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    addi r1, r1, 32
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+;
+; FAST-LABEL: test_lrintf:
+; FAST:       # %bb.0: # %entry
+; FAST-NEXT:    fctid f0, f1
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    blr
+entry:
+  %0 = tail call i64 @llvm.lrint.i64.f32(float %f)
+  ret i64 %0
+}
+
+declare i64 @llvm.lrint.i64.f32(float)
+
+define dso_local i64 @test_llrint(double %d) local_unnamed_addr {
+; BE-LABEL: test_llrint:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    mflr r0
+; BE-NEXT:    std r0, 16(r1)
+; BE-NEXT:    stdu r1, -112(r1)
+; BE-NEXT:    .cfi_def_cfa_offset 112
+; BE-NEXT:    .cfi_offset lr, 16
+; BE-NEXT:    bl llrint
+; BE-NEXT:    nop
+; BE-NEXT:    addi r1, r1, 112
+; BE-NEXT:    ld r0, 16(r1)
+; BE-NEXT:    mtlr r0
+; BE-NEXT:    blr
+;
+; CHECK-LABEL: test_llrint:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    std r0, 16(r1)
+; CHECK-NEXT:    stdu r1, -32(r1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    .cfi_offset lr, 16
+; CHECK-NEXT:    bl llrint
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    addi r1, r1, 32
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+;
+; FAST-LABEL: test_llrint:
+; FAST:       # %bb.0: # %entry
+; FAST-NEXT:    fctid f0, f1
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    blr
+entry:
+  %0 = tail call i64 @llvm.llrint.i64.f64(double %d)
+  ret i64 %0
+}
+
+declare i64 @llvm.llrint.i64.f64(double)
+
+define dso_local i64 @test_llrintf(float %f) local_unnamed_addr {
+; BE-LABEL: test_llrintf:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    mflr r0
+; BE-NEXT:    std r0, 16(r1)
+; BE-NEXT:    stdu r1, -112(r1)
+; BE-NEXT:    .cfi_def_cfa_offset 112
+; BE-NEXT:    .cfi_offset lr, 16
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    addi r1, r1, 112
+; BE-NEXT:    ld r0, 16(r1)
+; BE-NEXT:    mtlr r0
+; BE-NEXT:    blr
+;
+; CHECK-LABEL: test_llrintf:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    std r0, 16(r1)
+; CHECK-NEXT:    stdu r1, -32(r1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    .cfi_offset lr, 16
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    addi r1, r1, 32
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+;
+; FAST-LABEL: test_llrintf:
+; FAST:       # %bb.0: # %entry
+; FAST-NEXT:    fctid f0, f1
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    blr
+entry:
+  %0 = tail call i64 @llvm.llrint.i64.f32(float %f)
+  ret i64 %0
+}
+
+declare i64 @llvm.llrint.i64.f32(float)
+
+define dso_local i64 @test_lround(double %d) local_unnamed_addr {
+; BE-LABEL: test_lround:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    mflr r0
+; BE-NEXT:    std r0, 16(r1)
+; BE-NEXT:    stdu r1, -112(r1)
+; BE-NEXT:    .cfi_def_cfa_offset 112
+; BE-NEXT:    .cfi_offset lr, 16
+; BE-NEXT:    bl lround
+; BE-NEXT:    nop
+; BE-NEXT:    addi r1, r1, 112
+; BE-NEXT:    ld r0, 16(r1)
+; BE-NEXT:    mtlr r0
+; BE-NEXT:    blr
+;
+; CHECK-LABEL: test_lround:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    std r0, 16(r1)
+; CHECK-NEXT:    stdu r1, -32(r1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    .cfi_offset lr, 16
+; CHECK-NEXT:    bl lround
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    addi r1, r1, 32
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+;
+; FAST-LABEL: test_lround:
+; FAST:       # %bb.0: # %entry
+; FAST-NEXT:    xsrdpi f0, f1
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    blr
+entry:
+  %0 = tail call i64 @llvm.lround.i64.f64(double %d)
+  ret i64 %0
+}
+
+declare i64 @llvm.lround.i64.f64(double)
+
+define dso_local i64 @test_lroundf(float %f) local_unnamed_addr {
+; BE-LABEL: test_lroundf:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    mflr r0
+; BE-NEXT:    std r0, 16(r1)
+; BE-NEXT:    stdu r1, -112(r1)
+; BE-NEXT:    .cfi_def_cfa_offset 112
+; BE-NEXT:    .cfi_offset lr, 16
+; BE-NEXT:    bl lroundf
+; BE-NEXT:    nop
+; BE-NEXT:    addi r1, r1, 112
+; BE-NEXT:    ld r0, 16(r1)
+; BE-NEXT:    mtlr r0
+; BE-NEXT:    blr
+;
+; CHECK-LABEL: test_lroundf:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    std r0, 16(r1)
+; CHECK-NEXT:    stdu r1, -32(r1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    .cfi_offset lr, 16
+; CHECK-NEXT:    bl lroundf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    addi r1, r1, 32
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+;
+; FAST-LABEL: test_lroundf:
+; FAST:       # %bb.0: # %entry
+; FAST-NEXT:    xsrdpi f0, f1
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    blr
+entry:
+  %0 = tail call i64 @llvm.lround.i64.f32(float %f)
+  ret i64 %0
+}
+
+declare i64 @llvm.lround.i64.f32(float)
+
+define dso_local i64 @test_llround(double %d) local_unnamed_addr {
+; BE-LABEL: test_llround:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    mflr r0
+; BE-NEXT:    std r0, 16(r1)
+; BE-NEXT:    stdu r1, -112(r1)
+; BE-NEXT:    .cfi_def_cfa_offset 112
+; BE-NEXT:    .cfi_offset lr, 16
+; BE-NEXT:    bl llround
+; BE-NEXT:    nop
+; BE-NEXT:    addi r1, r1, 112
+; BE-NEXT:    ld r0, 16(r1)
+; BE-NEXT:    mtlr r0
+; BE-NEXT:    blr
+;
+; CHECK-LABEL: test_llround:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    std r0, 16(r1)
+; CHECK-NEXT:    stdu r1, -32(r1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    .cfi_offset lr, 16
+; CHECK-NEXT:    bl llround
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    addi r1, r1, 32
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+;
+; FAST-LABEL: test_llround:
+; FAST:       # %bb.0: # %entry
+; FAST-NEXT:    xsrdpi f0, f1
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    blr
+entry:
+  %0 = tail call i64 @llvm.llround.i64.f64(double %d)
+  ret i64 %0
+}
+
+declare i64 @llvm.llround.i64.f64(double)
+
+define dso_local i64 @test_llroundf(float %f) local_unnamed_addr {
+; BE-LABEL: test_llroundf:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    mflr r0
+; BE-NEXT:    std r0, 16(r1)
+; BE-NEXT:    stdu r1, -112(r1)
+; BE-NEXT:    .cfi_def_cfa_offset 112
+; BE-NEXT:    .cfi_offset lr, 16
+; BE-NEXT:    bl llroundf
+; BE-NEXT:    nop
+; BE-NEXT:    addi r1, r1, 112
+; BE-NEXT:    ld r0, 16(r1)
+; BE-NEXT:    mtlr r0
+; BE-NEXT:    blr
+;
+; CHECK-LABEL: test_llroundf:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    std r0, 16(r1)
+; CHECK-NEXT:    stdu r1, -32(r1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    .cfi_offset lr, 16
+; CHECK-NEXT:    bl llroundf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    addi r1, r1, 32
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+;
+; FAST-LABEL: test_llroundf:
+; FAST:       # %bb.0: # %entry
+; FAST-NEXT:    xsrdpi f0, f1
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    blr
+entry:
+  %0 = tail call i64 @llvm.llround.i64.f32(float %f)
+  ret i64 %0
+}
+
+declare i64 @llvm.llround.i64.f32(float)
+
+define dso_local double @test_nearbyint(double %d) local_unnamed_addr {
+; BE-LABEL: test_nearbyint:
+; BE:    # %bb.0: # %entry
+; BE:    bl nearbyint
+; BE:    blr
+;
+; CHECK-LABEL: test_nearbyint:
+; CHECK:    # %bb.0: # %entry
+; CHECK:    bl nearbyint
+; CHECK:    blr
+;
+; FAST-LABEL: test_nearbyint:
+; FAST:       # %bb.0: # %entry
+; FAST-NEXT:    xsrdpic f1, f1
+; FAST-NEXT:    blr
+entry:
+  %0 = tail call double @llvm.nearbyint.f64(double %d)
+  ret double %0
+}
+
+declare double @llvm.nearbyint.f64(double)
+
+define dso_local float @test_nearbyintf(float %f) local_unnamed_addr {
+; BE-LABEL: test_nearbyintf:
+; BE:    # %bb.0: # %entry
+; BE:    bl nearbyint
+; BE:    blr
+;
+; CHECK-LABEL: test_nearbyintf:
+; CHECK:    # %bb.0: # %entry
+; CHECK:    bl nearbyintf
+; CHECK:    blr
+;
+; FAST-LABEL: test_nearbyintf:
+; FAST:       # %bb.0: # %entry
+; FAST-NEXT:    xsrdpic f1, f1
+; FAST-NEXT:    blr
+entry:
+  %0 = tail call float @llvm.nearbyint.f32(float %f)
+  ret float %0
+}
+
+declare float @llvm.nearbyint.f32(float)
+
+define dso_local double @test_round(double %d) local_unnamed_addr {
+; BE-LABEL: test_round:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    xsrdpi f1, f1
+; BE-NEXT:    blr
+;
+; CHECK-LABEL: test_round:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xsrdpi f1, f1
+; CHECK-NEXT:    blr
+;
+; FAST-LABEL: test_round:
+; FAST:       # %bb.0: # %entry
+; FAST-NEXT:    xsrdpi f1, f1
+; FAST-NEXT:    blr
+entry:
+  %0 = tail call double @llvm.round.f64(double %d)
+  ret double %0
+}
+
+declare double @llvm.round.f64(double)
+
+define dso_local float @test_roundf(float %f) local_unnamed_addr {
+; BE-LABEL: test_roundf:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    xsrdpi f1, f1
+; BE-NEXT:    blr
+;
+; CHECK-LABEL: test_roundf:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xsrdpi f1, f1
+; CHECK-NEXT:    blr
+;
+; FAST-LABEL: test_roundf:
+; FAST:       # %bb.0: # %entry
+; FAST-NEXT:    xsrdpi f1, f1
+; FAST-NEXT:    blr
+entry:
+  %0 = tail call float @llvm.round.f32(float %f)
+  ret float %0
+}
+
+declare float @llvm.round.f32(float)
+
+define dso_local double @test_trunc(double %d) local_unnamed_addr {
+; BE-LABEL: test_trunc:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    xsrdpiz f1, f1
+; BE-NEXT:    blr
+;
+; CHECK-LABEL: test_trunc:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xsrdpiz f1, f1
+; CHECK-NEXT:    blr
+;
+; FAST-LABEL: test_trunc:
+; FAST:       # %bb.0: # %entry
+; FAST-NEXT:    xsrdpiz f1, f1
+; FAST-NEXT:    blr
+entry:
+  %0 = tail call double @llvm.trunc.f64(double %d)
+  ret double %0
+}
+
+declare double @llvm.trunc.f64(double)
+
+define dso_local float @test_truncf(float %f) local_unnamed_addr {
+; BE-LABEL: test_truncf:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    xsrdpiz f1, f1
+; BE-NEXT:    blr
+;
+; CHECK-LABEL: test_truncf:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xsrdpiz f1, f1
+; CHECK-NEXT:    blr
+;
+; FAST-LABEL: test_truncf:
+; FAST:       # %bb.0: # %entry
+; FAST-NEXT:    xsrdpiz f1, f1
+; FAST-NEXT:    blr
+entry:
+  %0 = tail call float @llvm.trunc.f32(float %f)
+  ret float %0
+}
+
+declare float @llvm.trunc.f32(float)
+
+define dso_local double @test_floor(double %d) local_unnamed_addr {
+; BE-LABEL: test_floor:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    xsrdpim f1, f1
+; BE-NEXT:    blr
+;
+; CHECK-LABEL: test_floor:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xsrdpim f1, f1
+; CHECK-NEXT:    blr
+;
+; FAST-LABEL: test_floor:
+; FAST:       # %bb.0: # %entry
+; FAST-NEXT:    xsrdpim f1, f1
+; FAST-NEXT:    blr
+entry:
+  %0 = tail call double @llvm.floor.f64(double %d)
+  ret double %0
+}
+
+declare double @llvm.floor.f64(double)
+
+define dso_local float @test_floorf(float %f) local_unnamed_addr {
+; BE-LABEL: test_floorf:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    xsrdpim f1, f1
+; BE-NEXT:    blr
+;
+; CHECK-LABEL: test_floorf:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xsrdpim f1, f1
+; CHECK-NEXT:    blr
+;
+; FAST-LABEL: test_floorf:
+; FAST:       # %bb.0: # %entry
+; FAST-NEXT:    xsrdpim f1, f1
+; FAST-NEXT:    blr
+entry:
+  %0 = tail call float @llvm.floor.f32(float %f)
+  ret float %0
+}
+
+declare float @llvm.floor.f32(float)
+
+define dso_local double @test_ceil(double %d) local_unnamed_addr {
+; BE-LABEL: test_ceil:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    xsrdpip f1, f1
+; BE-NEXT:    blr
+;
+; CHECK-LABEL: test_ceil:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xsrdpip f1, f1
+; CHECK-NEXT:    blr
+;
+; FAST-LABEL: test_ceil:
+; FAST:       # %bb.0: # %entry
+; FAST-NEXT:    xsrdpip f1, f1
+; FAST-NEXT:    blr
+entry:
+  %0 = tail call double @llvm.ceil.f64(double %d)
+  ret double %0
+}
+
+declare double @llvm.ceil.f64(double)
+
+define dso_local float @test_ceilf(float %f) local_unnamed_addr {
+; BE-LABEL: test_ceilf:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    xsrdpip f1, f1
+; BE-NEXT:    blr
+;
+; CHECK-LABEL: test_ceilf:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xsrdpip f1, f1
+; CHECK-NEXT:    blr
+;
+; FAST-LABEL: test_ceilf:
+; FAST:       # %bb.0: # %entry
+; FAST-NEXT:    xsrdpip f1, f1
+; FAST-NEXT:    blr
+entry:
+  %0 = tail call float @llvm.ceil.f32(float %f)
+  ret float %0
+}
+
+declare float @llvm.ceil.f32(float)
diff --git a/llvm/test/CodeGen/PowerPC/scheduling-mem-dependency.ll b/llvm/test/CodeGen/PowerPC/scheduling-mem-dependency.ll
index 40167e18a..4c6f9207a 100644
--- a/llvm/test/CodeGen/PowerPC/scheduling-mem-dependency.ll
+++ b/llvm/test/CodeGen/PowerPC/scheduling-mem-dependency.ll
@@ -1,19 +1,58 @@
 ; REQUIRES: asserts
 ; RUN: llc < %s -mtriple=powerpc64le-unknown-linux-gnu -verify-misched -debug-only=machine-scheduler -o - 2>&1 > /dev/null | FileCheck %s
+; RUN: llc < %s -mtriple=powerpc64le-unknown-linux-gnu -mcpu=pwr9 -verify-misched -debug-only=machine-scheduler -o - 2>&1 > /dev/null | FileCheck %s --check-prefix=CHECK-P9
 
 define i64 @store_disjoint_memory(i64* nocapture %P, i64 %v) {
 entry:
 ; CHECK: ********** MI Scheduling **********
 ; CHECK-LABEL: store_disjoint_memory:%bb.0
-; CHECK:SU(2):   STD renamable $x4, 24, renamable $x5 :: (store 8 into %ir.arrayidx)
+; CHECK:SU([[REG2:[0-9]+]]):   STD renamable $x{{[0-9]+}}, 24, renamable $x[[REG5:[0-9]+]]
 ; CHECK-NOT: Successors:
-; CHECK-NOT:    SU(3): Ord  Latency=0 Memory
-; CHECK:SU(3):   STD renamable $x4, 16, renamable $x5 :: (store 8 into %ir.arrayidx1)
+; CHECK-NOT:    SU([[REG3]]): Ord  Latency=0 Memory
+; CHECK:SU([[REG3:[0-9]+]]):   STD renamable $x{{[0-9]+}}, 16, renamable $x[[REG5]]
 ; CHECK: Predecessors:
-; CHECK-NOT:    SU(2): Ord  Latency=0 Memory
+; CHECK-NOT:    SU([[REG2]]): Ord  Latency=0 Memory
   %arrayidx = getelementptr inbounds i64, i64* %P, i64 3
   store i64 %v, i64* %arrayidx
   %arrayidx1 = getelementptr inbounds i64, i64* %P, i64 2
   store i64 %v, i64* %arrayidx1
   ret i64 %v
 }
+
+; LXSD is an instruction that can be modeled.
+@gd = external local_unnamed_addr global [500 x double], align 8
+@gf = external local_unnamed_addr global [500 x float], align 4
+
+define double @test_lxsd_no_barrier(double %a, double %b, double %c, double %d, double %e, double %f, double %g, double %h, double %i, double %j, double %k, double %l, double %m) {
+entry:
+  %0 = load double, double* getelementptr inbounds ([500 x double], [500 x double]* @gd, i64 0, i64 10), align 8
+  %1 = load double, double* getelementptr inbounds ([500 x double], [500 x double]* @gd, i64 0, i64 17), align 8
+  %2 = load double, double* getelementptr inbounds ([500 x double], [500 x double]* @gd, i64 0, i64 87), align 8
+  %3 = load double, double* getelementptr inbounds ([500 x double], [500 x double]* @gd, i64 0, i64 97), align 8
+  %4 = load double, double* getelementptr inbounds ([500 x double], [500 x double]* @gd, i64 0, i64 77), align 8
+  %add = fadd double %a, %b
+  %add1 = fadd double %add, %c
+  %add2 = fadd double %add1, %d
+  %add3 = fadd double %add2, %e
+  %add4 = fadd double %add3, %f
+  %add5 = fadd double %add4, %g
+  %add6 = fadd double %add5, %h
+  %add7 = fadd double %add6, %i
+  %add8 = fadd double %add7, %j
+  %add9 = fadd double %add8, %k
+  %add10 = fadd double %add9, %l
+  %add11 = fadd double %add10, %m
+  %add12 = fadd double %add11, %0
+  %add13 = fadd double %add12, %1
+  %add14 = fadd double %add13, %2
+  %add15 = fadd double %add14, %3
+  %add16 = fadd double %add15, %4
+  ret double %add16
+; CHECK-P9: ********** MI Scheduling **********
+; CHECK-P9-LABEL: test_lxsd_no_barrier:%bb.0 entry
+; CHECK-P9-NOT:Global memory object and new barrier chain: SU({{[0-9]+}}).
+; CHECK-P9:SU({{[0-9]+}}):   renamable $vf{{[0-9]+}} = LXSD 136
+; CHECK-P9:SU({{[0-9]+}}):   renamable $vf{{[0-9]+}} = LXSD 696
+; CHECK-P9:SU({{[0-9]+}}):   renamable $vf{{[0-9]+}} = LXSD 776
+; CHECK-P9:SU({{[0-9]+}}):   renamable $vf{{[0-9]+}} = LXSD 616
+}
diff --git a/llvm/test/CodeGen/PowerPC/select-addrRegRegOnly.ll b/llvm/test/CodeGen/PowerPC/select-addrRegRegOnly.ll
index d1ecdae58..29b11dfa2 100644
--- a/llvm/test/CodeGen/PowerPC/select-addrRegRegOnly.ll
+++ b/llvm/test/CodeGen/PowerPC/select-addrRegRegOnly.ll
@@ -23,7 +23,7 @@ define float @testMultipleAccess(i32* nocapture readonly %arr) local_unnamed_add
 ; CHECK-NEXT:    lwz 4, 8(3)
 ; CHECK-NEXT:    lwz 3, 12(3)
 ; CHECK-NEXT:    add 3, 3, 4
-; CHECK-NEXT:    mtvsrwa 0, 3
+; CHECK-NEXT:    mtfprwa 0, 3
 ; CHECK-NEXT:    xscvsxdsp 1, 0
 ; CHECK-NEXT:    blr
 entry:
diff --git a/llvm/test/CodeGen/PowerPC/setcc-logic.ll b/llvm/test/CodeGen/PowerPC/setcc-logic.ll
index 87f73e7b2..05eaad8b5 100644
--- a/llvm/test/CodeGen/PowerPC/setcc-logic.ll
+++ b/llvm/test/CodeGen/PowerPC/setcc-logic.ll
@@ -325,7 +325,7 @@ define <4 x i1> @all_bits_clear_vec(<4 x i32> %P, <4 x i32> %Q) {
 define <4 x i1> @all_sign_bits_clear_vec(<4 x i32> %P, <4 x i32> %Q) {
 ; CHECK-LABEL: all_sign_bits_clear_vec:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vspltisb 4, -1
+; CHECK-NEXT:    xxleqv 36, 36, 36
 ; CHECK-NEXT:    xxlor 34, 34, 35
 ; CHECK-NEXT:    vcmpgtsw 2, 2, 4
 ; CHECK-NEXT:    blr
@@ -338,7 +338,7 @@ define <4 x i1> @all_sign_bits_clear_vec(<4 x i32> %P, <4 x i32> %Q) {
 define <4 x i1> @all_bits_set_vec(<4 x i32> %P, <4 x i32> %Q) {
 ; CHECK-LABEL: all_bits_set_vec:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vspltisb 4, -1
+; CHECK-NEXT:    xxleqv 36, 36, 36
 ; CHECK-NEXT:    xxland 34, 34, 35
 ; CHECK-NEXT:    vcmpequw 2, 2, 4
 ; CHECK-NEXT:    blr
@@ -391,7 +391,7 @@ define <4 x i1> @any_sign_bits_set_vec(<4 x i32> %P, <4 x i32> %Q) {
 define <4 x i1> @any_bits_clear_vec(<4 x i32> %P, <4 x i32> %Q) {
 ; CHECK-LABEL: any_bits_clear_vec:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vspltisb 4, -1
+; CHECK-NEXT:    xxleqv 36, 36, 36
 ; CHECK-NEXT:    xxland 34, 34, 35
 ; CHECK-NEXT:    vcmpequw 2, 2, 4
 ; CHECK-NEXT:    xxlnor 34, 34, 34
@@ -405,7 +405,7 @@ define <4 x i1> @any_bits_clear_vec(<4 x i32> %P, <4 x i32> %Q) {
 define <4 x i1> @any_sign_bits_clear_vec(<4 x i32> %P, <4 x i32> %Q) {
 ; CHECK-LABEL: any_sign_bits_clear_vec:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vspltisb 4, -1
+; CHECK-NEXT:    xxleqv 36, 36, 36
 ; CHECK-NEXT:    xxland 34, 34, 35
 ; CHECK-NEXT:    vcmpgtsw 2, 2, 4
 ; CHECK-NEXT:    blr
diff --git a/llvm/test/CodeGen/PowerPC/setrnd.ll b/llvm/test/CodeGen/PowerPC/setrnd.ll
index c0f242ce9..40d99c878 100644
--- a/llvm/test/CodeGen/PowerPC/setrnd.ll
+++ b/llvm/test/CodeGen/PowerPC/setrnd.ll
@@ -26,7 +26,7 @@ entry:
 ; CHECK-DAG:   mffs 1 
 ; CHECK-DAG:   mffprd [[REG1:[0-9]+]], 1 
 ; CHECK-DAG:   rldimi [[REG1]], 3, 0, 62
-; CHECK-DAG:   mtvsrd [[REG2:[0-9]+]], [[REG1]]
+; CHECK-DAG:   mtfprd [[REG2:[0-9]+]], [[REG1]]
 ; CHECK-DAG:   mtfsf 255, [[REG2]] 
 ; CHECK:       blr
 
diff --git a/llvm/test/CodeGen/PowerPC/sext-vector-inreg.ll b/llvm/test/CodeGen/PowerPC/sext-vector-inreg.ll
new file mode 100644
index 000000000..11d4d1529
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/sext-vector-inreg.ll
@@ -0,0 +1,22 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -mcpu=pwr9 < %s | FileCheck -check-prefix=CHECK-P9 %s
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -mcpu=pwr8 < %s | FileCheck -check-prefix=CHECK-P8 %s
+define <4 x i32> @test_signext_vector_inreg(<4 x i16> %n) {
+; CHECK-P9-LABEL: test_signext_vector_inreg:
+; CHECK-P9:       # %bb.0: # %entry
+; CHECK-P9-NEXT:    vmrglh 2, 2, 2
+; CHECK-P9-NEXT:    vextsh2w 2, 2
+; CHECK-P9-NEXT:    blr
+;
+; CHECK-P8-LABEL: test_signext_vector_inreg:
+; CHECK-P8:       # %bb.0: # %entry
+; CHECK-P8-NEXT:    vmrglh 2, 2, 2
+; CHECK-P8-NEXT:    vspltisw 3, 8
+; CHECK-P8-NEXT:    vadduwm 3, 3, 3
+; CHECK-P8-NEXT:    vslw 2, 2, 3
+; CHECK-P8-NEXT:    vsraw 2, 2, 3
+; CHECK-P8-NEXT:    blr
+entry:
+  %0 = sext <4 x i16> %n to <4 x i32>
+  ret <4 x i32> %0
+}
diff --git a/llvm/test/CodeGen/PowerPC/shift-cmp.ll b/llvm/test/CodeGen/PowerPC/shift-cmp.ll
index ffbcd6b85..d580ad007 100644
--- a/llvm/test/CodeGen/PowerPC/shift-cmp.ll
+++ b/llvm/test/CodeGen/PowerPC/shift-cmp.ll
@@ -6,8 +6,7 @@ target triple = "powerpc64le-unknown-linux-gnu"
 define i1 @and_cmp_variable_power_of_two(i32 %x, i32 %y) {
 ; CHECK-LABEL: and_cmp_variable_power_of_two:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    subfic 4, 4, 32
-; CHECK-NEXT:    rlwnm 3, 3, 4, 31, 31
+; CHECK-NEXT:    srw 3, 3, 4
 ; CHECK-NEXT:    blr
   %shl = shl i32 1, %y
   %and = and i32 %x, %shl
@@ -18,8 +17,7 @@ define i1 @and_cmp_variable_power_of_two(i32 %x, i32 %y) {
 define i1 @and_cmp_variable_power_of_two_64(i64 %x, i64 %y) {
 ; CHECK-LABEL: and_cmp_variable_power_of_two_64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    subfic 4, 4, 64
-; CHECK-NEXT:    rldcl 3, 3, 4, 63
+; CHECK-NEXT:    srd 3, 3, 4
 ; CHECK-NEXT:    blr
   %shl = shl i64 1, %y
   %and = and i64 %x, %shl
@@ -30,9 +28,8 @@ define i1 @and_cmp_variable_power_of_two_64(i64 %x, i64 %y) {
 define i1 @and_ncmp_variable_power_of_two(i32 %x, i32 %y) {
 ; CHECK-LABEL: and_ncmp_variable_power_of_two:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    subfic 4, 4, 32
-; CHECK-NEXT:    nor 3, 3, 3
-; CHECK-NEXT:    rlwnm 3, 3, 4, 31, 31
+; CHECK-NEXT:    srw 3, 3, 4
+; CHECK-NEXT:    xori 3, 3, 1
 ; CHECK-NEXT:    blr
   %shl = shl i32 1, %y
   %and = and i32 %x, %shl
@@ -43,9 +40,8 @@ define i1 @and_ncmp_variable_power_of_two(i32 %x, i32 %y) {
 define i1 @and_ncmp_variable_power_of_two_64(i64 %x, i64 %y) {
 ; CHECK-LABEL: and_ncmp_variable_power_of_two_64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    not 3, 3
-; CHECK-NEXT:    subfic 4, 4, 64
-; CHECK-NEXT:    rldcl 3, 3, 4, 63
+; CHECK-NEXT:    srd 3, 3, 4
+; CHECK-NEXT:    xori 3, 3, 1
 ; CHECK-NEXT:    blr
   %shl = shl i64 1, %y
   %and = and i64 %x, %shl
diff --git a/llvm/test/CodeGen/PowerPC/shrink-wrap.ll b/llvm/test/CodeGen/PowerPC/shrink-wrap.ll
new file mode 100644
index 000000000..74a83fee1
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/shrink-wrap.ll
@@ -0,0 +1,54 @@
+; RUN: llc -verify-machineinstrs < %s -mtriple=powerpc64le-unknown-unknown -mcpu=pwr9 | FileCheck  %s
+define signext i32 @shrinkwrapme(i32 signext %a, i32 signext %lim) {
+entry:
+  %cmp5 = icmp sgt i32 %lim, 0
+  br i1 %cmp5, label %for.body.preheader, label %for.cond.cleanup
+
+ for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+ for.cond.cleanup.loopexit:                        ; preds = %for.body
+  br label %for.cond.cleanup
+
+ for.cond.cleanup:                                 ; preds = %for.cond.cleanup.loopexit, %entry
+  %Ret.0.lcssa = phi i32 [ 0, %entry ], [ %0, %for.cond.cleanup.loopexit ]
+  ret i32 %Ret.0.lcssa
+
+ for.body:                                         ; preds = %for.body.preheader, %for.body
+  %i.07 = phi i32 [ %inc, %for.body ], [ 0, %for.body.preheader ]
+  %Ret.06 = phi i32 [ %0, %for.body ], [ 0, %for.body.preheader ]
+  %0 = tail call i32 asm "add $0, $1, $2", "=r,r,r,~{r14},~{r15},~{r16},~{r17},~{r18},~{r19},~{r20},~{r21},~{r22},~{r23},~{r24},~{r25},~{r26},~{r27},~{r28},~{r29},~{r30},~{r31}"(i32 %a, i32 %Ret.06)
+  %inc = add nuw nsw i32 %i.07, 1
+  %exitcond = icmp eq i32 %inc, %lim
+  br i1 %exitcond, label %for.cond.cleanup.loopexit, label %for.body
+
+; CHECK-LABEL: shrinkwrapme
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    cmpwi
+; Prolog code
+; CHECK:         std 
+; CHECK:         std 
+; CHECK:         std 
+; CHECK:         std 
+; CHECK:         blt 0, .LBB0_3
+; CHECK:       # %bb.1:
+; CHECK-NEXT:    addi
+; CHECK-NEXT:    clrldi
+; CHECK-NEXT:    addi
+; CHECK-NEXT:    mtctr
+; CHECK-NEXT:    li 
+; CHECK:       .LBB0_2: 
+; CHECK:         add
+; CHECK:         bdnz .LBB0_2 
+; CHECK-NEXT:    b .LBB0_4
+; CHECK:       .LBB0_3: 
+; CHECK-NEXT:    li 
+; CHECK:       .LBB0_4: 
+; Epilog code
+; CHECK:         ld 
+; CHECK:         ld 
+; CHECK:         extsw
+; CHECK:         ld 
+; CHECK:         ld 
+; CHECK:         blr
+}
diff --git a/llvm/test/CodeGen/PowerPC/signbit-shift.ll b/llvm/test/CodeGen/PowerPC/signbit-shift.ll
index d87a24128..d66b7fc51 100644
--- a/llvm/test/CodeGen/PowerPC/signbit-shift.ll
+++ b/llvm/test/CodeGen/PowerPC/signbit-shift.ll
@@ -29,7 +29,7 @@ define i32 @add_zext_ifpos(i32 %x) {
 define <4 x i32> @add_zext_ifpos_vec_splat(<4 x i32> %x) {
 ; CHECK-LABEL: add_zext_ifpos_vec_splat:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vspltisb 3, -1
+; CHECK-NEXT:    xxleqv 35, 35, 35
 ; CHECK-NEXT:    addis 3, 2, .LCPI2_0@toc@ha
 ; CHECK-NEXT:    addi 3, 3, .LCPI2_0@toc@l
 ; CHECK-NEXT:    vcmpgtsw 2, 2, 3
@@ -81,7 +81,7 @@ define i32 @add_sext_ifpos(i32 %x) {
 define <4 x i32> @add_sext_ifpos_vec_splat(<4 x i32> %x) {
 ; CHECK-LABEL: add_sext_ifpos_vec_splat:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vspltisb 3, -1
+; CHECK-NEXT:    xxleqv 35, 35, 35
 ; CHECK-NEXT:    addis 3, 2, .LCPI6_0@toc@ha
 ; CHECK-NEXT:    addi 3, 3, .LCPI6_0@toc@l
 ; CHECK-NEXT:    vcmpgtsw 2, 2, 3
diff --git a/llvm/test/CodeGen/PowerPC/sjlj.ll b/llvm/test/CodeGen/PowerPC/sjlj.ll
index c5dba12c1..7c8f83e6e 100644
--- a/llvm/test/CodeGen/PowerPC/sjlj.ll
+++ b/llvm/test/CodeGen/PowerPC/sjlj.ll
@@ -81,8 +81,8 @@ return:                                           ; preds = %if.end, %if.then
 ; CHECK: # %bb.1:
 
 ; CHECK: .LBB1_3:
-; CHECK: mflr [[REGL:[0-9]+]]
 ; CHECK: ld [[REG2:[0-9]+]], [[OFF]](31)                   # 8-byte Folded Reload
+; CHECK: mflr [[REGL:[0-9]+]]
 ; CHECK: std [[REGL]], 8([[REG2]])
 ; CHECK: li 3, 0
 
@@ -172,7 +172,7 @@ declare i8* @llvm.stacksave() #3
 
 declare i32 @llvm.eh.sjlj.setjmp(i8*) #3
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { noreturn nounwind }
 attributes #2 = { nounwind readnone }
 attributes #3 = { nounwind }
diff --git a/llvm/test/CodeGen/PowerPC/sms-cpy-1.ll b/llvm/test/CodeGen/PowerPC/sms-cpy-1.ll
new file mode 100644
index 000000000..7804b0a3f
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/sms-cpy-1.ll
@@ -0,0 +1,113 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=powerpc64le-unknown-linux-gnu -verify-machineinstrs\
+; RUN:       -mcpu=pwr9 --ppc-enable-pipeliner 2>&1 | FileCheck %s
+
+@.str.28 = external unnamed_addr constant [69 x i8], align 1
+
+define void @print_res() nounwind {
+; CHECK-LABEL: print_res:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mflr 0
+; CHECK-NEXT:    std 0, 16(1)
+; CHECK-NEXT:    stdu 1, -128(1)
+; CHECK-NEXT:    lwz 3, 0(3)
+; CHECK-NEXT:    addi 3, 3, -1
+; CHECK-NEXT:    clrldi 4, 3, 32
+; CHECK-NEXT:    cmplwi 0, 3, 1
+; CHECK-NEXT:    li 3, 1
+; CHECK-NEXT:    isel 3, 4, 3, 1
+; CHECK-NEXT:    li 4, 2
+; CHECK-NEXT:    addi 3, 3, -1
+; CHECK-NEXT:    cmpldi 3, 2
+; CHECK-NEXT:    isel 3, 3, 4, 0
+; CHECK-NEXT:    li 4, 0
+; CHECK-NEXT:    addi 3, 3, 1
+; CHECK-NEXT:    li 5, 0
+; CHECK-NEXT:    li 7, -1
+; CHECK-NEXT:    mtctr 3
+; CHECK-NEXT:    lbz 5, 0(5)
+; CHECK-NEXT:    li 3, 1
+; CHECK-NEXT:    bdz .LBB0_6
+; CHECK-NEXT:  # %bb.1:
+; CHECK-NEXT:    xori 6, 5, 84
+; CHECK-NEXT:    clrldi 5, 7, 32
+; CHECK-NEXT:    lbz 5, 0(5)
+; CHECK-NEXT:    addi 3, 3, 1
+; CHECK-NEXT:    addi 8, 7, -1
+; CHECK-NEXT:    bdz .LBB0_5
+; CHECK-NEXT:  # %bb.2:
+; CHECK-NEXT:    cntlzw 6, 6
+; CHECK-NEXT:    srwi 7, 6, 5
+; CHECK-NEXT:    xori 6, 5, 84
+; CHECK-NEXT:    clrldi 5, 8, 32
+; CHECK-NEXT:    addi 8, 8, -1
+; CHECK-NEXT:    lbz 5, 0(5)
+; CHECK-NEXT:    addi 3, 3, 1
+; CHECK-NEXT:    bdz .LBB0_4
+; CHECK-NEXT:  .LBB0_3: #
+; CHECK-NEXT:    clrldi 10, 8, 32
+; CHECK-NEXT:    cntlzw 9, 6
+; CHECK-NEXT:    xori 6, 5, 84
+; CHECK-NEXT:    lbz 5, 0(10)
+; CHECK-NEXT:    addi 8, 8, -1
+; CHECK-NEXT:    addi 3, 3, 1
+; CHECK-NEXT:    add 4, 4, 7
+; CHECK-NEXT:    srwi 7, 9, 5
+; CHECK-NEXT:    bdnz .LBB0_3
+; CHECK-NEXT:  .LBB0_4:
+; CHECK-NEXT:    add 4, 4, 7
+; CHECK-NEXT:  .LBB0_5:
+; CHECK-NEXT:    cntlzw 6, 6
+; CHECK-NEXT:    srwi 6, 6, 5
+; CHECK-NEXT:    add 4, 4, 6
+; CHECK-NEXT:  .LBB0_6:
+; CHECK-NEXT:    xori 5, 5, 84
+; CHECK-NEXT:    cntlzw 5, 5
+; CHECK-NEXT:    clrldi 3, 3, 32
+; CHECK-NEXT:    std 3, 104(1)
+; CHECK-NEXT:    addis 3, 2, .LC0@toc@ha
+; CHECK-NEXT:    ld 3, .LC0@toc@l(3)
+; CHECK-NEXT:    li 7, 0
+; CHECK-NEXT:    li 8, 3
+; CHECK-NEXT:    srwi 5, 5, 5
+; CHECK-NEXT:    add 4, 4, 5
+; CHECK-NEXT:    li 5, 0
+; CHECK-NEXT:    std 5, 120(1)
+; CHECK-NEXT:    li 5, 3
+; CHECK-NEXT:    std 5, 96(1)
+; CHECK-NEXT:    clrldi 6, 4, 32
+; CHECK-NEXT:    li 4, 3
+; CHECK-NEXT:    li 5, 0
+; CHECK-NEXT:    li 10, 0
+; CHECK-NEXT:    bl printf
+; CHECK-NEXT:    nop
+  %1 = load i32, i32* undef, align 4
+  %2 = add i32 %1, -1
+  %3 = zext i32 %2 to i64
+  %4 = zext i32 3 to i64
+  br label %5
+
+5:                                                ; preds = %5, %0
+  %6 = phi i64 [ %16, %5 ], [ 0, %0 ]
+  %7 = phi i32 [ %15, %5 ], [ 0, %0 ]
+  %8 = trunc i64 %6 to i32
+  %9 = sub i32 0, %8
+  %10 = zext i32 %9 to i64
+  %11 = getelementptr inbounds i8, i8* null, i64 %10
+  %12 = load i8, i8* %11, align 1
+  %13 = icmp eq i8 %12, 84
+  %14 = zext i1 %13 to i32
+  %15 = add i32 %7, %14
+  %16 = add nuw nsw i64 %6, 1
+  %17 = icmp ult i64 %16, %3
+  %18 = icmp ult i64 %16, %4
+  %19 = and i1 %18, %17
+  br i1 %19, label %5, label %20
+
+20:                                               ; preds = %5
+  %21 = trunc i64 %16 to i32
+  call void (i8*, ...) @printf(i8* getelementptr inbounds ([69 x i8], [69 x i8]* @.str.28, i64 0, i64 0), i32 zeroext 3, i32 zeroext undef, i32 zeroext %15, i32 zeroext undef, i32 zeroext 3, i8* undef, i32 zeroext undef, i32 zeroext 3, i32 zeroext %21, i8* undef, i32 zeroext undef) #1
+  unreachable
+}
+
+declare void @printf(i8*, ...) local_unnamed_addr #0
diff --git a/llvm/test/CodeGen/PowerPC/sms-grp-order.ll b/llvm/test/CodeGen/PowerPC/sms-grp-order.ll
index a3e6de1c5..c462e18d9 100644
--- a/llvm/test/CodeGen/PowerPC/sms-grp-order.ll
+++ b/llvm/test/CodeGen/PowerPC/sms-grp-order.ll
@@ -4,25 +4,24 @@
 
 define void @lame_encode_buffer_interleaved() local_unnamed_addr {
 ; CHECK-LABEL: lame_encode_buffer_interleaved:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    lhz 3, 0(0)
-; CHECK-NEXT:    li 5, 1
-; CHECK-NEXT:    sldi 5, 5, 62
-; CHECK-NEXT:    lhz 4, 0(3)
-; CHECK-NEXT:    mtctr 5
-; CHECK-NEXT:    .p2align 5
-; CHECK-NEXT:  .LBB0_1: #
-; CHECK-NEXT:    extsh 3, 3
-; CHECK-NEXT:    extsh 4, 4
-; CHECK-NEXT:    srawi 3, 3, 1
-; CHECK-NEXT:    addze 3, 3
-; CHECK-NEXT:    srawi 4, 4, 1
-; CHECK-NEXT:    addze 4, 4
-; CHECK-NEXT:    bdnz .LBB0_1
-; CHECK-NEXT:  # %bb.2:
-; CHECK-NEXT:    sth 3, 0(0)
-; CHECK-NEXT:    sth 4, 0(3)
-; CHECK-NEXT:    blr
+; CHECK:      # %bb.0:
+; CHECK-NEXT:   lha 3, 0(3)
+; CHECK-NEXT:   li 5, 1
+; CHECK-NEXT:   sldi 5, 5, 62
+; CHECK-NEXT:   lhz 4, 0(0)
+; CHECK-NEXT:   mtctr 5
+; CHECK-NEXT:   srawi 3, 3, 1
+; CHECK-NEXT:   addze 3, 3
+; CHECK-NEXT:   .p2align 4
+; CHECK-NEXT: .LBB0_1:
+; CHECK-NEXT:   extsh 4, 4
+; CHECK-NEXT:   srawi 4, 4, 1
+; CHECK-NEXT:   addze 4, 4
+; CHECK-NEXT:   bdnz .LBB0_1
+; CHECK-NEXT: # %bb.2:
+; CHECK-NEXT:   sth 4, 0(0)
+; CHECK-NEXT:   sth 3, 0(3)
+; CHECK-NEXT:   blr
   br label %1
 
 1:                                                ; preds = %1, %0
diff --git a/llvm/test/CodeGen/PowerPC/sms-phi-1.ll b/llvm/test/CodeGen/PowerPC/sms-phi-1.ll
new file mode 100644
index 000000000..5cb685e98
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/sms-phi-1.ll
@@ -0,0 +1,64 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=powerpc64le-unknown-linux-gnu -verify-machineinstrs\
+; RUN:       -mcpu=pwr9 --ppc-enable-pipeliner 2>&1 | FileCheck %s
+
+define void @main() nounwind #0 {
+; CHECK-LABEL: main:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mflr 0
+; CHECK-NEXT:    std 30, -16(1) # 8-byte Folded Spill
+; CHECK-NEXT:    std 0, 16(1)
+; CHECK-NEXT:    stdu 1, -48(1)
+; CHECK-NEXT:    bl strtol
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mr 30, 3
+; CHECK-NEXT:    bl calloc
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi 4, 30, 32
+; CHECK-NEXT:    li 5, 0
+; CHECK-NEXT:    addi 3, 3, -4
+; CHECK-NEXT:    mtctr 4
+; CHECK-NEXT:    mullw 4, 5, 5
+; CHECK-NEXT:    li 6, 1
+; CHECK-NEXT:    bdz .LBB0_3
+; CHECK-NEXT:  # %bb.1:
+; CHECK-NEXT:    stwu 4, 4(3)
+; CHECK-NEXT:    mullw 4, 6, 6
+; CHECK-NEXT:    addi 5, 6, 1
+; CHECK-NEXT:    bdz .LBB0_3
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  .LBB0_2: #
+; CHECK-NEXT:    stwu 4, 4(3)
+; CHECK-NEXT:    mullw 4, 5, 5
+; CHECK-NEXT:    addi 5, 5, 1
+; CHECK-NEXT:    bdnz .LBB0_2
+; CHECK-NEXT:  .LBB0_3:
+; CHECK-NEXT:    stwu 4, 4(3)
+; CHECK-NEXT:    addi 1, 1, 48
+; CHECK-NEXT:    ld 0, 16(1)
+; CHECK-NEXT:    ld 30, -16(1) # 8-byte Folded Reload
+; CHECK-NEXT:    mtlr 0
+; CHECK-NEXT:    blr
+  %1 = tail call i64 @strtol()
+  %2 = trunc i64 %1 to i32
+  %3 = tail call noalias i8* @calloc()
+  %4 = bitcast i8* %3 to i32*
+  %5 = zext i32 %2 to i64
+  br label %6
+
+6:                                                ; preds = %6, %0
+  %7 = phi i64 [ %11, %6 ], [ 0, %0 ]
+  %8 = trunc i64 %7 to i32
+  %9 = mul nsw i32 %8, %8
+  %10 = getelementptr inbounds i32, i32* %4, i64 %7
+  store i32 %9, i32* %10, align 4
+  %11 = add nuw nsw i64 %7, 1
+  %12 = icmp eq i64 %11, %5
+  br i1 %12, label %13, label %6
+
+13:                                               ; preds = %6
+  ret void
+}
+
+declare i8* @calloc() local_unnamed_addr
+declare i64 @strtol() local_unnamed_addr
diff --git a/llvm/test/CodeGen/PowerPC/sms-phi-2.ll b/llvm/test/CodeGen/PowerPC/sms-phi-2.ll
new file mode 100644
index 000000000..eb60903f8
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/sms-phi-2.ll
@@ -0,0 +1,69 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=powerpc64le-unknown-linux-gnu -verify-machineinstrs\
+; RUN:       -mcpu=pwr9 --ppc-enable-pipeliner 2>&1 | FileCheck %s
+
+define void @phi2(i32, i32, i8*) local_unnamed_addr {
+; CHECK-LABEL: phi2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    divw 8, 3, 4
+; CHECK-NEXT:    li 5, 55
+; CHECK-NEXT:    li 6, 48
+; CHECK-NEXT:    mtctr 3
+; CHECK-NEXT:    bdz .LBB0_4
+; CHECK-NEXT:  # %bb.1:
+; CHECK-NEXT:    divw 9, 8, 4
+; CHECK-NEXT:    mullw 7, 8, 4
+; CHECK-NEXT:    subf 3, 7, 3
+; CHECK-NEXT:    cmplwi 3, 10
+; CHECK-NEXT:    isel 7, 6, 5, 0
+; CHECK-NEXT:    add 3, 7, 3
+; CHECK-NEXT:    stbu 3, -1(7)
+; CHECK-NEXT:    mr 3, 8
+; CHECK-NEXT:    bdz .LBB0_3
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  .LBB0_2: #
+; CHECK-NEXT:    mr 3, 9
+; CHECK-NEXT:    mullw 9, 9, 4
+; CHECK-NEXT:    divw 10, 3, 4
+; CHECK-NEXT:    subf 8, 9, 8
+; CHECK-NEXT:    cmplwi 8, 10
+; CHECK-NEXT:    isel 9, 6, 5, 0
+; CHECK-NEXT:    add 8, 9, 8
+; CHECK-NEXT:    mr 9, 10
+; CHECK-NEXT:    stbu 8, -1(7)
+; CHECK-NEXT:    mr 8, 3
+; CHECK-NEXT:    bdnz .LBB0_2
+; CHECK-NEXT:  .LBB0_3:
+; CHECK-NEXT:    mr 8, 9
+; CHECK-NEXT:    b .LBB0_5
+; CHECK-NEXT:  .LBB0_4:
+; CHECK-NEXT:    # implicit-def: $x7
+; CHECK-NEXT:  .LBB0_5:
+; CHECK-NEXT:    mullw 4, 8, 4
+; CHECK-NEXT:    subf 3, 4, 3
+; CHECK-NEXT:    cmplwi 3, 10
+; CHECK-NEXT:    isel 4, 6, 5, 0
+; CHECK-NEXT:    add 3, 4, 3
+; CHECK-NEXT:    stbu 3, -1(7)
+; CHECK-NEXT:    blr
+  br label %4
+
+4:                                                ; preds = %4, %3
+  %5 = phi i64 [ %7, %4 ], [ undef, %3 ]
+  %6 = phi i32 [ %8, %4 ], [ %0, %3 ]
+  %7 = add nsw i64 %5, -1
+  %8 = sdiv i32 %6, %1
+  %9 = mul nsw i32 %8, %1
+  %10 = sub nsw i32 %6, %9
+  %11 = icmp ult i32 %10, 10
+  %12 = trunc i32 %10 to i8
+  %13 = select i1 %11, i8 48, i8 55
+  %14 = add i8 %13, %12
+  %15 = getelementptr inbounds i8, i8* %2, i64 %7
+  store i8 %14, i8* %15, align 1
+  %16 = icmp sgt i64 %5, 1
+  br i1 %16, label %4, label %17
+
+17:                                               ; preds = %4
+  ret void
+}
diff --git a/llvm/test/CodeGen/PowerPC/sms-phi-3.ll b/llvm/test/CodeGen/PowerPC/sms-phi-3.ll
new file mode 100644
index 000000000..cdb5100b2
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/sms-phi-3.ll
@@ -0,0 +1,85 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=powerpc64le-unknown-linux-gnu -verify-machineinstrs\
+; RUN:       -mcpu=pwr9 --ppc-enable-pipeliner 2>&1 | FileCheck %s
+
+%0 = type { double, double, double, i32, i32 }
+declare i8* @malloc() local_unnamed_addr
+
+define void @phi3(i32*) nounwind {
+; CHECK-LABEL: phi3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mflr 0
+; CHECK-NEXT:    std 29, -24(1) # 8-byte Folded Spill
+; CHECK-NEXT:    std 30, -16(1) # 8-byte Folded Spill
+; CHECK-NEXT:    std 0, 16(1)
+; CHECK-NEXT:    stdu 1, -64(1)
+; CHECK-NEXT:    mr 30, 3
+; CHECK-NEXT:    bl malloc
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mr 29, 3
+; CHECK-NEXT:    bl malloc
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    addi 7, 30, -4
+; CHECK-NEXT:    mtctr 3
+; CHECK-NEXT:    lwzu 8, 4(7)
+; CHECK-NEXT:    addi 4, 29, -8
+; CHECK-NEXT:    li 5, 0
+; CHECK-NEXT:    bdz .LBB0_5
+; CHECK-NEXT:  # %bb.1:
+; CHECK-NEXT:    extswsli 6, 5, 5
+; CHECK-NEXT:    add 5, 8, 5
+; CHECK-NEXT:    lwzu 8, 4(7)
+; CHECK-NEXT:    bdz .LBB0_4
+; CHECK-NEXT:  # %bb.2:
+; CHECK-NEXT:    add 6, 3, 6
+; CHECK-NEXT:    stdu 6, 8(4)
+; CHECK-NEXT:    extswsli 6, 5, 5
+; CHECK-NEXT:    add 5, 8, 5
+; CHECK-NEXT:    lwzu 8, 4(7)
+; CHECK-NEXT:    bdz .LBB0_4
+; CHECK-NEXT:    .p2align 5
+; CHECK-NEXT:  .LBB0_3: #
+; CHECK-NEXT:    add 9, 3, 6
+; CHECK-NEXT:    extswsli 6, 5, 5
+; CHECK-NEXT:    add 5, 8, 5
+; CHECK-NEXT:    lwzu 8, 4(7)
+; CHECK-NEXT:    stdu 9, 8(4)
+; CHECK-NEXT:    bdnz .LBB0_3
+; CHECK-NEXT:  .LBB0_4:
+; CHECK-NEXT:    add 6, 3, 6
+; CHECK-NEXT:    stdu 6, 8(4)
+; CHECK-NEXT:  .LBB0_5:
+; CHECK-NEXT:    extswsli 5, 5, 5
+; CHECK-NEXT:    add 3, 3, 5
+; CHECK-NEXT:    stdu 3, 8(4)
+; CHECK-NEXT:    addi 1, 1, 64
+; CHECK-NEXT:    ld 0, 16(1)
+; CHECK-NEXT:    ld 30, -16(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 29, -24(1) # 8-byte Folded Reload
+; CHECK-NEXT:    mtlr 0
+; CHECK-NEXT:    blr
+  %2 = tail call noalias i8* @malloc()
+  %3 = bitcast i8* %2 to %0**
+  %4 = tail call noalias i8* @malloc()
+  %5 = bitcast i8* %4 to %0*
+  br label %6
+
+6:                                                ; preds = %6, %1
+  %7 = phi i64 [ %16, %6 ], [ 0, %1 ]
+  %8 = phi i32 [ %15, %6 ], [ 0, %1 ]
+  %9 = phi i64 [ %17, %6 ], [ undef, %1 ]
+  %10 = sext i32 %8 to i64
+  %11 = getelementptr inbounds %0, %0* %5, i64 %10
+  %12 = getelementptr inbounds %0*, %0** %3, i64 %7
+  store %0* %11, %0** %12, align 8
+  %13 = getelementptr inbounds i32, i32* %0, i64 %7
+  %14 = load i32, i32* %13, align 4
+  %15 = add nsw i32 %14, %8
+  %16 = add nuw nsw i64 %7, 1
+  %17 = add i64 %9, -1
+  %18 = icmp eq i64 %17, 0
+  br i1 %18, label %19, label %6
+
+19:                                               ; preds = %6
+  ret void
+}
diff --git a/llvm/test/CodeGen/PowerPC/sms-phi-5.ll b/llvm/test/CodeGen/PowerPC/sms-phi-5.ll
new file mode 100644
index 000000000..bdc773de8
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/sms-phi-5.ll
@@ -0,0 +1,56 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=powerpc64le-unknown-linux-gnu -verify-machineinstrs\
+; RUN:       -mcpu=pwr9 --ppc-enable-pipeliner 2>&1 | FileCheck %s
+
+define void @phi5() unnamed_addr {
+; CHECK-LABEL: phi5:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:  # %bb.1:
+; CHECK-NEXT:    li 3, 0
+; CHECK-NEXT:    li 4, 1
+; CHECK-NEXT:    slw 3, 4, 3
+; CHECK-NEXT:    andi. 3, 3, 6336
+; CHECK-NEXT:    beqlr 0
+; CHECK-NEXT:  # %bb.2:
+; CHECK-NEXT:    lhz 3, 0(3)
+; CHECK-NEXT:    slwi 3, 3, 15
+; CHECK-NEXT:    clrlwi 3, 3, 31
+; CHECK-NEXT:    rlwinm 4, 3, 31, 17, 31
+; CHECK-NEXT:    or 3, 3, 4
+; CHECK-NEXT:    rlwimi 3, 3, 15, 0, 16
+; CHECK-NEXT:  # %bb.3:
+; CHECK-NEXT:    blr
+  switch i12 undef, label %21 [
+    i12 6, label %1
+    i12 7, label %1
+    i12 12, label %1
+    i12 11, label %1
+  ]
+
+1:                                                ; preds = %0, %0, %0, %0
+  %2 = load i16, i16* undef, align 2
+  br label %3
+
+3:                                                ; preds = %3, %1
+  %4 = phi i16 [ %18, %3 ], [ undef, %1 ]
+  %5 = phi i16 [ %13, %3 ], [ undef, %1 ]
+  %6 = phi i16 [ %11, %3 ], [ undef, %1 ]
+  %7 = phi i16 [ undef, %3 ], [ %2, %1 ]
+  %8 = phi i32 [ %19, %3 ], [ undef, %1 ]
+  %9 = lshr i16 %6, 1
+  %10 = shl i16 %7, 15
+  %11 = or i16 %10, %9
+  %12 = shl i16 %6, 15
+  %13 = or i16 %12, 0
+  %14 = and i16 %4, 1
+  %15 = lshr i16 %4, 1
+  %16 = shl i16 %5, 15
+  %17 = or i16 %14, %15
+  %18 = or i16 %17, %16
+  %19 = add i32 %8, -1
+  %20 = icmp eq i32 %19, 0
+  br i1 %20, label %21, label %3
+
+21:                                               ; preds = %3, %0
+  ret void
+}
diff --git a/llvm/test/CodeGen/PowerPC/sms-simple.ll b/llvm/test/CodeGen/PowerPC/sms-simple.ll
index 6b1f0e453..a5b5acdc9 100644
--- a/llvm/test/CodeGen/PowerPC/sms-simple.ll
+++ b/llvm/test/CodeGen/PowerPC/sms-simple.ll
@@ -4,10 +4,9 @@
 ; RUN:       | FileCheck %s
 
 @x = dso_local local_unnamed_addr global <{ i32, i32, i32, i32, [1020 x i32] }> <{ i32 1, i32 2, i32 3, i32 4, [1020 x i32] zeroinitializer }>, align 4
-@y = common dso_local global [1024 x i32] zeroinitializer, align 4
+@y = dso_local global [1024 x i32] zeroinitializer, align 4
 
-; Function Attrs: norecurse nounwind
-define dso_local i32* @foo() local_unnamed_addr #0 {
+define dso_local i32* @foo() local_unnamed_addr {
 ; CHECK-LABEL: foo:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    addis r5, r2, x@toc@ha
@@ -27,7 +26,8 @@ define dso_local i32* @foo() local_unnamed_addr #0 {
 ; CHECK-NEXT:    lwz r7, 8(r5)
 ; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_1: # %for.body
-; CHECK:         maddld r7, r7, r7, r6
+; CHECK-NEXT:    #
+; CHECK-NEXT:    maddld r7, r7, r7, r6
 ; CHECK-NEXT:    lwzu r8, 12(r5)
 ; CHECK-NEXT:    maddld r8, r8, r8, r7
 ; CHECK-NEXT:    stw r6, 4(r4)
diff --git a/llvm/test/CodeGen/PowerPC/smulfixsat.ll b/llvm/test/CodeGen/PowerPC/smulfixsat.ll
new file mode 100644
index 000000000..cbba6f8f8
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/smulfixsat.ll
@@ -0,0 +1,52 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=ppc32 | FileCheck %s
+
+declare  i32 @llvm.smul.fix.sat.i32  (i32, i32, i32)
+
+define i32 @func1(i32 %x, i32 %y) nounwind {
+; CHECK-LABEL: func1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lis 5, 32767
+; CHECK-NEXT:    mulhw. 6, 3, 4
+; CHECK-NEXT:    lis 7, -32768
+; CHECK-NEXT:    mullw 3, 3, 4
+; CHECK-NEXT:    ori 4, 5, 65535
+; CHECK-NEXT:    srawi 5, 3, 31
+; CHECK-NEXT:    cmplw 1, 6, 5
+; CHECK-NEXT:    bc 12, 0, .LBB0_1
+; CHECK-NEXT:    b .LBB0_2
+; CHECK-NEXT:  .LBB0_1:
+; CHECK-NEXT:    addi 4, 7, 0
+; CHECK-NEXT:  .LBB0_2:
+; CHECK-NEXT:    bclr 12, 6, 0
+; CHECK-NEXT:  # %bb.3:
+; CHECK-NEXT:    ori 3, 4, 0
+; CHECK-NEXT:    blr
+  %tmp = call i32 @llvm.smul.fix.sat.i32(i32 %x, i32 %y, i32 0)
+  ret i32 %tmp
+}
+
+define i32 @func2(i32 %x, i32 %y) nounwind {
+; CHECK-LABEL: func2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mulhw. 6, 3, 4
+; CHECK-NEXT:    lis 5, 32767
+; CHECK-NEXT:    mullw 3, 3, 4
+; CHECK-NEXT:    rotlwi 3, 3, 31
+; CHECK-NEXT:    ori 4, 5, 65535
+; CHECK-NEXT:    rlwimi 3, 6, 31, 0, 0
+; CHECK-NEXT:    bc 12, 1, .LBB1_1
+; CHECK-NEXT:    b .LBB1_2
+; CHECK-NEXT:  .LBB1_1:
+; CHECK-NEXT:    addi 3, 4, 0
+; CHECK-NEXT:  .LBB1_2:
+; CHECK-NEXT:    cmpwi 6, -1
+; CHECK-NEXT:    lis 4, -32768
+; CHECK-NEXT:    bc 12, 0, .LBB1_3
+; CHECK-NEXT:    blr
+; CHECK-NEXT:  .LBB1_3:
+; CHECK-NEXT:    addi 3, 4, 0
+; CHECK-NEXT:    blr
+  %tmp = call i32 @llvm.smul.fix.sat.i32(i32 %x, i32 %y, i32 1)
+  ret i32 %tmp
+}
diff --git a/llvm/test/CodeGen/PowerPC/spe.ll b/llvm/test/CodeGen/PowerPC/spe.ll
index bd03fa30e..90852ad43 100644
--- a/llvm/test/CodeGen/PowerPC/spe.ll
+++ b/llvm/test/CodeGen/PowerPC/spe.ll
@@ -1,273 +1,590 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -verify-machineinstrs < %s -mtriple=powerpc-unknown-linux-gnu \
 ; RUN:          -mattr=+spe |  FileCheck %s
 
 declare float @llvm.fabs.float(float)
 define float @test_float_abs(float %a) #0 {
+; CHECK-LABEL: test_float_abs:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    efsabs 3, 3
+; CHECK-NEXT:    blr
   entry:
     %0 = tail call float @llvm.fabs.float(float %a)
     ret float %0
-; CHECK-LABEL: test_float_abs
-; CHECK: efsabs 3, 3
-; CHECK: blr
 }
 
 define float @test_fnabs(float %a) #0 {
+; CHECK-LABEL: test_fnabs:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    efsnabs 3, 3
+; CHECK-NEXT:    blr
   entry:
     %0 = tail call float @llvm.fabs.float(float %a)
     %sub = fsub float -0.000000e+00, %0
     ret float %sub
-; CHECK-LABEL: @test_fnabs
-; CHECK: efsnabs
-; CHECK: blr
 }
 
 define float @test_fdiv(float %a, float %b) {
+; CHECK-LABEL: test_fdiv:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    efsdiv 3, 3, 4
+; CHECK-NEXT:    blr
 entry:
   %v = fdiv float %a, %b
   ret float %v
 
-; CHECK-LABEL: test_fdiv
-; CHECK: efsdiv
-; CHECK: blr
 }
 
 define float @test_fmul(float %a, float %b) {
+; CHECK-LABEL: test_fmul:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    efsmul 3, 3, 4
+; CHECK-NEXT:    blr
   entry:
   %v = fmul float %a, %b
   ret float %v
 ; CHECK-LABEL @test_fmul
-; CHECK: efsmul
-; CHECK: blr
 }
 
 define float @test_fadd(float %a, float %b) {
+; CHECK-LABEL: test_fadd:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    efsadd 3, 3, 4
+; CHECK-NEXT:    blr
   entry:
   %v = fadd float %a, %b
   ret float %v
 ; CHECK-LABEL @test_fadd
-; CHECK: efsadd
-; CHECK: blr
 }
 
 define float @test_fsub(float %a, float %b) {
+; CHECK-LABEL: test_fsub:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    efssub 3, 3, 4
+; CHECK-NEXT:    blr
   entry:
   %v = fsub float %a, %b
   ret float %v
 ; CHECK-LABEL @test_fsub
-; CHECK: efssub
-; CHECK: blr
 }
 
 define float @test_fneg(float %a) {
+; CHECK-LABEL: test_fneg:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    efsneg 3, 3
+; CHECK-NEXT:    blr
   entry:
   %v = fsub float -0.0, %a
   ret float %v
 
 ; CHECK-LABEL @test_fneg
-; CHECK: efsneg
-; CHECK: blr
 }
 
 define float @test_dtos(double %a) {
+; CHECK-LABEL: test_dtos:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    evmergelo 3, 3, 4
+; CHECK-NEXT:    efscfd 3, 3
+; CHECK-NEXT:    blr
   entry:
   %v = fptrunc double %a to float
   ret float %v
-; CHECK-LABEL: test_dtos
-; CHECK: efscfd
-; CHECK: blr
 }
 
-define i1 @test_fcmpgt(float %a, float %b) {
+define i32 @test_fcmpgt(float %a, float %b) {
+; CHECK-LABEL: test_fcmpgt:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    stwu 1, -16(1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    efscmpgt 0, 3, 4
+; CHECK-NEXT:    ble 0, .LBB8_2
+; CHECK-NEXT:  # %bb.1: # %tr
+; CHECK-NEXT:    li 3, 1
+; CHECK-NEXT:    b .LBB8_3
+; CHECK-NEXT:  .LBB8_2: # %fa
+; CHECK-NEXT:    li 3, 0
+; CHECK-NEXT:  .LBB8_3: # %ret
+; CHECK-NEXT:    stw 3, 12(1)
+; CHECK-NEXT:    lwz 3, 12(1)
+; CHECK-NEXT:    addi 1, 1, 16
+; CHECK-NEXT:    blr
   entry:
-  %r = fcmp ogt float %a, %b
-  ret i1 %r
-; CHECK-LABEL: test_fcmpgt
-; CHECK: efscmpgt
-; CHECK: blr
+  %r = alloca i32, align 4
+  %c = fcmp ogt float %a, %b
+  br i1 %c, label %tr, label %fa
+tr:
+  store i32 1, i32* %r, align 4
+  br label %ret
+fa:
+  store i32 0, i32* %r, align 4
+  br label %ret
+ret:
+  %0 = load i32, i32* %r, align 4
+  ret i32 %0
 }
 
-define i1 @test_fcmpugt(float %a, float %b) {
+define i32 @test_fcmpugt(float %a, float %b) {
+; CHECK-LABEL: test_fcmpugt:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    stwu 1, -16(1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    efscmpeq 0, 4, 4
+; CHECK-NEXT:    bc 4, 1, .LBB9_4
+; CHECK-NEXT:  # %bb.1: # %entry
+; CHECK-NEXT:    efscmpeq 0, 3, 3
+; CHECK-NEXT:    bc 4, 1, .LBB9_4
+; CHECK-NEXT:  # %bb.2: # %entry
+; CHECK-NEXT:    efscmpgt 0, 3, 4
+; CHECK-NEXT:    bc 12, 1, .LBB9_4
+; CHECK-NEXT:  # %bb.3: # %fa
+; CHECK-NEXT:    li 3, 0
+; CHECK-NEXT:    b .LBB9_5
+; CHECK-NEXT:  .LBB9_4: # %tr
+; CHECK-NEXT:    li 3, 1
+; CHECK-NEXT:  .LBB9_5: # %ret
+; CHECK-NEXT:    stw 3, 12(1)
+; CHECK-NEXT:    lwz 3, 12(1)
+; CHECK-NEXT:    addi 1, 1, 16
+; CHECK-NEXT:    blr
   entry:
-  %r = fcmp ugt float %a, %b
-  ret i1 %r
-; CHECK-LABEL: test_fcmpugt
-; CHECK: efscmpgt
-; CHECK: blr
+  %r = alloca i32, align 4
+  %c = fcmp ugt float %a, %b
+  br i1 %c, label %tr, label %fa
+tr:
+  store i32 1, i32* %r, align 4
+  br label %ret
+fa:
+  store i32 0, i32* %r, align 4
+  br label %ret
+ret:
+  %0 = load i32, i32* %r, align 4
+  ret i32 %0
 }
 
-define i1 @test_fcmple(float %a, float %b) {
+define i32 @test_fcmple(float %a, float %b) {
+; CHECK-LABEL: test_fcmple:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    stwu 1, -16(1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    efscmpeq 0, 3, 3
+; CHECK-NEXT:    bc 4, 1, .LBB10_4
+; CHECK-NEXT:  # %bb.1: # %entry
+; CHECK-NEXT:    efscmpeq 0, 4, 4
+; CHECK-NEXT:    bc 4, 1, .LBB10_4
+; CHECK-NEXT:  # %bb.2: # %entry
+; CHECK-NEXT:    efscmpgt 0, 3, 4
+; CHECK-NEXT:    bc 12, 1, .LBB10_4
+; CHECK-NEXT:  # %bb.3: # %tr
+; CHECK-NEXT:    li 3, 1
+; CHECK-NEXT:    b .LBB10_5
+; CHECK-NEXT:  .LBB10_4: # %fa
+; CHECK-NEXT:    li 3, 0
+; CHECK-NEXT:  .LBB10_5: # %ret
+; CHECK-NEXT:    stw 3, 12(1)
+; CHECK-NEXT:    lwz 3, 12(1)
+; CHECK-NEXT:    addi 1, 1, 16
+; CHECK-NEXT:    blr
   entry:
-  %r = fcmp ole float %a, %b
-  ret i1 %r
-; CHECK-LABEL: test_fcmple
-; CHECK: efscmpgt
-; CHECK: blr
+  %r = alloca i32, align 4
+  %c = fcmp ole float %a, %b
+  br i1 %c, label %tr, label %fa
+tr:
+  store i32 1, i32* %r, align 4
+  br label %ret
+fa:
+  store i32 0, i32* %r, align 4
+  br label %ret
+ret:
+  %0 = load i32, i32* %r, align 4
+  ret i32 %0
 }
 
-define i1 @test_fcmpule(float %a, float %b) {
+define i32 @test_fcmpule(float %a, float %b) {
+; CHECK-LABEL: test_fcmpule:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    stwu 1, -16(1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    efscmpgt 0, 3, 4
+; CHECK-NEXT:    bgt 0, .LBB11_2
+; CHECK-NEXT:  # %bb.1: # %tr
+; CHECK-NEXT:    li 3, 1
+; CHECK-NEXT:    b .LBB11_3
+; CHECK-NEXT:  .LBB11_2: # %fa
+; CHECK-NEXT:    li 3, 0
+; CHECK-NEXT:  .LBB11_3: # %ret
+; CHECK-NEXT:    stw 3, 12(1)
+; CHECK-NEXT:    lwz 3, 12(1)
+; CHECK-NEXT:    addi 1, 1, 16
+; CHECK-NEXT:    blr
   entry:
-  %r = fcmp ule float %a, %b
-  ret i1 %r
-; CHECK-LABEL: test_fcmpule
-; CHECK: efscmpgt
-; CHECK: blr
+  %r = alloca i32, align 4
+  %c = fcmp ule float %a, %b
+  br i1 %c, label %tr, label %fa
+tr:
+  store i32 1, i32* %r, align 4
+  br label %ret
+fa:
+  store i32 0, i32* %r, align 4
+  br label %ret
+ret:
+  %0 = load i32, i32* %r, align 4
+  ret i32 %0
 }
 
-define i1 @test_fcmpeq(float %a, float %b) {
+; The type of comparison found in C's if (x == y)
+define i32 @test_fcmpeq(float %a, float %b) {
+; CHECK-LABEL: test_fcmpeq:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    stwu 1, -16(1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    efscmpeq 0, 3, 4
+; CHECK-NEXT:    ble 0, .LBB12_2
+; CHECK-NEXT:  # %bb.1: # %tr
+; CHECK-NEXT:    li 3, 1
+; CHECK-NEXT:    b .LBB12_3
+; CHECK-NEXT:  .LBB12_2: # %fa
+; CHECK-NEXT:    li 3, 0
+; CHECK-NEXT:  .LBB12_3: # %ret
+; CHECK-NEXT:    stw 3, 12(1)
+; CHECK-NEXT:    lwz 3, 12(1)
+; CHECK-NEXT:    addi 1, 1, 16
+; CHECK-NEXT:    blr
   entry:
-  %r = fcmp oeq float %a, %b
-  ret i1 %r
-; CHECK-LABEL: test_fcmpeq
-; CHECK: efscmpeq
-; CHECK: blr
+  %r = alloca i32, align 4
+  %c = fcmp oeq float %a, %b
+  br i1 %c, label %tr, label %fa
+tr:
+  store i32 1, i32* %r, align 4
+  br label %ret
+fa:
+  store i32 0, i32* %r, align 4
+  br label %ret
+ret:
+  %0 = load i32, i32* %r, align 4
+  ret i32 %0
 }
 
 ; (un)ordered tests are expanded to une and oeq so verify
 define i1 @test_fcmpuno(float %a, float %b) {
+; CHECK-LABEL: test_fcmpuno:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    efscmpeq 0, 3, 3
+; CHECK-NEXT:    efscmpeq 1, 4, 4
+; CHECK-NEXT:    li 5, 1
+; CHECK-NEXT:    crand 20, 5, 1
+; CHECK-NEXT:    bc 12, 20, .LBB13_2
+; CHECK-NEXT:  # %bb.1: # %entry
+; CHECK-NEXT:    ori 3, 5, 0
+; CHECK-NEXT:    blr
+; CHECK-NEXT:  .LBB13_2: # %entry
+; CHECK-NEXT:    addi 3, 0, 0
+; CHECK-NEXT:    blr
   entry:
   %r = fcmp uno float %a, %b
   ret i1 %r
-; CHECK-LABEL: test_fcmpuno
-; CHECK: efscmpeq
-; CHECK: efscmpeq
-; CHECK: crand
-; CHECK: blr
 }
 
 define i1 @test_fcmpord(float %a, float %b) {
+; CHECK-LABEL: test_fcmpord:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    efscmpeq 0, 4, 4
+; CHECK-NEXT:    efscmpeq 1, 3, 3
+; CHECK-NEXT:    li 5, 1
+; CHECK-NEXT:    crnand 20, 5, 1
+; CHECK-NEXT:    bc 12, 20, .LBB14_2
+; CHECK-NEXT:  # %bb.1: # %entry
+; CHECK-NEXT:    ori 3, 5, 0
+; CHECK-NEXT:    blr
+; CHECK-NEXT:  .LBB14_2: # %entry
+; CHECK-NEXT:    addi 3, 0, 0
+; CHECK-NEXT:    blr
   entry:
   %r = fcmp ord float %a, %b
   ret i1 %r
-; CHECK-LABEL: test_fcmpord
-; CHECK: efscmpeq
-; CHECK: efscmpeq
-; CHECK: crnand
-; CHECK: blr
 }
 
 define i1 @test_fcmpueq(float %a, float %b) {
+; CHECK-LABEL: test_fcmpueq:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    efscmpeq 0, 3, 3
+; CHECK-NEXT:    efscmpeq 1, 4, 4
+; CHECK-NEXT:    crnand 20, 5, 1
+; CHECK-NEXT:    efscmpeq 0, 3, 4
+; CHECK-NEXT:    li 5, 1
+; CHECK-NEXT:    crnor 20, 1, 20
+; CHECK-NEXT:    bc 12, 20, .LBB15_2
+; CHECK-NEXT:  # %bb.1: # %entry
+; CHECK-NEXT:    ori 3, 5, 0
+; CHECK-NEXT:    blr
+; CHECK-NEXT:  .LBB15_2: # %entry
+; CHECK-NEXT:    addi 3, 0, 0
+; CHECK-NEXT:    blr
   entry:
   %r = fcmp ueq float %a, %b
   ret i1 %r
-; CHECK-LABEL: test_fcmpueq
-; CHECK: efscmpeq
-; CHECK: blr
 }
 
 define i1 @test_fcmpne(float %a, float %b) {
+; CHECK-LABEL: test_fcmpne:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    efscmpeq 0, 4, 4
+; CHECK-NEXT:    efscmpeq 1, 3, 3
+; CHECK-NEXT:    crand 20, 5, 1
+; CHECK-NEXT:    efscmpeq 0, 3, 4
+; CHECK-NEXT:    li 5, 1
+; CHECK-NEXT:    crorc 20, 1, 20
+; CHECK-NEXT:    bc 12, 20, .LBB16_2
+; CHECK-NEXT:  # %bb.1: # %entry
+; CHECK-NEXT:    ori 3, 5, 0
+; CHECK-NEXT:    blr
+; CHECK-NEXT:  .LBB16_2: # %entry
+; CHECK-NEXT:    addi 3, 0, 0
+; CHECK-NEXT:    blr
   entry:
   %r = fcmp one float %a, %b
   ret i1 %r
-; CHECK-LABEL: test_fcmpne
-; CHECK: efscmpeq
-; CHECK: blr
 }
 
-define i1 @test_fcmpune(float %a, float %b) {
+define i32 @test_fcmpune(float %a, float %b) {
+; CHECK-LABEL: test_fcmpune:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    stwu 1, -16(1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    efscmpeq 0, 3, 4
+; CHECK-NEXT:    bgt 0, .LBB17_2
+; CHECK-NEXT:  # %bb.1: # %tr
+; CHECK-NEXT:    li 3, 1
+; CHECK-NEXT:    b .LBB17_3
+; CHECK-NEXT:  .LBB17_2: # %fa
+; CHECK-NEXT:    li 3, 0
+; CHECK-NEXT:  .LBB17_3: # %ret
+; CHECK-NEXT:    stw 3, 12(1)
+; CHECK-NEXT:    lwz 3, 12(1)
+; CHECK-NEXT:    addi 1, 1, 16
+; CHECK-NEXT:    blr
   entry:
-  %r = fcmp une float %a, %b
-  ret i1 %r
-; CHECK-LABEL: test_fcmpune
-; CHECK: efscmpeq
-; CHECK: blr
+  %r = alloca i32, align 4
+  %c = fcmp une float %a, %b
+  br i1 %c, label %tr, label %fa
+tr:
+  store i32 1, i32* %r, align 4
+  br label %ret
+fa:
+  store i32 0, i32* %r, align 4
+  br label %ret
+ret:
+  %0 = load i32, i32* %r, align 4
+  ret i32 %0
 }
 
-define i1 @test_fcmplt(float %a, float %b) {
+define i32 @test_fcmplt(float %a, float %b) {
+; CHECK-LABEL: test_fcmplt:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    stwu 1, -16(1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    efscmplt 0, 3, 4
+; CHECK-NEXT:    ble 0, .LBB18_2
+; CHECK-NEXT:  # %bb.1: # %tr
+; CHECK-NEXT:    li 3, 1
+; CHECK-NEXT:    b .LBB18_3
+; CHECK-NEXT:  .LBB18_2: # %fa
+; CHECK-NEXT:    li 3, 0
+; CHECK-NEXT:  .LBB18_3: # %ret
+; CHECK-NEXT:    stw 3, 12(1)
+; CHECK-NEXT:    lwz 3, 12(1)
+; CHECK-NEXT:    addi 1, 1, 16
+; CHECK-NEXT:    blr
   entry:
-  %r = fcmp olt float %a, %b
-  ret i1 %r
-; CHECK-LABEL: test_fcmplt
-; CHECK: efscmplt
-; CHECK: blr
+  %r = alloca i32, align 4
+  %c = fcmp olt float %a, %b
+  br i1 %c, label %tr, label %fa
+tr:
+  store i32 1, i32* %r, align 4
+  br label %ret
+fa:
+  store i32 0, i32* %r, align 4
+  br label %ret
+ret:
+  %0 = load i32, i32* %r, align 4
+  ret i32 %0
 }
 
 define i1 @test_fcmpult(float %a, float %b) {
+; CHECK-LABEL: test_fcmpult:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    efscmpeq 0, 3, 3
+; CHECK-NEXT:    efscmpeq 1, 4, 4
+; CHECK-NEXT:    crnand 20, 5, 1
+; CHECK-NEXT:    efscmplt 0, 3, 4
+; CHECK-NEXT:    li 5, 1
+; CHECK-NEXT:    crnor 20, 1, 20
+; CHECK-NEXT:    bc 12, 20, .LBB19_2
+; CHECK-NEXT:  # %bb.1: # %entry
+; CHECK-NEXT:    ori 3, 5, 0
+; CHECK-NEXT:    blr
+; CHECK-NEXT:  .LBB19_2: # %entry
+; CHECK-NEXT:    addi 3, 0, 0
+; CHECK-NEXT:    blr
   entry:
   %r = fcmp ult float %a, %b
   ret i1 %r
-; CHECK-LABEL: test_fcmpult
-; CHECK: efscmplt
-; CHECK: blr
 }
 
-define i1 @test_fcmpge(float %a, float %b) {
+define i32 @test_fcmpge(float %a, float %b) {
+; CHECK-LABEL: test_fcmpge:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    stwu 1, -16(1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    efscmpeq 0, 3, 3
+; CHECK-NEXT:    bc 4, 1, .LBB20_4
+; CHECK-NEXT:  # %bb.1: # %entry
+; CHECK-NEXT:    efscmpeq 0, 4, 4
+; CHECK-NEXT:    bc 4, 1, .LBB20_4
+; CHECK-NEXT:  # %bb.2: # %entry
+; CHECK-NEXT:    efscmplt 0, 3, 4
+; CHECK-NEXT:    bc 12, 1, .LBB20_4
+; CHECK-NEXT:  # %bb.3: # %tr
+; CHECK-NEXT:    li 3, 1
+; CHECK-NEXT:    b .LBB20_5
+; CHECK-NEXT:  .LBB20_4: # %fa
+; CHECK-NEXT:    li 3, 0
+; CHECK-NEXT:  .LBB20_5: # %ret
+; CHECK-NEXT:    stw 3, 12(1)
+; CHECK-NEXT:    lwz 3, 12(1)
+; CHECK-NEXT:    addi 1, 1, 16
+; CHECK-NEXT:    blr
   entry:
-  %r = fcmp oge float %a, %b
-  ret i1 %r
-; CHECK-LABEL: test_fcmpge
-; CHECK: efscmplt
-; CHECK: blr
+  %r = alloca i32, align 4
+  %c = fcmp oge float %a, %b
+  br i1 %c, label %tr, label %fa
+tr:
+  store i32 1, i32* %r, align 4
+  br label %ret
+fa:
+  store i32 0, i32* %r, align 4
+  br label %ret
+ret:
+  %0 = load i32, i32* %r, align 4
+  ret i32 %0
 }
 
-define i1 @test_fcmpuge(float %a, float %b) {
+define i32 @test_fcmpuge(float %a, float %b) {
+; CHECK-LABEL: test_fcmpuge:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    stwu 1, -16(1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    efscmplt 0, 3, 4
+; CHECK-NEXT:    bgt 0, .LBB21_2
+; CHECK-NEXT:  # %bb.1: # %tr
+; CHECK-NEXT:    li 3, 1
+; CHECK-NEXT:    b .LBB21_3
+; CHECK-NEXT:  .LBB21_2: # %fa
+; CHECK-NEXT:    li 3, 0
+; CHECK-NEXT:  .LBB21_3: # %ret
+; CHECK-NEXT:    stw 3, 12(1)
+; CHECK-NEXT:    lwz 3, 12(1)
+; CHECK-NEXT:    addi 1, 1, 16
+; CHECK-NEXT:    blr
   entry:
-  %r = fcmp uge float %a, %b
-  ret i1 %r
-; CHECK-LABEL: test_fcmpuge
-; CHECK: efscmplt
-; CHECK: blr
+  %r = alloca i32, align 4
+  %c = fcmp uge float %a, %b
+  br i1 %c, label %tr, label %fa
+tr:
+  store i32 1, i32* %r, align 4
+  br label %ret
+fa:
+  store i32 0, i32* %r, align 4
+  br label %ret
+ret:
+  %0 = load i32, i32* %r, align 4
+  ret i32 %0
 }
 
+
 define i32 @test_ftoui(float %a) {
+; CHECK-LABEL: test_ftoui:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    efsctuiz 3, 3
+; CHECK-NEXT:    blr
   %v = fptoui float %a to i32
   ret i32 %v
-; CHECK-LABEL: test_ftoui
-; CHECK: efsctuiz
 }
 
 define i32 @test_ftosi(float %a) {
+; CHECK-LABEL: test_ftosi:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    efsctsiz 3, 3
+; CHECK-NEXT:    blr
   %v = fptosi float %a to i32
   ret i32 %v
-; CHECK-LABEL: test_ftosi
-; CHECK: efsctsiz
 }
 
 define float @test_ffromui(i32 %a) {
+; CHECK-LABEL: test_ffromui:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    efscfui 3, 3
+; CHECK-NEXT:    blr
   %v = uitofp i32 %a to float
   ret float %v
-; CHECK-LABEL: test_ffromui
-; CHECK: efscfui
 }
 
 define float @test_ffromsi(i32 %a) {
+; CHECK-LABEL: test_ffromsi:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    efscfsi 3, 3
+; CHECK-NEXT:    blr
   %v = sitofp i32 %a to float
   ret float %v
-; CHECK-LABEL: test_ffromsi
-; CHECK: efscfsi
 }
 
 define i32 @test_fasmconst(float %x) {
+; CHECK-LABEL: test_fasmconst:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    stwu 1, -32(1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    stw 3, 20(1)
+; CHECK-NEXT:    stw 3, 24(1)
+; CHECK-NEXT:    lwz 3, 20(1)
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    efsctsi 3, 3
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    addi 1, 1, 32
+; CHECK-NEXT:    blr
 entry:
   %x.addr = alloca float, align 8
   store float %x, float* %x.addr, align 8
   %0 = load float, float* %x.addr, align 8
   %1 = call i32 asm sideeffect "efsctsi $0, $1", "=f,f"(float %0)
   ret i32 %1
-; CHECK-LABEL: test_fasmconst
 ; Check that it's not loading a double
-; CHECK-NOT: evldd
-; CHECK: #APP
-; CHECK: efsctsi
-; CHECK: #NO_APP
 }
 
 ; Double tests
 
 define void @test_double_abs(double * %aa) #0 {
+; CHECK-LABEL: test_double_abs:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    evldd 4, 0(3)
+; CHECK-NEXT:    efdabs 4, 4
+; CHECK-NEXT:    evstdd 4, 0(3)
+; CHECK-NEXT:    blr
   entry:
     %0 = load double, double * %aa
     %1 = tail call double @llvm.fabs.f64(double %0) #2
     store double %1, double * %aa
     ret void
-; CHECK-LABEL: test_double_abs
-; CHECK: efdabs
-; CHECK: blr
 }
 
 ; Function Attrs: nounwind readnone
 declare double @llvm.fabs.f64(double) #1
 
 define void @test_dnabs(double * %aa) #0 {
+; CHECK-LABEL: test_dnabs:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    evldd 4, 0(3)
+; CHECK-NEXT:    efdnabs 4, 4
+; CHECK-NEXT:    evstdd 4, 0(3)
+; CHECK-NEXT:    blr
   entry:
     %0 = load double, double * %aa
     %1 = tail call double @llvm.fabs.f64(double %0) #2
@@ -275,256 +592,635 @@ define void @test_dnabs(double * %aa) #0 {
     store double %sub, double * %aa
     ret void
 }
-; CHECK-LABEL: @test_dnabs
-; CHECK: efdnabs
-; CHECK: blr
 
 define double @test_ddiv(double %a, double %b) {
+; CHECK-LABEL: test_ddiv:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    evmergelo 5, 5, 6
+; CHECK-NEXT:    evmergelo 3, 3, 4
+; CHECK-NEXT:    efddiv 4, 3, 5
+; CHECK-NEXT:    evmergehi 3, 4, 4
+; CHECK-NEXT:    # kill: def $r4 killed $r4 killed $s4
+; CHECK-NEXT:    # kill: def $r3 killed $r3 killed $s3
+; CHECK-NEXT:    blr
 entry:
   %v = fdiv double %a, %b
   ret double %v
 
-; CHECK-LABEL: test_ddiv
-; CHECK: efddiv
-; CHECK: blr
 }
 
 define double @test_dmul(double %a, double %b) {
+; CHECK-LABEL: test_dmul:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    evmergelo 5, 5, 6
+; CHECK-NEXT:    evmergelo 3, 3, 4
+; CHECK-NEXT:    efdmul 4, 3, 5
+; CHECK-NEXT:    evmergehi 3, 4, 4
+; CHECK-NEXT:    # kill: def $r4 killed $r4 killed $s4
+; CHECK-NEXT:    # kill: def $r3 killed $r3 killed $s3
+; CHECK-NEXT:    blr
   entry:
   %v = fmul double %a, %b
   ret double %v
-; CHECK-LABEL @test_dmul
-; CHECK: efdmul
-; CHECK: blr
 }
 
 define double @test_dadd(double %a, double %b) {
+; CHECK-LABEL: test_dadd:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    evmergelo 5, 5, 6
+; CHECK-NEXT:    evmergelo 3, 3, 4
+; CHECK-NEXT:    efdadd 4, 3, 5
+; CHECK-NEXT:    evmergehi 3, 4, 4
+; CHECK-NEXT:    # kill: def $r4 killed $r4 killed $s4
+; CHECK-NEXT:    # kill: def $r3 killed $r3 killed $s3
+; CHECK-NEXT:    blr
   entry:
   %v = fadd double %a, %b
   ret double %v
-; CHECK-LABEL @test_dadd
-; CHECK: efdadd
-; CHECK: blr
 }
 
 define double @test_dsub(double %a, double %b) {
+; CHECK-LABEL: test_dsub:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    evmergelo 5, 5, 6
+; CHECK-NEXT:    evmergelo 3, 3, 4
+; CHECK-NEXT:    efdsub 4, 3, 5
+; CHECK-NEXT:    evmergehi 3, 4, 4
+; CHECK-NEXT:    # kill: def $r4 killed $r4 killed $s4
+; CHECK-NEXT:    # kill: def $r3 killed $r3 killed $s3
+; CHECK-NEXT:    blr
   entry:
   %v = fsub double %a, %b
   ret double %v
-; CHECK-LABEL @test_dsub
-; CHECK: efdsub
-; CHECK: blr
 }
 
 define double @test_dneg(double %a) {
+; CHECK-LABEL: test_dneg:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    evmergelo 3, 3, 4
+; CHECK-NEXT:    efdneg 4, 3
+; CHECK-NEXT:    evmergehi 3, 4, 4
+; CHECK-NEXT:    # kill: def $r4 killed $r4 killed $s4
+; CHECK-NEXT:    # kill: def $r3 killed $r3 killed $s3
+; CHECK-NEXT:    blr
   entry:
   %v = fsub double -0.0, %a
   ret double %v
-
-; CHECK-LABEL @test_dneg
-; CHECK: blr
 }
 
 define double @test_stod(float %a) {
+; CHECK-LABEL: test_stod:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    efdcfs 4, 3
+; CHECK-NEXT:    evmergehi 3, 4, 4
+; CHECK-NEXT:    # kill: def $r4 killed $r4 killed $s4
+; CHECK-NEXT:    # kill: def $r3 killed $r3 killed $s3
+; CHECK-NEXT:    blr
   entry:
   %v = fpext float %a to double
   ret double %v
-; CHECK-LABEL: test_stod
-; CHECK: efdcfs
-; CHECK: blr
 }
 
 ; (un)ordered tests are expanded to une and oeq so verify
 define i1 @test_dcmpuno(double %a, double %b) {
+; CHECK-LABEL: test_dcmpuno:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    evmergelo 5, 5, 6
+; CHECK-NEXT:    evmergelo 3, 3, 4
+; CHECK-NEXT:    li 7, 1
+; CHECK-NEXT:    efdcmpeq 0, 3, 3
+; CHECK-NEXT:    efdcmpeq 1, 5, 5
+; CHECK-NEXT:    crand 20, 5, 1
+; CHECK-NEXT:    bc 12, 20, .LBB35_2
+; CHECK-NEXT:  # %bb.1: # %entry
+; CHECK-NEXT:    ori 3, 7, 0
+; CHECK-NEXT:    blr
+; CHECK-NEXT:  .LBB35_2: # %entry
+; CHECK-NEXT:    addi 3, 0, 0
+; CHECK-NEXT:    blr
   entry:
   %r = fcmp uno double %a, %b
   ret i1 %r
-; CHECK-LABEL: test_dcmpuno
-; CHECK: efdcmpeq
-; CHECK: efdcmpeq
-; CHECK: crand
-; CHECK: blr
 }
 
 define i1 @test_dcmpord(double %a, double %b) {
+; CHECK-LABEL: test_dcmpord:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    evmergelo 3, 3, 4
+; CHECK-NEXT:    evmergelo 4, 5, 6
+; CHECK-NEXT:    li 7, 1
+; CHECK-NEXT:    efdcmpeq 0, 4, 4
+; CHECK-NEXT:    efdcmpeq 1, 3, 3
+; CHECK-NEXT:    crnand 20, 5, 1
+; CHECK-NEXT:    bc 12, 20, .LBB36_2
+; CHECK-NEXT:  # %bb.1: # %entry
+; CHECK-NEXT:    ori 3, 7, 0
+; CHECK-NEXT:    blr
+; CHECK-NEXT:  .LBB36_2: # %entry
+; CHECK-NEXT:    addi 3, 0, 0
+; CHECK-NEXT:    blr
   entry:
   %r = fcmp ord double %a, %b
   ret i1 %r
-; CHECK-LABEL: test_dcmpord
-; CHECK: efdcmpeq
-; CHECK: efdcmpeq
-; CHECK: crnand
-; CHECK: blr
 }
 
-define i1 @test_dcmpgt(double %a, double %b) {
+define i32 @test_dcmpgt(double %a, double %b) {
+; CHECK-LABEL: test_dcmpgt:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    stwu 1, -16(1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    evmergelo 5, 5, 6
+; CHECK-NEXT:    evmergelo 3, 3, 4
+; CHECK-NEXT:    efdcmpgt 0, 3, 5
+; CHECK-NEXT:    ble 0, .LBB37_2
+; CHECK-NEXT:  # %bb.1: # %tr
+; CHECK-NEXT:    li 3, 1
+; CHECK-NEXT:    b .LBB37_3
+; CHECK-NEXT:  .LBB37_2: # %fa
+; CHECK-NEXT:    li 3, 0
+; CHECK-NEXT:  .LBB37_3: # %ret
+; CHECK-NEXT:    stw 3, 12(1)
+; CHECK-NEXT:    lwz 3, 12(1)
+; CHECK-NEXT:    addi 1, 1, 16
+; CHECK-NEXT:    blr
   entry:
-  %r = fcmp ogt double %a, %b
-  ret i1 %r
-; CHECK-LABEL: test_dcmpgt
-; CHECK: efdcmpgt
-; CHECK: blr
+  %r = alloca i32, align 4
+  %c = fcmp ogt double %a, %b
+  br i1 %c, label %tr, label %fa
+tr:
+  store i32 1, i32* %r, align 4
+  br label %ret
+fa:
+  store i32 0, i32* %r, align 4
+  br label %ret
+ret:
+  %0 = load i32, i32* %r, align 4
+  ret i32 %0
 }
 
-define i1 @test_dcmpugt(double %a, double %b) {
+define i32 @test_dcmpugt(double %a, double %b) {
+; CHECK-LABEL: test_dcmpugt:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    stwu 1, -16(1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    evmergelo 3, 3, 4
+; CHECK-NEXT:    evmergelo 4, 5, 6
+; CHECK-NEXT:    efdcmpeq 0, 4, 4
+; CHECK-NEXT:    bc 4, 1, .LBB38_4
+; CHECK-NEXT:  # %bb.1: # %entry
+; CHECK-NEXT:    efdcmpeq 0, 3, 3
+; CHECK-NEXT:    bc 4, 1, .LBB38_4
+; CHECK-NEXT:  # %bb.2: # %entry
+; CHECK-NEXT:    efdcmpgt 0, 3, 4
+; CHECK-NEXT:    bc 12, 1, .LBB38_4
+; CHECK-NEXT:  # %bb.3: # %fa
+; CHECK-NEXT:    li 3, 0
+; CHECK-NEXT:    b .LBB38_5
+; CHECK-NEXT:  .LBB38_4: # %tr
+; CHECK-NEXT:    li 3, 1
+; CHECK-NEXT:  .LBB38_5: # %ret
+; CHECK-NEXT:    stw 3, 12(1)
+; CHECK-NEXT:    lwz 3, 12(1)
+; CHECK-NEXT:    addi 1, 1, 16
+; CHECK-NEXT:    blr
   entry:
-  %r = fcmp ugt double %a, %b
-  ret i1 %r
-; CHECK-LABEL: test_dcmpugt
-; CHECK: efdcmpgt
-; CHECK: blr
+  %r = alloca i32, align 4
+  %c = fcmp ugt double %a, %b
+  br i1 %c, label %tr, label %fa
+tr:
+  store i32 1, i32* %r, align 4
+  br label %ret
+fa:
+  store i32 0, i32* %r, align 4
+  br label %ret
+ret:
+  %0 = load i32, i32* %r, align 4
+  ret i32 %0
 }
 
-define i1 @test_dcmple(double %a, double %b) {
+define i32 @test_dcmple(double %a, double %b) {
+; CHECK-LABEL: test_dcmple:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    stwu 1, -16(1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    evmergelo 5, 5, 6
+; CHECK-NEXT:    evmergelo 3, 3, 4
+; CHECK-NEXT:    efdcmpgt 0, 3, 5
+; CHECK-NEXT:    bgt 0, .LBB39_2
+; CHECK-NEXT:  # %bb.1: # %tr
+; CHECK-NEXT:    li 3, 1
+; CHECK-NEXT:    b .LBB39_3
+; CHECK-NEXT:  .LBB39_2: # %fa
+; CHECK-NEXT:    li 3, 0
+; CHECK-NEXT:  .LBB39_3: # %ret
+; CHECK-NEXT:    stw 3, 12(1)
+; CHECK-NEXT:    lwz 3, 12(1)
+; CHECK-NEXT:    addi 1, 1, 16
+; CHECK-NEXT:    blr
   entry:
-  %r = fcmp ole double %a, %b
-  ret i1 %r
-; CHECK-LABEL: test_dcmple
-; CHECK: efdcmpgt
-; CHECK: blr
+  %r = alloca i32, align 4
+  %c = fcmp ule double %a, %b
+  br i1 %c, label %tr, label %fa
+tr:
+  store i32 1, i32* %r, align 4
+  br label %ret
+fa:
+  store i32 0, i32* %r, align 4
+  br label %ret
+ret:
+  %0 = load i32, i32* %r, align 4
+  ret i32 %0
 }
 
-define i1 @test_dcmpule(double %a, double %b) {
+define i32 @test_dcmpule(double %a, double %b) {
+; CHECK-LABEL: test_dcmpule:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    stwu 1, -16(1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    evmergelo 5, 5, 6
+; CHECK-NEXT:    evmergelo 3, 3, 4
+; CHECK-NEXT:    efdcmpgt 0, 3, 5
+; CHECK-NEXT:    bgt 0, .LBB40_2
+; CHECK-NEXT:  # %bb.1: # %tr
+; CHECK-NEXT:    li 3, 1
+; CHECK-NEXT:    b .LBB40_3
+; CHECK-NEXT:  .LBB40_2: # %fa
+; CHECK-NEXT:    li 3, 0
+; CHECK-NEXT:  .LBB40_3: # %ret
+; CHECK-NEXT:    stw 3, 12(1)
+; CHECK-NEXT:    lwz 3, 12(1)
+; CHECK-NEXT:    addi 1, 1, 16
+; CHECK-NEXT:    blr
   entry:
-  %r = fcmp ule double %a, %b
-  ret i1 %r
-; CHECK-LABEL: test_dcmpule
-; CHECK: efdcmpgt
-; CHECK: blr
+  %r = alloca i32, align 4
+  %c = fcmp ule double %a, %b
+  br i1 %c, label %tr, label %fa
+tr:
+  store i32 1, i32* %r, align 4
+  br label %ret
+fa:
+  store i32 0, i32* %r, align 4
+  br label %ret
+ret:
+  %0 = load i32, i32* %r, align 4
+  ret i32 %0
 }
 
-define i1 @test_dcmpeq(double %a, double %b) {
+; The type of comparison found in C's if (x == y)
+define i32 @test_dcmpeq(double %a, double %b) {
+; CHECK-LABEL: test_dcmpeq:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    stwu 1, -16(1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    evmergelo 5, 5, 6
+; CHECK-NEXT:    evmergelo 3, 3, 4
+; CHECK-NEXT:    efdcmpeq 0, 3, 5
+; CHECK-NEXT:    ble 0, .LBB41_2
+; CHECK-NEXT:  # %bb.1: # %tr
+; CHECK-NEXT:    li 3, 1
+; CHECK-NEXT:    b .LBB41_3
+; CHECK-NEXT:  .LBB41_2: # %fa
+; CHECK-NEXT:    li 3, 0
+; CHECK-NEXT:  .LBB41_3: # %ret
+; CHECK-NEXT:    stw 3, 12(1)
+; CHECK-NEXT:    lwz 3, 12(1)
+; CHECK-NEXT:    addi 1, 1, 16
+; CHECK-NEXT:    blr
   entry:
-  %r = fcmp oeq double %a, %b
-  ret i1 %r
-; CHECK-LABEL: test_dcmpeq
-; CHECK: efdcmpeq
-; CHECK: blr
+  %r = alloca i32, align 4
+  %c = fcmp oeq double %a, %b
+  br i1 %c, label %tr, label %fa
+tr:
+  store i32 1, i32* %r, align 4
+  br label %ret
+fa:
+  store i32 0, i32* %r, align 4
+  br label %ret
+ret:
+  %0 = load i32, i32* %r, align 4
+  ret i32 %0
 }
 
-define i1 @test_dcmpueq(double %a, double %b) {
+define i32 @test_dcmpueq(double %a, double %b) {
+; CHECK-LABEL: test_dcmpueq:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    stwu 1, -16(1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    evmergelo 3, 3, 4
+; CHECK-NEXT:    evmergelo 4, 5, 6
+; CHECK-NEXT:    efdcmpeq 0, 4, 4
+; CHECK-NEXT:    bc 4, 1, .LBB42_4
+; CHECK-NEXT:  # %bb.1: # %entry
+; CHECK-NEXT:    efdcmpeq 0, 3, 3
+; CHECK-NEXT:    bc 4, 1, .LBB42_4
+; CHECK-NEXT:  # %bb.2: # %entry
+; CHECK-NEXT:    efdcmpeq 0, 3, 4
+; CHECK-NEXT:    bc 12, 1, .LBB42_4
+; CHECK-NEXT:  # %bb.3: # %fa
+; CHECK-NEXT:    li 3, 0
+; CHECK-NEXT:    b .LBB42_5
+; CHECK-NEXT:  .LBB42_4: # %tr
+; CHECK-NEXT:    li 3, 1
+; CHECK-NEXT:  .LBB42_5: # %ret
+; CHECK-NEXT:    stw 3, 12(1)
+; CHECK-NEXT:    lwz 3, 12(1)
+; CHECK-NEXT:    addi 1, 1, 16
+; CHECK-NEXT:    blr
   entry:
-  %r = fcmp ueq double %a, %b
-  ret i1 %r
-; CHECK-LABEL: test_dcmpueq
-; CHECK: efdcmpeq
-; CHECK: blr
+  %r = alloca i32, align 4
+  %c = fcmp ueq double %a, %b
+  br i1 %c, label %tr, label %fa
+tr:
+  store i32 1, i32* %r, align 4
+  br label %ret
+fa:
+  store i32 0, i32* %r, align 4
+  br label %ret
+ret:
+  %0 = load i32, i32* %r, align 4
+  ret i32 %0
 }
 
 define i1 @test_dcmpne(double %a, double %b) {
+; CHECK-LABEL: test_dcmpne:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    evmergelo 3, 3, 4
+; CHECK-NEXT:    evmergelo 4, 5, 6
+; CHECK-NEXT:    li 7, 1
+; CHECK-NEXT:    efdcmpeq 0, 4, 4
+; CHECK-NEXT:    efdcmpeq 1, 3, 3
+; CHECK-NEXT:    efdcmpeq 5, 3, 4
+; CHECK-NEXT:    crand 24, 5, 1
+; CHECK-NEXT:    crorc 20, 21, 24
+; CHECK-NEXT:    bc 12, 20, .LBB43_2
+; CHECK-NEXT:  # %bb.1: # %entry
+; CHECK-NEXT:    ori 3, 7, 0
+; CHECK-NEXT:    blr
+; CHECK-NEXT:  .LBB43_2: # %entry
+; CHECK-NEXT:    addi 3, 0, 0
+; CHECK-NEXT:    blr
   entry:
   %r = fcmp one double %a, %b
   ret i1 %r
-; CHECK-LABEL: test_dcmpne
-; CHECK: efdcmpeq
-; CHECK: blr
 }
 
-define i1 @test_dcmpune(double %a, double %b) {
+define i32 @test_dcmpune(double %a, double %b) {
+; CHECK-LABEL: test_dcmpune:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    stwu 1, -16(1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    evmergelo 5, 5, 6
+; CHECK-NEXT:    evmergelo 3, 3, 4
+; CHECK-NEXT:    efdcmpeq 0, 3, 5
+; CHECK-NEXT:    bgt 0, .LBB44_2
+; CHECK-NEXT:  # %bb.1: # %tr
+; CHECK-NEXT:    li 3, 1
+; CHECK-NEXT:    b .LBB44_3
+; CHECK-NEXT:  .LBB44_2: # %fa
+; CHECK-NEXT:    li 3, 0
+; CHECK-NEXT:  .LBB44_3: # %ret
+; CHECK-NEXT:    stw 3, 12(1)
+; CHECK-NEXT:    lwz 3, 12(1)
+; CHECK-NEXT:    addi 1, 1, 16
+; CHECK-NEXT:    blr
   entry:
-  %r = fcmp une double %a, %b
-  ret i1 %r
-; CHECK-LABEL: test_dcmpune
-; CHECK: efdcmpeq
-; CHECK: blr
+  %r = alloca i32, align 4
+  %c = fcmp une double %a, %b
+  br i1 %c, label %tr, label %fa
+tr:
+  store i32 1, i32* %r, align 4
+  br label %ret
+fa:
+  store i32 0, i32* %r, align 4
+  br label %ret
+ret:
+  %0 = load i32, i32* %r, align 4
+  ret i32 %0
 }
 
-define i1 @test_dcmplt(double %a, double %b) {
+define i32 @test_dcmplt(double %a, double %b) {
+; CHECK-LABEL: test_dcmplt:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    stwu 1, -16(1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    evmergelo 5, 5, 6
+; CHECK-NEXT:    evmergelo 3, 3, 4
+; CHECK-NEXT:    efdcmplt 0, 3, 5
+; CHECK-NEXT:    ble 0, .LBB45_2
+; CHECK-NEXT:  # %bb.1: # %tr
+; CHECK-NEXT:    li 3, 1
+; CHECK-NEXT:    b .LBB45_3
+; CHECK-NEXT:  .LBB45_2: # %fa
+; CHECK-NEXT:    li 3, 0
+; CHECK-NEXT:  .LBB45_3: # %ret
+; CHECK-NEXT:    stw 3, 12(1)
+; CHECK-NEXT:    lwz 3, 12(1)
+; CHECK-NEXT:    addi 1, 1, 16
+; CHECK-NEXT:    blr
   entry:
-  %r = fcmp olt double %a, %b
-  ret i1 %r
-; CHECK-LABEL: test_dcmplt
-; CHECK: efdcmplt
-; CHECK: blr
+  %r = alloca i32, align 4
+  %c = fcmp olt double %a, %b
+  br i1 %c, label %tr, label %fa
+tr:
+  store i32 1, i32* %r, align 4
+  br label %ret
+fa:
+  store i32 0, i32* %r, align 4
+  br label %ret
+ret:
+  %0 = load i32, i32* %r, align 4
+  ret i32 %0
 }
 
-define i1 @test_dcmpult(double %a, double %b) {
+define i32 @test_dcmpult(double %a, double %b) {
+; CHECK-LABEL: test_dcmpult:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    stwu 1, -16(1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    evmergelo 3, 3, 4
+; CHECK-NEXT:    evmergelo 4, 5, 6
+; CHECK-NEXT:    efdcmpeq 0, 4, 4
+; CHECK-NEXT:    bc 4, 1, .LBB46_4
+; CHECK-NEXT:  # %bb.1: # %entry
+; CHECK-NEXT:    efdcmpeq 0, 3, 3
+; CHECK-NEXT:    bc 4, 1, .LBB46_4
+; CHECK-NEXT:  # %bb.2: # %entry
+; CHECK-NEXT:    efdcmplt 0, 3, 4
+; CHECK-NEXT:    bc 12, 1, .LBB46_4
+; CHECK-NEXT:  # %bb.3: # %fa
+; CHECK-NEXT:    li 3, 0
+; CHECK-NEXT:    b .LBB46_5
+; CHECK-NEXT:  .LBB46_4: # %tr
+; CHECK-NEXT:    li 3, 1
+; CHECK-NEXT:  .LBB46_5: # %ret
+; CHECK-NEXT:    stw 3, 12(1)
+; CHECK-NEXT:    lwz 3, 12(1)
+; CHECK-NEXT:    addi 1, 1, 16
+; CHECK-NEXT:    blr
   entry:
-  %r = fcmp ult double %a, %b
-  ret i1 %r
-; CHECK-LABEL: test_dcmpult
-; CHECK: efdcmplt
-; CHECK: blr
+  %r = alloca i32, align 4
+  %c = fcmp ult double %a, %b
+  br i1 %c, label %tr, label %fa
+tr:
+  store i32 1, i32* %r, align 4
+  br label %ret
+fa:
+  store i32 0, i32* %r, align 4
+  br label %ret
+ret:
+  %0 = load i32, i32* %r, align 4
+  ret i32 %0
 }
 
 define i1 @test_dcmpge(double %a, double %b) {
+; CHECK-LABEL: test_dcmpge:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    evmergelo 3, 3, 4
+; CHECK-NEXT:    evmergelo 4, 5, 6
+; CHECK-NEXT:    li 7, 1
+; CHECK-NEXT:    efdcmpeq 0, 4, 4
+; CHECK-NEXT:    efdcmpeq 1, 3, 3
+; CHECK-NEXT:    efdcmplt 5, 3, 4
+; CHECK-NEXT:    crand 24, 5, 1
+; CHECK-NEXT:    crorc 20, 21, 24
+; CHECK-NEXT:    bc 12, 20, .LBB47_2
+; CHECK-NEXT:  # %bb.1: # %entry
+; CHECK-NEXT:    ori 3, 7, 0
+; CHECK-NEXT:    blr
+; CHECK-NEXT:  .LBB47_2: # %entry
+; CHECK-NEXT:    addi 3, 0, 0
+; CHECK-NEXT:    blr
   entry:
   %r = fcmp oge double %a, %b
   ret i1 %r
-; CHECK-LABEL: test_dcmpge
-; CHECK: efdcmplt
-; CHECK: blr
 }
 
-define i1 @test_dcmpuge(double %a, double %b) {
+define i32 @test_dcmpuge(double %a, double %b) {
+; CHECK-LABEL: test_dcmpuge:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    stwu 1, -16(1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    evmergelo 5, 5, 6
+; CHECK-NEXT:    evmergelo 3, 3, 4
+; CHECK-NEXT:    efdcmplt 0, 3, 5
+; CHECK-NEXT:    bgt 0, .LBB48_2
+; CHECK-NEXT:  # %bb.1: # %tr
+; CHECK-NEXT:    li 3, 1
+; CHECK-NEXT:    b .LBB48_3
+; CHECK-NEXT:  .LBB48_2: # %fa
+; CHECK-NEXT:    li 3, 0
+; CHECK-NEXT:  .LBB48_3: # %ret
+; CHECK-NEXT:    stw 3, 12(1)
+; CHECK-NEXT:    lwz 3, 12(1)
+; CHECK-NEXT:    addi 1, 1, 16
+; CHECK-NEXT:    blr
   entry:
-  %r = fcmp uge double %a, %b
-  ret i1 %r
-; CHECK-LABEL: test_dcmpuge
-; CHECK: efdcmplt
-; CHECK: blr
+  %r = alloca i32, align 4
+  %c = fcmp uge double %a, %b
+  br i1 %c, label %tr, label %fa
+tr:
+  store i32 1, i32* %r, align 4
+  br label %ret
+fa:
+  store i32 0, i32* %r, align 4
+  br label %ret
+ret:
+  %0 = load i32, i32* %r, align 4
+  ret i32 %0
 }
 
 define double @test_dselect(double %a, double %b, i1 %c) {
+; CHECK-LABEL: test_dselect:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    andi. 7, 7, 1
+; CHECK-NEXT:    evmergelo 5, 5, 6
+; CHECK-NEXT:    evmergelo 4, 3, 4
+; CHECK-NEXT:    bc 12, 1, .LBB49_2
+; CHECK-NEXT:  # %bb.1: # %entry
+; CHECK-NEXT:    evor 4, 5, 5
+; CHECK-NEXT:  .LBB49_2: # %entry
+; CHECK-NEXT:    evmergehi 3, 4, 4
+; CHECK-NEXT:    # kill: def $r4 killed $r4 killed $s4
+; CHECK-NEXT:    # kill: def $r3 killed $r3 killed $s3
+; CHECK-NEXT:    blr
 entry:
   %r = select i1 %c, double %a, double %b
   ret double %r
-; CHECK-LABEL: test_dselect
-; CHECK: andi.
-; CHECK: bc
-; CHECK: evor
-; CHECK: evmergehi
-; CHECK: blr
 }
 
 define i32 @test_dtoui(double %a) {
+; CHECK-LABEL: test_dtoui:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    evmergelo 3, 3, 4
+; CHECK-NEXT:    efdctuiz 3, 3
+; CHECK-NEXT:    blr
 entry:
   %v = fptoui double %a to i32
   ret i32 %v
-; CHECK-LABEL: test_dtoui
-; CHECK: efdctuiz
 }
 
 define i32 @test_dtosi(double %a) {
+; CHECK-LABEL: test_dtosi:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    evmergelo 3, 3, 4
+; CHECK-NEXT:    efdctsiz 3, 3
+; CHECK-NEXT:    blr
 entry:
   %v = fptosi double %a to i32
   ret i32 %v
-; CHECK-LABEL: test_dtosi
-; CHECK: efdctsiz
 }
 
 define double @test_dfromui(i32 %a) {
+; CHECK-LABEL: test_dfromui:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    efdcfui 4, 3
+; CHECK-NEXT:    evmergehi 3, 4, 4
+; CHECK-NEXT:    # kill: def $r4 killed $r4 killed $s4
+; CHECK-NEXT:    # kill: def $r3 killed $r3 killed $s3
+; CHECK-NEXT:    blr
 entry:
   %v = uitofp i32 %a to double
   ret double %v
-; CHECK-LABEL: test_dfromui
-; CHECK: efdcfui
 }
 
 define double @test_dfromsi(i32 %a) {
+; CHECK-LABEL: test_dfromsi:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    efdcfsi 4, 3
+; CHECK-NEXT:    evmergehi 3, 4, 4
+; CHECK-NEXT:    # kill: def $r4 killed $r4 killed $s4
+; CHECK-NEXT:    # kill: def $r3 killed $r3 killed $s3
+; CHECK-NEXT:    blr
 entry:
   %v = sitofp i32 %a to double
   ret double %v
-; CHECK-LABEL: test_dfromsi
-; CHECK: efdcfsi
 }
 
 define i32 @test_dasmconst(double %x) {
+; CHECK-LABEL: test_dasmconst:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    stwu 1, -16(1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    evmergelo 3, 3, 4
+; CHECK-NEXT:    evstdd 3, 8(1)
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    efdctsi 3, 3
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    addi 1, 1, 16
+; CHECK-NEXT:    blr
 entry:
   %x.addr = alloca double, align 8
   store double %x, double* %x.addr, align 8
   %0 = load double, double* %x.addr, align 8
   %1 = call i32 asm sideeffect "efdctsi $0, $1", "=d,d"(double %0)
   ret i32 %1
-; CHECK-LABEL: test_dasmconst
-; CHECK: evmergelo
-; CHECK: #APP
-; CHECK: efdctsi
-; CHECK: #NO_APP
 }
 
 declare double @test_spill_spe_regs(double, double);
 define dso_local void @test_func2() #0 {
+; CHECK-LABEL: test_func2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    blr
 entry:
   ret void
 }
@@ -532,6 +1228,120 @@ entry:
 declare void @test_memset(i8* nocapture writeonly, i8, i32, i1)
 @global_var1 = global i32 0, align 4
 define double @test_spill(double %a, i32 %a1, i64 %a2, i8 * %a3, i32 *%a4, i32* %a5) nounwind {
+; CHECK-LABEL: test_spill:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    mflr 0
+; CHECK-NEXT:    stw 0, 4(1)
+; CHECK-NEXT:    stwu 1, -352(1)
+; CHECK-NEXT:    li 5, 256
+; CHECK-NEXT:    evstddx 30, 1, 5 # 8-byte Folded Spill
+; CHECK-NEXT:    li 5, 264
+; CHECK-NEXT:    evstddx 31, 1, 5 # 8-byte Folded Spill
+; CHECK-NEXT:    li 5, .LCPI56_0@l
+; CHECK-NEXT:    lis 6, .LCPI56_0@ha
+; CHECK-NEXT:    evlddx 5, 6, 5
+; CHECK-NEXT:    stw 14, 280(1) # 4-byte Folded Spill
+; CHECK-NEXT:    stw 15, 284(1) # 4-byte Folded Spill
+; CHECK-NEXT:    stw 16, 288(1) # 4-byte Folded Spill
+; CHECK-NEXT:    stw 17, 292(1) # 4-byte Folded Spill
+; CHECK-NEXT:    stw 18, 296(1) # 4-byte Folded Spill
+; CHECK-NEXT:    stw 19, 300(1) # 4-byte Folded Spill
+; CHECK-NEXT:    stw 20, 304(1) # 4-byte Folded Spill
+; CHECK-NEXT:    stw 21, 308(1) # 4-byte Folded Spill
+; CHECK-NEXT:    stw 22, 312(1) # 4-byte Folded Spill
+; CHECK-NEXT:    stw 23, 316(1) # 4-byte Folded Spill
+; CHECK-NEXT:    stw 24, 320(1) # 4-byte Folded Spill
+; CHECK-NEXT:    stw 25, 324(1) # 4-byte Folded Spill
+; CHECK-NEXT:    stw 26, 328(1) # 4-byte Folded Spill
+; CHECK-NEXT:    stw 27, 332(1) # 4-byte Folded Spill
+; CHECK-NEXT:    stw 28, 336(1) # 4-byte Folded Spill
+; CHECK-NEXT:    stw 29, 340(1) # 4-byte Folded Spill
+; CHECK-NEXT:    stw 30, 344(1) # 4-byte Folded Spill
+; CHECK-NEXT:    stw 31, 348(1) # 4-byte Folded Spill
+; CHECK-NEXT:    evstdd 14, 128(1) # 8-byte Folded Spill
+; CHECK-NEXT:    evstdd 15, 136(1) # 8-byte Folded Spill
+; CHECK-NEXT:    evstdd 16, 144(1) # 8-byte Folded Spill
+; CHECK-NEXT:    evstdd 17, 152(1) # 8-byte Folded Spill
+; CHECK-NEXT:    evstdd 18, 160(1) # 8-byte Folded Spill
+; CHECK-NEXT:    evstdd 19, 168(1) # 8-byte Folded Spill
+; CHECK-NEXT:    evstdd 20, 176(1) # 8-byte Folded Spill
+; CHECK-NEXT:    evstdd 21, 184(1) # 8-byte Folded Spill
+; CHECK-NEXT:    evstdd 22, 192(1) # 8-byte Folded Spill
+; CHECK-NEXT:    evstdd 23, 200(1) # 8-byte Folded Spill
+; CHECK-NEXT:    evstdd 24, 208(1) # 8-byte Folded Spill
+; CHECK-NEXT:    evstdd 25, 216(1) # 8-byte Folded Spill
+; CHECK-NEXT:    evstdd 26, 224(1) # 8-byte Folded Spill
+; CHECK-NEXT:    evstdd 27, 232(1) # 8-byte Folded Spill
+; CHECK-NEXT:    evstdd 28, 240(1) # 8-byte Folded Spill
+; CHECK-NEXT:    evstdd 29, 248(1) # 8-byte Folded Spill
+; CHECK-NEXT:    evmergelo 3, 3, 4
+; CHECK-NEXT:    lwz 4, 360(1)
+; CHECK-NEXT:    efdadd 3, 3, 3
+; CHECK-NEXT:    efdadd 3, 3, 5
+; CHECK-NEXT:    evstdd 3, 24(1) # 8-byte Folded Spill
+; CHECK-NEXT:    stw 4, 20(1) # 4-byte Folded Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    addi 3, 1, 76
+; CHECK-NEXT:    li 4, 0
+; CHECK-NEXT:    li 5, 24
+; CHECK-NEXT:    li 6, 1
+; CHECK-NEXT:    li 30, 0
+; CHECK-NEXT:    bl test_memset
+; CHECK-NEXT:    lwz 3, 20(1) # 4-byte Folded Reload
+; CHECK-NEXT:    stw 30, 0(3)
+; CHECK-NEXT:    bl test_func2
+; CHECK-NEXT:    addi 3, 1, 32
+; CHECK-NEXT:    li 4, 0
+; CHECK-NEXT:    li 5, 20
+; CHECK-NEXT:    li 6, 1
+; CHECK-NEXT:    bl test_memset
+; CHECK-NEXT:    evldd 4, 24(1) # 8-byte Folded Reload
+; CHECK-NEXT:    li 5, 264
+; CHECK-NEXT:    evmergehi 3, 4, 4
+; CHECK-NEXT:    evlddx 31, 1, 5 # 8-byte Folded Reload
+; CHECK-NEXT:    li 5, 256
+; CHECK-NEXT:    evlddx 30, 1, 5 # 8-byte Folded Reload
+; CHECK-NEXT:    evldd 29, 248(1) # 8-byte Folded Reload
+; CHECK-NEXT:    evldd 28, 240(1) # 8-byte Folded Reload
+; CHECK-NEXT:    evldd 27, 232(1) # 8-byte Folded Reload
+; CHECK-NEXT:    evldd 26, 224(1) # 8-byte Folded Reload
+; CHECK-NEXT:    evldd 25, 216(1) # 8-byte Folded Reload
+; CHECK-NEXT:    evldd 24, 208(1) # 8-byte Folded Reload
+; CHECK-NEXT:    evldd 23, 200(1) # 8-byte Folded Reload
+; CHECK-NEXT:    evldd 22, 192(1) # 8-byte Folded Reload
+; CHECK-NEXT:    evldd 21, 184(1) # 8-byte Folded Reload
+; CHECK-NEXT:    evldd 20, 176(1) # 8-byte Folded Reload
+; CHECK-NEXT:    evldd 19, 168(1) # 8-byte Folded Reload
+; CHECK-NEXT:    evldd 18, 160(1) # 8-byte Folded Reload
+; CHECK-NEXT:    evldd 17, 152(1) # 8-byte Folded Reload
+; CHECK-NEXT:    evldd 16, 144(1) # 8-byte Folded Reload
+; CHECK-NEXT:    evldd 15, 136(1) # 8-byte Folded Reload
+; CHECK-NEXT:    evldd 14, 128(1) # 8-byte Folded Reload
+; CHECK-NEXT:    # kill: def $r3 killed $r3 killed $s3
+; CHECK-NEXT:    # kill: def $r4 killed $r4 killed $s4
+; CHECK-NEXT:    lwz 31, 348(1) # 4-byte Folded Reload
+; CHECK-NEXT:    lwz 30, 344(1) # 4-byte Folded Reload
+; CHECK-NEXT:    lwz 29, 340(1) # 4-byte Folded Reload
+; CHECK-NEXT:    lwz 28, 336(1) # 4-byte Folded Reload
+; CHECK-NEXT:    lwz 27, 332(1) # 4-byte Folded Reload
+; CHECK-NEXT:    lwz 26, 328(1) # 4-byte Folded Reload
+; CHECK-NEXT:    lwz 25, 324(1) # 4-byte Folded Reload
+; CHECK-NEXT:    lwz 24, 320(1) # 4-byte Folded Reload
+; CHECK-NEXT:    lwz 23, 316(1) # 4-byte Folded Reload
+; CHECK-NEXT:    lwz 22, 312(1) # 4-byte Folded Reload
+; CHECK-NEXT:    lwz 21, 308(1) # 4-byte Folded Reload
+; CHECK-NEXT:    lwz 20, 304(1) # 4-byte Folded Reload
+; CHECK-NEXT:    lwz 19, 300(1) # 4-byte Folded Reload
+; CHECK-NEXT:    lwz 18, 296(1) # 4-byte Folded Reload
+; CHECK-NEXT:    lwz 17, 292(1) # 4-byte Folded Reload
+; CHECK-NEXT:    lwz 16, 288(1) # 4-byte Folded Reload
+; CHECK-NEXT:    lwz 15, 284(1) # 4-byte Folded Reload
+; CHECK-NEXT:    lwz 14, 280(1) # 4-byte Folded Reload
+; CHECK-NEXT:    lwz 0, 356(1)
+; CHECK-NEXT:    addi 1, 1, 352
+; CHECK-NEXT:    mtlr 0
+; CHECK-NEXT:    blr
 entry:
   %v1 = alloca [13 x i32], align 4
   %v2 = alloca [11 x i32], align 4
@@ -549,11 +1359,4 @@ entry:
 return:
   ret double %1
 
-; CHECK-LABEL: test_spill
-; CHECK: li [[VREG:[0-9]+]], 256
-; CHECK: evstddx {{[0-9]+}}, {{[0-9]+}}, [[VREG]]
-; CHECK-NOT: evstdd {{[0-9]+}}, 256({{[0-9]+}}
-; CHECK: evstdd
-; CHECK: efdadd
-; CHECK: evldd
 }
diff --git a/llvm/test/CodeGen/PowerPC/spill_p9_setb.ll b/llvm/test/CodeGen/PowerPC/spill_p9_setb.ll
new file mode 100644
index 000000000..ad3deca5b
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/spill_p9_setb.ll
@@ -0,0 +1,56 @@
+; NOTE: This test case aims to test the sequence of spilling the CR[0-7]LT bits
+; NOTE: on POWER9 using the setb instruction.
+
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu \
+; RUN:     -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr -mcpu=pwr9 < %s \
+; RUN:     | FileCheck %s --check-prefix=CHECK-P9
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu \
+; RUN:     -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr -mcpu=pwr9 < %s \
+; RUN:     | FileCheck %s --check-prefix=CHECK-P9
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu \
+; RUN:     -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr -mcpu=pwr8 < %s \
+; RUN:     | FileCheck %s --check-prefix=CHECK-P8
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu \
+; RUN:     -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr -mcpu=pwr8 < %s \
+; RUN:     | FileCheck %s --check-prefix=CHECK-P8
+
+define void @p9_setb_spill() {
+; CHECK-P9-LABEL: p9_setb_spill:
+; CHECK-P9:       # %bb.1: # %if.then
+; CHECK-P9-DAG:    crnor 4*cr[[CREG:.*]]+lt, eq, eq
+; CHECK-P9-DAG:    setb [[REG1:.*]], cr[[CREG]]
+; CHECK-P9-DAG:    stw [[REG1]]
+; CHECK-P9:        blr
+; CHECK-P9:        .LBB0_4: # %if.then1
+;
+; CHECK-P8-LABEL: p9_setb_spill:
+; CHECK-P8:       # %bb.1: # %if.then
+; CHECK-P8-DAG:    crnor 4*cr[[CREG2:.*]]+lt, eq, eq
+; CHECK-P8-DAG:    mfocrf [[REG2:.*]],
+; CHECK-P8-DAG:    rlwinm [[REG2]], [[REG2]]
+; CHECK-P8-DAG:    stw [[REG2]]
+; CHECK-P8:        blr
+; CHECK-P8:        .LBB0_4: # %if.then1
+entry:
+  br i1 undef, label %if.end, label %if.then
+
+if.then:                                          ; preds = %entry
+  %call = tail call signext i32 bitcast (i32 (...)* @fn_call to i32 ()*)()
+  %cmp1 = icmp ne i32 %call, 0
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %entry
+  %off0 = phi i1 [ %cmp1, %if.then ], [ false, %entry ]
+  tail call void asm sideeffect "#Clobber", "~{cr0},~{cr1},~{cr2},~{cr3},~{cr4},~{cr5},~{cr6},~{cr7}"()
+  %off0.not = xor i1 %off0, true
+  %or = or i1 false, %off0.not
+  br i1 %or, label %if.end2, label %if.then1
+
+if.then1:                                         ; preds = %if.end
+  unreachable
+
+if.end2:                                         ; preds = %if.end
+  ret void
+}
+
+declare signext i32 @fn_call(...)
diff --git a/llvm/test/CodeGen/PowerPC/srem-lkk.ll b/llvm/test/CodeGen/PowerPC/srem-lkk.ll
new file mode 100644
index 000000000..97c9089f3
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/srem-lkk.ll
@@ -0,0 +1,149 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -verify-machineinstrs -mtriple=powerpc-unknown-linux-gnu -mcpu=ppc64 < %s | FileCheck -check-prefixes=CHECK,CHECK64 %s
+; RUN: llc -verify-machineinstrs -mtriple=powerpc-unknown-linux-gnu -mcpu=ppc < %s | FileCheck -check-prefixes=CHECK,CHECK32 %s
+
+define i32 @fold_srem_positive_odd(i32 %x) {
+; CHECK-LABEL: fold_srem_positive_odd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lis 4, -21386
+; CHECK-NEXT:    ori 4, 4, 37253
+; CHECK-NEXT:    mulhw 4, 3, 4
+; CHECK-NEXT:    add 4, 4, 3
+; CHECK-NEXT:    srwi 5, 4, 31
+; CHECK-NEXT:    srawi 4, 4, 6
+; CHECK-NEXT:    add 4, 4, 5
+; CHECK-NEXT:    mulli 4, 4, 95
+; CHECK-NEXT:    subf 3, 4, 3
+; CHECK-NEXT:    blr
+  %1 = srem i32 %x, 95
+  ret i32 %1
+}
+
+
+define i32 @fold_srem_positive_even(i32 %x) {
+; CHECK-LABEL: fold_srem_positive_even:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lis 4, 15827
+; CHECK-NEXT:    ori 4, 4, 36849
+; CHECK-NEXT:    mulhw 4, 3, 4
+; CHECK-NEXT:    srwi 5, 4, 31
+; CHECK-NEXT:    srawi 4, 4, 8
+; CHECK-NEXT:    add 4, 4, 5
+; CHECK-NEXT:    mulli 4, 4, 1060
+; CHECK-NEXT:    subf 3, 4, 3
+; CHECK-NEXT:    blr
+  %1 = srem i32 %x, 1060
+  ret i32 %1
+}
+
+
+define i32 @fold_srem_negative_odd(i32 %x) {
+; CHECK-LABEL: fold_srem_negative_odd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lis 4, -23206
+; CHECK-NEXT:    ori 4, 4, 65445
+; CHECK-NEXT:    mulhw 4, 3, 4
+; CHECK-NEXT:    srwi 5, 4, 31
+; CHECK-NEXT:    srawi 4, 4, 8
+; CHECK-NEXT:    add 4, 4, 5
+; CHECK-NEXT:    mulli 4, 4, -723
+; CHECK-NEXT:    subf 3, 4, 3
+; CHECK-NEXT:    blr
+  %1 = srem i32 %x, -723
+  ret i32 %1
+}
+
+
+define i32 @fold_srem_negative_even(i32 %x) {
+; CHECK-LABEL: fold_srem_negative_even:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lis 4, -731
+; CHECK-NEXT:    ori 4, 4, 62439
+; CHECK-NEXT:    mulhw 4, 3, 4
+; CHECK-NEXT:    srwi 5, 4, 31
+; CHECK-NEXT:    srawi 4, 4, 8
+; CHECK-NEXT:    add 4, 4, 5
+; CHECK-NEXT:    mulli 4, 4, -22981
+; CHECK-NEXT:    subf 3, 4, 3
+; CHECK-NEXT:    blr
+  %1 = srem i32 %x, -22981
+  ret i32 %1
+}
+
+
+; Don't fold if we can combine srem with sdiv.
+define i32 @combine_srem_sdiv(i32 %x) {
+; CHECK-LABEL: combine_srem_sdiv:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lis 4, -21386
+; CHECK-NEXT:    ori 4, 4, 37253
+; CHECK-NEXT:    mulhw 4, 3, 4
+; CHECK-NEXT:    add 4, 4, 3
+; CHECK-NEXT:    srwi 5, 4, 31
+; CHECK-NEXT:    srawi 4, 4, 6
+; CHECK-NEXT:    add 4, 4, 5
+; CHECK-NEXT:    mulli 5, 4, 95
+; CHECK-NEXT:    subf 3, 5, 3
+; CHECK-NEXT:    add 3, 3, 4
+; CHECK-NEXT:    blr
+  %1 = srem i32 %x, 95
+  %2 = sdiv i32 %x, 95
+  %3 = add i32 %1, %2
+  ret i32 %3
+}
+
+; Don't fold for divisors that are a power of two.
+define i32 @dont_fold_srem_power_of_two(i32 %x) {
+; CHECK-LABEL: dont_fold_srem_power_of_two:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    srawi 4, 3, 6
+; CHECK-NEXT:    addze 4, 4
+; CHECK-NEXT:    slwi 4, 4, 6
+; CHECK-NEXT:    subf 3, 4, 3
+; CHECK-NEXT:    blr
+  %1 = srem i32 %x, 64
+  ret i32 %1
+}
+
+; Don't fold if the divisor is one.
+define i32 @dont_fold_srem_one(i32 %x) {
+; CHECK-LABEL: dont_fold_srem_one:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li 3, 0
+; CHECK-NEXT:    blr
+  %1 = srem i32 %x, 1
+  ret i32 %1
+}
+
+; Don't fold if the divisor is 2^31.
+define i32 @dont_fold_srem_i32_smax(i32 %x) {
+; CHECK-LABEL: dont_fold_srem_i32_smax:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    srawi 4, 3, 31
+; CHECK-NEXT:    addze 4, 4
+; CHECK-NEXT:    slwi 4, 4, 31
+; CHECK-NEXT:    add 3, 3, 4
+; CHECK-NEXT:    blr
+  %1 = srem i32 %x, 2147483648
+  ret i32 %1
+}
+
+; Don't fold i64 srem
+define i64 @dont_fold_srem_i64(i64 %x) {
+; CHECK-LABEL: dont_fold_srem_i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mflr 0
+; CHECK-NEXT:    stw 0, 4(1)
+; CHECK-NEXT:    stwu 1, -16(1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset lr, 4
+; CHECK-NEXT:    li 5, 0
+; CHECK-NEXT:    li 6, 98
+; CHECK-NEXT:    bl __moddi3
+; CHECK-NEXT:    lwz 0, 20(1)
+; CHECK-NEXT:    addi 1, 1, 16
+; CHECK-NEXT:    mtlr 0
+; CHECK-NEXT:    blr
+  %1 = srem i64 %x, 98
+  ret i64 %1
+}
diff --git a/llvm/test/CodeGen/PowerPC/srem-vector-lkk.ll b/llvm/test/CodeGen/PowerPC/srem-vector-lkk.ll
new file mode 100644
index 000000000..d795f6b62
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/srem-vector-lkk.ll
@@ -0,0 +1,1675 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mcpu=pwr9 -verify-machineinstrs -ppc-vsr-nums-as-vr -ppc-asm-full-reg-names \
+; RUN:		-mtriple=powerpc64le-unknown-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,P9LE
+; RUN: llc -mcpu=pwr9 -verify-machineinstrs -ppc-vsr-nums-as-vr -ppc-asm-full-reg-names \
+; RUN:    -mtriple=powerpc64-unknown-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,P9BE
+; RUN: llc -mcpu=pwr8 -verify-machineinstrs -ppc-vsr-nums-as-vr -ppc-asm-full-reg-names \
+; RUN:    -mtriple=powerpc64le-unknown-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,P8LE
+; RUN: llc -mcpu=pwr8 -verify-machineinstrs -ppc-vsr-nums-as-vr -ppc-asm-full-reg-names \
+; RUN:    -mtriple=powerpc64-unknown-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,P8BE
+
+define <4 x i16> @fold_srem_vec_1(<4 x i16> %x) {
+; P9LE-LABEL: fold_srem_vec_1:
+; P9LE:       # %bb.0:
+; P9LE-NEXT:    li r3, 0
+; P9LE-NEXT:    vextuhrx r3, r3, v2
+; P9LE-NEXT:    extsh r4, r3
+; P9LE-NEXT:    lis r5, -21386
+; P9LE-NEXT:    ori r5, r5, 37253
+; P9LE-NEXT:    extsw r4, r4
+; P9LE-NEXT:    mulld r5, r4, r5
+; P9LE-NEXT:    rldicl r5, r5, 32, 32
+; P9LE-NEXT:    add r4, r5, r4
+; P9LE-NEXT:    srwi r5, r4, 31
+; P9LE-NEXT:    srawi r4, r4, 6
+; P9LE-NEXT:    add r4, r4, r5
+; P9LE-NEXT:    lis r5, 31710
+; P9LE-NEXT:    mulli r4, r4, 95
+; P9LE-NEXT:    subf r3, r4, r3
+; P9LE-NEXT:    mtvsrd f0, r3
+; P9LE-NEXT:    li r3, 2
+; P9LE-NEXT:    vextuhrx r3, r3, v2
+; P9LE-NEXT:    extsh r4, r3
+; P9LE-NEXT:    extsw r4, r4
+; P9LE-NEXT:    ori r5, r5, 63421
+; P9LE-NEXT:    mulld r5, r4, r5
+; P9LE-NEXT:    rldicl r5, r5, 32, 32
+; P9LE-NEXT:    subf r4, r4, r5
+; P9LE-NEXT:    srwi r5, r4, 31
+; P9LE-NEXT:    srawi r4, r4, 6
+; P9LE-NEXT:    add r4, r4, r5
+; P9LE-NEXT:    lis r5, 21399
+; P9LE-NEXT:    mulli r4, r4, -124
+; P9LE-NEXT:    subf r3, r4, r3
+; P9LE-NEXT:    xxswapd v3, vs0
+; P9LE-NEXT:    mtvsrd f0, r3
+; P9LE-NEXT:    li r3, 4
+; P9LE-NEXT:    vextuhrx r3, r3, v2
+; P9LE-NEXT:    extsh r4, r3
+; P9LE-NEXT:    extsw r4, r4
+; P9LE-NEXT:    ori r5, r5, 33437
+; P9LE-NEXT:    mulld r4, r4, r5
+; P9LE-NEXT:    rldicl r5, r4, 1, 63
+; P9LE-NEXT:    rldicl r4, r4, 32, 32
+; P9LE-NEXT:    srawi r4, r4, 5
+; P9LE-NEXT:    add r4, r4, r5
+; P9LE-NEXT:    lis r5, -16728
+; P9LE-NEXT:    mulli r4, r4, 98
+; P9LE-NEXT:    subf r3, r4, r3
+; P9LE-NEXT:    xxswapd v4, vs0
+; P9LE-NEXT:    mtvsrd f0, r3
+; P9LE-NEXT:    li r3, 6
+; P9LE-NEXT:    vextuhrx r3, r3, v2
+; P9LE-NEXT:    extsh r4, r3
+; P9LE-NEXT:    extsw r4, r4
+; P9LE-NEXT:    ori r5, r5, 63249
+; P9LE-NEXT:    mulld r4, r4, r5
+; P9LE-NEXT:    rldicl r5, r4, 1, 63
+; P9LE-NEXT:    rldicl r4, r4, 32, 32
+; P9LE-NEXT:    srawi r4, r4, 8
+; P9LE-NEXT:    add r4, r4, r5
+; P9LE-NEXT:    mulli r4, r4, -1003
+; P9LE-NEXT:    subf r3, r4, r3
+; P9LE-NEXT:    vmrglh v3, v4, v3
+; P9LE-NEXT:    xxswapd v4, vs0
+; P9LE-NEXT:    mtvsrd f0, r3
+; P9LE-NEXT:    xxswapd v2, vs0
+; P9LE-NEXT:    vmrglh v2, v2, v4
+; P9LE-NEXT:    vmrglw v2, v2, v3
+; P9LE-NEXT:    blr
+;
+; P9BE-LABEL: fold_srem_vec_1:
+; P9BE:       # %bb.0:
+; P9BE-NEXT:    li r3, 2
+; P9BE-NEXT:    vextuhlx r3, r3, v2
+; P9BE-NEXT:    extsh r3, r3
+; P9BE-NEXT:    lis r4, 31710
+; P9BE-NEXT:    ori r4, r4, 63421
+; P9BE-NEXT:    extsw r3, r3
+; P9BE-NEXT:    mulld r4, r3, r4
+; P9BE-NEXT:    rldicl r4, r4, 32, 32
+; P9BE-NEXT:    subf r4, r3, r4
+; P9BE-NEXT:    srwi r5, r4, 31
+; P9BE-NEXT:    srawi r4, r4, 6
+; P9BE-NEXT:    add r4, r4, r5
+; P9BE-NEXT:    mulli r4, r4, -124
+; P9BE-NEXT:    subf r3, r4, r3
+; P9BE-NEXT:    lis r4, -21386
+; P9BE-NEXT:    sldi r3, r3, 48
+; P9BE-NEXT:    mtvsrd v3, r3
+; P9BE-NEXT:    li r3, 0
+; P9BE-NEXT:    vextuhlx r3, r3, v2
+; P9BE-NEXT:    extsh r3, r3
+; P9BE-NEXT:    extsw r3, r3
+; P9BE-NEXT:    ori r4, r4, 37253
+; P9BE-NEXT:    mulld r4, r3, r4
+; P9BE-NEXT:    rldicl r4, r4, 32, 32
+; P9BE-NEXT:    add r4, r4, r3
+; P9BE-NEXT:    srwi r5, r4, 31
+; P9BE-NEXT:    srawi r4, r4, 6
+; P9BE-NEXT:    add r4, r4, r5
+; P9BE-NEXT:    mulli r4, r4, 95
+; P9BE-NEXT:    subf r3, r4, r3
+; P9BE-NEXT:    lis r4, -16728
+; P9BE-NEXT:    sldi r3, r3, 48
+; P9BE-NEXT:    mtvsrd v4, r3
+; P9BE-NEXT:    li r3, 6
+; P9BE-NEXT:    vextuhlx r3, r3, v2
+; P9BE-NEXT:    extsh r3, r3
+; P9BE-NEXT:    extsw r3, r3
+; P9BE-NEXT:    ori r4, r4, 63249
+; P9BE-NEXT:    mulld r4, r3, r4
+; P9BE-NEXT:    rldicl r5, r4, 1, 63
+; P9BE-NEXT:    rldicl r4, r4, 32, 32
+; P9BE-NEXT:    srawi r4, r4, 8
+; P9BE-NEXT:    add r4, r4, r5
+; P9BE-NEXT:    mulli r4, r4, -1003
+; P9BE-NEXT:    subf r3, r4, r3
+; P9BE-NEXT:    lis r4, 21399
+; P9BE-NEXT:    sldi r3, r3, 48
+; P9BE-NEXT:    vmrghh v3, v4, v3
+; P9BE-NEXT:    mtvsrd v4, r3
+; P9BE-NEXT:    li r3, 4
+; P9BE-NEXT:    vextuhlx r3, r3, v2
+; P9BE-NEXT:    extsh r3, r3
+; P9BE-NEXT:    extsw r3, r3
+; P9BE-NEXT:    ori r4, r4, 33437
+; P9BE-NEXT:    mulld r4, r3, r4
+; P9BE-NEXT:    rldicl r5, r4, 1, 63
+; P9BE-NEXT:    rldicl r4, r4, 32, 32
+; P9BE-NEXT:    srawi r4, r4, 5
+; P9BE-NEXT:    add r4, r4, r5
+; P9BE-NEXT:    mulli r4, r4, 98
+; P9BE-NEXT:    subf r3, r4, r3
+; P9BE-NEXT:    sldi r3, r3, 48
+; P9BE-NEXT:    mtvsrd v2, r3
+; P9BE-NEXT:    vmrghh v2, v2, v4
+; P9BE-NEXT:    vmrghw v2, v3, v2
+; P9BE-NEXT:    blr
+;
+; P8LE-LABEL: fold_srem_vec_1:
+; P8LE:       # %bb.0:
+; P8LE-NEXT:    xxswapd vs0, v2
+; P8LE-NEXT:    lis r4, 21399
+; P8LE-NEXT:    lis r9, -16728
+; P8LE-NEXT:    lis r11, -21386
+; P8LE-NEXT:    std r30, -16(r1) # 8-byte Folded Spill
+; P8LE-NEXT:    ori r4, r4, 33437
+; P8LE-NEXT:    ori r9, r9, 63249
+; P8LE-NEXT:    ori r11, r11, 37253
+; P8LE-NEXT:    mfvsrd r5, f0
+; P8LE-NEXT:    rldicl r3, r5, 32, 48
+; P8LE-NEXT:    rldicl r6, r5, 16, 48
+; P8LE-NEXT:    clrldi r7, r5, 48
+; P8LE-NEXT:    extsh r8, r3
+; P8LE-NEXT:    extsh r10, r6
+; P8LE-NEXT:    rldicl r5, r5, 48, 48
+; P8LE-NEXT:    extsw r8, r8
+; P8LE-NEXT:    extsh r12, r7
+; P8LE-NEXT:    extsw r10, r10
+; P8LE-NEXT:    mulld r4, r8, r4
+; P8LE-NEXT:    lis r8, 31710
+; P8LE-NEXT:    extsh r0, r5
+; P8LE-NEXT:    extsw r12, r12
+; P8LE-NEXT:    mulld r9, r10, r9
+; P8LE-NEXT:    ori r8, r8, 63421
+; P8LE-NEXT:    extsw r10, r0
+; P8LE-NEXT:    mulld r11, r12, r11
+; P8LE-NEXT:    mulld r8, r10, r8
+; P8LE-NEXT:    rldicl r0, r4, 1, 63
+; P8LE-NEXT:    rldicl r4, r4, 32, 32
+; P8LE-NEXT:    rldicl r30, r9, 1, 63
+; P8LE-NEXT:    rldicl r9, r9, 32, 32
+; P8LE-NEXT:    rldicl r11, r11, 32, 32
+; P8LE-NEXT:    rldicl r8, r8, 32, 32
+; P8LE-NEXT:    add r11, r11, r12
+; P8LE-NEXT:    srawi r4, r4, 5
+; P8LE-NEXT:    subf r8, r10, r8
+; P8LE-NEXT:    srawi r9, r9, 8
+; P8LE-NEXT:    srwi r10, r11, 31
+; P8LE-NEXT:    add r4, r4, r0
+; P8LE-NEXT:    srawi r11, r11, 6
+; P8LE-NEXT:    add r9, r9, r30
+; P8LE-NEXT:    ld r30, -16(r1) # 8-byte Folded Reload
+; P8LE-NEXT:    add r10, r11, r10
+; P8LE-NEXT:    srwi r11, r8, 31
+; P8LE-NEXT:    srawi r8, r8, 6
+; P8LE-NEXT:    mulli r4, r4, 98
+; P8LE-NEXT:    mulli r9, r9, -1003
+; P8LE-NEXT:    add r8, r8, r11
+; P8LE-NEXT:    mulli r10, r10, 95
+; P8LE-NEXT:    mulli r8, r8, -124
+; P8LE-NEXT:    subf r3, r4, r3
+; P8LE-NEXT:    subf r4, r9, r6
+; P8LE-NEXT:    mtvsrd f0, r3
+; P8LE-NEXT:    subf r3, r10, r7
+; P8LE-NEXT:    mtvsrd f1, r4
+; P8LE-NEXT:    subf r4, r8, r5
+; P8LE-NEXT:    mtvsrd f2, r3
+; P8LE-NEXT:    xxswapd v2, vs0
+; P8LE-NEXT:    mtvsrd f3, r4
+; P8LE-NEXT:    xxswapd v3, vs1
+; P8LE-NEXT:    xxswapd v4, vs2
+; P8LE-NEXT:    xxswapd v5, vs3
+; P8LE-NEXT:    vmrglh v2, v3, v2
+; P8LE-NEXT:    vmrglh v3, v5, v4
+; P8LE-NEXT:    vmrglw v2, v2, v3
+; P8LE-NEXT:    blr
+;
+; P8BE-LABEL: fold_srem_vec_1:
+; P8BE:       # %bb.0:
+; P8BE-NEXT:    mfvsrd r4, v2
+; P8BE-NEXT:    lis r3, -16728
+; P8BE-NEXT:    lis r9, 31710
+; P8BE-NEXT:    lis r8, 21399
+; P8BE-NEXT:    lis r10, -21386
+; P8BE-NEXT:    ori r3, r3, 63249
+; P8BE-NEXT:    ori r9, r9, 63421
+; P8BE-NEXT:    ori r8, r8, 33437
+; P8BE-NEXT:    ori r10, r10, 37253
+; P8BE-NEXT:    clrldi r5, r4, 48
+; P8BE-NEXT:    rldicl r7, r4, 32, 48
+; P8BE-NEXT:    rldicl r6, r4, 48, 48
+; P8BE-NEXT:    rldicl r4, r4, 16, 48
+; P8BE-NEXT:    extsh r5, r5
+; P8BE-NEXT:    extsh r7, r7
+; P8BE-NEXT:    extsh r6, r6
+; P8BE-NEXT:    extsw r5, r5
+; P8BE-NEXT:    extsh r4, r4
+; P8BE-NEXT:    extsw r7, r7
+; P8BE-NEXT:    extsw r6, r6
+; P8BE-NEXT:    mulld r3, r5, r3
+; P8BE-NEXT:    extsw r4, r4
+; P8BE-NEXT:    mulld r9, r7, r9
+; P8BE-NEXT:    mulld r8, r6, r8
+; P8BE-NEXT:    mulld r10, r4, r10
+; P8BE-NEXT:    rldicl r11, r3, 1, 63
+; P8BE-NEXT:    rldicl r3, r3, 32, 32
+; P8BE-NEXT:    rldicl r9, r9, 32, 32
+; P8BE-NEXT:    rldicl r12, r8, 1, 63
+; P8BE-NEXT:    rldicl r8, r8, 32, 32
+; P8BE-NEXT:    rldicl r10, r10, 32, 32
+; P8BE-NEXT:    subf r9, r7, r9
+; P8BE-NEXT:    srawi r3, r3, 8
+; P8BE-NEXT:    srawi r8, r8, 5
+; P8BE-NEXT:    add r10, r10, r4
+; P8BE-NEXT:    add r3, r3, r11
+; P8BE-NEXT:    srwi r11, r9, 31
+; P8BE-NEXT:    add r8, r8, r12
+; P8BE-NEXT:    srawi r9, r9, 6
+; P8BE-NEXT:    mulli r3, r3, -1003
+; P8BE-NEXT:    add r9, r9, r11
+; P8BE-NEXT:    srwi r11, r10, 31
+; P8BE-NEXT:    srawi r10, r10, 6
+; P8BE-NEXT:    mulli r8, r8, 98
+; P8BE-NEXT:    add r10, r10, r11
+; P8BE-NEXT:    mulli r9, r9, -124
+; P8BE-NEXT:    mulli r10, r10, 95
+; P8BE-NEXT:    subf r3, r3, r5
+; P8BE-NEXT:    sldi r3, r3, 48
+; P8BE-NEXT:    subf r5, r8, r6
+; P8BE-NEXT:    mtvsrd v2, r3
+; P8BE-NEXT:    subf r6, r9, r7
+; P8BE-NEXT:    sldi r3, r5, 48
+; P8BE-NEXT:    subf r4, r10, r4
+; P8BE-NEXT:    mtvsrd v3, r3
+; P8BE-NEXT:    sldi r3, r6, 48
+; P8BE-NEXT:    sldi r4, r4, 48
+; P8BE-NEXT:    mtvsrd v4, r3
+; P8BE-NEXT:    mtvsrd v5, r4
+; P8BE-NEXT:    vmrghh v2, v3, v2
+; P8BE-NEXT:    vmrghh v3, v5, v4
+; P8BE-NEXT:    vmrghw v2, v3, v2
+; P8BE-NEXT:    blr
+  %1 = srem <4 x i16> %x, <i16 95, i16 -124, i16 98, i16 -1003>
+  ret <4 x i16> %1
+}
+
+define <4 x i16> @fold_srem_vec_2(<4 x i16> %x) {
+; P9LE-LABEL: fold_srem_vec_2:
+; P9LE:       # %bb.0:
+; P9LE-NEXT:    li r3, 0
+; P9LE-NEXT:    vextuhrx r3, r3, v2
+; P9LE-NEXT:    extsh r4, r3
+; P9LE-NEXT:    lis r5, -21386
+; P9LE-NEXT:    ori r5, r5, 37253
+; P9LE-NEXT:    extsw r4, r4
+; P9LE-NEXT:    mulld r6, r4, r5
+; P9LE-NEXT:    rldicl r6, r6, 32, 32
+; P9LE-NEXT:    add r4, r6, r4
+; P9LE-NEXT:    srwi r6, r4, 31
+; P9LE-NEXT:    srawi r4, r4, 6
+; P9LE-NEXT:    add r4, r4, r6
+; P9LE-NEXT:    mulli r4, r4, 95
+; P9LE-NEXT:    subf r3, r4, r3
+; P9LE-NEXT:    mtvsrd f0, r3
+; P9LE-NEXT:    li r3, 2
+; P9LE-NEXT:    vextuhrx r3, r3, v2
+; P9LE-NEXT:    extsh r4, r3
+; P9LE-NEXT:    extsw r4, r4
+; P9LE-NEXT:    mulld r6, r4, r5
+; P9LE-NEXT:    rldicl r6, r6, 32, 32
+; P9LE-NEXT:    add r4, r6, r4
+; P9LE-NEXT:    srwi r6, r4, 31
+; P9LE-NEXT:    srawi r4, r4, 6
+; P9LE-NEXT:    add r4, r4, r6
+; P9LE-NEXT:    mulli r4, r4, 95
+; P9LE-NEXT:    subf r3, r4, r3
+; P9LE-NEXT:    xxswapd v3, vs0
+; P9LE-NEXT:    mtvsrd f0, r3
+; P9LE-NEXT:    li r3, 4
+; P9LE-NEXT:    vextuhrx r3, r3, v2
+; P9LE-NEXT:    extsh r4, r3
+; P9LE-NEXT:    extsw r4, r4
+; P9LE-NEXT:    mulld r6, r4, r5
+; P9LE-NEXT:    rldicl r6, r6, 32, 32
+; P9LE-NEXT:    add r4, r6, r4
+; P9LE-NEXT:    srwi r6, r4, 31
+; P9LE-NEXT:    srawi r4, r4, 6
+; P9LE-NEXT:    add r4, r4, r6
+; P9LE-NEXT:    mulli r4, r4, 95
+; P9LE-NEXT:    subf r3, r4, r3
+; P9LE-NEXT:    xxswapd v4, vs0
+; P9LE-NEXT:    mtvsrd f0, r3
+; P9LE-NEXT:    li r3, 6
+; P9LE-NEXT:    vextuhrx r3, r3, v2
+; P9LE-NEXT:    extsh r4, r3
+; P9LE-NEXT:    extsw r4, r4
+; P9LE-NEXT:    mulld r5, r4, r5
+; P9LE-NEXT:    rldicl r5, r5, 32, 32
+; P9LE-NEXT:    add r4, r5, r4
+; P9LE-NEXT:    srwi r5, r4, 31
+; P9LE-NEXT:    srawi r4, r4, 6
+; P9LE-NEXT:    add r4, r4, r5
+; P9LE-NEXT:    mulli r4, r4, 95
+; P9LE-NEXT:    subf r3, r4, r3
+; P9LE-NEXT:    vmrglh v3, v4, v3
+; P9LE-NEXT:    xxswapd v4, vs0
+; P9LE-NEXT:    mtvsrd f0, r3
+; P9LE-NEXT:    xxswapd v2, vs0
+; P9LE-NEXT:    vmrglh v2, v2, v4
+; P9LE-NEXT:    vmrglw v2, v2, v3
+; P9LE-NEXT:    blr
+;
+; P9BE-LABEL: fold_srem_vec_2:
+; P9BE:       # %bb.0:
+; P9BE-NEXT:    li r3, 6
+; P9BE-NEXT:    vextuhlx r3, r3, v2
+; P9BE-NEXT:    extsh r3, r3
+; P9BE-NEXT:    lis r4, -21386
+; P9BE-NEXT:    ori r4, r4, 37253
+; P9BE-NEXT:    extsw r3, r3
+; P9BE-NEXT:    mulld r5, r3, r4
+; P9BE-NEXT:    rldicl r5, r5, 32, 32
+; P9BE-NEXT:    add r5, r5, r3
+; P9BE-NEXT:    srwi r6, r5, 31
+; P9BE-NEXT:    srawi r5, r5, 6
+; P9BE-NEXT:    add r5, r5, r6
+; P9BE-NEXT:    mulli r5, r5, 95
+; P9BE-NEXT:    subf r3, r5, r3
+; P9BE-NEXT:    sldi r3, r3, 48
+; P9BE-NEXT:    mtvsrd v3, r3
+; P9BE-NEXT:    li r3, 4
+; P9BE-NEXT:    vextuhlx r3, r3, v2
+; P9BE-NEXT:    extsh r3, r3
+; P9BE-NEXT:    extsw r3, r3
+; P9BE-NEXT:    mulld r5, r3, r4
+; P9BE-NEXT:    rldicl r5, r5, 32, 32
+; P9BE-NEXT:    add r5, r5, r3
+; P9BE-NEXT:    srwi r6, r5, 31
+; P9BE-NEXT:    srawi r5, r5, 6
+; P9BE-NEXT:    add r5, r5, r6
+; P9BE-NEXT:    mulli r5, r5, 95
+; P9BE-NEXT:    subf r3, r5, r3
+; P9BE-NEXT:    sldi r3, r3, 48
+; P9BE-NEXT:    mtvsrd v4, r3
+; P9BE-NEXT:    li r3, 2
+; P9BE-NEXT:    vextuhlx r3, r3, v2
+; P9BE-NEXT:    extsh r3, r3
+; P9BE-NEXT:    extsw r3, r3
+; P9BE-NEXT:    mulld r5, r3, r4
+; P9BE-NEXT:    rldicl r5, r5, 32, 32
+; P9BE-NEXT:    add r5, r5, r3
+; P9BE-NEXT:    srwi r6, r5, 31
+; P9BE-NEXT:    srawi r5, r5, 6
+; P9BE-NEXT:    add r5, r5, r6
+; P9BE-NEXT:    mulli r5, r5, 95
+; P9BE-NEXT:    subf r3, r5, r3
+; P9BE-NEXT:    sldi r3, r3, 48
+; P9BE-NEXT:    vmrghh v3, v4, v3
+; P9BE-NEXT:    mtvsrd v4, r3
+; P9BE-NEXT:    li r3, 0
+; P9BE-NEXT:    vextuhlx r3, r3, v2
+; P9BE-NEXT:    extsh r3, r3
+; P9BE-NEXT:    extsw r3, r3
+; P9BE-NEXT:    mulld r4, r3, r4
+; P9BE-NEXT:    rldicl r4, r4, 32, 32
+; P9BE-NEXT:    add r4, r4, r3
+; P9BE-NEXT:    srwi r5, r4, 31
+; P9BE-NEXT:    srawi r4, r4, 6
+; P9BE-NEXT:    add r4, r4, r5
+; P9BE-NEXT:    mulli r4, r4, 95
+; P9BE-NEXT:    subf r3, r4, r3
+; P9BE-NEXT:    sldi r3, r3, 48
+; P9BE-NEXT:    mtvsrd v2, r3
+; P9BE-NEXT:    vmrghh v2, v2, v4
+; P9BE-NEXT:    vmrghw v2, v2, v3
+; P9BE-NEXT:    blr
+;
+; P8LE-LABEL: fold_srem_vec_2:
+; P8LE:       # %bb.0:
+; P8LE-NEXT:    xxswapd vs0, v2
+; P8LE-NEXT:    lis r4, -21386
+; P8LE-NEXT:    std r30, -16(r1) # 8-byte Folded Spill
+; P8LE-NEXT:    ori r4, r4, 37253
+; P8LE-NEXT:    mfvsrd r5, f0
+; P8LE-NEXT:    clrldi r3, r5, 48
+; P8LE-NEXT:    rldicl r7, r5, 32, 48
+; P8LE-NEXT:    extsh r8, r3
+; P8LE-NEXT:    rldicl r6, r5, 48, 48
+; P8LE-NEXT:    extsh r10, r7
+; P8LE-NEXT:    rldicl r5, r5, 16, 48
+; P8LE-NEXT:    extsw r8, r8
+; P8LE-NEXT:    extsh r9, r6
+; P8LE-NEXT:    extsw r10, r10
+; P8LE-NEXT:    extsh r11, r5
+; P8LE-NEXT:    mulld r12, r8, r4
+; P8LE-NEXT:    extsw r9, r9
+; P8LE-NEXT:    extsw r11, r11
+; P8LE-NEXT:    mulld r30, r10, r4
+; P8LE-NEXT:    mulld r0, r9, r4
+; P8LE-NEXT:    mulld r4, r11, r4
+; P8LE-NEXT:    rldicl r12, r12, 32, 32
+; P8LE-NEXT:    add r8, r12, r8
+; P8LE-NEXT:    rldicl r12, r30, 32, 32
+; P8LE-NEXT:    ld r30, -16(r1) # 8-byte Folded Reload
+; P8LE-NEXT:    rldicl r0, r0, 32, 32
+; P8LE-NEXT:    rldicl r4, r4, 32, 32
+; P8LE-NEXT:    add r10, r12, r10
+; P8LE-NEXT:    add r9, r0, r9
+; P8LE-NEXT:    srwi r0, r8, 31
+; P8LE-NEXT:    add r4, r4, r11
+; P8LE-NEXT:    srwi r11, r10, 31
+; P8LE-NEXT:    srawi r8, r8, 6
+; P8LE-NEXT:    srawi r10, r10, 6
+; P8LE-NEXT:    srwi r12, r9, 31
+; P8LE-NEXT:    add r8, r8, r0
+; P8LE-NEXT:    srawi r9, r9, 6
+; P8LE-NEXT:    add r10, r10, r11
+; P8LE-NEXT:    srwi r11, r4, 31
+; P8LE-NEXT:    srawi r4, r4, 6
+; P8LE-NEXT:    add r9, r9, r12
+; P8LE-NEXT:    mulli r8, r8, 95
+; P8LE-NEXT:    add r4, r4, r11
+; P8LE-NEXT:    mulli r9, r9, 95
+; P8LE-NEXT:    mulli r10, r10, 95
+; P8LE-NEXT:    mulli r4, r4, 95
+; P8LE-NEXT:    subf r3, r8, r3
+; P8LE-NEXT:    subf r6, r9, r6
+; P8LE-NEXT:    mtvsrd f0, r3
+; P8LE-NEXT:    subf r3, r10, r7
+; P8LE-NEXT:    subf r4, r4, r5
+; P8LE-NEXT:    mtvsrd f1, r6
+; P8LE-NEXT:    mtvsrd f2, r3
+; P8LE-NEXT:    xxswapd v2, vs0
+; P8LE-NEXT:    mtvsrd f3, r4
+; P8LE-NEXT:    xxswapd v3, vs1
+; P8LE-NEXT:    xxswapd v4, vs2
+; P8LE-NEXT:    xxswapd v5, vs3
+; P8LE-NEXT:    vmrglh v2, v3, v2
+; P8LE-NEXT:    vmrglh v3, v5, v4
+; P8LE-NEXT:    vmrglw v2, v3, v2
+; P8LE-NEXT:    blr
+;
+; P8BE-LABEL: fold_srem_vec_2:
+; P8BE:       # %bb.0:
+; P8BE-NEXT:    mfvsrd r4, v2
+; P8BE-NEXT:    lis r3, -21386
+; P8BE-NEXT:    ori r3, r3, 37253
+; P8BE-NEXT:    clrldi r5, r4, 48
+; P8BE-NEXT:    rldicl r6, r4, 48, 48
+; P8BE-NEXT:    extsh r5, r5
+; P8BE-NEXT:    rldicl r7, r4, 32, 48
+; P8BE-NEXT:    extsh r6, r6
+; P8BE-NEXT:    extsw r5, r5
+; P8BE-NEXT:    rldicl r4, r4, 16, 48
+; P8BE-NEXT:    extsh r7, r7
+; P8BE-NEXT:    extsw r6, r6
+; P8BE-NEXT:    mulld r8, r5, r3
+; P8BE-NEXT:    extsh r4, r4
+; P8BE-NEXT:    extsw r7, r7
+; P8BE-NEXT:    mulld r9, r6, r3
+; P8BE-NEXT:    extsw r4, r4
+; P8BE-NEXT:    mulld r10, r7, r3
+; P8BE-NEXT:    mulld r3, r4, r3
+; P8BE-NEXT:    rldicl r8, r8, 32, 32
+; P8BE-NEXT:    rldicl r9, r9, 32, 32
+; P8BE-NEXT:    add r8, r8, r5
+; P8BE-NEXT:    rldicl r10, r10, 32, 32
+; P8BE-NEXT:    add r9, r9, r6
+; P8BE-NEXT:    srwi r11, r8, 31
+; P8BE-NEXT:    srawi r8, r8, 6
+; P8BE-NEXT:    rldicl r3, r3, 32, 32
+; P8BE-NEXT:    add r10, r10, r7
+; P8BE-NEXT:    add r8, r8, r11
+; P8BE-NEXT:    srwi r11, r9, 31
+; P8BE-NEXT:    add r3, r3, r4
+; P8BE-NEXT:    srawi r9, r9, 6
+; P8BE-NEXT:    mulli r8, r8, 95
+; P8BE-NEXT:    add r9, r9, r11
+; P8BE-NEXT:    srwi r11, r10, 31
+; P8BE-NEXT:    srawi r10, r10, 6
+; P8BE-NEXT:    mulli r9, r9, 95
+; P8BE-NEXT:    add r10, r10, r11
+; P8BE-NEXT:    srwi r11, r3, 31
+; P8BE-NEXT:    srawi r3, r3, 6
+; P8BE-NEXT:    mulli r10, r10, 95
+; P8BE-NEXT:    subf r5, r8, r5
+; P8BE-NEXT:    add r3, r3, r11
+; P8BE-NEXT:    sldi r5, r5, 48
+; P8BE-NEXT:    mulli r3, r3, 95
+; P8BE-NEXT:    subf r6, r9, r6
+; P8BE-NEXT:    mtvsrd v2, r5
+; P8BE-NEXT:    sldi r6, r6, 48
+; P8BE-NEXT:    subf r7, r10, r7
+; P8BE-NEXT:    mtvsrd v3, r6
+; P8BE-NEXT:    subf r3, r3, r4
+; P8BE-NEXT:    sldi r4, r7, 48
+; P8BE-NEXT:    vmrghh v2, v3, v2
+; P8BE-NEXT:    sldi r3, r3, 48
+; P8BE-NEXT:    mtvsrd v4, r4
+; P8BE-NEXT:    mtvsrd v5, r3
+; P8BE-NEXT:    vmrghh v3, v5, v4
+; P8BE-NEXT:    vmrghw v2, v3, v2
+; P8BE-NEXT:    blr
+  %1 = srem <4 x i16> %x, <i16 95, i16 95, i16 95, i16 95>
+  ret <4 x i16> %1
+}
+
+
+; Don't fold if we can combine srem with sdiv.
+define <4 x i16> @combine_srem_sdiv(<4 x i16> %x) {
+; P9LE-LABEL: combine_srem_sdiv:
+; P9LE:       # %bb.0:
+; P9LE-NEXT:    li r3, 0
+; P9LE-NEXT:    vextuhrx r3, r3, v2
+; P9LE-NEXT:    extsh r4, r3
+; P9LE-NEXT:    lis r5, -21386
+; P9LE-NEXT:    ori r5, r5, 37253
+; P9LE-NEXT:    extsw r4, r4
+; P9LE-NEXT:    mulld r6, r4, r5
+; P9LE-NEXT:    rldicl r6, r6, 32, 32
+; P9LE-NEXT:    add r4, r6, r4
+; P9LE-NEXT:    srwi r6, r4, 31
+; P9LE-NEXT:    srawi r4, r4, 6
+; P9LE-NEXT:    add r4, r4, r6
+; P9LE-NEXT:    mulli r6, r4, 95
+; P9LE-NEXT:    subf r3, r6, r3
+; P9LE-NEXT:    mtvsrd f0, r3
+; P9LE-NEXT:    li r3, 2
+; P9LE-NEXT:    vextuhrx r3, r3, v2
+; P9LE-NEXT:    extsh r6, r3
+; P9LE-NEXT:    extsw r6, r6
+; P9LE-NEXT:    mulld r7, r6, r5
+; P9LE-NEXT:    rldicl r7, r7, 32, 32
+; P9LE-NEXT:    add r6, r7, r6
+; P9LE-NEXT:    srwi r7, r6, 31
+; P9LE-NEXT:    srawi r6, r6, 6
+; P9LE-NEXT:    add r6, r6, r7
+; P9LE-NEXT:    mulli r7, r6, 95
+; P9LE-NEXT:    subf r3, r7, r3
+; P9LE-NEXT:    xxswapd v3, vs0
+; P9LE-NEXT:    mtvsrd f0, r3
+; P9LE-NEXT:    li r3, 4
+; P9LE-NEXT:    vextuhrx r3, r3, v2
+; P9LE-NEXT:    extsh r7, r3
+; P9LE-NEXT:    extsw r7, r7
+; P9LE-NEXT:    mulld r8, r7, r5
+; P9LE-NEXT:    rldicl r8, r8, 32, 32
+; P9LE-NEXT:    add r7, r8, r7
+; P9LE-NEXT:    srwi r8, r7, 31
+; P9LE-NEXT:    srawi r7, r7, 6
+; P9LE-NEXT:    add r7, r7, r8
+; P9LE-NEXT:    mulli r8, r7, 95
+; P9LE-NEXT:    subf r3, r8, r3
+; P9LE-NEXT:    xxswapd v4, vs0
+; P9LE-NEXT:    mtvsrd f0, r3
+; P9LE-NEXT:    li r3, 6
+; P9LE-NEXT:    vextuhrx r3, r3, v2
+; P9LE-NEXT:    extsh r8, r3
+; P9LE-NEXT:    extsw r8, r8
+; P9LE-NEXT:    mulld r5, r8, r5
+; P9LE-NEXT:    rldicl r5, r5, 32, 32
+; P9LE-NEXT:    add r5, r5, r8
+; P9LE-NEXT:    srwi r8, r5, 31
+; P9LE-NEXT:    srawi r5, r5, 6
+; P9LE-NEXT:    add r5, r5, r8
+; P9LE-NEXT:    mulli r8, r5, 95
+; P9LE-NEXT:    subf r3, r8, r3
+; P9LE-NEXT:    vmrglh v3, v4, v3
+; P9LE-NEXT:    xxswapd v4, vs0
+; P9LE-NEXT:    mtvsrd f0, r3
+; P9LE-NEXT:    xxswapd v2, vs0
+; P9LE-NEXT:    mtvsrd f0, r4
+; P9LE-NEXT:    vmrglh v2, v2, v4
+; P9LE-NEXT:    vmrglw v2, v2, v3
+; P9LE-NEXT:    xxswapd v3, vs0
+; P9LE-NEXT:    mtvsrd f0, r6
+; P9LE-NEXT:    xxswapd v4, vs0
+; P9LE-NEXT:    mtvsrd f0, r7
+; P9LE-NEXT:    vmrglh v3, v4, v3
+; P9LE-NEXT:    xxswapd v4, vs0
+; P9LE-NEXT:    mtvsrd f0, r5
+; P9LE-NEXT:    xxswapd v5, vs0
+; P9LE-NEXT:    vmrglh v4, v5, v4
+; P9LE-NEXT:    vmrglw v3, v4, v3
+; P9LE-NEXT:    vadduhm v2, v2, v3
+; P9LE-NEXT:    blr
+;
+; P9BE-LABEL: combine_srem_sdiv:
+; P9BE:       # %bb.0:
+; P9BE-NEXT:    li r3, 6
+; P9BE-NEXT:    vextuhlx r3, r3, v2
+; P9BE-NEXT:    extsh r4, r3
+; P9BE-NEXT:    lis r5, -21386
+; P9BE-NEXT:    ori r5, r5, 37253
+; P9BE-NEXT:    extsw r4, r4
+; P9BE-NEXT:    mulld r6, r4, r5
+; P9BE-NEXT:    rldicl r6, r6, 32, 32
+; P9BE-NEXT:    add r4, r6, r4
+; P9BE-NEXT:    srwi r6, r4, 31
+; P9BE-NEXT:    srawi r4, r4, 6
+; P9BE-NEXT:    add r4, r4, r6
+; P9BE-NEXT:    mulli r6, r4, 95
+; P9BE-NEXT:    subf r3, r6, r3
+; P9BE-NEXT:    sldi r3, r3, 48
+; P9BE-NEXT:    mtvsrd v3, r3
+; P9BE-NEXT:    li r3, 4
+; P9BE-NEXT:    vextuhlx r3, r3, v2
+; P9BE-NEXT:    extsh r6, r3
+; P9BE-NEXT:    extsw r6, r6
+; P9BE-NEXT:    mulld r7, r6, r5
+; P9BE-NEXT:    rldicl r7, r7, 32, 32
+; P9BE-NEXT:    add r6, r7, r6
+; P9BE-NEXT:    srwi r7, r6, 31
+; P9BE-NEXT:    srawi r6, r6, 6
+; P9BE-NEXT:    add r6, r6, r7
+; P9BE-NEXT:    mulli r7, r6, 95
+; P9BE-NEXT:    subf r3, r7, r3
+; P9BE-NEXT:    sldi r3, r3, 48
+; P9BE-NEXT:    mtvsrd v4, r3
+; P9BE-NEXT:    li r3, 2
+; P9BE-NEXT:    vextuhlx r3, r3, v2
+; P9BE-NEXT:    extsh r7, r3
+; P9BE-NEXT:    extsw r7, r7
+; P9BE-NEXT:    mulld r8, r7, r5
+; P9BE-NEXT:    rldicl r8, r8, 32, 32
+; P9BE-NEXT:    add r7, r8, r7
+; P9BE-NEXT:    srwi r8, r7, 31
+; P9BE-NEXT:    srawi r7, r7, 6
+; P9BE-NEXT:    add r7, r7, r8
+; P9BE-NEXT:    mulli r8, r7, 95
+; P9BE-NEXT:    subf r3, r8, r3
+; P9BE-NEXT:    sldi r3, r3, 48
+; P9BE-NEXT:    vmrghh v3, v4, v3
+; P9BE-NEXT:    mtvsrd v4, r3
+; P9BE-NEXT:    li r3, 0
+; P9BE-NEXT:    vextuhlx r3, r3, v2
+; P9BE-NEXT:    extsh r3, r3
+; P9BE-NEXT:    extsw r3, r3
+; P9BE-NEXT:    mulld r5, r3, r5
+; P9BE-NEXT:    rldicl r5, r5, 32, 32
+; P9BE-NEXT:    add r5, r5, r3
+; P9BE-NEXT:    srwi r8, r5, 31
+; P9BE-NEXT:    srawi r5, r5, 6
+; P9BE-NEXT:    add r5, r5, r8
+; P9BE-NEXT:    mulli r8, r5, 95
+; P9BE-NEXT:    subf r3, r8, r3
+; P9BE-NEXT:    sldi r3, r3, 48
+; P9BE-NEXT:    mtvsrd v2, r3
+; P9BE-NEXT:    sldi r3, r4, 48
+; P9BE-NEXT:    vmrghh v2, v2, v4
+; P9BE-NEXT:    vmrghw v2, v2, v3
+; P9BE-NEXT:    mtvsrd v3, r3
+; P9BE-NEXT:    sldi r3, r6, 48
+; P9BE-NEXT:    mtvsrd v4, r3
+; P9BE-NEXT:    sldi r3, r7, 48
+; P9BE-NEXT:    vmrghh v3, v4, v3
+; P9BE-NEXT:    mtvsrd v4, r3
+; P9BE-NEXT:    sldi r3, r5, 48
+; P9BE-NEXT:    mtvsrd v5, r3
+; P9BE-NEXT:    vmrghh v4, v5, v4
+; P9BE-NEXT:    vmrghw v3, v4, v3
+; P9BE-NEXT:    vadduhm v2, v2, v3
+; P9BE-NEXT:    blr
+;
+; P8LE-LABEL: combine_srem_sdiv:
+; P8LE:       # %bb.0:
+; P8LE-NEXT:    xxswapd vs0, v2
+; P8LE-NEXT:    lis r5, -21386
+; P8LE-NEXT:    std r30, -16(r1) # 8-byte Folded Spill
+; P8LE-NEXT:    ori r5, r5, 37253
+; P8LE-NEXT:    mfvsrd r6, f0
+; P8LE-NEXT:    clrldi r3, r6, 48
+; P8LE-NEXT:    rldicl r4, r6, 48, 48
+; P8LE-NEXT:    rldicl r7, r6, 32, 48
+; P8LE-NEXT:    extsh r8, r3
+; P8LE-NEXT:    extsh r9, r4
+; P8LE-NEXT:    rldicl r6, r6, 16, 48
+; P8LE-NEXT:    extsh r10, r7
+; P8LE-NEXT:    extsw r8, r8
+; P8LE-NEXT:    extsw r9, r9
+; P8LE-NEXT:    extsh r11, r6
+; P8LE-NEXT:    extsw r10, r10
+; P8LE-NEXT:    mulld r12, r8, r5
+; P8LE-NEXT:    extsw r11, r11
+; P8LE-NEXT:    mulld r0, r9, r5
+; P8LE-NEXT:    mulld r30, r10, r5
+; P8LE-NEXT:    mulld r5, r11, r5
+; P8LE-NEXT:    rldicl r12, r12, 32, 32
+; P8LE-NEXT:    rldicl r0, r0, 32, 32
+; P8LE-NEXT:    rldicl r30, r30, 32, 32
+; P8LE-NEXT:    add r8, r12, r8
+; P8LE-NEXT:    rldicl r5, r5, 32, 32
+; P8LE-NEXT:    add r9, r0, r9
+; P8LE-NEXT:    add r10, r30, r10
+; P8LE-NEXT:    srwi r12, r8, 31
+; P8LE-NEXT:    ld r30, -16(r1) # 8-byte Folded Reload
+; P8LE-NEXT:    srawi r8, r8, 6
+; P8LE-NEXT:    srawi r0, r9, 6
+; P8LE-NEXT:    srwi r9, r9, 31
+; P8LE-NEXT:    add r5, r5, r11
+; P8LE-NEXT:    add r8, r8, r12
+; P8LE-NEXT:    srawi r12, r10, 6
+; P8LE-NEXT:    srwi r10, r10, 31
+; P8LE-NEXT:    add r9, r0, r9
+; P8LE-NEXT:    mulli r0, r8, 95
+; P8LE-NEXT:    add r10, r12, r10
+; P8LE-NEXT:    mtvsrd f0, r8
+; P8LE-NEXT:    srwi r8, r5, 31
+; P8LE-NEXT:    srawi r5, r5, 6
+; P8LE-NEXT:    mulli r11, r9, 95
+; P8LE-NEXT:    mtvsrd f1, r9
+; P8LE-NEXT:    mulli r9, r10, 95
+; P8LE-NEXT:    add r5, r5, r8
+; P8LE-NEXT:    xxswapd v2, vs0
+; P8LE-NEXT:    mtvsrd f2, r10
+; P8LE-NEXT:    mtvsrd f3, r5
+; P8LE-NEXT:    mulli r5, r5, 95
+; P8LE-NEXT:    xxswapd v3, vs1
+; P8LE-NEXT:    subf r3, r0, r3
+; P8LE-NEXT:    xxswapd v1, vs2
+; P8LE-NEXT:    mtvsrd f0, r3
+; P8LE-NEXT:    subf r4, r11, r4
+; P8LE-NEXT:    xxswapd v6, vs3
+; P8LE-NEXT:    subf r3, r9, r7
+; P8LE-NEXT:    mtvsrd f1, r4
+; P8LE-NEXT:    mtvsrd f4, r3
+; P8LE-NEXT:    subf r3, r5, r6
+; P8LE-NEXT:    mtvsrd f5, r3
+; P8LE-NEXT:    xxswapd v4, vs1
+; P8LE-NEXT:    vmrglh v2, v3, v2
+; P8LE-NEXT:    xxswapd v3, vs0
+; P8LE-NEXT:    xxswapd v5, vs4
+; P8LE-NEXT:    xxswapd v0, vs5
+; P8LE-NEXT:    vmrglh v3, v4, v3
+; P8LE-NEXT:    vmrglh v4, v0, v5
+; P8LE-NEXT:    vmrglh v5, v6, v1
+; P8LE-NEXT:    vmrglw v3, v4, v3
+; P8LE-NEXT:    vmrglw v2, v5, v2
+; P8LE-NEXT:    vadduhm v2, v3, v2
+; P8LE-NEXT:    blr
+;
+; P8BE-LABEL: combine_srem_sdiv:
+; P8BE:       # %bb.0:
+; P8BE-NEXT:    mfvsrd r6, v2
+; P8BE-NEXT:    lis r5, -21386
+; P8BE-NEXT:    ori r5, r5, 37253
+; P8BE-NEXT:    clrldi r3, r6, 48
+; P8BE-NEXT:    rldicl r4, r6, 48, 48
+; P8BE-NEXT:    extsh r8, r3
+; P8BE-NEXT:    rldicl r7, r6, 32, 48
+; P8BE-NEXT:    extsh r9, r4
+; P8BE-NEXT:    rldicl r6, r6, 16, 48
+; P8BE-NEXT:    extsw r8, r8
+; P8BE-NEXT:    extsh r10, r7
+; P8BE-NEXT:    extsw r9, r9
+; P8BE-NEXT:    extsh r6, r6
+; P8BE-NEXT:    mulld r11, r8, r5
+; P8BE-NEXT:    extsw r10, r10
+; P8BE-NEXT:    extsw r6, r6
+; P8BE-NEXT:    mulld r12, r9, r5
+; P8BE-NEXT:    mulld r0, r10, r5
+; P8BE-NEXT:    mulld r5, r6, r5
+; P8BE-NEXT:    rldicl r11, r11, 32, 32
+; P8BE-NEXT:    rldicl r12, r12, 32, 32
+; P8BE-NEXT:    add r8, r11, r8
+; P8BE-NEXT:    rldicl r0, r0, 32, 32
+; P8BE-NEXT:    rldicl r5, r5, 32, 32
+; P8BE-NEXT:    add r9, r12, r9
+; P8BE-NEXT:    srawi r11, r8, 6
+; P8BE-NEXT:    srwi r8, r8, 31
+; P8BE-NEXT:    add r10, r0, r10
+; P8BE-NEXT:    add r5, r5, r6
+; P8BE-NEXT:    srawi r12, r9, 6
+; P8BE-NEXT:    srwi r9, r9, 31
+; P8BE-NEXT:    add r8, r11, r8
+; P8BE-NEXT:    srawi r0, r10, 6
+; P8BE-NEXT:    srawi r11, r5, 6
+; P8BE-NEXT:    srwi r10, r10, 31
+; P8BE-NEXT:    add r9, r12, r9
+; P8BE-NEXT:    srwi r5, r5, 31
+; P8BE-NEXT:    mulli r12, r8, 95
+; P8BE-NEXT:    add r10, r0, r10
+; P8BE-NEXT:    add r5, r11, r5
+; P8BE-NEXT:    mulli r0, r9, 95
+; P8BE-NEXT:    sldi r9, r9, 48
+; P8BE-NEXT:    sldi r8, r8, 48
+; P8BE-NEXT:    mtvsrd v3, r9
+; P8BE-NEXT:    mulli r9, r5, 95
+; P8BE-NEXT:    mtvsrd v2, r8
+; P8BE-NEXT:    mulli r8, r10, 95
+; P8BE-NEXT:    sldi r10, r10, 48
+; P8BE-NEXT:    subf r3, r12, r3
+; P8BE-NEXT:    mtvsrd v4, r10
+; P8BE-NEXT:    subf r4, r0, r4
+; P8BE-NEXT:    sldi r3, r3, 48
+; P8BE-NEXT:    vmrghh v2, v3, v2
+; P8BE-NEXT:    sldi r4, r4, 48
+; P8BE-NEXT:    mtvsrd v3, r3
+; P8BE-NEXT:    subf r3, r9, r6
+; P8BE-NEXT:    subf r7, r8, r7
+; P8BE-NEXT:    mtvsrd v5, r4
+; P8BE-NEXT:    sldi r3, r3, 48
+; P8BE-NEXT:    sldi r6, r7, 48
+; P8BE-NEXT:    mtvsrd v1, r3
+; P8BE-NEXT:    sldi r3, r5, 48
+; P8BE-NEXT:    mtvsrd v0, r6
+; P8BE-NEXT:    vmrghh v3, v5, v3
+; P8BE-NEXT:    mtvsrd v5, r3
+; P8BE-NEXT:    vmrghh v0, v1, v0
+; P8BE-NEXT:    vmrghh v4, v5, v4
+; P8BE-NEXT:    vmrghw v3, v0, v3
+; P8BE-NEXT:    vmrghw v2, v4, v2
+; P8BE-NEXT:    vadduhm v2, v3, v2
+; P8BE-NEXT:    blr
+  %1 = srem <4 x i16> %x, <i16 95, i16 95, i16 95, i16 95>
+  %2 = sdiv <4 x i16> %x, <i16 95, i16 95, i16 95, i16 95>
+  %3 = add <4 x i16> %1, %2
+  ret <4 x i16> %3
+}
+
+; Don't fold for divisors that are a power of two.
+define <4 x i16> @dont_fold_srem_power_of_two(<4 x i16> %x) {
+; P9LE-LABEL: dont_fold_srem_power_of_two:
+; P9LE:       # %bb.0:
+; P9LE-NEXT:    li r3, 0
+; P9LE-NEXT:    vextuhrx r3, r3, v2
+; P9LE-NEXT:    extsh r4, r3
+; P9LE-NEXT:    srawi r4, r4, 6
+; P9LE-NEXT:    addze r4, r4
+; P9LE-NEXT:    slwi r4, r4, 6
+; P9LE-NEXT:    subf r3, r4, r3
+; P9LE-NEXT:    mtvsrd f0, r3
+; P9LE-NEXT:    li r3, 2
+; P9LE-NEXT:    vextuhrx r3, r3, v2
+; P9LE-NEXT:    extsh r4, r3
+; P9LE-NEXT:    srawi r4, r4, 5
+; P9LE-NEXT:    addze r4, r4
+; P9LE-NEXT:    slwi r4, r4, 5
+; P9LE-NEXT:    subf r3, r4, r3
+; P9LE-NEXT:    xxswapd v3, vs0
+; P9LE-NEXT:    mtvsrd f0, r3
+; P9LE-NEXT:    li r3, 6
+; P9LE-NEXT:    vextuhrx r3, r3, v2
+; P9LE-NEXT:    extsh r4, r3
+; P9LE-NEXT:    lis r5, -21386
+; P9LE-NEXT:    ori r5, r5, 37253
+; P9LE-NEXT:    xxswapd v4, vs0
+; P9LE-NEXT:    vmrglh v3, v4, v3
+; P9LE-NEXT:    extsw r4, r4
+; P9LE-NEXT:    mulld r5, r4, r5
+; P9LE-NEXT:    rldicl r5, r5, 32, 32
+; P9LE-NEXT:    add r4, r5, r4
+; P9LE-NEXT:    srwi r5, r4, 31
+; P9LE-NEXT:    srawi r4, r4, 6
+; P9LE-NEXT:    add r4, r4, r5
+; P9LE-NEXT:    mulli r4, r4, 95
+; P9LE-NEXT:    subf r3, r4, r3
+; P9LE-NEXT:    mtvsrd f0, r3
+; P9LE-NEXT:    li r3, 4
+; P9LE-NEXT:    vextuhrx r3, r3, v2
+; P9LE-NEXT:    extsh r4, r3
+; P9LE-NEXT:    srawi r4, r4, 3
+; P9LE-NEXT:    addze r4, r4
+; P9LE-NEXT:    slwi r4, r4, 3
+; P9LE-NEXT:    subf r3, r4, r3
+; P9LE-NEXT:    xxswapd v4, vs0
+; P9LE-NEXT:    mtvsrd f0, r3
+; P9LE-NEXT:    xxswapd v2, vs0
+; P9LE-NEXT:    vmrglh v2, v4, v2
+; P9LE-NEXT:    vmrglw v2, v2, v3
+; P9LE-NEXT:    blr
+;
+; P9BE-LABEL: dont_fold_srem_power_of_two:
+; P9BE:       # %bb.0:
+; P9BE-NEXT:    li r3, 2
+; P9BE-NEXT:    vextuhlx r3, r3, v2
+; P9BE-NEXT:    extsh r3, r3
+; P9BE-NEXT:    srawi r4, r3, 5
+; P9BE-NEXT:    addze r4, r4
+; P9BE-NEXT:    slwi r4, r4, 5
+; P9BE-NEXT:    subf r3, r4, r3
+; P9BE-NEXT:    sldi r3, r3, 48
+; P9BE-NEXT:    mtvsrd v3, r3
+; P9BE-NEXT:    li r3, 0
+; P9BE-NEXT:    vextuhlx r3, r3, v2
+; P9BE-NEXT:    extsh r3, r3
+; P9BE-NEXT:    srawi r4, r3, 6
+; P9BE-NEXT:    addze r4, r4
+; P9BE-NEXT:    slwi r4, r4, 6
+; P9BE-NEXT:    subf r3, r4, r3
+; P9BE-NEXT:    lis r4, -21386
+; P9BE-NEXT:    sldi r3, r3, 48
+; P9BE-NEXT:    mtvsrd v4, r3
+; P9BE-NEXT:    li r3, 6
+; P9BE-NEXT:    vextuhlx r3, r3, v2
+; P9BE-NEXT:    extsh r3, r3
+; P9BE-NEXT:    extsw r3, r3
+; P9BE-NEXT:    ori r4, r4, 37253
+; P9BE-NEXT:    mulld r4, r3, r4
+; P9BE-NEXT:    rldicl r4, r4, 32, 32
+; P9BE-NEXT:    add r4, r4, r3
+; P9BE-NEXT:    srwi r5, r4, 31
+; P9BE-NEXT:    srawi r4, r4, 6
+; P9BE-NEXT:    add r4, r4, r5
+; P9BE-NEXT:    mulli r4, r4, 95
+; P9BE-NEXT:    subf r3, r4, r3
+; P9BE-NEXT:    sldi r3, r3, 48
+; P9BE-NEXT:    vmrghh v3, v4, v3
+; P9BE-NEXT:    mtvsrd v4, r3
+; P9BE-NEXT:    li r3, 4
+; P9BE-NEXT:    vextuhlx r3, r3, v2
+; P9BE-NEXT:    extsh r3, r3
+; P9BE-NEXT:    srawi r4, r3, 3
+; P9BE-NEXT:    addze r4, r4
+; P9BE-NEXT:    slwi r4, r4, 3
+; P9BE-NEXT:    subf r3, r4, r3
+; P9BE-NEXT:    sldi r3, r3, 48
+; P9BE-NEXT:    mtvsrd v2, r3
+; P9BE-NEXT:    vmrghh v2, v2, v4
+; P9BE-NEXT:    vmrghw v2, v3, v2
+; P9BE-NEXT:    blr
+;
+; P8LE-LABEL: dont_fold_srem_power_of_two:
+; P8LE:       # %bb.0:
+; P8LE-NEXT:    xxswapd vs0, v2
+; P8LE-NEXT:    lis r3, -21386
+; P8LE-NEXT:    ori r3, r3, 37253
+; P8LE-NEXT:    mfvsrd r4, f0
+; P8LE-NEXT:    rldicl r5, r4, 16, 48
+; P8LE-NEXT:    clrldi r7, r4, 48
+; P8LE-NEXT:    extsh r6, r5
+; P8LE-NEXT:    extsh r8, r7
+; P8LE-NEXT:    extsw r6, r6
+; P8LE-NEXT:    rldicl r9, r4, 48, 48
+; P8LE-NEXT:    mulld r3, r6, r3
+; P8LE-NEXT:    srawi r8, r8, 6
+; P8LE-NEXT:    extsh r10, r9
+; P8LE-NEXT:    addze r8, r8
+; P8LE-NEXT:    rldicl r4, r4, 32, 48
+; P8LE-NEXT:    srawi r10, r10, 5
+; P8LE-NEXT:    slwi r8, r8, 6
+; P8LE-NEXT:    subf r7, r8, r7
+; P8LE-NEXT:    rldicl r3, r3, 32, 32
+; P8LE-NEXT:    mtvsrd f0, r7
+; P8LE-NEXT:    add r3, r3, r6
+; P8LE-NEXT:    addze r6, r10
+; P8LE-NEXT:    srwi r10, r3, 31
+; P8LE-NEXT:    srawi r3, r3, 6
+; P8LE-NEXT:    slwi r6, r6, 5
+; P8LE-NEXT:    xxswapd v2, vs0
+; P8LE-NEXT:    add r3, r3, r10
+; P8LE-NEXT:    extsh r10, r4
+; P8LE-NEXT:    subf r6, r6, r9
+; P8LE-NEXT:    mulli r3, r3, 95
+; P8LE-NEXT:    srawi r8, r10, 3
+; P8LE-NEXT:    mtvsrd f1, r6
+; P8LE-NEXT:    addze r7, r8
+; P8LE-NEXT:    xxswapd v3, vs1
+; P8LE-NEXT:    subf r3, r3, r5
+; P8LE-NEXT:    slwi r5, r7, 3
+; P8LE-NEXT:    subf r4, r5, r4
+; P8LE-NEXT:    mtvsrd f2, r3
+; P8LE-NEXT:    mtvsrd f3, r4
+; P8LE-NEXT:    xxswapd v4, vs2
+; P8LE-NEXT:    vmrglh v2, v3, v2
+; P8LE-NEXT:    xxswapd v5, vs3
+; P8LE-NEXT:    vmrglh v3, v4, v5
+; P8LE-NEXT:    vmrglw v2, v3, v2
+; P8LE-NEXT:    blr
+;
+; P8BE-LABEL: dont_fold_srem_power_of_two:
+; P8BE:       # %bb.0:
+; P8BE-NEXT:    mfvsrd r4, v2
+; P8BE-NEXT:    lis r3, -21386
+; P8BE-NEXT:    ori r3, r3, 37253
+; P8BE-NEXT:    clrldi r5, r4, 48
+; P8BE-NEXT:    rldicl r6, r4, 32, 48
+; P8BE-NEXT:    extsh r5, r5
+; P8BE-NEXT:    extsh r6, r6
+; P8BE-NEXT:    extsw r5, r5
+; P8BE-NEXT:    rldicl r7, r4, 16, 48
+; P8BE-NEXT:    mulld r3, r5, r3
+; P8BE-NEXT:    srawi r8, r6, 5
+; P8BE-NEXT:    extsh r7, r7
+; P8BE-NEXT:    addze r8, r8
+; P8BE-NEXT:    rldicl r4, r4, 48, 48
+; P8BE-NEXT:    srawi r9, r7, 6
+; P8BE-NEXT:    extsh r4, r4
+; P8BE-NEXT:    slwi r8, r8, 5
+; P8BE-NEXT:    addze r9, r9
+; P8BE-NEXT:    subf r6, r8, r6
+; P8BE-NEXT:    rldicl r3, r3, 32, 32
+; P8BE-NEXT:    slwi r8, r9, 6
+; P8BE-NEXT:    add r3, r3, r5
+; P8BE-NEXT:    subf r7, r8, r7
+; P8BE-NEXT:    srwi r10, r3, 31
+; P8BE-NEXT:    srawi r3, r3, 6
+; P8BE-NEXT:    add r3, r3, r10
+; P8BE-NEXT:    srawi r9, r4, 3
+; P8BE-NEXT:    mulli r3, r3, 95
+; P8BE-NEXT:    sldi r6, r6, 48
+; P8BE-NEXT:    addze r8, r9
+; P8BE-NEXT:    mtvsrd v2, r6
+; P8BE-NEXT:    slwi r6, r8, 3
+; P8BE-NEXT:    subf r4, r6, r4
+; P8BE-NEXT:    sldi r4, r4, 48
+; P8BE-NEXT:    subf r3, r3, r5
+; P8BE-NEXT:    sldi r5, r7, 48
+; P8BE-NEXT:    mtvsrd v5, r4
+; P8BE-NEXT:    sldi r3, r3, 48
+; P8BE-NEXT:    mtvsrd v3, r5
+; P8BE-NEXT:    mtvsrd v4, r3
+; P8BE-NEXT:    vmrghh v2, v3, v2
+; P8BE-NEXT:    vmrghh v3, v5, v4
+; P8BE-NEXT:    vmrghw v2, v2, v3
+; P8BE-NEXT:    blr
+  %1 = srem <4 x i16> %x, <i16 64, i16 32, i16 8, i16 95>
+  ret <4 x i16> %1
+}
+
+; Don't fold if the divisor is one.
+define <4 x i16> @dont_fold_srem_one(<4 x i16> %x) {
+; P9LE-LABEL: dont_fold_srem_one:
+; P9LE:       # %bb.0:
+; P9LE-NEXT:    li r3, 2
+; P9LE-NEXT:    vextuhrx r3, r3, v2
+; P9LE-NEXT:    extsh r4, r3
+; P9LE-NEXT:    lis r5, -14230
+; P9LE-NEXT:    ori r5, r5, 30865
+; P9LE-NEXT:    extsw r4, r4
+; P9LE-NEXT:    mulld r5, r4, r5
+; P9LE-NEXT:    rldicl r5, r5, 32, 32
+; P9LE-NEXT:    xxlxor v4, v4, v4
+; P9LE-NEXT:    add r4, r5, r4
+; P9LE-NEXT:    srwi r5, r4, 31
+; P9LE-NEXT:    srawi r4, r4, 9
+; P9LE-NEXT:    add r4, r4, r5
+; P9LE-NEXT:    lis r5, -19946
+; P9LE-NEXT:    mulli r4, r4, 654
+; P9LE-NEXT:    subf r3, r4, r3
+; P9LE-NEXT:    mtvsrd f0, r3
+; P9LE-NEXT:    li r3, 4
+; P9LE-NEXT:    vextuhrx r3, r3, v2
+; P9LE-NEXT:    extsh r4, r3
+; P9LE-NEXT:    extsw r4, r4
+; P9LE-NEXT:    ori r5, r5, 17097
+; P9LE-NEXT:    mulld r5, r4, r5
+; P9LE-NEXT:    rldicl r5, r5, 32, 32
+; P9LE-NEXT:    add r4, r5, r4
+; P9LE-NEXT:    srwi r5, r4, 31
+; P9LE-NEXT:    srawi r4, r4, 4
+; P9LE-NEXT:    add r4, r4, r5
+; P9LE-NEXT:    lis r5, 24749
+; P9LE-NEXT:    mulli r4, r4, 23
+; P9LE-NEXT:    subf r3, r4, r3
+; P9LE-NEXT:    xxswapd v3, vs0
+; P9LE-NEXT:    mtvsrd f0, r3
+; P9LE-NEXT:    li r3, 6
+; P9LE-NEXT:    vextuhrx r3, r3, v2
+; P9LE-NEXT:    extsh r4, r3
+; P9LE-NEXT:    extsw r4, r4
+; P9LE-NEXT:    ori r5, r5, 47143
+; P9LE-NEXT:    mulld r4, r4, r5
+; P9LE-NEXT:    rldicl r5, r4, 1, 63
+; P9LE-NEXT:    rldicl r4, r4, 32, 32
+; P9LE-NEXT:    srawi r4, r4, 11
+; P9LE-NEXT:    add r4, r4, r5
+; P9LE-NEXT:    mulli r4, r4, 5423
+; P9LE-NEXT:    subf r3, r4, r3
+; P9LE-NEXT:    vmrglh v3, v3, v4
+; P9LE-NEXT:    xxswapd v4, vs0
+; P9LE-NEXT:    mtvsrd f0, r3
+; P9LE-NEXT:    xxswapd v2, vs0
+; P9LE-NEXT:    vmrglh v2, v2, v4
+; P9LE-NEXT:    vmrglw v2, v2, v3
+; P9LE-NEXT:    blr
+;
+; P9BE-LABEL: dont_fold_srem_one:
+; P9BE:       # %bb.0:
+; P9BE-NEXT:    li r3, 4
+; P9BE-NEXT:    vextuhlx r3, r3, v2
+; P9BE-NEXT:    extsh r3, r3
+; P9BE-NEXT:    lis r4, -19946
+; P9BE-NEXT:    ori r4, r4, 17097
+; P9BE-NEXT:    extsw r3, r3
+; P9BE-NEXT:    mulld r4, r3, r4
+; P9BE-NEXT:    rldicl r4, r4, 32, 32
+; P9BE-NEXT:    add r4, r4, r3
+; P9BE-NEXT:    srwi r5, r4, 31
+; P9BE-NEXT:    srawi r4, r4, 4
+; P9BE-NEXT:    add r4, r4, r5
+; P9BE-NEXT:    mulli r4, r4, 23
+; P9BE-NEXT:    subf r3, r4, r3
+; P9BE-NEXT:    lis r4, 24749
+; P9BE-NEXT:    sldi r3, r3, 48
+; P9BE-NEXT:    mtvsrd v3, r3
+; P9BE-NEXT:    li r3, 6
+; P9BE-NEXT:    vextuhlx r3, r3, v2
+; P9BE-NEXT:    extsh r3, r3
+; P9BE-NEXT:    extsw r3, r3
+; P9BE-NEXT:    ori r4, r4, 47143
+; P9BE-NEXT:    mulld r4, r3, r4
+; P9BE-NEXT:    rldicl r5, r4, 1, 63
+; P9BE-NEXT:    rldicl r4, r4, 32, 32
+; P9BE-NEXT:    srawi r4, r4, 11
+; P9BE-NEXT:    add r4, r4, r5
+; P9BE-NEXT:    mulli r4, r4, 5423
+; P9BE-NEXT:    subf r3, r4, r3
+; P9BE-NEXT:    lis r4, -14230
+; P9BE-NEXT:    sldi r3, r3, 48
+; P9BE-NEXT:    mtvsrd v4, r3
+; P9BE-NEXT:    li r3, 2
+; P9BE-NEXT:    vextuhlx r3, r3, v2
+; P9BE-NEXT:    extsh r3, r3
+; P9BE-NEXT:    extsw r3, r3
+; P9BE-NEXT:    ori r4, r4, 30865
+; P9BE-NEXT:    mulld r4, r3, r4
+; P9BE-NEXT:    rldicl r4, r4, 32, 32
+; P9BE-NEXT:    add r4, r4, r3
+; P9BE-NEXT:    srwi r5, r4, 31
+; P9BE-NEXT:    srawi r4, r4, 9
+; P9BE-NEXT:    add r4, r4, r5
+; P9BE-NEXT:    mulli r4, r4, 654
+; P9BE-NEXT:    subf r3, r4, r3
+; P9BE-NEXT:    sldi r3, r3, 48
+; P9BE-NEXT:    mtvsrd v2, r3
+; P9BE-NEXT:    li r3, 0
+; P9BE-NEXT:    sldi r3, r3, 48
+; P9BE-NEXT:    vmrghh v3, v3, v4
+; P9BE-NEXT:    mtvsrd v4, r3
+; P9BE-NEXT:    vmrghh v2, v4, v2
+; P9BE-NEXT:    vmrghw v2, v2, v3
+; P9BE-NEXT:    blr
+;
+; P8LE-LABEL: dont_fold_srem_one:
+; P8LE:       # %bb.0:
+; P8LE-NEXT:    xxswapd vs0, v2
+; P8LE-NEXT:    lis r3, 24749
+; P8LE-NEXT:    lis r8, -19946
+; P8LE-NEXT:    lis r10, -14230
+; P8LE-NEXT:    xxlxor v5, v5, v5
+; P8LE-NEXT:    ori r3, r3, 47143
+; P8LE-NEXT:    ori r8, r8, 17097
+; P8LE-NEXT:    mfvsrd r4, f0
+; P8LE-NEXT:    rldicl r5, r4, 16, 48
+; P8LE-NEXT:    rldicl r6, r4, 32, 48
+; P8LE-NEXT:    rldicl r4, r4, 48, 48
+; P8LE-NEXT:    extsh r7, r5
+; P8LE-NEXT:    extsh r9, r6
+; P8LE-NEXT:    extsw r7, r7
+; P8LE-NEXT:    extsh r11, r4
+; P8LE-NEXT:    extsw r9, r9
+; P8LE-NEXT:    mulld r3, r7, r3
+; P8LE-NEXT:    ori r7, r10, 30865
+; P8LE-NEXT:    extsw r10, r11
+; P8LE-NEXT:    mulld r8, r9, r8
+; P8LE-NEXT:    mulld r7, r10, r7
+; P8LE-NEXT:    rldicl r11, r3, 1, 63
+; P8LE-NEXT:    rldicl r3, r3, 32, 32
+; P8LE-NEXT:    rldicl r8, r8, 32, 32
+; P8LE-NEXT:    rldicl r7, r7, 32, 32
+; P8LE-NEXT:    add r8, r8, r9
+; P8LE-NEXT:    srawi r3, r3, 11
+; P8LE-NEXT:    add r7, r7, r10
+; P8LE-NEXT:    srwi r9, r8, 31
+; P8LE-NEXT:    srawi r8, r8, 4
+; P8LE-NEXT:    add r3, r3, r11
+; P8LE-NEXT:    add r8, r8, r9
+; P8LE-NEXT:    srwi r9, r7, 31
+; P8LE-NEXT:    srawi r7, r7, 9
+; P8LE-NEXT:    mulli r3, r3, 5423
+; P8LE-NEXT:    add r7, r7, r9
+; P8LE-NEXT:    mulli r8, r8, 23
+; P8LE-NEXT:    mulli r7, r7, 654
+; P8LE-NEXT:    subf r3, r3, r5
+; P8LE-NEXT:    mtvsrd f0, r3
+; P8LE-NEXT:    subf r3, r8, r6
+; P8LE-NEXT:    subf r4, r7, r4
+; P8LE-NEXT:    mtvsrd f1, r3
+; P8LE-NEXT:    mtvsrd f2, r4
+; P8LE-NEXT:    xxswapd v2, vs0
+; P8LE-NEXT:    xxswapd v3, vs1
+; P8LE-NEXT:    xxswapd v4, vs2
+; P8LE-NEXT:    vmrglh v2, v2, v3
+; P8LE-NEXT:    vmrglh v3, v4, v5
+; P8LE-NEXT:    vmrglw v2, v2, v3
+; P8LE-NEXT:    blr
+;
+; P8BE-LABEL: dont_fold_srem_one:
+; P8BE:       # %bb.0:
+; P8BE-NEXT:    mfvsrd r4, v2
+; P8BE-NEXT:    lis r3, 24749
+; P8BE-NEXT:    lis r7, -19946
+; P8BE-NEXT:    lis r8, -14230
+; P8BE-NEXT:    ori r3, r3, 47143
+; P8BE-NEXT:    ori r7, r7, 17097
+; P8BE-NEXT:    ori r8, r8, 30865
+; P8BE-NEXT:    clrldi r5, r4, 48
+; P8BE-NEXT:    rldicl r6, r4, 48, 48
+; P8BE-NEXT:    rldicl r4, r4, 32, 48
+; P8BE-NEXT:    extsh r5, r5
+; P8BE-NEXT:    extsh r6, r6
+; P8BE-NEXT:    extsh r4, r4
+; P8BE-NEXT:    extsw r5, r5
+; P8BE-NEXT:    extsw r6, r6
+; P8BE-NEXT:    extsw r4, r4
+; P8BE-NEXT:    mulld r3, r5, r3
+; P8BE-NEXT:    mulld r7, r6, r7
+; P8BE-NEXT:    mulld r8, r4, r8
+; P8BE-NEXT:    rldicl r9, r3, 1, 63
+; P8BE-NEXT:    rldicl r3, r3, 32, 32
+; P8BE-NEXT:    rldicl r7, r7, 32, 32
+; P8BE-NEXT:    rldicl r8, r8, 32, 32
+; P8BE-NEXT:    srawi r3, r3, 11
+; P8BE-NEXT:    add r7, r7, r6
+; P8BE-NEXT:    add r8, r8, r4
+; P8BE-NEXT:    add r3, r3, r9
+; P8BE-NEXT:    srwi r9, r7, 31
+; P8BE-NEXT:    srawi r7, r7, 4
+; P8BE-NEXT:    mulli r3, r3, 5423
+; P8BE-NEXT:    add r7, r7, r9
+; P8BE-NEXT:    srwi r9, r8, 31
+; P8BE-NEXT:    srawi r8, r8, 9
+; P8BE-NEXT:    mulli r7, r7, 23
+; P8BE-NEXT:    add r8, r8, r9
+; P8BE-NEXT:    li r9, 0
+; P8BE-NEXT:    mulli r8, r8, 654
+; P8BE-NEXT:    subf r3, r3, r5
+; P8BE-NEXT:    sldi r5, r9, 48
+; P8BE-NEXT:    sldi r3, r3, 48
+; P8BE-NEXT:    mtvsrd v2, r5
+; P8BE-NEXT:    subf r5, r7, r6
+; P8BE-NEXT:    mtvsrd v3, r3
+; P8BE-NEXT:    sldi r3, r5, 48
+; P8BE-NEXT:    subf r4, r8, r4
+; P8BE-NEXT:    mtvsrd v4, r3
+; P8BE-NEXT:    sldi r4, r4, 48
+; P8BE-NEXT:    mtvsrd v5, r4
+; P8BE-NEXT:    vmrghh v3, v4, v3
+; P8BE-NEXT:    vmrghh v2, v2, v5
+; P8BE-NEXT:    vmrghw v2, v2, v3
+; P8BE-NEXT:    blr
+  %1 = srem <4 x i16> %x, <i16 1, i16 654, i16 23, i16 5423>
+  ret <4 x i16> %1
+}
+
+; Don't fold if the divisor is 2^15.
+define <4 x i16> @dont_fold_urem_i16_smax(<4 x i16> %x) {
+; P9LE-LABEL: dont_fold_urem_i16_smax:
+; P9LE:       # %bb.0:
+; P9LE-NEXT:    li r3, 4
+; P9LE-NEXT:    vextuhrx r3, r3, v2
+; P9LE-NEXT:    extsh r4, r3
+; P9LE-NEXT:    lis r5, -19946
+; P9LE-NEXT:    ori r5, r5, 17097
+; P9LE-NEXT:    extsw r4, r4
+; P9LE-NEXT:    mulld r5, r4, r5
+; P9LE-NEXT:    rldicl r5, r5, 32, 32
+; P9LE-NEXT:    add r4, r5, r4
+; P9LE-NEXT:    srwi r5, r4, 31
+; P9LE-NEXT:    srawi r4, r4, 4
+; P9LE-NEXT:    add r4, r4, r5
+; P9LE-NEXT:    lis r5, 24749
+; P9LE-NEXT:    mulli r4, r4, 23
+; P9LE-NEXT:    subf r3, r4, r3
+; P9LE-NEXT:    mtvsrd f0, r3
+; P9LE-NEXT:    li r3, 6
+; P9LE-NEXT:    vextuhrx r3, r3, v2
+; P9LE-NEXT:    extsh r4, r3
+; P9LE-NEXT:    extsw r4, r4
+; P9LE-NEXT:    ori r5, r5, 47143
+; P9LE-NEXT:    mulld r4, r4, r5
+; P9LE-NEXT:    rldicl r5, r4, 1, 63
+; P9LE-NEXT:    rldicl r4, r4, 32, 32
+; P9LE-NEXT:    srawi r4, r4, 11
+; P9LE-NEXT:    add r4, r4, r5
+; P9LE-NEXT:    mulli r4, r4, 5423
+; P9LE-NEXT:    subf r3, r4, r3
+; P9LE-NEXT:    xxswapd v3, vs0
+; P9LE-NEXT:    mtvsrd f0, r3
+; P9LE-NEXT:    li r3, 2
+; P9LE-NEXT:    vextuhrx r3, r3, v2
+; P9LE-NEXT:    extsh r4, r3
+; P9LE-NEXT:    srawi r4, r4, 15
+; P9LE-NEXT:    addze r4, r4
+; P9LE-NEXT:    slwi r4, r4, 15
+; P9LE-NEXT:    subf r3, r4, r3
+; P9LE-NEXT:    xxswapd v4, vs0
+; P9LE-NEXT:    mtvsrd f0, r3
+; P9LE-NEXT:    xxswapd v2, vs0
+; P9LE-NEXT:    vmrglh v3, v4, v3
+; P9LE-NEXT:    xxlxor v4, v4, v4
+; P9LE-NEXT:    vmrglh v2, v2, v4
+; P9LE-NEXT:    vmrglw v2, v3, v2
+; P9LE-NEXT:    blr
+;
+; P9BE-LABEL: dont_fold_urem_i16_smax:
+; P9BE:       # %bb.0:
+; P9BE-NEXT:    li r3, 4
+; P9BE-NEXT:    vextuhlx r3, r3, v2
+; P9BE-NEXT:    extsh r3, r3
+; P9BE-NEXT:    lis r4, -19946
+; P9BE-NEXT:    ori r4, r4, 17097
+; P9BE-NEXT:    extsw r3, r3
+; P9BE-NEXT:    mulld r4, r3, r4
+; P9BE-NEXT:    rldicl r4, r4, 32, 32
+; P9BE-NEXT:    add r4, r4, r3
+; P9BE-NEXT:    srwi r5, r4, 31
+; P9BE-NEXT:    srawi r4, r4, 4
+; P9BE-NEXT:    add r4, r4, r5
+; P9BE-NEXT:    mulli r4, r4, 23
+; P9BE-NEXT:    subf r3, r4, r3
+; P9BE-NEXT:    lis r4, 24749
+; P9BE-NEXT:    sldi r3, r3, 48
+; P9BE-NEXT:    mtvsrd v3, r3
+; P9BE-NEXT:    li r3, 6
+; P9BE-NEXT:    vextuhlx r3, r3, v2
+; P9BE-NEXT:    extsh r3, r3
+; P9BE-NEXT:    extsw r3, r3
+; P9BE-NEXT:    ori r4, r4, 47143
+; P9BE-NEXT:    mulld r4, r3, r4
+; P9BE-NEXT:    rldicl r5, r4, 1, 63
+; P9BE-NEXT:    rldicl r4, r4, 32, 32
+; P9BE-NEXT:    srawi r4, r4, 11
+; P9BE-NEXT:    add r4, r4, r5
+; P9BE-NEXT:    mulli r4, r4, 5423
+; P9BE-NEXT:    subf r3, r4, r3
+; P9BE-NEXT:    sldi r3, r3, 48
+; P9BE-NEXT:    mtvsrd v4, r3
+; P9BE-NEXT:    li r3, 2
+; P9BE-NEXT:    vextuhlx r3, r3, v2
+; P9BE-NEXT:    extsh r3, r3
+; P9BE-NEXT:    srawi r4, r3, 15
+; P9BE-NEXT:    addze r4, r4
+; P9BE-NEXT:    slwi r4, r4, 15
+; P9BE-NEXT:    subf r3, r4, r3
+; P9BE-NEXT:    sldi r3, r3, 48
+; P9BE-NEXT:    mtvsrd v2, r3
+; P9BE-NEXT:    li r3, 0
+; P9BE-NEXT:    sldi r3, r3, 48
+; P9BE-NEXT:    vmrghh v3, v3, v4
+; P9BE-NEXT:    mtvsrd v4, r3
+; P9BE-NEXT:    vmrghh v2, v4, v2
+; P9BE-NEXT:    vmrghw v2, v2, v3
+; P9BE-NEXT:    blr
+;
+; P8LE-LABEL: dont_fold_urem_i16_smax:
+; P8LE:       # %bb.0:
+; P8LE-NEXT:    xxswapd vs0, v2
+; P8LE-NEXT:    lis r6, 24749
+; P8LE-NEXT:    lis r7, -19946
+; P8LE-NEXT:    xxlxor v5, v5, v5
+; P8LE-NEXT:    ori r6, r6, 47143
+; P8LE-NEXT:    ori r7, r7, 17097
+; P8LE-NEXT:    mfvsrd r3, f0
+; P8LE-NEXT:    rldicl r4, r3, 16, 48
+; P8LE-NEXT:    rldicl r5, r3, 32, 48
+; P8LE-NEXT:    extsh r8, r4
+; P8LE-NEXT:    extsh r9, r5
+; P8LE-NEXT:    extsw r8, r8
+; P8LE-NEXT:    extsw r9, r9
+; P8LE-NEXT:    mulld r6, r8, r6
+; P8LE-NEXT:    mulld r7, r9, r7
+; P8LE-NEXT:    rldicl r3, r3, 48, 48
+; P8LE-NEXT:    rldicl r8, r6, 32, 32
+; P8LE-NEXT:    rldicl r7, r7, 32, 32
+; P8LE-NEXT:    rldicl r6, r6, 1, 63
+; P8LE-NEXT:    srawi r8, r8, 11
+; P8LE-NEXT:    add r7, r7, r9
+; P8LE-NEXT:    add r6, r8, r6
+; P8LE-NEXT:    srwi r8, r7, 31
+; P8LE-NEXT:    srawi r7, r7, 4
+; P8LE-NEXT:    mulli r6, r6, 5423
+; P8LE-NEXT:    add r7, r7, r8
+; P8LE-NEXT:    extsh r8, r3
+; P8LE-NEXT:    mulli r7, r7, 23
+; P8LE-NEXT:    srawi r8, r8, 15
+; P8LE-NEXT:    subf r4, r6, r4
+; P8LE-NEXT:    addze r6, r8
+; P8LE-NEXT:    mtvsrd f0, r4
+; P8LE-NEXT:    slwi r4, r6, 15
+; P8LE-NEXT:    subf r5, r7, r5
+; P8LE-NEXT:    subf r3, r4, r3
+; P8LE-NEXT:    mtvsrd f1, r5
+; P8LE-NEXT:    xxswapd v2, vs0
+; P8LE-NEXT:    mtvsrd f2, r3
+; P8LE-NEXT:    xxswapd v3, vs1
+; P8LE-NEXT:    xxswapd v4, vs2
+; P8LE-NEXT:    vmrglh v2, v2, v3
+; P8LE-NEXT:    vmrglh v3, v4, v5
+; P8LE-NEXT:    vmrglw v2, v2, v3
+; P8LE-NEXT:    blr
+;
+; P8BE-LABEL: dont_fold_urem_i16_smax:
+; P8BE:       # %bb.0:
+; P8BE-NEXT:    mfvsrd r4, v2
+; P8BE-NEXT:    lis r3, 24749
+; P8BE-NEXT:    lis r7, -19946
+; P8BE-NEXT:    ori r3, r3, 47143
+; P8BE-NEXT:    ori r7, r7, 17097
+; P8BE-NEXT:    clrldi r5, r4, 48
+; P8BE-NEXT:    rldicl r6, r4, 48, 48
+; P8BE-NEXT:    extsh r5, r5
+; P8BE-NEXT:    extsh r6, r6
+; P8BE-NEXT:    extsw r5, r5
+; P8BE-NEXT:    extsw r6, r6
+; P8BE-NEXT:    mulld r3, r5, r3
+; P8BE-NEXT:    mulld r7, r6, r7
+; P8BE-NEXT:    rldicl r4, r4, 32, 48
+; P8BE-NEXT:    extsh r4, r4
+; P8BE-NEXT:    rldicl r8, r3, 1, 63
+; P8BE-NEXT:    rldicl r3, r3, 32, 32
+; P8BE-NEXT:    rldicl r7, r7, 32, 32
+; P8BE-NEXT:    srawi r3, r3, 11
+; P8BE-NEXT:    add r7, r7, r6
+; P8BE-NEXT:    add r3, r3, r8
+; P8BE-NEXT:    srwi r8, r7, 31
+; P8BE-NEXT:    srawi r7, r7, 4
+; P8BE-NEXT:    mulli r3, r3, 5423
+; P8BE-NEXT:    add r7, r7, r8
+; P8BE-NEXT:    li r8, 0
+; P8BE-NEXT:    mulli r7, r7, 23
+; P8BE-NEXT:    srawi r9, r4, 15
+; P8BE-NEXT:    subf r3, r3, r5
+; P8BE-NEXT:    sldi r5, r8, 48
+; P8BE-NEXT:    addze r8, r9
+; P8BE-NEXT:    mtvsrd v2, r5
+; P8BE-NEXT:    subf r5, r7, r6
+; P8BE-NEXT:    slwi r6, r8, 15
+; P8BE-NEXT:    sldi r3, r3, 48
+; P8BE-NEXT:    subf r4, r6, r4
+; P8BE-NEXT:    mtvsrd v3, r3
+; P8BE-NEXT:    sldi r3, r5, 48
+; P8BE-NEXT:    sldi r4, r4, 48
+; P8BE-NEXT:    mtvsrd v4, r3
+; P8BE-NEXT:    mtvsrd v5, r4
+; P8BE-NEXT:    vmrghh v3, v4, v3
+; P8BE-NEXT:    vmrghh v2, v2, v5
+; P8BE-NEXT:    vmrghw v2, v2, v3
+; P8BE-NEXT:    blr
+  %1 = srem <4 x i16> %x, <i16 1, i16 32768, i16 23, i16 5423>
+  ret <4 x i16> %1
+}
+
+; Don't fold i64 srem.
+define <4 x i64> @dont_fold_srem_i64(<4 x i64> %x) {
+; P9LE-LABEL: dont_fold_srem_i64:
+; P9LE:       # %bb.0:
+; P9LE-NEXT:    lis r4, 24749
+; P9LE-NEXT:    ori r4, r4, 47142
+; P9LE-NEXT:    sldi r4, r4, 32
+; P9LE-NEXT:    oris r4, r4, 58853
+; P9LE-NEXT:    mfvsrd r3, v3
+; P9LE-NEXT:    ori r4, r4, 6055
+; P9LE-NEXT:    mulhd r4, r3, r4
+; P9LE-NEXT:    rldicl r5, r4, 1, 63
+; P9LE-NEXT:    sradi r4, r4, 11
+; P9LE-NEXT:    add r4, r4, r5
+; P9LE-NEXT:    lis r5, -19946
+; P9LE-NEXT:    mulli r4, r4, 5423
+; P9LE-NEXT:    ori r5, r5, 17096
+; P9LE-NEXT:    sldi r5, r5, 32
+; P9LE-NEXT:    oris r5, r5, 22795
+; P9LE-NEXT:    sub r3, r3, r4
+; P9LE-NEXT:    mfvsrld r4, v3
+; P9LE-NEXT:    ori r5, r5, 8549
+; P9LE-NEXT:    mulhd r5, r4, r5
+; P9LE-NEXT:    add r5, r5, r4
+; P9LE-NEXT:    rldicl r6, r5, 1, 63
+; P9LE-NEXT:    sradi r5, r5, 4
+; P9LE-NEXT:    add r5, r5, r6
+; P9LE-NEXT:    mulli r5, r5, 23
+; P9LE-NEXT:    sub r4, r4, r5
+; P9LE-NEXT:    mtvsrdd v3, r3, r4
+; P9LE-NEXT:    lis r4, 25653
+; P9LE-NEXT:    ori r4, r4, 15432
+; P9LE-NEXT:    sldi r4, r4, 32
+; P9LE-NEXT:    oris r4, r4, 1603
+; P9LE-NEXT:    mfvsrd r3, v2
+; P9LE-NEXT:    ori r4, r4, 21445
+; P9LE-NEXT:    mulhd r4, r3, r4
+; P9LE-NEXT:    rldicl r5, r4, 1, 63
+; P9LE-NEXT:    sradi r4, r4, 8
+; P9LE-NEXT:    add r4, r4, r5
+; P9LE-NEXT:    mulli r4, r4, 654
+; P9LE-NEXT:    sub r3, r3, r4
+; P9LE-NEXT:    li r4, 0
+; P9LE-NEXT:    mtvsrdd v2, r3, r4
+; P9LE-NEXT:    blr
+;
+; P9BE-LABEL: dont_fold_srem_i64:
+; P9BE:       # %bb.0:
+; P9BE-NEXT:    lis r4, 24749
+; P9BE-NEXT:    ori r4, r4, 47142
+; P9BE-NEXT:    sldi r4, r4, 32
+; P9BE-NEXT:    oris r4, r4, 58853
+; P9BE-NEXT:    mfvsrld r3, v3
+; P9BE-NEXT:    ori r4, r4, 6055
+; P9BE-NEXT:    mulhd r4, r3, r4
+; P9BE-NEXT:    rldicl r5, r4, 1, 63
+; P9BE-NEXT:    sradi r4, r4, 11
+; P9BE-NEXT:    add r4, r4, r5
+; P9BE-NEXT:    lis r5, -19946
+; P9BE-NEXT:    ori r5, r5, 17096
+; P9BE-NEXT:    mulli r4, r4, 5423
+; P9BE-NEXT:    sldi r5, r5, 32
+; P9BE-NEXT:    oris r5, r5, 22795
+; P9BE-NEXT:    sub r3, r3, r4
+; P9BE-NEXT:    mfvsrd r4, v3
+; P9BE-NEXT:    ori r5, r5, 8549
+; P9BE-NEXT:    mulhd r5, r4, r5
+; P9BE-NEXT:    add r5, r5, r4
+; P9BE-NEXT:    rldicl r6, r5, 1, 63
+; P9BE-NEXT:    sradi r5, r5, 4
+; P9BE-NEXT:    add r5, r5, r6
+; P9BE-NEXT:    mulli r5, r5, 23
+; P9BE-NEXT:    sub r4, r4, r5
+; P9BE-NEXT:    mtvsrdd v3, r4, r3
+; P9BE-NEXT:    lis r4, 25653
+; P9BE-NEXT:    ori r4, r4, 15432
+; P9BE-NEXT:    sldi r4, r4, 32
+; P9BE-NEXT:    oris r4, r4, 1603
+; P9BE-NEXT:    mfvsrld r3, v2
+; P9BE-NEXT:    ori r4, r4, 21445
+; P9BE-NEXT:    mulhd r4, r3, r4
+; P9BE-NEXT:    rldicl r5, r4, 1, 63
+; P9BE-NEXT:    sradi r4, r4, 8
+; P9BE-NEXT:    add r4, r4, r5
+; P9BE-NEXT:    mulli r4, r4, 654
+; P9BE-NEXT:    sub r3, r3, r4
+; P9BE-NEXT:    mtvsrdd v2, 0, r3
+; P9BE-NEXT:    blr
+;
+; P8LE-LABEL: dont_fold_srem_i64:
+; P8LE:       # %bb.0:
+; P8LE-NEXT:    lis r3, 24749
+; P8LE-NEXT:    lis r4, -19946
+; P8LE-NEXT:    lis r5, 25653
+; P8LE-NEXT:    xxswapd vs0, v3
+; P8LE-NEXT:    mfvsrd r6, v3
+; P8LE-NEXT:    ori r3, r3, 47142
+; P8LE-NEXT:    ori r4, r4, 17096
+; P8LE-NEXT:    ori r5, r5, 15432
+; P8LE-NEXT:    mfvsrd r7, v2
+; P8LE-NEXT:    sldi r3, r3, 32
+; P8LE-NEXT:    sldi r4, r4, 32
+; P8LE-NEXT:    sldi r5, r5, 32
+; P8LE-NEXT:    oris r3, r3, 58853
+; P8LE-NEXT:    oris r4, r4, 22795
+; P8LE-NEXT:    mfvsrd r8, f0
+; P8LE-NEXT:    oris r5, r5, 1603
+; P8LE-NEXT:    ori r3, r3, 6055
+; P8LE-NEXT:    ori r4, r4, 8549
+; P8LE-NEXT:    ori r5, r5, 21445
+; P8LE-NEXT:    mulhd r3, r6, r3
+; P8LE-NEXT:    mulhd r5, r7, r5
+; P8LE-NEXT:    mulhd r4, r8, r4
+; P8LE-NEXT:    rldicl r9, r3, 1, 63
+; P8LE-NEXT:    sradi r3, r3, 11
+; P8LE-NEXT:    add r3, r3, r9
+; P8LE-NEXT:    rldicl r9, r5, 1, 63
+; P8LE-NEXT:    add r4, r4, r8
+; P8LE-NEXT:    sradi r5, r5, 8
+; P8LE-NEXT:    mulli r3, r3, 5423
+; P8LE-NEXT:    add r5, r5, r9
+; P8LE-NEXT:    rldicl r9, r4, 1, 63
+; P8LE-NEXT:    sradi r4, r4, 4
+; P8LE-NEXT:    mulli r5, r5, 654
+; P8LE-NEXT:    add r4, r4, r9
+; P8LE-NEXT:    mulli r4, r4, 23
+; P8LE-NEXT:    sub r3, r6, r3
+; P8LE-NEXT:    mtvsrd f0, r3
+; P8LE-NEXT:    sub r5, r7, r5
+; P8LE-NEXT:    mtvsrd f1, r5
+; P8LE-NEXT:    sub r3, r8, r4
+; P8LE-NEXT:    li r4, 0
+; P8LE-NEXT:    mtvsrd f2, r3
+; P8LE-NEXT:    mtvsrd f3, r4
+; P8LE-NEXT:    xxmrghd v3, vs0, vs2
+; P8LE-NEXT:    xxmrghd v2, vs1, vs3
+; P8LE-NEXT:    blr
+;
+; P8BE-LABEL: dont_fold_srem_i64:
+; P8BE:       # %bb.0:
+; P8BE-NEXT:    lis r4, -19946
+; P8BE-NEXT:    lis r3, 24749
+; P8BE-NEXT:    xxswapd vs0, v3
+; P8BE-NEXT:    lis r5, 25653
+; P8BE-NEXT:    xxswapd vs1, v2
+; P8BE-NEXT:    ori r4, r4, 17096
+; P8BE-NEXT:    ori r3, r3, 47142
+; P8BE-NEXT:    ori r5, r5, 15432
+; P8BE-NEXT:    mfvsrd r6, v3
+; P8BE-NEXT:    sldi r4, r4, 32
+; P8BE-NEXT:    sldi r3, r3, 32
+; P8BE-NEXT:    oris r4, r4, 22795
+; P8BE-NEXT:    sldi r5, r5, 32
+; P8BE-NEXT:    oris r3, r3, 58853
+; P8BE-NEXT:    mfvsrd r7, f0
+; P8BE-NEXT:    ori r4, r4, 8549
+; P8BE-NEXT:    ori r3, r3, 6055
+; P8BE-NEXT:    oris r5, r5, 1603
+; P8BE-NEXT:    mfvsrd r8, f1
+; P8BE-NEXT:    mulhd r4, r6, r4
+; P8BE-NEXT:    mulhd r3, r7, r3
+; P8BE-NEXT:    ori r5, r5, 21445
+; P8BE-NEXT:    mulhd r5, r8, r5
+; P8BE-NEXT:    add r4, r4, r6
+; P8BE-NEXT:    rldicl r9, r3, 1, 63
+; P8BE-NEXT:    sradi r3, r3, 11
+; P8BE-NEXT:    rldicl r10, r4, 1, 63
+; P8BE-NEXT:    sradi r4, r4, 4
+; P8BE-NEXT:    add r3, r3, r9
+; P8BE-NEXT:    rldicl r9, r5, 1, 63
+; P8BE-NEXT:    add r4, r4, r10
+; P8BE-NEXT:    sradi r5, r5, 8
+; P8BE-NEXT:    mulli r3, r3, 5423
+; P8BE-NEXT:    add r5, r5, r9
+; P8BE-NEXT:    mulli r4, r4, 23
+; P8BE-NEXT:    mulli r5, r5, 654
+; P8BE-NEXT:    sub r3, r7, r3
+; P8BE-NEXT:    sub r4, r6, r4
+; P8BE-NEXT:    mtvsrd f0, r3
+; P8BE-NEXT:    sub r3, r8, r5
+; P8BE-NEXT:    mtvsrd f1, r4
+; P8BE-NEXT:    li r4, 0
+; P8BE-NEXT:    mtvsrd f2, r3
+; P8BE-NEXT:    mtvsrd f3, r4
+; P8BE-NEXT:    xxmrghd v3, vs1, vs0
+; P8BE-NEXT:    xxmrghd v2, vs3, vs2
+; P8BE-NEXT:    blr
+  %1 = srem <4 x i64> %x, <i64 1, i64 654, i64 23, i64 5423>
+  ret <4 x i64> %1
+}
diff --git a/llvm/test/CodeGen/PowerPC/stack-guard-reassign.ll b/llvm/test/CodeGen/PowerPC/stack-guard-reassign.ll
index 8128b63d5..e20a8cd11 100644
--- a/llvm/test/CodeGen/PowerPC/stack-guard-reassign.ll
+++ b/llvm/test/CodeGen/PowerPC/stack-guard-reassign.ll
@@ -9,9 +9,8 @@
 ; CHECK-NEXT: ori 0, 0, 65488
 ; CHECK-NEXT: stwux 1, 1, 0
 ; CHECK-NEXT: subf 0, 0, 1
-; CHECK-NEXT: lis 4, 1
-; CHECK-NEXT: ori 4, 4, 44
-; CHECK-NEXT: add 4, 1, 4
-; CHECK-NEXT: lis 5, __stack_chk_guard@ha
-; CHECK-NEXT: lwz 6, __stack_chk_guard@l(5)
-; CHECK-NEXT: stw 6, 0(4)
+; CHECK-NEXT: lis 4, __stack_chk_guard@ha
+; CHECK-NEXT: lwz 5, __stack_chk_guard@l(4)
+; CHECK-NEXT: lis 6, 1
+; CHECK-NEXT: ori 6, 6, 44
+; CHECK-NEXT: stwx 5, 1, 6
diff --git a/llvm/test/CodeGen/PowerPC/store-combine.ll b/llvm/test/CodeGen/PowerPC/store-combine.ll
index ed8f2309c..86be121eb 100644
--- a/llvm/test/CodeGen/PowerPC/store-combine.ll
+++ b/llvm/test/CodeGen/PowerPC/store-combine.ll
@@ -15,7 +15,7 @@ define void @store_i32_by_i8(i32 signext %m, i8* %p) {
 ;
 ; CHECK-PPC64-LABEL: store_i32_by_i8:
 ; CHECK-PPC64:       # %bb.0: # %entry
-; CHECK-PPC64-NEXT:    stwbrx 3, 0, 4 
+; CHECK-PPC64-NEXT:    stwbrx 3, 0, 4
 ; CHECK-PPC64-NEXT:    blr
 entry:
   %conv = trunc i32 %m to i8
@@ -187,18 +187,18 @@ entry:
 define void @store_i64_by_i8_bswap_uses(i32 signext %t, i8* %p) {
 ; CHECK-PPC64LE-LABEL: store_i64_by_i8_bswap_uses:
 ; CHECK-PPC64LE:       # %bb.0: # %entry
-; CHECK-PPC64LE-NEXT:    slwi [[REG:[0-9]+]], 3, 3
-; CHECK-PPC64LE-NEXT:    subf [[REG1:[0-9]+]], 3, [[REG]] 
-; CHECK-PPC64LE-NEXT:    extsw [[REG2:[0-9]+]], [[REG1]]
-; CHECK-PPC64LE-NEXT:    stdbrx [[REG2]], 0, 4
+; CHECK-PPC64LE-NEXT:    slwi 5, 3, 3
+; CHECK-PPC64LE-NEXT:    subf 3, 3, 5
+; CHECK-PPC64LE-NEXT:    extsw 3, 3
+; CHECK-PPC64LE-NEXT:    stdbrx 3, 0, 4
 ; CHECK-PPC64LE-NEXT:    blr
 ;
 ; CHECK-PPC64-LABEL: store_i64_by_i8_bswap_uses:
 ; CHECK-PPC64:       # %bb.0: # %entry
-; CHECK-PPC64-NEXT:    slwi [[REG:[0-9]+]], 3, 3
-; CHECK-PPC64-NEXT:    subf [[REG1:[0-9]+]], 3, [[REG]]
-; CHECK-PPC64-NEXT:    extsw [[REG2:[0-9]+]], [[REG1]]
-; CHECK-PPC64-NEXT:    stdx [[REG2]], 0, 4
+; CHECK-PPC64-NEXT:    slwi 5, 3, 3
+; CHECK-PPC64-NEXT:    subf 3, 3, 5
+; CHECK-PPC64-NEXT:    extsw 3, 3
+; CHECK-PPC64-NEXT:    stdx 3, 0, 4
 ; CHECK-PPC64-NEXT:    blr
 entry:
   %mul = mul nsw i32 %t, 7
@@ -234,8 +234,6 @@ entry:
   %conv28 = trunc i64 %shr26 to i8
   store i8 %conv28, i8* %p, align 1
   ret void
-; CEHCK-PPC64LE: stdbrx [[REG2]], 0, 4
-; CEHCK-PPC64: stdx [[REG2]], 0, 4
 }
 
 ; One of the stores is volatile
@@ -248,11 +246,22 @@ entry:
 define void @store_i32_by_i8_bswap_volatile(i32 signext %m, i8* %p) {
 ; CHECK-PPC64LE-LABEL: store_i32_by_i8_bswap_volatile:
 ; CHECK-PPC64LE:       # %bb.0: # %entry
-; CHECK-PPC64LE-NOT:   stwbrx 
+; CHECK-PPC64LE-NEXT:    li 5, 2
+; CHECK-PPC64LE-NEXT:    sthbrx 3, 4, 5
+; CHECK-PPC64LE-NEXT:    srwi 5, 3, 16
+; CHECK-PPC64LE-NEXT:    srwi 3, 3, 24
+; CHECK-PPC64LE-NEXT:    stb 5, 1(4)
+; CHECK-PPC64LE-NEXT:    stb 3, 0(4)
+; CHECK-PPC64LE-NEXT:    blr
 ;
 ; CHECK-PPC64-LABEL: store_i32_by_i8_bswap_volatile:
 ; CHECK-PPC64:       # %bb.0: # %entry
-; CHECK-PPC64-NOT:   stw 
+; CHECK-PPC64-NEXT:    sth 3, 2(4)
+; CHECK-PPC64-NEXT:    srwi 5, 3, 16
+; CHECK-PPC64-NEXT:    srwi 3, 3, 24
+; CHECK-PPC64-NEXT:    stb 5, 1(4)
+; CHECK-PPC64-NEXT:    stb 3, 0(4)
+; CHECK-PPC64-NEXT:    blr
 entry:
   %conv = trunc i32 %m to i8
   %arrayidx = getelementptr inbounds i8, i8* %p, i64 3
@@ -281,11 +290,26 @@ entry:
 define void @store_i32_by_i8_bswap_store_in_between(i32 signext %m, i8* %p, i8* %q) {
 ; CHECK-PPC64LE-LABEL: store_i32_by_i8_bswap_store_in_between:
 ; CHECK-PPC64LE:       # %bb.0: # %entry
-; CHECK-PPC64LE-NOT:   stwbrx 
+; CHECK-PPC64LE-NEXT:    li 6, 2
+; CHECK-PPC64LE-NEXT:    sthbrx 3, 4, 6
+; CHECK-PPC64LE-NEXT:    li 6, 3
+; CHECK-PPC64LE-NEXT:    stb 6, 0(5)
+; CHECK-PPC64LE-NEXT:    srwi 5, 3, 16
+; CHECK-PPC64LE-NEXT:    srwi 3, 3, 24
+; CHECK-PPC64LE-NEXT:    stb 5, 1(4)
+; CHECK-PPC64LE-NEXT:    stb 3, 0(4)
+; CHECK-PPC64LE-NEXT:    blr
 ;
 ; CHECK-PPC64-LABEL: store_i32_by_i8_bswap_store_in_between:
 ; CHECK-PPC64:       # %bb.0: # %entry
-; CHECK-PPC64-NOT:   stw 
+; CHECK-PPC64-NEXT:    li 6, 3
+; CHECK-PPC64-NEXT:    sth 3, 2(4)
+; CHECK-PPC64-NEXT:    stb 6, 0(5)
+; CHECK-PPC64-NEXT:    srwi 5, 3, 16
+; CHECK-PPC64-NEXT:    srwi 3, 3, 24
+; CHECK-PPC64-NEXT:    stb 5, 1(4)
+; CHECK-PPC64-NEXT:    stb 3, 0(4)
+; CHECK-PPC64-NEXT:    blr
 entry:
   %conv = trunc i32 %m to i8
   %arrayidx = getelementptr inbounds i8, i8* %p, i64 3
@@ -308,11 +332,25 @@ entry:
 define void @store_i32_by_i8_bswap_unrelated_store(i32 signext %m, i8* %p, i8* %q) {
 ; CHECK-PPC64LE-LABEL: store_i32_by_i8_bswap_unrelated_store:
 ; CHECK-PPC64LE:       # %bb.0: # %entry
-; CHECK-PPC64LE-NOT:   stwbrx 
+; CHECK-PPC64LE-NEXT:    srwi 6, 3, 8
+; CHECK-PPC64LE-NEXT:    stb 3, 3(4)
+; CHECK-PPC64LE-NEXT:    stb 6, 2(5)
+; CHECK-PPC64LE-NEXT:    srwi 5, 3, 16
+; CHECK-PPC64LE-NEXT:    srwi 3, 3, 24
+; CHECK-PPC64LE-NEXT:    stb 5, 1(4)
+; CHECK-PPC64LE-NEXT:    stb 3, 0(4)
+; CHECK-PPC64LE-NEXT:    blr
 ;
 ; CHECK-PPC64-LABEL: store_i32_by_i8_bswap_unrelated_store:
 ; CHECK-PPC64:       # %bb.0: # %entry
-; CHECK-PPC64-NOT:   stw 
+; CHECK-PPC64-NEXT:    srwi 6, 3, 8
+; CHECK-PPC64-NEXT:    stb 3, 3(4)
+; CHECK-PPC64-NEXT:    stb 6, 2(5)
+; CHECK-PPC64-NEXT:    srwi 5, 3, 16
+; CHECK-PPC64-NEXT:    srwi 3, 3, 24
+; CHECK-PPC64-NEXT:    stb 5, 1(4)
+; CHECK-PPC64-NEXT:    stb 3, 0(4)
+; CHECK-PPC64-NEXT:    blr
 entry:
   %conv = trunc i32 %m to i8
   %arrayidx = getelementptr inbounds i8, i8* %p, i64 3
@@ -339,8 +377,8 @@ entry:
 define void @store_i32_by_i8_bswap_nonzero_offset(i32 signext %m, i8* %p) {
 ; CHECK-PPC64LE-LABEL: store_i32_by_i8_bswap_nonzero_offset:
 ; CHECK-PPC64LE:       # %bb.0: # %entry
-; CHECK-PPC64LE-NEXT:    addi [[REG1:[0-9]+]], 4, 1
-; CHECK-PPC64LE-NEXT:    stwbrx 3, 0, [[REG1]] 
+; CHECK-PPC64LE-NEXT:    addi 4, 4, 1
+; CHECK-PPC64LE-NEXT:    stwbrx 3, 0, 4
 ; CHECK-PPC64LE-NEXT:    blr
 ;
 ; CHECK-PPC64-LABEL: store_i32_by_i8_bswap_nonzero_offset:
@@ -379,8 +417,8 @@ define void @store_i32_by_i8_neg_offset(i32 signext %m, i8* %p) {
 ;
 ; CHECK-PPC64-LABEL: store_i32_by_i8_neg_offset:
 ; CHECK-PPC64:       # %bb.0: # %entry
-; CHECK-PPC64-NEXT:    addi [[REG1:[0-9]+]], 4, -4
-; CHECK-PPC64-NEXT:    stwbrx 3, 0, [[REG1]] 
+; CHECK-PPC64-NEXT:    addi 4, 4, -4
+; CHECK-PPC64-NEXT:    stwbrx 3, 0, 4
 ; CHECK-PPC64-NEXT:    blr
 entry:
   %0 = lshr i32 %m, 8
@@ -409,8 +447,8 @@ entry:
 define void @store_i32_by_i8_bswap_neg_offset(i32 signext %m, i8* %p) {
 ; CHECK-PPC64LE-LABEL: store_i32_by_i8_bswap_neg_offset:
 ; CHECK-PPC64LE:       # %bb.0: # %entry
-; CHECK-PPC64LE-NEXT:    addi [[REG1:[0-9]+]], 4, -4
-; CHECK-PPC64LE-NEXT:    stwbrx 3, 0, [[REG1]] 
+; CHECK-PPC64LE-NEXT:    addi 4, 4, -4
+; CHECK-PPC64LE-NEXT:    stwbrx 3, 0, 4
 ; CHECK-PPC64LE-NEXT:    blr
 ;
 ; CHECK-PPC64-LABEL: store_i32_by_i8_bswap_neg_offset:
@@ -444,17 +482,17 @@ entry:
 define void @store_i32_by_i8_bswap_base_index_offset(i32 %m, i32 %i, i8* %p) {
 ; CHECK-PPC64LE-LABEL: store_i32_by_i8_bswap_base_index_offset:
 ; CHECK-PPC64LE:       # %bb.0: # %entry
-; CHECK-PPC64LE-NEXT:    extsw [[REG1:[0-9]+]], 4
-; CHECK-PPC64LE-NEXT:    add [[REG2:[0-9]+]], 5, [[REG1]] 
-; CHECK-PPC64LE-NEXT:    addi [[REG3:[0-9]+]], [[REG2]], -4
-; CHECK-PPC64LE-NEXT:    stwbrx 3, 0, [[REG3]] 
+; CHECK-PPC64LE-NEXT:    extsw 4, 4
+; CHECK-PPC64LE-NEXT:    add 4, 5, 4
+; CHECK-PPC64LE-NEXT:    addi 4, 4, -4
+; CHECK-PPC64LE-NEXT:    stwbrx 3, 0, 4
 ; CHECK-PPC64LE-NEXT:    blr
 ;
 ; CHECK-PPC64-LABEL: store_i32_by_i8_bswap_base_index_offset:
 ; CHECK-PPC64:       # %bb.0: # %entry
-; CHECK-PPC64-NEXT:    extsw [[REG1:[0-9]+]], 4
-; CHECK-PPC64-NEXT:    add [[REG2:[0-9]+]], 5, [[REG1]] 
-; CHECK-PPC64-NEXT:    stw 3, -4([[REG2]])
+; CHECK-PPC64-NEXT:    extsw 4, 4
+; CHECK-PPC64-NEXT:    add 4, 5, 4
+; CHECK-PPC64-NEXT:    stw 3, -4(4)
 ; CHECK-PPC64-NEXT:    blr
 entry:
   %0 = lshr i32 %m, 16
@@ -496,17 +534,17 @@ entry:
 define void @store_i32_by_i8_bswap_complicated(i32 %m, i32 %i, i8* %p) {
 ; CHECK-PPC64LE-LABEL: store_i32_by_i8_bswap_complicated:
 ; CHECK-PPC64LE:       # %bb.0: # %entry
-; CHECK-PPC64LE-NEXT:    extsw [[REG1:[0-9]+]], 4
-; CHECK-PPC64LE-NEXT:    add [[REG2:[0-9]+]], 5, [[REG1]] 
-; CHECK-PPC64LE-NEXT:    addi [[REG3:[0-9]+]], [[REG2]], 3 
-; CHECK-PPC64LE-NEXT:    stwbrx 3, 0, [[REG3]] 
+; CHECK-PPC64LE-NEXT:    extsw 4, 4
+; CHECK-PPC64LE-NEXT:    add 4, 5, 4
+; CHECK-PPC64LE-NEXT:    addi 4, 4, 3
+; CHECK-PPC64LE-NEXT:    stwbrx 3, 0, 4
 ; CHECK-PPC64LE-NEXT:    blr
 ;
 ; CHECK-PPC64-LABEL: store_i32_by_i8_bswap_complicated:
 ; CHECK-PPC64:       # %bb.0: # %entry
-; CHECK-PPC64-NEXT:    extsw [[REG1:[0-9]+]], 4
-; CHECK-PPC64-NEXT:    add [[REG2:[0-9]+]], 5, [[REG1]] 
-; CHECK-PPC64-NEXT:    stw 3, 3([[REG2]])
+; CHECK-PPC64-NEXT:    extsw 4, 4
+; CHECK-PPC64-NEXT:    add 4, 5, 4
+; CHECK-PPC64-NEXT:    stw 3, 3(4)
 ; CHECK-PPC64-NEXT:    blr
 entry:
   %idx.ext = sext i32 %i to i64
@@ -579,15 +617,17 @@ entry:
 ; p[0] = v;
 ; p[1] = v;
 define void @store_same_value_to_consecutive_mem(i8* %p, i8 zeroext %v) {
-; CHECK-PPC64LE-LABEL: store_same_value_to_consecutive_mem 
-; CHECK-PPC64LE:       # %bb.0: # %entry 
-; CHECK-PPC64LE-NEXT:    stb 4, 0(3) 
-; CHECK-PPC64LE-NEXT:    stb 4, 1(3) 
+; CHECK-PPC64LE-LABEL: store_same_value_to_consecutive_mem:
+; CHECK-PPC64LE:       # %bb.0: # %entry
+; CHECK-PPC64LE-NEXT:    stb 4, 0(3)
+; CHECK-PPC64LE-NEXT:    stb 4, 1(3)
+; CHECK-PPC64LE-NEXT:    blr
 ;
-; CHECK-PPC64-LABEL: store_same_value_to_consecutive_mem 
+; CHECK-PPC64-LABEL: store_same_value_to_consecutive_mem:
 ; CHECK-PPC64:       # %bb.0: # %entry
 ; CHECK-PPC64-NEXT:    stb 4, 0(3)
 ; CHECK-PPC64-NEXT:    stb 4, 1(3)
+; CHECK-PPC64-NEXT:    blr
 entry:
   store i8 %v, i8* %p, align 1
   %arrayidx1 = getelementptr inbounds i8, i8* %p, i64 1
diff --git a/llvm/test/CodeGen/PowerPC/store-constant.ll b/llvm/test/CodeGen/PowerPC/store-constant.ll
index bc847028e..7e562eb38 100644
--- a/llvm/test/CodeGen/PowerPC/store-constant.ll
+++ b/llvm/test/CodeGen/PowerPC/store-constant.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr8 -verify-machineinstrs | FileCheck %s
 
 @CVal = external local_unnamed_addr global i8, align 1
@@ -13,6 +14,14 @@
 %struct.S = type { i64, i8, i16, i32 }
 
 define void @foo(%struct.S* %p) {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li 4, 0
+; CHECK-NEXT:    stb 4, 8(3)
+; CHECK-NEXT:    std 4, 0(3)
+; CHECK-NEXT:    sth 4, 10(3)
+; CHECK-NEXT:    stw 4, 12(3)
+; CHECK-NEXT:    blr
   %l4 = bitcast %struct.S* %p to i64*
   store i64 0, i64* %l4, align 8
   %c = getelementptr %struct.S, %struct.S* %p, i64 0, i32 1
@@ -23,15 +32,17 @@ define void @foo(%struct.S* %p) {
   store i32 0, i32* %i, align 4
   ret void
 
-; CHECK-LABEL: @foo
-; CHECK:       li 4, 0
-; CHECK:       stb 4, 8(3)
-; CHECK:       std 4, 0(3)
-; CHECK:       sth 4, 10(3)
-; CHECK:       stw 4, 12(3)
 }
 
 define void @bar(%struct.S* %p) {
+; CHECK-LABEL: bar:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li 4, 2
+; CHECK-NEXT:    stw 4, 12(3)
+; CHECK-NEXT:    sth 4, 10(3)
+; CHECK-NEXT:    std 4, 0(3)
+; CHECK-NEXT:    stb 4, 8(3)
+; CHECK-NEXT:    blr
   %i = getelementptr %struct.S, %struct.S* %p, i64 0, i32 3
   store i32 2, i32* %i, align 4
   %s = getelementptr %struct.S, %struct.S* %p, i64 0, i32 2
@@ -42,103 +53,171 @@ define void @bar(%struct.S* %p) {
   store i64 2, i64* %l4, align 8
   ret void
 
-; CHECK-LABEL: @bar
-; CHECK:       li 4, 2
-; CHECK-DAG:       stw 4, 12(3)
-; CHECK-DAG:       sth 4, 10(3)
-; CHECK-DAG:       std 4, 0(3)
-; CHECK-DAG:       stb 4, 8(3)
 }
 
 ; Function Attrs: norecurse nounwind
 define void @setSmallNeg() {
+; CHECK-LABEL: setSmallNeg:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addis 3, 2, .LC0@toc@ha
+; CHECK-NEXT:    addis 4, 2, .LC1@toc@ha
+; CHECK-NEXT:    addis 5, 2, .LC2@toc@ha
+; CHECK-NEXT:    addis 6, 2, .LC3@toc@ha
+; CHECK-NEXT:    li 7, -7
+; CHECK-NEXT:    ld 3, .LC0@toc@l(3)
+; CHECK-NEXT:    ld 4, .LC1@toc@l(4)
+; CHECK-NEXT:    ld 5, .LC2@toc@l(5)
+; CHECK-NEXT:    ld 6, .LC3@toc@l(6)
+; CHECK-NEXT:    stb 7, 0(3)
+; CHECK-NEXT:    sth 7, 0(4)
+; CHECK-NEXT:    std 7, 0(6)
+; CHECK-NEXT:    stw 7, 0(5)
+; CHECK-NEXT:    blr
 entry:
   store i8 -7, i8* @CVal, align 1
   store i16 -7, i16* @SVal, align 2
   store i32 -7, i32* @IVal, align 4
   store i64 -7, i64* @LVal, align 8
   ret void
-; CHECK-LABEL: setSmallNeg
-; CHECK: li 7, -7
-; CHECK-DAG: stb 7,
-; CHECK-DAG: sth 7,
-; CHECK-DAG: stw 7,
-; CHECK-DAG: std 7,
 }
 
 ; Function Attrs: norecurse nounwind
 define void @setSmallPos() {
+; CHECK-LABEL: setSmallPos:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addis 3, 2, .LC0@toc@ha
+; CHECK-NEXT:    addis 4, 2, .LC1@toc@ha
+; CHECK-NEXT:    addis 5, 2, .LC2@toc@ha
+; CHECK-NEXT:    addis 6, 2, .LC3@toc@ha
+; CHECK-NEXT:    li 7, 8
+; CHECK-NEXT:    ld 3, .LC0@toc@l(3)
+; CHECK-NEXT:    ld 4, .LC1@toc@l(4)
+; CHECK-NEXT:    ld 5, .LC2@toc@l(5)
+; CHECK-NEXT:    ld 6, .LC3@toc@l(6)
+; CHECK-NEXT:    stb 7, 0(3)
+; CHECK-NEXT:    sth 7, 0(4)
+; CHECK-NEXT:    std 7, 0(6)
+; CHECK-NEXT:    stw 7, 0(5)
+; CHECK-NEXT:    blr
 entry:
   store i8 8, i8* @CVal, align 1
   store i16 8, i16* @SVal, align 2
   store i32 8, i32* @IVal, align 4
   store i64 8, i64* @LVal, align 8
   ret void
-; CHECK-LABEL: setSmallPos
-; CHECK: li 7, 8
-; CHECK-DAG: stb 7,
-; CHECK-DAG: sth 7,
-; CHECK-DAG: stw 7,
-; CHECK-DAG: std 7,
 }
 
 ; Function Attrs: norecurse nounwind
 define void @setMaxNeg() {
+; CHECK-LABEL: setMaxNeg:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addis 3, 2, .LC1@toc@ha
+; CHECK-NEXT:    addis 4, 2, .LC2@toc@ha
+; CHECK-NEXT:    addis 5, 2, .LC3@toc@ha
+; CHECK-NEXT:    li 6, -32768
+; CHECK-NEXT:    ld 3, .LC1@toc@l(3)
+; CHECK-NEXT:    ld 4, .LC2@toc@l(4)
+; CHECK-NEXT:    ld 5, .LC3@toc@l(5)
+; CHECK-NEXT:    sth 6, 0(3)
+; CHECK-NEXT:    stw 6, 0(4)
+; CHECK-NEXT:    std 6, 0(5)
+; CHECK-NEXT:    blr
 entry:
   store i16 -32768, i16* @SVal, align 2
   store i32 -32768, i32* @IVal, align 4
   store i64 -32768, i64* @LVal, align 8
   ret void
-; CHECK-LABEL: setMaxNeg
-; CHECK: li 6, -32768
-; CHECK-DAG: sth 6,
-; CHECK-DAG: stw 6,
-; CHECK-DAG: std 6,
 }
 
 ; Function Attrs: norecurse nounwind
 define void @setMaxPos() {
+; CHECK-LABEL: setMaxPos:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addis 3, 2, .LC1@toc@ha
+; CHECK-NEXT:    addis 4, 2, .LC2@toc@ha
+; CHECK-NEXT:    addis 5, 2, .LC3@toc@ha
+; CHECK-NEXT:    li 6, 32767
+; CHECK-NEXT:    ld 3, .LC1@toc@l(3)
+; CHECK-NEXT:    ld 4, .LC2@toc@l(4)
+; CHECK-NEXT:    ld 5, .LC3@toc@l(5)
+; CHECK-NEXT:    sth 6, 0(3)
+; CHECK-NEXT:    stw 6, 0(4)
+; CHECK-NEXT:    std 6, 0(5)
+; CHECK-NEXT:    blr
 entry:
   store i16 32767, i16* @SVal, align 2
   store i32 32767, i32* @IVal, align 4
   store i64 32767, i64* @LVal, align 8
   ret void
-; CHECK-LABEL: setMaxPos
-; CHECK: li 6, 32767
-; CHECK-DAG: sth 6,
-; CHECK-DAG: stw 6,
-; CHECK-DAG: std 6,
 }
 
 ; Function Attrs: norecurse nounwind
 define void @setExcessiveNeg() {
+; CHECK-LABEL: setExcessiveNeg:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addis 3, 2, .LC2@toc@ha
+; CHECK-NEXT:    addis 4, 2, .LC3@toc@ha
+; CHECK-NEXT:    lis 5, -1
+; CHECK-NEXT:    ld 3, .LC2@toc@l(3)
+; CHECK-NEXT:    ld 4, .LC3@toc@l(4)
+; CHECK-NEXT:    ori 5, 5, 32767
+; CHECK-NEXT:    stw 5, 0(3)
+; CHECK-NEXT:    std 5, 0(4)
+; CHECK-NEXT:    blr
 entry:
   store i32 -32769, i32* @IVal, align 4
   store i64 -32769, i64* @LVal, align 8
   ret void
-; CHECK-LABEL: setExcessiveNeg
-; CHECK: lis 5, -1
-; CHECK: ori 5, 5, 32767
-; CHECK-DAG: stw 5,
-; CHECK-DAG: std 5,
 }
 
 ; Function Attrs: norecurse nounwind
 define void @setExcessivePos() {
+; CHECK-LABEL: setExcessivePos:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addis 3, 2, .LC4@toc@ha
+; CHECK-NEXT:    addis 4, 2, .LC2@toc@ha
+; CHECK-NEXT:    addis 5, 2, .LC3@toc@ha
+; CHECK-NEXT:    li 6, 0
+; CHECK-NEXT:    ld 3, .LC4@toc@l(3)
+; CHECK-NEXT:    ld 4, .LC2@toc@l(4)
+; CHECK-NEXT:    ld 5, .LC3@toc@l(5)
+; CHECK-NEXT:    ori 6, 6, 32768
+; CHECK-NEXT:    sth 6, 0(3)
+; CHECK-NEXT:    stw 6, 0(4)
+; CHECK-NEXT:    std 6, 0(5)
+; CHECK-NEXT:    blr
 entry:
   store i16 -32768, i16* @USVal, align 2
   store i32 32768, i32* @IVal, align 4
   store i64 32768, i64* @LVal, align 8
   ret void
-; CHECK-LABEL: setExcessivePos
-; CHECK: li 6, 0
-; CHECK: ori 6, 6, 32768
-; CHECK-DAG: sth 6,
-; CHECK-DAG: stw 6,
-; CHECK-DAG: std 6,
 }
 
 define void @SetArr(i32 signext %Len) {
+; CHECK-LABEL: SetArr:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cmpwi 3, 1
+; CHECK-NEXT:    bltlr 0
+; CHECK-NEXT:  # %bb.1: # %for.body.lr.ph
+; CHECK-NEXT:    addis 4, 2, .LC5@toc@ha
+; CHECK-NEXT:    addis 5, 2, .LC6@toc@ha
+; CHECK-NEXT:    clrldi 6, 3, 32
+; CHECK-NEXT:    ld 4, .LC5@toc@l(4)
+; CHECK-NEXT:    ld 5, .LC6@toc@l(5)
+; CHECK-NEXT:    ld 4, 0(4)
+; CHECK-NEXT:    ld 5, 0(5)
+; CHECK-NEXT:    mtctr 6
+; CHECK-NEXT:    addi 3, 4, -8
+; CHECK-NEXT:    addi 4, 5, -4
+; CHECK-NEXT:    li 5, -7
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  .LBB8_2: # %for.body
+; CHECK-NEXT:    #
+; CHECK-NEXT:    stdu 5, 8(3)
+; CHECK-NEXT:    stwu 5, 4(4)
+; CHECK-NEXT:    bdnz .LBB8_2
+; CHECK-NEXT:  # %bb.3: # %for.cond.cleanup
+; CHECK-NEXT:    blr
 entry:
   %cmp7 = icmp sgt i32 %Len, 0
   br i1 %cmp7, label %for.body.lr.ph, label %for.cond.cleanup
@@ -161,31 +240,39 @@ for.body:                                         ; preds = %for.body, %for.body
   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
   %exitcond = icmp eq i64 %indvars.iv.next, %wide.trip.count
   br i1 %exitcond, label %for.cond.cleanup, label %for.body
-; CHECK-LABEL: SetArr
-; CHECK: li 5, -7
-; CHECK: stdu 5, 8(3)
-; CHECK: stwu 5, 4(4)
 }
 
 define void @setSameValDiffSizeCI() {
+; CHECK-LABEL: setSameValDiffSizeCI:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addis 3, 2, .LC2@toc@ha
+; CHECK-NEXT:    addis 4, 2, .LC0@toc@ha
+; CHECK-NEXT:    li 5, 255
+; CHECK-NEXT:    ld 3, .LC2@toc@l(3)
+; CHECK-NEXT:    ld 4, .LC0@toc@l(4)
+; CHECK-NEXT:    stw 5, 0(3)
+; CHECK-NEXT:    stb 5, 0(4)
+; CHECK-NEXT:    blr
 entry:
   store i32 255, i32* @IVal, align 4
   store i8 -1, i8* @CVal, align 1
   ret void
-; CHECK-LABEL: setSameValDiffSizeCI
-; CHECK: li 5, 255
-; CHECK-DAG: stb 5,
-; CHECK-DAG: stw 5,
 }
 
 define void @setSameValDiffSizeSI() {
+; CHECK-LABEL: setSameValDiffSizeSI:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addis 3, 2, .LC2@toc@ha
+; CHECK-NEXT:    addis 4, 2, .LC1@toc@ha
+; CHECK-NEXT:    li 5, 0
+; CHECK-NEXT:    ld 3, .LC2@toc@l(3)
+; CHECK-NEXT:    ld 4, .LC1@toc@l(4)
+; CHECK-NEXT:    ori 5, 5, 65535
+; CHECK-NEXT:    stw 5, 0(3)
+; CHECK-NEXT:    sth 5, 0(4)
+; CHECK-NEXT:    blr
 entry:
   store i32 65535, i32* @IVal, align 4
   store i16 -1, i16* @SVal, align 2
   ret void
-; CHECK-LABEL: setSameValDiffSizeSI
-; CHECK: li 5, 0
-; CHECK: ori 5, 5, 65535
-; CHECK-DAG: sth 5,
-; CHECK-DAG: stw 5,
 }
diff --git a/llvm/test/CodeGen/PowerPC/store_fptoi.ll b/llvm/test/CodeGen/PowerPC/store_fptoi.ll
index 097806137..e4f47ab76 100644
--- a/llvm/test/CodeGen/PowerPC/store_fptoi.ll
+++ b/llvm/test/CodeGen/PowerPC/store_fptoi.ll
@@ -66,7 +66,7 @@ entry:
 ; CHECK-PWR8-LABEL: dpConv2shw
 ; CHECK-PWR8: lfdx [[LD:[0-9]+]], 0, 3
 ; CHECK-PWR8-NEXT: xscvdpsxws [[CONV:[0-9]+]], [[LD]]
-; CHECK-PWR8-NEXT: mfvsrwz [[REG:[0-9]+]], [[CONV]]
+; CHECK-PWR8-NEXT: mffprwz [[REG:[0-9]+]], [[CONV]]
 ; CHECK-PWR8-NEXT: sth [[REG]], 0(4)
 ; CHECK-PWR8-NEXT: blr
 }
@@ -88,7 +88,7 @@ entry:
 ; CHECK-PWR8-LABEL: dpConv2sb
 ; CHECK-PWR8: lfdx [[LD:[0-9]+]], 0, 3
 ; CHECK-PWR8-NEXT: xscvdpsxws [[CONV:[0-9]+]], [[LD]]
-; CHECK-PWR8-NEXT: mfvsrwz [[REG:[0-9]+]], [[CONV]]
+; CHECK-PWR8-NEXT: mffprwz [[REG:[0-9]+]], [[CONV]]
 ; CHECK-PWR8-NEXT: stb [[REG]], 0(4)
 ; CHECK-PWR8-NEXT: blr
 }
@@ -152,7 +152,7 @@ entry:
 ; CHECK-PWR8-LABEL: spConv2shw
 ; CHECK-PWR8: lfsx [[LD:[0-9]+]], 0, 3
 ; CHECK-PWR8-NEXT: xscvdpsxws [[CONV:[0-9]+]], [[LD]]
-; CHECK-PWR8-NEXT: mfvsrwz [[REG:[0-9]+]], [[CONV]]
+; CHECK-PWR8-NEXT: mffprwz [[REG:[0-9]+]], [[CONV]]
 ; CHECK-PWR8-NEXT: sth [[REG]], 0(4)
 ; CHECK-PWR8-NEXT: blr
 }
@@ -174,7 +174,7 @@ entry:
 ; CHECK-PWR8-LABEL: spConv2sb
 ; CHECK-PWR8: lfsx [[LD:[0-9]+]], 0, 3
 ; CHECK-PWR8-NEXT: xscvdpsxws [[CONV:[0-9]+]], [[LD]]
-; CHECK-PWR8-NEXT: mfvsrwz [[REG:[0-9]+]], [[CONV]]
+; CHECK-PWR8-NEXT: mffprwz [[REG:[0-9]+]], [[CONV]]
 ; CHECK-PWR8-NEXT: stb [[REG]], 0(4)
 ; CHECK-PWR8-NEXT: blr
 }
@@ -253,7 +253,7 @@ entry:
 ; CHECK-PWR8: lfdx [[LD:[0-9]+]], 0, 3
 ; CHECK-PWR8-NEXT: sldi [[REG:[0-9]+]], 5, 1
 ; CHECK-PWR8-NEXT: xscvdpsxws [[CONV:[0-9]+]], [[LD]]
-; CHECK-PWR8-NEXT: mfvsrwz [[REG:[0-9]+]], [[CONV]]
+; CHECK-PWR8-NEXT: mffprwz [[REG:[0-9]+]], [[CONV]]
 ; CHECK-PWR8-NEXT: sthx [[REG]], 4, 5
 ; CHECK-PWR8-NEXT: blr
 }
@@ -278,7 +278,7 @@ entry:
 ; CHECK-PWR8-LABEL: dpConv2sb_x
 ; CHECK-PWR8: lfdx [[LD:[0-9]+]], 0, 3
 ; CHECK-PWR8-NEXT: xscvdpsxws [[CONV:[0-9]+]], [[LD]]
-; CHECK-PWR8-NEXT: mfvsrwz [[REG:[0-9]+]], [[CONV]]
+; CHECK-PWR8-NEXT: mffprwz [[REG:[0-9]+]], [[CONV]]
 ; CHECK-PWR8-NEXT: stbx [[REG]], 4, 5
 ; CHECK-PWR8-NEXT: blr
 }
@@ -357,7 +357,7 @@ entry:
 ; CHECK-PWR8: lfsx [[LD:[0-9]+]], 0, 3
 ; CHECK-PWR8-NEXT: sldi [[REG:[0-9]+]], 5, 1
 ; CHECK-PWR8-NEXT: xscvdpsxws [[CONV:[0-9]+]], [[LD]]
-; CHECK-PWR8-NEXT: mfvsrwz [[REG2:[0-9]+]], [[CONV]]
+; CHECK-PWR8-NEXT: mffprwz [[REG2:[0-9]+]], [[CONV]]
 ; CHECK-PWR8-NEXT: sthx [[REG2]], 4, [[REG]]
 ; CHECK-PWR8-NEXT: blr
 }
@@ -382,7 +382,7 @@ entry:
 ; CHECK-PWR8-LABEL: spConv2sb_x
 ; CHECK-PWR8: lfsx [[LD:[0-9]+]], 0, 3
 ; CHECK-PWR8-NEXT: xscvdpsxws [[CONV:[0-9]+]], [[LD]]
-; CHECK-PWR8-NEXT: mfvsrwz [[REG:[0-9]+]], [[CONV]]
+; CHECK-PWR8-NEXT: mffprwz [[REG:[0-9]+]], [[CONV]]
 ; CHECK-PWR8-NEXT: stbx [[REG]], 4, 5
 ; CHECK-PWR8-NEXT: blr
 }
@@ -450,7 +450,7 @@ entry:
 ; CHECK-PWR8-LABEL: dpConv2uhw
 ; CHECK-PWR8: lfdx [[LD:[0-9]+]], 0, 3
 ; CHECK-PWR8-NEXT: xscvdpsxws [[CONV:[0-9]+]], [[LD]]
-; CHECK-PWR8-NEXT: mfvsrwz [[REG:[0-9]+]], [[CONV]]
+; CHECK-PWR8-NEXT: mffprwz [[REG:[0-9]+]], [[CONV]]
 ; CHECK-PWR8-NEXT: sth [[REG]], 0(4)
 ; CHECK-PWR8-NEXT: blr
 }
@@ -472,7 +472,7 @@ entry:
 ; CHECK-PWR8-LABEL: dpConv2ub
 ; CHECK-PWR8: lfdx [[LD:[0-9]+]], 0, 3
 ; CHECK-PWR8-NEXT: xscvdpsxws [[CONV:[0-9]+]], [[LD]]
-; CHECK-PWR8-NEXT: mfvsrwz [[REG:[0-9]+]], [[CONV]]
+; CHECK-PWR8-NEXT: mffprwz [[REG:[0-9]+]], [[CONV]]
 ; CHECK-PWR8-NEXT: stb [[REG]], 0(4)
 ; CHECK-PWR8-NEXT: blr
 }
@@ -536,7 +536,7 @@ entry:
 ; CHECK-PWR8-LABEL: spConv2uhw
 ; CHECK-PWR8: lfsx [[LD:[0-9]+]], 0, 3
 ; CHECK-PWR8-NEXT: xscvdpsxws [[CONV:[0-9]+]], [[LD]]
-; CHECK-PWR8-NEXT: mfvsrwz [[REG:[0-9]+]], [[CONV]]
+; CHECK-PWR8-NEXT: mffprwz [[REG:[0-9]+]], [[CONV]]
 ; CHECK-PWR8-NEXT: sth [[REG]], 0(4)
 ; CHECK-PWR8-NEXT: blr
 }
@@ -558,7 +558,7 @@ entry:
 ; CHECK-PWR8-LABEL: spConv2ub
 ; CHECK-PWR8: lfsx [[LD:[0-9]+]], 0, 3
 ; CHECK-PWR8-NEXT: xscvdpsxws [[CONV:[0-9]+]], [[LD]]
-; CHECK-PWR8-NEXT: mfvsrwz [[REG:[0-9]+]], [[CONV]]
+; CHECK-PWR8-NEXT: mffprwz [[REG:[0-9]+]], [[CONV]]
 ; CHECK-PWR8-NEXT: stb [[REG]], 0(4)
 ; CHECK-PWR8-NEXT: blr
 }
@@ -637,7 +637,7 @@ entry:
 ; CHECK-PWR8: lfdx [[LD:[0-9]+]], 0, 3
 ; CHECK-PWR8-NEXT: sldi [[REG:[0-9]+]], 5, 1
 ; CHECK-PWR8-NEXT: xscvdpsxws [[CONV:[0-9]+]], [[LD]]
-; CHECK-PWR8-NEXT: mfvsrwz [[REG:[0-9]+]], [[CONV]]
+; CHECK-PWR8-NEXT: mffprwz [[REG:[0-9]+]], [[CONV]]
 ; CHECK-PWR8-NEXT: sthx [[REG]], 4, 5
 ; CHECK-PWR8-NEXT: blr
 }
@@ -662,7 +662,7 @@ entry:
 ; CHECK-PWR8-LABEL: dpConv2ub_x
 ; CHECK-PWR8: lfdx [[LD:[0-9]+]], 0, 3
 ; CHECK-PWR8-NEXT: xscvdpsxws [[CONV:[0-9]+]], [[LD]]
-; CHECK-PWR8-NEXT: mfvsrwz [[REG:[0-9]+]], [[CONV]]
+; CHECK-PWR8-NEXT: mffprwz [[REG:[0-9]+]], [[CONV]]
 ; CHECK-PWR8-NEXT: stbx [[REG]], 4, 5
 ; CHECK-PWR8-NEXT: blr
 }
@@ -741,7 +741,7 @@ entry:
 ; CHECK-PWR8: lfsx [[LD:[0-9]+]], 0, 3
 ; CHECK-PWR8-NEXT: sldi [[REG:[0-9]+]], 5, 1
 ; CHECK-PWR8-NEXT: xscvdpsxws [[CONV:[0-9]+]], [[LD]]
-; CHECK-PWR8-NEXT: mfvsrwz [[REG2:[0-9]+]], [[CONV]]
+; CHECK-PWR8-NEXT: mffprwz [[REG2:[0-9]+]], [[CONV]]
 ; CHECK-PWR8-NEXT: sthx [[REG2]], 4, [[REG]]
 ; CHECK-PWR8-NEXT: blr
 }
@@ -766,7 +766,7 @@ entry:
 ; CHECK-PWR8-LABEL: spConv2ub_x
 ; CHECK-PWR8: lfsx [[LD:[0-9]+]], 0, 3
 ; CHECK-PWR8-NEXT: xscvdpsxws [[CONV:[0-9]+]], [[LD]]
-; CHECK-PWR8-NEXT: mfvsrwz [[REG:[0-9]+]], [[CONV]]
+; CHECK-PWR8-NEXT: mffprwz [[REG:[0-9]+]], [[CONV]]
 ; CHECK-PWR8-NEXT: stbx [[REG]], 4, 5
 ; CHECK-PWR8-NEXT: blr
 }
diff --git a/llvm/test/CodeGen/PowerPC/stubs.ll b/llvm/test/CodeGen/PowerPC/stubs.ll
index 33499f1bc..d8772b9b7 100644
--- a/llvm/test/CodeGen/PowerPC/stubs.ll
+++ b/llvm/test/CodeGen/PowerPC/stubs.ll
@@ -6,4 +6,4 @@ entry:
 }
 
 ; CHECK: test1:
-; CHECK: bl __floatditf@PLT
+; CHECK: bl __floatditf
diff --git a/llvm/test/CodeGen/PowerPC/stwu-sched.ll b/llvm/test/CodeGen/PowerPC/stwu-sched.ll
index f88869b19..0afd2ee40 100644
--- a/llvm/test/CodeGen/PowerPC/stwu-sched.ll
+++ b/llvm/test/CodeGen/PowerPC/stwu-sched.ll
@@ -58,7 +58,7 @@ define void @initCombList(%0* nocapture, i32 signext) local_unnamed_addr #0 {
   ret void
 }
 
-attributes #0 = { norecurse nounwind writeonly "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { norecurse nounwind writeonly "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.ident = !{!0}
 
diff --git a/llvm/test/CodeGen/PowerPC/sub-of-not.ll b/llvm/test/CodeGen/PowerPC/sub-of-not.ll
index 992781887..a155d87f9 100644
--- a/llvm/test/CodeGen/PowerPC/sub-of-not.ll
+++ b/llvm/test/CodeGen/PowerPC/sub-of-not.ll
@@ -170,7 +170,6 @@ define <16 x i8> @vector_i128_i8(<16 x i8> %x, <16 x i8> %y) nounwind {
 ; PPC64BE-NEXT:    std 23, -72(1) # 8-byte Folded Spill
 ; PPC64BE-NEXT:    std 22, -80(1) # 8-byte Folded Spill
 ; PPC64BE-NEXT:    std 26, -48(1) # 8-byte Folded Spill
-; PPC64BE-NEXT:    std 2, -96(1) # 8-byte Folded Spill
 ; PPC64BE-NEXT:    std 25, -56(1) # 8-byte Folded Spill
 ; PPC64BE-NEXT:    std 24, -64(1) # 8-byte Folded Spill
 ; PPC64BE-NEXT:    std 29, -24(1) # 8-byte Folded Spill
@@ -191,8 +190,8 @@ define <16 x i8> @vector_i128_i8(<16 x i8> %x, <16 x i8> %y) nounwind {
 ; PPC64BE-NEXT:    add 7, 12, 7
 ; PPC64BE-NEXT:    lbz 12, 239(1)
 ; PPC64BE-NEXT:    lbz 26, 151(1)
-; PPC64BE-NEXT:    add 2, 21, 23
-; PPC64BE-NEXT:    lbz 23, 279(1)
+; PPC64BE-NEXT:    add 23, 21, 23
+; PPC64BE-NEXT:    lbz 21, 279(1)
 ; PPC64BE-NEXT:    lbz 25, 143(1)
 ; PPC64BE-NEXT:    add 11, 11, 22
 ; PPC64BE-NEXT:    lbz 22, 271(1)
@@ -201,8 +200,8 @@ define <16 x i8> @vector_i128_i8(<16 x i8> %x, <16 x i8> %y) nounwind {
 ; PPC64BE-NEXT:    lbz 12, 263(1)
 ; PPC64BE-NEXT:    lbz 30, 175(1)
 ; PPC64BE-NEXT:    lbz 29, 303(1)
-; PPC64BE-NEXT:    add 26, 23, 26
-; PPC64BE-NEXT:    lbz 23, 311(1)
+; PPC64BE-NEXT:    add 26, 21, 26
+; PPC64BE-NEXT:    lbz 21, 311(1)
 ; PPC64BE-NEXT:    std 28, -32(1) # 8-byte Folded Spill
 ; PPC64BE-NEXT:    add 25, 22, 25
 ; PPC64BE-NEXT:    lbz 28, 167(1)
@@ -212,7 +211,7 @@ define <16 x i8> @vector_i128_i8(<16 x i8> %x, <16 x i8> %y) nounwind {
 ; PPC64BE-NEXT:    lbz 27, 159(1)
 ; PPC64BE-NEXT:    lbz 24, 287(1)
 ; PPC64BE-NEXT:    add 30, 29, 30
-; PPC64BE-NEXT:    add 29, 23, 0
+; PPC64BE-NEXT:    add 29, 21, 0
 ; PPC64BE-NEXT:    addi 0, 29, 1
 ; PPC64BE-NEXT:    add 28, 22, 28
 ; PPC64BE-NEXT:    stb 0, 15(3)
@@ -228,7 +227,7 @@ define <16 x i8> @vector_i128_i8(<16 x i8> %x, <16 x i8> %y) nounwind {
 ; PPC64BE-NEXT:    stb 0, 11(3)
 ; PPC64BE-NEXT:    addi 0, 25, 1
 ; PPC64BE-NEXT:    stb 12, 9(3)
-; PPC64BE-NEXT:    addi 12, 2, 1
+; PPC64BE-NEXT:    addi 12, 23, 1
 ; PPC64BE-NEXT:    addi 11, 11, 1
 ; PPC64BE-NEXT:    addi 10, 10, 1
 ; PPC64BE-NEXT:    addi 9, 9, 1
@@ -247,7 +246,6 @@ define <16 x i8> @vector_i128_i8(<16 x i8> %x, <16 x i8> %y) nounwind {
 ; PPC64BE-NEXT:    stb 6, 2(3)
 ; PPC64BE-NEXT:    stb 5, 1(3)
 ; PPC64BE-NEXT:    stb 4, 0(3)
-; PPC64BE-NEXT:    ld 2, -96(1) # 8-byte Folded Reload
 ; PPC64BE-NEXT:    ld 30, -16(1) # 8-byte Folded Reload
 ; PPC64BE-NEXT:    ld 29, -24(1) # 8-byte Folded Reload
 ; PPC64BE-NEXT:    ld 28, -32(1) # 8-byte Folded Reload
diff --git a/llvm/test/CodeGen/PowerPC/swaps-le-1.ll b/llvm/test/CodeGen/PowerPC/swaps-le-1.ll
index 2855287fb..bea0d9ef3 100644
--- a/llvm/test/CodeGen/PowerPC/swaps-le-1.ll
+++ b/llvm/test/CodeGen/PowerPC/swaps-le-1.ll
@@ -1,17 +1,17 @@
 ; RUN: llc -verify-machineinstrs -O3 -mcpu=pwr8 \
-; RUN:   -mtriple=powerpc64le-unknown-linux-gnu < %s | FileCheck -allow-deprecated-dag-overlap %s
+; RUN:   -mtriple=powerpc64le-unknown-linux-gnu < %s | FileCheck  %s
 
 ; RUN: llc -verify-machineinstrs -O3 -mcpu=pwr8 -disable-ppc-vsx-swap-removal \
-; RUN:   -mtriple=powerpc64le-unknown-linux-gnu < %s | FileCheck -allow-deprecated-dag-overlap \
+; RUN:   -mtriple=powerpc64le-unknown-linux-gnu < %s | FileCheck  \
 ; RUN:   -check-prefix=NOOPTSWAP %s
 
 ; RUN: llc -O3 -mcpu=pwr9 -mtriple=powerpc64le-unknown-linux-gnu \
-; RUN:  -verify-machineinstrs -ppc-vsr-nums-as-vr < %s | FileCheck -allow-deprecated-dag-overlap \
+; RUN:  -verify-machineinstrs -ppc-vsr-nums-as-vr < %s | FileCheck  \
 ; RUN:  -check-prefix=CHECK-P9 --implicit-check-not xxswapd %s
 
 ; RUN: llc -O3 -mcpu=pwr9 -disable-ppc-vsx-swap-removal -mattr=-power9-vector \
 ; RUN:  -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu < %s \
-; RUN:  | FileCheck -allow-deprecated-dag-overlap -check-prefix=NOOPTSWAP %s
+; RUN:  | FileCheck  -check-prefix=NOOPTSWAP %s
 
 ; LH: 2016-11-17
 ;   Updated align attritue from 16 to 8 to keep swap instructions tests.
@@ -164,18 +164,18 @@ for.end:
 ; NOOPTSWAP: stxvd2x
 
 ; CHECK-P9-LABEL: @foo
-; CHECK-P9-DAG: lxvx
-; CHECK-P9-DAG: lxvx
-; CHECK-P9-DAG: lxvx
-; CHECK-P9-DAG: lxvx
-; CHECK-P9-DAG: lxvx
-; CHECK-P9-DAG: lxvx
-; CHECK-P9-DAG: lxvx
-; CHECK-P9-DAG: lxvx
-; CHECK-P9-DAG: lxvx
-; CHECK-P9-DAG: lxvx
-; CHECK-P9-DAG: lxvx
-; CHECK-P9-DAG: lxvx
+; CHECK-P9-DAG: lxv
+; CHECK-P9-DAG: lxv
+; CHECK-P9-DAG: lxv
+; CHECK-P9-DAG: lxv
+; CHECK-P9-DAG: lxv
+; CHECK-P9-DAG: lxv
+; CHECK-P9-DAG: lxv
+; CHECK-P9-DAG: lxv
+; CHECK-P9-DAG: lxv
+; CHECK-P9-DAG: lxv
+; CHECK-P9-DAG: lxv
+; CHECK-P9-DAG: lxv
 ; CHECK-P9-DAG: vadduwm
 ; CHECK-P9-DAG: vadduwm
 ; CHECK-P9-DAG: vadduwm
@@ -184,8 +184,8 @@ for.end:
 ; CHECK-P9-DAG: vmuluwm
 ; CHECK-P9-DAG: vmuluwm
 ; CHECK-P9-DAG: vmuluwm
-; CHECK-P9-DAG: stxvx
-; CHECK-P9-DAG: stxvx
-; CHECK-P9-DAG: stxvx
-; CHECK-P9-DAG: stxvx
+; CHECK-P9-DAG: stxv
+; CHECK-P9-DAG: stxv
+; CHECK-P9-DAG: stxv
+; CHECK-P9-DAG: stxv
 
diff --git a/llvm/test/CodeGen/PowerPC/swaps-le-7.ll b/llvm/test/CodeGen/PowerPC/swaps-le-7.ll
index 3c8813bc9..6f5445197 100644
--- a/llvm/test/CodeGen/PowerPC/swaps-le-7.ll
+++ b/llvm/test/CodeGen/PowerPC/swaps-le-7.ll
@@ -9,8 +9,8 @@
 @G4 = global <2 x double> <double 7.0, double 8.0>
 
 ; CHECK-LABEL: @zg
-; CHECK: xxspltd
-; CHECK-NEXT: xxspltd
+; CHECK: lxvdsx
+; CHECK-NEXT: lxvdsx
 ; CHECK-NEXT: xvmuldp
 ; CHECK-DAG: xvmuldp
 ; CHECK-DAG: xvsubdp
diff --git a/llvm/test/CodeGen/PowerPC/testComparesi32gtu.ll b/llvm/test/CodeGen/PowerPC/testComparesi32gtu.ll
index 62d66c574..855e175c6 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesi32gtu.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesi32gtu.ll
@@ -1,25 +1,61 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -relocation-model=pic -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
-; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
+; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
+; RUN:  --check-prefixes=CHECK,BE
 ; RUN: llc -relocation-model=pic -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
-; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
+; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
+; RUN:  --check-prefixes=CHECK,LE
 
 %struct.tree_common = type { i8, [3 x i8] }
 declare signext i32 @fn2(...) local_unnamed_addr #1
 
 ; Function Attrs: nounwind
-define i32 @testCompare1(%struct.tree_common* nocapture readonly %arg1) {
-; CHECK-LABEL: testCompare1:
-; CHECK:       # %bb.0: # %entry
-; CHECK:         lbz r3, 0(r3)
-; CHECK-DAG:     clrlwi r3, r3, 31
-; CHECK-DAG:     clrldi r3, r3, 32
-; CHECK:         lbz  r4, 0(r4)
-; CHECK-DAG:     clrlwi r4, r4, 31
-; CHECK-DAG:     clrldi r4, r4, 32
-; CHECK:         sub r3, r3, r4
-; CHECK-NEXT:    rldicl r3, r3, 1, 63
+define i32 @testCompare1(%struct.tree_common* nocapture readonly %arg1) nounwind {
+; BE-LABEL: testCompare1:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    mflr r0
+; BE-NEXT:    std r0, 16(r1)
+; BE-NEXT:    stdu r1, -112(r1)
+; BE-NEXT:    addis r4, r2, .LC0@toc@ha
+; BE-NEXT:    lbz r3, 0(r3)
+; BE-NEXT:    ld r4, .LC0@toc@l(r4)
+; BE-NEXT:    clrlwi r3, r3, 31
+; BE-NEXT:    clrldi r3, r3, 32
+; BE-NEXT:    lbz r4, 0(r4)
+; BE-NEXT:    clrlwi r4, r4, 31
+; BE-NEXT:    clrldi r4, r4, 32
+; BE-NEXT:    sub r3, r3, r4
+; BE-NEXT:    rldicl r3, r3, 1, 63
+; BE-NEXT:    bl fn2
+; BE-NEXT:    nop
+; BE-NEXT:    addi r1, r1, 112
+; BE-NEXT:    ld r0, 16(r1)
+; BE-NEXT:    mtlr r0
+; BE-NEXT:    blr
+;
+; LE-LABEL: testCompare1:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    mflr r0
+; LE-NEXT:    std r0, 16(r1)
+; LE-NEXT:    stdu r1, -32(r1)
+; LE-NEXT:    addis r4, r2, .LC0@toc@ha
+; LE-NEXT:    lbz r3, 0(r3)
+; LE-NEXT:    ld r4, .LC0@toc@l(r4)
+; LE-NEXT:    clrlwi r3, r3, 31
+; LE-NEXT:    clrldi r3, r3, 32
+; LE-NEXT:    lbz r4, 0(r4)
+; LE-NEXT:    clrlwi r4, r4, 31
+; LE-NEXT:    clrldi r4, r4, 32
+; LE-NEXT:    sub r3, r3, r4
+; LE-NEXT:    rldicl r3, r3, 1, 63
+; LE-NEXT:    bl fn2
+; LE-NEXT:    nop
+; LE-NEXT:    addi r1, r1, 32
+; LE-NEXT:    ld r0, 16(r1)
+; LE-NEXT:    mtlr r0
+; LE-NEXT:    blr
 entry:
   %bf.load = load i8, i8* bitcast (i32 (%struct.tree_common*)* @testCompare1 to i8*), align 4
   %bf.clear = and i8 %bf.load, 1
@@ -36,11 +72,11 @@ entry:
 define signext i32 @testCompare2(i32 zeroext %a, i32 zeroext %b) {
 ; CHECK-LABEL: testCompare2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-DAG:     rlwinm r3, r3, 0, 31, 31
-; CHECK-DAG:     rlwinm r4, r4, 0, 31, 31
-; CHECK-DAG:     clrldi r3, r3, 32
-; CHECK-DAG:     clrldi r4, r4, 32
-; CHECK:         sub r3, r4, r3
+; CHECK-NEXT:    rlwinm r3, r3, 0, 31, 31
+; CHECK-NEXT:    rlwinm r4, r4, 0, 31, 31
+; CHECK-NEXT:    clrldi r3, r3, 32
+; CHECK-NEXT:    clrldi r4, r4, 32
+; CHECK-NEXT:    sub r3, r4, r3
 ; CHECK-NEXT:    rldicl r3, r3, 1, 63
 ; CHECK-NEXT:    blr
 entry:
diff --git a/llvm/test/CodeGen/PowerPC/testComparesi32ltu.ll b/llvm/test/CodeGen/PowerPC/testComparesi32ltu.ll
index 2b9d0ef1f..0c399ff42 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesi32ltu.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesi32ltu.ll
@@ -1,25 +1,61 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -relocation-model=pic -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
-; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
+; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
+; RUN:  --check-prefixes=CHECK,BE
 ; RUN: llc -relocation-model=pic -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
-; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
+; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
+; RUN:  --check-prefixes=CHECK,LE
 
 %struct.tree_common = type { i8, [3 x i8] }
 declare signext i32 @fn2(...) local_unnamed_addr #1
 
 ; Function Attrs: nounwind
-define i32 @testCompare1(%struct.tree_common* nocapture readonly %arg1) {
-; CHECK-LABEL: testCompare1:
-; CHECK:       # %bb.0: # %entry
-; CHECK:         lbz r3, 0(r3)
-; CHECK-DAG:     clrlwi r3, r3, 31
-; CHECK-DAG:     clrldi r3, r3, 32
-; CHECK:         lbz  r4, 0(r4)
-; CHECK-DAG:     clrlwi r4, r4, 31
-; CHECK-DAG:     clrldi r4, r4, 32
-; CHECK:         sub r3, r4, r3
-; CHECK-NEXT:    rldicl r3, r3, 1, 63
+define i32 @testCompare1(%struct.tree_common* nocapture readonly %arg1) nounwind {
+; BE-LABEL: testCompare1:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    mflr r0
+; BE-NEXT:    std r0, 16(r1)
+; BE-NEXT:    stdu r1, -112(r1)
+; BE-NEXT:    addis r4, r2, .LC0@toc@ha
+; BE-NEXT:    lbz r3, 0(r3)
+; BE-NEXT:    ld r4, .LC0@toc@l(r4)
+; BE-NEXT:    clrlwi r3, r3, 31
+; BE-NEXT:    clrldi r3, r3, 32
+; BE-NEXT:    lbz r4, 0(r4)
+; BE-NEXT:    clrlwi r4, r4, 31
+; BE-NEXT:    clrldi r4, r4, 32
+; BE-NEXT:    sub r3, r4, r3
+; BE-NEXT:    rldicl r3, r3, 1, 63
+; BE-NEXT:    bl fn2
+; BE-NEXT:    nop
+; BE-NEXT:    addi r1, r1, 112
+; BE-NEXT:    ld r0, 16(r1)
+; BE-NEXT:    mtlr r0
+; BE-NEXT:    blr
+;
+; LE-LABEL: testCompare1:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    mflr r0
+; LE-NEXT:    std r0, 16(r1)
+; LE-NEXT:    stdu r1, -32(r1)
+; LE-NEXT:    addis r4, r2, .LC0@toc@ha
+; LE-NEXT:    lbz r3, 0(r3)
+; LE-NEXT:    ld r4, .LC0@toc@l(r4)
+; LE-NEXT:    clrlwi r3, r3, 31
+; LE-NEXT:    clrldi r3, r3, 32
+; LE-NEXT:    lbz r4, 0(r4)
+; LE-NEXT:    clrlwi r4, r4, 31
+; LE-NEXT:    clrldi r4, r4, 32
+; LE-NEXT:    sub r3, r4, r3
+; LE-NEXT:    rldicl r3, r3, 1, 63
+; LE-NEXT:    bl fn2
+; LE-NEXT:    nop
+; LE-NEXT:    addi r1, r1, 32
+; LE-NEXT:    ld r0, 16(r1)
+; LE-NEXT:    mtlr r0
+; LE-NEXT:    blr
 entry:
   %bf.load = load i8, i8* bitcast (i32 (%struct.tree_common*)* @testCompare1 to i8*), align 4
   %bf.clear = and i8 %bf.load, 1
@@ -36,11 +72,11 @@ entry:
 define signext i32 @testCompare2(i32 zeroext %a, i32 zeroext %b) {
 ; CHECK-LABEL: testCompare2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-DAG:     rlwinm r3, r3, 0, 31, 31
-; CHECK-DAG:     rlwinm r4, r4, 0, 31, 31
-; CHECK-DAG:     clrldi r3, r3, 32
-; CHECK-DAG:     clrldi r4, r4, 32
-; CHECK:         sub r3, r3, r4
+; CHECK-NEXT:    rlwinm r3, r3, 0, 31, 31
+; CHECK-NEXT:    rlwinm r4, r4, 0, 31, 31
+; CHECK-NEXT:    clrldi r3, r3, 32
+; CHECK-NEXT:    clrldi r4, r4, 32
+; CHECK-NEXT:    sub r3, r3, r4
 ; CHECK-NEXT:    rldicl r3, r3, 1, 63
 ; CHECK-NEXT:    blr
 entry:
diff --git a/llvm/test/CodeGen/PowerPC/testComparesieqsc.ll b/llvm/test/CodeGen/PowerPC/testComparesieqsc.ll
index 893b374d8..8b5cade8f 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesieqsc.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesieqsc.ll
@@ -7,7 +7,7 @@
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 ; ModuleID = 'ComparisonTestCases/testComparesieqsc.c'
 
-@glob = common local_unnamed_addr global i8 0, align 1
+@glob = local_unnamed_addr global i8 0, align 1
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_ieqsc(i8 signext %a, i8 signext %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testComparesieqsi.ll b/llvm/test/CodeGen/PowerPC/testComparesieqsi.ll
index d75485fcb..d40574172 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesieqsi.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesieqsi.ll
@@ -7,7 +7,7 @@
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 ; ModuleID = 'ComparisonTestCases/testComparesieqsi.c'
 
-@glob = common local_unnamed_addr global i32 0, align 4
+@glob = local_unnamed_addr global i32 0, align 4
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_ieqsi(i32 signext %a, i32 signext %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testComparesieqsll.ll b/llvm/test/CodeGen/PowerPC/testComparesieqsll.ll
index f5627ff76..9d6e0b678 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesieqsll.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesieqsll.ll
@@ -7,7 +7,7 @@
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 ; ModuleID = 'ComparisonTestCases/testComparesieqsll.c'
 
-@glob = common local_unnamed_addr global i64 0, align 8
+@glob = local_unnamed_addr global i64 0, align 8
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_ieqsll(i64 %a, i64 %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testComparesieqss.ll b/llvm/test/CodeGen/PowerPC/testComparesieqss.ll
index 2a5ee0c0f..9acfcc440 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesieqss.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesieqss.ll
@@ -7,7 +7,7 @@
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 ; ModuleID = 'ComparisonTestCases/testComparesieqss.c'
 
-@glob = common local_unnamed_addr global i16 0, align 2
+@glob = local_unnamed_addr global i16 0, align 2
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_ieqss(i16 signext %a, i16 signext %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testComparesiequc.ll b/llvm/test/CodeGen/PowerPC/testComparesiequc.ll
index 18243d366..7f938dff8 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesiequc.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesiequc.ll
@@ -7,7 +7,7 @@
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 ; ModuleID = 'ComparisonTestCases/testComparesiequc.c'
 
-@glob = common local_unnamed_addr global i8 0, align 1
+@glob = local_unnamed_addr global i8 0, align 1
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_iequc(i8 zeroext %a, i8 zeroext %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testComparesiequi.ll b/llvm/test/CodeGen/PowerPC/testComparesiequi.ll
index afa447525..b40e3af36 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesiequi.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesiequi.ll
@@ -7,7 +7,7 @@
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 ; ModuleID = 'ComparisonTestCases/testComparesiequi.c'
 
-@glob = common local_unnamed_addr global i32 0, align 4
+@glob = local_unnamed_addr global i32 0, align 4
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_iequi(i32 zeroext %a, i32 zeroext %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testComparesiequll.ll b/llvm/test/CodeGen/PowerPC/testComparesiequll.ll
index 163997153..6ef9db9bd 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesiequll.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesiequll.ll
@@ -7,7 +7,7 @@
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 ; ModuleID = 'ComparisonTestCases/testComparesiequll.c'
 
-@glob = common local_unnamed_addr global i64 0, align 8
+@glob = local_unnamed_addr global i64 0, align 8
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_iequll(i64 %a, i64 %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testComparesiequs.ll b/llvm/test/CodeGen/PowerPC/testComparesiequs.ll
index 99b388313..90bcee8d7 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesiequs.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesiequs.ll
@@ -7,7 +7,7 @@
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 ; ModuleID = 'ComparisonTestCases/testComparesiequs.c'
 
-@glob = common local_unnamed_addr global i16 0, align 2
+@glob = local_unnamed_addr global i16 0, align 2
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_iequs(i16 zeroext %a, i16 zeroext %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testComparesigesc.ll b/llvm/test/CodeGen/PowerPC/testComparesigesc.ll
index eb457951f..d9a397cf0 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesigesc.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesigesc.ll
@@ -5,7 +5,7 @@
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s --check-prefix=CHECK-LE \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
-@glob = common local_unnamed_addr global i8 0, align 1
+@glob = local_unnamed_addr global i8 0, align 1
 
 define signext i32 @test_igesc(i8 signext %a, i8 signext %b) {
 ; CHECK-LABEL: test_igesc:
diff --git a/llvm/test/CodeGen/PowerPC/testComparesigesi.ll b/llvm/test/CodeGen/PowerPC/testComparesigesi.ll
index 93e6c1e1f..bff6cb243 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesigesi.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesigesi.ll
@@ -5,7 +5,7 @@
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s --check-prefix=CHECK-LE \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
-@glob = common local_unnamed_addr global i32 0, align 4
+@glob = local_unnamed_addr global i32 0, align 4
 
 define signext i32 @test_igesi(i32 signext %a, i32 signext %b) {
 ; CHECK-LABEL: test_igesi:
diff --git a/llvm/test/CodeGen/PowerPC/testComparesigesll.ll b/llvm/test/CodeGen/PowerPC/testComparesigesll.ll
index 0f0d792ed..541ee87da 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesigesll.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesigesll.ll
@@ -5,7 +5,7 @@
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s --check-prefix=CHECK-LE \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
-@glob = common local_unnamed_addr global i64 0, align 8
+@glob = local_unnamed_addr global i64 0, align 8
 
 define signext i32 @test_igesll(i64 %a, i64 %b) {
 ; CHECK-LABEL: test_igesll:
diff --git a/llvm/test/CodeGen/PowerPC/testComparesigess.ll b/llvm/test/CodeGen/PowerPC/testComparesigess.ll
index ac04e9e86..c0e1ab8ec 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesigess.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesigess.ll
@@ -5,7 +5,7 @@
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s --check-prefix=CHECK-LE \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
-@glob = common local_unnamed_addr global i16 0, align 2
+@glob = local_unnamed_addr global i16 0, align 2
 
 define signext i32 @test_igess(i16 signext %a, i16 signext %b) {
 ; CHECK-LABEL: test_igess:
diff --git a/llvm/test/CodeGen/PowerPC/testComparesigeuc.ll b/llvm/test/CodeGen/PowerPC/testComparesigeuc.ll
index c36b6a5e2..3b9314bf6 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesigeuc.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesigeuc.ll
@@ -1,77 +1,116 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
-; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
+; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
+; RUN:  --check-prefixes=CHECK,BE
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
-; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
+; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
+; RUN:  --check-prefixes=CHECK,LE
 
-@glob = common local_unnamed_addr global i8 0, align 1
+@glob = local_unnamed_addr global i8 0, align 1
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_igeuc(i8 zeroext %a, i8 zeroext %b) {
+; CHECK-LABEL: test_igeuc:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    sub r3, r3, r4
+; CHECK-NEXT:    not r3, r3
+; CHECK-NEXT:    rldicl r3, r3, 1, 63
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp uge i8 %a, %b
   %conv2 = zext i1 %cmp to i32
   ret i32 %conv2
-; CHECK-LABEL: test_igeuc:
-; CHECK: sub [[REG1:r[0-9]+]], r3, r4
-; CHECK-NEXT: not [[REG2:r[0-9]+]], [[REG1]]
-; CHECK-NEXT: rldicl r3, [[REG2]], 1, 63
-; CHECK: blr
 }
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_igeuc_sext(i8 zeroext %a, i8 zeroext %b) {
+; CHECK-LABEL: test_igeuc_sext:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    sub r3, r3, r4
+; CHECK-NEXT:    rldicl r3, r3, 1, 63
+; CHECK-NEXT:    addi r3, r3, -1
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp uge i8 %a, %b
   %sub = sext i1 %cmp to i32
   ret i32 %sub
-; CHECK-LABEL: @test_igeuc_sext
-; CHECK: sub [[REG1:r[0-9]+]], r3, r4
-; CHECK-NEXT: rldicl [[REG2:r[0-9]+]], [[REG1]], 1, 63
-; CHECK-NEXT: addi [[REG3:r[0-9]+]], [[REG2]], -1
-; CHECK-NEXT: blr  
-  
+
 }
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_igeuc_z(i8 zeroext %a) {
+; CHECK-LABEL: test_igeuc_z:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    li r3, 1
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp uge i8 %a, 0
   %conv2 = zext i1 %cmp to i32
   ret i32 %conv2
-; CHECK-LABEL: @test_igeuc_z
-; CHECK: li r3, 1
-; CHECK-NEXT: blr  
 }
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_igeuc_sext_z(i8 zeroext %a) {
+; CHECK-LABEL: test_igeuc_sext_z:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    li r3, -1
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp uge i8 %a, 0
   %conv2 = sext i1 %cmp to i32
   ret i32 %conv2
-; CHECK-LABEL: @test_igeuc_sext_z
-; CHECK: li r3, -1
-; CHECK-NEXT: blr  
 }
 
 ; Function Attrs: norecurse nounwind
 define void @test_igeuc_store(i8 zeroext %a, i8 zeroext %b) {
+; BE-LABEL: test_igeuc_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r5, r2, .LC0@toc@ha
+; BE-NEXT:    sub r3, r3, r4
+; BE-NEXT:    ld r4, .LC0@toc@l(r5)
+; BE-NEXT:    not r3, r3
+; BE-NEXT:    rldicl r3, r3, 1, 63
+; BE-NEXT:    stb r3, 0(r4)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_igeuc_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    sub r3, r3, r4
+; LE-NEXT:    addis r5, r2, glob@toc@ha
+; LE-NEXT:    not r3, r3
+; LE-NEXT:    rldicl r3, r3, 1, 63
+; LE-NEXT:    stb r3, glob@toc@l(r5)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp uge i8 %a, %b
   %conv3 = zext i1 %cmp to i8
   store i8 %conv3, i8* @glob
   ret void
 ; CHECK_LABEL: test_igeuc_store:
-; CHECK: sub [[REG1:r[0-9]+]], r3, r4
-; CHECK: not [[REG2:r[0-9]+]], [[REG1]]
-; CHECK: rldicl r3, [[REG2]], 1, 63
-; CHECK: blr
 }
 
 ; Function Attrs: norecurse nounwind
 define void @test_igeuc_sext_store(i8 zeroext %a, i8 zeroext %b) {
+; BE-LABEL: test_igeuc_sext_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r5, r2, .LC0@toc@ha
+; BE-NEXT:    sub r3, r3, r4
+; BE-NEXT:    ld r4, .LC0@toc@l(r5)
+; BE-NEXT:    rldicl r3, r3, 1, 63
+; BE-NEXT:    addi r3, r3, -1
+; BE-NEXT:    stb r3, 0(r4)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_igeuc_sext_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    sub r3, r3, r4
+; LE-NEXT:    addis r5, r2, glob@toc@ha
+; LE-NEXT:    rldicl r3, r3, 1, 63
+; LE-NEXT:    addi r3, r3, -1
+; LE-NEXT:    stb r3, glob@toc@l(r5)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp uge i8 %a, %b
   %conv3 = sext i1 %cmp to i8
@@ -82,31 +121,51 @@ entry:
 ; CHECK-TBD: rldicl [[REG2:r[0-9]+]], [[REG1]], 1, 63
 ; CHECK-TBD: addi [[REG3:r[0-9]+]], [[REG2]], -1
 ; CHECK-TBD: stb  [[REG3]]
-; CHECK-TBD: blr    
+; CHECK-TBD: blr
 }
 
 ; Function Attrs : norecurse nounwind
 define void @test_igeuc_z_store(i8 zeroext %a) {
+; BE-LABEL: test_igeuc_z_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r3, r2, .LC0@toc@ha
+; BE-NEXT:    li r4, 1
+; BE-NEXT:    ld r3, .LC0@toc@l(r3)
+; BE-NEXT:    stb r4, 0(r3)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_igeuc_z_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    addis r3, r2, glob@toc@ha
+; LE-NEXT:    li r4, 1
+; LE-NEXT:    stb r4, glob@toc@l(r3)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp uge i8 %a, 0
   %conv3 = zext i1 %cmp to i8
   store i8 %conv3, i8* @glob
   ret void
-; CHECK-LABEL: @test_igeuc_z_store
-; CHECK: li [[REG1:r[0-9]+]], 1
-; CHECK: stb [[REG1]]
-; CHECK: blr    
 }
 
 ; Function Attrs: norecurse nounwind
 define void @test_igeuc_sext_z_store(i8 zeroext %a) {
+; BE-LABEL: test_igeuc_sext_z_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r3, r2, .LC0@toc@ha
+; BE-NEXT:    li r4, -1
+; BE-NEXT:    ld r3, .LC0@toc@l(r3)
+; BE-NEXT:    stb r4, 0(r3)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_igeuc_sext_z_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    addis r3, r2, glob@toc@ha
+; LE-NEXT:    li r4, -1
+; LE-NEXT:    stb r4, glob@toc@l(r3)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp uge i8 %a, 0
   %conv3 = sext i1 %cmp to i8
   store i8 %conv3, i8* @glob
   ret void
-; CHECK-LABEL: @test_igeuc_sext_z_store
-; CHECK: li [[REG1:r[0-9]+]], -1
-; CHECK: stb [[REG1]]
-; CHECK: blr
 }
diff --git a/llvm/test/CodeGen/PowerPC/testComparesigeui.ll b/llvm/test/CodeGen/PowerPC/testComparesigeui.ll
index bec1e21c5..13d99c435 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesigeui.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesigeui.ll
@@ -1,112 +1,165 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
-; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
+; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
+; RUN:  --check-prefixes=CHECK,BE
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
-; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
+; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
+; RUN:  --check-prefixes=CHECK,LE
 
-@glob = common local_unnamed_addr global i32 0, align 4
+@glob = local_unnamed_addr global i32 0, align 4
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_igeui(i32 zeroext %a, i32 zeroext %b) {
+; CHECK-LABEL: test_igeui:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    sub r3, r3, r4
+; CHECK-NEXT:    not r3, r3
+; CHECK-NEXT:    rldicl r3, r3, 1, 63
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp uge i32 %a, %b
   %conv = zext i1 %cmp to i32
   ret i32 %conv
-; CHECK-LABEL: test_igeui:
-; CHECK: sub [[REG1:r[0-9]+]], r3, r4
-; CHECK-NEXT: not [[REG2:r[0-9]+]], [[REG1]]
-; CHECK-NEXT: rldicl r3, [[REG2]], 1, 63
-; CHECK: blr
 }
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_igeui_sext(i32 zeroext %a, i32 zeroext %b) {
+; CHECK-LABEL: test_igeui_sext:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    sub r3, r3, r4
+; CHECK-NEXT:    rldicl r3, r3, 1, 63
+; CHECK-NEXT:    addi r3, r3, -1
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp uge i32 %a, %b
   %sub = sext i1 %cmp to i32
   ret i32 %sub
-; CHECK-LABEL: @test_igeui_sext
-; CHECK: sub [[REG1:r[0-9]+]], r3, r4
-; CHECK-NEXT: rldicl [[REG2:r[0-9]+]], [[REG1]], 1, 63
-; CHECK-NEXT: addi [[REG3:r[0-9]+]], [[REG2]], -1
-; CHECK-NEXT: blr    
 }
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_igeui_z(i32 zeroext %a) {
+; CHECK-LABEL: test_igeui_z:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    li r3, 1
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp uge i32 %a, 0
   %sub = zext i1 %cmp to i32
   ret i32 %sub
-; CHECK-LABEL: @test_igeui_z
-; CHECK: li r3, 1
-; CHECK-NEXT: blr
 }
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_igeui_sext_z(i32 zeroext %a) {
+; CHECK-LABEL: test_igeui_sext_z:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    li r3, -1
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp uge i32 %a, 0
   %sub = sext i1 %cmp to i32
   ret i32 %sub
-; CHECK-LABEL: @test_igeui_sext_z
-; CHECK: li r3, -1
-; CHECK-NEXT: blr
 }
 
 ; Function Attrs: norecurse nounwind
 define void @test_igeui_store(i32 zeroext %a, i32 zeroext %b) {
+; BE-LABEL: test_igeui_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r5, r2, .LC0@toc@ha
+; BE-NEXT:    sub r3, r3, r4
+; BE-NEXT:    ld r4, .LC0@toc@l(r5)
+; BE-NEXT:    not r3, r3
+; BE-NEXT:    rldicl r3, r3, 1, 63
+; BE-NEXT:    stw r3, 0(r4)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_igeui_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    sub r3, r3, r4
+; LE-NEXT:    addis r5, r2, glob@toc@ha
+; LE-NEXT:    not r3, r3
+; LE-NEXT:    rldicl r3, r3, 1, 63
+; LE-NEXT:    stw r3, glob@toc@l(r5)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp uge i32 %a, %b
   %conv = zext i1 %cmp to i32
   store i32 %conv, i32* @glob
   ret void
 ; CHECK_LABEL: test_igeuc_store:
-; CHECK: sub [[REG1:r[0-9]+]], r3, r4
-; CHECK: not [[REG2:r[0-9]+]], [[REG1]]
-; CHECK-NEXT: rldicl r3, [[REG2]], 1, 63
-; CHECK: blr
 }
 
 ; Function Attrs: norecurse nounwind
 define void @test_igeui_sext_store(i32 zeroext %a, i32 zeroext %b) {
+; BE-LABEL: test_igeui_sext_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r5, r2, .LC0@toc@ha
+; BE-NEXT:    sub r3, r3, r4
+; BE-NEXT:    ld r4, .LC0@toc@l(r5)
+; BE-NEXT:    rldicl r3, r3, 1, 63
+; BE-NEXT:    addi r3, r3, -1
+; BE-NEXT:    stw r3, 0(r4)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_igeui_sext_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    sub r3, r3, r4
+; LE-NEXT:    addis r5, r2, glob@toc@ha
+; LE-NEXT:    rldicl r3, r3, 1, 63
+; LE-NEXT:    addi r3, r3, -1
+; LE-NEXT:    stw r3, glob@toc@l(r5)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp uge i32 %a, %b
   %sub = sext i1 %cmp to i32
   store i32 %sub, i32* @glob
   ret void
-; CHECK-LABEL: @test_igeui_sext_store
-; CHECK: sub [[REG1:r[0-9]+]], r3, r4
-; CHECK: rldicl [[REG2:r[0-9]+]], [[REG1]], 1, 63
-; CHECK: addi [[REG3:r[0-9]+]], [[REG2]], -1
-; CHECK: stw  [[REG3]]
-; CHECK: blr    
 }
 
 ; Function Attrs: norecurse nounwind
 define void @test_igeui_z_store(i32 zeroext %a) {
+; BE-LABEL: test_igeui_z_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r3, r2, .LC0@toc@ha
+; BE-NEXT:    li r4, 1
+; BE-NEXT:    ld r3, .LC0@toc@l(r3)
+; BE-NEXT:    stw r4, 0(r3)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_igeui_z_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    addis r3, r2, glob@toc@ha
+; LE-NEXT:    li r4, 1
+; LE-NEXT:    stw r4, glob@toc@l(r3)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp uge i32 %a, 0
   %conv1 = zext i1 %cmp to i32
   store i32 %conv1, i32* @glob
   ret void
-; CHECK-LABEL: @test_igeui_z_store
-; CHECK: li [[REG1:r[0-9]+]], 1
-; CHECK: stw [[REG1]]
-; CHECK: blr  
 }
 
 ; Function Attrs: norecurse nounwind
 define void @test_igeui_sext_z_store(i32 zeroext %a) {
+; BE-LABEL: test_igeui_sext_z_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r3, r2, .LC0@toc@ha
+; BE-NEXT:    li r4, -1
+; BE-NEXT:    ld r3, .LC0@toc@l(r3)
+; BE-NEXT:    stw r4, 0(r3)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_igeui_sext_z_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    addis r3, r2, glob@toc@ha
+; LE-NEXT:    li r4, -1
+; LE-NEXT:    stw r4, glob@toc@l(r3)
+; LE-NEXT:    blr
 entry:
-  %cmp = icmp uge i32 %a, 0 
+  %cmp = icmp uge i32 %a, 0
   %conv1 = sext i1 %cmp to i32
   store i32 %conv1, i32* @glob
   ret void
-; CHECK-LABEL: @test_igeui_sext_z_store
-; CHECK: li [[REG1:r[0-9]+]], -1
-; CHECK: stw [[REG1]]
-; CHECK: blr
 }
 
diff --git a/llvm/test/CodeGen/PowerPC/testComparesigeull.ll b/llvm/test/CodeGen/PowerPC/testComparesigeull.ll
index 103eeb1c7..4d430db0b 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesigeull.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesigeull.ll
@@ -1,111 +1,163 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
-; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
+; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
+; RUN:  --check-prefixes=CHECK,BE
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
-; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
+; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
+; RUN:  --check-prefixes=CHECK,LE
 
-@glob = common local_unnamed_addr global i64 0, align 8
+@glob = local_unnamed_addr global i64 0, align 8
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_igeull(i64 %a, i64 %b) {
+; CHECK-LABEL: test_igeull:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    subfc r3, r4, r3
+; CHECK-NEXT:    subfe r3, r4, r4
+; CHECK-NEXT:    addi r3, r3, 1
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp uge i64 %a, %b
   %conv = zext i1 %cmp to i32
   ret i32 %conv
-; CHECK-LABEL: test_igeull:
-; CHECK: subfc {{r[0-9]+}}, r4, r3
-; CHECK-NEXT: subfe [[REG1:r[0-9]+]], r4, r4
-; CHECK-NEXT: addi r3, [[REG1]], 1
-; CHECK: blr
 }
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_igeull_sext(i64 %a, i64 %b) {
+; CHECK-LABEL: test_igeull_sext:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    subfc r3, r4, r3
+; CHECK-NEXT:    subfe r3, r4, r4
+; CHECK-NEXT:    not r3, r3
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp uge i64 %a, %b
   %sub = sext i1 %cmp to i32
   ret i32 %sub
-; CHECK-LABEL: @test_igeull_sext
-; CHECK: subfc [[REG1:r[0-9]+]], r4, r3
-; CHECK: subfe [[REG2:r[0-9]+]], {{r[0-9]+}}, {{r[0-9]+}}
-; CHECK: not r3, [[REG2]]
-; CHECK: blr
 }
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_igeull_z(i64 %a) {
+; CHECK-LABEL: test_igeull_z:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    li r3, 1
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp uge i64 %a, 0
   %sub = zext i1 %cmp to i32
   ret i32 %sub
-; CHECK-LABEL: @test_igeull_z
-; CHECK: li r3, 1
-; CHECK-NEXT: blr
 }
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_igeull_sext_z(i64 %a) {
+; CHECK-LABEL: test_igeull_sext_z:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    li r3, -1
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp uge i64 %a, 0
   %sub = sext i1 %cmp to i32
   ret i32 %sub
-; CHECK-LABEL: @test_igeull_sext_z
-; CHECK: li r3, -1
-; CHECK-NEXT: blr
 }
 
 ; Function Attrs: norecurse nounwind
 define void @test_igeull_store(i64 %a, i64 %b) {
+; BE-LABEL: test_igeull_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r5, r2, .LC0@toc@ha
+; BE-NEXT:    subfc r3, r4, r3
+; BE-NEXT:    ld r3, .LC0@toc@l(r5)
+; BE-NEXT:    subfe r4, r4, r4
+; BE-NEXT:    addi r4, r4, 1
+; BE-NEXT:    std r4, 0(r3)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_igeull_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    subfc r3, r4, r3
+; LE-NEXT:    addis r5, r2, glob@toc@ha
+; LE-NEXT:    subfe r3, r4, r4
+; LE-NEXT:    addi r3, r3, 1
+; LE-NEXT:    std r3, glob@toc@l(r5)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp uge i64 %a, %b
   %conv1 = zext i1 %cmp to i64
   store i64 %conv1, i64* @glob
   ret void
-; CHECK-LABEL: test_igeull_store:
-; CHECK: subfc {{r[0-9]+}}, r4, r3
-; CHECK: subfe [[REG1:r[0-9]+]], r4, r4
-; CHECK: addi {{r[0-9]+}}, [[REG1]], 1
-; CHECK: blr
 }
 
 ; Function Attrs: norecurse nounwind
 define void @test_igeull_sext_store(i64 %a, i64 %b) {
+; BE-LABEL: test_igeull_sext_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r5, r2, .LC0@toc@ha
+; BE-NEXT:    subfc r3, r4, r3
+; BE-NEXT:    ld r3, .LC0@toc@l(r5)
+; BE-NEXT:    subfe r4, r4, r4
+; BE-NEXT:    not r4, r4
+; BE-NEXT:    std r4, 0(r3)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_igeull_sext_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    subfc r3, r4, r3
+; LE-NEXT:    addis r5, r2, glob@toc@ha
+; LE-NEXT:    subfe r3, r4, r4
+; LE-NEXT:    not r3, r3
+; LE-NEXT:    std r3, glob@toc@l(r5)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp uge i64 %a, %b
   %conv1 = sext i1 %cmp to i64
   store i64 %conv1, i64* @glob
   ret void
-; CHECK-LABEL: @test_igeull_sext_store
-; CHECK: subfc [[REG1:r[0-9]+]], r4, r3
-; CHECK: subfe [[REG2:r[0-9]+]], {{r[0-9]+}}, {{r[0-9]+}}
-; CHECK: not [[REG3:r[0-9]+]], [[REG2]]
-; CHECK: std [[REG3]]
-; CHECK: blr
 }
 
 ; Function Attrs: norecurse nounwind
 define void @test_igeull_z_store(i64 %a) {
+; BE-LABEL: test_igeull_z_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r3, r2, .LC0@toc@ha
+; BE-NEXT:    li r4, 1
+; BE-NEXT:    ld r3, .LC0@toc@l(r3)
+; BE-NEXT:    std r4, 0(r3)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_igeull_z_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    addis r3, r2, glob@toc@ha
+; LE-NEXT:    li r4, 1
+; LE-NEXT:    std r4, glob@toc@l(r3)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp uge i64 %a, 0
   %conv1 = zext i1 %cmp to i64
   store i64 %conv1, i64* @glob
   ret void
-; CHECK-LABEL: @test_igeull_z_store
-; CHECK: li [[REG1:r[0-9]+]], 1
-; CHECK: std [[REG1]]
-; CHECK: blr
 }
 
 ; Function Attrs: norecurse nounwind
 define void @test_igeull_sext_z_store(i64 %a) {
+; BE-LABEL: test_igeull_sext_z_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r3, r2, .LC0@toc@ha
+; BE-NEXT:    li r4, -1
+; BE-NEXT:    ld r3, .LC0@toc@l(r3)
+; BE-NEXT:    std r4, 0(r3)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_igeull_sext_z_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    addis r3, r2, glob@toc@ha
+; LE-NEXT:    li r4, -1
+; LE-NEXT:    std r4, glob@toc@l(r3)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp uge i64 %a, 0
   %conv1 = sext i1 %cmp to i64
   store i64 %conv1, i64* @glob
   ret void
-; CHECK-LABEL: @test_igeull_sext_z_store
-; CHECK: li [[REG1:r[0-9]+]], -1
-; CHECK: std [[REG1]]
-; CHECK: blr
 }
diff --git a/llvm/test/CodeGen/PowerPC/testComparesigeus.ll b/llvm/test/CodeGen/PowerPC/testComparesigeus.ll
index 33266e04a..3e5063dc3 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesigeus.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesigeus.ll
@@ -1,112 +1,165 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
-; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
+; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
+; RUN:  --check-prefixes=CHECK,BE
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
-; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
+; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
+; RUN:  --check-prefixes=CHECK,LE
 
-@glob = common local_unnamed_addr global i16 0, align 2
+@glob = local_unnamed_addr global i16 0, align 2
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_igeus(i16 zeroext %a, i16 zeroext %b) {
+; CHECK-LABEL: test_igeus:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    sub r3, r3, r4
+; CHECK-NEXT:    not r3, r3
+; CHECK-NEXT:    rldicl r3, r3, 1, 63
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp uge i16 %a, %b
   %conv2 = zext i1 %cmp to i32
   ret i32 %conv2
-; CHECK-LABEL: test_igeus:
-; CHECK: sub [[REG1:r[0-9]+]], r3, r4
-; CHECK-NEXT: not [[REG2:r[0-9]+]], [[REG1]]
-; CHECK-NEXT: rldicl r3, [[REG2]], 1, 63
-; CHECK: blr
 }
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_igeus_sext(i16 zeroext %a, i16 zeroext %b) {
+; CHECK-LABEL: test_igeus_sext:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    sub r3, r3, r4
+; CHECK-NEXT:    rldicl r3, r3, 1, 63
+; CHECK-NEXT:    addi r3, r3, -1
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp uge i16 %a, %b
   %sub = sext i1 %cmp to i32
   ret i32 %sub
-; CHECK-LABEL: @test_igeus_sext
-; CHECK: sub [[REG1:r[0-9]+]], r3, r4
-; CHECK-NEXT: rldicl [[REG2:r[0-9]+]], [[REG1]], 1, 63
-; CHECK-NEXT: addi [[REG3:r[0-9]+]], [[REG2]], -1
-; CHECK-NEXT: blr
 }
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_igeus_z(i16 zeroext %a) {
+; CHECK-LABEL: test_igeus_z:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    li r3, 1
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp uge i16 %a, 0
   %conv2 = zext i1 %cmp to i32
   ret i32 %conv2
-; CHECK-LABEL: @test_igeus_z
-; CHECK: li r3, 1
-; CHECK-NEXT: blr
 }
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_igeus_sext_z(i16 zeroext %a) {
+; CHECK-LABEL: test_igeus_sext_z:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    li r3, 1
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp uge i16 %a, 0
   %conv2 = zext i1 %cmp to i32
   ret i32 %conv2
-; CHECK-LABEL: @test_igeus_sext_z
-; CHECK: li r3, 1
-; CHECK-NEXT: blr
 }
 
 ; Function Attrs: norecurse nounwind
 define void @test_igeus_store(i16 zeroext %a, i16 zeroext %b) {
+; BE-LABEL: test_igeus_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r5, r2, .LC0@toc@ha
+; BE-NEXT:    sub r3, r3, r4
+; BE-NEXT:    ld r4, .LC0@toc@l(r5)
+; BE-NEXT:    not r3, r3
+; BE-NEXT:    rldicl r3, r3, 1, 63
+; BE-NEXT:    sth r3, 0(r4)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_igeus_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    sub r3, r3, r4
+; LE-NEXT:    addis r5, r2, glob@toc@ha
+; LE-NEXT:    not r3, r3
+; LE-NEXT:    rldicl r3, r3, 1, 63
+; LE-NEXT:    sth r3, glob@toc@l(r5)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp uge i16 %a, %b
   %conv3 = zext i1 %cmp to i16
   store i16 %conv3, i16* @glob
   ret void
 ; CHECK_LABEL: test_igeus_store:
-; CHECK: sub [[REG1:r[0-9]+]], r3, r4
-; CHECK: not [[REG2:r[0-9]+]], [[REG1]]
-; CHECK-NEXT: rldicl r3, [[REG2]], 1, 63
-; CHECK: blr
 }
 
 ; Function Attrs: norecurse nounwind
 define void @test_igeus_sext_store(i16 zeroext %a, i16 zeroext %b) {
+; BE-LABEL: test_igeus_sext_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r5, r2, .LC0@toc@ha
+; BE-NEXT:    sub r3, r3, r4
+; BE-NEXT:    ld r4, .LC0@toc@l(r5)
+; BE-NEXT:    rldicl r3, r3, 1, 63
+; BE-NEXT:    addi r3, r3, -1
+; BE-NEXT:    sth r3, 0(r4)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_igeus_sext_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    sub r3, r3, r4
+; LE-NEXT:    addis r5, r2, glob@toc@ha
+; LE-NEXT:    rldicl r3, r3, 1, 63
+; LE-NEXT:    addi r3, r3, -1
+; LE-NEXT:    sth r3, glob@toc@l(r5)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp uge i16 %a, %b
   %conv3 = sext i1 %cmp to i16
   store i16 %conv3, i16* @glob
   ret void
-; CHECK-LABEL: @test_igeus_sext_store
-; CHECK: sub [[REG1:r[0-9]+]], r3, r4
-; CHECK: rldicl [[REG2:r[0-9]+]], [[REG1]], 1, 63
-; CHECK: addi [[REG3:r[0-9]+]], [[REG2]], -1
-; CHECK: sth  [[REG3]]
-; CHECK: blr
 }
 
 ; Function Attrs: norecurse nounwind
 define void @test_igeus_z_store(i16 zeroext %a) {
+; BE-LABEL: test_igeus_z_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r3, r2, .LC0@toc@ha
+; BE-NEXT:    li r4, 1
+; BE-NEXT:    ld r3, .LC0@toc@l(r3)
+; BE-NEXT:    sth r4, 0(r3)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_igeus_z_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    addis r3, r2, glob@toc@ha
+; LE-NEXT:    li r4, 1
+; LE-NEXT:    sth r4, glob@toc@l(r3)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp uge i16 %a, 0
   %conv3 = zext i1 %cmp to i16
   store i16 %conv3, i16* @glob
   ret void
-; CHECK-LABEL: @test_igeus_z_store
-; CHECK: li [[REG1:r[0-9]+]], 1
-; CHECK: sth [[REG1]]
-; CHECK: blr
 }
 
 ; Function Attrs: norecurse nounwind
 define void @test_igeus_sext_z_store(i16 zeroext %a) {
+; BE-LABEL: test_igeus_sext_z_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r3, r2, .LC0@toc@ha
+; BE-NEXT:    li r4, -1
+; BE-NEXT:    ld r3, .LC0@toc@l(r3)
+; BE-NEXT:    sth r4, 0(r3)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_igeus_sext_z_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    addis r3, r2, glob@toc@ha
+; LE-NEXT:    li r4, -1
+; LE-NEXT:    sth r4, glob@toc@l(r3)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp uge i16 %a, 0
   %conv3 = sext i1 %cmp to i16
   store i16 %conv3, i16* @glob
   ret void
-; CHECK-LABEL: @test_igeus_sext_z_store
-; CHECK: li [[REG1:r[0-9]+]], -1
-; CHECK: sth [[REG1]]
-; CHECK: blr
 }
 
diff --git a/llvm/test/CodeGen/PowerPC/testComparesigtsc.ll b/llvm/test/CodeGen/PowerPC/testComparesigtsc.ll
index c669d694c..19a0a1d96 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesigtsc.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesigtsc.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -relocation-model=pic -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
 ; RUN:   --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
@@ -5,14 +6,14 @@
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
 ; RUN:   --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 
-@glob = common local_unnamed_addr global i8 0, align 1
+@glob = local_unnamed_addr global i8 0, align 1
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_igtsc(i8 signext %a, i8 signext %b) {
 ; CHECK-LABEL: test_igtsc:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    sub [[REG:r[0-9]+]], r4, r3
-; CHECK-NEXT:    rldicl r3, [[REG]], 1, 63
+; CHECK-NEXT:    sub r3, r4, r3
+; CHECK-NEXT:    rldicl r3, r3, 1, 63
 ; CHECK-NEXT:    blr
 entry:
   %cmp = icmp sgt i8 %a, %b
@@ -24,8 +25,8 @@ entry:
 define signext i32 @test_igtsc_sext(i8 signext %a, i8 signext %b) {
 ; CHECK-LABEL: test_igtsc_sext:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    sub [[REG:r[0-9]+]], r4, r3
-; CHECK-NEXT:    sradi r3, [[REG]], 63
+; CHECK-NEXT:    sub r3, r4, r3
+; CHECK-NEXT:    sradi r3, r3, 63
 ; CHECK-NEXT:    blr
 entry:
   %cmp = icmp sgt i8 %a, %b
@@ -50,9 +51,10 @@ entry:
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_igtsc_sext_z(i8 signext %a) {
 ; CHECK-LABEL: test_igtsc_sext_z:
-; CHECK: neg [[REG2:r[0-9]+]], r3
-; CHECK-NEXT: sradi r3, [[REG2]], 63
-; CHECK-NEXT: blr
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    neg r3, r3
+; CHECK-NEXT:    sradi r3, r3, 63
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp sgt i8 %a, 0
   %sub = sext i1 %cmp to i32
@@ -63,8 +65,12 @@ entry:
 define void @test_igtsc_store(i8 signext %a, i8 signext %b) {
 ; CHECK-LABEL: test_igtsc_store:
 ; CHECK:       # %bb.0: # %entry
-; CHECK:         sub [[REG:r[0-9]+]], r4, r3
-; CHECK:         rldicl {{r[0-9]+}}, [[REG]], 1, 63
+; CHECK-NEXT:    addis r5, r2, .LC0@toc@ha
+; CHECK-NEXT:    sub r3, r4, r3
+; CHECK-NEXT:    ld r5, .LC0@toc@l(r5)
+; CHECK-NEXT:    rldicl r3, r3, 1, 63
+; CHECK-NEXT:    stb r3, 0(r5)
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp sgt i8 %a, %b
   %conv3 = zext i1 %cmp to i8
@@ -76,8 +82,12 @@ entry:
 define void @test_igtsc_sext_store(i8 signext %a, i8 signext %b) {
 ; CHECK-LABEL: test_igtsc_sext_store:
 ; CHECK:       # %bb.0: # %entry
-; CHECK:         sub [[REG:r[0-9]+]], r4, r3
-; CHECK:         sradi {{r[0-9]+}}, [[REG]], 63
+; CHECK-NEXT:    addis r5, r2, .LC0@toc@ha
+; CHECK-NEXT:    sub r3, r4, r3
+; CHECK-NEXT:    ld r5, .LC0@toc@l(r5)
+; CHECK-NEXT:    sradi r3, r3, 63
+; CHECK-NEXT:    stb r3, 0(r5)
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp sgt i8 %a, %b
   %conv3 = sext i1 %cmp to i8
@@ -106,8 +116,13 @@ entry:
 ; Function Attrs: norecurse nounwind
 define void @test_igtsc_sext_z_store(i8 signext %a) {
 ; CHECK-LABEL: test_igtsc_sext_z_store:
-; CHECK:       neg [[REG2:r[0-9]+]], r3
-; CHECK:       sradi {{r[0-9]+}}, [[REG2]], 63
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addis r4, r2, .LC0@toc@ha
+; CHECK-NEXT:    neg r3, r3
+; CHECK-NEXT:    ld r4, .LC0@toc@l(r4)
+; CHECK-NEXT:    sradi r3, r3, 63
+; CHECK-NEXT:    stb r3, 0(r4)
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp sgt i8 %a, 0
   %conv2 = sext i1 %cmp to i8
diff --git a/llvm/test/CodeGen/PowerPC/testComparesigtsi.ll b/llvm/test/CodeGen/PowerPC/testComparesigtsi.ll
index 9b5f1f43c..21a773f1f 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesigtsi.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesigtsi.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -relocation-model=pic -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
@@ -5,14 +6,14 @@
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 
-@glob = common local_unnamed_addr global i32 0, align 4
+@glob = local_unnamed_addr global i32 0, align 4
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_igtsi(i32 signext %a, i32 signext %b) {
 ; CHECK-LABEL: test_igtsi:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    sub [[REG:r[0-9]+]], r4, r3
-; CHECK-NEXT:    rldicl r3, [[REG]], 1, 63
+; CHECK-NEXT:    sub r3, r4, r3
+; CHECK-NEXT:    rldicl r3, r3, 1, 63
 ; CHECK-NEXT:    blr
 entry:
   %cmp = icmp sgt i32 %a, %b
@@ -24,8 +25,8 @@ entry:
 define signext i32 @test_igtsi_sext(i32 signext %a, i32 signext %b) {
 ; CHECK-LABEL: test_igtsi_sext:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    sub [[REG:r[0-9]+]], r4, r3
-; CHECK-NEXT:    sradi r3, [[REG]], 63
+; CHECK-NEXT:    sub r3, r4, r3
+; CHECK-NEXT:    sradi r3, r3, 63
 ; CHECK-NEXT:    blr
 entry:
   %cmp = icmp sgt i32 %a, %b
@@ -50,9 +51,10 @@ entry:
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_igtsi_sext_z(i32 signext %a) {
 ; CHECK-LABEL: test_igtsi_sext_z:
-; CHECK:  neg [[REG2:r[0-9]+]], r3
-; CHECK-NEXT: sradi r3, [[REG2]], 63
-; CHECK-NEXT: blr
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    neg r3, r3
+; CHECK-NEXT:    sradi r3, r3, 63
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp sgt i32 %a, 0
   %sub = sext i1 %cmp to i32
@@ -63,8 +65,12 @@ entry:
 define void @test_igtsi_store(i32 signext %a, i32 signext %b) {
 ; CHECK-LABEL: test_igtsi_store:
 ; CHECK:       # %bb.0: # %entry
-; CHECK:         sub [[REG:r[0-9]+]], r4, r3
-; CHECK:         rldicl {{r[0-9]+}}, [[REG]], 1, 63
+; CHECK-NEXT:    addis r5, r2, .LC0@toc@ha
+; CHECK-NEXT:    sub r3, r4, r3
+; CHECK-NEXT:    ld r5, .LC0@toc@l(r5)
+; CHECK-NEXT:    rldicl r3, r3, 1, 63
+; CHECK-NEXT:    stw r3, 0(r5)
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp sgt i32 %a, %b
   %conv = zext i1 %cmp to i32
@@ -76,8 +82,12 @@ entry:
 define void @test_igtsi_sext_store(i32 signext %a, i32 signext %b) {
 ; CHECK-LABEL: test_igtsi_sext_store:
 ; CHECK:       # %bb.0: # %entry
-; CHECK:         sub [[REG:r[0-9]+]], r4, r3
-; CHECK:         sradi {{r[0-9]+}}, [[REG]], 63
+; CHECK-NEXT:    addis r5, r2, .LC0@toc@ha
+; CHECK-NEXT:    sub r3, r4, r3
+; CHECK-NEXT:    ld r5, .LC0@toc@l(r5)
+; CHECK-NEXT:    sradi r3, r3, 63
+; CHECK-NEXT:    stw r3, 0(r5)
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp sgt i32 %a, %b
   %sub = sext i1 %cmp to i32
@@ -106,8 +116,13 @@ entry:
 ; Function Attrs: norecurse nounwind
 define void @test_igtsi_sext_z_store(i32 signext %a) {
 ; CHECK-LABEL: test_igtsi_sext_z_store:
-; CHECK: neg [[REG:r[0-9]+]], r3
-; CHECK: sradi {{r[0-9]+}}, [[REG]], 63
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addis r4, r2, .LC0@toc@ha
+; CHECK-NEXT:    neg r3, r3
+; CHECK-NEXT:    ld r4, .LC0@toc@l(r4)
+; CHECK-NEXT:    sradi r3, r3, 63
+; CHECK-NEXT:    stw r3, 0(r4)
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp sgt i32 %a, 0
   %sub = sext i1 %cmp to i32
diff --git a/llvm/test/CodeGen/PowerPC/testComparesigtsll.ll b/llvm/test/CodeGen/PowerPC/testComparesigtsll.ll
index c2198d10b..f96a28514 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesigtsll.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesigtsll.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -relocation-model=pic -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
@@ -5,17 +6,17 @@
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 
-@glob = common local_unnamed_addr global i64 0, align 8
+@glob = local_unnamed_addr global i64 0, align 8
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_igtsll(i64 %a, i64 %b) {
 ; CHECK-LABEL: test_igtsll:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    sradi [[REG1:r[0-9]+]], r4, 63
-; CHECK-NEXT:    rldicl [[REG2:r[0-9]+]], r3, 1, 63
-; CHECK-NEXT:    subfc [[REG3:r[0-9]+]], r3, r4
-; CHECK-NEXT:    adde [[REG4:r[0-9]+]], [[REG2]], [[REG1]]
-; CHECK-NEXT:    xori r3, [[REG4]], 1
+; CHECK-NEXT:    sradi r5, r4, 63
+; CHECK-NEXT:    rldicl r6, r3, 1, 63
+; CHECK-NEXT:    subfc r3, r3, r4
+; CHECK-NEXT:    adde r3, r6, r5
+; CHECK-NEXT:    xori r3, r3, 1
 ; CHECK-NEXT:    blr
 entry:
   %cmp = icmp sgt i64 %a, %b
@@ -27,12 +28,12 @@ entry:
 define signext i32 @test_igtsll_sext(i64 %a, i64 %b) {
 ; CHECK-LABEL: test_igtsll_sext:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    sradi [[REG1:r[0-9]+]], r4, 63
-; CHECK-NEXT:    rldicl [[REG2:r[0-9]+]], r3, 1, 63
-; CHECK-NEXT:    subfc [[REG3:r[0-9]+]], r3, r4
-; CHECK-NEXT:    adde [[REG4:r[0-9]+]], [[REG2]], [[REG1]]
-; CHECK-NEXT:    xori [[REG5:r[0-9]+]], [[REG4]], 1
-; CHECK-NEXT:    neg r3, [[REG5]]
+; CHECK-NEXT:    sradi r5, r4, 63
+; CHECK-NEXT:    rldicl r6, r3, 1, 63
+; CHECK-NEXT:    subfc r3, r3, r4
+; CHECK-NEXT:    adde r3, r6, r5
+; CHECK-NEXT:    xori r3, r3, 1
+; CHECK-NEXT:    neg r3, r3
 ; CHECK-NEXT:    blr
 entry:
   %cmp = icmp sgt i64 %a, %b
@@ -58,9 +59,11 @@ entry:
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_igtsll_sext_z(i64 %a) {
 ; CHECK-LABEL: test_igtsll_sext_z:
-; CHECK:    addi [[REG1:r[0-9]+]], r3, -1
-; CHECK-NEXT:    nor [[REG2:r[0-9]+]], [[REG1]], r3
-; CHECK-NEXT:    sradi r3, [[REG2]], 63
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addi r4, r3, -1
+; CHECK-NEXT:    nor r3, r4, r3
+; CHECK-NEXT:    sradi r3, r3, 63
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp sgt i64 %a, 0
   %sub = sext i1 %cmp to i32
@@ -71,12 +74,16 @@ entry:
 define void @test_igtsll_store(i64 %a, i64 %b) {
 ; CHECK-LABEL: test_igtsll_store:
 ; CHECK:       # %bb.0: # %entry
-; CHECK:         sradi [[REG1:r[0-9]+]], r4, 63
-; CHECK:         rldicl [[REG2:r[0-9]+]], r3, 1, 63
+; CHECK-NEXT:    sradi r6, r4, 63
+; CHECK-NEXT:    addis r5, r2, .LC0@toc@ha
+; CHECK-NEXT:    subfc r4, r3, r4
+; CHECK-NEXT:    rldicl r3, r3, 1, 63
+; CHECK-NEXT:    ld r4, .LC0@toc@l(r5)
+; CHECK-NEXT:    adde r3, r3, r6
+; CHECK-NEXT:    xori r3, r3, 1
+; CHECK-NEXT:    std r3, 0(r4)
+; CHECK-NEXT:    blr
 ; CHECK-DIAG:    subfc [[REG3:r[0-9]+]], r3, r4
-; CHECK:         adde [[REG4:r[0-9]+]], [[REG2]], [[REG1]]
-; CHECK:         xori [[REG5:r[0-9]+]], [[REG4]], 1
-; CHECK-NOT:     neg
 entry:
   %cmp = icmp sgt i64 %a, %b
   %conv1 = zext i1 %cmp to i64
@@ -88,12 +95,17 @@ entry:
 define void @test_igtsll_sext_store(i64 %a, i64 %b) {
 ; CHECK-LABEL: test_igtsll_sext_store:
 ; CHECK:       # %bb.0: # %entry
-; CHECK:         sradi [[REG1:r[0-9]+]], r4, 63
-; CHECK:         rldicl [[REG2:r[0-9]+]], r3, 1, 63
+; CHECK-NEXT:    sradi r6, r4, 63
+; CHECK-NEXT:    addis r5, r2, .LC0@toc@ha
+; CHECK-NEXT:    subfc r4, r3, r4
+; CHECK-NEXT:    rldicl r3, r3, 1, 63
+; CHECK-NEXT:    adde r3, r3, r6
+; CHECK-NEXT:    ld r4, .LC0@toc@l(r5)
+; CHECK-NEXT:    xori r3, r3, 1
+; CHECK-NEXT:    neg r3, r3
+; CHECK-NEXT:    std r3, 0(r4)
+; CHECK-NEXT:    blr
 ; CHECK-DIAG:    subfc [[REG3:r[0-9]+]], r3, r4
-; CHECK:         adde [[REG4:r[0-9]+]], [[REG2]], [[REG1]]
-; CHECK:         xori [[REG5:r[0-9]+]], [[REG4]], 1
-; CHECK:         neg {{r[0-9]+}}, [[REG5]]
 entry:
   %cmp = icmp sgt i64 %a, %b
   %conv1 = sext i1 %cmp to i64
@@ -123,9 +135,14 @@ entry:
 ; Function Attrs: norecurse nounwind
 define void @test_igtsll_sext_z_store(i64 %a) {
 ; CHECK-LABEL: test_igtsll_sext_z_store:
-; CHECK:    addi [[REG1:r[0-9]+]], r3, -1
-; CHECK:    nor [[REG2:r[0-9]+]], [[REG1]], r3
-; CHECK:    sradi [[REG3:r[0-9]+]], [[REG2]], 63
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addis r4, r2, .LC0@toc@ha
+; CHECK-NEXT:    addi r5, r3, -1
+; CHECK-NEXT:    ld r4, .LC0@toc@l(r4)
+; CHECK-NEXT:    nor r3, r5, r3
+; CHECK-NEXT:    sradi r3, r3, 63
+; CHECK-NEXT:    std r3, 0(r4)
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp sgt i64 %a, 0
   %conv1 = sext i1 %cmp to i64
diff --git a/llvm/test/CodeGen/PowerPC/testComparesigtss.ll b/llvm/test/CodeGen/PowerPC/testComparesigtss.ll
index 93e6ccd53..fe9512719 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesigtss.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesigtss.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -relocation-model=pic -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
 ; RUN:   --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
@@ -5,14 +6,14 @@
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
 ; RUN:   --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 
-@glob = common local_unnamed_addr global i16 0, align 2
+@glob = local_unnamed_addr global i16 0, align 2
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_igtss(i16 signext %a, i16 signext %b) {
 ; CHECK-LABEL: test_igtss:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    sub [[REG1:r[0-9]+]], r4, r3
-; CHECK-NEXT:    rldicl r3, [[REG1]], 1, 63
+; CHECK-NEXT:    sub r3, r4, r3
+; CHECK-NEXT:    rldicl r3, r3, 1, 63
 ; CHECK-NEXT:    blr
 entry:
   %cmp = icmp sgt i16 %a, %b
@@ -24,8 +25,8 @@ entry:
 define signext i32 @test_igtss_sext(i16 signext %a, i16 signext %b) {
 ; CHECK-LABEL: test_igtss_sext:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    sub [[REG:r[0-9]+]], r4, r3
-; CHECK-NEXT:    sradi r3, [[REG]], 63
+; CHECK-NEXT:    sub r3, r4, r3
+; CHECK-NEXT:    sradi r3, r3, 63
 ; CHECK-NEXT:    blr
 entry:
   %cmp = icmp sgt i16 %a, %b
@@ -51,8 +52,8 @@ entry:
 define signext i32 @test_igtss_sext_z(i16 signext %a) {
 ; CHECK-LABEL: test_igtss_sext_z:
 ; CHECK:       # %bb.0: # %entry
-; CHECK:    neg [[REG2:r[0-9]+]], r3
-; CHECK-NEXT:    sradi r3, [[REG2]], 63
+; CHECK-NEXT:    neg r3, r3
+; CHECK-NEXT:    sradi r3, r3, 63
 ; CHECK-NEXT:    blr
 entry:
   %cmp = icmp sgt i16 %a, 0
@@ -64,8 +65,12 @@ entry:
 define void @test_igtss_store(i16 signext %a, i16 signext %b) {
 ; CHECK-LABEL: test_igtss_store:
 ; CHECK:       # %bb.0: # %entry
-; CHECK:         sub [[REG1:r[0-9]+]], r4, r3
-; CHECK:         rldicl {{r[0-9]+}}, [[REG1]], 1, 63
+; CHECK-NEXT:    addis r5, r2, .LC0@toc@ha
+; CHECK-NEXT:    sub r3, r4, r3
+; CHECK-NEXT:    ld r5, .LC0@toc@l(r5)
+; CHECK-NEXT:    rldicl r3, r3, 1, 63
+; CHECK-NEXT:    sth r3, 0(r5)
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp sgt i16 %a, %b
   %conv3 = zext i1 %cmp to i16
@@ -77,8 +82,12 @@ entry:
 define void @test_igtss_sext_store(i16 signext %a, i16 signext %b) {
 ; CHECK-LABEL: test_igtss_sext_store:
 ; CHECK:       # %bb.0: # %entry
-; CHECK:         sub [[REG:r[0-9]+]], r4, r3
-; CHECK:         sradi {{r[0-9]+}}, [[REG]], 63
+; CHECK-NEXT:    addis r5, r2, .LC0@toc@ha
+; CHECK-NEXT:    sub r3, r4, r3
+; CHECK-NEXT:    ld r5, .LC0@toc@l(r5)
+; CHECK-NEXT:    sradi r3, r3, 63
+; CHECK-NEXT:    sth r3, 0(r5)
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp sgt i16 %a, %b
   %conv3 = sext i1 %cmp to i16
@@ -107,8 +116,13 @@ entry:
 ; Function Attrs: norecurse nounwind
 define void @test_igtss_sext_z_store(i16 signext %a) {
 ; CHECK-LABEL: test_igtss_sext_z_store:
-; CHECK:       neg [[REG2:r[0-9]+]], r3
-; CHECK:       sradi {{r[0-9]+}}, [[REG2]], 63
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addis r4, r2, .LC0@toc@ha
+; CHECK-NEXT:    neg r3, r3
+; CHECK-NEXT:    ld r4, .LC0@toc@l(r4)
+; CHECK-NEXT:    sradi r3, r3, 63
+; CHECK-NEXT:    sth r3, 0(r4)
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp sgt i16 %a, 0
   %conv2 = sext i1 %cmp to i16
diff --git a/llvm/test/CodeGen/PowerPC/testComparesigtuc.ll b/llvm/test/CodeGen/PowerPC/testComparesigtuc.ll
index 2886130df..2aca2dd2b 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesigtuc.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesigtuc.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -relocation-model=pic -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
@@ -5,13 +6,14 @@
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 
-@glob = common local_unnamed_addr global i8 0, align 1
+@glob = local_unnamed_addr global i8 0, align 1
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_igtuc(i8 zeroext %a, i8 zeroext %b) {
 ; CHECK-LABEL: test_igtuc:
-; CHECK:    sub [[REG:r[0-9]+]], r4, r3
-; CHECK-NEXT:    rldicl r3, [[REG]], 1, 63
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    sub r3, r4, r3
+; CHECK-NEXT:    rldicl r3, r3, 1, 63
 ; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ugt i8 %a, %b
@@ -22,8 +24,9 @@ entry:
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_igtuc_sext(i8 zeroext %a, i8 zeroext %b) {
 ; CHECK-LABEL: test_igtuc_sext:
-; CHECK:    sub [[REG:r[0-9]+]], r4, r3
-; CHECK-NEXT:    sradi r3, [[REG]], 63
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    sub r3, r4, r3
+; CHECK-NEXT:    sradi r3, r3, 63
 ; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ugt i8 %a, %b
@@ -34,7 +37,8 @@ entry:
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_igtuc_z(i8 zeroext %a) {
 ; CHECK-LABEL: test_igtuc_z:
-; CHECK:    cntlzw r3, r3
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cntlzw r3, r3
 ; CHECK-NEXT:    srwi r3, r3, 5
 ; CHECK-NEXT:    xori r3, r3, 1
 ; CHECK-NEXT:    blr
@@ -47,7 +51,8 @@ entry:
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_igtuc_sext_z(i8 zeroext %a) {
 ; CHECK-LABEL: test_igtuc_sext_z:
-; CHECK:    cntlzw r3, r3
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cntlzw r3, r3
 ; CHECK-NEXT:    srwi r3, r3, 5
 ; CHECK-NEXT:    xori r3, r3, 1
 ; CHECK-NEXT:    neg r3, r3
@@ -61,8 +66,13 @@ entry:
 ; Function Attrs: norecurse nounwind
 define void @test_igtuc_store(i8 zeroext %a, i8 zeroext %b) {
 ; CHECK-LABEL: test_igtuc_store:
-; CHECK:         sub [[REG:r[0-9]+]], r4, r3
-; CHECK:         rldicl {{r[0-9]+}}, [[REG]], 1, 63
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addis r5, r2, .LC0@toc@ha
+; CHECK-NEXT:    sub r3, r4, r3
+; CHECK-NEXT:    ld r5, .LC0@toc@l(r5)
+; CHECK-NEXT:    rldicl r3, r3, 1, 63
+; CHECK-NEXT:    stb r3, 0(r5)
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ugt i8 %a, %b
   %conv3 = zext i1 %cmp to i8
@@ -73,8 +83,13 @@ entry:
 ; Function Attrs: norecurse nounwind
 define void @test_igtuc_sext_store(i8 zeroext %a, i8 zeroext %b) {
 ; CHECK-LABEL: test_igtuc_sext_store:
-; CHECK:         sub [[REG:r[0-9]+]], r4, r3
-; CHECK:         sradi {{r[0-9]+}}, [[REG]], 63
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addis r5, r2, .LC0@toc@ha
+; CHECK-NEXT:    sub r3, r4, r3
+; CHECK-NEXT:    ld r5, .LC0@toc@l(r5)
+; CHECK-NEXT:    sradi r3, r3, 63
+; CHECK-NEXT:    stb r3, 0(r5)
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ugt i8 %a, %b
   %conv3 = sext i1 %cmp to i8
@@ -85,10 +100,13 @@ entry:
 ; Function Attrs: norecurse nounwind
 define void @test_igtuc_z_store(i8 zeroext %a) {
 ; CHECK-LABEL: test_igtuc_z_store:
-; CHECK:    cntlzw r3, r3
-; CHECK:    srwi r3, r3, 5
-; CHECK:    xori r3, r3, 1
-; CHECK:    stb r3, 0(r4)
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addis r4, r2, .LC0@toc@ha
+; CHECK-NEXT:    cntlzw r3, r3
+; CHECK-NEXT:    ld r4, .LC0@toc@l(r4)
+; CHECK-NEXT:    srwi r3, r3, 5
+; CHECK-NEXT:    xori r3, r3, 1
+; CHECK-NEXT:    stb r3, 0(r4)
 ; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ne i8 %a, 0
@@ -100,11 +118,14 @@ entry:
 ; Function Attrs: norecurse nounwind
 define void @test_igtuc_sext_z_store(i8 zeroext %a) {
 ; CHECK-LABEL: test_igtuc_sext_z_store:
-; CHECK:    cntlzw r3, r3
-; CHECK:    srwi r3, r3, 5
-; CHECK:    xori r3, r3, 1
-; CHECK:    neg r3, r3
-; CHECK:    stb r3, 0(r4)
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addis r4, r2, .LC0@toc@ha
+; CHECK-NEXT:    cntlzw r3, r3
+; CHECK-NEXT:    srwi r3, r3, 5
+; CHECK-NEXT:    ld r4, .LC0@toc@l(r4)
+; CHECK-NEXT:    xori r3, r3, 1
+; CHECK-NEXT:    neg r3, r3
+; CHECK-NEXT:    stb r3, 0(r4)
 ; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ne i8 %a, 0
diff --git a/llvm/test/CodeGen/PowerPC/testComparesigtui.ll b/llvm/test/CodeGen/PowerPC/testComparesigtui.ll
index a81a1a64f..aa1c2edc5 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesigtui.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesigtui.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -relocation-model=pic -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
@@ -5,13 +6,14 @@
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 
-@glob = common local_unnamed_addr global i32 0, align 4
+@glob = local_unnamed_addr global i32 0, align 4
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_igtui(i32 zeroext %a, i32 zeroext %b) {
 ; CHECK-LABEL: test_igtui:
-; CHECK:    sub [[REG:r[0-9]+]], r4, r3
-; CHECK-NEXT:    rldicl r3, [[REG]], 1, 63
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    sub r3, r4, r3
+; CHECK-NEXT:    rldicl r3, r3, 1, 63
 ; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ugt i32 %a, %b
@@ -22,8 +24,9 @@ entry:
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_igtui_sext(i32 zeroext %a, i32 zeroext %b) {
 ; CHECK-LABEL: test_igtui_sext:
-; CHECK:    sub [[REG:r[0-9]+]], r4, r3
-; CHECK-NEXT:    sradi r3, [[REG]], 63
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    sub r3, r4, r3
+; CHECK-NEXT:    sradi r3, r3, 63
 ; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ugt i32 %a, %b
@@ -34,7 +37,8 @@ entry:
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_igtui_z(i32 zeroext %a) {
 ; CHECK-LABEL: test_igtui_z:
-; CHECK:    cntlzw r3, r3
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cntlzw r3, r3
 ; CHECK-NEXT:    srwi r3, r3, 5
 ; CHECK-NEXT:    xori r3, r3, 1
 ; CHECK-NEXT:    blr
@@ -47,7 +51,8 @@ entry:
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_igtui_sext_z(i32 zeroext %a) {
 ; CHECK-LABEL: test_igtui_sext_z:
-; CHECK:    cntlzw r3, r3
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cntlzw r3, r3
 ; CHECK-NEXT:    srwi r3, r3, 5
 ; CHECK-NEXT:    xori r3, r3, 1
 ; CHECK-NEXT:    neg r3, r3
@@ -61,8 +66,13 @@ entry:
 ; Function Attrs: norecurse nounwind
 define void @test_igtui_store(i32 zeroext %a, i32 zeroext %b) {
 ; CHECK-LABEL: test_igtui_store:
-; CHECK:         sub [[REG:r[0-9]+]], r4, r3
-; CHECK:         rldicl {{r[0-9]+}}, [[REG]], 1, 63
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addis r5, r2, .LC0@toc@ha
+; CHECK-NEXT:    sub r3, r4, r3
+; CHECK-NEXT:    ld r5, .LC0@toc@l(r5)
+; CHECK-NEXT:    rldicl r3, r3, 1, 63
+; CHECK-NEXT:    stw r3, 0(r5)
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ugt i32 %a, %b
   %conv = zext i1 %cmp to i32
@@ -73,8 +83,13 @@ entry:
 ; Function Attrs: norecurse nounwind
 define void @test_igtui_sext_store(i32 zeroext %a, i32 zeroext %b) {
 ; CHECK-LABEL: test_igtui_sext_store:
-; CHECK:         sub [[REG:r[0-9]+]], r4, r3
-; CHECK:         sradi {{r[0-9]+}}, [[REG]], 63
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addis r5, r2, .LC0@toc@ha
+; CHECK-NEXT:    sub r3, r4, r3
+; CHECK-NEXT:    ld r5, .LC0@toc@l(r5)
+; CHECK-NEXT:    sradi r3, r3, 63
+; CHECK-NEXT:    stw r3, 0(r5)
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ugt i32 %a, %b
   %sub = sext i1 %cmp to i32
@@ -85,10 +100,13 @@ entry:
 ; Function Attrs: norecurse nounwind
 define void @test_igtui_z_store(i32 zeroext %a) {
 ; CHECK-LABEL: test_igtui_z_store:
-; CHECK:    cntlzw r3, r3
-; CHECK:    srwi r3, r3, 5
-; CHECK:    xori r3, r3, 1
-; CHECK:    stw r3, 0(r4)
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addis r4, r2, .LC0@toc@ha
+; CHECK-NEXT:    cntlzw r3, r3
+; CHECK-NEXT:    ld r4, .LC0@toc@l(r4)
+; CHECK-NEXT:    srwi r3, r3, 5
+; CHECK-NEXT:    xori r3, r3, 1
+; CHECK-NEXT:    stw r3, 0(r4)
 ; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ne i32 %a, 0
@@ -100,11 +118,14 @@ entry:
 ; Function Attrs: norecurse nounwind
 define void @test_igtui_sext_z_store(i32 zeroext %a) {
 ; CHECK-LABEL: test_igtui_sext_z_store:
-; CHECK:    cntlzw r3, r3
-; CHECK:    srwi r3, r3, 5
-; CHECK:    xori r3, r3, 1
-; CHECK:    neg r3, r3
-; CHECK:    stw r3, 0(r4)
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addis r4, r2, .LC0@toc@ha
+; CHECK-NEXT:    cntlzw r3, r3
+; CHECK-NEXT:    srwi r3, r3, 5
+; CHECK-NEXT:    ld r4, .LC0@toc@l(r4)
+; CHECK-NEXT:    xori r3, r3, 1
+; CHECK-NEXT:    neg r3, r3
+; CHECK-NEXT:    stw r3, 0(r4)
 ; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ne i32 %a, 0
diff --git a/llvm/test/CodeGen/PowerPC/testComparesigtus.ll b/llvm/test/CodeGen/PowerPC/testComparesigtus.ll
index 7beca852c..877753c1a 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesigtus.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesigtus.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -relocation-model=pic -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
@@ -5,13 +6,14 @@
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 
-@glob = common local_unnamed_addr global i16 0, align 2
+@glob = local_unnamed_addr global i16 0, align 2
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_igtus(i16 zeroext %a, i16 zeroext %b) {
 ; CHECK-LABEL: test_igtus:
-; CHECK:    sub [[REG:r[0-9]+]], r4, r3
-; CHECK-NEXT:    rldicl r3, [[REG]], 1, 63
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    sub r3, r4, r3
+; CHECK-NEXT:    rldicl r3, r3, 1, 63
 ; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ugt i16 %a, %b
@@ -22,8 +24,9 @@ entry:
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_igtus_sext(i16 zeroext %a, i16 zeroext %b) {
 ; CHECK-LABEL: test_igtus_sext:
-; CHECK:    sub [[REG:r[0-9]+]], r4, r3
-; CHECK-NEXT:    sradi r3, [[REG]], 63
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    sub r3, r4, r3
+; CHECK-NEXT:    sradi r3, r3, 63
 ; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ugt i16 %a, %b
@@ -34,7 +37,8 @@ entry:
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_igtus_z(i16 zeroext %a) {
 ; CHECK-LABEL: test_igtus_z:
-; CHECK:    cntlzw r3, r3
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cntlzw r3, r3
 ; CHECK-NEXT:    srwi r3, r3, 5
 ; CHECK-NEXT:    xori r3, r3, 1
 ; CHECK-NEXT:    blr
@@ -47,7 +51,8 @@ entry:
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_igtus_sext_z(i16 zeroext %a) {
 ; CHECK-LABEL: test_igtus_sext_z:
-; CHECK:    cntlzw r3, r3
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cntlzw r3, r3
 ; CHECK-NEXT:    srwi r3, r3, 5
 ; CHECK-NEXT:    xori r3, r3, 1
 ; CHECK-NEXT:    neg r3, r3
@@ -61,9 +66,13 @@ entry:
 ; Function Attrs: norecurse nounwind
 define void @test_igtus_store(i16 zeroext %a, i16 zeroext %b) {
 ; CHECK-LABEL: test_igtus_store:
-; CHECK:         sub [[REG:r[0-9]+]], r4, r3
-; CHECK:         rldicl {{r[0-9]+}}, [[REG]], 1, 63
-; CHECK:         blr
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addis r5, r2, .LC0@toc@ha
+; CHECK-NEXT:    sub r3, r4, r3
+; CHECK-NEXT:    ld r5, .LC0@toc@l(r5)
+; CHECK-NEXT:    rldicl r3, r3, 1, 63
+; CHECK-NEXT:    sth r3, 0(r5)
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ugt i16 %a, %b
   %conv3 = zext i1 %cmp to i16
@@ -74,9 +83,13 @@ entry:
 ; Function Attrs: norecurse nounwind
 define void @test_igtus_sext_store(i16 zeroext %a, i16 zeroext %b) {
 ; CHECK-LABEL: test_igtus_sext_store:
-; CHECK:         sub [[REG:r[0-9]+]], r4, r3
-; CHECK:         sradi {{r[0-9]+}}, [[REG]], 63
-; CHECK:         blr
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addis r5, r2, .LC0@toc@ha
+; CHECK-NEXT:    sub r3, r4, r3
+; CHECK-NEXT:    ld r5, .LC0@toc@l(r5)
+; CHECK-NEXT:    sradi r3, r3, 63
+; CHECK-NEXT:    sth r3, 0(r5)
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ugt i16 %a, %b
   %conv3 = sext i1 %cmp to i16
@@ -87,10 +100,13 @@ entry:
 ; Function Attrs: norecurse nounwind
 define void @test_igtus_z_store(i16 zeroext %a) {
 ; CHECK-LABEL: test_igtus_z_store:
-; CHECK:    cntlzw r3, r3
-; CHECK:    srwi r3, r3, 5
-; CHECK:    xori r3, r3, 1
-; CHECK:    sth r3, 0(r4)
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addis r4, r2, .LC0@toc@ha
+; CHECK-NEXT:    cntlzw r3, r3
+; CHECK-NEXT:    ld r4, .LC0@toc@l(r4)
+; CHECK-NEXT:    srwi r3, r3, 5
+; CHECK-NEXT:    xori r3, r3, 1
+; CHECK-NEXT:    sth r3, 0(r4)
 ; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ne i16 %a, 0
@@ -102,11 +118,14 @@ entry:
 ; Function Attrs: norecurse nounwind
 define void @test_igtus_sext_z_store(i16 zeroext %a) {
 ; CHECK-LABEL: test_igtus_sext_z_store:
-; CHECK:    cntlzw r3, r3
-; CHECK:    srwi r3, r3, 5
-; CHECK:    xori r3, r3, 1
-; CHECK:    neg r3, r3
-; CHECK:    sth r3, 0(r4)
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addis r4, r2, .LC0@toc@ha
+; CHECK-NEXT:    cntlzw r3, r3
+; CHECK-NEXT:    srwi r3, r3, 5
+; CHECK-NEXT:    ld r4, .LC0@toc@l(r4)
+; CHECK-NEXT:    xori r3, r3, 1
+; CHECK-NEXT:    neg r3, r3
+; CHECK-NEXT:    sth r3, 0(r4)
 ; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ne i16 %a, 0
diff --git a/llvm/test/CodeGen/PowerPC/testComparesilesc.ll b/llvm/test/CodeGen/PowerPC/testComparesilesc.ll
index 94118d642..bd5a16fb5 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesilesc.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesilesc.ll
@@ -5,7 +5,7 @@
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s --check-prefix=CHECK-LE \
 ; RUN:   --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
-@glob = common local_unnamed_addr global i8 0, align 1
+@glob = local_unnamed_addr global i8 0, align 1
 
 define signext i32 @test_ilesc(i8 signext %a, i8 signext %b) {
 ; CHECK-LABEL: test_ilesc:
diff --git a/llvm/test/CodeGen/PowerPC/testComparesilesi.ll b/llvm/test/CodeGen/PowerPC/testComparesilesi.ll
index a5f2f84a5..8717e237e 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesilesi.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesilesi.ll
@@ -5,7 +5,7 @@
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s --check-prefix=CHECK-LE \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
-@glob = common local_unnamed_addr global i32 0, align 4
+@glob = local_unnamed_addr global i32 0, align 4
 
 define signext i32 @test_ilesi(i32 signext %a, i32 signext %b) {
 ; CHECK-LABEL: test_ilesi:
diff --git a/llvm/test/CodeGen/PowerPC/testComparesilesll.ll b/llvm/test/CodeGen/PowerPC/testComparesilesll.ll
index 532df4c3c..b1827661a 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesilesll.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesilesll.ll
@@ -5,7 +5,7 @@
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s --check-prefix=CHECK-LE \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
-@glob = common local_unnamed_addr global i64 0, align 8
+@glob = local_unnamed_addr global i64 0, align 8
 
 define signext i32 @test_ilesll(i64 %a, i64 %b) {
 ; CHECK-LABEL: test_ilesll:
diff --git a/llvm/test/CodeGen/PowerPC/testComparesiless.ll b/llvm/test/CodeGen/PowerPC/testComparesiless.ll
index 45c887d21..b9b029496 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesiless.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesiless.ll
@@ -5,7 +5,7 @@
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s --check-prefix=CHECK-LE \
 ; RUN:   --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
-@glob = common local_unnamed_addr global i16 0, align 2
+@glob = local_unnamed_addr global i16 0, align 2
 
 define signext i32 @test_iless(i16 signext %a, i16 signext %b) {
 ; CHECK-LABEL: test_iless:
diff --git a/llvm/test/CodeGen/PowerPC/testComparesileuc.ll b/llvm/test/CodeGen/PowerPC/testComparesileuc.ll
index 3084b7ee9..4461fb66c 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesileuc.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesileuc.ll
@@ -1,117 +1,173 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
-; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
+; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
+; RUN:  --check-prefixes=CHECK,BE
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
-; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
+; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
+; RUN:  --check-prefixes=CHECK,LE
 
-@glob = common local_unnamed_addr global i8 0, align 1
+@glob = local_unnamed_addr global i8 0, align 1
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_ileuc(i8 zeroext %a, i8 zeroext %b) {
+; CHECK-LABEL: test_ileuc:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    sub r3, r4, r3
+; CHECK-NEXT:    not r3, r3
+; CHECK-NEXT:    rldicl r3, r3, 1, 63
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ule i8 %a, %b
   %conv2 = zext i1 %cmp to i32
   ret i32 %conv2
-; CHECK-LABEL: test_ileuc:
-; CHECK: sub [[REG1:r[0-9]+]], r4, r3
-; CHECK-NEXT: not [[REG2:r[0-9]+]], [[REG1]]
-; CHECK-NEXT: rldicl r3, [[REG2]], 1, 63
-; CHECK: blr
 }
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_ileuc_sext(i8 zeroext %a, i8 zeroext %b) {
+; CHECK-LABEL: test_ileuc_sext:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    sub r3, r4, r3
+; CHECK-NEXT:    rldicl r3, r3, 1, 63
+; CHECK-NEXT:    addi r3, r3, -1
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ule i8 %a, %b
   %sub = sext i1 %cmp to i32
   ret i32 %sub
-; CHECK-LABEL: @test_ileuc_sext
-; CHECK: sub [[REG1:r[0-9]+]], r4, r3
-; CHECK-NEXT: rldicl [[REG2:r[0-9]+]], [[REG1]], 1, 63
-; CHECK-NEXT: addi [[REG3:r[0-9]+]], [[REG2]], -1
-; CHECK-NEXT: blr
 }
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_ileuc_z(i8 zeroext %a) {
+; CHECK-LABEL: test_ileuc_z:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cntlzw r3, r3
+; CHECK-NEXT:    srwi r3, r3, 5
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp eq i8 %a, 0
   %conv1 = zext i1 %cmp to i32
   ret i32 %conv1
-; CHECK-LABEL: test_ileuc_z:
-; CHECK: cntlzw [[REG1:r[0-9]+]], r3
-; CHECK: srwi r3, [[REG1]], 5
-; CHECK: blr
 }
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_ileuc_sext_z(i8 zeroext %a) {
+; CHECK-LABEL: test_ileuc_sext_z:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cntlzw r3, r3
+; CHECK-NEXT:    srwi r3, r3, 5
+; CHECK-NEXT:    neg r3, r3
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ule i8 %a, 0
   %sub = sext i1 %cmp to i32
   ret i32 %sub
-; CHECK-LABEL: @test_ileuc_sext_z
-; CHECK: cntlzw [[REG1:r[0-9]+]], r3
-; CHECK-NEXT: srwi [[REG2:r[0-9]+]], [[REG1]], 5
-; CHECK-NEXT: neg r3, [[REG2]]
-; CHECK-NEXT: blr
 }
 
 ; Function Attrs: norecurse nounwind
 define void @test_ileuc_store(i8 zeroext %a, i8 zeroext %b) {
+; BE-LABEL: test_ileuc_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r5, r2, .LC0@toc@ha
+; BE-NEXT:    sub r3, r4, r3
+; BE-NEXT:    ld r4, .LC0@toc@l(r5)
+; BE-NEXT:    not r3, r3
+; BE-NEXT:    rldicl r3, r3, 1, 63
+; BE-NEXT:    stb r3, 0(r4)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_ileuc_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    sub r3, r4, r3
+; LE-NEXT:    addis r5, r2, glob@toc@ha
+; LE-NEXT:    not r3, r3
+; LE-NEXT:    rldicl r3, r3, 1, 63
+; LE-NEXT:    stb r3, glob@toc@l(r5)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp ule i8 %a, %b
   %conv3 = zext i1 %cmp to i8
   store i8 %conv3, i8* @glob
   ret void
-; CHECK-LABEL: test_ileuc_store:
-; CHECK: sub [[REG1:r[0-9]+]], r4, r3
-; CHECK: not [[REG2:r[0-9]+]], [[REG1]]
-; CHECK-NEXT: rldicl r3, [[REG2]], 1, 63
-; CHECK: blr
 }
 
 ; Function Attrs: norecurse nounwind
 define void @test_ileuc_sext_store(i8 zeroext %a, i8 zeroext %b) {
+; BE-LABEL: test_ileuc_sext_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r5, r2, .LC0@toc@ha
+; BE-NEXT:    sub r3, r4, r3
+; BE-NEXT:    ld r4, .LC0@toc@l(r5)
+; BE-NEXT:    rldicl r3, r3, 1, 63
+; BE-NEXT:    addi r3, r3, -1
+; BE-NEXT:    stb r3, 0(r4)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_ileuc_sext_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    sub r3, r4, r3
+; LE-NEXT:    addis r5, r2, glob@toc@ha
+; LE-NEXT:    rldicl r3, r3, 1, 63
+; LE-NEXT:    addi r3, r3, -1
+; LE-NEXT:    stb r3, glob@toc@l(r5)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp ule i8 %a, %b
   %conv3 = sext i1 %cmp to i8
   store i8 %conv3, i8* @glob
   ret void
-; CHECK-LABEL: @test_ileuc_sext_store
-; CHECK: sub [[REG1:r[0-9]+]], r4, r3
-; CHECK: rldicl [[REG2:r[0-9]+]], [[REG1]], 1, 63
-; CHECK: addi [[REG3:r[0-9]+]], [[REG2]], -1
-; CHECK: stb  [[REG3]]
-; CHECK: blr
 }
 
 ; Function Attrs: norecurse nounwind
 define void @test_ileuc_z_store(i8 zeroext %a) {
+; BE-LABEL: test_ileuc_z_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r4, r2, .LC0@toc@ha
+; BE-NEXT:    cntlzw r3, r3
+; BE-NEXT:    ld r4, .LC0@toc@l(r4)
+; BE-NEXT:    srwi r3, r3, 5
+; BE-NEXT:    stb r3, 0(r4)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_ileuc_z_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    cntlzw r3, r3
+; LE-NEXT:    addis r4, r2, glob@toc@ha
+; LE-NEXT:    srwi r3, r3, 5
+; LE-NEXT:    stb r3, glob@toc@l(r4)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp eq i8 %a, 0
   %conv2 = zext i1 %cmp to i8
   store i8 %conv2, i8* @glob
   ret void
-; CHECK-LABEL: test_ileuc_z_store:
-; CHECK: cntlzw [[REG1:r[0-9]+]], r3
-; CHECK: srwi {{r[0-9]+}}, [[REG1]], 5
-; CHECK: blr
 }
 
 ; Function Attrs: norecurse nounwind
 define void @test_ileuc_sext_z_store(i8 zeroext %a) {
+; BE-LABEL: test_ileuc_sext_z_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r4, r2, .LC0@toc@ha
+; BE-NEXT:    cntlzw r3, r3
+; BE-NEXT:    ld r4, .LC0@toc@l(r4)
+; BE-NEXT:    srwi r3, r3, 5
+; BE-NEXT:    neg r3, r3
+; BE-NEXT:    stb r3, 0(r4)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_ileuc_sext_z_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    cntlzw r3, r3
+; LE-NEXT:    addis r4, r2, glob@toc@ha
+; LE-NEXT:    srwi r3, r3, 5
+; LE-NEXT:    neg r3, r3
+; LE-NEXT:    stb r3, glob@toc@l(r4)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp eq i8 %a, 0
   %conv2 = sext i1 %cmp to i8
   store i8 %conv2, i8* @glob
   ret void
-; CHECK-LABEL: @test_ileuc_sext_z_store
-; CHECK: cntlzw [[REG1:r[0-9]+]], r3
-; CHECK: srwi [[REG2:r[0-9]+]], [[REG1]], 5
-; CHECK: neg [[REG3:r[0-9]+]], [[REG2]]
-; CHECK: stb [[REG3]]
-; CHECK: blr
 }
 
diff --git a/llvm/test/CodeGen/PowerPC/testComparesileui.ll b/llvm/test/CodeGen/PowerPC/testComparesileui.ll
index addaa9b99..ad6418c10 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesileui.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesileui.ll
@@ -1,117 +1,173 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
-; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
+; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
+; RUN:  --check-prefixes=CHECK,BE
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
-; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
+; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
+; RUN:  --check-prefixes=CHECK,LE
 
-@glob = common local_unnamed_addr global i32 0, align 4
+@glob = local_unnamed_addr global i32 0, align 4
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_ileui(i32 zeroext %a, i32 zeroext %b) {
+; CHECK-LABEL: test_ileui:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    sub r3, r4, r3
+; CHECK-NEXT:    not r3, r3
+; CHECK-NEXT:    rldicl r3, r3, 1, 63
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ule i32 %a, %b
   %sub = zext i1 %cmp to i32
   ret i32 %sub
-; CHECK-LABEL: test_ileui:
-; CHECK: sub [[REG1:r[0-9]+]], r4, r3
-; CHECK-NEXT: not [[REG2:r[0-9]+]], [[REG1]]
-; CHECK-NEXT: rldicl r3, [[REG2]], 1, 63
-; CHECK: blr
 }
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_ileui_sext(i32 zeroext %a, i32 zeroext %b) {
+; CHECK-LABEL: test_ileui_sext:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    sub r3, r4, r3
+; CHECK-NEXT:    rldicl r3, r3, 1, 63
+; CHECK-NEXT:    addi r3, r3, -1
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ule i32 %a, %b
   %sub = sext i1 %cmp to i32
   ret i32 %sub
-; CHECK-LABEL: @test_ileui_sext
-; CHECK: sub [[REG1:r[0-9]+]], r4, r3
-; CHECK-NEXT: rldicl [[REG2:r[0-9]+]], [[REG1]], 1, 63
-; CHECK-NEXT: addi [[REG3:r[0-9]+]], [[REG2]], -1
-; CHECK-NEXT: blr
 }
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_ileui_z(i32 zeroext %a) {
+; CHECK-LABEL: test_ileui_z:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cntlzw r3, r3
+; CHECK-NEXT:    srwi r3, r3, 5
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp eq i32 %a, 0
   %sub = zext i1 %cmp to i32
   ret i32 %sub
-; CHECK-LABEL: test_ileui_z:
-; CHECK: cntlzw [[REG1:r[0-9]+]], r3
-; CHECK: srwi r3, [[REG1]], 5
-; CHECK: blr
 }
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_ileui_sext_z(i32 zeroext %a) {
+; CHECK-LABEL: test_ileui_sext_z:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cntlzw r3, r3
+; CHECK-NEXT:    srwi r3, r3, 5
+; CHECK-NEXT:    neg r3, r3
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp eq i32 %a, 0
   %sub = sext i1 %cmp to i32
   ret i32 %sub
-; CHECK-LABEL: @test_ileui_sext_z
-; CHECK: cntlzw [[REG1:r[0-9]+]], r3
-; CHECK-NEXT: srwi [[REG2:r[0-9]+]], [[REG1]], 5
-; CHECK-NEXT: neg r3, [[REG2]]
-; CHECK: blr
 }
 
 ; Function Attrs: norecurse nounwind
 define void @test_ileui_store(i32 zeroext %a, i32 zeroext %b) {
+; BE-LABEL: test_ileui_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r5, r2, .LC0@toc@ha
+; BE-NEXT:    sub r3, r4, r3
+; BE-NEXT:    ld r4, .LC0@toc@l(r5)
+; BE-NEXT:    not r3, r3
+; BE-NEXT:    rldicl r3, r3, 1, 63
+; BE-NEXT:    stw r3, 0(r4)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_ileui_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    sub r3, r4, r3
+; LE-NEXT:    addis r5, r2, glob@toc@ha
+; LE-NEXT:    not r3, r3
+; LE-NEXT:    rldicl r3, r3, 1, 63
+; LE-NEXT:    stw r3, glob@toc@l(r5)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp ule i32 %a, %b
   %sub = zext i1 %cmp to i32
   store i32 %sub, i32* @glob
   ret void
-; CHECK-LABEL: test_ileui_store:
-; CHECK: sub [[REG1:r[0-9]+]], r4, r3
-; CHECK: not [[REG2:r[0-9]+]], [[REG1]]
-; CHECK-NEXT: rldicl r3, [[REG2]], 1, 63
-; CHECK: blr
 }
 
 ; Function Attrs: norecurse nounwind
 define void @test_ileui_sext_store(i32 zeroext %a, i32 zeroext %b) {
+; BE-LABEL: test_ileui_sext_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r5, r2, .LC0@toc@ha
+; BE-NEXT:    sub r3, r4, r3
+; BE-NEXT:    ld r4, .LC0@toc@l(r5)
+; BE-NEXT:    rldicl r3, r3, 1, 63
+; BE-NEXT:    addi r3, r3, -1
+; BE-NEXT:    stw r3, 0(r4)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_ileui_sext_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    sub r3, r4, r3
+; LE-NEXT:    addis r5, r2, glob@toc@ha
+; LE-NEXT:    rldicl r3, r3, 1, 63
+; LE-NEXT:    addi r3, r3, -1
+; LE-NEXT:    stw r3, glob@toc@l(r5)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp ule i32 %a, %b
   %sub = sext i1 %cmp to i32
   store i32 %sub, i32* @glob
   ret void
-; CHECK-LABEL: @test_ileui_sext_store
-; CHECK: sub [[REG1:r[0-9]+]], r4, r3
-; CHECK: rldicl [[REG2:r[0-9]+]], [[REG1]], 1, 63
-; CHECK: addi [[REG3:r[0-9]+]], [[REG2]], -1
-; CHECK: stw  [[REG3]]
-; CHECK: blr
 }
 
 ; Function Attrs: norecurse nounwind
 define void @test_ileui_z_store(i32 zeroext %a) {
+; BE-LABEL: test_ileui_z_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r4, r2, .LC0@toc@ha
+; BE-NEXT:    cntlzw r3, r3
+; BE-NEXT:    ld r4, .LC0@toc@l(r4)
+; BE-NEXT:    srwi r3, r3, 5
+; BE-NEXT:    stw r3, 0(r4)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_ileui_z_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    cntlzw r3, r3
+; LE-NEXT:    addis r4, r2, glob@toc@ha
+; LE-NEXT:    srwi r3, r3, 5
+; LE-NEXT:    stw r3, glob@toc@l(r4)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp eq i32 %a, 0
   %sub = zext i1 %cmp to i32
   store i32 %sub, i32* @glob
   ret void
-; CHECK-LABEL: test_ileui_z_store:
-; CHECK: cntlzw [[REG1:r[0-9]+]], r3
-; CHECK: srwi {{r[0-9]+}}, [[REG1]], 5
-; CHECK: blr
 }
 
 ; Function Attrs: norecurse nounwind
 define void @test_ileui_sext_z_store(i32 zeroext %a) {
+; BE-LABEL: test_ileui_sext_z_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r4, r2, .LC0@toc@ha
+; BE-NEXT:    cntlzw r3, r3
+; BE-NEXT:    ld r4, .LC0@toc@l(r4)
+; BE-NEXT:    srwi r3, r3, 5
+; BE-NEXT:    neg r3, r3
+; BE-NEXT:    stw r3, 0(r4)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_ileui_sext_z_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    cntlzw r3, r3
+; LE-NEXT:    addis r4, r2, glob@toc@ha
+; LE-NEXT:    srwi r3, r3, 5
+; LE-NEXT:    neg r3, r3
+; LE-NEXT:    stw r3, glob@toc@l(r4)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp eq i32 %a, 0
   %sub = sext i1 %cmp to i32
   store i32 %sub, i32* @glob
   ret void
-; CHECK-LABEL: @test_ileui_sext_z_store
-; CHECK: cntlzw [[REG1:r[0-9]+]], r3
-; CHECK: srwi [[REG2:r[0-9]+]], [[REG1]], 5
-; CHECK: neg [[REG3:r[0-9]+]], [[REG2]]
-; CHECK: stw [[REG3]]
-; CHECK: blr
 }
 
diff --git a/llvm/test/CodeGen/PowerPC/testComparesileull.ll b/llvm/test/CodeGen/PowerPC/testComparesileull.ll
index afcf0ce4b..54ee2561a 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesileull.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesileull.ll
@@ -1,115 +1,170 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
-; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
+; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
+; RUN:  --check-prefixes=CHECK,BE
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
-; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
+; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
+; RUN:  --check-prefixes=CHECK,LE
 
-@glob = common local_unnamed_addr global i64 0, align 8
+@glob = local_unnamed_addr global i64 0, align 8
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_ileull(i64 %a, i64 %b) {
+; CHECK-LABEL: test_ileull:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    subfc r4, r3, r4
+; CHECK-NEXT:    subfe r3, r3, r3
+; CHECK-NEXT:    addi r3, r3, 1
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ule i64 %a, %b
   %conv = zext i1 %cmp to i32
   ret i32 %conv
-; CHECK-LABEL: test_ileull:
-; CHECK: subfc {{r[0-9]+}}, r3, r4
-; CHECK-NEXT: subfe [[REG1:r[0-9]+]], r3, r3
-; CHECK-NEXT: addi r3, [[REG1]], 1
-; CHECK-NEXT: blr
 }
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_ileull_sext(i64 %a, i64 %b) {
+; CHECK-LABEL: test_ileull_sext:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    subfc r4, r3, r4
+; CHECK-NEXT:    subfe r3, r3, r3
+; CHECK-NEXT:    not r3, r3
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ule i64 %a, %b
   %sub = sext i1 %cmp to i32
   ret i32 %sub
-; CHECK-LABEL: @test_ileull_sext
-; CHECK: subfc [[REG1:r[0-9]+]], r3, r4
-; CHECK: subfe [[REG2:r[0-9]+]], {{r[0-9]+}}, {{r[0-9]+}}
-; CHECK: not r3, [[REG2]]
-; CHECK: blr
 }
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_ileull_z(i64 %a) {
+; CHECK-LABEL: test_ileull_z:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cntlzd r3, r3
+; CHECK-NEXT:    rldicl r3, r3, 58, 63
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ule i64 %a, 0
   %conv = zext i1 %cmp to i32
   ret i32 %conv
-; CHECK-LABEL: test_ileull_z
-; CHECK: cntlzd [[REG1:r[0-9]+]], r3
-; CHECK-NEXT: rldicl r3, [[REG1]], 58, 63
-; CHECK-NEXT: blr
 }
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_ileull_sext_z(i64 %a) {
+; CHECK-LABEL: test_ileull_sext_z:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addic r3, r3, -1
+; CHECK-NEXT:    subfe r3, r3, r3
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ule i64 %a, 0
   %sub = sext i1 %cmp to i32
   ret i32 %sub
-; CHECK-LABEL: @test_ileull_sext_z
-; CHECK: addic [[REG1:r[0-9]+]], r3, -1
-; CHECK: subfe r3, [[REG1]]
-; CHECK: blr
 }
 
 ; Function Attrs: norecurse nounwind
 define void @test_ileull_store(i64 %a, i64 %b) {
+; BE-LABEL: test_ileull_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r5, r2, .LC0@toc@ha
+; BE-NEXT:    subfc r4, r3, r4
+; BE-NEXT:    ld r4, .LC0@toc@l(r5)
+; BE-NEXT:    subfe r3, r3, r3
+; BE-NEXT:    addi r3, r3, 1
+; BE-NEXT:    std r3, 0(r4)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_ileull_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    subfc r4, r3, r4
+; LE-NEXT:    addis r5, r2, glob@toc@ha
+; LE-NEXT:    subfe r3, r3, r3
+; LE-NEXT:    addi r3, r3, 1
+; LE-NEXT:    std r3, glob@toc@l(r5)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp ule i64 %a, %b
   %conv1 = zext i1 %cmp to i64
   store i64 %conv1, i64* @glob
   ret void
-; CHECK-LABEL: test_ileull_store:
-; CHECK: subfc {{r[0-9]+}}, r3, r4
-; CHECK: subfe [[REG1:r[0-9]+]], r3, r3
-; CHECK: addi {{r[0-9]+}}, [[REG1]], 1
-; CHECK: blr
 }
 
 ; Function Attrs: norecurse nounwind
 define void @test_ileull_sext_store(i64 %a, i64 %b) {
+; BE-LABEL: test_ileull_sext_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r5, r2, .LC0@toc@ha
+; BE-NEXT:    subfc r4, r3, r4
+; BE-NEXT:    ld r4, .LC0@toc@l(r5)
+; BE-NEXT:    subfe r3, r3, r3
+; BE-NEXT:    not r3, r3
+; BE-NEXT:    std r3, 0(r4)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_ileull_sext_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    subfc r4, r3, r4
+; LE-NEXT:    addis r5, r2, glob@toc@ha
+; LE-NEXT:    subfe r3, r3, r3
+; LE-NEXT:    not r3, r3
+; LE-NEXT:    std r3, glob@toc@l(r5)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp ule i64 %a, %b
   %conv1 = sext i1 %cmp to i64
   store i64 %conv1, i64* @glob
   ret void
-; CHECK-LABEL: @test_ileull_sext_store
-; CHECK: subfc [[REG1:r[0-9]+]], r3, r4
-; CHECK: subfe [[REG2:r[0-9]+]], {{r[0-9]+}}, {{r[0-9]+}}
-; CHECK: not [[REG3:r[0-9]+]], [[REG2]]
-; CHECK: std [[REG3]]
-; CHECK: blr
 }
 
 ; Function Attrs: norecurse nounwind
 define void @test_ileull_z_store(i64 %a) {
+; BE-LABEL: test_ileull_z_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r4, r2, .LC0@toc@ha
+; BE-NEXT:    cntlzd r3, r3
+; BE-NEXT:    ld r4, .LC0@toc@l(r4)
+; BE-NEXT:    rldicl r3, r3, 58, 63
+; BE-NEXT:    std r3, 0(r4)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_ileull_z_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    cntlzd r3, r3
+; LE-NEXT:    addis r4, r2, glob@toc@ha
+; LE-NEXT:    rldicl r3, r3, 58, 63
+; LE-NEXT:    std r3, glob@toc@l(r4)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp ule i64 %a, 0
   %conv1 = zext i1 %cmp to i64
   store i64 %conv1, i64* @glob
   ret void
-; CHECK-LABEL: test_ileull_z_store:
-; CHECK: cntlzd [[REG1:r[0-9]+]], r3
-; CHECK: rldicl {{r[0-9]+}}, [[REG1]], 58, 63
-; CHECK: blr
 }
 
 ; Function Attrs: norecurse nounwind
 define void @test_ileull_sext_z_store(i64 %a) {
+; BE-LABEL: test_ileull_sext_z_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r4, r2, .LC0@toc@ha
+; BE-NEXT:    addic r3, r3, -1
+; BE-NEXT:    ld r4, .LC0@toc@l(r4)
+; BE-NEXT:    subfe r3, r3, r3
+; BE-NEXT:    std r3, 0(r4)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_ileull_sext_z_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    addic r3, r3, -1
+; LE-NEXT:    addis r4, r2, glob@toc@ha
+; LE-NEXT:    subfe r3, r3, r3
+; LE-NEXT:    std r3, glob@toc@l(r4)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp ule i64 %a, 0
   %conv1 = sext i1 %cmp to i64
   store i64 %conv1, i64* @glob
   ret void
-; CHECK-LABEL: @test_ileull_sext_z_store
-; CHECK: addic [[REG1:r[0-9]+]], r3, -1
-; CHECK: subfe [[REG2:r[0-9]+]], {{r[0-9]+}}, {{r[0-9]+}}
-; CHECK: std [[REG2]]
-; CHECK: blr
 }
 
diff --git a/llvm/test/CodeGen/PowerPC/testComparesileus.ll b/llvm/test/CodeGen/PowerPC/testComparesileus.ll
index 122d46f24..e4b68bb03 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesileus.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesileus.ll
@@ -1,117 +1,173 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
-; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
+; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
+; RUN:  --check-prefixes=CHECK,BE
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
-; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
+; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
+; RUN:  --check-prefixes=CHECK,LE
 
-@glob = common local_unnamed_addr global i16 0, align 2
+@glob = local_unnamed_addr global i16 0, align 2
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_ileus(i16 zeroext %a, i16 zeroext %b) {
+; CHECK-LABEL: test_ileus:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    sub r3, r4, r3
+; CHECK-NEXT:    not r3, r3
+; CHECK-NEXT:    rldicl r3, r3, 1, 63
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ule i16 %a, %b
   %conv2 = zext i1 %cmp to i32
   ret i32 %conv2
-; CHECK-LABEL: test_ileus:
-; CHECK: sub [[REG1:r[0-9]+]], r4, r3
-; CHECK-NEXT: not [[REG2:r[0-9]+]], [[REG1]]
-; CHECK-NEXT: rldicl r3, [[REG2]], 1, 63
-; CHECK-NEXT: blr
 }
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_ileus_sext(i16 zeroext %a, i16 zeroext %b) {
+; CHECK-LABEL: test_ileus_sext:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    sub r3, r4, r3
+; CHECK-NEXT:    rldicl r3, r3, 1, 63
+; CHECK-NEXT:    addi r3, r3, -1
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ule i16 %a, %b
   %sub = sext i1 %cmp to i32
   ret i32 %sub
-; CHECK-LABEL: @test_ileus_sext
-; CHECK: sub [[REG1:r[0-9]+]], r4, r3
-; CHECK-NEXT: rldicl [[REG2:r[0-9]+]], [[REG1]], 1, 63
-; CHECK-NEXT: addi [[REG3:r[0-9]+]], [[REG2]], -1
-; CHECK-NEXT: blr
 }
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_ileus_z(i16 zeroext %a) {
+; CHECK-LABEL: test_ileus_z:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cntlzw r3, r3
+; CHECK-NEXT:    srwi r3, r3, 5
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ule i16 %a, 0
   %conv1 = zext i1 %cmp to i32
   ret i32 %conv1
-; CHECK-LABEL: test_ileus_z:
-; CHECK: cntlzw [[REG1:r[0-9]+]], r3
-; CHECK: srwi r3, [[REG1]], 5
-; CHECK-NEXT: blr
 }
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_ileus_sext_z(i16 zeroext %a) {
+; CHECK-LABEL: test_ileus_sext_z:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cntlzw r3, r3
+; CHECK-NEXT:    srwi r3, r3, 5
+; CHECK-NEXT:    neg r3, r3
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ule i16 %a, 0
   %sub = sext i1 %cmp to i32
   ret i32 %sub
-; CHECK-LABEL: @test_ileus_sext_z
-; CHECK: cntlzw [[REG1:r[0-9]+]], r3
-; CHECK-NEXT: srwi [[REG2:r[0-9]+]], [[REG1]], 5
-; CHECK-NEXT: neg r3, [[REG2]]
-; CHECK-NEXT: blr
 }
 
 ; Function Attrs: norecurse nounwind
 define void @test_ileus_store(i16 zeroext %a, i16 zeroext %b) {
+; BE-LABEL: test_ileus_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r5, r2, .LC0@toc@ha
+; BE-NEXT:    sub r3, r4, r3
+; BE-NEXT:    ld r4, .LC0@toc@l(r5)
+; BE-NEXT:    not r3, r3
+; BE-NEXT:    rldicl r3, r3, 1, 63
+; BE-NEXT:    sth r3, 0(r4)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_ileus_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    sub r3, r4, r3
+; LE-NEXT:    addis r5, r2, glob@toc@ha
+; LE-NEXT:    not r3, r3
+; LE-NEXT:    rldicl r3, r3, 1, 63
+; LE-NEXT:    sth r3, glob@toc@l(r5)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp ule i16 %a, %b
   %conv3 = zext i1 %cmp to i16
   store i16 %conv3, i16* @glob
   ret void
-; CHECK-LABEL: test_ileus_store:
-; CHECK: sub [[REG1:r[0-9]+]], r4, r3
-; CHECK: not [[REG2:r[0-9]+]], [[REG1]]
-; CHECK-NEXT: rldicl r3, [[REG2]], 1, 63
-; CHECK: blr
 }
 
 ; Function Attrs: norecurse nounwind
 define void @test_ileus_sext_store(i16 zeroext %a, i16 zeroext %b) {
+; BE-LABEL: test_ileus_sext_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r5, r2, .LC0@toc@ha
+; BE-NEXT:    sub r3, r4, r3
+; BE-NEXT:    ld r4, .LC0@toc@l(r5)
+; BE-NEXT:    rldicl r3, r3, 1, 63
+; BE-NEXT:    addi r3, r3, -1
+; BE-NEXT:    sth r3, 0(r4)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_ileus_sext_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    sub r3, r4, r3
+; LE-NEXT:    addis r5, r2, glob@toc@ha
+; LE-NEXT:    rldicl r3, r3, 1, 63
+; LE-NEXT:    addi r3, r3, -1
+; LE-NEXT:    sth r3, glob@toc@l(r5)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp ule i16 %a, %b
   %conv3 = sext i1 %cmp to i16
   store i16 %conv3, i16* @glob
   ret void
-; CHECK-LABEL: @test_ileus_sext_store
-; CHECK: sub [[REG1:r[0-9]+]], r4, r3
-; CHECK: rldicl [[REG2:r[0-9]+]], [[REG1]], 1, 63
-; CHECK: addi [[REG3:r[0-9]+]], [[REG2]], -1
-; CHECK: sth  [[REG3]]
-; CHECK: blr
 }
 
 ; Function Attrs: norecurse nounwind
 define void @test_ileus_z_store(i16 zeroext %a) {
+; BE-LABEL: test_ileus_z_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r4, r2, .LC0@toc@ha
+; BE-NEXT:    cntlzw r3, r3
+; BE-NEXT:    ld r4, .LC0@toc@l(r4)
+; BE-NEXT:    srwi r3, r3, 5
+; BE-NEXT:    sth r3, 0(r4)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_ileus_z_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    cntlzw r3, r3
+; LE-NEXT:    addis r4, r2, glob@toc@ha
+; LE-NEXT:    srwi r3, r3, 5
+; LE-NEXT:    sth r3, glob@toc@l(r4)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp ule i16 %a, 0
   %conv2 = zext i1 %cmp to i16
   store i16 %conv2, i16* @glob
   ret void
-; CHECK-LABEL: test_ileus_z_store:
-; CHECK: cntlzw [[REG1:r[0-9]+]], r3
-; CHECK: srwi {{r[0-9]+}}, [[REG1]], 5
-; CHECK: blr
 }
 
 ; Function Attrs: norecurse nounwind
 define void @test_ileus_sext_z_store(i16 zeroext %a) {
+; BE-LABEL: test_ileus_sext_z_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r4, r2, .LC0@toc@ha
+; BE-NEXT:    cntlzw r3, r3
+; BE-NEXT:    ld r4, .LC0@toc@l(r4)
+; BE-NEXT:    srwi r3, r3, 5
+; BE-NEXT:    neg r3, r3
+; BE-NEXT:    sth r3, 0(r4)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_ileus_sext_z_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    cntlzw r3, r3
+; LE-NEXT:    addis r4, r2, glob@toc@ha
+; LE-NEXT:    srwi r3, r3, 5
+; LE-NEXT:    neg r3, r3
+; LE-NEXT:    sth r3, glob@toc@l(r4)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp ule i16 %a, 0
   %conv2 = sext i1 %cmp to i16
   store i16 %conv2, i16* @glob
   ret void
-; CHECK-LABEL: @test_ileus_sext_z_store
-; CHECK: cntlzw [[REG1:r[0-9]+]], r3
-; CHECK: srwi [[REG2:r[0-9]+]], [[REG1]], 5
-; CHECK: neg [[REG3:r[0-9]+]], [[REG2]]
-; CHECK: sth [[REG3]]
-; CHECK: blr
 }
 
diff --git a/llvm/test/CodeGen/PowerPC/testComparesiltsc.ll b/llvm/test/CodeGen/PowerPC/testComparesiltsc.ll
index 08a023302..b1a1b96e6 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesiltsc.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesiltsc.ll
@@ -1,19 +1,21 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
-; RUN:   --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
+; RUN:   --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
+; RUN:  --check-prefixes=CHECK,BE
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
-; RUN:   --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN:   --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
+; RUN:  --check-prefixes=CHECK,LE
 
-@glob = common local_unnamed_addr global i8 0, align 1
+@glob = local_unnamed_addr global i8 0, align 1
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_iltsc(i8 signext %a, i8 signext %b) {
 ; CHECK-LABEL: test_iltsc:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    sub [[REG:r[0-9]+]], r3, r4
-; CHECK-NEXT:    rldicl r3, [[REG]], 1, 63
+; CHECK-NEXT:    sub r3, r3, r4
+; CHECK-NEXT:    rldicl r3, r3, 1, 63
 ; CHECK-NEXT:    blr
 entry:
   %cmp = icmp slt i8 %a, %b
@@ -25,8 +27,8 @@ entry:
 define signext i32 @test_iltsc_sext(i8 signext %a, i8 signext %b) {
 ; CHECK-LABEL: test_iltsc_sext:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    sub [[REG:r[0-9]+]], r3, r4
-; CHECK-NEXT:    sradi r3, [[REG]], 63
+; CHECK-NEXT:    sub r3, r3, r4
+; CHECK-NEXT:    sradi r3, r3, 63
 ; CHECK-NEXT:    blr
 entry:
   %cmp = icmp slt i8 %a, %b
@@ -37,8 +39,9 @@ entry:
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_iltsc_sext_z(i8 signext %a) {
 ; CHECK-LABEL: test_iltsc_sext_z:
-; CHECK:       srawi r3, r3, 31
-; CHECK-NEXT:  blr
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    srawi r3, r3, 31
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp slt i8 %a, 0
   %sub = sext i1 %cmp to i32
@@ -47,10 +50,22 @@ entry:
 
 ; Function Attrs: norecurse nounwind
 define void @test_iltsc_store(i8 signext %a, i8 signext %b) {
-; CHECK-LABEL: test_iltsc_store:
-; CHECK:       # %bb.0: # %entry
-; CHECK:         sub [[REG:r[0-9]+]], r3, r4
-; CHECK:         rldicl {{r[0-9]+}}, [[REG]], 1, 63
+; BE-LABEL: test_iltsc_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r5, r2, .LC0@toc@ha
+; BE-NEXT:    sub r3, r3, r4
+; BE-NEXT:    ld r5, .LC0@toc@l(r5)
+; BE-NEXT:    rldicl r3, r3, 1, 63
+; BE-NEXT:    stb r3, 0(r5)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_iltsc_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    sub r3, r3, r4
+; LE-NEXT:    addis r5, r2, glob@toc@ha
+; LE-NEXT:    rldicl r3, r3, 1, 63
+; LE-NEXT:    stb r3, glob@toc@l(r5)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp slt i8 %a, %b
   %conv3 = zext i1 %cmp to i8
@@ -60,10 +75,22 @@ entry:
 
 ; Function Attrs: norecurse nounwind
 define void @test_iltsc_sext_store(i8 signext %a, i8 signext %b) {
-; CHECK-LABEL: test_iltsc_sext_store:
-; CHECK:       # %bb.0: # %entry
-; CHECK:         sub [[REG:r[0-9]+]], r3, r4
-; CHECK:         sradi {{r[0-9]+}}, [[REG]], 63
+; BE-LABEL: test_iltsc_sext_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r5, r2, .LC0@toc@ha
+; BE-NEXT:    sub r3, r3, r4
+; BE-NEXT:    ld r5, .LC0@toc@l(r5)
+; BE-NEXT:    sradi r3, r3, 63
+; BE-NEXT:    stb r3, 0(r5)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_iltsc_sext_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    sub r3, r3, r4
+; LE-NEXT:    addis r5, r2, glob@toc@ha
+; LE-NEXT:    sradi r3, r3, 63
+; LE-NEXT:    stb r3, glob@toc@l(r5)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp slt i8 %a, %b
   %conv3 = sext i1 %cmp to i8
@@ -73,8 +100,20 @@ entry:
 
 ; Function Attrs: norecurse nounwind
 define void @test_iltsc_sext_z_store(i8 signext %a) {
-; CHECK-LABEL: test_iltsc_sext_z_store:
-; CHECK: srwi {{r[0-9]+}}, r3, 7
+; BE-LABEL: test_iltsc_sext_z_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r4, r2, .LC0@toc@ha
+; BE-NEXT:    srwi r3, r3, 7
+; BE-NEXT:    ld r4, .LC0@toc@l(r4)
+; BE-NEXT:    stb r3, 0(r4)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_iltsc_sext_z_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    addis r4, r2, glob@toc@ha
+; LE-NEXT:    srwi r3, r3, 7
+; LE-NEXT:    stb r3, glob@toc@l(r4)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp slt i8 %a, 0
   %conv2 = sext i1 %cmp to i8
diff --git a/llvm/test/CodeGen/PowerPC/testComparesiltsi.ll b/llvm/test/CodeGen/PowerPC/testComparesiltsi.ll
index 39f37387f..252e9c323 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesiltsi.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesiltsi.ll
@@ -1,19 +1,21 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
-; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
+; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
+; RUN:  --check-prefixes=CHECK,BE
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
-; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
+; RUN:  --check-prefixes=CHECK,LE
 
-@glob = common local_unnamed_addr global i32 0, align 4
+@glob = local_unnamed_addr global i32 0, align 4
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_iltsi(i32 signext %a, i32 signext %b) {
 ; CHECK-LABEL: test_iltsi:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    sub [[REG:r[0-9]+]], r3, r4
-; CHECK-NEXT:    rldicl r3, [[REG]], 1, 63
+; CHECK-NEXT:    sub r3, r3, r4
+; CHECK-NEXT:    rldicl r3, r3, 1, 63
 ; CHECK-NEXT:    blr
 entry:
   %cmp = icmp slt i32 %a, %b
@@ -25,8 +27,8 @@ entry:
 define signext i32 @test_iltsi_sext(i32 signext %a, i32 signext %b) {
 ; CHECK-LABEL: test_iltsi_sext:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    sub [[REG:r[0-9]+]], r3, r4
-; CHECK-NEXT:    sradi r3, [[REG]], 63
+; CHECK-NEXT:    sub r3, r3, r4
+; CHECK-NEXT:    sradi r3, r3, 63
 ; CHECK-NEXT:    blr
 entry:
   %cmp = icmp slt i32 %a, %b
@@ -48,10 +50,22 @@ entry:
 
 ; Function Attrs: norecurse nounwind
 define void @test_iltsi_store(i32 signext %a, i32 signext %b) {
-; CHECK-LABEL: test_iltsi_store:
-; CHECK:       # %bb.0: # %entry
-; CHECK:         sub [[REG:r[0-9]+]], r3, r4
-; CHECK:         rldicl {{r[0-9]+}}, [[REG]], 1, 63
+; BE-LABEL: test_iltsi_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r5, r2, .LC0@toc@ha
+; BE-NEXT:    sub r3, r3, r4
+; BE-NEXT:    ld r5, .LC0@toc@l(r5)
+; BE-NEXT:    rldicl r3, r3, 1, 63
+; BE-NEXT:    stw r3, 0(r5)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_iltsi_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    sub r3, r3, r4
+; LE-NEXT:    addis r5, r2, glob@toc@ha
+; LE-NEXT:    rldicl r3, r3, 1, 63
+; LE-NEXT:    stw r3, glob@toc@l(r5)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp slt i32 %a, %b
   %conv = zext i1 %cmp to i32
@@ -61,10 +75,22 @@ entry:
 
 ; Function Attrs: norecurse nounwind
 define void @test_iltsi_sext_store(i32 signext %a, i32 signext %b) {
-; CHECK-LABEL: test_iltsi_sext_store:
-; CHECK:       # %bb.0: # %entry
-; CHECK:         sub [[REG:r[0-9]+]], r3, r4
-; CHECK:         sradi {{r[0-9]+}}, [[REG]], 63
+; BE-LABEL: test_iltsi_sext_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r5, r2, .LC0@toc@ha
+; BE-NEXT:    sub r3, r3, r4
+; BE-NEXT:    ld r5, .LC0@toc@l(r5)
+; BE-NEXT:    sradi r3, r3, 63
+; BE-NEXT:    stw r3, 0(r5)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_iltsi_sext_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    sub r3, r3, r4
+; LE-NEXT:    addis r5, r2, glob@toc@ha
+; LE-NEXT:    sradi r3, r3, 63
+; LE-NEXT:    stw r3, glob@toc@l(r5)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp slt i32 %a, %b
   %sub = sext i1 %cmp to i32
@@ -74,9 +100,20 @@ entry:
 
 ; Function Attrs: norecurse nounwind
 define void @test_iltsi_sext_z_store(i32 signext %a) {
-; CHECK-LABEL: test_iltsi_sext_z_store:
-; CHECK:    srawi {{r[0-9]+}}, r3, 31
-; CHECK:    blr
+; BE-LABEL: test_iltsi_sext_z_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r4, r2, .LC0@toc@ha
+; BE-NEXT:    srawi r3, r3, 31
+; BE-NEXT:    ld r4, .LC0@toc@l(r4)
+; BE-NEXT:    stw r3, 0(r4)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_iltsi_sext_z_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    addis r4, r2, glob@toc@ha
+; LE-NEXT:    srawi r3, r3, 31
+; LE-NEXT:    stw r3, glob@toc@l(r4)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp slt i32 %a, 0
   %sub = sext i1 %cmp to i32
diff --git a/llvm/test/CodeGen/PowerPC/testComparesiltsll.ll b/llvm/test/CodeGen/PowerPC/testComparesiltsll.ll
index 4152b8556..0a31d051e 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesiltsll.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesiltsll.ll
@@ -1,22 +1,24 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
-; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
+; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
+; RUN:  --check-prefixes=CHECK,BE
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
-; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
+; RUN:  --check-prefixes=CHECK,LE
 
-@glob = common local_unnamed_addr global i64 0, align 8
+@glob = local_unnamed_addr global i64 0, align 8
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_iltsll(i64 %a, i64 %b) {
 ; CHECK-LABEL: test_iltsll:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    sradi [[REG1:r[0-9]+]], r3, 63
-; CHECK-NEXT:    rldicl [[REG2:r[0-9]+]], r4, 1, 63
-; CHECK-NEXT:    subfc [[REG3:r[0-9]+]], r4, r3
-; CHECK-NEXT:    adde [[REG4:r[0-9]+]], [[REG2]], [[REG1]]
-; CHECK-NEXT:    xori r3, [[REG4]], 1
+; CHECK-NEXT:    sradi r5, r3, 63
+; CHECK-NEXT:    rldicl r6, r4, 1, 63
+; CHECK-NEXT:    subfc r3, r4, r3
+; CHECK-NEXT:    adde r3, r6, r5
+; CHECK-NEXT:    xori r3, r3, 1
 ; CHECK-NEXT:    blr
 entry:
   %cmp = icmp slt i64 %a, %b
@@ -28,12 +30,12 @@ entry:
 define signext i32 @test_iltsll_sext(i64 %a, i64 %b) {
 ; CHECK-LABEL: test_iltsll_sext:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    sradi [[REG1:r[0-9]+]], r3, 63
-; CHECK-NEXT:    rldicl [[REG2:r[0-9]+]], r4, 1, 63
-; CHECK-NEXT:    subfc [[REG3:r[0-9]+]], r4, r3
-; CHECK-NEXT:    adde [[REG4:r[0-9]+]], [[REG2]], [[REG1]]
-; CHECK-NEXT:    xori [[REG5:r[0-9]+]], [[REG4]], 1
-; CHECK-NEXT:    neg r3, [[REG5]]
+; CHECK-NEXT:    sradi r5, r3, 63
+; CHECK-NEXT:    rldicl r6, r4, 1, 63
+; CHECK-NEXT:    subfc r3, r4, r3
+; CHECK-NEXT:    adde r3, r6, r5
+; CHECK-NEXT:    xori r3, r3, 1
+; CHECK-NEXT:    neg r3, r3
 ; CHECK-NEXT:    blr
 entry:
   %cmp = icmp slt i64 %a, %b
@@ -55,14 +57,29 @@ entry:
 
 ; Function Attrs: norecurse nounwind
 define void @test_iltsll_store(i64 %a, i64 %b) {
-; CHECK-LABEL: test_iltsll_store:
-; CHECK:       # %bb.0: # %entry
-; CHECK:         sradi [[REG1:r[0-9]+]], r3, 63
-; CHECK:         rldicl [[REG2:r[0-9]+]], r4, 1, 63
+; BE-LABEL: test_iltsll_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    sradi r6, r3, 63
+; BE-NEXT:    addis r5, r2, .LC0@toc@ha
+; BE-NEXT:    subfc r3, r4, r3
+; BE-NEXT:    rldicl r3, r4, 1, 63
+; BE-NEXT:    ld r4, .LC0@toc@l(r5)
+; BE-NEXT:    adde r3, r3, r6
+; BE-NEXT:    xori r3, r3, 1
+; BE-NEXT:    std r3, 0(r4)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_iltsll_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    sradi r6, r3, 63
+; LE-NEXT:    addis r5, r2, glob@toc@ha
+; LE-NEXT:    subfc r3, r4, r3
+; LE-NEXT:    rldicl r3, r4, 1, 63
+; LE-NEXT:    adde r3, r3, r6
+; LE-NEXT:    xori r3, r3, 1
+; LE-NEXT:    std r3, glob@toc@l(r5)
+; LE-NEXT:    blr
 ; CHECK-DIAG:    subfc [[REG3:r[0-9]+]], r4, r3
-; CHECK:         adde [[REG4:r[0-9]+]], [[REG2]], [[REG1]]
-; CHECK:         xori [[REG5:r[0-9]+]], [[REG4]], 1
-; CHECK-NOT:     neg {{r[0-9]+}}, [[REG5]]
 entry:
   %cmp = icmp slt i64 %a, %b
   %conv1 = zext i1 %cmp to i64
@@ -72,14 +89,31 @@ entry:
 
 ; Function Attrs: norecurse nounwind
 define void @test_iltsll_sext_store(i64 %a, i64 %b) {
-; CHECK-LABEL: test_iltsll_sext_store:
-; CHECK:       # %bb.0: # %entry
-; CHECK:         sradi [[REG1:r[0-9]+]], r3, 63
-; CHECK:         rldicl [[REG2:r[0-9]+]], r4, 1, 63
+; BE-LABEL: test_iltsll_sext_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    sradi r6, r3, 63
+; BE-NEXT:    addis r5, r2, .LC0@toc@ha
+; BE-NEXT:    subfc r3, r4, r3
+; BE-NEXT:    rldicl r3, r4, 1, 63
+; BE-NEXT:    ld r4, .LC0@toc@l(r5)
+; BE-NEXT:    adde r3, r3, r6
+; BE-NEXT:    xori r3, r3, 1
+; BE-NEXT:    neg r3, r3
+; BE-NEXT:    std r3, 0(r4)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_iltsll_sext_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    sradi r6, r3, 63
+; LE-NEXT:    addis r5, r2, glob@toc@ha
+; LE-NEXT:    subfc r3, r4, r3
+; LE-NEXT:    rldicl r3, r4, 1, 63
+; LE-NEXT:    adde r3, r3, r6
+; LE-NEXT:    xori r3, r3, 1
+; LE-NEXT:    neg r3, r3
+; LE-NEXT:    std r3, glob@toc@l(r5)
+; LE-NEXT:    blr
 ; CHECK-DIAG:    subfc [[REG3:r[0-9]+]], r4, r3
-; CHECK:         adde [[REG4:r[0-9]+]], [[REG2]], [[REG1]]
-; CHECK:         xori [[REG5:r[0-9]+]], [[REG4]], 1
-; CHECK:         neg {{r[0-9]+}}, [[REG5]]
 entry:
   %cmp = icmp slt i64 %a, %b
   %conv1 = sext i1 %cmp to i64
@@ -89,8 +123,20 @@ entry:
 
 ; Function Attrs: norecurse nounwind
 define void @test_iltsll_sext_z_store(i64 %a) {
-; CHECK-LABEL: test_iltsll_sext_z_store:
-; CHECK:    sradi r3, r3, 63
+; BE-LABEL: test_iltsll_sext_z_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r4, r2, .LC0@toc@ha
+; BE-NEXT:    sradi r3, r3, 63
+; BE-NEXT:    ld r4, .LC0@toc@l(r4)
+; BE-NEXT:    std r3, 0(r4)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_iltsll_sext_z_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    addis r4, r2, glob@toc@ha
+; LE-NEXT:    sradi r3, r3, 63
+; LE-NEXT:    std r3, glob@toc@l(r4)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp slt i64 %a, 0
   %conv2 = sext i1 %cmp to i64
diff --git a/llvm/test/CodeGen/PowerPC/testComparesiltss.ll b/llvm/test/CodeGen/PowerPC/testComparesiltss.ll
index db5a60dfb..f6a912513 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesiltss.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesiltss.ll
@@ -1,19 +1,21 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
-; RUN:   --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
+; RUN:   --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
+; RUN:  --check-prefixes=CHECK,BE
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
-; RUN:   --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN:   --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
+; RUN:  --check-prefixes=CHECK,LE
 
-@glob = common local_unnamed_addr global i16 0, align 2
+@glob = local_unnamed_addr global i16 0, align 2
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_iltss(i16 signext %a, i16 signext %b) {
 ; CHECK-LABEL: test_iltss:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    sub [[REG:r[0-9]+]], r3, r4
-; CHECK-NEXT:    rldicl r3, [[REG]], 1, 63
+; CHECK-NEXT:    sub r3, r3, r4
+; CHECK-NEXT:    rldicl r3, r3, 1, 63
 ; CHECK-NEXT:    blr
 entry:
   %cmp = icmp slt i16 %a, %b
@@ -25,8 +27,8 @@ entry:
 define signext i32 @test_iltss_sext(i16 signext %a, i16 signext %b) {
 ; CHECK-LABEL: test_iltss_sext:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    sub [[REG:r[0-9]+]], r3, r4
-; CHECK-NEXT:    sradi r3, [[REG]], 63
+; CHECK-NEXT:    sub r3, r3, r4
+; CHECK-NEXT:    sradi r3, r3, 63
 ; CHECK-NEXT:    blr
 entry:
   %cmp = icmp slt i16 %a, %b
@@ -37,8 +39,9 @@ entry:
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_iltss_sext_z(i16 signext %a) {
 ; CHECK-LABEL: test_iltss_sext_z:
-; CHECK:       srawi r3, r3, 31
-; CHECK-NEXT:  blr
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    srawi r3, r3, 31
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp slt i16 %a, 0
   %sub = sext i1 %cmp to i32
@@ -47,10 +50,22 @@ entry:
 
 ; Function Attrs: norecurse nounwind
 define void @test_iltss_store(i16 signext %a, i16 signext %b) {
-; CHECK-LABEL: test_iltss_store:
-; CHECK:       # %bb.0: # %entry
-; CHECK:         sub [[REG:r[0-9]+]], r3, r4
-; CHECK:         rldicl {{r[0-9]+}}, [[REG]], 1, 63
+; BE-LABEL: test_iltss_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r5, r2, .LC0@toc@ha
+; BE-NEXT:    sub r3, r3, r4
+; BE-NEXT:    ld r5, .LC0@toc@l(r5)
+; BE-NEXT:    rldicl r3, r3, 1, 63
+; BE-NEXT:    sth r3, 0(r5)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_iltss_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    sub r3, r3, r4
+; LE-NEXT:    addis r5, r2, glob@toc@ha
+; LE-NEXT:    rldicl r3, r3, 1, 63
+; LE-NEXT:    sth r3, glob@toc@l(r5)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp slt i16 %a, %b
   %conv3 = zext i1 %cmp to i16
@@ -60,10 +75,22 @@ entry:
 
 ; Function Attrs: norecurse nounwind
 define void @test_iltss_sext_store(i16 signext %a, i16 signext %b) {
-; CHECK-LABEL: test_iltss_sext_store:
-; CHECK:       # %bb.0: # %entry
-; CHECK:         sub [[REG:r[0-9]+]], r3, r4
-; CHECK:         sradi {{r[0-9]+}}, [[REG]], 63
+; BE-LABEL: test_iltss_sext_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r5, r2, .LC0@toc@ha
+; BE-NEXT:    sub r3, r3, r4
+; BE-NEXT:    ld r5, .LC0@toc@l(r5)
+; BE-NEXT:    sradi r3, r3, 63
+; BE-NEXT:    sth r3, 0(r5)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_iltss_sext_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    sub r3, r3, r4
+; LE-NEXT:    addis r5, r2, glob@toc@ha
+; LE-NEXT:    sradi r3, r3, 63
+; LE-NEXT:    sth r3, glob@toc@l(r5)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp slt i16 %a, %b
   %conv3 = sext i1 %cmp to i16
@@ -73,8 +100,20 @@ entry:
 
 ; Function Attrs: norecurse nounwind
 define void @test_iltss_sext_z_store(i16 signext %a) {
-; CHECK-LABEL: test_iltss_sext_z_store:
-; CHECK:       srwi {{r[0-9]+}}, r3, 15
+; BE-LABEL: test_iltss_sext_z_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r4, r2, .LC0@toc@ha
+; BE-NEXT:    srwi r3, r3, 15
+; BE-NEXT:    ld r4, .LC0@toc@l(r4)
+; BE-NEXT:    sth r3, 0(r4)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_iltss_sext_z_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    addis r4, r2, glob@toc@ha
+; LE-NEXT:    srwi r3, r3, 15
+; LE-NEXT:    sth r3, glob@toc@l(r4)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp slt i16 %a, 0
   %sub = sext i1 %cmp to i16
diff --git a/llvm/test/CodeGen/PowerPC/testComparesiltuc.ll b/llvm/test/CodeGen/PowerPC/testComparesiltuc.ll
index 19db29a50..de2dc1cbe 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesiltuc.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesiltuc.ll
@@ -1,17 +1,21 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
-; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
+; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
+; RUN:  --check-prefixes=CHECK,BE
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
-; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
+; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
+; RUN:  --check-prefixes=CHECK,LE
 
-@glob = common local_unnamed_addr global i8 0, align 1
+@glob = local_unnamed_addr global i8 0, align 1
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_iltuc(i8 zeroext %a, i8 zeroext %b) {
 ; CHECK-LABEL: test_iltuc:
-; CHECK:    sub [[REG:r[0-9]+]], r3, r4
-; CHECK-NEXT:    rldicl r3, [[REG]], 1, 63
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    sub r3, r3, r4
+; CHECK-NEXT:    rldicl r3, r3, 1, 63
 ; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ult i8 %a, %b
@@ -22,8 +26,9 @@ entry:
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_iltuc_sext(i8 zeroext %a, i8 zeroext %b) {
 ; CHECK-LABEL: test_iltuc_sext:
-; CHECK:    sub [[REG:r[0-9]+]], r3, r4
-; CHECK-NEXT:    sradi r3, [[REG]], 63
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    sub r3, r3, r4
+; CHECK-NEXT:    sradi r3, r3, 63
 ; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ult i8 %a, %b
@@ -33,9 +38,22 @@ entry:
 
 ; Function Attrs: norecurse nounwind
 define void @test_iltuc_store(i8 zeroext %a, i8 zeroext %b) {
-; CHECK-LABEL: test_iltuc_store:
-; CHECK:         sub [[REG:r[2-9]+]], r3, r4
-; CHECK:    rldicl {{r[0-9]+}}, [[REG]], 1, 63
+; BE-LABEL: test_iltuc_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r5, r2, .LC0@toc@ha
+; BE-NEXT:    sub r3, r3, r4
+; BE-NEXT:    ld r5, .LC0@toc@l(r5)
+; BE-NEXT:    rldicl r3, r3, 1, 63
+; BE-NEXT:    stb r3, 0(r5)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_iltuc_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    sub r3, r3, r4
+; LE-NEXT:    addis r5, r2, glob@toc@ha
+; LE-NEXT:    rldicl r3, r3, 1, 63
+; LE-NEXT:    stb r3, glob@toc@l(r5)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp ult i8 %a, %b
   %conv3 = zext i1 %cmp to i8
@@ -45,9 +63,22 @@ entry:
 
 ; Function Attrs: norecurse nounwind
 define void @test_iltuc_sext_store(i8 zeroext %a, i8 zeroext %b) {
-; CHECK-LABEL: test_iltuc_sext_store:
-; CHECK:         sub [[REG:r[0-9]+]], r3, r4
-; CHECK:         sradi {{r[0-9]+}}, [[REG]], 63
+; BE-LABEL: test_iltuc_sext_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r5, r2, .LC0@toc@ha
+; BE-NEXT:    sub r3, r3, r4
+; BE-NEXT:    ld r5, .LC0@toc@l(r5)
+; BE-NEXT:    sradi r3, r3, 63
+; BE-NEXT:    stb r3, 0(r5)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_iltuc_sext_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    sub r3, r3, r4
+; LE-NEXT:    addis r5, r2, glob@toc@ha
+; LE-NEXT:    sradi r3, r3, 63
+; LE-NEXT:    stb r3, glob@toc@l(r5)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp ult i8 %a, %b
   %conv3 = sext i1 %cmp to i8
diff --git a/llvm/test/CodeGen/PowerPC/testComparesiltui.ll b/llvm/test/CodeGen/PowerPC/testComparesiltui.ll
index dd9a202cb..a0d40b62a 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesiltui.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesiltui.ll
@@ -1,17 +1,21 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
-; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
+; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
+; RUN:  --check-prefixes=CHECK,BE
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
-; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
+; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
+; RUN:  --check-prefixes=CHECK,LE
 
-@glob = common local_unnamed_addr global i32 0, align 4
+@glob = local_unnamed_addr global i32 0, align 4
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_iltui(i32 zeroext %a, i32 zeroext %b) {
 ; CHECK-LABEL: test_iltui:
-; CHECK:    sub [[REG:r[0-9]+]], r3, r4
-; CHECK-NEXT:    rldicl r3, [[REG]], 1, 63
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    sub r3, r3, r4
+; CHECK-NEXT:    rldicl r3, r3, 1, 63
 ; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ult i32 %a, %b
@@ -22,8 +26,9 @@ entry:
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_iltui_sext(i32 zeroext %a, i32 zeroext %b) {
 ; CHECK-LABEL: test_iltui_sext:
-; CHECK:    sub [[REG:r[0-9]+]], r3, r4
-; CHECK-NEXT:    sradi r3, [[REG]], 63
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    sub r3, r3, r4
+; CHECK-NEXT:    sradi r3, r3, 63
 ; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ult i32 %a, %b
@@ -33,9 +38,22 @@ entry:
 
 ; Function Attrs: norecurse nounwind
 define void @test_iltui_store(i32 zeroext %a, i32 zeroext %b) {
-; CHECK-LABEL: test_iltui_store:
-; CHECK:         sub [[REG:r[2-9]+]], r3, r4
-; CHECK:         rldicl {{r[0-9]+}}, [[REG]], 1, 63
+; BE-LABEL: test_iltui_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r5, r2, .LC0@toc@ha
+; BE-NEXT:    sub r3, r3, r4
+; BE-NEXT:    ld r5, .LC0@toc@l(r5)
+; BE-NEXT:    rldicl r3, r3, 1, 63
+; BE-NEXT:    stw r3, 0(r5)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_iltui_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    sub r3, r3, r4
+; LE-NEXT:    addis r5, r2, glob@toc@ha
+; LE-NEXT:    rldicl r3, r3, 1, 63
+; LE-NEXT:    stw r3, glob@toc@l(r5)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp ult i32 %a, %b
   %conv = zext i1 %cmp to i32
@@ -45,9 +63,22 @@ entry:
 
 ; Function Attrs: norecurse nounwind
 define void @test_iltui_sext_store(i32 zeroext %a, i32 zeroext %b) {
-; CHECK-LABEL: test_iltui_sext_store:
-; CHECK:         sub [[REG:r[0-9]+]], r3, r4
-; CHECK:         sradi {{r[0-9]+}}, [[REG]], 63
+; BE-LABEL: test_iltui_sext_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r5, r2, .LC0@toc@ha
+; BE-NEXT:    sub r3, r3, r4
+; BE-NEXT:    ld r5, .LC0@toc@l(r5)
+; BE-NEXT:    sradi r3, r3, 63
+; BE-NEXT:    stw r3, 0(r5)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_iltui_sext_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    sub r3, r3, r4
+; LE-NEXT:    addis r5, r2, glob@toc@ha
+; LE-NEXT:    sradi r3, r3, 63
+; LE-NEXT:    stw r3, glob@toc@l(r5)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp ult i32 %a, %b
   %sub = sext i1 %cmp to i32
diff --git a/llvm/test/CodeGen/PowerPC/testComparesiltus.ll b/llvm/test/CodeGen/PowerPC/testComparesiltus.ll
index 117457250..98b8f3553 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesiltus.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesiltus.ll
@@ -1,17 +1,21 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
-; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
+; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
+; RUN:  --check-prefixes=CHECK,BE
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
-; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
+; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
+; RUN:  --check-prefixes=CHECK,LE
 
-@glob = common local_unnamed_addr global i16 0, align 2
+@glob = local_unnamed_addr global i16 0, align 2
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_iltus(i16 zeroext %a, i16 zeroext %b) {
 ; CHECK-LABEL: test_iltus:
-; CHECK:    sub [[REG:r[0-9]+]], r3, r4
-; CHECK-NEXT:    rldicl r3, [[REG]], 1, 63
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    sub r3, r3, r4
+; CHECK-NEXT:    rldicl r3, r3, 1, 63
 ; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ult i16 %a, %b
@@ -22,8 +26,9 @@ entry:
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_iltus_sext(i16 zeroext %a, i16 zeroext %b) {
 ; CHECK-LABEL: test_iltus_sext:
-; CHECK:    sub [[REG:r[0-9]+]], r3, r4
-; CHECK-NEXT:    sradi r3, [[REG]], 63
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    sub r3, r3, r4
+; CHECK-NEXT:    sradi r3, r3, 63
 ; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ult i16 %a, %b
@@ -33,9 +38,22 @@ entry:
 
 ; Function Attrs: norecurse nounwind
 define void @test_iltus_store(i16 zeroext %a, i16 zeroext %b) {
-; CHECK-LABEL: test_iltus_store:
-; CHECK:         sub [[REG:r[2-9]+]], r3, r4
-; CHECK:         rldicl {{r[0-9]+}}, [[REG]], 1, 63
+; BE-LABEL: test_iltus_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r5, r2, .LC0@toc@ha
+; BE-NEXT:    sub r3, r3, r4
+; BE-NEXT:    ld r5, .LC0@toc@l(r5)
+; BE-NEXT:    rldicl r3, r3, 1, 63
+; BE-NEXT:    sth r3, 0(r5)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_iltus_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    sub r3, r3, r4
+; LE-NEXT:    addis r5, r2, glob@toc@ha
+; LE-NEXT:    rldicl r3, r3, 1, 63
+; LE-NEXT:    sth r3, glob@toc@l(r5)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp ult i16 %a, %b
   %conv3 = zext i1 %cmp to i16
@@ -45,9 +63,22 @@ entry:
 
 ; Function Attrs: norecurse nounwind
 define void @test_iltus_sext_store(i16 zeroext %a, i16 zeroext %b) {
-; CHECK-LABEL: test_iltus_sext_store:
-; CHECK:         sub [[REG:r[0-9]+]], r3, r4
-; CHECK:         sradi {{r[0-9]+}}, [[REG]], 63
+; BE-LABEL: test_iltus_sext_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r5, r2, .LC0@toc@ha
+; BE-NEXT:    sub r3, r3, r4
+; BE-NEXT:    ld r5, .LC0@toc@l(r5)
+; BE-NEXT:    sradi r3, r3, 63
+; BE-NEXT:    sth r3, 0(r5)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_iltus_sext_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    sub r3, r3, r4
+; LE-NEXT:    addis r5, r2, glob@toc@ha
+; LE-NEXT:    sradi r3, r3, 63
+; LE-NEXT:    sth r3, glob@toc@l(r5)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp ult i16 %a, %b
   %conv3 = sext i1 %cmp to i16
diff --git a/llvm/test/CodeGen/PowerPC/testComparesinesc.ll b/llvm/test/CodeGen/PowerPC/testComparesinesc.ll
index f5ad93440..8c81eadc1 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesinesc.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesinesc.ll
@@ -6,7 +6,7 @@
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s --check-prefix=CHECK-LE \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 
-@glob = common local_unnamed_addr global i8 0, align 1
+@glob = local_unnamed_addr global i8 0, align 1
 
 define signext i32 @test_inesc(i8 signext %a, i8 signext %b) {
 ; CHECK-LABEL: test_inesc:
diff --git a/llvm/test/CodeGen/PowerPC/testComparesinesi.ll b/llvm/test/CodeGen/PowerPC/testComparesinesi.ll
index 3b0fe78e9..ae8d981cd 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesinesi.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesinesi.ll
@@ -6,7 +6,7 @@
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s --check-prefix=CHECK-LE \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 
-@glob = common local_unnamed_addr global i32 0, align 4
+@glob = local_unnamed_addr global i32 0, align 4
 
 define signext i32 @test_inesi(i32 signext %a, i32 signext %b) {
 ; CHECK-LABEL: test_inesi:
diff --git a/llvm/test/CodeGen/PowerPC/testComparesinesll.ll b/llvm/test/CodeGen/PowerPC/testComparesinesll.ll
index bfc0b9d3f..c360fb991 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesinesll.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesinesll.ll
@@ -6,7 +6,7 @@
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s --check-prefix=CHECK-LE \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 
-@glob = common local_unnamed_addr global i64 0, align 8
+@glob = local_unnamed_addr global i64 0, align 8
 
 define signext i32 @test_inesll(i64 %a, i64 %b) {
 ; CHECK-LABEL: test_inesll:
diff --git a/llvm/test/CodeGen/PowerPC/testComparesiness.ll b/llvm/test/CodeGen/PowerPC/testComparesiness.ll
index 9e570949e..cdfeaa9d7 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesiness.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesiness.ll
@@ -6,7 +6,7 @@
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s --check-prefix=CHECK-LE \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 
-@glob = common local_unnamed_addr global i16 0, align 2
+@glob = local_unnamed_addr global i16 0, align 2
 
 define signext i32 @test_iness(i16 signext %a, i16 signext %b) {
 ; CHECK-LABEL: test_iness:
diff --git a/llvm/test/CodeGen/PowerPC/testComparesineuc.ll b/llvm/test/CodeGen/PowerPC/testComparesineuc.ll
index 4e7f5c05c..e63ea46b0 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesineuc.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesineuc.ll
@@ -5,7 +5,7 @@
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s --check-prefix=CHECK-LE \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
-@glob = common local_unnamed_addr global i8 0, align 1
+@glob = local_unnamed_addr global i8 0, align 1
 
 define signext i32 @test_ineuc(i8 zeroext %a, i8 zeroext %b) {
 ; CHECK-LABEL: test_ineuc:
diff --git a/llvm/test/CodeGen/PowerPC/testComparesineui.ll b/llvm/test/CodeGen/PowerPC/testComparesineui.ll
index e8fb3dd85..86e6fb9c7 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesineui.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesineui.ll
@@ -6,7 +6,7 @@
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s --check-prefix=CHECK-LE \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 
-@glob = common local_unnamed_addr global i32 0, align 4
+@glob = local_unnamed_addr global i32 0, align 4
 
 define signext i32 @test_ineui(i32 zeroext %a, i32 zeroext %b) {
 ; CHECK-LABEL: test_ineui:
diff --git a/llvm/test/CodeGen/PowerPC/testComparesineull.ll b/llvm/test/CodeGen/PowerPC/testComparesineull.ll
index e288988bd..c0b04c273 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesineull.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesineull.ll
@@ -6,7 +6,7 @@
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s --check-prefix=CHECK-LE \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 
-@glob = common local_unnamed_addr global i64 0, align 8
+@glob = local_unnamed_addr global i64 0, align 8
 
 define signext i32 @test_ineull(i64 %a, i64 %b) {
 ; CHECK-LABEL: test_ineull:
diff --git a/llvm/test/CodeGen/PowerPC/testComparesineus.ll b/llvm/test/CodeGen/PowerPC/testComparesineus.ll
index 4a2f851fa..df208eebf 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesineus.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesineus.ll
@@ -6,7 +6,7 @@
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s --check-prefix=CHECK-LE \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 
-@glob = common local_unnamed_addr global i16 0, align 2
+@glob = local_unnamed_addr global i16 0, align 2
 
 define signext i32 @test_ineus(i16 zeroext %a, i16 zeroext %b) {
 ; CHECK-LABEL: test_ineus:
diff --git a/llvm/test/CodeGen/PowerPC/testCompareslleqsc.ll b/llvm/test/CodeGen/PowerPC/testCompareslleqsc.ll
index 3e519edcd..7759f0ad9 100644
--- a/llvm/test/CodeGen/PowerPC/testCompareslleqsc.ll
+++ b/llvm/test/CodeGen/PowerPC/testCompareslleqsc.ll
@@ -7,7 +7,7 @@
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 ; ModuleID = 'ComparisonTestCases/testCompareslleqsc.c'
 
-@glob = common local_unnamed_addr global i8 0, align 1
+@glob = local_unnamed_addr global i8 0, align 1
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_lleqsc(i8 signext %a, i8 signext %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testCompareslleqsi.ll b/llvm/test/CodeGen/PowerPC/testCompareslleqsi.ll
index 4de1d5882..63612aec1 100644
--- a/llvm/test/CodeGen/PowerPC/testCompareslleqsi.ll
+++ b/llvm/test/CodeGen/PowerPC/testCompareslleqsi.ll
@@ -6,7 +6,7 @@
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s --check-prefix=CHECK-LE \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 
-@glob = common local_unnamed_addr global i32 0, align 4
+@glob = local_unnamed_addr global i32 0, align 4
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_lleqsi(i32 signext %a, i32 signext %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testCompareslleqsll.ll b/llvm/test/CodeGen/PowerPC/testCompareslleqsll.ll
index ac92d171a..18b8d0b24 100644
--- a/llvm/test/CodeGen/PowerPC/testCompareslleqsll.ll
+++ b/llvm/test/CodeGen/PowerPC/testCompareslleqsll.ll
@@ -6,7 +6,7 @@
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s --check-prefix=CHECK-LE \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 
-@glob = common local_unnamed_addr global i64 0, align 8
+@glob = local_unnamed_addr global i64 0, align 8
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_lleqsll(i64 %a, i64 %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testCompareslleqss.ll b/llvm/test/CodeGen/PowerPC/testCompareslleqss.ll
index d65990f9d..13e0759e8 100644
--- a/llvm/test/CodeGen/PowerPC/testCompareslleqss.ll
+++ b/llvm/test/CodeGen/PowerPC/testCompareslleqss.ll
@@ -6,7 +6,7 @@
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s --check-prefix=CHECK-LE \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 
-@glob = common local_unnamed_addr global i16 0, align 2
+@glob = local_unnamed_addr global i16 0, align 2
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_lleqss(i16 signext %a, i16 signext %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testComparesllequc.ll b/llvm/test/CodeGen/PowerPC/testComparesllequc.ll
index 1672d19a4..4c3c1d611 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesllequc.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesllequc.ll
@@ -6,7 +6,7 @@
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s --check-prefix=CHECK-LE \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 
-@glob = common local_unnamed_addr global i8 0, align 1
+@glob = local_unnamed_addr global i8 0, align 1
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llequc(i8 zeroext %a, i8 zeroext %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testComparesllequi.ll b/llvm/test/CodeGen/PowerPC/testComparesllequi.ll
index fd1d13bae..16741ea02 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesllequi.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesllequi.ll
@@ -6,7 +6,7 @@
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s --check-prefix=CHECK-LE \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 
-@glob = common local_unnamed_addr global i32 0, align 4
+@glob = local_unnamed_addr global i32 0, align 4
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llequi(i32 zeroext %a, i32 zeroext %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testComparesllequll.ll b/llvm/test/CodeGen/PowerPC/testComparesllequll.ll
index a42b5e4c7..16bd2d1a4 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesllequll.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesllequll.ll
@@ -6,7 +6,7 @@
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s --check-prefix=CHECK-LE \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 
-@glob = common local_unnamed_addr global i64 0, align 8
+@glob = local_unnamed_addr global i64 0, align 8
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llequll(i64 %a, i64 %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testComparesllequs.ll b/llvm/test/CodeGen/PowerPC/testComparesllequs.ll
index e3dde77ab..4470b4f95 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesllequs.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesllequs.ll
@@ -6,7 +6,7 @@
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s --check-prefix=CHECK-LE \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 
-@glob = common local_unnamed_addr global i16 0, align 2
+@glob = local_unnamed_addr global i16 0, align 2
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llequs(i16 zeroext %a, i16 zeroext %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testComparesllgesc.ll b/llvm/test/CodeGen/PowerPC/testComparesllgesc.ll
index b97da2503..f7c798596 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesllgesc.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesllgesc.ll
@@ -5,7 +5,7 @@
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s --check-prefix=CHECK-LE \
 ; RUN:   --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
-@glob = common local_unnamed_addr global i8 0, align 1
+@glob = local_unnamed_addr global i8 0, align 1
 
 define i64 @test_llgesc(i8 signext %a, i8 signext %b) {
 ; CHECK-LABEL: test_llgesc:
diff --git a/llvm/test/CodeGen/PowerPC/testComparesllgesi.ll b/llvm/test/CodeGen/PowerPC/testComparesllgesi.ll
index cc23babc0..046c037bb 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesllgesi.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesllgesi.ll
@@ -5,7 +5,7 @@
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s --check-prefix=CHECK-LE \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
-@glob = common local_unnamed_addr global i32 0, align 4
+@glob = local_unnamed_addr global i32 0, align 4
 
 define i64 @test_llgesi(i32 signext %a, i32 signext %b) {
 ; CHECK-LABEL: test_llgesi:
diff --git a/llvm/test/CodeGen/PowerPC/testComparesllgesll.ll b/llvm/test/CodeGen/PowerPC/testComparesllgesll.ll
index 1c119b950..2b1dc4a68 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesllgesll.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesllgesll.ll
@@ -5,7 +5,7 @@
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s --check-prefix=CHECK-LE \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
-@glob = common local_unnamed_addr global i64 0, align 8
+@glob = local_unnamed_addr global i64 0, align 8
 
 define i64 @test_llgesll(i64 %a, i64 %b) {
 ; CHECK-LABEL: test_llgesll:
diff --git a/llvm/test/CodeGen/PowerPC/testComparesllgess.ll b/llvm/test/CodeGen/PowerPC/testComparesllgess.ll
index 400c11bef..50583cd7f 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesllgess.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesllgess.ll
@@ -5,7 +5,7 @@
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s --check-prefix=CHECK-LE \
 ; RUN:   --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
-@glob = common local_unnamed_addr global i16 0, align 2
+@glob = local_unnamed_addr global i16 0, align 2
 
 define i64 @test_llgess(i16 signext %a, i16 signext %b) {
 ; CHECK-LABEL: test_llgess:
diff --git a/llvm/test/CodeGen/PowerPC/testComparesllgeuc.ll b/llvm/test/CodeGen/PowerPC/testComparesllgeuc.ll
index f1f50dd51..85a9df5c2 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesllgeuc.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesllgeuc.ll
@@ -1,112 +1,165 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
-; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
+; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
+; RUN:  --check-prefixes=CHECK,BE
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
-; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
+; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
+; RUN:  --check-prefixes=CHECK,LE
 
-@glob = common local_unnamed_addr global i8 0, align 1
+@glob = local_unnamed_addr global i8 0, align 1
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llgeuc(i8 zeroext %a, i8 zeroext %b) {
+; CHECK-LABEL: test_llgeuc:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    sub r3, r3, r4
+; CHECK-NEXT:    not r3, r3
+; CHECK-NEXT:    rldicl r3, r3, 1, 63
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp uge i8 %a, %b
   %conv3 = zext i1 %cmp to i64
   ret i64 %conv3
-; CHECK-LABEL: test_llgeuc:
-; CHECK: sub [[REG1:r[0-9]+]], r3, r4
-; CHECK-NEXT: not [[REG2:r[0-9]+]], [[REG1]]
-; CHECK-NEXT: rldicl r3, [[REG2]], 1, 63
-; CHECK: blr
 }
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llgeuc_sext(i8 zeroext %a, i8 zeroext %b) {
+; CHECK-LABEL: test_llgeuc_sext:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    sub r3, r3, r4
+; CHECK-NEXT:    rldicl r3, r3, 1, 63
+; CHECK-NEXT:    addi r3, r3, -1
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp uge i8 %a, %b
   %conv3 = sext i1 %cmp to i64
   ret i64 %conv3
-; CHECK-LABEL: @test_llgeuc_sext
-; CHECK: sub [[REG1:r[0-9]+]], r3, r4
-; CHECK-NEXT: rldicl [[REG2:r[0-9]+]], [[REG1]], 1, 63
-; CHECK-NEXT: addi [[REG3:r[0-9]+]], [[REG2]], -1
-; CHECK-NEXT: blr
 }
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llgeuc_z(i8 zeroext %a) {
+; CHECK-LABEL: test_llgeuc_z:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    li r3, 1
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp uge i8 %a, 0
   %conv1 = zext i1 %cmp to i64
   ret i64 %conv1
-; CHECK-LABEL: @test_llgeuc_z
-; CHECK: li r3, 1
-; CHECK: blr
 }
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llgeuc_sext_z(i8 zeroext %a) {
+; CHECK-LABEL: test_llgeuc_sext_z:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    li r3, -1
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp uge i8 %a, 0
   %conv1 = sext i1 %cmp to i64
   ret i64 %conv1
-; CHECK-LABEL: @test_llgeuc_sext_z
-; CHECK: li r3, -1
-; CHECK-NEXT: blr
 }
 
 ; Function Attrs: norecurse nounwind
 define void @test_llgeuc_store(i8 zeroext %a, i8 zeroext %b) {
+; BE-LABEL: test_llgeuc_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r5, r2, .LC0@toc@ha
+; BE-NEXT:    sub r3, r3, r4
+; BE-NEXT:    ld r4, .LC0@toc@l(r5)
+; BE-NEXT:    not r3, r3
+; BE-NEXT:    rldicl r3, r3, 1, 63
+; BE-NEXT:    stb r3, 0(r4)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_llgeuc_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    sub r3, r3, r4
+; LE-NEXT:    addis r5, r2, glob@toc@ha
+; LE-NEXT:    not r3, r3
+; LE-NEXT:    rldicl r3, r3, 1, 63
+; LE-NEXT:    stb r3, glob@toc@l(r5)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp uge i8 %a, %b
   %conv3 = zext i1 %cmp to i8
   store i8 %conv3, i8* @glob
   ret void
 ; CHECK_LABEL: test_llgeuc_store:
-; CHECK: sub [[REG1:r[0-9]+]], r3, r4
-; CHECK: not [[REG2:r[0-9]+]], [[REG1]]
-; CHECK-NEXT: rldicl r3, [[REG2]], 1, 63
-; CHECK: blr
 }
 
 ; Function Attrs: norecurse nounwind
 define void @test_llgeuc_sext_store(i8 zeroext %a, i8 zeroext %b) {
+; BE-LABEL: test_llgeuc_sext_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r5, r2, .LC0@toc@ha
+; BE-NEXT:    sub r3, r3, r4
+; BE-NEXT:    ld r4, .LC0@toc@l(r5)
+; BE-NEXT:    rldicl r3, r3, 1, 63
+; BE-NEXT:    addi r3, r3, -1
+; BE-NEXT:    stb r3, 0(r4)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_llgeuc_sext_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    sub r3, r3, r4
+; LE-NEXT:    addis r5, r2, glob@toc@ha
+; LE-NEXT:    rldicl r3, r3, 1, 63
+; LE-NEXT:    addi r3, r3, -1
+; LE-NEXT:    stb r3, glob@toc@l(r5)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp uge i8 %a, %b
   %conv3 = sext i1 %cmp to i8
   store i8 %conv3, i8* @glob
   ret void
-; CHECK-LABEL: @test_llgeuc_sext_store
-; CHECK: sub [[REG1:r[0-9]+]], r3, r4
-; CHECK: rldicl [[REG2:r[0-9]+]], [[REG1]], 1, 63
-; CHECK: addi [[REG3:r[0-9]+]], [[REG2]], -1
-; CHECK: stb  [[REG3]]
-; CHECK: blr
 }
 
 ; Function Attrs: norecurse nounwind
 define void @test_llgeuc_z_store(i8 zeroext %a) {
+; BE-LABEL: test_llgeuc_z_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r3, r2, .LC0@toc@ha
+; BE-NEXT:    li r4, 1
+; BE-NEXT:    ld r3, .LC0@toc@l(r3)
+; BE-NEXT:    stb r4, 0(r3)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_llgeuc_z_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    addis r3, r2, glob@toc@ha
+; LE-NEXT:    li r4, 1
+; LE-NEXT:    stb r4, glob@toc@l(r3)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp uge i8 %a, 0
   %conv1 = zext i1 %cmp to i8
   store i8 %conv1, i8* @glob
   ret void
-; CHECK-LABEL: @test_llgeuc_z_store
-; CHECK: li [[REG1:r[0-9]+]], 1
-; CHECK: stb [[REG1]]
-; CHECK: blr
 }
 
 ; Function Attrs: norecurse nounwind
 define void @test_llgeuc_sext_z_store(i8 zeroext %a) {
+; BE-LABEL: test_llgeuc_sext_z_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r3, r2, .LC0@toc@ha
+; BE-NEXT:    li r4, -1
+; BE-NEXT:    ld r3, .LC0@toc@l(r3)
+; BE-NEXT:    stb r4, 0(r3)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_llgeuc_sext_z_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    addis r3, r2, glob@toc@ha
+; LE-NEXT:    li r4, -1
+; LE-NEXT:    stb r4, glob@toc@l(r3)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp uge i8 %a, 0
   %conv1 = sext i1 %cmp to i8
   store i8 %conv1, i8* @glob
   ret void
-; CHECK-LABEL: @test_llgeuc_sext_z_store
-; CHECK: li [[REG1:r[0-9]+]], -1
-; CHECK: stb [[REG1]]
-; CHECK: blr
 }
 
diff --git a/llvm/test/CodeGen/PowerPC/testComparesllgeui.ll b/llvm/test/CodeGen/PowerPC/testComparesllgeui.ll
index 116cc9464..19da9da01 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesllgeui.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesllgeui.ll
@@ -1,112 +1,165 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
-; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
+; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
+; RUN:  --check-prefixes=CHECK,BE
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
-; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
+; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
+; RUN:  --check-prefixes=CHECK,LE
 
-@glob = common local_unnamed_addr global i32 0, align 4
+@glob = local_unnamed_addr global i32 0, align 4
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llgeui(i32 zeroext %a, i32 zeroext %b) {
+; CHECK-LABEL: test_llgeui:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    sub r3, r3, r4
+; CHECK-NEXT:    not r3, r3
+; CHECK-NEXT:    rldicl r3, r3, 1, 63
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp uge i32 %a, %b
   %conv1 = zext i1 %cmp to i64
   ret i64 %conv1
-; CHECK-LABEL: test_llgeui:
-; CHECK: sub [[REG1:r[0-9]+]], r3, r4
-; CHECK-NEXT: not [[REG2:r[0-9]+]], [[REG1]]
-; CHECK-NEXT: rldicl r3, [[REG2]], 1, 63
-; CHECK: blr
 }
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llgeui_sext(i32 zeroext %a, i32 zeroext %b) {
+; CHECK-LABEL: test_llgeui_sext:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    sub r3, r3, r4
+; CHECK-NEXT:    rldicl r3, r3, 1, 63
+; CHECK-NEXT:    addi r3, r3, -1
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp uge i32 %a, %b
   %conv1 = sext i1 %cmp to i64
   ret i64 %conv1
-; CHECK-LABEL: @test_llgeui_sext
-; CHECK: sub [[REG1:r[0-9]+]], r3, r4
-; CHECK-NEXT: rldicl [[REG2:r[0-9]+]], [[REG1]], 1, 63
-; CHECK-NEXT: addi [[REG3:r[0-9]+]], [[REG2]], -1
-; CHECK-NEXT: blr    
 }
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llgeui_z(i32 zeroext %a) {
+; CHECK-LABEL: test_llgeui_z:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    li r3, 1
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp uge i32 %a, 0
   %conv1 = zext i1 %cmp to i64
   ret i64 %conv1
-; CHECK-LABEL: @test_llgeui_z
-; CHECK: li r3, 1
-; CHECK: blr    
 }
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llgeui_sext_z(i32 zeroext %a) {
+; CHECK-LABEL: test_llgeui_sext_z:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    li r3, -1
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp uge i32 %a, 0
   %conv1 = sext i1 %cmp to i64
   ret i64 %conv1
-; CHECK-LABEL: @test_llgeui_sext_z
-; CHECK: li r3, -1
-; CHECK-NEXT: blr    
 }
 
 ; Function Attrs: norecurse nounwind
 define void @test_llgeui_store(i32 zeroext %a, i32 zeroext %b) {
+; BE-LABEL: test_llgeui_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r5, r2, .LC0@toc@ha
+; BE-NEXT:    sub r3, r3, r4
+; BE-NEXT:    ld r4, .LC0@toc@l(r5)
+; BE-NEXT:    not r3, r3
+; BE-NEXT:    rldicl r3, r3, 1, 63
+; BE-NEXT:    stw r3, 0(r4)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_llgeui_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    sub r3, r3, r4
+; LE-NEXT:    addis r5, r2, glob@toc@ha
+; LE-NEXT:    not r3, r3
+; LE-NEXT:    rldicl r3, r3, 1, 63
+; LE-NEXT:    stw r3, glob@toc@l(r5)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp uge i32 %a, %b
   %conv = zext i1 %cmp to i32
   store i32 %conv, i32* @glob
   ret void
 ; CHECK_LABEL: test_igeuc_store:
-; CHECK: sub [[REG1:r[0-9]+]], r3, r4
-; CHECK: not [[REG2:r[0-9]+]], [[REG1]]
-; CHECK-NEXT: rldicl r3, [[REG2]], 1, 63
-; CHECK: blr
 }
 
 ; Function Attrs: norecurse nounwind
 define void @test_llgeui_sext_store(i32 zeroext %a, i32 zeroext %b) {
+; BE-LABEL: test_llgeui_sext_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r5, r2, .LC0@toc@ha
+; BE-NEXT:    sub r3, r3, r4
+; BE-NEXT:    ld r4, .LC0@toc@l(r5)
+; BE-NEXT:    rldicl r3, r3, 1, 63
+; BE-NEXT:    addi r3, r3, -1
+; BE-NEXT:    stw r3, 0(r4)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_llgeui_sext_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    sub r3, r3, r4
+; LE-NEXT:    addis r5, r2, glob@toc@ha
+; LE-NEXT:    rldicl r3, r3, 1, 63
+; LE-NEXT:    addi r3, r3, -1
+; LE-NEXT:    stw r3, glob@toc@l(r5)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp uge i32 %a, %b
   %sub = sext i1 %cmp to i32
   store i32 %sub, i32* @glob
   ret void
-; CHECK-LABEL: @test_llgeui_sext_store
-; CHECK: sub [[REG1:r[0-9]+]], r3, r4
-; CHECK: rldicl [[REG2:r[0-9]+]], [[REG1]], 1, 63
-; CHECK: addi [[REG3:r[0-9]+]], [[REG2]], -1
-; CHECK: stw  [[REG3]]
-; CHECK: blr  
 }
 
 ; Function Attrs: norecurse nounwind
 define void @test_llgeui_z_store(i32 zeroext %a) {
+; BE-LABEL: test_llgeui_z_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r3, r2, .LC0@toc@ha
+; BE-NEXT:    li r4, 1
+; BE-NEXT:    ld r3, .LC0@toc@l(r3)
+; BE-NEXT:    stw r4, 0(r3)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_llgeui_z_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    addis r3, r2, glob@toc@ha
+; LE-NEXT:    li r4, 1
+; LE-NEXT:    stw r4, glob@toc@l(r3)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp uge i32 %a, 0
   %sub = zext i1 %cmp to i32
   store i32 %sub, i32* @glob
   ret void
-; CHECK-LABEL: @test_llgeui_z_store
-; CHECK: li [[REG1:r[0-9]+]], 1
-; CHECK: stw [[REG1]]
-; CHECK: blr  
 }
 
 ; Function Attrs: norecurse nounwind
 define void @test_llgeui_sext_z_store(i32 zeroext %a) {
+; BE-LABEL: test_llgeui_sext_z_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r3, r2, .LC0@toc@ha
+; BE-NEXT:    li r4, -1
+; BE-NEXT:    ld r3, .LC0@toc@l(r3)
+; BE-NEXT:    stw r4, 0(r3)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_llgeui_sext_z_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    addis r3, r2, glob@toc@ha
+; LE-NEXT:    li r4, -1
+; LE-NEXT:    stw r4, glob@toc@l(r3)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp uge i32 %a, 0
   %sub = sext i1 %cmp to i32
   store i32 %sub, i32* @glob
   ret void
-; CHECK-LABEL: @test_llgeui_sext_z_store
-; CHECK: li [[REG1:r[0-9]+]], -1
-; CHECK: stw [[REG1]]
-; CHECK: blr
 }
 
diff --git a/llvm/test/CodeGen/PowerPC/testComparesllgeull.ll b/llvm/test/CodeGen/PowerPC/testComparesllgeull.ll
index be1db4f75..68d908a4e 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesllgeull.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesllgeull.ll
@@ -1,110 +1,162 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
-; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
+; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
+; RUN:  --check-prefixes=CHECK,BE
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
-; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
+; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
+; RUN:  --check-prefixes=CHECK,LE
 
-@glob = common local_unnamed_addr global i64 0, align 8
+@glob = local_unnamed_addr global i64 0, align 8
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llgeull(i64 %a, i64 %b) {
+; CHECK-LABEL: test_llgeull:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    subfc r3, r4, r3
+; CHECK-NEXT:    subfe r3, r4, r4
+; CHECK-NEXT:    addi r3, r3, 1
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp uge i64 %a, %b
   %conv1 = zext i1 %cmp to i64
   ret i64 %conv1
-; CHECK-LABEL: test_llgeull:
-; CHECK: subfc {{r[0-9]+}}, r4, r3
-; CHECK-NEXT: subfe [[REG1:r[0-9]+]], r4, r4
-; CHECK-NEXT: addi r3, [[REG1]], 1
-; CHECK: blr
 }
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llgeull_sext(i64 %a, i64 %b) {
+; CHECK-LABEL: test_llgeull_sext:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    subfc r3, r4, r3
+; CHECK-NEXT:    subfe r3, r4, r4
+; CHECK-NEXT:    not r3, r3
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp uge i64 %a, %b
   %conv1 = sext i1 %cmp to i64
   ret i64 %conv1
-; CHECK-LABEL: @test_llgeull_sext
-; CHECK: subfc [[REG1:r[0-9]+]], r4, r3
-; CHECK: subfe [[REG2:r[0-9]+]], {{r[0-9]+}}, {{r[0-9]+}}
-; CHECK: not [[REG3:r[0-9]+]], [[REG2]]
-; CHECK: blr
 }
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llgeull_z(i64 %a) {
+; CHECK-LABEL: test_llgeull_z:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    li r3, 1
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp uge i64 %a, 0
   %conv1 = zext i1 %cmp to i64
   ret i64 %conv1
-; CHECK-LABEL: @test_llgeull_z
-; CHECK: li r3, 1
-; CHECK-NEXT: blr
 }
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llgeull_sext_z(i64 %a) {
+; CHECK-LABEL: test_llgeull_sext_z:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    li r3, -1
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp uge i64 %a, 0
   %conv1 = sext i1 %cmp to i64
   ret i64 %conv1
-; CHECK-LABEL: @test_llgeull_sext_z
-; CHECK: li r3, -1
-; CHECK-NEXT: blr
 }
 
 ; Function Attrs: norecurse nounwind
 define void @test_llgeull_store(i64 %a, i64 %b) {
+; BE-LABEL: test_llgeull_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r5, r2, .LC0@toc@ha
+; BE-NEXT:    subfc r3, r4, r3
+; BE-NEXT:    ld r3, .LC0@toc@l(r5)
+; BE-NEXT:    subfe r4, r4, r4
+; BE-NEXT:    addi r4, r4, 1
+; BE-NEXT:    std r4, 0(r3)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_llgeull_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    subfc r3, r4, r3
+; LE-NEXT:    addis r5, r2, glob@toc@ha
+; LE-NEXT:    subfe r3, r4, r4
+; LE-NEXT:    addi r3, r3, 1
+; LE-NEXT:    std r3, glob@toc@l(r5)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp uge i64 %a, %b
   %conv1 = zext i1 %cmp to i64
   store i64 %conv1, i64* @glob
   ret void
-; CHECK-LABEL: test_llgeull_store:
-; CHECK: subfc {{r[0-9]+}}, r4, r3
-; CHECK: subfe [[REG1:r[0-9]+]], r4, r4
-; CHECK: addi {{r[0-9]+}}, [[REG1]], 1
-; CHECK: blr
 }
 
 ; Function Attrs: norecurse nounwind
 define void @test_llgeull_sext_store(i64 %a, i64 %b) {
+; BE-LABEL: test_llgeull_sext_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r5, r2, .LC0@toc@ha
+; BE-NEXT:    subfc r3, r4, r3
+; BE-NEXT:    ld r3, .LC0@toc@l(r5)
+; BE-NEXT:    subfe r4, r4, r4
+; BE-NEXT:    not r4, r4
+; BE-NEXT:    std r4, 0(r3)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_llgeull_sext_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    subfc r3, r4, r3
+; LE-NEXT:    addis r5, r2, glob@toc@ha
+; LE-NEXT:    subfe r3, r4, r4
+; LE-NEXT:    not r3, r3
+; LE-NEXT:    std r3, glob@toc@l(r5)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp uge i64 %a, %b
   %conv1 = sext i1 %cmp to i64
   store i64 %conv1, i64* @glob
   ret void
-; CHECK-LABEL: @test_llgeull_sext_store
-; CHECK: subfc [[REG1:r[0-9]+]], r4, r3
-; CHECK: subfe [[REG2:r[0-9]+]], {{r[0-9]+}}, {{r[0-9]+}}
-; CHECK: not [[REG3:r[0-9]+]], [[REG2]]
-; CHECK: std [[REG3]]
-; CHECK: blr
 }
 
 ; Function Attrs: norecurse nounwind
 define void @test_llgeull_z_store(i64 %a) {
+; BE-LABEL: test_llgeull_z_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r3, r2, .LC0@toc@ha
+; BE-NEXT:    li r4, 1
+; BE-NEXT:    ld r3, .LC0@toc@l(r3)
+; BE-NEXT:    std r4, 0(r3)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_llgeull_z_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    addis r3, r2, glob@toc@ha
+; LE-NEXT:    li r4, 1
+; LE-NEXT:    std r4, glob@toc@l(r3)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp uge i64 %a, 0
   %conv1 = zext i1 %cmp to i64
   store i64 %conv1, i64* @glob
   ret void
-; CHECK-LABEL: @test_llgeull_z_store
-; CHECK: li [[REG1:r[0-9]+]], 1
-; CHECK: std [[REG1]]
-; CHECK: blr
 }
 
 ; Function Attrs: norecurse nounwind
 define void @test_llgeull_sext_z_store(i64 %a) {
+; BE-LABEL: test_llgeull_sext_z_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r3, r2, .LC0@toc@ha
+; BE-NEXT:    li r4, -1
+; BE-NEXT:    ld r3, .LC0@toc@l(r3)
+; BE-NEXT:    std r4, 0(r3)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_llgeull_sext_z_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    addis r3, r2, glob@toc@ha
+; LE-NEXT:    li r4, -1
+; LE-NEXT:    std r4, glob@toc@l(r3)
+; LE-NEXT:    blr
 entry:
   store i64 -1, i64* @glob
   ret void
-; CHECK-LABEL: @test_llgeull_sext_z_store
-; CHECK: li [[REG1:r[0-9]+]], -1
-; CHECK: std [[REG1]]
-; CHECK: blr
 }
 
diff --git a/llvm/test/CodeGen/PowerPC/testComparesllgeus.ll b/llvm/test/CodeGen/PowerPC/testComparesllgeus.ll
index 67d6d1906..56099c63d 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesllgeus.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesllgeus.ll
@@ -1,112 +1,165 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
-; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
+; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
+; RUN:  --check-prefixes=CHECK,BE
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
-; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
+; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
+; RUN:  --check-prefixes=CHECK,LE
 
-@glob = common local_unnamed_addr global i16 0, align 2
+@glob = local_unnamed_addr global i16 0, align 2
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llgeus(i16 zeroext %a, i16 zeroext %b) {
+; CHECK-LABEL: test_llgeus:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    sub r3, r3, r4
+; CHECK-NEXT:    not r3, r3
+; CHECK-NEXT:    rldicl r3, r3, 1, 63
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp uge i16 %a, %b
   %conv3 = zext i1 %cmp to i64
   ret i64 %conv3
-; CHECK-LABEL: test_llgeus:
-; CHECK: sub [[REG1:r[0-9]+]], r3, r4
-; CHECK-NEXT: not [[REG2:r[0-9]+]], [[REG1]]
-; CHECK-NEXT: rldicl r3, [[REG2]], 1, 63
-; CHECK: blr
 }
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llgeus_sext(i16 zeroext %a, i16 zeroext %b) {
+; CHECK-LABEL: test_llgeus_sext:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    sub r3, r3, r4
+; CHECK-NEXT:    rldicl r3, r3, 1, 63
+; CHECK-NEXT:    addi r3, r3, -1
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp uge i16 %a, %b
   %conv3 = sext i1 %cmp to i64
   ret i64 %conv3
-; CHECK-LABEL: @test_llgeus_sext
-; CHECK: sub [[REG1:r[0-9]+]], r3, r4
-; CHECK-NEXT: rldicl [[REG2:r[0-9]+]], [[REG1]], 1, 63
-; CHECK-NEXT: addi [[REG3:r[0-9]+]], [[REG2]], -1
-; CHECK-NEXT: blr
 }
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llgeus_z(i16 zeroext %a) {
+; CHECK-LABEL: test_llgeus_z:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    li r3, 1
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp uge i16 %a, 0
   %conv1 = zext i1 %cmp to i64
   ret i64 %conv1
-; CHECK-LABEL: @test_llgeus_z
-; CHECK: li r3, 1
-; CHECK-NEXT: blr
 }
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llgeus_sext_z(i16 zeroext %a) {
+; CHECK-LABEL: test_llgeus_sext_z:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    li r3, -1
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp uge i16 %a, 0
   %conv1 = sext i1 %cmp to i64
   ret i64 %conv1
-; CHECK-LABEL: @test_llgeus_sext_z
-; CHECK: li r3, -1
-; CHECK: blr
 }
 
 ; Function Attrs: norecurse nounwind
 define void @test_llgeus_store(i16 zeroext %a, i16 zeroext %b) {
+; BE-LABEL: test_llgeus_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r5, r2, .LC0@toc@ha
+; BE-NEXT:    sub r3, r3, r4
+; BE-NEXT:    ld r4, .LC0@toc@l(r5)
+; BE-NEXT:    not r3, r3
+; BE-NEXT:    rldicl r3, r3, 1, 63
+; BE-NEXT:    sth r3, 0(r4)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_llgeus_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    sub r3, r3, r4
+; LE-NEXT:    addis r5, r2, glob@toc@ha
+; LE-NEXT:    not r3, r3
+; LE-NEXT:    rldicl r3, r3, 1, 63
+; LE-NEXT:    sth r3, glob@toc@l(r5)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp uge i16 %a, %b
   %conv3 = zext i1 %cmp to i16
   store i16 %conv3, i16* @glob
   ret void
 ; CHECK_LABEL: test_llgeus_store:
-; CHECK: sub [[REG1:r[0-9]+]], r3, r4
-; CHECK: not [[REG2:r[0-9]+]], [[REG1]]
-; CHECK-NEXT: rldicl r3, [[REG2]], 1, 63
-; CHECK: blr
 }
 
 ; Function Attrs: norecurse nounwind
 define void @test_llgeus_sext_store(i16 zeroext %a, i16 zeroext %b) {
+; BE-LABEL: test_llgeus_sext_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r5, r2, .LC0@toc@ha
+; BE-NEXT:    sub r3, r3, r4
+; BE-NEXT:    ld r4, .LC0@toc@l(r5)
+; BE-NEXT:    rldicl r3, r3, 1, 63
+; BE-NEXT:    addi r3, r3, -1
+; BE-NEXT:    sth r3, 0(r4)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_llgeus_sext_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    sub r3, r3, r4
+; LE-NEXT:    addis r5, r2, glob@toc@ha
+; LE-NEXT:    rldicl r3, r3, 1, 63
+; LE-NEXT:    addi r3, r3, -1
+; LE-NEXT:    sth r3, glob@toc@l(r5)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp uge i16 %a, %b
   %conv3 = sext i1 %cmp to i16
   store i16 %conv3, i16* @glob
   ret void
-; CHECK-LABEL: @test_llgeus_sext_store
-; CHECK: sub [[REG1:r[0-9]+]], r3, r4
-; CHECK: rldicl [[REG2:r[0-9]+]], [[REG1]], 1, 63
-; CHECK: addi [[REG3:r[0-9]+]], [[REG2]], -1
-; CHECK: sth  [[REG3]]
-; CHECK: blr    
 }
 
 ; Function Attrs: norecurse nounwind
 define void @test_llgeus_z_store(i16 zeroext %a) {
+; BE-LABEL: test_llgeus_z_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r3, r2, .LC0@toc@ha
+; BE-NEXT:    li r4, 1
+; BE-NEXT:    ld r3, .LC0@toc@l(r3)
+; BE-NEXT:    sth r4, 0(r3)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_llgeus_z_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    addis r3, r2, glob@toc@ha
+; LE-NEXT:    li r4, 1
+; LE-NEXT:    sth r4, glob@toc@l(r3)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp uge i16 %a, 0
   %conv1 = zext i1 %cmp to i16
   store i16 %conv1, i16* @glob
   ret void
-; CHECK-LABEL: @test_llgeus_z_store
-; CHECK: li [[REG1:r[0-9]+]], 1
-; CHECK: sth [[REG1]]
-; CHECK: blr  
 }
 
 ; Function Attrs: norecurse nounwind
 define void @test_llgeus_sext_z_store(i16 zeroext %a) {
+; BE-LABEL: test_llgeus_sext_z_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r3, r2, .LC0@toc@ha
+; BE-NEXT:    li r4, -1
+; BE-NEXT:    ld r3, .LC0@toc@l(r3)
+; BE-NEXT:    sth r4, 0(r3)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_llgeus_sext_z_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    addis r3, r2, glob@toc@ha
+; LE-NEXT:    li r4, -1
+; LE-NEXT:    sth r4, glob@toc@l(r3)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp uge i16 %a, 0
   %conv1 = sext i1 %cmp to i16
   store i16 %conv1, i16* @glob
   ret void
-; CHECK-LABEL: @test_llgeus_sext_z_store
-; CHECK: li [[REG1:r[0-9]+]], -1
-; CHECK: sth [[REG1]]
-; CHECK: blr  
 }
 
diff --git a/llvm/test/CodeGen/PowerPC/testComparesllgtsll.ll b/llvm/test/CodeGen/PowerPC/testComparesllgtsll.ll
index deb9ad562..6f4c9b00b 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesllgtsll.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesllgtsll.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -relocation-model=pic -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
@@ -5,17 +6,17 @@
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 
-@glob = common local_unnamed_addr global i64 0, align 8
+@glob = local_unnamed_addr global i64 0, align 8
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llgtsll(i64 %a, i64 %b) {
 ; CHECK-LABEL: test_llgtsll:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    sradi [[REG1:r[0-9]+]], r4, 63
-; CHECK-NEXT:    rldicl [[REG2:r[0-9]+]], r3, 1, 63
-; CHECK-NEXT:    subfc [[REG3:r[0-9]+]], r3, r4
-; CHECK-NEXT:    adde [[REG4:r[0-9]+]], [[REG2]], [[REG1]]
-; CHECK-NEXT:    xori r3, [[REG4]], 1
+; CHECK-NEXT:    sradi r5, r4, 63
+; CHECK-NEXT:    rldicl r6, r3, 1, 63
+; CHECK-NEXT:    subfc r3, r3, r4
+; CHECK-NEXT:    adde r3, r6, r5
+; CHECK-NEXT:    xori r3, r3, 1
 ; CHECK-NEXT:    blr
 entry:
   %cmp = icmp sgt i64 %a, %b
@@ -27,12 +28,12 @@ entry:
 define i64 @test_llgtsll_sext(i64 %a, i64 %b) {
 ; CHECK-LABEL: test_llgtsll_sext:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    sradi [[REG1:r[0-9]+]], r4, 63
-; CHECK-NEXT:    rldicl [[REG2:r[0-9]+]], r3, 1, 63
-; CHECK-NEXT:    subfc [[REG3:r[0-9]+]], r3, r4
-; CHECK-NEXT:    adde [[REG4:r[0-9]+]], [[REG2]], [[REG1]]
-; CHECK-NEXT:    xori [[REG5:r[0-9]+]], [[REG4]], 1
-; CHECK-NEXT:    neg r3, [[REG5]]
+; CHECK-NEXT:    sradi r5, r4, 63
+; CHECK-NEXT:    rldicl r6, r3, 1, 63
+; CHECK-NEXT:    subfc r3, r3, r4
+; CHECK-NEXT:    adde r3, r6, r5
+; CHECK-NEXT:    xori r3, r3, 1
+; CHECK-NEXT:    neg r3, r3
 ; CHECK-NEXT:    blr
 entry:
   %cmp = icmp sgt i64 %a, %b
@@ -58,9 +59,11 @@ entry:
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llgtsll_sext_z(i64 %a) {
 ; CHECK-LABEL: test_llgtsll_sext_z:
-; CHECK:       addi [[REG1:r[0-9]+]], r3, -1
-; CHECK-NEXT:  nor [[REG2:r[0-9]+]], [[REG1]], r3
-; CHECK-NEXT:  sradi r3, [[REG2]], 63
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addi r4, r3, -1
+; CHECK-NEXT:    nor r3, r4, r3
+; CHECK-NEXT:    sradi r3, r3, 63
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp sgt i64 %a, 0
   %conv1 = sext i1 %cmp to i64
@@ -71,12 +74,16 @@ entry:
 define void @test_llgtsll_store(i64 %a, i64 %b) {
 ; CHECK-LABEL: test_llgtsll_store:
 ; CHECK:       # %bb.0: # %entry
-; CHECK:         sradi [[REG1:r[0-9]+]], r4, 63
-; CHECK:         rldicl [[REG2:r[0-9]+]], r3, 1, 63
+; CHECK-NEXT:    sradi r6, r4, 63
+; CHECK-NEXT:    addis r5, r2, .LC0@toc@ha
+; CHECK-NEXT:    subfc r4, r3, r4
+; CHECK-NEXT:    rldicl r3, r3, 1, 63
+; CHECK-NEXT:    ld r4, .LC0@toc@l(r5)
+; CHECK-NEXT:    adde r3, r3, r6
+; CHECK-NEXT:    xori r3, r3, 1
+; CHECK-NEXT:    std r3, 0(r4)
+; CHECK-NEXT:    blr
 ; CHECK-DIAG:    subfc [[REG3:r[0-9]+]], r3, r4
-; CHECK:         adde [[REG4:r[0-9]+]], [[REG2]], [[REG1]]
-; CHECK:         xori [[REG5:r[0-9]+]], [[REG4]], 1
-; CHECK-NOT:     neg 
 entry:
   %cmp = icmp sgt i64 %a, %b
   %conv1 = zext i1 %cmp to i64
@@ -88,12 +95,17 @@ entry:
 define void @test_llgtsll_sext_store(i64 %a, i64 %b) {
 ; CHECK-LABEL: test_llgtsll_sext_store:
 ; CHECK:       # %bb.0: # %entry
-; CHECK:         sradi [[REG1:r[0-9]+]], r4, 63
-; CHECK:         rldicl [[REG2:r[0-9]+]], r3, 1, 63
+; CHECK-NEXT:    sradi r6, r4, 63
+; CHECK-NEXT:    addis r5, r2, .LC0@toc@ha
+; CHECK-NEXT:    subfc r4, r3, r4
+; CHECK-NEXT:    rldicl r3, r3, 1, 63
+; CHECK-NEXT:    adde r3, r3, r6
+; CHECK-NEXT:    ld r4, .LC0@toc@l(r5)
+; CHECK-NEXT:    xori r3, r3, 1
+; CHECK-NEXT:    neg r3, r3
+; CHECK-NEXT:    std r3, 0(r4)
+; CHECK-NEXT:    blr
 ; CHECK-DIAG:    subfc [[REG3:r[0-9]+]], r3, r4
-; CHECK:         adde [[REG4:r[0-9]+]], [[REG2]], [[REG1]]
-; CHECK:         xori [[REG5:r[0-9]+]], [[REG4]], 1
-; CHECK:         neg {{r[0-9]+}}, [[REG5]]
 entry:
   %cmp = icmp sgt i64 %a, %b
   %conv1 = sext i1 %cmp to i64
@@ -123,9 +135,14 @@ entry:
 ; Function Attrs: norecurse nounwind
 define void @test_llgtsll_sext_z_store(i64 %a) {
 ; CHECK-LABEL: test_llgtsll_sext_z_store:
-; CHECK:    addi [[REG1:r[0-9]+]], r3, -1
-; CHECK:    nor [[REG2:r[0-9]+]], [[REG1]], r3
-; CHECK:    sradi [[REG3:r[0-9]+]], [[REG2]], 63
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addis r4, r2, .LC0@toc@ha
+; CHECK-NEXT:    addi r5, r3, -1
+; CHECK-NEXT:    ld r4, .LC0@toc@l(r4)
+; CHECK-NEXT:    nor r3, r5, r3
+; CHECK-NEXT:    sradi r3, r3, 63
+; CHECK-NEXT:    std r3, 0(r4)
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp sgt i64 %a, 0
   %conv1 = sext i1 %cmp to i64
diff --git a/llvm/test/CodeGen/PowerPC/testComparesllgtuc.ll b/llvm/test/CodeGen/PowerPC/testComparesllgtuc.ll
index 685026ad0..e535d30d2 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesllgtuc.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesllgtuc.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -relocation-model=pic -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
@@ -5,13 +6,14 @@
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 
-@glob = common local_unnamed_addr global i8 0, align 1
+@glob = local_unnamed_addr global i8 0, align 1
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llgtuc(i8 zeroext %a, i8 zeroext %b) {
 ; CHECK-LABEL: test_llgtuc:
-; CHECK:    sub [[REG:r[0-9]+]], r4, r3
-; CHECK:    rldicl r3, [[REG]], 1, 63
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    sub r3, r4, r3
+; CHECK-NEXT:    rldicl r3, r3, 1, 63
 ; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ugt i8 %a, %b
@@ -22,8 +24,9 @@ entry:
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llgtuc_sext(i8 zeroext %a, i8 zeroext %b) {
 ; CHECK-LABEL: test_llgtuc_sext:
-; CHECK:    sub [[REG:r[0-9]+]], r4, r3
-; CHECK:    sradi r3, [[REG]], 63
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    sub r3, r4, r3
+; CHECK-NEXT:    sradi r3, r3, 63
 ; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ugt i8 %a, %b
@@ -34,9 +37,10 @@ entry:
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llgtuc_z(i8 zeroext %a) {
 ; CHECK-LABEL: test_llgtuc_z:
-; CHECK:    cntlzw r3, r3
-; CHECK:    srwi r3, r3, 5
-; CHECK:    xori r3, r3, 1
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cntlzw r3, r3
+; CHECK-NEXT:    srwi r3, r3, 5
+; CHECK-NEXT:    xori r3, r3, 1
 ; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ne i8 %a, 0
@@ -47,10 +51,11 @@ entry:
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llgtuc_sext_z(i8 zeroext %a) {
 ; CHECK-LABEL: test_llgtuc_sext_z:
-; CHECK:    cntlzw r3, r3
-; CHECK:    srwi r3, r3, 5
-; CHECK:    xori r3, r3, 1
-; CHECK:    neg r3, r3
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cntlzw r3, r3
+; CHECK-NEXT:    srwi r3, r3, 5
+; CHECK-NEXT:    xori r3, r3, 1
+; CHECK-NEXT:    neg r3, r3
 ; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ne i8 %a, 0
@@ -61,8 +66,13 @@ entry:
 ; Function Attrs: norecurse nounwind
 define void @test_llgtuc_store(i8 zeroext %a, i8 zeroext %b) {
 ; CHECK-LABEL: test_llgtuc_store:
-; CHECK:         sub [[REG:r[0-9]+]], r4, r3
-; CHECK:         rldicl {{r[0-9]+}}, [[REG]], 1, 63
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addis r5, r2, .LC0@toc@ha
+; CHECK-NEXT:    sub r3, r4, r3
+; CHECK-NEXT:    ld r5, .LC0@toc@l(r5)
+; CHECK-NEXT:    rldicl r3, r3, 1, 63
+; CHECK-NEXT:    stb r3, 0(r5)
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ugt i8 %a, %b
   %conv3 = zext i1 %cmp to i8
@@ -73,8 +83,13 @@ entry:
 ; Function Attrs: norecurse nounwind
 define void @test_llgtuc_sext_store(i8 zeroext %a, i8 zeroext %b) {
 ; CHECK-LABEL: test_llgtuc_sext_store:
-; CHECK:         sub [[REG:r[0-9]+]], r4, r3
-; CHECK:         sradi {{r[0-9]+}}, [[REG]], 63
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addis r5, r2, .LC0@toc@ha
+; CHECK-NEXT:    sub r3, r4, r3
+; CHECK-NEXT:    ld r5, .LC0@toc@l(r5)
+; CHECK-NEXT:    sradi r3, r3, 63
+; CHECK-NEXT:    stb r3, 0(r5)
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ugt i8 %a, %b
   %conv3 = sext i1 %cmp to i8
@@ -85,10 +100,13 @@ entry:
 ; Function Attrs: norecurse nounwind
 define void @test_llgtuc_z_store(i8 zeroext %a) {
 ; CHECK-LABEL: test_llgtuc_z_store:
-; CHECK:    cntlzw r3, r3
-; CHECK:    srwi r3, r3, 5
-; CHECK:    xori r3, r3, 1
-; CHECK:    stb r3, 0(r4)
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addis r4, r2, .LC0@toc@ha
+; CHECK-NEXT:    cntlzw r3, r3
+; CHECK-NEXT:    ld r4, .LC0@toc@l(r4)
+; CHECK-NEXT:    srwi r3, r3, 5
+; CHECK-NEXT:    xori r3, r3, 1
+; CHECK-NEXT:    stb r3, 0(r4)
 ; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ne i8 %a, 0
@@ -100,11 +118,14 @@ entry:
 ; Function Attrs: norecurse nounwind
 define void @test_llgtuc_sext_z_store(i8 zeroext %a) {
 ; CHECK-LABEL: test_llgtuc_sext_z_store:
-; CHECK:    cntlzw r3, r3
-; CHECK:    srwi r3, r3, 5
-; CHECK:    xori r3, r3, 1
-; CHECK:    neg r3, r3
-; CHECK:    stb r3, 0(r4)
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addis r4, r2, .LC0@toc@ha
+; CHECK-NEXT:    cntlzw r3, r3
+; CHECK-NEXT:    srwi r3, r3, 5
+; CHECK-NEXT:    ld r4, .LC0@toc@l(r4)
+; CHECK-NEXT:    xori r3, r3, 1
+; CHECK-NEXT:    neg r3, r3
+; CHECK-NEXT:    stb r3, 0(r4)
 ; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ne i8 %a, 0
diff --git a/llvm/test/CodeGen/PowerPC/testComparesllgtui.ll b/llvm/test/CodeGen/PowerPC/testComparesllgtui.ll
index a1c8777e3..3c6b2f1c2 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesllgtui.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesllgtui.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -relocation-model=pic -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
@@ -5,14 +6,15 @@
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 
-@glob = common local_unnamed_addr global i32 0, align 4
+@glob = local_unnamed_addr global i32 0, align 4
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llgtui(i32 zeroext %a, i32 zeroext %b) {
 ; CHECK-LABEL: test_llgtui:
-; CHECK-NOT:     clrldi
-; CHECK:    sub [[REG:r[0-9]+]], r4, r3
-; CHECK-NEXT:    rldicl r3, [[REG]], 1, 63
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    sub r3, r4, r3
+; CHECK-NEXT:    rldicl r3, r3, 1, 63
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ugt i32 %a, %b
   %conv1 = zext i1 %cmp to i64
@@ -22,8 +24,9 @@ entry:
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llgtui_sext(i32 zeroext %a, i32 zeroext %b) {
 ; CHECK-LABEL: test_llgtui_sext:
-; CHECK:    sub [[REG:r[0-9]+]], r4, r3
-; CHECK-NEXT:    sradi r3, [[REG]], 63
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    sub r3, r4, r3
+; CHECK-NEXT:    sradi r3, r3, 63
 ; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ugt i32 %a, %b
@@ -34,7 +37,8 @@ entry:
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llgtui_z(i32 zeroext %a) {
 ; CHECK-LABEL: test_llgtui_z:
-; CHECK:    cntlzw r3, r3
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cntlzw r3, r3
 ; CHECK-NEXT:    srwi r3, r3, 5
 ; CHECK-NEXT:    xori r3, r3, 1
 ; CHECK-NEXT:    blr
@@ -47,7 +51,8 @@ entry:
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llgtui_sext_z(i32 zeroext %a) {
 ; CHECK-LABEL: test_llgtui_sext_z:
-; CHECK:    cntlzw r3, r3
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cntlzw r3, r3
 ; CHECK-NEXT:    srwi r3, r3, 5
 ; CHECK-NEXT:    xori r3, r3, 1
 ; CHECK-NEXT:    neg r3, r3
@@ -61,8 +66,13 @@ entry:
 ; Function Attrs: norecurse nounwind
 define void @test_llgtui_store(i32 zeroext %a, i32 zeroext %b) {
 ; CHECK-LABEL: test_llgtui_store:
-; CHECK:         sub [[REG:r[0-9]+]], r4, r3
-; CHECK:         rldicl {{r[0-9]+}}, [[REG]], 1, 63
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addis r5, r2, .LC0@toc@ha
+; CHECK-NEXT:    sub r3, r4, r3
+; CHECK-NEXT:    ld r5, .LC0@toc@l(r5)
+; CHECK-NEXT:    rldicl r3, r3, 1, 63
+; CHECK-NEXT:    stw r3, 0(r5)
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ugt i32 %a, %b
   %conv = zext i1 %cmp to i32
@@ -73,8 +83,13 @@ entry:
 ; Function Attrs: norecurse nounwind
 define void @test_llgtui_sext_store(i32 zeroext %a, i32 zeroext %b) {
 ; CHECK-LABEL: test_llgtui_sext_store:
-; CHECK:         sub [[REG:r[0-9]+]], r4, r3
-; CHECK:         sradi {{r[0-9]+}}, [[REG]], 63
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addis r5, r2, .LC0@toc@ha
+; CHECK-NEXT:    sub r3, r4, r3
+; CHECK-NEXT:    ld r5, .LC0@toc@l(r5)
+; CHECK-NEXT:    sradi r3, r3, 63
+; CHECK-NEXT:    stw r3, 0(r5)
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ugt i32 %a, %b
   %sub = sext i1 %cmp to i32
@@ -85,10 +100,13 @@ entry:
 ; Function Attrs: norecurse nounwind
 define void @test_llgtui_z_store(i32 zeroext %a) {
 ; CHECK-LABEL: test_llgtui_z_store:
-; CHECK:    cntlzw r3, r3
-; CHECK:    srwi r3, r3, 5
-; CHECK:    xori r3, r3, 1
-; CHECK:    stw r3, 0(r4)
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addis r4, r2, .LC0@toc@ha
+; CHECK-NEXT:    cntlzw r3, r3
+; CHECK-NEXT:    ld r4, .LC0@toc@l(r4)
+; CHECK-NEXT:    srwi r3, r3, 5
+; CHECK-NEXT:    xori r3, r3, 1
+; CHECK-NEXT:    stw r3, 0(r4)
 ; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ne i32 %a, 0
@@ -100,11 +118,14 @@ entry:
 ; Function Attrs: norecurse nounwind
 define void @test_llgtui_sext_z_store(i32 zeroext %a) {
 ; CHECK-LABEL: test_llgtui_sext_z_store:
-; CHECK:    cntlzw r3, r3
-; CHECK:    srwi r3, r3, 5
-; CHECK:    xori r3, r3, 1
-; CHECK:    neg r3, r3
-; CHECK:    stw r3, 0(r4)
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addis r4, r2, .LC0@toc@ha
+; CHECK-NEXT:    cntlzw r3, r3
+; CHECK-NEXT:    srwi r3, r3, 5
+; CHECK-NEXT:    ld r4, .LC0@toc@l(r4)
+; CHECK-NEXT:    xori r3, r3, 1
+; CHECK-NEXT:    neg r3, r3
+; CHECK-NEXT:    stw r3, 0(r4)
 ; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ne i32 %a, 0
diff --git a/llvm/test/CodeGen/PowerPC/testComparesllgtus.ll b/llvm/test/CodeGen/PowerPC/testComparesllgtus.ll
index 94bb14546..1dbd9ca80 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesllgtus.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesllgtus.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -relocation-model=pic -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
@@ -5,14 +6,14 @@
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 
-@glob = common local_unnamed_addr global i16 0, align 2
+@glob = local_unnamed_addr global i16 0, align 2
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llgtus(i16 zeroext %a, i16 zeroext %b) {
 ; CHECK-LABEL: test_llgtus:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    sub [[REG:r[0-9]+]], r4, r3
-; CHECK-NEXT:    rldicl r3, [[REG]], 1, 63
+; CHECK-NEXT:    sub r3, r4, r3
+; CHECK-NEXT:    rldicl r3, r3, 1, 63
 ; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ugt i16 %a, %b
@@ -24,8 +25,8 @@ entry:
 define i64 @test_llgtus_sext(i16 zeroext %a, i16 zeroext %b) {
 ; CHECK-LABEL: test_llgtus_sext:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    sub [[REG:r[0-9]+]], r4, r3
-; CHECK-NEXT:    sradi r3, [[REG]], 63
+; CHECK-NEXT:    sub r3, r4, r3
+; CHECK-NEXT:    sradi r3, r3, 63
 ; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ugt i16 %a, %b
@@ -66,8 +67,12 @@ entry:
 define void @test_llgtus_store(i16 zeroext %a, i16 zeroext %b) {
 ; CHECK-LABEL: test_llgtus_store:
 ; CHECK:       # %bb.0: # %entry
-; CHECK:         sub [[REG:r[0-9]+]], r4, r3
-; CHECK:         rldicl {{r[0-9]+}}, [[REG]], 1, 63
+; CHECK-NEXT:    addis r5, r2, .LC0@toc@ha
+; CHECK-NEXT:    sub r3, r4, r3
+; CHECK-NEXT:    ld r5, .LC0@toc@l(r5)
+; CHECK-NEXT:    rldicl r3, r3, 1, 63
+; CHECK-NEXT:    sth r3, 0(r5)
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ugt i16 %a, %b
   %conv3 = zext i1 %cmp to i16
@@ -79,8 +84,12 @@ entry:
 define void @test_llgtus_sext_store(i16 zeroext %a, i16 zeroext %b) {
 ; CHECK-LABEL: test_llgtus_sext_store:
 ; CHECK:       # %bb.0: # %entry
-; CHECK:         sub [[REG:r[0-9]+]], r4, r3
-; CHECK:         sradi {{r[0-9]+}}, [[REG]], 63
+; CHECK-NEXT:    addis r5, r2, .LC0@toc@ha
+; CHECK-NEXT:    sub r3, r4, r3
+; CHECK-NEXT:    ld r5, .LC0@toc@l(r5)
+; CHECK-NEXT:    sradi r3, r3, 63
+; CHECK-NEXT:    sth r3, 0(r5)
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ugt i16 %a, %b
   %conv3 = sext i1 %cmp to i16
diff --git a/llvm/test/CodeGen/PowerPC/testCompareslllesc.ll b/llvm/test/CodeGen/PowerPC/testCompareslllesc.ll
index 1a5ad44ee..1ec226a86 100644
--- a/llvm/test/CodeGen/PowerPC/testCompareslllesc.ll
+++ b/llvm/test/CodeGen/PowerPC/testCompareslllesc.ll
@@ -6,7 +6,7 @@
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s --check-prefix=CHECK-LE \
 ; RUN:   --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 
-@glob = common local_unnamed_addr global i8 0, align 1
+@glob = local_unnamed_addr global i8 0, align 1
 
 define i64 @test_lllesc(i8 signext %a, i8 signext %b) {
 ; CHECK-LABEL: test_lllesc:
diff --git a/llvm/test/CodeGen/PowerPC/testCompareslllesi.ll b/llvm/test/CodeGen/PowerPC/testCompareslllesi.ll
index 9b79c1739..3ff490774 100644
--- a/llvm/test/CodeGen/PowerPC/testCompareslllesi.ll
+++ b/llvm/test/CodeGen/PowerPC/testCompareslllesi.ll
@@ -6,7 +6,7 @@
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s --check-prefix=CHECK-LE \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 
-@glob = common local_unnamed_addr global i32 0, align 4
+@glob = local_unnamed_addr global i32 0, align 4
 
 define i64 @test_lllesi(i32 signext %a, i32 signext %b)  {
 ; CHECK-LABEL: test_lllesi:
diff --git a/llvm/test/CodeGen/PowerPC/testCompareslllesll.ll b/llvm/test/CodeGen/PowerPC/testCompareslllesll.ll
index 83ce2f812..ca7ce5e56 100644
--- a/llvm/test/CodeGen/PowerPC/testCompareslllesll.ll
+++ b/llvm/test/CodeGen/PowerPC/testCompareslllesll.ll
@@ -5,7 +5,7 @@
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s --check-prefix=CHECK-LE \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
-@glob = common local_unnamed_addr global i64 0, align 8
+@glob = local_unnamed_addr global i64 0, align 8
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_lllesll(i64 %a, i64 %b)  {
diff --git a/llvm/test/CodeGen/PowerPC/testComparesllless.ll b/llvm/test/CodeGen/PowerPC/testComparesllless.ll
index 6f8657dac..c67ef49e8 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesllless.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesllless.ll
@@ -6,7 +6,7 @@
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s --check-prefix=CHECK-LE \
 ; RUN:   --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 
-@glob = common local_unnamed_addr global i16 0, align 2
+@glob = local_unnamed_addr global i16 0, align 2
 
 define i64 @test_llless(i16 signext %a, i16 signext %b)  {
 ; CHECK-LABEL: test_llless:
diff --git a/llvm/test/CodeGen/PowerPC/testComparesllleuc.ll b/llvm/test/CodeGen/PowerPC/testComparesllleuc.ll
index b2df0167f..9f0bf574e 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesllleuc.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesllleuc.ll
@@ -1,116 +1,172 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
-; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
+; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
+; RUN:  --check-prefixes=CHECK,BE
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
-; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
+; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
+; RUN:  --check-prefixes=CHECK,LE
 
-@glob = common local_unnamed_addr global i8 0, align 1
+@glob = local_unnamed_addr global i8 0, align 1
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llleuc(i8 zeroext %a, i8 zeroext %b) {
+; CHECK-LABEL: test_llleuc:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    sub r3, r4, r3
+; CHECK-NEXT:    not r3, r3
+; CHECK-NEXT:    rldicl r3, r3, 1, 63
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ule i8 %a, %b
   %conv3 = zext i1 %cmp to i64
   ret i64 %conv3
-; CHECK-LABEL: test_llleuc:
-; CHECK: sub [[REG1:r[0-9]+]], r4, r3
-; CHECK-NEXT: not [[REG2:r[0-9]+]], [[REG1]]
-; CHECK-NEXT: rldicl r3, [[REG2]], 1, 63
-; CHECK-NEXT: blr
 }
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llleuc_sext(i8 zeroext %a, i8 zeroext %b) {
+; CHECK-LABEL: test_llleuc_sext:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    sub r3, r4, r3
+; CHECK-NEXT:    rldicl r3, r3, 1, 63
+; CHECK-NEXT:    addi r3, r3, -1
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ule i8 %a, %b
   %conv3 = sext i1 %cmp to i64
   ret i64 %conv3
-; CHECK-LABEL: @test_llleuc_sext
-; CHECK: sub [[REG1:r[0-9]+]], r4, r3
-; CHECK-NEXT: rldicl [[REG2:r[0-9]+]], [[REG1]], 1, 63
-; CHECK-NEXT: addi [[REG3:r[0-9]+]], [[REG2]], -1
-; CHECK-NEXT: blr
 }
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llleuc_z(i8 zeroext %a) {
+; CHECK-LABEL: test_llleuc_z:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cntlzw r3, r3
+; CHECK-NEXT:    srwi r3, r3, 5
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ule i8 %a, 0
   %conv2 = zext i1 %cmp to i64
   ret i64 %conv2
-; CHECK-LABEL: test_llleuc_z:
-; CHECK: cntlzw r3, r3
-; CHECK-NEXT: srwi r3, r3, 5
-; CHECK-NEXT: blr
 }
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llleuc_sext_z(i8 zeroext %a) {
+; CHECK-LABEL: test_llleuc_sext_z:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cntlzw r3, r3
+; CHECK-NEXT:    srwi r3, r3, 5
+; CHECK-NEXT:    neg r3, r3
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ule i8 %a, 0
   %conv2 = sext i1 %cmp to i64
   ret i64 %conv2
-; CHECK-LABEL: @test_llleuc_sext_z
-; CHECK: cntlzw [[REG1:r[0-9]+]], r3
-; CHECK-NEXT: srwi [[REG2:r[0-9]+]], [[REG1]], 5
-; CHECK-NEXT: neg r3, [[REG2]]
-; CHECK-NEXT: blr
 }
 
 ; Function Attrs: norecurse nounwind
 define void @test_llleuc_store(i8 zeroext %a, i8 zeroext %b) {
+; BE-LABEL: test_llleuc_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r5, r2, .LC0@toc@ha
+; BE-NEXT:    sub r3, r4, r3
+; BE-NEXT:    ld r4, .LC0@toc@l(r5)
+; BE-NEXT:    not r3, r3
+; BE-NEXT:    rldicl r3, r3, 1, 63
+; BE-NEXT:    stb r3, 0(r4)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_llleuc_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    sub r3, r4, r3
+; LE-NEXT:    addis r5, r2, glob@toc@ha
+; LE-NEXT:    not r3, r3
+; LE-NEXT:    rldicl r3, r3, 1, 63
+; LE-NEXT:    stb r3, glob@toc@l(r5)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp ule i8 %a, %b
   %conv3 = zext i1 %cmp to i8
   store i8 %conv3, i8* @glob
   ret void
-; CHECK-LABEL: test_llleuc_store:
-; CHECK: sub [[REG1:r[0-9]+]], r4, r3
-; CHECK: not [[REG2:r[0-9]+]], [[REG1]]
-; CHECK-NEXT: rldicl r3, [[REG2]], 1, 63
-; CHECK: blr
 }
 
 ; Function Attrs: norecurse nounwind
 define void @test_llleuc_sext_store(i8 zeroext %a, i8 zeroext %b) {
+; BE-LABEL: test_llleuc_sext_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r5, r2, .LC0@toc@ha
+; BE-NEXT:    sub r3, r4, r3
+; BE-NEXT:    ld r4, .LC0@toc@l(r5)
+; BE-NEXT:    rldicl r3, r3, 1, 63
+; BE-NEXT:    addi r3, r3, -1
+; BE-NEXT:    stb r3, 0(r4)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_llleuc_sext_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    sub r3, r4, r3
+; LE-NEXT:    addis r5, r2, glob@toc@ha
+; LE-NEXT:    rldicl r3, r3, 1, 63
+; LE-NEXT:    addi r3, r3, -1
+; LE-NEXT:    stb r3, glob@toc@l(r5)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp ule i8 %a, %b
   %conv3 = sext i1 %cmp to i8
   store i8 %conv3, i8* @glob
   ret void
-; CHECK-LABEL: @test_llleuc_sext_store
-; CHECK: sub [[REG1:r[0-9]+]], r4, r3
-; CHECK: rldicl [[REG2:r[0-9]+]], [[REG1]], 1, 63
-; CHECK: addi [[REG3:r[0-9]+]], [[REG2]], -1
-; CHECK: stb  [[REG3]]
-; CHECK: blr
 }
 
 ; Function Attrs: norecurse nounwind
 define void @test_llleuc_z_store(i8 zeroext %a) {
+; BE-LABEL: test_llleuc_z_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r4, r2, .LC0@toc@ha
+; BE-NEXT:    cntlzw r3, r3
+; BE-NEXT:    ld r4, .LC0@toc@l(r4)
+; BE-NEXT:    srwi r3, r3, 5
+; BE-NEXT:    stb r3, 0(r4)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_llleuc_z_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    cntlzw r3, r3
+; LE-NEXT:    addis r4, r2, glob@toc@ha
+; LE-NEXT:    srwi r3, r3, 5
+; LE-NEXT:    stb r3, glob@toc@l(r4)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp ule i8 %a, 0
   %conv2 = zext i1 %cmp to i8
   store i8 %conv2, i8* @glob
   ret void
-; CHECK-LABEL: test_llleuc_z_store:
-; CHECK: cntlzw r3, r3
-; CHECK: srwi {{r[0-9]}}, r3, 5
-; CHECK: blr
 }
 
 ; Function Attrs: norecurse nounwind
 define void @test_llleuc_sext_z_store(i8 zeroext %a) {
+; BE-LABEL: test_llleuc_sext_z_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r4, r2, .LC0@toc@ha
+; BE-NEXT:    cntlzw r3, r3
+; BE-NEXT:    ld r4, .LC0@toc@l(r4)
+; BE-NEXT:    srwi r3, r3, 5
+; BE-NEXT:    neg r3, r3
+; BE-NEXT:    stb r3, 0(r4)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_llleuc_sext_z_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    cntlzw r3, r3
+; LE-NEXT:    addis r4, r2, glob@toc@ha
+; LE-NEXT:    srwi r3, r3, 5
+; LE-NEXT:    neg r3, r3
+; LE-NEXT:    stb r3, glob@toc@l(r4)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp ule i8 %a, 0
   %conv2 = sext i1 %cmp to i8
   store i8 %conv2, i8* @glob
   ret void
-; CHECK-LABEL: @test_llleuc_sext_z_store
-; CHECK: cntlzw [[REG1:r[0-9]+]], r3
-; CHECK: srwi [[REG2:r[0-9]+]], [[REG1]], 5
-; CHECK: neg [[REG3:r[0-9]+]], [[REG2]]
-; CHECK: stb [[REG3]]
-; CHECK: blr
 }
diff --git a/llvm/test/CodeGen/PowerPC/testComparesllleui.ll b/llvm/test/CodeGen/PowerPC/testComparesllleui.ll
index fa77495d7..b6c43e4a3 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesllleui.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesllleui.ll
@@ -1,117 +1,173 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
-; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
+; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
+; RUN:  --check-prefixes=CHECK,BE
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
-; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
+; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
+; RUN:  --check-prefixes=CHECK,LE
 
-@glob = common local_unnamed_addr global i32 0, align 4
+@glob = local_unnamed_addr global i32 0, align 4
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llleui(i32 zeroext %a, i32 zeroext %b) {
+; CHECK-LABEL: test_llleui:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    sub r3, r4, r3
+; CHECK-NEXT:    not r3, r3
+; CHECK-NEXT:    rldicl r3, r3, 1, 63
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ule i32 %a, %b
   %conv1 = zext i1 %cmp to i64
   ret i64 %conv1
-; CHECK-LABEL: test_llleui:
-; CHECK: sub [[REG1:r[0-9]+]], r4, r3
-; CHECK-NEXT: not [[REG2:r[0-9]+]], [[REG1]]
-; CHECK-NEXT: rldicl r3, [[REG2]], 1, 63
-; CHECK: blr
 }
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llleui_sext(i32 zeroext %a, i32 zeroext %b) {
+; CHECK-LABEL: test_llleui_sext:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    sub r3, r4, r3
+; CHECK-NEXT:    rldicl r3, r3, 1, 63
+; CHECK-NEXT:    addi r3, r3, -1
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ule i32 %a, %b
   %conv1 = sext i1 %cmp to i64
   ret i64 %conv1
-; CHECK-LABEL: @test_llleui_sext
-; CHECK: sub [[REG1:r[0-9]+]], r4, r3
-; CHECK-NEXT: rldicl [[REG2:r[0-9]+]], [[REG1]], 1, 63
-; CHECK-NEXT: addi [[REG3:r[0-9]+]], [[REG2]], -1
-; CHECK-NEXT: blr
 }
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llleui_z(i32 zeroext %a) {
+; CHECK-LABEL: test_llleui_z:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cntlzw r3, r3
+; CHECK-NEXT:    srwi r3, r3, 5
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ule i32 %a, 0
   %conv1 = zext i1 %cmp to i64
   ret i64 %conv1
-; CHECK-LABEL: test_llleui_z:
-; CHECK: cntlzw r3, r3
-; CHECK-NEXT: srwi r3, r3, 5
-; CHECK-NEXT: blr
 }
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llleui_sext_z(i32 zeroext %a) {
+; CHECK-LABEL: test_llleui_sext_z:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cntlzw r3, r3
+; CHECK-NEXT:    srwi r3, r3, 5
+; CHECK-NEXT:    neg r3, r3
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ule i32 %a, 0
   %conv1 = sext i1 %cmp to i64
   ret i64 %conv1
-; CHECK-LABEL: @test_llleui_sext_z
-; CHECK: cntlzw [[REG1:r[0-9]+]], r3
-; CHECK-NEXT: srwi [[REG2:r[0-9]+]], [[REG1]], 5
-; CHECK-NEXT: neg r3, [[REG2]]
-; CHECK: blr
 }
 
 ; Function Attrs: norecurse nounwind
 define void @test_llleui_store(i32 zeroext %a, i32 zeroext %b) {
+; BE-LABEL: test_llleui_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r5, r2, .LC0@toc@ha
+; BE-NEXT:    sub r3, r4, r3
+; BE-NEXT:    ld r4, .LC0@toc@l(r5)
+; BE-NEXT:    not r3, r3
+; BE-NEXT:    rldicl r3, r3, 1, 63
+; BE-NEXT:    stw r3, 0(r4)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_llleui_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    sub r3, r4, r3
+; LE-NEXT:    addis r5, r2, glob@toc@ha
+; LE-NEXT:    not r3, r3
+; LE-NEXT:    rldicl r3, r3, 1, 63
+; LE-NEXT:    stw r3, glob@toc@l(r5)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp ule i32 %a, %b
   %conv = zext i1 %cmp to i32
   store i32 %conv, i32* @glob
   ret void
-; CHECK-LABEL: test_llleui_store:
-; CHECK: sub [[REG1:r[0-9]+]], r4, r3
-; CHECK: not [[REG2:r[0-9]+]], [[REG1]]
-; CHECK-NEXT: rldicl r3, [[REG2]], 1, 63
-; CHECK: blr
 }
 
 ; Function Attrs: norecurse nounwind
 define void @test_llleui_sext_store(i32 zeroext %a, i32 zeroext %b) {
+; BE-LABEL: test_llleui_sext_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r5, r2, .LC0@toc@ha
+; BE-NEXT:    sub r3, r4, r3
+; BE-NEXT:    ld r4, .LC0@toc@l(r5)
+; BE-NEXT:    rldicl r3, r3, 1, 63
+; BE-NEXT:    addi r3, r3, -1
+; BE-NEXT:    stw r3, 0(r4)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_llleui_sext_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    sub r3, r4, r3
+; LE-NEXT:    addis r5, r2, glob@toc@ha
+; LE-NEXT:    rldicl r3, r3, 1, 63
+; LE-NEXT:    addi r3, r3, -1
+; LE-NEXT:    stw r3, glob@toc@l(r5)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp ule i32 %a, %b
   %sub = sext i1 %cmp to i32
   store i32 %sub, i32* @glob
   ret void
-; CHECK-LABEL: @test_llleui_sext_store
-; CHECK: sub [[REG1:r[0-9]+]], r4, r3
-; CHECK: rldicl [[REG2:r[0-9]+]], [[REG1]], 1, 63
-; CHECK: addi [[REG3:r[0-9]+]], [[REG2]], -1
-; CHECK: stw  [[REG3]]
-; CHECK: blr
 }
 
 ; Function Attrs: norecurse nounwind
 define void @test_llleui_z_store(i32 zeroext %a) {
+; BE-LABEL: test_llleui_z_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r4, r2, .LC0@toc@ha
+; BE-NEXT:    cntlzw r3, r3
+; BE-NEXT:    ld r4, .LC0@toc@l(r4)
+; BE-NEXT:    srwi r3, r3, 5
+; BE-NEXT:    stw r3, 0(r4)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_llleui_z_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    cntlzw r3, r3
+; LE-NEXT:    addis r4, r2, glob@toc@ha
+; LE-NEXT:    srwi r3, r3, 5
+; LE-NEXT:    stw r3, glob@toc@l(r4)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp ule i32 %a, 0
   %conv = zext i1 %cmp to i32
   store i32 %conv, i32* @glob
   ret void
-; CHECK-LABEL: test_llleui_z_store:
-; CHECK: cntlzw r3, r3
-; CHECK: srwi r3, r3, 5
-; CHECK: blr
 }
 
 ; Function Attrs: norecurse nounwind
 define void @test_llleui_sext_z_store(i32 zeroext %a) {
+; BE-LABEL: test_llleui_sext_z_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r4, r2, .LC0@toc@ha
+; BE-NEXT:    cntlzw r3, r3
+; BE-NEXT:    ld r4, .LC0@toc@l(r4)
+; BE-NEXT:    srwi r3, r3, 5
+; BE-NEXT:    neg r3, r3
+; BE-NEXT:    stw r3, 0(r4)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_llleui_sext_z_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    cntlzw r3, r3
+; LE-NEXT:    addis r4, r2, glob@toc@ha
+; LE-NEXT:    srwi r3, r3, 5
+; LE-NEXT:    neg r3, r3
+; LE-NEXT:    stw r3, glob@toc@l(r4)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp ule i32 %a, 0
   %sub = sext i1 %cmp to i32
   store i32 %sub, i32* @glob
   ret void
-; CHECK-LABEL: @test_llleui_sext_z_store
-; CHECK: cntlzw [[REG1:r[0-9]+]], r3
-; CHECK: srwi [[REG2:r[0-9]+]], [[REG1]], 5
-; CHECK: neg [[REG3:r[0-9]+]], [[REG2]]
-; CHECK: stw [[REG3]]
-; CHECK: blr
 }
 
diff --git a/llvm/test/CodeGen/PowerPC/testComparesllleull.ll b/llvm/test/CodeGen/PowerPC/testComparesllleull.ll
index bd75f8c73..412d26497 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesllleull.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesllleull.ll
@@ -1,115 +1,170 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
-; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
+; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
+; RUN:  --check-prefixes=CHECK,BE
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
-; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
+; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
+; RUN:  --check-prefixes=CHECK,LE
 
-@glob = common local_unnamed_addr global i64 0, align 8
+@glob = local_unnamed_addr global i64 0, align 8
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llleull(i64 %a, i64 %b) {
+; CHECK-LABEL: test_llleull:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    subfc r4, r3, r4
+; CHECK-NEXT:    subfe r3, r3, r3
+; CHECK-NEXT:    addi r3, r3, 1
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ule i64 %a, %b
   %conv1 = zext i1 %cmp to i64
   ret i64 %conv1
-; CHECK-LABEL: test_llleull:
-; CHECK: subfc {{r[0-9]+}}, r3, r4
-; CHECK-NEXT: subfe [[REG1:r[0-9]+]], r3, r3
-; CHECK-NEXT: addi r3, [[REG1]], 1
-; CHECK: blr
 }
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llleull_sext(i64 %a, i64 %b) {
+; CHECK-LABEL: test_llleull_sext:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    subfc r4, r3, r4
+; CHECK-NEXT:    subfe r3, r3, r3
+; CHECK-NEXT:    not r3, r3
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ule i64 %a, %b
   %conv1 = sext i1 %cmp to i64
   ret i64 %conv1
-; CHECK-LABEL: @test_llleull_sext
-; CHECK: subfc [[REG1:r[0-9]+]], r3, r4
-; CHECK: subfe [[REG2:r[0-9]+]], {{r[0-9]+}}, {{r[0-9]+}}
-; CHECK: not [[REG3:r[0-9]+]], [[REG2]]
-; CHECK: blr
 }
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llleull_z(i64 %a) {
+; CHECK-LABEL: test_llleull_z:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cntlzd r3, r3
+; CHECK-NEXT:    rldicl r3, r3, 58, 63
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ule i64 %a, 0
   %conv1 = zext i1 %cmp to i64
   ret i64 %conv1
-; CHECK-LABEL: test_llleull_z
-; CHECK: cntlzd [[REG1:r[0-9]+]], r3
-; CHECK-NEXT: rldicl r3, [[REG1]], 58, 63
-; CHECK-NEXT: blr
 }
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llleull_sext_z(i64 %a) {
+; CHECK-LABEL: test_llleull_sext_z:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addic r3, r3, -1
+; CHECK-NEXT:    subfe r3, r3, r3
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ule i64 %a, 0
   %conv1 = sext i1 %cmp to i64
   ret i64 %conv1
-; CHECK-LABEL: @test_llleull_sext_z
-; CHECK: addic [[REG1:r[0-9]+]], r3, -1
-; CHECK: subfe r3, [[REG1]]
-; CHECK: blr
 }
 
 ; Function Attrs: norecurse nounwind
 define void @test_llleull_store(i64 %a, i64 %b) {
+; BE-LABEL: test_llleull_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r5, r2, .LC0@toc@ha
+; BE-NEXT:    subfc r4, r3, r4
+; BE-NEXT:    ld r4, .LC0@toc@l(r5)
+; BE-NEXT:    subfe r3, r3, r3
+; BE-NEXT:    addi r3, r3, 1
+; BE-NEXT:    std r3, 0(r4)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_llleull_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    subfc r4, r3, r4
+; LE-NEXT:    addis r5, r2, glob@toc@ha
+; LE-NEXT:    subfe r3, r3, r3
+; LE-NEXT:    addi r3, r3, 1
+; LE-NEXT:    std r3, glob@toc@l(r5)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp ule i64 %a, %b
   %conv1 = zext i1 %cmp to i64
   store i64 %conv1, i64* @glob
   ret void
-; CHECK-LABEL: test_llleull_store:
-; CHECK: subfc {{r[0-9]+}}, r3, r4
-; CHECK: subfe [[REG1:r[0-9]+]], r3, r3
-; CHECK: addi r3, [[REG1]], 1
-; CHECK: blr
 }
 
 ; Function Attrs: norecurse nounwind
 define void @test_llleull_sext_store(i64 %a, i64 %b) {
+; BE-LABEL: test_llleull_sext_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r5, r2, .LC0@toc@ha
+; BE-NEXT:    subfc r4, r3, r4
+; BE-NEXT:    ld r4, .LC0@toc@l(r5)
+; BE-NEXT:    subfe r3, r3, r3
+; BE-NEXT:    not r3, r3
+; BE-NEXT:    std r3, 0(r4)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_llleull_sext_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    subfc r4, r3, r4
+; LE-NEXT:    addis r5, r2, glob@toc@ha
+; LE-NEXT:    subfe r3, r3, r3
+; LE-NEXT:    not r3, r3
+; LE-NEXT:    std r3, glob@toc@l(r5)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp ule i64 %a, %b
   %conv1 = sext i1 %cmp to i64
   store i64 %conv1, i64* @glob
   ret void
-; CHECK-LABEL: @test_llleull_sext_store
-; CHECK: subfc [[REG1:r[0-9]+]], r3, r4
-; CHECK: subfe [[REG2:r[0-9]+]], {{r[0-9]+}}, {{r[0-9]+}}
-; CHECK: not [[REG3:r[0-9]+]], [[REG2]]
-; CHECK: std [[REG3]]
-; CHECK: blr
 }
 
 ; Function Attrs: norecurse nounwind
 define void @test_llleull_z_store(i64 %a) {
+; BE-LABEL: test_llleull_z_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r4, r2, .LC0@toc@ha
+; BE-NEXT:    cntlzd r3, r3
+; BE-NEXT:    ld r4, .LC0@toc@l(r4)
+; BE-NEXT:    rldicl r3, r3, 58, 63
+; BE-NEXT:    std r3, 0(r4)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_llleull_z_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    cntlzd r3, r3
+; LE-NEXT:    addis r4, r2, glob@toc@ha
+; LE-NEXT:    rldicl r3, r3, 58, 63
+; LE-NEXT:    std r3, glob@toc@l(r4)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp ule i64 %a, 0
   %conv1 = zext i1 %cmp to i64
   store i64 %conv1, i64* @glob
   ret void
-; CHECK-LABEL: test_llleull_z_store:
-; CHECK: cntlzd [[REG1:r[0-9]+]], r3
-; CHECK: rldicl {{r[0-9]+}}, [[REG1]], 58, 63
-; CHECK: blr
 }
 
 ; Function Attrs: norecurse nounwind
 define void @test_llleull_sext_z_store(i64 %a) {
+; BE-LABEL: test_llleull_sext_z_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r4, r2, .LC0@toc@ha
+; BE-NEXT:    addic r3, r3, -1
+; BE-NEXT:    ld r4, .LC0@toc@l(r4)
+; BE-NEXT:    subfe r3, r3, r3
+; BE-NEXT:    std r3, 0(r4)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_llleull_sext_z_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    addic r3, r3, -1
+; LE-NEXT:    addis r4, r2, glob@toc@ha
+; LE-NEXT:    subfe r3, r3, r3
+; LE-NEXT:    std r3, glob@toc@l(r4)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp ule i64 %a, 0
   %conv1 = sext i1 %cmp to i64
   store i64 %conv1, i64* @glob
   ret void
-; CHECK-LABEL: @test_llleull_sext_z_store
-; CHECK: addic [[REG1:r[0-9]+]], r3, -1
-; CHECK: subfe [[REG2:r[0-9]+]], {{r[0-9]+}}, {{r[0-9]+}}
-; CHECK: std [[REG2]]
-; CHECK: blr
 }
 
diff --git a/llvm/test/CodeGen/PowerPC/testComparesllleus.ll b/llvm/test/CodeGen/PowerPC/testComparesllleus.ll
index 51ebb5e57..8c852f16d 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesllleus.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesllleus.ll
@@ -1,117 +1,173 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
-; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
+; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
+; RUN:  --check-prefixes=CHECK,BE
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
-; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
+; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
+; RUN:  --check-prefixes=CHECK,LE
 
-@glob = common local_unnamed_addr global i16 0, align 2
+@glob = local_unnamed_addr global i16 0, align 2
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llleus(i16 zeroext %a, i16 zeroext %b) {
+; CHECK-LABEL: test_llleus:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    sub r3, r4, r3
+; CHECK-NEXT:    not r3, r3
+; CHECK-NEXT:    rldicl r3, r3, 1, 63
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ule i16 %a, %b
   %conv3 = zext i1 %cmp to i64
   ret i64 %conv3
-; CHECK-LABEL: test_llleus:
-; CHECK: sub [[REG1:r[0-9]+]], r4, r3
-; CHECK-NEXT: not [[REG2:r[0-9]+]], [[REG1]]
-; CHECK-NEXT: rldicl r3, [[REG2]], 1, 63
-; CHECK: blr
 }
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llleus_sext(i16 zeroext %a, i16 zeroext %b) {
+; CHECK-LABEL: test_llleus_sext:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    sub r3, r4, r3
+; CHECK-NEXT:    rldicl r3, r3, 1, 63
+; CHECK-NEXT:    addi r3, r3, -1
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ule i16 %a, %b
   %conv3 = sext i1 %cmp to i64
   ret i64 %conv3
-; CHECK-LABEL: @test_llleus_sext
-; CHECK: sub [[REG1:r[0-9]+]], r4, r3
-; CHECK-NEXT: rldicl [[REG2:r[0-9]+]], [[REG1]], 1, 63
-; CHECK-NEXT: addi [[REG3:r[0-9]+]], [[REG2]], -1
-; CHECK-NEXT: blr
 }
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llleus_z(i16 zeroext %a) {
+; CHECK-LABEL: test_llleus_z:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cntlzw r3, r3
+; CHECK-NEXT:    srwi r3, r3, 5
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ule i16 %a, 0
   %conv2 = zext i1 %cmp to i64
   ret i64 %conv2
-; CHECK-LABEL: test_llleus_z:
-; CHECK: cntlzw r3, r3
-; CHECK-NEXT: srwi r3, r3, 5
-; CHECK: blr
 }
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llleus_sext_z(i16 zeroext %a) {
+; CHECK-LABEL: test_llleus_sext_z:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cntlzw r3, r3
+; CHECK-NEXT:    srwi r3, r3, 5
+; CHECK-NEXT:    neg r3, r3
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ule i16 %a, 0
   %conv2 = sext i1 %cmp to i64
   ret i64 %conv2
-; CHECK-LABEL: @test_llleus_sext_z
-; CHECK: cntlzw [[REG1:r[0-9]+]], r3
-; CHECK-NEXT: srwi [[REG2:r[0-9]+]], [[REG1]], 5
-; CHECK-NEXT: neg r3, [[REG2]]
-; CHECK-NEXT: blr
 }
 
 ; Function Attrs: norecurse nounwind
 define void @test_llleus_store(i16 zeroext %a, i16 zeroext %b) {
+; BE-LABEL: test_llleus_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r5, r2, .LC0@toc@ha
+; BE-NEXT:    sub r3, r4, r3
+; BE-NEXT:    ld r4, .LC0@toc@l(r5)
+; BE-NEXT:    not r3, r3
+; BE-NEXT:    rldicl r3, r3, 1, 63
+; BE-NEXT:    sth r3, 0(r4)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_llleus_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    sub r3, r4, r3
+; LE-NEXT:    addis r5, r2, glob@toc@ha
+; LE-NEXT:    not r3, r3
+; LE-NEXT:    rldicl r3, r3, 1, 63
+; LE-NEXT:    sth r3, glob@toc@l(r5)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp ule i16 %a, %b
   %conv3 = zext i1 %cmp to i16
   store i16 %conv3, i16* @glob
   ret void
-; CHECK-LABEL: test_llleus_store:
-; CHECK: sub [[REG1:r[0-9]+]], r4, r3
-; CHECK: not [[REG2:r[0-9]+]], [[REG1]]
-; CHECK-NEXT: rldicl r3, [[REG2]], 1, 63
-; CHECK: blr
 }
 
 ; Function Attrs: norecurse nounwind
 define void @test_llleus_sext_store(i16 zeroext %a, i16 zeroext %b) {
+; BE-LABEL: test_llleus_sext_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r5, r2, .LC0@toc@ha
+; BE-NEXT:    sub r3, r4, r3
+; BE-NEXT:    ld r4, .LC0@toc@l(r5)
+; BE-NEXT:    rldicl r3, r3, 1, 63
+; BE-NEXT:    addi r3, r3, -1
+; BE-NEXT:    sth r3, 0(r4)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_llleus_sext_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    sub r3, r4, r3
+; LE-NEXT:    addis r5, r2, glob@toc@ha
+; LE-NEXT:    rldicl r3, r3, 1, 63
+; LE-NEXT:    addi r3, r3, -1
+; LE-NEXT:    sth r3, glob@toc@l(r5)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp ule i16 %a, %b
   %conv3 = sext i1 %cmp to i16
   store i16 %conv3, i16* @glob
   ret void
-; CHECK-LABEL: @test_llleus_sext_store
-; CHECK: sub [[REG1:r[0-9]+]], r4, r3
-; CHECK: rldicl [[REG2:r[0-9]+]], [[REG1]], 1, 63
-; CHECK: addi [[REG3:r[0-9]+]], [[REG2]], -1
-; CHECK: sth  [[REG3]]
-; CHECK: blr
 }
 
 ; Function Attrs: norecurse nounwind
 define void @test_llleus_z_store(i16 zeroext %a) {
+; BE-LABEL: test_llleus_z_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r4, r2, .LC0@toc@ha
+; BE-NEXT:    cntlzw r3, r3
+; BE-NEXT:    ld r4, .LC0@toc@l(r4)
+; BE-NEXT:    srwi r3, r3, 5
+; BE-NEXT:    sth r3, 0(r4)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_llleus_z_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    cntlzw r3, r3
+; LE-NEXT:    addis r4, r2, glob@toc@ha
+; LE-NEXT:    srwi r3, r3, 5
+; LE-NEXT:    sth r3, glob@toc@l(r4)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp ule i16 %a, 0
   %conv2 = zext i1 %cmp to i16
   store i16 %conv2, i16* @glob
   ret void
-; CHECK-LABEL: test_llleus_z_store:
-; CHECK: cntlzw r3, r3
-; CHECK: srwi r3, r3, 5
-; CHECK: blr
 }
 
 ; Function Attrs: norecurse nounwind
 define void @test_llleus_sext_z_store(i16 zeroext %a) {
+; BE-LABEL: test_llleus_sext_z_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r4, r2, .LC0@toc@ha
+; BE-NEXT:    cntlzw r3, r3
+; BE-NEXT:    ld r4, .LC0@toc@l(r4)
+; BE-NEXT:    srwi r3, r3, 5
+; BE-NEXT:    neg r3, r3
+; BE-NEXT:    sth r3, 0(r4)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_llleus_sext_z_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    cntlzw r3, r3
+; LE-NEXT:    addis r4, r2, glob@toc@ha
+; LE-NEXT:    srwi r3, r3, 5
+; LE-NEXT:    neg r3, r3
+; LE-NEXT:    sth r3, glob@toc@l(r4)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp ule i16 %a, 0
   %conv2 = sext i1 %cmp to i16
   store i16 %conv2, i16* @glob
   ret void
-; CHECK-LABEL: @test_llleus_sext_z_store
-; CHECK: cntlzw [[REG1:r[0-9]+]], r3
-; CHECK: srwi [[REG2:r[0-9]+]], [[REG1]], 5
-; CHECK: neg [[REG3:r[0-9]+]], [[REG2]]
-; CHECK: sth [[REG3]]
-; CHECK: blr
 }
 
diff --git a/llvm/test/CodeGen/PowerPC/testComparesllltsll.ll b/llvm/test/CodeGen/PowerPC/testComparesllltsll.ll
index 3e37daf04..aeb189112 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesllltsll.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesllltsll.ll
@@ -1,22 +1,24 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
-; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
+; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
+; RUN:  --check-prefixes=CHECK,BE
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
-; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
+; RUN:  --check-prefixes=CHECK,LE
 
-@glob = common local_unnamed_addr global i64 0, align 8
+@glob = local_unnamed_addr global i64 0, align 8
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llltsll(i64 %a, i64 %b) {
 ; CHECK-LABEL: test_llltsll:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    sradi [[REG1:r[0-9]+]], r3, 63
-; CHECK-NEXT:    rldicl [[REG2:r[0-9]+]], r4, 1, 63
-; CHECK-NEXT:    subfc [[REG3:r[0-9]+]], r4, r3
-; CHECK-NEXT:    adde [[REG4:r[0-9]+]], [[REG2]], [[REG1]]
-; CHECK-NEXT:    xori r3, [[REG4]], 1
+; CHECK-NEXT:    sradi r5, r3, 63
+; CHECK-NEXT:    rldicl r6, r4, 1, 63
+; CHECK-NEXT:    subfc r3, r4, r3
+; CHECK-NEXT:    adde r3, r6, r5
+; CHECK-NEXT:    xori r3, r3, 1
 ; CHECK-NEXT:    blr
 entry:
   %cmp = icmp slt i64 %a, %b
@@ -28,12 +30,12 @@ entry:
 define i64 @test_llltsll_sext(i64 %a, i64 %b) {
 ; CHECK-LABEL: test_llltsll_sext:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    sradi [[REG1:r[0-9]+]], r3, 63
-; CHECK-NEXT:    rldicl [[REG2:r[0-9]+]], r4, 1, 63
-; CHECK-NEXT:    subfc [[REG3:r[0-9]+]], r4, r3
-; CHECK-NEXT:    adde [[REG4:r[0-9]+]], [[REG2]], [[REG1]]
-; CHECK-NEXT:    xori [[REG5:r[0-9]+]], [[REG4]], 1
-; CHECK-NEXT:    neg r3, [[REG5]]
+; CHECK-NEXT:    sradi r5, r3, 63
+; CHECK-NEXT:    rldicl r6, r4, 1, 63
+; CHECK-NEXT:    subfc r3, r4, r3
+; CHECK-NEXT:    adde r3, r6, r5
+; CHECK-NEXT:    xori r3, r3, 1
+; CHECK-NEXT:    neg r3, r3
 ; CHECK-NEXT:    blr
 entry:
   %cmp = icmp slt i64 %a, %b
@@ -55,14 +57,29 @@ entry:
 
 ; Function Attrs: norecurse nounwind
 define void @test_llltsll_store(i64 %a, i64 %b) {
-; CHECK-LABEL: test_llltsll_store:
-; CHECK:       # %bb.0: # %entry
-; CHECK:         sradi [[REG1:r[0-9]+]], r3, 63
-; CHECK:         rldicl [[REG2:r[0-9]+]], r4, 1, 63
+; BE-LABEL: test_llltsll_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    sradi r6, r3, 63
+; BE-NEXT:    addis r5, r2, .LC0@toc@ha
+; BE-NEXT:    subfc r3, r4, r3
+; BE-NEXT:    rldicl r3, r4, 1, 63
+; BE-NEXT:    ld r4, .LC0@toc@l(r5)
+; BE-NEXT:    adde r3, r3, r6
+; BE-NEXT:    xori r3, r3, 1
+; BE-NEXT:    std r3, 0(r4)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_llltsll_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    sradi r6, r3, 63
+; LE-NEXT:    addis r5, r2, glob@toc@ha
+; LE-NEXT:    subfc r3, r4, r3
+; LE-NEXT:    rldicl r3, r4, 1, 63
+; LE-NEXT:    adde r3, r3, r6
+; LE-NEXT:    xori r3, r3, 1
+; LE-NEXT:    std r3, glob@toc@l(r5)
+; LE-NEXT:    blr
 ; CHECK-DIAG:    subfc [[REG3:r[0-9]+]], r4, r3
-; CHECK:         adde [[REG4:r[0-9]+]], [[REG2]], [[REG1]]
-; CHECK:         xori [[REG5:r[0-9]+]], [[REG4]], 1
-; CHECK-NOT:     neg
 entry:
   %cmp = icmp slt i64 %a, %b
   %conv1 = zext i1 %cmp to i64
@@ -72,14 +89,31 @@ entry:
 
 ; Function Attrs: norecurse nounwind
 define void @test_llltsll_sext_store(i64 %a, i64 %b) {
-; CHECK-LABEL: test_llltsll_sext_store:
-; CHECK:       # %bb.0: # %entry
-; CHECK:         sradi [[REG1:r[0-9]+]], r3, 63
-; CHECK:         rldicl [[REG2:r[0-9]+]], r4, 1, 63
+; BE-LABEL: test_llltsll_sext_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    sradi r6, r3, 63
+; BE-NEXT:    addis r5, r2, .LC0@toc@ha
+; BE-NEXT:    subfc r3, r4, r3
+; BE-NEXT:    rldicl r3, r4, 1, 63
+; BE-NEXT:    ld r4, .LC0@toc@l(r5)
+; BE-NEXT:    adde r3, r3, r6
+; BE-NEXT:    xori r3, r3, 1
+; BE-NEXT:    neg r3, r3
+; BE-NEXT:    std r3, 0(r4)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_llltsll_sext_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    sradi r6, r3, 63
+; LE-NEXT:    addis r5, r2, glob@toc@ha
+; LE-NEXT:    subfc r3, r4, r3
+; LE-NEXT:    rldicl r3, r4, 1, 63
+; LE-NEXT:    adde r3, r3, r6
+; LE-NEXT:    xori r3, r3, 1
+; LE-NEXT:    neg r3, r3
+; LE-NEXT:    std r3, glob@toc@l(r5)
+; LE-NEXT:    blr
 ; CHECK-DIAG:    subfc [[REG3:r[0-9]+]], r4, r3
-; CHECK:         adde [[REG4:r[0-9]+]], [[REG2]], [[REG1]]
-; CHECK:         xori [[REG5:r[0-9]+]], [[REG4]], 1
-; CHECK:         neg {{r[0-9]+}}, [[REG5]]
 entry:
   %cmp = icmp slt i64 %a, %b
   %conv1 = sext i1 %cmp to i64
@@ -89,8 +123,20 @@ entry:
 
 ; Function Attrs: norecurse nounwind
 define void @test_llltsll_sext_z_store(i64 %a) {
-; CHECK-LABEL: test_llltsll_sext_z_store:
-; CHECK:    sradi r3, r3, 63
+; BE-LABEL: test_llltsll_sext_z_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r4, r2, .LC0@toc@ha
+; BE-NEXT:    sradi r3, r3, 63
+; BE-NEXT:    ld r4, .LC0@toc@l(r4)
+; BE-NEXT:    std r3, 0(r4)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_llltsll_sext_z_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    addis r4, r2, glob@toc@ha
+; LE-NEXT:    sradi r3, r3, 63
+; LE-NEXT:    std r3, glob@toc@l(r4)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp slt i64 %a, 0
   %sub = sext i1 %cmp to i64
diff --git a/llvm/test/CodeGen/PowerPC/testComparesllltuc.ll b/llvm/test/CodeGen/PowerPC/testComparesllltuc.ll
index a8244e757..dafbbc1f4 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesllltuc.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesllltuc.ll
@@ -1,18 +1,21 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
-; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
+; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
+; RUN:  --check-prefixes=CHECK,BE
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
-; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
+; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
+; RUN:  --check-prefixes=CHECK,LE
 
-@glob = common local_unnamed_addr global i8 0, align 1
+@glob = local_unnamed_addr global i8 0, align 1
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llltuc(i8 zeroext %a, i8 zeroext %b) {
 ; CHECK-LABEL: test_llltuc:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    sub [[REG:r[0-9]+]], r3, r4
-; CHECK-NEXT:    rldicl r3, [[REG]], 1, 63
+; CHECK-NEXT:    sub r3, r3, r4
+; CHECK-NEXT:    rldicl r3, r3, 1, 63
 ; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ult i8 %a, %b
@@ -24,8 +27,8 @@ entry:
 define i64 @test_llltuc_sext(i8 zeroext %a, i8 zeroext %b) {
 ; CHECK-LABEL: test_llltuc_sext:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    sub [[REG:r[0-9]+]], r3, r4
-; CHECK-NEXT:    sradi r3, [[REG]], 63
+; CHECK-NEXT:    sub r3, r3, r4
+; CHECK-NEXT:    sradi r3, r3, 63
 ; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ult i8 %a, %b
@@ -35,10 +38,22 @@ entry:
 
 ; Function Attrs: norecurse nounwind
 define void @test_llltuc_store(i8 zeroext %a, i8 zeroext %b) {
-; CHECK-LABEL: test_llltuc_store:
-; CHECK:       # %bb.0: # %entry
-; CHECK:         sub [[REG:r[2-9]+]], r3, r4
-; CHECK:         rldicl {{r[0-9]+}}, [[REG]], 1, 63
+; BE-LABEL: test_llltuc_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r5, r2, .LC0@toc@ha
+; BE-NEXT:    sub r3, r3, r4
+; BE-NEXT:    ld r5, .LC0@toc@l(r5)
+; BE-NEXT:    rldicl r3, r3, 1, 63
+; BE-NEXT:    stb r3, 0(r5)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_llltuc_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    sub r3, r3, r4
+; LE-NEXT:    addis r5, r2, glob@toc@ha
+; LE-NEXT:    rldicl r3, r3, 1, 63
+; LE-NEXT:    stb r3, glob@toc@l(r5)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp ult i8 %a, %b
   %conv3 = zext i1 %cmp to i8
@@ -48,10 +63,22 @@ entry:
 
 ; Function Attrs: norecurse nounwind
 define void @test_llltuc_sext_store(i8 zeroext %a, i8 zeroext %b) {
-; CHECK-LABEL: test_llltuc_sext_store:
-; CHECK:       # %bb.0: # %entry
-; CHECK:         sub [[REG:r[0-9]+]], r3, r4
-; CHECK:         sradi {{r[0-9]+}}, [[REG]], 63
+; BE-LABEL: test_llltuc_sext_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r5, r2, .LC0@toc@ha
+; BE-NEXT:    sub r3, r3, r4
+; BE-NEXT:    ld r5, .LC0@toc@l(r5)
+; BE-NEXT:    sradi r3, r3, 63
+; BE-NEXT:    stb r3, 0(r5)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_llltuc_sext_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    sub r3, r3, r4
+; LE-NEXT:    addis r5, r2, glob@toc@ha
+; LE-NEXT:    sradi r3, r3, 63
+; LE-NEXT:    stb r3, glob@toc@l(r5)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp ult i8 %a, %b
   %conv3 = sext i1 %cmp to i8
diff --git a/llvm/test/CodeGen/PowerPC/testComparesllltui.ll b/llvm/test/CodeGen/PowerPC/testComparesllltui.ll
index 126d110d0..50a0ff119 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesllltui.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesllltui.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -relocation-model=pic -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
@@ -5,15 +6,14 @@
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 
-@glob = common local_unnamed_addr global i32 0, align 4
+@glob = local_unnamed_addr global i32 0, align 4
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llltui(i32 zeroext %a, i32 zeroext %b) {
 ; CHECK-LABEL: test_llltui:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NOT:     clrldi
-; CHECK-NEXT:    sub [[REG:r[0-9]+]], r3, r4
-; CHECK-NEXT:    rldicl r3, [[REG]], 1, 63
+; CHECK-NEXT:    sub r3, r3, r4
+; CHECK-NEXT:    rldicl r3, r3, 1, 63
 ; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ult i32 %a, %b
@@ -25,8 +25,8 @@ entry:
 define i64 @test_llltui_sext(i32 zeroext %a, i32 zeroext %b) {
 ; CHECK-LABEL: test_llltui_sext:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    sub [[REG:r[0-9]+]], r3, r4
-; CHECK-NEXT:    sradi r3, [[REG]], 63
+; CHECK-NEXT:    sub r3, r3, r4
+; CHECK-NEXT:    sradi r3, r3, 63
 ; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ult i32 %a, %b
@@ -58,9 +58,12 @@ entry:
 define void @test_llltui_store(i32 zeroext %a, i32 zeroext %b) {
 ; CHECK-LABEL: test_llltui_store:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NOT:     clrldi
-; CHECK:         sub [[REG:r[2-9]+]], r3, r4
-; CHECK:         rldicl {{r[0-9]+}}, [[REG]], 1, 63
+; CHECK-NEXT:    addis r5, r2, .LC0@toc@ha
+; CHECK-NEXT:    sub r3, r3, r4
+; CHECK-NEXT:    ld r5, .LC0@toc@l(r5)
+; CHECK-NEXT:    rldicl r3, r3, 1, 63
+; CHECK-NEXT:    stw r3, 0(r5)
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ult i32 %a, %b
   %conv = zext i1 %cmp to i32
@@ -72,9 +75,12 @@ entry:
 define void @test_llltui_sext_store(i32 zeroext %a, i32 zeroext %b) {
 ; CHECK-LABEL: test_llltui_sext_store:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NOT:     clrldi
-; CHECK:         sub [[REG:r[0-9]+]], r3, r4
-; CHECK:         sradi {{r[0-9]+}}, [[REG]], 63
+; CHECK-NEXT:    addis r5, r2, .LC0@toc@ha
+; CHECK-NEXT:    sub r3, r3, r4
+; CHECK-NEXT:    ld r5, .LC0@toc@l(r5)
+; CHECK-NEXT:    sradi r3, r3, 63
+; CHECK-NEXT:    stw r3, 0(r5)
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ult i32 %a, %b
   %sub = sext i1 %cmp to i32
@@ -86,8 +92,10 @@ entry:
 define void @test_llltui_z_store(i32 zeroext %a) {
 ; CHECK-LABEL: test_llltui_z_store:
 ; CHECK:       # %bb.0: # %entry
-; CHECK:         li [[REG:r[0-9]+]], 0
-; CHECK:         stw [[REG]], 0(r3)
+; CHECK-NEXT:    addis r3, r2, .LC0@toc@ha
+; CHECK-NEXT:    li r4, 0
+; CHECK-NEXT:    ld r3, .LC0@toc@l(r3)
+; CHECK-NEXT:    stw r4, 0(r3)
 ; CHECK-NEXT:    blr
 entry:
   store i32 0, i32* @glob, align 4
@@ -98,8 +106,10 @@ entry:
 define void @test_llltui_sext_z_store(i32 zeroext %a) {
 ; CHECK-LABEL: test_llltui_sext_z_store:
 ; CHECK:       # %bb.0: # %entry
-; CHECK:         li [[REG:r[0-9]+]], 0
-; CHECK:         stw [[REG]], 0(r3)
+; CHECK-NEXT:    addis r3, r2, .LC0@toc@ha
+; CHECK-NEXT:    li r4, 0
+; CHECK-NEXT:    ld r3, .LC0@toc@l(r3)
+; CHECK-NEXT:    stw r4, 0(r3)
 ; CHECK-NEXT:    blr
 entry:
   store i32 0, i32* @glob, align 4
diff --git a/llvm/test/CodeGen/PowerPC/testComparesllltus.ll b/llvm/test/CodeGen/PowerPC/testComparesllltus.ll
index e997d0aa8..217821b66 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesllltus.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesllltus.ll
@@ -1,18 +1,21 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
-; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
+; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
+; RUN:  --check-prefixes=CHECK,BE
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
-; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
+; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
+; RUN:  --check-prefixes=CHECK,LE
 
-@glob = common local_unnamed_addr global i16 0, align 2
+@glob = local_unnamed_addr global i16 0, align 2
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llltus(i16 zeroext %a, i16 zeroext %b) {
 ; CHECK-LABEL: test_llltus:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    sub [[REG:r[0-9]+]], r3, r4
-; CHECK-NEXT:    rldicl r3, [[REG]], 1, 63
+; CHECK-NEXT:    sub r3, r3, r4
+; CHECK-NEXT:    rldicl r3, r3, 1, 63
 ; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ult i16 %a, %b
@@ -24,8 +27,8 @@ entry:
 define i64 @test_llltus_sext(i16 zeroext %a, i16 zeroext %b) {
 ; CHECK-LABEL: test_llltus_sext:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    sub [[REG:r[0-9]+]], r3, r4
-; CHECK-NEXT:    sradi r3, [[REG]], 63
+; CHECK-NEXT:    sub r3, r3, r4
+; CHECK-NEXT:    sradi r3, r3, 63
 ; CHECK-NEXT:    blr
 entry:
   %cmp = icmp ult i16 %a, %b
@@ -35,9 +38,22 @@ entry:
 
 ; Function Attrs: norecurse nounwind
 define void @test_llltus_store(i16 zeroext %a, i16 zeroext %b) {
-; CHECK-LABEL: test_llltus_store:
-; CHECK:         sub [[REG:r[2-9]+]], r3, r4
-; CHECK:         rldicl {{r[0-9]+}}, [[REG]], 1, 63
+; BE-LABEL: test_llltus_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r5, r2, .LC0@toc@ha
+; BE-NEXT:    sub r3, r3, r4
+; BE-NEXT:    ld r5, .LC0@toc@l(r5)
+; BE-NEXT:    rldicl r3, r3, 1, 63
+; BE-NEXT:    sth r3, 0(r5)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_llltus_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    sub r3, r3, r4
+; LE-NEXT:    addis r5, r2, glob@toc@ha
+; LE-NEXT:    rldicl r3, r3, 1, 63
+; LE-NEXT:    sth r3, glob@toc@l(r5)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp ult i16 %a, %b
   %conv3 = zext i1 %cmp to i16
@@ -47,10 +63,22 @@ entry:
 
 ; Function Attrs: norecurse nounwind
 define void @test_llltus_sext_store(i16 zeroext %a, i16 zeroext %b) {
-; CHECK-LABEL: test_llltus_sext_store:
-; CHECK:       # %bb.0: # %entry
-; CHECK:         sub [[REG:r[0-9]+]], r3, r4
-; CHECK:         sradi {{r[0-9]+}}, [[REG]], 63
+; BE-LABEL: test_llltus_sext_store:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    addis r5, r2, .LC0@toc@ha
+; BE-NEXT:    sub r3, r3, r4
+; BE-NEXT:    ld r5, .LC0@toc@l(r5)
+; BE-NEXT:    sradi r3, r3, 63
+; BE-NEXT:    sth r3, 0(r5)
+; BE-NEXT:    blr
+;
+; LE-LABEL: test_llltus_sext_store:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    sub r3, r3, r4
+; LE-NEXT:    addis r5, r2, glob@toc@ha
+; LE-NEXT:    sradi r3, r3, 63
+; LE-NEXT:    sth r3, glob@toc@l(r5)
+; LE-NEXT:    blr
 entry:
   %cmp = icmp ult i16 %a, %b
   %conv3 = sext i1 %cmp to i16
diff --git a/llvm/test/CodeGen/PowerPC/testComparesllnesll.ll b/llvm/test/CodeGen/PowerPC/testComparesllnesll.ll
index 10bff2f7d..76ea6262d 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesllnesll.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesllnesll.ll
@@ -6,7 +6,7 @@
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s --check-prefix=CHECK-LE \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 
-@glob = common local_unnamed_addr global i64 0, align 8
+@glob = local_unnamed_addr global i64 0, align 8
 
 define i64 @test_llnesll(i64 %a, i64 %b) {
 ; CHECK-LABEL: test_llnesll:
diff --git a/llvm/test/CodeGen/PowerPC/testComparesllneull.ll b/llvm/test/CodeGen/PowerPC/testComparesllneull.ll
index 9e6be97e8..e809a0b91 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesllneull.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesllneull.ll
@@ -6,7 +6,7 @@
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s --check-prefix=CHECK-LE \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 
-@glob = common local_unnamed_addr global i64 0, align 8
+@glob = local_unnamed_addr global i64 0, align 8
 
 define i64 @test_llneull(i64 %a, i64 %b) {
 ; CHECK-LABEL: test_llneull:
diff --git a/llvm/test/CodeGen/PowerPC/test_call_aix.ll b/llvm/test/CodeGen/PowerPC/test_call_aix.ll
index 6f98843e9..ba363b75b 100644
--- a/llvm/test/CodeGen/PowerPC/test_call_aix.ll
+++ b/llvm/test/CodeGen/PowerPC/test_call_aix.ll
@@ -4,6 +4,9 @@
 ; RUN: llc -mtriple powerpc64-ibm-aix-xcoff -stop-after=machine-cp < %s | \
 ; RUN: FileCheck --check-prefix=64BIT %s
 
+; RUN: llc -mtriple powerpc-ibm-aix-xcoff < %s | FileCheck %s
+; RUN: llc -mtriple powerpc64-ibm-aix-xcoff < %s | FileCheck %s
+
 declare void @foo(...)
 
 define void @test_call() {
@@ -16,6 +19,10 @@ entry:
 ; 64BIT: BL8_NOP <mcsymbol .foo>, csr_aix64, implicit-def dead $lr8, implicit $rm, implicit $x2, implicit-def $r1
 ; 64BIT: ADJCALLSTACKUP 112, 0, implicit-def dead $r1, implicit $r1
 
+; CHECK-LABEL: test_call
+; CHECK: bl .foo
+; CHECK-NEXT: nop
+
   call void bitcast (void (...)* @foo to void ()*)()
   ret void
 }
@@ -35,6 +42,10 @@ entry:
 ; 64BIT: BL8 <mcsymbol .foo_local>, csr_aix64, implicit-def dead $lr8, implicit $rm, implicit $x2, implicit-def $r1
 ; 64BIT: ADJCALLSTACKUP 112, 0, implicit-def dead $r1, implicit $r1
 
+; CHECK-LABEL: test_local_call
+; CHECK: bl .foo_local
+; CHECK-NOT: nop
+
   call void @foo_local()
   ret void
 }
diff --git a/llvm/test/CodeGen/PowerPC/test_func_desc.ll b/llvm/test/CodeGen/PowerPC/test_func_desc.ll
new file mode 100644
index 000000000..3cd16b94f
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/test_func_desc.ll
@@ -0,0 +1,74 @@
+; RUN: llc -mtriple powerpc-ibm-aix-xcoff < %s | \
+; RUN: FileCheck --check-prefixes=CHECK,32BIT %s
+
+; RUN: llc -mtriple powerpc64-ibm-aix-xcoff < %s | \
+; RUN: FileCheck --check-prefixes=CHECK,64BIT %s
+
+
+define i32 @foo() {
+entry:
+  ret i32 3
+}
+
+define i32 @main() {
+entry:
+  %0 = call i32 @foo()
+  %1 = call i32 bitcast (i32 (...)* @extern_foo to i32 ()*)()
+  %2 = call i32 @static_foo()
+  %3 = add nsw i32 %0, %1
+  %4 = add nsw i32 %3, %2
+  ret i32 %4
+}
+
+declare i32 @extern_foo(...)
+
+define internal i32 @static_foo() {
+entry:
+  ret i32 3
+}
+
+; CHECK: .globl foo
+; CHECK: .globl .foo
+; CHECK: .csect foo[DS]
+; CHECK-NEXT: foo:
+; 32BIT: .long .foo
+; 32BIT-NEXT: .long TOC[TC0]
+; 32BIT-NEXT: .long 0
+; 64BIT: .llong .foo
+; 64BIT-NEXT: .llong TOC[TC0]
+; 64BIT-NEXT: .llong 0
+; CHECK-NEXT: .csect .text[PR]
+; CHECK-LABEL: .foo:
+
+; CHECK: .globl main
+; CHECK: .globl .main
+; CHECK: .csect main[DS]
+; CHECK-NEXT: main:
+; 32BIT: .long .main
+; 32BIT-NEXT: .long TOC[TC0]
+; 32BIT-NEXT: .long 0
+; 64BIT: .llong .main
+; 64BIT-NEXT: .llong TOC[TC0]
+; 64BIT-NEXT: .llong 0
+; CHECK-NEXT: .csect .text[PR]
+; CHECK-LABEL: .main:
+; CHECK: bl .foo
+; CHECK: bl .extern_foo
+; CHECK: bl .static_foo
+
+; CHECK: .lglobl .static_foo
+; CHECK: .csect static_foo[DS]
+; CHECK-NEXT: static_foo:
+; 32BIT: .long .static_foo
+; 32BIT-NEXT: .long TOC[TC0]
+; 32BIT-NEXT: .long 0
+; 64BIT: .llong .static_foo
+; 64BIT-NEXT: .llong TOC[TC0]
+; 64BIT-NEXT: .llong 0
+; CHECK-NEXT: .csect .text[PR]
+; CHECK-LABEL: .static_foo:
+
+; CHECK-NOT: .csect extern_foo
+
+; CHECK: .toc
+; CHECK-NOT: .tc
diff --git a/llvm/test/CodeGen/PowerPC/tls.ll b/llvm/test/CodeGen/PowerPC/tls.ll
index 1d6b22e3f..18ca23b5e 100644
--- a/llvm/test/CodeGen/PowerPC/tls.ll
+++ b/llvm/test/CodeGen/PowerPC/tls.ll
@@ -1,10 +1,10 @@
-; RUN: llc -relocation-model=static -verify-machineinstrs -O0 < %s -mcpu=ppc64 | FileCheck -check-prefix=OPT0 %s
-; RUN: llc -relocation-model=static -verify-machineinstrs -O1 < %s -mcpu=ppc64 | FileCheck -check-prefix=OPT1 %s
+; RUN: llc -relocation-model=static -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu \
+; RUN:     -O0 < %s -mcpu=ppc64 | FileCheck -check-prefix=OPT0 %s
+; RUN: llc -relocation-model=static -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu \
+; RUN:     -O1 < %s -mcpu=ppc64 | FileCheck -check-prefix=OPT1 %s
 ; RUN: llc -verify-machineinstrs -O0 < %s -mtriple=ppc32-- -mcpu=ppc | FileCheck -check-prefix=OPT0-PPC32 %s
 ; RUN: llc -relocation-model=pic -verify-machineinstrs -O0 < %s -mtriple=ppc32-- -mcpu=ppc | FileCheck -check-prefix=OPT0-PPC32-PIC %s
 
-target triple = "powerpc64-unknown-linux-gnu"
-
 @a = thread_local global i32 0, align 4
 
 ;OPT0-LABEL:          localexec:
diff --git a/llvm/test/CodeGen/PowerPC/toc-load-sched-bug.ll b/llvm/test/CodeGen/PowerPC/toc-load-sched-bug.ll
index 8e2aadf87..1ebc8d514 100644
--- a/llvm/test/CodeGen/PowerPC/toc-load-sched-bug.ll
+++ b/llvm/test/CodeGen/PowerPC/toc-load-sched-bug.ll
@@ -473,13 +473,13 @@ declare void @_ZNSs4swapERSs(%"class.std::basic_string"*, %"class.std::basic_str
 ; Function Attrs: nounwind
 declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i1) #3
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #2 = { inlinehint nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { inlinehint nounwind "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #3 = { nounwind }
-attributes #4 = { noreturn nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #5 = { nounwind readonly "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #6 = { nobuiltin nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #4 = { noreturn nounwind "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #5 = { nounwind readonly "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #6 = { nobuiltin nounwind "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #7 = { noreturn nounwind }
 
 !llvm.ident = !{!0}
diff --git a/llvm/test/CodeGen/PowerPC/tocSaveInPrologue.ll b/llvm/test/CodeGen/PowerPC/tocSaveInPrologue.ll
index d9f85f177..5235f3359 100644
--- a/llvm/test/CodeGen/PowerPC/tocSaveInPrologue.ll
+++ b/llvm/test/CodeGen/PowerPC/tocSaveInPrologue.ll
@@ -16,16 +16,17 @@ define dso_local void @test(void (i32)* nocapture %fp, i32 signext %Arg, i32 sig
 ; CHECK-NEXT:    std r0, 16(r1)
 ; CHECK-NEXT:    stdu r1, -64(r1)
 ; CHECK-NEXT:    mr r29, r5
-; CHECK-NEXT:    cmpwi cr1, r4, 11
 ; CHECK-NEXT:    mr r30, r3
 ; CHECK-NEXT:    extsw r28, r4
 ; CHECK-NEXT:    std r2, 24(r1)
 ; CHECK-NEXT:    cmpwi r29, 1
-; CHECK-NEXT:    cror 4*cr5+lt, lt, 4*cr1+lt
-; CHECK-NEXT:    bc 12, 4*cr5+lt, .LBB0_2
+; CHECK-NEXT:    bc 12, lt, .LBB0_3
+; CHECK-NEXT:  # %bb.1: # %entry
+; CHECK-NEXT:    cmpwi cr0, r4, 11
+; CHECK-NEXT:    bc 12, lt, .LBB0_3
 ; CHECK-NEXT:    .p2align 5
-; CHECK-NEXT:  .LBB0_1: # %for.body.us
-; CHECK-NEXT:  # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:  .LBB0_2: # %for.body.us
+; CHECK-NEXT:    #
 ; CHECK-NEXT:    mtctr r30
 ; CHECK-NEXT:    mr r3, r28
 ; CHECK-NEXT:    mr r12, r30
@@ -33,8 +34,8 @@ define dso_local void @test(void (i32)* nocapture %fp, i32 signext %Arg, i32 sig
 ; CHECK-NEXT:    ld 2, 24(r1)
 ; CHECK-NEXT:    addi r29, r29, -1
 ; CHECK-NEXT:    cmplwi r29, 0
-; CHECK-NEXT:    bne cr0, .LBB0_1
-; CHECK-NEXT:  .LBB0_2: # %for.cond.cleanup
+; CHECK-NEXT:    bne cr0, .LBB0_2
+; CHECK-NEXT:  .LBB0_3: # %for.cond.cleanup
 ; CHECK-NEXT:    mtctr r30
 ; CHECK-NEXT:    mr r3, r28
 ; CHECK-NEXT:    mr r12, r30
@@ -42,10 +43,10 @@ define dso_local void @test(void (i32)* nocapture %fp, i32 signext %Arg, i32 sig
 ; CHECK-NEXT:    ld 2, 24(r1)
 ; CHECK-NEXT:    addi r1, r1, 64
 ; CHECK-NEXT:    ld r0, 16(r1)
-; CHECK-NEXT:    mtlr r0
 ; CHECK-NEXT:    ld r30, -16(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld r29, -24(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld r28, -32(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    mtlr r0
 ; CHECK-NEXT:    blr
 entry:
   %cmp7 = icmp sgt i32 %Len, 0
diff --git a/llvm/test/CodeGen/PowerPC/trunc-srl-load.ll b/llvm/test/CodeGen/PowerPC/trunc-srl-load.ll
index 4cb557f03..5dc0534ef 100644
--- a/llvm/test/CodeGen/PowerPC/trunc-srl-load.ll
+++ b/llvm/test/CodeGen/PowerPC/trunc-srl-load.ll
@@ -1,9 +1,14 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=powerpc64-unknown-unknown %s -o - | FileCheck %s
 
-; CHECK-LABEL: trunc_srl_load
-; CHECK-NOT: lhz 4, 4(0)
-; CHECK: lhz 4, 2(0)
 define dso_local fastcc void @trunc_srl_load(i32 zeroext %AttrArgNo) {
+; CHECK-LABEL: trunc_srl_load:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lhz 4, 2(0)
+; CHECK-NEXT:    cmplw 0, 4, 3
+; CHECK-NEXT:    ble 0, .LBB0_2
+; CHECK-NEXT:  # %bb.1: # %exit
+; CHECK-NEXT:  .LBB0_2: # %cond.false
 entry:
   %bf.load.i = load i64, i64* null, align 8
   %bf.lshr.i = lshr i64 %bf.load.i, 32
@@ -16,3 +21,14 @@ exit:       ; preds = %entry
 cond.false:                                       ; preds = %entry
   unreachable
 }
+
+define i32 @sh_trunc_sh(i64 %x) {
+; CHECK-LABEL: sh_trunc_sh:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    rldicl 3, 3, 47, 36
+; CHECK-NEXT:    blr
+  %s = lshr i64 %x, 13
+  %t = trunc i64 %s to i32
+  %r = lshr i32 %t, 4
+  ret i32 %r
+}
diff --git a/llvm/test/CodeGen/PowerPC/uint-to-fp-v4i32.ll b/llvm/test/CodeGen/PowerPC/uint-to-fp-v4i32.ll
index 895cb07cd..4afd0e204 100644
--- a/llvm/test/CodeGen/PowerPC/uint-to-fp-v4i32.ll
+++ b/llvm/test/CodeGen/PowerPC/uint-to-fp-v4i32.ll
@@ -1,15 +1,15 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mcpu=pwr9 -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr \
-; RUN:   -mtriple=powerpc64-unknown-unknown < %s | FileCheck -allow-deprecated-dag-overlap %s \
+; RUN:   -mtriple=powerpc64-unknown-unknown < %s | FileCheck  %s \
 ; RUN:   -check-prefix=P9BE
 ; RUN: llc -mcpu=pwr9 -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr \
-; RUN:   -mtriple=powerpc64le-unknown-unknown < %s | FileCheck -allow-deprecated-dag-overlap %s \
+; RUN:   -mtriple=powerpc64le-unknown-unknown < %s | FileCheck  %s \
 ; RUN:   -check-prefix=P9LE
 ; RUN: llc -mcpu=pwr8 -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr \
-; RUN:   -mtriple=powerpc64-unknown-unknown < %s | FileCheck -allow-deprecated-dag-overlap %s \
+; RUN:   -mtriple=powerpc64-unknown-unknown < %s | FileCheck  %s \
 ; RUN:   -check-prefix=P8BE
 ; RUN: llc -mcpu=pwr8 -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr \
-; RUN:   -mtriple=powerpc64le-unknown-unknown < %s | FileCheck -allow-deprecated-dag-overlap %s \
+; RUN:   -mtriple=powerpc64le-unknown-unknown < %s | FileCheck  %s \
 ; RUN:   -check-prefix=P8LE
 define dso_local <2 x double> @test1(<8 x i16> %a) {
 ; P9BE-LABEL: test1:
@@ -17,11 +17,11 @@ define dso_local <2 x double> @test1(<8 x i16> %a) {
 ; P9BE-NEXT:    li r3, 0
 ; P9BE-NEXT:    vextuhlx r3, r3, v2
 ; P9BE-NEXT:    rlwinm r3, r3, 0, 16, 31
-; P9BE-NEXT:    mtvsrwz f0, r3
+; P9BE-NEXT:    mtfprwz f0, r3
 ; P9BE-NEXT:    li r3, 2
 ; P9BE-NEXT:    vextuhlx r3, r3, v2
 ; P9BE-NEXT:    rlwinm r3, r3, 0, 16, 31
-; P9BE-NEXT:    mtvsrwz f1, r3
+; P9BE-NEXT:    mtfprwz f1, r3
 ; P9BE-NEXT:    xscvuxddp f0, f0
 ; P9BE-NEXT:    xscvuxddp f1, f1
 ; P9BE-NEXT:    xxmrghd v2, vs0, vs1
@@ -32,11 +32,11 @@ define dso_local <2 x double> @test1(<8 x i16> %a) {
 ; P9LE-NEXT:    li r3, 0
 ; P9LE-NEXT:    vextuhrx r3, r3, v2
 ; P9LE-NEXT:    rlwinm r3, r3, 0, 16, 31
-; P9LE-NEXT:    mtvsrwz f0, r3
+; P9LE-NEXT:    mtfprwz f0, r3
 ; P9LE-NEXT:    li r3, 2
 ; P9LE-NEXT:    vextuhrx r3, r3, v2
 ; P9LE-NEXT:    rlwinm r3, r3, 0, 16, 31
-; P9LE-NEXT:    mtvsrwz f1, r3
+; P9LE-NEXT:    mtfprwz f1, r3
 ; P9LE-NEXT:    xscvuxddp f0, f0
 ; P9LE-NEXT:    xscvuxddp f1, f1
 ; P9LE-NEXT:    xxmrghd v2, vs1, vs0
@@ -49,8 +49,8 @@ define dso_local <2 x double> @test1(<8 x i16> %a) {
 ; P8BE-NEXT:    rldicl r3, r3, 32, 48
 ; P8BE-NEXT:    rlwinm r4, r4, 0, 16, 31
 ; P8BE-NEXT:    rlwinm r3, r3, 0, 16, 31
-; P8BE-NEXT:    mtvsrwz f0, r4
-; P8BE-NEXT:    mtvsrwz f1, r3
+; P8BE-NEXT:    mtfprwz f0, r4
+; P8BE-NEXT:    mtfprwz f1, r3
 ; P8BE-NEXT:    xscvuxddp f0, f0
 ; P8BE-NEXT:    xscvuxddp f1, f1
 ; P8BE-NEXT:    xxmrghd v2, vs0, vs1
@@ -64,8 +64,8 @@ define dso_local <2 x double> @test1(<8 x i16> %a) {
 ; P8LE-NEXT:    rldicl r3, r3, 48, 48
 ; P8LE-NEXT:    rlwinm r4, r4, 0, 16, 31
 ; P8LE-NEXT:    rlwinm r3, r3, 0, 16, 31
-; P8LE-NEXT:    mtvsrwz f0, r4
-; P8LE-NEXT:    mtvsrwz f1, r3
+; P8LE-NEXT:    mtfprwz f0, r4
+; P8LE-NEXT:    mtfprwz f1, r3
 ; P8LE-NEXT:    xscvuxddp f0, f0
 ; P8LE-NEXT:    xscvuxddp f1, f1
 ; P8LE-NEXT:    xxmrghd v2, vs1, vs0
@@ -103,10 +103,10 @@ define dso_local <2 x double> @test2(<4 x i32> %a, <4 x i32> %b) {
 ; P8BE:       # %bb.0: # %entry
 ; P8BE-NEXT:    xxsldwi vs0, v2, v2, 3
 ; P8BE-NEXT:    mfvsrwz r4, v3
-; P8BE-NEXT:    mtvsrwz f1, r4
+; P8BE-NEXT:    mtfprwz f1, r4
 ; P8BE-NEXT:    mfvsrwz r3, f0
 ; P8BE-NEXT:    xscvuxddp f1, f1
-; P8BE-NEXT:    mtvsrwz f0, r3
+; P8BE-NEXT:    mtfprwz f0, r3
 ; P8BE-NEXT:    xscvuxddp f0, f0
 ; P8BE-NEXT:    xxmrghd v2, vs0, vs1
 ; P8BE-NEXT:    blr
@@ -117,8 +117,8 @@ define dso_local <2 x double> @test2(<4 x i32> %a, <4 x i32> %b) {
 ; P8LE-NEXT:    xxsldwi vs1, v3, v3, 1
 ; P8LE-NEXT:    mfvsrwz r3, f0
 ; P8LE-NEXT:    mfvsrwz r4, f1
-; P8LE-NEXT:    mtvsrwz f0, r3
-; P8LE-NEXT:    mtvsrwz f1, r4
+; P8LE-NEXT:    mtfprwz f0, r3
+; P8LE-NEXT:    mtfprwz f1, r4
 ; P8LE-NEXT:    xscvuxddp f0, f0
 ; P8LE-NEXT:    xscvuxddp f1, f1
 ; P8LE-NEXT:    xxmrghd v2, vs1, vs0
diff --git a/llvm/test/CodeGen/PowerPC/uint-to-ppcfp128-crash.ll b/llvm/test/CodeGen/PowerPC/uint-to-ppcfp128-crash.ll
index ad8dd90ea..bb15367e4 100644
--- a/llvm/test/CodeGen/PowerPC/uint-to-ppcfp128-crash.ll
+++ b/llvm/test/CodeGen/PowerPC/uint-to-ppcfp128-crash.ll
@@ -9,7 +9,7 @@ entry:
   %conv = uitofp i16 %0 to ppc_fp128
   ret ppc_fp128 %conv
 ; CHECK: lhz [[LD:[0-9]+]], 0(3)
-; CHECK: mtvsrwa [[MV:[0-9]+]], [[LD]]
+; CHECK: mtfprwa [[MV:[0-9]+]], [[LD]]
 ; CHECK: xscvsxddp [[CONV:[0-9]+]], [[MV]]
 ; CHECK: bl __gcc_qadd
 }
diff --git a/llvm/test/CodeGen/PowerPC/umulfixsat.ll b/llvm/test/CodeGen/PowerPC/umulfixsat.ll
new file mode 100644
index 000000000..bc41da99e
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/umulfixsat.ll
@@ -0,0 +1,36 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=ppc32 | FileCheck %s
+
+declare  i32 @llvm.umul.fix.sat.i32(i32, i32, i32)
+
+define i32 @func1(i32 %x, i32 %y) nounwind {
+; CHECK-LABEL: func1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li 5, -1
+; CHECK-NEXT:    mulhwu. 6, 3, 4
+; CHECK-NEXT:    mullw 3, 3, 4
+; CHECK-NEXT:    bclr 12, 2, 0
+; CHECK-NEXT:  # %bb.1:
+; CHECK-NEXT:    ori 3, 5, 0
+; CHECK-NEXT:    blr
+  %tmp = call i32 @llvm.umul.fix.sat.i32(i32 %x, i32 %y, i32 0)
+  ret i32 %tmp
+}
+
+define i32 @func2(i32 %x, i32 %y) nounwind {
+; CHECK-LABEL: func2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mulhwu 6, 3, 4
+; CHECK-NEXT:    li 5, -1
+; CHECK-NEXT:    cmplwi 6, 1
+; CHECK-NEXT:    mullw 3, 3, 4
+; CHECK-NEXT:    rotlwi 3, 3, 31
+; CHECK-NEXT:    rlwimi 3, 6, 31, 0, 0
+; CHECK-NEXT:    bc 12, 1, .LBB1_1
+; CHECK-NEXT:    blr
+; CHECK-NEXT:  .LBB1_1:
+; CHECK-NEXT:    addi 3, 5, 0
+; CHECK-NEXT:    blr
+  %tmp = call i32 @llvm.umul.fix.sat.i32(i32 %x, i32 %y, i32 1)
+  ret i32 %tmp
+}
diff --git a/llvm/test/CodeGen/PowerPC/umulo-128-legalisation-lowering.ll b/llvm/test/CodeGen/PowerPC/umulo-128-legalisation-lowering.ll
index 2ca96df78..a6df2661d 100644
--- a/llvm/test/CodeGen/PowerPC/umulo-128-legalisation-lowering.ll
+++ b/llvm/test/CodeGen/PowerPC/umulo-128-legalisation-lowering.ll
@@ -1,164 +1,157 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=powerpc64-unknown-linux-gnu | FileCheck %s --check-prefixes=PPC64
 ; RUN: llc < %s -mtriple=powerpc-unknown-linux-gnu | FileCheck %s --check-prefixes=PPC32
 
 define { i128, i8 } @muloti_test(i128 %l, i128 %r) unnamed_addr #0 {
-
 ; PPC64-LABEL muloti_test:
-; PPC64: mulld 8, 5, 4
-; PPC64-NEXT: cmpdi 5, 3, 0
-; PPC64-NEXT: mulhdu. 9, 3, 6
-; PPC64-NEXT: mulld 3, 3, 6
-; PPC64-NEXT: mcrf 1, 0
-; PPC64-NEXT: add 3, 3, 8
-; PPC64-NEXT: cmpdi   5, 0
-; PPC64-NEXT: crnor 20, 2, 22
-; PPC64-NEXT: cmpldi  3, 0
-; PPC64-NEXT: mulhdu 8, 4, 6
-; PPC64-NEXT: add 3, 8, 3
-; PPC64-NEXT: cmpld 6, 3, 8
-; PPC64-NEXT: crandc 21, 24, 2
-; PPC64-NEXT: crorc 20, 20, 6
-; PPC64-NEXT: li 7, 1
-; PPC64-NEXT: mulhdu. 5, 5, 4
-; PPC64-NEXT: crorc 20, 20, 2
-; PPC64-NEXT: crnor 20, 20, 21
-; PPC64-NEXT: mulld 4, 4, 6
-; PPC64-NEXT: bc 12, 20, .LBB0_2
-; PPC64: ori 5, 7, 0
-; PPC64-NEXT: blr
-; PPC64-NEXT: .LBB0_2:
-; PPC64-NEXT: addi 5, 0, 0
-; PPC64-NEXT: blr
+; PPC64-LABEL: muloti_test:
+; PPC64:       # %bb.0: # %start
+; PPC64-NEXT:    mulld 8, 5, 4
+; PPC64-NEXT:    cmpdi 5, 3, 0
+; PPC64-NEXT:    mulhdu. 9, 3, 6
+; PPC64-NEXT:    mulld 3, 3, 6
+; PPC64-NEXT:    mcrf 1, 0
+; PPC64-NEXT:    add 3, 3, 8
+; PPC64-NEXT:    cmpdi 5, 0
+; PPC64-NEXT:    crnor 20, 2, 22
+; PPC64-NEXT:    cmpldi 3, 0
+; PPC64-NEXT:    mulhdu 8, 4, 6
+; PPC64-NEXT:    add 3, 8, 3
+; PPC64-NEXT:    cmpld 6, 3, 8
+; PPC64-NEXT:    crandc 21, 24, 2
+; PPC64-NEXT:    crorc 20, 20, 6
+; PPC64-NEXT:    li 7, 1
+; PPC64-NEXT:    mulhdu. 5, 5, 4
+; PPC64-NEXT:    crorc 20, 20, 2
+; PPC64-NEXT:    crnor 20, 20, 21
+; PPC64-NEXT:    mulld 4, 4, 6
+; PPC64-NEXT:    bc 12, 20, .LBB0_2
+; PPC64-NEXT:  # %bb.1: # %start
+; PPC64-NEXT:    ori 5, 7, 0
+; PPC64-NEXT:    blr
+; PPC64-NEXT:  .LBB0_2: # %start
+; PPC64-NEXT:    addi 5, 0, 0
+; PPC64-NEXT:    blr
 ;
+; PPC32-LABEL: muloti_test:
+; PPC32:       # %bb.0: # %start
+; PPC32-NEXT:    mflr 0
+; PPC32-NEXT:    stw 0, 4(1)
+; PPC32-NEXT:    stwu 1, -80(1)
+; PPC32-NEXT:    stw 26, 56(1) # 4-byte Folded Spill
+; PPC32-NEXT:    stw 27, 60(1) # 4-byte Folded Spill
+; PPC32-NEXT:    stw 29, 68(1) # 4-byte Folded Spill
+; PPC32-NEXT:    stw 30, 72(1) # 4-byte Folded Spill
+; PPC32-NEXT:    mfcr 12
+; PPC32-NEXT:    mr 30, 8
+; PPC32-NEXT:    mr 29, 7
+; PPC32-NEXT:    mr 27, 4
+; PPC32-NEXT:    mr 26, 3
+; PPC32-NEXT:    li 3, 0
+; PPC32-NEXT:    li 4, 0
+; PPC32-NEXT:    li 7, 0
+; PPC32-NEXT:    li 8, 0
+; PPC32-NEXT:    stw 20, 32(1) # 4-byte Folded Spill
+; PPC32-NEXT:    stw 21, 36(1) # 4-byte Folded Spill
+; PPC32-NEXT:    stw 22, 40(1) # 4-byte Folded Spill
+; PPC32-NEXT:    stw 23, 44(1) # 4-byte Folded Spill
+; PPC32-NEXT:    stw 24, 48(1) # 4-byte Folded Spill
+; PPC32-NEXT:    stw 25, 52(1) # 4-byte Folded Spill
+; PPC32-NEXT:    stw 28, 64(1) # 4-byte Folded Spill
+; PPC32-NEXT:    mr 25, 10
+; PPC32-NEXT:    stw 12, 28(1)
+; PPC32-NEXT:    mr 28, 9
+; PPC32-NEXT:    mr 23, 6
+; PPC32-NEXT:    mr 24, 5
+; PPC32-NEXT:    bl __multi3
+; PPC32-NEXT:    mr 7, 4
+; PPC32-NEXT:    mullw 4, 24, 30
+; PPC32-NEXT:    mullw 8, 29, 23
+; PPC32-NEXT:    mullw 10, 28, 27
+; PPC32-NEXT:    mullw 11, 26, 25
+; PPC32-NEXT:    mulhwu 9, 30, 23
+; PPC32-NEXT:    mulhwu 12, 27, 25
+; PPC32-NEXT:    mullw 0, 30, 23
+; PPC32-NEXT:    mullw 22, 27, 25
+; PPC32-NEXT:    add 21, 8, 4
+; PPC32-NEXT:    add 10, 11, 10
+; PPC32-NEXT:    addc 4, 22, 0
+; PPC32-NEXT:    add 11, 9, 21
+; PPC32-NEXT:    add 0, 12, 10
+; PPC32-NEXT:    adde 8, 0, 11
+; PPC32-NEXT:    addc 4, 7, 4
+; PPC32-NEXT:    adde 8, 3, 8
+; PPC32-NEXT:    xor 22, 4, 7
+; PPC32-NEXT:    xor 20, 8, 3
+; PPC32-NEXT:    or. 22, 22, 20
+; PPC32-NEXT:    mcrf 1, 0
+; PPC32-NEXT:    cmpwi 29, 0
+; PPC32-NEXT:    cmpwi 5, 24, 0
+; PPC32-NEXT:    cmpwi 6, 26, 0
+; PPC32-NEXT:    cmpwi 7, 28, 0
+; PPC32-NEXT:    crnor 8, 22, 2
+; PPC32-NEXT:    mulhwu. 23, 29, 23
+; PPC32-NEXT:    crnor 9, 30, 26
+; PPC32-NEXT:    mcrf 5, 0
+; PPC32-NEXT:    cmplwi 21, 0
+; PPC32-NEXT:    cmplw 6, 11, 9
+; PPC32-NEXT:    cmplwi 7, 10, 0
+; PPC32-NEXT:    crandc 10, 24, 2
+; PPC32-NEXT:    cmplw 3, 0, 12
+; PPC32-NEXT:    mulhwu. 9, 24, 30
+; PPC32-NEXT:    mcrf 6, 0
+; PPC32-NEXT:    crandc 11, 12, 30
+; PPC32-NEXT:    cmplw 4, 7
+; PPC32-NEXT:    cmplw 7, 8, 3
+; PPC32-NEXT:    crand 12, 30, 0
+; PPC32-NEXT:    crandc 13, 28, 30
+; PPC32-NEXT:    mulhwu. 3, 26, 25
+; PPC32-NEXT:    mcrf 7, 0
+; PPC32-NEXT:    cror 0, 12, 13
+; PPC32-NEXT:    crandc 12, 0, 6
+; PPC32-NEXT:    crorc 20, 8, 22
+; PPC32-NEXT:    crorc 20, 20, 26
+; PPC32-NEXT:    mulhwu. 3, 28, 27
+; PPC32-NEXT:    mcrf 1, 0
+; PPC32-NEXT:    crorc 25, 9, 30
+; PPC32-NEXT:    or. 3, 27, 26
+; PPC32-NEXT:    cror 24, 20, 10
+; PPC32-NEXT:    mcrf 5, 0
+; PPC32-NEXT:    crorc 25, 25, 6
+; PPC32-NEXT:    or. 3, 30, 29
+; PPC32-NEXT:    cror 25, 25, 11
+; PPC32-NEXT:    crnor 20, 2, 22
+; PPC32-NEXT:    lwz 12, 28(1)
+; PPC32-NEXT:    cror 20, 20, 25
+; PPC32-NEXT:    cror 20, 20, 24
+; PPC32-NEXT:    crnor 20, 20, 12
+; PPC32-NEXT:    li 3, 1
+; PPC32-NEXT:    bc 12, 20, .LBB0_2
+; PPC32-NEXT:  # %bb.1: # %start
+; PPC32-NEXT:    ori 7, 3, 0
+; PPC32-NEXT:    b .LBB0_3
+; PPC32-NEXT:  .LBB0_2: # %start
+; PPC32-NEXT:    addi 7, 0, 0
+; PPC32-NEXT:  .LBB0_3: # %start
+; PPC32-NEXT:    mr 3, 8
+; PPC32-NEXT:    mtcrf 32, 12 # cr2
+; PPC32-NEXT:    mtcrf 16, 12 # cr3
+; PPC32-NEXT:    lwz 30, 72(1) # 4-byte Folded Reload
+; PPC32-NEXT:    lwz 29, 68(1) # 4-byte Folded Reload
+; PPC32-NEXT:    lwz 28, 64(1) # 4-byte Folded Reload
+; PPC32-NEXT:    lwz 27, 60(1) # 4-byte Folded Reload
+; PPC32-NEXT:    lwz 26, 56(1) # 4-byte Folded Reload
+; PPC32-NEXT:    lwz 25, 52(1) # 4-byte Folded Reload
+; PPC32-NEXT:    lwz 24, 48(1) # 4-byte Folded Reload
+; PPC32-NEXT:    lwz 23, 44(1) # 4-byte Folded Reload
+; PPC32-NEXT:    lwz 22, 40(1) # 4-byte Folded Reload
+; PPC32-NEXT:    lwz 21, 36(1) # 4-byte Folded Reload
+; PPC32-NEXT:    lwz 20, 32(1) # 4-byte Folded Reload
+; PPC32-NEXT:    lwz 0, 84(1)
+; PPC32-NEXT:    addi 1, 1, 80
+; PPC32-NEXT:    mtlr 0
+; PPC32-NEXT:    blr
 ; PPC32-LABEL muloti_test:
-; PPC32: mflr 0
-; PPC32-NEXT: stw 0, 4(1)
-; PPC32-NEXT: stwu 1, -80(1)
-; PPC32-NEXT: .cfi_def_cfa_offset 80
-; PPC32-NEXT: .cfi_offset lr, 4
-; PPC32-NEXT: .cfi_offset r20, -48
-; PPC32-NEXT: .cfi_offset r21, -44
-; PPC32-NEXT: .cfi_offset r22, -40
-; PPC32-NEXT: .cfi_offset r23, -36
-; PPC32-NEXT: .cfi_offset r24, -32
-; PPC32-NEXT: .cfi_offset r25, -28
-; PPC32-NEXT: .cfi_offset r26, -24
-; PPC32-NEXT: .cfi_offset r27, -20
-; PPC32-NEXT: .cfi_offset r28, -16
-; PPC32-NEXT: .cfi_offset r29, -12
-; PPC32-NEXT: .cfi_offset r30, -8
-; PPC32-NEXT: stw 26, 56(1)
-; PPC32-NEXT: stw 27, 60(1)
-; PPC32-NEXT: stw 29, 68(1)
-; PPC32-NEXT: stw 30, 72(1)
-; PPC32-NEXT: mfcr 12
-; PPC32-NEXT: mr 30, 8
-; PPC32-NEXT: mr 29, 7
-; PPC32-NEXT: mr 27, 4
-; PPC32-NEXT: mr 26, 3
-; PPC32-NEXT: li 3, 0
-; PPC32-NEXT: li 4, 0
-; PPC32-NEXT: li 7, 0
-; PPC32-NEXT: li 8, 0
-; PPC32-NEXT: stw 20, 32(1)
-; PPC32-NEXT: stw 21, 36(1)
-; PPC32-NEXT: stw 22, 40(1)
-; PPC32-NEXT: stw 23, 44(1)
-; PPC32-NEXT: stw 24, 48(1)
-; PPC32-NEXT: stw 25, 52(1)
-; PPC32-NEXT: stw 28, 64(1)
-; PPC32-NEXT: mr 25, 10
-; PPC32-NEXT: stw 12, 28(1)
-; PPC32-NEXT: mr 28, 9
-; PPC32-NEXT: mr 23, 6
-; PPC32-NEXT: mr 24, 5
-; PPC32-NEXT: bl __multi3@PLT
-; PPC32-NEXT: mr 7, 4
-; PPC32-NEXT: mullw 4, 24, 30
-; PPC32-NEXT: mullw 8, 29, 23
-; PPC32-NEXT: mullw 10, 28, 27
-; PPC32-NEXT: mullw 11, 26, 25
-; PPC32-NEXT: mulhwu 9, 30, 23
-; PPC32-NEXT: mulhwu 12, 27, 25
-; PPC32-NEXT: mullw 0, 30, 23
-; PPC32-NEXT: mullw 22, 27, 25
-; PPC32-NEXT: add 21, 8, 4
-; PPC32-NEXT: add 10, 11, 10
-; PPC32-NEXT: addc 4, 22, 0
-; PPC32-NEXT: add 11, 9, 21
-; PPC32-NEXT: add 0, 12, 10
-; PPC32-NEXT: adde 8, 0, 11
-; PPC32-NEXT: addc 4, 7, 4
-; PPC32-NEXT: adde 8, 3, 8
-; PPC32-NEXT: xor 22, 4, 7
-; PPC32-NEXT: xor 20, 8, 3
-; PPC32-NEXT: or. 22, 22, 20
-; PPC32-NEXT: mcrf 1, 0
-; PPC32-NEXT: cmpwi   29, 0
-; PPC32-NEXT: cmpwi 5, 24, 0
-; PPC32-NEXT: cmpwi 6, 26, 0
-; PPC32-NEXT: cmpwi 7, 28, 0
-; PPC32-NEXT: crnor 8, 22, 2
-; PPC32-NEXT: mulhwu. 23, 29, 23
-; PPC32-NEXT: crnor 9, 30, 26
-; PPC32-NEXT: mcrf 5, 0
-; PPC32-NEXT: cmplwi  21, 0
-; PPC32-NEXT: cmplw 6, 11, 9
-; PPC32-NEXT: cmplwi 7, 10, 0
-; PPC32-NEXT: crandc 10, 24, 2
-; PPC32-NEXT: cmplw 3, 0, 12
-; PPC32-NEXT: mulhwu. 9, 24, 30
-; PPC32-NEXT: mcrf 6, 0
-; PPC32-NEXT: crandc 11, 12, 30
-; PPC32-NEXT: cmplw   4, 7
-; PPC32-NEXT: cmplw 7, 8, 3
-; PPC32-NEXT: crand 12, 30, 0
-; PPC32-NEXT: crandc 13, 28, 30
-; PPC32-NEXT: mulhwu. 3, 26, 25
-; PPC32-NEXT: mcrf 7, 0
-; PPC32-NEXT: cror 0, 12, 13
-; PPC32-NEXT: crandc 12, 0, 6
-; PPC32-NEXT: crorc 20, 8, 22
-; PPC32-NEXT: crorc 20, 20, 26
-; PPC32-NEXT: mulhwu. 3, 28, 27
-; PPC32-NEXT: mcrf 1, 0
-; PPC32-NEXT: crorc 25, 9, 30
-; PPC32-NEXT: or. 3, 27, 26
-; PPC32-NEXT: cror 24, 20, 10
-; PPC32-NEXT: mcrf 5, 0
-; PPC32-NEXT: crorc 25, 25, 6
-; PPC32-NEXT: or. 3, 30, 29
-; PPC32-NEXT: cror 25, 25, 11
-; PPC32-NEXT: crnor 20, 2, 22
-; PPC32-NEXT: lwz 12, 28(1)
-; PPC32-NEXT: cror 20, 20, 25
-; PPC32-NEXT: cror 20, 20, 24
-; PPC32-NEXT: crnor 20, 20, 12
-; PPC32-NEXT: li 3, 1
-; PPC32-NEXT: bc 12, 20, .LBB0_2
-; PPC32: ori 7, 3, 0
-; PPC32-NEXT: b .LBB0_3
-; PPC32-NEXT:.LBB0_2:
-; PPC32-NEXT: addi 7, 0, 0
-; PPC32-NEXT:.LBB0_3:
-; PPC32-NEXT: mr 3, 8
-; PPC32-NEXT: mtcrf 32, 12
-; PPC32-NEXT: mtcrf 16, 12
-; PPC32-NEXT: lwz 30, 72(1)
-; PPC32-NEXT: lwz 29, 68(1)
-; PPC32-NEXT: lwz 28, 64(1)
-; PPC32-NEXT: lwz 27, 60(1)
-; PPC32-NEXT: lwz 26, 56(1)
-; PPC32-NEXT: lwz 25, 52(1)
-; PPC32-NEXT: lwz 24, 48(1)
-; PPC32-NEXT: lwz 23, 44(1)
-; PPC32-NEXT: lwz 22, 40(1)
-; PPC32-NEXT: lwz 21, 36(1)
-; PPC32-NEXT: lwz 20, 32(1)
-; PPC32-NEXT: lwz 0, 84(1)
-; PPC32-NEXT: addi 1, 1, 80
-; PPC32-NEXT: mtlr 0
-; PPC32-NEXT: blr
 start:
   %0 = tail call { i128, i1 } @llvm.umul.with.overflow.i128(i128 %l, i128 %r) #2
   %1 = extractvalue { i128, i1 } %0, 0
@@ -172,6 +165,6 @@ start:
 ; Function Attrs: nounwind readnone speculatable
 declare { i128, i1 } @llvm.umul.with.overflow.i128(i128, i128) #1
 
-attributes #0 = { nounwind readnone uwtable }
+attributes #0 = { nounwind readnone }
 attributes #1 = { nounwind readnone speculatable }
 attributes #2 = { nounwind }
diff --git a/llvm/test/CodeGen/PowerPC/unal-vec-ldst.ll b/llvm/test/CodeGen/PowerPC/unal-vec-ldst.ll
index f1bd2af4c..497add38e 100644
--- a/llvm/test/CodeGen/PowerPC/unal-vec-ldst.ll
+++ b/llvm/test/CodeGen/PowerPC/unal-vec-ldst.ll
@@ -1,577 +1,622 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -verify-machineinstrs < %s | FileCheck %s
 target datalayout = "E-m:e-i64:64-n32:64"
 target triple = "powerpc64-unknown-linux-gnu"
 
 define <16 x i8> @test_l_v16i8(<16 x i8>* %p) #0 {
+; CHECK-LABEL: test_l_v16i8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    li 4, 15
+; CHECK-NEXT:    lvsl 3, 0, 3
+; CHECK-NEXT:    lvx 2, 3, 4
+; CHECK-NEXT:    lvx 4, 0, 3
+; CHECK-NEXT:    vperm 2, 4, 2, 3
+; CHECK-NEXT:    blr
 entry:
   %r = load <16 x i8>, <16 x i8>* %p, align 1
   ret <16 x i8> %r
 
-; CHECK-LABEL: @test_l_v16i8
-; CHECK-DAG: li [[REG1:[0-9]+]], 15
-; CHECK-DAG: lvsl [[REG2:[0-9]+]], 0, 3
-; CHECK-DAG: lvx [[REG3:[0-9]+]], 3, [[REG1]]
-; CHECK-DAG: lvx [[REG4:[0-9]+]], 0, 3
-; CHECK: vperm 2, [[REG4]], [[REG3]], [[REG2]]
-; CHECK: blr
 }
 
 define <32 x i8> @test_l_v32i8(<32 x i8>* %p) #0 {
+; CHECK-LABEL: test_l_v32i8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    li 4, 31
+; CHECK-NEXT:    lvsl 5, 0, 3
+; CHECK-NEXT:    lvx 2, 3, 4
+; CHECK-NEXT:    li 4, 16
+; CHECK-NEXT:    lvx 4, 3, 4
+; CHECK-NEXT:    lvx 0, 0, 3
+; CHECK-NEXT:    vperm 3, 4, 2, 5
+; CHECK-NEXT:    vperm 2, 0, 4, 5
+; CHECK-NEXT:    blr
 entry:
   %r = load <32 x i8>, <32 x i8>* %p, align 1
   ret <32 x i8> %r
 
-; CHECK-LABEL: @test_l_v32i8
-; CHECK-DAG: li [[REG1:[0-9]+]], 31
-; CHECK-DAG: li [[REG2:[0-9]+]], 16
-; CHECK-DAG: lvsl [[REG3:[0-9]+]], 0, 3
-; CHECK-DAG: lvx [[REG4:[0-9]+]], 3, [[REG1]]
-; CHECK-DAG: lvx [[REG5:[0-9]+]], 3, [[REG2]]
-; CHECK-DAG: lvx [[REG6:[0-9]+]], 0, 3
-; CHECK-DAG: vperm 3, {{[0-9]+}}, {{[0-9]+}}, [[REG3]]
-; CHECK-DAG: vperm 2, {{[0-9]+}}, {{[0-9]+}}, [[REG3]]
-; CHECK: blr
 }
 
 define <8 x i16> @test_l_v8i16(<8 x i16>* %p) #0 {
+; CHECK-LABEL: test_l_v8i16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    li 4, 15
+; CHECK-NEXT:    lvsl 3, 0, 3
+; CHECK-NEXT:    lvx 2, 3, 4
+; CHECK-NEXT:    lvx 4, 0, 3
+; CHECK-NEXT:    vperm 2, 4, 2, 3
+; CHECK-NEXT:    blr
 entry:
   %r = load <8 x i16>, <8 x i16>* %p, align 2
   ret <8 x i16> %r
 
-; CHECK-LABEL: @test_l_v8i16
-; CHECK-DAG: li [[REG1:[0-9]+]], 15
-; CHECK-DAG: lvsl [[REG2:[0-9]+]], 0, 3
-; CHECK-DAG: lvx [[REG3:[0-9]+]], 3, [[REG1]]
-; CHECK-DAG: lvx [[REG4:[0-9]+]], 0, 3
-; CHECK: vperm 2, [[REG4]], [[REG3]], [[REG2]]
-; CHECK: blr
 }
 
 define <16 x i16> @test_l_v16i16(<16 x i16>* %p) #0 {
+; CHECK-LABEL: test_l_v16i16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    li 4, 31
+; CHECK-NEXT:    lvsl 5, 0, 3
+; CHECK-NEXT:    lvx 2, 3, 4
+; CHECK-NEXT:    li 4, 16
+; CHECK-NEXT:    lvx 4, 3, 4
+; CHECK-NEXT:    lvx 0, 0, 3
+; CHECK-NEXT:    vperm 3, 4, 2, 5
+; CHECK-NEXT:    vperm 2, 0, 4, 5
+; CHECK-NEXT:    blr
 entry:
   %r = load <16 x i16>, <16 x i16>* %p, align 2
   ret <16 x i16> %r
 
-; CHECK-LABEL: @test_l_v16i16
-; CHECK-DAG: li [[REG1:[0-9]+]], 31
-; CHECK-DAG: li [[REG2:[0-9]+]], 16
-; CHECK-DAG: lvsl [[REG3:[0-9]+]], 0, 3
-; CHECK-DAG: lvx [[REG4:[0-9]+]], 3, [[REG1]]
-; CHECK-DAG: lvx [[REG5:[0-9]+]], 3, [[REG2]]
-; CHECK-DAG: lvx [[REG6:[0-9]+]], 0, 3
-; CHECK-DAG: vperm 3, {{[0-9]+}}, {{[0-9]+}}, [[REG3]]
-; CHECK-DAG: vperm 2, {{[0-9]+}}, {{[0-9]+}}, [[REG3]]
-; CHECK: blr
 }
 
 define <4 x i32> @test_l_v4i32(<4 x i32>* %p) #0 {
+; CHECK-LABEL: test_l_v4i32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    li 4, 15
+; CHECK-NEXT:    lvsl 3, 0, 3
+; CHECK-NEXT:    lvx 2, 3, 4
+; CHECK-NEXT:    lvx 4, 0, 3
+; CHECK-NEXT:    vperm 2, 4, 2, 3
+; CHECK-NEXT:    blr
 entry:
   %r = load <4 x i32>, <4 x i32>* %p, align 4
   ret <4 x i32> %r
 
-; CHECK-LABEL: @test_l_v4i32
-; CHECK-DAG: li [[REG1:[0-9]+]], 15
-; CHECK-DAG: lvsl [[REG2:[0-9]+]], 0, 3
-; CHECK-DAG: lvx [[REG3:[0-9]+]], 3, [[REG1]]
-; CHECK-DAG: lvx [[REG4:[0-9]+]], 0, 3
-; CHECK: vperm 2, [[REG4]], [[REG3]], [[REG2]]
-; CHECK: blr
 }
 
 define <8 x i32> @test_l_v8i32(<8 x i32>* %p) #0 {
+; CHECK-LABEL: test_l_v8i32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    li 4, 31
+; CHECK-NEXT:    lvsl 5, 0, 3
+; CHECK-NEXT:    lvx 2, 3, 4
+; CHECK-NEXT:    li 4, 16
+; CHECK-NEXT:    lvx 4, 3, 4
+; CHECK-NEXT:    lvx 0, 0, 3
+; CHECK-NEXT:    vperm 3, 4, 2, 5
+; CHECK-NEXT:    vperm 2, 0, 4, 5
+; CHECK-NEXT:    blr
 entry:
   %r = load <8 x i32>, <8 x i32>* %p, align 4
   ret <8 x i32> %r
 
-; CHECK-LABEL: @test_l_v8i32
-; CHECK-DAG: li [[REG1:[0-9]+]], 31
-; CHECK-DAG: li [[REG2:[0-9]+]], 16
-; CHECK-DAG: lvsl [[REG3:[0-9]+]], 0, 3
-; CHECK-DAG: lvx [[REG4:[0-9]+]], 3, [[REG1]]
-; CHECK-DAG: lvx [[REG5:[0-9]+]], 3, [[REG2]]
-; CHECK-DAG: lvx [[REG6:[0-9]+]], 0, 3
-; CHECK-DAG: vperm 3, {{[0-9]+}}, {{[0-9]+}}, [[REG3]]
-; CHECK-DAG: vperm 2, {{[0-9]+}}, {{[0-9]+}}, [[REG3]]
-; CHECK: blr
 }
 
 define <2 x i64> @test_l_v2i64(<2 x i64>* %p) #0 {
+; CHECK-LABEL: test_l_v2i64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lxvd2x 34, 0, 3
+; CHECK-NEXT:    blr
 entry:
   %r = load <2 x i64>, <2 x i64>* %p, align 8
   ret <2 x i64> %r
 
-; CHECK-LABEL: @test_l_v2i64
-; CHECK: lxvd2x 34, 0, 3
-; CHECK: blr
 }
 
 define <4 x i64> @test_l_v4i64(<4 x i64>* %p) #0 {
+; CHECK-LABEL: test_l_v4i64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    li 4, 16
+; CHECK-NEXT:    lxvd2x 34, 0, 3
+; CHECK-NEXT:    lxvd2x 35, 3, 4
+; CHECK-NEXT:    blr
 entry:
   %r = load <4 x i64>, <4 x i64>* %p, align 8
   ret <4 x i64> %r
 
-; CHECK-LABEL: @test_l_v4i64
-; CHECK-DAG: li [[REG1:[0-9]+]], 16
-; CHECK-DAG: lxvd2x 34, 0, 3
-; CHECK-DAG: lxvd2x 35, 3, [[REG1]]
-; CHECK: blr
 }
 
 define <4 x float> @test_l_v4float(<4 x float>* %p) #0 {
+; CHECK-LABEL: test_l_v4float:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    li 4, 15
+; CHECK-NEXT:    lvsl 3, 0, 3
+; CHECK-NEXT:    lvx 2, 3, 4
+; CHECK-NEXT:    lvx 4, 0, 3
+; CHECK-NEXT:    vperm 2, 4, 2, 3
+; CHECK-NEXT:    blr
 entry:
   %r = load <4 x float>, <4 x float>* %p, align 4
   ret <4 x float> %r
 
-; CHECK-LABEL: @test_l_v4float
-; CHECK-DAG: li [[REG1:[0-9]+]], 15
-; CHECK-DAG: lvsl [[REG2:[0-9]+]], 0, 3
-; CHECK-DAG: lvx [[REG3:[0-9]+]], 3, [[REG1]]
-; CHECK-DAG: lvx [[REG4:[0-9]+]], 0, 3
-; CHECK: vperm 2, [[REG4]], [[REG3]], [[REG2]]
-; CHECK: blr
 }
 
 define <8 x float> @test_l_v8float(<8 x float>* %p) #0 {
+; CHECK-LABEL: test_l_v8float:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    li 4, 31
+; CHECK-NEXT:    lvsl 5, 0, 3
+; CHECK-NEXT:    lvx 2, 3, 4
+; CHECK-NEXT:    li 4, 16
+; CHECK-NEXT:    lvx 4, 3, 4
+; CHECK-NEXT:    lvx 0, 0, 3
+; CHECK-NEXT:    vperm 3, 4, 2, 5
+; CHECK-NEXT:    vperm 2, 0, 4, 5
+; CHECK-NEXT:    blr
 entry:
   %r = load <8 x float>, <8 x float>* %p, align 4
   ret <8 x float> %r
 
-; CHECK-LABEL: @test_l_v8float
-; CHECK-DAG: li [[REG1:[0-9]+]], 31
-; CHECK-DAG: li [[REG2:[0-9]+]], 16
-; CHECK-DAG: lvsl [[REG3:[0-9]+]], 0, 3
-; CHECK-DAG: lvx [[REG4:[0-9]+]], 3, [[REG1]]
-; CHECK-DAG: lvx [[REG5:[0-9]+]], 3, [[REG2]]
-; CHECK-DAG: lvx [[REG6:[0-9]+]], 0, 3
-; CHECK-DAG: vperm 3, {{[0-9]+}}, {{[0-9]+}}, [[REG3]]
-; CHECK-DAG: vperm 2, {{[0-9]+}}, {{[0-9]+}}, [[REG3]]
-; CHECK: blr
 }
 
 define <2 x double> @test_l_v2double(<2 x double>* %p) #0 {
+; CHECK-LABEL: test_l_v2double:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lxvd2x 34, 0, 3
+; CHECK-NEXT:    blr
 entry:
   %r = load <2 x double>, <2 x double>* %p, align 8
   ret <2 x double> %r
 
-; CHECK-LABEL: @test_l_v2double
-; CHECK: lxvd2x 34, 0, 3
-; CHECK: blr
 }
 
 define <4 x double> @test_l_v4double(<4 x double>* %p) #0 {
+; CHECK-LABEL: test_l_v4double:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    li 4, 16
+; CHECK-NEXT:    lxvd2x 34, 0, 3
+; CHECK-NEXT:    lxvd2x 35, 3, 4
+; CHECK-NEXT:    blr
 entry:
   %r = load <4 x double>, <4 x double>* %p, align 8
   ret <4 x double> %r
 
-; CHECK-LABEL: @test_l_v4double
-; CHECK-DAG: li [[REG1:[0-9]+]], 16
-; CHECK-DAG: lxvd2x 34, 0, 3
-; CHECK-DAG: lxvd2x 35, 3, [[REG1]]
-; CHECK: blr
 }
 
 define <16 x i8> @test_l_p8v16i8(<16 x i8>* %p) #2 {
+; CHECK-LABEL: test_l_p8v16i8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lxvw4x 34, 0, 3
+; CHECK-NEXT:    blr
 entry:
   %r = load <16 x i8>, <16 x i8>* %p, align 1
   ret <16 x i8> %r
 
-; CHECK-LABEL: @test_l_p8v16i8
-; CHECK: lxvw4x 34, 0, 3
-; CHECK: blr
 }
 
 define <32 x i8> @test_l_p8v32i8(<32 x i8>* %p) #2 {
+; CHECK-LABEL: test_l_p8v32i8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    li 4, 16
+; CHECK-NEXT:    lxvw4x 34, 0, 3
+; CHECK-NEXT:    lxvw4x 35, 3, 4
+; CHECK-NEXT:    blr
 entry:
   %r = load <32 x i8>, <32 x i8>* %p, align 1
   ret <32 x i8> %r
 
-; CHECK-LABEL: @test_l_p8v32i8
-; CHECK-DAG: li [[REG1:[0-9]+]], 16
-; CHECK-DAG: lxvw4x 34, 0, 3
-; CHECK-DAG: lxvw4x 35, 3, [[REG1]]
-; CHECK: blr
 }
 
 define <8 x i16> @test_l_p8v8i16(<8 x i16>* %p) #2 {
+; CHECK-LABEL: test_l_p8v8i16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lxvw4x 34, 0, 3
+; CHECK-NEXT:    blr
 entry:
   %r = load <8 x i16>, <8 x i16>* %p, align 2
   ret <8 x i16> %r
 
-; CHECK-LABEL: @test_l_p8v8i16
-; CHECK: lxvw4x 34, 0, 3
-; CHECK: blr
 }
 
 define <16 x i16> @test_l_p8v16i16(<16 x i16>* %p) #2 {
+; CHECK-LABEL: test_l_p8v16i16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    li 4, 16
+; CHECK-NEXT:    lxvw4x 34, 0, 3
+; CHECK-NEXT:    lxvw4x 35, 3, 4
+; CHECK-NEXT:    blr
 entry:
   %r = load <16 x i16>, <16 x i16>* %p, align 2
   ret <16 x i16> %r
 
-; CHECK-LABEL: @test_l_p8v16i16
-; CHECK-DAG: li [[REG1:[0-9]+]], 16
-; CHECK-DAG: lxvw4x 34, 0, 3
-; CHECK-DAG: lxvw4x 35, 3, [[REG1]]
-; CHECK: blr
 }
 
 define <4 x i32> @test_l_p8v4i32(<4 x i32>* %p) #2 {
+; CHECK-LABEL: test_l_p8v4i32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lxvw4x 34, 0, 3
+; CHECK-NEXT:    blr
 entry:
   %r = load <4 x i32>, <4 x i32>* %p, align 4
   ret <4 x i32> %r
 
-; CHECK-LABEL: @test_l_p8v4i32
-; CHECK: lxvw4x 34, 0, 3
-; CHECK: blr
 }
 
 define <8 x i32> @test_l_p8v8i32(<8 x i32>* %p) #2 {
+; CHECK-LABEL: test_l_p8v8i32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    li 4, 16
+; CHECK-NEXT:    lxvw4x 34, 0, 3
+; CHECK-NEXT:    lxvw4x 35, 3, 4
+; CHECK-NEXT:    blr
 entry:
   %r = load <8 x i32>, <8 x i32>* %p, align 4
   ret <8 x i32> %r
 
-; CHECK-LABEL: @test_l_p8v8i32
-; CHECK-DAG: li [[REG1:[0-9]+]], 16
-; CHECK-DAG: lxvw4x 34, 0, 3
-; CHECK-DAG: lxvw4x 35, 3, [[REG1]]
-; CHECK: blr
 }
 
 define <2 x i64> @test_l_p8v2i64(<2 x i64>* %p) #2 {
+; CHECK-LABEL: test_l_p8v2i64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lxvd2x 34, 0, 3
+; CHECK-NEXT:    blr
 entry:
   %r = load <2 x i64>, <2 x i64>* %p, align 8
   ret <2 x i64> %r
 
-; CHECK-LABEL: @test_l_p8v2i64
-; CHECK: lxvd2x 34, 0, 3
-; CHECK: blr
 }
 
 define <4 x i64> @test_l_p8v4i64(<4 x i64>* %p) #2 {
+; CHECK-LABEL: test_l_p8v4i64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    li 4, 16
+; CHECK-NEXT:    lxvd2x 34, 0, 3
+; CHECK-NEXT:    lxvd2x 35, 3, 4
+; CHECK-NEXT:    blr
 entry:
   %r = load <4 x i64>, <4 x i64>* %p, align 8
   ret <4 x i64> %r
 
-; CHECK-LABEL: @test_l_p8v4i64
-; CHECK-DAG: li [[REG1:[0-9]+]], 16
-; CHECK-DAG: lxvd2x 34, 0, 3
-; CHECK-DAG: lxvd2x 35, 3, [[REG1]]
-; CHECK: blr
 }
 
 define <4 x float> @test_l_p8v4float(<4 x float>* %p) #2 {
+; CHECK-LABEL: test_l_p8v4float:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lxvw4x 34, 0, 3
+; CHECK-NEXT:    blr
 entry:
   %r = load <4 x float>, <4 x float>* %p, align 4
   ret <4 x float> %r
 
-; CHECK-LABEL: @test_l_p8v4float
-; CHECK: lxvw4x 34, 0, 3
-; CHECK: blr
 }
 
 define <8 x float> @test_l_p8v8float(<8 x float>* %p) #2 {
+; CHECK-LABEL: test_l_p8v8float:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    li 4, 16
+; CHECK-NEXT:    lxvw4x 34, 0, 3
+; CHECK-NEXT:    lxvw4x 35, 3, 4
+; CHECK-NEXT:    blr
 entry:
   %r = load <8 x float>, <8 x float>* %p, align 4
   ret <8 x float> %r
 
-; CHECK-LABEL: @test_l_p8v8float
-; CHECK-DAG: li [[REG1:[0-9]+]], 16
-; CHECK-DAG: lxvw4x 34, 0, 3
-; CHECK-DAG: lxvw4x 35, 3, [[REG1]]
-; CHECK: blr
 }
 
 define <2 x double> @test_l_p8v2double(<2 x double>* %p) #2 {
+; CHECK-LABEL: test_l_p8v2double:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lxvd2x 34, 0, 3
+; CHECK-NEXT:    blr
 entry:
   %r = load <2 x double>, <2 x double>* %p, align 8
   ret <2 x double> %r
 
-; CHECK-LABEL: @test_l_p8v2double
-; CHECK: lxvd2x 34, 0, 3
-; CHECK: blr
 }
 
 define <4 x double> @test_l_p8v4double(<4 x double>* %p) #2 {
+; CHECK-LABEL: test_l_p8v4double:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    li 4, 16
+; CHECK-NEXT:    lxvd2x 34, 0, 3
+; CHECK-NEXT:    lxvd2x 35, 3, 4
+; CHECK-NEXT:    blr
 entry:
   %r = load <4 x double>, <4 x double>* %p, align 8
   ret <4 x double> %r
 
-; CHECK-LABEL: @test_l_p8v4double
-; CHECK-DAG: li [[REG1:[0-9]+]], 16
-; CHECK-DAG: lxvd2x 34, 0, 3
-; CHECK-DAG: lxvd2x 35, 3, [[REG1]]
-; CHECK: blr
 }
 
 define <4 x float> @test_l_qv4float(<4 x float>* %p) #1 {
+; CHECK-LABEL: test_l_qv4float:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    li 4, 15
+; CHECK-NEXT:    qvlpclsx 0, 0, 3
+; CHECK-NEXT:    qvlfsx 1, 3, 4
+; CHECK-NEXT:    qvlfsx 2, 0, 3
+; CHECK-NEXT:    qvfperm 1, 2, 1, 0
+; CHECK-NEXT:    blr
 entry:
   %r = load <4 x float>, <4 x float>* %p, align 4
   ret <4 x float> %r
 
-; CHECK-LABEL: @test_l_qv4float
-; CHECK-DAG: li [[REG1:[0-9]+]], 15
-; CHECK-DAG: qvlpclsx 0, 0, 3
-; CHECK-DAG: qvlfsx [[REG2:[0-9]+]], 3, [[REG1]]
-; CHECK-DAG: qvlfsx [[REG3:[0-9]+]], 0, 3
-; CHECK: qvfperm 1, [[REG3]], [[REG2]], 0
-; CHECK: blr
 }
 
 define <8 x float> @test_l_qv8float(<8 x float>* %p) #1 {
+; CHECK-LABEL: test_l_qv8float:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    li 4, 31
+; CHECK-NEXT:    qvlpclsx 1, 0, 3
+; CHECK-NEXT:    qvlfsx 0, 3, 4
+; CHECK-NEXT:    li 4, 16
+; CHECK-NEXT:    qvlfsx 3, 3, 4
+; CHECK-NEXT:    qvlfsx 4, 0, 3
+; CHECK-NEXT:    qvfperm 2, 3, 0, 1
+; CHECK-NEXT:    qvfperm 1, 4, 3, 1
+; CHECK-NEXT:    blr
 entry:
   %r = load <8 x float>, <8 x float>* %p, align 4
   ret <8 x float> %r
 
-; CHECK-LABEL: @test_l_qv8float
-; CHECK-DAG: li [[REG1:[0-9]+]], 31
-; CHECK-DAG: li [[REG2:[0-9]+]], 16
-; CHECK-DAG: qvlfsx [[REG3:[0-9]+]], 3, [[REG1]]
-; CHECK-DAG: qvlfsx [[REG4:[0-9]+]], 3, [[REG2]]
-; CHECK-DAG: qvlpclsx [[REG5:[0-5]+]], 0, 3
-; CHECK-DAG: qvlfsx [[REG6:[0-9]+]], 0, 3
-; CHECK-DAG: qvfperm 2, {{[0-9]+}}, {{[0-9]+}}, [[REG5]]
-; CHECK-DAG: qvfperm 1, {{[0-9]+}}, {{[0-9]+}}, [[REG5]]
-; CHECK: blr
 }
 
 define <4 x double> @test_l_qv4double(<4 x double>* %p) #1 {
+; CHECK-LABEL: test_l_qv4double:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    li 4, 31
+; CHECK-NEXT:    qvlpcldx 0, 0, 3
+; CHECK-NEXT:    qvlfdx 1, 3, 4
+; CHECK-NEXT:    qvlfdx 2, 0, 3
+; CHECK-NEXT:    qvfperm 1, 2, 1, 0
+; CHECK-NEXT:    blr
 entry:
   %r = load <4 x double>, <4 x double>* %p, align 8
   ret <4 x double> %r
 
-; CHECK-LABEL: @test_l_qv4double
-; CHECK-DAG: li [[REG1:[0-9]+]], 31
-; CHECK-DAG: qvlpcldx 0, 0, 3
-; CHECK-DAG: qvlfdx [[REG2:[0-9]+]], 3, [[REG1]]
-; CHECK-DAG: qvlfdx [[REG3:[0-9]+]], 0, 3
-; CHECK: qvfperm 1, [[REG3]], [[REG2]], 0
-; CHECK: blr
 }
 
 define <8 x double> @test_l_qv8double(<8 x double>* %p) #1 {
+; CHECK-LABEL: test_l_qv8double:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    li 4, 63
+; CHECK-NEXT:    qvlpcldx 1, 0, 3
+; CHECK-NEXT:    qvlfdx 0, 3, 4
+; CHECK-NEXT:    li 4, 32
+; CHECK-NEXT:    qvlfdx 3, 3, 4
+; CHECK-NEXT:    qvlfdx 4, 0, 3
+; CHECK-NEXT:    qvfperm 2, 3, 0, 1
+; CHECK-NEXT:    qvfperm 1, 4, 3, 1
+; CHECK-NEXT:    blr
 entry:
   %r = load <8 x double>, <8 x double>* %p, align 8
   ret <8 x double> %r
 
-; CHECK-LABEL: @test_l_qv8double
-; CHECK-DAG: li [[REG1:[0-9]+]], 63
-; CHECK-DAG: li [[REG2:[0-9]+]], 32
-; CHECK-DAG: qvlfdx [[REG3:[0-9]+]], 3, [[REG1]]
-; CHECK-DAG: qvlfdx [[REG4:[0-9]+]], 3, [[REG2]]
-; CHECK-DAG: qvlpcldx [[REG5:[0-5]+]], 0, 3
-; CHECK-DAG: qvlfdx [[REG6:[0-9]+]], 0, 3
-; CHECK-DAG: qvfperm 2, {{[0-9]+}}, {{[0-9]+}}, [[REG5]]
-; CHECK-DAG: qvfperm 1, {{[0-9]+}}, {{[0-9]+}}, [[REG5]]
-; CHECK: blr
 }
 
 define void @test_s_v16i8(<16 x i8>* %p, <16 x i8> %v) #0 {
+; CHECK-LABEL: test_s_v16i8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    stxvw4x 34, 0, 3
+; CHECK-NEXT:    blr
 entry:
   store <16 x i8> %v, <16 x i8>* %p, align 1
   ret void
 
-; CHECK-LABEL: @test_s_v16i8
-; CHECK: stxvw4x 34, 0, 3
-; CHECK: blr
 }
 
 define void @test_s_v32i8(<32 x i8>* %p, <32 x i8> %v) #0 {
+; CHECK-LABEL: test_s_v32i8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    li 4, 16
+; CHECK-NEXT:    stxvw4x 34, 0, 3
+; CHECK-NEXT:    stxvw4x 35, 3, 4
+; CHECK-NEXT:    blr
 entry:
   store <32 x i8> %v, <32 x i8>* %p, align 1
   ret void
 
-; CHECK-LABEL: @test_s_v32i8
-; CHECK-DAG: li [[REG1:[0-9]+]], 16
-; CHECK-DAG: stxvw4x 35, 3, [[REG1]]
-; CHECK-DAG: stxvw4x 34, 0, 3
-; CHECK: blr
 }
 
 define void @test_s_v8i16(<8 x i16>* %p, <8 x i16> %v) #0 {
+; CHECK-LABEL: test_s_v8i16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    stxvw4x 34, 0, 3
+; CHECK-NEXT:    blr
 entry:
   store <8 x i16> %v, <8 x i16>* %p, align 2
   ret void
 
-; CHECK-LABEL: @test_s_v8i16
-; CHECK: stxvw4x 34, 0, 3
-; CHECK: blr
 }
 
 define void @test_s_v16i16(<16 x i16>* %p, <16 x i16> %v) #0 {
+; CHECK-LABEL: test_s_v16i16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    li 4, 16
+; CHECK-NEXT:    stxvw4x 34, 0, 3
+; CHECK-NEXT:    stxvw4x 35, 3, 4
+; CHECK-NEXT:    blr
 entry:
   store <16 x i16> %v, <16 x i16>* %p, align 2
   ret void
 
-; CHECK-LABEL: @test_s_v16i16
-; CHECK-DAG: li [[REG1:[0-9]+]], 16
-; CHECK-DAG: stxvw4x 35, 3, [[REG1]]
-; CHECK-DAG: stxvw4x 34, 0, 3
-; CHECK: blr
 }
 
 define void @test_s_v4i32(<4 x i32>* %p, <4 x i32> %v) #0 {
+; CHECK-LABEL: test_s_v4i32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    stxvw4x 34, 0, 3
+; CHECK-NEXT:    blr
 entry:
   store <4 x i32> %v, <4 x i32>* %p, align 4
   ret void
 
-; CHECK-LABEL: @test_s_v4i32
-; CHECK: stxvw4x 34, 0, 3
-; CHECK: blr
 }
 
 define void @test_s_v8i32(<8 x i32>* %p, <8 x i32> %v) #0 {
+; CHECK-LABEL: test_s_v8i32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    li 4, 16
+; CHECK-NEXT:    stxvw4x 34, 0, 3
+; CHECK-NEXT:    stxvw4x 35, 3, 4
+; CHECK-NEXT:    blr
 entry:
   store <8 x i32> %v, <8 x i32>* %p, align 4
   ret void
 
-; CHECK-LABEL: @test_s_v8i32
-; CHECK-DAG: li [[REG1:[0-9]+]], 16
-; CHECK-DAG: stxvw4x 35, 3, [[REG1]]
-; CHECK-DAG: stxvw4x 34, 0, 3
-; CHECK: blr
 }
 
 define void @test_s_v2i64(<2 x i64>* %p, <2 x i64> %v) #0 {
+; CHECK-LABEL: test_s_v2i64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    stxvd2x 34, 0, 3
+; CHECK-NEXT:    blr
 entry:
   store <2 x i64> %v, <2 x i64>* %p, align 8
   ret void
 
-; CHECK-LABEL: @test_s_v2i64
-; CHECK: stxvd2x 34, 0, 3
-; CHECK: blr
 }
 
 define void @test_s_v4i64(<4 x i64>* %p, <4 x i64> %v) #0 {
+; CHECK-LABEL: test_s_v4i64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    li 4, 16
+; CHECK-NEXT:    stxvd2x 34, 0, 3
+; CHECK-NEXT:    stxvd2x 35, 3, 4
+; CHECK-NEXT:    blr
 entry:
   store <4 x i64> %v, <4 x i64>* %p, align 8
   ret void
 
-; CHECK-LABEL: @test_s_v4i64
-; CHECK-DAG: li [[REG1:[0-9]+]], 16
-; CHECK-DAG: stxvd2x 35, 3, [[REG1]]
-; CHECK-DAG: stxvd2x 34, 0, 3
-; CHECK: blr
 }
 
 define void @test_s_v4float(<4 x float>* %p, <4 x float> %v) #0 {
+; CHECK-LABEL: test_s_v4float:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    stxvw4x 34, 0, 3
+; CHECK-NEXT:    blr
 entry:
   store <4 x float> %v, <4 x float>* %p, align 4
   ret void
 
-; CHECK-LABEL: @test_s_v4float
-; CHECK: stxvw4x 34, 0, 3
-; CHECK: blr
 }
 
 define void @test_s_v8float(<8 x float>* %p, <8 x float> %v) #0 {
+; CHECK-LABEL: test_s_v8float:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    li 4, 16
+; CHECK-NEXT:    stxvw4x 34, 0, 3
+; CHECK-NEXT:    stxvw4x 35, 3, 4
+; CHECK-NEXT:    blr
 entry:
   store <8 x float> %v, <8 x float>* %p, align 4
   ret void
 
-; CHECK-LABEL: @test_s_v8float
-; CHECK-DAG: li [[REG1:[0-9]+]], 16
-; CHECK-DAG: stxvw4x 35, 3, [[REG1]]
-; CHECK-DAG: stxvw4x 34, 0, 3
-; CHECK: blr
 }
 
 define void @test_s_v2double(<2 x double>* %p, <2 x double> %v) #0 {
+; CHECK-LABEL: test_s_v2double:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    stxvd2x 34, 0, 3
+; CHECK-NEXT:    blr
 entry:
   store <2 x double> %v, <2 x double>* %p, align 8
   ret void
 
-; CHECK-LABEL: @test_s_v2double
-; CHECK: stxvd2x 34, 0, 3
-; CHECK: blr
 }
 
 define void @test_s_v4double(<4 x double>* %p, <4 x double> %v) #0 {
+; CHECK-LABEL: test_s_v4double:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    li 4, 16
+; CHECK-NEXT:    stxvd2x 34, 0, 3
+; CHECK-NEXT:    stxvd2x 35, 3, 4
+; CHECK-NEXT:    blr
 entry:
   store <4 x double> %v, <4 x double>* %p, align 8
   ret void
 
-; CHECK-LABEL: @test_s_v4double
-; CHECK-DAG: li [[REG1:[0-9]+]], 16
-; CHECK-DAG: stxvd2x 35, 3, [[REG1]]
-; CHECK-DAG: stxvd2x 34, 0, 3
-; CHECK: blr
 }
 
 define void @test_s_qv4float(<4 x float>* %p, <4 x float> %v) #1 {
+; CHECK-LABEL: test_s_qv4float:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    qvesplati 0, 1, 3
+; CHECK-NEXT:    stfs 1, 0(3)
+; CHECK-NEXT:    stfs 0, 12(3)
+; CHECK-NEXT:    qvesplati 0, 1, 2
+; CHECK-NEXT:    qvesplati 1, 1, 1
+; CHECK-NEXT:    stfs 0, 8(3)
+; CHECK-NEXT:    stfs 1, 4(3)
+; CHECK-NEXT:    blr
 entry:
   store <4 x float> %v, <4 x float>* %p, align 4
   ret void
 
-; CHECK-LABEL: @test_s_qv4float
-; CHECK-DAG: qvesplati [[REG1:[0-9]+]], 1, 3
-; CHECK-DAG: qvesplati [[REG2:[0-9]+]], 1, 2
-; CHECK-DAG: qvesplati [[REG3:[0-9]+]], 1, 1
-; CHECK-DAG: stfs 1, 0(3)
-; CHECK-DAG: stfs [[REG1]], 12(3)
-; CHECK-DAG: stfs [[REG2]], 8(3)
-; CHECK-DAG: stfs [[REG3]], 4(3)
-; CHECK: blr
 }
 
 define void @test_s_qv8float(<8 x float>* %p, <8 x float> %v) #1 {
+; CHECK-LABEL: test_s_qv8float:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    qvesplati 0, 2, 3
+; CHECK-NEXT:    stfs 2, 16(3)
+; CHECK-NEXT:    stfs 0, 28(3)
+; CHECK-NEXT:    qvesplati 0, 2, 2
+; CHECK-NEXT:    qvesplati 2, 2, 1
+; CHECK-NEXT:    stfs 1, 0(3)
+; CHECK-NEXT:    stfs 0, 24(3)
+; CHECK-NEXT:    qvesplati 0, 1, 3
+; CHECK-NEXT:    stfs 2, 20(3)
+; CHECK-NEXT:    qvesplati 2, 1, 2
+; CHECK-NEXT:    qvesplati 1, 1, 1
+; CHECK-NEXT:    stfs 0, 12(3)
+; CHECK-NEXT:    stfs 2, 8(3)
+; CHECK-NEXT:    stfs 1, 4(3)
+; CHECK-NEXT:    blr
 entry:
   store <8 x float> %v, <8 x float>* %p, align 4
   ret void
 
-; CHECK-LABEL: @test_s_qv8float
-; CHECK-DAG: qvesplati [[REG1:[0-9]+]], 2, 3
-; CHECK-DAG: qvesplati [[REG2:[0-9]+]], 2, 2
-; CHECK-DAG: qvesplati [[REG3:[0-9]+]], 2, 1
-; CHECK-DAG: qvesplati [[REG4:[0-9]+]], 1, 3
-; CHECK-DAG: qvesplati [[REG5:[0-9]+]], 1, 2
-; CHECK-DAG: qvesplati [[REG6:[0-9]+]], 1, 1
-; CHECK-DAG: stfs 2, 16(3)
-; CHECK-DAG: stfs 1, 0(3)
-; CHECK-DAG: stfs [[REG1]], 28(3)
-; CHECK-DAG: stfs [[REG2]], 24(3)
-; CHECK-DAG: stfs [[REG3]], 20(3)
-; CHECK-DAG: stfs [[REG4]], 12(3)
-; CHECK-DAG: stfs [[REG5]], 8(3)
-; CHECK-DAG: stfs [[REG6]], 4(3)
-; CHECK: blr
 }
 
 define void @test_s_qv4double(<4 x double>* %p, <4 x double> %v) #1 {
+; CHECK-LABEL: test_s_qv4double:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    qvesplati 0, 1, 3
+; CHECK-NEXT:    stfd 1, 0(3)
+; CHECK-NEXT:    stfd 0, 24(3)
+; CHECK-NEXT:    qvesplati 0, 1, 2
+; CHECK-NEXT:    qvesplati 1, 1, 1
+; CHECK-NEXT:    stfd 0, 16(3)
+; CHECK-NEXT:    stfd 1, 8(3)
+; CHECK-NEXT:    blr
 entry:
   store <4 x double> %v, <4 x double>* %p, align 8
   ret void
 
-; CHECK-LABEL: @test_s_qv4double
-; CHECK-DAG: qvesplati [[REG1:[0-9]+]], 1, 3
-; CHECK-DAG: qvesplati [[REG2:[0-9]+]], 1, 2
-; CHECK-DAG: qvesplati [[REG3:[0-9]+]], 1, 1
-; CHECK-DAG: stfd 1, 0(3)
-; CHECK-DAG: stfd [[REG1]], 24(3)
-; CHECK-DAG: stfd [[REG2]], 16(3)
-; CHECK-DAG: stfd [[REG3]], 8(3)
-; CHECK: blr
 }
 
 define void @test_s_qv8double(<8 x double>* %p, <8 x double> %v) #1 {
+; CHECK-LABEL: test_s_qv8double:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    qvesplati 0, 2, 3
+; CHECK-NEXT:    stfd 2, 32(3)
+; CHECK-NEXT:    stfd 0, 56(3)
+; CHECK-NEXT:    qvesplati 0, 2, 2
+; CHECK-NEXT:    qvesplati 2, 2, 1
+; CHECK-NEXT:    stfd 1, 0(3)
+; CHECK-NEXT:    stfd 0, 48(3)
+; CHECK-NEXT:    qvesplati 0, 1, 3
+; CHECK-NEXT:    stfd 2, 40(3)
+; CHECK-NEXT:    qvesplati 2, 1, 2
+; CHECK-NEXT:    qvesplati 1, 1, 1
+; CHECK-NEXT:    stfd 0, 24(3)
+; CHECK-NEXT:    stfd 2, 16(3)
+; CHECK-NEXT:    stfd 1, 8(3)
+; CHECK-NEXT:    blr
 entry:
   store <8 x double> %v, <8 x double>* %p, align 8
   ret void
 
-; CHECK-LABEL: @test_s_qv8double
-; CHECK-DAG: qvesplati [[REG1:[0-9]+]], 2, 3
-; CHECK-DAG: qvesplati [[REG2:[0-9]+]], 2, 2
-; CHECK-DAG: qvesplati [[REG3:[0-9]+]], 2, 1
-; CHECK-DAG: qvesplati [[REG4:[0-9]+]], 1, 3
-; CHECK-DAG: qvesplati [[REG5:[0-9]+]], 1, 2
-; CHECK-DAG: qvesplati [[REG6:[0-9]+]], 1, 1
-; CHECK-DAG: stfd 2, 32(3)
-; CHECK-DAG: stfd 1, 0(3)
-; CHECK-DAG: stfd [[REG1]], 56(3)
-; CHECK-DAG: stfd [[REG2]], 48(3)
-; CHECK-DAG: stfd [[REG3]], 40(3)
-; CHECK-DAG: stfd [[REG4]], 24(3)
-; CHECK-DAG: stfd [[REG5]], 16(3)
-; CHECK-DAG: stfd [[REG6]], 8(3)
-; CHECK: blr
 }
 
 attributes #0 = { nounwind "target-cpu"="pwr7" }
diff --git a/llvm/test/CodeGen/PowerPC/unal4-std.ll b/llvm/test/CodeGen/PowerPC/unal4-std.ll
index f27aaf4ed..f843b6b58 100644
--- a/llvm/test/CodeGen/PowerPC/unal4-std.ll
+++ b/llvm/test/CodeGen/PowerPC/unal4-std.ll
@@ -21,4 +21,4 @@ if.end210:                                        ; preds = %entry
 ; CHECK-VSX: stxvw4x {{[0-9]+}}, 0,
 }
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/PowerPC/unaligned-floats.ll b/llvm/test/CodeGen/PowerPC/unaligned-floats.ll
new file mode 100644
index 000000000..af7d36ac1
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/unaligned-floats.ll
@@ -0,0 +1,43 @@
+; RUN: llc -mcpu=pwr7 -ppc-asm-full-reg-names -mtriple=powerpc-- \
+; RUN:   -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mattr=allow-unaligned-fp-access -ppc-asm-full-reg-names \
+; RUN:   -mtriple=powerpc-- -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -ppc-asm-full-reg-names -mtriple=powerpc-- \
+; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefix=UNALIGN
+
+; Test case as provided by author in https://bugs.llvm.org/show_bug.cgi?id=40554
+%struct.anon = type { i32, [5 x i8] }
+
+@s = dso_local local_unnamed_addr global %struct.anon { i32 0, [5 x i8] c"\00B\F6\E9y" }, align 4
+@.str = private unnamed_addr constant [4 x i8] c"%g\0A\00", align 1
+; Function Attrs: nofree nounwind
+define dso_local i32 @main() local_unnamed_addr {
+; CHECK-LABEL: main:
+; CHECK:       lfs f1, 5(r3)
+; CHECK:       blr
+;
+; UNALIGN-LABEL: main:
+; UNALIGN:       lfs f1, 12(r1)
+; UNALIGN:       blr
+entry:
+  %0 = load i8, i8* getelementptr inbounds (%struct.anon, %struct.anon* @s, i32 0, i32 1, i32 1), align 1
+  %conv = zext i8 %0 to i32
+  %shl = shl nuw i32 %conv, 24
+  %1 = load i8, i8* getelementptr inbounds (%struct.anon, %struct.anon* @s, i32 0, i32 1, i32 2), align 2
+  %conv1 = zext i8 %1 to i32
+  %shl2 = shl nuw nsw i32 %conv1, 16
+  %add = or i32 %shl2, %shl
+  %2 = load i8, i8* getelementptr inbounds (%struct.anon, %struct.anon* @s, i32 0, i32 1, i32 3), align 1
+  %conv3 = zext i8 %2 to i32
+  %shl4 = shl nuw nsw i32 %conv3, 8
+  %add5 = or i32 %add, %shl4
+  %3 = load i8, i8* getelementptr inbounds (%struct.anon, %struct.anon* @s, i32 0, i32 1, i32 4), align 4
+  %conv6 = zext i8 %3 to i32
+  %add7 = or i32 %add5, %conv6
+  %4 = bitcast i32 %add7 to float
+  %conv8 = fpext float %4 to double
+  %call = tail call i32 (i8*, ...) @printf(i8* nonnull dereferenceable(1) getelementptr inbounds ([4 x i8], [4 x i8]* @.str, i32 0, i32 0), double %conv8)
+  ret i32 0
+}
+; Function Attrs: nofree nounwind
+declare i32 @printf(i8* nocapture readonly, ...) local_unnamed_addr
diff --git a/llvm/test/CodeGen/PowerPC/urem-lkk.ll b/llvm/test/CodeGen/PowerPC/urem-lkk.ll
new file mode 100644
index 000000000..307af5ec3
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/urem-lkk.ll
@@ -0,0 +1,106 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -verify-machineinstrs -mtriple=powerpc-unknown-linux-gnu -mcpu=ppc64 < %s | FileCheck -check-prefixes=CHECK,CHECK64 %s
+; RUN: llc -verify-machineinstrs -mtriple=powerpc-unknown-linux-gnu -mcpu=ppc < %s | FileCheck -check-prefixes=CHECK,CHECK32 %s
+
+define i32 @fold_urem_positive_odd(i32 %x) {
+; CHECK-LABEL: fold_urem_positive_odd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lis 4, 22765
+; CHECK-NEXT:    ori 4, 4, 8969
+; CHECK-NEXT:    mulhwu 4, 3, 4
+; CHECK-NEXT:    subf 5, 4, 3
+; CHECK-NEXT:    srwi 5, 5, 1
+; CHECK-NEXT:    add 4, 5, 4
+; CHECK-NEXT:    srwi 4, 4, 6
+; CHECK-NEXT:    mulli 4, 4, 95
+; CHECK-NEXT:    subf 3, 4, 3
+; CHECK-NEXT:    blr
+  %1 = urem i32 %x, 95
+  ret i32 %1
+}
+
+
+define i32 @fold_urem_positive_even(i32 %x) {
+; CHECK-LABEL: fold_urem_positive_even:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lis 4, -2226
+; CHECK-NEXT:    ori 4, 4, 16323
+; CHECK-NEXT:    mulhwu 4, 3, 4
+; CHECK-NEXT:    srwi 4, 4, 10
+; CHECK-NEXT:    mulli 4, 4, 1060
+; CHECK-NEXT:    subf 3, 4, 3
+; CHECK-NEXT:    blr
+  %1 = urem i32 %x, 1060
+  ret i32 %1
+}
+
+
+; Don't fold if we can combine urem with udiv.
+define i32 @combine_urem_udiv(i32 %x) {
+; CHECK-LABEL: combine_urem_udiv:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lis 4, 22765
+; CHECK-NEXT:    ori 4, 4, 8969
+; CHECK-NEXT:    mulhwu 4, 3, 4
+; CHECK-NEXT:    subf 5, 4, 3
+; CHECK-NEXT:    srwi 5, 5, 1
+; CHECK-NEXT:    add 4, 5, 4
+; CHECK-NEXT:    srwi 4, 4, 6
+; CHECK-NEXT:    mulli 5, 4, 95
+; CHECK-NEXT:    subf 3, 5, 3
+; CHECK-NEXT:    add 3, 3, 4
+; CHECK-NEXT:    blr
+  %1 = urem i32 %x, 95
+  %2 = udiv i32 %x, 95
+  %3 = add i32 %1, %2
+  ret i32 %3
+}
+
+; Don't fold for divisors that are a power of two.
+define i32 @dont_fold_urem_power_of_two(i32 %x) {
+; CHECK-LABEL: dont_fold_urem_power_of_two:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    clrlwi 3, 3, 26
+; CHECK-NEXT:    blr
+  %1 = urem i32 %x, 64
+  ret i32 %1
+}
+
+; Don't fold if the divisor is one.
+define i32 @dont_fold_urem_one(i32 %x) {
+; CHECK-LABEL: dont_fold_urem_one:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li 3, 0
+; CHECK-NEXT:    blr
+  %1 = urem i32 %x, 1
+  ret i32 %1
+}
+
+; Don't fold if the divisor is 2^32.
+define i32 @dont_fold_urem_i32_umax(i32 %x) {
+; CHECK-LABEL: dont_fold_urem_i32_umax:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    blr
+  %1 = urem i32 %x, 4294967296
+  ret i32 %1
+}
+
+; Don't fold i64 urem
+define i64 @dont_fold_urem_i64(i64 %x) {
+; CHECK-LABEL: dont_fold_urem_i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mflr 0
+; CHECK-NEXT:    stw 0, 4(1)
+; CHECK-NEXT:    stwu 1, -16(1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset lr, 4
+; CHECK-NEXT:    li 5, 0
+; CHECK-NEXT:    li 6, 98
+; CHECK-NEXT:    bl __umoddi3
+; CHECK-NEXT:    lwz 0, 20(1)
+; CHECK-NEXT:    addi 1, 1, 16
+; CHECK-NEXT:    mtlr 0
+; CHECK-NEXT:    blr
+  %1 = urem i64 %x, 98
+  ret i64 %1
+}
diff --git a/llvm/test/CodeGen/PowerPC/urem-vector-lkk.ll b/llvm/test/CodeGen/PowerPC/urem-vector-lkk.ll
new file mode 100644
index 000000000..e3d9027d9
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/urem-vector-lkk.ll
@@ -0,0 +1,1338 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mcpu=pwr9 -verify-machineinstrs -ppc-vsr-nums-as-vr -ppc-asm-full-reg-names \
+; RUN:		-mtriple=powerpc64le-unknown-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,P9LE
+; RUN: llc -mcpu=pwr9 -verify-machineinstrs -ppc-vsr-nums-as-vr -ppc-asm-full-reg-names \
+; RUN:    -mtriple=powerpc64-unknown-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,P9BE
+; RUN: llc -mcpu=pwr8 -verify-machineinstrs -ppc-vsr-nums-as-vr -ppc-asm-full-reg-names \
+; RUN:    -mtriple=powerpc64le-unknown-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,P8LE
+; RUN: llc -mcpu=pwr8 -verify-machineinstrs -ppc-vsr-nums-as-vr -ppc-asm-full-reg-names \
+; RUN:    -mtriple=powerpc64-unknown-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,P8BE
+
+define <4 x i16> @fold_urem_vec_1(<4 x i16> %x) {
+; P9LE-LABEL: fold_urem_vec_1:
+; P9LE:       # %bb.0:
+; P9LE-NEXT:    li r3, 4
+; P9LE-NEXT:    vextuhrx r3, r3, v2
+; P9LE-NEXT:    lis r5, 21399
+; P9LE-NEXT:    ori r5, r5, 33437
+; P9LE-NEXT:    rlwinm r4, r3, 0, 16, 31
+; P9LE-NEXT:    mulld r4, r4, r5
+; P9LE-NEXT:    lis r5, 16727
+; P9LE-NEXT:    ori r5, r5, 2287
+; P9LE-NEXT:    rldicl r4, r4, 27, 37
+; P9LE-NEXT:    mulli r4, r4, 98
+; P9LE-NEXT:    subf r3, r4, r3
+; P9LE-NEXT:    mtvsrd f0, r3
+; P9LE-NEXT:    li r3, 6
+; P9LE-NEXT:    vextuhrx r3, r3, v2
+; P9LE-NEXT:    rlwinm r4, r3, 0, 16, 31
+; P9LE-NEXT:    mulld r4, r4, r5
+; P9LE-NEXT:    lis r5, 8456
+; P9LE-NEXT:    ori r5, r5, 16913
+; P9LE-NEXT:    rldicl r4, r4, 24, 40
+; P9LE-NEXT:    mulli r4, r4, 1003
+; P9LE-NEXT:    subf r3, r4, r3
+; P9LE-NEXT:    xxswapd v3, vs0
+; P9LE-NEXT:    mtvsrd f0, r3
+; P9LE-NEXT:    li r3, 2
+; P9LE-NEXT:    vextuhrx r3, r3, v2
+; P9LE-NEXT:    rlwinm r4, r3, 30, 18, 31
+; P9LE-NEXT:    mulld r4, r4, r5
+; P9LE-NEXT:    rldicl r4, r4, 30, 34
+; P9LE-NEXT:    mulli r4, r4, 124
+; P9LE-NEXT:    subf r3, r4, r3
+; P9LE-NEXT:    xxswapd v4, vs0
+; P9LE-NEXT:    mtvsrd f0, r3
+; P9LE-NEXT:    li r3, 0
+; P9LE-NEXT:    vextuhrx r3, r3, v2
+; P9LE-NEXT:    rlwinm r4, r3, 0, 16, 31
+; P9LE-NEXT:    lis r6, 22765
+; P9LE-NEXT:    ori r6, r6, 8969
+; P9LE-NEXT:    vmrglh v3, v4, v3
+; P9LE-NEXT:    xxswapd v4, vs0
+; P9LE-NEXT:    clrldi r5, r4, 32
+; P9LE-NEXT:    mulld r5, r5, r6
+; P9LE-NEXT:    rldicl r5, r5, 32, 32
+; P9LE-NEXT:    subf r4, r5, r4
+; P9LE-NEXT:    srwi r4, r4, 1
+; P9LE-NEXT:    add r4, r4, r5
+; P9LE-NEXT:    srwi r4, r4, 6
+; P9LE-NEXT:    mulli r4, r4, 95
+; P9LE-NEXT:    subf r3, r4, r3
+; P9LE-NEXT:    mtvsrd f0, r3
+; P9LE-NEXT:    xxswapd v2, vs0
+; P9LE-NEXT:    vmrglh v2, v4, v2
+; P9LE-NEXT:    vmrglw v2, v3, v2
+; P9LE-NEXT:    blr
+;
+; P9BE-LABEL: fold_urem_vec_1:
+; P9BE:       # %bb.0:
+; P9BE-NEXT:    li r3, 6
+; P9BE-NEXT:    vextuhlx r3, r3, v2
+; P9BE-NEXT:    rlwinm r3, r3, 0, 16, 31
+; P9BE-NEXT:    lis r5, 16727
+; P9BE-NEXT:    ori r5, r5, 2287
+; P9BE-NEXT:    clrldi r4, r3, 32
+; P9BE-NEXT:    mulld r4, r4, r5
+; P9BE-NEXT:    lis r5, 21399
+; P9BE-NEXT:    ori r5, r5, 33437
+; P9BE-NEXT:    rldicl r4, r4, 24, 40
+; P9BE-NEXT:    mulli r4, r4, 1003
+; P9BE-NEXT:    subf r3, r4, r3
+; P9BE-NEXT:    sldi r3, r3, 48
+; P9BE-NEXT:    mtvsrd v3, r3
+; P9BE-NEXT:    li r3, 4
+; P9BE-NEXT:    vextuhlx r3, r3, v2
+; P9BE-NEXT:    rlwinm r3, r3, 0, 16, 31
+; P9BE-NEXT:    clrldi r4, r3, 32
+; P9BE-NEXT:    mulld r4, r4, r5
+; P9BE-NEXT:    lis r5, 8456
+; P9BE-NEXT:    ori r5, r5, 16913
+; P9BE-NEXT:    rldicl r4, r4, 27, 37
+; P9BE-NEXT:    mulli r4, r4, 98
+; P9BE-NEXT:    subf r3, r4, r3
+; P9BE-NEXT:    sldi r3, r3, 48
+; P9BE-NEXT:    mtvsrd v4, r3
+; P9BE-NEXT:    li r3, 2
+; P9BE-NEXT:    vextuhlx r3, r3, v2
+; P9BE-NEXT:    clrlwi r4, r3, 16
+; P9BE-NEXT:    rlwinm r3, r3, 30, 18, 31
+; P9BE-NEXT:    mulld r3, r3, r5
+; P9BE-NEXT:    lis r5, 22765
+; P9BE-NEXT:    ori r5, r5, 8969
+; P9BE-NEXT:    rldicl r3, r3, 30, 34
+; P9BE-NEXT:    mulli r3, r3, 124
+; P9BE-NEXT:    subf r3, r3, r4
+; P9BE-NEXT:    sldi r3, r3, 48
+; P9BE-NEXT:    vmrghh v3, v4, v3
+; P9BE-NEXT:    mtvsrd v4, r3
+; P9BE-NEXT:    li r3, 0
+; P9BE-NEXT:    vextuhlx r3, r3, v2
+; P9BE-NEXT:    rlwinm r3, r3, 0, 16, 31
+; P9BE-NEXT:    clrldi r4, r3, 32
+; P9BE-NEXT:    mulld r4, r4, r5
+; P9BE-NEXT:    rldicl r4, r4, 32, 32
+; P9BE-NEXT:    subf r5, r4, r3
+; P9BE-NEXT:    srwi r5, r5, 1
+; P9BE-NEXT:    add r4, r5, r4
+; P9BE-NEXT:    srwi r4, r4, 6
+; P9BE-NEXT:    mulli r4, r4, 95
+; P9BE-NEXT:    subf r3, r4, r3
+; P9BE-NEXT:    sldi r3, r3, 48
+; P9BE-NEXT:    mtvsrd v2, r3
+; P9BE-NEXT:    vmrghh v2, v2, v4
+; P9BE-NEXT:    vmrghw v2, v2, v3
+; P9BE-NEXT:    blr
+;
+; P8LE-LABEL: fold_urem_vec_1:
+; P8LE:       # %bb.0:
+; P8LE-NEXT:    xxswapd vs0, v2
+; P8LE-NEXT:    lis r3, 22765
+; P8LE-NEXT:    lis r8, 21399
+; P8LE-NEXT:    ori r3, r3, 8969
+; P8LE-NEXT:    ori r8, r8, 33437
+; P8LE-NEXT:    mfvsrd r4, f0
+; P8LE-NEXT:    clrldi r5, r4, 48
+; P8LE-NEXT:    rldicl r9, r4, 32, 48
+; P8LE-NEXT:    rlwinm r6, r5, 0, 16, 31
+; P8LE-NEXT:    rldicl r10, r4, 16, 48
+; P8LE-NEXT:    rlwinm r11, r9, 0, 16, 31
+; P8LE-NEXT:    clrldi r7, r6, 32
+; P8LE-NEXT:    rlwinm r12, r10, 0, 16, 31
+; P8LE-NEXT:    mulld r3, r7, r3
+; P8LE-NEXT:    lis r7, 16727
+; P8LE-NEXT:    ori r7, r7, 2287
+; P8LE-NEXT:    mulld r8, r11, r8
+; P8LE-NEXT:    lis r11, 8456
+; P8LE-NEXT:    rldicl r4, r4, 48, 48
+; P8LE-NEXT:    mulld r7, r12, r7
+; P8LE-NEXT:    ori r11, r11, 16913
+; P8LE-NEXT:    rlwinm r12, r4, 30, 18, 31
+; P8LE-NEXT:    rldicl r3, r3, 32, 32
+; P8LE-NEXT:    mulld r11, r12, r11
+; P8LE-NEXT:    subf r6, r3, r6
+; P8LE-NEXT:    rldicl r8, r8, 27, 37
+; P8LE-NEXT:    srwi r6, r6, 1
+; P8LE-NEXT:    add r3, r6, r3
+; P8LE-NEXT:    rldicl r6, r7, 24, 40
+; P8LE-NEXT:    mulli r7, r8, 98
+; P8LE-NEXT:    srwi r3, r3, 6
+; P8LE-NEXT:    rldicl r8, r11, 30, 34
+; P8LE-NEXT:    mulli r6, r6, 1003
+; P8LE-NEXT:    mulli r3, r3, 95
+; P8LE-NEXT:    mulli r8, r8, 124
+; P8LE-NEXT:    subf r7, r7, r9
+; P8LE-NEXT:    subf r6, r6, r10
+; P8LE-NEXT:    mtvsrd f0, r7
+; P8LE-NEXT:    subf r3, r3, r5
+; P8LE-NEXT:    subf r4, r8, r4
+; P8LE-NEXT:    mtvsrd f1, r6
+; P8LE-NEXT:    mtvsrd f2, r3
+; P8LE-NEXT:    xxswapd v2, vs0
+; P8LE-NEXT:    mtvsrd f3, r4
+; P8LE-NEXT:    xxswapd v3, vs1
+; P8LE-NEXT:    xxswapd v4, vs2
+; P8LE-NEXT:    xxswapd v5, vs3
+; P8LE-NEXT:    vmrglh v2, v3, v2
+; P8LE-NEXT:    vmrglh v3, v5, v4
+; P8LE-NEXT:    vmrglw v2, v2, v3
+; P8LE-NEXT:    blr
+;
+; P8BE-LABEL: fold_urem_vec_1:
+; P8BE:       # %bb.0:
+; P8BE-NEXT:    mfvsrd r4, v2
+; P8BE-NEXT:    lis r3, 22765
+; P8BE-NEXT:    lis r9, 16727
+; P8BE-NEXT:    ori r3, r3, 8969
+; P8BE-NEXT:    ori r9, r9, 2287
+; P8BE-NEXT:    rldicl r5, r4, 16, 48
+; P8BE-NEXT:    clrldi r6, r4, 48
+; P8BE-NEXT:    rlwinm r5, r5, 0, 16, 31
+; P8BE-NEXT:    rldicl r7, r4, 48, 48
+; P8BE-NEXT:    rlwinm r6, r6, 0, 16, 31
+; P8BE-NEXT:    clrldi r8, r5, 32
+; P8BE-NEXT:    rlwinm r7, r7, 0, 16, 31
+; P8BE-NEXT:    mulld r3, r8, r3
+; P8BE-NEXT:    lis r8, 21399
+; P8BE-NEXT:    clrldi r10, r6, 32
+; P8BE-NEXT:    ori r8, r8, 33437
+; P8BE-NEXT:    clrldi r11, r7, 32
+; P8BE-NEXT:    mulld r9, r10, r9
+; P8BE-NEXT:    lis r10, 8456
+; P8BE-NEXT:    rldicl r4, r4, 32, 48
+; P8BE-NEXT:    mulld r8, r11, r8
+; P8BE-NEXT:    ori r10, r10, 16913
+; P8BE-NEXT:    rlwinm r11, r4, 30, 18, 31
+; P8BE-NEXT:    rldicl r3, r3, 32, 32
+; P8BE-NEXT:    rlwinm r4, r4, 0, 16, 31
+; P8BE-NEXT:    mulld r10, r11, r10
+; P8BE-NEXT:    subf r11, r3, r5
+; P8BE-NEXT:    srwi r11, r11, 1
+; P8BE-NEXT:    rldicl r9, r9, 24, 40
+; P8BE-NEXT:    add r3, r11, r3
+; P8BE-NEXT:    rldicl r8, r8, 27, 37
+; P8BE-NEXT:    srwi r3, r3, 6
+; P8BE-NEXT:    mulli r9, r9, 1003
+; P8BE-NEXT:    rldicl r10, r10, 30, 34
+; P8BE-NEXT:    mulli r8, r8, 98
+; P8BE-NEXT:    mulli r3, r3, 95
+; P8BE-NEXT:    mulli r10, r10, 124
+; P8BE-NEXT:    subf r6, r9, r6
+; P8BE-NEXT:    subf r7, r8, r7
+; P8BE-NEXT:    sldi r6, r6, 48
+; P8BE-NEXT:    subf r3, r3, r5
+; P8BE-NEXT:    subf r4, r10, r4
+; P8BE-NEXT:    mtvsrd v2, r6
+; P8BE-NEXT:    sldi r5, r7, 48
+; P8BE-NEXT:    sldi r3, r3, 48
+; P8BE-NEXT:    sldi r4, r4, 48
+; P8BE-NEXT:    mtvsrd v3, r5
+; P8BE-NEXT:    mtvsrd v4, r3
+; P8BE-NEXT:    mtvsrd v5, r4
+; P8BE-NEXT:    vmrghh v2, v3, v2
+; P8BE-NEXT:    vmrghh v3, v4, v5
+; P8BE-NEXT:    vmrghw v2, v3, v2
+; P8BE-NEXT:    blr
+  %1 = urem <4 x i16> %x, <i16 95, i16 124, i16 98, i16 1003>
+  ret <4 x i16> %1
+}
+
+define <4 x i16> @fold_urem_vec_2(<4 x i16> %x) {
+; P9LE-LABEL: fold_urem_vec_2:
+; P9LE:       # %bb.0:
+; P9LE-NEXT:    li r3, 0
+; P9LE-NEXT:    vextuhrx r3, r3, v2
+; P9LE-NEXT:    rlwinm r4, r3, 0, 16, 31
+; P9LE-NEXT:    lis r6, 22765
+; P9LE-NEXT:    ori r6, r6, 8969
+; P9LE-NEXT:    clrldi r5, r4, 32
+; P9LE-NEXT:    mulld r5, r5, r6
+; P9LE-NEXT:    rldicl r5, r5, 32, 32
+; P9LE-NEXT:    subf r4, r5, r4
+; P9LE-NEXT:    srwi r4, r4, 1
+; P9LE-NEXT:    add r4, r4, r5
+; P9LE-NEXT:    srwi r4, r4, 6
+; P9LE-NEXT:    mulli r4, r4, 95
+; P9LE-NEXT:    subf r3, r4, r3
+; P9LE-NEXT:    mtvsrd f0, r3
+; P9LE-NEXT:    li r3, 2
+; P9LE-NEXT:    vextuhrx r3, r3, v2
+; P9LE-NEXT:    rlwinm r4, r3, 0, 16, 31
+; P9LE-NEXT:    clrldi r5, r4, 32
+; P9LE-NEXT:    mulld r5, r5, r6
+; P9LE-NEXT:    rldicl r5, r5, 32, 32
+; P9LE-NEXT:    subf r4, r5, r4
+; P9LE-NEXT:    srwi r4, r4, 1
+; P9LE-NEXT:    add r4, r4, r5
+; P9LE-NEXT:    srwi r4, r4, 6
+; P9LE-NEXT:    mulli r4, r4, 95
+; P9LE-NEXT:    subf r3, r4, r3
+; P9LE-NEXT:    xxswapd v3, vs0
+; P9LE-NEXT:    mtvsrd f0, r3
+; P9LE-NEXT:    li r3, 4
+; P9LE-NEXT:    vextuhrx r3, r3, v2
+; P9LE-NEXT:    rlwinm r4, r3, 0, 16, 31
+; P9LE-NEXT:    clrldi r5, r4, 32
+; P9LE-NEXT:    mulld r5, r5, r6
+; P9LE-NEXT:    rldicl r5, r5, 32, 32
+; P9LE-NEXT:    subf r4, r5, r4
+; P9LE-NEXT:    srwi r4, r4, 1
+; P9LE-NEXT:    add r4, r4, r5
+; P9LE-NEXT:    srwi r4, r4, 6
+; P9LE-NEXT:    mulli r4, r4, 95
+; P9LE-NEXT:    subf r3, r4, r3
+; P9LE-NEXT:    xxswapd v4, vs0
+; P9LE-NEXT:    mtvsrd f0, r3
+; P9LE-NEXT:    li r3, 6
+; P9LE-NEXT:    vextuhrx r3, r3, v2
+; P9LE-NEXT:    rlwinm r4, r3, 0, 16, 31
+; P9LE-NEXT:    clrldi r5, r4, 32
+; P9LE-NEXT:    mulld r5, r5, r6
+; P9LE-NEXT:    rldicl r5, r5, 32, 32
+; P9LE-NEXT:    subf r4, r5, r4
+; P9LE-NEXT:    srwi r4, r4, 1
+; P9LE-NEXT:    add r4, r4, r5
+; P9LE-NEXT:    srwi r4, r4, 6
+; P9LE-NEXT:    mulli r4, r4, 95
+; P9LE-NEXT:    subf r3, r4, r3
+; P9LE-NEXT:    vmrglh v3, v4, v3
+; P9LE-NEXT:    xxswapd v4, vs0
+; P9LE-NEXT:    mtvsrd f0, r3
+; P9LE-NEXT:    xxswapd v2, vs0
+; P9LE-NEXT:    vmrglh v2, v2, v4
+; P9LE-NEXT:    vmrglw v2, v2, v3
+; P9LE-NEXT:    blr
+;
+; P9BE-LABEL: fold_urem_vec_2:
+; P9BE:       # %bb.0:
+; P9BE-NEXT:    li r3, 6
+; P9BE-NEXT:    vextuhlx r3, r3, v2
+; P9BE-NEXT:    rlwinm r3, r3, 0, 16, 31
+; P9BE-NEXT:    lis r5, 22765
+; P9BE-NEXT:    ori r5, r5, 8969
+; P9BE-NEXT:    clrldi r4, r3, 32
+; P9BE-NEXT:    mulld r4, r4, r5
+; P9BE-NEXT:    rldicl r4, r4, 32, 32
+; P9BE-NEXT:    subf r6, r4, r3
+; P9BE-NEXT:    srwi r6, r6, 1
+; P9BE-NEXT:    add r4, r6, r4
+; P9BE-NEXT:    srwi r4, r4, 6
+; P9BE-NEXT:    mulli r4, r4, 95
+; P9BE-NEXT:    subf r3, r4, r3
+; P9BE-NEXT:    sldi r3, r3, 48
+; P9BE-NEXT:    mtvsrd v3, r3
+; P9BE-NEXT:    li r3, 4
+; P9BE-NEXT:    vextuhlx r3, r3, v2
+; P9BE-NEXT:    rlwinm r3, r3, 0, 16, 31
+; P9BE-NEXT:    clrldi r4, r3, 32
+; P9BE-NEXT:    mulld r4, r4, r5
+; P9BE-NEXT:    rldicl r4, r4, 32, 32
+; P9BE-NEXT:    subf r6, r4, r3
+; P9BE-NEXT:    srwi r6, r6, 1
+; P9BE-NEXT:    add r4, r6, r4
+; P9BE-NEXT:    srwi r4, r4, 6
+; P9BE-NEXT:    mulli r4, r4, 95
+; P9BE-NEXT:    subf r3, r4, r3
+; P9BE-NEXT:    sldi r3, r3, 48
+; P9BE-NEXT:    mtvsrd v4, r3
+; P9BE-NEXT:    li r3, 2
+; P9BE-NEXT:    vextuhlx r3, r3, v2
+; P9BE-NEXT:    rlwinm r3, r3, 0, 16, 31
+; P9BE-NEXT:    clrldi r4, r3, 32
+; P9BE-NEXT:    mulld r4, r4, r5
+; P9BE-NEXT:    rldicl r4, r4, 32, 32
+; P9BE-NEXT:    subf r6, r4, r3
+; P9BE-NEXT:    srwi r6, r6, 1
+; P9BE-NEXT:    add r4, r6, r4
+; P9BE-NEXT:    srwi r4, r4, 6
+; P9BE-NEXT:    mulli r4, r4, 95
+; P9BE-NEXT:    subf r3, r4, r3
+; P9BE-NEXT:    sldi r3, r3, 48
+; P9BE-NEXT:    vmrghh v3, v4, v3
+; P9BE-NEXT:    mtvsrd v4, r3
+; P9BE-NEXT:    li r3, 0
+; P9BE-NEXT:    vextuhlx r3, r3, v2
+; P9BE-NEXT:    rlwinm r3, r3, 0, 16, 31
+; P9BE-NEXT:    clrldi r4, r3, 32
+; P9BE-NEXT:    mulld r4, r4, r5
+; P9BE-NEXT:    rldicl r4, r4, 32, 32
+; P9BE-NEXT:    subf r5, r4, r3
+; P9BE-NEXT:    srwi r5, r5, 1
+; P9BE-NEXT:    add r4, r5, r4
+; P9BE-NEXT:    srwi r4, r4, 6
+; P9BE-NEXT:    mulli r4, r4, 95
+; P9BE-NEXT:    subf r3, r4, r3
+; P9BE-NEXT:    sldi r3, r3, 48
+; P9BE-NEXT:    mtvsrd v2, r3
+; P9BE-NEXT:    vmrghh v2, v2, v4
+; P9BE-NEXT:    vmrghw v2, v2, v3
+; P9BE-NEXT:    blr
+;
+; P8LE-LABEL: fold_urem_vec_2:
+; P8LE:       # %bb.0:
+; P8LE-NEXT:    xxswapd vs0, v2
+; P8LE-NEXT:    lis r4, 22765
+; P8LE-NEXT:    std r29, -24(r1) # 8-byte Folded Spill
+; P8LE-NEXT:    std r30, -16(r1) # 8-byte Folded Spill
+; P8LE-NEXT:    ori r4, r4, 8969
+; P8LE-NEXT:    mfvsrd r5, f0
+; P8LE-NEXT:    clrldi r3, r5, 48
+; P8LE-NEXT:    rldicl r6, r5, 48, 48
+; P8LE-NEXT:    rlwinm r8, r3, 0, 16, 31
+; P8LE-NEXT:    rldicl r7, r5, 32, 48
+; P8LE-NEXT:    rlwinm r9, r6, 0, 16, 31
+; P8LE-NEXT:    rldicl r5, r5, 16, 48
+; P8LE-NEXT:    clrldi r11, r8, 32
+; P8LE-NEXT:    rlwinm r10, r7, 0, 16, 31
+; P8LE-NEXT:    rlwinm r12, r5, 0, 16, 31
+; P8LE-NEXT:    mulld r11, r11, r4
+; P8LE-NEXT:    clrldi r0, r9, 32
+; P8LE-NEXT:    clrldi r30, r10, 32
+; P8LE-NEXT:    clrldi r29, r12, 32
+; P8LE-NEXT:    mulld r0, r0, r4
+; P8LE-NEXT:    mulld r30, r30, r4
+; P8LE-NEXT:    mulld r4, r29, r4
+; P8LE-NEXT:    ld r29, -24(r1) # 8-byte Folded Reload
+; P8LE-NEXT:    rldicl r11, r11, 32, 32
+; P8LE-NEXT:    subf r8, r11, r8
+; P8LE-NEXT:    rldicl r0, r0, 32, 32
+; P8LE-NEXT:    srwi r8, r8, 1
+; P8LE-NEXT:    rldicl r30, r30, 32, 32
+; P8LE-NEXT:    rldicl r4, r4, 32, 32
+; P8LE-NEXT:    subf r9, r0, r9
+; P8LE-NEXT:    add r8, r8, r11
+; P8LE-NEXT:    subf r10, r30, r10
+; P8LE-NEXT:    subf r11, r4, r12
+; P8LE-NEXT:    srwi r9, r9, 1
+; P8LE-NEXT:    srwi r8, r8, 6
+; P8LE-NEXT:    srwi r10, r10, 1
+; P8LE-NEXT:    srwi r11, r11, 1
+; P8LE-NEXT:    add r9, r9, r0
+; P8LE-NEXT:    add r10, r10, r30
+; P8LE-NEXT:    add r4, r11, r4
+; P8LE-NEXT:    srwi r9, r9, 6
+; P8LE-NEXT:    ld r30, -16(r1) # 8-byte Folded Reload
+; P8LE-NEXT:    mulli r8, r8, 95
+; P8LE-NEXT:    srwi r10, r10, 6
+; P8LE-NEXT:    srwi r4, r4, 6
+; P8LE-NEXT:    mulli r9, r9, 95
+; P8LE-NEXT:    mulli r10, r10, 95
+; P8LE-NEXT:    mulli r4, r4, 95
+; P8LE-NEXT:    subf r3, r8, r3
+; P8LE-NEXT:    subf r6, r9, r6
+; P8LE-NEXT:    mtvsrd f0, r3
+; P8LE-NEXT:    subf r3, r10, r7
+; P8LE-NEXT:    subf r4, r4, r5
+; P8LE-NEXT:    mtvsrd f1, r6
+; P8LE-NEXT:    mtvsrd f2, r3
+; P8LE-NEXT:    xxswapd v2, vs0
+; P8LE-NEXT:    mtvsrd f3, r4
+; P8LE-NEXT:    xxswapd v3, vs1
+; P8LE-NEXT:    xxswapd v4, vs2
+; P8LE-NEXT:    xxswapd v5, vs3
+; P8LE-NEXT:    vmrglh v2, v3, v2
+; P8LE-NEXT:    vmrglh v3, v5, v4
+; P8LE-NEXT:    vmrglw v2, v3, v2
+; P8LE-NEXT:    blr
+;
+; P8BE-LABEL: fold_urem_vec_2:
+; P8BE:       # %bb.0:
+; P8BE-NEXT:    mfvsrd r4, v2
+; P8BE-NEXT:    lis r3, 22765
+; P8BE-NEXT:    ori r3, r3, 8969
+; P8BE-NEXT:    clrldi r5, r4, 48
+; P8BE-NEXT:    rldicl r6, r4, 48, 48
+; P8BE-NEXT:    rlwinm r5, r5, 0, 16, 31
+; P8BE-NEXT:    rldicl r7, r4, 32, 48
+; P8BE-NEXT:    rlwinm r6, r6, 0, 16, 31
+; P8BE-NEXT:    clrldi r8, r5, 32
+; P8BE-NEXT:    rldicl r4, r4, 16, 48
+; P8BE-NEXT:    rlwinm r7, r7, 0, 16, 31
+; P8BE-NEXT:    clrldi r9, r6, 32
+; P8BE-NEXT:    mulld r8, r8, r3
+; P8BE-NEXT:    rlwinm r4, r4, 0, 16, 31
+; P8BE-NEXT:    clrldi r10, r7, 32
+; P8BE-NEXT:    mulld r9, r9, r3
+; P8BE-NEXT:    clrldi r11, r4, 32
+; P8BE-NEXT:    mulld r10, r10, r3
+; P8BE-NEXT:    mulld r3, r11, r3
+; P8BE-NEXT:    rldicl r8, r8, 32, 32
+; P8BE-NEXT:    rldicl r9, r9, 32, 32
+; P8BE-NEXT:    subf r11, r8, r5
+; P8BE-NEXT:    rldicl r10, r10, 32, 32
+; P8BE-NEXT:    subf r12, r9, r6
+; P8BE-NEXT:    srwi r11, r11, 1
+; P8BE-NEXT:    rldicl r3, r3, 32, 32
+; P8BE-NEXT:    add r8, r11, r8
+; P8BE-NEXT:    subf r11, r10, r7
+; P8BE-NEXT:    srwi r12, r12, 1
+; P8BE-NEXT:    add r9, r12, r9
+; P8BE-NEXT:    subf r12, r3, r4
+; P8BE-NEXT:    srwi r11, r11, 1
+; P8BE-NEXT:    srwi r8, r8, 6
+; P8BE-NEXT:    add r10, r11, r10
+; P8BE-NEXT:    srwi r11, r12, 1
+; P8BE-NEXT:    srwi r9, r9, 6
+; P8BE-NEXT:    add r3, r11, r3
+; P8BE-NEXT:    srwi r10, r10, 6
+; P8BE-NEXT:    srwi r3, r3, 6
+; P8BE-NEXT:    mulli r8, r8, 95
+; P8BE-NEXT:    mulli r9, r9, 95
+; P8BE-NEXT:    mulli r10, r10, 95
+; P8BE-NEXT:    mulli r3, r3, 95
+; P8BE-NEXT:    subf r5, r8, r5
+; P8BE-NEXT:    subf r6, r9, r6
+; P8BE-NEXT:    subf r7, r10, r7
+; P8BE-NEXT:    subf r3, r3, r4
+; P8BE-NEXT:    sldi r5, r5, 48
+; P8BE-NEXT:    sldi r6, r6, 48
+; P8BE-NEXT:    sldi r4, r7, 48
+; P8BE-NEXT:    mtvsrd v2, r5
+; P8BE-NEXT:    sldi r3, r3, 48
+; P8BE-NEXT:    mtvsrd v3, r6
+; P8BE-NEXT:    mtvsrd v4, r4
+; P8BE-NEXT:    mtvsrd v5, r3
+; P8BE-NEXT:    vmrghh v2, v3, v2
+; P8BE-NEXT:    vmrghh v3, v5, v4
+; P8BE-NEXT:    vmrghw v2, v3, v2
+; P8BE-NEXT:    blr
+  %1 = urem <4 x i16> %x, <i16 95, i16 95, i16 95, i16 95>
+  ret <4 x i16> %1
+}
+
+
+; Don't fold if we can combine urem with udiv.
+define <4 x i16> @combine_urem_udiv(<4 x i16> %x) {
+; P9LE-LABEL: combine_urem_udiv:
+; P9LE:       # %bb.0:
+; P9LE-NEXT:    li r3, 0
+; P9LE-NEXT:    vextuhrx r3, r3, v2
+; P9LE-NEXT:    rlwinm r4, r3, 0, 16, 31
+; P9LE-NEXT:    lis r6, 22765
+; P9LE-NEXT:    ori r6, r6, 8969
+; P9LE-NEXT:    clrldi r5, r4, 32
+; P9LE-NEXT:    mulld r5, r5, r6
+; P9LE-NEXT:    rldicl r5, r5, 32, 32
+; P9LE-NEXT:    subf r4, r5, r4
+; P9LE-NEXT:    srwi r4, r4, 1
+; P9LE-NEXT:    add r4, r4, r5
+; P9LE-NEXT:    srwi r4, r4, 6
+; P9LE-NEXT:    mulli r5, r4, 95
+; P9LE-NEXT:    subf r3, r5, r3
+; P9LE-NEXT:    mtvsrd f0, r3
+; P9LE-NEXT:    li r3, 2
+; P9LE-NEXT:    vextuhrx r3, r3, v2
+; P9LE-NEXT:    rlwinm r5, r3, 0, 16, 31
+; P9LE-NEXT:    clrldi r7, r5, 32
+; P9LE-NEXT:    mulld r7, r7, r6
+; P9LE-NEXT:    rldicl r7, r7, 32, 32
+; P9LE-NEXT:    subf r5, r7, r5
+; P9LE-NEXT:    srwi r5, r5, 1
+; P9LE-NEXT:    add r5, r5, r7
+; P9LE-NEXT:    srwi r5, r5, 6
+; P9LE-NEXT:    mulli r7, r5, 95
+; P9LE-NEXT:    subf r3, r7, r3
+; P9LE-NEXT:    xxswapd v3, vs0
+; P9LE-NEXT:    mtvsrd f0, r3
+; P9LE-NEXT:    li r3, 4
+; P9LE-NEXT:    vextuhrx r3, r3, v2
+; P9LE-NEXT:    rlwinm r7, r3, 0, 16, 31
+; P9LE-NEXT:    clrldi r8, r7, 32
+; P9LE-NEXT:    mulld r8, r8, r6
+; P9LE-NEXT:    rldicl r8, r8, 32, 32
+; P9LE-NEXT:    subf r7, r8, r7
+; P9LE-NEXT:    srwi r7, r7, 1
+; P9LE-NEXT:    add r7, r7, r8
+; P9LE-NEXT:    srwi r7, r7, 6
+; P9LE-NEXT:    mulli r8, r7, 95
+; P9LE-NEXT:    subf r3, r8, r3
+; P9LE-NEXT:    xxswapd v4, vs0
+; P9LE-NEXT:    mtvsrd f0, r3
+; P9LE-NEXT:    li r3, 6
+; P9LE-NEXT:    vextuhrx r3, r3, v2
+; P9LE-NEXT:    rlwinm r8, r3, 0, 16, 31
+; P9LE-NEXT:    clrldi r9, r8, 32
+; P9LE-NEXT:    mulld r6, r9, r6
+; P9LE-NEXT:    rldicl r6, r6, 32, 32
+; P9LE-NEXT:    subf r8, r6, r8
+; P9LE-NEXT:    srwi r8, r8, 1
+; P9LE-NEXT:    add r6, r8, r6
+; P9LE-NEXT:    srwi r6, r6, 6
+; P9LE-NEXT:    mulli r8, r6, 95
+; P9LE-NEXT:    subf r3, r8, r3
+; P9LE-NEXT:    vmrglh v3, v4, v3
+; P9LE-NEXT:    xxswapd v4, vs0
+; P9LE-NEXT:    mtvsrd f0, r3
+; P9LE-NEXT:    xxswapd v2, vs0
+; P9LE-NEXT:    mtvsrd f0, r4
+; P9LE-NEXT:    vmrglh v2, v2, v4
+; P9LE-NEXT:    vmrglw v2, v2, v3
+; P9LE-NEXT:    xxswapd v3, vs0
+; P9LE-NEXT:    mtvsrd f0, r5
+; P9LE-NEXT:    xxswapd v4, vs0
+; P9LE-NEXT:    mtvsrd f0, r7
+; P9LE-NEXT:    vmrglh v3, v4, v3
+; P9LE-NEXT:    xxswapd v4, vs0
+; P9LE-NEXT:    mtvsrd f0, r6
+; P9LE-NEXT:    xxswapd v5, vs0
+; P9LE-NEXT:    vmrglh v4, v5, v4
+; P9LE-NEXT:    vmrglw v3, v4, v3
+; P9LE-NEXT:    vadduhm v2, v2, v3
+; P9LE-NEXT:    blr
+;
+; P9BE-LABEL: combine_urem_udiv:
+; P9BE:       # %bb.0:
+; P9BE-NEXT:    li r3, 6
+; P9BE-NEXT:    vextuhlx r3, r3, v2
+; P9BE-NEXT:    rlwinm r4, r3, 0, 16, 31
+; P9BE-NEXT:    lis r6, 22765
+; P9BE-NEXT:    ori r6, r6, 8969
+; P9BE-NEXT:    clrldi r5, r4, 32
+; P9BE-NEXT:    mulld r5, r5, r6
+; P9BE-NEXT:    rldicl r5, r5, 32, 32
+; P9BE-NEXT:    subf r4, r5, r4
+; P9BE-NEXT:    srwi r4, r4, 1
+; P9BE-NEXT:    add r4, r4, r5
+; P9BE-NEXT:    srwi r4, r4, 6
+; P9BE-NEXT:    mulli r5, r4, 95
+; P9BE-NEXT:    subf r3, r5, r3
+; P9BE-NEXT:    sldi r3, r3, 48
+; P9BE-NEXT:    mtvsrd v3, r3
+; P9BE-NEXT:    li r3, 4
+; P9BE-NEXT:    vextuhlx r3, r3, v2
+; P9BE-NEXT:    rlwinm r5, r3, 0, 16, 31
+; P9BE-NEXT:    clrldi r7, r5, 32
+; P9BE-NEXT:    mulld r7, r7, r6
+; P9BE-NEXT:    rldicl r7, r7, 32, 32
+; P9BE-NEXT:    subf r5, r7, r5
+; P9BE-NEXT:    srwi r5, r5, 1
+; P9BE-NEXT:    add r5, r5, r7
+; P9BE-NEXT:    srwi r5, r5, 6
+; P9BE-NEXT:    mulli r7, r5, 95
+; P9BE-NEXT:    subf r3, r7, r3
+; P9BE-NEXT:    sldi r3, r3, 48
+; P9BE-NEXT:    mtvsrd v4, r3
+; P9BE-NEXT:    li r3, 2
+; P9BE-NEXT:    vextuhlx r3, r3, v2
+; P9BE-NEXT:    rlwinm r7, r3, 0, 16, 31
+; P9BE-NEXT:    clrldi r8, r7, 32
+; P9BE-NEXT:    mulld r8, r8, r6
+; P9BE-NEXT:    rldicl r8, r8, 32, 32
+; P9BE-NEXT:    subf r7, r8, r7
+; P9BE-NEXT:    srwi r7, r7, 1
+; P9BE-NEXT:    add r7, r7, r8
+; P9BE-NEXT:    srwi r7, r7, 6
+; P9BE-NEXT:    mulli r8, r7, 95
+; P9BE-NEXT:    subf r3, r8, r3
+; P9BE-NEXT:    sldi r3, r3, 48
+; P9BE-NEXT:    vmrghh v3, v4, v3
+; P9BE-NEXT:    mtvsrd v4, r3
+; P9BE-NEXT:    li r3, 0
+; P9BE-NEXT:    vextuhlx r3, r3, v2
+; P9BE-NEXT:    rlwinm r3, r3, 0, 16, 31
+; P9BE-NEXT:    clrldi r8, r3, 32
+; P9BE-NEXT:    mulld r6, r8, r6
+; P9BE-NEXT:    rldicl r6, r6, 32, 32
+; P9BE-NEXT:    subf r8, r6, r3
+; P9BE-NEXT:    srwi r8, r8, 1
+; P9BE-NEXT:    add r6, r8, r6
+; P9BE-NEXT:    srwi r6, r6, 6
+; P9BE-NEXT:    mulli r8, r6, 95
+; P9BE-NEXT:    subf r3, r8, r3
+; P9BE-NEXT:    sldi r3, r3, 48
+; P9BE-NEXT:    mtvsrd v2, r3
+; P9BE-NEXT:    sldi r3, r4, 48
+; P9BE-NEXT:    vmrghh v2, v2, v4
+; P9BE-NEXT:    vmrghw v2, v2, v3
+; P9BE-NEXT:    mtvsrd v3, r3
+; P9BE-NEXT:    sldi r3, r5, 48
+; P9BE-NEXT:    mtvsrd v4, r3
+; P9BE-NEXT:    sldi r3, r7, 48
+; P9BE-NEXT:    vmrghh v3, v4, v3
+; P9BE-NEXT:    mtvsrd v4, r3
+; P9BE-NEXT:    sldi r3, r6, 48
+; P9BE-NEXT:    mtvsrd v5, r3
+; P9BE-NEXT:    vmrghh v4, v5, v4
+; P9BE-NEXT:    vmrghw v3, v4, v3
+; P9BE-NEXT:    vadduhm v2, v2, v3
+; P9BE-NEXT:    blr
+;
+; P8LE-LABEL: combine_urem_udiv:
+; P8LE:       # %bb.0:
+; P8LE-NEXT:    xxswapd vs0, v2
+; P8LE-NEXT:    lis r5, 22765
+; P8LE-NEXT:    std r30, -16(r1) # 8-byte Folded Spill
+; P8LE-NEXT:    std r29, -24(r1) # 8-byte Folded Spill
+; P8LE-NEXT:    ori r5, r5, 8969
+; P8LE-NEXT:    mfvsrd r6, f0
+; P8LE-NEXT:    clrldi r3, r6, 48
+; P8LE-NEXT:    rldicl r4, r6, 48, 48
+; P8LE-NEXT:    rldicl r7, r6, 32, 48
+; P8LE-NEXT:    rlwinm r8, r3, 0, 16, 31
+; P8LE-NEXT:    rlwinm r9, r4, 0, 16, 31
+; P8LE-NEXT:    rldicl r6, r6, 16, 48
+; P8LE-NEXT:    rlwinm r10, r7, 0, 16, 31
+; P8LE-NEXT:    clrldi r11, r8, 32
+; P8LE-NEXT:    rlwinm r12, r6, 0, 16, 31
+; P8LE-NEXT:    clrldi r0, r9, 32
+; P8LE-NEXT:    clrldi r30, r10, 32
+; P8LE-NEXT:    mulld r11, r11, r5
+; P8LE-NEXT:    clrldi r29, r12, 32
+; P8LE-NEXT:    mulld r0, r0, r5
+; P8LE-NEXT:    mulld r30, r30, r5
+; P8LE-NEXT:    mulld r5, r29, r5
+; P8LE-NEXT:    ld r29, -24(r1) # 8-byte Folded Reload
+; P8LE-NEXT:    rldicl r11, r11, 32, 32
+; P8LE-NEXT:    rldicl r0, r0, 32, 32
+; P8LE-NEXT:    rldicl r30, r30, 32, 32
+; P8LE-NEXT:    subf r8, r11, r8
+; P8LE-NEXT:    rldicl r5, r5, 32, 32
+; P8LE-NEXT:    subf r9, r0, r9
+; P8LE-NEXT:    srwi r8, r8, 1
+; P8LE-NEXT:    subf r10, r30, r10
+; P8LE-NEXT:    add r8, r8, r11
+; P8LE-NEXT:    srwi r9, r9, 1
+; P8LE-NEXT:    srwi r10, r10, 1
+; P8LE-NEXT:    subf r11, r5, r12
+; P8LE-NEXT:    add r9, r9, r0
+; P8LE-NEXT:    srwi r8, r8, 6
+; P8LE-NEXT:    add r10, r10, r30
+; P8LE-NEXT:    srwi r11, r11, 1
+; P8LE-NEXT:    ld r30, -16(r1) # 8-byte Folded Reload
+; P8LE-NEXT:    srwi r9, r9, 6
+; P8LE-NEXT:    mulli r12, r8, 95
+; P8LE-NEXT:    srwi r10, r10, 6
+; P8LE-NEXT:    add r5, r11, r5
+; P8LE-NEXT:    mtvsrd f0, r8
+; P8LE-NEXT:    mulli r8, r9, 95
+; P8LE-NEXT:    mtvsrd f1, r9
+; P8LE-NEXT:    mulli r9, r10, 95
+; P8LE-NEXT:    srwi r5, r5, 6
+; P8LE-NEXT:    mtvsrd f3, r5
+; P8LE-NEXT:    mulli r5, r5, 95
+; P8LE-NEXT:    xxswapd v2, vs0
+; P8LE-NEXT:    xxswapd v3, vs1
+; P8LE-NEXT:    mtvsrd f2, r10
+; P8LE-NEXT:    subf r3, r12, r3
+; P8LE-NEXT:    xxswapd v6, vs3
+; P8LE-NEXT:    mtvsrd f0, r3
+; P8LE-NEXT:    subf r3, r9, r7
+; P8LE-NEXT:    subf r4, r8, r4
+; P8LE-NEXT:    xxswapd v1, vs2
+; P8LE-NEXT:    mtvsrd f4, r3
+; P8LE-NEXT:    subf r3, r5, r6
+; P8LE-NEXT:    mtvsrd f1, r4
+; P8LE-NEXT:    mtvsrd f5, r3
+; P8LE-NEXT:    xxswapd v5, vs4
+; P8LE-NEXT:    vmrglh v2, v3, v2
+; P8LE-NEXT:    xxswapd v3, vs0
+; P8LE-NEXT:    xxswapd v4, vs1
+; P8LE-NEXT:    xxswapd v0, vs5
+; P8LE-NEXT:    vmrglh v3, v4, v3
+; P8LE-NEXT:    vmrglh v4, v0, v5
+; P8LE-NEXT:    vmrglh v5, v6, v1
+; P8LE-NEXT:    vmrglw v3, v4, v3
+; P8LE-NEXT:    vmrglw v2, v5, v2
+; P8LE-NEXT:    vadduhm v2, v3, v2
+; P8LE-NEXT:    blr
+;
+; P8BE-LABEL: combine_urem_udiv:
+; P8BE:       # %bb.0:
+; P8BE-NEXT:    mfvsrd r6, v2
+; P8BE-NEXT:    lis r5, 22765
+; P8BE-NEXT:    std r30, -16(r1) # 8-byte Folded Spill
+; P8BE-NEXT:    ori r5, r5, 8969
+; P8BE-NEXT:    clrldi r3, r6, 48
+; P8BE-NEXT:    rldicl r4, r6, 48, 48
+; P8BE-NEXT:    rlwinm r8, r3, 0, 16, 31
+; P8BE-NEXT:    rldicl r7, r6, 32, 48
+; P8BE-NEXT:    rlwinm r9, r4, 0, 16, 31
+; P8BE-NEXT:    rldicl r6, r6, 16, 48
+; P8BE-NEXT:    clrldi r11, r8, 32
+; P8BE-NEXT:    rlwinm r10, r7, 0, 16, 31
+; P8BE-NEXT:    rlwinm r6, r6, 0, 16, 31
+; P8BE-NEXT:    clrldi r12, r9, 32
+; P8BE-NEXT:    mulld r11, r11, r5
+; P8BE-NEXT:    clrldi r0, r10, 32
+; P8BE-NEXT:    clrldi r30, r6, 32
+; P8BE-NEXT:    mulld r12, r12, r5
+; P8BE-NEXT:    mulld r0, r0, r5
+; P8BE-NEXT:    mulld r5, r30, r5
+; P8BE-NEXT:    ld r30, -16(r1) # 8-byte Folded Reload
+; P8BE-NEXT:    rldicl r11, r11, 32, 32
+; P8BE-NEXT:    rldicl r12, r12, 32, 32
+; P8BE-NEXT:    subf r8, r11, r8
+; P8BE-NEXT:    rldicl r5, r5, 32, 32
+; P8BE-NEXT:    subf r9, r12, r9
+; P8BE-NEXT:    srwi r8, r8, 1
+; P8BE-NEXT:    rldicl r0, r0, 32, 32
+; P8BE-NEXT:    add r8, r8, r11
+; P8BE-NEXT:    srwi r9, r9, 1
+; P8BE-NEXT:    subf r11, r5, r6
+; P8BE-NEXT:    subf r10, r0, r10
+; P8BE-NEXT:    add r9, r9, r12
+; P8BE-NEXT:    srwi r8, r8, 6
+; P8BE-NEXT:    srwi r11, r11, 1
+; P8BE-NEXT:    srwi r10, r10, 1
+; P8BE-NEXT:    srwi r9, r9, 6
+; P8BE-NEXT:    add r5, r11, r5
+; P8BE-NEXT:    mulli r12, r8, 95
+; P8BE-NEXT:    add r10, r10, r0
+; P8BE-NEXT:    srwi r5, r5, 6
+; P8BE-NEXT:    mulli r11, r9, 95
+; P8BE-NEXT:    sldi r9, r9, 48
+; P8BE-NEXT:    srwi r10, r10, 6
+; P8BE-NEXT:    sldi r8, r8, 48
+; P8BE-NEXT:    mtvsrd v3, r9
+; P8BE-NEXT:    mulli r9, r5, 95
+; P8BE-NEXT:    mtvsrd v2, r8
+; P8BE-NEXT:    mulli r8, r10, 95
+; P8BE-NEXT:    sldi r10, r10, 48
+; P8BE-NEXT:    subf r3, r12, r3
+; P8BE-NEXT:    mtvsrd v4, r10
+; P8BE-NEXT:    subf r4, r11, r4
+; P8BE-NEXT:    sldi r3, r3, 48
+; P8BE-NEXT:    vmrghh v2, v3, v2
+; P8BE-NEXT:    sldi r4, r4, 48
+; P8BE-NEXT:    mtvsrd v3, r3
+; P8BE-NEXT:    subf r3, r9, r6
+; P8BE-NEXT:    subf r7, r8, r7
+; P8BE-NEXT:    mtvsrd v5, r4
+; P8BE-NEXT:    sldi r3, r3, 48
+; P8BE-NEXT:    sldi r6, r7, 48
+; P8BE-NEXT:    mtvsrd v1, r3
+; P8BE-NEXT:    sldi r3, r5, 48
+; P8BE-NEXT:    mtvsrd v0, r6
+; P8BE-NEXT:    vmrghh v3, v5, v3
+; P8BE-NEXT:    mtvsrd v5, r3
+; P8BE-NEXT:    vmrghh v0, v1, v0
+; P8BE-NEXT:    vmrghh v4, v5, v4
+; P8BE-NEXT:    vmrghw v3, v0, v3
+; P8BE-NEXT:    vmrghw v2, v4, v2
+; P8BE-NEXT:    vadduhm v2, v3, v2
+; P8BE-NEXT:    blr
+  %1 = urem <4 x i16> %x, <i16 95, i16 95, i16 95, i16 95>
+  %2 = udiv <4 x i16> %x, <i16 95, i16 95, i16 95, i16 95>
+  %3 = add <4 x i16> %1, %2
+  ret <4 x i16> %3
+}
+
+; Don't fold for divisors that are a power of two.
+define <4 x i16> @dont_fold_urem_power_of_two(<4 x i16> %x) {
+; P9LE-LABEL: dont_fold_urem_power_of_two:
+; P9LE:       # %bb.0:
+; P9LE-NEXT:    li r3, 0
+; P9LE-NEXT:    vextuhrx r3, r3, v2
+; P9LE-NEXT:    rlwinm r3, r3, 0, 26, 31
+; P9LE-NEXT:    mtvsrd f0, r3
+; P9LE-NEXT:    li r3, 2
+; P9LE-NEXT:    vextuhrx r3, r3, v2
+; P9LE-NEXT:    rlwinm r3, r3, 0, 27, 31
+; P9LE-NEXT:    xxswapd v3, vs0
+; P9LE-NEXT:    mtvsrd f0, r3
+; P9LE-NEXT:    li r3, 6
+; P9LE-NEXT:    vextuhrx r3, r3, v2
+; P9LE-NEXT:    rlwinm r4, r3, 0, 16, 31
+; P9LE-NEXT:    lis r6, 22765
+; P9LE-NEXT:    ori r6, r6, 8969
+; P9LE-NEXT:    xxswapd v4, vs0
+; P9LE-NEXT:    vmrglh v3, v4, v3
+; P9LE-NEXT:    clrldi r5, r4, 32
+; P9LE-NEXT:    mulld r5, r5, r6
+; P9LE-NEXT:    rldicl r5, r5, 32, 32
+; P9LE-NEXT:    subf r4, r5, r4
+; P9LE-NEXT:    srwi r4, r4, 1
+; P9LE-NEXT:    add r4, r4, r5
+; P9LE-NEXT:    srwi r4, r4, 6
+; P9LE-NEXT:    mulli r4, r4, 95
+; P9LE-NEXT:    subf r3, r4, r3
+; P9LE-NEXT:    mtvsrd f0, r3
+; P9LE-NEXT:    li r3, 4
+; P9LE-NEXT:    vextuhrx r3, r3, v2
+; P9LE-NEXT:    rlwinm r3, r3, 0, 29, 31
+; P9LE-NEXT:    xxswapd v4, vs0
+; P9LE-NEXT:    mtvsrd f0, r3
+; P9LE-NEXT:    xxswapd v2, vs0
+; P9LE-NEXT:    vmrglh v2, v4, v2
+; P9LE-NEXT:    vmrglw v2, v2, v3
+; P9LE-NEXT:    blr
+;
+; P9BE-LABEL: dont_fold_urem_power_of_two:
+; P9BE:       # %bb.0:
+; P9BE-NEXT:    li r3, 2
+; P9BE-NEXT:    vextuhlx r3, r3, v2
+; P9BE-NEXT:    rlwinm r3, r3, 0, 27, 31
+; P9BE-NEXT:    sldi r3, r3, 48
+; P9BE-NEXT:    mtvsrd v3, r3
+; P9BE-NEXT:    li r3, 0
+; P9BE-NEXT:    vextuhlx r3, r3, v2
+; P9BE-NEXT:    rlwinm r3, r3, 0, 26, 31
+; P9BE-NEXT:    sldi r3, r3, 48
+; P9BE-NEXT:    mtvsrd v4, r3
+; P9BE-NEXT:    li r3, 6
+; P9BE-NEXT:    vextuhlx r3, r3, v2
+; P9BE-NEXT:    rlwinm r3, r3, 0, 16, 31
+; P9BE-NEXT:    lis r5, 22765
+; P9BE-NEXT:    ori r5, r5, 8969
+; P9BE-NEXT:    vmrghh v3, v4, v3
+; P9BE-NEXT:    clrldi r4, r3, 32
+; P9BE-NEXT:    mulld r4, r4, r5
+; P9BE-NEXT:    rldicl r4, r4, 32, 32
+; P9BE-NEXT:    subf r5, r4, r3
+; P9BE-NEXT:    srwi r5, r5, 1
+; P9BE-NEXT:    add r4, r5, r4
+; P9BE-NEXT:    srwi r4, r4, 6
+; P9BE-NEXT:    mulli r4, r4, 95
+; P9BE-NEXT:    subf r3, r4, r3
+; P9BE-NEXT:    sldi r3, r3, 48
+; P9BE-NEXT:    mtvsrd v4, r3
+; P9BE-NEXT:    li r3, 4
+; P9BE-NEXT:    vextuhlx r3, r3, v2
+; P9BE-NEXT:    rlwinm r3, r3, 0, 29, 31
+; P9BE-NEXT:    sldi r3, r3, 48
+; P9BE-NEXT:    mtvsrd v2, r3
+; P9BE-NEXT:    vmrghh v2, v2, v4
+; P9BE-NEXT:    vmrghw v2, v3, v2
+; P9BE-NEXT:    blr
+;
+; P8LE-LABEL: dont_fold_urem_power_of_two:
+; P8LE:       # %bb.0:
+; P8LE-NEXT:    xxswapd vs0, v2
+; P8LE-NEXT:    lis r3, 22765
+; P8LE-NEXT:    ori r3, r3, 8969
+; P8LE-NEXT:    mfvsrd r4, f0
+; P8LE-NEXT:    rldicl r5, r4, 16, 48
+; P8LE-NEXT:    rlwinm r6, r5, 0, 16, 31
+; P8LE-NEXT:    clrldi r7, r6, 32
+; P8LE-NEXT:    mulld r3, r7, r3
+; P8LE-NEXT:    rldicl r7, r4, 48, 48
+; P8LE-NEXT:    rlwinm r7, r7, 0, 27, 31
+; P8LE-NEXT:    mtvsrd f1, r7
+; P8LE-NEXT:    rldicl r3, r3, 32, 32
+; P8LE-NEXT:    xxswapd v3, vs1
+; P8LE-NEXT:    subf r6, r3, r6
+; P8LE-NEXT:    srwi r6, r6, 1
+; P8LE-NEXT:    add r3, r6, r3
+; P8LE-NEXT:    clrldi r6, r4, 48
+; P8LE-NEXT:    srwi r3, r3, 6
+; P8LE-NEXT:    rldicl r4, r4, 32, 48
+; P8LE-NEXT:    rlwinm r6, r6, 0, 26, 31
+; P8LE-NEXT:    mulli r3, r3, 95
+; P8LE-NEXT:    rlwinm r4, r4, 0, 29, 31
+; P8LE-NEXT:    mtvsrd f0, r6
+; P8LE-NEXT:    mtvsrd f3, r4
+; P8LE-NEXT:    xxswapd v2, vs0
+; P8LE-NEXT:    xxswapd v5, vs3
+; P8LE-NEXT:    subf r3, r3, r5
+; P8LE-NEXT:    mtvsrd f2, r3
+; P8LE-NEXT:    vmrglh v2, v3, v2
+; P8LE-NEXT:    xxswapd v4, vs2
+; P8LE-NEXT:    vmrglh v3, v4, v5
+; P8LE-NEXT:    vmrglw v2, v3, v2
+; P8LE-NEXT:    blr
+;
+; P8BE-LABEL: dont_fold_urem_power_of_two:
+; P8BE:       # %bb.0:
+; P8BE-NEXT:    mfvsrd r4, v2
+; P8BE-NEXT:    lis r3, 22765
+; P8BE-NEXT:    ori r3, r3, 8969
+; P8BE-NEXT:    clrldi r5, r4, 48
+; P8BE-NEXT:    rldicl r7, r4, 16, 48
+; P8BE-NEXT:    rlwinm r5, r5, 0, 16, 31
+; P8BE-NEXT:    rlwinm r7, r7, 0, 26, 31
+; P8BE-NEXT:    clrldi r6, r5, 32
+; P8BE-NEXT:    mulld r3, r6, r3
+; P8BE-NEXT:    rldicl r3, r3, 32, 32
+; P8BE-NEXT:    subf r6, r3, r5
+; P8BE-NEXT:    srwi r6, r6, 1
+; P8BE-NEXT:    add r3, r6, r3
+; P8BE-NEXT:    rldicl r6, r4, 32, 48
+; P8BE-NEXT:    srwi r3, r3, 6
+; P8BE-NEXT:    rldicl r4, r4, 48, 48
+; P8BE-NEXT:    rlwinm r6, r6, 0, 27, 31
+; P8BE-NEXT:    mulli r3, r3, 95
+; P8BE-NEXT:    sldi r6, r6, 48
+; P8BE-NEXT:    rlwinm r4, r4, 0, 29, 31
+; P8BE-NEXT:    mtvsrd v2, r6
+; P8BE-NEXT:    sldi r6, r7, 48
+; P8BE-NEXT:    sldi r4, r4, 48
+; P8BE-NEXT:    mtvsrd v3, r6
+; P8BE-NEXT:    mtvsrd v5, r4
+; P8BE-NEXT:    subf r3, r3, r5
+; P8BE-NEXT:    vmrghh v2, v3, v2
+; P8BE-NEXT:    sldi r3, r3, 48
+; P8BE-NEXT:    mtvsrd v4, r3
+; P8BE-NEXT:    vmrghh v3, v5, v4
+; P8BE-NEXT:    vmrghw v2, v2, v3
+; P8BE-NEXT:    blr
+  %1 = urem <4 x i16> %x, <i16 64, i16 32, i16 8, i16 95>
+  ret <4 x i16> %1
+}
+
+; Don't fold if the divisor is one.
+define <4 x i16> @dont_fold_urem_one(<4 x i16> %x) {
+; P9LE-LABEL: dont_fold_urem_one:
+; P9LE:       # %bb.0:
+; P9LE-NEXT:    li r3, 4
+; P9LE-NEXT:    li r5, 0
+; P9LE-NEXT:    vextuhrx r3, r3, v2
+; P9LE-NEXT:    oris r6, r5, 45590
+; P9LE-NEXT:    oris r5, r5, 51306
+; P9LE-NEXT:    ori r6, r6, 17097
+; P9LE-NEXT:    ori r5, r5, 30865
+; P9LE-NEXT:    rlwinm r4, r3, 0, 16, 31
+; P9LE-NEXT:    mulld r4, r4, r6
+; P9LE-NEXT:    lis r6, 24749
+; P9LE-NEXT:    ori r6, r6, 47143
+; P9LE-NEXT:    rldicl r4, r4, 28, 36
+; P9LE-NEXT:    mulli r4, r4, 23
+; P9LE-NEXT:    subf r3, r4, r3
+; P9LE-NEXT:    mtvsrd f0, r3
+; P9LE-NEXT:    li r3, 6
+; P9LE-NEXT:    vextuhrx r3, r3, v2
+; P9LE-NEXT:    rlwinm r4, r3, 0, 16, 31
+; P9LE-NEXT:    mulld r4, r4, r6
+; P9LE-NEXT:    rldicl r4, r4, 21, 43
+; P9LE-NEXT:    mulli r4, r4, 5423
+; P9LE-NEXT:    subf r3, r4, r3
+; P9LE-NEXT:    xxswapd v3, vs0
+; P9LE-NEXT:    mtvsrd f0, r3
+; P9LE-NEXT:    li r3, 2
+; P9LE-NEXT:    vextuhrx r3, r3, v2
+; P9LE-NEXT:    rlwinm r4, r3, 31, 17, 31
+; P9LE-NEXT:    mulld r4, r4, r5
+; P9LE-NEXT:    rldicl r4, r4, 24, 40
+; P9LE-NEXT:    mulli r4, r4, 654
+; P9LE-NEXT:    subf r3, r4, r3
+; P9LE-NEXT:    xxswapd v4, vs0
+; P9LE-NEXT:    mtvsrd f0, r3
+; P9LE-NEXT:    xxswapd v2, vs0
+; P9LE-NEXT:    vmrglh v3, v4, v3
+; P9LE-NEXT:    xxlxor v4, v4, v4
+; P9LE-NEXT:    vmrglh v2, v2, v4
+; P9LE-NEXT:    vmrglw v2, v3, v2
+; P9LE-NEXT:    blr
+;
+; P9BE-LABEL: dont_fold_urem_one:
+; P9BE:       # %bb.0:
+; P9BE-NEXT:    li r3, 6
+; P9BE-NEXT:    vextuhlx r3, r3, v2
+; P9BE-NEXT:    rlwinm r3, r3, 0, 16, 31
+; P9BE-NEXT:    lis r5, 24749
+; P9BE-NEXT:    ori r5, r5, 47143
+; P9BE-NEXT:    clrldi r4, r3, 32
+; P9BE-NEXT:    mulld r4, r4, r5
+; P9BE-NEXT:    li r5, 0
+; P9BE-NEXT:    oris r6, r5, 45590
+; P9BE-NEXT:    oris r5, r5, 51306
+; P9BE-NEXT:    ori r6, r6, 17097
+; P9BE-NEXT:    ori r5, r5, 30865
+; P9BE-NEXT:    rldicl r4, r4, 21, 43
+; P9BE-NEXT:    mulli r4, r4, 5423
+; P9BE-NEXT:    subf r3, r4, r3
+; P9BE-NEXT:    sldi r3, r3, 48
+; P9BE-NEXT:    mtvsrd v3, r3
+; P9BE-NEXT:    li r3, 4
+; P9BE-NEXT:    vextuhlx r3, r3, v2
+; P9BE-NEXT:    rlwinm r3, r3, 0, 16, 31
+; P9BE-NEXT:    clrldi r4, r3, 32
+; P9BE-NEXT:    mulld r4, r4, r6
+; P9BE-NEXT:    rldicl r4, r4, 28, 36
+; P9BE-NEXT:    mulli r4, r4, 23
+; P9BE-NEXT:    subf r3, r4, r3
+; P9BE-NEXT:    sldi r3, r3, 48
+; P9BE-NEXT:    mtvsrd v4, r3
+; P9BE-NEXT:    li r3, 2
+; P9BE-NEXT:    vextuhlx r3, r3, v2
+; P9BE-NEXT:    clrlwi r4, r3, 16
+; P9BE-NEXT:    rlwinm r3, r3, 31, 17, 31
+; P9BE-NEXT:    mulld r3, r3, r5
+; P9BE-NEXT:    rldicl r3, r3, 24, 40
+; P9BE-NEXT:    mulli r3, r3, 654
+; P9BE-NEXT:    subf r3, r3, r4
+; P9BE-NEXT:    sldi r3, r3, 48
+; P9BE-NEXT:    mtvsrd v2, r3
+; P9BE-NEXT:    li r3, 0
+; P9BE-NEXT:    sldi r3, r3, 48
+; P9BE-NEXT:    vmrghh v3, v4, v3
+; P9BE-NEXT:    mtvsrd v4, r3
+; P9BE-NEXT:    vmrghh v2, v4, v2
+; P9BE-NEXT:    vmrghw v2, v2, v3
+; P9BE-NEXT:    blr
+;
+; P8LE-LABEL: dont_fold_urem_one:
+; P8LE:       # %bb.0:
+; P8LE-NEXT:    xxswapd vs0, v2
+; P8LE-NEXT:    li r3, 0
+; P8LE-NEXT:    lis r8, 24749
+; P8LE-NEXT:    xxlxor v5, v5, v5
+; P8LE-NEXT:    oris r5, r3, 45590
+; P8LE-NEXT:    ori r8, r8, 47143
+; P8LE-NEXT:    oris r3, r3, 51306
+; P8LE-NEXT:    ori r5, r5, 17097
+; P8LE-NEXT:    ori r3, r3, 30865
+; P8LE-NEXT:    mfvsrd r4, f0
+; P8LE-NEXT:    rldicl r6, r4, 32, 48
+; P8LE-NEXT:    rldicl r7, r4, 16, 48
+; P8LE-NEXT:    rlwinm r9, r6, 0, 16, 31
+; P8LE-NEXT:    rldicl r4, r4, 48, 48
+; P8LE-NEXT:    mulld r5, r9, r5
+; P8LE-NEXT:    rlwinm r9, r7, 0, 16, 31
+; P8LE-NEXT:    mulld r8, r9, r8
+; P8LE-NEXT:    rlwinm r9, r4, 31, 17, 31
+; P8LE-NEXT:    mulld r3, r9, r3
+; P8LE-NEXT:    rldicl r5, r5, 28, 36
+; P8LE-NEXT:    rldicl r8, r8, 21, 43
+; P8LE-NEXT:    mulli r5, r5, 23
+; P8LE-NEXT:    rldicl r3, r3, 24, 40
+; P8LE-NEXT:    mulli r8, r8, 5423
+; P8LE-NEXT:    mulli r3, r3, 654
+; P8LE-NEXT:    subf r5, r5, r6
+; P8LE-NEXT:    subf r6, r8, r7
+; P8LE-NEXT:    mtvsrd f0, r5
+; P8LE-NEXT:    subf r3, r3, r4
+; P8LE-NEXT:    mtvsrd f1, r6
+; P8LE-NEXT:    mtvsrd f2, r3
+; P8LE-NEXT:    xxswapd v2, vs0
+; P8LE-NEXT:    xxswapd v3, vs1
+; P8LE-NEXT:    xxswapd v4, vs2
+; P8LE-NEXT:    vmrglh v2, v3, v2
+; P8LE-NEXT:    vmrglh v3, v4, v5
+; P8LE-NEXT:    vmrglw v2, v2, v3
+; P8LE-NEXT:    blr
+;
+; P8BE-LABEL: dont_fold_urem_one:
+; P8BE:       # %bb.0:
+; P8BE-NEXT:    mfvsrd r4, v2
+; P8BE-NEXT:    li r3, 0
+; P8BE-NEXT:    lis r8, 24749
+; P8BE-NEXT:    oris r6, r3, 51306
+; P8BE-NEXT:    ori r8, r8, 47143
+; P8BE-NEXT:    oris r3, r3, 45590
+; P8BE-NEXT:    rldicl r5, r4, 32, 48
+; P8BE-NEXT:    clrldi r7, r4, 48
+; P8BE-NEXT:    ori r6, r6, 30865
+; P8BE-NEXT:    ori r3, r3, 17097
+; P8BE-NEXT:    rldicl r4, r4, 48, 48
+; P8BE-NEXT:    rlwinm r9, r5, 31, 17, 31
+; P8BE-NEXT:    rlwinm r7, r7, 0, 16, 31
+; P8BE-NEXT:    rlwinm r5, r5, 0, 16, 31
+; P8BE-NEXT:    rlwinm r4, r4, 0, 16, 31
+; P8BE-NEXT:    mulld r6, r9, r6
+; P8BE-NEXT:    clrldi r9, r7, 32
+; P8BE-NEXT:    mulld r8, r9, r8
+; P8BE-NEXT:    clrldi r9, r4, 32
+; P8BE-NEXT:    mulld r3, r9, r3
+; P8BE-NEXT:    li r9, 0
+; P8BE-NEXT:    rldicl r6, r6, 24, 40
+; P8BE-NEXT:    mulli r6, r6, 654
+; P8BE-NEXT:    rldicl r8, r8, 21, 43
+; P8BE-NEXT:    rldicl r3, r3, 28, 36
+; P8BE-NEXT:    mulli r8, r8, 5423
+; P8BE-NEXT:    mulli r3, r3, 23
+; P8BE-NEXT:    subf r5, r6, r5
+; P8BE-NEXT:    sldi r6, r9, 48
+; P8BE-NEXT:    mtvsrd v2, r6
+; P8BE-NEXT:    sldi r5, r5, 48
+; P8BE-NEXT:    subf r6, r8, r7
+; P8BE-NEXT:    mtvsrd v3, r5
+; P8BE-NEXT:    subf r3, r3, r4
+; P8BE-NEXT:    sldi r4, r6, 48
+; P8BE-NEXT:    sldi r3, r3, 48
+; P8BE-NEXT:    mtvsrd v4, r4
+; P8BE-NEXT:    mtvsrd v5, r3
+; P8BE-NEXT:    vmrghh v2, v2, v3
+; P8BE-NEXT:    vmrghh v3, v5, v4
+; P8BE-NEXT:    vmrghw v2, v2, v3
+; P8BE-NEXT:    blr
+  %1 = urem <4 x i16> %x, <i16 1, i16 654, i16 23, i16 5423>
+  ret <4 x i16> %1
+}
+
+; Don't fold if the divisor is 2^16.
+define <4 x i16> @dont_fold_urem_i16_smax(<4 x i16> %x) {
+; CHECK-LABEL: dont_fold_urem_i16_smax:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    blr
+  %1 = urem <4 x i16> %x, <i16 1, i16 65536, i16 23, i16 5423>
+  ret <4 x i16> %1
+}
+
+; Don't fold i64 urem.
+define <4 x i64> @dont_fold_urem_i64(<4 x i64> %x) {
+; P9LE-LABEL: dont_fold_urem_i64:
+; P9LE:       # %bb.0:
+; P9LE-NEXT:    lis r4, 25644
+; P9LE-NEXT:    ori r4, r4, 34192
+; P9LE-NEXT:    sldi r4, r4, 32
+; P9LE-NEXT:    oris r4, r4, 45590
+; P9LE-NEXT:    mfvsrld r3, v3
+; P9LE-NEXT:    ori r4, r4, 17097
+; P9LE-NEXT:    mulhdu r4, r3, r4
+; P9LE-NEXT:    sub r5, r3, r4
+; P9LE-NEXT:    rldicl r5, r5, 63, 1
+; P9LE-NEXT:    add r4, r5, r4
+; P9LE-NEXT:    lis r5, -16037
+; P9LE-NEXT:    rldicl r4, r4, 60, 4
+; P9LE-NEXT:    ori r5, r5, 28749
+; P9LE-NEXT:    mulli r4, r4, 23
+; P9LE-NEXT:    sldi r5, r5, 32
+; P9LE-NEXT:    oris r5, r5, 52170
+; P9LE-NEXT:    ori r5, r5, 12109
+; P9LE-NEXT:    sub r3, r3, r4
+; P9LE-NEXT:    mfvsrd r4, v3
+; P9LE-NEXT:    mulhdu r5, r4, r5
+; P9LE-NEXT:    rldicl r5, r5, 52, 12
+; P9LE-NEXT:    mulli r5, r5, 5423
+; P9LE-NEXT:    sub r4, r4, r5
+; P9LE-NEXT:    lis r5, 25653
+; P9LE-NEXT:    ori r5, r5, 15432
+; P9LE-NEXT:    sldi r5, r5, 32
+; P9LE-NEXT:    mtvsrdd v3, r4, r3
+; P9LE-NEXT:    mfvsrd r3, v2
+; P9LE-NEXT:    rldicl r4, r3, 63, 1
+; P9LE-NEXT:    oris r5, r5, 1603
+; P9LE-NEXT:    ori r5, r5, 21445
+; P9LE-NEXT:    mulhdu r4, r4, r5
+; P9LE-NEXT:    rldicl r4, r4, 57, 7
+; P9LE-NEXT:    mulli r4, r4, 654
+; P9LE-NEXT:    sub r3, r3, r4
+; P9LE-NEXT:    li r4, 0
+; P9LE-NEXT:    mtvsrdd v2, r3, r4
+; P9LE-NEXT:    blr
+;
+; P9BE-LABEL: dont_fold_urem_i64:
+; P9BE:       # %bb.0:
+; P9BE-NEXT:    lis r4, 25644
+; P9BE-NEXT:    ori r4, r4, 34192
+; P9BE-NEXT:    sldi r4, r4, 32
+; P9BE-NEXT:    oris r4, r4, 45590
+; P9BE-NEXT:    mfvsrd r3, v3
+; P9BE-NEXT:    ori r4, r4, 17097
+; P9BE-NEXT:    mulhdu r4, r3, r4
+; P9BE-NEXT:    sub r5, r3, r4
+; P9BE-NEXT:    rldicl r5, r5, 63, 1
+; P9BE-NEXT:    add r4, r5, r4
+; P9BE-NEXT:    lis r5, -16037
+; P9BE-NEXT:    rldicl r4, r4, 60, 4
+; P9BE-NEXT:    mulli r4, r4, 23
+; P9BE-NEXT:    ori r5, r5, 28749
+; P9BE-NEXT:    sldi r5, r5, 32
+; P9BE-NEXT:    oris r5, r5, 52170
+; P9BE-NEXT:    ori r5, r5, 12109
+; P9BE-NEXT:    sub r3, r3, r4
+; P9BE-NEXT:    mfvsrld r4, v3
+; P9BE-NEXT:    mulhdu r5, r4, r5
+; P9BE-NEXT:    rldicl r5, r5, 52, 12
+; P9BE-NEXT:    mulli r5, r5, 5423
+; P9BE-NEXT:    sub r4, r4, r5
+; P9BE-NEXT:    lis r5, 25653
+; P9BE-NEXT:    ori r5, r5, 15432
+; P9BE-NEXT:    sldi r5, r5, 32
+; P9BE-NEXT:    mtvsrdd v3, r3, r4
+; P9BE-NEXT:    mfvsrld r3, v2
+; P9BE-NEXT:    rldicl r4, r3, 63, 1
+; P9BE-NEXT:    oris r5, r5, 1603
+; P9BE-NEXT:    ori r5, r5, 21445
+; P9BE-NEXT:    mulhdu r4, r4, r5
+; P9BE-NEXT:    rldicl r4, r4, 57, 7
+; P9BE-NEXT:    mulli r4, r4, 654
+; P9BE-NEXT:    sub r3, r3, r4
+; P9BE-NEXT:    mtvsrdd v2, 0, r3
+; P9BE-NEXT:    blr
+;
+; P8LE-LABEL: dont_fold_urem_i64:
+; P8LE:       # %bb.0:
+; P8LE-NEXT:    lis r3, 25644
+; P8LE-NEXT:    xxswapd vs0, v3
+; P8LE-NEXT:    lis r4, -16037
+; P8LE-NEXT:    lis r5, 25653
+; P8LE-NEXT:    mfvsrd r6, v2
+; P8LE-NEXT:    ori r3, r3, 34192
+; P8LE-NEXT:    ori r4, r4, 28749
+; P8LE-NEXT:    ori r5, r5, 15432
+; P8LE-NEXT:    mfvsrd r8, v3
+; P8LE-NEXT:    sldi r3, r3, 32
+; P8LE-NEXT:    sldi r4, r4, 32
+; P8LE-NEXT:    oris r3, r3, 45590
+; P8LE-NEXT:    mfvsrd r7, f0
+; P8LE-NEXT:    sldi r5, r5, 32
+; P8LE-NEXT:    oris r4, r4, 52170
+; P8LE-NEXT:    ori r3, r3, 17097
+; P8LE-NEXT:    oris r5, r5, 1603
+; P8LE-NEXT:    ori r4, r4, 12109
+; P8LE-NEXT:    mulhdu r3, r7, r3
+; P8LE-NEXT:    rldicl r9, r6, 63, 1
+; P8LE-NEXT:    ori r5, r5, 21445
+; P8LE-NEXT:    mulhdu r4, r8, r4
+; P8LE-NEXT:    mulhdu r5, r9, r5
+; P8LE-NEXT:    sub r9, r7, r3
+; P8LE-NEXT:    rldicl r9, r9, 63, 1
+; P8LE-NEXT:    rldicl r4, r4, 52, 12
+; P8LE-NEXT:    add r3, r9, r3
+; P8LE-NEXT:    rldicl r5, r5, 57, 7
+; P8LE-NEXT:    mulli r4, r4, 5423
+; P8LE-NEXT:    rldicl r3, r3, 60, 4
+; P8LE-NEXT:    mulli r5, r5, 654
+; P8LE-NEXT:    mulli r3, r3, 23
+; P8LE-NEXT:    sub r4, r8, r4
+; P8LE-NEXT:    sub r5, r6, r5
+; P8LE-NEXT:    mtvsrd f0, r4
+; P8LE-NEXT:    sub r3, r7, r3
+; P8LE-NEXT:    li r4, 0
+; P8LE-NEXT:    mtvsrd f1, r5
+; P8LE-NEXT:    mtvsrd f2, r3
+; P8LE-NEXT:    mtvsrd f3, r4
+; P8LE-NEXT:    xxmrghd v3, vs0, vs2
+; P8LE-NEXT:    xxmrghd v2, vs1, vs3
+; P8LE-NEXT:    blr
+;
+; P8BE-LABEL: dont_fold_urem_i64:
+; P8BE:       # %bb.0:
+; P8BE-NEXT:    lis r3, 25644
+; P8BE-NEXT:    lis r4, -16037
+; P8BE-NEXT:    xxswapd vs0, v3
+; P8BE-NEXT:    xxswapd vs1, v2
+; P8BE-NEXT:    lis r5, 25653
+; P8BE-NEXT:    ori r3, r3, 34192
+; P8BE-NEXT:    ori r4, r4, 28749
+; P8BE-NEXT:    mfvsrd r6, v3
+; P8BE-NEXT:    ori r5, r5, 15432
+; P8BE-NEXT:    sldi r3, r3, 32
+; P8BE-NEXT:    sldi r4, r4, 32
+; P8BE-NEXT:    oris r3, r3, 45590
+; P8BE-NEXT:    sldi r5, r5, 32
+; P8BE-NEXT:    mfvsrd r7, f0
+; P8BE-NEXT:    oris r4, r4, 52170
+; P8BE-NEXT:    ori r3, r3, 17097
+; P8BE-NEXT:    mfvsrd r8, f1
+; P8BE-NEXT:    oris r5, r5, 1603
+; P8BE-NEXT:    ori r4, r4, 12109
+; P8BE-NEXT:    mulhdu r3, r6, r3
+; P8BE-NEXT:    ori r5, r5, 21445
+; P8BE-NEXT:    mulhdu r4, r7, r4
+; P8BE-NEXT:    rldicl r9, r8, 63, 1
+; P8BE-NEXT:    mulhdu r5, r9, r5
+; P8BE-NEXT:    sub r9, r6, r3
+; P8BE-NEXT:    rldicl r9, r9, 63, 1
+; P8BE-NEXT:    rldicl r4, r4, 52, 12
+; P8BE-NEXT:    add r3, r9, r3
+; P8BE-NEXT:    mulli r4, r4, 5423
+; P8BE-NEXT:    rldicl r5, r5, 57, 7
+; P8BE-NEXT:    rldicl r3, r3, 60, 4
+; P8BE-NEXT:    mulli r5, r5, 654
+; P8BE-NEXT:    mulli r3, r3, 23
+; P8BE-NEXT:    sub r4, r7, r4
+; P8BE-NEXT:    mtvsrd f0, r4
+; P8BE-NEXT:    sub r4, r8, r5
+; P8BE-NEXT:    sub r3, r6, r3
+; P8BE-NEXT:    mtvsrd f1, r4
+; P8BE-NEXT:    li r4, 0
+; P8BE-NEXT:    mtvsrd f2, r3
+; P8BE-NEXT:    mtvsrd f3, r4
+; P8BE-NEXT:    xxmrghd v3, vs2, vs0
+; P8BE-NEXT:    xxmrghd v2, vs3, vs1
+; P8BE-NEXT:    blr
+  %1 = urem <4 x i64> %x, <i64 1, i64 654, i64 23, i64 5423>
+  ret <4 x i64> %1
+}
diff --git a/llvm/test/CodeGen/PowerPC/use-cr-result-of-dom-icmp-st.ll b/llvm/test/CodeGen/PowerPC/use-cr-result-of-dom-icmp-st.ll
index 7839669be..f0e2ddaeb 100644
--- a/llvm/test/CodeGen/PowerPC/use-cr-result-of-dom-icmp-st.ll
+++ b/llvm/test/CodeGen/PowerPC/use-cr-result-of-dom-icmp-st.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -O3 < %s | FileCheck %s -check-prefix=PPC64LE
+; RUN: llc -mcpu=pwr9 -mtriple=powerpc64le-unknown-linux-gnu -O3 -ppc-asm-full-reg-names -verify-machineinstrs < %s | FileCheck %s
 
 ; Test cases are generated from:
 ; long long NAME(PARAM a, PARAM b) {
@@ -13,22 +13,21 @@
 ; for example. ll is PARAM, a_op_b (i.e., a << b) is LHS, _1 (i.e., -1) is RHS.
 
 target datalayout = "e-m:e-i64:64-n32:64"
-target triple = "powerpc64le-unknown-linux-gnu"
 
 define i64 @ll_a_op_b__2(i64 %a, i64 %b) {
-; PPC64LE-LABEL: ll_a_op_b__2:
-; PPC64LE:       # %bb.0: # %entry
-; PPC64LE-NEXT:    sld 5, 3, 4
-; PPC64LE-NEXT:    cmpdi 5, -2
-; PPC64LE-NEXT:    ble 0, .LBB0_2
-; PPC64LE-NEXT:  # %bb.1: # %return
-; PPC64LE-NEXT:    mr 3, 4
-; PPC64LE-NEXT:    blr
-; PPC64LE-NEXT:  .LBB0_2: # %if.end
-; PPC64LE-NEXT:    li 5, 1
-; PPC64LE-NEXT:    isel 4, 4, 5, 2
-; PPC64LE-NEXT:    mulld 3, 4, 3
-; PPC64LE-NEXT:    blr
+; CHECK-LABEL: ll_a_op_b__2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    sld r5, r3, r4
+; CHECK-NEXT:    cmpdi r5, -2
+; CHECK-NEXT:    ble cr0, .LBB0_2
+; CHECK-NEXT:  # %bb.1: # %return
+; CHECK-NEXT:    mr r3, r4
+; CHECK-NEXT:    blr
+; CHECK-NEXT:  .LBB0_2: # %if.end
+; CHECK-NEXT:    li r5, 1
+; CHECK-NEXT:    isel r4, r5, r4, lt
+; CHECK-NEXT:    mulld r3, r4, r3
+; CHECK-NEXT:    blr
 entry:
   %shl = shl i64 %a, %b
   %cmp = icmp sgt i64 %shl, -2
@@ -45,19 +44,19 @@ return:                                           ; preds = %entry
 }
 
 define i64 @ll_a_op_b__1(i64 %a, i64 %b) {
-; PPC64LE-LABEL: ll_a_op_b__1:
-; PPC64LE:       # %bb.0: # %entry
-; PPC64LE-NEXT:    sld 5, 3, 4
-; PPC64LE-NEXT:    cmpdi 5, -1
-; PPC64LE-NEXT:    ble 0, .LBB1_2
-; PPC64LE-NEXT:  # %bb.1: # %return
-; PPC64LE-NEXT:    mr 3, 4
-; PPC64LE-NEXT:    blr
-; PPC64LE-NEXT:  .LBB1_2: # %if.end
-; PPC64LE-NEXT:    li 5, 1
-; PPC64LE-NEXT:    isel 4, 4, 5, 2
-; PPC64LE-NEXT:    mulld 3, 4, 3
-; PPC64LE-NEXT:    blr
+; CHECK-LABEL: ll_a_op_b__1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    sld r5, r3, r4
+; CHECK-NEXT:    cmpdi r5, -1
+; CHECK-NEXT:    ble cr0, .LBB1_2
+; CHECK-NEXT:  # %bb.1: # %return
+; CHECK-NEXT:    mr r3, r4
+; CHECK-NEXT:    blr
+; CHECK-NEXT:  .LBB1_2: # %if.end
+; CHECK-NEXT:    li r5, 1
+; CHECK-NEXT:    isel r4, r5, r4, lt
+; CHECK-NEXT:    mulld r3, r4, r3
+; CHECK-NEXT:    blr
 entry:
   %shl = shl i64 %a, %b
   %cmp = icmp sgt i64 %shl, -1
@@ -74,19 +73,18 @@ return:                                           ; preds = %entry
 }
 
 define i64 @ll_a_op_b_0(i64 %a, i64 %b) {
-; PPC64LE-LABEL: ll_a_op_b_0:
-; PPC64LE:       # %bb.0: # %entry
-; PPC64LE-NEXT:    sld. 5, 3, 4
-; PPC64LE-NEXT:    ble 0, .LBB2_2
-; PPC64LE-NEXT:  # %bb.1: # %return
-; PPC64LE-NEXT:    mr 3, 4
-; PPC64LE-NEXT:    blr
-; PPC64LE-NEXT:  .LBB2_2: # %if.end
-; PPC64LE-NEXT:    cmpldi 5, 0
-; PPC64LE-NEXT:    li 5, 1
-; PPC64LE-NEXT:    isel 4, 4, 5, 2
-; PPC64LE-NEXT:    mulld 3, 4, 3
-; PPC64LE-NEXT:    blr
+; CHECK-LABEL: ll_a_op_b_0:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    sld. r5, r3, r4
+; CHECK-NEXT:    ble cr0, .LBB2_2
+; CHECK-NEXT:  # %bb.1: # %return
+; CHECK-NEXT:    mr r3, r4
+; CHECK-NEXT:    blr
+; CHECK-NEXT:  .LBB2_2: # %if.end
+; CHECK-NEXT:    li r5, 1
+; CHECK-NEXT:    isel r4, r5, r4, lt
+; CHECK-NEXT:    mulld r3, r4, r3
+; CHECK-NEXT:    blr
 entry:
   %shl = shl i64 %a, %b
   %cmp = icmp sgt i64 %shl, 0
@@ -103,20 +101,19 @@ return:                                           ; preds = %entry
 }
 
 define i64 @ll_a_op_b_1(i64 %a, i64 %b) {
-; PPC64LE-LABEL: ll_a_op_b_1:
-; PPC64LE:       # %bb.0: # %entry
-; PPC64LE-NEXT:    sld 5, 3, 4
-; PPC64LE-NEXT:    cmpdi 5, 1
-; PPC64LE-NEXT:    ble 0, .LBB3_2
-; PPC64LE-NEXT:  # %bb.1: # %return
-; PPC64LE-NEXT:    mr 3, 4
-; PPC64LE-NEXT:    blr
-; PPC64LE-NEXT:  .LBB3_2: # %if.end
-; PPC64LE-NEXT:    cmpldi 5, 1
-; PPC64LE-NEXT:    li 5, 1
-; PPC64LE-NEXT:    isel 4, 4, 5, 2
-; PPC64LE-NEXT:    mulld 3, 4, 3
-; PPC64LE-NEXT:    blr
+; CHECK-LABEL: ll_a_op_b_1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    sld r5, r3, r4
+; CHECK-NEXT:    cmpdi r5, 1
+; CHECK-NEXT:    ble cr0, .LBB3_2
+; CHECK-NEXT:  # %bb.1: # %return
+; CHECK-NEXT:    mr r3, r4
+; CHECK-NEXT:    blr
+; CHECK-NEXT:  .LBB3_2: # %if.end
+; CHECK-NEXT:    li r5, 1
+; CHECK-NEXT:    isel r4, r5, r4, lt
+; CHECK-NEXT:    mulld r3, r4, r3
+; CHECK-NEXT:    blr
 entry:
   %shl = shl i64 %a, %b
   %cmp = icmp sgt i64 %shl, 1
@@ -133,20 +130,19 @@ return:                                           ; preds = %entry
 }
 
 define i64 @ll_a_op_b_2(i64 %a, i64 %b) {
-; PPC64LE-LABEL: ll_a_op_b_2:
-; PPC64LE:       # %bb.0: # %entry
-; PPC64LE-NEXT:    sld 5, 3, 4
-; PPC64LE-NEXT:    cmpdi 5, 2
-; PPC64LE-NEXT:    ble 0, .LBB4_2
-; PPC64LE-NEXT:  # %bb.1: # %return
-; PPC64LE-NEXT:    mr 3, 4
-; PPC64LE-NEXT:    blr
-; PPC64LE-NEXT:  .LBB4_2: # %if.end
-; PPC64LE-NEXT:    cmpldi 5, 2
-; PPC64LE-NEXT:    li 5, 1
-; PPC64LE-NEXT:    isel 4, 4, 5, 2
-; PPC64LE-NEXT:    mulld 3, 4, 3
-; PPC64LE-NEXT:    blr
+; CHECK-LABEL: ll_a_op_b_2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    sld r5, r3, r4
+; CHECK-NEXT:    cmpdi r5, 2
+; CHECK-NEXT:    ble cr0, .LBB4_2
+; CHECK-NEXT:  # %bb.1: # %return
+; CHECK-NEXT:    mr r3, r4
+; CHECK-NEXT:    blr
+; CHECK-NEXT:  .LBB4_2: # %if.end
+; CHECK-NEXT:    li r5, 1
+; CHECK-NEXT:    isel r4, r5, r4, lt
+; CHECK-NEXT:    mulld r3, r4, r3
+; CHECK-NEXT:    blr
 entry:
   %shl = shl i64 %a, %b
   %cmp = icmp sgt i64 %shl, 2
@@ -163,18 +159,18 @@ return:                                           ; preds = %entry
 }
 
 define i64 @ll_a__2(i64 %a, i64 %b) {
-; PPC64LE-LABEL: ll_a__2:
-; PPC64LE:       # %bb.0: # %entry
-; PPC64LE-NEXT:    cmpdi 3, -2
-; PPC64LE-NEXT:    ble 0, .LBB5_2
-; PPC64LE-NEXT:  # %bb.1: # %return
-; PPC64LE-NEXT:    mr 3, 4
-; PPC64LE-NEXT:    blr
-; PPC64LE-NEXT:  .LBB5_2: # %if.end
-; PPC64LE-NEXT:    li 5, 1
-; PPC64LE-NEXT:    isel 4, 4, 5, 2
-; PPC64LE-NEXT:    mulld 3, 4, 3
-; PPC64LE-NEXT:    blr
+; CHECK-LABEL: ll_a__2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cmpdi r3, -2
+; CHECK-NEXT:    ble cr0, .LBB5_2
+; CHECK-NEXT:  # %bb.1: # %return
+; CHECK-NEXT:    mr r3, r4
+; CHECK-NEXT:    blr
+; CHECK-NEXT:  .LBB5_2: # %if.end
+; CHECK-NEXT:    li r5, 1
+; CHECK-NEXT:    isel r4, r5, r4, lt
+; CHECK-NEXT:    mulld r3, r4, r3
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp sgt i64 %a, -2
   br i1 %cmp, label %return, label %if.end
@@ -190,18 +186,18 @@ return:                                           ; preds = %entry
 }
 
 define i64 @ll_a__1(i64 %a, i64 %b) {
-; PPC64LE-LABEL: ll_a__1:
-; PPC64LE:       # %bb.0: # %entry
-; PPC64LE-NEXT:    cmpdi 3, -1
-; PPC64LE-NEXT:    ble 0, .LBB6_2
-; PPC64LE-NEXT:  # %bb.1: # %return
-; PPC64LE-NEXT:    mr 3, 4
-; PPC64LE-NEXT:    blr
-; PPC64LE-NEXT:  .LBB6_2: # %if.end
-; PPC64LE-NEXT:    li 5, 1
-; PPC64LE-NEXT:    isel 4, 4, 5, 2
-; PPC64LE-NEXT:    mulld 3, 4, 3
-; PPC64LE-NEXT:    blr
+; CHECK-LABEL: ll_a__1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cmpdi r3, -1
+; CHECK-NEXT:    ble cr0, .LBB6_2
+; CHECK-NEXT:  # %bb.1: # %return
+; CHECK-NEXT:    mr r3, r4
+; CHECK-NEXT:    blr
+; CHECK-NEXT:  .LBB6_2: # %if.end
+; CHECK-NEXT:    li r5, 1
+; CHECK-NEXT:    isel r4, r5, r4, lt
+; CHECK-NEXT:    mulld r3, r4, r3
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp sgt i64 %a, -1
   br i1 %cmp, label %return, label %if.end
@@ -217,19 +213,18 @@ return:                                           ; preds = %entry
 }
 
 define i64 @ll_a_0(i64 %a, i64 %b) {
-; PPC64LE-LABEL: ll_a_0:
-; PPC64LE:       # %bb.0: # %entry
-; PPC64LE-NEXT:    cmpdi 3, 0
-; PPC64LE-NEXT:    ble 0, .LBB7_2
-; PPC64LE-NEXT:  # %bb.1: # %return
-; PPC64LE-NEXT:    mr 3, 4
-; PPC64LE-NEXT:    blr
-; PPC64LE-NEXT:  .LBB7_2: # %if.end
-; PPC64LE-NEXT:    cmpldi 3, 0
-; PPC64LE-NEXT:    li 5, 1
-; PPC64LE-NEXT:    isel 4, 4, 5, 2
-; PPC64LE-NEXT:    mulld 3, 4, 3
-; PPC64LE-NEXT:    blr
+; CHECK-LABEL: ll_a_0:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cmpdi r3, 0
+; CHECK-NEXT:    ble cr0, .LBB7_2
+; CHECK-NEXT:  # %bb.1: # %return
+; CHECK-NEXT:    mr r3, r4
+; CHECK-NEXT:    blr
+; CHECK-NEXT:  .LBB7_2: # %if.end
+; CHECK-NEXT:    li r5, 1
+; CHECK-NEXT:    isel r4, r5, r4, lt
+; CHECK-NEXT:    mulld r3, r4, r3
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp sgt i64 %a, 0
   br i1 %cmp, label %return, label %if.end
@@ -245,19 +240,18 @@ return:                                           ; preds = %entry
 }
 
 define i64 @ll_a_1(i64 %a, i64 %b) {
-; PPC64LE-LABEL: ll_a_1:
-; PPC64LE:       # %bb.0: # %entry
-; PPC64LE-NEXT:    cmpdi 3, 1
-; PPC64LE-NEXT:    ble 0, .LBB8_2
-; PPC64LE-NEXT:  # %bb.1: # %return
-; PPC64LE-NEXT:    mr 3, 4
-; PPC64LE-NEXT:    blr
-; PPC64LE-NEXT:  .LBB8_2: # %if.end
-; PPC64LE-NEXT:    cmpldi 3, 1
-; PPC64LE-NEXT:    li 5, 1
-; PPC64LE-NEXT:    isel 4, 4, 5, 2
-; PPC64LE-NEXT:    mulld 3, 4, 3
-; PPC64LE-NEXT:    blr
+; CHECK-LABEL: ll_a_1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cmpdi r3, 1
+; CHECK-NEXT:    ble cr0, .LBB8_2
+; CHECK-NEXT:  # %bb.1: # %return
+; CHECK-NEXT:    mr r3, r4
+; CHECK-NEXT:    blr
+; CHECK-NEXT:  .LBB8_2: # %if.end
+; CHECK-NEXT:    li r5, 1
+; CHECK-NEXT:    isel r4, r5, r4, lt
+; CHECK-NEXT:    mulld r3, r4, r3
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp sgt i64 %a, 1
   br i1 %cmp, label %return, label %if.end
@@ -273,19 +267,18 @@ return:                                           ; preds = %entry
 }
 
 define i64 @ll_a_2(i64 %a, i64 %b) {
-; PPC64LE-LABEL: ll_a_2:
-; PPC64LE:       # %bb.0: # %entry
-; PPC64LE-NEXT:    cmpdi 3, 2
-; PPC64LE-NEXT:    ble 0, .LBB9_2
-; PPC64LE-NEXT:  # %bb.1: # %return
-; PPC64LE-NEXT:    mr 3, 4
-; PPC64LE-NEXT:    blr
-; PPC64LE-NEXT:  .LBB9_2: # %if.end
-; PPC64LE-NEXT:    cmpldi 3, 2
-; PPC64LE-NEXT:    li 5, 1
-; PPC64LE-NEXT:    isel 4, 4, 5, 2
-; PPC64LE-NEXT:    mulld 3, 4, 3
-; PPC64LE-NEXT:    blr
+; CHECK-LABEL: ll_a_2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cmpdi r3, 2
+; CHECK-NEXT:    ble cr0, .LBB9_2
+; CHECK-NEXT:  # %bb.1: # %return
+; CHECK-NEXT:    mr r3, r4
+; CHECK-NEXT:    blr
+; CHECK-NEXT:  .LBB9_2: # %if.end
+; CHECK-NEXT:    li r5, 1
+; CHECK-NEXT:    isel r4, r5, r4, lt
+; CHECK-NEXT:    mulld r3, r4, r3
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp sgt i64 %a, 2
   br i1 %cmp, label %return, label %if.end
@@ -301,16 +294,18 @@ return:                                           ; preds = %entry
 }
 
 define i64 @i_a_op_b__2(i32 signext %a, i32 signext %b) {
-; PPC64LE-LABEL: i_a_op_b__2:
-; PPC64LE:       # %bb.0: # %entry
-; PPC64LE-NEXT:    slw 6, 3, 4
-; PPC64LE-NEXT:    li 5, 1
-; PPC64LE-NEXT:    cmpwi 6, -2
-; PPC64LE-NEXT:    isel 5, 4, 5, 2
-; PPC64LE-NEXT:    mullw 3, 5, 3
-; PPC64LE-NEXT:    isel 3, 4, 3, 1
-; PPC64LE-NEXT:    extsw 3, 3
-; PPC64LE-NEXT:    blr
+; CHECK-LABEL: i_a_op_b__2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    slw r5, r3, r4
+; CHECK-NEXT:    cmpwi r5, -2
+; CHECK-NEXT:    bgt cr0, .LBB10_2
+; CHECK-NEXT:  # %bb.1: # %if.end
+; CHECK-NEXT:    li r5, 1
+; CHECK-NEXT:    isel r4, r5, r4, lt
+; CHECK-NEXT:    mullw r4, r4, r3
+; CHECK-NEXT:  .LBB10_2: # %return
+; CHECK-NEXT:    extsw r3, r4
+; CHECK-NEXT:    blr
 entry:
   %shl = shl i32 %a, %b
   %cmp = icmp sgt i32 %shl, -2
@@ -329,16 +324,20 @@ return:                                           ; preds = %if.end, %entry
 }
 
 define i64 @i_a_op_b__1(i32 signext %a, i32 signext %b) {
-; PPC64LE-LABEL: i_a_op_b__1:
-; PPC64LE:       # %bb.0: # %entry
-; PPC64LE-NEXT:    slw 6, 3, 4
-; PPC64LE-NEXT:    li 5, 1
-; PPC64LE-NEXT:    cmpwi 6, -1
-; PPC64LE-NEXT:    isel 5, 4, 5, 2
-; PPC64LE-NEXT:    mullw 3, 5, 3
-; PPC64LE-NEXT:    isel 3, 4, 3, 1
-; PPC64LE-NEXT:    extsw 3, 3
-; PPC64LE-NEXT:    blr
+; CHECK-LABEL: i_a_op_b__1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    slw r5, r3, r4
+; CHECK-NEXT:    cmpwi r5, -1
+; CHECK-NEXT:    ble cr0, .LBB11_2
+; CHECK-NEXT:  # %bb.1: # %return
+; CHECK-NEXT:    extsw r3, r4
+; CHECK-NEXT:    blr
+; CHECK-NEXT:  .LBB11_2: # %if.end
+; CHECK-NEXT:    li r5, 1
+; CHECK-NEXT:    isel r4, r5, r4, lt
+; CHECK-NEXT:    mullw r4, r4, r3
+; CHECK-NEXT:    extsw r3, r4
+; CHECK-NEXT:    blr
 entry:
   %shl = shl i32 %a, %b
   %cmp = icmp sgt i32 %shl, -1
@@ -357,16 +356,20 @@ return:                                           ; preds = %if.end, %entry
 }
 
 define i64 @i_a_op_b_0(i32 signext %a, i32 signext %b) {
-; PPC64LE-LABEL: i_a_op_b_0:
-; PPC64LE:       # %bb.0: # %entry
-; PPC64LE-NEXT:    slw. 5, 3, 4
-; PPC64LE-NEXT:    li 6, 1
-; PPC64LE-NEXT:    isel 6, 4, 6, 2
-; PPC64LE-NEXT:    cmpwi 5, 0
-; PPC64LE-NEXT:    mullw 3, 6, 3
-; PPC64LE-NEXT:    isel 3, 4, 3, 1
-; PPC64LE-NEXT:    extsw 3, 3
-; PPC64LE-NEXT:    blr
+; CHECK-LABEL: i_a_op_b_0:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    slw r5, r3, r4
+; CHECK-NEXT:    cmpwi r5, 0
+; CHECK-NEXT:    ble cr0, .LBB12_2
+; CHECK-NEXT:  # %bb.1: # %return
+; CHECK-NEXT:    extsw r3, r4
+; CHECK-NEXT:    blr
+; CHECK-NEXT:  .LBB12_2: # %if.end
+; CHECK-NEXT:    li r5, 1
+; CHECK-NEXT:    isel r4, r5, r4, lt
+; CHECK-NEXT:    mullw r4, r4, r3
+; CHECK-NEXT:    extsw r3, r4
+; CHECK-NEXT:    blr
 entry:
   %shl = shl i32 %a, %b
   %cmp = icmp sgt i32 %shl, 0
@@ -385,17 +388,18 @@ return:                                           ; preds = %if.end, %entry
 }
 
 define i64 @i_a_op_b_1(i32 signext %a, i32 signext %b) {
-; PPC64LE-LABEL: i_a_op_b_1:
-; PPC64LE:       # %bb.0: # %entry
-; PPC64LE-NEXT:    slw 6, 3, 4
-; PPC64LE-NEXT:    li 5, 1
-; PPC64LE-NEXT:    cmplwi 6, 1
-; PPC64LE-NEXT:    isel 5, 4, 5, 2
-; PPC64LE-NEXT:    cmpwi 6, 1
-; PPC64LE-NEXT:    mullw 3, 5, 3
-; PPC64LE-NEXT:    isel 3, 4, 3, 1
-; PPC64LE-NEXT:    extsw 3, 3
-; PPC64LE-NEXT:    blr
+; CHECK-LABEL: i_a_op_b_1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    slw r5, r3, r4
+; CHECK-NEXT:    cmpwi r5, 1
+; CHECK-NEXT:    bgt cr0, .LBB13_2
+; CHECK-NEXT:  # %bb.1: # %if.end
+; CHECK-NEXT:    li r5, 1
+; CHECK-NEXT:    isel r4, r5, r4, lt
+; CHECK-NEXT:    mullw r4, r4, r3
+; CHECK-NEXT:  .LBB13_2: # %return
+; CHECK-NEXT:    extsw r3, r4
+; CHECK-NEXT:    blr
 entry:
   %shl = shl i32 %a, %b
   %cmp = icmp sgt i32 %shl, 1
@@ -414,17 +418,18 @@ return:                                           ; preds = %if.end, %entry
 }
 
 define i64 @i_a_op_b_2(i32 signext %a, i32 signext %b) {
-; PPC64LE-LABEL: i_a_op_b_2:
-; PPC64LE:       # %bb.0: # %entry
-; PPC64LE-NEXT:    slw 6, 3, 4
-; PPC64LE-NEXT:    li 5, 1
-; PPC64LE-NEXT:    cmplwi 6, 2
-; PPC64LE-NEXT:    isel 5, 4, 5, 2
-; PPC64LE-NEXT:    cmpwi 6, 2
-; PPC64LE-NEXT:    mullw 3, 5, 3
-; PPC64LE-NEXT:    isel 3, 4, 3, 1
-; PPC64LE-NEXT:    extsw 3, 3
-; PPC64LE-NEXT:    blr
+; CHECK-LABEL: i_a_op_b_2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    slw r5, r3, r4
+; CHECK-NEXT:    cmpwi r5, 2
+; CHECK-NEXT:    bgt cr0, .LBB14_2
+; CHECK-NEXT:  # %bb.1: # %if.end
+; CHECK-NEXT:    li r5, 1
+; CHECK-NEXT:    isel r4, r5, r4, lt
+; CHECK-NEXT:    mullw r4, r4, r3
+; CHECK-NEXT:  .LBB14_2: # %return
+; CHECK-NEXT:    extsw r3, r4
+; CHECK-NEXT:    blr
 entry:
   %shl = shl i32 %a, %b
   %cmp = icmp sgt i32 %shl, 2
@@ -443,15 +448,17 @@ return:                                           ; preds = %if.end, %entry
 }
 
 define i64 @i_a__2(i32 signext %a, i32 signext %b) {
-; PPC64LE-LABEL: i_a__2:
-; PPC64LE:       # %bb.0: # %entry
-; PPC64LE-NEXT:    li 5, 1
-; PPC64LE-NEXT:    cmpwi 3, -2
-; PPC64LE-NEXT:    isel 5, 4, 5, 2
-; PPC64LE-NEXT:    mullw 3, 5, 3
-; PPC64LE-NEXT:    isel 3, 4, 3, 1
-; PPC64LE-NEXT:    extsw 3, 3
-; PPC64LE-NEXT:    blr
+; CHECK-LABEL: i_a__2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cmpwi r3, -2
+; CHECK-NEXT:    bgt cr0, .LBB15_2
+; CHECK-NEXT:  # %bb.1: # %if.end
+; CHECK-NEXT:    li r5, 1
+; CHECK-NEXT:    isel r4, r5, r4, lt
+; CHECK-NEXT:    mullw r4, r4, r3
+; CHECK-NEXT:  .LBB15_2: # %return
+; CHECK-NEXT:    extsw r3, r4
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp sgt i32 %a, -2
   br i1 %cmp, label %return, label %if.end
@@ -469,15 +476,19 @@ return:                                           ; preds = %if.end, %entry
 }
 
 define i64 @i_a__1(i32 signext %a, i32 signext %b) {
-; PPC64LE-LABEL: i_a__1:
-; PPC64LE:       # %bb.0: # %entry
-; PPC64LE-NEXT:    li 5, 1
-; PPC64LE-NEXT:    cmpwi 3, -1
-; PPC64LE-NEXT:    isel 5, 4, 5, 2
-; PPC64LE-NEXT:    mullw 3, 5, 3
-; PPC64LE-NEXT:    isel 3, 4, 3, 1
-; PPC64LE-NEXT:    extsw 3, 3
-; PPC64LE-NEXT:    blr
+; CHECK-LABEL: i_a__1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cmpwi r3, -1
+; CHECK-NEXT:    ble cr0, .LBB16_2
+; CHECK-NEXT:  # %bb.1: # %return
+; CHECK-NEXT:    extsw r3, r4
+; CHECK-NEXT:    blr
+; CHECK-NEXT:  .LBB16_2: # %if.end
+; CHECK-NEXT:    li r5, 1
+; CHECK-NEXT:    isel r4, r5, r4, lt
+; CHECK-NEXT:    mullw r4, r4, r3
+; CHECK-NEXT:    extsw r3, r4
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp sgt i32 %a, -1
   br i1 %cmp, label %return, label %if.end
@@ -495,16 +506,19 @@ return:                                           ; preds = %if.end, %entry
 }
 
 define i64 @i_a_0(i32 signext %a, i32 signext %b) {
-; PPC64LE-LABEL: i_a_0:
-; PPC64LE:       # %bb.0: # %entry
-; PPC64LE-NEXT:    li 5, 1
-; PPC64LE-NEXT:    cmplwi 3, 0
-; PPC64LE-NEXT:    isel 5, 4, 5, 2
-; PPC64LE-NEXT:    cmpwi 0, 3, 0
-; PPC64LE-NEXT:    mullw 5, 5, 3
-; PPC64LE-NEXT:    isel 3, 4, 5, 1
-; PPC64LE-NEXT:    extsw 3, 3
-; PPC64LE-NEXT:    blr
+; CHECK-LABEL: i_a_0:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cmpwi r3, 0
+; CHECK-NEXT:    ble cr0, .LBB17_2
+; CHECK-NEXT:  # %bb.1: # %return
+; CHECK-NEXT:    extsw r3, r4
+; CHECK-NEXT:    blr
+; CHECK-NEXT:  .LBB17_2: # %if.end
+; CHECK-NEXT:    li r5, 1
+; CHECK-NEXT:    isel r4, r5, r4, lt
+; CHECK-NEXT:    mullw r4, r4, r3
+; CHECK-NEXT:    extsw r3, r4
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp sgt i32 %a, 0
   br i1 %cmp, label %return, label %if.end
@@ -522,16 +536,17 @@ return:                                           ; preds = %if.end, %entry
 }
 
 define i64 @i_a_1(i32 signext %a, i32 signext %b) {
-; PPC64LE-LABEL: i_a_1:
-; PPC64LE:       # %bb.0: # %entry
-; PPC64LE-NEXT:    li 5, 1
-; PPC64LE-NEXT:    cmplwi 3, 1
-; PPC64LE-NEXT:    isel 5, 4, 5, 2
-; PPC64LE-NEXT:    cmpwi 0, 3, 1
-; PPC64LE-NEXT:    mullw 5, 5, 3
-; PPC64LE-NEXT:    isel 3, 4, 5, 1
-; PPC64LE-NEXT:    extsw 3, 3
-; PPC64LE-NEXT:    blr
+; CHECK-LABEL: i_a_1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cmpwi r3, 1
+; CHECK-NEXT:    bgt cr0, .LBB18_2
+; CHECK-NEXT:  # %bb.1: # %if.end
+; CHECK-NEXT:    li r5, 1
+; CHECK-NEXT:    isel r4, r5, r4, lt
+; CHECK-NEXT:    mullw r4, r4, r3
+; CHECK-NEXT:  .LBB18_2: # %return
+; CHECK-NEXT:    extsw r3, r4
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp sgt i32 %a, 1
   br i1 %cmp, label %return, label %if.end
@@ -549,16 +564,17 @@ return:                                           ; preds = %if.end, %entry
 }
 
 define i64 @i_a_2(i32 signext %a, i32 signext %b) {
-; PPC64LE-LABEL: i_a_2:
-; PPC64LE:       # %bb.0: # %entry
-; PPC64LE-NEXT:    li 5, 1
-; PPC64LE-NEXT:    cmplwi 3, 2
-; PPC64LE-NEXT:    isel 5, 4, 5, 2
-; PPC64LE-NEXT:    cmpwi 0, 3, 2
-; PPC64LE-NEXT:    mullw 5, 5, 3
-; PPC64LE-NEXT:    isel 3, 4, 5, 1
-; PPC64LE-NEXT:    extsw 3, 3
-; PPC64LE-NEXT:    blr
+; CHECK-LABEL: i_a_2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cmpwi r3, 2
+; CHECK-NEXT:    bgt cr0, .LBB19_2
+; CHECK-NEXT:  # %bb.1: # %if.end
+; CHECK-NEXT:    li r5, 1
+; CHECK-NEXT:    isel r4, r5, r4, lt
+; CHECK-NEXT:    mullw r4, r4, r3
+; CHECK-NEXT:  .LBB19_2: # %return
+; CHECK-NEXT:    extsw r3, r4
+; CHECK-NEXT:    blr
 entry:
   %cmp = icmp sgt i32 %a, 2
   br i1 %cmp, label %return, label %if.end
diff --git a/llvm/test/CodeGen/PowerPC/uwtables.ll b/llvm/test/CodeGen/PowerPC/uwtables.ll
index 1968b715a..7523d04d7 100644
--- a/llvm/test/CodeGen/PowerPC/uwtables.ll
+++ b/llvm/test/CodeGen/PowerPC/uwtables.ll
@@ -47,5 +47,5 @@ declare i32 @__gxx_personality_v0(...)
 declare void @__cxa_call_unexpected(i8*) local_unnamed_addr
 
 
-attributes #0 = { noreturn nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noreturn nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
diff --git a/llvm/test/CodeGen/PowerPC/vavg.ll b/llvm/test/CodeGen/PowerPC/vavg.ll
new file mode 100644
index 000000000..735b39da8
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/vavg.ll
@@ -0,0 +1,240 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -mcpu=pwr9 < %s | FileCheck -check-prefix=CHECK-P9 %s
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -mcpu=pwr8 < %s | FileCheck -check-prefix=CHECK-P8 %s
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -mcpu=pwr7 < %s | FileCheck -check-prefix=CHECK-P7 %s
+define <8 x i16> @test_v8i16(<8 x i16> %m, <8 x i16> %n) {
+; CHECK-P9-LABEL: test_v8i16:
+; CHECK-P9:       # %bb.0: # %entry
+; CHECK-P9-NEXT:    vavguh 2, 3, 2
+; CHECK-P9-NEXT:    blr
+;
+; CHECK-P8-LABEL: test_v8i16:
+; CHECK-P8:       # %bb.0: # %entry
+; CHECK-P8-NEXT:    vavguh 2, 3, 2
+; CHECK-P8-NEXT:    blr
+;
+; CHECK-P7-LABEL: test_v8i16:
+; CHECK-P7:       # %bb.0: # %entry
+; CHECK-P7-NEXT:    vavguh 2, 3, 2
+; CHECK-P7-NEXT:    blr
+entry:
+  %add = add <8 x i16> %m, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
+  %add1 = add <8 x i16> %add, %n
+  %shr = lshr <8 x i16> %add1, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
+  ret <8 x i16> %shr
+}
+
+define <8 x i16> @test_v8i16_sign(<8 x i16> %m, <8 x i16> %n) {
+; CHECK-P9-LABEL: test_v8i16_sign:
+; CHECK-P9:       # %bb.0: # %entry
+; CHECK-P9-NEXT:    vavgsh 2, 3, 2
+; CHECK-P9-NEXT:    blr
+;
+; CHECK-P8-LABEL: test_v8i16_sign:
+; CHECK-P8:       # %bb.0: # %entry
+; CHECK-P8-NEXT:    vavgsh 2, 3, 2
+; CHECK-P8-NEXT:    blr
+;
+; CHECK-P7-LABEL: test_v8i16_sign:
+; CHECK-P7:       # %bb.0: # %entry
+; CHECK-P7-NEXT:    vavgsh 2, 3, 2
+; CHECK-P7-NEXT:    blr
+entry:
+  %add = add <8 x i16> %m, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
+  %add1 = add <8 x i16> %add, %n
+  %shr = ashr <8 x i16> %add1, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
+  ret <8 x i16> %shr
+}
+
+define <4 x i32> @test_v4i32(<4 x i32> %m, <4 x i32> %n) {
+; CHECK-P9-LABEL: test_v4i32:
+; CHECK-P9:       # %bb.0: # %entry
+; CHECK-P9-NEXT:    vavguw 2, 3, 2
+; CHECK-P9-NEXT:    blr
+;
+; CHECK-P8-LABEL: test_v4i32:
+; CHECK-P8:       # %bb.0: # %entry
+; CHECK-P8-NEXT:    vavguw 2, 3, 2
+; CHECK-P8-NEXT:    blr
+;
+; CHECK-P7-LABEL: test_v4i32:
+; CHECK-P7:       # %bb.0: # %entry
+; CHECK-P7-NEXT:    vavguw 2, 3, 2
+; CHECK-P7-NEXT:    blr
+entry:
+  %add = add <4 x i32> %m, <i32 1, i32 1, i32 1, i32 1>
+  %add1 = add <4 x i32> %add, %n
+  %shr = lshr <4 x i32> %add1, <i32 1, i32 1, i32 1, i32 1>
+  ret <4 x i32> %shr
+}
+
+define <4 x i32> @test_v4i32_sign(<4 x i32> %m, <4 x i32> %n) {
+; CHECK-P9-LABEL: test_v4i32_sign:
+; CHECK-P9:       # %bb.0: # %entry
+; CHECK-P9-NEXT:    vavgsw 2, 3, 2
+; CHECK-P9-NEXT:    blr
+;
+; CHECK-P8-LABEL: test_v4i32_sign:
+; CHECK-P8:       # %bb.0: # %entry
+; CHECK-P8-NEXT:    vavgsw 2, 3, 2
+; CHECK-P8-NEXT:    blr
+;
+; CHECK-P7-LABEL: test_v4i32_sign:
+; CHECK-P7:       # %bb.0: # %entry
+; CHECK-P7-NEXT:    vavgsw 2, 3, 2
+; CHECK-P7-NEXT:    blr
+entry:
+  %add = add <4 x i32> %m, <i32 1, i32 1, i32 1, i32 1>
+  %add1 = add <4 x i32> %add, %n
+  %shr = ashr <4 x i32> %add1, <i32 1, i32 1, i32 1, i32 1>
+  ret <4 x i32> %shr
+}
+
+define <16 x i8> @test_v16i8(<16 x i8> %m, <16 x i8> %n) {
+; CHECK-P9-LABEL: test_v16i8:
+; CHECK-P9:       # %bb.0: # %entry
+; CHECK-P9-NEXT:    vavgub 2, 3, 2
+; CHECK-P9-NEXT:    blr
+;
+; CHECK-P8-LABEL: test_v16i8:
+; CHECK-P8:       # %bb.0: # %entry
+; CHECK-P8-NEXT:    vavgub 2, 3, 2
+; CHECK-P8-NEXT:    blr
+;
+; CHECK-P7-LABEL: test_v16i8:
+; CHECK-P7:       # %bb.0: # %entry
+; CHECK-P7-NEXT:    vavgub 2, 3, 2
+; CHECK-P7-NEXT:    blr
+entry:
+  %add = add <16 x i8> %m, <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
+  %add1 = add <16 x i8> %add, %n
+  %shr = lshr <16 x i8> %add1, <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
+  ret <16 x i8> %shr
+}
+
+define <16 x i8> @test_v16i8_sign(<16 x i8> %m, <16 x i8> %n) {
+; CHECK-P9-LABEL: test_v16i8_sign:
+; CHECK-P9:       # %bb.0: # %entry
+; CHECK-P9-NEXT:    vavgsb 2, 3, 2
+; CHECK-P9-NEXT:    blr
+;
+; CHECK-P8-LABEL: test_v16i8_sign:
+; CHECK-P8:       # %bb.0: # %entry
+; CHECK-P8-NEXT:    vavgsb 2, 3, 2
+; CHECK-P8-NEXT:    blr
+;
+; CHECK-P7-LABEL: test_v16i8_sign:
+; CHECK-P7:       # %bb.0: # %entry
+; CHECK-P7-NEXT:    vavgsb 2, 3, 2
+; CHECK-P7-NEXT:    blr
+entry:
+  %add = add <16 x i8> %m, <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
+  %add1 = add <16 x i8> %add, %n
+  %shr = ashr <16 x i8> %add1, <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
+  ret <16 x i8> %shr
+}
+
+define <8 x i16> @test_v8i16_sign_negative(<8 x i16> %m, <8 x i16> %n) {
+; CHECK-P9-LABEL: test_v8i16_sign_negative:
+; CHECK-P9:       # %bb.0: # %entry
+; CHECK-P9-NEXT:    addis 3, 2, .LCPI6_0@toc@ha
+; CHECK-P9-NEXT:    addi 3, 3, .LCPI6_0@toc@l
+; CHECK-P9-NEXT:    vadduhm 2, 2, 3
+; CHECK-P9-NEXT:    lxvx 35, 0, 3
+; CHECK-P9-NEXT:    vadduhm 2, 2, 3
+; CHECK-P9-NEXT:    vspltish 3, 1
+; CHECK-P9-NEXT:    vsrah 2, 2, 3
+; CHECK-P9-NEXT:    blr
+;
+; CHECK-P8-LABEL: test_v8i16_sign_negative:
+; CHECK-P8:       # %bb.0: # %entry
+; CHECK-P8-NEXT:    addis 3, 2, .LCPI6_0@toc@ha
+; CHECK-P8-NEXT:    vadduhm 2, 2, 3
+; CHECK-P8-NEXT:    vspltish 4, 1
+; CHECK-P8-NEXT:    addi 3, 3, .LCPI6_0@toc@l
+; CHECK-P8-NEXT:    lvx 3, 0, 3
+; CHECK-P8-NEXT:    vadduhm 2, 2, 3
+; CHECK-P8-NEXT:    vsrah 2, 2, 4
+; CHECK-P8-NEXT:    blr
+;
+; CHECK-P7-LABEL: test_v8i16_sign_negative:
+; CHECK-P7:       # %bb.0: # %entry
+; CHECK-P7-NEXT:    addis 3, 2, .LCPI6_0@toc@ha
+; CHECK-P7-NEXT:    vadduhm 2, 2, 3
+; CHECK-P7-NEXT:    vspltish 4, 1
+; CHECK-P7-NEXT:    addi 3, 3, .LCPI6_0@toc@l
+; CHECK-P7-NEXT:    lvx 3, 0, 3
+; CHECK-P7-NEXT:    vadduhm 2, 2, 3
+; CHECK-P7-NEXT:    vsrah 2, 2, 4
+; CHECK-P7-NEXT:    blr
+entry:
+  %add = add <8 x i16> %m, <i16 1, i16 1, i16 1, i16 -1, i16 1, i16 1, i16 1, i16 1>
+  %add1 = add <8 x i16> %add, %n
+  %shr = ashr <8 x i16> %add1, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
+  ret <8 x i16> %shr
+}
+
+define <4 x i32> @test_v4i32_negative(<4 x i32> %m, <4 x i32> %n) {
+; CHECK-P9-LABEL: test_v4i32_negative:
+; CHECK-P9:       # %bb.0: # %entry
+; CHECK-P9-NEXT:    xxlnor 34, 34, 34
+; CHECK-P9-NEXT:    vsubuwm 2, 3, 2
+; CHECK-P9-NEXT:    vspltisw 3, 2
+; CHECK-P9-NEXT:    vsrw 2, 2, 3
+; CHECK-P9-NEXT:    blr
+;
+; CHECK-P8-LABEL: test_v4i32_negative:
+; CHECK-P8:       # %bb.0: # %entry
+; CHECK-P8-NEXT:    xxlnor 34, 34, 34
+; CHECK-P8-NEXT:    vspltisw 4, 2
+; CHECK-P8-NEXT:    vsubuwm 2, 3, 2
+; CHECK-P8-NEXT:    vsrw 2, 2, 4
+; CHECK-P8-NEXT:    blr
+;
+; CHECK-P7-LABEL: test_v4i32_negative:
+; CHECK-P7:       # %bb.0: # %entry
+; CHECK-P7-NEXT:    xxlnor 34, 34, 34
+; CHECK-P7-NEXT:    vspltisw 4, 2
+; CHECK-P7-NEXT:    vsubuwm 2, 3, 2
+; CHECK-P7-NEXT:    vsrw 2, 2, 4
+; CHECK-P7-NEXT:    blr
+entry:
+  %add = add <4 x i32> %m, <i32 1, i32 1, i32 1, i32 1>
+  %add1 = add <4 x i32> %add, %n
+  %shr = lshr <4 x i32> %add1, <i32 2, i32 2, i32 2, i32 2>
+  ret <4 x i32> %shr
+}
+
+define <4 x i32> @test_v4i32_sign_negative(<4 x i32> %m, <4 x i32> %n) {
+; CHECK-P9-LABEL: test_v4i32_sign_negative:
+; CHECK-P9:       # %bb.0: # %entry
+; CHECK-P9-NEXT:    vadduwm 2, 2, 3
+; CHECK-P9-NEXT:    xxleqv 35, 35, 35
+; CHECK-P9-NEXT:    vadduwm 2, 2, 3
+; CHECK-P9-NEXT:    vspltisw 3, 1
+; CHECK-P9-NEXT:    vsraw 2, 2, 3
+; CHECK-P9-NEXT:    blr
+;
+; CHECK-P8-LABEL: test_v4i32_sign_negative:
+; CHECK-P8:       # %bb.0: # %entry
+; CHECK-P8-NEXT:    xxleqv 36, 36, 36
+; CHECK-P8-NEXT:    vadduwm 2, 2, 3
+; CHECK-P8-NEXT:    vspltisw 3, 1
+; CHECK-P8-NEXT:    vadduwm 2, 2, 4
+; CHECK-P8-NEXT:    vsraw 2, 2, 3
+; CHECK-P8-NEXT:    blr
+;
+; CHECK-P7-LABEL: test_v4i32_sign_negative:
+; CHECK-P7:       # %bb.0: # %entry
+; CHECK-P7-NEXT:    vspltisb 4, -1
+; CHECK-P7-NEXT:    vadduwm 2, 2, 3
+; CHECK-P7-NEXT:    vspltisw 3, 1
+; CHECK-P7-NEXT:    vadduwm 2, 2, 4
+; CHECK-P7-NEXT:    vsraw 2, 2, 3
+; CHECK-P7-NEXT:    blr
+entry:
+  %add = add <4 x i32> %m, <i32 -1, i32 -1, i32 -1, i32 -1>
+  %add1 = add <4 x i32> %add, %n
+  %shr = ashr <4 x i32> %add1, <i32 1, i32 1, i32 1, i32 1>
+  ret <4 x i32> %shr
+}
diff --git a/llvm/test/CodeGen/PowerPC/vec-bswap.ll b/llvm/test/CodeGen/PowerPC/vec-bswap.ll
new file mode 100644
index 000000000..e6864d825
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/vec-bswap.ll
@@ -0,0 +1,115 @@
+; RUN: llc < %s -mtriple=powerpc64le-unknown-unknown -mcpu=pwr9 \
+; RUN:   -verify-machineinstrs -ppc-asm-full-reg-names | FileCheck %s
+define dso_local void @test(i32* %Arr, i32 signext %Len) {
+; CHECK-LABEL: test:
+; CHECK:         lxvx [[REG:vs[0-9]+]], r{{[0-9]+}}, r{{[0-9]+}}
+; CHECK-NEXT:    xxbrw vs{{[0-9]+}}, [[REG]]
+entry:
+  %cmp1 = icmp slt i32 0, %Len
+  br i1 %cmp1, label %for.body.lr.ph, label %for.cond.cleanup
+
+for.body.lr.ph:                                   ; preds = %entry
+  %min.iters.check = icmp ult i32 %Len, 4
+  br i1 %min.iters.check, label %scalar.ph, label %vector.ph
+
+vector.ph:                                        ; preds = %for.body.lr.ph
+  %n.mod.vf = urem i32 %Len, 4
+  %n.vec = sub i32 %Len, %n.mod.vf
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+  %0 = add i32 %index, 0
+  %1 = sext i32 %0 to i64
+  %2 = getelementptr inbounds i32, i32* %Arr, i64 %1
+  %3 = getelementptr inbounds i32, i32* %2, i32 0
+  %4 = bitcast i32* %3 to <4 x i32>*
+  %wide.load = load <4 x i32>, <4 x i32>* %4, align 4
+  %5 = call <4 x i32> @llvm.bswap.v4i32(<4 x i32> %wide.load)
+  %6 = sext i32 %0 to i64
+  %7 = getelementptr inbounds i32, i32* %Arr, i64 %6
+  %8 = getelementptr inbounds i32, i32* %7, i32 0
+  %9 = bitcast i32* %8 to <4 x i32>*
+  store <4 x i32> %5, <4 x i32>* %9, align 4
+  %index.next = add i32 %index, 4
+  %10 = icmp eq i32 %index.next, %n.vec
+  br i1 %10, label %middle.block, label %vector.body
+
+middle.block:                                     ; preds = %vector.body
+  %cmp.n = icmp eq i32 %Len, %n.vec
+  br i1 %cmp.n, label %for.cond.for.cond.cleanup_crit_edge, label %scalar.ph
+
+scalar.ph:                                        ; preds = %middle.block, %for.body.lr.ph
+  %bc.resume.val = phi i32 [ %n.vec, %middle.block ], [ 0, %for.body.lr.ph ]
+  br label %for.body
+
+for.cond.for.cond.cleanup_crit_edge:              ; preds = %middle.block, %for.inc
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.cond.for.cond.cleanup_crit_edge, %entry
+  br label %for.end
+
+for.body:                                         ; preds = %for.inc, %scalar.ph
+  %i.02 = phi i32 [ %bc.resume.val, %scalar.ph ], [ %inc, %for.inc ]
+  %idxprom = sext i32 %i.02 to i64
+  %arrayidx = getelementptr inbounds i32, i32* %Arr, i64 %idxprom
+  %11 = load i32, i32* %arrayidx, align 4
+  %12 = call i32 @llvm.bswap.i32(i32 %11)
+  %idxprom1 = sext i32 %i.02 to i64
+  %arrayidx2 = getelementptr inbounds i32, i32* %Arr, i64 %idxprom1
+  store i32 %12, i32* %arrayidx2, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body
+  %inc = add nsw i32 %i.02, 1
+  %cmp = icmp slt i32 %inc, %Len
+  br i1 %cmp, label %for.body, label %for.cond.for.cond.cleanup_crit_edge
+
+for.end:                                          ; preds = %for.cond.cleanup
+  ret void
+}
+
+define dso_local <8 x i16> @test_halfword(<8 x i16> %a) local_unnamed_addr {
+; CHECK-LABEL: test_halfword:
+; CHECK:       xxbrh vs34, vs34
+; CHECK-NEXT:  blr
+entry:
+  %0 = call <8 x i16> @llvm.bswap.v8i16(<8 x i16> %a)
+  ret <8 x i16> %0
+}
+
+define dso_local <2 x i64> @test_doubleword(<2 x i64> %a) local_unnamed_addr {
+; CHECK-LABEL: test_doubleword:
+; CHECK:       xxbrd vs34, vs34
+; CHECK-NEXT:  blr
+entry:
+  %0 = call <2 x i64> @llvm.bswap.v2i64(<2 x i64> %a)
+  ret <2 x i64> %0
+}
+
+define dso_local <1 x i128> @test_quadword(<1 x i128> %a) local_unnamed_addr {
+; CHECK-LABEL: test_quadword:
+; CHECK:       xxbrq vs34, vs34
+; CHECK-NEXT:  blr
+entry:
+  %0 = call <1 x i128> @llvm.bswap.v1i128(<1 x i128> %a)
+  ret <1 x i128> %0
+}
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare <1 x i128> @llvm.bswap.v1i128(<1 x i128>)
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare <2 x i64> @llvm.bswap.v2i64(<2 x i64>)
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare <8 x i16> @llvm.bswap.v8i16(<8 x i16>)
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare i32 @llvm.bswap.i32(i32)
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare <4 x i32> @llvm.bswap.v4i32(<4 x i32>)
diff --git a/llvm/test/CodeGen/PowerPC/vec-min-max.ll b/llvm/test/CodeGen/PowerPC/vec-min-max.ll
index 4b53d15e0..e29ef3368 100644
--- a/llvm/test/CodeGen/PowerPC/vec-min-max.ll
+++ b/llvm/test/CodeGen/PowerPC/vec-min-max.ll
@@ -237,3 +237,52 @@ entry:
   ret <2 x double> %1
 }
 
+define i128 @invalidv1i128(<2 x i128> %v1, <2 x i128> %v2) {
+; CHECK-LABEL: invalidv1i128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mfvsrd 3, 36
+; CHECK-NEXT:    xxswapd 0, 36
+; CHECK-NEXT:    mfvsrd 4, 34
+; CHECK-NEXT:    xxswapd 1, 34
+; CHECK-NEXT:    cmpld 4, 3
+; CHECK-NEXT:    cmpd 1, 4, 3
+; CHECK-NEXT:    mfvsrd 3, 0
+; CHECK-NEXT:    crandc 20, 4, 2
+; CHECK-NEXT:    mfvsrd 4, 1
+; CHECK-NEXT:    cmpld 1, 4, 3
+; CHECK-NEXT:    bc 12, 20, .LBB12_3
+; CHECK-NEXT:  # %bb.1:
+; CHECK-NEXT:    crand 20, 2, 4
+; CHECK-NEXT:    bc 12, 20, .LBB12_3
+; CHECK-NEXT:  # %bb.2:
+; CHECK-NEXT:    vmr 2, 4
+; CHECK-NEXT:  .LBB12_3:
+; CHECK-NEXT:    xxswapd 0, 34
+; CHECK-NEXT:    mfvsrd 4, 34
+; CHECK-NEXT:    mfvsrd 3, 0
+; CHECK-NEXT:    blr
+;
+; NOP8VEC-LABEL: invalidv1i128:
+; NOP8VEC:       # %bb.0:
+; NOP8VEC-NEXT:    cmpld 4, 8
+; NOP8VEC-NEXT:    cmpd 1, 4, 8
+; NOP8VEC-NEXT:    addi 5, 1, -32
+; NOP8VEC-NEXT:    crandc 20, 4, 2
+; NOP8VEC-NEXT:    cmpld 1, 3, 7
+; NOP8VEC-NEXT:    crand 21, 2, 4
+; NOP8VEC-NEXT:    cror 20, 21, 20
+; NOP8VEC-NEXT:    isel 3, 3, 7, 20
+; NOP8VEC-NEXT:    isel 4, 4, 8, 20
+; NOP8VEC-NEXT:    std 3, -32(1)
+; NOP8VEC-NEXT:    addi 3, 1, -16
+; NOP8VEC-NEXT:    std 4, -24(1)
+; NOP8VEC-NEXT:    lxvd2x 0, 0, 5
+; NOP8VEC-NEXT:    stxvd2x 0, 0, 3
+; NOP8VEC-NEXT:    ld 3, -16(1)
+; NOP8VEC-NEXT:    ld 4, -8(1)
+; NOP8VEC-NEXT:    blr
+%1 = icmp slt <2 x i128> %v1, %v2
+%2 = select <2 x i1> %1, <2 x i128> %v1, <2 x i128> %v2
+%3 = extractelement <2 x i128> %2, i32 0
+ret i128 %3
+}
diff --git a/llvm/test/CodeGen/PowerPC/vec_add_sub_doubleword.ll b/llvm/test/CodeGen/PowerPC/vec_add_sub_doubleword.ll
index 62f17acb8..cba2b5a5b 100644
--- a/llvm/test/CodeGen/PowerPC/vec_add_sub_doubleword.ll
+++ b/llvm/test/CodeGen/PowerPC/vec_add_sub_doubleword.ll
@@ -80,7 +80,7 @@ define <2 x i64> @test_sub(<2 x i64> %x, <2 x i64> %y) nounwind {
 define <2 x i64> @decrement_by_one(<2 x i64> %x) nounwind {
 ; VSX-LABEL: decrement_by_one:
 ; VSX:       # %bb.0:
-; VSX-NEXT:    vspltisb 3, -1
+; VSX-NEXT:    xxleqv 35, 35, 35
 ; VSX-NEXT:    vsubudm 2, 2, 3
 ; VSX-NEXT:    blr
 ;
diff --git a/llvm/test/CodeGen/PowerPC/vec_buildvector_loadstore.ll b/llvm/test/CodeGen/PowerPC/vec_buildvector_loadstore.ll
index aef43b41a..0a00dc447 100644
--- a/llvm/test/CodeGen/PowerPC/vec_buildvector_loadstore.ll
+++ b/llvm/test/CodeGen/PowerPC/vec_buildvector_loadstore.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -verify-machineinstrs < %s -mtriple=powerpc-unknown-linux-gnu -mattr=+altivec -disable-ppc-ilp-pref  | FileCheck %s
 ; Formerly this did byte loads and word stores.
 @a = external global <16 x i8>
@@ -5,8 +6,20 @@
 @c = external global <16 x i8>
 
 define void @foo() nounwind ssp {
-; CHECK: foo:
-; CHECK-NOT: stw
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    li 3, a@l
+; CHECK-NEXT:    lis 4, a@ha
+; CHECK-NEXT:    lvx 2, 4, 3
+; CHECK-NEXT:    li 3, .LCPI0_0@l
+; CHECK-NEXT:    lis 4, .LCPI0_0@ha
+; CHECK-NEXT:    lvx 3, 4, 3
+; CHECK-NEXT:    vxor 4, 4, 4
+; CHECK-NEXT:    li 3, c@l
+; CHECK-NEXT:    lis 4, c@ha
+; CHECK-NEXT:    vperm 2, 4, 2, 3
+; CHECK-NEXT:    stvx 2, 4, 3
+; CHECK-NEXT:    blr
 entry:
     %tmp0 = load <16 x i8>, <16 x i8>* @a, align 16
   %tmp180.i = extractelement <16 x i8> %tmp0, i32 0 ; <i8> [#uses=1]
@@ -33,5 +46,4 @@ entry:
 
 return:		; preds = %bb2
 	ret void
-; CHECK: blr
 }
diff --git a/llvm/test/CodeGen/PowerPC/vec_conv_fp32_to_i64_elts.ll b/llvm/test/CodeGen/PowerPC/vec_conv_fp32_to_i64_elts.ll
index 6b945d468..cf4a6d636 100644
--- a/llvm/test/CodeGen/PowerPC/vec_conv_fp32_to_i64_elts.ll
+++ b/llvm/test/CodeGen/PowerPC/vec_conv_fp32_to_i64_elts.ll
@@ -328,27 +328,27 @@ define void @test16elt(<16 x i64>* noalias nocapture sret %agg.result, <16 x flo
 ; CHECK-P9-NEXT:    lxv vs2, 48(r4)
 ; CHECK-P9-NEXT:    xxswapd vs8, vs2
 ; CHECK-P9-NEXT:    xscvspdpn f8, vs8
-; CHECK-P9-NEXT:    xvcvdpuxds vs0, vs0
-; CHECK-P9-NEXT:    stxv vs5, 32(r3)
-; CHECK-P9-NEXT:    xvcvdpuxds vs6, vs6
+; CHECK-P9-NEXT:    xvcvdpuxds vs4, vs4
 ; CHECK-P9-NEXT:    xscvspdpn f3, vs3
 ; CHECK-P9-NEXT:    xxmrghd vs3, vs7, vs3
 ; CHECK-P9-NEXT:    xxsldwi vs7, vs2, vs2, 3
+; CHECK-P9-NEXT:    xvcvdpuxds vs0, vs0
+; CHECK-P9-NEXT:    xvcvdpuxds vs6, vs6
+; CHECK-P9-NEXT:    stxv vs6, 64(r3)
 ; CHECK-P9-NEXT:    xscvspdpn f7, vs7
 ; CHECK-P9-NEXT:    xxmrghd vs7, vs8, vs7
 ; CHECK-P9-NEXT:    xscvspdpn f8, vs2
 ; CHECK-P9-NEXT:    xxsldwi vs2, vs2, vs2, 1
-; CHECK-P9-NEXT:    stxv vs6, 64(r3)
-; CHECK-P9-NEXT:    xvcvdpuxds vs4, vs4
+; CHECK-P9-NEXT:    xscvspdpn f2, vs2
+; CHECK-P9-NEXT:    xxmrghd vs2, vs8, vs2
 ; CHECK-P9-NEXT:    xvcvdpuxds vs3, vs3
 ; CHECK-P9-NEXT:    xvcvdpuxds vs7, vs7
-; CHECK-P9-NEXT:    xscvspdpn f2, vs2
 ; CHECK-P9-NEXT:    stxv vs3, 80(r3)
-; CHECK-P9-NEXT:    xxmrghd vs2, vs8, vs2
 ; CHECK-P9-NEXT:    xvcvdpuxds vs2, vs2
-; CHECK-P9-NEXT:    stxv vs2, 112(r3)
 ; CHECK-P9-NEXT:    stxv vs7, 96(r3)
+; CHECK-P9-NEXT:    stxv vs2, 112(r3)
 ; CHECK-P9-NEXT:    stxv vs4, 48(r3)
+; CHECK-P9-NEXT:    stxv vs5, 32(r3)
 ; CHECK-P9-NEXT:    stxv vs0, 16(r3)
 ; CHECK-P9-NEXT:    stxv vs1, 0(r3)
 ; CHECK-P9-NEXT:    blr
@@ -738,27 +738,27 @@ define void @test16elt_signed(<16 x i64>* noalias nocapture sret %agg.result, <1
 ; CHECK-P9-NEXT:    lxv vs2, 48(r4)
 ; CHECK-P9-NEXT:    xxswapd vs8, vs2
 ; CHECK-P9-NEXT:    xscvspdpn f8, vs8
-; CHECK-P9-NEXT:    xvcvdpuxds vs0, vs0
-; CHECK-P9-NEXT:    stxv vs5, 32(r3)
-; CHECK-P9-NEXT:    xvcvdpuxds vs6, vs6
+; CHECK-P9-NEXT:    xvcvdpuxds vs4, vs4
 ; CHECK-P9-NEXT:    xscvspdpn f3, vs3
 ; CHECK-P9-NEXT:    xxmrghd vs3, vs7, vs3
 ; CHECK-P9-NEXT:    xxsldwi vs7, vs2, vs2, 3
+; CHECK-P9-NEXT:    xvcvdpuxds vs0, vs0
+; CHECK-P9-NEXT:    xvcvdpuxds vs6, vs6
+; CHECK-P9-NEXT:    stxv vs6, 64(r3)
 ; CHECK-P9-NEXT:    xscvspdpn f7, vs7
 ; CHECK-P9-NEXT:    xxmrghd vs7, vs8, vs7
 ; CHECK-P9-NEXT:    xscvspdpn f8, vs2
 ; CHECK-P9-NEXT:    xxsldwi vs2, vs2, vs2, 1
-; CHECK-P9-NEXT:    stxv vs6, 64(r3)
-; CHECK-P9-NEXT:    xvcvdpuxds vs4, vs4
+; CHECK-P9-NEXT:    xscvspdpn f2, vs2
+; CHECK-P9-NEXT:    xxmrghd vs2, vs8, vs2
 ; CHECK-P9-NEXT:    xvcvdpuxds vs3, vs3
 ; CHECK-P9-NEXT:    xvcvdpuxds vs7, vs7
-; CHECK-P9-NEXT:    xscvspdpn f2, vs2
 ; CHECK-P9-NEXT:    stxv vs3, 80(r3)
-; CHECK-P9-NEXT:    xxmrghd vs2, vs8, vs2
 ; CHECK-P9-NEXT:    xvcvdpuxds vs2, vs2
-; CHECK-P9-NEXT:    stxv vs2, 112(r3)
 ; CHECK-P9-NEXT:    stxv vs7, 96(r3)
+; CHECK-P9-NEXT:    stxv vs2, 112(r3)
 ; CHECK-P9-NEXT:    stxv vs4, 48(r3)
+; CHECK-P9-NEXT:    stxv vs5, 32(r3)
 ; CHECK-P9-NEXT:    stxv vs0, 16(r3)
 ; CHECK-P9-NEXT:    stxv vs1, 0(r3)
 ; CHECK-P9-NEXT:    blr
diff --git a/llvm/test/CodeGen/PowerPC/vec_conv_fp64_to_i32_elts.ll b/llvm/test/CodeGen/PowerPC/vec_conv_fp64_to_i32_elts.ll
index f3431f862..73ae5174e 100644
--- a/llvm/test/CodeGen/PowerPC/vec_conv_fp64_to_i32_elts.ll
+++ b/llvm/test/CodeGen/PowerPC/vec_conv_fp64_to_i32_elts.ll
@@ -29,11 +29,11 @@ define i64 @test2elt(<2 x double> %a) local_unnamed_addr #0 {
 ; CHECK-P9-LABEL: test2elt:
 ; CHECK-P9:       # %bb.0: # %entry
 ; CHECK-P9-NEXT:    xscvdpuxws f0, v2
-; CHECK-P9-NEXT:    mfvsrwz r3, f0
+; CHECK-P9-NEXT:    mffprwz r3, f0
 ; CHECK-P9-NEXT:    xxswapd vs0, v2
 ; CHECK-P9-NEXT:    mtvsrws v3, r3
 ; CHECK-P9-NEXT:    xscvdpuxws f0, f0
-; CHECK-P9-NEXT:    mfvsrwz r3, f0
+; CHECK-P9-NEXT:    mffprwz r3, f0
 ; CHECK-P9-NEXT:    mtvsrws v2, r3
 ; CHECK-P9-NEXT:    vmrglw v2, v3, v2
 ; CHECK-P9-NEXT:    mfvsrld r3, v2
@@ -42,11 +42,11 @@ define i64 @test2elt(<2 x double> %a) local_unnamed_addr #0 {
 ; CHECK-BE-LABEL: test2elt:
 ; CHECK-BE:       # %bb.0: # %entry
 ; CHECK-BE-NEXT:    xscvdpuxws f0, v2
-; CHECK-BE-NEXT:    mfvsrwz r3, f0
+; CHECK-BE-NEXT:    mffprwz r3, f0
 ; CHECK-BE-NEXT:    xxswapd vs0, v2
 ; CHECK-BE-NEXT:    mtvsrws v3, r3
 ; CHECK-BE-NEXT:    xscvdpuxws f0, f0
-; CHECK-BE-NEXT:    mfvsrwz r3, f0
+; CHECK-BE-NEXT:    mffprwz r3, f0
 ; CHECK-BE-NEXT:    mtvsrws v2, r3
 ; CHECK-BE-NEXT:    vmrghw v2, v3, v2
 ; CHECK-BE-NEXT:    mfvsrd r3, v2
@@ -323,11 +323,11 @@ define i64 @test2elt_signed(<2 x double> %a) local_unnamed_addr #0 {
 ; CHECK-P9-LABEL: test2elt_signed:
 ; CHECK-P9:       # %bb.0: # %entry
 ; CHECK-P9-NEXT:    xscvdpsxws f0, v2
-; CHECK-P9-NEXT:    mfvsrwz r3, f0
+; CHECK-P9-NEXT:    mffprwz r3, f0
 ; CHECK-P9-NEXT:    xxswapd vs0, v2
 ; CHECK-P9-NEXT:    mtvsrws v3, r3
 ; CHECK-P9-NEXT:    xscvdpsxws f0, f0
-; CHECK-P9-NEXT:    mfvsrwz r3, f0
+; CHECK-P9-NEXT:    mffprwz r3, f0
 ; CHECK-P9-NEXT:    mtvsrws v2, r3
 ; CHECK-P9-NEXT:    vmrglw v2, v3, v2
 ; CHECK-P9-NEXT:    mfvsrld r3, v2
@@ -336,11 +336,11 @@ define i64 @test2elt_signed(<2 x double> %a) local_unnamed_addr #0 {
 ; CHECK-BE-LABEL: test2elt_signed:
 ; CHECK-BE:       # %bb.0: # %entry
 ; CHECK-BE-NEXT:    xscvdpsxws f0, v2
-; CHECK-BE-NEXT:    mfvsrwz r3, f0
+; CHECK-BE-NEXT:    mffprwz r3, f0
 ; CHECK-BE-NEXT:    xxswapd vs0, v2
 ; CHECK-BE-NEXT:    mtvsrws v3, r3
 ; CHECK-BE-NEXT:    xscvdpsxws f0, f0
-; CHECK-BE-NEXT:    mfvsrwz r3, f0
+; CHECK-BE-NEXT:    mffprwz r3, f0
 ; CHECK-BE-NEXT:    mtvsrws v2, r3
 ; CHECK-BE-NEXT:    vmrghw v2, v3, v2
 ; CHECK-BE-NEXT:    mfvsrd r3, v2
diff --git a/llvm/test/CodeGen/PowerPC/vec_conv_i16_to_fp32_elts.ll b/llvm/test/CodeGen/PowerPC/vec_conv_i16_to_fp32_elts.ll
index f8538ac4c..2179db537 100644
--- a/llvm/test/CodeGen/PowerPC/vec_conv_i16_to_fp32_elts.ll
+++ b/llvm/test/CodeGen/PowerPC/vec_conv_i16_to_fp32_elts.ll
@@ -18,8 +18,8 @@ define i64 @test2elt(i32 %a.coerce) local_unnamed_addr #0 {
 ; CHECK-P8-NEXT:    rldicl r3, r3, 48, 48
 ; CHECK-P8-NEXT:    rlwinm r4, r4, 0, 16, 31
 ; CHECK-P8-NEXT:    rlwinm r3, r3, 0, 16, 31
-; CHECK-P8-NEXT:    mtvsrwz f0, r4
-; CHECK-P8-NEXT:    mtvsrwz f1, r3
+; CHECK-P8-NEXT:    mtfprwz f0, r4
+; CHECK-P8-NEXT:    mtfprwz f1, r3
 ; CHECK-P8-NEXT:    xscvuxdsp f0, f0
 ; CHECK-P8-NEXT:    xscvuxdsp f1, f1
 ; CHECK-P8-NEXT:    xscvdpspn vs0, f0
@@ -37,14 +37,14 @@ define i64 @test2elt(i32 %a.coerce) local_unnamed_addr #0 {
 ; CHECK-P9-NEXT:    li r3, 0
 ; CHECK-P9-NEXT:    vextuhrx r3, r3, v2
 ; CHECK-P9-NEXT:    rlwinm r3, r3, 0, 16, 31
-; CHECK-P9-NEXT:    mtvsrwz f0, r3
+; CHECK-P9-NEXT:    mtfprwz f0, r3
 ; CHECK-P9-NEXT:    li r3, 2
 ; CHECK-P9-NEXT:    xscvuxdsp f0, f0
 ; CHECK-P9-NEXT:    xscvdpspn vs0, f0
 ; CHECK-P9-NEXT:    vextuhrx r3, r3, v2
 ; CHECK-P9-NEXT:    rlwinm r3, r3, 0, 16, 31
 ; CHECK-P9-NEXT:    xxsldwi v3, vs0, vs0, 1
-; CHECK-P9-NEXT:    mtvsrwz f0, r3
+; CHECK-P9-NEXT:    mtfprwz f0, r3
 ; CHECK-P9-NEXT:    xscvuxdsp f0, f0
 ; CHECK-P9-NEXT:    xscvdpspn vs0, f0
 ; CHECK-P9-NEXT:    xxsldwi v2, vs0, vs0, 1
@@ -58,13 +58,13 @@ define i64 @test2elt(i32 %a.coerce) local_unnamed_addr #0 {
 ; CHECK-BE-NEXT:    li r3, 2
 ; CHECK-BE-NEXT:    vextuhlx r3, r3, v2
 ; CHECK-BE-NEXT:    rlwinm r3, r3, 0, 16, 31
-; CHECK-BE-NEXT:    mtvsrwz f0, r3
+; CHECK-BE-NEXT:    mtfprwz f0, r3
 ; CHECK-BE-NEXT:    li r3, 0
 ; CHECK-BE-NEXT:    xscvuxdsp f0, f0
 ; CHECK-BE-NEXT:    vextuhlx r3, r3, v2
 ; CHECK-BE-NEXT:    rlwinm r3, r3, 0, 16, 31
 ; CHECK-BE-NEXT:    xscvdpspn v3, f0
-; CHECK-BE-NEXT:    mtvsrwz f0, r3
+; CHECK-BE-NEXT:    mtfprwz f0, r3
 ; CHECK-BE-NEXT:    xscvuxdsp f0, f0
 ; CHECK-BE-NEXT:    xscvdpspn v2, f0
 ; CHECK-BE-NEXT:    vmrghw v2, v2, v3
@@ -270,8 +270,8 @@ define i64 @test2elt_signed(i32 %a.coerce) local_unnamed_addr #0 {
 ; CHECK-P8-NEXT:    rldicl r3, r3, 48, 48
 ; CHECK-P8-NEXT:    extsh r4, r4
 ; CHECK-P8-NEXT:    extsh r3, r3
-; CHECK-P8-NEXT:    mtvsrwa f0, r4
-; CHECK-P8-NEXT:    mtvsrwa f1, r3
+; CHECK-P8-NEXT:    mtfprwa f0, r4
+; CHECK-P8-NEXT:    mtfprwa f1, r3
 ; CHECK-P8-NEXT:    xscvsxdsp f0, f0
 ; CHECK-P8-NEXT:    xscvsxdsp f1, f1
 ; CHECK-P8-NEXT:    xscvdpspn vs0, f0
@@ -289,14 +289,14 @@ define i64 @test2elt_signed(i32 %a.coerce) local_unnamed_addr #0 {
 ; CHECK-P9-NEXT:    li r3, 0
 ; CHECK-P9-NEXT:    vextuhrx r3, r3, v2
 ; CHECK-P9-NEXT:    extsh r3, r3
-; CHECK-P9-NEXT:    mtvsrwa f0, r3
+; CHECK-P9-NEXT:    mtfprwa f0, r3
 ; CHECK-P9-NEXT:    li r3, 2
 ; CHECK-P9-NEXT:    xscvsxdsp f0, f0
 ; CHECK-P9-NEXT:    xscvdpspn vs0, f0
 ; CHECK-P9-NEXT:    vextuhrx r3, r3, v2
 ; CHECK-P9-NEXT:    extsh r3, r3
 ; CHECK-P9-NEXT:    xxsldwi v3, vs0, vs0, 1
-; CHECK-P9-NEXT:    mtvsrwa f0, r3
+; CHECK-P9-NEXT:    mtfprwa f0, r3
 ; CHECK-P9-NEXT:    xscvsxdsp f0, f0
 ; CHECK-P9-NEXT:    xscvdpspn vs0, f0
 ; CHECK-P9-NEXT:    xxsldwi v2, vs0, vs0, 1
@@ -310,13 +310,13 @@ define i64 @test2elt_signed(i32 %a.coerce) local_unnamed_addr #0 {
 ; CHECK-BE-NEXT:    li r3, 2
 ; CHECK-BE-NEXT:    vextuhlx r3, r3, v2
 ; CHECK-BE-NEXT:    extsh r3, r3
-; CHECK-BE-NEXT:    mtvsrwa f0, r3
+; CHECK-BE-NEXT:    mtfprwa f0, r3
 ; CHECK-BE-NEXT:    li r3, 0
 ; CHECK-BE-NEXT:    xscvsxdsp f0, f0
 ; CHECK-BE-NEXT:    vextuhlx r3, r3, v2
 ; CHECK-BE-NEXT:    extsh r3, r3
 ; CHECK-BE-NEXT:    xscvdpspn v3, f0
-; CHECK-BE-NEXT:    mtvsrwa f0, r3
+; CHECK-BE-NEXT:    mtfprwa f0, r3
 ; CHECK-BE-NEXT:    xscvsxdsp f0, f0
 ; CHECK-BE-NEXT:    xscvdpspn v2, f0
 ; CHECK-BE-NEXT:    vmrghw v2, v2, v3
diff --git a/llvm/test/CodeGen/PowerPC/vec_conv_i16_to_fp64_elts.ll b/llvm/test/CodeGen/PowerPC/vec_conv_i16_to_fp64_elts.ll
index f03d0696f..883cf7e51 100644
--- a/llvm/test/CodeGen/PowerPC/vec_conv_i16_to_fp64_elts.ll
+++ b/llvm/test/CodeGen/PowerPC/vec_conv_i16_to_fp64_elts.ll
@@ -712,24 +712,24 @@ define void @test16elt_signed(<16 x double>* noalias nocapture sret %agg.result,
 ; CHECK-P9-NEXT:    vextsh2d v2, v2
 ; CHECK-P9-NEXT:    xvcvsxddp vs3, v2
 ; CHECK-P9-NEXT:    vperm v2, v4, v4, v3
+; CHECK-P9-NEXT:    stxv vs2, 32(r3)
 ; CHECK-P9-NEXT:    vextsh2d v2, v2
 ; CHECK-P9-NEXT:    stxv vs3, 48(r3)
+; CHECK-P9-NEXT:    stxv vs1, 16(r3)
 ; CHECK-P9-NEXT:    xvcvsxddp vs4, v2
 ; CHECK-P9-NEXT:    vperm v2, v4, v4, v5
 ; CHECK-P9-NEXT:    vextsh2d v2, v2
 ; CHECK-P9-NEXT:    xvcvsxddp vs5, v2
 ; CHECK-P9-NEXT:    vperm v2, v4, v4, v0
 ; CHECK-P9-NEXT:    stxv vs4, 64(r3)
-; CHECK-P9-NEXT:    stxv vs5, 80(r3)
 ; CHECK-P9-NEXT:    vextsh2d v2, v2
 ; CHECK-P9-NEXT:    xvcvsxddp vs6, v2
 ; CHECK-P9-NEXT:    vperm v2, v4, v4, v1
-; CHECK-P9-NEXT:    vextsh2d v2, v2
+; CHECK-P9-NEXT:    stxv vs5, 80(r3)
 ; CHECK-P9-NEXT:    stxv vs6, 96(r3)
+; CHECK-P9-NEXT:    vextsh2d v2, v2
 ; CHECK-P9-NEXT:    xvcvsxddp vs7, v2
 ; CHECK-P9-NEXT:    stxv vs7, 112(r3)
-; CHECK-P9-NEXT:    stxv vs2, 32(r3)
-; CHECK-P9-NEXT:    stxv vs1, 16(r3)
 ; CHECK-P9-NEXT:    stxv vs0, 0(r3)
 ; CHECK-P9-NEXT:    blr
 ;
diff --git a/llvm/test/CodeGen/PowerPC/vec_conv_i64_to_fp32_elts.ll b/llvm/test/CodeGen/PowerPC/vec_conv_i64_to_fp32_elts.ll
index 08e6f70bb..66e85e9f8 100644
--- a/llvm/test/CodeGen/PowerPC/vec_conv_i64_to_fp32_elts.ll
+++ b/llvm/test/CodeGen/PowerPC/vec_conv_i64_to_fp32_elts.ll
@@ -239,18 +239,18 @@ define void @test16elt(<16 x float>* noalias nocapture sret %agg.result, <16 x i
 ; CHECK-P9-NEXT:    xxsldwi v6, vs0, vs0, 3
 ; CHECK-P9-NEXT:    xvcvuxdsp vs0, v0
 ; CHECK-P9-NEXT:    lxv v5, 64(r4)
+; CHECK-P9-NEXT:    stxv v1, 0(r3)
 ; CHECK-P9-NEXT:    xxsldwi v0, vs0, vs0, 3
 ; CHECK-P9-NEXT:    xvcvuxdsp vs0, v5
 ; CHECK-P9-NEXT:    lxv v4, 80(r4)
 ; CHECK-P9-NEXT:    vpkudum v0, v0, v6
+; CHECK-P9-NEXT:    stxv v0, 16(r3)
 ; CHECK-P9-NEXT:    xxsldwi v5, vs0, vs0, 3
-; CHECK-P9-NEXT:    lxv v3, 96(r4)
 ; CHECK-P9-NEXT:    xvcvuxdsp vs0, v4
+; CHECK-P9-NEXT:    lxv v3, 96(r4)
 ; CHECK-P9-NEXT:    xxsldwi v4, vs0, vs0, 3
 ; CHECK-P9-NEXT:    xvcvuxdsp vs0, v3
 ; CHECK-P9-NEXT:    lxv v2, 112(r4)
-; CHECK-P9-NEXT:    stxv v0, 16(r3)
-; CHECK-P9-NEXT:    stxv v1, 0(r3)
 ; CHECK-P9-NEXT:    vpkudum v4, v4, v5
 ; CHECK-P9-NEXT:    stxv v4, 32(r3)
 ; CHECK-P9-NEXT:    xxsldwi v3, vs0, vs0, 3
@@ -275,18 +275,18 @@ define void @test16elt(<16 x float>* noalias nocapture sret %agg.result, <16 x i
 ; CHECK-BE-NEXT:    xxsldwi v6, vs0, vs0, 3
 ; CHECK-BE-NEXT:    xvcvuxdsp vs0, v0
 ; CHECK-BE-NEXT:    lxv v5, 80(r4)
+; CHECK-BE-NEXT:    stxv v1, 0(r3)
 ; CHECK-BE-NEXT:    xxsldwi v0, vs0, vs0, 3
 ; CHECK-BE-NEXT:    xvcvuxdsp vs0, v5
 ; CHECK-BE-NEXT:    lxv v4, 64(r4)
 ; CHECK-BE-NEXT:    vpkudum v0, v0, v6
+; CHECK-BE-NEXT:    stxv v0, 16(r3)
 ; CHECK-BE-NEXT:    xxsldwi v5, vs0, vs0, 3
-; CHECK-BE-NEXT:    lxv v3, 112(r4)
 ; CHECK-BE-NEXT:    xvcvuxdsp vs0, v4
+; CHECK-BE-NEXT:    lxv v3, 112(r4)
 ; CHECK-BE-NEXT:    xxsldwi v4, vs0, vs0, 3
 ; CHECK-BE-NEXT:    xvcvuxdsp vs0, v3
 ; CHECK-BE-NEXT:    lxv v2, 96(r4)
-; CHECK-BE-NEXT:    stxv v0, 16(r3)
-; CHECK-BE-NEXT:    stxv v1, 0(r3)
 ; CHECK-BE-NEXT:    vpkudum v4, v4, v5
 ; CHECK-BE-NEXT:    stxv v4, 32(r3)
 ; CHECK-BE-NEXT:    xxsldwi v3, vs0, vs0, 3
@@ -532,18 +532,18 @@ define void @test16elt_signed(<16 x float>* noalias nocapture sret %agg.result,
 ; CHECK-P9-NEXT:    xxsldwi v6, vs0, vs0, 3
 ; CHECK-P9-NEXT:    xvcvsxdsp vs0, v0
 ; CHECK-P9-NEXT:    lxv v5, 64(r4)
+; CHECK-P9-NEXT:    stxv v1, 0(r3)
 ; CHECK-P9-NEXT:    xxsldwi v0, vs0, vs0, 3
 ; CHECK-P9-NEXT:    xvcvsxdsp vs0, v5
 ; CHECK-P9-NEXT:    lxv v4, 80(r4)
 ; CHECK-P9-NEXT:    vpkudum v0, v0, v6
+; CHECK-P9-NEXT:    stxv v0, 16(r3)
 ; CHECK-P9-NEXT:    xxsldwi v5, vs0, vs0, 3
-; CHECK-P9-NEXT:    lxv v3, 96(r4)
 ; CHECK-P9-NEXT:    xvcvsxdsp vs0, v4
+; CHECK-P9-NEXT:    lxv v3, 96(r4)
 ; CHECK-P9-NEXT:    xxsldwi v4, vs0, vs0, 3
 ; CHECK-P9-NEXT:    xvcvsxdsp vs0, v3
 ; CHECK-P9-NEXT:    lxv v2, 112(r4)
-; CHECK-P9-NEXT:    stxv v0, 16(r3)
-; CHECK-P9-NEXT:    stxv v1, 0(r3)
 ; CHECK-P9-NEXT:    vpkudum v4, v4, v5
 ; CHECK-P9-NEXT:    stxv v4, 32(r3)
 ; CHECK-P9-NEXT:    xxsldwi v3, vs0, vs0, 3
@@ -568,18 +568,18 @@ define void @test16elt_signed(<16 x float>* noalias nocapture sret %agg.result,
 ; CHECK-BE-NEXT:    xxsldwi v6, vs0, vs0, 3
 ; CHECK-BE-NEXT:    xvcvsxdsp vs0, v0
 ; CHECK-BE-NEXT:    lxv v5, 80(r4)
+; CHECK-BE-NEXT:    stxv v1, 0(r3)
 ; CHECK-BE-NEXT:    xxsldwi v0, vs0, vs0, 3
 ; CHECK-BE-NEXT:    xvcvsxdsp vs0, v5
 ; CHECK-BE-NEXT:    lxv v4, 64(r4)
 ; CHECK-BE-NEXT:    vpkudum v0, v0, v6
+; CHECK-BE-NEXT:    stxv v0, 16(r3)
 ; CHECK-BE-NEXT:    xxsldwi v5, vs0, vs0, 3
-; CHECK-BE-NEXT:    lxv v3, 112(r4)
 ; CHECK-BE-NEXT:    xvcvsxdsp vs0, v4
+; CHECK-BE-NEXT:    lxv v3, 112(r4)
 ; CHECK-BE-NEXT:    xxsldwi v4, vs0, vs0, 3
 ; CHECK-BE-NEXT:    xvcvsxdsp vs0, v3
 ; CHECK-BE-NEXT:    lxv v2, 96(r4)
-; CHECK-BE-NEXT:    stxv v0, 16(r3)
-; CHECK-BE-NEXT:    stxv v1, 0(r3)
 ; CHECK-BE-NEXT:    vpkudum v4, v4, v5
 ; CHECK-BE-NEXT:    stxv v4, 32(r3)
 ; CHECK-BE-NEXT:    xxsldwi v3, vs0, vs0, 3
diff --git a/llvm/test/CodeGen/PowerPC/vec_conv_i8_to_fp32_elts.ll b/llvm/test/CodeGen/PowerPC/vec_conv_i8_to_fp32_elts.ll
index 432598fb8..a34dfb7b0 100644
--- a/llvm/test/CodeGen/PowerPC/vec_conv_i8_to_fp32_elts.ll
+++ b/llvm/test/CodeGen/PowerPC/vec_conv_i8_to_fp32_elts.ll
@@ -18,8 +18,8 @@ define i64 @test2elt(i16 %a.coerce) local_unnamed_addr #0 {
 ; CHECK-P8-NEXT:    rldicl r3, r3, 56, 56
 ; CHECK-P8-NEXT:    rlwinm r4, r4, 0, 24, 31
 ; CHECK-P8-NEXT:    rlwinm r3, r3, 0, 24, 31
-; CHECK-P8-NEXT:    mtvsrwz f0, r4
-; CHECK-P8-NEXT:    mtvsrwz f1, r3
+; CHECK-P8-NEXT:    mtfprwz f0, r4
+; CHECK-P8-NEXT:    mtfprwz f1, r3
 ; CHECK-P8-NEXT:    xscvuxdsp f0, f0
 ; CHECK-P8-NEXT:    xscvuxdsp f1, f1
 ; CHECK-P8-NEXT:    xscvdpspn vs0, f0
@@ -37,14 +37,14 @@ define i64 @test2elt(i16 %a.coerce) local_unnamed_addr #0 {
 ; CHECK-P9-NEXT:    li r3, 0
 ; CHECK-P9-NEXT:    vextubrx r3, r3, v2
 ; CHECK-P9-NEXT:    rlwinm r3, r3, 0, 24, 31
-; CHECK-P9-NEXT:    mtvsrwz f0, r3
+; CHECK-P9-NEXT:    mtfprwz f0, r3
 ; CHECK-P9-NEXT:    li r3, 1
 ; CHECK-P9-NEXT:    xscvuxdsp f0, f0
 ; CHECK-P9-NEXT:    xscvdpspn vs0, f0
 ; CHECK-P9-NEXT:    vextubrx r3, r3, v2
 ; CHECK-P9-NEXT:    rlwinm r3, r3, 0, 24, 31
 ; CHECK-P9-NEXT:    xxsldwi v3, vs0, vs0, 1
-; CHECK-P9-NEXT:    mtvsrwz f0, r3
+; CHECK-P9-NEXT:    mtfprwz f0, r3
 ; CHECK-P9-NEXT:    xscvuxdsp f0, f0
 ; CHECK-P9-NEXT:    xscvdpspn vs0, f0
 ; CHECK-P9-NEXT:    xxsldwi v2, vs0, vs0, 1
@@ -58,13 +58,13 @@ define i64 @test2elt(i16 %a.coerce) local_unnamed_addr #0 {
 ; CHECK-BE-NEXT:    li r3, 1
 ; CHECK-BE-NEXT:    vextublx r3, r3, v2
 ; CHECK-BE-NEXT:    rlwinm r3, r3, 0, 24, 31
-; CHECK-BE-NEXT:    mtvsrwz f0, r3
+; CHECK-BE-NEXT:    mtfprwz f0, r3
 ; CHECK-BE-NEXT:    li r3, 0
 ; CHECK-BE-NEXT:    xscvuxdsp f0, f0
 ; CHECK-BE-NEXT:    vextublx r3, r3, v2
 ; CHECK-BE-NEXT:    rlwinm r3, r3, 0, 24, 31
 ; CHECK-BE-NEXT:    xscvdpspn v3, f0
-; CHECK-BE-NEXT:    mtvsrwz f0, r3
+; CHECK-BE-NEXT:    mtfprwz f0, r3
 ; CHECK-BE-NEXT:    xscvuxdsp f0, f0
 ; CHECK-BE-NEXT:    xscvdpspn v2, f0
 ; CHECK-BE-NEXT:    vmrghw v2, v2, v3
@@ -286,8 +286,8 @@ define i64 @test2elt_signed(i16 %a.coerce) local_unnamed_addr #0 {
 ; CHECK-P8-NEXT:    rldicl r3, r3, 56, 56
 ; CHECK-P8-NEXT:    extsb r4, r4
 ; CHECK-P8-NEXT:    extsb r3, r3
-; CHECK-P8-NEXT:    mtvsrwa f0, r4
-; CHECK-P8-NEXT:    mtvsrwa f1, r3
+; CHECK-P8-NEXT:    mtfprwa f0, r4
+; CHECK-P8-NEXT:    mtfprwa f1, r3
 ; CHECK-P8-NEXT:    xscvsxdsp f0, f0
 ; CHECK-P8-NEXT:    xscvsxdsp f1, f1
 ; CHECK-P8-NEXT:    xscvdpspn vs0, f0
@@ -305,14 +305,14 @@ define i64 @test2elt_signed(i16 %a.coerce) local_unnamed_addr #0 {
 ; CHECK-P9-NEXT:    li r3, 0
 ; CHECK-P9-NEXT:    vextubrx r3, r3, v2
 ; CHECK-P9-NEXT:    extsb r3, r3
-; CHECK-P9-NEXT:    mtvsrwa f0, r3
+; CHECK-P9-NEXT:    mtfprwa f0, r3
 ; CHECK-P9-NEXT:    li r3, 1
 ; CHECK-P9-NEXT:    xscvsxdsp f0, f0
 ; CHECK-P9-NEXT:    xscvdpspn vs0, f0
 ; CHECK-P9-NEXT:    vextubrx r3, r3, v2
 ; CHECK-P9-NEXT:    extsb r3, r3
 ; CHECK-P9-NEXT:    xxsldwi v3, vs0, vs0, 1
-; CHECK-P9-NEXT:    mtvsrwa f0, r3
+; CHECK-P9-NEXT:    mtfprwa f0, r3
 ; CHECK-P9-NEXT:    xscvsxdsp f0, f0
 ; CHECK-P9-NEXT:    xscvdpspn vs0, f0
 ; CHECK-P9-NEXT:    xxsldwi v2, vs0, vs0, 1
@@ -326,13 +326,13 @@ define i64 @test2elt_signed(i16 %a.coerce) local_unnamed_addr #0 {
 ; CHECK-BE-NEXT:    li r3, 1
 ; CHECK-BE-NEXT:    vextublx r3, r3, v2
 ; CHECK-BE-NEXT:    extsb r3, r3
-; CHECK-BE-NEXT:    mtvsrwa f0, r3
+; CHECK-BE-NEXT:    mtfprwa f0, r3
 ; CHECK-BE-NEXT:    li r3, 0
 ; CHECK-BE-NEXT:    xscvsxdsp f0, f0
 ; CHECK-BE-NEXT:    vextublx r3, r3, v2
 ; CHECK-BE-NEXT:    extsb r3, r3
 ; CHECK-BE-NEXT:    xscvdpspn v3, f0
-; CHECK-BE-NEXT:    mtvsrwa f0, r3
+; CHECK-BE-NEXT:    mtfprwa f0, r3
 ; CHECK-BE-NEXT:    xscvsxdsp f0, f0
 ; CHECK-BE-NEXT:    xscvdpspn v2, f0
 ; CHECK-BE-NEXT:    vmrghw v2, v2, v3
diff --git a/llvm/test/CodeGen/PowerPC/vec_revb.ll b/llvm/test/CodeGen/PowerPC/vec_revb.ll
index 00c08a120..644b4e1f5 100644
--- a/llvm/test/CodeGen/PowerPC/vec_revb.ll
+++ b/llvm/test/CodeGen/PowerPC/vec_revb.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr9 < %s | FileCheck %s
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -mcpu=pwr9 < %s | FileCheck %s
 
@@ -52,3 +53,19 @@ entry:
   %2 = bitcast <16 x i8> %1 to <1 x i128>
   ret <1 x i128> %2
 }
+
+define <4 x i32> @testXXBRD_With_LogicalOp(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: testXXBRD_With_LogicalOp:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxland 0, 34, 35
+; CHECK-NEXT:    xxbrw 34, 0
+; CHECK-NEXT:    blr
+entry:
+  %0 = bitcast <4 x i32> %a to <16 x i8>
+  %1 = shufflevector <16 x i8> %0, <16 x i8> undef, <16 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 11, i32 10, i32 9, i32 8, i32 15, i32 14, i32 13, i32 12>
+  %2 = bitcast <16 x i8> %1 to <4 x i32>
+  %3 = call <4 x i32> @llvm.bswap.v4i32(<4 x i32> %b)
+  %4 = and <4 x i32> %2, %3
+  ret <4 x i32> %4
+}
+declare <4 x i32> @llvm.bswap.v4i32(<4 x i32>)
diff --git a/llvm/test/CodeGen/PowerPC/vec_shuffle.ll b/llvm/test/CodeGen/PowerPC/vec_shuffle.ll
index 2ae98fcf0..40885f803 100644
--- a/llvm/test/CodeGen/PowerPC/vec_shuffle.ll
+++ b/llvm/test/CodeGen/PowerPC/vec_shuffle.ll
@@ -1,13 +1,16 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: opt -mtriple=powerpc-unknown-linux-gnu < %s -instcombine | \
 ; RUN:   llc -mtriple=ppc32-- -mcpu=g5 | not grep vperm
-; RUN: llc -verify-machineinstrs < %s -mtriple=ppc32-- -mcpu=g5 > %t
-; RUN: grep vsldoi  %t | count 2
-; RUN: grep vmrgh   %t | count 7
-; RUN: grep vmrgl   %t | count 6
-; RUN: grep vpkuhum %t | count 1
-; RUN: grep vpkuwum %t | count 1
+; RUN: llc -verify-machineinstrs < %s -mtriple=ppc32-- -mcpu=g5 | FileCheck %s
 
 define void @VSLDOI_xy(<8 x i16>* %A, <8 x i16>* %B) {
+; CHECK-LABEL: VSLDOI_xy:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lvx 2, 0, 3
+; CHECK-NEXT:    lvx 3, 0, 4
+; CHECK-NEXT:    vsldoi 2, 2, 3, 5
+; CHECK-NEXT:    stvx 2, 0, 3
+; CHECK-NEXT:    blr
 entry:
 	%tmp = load <8 x i16>, <8 x i16>* %A		; <<8 x i16>> [#uses=1]
 	%tmp2 = load <8 x i16>, <8 x i16>* %B		; <<8 x i16>> [#uses=1]
@@ -51,6 +54,12 @@ entry:
 }
 
 define void @VSLDOI_xx(<8 x i16>* %A, <8 x i16>* %B) {
+; CHECK-LABEL: VSLDOI_xx:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lvx 2, 0, 3
+; CHECK-NEXT:    vsldoi 2, 2, 2, 5
+; CHECK-NEXT:    stvx 2, 0, 3
+; CHECK-NEXT:    blr
 	%tmp = load <8 x i16>, <8 x i16>* %A		; <<8 x i16>> [#uses=1]
 	%tmp2 = load <8 x i16>, <8 x i16>* %A		; <<8 x i16>> [#uses=1]
 	%tmp.upgrd.5 = bitcast <8 x i16> %tmp to <16 x i8>		; <<16 x i8>> [#uses=11]
@@ -93,6 +102,14 @@ define void @VSLDOI_xx(<8 x i16>* %A, <8 x i16>* %B) {
 }
 
 define void @VPERM_promote(<8 x i16>* %A, <8 x i16>* %B) {
+; CHECK-LABEL: VPERM_promote:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lvx 2, 0, 3
+; CHECK-NEXT:    vspltisb 4, 14
+; CHECK-NEXT:    lvx 3, 0, 4
+; CHECK-NEXT:    vperm 2, 2, 3, 4
+; CHECK-NEXT:    stvx 2, 0, 3
+; CHECK-NEXT:    blr
 entry:
 	%tmp = load <8 x i16>, <8 x i16>* %A		; <<8 x i16>> [#uses=1]
 	%tmp.upgrd.9 = bitcast <8 x i16> %tmp to <4 x i32>		; <<4 x i32>> [#uses=1]
@@ -107,6 +124,13 @@ entry:
 declare <4 x i32> @llvm.ppc.altivec.vperm(<4 x i32>, <4 x i32>, <16 x i8>)
 
 define void @tb_l(<16 x i8>* %A, <16 x i8>* %B) {
+; CHECK-LABEL: tb_l:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lvx 2, 0, 3
+; CHECK-NEXT:    lvx 3, 0, 4
+; CHECK-NEXT:    vmrglb 2, 2, 3
+; CHECK-NEXT:    stvx 2, 0, 3
+; CHECK-NEXT:    blr
 entry:
 	%tmp = load <16 x i8>, <16 x i8>* %A		; <<16 x i8>> [#uses=8]
 	%tmp2 = load <16 x i8>, <16 x i8>* %B		; <<16 x i8>> [#uses=8]
@@ -147,6 +171,13 @@ entry:
 }
 
 define void @th_l(<8 x i16>* %A, <8 x i16>* %B) {
+; CHECK-LABEL: th_l:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lvx 2, 0, 3
+; CHECK-NEXT:    lvx 3, 0, 4
+; CHECK-NEXT:    vmrglh 2, 2, 3
+; CHECK-NEXT:    stvx 2, 0, 3
+; CHECK-NEXT:    blr
 entry:
 	%tmp = load <8 x i16>, <8 x i16>* %A		; <<8 x i16>> [#uses=4]
 	%tmp2 = load <8 x i16>, <8 x i16>* %B		; <<8 x i16>> [#uses=4]
@@ -171,6 +202,13 @@ entry:
 }
 
 define void @tw_l(<4 x i32>* %A, <4 x i32>* %B) {
+; CHECK-LABEL: tw_l:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lvx 2, 0, 3
+; CHECK-NEXT:    lvx 3, 0, 4
+; CHECK-NEXT:    vmrglw 2, 2, 3
+; CHECK-NEXT:    stvx 2, 0, 3
+; CHECK-NEXT:    blr
 entry:
 	%tmp = load <4 x i32>, <4 x i32>* %A		; <<4 x i32>> [#uses=2]
 	%tmp2 = load <4 x i32>, <4 x i32>* %B		; <<4 x i32>> [#uses=2]
@@ -187,6 +225,13 @@ entry:
 }
 
 define void @tb_h(<16 x i8>* %A, <16 x i8>* %B) {
+; CHECK-LABEL: tb_h:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lvx 2, 0, 3
+; CHECK-NEXT:    lvx 3, 0, 4
+; CHECK-NEXT:    vmrghb 2, 2, 3
+; CHECK-NEXT:    stvx 2, 0, 3
+; CHECK-NEXT:    blr
 entry:
 	%tmp = load <16 x i8>, <16 x i8>* %A		; <<16 x i8>> [#uses=8]
 	%tmp2 = load <16 x i8>, <16 x i8>* %B		; <<16 x i8>> [#uses=8]
@@ -227,6 +272,13 @@ entry:
 }
 
 define void @th_h(<8 x i16>* %A, <8 x i16>* %B) {
+; CHECK-LABEL: th_h:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lvx 2, 0, 3
+; CHECK-NEXT:    lvx 3, 0, 4
+; CHECK-NEXT:    vmrghh 2, 2, 3
+; CHECK-NEXT:    stvx 2, 0, 3
+; CHECK-NEXT:    blr
 entry:
 	%tmp = load <8 x i16>, <8 x i16>* %A		; <<8 x i16>> [#uses=4]
 	%tmp2 = load <8 x i16>, <8 x i16>* %B		; <<8 x i16>> [#uses=4]
@@ -251,6 +303,13 @@ entry:
 }
 
 define void @tw_h(<4 x i32>* %A, <4 x i32>* %B) {
+; CHECK-LABEL: tw_h:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lvx 2, 0, 3
+; CHECK-NEXT:    lvx 3, 0, 4
+; CHECK-NEXT:    vmrghw 2, 3, 2
+; CHECK-NEXT:    stvx 2, 0, 3
+; CHECK-NEXT:    blr
 entry:
 	%tmp = load <4 x i32>, <4 x i32>* %A		; <<4 x i32>> [#uses=2]
 	%tmp2 = load <4 x i32>, <4 x i32>* %B		; <<4 x i32>> [#uses=2]
@@ -267,6 +326,13 @@ entry:
 }
 
 define void @tw_h_flop(<4 x i32>* %A, <4 x i32>* %B) {
+; CHECK-LABEL: tw_h_flop:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lvx 2, 0, 3
+; CHECK-NEXT:    lvx 3, 0, 4
+; CHECK-NEXT:    vmrghw 2, 2, 3
+; CHECK-NEXT:    stvx 2, 0, 3
+; CHECK-NEXT:    blr
 	%tmp = load <4 x i32>, <4 x i32>* %A		; <<4 x i32>> [#uses=2]
 	%tmp2 = load <4 x i32>, <4 x i32>* %B		; <<4 x i32>> [#uses=2]
 	%tmp.upgrd.18 = extractelement <4 x i32> %tmp, i32 0		; <i32> [#uses=1]
@@ -282,6 +348,12 @@ define void @tw_h_flop(<4 x i32>* %A, <4 x i32>* %B) {
 }
 
 define void @VMRG_UNARY_tb_l(<16 x i8>* %A, <16 x i8>* %B) {
+; CHECK-LABEL: VMRG_UNARY_tb_l:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lvx 2, 0, 3
+; CHECK-NEXT:    vmrglb 2, 2, 2
+; CHECK-NEXT:    stvx 2, 0, 3
+; CHECK-NEXT:    blr
 entry:
 	%tmp = load <16 x i8>, <16 x i8>* %A		; <<16 x i8>> [#uses=16]
 	%tmp.upgrd.19 = extractelement <16 x i8> %tmp, i32 8		; <i8> [#uses=1]
@@ -321,6 +393,12 @@ entry:
 }
 
 define void @VMRG_UNARY_th_l(<8 x i16>* %A, <8 x i16>* %B) {
+; CHECK-LABEL: VMRG_UNARY_th_l:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lvx 2, 0, 3
+; CHECK-NEXT:    vmrglh 2, 2, 2
+; CHECK-NEXT:    stvx 2, 0, 3
+; CHECK-NEXT:    blr
 entry:
 	%tmp = load <8 x i16>, <8 x i16>* %A		; <<8 x i16>> [#uses=8]
 	%tmp.upgrd.20 = extractelement <8 x i16> %tmp, i32 4		; <i16> [#uses=1]
@@ -344,6 +422,12 @@ entry:
 }
 
 define void @VMRG_UNARY_tw_l(<4 x i32>* %A, <4 x i32>* %B) {
+; CHECK-LABEL: VMRG_UNARY_tw_l:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lvx 2, 0, 3
+; CHECK-NEXT:    vmrglw 2, 2, 2
+; CHECK-NEXT:    stvx 2, 0, 3
+; CHECK-NEXT:    blr
 entry:
 	%tmp = load <4 x i32>, <4 x i32>* %A		; <<4 x i32>> [#uses=4]
 	%tmp.upgrd.21 = extractelement <4 x i32> %tmp, i32 2		; <i32> [#uses=1]
@@ -359,6 +443,12 @@ entry:
 }
 
 define void @VMRG_UNARY_tb_h(<16 x i8>* %A, <16 x i8>* %B) {
+; CHECK-LABEL: VMRG_UNARY_tb_h:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lvx 2, 0, 3
+; CHECK-NEXT:    vmrghb 2, 2, 2
+; CHECK-NEXT:    stvx 2, 0, 3
+; CHECK-NEXT:    blr
 entry:
 	%tmp = load <16 x i8>, <16 x i8>* %A		; <<16 x i8>> [#uses=16]
 	%tmp.upgrd.22 = extractelement <16 x i8> %tmp, i32 0		; <i8> [#uses=1]
@@ -398,6 +488,12 @@ entry:
 }
 
 define void @VMRG_UNARY_th_h(<8 x i16>* %A, <8 x i16>* %B) {
+; CHECK-LABEL: VMRG_UNARY_th_h:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lvx 2, 0, 3
+; CHECK-NEXT:    vmrghh 2, 2, 2
+; CHECK-NEXT:    stvx 2, 0, 3
+; CHECK-NEXT:    blr
 entry:
 	%tmp = load <8 x i16>, <8 x i16>* %A		; <<8 x i16>> [#uses=8]
 	%tmp.upgrd.23 = extractelement <8 x i16> %tmp, i32 0		; <i16> [#uses=1]
@@ -421,6 +517,12 @@ entry:
 }
 
 define void @VMRG_UNARY_tw_h(<4 x i32>* %A, <4 x i32>* %B) {
+; CHECK-LABEL: VMRG_UNARY_tw_h:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lvx 2, 0, 3
+; CHECK-NEXT:    vmrghw 2, 2, 2
+; CHECK-NEXT:    stvx 2, 0, 3
+; CHECK-NEXT:    blr
 entry:
 	%tmp = load <4 x i32>, <4 x i32>* %A		; <<4 x i32>> [#uses=4]
 	%tmp.upgrd.24 = extractelement <4 x i32> %tmp, i32 0		; <i32> [#uses=1]
@@ -436,6 +538,12 @@ entry:
 }
 
 define void @VPCKUHUM_unary(<8 x i16>* %A, <8 x i16>* %B) {
+; CHECK-LABEL: VPCKUHUM_unary:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lvx 2, 0, 3
+; CHECK-NEXT:    vpkuhum 2, 2, 2
+; CHECK-NEXT:    stvx 2, 0, 3
+; CHECK-NEXT:    blr
 entry:
 	%tmp = load <8 x i16>, <8 x i16>* %A		; <<8 x i16>> [#uses=2]
 	%tmp.upgrd.25 = bitcast <8 x i16> %tmp to <16 x i8>		; <<16 x i8>> [#uses=8]
@@ -478,6 +586,12 @@ entry:
 }
 
 define void @VPCKUWUM_unary(<4 x i32>* %A, <4 x i32>* %B) {
+; CHECK-LABEL: VPCKUWUM_unary:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lvx 2, 0, 3
+; CHECK-NEXT:    vpkuwum 2, 2, 2
+; CHECK-NEXT:    stvx 2, 0, 3
+; CHECK-NEXT:    blr
 entry:
 	%tmp = load <4 x i32>, <4 x i32>* %A		; <<4 x i32>> [#uses=2]
 	%tmp.upgrd.28 = bitcast <4 x i32> %tmp to <8 x i16>		; <<8 x i16>> [#uses=4]
diff --git a/llvm/test/CodeGen/PowerPC/vec_shuffle_le.ll b/llvm/test/CodeGen/PowerPC/vec_shuffle_le.ll
index be79ea9fb..2d456d06b 100644
--- a/llvm/test/CodeGen/PowerPC/vec_shuffle_le.ll
+++ b/llvm/test/CodeGen/PowerPC/vec_shuffle_le.ll
@@ -1,208 +1,272 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -verify-machineinstrs < %s -mtriple=powerpc64le-unknown-linux-gnu -mattr=+altivec -mattr=-vsx -mcpu=pwr7 | FileCheck %s
 
 define void @VPKUHUM_xy(<16 x i8>* %A, <16 x i8>* %B) {
+; CHECK-LABEL: VPKUHUM_xy:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lvx 2, 0, 3
+; CHECK-NEXT:    lvx 3, 0, 4
+; CHECK-NEXT:    vpkuhum 2, 3, 2
+; CHECK-NEXT:    stvx 2, 0, 3
+; CHECK-NEXT:    blr
 entry:
-; CHECK: VPKUHUM_xy:
         %tmp = load <16 x i8>, <16 x i8>* %A
         %tmp2 = load <16 x i8>, <16 x i8>* %B
         %tmp3 = shufflevector <16 x i8> %tmp, <16 x i8> %tmp2, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
-; CHECK: lvx [[REG1:[0-9]+]]
-; CHECK: lvx [[REG2:[0-9]+]]
-; CHECK: vpkuhum [[REG3:[0-9]+]], [[REG2]], [[REG1]]
         store <16 x i8> %tmp3, <16 x i8>* %A
         ret void
 }
 
 define void @VPKUHUM_xx(<16 x i8>* %A) {
+; CHECK-LABEL: VPKUHUM_xx:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lvx 2, 0, 3
+; CHECK-NEXT:    vpkuhum 2, 2, 2
+; CHECK-NEXT:    stvx 2, 0, 3
+; CHECK-NEXT:    blr
 entry:
-; CHECK: VPKUHUM_xx:
         %tmp = load <16 x i8>, <16 x i8>* %A
         %tmp2 = shufflevector <16 x i8> %tmp, <16 x i8> %tmp, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
-; CHECK: vpkuhum
         store <16 x i8> %tmp2, <16 x i8>* %A
         ret void
 }
 
 define void @VPKUWUM_xy(<16 x i8>* %A, <16 x i8>* %B) {
+; CHECK-LABEL: VPKUWUM_xy:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lvx 2, 0, 3
+; CHECK-NEXT:    lvx 3, 0, 4
+; CHECK-NEXT:    vpkuwum 2, 3, 2
+; CHECK-NEXT:    stvx 2, 0, 3
+; CHECK-NEXT:    blr
 entry:
-; CHECK: VPKUWUM_xy:
         %tmp = load <16 x i8>, <16 x i8>* %A
         %tmp2 = load <16 x i8>, <16 x i8>* %B
         %tmp3 = shufflevector <16 x i8> %tmp, <16 x i8> %tmp2, <16 x i32> <i32 0, i32 1, i32 4, i32 5, i32 8, i32 9, i32 12, i32 13, i32 16, i32 17, i32 20, i32 21, i32 24, i32 25, i32 28, i32 29>
-; CHECK: lvx [[REG1:[0-9]+]]
-; CHECK: lvx [[REG2:[0-9]+]]
-; CHECK: vpkuwum [[REG3:[0-9]+]], [[REG2]], [[REG1]]
         store <16 x i8> %tmp3, <16 x i8>* %A
         ret void
 }
 
 define void @VPKUWUM_xx(<16 x i8>* %A) {
+; CHECK-LABEL: VPKUWUM_xx:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lvx 2, 0, 3
+; CHECK-NEXT:    vpkuwum 2, 2, 2
+; CHECK-NEXT:    stvx 2, 0, 3
+; CHECK-NEXT:    blr
 entry:
-; CHECK: VPKUWUM_xx:
         %tmp = load <16 x i8>, <16 x i8>* %A
         %tmp2 = shufflevector <16 x i8> %tmp, <16 x i8> %tmp, <16 x i32> <i32 0, i32 1, i32 4, i32 5, i32 8, i32 9, i32 12, i32 13, i32 0, i32 1, i32 4, i32 5, i32 8, i32 9, i32 12, i32 13>
-; CHECK: vpkuwum
         store <16 x i8> %tmp2, <16 x i8>* %A
         ret void
 }
 
 define void @VMRGLB_xy(<16 x i8>* %A, <16 x i8>* %B) {
+; CHECK-LABEL: VMRGLB_xy:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lvx 2, 0, 3
+; CHECK-NEXT:    lvx 3, 0, 4
+; CHECK-NEXT:    vmrglb 2, 3, 2
+; CHECK-NEXT:    stvx 2, 0, 3
+; CHECK-NEXT:    blr
 entry:
-; CHECK: VMRGLB_xy:
         %tmp = load <16 x i8>, <16 x i8>* %A
         %tmp2 = load <16 x i8>, <16 x i8>* %B
         %tmp3 = shufflevector <16 x i8> %tmp, <16 x i8> %tmp2, <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23>
-; CHECK: lvx [[REG1:[0-9]+]]
-; CHECK: lvx [[REG2:[0-9]+]]
-; CHECK: vmrglb [[REG3:[0-9]+]], [[REG2]], [[REG1]]
         store <16 x i8> %tmp3, <16 x i8>* %A
         ret void
 }
 
 define void @VMRGLB_xx(<16 x i8>* %A) {
+; CHECK-LABEL: VMRGLB_xx:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lvx 2, 0, 3
+; CHECK-NEXT:    vmrglb 2, 2, 2
+; CHECK-NEXT:    stvx 2, 0, 3
+; CHECK-NEXT:    blr
 entry:
-; CHECK: VMRGLB_xx:
         %tmp = load <16 x i8>, <16 x i8>* %A
         %tmp2 = shufflevector <16 x i8> %tmp, <16 x i8> %tmp, <16 x i32> <i32 0, i32 0, i32 1, i32 1, i32 2, i32 2, i32 3, i32 3, i32 4, i32 4, i32 5, i32 5, i32 6, i32 6, i32 7, i32 7>
-; CHECK: vmrglb
         store <16 x i8> %tmp2, <16 x i8>* %A
         ret void
 }
 
 define void @VMRGHB_xy(<16 x i8>* %A, <16 x i8>* %B) {
+; CHECK-LABEL: VMRGHB_xy:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lvx 2, 0, 3
+; CHECK-NEXT:    lvx 3, 0, 4
+; CHECK-NEXT:    vmrghb 2, 3, 2
+; CHECK-NEXT:    stvx 2, 0, 3
+; CHECK-NEXT:    blr
 entry:
-; CHECK: VMRGHB_xy:
         %tmp = load <16 x i8>, <16 x i8>* %A
         %tmp2 = load <16 x i8>, <16 x i8>* %B
         %tmp3 = shufflevector <16 x i8> %tmp, <16 x i8> %tmp2, <16 x i32> <i32 8, i32 24, i32 9, i32 25, i32 10, i32 26, i32 11, i32 27, i32 12, i32 28, i32 13, i32 29, i32 14, i32 30, i32 15, i32 31>
-; CHECK: lvx [[REG1:[0-9]+]]
-; CHECK: lvx [[REG2:[0-9]+]]
-; CHECK: vmrghb [[REG3:[0-9]+]], [[REG2]], [[REG1]]
         store <16 x i8> %tmp3, <16 x i8>* %A
         ret void
 }
 
 define void @VMRGHB_xx(<16 x i8>* %A) {
+; CHECK-LABEL: VMRGHB_xx:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lvx 2, 0, 3
+; CHECK-NEXT:    vmrghb 2, 2, 2
+; CHECK-NEXT:    stvx 2, 0, 3
+; CHECK-NEXT:    blr
 entry:
-; CHECK: VMRGHB_xx:
         %tmp = load <16 x i8>, <16 x i8>* %A
         %tmp2 = shufflevector <16 x i8> %tmp, <16 x i8> %tmp, <16 x i32> <i32 8, i32 8, i32 9, i32 9, i32 10, i32 10, i32 11, i32 11, i32 12, i32 12, i32 13, i32 13, i32 14, i32 14, i32 15, i32 15>
-; CHECK: vmrghb
         store <16 x i8> %tmp2, <16 x i8>* %A
         ret void
 }
 
 define void @VMRGLH_xy(<16 x i8>* %A, <16 x i8>* %B) {
+; CHECK-LABEL: VMRGLH_xy:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lvx 2, 0, 3
+; CHECK-NEXT:    lvx 3, 0, 4
+; CHECK-NEXT:    vmrglh 2, 3, 2
+; CHECK-NEXT:    stvx 2, 0, 3
+; CHECK-NEXT:    blr
 entry:
-; CHECK: VMRGLH_xy:
         %tmp = load <16 x i8>, <16 x i8>* %A
         %tmp2 = load <16 x i8>, <16 x i8>* %B
         %tmp3 = shufflevector <16 x i8> %tmp, <16 x i8> %tmp2, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 2, i32 3, i32 18, i32 19, i32 4, i32 5, i32 20, i32 21, i32 6, i32 7, i32 22, i32 23>
-; CHECK: lvx [[REG1:[0-9]+]]
-; CHECK: lvx [[REG2:[0-9]+]]
-; CHECK: vmrglh [[REG3:[0-9]+]], [[REG2]], [[REG1]]
         store <16 x i8> %tmp3, <16 x i8>* %A
         ret void
 }
 
 define void @VMRGLH_xx(<16 x i8>* %A) {
+; CHECK-LABEL: VMRGLH_xx:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lvx 2, 0, 3
+; CHECK-NEXT:    vmrglh 2, 2, 2
+; CHECK-NEXT:    stvx 2, 0, 3
+; CHECK-NEXT:    blr
 entry:
-; CHECK: VMRGLH_xx:
         %tmp = load <16 x i8>, <16 x i8>* %A
         %tmp2 = shufflevector <16 x i8> %tmp, <16 x i8> %tmp, <16 x i32> <i32 0, i32 1, i32 0, i32 1, i32 2, i32 3, i32 2, i32 3, i32 4, i32 5, i32 4, i32 5, i32 6, i32 7, i32 6, i32 7>
-; CHECK: vmrglh
         store <16 x i8> %tmp2, <16 x i8>* %A
         ret void
 }
 
 define void @VMRGHH_xy(<16 x i8>* %A, <16 x i8>* %B) {
+; CHECK-LABEL: VMRGHH_xy:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lvx 2, 0, 3
+; CHECK-NEXT:    lvx 3, 0, 4
+; CHECK-NEXT:    vmrghh 2, 3, 2
+; CHECK-NEXT:    stvx 2, 0, 3
+; CHECK-NEXT:    blr
 entry:
-; CHECK: VMRGHH_xy:
         %tmp = load <16 x i8>, <16 x i8>* %A
         %tmp2 = load <16 x i8>, <16 x i8>* %B
         %tmp3 = shufflevector <16 x i8> %tmp, <16 x i8> %tmp2, <16 x i32> <i32 8, i32 9, i32 24, i32 25, i32 10, i32 11, i32 26, i32 27, i32 12, i32 13, i32 28, i32 29, i32 14, i32 15, i32 30, i32 31>
-; CHECK: lvx [[REG1:[0-9]+]]
-; CHECK: lvx [[REG2:[0-9]+]]
-; CHECK: vmrghh [[REG3:[0-9]+]], [[REG2]], [[REG1]]
         store <16 x i8> %tmp3, <16 x i8>* %A
         ret void
 }
 
 define void @VMRGHH_xx(<16 x i8>* %A) {
+; CHECK-LABEL: VMRGHH_xx:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lvx 2, 0, 3
+; CHECK-NEXT:    vmrghh 2, 2, 2
+; CHECK-NEXT:    stvx 2, 0, 3
+; CHECK-NEXT:    blr
 entry:
-; CHECK: VMRGHH_xx:
         %tmp = load <16 x i8>, <16 x i8>* %A
         %tmp2 = shufflevector <16 x i8> %tmp, <16 x i8> %tmp, <16 x i32> <i32 8, i32 9, i32 8, i32 9, i32 10, i32 11, i32 10, i32 11, i32 12, i32 13, i32 12, i32 13, i32 14, i32 15, i32 14, i32 15>
-; CHECK: vmrghh
         store <16 x i8> %tmp2, <16 x i8>* %A
         ret void
 }
 
 define void @VMRGLW_xy(<16 x i8>* %A, <16 x i8>* %B) {
+; CHECK-LABEL: VMRGLW_xy:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lvx 2, 0, 3
+; CHECK-NEXT:    lvx 3, 0, 4
+; CHECK-NEXT:    vmrglw 2, 3, 2
+; CHECK-NEXT:    stvx 2, 0, 3
+; CHECK-NEXT:    blr
 entry:
-; CHECK: VMRGLW_xy:
         %tmp = load <16 x i8>, <16 x i8>* %A
         %tmp2 = load <16 x i8>, <16 x i8>* %B
         %tmp3 = shufflevector <16 x i8> %tmp, <16 x i8> %tmp2, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 20, i32 21, i32 22, i32 23>
-; CHECK: lvx [[REG1:[0-9]+]]
-; CHECK: lvx [[REG2:[0-9]+]]
-; CHECK: vmrglw [[REG3:[0-9]+]], [[REG2]], [[REG1]]
         store <16 x i8> %tmp3, <16 x i8>* %A
         ret void
 }
 
 define void @VMRGLW_xx(<16 x i8>* %A) {
+; CHECK-LABEL: VMRGLW_xx:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lvx 2, 0, 3
+; CHECK-NEXT:    vmrglw 2, 2, 2
+; CHECK-NEXT:    stvx 2, 0, 3
+; CHECK-NEXT:    blr
 entry:
-; CHECK: VMRGLW_xx:
         %tmp = load <16 x i8>, <16 x i8>* %A
         %tmp2 = shufflevector <16 x i8> %tmp, <16 x i8> %tmp, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7>
-; CHECK: vmrglw
         store <16 x i8> %tmp2, <16 x i8>* %A
         ret void
 }
 
 define void @VMRGHW_xy(<16 x i8>* %A, <16 x i8>* %B) {
+; CHECK-LABEL: VMRGHW_xy:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lvx 2, 0, 3
+; CHECK-NEXT:    lvx 3, 0, 4
+; CHECK-NEXT:    vmrghw 2, 3, 2
+; CHECK-NEXT:    stvx 2, 0, 3
+; CHECK-NEXT:    blr
 entry:
-; CHECK: VMRGHW_xy:
         %tmp = load <16 x i8>, <16 x i8>* %A
         %tmp2 = load <16 x i8>, <16 x i8>* %B
         %tmp3 = shufflevector <16 x i8> %tmp, <16 x i8> %tmp2, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 24, i32 25, i32 26, i32 27, i32 12, i32 13, i32 14, i32 15, i32 28, i32 29, i32 30, i32 31>
-; CHECK: lvx [[REG1:[0-9]+]]
-; CHECK: lvx [[REG2:[0-9]+]]
-; CHECK: vmrghw [[REG3:[0-9]+]], [[REG2]], [[REG1]]
         store <16 x i8> %tmp3, <16 x i8>* %A
         ret void
 }
 
 define void @VMRGHW_xx(<16 x i8>* %A) {
+; CHECK-LABEL: VMRGHW_xx:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lvx 2, 0, 3
+; CHECK-NEXT:    vmrghw 2, 2, 2
+; CHECK-NEXT:    stvx 2, 0, 3
+; CHECK-NEXT:    blr
 entry:
-; CHECK: VMRGHW_xx:
         %tmp = load <16 x i8>, <16 x i8>* %A
         %tmp2 = shufflevector <16 x i8> %tmp, <16 x i8> %tmp, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 12, i32 13, i32 14, i32 15>
-; CHECK: vmrghw
         store <16 x i8> %tmp2, <16 x i8>* %A
         ret void
 }
 
 define void @VSLDOI_xy(<16 x i8>* %A, <16 x i8>* %B) {
+; CHECK-LABEL: VSLDOI_xy:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lvx 2, 0, 3
+; CHECK-NEXT:    lvx 3, 0, 4
+; CHECK-NEXT:    vsldoi 2, 3, 2, 4
+; CHECK-NEXT:    stvx 2, 0, 3
+; CHECK-NEXT:    blr
 entry:
-; CHECK: VSLDOI_xy:
         %tmp = load <16 x i8>, <16 x i8>* %A
         %tmp2 = load <16 x i8>, <16 x i8>* %B
         %tmp3 = shufflevector <16 x i8> %tmp, <16 x i8> %tmp2, <16 x i32> <i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27>
-; CHECK: lvx [[REG1:[0-9]+]]
-; CHECK: lvx [[REG2:[0-9]+]]
-; CHECK: vsldoi [[REG3:[0-9]+]], [[REG2]], [[REG1]], 4
         store <16 x i8> %tmp3, <16 x i8>* %A
         ret void
 }
 
 define void @VSLDOI_xx(<16 x i8>* %A) {
+; CHECK-LABEL: VSLDOI_xx:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lvx 2, 0, 3
+; CHECK-NEXT:    vsldoi 2, 2, 2, 4
+; CHECK-NEXT:    stvx 2, 0, 3
+; CHECK-NEXT:    blr
 entry:
-; CHECK: VSLDOI_xx:
         %tmp = load <16 x i8>, <16 x i8>* %A
         %tmp2 = shufflevector <16 x i8> %tmp, <16 x i8> %tmp, <16 x i32> <i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-; CHECK: vsldoi {{[0-9]+}}, [[REG1:[0-9]+]], [[REG1]], 4
         store <16 x i8> %tmp2, <16 x i8>* %A
         ret void
 }
diff --git a/llvm/test/CodeGen/PowerPC/vec_shuffle_p8vector.ll b/llvm/test/CodeGen/PowerPC/vec_shuffle_p8vector.ll
index e1be6663a..33226e8a0 100644
--- a/llvm/test/CodeGen/PowerPC/vec_shuffle_p8vector.ll
+++ b/llvm/test/CodeGen/PowerPC/vec_shuffle_p8vector.ll
@@ -1,7 +1,23 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -verify-machineinstrs -mcpu=pwr8 -mtriple=powerpc64-unknown-linux-gnu -mattr=+power8-vector < %s | FileCheck %s
 ; RUN: llc -verify-machineinstrs -mcpu=pwr7 -mtriple=powerpc64-unknown-linux-gnu < %s | FileCheck -check-prefix=CHECK-PWR7 %s
 
 define void @VPKUDUM_unary(<2 x i64>* %A) {
+; CHECK-LABEL: VPKUDUM_unary:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lxvw4x 34, 0, 3
+; CHECK-NEXT:    vpkudum 2, 2, 2
+; CHECK-NEXT:    stxvw4x 34, 0, 3
+; CHECK-NEXT:    blr
+;
+; CHECK-PWR7-LABEL: VPKUDUM_unary:
+; CHECK-PWR7:       # %bb.0: # %entry
+; CHECK-PWR7-NEXT:    lxvw4x 34, 0, 3
+; CHECK-PWR7-NEXT:    vmrglw 3, 2, 2
+; CHECK-PWR7-NEXT:    vmrghw 2, 2, 2
+; CHECK-PWR7-NEXT:    vmrglw 2, 2, 3
+; CHECK-PWR7-NEXT:    stxvw4x 34, 0, 3
+; CHECK-PWR7-NEXT:    blr
 entry:
         %tmp = load <2 x i64>, <2 x i64>* %A
         %tmp2 = bitcast <2 x i64> %tmp to <4 x i32>
@@ -16,16 +32,24 @@ entry:
         ret void
 }
 
-; CHECK-LABEL: @VPKUDUM_unary
-; CHECK-NOT:   vperm
-; CHECK-NOT:   vmrglw
-; CHECK-NOT:   vmrghw
-; CHECK:       vpkudum
-; CHECK-PWR7:  vmrglw
-; CHECK-PWR7:  vmrghw
-; CHECK-PWR7:  vmrglw
-
 define void @VPKUDUM(<2 x i64>* %A, <2 x i64>* %B) {
+; CHECK-LABEL: VPKUDUM:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lxvw4x 34, 0, 3
+; CHECK-NEXT:    lxvw4x 35, 0, 4
+; CHECK-NEXT:    vpkudum 2, 2, 3
+; CHECK-NEXT:    stxvw4x 34, 0, 3
+; CHECK-NEXT:    blr
+;
+; CHECK-PWR7-LABEL: VPKUDUM:
+; CHECK-PWR7:       # %bb.0: # %entry
+; CHECK-PWR7-NEXT:    lxvw4x 34, 0, 3
+; CHECK-PWR7-NEXT:    lxvw4x 35, 0, 4
+; CHECK-PWR7-NEXT:    vmrglw 4, 2, 3
+; CHECK-PWR7-NEXT:    vmrghw 2, 2, 3
+; CHECK-PWR7-NEXT:    vmrglw 2, 2, 4
+; CHECK-PWR7-NEXT:    stxvw4x 34, 0, 3
+; CHECK-PWR7-NEXT:    blr
 entry:
         %tmp = load <2 x i64>, <2 x i64>* %A
         %tmp2 = bitcast <2 x i64> %tmp to <4 x i32>
@@ -44,11 +68,3 @@ entry:
         ret void
 }
 
-; CHECK-LABEL: @VPKUDUM
-; CHECK-NOT:   vperm
-; CHECK-NOT:   vmrglw
-; CHECK-NOT:   vmrghw
-; CHECK:       vpkudum
-; CHECK-PWR7:  vmrglw
-; CHECK-PWR7:  vmrghw
-; CHECK-PWR7:  vmrglw
diff --git a/llvm/test/CodeGen/PowerPC/vec_shuffle_p8vector_le.ll b/llvm/test/CodeGen/PowerPC/vec_shuffle_p8vector_le.ll
index df373aef4..65200298e 100644
--- a/llvm/test/CodeGen/PowerPC/vec_shuffle_p8vector_le.ll
+++ b/llvm/test/CodeGen/PowerPC/vec_shuffle_p8vector_le.ll
@@ -1,6 +1,13 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -verify-machineinstrs -mcpu=pwr8 -mtriple=powerpc64le-unknown-linux-gnu -mattr=+power8-vector < %s | FileCheck %s
 
 define void @VPKUDUM_unary(<2 x i64>* %A) {
+; CHECK-LABEL: VPKUDUM_unary:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lvx 2, 0, 3
+; CHECK-NEXT:    vpkudum 2, 2, 2
+; CHECK-NEXT:    stvx 2, 0, 3
+; CHECK-NEXT:    blr
 entry:
 	%tmp = load <2 x i64>, <2 x i64>* %A
 	%tmp2 = bitcast <2 x i64> %tmp to <4 x i32>
@@ -15,11 +22,14 @@ entry:
 	ret void
 }
 
-; CHECK-LABEL: @VPKUDUM_unary
-; CHECK-NOT:   vperm
-; CHECK:       vpkudum
-
 define void @VPKUDUM(<2 x i64>* %A, <2 x i64>* %B) {
+; CHECK-LABEL: VPKUDUM:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lvx 2, 0, 3
+; CHECK-NEXT:    lvx 3, 0, 4
+; CHECK-NEXT:    vpkudum 2, 3, 2
+; CHECK-NEXT:    stvx 2, 0, 3
+; CHECK-NEXT:    blr
 entry:
 	%tmp = load <2 x i64>, <2 x i64>* %A
 	%tmp2 = bitcast <2 x i64> %tmp to <4 x i32>
@@ -38,6 +48,3 @@ entry:
 	ret void
 }
 
-; CHECK-LABEL: @VPKUDUM
-; CHECK-NOT:   vperm
-; CHECK:       vpkudum
diff --git a/llvm/test/CodeGen/PowerPC/vector-constrained-fp-intrinsics.ll b/llvm/test/CodeGen/PowerPC/vector-constrained-fp-intrinsics.ll
index a35b6f935..253e74cf0 100644
--- a/llvm/test/CodeGen/PowerPC/vector-constrained-fp-intrinsics.ll
+++ b/llvm/test/CodeGen/PowerPC/vector-constrained-fp-intrinsics.ll
@@ -2,7 +2,7 @@
 ; RUN: llc -O3 -mtriple=powerpc64le-linux-gnu < %s | FileCheck --check-prefix=PC64LE %s
 ; RUN: llc -O3 -mtriple=powerpc64le-linux-gnu -mcpu=pwr9 < %s | FileCheck --check-prefix=PC64LE9 %s
 
-define <1 x float> @constrained_vector_fdiv_v1f32() {
+define <1 x float> @constrained_vector_fdiv_v1f32() #0 {
 ; PC64LE-LABEL: constrained_vector_fdiv_v1f32:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    addis 3, 2, .LCPI0_0@toc@ha
@@ -29,11 +29,11 @@ entry:
            <1 x float> <float 1.000000e+00>,
            <1 x float> <float 1.000000e+01>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #1
   ret <1 x float> %div
 }
 
-define <2 x double> @constrained_vector_fdiv_v2f64() {
+define <2 x double> @constrained_vector_fdiv_v2f64() #0 {
 ; PC64LE-LABEL: constrained_vector_fdiv_v2f64:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    addis 3, 2, .LCPI1_0@toc@ha
@@ -62,11 +62,11 @@ entry:
            <2 x double> <double 1.000000e+00, double 2.000000e+00>,
            <2 x double> <double 1.000000e+01, double 1.000000e+01>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #1
   ret <2 x double> %div
 }
 
-define <3 x float> @constrained_vector_fdiv_v3f32() {
+define <3 x float> @constrained_vector_fdiv_v3f32() #0 {
 ; PC64LE-LABEL: constrained_vector_fdiv_v3f32:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    addis 3, 2, .LCPI2_0@toc@ha
@@ -123,11 +123,11 @@ entry:
            <3 x float> <float 1.000000e+00, float 2.000000e+00, float 3.000000e+00>,
            <3 x float> <float 1.000000e+01, float 1.000000e+01, float 1.000000e+01>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #1
   ret <3 x float> %div
 }
 
-define <3 x double> @constrained_vector_fdiv_v3f64() {
+define <3 x double> @constrained_vector_fdiv_v3f64() #0 {
 ; PC64LE-LABEL: constrained_vector_fdiv_v3f64:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    addis 3, 2, .LCPI3_2@toc@ha
@@ -172,11 +172,11 @@ entry:
            <3 x double> <double 1.000000e+00, double 2.000000e+00, double 3.000000e+00>,
            <3 x double> <double 1.000000e+01, double 1.000000e+01, double 1.000000e+01>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #1
   ret <3 x double> %div
 }
 
-define <4 x double> @constrained_vector_fdiv_v4f64() {
+define <4 x double> @constrained_vector_fdiv_v4f64() #0 {
 ; PC64LE-LABEL: constrained_vector_fdiv_v4f64:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    addis 3, 2, .LCPI4_0@toc@ha
@@ -216,18 +216,16 @@ entry:
            <4 x double> <double 1.000000e+01, double 1.000000e+01,
                          double 1.000000e+01, double 1.000000e+01>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #1
   ret <4 x double> %div
 }
 
-define <1 x float> @constrained_vector_frem_v1f32() {
+define <1 x float> @constrained_vector_frem_v1f32() #0 {
 ; PC64LE-LABEL: constrained_vector_frem_v1f32:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    mflr 0
 ; PC64LE-NEXT:    std 0, 16(1)
 ; PC64LE-NEXT:    stdu 1, -32(1)
-; PC64LE-NEXT:    .cfi_def_cfa_offset 32
-; PC64LE-NEXT:    .cfi_offset lr, 16
 ; PC64LE-NEXT:    addis 3, 2, .LCPI5_0@toc@ha
 ; PC64LE-NEXT:    addis 4, 2, .LCPI5_1@toc@ha
 ; PC64LE-NEXT:    lfs 1, .LCPI5_0@toc@l(3)
@@ -246,8 +244,6 @@ define <1 x float> @constrained_vector_frem_v1f32() {
 ; PC64LE9-NEXT:    mflr 0
 ; PC64LE9-NEXT:    std 0, 16(1)
 ; PC64LE9-NEXT:    stdu 1, -32(1)
-; PC64LE9-NEXT:    .cfi_def_cfa_offset 32
-; PC64LE9-NEXT:    .cfi_offset lr, 16
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI5_0@toc@ha
 ; PC64LE9-NEXT:    lfs 1, .LCPI5_0@toc@l(3)
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI5_1@toc@ha
@@ -265,17 +261,14 @@ entry:
            <1 x float> <float 1.000000e+00>,
            <1 x float> <float 1.000000e+01>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #1
   ret <1 x float> %rem
 }
 
-define <2 x double> @constrained_vector_frem_v2f64() {
+define <2 x double> @constrained_vector_frem_v2f64() #0 {
 ; PC64LE-LABEL: constrained_vector_frem_v2f64:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    mflr 0
-; PC64LE-NEXT:    .cfi_def_cfa_offset 80
-; PC64LE-NEXT:    .cfi_offset lr, 16
-; PC64LE-NEXT:    .cfi_offset f31, -8
 ; PC64LE-NEXT:    stfd 31, -8(1) # 8-byte Folded Spill
 ; PC64LE-NEXT:    std 0, 16(1)
 ; PC64LE-NEXT:    stdu 1, -80(1)
@@ -300,16 +293,13 @@ define <2 x double> @constrained_vector_frem_v2f64() {
 ; PC64LE-NEXT:    xxmrghd 34, 1, 0
 ; PC64LE-NEXT:    addi 1, 1, 80
 ; PC64LE-NEXT:    ld 0, 16(1)
-; PC64LE-NEXT:    mtlr 0
 ; PC64LE-NEXT:    lfd 31, -8(1) # 8-byte Folded Reload
+; PC64LE-NEXT:    mtlr 0
 ; PC64LE-NEXT:    blr
 ;
 ; PC64LE9-LABEL: constrained_vector_frem_v2f64:
 ; PC64LE9:       # %bb.0: # %entry
 ; PC64LE9-NEXT:    mflr 0
-; PC64LE9-NEXT:    .cfi_def_cfa_offset 64
-; PC64LE9-NEXT:    .cfi_offset lr, 16
-; PC64LE9-NEXT:    .cfi_offset f31, -8
 ; PC64LE9-NEXT:    stfd 31, -8(1) # 8-byte Folded Spill
 ; PC64LE9-NEXT:    std 0, 16(1)
 ; PC64LE9-NEXT:    stdu 1, -64(1)
@@ -332,27 +322,22 @@ define <2 x double> @constrained_vector_frem_v2f64() {
 ; PC64LE9-NEXT:    xxmrghd 34, 1, 0
 ; PC64LE9-NEXT:    addi 1, 1, 64
 ; PC64LE9-NEXT:    ld 0, 16(1)
-; PC64LE9-NEXT:    mtlr 0
 ; PC64LE9-NEXT:    lfd 31, -8(1) # 8-byte Folded Reload
+; PC64LE9-NEXT:    mtlr 0
 ; PC64LE9-NEXT:    blr
 entry:
   %rem = call <2 x double> @llvm.experimental.constrained.frem.v2f64(
            <2 x double> <double 1.000000e+00, double 2.000000e+00>,
            <2 x double> <double 1.000000e+01, double 1.000000e+01>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #1
   ret <2 x double> %rem
 }
 
-define <3 x float> @constrained_vector_frem_v3f32() {
+define <3 x float> @constrained_vector_frem_v3f32() #0 {
 ; PC64LE-LABEL: constrained_vector_frem_v3f32:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    mflr 0
-; PC64LE-NEXT:    .cfi_def_cfa_offset 64
-; PC64LE-NEXT:    .cfi_offset lr, 16
-; PC64LE-NEXT:    .cfi_offset f29, -24
-; PC64LE-NEXT:    .cfi_offset f30, -16
-; PC64LE-NEXT:    .cfi_offset f31, -8
 ; PC64LE-NEXT:    stfd 29, -24(1) # 8-byte Folded Spill
 ; PC64LE-NEXT:    stfd 30, -16(1) # 8-byte Folded Spill
 ; PC64LE-NEXT:    stfd 31, -8(1) # 8-byte Folded Spill
@@ -390,20 +375,15 @@ define <3 x float> @constrained_vector_frem_v3f32() {
 ; PC64LE-NEXT:    vperm 2, 3, 2, 4
 ; PC64LE-NEXT:    addi 1, 1, 64
 ; PC64LE-NEXT:    ld 0, 16(1)
-; PC64LE-NEXT:    mtlr 0
 ; PC64LE-NEXT:    lfd 31, -8(1) # 8-byte Folded Reload
 ; PC64LE-NEXT:    lfd 30, -16(1) # 8-byte Folded Reload
 ; PC64LE-NEXT:    lfd 29, -24(1) # 8-byte Folded Reload
+; PC64LE-NEXT:    mtlr 0
 ; PC64LE-NEXT:    blr
 ;
 ; PC64LE9-LABEL: constrained_vector_frem_v3f32:
 ; PC64LE9:       # %bb.0: # %entry
 ; PC64LE9-NEXT:    mflr 0
-; PC64LE9-NEXT:    .cfi_def_cfa_offset 64
-; PC64LE9-NEXT:    .cfi_offset lr, 16
-; PC64LE9-NEXT:    .cfi_offset f29, -24
-; PC64LE9-NEXT:    .cfi_offset f30, -16
-; PC64LE9-NEXT:    .cfi_offset f31, -8
 ; PC64LE9-NEXT:    stfd 29, -24(1) # 8-byte Folded Spill
 ; PC64LE9-NEXT:    stfd 30, -16(1) # 8-byte Folded Spill
 ; PC64LE9-NEXT:    stfd 31, -8(1) # 8-byte Folded Spill
@@ -441,9 +421,9 @@ define <3 x float> @constrained_vector_frem_v3f32() {
 ; PC64LE9-NEXT:    vperm 2, 3, 2, 4
 ; PC64LE9-NEXT:    addi 1, 1, 64
 ; PC64LE9-NEXT:    ld 0, 16(1)
-; PC64LE9-NEXT:    mtlr 0
 ; PC64LE9-NEXT:    lfd 31, -8(1) # 8-byte Folded Reload
 ; PC64LE9-NEXT:    lfd 30, -16(1) # 8-byte Folded Reload
+; PC64LE9-NEXT:    mtlr 0
 ; PC64LE9-NEXT:    lfd 29, -24(1) # 8-byte Folded Reload
 ; PC64LE9-NEXT:    blr
 entry:
@@ -451,20 +431,16 @@ entry:
            <3 x float> <float 1.000000e+00, float 2.000000e+00, float 3.000000e+00>,
            <3 x float> <float 1.000000e+01, float 1.000000e+01, float 1.000000e+01>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #1
   ret <3 x float> %rem
 }
 
-define <3 x double> @constrained_vector_frem_v3f64() {
+define <3 x double> @constrained_vector_frem_v3f64() #0 {
 ; PC64LE-LABEL: constrained_vector_frem_v3f64:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    mflr 0
 ; PC64LE-NEXT:    std 0, 16(1)
 ; PC64LE-NEXT:    stdu 1, -96(1)
-; PC64LE-NEXT:    .cfi_def_cfa_offset 96
-; PC64LE-NEXT:    .cfi_offset lr, 16
-; PC64LE-NEXT:    .cfi_offset f31, -8
-; PC64LE-NEXT:    .cfi_offset v31, -32
 ; PC64LE-NEXT:    addis 4, 2, .LCPI8_1@toc@ha
 ; PC64LE-NEXT:    stfd 31, 88(1) # 8-byte Folded Spill
 ; PC64LE-NEXT:    li 3, 64
@@ -492,12 +468,13 @@ define <3 x double> @constrained_vector_frem_v3f64() {
 ; PC64LE-NEXT:    lfs 1, .LCPI8_3@toc@l(3)
 ; PC64LE-NEXT:    bl fmod
 ; PC64LE-NEXT:    nop
+; PC64LE-NEXT:    xxswapd 0, 63
 ; PC64LE-NEXT:    li 3, 64
-; PC64LE-NEXT:    fmr 3, 1
-; PC64LE-NEXT:    xxlor 1, 63, 63
-; PC64LE-NEXT:    lfd 31, 88(1) # 8-byte Folded Reload
 ; PC64LE-NEXT:    xxlor 2, 63, 63
+; PC64LE-NEXT:    lfd 31, 88(1) # 8-byte Folded Reload
 ; PC64LE-NEXT:    lxvd2x 63, 1, 3 # 16-byte Folded Reload
+; PC64LE-NEXT:    fmr 3, 1
+; PC64LE-NEXT:    fmr 1, 0
 ; PC64LE-NEXT:    addi 1, 1, 96
 ; PC64LE-NEXT:    ld 0, 16(1)
 ; PC64LE-NEXT:    mtlr 0
@@ -508,10 +485,6 @@ define <3 x double> @constrained_vector_frem_v3f64() {
 ; PC64LE9-NEXT:    mflr 0
 ; PC64LE9-NEXT:    std 0, 16(1)
 ; PC64LE9-NEXT:    stdu 1, -80(1)
-; PC64LE9-NEXT:    .cfi_def_cfa_offset 80
-; PC64LE9-NEXT:    .cfi_offset lr, 16
-; PC64LE9-NEXT:    .cfi_offset f31, -8
-; PC64LE9-NEXT:    .cfi_offset v31, -32
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI8_0@toc@ha
 ; PC64LE9-NEXT:    lfs 1, .LCPI8_0@toc@l(3)
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI8_1@toc@ha
@@ -537,10 +510,11 @@ define <3 x double> @constrained_vector_frem_v3f64() {
 ; PC64LE9-NEXT:    bl fmod
 ; PC64LE9-NEXT:    nop
 ; PC64LE9-NEXT:    fmr 3, 1
-; PC64LE9-NEXT:    xscpsgndp 1, 63, 63
+; PC64LE9-NEXT:    xxswapd 1, 63
 ; PC64LE9-NEXT:    xscpsgndp 2, 63, 63
 ; PC64LE9-NEXT:    lxv 63, 48(1) # 16-byte Folded Reload
 ; PC64LE9-NEXT:    lfd 31, 72(1) # 8-byte Folded Reload
+; PC64LE9-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
 ; PC64LE9-NEXT:    addi 1, 1, 80
 ; PC64LE9-NEXT:    ld 0, 16(1)
 ; PC64LE9-NEXT:    mtlr 0
@@ -550,20 +524,16 @@ entry:
            <3 x double> <double 1.000000e+00, double 2.000000e+00, double 3.000000e+00>,
            <3 x double> <double 1.000000e+01, double 1.000000e+01, double 1.000000e+01>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #1
   ret <3 x double> %rem
 }
 
-define <4 x double> @constrained_vector_frem_v4f64() {
+define <4 x double> @constrained_vector_frem_v4f64() #0 {
 ; PC64LE-LABEL: constrained_vector_frem_v4f64:
 ; PC64LE:       # %bb.0:
 ; PC64LE-NEXT:    mflr 0
 ; PC64LE-NEXT:    std 0, 16(1)
 ; PC64LE-NEXT:    stdu 1, -96(1)
-; PC64LE-NEXT:    .cfi_def_cfa_offset 96
-; PC64LE-NEXT:    .cfi_offset lr, 16
-; PC64LE-NEXT:    .cfi_offset f31, -8
-; PC64LE-NEXT:    .cfi_offset v31, -32
 ; PC64LE-NEXT:    addis 4, 2, .LCPI9_1@toc@ha
 ; PC64LE-NEXT:    stfd 31, 88(1) # 8-byte Folded Spill
 ; PC64LE-NEXT:    li 3, 64
@@ -617,10 +587,6 @@ define <4 x double> @constrained_vector_frem_v4f64() {
 ; PC64LE9-NEXT:    mflr 0
 ; PC64LE9-NEXT:    std 0, 16(1)
 ; PC64LE9-NEXT:    stdu 1, -80(1)
-; PC64LE9-NEXT:    .cfi_def_cfa_offset 80
-; PC64LE9-NEXT:    .cfi_offset lr, 16
-; PC64LE9-NEXT:    .cfi_offset f31, -8
-; PC64LE9-NEXT:    .cfi_offset v31, -32
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI9_0@toc@ha
 ; PC64LE9-NEXT:    lfs 1, .LCPI9_0@toc@l(3)
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI9_1@toc@ha
@@ -668,11 +634,11 @@ define <4 x double> @constrained_vector_frem_v4f64() {
            <4 x double> <double 1.000000e+01, double 1.000000e+01,
                          double 1.000000e+01, double 1.000000e+01>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #1
   ret <4 x double> %rem
 }
 
-define <1 x float> @constrained_vector_fmul_v1f32() {
+define <1 x float> @constrained_vector_fmul_v1f32() #0 {
 ; PC64LE-LABEL: constrained_vector_fmul_v1f32:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    addis 3, 2, .LCPI10_0@toc@ha
@@ -699,11 +665,11 @@ entry:
            <1 x float> <float 0x7FF0000000000000>,
            <1 x float> <float 2.000000e+00>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #1
   ret <1 x float> %mul
 }
 
-define <2 x double> @constrained_vector_fmul_v2f64() {
+define <2 x double> @constrained_vector_fmul_v2f64() #0 {
 ; PC64LE-LABEL: constrained_vector_fmul_v2f64:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    addis 3, 2, .LCPI11_0@toc@ha
@@ -732,11 +698,11 @@ entry:
            <2 x double> <double 0x7FEFFFFFFFFFFFFF, double 0x7FEFFFFFFFFFFFFF>,
            <2 x double> <double 2.000000e+00, double 3.000000e+00>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #1
   ret <2 x double> %mul
 }
 
-define <3 x float> @constrained_vector_fmul_v3f32() {
+define <3 x float> @constrained_vector_fmul_v3f32() #0 {
 ; PC64LE-LABEL: constrained_vector_fmul_v3f32:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    addis 3, 2, .LCPI12_1@toc@ha
@@ -794,11 +760,11 @@ entry:
                         float 0x7FF0000000000000>,
            <3 x float> <float 1.000000e+00, float 1.000000e+01, float 1.000000e+02>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #1
   ret <3 x float> %mul
 }
 
-define <3 x double> @constrained_vector_fmul_v3f64() {
+define <3 x double> @constrained_vector_fmul_v3f64() #0 {
 ; PC64LE-LABEL: constrained_vector_fmul_v3f64:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    addis 3, 2, .LCPI13_2@toc@ha
@@ -844,11 +810,11 @@ entry:
                          double 0x7FEFFFFFFFFFFFFF>,
            <3 x double> <double 1.000000e+00, double 1.000000e+01, double 1.000000e+02>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #1
   ret <3 x double> %mul
 }
 
-define <4 x double> @constrained_vector_fmul_v4f64() {
+define <4 x double> @constrained_vector_fmul_v4f64() #0 {
 ; PC64LE-LABEL: constrained_vector_fmul_v4f64:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    addis 3, 2, .LCPI14_0@toc@ha
@@ -888,11 +854,11 @@ entry:
            <4 x double> <double 2.000000e+00, double 3.000000e+00,
                          double 4.000000e+00, double 5.000000e+00>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #1
   ret <4 x double> %mul
 }
 
-define <1 x float> @constrained_vector_fadd_v1f32() {
+define <1 x float> @constrained_vector_fadd_v1f32() #0 {
 ; PC64LE-LABEL: constrained_vector_fadd_v1f32:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    addis 3, 2, .LCPI15_0@toc@ha
@@ -919,11 +885,11 @@ entry:
            <1 x float> <float 0x7FF0000000000000>,
            <1 x float> <float 1.0>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #1
   ret <1 x float> %add
 }
 
-define <2 x double> @constrained_vector_fadd_v2f64() {
+define <2 x double> @constrained_vector_fadd_v2f64() #0 {
 ; PC64LE-LABEL: constrained_vector_fadd_v2f64:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    addis 3, 2, .LCPI16_0@toc@ha
@@ -952,11 +918,11 @@ entry:
            <2 x double> <double 0x7FEFFFFFFFFFFFFF, double 0x7FEFFFFFFFFFFFFF>,
            <2 x double> <double 1.000000e+00, double 1.000000e-01>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #1
   ret <2 x double> %add
 }
 
-define <3 x float> @constrained_vector_fadd_v3f32() {
+define <3 x float> @constrained_vector_fadd_v3f32() #0 {
 ; PC64LE-LABEL: constrained_vector_fadd_v3f32:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    addis 3, 2, .LCPI17_0@toc@ha
@@ -1012,11 +978,11 @@ entry:
                         float 0xFFFFFFFFE0000000>,
            <3 x float> <float 2.0, float 1.0, float 0.0>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #1
   ret <3 x float> %add
 }
 
-define <3 x double> @constrained_vector_fadd_v3f64() {
+define <3 x double> @constrained_vector_fadd_v3f64() #0 {
 ; PC64LE-LABEL: constrained_vector_fadd_v3f64:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    addis 3, 2, .LCPI18_1@toc@ha
@@ -1060,11 +1026,11 @@ entry:
                          double 0x7FEFFFFFFFFFFFFF>,
            <3 x double> <double 2.0, double 1.0, double 0.0>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #1
   ret <3 x double> %add
 }
 
-define <4 x double> @constrained_vector_fadd_v4f64() {
+define <4 x double> @constrained_vector_fadd_v4f64() #0 {
 ; PC64LE-LABEL: constrained_vector_fadd_v4f64:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    addis 3, 2, .LCPI19_0@toc@ha
@@ -1104,11 +1070,11 @@ entry:
            <4 x double> <double 1.000000e+00, double 1.000000e-01,
                          double 2.000000e+00, double 2.000000e-01>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #1
   ret <4 x double> %add
 }
 
-define <1 x float> @constrained_vector_fsub_v1f32() {
+define <1 x float> @constrained_vector_fsub_v1f32() #0 {
 ; PC64LE-LABEL: constrained_vector_fsub_v1f32:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    addis 3, 2, .LCPI20_0@toc@ha
@@ -1135,11 +1101,11 @@ entry:
            <1 x float> <float 0x7FF0000000000000>,
            <1 x float> <float 1.000000e+00>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #1
   ret <1 x float> %sub
 }
 
-define <2 x double> @constrained_vector_fsub_v2f64() {
+define <2 x double> @constrained_vector_fsub_v2f64() #0 {
 ; PC64LE-LABEL: constrained_vector_fsub_v2f64:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    addis 3, 2, .LCPI21_0@toc@ha
@@ -1168,11 +1134,11 @@ entry:
            <2 x double> <double 0xFFEFFFFFFFFFFFFF, double 0xFFEFFFFFFFFFFFFF>,
            <2 x double> <double 1.000000e+00, double 1.000000e-01>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #1
   ret <2 x double> %sub
 }
 
-define <3 x float> @constrained_vector_fsub_v3f32() {
+define <3 x float> @constrained_vector_fsub_v3f32() #0 {
 ; PC64LE-LABEL: constrained_vector_fsub_v3f32:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    addis 3, 2, .LCPI22_0@toc@ha
@@ -1228,11 +1194,11 @@ entry:
                         float 0xFFFFFFFFE0000000>,
            <3 x float> <float 2.0, float 1.0, float 0.0>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #1
   ret <3 x float> %sub
 }
 
-define <3 x double> @constrained_vector_fsub_v3f64() {
+define <3 x double> @constrained_vector_fsub_v3f64() #0 {
 ; PC64LE-LABEL: constrained_vector_fsub_v3f64:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    addis 3, 2, .LCPI23_1@toc@ha
@@ -1276,11 +1242,11 @@ entry:
                          double 0xFFEFFFFFFFFFFFFF>,
            <3 x double> <double 2.0, double 1.0, double 0.0>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #1
   ret <3 x double> %sub
 }
 
-define <4 x double> @constrained_vector_fsub_v4f64() {
+define <4 x double> @constrained_vector_fsub_v4f64() #0 {
 ; PC64LE-LABEL: constrained_vector_fsub_v4f64:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    addis 3, 2, .LCPI24_0@toc@ha
@@ -1320,11 +1286,11 @@ entry:
            <4 x double> <double 1.000000e+00, double 1.000000e-01,
                          double 2.000000e+00, double 2.000000e-01>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #1
   ret <4 x double> %sub
 }
 
-define <1 x float> @constrained_vector_sqrt_v1f32() {
+define <1 x float> @constrained_vector_sqrt_v1f32() #0 {
 ; PC64LE-LABEL: constrained_vector_sqrt_v1f32:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    addis 3, 2, .LCPI25_0@toc@ha
@@ -1346,11 +1312,11 @@ entry:
   %sqrt = call <1 x float> @llvm.experimental.constrained.sqrt.v1f32(
                               <1 x float> <float 42.0>,
                               metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #1
   ret <1 x float> %sqrt
 }
 
-define <2 x double> @constrained_vector_sqrt_v2f64() {
+define <2 x double> @constrained_vector_sqrt_v2f64() #0 {
 ; PC64LE-LABEL: constrained_vector_sqrt_v2f64:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    addis 3, 2, .LCPI26_0@toc@ha
@@ -1371,11 +1337,11 @@ entry:
   %sqrt = call <2 x double> @llvm.experimental.constrained.sqrt.v2f64(
                               <2 x double> <double 42.0, double 42.1>,
                               metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #1
   ret <2 x double> %sqrt
 }
 
-define <3 x float> @constrained_vector_sqrt_v3f32() {
+define <3 x float> @constrained_vector_sqrt_v3f32() #0 {
 ; PC64LE-LABEL: constrained_vector_sqrt_v3f32:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    addis 3, 2, .LCPI27_2@toc@ha
@@ -1427,11 +1393,11 @@ entry:
   %sqrt = call <3 x float> @llvm.experimental.constrained.sqrt.v3f32(
                               <3 x float> <float 42.0, float 43.0, float 44.0>,
                               metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #1
   ret <3 x float> %sqrt
 }
 
-define <3 x double> @constrained_vector_sqrt_v3f64() {
+define <3 x double> @constrained_vector_sqrt_v3f64() #0 {
 ; PC64LE-LABEL: constrained_vector_sqrt_v3f64:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    addis 3, 2, .LCPI28_1@toc@ha
@@ -1464,11 +1430,11 @@ entry:
   %sqrt = call <3 x double> @llvm.experimental.constrained.sqrt.v3f64(
                           <3 x double> <double 42.0, double 42.1, double 42.2>,
                           metadata !"round.dynamic",
-                          metadata !"fpexcept.strict")
+                          metadata !"fpexcept.strict") #1
   ret <3 x double> %sqrt
 }
 
-define <4 x double> @constrained_vector_sqrt_v4f64() {
+define <4 x double> @constrained_vector_sqrt_v4f64() #0 {
 ; PC64LE-LABEL: constrained_vector_sqrt_v4f64:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    addis 3, 2, .LCPI29_0@toc@ha
@@ -1499,18 +1465,16 @@ define <4 x double> @constrained_vector_sqrt_v4f64() {
                               <4 x double> <double 42.0, double 42.1,
                                             double 42.2, double 42.3>,
                               metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #1
   ret <4 x double> %sqrt
 }
 
-define <1 x float> @constrained_vector_pow_v1f32() {
+define <1 x float> @constrained_vector_pow_v1f32() #0 {
 ; PC64LE-LABEL: constrained_vector_pow_v1f32:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    mflr 0
 ; PC64LE-NEXT:    std 0, 16(1)
 ; PC64LE-NEXT:    stdu 1, -32(1)
-; PC64LE-NEXT:    .cfi_def_cfa_offset 32
-; PC64LE-NEXT:    .cfi_offset lr, 16
 ; PC64LE-NEXT:    addis 3, 2, .LCPI30_0@toc@ha
 ; PC64LE-NEXT:    addis 4, 2, .LCPI30_1@toc@ha
 ; PC64LE-NEXT:    lfs 1, .LCPI30_0@toc@l(3)
@@ -1529,8 +1493,6 @@ define <1 x float> @constrained_vector_pow_v1f32() {
 ; PC64LE9-NEXT:    mflr 0
 ; PC64LE9-NEXT:    std 0, 16(1)
 ; PC64LE9-NEXT:    stdu 1, -32(1)
-; PC64LE9-NEXT:    .cfi_def_cfa_offset 32
-; PC64LE9-NEXT:    .cfi_offset lr, 16
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI30_0@toc@ha
 ; PC64LE9-NEXT:    lfs 1, .LCPI30_0@toc@l(3)
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI30_1@toc@ha
@@ -1548,17 +1510,14 @@ entry:
                              <1 x float> <float 42.0>,
                              <1 x float> <float 3.0>,
                              metadata !"round.dynamic",
-                             metadata !"fpexcept.strict")
+                             metadata !"fpexcept.strict") #1
   ret <1 x float> %pow
 }
 
-define <2 x double> @constrained_vector_pow_v2f64() {
+define <2 x double> @constrained_vector_pow_v2f64() #0 {
 ; PC64LE-LABEL: constrained_vector_pow_v2f64:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    mflr 0
-; PC64LE-NEXT:    .cfi_def_cfa_offset 80
-; PC64LE-NEXT:    .cfi_offset lr, 16
-; PC64LE-NEXT:    .cfi_offset f31, -8
 ; PC64LE-NEXT:    stfd 31, -8(1) # 8-byte Folded Spill
 ; PC64LE-NEXT:    std 0, 16(1)
 ; PC64LE-NEXT:    stdu 1, -80(1)
@@ -1583,16 +1542,13 @@ define <2 x double> @constrained_vector_pow_v2f64() {
 ; PC64LE-NEXT:    xxmrghd 34, 1, 0
 ; PC64LE-NEXT:    addi 1, 1, 80
 ; PC64LE-NEXT:    ld 0, 16(1)
-; PC64LE-NEXT:    mtlr 0
 ; PC64LE-NEXT:    lfd 31, -8(1) # 8-byte Folded Reload
+; PC64LE-NEXT:    mtlr 0
 ; PC64LE-NEXT:    blr
 ;
 ; PC64LE9-LABEL: constrained_vector_pow_v2f64:
 ; PC64LE9:       # %bb.0: # %entry
 ; PC64LE9-NEXT:    mflr 0
-; PC64LE9-NEXT:    .cfi_def_cfa_offset 64
-; PC64LE9-NEXT:    .cfi_offset lr, 16
-; PC64LE9-NEXT:    .cfi_offset f31, -8
 ; PC64LE9-NEXT:    stfd 31, -8(1) # 8-byte Folded Spill
 ; PC64LE9-NEXT:    std 0, 16(1)
 ; PC64LE9-NEXT:    stdu 1, -64(1)
@@ -1615,27 +1571,22 @@ define <2 x double> @constrained_vector_pow_v2f64() {
 ; PC64LE9-NEXT:    xxmrghd 34, 1, 0
 ; PC64LE9-NEXT:    addi 1, 1, 64
 ; PC64LE9-NEXT:    ld 0, 16(1)
-; PC64LE9-NEXT:    mtlr 0
 ; PC64LE9-NEXT:    lfd 31, -8(1) # 8-byte Folded Reload
+; PC64LE9-NEXT:    mtlr 0
 ; PC64LE9-NEXT:    blr
 entry:
   %pow = call <2 x double> @llvm.experimental.constrained.pow.v2f64(
                              <2 x double> <double 42.1, double 42.2>,
                              <2 x double> <double 3.0, double 3.0>,
                              metadata !"round.dynamic",
-                             metadata !"fpexcept.strict")
+                             metadata !"fpexcept.strict") #1
   ret <2 x double> %pow
 }
 
-define <3 x float> @constrained_vector_pow_v3f32() {
+define <3 x float> @constrained_vector_pow_v3f32() #0 {
 ; PC64LE-LABEL: constrained_vector_pow_v3f32:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    mflr 0
-; PC64LE-NEXT:    .cfi_def_cfa_offset 64
-; PC64LE-NEXT:    .cfi_offset lr, 16
-; PC64LE-NEXT:    .cfi_offset f29, -24
-; PC64LE-NEXT:    .cfi_offset f30, -16
-; PC64LE-NEXT:    .cfi_offset f31, -8
 ; PC64LE-NEXT:    stfd 29, -24(1) # 8-byte Folded Spill
 ; PC64LE-NEXT:    stfd 30, -16(1) # 8-byte Folded Spill
 ; PC64LE-NEXT:    stfd 31, -8(1) # 8-byte Folded Spill
@@ -1673,20 +1624,15 @@ define <3 x float> @constrained_vector_pow_v3f32() {
 ; PC64LE-NEXT:    vperm 2, 3, 2, 4
 ; PC64LE-NEXT:    addi 1, 1, 64
 ; PC64LE-NEXT:    ld 0, 16(1)
-; PC64LE-NEXT:    mtlr 0
 ; PC64LE-NEXT:    lfd 31, -8(1) # 8-byte Folded Reload
 ; PC64LE-NEXT:    lfd 30, -16(1) # 8-byte Folded Reload
 ; PC64LE-NEXT:    lfd 29, -24(1) # 8-byte Folded Reload
+; PC64LE-NEXT:    mtlr 0
 ; PC64LE-NEXT:    blr
 ;
 ; PC64LE9-LABEL: constrained_vector_pow_v3f32:
 ; PC64LE9:       # %bb.0: # %entry
 ; PC64LE9-NEXT:    mflr 0
-; PC64LE9-NEXT:    .cfi_def_cfa_offset 64
-; PC64LE9-NEXT:    .cfi_offset lr, 16
-; PC64LE9-NEXT:    .cfi_offset f29, -24
-; PC64LE9-NEXT:    .cfi_offset f30, -16
-; PC64LE9-NEXT:    .cfi_offset f31, -8
 ; PC64LE9-NEXT:    stfd 29, -24(1) # 8-byte Folded Spill
 ; PC64LE9-NEXT:    stfd 30, -16(1) # 8-byte Folded Spill
 ; PC64LE9-NEXT:    stfd 31, -8(1) # 8-byte Folded Spill
@@ -1724,9 +1670,9 @@ define <3 x float> @constrained_vector_pow_v3f32() {
 ; PC64LE9-NEXT:    vperm 2, 3, 2, 4
 ; PC64LE9-NEXT:    addi 1, 1, 64
 ; PC64LE9-NEXT:    ld 0, 16(1)
-; PC64LE9-NEXT:    mtlr 0
 ; PC64LE9-NEXT:    lfd 31, -8(1) # 8-byte Folded Reload
 ; PC64LE9-NEXT:    lfd 30, -16(1) # 8-byte Folded Reload
+; PC64LE9-NEXT:    mtlr 0
 ; PC64LE9-NEXT:    lfd 29, -24(1) # 8-byte Folded Reload
 ; PC64LE9-NEXT:    blr
 entry:
@@ -1734,20 +1680,16 @@ entry:
                              <3 x float> <float 42.0, float 43.0, float 44.0>,
                              <3 x float> <float 3.0, float 3.0, float 3.0>,
                              metadata !"round.dynamic",
-                             metadata !"fpexcept.strict")
+                             metadata !"fpexcept.strict") #1
   ret <3 x float> %pow
 }
 
-define <3 x double> @constrained_vector_pow_v3f64() {
+define <3 x double> @constrained_vector_pow_v3f64() #0 {
 ; PC64LE-LABEL: constrained_vector_pow_v3f64:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    mflr 0
 ; PC64LE-NEXT:    std 0, 16(1)
 ; PC64LE-NEXT:    stdu 1, -96(1)
-; PC64LE-NEXT:    .cfi_def_cfa_offset 96
-; PC64LE-NEXT:    .cfi_offset lr, 16
-; PC64LE-NEXT:    .cfi_offset f31, -8
-; PC64LE-NEXT:    .cfi_offset v31, -32
 ; PC64LE-NEXT:    addis 4, 2, .LCPI33_1@toc@ha
 ; PC64LE-NEXT:    stfd 31, 88(1) # 8-byte Folded Spill
 ; PC64LE-NEXT:    li 3, 64
@@ -1775,12 +1717,13 @@ define <3 x double> @constrained_vector_pow_v3f64() {
 ; PC64LE-NEXT:    lfd 1, .LCPI33_3@toc@l(3)
 ; PC64LE-NEXT:    bl pow
 ; PC64LE-NEXT:    nop
+; PC64LE-NEXT:    xxswapd 0, 63
 ; PC64LE-NEXT:    li 3, 64
-; PC64LE-NEXT:    fmr 3, 1
-; PC64LE-NEXT:    xxlor 1, 63, 63
-; PC64LE-NEXT:    lfd 31, 88(1) # 8-byte Folded Reload
 ; PC64LE-NEXT:    xxlor 2, 63, 63
+; PC64LE-NEXT:    lfd 31, 88(1) # 8-byte Folded Reload
 ; PC64LE-NEXT:    lxvd2x 63, 1, 3 # 16-byte Folded Reload
+; PC64LE-NEXT:    fmr 3, 1
+; PC64LE-NEXT:    fmr 1, 0
 ; PC64LE-NEXT:    addi 1, 1, 96
 ; PC64LE-NEXT:    ld 0, 16(1)
 ; PC64LE-NEXT:    mtlr 0
@@ -1791,10 +1734,6 @@ define <3 x double> @constrained_vector_pow_v3f64() {
 ; PC64LE9-NEXT:    mflr 0
 ; PC64LE9-NEXT:    std 0, 16(1)
 ; PC64LE9-NEXT:    stdu 1, -80(1)
-; PC64LE9-NEXT:    .cfi_def_cfa_offset 80
-; PC64LE9-NEXT:    .cfi_offset lr, 16
-; PC64LE9-NEXT:    .cfi_offset f31, -8
-; PC64LE9-NEXT:    .cfi_offset v31, -32
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI33_0@toc@ha
 ; PC64LE9-NEXT:    lfs 1, .LCPI33_0@toc@l(3)
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI33_1@toc@ha
@@ -1820,10 +1759,11 @@ define <3 x double> @constrained_vector_pow_v3f64() {
 ; PC64LE9-NEXT:    bl pow
 ; PC64LE9-NEXT:    nop
 ; PC64LE9-NEXT:    fmr 3, 1
-; PC64LE9-NEXT:    xscpsgndp 1, 63, 63
+; PC64LE9-NEXT:    xxswapd 1, 63
 ; PC64LE9-NEXT:    xscpsgndp 2, 63, 63
 ; PC64LE9-NEXT:    lxv 63, 48(1) # 16-byte Folded Reload
 ; PC64LE9-NEXT:    lfd 31, 72(1) # 8-byte Folded Reload
+; PC64LE9-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
 ; PC64LE9-NEXT:    addi 1, 1, 80
 ; PC64LE9-NEXT:    ld 0, 16(1)
 ; PC64LE9-NEXT:    mtlr 0
@@ -1833,20 +1773,16 @@ entry:
                           <3 x double> <double 42.0, double 42.1, double 42.2>,
                           <3 x double> <double 3.0, double 3.0, double 3.0>,
                           metadata !"round.dynamic",
-                          metadata !"fpexcept.strict")
+                          metadata !"fpexcept.strict") #1
   ret <3 x double> %pow
 }
 
-define <4 x double> @constrained_vector_pow_v4f64() {
+define <4 x double> @constrained_vector_pow_v4f64() #0 {
 ; PC64LE-LABEL: constrained_vector_pow_v4f64:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    mflr 0
 ; PC64LE-NEXT:    std 0, 16(1)
 ; PC64LE-NEXT:    stdu 1, -96(1)
-; PC64LE-NEXT:    .cfi_def_cfa_offset 96
-; PC64LE-NEXT:    .cfi_offset lr, 16
-; PC64LE-NEXT:    .cfi_offset f31, -8
-; PC64LE-NEXT:    .cfi_offset v31, -32
 ; PC64LE-NEXT:    addis 4, 2, .LCPI34_1@toc@ha
 ; PC64LE-NEXT:    stfd 31, 88(1) # 8-byte Folded Spill
 ; PC64LE-NEXT:    li 3, 64
@@ -1900,10 +1836,6 @@ define <4 x double> @constrained_vector_pow_v4f64() {
 ; PC64LE9-NEXT:    mflr 0
 ; PC64LE9-NEXT:    std 0, 16(1)
 ; PC64LE9-NEXT:    stdu 1, -80(1)
-; PC64LE9-NEXT:    .cfi_def_cfa_offset 80
-; PC64LE9-NEXT:    .cfi_offset lr, 16
-; PC64LE9-NEXT:    .cfi_offset f31, -8
-; PC64LE9-NEXT:    .cfi_offset v31, -32
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI34_0@toc@ha
 ; PC64LE9-NEXT:    lfd 1, .LCPI34_0@toc@l(3)
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI34_1@toc@ha
@@ -1952,18 +1884,16 @@ entry:
                              <4 x double> <double 3.0, double 3.0,
                                            double 3.0, double 3.0>,
                              metadata !"round.dynamic",
-                             metadata !"fpexcept.strict")
+                             metadata !"fpexcept.strict") #1
   ret <4 x double> %pow
 }
 
-define <1 x float> @constrained_vector_powi_v1f32() {
+define <1 x float> @constrained_vector_powi_v1f32() #0 {
 ; PC64LE-LABEL: constrained_vector_powi_v1f32:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    mflr 0
 ; PC64LE-NEXT:    std 0, 16(1)
 ; PC64LE-NEXT:    stdu 1, -32(1)
-; PC64LE-NEXT:    .cfi_def_cfa_offset 32
-; PC64LE-NEXT:    .cfi_offset lr, 16
 ; PC64LE-NEXT:    addis 3, 2, .LCPI35_0@toc@ha
 ; PC64LE-NEXT:    li 4, 3
 ; PC64LE-NEXT:    lfs 1, .LCPI35_0@toc@l(3)
@@ -1981,8 +1911,6 @@ define <1 x float> @constrained_vector_powi_v1f32() {
 ; PC64LE9-NEXT:    mflr 0
 ; PC64LE9-NEXT:    std 0, 16(1)
 ; PC64LE9-NEXT:    stdu 1, -32(1)
-; PC64LE9-NEXT:    .cfi_def_cfa_offset 32
-; PC64LE9-NEXT:    .cfi_offset lr, 16
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI35_0@toc@ha
 ; PC64LE9-NEXT:    lfs 1, .LCPI35_0@toc@l(3)
 ; PC64LE9-NEXT:    li 4, 3
@@ -1999,18 +1927,16 @@ entry:
                               <1 x float> <float 42.0>,
                               i32 3,
                               metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #1
   ret <1 x float> %powi
 }
 
-define <2 x double> @constrained_vector_powi_v2f64() {
+define <2 x double> @constrained_vector_powi_v2f64() #0 {
 ; PC64LE-LABEL: constrained_vector_powi_v2f64:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    mflr 0
 ; PC64LE-NEXT:    std 0, 16(1)
 ; PC64LE-NEXT:    stdu 1, -64(1)
-; PC64LE-NEXT:    .cfi_def_cfa_offset 64
-; PC64LE-NEXT:    .cfi_offset lr, 16
 ; PC64LE-NEXT:    addis 3, 2, .LCPI36_0@toc@ha
 ; PC64LE-NEXT:    li 4, 3
 ; PC64LE-NEXT:    lfd 1, .LCPI36_0@toc@l(3)
@@ -2038,8 +1964,6 @@ define <2 x double> @constrained_vector_powi_v2f64() {
 ; PC64LE9-NEXT:    mflr 0
 ; PC64LE9-NEXT:    std 0, 16(1)
 ; PC64LE9-NEXT:    stdu 1, -48(1)
-; PC64LE9-NEXT:    .cfi_def_cfa_offset 48
-; PC64LE9-NEXT:    .cfi_offset lr, 16
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI36_0@toc@ha
 ; PC64LE9-NEXT:    lfd 1, .LCPI36_0@toc@l(3)
 ; PC64LE9-NEXT:    li 4, 3
@@ -2064,20 +1988,16 @@ entry:
                               <2 x double> <double 42.1, double 42.2>,
                               i32 3,
                               metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #1
   ret <2 x double> %powi
 }
 
-define <3 x float> @constrained_vector_powi_v3f32() {
+define <3 x float> @constrained_vector_powi_v3f32() #0 {
 ;
 ;
 ; PC64LE-LABEL: constrained_vector_powi_v3f32:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    mflr 0
-; PC64LE-NEXT:    .cfi_def_cfa_offset 48
-; PC64LE-NEXT:    .cfi_offset lr, 16
-; PC64LE-NEXT:    .cfi_offset f30, -16
-; PC64LE-NEXT:    .cfi_offset f31, -8
 ; PC64LE-NEXT:    stfd 30, -16(1) # 8-byte Folded Spill
 ; PC64LE-NEXT:    stfd 31, -8(1) # 8-byte Folded Spill
 ; PC64LE-NEXT:    std 0, 16(1)
@@ -2112,18 +2032,14 @@ define <3 x float> @constrained_vector_powi_v3f32() {
 ; PC64LE-NEXT:    vperm 2, 3, 2, 4
 ; PC64LE-NEXT:    addi 1, 1, 48
 ; PC64LE-NEXT:    ld 0, 16(1)
-; PC64LE-NEXT:    mtlr 0
 ; PC64LE-NEXT:    lfd 31, -8(1) # 8-byte Folded Reload
 ; PC64LE-NEXT:    lfd 30, -16(1) # 8-byte Folded Reload
+; PC64LE-NEXT:    mtlr 0
 ; PC64LE-NEXT:    blr
 ;
 ; PC64LE9-LABEL: constrained_vector_powi_v3f32:
 ; PC64LE9:       # %bb.0: # %entry
 ; PC64LE9-NEXT:    mflr 0
-; PC64LE9-NEXT:    .cfi_def_cfa_offset 48
-; PC64LE9-NEXT:    .cfi_offset lr, 16
-; PC64LE9-NEXT:    .cfi_offset f30, -16
-; PC64LE9-NEXT:    .cfi_offset f31, -8
 ; PC64LE9-NEXT:    stfd 30, -16(1) # 8-byte Folded Spill
 ; PC64LE9-NEXT:    stfd 31, -8(1) # 8-byte Folded Spill
 ; PC64LE9-NEXT:    std 0, 16(1)
@@ -2158,28 +2074,25 @@ define <3 x float> @constrained_vector_powi_v3f32() {
 ; PC64LE9-NEXT:    vperm 2, 3, 2, 4
 ; PC64LE9-NEXT:    addi 1, 1, 48
 ; PC64LE9-NEXT:    ld 0, 16(1)
-; PC64LE9-NEXT:    mtlr 0
 ; PC64LE9-NEXT:    lfd 31, -8(1) # 8-byte Folded Reload
 ; PC64LE9-NEXT:    lfd 30, -16(1) # 8-byte Folded Reload
+; PC64LE9-NEXT:    mtlr 0
 ; PC64LE9-NEXT:    blr
 entry:
   %powi = call <3 x float> @llvm.experimental.constrained.powi.v3f32(
                               <3 x float> <float 42.0, float 43.0, float 44.0>,
                               i32 3,
                               metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #1
   ret <3 x float> %powi
 }
 
-define <3 x double> @constrained_vector_powi_v3f64() {
+define <3 x double> @constrained_vector_powi_v3f64() #0 {
 ; PC64LE-LABEL: constrained_vector_powi_v3f64:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    mflr 0
 ; PC64LE-NEXT:    std 0, 16(1)
 ; PC64LE-NEXT:    stdu 1, -80(1)
-; PC64LE-NEXT:    .cfi_def_cfa_offset 80
-; PC64LE-NEXT:    .cfi_offset lr, 16
-; PC64LE-NEXT:    .cfi_offset v31, -16
 ; PC64LE-NEXT:    li 3, 64
 ; PC64LE-NEXT:    li 4, 3
 ; PC64LE-NEXT:    stxvd2x 63, 1, 3 # 16-byte Folded Spill
@@ -2204,11 +2117,12 @@ define <3 x double> @constrained_vector_powi_v3f64() {
 ; PC64LE-NEXT:    lfd 1, .LCPI38_2@toc@l(3)
 ; PC64LE-NEXT:    bl __powidf2
 ; PC64LE-NEXT:    nop
+; PC64LE-NEXT:    xxswapd 0, 63
 ; PC64LE-NEXT:    li 3, 64
-; PC64LE-NEXT:    fmr 3, 1
-; PC64LE-NEXT:    xxlor 1, 63, 63
 ; PC64LE-NEXT:    xxlor 2, 63, 63
 ; PC64LE-NEXT:    lxvd2x 63, 1, 3 # 16-byte Folded Reload
+; PC64LE-NEXT:    fmr 3, 1
+; PC64LE-NEXT:    fmr 1, 0
 ; PC64LE-NEXT:    addi 1, 1, 80
 ; PC64LE-NEXT:    ld 0, 16(1)
 ; PC64LE-NEXT:    mtlr 0
@@ -2219,9 +2133,6 @@ define <3 x double> @constrained_vector_powi_v3f64() {
 ; PC64LE9-NEXT:    mflr 0
 ; PC64LE9-NEXT:    std 0, 16(1)
 ; PC64LE9-NEXT:    stdu 1, -64(1)
-; PC64LE9-NEXT:    .cfi_def_cfa_offset 64
-; PC64LE9-NEXT:    .cfi_offset lr, 16
-; PC64LE9-NEXT:    .cfi_offset v31, -16
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI38_0@toc@ha
 ; PC64LE9-NEXT:    lfd 1, .LCPI38_0@toc@l(3)
 ; PC64LE9-NEXT:    li 4, 3
@@ -2244,9 +2155,10 @@ define <3 x double> @constrained_vector_powi_v3f64() {
 ; PC64LE9-NEXT:    bl __powidf2
 ; PC64LE9-NEXT:    nop
 ; PC64LE9-NEXT:    fmr 3, 1
-; PC64LE9-NEXT:    xscpsgndp 1, 63, 63
+; PC64LE9-NEXT:    xxswapd 1, 63
 ; PC64LE9-NEXT:    xscpsgndp 2, 63, 63
 ; PC64LE9-NEXT:    lxv 63, 48(1) # 16-byte Folded Reload
+; PC64LE9-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
 ; PC64LE9-NEXT:    addi 1, 1, 64
 ; PC64LE9-NEXT:    ld 0, 16(1)
 ; PC64LE9-NEXT:    mtlr 0
@@ -2256,19 +2168,16 @@ entry:
                           <3 x double> <double 42.0, double 42.1, double 42.2>,
                           i32 3,
                           metadata !"round.dynamic",
-                          metadata !"fpexcept.strict")
+                          metadata !"fpexcept.strict") #1
   ret <3 x double> %powi
 }
 
-define <4 x double> @constrained_vector_powi_v4f64() {
+define <4 x double> @constrained_vector_powi_v4f64() #0 {
 ; PC64LE-LABEL: constrained_vector_powi_v4f64:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    mflr 0
 ; PC64LE-NEXT:    std 0, 16(1)
 ; PC64LE-NEXT:    stdu 1, -80(1)
-; PC64LE-NEXT:    .cfi_def_cfa_offset 80
-; PC64LE-NEXT:    .cfi_offset lr, 16
-; PC64LE-NEXT:    .cfi_offset v31, -16
 ; PC64LE-NEXT:    li 3, 64
 ; PC64LE-NEXT:    li 4, 3
 ; PC64LE-NEXT:    stxvd2x 63, 1, 3 # 16-byte Folded Spill
@@ -2318,9 +2227,6 @@ define <4 x double> @constrained_vector_powi_v4f64() {
 ; PC64LE9-NEXT:    mflr 0
 ; PC64LE9-NEXT:    std 0, 16(1)
 ; PC64LE9-NEXT:    stdu 1, -64(1)
-; PC64LE9-NEXT:    .cfi_def_cfa_offset 64
-; PC64LE9-NEXT:    .cfi_offset lr, 16
-; PC64LE9-NEXT:    .cfi_offset v31, -16
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI39_0@toc@ha
 ; PC64LE9-NEXT:    lfd 1, .LCPI39_0@toc@l(3)
 ; PC64LE9-NEXT:    li 4, 3
@@ -2364,18 +2270,16 @@ entry:
                                             double 42.3, double 42.4>,
                               i32 3,
                               metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #1
   ret <4 x double> %powi
 }
 
-define <1 x float> @constrained_vector_sin_v1f32() {
+define <1 x float> @constrained_vector_sin_v1f32() #0 {
 ; PC64LE-LABEL: constrained_vector_sin_v1f32:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    mflr 0
 ; PC64LE-NEXT:    std 0, 16(1)
 ; PC64LE-NEXT:    stdu 1, -32(1)
-; PC64LE-NEXT:    .cfi_def_cfa_offset 32
-; PC64LE-NEXT:    .cfi_offset lr, 16
 ; PC64LE-NEXT:    addis 3, 2, .LCPI40_0@toc@ha
 ; PC64LE-NEXT:    lfs 1, .LCPI40_0@toc@l(3)
 ; PC64LE-NEXT:    bl sinf
@@ -2392,8 +2296,6 @@ define <1 x float> @constrained_vector_sin_v1f32() {
 ; PC64LE9-NEXT:    mflr 0
 ; PC64LE9-NEXT:    std 0, 16(1)
 ; PC64LE9-NEXT:    stdu 1, -32(1)
-; PC64LE9-NEXT:    .cfi_def_cfa_offset 32
-; PC64LE9-NEXT:    .cfi_offset lr, 16
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI40_0@toc@ha
 ; PC64LE9-NEXT:    lfs 1, .LCPI40_0@toc@l(3)
 ; PC64LE9-NEXT:    bl sinf
@@ -2408,18 +2310,16 @@ entry:
   %sin = call <1 x float> @llvm.experimental.constrained.sin.v1f32(
                              <1 x float> <float 42.0>,
                              metadata !"round.dynamic",
-                             metadata !"fpexcept.strict")
+                             metadata !"fpexcept.strict") #1
   ret <1 x float> %sin
 }
 
-define <2 x double> @constrained_vector_sin_v2f64() {
+define <2 x double> @constrained_vector_sin_v2f64() #0 {
 ; PC64LE-LABEL: constrained_vector_sin_v2f64:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    mflr 0
 ; PC64LE-NEXT:    std 0, 16(1)
 ; PC64LE-NEXT:    stdu 1, -64(1)
-; PC64LE-NEXT:    .cfi_def_cfa_offset 64
-; PC64LE-NEXT:    .cfi_offset lr, 16
 ; PC64LE-NEXT:    addis 3, 2, .LCPI41_0@toc@ha
 ; PC64LE-NEXT:    lfd 1, .LCPI41_0@toc@l(3)
 ; PC64LE-NEXT:    bl sin
@@ -2445,8 +2345,6 @@ define <2 x double> @constrained_vector_sin_v2f64() {
 ; PC64LE9-NEXT:    mflr 0
 ; PC64LE9-NEXT:    std 0, 16(1)
 ; PC64LE9-NEXT:    stdu 1, -48(1)
-; PC64LE9-NEXT:    .cfi_def_cfa_offset 48
-; PC64LE9-NEXT:    .cfi_offset lr, 16
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI41_0@toc@ha
 ; PC64LE9-NEXT:    lfd 1, .LCPI41_0@toc@l(3)
 ; PC64LE9-NEXT:    bl sin
@@ -2468,18 +2366,14 @@ entry:
   %sin = call <2 x double> @llvm.experimental.constrained.sin.v2f64(
                              <2 x double> <double 42.0, double 42.1>,
                              metadata !"round.dynamic",
-                             metadata !"fpexcept.strict")
+                             metadata !"fpexcept.strict") #1
   ret <2 x double> %sin
 }
 
-define <3 x float> @constrained_vector_sin_v3f32() {
+define <3 x float> @constrained_vector_sin_v3f32() #0 {
 ; PC64LE-LABEL: constrained_vector_sin_v3f32:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    mflr 0
-; PC64LE-NEXT:    .cfi_def_cfa_offset 48
-; PC64LE-NEXT:    .cfi_offset lr, 16
-; PC64LE-NEXT:    .cfi_offset f30, -16
-; PC64LE-NEXT:    .cfi_offset f31, -8
 ; PC64LE-NEXT:    stfd 30, -16(1) # 8-byte Folded Spill
 ; PC64LE-NEXT:    stfd 31, -8(1) # 8-byte Folded Spill
 ; PC64LE-NEXT:    std 0, 16(1)
@@ -2511,18 +2405,14 @@ define <3 x float> @constrained_vector_sin_v3f32() {
 ; PC64LE-NEXT:    vperm 2, 4, 2, 3
 ; PC64LE-NEXT:    addi 1, 1, 48
 ; PC64LE-NEXT:    ld 0, 16(1)
-; PC64LE-NEXT:    mtlr 0
 ; PC64LE-NEXT:    lfd 31, -8(1) # 8-byte Folded Reload
 ; PC64LE-NEXT:    lfd 30, -16(1) # 8-byte Folded Reload
+; PC64LE-NEXT:    mtlr 0
 ; PC64LE-NEXT:    blr
 ;
 ; PC64LE9-LABEL: constrained_vector_sin_v3f32:
 ; PC64LE9:       # %bb.0: # %entry
 ; PC64LE9-NEXT:    mflr 0
-; PC64LE9-NEXT:    .cfi_def_cfa_offset 48
-; PC64LE9-NEXT:    .cfi_offset lr, 16
-; PC64LE9-NEXT:    .cfi_offset f30, -16
-; PC64LE9-NEXT:    .cfi_offset f31, -8
 ; PC64LE9-NEXT:    stfd 30, -16(1) # 8-byte Folded Spill
 ; PC64LE9-NEXT:    stfd 31, -8(1) # 8-byte Folded Spill
 ; PC64LE9-NEXT:    std 0, 16(1)
@@ -2554,27 +2444,24 @@ define <3 x float> @constrained_vector_sin_v3f32() {
 ; PC64LE9-NEXT:    vperm 2, 4, 2, 3
 ; PC64LE9-NEXT:    addi 1, 1, 48
 ; PC64LE9-NEXT:    ld 0, 16(1)
-; PC64LE9-NEXT:    mtlr 0
 ; PC64LE9-NEXT:    lfd 31, -8(1) # 8-byte Folded Reload
 ; PC64LE9-NEXT:    lfd 30, -16(1) # 8-byte Folded Reload
+; PC64LE9-NEXT:    mtlr 0
 ; PC64LE9-NEXT:    blr
 entry:
   %sin = call <3 x float> @llvm.experimental.constrained.sin.v3f32(
                               <3 x float> <float 42.0, float 43.0, float 44.0>,
                               metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #1
   ret <3 x float> %sin
 }
 
-define <3 x double> @constrained_vector_sin_v3f64() {
+define <3 x double> @constrained_vector_sin_v3f64() #0 {
 ; PC64LE-LABEL: constrained_vector_sin_v3f64:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    mflr 0
 ; PC64LE-NEXT:    std 0, 16(1)
 ; PC64LE-NEXT:    stdu 1, -80(1)
-; PC64LE-NEXT:    .cfi_def_cfa_offset 80
-; PC64LE-NEXT:    .cfi_offset lr, 16
-; PC64LE-NEXT:    .cfi_offset v31, -16
 ; PC64LE-NEXT:    li 3, 64
 ; PC64LE-NEXT:    stxvd2x 63, 1, 3 # 16-byte Folded Spill
 ; PC64LE-NEXT:    addis 3, 2, .LCPI43_0@toc@ha
@@ -2596,11 +2483,12 @@ define <3 x double> @constrained_vector_sin_v3f64() {
 ; PC64LE-NEXT:    lfd 1, .LCPI43_2@toc@l(3)
 ; PC64LE-NEXT:    bl sin
 ; PC64LE-NEXT:    nop
+; PC64LE-NEXT:    xxswapd 0, 63
 ; PC64LE-NEXT:    li 3, 64
-; PC64LE-NEXT:    fmr 3, 1
-; PC64LE-NEXT:    xxlor 1, 63, 63
 ; PC64LE-NEXT:    xxlor 2, 63, 63
 ; PC64LE-NEXT:    lxvd2x 63, 1, 3 # 16-byte Folded Reload
+; PC64LE-NEXT:    fmr 3, 1
+; PC64LE-NEXT:    fmr 1, 0
 ; PC64LE-NEXT:    addi 1, 1, 80
 ; PC64LE-NEXT:    ld 0, 16(1)
 ; PC64LE-NEXT:    mtlr 0
@@ -2611,9 +2499,6 @@ define <3 x double> @constrained_vector_sin_v3f64() {
 ; PC64LE9-NEXT:    mflr 0
 ; PC64LE9-NEXT:    std 0, 16(1)
 ; PC64LE9-NEXT:    stdu 1, -64(1)
-; PC64LE9-NEXT:    .cfi_def_cfa_offset 64
-; PC64LE9-NEXT:    .cfi_offset lr, 16
-; PC64LE9-NEXT:    .cfi_offset v31, -16
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI43_0@toc@ha
 ; PC64LE9-NEXT:    lfd 1, .LCPI43_0@toc@l(3)
 ; PC64LE9-NEXT:    stxv 63, 48(1) # 16-byte Folded Spill
@@ -2633,9 +2518,10 @@ define <3 x double> @constrained_vector_sin_v3f64() {
 ; PC64LE9-NEXT:    bl sin
 ; PC64LE9-NEXT:    nop
 ; PC64LE9-NEXT:    fmr 3, 1
-; PC64LE9-NEXT:    xscpsgndp 1, 63, 63
+; PC64LE9-NEXT:    xxswapd 1, 63
 ; PC64LE9-NEXT:    xscpsgndp 2, 63, 63
 ; PC64LE9-NEXT:    lxv 63, 48(1) # 16-byte Folded Reload
+; PC64LE9-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
 ; PC64LE9-NEXT:    addi 1, 1, 64
 ; PC64LE9-NEXT:    ld 0, 16(1)
 ; PC64LE9-NEXT:    mtlr 0
@@ -2644,19 +2530,16 @@ entry:
   %sin = call <3 x double> @llvm.experimental.constrained.sin.v3f64(
                           <3 x double> <double 42.0, double 42.1, double 42.2>,
                           metadata !"round.dynamic",
-                          metadata !"fpexcept.strict")
+                          metadata !"fpexcept.strict") #1
   ret <3 x double> %sin
 }
 
-define <4 x double> @constrained_vector_sin_v4f64() {
+define <4 x double> @constrained_vector_sin_v4f64() #0 {
 ; PC64LE-LABEL: constrained_vector_sin_v4f64:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    mflr 0
 ; PC64LE-NEXT:    std 0, 16(1)
 ; PC64LE-NEXT:    stdu 1, -80(1)
-; PC64LE-NEXT:    .cfi_def_cfa_offset 80
-; PC64LE-NEXT:    .cfi_offset lr, 16
-; PC64LE-NEXT:    .cfi_offset v31, -16
 ; PC64LE-NEXT:    li 3, 64
 ; PC64LE-NEXT:    stxvd2x 63, 1, 3 # 16-byte Folded Spill
 ; PC64LE-NEXT:    addis 3, 2, .LCPI44_0@toc@ha
@@ -2702,9 +2585,6 @@ define <4 x double> @constrained_vector_sin_v4f64() {
 ; PC64LE9-NEXT:    mflr 0
 ; PC64LE9-NEXT:    std 0, 16(1)
 ; PC64LE9-NEXT:    stdu 1, -64(1)
-; PC64LE9-NEXT:    .cfi_def_cfa_offset 64
-; PC64LE9-NEXT:    .cfi_offset lr, 16
-; PC64LE9-NEXT:    .cfi_offset v31, -16
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI44_0@toc@ha
 ; PC64LE9-NEXT:    lfd 1, .LCPI44_0@toc@l(3)
 ; PC64LE9-NEXT:    stxv 63, 48(1) # 16-byte Folded Spill
@@ -2743,18 +2623,16 @@ entry:
                              <4 x double> <double 42.0, double 42.1,
                                            double 42.2, double 42.3>,
                              metadata !"round.dynamic",
-                             metadata !"fpexcept.strict")
+                             metadata !"fpexcept.strict") #1
   ret <4 x double> %sin
 }
 
-define <1 x float> @constrained_vector_cos_v1f32() {
+define <1 x float> @constrained_vector_cos_v1f32() #0 {
 ; PC64LE-LABEL: constrained_vector_cos_v1f32:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    mflr 0
 ; PC64LE-NEXT:    std 0, 16(1)
 ; PC64LE-NEXT:    stdu 1, -32(1)
-; PC64LE-NEXT:    .cfi_def_cfa_offset 32
-; PC64LE-NEXT:    .cfi_offset lr, 16
 ; PC64LE-NEXT:    addis 3, 2, .LCPI45_0@toc@ha
 ; PC64LE-NEXT:    lfs 1, .LCPI45_0@toc@l(3)
 ; PC64LE-NEXT:    bl cosf
@@ -2771,8 +2649,6 @@ define <1 x float> @constrained_vector_cos_v1f32() {
 ; PC64LE9-NEXT:    mflr 0
 ; PC64LE9-NEXT:    std 0, 16(1)
 ; PC64LE9-NEXT:    stdu 1, -32(1)
-; PC64LE9-NEXT:    .cfi_def_cfa_offset 32
-; PC64LE9-NEXT:    .cfi_offset lr, 16
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI45_0@toc@ha
 ; PC64LE9-NEXT:    lfs 1, .LCPI45_0@toc@l(3)
 ; PC64LE9-NEXT:    bl cosf
@@ -2787,18 +2663,16 @@ entry:
   %cos = call <1 x float> @llvm.experimental.constrained.cos.v1f32(
                              <1 x float> <float 42.0>,
                              metadata !"round.dynamic",
-                             metadata !"fpexcept.strict")
+                             metadata !"fpexcept.strict") #1
   ret <1 x float> %cos
 }
 
-define <2 x double> @constrained_vector_cos_v2f64() {
+define <2 x double> @constrained_vector_cos_v2f64() #0 {
 ; PC64LE-LABEL: constrained_vector_cos_v2f64:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    mflr 0
 ; PC64LE-NEXT:    std 0, 16(1)
 ; PC64LE-NEXT:    stdu 1, -64(1)
-; PC64LE-NEXT:    .cfi_def_cfa_offset 64
-; PC64LE-NEXT:    .cfi_offset lr, 16
 ; PC64LE-NEXT:    addis 3, 2, .LCPI46_0@toc@ha
 ; PC64LE-NEXT:    lfd 1, .LCPI46_0@toc@l(3)
 ; PC64LE-NEXT:    bl cos
@@ -2824,8 +2698,6 @@ define <2 x double> @constrained_vector_cos_v2f64() {
 ; PC64LE9-NEXT:    mflr 0
 ; PC64LE9-NEXT:    std 0, 16(1)
 ; PC64LE9-NEXT:    stdu 1, -48(1)
-; PC64LE9-NEXT:    .cfi_def_cfa_offset 48
-; PC64LE9-NEXT:    .cfi_offset lr, 16
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI46_0@toc@ha
 ; PC64LE9-NEXT:    lfd 1, .LCPI46_0@toc@l(3)
 ; PC64LE9-NEXT:    bl cos
@@ -2847,18 +2719,14 @@ entry:
   %cos = call <2 x double> @llvm.experimental.constrained.cos.v2f64(
                              <2 x double> <double 42.0, double 42.1>,
                              metadata !"round.dynamic",
-                             metadata !"fpexcept.strict")
+                             metadata !"fpexcept.strict") #1
   ret <2 x double> %cos
 }
 
-define <3 x float> @constrained_vector_cos_v3f32() {
+define <3 x float> @constrained_vector_cos_v3f32() #0 {
 ; PC64LE-LABEL: constrained_vector_cos_v3f32:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    mflr 0
-; PC64LE-NEXT:    .cfi_def_cfa_offset 48
-; PC64LE-NEXT:    .cfi_offset lr, 16
-; PC64LE-NEXT:    .cfi_offset f30, -16
-; PC64LE-NEXT:    .cfi_offset f31, -8
 ; PC64LE-NEXT:    stfd 30, -16(1) # 8-byte Folded Spill
 ; PC64LE-NEXT:    stfd 31, -8(1) # 8-byte Folded Spill
 ; PC64LE-NEXT:    std 0, 16(1)
@@ -2890,18 +2758,14 @@ define <3 x float> @constrained_vector_cos_v3f32() {
 ; PC64LE-NEXT:    vperm 2, 4, 2, 3
 ; PC64LE-NEXT:    addi 1, 1, 48
 ; PC64LE-NEXT:    ld 0, 16(1)
-; PC64LE-NEXT:    mtlr 0
 ; PC64LE-NEXT:    lfd 31, -8(1) # 8-byte Folded Reload
 ; PC64LE-NEXT:    lfd 30, -16(1) # 8-byte Folded Reload
+; PC64LE-NEXT:    mtlr 0
 ; PC64LE-NEXT:    blr
 ;
 ; PC64LE9-LABEL: constrained_vector_cos_v3f32:
 ; PC64LE9:       # %bb.0: # %entry
 ; PC64LE9-NEXT:    mflr 0
-; PC64LE9-NEXT:    .cfi_def_cfa_offset 48
-; PC64LE9-NEXT:    .cfi_offset lr, 16
-; PC64LE9-NEXT:    .cfi_offset f30, -16
-; PC64LE9-NEXT:    .cfi_offset f31, -8
 ; PC64LE9-NEXT:    stfd 30, -16(1) # 8-byte Folded Spill
 ; PC64LE9-NEXT:    stfd 31, -8(1) # 8-byte Folded Spill
 ; PC64LE9-NEXT:    std 0, 16(1)
@@ -2933,27 +2797,24 @@ define <3 x float> @constrained_vector_cos_v3f32() {
 ; PC64LE9-NEXT:    vperm 2, 4, 2, 3
 ; PC64LE9-NEXT:    addi 1, 1, 48
 ; PC64LE9-NEXT:    ld 0, 16(1)
-; PC64LE9-NEXT:    mtlr 0
 ; PC64LE9-NEXT:    lfd 31, -8(1) # 8-byte Folded Reload
 ; PC64LE9-NEXT:    lfd 30, -16(1) # 8-byte Folded Reload
+; PC64LE9-NEXT:    mtlr 0
 ; PC64LE9-NEXT:    blr
 entry:
   %cos = call <3 x float> @llvm.experimental.constrained.cos.v3f32(
                               <3 x float> <float 42.0, float 43.0, float 44.0>,
                               metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #1
   ret <3 x float> %cos
 }
 
-define <3 x double> @constrained_vector_cos_v3f64() {
+define <3 x double> @constrained_vector_cos_v3f64() #0 {
 ; PC64LE-LABEL: constrained_vector_cos_v3f64:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    mflr 0
 ; PC64LE-NEXT:    std 0, 16(1)
 ; PC64LE-NEXT:    stdu 1, -80(1)
-; PC64LE-NEXT:    .cfi_def_cfa_offset 80
-; PC64LE-NEXT:    .cfi_offset lr, 16
-; PC64LE-NEXT:    .cfi_offset v31, -16
 ; PC64LE-NEXT:    li 3, 64
 ; PC64LE-NEXT:    stxvd2x 63, 1, 3 # 16-byte Folded Spill
 ; PC64LE-NEXT:    addis 3, 2, .LCPI48_0@toc@ha
@@ -2975,11 +2836,12 @@ define <3 x double> @constrained_vector_cos_v3f64() {
 ; PC64LE-NEXT:    lfd 1, .LCPI48_2@toc@l(3)
 ; PC64LE-NEXT:    bl cos
 ; PC64LE-NEXT:    nop
+; PC64LE-NEXT:    xxswapd 0, 63
 ; PC64LE-NEXT:    li 3, 64
-; PC64LE-NEXT:    fmr 3, 1
-; PC64LE-NEXT:    xxlor 1, 63, 63
 ; PC64LE-NEXT:    xxlor 2, 63, 63
 ; PC64LE-NEXT:    lxvd2x 63, 1, 3 # 16-byte Folded Reload
+; PC64LE-NEXT:    fmr 3, 1
+; PC64LE-NEXT:    fmr 1, 0
 ; PC64LE-NEXT:    addi 1, 1, 80
 ; PC64LE-NEXT:    ld 0, 16(1)
 ; PC64LE-NEXT:    mtlr 0
@@ -2990,9 +2852,6 @@ define <3 x double> @constrained_vector_cos_v3f64() {
 ; PC64LE9-NEXT:    mflr 0
 ; PC64LE9-NEXT:    std 0, 16(1)
 ; PC64LE9-NEXT:    stdu 1, -64(1)
-; PC64LE9-NEXT:    .cfi_def_cfa_offset 64
-; PC64LE9-NEXT:    .cfi_offset lr, 16
-; PC64LE9-NEXT:    .cfi_offset v31, -16
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI48_0@toc@ha
 ; PC64LE9-NEXT:    lfd 1, .LCPI48_0@toc@l(3)
 ; PC64LE9-NEXT:    stxv 63, 48(1) # 16-byte Folded Spill
@@ -3012,9 +2871,10 @@ define <3 x double> @constrained_vector_cos_v3f64() {
 ; PC64LE9-NEXT:    bl cos
 ; PC64LE9-NEXT:    nop
 ; PC64LE9-NEXT:    fmr 3, 1
-; PC64LE9-NEXT:    xscpsgndp 1, 63, 63
+; PC64LE9-NEXT:    xxswapd 1, 63
 ; PC64LE9-NEXT:    xscpsgndp 2, 63, 63
 ; PC64LE9-NEXT:    lxv 63, 48(1) # 16-byte Folded Reload
+; PC64LE9-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
 ; PC64LE9-NEXT:    addi 1, 1, 64
 ; PC64LE9-NEXT:    ld 0, 16(1)
 ; PC64LE9-NEXT:    mtlr 0
@@ -3023,19 +2883,16 @@ entry:
   %cos = call <3 x double> @llvm.experimental.constrained.cos.v3f64(
                           <3 x double> <double 42.0, double 42.1, double 42.2>,
                           metadata !"round.dynamic",
-                          metadata !"fpexcept.strict")
+                          metadata !"fpexcept.strict") #1
   ret <3 x double> %cos
 }
 
-define <4 x double> @constrained_vector_cos_v4f64() {
+define <4 x double> @constrained_vector_cos_v4f64() #0 {
 ; PC64LE-LABEL: constrained_vector_cos_v4f64:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    mflr 0
 ; PC64LE-NEXT:    std 0, 16(1)
 ; PC64LE-NEXT:    stdu 1, -80(1)
-; PC64LE-NEXT:    .cfi_def_cfa_offset 80
-; PC64LE-NEXT:    .cfi_offset lr, 16
-; PC64LE-NEXT:    .cfi_offset v31, -16
 ; PC64LE-NEXT:    li 3, 64
 ; PC64LE-NEXT:    stxvd2x 63, 1, 3 # 16-byte Folded Spill
 ; PC64LE-NEXT:    addis 3, 2, .LCPI49_0@toc@ha
@@ -3081,9 +2938,6 @@ define <4 x double> @constrained_vector_cos_v4f64() {
 ; PC64LE9-NEXT:    mflr 0
 ; PC64LE9-NEXT:    std 0, 16(1)
 ; PC64LE9-NEXT:    stdu 1, -64(1)
-; PC64LE9-NEXT:    .cfi_def_cfa_offset 64
-; PC64LE9-NEXT:    .cfi_offset lr, 16
-; PC64LE9-NEXT:    .cfi_offset v31, -16
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI49_0@toc@ha
 ; PC64LE9-NEXT:    lfd 1, .LCPI49_0@toc@l(3)
 ; PC64LE9-NEXT:    stxv 63, 48(1) # 16-byte Folded Spill
@@ -3122,18 +2976,16 @@ entry:
                              <4 x double> <double 42.0, double 42.1,
                                            double 42.2, double 42.3>,
                              metadata !"round.dynamic",
-                             metadata !"fpexcept.strict")
+                             metadata !"fpexcept.strict") #1
   ret <4 x double> %cos
 }
 
-define <1 x float> @constrained_vector_exp_v1f32() {
+define <1 x float> @constrained_vector_exp_v1f32() #0 {
 ; PC64LE-LABEL: constrained_vector_exp_v1f32:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    mflr 0
 ; PC64LE-NEXT:    std 0, 16(1)
 ; PC64LE-NEXT:    stdu 1, -32(1)
-; PC64LE-NEXT:    .cfi_def_cfa_offset 32
-; PC64LE-NEXT:    .cfi_offset lr, 16
 ; PC64LE-NEXT:    addis 3, 2, .LCPI50_0@toc@ha
 ; PC64LE-NEXT:    lfs 1, .LCPI50_0@toc@l(3)
 ; PC64LE-NEXT:    bl expf
@@ -3150,8 +3002,6 @@ define <1 x float> @constrained_vector_exp_v1f32() {
 ; PC64LE9-NEXT:    mflr 0
 ; PC64LE9-NEXT:    std 0, 16(1)
 ; PC64LE9-NEXT:    stdu 1, -32(1)
-; PC64LE9-NEXT:    .cfi_def_cfa_offset 32
-; PC64LE9-NEXT:    .cfi_offset lr, 16
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI50_0@toc@ha
 ; PC64LE9-NEXT:    lfs 1, .LCPI50_0@toc@l(3)
 ; PC64LE9-NEXT:    bl expf
@@ -3166,18 +3016,16 @@ entry:
   %exp = call <1 x float> @llvm.experimental.constrained.exp.v1f32(
                              <1 x float> <float 42.0>,
                              metadata !"round.dynamic",
-                             metadata !"fpexcept.strict")
+                             metadata !"fpexcept.strict") #1
   ret <1 x float> %exp
 }
 
-define <2 x double> @constrained_vector_exp_v2f64() {
+define <2 x double> @constrained_vector_exp_v2f64() #0 {
 ; PC64LE-LABEL: constrained_vector_exp_v2f64:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    mflr 0
 ; PC64LE-NEXT:    std 0, 16(1)
 ; PC64LE-NEXT:    stdu 1, -64(1)
-; PC64LE-NEXT:    .cfi_def_cfa_offset 64
-; PC64LE-NEXT:    .cfi_offset lr, 16
 ; PC64LE-NEXT:    addis 3, 2, .LCPI51_0@toc@ha
 ; PC64LE-NEXT:    lfd 1, .LCPI51_0@toc@l(3)
 ; PC64LE-NEXT:    bl exp
@@ -3203,8 +3051,6 @@ define <2 x double> @constrained_vector_exp_v2f64() {
 ; PC64LE9-NEXT:    mflr 0
 ; PC64LE9-NEXT:    std 0, 16(1)
 ; PC64LE9-NEXT:    stdu 1, -48(1)
-; PC64LE9-NEXT:    .cfi_def_cfa_offset 48
-; PC64LE9-NEXT:    .cfi_offset lr, 16
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI51_0@toc@ha
 ; PC64LE9-NEXT:    lfd 1, .LCPI51_0@toc@l(3)
 ; PC64LE9-NEXT:    bl exp
@@ -3226,18 +3072,14 @@ entry:
   %exp = call <2 x double> @llvm.experimental.constrained.exp.v2f64(
                              <2 x double> <double 42.0, double 42.1>,
                              metadata !"round.dynamic",
-                             metadata !"fpexcept.strict")
+                             metadata !"fpexcept.strict") #1
   ret <2 x double> %exp
 }
 
-define <3 x float> @constrained_vector_exp_v3f32() {
+define <3 x float> @constrained_vector_exp_v3f32() #0 {
 ; PC64LE-LABEL: constrained_vector_exp_v3f32:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    mflr 0
-; PC64LE-NEXT:    .cfi_def_cfa_offset 48
-; PC64LE-NEXT:    .cfi_offset lr, 16
-; PC64LE-NEXT:    .cfi_offset f30, -16
-; PC64LE-NEXT:    .cfi_offset f31, -8
 ; PC64LE-NEXT:    stfd 30, -16(1) # 8-byte Folded Spill
 ; PC64LE-NEXT:    stfd 31, -8(1) # 8-byte Folded Spill
 ; PC64LE-NEXT:    std 0, 16(1)
@@ -3269,18 +3111,14 @@ define <3 x float> @constrained_vector_exp_v3f32() {
 ; PC64LE-NEXT:    vperm 2, 4, 2, 3
 ; PC64LE-NEXT:    addi 1, 1, 48
 ; PC64LE-NEXT:    ld 0, 16(1)
-; PC64LE-NEXT:    mtlr 0
 ; PC64LE-NEXT:    lfd 31, -8(1) # 8-byte Folded Reload
 ; PC64LE-NEXT:    lfd 30, -16(1) # 8-byte Folded Reload
+; PC64LE-NEXT:    mtlr 0
 ; PC64LE-NEXT:    blr
 ;
 ; PC64LE9-LABEL: constrained_vector_exp_v3f32:
 ; PC64LE9:       # %bb.0: # %entry
 ; PC64LE9-NEXT:    mflr 0
-; PC64LE9-NEXT:    .cfi_def_cfa_offset 48
-; PC64LE9-NEXT:    .cfi_offset lr, 16
-; PC64LE9-NEXT:    .cfi_offset f30, -16
-; PC64LE9-NEXT:    .cfi_offset f31, -8
 ; PC64LE9-NEXT:    stfd 30, -16(1) # 8-byte Folded Spill
 ; PC64LE9-NEXT:    stfd 31, -8(1) # 8-byte Folded Spill
 ; PC64LE9-NEXT:    std 0, 16(1)
@@ -3312,27 +3150,24 @@ define <3 x float> @constrained_vector_exp_v3f32() {
 ; PC64LE9-NEXT:    vperm 2, 4, 2, 3
 ; PC64LE9-NEXT:    addi 1, 1, 48
 ; PC64LE9-NEXT:    ld 0, 16(1)
-; PC64LE9-NEXT:    mtlr 0
 ; PC64LE9-NEXT:    lfd 31, -8(1) # 8-byte Folded Reload
 ; PC64LE9-NEXT:    lfd 30, -16(1) # 8-byte Folded Reload
+; PC64LE9-NEXT:    mtlr 0
 ; PC64LE9-NEXT:    blr
 entry:
   %exp = call <3 x float> @llvm.experimental.constrained.exp.v3f32(
                               <3 x float> <float 42.0, float 43.0, float 44.0>,
                               metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #1
   ret <3 x float> %exp
 }
 
-define <3 x double> @constrained_vector_exp_v3f64() {
+define <3 x double> @constrained_vector_exp_v3f64() #0 {
 ; PC64LE-LABEL: constrained_vector_exp_v3f64:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    mflr 0
 ; PC64LE-NEXT:    std 0, 16(1)
 ; PC64LE-NEXT:    stdu 1, -80(1)
-; PC64LE-NEXT:    .cfi_def_cfa_offset 80
-; PC64LE-NEXT:    .cfi_offset lr, 16
-; PC64LE-NEXT:    .cfi_offset v31, -16
 ; PC64LE-NEXT:    li 3, 64
 ; PC64LE-NEXT:    stxvd2x 63, 1, 3 # 16-byte Folded Spill
 ; PC64LE-NEXT:    addis 3, 2, .LCPI53_0@toc@ha
@@ -3354,11 +3189,12 @@ define <3 x double> @constrained_vector_exp_v3f64() {
 ; PC64LE-NEXT:    lfd 1, .LCPI53_2@toc@l(3)
 ; PC64LE-NEXT:    bl exp
 ; PC64LE-NEXT:    nop
+; PC64LE-NEXT:    xxswapd 0, 63
 ; PC64LE-NEXT:    li 3, 64
-; PC64LE-NEXT:    fmr 3, 1
-; PC64LE-NEXT:    xxlor 1, 63, 63
 ; PC64LE-NEXT:    xxlor 2, 63, 63
 ; PC64LE-NEXT:    lxvd2x 63, 1, 3 # 16-byte Folded Reload
+; PC64LE-NEXT:    fmr 3, 1
+; PC64LE-NEXT:    fmr 1, 0
 ; PC64LE-NEXT:    addi 1, 1, 80
 ; PC64LE-NEXT:    ld 0, 16(1)
 ; PC64LE-NEXT:    mtlr 0
@@ -3369,9 +3205,6 @@ define <3 x double> @constrained_vector_exp_v3f64() {
 ; PC64LE9-NEXT:    mflr 0
 ; PC64LE9-NEXT:    std 0, 16(1)
 ; PC64LE9-NEXT:    stdu 1, -64(1)
-; PC64LE9-NEXT:    .cfi_def_cfa_offset 64
-; PC64LE9-NEXT:    .cfi_offset lr, 16
-; PC64LE9-NEXT:    .cfi_offset v31, -16
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI53_0@toc@ha
 ; PC64LE9-NEXT:    lfd 1, .LCPI53_0@toc@l(3)
 ; PC64LE9-NEXT:    stxv 63, 48(1) # 16-byte Folded Spill
@@ -3391,9 +3224,10 @@ define <3 x double> @constrained_vector_exp_v3f64() {
 ; PC64LE9-NEXT:    bl exp
 ; PC64LE9-NEXT:    nop
 ; PC64LE9-NEXT:    fmr 3, 1
-; PC64LE9-NEXT:    xscpsgndp 1, 63, 63
+; PC64LE9-NEXT:    xxswapd 1, 63
 ; PC64LE9-NEXT:    xscpsgndp 2, 63, 63
 ; PC64LE9-NEXT:    lxv 63, 48(1) # 16-byte Folded Reload
+; PC64LE9-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
 ; PC64LE9-NEXT:    addi 1, 1, 64
 ; PC64LE9-NEXT:    ld 0, 16(1)
 ; PC64LE9-NEXT:    mtlr 0
@@ -3402,19 +3236,16 @@ entry:
   %exp = call <3 x double> @llvm.experimental.constrained.exp.v3f64(
                           <3 x double> <double 42.0, double 42.1, double 42.2>,
                           metadata !"round.dynamic",
-                          metadata !"fpexcept.strict")
+                          metadata !"fpexcept.strict") #1
   ret <3 x double> %exp
 }
 
-define <4 x double> @constrained_vector_exp_v4f64() {
+define <4 x double> @constrained_vector_exp_v4f64() #0 {
 ; PC64LE-LABEL: constrained_vector_exp_v4f64:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    mflr 0
 ; PC64LE-NEXT:    std 0, 16(1)
 ; PC64LE-NEXT:    stdu 1, -80(1)
-; PC64LE-NEXT:    .cfi_def_cfa_offset 80
-; PC64LE-NEXT:    .cfi_offset lr, 16
-; PC64LE-NEXT:    .cfi_offset v31, -16
 ; PC64LE-NEXT:    li 3, 64
 ; PC64LE-NEXT:    stxvd2x 63, 1, 3 # 16-byte Folded Spill
 ; PC64LE-NEXT:    addis 3, 2, .LCPI54_0@toc@ha
@@ -3460,9 +3291,6 @@ define <4 x double> @constrained_vector_exp_v4f64() {
 ; PC64LE9-NEXT:    mflr 0
 ; PC64LE9-NEXT:    std 0, 16(1)
 ; PC64LE9-NEXT:    stdu 1, -64(1)
-; PC64LE9-NEXT:    .cfi_def_cfa_offset 64
-; PC64LE9-NEXT:    .cfi_offset lr, 16
-; PC64LE9-NEXT:    .cfi_offset v31, -16
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI54_0@toc@ha
 ; PC64LE9-NEXT:    lfd 1, .LCPI54_0@toc@l(3)
 ; PC64LE9-NEXT:    stxv 63, 48(1) # 16-byte Folded Spill
@@ -3501,18 +3329,16 @@ entry:
                              <4 x double> <double 42.0, double 42.1,
                                            double 42.2, double 42.3>,
                              metadata !"round.dynamic",
-                             metadata !"fpexcept.strict")
+                             metadata !"fpexcept.strict") #1
   ret <4 x double> %exp
 }
 
-define <1 x float> @constrained_vector_exp2_v1f32() {
+define <1 x float> @constrained_vector_exp2_v1f32() #0 {
 ; PC64LE-LABEL: constrained_vector_exp2_v1f32:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    mflr 0
 ; PC64LE-NEXT:    std 0, 16(1)
 ; PC64LE-NEXT:    stdu 1, -32(1)
-; PC64LE-NEXT:    .cfi_def_cfa_offset 32
-; PC64LE-NEXT:    .cfi_offset lr, 16
 ; PC64LE-NEXT:    addis 3, 2, .LCPI55_0@toc@ha
 ; PC64LE-NEXT:    lfs 1, .LCPI55_0@toc@l(3)
 ; PC64LE-NEXT:    bl exp2f
@@ -3529,8 +3355,6 @@ define <1 x float> @constrained_vector_exp2_v1f32() {
 ; PC64LE9-NEXT:    mflr 0
 ; PC64LE9-NEXT:    std 0, 16(1)
 ; PC64LE9-NEXT:    stdu 1, -32(1)
-; PC64LE9-NEXT:    .cfi_def_cfa_offset 32
-; PC64LE9-NEXT:    .cfi_offset lr, 16
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI55_0@toc@ha
 ; PC64LE9-NEXT:    lfs 1, .LCPI55_0@toc@l(3)
 ; PC64LE9-NEXT:    bl exp2f
@@ -3545,18 +3369,16 @@ entry:
   %exp2 = call <1 x float> @llvm.experimental.constrained.exp2.v1f32(
                              <1 x float> <float 42.0>,
                              metadata !"round.dynamic",
-                             metadata !"fpexcept.strict")
+                             metadata !"fpexcept.strict") #1
   ret <1 x float> %exp2
 }
 
-define <2 x double> @constrained_vector_exp2_v2f64() {
+define <2 x double> @constrained_vector_exp2_v2f64() #0 {
 ; PC64LE-LABEL: constrained_vector_exp2_v2f64:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    mflr 0
 ; PC64LE-NEXT:    std 0, 16(1)
 ; PC64LE-NEXT:    stdu 1, -64(1)
-; PC64LE-NEXT:    .cfi_def_cfa_offset 64
-; PC64LE-NEXT:    .cfi_offset lr, 16
 ; PC64LE-NEXT:    addis 3, 2, .LCPI56_0@toc@ha
 ; PC64LE-NEXT:    lfd 1, .LCPI56_0@toc@l(3)
 ; PC64LE-NEXT:    bl exp2
@@ -3582,8 +3404,6 @@ define <2 x double> @constrained_vector_exp2_v2f64() {
 ; PC64LE9-NEXT:    mflr 0
 ; PC64LE9-NEXT:    std 0, 16(1)
 ; PC64LE9-NEXT:    stdu 1, -48(1)
-; PC64LE9-NEXT:    .cfi_def_cfa_offset 48
-; PC64LE9-NEXT:    .cfi_offset lr, 16
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI56_0@toc@ha
 ; PC64LE9-NEXT:    lfd 1, .LCPI56_0@toc@l(3)
 ; PC64LE9-NEXT:    bl exp2
@@ -3605,18 +3425,14 @@ entry:
   %exp2 = call <2 x double> @llvm.experimental.constrained.exp2.v2f64(
                               <2 x double> <double 42.1, double 42.0>,
                               metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #1
   ret <2 x double> %exp2
 }
 
-define <3 x float> @constrained_vector_exp2_v3f32() {
+define <3 x float> @constrained_vector_exp2_v3f32() #0 {
 ; PC64LE-LABEL: constrained_vector_exp2_v3f32:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    mflr 0
-; PC64LE-NEXT:    .cfi_def_cfa_offset 48
-; PC64LE-NEXT:    .cfi_offset lr, 16
-; PC64LE-NEXT:    .cfi_offset f30, -16
-; PC64LE-NEXT:    .cfi_offset f31, -8
 ; PC64LE-NEXT:    stfd 30, -16(1) # 8-byte Folded Spill
 ; PC64LE-NEXT:    stfd 31, -8(1) # 8-byte Folded Spill
 ; PC64LE-NEXT:    std 0, 16(1)
@@ -3648,18 +3464,14 @@ define <3 x float> @constrained_vector_exp2_v3f32() {
 ; PC64LE-NEXT:    vperm 2, 4, 2, 3
 ; PC64LE-NEXT:    addi 1, 1, 48
 ; PC64LE-NEXT:    ld 0, 16(1)
-; PC64LE-NEXT:    mtlr 0
 ; PC64LE-NEXT:    lfd 31, -8(1) # 8-byte Folded Reload
 ; PC64LE-NEXT:    lfd 30, -16(1) # 8-byte Folded Reload
+; PC64LE-NEXT:    mtlr 0
 ; PC64LE-NEXT:    blr
 ;
 ; PC64LE9-LABEL: constrained_vector_exp2_v3f32:
 ; PC64LE9:       # %bb.0: # %entry
 ; PC64LE9-NEXT:    mflr 0
-; PC64LE9-NEXT:    .cfi_def_cfa_offset 48
-; PC64LE9-NEXT:    .cfi_offset lr, 16
-; PC64LE9-NEXT:    .cfi_offset f30, -16
-; PC64LE9-NEXT:    .cfi_offset f31, -8
 ; PC64LE9-NEXT:    stfd 30, -16(1) # 8-byte Folded Spill
 ; PC64LE9-NEXT:    stfd 31, -8(1) # 8-byte Folded Spill
 ; PC64LE9-NEXT:    std 0, 16(1)
@@ -3691,27 +3503,24 @@ define <3 x float> @constrained_vector_exp2_v3f32() {
 ; PC64LE9-NEXT:    vperm 2, 4, 2, 3
 ; PC64LE9-NEXT:    addi 1, 1, 48
 ; PC64LE9-NEXT:    ld 0, 16(1)
-; PC64LE9-NEXT:    mtlr 0
 ; PC64LE9-NEXT:    lfd 31, -8(1) # 8-byte Folded Reload
 ; PC64LE9-NEXT:    lfd 30, -16(1) # 8-byte Folded Reload
+; PC64LE9-NEXT:    mtlr 0
 ; PC64LE9-NEXT:    blr
 entry:
   %exp2 = call <3 x float> @llvm.experimental.constrained.exp2.v3f32(
                               <3 x float> <float 42.0, float 43.0, float 44.0>,
                               metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #1
   ret <3 x float> %exp2
 }
 
-define <3 x double> @constrained_vector_exp2_v3f64() {
+define <3 x double> @constrained_vector_exp2_v3f64() #0 {
 ; PC64LE-LABEL: constrained_vector_exp2_v3f64:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    mflr 0
 ; PC64LE-NEXT:    std 0, 16(1)
 ; PC64LE-NEXT:    stdu 1, -80(1)
-; PC64LE-NEXT:    .cfi_def_cfa_offset 80
-; PC64LE-NEXT:    .cfi_offset lr, 16
-; PC64LE-NEXT:    .cfi_offset v31, -16
 ; PC64LE-NEXT:    li 3, 64
 ; PC64LE-NEXT:    stxvd2x 63, 1, 3 # 16-byte Folded Spill
 ; PC64LE-NEXT:    addis 3, 2, .LCPI58_0@toc@ha
@@ -3733,11 +3542,12 @@ define <3 x double> @constrained_vector_exp2_v3f64() {
 ; PC64LE-NEXT:    lfd 1, .LCPI58_2@toc@l(3)
 ; PC64LE-NEXT:    bl exp2
 ; PC64LE-NEXT:    nop
+; PC64LE-NEXT:    xxswapd 0, 63
 ; PC64LE-NEXT:    li 3, 64
-; PC64LE-NEXT:    fmr 3, 1
-; PC64LE-NEXT:    xxlor 1, 63, 63
 ; PC64LE-NEXT:    xxlor 2, 63, 63
 ; PC64LE-NEXT:    lxvd2x 63, 1, 3 # 16-byte Folded Reload
+; PC64LE-NEXT:    fmr 3, 1
+; PC64LE-NEXT:    fmr 1, 0
 ; PC64LE-NEXT:    addi 1, 1, 80
 ; PC64LE-NEXT:    ld 0, 16(1)
 ; PC64LE-NEXT:    mtlr 0
@@ -3748,9 +3558,6 @@ define <3 x double> @constrained_vector_exp2_v3f64() {
 ; PC64LE9-NEXT:    mflr 0
 ; PC64LE9-NEXT:    std 0, 16(1)
 ; PC64LE9-NEXT:    stdu 1, -64(1)
-; PC64LE9-NEXT:    .cfi_def_cfa_offset 64
-; PC64LE9-NEXT:    .cfi_offset lr, 16
-; PC64LE9-NEXT:    .cfi_offset v31, -16
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI58_0@toc@ha
 ; PC64LE9-NEXT:    lfd 1, .LCPI58_0@toc@l(3)
 ; PC64LE9-NEXT:    stxv 63, 48(1) # 16-byte Folded Spill
@@ -3770,9 +3577,10 @@ define <3 x double> @constrained_vector_exp2_v3f64() {
 ; PC64LE9-NEXT:    bl exp2
 ; PC64LE9-NEXT:    nop
 ; PC64LE9-NEXT:    fmr 3, 1
-; PC64LE9-NEXT:    xscpsgndp 1, 63, 63
+; PC64LE9-NEXT:    xxswapd 1, 63
 ; PC64LE9-NEXT:    xscpsgndp 2, 63, 63
 ; PC64LE9-NEXT:    lxv 63, 48(1) # 16-byte Folded Reload
+; PC64LE9-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
 ; PC64LE9-NEXT:    addi 1, 1, 64
 ; PC64LE9-NEXT:    ld 0, 16(1)
 ; PC64LE9-NEXT:    mtlr 0
@@ -3781,19 +3589,16 @@ entry:
   %exp2 = call <3 x double> @llvm.experimental.constrained.exp2.v3f64(
                           <3 x double> <double 42.0, double 42.1, double 42.2>,
                           metadata !"round.dynamic",
-                          metadata !"fpexcept.strict")
+                          metadata !"fpexcept.strict") #1
   ret <3 x double> %exp2
 }
 
-define <4 x double> @constrained_vector_exp2_v4f64() {
+define <4 x double> @constrained_vector_exp2_v4f64() #0 {
 ; PC64LE-LABEL: constrained_vector_exp2_v4f64:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    mflr 0
 ; PC64LE-NEXT:    std 0, 16(1)
 ; PC64LE-NEXT:    stdu 1, -80(1)
-; PC64LE-NEXT:    .cfi_def_cfa_offset 80
-; PC64LE-NEXT:    .cfi_offset lr, 16
-; PC64LE-NEXT:    .cfi_offset v31, -16
 ; PC64LE-NEXT:    li 3, 64
 ; PC64LE-NEXT:    stxvd2x 63, 1, 3 # 16-byte Folded Spill
 ; PC64LE-NEXT:    addis 3, 2, .LCPI59_0@toc@ha
@@ -3839,9 +3644,6 @@ define <4 x double> @constrained_vector_exp2_v4f64() {
 ; PC64LE9-NEXT:    mflr 0
 ; PC64LE9-NEXT:    std 0, 16(1)
 ; PC64LE9-NEXT:    stdu 1, -64(1)
-; PC64LE9-NEXT:    .cfi_def_cfa_offset 64
-; PC64LE9-NEXT:    .cfi_offset lr, 16
-; PC64LE9-NEXT:    .cfi_offset v31, -16
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI59_0@toc@ha
 ; PC64LE9-NEXT:    lfd 1, .LCPI59_0@toc@l(3)
 ; PC64LE9-NEXT:    stxv 63, 48(1) # 16-byte Folded Spill
@@ -3880,18 +3682,16 @@ entry:
                               <4 x double> <double 42.1, double 42.2,
                                             double 42.3, double 42.4>,
                               metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #1
   ret <4 x double> %exp2
 }
 
-define <1 x float> @constrained_vector_log_v1f32() {
+define <1 x float> @constrained_vector_log_v1f32() #0 {
 ; PC64LE-LABEL: constrained_vector_log_v1f32:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    mflr 0
 ; PC64LE-NEXT:    std 0, 16(1)
 ; PC64LE-NEXT:    stdu 1, -32(1)
-; PC64LE-NEXT:    .cfi_def_cfa_offset 32
-; PC64LE-NEXT:    .cfi_offset lr, 16
 ; PC64LE-NEXT:    addis 3, 2, .LCPI60_0@toc@ha
 ; PC64LE-NEXT:    lfs 1, .LCPI60_0@toc@l(3)
 ; PC64LE-NEXT:    bl logf
@@ -3908,8 +3708,6 @@ define <1 x float> @constrained_vector_log_v1f32() {
 ; PC64LE9-NEXT:    mflr 0
 ; PC64LE9-NEXT:    std 0, 16(1)
 ; PC64LE9-NEXT:    stdu 1, -32(1)
-; PC64LE9-NEXT:    .cfi_def_cfa_offset 32
-; PC64LE9-NEXT:    .cfi_offset lr, 16
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI60_0@toc@ha
 ; PC64LE9-NEXT:    lfs 1, .LCPI60_0@toc@l(3)
 ; PC64LE9-NEXT:    bl logf
@@ -3924,18 +3722,16 @@ entry:
   %log = call <1 x float> @llvm.experimental.constrained.log.v1f32(
                              <1 x float> <float 42.0>,
                              metadata !"round.dynamic",
-                             metadata !"fpexcept.strict")
+                             metadata !"fpexcept.strict") #1
   ret <1 x float> %log
 }
 
-define <2 x double> @constrained_vector_log_v2f64() {
+define <2 x double> @constrained_vector_log_v2f64() #0 {
 ; PC64LE-LABEL: constrained_vector_log_v2f64:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    mflr 0
 ; PC64LE-NEXT:    std 0, 16(1)
 ; PC64LE-NEXT:    stdu 1, -64(1)
-; PC64LE-NEXT:    .cfi_def_cfa_offset 64
-; PC64LE-NEXT:    .cfi_offset lr, 16
 ; PC64LE-NEXT:    addis 3, 2, .LCPI61_0@toc@ha
 ; PC64LE-NEXT:    lfd 1, .LCPI61_0@toc@l(3)
 ; PC64LE-NEXT:    bl log
@@ -3961,8 +3757,6 @@ define <2 x double> @constrained_vector_log_v2f64() {
 ; PC64LE9-NEXT:    mflr 0
 ; PC64LE9-NEXT:    std 0, 16(1)
 ; PC64LE9-NEXT:    stdu 1, -48(1)
-; PC64LE9-NEXT:    .cfi_def_cfa_offset 48
-; PC64LE9-NEXT:    .cfi_offset lr, 16
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI61_0@toc@ha
 ; PC64LE9-NEXT:    lfd 1, .LCPI61_0@toc@l(3)
 ; PC64LE9-NEXT:    bl log
@@ -3984,18 +3778,14 @@ entry:
   %log = call <2 x double> @llvm.experimental.constrained.log.v2f64(
                              <2 x double> <double 42.0, double 42.1>,
                              metadata !"round.dynamic",
-                             metadata !"fpexcept.strict")
+                             metadata !"fpexcept.strict") #1
   ret <2 x double> %log
 }
 
-define <3 x float> @constrained_vector_log_v3f32() {
+define <3 x float> @constrained_vector_log_v3f32() #0 {
 ; PC64LE-LABEL: constrained_vector_log_v3f32:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    mflr 0
-; PC64LE-NEXT:    .cfi_def_cfa_offset 48
-; PC64LE-NEXT:    .cfi_offset lr, 16
-; PC64LE-NEXT:    .cfi_offset f30, -16
-; PC64LE-NEXT:    .cfi_offset f31, -8
 ; PC64LE-NEXT:    stfd 30, -16(1) # 8-byte Folded Spill
 ; PC64LE-NEXT:    stfd 31, -8(1) # 8-byte Folded Spill
 ; PC64LE-NEXT:    std 0, 16(1)
@@ -4027,18 +3817,14 @@ define <3 x float> @constrained_vector_log_v3f32() {
 ; PC64LE-NEXT:    vperm 2, 4, 2, 3
 ; PC64LE-NEXT:    addi 1, 1, 48
 ; PC64LE-NEXT:    ld 0, 16(1)
-; PC64LE-NEXT:    mtlr 0
 ; PC64LE-NEXT:    lfd 31, -8(1) # 8-byte Folded Reload
 ; PC64LE-NEXT:    lfd 30, -16(1) # 8-byte Folded Reload
+; PC64LE-NEXT:    mtlr 0
 ; PC64LE-NEXT:    blr
 ;
 ; PC64LE9-LABEL: constrained_vector_log_v3f32:
 ; PC64LE9:       # %bb.0: # %entry
 ; PC64LE9-NEXT:    mflr 0
-; PC64LE9-NEXT:    .cfi_def_cfa_offset 48
-; PC64LE9-NEXT:    .cfi_offset lr, 16
-; PC64LE9-NEXT:    .cfi_offset f30, -16
-; PC64LE9-NEXT:    .cfi_offset f31, -8
 ; PC64LE9-NEXT:    stfd 30, -16(1) # 8-byte Folded Spill
 ; PC64LE9-NEXT:    stfd 31, -8(1) # 8-byte Folded Spill
 ; PC64LE9-NEXT:    std 0, 16(1)
@@ -4070,27 +3856,24 @@ define <3 x float> @constrained_vector_log_v3f32() {
 ; PC64LE9-NEXT:    vperm 2, 4, 2, 3
 ; PC64LE9-NEXT:    addi 1, 1, 48
 ; PC64LE9-NEXT:    ld 0, 16(1)
-; PC64LE9-NEXT:    mtlr 0
 ; PC64LE9-NEXT:    lfd 31, -8(1) # 8-byte Folded Reload
 ; PC64LE9-NEXT:    lfd 30, -16(1) # 8-byte Folded Reload
+; PC64LE9-NEXT:    mtlr 0
 ; PC64LE9-NEXT:    blr
 entry:
   %log = call <3 x float> @llvm.experimental.constrained.log.v3f32(
                               <3 x float> <float 42.0, float 43.0, float 44.0>,
                               metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #1
   ret <3 x float> %log
 }
 
-define <3 x double> @constrained_vector_log_v3f64() {
+define <3 x double> @constrained_vector_log_v3f64() #0 {
 ; PC64LE-LABEL: constrained_vector_log_v3f64:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    mflr 0
 ; PC64LE-NEXT:    std 0, 16(1)
 ; PC64LE-NEXT:    stdu 1, -80(1)
-; PC64LE-NEXT:    .cfi_def_cfa_offset 80
-; PC64LE-NEXT:    .cfi_offset lr, 16
-; PC64LE-NEXT:    .cfi_offset v31, -16
 ; PC64LE-NEXT:    li 3, 64
 ; PC64LE-NEXT:    stxvd2x 63, 1, 3 # 16-byte Folded Spill
 ; PC64LE-NEXT:    addis 3, 2, .LCPI63_0@toc@ha
@@ -4112,11 +3895,12 @@ define <3 x double> @constrained_vector_log_v3f64() {
 ; PC64LE-NEXT:    lfd 1, .LCPI63_2@toc@l(3)
 ; PC64LE-NEXT:    bl log
 ; PC64LE-NEXT:    nop
+; PC64LE-NEXT:    xxswapd 0, 63
 ; PC64LE-NEXT:    li 3, 64
-; PC64LE-NEXT:    fmr 3, 1
-; PC64LE-NEXT:    xxlor 1, 63, 63
 ; PC64LE-NEXT:    xxlor 2, 63, 63
 ; PC64LE-NEXT:    lxvd2x 63, 1, 3 # 16-byte Folded Reload
+; PC64LE-NEXT:    fmr 3, 1
+; PC64LE-NEXT:    fmr 1, 0
 ; PC64LE-NEXT:    addi 1, 1, 80
 ; PC64LE-NEXT:    ld 0, 16(1)
 ; PC64LE-NEXT:    mtlr 0
@@ -4127,9 +3911,6 @@ define <3 x double> @constrained_vector_log_v3f64() {
 ; PC64LE9-NEXT:    mflr 0
 ; PC64LE9-NEXT:    std 0, 16(1)
 ; PC64LE9-NEXT:    stdu 1, -64(1)
-; PC64LE9-NEXT:    .cfi_def_cfa_offset 64
-; PC64LE9-NEXT:    .cfi_offset lr, 16
-; PC64LE9-NEXT:    .cfi_offset v31, -16
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI63_0@toc@ha
 ; PC64LE9-NEXT:    lfd 1, .LCPI63_0@toc@l(3)
 ; PC64LE9-NEXT:    stxv 63, 48(1) # 16-byte Folded Spill
@@ -4149,9 +3930,10 @@ define <3 x double> @constrained_vector_log_v3f64() {
 ; PC64LE9-NEXT:    bl log
 ; PC64LE9-NEXT:    nop
 ; PC64LE9-NEXT:    fmr 3, 1
-; PC64LE9-NEXT:    xscpsgndp 1, 63, 63
+; PC64LE9-NEXT:    xxswapd 1, 63
 ; PC64LE9-NEXT:    xscpsgndp 2, 63, 63
 ; PC64LE9-NEXT:    lxv 63, 48(1) # 16-byte Folded Reload
+; PC64LE9-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
 ; PC64LE9-NEXT:    addi 1, 1, 64
 ; PC64LE9-NEXT:    ld 0, 16(1)
 ; PC64LE9-NEXT:    mtlr 0
@@ -4160,19 +3942,16 @@ entry:
   %log = call <3 x double> @llvm.experimental.constrained.log.v3f64(
                           <3 x double> <double 42.0, double 42.1, double 42.2>,
                           metadata !"round.dynamic",
-                          metadata !"fpexcept.strict")
+                          metadata !"fpexcept.strict") #1
   ret <3 x double> %log
 }
 
-define <4 x double> @constrained_vector_log_v4f64() {
+define <4 x double> @constrained_vector_log_v4f64() #0 {
 ; PC64LE-LABEL: constrained_vector_log_v4f64:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    mflr 0
 ; PC64LE-NEXT:    std 0, 16(1)
 ; PC64LE-NEXT:    stdu 1, -80(1)
-; PC64LE-NEXT:    .cfi_def_cfa_offset 80
-; PC64LE-NEXT:    .cfi_offset lr, 16
-; PC64LE-NEXT:    .cfi_offset v31, -16
 ; PC64LE-NEXT:    li 3, 64
 ; PC64LE-NEXT:    stxvd2x 63, 1, 3 # 16-byte Folded Spill
 ; PC64LE-NEXT:    addis 3, 2, .LCPI64_0@toc@ha
@@ -4218,9 +3997,6 @@ define <4 x double> @constrained_vector_log_v4f64() {
 ; PC64LE9-NEXT:    mflr 0
 ; PC64LE9-NEXT:    std 0, 16(1)
 ; PC64LE9-NEXT:    stdu 1, -64(1)
-; PC64LE9-NEXT:    .cfi_def_cfa_offset 64
-; PC64LE9-NEXT:    .cfi_offset lr, 16
-; PC64LE9-NEXT:    .cfi_offset v31, -16
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI64_0@toc@ha
 ; PC64LE9-NEXT:    lfd 1, .LCPI64_0@toc@l(3)
 ; PC64LE9-NEXT:    stxv 63, 48(1) # 16-byte Folded Spill
@@ -4259,18 +4035,16 @@ entry:
                              <4 x double> <double 42.0, double 42.1,
                                            double 42.2, double 42.3>,
                              metadata !"round.dynamic",
-                             metadata !"fpexcept.strict")
+                             metadata !"fpexcept.strict") #1
   ret <4 x double> %log
 }
 
-define <1 x float> @constrained_vector_log10_v1f32() {
+define <1 x float> @constrained_vector_log10_v1f32() #0 {
 ; PC64LE-LABEL: constrained_vector_log10_v1f32:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    mflr 0
 ; PC64LE-NEXT:    std 0, 16(1)
 ; PC64LE-NEXT:    stdu 1, -32(1)
-; PC64LE-NEXT:    .cfi_def_cfa_offset 32
-; PC64LE-NEXT:    .cfi_offset lr, 16
 ; PC64LE-NEXT:    addis 3, 2, .LCPI65_0@toc@ha
 ; PC64LE-NEXT:    lfs 1, .LCPI65_0@toc@l(3)
 ; PC64LE-NEXT:    bl log10f
@@ -4287,8 +4061,6 @@ define <1 x float> @constrained_vector_log10_v1f32() {
 ; PC64LE9-NEXT:    mflr 0
 ; PC64LE9-NEXT:    std 0, 16(1)
 ; PC64LE9-NEXT:    stdu 1, -32(1)
-; PC64LE9-NEXT:    .cfi_def_cfa_offset 32
-; PC64LE9-NEXT:    .cfi_offset lr, 16
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI65_0@toc@ha
 ; PC64LE9-NEXT:    lfs 1, .LCPI65_0@toc@l(3)
 ; PC64LE9-NEXT:    bl log10f
@@ -4303,18 +4075,16 @@ entry:
   %log10 = call <1 x float> @llvm.experimental.constrained.log10.v1f32(
                              <1 x float> <float 42.0>,
                              metadata !"round.dynamic",
-                             metadata !"fpexcept.strict")
+                             metadata !"fpexcept.strict") #1
   ret <1 x float> %log10
 }
 
-define <2 x double> @constrained_vector_log10_v2f64() {
+define <2 x double> @constrained_vector_log10_v2f64() #0 {
 ; PC64LE-LABEL: constrained_vector_log10_v2f64:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    mflr 0
 ; PC64LE-NEXT:    std 0, 16(1)
 ; PC64LE-NEXT:    stdu 1, -64(1)
-; PC64LE-NEXT:    .cfi_def_cfa_offset 64
-; PC64LE-NEXT:    .cfi_offset lr, 16
 ; PC64LE-NEXT:    addis 3, 2, .LCPI66_0@toc@ha
 ; PC64LE-NEXT:    lfd 1, .LCPI66_0@toc@l(3)
 ; PC64LE-NEXT:    bl log10
@@ -4340,8 +4110,6 @@ define <2 x double> @constrained_vector_log10_v2f64() {
 ; PC64LE9-NEXT:    mflr 0
 ; PC64LE9-NEXT:    std 0, 16(1)
 ; PC64LE9-NEXT:    stdu 1, -48(1)
-; PC64LE9-NEXT:    .cfi_def_cfa_offset 48
-; PC64LE9-NEXT:    .cfi_offset lr, 16
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI66_0@toc@ha
 ; PC64LE9-NEXT:    lfd 1, .LCPI66_0@toc@l(3)
 ; PC64LE9-NEXT:    bl log10
@@ -4363,18 +4131,14 @@ entry:
   %log10 = call <2 x double> @llvm.experimental.constrained.log10.v2f64(
                                <2 x double> <double 42.0, double 42.1>,
                                metadata !"round.dynamic",
-                               metadata !"fpexcept.strict")
+                               metadata !"fpexcept.strict") #1
   ret <2 x double> %log10
 }
 
-define <3 x float> @constrained_vector_log10_v3f32() {
+define <3 x float> @constrained_vector_log10_v3f32() #0 {
 ; PC64LE-LABEL: constrained_vector_log10_v3f32:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    mflr 0
-; PC64LE-NEXT:    .cfi_def_cfa_offset 48
-; PC64LE-NEXT:    .cfi_offset lr, 16
-; PC64LE-NEXT:    .cfi_offset f30, -16
-; PC64LE-NEXT:    .cfi_offset f31, -8
 ; PC64LE-NEXT:    stfd 30, -16(1) # 8-byte Folded Spill
 ; PC64LE-NEXT:    stfd 31, -8(1) # 8-byte Folded Spill
 ; PC64LE-NEXT:    std 0, 16(1)
@@ -4406,18 +4170,14 @@ define <3 x float> @constrained_vector_log10_v3f32() {
 ; PC64LE-NEXT:    vperm 2, 4, 2, 3
 ; PC64LE-NEXT:    addi 1, 1, 48
 ; PC64LE-NEXT:    ld 0, 16(1)
-; PC64LE-NEXT:    mtlr 0
 ; PC64LE-NEXT:    lfd 31, -8(1) # 8-byte Folded Reload
 ; PC64LE-NEXT:    lfd 30, -16(1) # 8-byte Folded Reload
+; PC64LE-NEXT:    mtlr 0
 ; PC64LE-NEXT:    blr
 ;
 ; PC64LE9-LABEL: constrained_vector_log10_v3f32:
 ; PC64LE9:       # %bb.0: # %entry
 ; PC64LE9-NEXT:    mflr 0
-; PC64LE9-NEXT:    .cfi_def_cfa_offset 48
-; PC64LE9-NEXT:    .cfi_offset lr, 16
-; PC64LE9-NEXT:    .cfi_offset f30, -16
-; PC64LE9-NEXT:    .cfi_offset f31, -8
 ; PC64LE9-NEXT:    stfd 30, -16(1) # 8-byte Folded Spill
 ; PC64LE9-NEXT:    stfd 31, -8(1) # 8-byte Folded Spill
 ; PC64LE9-NEXT:    std 0, 16(1)
@@ -4449,27 +4209,24 @@ define <3 x float> @constrained_vector_log10_v3f32() {
 ; PC64LE9-NEXT:    vperm 2, 4, 2, 3
 ; PC64LE9-NEXT:    addi 1, 1, 48
 ; PC64LE9-NEXT:    ld 0, 16(1)
-; PC64LE9-NEXT:    mtlr 0
 ; PC64LE9-NEXT:    lfd 31, -8(1) # 8-byte Folded Reload
 ; PC64LE9-NEXT:    lfd 30, -16(1) # 8-byte Folded Reload
+; PC64LE9-NEXT:    mtlr 0
 ; PC64LE9-NEXT:    blr
 entry:
   %log10 = call <3 x float> @llvm.experimental.constrained.log10.v3f32(
                               <3 x float> <float 42.0, float 43.0, float 44.0>,
                               metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #1
   ret <3 x float> %log10
 }
 
-define <3 x double> @constrained_vector_log10_v3f64() {
+define <3 x double> @constrained_vector_log10_v3f64() #0 {
 ; PC64LE-LABEL: constrained_vector_log10_v3f64:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    mflr 0
 ; PC64LE-NEXT:    std 0, 16(1)
 ; PC64LE-NEXT:    stdu 1, -80(1)
-; PC64LE-NEXT:    .cfi_def_cfa_offset 80
-; PC64LE-NEXT:    .cfi_offset lr, 16
-; PC64LE-NEXT:    .cfi_offset v31, -16
 ; PC64LE-NEXT:    li 3, 64
 ; PC64LE-NEXT:    stxvd2x 63, 1, 3 # 16-byte Folded Spill
 ; PC64LE-NEXT:    addis 3, 2, .LCPI68_0@toc@ha
@@ -4491,11 +4248,12 @@ define <3 x double> @constrained_vector_log10_v3f64() {
 ; PC64LE-NEXT:    lfd 1, .LCPI68_2@toc@l(3)
 ; PC64LE-NEXT:    bl log10
 ; PC64LE-NEXT:    nop
+; PC64LE-NEXT:    xxswapd 0, 63
 ; PC64LE-NEXT:    li 3, 64
-; PC64LE-NEXT:    fmr 3, 1
-; PC64LE-NEXT:    xxlor 1, 63, 63
 ; PC64LE-NEXT:    xxlor 2, 63, 63
 ; PC64LE-NEXT:    lxvd2x 63, 1, 3 # 16-byte Folded Reload
+; PC64LE-NEXT:    fmr 3, 1
+; PC64LE-NEXT:    fmr 1, 0
 ; PC64LE-NEXT:    addi 1, 1, 80
 ; PC64LE-NEXT:    ld 0, 16(1)
 ; PC64LE-NEXT:    mtlr 0
@@ -4506,9 +4264,6 @@ define <3 x double> @constrained_vector_log10_v3f64() {
 ; PC64LE9-NEXT:    mflr 0
 ; PC64LE9-NEXT:    std 0, 16(1)
 ; PC64LE9-NEXT:    stdu 1, -64(1)
-; PC64LE9-NEXT:    .cfi_def_cfa_offset 64
-; PC64LE9-NEXT:    .cfi_offset lr, 16
-; PC64LE9-NEXT:    .cfi_offset v31, -16
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI68_0@toc@ha
 ; PC64LE9-NEXT:    lfd 1, .LCPI68_0@toc@l(3)
 ; PC64LE9-NEXT:    stxv 63, 48(1) # 16-byte Folded Spill
@@ -4528,9 +4283,10 @@ define <3 x double> @constrained_vector_log10_v3f64() {
 ; PC64LE9-NEXT:    bl log10
 ; PC64LE9-NEXT:    nop
 ; PC64LE9-NEXT:    fmr 3, 1
-; PC64LE9-NEXT:    xscpsgndp 1, 63, 63
+; PC64LE9-NEXT:    xxswapd 1, 63
 ; PC64LE9-NEXT:    xscpsgndp 2, 63, 63
 ; PC64LE9-NEXT:    lxv 63, 48(1) # 16-byte Folded Reload
+; PC64LE9-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
 ; PC64LE9-NEXT:    addi 1, 1, 64
 ; PC64LE9-NEXT:    ld 0, 16(1)
 ; PC64LE9-NEXT:    mtlr 0
@@ -4539,19 +4295,16 @@ entry:
   %log10 = call <3 x double> @llvm.experimental.constrained.log10.v3f64(
                           <3 x double> <double 42.0, double 42.1, double 42.2>,
                           metadata !"round.dynamic",
-                          metadata !"fpexcept.strict")
+                          metadata !"fpexcept.strict") #1
   ret <3 x double> %log10
 }
 
-define <4 x double> @constrained_vector_log10_v4f64() {
+define <4 x double> @constrained_vector_log10_v4f64() #0 {
 ; PC64LE-LABEL: constrained_vector_log10_v4f64:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    mflr 0
 ; PC64LE-NEXT:    std 0, 16(1)
 ; PC64LE-NEXT:    stdu 1, -80(1)
-; PC64LE-NEXT:    .cfi_def_cfa_offset 80
-; PC64LE-NEXT:    .cfi_offset lr, 16
-; PC64LE-NEXT:    .cfi_offset v31, -16
 ; PC64LE-NEXT:    li 3, 64
 ; PC64LE-NEXT:    stxvd2x 63, 1, 3 # 16-byte Folded Spill
 ; PC64LE-NEXT:    addis 3, 2, .LCPI69_0@toc@ha
@@ -4597,9 +4350,6 @@ define <4 x double> @constrained_vector_log10_v4f64() {
 ; PC64LE9-NEXT:    mflr 0
 ; PC64LE9-NEXT:    std 0, 16(1)
 ; PC64LE9-NEXT:    stdu 1, -64(1)
-; PC64LE9-NEXT:    .cfi_def_cfa_offset 64
-; PC64LE9-NEXT:    .cfi_offset lr, 16
-; PC64LE9-NEXT:    .cfi_offset v31, -16
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI69_0@toc@ha
 ; PC64LE9-NEXT:    lfd 1, .LCPI69_0@toc@l(3)
 ; PC64LE9-NEXT:    stxv 63, 48(1) # 16-byte Folded Spill
@@ -4638,18 +4388,16 @@ entry:
                                <4 x double> <double 42.0, double 42.1,
                                              double 42.2, double 42.3>,
                                metadata !"round.dynamic",
-                               metadata !"fpexcept.strict")
+                               metadata !"fpexcept.strict") #1
   ret <4 x double> %log10
 }
 
-define <1 x float> @constrained_vector_log2_v1f32() {
+define <1 x float> @constrained_vector_log2_v1f32() #0 {
 ; PC64LE-LABEL: constrained_vector_log2_v1f32:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    mflr 0
 ; PC64LE-NEXT:    std 0, 16(1)
 ; PC64LE-NEXT:    stdu 1, -32(1)
-; PC64LE-NEXT:    .cfi_def_cfa_offset 32
-; PC64LE-NEXT:    .cfi_offset lr, 16
 ; PC64LE-NEXT:    addis 3, 2, .LCPI70_0@toc@ha
 ; PC64LE-NEXT:    lfs 1, .LCPI70_0@toc@l(3)
 ; PC64LE-NEXT:    bl log2f
@@ -4666,8 +4414,6 @@ define <1 x float> @constrained_vector_log2_v1f32() {
 ; PC64LE9-NEXT:    mflr 0
 ; PC64LE9-NEXT:    std 0, 16(1)
 ; PC64LE9-NEXT:    stdu 1, -32(1)
-; PC64LE9-NEXT:    .cfi_def_cfa_offset 32
-; PC64LE9-NEXT:    .cfi_offset lr, 16
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI70_0@toc@ha
 ; PC64LE9-NEXT:    lfs 1, .LCPI70_0@toc@l(3)
 ; PC64LE9-NEXT:    bl log2f
@@ -4682,18 +4428,16 @@ entry:
   %log2 = call <1 x float> @llvm.experimental.constrained.log2.v1f32(
                              <1 x float> <float 42.0>,
                              metadata !"round.dynamic",
-                             metadata !"fpexcept.strict")
+                             metadata !"fpexcept.strict") #1
   ret <1 x float> %log2
 }
 
-define <2 x double> @constrained_vector_log2_v2f64() {
+define <2 x double> @constrained_vector_log2_v2f64() #0 {
 ; PC64LE-LABEL: constrained_vector_log2_v2f64:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    mflr 0
 ; PC64LE-NEXT:    std 0, 16(1)
 ; PC64LE-NEXT:    stdu 1, -64(1)
-; PC64LE-NEXT:    .cfi_def_cfa_offset 64
-; PC64LE-NEXT:    .cfi_offset lr, 16
 ; PC64LE-NEXT:    addis 3, 2, .LCPI71_0@toc@ha
 ; PC64LE-NEXT:    lfd 1, .LCPI71_0@toc@l(3)
 ; PC64LE-NEXT:    bl log2
@@ -4719,8 +4463,6 @@ define <2 x double> @constrained_vector_log2_v2f64() {
 ; PC64LE9-NEXT:    mflr 0
 ; PC64LE9-NEXT:    std 0, 16(1)
 ; PC64LE9-NEXT:    stdu 1, -48(1)
-; PC64LE9-NEXT:    .cfi_def_cfa_offset 48
-; PC64LE9-NEXT:    .cfi_offset lr, 16
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI71_0@toc@ha
 ; PC64LE9-NEXT:    lfd 1, .LCPI71_0@toc@l(3)
 ; PC64LE9-NEXT:    bl log2
@@ -4742,18 +4484,14 @@ entry:
   %log2 = call <2 x double> @llvm.experimental.constrained.log2.v2f64(
                               <2 x double> <double 42.0, double 42.1>,
                               metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #1
   ret <2 x double> %log2
 }
 
-define <3 x float> @constrained_vector_log2_v3f32() {
+define <3 x float> @constrained_vector_log2_v3f32() #0 {
 ; PC64LE-LABEL: constrained_vector_log2_v3f32:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    mflr 0
-; PC64LE-NEXT:    .cfi_def_cfa_offset 48
-; PC64LE-NEXT:    .cfi_offset lr, 16
-; PC64LE-NEXT:    .cfi_offset f30, -16
-; PC64LE-NEXT:    .cfi_offset f31, -8
 ; PC64LE-NEXT:    stfd 30, -16(1) # 8-byte Folded Spill
 ; PC64LE-NEXT:    stfd 31, -8(1) # 8-byte Folded Spill
 ; PC64LE-NEXT:    std 0, 16(1)
@@ -4785,18 +4523,14 @@ define <3 x float> @constrained_vector_log2_v3f32() {
 ; PC64LE-NEXT:    vperm 2, 4, 2, 3
 ; PC64LE-NEXT:    addi 1, 1, 48
 ; PC64LE-NEXT:    ld 0, 16(1)
-; PC64LE-NEXT:    mtlr 0
 ; PC64LE-NEXT:    lfd 31, -8(1) # 8-byte Folded Reload
 ; PC64LE-NEXT:    lfd 30, -16(1) # 8-byte Folded Reload
+; PC64LE-NEXT:    mtlr 0
 ; PC64LE-NEXT:    blr
 ;
 ; PC64LE9-LABEL: constrained_vector_log2_v3f32:
 ; PC64LE9:       # %bb.0: # %entry
 ; PC64LE9-NEXT:    mflr 0
-; PC64LE9-NEXT:    .cfi_def_cfa_offset 48
-; PC64LE9-NEXT:    .cfi_offset lr, 16
-; PC64LE9-NEXT:    .cfi_offset f30, -16
-; PC64LE9-NEXT:    .cfi_offset f31, -8
 ; PC64LE9-NEXT:    stfd 30, -16(1) # 8-byte Folded Spill
 ; PC64LE9-NEXT:    stfd 31, -8(1) # 8-byte Folded Spill
 ; PC64LE9-NEXT:    std 0, 16(1)
@@ -4828,27 +4562,24 @@ define <3 x float> @constrained_vector_log2_v3f32() {
 ; PC64LE9-NEXT:    vperm 2, 4, 2, 3
 ; PC64LE9-NEXT:    addi 1, 1, 48
 ; PC64LE9-NEXT:    ld 0, 16(1)
-; PC64LE9-NEXT:    mtlr 0
 ; PC64LE9-NEXT:    lfd 31, -8(1) # 8-byte Folded Reload
 ; PC64LE9-NEXT:    lfd 30, -16(1) # 8-byte Folded Reload
+; PC64LE9-NEXT:    mtlr 0
 ; PC64LE9-NEXT:    blr
 entry:
   %log2 = call <3 x float> @llvm.experimental.constrained.log2.v3f32(
                               <3 x float> <float 42.0, float 43.0, float 44.0>,
                               metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #1
   ret <3 x float> %log2
 }
 
-define <3 x double> @constrained_vector_log2_v3f64() {
+define <3 x double> @constrained_vector_log2_v3f64() #0 {
 ; PC64LE-LABEL: constrained_vector_log2_v3f64:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    mflr 0
 ; PC64LE-NEXT:    std 0, 16(1)
 ; PC64LE-NEXT:    stdu 1, -80(1)
-; PC64LE-NEXT:    .cfi_def_cfa_offset 80
-; PC64LE-NEXT:    .cfi_offset lr, 16
-; PC64LE-NEXT:    .cfi_offset v31, -16
 ; PC64LE-NEXT:    li 3, 64
 ; PC64LE-NEXT:    stxvd2x 63, 1, 3 # 16-byte Folded Spill
 ; PC64LE-NEXT:    addis 3, 2, .LCPI73_0@toc@ha
@@ -4870,11 +4601,12 @@ define <3 x double> @constrained_vector_log2_v3f64() {
 ; PC64LE-NEXT:    lfd 1, .LCPI73_2@toc@l(3)
 ; PC64LE-NEXT:    bl log2
 ; PC64LE-NEXT:    nop
+; PC64LE-NEXT:    xxswapd 0, 63
 ; PC64LE-NEXT:    li 3, 64
-; PC64LE-NEXT:    fmr 3, 1
-; PC64LE-NEXT:    xxlor 1, 63, 63
 ; PC64LE-NEXT:    xxlor 2, 63, 63
 ; PC64LE-NEXT:    lxvd2x 63, 1, 3 # 16-byte Folded Reload
+; PC64LE-NEXT:    fmr 3, 1
+; PC64LE-NEXT:    fmr 1, 0
 ; PC64LE-NEXT:    addi 1, 1, 80
 ; PC64LE-NEXT:    ld 0, 16(1)
 ; PC64LE-NEXT:    mtlr 0
@@ -4885,9 +4617,6 @@ define <3 x double> @constrained_vector_log2_v3f64() {
 ; PC64LE9-NEXT:    mflr 0
 ; PC64LE9-NEXT:    std 0, 16(1)
 ; PC64LE9-NEXT:    stdu 1, -64(1)
-; PC64LE9-NEXT:    .cfi_def_cfa_offset 64
-; PC64LE9-NEXT:    .cfi_offset lr, 16
-; PC64LE9-NEXT:    .cfi_offset v31, -16
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI73_0@toc@ha
 ; PC64LE9-NEXT:    lfd 1, .LCPI73_0@toc@l(3)
 ; PC64LE9-NEXT:    stxv 63, 48(1) # 16-byte Folded Spill
@@ -4907,9 +4636,10 @@ define <3 x double> @constrained_vector_log2_v3f64() {
 ; PC64LE9-NEXT:    bl log2
 ; PC64LE9-NEXT:    nop
 ; PC64LE9-NEXT:    fmr 3, 1
-; PC64LE9-NEXT:    xscpsgndp 1, 63, 63
+; PC64LE9-NEXT:    xxswapd 1, 63
 ; PC64LE9-NEXT:    xscpsgndp 2, 63, 63
 ; PC64LE9-NEXT:    lxv 63, 48(1) # 16-byte Folded Reload
+; PC64LE9-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
 ; PC64LE9-NEXT:    addi 1, 1, 64
 ; PC64LE9-NEXT:    ld 0, 16(1)
 ; PC64LE9-NEXT:    mtlr 0
@@ -4918,19 +4648,16 @@ entry:
   %log2 = call <3 x double> @llvm.experimental.constrained.log2.v3f64(
                           <3 x double> <double 42.0, double 42.1, double 42.2>,
                           metadata !"round.dynamic",
-                          metadata !"fpexcept.strict")
+                          metadata !"fpexcept.strict") #1
   ret <3 x double> %log2
 }
 
-define <4 x double> @constrained_vector_log2_v4f64() {
+define <4 x double> @constrained_vector_log2_v4f64() #0 {
 ; PC64LE-LABEL: constrained_vector_log2_v4f64:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    mflr 0
 ; PC64LE-NEXT:    std 0, 16(1)
 ; PC64LE-NEXT:    stdu 1, -80(1)
-; PC64LE-NEXT:    .cfi_def_cfa_offset 80
-; PC64LE-NEXT:    .cfi_offset lr, 16
-; PC64LE-NEXT:    .cfi_offset v31, -16
 ; PC64LE-NEXT:    li 3, 64
 ; PC64LE-NEXT:    stxvd2x 63, 1, 3 # 16-byte Folded Spill
 ; PC64LE-NEXT:    addis 3, 2, .LCPI74_0@toc@ha
@@ -4976,9 +4703,6 @@ define <4 x double> @constrained_vector_log2_v4f64() {
 ; PC64LE9-NEXT:    mflr 0
 ; PC64LE9-NEXT:    std 0, 16(1)
 ; PC64LE9-NEXT:    stdu 1, -64(1)
-; PC64LE9-NEXT:    .cfi_def_cfa_offset 64
-; PC64LE9-NEXT:    .cfi_offset lr, 16
-; PC64LE9-NEXT:    .cfi_offset v31, -16
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI74_0@toc@ha
 ; PC64LE9-NEXT:    lfd 1, .LCPI74_0@toc@l(3)
 ; PC64LE9-NEXT:    stxv 63, 48(1) # 16-byte Folded Spill
@@ -5017,18 +4741,16 @@ entry:
                               <4 x double> <double 42.0, double 42.1,
                                             double 42.2, double 42.3>,
                               metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #1
   ret <4 x double> %log2
 }
 
-define <1 x float> @constrained_vector_rint_v1f32() {
+define <1 x float> @constrained_vector_rint_v1f32() #0 {
 ; PC64LE-LABEL: constrained_vector_rint_v1f32:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    mflr 0
 ; PC64LE-NEXT:    std 0, 16(1)
 ; PC64LE-NEXT:    stdu 1, -32(1)
-; PC64LE-NEXT:    .cfi_def_cfa_offset 32
-; PC64LE-NEXT:    .cfi_offset lr, 16
 ; PC64LE-NEXT:    addis 3, 2, .LCPI75_0@toc@ha
 ; PC64LE-NEXT:    lfs 1, .LCPI75_0@toc@l(3)
 ; PC64LE-NEXT:    bl rintf
@@ -5045,8 +4767,6 @@ define <1 x float> @constrained_vector_rint_v1f32() {
 ; PC64LE9-NEXT:    mflr 0
 ; PC64LE9-NEXT:    std 0, 16(1)
 ; PC64LE9-NEXT:    stdu 1, -32(1)
-; PC64LE9-NEXT:    .cfi_def_cfa_offset 32
-; PC64LE9-NEXT:    .cfi_offset lr, 16
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI75_0@toc@ha
 ; PC64LE9-NEXT:    lfs 1, .LCPI75_0@toc@l(3)
 ; PC64LE9-NEXT:    bl rintf
@@ -5061,18 +4781,16 @@ entry:
   %rint = call <1 x float> @llvm.experimental.constrained.rint.v1f32(
                              <1 x float> <float 42.0>,
                              metadata !"round.dynamic",
-                             metadata !"fpexcept.strict")
+                             metadata !"fpexcept.strict") #1
   ret <1 x float> %rint
 }
 
-define <2 x double> @constrained_vector_rint_v2f64() {
+define <2 x double> @constrained_vector_rint_v2f64() #0 {
 ; PC64LE-LABEL: constrained_vector_rint_v2f64:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    mflr 0
 ; PC64LE-NEXT:    std 0, 16(1)
 ; PC64LE-NEXT:    stdu 1, -64(1)
-; PC64LE-NEXT:    .cfi_def_cfa_offset 64
-; PC64LE-NEXT:    .cfi_offset lr, 16
 ; PC64LE-NEXT:    addis 3, 2, .LCPI76_0@toc@ha
 ; PC64LE-NEXT:    lfd 1, .LCPI76_0@toc@l(3)
 ; PC64LE-NEXT:    bl rint
@@ -5098,8 +4816,6 @@ define <2 x double> @constrained_vector_rint_v2f64() {
 ; PC64LE9-NEXT:    mflr 0
 ; PC64LE9-NEXT:    std 0, 16(1)
 ; PC64LE9-NEXT:    stdu 1, -48(1)
-; PC64LE9-NEXT:    .cfi_def_cfa_offset 48
-; PC64LE9-NEXT:    .cfi_offset lr, 16
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI76_0@toc@ha
 ; PC64LE9-NEXT:    lfd 1, .LCPI76_0@toc@l(3)
 ; PC64LE9-NEXT:    bl rint
@@ -5121,18 +4837,14 @@ entry:
   %rint = call <2 x double> @llvm.experimental.constrained.rint.v2f64(
                         <2 x double> <double 42.1, double 42.0>,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #1
   ret <2 x double> %rint
 }
 
-define <3 x float> @constrained_vector_rint_v3f32() {
+define <3 x float> @constrained_vector_rint_v3f32() #0 {
 ; PC64LE-LABEL: constrained_vector_rint_v3f32:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    mflr 0
-; PC64LE-NEXT:    .cfi_def_cfa_offset 48
-; PC64LE-NEXT:    .cfi_offset lr, 16
-; PC64LE-NEXT:    .cfi_offset f30, -16
-; PC64LE-NEXT:    .cfi_offset f31, -8
 ; PC64LE-NEXT:    stfd 30, -16(1) # 8-byte Folded Spill
 ; PC64LE-NEXT:    stfd 31, -8(1) # 8-byte Folded Spill
 ; PC64LE-NEXT:    std 0, 16(1)
@@ -5164,18 +4876,14 @@ define <3 x float> @constrained_vector_rint_v3f32() {
 ; PC64LE-NEXT:    vperm 2, 4, 2, 3
 ; PC64LE-NEXT:    addi 1, 1, 48
 ; PC64LE-NEXT:    ld 0, 16(1)
-; PC64LE-NEXT:    mtlr 0
 ; PC64LE-NEXT:    lfd 31, -8(1) # 8-byte Folded Reload
 ; PC64LE-NEXT:    lfd 30, -16(1) # 8-byte Folded Reload
+; PC64LE-NEXT:    mtlr 0
 ; PC64LE-NEXT:    blr
 ;
 ; PC64LE9-LABEL: constrained_vector_rint_v3f32:
 ; PC64LE9:       # %bb.0: # %entry
 ; PC64LE9-NEXT:    mflr 0
-; PC64LE9-NEXT:    .cfi_def_cfa_offset 48
-; PC64LE9-NEXT:    .cfi_offset lr, 16
-; PC64LE9-NEXT:    .cfi_offset f30, -16
-; PC64LE9-NEXT:    .cfi_offset f31, -8
 ; PC64LE9-NEXT:    stfd 30, -16(1) # 8-byte Folded Spill
 ; PC64LE9-NEXT:    stfd 31, -8(1) # 8-byte Folded Spill
 ; PC64LE9-NEXT:    std 0, 16(1)
@@ -5207,27 +4915,24 @@ define <3 x float> @constrained_vector_rint_v3f32() {
 ; PC64LE9-NEXT:    vperm 2, 4, 2, 3
 ; PC64LE9-NEXT:    addi 1, 1, 48
 ; PC64LE9-NEXT:    ld 0, 16(1)
-; PC64LE9-NEXT:    mtlr 0
 ; PC64LE9-NEXT:    lfd 31, -8(1) # 8-byte Folded Reload
 ; PC64LE9-NEXT:    lfd 30, -16(1) # 8-byte Folded Reload
+; PC64LE9-NEXT:    mtlr 0
 ; PC64LE9-NEXT:    blr
  entry:
   %rint = call <3 x float> @llvm.experimental.constrained.rint.v3f32(
                               <3 x float> <float 42.0, float 43.0, float 44.0>,
                               metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #1
   ret <3 x float> %rint
 }
 
-define <3 x double> @constrained_vector_rint_v3f64() {
+define <3 x double> @constrained_vector_rint_v3f64() #0 {
 ; PC64LE-LABEL: constrained_vector_rint_v3f64:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    mflr 0
 ; PC64LE-NEXT:    std 0, 16(1)
 ; PC64LE-NEXT:    stdu 1, -80(1)
-; PC64LE-NEXT:    .cfi_def_cfa_offset 80
-; PC64LE-NEXT:    .cfi_offset lr, 16
-; PC64LE-NEXT:    .cfi_offset v31, -16
 ; PC64LE-NEXT:    li 3, 64
 ; PC64LE-NEXT:    stxvd2x 63, 1, 3 # 16-byte Folded Spill
 ; PC64LE-NEXT:    addis 3, 2, .LCPI78_0@toc@ha
@@ -5249,11 +4954,12 @@ define <3 x double> @constrained_vector_rint_v3f64() {
 ; PC64LE-NEXT:    lfd 1, .LCPI78_2@toc@l(3)
 ; PC64LE-NEXT:    bl rint
 ; PC64LE-NEXT:    nop
+; PC64LE-NEXT:    xxswapd 0, 63
 ; PC64LE-NEXT:    li 3, 64
-; PC64LE-NEXT:    fmr 3, 1
-; PC64LE-NEXT:    xxlor 1, 63, 63
 ; PC64LE-NEXT:    xxlor 2, 63, 63
 ; PC64LE-NEXT:    lxvd2x 63, 1, 3 # 16-byte Folded Reload
+; PC64LE-NEXT:    fmr 3, 1
+; PC64LE-NEXT:    fmr 1, 0
 ; PC64LE-NEXT:    addi 1, 1, 80
 ; PC64LE-NEXT:    ld 0, 16(1)
 ; PC64LE-NEXT:    mtlr 0
@@ -5264,9 +4970,6 @@ define <3 x double> @constrained_vector_rint_v3f64() {
 ; PC64LE9-NEXT:    mflr 0
 ; PC64LE9-NEXT:    std 0, 16(1)
 ; PC64LE9-NEXT:    stdu 1, -64(1)
-; PC64LE9-NEXT:    .cfi_def_cfa_offset 64
-; PC64LE9-NEXT:    .cfi_offset lr, 16
-; PC64LE9-NEXT:    .cfi_offset v31, -16
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI78_0@toc@ha
 ; PC64LE9-NEXT:    lfd 1, .LCPI78_0@toc@l(3)
 ; PC64LE9-NEXT:    stxv 63, 48(1) # 16-byte Folded Spill
@@ -5286,9 +4989,10 @@ define <3 x double> @constrained_vector_rint_v3f64() {
 ; PC64LE9-NEXT:    bl rint
 ; PC64LE9-NEXT:    nop
 ; PC64LE9-NEXT:    fmr 3, 1
-; PC64LE9-NEXT:    xscpsgndp 1, 63, 63
+; PC64LE9-NEXT:    xxswapd 1, 63
 ; PC64LE9-NEXT:    xscpsgndp 2, 63, 63
 ; PC64LE9-NEXT:    lxv 63, 48(1) # 16-byte Folded Reload
+; PC64LE9-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
 ; PC64LE9-NEXT:    addi 1, 1, 64
 ; PC64LE9-NEXT:    ld 0, 16(1)
 ; PC64LE9-NEXT:    mtlr 0
@@ -5297,19 +5001,16 @@ entry:
   %rint = call <3 x double> @llvm.experimental.constrained.rint.v3f64(
                           <3 x double> <double 42.0, double 42.1, double 42.2>,
                           metadata !"round.dynamic",
-                          metadata !"fpexcept.strict")
+                          metadata !"fpexcept.strict") #1
   ret <3 x double> %rint
 }
 
-define <4 x double> @constrained_vector_rint_v4f64() {
+define <4 x double> @constrained_vector_rint_v4f64() #0 {
 ; PC64LE-LABEL: constrained_vector_rint_v4f64:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    mflr 0
 ; PC64LE-NEXT:    std 0, 16(1)
 ; PC64LE-NEXT:    stdu 1, -80(1)
-; PC64LE-NEXT:    .cfi_def_cfa_offset 80
-; PC64LE-NEXT:    .cfi_offset lr, 16
-; PC64LE-NEXT:    .cfi_offset v31, -16
 ; PC64LE-NEXT:    li 3, 64
 ; PC64LE-NEXT:    stxvd2x 63, 1, 3 # 16-byte Folded Spill
 ; PC64LE-NEXT:    addis 3, 2, .LCPI79_0@toc@ha
@@ -5355,9 +5056,6 @@ define <4 x double> @constrained_vector_rint_v4f64() {
 ; PC64LE9-NEXT:    mflr 0
 ; PC64LE9-NEXT:    std 0, 16(1)
 ; PC64LE9-NEXT:    stdu 1, -64(1)
-; PC64LE9-NEXT:    .cfi_def_cfa_offset 64
-; PC64LE9-NEXT:    .cfi_offset lr, 16
-; PC64LE9-NEXT:    .cfi_offset v31, -16
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI79_0@toc@ha
 ; PC64LE9-NEXT:    lfd 1, .LCPI79_0@toc@l(3)
 ; PC64LE9-NEXT:    stxv 63, 48(1) # 16-byte Folded Spill
@@ -5396,18 +5094,16 @@ entry:
                         <4 x double> <double 42.1, double 42.2,
                                       double 42.3, double 42.4>,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #1
   ret <4 x double> %rint
 }
 
-define <1 x float> @constrained_vector_nearbyint_v1f32() {
+define <1 x float> @constrained_vector_nearbyint_v1f32() #0 {
 ; PC64LE-LABEL: constrained_vector_nearbyint_v1f32:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    mflr 0
 ; PC64LE-NEXT:    std 0, 16(1)
 ; PC64LE-NEXT:    stdu 1, -32(1)
-; PC64LE-NEXT:    .cfi_def_cfa_offset 32
-; PC64LE-NEXT:    .cfi_offset lr, 16
 ; PC64LE-NEXT:    addis 3, 2, .LCPI80_0@toc@ha
 ; PC64LE-NEXT:    lfs 1, .LCPI80_0@toc@l(3)
 ; PC64LE-NEXT:    bl nearbyintf
@@ -5424,8 +5120,6 @@ define <1 x float> @constrained_vector_nearbyint_v1f32() {
 ; PC64LE9-NEXT:    mflr 0
 ; PC64LE9-NEXT:    std 0, 16(1)
 ; PC64LE9-NEXT:    stdu 1, -32(1)
-; PC64LE9-NEXT:    .cfi_def_cfa_offset 32
-; PC64LE9-NEXT:    .cfi_offset lr, 16
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI80_0@toc@ha
 ; PC64LE9-NEXT:    lfs 1, .LCPI80_0@toc@l(3)
 ; PC64LE9-NEXT:    bl nearbyintf
@@ -5440,43 +5134,70 @@ entry:
   %nearby = call <1 x float> @llvm.experimental.constrained.nearbyint.v1f32(
                                <1 x float> <float 42.0>,
                                metadata !"round.dynamic",
-                               metadata !"fpexcept.strict")
+                               metadata !"fpexcept.strict") #1
   ret <1 x float> %nearby
 }
 
-define <2 x double> @constrained_vector_nearbyint_v2f64() {
+define <2 x double> @constrained_vector_nearbyint_v2f64() #0 {
 ; PC64LE-LABEL: constrained_vector_nearbyint_v2f64:
 ; PC64LE:       # %bb.0: # %entry
+; PC64LE-NEXT:    mflr 0
+; PC64LE-NEXT:    std 0, 16(1)
+; PC64LE-NEXT:    stdu 1, -64(1)
 ; PC64LE-NEXT:    addis 3, 2, .LCPI81_0@toc@ha
-; PC64LE-NEXT:    addi 3, 3, .LCPI81_0@toc@l
-; PC64LE-NEXT:    lxvd2x 0, 0, 3
-; PC64LE-NEXT:    xxswapd 0, 0
-; PC64LE-NEXT:    xvrdpic 34, 0
+; PC64LE-NEXT:    lfd 1, .LCPI81_0@toc@l(3)
+; PC64LE-NEXT:    bl nearbyint
+; PC64LE-NEXT:    nop
+; PC64LE-NEXT:    li 3, 48
+; PC64LE-NEXT:    # kill: def $f1 killed $f1 def $vsl1
+; PC64LE-NEXT:    stxvd2x 1, 1, 3 # 16-byte Folded Spill
+; PC64LE-NEXT:    addis 3, 2, .LCPI81_1@toc@ha
+; PC64LE-NEXT:    lfs 1, .LCPI81_1@toc@l(3)
+; PC64LE-NEXT:    bl nearbyint
+; PC64LE-NEXT:    nop
+; PC64LE-NEXT:    li 3, 48
+; PC64LE-NEXT:    # kill: def $f1 killed $f1 def $vsl1
+; PC64LE-NEXT:    lxvd2x 0, 1, 3 # 16-byte Folded Reload
+; PC64LE-NEXT:    xxmrghd 34, 1, 0
+; PC64LE-NEXT:    addi 1, 1, 64
+; PC64LE-NEXT:    ld 0, 16(1)
+; PC64LE-NEXT:    mtlr 0
 ; PC64LE-NEXT:    blr
 ;
 ; PC64LE9-LABEL: constrained_vector_nearbyint_v2f64:
 ; PC64LE9:       # %bb.0: # %entry
+; PC64LE9-NEXT:    mflr 0
+; PC64LE9-NEXT:    std 0, 16(1)
+; PC64LE9-NEXT:    stdu 1, -48(1)
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI81_0@toc@ha
-; PC64LE9-NEXT:    addi 3, 3, .LCPI81_0@toc@l
-; PC64LE9-NEXT:    lxvx 0, 0, 3
-; PC64LE9-NEXT:    xvrdpic 34, 0
+; PC64LE9-NEXT:    lfd 1, .LCPI81_0@toc@l(3)
+; PC64LE9-NEXT:    bl nearbyint
+; PC64LE9-NEXT:    nop
+; PC64LE9-NEXT:    addis 3, 2, .LCPI81_1@toc@ha
+; PC64LE9-NEXT:    # kill: def $f1 killed $f1 def $vsl1
+; PC64LE9-NEXT:    stxv 1, 32(1) # 16-byte Folded Spill
+; PC64LE9-NEXT:    lfs 1, .LCPI81_1@toc@l(3)
+; PC64LE9-NEXT:    bl nearbyint
+; PC64LE9-NEXT:    nop
+; PC64LE9-NEXT:    lxv 0, 32(1) # 16-byte Folded Reload
+; PC64LE9-NEXT:    # kill: def $f1 killed $f1 def $vsl1
+; PC64LE9-NEXT:    xxmrghd 34, 1, 0
+; PC64LE9-NEXT:    addi 1, 1, 48
+; PC64LE9-NEXT:    ld 0, 16(1)
+; PC64LE9-NEXT:    mtlr 0
 ; PC64LE9-NEXT:    blr
 entry:
   %nearby = call <2 x double> @llvm.experimental.constrained.nearbyint.v2f64(
                                 <2 x double> <double 42.1, double 42.0>,
                                 metadata !"round.dynamic",
-                                metadata !"fpexcept.strict")
+                                metadata !"fpexcept.strict") #1
   ret <2 x double> %nearby
 }
 
-define <3 x float> @constrained_vector_nearbyint_v3f32() {
+define <3 x float> @constrained_vector_nearbyint_v3f32() #0 {
 ; PC64LE-LABEL: constrained_vector_nearbyint_v3f32:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    mflr 0
-; PC64LE-NEXT:    .cfi_def_cfa_offset 48
-; PC64LE-NEXT:    .cfi_offset lr, 16
-; PC64LE-NEXT:    .cfi_offset f30, -16
-; PC64LE-NEXT:    .cfi_offset f31, -8
 ; PC64LE-NEXT:    stfd 30, -16(1) # 8-byte Folded Spill
 ; PC64LE-NEXT:    stfd 31, -8(1) # 8-byte Folded Spill
 ; PC64LE-NEXT:    std 0, 16(1)
@@ -5508,18 +5229,14 @@ define <3 x float> @constrained_vector_nearbyint_v3f32() {
 ; PC64LE-NEXT:    vperm 2, 4, 2, 3
 ; PC64LE-NEXT:    addi 1, 1, 48
 ; PC64LE-NEXT:    ld 0, 16(1)
-; PC64LE-NEXT:    mtlr 0
 ; PC64LE-NEXT:    lfd 31, -8(1) # 8-byte Folded Reload
 ; PC64LE-NEXT:    lfd 30, -16(1) # 8-byte Folded Reload
+; PC64LE-NEXT:    mtlr 0
 ; PC64LE-NEXT:    blr
 ;
 ; PC64LE9-LABEL: constrained_vector_nearbyint_v3f32:
 ; PC64LE9:       # %bb.0: # %entry
 ; PC64LE9-NEXT:    mflr 0
-; PC64LE9-NEXT:    .cfi_def_cfa_offset 48
-; PC64LE9-NEXT:    .cfi_offset lr, 16
-; PC64LE9-NEXT:    .cfi_offset f30, -16
-; PC64LE9-NEXT:    .cfi_offset f31, -8
 ; PC64LE9-NEXT:    stfd 30, -16(1) # 8-byte Folded Spill
 ; PC64LE9-NEXT:    stfd 31, -8(1) # 8-byte Folded Spill
 ; PC64LE9-NEXT:    std 0, 16(1)
@@ -5551,40 +5268,52 @@ define <3 x float> @constrained_vector_nearbyint_v3f32() {
 ; PC64LE9-NEXT:    vperm 2, 4, 2, 3
 ; PC64LE9-NEXT:    addi 1, 1, 48
 ; PC64LE9-NEXT:    ld 0, 16(1)
-; PC64LE9-NEXT:    mtlr 0
 ; PC64LE9-NEXT:    lfd 31, -8(1) # 8-byte Folded Reload
 ; PC64LE9-NEXT:    lfd 30, -16(1) # 8-byte Folded Reload
+; PC64LE9-NEXT:    mtlr 0
 ; PC64LE9-NEXT:    blr
 entry:
   %nearby = call <3 x float> @llvm.experimental.constrained.nearbyint.v3f32(
                               <3 x float> <float 42.0, float 43.0, float 44.0>,
                               metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #1
   ret <3 x float> %nearby
 }
 
-define <3 x double> @constrained_vector_nearby_v3f64() {
+define <3 x double> @constrained_vector_nearby_v3f64() #0 {
 ; PC64LE-LABEL: constrained_vector_nearby_v3f64:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    mflr 0
 ; PC64LE-NEXT:    std 0, 16(1)
-; PC64LE-NEXT:    stdu 1, -32(1)
-; PC64LE-NEXT:    .cfi_def_cfa_offset 32
-; PC64LE-NEXT:    .cfi_offset lr, 16
+; PC64LE-NEXT:    stdu 1, -80(1)
+; PC64LE-NEXT:    li 3, 64
+; PC64LE-NEXT:    stxvd2x 63, 1, 3 # 16-byte Folded Spill
 ; PC64LE-NEXT:    addis 3, 2, .LCPI83_0@toc@ha
 ; PC64LE-NEXT:    lfd 1, .LCPI83_0@toc@l(3)
 ; PC64LE-NEXT:    bl nearbyint
 ; PC64LE-NEXT:    nop
+; PC64LE-NEXT:    li 3, 48
+; PC64LE-NEXT:    # kill: def $f1 killed $f1 def $vsl1
+; PC64LE-NEXT:    stxvd2x 1, 1, 3 # 16-byte Folded Spill
 ; PC64LE-NEXT:    addis 3, 2, .LCPI83_1@toc@ha
+; PC64LE-NEXT:    lfs 1, .LCPI83_1@toc@l(3)
+; PC64LE-NEXT:    bl nearbyint
+; PC64LE-NEXT:    nop
+; PC64LE-NEXT:    li 3, 48
+; PC64LE-NEXT:    # kill: def $f1 killed $f1 def $vsl1
+; PC64LE-NEXT:    lxvd2x 0, 1, 3 # 16-byte Folded Reload
+; PC64LE-NEXT:    addis 3, 2, .LCPI83_2@toc@ha
+; PC64LE-NEXT:    xxmrghd 63, 0, 1
+; PC64LE-NEXT:    lfd 1, .LCPI83_2@toc@l(3)
+; PC64LE-NEXT:    bl nearbyint
+; PC64LE-NEXT:    nop
+; PC64LE-NEXT:    xxswapd 0, 63
+; PC64LE-NEXT:    li 3, 64
+; PC64LE-NEXT:    xxlor 2, 63, 63
+; PC64LE-NEXT:    lxvd2x 63, 1, 3 # 16-byte Folded Reload
 ; PC64LE-NEXT:    fmr 3, 1
-; PC64LE-NEXT:    addi 3, 3, .LCPI83_1@toc@l
-; PC64LE-NEXT:    lxvd2x 0, 0, 3
-; PC64LE-NEXT:    xxswapd 0, 0
-; PC64LE-NEXT:    xvrdpic 2, 0
-; PC64LE-NEXT:    xxswapd 0, 2
-; PC64LE-NEXT:    # kill: def $f2 killed $f2 killed $vsl2
 ; PC64LE-NEXT:    fmr 1, 0
-; PC64LE-NEXT:    addi 1, 1, 32
+; PC64LE-NEXT:    addi 1, 1, 80
 ; PC64LE-NEXT:    ld 0, 16(1)
 ; PC64LE-NEXT:    mtlr 0
 ; PC64LE-NEXT:    blr
@@ -5593,22 +5322,31 @@ define <3 x double> @constrained_vector_nearby_v3f64() {
 ; PC64LE9:       # %bb.0: # %entry
 ; PC64LE9-NEXT:    mflr 0
 ; PC64LE9-NEXT:    std 0, 16(1)
-; PC64LE9-NEXT:    stdu 1, -32(1)
-; PC64LE9-NEXT:    .cfi_def_cfa_offset 32
-; PC64LE9-NEXT:    .cfi_offset lr, 16
+; PC64LE9-NEXT:    stdu 1, -64(1)
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI83_0@toc@ha
 ; PC64LE9-NEXT:    lfd 1, .LCPI83_0@toc@l(3)
+; PC64LE9-NEXT:    stxv 63, 48(1) # 16-byte Folded Spill
 ; PC64LE9-NEXT:    bl nearbyint
 ; PC64LE9-NEXT:    nop
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI83_1@toc@ha
-; PC64LE9-NEXT:    addi 3, 3, .LCPI83_1@toc@l
-; PC64LE9-NEXT:    lxvx 0, 0, 3
-; PC64LE9-NEXT:    xvrdpic 2, 0
+; PC64LE9-NEXT:    # kill: def $f1 killed $f1 def $vsl1
+; PC64LE9-NEXT:    stxv 1, 32(1) # 16-byte Folded Spill
+; PC64LE9-NEXT:    lfs 1, .LCPI83_1@toc@l(3)
+; PC64LE9-NEXT:    bl nearbyint
+; PC64LE9-NEXT:    nop
+; PC64LE9-NEXT:    lxv 0, 32(1) # 16-byte Folded Reload
+; PC64LE9-NEXT:    addis 3, 2, .LCPI83_2@toc@ha
+; PC64LE9-NEXT:    # kill: def $f1 killed $f1 def $vsl1
+; PC64LE9-NEXT:    xxmrghd 63, 0, 1
+; PC64LE9-NEXT:    lfd 1, .LCPI83_2@toc@l(3)
+; PC64LE9-NEXT:    bl nearbyint
+; PC64LE9-NEXT:    nop
 ; PC64LE9-NEXT:    fmr 3, 1
-; PC64LE9-NEXT:    xxswapd 1, 2
+; PC64LE9-NEXT:    xxswapd 1, 63
+; PC64LE9-NEXT:    xscpsgndp 2, 63, 63
+; PC64LE9-NEXT:    lxv 63, 48(1) # 16-byte Folded Reload
 ; PC64LE9-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
-; PC64LE9-NEXT:    # kill: def $f2 killed $f2 killed $vsl2
-; PC64LE9-NEXT:    addi 1, 1, 32
+; PC64LE9-NEXT:    addi 1, 1, 64
 ; PC64LE9-NEXT:    ld 0, 16(1)
 ; PC64LE9-NEXT:    mtlr 0
 ; PC64LE9-NEXT:    blr
@@ -5616,53 +5354,109 @@ entry:
   %nearby = call <3 x double> @llvm.experimental.constrained.nearbyint.v3f64(
                           <3 x double> <double 42.0, double 42.1, double 42.2>,
                           metadata !"round.dynamic",
-                          metadata !"fpexcept.strict")
+                          metadata !"fpexcept.strict") #1
   ret <3 x double> %nearby
 }
 
-define <4 x double> @constrained_vector_nearbyint_v4f64() {
+define <4 x double> @constrained_vector_nearbyint_v4f64() #0 {
 ; PC64LE-LABEL: constrained_vector_nearbyint_v4f64:
 ; PC64LE:       # %bb.0: # %entry
+; PC64LE-NEXT:    mflr 0
+; PC64LE-NEXT:    std 0, 16(1)
+; PC64LE-NEXT:    stdu 1, -80(1)
+; PC64LE-NEXT:    li 3, 64
+; PC64LE-NEXT:    stxvd2x 63, 1, 3 # 16-byte Folded Spill
 ; PC64LE-NEXT:    addis 3, 2, .LCPI84_0@toc@ha
-; PC64LE-NEXT:    addis 4, 2, .LCPI84_1@toc@ha
-; PC64LE-NEXT:    addi 3, 3, .LCPI84_0@toc@l
-; PC64LE-NEXT:    lxvd2x 0, 0, 3
-; PC64LE-NEXT:    addi 3, 4, .LCPI84_1@toc@l
-; PC64LE-NEXT:    lxvd2x 1, 0, 3
-; PC64LE-NEXT:    xxswapd 0, 0
-; PC64LE-NEXT:    xxswapd 1, 1
-; PC64LE-NEXT:    xvrdpic 34, 0
-; PC64LE-NEXT:    xvrdpic 35, 1
+; PC64LE-NEXT:    lfd 1, .LCPI84_0@toc@l(3)
+; PC64LE-NEXT:    bl nearbyint
+; PC64LE-NEXT:    nop
+; PC64LE-NEXT:    li 3, 48
+; PC64LE-NEXT:    # kill: def $f1 killed $f1 def $vsl1
+; PC64LE-NEXT:    stxvd2x 1, 1, 3 # 16-byte Folded Spill
+; PC64LE-NEXT:    addis 3, 2, .LCPI84_1@toc@ha
+; PC64LE-NEXT:    lfd 1, .LCPI84_1@toc@l(3)
+; PC64LE-NEXT:    bl nearbyint
+; PC64LE-NEXT:    nop
+; PC64LE-NEXT:    li 3, 48
+; PC64LE-NEXT:    # kill: def $f1 killed $f1 def $vsl1
+; PC64LE-NEXT:    lxvd2x 0, 1, 3 # 16-byte Folded Reload
+; PC64LE-NEXT:    addis 3, 2, .LCPI84_2@toc@ha
+; PC64LE-NEXT:    xxmrghd 63, 1, 0
+; PC64LE-NEXT:    lfd 1, .LCPI84_2@toc@l(3)
+; PC64LE-NEXT:    bl nearbyint
+; PC64LE-NEXT:    nop
+; PC64LE-NEXT:    li 3, 48
+; PC64LE-NEXT:    # kill: def $f1 killed $f1 def $vsl1
+; PC64LE-NEXT:    stxvd2x 1, 1, 3 # 16-byte Folded Spill
+; PC64LE-NEXT:    addis 3, 2, .LCPI84_3@toc@ha
+; PC64LE-NEXT:    lfd 1, .LCPI84_3@toc@l(3)
+; PC64LE-NEXT:    bl nearbyint
+; PC64LE-NEXT:    nop
+; PC64LE-NEXT:    li 3, 48
+; PC64LE-NEXT:    vmr 2, 31
+; PC64LE-NEXT:    # kill: def $f1 killed $f1 def $vsl1
+; PC64LE-NEXT:    lxvd2x 0, 1, 3 # 16-byte Folded Reload
+; PC64LE-NEXT:    li 3, 64
+; PC64LE-NEXT:    lxvd2x 63, 1, 3 # 16-byte Folded Reload
+; PC64LE-NEXT:    xxmrghd 35, 1, 0
+; PC64LE-NEXT:    addi 1, 1, 80
+; PC64LE-NEXT:    ld 0, 16(1)
+; PC64LE-NEXT:    mtlr 0
 ; PC64LE-NEXT:    blr
 ;
 ; PC64LE9-LABEL: constrained_vector_nearbyint_v4f64:
 ; PC64LE9:       # %bb.0: # %entry
+; PC64LE9-NEXT:    mflr 0
+; PC64LE9-NEXT:    std 0, 16(1)
+; PC64LE9-NEXT:    stdu 1, -64(1)
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI84_0@toc@ha
-; PC64LE9-NEXT:    addi 3, 3, .LCPI84_0@toc@l
-; PC64LE9-NEXT:    lxvx 0, 0, 3
+; PC64LE9-NEXT:    lfd 1, .LCPI84_0@toc@l(3)
+; PC64LE9-NEXT:    stxv 63, 48(1) # 16-byte Folded Spill
+; PC64LE9-NEXT:    bl nearbyint
+; PC64LE9-NEXT:    nop
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI84_1@toc@ha
-; PC64LE9-NEXT:    addi 3, 3, .LCPI84_1@toc@l
-; PC64LE9-NEXT:    xvrdpic 34, 0
-; PC64LE9-NEXT:    lxvx 0, 0, 3
-; PC64LE9-NEXT:    xvrdpic 35, 0
+; PC64LE9-NEXT:    # kill: def $f1 killed $f1 def $vsl1
+; PC64LE9-NEXT:    stxv 1, 32(1) # 16-byte Folded Spill
+; PC64LE9-NEXT:    lfd 1, .LCPI84_1@toc@l(3)
+; PC64LE9-NEXT:    bl nearbyint
+; PC64LE9-NEXT:    nop
+; PC64LE9-NEXT:    lxv 0, 32(1) # 16-byte Folded Reload
+; PC64LE9-NEXT:    addis 3, 2, .LCPI84_2@toc@ha
+; PC64LE9-NEXT:    # kill: def $f1 killed $f1 def $vsl1
+; PC64LE9-NEXT:    xxmrghd 63, 1, 0
+; PC64LE9-NEXT:    lfd 1, .LCPI84_2@toc@l(3)
+; PC64LE9-NEXT:    bl nearbyint
+; PC64LE9-NEXT:    nop
+; PC64LE9-NEXT:    addis 3, 2, .LCPI84_3@toc@ha
+; PC64LE9-NEXT:    # kill: def $f1 killed $f1 def $vsl1
+; PC64LE9-NEXT:    stxv 1, 32(1) # 16-byte Folded Spill
+; PC64LE9-NEXT:    lfd 1, .LCPI84_3@toc@l(3)
+; PC64LE9-NEXT:    bl nearbyint
+; PC64LE9-NEXT:    nop
+; PC64LE9-NEXT:    lxv 0, 32(1) # 16-byte Folded Reload
+; PC64LE9-NEXT:    vmr 2, 31
+; PC64LE9-NEXT:    lxv 63, 48(1) # 16-byte Folded Reload
+; PC64LE9-NEXT:    # kill: def $f1 killed $f1 def $vsl1
+; PC64LE9-NEXT:    xxmrghd 35, 1, 0
+; PC64LE9-NEXT:    addi 1, 1, 64
+; PC64LE9-NEXT:    ld 0, 16(1)
+; PC64LE9-NEXT:    mtlr 0
 ; PC64LE9-NEXT:    blr
 entry:
   %nearby = call <4 x double> @llvm.experimental.constrained.nearbyint.v4f64(
                                 <4 x double> <double 42.1, double 42.2,
                                               double 42.3, double 42.4>,
                                 metadata !"round.dynamic",
-                                metadata !"fpexcept.strict")
+                                metadata !"fpexcept.strict") #1
   ret <4 x double> %nearby
 }
 
-define <1 x float> @constrained_vector_maxnum_v1f32() {
+define <1 x float> @constrained_vector_maxnum_v1f32() #0 {
 ; PC64LE-LABEL: constrained_vector_maxnum_v1f32:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    mflr 0
 ; PC64LE-NEXT:    std 0, 16(1)
 ; PC64LE-NEXT:    stdu 1, -32(1)
-; PC64LE-NEXT:    .cfi_def_cfa_offset 32
-; PC64LE-NEXT:    .cfi_offset lr, 16
 ; PC64LE-NEXT:    addis 3, 2, .LCPI85_0@toc@ha
 ; PC64LE-NEXT:    addis 4, 2, .LCPI85_1@toc@ha
 ; PC64LE-NEXT:    lfs 1, .LCPI85_0@toc@l(3)
@@ -5681,8 +5475,6 @@ define <1 x float> @constrained_vector_maxnum_v1f32() {
 ; PC64LE9-NEXT:    mflr 0
 ; PC64LE9-NEXT:    std 0, 16(1)
 ; PC64LE9-NEXT:    stdu 1, -32(1)
-; PC64LE9-NEXT:    .cfi_def_cfa_offset 32
-; PC64LE9-NEXT:    .cfi_offset lr, 16
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI85_0@toc@ha
 ; PC64LE9-NEXT:    lfs 1, .LCPI85_0@toc@l(3)
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI85_1@toc@ha
@@ -5698,53 +5490,78 @@ define <1 x float> @constrained_vector_maxnum_v1f32() {
 entry:
   %max = call <1 x float> @llvm.experimental.constrained.maxnum.v1f32(
                                <1 x float> <float 42.0>, <1 x float> <float 41.0>,
-                               metadata !"round.dynamic",
-                               metadata !"fpexcept.strict")
+                               metadata !"fpexcept.strict") #1
   ret <1 x float> %max
 }
 
-define <2 x double> @constrained_vector_maxnum_v2f64() {
+define <2 x double> @constrained_vector_maxnum_v2f64() #0 {
 ; PC64LE-LABEL: constrained_vector_maxnum_v2f64:
 ; PC64LE:       # %bb.0: # %entry
+; PC64LE-NEXT:    mflr 0
+; PC64LE-NEXT:    std 0, 16(1)
+; PC64LE-NEXT:    stdu 1, -64(1)
 ; PC64LE-NEXT:    addis 3, 2, .LCPI86_0@toc@ha
 ; PC64LE-NEXT:    addis 4, 2, .LCPI86_1@toc@ha
-; PC64LE-NEXT:    addi 3, 3, .LCPI86_0@toc@l
-; PC64LE-NEXT:    addi 4, 4, .LCPI86_1@toc@l
-; PC64LE-NEXT:    lxvd2x 0, 0, 3
-; PC64LE-NEXT:    lxvd2x 1, 0, 4
-; PC64LE-NEXT:    xxswapd 0, 0
-; PC64LE-NEXT:    xxswapd 1, 1
-; PC64LE-NEXT:    xvmaxdp 34, 1, 0
-; PC64LE-NEXT:    blr
+; PC64LE-NEXT:    lfs 1, .LCPI86_0@toc@l(3)
+; PC64LE-NEXT:    lfs 2, .LCPI86_1@toc@l(4)
+; PC64LE-NEXT:    bl fmax
+; PC64LE-NEXT:    nop
+; PC64LE-NEXT:    li 3, 48
+; PC64LE-NEXT:    # kill: def $f1 killed $f1 def $vsl1
+; PC64LE-NEXT:    addis 4, 2, .LCPI86_3@toc@ha
+; PC64LE-NEXT:    stxvd2x 1, 1, 3 # 16-byte Folded Spill
+; PC64LE-NEXT:    addis 3, 2, .LCPI86_2@toc@ha
+; PC64LE-NEXT:    lfs 2, .LCPI86_3@toc@l(4)
+; PC64LE-NEXT:    lfs 1, .LCPI86_2@toc@l(3)
+; PC64LE-NEXT:    bl fmax
+; PC64LE-NEXT:    nop
+; PC64LE-NEXT:    li 3, 48
+; PC64LE-NEXT:    # kill: def $f1 killed $f1 def $vsl1
+; PC64LE-NEXT:    lxvd2x 0, 1, 3 # 16-byte Folded Reload
+; PC64LE-NEXT:    xxmrghd 34, 1, 0
+; PC64LE-NEXT:    addi 1, 1, 64
+; PC64LE-NEXT:    ld 0, 16(1)
+; PC64LE-NEXT:    mtlr 0
+; PC64LE-NEXT:    blr
 ;
 ; PC64LE9-LABEL: constrained_vector_maxnum_v2f64:
 ; PC64LE9:       # %bb.0: # %entry
+; PC64LE9-NEXT:    mflr 0
+; PC64LE9-NEXT:    std 0, 16(1)
+; PC64LE9-NEXT:    stdu 1, -48(1)
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI86_0@toc@ha
-; PC64LE9-NEXT:    addi 3, 3, .LCPI86_0@toc@l
-; PC64LE9-NEXT:    lxvx 0, 0, 3
+; PC64LE9-NEXT:    lfs 1, .LCPI86_0@toc@l(3)
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI86_1@toc@ha
-; PC64LE9-NEXT:    addi 3, 3, .LCPI86_1@toc@l
-; PC64LE9-NEXT:    lxvx 1, 0, 3
-; PC64LE9-NEXT:    xvmaxdp 34, 1, 0
+; PC64LE9-NEXT:    lfs 2, .LCPI86_1@toc@l(3)
+; PC64LE9-NEXT:    bl fmax
+; PC64LE9-NEXT:    nop
+; PC64LE9-NEXT:    addis 3, 2, .LCPI86_2@toc@ha
+; PC64LE9-NEXT:    # kill: def $f1 killed $f1 def $vsl1
+; PC64LE9-NEXT:    stxv 1, 32(1) # 16-byte Folded Spill
+; PC64LE9-NEXT:    lfs 1, .LCPI86_2@toc@l(3)
+; PC64LE9-NEXT:    addis 3, 2, .LCPI86_3@toc@ha
+; PC64LE9-NEXT:    lfs 2, .LCPI86_3@toc@l(3)
+; PC64LE9-NEXT:    bl fmax
+; PC64LE9-NEXT:    nop
+; PC64LE9-NEXT:    lxv 0, 32(1) # 16-byte Folded Reload
+; PC64LE9-NEXT:    # kill: def $f1 killed $f1 def $vsl1
+; PC64LE9-NEXT:    xxmrghd 34, 1, 0
+; PC64LE9-NEXT:    addi 1, 1, 48
+; PC64LE9-NEXT:    ld 0, 16(1)
+; PC64LE9-NEXT:    mtlr 0
 ; PC64LE9-NEXT:    blr
 entry:
   %max = call <2 x double> @llvm.experimental.constrained.maxnum.v2f64(
                                 <2 x double> <double 43.0, double 42.0>,
                                 <2 x double> <double 41.0, double 40.0>,
-                                metadata !"round.dynamic",
-                                metadata !"fpexcept.strict")
+                                metadata !"fpexcept.strict") #1
   ret <2 x double> %max
 }
 
-define <3 x float> @constrained_vector_maxnum_v3f32() {
+define <3 x float> @constrained_vector_maxnum_v3f32() #0 {
 ; PC64LE-LABEL: constrained_vector_maxnum_v3f32:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    mflr 0
-; PC64LE-NEXT:    .cfi_def_cfa_offset 64
-; PC64LE-NEXT:    .cfi_offset lr, 16
-; PC64LE-NEXT:    .cfi_offset f29, -24
-; PC64LE-NEXT:    .cfi_offset f30, -16
-; PC64LE-NEXT:    .cfi_offset f31, -8
 ; PC64LE-NEXT:    stfd 29, -24(1) # 8-byte Folded Spill
 ; PC64LE-NEXT:    stfd 30, -16(1) # 8-byte Folded Spill
 ; PC64LE-NEXT:    stfd 31, -8(1) # 8-byte Folded Spill
@@ -5783,20 +5600,15 @@ define <3 x float> @constrained_vector_maxnum_v3f32() {
 ; PC64LE-NEXT:    vperm 2, 3, 2, 4
 ; PC64LE-NEXT:    addi 1, 1, 64
 ; PC64LE-NEXT:    ld 0, 16(1)
-; PC64LE-NEXT:    mtlr 0
 ; PC64LE-NEXT:    lfd 31, -8(1) # 8-byte Folded Reload
 ; PC64LE-NEXT:    lfd 30, -16(1) # 8-byte Folded Reload
 ; PC64LE-NEXT:    lfd 29, -24(1) # 8-byte Folded Reload
+; PC64LE-NEXT:    mtlr 0
 ; PC64LE-NEXT:    blr
 ;
 ; PC64LE9-LABEL: constrained_vector_maxnum_v3f32:
 ; PC64LE9:       # %bb.0: # %entry
 ; PC64LE9-NEXT:    mflr 0
-; PC64LE9-NEXT:    .cfi_def_cfa_offset 64
-; PC64LE9-NEXT:    .cfi_offset lr, 16
-; PC64LE9-NEXT:    .cfi_offset f29, -24
-; PC64LE9-NEXT:    .cfi_offset f30, -16
-; PC64LE9-NEXT:    .cfi_offset f31, -8
 ; PC64LE9-NEXT:    stfd 29, -24(1) # 8-byte Folded Spill
 ; PC64LE9-NEXT:    stfd 30, -16(1) # 8-byte Folded Spill
 ; PC64LE9-NEXT:    stfd 31, -8(1) # 8-byte Folded Spill
@@ -5835,48 +5647,59 @@ define <3 x float> @constrained_vector_maxnum_v3f32() {
 ; PC64LE9-NEXT:    vperm 2, 3, 2, 4
 ; PC64LE9-NEXT:    addi 1, 1, 64
 ; PC64LE9-NEXT:    ld 0, 16(1)
-; PC64LE9-NEXT:    mtlr 0
 ; PC64LE9-NEXT:    lfd 31, -8(1) # 8-byte Folded Reload
 ; PC64LE9-NEXT:    lfd 30, -16(1) # 8-byte Folded Reload
+; PC64LE9-NEXT:    mtlr 0
 ; PC64LE9-NEXT:    lfd 29, -24(1) # 8-byte Folded Reload
 ; PC64LE9-NEXT:    blr
 entry:
   %max = call <3 x float> @llvm.experimental.constrained.maxnum.v3f32(
                               <3 x float> <float 43.0, float 44.0, float 45.0>,
                               <3 x float> <float 41.0, float 42.0, float 43.0>,
-                              metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #1
   ret <3 x float> %max
 }
 
-define <3 x double> @constrained_vector_max_v3f64() {
+define <3 x double> @constrained_vector_max_v3f64() #0 {
 ; PC64LE-LABEL: constrained_vector_max_v3f64:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    mflr 0
 ; PC64LE-NEXT:    std 0, 16(1)
-; PC64LE-NEXT:    stdu 1, -32(1)
-; PC64LE-NEXT:    .cfi_def_cfa_offset 32
-; PC64LE-NEXT:    .cfi_offset lr, 16
-; PC64LE-NEXT:    addis 3, 2, .LCPI88_0@toc@ha
+; PC64LE-NEXT:    stdu 1, -80(1)
+; PC64LE-NEXT:    li 3, 64
 ; PC64LE-NEXT:    addis 4, 2, .LCPI88_1@toc@ha
-; PC64LE-NEXT:    lfs 1, .LCPI88_0@toc@l(3)
+; PC64LE-NEXT:    stxvd2x 63, 1, 3 # 16-byte Folded Spill
+; PC64LE-NEXT:    addis 3, 2, .LCPI88_0@toc@ha
 ; PC64LE-NEXT:    lfs 2, .LCPI88_1@toc@l(4)
+; PC64LE-NEXT:    lfs 1, .LCPI88_0@toc@l(3)
 ; PC64LE-NEXT:    bl fmax
 ; PC64LE-NEXT:    nop
-; PC64LE-NEXT:    addis 3, 2, .LCPI88_2@toc@ha
+; PC64LE-NEXT:    li 3, 48
+; PC64LE-NEXT:    # kill: def $f1 killed $f1 def $vsl1
 ; PC64LE-NEXT:    addis 4, 2, .LCPI88_3@toc@ha
+; PC64LE-NEXT:    stxvd2x 1, 1, 3 # 16-byte Folded Spill
+; PC64LE-NEXT:    addis 3, 2, .LCPI88_2@toc@ha
+; PC64LE-NEXT:    lfs 2, .LCPI88_3@toc@l(4)
+; PC64LE-NEXT:    lfs 1, .LCPI88_2@toc@l(3)
+; PC64LE-NEXT:    bl fmax
+; PC64LE-NEXT:    nop
+; PC64LE-NEXT:    li 3, 48
+; PC64LE-NEXT:    addis 4, 2, .LCPI88_5@toc@ha
+; PC64LE-NEXT:    # kill: def $f1 killed $f1 def $vsl1
+; PC64LE-NEXT:    lxvd2x 0, 1, 3 # 16-byte Folded Reload
+; PC64LE-NEXT:    addis 3, 2, .LCPI88_4@toc@ha
+; PC64LE-NEXT:    lfs 2, .LCPI88_5@toc@l(4)
+; PC64LE-NEXT:    xxmrghd 63, 1, 0
+; PC64LE-NEXT:    lfs 1, .LCPI88_4@toc@l(3)
+; PC64LE-NEXT:    bl fmax
+; PC64LE-NEXT:    nop
+; PC64LE-NEXT:    xxswapd 0, 63
+; PC64LE-NEXT:    li 3, 64
+; PC64LE-NEXT:    xxlor 2, 63, 63
+; PC64LE-NEXT:    lxvd2x 63, 1, 3 # 16-byte Folded Reload
 ; PC64LE-NEXT:    fmr 3, 1
-; PC64LE-NEXT:    addi 3, 3, .LCPI88_2@toc@l
-; PC64LE-NEXT:    addi 4, 4, .LCPI88_3@toc@l
-; PC64LE-NEXT:    lxvd2x 0, 0, 3
-; PC64LE-NEXT:    lxvd2x 2, 0, 4
-; PC64LE-NEXT:    xxswapd 0, 0
-; PC64LE-NEXT:    xxswapd 2, 2
-; PC64LE-NEXT:    xvmaxdp 2, 2, 0
-; PC64LE-NEXT:    xxswapd 0, 2
-; PC64LE-NEXT:    # kill: def $f2 killed $f2 killed $vsl2
 ; PC64LE-NEXT:    fmr 1, 0
-; PC64LE-NEXT:    addi 1, 1, 32
+; PC64LE-NEXT:    addi 1, 1, 80
 ; PC64LE-NEXT:    ld 0, 16(1)
 ; PC64LE-NEXT:    mtlr 0
 ; PC64LE-NEXT:    blr
@@ -5885,27 +5708,37 @@ define <3 x double> @constrained_vector_max_v3f64() {
 ; PC64LE9:       # %bb.0: # %entry
 ; PC64LE9-NEXT:    mflr 0
 ; PC64LE9-NEXT:    std 0, 16(1)
-; PC64LE9-NEXT:    stdu 1, -32(1)
-; PC64LE9-NEXT:    .cfi_def_cfa_offset 32
-; PC64LE9-NEXT:    .cfi_offset lr, 16
+; PC64LE9-NEXT:    stdu 1, -64(1)
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI88_0@toc@ha
 ; PC64LE9-NEXT:    lfs 1, .LCPI88_0@toc@l(3)
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI88_1@toc@ha
 ; PC64LE9-NEXT:    lfs 2, .LCPI88_1@toc@l(3)
+; PC64LE9-NEXT:    stxv 63, 48(1) # 16-byte Folded Spill
 ; PC64LE9-NEXT:    bl fmax
 ; PC64LE9-NEXT:    nop
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI88_2@toc@ha
-; PC64LE9-NEXT:    addi 3, 3, .LCPI88_2@toc@l
-; PC64LE9-NEXT:    lxvx 0, 0, 3
+; PC64LE9-NEXT:    # kill: def $f1 killed $f1 def $vsl1
+; PC64LE9-NEXT:    stxv 1, 32(1) # 16-byte Folded Spill
+; PC64LE9-NEXT:    lfs 1, .LCPI88_2@toc@l(3)
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI88_3@toc@ha
-; PC64LE9-NEXT:    addi 3, 3, .LCPI88_3@toc@l
+; PC64LE9-NEXT:    lfs 2, .LCPI88_3@toc@l(3)
+; PC64LE9-NEXT:    bl fmax
+; PC64LE9-NEXT:    nop
+; PC64LE9-NEXT:    lxv 0, 32(1) # 16-byte Folded Reload
+; PC64LE9-NEXT:    addis 3, 2, .LCPI88_4@toc@ha
+; PC64LE9-NEXT:    # kill: def $f1 killed $f1 def $vsl1
+; PC64LE9-NEXT:    xxmrghd 63, 1, 0
+; PC64LE9-NEXT:    lfs 1, .LCPI88_4@toc@l(3)
+; PC64LE9-NEXT:    addis 3, 2, .LCPI88_5@toc@ha
+; PC64LE9-NEXT:    lfs 2, .LCPI88_5@toc@l(3)
+; PC64LE9-NEXT:    bl fmax
+; PC64LE9-NEXT:    nop
 ; PC64LE9-NEXT:    fmr 3, 1
-; PC64LE9-NEXT:    lxvx 1, 0, 3
-; PC64LE9-NEXT:    xvmaxdp 2, 1, 0
-; PC64LE9-NEXT:    xxswapd 1, 2
+; PC64LE9-NEXT:    xxswapd 1, 63
+; PC64LE9-NEXT:    xscpsgndp 2, 63, 63
+; PC64LE9-NEXT:    lxv 63, 48(1) # 16-byte Folded Reload
 ; PC64LE9-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
-; PC64LE9-NEXT:    # kill: def $f2 killed $f2 killed $vsl2
-; PC64LE9-NEXT:    addi 1, 1, 32
+; PC64LE9-NEXT:    addi 1, 1, 64
 ; PC64LE9-NEXT:    ld 0, 16(1)
 ; PC64LE9-NEXT:    mtlr 0
 ; PC64LE9-NEXT:    blr
@@ -5913,50 +5746,109 @@ entry:
   %max = call <3 x double> @llvm.experimental.constrained.maxnum.v3f64(
                           <3 x double> <double 43.0, double 44.0, double 45.0>,
                           <3 x double> <double 40.0, double 41.0, double 42.0>,
-                          metadata !"round.dynamic",
-                          metadata !"fpexcept.strict")
+                          metadata !"fpexcept.strict") #1
   ret <3 x double> %max
 }
 
-define <4 x double> @constrained_vector_maxnum_v4f64() {
+define <4 x double> @constrained_vector_maxnum_v4f64() #0 {
 ; PC64LE-LABEL: constrained_vector_maxnum_v4f64:
 ; PC64LE:       # %bb.0: # %entry
-; PC64LE-NEXT:    addis 3, 2, .LCPI89_0@toc@ha
+; PC64LE-NEXT:    mflr 0
+; PC64LE-NEXT:    std 0, 16(1)
+; PC64LE-NEXT:    stdu 1, -80(1)
+; PC64LE-NEXT:    li 3, 64
 ; PC64LE-NEXT:    addis 4, 2, .LCPI89_1@toc@ha
-; PC64LE-NEXT:    addis 5, 2, .LCPI89_2@toc@ha
-; PC64LE-NEXT:    addis 6, 2, .LCPI89_3@toc@ha
-; PC64LE-NEXT:    addi 3, 3, .LCPI89_0@toc@l
-; PC64LE-NEXT:    addi 4, 4, .LCPI89_1@toc@l
-; PC64LE-NEXT:    lxvd2x 0, 0, 3
-; PC64LE-NEXT:    lxvd2x 1, 0, 4
-; PC64LE-NEXT:    addi 3, 5, .LCPI89_2@toc@l
-; PC64LE-NEXT:    addi 4, 6, .LCPI89_3@toc@l
-; PC64LE-NEXT:    lxvd2x 2, 0, 3
-; PC64LE-NEXT:    lxvd2x 3, 0, 4
-; PC64LE-NEXT:    xxswapd 0, 0
-; PC64LE-NEXT:    xxswapd 1, 1
-; PC64LE-NEXT:    xxswapd 2, 2
-; PC64LE-NEXT:    xxswapd 3, 3
-; PC64LE-NEXT:    xvmaxdp 34, 1, 0
-; PC64LE-NEXT:    xvmaxdp 35, 3, 2
+; PC64LE-NEXT:    stxvd2x 63, 1, 3 # 16-byte Folded Spill
+; PC64LE-NEXT:    addis 3, 2, .LCPI89_0@toc@ha
+; PC64LE-NEXT:    lfs 2, .LCPI89_1@toc@l(4)
+; PC64LE-NEXT:    lfs 1, .LCPI89_0@toc@l(3)
+; PC64LE-NEXT:    bl fmax
+; PC64LE-NEXT:    nop
+; PC64LE-NEXT:    li 3, 48
+; PC64LE-NEXT:    # kill: def $f1 killed $f1 def $vsl1
+; PC64LE-NEXT:    addis 4, 2, .LCPI89_3@toc@ha
+; PC64LE-NEXT:    stxvd2x 1, 1, 3 # 16-byte Folded Spill
+; PC64LE-NEXT:    addis 3, 2, .LCPI89_2@toc@ha
+; PC64LE-NEXT:    lfs 2, .LCPI89_3@toc@l(4)
+; PC64LE-NEXT:    lfs 1, .LCPI89_2@toc@l(3)
+; PC64LE-NEXT:    bl fmax
+; PC64LE-NEXT:    nop
+; PC64LE-NEXT:    li 3, 48
+; PC64LE-NEXT:    addis 4, 2, .LCPI89_5@toc@ha
+; PC64LE-NEXT:    # kill: def $f1 killed $f1 def $vsl1
+; PC64LE-NEXT:    lxvd2x 0, 1, 3 # 16-byte Folded Reload
+; PC64LE-NEXT:    addis 3, 2, .LCPI89_4@toc@ha
+; PC64LE-NEXT:    lfs 2, .LCPI89_5@toc@l(4)
+; PC64LE-NEXT:    xxmrghd 63, 1, 0
+; PC64LE-NEXT:    lfs 1, .LCPI89_4@toc@l(3)
+; PC64LE-NEXT:    bl fmax
+; PC64LE-NEXT:    nop
+; PC64LE-NEXT:    li 3, 48
+; PC64LE-NEXT:    # kill: def $f1 killed $f1 def $vsl1
+; PC64LE-NEXT:    addis 4, 2, .LCPI89_7@toc@ha
+; PC64LE-NEXT:    stxvd2x 1, 1, 3 # 16-byte Folded Spill
+; PC64LE-NEXT:    addis 3, 2, .LCPI89_6@toc@ha
+; PC64LE-NEXT:    lfs 2, .LCPI89_7@toc@l(4)
+; PC64LE-NEXT:    lfs 1, .LCPI89_6@toc@l(3)
+; PC64LE-NEXT:    bl fmax
+; PC64LE-NEXT:    nop
+; PC64LE-NEXT:    li 3, 48
+; PC64LE-NEXT:    vmr 2, 31
+; PC64LE-NEXT:    # kill: def $f1 killed $f1 def $vsl1
+; PC64LE-NEXT:    lxvd2x 0, 1, 3 # 16-byte Folded Reload
+; PC64LE-NEXT:    li 3, 64
+; PC64LE-NEXT:    lxvd2x 63, 1, 3 # 16-byte Folded Reload
+; PC64LE-NEXT:    xxmrghd 35, 1, 0
+; PC64LE-NEXT:    addi 1, 1, 80
+; PC64LE-NEXT:    ld 0, 16(1)
+; PC64LE-NEXT:    mtlr 0
 ; PC64LE-NEXT:    blr
 ;
 ; PC64LE9-LABEL: constrained_vector_maxnum_v4f64:
 ; PC64LE9:       # %bb.0: # %entry
+; PC64LE9-NEXT:    mflr 0
+; PC64LE9-NEXT:    std 0, 16(1)
+; PC64LE9-NEXT:    stdu 1, -64(1)
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI89_0@toc@ha
-; PC64LE9-NEXT:    addi 3, 3, .LCPI89_0@toc@l
-; PC64LE9-NEXT:    lxvx 0, 0, 3
+; PC64LE9-NEXT:    lfs 1, .LCPI89_0@toc@l(3)
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI89_1@toc@ha
-; PC64LE9-NEXT:    addi 3, 3, .LCPI89_1@toc@l
-; PC64LE9-NEXT:    lxvx 1, 0, 3
+; PC64LE9-NEXT:    lfs 2, .LCPI89_1@toc@l(3)
+; PC64LE9-NEXT:    stxv 63, 48(1) # 16-byte Folded Spill
+; PC64LE9-NEXT:    bl fmax
+; PC64LE9-NEXT:    nop
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI89_2@toc@ha
-; PC64LE9-NEXT:    addi 3, 3, .LCPI89_2@toc@l
-; PC64LE9-NEXT:    xvmaxdp 34, 1, 0
-; PC64LE9-NEXT:    lxvx 0, 0, 3
+; PC64LE9-NEXT:    # kill: def $f1 killed $f1 def $vsl1
+; PC64LE9-NEXT:    stxv 1, 32(1) # 16-byte Folded Spill
+; PC64LE9-NEXT:    lfs 1, .LCPI89_2@toc@l(3)
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI89_3@toc@ha
-; PC64LE9-NEXT:    addi 3, 3, .LCPI89_3@toc@l
-; PC64LE9-NEXT:    lxvx 1, 0, 3
-; PC64LE9-NEXT:    xvmaxdp 35, 1, 0
+; PC64LE9-NEXT:    lfs 2, .LCPI89_3@toc@l(3)
+; PC64LE9-NEXT:    bl fmax
+; PC64LE9-NEXT:    nop
+; PC64LE9-NEXT:    lxv 0, 32(1) # 16-byte Folded Reload
+; PC64LE9-NEXT:    addis 3, 2, .LCPI89_4@toc@ha
+; PC64LE9-NEXT:    # kill: def $f1 killed $f1 def $vsl1
+; PC64LE9-NEXT:    xxmrghd 63, 1, 0
+; PC64LE9-NEXT:    lfs 1, .LCPI89_4@toc@l(3)
+; PC64LE9-NEXT:    addis 3, 2, .LCPI89_5@toc@ha
+; PC64LE9-NEXT:    lfs 2, .LCPI89_5@toc@l(3)
+; PC64LE9-NEXT:    bl fmax
+; PC64LE9-NEXT:    nop
+; PC64LE9-NEXT:    addis 3, 2, .LCPI89_6@toc@ha
+; PC64LE9-NEXT:    # kill: def $f1 killed $f1 def $vsl1
+; PC64LE9-NEXT:    stxv 1, 32(1) # 16-byte Folded Spill
+; PC64LE9-NEXT:    lfs 1, .LCPI89_6@toc@l(3)
+; PC64LE9-NEXT:    addis 3, 2, .LCPI89_7@toc@ha
+; PC64LE9-NEXT:    lfs 2, .LCPI89_7@toc@l(3)
+; PC64LE9-NEXT:    bl fmax
+; PC64LE9-NEXT:    nop
+; PC64LE9-NEXT:    lxv 0, 32(1) # 16-byte Folded Reload
+; PC64LE9-NEXT:    vmr 2, 31
+; PC64LE9-NEXT:    lxv 63, 48(1) # 16-byte Folded Reload
+; PC64LE9-NEXT:    # kill: def $f1 killed $f1 def $vsl1
+; PC64LE9-NEXT:    xxmrghd 35, 1, 0
+; PC64LE9-NEXT:    addi 1, 1, 64
+; PC64LE9-NEXT:    ld 0, 16(1)
+; PC64LE9-NEXT:    mtlr 0
 ; PC64LE9-NEXT:    blr
 entry:
   %max = call <4 x double> @llvm.experimental.constrained.maxnum.v4f64(
@@ -5964,19 +5856,16 @@ entry:
                                               double 46.0, double 47.0>,
                                 <4 x double> <double 40.0, double 41.0,
                                               double 42.0, double 43.0>,
-                                metadata !"round.dynamic",
-                                metadata !"fpexcept.strict")
+                                metadata !"fpexcept.strict") #1
   ret <4 x double> %max
 }
 
-define <1 x float> @constrained_vector_minnum_v1f32() {
+define <1 x float> @constrained_vector_minnum_v1f32() #0 {
 ; PC64LE-LABEL: constrained_vector_minnum_v1f32:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    mflr 0
 ; PC64LE-NEXT:    std 0, 16(1)
 ; PC64LE-NEXT:    stdu 1, -32(1)
-; PC64LE-NEXT:    .cfi_def_cfa_offset 32
-; PC64LE-NEXT:    .cfi_offset lr, 16
 ; PC64LE-NEXT:    addis 3, 2, .LCPI90_0@toc@ha
 ; PC64LE-NEXT:    addis 4, 2, .LCPI90_1@toc@ha
 ; PC64LE-NEXT:    lfs 1, .LCPI90_0@toc@l(3)
@@ -5995,8 +5884,6 @@ define <1 x float> @constrained_vector_minnum_v1f32() {
 ; PC64LE9-NEXT:    mflr 0
 ; PC64LE9-NEXT:    std 0, 16(1)
 ; PC64LE9-NEXT:    stdu 1, -32(1)
-; PC64LE9-NEXT:    .cfi_def_cfa_offset 32
-; PC64LE9-NEXT:    .cfi_offset lr, 16
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI90_0@toc@ha
 ; PC64LE9-NEXT:    lfs 1, .LCPI90_0@toc@l(3)
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI90_1@toc@ha
@@ -6012,53 +5899,78 @@ define <1 x float> @constrained_vector_minnum_v1f32() {
  entry:
   %min = call <1 x float> @llvm.experimental.constrained.minnum.v1f32(
                                <1 x float> <float 42.0>, <1 x float> <float 41.0>,
-                               metadata !"round.dynamic",
-                               metadata !"fpexcept.strict")
+                               metadata !"fpexcept.strict") #1
   ret <1 x float> %min
 }
 
-define <2 x double> @constrained_vector_minnum_v2f64() {
+define <2 x double> @constrained_vector_minnum_v2f64() #0 {
 ; PC64LE-LABEL: constrained_vector_minnum_v2f64:
 ; PC64LE:       # %bb.0: # %entry
+; PC64LE-NEXT:    mflr 0
+; PC64LE-NEXT:    std 0, 16(1)
+; PC64LE-NEXT:    stdu 1, -64(1)
 ; PC64LE-NEXT:    addis 3, 2, .LCPI91_0@toc@ha
 ; PC64LE-NEXT:    addis 4, 2, .LCPI91_1@toc@ha
-; PC64LE-NEXT:    addi 3, 3, .LCPI91_0@toc@l
-; PC64LE-NEXT:    addi 4, 4, .LCPI91_1@toc@l
-; PC64LE-NEXT:    lxvd2x 0, 0, 3
-; PC64LE-NEXT:    lxvd2x 1, 0, 4
-; PC64LE-NEXT:    xxswapd 0, 0
-; PC64LE-NEXT:    xxswapd 1, 1
-; PC64LE-NEXT:    xvmindp 34, 1, 0
+; PC64LE-NEXT:    lfs 1, .LCPI91_0@toc@l(3)
+; PC64LE-NEXT:    lfs 2, .LCPI91_1@toc@l(4)
+; PC64LE-NEXT:    bl fmin
+; PC64LE-NEXT:    nop
+; PC64LE-NEXT:    li 3, 48
+; PC64LE-NEXT:    # kill: def $f1 killed $f1 def $vsl1
+; PC64LE-NEXT:    addis 4, 2, .LCPI91_3@toc@ha
+; PC64LE-NEXT:    stxvd2x 1, 1, 3 # 16-byte Folded Spill
+; PC64LE-NEXT:    addis 3, 2, .LCPI91_2@toc@ha
+; PC64LE-NEXT:    lfs 2, .LCPI91_3@toc@l(4)
+; PC64LE-NEXT:    lfs 1, .LCPI91_2@toc@l(3)
+; PC64LE-NEXT:    bl fmin
+; PC64LE-NEXT:    nop
+; PC64LE-NEXT:    li 3, 48
+; PC64LE-NEXT:    # kill: def $f1 killed $f1 def $vsl1
+; PC64LE-NEXT:    lxvd2x 0, 1, 3 # 16-byte Folded Reload
+; PC64LE-NEXT:    xxmrghd 34, 1, 0
+; PC64LE-NEXT:    addi 1, 1, 64
+; PC64LE-NEXT:    ld 0, 16(1)
+; PC64LE-NEXT:    mtlr 0
 ; PC64LE-NEXT:    blr
 ;
 ; PC64LE9-LABEL: constrained_vector_minnum_v2f64:
 ; PC64LE9:       # %bb.0: # %entry
+; PC64LE9-NEXT:    mflr 0
+; PC64LE9-NEXT:    std 0, 16(1)
+; PC64LE9-NEXT:    stdu 1, -48(1)
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI91_0@toc@ha
-; PC64LE9-NEXT:    addi 3, 3, .LCPI91_0@toc@l
-; PC64LE9-NEXT:    lxvx 0, 0, 3
+; PC64LE9-NEXT:    lfs 1, .LCPI91_0@toc@l(3)
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI91_1@toc@ha
-; PC64LE9-NEXT:    addi 3, 3, .LCPI91_1@toc@l
-; PC64LE9-NEXT:    lxvx 1, 0, 3
-; PC64LE9-NEXT:    xvmindp 34, 1, 0
+; PC64LE9-NEXT:    lfs 2, .LCPI91_1@toc@l(3)
+; PC64LE9-NEXT:    bl fmin
+; PC64LE9-NEXT:    nop
+; PC64LE9-NEXT:    addis 3, 2, .LCPI91_2@toc@ha
+; PC64LE9-NEXT:    # kill: def $f1 killed $f1 def $vsl1
+; PC64LE9-NEXT:    stxv 1, 32(1) # 16-byte Folded Spill
+; PC64LE9-NEXT:    lfs 1, .LCPI91_2@toc@l(3)
+; PC64LE9-NEXT:    addis 3, 2, .LCPI91_3@toc@ha
+; PC64LE9-NEXT:    lfs 2, .LCPI91_3@toc@l(3)
+; PC64LE9-NEXT:    bl fmin
+; PC64LE9-NEXT:    nop
+; PC64LE9-NEXT:    lxv 0, 32(1) # 16-byte Folded Reload
+; PC64LE9-NEXT:    # kill: def $f1 killed $f1 def $vsl1
+; PC64LE9-NEXT:    xxmrghd 34, 1, 0
+; PC64LE9-NEXT:    addi 1, 1, 48
+; PC64LE9-NEXT:    ld 0, 16(1)
+; PC64LE9-NEXT:    mtlr 0
 ; PC64LE9-NEXT:    blr
 entry:
   %min = call <2 x double> @llvm.experimental.constrained.minnum.v2f64(
                                 <2 x double> <double 43.0, double 42.0>,
                                 <2 x double> <double 41.0, double 40.0>,
-                                metadata !"round.dynamic",
-                                metadata !"fpexcept.strict")
+                                metadata !"fpexcept.strict") #1
   ret <2 x double> %min
 }
 
-define <3 x float> @constrained_vector_minnum_v3f32() {
+define <3 x float> @constrained_vector_minnum_v3f32() #0 {
 ; PC64LE-LABEL: constrained_vector_minnum_v3f32:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    mflr 0
-; PC64LE-NEXT:    .cfi_def_cfa_offset 64
-; PC64LE-NEXT:    .cfi_offset lr, 16
-; PC64LE-NEXT:    .cfi_offset f29, -24
-; PC64LE-NEXT:    .cfi_offset f30, -16
-; PC64LE-NEXT:    .cfi_offset f31, -8
 ; PC64LE-NEXT:    stfd 29, -24(1) # 8-byte Folded Spill
 ; PC64LE-NEXT:    stfd 30, -16(1) # 8-byte Folded Spill
 ; PC64LE-NEXT:    stfd 31, -8(1) # 8-byte Folded Spill
@@ -6097,20 +6009,15 @@ define <3 x float> @constrained_vector_minnum_v3f32() {
 ; PC64LE-NEXT:    vperm 2, 3, 2, 4
 ; PC64LE-NEXT:    addi 1, 1, 64
 ; PC64LE-NEXT:    ld 0, 16(1)
-; PC64LE-NEXT:    mtlr 0
 ; PC64LE-NEXT:    lfd 31, -8(1) # 8-byte Folded Reload
 ; PC64LE-NEXT:    lfd 30, -16(1) # 8-byte Folded Reload
 ; PC64LE-NEXT:    lfd 29, -24(1) # 8-byte Folded Reload
+; PC64LE-NEXT:    mtlr 0
 ; PC64LE-NEXT:    blr
 ;
 ; PC64LE9-LABEL: constrained_vector_minnum_v3f32:
 ; PC64LE9:       # %bb.0: # %entry
 ; PC64LE9-NEXT:    mflr 0
-; PC64LE9-NEXT:    .cfi_def_cfa_offset 64
-; PC64LE9-NEXT:    .cfi_offset lr, 16
-; PC64LE9-NEXT:    .cfi_offset f29, -24
-; PC64LE9-NEXT:    .cfi_offset f30, -16
-; PC64LE9-NEXT:    .cfi_offset f31, -8
 ; PC64LE9-NEXT:    stfd 29, -24(1) # 8-byte Folded Spill
 ; PC64LE9-NEXT:    stfd 30, -16(1) # 8-byte Folded Spill
 ; PC64LE9-NEXT:    stfd 31, -8(1) # 8-byte Folded Spill
@@ -6149,48 +6056,59 @@ define <3 x float> @constrained_vector_minnum_v3f32() {
 ; PC64LE9-NEXT:    vperm 2, 3, 2, 4
 ; PC64LE9-NEXT:    addi 1, 1, 64
 ; PC64LE9-NEXT:    ld 0, 16(1)
-; PC64LE9-NEXT:    mtlr 0
 ; PC64LE9-NEXT:    lfd 31, -8(1) # 8-byte Folded Reload
 ; PC64LE9-NEXT:    lfd 30, -16(1) # 8-byte Folded Reload
+; PC64LE9-NEXT:    mtlr 0
 ; PC64LE9-NEXT:    lfd 29, -24(1) # 8-byte Folded Reload
 ; PC64LE9-NEXT:    blr
 entry:
   %min = call <3 x float> @llvm.experimental.constrained.minnum.v3f32(
                               <3 x float> <float 43.0, float 44.0, float 45.0>,
                               <3 x float> <float 41.0, float 42.0, float 43.0>,
-                              metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #1
   ret <3 x float> %min
 }
 
-define <3 x double> @constrained_vector_min_v3f64() {
+define <3 x double> @constrained_vector_min_v3f64() #0 {
 ; PC64LE-LABEL: constrained_vector_min_v3f64:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    mflr 0
 ; PC64LE-NEXT:    std 0, 16(1)
-; PC64LE-NEXT:    stdu 1, -32(1)
-; PC64LE-NEXT:    .cfi_def_cfa_offset 32
-; PC64LE-NEXT:    .cfi_offset lr, 16
-; PC64LE-NEXT:    addis 3, 2, .LCPI93_0@toc@ha
+; PC64LE-NEXT:    stdu 1, -80(1)
+; PC64LE-NEXT:    li 3, 64
 ; PC64LE-NEXT:    addis 4, 2, .LCPI93_1@toc@ha
-; PC64LE-NEXT:    lfs 1, .LCPI93_0@toc@l(3)
+; PC64LE-NEXT:    stxvd2x 63, 1, 3 # 16-byte Folded Spill
+; PC64LE-NEXT:    addis 3, 2, .LCPI93_0@toc@ha
 ; PC64LE-NEXT:    lfs 2, .LCPI93_1@toc@l(4)
+; PC64LE-NEXT:    lfs 1, .LCPI93_0@toc@l(3)
 ; PC64LE-NEXT:    bl fmin
 ; PC64LE-NEXT:    nop
-; PC64LE-NEXT:    addis 3, 2, .LCPI93_2@toc@ha
+; PC64LE-NEXT:    li 3, 48
+; PC64LE-NEXT:    # kill: def $f1 killed $f1 def $vsl1
 ; PC64LE-NEXT:    addis 4, 2, .LCPI93_3@toc@ha
+; PC64LE-NEXT:    stxvd2x 1, 1, 3 # 16-byte Folded Spill
+; PC64LE-NEXT:    addis 3, 2, .LCPI93_2@toc@ha
+; PC64LE-NEXT:    lfs 2, .LCPI93_3@toc@l(4)
+; PC64LE-NEXT:    lfs 1, .LCPI93_2@toc@l(3)
+; PC64LE-NEXT:    bl fmin
+; PC64LE-NEXT:    nop
+; PC64LE-NEXT:    li 3, 48
+; PC64LE-NEXT:    addis 4, 2, .LCPI93_5@toc@ha
+; PC64LE-NEXT:    # kill: def $f1 killed $f1 def $vsl1
+; PC64LE-NEXT:    lxvd2x 0, 1, 3 # 16-byte Folded Reload
+; PC64LE-NEXT:    addis 3, 2, .LCPI93_4@toc@ha
+; PC64LE-NEXT:    lfs 2, .LCPI93_5@toc@l(4)
+; PC64LE-NEXT:    xxmrghd 63, 1, 0
+; PC64LE-NEXT:    lfs 1, .LCPI93_4@toc@l(3)
+; PC64LE-NEXT:    bl fmin
+; PC64LE-NEXT:    nop
+; PC64LE-NEXT:    xxswapd 0, 63
+; PC64LE-NEXT:    li 3, 64
+; PC64LE-NEXT:    xxlor 2, 63, 63
+; PC64LE-NEXT:    lxvd2x 63, 1, 3 # 16-byte Folded Reload
 ; PC64LE-NEXT:    fmr 3, 1
-; PC64LE-NEXT:    addi 3, 3, .LCPI93_2@toc@l
-; PC64LE-NEXT:    addi 4, 4, .LCPI93_3@toc@l
-; PC64LE-NEXT:    lxvd2x 0, 0, 3
-; PC64LE-NEXT:    lxvd2x 2, 0, 4
-; PC64LE-NEXT:    xxswapd 0, 0
-; PC64LE-NEXT:    xxswapd 2, 2
-; PC64LE-NEXT:    xvmindp 2, 2, 0
-; PC64LE-NEXT:    xxswapd 0, 2
-; PC64LE-NEXT:    # kill: def $f2 killed $f2 killed $vsl2
 ; PC64LE-NEXT:    fmr 1, 0
-; PC64LE-NEXT:    addi 1, 1, 32
+; PC64LE-NEXT:    addi 1, 1, 80
 ; PC64LE-NEXT:    ld 0, 16(1)
 ; PC64LE-NEXT:    mtlr 0
 ; PC64LE-NEXT:    blr
@@ -6199,27 +6117,37 @@ define <3 x double> @constrained_vector_min_v3f64() {
 ; PC64LE9:       # %bb.0: # %entry
 ; PC64LE9-NEXT:    mflr 0
 ; PC64LE9-NEXT:    std 0, 16(1)
-; PC64LE9-NEXT:    stdu 1, -32(1)
-; PC64LE9-NEXT:    .cfi_def_cfa_offset 32
-; PC64LE9-NEXT:    .cfi_offset lr, 16
+; PC64LE9-NEXT:    stdu 1, -64(1)
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI93_0@toc@ha
 ; PC64LE9-NEXT:    lfs 1, .LCPI93_0@toc@l(3)
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI93_1@toc@ha
 ; PC64LE9-NEXT:    lfs 2, .LCPI93_1@toc@l(3)
+; PC64LE9-NEXT:    stxv 63, 48(1) # 16-byte Folded Spill
 ; PC64LE9-NEXT:    bl fmin
 ; PC64LE9-NEXT:    nop
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI93_2@toc@ha
-; PC64LE9-NEXT:    addi 3, 3, .LCPI93_2@toc@l
-; PC64LE9-NEXT:    lxvx 0, 0, 3
+; PC64LE9-NEXT:    # kill: def $f1 killed $f1 def $vsl1
+; PC64LE9-NEXT:    stxv 1, 32(1) # 16-byte Folded Spill
+; PC64LE9-NEXT:    lfs 1, .LCPI93_2@toc@l(3)
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI93_3@toc@ha
-; PC64LE9-NEXT:    addi 3, 3, .LCPI93_3@toc@l
+; PC64LE9-NEXT:    lfs 2, .LCPI93_3@toc@l(3)
+; PC64LE9-NEXT:    bl fmin
+; PC64LE9-NEXT:    nop
+; PC64LE9-NEXT:    lxv 0, 32(1) # 16-byte Folded Reload
+; PC64LE9-NEXT:    addis 3, 2, .LCPI93_4@toc@ha
+; PC64LE9-NEXT:    # kill: def $f1 killed $f1 def $vsl1
+; PC64LE9-NEXT:    xxmrghd 63, 1, 0
+; PC64LE9-NEXT:    lfs 1, .LCPI93_4@toc@l(3)
+; PC64LE9-NEXT:    addis 3, 2, .LCPI93_5@toc@ha
+; PC64LE9-NEXT:    lfs 2, .LCPI93_5@toc@l(3)
+; PC64LE9-NEXT:    bl fmin
+; PC64LE9-NEXT:    nop
 ; PC64LE9-NEXT:    fmr 3, 1
-; PC64LE9-NEXT:    lxvx 1, 0, 3
-; PC64LE9-NEXT:    xvmindp 2, 1, 0
-; PC64LE9-NEXT:    xxswapd 1, 2
+; PC64LE9-NEXT:    xxswapd 1, 63
+; PC64LE9-NEXT:    xscpsgndp 2, 63, 63
+; PC64LE9-NEXT:    lxv 63, 48(1) # 16-byte Folded Reload
 ; PC64LE9-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
-; PC64LE9-NEXT:    # kill: def $f2 killed $f2 killed $vsl2
-; PC64LE9-NEXT:    addi 1, 1, 32
+; PC64LE9-NEXT:    addi 1, 1, 64
 ; PC64LE9-NEXT:    ld 0, 16(1)
 ; PC64LE9-NEXT:    mtlr 0
 ; PC64LE9-NEXT:    blr
@@ -6227,50 +6155,109 @@ entry:
  %min = call <3 x double> @llvm.experimental.constrained.minnum.v3f64(
                           <3 x double> <double 43.0, double 44.0, double 45.0>,
                           <3 x double> <double 40.0, double 41.0, double 42.0>,
-                          metadata !"round.dynamic",
-                          metadata !"fpexcept.strict")
+                          metadata !"fpexcept.strict") #1
   ret <3 x double> %min
 }
 
-define <4 x double> @constrained_vector_minnum_v4f64() {
+define <4 x double> @constrained_vector_minnum_v4f64() #0 {
 ; PC64LE-LABEL: constrained_vector_minnum_v4f64:
 ; PC64LE:       # %bb.0: # %entry
-; PC64LE-NEXT:    addis 3, 2, .LCPI94_0@toc@ha
+; PC64LE-NEXT:    mflr 0
+; PC64LE-NEXT:    std 0, 16(1)
+; PC64LE-NEXT:    stdu 1, -80(1)
+; PC64LE-NEXT:    li 3, 64
 ; PC64LE-NEXT:    addis 4, 2, .LCPI94_1@toc@ha
-; PC64LE-NEXT:    addis 5, 2, .LCPI94_2@toc@ha
-; PC64LE-NEXT:    addis 6, 2, .LCPI94_3@toc@ha
-; PC64LE-NEXT:    addi 3, 3, .LCPI94_0@toc@l
-; PC64LE-NEXT:    addi 4, 4, .LCPI94_1@toc@l
-; PC64LE-NEXT:    lxvd2x 0, 0, 3
-; PC64LE-NEXT:    lxvd2x 1, 0, 4
-; PC64LE-NEXT:    addi 3, 5, .LCPI94_2@toc@l
-; PC64LE-NEXT:    addi 4, 6, .LCPI94_3@toc@l
-; PC64LE-NEXT:    lxvd2x 2, 0, 3
-; PC64LE-NEXT:    lxvd2x 3, 0, 4
-; PC64LE-NEXT:    xxswapd 0, 0
-; PC64LE-NEXT:    xxswapd 1, 1
-; PC64LE-NEXT:    xxswapd 2, 2
-; PC64LE-NEXT:    xxswapd 3, 3
-; PC64LE-NEXT:    xvmindp 34, 1, 0
-; PC64LE-NEXT:    xvmindp 35, 3, 2
+; PC64LE-NEXT:    stxvd2x 63, 1, 3 # 16-byte Folded Spill
+; PC64LE-NEXT:    addis 3, 2, .LCPI94_0@toc@ha
+; PC64LE-NEXT:    lfs 2, .LCPI94_1@toc@l(4)
+; PC64LE-NEXT:    lfs 1, .LCPI94_0@toc@l(3)
+; PC64LE-NEXT:    bl fmin
+; PC64LE-NEXT:    nop
+; PC64LE-NEXT:    li 3, 48
+; PC64LE-NEXT:    # kill: def $f1 killed $f1 def $vsl1
+; PC64LE-NEXT:    addis 4, 2, .LCPI94_3@toc@ha
+; PC64LE-NEXT:    stxvd2x 1, 1, 3 # 16-byte Folded Spill
+; PC64LE-NEXT:    addis 3, 2, .LCPI94_2@toc@ha
+; PC64LE-NEXT:    lfs 2, .LCPI94_3@toc@l(4)
+; PC64LE-NEXT:    lfs 1, .LCPI94_2@toc@l(3)
+; PC64LE-NEXT:    bl fmin
+; PC64LE-NEXT:    nop
+; PC64LE-NEXT:    li 3, 48
+; PC64LE-NEXT:    addis 4, 2, .LCPI94_5@toc@ha
+; PC64LE-NEXT:    # kill: def $f1 killed $f1 def $vsl1
+; PC64LE-NEXT:    lxvd2x 0, 1, 3 # 16-byte Folded Reload
+; PC64LE-NEXT:    addis 3, 2, .LCPI94_4@toc@ha
+; PC64LE-NEXT:    lfs 2, .LCPI94_5@toc@l(4)
+; PC64LE-NEXT:    xxmrghd 63, 1, 0
+; PC64LE-NEXT:    lfs 1, .LCPI94_4@toc@l(3)
+; PC64LE-NEXT:    bl fmin
+; PC64LE-NEXT:    nop
+; PC64LE-NEXT:    li 3, 48
+; PC64LE-NEXT:    # kill: def $f1 killed $f1 def $vsl1
+; PC64LE-NEXT:    addis 4, 2, .LCPI94_7@toc@ha
+; PC64LE-NEXT:    stxvd2x 1, 1, 3 # 16-byte Folded Spill
+; PC64LE-NEXT:    addis 3, 2, .LCPI94_6@toc@ha
+; PC64LE-NEXT:    lfs 2, .LCPI94_7@toc@l(4)
+; PC64LE-NEXT:    lfs 1, .LCPI94_6@toc@l(3)
+; PC64LE-NEXT:    bl fmin
+; PC64LE-NEXT:    nop
+; PC64LE-NEXT:    li 3, 48
+; PC64LE-NEXT:    vmr 2, 31
+; PC64LE-NEXT:    # kill: def $f1 killed $f1 def $vsl1
+; PC64LE-NEXT:    lxvd2x 0, 1, 3 # 16-byte Folded Reload
+; PC64LE-NEXT:    li 3, 64
+; PC64LE-NEXT:    lxvd2x 63, 1, 3 # 16-byte Folded Reload
+; PC64LE-NEXT:    xxmrghd 35, 1, 0
+; PC64LE-NEXT:    addi 1, 1, 80
+; PC64LE-NEXT:    ld 0, 16(1)
+; PC64LE-NEXT:    mtlr 0
 ; PC64LE-NEXT:    blr
 ;
 ; PC64LE9-LABEL: constrained_vector_minnum_v4f64:
 ; PC64LE9:       # %bb.0: # %entry
+; PC64LE9-NEXT:    mflr 0
+; PC64LE9-NEXT:    std 0, 16(1)
+; PC64LE9-NEXT:    stdu 1, -64(1)
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI94_0@toc@ha
-; PC64LE9-NEXT:    addi 3, 3, .LCPI94_0@toc@l
-; PC64LE9-NEXT:    lxvx 0, 0, 3
+; PC64LE9-NEXT:    lfs 1, .LCPI94_0@toc@l(3)
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI94_1@toc@ha
-; PC64LE9-NEXT:    addi 3, 3, .LCPI94_1@toc@l
-; PC64LE9-NEXT:    lxvx 1, 0, 3
+; PC64LE9-NEXT:    lfs 2, .LCPI94_1@toc@l(3)
+; PC64LE9-NEXT:    stxv 63, 48(1) # 16-byte Folded Spill
+; PC64LE9-NEXT:    bl fmin
+; PC64LE9-NEXT:    nop
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI94_2@toc@ha
-; PC64LE9-NEXT:    addi 3, 3, .LCPI94_2@toc@l
-; PC64LE9-NEXT:    xvmindp 34, 1, 0
-; PC64LE9-NEXT:    lxvx 0, 0, 3
+; PC64LE9-NEXT:    # kill: def $f1 killed $f1 def $vsl1
+; PC64LE9-NEXT:    stxv 1, 32(1) # 16-byte Folded Spill
+; PC64LE9-NEXT:    lfs 1, .LCPI94_2@toc@l(3)
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI94_3@toc@ha
-; PC64LE9-NEXT:    addi 3, 3, .LCPI94_3@toc@l
-; PC64LE9-NEXT:    lxvx 1, 0, 3
-; PC64LE9-NEXT:    xvmindp 35, 1, 0
+; PC64LE9-NEXT:    lfs 2, .LCPI94_3@toc@l(3)
+; PC64LE9-NEXT:    bl fmin
+; PC64LE9-NEXT:    nop
+; PC64LE9-NEXT:    lxv 0, 32(1) # 16-byte Folded Reload
+; PC64LE9-NEXT:    addis 3, 2, .LCPI94_4@toc@ha
+; PC64LE9-NEXT:    # kill: def $f1 killed $f1 def $vsl1
+; PC64LE9-NEXT:    xxmrghd 63, 1, 0
+; PC64LE9-NEXT:    lfs 1, .LCPI94_4@toc@l(3)
+; PC64LE9-NEXT:    addis 3, 2, .LCPI94_5@toc@ha
+; PC64LE9-NEXT:    lfs 2, .LCPI94_5@toc@l(3)
+; PC64LE9-NEXT:    bl fmin
+; PC64LE9-NEXT:    nop
+; PC64LE9-NEXT:    addis 3, 2, .LCPI94_6@toc@ha
+; PC64LE9-NEXT:    # kill: def $f1 killed $f1 def $vsl1
+; PC64LE9-NEXT:    stxv 1, 32(1) # 16-byte Folded Spill
+; PC64LE9-NEXT:    lfs 1, .LCPI94_6@toc@l(3)
+; PC64LE9-NEXT:    addis 3, 2, .LCPI94_7@toc@ha
+; PC64LE9-NEXT:    lfs 2, .LCPI94_7@toc@l(3)
+; PC64LE9-NEXT:    bl fmin
+; PC64LE9-NEXT:    nop
+; PC64LE9-NEXT:    lxv 0, 32(1) # 16-byte Folded Reload
+; PC64LE9-NEXT:    vmr 2, 31
+; PC64LE9-NEXT:    lxv 63, 48(1) # 16-byte Folded Reload
+; PC64LE9-NEXT:    # kill: def $f1 killed $f1 def $vsl1
+; PC64LE9-NEXT:    xxmrghd 35, 1, 0
+; PC64LE9-NEXT:    addi 1, 1, 64
+; PC64LE9-NEXT:    ld 0, 16(1)
+; PC64LE9-NEXT:    mtlr 0
 ; PC64LE9-NEXT:    blr
 entry:
   %min = call <4 x double> @llvm.experimental.constrained.minnum.v4f64(
@@ -6278,12 +6265,11 @@ entry:
                                               double 46.0, double 47.0>,
                                 <4 x double> <double 40.0, double 41.0,
                                               double 42.0, double 43.0>,
-                                metadata !"round.dynamic",
-                                metadata !"fpexcept.strict")
+                                metadata !"fpexcept.strict") #1
   ret <4 x double> %min
 }
 
-define <1 x float> @constrained_vector_fptrunc_v1f64() {
+define <1 x float> @constrained_vector_fptrunc_v1f64() #0 {
 ; PC64LE-LABEL: constrained_vector_fptrunc_v1f64:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    addis 3, 2, .LCPI95_0@toc@ha
@@ -6305,11 +6291,11 @@ entry:
   %result = call <1 x float> @llvm.experimental.constrained.fptrunc.v1f32.v1f64(
                                 <1 x double><double 42.1>,
                                 metadata !"round.dynamic",
-                                metadata !"fpexcept.strict")
+                                metadata !"fpexcept.strict") #1
   ret <1 x float> %result
 }
 
-define <2 x float> @constrained_vector_fptrunc_v2f64() {
+define <2 x float> @constrained_vector_fptrunc_v2f64() #0 {
 ; PC64LE-LABEL: constrained_vector_fptrunc_v2f64:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    addis 3, 2, .LCPI96_0@toc@ha
@@ -6343,11 +6329,11 @@ entry:
   %result = call <2 x float> @llvm.experimental.constrained.fptrunc.v2f32.v2f64(
                                 <2 x double><double 42.1, double 42.2>,
                                 metadata !"round.dynamic",
-                                metadata !"fpexcept.strict")
+                                metadata !"fpexcept.strict") #1
   ret <2 x float> %result
 }
 
-define <3 x float> @constrained_vector_fptrunc_v3f64() {
+define <3 x float> @constrained_vector_fptrunc_v3f64() #0 {
 ; PC64LE-LABEL: constrained_vector_fptrunc_v3f64:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    addis 3, 2, .LCPI97_0@toc@ha
@@ -6400,11 +6386,11 @@ entry:
                                 <3 x double><double 42.1, double 42.2,
                                              double 42.3>,
                                 metadata !"round.dynamic",
-                                metadata !"fpexcept.strict")
+                                metadata !"fpexcept.strict") #1
   ret <3 x float> %result
 }
 
-define <4 x float> @constrained_vector_fptrunc_v4f64() {
+define <4 x float> @constrained_vector_fptrunc_v4f64() #0 {
 ; PC64LE-LABEL: constrained_vector_fptrunc_v4f64:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    addis 3, 2, .LCPI98_0@toc@ha
@@ -6443,11 +6429,11 @@ entry:
                                 <4 x double><double 42.1, double 42.2,
                                              double 42.3, double 42.4>,
                                 metadata !"round.dynamic",
-                                metadata !"fpexcept.strict")
+                                metadata !"fpexcept.strict") #1
   ret <4 x float> %result
 }
 
-define <1 x double> @constrained_vector_fpext_v1f32() {
+define <1 x double> @constrained_vector_fpext_v1f32() #0 {
 ; PC64LE-LABEL: constrained_vector_fpext_v1f32:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    addis 3, 2, .LCPI99_0@toc@ha
@@ -6464,11 +6450,11 @@ define <1 x double> @constrained_vector_fpext_v1f32() {
 entry:
   %result = call <1 x double> @llvm.experimental.constrained.fpext.v1f64.v1f32(
                                 <1 x float><float 42.0>,
-                                metadata !"fpexcept.strict")
+                                metadata !"fpexcept.strict") #1
   ret <1 x double> %result
 }
 
-define <2 x double> @constrained_vector_fpext_v2f32() {
+define <2 x double> @constrained_vector_fpext_v2f32() #0 {
 ; PC64LE-LABEL: constrained_vector_fpext_v2f32:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    addis 3, 2, .LCPI100_0@toc@ha
@@ -6489,11 +6475,11 @@ define <2 x double> @constrained_vector_fpext_v2f32() {
 entry:
   %result = call <2 x double> @llvm.experimental.constrained.fpext.v2f64.v2f32(
                                 <2 x float><float 42.0, float 43.0>,
-                                metadata !"fpexcept.strict")
+                                metadata !"fpexcept.strict") #1
   ret <2 x double> %result
 }
 
-define <3 x double> @constrained_vector_fpext_v3f32() {
+define <3 x double> @constrained_vector_fpext_v3f32() #0 {
 ; PC64LE-LABEL: constrained_vector_fpext_v3f32:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    addis 3, 2, .LCPI101_0@toc@ha
@@ -6517,11 +6503,11 @@ entry:
   %result = call <3 x double> @llvm.experimental.constrained.fpext.v3f64.v3f32(
                                 <3 x float><float 42.0, float 43.0,
                                             float 44.0>,
-                                metadata !"fpexcept.strict")
+                                metadata !"fpexcept.strict") #1
   ret <3 x double> %result
 }
 
-define <4 x double> @constrained_vector_fpext_v4f32() {
+define <4 x double> @constrained_vector_fpext_v4f32() #0 {
 ; PC64LE-LABEL: constrained_vector_fpext_v4f32:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    addis 3, 2, .LCPI102_0@toc@ha
@@ -6553,16 +6539,16 @@ entry:
   %result = call <4 x double> @llvm.experimental.constrained.fpext.v4f64.v4f32(
                                 <4 x float><float 42.0, float 43.0,
                                             float 44.0, float 45.0>,
-                                metadata !"fpexcept.strict")
+                                metadata !"fpexcept.strict") #1
   ret <4 x double> %result
 }
 
-define <1 x float> @constrained_vector_ceil_v1f32() {
+define <1 x float> @constrained_vector_ceil_v1f32() #0 {
 ; PC64LE-LABEL: constrained_vector_ceil_v1f32:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    addis 3, 2, .LCPI103_0@toc@ha
 ; PC64LE-NEXT:    lfs 0, .LCPI103_0@toc@l(3)
-; PC64LE-NEXT:    frip 0, 0
+; PC64LE-NEXT:    xsrdpip 0, 0
 ; PC64LE-NEXT:    xscvdpspn 0, 0
 ; PC64LE-NEXT:    xxsldwi 34, 0, 0, 1
 ; PC64LE-NEXT:    blr
@@ -6571,19 +6557,18 @@ define <1 x float> @constrained_vector_ceil_v1f32() {
 ; PC64LE9:       # %bb.0: # %entry
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI103_0@toc@ha
 ; PC64LE9-NEXT:    lfs 0, .LCPI103_0@toc@l(3)
-; PC64LE9-NEXT:    frip 0, 0
+; PC64LE9-NEXT:    xsrdpip 0, 0
 ; PC64LE9-NEXT:    xscvdpspn 0, 0
 ; PC64LE9-NEXT:    xxsldwi 34, 0, 0, 1
 ; PC64LE9-NEXT:    blr
 entry:
   %ceil = call <1 x float> @llvm.experimental.constrained.ceil.v1f32(
                                <1 x float> <float 1.5>,
-                               metadata !"round.dynamic",
-                               metadata !"fpexcept.strict")
+                               metadata !"fpexcept.strict") #1
   ret <1 x float> %ceil
 }
 
-define <2 x double> @constrained_vector_ceil_v2f64() {
+define <2 x double> @constrained_vector_ceil_v2f64() #0 {
 ; PC64LE-LABEL: constrained_vector_ceil_v2f64:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    addis 3, 2, .LCPI104_0@toc@ha
@@ -6603,12 +6588,11 @@ define <2 x double> @constrained_vector_ceil_v2f64() {
 entry:
   %ceil = call <2 x double> @llvm.experimental.constrained.ceil.v2f64(
                                 <2 x double> <double 1.1, double 1.9>,
-                                metadata !"round.dynamic",
-                                metadata !"fpexcept.strict")
+                                metadata !"fpexcept.strict") #1
   ret <2 x double> %ceil
 }
 
-define <3 x float> @constrained_vector_ceil_v3f32() {
+define <3 x float> @constrained_vector_ceil_v3f32() #0 {
 ; PC64LE-LABEL: constrained_vector_ceil_v3f32:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    addis 3, 2, .LCPI105_2@toc@ha
@@ -6616,12 +6600,12 @@ define <3 x float> @constrained_vector_ceil_v3f32() {
 ; PC64LE-NEXT:    lfs 0, .LCPI105_2@toc@l(3)
 ; PC64LE-NEXT:    lfs 1, .LCPI105_1@toc@l(4)
 ; PC64LE-NEXT:    addis 3, 2, .LCPI105_0@toc@ha
-; PC64LE-NEXT:    frip 0, 0
+; PC64LE-NEXT:    xsrdpip 0, 0
 ; PC64LE-NEXT:    lfs 2, .LCPI105_0@toc@l(3)
 ; PC64LE-NEXT:    addis 3, 2, .LCPI105_3@toc@ha
-; PC64LE-NEXT:    frip 1, 1
+; PC64LE-NEXT:    xsrdpip 1, 1
 ; PC64LE-NEXT:    addi 3, 3, .LCPI105_3@toc@l
-; PC64LE-NEXT:    frip 2, 2
+; PC64LE-NEXT:    xsrdpip 2, 2
 ; PC64LE-NEXT:    xscvdpspn 0, 0
 ; PC64LE-NEXT:    xscvdpspn 1, 1
 ; PC64LE-NEXT:    xxsldwi 34, 0, 0, 1
@@ -6640,12 +6624,12 @@ define <3 x float> @constrained_vector_ceil_v3f32() {
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI105_1@toc@ha
 ; PC64LE9-NEXT:    lfs 1, .LCPI105_1@toc@l(3)
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI105_2@toc@ha
-; PC64LE9-NEXT:    frip 0, 0
+; PC64LE9-NEXT:    xsrdpip 0, 0
 ; PC64LE9-NEXT:    lfs 2, .LCPI105_2@toc@l(3)
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI105_3@toc@ha
 ; PC64LE9-NEXT:    addi 3, 3, .LCPI105_3@toc@l
-; PC64LE9-NEXT:    frip 1, 1
-; PC64LE9-NEXT:    frip 2, 2
+; PC64LE9-NEXT:    xsrdpip 1, 1
+; PC64LE9-NEXT:    xsrdpip 2, 2
 ; PC64LE9-NEXT:    xscvdpspn 0, 0
 ; PC64LE9-NEXT:    xscvdpspn 1, 1
 ; PC64LE9-NEXT:    xscvdpspn 2, 2
@@ -6659,12 +6643,11 @@ define <3 x float> @constrained_vector_ceil_v3f32() {
 entry:
   %ceil = call <3 x float> @llvm.experimental.constrained.ceil.v3f32(
                               <3 x float> <float 1.5, float 2.5, float 3.5>,
-                              metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #1
   ret <3 x float> %ceil
 }
 
-define <3 x double> @constrained_vector_ceil_v3f64() {
+define <3 x double> @constrained_vector_ceil_v3f64() #0 {
 ; PC64LE-LABEL: constrained_vector_ceil_v3f64:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    addis 3, 2, .LCPI106_1@toc@ha
@@ -6696,17 +6679,16 @@ define <3 x double> @constrained_vector_ceil_v3f64() {
 entry:
   %ceil = call <3 x double> @llvm.experimental.constrained.ceil.v3f64(
                           <3 x double> <double 1.1, double 1.9, double 1.5>,
-                          metadata !"round.dynamic",
-                          metadata !"fpexcept.strict")
+                          metadata !"fpexcept.strict") #1
   ret <3 x double> %ceil
 }
 
-define <1 x float> @constrained_vector_floor_v1f32() {
+define <1 x float> @constrained_vector_floor_v1f32() #0 {
 ; PC64LE-LABEL: constrained_vector_floor_v1f32:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    addis 3, 2, .LCPI107_0@toc@ha
 ; PC64LE-NEXT:    lfs 0, .LCPI107_0@toc@l(3)
-; PC64LE-NEXT:    frim 0, 0
+; PC64LE-NEXT:    xsrdpim 0, 0
 ; PC64LE-NEXT:    xscvdpspn 0, 0
 ; PC64LE-NEXT:    xxsldwi 34, 0, 0, 1
 ; PC64LE-NEXT:    blr
@@ -6715,20 +6697,19 @@ define <1 x float> @constrained_vector_floor_v1f32() {
 ; PC64LE9:       # %bb.0: # %entry
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI107_0@toc@ha
 ; PC64LE9-NEXT:    lfs 0, .LCPI107_0@toc@l(3)
-; PC64LE9-NEXT:    frim 0, 0
+; PC64LE9-NEXT:    xsrdpim 0, 0
 ; PC64LE9-NEXT:    xscvdpspn 0, 0
 ; PC64LE9-NEXT:    xxsldwi 34, 0, 0, 1
 ; PC64LE9-NEXT:    blr
 entry:
   %floor = call <1 x float> @llvm.experimental.constrained.floor.v1f32(
                                <1 x float> <float 1.5>,
-                               metadata !"round.dynamic",
-                               metadata !"fpexcept.strict")
+                               metadata !"fpexcept.strict") #1
   ret <1 x float> %floor
 }
 
 
-define <2 x double> @constrained_vector_floor_v2f64() {
+define <2 x double> @constrained_vector_floor_v2f64() #0 {
 ; PC64LE-LABEL: constrained_vector_floor_v2f64:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    addis 3, 2, .LCPI108_0@toc@ha
@@ -6748,12 +6729,11 @@ define <2 x double> @constrained_vector_floor_v2f64() {
 entry:
   %floor = call <2 x double> @llvm.experimental.constrained.floor.v2f64(
                                 <2 x double> <double 1.1, double 1.9>,
-                                metadata !"round.dynamic",
-                                metadata !"fpexcept.strict")
+                                metadata !"fpexcept.strict") #1
   ret <2 x double> %floor
 }
 
-define <3 x float> @constrained_vector_floor_v3f32() {
+define <3 x float> @constrained_vector_floor_v3f32() #0 {
 ; PC64LE-LABEL: constrained_vector_floor_v3f32:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    addis 3, 2, .LCPI109_2@toc@ha
@@ -6761,12 +6741,12 @@ define <3 x float> @constrained_vector_floor_v3f32() {
 ; PC64LE-NEXT:    lfs 0, .LCPI109_2@toc@l(3)
 ; PC64LE-NEXT:    lfs 1, .LCPI109_1@toc@l(4)
 ; PC64LE-NEXT:    addis 3, 2, .LCPI109_0@toc@ha
-; PC64LE-NEXT:    frim 0, 0
+; PC64LE-NEXT:    xsrdpim 0, 0
 ; PC64LE-NEXT:    lfs 2, .LCPI109_0@toc@l(3)
 ; PC64LE-NEXT:    addis 3, 2, .LCPI109_3@toc@ha
-; PC64LE-NEXT:    frim 1, 1
+; PC64LE-NEXT:    xsrdpim 1, 1
 ; PC64LE-NEXT:    addi 3, 3, .LCPI109_3@toc@l
-; PC64LE-NEXT:    frim 2, 2
+; PC64LE-NEXT:    xsrdpim 2, 2
 ; PC64LE-NEXT:    xscvdpspn 0, 0
 ; PC64LE-NEXT:    xscvdpspn 1, 1
 ; PC64LE-NEXT:    xxsldwi 34, 0, 0, 1
@@ -6785,12 +6765,12 @@ define <3 x float> @constrained_vector_floor_v3f32() {
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI109_1@toc@ha
 ; PC64LE9-NEXT:    lfs 1, .LCPI109_1@toc@l(3)
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI109_2@toc@ha
-; PC64LE9-NEXT:    frim 0, 0
+; PC64LE9-NEXT:    xsrdpim 0, 0
 ; PC64LE9-NEXT:    lfs 2, .LCPI109_2@toc@l(3)
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI109_3@toc@ha
 ; PC64LE9-NEXT:    addi 3, 3, .LCPI109_3@toc@l
-; PC64LE9-NEXT:    frim 1, 1
-; PC64LE9-NEXT:    frim 2, 2
+; PC64LE9-NEXT:    xsrdpim 1, 1
+; PC64LE9-NEXT:    xsrdpim 2, 2
 ; PC64LE9-NEXT:    xscvdpspn 0, 0
 ; PC64LE9-NEXT:    xscvdpspn 1, 1
 ; PC64LE9-NEXT:    xscvdpspn 2, 2
@@ -6804,12 +6784,11 @@ define <3 x float> @constrained_vector_floor_v3f32() {
 entry:
   %floor = call <3 x float> @llvm.experimental.constrained.floor.v3f32(
                               <3 x float> <float 1.5, float 2.5, float 3.5>,
-                              metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #1
   ret <3 x float> %floor
 }
 
-define <3 x double> @constrained_vector_floor_v3f64() {
+define <3 x double> @constrained_vector_floor_v3f64() #0 {
 ; PC64LE-LABEL: constrained_vector_floor_v3f64:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    addis 3, 2, .LCPI110_1@toc@ha
@@ -6841,17 +6820,16 @@ define <3 x double> @constrained_vector_floor_v3f64() {
 entry:
   %floor = call <3 x double> @llvm.experimental.constrained.floor.v3f64(
                           <3 x double> <double 1.1, double 1.9, double 1.5>,
-                          metadata !"round.dynamic",
-                          metadata !"fpexcept.strict")
+                          metadata !"fpexcept.strict") #1
   ret <3 x double> %floor
 }
 
-define <1 x float> @constrained_vector_round_v1f32() {
+define <1 x float> @constrained_vector_round_v1f32() #0 {
 ; PC64LE-LABEL: constrained_vector_round_v1f32:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    addis 3, 2, .LCPI111_0@toc@ha
 ; PC64LE-NEXT:    lfs 0, .LCPI111_0@toc@l(3)
-; PC64LE-NEXT:    frin 0, 0
+; PC64LE-NEXT:    xsrdpi 0, 0
 ; PC64LE-NEXT:    xscvdpspn 0, 0
 ; PC64LE-NEXT:    xxsldwi 34, 0, 0, 1
 ; PC64LE-NEXT:    blr
@@ -6860,19 +6838,18 @@ define <1 x float> @constrained_vector_round_v1f32() {
 ; PC64LE9:       # %bb.0: # %entry
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI111_0@toc@ha
 ; PC64LE9-NEXT:    lfs 0, .LCPI111_0@toc@l(3)
-; PC64LE9-NEXT:    frin 0, 0
+; PC64LE9-NEXT:    xsrdpi 0, 0
 ; PC64LE9-NEXT:    xscvdpspn 0, 0
 ; PC64LE9-NEXT:    xxsldwi 34, 0, 0, 1
 ; PC64LE9-NEXT:    blr
 entry:
   %round = call <1 x float> @llvm.experimental.constrained.round.v1f32(
                                <1 x float> <float 1.5>,
-                               metadata !"round.dynamic",
-                               metadata !"fpexcept.strict")
+                               metadata !"fpexcept.strict") #1
   ret <1 x float> %round
 }
 
-define <2 x double> @constrained_vector_round_v2f64() {
+define <2 x double> @constrained_vector_round_v2f64() #0 {
 ; PC64LE-LABEL: constrained_vector_round_v2f64:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    addis 3, 2, .LCPI112_0@toc@ha
@@ -6892,12 +6869,11 @@ define <2 x double> @constrained_vector_round_v2f64() {
 entry:
   %round = call <2 x double> @llvm.experimental.constrained.round.v2f64(
                                 <2 x double> <double 1.1, double 1.9>,
-                                metadata !"round.dynamic",
-                                metadata !"fpexcept.strict")
+                                metadata !"fpexcept.strict") #1
   ret <2 x double> %round
 }
 
-define <3 x float> @constrained_vector_round_v3f32() {
+define <3 x float> @constrained_vector_round_v3f32() #0 {
 ; PC64LE-LABEL: constrained_vector_round_v3f32:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    addis 3, 2, .LCPI113_2@toc@ha
@@ -6905,12 +6881,12 @@ define <3 x float> @constrained_vector_round_v3f32() {
 ; PC64LE-NEXT:    lfs 0, .LCPI113_2@toc@l(3)
 ; PC64LE-NEXT:    lfs 1, .LCPI113_1@toc@l(4)
 ; PC64LE-NEXT:    addis 3, 2, .LCPI113_0@toc@ha
-; PC64LE-NEXT:    frin 0, 0
+; PC64LE-NEXT:    xsrdpi 0, 0
 ; PC64LE-NEXT:    lfs 2, .LCPI113_0@toc@l(3)
 ; PC64LE-NEXT:    addis 3, 2, .LCPI113_3@toc@ha
-; PC64LE-NEXT:    frin 1, 1
+; PC64LE-NEXT:    xsrdpi 1, 1
 ; PC64LE-NEXT:    addi 3, 3, .LCPI113_3@toc@l
-; PC64LE-NEXT:    frin 2, 2
+; PC64LE-NEXT:    xsrdpi 2, 2
 ; PC64LE-NEXT:    xscvdpspn 0, 0
 ; PC64LE-NEXT:    xscvdpspn 1, 1
 ; PC64LE-NEXT:    xxsldwi 34, 0, 0, 1
@@ -6929,12 +6905,12 @@ define <3 x float> @constrained_vector_round_v3f32() {
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI113_1@toc@ha
 ; PC64LE9-NEXT:    lfs 1, .LCPI113_1@toc@l(3)
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI113_2@toc@ha
-; PC64LE9-NEXT:    frin 0, 0
+; PC64LE9-NEXT:    xsrdpi 0, 0
 ; PC64LE9-NEXT:    lfs 2, .LCPI113_2@toc@l(3)
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI113_3@toc@ha
 ; PC64LE9-NEXT:    addi 3, 3, .LCPI113_3@toc@l
-; PC64LE9-NEXT:    frin 1, 1
-; PC64LE9-NEXT:    frin 2, 2
+; PC64LE9-NEXT:    xsrdpi 1, 1
+; PC64LE9-NEXT:    xsrdpi 2, 2
 ; PC64LE9-NEXT:    xscvdpspn 0, 0
 ; PC64LE9-NEXT:    xscvdpspn 1, 1
 ; PC64LE9-NEXT:    xscvdpspn 2, 2
@@ -6948,13 +6924,12 @@ define <3 x float> @constrained_vector_round_v3f32() {
 entry:
   %round = call <3 x float> @llvm.experimental.constrained.round.v3f32(
                               <3 x float> <float 1.5, float 2.5, float 3.5>,
-                              metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #1
   ret <3 x float> %round
 }
 
 
-define <3 x double> @constrained_vector_round_v3f64() {
+define <3 x double> @constrained_vector_round_v3f64() #0 {
 ; PC64LE-LABEL: constrained_vector_round_v3f64:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    addis 3, 2, .LCPI114_1@toc@ha
@@ -6986,17 +6961,16 @@ define <3 x double> @constrained_vector_round_v3f64() {
 entry:
   %round = call <3 x double> @llvm.experimental.constrained.round.v3f64(
                           <3 x double> <double 1.1, double 1.9, double 1.5>,
-                          metadata !"round.dynamic",
-                          metadata !"fpexcept.strict")
+                          metadata !"fpexcept.strict") #1
   ret <3 x double> %round
 }
 
-define <1 x float> @constrained_vector_trunc_v1f32() {
+define <1 x float> @constrained_vector_trunc_v1f32() #0 {
 ; PC64LE-LABEL: constrained_vector_trunc_v1f32:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    addis 3, 2, .LCPI115_0@toc@ha
 ; PC64LE-NEXT:    lfs 0, .LCPI115_0@toc@l(3)
-; PC64LE-NEXT:    friz 0, 0
+; PC64LE-NEXT:    xsrdpiz 0, 0
 ; PC64LE-NEXT:    xscvdpspn 0, 0
 ; PC64LE-NEXT:    xxsldwi 34, 0, 0, 1
 ; PC64LE-NEXT:    blr
@@ -7005,19 +6979,18 @@ define <1 x float> @constrained_vector_trunc_v1f32() {
 ; PC64LE9:       # %bb.0: # %entry
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI115_0@toc@ha
 ; PC64LE9-NEXT:    lfs 0, .LCPI115_0@toc@l(3)
-; PC64LE9-NEXT:    friz 0, 0
+; PC64LE9-NEXT:    xsrdpiz 0, 0
 ; PC64LE9-NEXT:    xscvdpspn 0, 0
 ; PC64LE9-NEXT:    xxsldwi 34, 0, 0, 1
 ; PC64LE9-NEXT:    blr
 entry:
   %trunc = call <1 x float> @llvm.experimental.constrained.trunc.v1f32(
                                <1 x float> <float 1.5>,
-                               metadata !"round.dynamic",
-                               metadata !"fpexcept.strict")
+                               metadata !"fpexcept.strict") #1
   ret <1 x float> %trunc
 }
 
-define <2 x double> @constrained_vector_trunc_v2f64() {
+define <2 x double> @constrained_vector_trunc_v2f64() #0 {
 ; PC64LE-LABEL: constrained_vector_trunc_v2f64:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    addis 3, 2, .LCPI116_0@toc@ha
@@ -7037,12 +7010,11 @@ define <2 x double> @constrained_vector_trunc_v2f64() {
 entry:
   %trunc = call <2 x double> @llvm.experimental.constrained.trunc.v2f64(
                                 <2 x double> <double 1.1, double 1.9>,
-                                metadata !"round.dynamic",
-                                metadata !"fpexcept.strict")
+                                metadata !"fpexcept.strict") #1
   ret <2 x double> %trunc
 }
 
-define <3 x float> @constrained_vector_trunc_v3f32() {
+define <3 x float> @constrained_vector_trunc_v3f32() #0 {
 ; PC64LE-LABEL: constrained_vector_trunc_v3f32:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    addis 3, 2, .LCPI117_2@toc@ha
@@ -7050,12 +7022,12 @@ define <3 x float> @constrained_vector_trunc_v3f32() {
 ; PC64LE-NEXT:    lfs 0, .LCPI117_2@toc@l(3)
 ; PC64LE-NEXT:    lfs 1, .LCPI117_1@toc@l(4)
 ; PC64LE-NEXT:    addis 3, 2, .LCPI117_0@toc@ha
-; PC64LE-NEXT:    friz 0, 0
+; PC64LE-NEXT:    xsrdpiz 0, 0
 ; PC64LE-NEXT:    lfs 2, .LCPI117_0@toc@l(3)
 ; PC64LE-NEXT:    addis 3, 2, .LCPI117_3@toc@ha
-; PC64LE-NEXT:    friz 1, 1
+; PC64LE-NEXT:    xsrdpiz 1, 1
 ; PC64LE-NEXT:    addi 3, 3, .LCPI117_3@toc@l
-; PC64LE-NEXT:    friz 2, 2
+; PC64LE-NEXT:    xsrdpiz 2, 2
 ; PC64LE-NEXT:    xscvdpspn 0, 0
 ; PC64LE-NEXT:    xscvdpspn 1, 1
 ; PC64LE-NEXT:    xxsldwi 34, 0, 0, 1
@@ -7074,12 +7046,12 @@ define <3 x float> @constrained_vector_trunc_v3f32() {
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI117_1@toc@ha
 ; PC64LE9-NEXT:    lfs 1, .LCPI117_1@toc@l(3)
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI117_2@toc@ha
-; PC64LE9-NEXT:    friz 0, 0
+; PC64LE9-NEXT:    xsrdpiz 0, 0
 ; PC64LE9-NEXT:    lfs 2, .LCPI117_2@toc@l(3)
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI117_3@toc@ha
 ; PC64LE9-NEXT:    addi 3, 3, .LCPI117_3@toc@l
-; PC64LE9-NEXT:    friz 1, 1
-; PC64LE9-NEXT:    friz 2, 2
+; PC64LE9-NEXT:    xsrdpiz 1, 1
+; PC64LE9-NEXT:    xsrdpiz 2, 2
 ; PC64LE9-NEXT:    xscvdpspn 0, 0
 ; PC64LE9-NEXT:    xscvdpspn 1, 1
 ; PC64LE9-NEXT:    xscvdpspn 2, 2
@@ -7093,12 +7065,11 @@ define <3 x float> @constrained_vector_trunc_v3f32() {
 entry:
   %trunc = call <3 x float> @llvm.experimental.constrained.trunc.v3f32(
                               <3 x float> <float 1.5, float 2.5, float 3.5>,
-                              metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #1
   ret <3 x float> %trunc
 }
 
-define <3 x double> @constrained_vector_trunc_v3f64() {
+define <3 x double> @constrained_vector_trunc_v3f64() #0 {
 ; PC64LE-LABEL: constrained_vector_trunc_v3f64:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    addis 3, 2, .LCPI118_1@toc@ha
@@ -7130,11 +7101,12 @@ define <3 x double> @constrained_vector_trunc_v3f64() {
 entry:
   %trunc = call <3 x double> @llvm.experimental.constrained.trunc.v3f64(
                           <3 x double> <double 1.1, double 1.9, double 1.5>,
-                          metadata !"round.dynamic",
-                          metadata !"fpexcept.strict")
+                          metadata !"fpexcept.strict") #1
   ret <3 x double> %trunc
 }
 
+attributes #0 = { nounwind strictfp noimplicitfloat }
+attributes #1 = { strictfp }
 
 ; Single width declarations
 declare <2 x double> @llvm.experimental.constrained.fadd.v2f64(<2 x double>, <2 x double>, metadata, metadata)
@@ -7154,14 +7126,14 @@ declare <2 x double> @llvm.experimental.constrained.log10.v2f64(<2 x double>, me
 declare <2 x double> @llvm.experimental.constrained.log2.v2f64(<2 x double>, metadata, metadata)
 declare <2 x double> @llvm.experimental.constrained.rint.v2f64(<2 x double>, metadata, metadata)
 declare <2 x double> @llvm.experimental.constrained.nearbyint.v2f64(<2 x double>, metadata, metadata)
-declare <2 x double> @llvm.experimental.constrained.maxnum.v2f64(<2 x double>, <2 x double>, metadata, metadata)
-declare <2 x double> @llvm.experimental.constrained.minnum.v2f64(<2 x double>, <2 x double>, metadata, metadata)
+declare <2 x double> @llvm.experimental.constrained.maxnum.v2f64(<2 x double>, <2 x double>, metadata)
+declare <2 x double> @llvm.experimental.constrained.minnum.v2f64(<2 x double>, <2 x double>, metadata)
 declare <2 x float> @llvm.experimental.constrained.fptrunc.v2f32.v2f64(<2 x double>, metadata, metadata)
 declare <2 x double> @llvm.experimental.constrained.fpext.v2f64.v2f32(<2 x float>, metadata)
-declare <2 x double> @llvm.experimental.constrained.ceil.v2f64(<2 x double>, metadata, metadata)
-declare <2 x double> @llvm.experimental.constrained.floor.v2f64(<2 x double>, metadata, metadata)
-declare <2 x double> @llvm.experimental.constrained.round.v2f64(<2 x double>, metadata, metadata)
-declare <2 x double> @llvm.experimental.constrained.trunc.v2f64(<2 x double>, metadata, metadata)
+declare <2 x double> @llvm.experimental.constrained.ceil.v2f64(<2 x double>, metadata)
+declare <2 x double> @llvm.experimental.constrained.floor.v2f64(<2 x double>, metadata)
+declare <2 x double> @llvm.experimental.constrained.round.v2f64(<2 x double>, metadata)
+declare <2 x double> @llvm.experimental.constrained.trunc.v2f64(<2 x double>, metadata)
 
 ; Scalar width declarations
 declare <1 x float> @llvm.experimental.constrained.fadd.v1f32(<1 x float>, <1 x float>, metadata, metadata)
@@ -7181,14 +7153,14 @@ declare <1 x float> @llvm.experimental.constrained.log10.v1f32(<1 x float>, meta
 declare <1 x float> @llvm.experimental.constrained.log2.v1f32(<1 x float>, metadata, metadata)
 declare <1 x float> @llvm.experimental.constrained.rint.v1f32(<1 x float>, metadata, metadata)
 declare <1 x float> @llvm.experimental.constrained.nearbyint.v1f32(<1 x float>, metadata, metadata)
-declare <1 x float> @llvm.experimental.constrained.maxnum.v1f32(<1 x float>, <1 x float>, metadata, metadata)
-declare <1 x float> @llvm.experimental.constrained.minnum.v1f32(<1 x float>, <1 x float>, metadata, metadata)
+declare <1 x float> @llvm.experimental.constrained.maxnum.v1f32(<1 x float>, <1 x float>, metadata)
+declare <1 x float> @llvm.experimental.constrained.minnum.v1f32(<1 x float>, <1 x float>, metadata)
 declare <1 x float> @llvm.experimental.constrained.fptrunc.v1f32.v1f64(<1 x double>, metadata, metadata)
 declare <1 x double> @llvm.experimental.constrained.fpext.v1f64.v1f32(<1 x float>, metadata)
-declare <1 x float> @llvm.experimental.constrained.ceil.v1f32(<1 x float>, metadata, metadata)
-declare <1 x float> @llvm.experimental.constrained.floor.v1f32(<1 x float>, metadata, metadata)
-declare <1 x float> @llvm.experimental.constrained.round.v1f32(<1 x float>, metadata, metadata)
-declare <1 x float> @llvm.experimental.constrained.trunc.v1f32(<1 x float>, metadata, metadata)
+declare <1 x float> @llvm.experimental.constrained.ceil.v1f32(<1 x float>, metadata)
+declare <1 x float> @llvm.experimental.constrained.floor.v1f32(<1 x float>, metadata)
+declare <1 x float> @llvm.experimental.constrained.round.v1f32(<1 x float>, metadata)
+declare <1 x float> @llvm.experimental.constrained.trunc.v1f32(<1 x float>, metadata)
 
 ; Illegal width declarations
 declare <3 x float> @llvm.experimental.constrained.fadd.v3f32(<3 x float>, <3 x float>, metadata, metadata)
@@ -7225,20 +7197,20 @@ declare <3 x float> @llvm.experimental.constrained.rint.v3f32(<3 x float>, metad
 declare <3 x double> @llvm.experimental.constrained.rint.v3f64(<3 x double>, metadata, metadata)
 declare <3 x float> @llvm.experimental.constrained.nearbyint.v3f32(<3 x float>, metadata, metadata)
 declare <3 x double> @llvm.experimental.constrained.nearbyint.v3f64(<3 x double>, metadata, metadata)
-declare <3 x float> @llvm.experimental.constrained.maxnum.v3f32(<3 x float>, <3 x float>, metadata, metadata)
-declare <3 x double> @llvm.experimental.constrained.maxnum.v3f64(<3 x double>, <3 x double>, metadata, metadata)
-declare <3 x float> @llvm.experimental.constrained.minnum.v3f32(<3 x float>, <3 x float>, metadata, metadata)
-declare <3 x double> @llvm.experimental.constrained.minnum.v3f64(<3 x double>, <3 x double>, metadata, metadata)
+declare <3 x float> @llvm.experimental.constrained.maxnum.v3f32(<3 x float>, <3 x float>, metadata)
+declare <3 x double> @llvm.experimental.constrained.maxnum.v3f64(<3 x double>, <3 x double>, metadata)
+declare <3 x float> @llvm.experimental.constrained.minnum.v3f32(<3 x float>, <3 x float>, metadata)
+declare <3 x double> @llvm.experimental.constrained.minnum.v3f64(<3 x double>, <3 x double>, metadata)
 declare <3 x float> @llvm.experimental.constrained.fptrunc.v3f32.v3f64(<3 x double>, metadata, metadata)
 declare <3 x double> @llvm.experimental.constrained.fpext.v3f64.v3f32(<3 x float>, metadata)
-declare <3 x float> @llvm.experimental.constrained.ceil.v3f32(<3 x float>, metadata, metadata)
-declare <3 x double> @llvm.experimental.constrained.ceil.v3f64(<3 x double>, metadata, metadata)
-declare <3 x float> @llvm.experimental.constrained.floor.v3f32(<3 x float>, metadata, metadata)
-declare <3 x double> @llvm.experimental.constrained.floor.v3f64(<3 x double>, metadata, metadata)
-declare <3 x float> @llvm.experimental.constrained.round.v3f32(<3 x float>, metadata, metadata)
-declare <3 x double> @llvm.experimental.constrained.round.v3f64(<3 x double>, metadata, metadata)
-declare <3 x float> @llvm.experimental.constrained.trunc.v3f32(<3 x float>, metadata, metadata)
-declare <3 x double> @llvm.experimental.constrained.trunc.v3f64(<3 x double>, metadata, metadata)
+declare <3 x float> @llvm.experimental.constrained.ceil.v3f32(<3 x float>, metadata)
+declare <3 x double> @llvm.experimental.constrained.ceil.v3f64(<3 x double>, metadata)
+declare <3 x float> @llvm.experimental.constrained.floor.v3f32(<3 x float>, metadata)
+declare <3 x double> @llvm.experimental.constrained.floor.v3f64(<3 x double>, metadata)
+declare <3 x float> @llvm.experimental.constrained.round.v3f32(<3 x float>, metadata)
+declare <3 x double> @llvm.experimental.constrained.round.v3f64(<3 x double>, metadata)
+declare <3 x float> @llvm.experimental.constrained.trunc.v3f32(<3 x float>, metadata)
+declare <3 x double> @llvm.experimental.constrained.trunc.v3f64(<3 x double>, metadata)
 
 ; Double width declarations
 declare <4 x double> @llvm.experimental.constrained.fadd.v4f64(<4 x double>, <4 x double>, metadata, metadata)
@@ -7258,11 +7230,11 @@ declare <4 x double> @llvm.experimental.constrained.log10.v4f64(<4 x double>, me
 declare <4 x double> @llvm.experimental.constrained.log2.v4f64(<4 x double>, metadata, metadata)
 declare <4 x double> @llvm.experimental.constrained.rint.v4f64(<4 x double>, metadata, metadata)
 declare <4 x double> @llvm.experimental.constrained.nearbyint.v4f64(<4 x double>, metadata, metadata)
-declare <4 x double> @llvm.experimental.constrained.maxnum.v4f64(<4 x double>, <4 x double>, metadata, metadata)
-declare <4 x double> @llvm.experimental.constrained.minnum.v4f64(<4 x double>, <4 x double>, metadata, metadata)
+declare <4 x double> @llvm.experimental.constrained.maxnum.v4f64(<4 x double>, <4 x double>, metadata)
+declare <4 x double> @llvm.experimental.constrained.minnum.v4f64(<4 x double>, <4 x double>, metadata)
 declare <4 x float> @llvm.experimental.constrained.fptrunc.v4f32.v4f64(<4 x double>, metadata, metadata)
 declare <4 x double> @llvm.experimental.constrained.fpext.v4f64.v4f32(<4 x float>, metadata)
-declare <4 x double> @llvm.experimental.constrained.ceil.v4f64(<4 x double>, metadata, metadata)
-declare <4 x double> @llvm.experimental.constrained.floor.v4f64(<4 x double>, metadata, metadata)
-declare <4 x double> @llvm.experimental.constrained.round.v4f64(<4 x double>, metadata, metadata)
-declare <4 x double> @llvm.experimental.constrained.trunc.v4f64(<4 x double>, metadata, metadata)
+declare <4 x double> @llvm.experimental.constrained.ceil.v4f64(<4 x double>, metadata)
+declare <4 x double> @llvm.experimental.constrained.floor.v4f64(<4 x double>, metadata)
+declare <4 x double> @llvm.experimental.constrained.round.v4f64(<4 x double>, metadata)
+declare <4 x double> @llvm.experimental.constrained.trunc.v4f64(<4 x double>, metadata)
diff --git a/llvm/test/CodeGen/PowerPC/vector-extend-sign.ll b/llvm/test/CodeGen/PowerPC/vector-extend-sign.ll
new file mode 100644
index 000000000..71928296d
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/vector-extend-sign.ll
@@ -0,0 +1,178 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu \
+; RUN:     -mcpu=pwr9 < %s | FileCheck %s --check-prefix=CHECK-P9
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu \
+; RUN:     -mcpu=pwr9 < %s | FileCheck %s --check-prefix=CHECK-P9
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu \
+; RUN:     -mcpu=pwr9 -mattr=-altivec < %s | FileCheck %s \
+; RUN:     --check-prefix=CHECK-P9-NOALTIVEC
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu \
+; RUN:     -mcpu=pwr8 < %s | FileCheck %s --check-prefix=CHECK-P8
+
+define <4 x i32> @test_vextsh2w(<4 x i32> %m) {
+; CHECK-P9-LABEL: test_vextsh2w:
+; CHECK-P9:       # %bb.0: # %entry
+; CHECK-P9-NEXT:    vextsh2w 2, 2
+; CHECK-P9-NEXT:    blr
+;
+; CHECK-P9-NOALTIVEC-LABEL: test_vextsh2w:
+; CHECK-P9-NOALTIVEC:       # %bb.0: # %entry
+; CHECK-P9-NOALTIVEC-NEXT:    extsh 6, 6
+; CHECK-P9-NOALTIVEC-NEXT:    extsh 5, 5
+; CHECK-P9-NOALTIVEC-NEXT:    extsh 4, 4
+; CHECK-P9-NOALTIVEC-NEXT:    extsh 3, 3
+; CHECK-P9-NOALTIVEC-NEXT:    blr
+;
+; CHECK-P8-LABEL: test_vextsh2w:
+; CHECK-P8:       # %bb.0: # %entry
+; CHECK-P8-NEXT:    vspltisw 3, 8
+; CHECK-P8-NEXT:    vadduwm 3, 3, 3
+; CHECK-P8-NEXT:    vslw 2, 2, 3
+; CHECK-P8-NEXT:    vsraw 2, 2, 3
+; CHECK-P8-NEXT:    blr
+entry:
+  %shl = shl <4 x i32> %m, <i32 16, i32 16, i32 16, i32 16>
+  %shr = ashr exact <4 x i32> %shl, <i32 16, i32 16, i32 16, i32 16>
+  ret <4 x i32> %shr
+}
+
+define <4 x i32> @test_vextsb2w(<4 x i32> %m) {
+; CHECK-P9-LABEL: test_vextsb2w:
+; CHECK-P9:       # %bb.0: # %entry
+; CHECK-P9-NEXT:    vextsb2w 2, 2
+; CHECK-P9-NEXT:    blr
+;
+; CHECK-P9-NOALTIVEC-LABEL: test_vextsb2w:
+; CHECK-P9-NOALTIVEC:       # %bb.0: # %entry
+; CHECK-P9-NOALTIVEC-NEXT:    extsb 6, 6
+; CHECK-P9-NOALTIVEC-NEXT:    extsb 5, 5
+; CHECK-P9-NOALTIVEC-NEXT:    extsb 4, 4
+; CHECK-P9-NOALTIVEC-NEXT:    extsb 3, 3
+; CHECK-P9-NOALTIVEC-NEXT:    blr
+;
+; CHECK-P8-LABEL: test_vextsb2w:
+; CHECK-P8:       # %bb.0: # %entry
+; CHECK-P8-NEXT:    vspltisw 3, 12
+; CHECK-P8-NEXT:    vadduwm 3, 3, 3
+; CHECK-P8-NEXT:    vslw 2, 2, 3
+; CHECK-P8-NEXT:    vsraw 2, 2, 3
+; CHECK-P8-NEXT:    blr
+entry:
+  %shl = shl <4 x i32> %m, <i32 24, i32 24, i32 24, i32 24>
+  %shr = ashr exact <4 x i32> %shl, <i32 24, i32 24, i32 24, i32 24>
+  ret <4 x i32> %shr
+}
+
+define <2 x i64> @test_vextsb2d(<2 x i64> %m) {
+; CHECK-P9-LABEL: test_vextsb2d:
+; CHECK-P9:       # %bb.0: # %entry
+; CHECK-P9-NEXT:    vextsb2d 2, 2
+; CHECK-P9-NEXT:    blr
+;
+; CHECK-P9-NOALTIVEC-LABEL: test_vextsb2d:
+; CHECK-P9-NOALTIVEC:       # %bb.0: # %entry
+; CHECK-P9-NOALTIVEC-NEXT:    extsb 3, 3
+; CHECK-P9-NOALTIVEC-NEXT:    extsb 4, 4
+; CHECK-P9-NOALTIVEC-NEXT:    blr
+;
+; CHECK-P8-LABEL: test_vextsb2d:
+; CHECK-P8:       # %bb.0: # %entry
+; CHECK-P8-NEXT:    addis 3, 2, .LCPI2_0@toc@ha
+; CHECK-P8-NEXT:    addi 3, 3, .LCPI2_0@toc@l
+; CHECK-P8-NEXT:    lxvd2x 0, 0, 3
+; CHECK-P8-NEXT:    xxswapd 35, 0
+; CHECK-P8-NEXT:    vsld 2, 2, 3
+; CHECK-P8-NEXT:    vsrad 2, 2, 3
+; CHECK-P8-NEXT:    blr
+entry:
+  %shl = shl <2 x i64> %m, <i64 56, i64 56>
+  %shr = ashr exact <2 x i64> %shl, <i64 56, i64 56>
+  ret <2 x i64> %shr
+}
+
+define <2 x i64> @test_vextsh2d(<2 x i64> %m) {
+; CHECK-P9-LABEL: test_vextsh2d:
+; CHECK-P9:       # %bb.0: # %entry
+; CHECK-P9-NEXT:    vextsh2d 2, 2
+; CHECK-P9-NEXT:    blr
+;
+; CHECK-P9-NOALTIVEC-LABEL: test_vextsh2d:
+; CHECK-P9-NOALTIVEC:       # %bb.0: # %entry
+; CHECK-P9-NOALTIVEC-NEXT:    extsh 3, 3
+; CHECK-P9-NOALTIVEC-NEXT:    extsh 4, 4
+; CHECK-P9-NOALTIVEC-NEXT:    blr
+;
+; CHECK-P8-LABEL: test_vextsh2d:
+; CHECK-P8:       # %bb.0: # %entry
+; CHECK-P8-NEXT:    addis 3, 2, .LCPI3_0@toc@ha
+; CHECK-P8-NEXT:    addi 3, 3, .LCPI3_0@toc@l
+; CHECK-P8-NEXT:    lxvd2x 0, 0, 3
+; CHECK-P8-NEXT:    xxswapd 35, 0
+; CHECK-P8-NEXT:    vsld 2, 2, 3
+; CHECK-P8-NEXT:    vsrad 2, 2, 3
+; CHECK-P8-NEXT:    blr
+entry:
+  %shl = shl <2 x i64> %m, <i64 48, i64 48>
+  %shr = ashr exact <2 x i64> %shl, <i64 48, i64 48>
+  ret <2 x i64> %shr
+}
+
+define <2 x i64> @test_vextsw2d(<2 x i64> %m) {
+; CHECK-P9-LABEL: test_vextsw2d:
+; CHECK-P9:       # %bb.0: # %entry
+; CHECK-P9-NEXT:    vextsw2d 2, 2
+; CHECK-P9-NEXT:    blr
+;
+; CHECK-P9-NOALTIVEC-LABEL: test_vextsw2d:
+; CHECK-P9-NOALTIVEC:       # %bb.0: # %entry
+; CHECK-P9-NOALTIVEC-NEXT:    extsw 3, 3
+; CHECK-P9-NOALTIVEC-NEXT:    extsw 4, 4
+; CHECK-P9-NOALTIVEC-NEXT:    blr
+;
+; CHECK-P8-LABEL: test_vextsw2d:
+; CHECK-P8:       # %bb.0: # %entry
+; CHECK-P8-NEXT:    addis 3, 2, .LCPI4_0@toc@ha
+; CHECK-P8-NEXT:    addi 3, 3, .LCPI4_0@toc@l
+; CHECK-P8-NEXT:    lxvd2x 0, 0, 3
+; CHECK-P8-NEXT:    xxswapd 35, 0
+; CHECK-P8-NEXT:    vsld 2, 2, 3
+; CHECK-P8-NEXT:    vsrad 2, 2, 3
+; CHECK-P8-NEXT:    blr
+entry:
+  %shl = shl <2 x i64> %m, <i64 32, i64 32>
+  %shr = ashr exact <2 x i64> %shl, <i64 32, i64 32>
+  ret <2 x i64> %shr
+}
+
+define <2 x i64> @test_none(<2 x i64> %m) {
+; CHECK-P9-LABEL: test_none:
+; CHECK-P9:       # %bb.0: # %entry
+; CHECK-P9-NEXT:    addis 3, 2, .LCPI5_0@toc@ha
+; CHECK-P9-NEXT:    addi 3, 3, .LCPI5_0@toc@l
+; CHECK-P9-NEXT:    lxvx 35, 0, 3
+; CHECK-P9-NEXT:    vsld 2, 2, 3
+; CHECK-P9-NEXT:    vsrad 2, 2, 3
+; CHECK-P9-NEXT:    blr
+;
+; CHECK-P9-NOALTIVEC-LABEL: test_none:
+; CHECK-P9-NOALTIVEC:       # %bb.0: # %entry
+; CHECK-P9-NOALTIVEC-NEXT:    sldi 3, 3, 16
+; CHECK-P9-NOALTIVEC-NEXT:    sldi 4, 4, 16
+; CHECK-P9-NOALTIVEC-NEXT:    sradi 3, 3, 16
+; CHECK-P9-NOALTIVEC-NEXT:    sradi 4, 4, 16
+; CHECK-P9-NOALTIVEC-NEXT:    blr
+;
+; CHECK-P8-LABEL: test_none:
+; CHECK-P8:       # %bb.0: # %entry
+; CHECK-P8-NEXT:    addis 3, 2, .LCPI5_0@toc@ha
+; CHECK-P8-NEXT:    addi 3, 3, .LCPI5_0@toc@l
+; CHECK-P8-NEXT:    lxvd2x 0, 0, 3
+; CHECK-P8-NEXT:    xxswapd 35, 0
+; CHECK-P8-NEXT:    vsld 2, 2, 3
+; CHECK-P8-NEXT:    vsrad 2, 2, 3
+; CHECK-P8-NEXT:    blr
+entry:
+  %shl = shl <2 x i64> %m, <i64 16, i64 16>
+  %shr = ashr exact <2 x i64> %shl, <i64 16, i64 16>
+  ret <2 x i64> %shr
+}
diff --git a/llvm/test/CodeGen/PowerPC/vector-rotates.ll b/llvm/test/CodeGen/PowerPC/vector-rotates.ll
new file mode 100644
index 000000000..d5fc48173
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/vector-rotates.ll
@@ -0,0 +1,136 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -O3 -mtriple=powerpc64le-unknown-unknown -ppc-asm-full-reg-names \
+; RUN:   -verify-machineinstrs -mcpu=pwr8 < %s | \
+; RUN:   FileCheck --check-prefix=CHECK-P8 %s
+; RUN: llc -O3 -mtriple=powerpc64-unknown-unknown -ppc-asm-full-reg-names \
+; RUN:   -verify-machineinstrs -mcpu=pwr7 < %s | \
+; RUN:   FileCheck --check-prefix=CHECK-P7 %s
+
+define <16 x i8> @rotl_v16i8(<16 x i8> %a) {
+; CHECK-P8-LABEL: rotl_v16i8:
+; CHECK-P8:       # %bb.0: # %entry
+; CHECK-P8-NEXT:    addis r3, r2, .LCPI0_0@toc@ha
+; CHECK-P8-NEXT:    addi r3, r3, .LCPI0_0@toc@l
+; CHECK-P8-NEXT:    lvx v3, 0, r3
+; CHECK-P8-NEXT:    vrlb v2, v2, v3
+; CHECK-P8-NEXT:    blr
+;
+; CHECK-P7-LABEL: rotl_v16i8:
+; CHECK-P7:       # %bb.0: # %entry
+; CHECK-P7-NEXT:    addis r3, r2, .LCPI0_0@toc@ha
+; CHECK-P7-NEXT:    addi r3, r3, .LCPI0_0@toc@l
+; CHECK-P7-NEXT:    lxvw4x vs35, 0, r3
+; CHECK-P7-NEXT:    vrlb v2, v2, v3
+; CHECK-P7-NEXT:    blr
+entry:
+  %b = shl <16 x i8> %a, <i8 1, i8 1, i8 2, i8 2, i8 3, i8 3, i8 4, i8 4, i8 5, i8 5, i8 6, i8 6, i8 7, i8 7, i8 8, i8 8>
+  %c = lshr <16 x i8> %a, <i8 7, i8 7, i8 6, i8 6, i8 5, i8 5, i8 4, i8 4, i8 3, i8 3, i8 2, i8 2, i8 1, i8 1, i8 0, i8 0>
+  %d = or <16 x i8> %b, %c
+  ret <16 x i8> %d
+}
+
+define <8 x i16> @rotl_v8i16(<8 x i16> %a) {
+; CHECK-P8-LABEL: rotl_v8i16:
+; CHECK-P8:       # %bb.0: # %entry
+; CHECK-P8-NEXT:    addis r3, r2, .LCPI1_0@toc@ha
+; CHECK-P8-NEXT:    addi r3, r3, .LCPI1_0@toc@l
+; CHECK-P8-NEXT:    lvx v3, 0, r3
+; CHECK-P8-NEXT:    vrlh v2, v2, v3
+; CHECK-P8-NEXT:    blr
+;
+; CHECK-P7-LABEL: rotl_v8i16:
+; CHECK-P7:       # %bb.0: # %entry
+; CHECK-P7-NEXT:    addis r3, r2, .LCPI1_0@toc@ha
+; CHECK-P7-NEXT:    addi r3, r3, .LCPI1_0@toc@l
+; CHECK-P7-NEXT:    lxvw4x vs35, 0, r3
+; CHECK-P7-NEXT:    vrlh v2, v2, v3
+; CHECK-P7-NEXT:    blr
+entry:
+  %b = shl <8 x i16> %a, <i16 1, i16 2, i16 3, i16 5, i16 7, i16 11, i16 13, i16 16>
+  %c = lshr <8 x i16> %a, <i16 15, i16 14, i16 13, i16 11, i16 9, i16 5, i16 3, i16 0>
+  %d = or <8 x i16> %b, %c
+  ret <8 x i16> %d
+}
+
+define <4 x i32> @rotl_v4i32_0(<4 x i32> %a) {
+; CHECK-P8-LABEL: rotl_v4i32_0:
+; CHECK-P8:       # %bb.0: # %entry
+; CHECK-P8-NEXT:    addis r3, r2, .LCPI2_0@toc@ha
+; CHECK-P8-NEXT:    addi r3, r3, .LCPI2_0@toc@l
+; CHECK-P8-NEXT:    lvx v3, 0, r3
+; CHECK-P8-NEXT:    vrlw v2, v2, v3
+; CHECK-P8-NEXT:    blr
+;
+; CHECK-P7-LABEL: rotl_v4i32_0:
+; CHECK-P7:       # %bb.0: # %entry
+; CHECK-P7-NEXT:    addis r3, r2, .LCPI2_0@toc@ha
+; CHECK-P7-NEXT:    addi r3, r3, .LCPI2_0@toc@l
+; CHECK-P7-NEXT:    lxvw4x vs35, 0, r3
+; CHECK-P7-NEXT:    vrlw v2, v2, v3
+; CHECK-P7-NEXT:    blr
+entry:
+  %b = shl <4 x i32> %a, <i32 29, i32 19, i32 17, i32 11>
+  %c = lshr <4 x i32> %a, <i32 3, i32 13, i32 15, i32 21>
+  %d = or <4 x i32> %b, %c
+  ret <4 x i32> %d
+}
+
+define <4 x i32> @rotl_v4i32_1(<4 x i32> %a) {
+; CHECK-P8-LABEL: rotl_v4i32_1:
+; CHECK-P8:       # %bb.0: # %entry
+; CHECK-P8-NEXT:    vspltisw v3, -16
+; CHECK-P8-NEXT:    vspltisw v4, 7
+; CHECK-P8-NEXT:    vsubuwm v3, v4, v3
+; CHECK-P8-NEXT:    vrlw v2, v2, v3
+; CHECK-P8-NEXT:    blr
+;
+; CHECK-P7-LABEL: rotl_v4i32_1:
+; CHECK-P7:       # %bb.0: # %entry
+; CHECK-P7-NEXT:    vspltisw v3, -16
+; CHECK-P7-NEXT:    vspltisw v4, 7
+; CHECK-P7-NEXT:    vsubuwm v3, v4, v3
+; CHECK-P7-NEXT:    vrlw v2, v2, v3
+; CHECK-P7-NEXT:    blr
+entry:
+  %b = shl <4 x i32> %a, <i32 23, i32 23, i32 23, i32 23>
+  %c = lshr <4 x i32> %a, <i32 9, i32 9, i32 9, i32 9>
+  %d = or <4 x i32> %b, %c
+  ret <4 x i32> %d
+}
+
+define <2 x i64> @rotl_v2i64(<2 x i64> %a) {
+; CHECK-P8-LABEL: rotl_v2i64:
+; CHECK-P8:       # %bb.0: # %entry
+; CHECK-P8-NEXT:    addis r3, r2, .LCPI4_0@toc@ha
+; CHECK-P8-NEXT:    addi r3, r3, .LCPI4_0@toc@l
+; CHECK-P8-NEXT:    lxvd2x vs0, 0, r3
+; CHECK-P8-NEXT:    xxswapd vs35, vs0
+; CHECK-P8-NEXT:    vrld v2, v2, v3
+; CHECK-P8-NEXT:    blr
+;
+; CHECK-P7-LABEL: rotl_v2i64:
+; CHECK-P7:       # %bb.0: # %entry
+; CHECK-P7-NEXT:    addi r3, r1, -48
+; CHECK-P7-NEXT:    stxvd2x vs34, 0, r3
+; CHECK-P7-NEXT:    ld r3, -40(r1)
+; CHECK-P7-NEXT:    sldi r4, r3, 53
+; CHECK-P7-NEXT:    rldicl r3, r3, 53, 11
+; CHECK-P7-NEXT:    std r4, -8(r1)
+; CHECK-P7-NEXT:    ld r4, -48(r1)
+; CHECK-P7-NEXT:    sldi r5, r4, 41
+; CHECK-P7-NEXT:    rldicl r4, r4, 41, 23
+; CHECK-P7-NEXT:    std r5, -16(r1)
+; CHECK-P7-NEXT:    addi r5, r1, -16
+; CHECK-P7-NEXT:    lxvw4x vs0, 0, r5
+; CHECK-P7-NEXT:    std r3, -24(r1)
+; CHECK-P7-NEXT:    addi r3, r1, -32
+; CHECK-P7-NEXT:    std r4, -32(r1)
+; CHECK-P7-NEXT:    lxvw4x vs1, 0, r3
+; CHECK-P7-NEXT:    xxlor vs34, vs0, vs1
+; CHECK-P7-NEXT:    blr
+entry:
+  %b = shl <2 x i64> %a, <i64 41, i64 53>
+  %c = lshr <2 x i64> %a, <i64 23, i64 11>
+  %d = or <2 x i64> %b, %c
+  ret <2 x i64> %d
+}
diff --git a/llvm/test/CodeGen/PowerPC/vselect-constants.ll b/llvm/test/CodeGen/PowerPC/vselect-constants.ll
index b4264e5ee..df939d066 100644
--- a/llvm/test/CodeGen/PowerPC/vselect-constants.ll
+++ b/llvm/test/CodeGen/PowerPC/vselect-constants.ll
@@ -130,9 +130,9 @@ define <4 x i32> @sel_0_or_minus1_vec(<4 x i1> %cond) {
 ; CHECK-LABEL: sel_0_or_minus1_vec:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vspltisw 3, 1
-; CHECK-NEXT:    vspltisb 4, -1
 ; CHECK-NEXT:    xxland 34, 34, 35
-; CHECK-NEXT:    vadduwm 2, 2, 4
+; CHECK-NEXT:    xxleqv 35, 35, 35
+; CHECK-NEXT:    vadduwm 2, 2, 3
 ; CHECK-NEXT:    blr
   %add = select <4 x i1> %cond, <4 x i32> <i32 0, i32 0, i32 0, i32 0>, <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>
   ret <4 x i32> %add
diff --git a/llvm/test/CodeGen/PowerPC/vsx-fma-mutate-undef.ll b/llvm/test/CodeGen/PowerPC/vsx-fma-mutate-undef.ll
index ed6b49dd7..301446a9e 100644
--- a/llvm/test/CodeGen/PowerPC/vsx-fma-mutate-undef.ll
+++ b/llvm/test/CodeGen/PowerPC/vsx-fma-mutate-undef.ll
@@ -3,12 +3,12 @@ target datalayout = "e-m:e-i64:64-n32:64"
 target triple = "powerpc64le-unknown-linux-gnu"
 
 ; Function Attrs: nounwind
-define void @acosh_float8(<4 x i32> %v1, <4 x i32> %v2) #0 {
+define void @acosh_float8(<4 x i32> %v1, <4 x i32> %v2, <4 x float> %x) #0 {
 entry:
   br i1 undef, label %if.then, label %if.end
 
 if.then:                                          ; preds = %entry
-  %0 = tail call <4 x float> @llvm.fmuladd.v4f32(<4 x float> undef, <4 x float> <float 0x3FE62E4200000000, float 0x3FE62E4200000000, float 0x3FE62E4200000000, float 0x3FE62E4200000000>, <4 x float> undef) #0
+  %0 = tail call <4 x float> @llvm.fmuladd.v4f32(<4 x float> %x, <4 x float> <float 0x3FE62E4200000000, float 0x3FE62E4200000000, float 0x3FE62E4200000000, float 0x3FE62E4200000000>, <4 x float> %x) #0
   %astype.i.i.74.i = bitcast <4 x float> %0 to <4 x i32>
   %and.i.i.76.i = and <4 x i32> %astype.i.i.74.i, %v1
   %or.i.i.79.i = or <4 x i32> %and.i.i.76.i, %v2
diff --git a/llvm/test/CodeGen/PowerPC/vsx_insert_extract_le.ll b/llvm/test/CodeGen/PowerPC/vsx_insert_extract_le.ll
index 0c081f744..7450be09f 100644
--- a/llvm/test/CodeGen/PowerPC/vsx_insert_extract_le.ll
+++ b/llvm/test/CodeGen/PowerPC/vsx_insert_extract_le.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -verify-machineinstrs -mcpu=pwr8 -mattr=+vsx -ppc-vsr-nums-as-vr \
 ; RUN:   -ppc-asm-full-reg-names -mtriple=powerpc64le-unknown-linux-gnu < %s \
 ; RUN:   | FileCheck %s
@@ -31,10 +32,10 @@ define <2 x double> @testi0(<2 x double>* %p1, double* %p2) {
 ;
 ; CHECK-P9-LABEL: testi0:
 ; CHECK-P9:       # %bb.0:
-; CHECK-P9-NEXT:    lfd [[REG:f[0-9]+]], 0(r4)
-; CHECK-P9-NEXT:    lxv [[REG1:vs[0-9]+]], 0(r3)
-; CHECK-P9-NEXT:    xxpermdi [[REG2:vs[0-9]+]], [[REG]], [[REG]], 2
-; CHECK-P9-NEXT:    xxpermdi v2, [[REG1]], [[REG2]], 1
+; CHECK-P9-NEXT:    lfd f1, 0(r4)
+; CHECK-P9-NEXT:    lxv vs0, 0(r3)
+; CHECK-P9-NEXT:    xxpermdi vs1, f1, f1, 2
+; CHECK-P9-NEXT:    xxpermdi v2, vs0, vs1, 1
 ; CHECK-P9-NEXT:    blr
   %v = load <2 x double>, <2 x double>* %p1
   %s = load double, double* %p2
@@ -65,10 +66,10 @@ define <2 x double> @testi1(<2 x double>* %p1, double* %p2) {
 ;
 ; CHECK-P9-LABEL: testi1:
 ; CHECK-P9:       # %bb.0:
-; CHECK-P9-NEXT:    lfd [[REG:f[0-9]+]], 0(r4)
-; CHECK-P9-NEXT:    lxv [[REG1:vs[0-9]+]], 0(r3)
-; CHECK-P9-NEXT:    xxpermdi [[REG2:vs[0-9]+]], [[REG]], [[REG]], 2
-; CHECK-P9-NEXT:    xxmrgld v2, [[REG2]], [[REG1]] 
+; CHECK-P9-NEXT:    lfd f1, 0(r4)
+; CHECK-P9-NEXT:    lxv vs0, 0(r3)
+; CHECK-P9-NEXT:    xxpermdi vs1, f1, f1, 2
+; CHECK-P9-NEXT:    xxmrgld v2, vs1, vs0
 ; CHECK-P9-NEXT:    blr
   %v = load <2 x double>, <2 x double>* %p1
   %s = load double, double* %p2
@@ -82,12 +83,14 @@ define double @teste0(<2 x double>* %p1) {
 ; CHECK-LABEL: teste0:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lxvd2x vs1, 0, r3
-; CHECK:         blr
+; CHECK-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
+; CHECK-NEXT:    blr
 ;
 ; CHECK-P9-VECTOR-LABEL: teste0:
 ; CHECK-P9-VECTOR:       # %bb.0:
 ; CHECK-P9-VECTOR-NEXT:    lxvd2x vs1, 0, r3
-; CHECK-P9-VECTOR:         blr
+; CHECK-P9-VECTOR-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
+; CHECK-P9-VECTOR-NEXT:    blr
 ;
 ; CHECK-P9-LABEL: teste0:
 ; CHECK-P9:       # %bb.0:
@@ -105,13 +108,15 @@ define double @teste1(<2 x double>* %p1) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lxvd2x vs0, 0, r3
 ; CHECK-NEXT:    xxswapd vs1, vs0
-; CHECK:         blr
+; CHECK-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
+; CHECK-NEXT:    blr
 ;
 ; CHECK-P9-VECTOR-LABEL: teste1:
 ; CHECK-P9-VECTOR:       # %bb.0:
 ; CHECK-P9-VECTOR-NEXT:    lxvd2x vs0, 0, r3
 ; CHECK-P9-VECTOR-NEXT:    xxswapd vs1, vs0
-; CHECK-P9-VECTOR:         blr
+; CHECK-P9-VECTOR-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
+; CHECK-P9-VECTOR-NEXT:    blr
 ;
 ; CHECK-P9-LABEL: teste1:
 ; CHECK-P9:       # %bb.0:
diff --git a/llvm/test/CodeGen/PowerPC/vsx_shuffle_le.ll b/llvm/test/CodeGen/PowerPC/vsx_shuffle_le.ll
index cfe201999..c2b886d60 100644
--- a/llvm/test/CodeGen/PowerPC/vsx_shuffle_le.ll
+++ b/llvm/test/CodeGen/PowerPC/vsx_shuffle_le.ll
@@ -85,8 +85,7 @@ define <2 x double> @test10(<2 x double>* %p1, <2 x double>* %p2) {
 ; CHECK: lxvd2x 34, 0, 3
 
 ; CHECK-P9-LABEL: @test10
-; CHECK-P9: lxv 0, 0(3)
-; CHECK-P9: xxswapd 34, 0
+; CHECK-P9: lxvd2x 34, 0, 3
 }
 
 define <2 x double> @test11(<2 x double>* %p1, <2 x double>* %p2) {
@@ -257,8 +256,7 @@ define <2 x double> @test32(<2 x double>* %p1, <2 x double>* %p2) {
 ; CHECK: lxvd2x 34, 0, 4
 
 ; CHECK-P9-LABEL: @test32
-; CHECK-P9: lxv 0, 0(4)
-; CHECK-P9: xxswapd 34, 0
+; CHECK-P9: lxvd2x 34, 0, 4
 }
 
 define <2 x double> @test33(<2 x double>* %p1, <2 x double>* %p2) {
diff --git a/llvm/test/CodeGen/PowerPC/zero-not-run.ll b/llvm/test/CodeGen/PowerPC/zero-not-run.ll
index 77a9f7229..3e09b3b77 100644
--- a/llvm/test/CodeGen/PowerPC/zero-not-run.ll
+++ b/llvm/test/CodeGen/PowerPC/zero-not-run.ll
@@ -24,4 +24,4 @@ for.end731:                                       ; preds = %entry
 ; Function Attrs: nounwind
 declare i64 @safe_mod_func_uint64_t_u_u(i64, i64) #0
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/PowerPC/zext-and-cmp.ll b/llvm/test/CodeGen/PowerPC/zext-and-cmp.ll
index b06a384d7..69474bd08 100644
--- a/llvm/test/CodeGen/PowerPC/zext-and-cmp.ll
+++ b/llvm/test/CodeGen/PowerPC/zext-and-cmp.ll
@@ -4,7 +4,7 @@
 ; to zero can be simplifed by using the record form when one of its operands
 ; is known to be zero extended.
 
-@k = common local_unnamed_addr global i32 0, align 4
+@k = local_unnamed_addr global i32 0, align 4
 
 ; Function Attrs: norecurse nounwind
 define signext i32 @cmplwi(i32* nocapture readonly %p, i32* nocapture readonly %q, i32 signext %j, i32 signext %r10) {
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/calllowering-ret.ll b/llvm/test/CodeGen/RISCV/GlobalISel/calllowering-ret.ll
new file mode 100644
index 000000000..44438c1cc
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/calllowering-ret.ll
@@ -0,0 +1,17 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -global-isel -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV32I %s
+; RUN: llc -mtriple=riscv64 -global-isel -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64I %s
+
+define void @foo() {
+  ; RV32I-LABEL: foo
+  ; RV32I:       # %bb.0: # %entry
+  ; RV32I:         ret
+
+  ; RV64I-LABEL: foo
+  ; RV64I:       # %bb.0: # %entry
+  ; RV64I:         ret
+entry:
+  ret void
+}
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/irtranslator-calllowering.ll b/llvm/test/CodeGen/RISCV/GlobalISel/irtranslator-calllowering.ll
new file mode 100644
index 000000000..33f34705c
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/irtranslator-calllowering.ll
@@ -0,0 +1,17 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -global-isel -stop-after=irtranslator -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV32I %s
+; RUN: llc -mtriple=riscv64 -global-isel -stop-after=irtranslator -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64I %s
+
+define void @foo() {
+  ; RV32I-LABEL: name: foo
+  ; RV32I: bb.1.entry:
+  ; RV32I-NEXT:   PseudoRET
+
+  ; RV64I-LABEL: name: foo
+  ; RV64I: bb.1.entry:
+  ; RV64I-NEXT:   PseudoRET
+entry:
+  ret void
+}
diff --git a/llvm/test/CodeGen/RISCV/add-before-shl.ll b/llvm/test/CodeGen/RISCV/add-before-shl.ll
index 3279de3ea..51e55923c 100644
--- a/llvm/test/CodeGen/RISCV/add-before-shl.ll
+++ b/llvm/test/CodeGen/RISCV/add-before-shl.ll
@@ -96,36 +96,36 @@ define i128 @add_wide_operand(i128 %a) nounwind {
 ; RV32I-LABEL: add_wide_operand:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    lw a2, 0(a1)
-; RV32I-NEXT:    srli a3, a2, 29
-; RV32I-NEXT:    lw a4, 4(a1)
-; RV32I-NEXT:    slli a5, a4, 3
-; RV32I-NEXT:    or a6, a5, a3
-; RV32I-NEXT:    srli a4, a4, 29
-; RV32I-NEXT:    lw a5, 8(a1)
-; RV32I-NEXT:    slli a3, a5, 3
-; RV32I-NEXT:    or a3, a3, a4
+; RV32I-NEXT:    lw a3, 4(a1)
+; RV32I-NEXT:    lw a6, 12(a1)
+; RV32I-NEXT:    lw a1, 8(a1)
+; RV32I-NEXT:    srli a5, a2, 29
+; RV32I-NEXT:    slli a4, a3, 3
+; RV32I-NEXT:    or a4, a4, a5
+; RV32I-NEXT:    srli a3, a3, 29
+; RV32I-NEXT:    slli a5, a1, 3
+; RV32I-NEXT:    or a3, a5, a3
+; RV32I-NEXT:    srli a1, a1, 29
+; RV32I-NEXT:    slli a5, a6, 3
+; RV32I-NEXT:    or a1, a5, a1
 ; RV32I-NEXT:    slli a2, a2, 3
+; RV32I-NEXT:    lui a5, 128
+; RV32I-NEXT:    add a1, a1, a5
 ; RV32I-NEXT:    sw a2, 0(a0)
 ; RV32I-NEXT:    sw a3, 8(a0)
-; RV32I-NEXT:    sw a6, 4(a0)
-; RV32I-NEXT:    srli a2, a5, 29
-; RV32I-NEXT:    lw a1, 12(a1)
-; RV32I-NEXT:    slli a1, a1, 3
-; RV32I-NEXT:    or a1, a1, a2
-; RV32I-NEXT:    lui a2, 128
-; RV32I-NEXT:    add a1, a1, a2
+; RV32I-NEXT:    sw a4, 4(a0)
 ; RV32I-NEXT:    sw a1, 12(a0)
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: add_wide_operand:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    slli a1, a1, 3
 ; RV64I-NEXT:    srli a2, a0, 61
+; RV64I-NEXT:    slli a1, a1, 3
 ; RV64I-NEXT:    or a1, a1, a2
+; RV64I-NEXT:    slli a0, a0, 3
 ; RV64I-NEXT:    addi a2, zero, 1
 ; RV64I-NEXT:    slli a2, a2, 51
 ; RV64I-NEXT:    add a1, a1, a2
-; RV64I-NEXT:    slli a0, a0, 3
 ; RV64I-NEXT:    ret
   %1 = add i128 %a, 5192296858534827628530496329220096
   %2 = shl i128 %1, 3
diff --git a/llvm/test/CodeGen/RISCV/addc-adde-sube-subc.ll b/llvm/test/CodeGen/RISCV/addc-adde-sube-subc.ll
index 068e52f61..5fd8261e2 100644
--- a/llvm/test/CodeGen/RISCV/addc-adde-sube-subc.ll
+++ b/llvm/test/CodeGen/RISCV/addc-adde-sube-subc.ll
@@ -20,9 +20,9 @@ define i64 @addc_adde(i64 %a, i64 %b) nounwind {
 define i64 @subc_sube(i64 %a, i64 %b) nounwind {
 ; RV32I-LABEL: subc_sube:
 ; RV32I:       # %bb.0:
+; RV32I-NEXT:    sltu a4, a0, a2
 ; RV32I-NEXT:    sub a1, a1, a3
-; RV32I-NEXT:    sltu a3, a0, a2
-; RV32I-NEXT:    sub a1, a1, a3
+; RV32I-NEXT:    sub a1, a1, a4
 ; RV32I-NEXT:    sub a0, a0, a2
 ; RV32I-NEXT:    ret
   %1 = sub i64 %a, %b
diff --git a/llvm/test/CodeGen/RISCV/addcarry.ll b/llvm/test/CodeGen/RISCV/addcarry.ll
index 15fe53e9f..71711f9eb 100644
--- a/llvm/test/CodeGen/RISCV/addcarry.ll
+++ b/llvm/test/CodeGen/RISCV/addcarry.ll
@@ -10,17 +10,17 @@ define i64 @addcarry(i64 %x, i64 %y) nounwind {
 ; RISCV32-LABEL: addcarry:
 ; RISCV32:       # %bb.0:
 ; RISCV32-NEXT:    mul a4, a0, a3
-; RISCV32-NEXT:    mulhu a5, a0, a2
-; RISCV32-NEXT:    add a4, a5, a4
-; RISCV32-NEXT:    sltu a6, a4, a5
-; RISCV32-NEXT:    mulhu a5, a0, a3
-; RISCV32-NEXT:    add a6, a5, a6
-; RISCV32-NEXT:    mulhu a5, a1, a2
-; RISCV32-NEXT:    add a7, a6, a5
+; RISCV32-NEXT:    mulhu a7, a0, a2
+; RISCV32-NEXT:    add a4, a7, a4
 ; RISCV32-NEXT:    mul a5, a1, a2
 ; RISCV32-NEXT:    add a6, a4, a5
-; RISCV32-NEXT:    sltu a4, a6, a4
-; RISCV32-NEXT:    add a4, a7, a4
+; RISCV32-NEXT:    sltu t0, a6, a4
+; RISCV32-NEXT:    sltu a4, a4, a7
+; RISCV32-NEXT:    mulhu a5, a0, a3
+; RISCV32-NEXT:    add a4, a5, a4
+; RISCV32-NEXT:    mulhu a5, a1, a2
+; RISCV32-NEXT:    add a4, a4, a5
+; RISCV32-NEXT:    add a4, a4, t0
 ; RISCV32-NEXT:    mul a5, a1, a3
 ; RISCV32-NEXT:    add a5, a4, a5
 ; RISCV32-NEXT:    bgez a1, .LBB0_2
@@ -31,13 +31,13 @@ define i64 @addcarry(i64 %x, i64 %y) nounwind {
 ; RISCV32-NEXT:  # %bb.3:
 ; RISCV32-NEXT:    sub a5, a5, a0
 ; RISCV32-NEXT:  .LBB0_4:
+; RISCV32-NEXT:    slli a1, a5, 30
+; RISCV32-NEXT:    srli a3, a6, 2
+; RISCV32-NEXT:    or a1, a1, a3
+; RISCV32-NEXT:    slli a3, a6, 30
 ; RISCV32-NEXT:    mul a0, a0, a2
 ; RISCV32-NEXT:    srli a0, a0, 2
-; RISCV32-NEXT:    slli a1, a6, 30
-; RISCV32-NEXT:    or a0, a0, a1
-; RISCV32-NEXT:    srli a1, a6, 2
-; RISCV32-NEXT:    slli a2, a5, 30
-; RISCV32-NEXT:    or a1, a1, a2
+; RISCV32-NEXT:    or a0, a3, a0
 ; RISCV32-NEXT:    ret
   %tmp = call i64 @llvm.smul.fix.i64(i64 %x, i64 %y, i32 2);
   ret i64 %tmp;
diff --git a/llvm/test/CodeGen/RISCV/alloca.ll b/llvm/test/CodeGen/RISCV/alloca.ll
index 22dd42ccb..6cd9bf463 100644
--- a/llvm/test/CodeGen/RISCV/alloca.ll
+++ b/llvm/test/CodeGen/RISCV/alloca.ll
@@ -82,8 +82,7 @@ define void @alloca_callframe(i32 %n) nounwind {
 ; RV32I-NEXT:    sw a1, 8(sp)
 ; RV32I-NEXT:    addi a1, zero, 10
 ; RV32I-NEXT:    sw a1, 4(sp)
-; RV32I-NEXT:    addi a1, zero, 9
-; RV32I-NEXT:    sw a1, 0(sp)
+; RV32I-NEXT:    addi t0, zero, 9
 ; RV32I-NEXT:    addi a1, zero, 2
 ; RV32I-NEXT:    addi a2, zero, 3
 ; RV32I-NEXT:    addi a3, zero, 4
@@ -91,6 +90,7 @@ define void @alloca_callframe(i32 %n) nounwind {
 ; RV32I-NEXT:    addi a5, zero, 6
 ; RV32I-NEXT:    addi a6, zero, 7
 ; RV32I-NEXT:    addi a7, zero, 8
+; RV32I-NEXT:    sw t0, 0(sp)
 ; RV32I-NEXT:    call func
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    addi sp, s0, -16
diff --git a/llvm/test/CodeGen/RISCV/alu32.ll b/llvm/test/CodeGen/RISCV/alu32.ll
index d6f667c9c..975716e53 100644
--- a/llvm/test/CodeGen/RISCV/alu32.ll
+++ b/llvm/test/CodeGen/RISCV/alu32.ll
@@ -153,7 +153,7 @@ define i32 @add(i32 %a, i32 %b) nounwind {
 ;
 ; RV64I-LABEL: add:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    add a0, a0, a1
+; RV64I-NEXT:    addw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = add i32 %a, %b
   ret i32 %1
@@ -167,7 +167,7 @@ define i32 @sub(i32 %a, i32 %b) nounwind {
 ;
 ; RV64I-LABEL: sub:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sub a0, a0, a1
+; RV64I-NEXT:    subw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = sub i32 %a, %b
   ret i32 %1
diff --git a/llvm/test/CodeGen/RISCV/alu64.ll b/llvm/test/CodeGen/RISCV/alu64.ll
index 0a44a380d..1160b4055 100644
--- a/llvm/test/CodeGen/RISCV/alu64.ll
+++ b/llvm/test/CodeGen/RISCV/alu64.ll
@@ -123,8 +123,8 @@ define i64 @slli(i64 %a) nounwind {
 ;
 ; RV32I-LABEL: slli:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    slli a1, a1, 7
 ; RV32I-NEXT:    srli a2, a0, 25
+; RV32I-NEXT:    slli a1, a1, 7
 ; RV32I-NEXT:    or a1, a1, a2
 ; RV32I-NEXT:    slli a0, a0, 7
 ; RV32I-NEXT:    ret
@@ -140,8 +140,8 @@ define i64 @srli(i64 %a) nounwind {
 ;
 ; RV32I-LABEL: srli:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    srli a0, a0, 8
 ; RV32I-NEXT:    slli a2, a1, 24
+; RV32I-NEXT:    srli a0, a0, 8
 ; RV32I-NEXT:    or a0, a0, a2
 ; RV32I-NEXT:    srli a1, a1, 8
 ; RV32I-NEXT:    ret
@@ -157,8 +157,8 @@ define i64 @srai(i64 %a) nounwind {
 ;
 ; RV32I-LABEL: srai:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    srli a0, a0, 9
 ; RV32I-NEXT:    slli a2, a1, 23
+; RV32I-NEXT:    srli a0, a0, 9
 ; RV32I-NEXT:    or a0, a0, a2
 ; RV32I-NEXT:    srai a1, a1, 9
 ; RV32I-NEXT:    ret
@@ -194,9 +194,9 @@ define i64 @sub(i64 %a, i64 %b) nounwind {
 ;
 ; RV32I-LABEL: sub:
 ; RV32I:       # %bb.0:
+; RV32I-NEXT:    sltu a4, a0, a2
 ; RV32I-NEXT:    sub a1, a1, a3
-; RV32I-NEXT:    sltu a3, a0, a2
-; RV32I-NEXT:    sub a1, a1, a3
+; RV32I-NEXT:    sub a1, a1, a4
 ; RV32I-NEXT:    sub a0, a0, a2
 ; RV32I-NEXT:    ret
   %1 = sub i64 %a, %b
@@ -218,11 +218,11 @@ define i64 @sll(i64 %a, i64 %b) nounwind {
 ; RV32I-NEXT:    mv a0, zero
 ; RV32I-NEXT:    ret
 ; RV32I-NEXT:  .LBB11_2:
+; RV32I-NEXT:    sll a1, a1, a2
 ; RV32I-NEXT:    addi a3, zero, 31
 ; RV32I-NEXT:    sub a3, a3, a2
 ; RV32I-NEXT:    srli a4, a0, 1
 ; RV32I-NEXT:    srl a3, a4, a3
-; RV32I-NEXT:    sll a1, a1, a2
 ; RV32I-NEXT:    or a1, a1, a3
 ; RV32I-NEXT:    sll a0, a0, a2
 ; RV32I-NEXT:    ret
@@ -304,11 +304,11 @@ define i64 @srl(i64 %a, i64 %b) nounwind {
 ; RV32I-NEXT:    mv a1, zero
 ; RV32I-NEXT:    ret
 ; RV32I-NEXT:  .LBB15_2:
+; RV32I-NEXT:    srl a0, a0, a2
 ; RV32I-NEXT:    addi a3, zero, 31
 ; RV32I-NEXT:    sub a3, a3, a2
 ; RV32I-NEXT:    slli a4, a1, 1
 ; RV32I-NEXT:    sll a3, a4, a3
-; RV32I-NEXT:    srl a0, a0, a2
 ; RV32I-NEXT:    or a0, a0, a3
 ; RV32I-NEXT:    srl a1, a1, a2
 ; RV32I-NEXT:    ret
@@ -331,11 +331,11 @@ define i64 @sra(i64 %a, i64 %b) nounwind {
 ; RV32I-NEXT:    srai a1, a1, 31
 ; RV32I-NEXT:    ret
 ; RV32I-NEXT:  .LBB16_2:
+; RV32I-NEXT:    srl a0, a0, a2
 ; RV32I-NEXT:    addi a3, zero, 31
 ; RV32I-NEXT:    sub a3, a3, a2
 ; RV32I-NEXT:    slli a4, a1, 1
 ; RV32I-NEXT:    sll a3, a4, a3
-; RV32I-NEXT:    srl a0, a0, a2
 ; RV32I-NEXT:    or a0, a0, a3
 ; RV32I-NEXT:    sra a1, a1, a2
 ; RV32I-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/arith-with-overflow.ll b/llvm/test/CodeGen/RISCV/arith-with-overflow.ll
index 3fb634213..4358fb5a1 100644
--- a/llvm/test/CodeGen/RISCV/arith-with-overflow.ll
+++ b/llvm/test/CodeGen/RISCV/arith-with-overflow.ll
@@ -11,16 +11,10 @@ define i1 @sadd(i32 %a, i32 %b, i32* %c) nounwind {
 ; RV32I-LABEL: sadd:
 ; RV32I:       # %bb.0: # %entry
 ; RV32I-NEXT:    add a3, a0, a1
+; RV32I-NEXT:    slt a0, a3, a0
+; RV32I-NEXT:    slti a1, a1, 0
+; RV32I-NEXT:    xor a0, a1, a0
 ; RV32I-NEXT:    sw a3, 0(a2)
-; RV32I-NEXT:    addi a2, zero, -1
-; RV32I-NEXT:    slt a1, a2, a1
-; RV32I-NEXT:    slt a0, a2, a0
-; RV32I-NEXT:    slt a2, a2, a3
-; RV32I-NEXT:    xor a2, a0, a2
-; RV32I-NEXT:    xor a0, a0, a1
-; RV32I-NEXT:    seqz a0, a0
-; RV32I-NEXT:    snez a1, a2
-; RV32I-NEXT:    and a0, a0, a1
 ; RV32I-NEXT:    ret
 entry:
   %x = call {i32, i1} @llvm.sadd.with.overflow.i32(i32 %a, i32 %b)
@@ -33,17 +27,11 @@ entry:
 define i1 @ssub(i32 %a, i32 %b, i32* %c) nounwind {
 ; RV32I-LABEL: ssub:
 ; RV32I:       # %bb.0: # %entry
-; RV32I-NEXT:    sub a3, a0, a1
-; RV32I-NEXT:    sw a3, 0(a2)
-; RV32I-NEXT:    addi a2, zero, -1
-; RV32I-NEXT:    slt a1, a2, a1
-; RV32I-NEXT:    slt a0, a2, a0
-; RV32I-NEXT:    slt a2, a2, a3
-; RV32I-NEXT:    xor a2, a0, a2
-; RV32I-NEXT:    xor a0, a0, a1
-; RV32I-NEXT:    snez a0, a0
-; RV32I-NEXT:    snez a1, a2
-; RV32I-NEXT:    and a0, a0, a1
+; RV32I-NEXT:    sgtz a3, a1
+; RV32I-NEXT:    sub a1, a0, a1
+; RV32I-NEXT:    slt a0, a1, a0
+; RV32I-NEXT:    xor a0, a3, a0
+; RV32I-NEXT:    sw a1, 0(a2)
 ; RV32I-NEXT:    ret
 entry:
   %x = call {i32, i1} @llvm.ssub.with.overflow.i32(i32 %a, i32 %b)
@@ -57,8 +45,8 @@ define i1 @uadd(i32 %a, i32 %b, i32* %c) nounwind {
 ; RV32I-LABEL: uadd:
 ; RV32I:       # %bb.0: # %entry
 ; RV32I-NEXT:    add a1, a0, a1
-; RV32I-NEXT:    sw a1, 0(a2)
 ; RV32I-NEXT:    sltu a0, a1, a0
+; RV32I-NEXT:    sw a1, 0(a2)
 ; RV32I-NEXT:    ret
 entry:
   %x = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %a, i32 %b)
@@ -72,8 +60,8 @@ define i1 @usub(i32 %a, i32 %b, i32* %c) nounwind {
 ; RV32I-LABEL: usub:
 ; RV32I:       # %bb.0: # %entry
 ; RV32I-NEXT:    sub a1, a0, a1
-; RV32I-NEXT:    sw a1, 0(a2)
 ; RV32I-NEXT:    sltu a0, a0, a1
+; RV32I-NEXT:    sw a1, 0(a2)
 ; RV32I-NEXT:    ret
 entry:
   %x = call {i32, i1} @llvm.usub.with.overflow.i32(i32 %a, i32 %b)
diff --git a/llvm/test/CodeGen/RISCV/atomic-cmpxchg-flag.ll b/llvm/test/CodeGen/RISCV/atomic-cmpxchg-flag.ll
index b331b4b99..082b92bb6 100644
--- a/llvm/test/CodeGen/RISCV/atomic-cmpxchg-flag.ll
+++ b/llvm/test/CodeGen/RISCV/atomic-cmpxchg-flag.ll
@@ -7,7 +7,6 @@
 ; higher bits were masked to zero for the comparison.
 
 define i1 @cmpxchg_i32_seq_cst_seq_cst(i32* %ptr, i32 signext %cmp,
-        i32 signext %val) nounwind {
 ; RV64IA-LABEL: cmpxchg_i32_seq_cst_seq_cst:
 ; RV64IA:       # %bb.0: # %entry
 ; RV64IA-NEXT:  .LBB0_1: # %entry
@@ -22,6 +21,7 @@ define i1 @cmpxchg_i32_seq_cst_seq_cst(i32* %ptr, i32 signext %cmp,
 ; RV64IA-NEXT:    xor a0, a3, a1
 ; RV64IA-NEXT:    seqz a0, a0
 ; RV64IA-NEXT:    ret
+        i32 signext %val) nounwind {
 entry:
   %0 = cmpxchg i32* %ptr, i32 %cmp, i32 %val seq_cst seq_cst
   %1 = extractvalue { i32, i1 } %0, 1
diff --git a/llvm/test/CodeGen/RISCV/atomic-cmpxchg.ll b/llvm/test/CodeGen/RISCV/atomic-cmpxchg.ll
index a4526b7f4..43da05ebe 100644
--- a/llvm/test/CodeGen/RISCV/atomic-cmpxchg.ll
+++ b/llvm/test/CodeGen/RISCV/atomic-cmpxchg.ll
@@ -24,24 +24,24 @@ define void @cmpxchg_i8_monotonic_monotonic(i8* %ptr, i8 %cmp, i8 %val) nounwind
 ;
 ; RV32IA-LABEL: cmpxchg_i8_monotonic_monotonic:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
+; RV32IA-NEXT:    andi a3, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a4, zero, 255
-; RV32IA-NEXT:    sll a4, a4, a3
-; RV32IA-NEXT:    andi a2, a2, 255
-; RV32IA-NEXT:    sll a2, a2, a3
+; RV32IA-NEXT:    sll a4, a4, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    andi a2, a2, 255
+; RV32IA-NEXT:    sll a0, a2, a0
 ; RV32IA-NEXT:  .LBB0_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a3, (a0)
-; RV32IA-NEXT:    and a5, a3, a4
+; RV32IA-NEXT:    lr.w a2, (a3)
+; RV32IA-NEXT:    and a5, a2, a4
 ; RV32IA-NEXT:    bne a5, a1, .LBB0_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB0_1 Depth=1
-; RV32IA-NEXT:    xor a5, a3, a2
+; RV32IA-NEXT:    xor a5, a2, a0
 ; RV32IA-NEXT:    and a5, a5, a4
-; RV32IA-NEXT:    xor a5, a3, a5
-; RV32IA-NEXT:    sc.w a5, a5, (a0)
+; RV32IA-NEXT:    xor a5, a2, a5
+; RV32IA-NEXT:    sc.w a5, a5, (a3)
 ; RV32IA-NEXT:    bnez a5, .LBB0_1
 ; RV32IA-NEXT:  .LBB0_3:
 ; RV32IA-NEXT:    ret
@@ -61,24 +61,24 @@ define void @cmpxchg_i8_monotonic_monotonic(i8* %ptr, i8 %cmp, i8 %val) nounwind
 ;
 ; RV64IA-LABEL: cmpxchg_i8_monotonic_monotonic:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
+; RV64IA-NEXT:    andi a3, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a4, zero, 255
-; RV64IA-NEXT:    sllw a4, a4, a3
-; RV64IA-NEXT:    andi a2, a2, 255
-; RV64IA-NEXT:    sllw a2, a2, a3
+; RV64IA-NEXT:    sllw a4, a4, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    andi a2, a2, 255
+; RV64IA-NEXT:    sllw a0, a2, a0
 ; RV64IA-NEXT:  .LBB0_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a3, (a0)
-; RV64IA-NEXT:    and a5, a3, a4
+; RV64IA-NEXT:    lr.w a2, (a3)
+; RV64IA-NEXT:    and a5, a2, a4
 ; RV64IA-NEXT:    bne a5, a1, .LBB0_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB0_1 Depth=1
-; RV64IA-NEXT:    xor a5, a3, a2
+; RV64IA-NEXT:    xor a5, a2, a0
 ; RV64IA-NEXT:    and a5, a5, a4
-; RV64IA-NEXT:    xor a5, a3, a5
-; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    xor a5, a2, a5
+; RV64IA-NEXT:    sc.w a5, a5, (a3)
 ; RV64IA-NEXT:    bnez a5, .LBB0_1
 ; RV64IA-NEXT:  .LBB0_3:
 ; RV64IA-NEXT:    ret
@@ -102,24 +102,24 @@ define void @cmpxchg_i8_acquire_monotonic(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ;
 ; RV32IA-LABEL: cmpxchg_i8_acquire_monotonic:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
+; RV32IA-NEXT:    andi a3, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a4, zero, 255
-; RV32IA-NEXT:    sll a4, a4, a3
-; RV32IA-NEXT:    andi a2, a2, 255
-; RV32IA-NEXT:    sll a2, a2, a3
+; RV32IA-NEXT:    sll a4, a4, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    andi a2, a2, 255
+; RV32IA-NEXT:    sll a0, a2, a0
 ; RV32IA-NEXT:  .LBB1_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a3, (a0)
-; RV32IA-NEXT:    and a5, a3, a4
+; RV32IA-NEXT:    lr.w.aq a2, (a3)
+; RV32IA-NEXT:    and a5, a2, a4
 ; RV32IA-NEXT:    bne a5, a1, .LBB1_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB1_1 Depth=1
-; RV32IA-NEXT:    xor a5, a3, a2
+; RV32IA-NEXT:    xor a5, a2, a0
 ; RV32IA-NEXT:    and a5, a5, a4
-; RV32IA-NEXT:    xor a5, a3, a5
-; RV32IA-NEXT:    sc.w a5, a5, (a0)
+; RV32IA-NEXT:    xor a5, a2, a5
+; RV32IA-NEXT:    sc.w a5, a5, (a3)
 ; RV32IA-NEXT:    bnez a5, .LBB1_1
 ; RV32IA-NEXT:  .LBB1_3:
 ; RV32IA-NEXT:    ret
@@ -139,24 +139,24 @@ define void @cmpxchg_i8_acquire_monotonic(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ;
 ; RV64IA-LABEL: cmpxchg_i8_acquire_monotonic:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
+; RV64IA-NEXT:    andi a3, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a4, zero, 255
-; RV64IA-NEXT:    sllw a4, a4, a3
-; RV64IA-NEXT:    andi a2, a2, 255
-; RV64IA-NEXT:    sllw a2, a2, a3
+; RV64IA-NEXT:    sllw a4, a4, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    andi a2, a2, 255
+; RV64IA-NEXT:    sllw a0, a2, a0
 ; RV64IA-NEXT:  .LBB1_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a3, (a0)
-; RV64IA-NEXT:    and a5, a3, a4
+; RV64IA-NEXT:    lr.w.aq a2, (a3)
+; RV64IA-NEXT:    and a5, a2, a4
 ; RV64IA-NEXT:    bne a5, a1, .LBB1_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB1_1 Depth=1
-; RV64IA-NEXT:    xor a5, a3, a2
+; RV64IA-NEXT:    xor a5, a2, a0
 ; RV64IA-NEXT:    and a5, a5, a4
-; RV64IA-NEXT:    xor a5, a3, a5
-; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    xor a5, a2, a5
+; RV64IA-NEXT:    sc.w a5, a5, (a3)
 ; RV64IA-NEXT:    bnez a5, .LBB1_1
 ; RV64IA-NEXT:  .LBB1_3:
 ; RV64IA-NEXT:    ret
@@ -180,24 +180,24 @@ define void @cmpxchg_i8_acquire_acquire(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ;
 ; RV32IA-LABEL: cmpxchg_i8_acquire_acquire:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
+; RV32IA-NEXT:    andi a3, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a4, zero, 255
-; RV32IA-NEXT:    sll a4, a4, a3
-; RV32IA-NEXT:    andi a2, a2, 255
-; RV32IA-NEXT:    sll a2, a2, a3
+; RV32IA-NEXT:    sll a4, a4, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    andi a2, a2, 255
+; RV32IA-NEXT:    sll a0, a2, a0
 ; RV32IA-NEXT:  .LBB2_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a3, (a0)
-; RV32IA-NEXT:    and a5, a3, a4
+; RV32IA-NEXT:    lr.w.aq a2, (a3)
+; RV32IA-NEXT:    and a5, a2, a4
 ; RV32IA-NEXT:    bne a5, a1, .LBB2_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB2_1 Depth=1
-; RV32IA-NEXT:    xor a5, a3, a2
+; RV32IA-NEXT:    xor a5, a2, a0
 ; RV32IA-NEXT:    and a5, a5, a4
-; RV32IA-NEXT:    xor a5, a3, a5
-; RV32IA-NEXT:    sc.w a5, a5, (a0)
+; RV32IA-NEXT:    xor a5, a2, a5
+; RV32IA-NEXT:    sc.w a5, a5, (a3)
 ; RV32IA-NEXT:    bnez a5, .LBB2_1
 ; RV32IA-NEXT:  .LBB2_3:
 ; RV32IA-NEXT:    ret
@@ -217,24 +217,24 @@ define void @cmpxchg_i8_acquire_acquire(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ;
 ; RV64IA-LABEL: cmpxchg_i8_acquire_acquire:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
+; RV64IA-NEXT:    andi a3, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a4, zero, 255
-; RV64IA-NEXT:    sllw a4, a4, a3
-; RV64IA-NEXT:    andi a2, a2, 255
-; RV64IA-NEXT:    sllw a2, a2, a3
+; RV64IA-NEXT:    sllw a4, a4, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    andi a2, a2, 255
+; RV64IA-NEXT:    sllw a0, a2, a0
 ; RV64IA-NEXT:  .LBB2_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a3, (a0)
-; RV64IA-NEXT:    and a5, a3, a4
+; RV64IA-NEXT:    lr.w.aq a2, (a3)
+; RV64IA-NEXT:    and a5, a2, a4
 ; RV64IA-NEXT:    bne a5, a1, .LBB2_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB2_1 Depth=1
-; RV64IA-NEXT:    xor a5, a3, a2
+; RV64IA-NEXT:    xor a5, a2, a0
 ; RV64IA-NEXT:    and a5, a5, a4
-; RV64IA-NEXT:    xor a5, a3, a5
-; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    xor a5, a2, a5
+; RV64IA-NEXT:    sc.w a5, a5, (a3)
 ; RV64IA-NEXT:    bnez a5, .LBB2_1
 ; RV64IA-NEXT:  .LBB2_3:
 ; RV64IA-NEXT:    ret
@@ -258,24 +258,24 @@ define void @cmpxchg_i8_release_monotonic(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ;
 ; RV32IA-LABEL: cmpxchg_i8_release_monotonic:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
+; RV32IA-NEXT:    andi a3, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a4, zero, 255
-; RV32IA-NEXT:    sll a4, a4, a3
-; RV32IA-NEXT:    andi a2, a2, 255
-; RV32IA-NEXT:    sll a2, a2, a3
+; RV32IA-NEXT:    sll a4, a4, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    andi a2, a2, 255
+; RV32IA-NEXT:    sll a0, a2, a0
 ; RV32IA-NEXT:  .LBB3_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a3, (a0)
-; RV32IA-NEXT:    and a5, a3, a4
+; RV32IA-NEXT:    lr.w a2, (a3)
+; RV32IA-NEXT:    and a5, a2, a4
 ; RV32IA-NEXT:    bne a5, a1, .LBB3_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB3_1 Depth=1
-; RV32IA-NEXT:    xor a5, a3, a2
+; RV32IA-NEXT:    xor a5, a2, a0
 ; RV32IA-NEXT:    and a5, a5, a4
-; RV32IA-NEXT:    xor a5, a3, a5
-; RV32IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV32IA-NEXT:    xor a5, a2, a5
+; RV32IA-NEXT:    sc.w.rl a5, a5, (a3)
 ; RV32IA-NEXT:    bnez a5, .LBB3_1
 ; RV32IA-NEXT:  .LBB3_3:
 ; RV32IA-NEXT:    ret
@@ -295,24 +295,24 @@ define void @cmpxchg_i8_release_monotonic(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ;
 ; RV64IA-LABEL: cmpxchg_i8_release_monotonic:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
+; RV64IA-NEXT:    andi a3, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a4, zero, 255
-; RV64IA-NEXT:    sllw a4, a4, a3
-; RV64IA-NEXT:    andi a2, a2, 255
-; RV64IA-NEXT:    sllw a2, a2, a3
+; RV64IA-NEXT:    sllw a4, a4, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    andi a2, a2, 255
+; RV64IA-NEXT:    sllw a0, a2, a0
 ; RV64IA-NEXT:  .LBB3_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a3, (a0)
-; RV64IA-NEXT:    and a5, a3, a4
+; RV64IA-NEXT:    lr.w a2, (a3)
+; RV64IA-NEXT:    and a5, a2, a4
 ; RV64IA-NEXT:    bne a5, a1, .LBB3_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB3_1 Depth=1
-; RV64IA-NEXT:    xor a5, a3, a2
+; RV64IA-NEXT:    xor a5, a2, a0
 ; RV64IA-NEXT:    and a5, a5, a4
-; RV64IA-NEXT:    xor a5, a3, a5
-; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    xor a5, a2, a5
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a3)
 ; RV64IA-NEXT:    bnez a5, .LBB3_1
 ; RV64IA-NEXT:  .LBB3_3:
 ; RV64IA-NEXT:    ret
@@ -336,24 +336,24 @@ define void @cmpxchg_i8_release_acquire(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ;
 ; RV32IA-LABEL: cmpxchg_i8_release_acquire:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
+; RV32IA-NEXT:    andi a3, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a4, zero, 255
-; RV32IA-NEXT:    sll a4, a4, a3
-; RV32IA-NEXT:    andi a2, a2, 255
-; RV32IA-NEXT:    sll a2, a2, a3
+; RV32IA-NEXT:    sll a4, a4, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    andi a2, a2, 255
+; RV32IA-NEXT:    sll a0, a2, a0
 ; RV32IA-NEXT:  .LBB4_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a3, (a0)
-; RV32IA-NEXT:    and a5, a3, a4
+; RV32IA-NEXT:    lr.w a2, (a3)
+; RV32IA-NEXT:    and a5, a2, a4
 ; RV32IA-NEXT:    bne a5, a1, .LBB4_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB4_1 Depth=1
-; RV32IA-NEXT:    xor a5, a3, a2
+; RV32IA-NEXT:    xor a5, a2, a0
 ; RV32IA-NEXT:    and a5, a5, a4
-; RV32IA-NEXT:    xor a5, a3, a5
-; RV32IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV32IA-NEXT:    xor a5, a2, a5
+; RV32IA-NEXT:    sc.w.rl a5, a5, (a3)
 ; RV32IA-NEXT:    bnez a5, .LBB4_1
 ; RV32IA-NEXT:  .LBB4_3:
 ; RV32IA-NEXT:    ret
@@ -373,24 +373,24 @@ define void @cmpxchg_i8_release_acquire(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ;
 ; RV64IA-LABEL: cmpxchg_i8_release_acquire:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
+; RV64IA-NEXT:    andi a3, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a4, zero, 255
-; RV64IA-NEXT:    sllw a4, a4, a3
-; RV64IA-NEXT:    andi a2, a2, 255
-; RV64IA-NEXT:    sllw a2, a2, a3
+; RV64IA-NEXT:    sllw a4, a4, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    andi a2, a2, 255
+; RV64IA-NEXT:    sllw a0, a2, a0
 ; RV64IA-NEXT:  .LBB4_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a3, (a0)
-; RV64IA-NEXT:    and a5, a3, a4
+; RV64IA-NEXT:    lr.w a2, (a3)
+; RV64IA-NEXT:    and a5, a2, a4
 ; RV64IA-NEXT:    bne a5, a1, .LBB4_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB4_1 Depth=1
-; RV64IA-NEXT:    xor a5, a3, a2
+; RV64IA-NEXT:    xor a5, a2, a0
 ; RV64IA-NEXT:    and a5, a5, a4
-; RV64IA-NEXT:    xor a5, a3, a5
-; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    xor a5, a2, a5
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a3)
 ; RV64IA-NEXT:    bnez a5, .LBB4_1
 ; RV64IA-NEXT:  .LBB4_3:
 ; RV64IA-NEXT:    ret
@@ -414,24 +414,24 @@ define void @cmpxchg_i8_acq_rel_monotonic(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ;
 ; RV32IA-LABEL: cmpxchg_i8_acq_rel_monotonic:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
+; RV32IA-NEXT:    andi a3, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a4, zero, 255
-; RV32IA-NEXT:    sll a4, a4, a3
-; RV32IA-NEXT:    andi a2, a2, 255
-; RV32IA-NEXT:    sll a2, a2, a3
+; RV32IA-NEXT:    sll a4, a4, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    andi a2, a2, 255
+; RV32IA-NEXT:    sll a0, a2, a0
 ; RV32IA-NEXT:  .LBB5_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a3, (a0)
-; RV32IA-NEXT:    and a5, a3, a4
+; RV32IA-NEXT:    lr.w.aq a2, (a3)
+; RV32IA-NEXT:    and a5, a2, a4
 ; RV32IA-NEXT:    bne a5, a1, .LBB5_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB5_1 Depth=1
-; RV32IA-NEXT:    xor a5, a3, a2
+; RV32IA-NEXT:    xor a5, a2, a0
 ; RV32IA-NEXT:    and a5, a5, a4
-; RV32IA-NEXT:    xor a5, a3, a5
-; RV32IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV32IA-NEXT:    xor a5, a2, a5
+; RV32IA-NEXT:    sc.w.rl a5, a5, (a3)
 ; RV32IA-NEXT:    bnez a5, .LBB5_1
 ; RV32IA-NEXT:  .LBB5_3:
 ; RV32IA-NEXT:    ret
@@ -451,24 +451,24 @@ define void @cmpxchg_i8_acq_rel_monotonic(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ;
 ; RV64IA-LABEL: cmpxchg_i8_acq_rel_monotonic:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
+; RV64IA-NEXT:    andi a3, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a4, zero, 255
-; RV64IA-NEXT:    sllw a4, a4, a3
-; RV64IA-NEXT:    andi a2, a2, 255
-; RV64IA-NEXT:    sllw a2, a2, a3
+; RV64IA-NEXT:    sllw a4, a4, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    andi a2, a2, 255
+; RV64IA-NEXT:    sllw a0, a2, a0
 ; RV64IA-NEXT:  .LBB5_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a3, (a0)
-; RV64IA-NEXT:    and a5, a3, a4
+; RV64IA-NEXT:    lr.w.aq a2, (a3)
+; RV64IA-NEXT:    and a5, a2, a4
 ; RV64IA-NEXT:    bne a5, a1, .LBB5_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB5_1 Depth=1
-; RV64IA-NEXT:    xor a5, a3, a2
+; RV64IA-NEXT:    xor a5, a2, a0
 ; RV64IA-NEXT:    and a5, a5, a4
-; RV64IA-NEXT:    xor a5, a3, a5
-; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    xor a5, a2, a5
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a3)
 ; RV64IA-NEXT:    bnez a5, .LBB5_1
 ; RV64IA-NEXT:  .LBB5_3:
 ; RV64IA-NEXT:    ret
@@ -492,24 +492,24 @@ define void @cmpxchg_i8_acq_rel_acquire(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ;
 ; RV32IA-LABEL: cmpxchg_i8_acq_rel_acquire:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
+; RV32IA-NEXT:    andi a3, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a4, zero, 255
-; RV32IA-NEXT:    sll a4, a4, a3
-; RV32IA-NEXT:    andi a2, a2, 255
-; RV32IA-NEXT:    sll a2, a2, a3
+; RV32IA-NEXT:    sll a4, a4, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    andi a2, a2, 255
+; RV32IA-NEXT:    sll a0, a2, a0
 ; RV32IA-NEXT:  .LBB6_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a3, (a0)
-; RV32IA-NEXT:    and a5, a3, a4
+; RV32IA-NEXT:    lr.w.aq a2, (a3)
+; RV32IA-NEXT:    and a5, a2, a4
 ; RV32IA-NEXT:    bne a5, a1, .LBB6_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB6_1 Depth=1
-; RV32IA-NEXT:    xor a5, a3, a2
+; RV32IA-NEXT:    xor a5, a2, a0
 ; RV32IA-NEXT:    and a5, a5, a4
-; RV32IA-NEXT:    xor a5, a3, a5
-; RV32IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV32IA-NEXT:    xor a5, a2, a5
+; RV32IA-NEXT:    sc.w.rl a5, a5, (a3)
 ; RV32IA-NEXT:    bnez a5, .LBB6_1
 ; RV32IA-NEXT:  .LBB6_3:
 ; RV32IA-NEXT:    ret
@@ -529,24 +529,24 @@ define void @cmpxchg_i8_acq_rel_acquire(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ;
 ; RV64IA-LABEL: cmpxchg_i8_acq_rel_acquire:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
+; RV64IA-NEXT:    andi a3, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a4, zero, 255
-; RV64IA-NEXT:    sllw a4, a4, a3
-; RV64IA-NEXT:    andi a2, a2, 255
-; RV64IA-NEXT:    sllw a2, a2, a3
+; RV64IA-NEXT:    sllw a4, a4, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    andi a2, a2, 255
+; RV64IA-NEXT:    sllw a0, a2, a0
 ; RV64IA-NEXT:  .LBB6_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a3, (a0)
-; RV64IA-NEXT:    and a5, a3, a4
+; RV64IA-NEXT:    lr.w.aq a2, (a3)
+; RV64IA-NEXT:    and a5, a2, a4
 ; RV64IA-NEXT:    bne a5, a1, .LBB6_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB6_1 Depth=1
-; RV64IA-NEXT:    xor a5, a3, a2
+; RV64IA-NEXT:    xor a5, a2, a0
 ; RV64IA-NEXT:    and a5, a5, a4
-; RV64IA-NEXT:    xor a5, a3, a5
-; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    xor a5, a2, a5
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a3)
 ; RV64IA-NEXT:    bnez a5, .LBB6_1
 ; RV64IA-NEXT:  .LBB6_3:
 ; RV64IA-NEXT:    ret
@@ -570,24 +570,24 @@ define void @cmpxchg_i8_seq_cst_monotonic(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ;
 ; RV32IA-LABEL: cmpxchg_i8_seq_cst_monotonic:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
+; RV32IA-NEXT:    andi a3, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a4, zero, 255
-; RV32IA-NEXT:    sll a4, a4, a3
-; RV32IA-NEXT:    andi a2, a2, 255
-; RV32IA-NEXT:    sll a2, a2, a3
+; RV32IA-NEXT:    sll a4, a4, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    andi a2, a2, 255
+; RV32IA-NEXT:    sll a0, a2, a0
 ; RV32IA-NEXT:  .LBB7_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aqrl a3, (a0)
-; RV32IA-NEXT:    and a5, a3, a4
+; RV32IA-NEXT:    lr.w.aqrl a2, (a3)
+; RV32IA-NEXT:    and a5, a2, a4
 ; RV32IA-NEXT:    bne a5, a1, .LBB7_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB7_1 Depth=1
-; RV32IA-NEXT:    xor a5, a3, a2
+; RV32IA-NEXT:    xor a5, a2, a0
 ; RV32IA-NEXT:    and a5, a5, a4
-; RV32IA-NEXT:    xor a5, a3, a5
-; RV32IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV32IA-NEXT:    xor a5, a2, a5
+; RV32IA-NEXT:    sc.w.aqrl a5, a5, (a3)
 ; RV32IA-NEXT:    bnez a5, .LBB7_1
 ; RV32IA-NEXT:  .LBB7_3:
 ; RV32IA-NEXT:    ret
@@ -607,24 +607,24 @@ define void @cmpxchg_i8_seq_cst_monotonic(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ;
 ; RV64IA-LABEL: cmpxchg_i8_seq_cst_monotonic:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
+; RV64IA-NEXT:    andi a3, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a4, zero, 255
-; RV64IA-NEXT:    sllw a4, a4, a3
-; RV64IA-NEXT:    andi a2, a2, 255
-; RV64IA-NEXT:    sllw a2, a2, a3
+; RV64IA-NEXT:    sllw a4, a4, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    andi a2, a2, 255
+; RV64IA-NEXT:    sllw a0, a2, a0
 ; RV64IA-NEXT:  .LBB7_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aqrl a3, (a0)
-; RV64IA-NEXT:    and a5, a3, a4
+; RV64IA-NEXT:    lr.w.aqrl a2, (a3)
+; RV64IA-NEXT:    and a5, a2, a4
 ; RV64IA-NEXT:    bne a5, a1, .LBB7_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB7_1 Depth=1
-; RV64IA-NEXT:    xor a5, a3, a2
+; RV64IA-NEXT:    xor a5, a2, a0
 ; RV64IA-NEXT:    and a5, a5, a4
-; RV64IA-NEXT:    xor a5, a3, a5
-; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV64IA-NEXT:    xor a5, a2, a5
+; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a3)
 ; RV64IA-NEXT:    bnez a5, .LBB7_1
 ; RV64IA-NEXT:  .LBB7_3:
 ; RV64IA-NEXT:    ret
@@ -648,24 +648,24 @@ define void @cmpxchg_i8_seq_cst_acquire(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ;
 ; RV32IA-LABEL: cmpxchg_i8_seq_cst_acquire:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
+; RV32IA-NEXT:    andi a3, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a4, zero, 255
-; RV32IA-NEXT:    sll a4, a4, a3
-; RV32IA-NEXT:    andi a2, a2, 255
-; RV32IA-NEXT:    sll a2, a2, a3
+; RV32IA-NEXT:    sll a4, a4, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    andi a2, a2, 255
+; RV32IA-NEXT:    sll a0, a2, a0
 ; RV32IA-NEXT:  .LBB8_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aqrl a3, (a0)
-; RV32IA-NEXT:    and a5, a3, a4
+; RV32IA-NEXT:    lr.w.aqrl a2, (a3)
+; RV32IA-NEXT:    and a5, a2, a4
 ; RV32IA-NEXT:    bne a5, a1, .LBB8_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB8_1 Depth=1
-; RV32IA-NEXT:    xor a5, a3, a2
+; RV32IA-NEXT:    xor a5, a2, a0
 ; RV32IA-NEXT:    and a5, a5, a4
-; RV32IA-NEXT:    xor a5, a3, a5
-; RV32IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV32IA-NEXT:    xor a5, a2, a5
+; RV32IA-NEXT:    sc.w.aqrl a5, a5, (a3)
 ; RV32IA-NEXT:    bnez a5, .LBB8_1
 ; RV32IA-NEXT:  .LBB8_3:
 ; RV32IA-NEXT:    ret
@@ -685,24 +685,24 @@ define void @cmpxchg_i8_seq_cst_acquire(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ;
 ; RV64IA-LABEL: cmpxchg_i8_seq_cst_acquire:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
+; RV64IA-NEXT:    andi a3, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a4, zero, 255
-; RV64IA-NEXT:    sllw a4, a4, a3
-; RV64IA-NEXT:    andi a2, a2, 255
-; RV64IA-NEXT:    sllw a2, a2, a3
+; RV64IA-NEXT:    sllw a4, a4, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    andi a2, a2, 255
+; RV64IA-NEXT:    sllw a0, a2, a0
 ; RV64IA-NEXT:  .LBB8_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aqrl a3, (a0)
-; RV64IA-NEXT:    and a5, a3, a4
+; RV64IA-NEXT:    lr.w.aqrl a2, (a3)
+; RV64IA-NEXT:    and a5, a2, a4
 ; RV64IA-NEXT:    bne a5, a1, .LBB8_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB8_1 Depth=1
-; RV64IA-NEXT:    xor a5, a3, a2
+; RV64IA-NEXT:    xor a5, a2, a0
 ; RV64IA-NEXT:    and a5, a5, a4
-; RV64IA-NEXT:    xor a5, a3, a5
-; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV64IA-NEXT:    xor a5, a2, a5
+; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a3)
 ; RV64IA-NEXT:    bnez a5, .LBB8_1
 ; RV64IA-NEXT:  .LBB8_3:
 ; RV64IA-NEXT:    ret
@@ -726,24 +726,24 @@ define void @cmpxchg_i8_seq_cst_seq_cst(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ;
 ; RV32IA-LABEL: cmpxchg_i8_seq_cst_seq_cst:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
+; RV32IA-NEXT:    andi a3, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a4, zero, 255
-; RV32IA-NEXT:    sll a4, a4, a3
-; RV32IA-NEXT:    andi a2, a2, 255
-; RV32IA-NEXT:    sll a2, a2, a3
+; RV32IA-NEXT:    sll a4, a4, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    andi a2, a2, 255
+; RV32IA-NEXT:    sll a0, a2, a0
 ; RV32IA-NEXT:  .LBB9_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aqrl a3, (a0)
-; RV32IA-NEXT:    and a5, a3, a4
+; RV32IA-NEXT:    lr.w.aqrl a2, (a3)
+; RV32IA-NEXT:    and a5, a2, a4
 ; RV32IA-NEXT:    bne a5, a1, .LBB9_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB9_1 Depth=1
-; RV32IA-NEXT:    xor a5, a3, a2
+; RV32IA-NEXT:    xor a5, a2, a0
 ; RV32IA-NEXT:    and a5, a5, a4
-; RV32IA-NEXT:    xor a5, a3, a5
-; RV32IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV32IA-NEXT:    xor a5, a2, a5
+; RV32IA-NEXT:    sc.w.aqrl a5, a5, (a3)
 ; RV32IA-NEXT:    bnez a5, .LBB9_1
 ; RV32IA-NEXT:  .LBB9_3:
 ; RV32IA-NEXT:    ret
@@ -763,24 +763,24 @@ define void @cmpxchg_i8_seq_cst_seq_cst(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ;
 ; RV64IA-LABEL: cmpxchg_i8_seq_cst_seq_cst:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
+; RV64IA-NEXT:    andi a3, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a4, zero, 255
-; RV64IA-NEXT:    sllw a4, a4, a3
-; RV64IA-NEXT:    andi a2, a2, 255
-; RV64IA-NEXT:    sllw a2, a2, a3
+; RV64IA-NEXT:    sllw a4, a4, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    andi a2, a2, 255
+; RV64IA-NEXT:    sllw a0, a2, a0
 ; RV64IA-NEXT:  .LBB9_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aqrl a3, (a0)
-; RV64IA-NEXT:    and a5, a3, a4
+; RV64IA-NEXT:    lr.w.aqrl a2, (a3)
+; RV64IA-NEXT:    and a5, a2, a4
 ; RV64IA-NEXT:    bne a5, a1, .LBB9_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB9_1 Depth=1
-; RV64IA-NEXT:    xor a5, a3, a2
+; RV64IA-NEXT:    xor a5, a2, a0
 ; RV64IA-NEXT:    and a5, a5, a4
-; RV64IA-NEXT:    xor a5, a3, a5
-; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV64IA-NEXT:    xor a5, a2, a5
+; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a3)
 ; RV64IA-NEXT:    bnez a5, .LBB9_1
 ; RV64IA-NEXT:  .LBB9_3:
 ; RV64IA-NEXT:    ret
@@ -804,26 +804,26 @@ define void @cmpxchg_i16_monotonic_monotonic(i16* %ptr, i16 %cmp, i16 %val) noun
 ;
 ; RV32IA-LABEL: cmpxchg_i16_monotonic_monotonic:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a3, 16
-; RV32IA-NEXT:    addi a3, a3, -1
-; RV32IA-NEXT:    and a1, a1, a3
-; RV32IA-NEXT:    and a2, a2, a3
-; RV32IA-NEXT:    slli a4, a0, 3
-; RV32IA-NEXT:    andi a4, a4, 24
-; RV32IA-NEXT:    sll a3, a3, a4
-; RV32IA-NEXT:    sll a2, a2, a4
-; RV32IA-NEXT:    sll a1, a1, a4
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a3, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a4, 16
+; RV32IA-NEXT:    addi a4, a4, -1
+; RV32IA-NEXT:    sll a5, a4, a0
+; RV32IA-NEXT:    and a1, a1, a4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    and a2, a2, a4
+; RV32IA-NEXT:    sll a0, a2, a0
 ; RV32IA-NEXT:  .LBB10_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a4, (a0)
-; RV32IA-NEXT:    and a5, a4, a3
-; RV32IA-NEXT:    bne a5, a1, .LBB10_3
+; RV32IA-NEXT:    lr.w a2, (a3)
+; RV32IA-NEXT:    and a4, a2, a5
+; RV32IA-NEXT:    bne a4, a1, .LBB10_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB10_1 Depth=1
-; RV32IA-NEXT:    xor a5, a4, a2
-; RV32IA-NEXT:    and a5, a5, a3
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w a5, a5, (a0)
-; RV32IA-NEXT:    bnez a5, .LBB10_1
+; RV32IA-NEXT:    xor a4, a2, a0
+; RV32IA-NEXT:    and a4, a4, a5
+; RV32IA-NEXT:    xor a4, a2, a4
+; RV32IA-NEXT:    sc.w a4, a4, (a3)
+; RV32IA-NEXT:    bnez a4, .LBB10_1
 ; RV32IA-NEXT:  .LBB10_3:
 ; RV32IA-NEXT:    ret
 ;
@@ -842,26 +842,26 @@ define void @cmpxchg_i16_monotonic_monotonic(i16* %ptr, i16 %cmp, i16 %val) noun
 ;
 ; RV64IA-LABEL: cmpxchg_i16_monotonic_monotonic:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a3, 16
-; RV64IA-NEXT:    addiw a3, a3, -1
-; RV64IA-NEXT:    and a1, a1, a3
-; RV64IA-NEXT:    and a2, a2, a3
-; RV64IA-NEXT:    slli a4, a0, 3
-; RV64IA-NEXT:    andi a4, a4, 24
-; RV64IA-NEXT:    sllw a3, a3, a4
-; RV64IA-NEXT:    sllw a2, a2, a4
-; RV64IA-NEXT:    sllw a1, a1, a4
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a3, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a4, 16
+; RV64IA-NEXT:    addiw a4, a4, -1
+; RV64IA-NEXT:    sllw a5, a4, a0
+; RV64IA-NEXT:    and a1, a1, a4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    and a2, a2, a4
+; RV64IA-NEXT:    sllw a0, a2, a0
 ; RV64IA-NEXT:  .LBB10_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a4, (a0)
-; RV64IA-NEXT:    and a5, a4, a3
-; RV64IA-NEXT:    bne a5, a1, .LBB10_3
+; RV64IA-NEXT:    lr.w a2, (a3)
+; RV64IA-NEXT:    and a4, a2, a5
+; RV64IA-NEXT:    bne a4, a1, .LBB10_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB10_1 Depth=1
-; RV64IA-NEXT:    xor a5, a4, a2
-; RV64IA-NEXT:    and a5, a5, a3
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w a5, a5, (a0)
-; RV64IA-NEXT:    bnez a5, .LBB10_1
+; RV64IA-NEXT:    xor a4, a2, a0
+; RV64IA-NEXT:    and a4, a4, a5
+; RV64IA-NEXT:    xor a4, a2, a4
+; RV64IA-NEXT:    sc.w a4, a4, (a3)
+; RV64IA-NEXT:    bnez a4, .LBB10_1
 ; RV64IA-NEXT:  .LBB10_3:
 ; RV64IA-NEXT:    ret
   %res = cmpxchg i16* %ptr, i16 %cmp, i16 %val monotonic monotonic
@@ -884,26 +884,26 @@ define void @cmpxchg_i16_acquire_monotonic(i16* %ptr, i16 %cmp, i16 %val) nounwi
 ;
 ; RV32IA-LABEL: cmpxchg_i16_acquire_monotonic:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a3, 16
-; RV32IA-NEXT:    addi a3, a3, -1
-; RV32IA-NEXT:    and a1, a1, a3
-; RV32IA-NEXT:    and a2, a2, a3
-; RV32IA-NEXT:    slli a4, a0, 3
-; RV32IA-NEXT:    andi a4, a4, 24
-; RV32IA-NEXT:    sll a3, a3, a4
-; RV32IA-NEXT:    sll a2, a2, a4
-; RV32IA-NEXT:    sll a1, a1, a4
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a3, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a4, 16
+; RV32IA-NEXT:    addi a4, a4, -1
+; RV32IA-NEXT:    sll a5, a4, a0
+; RV32IA-NEXT:    and a1, a1, a4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    and a2, a2, a4
+; RV32IA-NEXT:    sll a0, a2, a0
 ; RV32IA-NEXT:  .LBB11_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a4, (a0)
-; RV32IA-NEXT:    and a5, a4, a3
-; RV32IA-NEXT:    bne a5, a1, .LBB11_3
+; RV32IA-NEXT:    lr.w.aq a2, (a3)
+; RV32IA-NEXT:    and a4, a2, a5
+; RV32IA-NEXT:    bne a4, a1, .LBB11_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB11_1 Depth=1
-; RV32IA-NEXT:    xor a5, a4, a2
-; RV32IA-NEXT:    and a5, a5, a3
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w a5, a5, (a0)
-; RV32IA-NEXT:    bnez a5, .LBB11_1
+; RV32IA-NEXT:    xor a4, a2, a0
+; RV32IA-NEXT:    and a4, a4, a5
+; RV32IA-NEXT:    xor a4, a2, a4
+; RV32IA-NEXT:    sc.w a4, a4, (a3)
+; RV32IA-NEXT:    bnez a4, .LBB11_1
 ; RV32IA-NEXT:  .LBB11_3:
 ; RV32IA-NEXT:    ret
 ;
@@ -922,26 +922,26 @@ define void @cmpxchg_i16_acquire_monotonic(i16* %ptr, i16 %cmp, i16 %val) nounwi
 ;
 ; RV64IA-LABEL: cmpxchg_i16_acquire_monotonic:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a3, 16
-; RV64IA-NEXT:    addiw a3, a3, -1
-; RV64IA-NEXT:    and a1, a1, a3
-; RV64IA-NEXT:    and a2, a2, a3
-; RV64IA-NEXT:    slli a4, a0, 3
-; RV64IA-NEXT:    andi a4, a4, 24
-; RV64IA-NEXT:    sllw a3, a3, a4
-; RV64IA-NEXT:    sllw a2, a2, a4
-; RV64IA-NEXT:    sllw a1, a1, a4
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a3, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a4, 16
+; RV64IA-NEXT:    addiw a4, a4, -1
+; RV64IA-NEXT:    sllw a5, a4, a0
+; RV64IA-NEXT:    and a1, a1, a4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    and a2, a2, a4
+; RV64IA-NEXT:    sllw a0, a2, a0
 ; RV64IA-NEXT:  .LBB11_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a4, (a0)
-; RV64IA-NEXT:    and a5, a4, a3
-; RV64IA-NEXT:    bne a5, a1, .LBB11_3
+; RV64IA-NEXT:    lr.w.aq a2, (a3)
+; RV64IA-NEXT:    and a4, a2, a5
+; RV64IA-NEXT:    bne a4, a1, .LBB11_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB11_1 Depth=1
-; RV64IA-NEXT:    xor a5, a4, a2
-; RV64IA-NEXT:    and a5, a5, a3
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w a5, a5, (a0)
-; RV64IA-NEXT:    bnez a5, .LBB11_1
+; RV64IA-NEXT:    xor a4, a2, a0
+; RV64IA-NEXT:    and a4, a4, a5
+; RV64IA-NEXT:    xor a4, a2, a4
+; RV64IA-NEXT:    sc.w a4, a4, (a3)
+; RV64IA-NEXT:    bnez a4, .LBB11_1
 ; RV64IA-NEXT:  .LBB11_3:
 ; RV64IA-NEXT:    ret
   %res = cmpxchg i16* %ptr, i16 %cmp, i16 %val acquire monotonic
@@ -964,26 +964,26 @@ define void @cmpxchg_i16_acquire_acquire(i16* %ptr, i16 %cmp, i16 %val) nounwind
 ;
 ; RV32IA-LABEL: cmpxchg_i16_acquire_acquire:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a3, 16
-; RV32IA-NEXT:    addi a3, a3, -1
-; RV32IA-NEXT:    and a1, a1, a3
-; RV32IA-NEXT:    and a2, a2, a3
-; RV32IA-NEXT:    slli a4, a0, 3
-; RV32IA-NEXT:    andi a4, a4, 24
-; RV32IA-NEXT:    sll a3, a3, a4
-; RV32IA-NEXT:    sll a2, a2, a4
-; RV32IA-NEXT:    sll a1, a1, a4
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a3, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a4, 16
+; RV32IA-NEXT:    addi a4, a4, -1
+; RV32IA-NEXT:    sll a5, a4, a0
+; RV32IA-NEXT:    and a1, a1, a4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    and a2, a2, a4
+; RV32IA-NEXT:    sll a0, a2, a0
 ; RV32IA-NEXT:  .LBB12_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a4, (a0)
-; RV32IA-NEXT:    and a5, a4, a3
-; RV32IA-NEXT:    bne a5, a1, .LBB12_3
+; RV32IA-NEXT:    lr.w.aq a2, (a3)
+; RV32IA-NEXT:    and a4, a2, a5
+; RV32IA-NEXT:    bne a4, a1, .LBB12_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB12_1 Depth=1
-; RV32IA-NEXT:    xor a5, a4, a2
-; RV32IA-NEXT:    and a5, a5, a3
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w a5, a5, (a0)
-; RV32IA-NEXT:    bnez a5, .LBB12_1
+; RV32IA-NEXT:    xor a4, a2, a0
+; RV32IA-NEXT:    and a4, a4, a5
+; RV32IA-NEXT:    xor a4, a2, a4
+; RV32IA-NEXT:    sc.w a4, a4, (a3)
+; RV32IA-NEXT:    bnez a4, .LBB12_1
 ; RV32IA-NEXT:  .LBB12_3:
 ; RV32IA-NEXT:    ret
 ;
@@ -1002,26 +1002,26 @@ define void @cmpxchg_i16_acquire_acquire(i16* %ptr, i16 %cmp, i16 %val) nounwind
 ;
 ; RV64IA-LABEL: cmpxchg_i16_acquire_acquire:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a3, 16
-; RV64IA-NEXT:    addiw a3, a3, -1
-; RV64IA-NEXT:    and a1, a1, a3
-; RV64IA-NEXT:    and a2, a2, a3
-; RV64IA-NEXT:    slli a4, a0, 3
-; RV64IA-NEXT:    andi a4, a4, 24
-; RV64IA-NEXT:    sllw a3, a3, a4
-; RV64IA-NEXT:    sllw a2, a2, a4
-; RV64IA-NEXT:    sllw a1, a1, a4
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a3, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a4, 16
+; RV64IA-NEXT:    addiw a4, a4, -1
+; RV64IA-NEXT:    sllw a5, a4, a0
+; RV64IA-NEXT:    and a1, a1, a4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    and a2, a2, a4
+; RV64IA-NEXT:    sllw a0, a2, a0
 ; RV64IA-NEXT:  .LBB12_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a4, (a0)
-; RV64IA-NEXT:    and a5, a4, a3
-; RV64IA-NEXT:    bne a5, a1, .LBB12_3
+; RV64IA-NEXT:    lr.w.aq a2, (a3)
+; RV64IA-NEXT:    and a4, a2, a5
+; RV64IA-NEXT:    bne a4, a1, .LBB12_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB12_1 Depth=1
-; RV64IA-NEXT:    xor a5, a4, a2
-; RV64IA-NEXT:    and a5, a5, a3
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w a5, a5, (a0)
-; RV64IA-NEXT:    bnez a5, .LBB12_1
+; RV64IA-NEXT:    xor a4, a2, a0
+; RV64IA-NEXT:    and a4, a4, a5
+; RV64IA-NEXT:    xor a4, a2, a4
+; RV64IA-NEXT:    sc.w a4, a4, (a3)
+; RV64IA-NEXT:    bnez a4, .LBB12_1
 ; RV64IA-NEXT:  .LBB12_3:
 ; RV64IA-NEXT:    ret
   %res = cmpxchg i16* %ptr, i16 %cmp, i16 %val acquire acquire
@@ -1044,26 +1044,26 @@ define void @cmpxchg_i16_release_monotonic(i16* %ptr, i16 %cmp, i16 %val) nounwi
 ;
 ; RV32IA-LABEL: cmpxchg_i16_release_monotonic:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a3, 16
-; RV32IA-NEXT:    addi a3, a3, -1
-; RV32IA-NEXT:    and a1, a1, a3
-; RV32IA-NEXT:    and a2, a2, a3
-; RV32IA-NEXT:    slli a4, a0, 3
-; RV32IA-NEXT:    andi a4, a4, 24
-; RV32IA-NEXT:    sll a3, a3, a4
-; RV32IA-NEXT:    sll a2, a2, a4
-; RV32IA-NEXT:    sll a1, a1, a4
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a3, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a4, 16
+; RV32IA-NEXT:    addi a4, a4, -1
+; RV32IA-NEXT:    sll a5, a4, a0
+; RV32IA-NEXT:    and a1, a1, a4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    and a2, a2, a4
+; RV32IA-NEXT:    sll a0, a2, a0
 ; RV32IA-NEXT:  .LBB13_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a4, (a0)
-; RV32IA-NEXT:    and a5, a4, a3
-; RV32IA-NEXT:    bne a5, a1, .LBB13_3
+; RV32IA-NEXT:    lr.w a2, (a3)
+; RV32IA-NEXT:    and a4, a2, a5
+; RV32IA-NEXT:    bne a4, a1, .LBB13_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB13_1 Depth=1
-; RV32IA-NEXT:    xor a5, a4, a2
-; RV32IA-NEXT:    and a5, a5, a3
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w.rl a5, a5, (a0)
-; RV32IA-NEXT:    bnez a5, .LBB13_1
+; RV32IA-NEXT:    xor a4, a2, a0
+; RV32IA-NEXT:    and a4, a4, a5
+; RV32IA-NEXT:    xor a4, a2, a4
+; RV32IA-NEXT:    sc.w.rl a4, a4, (a3)
+; RV32IA-NEXT:    bnez a4, .LBB13_1
 ; RV32IA-NEXT:  .LBB13_3:
 ; RV32IA-NEXT:    ret
 ;
@@ -1082,26 +1082,26 @@ define void @cmpxchg_i16_release_monotonic(i16* %ptr, i16 %cmp, i16 %val) nounwi
 ;
 ; RV64IA-LABEL: cmpxchg_i16_release_monotonic:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a3, 16
-; RV64IA-NEXT:    addiw a3, a3, -1
-; RV64IA-NEXT:    and a1, a1, a3
-; RV64IA-NEXT:    and a2, a2, a3
-; RV64IA-NEXT:    slli a4, a0, 3
-; RV64IA-NEXT:    andi a4, a4, 24
-; RV64IA-NEXT:    sllw a3, a3, a4
-; RV64IA-NEXT:    sllw a2, a2, a4
-; RV64IA-NEXT:    sllw a1, a1, a4
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a3, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a4, 16
+; RV64IA-NEXT:    addiw a4, a4, -1
+; RV64IA-NEXT:    sllw a5, a4, a0
+; RV64IA-NEXT:    and a1, a1, a4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    and a2, a2, a4
+; RV64IA-NEXT:    sllw a0, a2, a0
 ; RV64IA-NEXT:  .LBB13_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a4, (a0)
-; RV64IA-NEXT:    and a5, a4, a3
-; RV64IA-NEXT:    bne a5, a1, .LBB13_3
+; RV64IA-NEXT:    lr.w a2, (a3)
+; RV64IA-NEXT:    and a4, a2, a5
+; RV64IA-NEXT:    bne a4, a1, .LBB13_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB13_1 Depth=1
-; RV64IA-NEXT:    xor a5, a4, a2
-; RV64IA-NEXT:    and a5, a5, a3
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
-; RV64IA-NEXT:    bnez a5, .LBB13_1
+; RV64IA-NEXT:    xor a4, a2, a0
+; RV64IA-NEXT:    and a4, a4, a5
+; RV64IA-NEXT:    xor a4, a2, a4
+; RV64IA-NEXT:    sc.w.rl a4, a4, (a3)
+; RV64IA-NEXT:    bnez a4, .LBB13_1
 ; RV64IA-NEXT:  .LBB13_3:
 ; RV64IA-NEXT:    ret
   %res = cmpxchg i16* %ptr, i16 %cmp, i16 %val release monotonic
@@ -1124,26 +1124,26 @@ define void @cmpxchg_i16_release_acquire(i16* %ptr, i16 %cmp, i16 %val) nounwind
 ;
 ; RV32IA-LABEL: cmpxchg_i16_release_acquire:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a3, 16
-; RV32IA-NEXT:    addi a3, a3, -1
-; RV32IA-NEXT:    and a1, a1, a3
-; RV32IA-NEXT:    and a2, a2, a3
-; RV32IA-NEXT:    slli a4, a0, 3
-; RV32IA-NEXT:    andi a4, a4, 24
-; RV32IA-NEXT:    sll a3, a3, a4
-; RV32IA-NEXT:    sll a2, a2, a4
-; RV32IA-NEXT:    sll a1, a1, a4
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a3, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a4, 16
+; RV32IA-NEXT:    addi a4, a4, -1
+; RV32IA-NEXT:    sll a5, a4, a0
+; RV32IA-NEXT:    and a1, a1, a4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    and a2, a2, a4
+; RV32IA-NEXT:    sll a0, a2, a0
 ; RV32IA-NEXT:  .LBB14_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a4, (a0)
-; RV32IA-NEXT:    and a5, a4, a3
-; RV32IA-NEXT:    bne a5, a1, .LBB14_3
+; RV32IA-NEXT:    lr.w a2, (a3)
+; RV32IA-NEXT:    and a4, a2, a5
+; RV32IA-NEXT:    bne a4, a1, .LBB14_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB14_1 Depth=1
-; RV32IA-NEXT:    xor a5, a4, a2
-; RV32IA-NEXT:    and a5, a5, a3
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w.rl a5, a5, (a0)
-; RV32IA-NEXT:    bnez a5, .LBB14_1
+; RV32IA-NEXT:    xor a4, a2, a0
+; RV32IA-NEXT:    and a4, a4, a5
+; RV32IA-NEXT:    xor a4, a2, a4
+; RV32IA-NEXT:    sc.w.rl a4, a4, (a3)
+; RV32IA-NEXT:    bnez a4, .LBB14_1
 ; RV32IA-NEXT:  .LBB14_3:
 ; RV32IA-NEXT:    ret
 ;
@@ -1162,26 +1162,26 @@ define void @cmpxchg_i16_release_acquire(i16* %ptr, i16 %cmp, i16 %val) nounwind
 ;
 ; RV64IA-LABEL: cmpxchg_i16_release_acquire:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a3, 16
-; RV64IA-NEXT:    addiw a3, a3, -1
-; RV64IA-NEXT:    and a1, a1, a3
-; RV64IA-NEXT:    and a2, a2, a3
-; RV64IA-NEXT:    slli a4, a0, 3
-; RV64IA-NEXT:    andi a4, a4, 24
-; RV64IA-NEXT:    sllw a3, a3, a4
-; RV64IA-NEXT:    sllw a2, a2, a4
-; RV64IA-NEXT:    sllw a1, a1, a4
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a3, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a4, 16
+; RV64IA-NEXT:    addiw a4, a4, -1
+; RV64IA-NEXT:    sllw a5, a4, a0
+; RV64IA-NEXT:    and a1, a1, a4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    and a2, a2, a4
+; RV64IA-NEXT:    sllw a0, a2, a0
 ; RV64IA-NEXT:  .LBB14_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a4, (a0)
-; RV64IA-NEXT:    and a5, a4, a3
-; RV64IA-NEXT:    bne a5, a1, .LBB14_3
+; RV64IA-NEXT:    lr.w a2, (a3)
+; RV64IA-NEXT:    and a4, a2, a5
+; RV64IA-NEXT:    bne a4, a1, .LBB14_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB14_1 Depth=1
-; RV64IA-NEXT:    xor a5, a4, a2
-; RV64IA-NEXT:    and a5, a5, a3
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
-; RV64IA-NEXT:    bnez a5, .LBB14_1
+; RV64IA-NEXT:    xor a4, a2, a0
+; RV64IA-NEXT:    and a4, a4, a5
+; RV64IA-NEXT:    xor a4, a2, a4
+; RV64IA-NEXT:    sc.w.rl a4, a4, (a3)
+; RV64IA-NEXT:    bnez a4, .LBB14_1
 ; RV64IA-NEXT:  .LBB14_3:
 ; RV64IA-NEXT:    ret
   %res = cmpxchg i16* %ptr, i16 %cmp, i16 %val release acquire
@@ -1204,26 +1204,26 @@ define void @cmpxchg_i16_acq_rel_monotonic(i16* %ptr, i16 %cmp, i16 %val) nounwi
 ;
 ; RV32IA-LABEL: cmpxchg_i16_acq_rel_monotonic:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a3, 16
-; RV32IA-NEXT:    addi a3, a3, -1
-; RV32IA-NEXT:    and a1, a1, a3
-; RV32IA-NEXT:    and a2, a2, a3
-; RV32IA-NEXT:    slli a4, a0, 3
-; RV32IA-NEXT:    andi a4, a4, 24
-; RV32IA-NEXT:    sll a3, a3, a4
-; RV32IA-NEXT:    sll a2, a2, a4
-; RV32IA-NEXT:    sll a1, a1, a4
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a3, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a4, 16
+; RV32IA-NEXT:    addi a4, a4, -1
+; RV32IA-NEXT:    sll a5, a4, a0
+; RV32IA-NEXT:    and a1, a1, a4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    and a2, a2, a4
+; RV32IA-NEXT:    sll a0, a2, a0
 ; RV32IA-NEXT:  .LBB15_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a4, (a0)
-; RV32IA-NEXT:    and a5, a4, a3
-; RV32IA-NEXT:    bne a5, a1, .LBB15_3
+; RV32IA-NEXT:    lr.w.aq a2, (a3)
+; RV32IA-NEXT:    and a4, a2, a5
+; RV32IA-NEXT:    bne a4, a1, .LBB15_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB15_1 Depth=1
-; RV32IA-NEXT:    xor a5, a4, a2
-; RV32IA-NEXT:    and a5, a5, a3
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w.rl a5, a5, (a0)
-; RV32IA-NEXT:    bnez a5, .LBB15_1
+; RV32IA-NEXT:    xor a4, a2, a0
+; RV32IA-NEXT:    and a4, a4, a5
+; RV32IA-NEXT:    xor a4, a2, a4
+; RV32IA-NEXT:    sc.w.rl a4, a4, (a3)
+; RV32IA-NEXT:    bnez a4, .LBB15_1
 ; RV32IA-NEXT:  .LBB15_3:
 ; RV32IA-NEXT:    ret
 ;
@@ -1242,26 +1242,26 @@ define void @cmpxchg_i16_acq_rel_monotonic(i16* %ptr, i16 %cmp, i16 %val) nounwi
 ;
 ; RV64IA-LABEL: cmpxchg_i16_acq_rel_monotonic:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a3, 16
-; RV64IA-NEXT:    addiw a3, a3, -1
-; RV64IA-NEXT:    and a1, a1, a3
-; RV64IA-NEXT:    and a2, a2, a3
-; RV64IA-NEXT:    slli a4, a0, 3
-; RV64IA-NEXT:    andi a4, a4, 24
-; RV64IA-NEXT:    sllw a3, a3, a4
-; RV64IA-NEXT:    sllw a2, a2, a4
-; RV64IA-NEXT:    sllw a1, a1, a4
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a3, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a4, 16
+; RV64IA-NEXT:    addiw a4, a4, -1
+; RV64IA-NEXT:    sllw a5, a4, a0
+; RV64IA-NEXT:    and a1, a1, a4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    and a2, a2, a4
+; RV64IA-NEXT:    sllw a0, a2, a0
 ; RV64IA-NEXT:  .LBB15_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a4, (a0)
-; RV64IA-NEXT:    and a5, a4, a3
-; RV64IA-NEXT:    bne a5, a1, .LBB15_3
+; RV64IA-NEXT:    lr.w.aq a2, (a3)
+; RV64IA-NEXT:    and a4, a2, a5
+; RV64IA-NEXT:    bne a4, a1, .LBB15_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB15_1 Depth=1
-; RV64IA-NEXT:    xor a5, a4, a2
-; RV64IA-NEXT:    and a5, a5, a3
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
-; RV64IA-NEXT:    bnez a5, .LBB15_1
+; RV64IA-NEXT:    xor a4, a2, a0
+; RV64IA-NEXT:    and a4, a4, a5
+; RV64IA-NEXT:    xor a4, a2, a4
+; RV64IA-NEXT:    sc.w.rl a4, a4, (a3)
+; RV64IA-NEXT:    bnez a4, .LBB15_1
 ; RV64IA-NEXT:  .LBB15_3:
 ; RV64IA-NEXT:    ret
   %res = cmpxchg i16* %ptr, i16 %cmp, i16 %val acq_rel monotonic
@@ -1284,26 +1284,26 @@ define void @cmpxchg_i16_acq_rel_acquire(i16* %ptr, i16 %cmp, i16 %val) nounwind
 ;
 ; RV32IA-LABEL: cmpxchg_i16_acq_rel_acquire:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a3, 16
-; RV32IA-NEXT:    addi a3, a3, -1
-; RV32IA-NEXT:    and a1, a1, a3
-; RV32IA-NEXT:    and a2, a2, a3
-; RV32IA-NEXT:    slli a4, a0, 3
-; RV32IA-NEXT:    andi a4, a4, 24
-; RV32IA-NEXT:    sll a3, a3, a4
-; RV32IA-NEXT:    sll a2, a2, a4
-; RV32IA-NEXT:    sll a1, a1, a4
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a3, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a4, 16
+; RV32IA-NEXT:    addi a4, a4, -1
+; RV32IA-NEXT:    sll a5, a4, a0
+; RV32IA-NEXT:    and a1, a1, a4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    and a2, a2, a4
+; RV32IA-NEXT:    sll a0, a2, a0
 ; RV32IA-NEXT:  .LBB16_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a4, (a0)
-; RV32IA-NEXT:    and a5, a4, a3
-; RV32IA-NEXT:    bne a5, a1, .LBB16_3
+; RV32IA-NEXT:    lr.w.aq a2, (a3)
+; RV32IA-NEXT:    and a4, a2, a5
+; RV32IA-NEXT:    bne a4, a1, .LBB16_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB16_1 Depth=1
-; RV32IA-NEXT:    xor a5, a4, a2
-; RV32IA-NEXT:    and a5, a5, a3
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w.rl a5, a5, (a0)
-; RV32IA-NEXT:    bnez a5, .LBB16_1
+; RV32IA-NEXT:    xor a4, a2, a0
+; RV32IA-NEXT:    and a4, a4, a5
+; RV32IA-NEXT:    xor a4, a2, a4
+; RV32IA-NEXT:    sc.w.rl a4, a4, (a3)
+; RV32IA-NEXT:    bnez a4, .LBB16_1
 ; RV32IA-NEXT:  .LBB16_3:
 ; RV32IA-NEXT:    ret
 ;
@@ -1322,26 +1322,26 @@ define void @cmpxchg_i16_acq_rel_acquire(i16* %ptr, i16 %cmp, i16 %val) nounwind
 ;
 ; RV64IA-LABEL: cmpxchg_i16_acq_rel_acquire:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a3, 16
-; RV64IA-NEXT:    addiw a3, a3, -1
-; RV64IA-NEXT:    and a1, a1, a3
-; RV64IA-NEXT:    and a2, a2, a3
-; RV64IA-NEXT:    slli a4, a0, 3
-; RV64IA-NEXT:    andi a4, a4, 24
-; RV64IA-NEXT:    sllw a3, a3, a4
-; RV64IA-NEXT:    sllw a2, a2, a4
-; RV64IA-NEXT:    sllw a1, a1, a4
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a3, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a4, 16
+; RV64IA-NEXT:    addiw a4, a4, -1
+; RV64IA-NEXT:    sllw a5, a4, a0
+; RV64IA-NEXT:    and a1, a1, a4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    and a2, a2, a4
+; RV64IA-NEXT:    sllw a0, a2, a0
 ; RV64IA-NEXT:  .LBB16_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a4, (a0)
-; RV64IA-NEXT:    and a5, a4, a3
-; RV64IA-NEXT:    bne a5, a1, .LBB16_3
+; RV64IA-NEXT:    lr.w.aq a2, (a3)
+; RV64IA-NEXT:    and a4, a2, a5
+; RV64IA-NEXT:    bne a4, a1, .LBB16_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB16_1 Depth=1
-; RV64IA-NEXT:    xor a5, a4, a2
-; RV64IA-NEXT:    and a5, a5, a3
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
-; RV64IA-NEXT:    bnez a5, .LBB16_1
+; RV64IA-NEXT:    xor a4, a2, a0
+; RV64IA-NEXT:    and a4, a4, a5
+; RV64IA-NEXT:    xor a4, a2, a4
+; RV64IA-NEXT:    sc.w.rl a4, a4, (a3)
+; RV64IA-NEXT:    bnez a4, .LBB16_1
 ; RV64IA-NEXT:  .LBB16_3:
 ; RV64IA-NEXT:    ret
   %res = cmpxchg i16* %ptr, i16 %cmp, i16 %val acq_rel acquire
@@ -1364,26 +1364,26 @@ define void @cmpxchg_i16_seq_cst_monotonic(i16* %ptr, i16 %cmp, i16 %val) nounwi
 ;
 ; RV32IA-LABEL: cmpxchg_i16_seq_cst_monotonic:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a3, 16
-; RV32IA-NEXT:    addi a3, a3, -1
-; RV32IA-NEXT:    and a1, a1, a3
-; RV32IA-NEXT:    and a2, a2, a3
-; RV32IA-NEXT:    slli a4, a0, 3
-; RV32IA-NEXT:    andi a4, a4, 24
-; RV32IA-NEXT:    sll a3, a3, a4
-; RV32IA-NEXT:    sll a2, a2, a4
-; RV32IA-NEXT:    sll a1, a1, a4
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a3, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a4, 16
+; RV32IA-NEXT:    addi a4, a4, -1
+; RV32IA-NEXT:    sll a5, a4, a0
+; RV32IA-NEXT:    and a1, a1, a4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    and a2, a2, a4
+; RV32IA-NEXT:    sll a0, a2, a0
 ; RV32IA-NEXT:  .LBB17_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aqrl a4, (a0)
-; RV32IA-NEXT:    and a5, a4, a3
-; RV32IA-NEXT:    bne a5, a1, .LBB17_3
+; RV32IA-NEXT:    lr.w.aqrl a2, (a3)
+; RV32IA-NEXT:    and a4, a2, a5
+; RV32IA-NEXT:    bne a4, a1, .LBB17_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB17_1 Depth=1
-; RV32IA-NEXT:    xor a5, a4, a2
-; RV32IA-NEXT:    and a5, a5, a3
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w.aqrl a5, a5, (a0)
-; RV32IA-NEXT:    bnez a5, .LBB17_1
+; RV32IA-NEXT:    xor a4, a2, a0
+; RV32IA-NEXT:    and a4, a4, a5
+; RV32IA-NEXT:    xor a4, a2, a4
+; RV32IA-NEXT:    sc.w.aqrl a4, a4, (a3)
+; RV32IA-NEXT:    bnez a4, .LBB17_1
 ; RV32IA-NEXT:  .LBB17_3:
 ; RV32IA-NEXT:    ret
 ;
@@ -1402,26 +1402,26 @@ define void @cmpxchg_i16_seq_cst_monotonic(i16* %ptr, i16 %cmp, i16 %val) nounwi
 ;
 ; RV64IA-LABEL: cmpxchg_i16_seq_cst_monotonic:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a3, 16
-; RV64IA-NEXT:    addiw a3, a3, -1
-; RV64IA-NEXT:    and a1, a1, a3
-; RV64IA-NEXT:    and a2, a2, a3
-; RV64IA-NEXT:    slli a4, a0, 3
-; RV64IA-NEXT:    andi a4, a4, 24
-; RV64IA-NEXT:    sllw a3, a3, a4
-; RV64IA-NEXT:    sllw a2, a2, a4
-; RV64IA-NEXT:    sllw a1, a1, a4
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a3, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a4, 16
+; RV64IA-NEXT:    addiw a4, a4, -1
+; RV64IA-NEXT:    sllw a5, a4, a0
+; RV64IA-NEXT:    and a1, a1, a4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    and a2, a2, a4
+; RV64IA-NEXT:    sllw a0, a2, a0
 ; RV64IA-NEXT:  .LBB17_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aqrl a4, (a0)
-; RV64IA-NEXT:    and a5, a4, a3
-; RV64IA-NEXT:    bne a5, a1, .LBB17_3
+; RV64IA-NEXT:    lr.w.aqrl a2, (a3)
+; RV64IA-NEXT:    and a4, a2, a5
+; RV64IA-NEXT:    bne a4, a1, .LBB17_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB17_1 Depth=1
-; RV64IA-NEXT:    xor a5, a4, a2
-; RV64IA-NEXT:    and a5, a5, a3
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a0)
-; RV64IA-NEXT:    bnez a5, .LBB17_1
+; RV64IA-NEXT:    xor a4, a2, a0
+; RV64IA-NEXT:    and a4, a4, a5
+; RV64IA-NEXT:    xor a4, a2, a4
+; RV64IA-NEXT:    sc.w.aqrl a4, a4, (a3)
+; RV64IA-NEXT:    bnez a4, .LBB17_1
 ; RV64IA-NEXT:  .LBB17_3:
 ; RV64IA-NEXT:    ret
   %res = cmpxchg i16* %ptr, i16 %cmp, i16 %val seq_cst monotonic
@@ -1444,26 +1444,26 @@ define void @cmpxchg_i16_seq_cst_acquire(i16* %ptr, i16 %cmp, i16 %val) nounwind
 ;
 ; RV32IA-LABEL: cmpxchg_i16_seq_cst_acquire:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a3, 16
-; RV32IA-NEXT:    addi a3, a3, -1
-; RV32IA-NEXT:    and a1, a1, a3
-; RV32IA-NEXT:    and a2, a2, a3
-; RV32IA-NEXT:    slli a4, a0, 3
-; RV32IA-NEXT:    andi a4, a4, 24
-; RV32IA-NEXT:    sll a3, a3, a4
-; RV32IA-NEXT:    sll a2, a2, a4
-; RV32IA-NEXT:    sll a1, a1, a4
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a3, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a4, 16
+; RV32IA-NEXT:    addi a4, a4, -1
+; RV32IA-NEXT:    sll a5, a4, a0
+; RV32IA-NEXT:    and a1, a1, a4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    and a2, a2, a4
+; RV32IA-NEXT:    sll a0, a2, a0
 ; RV32IA-NEXT:  .LBB18_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aqrl a4, (a0)
-; RV32IA-NEXT:    and a5, a4, a3
-; RV32IA-NEXT:    bne a5, a1, .LBB18_3
+; RV32IA-NEXT:    lr.w.aqrl a2, (a3)
+; RV32IA-NEXT:    and a4, a2, a5
+; RV32IA-NEXT:    bne a4, a1, .LBB18_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB18_1 Depth=1
-; RV32IA-NEXT:    xor a5, a4, a2
-; RV32IA-NEXT:    and a5, a5, a3
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w.aqrl a5, a5, (a0)
-; RV32IA-NEXT:    bnez a5, .LBB18_1
+; RV32IA-NEXT:    xor a4, a2, a0
+; RV32IA-NEXT:    and a4, a4, a5
+; RV32IA-NEXT:    xor a4, a2, a4
+; RV32IA-NEXT:    sc.w.aqrl a4, a4, (a3)
+; RV32IA-NEXT:    bnez a4, .LBB18_1
 ; RV32IA-NEXT:  .LBB18_3:
 ; RV32IA-NEXT:    ret
 ;
@@ -1482,26 +1482,26 @@ define void @cmpxchg_i16_seq_cst_acquire(i16* %ptr, i16 %cmp, i16 %val) nounwind
 ;
 ; RV64IA-LABEL: cmpxchg_i16_seq_cst_acquire:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a3, 16
-; RV64IA-NEXT:    addiw a3, a3, -1
-; RV64IA-NEXT:    and a1, a1, a3
-; RV64IA-NEXT:    and a2, a2, a3
-; RV64IA-NEXT:    slli a4, a0, 3
-; RV64IA-NEXT:    andi a4, a4, 24
-; RV64IA-NEXT:    sllw a3, a3, a4
-; RV64IA-NEXT:    sllw a2, a2, a4
-; RV64IA-NEXT:    sllw a1, a1, a4
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a3, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a4, 16
+; RV64IA-NEXT:    addiw a4, a4, -1
+; RV64IA-NEXT:    sllw a5, a4, a0
+; RV64IA-NEXT:    and a1, a1, a4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    and a2, a2, a4
+; RV64IA-NEXT:    sllw a0, a2, a0
 ; RV64IA-NEXT:  .LBB18_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aqrl a4, (a0)
-; RV64IA-NEXT:    and a5, a4, a3
-; RV64IA-NEXT:    bne a5, a1, .LBB18_3
+; RV64IA-NEXT:    lr.w.aqrl a2, (a3)
+; RV64IA-NEXT:    and a4, a2, a5
+; RV64IA-NEXT:    bne a4, a1, .LBB18_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB18_1 Depth=1
-; RV64IA-NEXT:    xor a5, a4, a2
-; RV64IA-NEXT:    and a5, a5, a3
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a0)
-; RV64IA-NEXT:    bnez a5, .LBB18_1
+; RV64IA-NEXT:    xor a4, a2, a0
+; RV64IA-NEXT:    and a4, a4, a5
+; RV64IA-NEXT:    xor a4, a2, a4
+; RV64IA-NEXT:    sc.w.aqrl a4, a4, (a3)
+; RV64IA-NEXT:    bnez a4, .LBB18_1
 ; RV64IA-NEXT:  .LBB18_3:
 ; RV64IA-NEXT:    ret
   %res = cmpxchg i16* %ptr, i16 %cmp, i16 %val seq_cst acquire
@@ -1524,26 +1524,26 @@ define void @cmpxchg_i16_seq_cst_seq_cst(i16* %ptr, i16 %cmp, i16 %val) nounwind
 ;
 ; RV32IA-LABEL: cmpxchg_i16_seq_cst_seq_cst:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a3, 16
-; RV32IA-NEXT:    addi a3, a3, -1
-; RV32IA-NEXT:    and a1, a1, a3
-; RV32IA-NEXT:    and a2, a2, a3
-; RV32IA-NEXT:    slli a4, a0, 3
-; RV32IA-NEXT:    andi a4, a4, 24
-; RV32IA-NEXT:    sll a3, a3, a4
-; RV32IA-NEXT:    sll a2, a2, a4
-; RV32IA-NEXT:    sll a1, a1, a4
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a3, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a4, 16
+; RV32IA-NEXT:    addi a4, a4, -1
+; RV32IA-NEXT:    sll a5, a4, a0
+; RV32IA-NEXT:    and a1, a1, a4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    and a2, a2, a4
+; RV32IA-NEXT:    sll a0, a2, a0
 ; RV32IA-NEXT:  .LBB19_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aqrl a4, (a0)
-; RV32IA-NEXT:    and a5, a4, a3
-; RV32IA-NEXT:    bne a5, a1, .LBB19_3
+; RV32IA-NEXT:    lr.w.aqrl a2, (a3)
+; RV32IA-NEXT:    and a4, a2, a5
+; RV32IA-NEXT:    bne a4, a1, .LBB19_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB19_1 Depth=1
-; RV32IA-NEXT:    xor a5, a4, a2
-; RV32IA-NEXT:    and a5, a5, a3
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w.aqrl a5, a5, (a0)
-; RV32IA-NEXT:    bnez a5, .LBB19_1
+; RV32IA-NEXT:    xor a4, a2, a0
+; RV32IA-NEXT:    and a4, a4, a5
+; RV32IA-NEXT:    xor a4, a2, a4
+; RV32IA-NEXT:    sc.w.aqrl a4, a4, (a3)
+; RV32IA-NEXT:    bnez a4, .LBB19_1
 ; RV32IA-NEXT:  .LBB19_3:
 ; RV32IA-NEXT:    ret
 ;
@@ -1562,26 +1562,26 @@ define void @cmpxchg_i16_seq_cst_seq_cst(i16* %ptr, i16 %cmp, i16 %val) nounwind
 ;
 ; RV64IA-LABEL: cmpxchg_i16_seq_cst_seq_cst:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a3, 16
-; RV64IA-NEXT:    addiw a3, a3, -1
-; RV64IA-NEXT:    and a1, a1, a3
-; RV64IA-NEXT:    and a2, a2, a3
-; RV64IA-NEXT:    slli a4, a0, 3
-; RV64IA-NEXT:    andi a4, a4, 24
-; RV64IA-NEXT:    sllw a3, a3, a4
-; RV64IA-NEXT:    sllw a2, a2, a4
-; RV64IA-NEXT:    sllw a1, a1, a4
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a3, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a4, 16
+; RV64IA-NEXT:    addiw a4, a4, -1
+; RV64IA-NEXT:    sllw a5, a4, a0
+; RV64IA-NEXT:    and a1, a1, a4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    and a2, a2, a4
+; RV64IA-NEXT:    sllw a0, a2, a0
 ; RV64IA-NEXT:  .LBB19_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aqrl a4, (a0)
-; RV64IA-NEXT:    and a5, a4, a3
-; RV64IA-NEXT:    bne a5, a1, .LBB19_3
+; RV64IA-NEXT:    lr.w.aqrl a2, (a3)
+; RV64IA-NEXT:    and a4, a2, a5
+; RV64IA-NEXT:    bne a4, a1, .LBB19_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB19_1 Depth=1
-; RV64IA-NEXT:    xor a5, a4, a2
-; RV64IA-NEXT:    and a5, a5, a3
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a0)
-; RV64IA-NEXT:    bnez a5, .LBB19_1
+; RV64IA-NEXT:    xor a4, a2, a0
+; RV64IA-NEXT:    and a4, a4, a5
+; RV64IA-NEXT:    xor a4, a2, a4
+; RV64IA-NEXT:    sc.w.aqrl a4, a4, (a3)
+; RV64IA-NEXT:    bnez a4, .LBB19_1
 ; RV64IA-NEXT:  .LBB19_3:
 ; RV64IA-NEXT:    ret
   %res = cmpxchg i16* %ptr, i16 %cmp, i16 %val seq_cst seq_cst
@@ -2173,12 +2173,13 @@ define void @cmpxchg_i64_acquire_monotonic(i64* %ptr, i64 %cmp, i64 %val) nounwi
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    mv a5, a4
 ; RV32I-NEXT:    sw a2, 4(sp)
 ; RV32I-NEXT:    sw a1, 0(sp)
 ; RV32I-NEXT:    mv a1, sp
+; RV32I-NEXT:    addi a4, zero, 2
 ; RV32I-NEXT:    mv a2, a3
-; RV32I-NEXT:    mv a3, a4
-; RV32I-NEXT:    addi  a4, zero, 2
+; RV32I-NEXT:    mv a3, a5
 ; RV32I-NEXT:    mv a5, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
 ; RV32I-NEXT:    lw ra, 12(sp)
@@ -2189,12 +2190,13 @@ define void @cmpxchg_i64_acquire_monotonic(i64* %ptr, i64 %cmp, i64 %val) nounwi
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
 ; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    mv a5, a4
 ; RV32IA-NEXT:    sw a2, 4(sp)
 ; RV32IA-NEXT:    sw a1, 0(sp)
 ; RV32IA-NEXT:    mv a1, sp
-; RV32IA-NEXT:    mv a2, a3
-; RV32IA-NEXT:    mv a3, a4
 ; RV32IA-NEXT:    addi a4, zero, 2
+; RV32IA-NEXT:    mv a2, a3
+; RV32IA-NEXT:    mv a3, a5
 ; RV32IA-NEXT:    mv a5, zero
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
 ; RV32IA-NEXT:    lw ra, 12(sp)
@@ -2233,13 +2235,14 @@ define void @cmpxchg_i64_acquire_acquire(i64* %ptr, i64 %cmp, i64 %val) nounwind
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    mv a6, a4
 ; RV32I-NEXT:    sw a2, 4(sp)
 ; RV32I-NEXT:    sw a1, 0(sp)
 ; RV32I-NEXT:    mv a1, sp
-; RV32I-NEXT:    mv a2, a3
-; RV32I-NEXT:    mv a3, a4
 ; RV32I-NEXT:    addi a4, zero, 2
 ; RV32I-NEXT:    addi a5, zero, 2
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    mv a3, a6
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
 ; RV32I-NEXT:    lw ra, 12(sp)
 ; RV32I-NEXT:    addi sp, sp, 16
@@ -2249,13 +2252,14 @@ define void @cmpxchg_i64_acquire_acquire(i64* %ptr, i64 %cmp, i64 %val) nounwind
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
 ; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    mv a6, a4
 ; RV32IA-NEXT:    sw a2, 4(sp)
 ; RV32IA-NEXT:    sw a1, 0(sp)
 ; RV32IA-NEXT:    mv a1, sp
-; RV32IA-NEXT:    mv a2, a3
-; RV32IA-NEXT:    mv a3, a4
 ; RV32IA-NEXT:    addi a4, zero, 2
 ; RV32IA-NEXT:    addi a5, zero, 2
+; RV32IA-NEXT:    mv a2, a3
+; RV32IA-NEXT:    mv a3, a6
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
 ; RV32IA-NEXT:    lw ra, 12(sp)
 ; RV32IA-NEXT:    addi sp, sp, 16
@@ -2293,12 +2297,13 @@ define void @cmpxchg_i64_release_monotonic(i64* %ptr, i64 %cmp, i64 %val) nounwi
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    mv a5, a4
 ; RV32I-NEXT:    sw a2, 4(sp)
 ; RV32I-NEXT:    sw a1, 0(sp)
 ; RV32I-NEXT:    mv a1, sp
-; RV32I-NEXT:    mv a2, a3
-; RV32I-NEXT:    mv a3, a4
 ; RV32I-NEXT:    addi a4, zero, 3
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    mv a3, a5
 ; RV32I-NEXT:    mv a5, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
 ; RV32I-NEXT:    lw ra, 12(sp)
@@ -2309,12 +2314,13 @@ define void @cmpxchg_i64_release_monotonic(i64* %ptr, i64 %cmp, i64 %val) nounwi
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
 ; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    mv a5, a4
 ; RV32IA-NEXT:    sw a2, 4(sp)
 ; RV32IA-NEXT:    sw a1, 0(sp)
 ; RV32IA-NEXT:    mv a1, sp
-; RV32IA-NEXT:    mv a2, a3
-; RV32IA-NEXT:    mv a3, a4
 ; RV32IA-NEXT:    addi a4, zero, 3
+; RV32IA-NEXT:    mv a2, a3
+; RV32IA-NEXT:    mv a3, a5
 ; RV32IA-NEXT:    mv a5, zero
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
 ; RV32IA-NEXT:    lw ra, 12(sp)
@@ -2353,13 +2359,14 @@ define void @cmpxchg_i64_release_acquire(i64* %ptr, i64 %cmp, i64 %val) nounwind
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    mv a6, a4
 ; RV32I-NEXT:    sw a2, 4(sp)
 ; RV32I-NEXT:    sw a1, 0(sp)
 ; RV32I-NEXT:    mv a1, sp
-; RV32I-NEXT:    mv a2, a3
-; RV32I-NEXT:    mv a3, a4
 ; RV32I-NEXT:    addi a4, zero, 3
 ; RV32I-NEXT:    addi a5, zero, 2
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    mv a3, a6
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
 ; RV32I-NEXT:    lw ra, 12(sp)
 ; RV32I-NEXT:    addi sp, sp, 16
@@ -2369,13 +2376,14 @@ define void @cmpxchg_i64_release_acquire(i64* %ptr, i64 %cmp, i64 %val) nounwind
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
 ; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    mv a6, a4
 ; RV32IA-NEXT:    sw a2, 4(sp)
 ; RV32IA-NEXT:    sw a1, 0(sp)
 ; RV32IA-NEXT:    mv a1, sp
-; RV32IA-NEXT:    mv a2, a3
-; RV32IA-NEXT:    mv a3, a4
 ; RV32IA-NEXT:    addi a4, zero, 3
 ; RV32IA-NEXT:    addi a5, zero, 2
+; RV32IA-NEXT:    mv a2, a3
+; RV32IA-NEXT:    mv a3, a6
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
 ; RV32IA-NEXT:    lw ra, 12(sp)
 ; RV32IA-NEXT:    addi sp, sp, 16
@@ -2413,12 +2421,13 @@ define void @cmpxchg_i64_acq_rel_monotonic(i64* %ptr, i64 %cmp, i64 %val) nounwi
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    mv a5, a4
 ; RV32I-NEXT:    sw a2, 4(sp)
 ; RV32I-NEXT:    sw a1, 0(sp)
 ; RV32I-NEXT:    mv a1, sp
-; RV32I-NEXT:    mv a2, a3
-; RV32I-NEXT:    mv a3, a4
 ; RV32I-NEXT:    addi a4, zero, 4
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    mv a3, a5
 ; RV32I-NEXT:    mv a5, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
 ; RV32I-NEXT:    lw ra, 12(sp)
@@ -2429,12 +2438,13 @@ define void @cmpxchg_i64_acq_rel_monotonic(i64* %ptr, i64 %cmp, i64 %val) nounwi
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
 ; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    mv a5, a4
 ; RV32IA-NEXT:    sw a2, 4(sp)
 ; RV32IA-NEXT:    sw a1, 0(sp)
 ; RV32IA-NEXT:    mv a1, sp
-; RV32IA-NEXT:    mv a2, a3
-; RV32IA-NEXT:    mv a3, a4
 ; RV32IA-NEXT:    addi a4, zero, 4
+; RV32IA-NEXT:    mv a2, a3
+; RV32IA-NEXT:    mv a3, a5
 ; RV32IA-NEXT:    mv a5, zero
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
 ; RV32IA-NEXT:    lw ra, 12(sp)
@@ -2473,13 +2483,14 @@ define void @cmpxchg_i64_acq_rel_acquire(i64* %ptr, i64 %cmp, i64 %val) nounwind
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    mv a6, a4
 ; RV32I-NEXT:    sw a2, 4(sp)
 ; RV32I-NEXT:    sw a1, 0(sp)
 ; RV32I-NEXT:    mv a1, sp
-; RV32I-NEXT:    mv a2, a3
-; RV32I-NEXT:    mv a3, a4
 ; RV32I-NEXT:    addi a4, zero, 4
 ; RV32I-NEXT:    addi a5, zero, 2
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    mv a3, a6
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
 ; RV32I-NEXT:    lw ra, 12(sp)
 ; RV32I-NEXT:    addi sp, sp, 16
@@ -2489,13 +2500,14 @@ define void @cmpxchg_i64_acq_rel_acquire(i64* %ptr, i64 %cmp, i64 %val) nounwind
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
 ; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    mv a6, a4
 ; RV32IA-NEXT:    sw a2, 4(sp)
 ; RV32IA-NEXT:    sw a1, 0(sp)
 ; RV32IA-NEXT:    mv a1, sp
-; RV32IA-NEXT:    mv a2, a3
-; RV32IA-NEXT:    mv a3, a4
 ; RV32IA-NEXT:    addi a4, zero, 4
 ; RV32IA-NEXT:    addi a5, zero, 2
+; RV32IA-NEXT:    mv a2, a3
+; RV32IA-NEXT:    mv a3, a6
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
 ; RV32IA-NEXT:    lw ra, 12(sp)
 ; RV32IA-NEXT:    addi sp, sp, 16
@@ -2533,12 +2545,13 @@ define void @cmpxchg_i64_seq_cst_monotonic(i64* %ptr, i64 %cmp, i64 %val) nounwi
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    mv a5, a4
 ; RV32I-NEXT:    sw a2, 4(sp)
 ; RV32I-NEXT:    sw a1, 0(sp)
 ; RV32I-NEXT:    mv a1, sp
-; RV32I-NEXT:    mv a2, a3
-; RV32I-NEXT:    mv a3, a4
 ; RV32I-NEXT:    addi a4, zero, 5
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    mv a3, a5
 ; RV32I-NEXT:    mv a5, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
 ; RV32I-NEXT:    lw ra, 12(sp)
@@ -2549,12 +2562,13 @@ define void @cmpxchg_i64_seq_cst_monotonic(i64* %ptr, i64 %cmp, i64 %val) nounwi
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
 ; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    mv a5, a4
 ; RV32IA-NEXT:    sw a2, 4(sp)
 ; RV32IA-NEXT:    sw a1, 0(sp)
 ; RV32IA-NEXT:    mv a1, sp
-; RV32IA-NEXT:    mv a2, a3
-; RV32IA-NEXT:    mv a3, a4
 ; RV32IA-NEXT:    addi a4, zero, 5
+; RV32IA-NEXT:    mv a2, a3
+; RV32IA-NEXT:    mv a3, a5
 ; RV32IA-NEXT:    mv a5, zero
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
 ; RV32IA-NEXT:    lw ra, 12(sp)
@@ -2593,13 +2607,14 @@ define void @cmpxchg_i64_seq_cst_acquire(i64* %ptr, i64 %cmp, i64 %val) nounwind
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    mv a6, a4
 ; RV32I-NEXT:    sw a2, 4(sp)
 ; RV32I-NEXT:    sw a1, 0(sp)
 ; RV32I-NEXT:    mv a1, sp
-; RV32I-NEXT:    mv a2, a3
-; RV32I-NEXT:    mv a3, a4
 ; RV32I-NEXT:    addi a4, zero, 5
 ; RV32I-NEXT:    addi a5, zero, 2
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    mv a3, a6
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
 ; RV32I-NEXT:    lw ra, 12(sp)
 ; RV32I-NEXT:    addi sp, sp, 16
@@ -2609,13 +2624,14 @@ define void @cmpxchg_i64_seq_cst_acquire(i64* %ptr, i64 %cmp, i64 %val) nounwind
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
 ; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    mv a6, a4
 ; RV32IA-NEXT:    sw a2, 4(sp)
 ; RV32IA-NEXT:    sw a1, 0(sp)
 ; RV32IA-NEXT:    mv a1, sp
-; RV32IA-NEXT:    mv a2, a3
-; RV32IA-NEXT:    mv a3, a4
 ; RV32IA-NEXT:    addi a4, zero, 5
 ; RV32IA-NEXT:    addi a5, zero, 2
+; RV32IA-NEXT:    mv a2, a3
+; RV32IA-NEXT:    mv a3, a6
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
 ; RV32IA-NEXT:    lw ra, 12(sp)
 ; RV32IA-NEXT:    addi sp, sp, 16
@@ -2653,13 +2669,14 @@ define void @cmpxchg_i64_seq_cst_seq_cst(i64* %ptr, i64 %cmp, i64 %val) nounwind
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    mv a6, a4
 ; RV32I-NEXT:    sw a2, 4(sp)
 ; RV32I-NEXT:    sw a1, 0(sp)
 ; RV32I-NEXT:    mv a1, sp
-; RV32I-NEXT:    mv a2, a3
-; RV32I-NEXT:    mv a3, a4
 ; RV32I-NEXT:    addi a4, zero, 5
 ; RV32I-NEXT:    addi a5, zero, 5
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    mv a3, a6
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
 ; RV32I-NEXT:    lw ra, 12(sp)
 ; RV32I-NEXT:    addi sp, sp, 16
@@ -2669,13 +2686,14 @@ define void @cmpxchg_i64_seq_cst_seq_cst(i64* %ptr, i64 %cmp, i64 %val) nounwind
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
 ; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    mv a6, a4
 ; RV32IA-NEXT:    sw a2, 4(sp)
 ; RV32IA-NEXT:    sw a1, 0(sp)
 ; RV32IA-NEXT:    mv a1, sp
-; RV32IA-NEXT:    mv a2, a3
-; RV32IA-NEXT:    mv a3, a4
 ; RV32IA-NEXT:    addi a4, zero, 5
 ; RV32IA-NEXT:    addi a5, zero, 5
+; RV32IA-NEXT:    mv a2, a3
+; RV32IA-NEXT:    mv a3, a6
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
 ; RV32IA-NEXT:    lw ra, 12(sp)
 ; RV32IA-NEXT:    addi sp, sp, 16
diff --git a/llvm/test/CodeGen/RISCV/atomic-rmw.ll b/llvm/test/CodeGen/RISCV/atomic-rmw.ll
index c12a298b1..a84d50b5d 100644
--- a/llvm/test/CodeGen/RISCV/atomic-rmw.ll
+++ b/llvm/test/CodeGen/RISCV/atomic-rmw.ll
@@ -21,23 +21,23 @@ define i8 @atomicrmw_xchg_i8_monotonic(i8* %a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_xchg_i8_monotonic:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a3, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB0_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a4, (a0)
-; RV32IA-NEXT:    add a5, zero, a1
+; RV32IA-NEXT:    lr.w a4, (a2)
+; RV32IA-NEXT:    mv a5, a1
 ; RV32IA-NEXT:    xor a5, a4, a5
 ; RV32IA-NEXT:    and a5, a5, a3
 ; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w a5, a5, (a0)
+; RV32IA-NEXT:    sc.w a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB0_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a2
+; RV32IA-NEXT:    srl a0, a4, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_xchg_i8_monotonic:
@@ -52,23 +52,23 @@ define i8 @atomicrmw_xchg_i8_monotonic(i8* %a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_xchg_i8_monotonic:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB0_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a4, (a0)
-; RV64IA-NEXT:    add a5, zero, a1
+; RV64IA-NEXT:    lr.w a4, (a2)
+; RV64IA-NEXT:    mv a5, a1
 ; RV64IA-NEXT:    xor a5, a4, a5
 ; RV64IA-NEXT:    and a5, a5, a3
 ; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    sc.w a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB0_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    srlw a0, a4, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw xchg i8* %a, i8 %b monotonic
   ret i8 %1
@@ -87,23 +87,23 @@ define i8 @atomicrmw_xchg_i8_acquire(i8* %a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_xchg_i8_acquire:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a3, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB1_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a4, (a0)
-; RV32IA-NEXT:    add a5, zero, a1
+; RV32IA-NEXT:    lr.w.aq a4, (a2)
+; RV32IA-NEXT:    mv a5, a1
 ; RV32IA-NEXT:    xor a5, a4, a5
 ; RV32IA-NEXT:    and a5, a5, a3
 ; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w a5, a5, (a0)
+; RV32IA-NEXT:    sc.w a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB1_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a2
+; RV32IA-NEXT:    srl a0, a4, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_xchg_i8_acquire:
@@ -118,23 +118,23 @@ define i8 @atomicrmw_xchg_i8_acquire(i8* %a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_xchg_i8_acquire:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB1_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a4, (a0)
-; RV64IA-NEXT:    add a5, zero, a1
+; RV64IA-NEXT:    lr.w.aq a4, (a2)
+; RV64IA-NEXT:    mv a5, a1
 ; RV64IA-NEXT:    xor a5, a4, a5
 ; RV64IA-NEXT:    and a5, a5, a3
 ; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    sc.w a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB1_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    srlw a0, a4, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw xchg i8* %a, i8 %b acquire
   ret i8 %1
@@ -153,23 +153,23 @@ define i8 @atomicrmw_xchg_i8_release(i8* %a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_xchg_i8_release:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a3, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB2_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a4, (a0)
-; RV32IA-NEXT:    add a5, zero, a1
+; RV32IA-NEXT:    lr.w a4, (a2)
+; RV32IA-NEXT:    mv a5, a1
 ; RV32IA-NEXT:    xor a5, a4, a5
 ; RV32IA-NEXT:    and a5, a5, a3
 ; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV32IA-NEXT:    sc.w.rl a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB2_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a2
+; RV32IA-NEXT:    srl a0, a4, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_xchg_i8_release:
@@ -184,23 +184,23 @@ define i8 @atomicrmw_xchg_i8_release(i8* %a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_xchg_i8_release:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB2_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a4, (a0)
-; RV64IA-NEXT:    add a5, zero, a1
+; RV64IA-NEXT:    lr.w a4, (a2)
+; RV64IA-NEXT:    mv a5, a1
 ; RV64IA-NEXT:    xor a5, a4, a5
 ; RV64IA-NEXT:    and a5, a5, a3
 ; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB2_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    srlw a0, a4, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw xchg i8* %a, i8 %b release
   ret i8 %1
@@ -219,23 +219,23 @@ define i8 @atomicrmw_xchg_i8_acq_rel(i8* %a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_xchg_i8_acq_rel:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a3, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB3_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a4, (a0)
-; RV32IA-NEXT:    add a5, zero, a1
+; RV32IA-NEXT:    lr.w.aq a4, (a2)
+; RV32IA-NEXT:    mv a5, a1
 ; RV32IA-NEXT:    xor a5, a4, a5
 ; RV32IA-NEXT:    and a5, a5, a3
 ; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV32IA-NEXT:    sc.w.rl a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB3_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a2
+; RV32IA-NEXT:    srl a0, a4, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_xchg_i8_acq_rel:
@@ -250,23 +250,23 @@ define i8 @atomicrmw_xchg_i8_acq_rel(i8* %a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_xchg_i8_acq_rel:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB3_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a4, (a0)
-; RV64IA-NEXT:    add a5, zero, a1
+; RV64IA-NEXT:    lr.w.aq a4, (a2)
+; RV64IA-NEXT:    mv a5, a1
 ; RV64IA-NEXT:    xor a5, a4, a5
 ; RV64IA-NEXT:    and a5, a5, a3
 ; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB3_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    srlw a0, a4, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw xchg i8* %a, i8 %b acq_rel
   ret i8 %1
@@ -285,23 +285,23 @@ define i8 @atomicrmw_xchg_i8_seq_cst(i8* %a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_xchg_i8_seq_cst:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a3, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB4_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aqrl a4, (a0)
-; RV32IA-NEXT:    add a5, zero, a1
+; RV32IA-NEXT:    lr.w.aqrl a4, (a2)
+; RV32IA-NEXT:    mv a5, a1
 ; RV32IA-NEXT:    xor a5, a4, a5
 ; RV32IA-NEXT:    and a5, a5, a3
 ; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV32IA-NEXT:    sc.w.aqrl a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB4_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a2
+; RV32IA-NEXT:    srl a0, a4, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_xchg_i8_seq_cst:
@@ -316,23 +316,23 @@ define i8 @atomicrmw_xchg_i8_seq_cst(i8* %a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_xchg_i8_seq_cst:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB4_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aqrl a4, (a0)
-; RV64IA-NEXT:    add a5, zero, a1
+; RV64IA-NEXT:    lr.w.aqrl a4, (a2)
+; RV64IA-NEXT:    mv a5, a1
 ; RV64IA-NEXT:    xor a5, a4, a5
 ; RV64IA-NEXT:    and a5, a5, a3
 ; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB4_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    srlw a0, a4, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw xchg i8* %a, i8 %b seq_cst
   ret i8 %1
@@ -351,23 +351,23 @@ define i8 @atomicrmw_add_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_add_i8_monotonic:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a3, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB5_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a4, (a0)
+; RV32IA-NEXT:    lr.w a4, (a2)
 ; RV32IA-NEXT:    add a5, a4, a1
 ; RV32IA-NEXT:    xor a5, a4, a5
 ; RV32IA-NEXT:    and a5, a5, a3
 ; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w a5, a5, (a0)
+; RV32IA-NEXT:    sc.w a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB5_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a2
+; RV32IA-NEXT:    srl a0, a4, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_add_i8_monotonic:
@@ -382,23 +382,23 @@ define i8 @atomicrmw_add_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_add_i8_monotonic:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB5_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a4, (a0)
+; RV64IA-NEXT:    lr.w a4, (a2)
 ; RV64IA-NEXT:    add a5, a4, a1
 ; RV64IA-NEXT:    xor a5, a4, a5
 ; RV64IA-NEXT:    and a5, a5, a3
 ; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    sc.w a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB5_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    srlw a0, a4, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw add i8* %a, i8 %b monotonic
   ret i8 %1
@@ -417,23 +417,23 @@ define i8 @atomicrmw_add_i8_acquire(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_add_i8_acquire:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a3, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB6_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a4, (a0)
+; RV32IA-NEXT:    lr.w.aq a4, (a2)
 ; RV32IA-NEXT:    add a5, a4, a1
 ; RV32IA-NEXT:    xor a5, a4, a5
 ; RV32IA-NEXT:    and a5, a5, a3
 ; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w a5, a5, (a0)
+; RV32IA-NEXT:    sc.w a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB6_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a2
+; RV32IA-NEXT:    srl a0, a4, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_add_i8_acquire:
@@ -448,23 +448,23 @@ define i8 @atomicrmw_add_i8_acquire(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_add_i8_acquire:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB6_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a4, (a0)
+; RV64IA-NEXT:    lr.w.aq a4, (a2)
 ; RV64IA-NEXT:    add a5, a4, a1
 ; RV64IA-NEXT:    xor a5, a4, a5
 ; RV64IA-NEXT:    and a5, a5, a3
 ; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    sc.w a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB6_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    srlw a0, a4, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw add i8* %a, i8 %b acquire
   ret i8 %1
@@ -483,23 +483,23 @@ define i8 @atomicrmw_add_i8_release(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_add_i8_release:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a3, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB7_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a4, (a0)
+; RV32IA-NEXT:    lr.w a4, (a2)
 ; RV32IA-NEXT:    add a5, a4, a1
 ; RV32IA-NEXT:    xor a5, a4, a5
 ; RV32IA-NEXT:    and a5, a5, a3
 ; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV32IA-NEXT:    sc.w.rl a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB7_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a2
+; RV32IA-NEXT:    srl a0, a4, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_add_i8_release:
@@ -514,23 +514,23 @@ define i8 @atomicrmw_add_i8_release(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_add_i8_release:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB7_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a4, (a0)
+; RV64IA-NEXT:    lr.w a4, (a2)
 ; RV64IA-NEXT:    add a5, a4, a1
 ; RV64IA-NEXT:    xor a5, a4, a5
 ; RV64IA-NEXT:    and a5, a5, a3
 ; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB7_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    srlw a0, a4, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw add i8* %a, i8 %b release
   ret i8 %1
@@ -549,23 +549,23 @@ define i8 @atomicrmw_add_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_add_i8_acq_rel:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a3, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB8_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a4, (a0)
+; RV32IA-NEXT:    lr.w.aq a4, (a2)
 ; RV32IA-NEXT:    add a5, a4, a1
 ; RV32IA-NEXT:    xor a5, a4, a5
 ; RV32IA-NEXT:    and a5, a5, a3
 ; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV32IA-NEXT:    sc.w.rl a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB8_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a2
+; RV32IA-NEXT:    srl a0, a4, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_add_i8_acq_rel:
@@ -580,23 +580,23 @@ define i8 @atomicrmw_add_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_add_i8_acq_rel:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB8_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a4, (a0)
+; RV64IA-NEXT:    lr.w.aq a4, (a2)
 ; RV64IA-NEXT:    add a5, a4, a1
 ; RV64IA-NEXT:    xor a5, a4, a5
 ; RV64IA-NEXT:    and a5, a5, a3
 ; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB8_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    srlw a0, a4, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw add i8* %a, i8 %b acq_rel
   ret i8 %1
@@ -615,23 +615,23 @@ define i8 @atomicrmw_add_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_add_i8_seq_cst:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a3, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB9_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aqrl a4, (a0)
+; RV32IA-NEXT:    lr.w.aqrl a4, (a2)
 ; RV32IA-NEXT:    add a5, a4, a1
 ; RV32IA-NEXT:    xor a5, a4, a5
 ; RV32IA-NEXT:    and a5, a5, a3
 ; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV32IA-NEXT:    sc.w.aqrl a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB9_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a2
+; RV32IA-NEXT:    srl a0, a4, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_add_i8_seq_cst:
@@ -646,23 +646,23 @@ define i8 @atomicrmw_add_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_add_i8_seq_cst:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB9_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aqrl a4, (a0)
+; RV64IA-NEXT:    lr.w.aqrl a4, (a2)
 ; RV64IA-NEXT:    add a5, a4, a1
 ; RV64IA-NEXT:    xor a5, a4, a5
 ; RV64IA-NEXT:    and a5, a5, a3
 ; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB9_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    srlw a0, a4, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw add i8* %a, i8 %b seq_cst
   ret i8 %1
@@ -681,23 +681,23 @@ define i8 @atomicrmw_sub_i8_monotonic(i8* %a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_sub_i8_monotonic:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a3, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB10_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a4, (a0)
+; RV32IA-NEXT:    lr.w a4, (a2)
 ; RV32IA-NEXT:    sub a5, a4, a1
 ; RV32IA-NEXT:    xor a5, a4, a5
 ; RV32IA-NEXT:    and a5, a5, a3
 ; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w a5, a5, (a0)
+; RV32IA-NEXT:    sc.w a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB10_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a2
+; RV32IA-NEXT:    srl a0, a4, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_sub_i8_monotonic:
@@ -712,23 +712,23 @@ define i8 @atomicrmw_sub_i8_monotonic(i8* %a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_sub_i8_monotonic:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB10_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a4, (a0)
+; RV64IA-NEXT:    lr.w a4, (a2)
 ; RV64IA-NEXT:    sub a5, a4, a1
 ; RV64IA-NEXT:    xor a5, a4, a5
 ; RV64IA-NEXT:    and a5, a5, a3
 ; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    sc.w a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB10_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    srlw a0, a4, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw sub i8* %a, i8 %b monotonic
   ret i8 %1
@@ -747,23 +747,23 @@ define i8 @atomicrmw_sub_i8_acquire(i8* %a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_sub_i8_acquire:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a3, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB11_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a4, (a0)
+; RV32IA-NEXT:    lr.w.aq a4, (a2)
 ; RV32IA-NEXT:    sub a5, a4, a1
 ; RV32IA-NEXT:    xor a5, a4, a5
 ; RV32IA-NEXT:    and a5, a5, a3
 ; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w a5, a5, (a0)
+; RV32IA-NEXT:    sc.w a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB11_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a2
+; RV32IA-NEXT:    srl a0, a4, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_sub_i8_acquire:
@@ -778,23 +778,23 @@ define i8 @atomicrmw_sub_i8_acquire(i8* %a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_sub_i8_acquire:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB11_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a4, (a0)
+; RV64IA-NEXT:    lr.w.aq a4, (a2)
 ; RV64IA-NEXT:    sub a5, a4, a1
 ; RV64IA-NEXT:    xor a5, a4, a5
 ; RV64IA-NEXT:    and a5, a5, a3
 ; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    sc.w a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB11_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    srlw a0, a4, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw sub i8* %a, i8 %b acquire
   ret i8 %1
@@ -813,23 +813,23 @@ define i8 @atomicrmw_sub_i8_release(i8* %a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_sub_i8_release:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a3, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB12_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a4, (a0)
+; RV32IA-NEXT:    lr.w a4, (a2)
 ; RV32IA-NEXT:    sub a5, a4, a1
 ; RV32IA-NEXT:    xor a5, a4, a5
 ; RV32IA-NEXT:    and a5, a5, a3
 ; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV32IA-NEXT:    sc.w.rl a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB12_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a2
+; RV32IA-NEXT:    srl a0, a4, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_sub_i8_release:
@@ -844,23 +844,23 @@ define i8 @atomicrmw_sub_i8_release(i8* %a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_sub_i8_release:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB12_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a4, (a0)
+; RV64IA-NEXT:    lr.w a4, (a2)
 ; RV64IA-NEXT:    sub a5, a4, a1
 ; RV64IA-NEXT:    xor a5, a4, a5
 ; RV64IA-NEXT:    and a5, a5, a3
 ; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB12_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    srlw a0, a4, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw sub i8* %a, i8 %b release
   ret i8 %1
@@ -879,23 +879,23 @@ define i8 @atomicrmw_sub_i8_acq_rel(i8* %a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_sub_i8_acq_rel:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a3, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB13_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a4, (a0)
+; RV32IA-NEXT:    lr.w.aq a4, (a2)
 ; RV32IA-NEXT:    sub a5, a4, a1
 ; RV32IA-NEXT:    xor a5, a4, a5
 ; RV32IA-NEXT:    and a5, a5, a3
 ; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV32IA-NEXT:    sc.w.rl a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB13_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a2
+; RV32IA-NEXT:    srl a0, a4, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_sub_i8_acq_rel:
@@ -910,23 +910,23 @@ define i8 @atomicrmw_sub_i8_acq_rel(i8* %a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_sub_i8_acq_rel:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB13_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a4, (a0)
+; RV64IA-NEXT:    lr.w.aq a4, (a2)
 ; RV64IA-NEXT:    sub a5, a4, a1
 ; RV64IA-NEXT:    xor a5, a4, a5
 ; RV64IA-NEXT:    and a5, a5, a3
 ; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB13_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    srlw a0, a4, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw sub i8* %a, i8 %b acq_rel
   ret i8 %1
@@ -945,23 +945,23 @@ define i8 @atomicrmw_sub_i8_seq_cst(i8* %a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_sub_i8_seq_cst:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a3, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB14_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aqrl a4, (a0)
+; RV32IA-NEXT:    lr.w.aqrl a4, (a2)
 ; RV32IA-NEXT:    sub a5, a4, a1
 ; RV32IA-NEXT:    xor a5, a4, a5
 ; RV32IA-NEXT:    and a5, a5, a3
 ; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV32IA-NEXT:    sc.w.aqrl a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB14_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a2
+; RV32IA-NEXT:    srl a0, a4, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_sub_i8_seq_cst:
@@ -976,23 +976,23 @@ define i8 @atomicrmw_sub_i8_seq_cst(i8* %a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_sub_i8_seq_cst:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB14_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aqrl a4, (a0)
+; RV64IA-NEXT:    lr.w.aqrl a4, (a2)
 ; RV64IA-NEXT:    sub a5, a4, a1
 ; RV64IA-NEXT:    xor a5, a4, a5
 ; RV64IA-NEXT:    and a5, a5, a3
 ; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB14_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    srlw a0, a4, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw sub i8* %a, i8 %b seq_cst
   ret i8 %1
@@ -1011,17 +1011,17 @@ define i8 @atomicrmw_and_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_and_i8_monotonic:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    sll a1, a1, a2
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a3, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    not a3, a3
+; RV32IA-NEXT:    andi a1, a1, 255
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:    or a1, a3, a1
-; RV32IA-NEXT:    andi a0, a0, -4
-; RV32IA-NEXT:    amoand.w a0, a1, (a0)
-; RV32IA-NEXT:    srl a0, a0, a2
+; RV32IA-NEXT:    amoand.w a1, a1, (a2)
+; RV32IA-NEXT:    srl a0, a1, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_and_i8_monotonic:
@@ -1036,17 +1036,17 @@ define i8 @atomicrmw_and_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_and_i8_monotonic:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    not a3, a3
+; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:    or a1, a3, a1
-; RV64IA-NEXT:    andi a0, a0, -4
-; RV64IA-NEXT:    amoand.w a0, a1, (a0)
-; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    amoand.w a1, a1, (a2)
+; RV64IA-NEXT:    srlw a0, a1, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw and i8* %a, i8 %b monotonic
   ret i8 %1
@@ -1065,17 +1065,17 @@ define i8 @atomicrmw_and_i8_acquire(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_and_i8_acquire:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    sll a1, a1, a2
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a3, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    not a3, a3
+; RV32IA-NEXT:    andi a1, a1, 255
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:    or a1, a3, a1
-; RV32IA-NEXT:    andi a0, a0, -4
-; RV32IA-NEXT:    amoand.w.aq a0, a1, (a0)
-; RV32IA-NEXT:    srl a0, a0, a2
+; RV32IA-NEXT:    amoand.w.aq a1, a1, (a2)
+; RV32IA-NEXT:    srl a0, a1, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_and_i8_acquire:
@@ -1090,17 +1090,17 @@ define i8 @atomicrmw_and_i8_acquire(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_and_i8_acquire:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    not a3, a3
+; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:    or a1, a3, a1
-; RV64IA-NEXT:    andi a0, a0, -4
-; RV64IA-NEXT:    amoand.w.aq a0, a1, (a0)
-; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    amoand.w.aq a1, a1, (a2)
+; RV64IA-NEXT:    srlw a0, a1, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw and i8* %a, i8 %b acquire
   ret i8 %1
@@ -1119,17 +1119,17 @@ define i8 @atomicrmw_and_i8_release(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_and_i8_release:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    sll a1, a1, a2
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a3, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    not a3, a3
+; RV32IA-NEXT:    andi a1, a1, 255
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:    or a1, a3, a1
-; RV32IA-NEXT:    andi a0, a0, -4
-; RV32IA-NEXT:    amoand.w.rl a0, a1, (a0)
-; RV32IA-NEXT:    srl a0, a0, a2
+; RV32IA-NEXT:    amoand.w.rl a1, a1, (a2)
+; RV32IA-NEXT:    srl a0, a1, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_and_i8_release:
@@ -1144,17 +1144,17 @@ define i8 @atomicrmw_and_i8_release(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_and_i8_release:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    not a3, a3
+; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:    or a1, a3, a1
-; RV64IA-NEXT:    andi a0, a0, -4
-; RV64IA-NEXT:    amoand.w.rl a0, a1, (a0)
-; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    amoand.w.rl a1, a1, (a2)
+; RV64IA-NEXT:    srlw a0, a1, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw and i8* %a, i8 %b release
   ret i8 %1
@@ -1173,17 +1173,17 @@ define i8 @atomicrmw_and_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_and_i8_acq_rel:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    sll a1, a1, a2
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a3, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    not a3, a3
+; RV32IA-NEXT:    andi a1, a1, 255
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:    or a1, a3, a1
-; RV32IA-NEXT:    andi a0, a0, -4
-; RV32IA-NEXT:    amoand.w.aqrl a0, a1, (a0)
-; RV32IA-NEXT:    srl a0, a0, a2
+; RV32IA-NEXT:    amoand.w.aqrl a1, a1, (a2)
+; RV32IA-NEXT:    srl a0, a1, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_and_i8_acq_rel:
@@ -1198,17 +1198,17 @@ define i8 @atomicrmw_and_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_and_i8_acq_rel:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    not a3, a3
+; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:    or a1, a3, a1
-; RV64IA-NEXT:    andi a0, a0, -4
-; RV64IA-NEXT:    amoand.w.aqrl a0, a1, (a0)
-; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    amoand.w.aqrl a1, a1, (a2)
+; RV64IA-NEXT:    srlw a0, a1, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw and i8* %a, i8 %b acq_rel
   ret i8 %1
@@ -1227,17 +1227,17 @@ define i8 @atomicrmw_and_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_and_i8_seq_cst:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    sll a1, a1, a2
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a3, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    not a3, a3
+; RV32IA-NEXT:    andi a1, a1, 255
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:    or a1, a3, a1
-; RV32IA-NEXT:    andi a0, a0, -4
-; RV32IA-NEXT:    amoand.w.aqrl a0, a1, (a0)
-; RV32IA-NEXT:    srl a0, a0, a2
+; RV32IA-NEXT:    amoand.w.aqrl a1, a1, (a2)
+; RV32IA-NEXT:    srl a0, a1, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_and_i8_seq_cst:
@@ -1252,17 +1252,17 @@ define i8 @atomicrmw_and_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_and_i8_seq_cst:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    not a3, a3
+; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:    or a1, a3, a1
-; RV64IA-NEXT:    andi a0, a0, -4
-; RV64IA-NEXT:    amoand.w.aqrl a0, a1, (a0)
-; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    amoand.w.aqrl a1, a1, (a2)
+; RV64IA-NEXT:    srlw a0, a1, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw and i8* %a, i8 %b seq_cst
   ret i8 %1
@@ -1281,24 +1281,24 @@ define i8 @atomicrmw_nand_i8_monotonic(i8* %a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_nand_i8_monotonic:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a3, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB20_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a4, (a0)
+; RV32IA-NEXT:    lr.w a4, (a2)
 ; RV32IA-NEXT:    and a5, a4, a1
 ; RV32IA-NEXT:    not a5, a5
 ; RV32IA-NEXT:    xor a5, a4, a5
 ; RV32IA-NEXT:    and a5, a5, a3
 ; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w a5, a5, (a0)
+; RV32IA-NEXT:    sc.w a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB20_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a2
+; RV32IA-NEXT:    srl a0, a4, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_nand_i8_monotonic:
@@ -1313,24 +1313,24 @@ define i8 @atomicrmw_nand_i8_monotonic(i8* %a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_nand_i8_monotonic:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB20_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a4, (a0)
+; RV64IA-NEXT:    lr.w a4, (a2)
 ; RV64IA-NEXT:    and a5, a4, a1
 ; RV64IA-NEXT:    not a5, a5
 ; RV64IA-NEXT:    xor a5, a4, a5
 ; RV64IA-NEXT:    and a5, a5, a3
 ; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    sc.w a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB20_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    srlw a0, a4, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw nand i8* %a, i8 %b monotonic
   ret i8 %1
@@ -1349,24 +1349,24 @@ define i8 @atomicrmw_nand_i8_acquire(i8* %a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_nand_i8_acquire:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a3, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB21_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a4, (a0)
+; RV32IA-NEXT:    lr.w.aq a4, (a2)
 ; RV32IA-NEXT:    and a5, a4, a1
 ; RV32IA-NEXT:    not a5, a5
 ; RV32IA-NEXT:    xor a5, a4, a5
 ; RV32IA-NEXT:    and a5, a5, a3
 ; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w a5, a5, (a0)
+; RV32IA-NEXT:    sc.w a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB21_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a2
+; RV32IA-NEXT:    srl a0, a4, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_nand_i8_acquire:
@@ -1381,24 +1381,24 @@ define i8 @atomicrmw_nand_i8_acquire(i8* %a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_nand_i8_acquire:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB21_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a4, (a0)
+; RV64IA-NEXT:    lr.w.aq a4, (a2)
 ; RV64IA-NEXT:    and a5, a4, a1
 ; RV64IA-NEXT:    not a5, a5
 ; RV64IA-NEXT:    xor a5, a4, a5
 ; RV64IA-NEXT:    and a5, a5, a3
 ; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    sc.w a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB21_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    srlw a0, a4, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw nand i8* %a, i8 %b acquire
   ret i8 %1
@@ -1417,24 +1417,24 @@ define i8 @atomicrmw_nand_i8_release(i8* %a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_nand_i8_release:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a3, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB22_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a4, (a0)
+; RV32IA-NEXT:    lr.w a4, (a2)
 ; RV32IA-NEXT:    and a5, a4, a1
 ; RV32IA-NEXT:    not a5, a5
 ; RV32IA-NEXT:    xor a5, a4, a5
 ; RV32IA-NEXT:    and a5, a5, a3
 ; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV32IA-NEXT:    sc.w.rl a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB22_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a2
+; RV32IA-NEXT:    srl a0, a4, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_nand_i8_release:
@@ -1449,24 +1449,24 @@ define i8 @atomicrmw_nand_i8_release(i8* %a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_nand_i8_release:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB22_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a4, (a0)
+; RV64IA-NEXT:    lr.w a4, (a2)
 ; RV64IA-NEXT:    and a5, a4, a1
 ; RV64IA-NEXT:    not a5, a5
 ; RV64IA-NEXT:    xor a5, a4, a5
 ; RV64IA-NEXT:    and a5, a5, a3
 ; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB22_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    srlw a0, a4, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw nand i8* %a, i8 %b release
   ret i8 %1
@@ -1485,24 +1485,24 @@ define i8 @atomicrmw_nand_i8_acq_rel(i8* %a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_nand_i8_acq_rel:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a3, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB23_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a4, (a0)
+; RV32IA-NEXT:    lr.w.aq a4, (a2)
 ; RV32IA-NEXT:    and a5, a4, a1
 ; RV32IA-NEXT:    not a5, a5
 ; RV32IA-NEXT:    xor a5, a4, a5
 ; RV32IA-NEXT:    and a5, a5, a3
 ; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV32IA-NEXT:    sc.w.rl a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB23_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a2
+; RV32IA-NEXT:    srl a0, a4, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_nand_i8_acq_rel:
@@ -1517,24 +1517,24 @@ define i8 @atomicrmw_nand_i8_acq_rel(i8* %a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_nand_i8_acq_rel:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB23_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a4, (a0)
+; RV64IA-NEXT:    lr.w.aq a4, (a2)
 ; RV64IA-NEXT:    and a5, a4, a1
 ; RV64IA-NEXT:    not a5, a5
 ; RV64IA-NEXT:    xor a5, a4, a5
 ; RV64IA-NEXT:    and a5, a5, a3
 ; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB23_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    srlw a0, a4, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw nand i8* %a, i8 %b acq_rel
   ret i8 %1
@@ -1553,24 +1553,24 @@ define i8 @atomicrmw_nand_i8_seq_cst(i8* %a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_nand_i8_seq_cst:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a3, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB24_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aqrl a4, (a0)
+; RV32IA-NEXT:    lr.w.aqrl a4, (a2)
 ; RV32IA-NEXT:    and a5, a4, a1
 ; RV32IA-NEXT:    not a5, a5
 ; RV32IA-NEXT:    xor a5, a4, a5
 ; RV32IA-NEXT:    and a5, a5, a3
 ; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV32IA-NEXT:    sc.w.aqrl a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB24_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a2
+; RV32IA-NEXT:    srl a0, a4, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_nand_i8_seq_cst:
@@ -1585,24 +1585,24 @@ define i8 @atomicrmw_nand_i8_seq_cst(i8* %a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_nand_i8_seq_cst:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB24_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aqrl a4, (a0)
+; RV64IA-NEXT:    lr.w.aqrl a4, (a2)
 ; RV64IA-NEXT:    and a5, a4, a1
 ; RV64IA-NEXT:    not a5, a5
 ; RV64IA-NEXT:    xor a5, a4, a5
 ; RV64IA-NEXT:    and a5, a5, a3
 ; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB24_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    srlw a0, a4, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw nand i8* %a, i8 %b seq_cst
   ret i8 %1
@@ -1621,13 +1621,13 @@ define i8 @atomicrmw_or_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_or_i8_monotonic:
 ; RV32IA:       # %bb.0:
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
-; RV32IA-NEXT:    amoor.w a0, a1, (a0)
-; RV32IA-NEXT:    srl a0, a0, a2
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    amoor.w a1, a1, (a2)
+; RV32IA-NEXT:    srl a0, a1, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_or_i8_monotonic:
@@ -1642,13 +1642,13 @@ define i8 @atomicrmw_or_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_or_i8_monotonic:
 ; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
-; RV64IA-NEXT:    amoor.w a0, a1, (a0)
-; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    amoor.w a1, a1, (a2)
+; RV64IA-NEXT:    srlw a0, a1, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw or i8* %a, i8 %b monotonic
   ret i8 %1
@@ -1667,13 +1667,13 @@ define i8 @atomicrmw_or_i8_acquire(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_or_i8_acquire:
 ; RV32IA:       # %bb.0:
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
-; RV32IA-NEXT:    amoor.w.aq a0, a1, (a0)
-; RV32IA-NEXT:    srl a0, a0, a2
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    amoor.w.aq a1, a1, (a2)
+; RV32IA-NEXT:    srl a0, a1, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_or_i8_acquire:
@@ -1688,13 +1688,13 @@ define i8 @atomicrmw_or_i8_acquire(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_or_i8_acquire:
 ; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
-; RV64IA-NEXT:    amoor.w.aq a0, a1, (a0)
-; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    amoor.w.aq a1, a1, (a2)
+; RV64IA-NEXT:    srlw a0, a1, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw or i8* %a, i8 %b acquire
   ret i8 %1
@@ -1713,13 +1713,13 @@ define i8 @atomicrmw_or_i8_release(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_or_i8_release:
 ; RV32IA:       # %bb.0:
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
-; RV32IA-NEXT:    amoor.w.rl a0, a1, (a0)
-; RV32IA-NEXT:    srl a0, a0, a2
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    amoor.w.rl a1, a1, (a2)
+; RV32IA-NEXT:    srl a0, a1, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_or_i8_release:
@@ -1734,13 +1734,13 @@ define i8 @atomicrmw_or_i8_release(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_or_i8_release:
 ; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
-; RV64IA-NEXT:    amoor.w.rl a0, a1, (a0)
-; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    amoor.w.rl a1, a1, (a2)
+; RV64IA-NEXT:    srlw a0, a1, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw or i8* %a, i8 %b release
   ret i8 %1
@@ -1759,13 +1759,13 @@ define i8 @atomicrmw_or_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_or_i8_acq_rel:
 ; RV32IA:       # %bb.0:
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
-; RV32IA-NEXT:    amoor.w.aqrl a0, a1, (a0)
-; RV32IA-NEXT:    srl a0, a0, a2
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    amoor.w.aqrl a1, a1, (a2)
+; RV32IA-NEXT:    srl a0, a1, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_or_i8_acq_rel:
@@ -1780,13 +1780,13 @@ define i8 @atomicrmw_or_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_or_i8_acq_rel:
 ; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
-; RV64IA-NEXT:    amoor.w.aqrl a0, a1, (a0)
-; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    amoor.w.aqrl a1, a1, (a2)
+; RV64IA-NEXT:    srlw a0, a1, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw or i8* %a, i8 %b acq_rel
   ret i8 %1
@@ -1805,13 +1805,13 @@ define i8 @atomicrmw_or_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_or_i8_seq_cst:
 ; RV32IA:       # %bb.0:
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
-; RV32IA-NEXT:    amoor.w.aqrl a0, a1, (a0)
-; RV32IA-NEXT:    srl a0, a0, a2
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    amoor.w.aqrl a1, a1, (a2)
+; RV32IA-NEXT:    srl a0, a1, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_or_i8_seq_cst:
@@ -1826,13 +1826,13 @@ define i8 @atomicrmw_or_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_or_i8_seq_cst:
 ; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
-; RV64IA-NEXT:    amoor.w.aqrl a0, a1, (a0)
-; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    amoor.w.aqrl a1, a1, (a2)
+; RV64IA-NEXT:    srlw a0, a1, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw or i8* %a, i8 %b seq_cst
   ret i8 %1
@@ -1851,13 +1851,13 @@ define i8 @atomicrmw_xor_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_xor_i8_monotonic:
 ; RV32IA:       # %bb.0:
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
-; RV32IA-NEXT:    amoxor.w a0, a1, (a0)
-; RV32IA-NEXT:    srl a0, a0, a2
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    amoxor.w a1, a1, (a2)
+; RV32IA-NEXT:    srl a0, a1, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_xor_i8_monotonic:
@@ -1872,13 +1872,13 @@ define i8 @atomicrmw_xor_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_xor_i8_monotonic:
 ; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
-; RV64IA-NEXT:    amoxor.w a0, a1, (a0)
-; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    amoxor.w a1, a1, (a2)
+; RV64IA-NEXT:    srlw a0, a1, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw xor i8* %a, i8 %b monotonic
   ret i8 %1
@@ -1897,13 +1897,13 @@ define i8 @atomicrmw_xor_i8_acquire(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_xor_i8_acquire:
 ; RV32IA:       # %bb.0:
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
-; RV32IA-NEXT:    amoxor.w.aq a0, a1, (a0)
-; RV32IA-NEXT:    srl a0, a0, a2
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    amoxor.w.aq a1, a1, (a2)
+; RV32IA-NEXT:    srl a0, a1, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_xor_i8_acquire:
@@ -1918,13 +1918,13 @@ define i8 @atomicrmw_xor_i8_acquire(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_xor_i8_acquire:
 ; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
-; RV64IA-NEXT:    amoxor.w.aq a0, a1, (a0)
-; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    amoxor.w.aq a1, a1, (a2)
+; RV64IA-NEXT:    srlw a0, a1, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw xor i8* %a, i8 %b acquire
   ret i8 %1
@@ -1943,13 +1943,13 @@ define i8 @atomicrmw_xor_i8_release(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_xor_i8_release:
 ; RV32IA:       # %bb.0:
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
-; RV32IA-NEXT:    amoxor.w.rl a0, a1, (a0)
-; RV32IA-NEXT:    srl a0, a0, a2
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    amoxor.w.rl a1, a1, (a2)
+; RV32IA-NEXT:    srl a0, a1, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_xor_i8_release:
@@ -1964,13 +1964,13 @@ define i8 @atomicrmw_xor_i8_release(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_xor_i8_release:
 ; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
-; RV64IA-NEXT:    amoxor.w.rl a0, a1, (a0)
-; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    amoxor.w.rl a1, a1, (a2)
+; RV64IA-NEXT:    srlw a0, a1, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw xor i8* %a, i8 %b release
   ret i8 %1
@@ -1989,13 +1989,13 @@ define i8 @atomicrmw_xor_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_xor_i8_acq_rel:
 ; RV32IA:       # %bb.0:
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
-; RV32IA-NEXT:    amoxor.w.aqrl a0, a1, (a0)
-; RV32IA-NEXT:    srl a0, a0, a2
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    amoxor.w.aqrl a1, a1, (a2)
+; RV32IA-NEXT:    srl a0, a1, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_xor_i8_acq_rel:
@@ -2010,13 +2010,13 @@ define i8 @atomicrmw_xor_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_xor_i8_acq_rel:
 ; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
-; RV64IA-NEXT:    amoxor.w.aqrl a0, a1, (a0)
-; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    amoxor.w.aqrl a1, a1, (a2)
+; RV64IA-NEXT:    srlw a0, a1, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw xor i8* %a, i8 %b acq_rel
   ret i8 %1
@@ -2035,13 +2035,13 @@ define i8 @atomicrmw_xor_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_xor_i8_seq_cst:
 ; RV32IA:       # %bb.0:
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
-; RV32IA-NEXT:    amoxor.w.aqrl a0, a1, (a0)
-; RV32IA-NEXT:    srl a0, a0, a2
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    amoxor.w.aqrl a1, a1, (a2)
+; RV32IA-NEXT:    srl a0, a1, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_xor_i8_seq_cst:
@@ -2056,13 +2056,13 @@ define i8 @atomicrmw_xor_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_xor_i8_seq_cst:
 ; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
-; RV64IA-NEXT:    amoxor.w.aqrl a0, a1, (a0)
-; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    amoxor.w.aqrl a1, a1, (a2)
+; RV64IA-NEXT:    srlw a0, a1, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw xor i8* %a, i8 %b seq_cst
   ret i8 %1
@@ -2079,33 +2079,33 @@ define i8 @atomicrmw_max_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    sw s3, 12(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lbu a0, 0(a0)
-; RV32I-NEXT:    slli a1, a1, 24
-; RV32I-NEXT:    srai s0, a1, 24
+; RV32I-NEXT:    lbu a1, 0(a0)
+; RV32I-NEXT:    slli a0, s2, 24
+; RV32I-NEXT:    srai s0, a0, 24
 ; RV32I-NEXT:    addi s3, sp, 11
 ; RV32I-NEXT:    j .LBB35_2
 ; RV32I-NEXT:  .LBB35_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB35_2 Depth=1
-; RV32I-NEXT:    sb a0, 11(sp)
+; RV32I-NEXT:    sb a1, 11(sp)
 ; RV32I-NEXT:    mv a0, s1
 ; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    mv a3, zero
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_1
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lb a0, 11(sp)
-; RV32I-NEXT:    bnez a1, .LBB35_4
+; RV32I-NEXT:    lb a1, 11(sp)
+; RV32I-NEXT:    bnez a0, .LBB35_4
 ; RV32I-NEXT:  .LBB35_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    slli a1, a0, 24
-; RV32I-NEXT:    srai a1, a1, 24
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    blt s0, a1, .LBB35_1
+; RV32I-NEXT:    slli a0, a1, 24
+; RV32I-NEXT:    srai a0, a0, 24
+; RV32I-NEXT:    mv a2, a1
+; RV32I-NEXT:    blt s0, a0, .LBB35_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB35_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:    j .LBB35_1
 ; RV32I-NEXT:  .LBB35_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -2116,32 +2116,32 @@ define i8 @atomicrmw_max_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_max_i8_monotonic:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    addi a3, zero, 24
-; RV32IA-NEXT:    sub a6, a3, a2
-; RV32IA-NEXT:    addi a4, zero, 255
-; RV32IA-NEXT:    sll a7, a4, a2
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    addi a3, zero, 255
+; RV32IA-NEXT:    sll a7, a3, a0
 ; RV32IA-NEXT:    slli a1, a1, 24
 ; RV32IA-NEXT:    srai a1, a1, 24
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    addi a4, zero, 24
+; RV32IA-NEXT:    sub a4, a4, a0
 ; RV32IA-NEXT:  .LBB35_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a5, (a0)
-; RV32IA-NEXT:    and a4, a5, a7
-; RV32IA-NEXT:    mv a3, a5
-; RV32IA-NEXT:    sll a4, a4, a6
-; RV32IA-NEXT:    sra a4, a4, a6
-; RV32IA-NEXT:    bge a4, a1, .LBB35_3
+; RV32IA-NEXT:    lr.w a5, (a6)
+; RV32IA-NEXT:    and a3, a5, a7
+; RV32IA-NEXT:    mv a2, a5
+; RV32IA-NEXT:    sll a3, a3, a4
+; RV32IA-NEXT:    sra a3, a3, a4
+; RV32IA-NEXT:    bge a3, a1, .LBB35_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB35_1 Depth=1
-; RV32IA-NEXT:    xor a3, a5, a1
-; RV32IA-NEXT:    and a3, a3, a7
-; RV32IA-NEXT:    xor a3, a5, a3
+; RV32IA-NEXT:    xor a2, a5, a1
+; RV32IA-NEXT:    and a2, a2, a7
+; RV32IA-NEXT:    xor a2, a5, a2
 ; RV32IA-NEXT:  .LBB35_3: # in Loop: Header=BB35_1 Depth=1
-; RV32IA-NEXT:    sc.w a3, a3, (a0)
-; RV32IA-NEXT:    bnez a3, .LBB35_1
+; RV32IA-NEXT:    sc.w a2, a2, (a6)
+; RV32IA-NEXT:    bnez a2, .LBB35_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a5, a2
+; RV32IA-NEXT:    srl a0, a5, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_max_i8_monotonic:
@@ -2154,33 +2154,33 @@ define i8 @atomicrmw_max_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    sd s3, 8(sp)
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lbu a0, 0(a0)
-; RV64I-NEXT:    slli a1, a1, 56
-; RV64I-NEXT:    srai s0, a1, 56
+; RV64I-NEXT:    lbu a1, 0(a0)
+; RV64I-NEXT:    slli a0, s2, 56
+; RV64I-NEXT:    srai s0, a0, 56
 ; RV64I-NEXT:    addi s3, sp, 7
 ; RV64I-NEXT:    j .LBB35_2
 ; RV64I-NEXT:  .LBB35_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB35_2 Depth=1
-; RV64I-NEXT:    sb a0, 7(sp)
+; RV64I-NEXT:    sb a1, 7(sp)
 ; RV64I-NEXT:    mv a0, s1
 ; RV64I-NEXT:    mv a1, s3
 ; RV64I-NEXT:    mv a3, zero
 ; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    call __atomic_compare_exchange_1
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lb a0, 7(sp)
-; RV64I-NEXT:    bnez a1, .LBB35_4
+; RV64I-NEXT:    lb a1, 7(sp)
+; RV64I-NEXT:    bnez a0, .LBB35_4
 ; RV64I-NEXT:  .LBB35_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    slli a1, a0, 56
-; RV64I-NEXT:    srai a1, a1, 56
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    blt s0, a1, .LBB35_1
+; RV64I-NEXT:    slli a0, a1, 56
+; RV64I-NEXT:    srai a0, a0, 56
+; RV64I-NEXT:    mv a2, a1
+; RV64I-NEXT:    blt s0, a0, .LBB35_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB35_2 Depth=1
 ; RV64I-NEXT:    mv a2, s2
 ; RV64I-NEXT:    j .LBB35_1
 ; RV64I-NEXT:  .LBB35_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -2191,32 +2191,32 @@ define i8 @atomicrmw_max_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_max_i8_monotonic:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    addi a3, zero, 56
-; RV64IA-NEXT:    sub a6, a3, a2
-; RV64IA-NEXT:    addi a4, zero, 255
-; RV64IA-NEXT:    sllw a7, a4, a2
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    addi a3, zero, 255
+; RV64IA-NEXT:    sllw a7, a3, a0
 ; RV64IA-NEXT:    slli a1, a1, 56
 ; RV64IA-NEXT:    srai a1, a1, 56
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    addi a4, zero, 56
+; RV64IA-NEXT:    sub a4, a4, a0
 ; RV64IA-NEXT:  .LBB35_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a5, (a0)
-; RV64IA-NEXT:    and a4, a5, a7
-; RV64IA-NEXT:    mv a3, a5
-; RV64IA-NEXT:    sll a4, a4, a6
-; RV64IA-NEXT:    sra a4, a4, a6
-; RV64IA-NEXT:    bge a4, a1, .LBB35_3
+; RV64IA-NEXT:    lr.w a5, (a6)
+; RV64IA-NEXT:    and a3, a5, a7
+; RV64IA-NEXT:    mv a2, a5
+; RV64IA-NEXT:    sll a3, a3, a4
+; RV64IA-NEXT:    sra a3, a3, a4
+; RV64IA-NEXT:    bge a3, a1, .LBB35_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB35_1 Depth=1
-; RV64IA-NEXT:    xor a3, a5, a1
-; RV64IA-NEXT:    and a3, a3, a7
-; RV64IA-NEXT:    xor a3, a5, a3
+; RV64IA-NEXT:    xor a2, a5, a1
+; RV64IA-NEXT:    and a2, a2, a7
+; RV64IA-NEXT:    xor a2, a5, a2
 ; RV64IA-NEXT:  .LBB35_3: # in Loop: Header=BB35_1 Depth=1
-; RV64IA-NEXT:    sc.w a3, a3, (a0)
-; RV64IA-NEXT:    bnez a3, .LBB35_1
+; RV64IA-NEXT:    sc.w a2, a2, (a6)
+; RV64IA-NEXT:    bnez a2, .LBB35_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a5, a2
+; RV64IA-NEXT:    srlw a0, a5, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw max i8* %a, i8 %b monotonic
   ret i8 %1
@@ -2233,33 +2233,33 @@ define i8 @atomicrmw_max_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    sw s3, 12(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lbu a0, 0(a0)
-; RV32I-NEXT:    slli a1, a1, 24
-; RV32I-NEXT:    srai s0, a1, 24
+; RV32I-NEXT:    lbu a1, 0(a0)
+; RV32I-NEXT:    slli a0, s2, 24
+; RV32I-NEXT:    srai s0, a0, 24
 ; RV32I-NEXT:    addi s3, sp, 11
 ; RV32I-NEXT:    j .LBB36_2
 ; RV32I-NEXT:  .LBB36_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB36_2 Depth=1
-; RV32I-NEXT:    sb a0, 11(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sb a1, 11(sp)
 ; RV32I-NEXT:    addi a3, zero, 2
 ; RV32I-NEXT:    addi a4, zero, 2
+; RV32I-NEXT:    mv a0, s1
+; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    call __atomic_compare_exchange_1
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lb a0, 11(sp)
-; RV32I-NEXT:    bnez a1, .LBB36_4
+; RV32I-NEXT:    lb a1, 11(sp)
+; RV32I-NEXT:    bnez a0, .LBB36_4
 ; RV32I-NEXT:  .LBB36_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    slli a1, a0, 24
-; RV32I-NEXT:    srai a1, a1, 24
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    blt s0, a1, .LBB36_1
+; RV32I-NEXT:    slli a0, a1, 24
+; RV32I-NEXT:    srai a0, a0, 24
+; RV32I-NEXT:    mv a2, a1
+; RV32I-NEXT:    blt s0, a0, .LBB36_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB36_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:    j .LBB36_1
 ; RV32I-NEXT:  .LBB36_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -2270,32 +2270,32 @@ define i8 @atomicrmw_max_i8_acquire(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_max_i8_acquire:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    addi a3, zero, 24
-; RV32IA-NEXT:    sub a6, a3, a2
-; RV32IA-NEXT:    addi a4, zero, 255
-; RV32IA-NEXT:    sll a7, a4, a2
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    addi a3, zero, 255
+; RV32IA-NEXT:    sll a7, a3, a0
 ; RV32IA-NEXT:    slli a1, a1, 24
 ; RV32IA-NEXT:    srai a1, a1, 24
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    addi a4, zero, 24
+; RV32IA-NEXT:    sub a4, a4, a0
 ; RV32IA-NEXT:  .LBB36_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a5, (a0)
-; RV32IA-NEXT:    and a4, a5, a7
-; RV32IA-NEXT:    mv a3, a5
-; RV32IA-NEXT:    sll a4, a4, a6
-; RV32IA-NEXT:    sra a4, a4, a6
-; RV32IA-NEXT:    bge a4, a1, .LBB36_3
+; RV32IA-NEXT:    lr.w.aq a5, (a6)
+; RV32IA-NEXT:    and a3, a5, a7
+; RV32IA-NEXT:    mv a2, a5
+; RV32IA-NEXT:    sll a3, a3, a4
+; RV32IA-NEXT:    sra a3, a3, a4
+; RV32IA-NEXT:    bge a3, a1, .LBB36_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB36_1 Depth=1
-; RV32IA-NEXT:    xor a3, a5, a1
-; RV32IA-NEXT:    and a3, a3, a7
-; RV32IA-NEXT:    xor a3, a5, a3
+; RV32IA-NEXT:    xor a2, a5, a1
+; RV32IA-NEXT:    and a2, a2, a7
+; RV32IA-NEXT:    xor a2, a5, a2
 ; RV32IA-NEXT:  .LBB36_3: # in Loop: Header=BB36_1 Depth=1
-; RV32IA-NEXT:    sc.w a3, a3, (a0)
-; RV32IA-NEXT:    bnez a3, .LBB36_1
+; RV32IA-NEXT:    sc.w a2, a2, (a6)
+; RV32IA-NEXT:    bnez a2, .LBB36_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a5, a2
+; RV32IA-NEXT:    srl a0, a5, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_max_i8_acquire:
@@ -2308,33 +2308,33 @@ define i8 @atomicrmw_max_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    sd s3, 8(sp)
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lbu a0, 0(a0)
-; RV64I-NEXT:    slli a1, a1, 56
-; RV64I-NEXT:    srai s0, a1, 56
+; RV64I-NEXT:    lbu a1, 0(a0)
+; RV64I-NEXT:    slli a0, s2, 56
+; RV64I-NEXT:    srai s0, a0, 56
 ; RV64I-NEXT:    addi s3, sp, 7
 ; RV64I-NEXT:    j .LBB36_2
 ; RV64I-NEXT:  .LBB36_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB36_2 Depth=1
-; RV64I-NEXT:    sb a0, 7(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sb a1, 7(sp)
 ; RV64I-NEXT:    addi a3, zero, 2
 ; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    mv a0, s1
+; RV64I-NEXT:    mv a1, s3
 ; RV64I-NEXT:    call __atomic_compare_exchange_1
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lb a0, 7(sp)
-; RV64I-NEXT:    bnez a1, .LBB36_4
+; RV64I-NEXT:    lb a1, 7(sp)
+; RV64I-NEXT:    bnez a0, .LBB36_4
 ; RV64I-NEXT:  .LBB36_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    slli a1, a0, 56
-; RV64I-NEXT:    srai a1, a1, 56
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    blt s0, a1, .LBB36_1
+; RV64I-NEXT:    slli a0, a1, 56
+; RV64I-NEXT:    srai a0, a0, 56
+; RV64I-NEXT:    mv a2, a1
+; RV64I-NEXT:    blt s0, a0, .LBB36_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB36_2 Depth=1
 ; RV64I-NEXT:    mv a2, s2
 ; RV64I-NEXT:    j .LBB36_1
 ; RV64I-NEXT:  .LBB36_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -2345,32 +2345,32 @@ define i8 @atomicrmw_max_i8_acquire(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_max_i8_acquire:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    addi a3, zero, 56
-; RV64IA-NEXT:    sub a6, a3, a2
-; RV64IA-NEXT:    addi a4, zero, 255
-; RV64IA-NEXT:    sllw a7, a4, a2
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    addi a3, zero, 255
+; RV64IA-NEXT:    sllw a7, a3, a0
 ; RV64IA-NEXT:    slli a1, a1, 56
 ; RV64IA-NEXT:    srai a1, a1, 56
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    addi a4, zero, 56
+; RV64IA-NEXT:    sub a4, a4, a0
 ; RV64IA-NEXT:  .LBB36_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a5, (a0)
-; RV64IA-NEXT:    and a4, a5, a7
-; RV64IA-NEXT:    mv a3, a5
-; RV64IA-NEXT:    sll a4, a4, a6
-; RV64IA-NEXT:    sra a4, a4, a6
-; RV64IA-NEXT:    bge a4, a1, .LBB36_3
+; RV64IA-NEXT:    lr.w.aq a5, (a6)
+; RV64IA-NEXT:    and a3, a5, a7
+; RV64IA-NEXT:    mv a2, a5
+; RV64IA-NEXT:    sll a3, a3, a4
+; RV64IA-NEXT:    sra a3, a3, a4
+; RV64IA-NEXT:    bge a3, a1, .LBB36_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB36_1 Depth=1
-; RV64IA-NEXT:    xor a3, a5, a1
-; RV64IA-NEXT:    and a3, a3, a7
-; RV64IA-NEXT:    xor a3, a5, a3
+; RV64IA-NEXT:    xor a2, a5, a1
+; RV64IA-NEXT:    and a2, a2, a7
+; RV64IA-NEXT:    xor a2, a5, a2
 ; RV64IA-NEXT:  .LBB36_3: # in Loop: Header=BB36_1 Depth=1
-; RV64IA-NEXT:    sc.w a3, a3, (a0)
-; RV64IA-NEXT:    bnez a3, .LBB36_1
+; RV64IA-NEXT:    sc.w a2, a2, (a6)
+; RV64IA-NEXT:    bnez a2, .LBB36_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a5, a2
+; RV64IA-NEXT:    srlw a0, a5, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw max i8* %a, i8 %b acquire
   ret i8 %1
@@ -2387,33 +2387,33 @@ define i8 @atomicrmw_max_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    sw s3, 12(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lbu a0, 0(a0)
-; RV32I-NEXT:    slli a1, a1, 24
-; RV32I-NEXT:    srai s0, a1, 24
+; RV32I-NEXT:    lbu a1, 0(a0)
+; RV32I-NEXT:    slli a0, s2, 24
+; RV32I-NEXT:    srai s0, a0, 24
 ; RV32I-NEXT:    addi s3, sp, 11
 ; RV32I-NEXT:    j .LBB37_2
 ; RV32I-NEXT:  .LBB37_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB37_2 Depth=1
-; RV32I-NEXT:    sb a0, 11(sp)
+; RV32I-NEXT:    sb a1, 11(sp)
+; RV32I-NEXT:    addi a3, zero, 3
 ; RV32I-NEXT:    mv a0, s1
 ; RV32I-NEXT:    mv a1, s3
-; RV32I-NEXT:    addi a3, zero, 3
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_1
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lb a0, 11(sp)
-; RV32I-NEXT:    bnez a1, .LBB37_4
+; RV32I-NEXT:    lb a1, 11(sp)
+; RV32I-NEXT:    bnez a0, .LBB37_4
 ; RV32I-NEXT:  .LBB37_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    slli a1, a0, 24
-; RV32I-NEXT:    srai a1, a1, 24
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    blt s0, a1, .LBB37_1
+; RV32I-NEXT:    slli a0, a1, 24
+; RV32I-NEXT:    srai a0, a0, 24
+; RV32I-NEXT:    mv a2, a1
+; RV32I-NEXT:    blt s0, a0, .LBB37_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB37_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:    j .LBB37_1
 ; RV32I-NEXT:  .LBB37_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -2424,32 +2424,32 @@ define i8 @atomicrmw_max_i8_release(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_max_i8_release:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    addi a3, zero, 24
-; RV32IA-NEXT:    sub a6, a3, a2
-; RV32IA-NEXT:    addi a4, zero, 255
-; RV32IA-NEXT:    sll a7, a4, a2
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    addi a3, zero, 255
+; RV32IA-NEXT:    sll a7, a3, a0
 ; RV32IA-NEXT:    slli a1, a1, 24
 ; RV32IA-NEXT:    srai a1, a1, 24
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    addi a4, zero, 24
+; RV32IA-NEXT:    sub a4, a4, a0
 ; RV32IA-NEXT:  .LBB37_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a5, (a0)
-; RV32IA-NEXT:    and a4, a5, a7
-; RV32IA-NEXT:    mv a3, a5
-; RV32IA-NEXT:    sll a4, a4, a6
-; RV32IA-NEXT:    sra a4, a4, a6
-; RV32IA-NEXT:    bge a4, a1, .LBB37_3
+; RV32IA-NEXT:    lr.w a5, (a6)
+; RV32IA-NEXT:    and a3, a5, a7
+; RV32IA-NEXT:    mv a2, a5
+; RV32IA-NEXT:    sll a3, a3, a4
+; RV32IA-NEXT:    sra a3, a3, a4
+; RV32IA-NEXT:    bge a3, a1, .LBB37_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB37_1 Depth=1
-; RV32IA-NEXT:    xor a3, a5, a1
-; RV32IA-NEXT:    and a3, a3, a7
-; RV32IA-NEXT:    xor a3, a5, a3
+; RV32IA-NEXT:    xor a2, a5, a1
+; RV32IA-NEXT:    and a2, a2, a7
+; RV32IA-NEXT:    xor a2, a5, a2
 ; RV32IA-NEXT:  .LBB37_3: # in Loop: Header=BB37_1 Depth=1
-; RV32IA-NEXT:    sc.w.rl a3, a3, (a0)
-; RV32IA-NEXT:    bnez a3, .LBB37_1
+; RV32IA-NEXT:    sc.w.rl a2, a2, (a6)
+; RV32IA-NEXT:    bnez a2, .LBB37_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a5, a2
+; RV32IA-NEXT:    srl a0, a5, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_max_i8_release:
@@ -2462,33 +2462,33 @@ define i8 @atomicrmw_max_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    sd s3, 8(sp)
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lbu a0, 0(a0)
-; RV64I-NEXT:    slli a1, a1, 56
-; RV64I-NEXT:    srai s0, a1, 56
+; RV64I-NEXT:    lbu a1, 0(a0)
+; RV64I-NEXT:    slli a0, s2, 56
+; RV64I-NEXT:    srai s0, a0, 56
 ; RV64I-NEXT:    addi s3, sp, 7
 ; RV64I-NEXT:    j .LBB37_2
 ; RV64I-NEXT:  .LBB37_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB37_2 Depth=1
-; RV64I-NEXT:    sb a0, 7(sp)
+; RV64I-NEXT:    sb a1, 7(sp)
+; RV64I-NEXT:    addi a3, zero, 3
 ; RV64I-NEXT:    mv a0, s1
 ; RV64I-NEXT:    mv a1, s3
-; RV64I-NEXT:    addi a3, zero, 3
 ; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    call __atomic_compare_exchange_1
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lb a0, 7(sp)
-; RV64I-NEXT:    bnez a1, .LBB37_4
+; RV64I-NEXT:    lb a1, 7(sp)
+; RV64I-NEXT:    bnez a0, .LBB37_4
 ; RV64I-NEXT:  .LBB37_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    slli a1, a0, 56
-; RV64I-NEXT:    srai a1, a1, 56
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    blt s0, a1, .LBB37_1
+; RV64I-NEXT:    slli a0, a1, 56
+; RV64I-NEXT:    srai a0, a0, 56
+; RV64I-NEXT:    mv a2, a1
+; RV64I-NEXT:    blt s0, a0, .LBB37_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB37_2 Depth=1
 ; RV64I-NEXT:    mv a2, s2
 ; RV64I-NEXT:    j .LBB37_1
 ; RV64I-NEXT:  .LBB37_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -2499,32 +2499,32 @@ define i8 @atomicrmw_max_i8_release(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_max_i8_release:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    addi a3, zero, 56
-; RV64IA-NEXT:    sub a6, a3, a2
-; RV64IA-NEXT:    addi a4, zero, 255
-; RV64IA-NEXT:    sllw a7, a4, a2
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    addi a3, zero, 255
+; RV64IA-NEXT:    sllw a7, a3, a0
 ; RV64IA-NEXT:    slli a1, a1, 56
 ; RV64IA-NEXT:    srai a1, a1, 56
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    addi a4, zero, 56
+; RV64IA-NEXT:    sub a4, a4, a0
 ; RV64IA-NEXT:  .LBB37_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a5, (a0)
-; RV64IA-NEXT:    and a4, a5, a7
-; RV64IA-NEXT:    mv a3, a5
-; RV64IA-NEXT:    sll a4, a4, a6
-; RV64IA-NEXT:    sra a4, a4, a6
-; RV64IA-NEXT:    bge a4, a1, .LBB37_3
+; RV64IA-NEXT:    lr.w a5, (a6)
+; RV64IA-NEXT:    and a3, a5, a7
+; RV64IA-NEXT:    mv a2, a5
+; RV64IA-NEXT:    sll a3, a3, a4
+; RV64IA-NEXT:    sra a3, a3, a4
+; RV64IA-NEXT:    bge a3, a1, .LBB37_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB37_1 Depth=1
-; RV64IA-NEXT:    xor a3, a5, a1
-; RV64IA-NEXT:    and a3, a3, a7
-; RV64IA-NEXT:    xor a3, a5, a3
+; RV64IA-NEXT:    xor a2, a5, a1
+; RV64IA-NEXT:    and a2, a2, a7
+; RV64IA-NEXT:    xor a2, a5, a2
 ; RV64IA-NEXT:  .LBB37_3: # in Loop: Header=BB37_1 Depth=1
-; RV64IA-NEXT:    sc.w.rl a3, a3, (a0)
-; RV64IA-NEXT:    bnez a3, .LBB37_1
+; RV64IA-NEXT:    sc.w.rl a2, a2, (a6)
+; RV64IA-NEXT:    bnez a2, .LBB37_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a5, a2
+; RV64IA-NEXT:    srlw a0, a5, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw max i8* %a, i8 %b release
   ret i8 %1
@@ -2541,33 +2541,33 @@ define i8 @atomicrmw_max_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    sw s3, 12(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lbu a0, 0(a0)
-; RV32I-NEXT:    slli a1, a1, 24
-; RV32I-NEXT:    srai s0, a1, 24
+; RV32I-NEXT:    lbu a1, 0(a0)
+; RV32I-NEXT:    slli a0, s2, 24
+; RV32I-NEXT:    srai s0, a0, 24
 ; RV32I-NEXT:    addi s3, sp, 11
 ; RV32I-NEXT:    j .LBB38_2
 ; RV32I-NEXT:  .LBB38_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB38_2 Depth=1
-; RV32I-NEXT:    sb a0, 11(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sb a1, 11(sp)
 ; RV32I-NEXT:    addi a3, zero, 4
 ; RV32I-NEXT:    addi a4, zero, 2
+; RV32I-NEXT:    mv a0, s1
+; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    call __atomic_compare_exchange_1
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lb a0, 11(sp)
-; RV32I-NEXT:    bnez a1, .LBB38_4
+; RV32I-NEXT:    lb a1, 11(sp)
+; RV32I-NEXT:    bnez a0, .LBB38_4
 ; RV32I-NEXT:  .LBB38_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    slli a1, a0, 24
-; RV32I-NEXT:    srai a1, a1, 24
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    blt s0, a1, .LBB38_1
+; RV32I-NEXT:    slli a0, a1, 24
+; RV32I-NEXT:    srai a0, a0, 24
+; RV32I-NEXT:    mv a2, a1
+; RV32I-NEXT:    blt s0, a0, .LBB38_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB38_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:    j .LBB38_1
 ; RV32I-NEXT:  .LBB38_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -2578,32 +2578,32 @@ define i8 @atomicrmw_max_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_max_i8_acq_rel:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    addi a3, zero, 24
-; RV32IA-NEXT:    sub a6, a3, a2
-; RV32IA-NEXT:    addi a4, zero, 255
-; RV32IA-NEXT:    sll a7, a4, a2
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    addi a3, zero, 255
+; RV32IA-NEXT:    sll a7, a3, a0
 ; RV32IA-NEXT:    slli a1, a1, 24
 ; RV32IA-NEXT:    srai a1, a1, 24
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    addi a4, zero, 24
+; RV32IA-NEXT:    sub a4, a4, a0
 ; RV32IA-NEXT:  .LBB38_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a5, (a0)
-; RV32IA-NEXT:    and a4, a5, a7
-; RV32IA-NEXT:    mv a3, a5
-; RV32IA-NEXT:    sll a4, a4, a6
-; RV32IA-NEXT:    sra a4, a4, a6
-; RV32IA-NEXT:    bge a4, a1, .LBB38_3
+; RV32IA-NEXT:    lr.w.aq a5, (a6)
+; RV32IA-NEXT:    and a3, a5, a7
+; RV32IA-NEXT:    mv a2, a5
+; RV32IA-NEXT:    sll a3, a3, a4
+; RV32IA-NEXT:    sra a3, a3, a4
+; RV32IA-NEXT:    bge a3, a1, .LBB38_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB38_1 Depth=1
-; RV32IA-NEXT:    xor a3, a5, a1
-; RV32IA-NEXT:    and a3, a3, a7
-; RV32IA-NEXT:    xor a3, a5, a3
+; RV32IA-NEXT:    xor a2, a5, a1
+; RV32IA-NEXT:    and a2, a2, a7
+; RV32IA-NEXT:    xor a2, a5, a2
 ; RV32IA-NEXT:  .LBB38_3: # in Loop: Header=BB38_1 Depth=1
-; RV32IA-NEXT:    sc.w.rl a3, a3, (a0)
-; RV32IA-NEXT:    bnez a3, .LBB38_1
+; RV32IA-NEXT:    sc.w.rl a2, a2, (a6)
+; RV32IA-NEXT:    bnez a2, .LBB38_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a5, a2
+; RV32IA-NEXT:    srl a0, a5, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_max_i8_acq_rel:
@@ -2616,33 +2616,33 @@ define i8 @atomicrmw_max_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    sd s3, 8(sp)
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lbu a0, 0(a0)
-; RV64I-NEXT:    slli a1, a1, 56
-; RV64I-NEXT:    srai s0, a1, 56
+; RV64I-NEXT:    lbu a1, 0(a0)
+; RV64I-NEXT:    slli a0, s2, 56
+; RV64I-NEXT:    srai s0, a0, 56
 ; RV64I-NEXT:    addi s3, sp, 7
 ; RV64I-NEXT:    j .LBB38_2
 ; RV64I-NEXT:  .LBB38_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB38_2 Depth=1
-; RV64I-NEXT:    sb a0, 7(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sb a1, 7(sp)
 ; RV64I-NEXT:    addi a3, zero, 4
 ; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    mv a0, s1
+; RV64I-NEXT:    mv a1, s3
 ; RV64I-NEXT:    call __atomic_compare_exchange_1
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lb a0, 7(sp)
-; RV64I-NEXT:    bnez a1, .LBB38_4
+; RV64I-NEXT:    lb a1, 7(sp)
+; RV64I-NEXT:    bnez a0, .LBB38_4
 ; RV64I-NEXT:  .LBB38_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    slli a1, a0, 56
-; RV64I-NEXT:    srai a1, a1, 56
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    blt s0, a1, .LBB38_1
+; RV64I-NEXT:    slli a0, a1, 56
+; RV64I-NEXT:    srai a0, a0, 56
+; RV64I-NEXT:    mv a2, a1
+; RV64I-NEXT:    blt s0, a0, .LBB38_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB38_2 Depth=1
 ; RV64I-NEXT:    mv a2, s2
 ; RV64I-NEXT:    j .LBB38_1
 ; RV64I-NEXT:  .LBB38_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -2653,32 +2653,32 @@ define i8 @atomicrmw_max_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_max_i8_acq_rel:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    addi a3, zero, 56
-; RV64IA-NEXT:    sub a6, a3, a2
-; RV64IA-NEXT:    addi a4, zero, 255
-; RV64IA-NEXT:    sllw a7, a4, a2
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    addi a3, zero, 255
+; RV64IA-NEXT:    sllw a7, a3, a0
 ; RV64IA-NEXT:    slli a1, a1, 56
 ; RV64IA-NEXT:    srai a1, a1, 56
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    addi a4, zero, 56
+; RV64IA-NEXT:    sub a4, a4, a0
 ; RV64IA-NEXT:  .LBB38_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a5, (a0)
-; RV64IA-NEXT:    and a4, a5, a7
-; RV64IA-NEXT:    mv a3, a5
-; RV64IA-NEXT:    sll a4, a4, a6
-; RV64IA-NEXT:    sra a4, a4, a6
-; RV64IA-NEXT:    bge a4, a1, .LBB38_3
+; RV64IA-NEXT:    lr.w.aq a5, (a6)
+; RV64IA-NEXT:    and a3, a5, a7
+; RV64IA-NEXT:    mv a2, a5
+; RV64IA-NEXT:    sll a3, a3, a4
+; RV64IA-NEXT:    sra a3, a3, a4
+; RV64IA-NEXT:    bge a3, a1, .LBB38_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB38_1 Depth=1
-; RV64IA-NEXT:    xor a3, a5, a1
-; RV64IA-NEXT:    and a3, a3, a7
-; RV64IA-NEXT:    xor a3, a5, a3
+; RV64IA-NEXT:    xor a2, a5, a1
+; RV64IA-NEXT:    and a2, a2, a7
+; RV64IA-NEXT:    xor a2, a5, a2
 ; RV64IA-NEXT:  .LBB38_3: # in Loop: Header=BB38_1 Depth=1
-; RV64IA-NEXT:    sc.w.rl a3, a3, (a0)
-; RV64IA-NEXT:    bnez a3, .LBB38_1
+; RV64IA-NEXT:    sc.w.rl a2, a2, (a6)
+; RV64IA-NEXT:    bnez a2, .LBB38_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a5, a2
+; RV64IA-NEXT:    srlw a0, a5, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw max i8* %a, i8 %b acq_rel
   ret i8 %1
@@ -2695,33 +2695,33 @@ define i8 @atomicrmw_max_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    sw s3, 12(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lbu a0, 0(a0)
-; RV32I-NEXT:    slli a1, a1, 24
-; RV32I-NEXT:    srai s0, a1, 24
+; RV32I-NEXT:    lbu a1, 0(a0)
+; RV32I-NEXT:    slli a0, s2, 24
+; RV32I-NEXT:    srai s0, a0, 24
 ; RV32I-NEXT:    addi s3, sp, 11
 ; RV32I-NEXT:    j .LBB39_2
 ; RV32I-NEXT:  .LBB39_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB39_2 Depth=1
-; RV32I-NEXT:    sb a0, 11(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sb a1, 11(sp)
 ; RV32I-NEXT:    addi a3, zero, 5
 ; RV32I-NEXT:    addi a4, zero, 5
+; RV32I-NEXT:    mv a0, s1
+; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    call __atomic_compare_exchange_1
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lb a0, 11(sp)
-; RV32I-NEXT:    bnez a1, .LBB39_4
+; RV32I-NEXT:    lb a1, 11(sp)
+; RV32I-NEXT:    bnez a0, .LBB39_4
 ; RV32I-NEXT:  .LBB39_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    slli a1, a0, 24
-; RV32I-NEXT:    srai a1, a1, 24
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    blt s0, a1, .LBB39_1
+; RV32I-NEXT:    slli a0, a1, 24
+; RV32I-NEXT:    srai a0, a0, 24
+; RV32I-NEXT:    mv a2, a1
+; RV32I-NEXT:    blt s0, a0, .LBB39_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB39_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:    j .LBB39_1
 ; RV32I-NEXT:  .LBB39_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -2732,32 +2732,32 @@ define i8 @atomicrmw_max_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_max_i8_seq_cst:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    addi a3, zero, 24
-; RV32IA-NEXT:    sub a6, a3, a2
-; RV32IA-NEXT:    addi a4, zero, 255
-; RV32IA-NEXT:    sll a7, a4, a2
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    addi a3, zero, 255
+; RV32IA-NEXT:    sll a7, a3, a0
 ; RV32IA-NEXT:    slli a1, a1, 24
 ; RV32IA-NEXT:    srai a1, a1, 24
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    addi a4, zero, 24
+; RV32IA-NEXT:    sub a4, a4, a0
 ; RV32IA-NEXT:  .LBB39_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aqrl a5, (a0)
-; RV32IA-NEXT:    and a4, a5, a7
-; RV32IA-NEXT:    mv a3, a5
-; RV32IA-NEXT:    sll a4, a4, a6
-; RV32IA-NEXT:    sra a4, a4, a6
-; RV32IA-NEXT:    bge a4, a1, .LBB39_3
+; RV32IA-NEXT:    lr.w.aqrl a5, (a6)
+; RV32IA-NEXT:    and a3, a5, a7
+; RV32IA-NEXT:    mv a2, a5
+; RV32IA-NEXT:    sll a3, a3, a4
+; RV32IA-NEXT:    sra a3, a3, a4
+; RV32IA-NEXT:    bge a3, a1, .LBB39_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB39_1 Depth=1
-; RV32IA-NEXT:    xor a3, a5, a1
-; RV32IA-NEXT:    and a3, a3, a7
-; RV32IA-NEXT:    xor a3, a5, a3
+; RV32IA-NEXT:    xor a2, a5, a1
+; RV32IA-NEXT:    and a2, a2, a7
+; RV32IA-NEXT:    xor a2, a5, a2
 ; RV32IA-NEXT:  .LBB39_3: # in Loop: Header=BB39_1 Depth=1
-; RV32IA-NEXT:    sc.w.aqrl a3, a3, (a0)
-; RV32IA-NEXT:    bnez a3, .LBB39_1
+; RV32IA-NEXT:    sc.w.aqrl a2, a2, (a6)
+; RV32IA-NEXT:    bnez a2, .LBB39_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a5, a2
+; RV32IA-NEXT:    srl a0, a5, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_max_i8_seq_cst:
@@ -2770,33 +2770,33 @@ define i8 @atomicrmw_max_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    sd s3, 8(sp)
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lbu a0, 0(a0)
-; RV64I-NEXT:    slli a1, a1, 56
-; RV64I-NEXT:    srai s0, a1, 56
+; RV64I-NEXT:    lbu a1, 0(a0)
+; RV64I-NEXT:    slli a0, s2, 56
+; RV64I-NEXT:    srai s0, a0, 56
 ; RV64I-NEXT:    addi s3, sp, 7
 ; RV64I-NEXT:    j .LBB39_2
 ; RV64I-NEXT:  .LBB39_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB39_2 Depth=1
-; RV64I-NEXT:    sb a0, 7(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sb a1, 7(sp)
 ; RV64I-NEXT:    addi a3, zero, 5
 ; RV64I-NEXT:    addi a4, zero, 5
+; RV64I-NEXT:    mv a0, s1
+; RV64I-NEXT:    mv a1, s3
 ; RV64I-NEXT:    call __atomic_compare_exchange_1
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lb a0, 7(sp)
-; RV64I-NEXT:    bnez a1, .LBB39_4
+; RV64I-NEXT:    lb a1, 7(sp)
+; RV64I-NEXT:    bnez a0, .LBB39_4
 ; RV64I-NEXT:  .LBB39_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    slli a1, a0, 56
-; RV64I-NEXT:    srai a1, a1, 56
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    blt s0, a1, .LBB39_1
+; RV64I-NEXT:    slli a0, a1, 56
+; RV64I-NEXT:    srai a0, a0, 56
+; RV64I-NEXT:    mv a2, a1
+; RV64I-NEXT:    blt s0, a0, .LBB39_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB39_2 Depth=1
 ; RV64I-NEXT:    mv a2, s2
 ; RV64I-NEXT:    j .LBB39_1
 ; RV64I-NEXT:  .LBB39_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -2807,32 +2807,32 @@ define i8 @atomicrmw_max_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_max_i8_seq_cst:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    addi a3, zero, 56
-; RV64IA-NEXT:    sub a6, a3, a2
-; RV64IA-NEXT:    addi a4, zero, 255
-; RV64IA-NEXT:    sllw a7, a4, a2
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    addi a3, zero, 255
+; RV64IA-NEXT:    sllw a7, a3, a0
 ; RV64IA-NEXT:    slli a1, a1, 56
 ; RV64IA-NEXT:    srai a1, a1, 56
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    addi a4, zero, 56
+; RV64IA-NEXT:    sub a4, a4, a0
 ; RV64IA-NEXT:  .LBB39_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aqrl a5, (a0)
-; RV64IA-NEXT:    and a4, a5, a7
-; RV64IA-NEXT:    mv a3, a5
-; RV64IA-NEXT:    sll a4, a4, a6
-; RV64IA-NEXT:    sra a4, a4, a6
-; RV64IA-NEXT:    bge a4, a1, .LBB39_3
+; RV64IA-NEXT:    lr.w.aqrl a5, (a6)
+; RV64IA-NEXT:    and a3, a5, a7
+; RV64IA-NEXT:    mv a2, a5
+; RV64IA-NEXT:    sll a3, a3, a4
+; RV64IA-NEXT:    sra a3, a3, a4
+; RV64IA-NEXT:    bge a3, a1, .LBB39_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB39_1 Depth=1
-; RV64IA-NEXT:    xor a3, a5, a1
-; RV64IA-NEXT:    and a3, a3, a7
-; RV64IA-NEXT:    xor a3, a5, a3
+; RV64IA-NEXT:    xor a2, a5, a1
+; RV64IA-NEXT:    and a2, a2, a7
+; RV64IA-NEXT:    xor a2, a5, a2
 ; RV64IA-NEXT:  .LBB39_3: # in Loop: Header=BB39_1 Depth=1
-; RV64IA-NEXT:    sc.w.aqrl a3, a3, (a0)
-; RV64IA-NEXT:    bnez a3, .LBB39_1
+; RV64IA-NEXT:    sc.w.aqrl a2, a2, (a6)
+; RV64IA-NEXT:    bnez a2, .LBB39_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a5, a2
+; RV64IA-NEXT:    srlw a0, a5, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw max i8* %a, i8 %b seq_cst
   ret i8 %1
@@ -2849,33 +2849,33 @@ define i8 @atomicrmw_min_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    sw s3, 12(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lbu a0, 0(a0)
-; RV32I-NEXT:    slli a1, a1, 24
-; RV32I-NEXT:    srai s0, a1, 24
+; RV32I-NEXT:    lbu a1, 0(a0)
+; RV32I-NEXT:    slli a0, s2, 24
+; RV32I-NEXT:    srai s0, a0, 24
 ; RV32I-NEXT:    addi s3, sp, 11
 ; RV32I-NEXT:    j .LBB40_2
 ; RV32I-NEXT:  .LBB40_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB40_2 Depth=1
-; RV32I-NEXT:    sb a0, 11(sp)
+; RV32I-NEXT:    sb a1, 11(sp)
 ; RV32I-NEXT:    mv a0, s1
 ; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    mv a3, zero
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_1
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lb a0, 11(sp)
-; RV32I-NEXT:    bnez a1, .LBB40_4
+; RV32I-NEXT:    lb a1, 11(sp)
+; RV32I-NEXT:    bnez a0, .LBB40_4
 ; RV32I-NEXT:  .LBB40_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    slli a1, a0, 24
-; RV32I-NEXT:    srai a1, a1, 24
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bge s0, a1, .LBB40_1
+; RV32I-NEXT:    slli a0, a1, 24
+; RV32I-NEXT:    srai a0, a0, 24
+; RV32I-NEXT:    mv a2, a1
+; RV32I-NEXT:    bge s0, a0, .LBB40_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB40_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:    j .LBB40_1
 ; RV32I-NEXT:  .LBB40_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -2886,32 +2886,32 @@ define i8 @atomicrmw_min_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_min_i8_monotonic:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    addi a3, zero, 24
-; RV32IA-NEXT:    sub a6, a3, a2
-; RV32IA-NEXT:    addi a4, zero, 255
-; RV32IA-NEXT:    sll a7, a4, a2
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    addi a3, zero, 255
+; RV32IA-NEXT:    sll a7, a3, a0
 ; RV32IA-NEXT:    slli a1, a1, 24
 ; RV32IA-NEXT:    srai a1, a1, 24
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    addi a4, zero, 24
+; RV32IA-NEXT:    sub a4, a4, a0
 ; RV32IA-NEXT:  .LBB40_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a5, (a0)
-; RV32IA-NEXT:    and a4, a5, a7
-; RV32IA-NEXT:    mv a3, a5
-; RV32IA-NEXT:    sll a4, a4, a6
-; RV32IA-NEXT:    sra a4, a4, a6
-; RV32IA-NEXT:    bge a1, a4, .LBB40_3
+; RV32IA-NEXT:    lr.w a5, (a6)
+; RV32IA-NEXT:    and a3, a5, a7
+; RV32IA-NEXT:    mv a2, a5
+; RV32IA-NEXT:    sll a3, a3, a4
+; RV32IA-NEXT:    sra a3, a3, a4
+; RV32IA-NEXT:    bge a1, a3, .LBB40_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB40_1 Depth=1
-; RV32IA-NEXT:    xor a3, a5, a1
-; RV32IA-NEXT:    and a3, a3, a7
-; RV32IA-NEXT:    xor a3, a5, a3
+; RV32IA-NEXT:    xor a2, a5, a1
+; RV32IA-NEXT:    and a2, a2, a7
+; RV32IA-NEXT:    xor a2, a5, a2
 ; RV32IA-NEXT:  .LBB40_3: # in Loop: Header=BB40_1 Depth=1
-; RV32IA-NEXT:    sc.w a3, a3, (a0)
-; RV32IA-NEXT:    bnez a3, .LBB40_1
+; RV32IA-NEXT:    sc.w a2, a2, (a6)
+; RV32IA-NEXT:    bnez a2, .LBB40_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a5, a2
+; RV32IA-NEXT:    srl a0, a5, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_min_i8_monotonic:
@@ -2924,33 +2924,33 @@ define i8 @atomicrmw_min_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    sd s3, 8(sp)
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lbu a0, 0(a0)
-; RV64I-NEXT:    slli a1, a1, 56
-; RV64I-NEXT:    srai s0, a1, 56
+; RV64I-NEXT:    lbu a1, 0(a0)
+; RV64I-NEXT:    slli a0, s2, 56
+; RV64I-NEXT:    srai s0, a0, 56
 ; RV64I-NEXT:    addi s3, sp, 7
 ; RV64I-NEXT:    j .LBB40_2
 ; RV64I-NEXT:  .LBB40_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB40_2 Depth=1
-; RV64I-NEXT:    sb a0, 7(sp)
+; RV64I-NEXT:    sb a1, 7(sp)
 ; RV64I-NEXT:    mv a0, s1
 ; RV64I-NEXT:    mv a1, s3
 ; RV64I-NEXT:    mv a3, zero
 ; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    call __atomic_compare_exchange_1
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lb a0, 7(sp)
-; RV64I-NEXT:    bnez a1, .LBB40_4
+; RV64I-NEXT:    lb a1, 7(sp)
+; RV64I-NEXT:    bnez a0, .LBB40_4
 ; RV64I-NEXT:  .LBB40_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    slli a1, a0, 56
-; RV64I-NEXT:    srai a1, a1, 56
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bge s0, a1, .LBB40_1
+; RV64I-NEXT:    slli a0, a1, 56
+; RV64I-NEXT:    srai a0, a0, 56
+; RV64I-NEXT:    mv a2, a1
+; RV64I-NEXT:    bge s0, a0, .LBB40_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB40_2 Depth=1
 ; RV64I-NEXT:    mv a2, s2
 ; RV64I-NEXT:    j .LBB40_1
 ; RV64I-NEXT:  .LBB40_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -2961,32 +2961,32 @@ define i8 @atomicrmw_min_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_min_i8_monotonic:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    addi a3, zero, 56
-; RV64IA-NEXT:    sub a6, a3, a2
-; RV64IA-NEXT:    addi a4, zero, 255
-; RV64IA-NEXT:    sllw a7, a4, a2
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    addi a3, zero, 255
+; RV64IA-NEXT:    sllw a7, a3, a0
 ; RV64IA-NEXT:    slli a1, a1, 56
 ; RV64IA-NEXT:    srai a1, a1, 56
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    addi a4, zero, 56
+; RV64IA-NEXT:    sub a4, a4, a0
 ; RV64IA-NEXT:  .LBB40_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a5, (a0)
-; RV64IA-NEXT:    and a4, a5, a7
-; RV64IA-NEXT:    mv a3, a5
-; RV64IA-NEXT:    sll a4, a4, a6
-; RV64IA-NEXT:    sra a4, a4, a6
-; RV64IA-NEXT:    bge a1, a4, .LBB40_3
+; RV64IA-NEXT:    lr.w a5, (a6)
+; RV64IA-NEXT:    and a3, a5, a7
+; RV64IA-NEXT:    mv a2, a5
+; RV64IA-NEXT:    sll a3, a3, a4
+; RV64IA-NEXT:    sra a3, a3, a4
+; RV64IA-NEXT:    bge a1, a3, .LBB40_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB40_1 Depth=1
-; RV64IA-NEXT:    xor a3, a5, a1
-; RV64IA-NEXT:    and a3, a3, a7
-; RV64IA-NEXT:    xor a3, a5, a3
+; RV64IA-NEXT:    xor a2, a5, a1
+; RV64IA-NEXT:    and a2, a2, a7
+; RV64IA-NEXT:    xor a2, a5, a2
 ; RV64IA-NEXT:  .LBB40_3: # in Loop: Header=BB40_1 Depth=1
-; RV64IA-NEXT:    sc.w a3, a3, (a0)
-; RV64IA-NEXT:    bnez a3, .LBB40_1
+; RV64IA-NEXT:    sc.w a2, a2, (a6)
+; RV64IA-NEXT:    bnez a2, .LBB40_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a5, a2
+; RV64IA-NEXT:    srlw a0, a5, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw min i8* %a, i8 %b monotonic
   ret i8 %1
@@ -3003,33 +3003,33 @@ define i8 @atomicrmw_min_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    sw s3, 12(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lbu a0, 0(a0)
-; RV32I-NEXT:    slli a1, a1, 24
-; RV32I-NEXT:    srai s0, a1, 24
+; RV32I-NEXT:    lbu a1, 0(a0)
+; RV32I-NEXT:    slli a0, s2, 24
+; RV32I-NEXT:    srai s0, a0, 24
 ; RV32I-NEXT:    addi s3, sp, 11
 ; RV32I-NEXT:    j .LBB41_2
 ; RV32I-NEXT:  .LBB41_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB41_2 Depth=1
-; RV32I-NEXT:    sb a0, 11(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sb a1, 11(sp)
 ; RV32I-NEXT:    addi a3, zero, 2
 ; RV32I-NEXT:    addi a4, zero, 2
+; RV32I-NEXT:    mv a0, s1
+; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    call __atomic_compare_exchange_1
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lb a0, 11(sp)
-; RV32I-NEXT:    bnez a1, .LBB41_4
+; RV32I-NEXT:    lb a1, 11(sp)
+; RV32I-NEXT:    bnez a0, .LBB41_4
 ; RV32I-NEXT:  .LBB41_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    slli a1, a0, 24
-; RV32I-NEXT:    srai a1, a1, 24
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bge s0, a1, .LBB41_1
+; RV32I-NEXT:    slli a0, a1, 24
+; RV32I-NEXT:    srai a0, a0, 24
+; RV32I-NEXT:    mv a2, a1
+; RV32I-NEXT:    bge s0, a0, .LBB41_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB41_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:    j .LBB41_1
 ; RV32I-NEXT:  .LBB41_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -3040,32 +3040,32 @@ define i8 @atomicrmw_min_i8_acquire(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_min_i8_acquire:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    addi a3, zero, 24
-; RV32IA-NEXT:    sub a6, a3, a2
-; RV32IA-NEXT:    addi a4, zero, 255
-; RV32IA-NEXT:    sll a7, a4, a2
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    addi a3, zero, 255
+; RV32IA-NEXT:    sll a7, a3, a0
 ; RV32IA-NEXT:    slli a1, a1, 24
 ; RV32IA-NEXT:    srai a1, a1, 24
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    addi a4, zero, 24
+; RV32IA-NEXT:    sub a4, a4, a0
 ; RV32IA-NEXT:  .LBB41_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a5, (a0)
-; RV32IA-NEXT:    and a4, a5, a7
-; RV32IA-NEXT:    mv a3, a5
-; RV32IA-NEXT:    sll a4, a4, a6
-; RV32IA-NEXT:    sra a4, a4, a6
-; RV32IA-NEXT:    bge a1, a4, .LBB41_3
+; RV32IA-NEXT:    lr.w.aq a5, (a6)
+; RV32IA-NEXT:    and a3, a5, a7
+; RV32IA-NEXT:    mv a2, a5
+; RV32IA-NEXT:    sll a3, a3, a4
+; RV32IA-NEXT:    sra a3, a3, a4
+; RV32IA-NEXT:    bge a1, a3, .LBB41_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB41_1 Depth=1
-; RV32IA-NEXT:    xor a3, a5, a1
-; RV32IA-NEXT:    and a3, a3, a7
-; RV32IA-NEXT:    xor a3, a5, a3
+; RV32IA-NEXT:    xor a2, a5, a1
+; RV32IA-NEXT:    and a2, a2, a7
+; RV32IA-NEXT:    xor a2, a5, a2
 ; RV32IA-NEXT:  .LBB41_3: # in Loop: Header=BB41_1 Depth=1
-; RV32IA-NEXT:    sc.w a3, a3, (a0)
-; RV32IA-NEXT:    bnez a3, .LBB41_1
+; RV32IA-NEXT:    sc.w a2, a2, (a6)
+; RV32IA-NEXT:    bnez a2, .LBB41_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a5, a2
+; RV32IA-NEXT:    srl a0, a5, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_min_i8_acquire:
@@ -3078,33 +3078,33 @@ define i8 @atomicrmw_min_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    sd s3, 8(sp)
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lbu a0, 0(a0)
-; RV64I-NEXT:    slli a1, a1, 56
-; RV64I-NEXT:    srai s0, a1, 56
+; RV64I-NEXT:    lbu a1, 0(a0)
+; RV64I-NEXT:    slli a0, s2, 56
+; RV64I-NEXT:    srai s0, a0, 56
 ; RV64I-NEXT:    addi s3, sp, 7
 ; RV64I-NEXT:    j .LBB41_2
 ; RV64I-NEXT:  .LBB41_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB41_2 Depth=1
-; RV64I-NEXT:    sb a0, 7(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sb a1, 7(sp)
 ; RV64I-NEXT:    addi a3, zero, 2
 ; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    mv a0, s1
+; RV64I-NEXT:    mv a1, s3
 ; RV64I-NEXT:    call __atomic_compare_exchange_1
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lb a0, 7(sp)
-; RV64I-NEXT:    bnez a1, .LBB41_4
+; RV64I-NEXT:    lb a1, 7(sp)
+; RV64I-NEXT:    bnez a0, .LBB41_4
 ; RV64I-NEXT:  .LBB41_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    slli a1, a0, 56
-; RV64I-NEXT:    srai a1, a1, 56
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bge s0, a1, .LBB41_1
+; RV64I-NEXT:    slli a0, a1, 56
+; RV64I-NEXT:    srai a0, a0, 56
+; RV64I-NEXT:    mv a2, a1
+; RV64I-NEXT:    bge s0, a0, .LBB41_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB41_2 Depth=1
 ; RV64I-NEXT:    mv a2, s2
 ; RV64I-NEXT:    j .LBB41_1
 ; RV64I-NEXT:  .LBB41_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -3115,32 +3115,32 @@ define i8 @atomicrmw_min_i8_acquire(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_min_i8_acquire:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    addi a3, zero, 56
-; RV64IA-NEXT:    sub a6, a3, a2
-; RV64IA-NEXT:    addi a4, zero, 255
-; RV64IA-NEXT:    sllw a7, a4, a2
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    addi a3, zero, 255
+; RV64IA-NEXT:    sllw a7, a3, a0
 ; RV64IA-NEXT:    slli a1, a1, 56
 ; RV64IA-NEXT:    srai a1, a1, 56
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    addi a4, zero, 56
+; RV64IA-NEXT:    sub a4, a4, a0
 ; RV64IA-NEXT:  .LBB41_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a5, (a0)
-; RV64IA-NEXT:    and a4, a5, a7
-; RV64IA-NEXT:    mv a3, a5
-; RV64IA-NEXT:    sll a4, a4, a6
-; RV64IA-NEXT:    sra a4, a4, a6
-; RV64IA-NEXT:    bge a1, a4, .LBB41_3
+; RV64IA-NEXT:    lr.w.aq a5, (a6)
+; RV64IA-NEXT:    and a3, a5, a7
+; RV64IA-NEXT:    mv a2, a5
+; RV64IA-NEXT:    sll a3, a3, a4
+; RV64IA-NEXT:    sra a3, a3, a4
+; RV64IA-NEXT:    bge a1, a3, .LBB41_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB41_1 Depth=1
-; RV64IA-NEXT:    xor a3, a5, a1
-; RV64IA-NEXT:    and a3, a3, a7
-; RV64IA-NEXT:    xor a3, a5, a3
+; RV64IA-NEXT:    xor a2, a5, a1
+; RV64IA-NEXT:    and a2, a2, a7
+; RV64IA-NEXT:    xor a2, a5, a2
 ; RV64IA-NEXT:  .LBB41_3: # in Loop: Header=BB41_1 Depth=1
-; RV64IA-NEXT:    sc.w a3, a3, (a0)
-; RV64IA-NEXT:    bnez a3, .LBB41_1
+; RV64IA-NEXT:    sc.w a2, a2, (a6)
+; RV64IA-NEXT:    bnez a2, .LBB41_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a5, a2
+; RV64IA-NEXT:    srlw a0, a5, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw min i8* %a, i8 %b acquire
   ret i8 %1
@@ -3157,33 +3157,33 @@ define i8 @atomicrmw_min_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    sw s3, 12(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lbu a0, 0(a0)
-; RV32I-NEXT:    slli a1, a1, 24
-; RV32I-NEXT:    srai s0, a1, 24
+; RV32I-NEXT:    lbu a1, 0(a0)
+; RV32I-NEXT:    slli a0, s2, 24
+; RV32I-NEXT:    srai s0, a0, 24
 ; RV32I-NEXT:    addi s3, sp, 11
 ; RV32I-NEXT:    j .LBB42_2
 ; RV32I-NEXT:  .LBB42_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB42_2 Depth=1
-; RV32I-NEXT:    sb a0, 11(sp)
+; RV32I-NEXT:    sb a1, 11(sp)
+; RV32I-NEXT:    addi a3, zero, 3
 ; RV32I-NEXT:    mv a0, s1
 ; RV32I-NEXT:    mv a1, s3
-; RV32I-NEXT:    addi a3, zero, 3
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_1
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lb a0, 11(sp)
-; RV32I-NEXT:    bnez a1, .LBB42_4
+; RV32I-NEXT:    lb a1, 11(sp)
+; RV32I-NEXT:    bnez a0, .LBB42_4
 ; RV32I-NEXT:  .LBB42_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    slli a1, a0, 24
-; RV32I-NEXT:    srai a1, a1, 24
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bge s0, a1, .LBB42_1
+; RV32I-NEXT:    slli a0, a1, 24
+; RV32I-NEXT:    srai a0, a0, 24
+; RV32I-NEXT:    mv a2, a1
+; RV32I-NEXT:    bge s0, a0, .LBB42_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB42_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:    j .LBB42_1
 ; RV32I-NEXT:  .LBB42_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -3194,32 +3194,32 @@ define i8 @atomicrmw_min_i8_release(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_min_i8_release:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    addi a3, zero, 24
-; RV32IA-NEXT:    sub a6, a3, a2
-; RV32IA-NEXT:    addi a4, zero, 255
-; RV32IA-NEXT:    sll a7, a4, a2
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    addi a3, zero, 255
+; RV32IA-NEXT:    sll a7, a3, a0
 ; RV32IA-NEXT:    slli a1, a1, 24
 ; RV32IA-NEXT:    srai a1, a1, 24
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    addi a4, zero, 24
+; RV32IA-NEXT:    sub a4, a4, a0
 ; RV32IA-NEXT:  .LBB42_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a5, (a0)
-; RV32IA-NEXT:    and a4, a5, a7
-; RV32IA-NEXT:    mv a3, a5
-; RV32IA-NEXT:    sll a4, a4, a6
-; RV32IA-NEXT:    sra a4, a4, a6
-; RV32IA-NEXT:    bge a1, a4, .LBB42_3
+; RV32IA-NEXT:    lr.w a5, (a6)
+; RV32IA-NEXT:    and a3, a5, a7
+; RV32IA-NEXT:    mv a2, a5
+; RV32IA-NEXT:    sll a3, a3, a4
+; RV32IA-NEXT:    sra a3, a3, a4
+; RV32IA-NEXT:    bge a1, a3, .LBB42_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB42_1 Depth=1
-; RV32IA-NEXT:    xor a3, a5, a1
-; RV32IA-NEXT:    and a3, a3, a7
-; RV32IA-NEXT:    xor a3, a5, a3
+; RV32IA-NEXT:    xor a2, a5, a1
+; RV32IA-NEXT:    and a2, a2, a7
+; RV32IA-NEXT:    xor a2, a5, a2
 ; RV32IA-NEXT:  .LBB42_3: # in Loop: Header=BB42_1 Depth=1
-; RV32IA-NEXT:    sc.w.rl a3, a3, (a0)
-; RV32IA-NEXT:    bnez a3, .LBB42_1
+; RV32IA-NEXT:    sc.w.rl a2, a2, (a6)
+; RV32IA-NEXT:    bnez a2, .LBB42_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a5, a2
+; RV32IA-NEXT:    srl a0, a5, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_min_i8_release:
@@ -3232,33 +3232,33 @@ define i8 @atomicrmw_min_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    sd s3, 8(sp)
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lbu a0, 0(a0)
-; RV64I-NEXT:    slli a1, a1, 56
-; RV64I-NEXT:    srai s0, a1, 56
+; RV64I-NEXT:    lbu a1, 0(a0)
+; RV64I-NEXT:    slli a0, s2, 56
+; RV64I-NEXT:    srai s0, a0, 56
 ; RV64I-NEXT:    addi s3, sp, 7
 ; RV64I-NEXT:    j .LBB42_2
 ; RV64I-NEXT:  .LBB42_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB42_2 Depth=1
-; RV64I-NEXT:    sb a0, 7(sp)
+; RV64I-NEXT:    sb a1, 7(sp)
+; RV64I-NEXT:    addi a3, zero, 3
 ; RV64I-NEXT:    mv a0, s1
 ; RV64I-NEXT:    mv a1, s3
-; RV64I-NEXT:    addi a3, zero, 3
 ; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    call __atomic_compare_exchange_1
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lb a0, 7(sp)
-; RV64I-NEXT:    bnez a1, .LBB42_4
+; RV64I-NEXT:    lb a1, 7(sp)
+; RV64I-NEXT:    bnez a0, .LBB42_4
 ; RV64I-NEXT:  .LBB42_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    slli a1, a0, 56
-; RV64I-NEXT:    srai a1, a1, 56
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bge s0, a1, .LBB42_1
+; RV64I-NEXT:    slli a0, a1, 56
+; RV64I-NEXT:    srai a0, a0, 56
+; RV64I-NEXT:    mv a2, a1
+; RV64I-NEXT:    bge s0, a0, .LBB42_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB42_2 Depth=1
 ; RV64I-NEXT:    mv a2, s2
 ; RV64I-NEXT:    j .LBB42_1
 ; RV64I-NEXT:  .LBB42_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -3269,32 +3269,32 @@ define i8 @atomicrmw_min_i8_release(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_min_i8_release:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    addi a3, zero, 56
-; RV64IA-NEXT:    sub a6, a3, a2
-; RV64IA-NEXT:    addi a4, zero, 255
-; RV64IA-NEXT:    sllw a7, a4, a2
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    addi a3, zero, 255
+; RV64IA-NEXT:    sllw a7, a3, a0
 ; RV64IA-NEXT:    slli a1, a1, 56
 ; RV64IA-NEXT:    srai a1, a1, 56
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    addi a4, zero, 56
+; RV64IA-NEXT:    sub a4, a4, a0
 ; RV64IA-NEXT:  .LBB42_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a5, (a0)
-; RV64IA-NEXT:    and a4, a5, a7
-; RV64IA-NEXT:    mv a3, a5
-; RV64IA-NEXT:    sll a4, a4, a6
-; RV64IA-NEXT:    sra a4, a4, a6
-; RV64IA-NEXT:    bge a1, a4, .LBB42_3
+; RV64IA-NEXT:    lr.w a5, (a6)
+; RV64IA-NEXT:    and a3, a5, a7
+; RV64IA-NEXT:    mv a2, a5
+; RV64IA-NEXT:    sll a3, a3, a4
+; RV64IA-NEXT:    sra a3, a3, a4
+; RV64IA-NEXT:    bge a1, a3, .LBB42_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB42_1 Depth=1
-; RV64IA-NEXT:    xor a3, a5, a1
-; RV64IA-NEXT:    and a3, a3, a7
-; RV64IA-NEXT:    xor a3, a5, a3
+; RV64IA-NEXT:    xor a2, a5, a1
+; RV64IA-NEXT:    and a2, a2, a7
+; RV64IA-NEXT:    xor a2, a5, a2
 ; RV64IA-NEXT:  .LBB42_3: # in Loop: Header=BB42_1 Depth=1
-; RV64IA-NEXT:    sc.w.rl a3, a3, (a0)
-; RV64IA-NEXT:    bnez a3, .LBB42_1
+; RV64IA-NEXT:    sc.w.rl a2, a2, (a6)
+; RV64IA-NEXT:    bnez a2, .LBB42_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a5, a2
+; RV64IA-NEXT:    srlw a0, a5, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw min i8* %a, i8 %b release
   ret i8 %1
@@ -3311,33 +3311,33 @@ define i8 @atomicrmw_min_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    sw s3, 12(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lbu a0, 0(a0)
-; RV32I-NEXT:    slli a1, a1, 24
-; RV32I-NEXT:    srai s0, a1, 24
+; RV32I-NEXT:    lbu a1, 0(a0)
+; RV32I-NEXT:    slli a0, s2, 24
+; RV32I-NEXT:    srai s0, a0, 24
 ; RV32I-NEXT:    addi s3, sp, 11
 ; RV32I-NEXT:    j .LBB43_2
 ; RV32I-NEXT:  .LBB43_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB43_2 Depth=1
-; RV32I-NEXT:    sb a0, 11(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sb a1, 11(sp)
 ; RV32I-NEXT:    addi a3, zero, 4
 ; RV32I-NEXT:    addi a4, zero, 2
+; RV32I-NEXT:    mv a0, s1
+; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    call __atomic_compare_exchange_1
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lb a0, 11(sp)
-; RV32I-NEXT:    bnez a1, .LBB43_4
+; RV32I-NEXT:    lb a1, 11(sp)
+; RV32I-NEXT:    bnez a0, .LBB43_4
 ; RV32I-NEXT:  .LBB43_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    slli a1, a0, 24
-; RV32I-NEXT:    srai a1, a1, 24
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bge s0, a1, .LBB43_1
+; RV32I-NEXT:    slli a0, a1, 24
+; RV32I-NEXT:    srai a0, a0, 24
+; RV32I-NEXT:    mv a2, a1
+; RV32I-NEXT:    bge s0, a0, .LBB43_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB43_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:    j .LBB43_1
 ; RV32I-NEXT:  .LBB43_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -3348,32 +3348,32 @@ define i8 @atomicrmw_min_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_min_i8_acq_rel:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    addi a3, zero, 24
-; RV32IA-NEXT:    sub a6, a3, a2
-; RV32IA-NEXT:    addi a4, zero, 255
-; RV32IA-NEXT:    sll a7, a4, a2
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    addi a3, zero, 255
+; RV32IA-NEXT:    sll a7, a3, a0
 ; RV32IA-NEXT:    slli a1, a1, 24
 ; RV32IA-NEXT:    srai a1, a1, 24
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    addi a4, zero, 24
+; RV32IA-NEXT:    sub a4, a4, a0
 ; RV32IA-NEXT:  .LBB43_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a5, (a0)
-; RV32IA-NEXT:    and a4, a5, a7
-; RV32IA-NEXT:    mv a3, a5
-; RV32IA-NEXT:    sll a4, a4, a6
-; RV32IA-NEXT:    sra a4, a4, a6
-; RV32IA-NEXT:    bge a1, a4, .LBB43_3
+; RV32IA-NEXT:    lr.w.aq a5, (a6)
+; RV32IA-NEXT:    and a3, a5, a7
+; RV32IA-NEXT:    mv a2, a5
+; RV32IA-NEXT:    sll a3, a3, a4
+; RV32IA-NEXT:    sra a3, a3, a4
+; RV32IA-NEXT:    bge a1, a3, .LBB43_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB43_1 Depth=1
-; RV32IA-NEXT:    xor a3, a5, a1
-; RV32IA-NEXT:    and a3, a3, a7
-; RV32IA-NEXT:    xor a3, a5, a3
+; RV32IA-NEXT:    xor a2, a5, a1
+; RV32IA-NEXT:    and a2, a2, a7
+; RV32IA-NEXT:    xor a2, a5, a2
 ; RV32IA-NEXT:  .LBB43_3: # in Loop: Header=BB43_1 Depth=1
-; RV32IA-NEXT:    sc.w.rl a3, a3, (a0)
-; RV32IA-NEXT:    bnez a3, .LBB43_1
+; RV32IA-NEXT:    sc.w.rl a2, a2, (a6)
+; RV32IA-NEXT:    bnez a2, .LBB43_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a5, a2
+; RV32IA-NEXT:    srl a0, a5, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_min_i8_acq_rel:
@@ -3386,33 +3386,33 @@ define i8 @atomicrmw_min_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    sd s3, 8(sp)
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lbu a0, 0(a0)
-; RV64I-NEXT:    slli a1, a1, 56
-; RV64I-NEXT:    srai s0, a1, 56
+; RV64I-NEXT:    lbu a1, 0(a0)
+; RV64I-NEXT:    slli a0, s2, 56
+; RV64I-NEXT:    srai s0, a0, 56
 ; RV64I-NEXT:    addi s3, sp, 7
 ; RV64I-NEXT:    j .LBB43_2
 ; RV64I-NEXT:  .LBB43_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB43_2 Depth=1
-; RV64I-NEXT:    sb a0, 7(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sb a1, 7(sp)
 ; RV64I-NEXT:    addi a3, zero, 4
 ; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    mv a0, s1
+; RV64I-NEXT:    mv a1, s3
 ; RV64I-NEXT:    call __atomic_compare_exchange_1
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lb a0, 7(sp)
-; RV64I-NEXT:    bnez a1, .LBB43_4
+; RV64I-NEXT:    lb a1, 7(sp)
+; RV64I-NEXT:    bnez a0, .LBB43_4
 ; RV64I-NEXT:  .LBB43_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    slli a1, a0, 56
-; RV64I-NEXT:    srai a1, a1, 56
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bge s0, a1, .LBB43_1
+; RV64I-NEXT:    slli a0, a1, 56
+; RV64I-NEXT:    srai a0, a0, 56
+; RV64I-NEXT:    mv a2, a1
+; RV64I-NEXT:    bge s0, a0, .LBB43_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB43_2 Depth=1
 ; RV64I-NEXT:    mv a2, s2
 ; RV64I-NEXT:    j .LBB43_1
 ; RV64I-NEXT:  .LBB43_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -3423,32 +3423,32 @@ define i8 @atomicrmw_min_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_min_i8_acq_rel:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    addi a3, zero, 56
-; RV64IA-NEXT:    sub a6, a3, a2
-; RV64IA-NEXT:    addi a4, zero, 255
-; RV64IA-NEXT:    sllw a7, a4, a2
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    addi a3, zero, 255
+; RV64IA-NEXT:    sllw a7, a3, a0
 ; RV64IA-NEXT:    slli a1, a1, 56
 ; RV64IA-NEXT:    srai a1, a1, 56
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    addi a4, zero, 56
+; RV64IA-NEXT:    sub a4, a4, a0
 ; RV64IA-NEXT:  .LBB43_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a5, (a0)
-; RV64IA-NEXT:    and a4, a5, a7
-; RV64IA-NEXT:    mv a3, a5
-; RV64IA-NEXT:    sll a4, a4, a6
-; RV64IA-NEXT:    sra a4, a4, a6
-; RV64IA-NEXT:    bge a1, a4, .LBB43_3
+; RV64IA-NEXT:    lr.w.aq a5, (a6)
+; RV64IA-NEXT:    and a3, a5, a7
+; RV64IA-NEXT:    mv a2, a5
+; RV64IA-NEXT:    sll a3, a3, a4
+; RV64IA-NEXT:    sra a3, a3, a4
+; RV64IA-NEXT:    bge a1, a3, .LBB43_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB43_1 Depth=1
-; RV64IA-NEXT:    xor a3, a5, a1
-; RV64IA-NEXT:    and a3, a3, a7
-; RV64IA-NEXT:    xor a3, a5, a3
+; RV64IA-NEXT:    xor a2, a5, a1
+; RV64IA-NEXT:    and a2, a2, a7
+; RV64IA-NEXT:    xor a2, a5, a2
 ; RV64IA-NEXT:  .LBB43_3: # in Loop: Header=BB43_1 Depth=1
-; RV64IA-NEXT:    sc.w.rl a3, a3, (a0)
-; RV64IA-NEXT:    bnez a3, .LBB43_1
+; RV64IA-NEXT:    sc.w.rl a2, a2, (a6)
+; RV64IA-NEXT:    bnez a2, .LBB43_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a5, a2
+; RV64IA-NEXT:    srlw a0, a5, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw min i8* %a, i8 %b acq_rel
   ret i8 %1
@@ -3465,33 +3465,33 @@ define i8 @atomicrmw_min_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    sw s3, 12(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lbu a0, 0(a0)
-; RV32I-NEXT:    slli a1, a1, 24
-; RV32I-NEXT:    srai s0, a1, 24
+; RV32I-NEXT:    lbu a1, 0(a0)
+; RV32I-NEXT:    slli a0, s2, 24
+; RV32I-NEXT:    srai s0, a0, 24
 ; RV32I-NEXT:    addi s3, sp, 11
 ; RV32I-NEXT:    j .LBB44_2
 ; RV32I-NEXT:  .LBB44_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB44_2 Depth=1
-; RV32I-NEXT:    sb a0, 11(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sb a1, 11(sp)
 ; RV32I-NEXT:    addi a3, zero, 5
 ; RV32I-NEXT:    addi a4, zero, 5
+; RV32I-NEXT:    mv a0, s1
+; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    call __atomic_compare_exchange_1
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lb a0, 11(sp)
-; RV32I-NEXT:    bnez a1, .LBB44_4
+; RV32I-NEXT:    lb a1, 11(sp)
+; RV32I-NEXT:    bnez a0, .LBB44_4
 ; RV32I-NEXT:  .LBB44_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    slli a1, a0, 24
-; RV32I-NEXT:    srai a1, a1, 24
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bge s0, a1, .LBB44_1
+; RV32I-NEXT:    slli a0, a1, 24
+; RV32I-NEXT:    srai a0, a0, 24
+; RV32I-NEXT:    mv a2, a1
+; RV32I-NEXT:    bge s0, a0, .LBB44_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB44_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:    j .LBB44_1
 ; RV32I-NEXT:  .LBB44_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -3502,32 +3502,32 @@ define i8 @atomicrmw_min_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_min_i8_seq_cst:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    addi a3, zero, 24
-; RV32IA-NEXT:    sub a6, a3, a2
-; RV32IA-NEXT:    addi a4, zero, 255
-; RV32IA-NEXT:    sll a7, a4, a2
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    addi a3, zero, 255
+; RV32IA-NEXT:    sll a7, a3, a0
 ; RV32IA-NEXT:    slli a1, a1, 24
 ; RV32IA-NEXT:    srai a1, a1, 24
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    addi a4, zero, 24
+; RV32IA-NEXT:    sub a4, a4, a0
 ; RV32IA-NEXT:  .LBB44_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aqrl a5, (a0)
-; RV32IA-NEXT:    and a4, a5, a7
-; RV32IA-NEXT:    mv a3, a5
-; RV32IA-NEXT:    sll a4, a4, a6
-; RV32IA-NEXT:    sra a4, a4, a6
-; RV32IA-NEXT:    bge a1, a4, .LBB44_3
+; RV32IA-NEXT:    lr.w.aqrl a5, (a6)
+; RV32IA-NEXT:    and a3, a5, a7
+; RV32IA-NEXT:    mv a2, a5
+; RV32IA-NEXT:    sll a3, a3, a4
+; RV32IA-NEXT:    sra a3, a3, a4
+; RV32IA-NEXT:    bge a1, a3, .LBB44_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB44_1 Depth=1
-; RV32IA-NEXT:    xor a3, a5, a1
-; RV32IA-NEXT:    and a3, a3, a7
-; RV32IA-NEXT:    xor a3, a5, a3
+; RV32IA-NEXT:    xor a2, a5, a1
+; RV32IA-NEXT:    and a2, a2, a7
+; RV32IA-NEXT:    xor a2, a5, a2
 ; RV32IA-NEXT:  .LBB44_3: # in Loop: Header=BB44_1 Depth=1
-; RV32IA-NEXT:    sc.w.aqrl a3, a3, (a0)
-; RV32IA-NEXT:    bnez a3, .LBB44_1
+; RV32IA-NEXT:    sc.w.aqrl a2, a2, (a6)
+; RV32IA-NEXT:    bnez a2, .LBB44_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a5, a2
+; RV32IA-NEXT:    srl a0, a5, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_min_i8_seq_cst:
@@ -3540,33 +3540,33 @@ define i8 @atomicrmw_min_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    sd s3, 8(sp)
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lbu a0, 0(a0)
-; RV64I-NEXT:    slli a1, a1, 56
-; RV64I-NEXT:    srai s0, a1, 56
+; RV64I-NEXT:    lbu a1, 0(a0)
+; RV64I-NEXT:    slli a0, s2, 56
+; RV64I-NEXT:    srai s0, a0, 56
 ; RV64I-NEXT:    addi s3, sp, 7
 ; RV64I-NEXT:    j .LBB44_2
 ; RV64I-NEXT:  .LBB44_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB44_2 Depth=1
-; RV64I-NEXT:    sb a0, 7(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sb a1, 7(sp)
 ; RV64I-NEXT:    addi a3, zero, 5
 ; RV64I-NEXT:    addi a4, zero, 5
+; RV64I-NEXT:    mv a0, s1
+; RV64I-NEXT:    mv a1, s3
 ; RV64I-NEXT:    call __atomic_compare_exchange_1
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lb a0, 7(sp)
-; RV64I-NEXT:    bnez a1, .LBB44_4
+; RV64I-NEXT:    lb a1, 7(sp)
+; RV64I-NEXT:    bnez a0, .LBB44_4
 ; RV64I-NEXT:  .LBB44_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    slli a1, a0, 56
-; RV64I-NEXT:    srai a1, a1, 56
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bge s0, a1, .LBB44_1
+; RV64I-NEXT:    slli a0, a1, 56
+; RV64I-NEXT:    srai a0, a0, 56
+; RV64I-NEXT:    mv a2, a1
+; RV64I-NEXT:    bge s0, a0, .LBB44_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB44_2 Depth=1
 ; RV64I-NEXT:    mv a2, s2
 ; RV64I-NEXT:    j .LBB44_1
 ; RV64I-NEXT:  .LBB44_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -3577,32 +3577,32 @@ define i8 @atomicrmw_min_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_min_i8_seq_cst:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    addi a3, zero, 56
-; RV64IA-NEXT:    sub a6, a3, a2
-; RV64IA-NEXT:    addi a4, zero, 255
-; RV64IA-NEXT:    sllw a7, a4, a2
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    addi a3, zero, 255
+; RV64IA-NEXT:    sllw a7, a3, a0
 ; RV64IA-NEXT:    slli a1, a1, 56
 ; RV64IA-NEXT:    srai a1, a1, 56
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    addi a4, zero, 56
+; RV64IA-NEXT:    sub a4, a4, a0
 ; RV64IA-NEXT:  .LBB44_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aqrl a5, (a0)
-; RV64IA-NEXT:    and a4, a5, a7
-; RV64IA-NEXT:    mv a3, a5
-; RV64IA-NEXT:    sll a4, a4, a6
-; RV64IA-NEXT:    sra a4, a4, a6
-; RV64IA-NEXT:    bge a1, a4, .LBB44_3
+; RV64IA-NEXT:    lr.w.aqrl a5, (a6)
+; RV64IA-NEXT:    and a3, a5, a7
+; RV64IA-NEXT:    mv a2, a5
+; RV64IA-NEXT:    sll a3, a3, a4
+; RV64IA-NEXT:    sra a3, a3, a4
+; RV64IA-NEXT:    bge a1, a3, .LBB44_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB44_1 Depth=1
-; RV64IA-NEXT:    xor a3, a5, a1
-; RV64IA-NEXT:    and a3, a3, a7
-; RV64IA-NEXT:    xor a3, a5, a3
+; RV64IA-NEXT:    xor a2, a5, a1
+; RV64IA-NEXT:    and a2, a2, a7
+; RV64IA-NEXT:    xor a2, a5, a2
 ; RV64IA-NEXT:  .LBB44_3: # in Loop: Header=BB44_1 Depth=1
-; RV64IA-NEXT:    sc.w.aqrl a3, a3, (a0)
-; RV64IA-NEXT:    bnez a3, .LBB44_1
+; RV64IA-NEXT:    sc.w.aqrl a2, a2, (a6)
+; RV64IA-NEXT:    bnez a2, .LBB44_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a5, a2
+; RV64IA-NEXT:    srlw a0, a5, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw min i8* %a, i8 %b seq_cst
   ret i8 %1
@@ -3617,33 +3617,33 @@ define i8 @atomicrmw_umax_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
 ; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lbu a0, 0(a0)
-; RV32I-NEXT:    andi s0, a1, 255
-; RV32I-NEXT:    addi s3, sp, 11
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lbu a3, 0(a0)
+; RV32I-NEXT:    mv s3, a1
+; RV32I-NEXT:    andi s1, a1, 255
+; RV32I-NEXT:    addi s2, sp, 11
 ; RV32I-NEXT:    j .LBB45_2
 ; RV32I-NEXT:  .LBB45_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB45_2 Depth=1
-; RV32I-NEXT:    sb a0, 11(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sb a3, 11(sp)
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s2
 ; RV32I-NEXT:    mv a3, zero
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_1
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lb a0, 11(sp)
-; RV32I-NEXT:    bnez a1, .LBB45_4
+; RV32I-NEXT:    lb a3, 11(sp)
+; RV32I-NEXT:    bnez a0, .LBB45_4
 ; RV32I-NEXT:  .LBB45_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    andi a1, a0, 255
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bltu s0, a1, .LBB45_1
+; RV32I-NEXT:    andi a0, a3, 255
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    bltu s1, a0, .LBB45_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB45_2 Depth=1
-; RV32I-NEXT:    mv a2, s2
+; RV32I-NEXT:    mv a2, s3
 ; RV32I-NEXT:    j .LBB45_1
 ; RV32I-NEXT:  .LBB45_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a3
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -3654,27 +3654,27 @@ define i8 @atomicrmw_umax_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_umax_i8_monotonic:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a6, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB45_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a4, (a0)
-; RV32IA-NEXT:    and a3, a4, a6
+; RV32IA-NEXT:    lr.w a4, (a6)
+; RV32IA-NEXT:    and a2, a4, a3
 ; RV32IA-NEXT:    mv a5, a4
-; RV32IA-NEXT:    bgeu a3, a1, .LBB45_3
+; RV32IA-NEXT:    bgeu a2, a1, .LBB45_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB45_1 Depth=1
 ; RV32IA-NEXT:    xor a5, a4, a1
-; RV32IA-NEXT:    and a5, a5, a6
+; RV32IA-NEXT:    and a5, a5, a3
 ; RV32IA-NEXT:    xor a5, a4, a5
 ; RV32IA-NEXT:  .LBB45_3: # in Loop: Header=BB45_1 Depth=1
-; RV32IA-NEXT:    sc.w a5, a5, (a0)
+; RV32IA-NEXT:    sc.w a5, a5, (a6)
 ; RV32IA-NEXT:    bnez a5, .LBB45_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a4, a2
+; RV32IA-NEXT:    srl a0, a4, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_umax_i8_monotonic:
@@ -3685,33 +3685,33 @@ define i8 @atomicrmw_umax_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
 ; RV64I-NEXT:    sd s3, 8(sp)
-; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lbu a0, 0(a0)
-; RV64I-NEXT:    andi s0, a1, 255
-; RV64I-NEXT:    addi s3, sp, 7
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    lbu a3, 0(a0)
+; RV64I-NEXT:    mv s3, a1
+; RV64I-NEXT:    andi s1, a1, 255
+; RV64I-NEXT:    addi s2, sp, 7
 ; RV64I-NEXT:    j .LBB45_2
 ; RV64I-NEXT:  .LBB45_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB45_2 Depth=1
-; RV64I-NEXT:    sb a0, 7(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sb a3, 7(sp)
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    mv a3, zero
 ; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    call __atomic_compare_exchange_1
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lb a0, 7(sp)
-; RV64I-NEXT:    bnez a1, .LBB45_4
+; RV64I-NEXT:    lb a3, 7(sp)
+; RV64I-NEXT:    bnez a0, .LBB45_4
 ; RV64I-NEXT:  .LBB45_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    andi a1, a0, 255
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bltu s0, a1, .LBB45_1
+; RV64I-NEXT:    andi a0, a3, 255
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bltu s1, a0, .LBB45_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB45_2 Depth=1
-; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:    mv a2, s3
 ; RV64I-NEXT:    j .LBB45_1
 ; RV64I-NEXT:  .LBB45_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -3722,27 +3722,27 @@ define i8 @atomicrmw_umax_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_umax_i8_monotonic:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a6, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB45_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a4, (a0)
-; RV64IA-NEXT:    and a3, a4, a6
+; RV64IA-NEXT:    lr.w a4, (a6)
+; RV64IA-NEXT:    and a2, a4, a3
 ; RV64IA-NEXT:    mv a5, a4
-; RV64IA-NEXT:    bgeu a3, a1, .LBB45_3
+; RV64IA-NEXT:    bgeu a2, a1, .LBB45_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB45_1 Depth=1
 ; RV64IA-NEXT:    xor a5, a4, a1
-; RV64IA-NEXT:    and a5, a5, a6
+; RV64IA-NEXT:    and a5, a5, a3
 ; RV64IA-NEXT:    xor a5, a4, a5
 ; RV64IA-NEXT:  .LBB45_3: # in Loop: Header=BB45_1 Depth=1
-; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    sc.w a5, a5, (a6)
 ; RV64IA-NEXT:    bnez a5, .LBB45_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    srlw a0, a4, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw umax i8* %a, i8 %b monotonic
   ret i8 %1
@@ -3757,33 +3757,33 @@ define i8 @atomicrmw_umax_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
 ; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lbu a0, 0(a0)
-; RV32I-NEXT:    andi s0, a1, 255
-; RV32I-NEXT:    addi s3, sp, 11
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lbu a3, 0(a0)
+; RV32I-NEXT:    mv s3, a1
+; RV32I-NEXT:    andi s1, a1, 255
+; RV32I-NEXT:    addi s2, sp, 11
 ; RV32I-NEXT:    j .LBB46_2
 ; RV32I-NEXT:  .LBB46_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB46_2 Depth=1
-; RV32I-NEXT:    sb a0, 11(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sb a3, 11(sp)
 ; RV32I-NEXT:    addi a3, zero, 2
 ; RV32I-NEXT:    addi a4, zero, 2
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s2
 ; RV32I-NEXT:    call __atomic_compare_exchange_1
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lb a0, 11(sp)
-; RV32I-NEXT:    bnez a1, .LBB46_4
+; RV32I-NEXT:    lb a3, 11(sp)
+; RV32I-NEXT:    bnez a0, .LBB46_4
 ; RV32I-NEXT:  .LBB46_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    andi a1, a0, 255
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bltu s0, a1, .LBB46_1
+; RV32I-NEXT:    andi a0, a3, 255
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    bltu s1, a0, .LBB46_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB46_2 Depth=1
-; RV32I-NEXT:    mv a2, s2
+; RV32I-NEXT:    mv a2, s3
 ; RV32I-NEXT:    j .LBB46_1
 ; RV32I-NEXT:  .LBB46_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a3
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -3794,27 +3794,27 @@ define i8 @atomicrmw_umax_i8_acquire(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_umax_i8_acquire:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a6, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB46_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a4, (a0)
-; RV32IA-NEXT:    and a3, a4, a6
+; RV32IA-NEXT:    lr.w.aq a4, (a6)
+; RV32IA-NEXT:    and a2, a4, a3
 ; RV32IA-NEXT:    mv a5, a4
-; RV32IA-NEXT:    bgeu a3, a1, .LBB46_3
+; RV32IA-NEXT:    bgeu a2, a1, .LBB46_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB46_1 Depth=1
 ; RV32IA-NEXT:    xor a5, a4, a1
-; RV32IA-NEXT:    and a5, a5, a6
+; RV32IA-NEXT:    and a5, a5, a3
 ; RV32IA-NEXT:    xor a5, a4, a5
 ; RV32IA-NEXT:  .LBB46_3: # in Loop: Header=BB46_1 Depth=1
-; RV32IA-NEXT:    sc.w a5, a5, (a0)
+; RV32IA-NEXT:    sc.w a5, a5, (a6)
 ; RV32IA-NEXT:    bnez a5, .LBB46_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a4, a2
+; RV32IA-NEXT:    srl a0, a4, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_umax_i8_acquire:
@@ -3825,33 +3825,33 @@ define i8 @atomicrmw_umax_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
 ; RV64I-NEXT:    sd s3, 8(sp)
-; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lbu a0, 0(a0)
-; RV64I-NEXT:    andi s0, a1, 255
-; RV64I-NEXT:    addi s3, sp, 7
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    lbu a3, 0(a0)
+; RV64I-NEXT:    mv s3, a1
+; RV64I-NEXT:    andi s1, a1, 255
+; RV64I-NEXT:    addi s2, sp, 7
 ; RV64I-NEXT:    j .LBB46_2
 ; RV64I-NEXT:  .LBB46_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB46_2 Depth=1
-; RV64I-NEXT:    sb a0, 7(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sb a3, 7(sp)
 ; RV64I-NEXT:    addi a3, zero, 2
 ; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    call __atomic_compare_exchange_1
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lb a0, 7(sp)
-; RV64I-NEXT:    bnez a1, .LBB46_4
+; RV64I-NEXT:    lb a3, 7(sp)
+; RV64I-NEXT:    bnez a0, .LBB46_4
 ; RV64I-NEXT:  .LBB46_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    andi a1, a0, 255
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bltu s0, a1, .LBB46_1
+; RV64I-NEXT:    andi a0, a3, 255
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bltu s1, a0, .LBB46_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB46_2 Depth=1
-; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:    mv a2, s3
 ; RV64I-NEXT:    j .LBB46_1
 ; RV64I-NEXT:  .LBB46_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -3862,27 +3862,27 @@ define i8 @atomicrmw_umax_i8_acquire(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_umax_i8_acquire:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a6, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB46_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a4, (a0)
-; RV64IA-NEXT:    and a3, a4, a6
+; RV64IA-NEXT:    lr.w.aq a4, (a6)
+; RV64IA-NEXT:    and a2, a4, a3
 ; RV64IA-NEXT:    mv a5, a4
-; RV64IA-NEXT:    bgeu a3, a1, .LBB46_3
+; RV64IA-NEXT:    bgeu a2, a1, .LBB46_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB46_1 Depth=1
 ; RV64IA-NEXT:    xor a5, a4, a1
-; RV64IA-NEXT:    and a5, a5, a6
+; RV64IA-NEXT:    and a5, a5, a3
 ; RV64IA-NEXT:    xor a5, a4, a5
 ; RV64IA-NEXT:  .LBB46_3: # in Loop: Header=BB46_1 Depth=1
-; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    sc.w a5, a5, (a6)
 ; RV64IA-NEXT:    bnez a5, .LBB46_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    srlw a0, a4, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw umax i8* %a, i8 %b acquire
   ret i8 %1
@@ -3897,33 +3897,33 @@ define i8 @atomicrmw_umax_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
 ; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lbu a0, 0(a0)
-; RV32I-NEXT:    andi s0, a1, 255
-; RV32I-NEXT:    addi s3, sp, 11
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lbu a3, 0(a0)
+; RV32I-NEXT:    mv s3, a1
+; RV32I-NEXT:    andi s1, a1, 255
+; RV32I-NEXT:    addi s2, sp, 11
 ; RV32I-NEXT:    j .LBB47_2
 ; RV32I-NEXT:  .LBB47_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB47_2 Depth=1
-; RV32I-NEXT:    sb a0, 11(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sb a3, 11(sp)
 ; RV32I-NEXT:    addi a3, zero, 3
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s2
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_1
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lb a0, 11(sp)
-; RV32I-NEXT:    bnez a1, .LBB47_4
+; RV32I-NEXT:    lb a3, 11(sp)
+; RV32I-NEXT:    bnez a0, .LBB47_4
 ; RV32I-NEXT:  .LBB47_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    andi a1, a0, 255
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bltu s0, a1, .LBB47_1
+; RV32I-NEXT:    andi a0, a3, 255
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    bltu s1, a0, .LBB47_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB47_2 Depth=1
-; RV32I-NEXT:    mv a2, s2
+; RV32I-NEXT:    mv a2, s3
 ; RV32I-NEXT:    j .LBB47_1
 ; RV32I-NEXT:  .LBB47_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a3
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -3934,27 +3934,27 @@ define i8 @atomicrmw_umax_i8_release(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_umax_i8_release:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a6, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB47_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a4, (a0)
-; RV32IA-NEXT:    and a3, a4, a6
+; RV32IA-NEXT:    lr.w a4, (a6)
+; RV32IA-NEXT:    and a2, a4, a3
 ; RV32IA-NEXT:    mv a5, a4
-; RV32IA-NEXT:    bgeu a3, a1, .LBB47_3
+; RV32IA-NEXT:    bgeu a2, a1, .LBB47_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB47_1 Depth=1
 ; RV32IA-NEXT:    xor a5, a4, a1
-; RV32IA-NEXT:    and a5, a5, a6
+; RV32IA-NEXT:    and a5, a5, a3
 ; RV32IA-NEXT:    xor a5, a4, a5
 ; RV32IA-NEXT:  .LBB47_3: # in Loop: Header=BB47_1 Depth=1
-; RV32IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV32IA-NEXT:    sc.w.rl a5, a5, (a6)
 ; RV32IA-NEXT:    bnez a5, .LBB47_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a4, a2
+; RV32IA-NEXT:    srl a0, a4, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_umax_i8_release:
@@ -3965,33 +3965,33 @@ define i8 @atomicrmw_umax_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
 ; RV64I-NEXT:    sd s3, 8(sp)
-; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lbu a0, 0(a0)
-; RV64I-NEXT:    andi s0, a1, 255
-; RV64I-NEXT:    addi s3, sp, 7
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    lbu a3, 0(a0)
+; RV64I-NEXT:    mv s3, a1
+; RV64I-NEXT:    andi s1, a1, 255
+; RV64I-NEXT:    addi s2, sp, 7
 ; RV64I-NEXT:    j .LBB47_2
 ; RV64I-NEXT:  .LBB47_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB47_2 Depth=1
-; RV64I-NEXT:    sb a0, 7(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sb a3, 7(sp)
 ; RV64I-NEXT:    addi a3, zero, 3
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    call __atomic_compare_exchange_1
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lb a0, 7(sp)
-; RV64I-NEXT:    bnez a1, .LBB47_4
+; RV64I-NEXT:    lb a3, 7(sp)
+; RV64I-NEXT:    bnez a0, .LBB47_4
 ; RV64I-NEXT:  .LBB47_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    andi a1, a0, 255
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bltu s0, a1, .LBB47_1
+; RV64I-NEXT:    andi a0, a3, 255
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bltu s1, a0, .LBB47_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB47_2 Depth=1
-; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:    mv a2, s3
 ; RV64I-NEXT:    j .LBB47_1
 ; RV64I-NEXT:  .LBB47_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -4002,27 +4002,27 @@ define i8 @atomicrmw_umax_i8_release(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_umax_i8_release:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a6, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB47_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a4, (a0)
-; RV64IA-NEXT:    and a3, a4, a6
+; RV64IA-NEXT:    lr.w a4, (a6)
+; RV64IA-NEXT:    and a2, a4, a3
 ; RV64IA-NEXT:    mv a5, a4
-; RV64IA-NEXT:    bgeu a3, a1, .LBB47_3
+; RV64IA-NEXT:    bgeu a2, a1, .LBB47_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB47_1 Depth=1
 ; RV64IA-NEXT:    xor a5, a4, a1
-; RV64IA-NEXT:    and a5, a5, a6
+; RV64IA-NEXT:    and a5, a5, a3
 ; RV64IA-NEXT:    xor a5, a4, a5
 ; RV64IA-NEXT:  .LBB47_3: # in Loop: Header=BB47_1 Depth=1
-; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a6)
 ; RV64IA-NEXT:    bnez a5, .LBB47_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    srlw a0, a4, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw umax i8* %a, i8 %b release
   ret i8 %1
@@ -4037,33 +4037,33 @@ define i8 @atomicrmw_umax_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
 ; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lbu a0, 0(a0)
-; RV32I-NEXT:    andi s0, a1, 255
-; RV32I-NEXT:    addi s3, sp, 11
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lbu a3, 0(a0)
+; RV32I-NEXT:    mv s3, a1
+; RV32I-NEXT:    andi s1, a1, 255
+; RV32I-NEXT:    addi s2, sp, 11
 ; RV32I-NEXT:    j .LBB48_2
 ; RV32I-NEXT:  .LBB48_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB48_2 Depth=1
-; RV32I-NEXT:    sb a0, 11(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sb a3, 11(sp)
 ; RV32I-NEXT:    addi a3, zero, 4
 ; RV32I-NEXT:    addi a4, zero, 2
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s2
 ; RV32I-NEXT:    call __atomic_compare_exchange_1
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lb a0, 11(sp)
-; RV32I-NEXT:    bnez a1, .LBB48_4
+; RV32I-NEXT:    lb a3, 11(sp)
+; RV32I-NEXT:    bnez a0, .LBB48_4
 ; RV32I-NEXT:  .LBB48_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    andi a1, a0, 255
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bltu s0, a1, .LBB48_1
+; RV32I-NEXT:    andi a0, a3, 255
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    bltu s1, a0, .LBB48_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB48_2 Depth=1
-; RV32I-NEXT:    mv a2, s2
+; RV32I-NEXT:    mv a2, s3
 ; RV32I-NEXT:    j .LBB48_1
 ; RV32I-NEXT:  .LBB48_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a3
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -4074,27 +4074,27 @@ define i8 @atomicrmw_umax_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_umax_i8_acq_rel:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a6, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB48_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a4, (a0)
-; RV32IA-NEXT:    and a3, a4, a6
+; RV32IA-NEXT:    lr.w.aq a4, (a6)
+; RV32IA-NEXT:    and a2, a4, a3
 ; RV32IA-NEXT:    mv a5, a4
-; RV32IA-NEXT:    bgeu a3, a1, .LBB48_3
+; RV32IA-NEXT:    bgeu a2, a1, .LBB48_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB48_1 Depth=1
 ; RV32IA-NEXT:    xor a5, a4, a1
-; RV32IA-NEXT:    and a5, a5, a6
+; RV32IA-NEXT:    and a5, a5, a3
 ; RV32IA-NEXT:    xor a5, a4, a5
 ; RV32IA-NEXT:  .LBB48_3: # in Loop: Header=BB48_1 Depth=1
-; RV32IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV32IA-NEXT:    sc.w.rl a5, a5, (a6)
 ; RV32IA-NEXT:    bnez a5, .LBB48_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a4, a2
+; RV32IA-NEXT:    srl a0, a4, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_umax_i8_acq_rel:
@@ -4105,33 +4105,33 @@ define i8 @atomicrmw_umax_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
 ; RV64I-NEXT:    sd s3, 8(sp)
-; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lbu a0, 0(a0)
-; RV64I-NEXT:    andi s0, a1, 255
-; RV64I-NEXT:    addi s3, sp, 7
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    lbu a3, 0(a0)
+; RV64I-NEXT:    mv s3, a1
+; RV64I-NEXT:    andi s1, a1, 255
+; RV64I-NEXT:    addi s2, sp, 7
 ; RV64I-NEXT:    j .LBB48_2
 ; RV64I-NEXT:  .LBB48_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB48_2 Depth=1
-; RV64I-NEXT:    sb a0, 7(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sb a3, 7(sp)
 ; RV64I-NEXT:    addi a3, zero, 4
 ; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    call __atomic_compare_exchange_1
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lb a0, 7(sp)
-; RV64I-NEXT:    bnez a1, .LBB48_4
+; RV64I-NEXT:    lb a3, 7(sp)
+; RV64I-NEXT:    bnez a0, .LBB48_4
 ; RV64I-NEXT:  .LBB48_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    andi a1, a0, 255
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bltu s0, a1, .LBB48_1
+; RV64I-NEXT:    andi a0, a3, 255
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bltu s1, a0, .LBB48_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB48_2 Depth=1
-; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:    mv a2, s3
 ; RV64I-NEXT:    j .LBB48_1
 ; RV64I-NEXT:  .LBB48_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -4142,27 +4142,27 @@ define i8 @atomicrmw_umax_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_umax_i8_acq_rel:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a6, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB48_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a4, (a0)
-; RV64IA-NEXT:    and a3, a4, a6
+; RV64IA-NEXT:    lr.w.aq a4, (a6)
+; RV64IA-NEXT:    and a2, a4, a3
 ; RV64IA-NEXT:    mv a5, a4
-; RV64IA-NEXT:    bgeu a3, a1, .LBB48_3
+; RV64IA-NEXT:    bgeu a2, a1, .LBB48_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB48_1 Depth=1
 ; RV64IA-NEXT:    xor a5, a4, a1
-; RV64IA-NEXT:    and a5, a5, a6
+; RV64IA-NEXT:    and a5, a5, a3
 ; RV64IA-NEXT:    xor a5, a4, a5
 ; RV64IA-NEXT:  .LBB48_3: # in Loop: Header=BB48_1 Depth=1
-; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a6)
 ; RV64IA-NEXT:    bnez a5, .LBB48_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    srlw a0, a4, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw umax i8* %a, i8 %b acq_rel
   ret i8 %1
@@ -4177,33 +4177,33 @@ define i8 @atomicrmw_umax_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
 ; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lbu a0, 0(a0)
-; RV32I-NEXT:    andi s0, a1, 255
-; RV32I-NEXT:    addi s3, sp, 11
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lbu a3, 0(a0)
+; RV32I-NEXT:    mv s3, a1
+; RV32I-NEXT:    andi s1, a1, 255
+; RV32I-NEXT:    addi s2, sp, 11
 ; RV32I-NEXT:    j .LBB49_2
 ; RV32I-NEXT:  .LBB49_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB49_2 Depth=1
-; RV32I-NEXT:    sb a0, 11(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sb a3, 11(sp)
 ; RV32I-NEXT:    addi a3, zero, 5
 ; RV32I-NEXT:    addi a4, zero, 5
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s2
 ; RV32I-NEXT:    call __atomic_compare_exchange_1
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lb a0, 11(sp)
-; RV32I-NEXT:    bnez a1, .LBB49_4
+; RV32I-NEXT:    lb a3, 11(sp)
+; RV32I-NEXT:    bnez a0, .LBB49_4
 ; RV32I-NEXT:  .LBB49_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    andi a1, a0, 255
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bltu s0, a1, .LBB49_1
+; RV32I-NEXT:    andi a0, a3, 255
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    bltu s1, a0, .LBB49_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB49_2 Depth=1
-; RV32I-NEXT:    mv a2, s2
+; RV32I-NEXT:    mv a2, s3
 ; RV32I-NEXT:    j .LBB49_1
 ; RV32I-NEXT:  .LBB49_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a3
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -4214,27 +4214,27 @@ define i8 @atomicrmw_umax_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_umax_i8_seq_cst:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a6, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB49_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aqrl a4, (a0)
-; RV32IA-NEXT:    and a3, a4, a6
+; RV32IA-NEXT:    lr.w.aqrl a4, (a6)
+; RV32IA-NEXT:    and a2, a4, a3
 ; RV32IA-NEXT:    mv a5, a4
-; RV32IA-NEXT:    bgeu a3, a1, .LBB49_3
+; RV32IA-NEXT:    bgeu a2, a1, .LBB49_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB49_1 Depth=1
 ; RV32IA-NEXT:    xor a5, a4, a1
-; RV32IA-NEXT:    and a5, a5, a6
+; RV32IA-NEXT:    and a5, a5, a3
 ; RV32IA-NEXT:    xor a5, a4, a5
 ; RV32IA-NEXT:  .LBB49_3: # in Loop: Header=BB49_1 Depth=1
-; RV32IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV32IA-NEXT:    sc.w.aqrl a5, a5, (a6)
 ; RV32IA-NEXT:    bnez a5, .LBB49_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a4, a2
+; RV32IA-NEXT:    srl a0, a4, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_umax_i8_seq_cst:
@@ -4245,33 +4245,33 @@ define i8 @atomicrmw_umax_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
 ; RV64I-NEXT:    sd s3, 8(sp)
-; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lbu a0, 0(a0)
-; RV64I-NEXT:    andi s0, a1, 255
-; RV64I-NEXT:    addi s3, sp, 7
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    lbu a3, 0(a0)
+; RV64I-NEXT:    mv s3, a1
+; RV64I-NEXT:    andi s1, a1, 255
+; RV64I-NEXT:    addi s2, sp, 7
 ; RV64I-NEXT:    j .LBB49_2
 ; RV64I-NEXT:  .LBB49_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB49_2 Depth=1
-; RV64I-NEXT:    sb a0, 7(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sb a3, 7(sp)
 ; RV64I-NEXT:    addi a3, zero, 5
 ; RV64I-NEXT:    addi a4, zero, 5
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    call __atomic_compare_exchange_1
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lb a0, 7(sp)
-; RV64I-NEXT:    bnez a1, .LBB49_4
+; RV64I-NEXT:    lb a3, 7(sp)
+; RV64I-NEXT:    bnez a0, .LBB49_4
 ; RV64I-NEXT:  .LBB49_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    andi a1, a0, 255
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bltu s0, a1, .LBB49_1
+; RV64I-NEXT:    andi a0, a3, 255
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bltu s1, a0, .LBB49_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB49_2 Depth=1
-; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:    mv a2, s3
 ; RV64I-NEXT:    j .LBB49_1
 ; RV64I-NEXT:  .LBB49_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -4282,27 +4282,27 @@ define i8 @atomicrmw_umax_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_umax_i8_seq_cst:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a6, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB49_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aqrl a4, (a0)
-; RV64IA-NEXT:    and a3, a4, a6
+; RV64IA-NEXT:    lr.w.aqrl a4, (a6)
+; RV64IA-NEXT:    and a2, a4, a3
 ; RV64IA-NEXT:    mv a5, a4
-; RV64IA-NEXT:    bgeu a3, a1, .LBB49_3
+; RV64IA-NEXT:    bgeu a2, a1, .LBB49_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB49_1 Depth=1
 ; RV64IA-NEXT:    xor a5, a4, a1
-; RV64IA-NEXT:    and a5, a5, a6
+; RV64IA-NEXT:    and a5, a5, a3
 ; RV64IA-NEXT:    xor a5, a4, a5
 ; RV64IA-NEXT:  .LBB49_3: # in Loop: Header=BB49_1 Depth=1
-; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a6)
 ; RV64IA-NEXT:    bnez a5, .LBB49_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    srlw a0, a4, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw umax i8* %a, i8 %b seq_cst
   ret i8 %1
@@ -4317,33 +4317,33 @@ define i8 @atomicrmw_umin_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
 ; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lbu a0, 0(a0)
-; RV32I-NEXT:    andi s0, a1, 255
-; RV32I-NEXT:    addi s3, sp, 11
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lbu a3, 0(a0)
+; RV32I-NEXT:    mv s3, a1
+; RV32I-NEXT:    andi s1, a1, 255
+; RV32I-NEXT:    addi s2, sp, 11
 ; RV32I-NEXT:    j .LBB50_2
 ; RV32I-NEXT:  .LBB50_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB50_2 Depth=1
-; RV32I-NEXT:    sb a0, 11(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sb a3, 11(sp)
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s2
 ; RV32I-NEXT:    mv a3, zero
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_1
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lb a0, 11(sp)
-; RV32I-NEXT:    bnez a1, .LBB50_4
+; RV32I-NEXT:    lb a3, 11(sp)
+; RV32I-NEXT:    bnez a0, .LBB50_4
 ; RV32I-NEXT:  .LBB50_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    andi a1, a0, 255
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bgeu s0, a1, .LBB50_1
+; RV32I-NEXT:    andi a0, a3, 255
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    bgeu s1, a0, .LBB50_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB50_2 Depth=1
-; RV32I-NEXT:    mv a2, s2
+; RV32I-NEXT:    mv a2, s3
 ; RV32I-NEXT:    j .LBB50_1
 ; RV32I-NEXT:  .LBB50_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a3
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -4354,27 +4354,27 @@ define i8 @atomicrmw_umin_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_umin_i8_monotonic:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a6, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB50_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a4, (a0)
-; RV32IA-NEXT:    and a3, a4, a6
+; RV32IA-NEXT:    lr.w a4, (a6)
+; RV32IA-NEXT:    and a2, a4, a3
 ; RV32IA-NEXT:    mv a5, a4
-; RV32IA-NEXT:    bgeu a1, a3, .LBB50_3
+; RV32IA-NEXT:    bgeu a1, a2, .LBB50_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB50_1 Depth=1
 ; RV32IA-NEXT:    xor a5, a4, a1
-; RV32IA-NEXT:    and a5, a5, a6
+; RV32IA-NEXT:    and a5, a5, a3
 ; RV32IA-NEXT:    xor a5, a4, a5
 ; RV32IA-NEXT:  .LBB50_3: # in Loop: Header=BB50_1 Depth=1
-; RV32IA-NEXT:    sc.w a5, a5, (a0)
+; RV32IA-NEXT:    sc.w a5, a5, (a6)
 ; RV32IA-NEXT:    bnez a5, .LBB50_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a4, a2
+; RV32IA-NEXT:    srl a0, a4, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_umin_i8_monotonic:
@@ -4385,33 +4385,33 @@ define i8 @atomicrmw_umin_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
 ; RV64I-NEXT:    sd s3, 8(sp)
-; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lbu a0, 0(a0)
-; RV64I-NEXT:    andi s0, a1, 255
-; RV64I-NEXT:    addi s3, sp, 7
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    lbu a3, 0(a0)
+; RV64I-NEXT:    mv s3, a1
+; RV64I-NEXT:    andi s1, a1, 255
+; RV64I-NEXT:    addi s2, sp, 7
 ; RV64I-NEXT:    j .LBB50_2
 ; RV64I-NEXT:  .LBB50_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB50_2 Depth=1
-; RV64I-NEXT:    sb a0, 7(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sb a3, 7(sp)
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    mv a3, zero
 ; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    call __atomic_compare_exchange_1
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lb a0, 7(sp)
-; RV64I-NEXT:    bnez a1, .LBB50_4
+; RV64I-NEXT:    lb a3, 7(sp)
+; RV64I-NEXT:    bnez a0, .LBB50_4
 ; RV64I-NEXT:  .LBB50_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    andi a1, a0, 255
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bgeu s0, a1, .LBB50_1
+; RV64I-NEXT:    andi a0, a3, 255
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bgeu s1, a0, .LBB50_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB50_2 Depth=1
-; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:    mv a2, s3
 ; RV64I-NEXT:    j .LBB50_1
 ; RV64I-NEXT:  .LBB50_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -4422,27 +4422,27 @@ define i8 @atomicrmw_umin_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_umin_i8_monotonic:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a6, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB50_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a4, (a0)
-; RV64IA-NEXT:    and a3, a4, a6
+; RV64IA-NEXT:    lr.w a4, (a6)
+; RV64IA-NEXT:    and a2, a4, a3
 ; RV64IA-NEXT:    mv a5, a4
-; RV64IA-NEXT:    bgeu a1, a3, .LBB50_3
+; RV64IA-NEXT:    bgeu a1, a2, .LBB50_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB50_1 Depth=1
 ; RV64IA-NEXT:    xor a5, a4, a1
-; RV64IA-NEXT:    and a5, a5, a6
+; RV64IA-NEXT:    and a5, a5, a3
 ; RV64IA-NEXT:    xor a5, a4, a5
 ; RV64IA-NEXT:  .LBB50_3: # in Loop: Header=BB50_1 Depth=1
-; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    sc.w a5, a5, (a6)
 ; RV64IA-NEXT:    bnez a5, .LBB50_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    srlw a0, a4, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw umin i8* %a, i8 %b monotonic
   ret i8 %1
@@ -4457,33 +4457,33 @@ define i8 @atomicrmw_umin_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
 ; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lbu a0, 0(a0)
-; RV32I-NEXT:    andi s0, a1, 255
-; RV32I-NEXT:    addi s3, sp, 11
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lbu a3, 0(a0)
+; RV32I-NEXT:    mv s3, a1
+; RV32I-NEXT:    andi s1, a1, 255
+; RV32I-NEXT:    addi s2, sp, 11
 ; RV32I-NEXT:    j .LBB51_2
 ; RV32I-NEXT:  .LBB51_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB51_2 Depth=1
-; RV32I-NEXT:    sb a0, 11(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sb a3, 11(sp)
 ; RV32I-NEXT:    addi a3, zero, 2
 ; RV32I-NEXT:    addi a4, zero, 2
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s2
 ; RV32I-NEXT:    call __atomic_compare_exchange_1
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lb a0, 11(sp)
-; RV32I-NEXT:    bnez a1, .LBB51_4
+; RV32I-NEXT:    lb a3, 11(sp)
+; RV32I-NEXT:    bnez a0, .LBB51_4
 ; RV32I-NEXT:  .LBB51_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    andi a1, a0, 255
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bgeu s0, a1, .LBB51_1
+; RV32I-NEXT:    andi a0, a3, 255
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    bgeu s1, a0, .LBB51_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB51_2 Depth=1
-; RV32I-NEXT:    mv a2, s2
+; RV32I-NEXT:    mv a2, s3
 ; RV32I-NEXT:    j .LBB51_1
 ; RV32I-NEXT:  .LBB51_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a3
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -4494,27 +4494,27 @@ define i8 @atomicrmw_umin_i8_acquire(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_umin_i8_acquire:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a6, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB51_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a4, (a0)
-; RV32IA-NEXT:    and a3, a4, a6
+; RV32IA-NEXT:    lr.w.aq a4, (a6)
+; RV32IA-NEXT:    and a2, a4, a3
 ; RV32IA-NEXT:    mv a5, a4
-; RV32IA-NEXT:    bgeu a1, a3, .LBB51_3
+; RV32IA-NEXT:    bgeu a1, a2, .LBB51_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB51_1 Depth=1
 ; RV32IA-NEXT:    xor a5, a4, a1
-; RV32IA-NEXT:    and a5, a5, a6
+; RV32IA-NEXT:    and a5, a5, a3
 ; RV32IA-NEXT:    xor a5, a4, a5
 ; RV32IA-NEXT:  .LBB51_3: # in Loop: Header=BB51_1 Depth=1
-; RV32IA-NEXT:    sc.w a5, a5, (a0)
+; RV32IA-NEXT:    sc.w a5, a5, (a6)
 ; RV32IA-NEXT:    bnez a5, .LBB51_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a4, a2
+; RV32IA-NEXT:    srl a0, a4, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_umin_i8_acquire:
@@ -4525,33 +4525,33 @@ define i8 @atomicrmw_umin_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
 ; RV64I-NEXT:    sd s3, 8(sp)
-; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lbu a0, 0(a0)
-; RV64I-NEXT:    andi s0, a1, 255
-; RV64I-NEXT:    addi s3, sp, 7
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    lbu a3, 0(a0)
+; RV64I-NEXT:    mv s3, a1
+; RV64I-NEXT:    andi s1, a1, 255
+; RV64I-NEXT:    addi s2, sp, 7
 ; RV64I-NEXT:    j .LBB51_2
 ; RV64I-NEXT:  .LBB51_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB51_2 Depth=1
-; RV64I-NEXT:    sb a0, 7(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sb a3, 7(sp)
 ; RV64I-NEXT:    addi a3, zero, 2
 ; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    call __atomic_compare_exchange_1
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lb a0, 7(sp)
-; RV64I-NEXT:    bnez a1, .LBB51_4
+; RV64I-NEXT:    lb a3, 7(sp)
+; RV64I-NEXT:    bnez a0, .LBB51_4
 ; RV64I-NEXT:  .LBB51_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    andi a1, a0, 255
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bgeu s0, a1, .LBB51_1
+; RV64I-NEXT:    andi a0, a3, 255
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bgeu s1, a0, .LBB51_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB51_2 Depth=1
-; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:    mv a2, s3
 ; RV64I-NEXT:    j .LBB51_1
 ; RV64I-NEXT:  .LBB51_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -4562,27 +4562,27 @@ define i8 @atomicrmw_umin_i8_acquire(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_umin_i8_acquire:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a6, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB51_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a4, (a0)
-; RV64IA-NEXT:    and a3, a4, a6
+; RV64IA-NEXT:    lr.w.aq a4, (a6)
+; RV64IA-NEXT:    and a2, a4, a3
 ; RV64IA-NEXT:    mv a5, a4
-; RV64IA-NEXT:    bgeu a1, a3, .LBB51_3
+; RV64IA-NEXT:    bgeu a1, a2, .LBB51_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB51_1 Depth=1
 ; RV64IA-NEXT:    xor a5, a4, a1
-; RV64IA-NEXT:    and a5, a5, a6
+; RV64IA-NEXT:    and a5, a5, a3
 ; RV64IA-NEXT:    xor a5, a4, a5
 ; RV64IA-NEXT:  .LBB51_3: # in Loop: Header=BB51_1 Depth=1
-; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    sc.w a5, a5, (a6)
 ; RV64IA-NEXT:    bnez a5, .LBB51_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    srlw a0, a4, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw umin i8* %a, i8 %b acquire
   ret i8 %1
@@ -4597,33 +4597,33 @@ define i8 @atomicrmw_umin_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
 ; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lbu a0, 0(a0)
-; RV32I-NEXT:    andi s0, a1, 255
-; RV32I-NEXT:    addi s3, sp, 11
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lbu a3, 0(a0)
+; RV32I-NEXT:    mv s3, a1
+; RV32I-NEXT:    andi s1, a1, 255
+; RV32I-NEXT:    addi s2, sp, 11
 ; RV32I-NEXT:    j .LBB52_2
 ; RV32I-NEXT:  .LBB52_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB52_2 Depth=1
-; RV32I-NEXT:    sb a0, 11(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sb a3, 11(sp)
 ; RV32I-NEXT:    addi a3, zero, 3
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s2
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_1
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lb a0, 11(sp)
-; RV32I-NEXT:    bnez a1, .LBB52_4
+; RV32I-NEXT:    lb a3, 11(sp)
+; RV32I-NEXT:    bnez a0, .LBB52_4
 ; RV32I-NEXT:  .LBB52_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    andi a1, a0, 255
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bgeu s0, a1, .LBB52_1
+; RV32I-NEXT:    andi a0, a3, 255
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    bgeu s1, a0, .LBB52_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB52_2 Depth=1
-; RV32I-NEXT:    mv a2, s2
+; RV32I-NEXT:    mv a2, s3
 ; RV32I-NEXT:    j .LBB52_1
 ; RV32I-NEXT:  .LBB52_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a3
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -4634,27 +4634,27 @@ define i8 @atomicrmw_umin_i8_release(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_umin_i8_release:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a6, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB52_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a4, (a0)
-; RV32IA-NEXT:    and a3, a4, a6
+; RV32IA-NEXT:    lr.w a4, (a6)
+; RV32IA-NEXT:    and a2, a4, a3
 ; RV32IA-NEXT:    mv a5, a4
-; RV32IA-NEXT:    bgeu a1, a3, .LBB52_3
+; RV32IA-NEXT:    bgeu a1, a2, .LBB52_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB52_1 Depth=1
 ; RV32IA-NEXT:    xor a5, a4, a1
-; RV32IA-NEXT:    and a5, a5, a6
+; RV32IA-NEXT:    and a5, a5, a3
 ; RV32IA-NEXT:    xor a5, a4, a5
 ; RV32IA-NEXT:  .LBB52_3: # in Loop: Header=BB52_1 Depth=1
-; RV32IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV32IA-NEXT:    sc.w.rl a5, a5, (a6)
 ; RV32IA-NEXT:    bnez a5, .LBB52_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a4, a2
+; RV32IA-NEXT:    srl a0, a4, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_umin_i8_release:
@@ -4665,33 +4665,33 @@ define i8 @atomicrmw_umin_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
 ; RV64I-NEXT:    sd s3, 8(sp)
-; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lbu a0, 0(a0)
-; RV64I-NEXT:    andi s0, a1, 255
-; RV64I-NEXT:    addi s3, sp, 7
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    lbu a3, 0(a0)
+; RV64I-NEXT:    mv s3, a1
+; RV64I-NEXT:    andi s1, a1, 255
+; RV64I-NEXT:    addi s2, sp, 7
 ; RV64I-NEXT:    j .LBB52_2
 ; RV64I-NEXT:  .LBB52_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB52_2 Depth=1
-; RV64I-NEXT:    sb a0, 7(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sb a3, 7(sp)
 ; RV64I-NEXT:    addi a3, zero, 3
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    call __atomic_compare_exchange_1
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lb a0, 7(sp)
-; RV64I-NEXT:    bnez a1, .LBB52_4
+; RV64I-NEXT:    lb a3, 7(sp)
+; RV64I-NEXT:    bnez a0, .LBB52_4
 ; RV64I-NEXT:  .LBB52_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    andi a1, a0, 255
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bgeu s0, a1, .LBB52_1
+; RV64I-NEXT:    andi a0, a3, 255
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bgeu s1, a0, .LBB52_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB52_2 Depth=1
-; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:    mv a2, s3
 ; RV64I-NEXT:    j .LBB52_1
 ; RV64I-NEXT:  .LBB52_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -4702,27 +4702,27 @@ define i8 @atomicrmw_umin_i8_release(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_umin_i8_release:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a6, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB52_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a4, (a0)
-; RV64IA-NEXT:    and a3, a4, a6
+; RV64IA-NEXT:    lr.w a4, (a6)
+; RV64IA-NEXT:    and a2, a4, a3
 ; RV64IA-NEXT:    mv a5, a4
-; RV64IA-NEXT:    bgeu a1, a3, .LBB52_3
+; RV64IA-NEXT:    bgeu a1, a2, .LBB52_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB52_1 Depth=1
 ; RV64IA-NEXT:    xor a5, a4, a1
-; RV64IA-NEXT:    and a5, a5, a6
+; RV64IA-NEXT:    and a5, a5, a3
 ; RV64IA-NEXT:    xor a5, a4, a5
 ; RV64IA-NEXT:  .LBB52_3: # in Loop: Header=BB52_1 Depth=1
-; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a6)
 ; RV64IA-NEXT:    bnez a5, .LBB52_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    srlw a0, a4, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw umin i8* %a, i8 %b release
   ret i8 %1
@@ -4737,33 +4737,33 @@ define i8 @atomicrmw_umin_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
 ; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lbu a0, 0(a0)
-; RV32I-NEXT:    andi s0, a1, 255
-; RV32I-NEXT:    addi s3, sp, 11
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lbu a3, 0(a0)
+; RV32I-NEXT:    mv s3, a1
+; RV32I-NEXT:    andi s1, a1, 255
+; RV32I-NEXT:    addi s2, sp, 11
 ; RV32I-NEXT:    j .LBB53_2
 ; RV32I-NEXT:  .LBB53_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB53_2 Depth=1
-; RV32I-NEXT:    sb a0, 11(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sb a3, 11(sp)
 ; RV32I-NEXT:    addi a3, zero, 4
 ; RV32I-NEXT:    addi a4, zero, 2
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s2
 ; RV32I-NEXT:    call __atomic_compare_exchange_1
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lb a0, 11(sp)
-; RV32I-NEXT:    bnez a1, .LBB53_4
+; RV32I-NEXT:    lb a3, 11(sp)
+; RV32I-NEXT:    bnez a0, .LBB53_4
 ; RV32I-NEXT:  .LBB53_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    andi a1, a0, 255
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bgeu s0, a1, .LBB53_1
+; RV32I-NEXT:    andi a0, a3, 255
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    bgeu s1, a0, .LBB53_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB53_2 Depth=1
-; RV32I-NEXT:    mv a2, s2
+; RV32I-NEXT:    mv a2, s3
 ; RV32I-NEXT:    j .LBB53_1
 ; RV32I-NEXT:  .LBB53_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a3
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -4774,27 +4774,27 @@ define i8 @atomicrmw_umin_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_umin_i8_acq_rel:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a6, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB53_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a4, (a0)
-; RV32IA-NEXT:    and a3, a4, a6
+; RV32IA-NEXT:    lr.w.aq a4, (a6)
+; RV32IA-NEXT:    and a2, a4, a3
 ; RV32IA-NEXT:    mv a5, a4
-; RV32IA-NEXT:    bgeu a1, a3, .LBB53_3
+; RV32IA-NEXT:    bgeu a1, a2, .LBB53_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB53_1 Depth=1
 ; RV32IA-NEXT:    xor a5, a4, a1
-; RV32IA-NEXT:    and a5, a5, a6
+; RV32IA-NEXT:    and a5, a5, a3
 ; RV32IA-NEXT:    xor a5, a4, a5
 ; RV32IA-NEXT:  .LBB53_3: # in Loop: Header=BB53_1 Depth=1
-; RV32IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV32IA-NEXT:    sc.w.rl a5, a5, (a6)
 ; RV32IA-NEXT:    bnez a5, .LBB53_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a4, a2
+; RV32IA-NEXT:    srl a0, a4, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_umin_i8_acq_rel:
@@ -4805,33 +4805,33 @@ define i8 @atomicrmw_umin_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
 ; RV64I-NEXT:    sd s3, 8(sp)
-; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lbu a0, 0(a0)
-; RV64I-NEXT:    andi s0, a1, 255
-; RV64I-NEXT:    addi s3, sp, 7
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    lbu a3, 0(a0)
+; RV64I-NEXT:    mv s3, a1
+; RV64I-NEXT:    andi s1, a1, 255
+; RV64I-NEXT:    addi s2, sp, 7
 ; RV64I-NEXT:    j .LBB53_2
 ; RV64I-NEXT:  .LBB53_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB53_2 Depth=1
-; RV64I-NEXT:    sb a0, 7(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sb a3, 7(sp)
 ; RV64I-NEXT:    addi a3, zero, 4
 ; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    call __atomic_compare_exchange_1
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lb a0, 7(sp)
-; RV64I-NEXT:    bnez a1, .LBB53_4
+; RV64I-NEXT:    lb a3, 7(sp)
+; RV64I-NEXT:    bnez a0, .LBB53_4
 ; RV64I-NEXT:  .LBB53_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    andi a1, a0, 255
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bgeu s0, a1, .LBB53_1
+; RV64I-NEXT:    andi a0, a3, 255
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bgeu s1, a0, .LBB53_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB53_2 Depth=1
-; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:    mv a2, s3
 ; RV64I-NEXT:    j .LBB53_1
 ; RV64I-NEXT:  .LBB53_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -4842,27 +4842,27 @@ define i8 @atomicrmw_umin_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_umin_i8_acq_rel:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a6, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB53_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a4, (a0)
-; RV64IA-NEXT:    and a3, a4, a6
+; RV64IA-NEXT:    lr.w.aq a4, (a6)
+; RV64IA-NEXT:    and a2, a4, a3
 ; RV64IA-NEXT:    mv a5, a4
-; RV64IA-NEXT:    bgeu a1, a3, .LBB53_3
+; RV64IA-NEXT:    bgeu a1, a2, .LBB53_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB53_1 Depth=1
 ; RV64IA-NEXT:    xor a5, a4, a1
-; RV64IA-NEXT:    and a5, a5, a6
+; RV64IA-NEXT:    and a5, a5, a3
 ; RV64IA-NEXT:    xor a5, a4, a5
 ; RV64IA-NEXT:  .LBB53_3: # in Loop: Header=BB53_1 Depth=1
-; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a6)
 ; RV64IA-NEXT:    bnez a5, .LBB53_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    srlw a0, a4, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw umin i8* %a, i8 %b acq_rel
   ret i8 %1
@@ -4877,33 +4877,33 @@ define i8 @atomicrmw_umin_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
 ; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lbu a0, 0(a0)
-; RV32I-NEXT:    andi s0, a1, 255
-; RV32I-NEXT:    addi s3, sp, 11
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lbu a3, 0(a0)
+; RV32I-NEXT:    mv s3, a1
+; RV32I-NEXT:    andi s1, a1, 255
+; RV32I-NEXT:    addi s2, sp, 11
 ; RV32I-NEXT:    j .LBB54_2
 ; RV32I-NEXT:  .LBB54_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB54_2 Depth=1
-; RV32I-NEXT:    sb a0, 11(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sb a3, 11(sp)
 ; RV32I-NEXT:    addi a3, zero, 5
 ; RV32I-NEXT:    addi a4, zero, 5
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s2
 ; RV32I-NEXT:    call __atomic_compare_exchange_1
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lb a0, 11(sp)
-; RV32I-NEXT:    bnez a1, .LBB54_4
+; RV32I-NEXT:    lb a3, 11(sp)
+; RV32I-NEXT:    bnez a0, .LBB54_4
 ; RV32I-NEXT:  .LBB54_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    andi a1, a0, 255
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bgeu s0, a1, .LBB54_1
+; RV32I-NEXT:    andi a0, a3, 255
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    bgeu s1, a0, .LBB54_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB54_2 Depth=1
-; RV32I-NEXT:    mv a2, s2
+; RV32I-NEXT:    mv a2, s3
 ; RV32I-NEXT:    j .LBB54_1
 ; RV32I-NEXT:  .LBB54_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a3
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -4914,27 +4914,27 @@ define i8 @atomicrmw_umin_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_umin_i8_seq_cst:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a6, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB54_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aqrl a4, (a0)
-; RV32IA-NEXT:    and a3, a4, a6
+; RV32IA-NEXT:    lr.w.aqrl a4, (a6)
+; RV32IA-NEXT:    and a2, a4, a3
 ; RV32IA-NEXT:    mv a5, a4
-; RV32IA-NEXT:    bgeu a1, a3, .LBB54_3
+; RV32IA-NEXT:    bgeu a1, a2, .LBB54_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB54_1 Depth=1
 ; RV32IA-NEXT:    xor a5, a4, a1
-; RV32IA-NEXT:    and a5, a5, a6
+; RV32IA-NEXT:    and a5, a5, a3
 ; RV32IA-NEXT:    xor a5, a4, a5
 ; RV32IA-NEXT:  .LBB54_3: # in Loop: Header=BB54_1 Depth=1
-; RV32IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV32IA-NEXT:    sc.w.aqrl a5, a5, (a6)
 ; RV32IA-NEXT:    bnez a5, .LBB54_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a4, a2
+; RV32IA-NEXT:    srl a0, a4, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_umin_i8_seq_cst:
@@ -4945,33 +4945,33 @@ define i8 @atomicrmw_umin_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
 ; RV64I-NEXT:    sd s3, 8(sp)
-; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lbu a0, 0(a0)
-; RV64I-NEXT:    andi s0, a1, 255
-; RV64I-NEXT:    addi s3, sp, 7
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    lbu a3, 0(a0)
+; RV64I-NEXT:    mv s3, a1
+; RV64I-NEXT:    andi s1, a1, 255
+; RV64I-NEXT:    addi s2, sp, 7
 ; RV64I-NEXT:    j .LBB54_2
 ; RV64I-NEXT:  .LBB54_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB54_2 Depth=1
-; RV64I-NEXT:    sb a0, 7(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sb a3, 7(sp)
 ; RV64I-NEXT:    addi a3, zero, 5
 ; RV64I-NEXT:    addi a4, zero, 5
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    call __atomic_compare_exchange_1
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lb a0, 7(sp)
-; RV64I-NEXT:    bnez a1, .LBB54_4
+; RV64I-NEXT:    lb a3, 7(sp)
+; RV64I-NEXT:    bnez a0, .LBB54_4
 ; RV64I-NEXT:  .LBB54_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    andi a1, a0, 255
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bgeu s0, a1, .LBB54_1
+; RV64I-NEXT:    andi a0, a3, 255
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bgeu s1, a0, .LBB54_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB54_2 Depth=1
-; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:    mv a2, s3
 ; RV64I-NEXT:    j .LBB54_1
 ; RV64I-NEXT:  .LBB54_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -4982,27 +4982,27 @@ define i8 @atomicrmw_umin_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_umin_i8_seq_cst:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a6, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB54_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aqrl a4, (a0)
-; RV64IA-NEXT:    and a3, a4, a6
+; RV64IA-NEXT:    lr.w.aqrl a4, (a6)
+; RV64IA-NEXT:    and a2, a4, a3
 ; RV64IA-NEXT:    mv a5, a4
-; RV64IA-NEXT:    bgeu a1, a3, .LBB54_3
+; RV64IA-NEXT:    bgeu a1, a2, .LBB54_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB54_1 Depth=1
 ; RV64IA-NEXT:    xor a5, a4, a1
-; RV64IA-NEXT:    and a5, a5, a6
+; RV64IA-NEXT:    and a5, a5, a3
 ; RV64IA-NEXT:    xor a5, a4, a5
 ; RV64IA-NEXT:  .LBB54_3: # in Loop: Header=BB54_1 Depth=1
-; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a6)
 ; RV64IA-NEXT:    bnez a5, .LBB54_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    srlw a0, a4, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw umin i8* %a, i8 %b seq_cst
   ret i8 %1
@@ -5021,24 +5021,24 @@ define i16 @atomicrmw_xchg_i16_monotonic(i16* %a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_xchg_i16_monotonic:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a2, a2, a3
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB55_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a4, (a0)
-; RV32IA-NEXT:    add a5, zero, a1
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    and a5, a5, a2
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w a5, a5, (a0)
+; RV32IA-NEXT:    lr.w a3, (a2)
+; RV32IA-NEXT:    mv a5, a1
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    and a5, a5, a4
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    sc.w a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB55_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a3
+; RV32IA-NEXT:    srl a0, a3, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_xchg_i16_monotonic:
@@ -5053,24 +5053,24 @@ define i16 @atomicrmw_xchg_i16_monotonic(i16* %a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_xchg_i16_monotonic:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a2, a2, a3
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB55_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a4, (a0)
-; RV64IA-NEXT:    add a5, zero, a1
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    and a5, a5, a2
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    lr.w a3, (a2)
+; RV64IA-NEXT:    mv a5, a1
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    sc.w a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB55_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    srlw a0, a3, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw xchg i16* %a, i16 %b monotonic
   ret i16 %1
@@ -5089,24 +5089,24 @@ define i16 @atomicrmw_xchg_i16_acquire(i16* %a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_xchg_i16_acquire:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a2, a2, a3
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB56_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a4, (a0)
-; RV32IA-NEXT:    add a5, zero, a1
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    and a5, a5, a2
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w a5, a5, (a0)
+; RV32IA-NEXT:    lr.w.aq a3, (a2)
+; RV32IA-NEXT:    mv a5, a1
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    and a5, a5, a4
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    sc.w a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB56_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a3
+; RV32IA-NEXT:    srl a0, a3, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_xchg_i16_acquire:
@@ -5121,24 +5121,24 @@ define i16 @atomicrmw_xchg_i16_acquire(i16* %a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_xchg_i16_acquire:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a2, a2, a3
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB56_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a4, (a0)
-; RV64IA-NEXT:    add a5, zero, a1
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    and a5, a5, a2
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    lr.w.aq a3, (a2)
+; RV64IA-NEXT:    mv a5, a1
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    sc.w a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB56_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    srlw a0, a3, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw xchg i16* %a, i16 %b acquire
   ret i16 %1
@@ -5157,24 +5157,24 @@ define i16 @atomicrmw_xchg_i16_release(i16* %a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_xchg_i16_release:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a2, a2, a3
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB57_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a4, (a0)
-; RV32IA-NEXT:    add a5, zero, a1
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    and a5, a5, a2
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV32IA-NEXT:    lr.w a3, (a2)
+; RV32IA-NEXT:    mv a5, a1
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    and a5, a5, a4
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    sc.w.rl a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB57_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a3
+; RV32IA-NEXT:    srl a0, a3, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_xchg_i16_release:
@@ -5189,24 +5189,24 @@ define i16 @atomicrmw_xchg_i16_release(i16* %a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_xchg_i16_release:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a2, a2, a3
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB57_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a4, (a0)
-; RV64IA-NEXT:    add a5, zero, a1
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    and a5, a5, a2
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    lr.w a3, (a2)
+; RV64IA-NEXT:    mv a5, a1
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB57_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    srlw a0, a3, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw xchg i16* %a, i16 %b release
   ret i16 %1
@@ -5225,24 +5225,24 @@ define i16 @atomicrmw_xchg_i16_acq_rel(i16* %a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_xchg_i16_acq_rel:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a2, a2, a3
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB58_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a4, (a0)
-; RV32IA-NEXT:    add a5, zero, a1
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    and a5, a5, a2
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV32IA-NEXT:    lr.w.aq a3, (a2)
+; RV32IA-NEXT:    mv a5, a1
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    and a5, a5, a4
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    sc.w.rl a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB58_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a3
+; RV32IA-NEXT:    srl a0, a3, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_xchg_i16_acq_rel:
@@ -5257,24 +5257,24 @@ define i16 @atomicrmw_xchg_i16_acq_rel(i16* %a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_xchg_i16_acq_rel:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a2, a2, a3
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB58_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a4, (a0)
-; RV64IA-NEXT:    add a5, zero, a1
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    and a5, a5, a2
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    lr.w.aq a3, (a2)
+; RV64IA-NEXT:    mv a5, a1
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB58_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    srlw a0, a3, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw xchg i16* %a, i16 %b acq_rel
   ret i16 %1
@@ -5293,24 +5293,24 @@ define i16 @atomicrmw_xchg_i16_seq_cst(i16* %a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_xchg_i16_seq_cst:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a2, a2, a3
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB59_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aqrl a4, (a0)
-; RV32IA-NEXT:    add a5, zero, a1
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    and a5, a5, a2
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV32IA-NEXT:    lr.w.aqrl a3, (a2)
+; RV32IA-NEXT:    mv a5, a1
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    and a5, a5, a4
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    sc.w.aqrl a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB59_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a3
+; RV32IA-NEXT:    srl a0, a3, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_xchg_i16_seq_cst:
@@ -5325,24 +5325,24 @@ define i16 @atomicrmw_xchg_i16_seq_cst(i16* %a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_xchg_i16_seq_cst:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a2, a2, a3
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB59_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aqrl a4, (a0)
-; RV64IA-NEXT:    add a5, zero, a1
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    and a5, a5, a2
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV64IA-NEXT:    lr.w.aqrl a3, (a2)
+; RV64IA-NEXT:    mv a5, a1
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB59_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    srlw a0, a3, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw xchg i16* %a, i16 %b seq_cst
   ret i16 %1
@@ -5361,24 +5361,24 @@ define i16 @atomicrmw_add_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_add_i16_monotonic:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a2, a2, a3
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB60_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a4, (a0)
-; RV32IA-NEXT:    add a5, a4, a1
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    and a5, a5, a2
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w a5, a5, (a0)
+; RV32IA-NEXT:    lr.w a3, (a2)
+; RV32IA-NEXT:    add a5, a3, a1
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    and a5, a5, a4
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    sc.w a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB60_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a3
+; RV32IA-NEXT:    srl a0, a3, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_add_i16_monotonic:
@@ -5393,24 +5393,24 @@ define i16 @atomicrmw_add_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_add_i16_monotonic:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a2, a2, a3
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB60_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a4, (a0)
-; RV64IA-NEXT:    add a5, a4, a1
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    and a5, a5, a2
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    lr.w a3, (a2)
+; RV64IA-NEXT:    add a5, a3, a1
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    sc.w a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB60_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    srlw a0, a3, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw add i16* %a, i16 %b monotonic
   ret i16 %1
@@ -5429,24 +5429,24 @@ define i16 @atomicrmw_add_i16_acquire(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_add_i16_acquire:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a2, a2, a3
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB61_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a4, (a0)
-; RV32IA-NEXT:    add a5, a4, a1
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    and a5, a5, a2
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w a5, a5, (a0)
+; RV32IA-NEXT:    lr.w.aq a3, (a2)
+; RV32IA-NEXT:    add a5, a3, a1
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    and a5, a5, a4
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    sc.w a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB61_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a3
+; RV32IA-NEXT:    srl a0, a3, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_add_i16_acquire:
@@ -5461,24 +5461,24 @@ define i16 @atomicrmw_add_i16_acquire(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_add_i16_acquire:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a2, a2, a3
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB61_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a4, (a0)
-; RV64IA-NEXT:    add a5, a4, a1
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    and a5, a5, a2
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    lr.w.aq a3, (a2)
+; RV64IA-NEXT:    add a5, a3, a1
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    sc.w a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB61_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    srlw a0, a3, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw add i16* %a, i16 %b acquire
   ret i16 %1
@@ -5497,24 +5497,24 @@ define i16 @atomicrmw_add_i16_release(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_add_i16_release:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a2, a2, a3
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB62_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a4, (a0)
-; RV32IA-NEXT:    add a5, a4, a1
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    and a5, a5, a2
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV32IA-NEXT:    lr.w a3, (a2)
+; RV32IA-NEXT:    add a5, a3, a1
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    and a5, a5, a4
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    sc.w.rl a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB62_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a3
+; RV32IA-NEXT:    srl a0, a3, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_add_i16_release:
@@ -5529,24 +5529,24 @@ define i16 @atomicrmw_add_i16_release(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_add_i16_release:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a2, a2, a3
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB62_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a4, (a0)
-; RV64IA-NEXT:    add a5, a4, a1
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    and a5, a5, a2
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    lr.w a3, (a2)
+; RV64IA-NEXT:    add a5, a3, a1
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB62_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    srlw a0, a3, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw add i16* %a, i16 %b release
   ret i16 %1
@@ -5565,24 +5565,24 @@ define i16 @atomicrmw_add_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_add_i16_acq_rel:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a2, a2, a3
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB63_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a4, (a0)
-; RV32IA-NEXT:    add a5, a4, a1
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    and a5, a5, a2
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV32IA-NEXT:    lr.w.aq a3, (a2)
+; RV32IA-NEXT:    add a5, a3, a1
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    and a5, a5, a4
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    sc.w.rl a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB63_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a3
+; RV32IA-NEXT:    srl a0, a3, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_add_i16_acq_rel:
@@ -5597,24 +5597,24 @@ define i16 @atomicrmw_add_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_add_i16_acq_rel:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a2, a2, a3
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB63_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a4, (a0)
-; RV64IA-NEXT:    add a5, a4, a1
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    and a5, a5, a2
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    lr.w.aq a3, (a2)
+; RV64IA-NEXT:    add a5, a3, a1
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB63_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    srlw a0, a3, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw add i16* %a, i16 %b acq_rel
   ret i16 %1
@@ -5633,24 +5633,24 @@ define i16 @atomicrmw_add_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_add_i16_seq_cst:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a2, a2, a3
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB64_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aqrl a4, (a0)
-; RV32IA-NEXT:    add a5, a4, a1
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    and a5, a5, a2
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV32IA-NEXT:    lr.w.aqrl a3, (a2)
+; RV32IA-NEXT:    add a5, a3, a1
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    and a5, a5, a4
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    sc.w.aqrl a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB64_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a3
+; RV32IA-NEXT:    srl a0, a3, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_add_i16_seq_cst:
@@ -5665,24 +5665,24 @@ define i16 @atomicrmw_add_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_add_i16_seq_cst:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a2, a2, a3
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB64_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aqrl a4, (a0)
-; RV64IA-NEXT:    add a5, a4, a1
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    and a5, a5, a2
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV64IA-NEXT:    lr.w.aqrl a3, (a2)
+; RV64IA-NEXT:    add a5, a3, a1
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB64_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    srlw a0, a3, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw add i16* %a, i16 %b seq_cst
   ret i16 %1
@@ -5701,24 +5701,24 @@ define i16 @atomicrmw_sub_i16_monotonic(i16* %a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_sub_i16_monotonic:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a2, a2, a3
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB65_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a4, (a0)
-; RV32IA-NEXT:    sub a5, a4, a1
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    and a5, a5, a2
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w a5, a5, (a0)
+; RV32IA-NEXT:    lr.w a3, (a2)
+; RV32IA-NEXT:    sub a5, a3, a1
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    and a5, a5, a4
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    sc.w a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB65_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a3
+; RV32IA-NEXT:    srl a0, a3, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_sub_i16_monotonic:
@@ -5733,24 +5733,24 @@ define i16 @atomicrmw_sub_i16_monotonic(i16* %a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_sub_i16_monotonic:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a2, a2, a3
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB65_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a4, (a0)
-; RV64IA-NEXT:    sub a5, a4, a1
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    and a5, a5, a2
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    lr.w a3, (a2)
+; RV64IA-NEXT:    sub a5, a3, a1
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    sc.w a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB65_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    srlw a0, a3, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw sub i16* %a, i16 %b monotonic
   ret i16 %1
@@ -5769,24 +5769,24 @@ define i16 @atomicrmw_sub_i16_acquire(i16* %a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_sub_i16_acquire:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a2, a2, a3
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB66_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a4, (a0)
-; RV32IA-NEXT:    sub a5, a4, a1
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    and a5, a5, a2
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w a5, a5, (a0)
+; RV32IA-NEXT:    lr.w.aq a3, (a2)
+; RV32IA-NEXT:    sub a5, a3, a1
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    and a5, a5, a4
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    sc.w a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB66_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a3
+; RV32IA-NEXT:    srl a0, a3, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_sub_i16_acquire:
@@ -5801,24 +5801,24 @@ define i16 @atomicrmw_sub_i16_acquire(i16* %a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_sub_i16_acquire:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a2, a2, a3
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB66_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a4, (a0)
-; RV64IA-NEXT:    sub a5, a4, a1
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    and a5, a5, a2
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    lr.w.aq a3, (a2)
+; RV64IA-NEXT:    sub a5, a3, a1
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    sc.w a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB66_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    srlw a0, a3, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw sub i16* %a, i16 %b acquire
   ret i16 %1
@@ -5837,24 +5837,24 @@ define i16 @atomicrmw_sub_i16_release(i16* %a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_sub_i16_release:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a2, a2, a3
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB67_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a4, (a0)
-; RV32IA-NEXT:    sub a5, a4, a1
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    and a5, a5, a2
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV32IA-NEXT:    lr.w a3, (a2)
+; RV32IA-NEXT:    sub a5, a3, a1
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    and a5, a5, a4
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    sc.w.rl a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB67_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a3
+; RV32IA-NEXT:    srl a0, a3, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_sub_i16_release:
@@ -5869,24 +5869,24 @@ define i16 @atomicrmw_sub_i16_release(i16* %a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_sub_i16_release:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a2, a2, a3
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB67_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a4, (a0)
-; RV64IA-NEXT:    sub a5, a4, a1
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    and a5, a5, a2
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    lr.w a3, (a2)
+; RV64IA-NEXT:    sub a5, a3, a1
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB67_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    srlw a0, a3, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw sub i16* %a, i16 %b release
   ret i16 %1
@@ -5905,24 +5905,24 @@ define i16 @atomicrmw_sub_i16_acq_rel(i16* %a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_sub_i16_acq_rel:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a2, a2, a3
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB68_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a4, (a0)
-; RV32IA-NEXT:    sub a5, a4, a1
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    and a5, a5, a2
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV32IA-NEXT:    lr.w.aq a3, (a2)
+; RV32IA-NEXT:    sub a5, a3, a1
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    and a5, a5, a4
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    sc.w.rl a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB68_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a3
+; RV32IA-NEXT:    srl a0, a3, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_sub_i16_acq_rel:
@@ -5937,24 +5937,24 @@ define i16 @atomicrmw_sub_i16_acq_rel(i16* %a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_sub_i16_acq_rel:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a2, a2, a3
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB68_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a4, (a0)
-; RV64IA-NEXT:    sub a5, a4, a1
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    and a5, a5, a2
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    lr.w.aq a3, (a2)
+; RV64IA-NEXT:    sub a5, a3, a1
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB68_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    srlw a0, a3, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw sub i16* %a, i16 %b acq_rel
   ret i16 %1
@@ -5973,24 +5973,24 @@ define i16 @atomicrmw_sub_i16_seq_cst(i16* %a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_sub_i16_seq_cst:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a2, a2, a3
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB69_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aqrl a4, (a0)
-; RV32IA-NEXT:    sub a5, a4, a1
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    and a5, a5, a2
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV32IA-NEXT:    lr.w.aqrl a3, (a2)
+; RV32IA-NEXT:    sub a5, a3, a1
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    and a5, a5, a4
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    sc.w.aqrl a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB69_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a3
+; RV32IA-NEXT:    srl a0, a3, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_sub_i16_seq_cst:
@@ -6005,24 +6005,24 @@ define i16 @atomicrmw_sub_i16_seq_cst(i16* %a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_sub_i16_seq_cst:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a2, a2, a3
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB69_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aqrl a4, (a0)
-; RV64IA-NEXT:    sub a5, a4, a1
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    and a5, a5, a2
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV64IA-NEXT:    lr.w.aqrl a3, (a2)
+; RV64IA-NEXT:    sub a5, a3, a1
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB69_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    srlw a0, a3, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw sub i16* %a, i16 %b seq_cst
   ret i16 %1
@@ -6041,18 +6041,18 @@ define i16 @atomicrmw_and_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_and_i16_monotonic:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    sll a2, a2, a3
-; RV32IA-NEXT:    not a2, a2
-; RV32IA-NEXT:    or a1, a2, a1
-; RV32IA-NEXT:    andi a0, a0, -4
-; RV32IA-NEXT:    amoand.w a0, a1, (a0)
-; RV32IA-NEXT:    srl a0, a0, a3
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    not a4, a4
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    or a1, a4, a1
+; RV32IA-NEXT:    amoand.w a1, a1, (a2)
+; RV32IA-NEXT:    srl a0, a1, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_and_i16_monotonic:
@@ -6067,18 +6067,18 @@ define i16 @atomicrmw_and_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_and_i16_monotonic:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    sllw a2, a2, a3
-; RV64IA-NEXT:    not a2, a2
-; RV64IA-NEXT:    or a1, a2, a1
-; RV64IA-NEXT:    andi a0, a0, -4
-; RV64IA-NEXT:    amoand.w a0, a1, (a0)
-; RV64IA-NEXT:    srlw a0, a0, a3
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    not a4, a4
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    or a1, a4, a1
+; RV64IA-NEXT:    amoand.w a1, a1, (a2)
+; RV64IA-NEXT:    srlw a0, a1, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw and i16* %a, i16 %b monotonic
   ret i16 %1
@@ -6097,18 +6097,18 @@ define i16 @atomicrmw_and_i16_acquire(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_and_i16_acquire:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    sll a2, a2, a3
-; RV32IA-NEXT:    not a2, a2
-; RV32IA-NEXT:    or a1, a2, a1
-; RV32IA-NEXT:    andi a0, a0, -4
-; RV32IA-NEXT:    amoand.w.aq a0, a1, (a0)
-; RV32IA-NEXT:    srl a0, a0, a3
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    not a4, a4
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    or a1, a4, a1
+; RV32IA-NEXT:    amoand.w.aq a1, a1, (a2)
+; RV32IA-NEXT:    srl a0, a1, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_and_i16_acquire:
@@ -6123,18 +6123,18 @@ define i16 @atomicrmw_and_i16_acquire(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_and_i16_acquire:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    sllw a2, a2, a3
-; RV64IA-NEXT:    not a2, a2
-; RV64IA-NEXT:    or a1, a2, a1
-; RV64IA-NEXT:    andi a0, a0, -4
-; RV64IA-NEXT:    amoand.w.aq a0, a1, (a0)
-; RV64IA-NEXT:    srlw a0, a0, a3
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    not a4, a4
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    or a1, a4, a1
+; RV64IA-NEXT:    amoand.w.aq a1, a1, (a2)
+; RV64IA-NEXT:    srlw a0, a1, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw and i16* %a, i16 %b acquire
   ret i16 %1
@@ -6153,18 +6153,18 @@ define i16 @atomicrmw_and_i16_release(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_and_i16_release:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    sll a2, a2, a3
-; RV32IA-NEXT:    not a2, a2
-; RV32IA-NEXT:    or a1, a2, a1
-; RV32IA-NEXT:    andi a0, a0, -4
-; RV32IA-NEXT:    amoand.w.rl a0, a1, (a0)
-; RV32IA-NEXT:    srl a0, a0, a3
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    not a4, a4
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    or a1, a4, a1
+; RV32IA-NEXT:    amoand.w.rl a1, a1, (a2)
+; RV32IA-NEXT:    srl a0, a1, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_and_i16_release:
@@ -6179,18 +6179,18 @@ define i16 @atomicrmw_and_i16_release(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_and_i16_release:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    sllw a2, a2, a3
-; RV64IA-NEXT:    not a2, a2
-; RV64IA-NEXT:    or a1, a2, a1
-; RV64IA-NEXT:    andi a0, a0, -4
-; RV64IA-NEXT:    amoand.w.rl a0, a1, (a0)
-; RV64IA-NEXT:    srlw a0, a0, a3
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    not a4, a4
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    or a1, a4, a1
+; RV64IA-NEXT:    amoand.w.rl a1, a1, (a2)
+; RV64IA-NEXT:    srlw a0, a1, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw and i16* %a, i16 %b release
   ret i16 %1
@@ -6209,18 +6209,18 @@ define i16 @atomicrmw_and_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_and_i16_acq_rel:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    sll a2, a2, a3
-; RV32IA-NEXT:    not a2, a2
-; RV32IA-NEXT:    or a1, a2, a1
-; RV32IA-NEXT:    andi a0, a0, -4
-; RV32IA-NEXT:    amoand.w.aqrl a0, a1, (a0)
-; RV32IA-NEXT:    srl a0, a0, a3
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    not a4, a4
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    or a1, a4, a1
+; RV32IA-NEXT:    amoand.w.aqrl a1, a1, (a2)
+; RV32IA-NEXT:    srl a0, a1, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_and_i16_acq_rel:
@@ -6235,18 +6235,18 @@ define i16 @atomicrmw_and_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_and_i16_acq_rel:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    sllw a2, a2, a3
-; RV64IA-NEXT:    not a2, a2
-; RV64IA-NEXT:    or a1, a2, a1
-; RV64IA-NEXT:    andi a0, a0, -4
-; RV64IA-NEXT:    amoand.w.aqrl a0, a1, (a0)
-; RV64IA-NEXT:    srlw a0, a0, a3
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    not a4, a4
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    or a1, a4, a1
+; RV64IA-NEXT:    amoand.w.aqrl a1, a1, (a2)
+; RV64IA-NEXT:    srlw a0, a1, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw and i16* %a, i16 %b acq_rel
   ret i16 %1
@@ -6265,18 +6265,18 @@ define i16 @atomicrmw_and_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_and_i16_seq_cst:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    sll a2, a2, a3
-; RV32IA-NEXT:    not a2, a2
-; RV32IA-NEXT:    or a1, a2, a1
-; RV32IA-NEXT:    andi a0, a0, -4
-; RV32IA-NEXT:    amoand.w.aqrl a0, a1, (a0)
-; RV32IA-NEXT:    srl a0, a0, a3
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    not a4, a4
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    or a1, a4, a1
+; RV32IA-NEXT:    amoand.w.aqrl a1, a1, (a2)
+; RV32IA-NEXT:    srl a0, a1, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_and_i16_seq_cst:
@@ -6291,18 +6291,18 @@ define i16 @atomicrmw_and_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_and_i16_seq_cst:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    sllw a2, a2, a3
-; RV64IA-NEXT:    not a2, a2
-; RV64IA-NEXT:    or a1, a2, a1
-; RV64IA-NEXT:    andi a0, a0, -4
-; RV64IA-NEXT:    amoand.w.aqrl a0, a1, (a0)
-; RV64IA-NEXT:    srlw a0, a0, a3
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    not a4, a4
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    or a1, a4, a1
+; RV64IA-NEXT:    amoand.w.aqrl a1, a1, (a2)
+; RV64IA-NEXT:    srlw a0, a1, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw and i16* %a, i16 %b seq_cst
   ret i16 %1
@@ -6321,25 +6321,25 @@ define i16 @atomicrmw_nand_i16_monotonic(i16* %a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_nand_i16_monotonic:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a2, a2, a3
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB75_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a4, (a0)
-; RV32IA-NEXT:    and a5, a4, a1
+; RV32IA-NEXT:    lr.w a3, (a2)
+; RV32IA-NEXT:    and a5, a3, a1
 ; RV32IA-NEXT:    not a5, a5
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    and a5, a5, a2
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w a5, a5, (a0)
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    and a5, a5, a4
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    sc.w a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB75_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a3
+; RV32IA-NEXT:    srl a0, a3, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_nand_i16_monotonic:
@@ -6354,25 +6354,25 @@ define i16 @atomicrmw_nand_i16_monotonic(i16* %a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_nand_i16_monotonic:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a2, a2, a3
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB75_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a4, (a0)
-; RV64IA-NEXT:    and a5, a4, a1
+; RV64IA-NEXT:    lr.w a3, (a2)
+; RV64IA-NEXT:    and a5, a3, a1
 ; RV64IA-NEXT:    not a5, a5
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    and a5, a5, a2
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    sc.w a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB75_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    srlw a0, a3, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw nand i16* %a, i16 %b monotonic
   ret i16 %1
@@ -6391,25 +6391,25 @@ define i16 @atomicrmw_nand_i16_acquire(i16* %a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_nand_i16_acquire:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a2, a2, a3
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB76_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a4, (a0)
-; RV32IA-NEXT:    and a5, a4, a1
+; RV32IA-NEXT:    lr.w.aq a3, (a2)
+; RV32IA-NEXT:    and a5, a3, a1
 ; RV32IA-NEXT:    not a5, a5
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    and a5, a5, a2
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w a5, a5, (a0)
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    and a5, a5, a4
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    sc.w a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB76_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a3
+; RV32IA-NEXT:    srl a0, a3, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_nand_i16_acquire:
@@ -6424,25 +6424,25 @@ define i16 @atomicrmw_nand_i16_acquire(i16* %a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_nand_i16_acquire:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a2, a2, a3
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB76_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a4, (a0)
-; RV64IA-NEXT:    and a5, a4, a1
+; RV64IA-NEXT:    lr.w.aq a3, (a2)
+; RV64IA-NEXT:    and a5, a3, a1
 ; RV64IA-NEXT:    not a5, a5
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    and a5, a5, a2
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    sc.w a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB76_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    srlw a0, a3, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw nand i16* %a, i16 %b acquire
   ret i16 %1
@@ -6461,25 +6461,25 @@ define i16 @atomicrmw_nand_i16_release(i16* %a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_nand_i16_release:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a2, a2, a3
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB77_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a4, (a0)
-; RV32IA-NEXT:    and a5, a4, a1
+; RV32IA-NEXT:    lr.w a3, (a2)
+; RV32IA-NEXT:    and a5, a3, a1
 ; RV32IA-NEXT:    not a5, a5
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    and a5, a5, a2
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    and a5, a5, a4
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    sc.w.rl a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB77_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a3
+; RV32IA-NEXT:    srl a0, a3, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_nand_i16_release:
@@ -6494,25 +6494,25 @@ define i16 @atomicrmw_nand_i16_release(i16* %a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_nand_i16_release:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a2, a2, a3
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB77_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a4, (a0)
-; RV64IA-NEXT:    and a5, a4, a1
+; RV64IA-NEXT:    lr.w a3, (a2)
+; RV64IA-NEXT:    and a5, a3, a1
 ; RV64IA-NEXT:    not a5, a5
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    and a5, a5, a2
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB77_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    srlw a0, a3, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw nand i16* %a, i16 %b release
   ret i16 %1
@@ -6531,25 +6531,25 @@ define i16 @atomicrmw_nand_i16_acq_rel(i16* %a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_nand_i16_acq_rel:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a2, a2, a3
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB78_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a4, (a0)
-; RV32IA-NEXT:    and a5, a4, a1
+; RV32IA-NEXT:    lr.w.aq a3, (a2)
+; RV32IA-NEXT:    and a5, a3, a1
 ; RV32IA-NEXT:    not a5, a5
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    and a5, a5, a2
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    and a5, a5, a4
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    sc.w.rl a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB78_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a3
+; RV32IA-NEXT:    srl a0, a3, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_nand_i16_acq_rel:
@@ -6564,25 +6564,25 @@ define i16 @atomicrmw_nand_i16_acq_rel(i16* %a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_nand_i16_acq_rel:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a2, a2, a3
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB78_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a4, (a0)
-; RV64IA-NEXT:    and a5, a4, a1
+; RV64IA-NEXT:    lr.w.aq a3, (a2)
+; RV64IA-NEXT:    and a5, a3, a1
 ; RV64IA-NEXT:    not a5, a5
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    and a5, a5, a2
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB78_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    srlw a0, a3, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw nand i16* %a, i16 %b acq_rel
   ret i16 %1
@@ -6601,25 +6601,25 @@ define i16 @atomicrmw_nand_i16_seq_cst(i16* %a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_nand_i16_seq_cst:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a2, a2, a3
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB79_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aqrl a4, (a0)
-; RV32IA-NEXT:    and a5, a4, a1
+; RV32IA-NEXT:    lr.w.aqrl a3, (a2)
+; RV32IA-NEXT:    and a5, a3, a1
 ; RV32IA-NEXT:    not a5, a5
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    and a5, a5, a2
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    and a5, a5, a4
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    sc.w.aqrl a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB79_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a3
+; RV32IA-NEXT:    srl a0, a3, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_nand_i16_seq_cst:
@@ -6634,25 +6634,25 @@ define i16 @atomicrmw_nand_i16_seq_cst(i16* %a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_nand_i16_seq_cst:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a2, a2, a3
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB79_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aqrl a4, (a0)
-; RV64IA-NEXT:    and a5, a4, a1
+; RV64IA-NEXT:    lr.w.aqrl a3, (a2)
+; RV64IA-NEXT:    and a5, a3, a1
 ; RV64IA-NEXT:    not a5, a5
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    and a5, a5, a2
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB79_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    srlw a0, a3, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw nand i16* %a, i16 %b seq_cst
   ret i16 %1
@@ -6671,15 +6671,15 @@ define i16 @atomicrmw_or_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_or_i16_monotonic:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
-; RV32IA-NEXT:    amoor.w a0, a1, (a0)
-; RV32IA-NEXT:    srl a0, a0, a2
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    amoor.w a1, a1, (a2)
+; RV32IA-NEXT:    srl a0, a1, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_or_i16_monotonic:
@@ -6694,15 +6694,15 @@ define i16 @atomicrmw_or_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_or_i16_monotonic:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
-; RV64IA-NEXT:    amoor.w a0, a1, (a0)
-; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    amoor.w a1, a1, (a2)
+; RV64IA-NEXT:    srlw a0, a1, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw or i16* %a, i16 %b monotonic
   ret i16 %1
@@ -6721,15 +6721,15 @@ define i16 @atomicrmw_or_i16_acquire(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_or_i16_acquire:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
-; RV32IA-NEXT:    amoor.w.aq a0, a1, (a0)
-; RV32IA-NEXT:    srl a0, a0, a2
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    amoor.w.aq a1, a1, (a2)
+; RV32IA-NEXT:    srl a0, a1, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_or_i16_acquire:
@@ -6744,15 +6744,15 @@ define i16 @atomicrmw_or_i16_acquire(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_or_i16_acquire:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
-; RV64IA-NEXT:    amoor.w.aq a0, a1, (a0)
-; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    amoor.w.aq a1, a1, (a2)
+; RV64IA-NEXT:    srlw a0, a1, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw or i16* %a, i16 %b acquire
   ret i16 %1
@@ -6771,15 +6771,15 @@ define i16 @atomicrmw_or_i16_release(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_or_i16_release:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
-; RV32IA-NEXT:    amoor.w.rl a0, a1, (a0)
-; RV32IA-NEXT:    srl a0, a0, a2
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    amoor.w.rl a1, a1, (a2)
+; RV32IA-NEXT:    srl a0, a1, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_or_i16_release:
@@ -6794,15 +6794,15 @@ define i16 @atomicrmw_or_i16_release(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_or_i16_release:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
-; RV64IA-NEXT:    amoor.w.rl a0, a1, (a0)
-; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    amoor.w.rl a1, a1, (a2)
+; RV64IA-NEXT:    srlw a0, a1, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw or i16* %a, i16 %b release
   ret i16 %1
@@ -6821,15 +6821,15 @@ define i16 @atomicrmw_or_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_or_i16_acq_rel:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
-; RV32IA-NEXT:    amoor.w.aqrl a0, a1, (a0)
-; RV32IA-NEXT:    srl a0, a0, a2
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    amoor.w.aqrl a1, a1, (a2)
+; RV32IA-NEXT:    srl a0, a1, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_or_i16_acq_rel:
@@ -6844,15 +6844,15 @@ define i16 @atomicrmw_or_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_or_i16_acq_rel:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
-; RV64IA-NEXT:    amoor.w.aqrl a0, a1, (a0)
-; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    amoor.w.aqrl a1, a1, (a2)
+; RV64IA-NEXT:    srlw a0, a1, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw or i16* %a, i16 %b acq_rel
   ret i16 %1
@@ -6871,15 +6871,15 @@ define i16 @atomicrmw_or_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_or_i16_seq_cst:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
-; RV32IA-NEXT:    amoor.w.aqrl a0, a1, (a0)
-; RV32IA-NEXT:    srl a0, a0, a2
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    amoor.w.aqrl a1, a1, (a2)
+; RV32IA-NEXT:    srl a0, a1, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_or_i16_seq_cst:
@@ -6894,15 +6894,15 @@ define i16 @atomicrmw_or_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_or_i16_seq_cst:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
-; RV64IA-NEXT:    amoor.w.aqrl a0, a1, (a0)
-; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    amoor.w.aqrl a1, a1, (a2)
+; RV64IA-NEXT:    srlw a0, a1, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw or i16* %a, i16 %b seq_cst
   ret i16 %1
@@ -6921,15 +6921,15 @@ define i16 @atomicrmw_xor_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_xor_i16_monotonic:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
-; RV32IA-NEXT:    amoxor.w a0, a1, (a0)
-; RV32IA-NEXT:    srl a0, a0, a2
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    amoxor.w a1, a1, (a2)
+; RV32IA-NEXT:    srl a0, a1, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_xor_i16_monotonic:
@@ -6944,15 +6944,15 @@ define i16 @atomicrmw_xor_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_xor_i16_monotonic:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
-; RV64IA-NEXT:    amoxor.w a0, a1, (a0)
-; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    amoxor.w a1, a1, (a2)
+; RV64IA-NEXT:    srlw a0, a1, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw xor i16* %a, i16 %b monotonic
   ret i16 %1
@@ -6971,15 +6971,15 @@ define i16 @atomicrmw_xor_i16_acquire(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_xor_i16_acquire:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
-; RV32IA-NEXT:    amoxor.w.aq a0, a1, (a0)
-; RV32IA-NEXT:    srl a0, a0, a2
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    amoxor.w.aq a1, a1, (a2)
+; RV32IA-NEXT:    srl a0, a1, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_xor_i16_acquire:
@@ -6994,15 +6994,15 @@ define i16 @atomicrmw_xor_i16_acquire(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_xor_i16_acquire:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
-; RV64IA-NEXT:    amoxor.w.aq a0, a1, (a0)
-; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    amoxor.w.aq a1, a1, (a2)
+; RV64IA-NEXT:    srlw a0, a1, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw xor i16* %a, i16 %b acquire
   ret i16 %1
@@ -7021,15 +7021,15 @@ define i16 @atomicrmw_xor_i16_release(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_xor_i16_release:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
-; RV32IA-NEXT:    amoxor.w.rl a0, a1, (a0)
-; RV32IA-NEXT:    srl a0, a0, a2
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    amoxor.w.rl a1, a1, (a2)
+; RV32IA-NEXT:    srl a0, a1, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_xor_i16_release:
@@ -7044,15 +7044,15 @@ define i16 @atomicrmw_xor_i16_release(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_xor_i16_release:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
-; RV64IA-NEXT:    amoxor.w.rl a0, a1, (a0)
-; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    amoxor.w.rl a1, a1, (a2)
+; RV64IA-NEXT:    srlw a0, a1, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw xor i16* %a, i16 %b release
   ret i16 %1
@@ -7071,15 +7071,15 @@ define i16 @atomicrmw_xor_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_xor_i16_acq_rel:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
-; RV32IA-NEXT:    amoxor.w.aqrl a0, a1, (a0)
-; RV32IA-NEXT:    srl a0, a0, a2
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    amoxor.w.aqrl a1, a1, (a2)
+; RV32IA-NEXT:    srl a0, a1, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_xor_i16_acq_rel:
@@ -7094,15 +7094,15 @@ define i16 @atomicrmw_xor_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_xor_i16_acq_rel:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
-; RV64IA-NEXT:    amoxor.w.aqrl a0, a1, (a0)
-; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    amoxor.w.aqrl a1, a1, (a2)
+; RV64IA-NEXT:    srlw a0, a1, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw xor i16* %a, i16 %b acq_rel
   ret i16 %1
@@ -7121,15 +7121,15 @@ define i16 @atomicrmw_xor_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_xor_i16_seq_cst:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
-; RV32IA-NEXT:    amoxor.w.aqrl a0, a1, (a0)
-; RV32IA-NEXT:    srl a0, a0, a2
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    amoxor.w.aqrl a1, a1, (a2)
+; RV32IA-NEXT:    srl a0, a1, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_xor_i16_seq_cst:
@@ -7144,15 +7144,15 @@ define i16 @atomicrmw_xor_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_xor_i16_seq_cst:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
-; RV64IA-NEXT:    amoxor.w.aqrl a0, a1, (a0)
-; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    amoxor.w.aqrl a1, a1, (a2)
+; RV64IA-NEXT:    srlw a0, a1, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw xor i16* %a, i16 %b seq_cst
   ret i16 %1
@@ -7169,33 +7169,33 @@ define i16 @atomicrmw_max_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    sw s3, 12(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lhu a0, 0(a0)
-; RV32I-NEXT:    slli a1, a1, 16
-; RV32I-NEXT:    srai s0, a1, 16
+; RV32I-NEXT:    lhu a1, 0(a0)
+; RV32I-NEXT:    slli a0, s2, 16
+; RV32I-NEXT:    srai s0, a0, 16
 ; RV32I-NEXT:    addi s3, sp, 10
 ; RV32I-NEXT:    j .LBB90_2
 ; RV32I-NEXT:  .LBB90_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB90_2 Depth=1
-; RV32I-NEXT:    sh a0, 10(sp)
+; RV32I-NEXT:    sh a1, 10(sp)
 ; RV32I-NEXT:    mv a0, s1
 ; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    mv a3, zero
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_2
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lh a0, 10(sp)
-; RV32I-NEXT:    bnez a1, .LBB90_4
+; RV32I-NEXT:    lh a1, 10(sp)
+; RV32I-NEXT:    bnez a0, .LBB90_4
 ; RV32I-NEXT:  .LBB90_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    slli a1, a0, 16
-; RV32I-NEXT:    srai a1, a1, 16
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    blt s0, a1, .LBB90_1
+; RV32I-NEXT:    slli a0, a1, 16
+; RV32I-NEXT:    srai a0, a0, 16
+; RV32I-NEXT:    mv a2, a1
+; RV32I-NEXT:    blt s0, a0, .LBB90_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB90_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:    j .LBB90_1
 ; RV32I-NEXT:  .LBB90_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -7206,33 +7206,33 @@ define i16 @atomicrmw_max_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_max_i16_monotonic:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    addi a3, zero, 16
-; RV32IA-NEXT:    sub a6, a3, a2
-; RV32IA-NEXT:    lui a4, 16
-; RV32IA-NEXT:    addi a4, a4, -1
-; RV32IA-NEXT:    sll a7, a4, a2
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a7, a3, a0
 ; RV32IA-NEXT:    slli a1, a1, 16
 ; RV32IA-NEXT:    srai a1, a1, 16
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    addi a4, zero, 16
+; RV32IA-NEXT:    sub a4, a4, a0
 ; RV32IA-NEXT:  .LBB90_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a5, (a0)
-; RV32IA-NEXT:    and a4, a5, a7
-; RV32IA-NEXT:    mv a3, a5
-; RV32IA-NEXT:    sll a4, a4, a6
-; RV32IA-NEXT:    sra a4, a4, a6
-; RV32IA-NEXT:    bge a4, a1, .LBB90_3
+; RV32IA-NEXT:    lr.w a5, (a6)
+; RV32IA-NEXT:    and a3, a5, a7
+; RV32IA-NEXT:    mv a2, a5
+; RV32IA-NEXT:    sll a3, a3, a4
+; RV32IA-NEXT:    sra a3, a3, a4
+; RV32IA-NEXT:    bge a3, a1, .LBB90_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB90_1 Depth=1
-; RV32IA-NEXT:    xor a3, a5, a1
-; RV32IA-NEXT:    and a3, a3, a7
-; RV32IA-NEXT:    xor a3, a5, a3
+; RV32IA-NEXT:    xor a2, a5, a1
+; RV32IA-NEXT:    and a2, a2, a7
+; RV32IA-NEXT:    xor a2, a5, a2
 ; RV32IA-NEXT:  .LBB90_3: # in Loop: Header=BB90_1 Depth=1
-; RV32IA-NEXT:    sc.w a3, a3, (a0)
-; RV32IA-NEXT:    bnez a3, .LBB90_1
+; RV32IA-NEXT:    sc.w a2, a2, (a6)
+; RV32IA-NEXT:    bnez a2, .LBB90_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a5, a2
+; RV32IA-NEXT:    srl a0, a5, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_max_i16_monotonic:
@@ -7245,33 +7245,33 @@ define i16 @atomicrmw_max_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    sd s3, 8(sp)
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lhu a0, 0(a0)
-; RV64I-NEXT:    slli a1, a1, 48
-; RV64I-NEXT:    srai s0, a1, 48
+; RV64I-NEXT:    lhu a1, 0(a0)
+; RV64I-NEXT:    slli a0, s2, 48
+; RV64I-NEXT:    srai s0, a0, 48
 ; RV64I-NEXT:    addi s3, sp, 6
 ; RV64I-NEXT:    j .LBB90_2
 ; RV64I-NEXT:  .LBB90_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB90_2 Depth=1
-; RV64I-NEXT:    sh a0, 6(sp)
+; RV64I-NEXT:    sh a1, 6(sp)
 ; RV64I-NEXT:    mv a0, s1
 ; RV64I-NEXT:    mv a1, s3
 ; RV64I-NEXT:    mv a3, zero
 ; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    call __atomic_compare_exchange_2
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lh a0, 6(sp)
-; RV64I-NEXT:    bnez a1, .LBB90_4
+; RV64I-NEXT:    lh a1, 6(sp)
+; RV64I-NEXT:    bnez a0, .LBB90_4
 ; RV64I-NEXT:  .LBB90_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    slli a1, a0, 48
-; RV64I-NEXT:    srai a1, a1, 48
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    blt s0, a1, .LBB90_1
+; RV64I-NEXT:    slli a0, a1, 48
+; RV64I-NEXT:    srai a0, a0, 48
+; RV64I-NEXT:    mv a2, a1
+; RV64I-NEXT:    blt s0, a0, .LBB90_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB90_2 Depth=1
 ; RV64I-NEXT:    mv a2, s2
 ; RV64I-NEXT:    j .LBB90_1
 ; RV64I-NEXT:  .LBB90_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -7282,33 +7282,33 @@ define i16 @atomicrmw_max_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_max_i16_monotonic:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    addi a3, zero, 48
-; RV64IA-NEXT:    sub a6, a3, a2
-; RV64IA-NEXT:    lui a4, 16
-; RV64IA-NEXT:    addiw a4, a4, -1
-; RV64IA-NEXT:    sllw a7, a4, a2
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a7, a3, a0
 ; RV64IA-NEXT:    slli a1, a1, 48
 ; RV64IA-NEXT:    srai a1, a1, 48
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    addi a4, zero, 48
+; RV64IA-NEXT:    sub a4, a4, a0
 ; RV64IA-NEXT:  .LBB90_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a5, (a0)
-; RV64IA-NEXT:    and a4, a5, a7
-; RV64IA-NEXT:    mv a3, a5
-; RV64IA-NEXT:    sll a4, a4, a6
-; RV64IA-NEXT:    sra a4, a4, a6
-; RV64IA-NEXT:    bge a4, a1, .LBB90_3
+; RV64IA-NEXT:    lr.w a5, (a6)
+; RV64IA-NEXT:    and a3, a5, a7
+; RV64IA-NEXT:    mv a2, a5
+; RV64IA-NEXT:    sll a3, a3, a4
+; RV64IA-NEXT:    sra a3, a3, a4
+; RV64IA-NEXT:    bge a3, a1, .LBB90_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB90_1 Depth=1
-; RV64IA-NEXT:    xor a3, a5, a1
-; RV64IA-NEXT:    and a3, a3, a7
-; RV64IA-NEXT:    xor a3, a5, a3
+; RV64IA-NEXT:    xor a2, a5, a1
+; RV64IA-NEXT:    and a2, a2, a7
+; RV64IA-NEXT:    xor a2, a5, a2
 ; RV64IA-NEXT:  .LBB90_3: # in Loop: Header=BB90_1 Depth=1
-; RV64IA-NEXT:    sc.w a3, a3, (a0)
-; RV64IA-NEXT:    bnez a3, .LBB90_1
+; RV64IA-NEXT:    sc.w a2, a2, (a6)
+; RV64IA-NEXT:    bnez a2, .LBB90_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a5, a2
+; RV64IA-NEXT:    srlw a0, a5, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw max i16* %a, i16 %b monotonic
   ret i16 %1
@@ -7325,33 +7325,33 @@ define i16 @atomicrmw_max_i16_acquire(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    sw s3, 12(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lhu a0, 0(a0)
-; RV32I-NEXT:    slli a1, a1, 16
-; RV32I-NEXT:    srai s0, a1, 16
+; RV32I-NEXT:    lhu a1, 0(a0)
+; RV32I-NEXT:    slli a0, s2, 16
+; RV32I-NEXT:    srai s0, a0, 16
 ; RV32I-NEXT:    addi s3, sp, 10
 ; RV32I-NEXT:    j .LBB91_2
 ; RV32I-NEXT:  .LBB91_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB91_2 Depth=1
-; RV32I-NEXT:    sh a0, 10(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sh a1, 10(sp)
 ; RV32I-NEXT:    addi a3, zero, 2
 ; RV32I-NEXT:    addi a4, zero, 2
+; RV32I-NEXT:    mv a0, s1
+; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    call __atomic_compare_exchange_2
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lh a0, 10(sp)
-; RV32I-NEXT:    bnez a1, .LBB91_4
+; RV32I-NEXT:    lh a1, 10(sp)
+; RV32I-NEXT:    bnez a0, .LBB91_4
 ; RV32I-NEXT:  .LBB91_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    slli a1, a0, 16
-; RV32I-NEXT:    srai a1, a1, 16
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    blt s0, a1, .LBB91_1
+; RV32I-NEXT:    slli a0, a1, 16
+; RV32I-NEXT:    srai a0, a0, 16
+; RV32I-NEXT:    mv a2, a1
+; RV32I-NEXT:    blt s0, a0, .LBB91_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB91_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:    j .LBB91_1
 ; RV32I-NEXT:  .LBB91_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -7362,33 +7362,33 @@ define i16 @atomicrmw_max_i16_acquire(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_max_i16_acquire:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    addi a3, zero, 16
-; RV32IA-NEXT:    sub a6, a3, a2
-; RV32IA-NEXT:    lui a4, 16
-; RV32IA-NEXT:    addi a4, a4, -1
-; RV32IA-NEXT:    sll a7, a4, a2
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a7, a3, a0
 ; RV32IA-NEXT:    slli a1, a1, 16
 ; RV32IA-NEXT:    srai a1, a1, 16
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    addi a4, zero, 16
+; RV32IA-NEXT:    sub a4, a4, a0
 ; RV32IA-NEXT:  .LBB91_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a5, (a0)
-; RV32IA-NEXT:    and a4, a5, a7
-; RV32IA-NEXT:    mv a3, a5
-; RV32IA-NEXT:    sll a4, a4, a6
-; RV32IA-NEXT:    sra a4, a4, a6
-; RV32IA-NEXT:    bge a4, a1, .LBB91_3
+; RV32IA-NEXT:    lr.w.aq a5, (a6)
+; RV32IA-NEXT:    and a3, a5, a7
+; RV32IA-NEXT:    mv a2, a5
+; RV32IA-NEXT:    sll a3, a3, a4
+; RV32IA-NEXT:    sra a3, a3, a4
+; RV32IA-NEXT:    bge a3, a1, .LBB91_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB91_1 Depth=1
-; RV32IA-NEXT:    xor a3, a5, a1
-; RV32IA-NEXT:    and a3, a3, a7
-; RV32IA-NEXT:    xor a3, a5, a3
+; RV32IA-NEXT:    xor a2, a5, a1
+; RV32IA-NEXT:    and a2, a2, a7
+; RV32IA-NEXT:    xor a2, a5, a2
 ; RV32IA-NEXT:  .LBB91_3: # in Loop: Header=BB91_1 Depth=1
-; RV32IA-NEXT:    sc.w a3, a3, (a0)
-; RV32IA-NEXT:    bnez a3, .LBB91_1
+; RV32IA-NEXT:    sc.w a2, a2, (a6)
+; RV32IA-NEXT:    bnez a2, .LBB91_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a5, a2
+; RV32IA-NEXT:    srl a0, a5, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_max_i16_acquire:
@@ -7401,33 +7401,33 @@ define i16 @atomicrmw_max_i16_acquire(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    sd s3, 8(sp)
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lhu a0, 0(a0)
-; RV64I-NEXT:    slli a1, a1, 48
-; RV64I-NEXT:    srai s0, a1, 48
+; RV64I-NEXT:    lhu a1, 0(a0)
+; RV64I-NEXT:    slli a0, s2, 48
+; RV64I-NEXT:    srai s0, a0, 48
 ; RV64I-NEXT:    addi s3, sp, 6
 ; RV64I-NEXT:    j .LBB91_2
 ; RV64I-NEXT:  .LBB91_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB91_2 Depth=1
-; RV64I-NEXT:    sh a0, 6(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sh a1, 6(sp)
 ; RV64I-NEXT:    addi a3, zero, 2
 ; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    mv a0, s1
+; RV64I-NEXT:    mv a1, s3
 ; RV64I-NEXT:    call __atomic_compare_exchange_2
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lh a0, 6(sp)
-; RV64I-NEXT:    bnez a1, .LBB91_4
+; RV64I-NEXT:    lh a1, 6(sp)
+; RV64I-NEXT:    bnez a0, .LBB91_4
 ; RV64I-NEXT:  .LBB91_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    slli a1, a0, 48
-; RV64I-NEXT:    srai a1, a1, 48
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    blt s0, a1, .LBB91_1
+; RV64I-NEXT:    slli a0, a1, 48
+; RV64I-NEXT:    srai a0, a0, 48
+; RV64I-NEXT:    mv a2, a1
+; RV64I-NEXT:    blt s0, a0, .LBB91_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB91_2 Depth=1
 ; RV64I-NEXT:    mv a2, s2
 ; RV64I-NEXT:    j .LBB91_1
 ; RV64I-NEXT:  .LBB91_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -7438,33 +7438,33 @@ define i16 @atomicrmw_max_i16_acquire(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_max_i16_acquire:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    addi a3, zero, 48
-; RV64IA-NEXT:    sub a6, a3, a2
-; RV64IA-NEXT:    lui a4, 16
-; RV64IA-NEXT:    addiw a4, a4, -1
-; RV64IA-NEXT:    sllw a7, a4, a2
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a7, a3, a0
 ; RV64IA-NEXT:    slli a1, a1, 48
 ; RV64IA-NEXT:    srai a1, a1, 48
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    addi a4, zero, 48
+; RV64IA-NEXT:    sub a4, a4, a0
 ; RV64IA-NEXT:  .LBB91_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a5, (a0)
-; RV64IA-NEXT:    and a4, a5, a7
-; RV64IA-NEXT:    mv a3, a5
-; RV64IA-NEXT:    sll a4, a4, a6
-; RV64IA-NEXT:    sra a4, a4, a6
-; RV64IA-NEXT:    bge a4, a1, .LBB91_3
+; RV64IA-NEXT:    lr.w.aq a5, (a6)
+; RV64IA-NEXT:    and a3, a5, a7
+; RV64IA-NEXT:    mv a2, a5
+; RV64IA-NEXT:    sll a3, a3, a4
+; RV64IA-NEXT:    sra a3, a3, a4
+; RV64IA-NEXT:    bge a3, a1, .LBB91_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB91_1 Depth=1
-; RV64IA-NEXT:    xor a3, a5, a1
-; RV64IA-NEXT:    and a3, a3, a7
-; RV64IA-NEXT:    xor a3, a5, a3
+; RV64IA-NEXT:    xor a2, a5, a1
+; RV64IA-NEXT:    and a2, a2, a7
+; RV64IA-NEXT:    xor a2, a5, a2
 ; RV64IA-NEXT:  .LBB91_3: # in Loop: Header=BB91_1 Depth=1
-; RV64IA-NEXT:    sc.w a3, a3, (a0)
-; RV64IA-NEXT:    bnez a3, .LBB91_1
+; RV64IA-NEXT:    sc.w a2, a2, (a6)
+; RV64IA-NEXT:    bnez a2, .LBB91_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a5, a2
+; RV64IA-NEXT:    srlw a0, a5, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw max i16* %a, i16 %b acquire
   ret i16 %1
@@ -7481,33 +7481,33 @@ define i16 @atomicrmw_max_i16_release(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    sw s3, 12(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lhu a0, 0(a0)
-; RV32I-NEXT:    slli a1, a1, 16
-; RV32I-NEXT:    srai s0, a1, 16
+; RV32I-NEXT:    lhu a1, 0(a0)
+; RV32I-NEXT:    slli a0, s2, 16
+; RV32I-NEXT:    srai s0, a0, 16
 ; RV32I-NEXT:    addi s3, sp, 10
 ; RV32I-NEXT:    j .LBB92_2
 ; RV32I-NEXT:  .LBB92_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB92_2 Depth=1
-; RV32I-NEXT:    sh a0, 10(sp)
+; RV32I-NEXT:    sh a1, 10(sp)
+; RV32I-NEXT:    addi a3, zero, 3
 ; RV32I-NEXT:    mv a0, s1
 ; RV32I-NEXT:    mv a1, s3
-; RV32I-NEXT:    addi a3, zero, 3
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_2
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lh a0, 10(sp)
-; RV32I-NEXT:    bnez a1, .LBB92_4
+; RV32I-NEXT:    lh a1, 10(sp)
+; RV32I-NEXT:    bnez a0, .LBB92_4
 ; RV32I-NEXT:  .LBB92_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    slli a1, a0, 16
-; RV32I-NEXT:    srai a1, a1, 16
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    blt s0, a1, .LBB92_1
+; RV32I-NEXT:    slli a0, a1, 16
+; RV32I-NEXT:    srai a0, a0, 16
+; RV32I-NEXT:    mv a2, a1
+; RV32I-NEXT:    blt s0, a0, .LBB92_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB92_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:    j .LBB92_1
 ; RV32I-NEXT:  .LBB92_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -7518,33 +7518,33 @@ define i16 @atomicrmw_max_i16_release(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_max_i16_release:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    addi a3, zero, 16
-; RV32IA-NEXT:    sub a6, a3, a2
-; RV32IA-NEXT:    lui a4, 16
-; RV32IA-NEXT:    addi a4, a4, -1
-; RV32IA-NEXT:    sll a7, a4, a2
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a7, a3, a0
 ; RV32IA-NEXT:    slli a1, a1, 16
 ; RV32IA-NEXT:    srai a1, a1, 16
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    addi a4, zero, 16
+; RV32IA-NEXT:    sub a4, a4, a0
 ; RV32IA-NEXT:  .LBB92_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a5, (a0)
-; RV32IA-NEXT:    and a4, a5, a7
-; RV32IA-NEXT:    mv a3, a5
-; RV32IA-NEXT:    sll a4, a4, a6
-; RV32IA-NEXT:    sra a4, a4, a6
-; RV32IA-NEXT:    bge a4, a1, .LBB92_3
+; RV32IA-NEXT:    lr.w a5, (a6)
+; RV32IA-NEXT:    and a3, a5, a7
+; RV32IA-NEXT:    mv a2, a5
+; RV32IA-NEXT:    sll a3, a3, a4
+; RV32IA-NEXT:    sra a3, a3, a4
+; RV32IA-NEXT:    bge a3, a1, .LBB92_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB92_1 Depth=1
-; RV32IA-NEXT:    xor a3, a5, a1
-; RV32IA-NEXT:    and a3, a3, a7
-; RV32IA-NEXT:    xor a3, a5, a3
+; RV32IA-NEXT:    xor a2, a5, a1
+; RV32IA-NEXT:    and a2, a2, a7
+; RV32IA-NEXT:    xor a2, a5, a2
 ; RV32IA-NEXT:  .LBB92_3: # in Loop: Header=BB92_1 Depth=1
-; RV32IA-NEXT:    sc.w.rl a3, a3, (a0)
-; RV32IA-NEXT:    bnez a3, .LBB92_1
+; RV32IA-NEXT:    sc.w.rl a2, a2, (a6)
+; RV32IA-NEXT:    bnez a2, .LBB92_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a5, a2
+; RV32IA-NEXT:    srl a0, a5, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_max_i16_release:
@@ -7557,33 +7557,33 @@ define i16 @atomicrmw_max_i16_release(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    sd s3, 8(sp)
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lhu a0, 0(a0)
-; RV64I-NEXT:    slli a1, a1, 48
-; RV64I-NEXT:    srai s0, a1, 48
+; RV64I-NEXT:    lhu a1, 0(a0)
+; RV64I-NEXT:    slli a0, s2, 48
+; RV64I-NEXT:    srai s0, a0, 48
 ; RV64I-NEXT:    addi s3, sp, 6
 ; RV64I-NEXT:    j .LBB92_2
 ; RV64I-NEXT:  .LBB92_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB92_2 Depth=1
-; RV64I-NEXT:    sh a0, 6(sp)
+; RV64I-NEXT:    sh a1, 6(sp)
+; RV64I-NEXT:    addi a3, zero, 3
 ; RV64I-NEXT:    mv a0, s1
 ; RV64I-NEXT:    mv a1, s3
-; RV64I-NEXT:    addi a3, zero, 3
 ; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    call __atomic_compare_exchange_2
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lh a0, 6(sp)
-; RV64I-NEXT:    bnez a1, .LBB92_4
+; RV64I-NEXT:    lh a1, 6(sp)
+; RV64I-NEXT:    bnez a0, .LBB92_4
 ; RV64I-NEXT:  .LBB92_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    slli a1, a0, 48
-; RV64I-NEXT:    srai a1, a1, 48
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    blt s0, a1, .LBB92_1
+; RV64I-NEXT:    slli a0, a1, 48
+; RV64I-NEXT:    srai a0, a0, 48
+; RV64I-NEXT:    mv a2, a1
+; RV64I-NEXT:    blt s0, a0, .LBB92_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB92_2 Depth=1
 ; RV64I-NEXT:    mv a2, s2
 ; RV64I-NEXT:    j .LBB92_1
 ; RV64I-NEXT:  .LBB92_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -7594,33 +7594,33 @@ define i16 @atomicrmw_max_i16_release(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_max_i16_release:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    addi a3, zero, 48
-; RV64IA-NEXT:    sub a6, a3, a2
-; RV64IA-NEXT:    lui a4, 16
-; RV64IA-NEXT:    addiw a4, a4, -1
-; RV64IA-NEXT:    sllw a7, a4, a2
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a7, a3, a0
 ; RV64IA-NEXT:    slli a1, a1, 48
 ; RV64IA-NEXT:    srai a1, a1, 48
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    addi a4, zero, 48
+; RV64IA-NEXT:    sub a4, a4, a0
 ; RV64IA-NEXT:  .LBB92_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a5, (a0)
-; RV64IA-NEXT:    and a4, a5, a7
-; RV64IA-NEXT:    mv a3, a5
-; RV64IA-NEXT:    sll a4, a4, a6
-; RV64IA-NEXT:    sra a4, a4, a6
-; RV64IA-NEXT:    bge a4, a1, .LBB92_3
+; RV64IA-NEXT:    lr.w a5, (a6)
+; RV64IA-NEXT:    and a3, a5, a7
+; RV64IA-NEXT:    mv a2, a5
+; RV64IA-NEXT:    sll a3, a3, a4
+; RV64IA-NEXT:    sra a3, a3, a4
+; RV64IA-NEXT:    bge a3, a1, .LBB92_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB92_1 Depth=1
-; RV64IA-NEXT:    xor a3, a5, a1
-; RV64IA-NEXT:    and a3, a3, a7
-; RV64IA-NEXT:    xor a3, a5, a3
+; RV64IA-NEXT:    xor a2, a5, a1
+; RV64IA-NEXT:    and a2, a2, a7
+; RV64IA-NEXT:    xor a2, a5, a2
 ; RV64IA-NEXT:  .LBB92_3: # in Loop: Header=BB92_1 Depth=1
-; RV64IA-NEXT:    sc.w.rl a3, a3, (a0)
-; RV64IA-NEXT:    bnez a3, .LBB92_1
+; RV64IA-NEXT:    sc.w.rl a2, a2, (a6)
+; RV64IA-NEXT:    bnez a2, .LBB92_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a5, a2
+; RV64IA-NEXT:    srlw a0, a5, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw max i16* %a, i16 %b release
   ret i16 %1
@@ -7637,33 +7637,33 @@ define i16 @atomicrmw_max_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    sw s3, 12(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lhu a0, 0(a0)
-; RV32I-NEXT:    slli a1, a1, 16
-; RV32I-NEXT:    srai s0, a1, 16
+; RV32I-NEXT:    lhu a1, 0(a0)
+; RV32I-NEXT:    slli a0, s2, 16
+; RV32I-NEXT:    srai s0, a0, 16
 ; RV32I-NEXT:    addi s3, sp, 10
 ; RV32I-NEXT:    j .LBB93_2
 ; RV32I-NEXT:  .LBB93_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB93_2 Depth=1
-; RV32I-NEXT:    sh a0, 10(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sh a1, 10(sp)
 ; RV32I-NEXT:    addi a3, zero, 4
 ; RV32I-NEXT:    addi a4, zero, 2
+; RV32I-NEXT:    mv a0, s1
+; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    call __atomic_compare_exchange_2
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lh a0, 10(sp)
-; RV32I-NEXT:    bnez a1, .LBB93_4
+; RV32I-NEXT:    lh a1, 10(sp)
+; RV32I-NEXT:    bnez a0, .LBB93_4
 ; RV32I-NEXT:  .LBB93_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    slli a1, a0, 16
-; RV32I-NEXT:    srai a1, a1, 16
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    blt s0, a1, .LBB93_1
+; RV32I-NEXT:    slli a0, a1, 16
+; RV32I-NEXT:    srai a0, a0, 16
+; RV32I-NEXT:    mv a2, a1
+; RV32I-NEXT:    blt s0, a0, .LBB93_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB93_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:    j .LBB93_1
 ; RV32I-NEXT:  .LBB93_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -7674,33 +7674,33 @@ define i16 @atomicrmw_max_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_max_i16_acq_rel:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    addi a3, zero, 16
-; RV32IA-NEXT:    sub a6, a3, a2
-; RV32IA-NEXT:    lui a4, 16
-; RV32IA-NEXT:    addi a4, a4, -1
-; RV32IA-NEXT:    sll a7, a4, a2
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a7, a3, a0
 ; RV32IA-NEXT:    slli a1, a1, 16
 ; RV32IA-NEXT:    srai a1, a1, 16
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    addi a4, zero, 16
+; RV32IA-NEXT:    sub a4, a4, a0
 ; RV32IA-NEXT:  .LBB93_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a5, (a0)
-; RV32IA-NEXT:    and a4, a5, a7
-; RV32IA-NEXT:    mv a3, a5
-; RV32IA-NEXT:    sll a4, a4, a6
-; RV32IA-NEXT:    sra a4, a4, a6
-; RV32IA-NEXT:    bge a4, a1, .LBB93_3
+; RV32IA-NEXT:    lr.w.aq a5, (a6)
+; RV32IA-NEXT:    and a3, a5, a7
+; RV32IA-NEXT:    mv a2, a5
+; RV32IA-NEXT:    sll a3, a3, a4
+; RV32IA-NEXT:    sra a3, a3, a4
+; RV32IA-NEXT:    bge a3, a1, .LBB93_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB93_1 Depth=1
-; RV32IA-NEXT:    xor a3, a5, a1
-; RV32IA-NEXT:    and a3, a3, a7
-; RV32IA-NEXT:    xor a3, a5, a3
+; RV32IA-NEXT:    xor a2, a5, a1
+; RV32IA-NEXT:    and a2, a2, a7
+; RV32IA-NEXT:    xor a2, a5, a2
 ; RV32IA-NEXT:  .LBB93_3: # in Loop: Header=BB93_1 Depth=1
-; RV32IA-NEXT:    sc.w.rl a3, a3, (a0)
-; RV32IA-NEXT:    bnez a3, .LBB93_1
+; RV32IA-NEXT:    sc.w.rl a2, a2, (a6)
+; RV32IA-NEXT:    bnez a2, .LBB93_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a5, a2
+; RV32IA-NEXT:    srl a0, a5, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_max_i16_acq_rel:
@@ -7713,33 +7713,33 @@ define i16 @atomicrmw_max_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    sd s3, 8(sp)
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lhu a0, 0(a0)
-; RV64I-NEXT:    slli a1, a1, 48
-; RV64I-NEXT:    srai s0, a1, 48
+; RV64I-NEXT:    lhu a1, 0(a0)
+; RV64I-NEXT:    slli a0, s2, 48
+; RV64I-NEXT:    srai s0, a0, 48
 ; RV64I-NEXT:    addi s3, sp, 6
 ; RV64I-NEXT:    j .LBB93_2
 ; RV64I-NEXT:  .LBB93_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB93_2 Depth=1
-; RV64I-NEXT:    sh a0, 6(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sh a1, 6(sp)
 ; RV64I-NEXT:    addi a3, zero, 4
 ; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    mv a0, s1
+; RV64I-NEXT:    mv a1, s3
 ; RV64I-NEXT:    call __atomic_compare_exchange_2
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lh a0, 6(sp)
-; RV64I-NEXT:    bnez a1, .LBB93_4
+; RV64I-NEXT:    lh a1, 6(sp)
+; RV64I-NEXT:    bnez a0, .LBB93_4
 ; RV64I-NEXT:  .LBB93_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    slli a1, a0, 48
-; RV64I-NEXT:    srai a1, a1, 48
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    blt s0, a1, .LBB93_1
+; RV64I-NEXT:    slli a0, a1, 48
+; RV64I-NEXT:    srai a0, a0, 48
+; RV64I-NEXT:    mv a2, a1
+; RV64I-NEXT:    blt s0, a0, .LBB93_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB93_2 Depth=1
 ; RV64I-NEXT:    mv a2, s2
 ; RV64I-NEXT:    j .LBB93_1
 ; RV64I-NEXT:  .LBB93_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -7750,33 +7750,33 @@ define i16 @atomicrmw_max_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_max_i16_acq_rel:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    addi a3, zero, 48
-; RV64IA-NEXT:    sub a6, a3, a2
-; RV64IA-NEXT:    lui a4, 16
-; RV64IA-NEXT:    addiw a4, a4, -1
-; RV64IA-NEXT:    sllw a7, a4, a2
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a7, a3, a0
 ; RV64IA-NEXT:    slli a1, a1, 48
 ; RV64IA-NEXT:    srai a1, a1, 48
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    addi a4, zero, 48
+; RV64IA-NEXT:    sub a4, a4, a0
 ; RV64IA-NEXT:  .LBB93_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a5, (a0)
-; RV64IA-NEXT:    and a4, a5, a7
-; RV64IA-NEXT:    mv a3, a5
-; RV64IA-NEXT:    sll a4, a4, a6
-; RV64IA-NEXT:    sra a4, a4, a6
-; RV64IA-NEXT:    bge a4, a1, .LBB93_3
+; RV64IA-NEXT:    lr.w.aq a5, (a6)
+; RV64IA-NEXT:    and a3, a5, a7
+; RV64IA-NEXT:    mv a2, a5
+; RV64IA-NEXT:    sll a3, a3, a4
+; RV64IA-NEXT:    sra a3, a3, a4
+; RV64IA-NEXT:    bge a3, a1, .LBB93_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB93_1 Depth=1
-; RV64IA-NEXT:    xor a3, a5, a1
-; RV64IA-NEXT:    and a3, a3, a7
-; RV64IA-NEXT:    xor a3, a5, a3
+; RV64IA-NEXT:    xor a2, a5, a1
+; RV64IA-NEXT:    and a2, a2, a7
+; RV64IA-NEXT:    xor a2, a5, a2
 ; RV64IA-NEXT:  .LBB93_3: # in Loop: Header=BB93_1 Depth=1
-; RV64IA-NEXT:    sc.w.rl a3, a3, (a0)
-; RV64IA-NEXT:    bnez a3, .LBB93_1
+; RV64IA-NEXT:    sc.w.rl a2, a2, (a6)
+; RV64IA-NEXT:    bnez a2, .LBB93_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a5, a2
+; RV64IA-NEXT:    srlw a0, a5, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw max i16* %a, i16 %b acq_rel
   ret i16 %1
@@ -7793,33 +7793,33 @@ define i16 @atomicrmw_max_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    sw s3, 12(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lhu a0, 0(a0)
-; RV32I-NEXT:    slli a1, a1, 16
-; RV32I-NEXT:    srai s0, a1, 16
+; RV32I-NEXT:    lhu a1, 0(a0)
+; RV32I-NEXT:    slli a0, s2, 16
+; RV32I-NEXT:    srai s0, a0, 16
 ; RV32I-NEXT:    addi s3, sp, 10
 ; RV32I-NEXT:    j .LBB94_2
 ; RV32I-NEXT:  .LBB94_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB94_2 Depth=1
-; RV32I-NEXT:    sh a0, 10(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sh a1, 10(sp)
 ; RV32I-NEXT:    addi a3, zero, 5
 ; RV32I-NEXT:    addi a4, zero, 5
+; RV32I-NEXT:    mv a0, s1
+; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    call __atomic_compare_exchange_2
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lh a0, 10(sp)
-; RV32I-NEXT:    bnez a1, .LBB94_4
+; RV32I-NEXT:    lh a1, 10(sp)
+; RV32I-NEXT:    bnez a0, .LBB94_4
 ; RV32I-NEXT:  .LBB94_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    slli a1, a0, 16
-; RV32I-NEXT:    srai a1, a1, 16
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    blt s0, a1, .LBB94_1
+; RV32I-NEXT:    slli a0, a1, 16
+; RV32I-NEXT:    srai a0, a0, 16
+; RV32I-NEXT:    mv a2, a1
+; RV32I-NEXT:    blt s0, a0, .LBB94_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB94_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:    j .LBB94_1
 ; RV32I-NEXT:  .LBB94_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -7830,33 +7830,33 @@ define i16 @atomicrmw_max_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_max_i16_seq_cst:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    addi a3, zero, 16
-; RV32IA-NEXT:    sub a6, a3, a2
-; RV32IA-NEXT:    lui a4, 16
-; RV32IA-NEXT:    addi a4, a4, -1
-; RV32IA-NEXT:    sll a7, a4, a2
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a7, a3, a0
 ; RV32IA-NEXT:    slli a1, a1, 16
 ; RV32IA-NEXT:    srai a1, a1, 16
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    addi a4, zero, 16
+; RV32IA-NEXT:    sub a4, a4, a0
 ; RV32IA-NEXT:  .LBB94_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aqrl a5, (a0)
-; RV32IA-NEXT:    and a4, a5, a7
-; RV32IA-NEXT:    mv a3, a5
-; RV32IA-NEXT:    sll a4, a4, a6
-; RV32IA-NEXT:    sra a4, a4, a6
-; RV32IA-NEXT:    bge a4, a1, .LBB94_3
+; RV32IA-NEXT:    lr.w.aqrl a5, (a6)
+; RV32IA-NEXT:    and a3, a5, a7
+; RV32IA-NEXT:    mv a2, a5
+; RV32IA-NEXT:    sll a3, a3, a4
+; RV32IA-NEXT:    sra a3, a3, a4
+; RV32IA-NEXT:    bge a3, a1, .LBB94_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB94_1 Depth=1
-; RV32IA-NEXT:    xor a3, a5, a1
-; RV32IA-NEXT:    and a3, a3, a7
-; RV32IA-NEXT:    xor a3, a5, a3
+; RV32IA-NEXT:    xor a2, a5, a1
+; RV32IA-NEXT:    and a2, a2, a7
+; RV32IA-NEXT:    xor a2, a5, a2
 ; RV32IA-NEXT:  .LBB94_3: # in Loop: Header=BB94_1 Depth=1
-; RV32IA-NEXT:    sc.w.aqrl a3, a3, (a0)
-; RV32IA-NEXT:    bnez a3, .LBB94_1
+; RV32IA-NEXT:    sc.w.aqrl a2, a2, (a6)
+; RV32IA-NEXT:    bnez a2, .LBB94_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a5, a2
+; RV32IA-NEXT:    srl a0, a5, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_max_i16_seq_cst:
@@ -7869,33 +7869,33 @@ define i16 @atomicrmw_max_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    sd s3, 8(sp)
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lhu a0, 0(a0)
-; RV64I-NEXT:    slli a1, a1, 48
-; RV64I-NEXT:    srai s0, a1, 48
+; RV64I-NEXT:    lhu a1, 0(a0)
+; RV64I-NEXT:    slli a0, s2, 48
+; RV64I-NEXT:    srai s0, a0, 48
 ; RV64I-NEXT:    addi s3, sp, 6
 ; RV64I-NEXT:    j .LBB94_2
 ; RV64I-NEXT:  .LBB94_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB94_2 Depth=1
-; RV64I-NEXT:    sh a0, 6(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sh a1, 6(sp)
 ; RV64I-NEXT:    addi a3, zero, 5
 ; RV64I-NEXT:    addi a4, zero, 5
+; RV64I-NEXT:    mv a0, s1
+; RV64I-NEXT:    mv a1, s3
 ; RV64I-NEXT:    call __atomic_compare_exchange_2
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lh a0, 6(sp)
-; RV64I-NEXT:    bnez a1, .LBB94_4
+; RV64I-NEXT:    lh a1, 6(sp)
+; RV64I-NEXT:    bnez a0, .LBB94_4
 ; RV64I-NEXT:  .LBB94_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    slli a1, a0, 48
-; RV64I-NEXT:    srai a1, a1, 48
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    blt s0, a1, .LBB94_1
+; RV64I-NEXT:    slli a0, a1, 48
+; RV64I-NEXT:    srai a0, a0, 48
+; RV64I-NEXT:    mv a2, a1
+; RV64I-NEXT:    blt s0, a0, .LBB94_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB94_2 Depth=1
 ; RV64I-NEXT:    mv a2, s2
 ; RV64I-NEXT:    j .LBB94_1
 ; RV64I-NEXT:  .LBB94_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -7906,33 +7906,33 @@ define i16 @atomicrmw_max_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_max_i16_seq_cst:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    addi a3, zero, 48
-; RV64IA-NEXT:    sub a6, a3, a2
-; RV64IA-NEXT:    lui a4, 16
-; RV64IA-NEXT:    addiw a4, a4, -1
-; RV64IA-NEXT:    sllw a7, a4, a2
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a7, a3, a0
 ; RV64IA-NEXT:    slli a1, a1, 48
 ; RV64IA-NEXT:    srai a1, a1, 48
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    addi a4, zero, 48
+; RV64IA-NEXT:    sub a4, a4, a0
 ; RV64IA-NEXT:  .LBB94_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aqrl a5, (a0)
-; RV64IA-NEXT:    and a4, a5, a7
-; RV64IA-NEXT:    mv a3, a5
-; RV64IA-NEXT:    sll a4, a4, a6
-; RV64IA-NEXT:    sra a4, a4, a6
-; RV64IA-NEXT:    bge a4, a1, .LBB94_3
+; RV64IA-NEXT:    lr.w.aqrl a5, (a6)
+; RV64IA-NEXT:    and a3, a5, a7
+; RV64IA-NEXT:    mv a2, a5
+; RV64IA-NEXT:    sll a3, a3, a4
+; RV64IA-NEXT:    sra a3, a3, a4
+; RV64IA-NEXT:    bge a3, a1, .LBB94_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB94_1 Depth=1
-; RV64IA-NEXT:    xor a3, a5, a1
-; RV64IA-NEXT:    and a3, a3, a7
-; RV64IA-NEXT:    xor a3, a5, a3
+; RV64IA-NEXT:    xor a2, a5, a1
+; RV64IA-NEXT:    and a2, a2, a7
+; RV64IA-NEXT:    xor a2, a5, a2
 ; RV64IA-NEXT:  .LBB94_3: # in Loop: Header=BB94_1 Depth=1
-; RV64IA-NEXT:    sc.w.aqrl a3, a3, (a0)
-; RV64IA-NEXT:    bnez a3, .LBB94_1
+; RV64IA-NEXT:    sc.w.aqrl a2, a2, (a6)
+; RV64IA-NEXT:    bnez a2, .LBB94_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a5, a2
+; RV64IA-NEXT:    srlw a0, a5, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw max i16* %a, i16 %b seq_cst
   ret i16 %1
@@ -7949,33 +7949,33 @@ define i16 @atomicrmw_min_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    sw s3, 12(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lhu a0, 0(a0)
-; RV32I-NEXT:    slli a1, a1, 16
-; RV32I-NEXT:    srai s0, a1, 16
+; RV32I-NEXT:    lhu a1, 0(a0)
+; RV32I-NEXT:    slli a0, s2, 16
+; RV32I-NEXT:    srai s0, a0, 16
 ; RV32I-NEXT:    addi s3, sp, 10
 ; RV32I-NEXT:    j .LBB95_2
 ; RV32I-NEXT:  .LBB95_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB95_2 Depth=1
-; RV32I-NEXT:    sh a0, 10(sp)
+; RV32I-NEXT:    sh a1, 10(sp)
 ; RV32I-NEXT:    mv a0, s1
 ; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    mv a3, zero
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_2
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lh a0, 10(sp)
-; RV32I-NEXT:    bnez a1, .LBB95_4
+; RV32I-NEXT:    lh a1, 10(sp)
+; RV32I-NEXT:    bnez a0, .LBB95_4
 ; RV32I-NEXT:  .LBB95_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    slli a1, a0, 16
-; RV32I-NEXT:    srai a1, a1, 16
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bge s0, a1, .LBB95_1
+; RV32I-NEXT:    slli a0, a1, 16
+; RV32I-NEXT:    srai a0, a0, 16
+; RV32I-NEXT:    mv a2, a1
+; RV32I-NEXT:    bge s0, a0, .LBB95_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB95_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:    j .LBB95_1
 ; RV32I-NEXT:  .LBB95_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -7986,33 +7986,33 @@ define i16 @atomicrmw_min_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_min_i16_monotonic:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    addi a3, zero, 16
-; RV32IA-NEXT:    sub a6, a3, a2
-; RV32IA-NEXT:    lui a4, 16
-; RV32IA-NEXT:    addi a4, a4, -1
-; RV32IA-NEXT:    sll a7, a4, a2
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a7, a3, a0
 ; RV32IA-NEXT:    slli a1, a1, 16
 ; RV32IA-NEXT:    srai a1, a1, 16
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    addi a4, zero, 16
+; RV32IA-NEXT:    sub a4, a4, a0
 ; RV32IA-NEXT:  .LBB95_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a5, (a0)
-; RV32IA-NEXT:    and a4, a5, a7
-; RV32IA-NEXT:    mv a3, a5
-; RV32IA-NEXT:    sll a4, a4, a6
-; RV32IA-NEXT:    sra a4, a4, a6
-; RV32IA-NEXT:    bge a1, a4, .LBB95_3
+; RV32IA-NEXT:    lr.w a5, (a6)
+; RV32IA-NEXT:    and a3, a5, a7
+; RV32IA-NEXT:    mv a2, a5
+; RV32IA-NEXT:    sll a3, a3, a4
+; RV32IA-NEXT:    sra a3, a3, a4
+; RV32IA-NEXT:    bge a1, a3, .LBB95_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB95_1 Depth=1
-; RV32IA-NEXT:    xor a3, a5, a1
-; RV32IA-NEXT:    and a3, a3, a7
-; RV32IA-NEXT:    xor a3, a5, a3
+; RV32IA-NEXT:    xor a2, a5, a1
+; RV32IA-NEXT:    and a2, a2, a7
+; RV32IA-NEXT:    xor a2, a5, a2
 ; RV32IA-NEXT:  .LBB95_3: # in Loop: Header=BB95_1 Depth=1
-; RV32IA-NEXT:    sc.w a3, a3, (a0)
-; RV32IA-NEXT:    bnez a3, .LBB95_1
+; RV32IA-NEXT:    sc.w a2, a2, (a6)
+; RV32IA-NEXT:    bnez a2, .LBB95_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a5, a2
+; RV32IA-NEXT:    srl a0, a5, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_min_i16_monotonic:
@@ -8025,33 +8025,33 @@ define i16 @atomicrmw_min_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    sd s3, 8(sp)
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lhu a0, 0(a0)
-; RV64I-NEXT:    slli a1, a1, 48
-; RV64I-NEXT:    srai s0, a1, 48
+; RV64I-NEXT:    lhu a1, 0(a0)
+; RV64I-NEXT:    slli a0, s2, 48
+; RV64I-NEXT:    srai s0, a0, 48
 ; RV64I-NEXT:    addi s3, sp, 6
 ; RV64I-NEXT:    j .LBB95_2
 ; RV64I-NEXT:  .LBB95_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB95_2 Depth=1
-; RV64I-NEXT:    sh a0, 6(sp)
+; RV64I-NEXT:    sh a1, 6(sp)
 ; RV64I-NEXT:    mv a0, s1
 ; RV64I-NEXT:    mv a1, s3
 ; RV64I-NEXT:    mv a3, zero
 ; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    call __atomic_compare_exchange_2
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lh a0, 6(sp)
-; RV64I-NEXT:    bnez a1, .LBB95_4
+; RV64I-NEXT:    lh a1, 6(sp)
+; RV64I-NEXT:    bnez a0, .LBB95_4
 ; RV64I-NEXT:  .LBB95_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    slli a1, a0, 48
-; RV64I-NEXT:    srai a1, a1, 48
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bge s0, a1, .LBB95_1
+; RV64I-NEXT:    slli a0, a1, 48
+; RV64I-NEXT:    srai a0, a0, 48
+; RV64I-NEXT:    mv a2, a1
+; RV64I-NEXT:    bge s0, a0, .LBB95_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB95_2 Depth=1
 ; RV64I-NEXT:    mv a2, s2
 ; RV64I-NEXT:    j .LBB95_1
 ; RV64I-NEXT:  .LBB95_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -8062,33 +8062,33 @@ define i16 @atomicrmw_min_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_min_i16_monotonic:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    addi a3, zero, 48
-; RV64IA-NEXT:    sub a6, a3, a2
-; RV64IA-NEXT:    lui a4, 16
-; RV64IA-NEXT:    addiw a4, a4, -1
-; RV64IA-NEXT:    sllw a7, a4, a2
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a7, a3, a0
 ; RV64IA-NEXT:    slli a1, a1, 48
 ; RV64IA-NEXT:    srai a1, a1, 48
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    addi a4, zero, 48
+; RV64IA-NEXT:    sub a4, a4, a0
 ; RV64IA-NEXT:  .LBB95_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a5, (a0)
-; RV64IA-NEXT:    and a4, a5, a7
-; RV64IA-NEXT:    mv a3, a5
-; RV64IA-NEXT:    sll a4, a4, a6
-; RV64IA-NEXT:    sra a4, a4, a6
-; RV64IA-NEXT:    bge a1, a4, .LBB95_3
+; RV64IA-NEXT:    lr.w a5, (a6)
+; RV64IA-NEXT:    and a3, a5, a7
+; RV64IA-NEXT:    mv a2, a5
+; RV64IA-NEXT:    sll a3, a3, a4
+; RV64IA-NEXT:    sra a3, a3, a4
+; RV64IA-NEXT:    bge a1, a3, .LBB95_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB95_1 Depth=1
-; RV64IA-NEXT:    xor a3, a5, a1
-; RV64IA-NEXT:    and a3, a3, a7
-; RV64IA-NEXT:    xor a3, a5, a3
+; RV64IA-NEXT:    xor a2, a5, a1
+; RV64IA-NEXT:    and a2, a2, a7
+; RV64IA-NEXT:    xor a2, a5, a2
 ; RV64IA-NEXT:  .LBB95_3: # in Loop: Header=BB95_1 Depth=1
-; RV64IA-NEXT:    sc.w a3, a3, (a0)
-; RV64IA-NEXT:    bnez a3, .LBB95_1
+; RV64IA-NEXT:    sc.w a2, a2, (a6)
+; RV64IA-NEXT:    bnez a2, .LBB95_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a5, a2
+; RV64IA-NEXT:    srlw a0, a5, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw min i16* %a, i16 %b monotonic
   ret i16 %1
@@ -8105,33 +8105,33 @@ define i16 @atomicrmw_min_i16_acquire(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    sw s3, 12(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lhu a0, 0(a0)
-; RV32I-NEXT:    slli a1, a1, 16
-; RV32I-NEXT:    srai s0, a1, 16
+; RV32I-NEXT:    lhu a1, 0(a0)
+; RV32I-NEXT:    slli a0, s2, 16
+; RV32I-NEXT:    srai s0, a0, 16
 ; RV32I-NEXT:    addi s3, sp, 10
 ; RV32I-NEXT:    j .LBB96_2
 ; RV32I-NEXT:  .LBB96_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB96_2 Depth=1
-; RV32I-NEXT:    sh a0, 10(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sh a1, 10(sp)
 ; RV32I-NEXT:    addi a3, zero, 2
 ; RV32I-NEXT:    addi a4, zero, 2
+; RV32I-NEXT:    mv a0, s1
+; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    call __atomic_compare_exchange_2
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lh a0, 10(sp)
-; RV32I-NEXT:    bnez a1, .LBB96_4
+; RV32I-NEXT:    lh a1, 10(sp)
+; RV32I-NEXT:    bnez a0, .LBB96_4
 ; RV32I-NEXT:  .LBB96_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    slli a1, a0, 16
-; RV32I-NEXT:    srai a1, a1, 16
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bge s0, a1, .LBB96_1
+; RV32I-NEXT:    slli a0, a1, 16
+; RV32I-NEXT:    srai a0, a0, 16
+; RV32I-NEXT:    mv a2, a1
+; RV32I-NEXT:    bge s0, a0, .LBB96_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB96_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:    j .LBB96_1
 ; RV32I-NEXT:  .LBB96_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -8142,33 +8142,33 @@ define i16 @atomicrmw_min_i16_acquire(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_min_i16_acquire:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    addi a3, zero, 16
-; RV32IA-NEXT:    sub a6, a3, a2
-; RV32IA-NEXT:    lui a4, 16
-; RV32IA-NEXT:    addi a4, a4, -1
-; RV32IA-NEXT:    sll a7, a4, a2
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a7, a3, a0
 ; RV32IA-NEXT:    slli a1, a1, 16
 ; RV32IA-NEXT:    srai a1, a1, 16
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    addi a4, zero, 16
+; RV32IA-NEXT:    sub a4, a4, a0
 ; RV32IA-NEXT:  .LBB96_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a5, (a0)
-; RV32IA-NEXT:    and a4, a5, a7
-; RV32IA-NEXT:    mv a3, a5
-; RV32IA-NEXT:    sll a4, a4, a6
-; RV32IA-NEXT:    sra a4, a4, a6
-; RV32IA-NEXT:    bge a1, a4, .LBB96_3
+; RV32IA-NEXT:    lr.w.aq a5, (a6)
+; RV32IA-NEXT:    and a3, a5, a7
+; RV32IA-NEXT:    mv a2, a5
+; RV32IA-NEXT:    sll a3, a3, a4
+; RV32IA-NEXT:    sra a3, a3, a4
+; RV32IA-NEXT:    bge a1, a3, .LBB96_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB96_1 Depth=1
-; RV32IA-NEXT:    xor a3, a5, a1
-; RV32IA-NEXT:    and a3, a3, a7
-; RV32IA-NEXT:    xor a3, a5, a3
+; RV32IA-NEXT:    xor a2, a5, a1
+; RV32IA-NEXT:    and a2, a2, a7
+; RV32IA-NEXT:    xor a2, a5, a2
 ; RV32IA-NEXT:  .LBB96_3: # in Loop: Header=BB96_1 Depth=1
-; RV32IA-NEXT:    sc.w a3, a3, (a0)
-; RV32IA-NEXT:    bnez a3, .LBB96_1
+; RV32IA-NEXT:    sc.w a2, a2, (a6)
+; RV32IA-NEXT:    bnez a2, .LBB96_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a5, a2
+; RV32IA-NEXT:    srl a0, a5, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_min_i16_acquire:
@@ -8181,33 +8181,33 @@ define i16 @atomicrmw_min_i16_acquire(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    sd s3, 8(sp)
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lhu a0, 0(a0)
-; RV64I-NEXT:    slli a1, a1, 48
-; RV64I-NEXT:    srai s0, a1, 48
+; RV64I-NEXT:    lhu a1, 0(a0)
+; RV64I-NEXT:    slli a0, s2, 48
+; RV64I-NEXT:    srai s0, a0, 48
 ; RV64I-NEXT:    addi s3, sp, 6
 ; RV64I-NEXT:    j .LBB96_2
 ; RV64I-NEXT:  .LBB96_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB96_2 Depth=1
-; RV64I-NEXT:    sh a0, 6(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sh a1, 6(sp)
 ; RV64I-NEXT:    addi a3, zero, 2
 ; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    mv a0, s1
+; RV64I-NEXT:    mv a1, s3
 ; RV64I-NEXT:    call __atomic_compare_exchange_2
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lh a0, 6(sp)
-; RV64I-NEXT:    bnez a1, .LBB96_4
+; RV64I-NEXT:    lh a1, 6(sp)
+; RV64I-NEXT:    bnez a0, .LBB96_4
 ; RV64I-NEXT:  .LBB96_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    slli a1, a0, 48
-; RV64I-NEXT:    srai a1, a1, 48
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bge s0, a1, .LBB96_1
+; RV64I-NEXT:    slli a0, a1, 48
+; RV64I-NEXT:    srai a0, a0, 48
+; RV64I-NEXT:    mv a2, a1
+; RV64I-NEXT:    bge s0, a0, .LBB96_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB96_2 Depth=1
 ; RV64I-NEXT:    mv a2, s2
 ; RV64I-NEXT:    j .LBB96_1
 ; RV64I-NEXT:  .LBB96_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -8218,33 +8218,33 @@ define i16 @atomicrmw_min_i16_acquire(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_min_i16_acquire:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    addi a3, zero, 48
-; RV64IA-NEXT:    sub a6, a3, a2
-; RV64IA-NEXT:    lui a4, 16
-; RV64IA-NEXT:    addiw a4, a4, -1
-; RV64IA-NEXT:    sllw a7, a4, a2
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a7, a3, a0
 ; RV64IA-NEXT:    slli a1, a1, 48
 ; RV64IA-NEXT:    srai a1, a1, 48
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    addi a4, zero, 48
+; RV64IA-NEXT:    sub a4, a4, a0
 ; RV64IA-NEXT:  .LBB96_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a5, (a0)
-; RV64IA-NEXT:    and a4, a5, a7
-; RV64IA-NEXT:    mv a3, a5
-; RV64IA-NEXT:    sll a4, a4, a6
-; RV64IA-NEXT:    sra a4, a4, a6
-; RV64IA-NEXT:    bge a1, a4, .LBB96_3
+; RV64IA-NEXT:    lr.w.aq a5, (a6)
+; RV64IA-NEXT:    and a3, a5, a7
+; RV64IA-NEXT:    mv a2, a5
+; RV64IA-NEXT:    sll a3, a3, a4
+; RV64IA-NEXT:    sra a3, a3, a4
+; RV64IA-NEXT:    bge a1, a3, .LBB96_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB96_1 Depth=1
-; RV64IA-NEXT:    xor a3, a5, a1
-; RV64IA-NEXT:    and a3, a3, a7
-; RV64IA-NEXT:    xor a3, a5, a3
+; RV64IA-NEXT:    xor a2, a5, a1
+; RV64IA-NEXT:    and a2, a2, a7
+; RV64IA-NEXT:    xor a2, a5, a2
 ; RV64IA-NEXT:  .LBB96_3: # in Loop: Header=BB96_1 Depth=1
-; RV64IA-NEXT:    sc.w a3, a3, (a0)
-; RV64IA-NEXT:    bnez a3, .LBB96_1
+; RV64IA-NEXT:    sc.w a2, a2, (a6)
+; RV64IA-NEXT:    bnez a2, .LBB96_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a5, a2
+; RV64IA-NEXT:    srlw a0, a5, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw min i16* %a, i16 %b acquire
   ret i16 %1
@@ -8261,33 +8261,33 @@ define i16 @atomicrmw_min_i16_release(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    sw s3, 12(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lhu a0, 0(a0)
-; RV32I-NEXT:    slli a1, a1, 16
-; RV32I-NEXT:    srai s0, a1, 16
+; RV32I-NEXT:    lhu a1, 0(a0)
+; RV32I-NEXT:    slli a0, s2, 16
+; RV32I-NEXT:    srai s0, a0, 16
 ; RV32I-NEXT:    addi s3, sp, 10
 ; RV32I-NEXT:    j .LBB97_2
 ; RV32I-NEXT:  .LBB97_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB97_2 Depth=1
-; RV32I-NEXT:    sh a0, 10(sp)
+; RV32I-NEXT:    sh a1, 10(sp)
+; RV32I-NEXT:    addi a3, zero, 3
 ; RV32I-NEXT:    mv a0, s1
 ; RV32I-NEXT:    mv a1, s3
-; RV32I-NEXT:    addi a3, zero, 3
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_2
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lh a0, 10(sp)
-; RV32I-NEXT:    bnez a1, .LBB97_4
+; RV32I-NEXT:    lh a1, 10(sp)
+; RV32I-NEXT:    bnez a0, .LBB97_4
 ; RV32I-NEXT:  .LBB97_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    slli a1, a0, 16
-; RV32I-NEXT:    srai a1, a1, 16
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bge s0, a1, .LBB97_1
+; RV32I-NEXT:    slli a0, a1, 16
+; RV32I-NEXT:    srai a0, a0, 16
+; RV32I-NEXT:    mv a2, a1
+; RV32I-NEXT:    bge s0, a0, .LBB97_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB97_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:    j .LBB97_1
 ; RV32I-NEXT:  .LBB97_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -8298,33 +8298,33 @@ define i16 @atomicrmw_min_i16_release(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_min_i16_release:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    addi a3, zero, 16
-; RV32IA-NEXT:    sub a6, a3, a2
-; RV32IA-NEXT:    lui a4, 16
-; RV32IA-NEXT:    addi a4, a4, -1
-; RV32IA-NEXT:    sll a7, a4, a2
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a7, a3, a0
 ; RV32IA-NEXT:    slli a1, a1, 16
 ; RV32IA-NEXT:    srai a1, a1, 16
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    addi a4, zero, 16
+; RV32IA-NEXT:    sub a4, a4, a0
 ; RV32IA-NEXT:  .LBB97_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a5, (a0)
-; RV32IA-NEXT:    and a4, a5, a7
-; RV32IA-NEXT:    mv a3, a5
-; RV32IA-NEXT:    sll a4, a4, a6
-; RV32IA-NEXT:    sra a4, a4, a6
-; RV32IA-NEXT:    bge a1, a4, .LBB97_3
+; RV32IA-NEXT:    lr.w a5, (a6)
+; RV32IA-NEXT:    and a3, a5, a7
+; RV32IA-NEXT:    mv a2, a5
+; RV32IA-NEXT:    sll a3, a3, a4
+; RV32IA-NEXT:    sra a3, a3, a4
+; RV32IA-NEXT:    bge a1, a3, .LBB97_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB97_1 Depth=1
-; RV32IA-NEXT:    xor a3, a5, a1
-; RV32IA-NEXT:    and a3, a3, a7
-; RV32IA-NEXT:    xor a3, a5, a3
+; RV32IA-NEXT:    xor a2, a5, a1
+; RV32IA-NEXT:    and a2, a2, a7
+; RV32IA-NEXT:    xor a2, a5, a2
 ; RV32IA-NEXT:  .LBB97_3: # in Loop: Header=BB97_1 Depth=1
-; RV32IA-NEXT:    sc.w.rl a3, a3, (a0)
-; RV32IA-NEXT:    bnez a3, .LBB97_1
+; RV32IA-NEXT:    sc.w.rl a2, a2, (a6)
+; RV32IA-NEXT:    bnez a2, .LBB97_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a5, a2
+; RV32IA-NEXT:    srl a0, a5, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_min_i16_release:
@@ -8337,33 +8337,33 @@ define i16 @atomicrmw_min_i16_release(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    sd s3, 8(sp)
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lhu a0, 0(a0)
-; RV64I-NEXT:    slli a1, a1, 48
-; RV64I-NEXT:    srai s0, a1, 48
+; RV64I-NEXT:    lhu a1, 0(a0)
+; RV64I-NEXT:    slli a0, s2, 48
+; RV64I-NEXT:    srai s0, a0, 48
 ; RV64I-NEXT:    addi s3, sp, 6
 ; RV64I-NEXT:    j .LBB97_2
 ; RV64I-NEXT:  .LBB97_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB97_2 Depth=1
-; RV64I-NEXT:    sh a0, 6(sp)
+; RV64I-NEXT:    sh a1, 6(sp)
+; RV64I-NEXT:    addi a3, zero, 3
 ; RV64I-NEXT:    mv a0, s1
 ; RV64I-NEXT:    mv a1, s3
-; RV64I-NEXT:    addi a3, zero, 3
 ; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    call __atomic_compare_exchange_2
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lh a0, 6(sp)
-; RV64I-NEXT:    bnez a1, .LBB97_4
+; RV64I-NEXT:    lh a1, 6(sp)
+; RV64I-NEXT:    bnez a0, .LBB97_4
 ; RV64I-NEXT:  .LBB97_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    slli a1, a0, 48
-; RV64I-NEXT:    srai a1, a1, 48
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bge s0, a1, .LBB97_1
+; RV64I-NEXT:    slli a0, a1, 48
+; RV64I-NEXT:    srai a0, a0, 48
+; RV64I-NEXT:    mv a2, a1
+; RV64I-NEXT:    bge s0, a0, .LBB97_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB97_2 Depth=1
 ; RV64I-NEXT:    mv a2, s2
 ; RV64I-NEXT:    j .LBB97_1
 ; RV64I-NEXT:  .LBB97_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -8374,33 +8374,33 @@ define i16 @atomicrmw_min_i16_release(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_min_i16_release:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    addi a3, zero, 48
-; RV64IA-NEXT:    sub a6, a3, a2
-; RV64IA-NEXT:    lui a4, 16
-; RV64IA-NEXT:    addiw a4, a4, -1
-; RV64IA-NEXT:    sllw a7, a4, a2
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a7, a3, a0
 ; RV64IA-NEXT:    slli a1, a1, 48
 ; RV64IA-NEXT:    srai a1, a1, 48
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    addi a4, zero, 48
+; RV64IA-NEXT:    sub a4, a4, a0
 ; RV64IA-NEXT:  .LBB97_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a5, (a0)
-; RV64IA-NEXT:    and a4, a5, a7
-; RV64IA-NEXT:    mv a3, a5
-; RV64IA-NEXT:    sll a4, a4, a6
-; RV64IA-NEXT:    sra a4, a4, a6
-; RV64IA-NEXT:    bge a1, a4, .LBB97_3
+; RV64IA-NEXT:    lr.w a5, (a6)
+; RV64IA-NEXT:    and a3, a5, a7
+; RV64IA-NEXT:    mv a2, a5
+; RV64IA-NEXT:    sll a3, a3, a4
+; RV64IA-NEXT:    sra a3, a3, a4
+; RV64IA-NEXT:    bge a1, a3, .LBB97_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB97_1 Depth=1
-; RV64IA-NEXT:    xor a3, a5, a1
-; RV64IA-NEXT:    and a3, a3, a7
-; RV64IA-NEXT:    xor a3, a5, a3
+; RV64IA-NEXT:    xor a2, a5, a1
+; RV64IA-NEXT:    and a2, a2, a7
+; RV64IA-NEXT:    xor a2, a5, a2
 ; RV64IA-NEXT:  .LBB97_3: # in Loop: Header=BB97_1 Depth=1
-; RV64IA-NEXT:    sc.w.rl a3, a3, (a0)
-; RV64IA-NEXT:    bnez a3, .LBB97_1
+; RV64IA-NEXT:    sc.w.rl a2, a2, (a6)
+; RV64IA-NEXT:    bnez a2, .LBB97_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a5, a2
+; RV64IA-NEXT:    srlw a0, a5, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw min i16* %a, i16 %b release
   ret i16 %1
@@ -8417,33 +8417,33 @@ define i16 @atomicrmw_min_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    sw s3, 12(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lhu a0, 0(a0)
-; RV32I-NEXT:    slli a1, a1, 16
-; RV32I-NEXT:    srai s0, a1, 16
+; RV32I-NEXT:    lhu a1, 0(a0)
+; RV32I-NEXT:    slli a0, s2, 16
+; RV32I-NEXT:    srai s0, a0, 16
 ; RV32I-NEXT:    addi s3, sp, 10
 ; RV32I-NEXT:    j .LBB98_2
 ; RV32I-NEXT:  .LBB98_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB98_2 Depth=1
-; RV32I-NEXT:    sh a0, 10(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sh a1, 10(sp)
 ; RV32I-NEXT:    addi a3, zero, 4
 ; RV32I-NEXT:    addi a4, zero, 2
+; RV32I-NEXT:    mv a0, s1
+; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    call __atomic_compare_exchange_2
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lh a0, 10(sp)
-; RV32I-NEXT:    bnez a1, .LBB98_4
+; RV32I-NEXT:    lh a1, 10(sp)
+; RV32I-NEXT:    bnez a0, .LBB98_4
 ; RV32I-NEXT:  .LBB98_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    slli a1, a0, 16
-; RV32I-NEXT:    srai a1, a1, 16
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bge s0, a1, .LBB98_1
+; RV32I-NEXT:    slli a0, a1, 16
+; RV32I-NEXT:    srai a0, a0, 16
+; RV32I-NEXT:    mv a2, a1
+; RV32I-NEXT:    bge s0, a0, .LBB98_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB98_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:    j .LBB98_1
 ; RV32I-NEXT:  .LBB98_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -8454,33 +8454,33 @@ define i16 @atomicrmw_min_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_min_i16_acq_rel:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    addi a3, zero, 16
-; RV32IA-NEXT:    sub a6, a3, a2
-; RV32IA-NEXT:    lui a4, 16
-; RV32IA-NEXT:    addi a4, a4, -1
-; RV32IA-NEXT:    sll a7, a4, a2
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a7, a3, a0
 ; RV32IA-NEXT:    slli a1, a1, 16
 ; RV32IA-NEXT:    srai a1, a1, 16
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    addi a4, zero, 16
+; RV32IA-NEXT:    sub a4, a4, a0
 ; RV32IA-NEXT:  .LBB98_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a5, (a0)
-; RV32IA-NEXT:    and a4, a5, a7
-; RV32IA-NEXT:    mv a3, a5
-; RV32IA-NEXT:    sll a4, a4, a6
-; RV32IA-NEXT:    sra a4, a4, a6
-; RV32IA-NEXT:    bge a1, a4, .LBB98_3
+; RV32IA-NEXT:    lr.w.aq a5, (a6)
+; RV32IA-NEXT:    and a3, a5, a7
+; RV32IA-NEXT:    mv a2, a5
+; RV32IA-NEXT:    sll a3, a3, a4
+; RV32IA-NEXT:    sra a3, a3, a4
+; RV32IA-NEXT:    bge a1, a3, .LBB98_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB98_1 Depth=1
-; RV32IA-NEXT:    xor a3, a5, a1
-; RV32IA-NEXT:    and a3, a3, a7
-; RV32IA-NEXT:    xor a3, a5, a3
+; RV32IA-NEXT:    xor a2, a5, a1
+; RV32IA-NEXT:    and a2, a2, a7
+; RV32IA-NEXT:    xor a2, a5, a2
 ; RV32IA-NEXT:  .LBB98_3: # in Loop: Header=BB98_1 Depth=1
-; RV32IA-NEXT:    sc.w.rl a3, a3, (a0)
-; RV32IA-NEXT:    bnez a3, .LBB98_1
+; RV32IA-NEXT:    sc.w.rl a2, a2, (a6)
+; RV32IA-NEXT:    bnez a2, .LBB98_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a5, a2
+; RV32IA-NEXT:    srl a0, a5, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_min_i16_acq_rel:
@@ -8493,33 +8493,33 @@ define i16 @atomicrmw_min_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    sd s3, 8(sp)
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lhu a0, 0(a0)
-; RV64I-NEXT:    slli a1, a1, 48
-; RV64I-NEXT:    srai s0, a1, 48
+; RV64I-NEXT:    lhu a1, 0(a0)
+; RV64I-NEXT:    slli a0, s2, 48
+; RV64I-NEXT:    srai s0, a0, 48
 ; RV64I-NEXT:    addi s3, sp, 6
 ; RV64I-NEXT:    j .LBB98_2
 ; RV64I-NEXT:  .LBB98_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB98_2 Depth=1
-; RV64I-NEXT:    sh a0, 6(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sh a1, 6(sp)
 ; RV64I-NEXT:    addi a3, zero, 4
 ; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    mv a0, s1
+; RV64I-NEXT:    mv a1, s3
 ; RV64I-NEXT:    call __atomic_compare_exchange_2
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lh a0, 6(sp)
-; RV64I-NEXT:    bnez a1, .LBB98_4
+; RV64I-NEXT:    lh a1, 6(sp)
+; RV64I-NEXT:    bnez a0, .LBB98_4
 ; RV64I-NEXT:  .LBB98_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    slli a1, a0, 48
-; RV64I-NEXT:    srai a1, a1, 48
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bge s0, a1, .LBB98_1
+; RV64I-NEXT:    slli a0, a1, 48
+; RV64I-NEXT:    srai a0, a0, 48
+; RV64I-NEXT:    mv a2, a1
+; RV64I-NEXT:    bge s0, a0, .LBB98_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB98_2 Depth=1
 ; RV64I-NEXT:    mv a2, s2
 ; RV64I-NEXT:    j .LBB98_1
 ; RV64I-NEXT:  .LBB98_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -8530,33 +8530,33 @@ define i16 @atomicrmw_min_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_min_i16_acq_rel:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    addi a3, zero, 48
-; RV64IA-NEXT:    sub a6, a3, a2
-; RV64IA-NEXT:    lui a4, 16
-; RV64IA-NEXT:    addiw a4, a4, -1
-; RV64IA-NEXT:    sllw a7, a4, a2
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a7, a3, a0
 ; RV64IA-NEXT:    slli a1, a1, 48
 ; RV64IA-NEXT:    srai a1, a1, 48
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    addi a4, zero, 48
+; RV64IA-NEXT:    sub a4, a4, a0
 ; RV64IA-NEXT:  .LBB98_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a5, (a0)
-; RV64IA-NEXT:    and a4, a5, a7
-; RV64IA-NEXT:    mv a3, a5
-; RV64IA-NEXT:    sll a4, a4, a6
-; RV64IA-NEXT:    sra a4, a4, a6
-; RV64IA-NEXT:    bge a1, a4, .LBB98_3
+; RV64IA-NEXT:    lr.w.aq a5, (a6)
+; RV64IA-NEXT:    and a3, a5, a7
+; RV64IA-NEXT:    mv a2, a5
+; RV64IA-NEXT:    sll a3, a3, a4
+; RV64IA-NEXT:    sra a3, a3, a4
+; RV64IA-NEXT:    bge a1, a3, .LBB98_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB98_1 Depth=1
-; RV64IA-NEXT:    xor a3, a5, a1
-; RV64IA-NEXT:    and a3, a3, a7
-; RV64IA-NEXT:    xor a3, a5, a3
+; RV64IA-NEXT:    xor a2, a5, a1
+; RV64IA-NEXT:    and a2, a2, a7
+; RV64IA-NEXT:    xor a2, a5, a2
 ; RV64IA-NEXT:  .LBB98_3: # in Loop: Header=BB98_1 Depth=1
-; RV64IA-NEXT:    sc.w.rl a3, a3, (a0)
-; RV64IA-NEXT:    bnez a3, .LBB98_1
+; RV64IA-NEXT:    sc.w.rl a2, a2, (a6)
+; RV64IA-NEXT:    bnez a2, .LBB98_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a5, a2
+; RV64IA-NEXT:    srlw a0, a5, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw min i16* %a, i16 %b acq_rel
   ret i16 %1
@@ -8573,33 +8573,33 @@ define i16 @atomicrmw_min_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    sw s3, 12(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lhu a0, 0(a0)
-; RV32I-NEXT:    slli a1, a1, 16
-; RV32I-NEXT:    srai s0, a1, 16
+; RV32I-NEXT:    lhu a1, 0(a0)
+; RV32I-NEXT:    slli a0, s2, 16
+; RV32I-NEXT:    srai s0, a0, 16
 ; RV32I-NEXT:    addi s3, sp, 10
 ; RV32I-NEXT:    j .LBB99_2
 ; RV32I-NEXT:  .LBB99_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB99_2 Depth=1
-; RV32I-NEXT:    sh a0, 10(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sh a1, 10(sp)
 ; RV32I-NEXT:    addi a3, zero, 5
 ; RV32I-NEXT:    addi a4, zero, 5
+; RV32I-NEXT:    mv a0, s1
+; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    call __atomic_compare_exchange_2
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lh a0, 10(sp)
-; RV32I-NEXT:    bnez a1, .LBB99_4
+; RV32I-NEXT:    lh a1, 10(sp)
+; RV32I-NEXT:    bnez a0, .LBB99_4
 ; RV32I-NEXT:  .LBB99_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    slli a1, a0, 16
-; RV32I-NEXT:    srai a1, a1, 16
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bge s0, a1, .LBB99_1
+; RV32I-NEXT:    slli a0, a1, 16
+; RV32I-NEXT:    srai a0, a0, 16
+; RV32I-NEXT:    mv a2, a1
+; RV32I-NEXT:    bge s0, a0, .LBB99_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB99_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:    j .LBB99_1
 ; RV32I-NEXT:  .LBB99_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -8610,33 +8610,33 @@ define i16 @atomicrmw_min_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_min_i16_seq_cst:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    addi a3, zero, 16
-; RV32IA-NEXT:    sub a6, a3, a2
-; RV32IA-NEXT:    lui a4, 16
-; RV32IA-NEXT:    addi a4, a4, -1
-; RV32IA-NEXT:    sll a7, a4, a2
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a7, a3, a0
 ; RV32IA-NEXT:    slli a1, a1, 16
 ; RV32IA-NEXT:    srai a1, a1, 16
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    addi a4, zero, 16
+; RV32IA-NEXT:    sub a4, a4, a0
 ; RV32IA-NEXT:  .LBB99_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aqrl a5, (a0)
-; RV32IA-NEXT:    and a4, a5, a7
-; RV32IA-NEXT:    mv a3, a5
-; RV32IA-NEXT:    sll a4, a4, a6
-; RV32IA-NEXT:    sra a4, a4, a6
-; RV32IA-NEXT:    bge a1, a4, .LBB99_3
+; RV32IA-NEXT:    lr.w.aqrl a5, (a6)
+; RV32IA-NEXT:    and a3, a5, a7
+; RV32IA-NEXT:    mv a2, a5
+; RV32IA-NEXT:    sll a3, a3, a4
+; RV32IA-NEXT:    sra a3, a3, a4
+; RV32IA-NEXT:    bge a1, a3, .LBB99_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB99_1 Depth=1
-; RV32IA-NEXT:    xor a3, a5, a1
-; RV32IA-NEXT:    and a3, a3, a7
-; RV32IA-NEXT:    xor a3, a5, a3
+; RV32IA-NEXT:    xor a2, a5, a1
+; RV32IA-NEXT:    and a2, a2, a7
+; RV32IA-NEXT:    xor a2, a5, a2
 ; RV32IA-NEXT:  .LBB99_3: # in Loop: Header=BB99_1 Depth=1
-; RV32IA-NEXT:    sc.w.aqrl a3, a3, (a0)
-; RV32IA-NEXT:    bnez a3, .LBB99_1
+; RV32IA-NEXT:    sc.w.aqrl a2, a2, (a6)
+; RV32IA-NEXT:    bnez a2, .LBB99_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a5, a2
+; RV32IA-NEXT:    srl a0, a5, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_min_i16_seq_cst:
@@ -8649,33 +8649,33 @@ define i16 @atomicrmw_min_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    sd s3, 8(sp)
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lhu a0, 0(a0)
-; RV64I-NEXT:    slli a1, a1, 48
-; RV64I-NEXT:    srai s0, a1, 48
+; RV64I-NEXT:    lhu a1, 0(a0)
+; RV64I-NEXT:    slli a0, s2, 48
+; RV64I-NEXT:    srai s0, a0, 48
 ; RV64I-NEXT:    addi s3, sp, 6
 ; RV64I-NEXT:    j .LBB99_2
 ; RV64I-NEXT:  .LBB99_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB99_2 Depth=1
-; RV64I-NEXT:    sh a0, 6(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sh a1, 6(sp)
 ; RV64I-NEXT:    addi a3, zero, 5
 ; RV64I-NEXT:    addi a4, zero, 5
+; RV64I-NEXT:    mv a0, s1
+; RV64I-NEXT:    mv a1, s3
 ; RV64I-NEXT:    call __atomic_compare_exchange_2
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lh a0, 6(sp)
-; RV64I-NEXT:    bnez a1, .LBB99_4
+; RV64I-NEXT:    lh a1, 6(sp)
+; RV64I-NEXT:    bnez a0, .LBB99_4
 ; RV64I-NEXT:  .LBB99_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    slli a1, a0, 48
-; RV64I-NEXT:    srai a1, a1, 48
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bge s0, a1, .LBB99_1
+; RV64I-NEXT:    slli a0, a1, 48
+; RV64I-NEXT:    srai a0, a0, 48
+; RV64I-NEXT:    mv a2, a1
+; RV64I-NEXT:    bge s0, a0, .LBB99_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB99_2 Depth=1
 ; RV64I-NEXT:    mv a2, s2
 ; RV64I-NEXT:    j .LBB99_1
 ; RV64I-NEXT:  .LBB99_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -8686,33 +8686,33 @@ define i16 @atomicrmw_min_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_min_i16_seq_cst:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    addi a3, zero, 48
-; RV64IA-NEXT:    sub a6, a3, a2
-; RV64IA-NEXT:    lui a4, 16
-; RV64IA-NEXT:    addiw a4, a4, -1
-; RV64IA-NEXT:    sllw a7, a4, a2
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a7, a3, a0
 ; RV64IA-NEXT:    slli a1, a1, 48
 ; RV64IA-NEXT:    srai a1, a1, 48
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    addi a4, zero, 48
+; RV64IA-NEXT:    sub a4, a4, a0
 ; RV64IA-NEXT:  .LBB99_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aqrl a5, (a0)
-; RV64IA-NEXT:    and a4, a5, a7
-; RV64IA-NEXT:    mv a3, a5
-; RV64IA-NEXT:    sll a4, a4, a6
-; RV64IA-NEXT:    sra a4, a4, a6
-; RV64IA-NEXT:    bge a1, a4, .LBB99_3
+; RV64IA-NEXT:    lr.w.aqrl a5, (a6)
+; RV64IA-NEXT:    and a3, a5, a7
+; RV64IA-NEXT:    mv a2, a5
+; RV64IA-NEXT:    sll a3, a3, a4
+; RV64IA-NEXT:    sra a3, a3, a4
+; RV64IA-NEXT:    bge a1, a3, .LBB99_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB99_1 Depth=1
-; RV64IA-NEXT:    xor a3, a5, a1
-; RV64IA-NEXT:    and a3, a3, a7
-; RV64IA-NEXT:    xor a3, a5, a3
+; RV64IA-NEXT:    xor a2, a5, a1
+; RV64IA-NEXT:    and a2, a2, a7
+; RV64IA-NEXT:    xor a2, a5, a2
 ; RV64IA-NEXT:  .LBB99_3: # in Loop: Header=BB99_1 Depth=1
-; RV64IA-NEXT:    sc.w.aqrl a3, a3, (a0)
-; RV64IA-NEXT:    bnez a3, .LBB99_1
+; RV64IA-NEXT:    sc.w.aqrl a2, a2, (a6)
+; RV64IA-NEXT:    bnez a2, .LBB99_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a5, a2
+; RV64IA-NEXT:    srlw a0, a5, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw min i16* %a, i16 %b seq_cst
   ret i16 %1
@@ -8730,33 +8730,33 @@ define i16 @atomicrmw_umax_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    sw s4, 8(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s4, a0
-; RV32I-NEXT:    lhu a0, 0(a0)
-; RV32I-NEXT:    lui a1, 16
-; RV32I-NEXT:    addi s0, a1, -1
+; RV32I-NEXT:    lhu a1, 0(a0)
+; RV32I-NEXT:    lui a0, 16
+; RV32I-NEXT:    addi s0, a0, -1
 ; RV32I-NEXT:    and s1, s2, s0
 ; RV32I-NEXT:    addi s3, sp, 6
 ; RV32I-NEXT:    j .LBB100_2
 ; RV32I-NEXT:  .LBB100_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB100_2 Depth=1
-; RV32I-NEXT:    sh a0, 6(sp)
+; RV32I-NEXT:    sh a1, 6(sp)
 ; RV32I-NEXT:    mv a0, s4
 ; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    mv a3, zero
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_2
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lh a0, 6(sp)
-; RV32I-NEXT:    bnez a1, .LBB100_4
+; RV32I-NEXT:    lh a1, 6(sp)
+; RV32I-NEXT:    bnez a0, .LBB100_4
 ; RV32I-NEXT:  .LBB100_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    and a1, a0, s0
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bltu s1, a1, .LBB100_1
+; RV32I-NEXT:    and a0, a1, s0
+; RV32I-NEXT:    mv a2, a1
+; RV32I-NEXT:    bltu s1, a0, .LBB100_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB100_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:    j .LBB100_1
 ; RV32I-NEXT:  .LBB100_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    lw s4, 8(sp)
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
@@ -8768,28 +8768,28 @@ define i16 @atomicrmw_umax_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_umax_i16_monotonic:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a6, a2, a3
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB100_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a4, (a0)
-; RV32IA-NEXT:    and a2, a4, a6
-; RV32IA-NEXT:    mv a5, a4
+; RV32IA-NEXT:    lr.w a3, (a6)
+; RV32IA-NEXT:    and a2, a3, a4
+; RV32IA-NEXT:    mv a5, a3
 ; RV32IA-NEXT:    bgeu a2, a1, .LBB100_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB100_1 Depth=1
-; RV32IA-NEXT:    xor a5, a4, a1
-; RV32IA-NEXT:    and a5, a5, a6
-; RV32IA-NEXT:    xor a5, a4, a5
+; RV32IA-NEXT:    xor a5, a3, a1
+; RV32IA-NEXT:    and a5, a5, a4
+; RV32IA-NEXT:    xor a5, a3, a5
 ; RV32IA-NEXT:  .LBB100_3: # in Loop: Header=BB100_1 Depth=1
-; RV32IA-NEXT:    sc.w a5, a5, (a0)
+; RV32IA-NEXT:    sc.w a5, a5, (a6)
 ; RV32IA-NEXT:    bnez a5, .LBB100_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a4, a3
+; RV32IA-NEXT:    srl a0, a3, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_umax_i16_monotonic:
@@ -8803,33 +8803,33 @@ define i16 @atomicrmw_umax_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    sd s4, 16(sp)
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s4, a0
-; RV64I-NEXT:    lhu a0, 0(a0)
-; RV64I-NEXT:    lui a1, 16
-; RV64I-NEXT:    addiw s0, a1, -1
+; RV64I-NEXT:    lhu a1, 0(a0)
+; RV64I-NEXT:    lui a0, 16
+; RV64I-NEXT:    addiw s0, a0, -1
 ; RV64I-NEXT:    and s1, s2, s0
 ; RV64I-NEXT:    addi s3, sp, 14
 ; RV64I-NEXT:    j .LBB100_2
 ; RV64I-NEXT:  .LBB100_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB100_2 Depth=1
-; RV64I-NEXT:    sh a0, 14(sp)
+; RV64I-NEXT:    sh a1, 14(sp)
 ; RV64I-NEXT:    mv a0, s4
 ; RV64I-NEXT:    mv a1, s3
 ; RV64I-NEXT:    mv a3, zero
 ; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    call __atomic_compare_exchange_2
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lh a0, 14(sp)
-; RV64I-NEXT:    bnez a1, .LBB100_4
+; RV64I-NEXT:    lh a1, 14(sp)
+; RV64I-NEXT:    bnez a0, .LBB100_4
 ; RV64I-NEXT:  .LBB100_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    and a1, a0, s0
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bltu s1, a1, .LBB100_1
+; RV64I-NEXT:    and a0, a1, s0
+; RV64I-NEXT:    mv a2, a1
+; RV64I-NEXT:    bltu s1, a0, .LBB100_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB100_2 Depth=1
 ; RV64I-NEXT:    mv a2, s2
 ; RV64I-NEXT:    j .LBB100_1
 ; RV64I-NEXT:  .LBB100_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ld s4, 16(sp)
 ; RV64I-NEXT:    ld s3, 24(sp)
 ; RV64I-NEXT:    ld s2, 32(sp)
@@ -8841,28 +8841,28 @@ define i16 @atomicrmw_umax_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_umax_i16_monotonic:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a6, a2, a3
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB100_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a4, (a0)
-; RV64IA-NEXT:    and a2, a4, a6
-; RV64IA-NEXT:    mv a5, a4
+; RV64IA-NEXT:    lr.w a3, (a6)
+; RV64IA-NEXT:    and a2, a3, a4
+; RV64IA-NEXT:    mv a5, a3
 ; RV64IA-NEXT:    bgeu a2, a1, .LBB100_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB100_1 Depth=1
-; RV64IA-NEXT:    xor a5, a4, a1
-; RV64IA-NEXT:    and a5, a5, a6
-; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    xor a5, a3, a1
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
 ; RV64IA-NEXT:  .LBB100_3: # in Loop: Header=BB100_1 Depth=1
-; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    sc.w a5, a5, (a6)
 ; RV64IA-NEXT:    bnez a5, .LBB100_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    srlw a0, a3, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw umax i16* %a, i16 %b monotonic
   ret i16 %1
@@ -8880,33 +8880,33 @@ define i16 @atomicrmw_umax_i16_acquire(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    sw s4, 8(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s4, a0
-; RV32I-NEXT:    lhu a0, 0(a0)
-; RV32I-NEXT:    lui a1, 16
-; RV32I-NEXT:    addi s0, a1, -1
+; RV32I-NEXT:    lhu a1, 0(a0)
+; RV32I-NEXT:    lui a0, 16
+; RV32I-NEXT:    addi s0, a0, -1
 ; RV32I-NEXT:    and s1, s2, s0
 ; RV32I-NEXT:    addi s3, sp, 6
 ; RV32I-NEXT:    j .LBB101_2
 ; RV32I-NEXT:  .LBB101_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB101_2 Depth=1
-; RV32I-NEXT:    sh a0, 6(sp)
-; RV32I-NEXT:    mv a0, s4
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sh a1, 6(sp)
 ; RV32I-NEXT:    addi a3, zero, 2
 ; RV32I-NEXT:    addi a4, zero, 2
+; RV32I-NEXT:    mv a0, s4
+; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    call __atomic_compare_exchange_2
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lh a0, 6(sp)
-; RV32I-NEXT:    bnez a1, .LBB101_4
+; RV32I-NEXT:    lh a1, 6(sp)
+; RV32I-NEXT:    bnez a0, .LBB101_4
 ; RV32I-NEXT:  .LBB101_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    and a1, a0, s0
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bltu s1, a1, .LBB101_1
+; RV32I-NEXT:    and a0, a1, s0
+; RV32I-NEXT:    mv a2, a1
+; RV32I-NEXT:    bltu s1, a0, .LBB101_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB101_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:    j .LBB101_1
 ; RV32I-NEXT:  .LBB101_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    lw s4, 8(sp)
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
@@ -8918,28 +8918,28 @@ define i16 @atomicrmw_umax_i16_acquire(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_umax_i16_acquire:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a6, a2, a3
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB101_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a4, (a0)
-; RV32IA-NEXT:    and a2, a4, a6
-; RV32IA-NEXT:    mv a5, a4
+; RV32IA-NEXT:    lr.w.aq a3, (a6)
+; RV32IA-NEXT:    and a2, a3, a4
+; RV32IA-NEXT:    mv a5, a3
 ; RV32IA-NEXT:    bgeu a2, a1, .LBB101_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB101_1 Depth=1
-; RV32IA-NEXT:    xor a5, a4, a1
-; RV32IA-NEXT:    and a5, a5, a6
-; RV32IA-NEXT:    xor a5, a4, a5
+; RV32IA-NEXT:    xor a5, a3, a1
+; RV32IA-NEXT:    and a5, a5, a4
+; RV32IA-NEXT:    xor a5, a3, a5
 ; RV32IA-NEXT:  .LBB101_3: # in Loop: Header=BB101_1 Depth=1
-; RV32IA-NEXT:    sc.w a5, a5, (a0)
+; RV32IA-NEXT:    sc.w a5, a5, (a6)
 ; RV32IA-NEXT:    bnez a5, .LBB101_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a4, a3
+; RV32IA-NEXT:    srl a0, a3, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_umax_i16_acquire:
@@ -8953,33 +8953,33 @@ define i16 @atomicrmw_umax_i16_acquire(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    sd s4, 16(sp)
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s4, a0
-; RV64I-NEXT:    lhu a0, 0(a0)
-; RV64I-NEXT:    lui a1, 16
-; RV64I-NEXT:    addiw s0, a1, -1
+; RV64I-NEXT:    lhu a1, 0(a0)
+; RV64I-NEXT:    lui a0, 16
+; RV64I-NEXT:    addiw s0, a0, -1
 ; RV64I-NEXT:    and s1, s2, s0
 ; RV64I-NEXT:    addi s3, sp, 14
 ; RV64I-NEXT:    j .LBB101_2
 ; RV64I-NEXT:  .LBB101_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB101_2 Depth=1
-; RV64I-NEXT:    sh a0, 14(sp)
-; RV64I-NEXT:    mv a0, s4
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sh a1, 14(sp)
 ; RV64I-NEXT:    addi a3, zero, 2
 ; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
 ; RV64I-NEXT:    call __atomic_compare_exchange_2
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lh a0, 14(sp)
-; RV64I-NEXT:    bnez a1, .LBB101_4
+; RV64I-NEXT:    lh a1, 14(sp)
+; RV64I-NEXT:    bnez a0, .LBB101_4
 ; RV64I-NEXT:  .LBB101_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    and a1, a0, s0
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bltu s1, a1, .LBB101_1
+; RV64I-NEXT:    and a0, a1, s0
+; RV64I-NEXT:    mv a2, a1
+; RV64I-NEXT:    bltu s1, a0, .LBB101_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB101_2 Depth=1
 ; RV64I-NEXT:    mv a2, s2
 ; RV64I-NEXT:    j .LBB101_1
 ; RV64I-NEXT:  .LBB101_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ld s4, 16(sp)
 ; RV64I-NEXT:    ld s3, 24(sp)
 ; RV64I-NEXT:    ld s2, 32(sp)
@@ -8991,28 +8991,28 @@ define i16 @atomicrmw_umax_i16_acquire(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_umax_i16_acquire:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a6, a2, a3
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB101_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a4, (a0)
-; RV64IA-NEXT:    and a2, a4, a6
-; RV64IA-NEXT:    mv a5, a4
+; RV64IA-NEXT:    lr.w.aq a3, (a6)
+; RV64IA-NEXT:    and a2, a3, a4
+; RV64IA-NEXT:    mv a5, a3
 ; RV64IA-NEXT:    bgeu a2, a1, .LBB101_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB101_1 Depth=1
-; RV64IA-NEXT:    xor a5, a4, a1
-; RV64IA-NEXT:    and a5, a5, a6
-; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    xor a5, a3, a1
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
 ; RV64IA-NEXT:  .LBB101_3: # in Loop: Header=BB101_1 Depth=1
-; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    sc.w a5, a5, (a6)
 ; RV64IA-NEXT:    bnez a5, .LBB101_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    srlw a0, a3, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw umax i16* %a, i16 %b acquire
   ret i16 %1
@@ -9030,33 +9030,33 @@ define i16 @atomicrmw_umax_i16_release(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    sw s4, 8(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s4, a0
-; RV32I-NEXT:    lhu a0, 0(a0)
-; RV32I-NEXT:    lui a1, 16
-; RV32I-NEXT:    addi s0, a1, -1
+; RV32I-NEXT:    lhu a1, 0(a0)
+; RV32I-NEXT:    lui a0, 16
+; RV32I-NEXT:    addi s0, a0, -1
 ; RV32I-NEXT:    and s1, s2, s0
 ; RV32I-NEXT:    addi s3, sp, 6
 ; RV32I-NEXT:    j .LBB102_2
 ; RV32I-NEXT:  .LBB102_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB102_2 Depth=1
-; RV32I-NEXT:    sh a0, 6(sp)
+; RV32I-NEXT:    sh a1, 6(sp)
+; RV32I-NEXT:    addi a3, zero, 3
 ; RV32I-NEXT:    mv a0, s4
 ; RV32I-NEXT:    mv a1, s3
-; RV32I-NEXT:    addi a3, zero, 3
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_2
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lh a0, 6(sp)
-; RV32I-NEXT:    bnez a1, .LBB102_4
+; RV32I-NEXT:    lh a1, 6(sp)
+; RV32I-NEXT:    bnez a0, .LBB102_4
 ; RV32I-NEXT:  .LBB102_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    and a1, a0, s0
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bltu s1, a1, .LBB102_1
+; RV32I-NEXT:    and a0, a1, s0
+; RV32I-NEXT:    mv a2, a1
+; RV32I-NEXT:    bltu s1, a0, .LBB102_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB102_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:    j .LBB102_1
 ; RV32I-NEXT:  .LBB102_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    lw s4, 8(sp)
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
@@ -9068,28 +9068,28 @@ define i16 @atomicrmw_umax_i16_release(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_umax_i16_release:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a6, a2, a3
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB102_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a4, (a0)
-; RV32IA-NEXT:    and a2, a4, a6
-; RV32IA-NEXT:    mv a5, a4
+; RV32IA-NEXT:    lr.w a3, (a6)
+; RV32IA-NEXT:    and a2, a3, a4
+; RV32IA-NEXT:    mv a5, a3
 ; RV32IA-NEXT:    bgeu a2, a1, .LBB102_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB102_1 Depth=1
-; RV32IA-NEXT:    xor a5, a4, a1
-; RV32IA-NEXT:    and a5, a5, a6
-; RV32IA-NEXT:    xor a5, a4, a5
+; RV32IA-NEXT:    xor a5, a3, a1
+; RV32IA-NEXT:    and a5, a5, a4
+; RV32IA-NEXT:    xor a5, a3, a5
 ; RV32IA-NEXT:  .LBB102_3: # in Loop: Header=BB102_1 Depth=1
-; RV32IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV32IA-NEXT:    sc.w.rl a5, a5, (a6)
 ; RV32IA-NEXT:    bnez a5, .LBB102_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a4, a3
+; RV32IA-NEXT:    srl a0, a3, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_umax_i16_release:
@@ -9103,33 +9103,33 @@ define i16 @atomicrmw_umax_i16_release(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    sd s4, 16(sp)
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s4, a0
-; RV64I-NEXT:    lhu a0, 0(a0)
-; RV64I-NEXT:    lui a1, 16
-; RV64I-NEXT:    addiw s0, a1, -1
+; RV64I-NEXT:    lhu a1, 0(a0)
+; RV64I-NEXT:    lui a0, 16
+; RV64I-NEXT:    addiw s0, a0, -1
 ; RV64I-NEXT:    and s1, s2, s0
 ; RV64I-NEXT:    addi s3, sp, 14
 ; RV64I-NEXT:    j .LBB102_2
 ; RV64I-NEXT:  .LBB102_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB102_2 Depth=1
-; RV64I-NEXT:    sh a0, 14(sp)
+; RV64I-NEXT:    sh a1, 14(sp)
+; RV64I-NEXT:    addi a3, zero, 3
 ; RV64I-NEXT:    mv a0, s4
 ; RV64I-NEXT:    mv a1, s3
-; RV64I-NEXT:    addi a3, zero, 3
 ; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    call __atomic_compare_exchange_2
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lh a0, 14(sp)
-; RV64I-NEXT:    bnez a1, .LBB102_4
+; RV64I-NEXT:    lh a1, 14(sp)
+; RV64I-NEXT:    bnez a0, .LBB102_4
 ; RV64I-NEXT:  .LBB102_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    and a1, a0, s0
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bltu s1, a1, .LBB102_1
+; RV64I-NEXT:    and a0, a1, s0
+; RV64I-NEXT:    mv a2, a1
+; RV64I-NEXT:    bltu s1, a0, .LBB102_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB102_2 Depth=1
 ; RV64I-NEXT:    mv a2, s2
 ; RV64I-NEXT:    j .LBB102_1
 ; RV64I-NEXT:  .LBB102_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ld s4, 16(sp)
 ; RV64I-NEXT:    ld s3, 24(sp)
 ; RV64I-NEXT:    ld s2, 32(sp)
@@ -9141,28 +9141,28 @@ define i16 @atomicrmw_umax_i16_release(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_umax_i16_release:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a6, a2, a3
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB102_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a4, (a0)
-; RV64IA-NEXT:    and a2, a4, a6
-; RV64IA-NEXT:    mv a5, a4
+; RV64IA-NEXT:    lr.w a3, (a6)
+; RV64IA-NEXT:    and a2, a3, a4
+; RV64IA-NEXT:    mv a5, a3
 ; RV64IA-NEXT:    bgeu a2, a1, .LBB102_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB102_1 Depth=1
-; RV64IA-NEXT:    xor a5, a4, a1
-; RV64IA-NEXT:    and a5, a5, a6
-; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    xor a5, a3, a1
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
 ; RV64IA-NEXT:  .LBB102_3: # in Loop: Header=BB102_1 Depth=1
-; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a6)
 ; RV64IA-NEXT:    bnez a5, .LBB102_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    srlw a0, a3, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw umax i16* %a, i16 %b release
   ret i16 %1
@@ -9180,33 +9180,33 @@ define i16 @atomicrmw_umax_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    sw s4, 8(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s4, a0
-; RV32I-NEXT:    lhu a0, 0(a0)
-; RV32I-NEXT:    lui a1, 16
-; RV32I-NEXT:    addi s0, a1, -1
+; RV32I-NEXT:    lhu a1, 0(a0)
+; RV32I-NEXT:    lui a0, 16
+; RV32I-NEXT:    addi s0, a0, -1
 ; RV32I-NEXT:    and s1, s2, s0
 ; RV32I-NEXT:    addi s3, sp, 6
 ; RV32I-NEXT:    j .LBB103_2
 ; RV32I-NEXT:  .LBB103_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB103_2 Depth=1
-; RV32I-NEXT:    sh a0, 6(sp)
-; RV32I-NEXT:    mv a0, s4
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sh a1, 6(sp)
 ; RV32I-NEXT:    addi a3, zero, 4
 ; RV32I-NEXT:    addi a4, zero, 2
+; RV32I-NEXT:    mv a0, s4
+; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    call __atomic_compare_exchange_2
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lh a0, 6(sp)
-; RV32I-NEXT:    bnez a1, .LBB103_4
+; RV32I-NEXT:    lh a1, 6(sp)
+; RV32I-NEXT:    bnez a0, .LBB103_4
 ; RV32I-NEXT:  .LBB103_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    and a1, a0, s0
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bltu s1, a1, .LBB103_1
+; RV32I-NEXT:    and a0, a1, s0
+; RV32I-NEXT:    mv a2, a1
+; RV32I-NEXT:    bltu s1, a0, .LBB103_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB103_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:    j .LBB103_1
 ; RV32I-NEXT:  .LBB103_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    lw s4, 8(sp)
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
@@ -9218,28 +9218,28 @@ define i16 @atomicrmw_umax_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_umax_i16_acq_rel:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a6, a2, a3
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB103_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a4, (a0)
-; RV32IA-NEXT:    and a2, a4, a6
-; RV32IA-NEXT:    mv a5, a4
+; RV32IA-NEXT:    lr.w.aq a3, (a6)
+; RV32IA-NEXT:    and a2, a3, a4
+; RV32IA-NEXT:    mv a5, a3
 ; RV32IA-NEXT:    bgeu a2, a1, .LBB103_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB103_1 Depth=1
-; RV32IA-NEXT:    xor a5, a4, a1
-; RV32IA-NEXT:    and a5, a5, a6
-; RV32IA-NEXT:    xor a5, a4, a5
+; RV32IA-NEXT:    xor a5, a3, a1
+; RV32IA-NEXT:    and a5, a5, a4
+; RV32IA-NEXT:    xor a5, a3, a5
 ; RV32IA-NEXT:  .LBB103_3: # in Loop: Header=BB103_1 Depth=1
-; RV32IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV32IA-NEXT:    sc.w.rl a5, a5, (a6)
 ; RV32IA-NEXT:    bnez a5, .LBB103_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a4, a3
+; RV32IA-NEXT:    srl a0, a3, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_umax_i16_acq_rel:
@@ -9253,33 +9253,33 @@ define i16 @atomicrmw_umax_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    sd s4, 16(sp)
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s4, a0
-; RV64I-NEXT:    lhu a0, 0(a0)
-; RV64I-NEXT:    lui a1, 16
-; RV64I-NEXT:    addiw s0, a1, -1
+; RV64I-NEXT:    lhu a1, 0(a0)
+; RV64I-NEXT:    lui a0, 16
+; RV64I-NEXT:    addiw s0, a0, -1
 ; RV64I-NEXT:    and s1, s2, s0
 ; RV64I-NEXT:    addi s3, sp, 14
 ; RV64I-NEXT:    j .LBB103_2
 ; RV64I-NEXT:  .LBB103_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB103_2 Depth=1
-; RV64I-NEXT:    sh a0, 14(sp)
-; RV64I-NEXT:    mv a0, s4
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sh a1, 14(sp)
 ; RV64I-NEXT:    addi a3, zero, 4
 ; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
 ; RV64I-NEXT:    call __atomic_compare_exchange_2
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lh a0, 14(sp)
-; RV64I-NEXT:    bnez a1, .LBB103_4
+; RV64I-NEXT:    lh a1, 14(sp)
+; RV64I-NEXT:    bnez a0, .LBB103_4
 ; RV64I-NEXT:  .LBB103_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    and a1, a0, s0
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bltu s1, a1, .LBB103_1
+; RV64I-NEXT:    and a0, a1, s0
+; RV64I-NEXT:    mv a2, a1
+; RV64I-NEXT:    bltu s1, a0, .LBB103_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB103_2 Depth=1
 ; RV64I-NEXT:    mv a2, s2
 ; RV64I-NEXT:    j .LBB103_1
 ; RV64I-NEXT:  .LBB103_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ld s4, 16(sp)
 ; RV64I-NEXT:    ld s3, 24(sp)
 ; RV64I-NEXT:    ld s2, 32(sp)
@@ -9291,28 +9291,28 @@ define i16 @atomicrmw_umax_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_umax_i16_acq_rel:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a6, a2, a3
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB103_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a4, (a0)
-; RV64IA-NEXT:    and a2, a4, a6
-; RV64IA-NEXT:    mv a5, a4
+; RV64IA-NEXT:    lr.w.aq a3, (a6)
+; RV64IA-NEXT:    and a2, a3, a4
+; RV64IA-NEXT:    mv a5, a3
 ; RV64IA-NEXT:    bgeu a2, a1, .LBB103_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB103_1 Depth=1
-; RV64IA-NEXT:    xor a5, a4, a1
-; RV64IA-NEXT:    and a5, a5, a6
-; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    xor a5, a3, a1
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
 ; RV64IA-NEXT:  .LBB103_3: # in Loop: Header=BB103_1 Depth=1
-; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a6)
 ; RV64IA-NEXT:    bnez a5, .LBB103_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    srlw a0, a3, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw umax i16* %a, i16 %b acq_rel
   ret i16 %1
@@ -9330,33 +9330,33 @@ define i16 @atomicrmw_umax_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    sw s4, 8(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s4, a0
-; RV32I-NEXT:    lhu a0, 0(a0)
-; RV32I-NEXT:    lui a1, 16
-; RV32I-NEXT:    addi s0, a1, -1
+; RV32I-NEXT:    lhu a1, 0(a0)
+; RV32I-NEXT:    lui a0, 16
+; RV32I-NEXT:    addi s0, a0, -1
 ; RV32I-NEXT:    and s1, s2, s0
 ; RV32I-NEXT:    addi s3, sp, 6
 ; RV32I-NEXT:    j .LBB104_2
 ; RV32I-NEXT:  .LBB104_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB104_2 Depth=1
-; RV32I-NEXT:    sh a0, 6(sp)
-; RV32I-NEXT:    mv a0, s4
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sh a1, 6(sp)
 ; RV32I-NEXT:    addi a3, zero, 5
 ; RV32I-NEXT:    addi a4, zero, 5
+; RV32I-NEXT:    mv a0, s4
+; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    call __atomic_compare_exchange_2
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lh a0, 6(sp)
-; RV32I-NEXT:    bnez a1, .LBB104_4
+; RV32I-NEXT:    lh a1, 6(sp)
+; RV32I-NEXT:    bnez a0, .LBB104_4
 ; RV32I-NEXT:  .LBB104_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    and a1, a0, s0
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bltu s1, a1, .LBB104_1
+; RV32I-NEXT:    and a0, a1, s0
+; RV32I-NEXT:    mv a2, a1
+; RV32I-NEXT:    bltu s1, a0, .LBB104_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB104_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:    j .LBB104_1
 ; RV32I-NEXT:  .LBB104_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    lw s4, 8(sp)
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
@@ -9368,28 +9368,28 @@ define i16 @atomicrmw_umax_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_umax_i16_seq_cst:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a6, a2, a3
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB104_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aqrl a4, (a0)
-; RV32IA-NEXT:    and a2, a4, a6
-; RV32IA-NEXT:    mv a5, a4
+; RV32IA-NEXT:    lr.w.aqrl a3, (a6)
+; RV32IA-NEXT:    and a2, a3, a4
+; RV32IA-NEXT:    mv a5, a3
 ; RV32IA-NEXT:    bgeu a2, a1, .LBB104_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB104_1 Depth=1
-; RV32IA-NEXT:    xor a5, a4, a1
-; RV32IA-NEXT:    and a5, a5, a6
-; RV32IA-NEXT:    xor a5, a4, a5
+; RV32IA-NEXT:    xor a5, a3, a1
+; RV32IA-NEXT:    and a5, a5, a4
+; RV32IA-NEXT:    xor a5, a3, a5
 ; RV32IA-NEXT:  .LBB104_3: # in Loop: Header=BB104_1 Depth=1
-; RV32IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV32IA-NEXT:    sc.w.aqrl a5, a5, (a6)
 ; RV32IA-NEXT:    bnez a5, .LBB104_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a4, a3
+; RV32IA-NEXT:    srl a0, a3, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_umax_i16_seq_cst:
@@ -9403,33 +9403,33 @@ define i16 @atomicrmw_umax_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    sd s4, 16(sp)
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s4, a0
-; RV64I-NEXT:    lhu a0, 0(a0)
-; RV64I-NEXT:    lui a1, 16
-; RV64I-NEXT:    addiw s0, a1, -1
+; RV64I-NEXT:    lhu a1, 0(a0)
+; RV64I-NEXT:    lui a0, 16
+; RV64I-NEXT:    addiw s0, a0, -1
 ; RV64I-NEXT:    and s1, s2, s0
 ; RV64I-NEXT:    addi s3, sp, 14
 ; RV64I-NEXT:    j .LBB104_2
 ; RV64I-NEXT:  .LBB104_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB104_2 Depth=1
-; RV64I-NEXT:    sh a0, 14(sp)
-; RV64I-NEXT:    mv a0, s4
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sh a1, 14(sp)
 ; RV64I-NEXT:    addi a3, zero, 5
 ; RV64I-NEXT:    addi a4, zero, 5
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
 ; RV64I-NEXT:    call __atomic_compare_exchange_2
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lh a0, 14(sp)
-; RV64I-NEXT:    bnez a1, .LBB104_4
+; RV64I-NEXT:    lh a1, 14(sp)
+; RV64I-NEXT:    bnez a0, .LBB104_4
 ; RV64I-NEXT:  .LBB104_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    and a1, a0, s0
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bltu s1, a1, .LBB104_1
+; RV64I-NEXT:    and a0, a1, s0
+; RV64I-NEXT:    mv a2, a1
+; RV64I-NEXT:    bltu s1, a0, .LBB104_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB104_2 Depth=1
 ; RV64I-NEXT:    mv a2, s2
 ; RV64I-NEXT:    j .LBB104_1
 ; RV64I-NEXT:  .LBB104_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ld s4, 16(sp)
 ; RV64I-NEXT:    ld s3, 24(sp)
 ; RV64I-NEXT:    ld s2, 32(sp)
@@ -9441,28 +9441,28 @@ define i16 @atomicrmw_umax_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_umax_i16_seq_cst:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a6, a2, a3
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB104_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aqrl a4, (a0)
-; RV64IA-NEXT:    and a2, a4, a6
-; RV64IA-NEXT:    mv a5, a4
+; RV64IA-NEXT:    lr.w.aqrl a3, (a6)
+; RV64IA-NEXT:    and a2, a3, a4
+; RV64IA-NEXT:    mv a5, a3
 ; RV64IA-NEXT:    bgeu a2, a1, .LBB104_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB104_1 Depth=1
-; RV64IA-NEXT:    xor a5, a4, a1
-; RV64IA-NEXT:    and a5, a5, a6
-; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    xor a5, a3, a1
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
 ; RV64IA-NEXT:  .LBB104_3: # in Loop: Header=BB104_1 Depth=1
-; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a6)
 ; RV64IA-NEXT:    bnez a5, .LBB104_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    srlw a0, a3, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw umax i16* %a, i16 %b seq_cst
   ret i16 %1
@@ -9480,33 +9480,33 @@ define i16 @atomicrmw_umin_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    sw s4, 8(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s4, a0
-; RV32I-NEXT:    lhu a0, 0(a0)
-; RV32I-NEXT:    lui a1, 16
-; RV32I-NEXT:    addi s0, a1, -1
+; RV32I-NEXT:    lhu a1, 0(a0)
+; RV32I-NEXT:    lui a0, 16
+; RV32I-NEXT:    addi s0, a0, -1
 ; RV32I-NEXT:    and s1, s2, s0
 ; RV32I-NEXT:    addi s3, sp, 6
 ; RV32I-NEXT:    j .LBB105_2
 ; RV32I-NEXT:  .LBB105_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB105_2 Depth=1
-; RV32I-NEXT:    sh a0, 6(sp)
+; RV32I-NEXT:    sh a1, 6(sp)
 ; RV32I-NEXT:    mv a0, s4
 ; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    mv a3, zero
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_2
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lh a0, 6(sp)
-; RV32I-NEXT:    bnez a1, .LBB105_4
+; RV32I-NEXT:    lh a1, 6(sp)
+; RV32I-NEXT:    bnez a0, .LBB105_4
 ; RV32I-NEXT:  .LBB105_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    and a1, a0, s0
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bgeu s1, a1, .LBB105_1
+; RV32I-NEXT:    and a0, a1, s0
+; RV32I-NEXT:    mv a2, a1
+; RV32I-NEXT:    bgeu s1, a0, .LBB105_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB105_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:    j .LBB105_1
 ; RV32I-NEXT:  .LBB105_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    lw s4, 8(sp)
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
@@ -9518,28 +9518,28 @@ define i16 @atomicrmw_umin_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_umin_i16_monotonic:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a6, a2, a3
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB105_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a4, (a0)
-; RV32IA-NEXT:    and a2, a4, a6
-; RV32IA-NEXT:    mv a5, a4
+; RV32IA-NEXT:    lr.w a3, (a6)
+; RV32IA-NEXT:    and a2, a3, a4
+; RV32IA-NEXT:    mv a5, a3
 ; RV32IA-NEXT:    bgeu a1, a2, .LBB105_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB105_1 Depth=1
-; RV32IA-NEXT:    xor a5, a4, a1
-; RV32IA-NEXT:    and a5, a5, a6
-; RV32IA-NEXT:    xor a5, a4, a5
+; RV32IA-NEXT:    xor a5, a3, a1
+; RV32IA-NEXT:    and a5, a5, a4
+; RV32IA-NEXT:    xor a5, a3, a5
 ; RV32IA-NEXT:  .LBB105_3: # in Loop: Header=BB105_1 Depth=1
-; RV32IA-NEXT:    sc.w a5, a5, (a0)
+; RV32IA-NEXT:    sc.w a5, a5, (a6)
 ; RV32IA-NEXT:    bnez a5, .LBB105_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a4, a3
+; RV32IA-NEXT:    srl a0, a3, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_umin_i16_monotonic:
@@ -9553,33 +9553,33 @@ define i16 @atomicrmw_umin_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    sd s4, 16(sp)
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s4, a0
-; RV64I-NEXT:    lhu a0, 0(a0)
-; RV64I-NEXT:    lui a1, 16
-; RV64I-NEXT:    addiw s0, a1, -1
+; RV64I-NEXT:    lhu a1, 0(a0)
+; RV64I-NEXT:    lui a0, 16
+; RV64I-NEXT:    addiw s0, a0, -1
 ; RV64I-NEXT:    and s1, s2, s0
 ; RV64I-NEXT:    addi s3, sp, 14
 ; RV64I-NEXT:    j .LBB105_2
 ; RV64I-NEXT:  .LBB105_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB105_2 Depth=1
-; RV64I-NEXT:    sh a0, 14(sp)
+; RV64I-NEXT:    sh a1, 14(sp)
 ; RV64I-NEXT:    mv a0, s4
 ; RV64I-NEXT:    mv a1, s3
 ; RV64I-NEXT:    mv a3, zero
 ; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    call __atomic_compare_exchange_2
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lh a0, 14(sp)
-; RV64I-NEXT:    bnez a1, .LBB105_4
+; RV64I-NEXT:    lh a1, 14(sp)
+; RV64I-NEXT:    bnez a0, .LBB105_4
 ; RV64I-NEXT:  .LBB105_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    and a1, a0, s0
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bgeu s1, a1, .LBB105_1
+; RV64I-NEXT:    and a0, a1, s0
+; RV64I-NEXT:    mv a2, a1
+; RV64I-NEXT:    bgeu s1, a0, .LBB105_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB105_2 Depth=1
 ; RV64I-NEXT:    mv a2, s2
 ; RV64I-NEXT:    j .LBB105_1
 ; RV64I-NEXT:  .LBB105_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ld s4, 16(sp)
 ; RV64I-NEXT:    ld s3, 24(sp)
 ; RV64I-NEXT:    ld s2, 32(sp)
@@ -9591,28 +9591,28 @@ define i16 @atomicrmw_umin_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_umin_i16_monotonic:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a6, a2, a3
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB105_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a4, (a0)
-; RV64IA-NEXT:    and a2, a4, a6
-; RV64IA-NEXT:    mv a5, a4
+; RV64IA-NEXT:    lr.w a3, (a6)
+; RV64IA-NEXT:    and a2, a3, a4
+; RV64IA-NEXT:    mv a5, a3
 ; RV64IA-NEXT:    bgeu a1, a2, .LBB105_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB105_1 Depth=1
-; RV64IA-NEXT:    xor a5, a4, a1
-; RV64IA-NEXT:    and a5, a5, a6
-; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    xor a5, a3, a1
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
 ; RV64IA-NEXT:  .LBB105_3: # in Loop: Header=BB105_1 Depth=1
-; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    sc.w a5, a5, (a6)
 ; RV64IA-NEXT:    bnez a5, .LBB105_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    srlw a0, a3, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw umin i16* %a, i16 %b monotonic
   ret i16 %1
@@ -9630,33 +9630,33 @@ define i16 @atomicrmw_umin_i16_acquire(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    sw s4, 8(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s4, a0
-; RV32I-NEXT:    lhu a0, 0(a0)
-; RV32I-NEXT:    lui a1, 16
-; RV32I-NEXT:    addi s0, a1, -1
+; RV32I-NEXT:    lhu a1, 0(a0)
+; RV32I-NEXT:    lui a0, 16
+; RV32I-NEXT:    addi s0, a0, -1
 ; RV32I-NEXT:    and s1, s2, s0
 ; RV32I-NEXT:    addi s3, sp, 6
 ; RV32I-NEXT:    j .LBB106_2
 ; RV32I-NEXT:  .LBB106_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB106_2 Depth=1
-; RV32I-NEXT:    sh a0, 6(sp)
-; RV32I-NEXT:    mv a0, s4
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sh a1, 6(sp)
 ; RV32I-NEXT:    addi a3, zero, 2
 ; RV32I-NEXT:    addi a4, zero, 2
+; RV32I-NEXT:    mv a0, s4
+; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    call __atomic_compare_exchange_2
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lh a0, 6(sp)
-; RV32I-NEXT:    bnez a1, .LBB106_4
+; RV32I-NEXT:    lh a1, 6(sp)
+; RV32I-NEXT:    bnez a0, .LBB106_4
 ; RV32I-NEXT:  .LBB106_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    and a1, a0, s0
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bgeu s1, a1, .LBB106_1
+; RV32I-NEXT:    and a0, a1, s0
+; RV32I-NEXT:    mv a2, a1
+; RV32I-NEXT:    bgeu s1, a0, .LBB106_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB106_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:    j .LBB106_1
 ; RV32I-NEXT:  .LBB106_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    lw s4, 8(sp)
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
@@ -9668,28 +9668,28 @@ define i16 @atomicrmw_umin_i16_acquire(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_umin_i16_acquire:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a6, a2, a3
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB106_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a4, (a0)
-; RV32IA-NEXT:    and a2, a4, a6
-; RV32IA-NEXT:    mv a5, a4
+; RV32IA-NEXT:    lr.w.aq a3, (a6)
+; RV32IA-NEXT:    and a2, a3, a4
+; RV32IA-NEXT:    mv a5, a3
 ; RV32IA-NEXT:    bgeu a1, a2, .LBB106_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB106_1 Depth=1
-; RV32IA-NEXT:    xor a5, a4, a1
-; RV32IA-NEXT:    and a5, a5, a6
-; RV32IA-NEXT:    xor a5, a4, a5
+; RV32IA-NEXT:    xor a5, a3, a1
+; RV32IA-NEXT:    and a5, a5, a4
+; RV32IA-NEXT:    xor a5, a3, a5
 ; RV32IA-NEXT:  .LBB106_3: # in Loop: Header=BB106_1 Depth=1
-; RV32IA-NEXT:    sc.w a5, a5, (a0)
+; RV32IA-NEXT:    sc.w a5, a5, (a6)
 ; RV32IA-NEXT:    bnez a5, .LBB106_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a4, a3
+; RV32IA-NEXT:    srl a0, a3, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_umin_i16_acquire:
@@ -9703,33 +9703,33 @@ define i16 @atomicrmw_umin_i16_acquire(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    sd s4, 16(sp)
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s4, a0
-; RV64I-NEXT:    lhu a0, 0(a0)
-; RV64I-NEXT:    lui a1, 16
-; RV64I-NEXT:    addiw s0, a1, -1
+; RV64I-NEXT:    lhu a1, 0(a0)
+; RV64I-NEXT:    lui a0, 16
+; RV64I-NEXT:    addiw s0, a0, -1
 ; RV64I-NEXT:    and s1, s2, s0
 ; RV64I-NEXT:    addi s3, sp, 14
 ; RV64I-NEXT:    j .LBB106_2
 ; RV64I-NEXT:  .LBB106_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB106_2 Depth=1
-; RV64I-NEXT:    sh a0, 14(sp)
-; RV64I-NEXT:    mv a0, s4
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sh a1, 14(sp)
 ; RV64I-NEXT:    addi a3, zero, 2
 ; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
 ; RV64I-NEXT:    call __atomic_compare_exchange_2
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lh a0, 14(sp)
-; RV64I-NEXT:    bnez a1, .LBB106_4
+; RV64I-NEXT:    lh a1, 14(sp)
+; RV64I-NEXT:    bnez a0, .LBB106_4
 ; RV64I-NEXT:  .LBB106_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    and a1, a0, s0
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bgeu s1, a1, .LBB106_1
+; RV64I-NEXT:    and a0, a1, s0
+; RV64I-NEXT:    mv a2, a1
+; RV64I-NEXT:    bgeu s1, a0, .LBB106_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB106_2 Depth=1
 ; RV64I-NEXT:    mv a2, s2
 ; RV64I-NEXT:    j .LBB106_1
 ; RV64I-NEXT:  .LBB106_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ld s4, 16(sp)
 ; RV64I-NEXT:    ld s3, 24(sp)
 ; RV64I-NEXT:    ld s2, 32(sp)
@@ -9741,28 +9741,28 @@ define i16 @atomicrmw_umin_i16_acquire(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_umin_i16_acquire:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a6, a2, a3
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB106_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a4, (a0)
-; RV64IA-NEXT:    and a2, a4, a6
-; RV64IA-NEXT:    mv a5, a4
+; RV64IA-NEXT:    lr.w.aq a3, (a6)
+; RV64IA-NEXT:    and a2, a3, a4
+; RV64IA-NEXT:    mv a5, a3
 ; RV64IA-NEXT:    bgeu a1, a2, .LBB106_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB106_1 Depth=1
-; RV64IA-NEXT:    xor a5, a4, a1
-; RV64IA-NEXT:    and a5, a5, a6
-; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    xor a5, a3, a1
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
 ; RV64IA-NEXT:  .LBB106_3: # in Loop: Header=BB106_1 Depth=1
-; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    sc.w a5, a5, (a6)
 ; RV64IA-NEXT:    bnez a5, .LBB106_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    srlw a0, a3, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw umin i16* %a, i16 %b acquire
   ret i16 %1
@@ -9780,33 +9780,33 @@ define i16 @atomicrmw_umin_i16_release(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    sw s4, 8(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s4, a0
-; RV32I-NEXT:    lhu a0, 0(a0)
-; RV32I-NEXT:    lui a1, 16
-; RV32I-NEXT:    addi s0, a1, -1
+; RV32I-NEXT:    lhu a1, 0(a0)
+; RV32I-NEXT:    lui a0, 16
+; RV32I-NEXT:    addi s0, a0, -1
 ; RV32I-NEXT:    and s1, s2, s0
 ; RV32I-NEXT:    addi s3, sp, 6
 ; RV32I-NEXT:    j .LBB107_2
 ; RV32I-NEXT:  .LBB107_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB107_2 Depth=1
-; RV32I-NEXT:    sh a0, 6(sp)
+; RV32I-NEXT:    sh a1, 6(sp)
+; RV32I-NEXT:    addi a3, zero, 3
 ; RV32I-NEXT:    mv a0, s4
 ; RV32I-NEXT:    mv a1, s3
-; RV32I-NEXT:    addi a3, zero, 3
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_2
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lh a0, 6(sp)
-; RV32I-NEXT:    bnez a1, .LBB107_4
+; RV32I-NEXT:    lh a1, 6(sp)
+; RV32I-NEXT:    bnez a0, .LBB107_4
 ; RV32I-NEXT:  .LBB107_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    and a1, a0, s0
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bgeu s1, a1, .LBB107_1
+; RV32I-NEXT:    and a0, a1, s0
+; RV32I-NEXT:    mv a2, a1
+; RV32I-NEXT:    bgeu s1, a0, .LBB107_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB107_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:    j .LBB107_1
 ; RV32I-NEXT:  .LBB107_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    lw s4, 8(sp)
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
@@ -9818,28 +9818,28 @@ define i16 @atomicrmw_umin_i16_release(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_umin_i16_release:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a6, a2, a3
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB107_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a4, (a0)
-; RV32IA-NEXT:    and a2, a4, a6
-; RV32IA-NEXT:    mv a5, a4
+; RV32IA-NEXT:    lr.w a3, (a6)
+; RV32IA-NEXT:    and a2, a3, a4
+; RV32IA-NEXT:    mv a5, a3
 ; RV32IA-NEXT:    bgeu a1, a2, .LBB107_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB107_1 Depth=1
-; RV32IA-NEXT:    xor a5, a4, a1
-; RV32IA-NEXT:    and a5, a5, a6
-; RV32IA-NEXT:    xor a5, a4, a5
+; RV32IA-NEXT:    xor a5, a3, a1
+; RV32IA-NEXT:    and a5, a5, a4
+; RV32IA-NEXT:    xor a5, a3, a5
 ; RV32IA-NEXT:  .LBB107_3: # in Loop: Header=BB107_1 Depth=1
-; RV32IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV32IA-NEXT:    sc.w.rl a5, a5, (a6)
 ; RV32IA-NEXT:    bnez a5, .LBB107_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a4, a3
+; RV32IA-NEXT:    srl a0, a3, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_umin_i16_release:
@@ -9853,33 +9853,33 @@ define i16 @atomicrmw_umin_i16_release(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    sd s4, 16(sp)
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s4, a0
-; RV64I-NEXT:    lhu a0, 0(a0)
-; RV64I-NEXT:    lui a1, 16
-; RV64I-NEXT:    addiw s0, a1, -1
+; RV64I-NEXT:    lhu a1, 0(a0)
+; RV64I-NEXT:    lui a0, 16
+; RV64I-NEXT:    addiw s0, a0, -1
 ; RV64I-NEXT:    and s1, s2, s0
 ; RV64I-NEXT:    addi s3, sp, 14
 ; RV64I-NEXT:    j .LBB107_2
 ; RV64I-NEXT:  .LBB107_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB107_2 Depth=1
-; RV64I-NEXT:    sh a0, 14(sp)
+; RV64I-NEXT:    sh a1, 14(sp)
+; RV64I-NEXT:    addi a3, zero, 3
 ; RV64I-NEXT:    mv a0, s4
 ; RV64I-NEXT:    mv a1, s3
-; RV64I-NEXT:    addi a3, zero, 3
 ; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    call __atomic_compare_exchange_2
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lh a0, 14(sp)
-; RV64I-NEXT:    bnez a1, .LBB107_4
+; RV64I-NEXT:    lh a1, 14(sp)
+; RV64I-NEXT:    bnez a0, .LBB107_4
 ; RV64I-NEXT:  .LBB107_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    and a1, a0, s0
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bgeu s1, a1, .LBB107_1
+; RV64I-NEXT:    and a0, a1, s0
+; RV64I-NEXT:    mv a2, a1
+; RV64I-NEXT:    bgeu s1, a0, .LBB107_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB107_2 Depth=1
 ; RV64I-NEXT:    mv a2, s2
 ; RV64I-NEXT:    j .LBB107_1
 ; RV64I-NEXT:  .LBB107_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ld s4, 16(sp)
 ; RV64I-NEXT:    ld s3, 24(sp)
 ; RV64I-NEXT:    ld s2, 32(sp)
@@ -9891,28 +9891,28 @@ define i16 @atomicrmw_umin_i16_release(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_umin_i16_release:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a6, a2, a3
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB107_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a4, (a0)
-; RV64IA-NEXT:    and a2, a4, a6
-; RV64IA-NEXT:    mv a5, a4
+; RV64IA-NEXT:    lr.w a3, (a6)
+; RV64IA-NEXT:    and a2, a3, a4
+; RV64IA-NEXT:    mv a5, a3
 ; RV64IA-NEXT:    bgeu a1, a2, .LBB107_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB107_1 Depth=1
-; RV64IA-NEXT:    xor a5, a4, a1
-; RV64IA-NEXT:    and a5, a5, a6
-; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    xor a5, a3, a1
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
 ; RV64IA-NEXT:  .LBB107_3: # in Loop: Header=BB107_1 Depth=1
-; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a6)
 ; RV64IA-NEXT:    bnez a5, .LBB107_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    srlw a0, a3, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw umin i16* %a, i16 %b release
   ret i16 %1
@@ -9930,33 +9930,33 @@ define i16 @atomicrmw_umin_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    sw s4, 8(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s4, a0
-; RV32I-NEXT:    lhu a0, 0(a0)
-; RV32I-NEXT:    lui a1, 16
-; RV32I-NEXT:    addi s0, a1, -1
+; RV32I-NEXT:    lhu a1, 0(a0)
+; RV32I-NEXT:    lui a0, 16
+; RV32I-NEXT:    addi s0, a0, -1
 ; RV32I-NEXT:    and s1, s2, s0
 ; RV32I-NEXT:    addi s3, sp, 6
 ; RV32I-NEXT:    j .LBB108_2
 ; RV32I-NEXT:  .LBB108_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB108_2 Depth=1
-; RV32I-NEXT:    sh a0, 6(sp)
-; RV32I-NEXT:    mv a0, s4
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sh a1, 6(sp)
 ; RV32I-NEXT:    addi a3, zero, 4
 ; RV32I-NEXT:    addi a4, zero, 2
+; RV32I-NEXT:    mv a0, s4
+; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    call __atomic_compare_exchange_2
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lh a0, 6(sp)
-; RV32I-NEXT:    bnez a1, .LBB108_4
+; RV32I-NEXT:    lh a1, 6(sp)
+; RV32I-NEXT:    bnez a0, .LBB108_4
 ; RV32I-NEXT:  .LBB108_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    and a1, a0, s0
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bgeu s1, a1, .LBB108_1
+; RV32I-NEXT:    and a0, a1, s0
+; RV32I-NEXT:    mv a2, a1
+; RV32I-NEXT:    bgeu s1, a0, .LBB108_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB108_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:    j .LBB108_1
 ; RV32I-NEXT:  .LBB108_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    lw s4, 8(sp)
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
@@ -9968,28 +9968,28 @@ define i16 @atomicrmw_umin_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_umin_i16_acq_rel:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a6, a2, a3
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB108_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a4, (a0)
-; RV32IA-NEXT:    and a2, a4, a6
-; RV32IA-NEXT:    mv a5, a4
+; RV32IA-NEXT:    lr.w.aq a3, (a6)
+; RV32IA-NEXT:    and a2, a3, a4
+; RV32IA-NEXT:    mv a5, a3
 ; RV32IA-NEXT:    bgeu a1, a2, .LBB108_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB108_1 Depth=1
-; RV32IA-NEXT:    xor a5, a4, a1
-; RV32IA-NEXT:    and a5, a5, a6
-; RV32IA-NEXT:    xor a5, a4, a5
+; RV32IA-NEXT:    xor a5, a3, a1
+; RV32IA-NEXT:    and a5, a5, a4
+; RV32IA-NEXT:    xor a5, a3, a5
 ; RV32IA-NEXT:  .LBB108_3: # in Loop: Header=BB108_1 Depth=1
-; RV32IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV32IA-NEXT:    sc.w.rl a5, a5, (a6)
 ; RV32IA-NEXT:    bnez a5, .LBB108_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a4, a3
+; RV32IA-NEXT:    srl a0, a3, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_umin_i16_acq_rel:
@@ -10003,33 +10003,33 @@ define i16 @atomicrmw_umin_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    sd s4, 16(sp)
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s4, a0
-; RV64I-NEXT:    lhu a0, 0(a0)
-; RV64I-NEXT:    lui a1, 16
-; RV64I-NEXT:    addiw s0, a1, -1
+; RV64I-NEXT:    lhu a1, 0(a0)
+; RV64I-NEXT:    lui a0, 16
+; RV64I-NEXT:    addiw s0, a0, -1
 ; RV64I-NEXT:    and s1, s2, s0
 ; RV64I-NEXT:    addi s3, sp, 14
 ; RV64I-NEXT:    j .LBB108_2
 ; RV64I-NEXT:  .LBB108_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB108_2 Depth=1
-; RV64I-NEXT:    sh a0, 14(sp)
-; RV64I-NEXT:    mv a0, s4
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sh a1, 14(sp)
 ; RV64I-NEXT:    addi a3, zero, 4
 ; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
 ; RV64I-NEXT:    call __atomic_compare_exchange_2
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lh a0, 14(sp)
-; RV64I-NEXT:    bnez a1, .LBB108_4
+; RV64I-NEXT:    lh a1, 14(sp)
+; RV64I-NEXT:    bnez a0, .LBB108_4
 ; RV64I-NEXT:  .LBB108_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    and a1, a0, s0
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bgeu s1, a1, .LBB108_1
+; RV64I-NEXT:    and a0, a1, s0
+; RV64I-NEXT:    mv a2, a1
+; RV64I-NEXT:    bgeu s1, a0, .LBB108_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB108_2 Depth=1
 ; RV64I-NEXT:    mv a2, s2
 ; RV64I-NEXT:    j .LBB108_1
 ; RV64I-NEXT:  .LBB108_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ld s4, 16(sp)
 ; RV64I-NEXT:    ld s3, 24(sp)
 ; RV64I-NEXT:    ld s2, 32(sp)
@@ -10041,28 +10041,28 @@ define i16 @atomicrmw_umin_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_umin_i16_acq_rel:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a6, a2, a3
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB108_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a4, (a0)
-; RV64IA-NEXT:    and a2, a4, a6
-; RV64IA-NEXT:    mv a5, a4
+; RV64IA-NEXT:    lr.w.aq a3, (a6)
+; RV64IA-NEXT:    and a2, a3, a4
+; RV64IA-NEXT:    mv a5, a3
 ; RV64IA-NEXT:    bgeu a1, a2, .LBB108_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB108_1 Depth=1
-; RV64IA-NEXT:    xor a5, a4, a1
-; RV64IA-NEXT:    and a5, a5, a6
-; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    xor a5, a3, a1
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
 ; RV64IA-NEXT:  .LBB108_3: # in Loop: Header=BB108_1 Depth=1
-; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a6)
 ; RV64IA-NEXT:    bnez a5, .LBB108_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    srlw a0, a3, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw umin i16* %a, i16 %b acq_rel
   ret i16 %1
@@ -10080,33 +10080,33 @@ define i16 @atomicrmw_umin_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    sw s4, 8(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s4, a0
-; RV32I-NEXT:    lhu a0, 0(a0)
-; RV32I-NEXT:    lui a1, 16
-; RV32I-NEXT:    addi s0, a1, -1
+; RV32I-NEXT:    lhu a1, 0(a0)
+; RV32I-NEXT:    lui a0, 16
+; RV32I-NEXT:    addi s0, a0, -1
 ; RV32I-NEXT:    and s1, s2, s0
 ; RV32I-NEXT:    addi s3, sp, 6
 ; RV32I-NEXT:    j .LBB109_2
 ; RV32I-NEXT:  .LBB109_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB109_2 Depth=1
-; RV32I-NEXT:    sh a0, 6(sp)
-; RV32I-NEXT:    mv a0, s4
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sh a1, 6(sp)
 ; RV32I-NEXT:    addi a3, zero, 5
 ; RV32I-NEXT:    addi a4, zero, 5
+; RV32I-NEXT:    mv a0, s4
+; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    call __atomic_compare_exchange_2
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lh a0, 6(sp)
-; RV32I-NEXT:    bnez a1, .LBB109_4
+; RV32I-NEXT:    lh a1, 6(sp)
+; RV32I-NEXT:    bnez a0, .LBB109_4
 ; RV32I-NEXT:  .LBB109_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    and a1, a0, s0
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bgeu s1, a1, .LBB109_1
+; RV32I-NEXT:    and a0, a1, s0
+; RV32I-NEXT:    mv a2, a1
+; RV32I-NEXT:    bgeu s1, a0, .LBB109_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB109_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:    j .LBB109_1
 ; RV32I-NEXT:  .LBB109_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    lw s4, 8(sp)
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
@@ -10118,28 +10118,28 @@ define i16 @atomicrmw_umin_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_umin_i16_seq_cst:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a6, a2, a3
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB109_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aqrl a4, (a0)
-; RV32IA-NEXT:    and a2, a4, a6
-; RV32IA-NEXT:    mv a5, a4
+; RV32IA-NEXT:    lr.w.aqrl a3, (a6)
+; RV32IA-NEXT:    and a2, a3, a4
+; RV32IA-NEXT:    mv a5, a3
 ; RV32IA-NEXT:    bgeu a1, a2, .LBB109_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB109_1 Depth=1
-; RV32IA-NEXT:    xor a5, a4, a1
-; RV32IA-NEXT:    and a5, a5, a6
-; RV32IA-NEXT:    xor a5, a4, a5
+; RV32IA-NEXT:    xor a5, a3, a1
+; RV32IA-NEXT:    and a5, a5, a4
+; RV32IA-NEXT:    xor a5, a3, a5
 ; RV32IA-NEXT:  .LBB109_3: # in Loop: Header=BB109_1 Depth=1
-; RV32IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV32IA-NEXT:    sc.w.aqrl a5, a5, (a6)
 ; RV32IA-NEXT:    bnez a5, .LBB109_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a4, a3
+; RV32IA-NEXT:    srl a0, a3, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_umin_i16_seq_cst:
@@ -10153,33 +10153,33 @@ define i16 @atomicrmw_umin_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    sd s4, 16(sp)
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s4, a0
-; RV64I-NEXT:    lhu a0, 0(a0)
-; RV64I-NEXT:    lui a1, 16
-; RV64I-NEXT:    addiw s0, a1, -1
+; RV64I-NEXT:    lhu a1, 0(a0)
+; RV64I-NEXT:    lui a0, 16
+; RV64I-NEXT:    addiw s0, a0, -1
 ; RV64I-NEXT:    and s1, s2, s0
 ; RV64I-NEXT:    addi s3, sp, 14
 ; RV64I-NEXT:    j .LBB109_2
 ; RV64I-NEXT:  .LBB109_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB109_2 Depth=1
-; RV64I-NEXT:    sh a0, 14(sp)
-; RV64I-NEXT:    mv a0, s4
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sh a1, 14(sp)
 ; RV64I-NEXT:    addi a3, zero, 5
 ; RV64I-NEXT:    addi a4, zero, 5
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
 ; RV64I-NEXT:    call __atomic_compare_exchange_2
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lh a0, 14(sp)
-; RV64I-NEXT:    bnez a1, .LBB109_4
+; RV64I-NEXT:    lh a1, 14(sp)
+; RV64I-NEXT:    bnez a0, .LBB109_4
 ; RV64I-NEXT:  .LBB109_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    and a1, a0, s0
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bgeu s1, a1, .LBB109_1
+; RV64I-NEXT:    and a0, a1, s0
+; RV64I-NEXT:    mv a2, a1
+; RV64I-NEXT:    bgeu s1, a0, .LBB109_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB109_2 Depth=1
 ; RV64I-NEXT:    mv a2, s2
 ; RV64I-NEXT:    j .LBB109_1
 ; RV64I-NEXT:  .LBB109_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ld s4, 16(sp)
 ; RV64I-NEXT:    ld s3, 24(sp)
 ; RV64I-NEXT:    ld s2, 32(sp)
@@ -10191,28 +10191,28 @@ define i16 @atomicrmw_umin_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_umin_i16_seq_cst:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a6, a2, a3
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB109_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aqrl a4, (a0)
-; RV64IA-NEXT:    and a2, a4, a6
-; RV64IA-NEXT:    mv a5, a4
+; RV64IA-NEXT:    lr.w.aqrl a3, (a6)
+; RV64IA-NEXT:    and a2, a3, a4
+; RV64IA-NEXT:    mv a5, a3
 ; RV64IA-NEXT:    bgeu a1, a2, .LBB109_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB109_1 Depth=1
-; RV64IA-NEXT:    xor a5, a4, a1
-; RV64IA-NEXT:    and a5, a5, a6
-; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    xor a5, a3, a1
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
 ; RV64IA-NEXT:  .LBB109_3: # in Loop: Header=BB109_1 Depth=1
-; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a6)
 ; RV64IA-NEXT:    bnez a5, .LBB109_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    srlw a0, a3, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw umin i16* %a, i16 %b seq_cst
   ret i16 %1
@@ -11496,30 +11496,31 @@ define i32 @atomicrmw_max_i32_monotonic(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    sw s0, 24(sp)
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
-; RV32I-NEXT:    mv s0, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a2, 0(a0)
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a3, 0(a0)
+; RV32I-NEXT:    mv s1, a1
 ; RV32I-NEXT:    addi s2, sp, 12
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    bge s0, a2, .LBB145_3
+; RV32I-NEXT:    j .LBB145_2
 ; RV32I-NEXT:  .LBB145_1: # %atomicrmw.start
-; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    mv a0, s1
+; RV32I-NEXT:    # in Loop: Header=BB145_2 Depth=1
+; RV32I-NEXT:    sw a3, 12(sp)
+; RV32I-NEXT:    mv a0, s0
 ; RV32I-NEXT:    mv a1, s2
 ; RV32I-NEXT:    mv a3, zero
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_4
-; RV32I-NEXT:    lw a2, 12(sp)
+; RV32I-NEXT:    lw a3, 12(sp)
 ; RV32I-NEXT:    bnez a0, .LBB145_4
-; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32I-NEXT:    # in Loop: Header=BB145_1 Depth=1
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    blt s0, a2, .LBB145_1
-; RV32I-NEXT:  .LBB145_3: # %atomicrmw.start
-; RV32I-NEXT:    mv a2, s0
+; RV32I-NEXT:  .LBB145_2: # %atomicrmw.start
+; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    blt s1, a3, .LBB145_1
+; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB145_2 Depth=1
+; RV32I-NEXT:    mv a2, s1
 ; RV32I-NEXT:    j .LBB145_1
 ; RV32I-NEXT:  .LBB145_4: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a3
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
 ; RV32I-NEXT:    lw s0, 24(sp)
@@ -11540,33 +11541,33 @@ define i32 @atomicrmw_max_i32_monotonic(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
 ; RV64I-NEXT:    sd s3, 8(sp)
-; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lwu a0, 0(a0)
-; RV64I-NEXT:    sext.w s0, a1
-; RV64I-NEXT:    addi s3, sp, 4
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    lwu a3, 0(a0)
+; RV64I-NEXT:    mv s3, a1
+; RV64I-NEXT:    sext.w s1, a1
+; RV64I-NEXT:    addi s2, sp, 4
 ; RV64I-NEXT:    j .LBB145_2
 ; RV64I-NEXT:  .LBB145_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB145_2 Depth=1
-; RV64I-NEXT:    sw a0, 4(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sw a3, 4(sp)
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    mv a3, zero
 ; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    call __atomic_compare_exchange_4
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lw a0, 4(sp)
-; RV64I-NEXT:    bnez a1, .LBB145_4
+; RV64I-NEXT:    lw a3, 4(sp)
+; RV64I-NEXT:    bnez a0, .LBB145_4
 ; RV64I-NEXT:  .LBB145_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    sext.w a1, a0
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    blt s0, a1, .LBB145_1
+; RV64I-NEXT:    sext.w a0, a3
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    blt s1, a0, .LBB145_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB145_2 Depth=1
-; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:    mv a2, s3
 ; RV64I-NEXT:    j .LBB145_1
 ; RV64I-NEXT:  .LBB145_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -11591,30 +11592,31 @@ define i32 @atomicrmw_max_i32_acquire(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    sw s0, 24(sp)
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
-; RV32I-NEXT:    mv s0, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a2, 0(a0)
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a3, 0(a0)
+; RV32I-NEXT:    mv s1, a1
 ; RV32I-NEXT:    addi s2, sp, 12
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    bge s0, a2, .LBB146_3
+; RV32I-NEXT:    j .LBB146_2
 ; RV32I-NEXT:  .LBB146_1: # %atomicrmw.start
-; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s2
+; RV32I-NEXT:    # in Loop: Header=BB146_2 Depth=1
+; RV32I-NEXT:    sw a3, 12(sp)
 ; RV32I-NEXT:    addi a3, zero, 2
 ; RV32I-NEXT:    addi a4, zero, 2
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s2
 ; RV32I-NEXT:    call __atomic_compare_exchange_4
-; RV32I-NEXT:    lw a2, 12(sp)
+; RV32I-NEXT:    lw a3, 12(sp)
 ; RV32I-NEXT:    bnez a0, .LBB146_4
-; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32I-NEXT:    # in Loop: Header=BB146_1 Depth=1
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    blt s0, a2, .LBB146_1
-; RV32I-NEXT:  .LBB146_3: # %atomicrmw.start
-; RV32I-NEXT:    mv a2, s0
+; RV32I-NEXT:  .LBB146_2: # %atomicrmw.start
+; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    blt s1, a3, .LBB146_1
+; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB146_2 Depth=1
+; RV32I-NEXT:    mv a2, s1
 ; RV32I-NEXT:    j .LBB146_1
 ; RV32I-NEXT:  .LBB146_4: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a3
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
 ; RV32I-NEXT:    lw s0, 24(sp)
@@ -11635,33 +11637,33 @@ define i32 @atomicrmw_max_i32_acquire(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
 ; RV64I-NEXT:    sd s3, 8(sp)
-; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lwu a0, 0(a0)
-; RV64I-NEXT:    sext.w s0, a1
-; RV64I-NEXT:    addi s3, sp, 4
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    lwu a3, 0(a0)
+; RV64I-NEXT:    mv s3, a1
+; RV64I-NEXT:    sext.w s1, a1
+; RV64I-NEXT:    addi s2, sp, 4
 ; RV64I-NEXT:    j .LBB146_2
 ; RV64I-NEXT:  .LBB146_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB146_2 Depth=1
-; RV64I-NEXT:    sw a0, 4(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sw a3, 4(sp)
 ; RV64I-NEXT:    addi a3, zero, 2
 ; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    call __atomic_compare_exchange_4
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lw a0, 4(sp)
-; RV64I-NEXT:    bnez a1, .LBB146_4
+; RV64I-NEXT:    lw a3, 4(sp)
+; RV64I-NEXT:    bnez a0, .LBB146_4
 ; RV64I-NEXT:  .LBB146_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    sext.w a1, a0
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    blt s0, a1, .LBB146_1
+; RV64I-NEXT:    sext.w a0, a3
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    blt s1, a0, .LBB146_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB146_2 Depth=1
-; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:    mv a2, s3
 ; RV64I-NEXT:    j .LBB146_1
 ; RV64I-NEXT:  .LBB146_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -11686,30 +11688,31 @@ define i32 @atomicrmw_max_i32_release(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    sw s0, 24(sp)
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
-; RV32I-NEXT:    mv s0, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a2, 0(a0)
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a3, 0(a0)
+; RV32I-NEXT:    mv s1, a1
 ; RV32I-NEXT:    addi s2, sp, 12
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    bge s0, a2, .LBB147_3
+; RV32I-NEXT:    j .LBB147_2
 ; RV32I-NEXT:  .LBB147_1: # %atomicrmw.start
-; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s2
+; RV32I-NEXT:    # in Loop: Header=BB147_2 Depth=1
+; RV32I-NEXT:    sw a3, 12(sp)
 ; RV32I-NEXT:    addi a3, zero, 3
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s2
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_4
-; RV32I-NEXT:    lw a2, 12(sp)
+; RV32I-NEXT:    lw a3, 12(sp)
 ; RV32I-NEXT:    bnez a0, .LBB147_4
-; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32I-NEXT:    # in Loop: Header=BB147_1 Depth=1
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    blt s0, a2, .LBB147_1
-; RV32I-NEXT:  .LBB147_3: # %atomicrmw.start
-; RV32I-NEXT:    mv a2, s0
+; RV32I-NEXT:  .LBB147_2: # %atomicrmw.start
+; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    blt s1, a3, .LBB147_1
+; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB147_2 Depth=1
+; RV32I-NEXT:    mv a2, s1
 ; RV32I-NEXT:    j .LBB147_1
 ; RV32I-NEXT:  .LBB147_4: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a3
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
 ; RV32I-NEXT:    lw s0, 24(sp)
@@ -11730,33 +11733,33 @@ define i32 @atomicrmw_max_i32_release(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
 ; RV64I-NEXT:    sd s3, 8(sp)
-; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lwu a0, 0(a0)
-; RV64I-NEXT:    sext.w s0, a1
-; RV64I-NEXT:    addi s3, sp, 4
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    lwu a3, 0(a0)
+; RV64I-NEXT:    mv s3, a1
+; RV64I-NEXT:    sext.w s1, a1
+; RV64I-NEXT:    addi s2, sp, 4
 ; RV64I-NEXT:    j .LBB147_2
 ; RV64I-NEXT:  .LBB147_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB147_2 Depth=1
-; RV64I-NEXT:    sw a0, 4(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sw a3, 4(sp)
 ; RV64I-NEXT:    addi a3, zero, 3
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    call __atomic_compare_exchange_4
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lw a0, 4(sp)
-; RV64I-NEXT:    bnez a1, .LBB147_4
+; RV64I-NEXT:    lw a3, 4(sp)
+; RV64I-NEXT:    bnez a0, .LBB147_4
 ; RV64I-NEXT:  .LBB147_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    sext.w a1, a0
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    blt s0, a1, .LBB147_1
+; RV64I-NEXT:    sext.w a0, a3
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    blt s1, a0, .LBB147_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB147_2 Depth=1
-; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:    mv a2, s3
 ; RV64I-NEXT:    j .LBB147_1
 ; RV64I-NEXT:  .LBB147_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -11781,30 +11784,31 @@ define i32 @atomicrmw_max_i32_acq_rel(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    sw s0, 24(sp)
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
-; RV32I-NEXT:    mv s0, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a2, 0(a0)
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a3, 0(a0)
+; RV32I-NEXT:    mv s1, a1
 ; RV32I-NEXT:    addi s2, sp, 12
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    bge s0, a2, .LBB148_3
+; RV32I-NEXT:    j .LBB148_2
 ; RV32I-NEXT:  .LBB148_1: # %atomicrmw.start
-; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s2
+; RV32I-NEXT:    # in Loop: Header=BB148_2 Depth=1
+; RV32I-NEXT:    sw a3, 12(sp)
 ; RV32I-NEXT:    addi a3, zero, 4
 ; RV32I-NEXT:    addi a4, zero, 2
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s2
 ; RV32I-NEXT:    call __atomic_compare_exchange_4
-; RV32I-NEXT:    lw a2, 12(sp)
+; RV32I-NEXT:    lw a3, 12(sp)
 ; RV32I-NEXT:    bnez a0, .LBB148_4
-; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32I-NEXT:    # in Loop: Header=BB148_1 Depth=1
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    blt s0, a2, .LBB148_1
-; RV32I-NEXT:  .LBB148_3: # %atomicrmw.start
-; RV32I-NEXT:    mv a2, s0
+; RV32I-NEXT:  .LBB148_2: # %atomicrmw.start
+; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    blt s1, a3, .LBB148_1
+; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB148_2 Depth=1
+; RV32I-NEXT:    mv a2, s1
 ; RV32I-NEXT:    j .LBB148_1
 ; RV32I-NEXT:  .LBB148_4: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a3
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
 ; RV32I-NEXT:    lw s0, 24(sp)
@@ -11825,33 +11829,33 @@ define i32 @atomicrmw_max_i32_acq_rel(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
 ; RV64I-NEXT:    sd s3, 8(sp)
-; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lwu a0, 0(a0)
-; RV64I-NEXT:    sext.w s0, a1
-; RV64I-NEXT:    addi s3, sp, 4
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    lwu a3, 0(a0)
+; RV64I-NEXT:    mv s3, a1
+; RV64I-NEXT:    sext.w s1, a1
+; RV64I-NEXT:    addi s2, sp, 4
 ; RV64I-NEXT:    j .LBB148_2
 ; RV64I-NEXT:  .LBB148_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB148_2 Depth=1
-; RV64I-NEXT:    sw a0, 4(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sw a3, 4(sp)
 ; RV64I-NEXT:    addi a3, zero, 4
 ; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    call __atomic_compare_exchange_4
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lw a0, 4(sp)
-; RV64I-NEXT:    bnez a1, .LBB148_4
+; RV64I-NEXT:    lw a3, 4(sp)
+; RV64I-NEXT:    bnez a0, .LBB148_4
 ; RV64I-NEXT:  .LBB148_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    sext.w a1, a0
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    blt s0, a1, .LBB148_1
+; RV64I-NEXT:    sext.w a0, a3
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    blt s1, a0, .LBB148_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB148_2 Depth=1
-; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:    mv a2, s3
 ; RV64I-NEXT:    j .LBB148_1
 ; RV64I-NEXT:  .LBB148_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -11876,30 +11880,31 @@ define i32 @atomicrmw_max_i32_seq_cst(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    sw s0, 24(sp)
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
-; RV32I-NEXT:    mv s0, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a2, 0(a0)
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a3, 0(a0)
+; RV32I-NEXT:    mv s1, a1
 ; RV32I-NEXT:    addi s2, sp, 12
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    bge s0, a2, .LBB149_3
+; RV32I-NEXT:    j .LBB149_2
 ; RV32I-NEXT:  .LBB149_1: # %atomicrmw.start
-; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s2
+; RV32I-NEXT:    # in Loop: Header=BB149_2 Depth=1
+; RV32I-NEXT:    sw a3, 12(sp)
 ; RV32I-NEXT:    addi a3, zero, 5
 ; RV32I-NEXT:    addi a4, zero, 5
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s2
 ; RV32I-NEXT:    call __atomic_compare_exchange_4
-; RV32I-NEXT:    lw a2, 12(sp)
+; RV32I-NEXT:    lw a3, 12(sp)
 ; RV32I-NEXT:    bnez a0, .LBB149_4
-; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32I-NEXT:    # in Loop: Header=BB149_1 Depth=1
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    blt s0, a2, .LBB149_1
-; RV32I-NEXT:  .LBB149_3: # %atomicrmw.start
-; RV32I-NEXT:    mv a2, s0
+; RV32I-NEXT:  .LBB149_2: # %atomicrmw.start
+; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    blt s1, a3, .LBB149_1
+; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB149_2 Depth=1
+; RV32I-NEXT:    mv a2, s1
 ; RV32I-NEXT:    j .LBB149_1
 ; RV32I-NEXT:  .LBB149_4: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a3
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
 ; RV32I-NEXT:    lw s0, 24(sp)
@@ -11920,33 +11925,33 @@ define i32 @atomicrmw_max_i32_seq_cst(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
 ; RV64I-NEXT:    sd s3, 8(sp)
-; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lwu a0, 0(a0)
-; RV64I-NEXT:    sext.w s0, a1
-; RV64I-NEXT:    addi s3, sp, 4
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    lwu a3, 0(a0)
+; RV64I-NEXT:    mv s3, a1
+; RV64I-NEXT:    sext.w s1, a1
+; RV64I-NEXT:    addi s2, sp, 4
 ; RV64I-NEXT:    j .LBB149_2
 ; RV64I-NEXT:  .LBB149_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB149_2 Depth=1
-; RV64I-NEXT:    sw a0, 4(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sw a3, 4(sp)
 ; RV64I-NEXT:    addi a3, zero, 5
 ; RV64I-NEXT:    addi a4, zero, 5
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    call __atomic_compare_exchange_4
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lw a0, 4(sp)
-; RV64I-NEXT:    bnez a1, .LBB149_4
+; RV64I-NEXT:    lw a3, 4(sp)
+; RV64I-NEXT:    bnez a0, .LBB149_4
 ; RV64I-NEXT:  .LBB149_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    sext.w a1, a0
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    blt s0, a1, .LBB149_1
+; RV64I-NEXT:    sext.w a0, a3
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    blt s1, a0, .LBB149_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB149_2 Depth=1
-; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:    mv a2, s3
 ; RV64I-NEXT:    j .LBB149_1
 ; RV64I-NEXT:  .LBB149_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -11971,30 +11976,31 @@ define i32 @atomicrmw_min_i32_monotonic(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    sw s0, 24(sp)
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
-; RV32I-NEXT:    mv s0, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a2, 0(a0)
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a3, 0(a0)
+; RV32I-NEXT:    mv s1, a1
 ; RV32I-NEXT:    addi s2, sp, 12
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    blt s0, a2, .LBB150_3
+; RV32I-NEXT:    j .LBB150_2
 ; RV32I-NEXT:  .LBB150_1: # %atomicrmw.start
-; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    mv a0, s1
+; RV32I-NEXT:    # in Loop: Header=BB150_2 Depth=1
+; RV32I-NEXT:    sw a3, 12(sp)
+; RV32I-NEXT:    mv a0, s0
 ; RV32I-NEXT:    mv a1, s2
 ; RV32I-NEXT:    mv a3, zero
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_4
-; RV32I-NEXT:    lw a2, 12(sp)
+; RV32I-NEXT:    lw a3, 12(sp)
 ; RV32I-NEXT:    bnez a0, .LBB150_4
-; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32I-NEXT:    # in Loop: Header=BB150_1 Depth=1
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    bge s0, a2, .LBB150_1
-; RV32I-NEXT:  .LBB150_3: # %atomicrmw.start
-; RV32I-NEXT:    mv a2, s0
+; RV32I-NEXT:  .LBB150_2: # %atomicrmw.start
+; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    bge s1, a3, .LBB150_1
+; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB150_2 Depth=1
+; RV32I-NEXT:    mv a2, s1
 ; RV32I-NEXT:    j .LBB150_1
 ; RV32I-NEXT:  .LBB150_4: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a3
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
 ; RV32I-NEXT:    lw s0, 24(sp)
@@ -12015,33 +12021,33 @@ define i32 @atomicrmw_min_i32_monotonic(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
 ; RV64I-NEXT:    sd s3, 8(sp)
-; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lwu a0, 0(a0)
-; RV64I-NEXT:    sext.w s0, a1
-; RV64I-NEXT:    addi s3, sp, 4
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    lwu a3, 0(a0)
+; RV64I-NEXT:    mv s3, a1
+; RV64I-NEXT:    sext.w s1, a1
+; RV64I-NEXT:    addi s2, sp, 4
 ; RV64I-NEXT:    j .LBB150_2
 ; RV64I-NEXT:  .LBB150_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB150_2 Depth=1
-; RV64I-NEXT:    sw a0, 4(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sw a3, 4(sp)
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    mv a3, zero
 ; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    call __atomic_compare_exchange_4
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lw a0, 4(sp)
-; RV64I-NEXT:    bnez a1, .LBB150_4
+; RV64I-NEXT:    lw a3, 4(sp)
+; RV64I-NEXT:    bnez a0, .LBB150_4
 ; RV64I-NEXT:  .LBB150_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    sext.w a1, a0
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bge s0, a1, .LBB150_1
+; RV64I-NEXT:    sext.w a0, a3
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bge s1, a0, .LBB150_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB150_2 Depth=1
-; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:    mv a2, s3
 ; RV64I-NEXT:    j .LBB150_1
 ; RV64I-NEXT:  .LBB150_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -12066,30 +12072,31 @@ define i32 @atomicrmw_min_i32_acquire(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    sw s0, 24(sp)
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
-; RV32I-NEXT:    mv s0, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a2, 0(a0)
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a3, 0(a0)
+; RV32I-NEXT:    mv s1, a1
 ; RV32I-NEXT:    addi s2, sp, 12
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    blt s0, a2, .LBB151_3
+; RV32I-NEXT:    j .LBB151_2
 ; RV32I-NEXT:  .LBB151_1: # %atomicrmw.start
-; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s2
+; RV32I-NEXT:    # in Loop: Header=BB151_2 Depth=1
+; RV32I-NEXT:    sw a3, 12(sp)
 ; RV32I-NEXT:    addi a3, zero, 2
 ; RV32I-NEXT:    addi a4, zero, 2
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s2
 ; RV32I-NEXT:    call __atomic_compare_exchange_4
-; RV32I-NEXT:    lw a2, 12(sp)
+; RV32I-NEXT:    lw a3, 12(sp)
 ; RV32I-NEXT:    bnez a0, .LBB151_4
-; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32I-NEXT:    # in Loop: Header=BB151_1 Depth=1
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    bge s0, a2, .LBB151_1
-; RV32I-NEXT:  .LBB151_3: # %atomicrmw.start
-; RV32I-NEXT:    mv a2, s0
+; RV32I-NEXT:  .LBB151_2: # %atomicrmw.start
+; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    bge s1, a3, .LBB151_1
+; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB151_2 Depth=1
+; RV32I-NEXT:    mv a2, s1
 ; RV32I-NEXT:    j .LBB151_1
 ; RV32I-NEXT:  .LBB151_4: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a3
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
 ; RV32I-NEXT:    lw s0, 24(sp)
@@ -12110,33 +12117,33 @@ define i32 @atomicrmw_min_i32_acquire(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
 ; RV64I-NEXT:    sd s3, 8(sp)
-; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lwu a0, 0(a0)
-; RV64I-NEXT:    sext.w s0, a1
-; RV64I-NEXT:    addi s3, sp, 4
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    lwu a3, 0(a0)
+; RV64I-NEXT:    mv s3, a1
+; RV64I-NEXT:    sext.w s1, a1
+; RV64I-NEXT:    addi s2, sp, 4
 ; RV64I-NEXT:    j .LBB151_2
 ; RV64I-NEXT:  .LBB151_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB151_2 Depth=1
-; RV64I-NEXT:    sw a0, 4(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sw a3, 4(sp)
 ; RV64I-NEXT:    addi a3, zero, 2
 ; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    call __atomic_compare_exchange_4
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lw a0, 4(sp)
-; RV64I-NEXT:    bnez a1, .LBB151_4
+; RV64I-NEXT:    lw a3, 4(sp)
+; RV64I-NEXT:    bnez a0, .LBB151_4
 ; RV64I-NEXT:  .LBB151_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    sext.w a1, a0
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bge s0, a1, .LBB151_1
+; RV64I-NEXT:    sext.w a0, a3
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bge s1, a0, .LBB151_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB151_2 Depth=1
-; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:    mv a2, s3
 ; RV64I-NEXT:    j .LBB151_1
 ; RV64I-NEXT:  .LBB151_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -12161,30 +12168,31 @@ define i32 @atomicrmw_min_i32_release(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    sw s0, 24(sp)
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
-; RV32I-NEXT:    mv s0, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a2, 0(a0)
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a3, 0(a0)
+; RV32I-NEXT:    mv s1, a1
 ; RV32I-NEXT:    addi s2, sp, 12
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    blt s0, a2, .LBB152_3
+; RV32I-NEXT:    j .LBB152_2
 ; RV32I-NEXT:  .LBB152_1: # %atomicrmw.start
-; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s2
+; RV32I-NEXT:    # in Loop: Header=BB152_2 Depth=1
+; RV32I-NEXT:    sw a3, 12(sp)
 ; RV32I-NEXT:    addi a3, zero, 3
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s2
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_4
-; RV32I-NEXT:    lw a2, 12(sp)
+; RV32I-NEXT:    lw a3, 12(sp)
 ; RV32I-NEXT:    bnez a0, .LBB152_4
-; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32I-NEXT:    # in Loop: Header=BB152_1 Depth=1
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    bge s0, a2, .LBB152_1
-; RV32I-NEXT:  .LBB152_3: # %atomicrmw.start
-; RV32I-NEXT:    mv a2, s0
+; RV32I-NEXT:  .LBB152_2: # %atomicrmw.start
+; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    bge s1, a3, .LBB152_1
+; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB152_2 Depth=1
+; RV32I-NEXT:    mv a2, s1
 ; RV32I-NEXT:    j .LBB152_1
 ; RV32I-NEXT:  .LBB152_4: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a3
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
 ; RV32I-NEXT:    lw s0, 24(sp)
@@ -12205,33 +12213,33 @@ define i32 @atomicrmw_min_i32_release(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
 ; RV64I-NEXT:    sd s3, 8(sp)
-; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lwu a0, 0(a0)
-; RV64I-NEXT:    sext.w s0, a1
-; RV64I-NEXT:    addi s3, sp, 4
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    lwu a3, 0(a0)
+; RV64I-NEXT:    mv s3, a1
+; RV64I-NEXT:    sext.w s1, a1
+; RV64I-NEXT:    addi s2, sp, 4
 ; RV64I-NEXT:    j .LBB152_2
 ; RV64I-NEXT:  .LBB152_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB152_2 Depth=1
-; RV64I-NEXT:    sw a0, 4(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sw a3, 4(sp)
 ; RV64I-NEXT:    addi a3, zero, 3
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    call __atomic_compare_exchange_4
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lw a0, 4(sp)
-; RV64I-NEXT:    bnez a1, .LBB152_4
+; RV64I-NEXT:    lw a3, 4(sp)
+; RV64I-NEXT:    bnez a0, .LBB152_4
 ; RV64I-NEXT:  .LBB152_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    sext.w a1, a0
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bge s0, a1, .LBB152_1
+; RV64I-NEXT:    sext.w a0, a3
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bge s1, a0, .LBB152_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB152_2 Depth=1
-; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:    mv a2, s3
 ; RV64I-NEXT:    j .LBB152_1
 ; RV64I-NEXT:  .LBB152_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -12256,30 +12264,31 @@ define i32 @atomicrmw_min_i32_acq_rel(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    sw s0, 24(sp)
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
-; RV32I-NEXT:    mv s0, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a2, 0(a0)
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a3, 0(a0)
+; RV32I-NEXT:    mv s1, a1
 ; RV32I-NEXT:    addi s2, sp, 12
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    blt s0, a2, .LBB153_3
+; RV32I-NEXT:    j .LBB153_2
 ; RV32I-NEXT:  .LBB153_1: # %atomicrmw.start
-; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s2
+; RV32I-NEXT:    # in Loop: Header=BB153_2 Depth=1
+; RV32I-NEXT:    sw a3, 12(sp)
 ; RV32I-NEXT:    addi a3, zero, 4
 ; RV32I-NEXT:    addi a4, zero, 2
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s2
 ; RV32I-NEXT:    call __atomic_compare_exchange_4
-; RV32I-NEXT:    lw a2, 12(sp)
+; RV32I-NEXT:    lw a3, 12(sp)
 ; RV32I-NEXT:    bnez a0, .LBB153_4
-; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32I-NEXT:    # in Loop: Header=BB153_1 Depth=1
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    bge s0, a2, .LBB153_1
-; RV32I-NEXT:  .LBB153_3: # %atomicrmw.start
-; RV32I-NEXT:    mv a2, s0
+; RV32I-NEXT:  .LBB153_2: # %atomicrmw.start
+; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    bge s1, a3, .LBB153_1
+; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB153_2 Depth=1
+; RV32I-NEXT:    mv a2, s1
 ; RV32I-NEXT:    j .LBB153_1
 ; RV32I-NEXT:  .LBB153_4: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a3
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
 ; RV32I-NEXT:    lw s0, 24(sp)
@@ -12300,33 +12309,33 @@ define i32 @atomicrmw_min_i32_acq_rel(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
 ; RV64I-NEXT:    sd s3, 8(sp)
-; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lwu a0, 0(a0)
-; RV64I-NEXT:    sext.w s0, a1
-; RV64I-NEXT:    addi s3, sp, 4
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    lwu a3, 0(a0)
+; RV64I-NEXT:    mv s3, a1
+; RV64I-NEXT:    sext.w s1, a1
+; RV64I-NEXT:    addi s2, sp, 4
 ; RV64I-NEXT:    j .LBB153_2
 ; RV64I-NEXT:  .LBB153_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB153_2 Depth=1
-; RV64I-NEXT:    sw a0, 4(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sw a3, 4(sp)
 ; RV64I-NEXT:    addi a3, zero, 4
 ; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    call __atomic_compare_exchange_4
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lw a0, 4(sp)
-; RV64I-NEXT:    bnez a1, .LBB153_4
+; RV64I-NEXT:    lw a3, 4(sp)
+; RV64I-NEXT:    bnez a0, .LBB153_4
 ; RV64I-NEXT:  .LBB153_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    sext.w a1, a0
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bge s0, a1, .LBB153_1
+; RV64I-NEXT:    sext.w a0, a3
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bge s1, a0, .LBB153_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB153_2 Depth=1
-; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:    mv a2, s3
 ; RV64I-NEXT:    j .LBB153_1
 ; RV64I-NEXT:  .LBB153_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -12351,30 +12360,31 @@ define i32 @atomicrmw_min_i32_seq_cst(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    sw s0, 24(sp)
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
-; RV32I-NEXT:    mv s0, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a2, 0(a0)
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a3, 0(a0)
+; RV32I-NEXT:    mv s1, a1
 ; RV32I-NEXT:    addi s2, sp, 12
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    blt s0, a2, .LBB154_3
+; RV32I-NEXT:    j .LBB154_2
 ; RV32I-NEXT:  .LBB154_1: # %atomicrmw.start
-; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s2
+; RV32I-NEXT:    # in Loop: Header=BB154_2 Depth=1
+; RV32I-NEXT:    sw a3, 12(sp)
 ; RV32I-NEXT:    addi a3, zero, 5
 ; RV32I-NEXT:    addi a4, zero, 5
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s2
 ; RV32I-NEXT:    call __atomic_compare_exchange_4
-; RV32I-NEXT:    lw a2, 12(sp)
+; RV32I-NEXT:    lw a3, 12(sp)
 ; RV32I-NEXT:    bnez a0, .LBB154_4
-; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32I-NEXT:    # in Loop: Header=BB154_1 Depth=1
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    bge s0, a2, .LBB154_1
-; RV32I-NEXT:  .LBB154_3: # %atomicrmw.start
-; RV32I-NEXT:    mv a2, s0
+; RV32I-NEXT:  .LBB154_2: # %atomicrmw.start
+; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    bge s1, a3, .LBB154_1
+; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB154_2 Depth=1
+; RV32I-NEXT:    mv a2, s1
 ; RV32I-NEXT:    j .LBB154_1
 ; RV32I-NEXT:  .LBB154_4: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a3
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
 ; RV32I-NEXT:    lw s0, 24(sp)
@@ -12395,33 +12405,33 @@ define i32 @atomicrmw_min_i32_seq_cst(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
 ; RV64I-NEXT:    sd s3, 8(sp)
-; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lwu a0, 0(a0)
-; RV64I-NEXT:    sext.w s0, a1
-; RV64I-NEXT:    addi s3, sp, 4
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    lwu a3, 0(a0)
+; RV64I-NEXT:    mv s3, a1
+; RV64I-NEXT:    sext.w s1, a1
+; RV64I-NEXT:    addi s2, sp, 4
 ; RV64I-NEXT:    j .LBB154_2
 ; RV64I-NEXT:  .LBB154_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB154_2 Depth=1
-; RV64I-NEXT:    sw a0, 4(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sw a3, 4(sp)
 ; RV64I-NEXT:    addi a3, zero, 5
 ; RV64I-NEXT:    addi a4, zero, 5
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    call __atomic_compare_exchange_4
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lw a0, 4(sp)
-; RV64I-NEXT:    bnez a1, .LBB154_4
+; RV64I-NEXT:    lw a3, 4(sp)
+; RV64I-NEXT:    bnez a0, .LBB154_4
 ; RV64I-NEXT:  .LBB154_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    sext.w a1, a0
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bge s0, a1, .LBB154_1
+; RV64I-NEXT:    sext.w a0, a3
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bge s1, a0, .LBB154_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB154_2 Depth=1
-; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:    mv a2, s3
 ; RV64I-NEXT:    j .LBB154_1
 ; RV64I-NEXT:  .LBB154_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -12446,30 +12456,31 @@ define i32 @atomicrmw_umax_i32_monotonic(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    sw s0, 24(sp)
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
-; RV32I-NEXT:    mv s0, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a2, 0(a0)
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a3, 0(a0)
+; RV32I-NEXT:    mv s1, a1
 ; RV32I-NEXT:    addi s2, sp, 12
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    bgeu s0, a2, .LBB155_3
+; RV32I-NEXT:    j .LBB155_2
 ; RV32I-NEXT:  .LBB155_1: # %atomicrmw.start
-; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    mv a0, s1
+; RV32I-NEXT:    # in Loop: Header=BB155_2 Depth=1
+; RV32I-NEXT:    sw a3, 12(sp)
+; RV32I-NEXT:    mv a0, s0
 ; RV32I-NEXT:    mv a1, s2
 ; RV32I-NEXT:    mv a3, zero
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_4
-; RV32I-NEXT:    lw a2, 12(sp)
+; RV32I-NEXT:    lw a3, 12(sp)
 ; RV32I-NEXT:    bnez a0, .LBB155_4
-; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32I-NEXT:    # in Loop: Header=BB155_1 Depth=1
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    bltu s0, a2, .LBB155_1
-; RV32I-NEXT:  .LBB155_3: # %atomicrmw.start
-; RV32I-NEXT:    mv a2, s0
+; RV32I-NEXT:  .LBB155_2: # %atomicrmw.start
+; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    bltu s1, a3, .LBB155_1
+; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB155_2 Depth=1
+; RV32I-NEXT:    mv a2, s1
 ; RV32I-NEXT:    j .LBB155_1
 ; RV32I-NEXT:  .LBB155_4: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a3
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
 ; RV32I-NEXT:    lw s0, 24(sp)
@@ -12490,33 +12501,33 @@ define i32 @atomicrmw_umax_i32_monotonic(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
 ; RV64I-NEXT:    sd s3, 8(sp)
-; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lwu a0, 0(a0)
-; RV64I-NEXT:    sext.w s0, a1
-; RV64I-NEXT:    addi s3, sp, 4
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    lwu a3, 0(a0)
+; RV64I-NEXT:    mv s3, a1
+; RV64I-NEXT:    sext.w s1, a1
+; RV64I-NEXT:    addi s2, sp, 4
 ; RV64I-NEXT:    j .LBB155_2
 ; RV64I-NEXT:  .LBB155_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB155_2 Depth=1
-; RV64I-NEXT:    sw a0, 4(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sw a3, 4(sp)
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    mv a3, zero
 ; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    call __atomic_compare_exchange_4
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lw a0, 4(sp)
-; RV64I-NEXT:    bnez a1, .LBB155_4
+; RV64I-NEXT:    lw a3, 4(sp)
+; RV64I-NEXT:    bnez a0, .LBB155_4
 ; RV64I-NEXT:  .LBB155_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    sext.w a1, a0
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bltu s0, a1, .LBB155_1
+; RV64I-NEXT:    sext.w a0, a3
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bltu s1, a0, .LBB155_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB155_2 Depth=1
-; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:    mv a2, s3
 ; RV64I-NEXT:    j .LBB155_1
 ; RV64I-NEXT:  .LBB155_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -12541,30 +12552,31 @@ define i32 @atomicrmw_umax_i32_acquire(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    sw s0, 24(sp)
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
-; RV32I-NEXT:    mv s0, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a2, 0(a0)
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a3, 0(a0)
+; RV32I-NEXT:    mv s1, a1
 ; RV32I-NEXT:    addi s2, sp, 12
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    bgeu s0, a2, .LBB156_3
+; RV32I-NEXT:    j .LBB156_2
 ; RV32I-NEXT:  .LBB156_1: # %atomicrmw.start
-; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s2
+; RV32I-NEXT:    # in Loop: Header=BB156_2 Depth=1
+; RV32I-NEXT:    sw a3, 12(sp)
 ; RV32I-NEXT:    addi a3, zero, 2
 ; RV32I-NEXT:    addi a4, zero, 2
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s2
 ; RV32I-NEXT:    call __atomic_compare_exchange_4
-; RV32I-NEXT:    lw a2, 12(sp)
+; RV32I-NEXT:    lw a3, 12(sp)
 ; RV32I-NEXT:    bnez a0, .LBB156_4
-; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32I-NEXT:    # in Loop: Header=BB156_1 Depth=1
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    bltu s0, a2, .LBB156_1
-; RV32I-NEXT:  .LBB156_3: # %atomicrmw.start
-; RV32I-NEXT:    mv a2, s0
+; RV32I-NEXT:  .LBB156_2: # %atomicrmw.start
+; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    bltu s1, a3, .LBB156_1
+; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB156_2 Depth=1
+; RV32I-NEXT:    mv a2, s1
 ; RV32I-NEXT:    j .LBB156_1
 ; RV32I-NEXT:  .LBB156_4: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a3
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
 ; RV32I-NEXT:    lw s0, 24(sp)
@@ -12585,33 +12597,33 @@ define i32 @atomicrmw_umax_i32_acquire(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
 ; RV64I-NEXT:    sd s3, 8(sp)
-; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lwu a0, 0(a0)
-; RV64I-NEXT:    sext.w s0, a1
-; RV64I-NEXT:    addi s3, sp, 4
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    lwu a3, 0(a0)
+; RV64I-NEXT:    mv s3, a1
+; RV64I-NEXT:    sext.w s1, a1
+; RV64I-NEXT:    addi s2, sp, 4
 ; RV64I-NEXT:    j .LBB156_2
 ; RV64I-NEXT:  .LBB156_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB156_2 Depth=1
-; RV64I-NEXT:    sw a0, 4(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sw a3, 4(sp)
 ; RV64I-NEXT:    addi a3, zero, 2
 ; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    call __atomic_compare_exchange_4
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lw a0, 4(sp)
-; RV64I-NEXT:    bnez a1, .LBB156_4
+; RV64I-NEXT:    lw a3, 4(sp)
+; RV64I-NEXT:    bnez a0, .LBB156_4
 ; RV64I-NEXT:  .LBB156_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    sext.w a1, a0
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bltu s0, a1, .LBB156_1
+; RV64I-NEXT:    sext.w a0, a3
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bltu s1, a0, .LBB156_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB156_2 Depth=1
-; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:    mv a2, s3
 ; RV64I-NEXT:    j .LBB156_1
 ; RV64I-NEXT:  .LBB156_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -12636,30 +12648,31 @@ define i32 @atomicrmw_umax_i32_release(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    sw s0, 24(sp)
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
-; RV32I-NEXT:    mv s0, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a2, 0(a0)
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a3, 0(a0)
+; RV32I-NEXT:    mv s1, a1
 ; RV32I-NEXT:    addi s2, sp, 12
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    bgeu s0, a2, .LBB157_3
+; RV32I-NEXT:    j .LBB157_2
 ; RV32I-NEXT:  .LBB157_1: # %atomicrmw.start
-; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s2
+; RV32I-NEXT:    # in Loop: Header=BB157_2 Depth=1
+; RV32I-NEXT:    sw a3, 12(sp)
 ; RV32I-NEXT:    addi a3, zero, 3
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s2
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_4
-; RV32I-NEXT:    lw a2, 12(sp)
+; RV32I-NEXT:    lw a3, 12(sp)
 ; RV32I-NEXT:    bnez a0, .LBB157_4
-; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32I-NEXT:    # in Loop: Header=BB157_1 Depth=1
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    bltu s0, a2, .LBB157_1
-; RV32I-NEXT:  .LBB157_3: # %atomicrmw.start
-; RV32I-NEXT:    mv a2, s0
+; RV32I-NEXT:  .LBB157_2: # %atomicrmw.start
+; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    bltu s1, a3, .LBB157_1
+; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB157_2 Depth=1
+; RV32I-NEXT:    mv a2, s1
 ; RV32I-NEXT:    j .LBB157_1
 ; RV32I-NEXT:  .LBB157_4: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a3
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
 ; RV32I-NEXT:    lw s0, 24(sp)
@@ -12680,33 +12693,33 @@ define i32 @atomicrmw_umax_i32_release(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
 ; RV64I-NEXT:    sd s3, 8(sp)
-; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lwu a0, 0(a0)
-; RV64I-NEXT:    sext.w s0, a1
-; RV64I-NEXT:    addi s3, sp, 4
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    lwu a3, 0(a0)
+; RV64I-NEXT:    mv s3, a1
+; RV64I-NEXT:    sext.w s1, a1
+; RV64I-NEXT:    addi s2, sp, 4
 ; RV64I-NEXT:    j .LBB157_2
 ; RV64I-NEXT:  .LBB157_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB157_2 Depth=1
-; RV64I-NEXT:    sw a0, 4(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sw a3, 4(sp)
 ; RV64I-NEXT:    addi a3, zero, 3
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    call __atomic_compare_exchange_4
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lw a0, 4(sp)
-; RV64I-NEXT:    bnez a1, .LBB157_4
+; RV64I-NEXT:    lw a3, 4(sp)
+; RV64I-NEXT:    bnez a0, .LBB157_4
 ; RV64I-NEXT:  .LBB157_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    sext.w a1, a0
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bltu s0, a1, .LBB157_1
+; RV64I-NEXT:    sext.w a0, a3
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bltu s1, a0, .LBB157_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB157_2 Depth=1
-; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:    mv a2, s3
 ; RV64I-NEXT:    j .LBB157_1
 ; RV64I-NEXT:  .LBB157_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -12731,30 +12744,31 @@ define i32 @atomicrmw_umax_i32_acq_rel(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    sw s0, 24(sp)
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
-; RV32I-NEXT:    mv s0, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a2, 0(a0)
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a3, 0(a0)
+; RV32I-NEXT:    mv s1, a1
 ; RV32I-NEXT:    addi s2, sp, 12
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    bgeu s0, a2, .LBB158_3
+; RV32I-NEXT:    j .LBB158_2
 ; RV32I-NEXT:  .LBB158_1: # %atomicrmw.start
-; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s2
+; RV32I-NEXT:    # in Loop: Header=BB158_2 Depth=1
+; RV32I-NEXT:    sw a3, 12(sp)
 ; RV32I-NEXT:    addi a3, zero, 4
 ; RV32I-NEXT:    addi a4, zero, 2
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s2
 ; RV32I-NEXT:    call __atomic_compare_exchange_4
-; RV32I-NEXT:    lw a2, 12(sp)
+; RV32I-NEXT:    lw a3, 12(sp)
 ; RV32I-NEXT:    bnez a0, .LBB158_4
-; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32I-NEXT:    # in Loop: Header=BB158_1 Depth=1
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    bltu s0, a2, .LBB158_1
-; RV32I-NEXT:  .LBB158_3: # %atomicrmw.start
-; RV32I-NEXT:    mv a2, s0
+; RV32I-NEXT:  .LBB158_2: # %atomicrmw.start
+; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    bltu s1, a3, .LBB158_1
+; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB158_2 Depth=1
+; RV32I-NEXT:    mv a2, s1
 ; RV32I-NEXT:    j .LBB158_1
 ; RV32I-NEXT:  .LBB158_4: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a3
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
 ; RV32I-NEXT:    lw s0, 24(sp)
@@ -12775,33 +12789,33 @@ define i32 @atomicrmw_umax_i32_acq_rel(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
 ; RV64I-NEXT:    sd s3, 8(sp)
-; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lwu a0, 0(a0)
-; RV64I-NEXT:    sext.w s0, a1
-; RV64I-NEXT:    addi s3, sp, 4
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    lwu a3, 0(a0)
+; RV64I-NEXT:    mv s3, a1
+; RV64I-NEXT:    sext.w s1, a1
+; RV64I-NEXT:    addi s2, sp, 4
 ; RV64I-NEXT:    j .LBB158_2
 ; RV64I-NEXT:  .LBB158_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB158_2 Depth=1
-; RV64I-NEXT:    sw a0, 4(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sw a3, 4(sp)
 ; RV64I-NEXT:    addi a3, zero, 4
 ; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    call __atomic_compare_exchange_4
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lw a0, 4(sp)
-; RV64I-NEXT:    bnez a1, .LBB158_4
+; RV64I-NEXT:    lw a3, 4(sp)
+; RV64I-NEXT:    bnez a0, .LBB158_4
 ; RV64I-NEXT:  .LBB158_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    sext.w a1, a0
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bltu s0, a1, .LBB158_1
+; RV64I-NEXT:    sext.w a0, a3
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bltu s1, a0, .LBB158_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB158_2 Depth=1
-; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:    mv a2, s3
 ; RV64I-NEXT:    j .LBB158_1
 ; RV64I-NEXT:  .LBB158_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -12826,30 +12840,31 @@ define i32 @atomicrmw_umax_i32_seq_cst(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    sw s0, 24(sp)
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
-; RV32I-NEXT:    mv s0, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a2, 0(a0)
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a3, 0(a0)
+; RV32I-NEXT:    mv s1, a1
 ; RV32I-NEXT:    addi s2, sp, 12
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    bgeu s0, a2, .LBB159_3
+; RV32I-NEXT:    j .LBB159_2
 ; RV32I-NEXT:  .LBB159_1: # %atomicrmw.start
-; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s2
+; RV32I-NEXT:    # in Loop: Header=BB159_2 Depth=1
+; RV32I-NEXT:    sw a3, 12(sp)
 ; RV32I-NEXT:    addi a3, zero, 5
 ; RV32I-NEXT:    addi a4, zero, 5
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s2
 ; RV32I-NEXT:    call __atomic_compare_exchange_4
-; RV32I-NEXT:    lw a2, 12(sp)
+; RV32I-NEXT:    lw a3, 12(sp)
 ; RV32I-NEXT:    bnez a0, .LBB159_4
-; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32I-NEXT:    # in Loop: Header=BB159_1 Depth=1
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    bltu s0, a2, .LBB159_1
-; RV32I-NEXT:  .LBB159_3: # %atomicrmw.start
-; RV32I-NEXT:    mv a2, s0
+; RV32I-NEXT:  .LBB159_2: # %atomicrmw.start
+; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    bltu s1, a3, .LBB159_1
+; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB159_2 Depth=1
+; RV32I-NEXT:    mv a2, s1
 ; RV32I-NEXT:    j .LBB159_1
 ; RV32I-NEXT:  .LBB159_4: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a3
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
 ; RV32I-NEXT:    lw s0, 24(sp)
@@ -12870,33 +12885,33 @@ define i32 @atomicrmw_umax_i32_seq_cst(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
 ; RV64I-NEXT:    sd s3, 8(sp)
-; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lwu a0, 0(a0)
-; RV64I-NEXT:    sext.w s0, a1
-; RV64I-NEXT:    addi s3, sp, 4
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    lwu a3, 0(a0)
+; RV64I-NEXT:    mv s3, a1
+; RV64I-NEXT:    sext.w s1, a1
+; RV64I-NEXT:    addi s2, sp, 4
 ; RV64I-NEXT:    j .LBB159_2
 ; RV64I-NEXT:  .LBB159_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB159_2 Depth=1
-; RV64I-NEXT:    sw a0, 4(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sw a3, 4(sp)
 ; RV64I-NEXT:    addi a3, zero, 5
 ; RV64I-NEXT:    addi a4, zero, 5
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    call __atomic_compare_exchange_4
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lw a0, 4(sp)
-; RV64I-NEXT:    bnez a1, .LBB159_4
+; RV64I-NEXT:    lw a3, 4(sp)
+; RV64I-NEXT:    bnez a0, .LBB159_4
 ; RV64I-NEXT:  .LBB159_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    sext.w a1, a0
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bltu s0, a1, .LBB159_1
+; RV64I-NEXT:    sext.w a0, a3
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bltu s1, a0, .LBB159_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB159_2 Depth=1
-; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:    mv a2, s3
 ; RV64I-NEXT:    j .LBB159_1
 ; RV64I-NEXT:  .LBB159_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -12921,30 +12936,31 @@ define i32 @atomicrmw_umin_i32_monotonic(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    sw s0, 24(sp)
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
-; RV32I-NEXT:    mv s0, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a2, 0(a0)
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a3, 0(a0)
+; RV32I-NEXT:    mv s1, a1
 ; RV32I-NEXT:    addi s2, sp, 12
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    bltu s0, a2, .LBB160_3
+; RV32I-NEXT:    j .LBB160_2
 ; RV32I-NEXT:  .LBB160_1: # %atomicrmw.start
-; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    mv a0, s1
+; RV32I-NEXT:    # in Loop: Header=BB160_2 Depth=1
+; RV32I-NEXT:    sw a3, 12(sp)
+; RV32I-NEXT:    mv a0, s0
 ; RV32I-NEXT:    mv a1, s2
 ; RV32I-NEXT:    mv a3, zero
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_4
-; RV32I-NEXT:    lw a2, 12(sp)
+; RV32I-NEXT:    lw a3, 12(sp)
 ; RV32I-NEXT:    bnez a0, .LBB160_4
-; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32I-NEXT:    # in Loop: Header=BB160_1 Depth=1
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    bgeu s0, a2, .LBB160_1
-; RV32I-NEXT:  .LBB160_3: # %atomicrmw.start
-; RV32I-NEXT:    mv a2, s0
+; RV32I-NEXT:  .LBB160_2: # %atomicrmw.start
+; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    bgeu s1, a3, .LBB160_1
+; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB160_2 Depth=1
+; RV32I-NEXT:    mv a2, s1
 ; RV32I-NEXT:    j .LBB160_1
 ; RV32I-NEXT:  .LBB160_4: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a3
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
 ; RV32I-NEXT:    lw s0, 24(sp)
@@ -12965,33 +12981,33 @@ define i32 @atomicrmw_umin_i32_monotonic(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
 ; RV64I-NEXT:    sd s3, 8(sp)
-; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lwu a0, 0(a0)
-; RV64I-NEXT:    sext.w s0, a1
-; RV64I-NEXT:    addi s3, sp, 4
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    lwu a3, 0(a0)
+; RV64I-NEXT:    mv s3, a1
+; RV64I-NEXT:    sext.w s1, a1
+; RV64I-NEXT:    addi s2, sp, 4
 ; RV64I-NEXT:    j .LBB160_2
 ; RV64I-NEXT:  .LBB160_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB160_2 Depth=1
-; RV64I-NEXT:    sw a0, 4(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sw a3, 4(sp)
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    mv a3, zero
 ; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    call __atomic_compare_exchange_4
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lw a0, 4(sp)
-; RV64I-NEXT:    bnez a1, .LBB160_4
+; RV64I-NEXT:    lw a3, 4(sp)
+; RV64I-NEXT:    bnez a0, .LBB160_4
 ; RV64I-NEXT:  .LBB160_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    sext.w a1, a0
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bgeu s0, a1, .LBB160_1
+; RV64I-NEXT:    sext.w a0, a3
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bgeu s1, a0, .LBB160_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB160_2 Depth=1
-; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:    mv a2, s3
 ; RV64I-NEXT:    j .LBB160_1
 ; RV64I-NEXT:  .LBB160_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -13016,30 +13032,31 @@ define i32 @atomicrmw_umin_i32_acquire(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    sw s0, 24(sp)
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
-; RV32I-NEXT:    mv s0, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a2, 0(a0)
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a3, 0(a0)
+; RV32I-NEXT:    mv s1, a1
 ; RV32I-NEXT:    addi s2, sp, 12
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    bltu s0, a2, .LBB161_3
+; RV32I-NEXT:    j .LBB161_2
 ; RV32I-NEXT:  .LBB161_1: # %atomicrmw.start
-; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s2
+; RV32I-NEXT:    # in Loop: Header=BB161_2 Depth=1
+; RV32I-NEXT:    sw a3, 12(sp)
 ; RV32I-NEXT:    addi a3, zero, 2
 ; RV32I-NEXT:    addi a4, zero, 2
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s2
 ; RV32I-NEXT:    call __atomic_compare_exchange_4
-; RV32I-NEXT:    lw a2, 12(sp)
+; RV32I-NEXT:    lw a3, 12(sp)
 ; RV32I-NEXT:    bnez a0, .LBB161_4
-; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32I-NEXT:    # in Loop: Header=BB161_1 Depth=1
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    bgeu s0, a2, .LBB161_1
-; RV32I-NEXT:  .LBB161_3: # %atomicrmw.start
-; RV32I-NEXT:    mv a2, s0
+; RV32I-NEXT:  .LBB161_2: # %atomicrmw.start
+; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    bgeu s1, a3, .LBB161_1
+; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB161_2 Depth=1
+; RV32I-NEXT:    mv a2, s1
 ; RV32I-NEXT:    j .LBB161_1
 ; RV32I-NEXT:  .LBB161_4: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a3
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
 ; RV32I-NEXT:    lw s0, 24(sp)
@@ -13060,33 +13077,33 @@ define i32 @atomicrmw_umin_i32_acquire(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
 ; RV64I-NEXT:    sd s3, 8(sp)
-; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lwu a0, 0(a0)
-; RV64I-NEXT:    sext.w s0, a1
-; RV64I-NEXT:    addi s3, sp, 4
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    lwu a3, 0(a0)
+; RV64I-NEXT:    mv s3, a1
+; RV64I-NEXT:    sext.w s1, a1
+; RV64I-NEXT:    addi s2, sp, 4
 ; RV64I-NEXT:    j .LBB161_2
 ; RV64I-NEXT:  .LBB161_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB161_2 Depth=1
-; RV64I-NEXT:    sw a0, 4(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sw a3, 4(sp)
 ; RV64I-NEXT:    addi a3, zero, 2
 ; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    call __atomic_compare_exchange_4
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lw a0, 4(sp)
-; RV64I-NEXT:    bnez a1, .LBB161_4
+; RV64I-NEXT:    lw a3, 4(sp)
+; RV64I-NEXT:    bnez a0, .LBB161_4
 ; RV64I-NEXT:  .LBB161_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    sext.w a1, a0
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bgeu s0, a1, .LBB161_1
+; RV64I-NEXT:    sext.w a0, a3
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bgeu s1, a0, .LBB161_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB161_2 Depth=1
-; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:    mv a2, s3
 ; RV64I-NEXT:    j .LBB161_1
 ; RV64I-NEXT:  .LBB161_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -13111,30 +13128,31 @@ define i32 @atomicrmw_umin_i32_release(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    sw s0, 24(sp)
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
-; RV32I-NEXT:    mv s0, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a2, 0(a0)
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a3, 0(a0)
+; RV32I-NEXT:    mv s1, a1
 ; RV32I-NEXT:    addi s2, sp, 12
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    bltu s0, a2, .LBB162_3
+; RV32I-NEXT:    j .LBB162_2
 ; RV32I-NEXT:  .LBB162_1: # %atomicrmw.start
-; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s2
+; RV32I-NEXT:    # in Loop: Header=BB162_2 Depth=1
+; RV32I-NEXT:    sw a3, 12(sp)
 ; RV32I-NEXT:    addi a3, zero, 3
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s2
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_4
-; RV32I-NEXT:    lw a2, 12(sp)
+; RV32I-NEXT:    lw a3, 12(sp)
 ; RV32I-NEXT:    bnez a0, .LBB162_4
-; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32I-NEXT:    # in Loop: Header=BB162_1 Depth=1
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    bgeu s0, a2, .LBB162_1
-; RV32I-NEXT:  .LBB162_3: # %atomicrmw.start
-; RV32I-NEXT:    mv a2, s0
+; RV32I-NEXT:  LBB162_2: # %atomicrmw.start
+; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    bgeu s1, a3, .LBB162_1
+; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB162_2 Depth=1
+; RV32I-NEXT:    mv a2, s1
 ; RV32I-NEXT:    j .LBB162_1
 ; RV32I-NEXT:  .LBB162_4: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a3
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
 ; RV32I-NEXT:    lw s0, 24(sp)
@@ -13155,33 +13173,33 @@ define i32 @atomicrmw_umin_i32_release(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
 ; RV64I-NEXT:    sd s3, 8(sp)
-; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lwu a0, 0(a0)
-; RV64I-NEXT:    sext.w s0, a1
-; RV64I-NEXT:    addi s3, sp, 4
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    lwu a3, 0(a0)
+; RV64I-NEXT:    mv s3, a1
+; RV64I-NEXT:    sext.w s1, a1
+; RV64I-NEXT:    addi s2, sp, 4
 ; RV64I-NEXT:    j .LBB162_2
 ; RV64I-NEXT:  .LBB162_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB162_2 Depth=1
-; RV64I-NEXT:    sw a0, 4(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sw a3, 4(sp)
 ; RV64I-NEXT:    addi a3, zero, 3
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    call __atomic_compare_exchange_4
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lw a0, 4(sp)
-; RV64I-NEXT:    bnez a1, .LBB162_4
+; RV64I-NEXT:    lw a3, 4(sp)
+; RV64I-NEXT:    bnez a0, .LBB162_4
 ; RV64I-NEXT:  .LBB162_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    sext.w a1, a0
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bgeu s0, a1, .LBB162_1
+; RV64I-NEXT:    sext.w a0, a3
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bgeu s1, a0, .LBB162_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB162_2 Depth=1
-; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:    mv a2, s3
 ; RV64I-NEXT:    j .LBB162_1
 ; RV64I-NEXT:  .LBB162_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -13206,30 +13224,31 @@ define i32 @atomicrmw_umin_i32_acq_rel(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    sw s0, 24(sp)
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
-; RV32I-NEXT:    mv s0, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a2, 0(a0)
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a3, 0(a0)
+; RV32I-NEXT:    mv s1, a1
 ; RV32I-NEXT:    addi s2, sp, 12
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    bltu s0, a2, .LBB163_3
+; RV32I-NEXT:    j .LBB163_2
 ; RV32I-NEXT:  .LBB163_1: # %atomicrmw.start
-; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s2
+; RV32I-NEXT:    # in Loop: Header=BB163_2 Depth=1
+; RV32I-NEXT:    sw a3, 12(sp)
 ; RV32I-NEXT:    addi a3, zero, 4
 ; RV32I-NEXT:    addi a4, zero, 2
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s2
 ; RV32I-NEXT:    call __atomic_compare_exchange_4
-; RV32I-NEXT:    lw a2, 12(sp)
+; RV32I-NEXT:    lw a3, 12(sp)
 ; RV32I-NEXT:    bnez a0, .LBB163_4
-; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32I-NEXT:    # in Loop: Header=BB163_1 Depth=1
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    bgeu s0, a2, .LBB163_1
-; RV32I-NEXT:  .LBB163_3: # %atomicrmw.start
-; RV32I-NEXT:    mv a2, s0
+; RV32I-NEXT:  .LBB163_2: # %atomicrmw.start
+; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    bgeu s1, a3, .LBB163_1
+; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB163_2 Depth=1
+; RV32I-NEXT:    mv a2, s1
 ; RV32I-NEXT:    j .LBB163_1
 ; RV32I-NEXT:  .LBB163_4: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a3
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
 ; RV32I-NEXT:    lw s0, 24(sp)
@@ -13250,33 +13269,33 @@ define i32 @atomicrmw_umin_i32_acq_rel(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
 ; RV64I-NEXT:    sd s3, 8(sp)
-; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lwu a0, 0(a0)
-; RV64I-NEXT:    sext.w s0, a1
-; RV64I-NEXT:    addi s3, sp, 4
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    lwu a3, 0(a0)
+; RV64I-NEXT:    mv s3, a1
+; RV64I-NEXT:    sext.w s1, a1
+; RV64I-NEXT:    addi s2, sp, 4
 ; RV64I-NEXT:    j .LBB163_2
 ; RV64I-NEXT:  .LBB163_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB163_2 Depth=1
-; RV64I-NEXT:    sw a0, 4(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sw a3, 4(sp)
 ; RV64I-NEXT:    addi a3, zero, 4
 ; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    call __atomic_compare_exchange_4
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lw a0, 4(sp)
-; RV64I-NEXT:    bnez a1, .LBB163_4
+; RV64I-NEXT:    lw a3, 4(sp)
+; RV64I-NEXT:    bnez a0, .LBB163_4
 ; RV64I-NEXT:  .LBB163_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    sext.w a1, a0
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bgeu s0, a1, .LBB163_1
+; RV64I-NEXT:    sext.w a0, a3
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bgeu s1, a0, .LBB163_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB163_2 Depth=1
-; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:    mv a2, s3
 ; RV64I-NEXT:    j .LBB163_1
 ; RV64I-NEXT:  .LBB163_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -13301,30 +13320,31 @@ define i32 @atomicrmw_umin_i32_seq_cst(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    sw s0, 24(sp)
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
-; RV32I-NEXT:    mv s0, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a2, 0(a0)
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a3, 0(a0)
+; RV32I-NEXT:    mv s1, a1
 ; RV32I-NEXT:    addi s2, sp, 12
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    bltu s0, a2, .LBB164_3
+; RV32I-NEXT:    j .LBB164_2
 ; RV32I-NEXT:  .LBB164_1: # %atomicrmw.start
-; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s2
+; RV32I-NEXT:    # in Loop: Header=BB164_2 Depth=1
+; RV32I-NEXT:    sw a3, 12(sp)
 ; RV32I-NEXT:    addi a3, zero, 5
 ; RV32I-NEXT:    addi a4, zero, 5
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s2
 ; RV32I-NEXT:    call __atomic_compare_exchange_4
-; RV32I-NEXT:    lw a2, 12(sp)
+; RV32I-NEXT:    lw a3, 12(sp)
 ; RV32I-NEXT:    bnez a0, .LBB164_4
-; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32I-NEXT:    # in Loop: Header=BB164_1 Depth=1
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    bgeu s0, a2, .LBB164_1
-; RV32I-NEXT:  .LBB164_3: # %atomicrmw.start
-; RV32I-NEXT:    mv a2, s0
+; RV32I-NEXT:  .LBB164_2: # %atomicrmw.start
+; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    bgeu s1, a3, .LBB164_1
+; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB164_2 Depth=1
+; RV32I-NEXT:    mv a2, s1
 ; RV32I-NEXT:    j .LBB164_1
 ; RV32I-NEXT:  .LBB164_4: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a3
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
 ; RV32I-NEXT:    lw s0, 24(sp)
@@ -13345,33 +13365,33 @@ define i32 @atomicrmw_umin_i32_seq_cst(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
 ; RV64I-NEXT:    sd s3, 8(sp)
-; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lwu a0, 0(a0)
-; RV64I-NEXT:    sext.w s0, a1
-; RV64I-NEXT:    addi s3, sp, 4
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    lwu a3, 0(a0)
+; RV64I-NEXT:    mv s3, a1
+; RV64I-NEXT:    sext.w s1, a1
+; RV64I-NEXT:    addi s2, sp, 4
 ; RV64I-NEXT:    j .LBB164_2
 ; RV64I-NEXT:  .LBB164_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB164_2 Depth=1
-; RV64I-NEXT:    sw a0, 4(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sw a3, 4(sp)
 ; RV64I-NEXT:    addi a3, zero, 5
 ; RV64I-NEXT:    addi a4, zero, 5
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    call __atomic_compare_exchange_4
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lw a0, 4(sp)
-; RV64I-NEXT:    bnez a1, .LBB164_4
+; RV64I-NEXT:    lw a3, 4(sp)
+; RV64I-NEXT:    bnez a0, .LBB164_4
 ; RV64I-NEXT:  .LBB164_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    sext.w a1, a0
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bgeu s0, a1, .LBB164_1
+; RV64I-NEXT:    sext.w a0, a3
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bgeu s1, a0, .LBB164_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB164_2 Depth=1
-; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:    mv a2, s3
 ; RV64I-NEXT:    j .LBB164_1
 ; RV64I-NEXT:  .LBB164_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -14802,41 +14822,47 @@ define i64 @atomicrmw_max_i64_monotonic(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
 ; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    mv s0, a2
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a5, 4(a0)
+; RV32I-NEXT:    lw a4, 0(a0)
+; RV32I-NEXT:    mv s1, a2
 ; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a1, 4(a0)
-; RV32I-NEXT:    lw a2, 0(a0)
 ; RV32I-NEXT:    mv s3, sp
-; RV32I-NEXT:    bne a1, s0, .LBB200_3
-; RV32I-NEXT:    j .LBB200_4
+; RV32I-NEXT:    j .LBB200_2
 ; RV32I-NEXT:  .LBB200_1: # %atomicrmw.start
-; RV32I-NEXT:    sw a1, 4(sp)
-; RV32I-NEXT:    mv a0, s1
+; RV32I-NEXT:    # in Loop: Header=BB200_2 Depth=1
+; RV32I-NEXT:    sw a4, 0(sp)
+; RV32I-NEXT:    sw a5, 4(sp)
+; RV32I-NEXT:    mv a0, s0
 ; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    mv a5, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
-; RV32I-NEXT:    lw a1, 4(sp)
-; RV32I-NEXT:    lw a2, 0(sp)
+; RV32I-NEXT:    lw a5, 4(sp)
+; RV32I-NEXT:    lw a4, 0(sp)
 ; RV32I-NEXT:    bnez a0, .LBB200_7
-; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32I-NEXT:    beq a1, s0, .LBB200_4
-; RV32I-NEXT:  .LBB200_3: # %atomicrmw.start
-; RV32I-NEXT:    slt a0, s0, a1
+; RV32I-NEXT:  .LBB200_2: # %atomicrmw.start
+; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32I-NEXT:    beq a5, s1, .LBB200_4
+; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB200_2 Depth=1
+; RV32I-NEXT:    slt a0, s1, a5
 ; RV32I-NEXT:    j .LBB200_5
 ; RV32I-NEXT:  .LBB200_4:
-; RV32I-NEXT:    sltu a0, s2, a2
+; RV32I-NEXT:    sltu a0, s2, a4
 ; RV32I-NEXT:  .LBB200_5: # %atomicrmw.start
-; RV32I-NEXT:    sw a2, 0(sp)
-; RV32I-NEXT:    mv a3, a1
+; RV32I-NEXT:    # in Loop: Header=BB200_2 Depth=1
+; RV32I-NEXT:    mv a2, a4
+; RV32I-NEXT:    mv a3, a5
 ; RV32I-NEXT:    bnez a0, .LBB200_1
 ; RV32I-NEXT:  # %bb.6: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB200_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
-; RV32I-NEXT:    mv a3, s0
+; RV32I-NEXT:    mv a3, s1
 ; RV32I-NEXT:    j .LBB200_1
 ; RV32I-NEXT:  .LBB200_7: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a4
+; RV32I-NEXT:    mv a1, a5
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -14853,41 +14879,47 @@ define i64 @atomicrmw_max_i64_monotonic(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    sw s1, 20(sp)
 ; RV32IA-NEXT:    sw s2, 16(sp)
 ; RV32IA-NEXT:    sw s3, 12(sp)
-; RV32IA-NEXT:    mv s0, a2
+; RV32IA-NEXT:    mv s0, a0
+; RV32IA-NEXT:    lw a5, 4(a0)
+; RV32IA-NEXT:    lw a4, 0(a0)
+; RV32IA-NEXT:    mv s1, a2
 ; RV32IA-NEXT:    mv s2, a1
-; RV32IA-NEXT:    mv s1, a0
-; RV32IA-NEXT:    lw a1, 4(a0)
-; RV32IA-NEXT:    lw a2, 0(a0)
 ; RV32IA-NEXT:    mv s3, sp
-; RV32IA-NEXT:    bne a1, s0, .LBB200_3
-; RV32IA-NEXT:    j .LBB200_4
+; RV32IA-NEXT:    j .LBB200_2
 ; RV32IA-NEXT:  .LBB200_1: # %atomicrmw.start
-; RV32IA-NEXT:    sw a1, 4(sp)
-; RV32IA-NEXT:    mv a0, s1
+; RV32IA-NEXT:    # in Loop: Header=BB200_2 Depth=1
+; RV32IA-NEXT:    sw a4, 0(sp)
+; RV32IA-NEXT:    sw a5, 4(sp)
+; RV32IA-NEXT:    mv a0, s0
 ; RV32IA-NEXT:    mv a1, s3
 ; RV32IA-NEXT:    mv a4, zero
 ; RV32IA-NEXT:    mv a5, zero
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
-; RV32IA-NEXT:    lw a1, 4(sp)
-; RV32IA-NEXT:    lw a2, 0(sp)
+; RV32IA-NEXT:    lw a5, 4(sp)
+; RV32IA-NEXT:    lw a4, 0(sp)
 ; RV32IA-NEXT:    bnez a0, .LBB200_7
-; RV32IA-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32IA-NEXT:    beq a1, s0, .LBB200_4
-; RV32IA-NEXT:  .LBB200_3: # %atomicrmw.start
-; RV32IA-NEXT:    slt a0, s0, a1
+; RV32IA-NEXT:  .LBB200_2: # %atomicrmw.start
+; RV32IA-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32IA-NEXT:    beq a5, s1, .LBB200_4
+; RV32IA-NEXT:  # %bb.3: # %atomicrmw.start
+; RV32IA-NEXT:    # in Loop: Header=BB200_2 Depth=1
+; RV32IA-NEXT:    slt a0, s1, a5
 ; RV32IA-NEXT:    j .LBB200_5
 ; RV32IA-NEXT:  .LBB200_4:
-; RV32IA-NEXT:    sltu a0, s2, a2
+; RV32IA-NEXT:    sltu a0, s2, a4
 ; RV32IA-NEXT:  .LBB200_5: # %atomicrmw.start
-; RV32IA-NEXT:    sw a2, 0(sp)
-; RV32IA-NEXT:    mv a3, a1
+; RV32IA-NEXT:    # in Loop: Header=BB200_2 Depth=1
+; RV32IA-NEXT:    mv a2, a4
+; RV32IA-NEXT:    mv a3, a5
 ; RV32IA-NEXT:    bnez a0, .LBB200_1
 ; RV32IA-NEXT:  # %bb.6: # %atomicrmw.start
+; RV32IA-NEXT:    # in Loop: Header=BB200_2 Depth=1
 ; RV32IA-NEXT:    mv a2, s2
-; RV32IA-NEXT:    mv a3, s0
+; RV32IA-NEXT:    mv a3, s1
 ; RV32IA-NEXT:    j .LBB200_1
 ; RV32IA-NEXT:  .LBB200_7: # %atomicrmw.end
-; RV32IA-NEXT:    mv a0, a2
+; RV32IA-NEXT:    mv a0, a4
+; RV32IA-NEXT:    mv a1, a5
 ; RV32IA-NEXT:    lw s3, 12(sp)
 ; RV32IA-NEXT:    lw s2, 16(sp)
 ; RV32IA-NEXT:    lw s1, 20(sp)
@@ -14903,30 +14935,31 @@ define i64 @atomicrmw_max_i64_monotonic(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    sd s0, 32(sp)
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
-; RV64I-NEXT:    mv s0, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    ld a2, 0(a0)
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    ld a3, 0(a0)
+; RV64I-NEXT:    mv s1, a1
 ; RV64I-NEXT:    addi s2, sp, 8
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    bge s0, a2, .LBB200_3
+; RV64I-NEXT:    j .LBB200_2
 ; RV64I-NEXT:  .LBB200_1: # %atomicrmw.start
-; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    mv a0, s1
+; RV64I-NEXT:    # in Loop: Header=BB200_2 Depth=1
+; RV64I-NEXT:    sd a3, 8(sp)
+; RV64I-NEXT:    mv a0, s0
 ; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    mv a3, zero
 ; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    call __atomic_compare_exchange_8
-; RV64I-NEXT:    ld a2, 8(sp)
+; RV64I-NEXT:    ld a3, 8(sp)
 ; RV64I-NEXT:    bnez a0, .LBB200_4
-; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV64I-NEXT:    # in Loop: Header=BB200_1 Depth=1
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    blt s0, a2, .LBB200_1
-; RV64I-NEXT:  .LBB200_3: # %atomicrmw.start
-; RV64I-NEXT:    mv a2, s0
+; RV64I-NEXT:  .LBB200_2: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    blt s1, a3, .LBB200_1
+; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB200_2 Depth=1
+; RV64I-NEXT:    mv a2, s1
 ; RV64I-NEXT:    j .LBB200_1
 ; RV64I-NEXT:  .LBB200_4: # %atomicrmw.end
-; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
 ; RV64I-NEXT:    ld s0, 32(sp)
@@ -14951,41 +14984,47 @@ define i64 @atomicrmw_max_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
 ; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    mv s0, a2
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a5, 4(a0)
+; RV32I-NEXT:    lw a4, 0(a0)
+; RV32I-NEXT:    mv s1, a2
 ; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a1, 4(a0)
-; RV32I-NEXT:    lw a2, 0(a0)
 ; RV32I-NEXT:    mv s3, sp
-; RV32I-NEXT:    bne a1, s0, .LBB201_3
-; RV32I-NEXT:    j .LBB201_4
+; RV32I-NEXT:    j .LBB201_2
 ; RV32I-NEXT:  .LBB201_1: # %atomicrmw.start
-; RV32I-NEXT:    sw a1, 4(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    # in Loop: Header=BB201_2 Depth=1
+; RV32I-NEXT:    sw a4, 0(sp)
+; RV32I-NEXT:    sw a5, 4(sp)
 ; RV32I-NEXT:    addi a4, zero, 2
 ; RV32I-NEXT:    addi a5, zero, 2
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
-; RV32I-NEXT:    lw a1, 4(sp)
-; RV32I-NEXT:    lw a2, 0(sp)
+; RV32I-NEXT:    lw a5, 4(sp)
+; RV32I-NEXT:    lw a4, 0(sp)
 ; RV32I-NEXT:    bnez a0, .LBB201_7
-; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32I-NEXT:    beq a1, s0, .LBB201_4
-; RV32I-NEXT:  .LBB201_3: # %atomicrmw.start
-; RV32I-NEXT:    slt a0, s0, a1
+; RV32I-NEXT:  .LBB201_2: # %atomicrmw.start
+; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32I-NEXT:    beq a5, s1, .LBB201_4
+; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB201_2 Depth=1
+; RV32I-NEXT:    slt a0, s1, a5
 ; RV32I-NEXT:    j .LBB201_5
 ; RV32I-NEXT:  .LBB201_4:
-; RV32I-NEXT:    sltu a0, s2, a2
+; RV32I-NEXT:    sltu a0, s2, a4
 ; RV32I-NEXT:  .LBB201_5: # %atomicrmw.start
-; RV32I-NEXT:    sw a2, 0(sp)
-; RV32I-NEXT:    mv a3, a1
+; RV32I-NEXT:    # in Loop: Header=BB201_2 Depth=1
+; RV32I-NEXT:    mv a2, a4
+; RV32I-NEXT:    mv a3, a5
 ; RV32I-NEXT:    bnez a0, .LBB201_1
 ; RV32I-NEXT:  # %bb.6: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB201_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
-; RV32I-NEXT:    mv a3, s0
+; RV32I-NEXT:    mv a3, s1
 ; RV32I-NEXT:    j .LBB201_1
 ; RV32I-NEXT:  .LBB201_7: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a4
+; RV32I-NEXT:    mv a1, a5
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -15002,41 +15041,47 @@ define i64 @atomicrmw_max_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    sw s1, 20(sp)
 ; RV32IA-NEXT:    sw s2, 16(sp)
 ; RV32IA-NEXT:    sw s3, 12(sp)
-; RV32IA-NEXT:    mv s0, a2
+; RV32IA-NEXT:    mv s0, a0
+; RV32IA-NEXT:    lw a5, 4(a0)
+; RV32IA-NEXT:    lw a4, 0(a0)
+; RV32IA-NEXT:    mv s1, a2
 ; RV32IA-NEXT:    mv s2, a1
-; RV32IA-NEXT:    mv s1, a0
-; RV32IA-NEXT:    lw a1, 4(a0)
-; RV32IA-NEXT:    lw a2, 0(a0)
 ; RV32IA-NEXT:    mv s3, sp
-; RV32IA-NEXT:    bne a1, s0, .LBB201_3
-; RV32IA-NEXT:    j .LBB201_4
+; RV32IA-NEXT:    j .LBB201_2
 ; RV32IA-NEXT:  .LBB201_1: # %atomicrmw.start
-; RV32IA-NEXT:    sw a1, 4(sp)
-; RV32IA-NEXT:    mv a0, s1
-; RV32IA-NEXT:    mv a1, s3
+; RV32IA-NEXT:    # in Loop: Header=BB201_2 Depth=1
+; RV32IA-NEXT:    sw a4, 0(sp)
+; RV32IA-NEXT:    sw a5, 4(sp)
 ; RV32IA-NEXT:    addi a4, zero, 2
 ; RV32IA-NEXT:    addi a5, zero, 2
+; RV32IA-NEXT:    mv a0, s0
+; RV32IA-NEXT:    mv a1, s3
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
-; RV32IA-NEXT:    lw a1, 4(sp)
-; RV32IA-NEXT:    lw a2, 0(sp)
+; RV32IA-NEXT:    lw a5, 4(sp)
+; RV32IA-NEXT:    lw a4, 0(sp)
 ; RV32IA-NEXT:    bnez a0, .LBB201_7
-; RV32IA-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32IA-NEXT:    beq a1, s0, .LBB201_4
-; RV32IA-NEXT:  .LBB201_3: # %atomicrmw.start
-; RV32IA-NEXT:    slt a0, s0, a1
+; RV32IA-NEXT:  .LBB201_2: # %atomicrmw.start
+; RV32IA-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32IA-NEXT:    beq a5, s1, .LBB201_4
+; RV32IA-NEXT:  # %bb.3: # %atomicrmw.start
+; RV32IA-NEXT:    # in Loop: Header=BB201_2 Depth=1
+; RV32IA-NEXT:    slt a0, s1, a5
 ; RV32IA-NEXT:    j .LBB201_5
 ; RV32IA-NEXT:  .LBB201_4:
-; RV32IA-NEXT:    sltu a0, s2, a2
+; RV32IA-NEXT:    sltu a0, s2, a4
 ; RV32IA-NEXT:  .LBB201_5: # %atomicrmw.start
-; RV32IA-NEXT:    sw a2, 0(sp)
-; RV32IA-NEXT:    mv a3, a1
+; RV32IA-NEXT:    # in Loop: Header=BB201_2 Depth=1
+; RV32IA-NEXT:    mv a2, a4
+; RV32IA-NEXT:    mv a3, a5
 ; RV32IA-NEXT:    bnez a0, .LBB201_1
 ; RV32IA-NEXT:  # %bb.6: # %atomicrmw.start
+; RV32IA-NEXT:    # in Loop: Header=BB201_2 Depth=1
 ; RV32IA-NEXT:    mv a2, s2
-; RV32IA-NEXT:    mv a3, s0
+; RV32IA-NEXT:    mv a3, s1
 ; RV32IA-NEXT:    j .LBB201_1
 ; RV32IA-NEXT:  .LBB201_7: # %atomicrmw.end
-; RV32IA-NEXT:    mv a0, a2
+; RV32IA-NEXT:    mv a0, a4
+; RV32IA-NEXT:    mv a1, a5
 ; RV32IA-NEXT:    lw s3, 12(sp)
 ; RV32IA-NEXT:    lw s2, 16(sp)
 ; RV32IA-NEXT:    lw s1, 20(sp)
@@ -15052,30 +15097,31 @@ define i64 @atomicrmw_max_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    sd s0, 32(sp)
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
-; RV64I-NEXT:    mv s0, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    ld a2, 0(a0)
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    ld a3, 0(a0)
+; RV64I-NEXT:    mv s1, a1
 ; RV64I-NEXT:    addi s2, sp, 8
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    bge s0, a2, .LBB201_3
+; RV64I-NEXT:    j .LBB201_2
 ; RV64I-NEXT:  .LBB201_1: # %atomicrmw.start
-; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s2
+; RV64I-NEXT:    # in Loop: Header=BB201_2 Depth=1
+; RV64I-NEXT:    sd a3, 8(sp)
 ; RV64I-NEXT:    addi a3, zero, 2
 ; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    call __atomic_compare_exchange_8
-; RV64I-NEXT:    ld a2, 8(sp)
+; RV64I-NEXT:    ld a3, 8(sp)
 ; RV64I-NEXT:    bnez a0, .LBB201_4
-; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV64I-NEXT:    # in Loop: Header=BB201_1 Depth=1
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    blt s0, a2, .LBB201_1
-; RV64I-NEXT:  .LBB201_3: # %atomicrmw.start
-; RV64I-NEXT:    mv a2, s0
+; RV64I-NEXT:  .LBB201_2: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    blt s1, a3, .LBB201_1
+; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB201_2 Depth=1
+; RV64I-NEXT:    mv a2, s1
 ; RV64I-NEXT:    j .LBB201_1
 ; RV64I-NEXT:  .LBB201_4: # %atomicrmw.end
-; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
 ; RV64I-NEXT:    ld s0, 32(sp)
@@ -15100,41 +15146,47 @@ define i64 @atomicrmw_max_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
 ; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    mv s0, a2
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a5, 4(a0)
+; RV32I-NEXT:    lw a4, 0(a0)
+; RV32I-NEXT:    mv s1, a2
 ; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a1, 4(a0)
-; RV32I-NEXT:    lw a2, 0(a0)
 ; RV32I-NEXT:    mv s3, sp
-; RV32I-NEXT:    bne a1, s0, .LBB202_3
-; RV32I-NEXT:    j .LBB202_4
+; RV32I-NEXT:    j .LBB202_2
 ; RV32I-NEXT:  .LBB202_1: # %atomicrmw.start
-; RV32I-NEXT:    sw a1, 4(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    # in Loop: Header=BB202_2 Depth=1
+; RV32I-NEXT:    sw a4, 0(sp)
+; RV32I-NEXT:    sw a5, 4(sp)
 ; RV32I-NEXT:    addi a4, zero, 3
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    mv a5, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
-; RV32I-NEXT:    lw a1, 4(sp)
-; RV32I-NEXT:    lw a2, 0(sp)
+; RV32I-NEXT:    lw a5, 4(sp)
+; RV32I-NEXT:    lw a4, 0(sp)
 ; RV32I-NEXT:    bnez a0, .LBB202_7
-; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32I-NEXT:    beq a1, s0, .LBB202_4
-; RV32I-NEXT:  .LBB202_3: # %atomicrmw.start
-; RV32I-NEXT:    slt a0, s0, a1
+; RV32I-NEXT:  .LBB202_2: # %atomicrmw.start
+; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32I-NEXT:    beq a5, s1, .LBB202_4
+; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB202_2 Depth=1
+; RV32I-NEXT:    slt a0, s1, a5
 ; RV32I-NEXT:    j .LBB202_5
 ; RV32I-NEXT:  .LBB202_4:
-; RV32I-NEXT:    sltu a0, s2, a2
+; RV32I-NEXT:    sltu a0, s2, a4
 ; RV32I-NEXT:  .LBB202_5: # %atomicrmw.start
-; RV32I-NEXT:    sw a2, 0(sp)
-; RV32I-NEXT:    mv a3, a1
+; RV32I-NEXT:    # in Loop: Header=BB202_2 Depth=1
+; RV32I-NEXT:    mv a2, a4
+; RV32I-NEXT:    mv a3, a5
 ; RV32I-NEXT:    bnez a0, .LBB202_1
 ; RV32I-NEXT:  # %bb.6: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB202_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
-; RV32I-NEXT:    mv a3, s0
+; RV32I-NEXT:    mv a3, s1
 ; RV32I-NEXT:    j .LBB202_1
 ; RV32I-NEXT:  .LBB202_7: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a4
+; RV32I-NEXT:    mv a1, a5
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -15151,41 +15203,47 @@ define i64 @atomicrmw_max_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    sw s1, 20(sp)
 ; RV32IA-NEXT:    sw s2, 16(sp)
 ; RV32IA-NEXT:    sw s3, 12(sp)
-; RV32IA-NEXT:    mv s0, a2
+; RV32IA-NEXT:    mv s0, a0
+; RV32IA-NEXT:    lw a5, 4(a0)
+; RV32IA-NEXT:    lw a4, 0(a0)
+; RV32IA-NEXT:    mv s1, a2
 ; RV32IA-NEXT:    mv s2, a1
-; RV32IA-NEXT:    mv s1, a0
-; RV32IA-NEXT:    lw a1, 4(a0)
-; RV32IA-NEXT:    lw a2, 0(a0)
 ; RV32IA-NEXT:    mv s3, sp
-; RV32IA-NEXT:    bne a1, s0, .LBB202_3
-; RV32IA-NEXT:    j .LBB202_4
+; RV32IA-NEXT:    j .LBB202_2
 ; RV32IA-NEXT:  .LBB202_1: # %atomicrmw.start
-; RV32IA-NEXT:    sw a1, 4(sp)
-; RV32IA-NEXT:    mv a0, s1
-; RV32IA-NEXT:    mv a1, s3
+; RV32IA-NEXT:    # in Loop: Header=BB202_2 Depth=1
+; RV32IA-NEXT:    sw a4, 0(sp)
+; RV32IA-NEXT:    sw a5, 4(sp)
 ; RV32IA-NEXT:    addi a4, zero, 3
+; RV32IA-NEXT:    mv a0, s0
+; RV32IA-NEXT:    mv a1, s3
 ; RV32IA-NEXT:    mv a5, zero
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
-; RV32IA-NEXT:    lw a1, 4(sp)
-; RV32IA-NEXT:    lw a2, 0(sp)
+; RV32IA-NEXT:    lw a5, 4(sp)
+; RV32IA-NEXT:    lw a4, 0(sp)
 ; RV32IA-NEXT:    bnez a0, .LBB202_7
-; RV32IA-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32IA-NEXT:    beq a1, s0, .LBB202_4
-; RV32IA-NEXT:  .LBB202_3: # %atomicrmw.start
-; RV32IA-NEXT:    slt a0, s0, a1
+; RV32IA-NEXT:  .LBB202_2: # %atomicrmw.start
+; RV32IA-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32IA-NEXT:    beq a5, s1, .LBB202_4
+; RV32IA-NEXT:  # %bb.3: # %atomicrmw.start
+; RV32IA-NEXT:    # in Loop: Header=BB202_2 Depth=1
+; RV32IA-NEXT:    slt a0, s1, a5
 ; RV32IA-NEXT:    j .LBB202_5
 ; RV32IA-NEXT:  .LBB202_4:
-; RV32IA-NEXT:    sltu a0, s2, a2
+; RV32IA-NEXT:    sltu a0, s2, a4
 ; RV32IA-NEXT:  .LBB202_5: # %atomicrmw.start
-; RV32IA-NEXT:    sw a2, 0(sp)
-; RV32IA-NEXT:    mv a3, a1
+; RV32IA-NEXT:    # in Loop: Header=BB202_2 Depth=1
+; RV32IA-NEXT:    mv a2, a4
+; RV32IA-NEXT:    mv a3, a5
 ; RV32IA-NEXT:    bnez a0, .LBB202_1
 ; RV32IA-NEXT:  # %bb.6: # %atomicrmw.start
+; RV32IA-NEXT:    # in Loop: Header=BB202_2 Depth=1
 ; RV32IA-NEXT:    mv a2, s2
-; RV32IA-NEXT:    mv a3, s0
+; RV32IA-NEXT:    mv a3, s1
 ; RV32IA-NEXT:    j .LBB202_1
 ; RV32IA-NEXT:  .LBB202_7: # %atomicrmw.end
-; RV32IA-NEXT:    mv a0, a2
+; RV32IA-NEXT:    mv a0, a4
+; RV32IA-NEXT:    mv a1, a5
 ; RV32IA-NEXT:    lw s3, 12(sp)
 ; RV32IA-NEXT:    lw s2, 16(sp)
 ; RV32IA-NEXT:    lw s1, 20(sp)
@@ -15201,30 +15259,31 @@ define i64 @atomicrmw_max_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    sd s0, 32(sp)
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
-; RV64I-NEXT:    mv s0, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    ld a2, 0(a0)
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    ld a3, 0(a0)
+; RV64I-NEXT:    mv s1, a1
 ; RV64I-NEXT:    addi s2, sp, 8
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    bge s0, a2, .LBB202_3
+; RV64I-NEXT:    j .LBB202_2
 ; RV64I-NEXT:  .LBB202_1: # %atomicrmw.start
-; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s2
+; RV64I-NEXT:    # in Loop: Header=BB202_2 Depth=1
+; RV64I-NEXT:    sd a3, 8(sp)
 ; RV64I-NEXT:    addi a3, zero, 3
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    call __atomic_compare_exchange_8
-; RV64I-NEXT:    ld a2, 8(sp)
+; RV64I-NEXT:    ld a3, 8(sp)
 ; RV64I-NEXT:    bnez a0, .LBB202_4
-; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV64I-NEXT:    # in Loop: Header=BB202_1 Depth=1
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    blt s0, a2, .LBB202_1
-; RV64I-NEXT:  .LBB202_3: # %atomicrmw.start
-; RV64I-NEXT:    mv a2, s0
+; RV64I-NEXT:  .LBB202_2: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    blt s1, a3, .LBB202_1
+; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB202_2 Depth=1
+; RV64I-NEXT:    mv a2, s1
 ; RV64I-NEXT:    j .LBB202_1
 ; RV64I-NEXT:  .LBB202_4: # %atomicrmw.end
-; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
 ; RV64I-NEXT:    ld s0, 32(sp)
@@ -15249,41 +15308,47 @@ define i64 @atomicrmw_max_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
 ; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    mv s0, a2
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a5, 4(a0)
+; RV32I-NEXT:    lw a4, 0(a0)
+; RV32I-NEXT:    mv s1, a2
 ; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a1, 4(a0)
-; RV32I-NEXT:    lw a2, 0(a0)
 ; RV32I-NEXT:    mv s3, sp
-; RV32I-NEXT:    bne a1, s0, .LBB203_3
-; RV32I-NEXT:    j .LBB203_4
+; RV32I-NEXT:    j .LBB203_2
 ; RV32I-NEXT:  .LBB203_1: # %atomicrmw.start
-; RV32I-NEXT:    sw a1, 4(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    # in Loop: Header=BB203_2 Depth=1
+; RV32I-NEXT:    sw a4, 0(sp)
+; RV32I-NEXT:    sw a5, 4(sp)
 ; RV32I-NEXT:    addi a4, zero, 4
 ; RV32I-NEXT:    addi a5, zero, 2
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
-; RV32I-NEXT:    lw a1, 4(sp)
-; RV32I-NEXT:    lw a2, 0(sp)
+; RV32I-NEXT:    lw a5, 4(sp)
+; RV32I-NEXT:    lw a4, 0(sp)
 ; RV32I-NEXT:    bnez a0, .LBB203_7
-; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32I-NEXT:    beq a1, s0, .LBB203_4
-; RV32I-NEXT:  .LBB203_3: # %atomicrmw.start
-; RV32I-NEXT:    slt a0, s0, a1
+; RV32I-NEXT:  .LBB203_2: # %atomicrmw.start
+; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32I-NEXT:    beq a5, s1, .LBB203_4
+; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB203_2 Depth=1
+; RV32I-NEXT:    slt a0, s1, a5
 ; RV32I-NEXT:    j .LBB203_5
 ; RV32I-NEXT:  .LBB203_4:
-; RV32I-NEXT:    sltu a0, s2, a2
+; RV32I-NEXT:    sltu a0, s2, a4
 ; RV32I-NEXT:  .LBB203_5: # %atomicrmw.start
-; RV32I-NEXT:    sw a2, 0(sp)
-; RV32I-NEXT:    mv a3, a1
+; RV32I-NEXT:    # in Loop: Header=BB203_2 Depth=1
+; RV32I-NEXT:    mv a2, a4
+; RV32I-NEXT:    mv a3, a5
 ; RV32I-NEXT:    bnez a0, .LBB203_1
 ; RV32I-NEXT:  # %bb.6: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB203_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
-; RV32I-NEXT:    mv a3, s0
+; RV32I-NEXT:    mv a3, s1
 ; RV32I-NEXT:    j .LBB203_1
 ; RV32I-NEXT:  .LBB203_7: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a4
+; RV32I-NEXT:    mv a1, a5
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -15300,41 +15365,47 @@ define i64 @atomicrmw_max_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    sw s1, 20(sp)
 ; RV32IA-NEXT:    sw s2, 16(sp)
 ; RV32IA-NEXT:    sw s3, 12(sp)
-; RV32IA-NEXT:    mv s0, a2
+; RV32IA-NEXT:    mv s0, a0
+; RV32IA-NEXT:    lw a5, 4(a0)
+; RV32IA-NEXT:    lw a4, 0(a0)
+; RV32IA-NEXT:    mv s1, a2
 ; RV32IA-NEXT:    mv s2, a1
-; RV32IA-NEXT:    mv s1, a0
-; RV32IA-NEXT:    lw a1, 4(a0)
-; RV32IA-NEXT:    lw a2, 0(a0)
 ; RV32IA-NEXT:    mv s3, sp
-; RV32IA-NEXT:    bne a1, s0, .LBB203_3
-; RV32IA-NEXT:    j .LBB203_4
+; RV32IA-NEXT:    j .LBB203_2
 ; RV32IA-NEXT:  .LBB203_1: # %atomicrmw.start
-; RV32IA-NEXT:    sw a1, 4(sp)
-; RV32IA-NEXT:    mv a0, s1
-; RV32IA-NEXT:    mv a1, s3
+; RV32IA-NEXT:    # in Loop: Header=BB203_2 Depth=1
+; RV32IA-NEXT:    sw a4, 0(sp)
+; RV32IA-NEXT:    sw a5, 4(sp)
 ; RV32IA-NEXT:    addi a4, zero, 4
 ; RV32IA-NEXT:    addi a5, zero, 2
+; RV32IA-NEXT:    mv a0, s0
+; RV32IA-NEXT:    mv a1, s3
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
-; RV32IA-NEXT:    lw a1, 4(sp)
-; RV32IA-NEXT:    lw a2, 0(sp)
+; RV32IA-NEXT:    lw a5, 4(sp)
+; RV32IA-NEXT:    lw a4, 0(sp)
 ; RV32IA-NEXT:    bnez a0, .LBB203_7
-; RV32IA-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32IA-NEXT:    beq a1, s0, .LBB203_4
-; RV32IA-NEXT:  .LBB203_3: # %atomicrmw.start
-; RV32IA-NEXT:    slt a0, s0, a1
+; RV32IA-NEXT:  .LBB203_2: # %atomicrmw.start
+; RV32IA-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32IA-NEXT:    beq a5, s1, .LBB203_4
+; RV32IA-NEXT:  # %bb.3: # %atomicrmw.start
+; RV32IA-NEXT:    # in Loop: Header=BB203_2 Depth=1
+; RV32IA-NEXT:    slt a0, s1, a5
 ; RV32IA-NEXT:    j .LBB203_5
 ; RV32IA-NEXT:  .LBB203_4:
-; RV32IA-NEXT:    sltu a0, s2, a2
+; RV32IA-NEXT:    sltu a0, s2, a4
 ; RV32IA-NEXT:  .LBB203_5: # %atomicrmw.start
-; RV32IA-NEXT:    sw a2, 0(sp)
-; RV32IA-NEXT:    mv a3, a1
+; RV32IA-NEXT:    # in Loop: Header=BB203_2 Depth=1
+; RV32IA-NEXT:    mv a2, a4
+; RV32IA-NEXT:    mv a3, a5
 ; RV32IA-NEXT:    bnez a0, .LBB203_1
 ; RV32IA-NEXT:  # %bb.6: # %atomicrmw.start
+; RV32IA-NEXT:    # in Loop: Header=BB203_2 Depth=1
 ; RV32IA-NEXT:    mv a2, s2
-; RV32IA-NEXT:    mv a3, s0
+; RV32IA-NEXT:    mv a3, s1
 ; RV32IA-NEXT:    j .LBB203_1
 ; RV32IA-NEXT:  .LBB203_7: # %atomicrmw.end
-; RV32IA-NEXT:    mv a0, a2
+; RV32IA-NEXT:    mv a0, a4
+; RV32IA-NEXT:    mv a1, a5
 ; RV32IA-NEXT:    lw s3, 12(sp)
 ; RV32IA-NEXT:    lw s2, 16(sp)
 ; RV32IA-NEXT:    lw s1, 20(sp)
@@ -15350,30 +15421,31 @@ define i64 @atomicrmw_max_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    sd s0, 32(sp)
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
-; RV64I-NEXT:    mv s0, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    ld a2, 0(a0)
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    ld a3, 0(a0)
+; RV64I-NEXT:    mv s1, a1
 ; RV64I-NEXT:    addi s2, sp, 8
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    bge s0, a2, .LBB203_3
+; RV64I-NEXT:    j .LBB203_2
 ; RV64I-NEXT:  .LBB203_1: # %atomicrmw.start
-; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s2
+; RV64I-NEXT:    # in Loop: Header=BB203_2 Depth=1
+; RV64I-NEXT:    sd a3, 8(sp)
 ; RV64I-NEXT:    addi a3, zero, 4
 ; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    call __atomic_compare_exchange_8
-; RV64I-NEXT:    ld a2, 8(sp)
+; RV64I-NEXT:    ld a3, 8(sp)
 ; RV64I-NEXT:    bnez a0, .LBB203_4
-; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV64I-NEXT:    # in Loop: Header=BB203_1 Depth=1
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    blt s0, a2, .LBB203_1
-; RV64I-NEXT:  .LBB203_3: # %atomicrmw.start
-; RV64I-NEXT:    mv a2, s0
+; RV64I-NEXT:  .LBB203_2: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    blt s1, a3, .LBB203_1
+; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB203_2 Depth=1
+; RV64I-NEXT:    mv a2, s1
 ; RV64I-NEXT:    j .LBB203_1
 ; RV64I-NEXT:  .LBB203_4: # %atomicrmw.end
-; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
 ; RV64I-NEXT:    ld s0, 32(sp)
@@ -15398,41 +15470,47 @@ define i64 @atomicrmw_max_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
 ; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    mv s0, a2
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a5, 4(a0)
+; RV32I-NEXT:    lw a4, 0(a0)
+; RV32I-NEXT:    mv s1, a2
 ; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a1, 4(a0)
-; RV32I-NEXT:    lw a2, 0(a0)
 ; RV32I-NEXT:    mv s3, sp
-; RV32I-NEXT:    bne a1, s0, .LBB204_3
-; RV32I-NEXT:    j .LBB204_4
+; RV32I-NEXT:    j .LBB204_2
 ; RV32I-NEXT:  .LBB204_1: # %atomicrmw.start
-; RV32I-NEXT:    sw a1, 4(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    # in Loop: Header=BB204_2 Depth=1
+; RV32I-NEXT:    sw a4, 0(sp)
+; RV32I-NEXT:    sw a5, 4(sp)
 ; RV32I-NEXT:    addi a4, zero, 5
 ; RV32I-NEXT:    addi a5, zero, 5
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
-; RV32I-NEXT:    lw a1, 4(sp)
-; RV32I-NEXT:    lw a2, 0(sp)
+; RV32I-NEXT:    lw a5, 4(sp)
+; RV32I-NEXT:    lw a4, 0(sp)
 ; RV32I-NEXT:    bnez a0, .LBB204_7
-; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32I-NEXT:    beq a1, s0, .LBB204_4
-; RV32I-NEXT:  .LBB204_3: # %atomicrmw.start
-; RV32I-NEXT:    slt a0, s0, a1
+; RV32I-NEXT:  .LBB204_2: # %atomicrmw.start
+; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32I-NEXT:    beq a5, s1, .LBB204_4
+; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB204_2 Depth=1
+; RV32I-NEXT:    slt a0, s1, a5
 ; RV32I-NEXT:    j .LBB204_5
 ; RV32I-NEXT:  .LBB204_4:
-; RV32I-NEXT:    sltu a0, s2, a2
+; RV32I-NEXT:    sltu a0, s2, a4
 ; RV32I-NEXT:  .LBB204_5: # %atomicrmw.start
-; RV32I-NEXT:    sw a2, 0(sp)
-; RV32I-NEXT:    mv a3, a1
+; RV32I-NEXT:    # in Loop: Header=BB204_2 Depth=1
+; RV32I-NEXT:    mv a2, a4
+; RV32I-NEXT:    mv a3, a5
 ; RV32I-NEXT:    bnez a0, .LBB204_1
 ; RV32I-NEXT:  # %bb.6: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB204_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
-; RV32I-NEXT:    mv a3, s0
+; RV32I-NEXT:    mv a3, s1
 ; RV32I-NEXT:    j .LBB204_1
 ; RV32I-NEXT:  .LBB204_7: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a4
+; RV32I-NEXT:    mv a1, a5
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -15449,41 +15527,47 @@ define i64 @atomicrmw_max_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    sw s1, 20(sp)
 ; RV32IA-NEXT:    sw s2, 16(sp)
 ; RV32IA-NEXT:    sw s3, 12(sp)
-; RV32IA-NEXT:    mv s0, a2
+; RV32IA-NEXT:    mv s0, a0
+; RV32IA-NEXT:    lw a5, 4(a0)
+; RV32IA-NEXT:    lw a4, 0(a0)
+; RV32IA-NEXT:    mv s1, a2
 ; RV32IA-NEXT:    mv s2, a1
-; RV32IA-NEXT:    mv s1, a0
-; RV32IA-NEXT:    lw a1, 4(a0)
-; RV32IA-NEXT:    lw a2, 0(a0)
 ; RV32IA-NEXT:    mv s3, sp
-; RV32IA-NEXT:    bne a1, s0, .LBB204_3
-; RV32IA-NEXT:    j .LBB204_4
+; RV32IA-NEXT:    j .LBB204_2
 ; RV32IA-NEXT:  .LBB204_1: # %atomicrmw.start
-; RV32IA-NEXT:    sw a1, 4(sp)
-; RV32IA-NEXT:    mv a0, s1
-; RV32IA-NEXT:    mv a1, s3
+; RV32IA-NEXT:    # in Loop: Header=BB204_2 Depth=1
+; RV32IA-NEXT:    sw a4, 0(sp)
+; RV32IA-NEXT:    sw a5, 4(sp)
 ; RV32IA-NEXT:    addi a4, zero, 5
 ; RV32IA-NEXT:    addi a5, zero, 5
+; RV32IA-NEXT:    mv a0, s0
+; RV32IA-NEXT:    mv a1, s3
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
-; RV32IA-NEXT:    lw a1, 4(sp)
-; RV32IA-NEXT:    lw a2, 0(sp)
+; RV32IA-NEXT:    lw a5, 4(sp)
+; RV32IA-NEXT:    lw a4, 0(sp)
 ; RV32IA-NEXT:    bnez a0, .LBB204_7
-; RV32IA-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32IA-NEXT:    beq a1, s0, .LBB204_4
-; RV32IA-NEXT:  .LBB204_3: # %atomicrmw.start
-; RV32IA-NEXT:    slt a0, s0, a1
+; RV32IA-NEXT:  .LBB204_2: # %atomicrmw.start
+; RV32IA-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32IA-NEXT:    beq a5, s1, .LBB204_4
+; RV32IA-NEXT:  # %bb.3: # %atomicrmw.start
+; RV32IA-NEXT:    # in Loop: Header=BB204_2 Depth=1
+; RV32IA-NEXT:    slt a0, s1, a5
 ; RV32IA-NEXT:    j .LBB204_5
 ; RV32IA-NEXT:  .LBB204_4:
-; RV32IA-NEXT:    sltu a0, s2, a2
+; RV32IA-NEXT:    sltu a0, s2, a4
 ; RV32IA-NEXT:  .LBB204_5: # %atomicrmw.start
-; RV32IA-NEXT:    sw a2, 0(sp)
-; RV32IA-NEXT:    mv a3, a1
+; RV32IA-NEXT:    # in Loop: Header=BB204_2 Depth=1
+; RV32IA-NEXT:    mv a2, a4
+; RV32IA-NEXT:    mv a3, a5
 ; RV32IA-NEXT:    bnez a0, .LBB204_1
 ; RV32IA-NEXT:  # %bb.6: # %atomicrmw.start
+; RV32IA-NEXT:    # in Loop: Header=BB204_2 Depth=1
 ; RV32IA-NEXT:    mv a2, s2
-; RV32IA-NEXT:    mv a3, s0
+; RV32IA-NEXT:    mv a3, s1
 ; RV32IA-NEXT:    j .LBB204_1
 ; RV32IA-NEXT:  .LBB204_7: # %atomicrmw.end
-; RV32IA-NEXT:    mv a0, a2
+; RV32IA-NEXT:    mv a0, a4
+; RV32IA-NEXT:    mv a1, a5
 ; RV32IA-NEXT:    lw s3, 12(sp)
 ; RV32IA-NEXT:    lw s2, 16(sp)
 ; RV32IA-NEXT:    lw s1, 20(sp)
@@ -15499,30 +15583,31 @@ define i64 @atomicrmw_max_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    sd s0, 32(sp)
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
-; RV64I-NEXT:    mv s0, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    ld a2, 0(a0)
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    ld a3, 0(a0)
+; RV64I-NEXT:    mv s1, a1
 ; RV64I-NEXT:    addi s2, sp, 8
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    bge s0, a2, .LBB204_3
+; RV64I-NEXT:    j .LBB204_2
 ; RV64I-NEXT:  .LBB204_1: # %atomicrmw.start
-; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s2
+; RV64I-NEXT:    # in Loop: Header=BB204_2 Depth=1
+; RV64I-NEXT:    sd a3, 8(sp)
 ; RV64I-NEXT:    addi a3, zero, 5
 ; RV64I-NEXT:    addi a4, zero, 5
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    call __atomic_compare_exchange_8
-; RV64I-NEXT:    ld a2, 8(sp)
+; RV64I-NEXT:    ld a3, 8(sp)
 ; RV64I-NEXT:    bnez a0, .LBB204_4
-; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV64I-NEXT:    # in Loop: Header=BB204_1 Depth=1
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    blt s0, a2, .LBB204_1
-; RV64I-NEXT:  .LBB204_3: # %atomicrmw.start
-; RV64I-NEXT:    mv a2, s0
+; RV64I-NEXT:  .LBB204_2: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    blt s1, a3, .LBB204_1
+; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB204_2 Depth=1
+; RV64I-NEXT:    mv a2, s1
 ; RV64I-NEXT:    j .LBB204_1
 ; RV64I-NEXT:  .LBB204_4: # %atomicrmw.end
-; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
 ; RV64I-NEXT:    ld s0, 32(sp)
@@ -15547,42 +15632,48 @@ define i64 @atomicrmw_min_i64_monotonic(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
 ; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    mv s0, a2
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a5, 4(a0)
+; RV32I-NEXT:    lw a4, 0(a0)
+; RV32I-NEXT:    mv s1, a2
 ; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a1, 4(a0)
-; RV32I-NEXT:    lw a2, 0(a0)
 ; RV32I-NEXT:    mv s3, sp
-; RV32I-NEXT:    bne a1, s0, .LBB205_3
-; RV32I-NEXT:    j .LBB205_4
+; RV32I-NEXT:    j .LBB205_2
 ; RV32I-NEXT:  .LBB205_1: # %atomicrmw.start
-; RV32I-NEXT:    sw a1, 4(sp)
-; RV32I-NEXT:    mv a0, s1
+; RV32I-NEXT:    # in Loop: Header=BB205_2 Depth=1
+; RV32I-NEXT:    sw a4, 0(sp)
+; RV32I-NEXT:    sw a5, 4(sp)
+; RV32I-NEXT:    mv a0, s0
 ; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    mv a5, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
-; RV32I-NEXT:    lw a1, 4(sp)
-; RV32I-NEXT:    lw a2, 0(sp)
+; RV32I-NEXT:    lw a5, 4(sp)
+; RV32I-NEXT:    lw a4, 0(sp)
 ; RV32I-NEXT:    bnez a0, .LBB205_7
-; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32I-NEXT:    beq a1, s0, .LBB205_4
-; RV32I-NEXT:  .LBB205_3: # %atomicrmw.start
-; RV32I-NEXT:    slt a0, s0, a1
+; RV32I-NEXT:  .LBB205_2: # %atomicrmw.start
+; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32I-NEXT:    beq a5, s1, .LBB205_4
+; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB205_2 Depth=1
+; RV32I-NEXT:    slt a0, s1, a5
 ; RV32I-NEXT:    j .LBB205_5
 ; RV32I-NEXT:  .LBB205_4:
-; RV32I-NEXT:    sltu a0, s2, a2
+; RV32I-NEXT:    sltu a0, s2, a4
 ; RV32I-NEXT:  .LBB205_5: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB205_2 Depth=1
 ; RV32I-NEXT:    xori a0, a0, 1
-; RV32I-NEXT:    sw a2, 0(sp)
-; RV32I-NEXT:    mv a3, a1
+; RV32I-NEXT:    mv a2, a4
+; RV32I-NEXT:    mv a3, a5
 ; RV32I-NEXT:    bnez a0, .LBB205_1
 ; RV32I-NEXT:  # %bb.6: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB205_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
-; RV32I-NEXT:    mv a3, s0
+; RV32I-NEXT:    mv a3, s1
 ; RV32I-NEXT:    j .LBB205_1
 ; RV32I-NEXT:  .LBB205_7: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a4
+; RV32I-NEXT:    mv a1, a5
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -15599,42 +15690,48 @@ define i64 @atomicrmw_min_i64_monotonic(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    sw s1, 20(sp)
 ; RV32IA-NEXT:    sw s2, 16(sp)
 ; RV32IA-NEXT:    sw s3, 12(sp)
-; RV32IA-NEXT:    mv s0, a2
+; RV32IA-NEXT:    mv s0, a0
+; RV32IA-NEXT:    lw a5, 4(a0)
+; RV32IA-NEXT:    lw a4, 0(a0)
+; RV32IA-NEXT:    mv s1, a2
 ; RV32IA-NEXT:    mv s2, a1
-; RV32IA-NEXT:    mv s1, a0
-; RV32IA-NEXT:    lw a1, 4(a0)
-; RV32IA-NEXT:    lw a2, 0(a0)
 ; RV32IA-NEXT:    mv s3, sp
-; RV32IA-NEXT:    bne a1, s0, .LBB205_3
-; RV32IA-NEXT:    j .LBB205_4
+; RV32IA-NEXT:    j .LBB205_2
 ; RV32IA-NEXT:  .LBB205_1: # %atomicrmw.start
-; RV32IA-NEXT:    sw a1, 4(sp)
-; RV32IA-NEXT:    mv a0, s1
+; RV32IA-NEXT:    # in Loop: Header=BB205_2 Depth=1
+; RV32IA-NEXT:    sw a4, 0(sp)
+; RV32IA-NEXT:    sw a5, 4(sp)
+; RV32IA-NEXT:    mv a0, s0
 ; RV32IA-NEXT:    mv a1, s3
 ; RV32IA-NEXT:    mv a4, zero
 ; RV32IA-NEXT:    mv a5, zero
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
-; RV32IA-NEXT:    lw a1, 4(sp)
-; RV32IA-NEXT:    lw a2, 0(sp)
+; RV32IA-NEXT:    lw a5, 4(sp)
+; RV32IA-NEXT:    lw a4, 0(sp)
 ; RV32IA-NEXT:    bnez a0, .LBB205_7
-; RV32IA-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32IA-NEXT:    beq a1, s0, .LBB205_4
-; RV32IA-NEXT:  .LBB205_3: # %atomicrmw.start
-; RV32IA-NEXT:    slt a0, s0, a1
+; RV32IA-NEXT:  .LBB205_2: # %atomicrmw.start
+; RV32IA-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32IA-NEXT:    beq a5, s1, .LBB205_4
+; RV32IA-NEXT:  # %bb.3: # %atomicrmw.start
+; RV32IA-NEXT:    # in Loop: Header=BB205_2 Depth=1
+; RV32IA-NEXT:    slt a0, s1, a5
 ; RV32IA-NEXT:    j .LBB205_5
 ; RV32IA-NEXT:  .LBB205_4:
-; RV32IA-NEXT:    sltu a0, s2, a2
+; RV32IA-NEXT:    sltu a0, s2, a4
 ; RV32IA-NEXT:  .LBB205_5: # %atomicrmw.start
+; RV32IA-NEXT:    # in Loop: Header=BB205_2 Depth=1
 ; RV32IA-NEXT:    xori a0, a0, 1
-; RV32IA-NEXT:    sw a2, 0(sp)
-; RV32IA-NEXT:    mv a3, a1
+; RV32IA-NEXT:    mv a2, a4
+; RV32IA-NEXT:    mv a3, a5
 ; RV32IA-NEXT:    bnez a0, .LBB205_1
 ; RV32IA-NEXT:  # %bb.6: # %atomicrmw.start
+; RV32IA-NEXT:    # in Loop: Header=BB205_2 Depth=1
 ; RV32IA-NEXT:    mv a2, s2
-; RV32IA-NEXT:    mv a3, s0
+; RV32IA-NEXT:    mv a3, s1
 ; RV32IA-NEXT:    j .LBB205_1
 ; RV32IA-NEXT:  .LBB205_7: # %atomicrmw.end
-; RV32IA-NEXT:    mv a0, a2
+; RV32IA-NEXT:    mv a0, a4
+; RV32IA-NEXT:    mv a1, a5
 ; RV32IA-NEXT:    lw s3, 12(sp)
 ; RV32IA-NEXT:    lw s2, 16(sp)
 ; RV32IA-NEXT:    lw s1, 20(sp)
@@ -15650,30 +15747,31 @@ define i64 @atomicrmw_min_i64_monotonic(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    sd s0, 32(sp)
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
-; RV64I-NEXT:    mv s0, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    ld a2, 0(a0)
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    ld a3, 0(a0)
+; RV64I-NEXT:    mv s1, a1
 ; RV64I-NEXT:    addi s2, sp, 8
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    blt s0, a2, .LBB205_3
+; RV64I-NEXT:    j .LBB205_2
 ; RV64I-NEXT:  .LBB205_1: # %atomicrmw.start
-; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    mv a0, s1
+; RV64I-NEXT:    # in Loop: Header=BB205_2 Depth=1
+; RV64I-NEXT:    sd a3, 8(sp)
+; RV64I-NEXT:    mv a0, s0
 ; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    mv a3, zero
 ; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    call __atomic_compare_exchange_8
-; RV64I-NEXT:    ld a2, 8(sp)
+; RV64I-NEXT:    ld a3, 8(sp)
 ; RV64I-NEXT:    bnez a0, .LBB205_4
-; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV64I-NEXT:    # in Loop: Header=BB205_1 Depth=1
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    bge s0, a2, .LBB205_1
-; RV64I-NEXT:  .LBB205_3: # %atomicrmw.start
-; RV64I-NEXT:    mv a2, s0
+; RV64I-NEXT:  .LBB205_2: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bge s1, a3, .LBB205_1
+; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB205_2 Depth=1
+; RV64I-NEXT:    mv a2, s1
 ; RV64I-NEXT:    j .LBB205_1
 ; RV64I-NEXT:  .LBB205_4: # %atomicrmw.end
-; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
 ; RV64I-NEXT:    ld s0, 32(sp)
@@ -15698,42 +15796,48 @@ define i64 @atomicrmw_min_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
 ; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    mv s0, a2
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a5, 4(a0)
+; RV32I-NEXT:    lw a4, 0(a0)
+; RV32I-NEXT:    mv s1, a2
 ; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a1, 4(a0)
-; RV32I-NEXT:    lw a2, 0(a0)
 ; RV32I-NEXT:    mv s3, sp
-; RV32I-NEXT:    bne a1, s0, .LBB206_3
-; RV32I-NEXT:    j .LBB206_4
+; RV32I-NEXT:    j .LBB206_2
 ; RV32I-NEXT:  .LBB206_1: # %atomicrmw.start
-; RV32I-NEXT:    sw a1, 4(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    # in Loop: Header=BB206_2 Depth=1
+; RV32I-NEXT:    sw a4, 0(sp)
+; RV32I-NEXT:    sw a5, 4(sp)
 ; RV32I-NEXT:    addi a4, zero, 2
 ; RV32I-NEXT:    addi a5, zero, 2
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
-; RV32I-NEXT:    lw a1, 4(sp)
-; RV32I-NEXT:    lw a2, 0(sp)
+; RV32I-NEXT:    lw a5, 4(sp)
+; RV32I-NEXT:    lw a4, 0(sp)
 ; RV32I-NEXT:    bnez a0, .LBB206_7
-; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32I-NEXT:    beq a1, s0, .LBB206_4
-; RV32I-NEXT:  .LBB206_3: # %atomicrmw.start
-; RV32I-NEXT:    slt a0, s0, a1
+; RV32I-NEXT:  .LBB206_2: # %atomicrmw.start
+; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32I-NEXT:    beq a5, s1, .LBB206_4
+; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB206_2 Depth=1
+; RV32I-NEXT:    slt a0, s1, a5
 ; RV32I-NEXT:    j .LBB206_5
 ; RV32I-NEXT:  .LBB206_4:
-; RV32I-NEXT:    sltu a0, s2, a2
+; RV32I-NEXT:    sltu a0, s2, a4
 ; RV32I-NEXT:  .LBB206_5: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB206_2 Depth=1
 ; RV32I-NEXT:    xori a0, a0, 1
-; RV32I-NEXT:    sw a2, 0(sp)
-; RV32I-NEXT:    mv a3, a1
+; RV32I-NEXT:    mv a2, a4
+; RV32I-NEXT:    mv a3, a5
 ; RV32I-NEXT:    bnez a0, .LBB206_1
 ; RV32I-NEXT:  # %bb.6: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB206_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
-; RV32I-NEXT:    mv a3, s0
+; RV32I-NEXT:    mv a3, s1
 ; RV32I-NEXT:    j .LBB206_1
 ; RV32I-NEXT:  .LBB206_7: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a4
+; RV32I-NEXT:    mv a1, a5
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -15750,42 +15854,48 @@ define i64 @atomicrmw_min_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    sw s1, 20(sp)
 ; RV32IA-NEXT:    sw s2, 16(sp)
 ; RV32IA-NEXT:    sw s3, 12(sp)
-; RV32IA-NEXT:    mv s0, a2
+; RV32IA-NEXT:    mv s0, a0
+; RV32IA-NEXT:    lw a5, 4(a0)
+; RV32IA-NEXT:    lw a4, 0(a0)
+; RV32IA-NEXT:    mv s1, a2
 ; RV32IA-NEXT:    mv s2, a1
-; RV32IA-NEXT:    mv s1, a0
-; RV32IA-NEXT:    lw a1, 4(a0)
-; RV32IA-NEXT:    lw a2, 0(a0)
 ; RV32IA-NEXT:    mv s3, sp
-; RV32IA-NEXT:    bne a1, s0, .LBB206_3
-; RV32IA-NEXT:    j .LBB206_4
+; RV32IA-NEXT:    j .LBB206_2
 ; RV32IA-NEXT:  .LBB206_1: # %atomicrmw.start
-; RV32IA-NEXT:    sw a1, 4(sp)
-; RV32IA-NEXT:    mv a0, s1
-; RV32IA-NEXT:    mv a1, s3
+; RV32IA-NEXT:    # in Loop: Header=BB206_2 Depth=1
+; RV32IA-NEXT:    sw a4, 0(sp)
+; RV32IA-NEXT:    sw a5, 4(sp)
 ; RV32IA-NEXT:    addi a4, zero, 2
 ; RV32IA-NEXT:    addi a5, zero, 2
+; RV32IA-NEXT:    mv a0, s0
+; RV32IA-NEXT:    mv a1, s3
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
-; RV32IA-NEXT:    lw a1, 4(sp)
-; RV32IA-NEXT:    lw a2, 0(sp)
+; RV32IA-NEXT:    lw a5, 4(sp)
+; RV32IA-NEXT:    lw a4, 0(sp)
 ; RV32IA-NEXT:    bnez a0, .LBB206_7
-; RV32IA-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32IA-NEXT:    beq a1, s0, .LBB206_4
-; RV32IA-NEXT:  .LBB206_3: # %atomicrmw.start
-; RV32IA-NEXT:    slt a0, s0, a1
+; RV32IA-NEXT:  .LBB206_2: # %atomicrmw.start
+; RV32IA-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32IA-NEXT:    beq a5, s1, .LBB206_4
+; RV32IA-NEXT:  # %bb.3: # %atomicrmw.start
+; RV32IA-NEXT:    # in Loop: Header=BB206_2 Depth=1
+; RV32IA-NEXT:    slt a0, s1, a5
 ; RV32IA-NEXT:    j .LBB206_5
 ; RV32IA-NEXT:  .LBB206_4:
-; RV32IA-NEXT:    sltu a0, s2, a2
+; RV32IA-NEXT:    sltu a0, s2, a4
 ; RV32IA-NEXT:  .LBB206_5: # %atomicrmw.start
+; RV32IA-NEXT:    # in Loop: Header=BB206_2 Depth=1
 ; RV32IA-NEXT:    xori a0, a0, 1
-; RV32IA-NEXT:    sw a2, 0(sp)
-; RV32IA-NEXT:    mv a3, a1
+; RV32IA-NEXT:    mv a2, a4
+; RV32IA-NEXT:    mv a3, a5
 ; RV32IA-NEXT:    bnez a0, .LBB206_1
 ; RV32IA-NEXT:  # %bb.6: # %atomicrmw.start
+; RV32IA-NEXT:    # in Loop: Header=BB206_2 Depth=1
 ; RV32IA-NEXT:    mv a2, s2
-; RV32IA-NEXT:    mv a3, s0
+; RV32IA-NEXT:    mv a3, s1
 ; RV32IA-NEXT:    j .LBB206_1
 ; RV32IA-NEXT:  .LBB206_7: # %atomicrmw.end
-; RV32IA-NEXT:    mv a0, a2
+; RV32IA-NEXT:    mv a0, a4
+; RV32IA-NEXT:    mv a1, a5
 ; RV32IA-NEXT:    lw s3, 12(sp)
 ; RV32IA-NEXT:    lw s2, 16(sp)
 ; RV32IA-NEXT:    lw s1, 20(sp)
@@ -15801,30 +15911,31 @@ define i64 @atomicrmw_min_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    sd s0, 32(sp)
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
-; RV64I-NEXT:    mv s0, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    ld a2, 0(a0)
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    ld a3, 0(a0)
+; RV64I-NEXT:    mv s1, a1
 ; RV64I-NEXT:    addi s2, sp, 8
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    blt s0, a2, .LBB206_3
+; RV64I-NEXT:    j .LBB206_2
 ; RV64I-NEXT:  .LBB206_1: # %atomicrmw.start
-; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s2
+; RV64I-NEXT:    # in Loop: Header=BB206_2 Depth=1
+; RV64I-NEXT:    sd a3, 8(sp)
 ; RV64I-NEXT:    addi a3, zero, 2
 ; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    call __atomic_compare_exchange_8
-; RV64I-NEXT:    ld a2, 8(sp)
+; RV64I-NEXT:    ld a3, 8(sp)
 ; RV64I-NEXT:    bnez a0, .LBB206_4
-; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV64I-NEXT:    # in Loop: Header=BB206_1 Depth=1
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    bge s0, a2, .LBB206_1
-; RV64I-NEXT:  .LBB206_3: # %atomicrmw.start
-; RV64I-NEXT:    mv a2, s0
+; RV64I-NEXT:  .LBB206_2: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bge s1, a3, .LBB206_1
+; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB206_2 Depth=1
+; RV64I-NEXT:    mv a2, s1
 ; RV64I-NEXT:    j .LBB206_1
 ; RV64I-NEXT:  .LBB206_4: # %atomicrmw.end
-; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
 ; RV64I-NEXT:    ld s0, 32(sp)
@@ -15849,42 +15960,48 @@ define i64 @atomicrmw_min_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
 ; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    mv s0, a2
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a5, 4(a0)
+; RV32I-NEXT:    lw a4, 0(a0)
+; RV32I-NEXT:    mv s1, a2
 ; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a1, 4(a0)
-; RV32I-NEXT:    lw a2, 0(a0)
 ; RV32I-NEXT:    mv s3, sp
-; RV32I-NEXT:    bne a1, s0, .LBB207_3
-; RV32I-NEXT:    j .LBB207_4
+; RV32I-NEXT:    j .LBB207_2
 ; RV32I-NEXT:  .LBB207_1: # %atomicrmw.start
-; RV32I-NEXT:    sw a1, 4(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    # in Loop: Header=BB207_2 Depth=1
+; RV32I-NEXT:    sw a4, 0(sp)
+; RV32I-NEXT:    sw a5, 4(sp)
 ; RV32I-NEXT:    addi a4, zero, 3
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    mv a5, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
-; RV32I-NEXT:    lw a1, 4(sp)
-; RV32I-NEXT:    lw a2, 0(sp)
+; RV32I-NEXT:    lw a5, 4(sp)
+; RV32I-NEXT:    lw a4, 0(sp)
 ; RV32I-NEXT:    bnez a0, .LBB207_7
-; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32I-NEXT:    beq a1, s0, .LBB207_4
-; RV32I-NEXT:  .LBB207_3: # %atomicrmw.start
-; RV32I-NEXT:    slt a0, s0, a1
+; RV32I-NEXT:  .LBB207_2: # %atomicrmw.start
+; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32I-NEXT:    beq a5, s1, .LBB207_4
+; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB207_2 Depth=1
+; RV32I-NEXT:    slt a0, s1, a5
 ; RV32I-NEXT:    j .LBB207_5
 ; RV32I-NEXT:  .LBB207_4:
-; RV32I-NEXT:    sltu a0, s2, a2
+; RV32I-NEXT:    sltu a0, s2, a4
 ; RV32I-NEXT:  .LBB207_5: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB207_2 Depth=1
 ; RV32I-NEXT:    xori a0, a0, 1
-; RV32I-NEXT:    sw a2, 0(sp)
-; RV32I-NEXT:    mv a3, a1
+; RV32I-NEXT:    mv a2, a4
+; RV32I-NEXT:    mv a3, a5
 ; RV32I-NEXT:    bnez a0, .LBB207_1
 ; RV32I-NEXT:  # %bb.6: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB207_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
-; RV32I-NEXT:    mv a3, s0
+; RV32I-NEXT:    mv a3, s1
 ; RV32I-NEXT:    j .LBB207_1
 ; RV32I-NEXT:  .LBB207_7: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a4
+; RV32I-NEXT:    mv a1, a5
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -15901,42 +16018,48 @@ define i64 @atomicrmw_min_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    sw s1, 20(sp)
 ; RV32IA-NEXT:    sw s2, 16(sp)
 ; RV32IA-NEXT:    sw s3, 12(sp)
-; RV32IA-NEXT:    mv s0, a2
+; RV32IA-NEXT:    mv s0, a0
+; RV32IA-NEXT:    lw a5, 4(a0)
+; RV32IA-NEXT:    lw a4, 0(a0)
+; RV32IA-NEXT:    mv s1, a2
 ; RV32IA-NEXT:    mv s2, a1
-; RV32IA-NEXT:    mv s1, a0
-; RV32IA-NEXT:    lw a1, 4(a0)
-; RV32IA-NEXT:    lw a2, 0(a0)
 ; RV32IA-NEXT:    mv s3, sp
-; RV32IA-NEXT:    bne a1, s0, .LBB207_3
-; RV32IA-NEXT:    j .LBB207_4
+; RV32IA-NEXT:    j .LBB207_2
 ; RV32IA-NEXT:  .LBB207_1: # %atomicrmw.start
-; RV32IA-NEXT:    sw a1, 4(sp)
-; RV32IA-NEXT:    mv a0, s1
-; RV32IA-NEXT:    mv a1, s3
+; RV32IA-NEXT:    # in Loop: Header=BB207_2 Depth=1
+; RV32IA-NEXT:    sw a4, 0(sp)
+; RV32IA-NEXT:    sw a5, 4(sp)
 ; RV32IA-NEXT:    addi a4, zero, 3
+; RV32IA-NEXT:    mv a0, s0
+; RV32IA-NEXT:    mv a1, s3
 ; RV32IA-NEXT:    mv a5, zero
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
-; RV32IA-NEXT:    lw a1, 4(sp)
-; RV32IA-NEXT:    lw a2, 0(sp)
+; RV32IA-NEXT:    lw a5, 4(sp)
+; RV32IA-NEXT:    lw a4, 0(sp)
 ; RV32IA-NEXT:    bnez a0, .LBB207_7
-; RV32IA-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32IA-NEXT:    beq a1, s0, .LBB207_4
-; RV32IA-NEXT:  .LBB207_3: # %atomicrmw.start
-; RV32IA-NEXT:    slt a0, s0, a1
+; RV32IA-NEXT:  .LBB207_2: # %atomicrmw.start
+; RV32IA-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32IA-NEXT:    beq a5, s1, .LBB207_4
+; RV32IA-NEXT:  # %bb.3: # %atomicrmw.start
+; RV32IA-NEXT:    # in Loop: Header=BB207_2 Depth=1
+; RV32IA-NEXT:    slt a0, s1, a5
 ; RV32IA-NEXT:    j .LBB207_5
 ; RV32IA-NEXT:  .LBB207_4:
-; RV32IA-NEXT:    sltu a0, s2, a2
+; RV32IA-NEXT:    sltu a0, s2, a4
 ; RV32IA-NEXT:  .LBB207_5: # %atomicrmw.start
+; RV32IA-NEXT:    # in Loop: Header=BB207_2 Depth=1
 ; RV32IA-NEXT:    xori a0, a0, 1
-; RV32IA-NEXT:    sw a2, 0(sp)
-; RV32IA-NEXT:    mv a3, a1
+; RV32IA-NEXT:    mv a2, a4
+; RV32IA-NEXT:    mv a3, a5
 ; RV32IA-NEXT:    bnez a0, .LBB207_1
 ; RV32IA-NEXT:  # %bb.6: # %atomicrmw.start
+; RV32IA-NEXT:    # in Loop: Header=BB207_2 Depth=1
 ; RV32IA-NEXT:    mv a2, s2
-; RV32IA-NEXT:    mv a3, s0
+; RV32IA-NEXT:    mv a3, s1
 ; RV32IA-NEXT:    j .LBB207_1
 ; RV32IA-NEXT:  .LBB207_7: # %atomicrmw.end
-; RV32IA-NEXT:    mv a0, a2
+; RV32IA-NEXT:    mv a0, a4
+; RV32IA-NEXT:    mv a1, a5
 ; RV32IA-NEXT:    lw s3, 12(sp)
 ; RV32IA-NEXT:    lw s2, 16(sp)
 ; RV32IA-NEXT:    lw s1, 20(sp)
@@ -15952,30 +16075,31 @@ define i64 @atomicrmw_min_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    sd s0, 32(sp)
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
-; RV64I-NEXT:    mv s0, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    ld a2, 0(a0)
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    ld a3, 0(a0)
+; RV64I-NEXT:    mv s1, a1
 ; RV64I-NEXT:    addi s2, sp, 8
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    blt s0, a2, .LBB207_3
+; RV64I-NEXT:    j .LBB207_2
 ; RV64I-NEXT:  .LBB207_1: # %atomicrmw.start
-; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s2
+; RV64I-NEXT:    # in Loop: Header=BB207_2 Depth=1
+; RV64I-NEXT:    sd a3, 8(sp)
 ; RV64I-NEXT:    addi a3, zero, 3
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    call __atomic_compare_exchange_8
-; RV64I-NEXT:    ld a2, 8(sp)
+; RV64I-NEXT:    ld a3, 8(sp)
 ; RV64I-NEXT:    bnez a0, .LBB207_4
-; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV64I-NEXT:    # in Loop: Header=BB207_1 Depth=1
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    bge s0, a2, .LBB207_1
-; RV64I-NEXT:  .LBB207_3: # %atomicrmw.start
-; RV64I-NEXT:    mv a2, s0
+; RV64I-NEXT:  .LBB207_2: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bge s1, a3, .LBB207_1
+; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB207_2 Depth=1
+; RV64I-NEXT:    mv a2, s1
 ; RV64I-NEXT:    j .LBB207_1
 ; RV64I-NEXT:  .LBB207_4: # %atomicrmw.end
-; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
 ; RV64I-NEXT:    ld s0, 32(sp)
@@ -16000,42 +16124,48 @@ define i64 @atomicrmw_min_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
 ; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    mv s0, a2
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a5, 4(a0)
+; RV32I-NEXT:    lw a4, 0(a0)
+; RV32I-NEXT:    mv s1, a2
 ; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a1, 4(a0)
-; RV32I-NEXT:    lw a2, 0(a0)
 ; RV32I-NEXT:    mv s3, sp
-; RV32I-NEXT:    bne a1, s0, .LBB208_3
-; RV32I-NEXT:    j .LBB208_4
+; RV32I-NEXT:    j .LBB208_2
 ; RV32I-NEXT:  .LBB208_1: # %atomicrmw.start
-; RV32I-NEXT:    sw a1, 4(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    # in Loop: Header=BB208_2 Depth=1
+; RV32I-NEXT:    sw a4, 0(sp)
+; RV32I-NEXT:    sw a5, 4(sp)
 ; RV32I-NEXT:    addi a4, zero, 4
 ; RV32I-NEXT:    addi a5, zero, 2
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
-; RV32I-NEXT:    lw a1, 4(sp)
-; RV32I-NEXT:    lw a2, 0(sp)
+; RV32I-NEXT:    lw a5, 4(sp)
+; RV32I-NEXT:    lw a4, 0(sp)
 ; RV32I-NEXT:    bnez a0, .LBB208_7
-; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32I-NEXT:    beq a1, s0, .LBB208_4
-; RV32I-NEXT:  .LBB208_3: # %atomicrmw.start
-; RV32I-NEXT:    slt a0, s0, a1
+; RV32I-NEXT:  .LBB208_2: # %atomicrmw.start
+; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32I-NEXT:    beq a5, s1, .LBB208_4
+; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB208_2 Depth=1
+; RV32I-NEXT:    slt a0, s1, a5
 ; RV32I-NEXT:    j .LBB208_5
 ; RV32I-NEXT:  .LBB208_4:
-; RV32I-NEXT:    sltu a0, s2, a2
+; RV32I-NEXT:    sltu a0, s2, a4
 ; RV32I-NEXT:  .LBB208_5: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB208_2 Depth=1
 ; RV32I-NEXT:    xori a0, a0, 1
-; RV32I-NEXT:    sw a2, 0(sp)
-; RV32I-NEXT:    mv a3, a1
+; RV32I-NEXT:    mv a2, a4
+; RV32I-NEXT:    mv a3, a5
 ; RV32I-NEXT:    bnez a0, .LBB208_1
 ; RV32I-NEXT:  # %bb.6: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB208_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
-; RV32I-NEXT:    mv a3, s0
+; RV32I-NEXT:    mv a3, s1
 ; RV32I-NEXT:    j .LBB208_1
 ; RV32I-NEXT:  .LBB208_7: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a4
+; RV32I-NEXT:    mv a1, a5
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -16052,42 +16182,48 @@ define i64 @atomicrmw_min_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    sw s1, 20(sp)
 ; RV32IA-NEXT:    sw s2, 16(sp)
 ; RV32IA-NEXT:    sw s3, 12(sp)
-; RV32IA-NEXT:    mv s0, a2
+; RV32IA-NEXT:    mv s0, a0
+; RV32IA-NEXT:    lw a5, 4(a0)
+; RV32IA-NEXT:    lw a4, 0(a0)
+; RV32IA-NEXT:    mv s1, a2
 ; RV32IA-NEXT:    mv s2, a1
-; RV32IA-NEXT:    mv s1, a0
-; RV32IA-NEXT:    lw a1, 4(a0)
-; RV32IA-NEXT:    lw a2, 0(a0)
 ; RV32IA-NEXT:    mv s3, sp
-; RV32IA-NEXT:    bne a1, s0, .LBB208_3
-; RV32IA-NEXT:    j .LBB208_4
+; RV32IA-NEXT:    j .LBB208_2
 ; RV32IA-NEXT:  .LBB208_1: # %atomicrmw.start
-; RV32IA-NEXT:    sw a1, 4(sp)
-; RV32IA-NEXT:    mv a0, s1
-; RV32IA-NEXT:    mv a1, s3
+; RV32IA-NEXT:    # in Loop: Header=BB208_2 Depth=1
+; RV32IA-NEXT:    sw a4, 0(sp)
+; RV32IA-NEXT:    sw a5, 4(sp)
 ; RV32IA-NEXT:    addi a4, zero, 4
 ; RV32IA-NEXT:    addi a5, zero, 2
+; RV32IA-NEXT:    mv a0, s0
+; RV32IA-NEXT:    mv a1, s3
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
-; RV32IA-NEXT:    lw a1, 4(sp)
-; RV32IA-NEXT:    lw a2, 0(sp)
+; RV32IA-NEXT:    lw a5, 4(sp)
+; RV32IA-NEXT:    lw a4, 0(sp)
 ; RV32IA-NEXT:    bnez a0, .LBB208_7
-; RV32IA-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32IA-NEXT:    beq a1, s0, .LBB208_4
-; RV32IA-NEXT:  .LBB208_3: # %atomicrmw.start
-; RV32IA-NEXT:    slt a0, s0, a1
+; RV32IA-NEXT:  .LBB208_2: # %atomicrmw.start
+; RV32IA-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32IA-NEXT:    beq a5, s1, .LBB208_4
+; RV32IA-NEXT:  # %bb.3: # %atomicrmw.start
+; RV32IA-NEXT:    # in Loop: Header=BB208_2 Depth=1
+; RV32IA-NEXT:    slt a0, s1, a5
 ; RV32IA-NEXT:    j .LBB208_5
 ; RV32IA-NEXT:  .LBB208_4:
-; RV32IA-NEXT:    sltu a0, s2, a2
+; RV32IA-NEXT:    sltu a0, s2, a4
 ; RV32IA-NEXT:  .LBB208_5: # %atomicrmw.start
+; RV32IA-NEXT:    # in Loop: Header=BB208_2 Depth=1
 ; RV32IA-NEXT:    xori a0, a0, 1
-; RV32IA-NEXT:    sw a2, 0(sp)
-; RV32IA-NEXT:    mv a3, a1
+; RV32IA-NEXT:    mv a2, a4
+; RV32IA-NEXT:    mv a3, a5
 ; RV32IA-NEXT:    bnez a0, .LBB208_1
 ; RV32IA-NEXT:  # %bb.6: # %atomicrmw.start
+; RV32IA-NEXT:    # in Loop: Header=BB208_2 Depth=1
 ; RV32IA-NEXT:    mv a2, s2
-; RV32IA-NEXT:    mv a3, s0
+; RV32IA-NEXT:    mv a3, s1
 ; RV32IA-NEXT:    j .LBB208_1
 ; RV32IA-NEXT:  .LBB208_7: # %atomicrmw.end
-; RV32IA-NEXT:    mv a0, a2
+; RV32IA-NEXT:    mv a0, a4
+; RV32IA-NEXT:    mv a1, a5
 ; RV32IA-NEXT:    lw s3, 12(sp)
 ; RV32IA-NEXT:    lw s2, 16(sp)
 ; RV32IA-NEXT:    lw s1, 20(sp)
@@ -16103,30 +16239,31 @@ define i64 @atomicrmw_min_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    sd s0, 32(sp)
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
-; RV64I-NEXT:    mv s0, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    ld a2, 0(a0)
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    ld a3, 0(a0)
+; RV64I-NEXT:    mv s1, a1
 ; RV64I-NEXT:    addi s2, sp, 8
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    blt s0, a2, .LBB208_3
+; RV64I-NEXT:    j .LBB208_2
 ; RV64I-NEXT:  .LBB208_1: # %atomicrmw.start
-; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s2
+; RV64I-NEXT:    # in Loop: Header=BB208_2 Depth=1
+; RV64I-NEXT:    sd a3, 8(sp)
 ; RV64I-NEXT:    addi a3, zero, 4
 ; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    call __atomic_compare_exchange_8
-; RV64I-NEXT:    ld a2, 8(sp)
+; RV64I-NEXT:    ld a3, 8(sp)
 ; RV64I-NEXT:    bnez a0, .LBB208_4
-; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV64I-NEXT:    # in Loop: Header=BB208_1 Depth=1
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    bge s0, a2, .LBB208_1
-; RV64I-NEXT:  .LBB208_3: # %atomicrmw.start
-; RV64I-NEXT:    mv a2, s0
+; RV64I-NEXT:  .LBB208_2: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bge s1, a3, .LBB208_1
+; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB208_2 Depth=1
+; RV64I-NEXT:    mv a2, s1
 ; RV64I-NEXT:    j .LBB208_1
 ; RV64I-NEXT:  .LBB208_4: # %atomicrmw.end
-; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
 ; RV64I-NEXT:    ld s0, 32(sp)
@@ -16151,42 +16288,48 @@ define i64 @atomicrmw_min_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
 ; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    mv s0, a2
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a5, 4(a0)
+; RV32I-NEXT:    lw a4, 0(a0)
+; RV32I-NEXT:    mv s1, a2
 ; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a1, 4(a0)
-; RV32I-NEXT:    lw a2, 0(a0)
 ; RV32I-NEXT:    mv s3, sp
-; RV32I-NEXT:    bne a1, s0, .LBB209_3
-; RV32I-NEXT:    j .LBB209_4
+; RV32I-NEXT:    j .LBB209_2
 ; RV32I-NEXT:  .LBB209_1: # %atomicrmw.start
-; RV32I-NEXT:    sw a1, 4(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    # in Loop: Header=BB209_2 Depth=1
+; RV32I-NEXT:    sw a4, 0(sp)
+; RV32I-NEXT:    sw a5, 4(sp)
 ; RV32I-NEXT:    addi a4, zero, 5
 ; RV32I-NEXT:    addi a5, zero, 5
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
-; RV32I-NEXT:    lw a1, 4(sp)
-; RV32I-NEXT:    lw a2, 0(sp)
+; RV32I-NEXT:    lw a5, 4(sp)
+; RV32I-NEXT:    lw a4, 0(sp)
 ; RV32I-NEXT:    bnez a0, .LBB209_7
-; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32I-NEXT:    beq a1, s0, .LBB209_4
-; RV32I-NEXT:  .LBB209_3: # %atomicrmw.start
-; RV32I-NEXT:    slt a0, s0, a1
+; RV32I-NEXT:  .LBB209_2: # %atomicrmw.start
+; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32I-NEXT:    beq a5, s1, .LBB209_4
+; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB209_2 Depth=1
+; RV32I-NEXT:    slt a0, s1, a5
 ; RV32I-NEXT:    j .LBB209_5
 ; RV32I-NEXT:  .LBB209_4:
-; RV32I-NEXT:    sltu a0, s2, a2
+; RV32I-NEXT:    sltu a0, s2, a4
 ; RV32I-NEXT:  .LBB209_5: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB209_2 Depth=1
 ; RV32I-NEXT:    xori a0, a0, 1
-; RV32I-NEXT:    sw a2, 0(sp)
-; RV32I-NEXT:    mv a3, a1
+; RV32I-NEXT:    mv a2, a4
+; RV32I-NEXT:    mv a3, a5
 ; RV32I-NEXT:    bnez a0, .LBB209_1
 ; RV32I-NEXT:  # %bb.6: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB209_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
-; RV32I-NEXT:    mv a3, s0
+; RV32I-NEXT:    mv a3, s1
 ; RV32I-NEXT:    j .LBB209_1
 ; RV32I-NEXT:  .LBB209_7: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a4
+; RV32I-NEXT:    mv a1, a5
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -16203,42 +16346,48 @@ define i64 @atomicrmw_min_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    sw s1, 20(sp)
 ; RV32IA-NEXT:    sw s2, 16(sp)
 ; RV32IA-NEXT:    sw s3, 12(sp)
-; RV32IA-NEXT:    mv s0, a2
+; RV32IA-NEXT:    mv s0, a0
+; RV32IA-NEXT:    lw a5, 4(a0)
+; RV32IA-NEXT:    lw a4, 0(a0)
+; RV32IA-NEXT:    mv s1, a2
 ; RV32IA-NEXT:    mv s2, a1
-; RV32IA-NEXT:    mv s1, a0
-; RV32IA-NEXT:    lw a1, 4(a0)
-; RV32IA-NEXT:    lw a2, 0(a0)
 ; RV32IA-NEXT:    mv s3, sp
-; RV32IA-NEXT:    bne a1, s0, .LBB209_3
-; RV32IA-NEXT:    j .LBB209_4
+; RV32IA-NEXT:    j .LBB209_2
 ; RV32IA-NEXT:  .LBB209_1: # %atomicrmw.start
-; RV32IA-NEXT:    sw a1, 4(sp)
-; RV32IA-NEXT:    mv a0, s1
-; RV32IA-NEXT:    mv a1, s3
+; RV32IA-NEXT:    # in Loop: Header=BB209_2 Depth=1
+; RV32IA-NEXT:    sw a4, 0(sp)
+; RV32IA-NEXT:    sw a5, 4(sp)
 ; RV32IA-NEXT:    addi a4, zero, 5
 ; RV32IA-NEXT:    addi a5, zero, 5
+; RV32IA-NEXT:    mv a0, s0
+; RV32IA-NEXT:    mv a1, s3
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
-; RV32IA-NEXT:    lw a1, 4(sp)
-; RV32IA-NEXT:    lw a2, 0(sp)
+; RV32IA-NEXT:    lw a5, 4(sp)
+; RV32IA-NEXT:    lw a4, 0(sp)
 ; RV32IA-NEXT:    bnez a0, .LBB209_7
-; RV32IA-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32IA-NEXT:    beq a1, s0, .LBB209_4
-; RV32IA-NEXT:  .LBB209_3: # %atomicrmw.start
-; RV32IA-NEXT:    slt a0, s0, a1
+; RV32IA-NEXT:  .LBB209_2: # %atomicrmw.start
+; RV32IA-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32IA-NEXT:    beq a5, s1, .LBB209_4
+; RV32IA-NEXT:  # %bb.3: # %atomicrmw.start
+; RV32IA-NEXT:    # in Loop: Header=BB209_2 Depth=1
+; RV32IA-NEXT:    slt a0, s1, a5
 ; RV32IA-NEXT:    j .LBB209_5
 ; RV32IA-NEXT:  .LBB209_4:
-; RV32IA-NEXT:    sltu a0, s2, a2
+; RV32IA-NEXT:    sltu a0, s2, a4
 ; RV32IA-NEXT:  .LBB209_5: # %atomicrmw.start
+; RV32IA-NEXT:    # in Loop: Header=BB209_2 Depth=1
 ; RV32IA-NEXT:    xori a0, a0, 1
-; RV32IA-NEXT:    sw a2, 0(sp)
-; RV32IA-NEXT:    mv a3, a1
+; RV32IA-NEXT:    mv a2, a4
+; RV32IA-NEXT:    mv a3, a5
 ; RV32IA-NEXT:    bnez a0, .LBB209_1
 ; RV32IA-NEXT:  # %bb.6: # %atomicrmw.start
+; RV32IA-NEXT:    # in Loop: Header=BB209_2 Depth=1
 ; RV32IA-NEXT:    mv a2, s2
-; RV32IA-NEXT:    mv a3, s0
+; RV32IA-NEXT:    mv a3, s1
 ; RV32IA-NEXT:    j .LBB209_1
 ; RV32IA-NEXT:  .LBB209_7: # %atomicrmw.end
-; RV32IA-NEXT:    mv a0, a2
+; RV32IA-NEXT:    mv a0, a4
+; RV32IA-NEXT:    mv a1, a5
 ; RV32IA-NEXT:    lw s3, 12(sp)
 ; RV32IA-NEXT:    lw s2, 16(sp)
 ; RV32IA-NEXT:    lw s1, 20(sp)
@@ -16254,30 +16403,31 @@ define i64 @atomicrmw_min_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    sd s0, 32(sp)
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
-; RV64I-NEXT:    mv s0, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    ld a2, 0(a0)
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    ld a3, 0(a0)
+; RV64I-NEXT:    mv s1, a1
 ; RV64I-NEXT:    addi s2, sp, 8
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    blt s0, a2, .LBB209_3
+; RV64I-NEXT:    j .LBB209_2
 ; RV64I-NEXT:  .LBB209_1: # %atomicrmw.start
-; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s2
+; RV64I-NEXT:    # in Loop: Header=BB209_2 Depth=1
+; RV64I-NEXT:    sd a3, 8(sp)
 ; RV64I-NEXT:    addi a3, zero, 5
 ; RV64I-NEXT:    addi a4, zero, 5
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    call __atomic_compare_exchange_8
-; RV64I-NEXT:    ld a2, 8(sp)
+; RV64I-NEXT:    ld a3, 8(sp)
 ; RV64I-NEXT:    bnez a0, .LBB209_4
-; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV64I-NEXT:    # in Loop: Header=BB209_1 Depth=1
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    bge s0, a2, .LBB209_1
-; RV64I-NEXT:  .LBB209_3: # %atomicrmw.start
-; RV64I-NEXT:    mv a2, s0
+; RV64I-NEXT:  .LBB209_2: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bge s1, a3, .LBB209_1
+; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB209_2 Depth=1
+; RV64I-NEXT:    mv a2, s1
 ; RV64I-NEXT:    j .LBB209_1
 ; RV64I-NEXT:  .LBB209_4: # %atomicrmw.end
-; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
 ; RV64I-NEXT:    ld s0, 32(sp)
@@ -16302,41 +16452,47 @@ define i64 @atomicrmw_umax_i64_monotonic(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
 ; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    mv s0, a2
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a5, 4(a0)
+; RV32I-NEXT:    lw a4, 0(a0)
+; RV32I-NEXT:    mv s1, a2
 ; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a1, 4(a0)
-; RV32I-NEXT:    lw a2, 0(a0)
 ; RV32I-NEXT:    mv s3, sp
-; RV32I-NEXT:    bne a1, s0, .LBB210_3
-; RV32I-NEXT:    j .LBB210_4
+; RV32I-NEXT:    j .LBB210_2
 ; RV32I-NEXT:  .LBB210_1: # %atomicrmw.start
-; RV32I-NEXT:    sw a1, 4(sp)
-; RV32I-NEXT:    mv a0, s1
+; RV32I-NEXT:    # in Loop: Header=BB210_2 Depth=1
+; RV32I-NEXT:    sw a4, 0(sp)
+; RV32I-NEXT:    sw a5, 4(sp)
+; RV32I-NEXT:    mv a0, s0
 ; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    mv a5, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
-; RV32I-NEXT:    lw a1, 4(sp)
-; RV32I-NEXT:    lw a2, 0(sp)
+; RV32I-NEXT:    lw a5, 4(sp)
+; RV32I-NEXT:    lw a4, 0(sp)
 ; RV32I-NEXT:    bnez a0, .LBB210_7
-; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32I-NEXT:    beq a1, s0, .LBB210_4
-; RV32I-NEXT:  .LBB210_3: # %atomicrmw.start
-; RV32I-NEXT:    sltu a0, s0, a1
+; RV32I-NEXT:  .LBB210_2: # %atomicrmw.start
+; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32I-NEXT:    beq a5, s1, .LBB210_4
+; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB210_2 Depth=1
+; RV32I-NEXT:    sltu a0, s1, a5
 ; RV32I-NEXT:    j .LBB210_5
 ; RV32I-NEXT:  .LBB210_4:
-; RV32I-NEXT:    sltu a0, s2, a2
+; RV32I-NEXT:    sltu a0, s2, a4
 ; RV32I-NEXT:  .LBB210_5: # %atomicrmw.start
-; RV32I-NEXT:    sw a2, 0(sp)
-; RV32I-NEXT:    mv a3, a1
+; RV32I-NEXT:    # in Loop: Header=BB210_2 Depth=1
+; RV32I-NEXT:    mv a2, a4
+; RV32I-NEXT:    mv a3, a5
 ; RV32I-NEXT:    bnez a0, .LBB210_1
 ; RV32I-NEXT:  # %bb.6: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB210_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
-; RV32I-NEXT:    mv a3, s0
+; RV32I-NEXT:    mv a3, s1
 ; RV32I-NEXT:    j .LBB210_1
 ; RV32I-NEXT:  .LBB210_7: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a4
+; RV32I-NEXT:    mv a1, a5
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -16353,41 +16509,47 @@ define i64 @atomicrmw_umax_i64_monotonic(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    sw s1, 20(sp)
 ; RV32IA-NEXT:    sw s2, 16(sp)
 ; RV32IA-NEXT:    sw s3, 12(sp)
-; RV32IA-NEXT:    mv s0, a2
+; RV32IA-NEXT:    mv s0, a0
+; RV32IA-NEXT:    lw a5, 4(a0)
+; RV32IA-NEXT:    lw a4, 0(a0)
+; RV32IA-NEXT:    mv s1, a2
 ; RV32IA-NEXT:    mv s2, a1
-; RV32IA-NEXT:    mv s1, a0
-; RV32IA-NEXT:    lw a1, 4(a0)
-; RV32IA-NEXT:    lw a2, 0(a0)
 ; RV32IA-NEXT:    mv s3, sp
-; RV32IA-NEXT:    bne a1, s0, .LBB210_3
-; RV32IA-NEXT:    j .LBB210_4
+; RV32IA-NEXT:    j .LBB210_2
 ; RV32IA-NEXT:  .LBB210_1: # %atomicrmw.start
-; RV32IA-NEXT:    sw a1, 4(sp)
-; RV32IA-NEXT:    mv a0, s1
+; RV32IA-NEXT:    # in Loop: Header=BB210_2 Depth=1
+; RV32IA-NEXT:    sw a4, 0(sp)
+; RV32IA-NEXT:    sw a5, 4(sp)
+; RV32IA-NEXT:    mv a0, s0
 ; RV32IA-NEXT:    mv a1, s3
 ; RV32IA-NEXT:    mv a4, zero
 ; RV32IA-NEXT:    mv a5, zero
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
-; RV32IA-NEXT:    lw a1, 4(sp)
-; RV32IA-NEXT:    lw a2, 0(sp)
+; RV32IA-NEXT:    lw a5, 4(sp)
+; RV32IA-NEXT:    lw a4, 0(sp)
 ; RV32IA-NEXT:    bnez a0, .LBB210_7
-; RV32IA-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32IA-NEXT:    beq a1, s0, .LBB210_4
-; RV32IA-NEXT:  .LBB210_3: # %atomicrmw.start
-; RV32IA-NEXT:    sltu a0, s0, a1
+; RV32IA-NEXT:  .LBB210_2: # %atomicrmw.start
+; RV32IA-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32IA-NEXT:    beq a5, s1, .LBB210_4
+; RV32IA-NEXT:  # %bb.3: # %atomicrmw.start
+; RV32IA-NEXT:    # in Loop: Header=BB210_2 Depth=1
+; RV32IA-NEXT:    sltu a0, s1, a5
 ; RV32IA-NEXT:    j .LBB210_5
 ; RV32IA-NEXT:  .LBB210_4:
-; RV32IA-NEXT:    sltu a0, s2, a2
+; RV32IA-NEXT:    sltu a0, s2, a4
 ; RV32IA-NEXT:  .LBB210_5: # %atomicrmw.start
-; RV32IA-NEXT:    sw a2, 0(sp)
-; RV32IA-NEXT:    mv a3, a1
+; RV32IA-NEXT:    # in Loop: Header=BB210_2 Depth=1
+; RV32IA-NEXT:    mv a2, a4
+; RV32IA-NEXT:    mv a3, a5
 ; RV32IA-NEXT:    bnez a0, .LBB210_1
 ; RV32IA-NEXT:  # %bb.6: # %atomicrmw.start
+; RV32IA-NEXT:    # in Loop: Header=BB210_2 Depth=1
 ; RV32IA-NEXT:    mv a2, s2
-; RV32IA-NEXT:    mv a3, s0
+; RV32IA-NEXT:    mv a3, s1
 ; RV32IA-NEXT:    j .LBB210_1
 ; RV32IA-NEXT:  .LBB210_7: # %atomicrmw.end
-; RV32IA-NEXT:    mv a0, a2
+; RV32IA-NEXT:    mv a0, a4
+; RV32IA-NEXT:    mv a1, a5
 ; RV32IA-NEXT:    lw s3, 12(sp)
 ; RV32IA-NEXT:    lw s2, 16(sp)
 ; RV32IA-NEXT:    lw s1, 20(sp)
@@ -16403,30 +16565,31 @@ define i64 @atomicrmw_umax_i64_monotonic(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    sd s0, 32(sp)
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
-; RV64I-NEXT:    mv s0, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    ld a2, 0(a0)
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    ld a3, 0(a0)
+; RV64I-NEXT:    mv s1, a1
 ; RV64I-NEXT:    addi s2, sp, 8
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    bgeu s0, a2, .LBB210_3
+; RV64I-NEXT:    j .LBB210_2
 ; RV64I-NEXT:  .LBB210_1: # %atomicrmw.start
-; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    mv a0, s1
+; RV64I-NEXT:    # in Loop: Header=BB210_2 Depth=1
+; RV64I-NEXT:    sd a3, 8(sp)
+; RV64I-NEXT:    mv a0, s0
 ; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    mv a3, zero
 ; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    call __atomic_compare_exchange_8
-; RV64I-NEXT:    ld a2, 8(sp)
+; RV64I-NEXT:    ld a3, 8(sp)
 ; RV64I-NEXT:    bnez a0, .LBB210_4
-; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV64I-NEXT:    # in Loop: Header=BB210_1 Depth=1
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    bltu s0, a2, .LBB210_1
-; RV64I-NEXT:  .LBB210_3: # %atomicrmw.start
-; RV64I-NEXT:    mv a2, s0
+; RV64I-NEXT:  .LBB210_2: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bltu s1, a3, .LBB210_1
+; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB210_2 Depth=1
+; RV64I-NEXT:    mv a2, s1
 ; RV64I-NEXT:    j .LBB210_1
 ; RV64I-NEXT:  .LBB210_4: # %atomicrmw.end
-; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
 ; RV64I-NEXT:    ld s0, 32(sp)
@@ -16451,41 +16614,47 @@ define i64 @atomicrmw_umax_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
 ; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    mv s0, a2
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a5, 4(a0)
+; RV32I-NEXT:    lw a4, 0(a0)
+; RV32I-NEXT:    mv s1, a2
 ; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a1, 4(a0)
-; RV32I-NEXT:    lw a2, 0(a0)
 ; RV32I-NEXT:    mv s3, sp
-; RV32I-NEXT:    bne a1, s0, .LBB211_3
-; RV32I-NEXT:    j .LBB211_4
+; RV32I-NEXT:    j .LBB211_2
 ; RV32I-NEXT:  .LBB211_1: # %atomicrmw.start
-; RV32I-NEXT:    sw a1, 4(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    # in Loop: Header=BB211_2 Depth=1
+; RV32I-NEXT:    sw a4, 0(sp)
+; RV32I-NEXT:    sw a5, 4(sp)
 ; RV32I-NEXT:    addi a4, zero, 2
 ; RV32I-NEXT:    addi a5, zero, 2
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
-; RV32I-NEXT:    lw a1, 4(sp)
-; RV32I-NEXT:    lw a2, 0(sp)
+; RV32I-NEXT:    lw a5, 4(sp)
+; RV32I-NEXT:    lw a4, 0(sp)
 ; RV32I-NEXT:    bnez a0, .LBB211_7
-; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32I-NEXT:    beq a1, s0, .LBB211_4
-; RV32I-NEXT:  .LBB211_3: # %atomicrmw.start
-; RV32I-NEXT:    sltu a0, s0, a1
+; RV32I-NEXT:  .LBB211_2: # %atomicrmw.start
+; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32I-NEXT:    beq a5, s1, .LBB211_4
+; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB211_2 Depth=1
+; RV32I-NEXT:    sltu a0, s1, a5
 ; RV32I-NEXT:    j .LBB211_5
 ; RV32I-NEXT:  .LBB211_4:
-; RV32I-NEXT:    sltu a0, s2, a2
+; RV32I-NEXT:    sltu a0, s2, a4
 ; RV32I-NEXT:  .LBB211_5: # %atomicrmw.start
-; RV32I-NEXT:    sw a2, 0(sp)
-; RV32I-NEXT:    mv a3, a1
+; RV32I-NEXT:    # in Loop: Header=BB211_2 Depth=1
+; RV32I-NEXT:    mv a2, a4
+; RV32I-NEXT:    mv a3, a5
 ; RV32I-NEXT:    bnez a0, .LBB211_1
 ; RV32I-NEXT:  # %bb.6: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB211_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
-; RV32I-NEXT:    mv a3, s0
+; RV32I-NEXT:    mv a3, s1
 ; RV32I-NEXT:    j .LBB211_1
 ; RV32I-NEXT:  .LBB211_7: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a4
+; RV32I-NEXT:    mv a1, a5
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -16502,41 +16671,47 @@ define i64 @atomicrmw_umax_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    sw s1, 20(sp)
 ; RV32IA-NEXT:    sw s2, 16(sp)
 ; RV32IA-NEXT:    sw s3, 12(sp)
-; RV32IA-NEXT:    mv s0, a2
+; RV32IA-NEXT:    mv s0, a0
+; RV32IA-NEXT:    lw a5, 4(a0)
+; RV32IA-NEXT:    lw a4, 0(a0)
+; RV32IA-NEXT:    mv s1, a2
 ; RV32IA-NEXT:    mv s2, a1
-; RV32IA-NEXT:    mv s1, a0
-; RV32IA-NEXT:    lw a1, 4(a0)
-; RV32IA-NEXT:    lw a2, 0(a0)
 ; RV32IA-NEXT:    mv s3, sp
-; RV32IA-NEXT:    bne a1, s0, .LBB211_3
-; RV32IA-NEXT:    j .LBB211_4
+; RV32IA-NEXT:    j .LBB211_2
 ; RV32IA-NEXT:  .LBB211_1: # %atomicrmw.start
-; RV32IA-NEXT:    sw a1, 4(sp)
-; RV32IA-NEXT:    mv a0, s1
-; RV32IA-NEXT:    mv a1, s3
+; RV32IA-NEXT:    # in Loop: Header=BB211_2 Depth=1
+; RV32IA-NEXT:    sw a4, 0(sp)
+; RV32IA-NEXT:    sw a5, 4(sp)
 ; RV32IA-NEXT:    addi a4, zero, 2
 ; RV32IA-NEXT:    addi a5, zero, 2
+; RV32IA-NEXT:    mv a0, s0
+; RV32IA-NEXT:    mv a1, s3
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
-; RV32IA-NEXT:    lw a1, 4(sp)
-; RV32IA-NEXT:    lw a2, 0(sp)
+; RV32IA-NEXT:    lw a5, 4(sp)
+; RV32IA-NEXT:    lw a4, 0(sp)
 ; RV32IA-NEXT:    bnez a0, .LBB211_7
-; RV32IA-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32IA-NEXT:    beq a1, s0, .LBB211_4
-; RV32IA-NEXT:  .LBB211_3: # %atomicrmw.start
-; RV32IA-NEXT:    sltu a0, s0, a1
+; RV32IA-NEXT:  .LBB211_2: # %atomicrmw.start
+; RV32IA-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32IA-NEXT:    beq a5, s1, .LBB211_4
+; RV32IA-NEXT:  # %bb.3: # %atomicrmw.start
+; RV32IA-NEXT:    # in Loop: Header=BB211_2 Depth=1
+; RV32IA-NEXT:    sltu a0, s1, a5
 ; RV32IA-NEXT:    j .LBB211_5
 ; RV32IA-NEXT:  .LBB211_4:
-; RV32IA-NEXT:    sltu a0, s2, a2
+; RV32IA-NEXT:    sltu a0, s2, a4
 ; RV32IA-NEXT:  .LBB211_5: # %atomicrmw.start
-; RV32IA-NEXT:    sw a2, 0(sp)
-; RV32IA-NEXT:    mv a3, a1
+; RV32IA-NEXT:    # in Loop: Header=BB211_2 Depth=1
+; RV32IA-NEXT:    mv a2, a4
+; RV32IA-NEXT:    mv a3, a5
 ; RV32IA-NEXT:    bnez a0, .LBB211_1
 ; RV32IA-NEXT:  # %bb.6: # %atomicrmw.start
+; RV32IA-NEXT:    # in Loop: Header=BB211_2 Depth=1
 ; RV32IA-NEXT:    mv a2, s2
-; RV32IA-NEXT:    mv a3, s0
+; RV32IA-NEXT:    mv a3, s1
 ; RV32IA-NEXT:    j .LBB211_1
 ; RV32IA-NEXT:  .LBB211_7: # %atomicrmw.end
-; RV32IA-NEXT:    mv a0, a2
+; RV32IA-NEXT:    mv a0, a4
+; RV32IA-NEXT:    mv a1, a5
 ; RV32IA-NEXT:    lw s3, 12(sp)
 ; RV32IA-NEXT:    lw s2, 16(sp)
 ; RV32IA-NEXT:    lw s1, 20(sp)
@@ -16552,30 +16727,31 @@ define i64 @atomicrmw_umax_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    sd s0, 32(sp)
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
-; RV64I-NEXT:    mv s0, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    ld a2, 0(a0)
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    ld a3, 0(a0)
+; RV64I-NEXT:    mv s1, a1
 ; RV64I-NEXT:    addi s2, sp, 8
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    bgeu s0, a2, .LBB211_3
+; RV64I-NEXT:    j .LBB211_2
 ; RV64I-NEXT:  .LBB211_1: # %atomicrmw.start
-; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s2
+; RV64I-NEXT:    # in Loop: Header=BB211_2 Depth=1
+; RV64I-NEXT:    sd a3, 8(sp)
 ; RV64I-NEXT:    addi a3, zero, 2
 ; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    call __atomic_compare_exchange_8
-; RV64I-NEXT:    ld a2, 8(sp)
+; RV64I-NEXT:    ld a3, 8(sp)
 ; RV64I-NEXT:    bnez a0, .LBB211_4
-; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV64I-NEXT:    # in Loop: Header=BB211_1 Depth=1
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    bltu s0, a2, .LBB211_1
-; RV64I-NEXT:  .LBB211_3: # %atomicrmw.start
-; RV64I-NEXT:    mv a2, s0
+; RV64I-NEXT:  .LBB211_2: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bltu s1, a3, .LBB211_1
+; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB211_2 Depth=1
+; RV64I-NEXT:    mv a2, s1
 ; RV64I-NEXT:    j .LBB211_1
 ; RV64I-NEXT:  .LBB211_4: # %atomicrmw.end
-; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
 ; RV64I-NEXT:    ld s0, 32(sp)
@@ -16600,41 +16776,47 @@ define i64 @atomicrmw_umax_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
 ; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    mv s0, a2
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a5, 4(a0)
+; RV32I-NEXT:    lw a4, 0(a0)
+; RV32I-NEXT:    mv s1, a2
 ; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a1, 4(a0)
-; RV32I-NEXT:    lw a2, 0(a0)
 ; RV32I-NEXT:    mv s3, sp
-; RV32I-NEXT:    bne a1, s0, .LBB212_3
-; RV32I-NEXT:    j .LBB212_4
+; RV32I-NEXT:    j .LBB212_2
 ; RV32I-NEXT:  .LBB212_1: # %atomicrmw.start
-; RV32I-NEXT:    sw a1, 4(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    # in Loop: Header=BB212_2 Depth=1
+; RV32I-NEXT:    sw a4, 0(sp)
+; RV32I-NEXT:    sw a5, 4(sp)
 ; RV32I-NEXT:    addi a4, zero, 3
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    mv a5, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
-; RV32I-NEXT:    lw a1, 4(sp)
-; RV32I-NEXT:    lw a2, 0(sp)
+; RV32I-NEXT:    lw a5, 4(sp)
+; RV32I-NEXT:    lw a4, 0(sp)
 ; RV32I-NEXT:    bnez a0, .LBB212_7
-; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32I-NEXT:    beq a1, s0, .LBB212_4
-; RV32I-NEXT:  .LBB212_3: # %atomicrmw.start
-; RV32I-NEXT:    sltu a0, s0, a1
+; RV32I-NEXT:  .LBB212_2: # %atomicrmw.start
+; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32I-NEXT:    beq a5, s1, .LBB212_4
+; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB212_2 Depth=1
+; RV32I-NEXT:    sltu a0, s1, a5
 ; RV32I-NEXT:    j .LBB212_5
 ; RV32I-NEXT:  .LBB212_4:
-; RV32I-NEXT:    sltu a0, s2, a2
+; RV32I-NEXT:    sltu a0, s2, a4
 ; RV32I-NEXT:  .LBB212_5: # %atomicrmw.start
-; RV32I-NEXT:    sw a2, 0(sp)
-; RV32I-NEXT:    mv a3, a1
+; RV32I-NEXT:    # in Loop: Header=BB212_2 Depth=1
+; RV32I-NEXT:    mv a2, a4
+; RV32I-NEXT:    mv a3, a5
 ; RV32I-NEXT:    bnez a0, .LBB212_1
 ; RV32I-NEXT:  # %bb.6: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB212_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
-; RV32I-NEXT:    mv a3, s0
+; RV32I-NEXT:    mv a3, s1
 ; RV32I-NEXT:    j .LBB212_1
 ; RV32I-NEXT:  .LBB212_7: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a4
+; RV32I-NEXT:    mv a1, a5
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -16651,41 +16833,47 @@ define i64 @atomicrmw_umax_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    sw s1, 20(sp)
 ; RV32IA-NEXT:    sw s2, 16(sp)
 ; RV32IA-NEXT:    sw s3, 12(sp)
-; RV32IA-NEXT:    mv s0, a2
+; RV32IA-NEXT:    mv s0, a0
+; RV32IA-NEXT:    lw a5, 4(a0)
+; RV32IA-NEXT:    lw a4, 0(a0)
+; RV32IA-NEXT:    mv s1, a2
 ; RV32IA-NEXT:    mv s2, a1
-; RV32IA-NEXT:    mv s1, a0
-; RV32IA-NEXT:    lw a1, 4(a0)
-; RV32IA-NEXT:    lw a2, 0(a0)
 ; RV32IA-NEXT:    mv s3, sp
-; RV32IA-NEXT:    bne a1, s0, .LBB212_3
-; RV32IA-NEXT:    j .LBB212_4
+; RV32IA-NEXT:    j .LBB212_2
 ; RV32IA-NEXT:  .LBB212_1: # %atomicrmw.start
-; RV32IA-NEXT:    sw a1, 4(sp)
-; RV32IA-NEXT:    mv a0, s1
-; RV32IA-NEXT:    mv a1, s3
+; RV32IA-NEXT:    # in Loop: Header=BB212_2 Depth=1
+; RV32IA-NEXT:    sw a4, 0(sp)
+; RV32IA-NEXT:    sw a5, 4(sp)
 ; RV32IA-NEXT:    addi a4, zero, 3
+; RV32IA-NEXT:    mv a0, s0
+; RV32IA-NEXT:    mv a1, s3
 ; RV32IA-NEXT:    mv a5, zero
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
-; RV32IA-NEXT:    lw a1, 4(sp)
-; RV32IA-NEXT:    lw a2, 0(sp)
+; RV32IA-NEXT:    lw a5, 4(sp)
+; RV32IA-NEXT:    lw a4, 0(sp)
 ; RV32IA-NEXT:    bnez a0, .LBB212_7
-; RV32IA-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32IA-NEXT:    beq a1, s0, .LBB212_4
-; RV32IA-NEXT:  .LBB212_3: # %atomicrmw.start
-; RV32IA-NEXT:    sltu a0, s0, a1
+; RV32IA-NEXT:  .LBB212_2: # %atomicrmw.start
+; RV32IA-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32IA-NEXT:    beq a5, s1, .LBB212_4
+; RV32IA-NEXT:  # %bb.3: # %atomicrmw.start
+; RV32IA-NEXT:    # in Loop: Header=BB212_2 Depth=1
+; RV32IA-NEXT:    sltu a0, s1, a5
 ; RV32IA-NEXT:    j .LBB212_5
 ; RV32IA-NEXT:  .LBB212_4:
-; RV32IA-NEXT:    sltu a0, s2, a2
+; RV32IA-NEXT:    sltu a0, s2, a4
 ; RV32IA-NEXT:  .LBB212_5: # %atomicrmw.start
-; RV32IA-NEXT:    sw a2, 0(sp)
-; RV32IA-NEXT:    mv a3, a1
+; RV32IA-NEXT:    # in Loop: Header=BB212_2 Depth=1
+; RV32IA-NEXT:    mv a2, a4
+; RV32IA-NEXT:    mv a3, a5
 ; RV32IA-NEXT:    bnez a0, .LBB212_1
 ; RV32IA-NEXT:  # %bb.6: # %atomicrmw.start
+; RV32IA-NEXT:    # in Loop: Header=BB212_2 Depth=1
 ; RV32IA-NEXT:    mv a2, s2
-; RV32IA-NEXT:    mv a3, s0
+; RV32IA-NEXT:    mv a3, s1
 ; RV32IA-NEXT:    j .LBB212_1
 ; RV32IA-NEXT:  .LBB212_7: # %atomicrmw.end
-; RV32IA-NEXT:    mv a0, a2
+; RV32IA-NEXT:    mv a0, a4
+; RV32IA-NEXT:    mv a1, a5
 ; RV32IA-NEXT:    lw s3, 12(sp)
 ; RV32IA-NEXT:    lw s2, 16(sp)
 ; RV32IA-NEXT:    lw s1, 20(sp)
@@ -16701,30 +16889,31 @@ define i64 @atomicrmw_umax_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    sd s0, 32(sp)
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
-; RV64I-NEXT:    mv s0, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    ld a2, 0(a0)
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    ld a3, 0(a0)
+; RV64I-NEXT:    mv s1, a1
 ; RV64I-NEXT:    addi s2, sp, 8
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    bgeu s0, a2, .LBB212_3
+; RV64I-NEXT:    j .LBB212_2
 ; RV64I-NEXT:  .LBB212_1: # %atomicrmw.start
-; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s2
+; RV64I-NEXT:    # in Loop: Header=BB212_2 Depth=1
+; RV64I-NEXT:    sd a3, 8(sp)
 ; RV64I-NEXT:    addi a3, zero, 3
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    call __atomic_compare_exchange_8
-; RV64I-NEXT:    ld a2, 8(sp)
+; RV64I-NEXT:    ld a3, 8(sp)
 ; RV64I-NEXT:    bnez a0, .LBB212_4
-; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV64I-NEXT:    # in Loop: Header=BB212_1 Depth=1
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    bltu s0, a2, .LBB212_1
-; RV64I-NEXT:  .LBB212_3: # %atomicrmw.start
-; RV64I-NEXT:    mv a2, s0
+; RV64I-NEXT:  .LBB212_2: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bltu s1, a3, .LBB212_1
+; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB212_2 Depth=1
+; RV64I-NEXT:    mv a2, s1
 ; RV64I-NEXT:    j .LBB212_1
 ; RV64I-NEXT:  .LBB212_4: # %atomicrmw.end
-; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
 ; RV64I-NEXT:    ld s0, 32(sp)
@@ -16749,41 +16938,47 @@ define i64 @atomicrmw_umax_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
 ; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    mv s0, a2
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a5, 4(a0)
+; RV32I-NEXT:    lw a4, 0(a0)
+; RV32I-NEXT:    mv s1, a2
 ; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a1, 4(a0)
-; RV32I-NEXT:    lw a2, 0(a0)
 ; RV32I-NEXT:    mv s3, sp
-; RV32I-NEXT:    bne a1, s0, .LBB213_3
-; RV32I-NEXT:    j .LBB213_4
+; RV32I-NEXT:    j .LBB213_2
 ; RV32I-NEXT:  .LBB213_1: # %atomicrmw.start
-; RV32I-NEXT:    sw a1, 4(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    # in Loop: Header=BB213_2 Depth=1
+; RV32I-NEXT:    sw a4, 0(sp)
+; RV32I-NEXT:    sw a5, 4(sp)
 ; RV32I-NEXT:    addi a4, zero, 4
 ; RV32I-NEXT:    addi a5, zero, 2
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
-; RV32I-NEXT:    lw a1, 4(sp)
-; RV32I-NEXT:    lw a2, 0(sp)
+; RV32I-NEXT:    lw a5, 4(sp)
+; RV32I-NEXT:    lw a4, 0(sp)
 ; RV32I-NEXT:    bnez a0, .LBB213_7
-; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32I-NEXT:    beq a1, s0, .LBB213_4
-; RV32I-NEXT:  .LBB213_3: # %atomicrmw.start
-; RV32I-NEXT:    sltu a0, s0, a1
+; RV32I-NEXT:  .LBB213_2: # %atomicrmw.start
+; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32I-NEXT:    beq a5, s1, .LBB213_4
+; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB213_2 Depth=1
+; RV32I-NEXT:    sltu a0, s1, a5
 ; RV32I-NEXT:    j .LBB213_5
 ; RV32I-NEXT:  .LBB213_4:
-; RV32I-NEXT:    sltu a0, s2, a2
+; RV32I-NEXT:    sltu a0, s2, a4
 ; RV32I-NEXT:  .LBB213_5: # %atomicrmw.start
-; RV32I-NEXT:    sw a2, 0(sp)
-; RV32I-NEXT:    mv a3, a1
+; RV32I-NEXT:    # in Loop: Header=BB213_2 Depth=1
+; RV32I-NEXT:    mv a2, a4
+; RV32I-NEXT:    mv a3, a5
 ; RV32I-NEXT:    bnez a0, .LBB213_1
 ; RV32I-NEXT:  # %bb.6: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB213_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
-; RV32I-NEXT:    mv a3, s0
+; RV32I-NEXT:    mv a3, s1
 ; RV32I-NEXT:    j .LBB213_1
 ; RV32I-NEXT:  .LBB213_7: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a4
+; RV32I-NEXT:    mv a1, a5
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -16800,41 +16995,47 @@ define i64 @atomicrmw_umax_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    sw s1, 20(sp)
 ; RV32IA-NEXT:    sw s2, 16(sp)
 ; RV32IA-NEXT:    sw s3, 12(sp)
-; RV32IA-NEXT:    mv s0, a2
+; RV32IA-NEXT:    mv s0, a0
+; RV32IA-NEXT:    lw a5, 4(a0)
+; RV32IA-NEXT:    lw a4, 0(a0)
+; RV32IA-NEXT:    mv s1, a2
 ; RV32IA-NEXT:    mv s2, a1
-; RV32IA-NEXT:    mv s1, a0
-; RV32IA-NEXT:    lw a1, 4(a0)
-; RV32IA-NEXT:    lw a2, 0(a0)
 ; RV32IA-NEXT:    mv s3, sp
-; RV32IA-NEXT:    bne a1, s0, .LBB213_3
-; RV32IA-NEXT:    j .LBB213_4
+; RV32IA-NEXT:    j .LBB213_2
 ; RV32IA-NEXT:  .LBB213_1: # %atomicrmw.start
-; RV32IA-NEXT:    sw a1, 4(sp)
-; RV32IA-NEXT:    mv a0, s1
-; RV32IA-NEXT:    mv a1, s3
+; RV32IA-NEXT:    # in Loop: Header=BB213_2 Depth=1
+; RV32IA-NEXT:    sw a4, 0(sp)
+; RV32IA-NEXT:    sw a5, 4(sp)
 ; RV32IA-NEXT:    addi a4, zero, 4
 ; RV32IA-NEXT:    addi a5, zero, 2
+; RV32IA-NEXT:    mv a0, s0
+; RV32IA-NEXT:    mv a1, s3
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
-; RV32IA-NEXT:    lw a1, 4(sp)
-; RV32IA-NEXT:    lw a2, 0(sp)
+; RV32IA-NEXT:    lw a5, 4(sp)
+; RV32IA-NEXT:    lw a4, 0(sp)
 ; RV32IA-NEXT:    bnez a0, .LBB213_7
-; RV32IA-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32IA-NEXT:    beq a1, s0, .LBB213_4
-; RV32IA-NEXT:  .LBB213_3: # %atomicrmw.start
-; RV32IA-NEXT:    sltu a0, s0, a1
+; RV32IA-NEXT:  .LBB213_2: # %atomicrmw.start
+; RV32IA-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32IA-NEXT:    beq a5, s1, .LBB213_4
+; RV32IA-NEXT:  # %bb.3: # %atomicrmw.start
+; RV32IA-NEXT:    # in Loop: Header=BB213_2 Depth=1
+; RV32IA-NEXT:    sltu a0, s1, a5
 ; RV32IA-NEXT:    j .LBB213_5
 ; RV32IA-NEXT:  .LBB213_4:
-; RV32IA-NEXT:    sltu a0, s2, a2
+; RV32IA-NEXT:    sltu a0, s2, a4
 ; RV32IA-NEXT:  .LBB213_5: # %atomicrmw.start
-; RV32IA-NEXT:    sw a2, 0(sp)
-; RV32IA-NEXT:    mv a3, a1
+; RV32IA-NEXT:    # in Loop: Header=BB213_2 Depth=1
+; RV32IA-NEXT:    mv a2, a4
+; RV32IA-NEXT:    mv a3, a5
 ; RV32IA-NEXT:    bnez a0, .LBB213_1
 ; RV32IA-NEXT:  # %bb.6: # %atomicrmw.start
+; RV32IA-NEXT:    # in Loop: Header=BB213_2 Depth=1
 ; RV32IA-NEXT:    mv a2, s2
-; RV32IA-NEXT:    mv a3, s0
+; RV32IA-NEXT:    mv a3, s1
 ; RV32IA-NEXT:    j .LBB213_1
 ; RV32IA-NEXT:  .LBB213_7: # %atomicrmw.end
-; RV32IA-NEXT:    mv a0, a2
+; RV32IA-NEXT:    mv a0, a4
+; RV32IA-NEXT:    mv a1, a5
 ; RV32IA-NEXT:    lw s3, 12(sp)
 ; RV32IA-NEXT:    lw s2, 16(sp)
 ; RV32IA-NEXT:    lw s1, 20(sp)
@@ -16850,30 +17051,31 @@ define i64 @atomicrmw_umax_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    sd s0, 32(sp)
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
-; RV64I-NEXT:    mv s0, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    ld a2, 0(a0)
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    ld a3, 0(a0)
+; RV64I-NEXT:    mv s1, a1
 ; RV64I-NEXT:    addi s2, sp, 8
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    bgeu s0, a2, .LBB213_3
+; RV64I-NEXT:    j .LBB213_2
 ; RV64I-NEXT:  .LBB213_1: # %atomicrmw.start
-; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s2
+; RV64I-NEXT:    # in Loop: Header=BB213_2 Depth=1
+; RV64I-NEXT:    sd a3, 8(sp)
 ; RV64I-NEXT:    addi a3, zero, 4
 ; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    call __atomic_compare_exchange_8
-; RV64I-NEXT:    ld a2, 8(sp)
+; RV64I-NEXT:    ld a3, 8(sp)
 ; RV64I-NEXT:    bnez a0, .LBB213_4
-; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV64I-NEXT:    # in Loop: Header=BB213_1 Depth=1
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    bltu s0, a2, .LBB213_1
-; RV64I-NEXT:  .LBB213_3: # %atomicrmw.start
-; RV64I-NEXT:    mv a2, s0
+; RV64I-NEXT:  .LBB213_2: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bltu s1, a3, .LBB213_1
+; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB213_2 Depth=1
+; RV64I-NEXT:    mv a2, s1
 ; RV64I-NEXT:    j .LBB213_1
 ; RV64I-NEXT:  .LBB213_4: # %atomicrmw.end
-; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
 ; RV64I-NEXT:    ld s0, 32(sp)
@@ -16898,41 +17100,47 @@ define i64 @atomicrmw_umax_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
 ; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    mv s0, a2
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a5, 4(a0)
+; RV32I-NEXT:    lw a4, 0(a0)
+; RV32I-NEXT:    mv s1, a2
 ; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a1, 4(a0)
-; RV32I-NEXT:    lw a2, 0(a0)
 ; RV32I-NEXT:    mv s3, sp
-; RV32I-NEXT:    bne a1, s0, .LBB214_3
-; RV32I-NEXT:    j .LBB214_4
+; RV32I-NEXT:    j .LBB214_2
 ; RV32I-NEXT:  .LBB214_1: # %atomicrmw.start
-; RV32I-NEXT:    sw a1, 4(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    # in Loop: Header=BB214_2 Depth=1
+; RV32I-NEXT:    sw a4, 0(sp)
+; RV32I-NEXT:    sw a5, 4(sp)
 ; RV32I-NEXT:    addi a4, zero, 5
 ; RV32I-NEXT:    addi a5, zero, 5
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
-; RV32I-NEXT:    lw a1, 4(sp)
-; RV32I-NEXT:    lw a2, 0(sp)
+; RV32I-NEXT:    lw a5, 4(sp)
+; RV32I-NEXT:    lw a4, 0(sp)
 ; RV32I-NEXT:    bnez a0, .LBB214_7
-; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32I-NEXT:    beq a1, s0, .LBB214_4
-; RV32I-NEXT:  .LBB214_3: # %atomicrmw.start
-; RV32I-NEXT:    sltu a0, s0, a1
+; RV32I-NEXT:  .LBB214_2: # %atomicrmw.start
+; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32I-NEXT:    beq a5, s1, .LBB214_4
+; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB214_2 Depth=1
+; RV32I-NEXT:    sltu a0, s1, a5
 ; RV32I-NEXT:    j .LBB214_5
 ; RV32I-NEXT:  .LBB214_4:
-; RV32I-NEXT:    sltu a0, s2, a2
+; RV32I-NEXT:    sltu a0, s2, a4
 ; RV32I-NEXT:  .LBB214_5: # %atomicrmw.start
-; RV32I-NEXT:    sw a2, 0(sp)
-; RV32I-NEXT:    mv a3, a1
+; RV32I-NEXT:    # in Loop: Header=BB214_2 Depth=1
+; RV32I-NEXT:    mv a2, a4
+; RV32I-NEXT:    mv a3, a5
 ; RV32I-NEXT:    bnez a0, .LBB214_1
 ; RV32I-NEXT:  # %bb.6: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB214_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
-; RV32I-NEXT:    mv a3, s0
+; RV32I-NEXT:    mv a3, s1
 ; RV32I-NEXT:    j .LBB214_1
 ; RV32I-NEXT:  .LBB214_7: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a4
+; RV32I-NEXT:    mv a1, a5
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -16949,41 +17157,47 @@ define i64 @atomicrmw_umax_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    sw s1, 20(sp)
 ; RV32IA-NEXT:    sw s2, 16(sp)
 ; RV32IA-NEXT:    sw s3, 12(sp)
-; RV32IA-NEXT:    mv s0, a2
+; RV32IA-NEXT:    mv s0, a0
+; RV32IA-NEXT:    lw a5, 4(a0)
+; RV32IA-NEXT:    lw a4, 0(a0)
+; RV32IA-NEXT:    mv s1, a2
 ; RV32IA-NEXT:    mv s2, a1
-; RV32IA-NEXT:    mv s1, a0
-; RV32IA-NEXT:    lw a1, 4(a0)
-; RV32IA-NEXT:    lw a2, 0(a0)
 ; RV32IA-NEXT:    mv s3, sp
-; RV32IA-NEXT:    bne a1, s0, .LBB214_3
-; RV32IA-NEXT:    j .LBB214_4
+; RV32IA-NEXT:    j .LBB214_2
 ; RV32IA-NEXT:  .LBB214_1: # %atomicrmw.start
-; RV32IA-NEXT:    sw a1, 4(sp)
-; RV32IA-NEXT:    mv a0, s1
-; RV32IA-NEXT:    mv a1, s3
+; RV32IA-NEXT:    # in Loop: Header=BB214_2 Depth=1
+; RV32IA-NEXT:    sw a4, 0(sp)
+; RV32IA-NEXT:    sw a5, 4(sp)
 ; RV32IA-NEXT:    addi a4, zero, 5
 ; RV32IA-NEXT:    addi a5, zero, 5
+; RV32IA-NEXT:    mv a0, s0
+; RV32IA-NEXT:    mv a1, s3
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
-; RV32IA-NEXT:    lw a1, 4(sp)
-; RV32IA-NEXT:    lw a2, 0(sp)
+; RV32IA-NEXT:    lw a5, 4(sp)
+; RV32IA-NEXT:    lw a4, 0(sp)
 ; RV32IA-NEXT:    bnez a0, .LBB214_7
-; RV32IA-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32IA-NEXT:    beq a1, s0, .LBB214_4
-; RV32IA-NEXT:  .LBB214_3: # %atomicrmw.start
-; RV32IA-NEXT:    sltu a0, s0, a1
+; RV32IA-NEXT:  .LBB214_2: # %atomicrmw.start
+; RV32IA-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32IA-NEXT:    beq a5, s1, .LBB214_4
+; RV32IA-NEXT:  # %bb.3: # %atomicrmw.start
+; RV32IA-NEXT:    # in Loop: Header=BB214_2 Depth=1
+; RV32IA-NEXT:    sltu a0, s1, a5
 ; RV32IA-NEXT:    j .LBB214_5
 ; RV32IA-NEXT:  .LBB214_4:
-; RV32IA-NEXT:    sltu a0, s2, a2
+; RV32IA-NEXT:    sltu a0, s2, a4
 ; RV32IA-NEXT:  .LBB214_5: # %atomicrmw.start
-; RV32IA-NEXT:    sw a2, 0(sp)
-; RV32IA-NEXT:    mv a3, a1
+; RV32IA-NEXT:    # in Loop: Header=BB214_2 Depth=1
+; RV32IA-NEXT:    mv a2, a4
+; RV32IA-NEXT:    mv a3, a5
 ; RV32IA-NEXT:    bnez a0, .LBB214_1
 ; RV32IA-NEXT:  # %bb.6: # %atomicrmw.start
+; RV32IA-NEXT:    # in Loop: Header=BB214_2 Depth=1
 ; RV32IA-NEXT:    mv a2, s2
-; RV32IA-NEXT:    mv a3, s0
+; RV32IA-NEXT:    mv a3, s1
 ; RV32IA-NEXT:    j .LBB214_1
 ; RV32IA-NEXT:  .LBB214_7: # %atomicrmw.end
-; RV32IA-NEXT:    mv a0, a2
+; RV32IA-NEXT:    mv a0, a4
+; RV32IA-NEXT:    mv a1, a5
 ; RV32IA-NEXT:    lw s3, 12(sp)
 ; RV32IA-NEXT:    lw s2, 16(sp)
 ; RV32IA-NEXT:    lw s1, 20(sp)
@@ -16999,30 +17213,31 @@ define i64 @atomicrmw_umax_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    sd s0, 32(sp)
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
-; RV64I-NEXT:    mv s0, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    ld a2, 0(a0)
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    ld a3, 0(a0)
+; RV64I-NEXT:    mv s1, a1
 ; RV64I-NEXT:    addi s2, sp, 8
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    bgeu s0, a2, .LBB214_3
+; RV64I-NEXT:    j .LBB214_2
 ; RV64I-NEXT:  .LBB214_1: # %atomicrmw.start
-; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s2
+; RV64I-NEXT:    # in Loop: Header=BB214_2 Depth=1
+; RV64I-NEXT:    sd a3, 8(sp)
 ; RV64I-NEXT:    addi a3, zero, 5
 ; RV64I-NEXT:    addi a4, zero, 5
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    call __atomic_compare_exchange_8
-; RV64I-NEXT:    ld a2, 8(sp)
+; RV64I-NEXT:    ld a3, 8(sp)
 ; RV64I-NEXT:    bnez a0, .LBB214_4
-; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV64I-NEXT:    # in Loop: Header=BB214_1 Depth=1
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    bltu s0, a2, .LBB214_1
-; RV64I-NEXT:  .LBB214_3: # %atomicrmw.start
-; RV64I-NEXT:    mv a2, s0
+; RV64I-NEXT:  .LBB214_2: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bltu s1, a3, .LBB214_1
+; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB214_2 Depth=1
+; RV64I-NEXT:    mv a2, s1
 ; RV64I-NEXT:    j .LBB214_1
 ; RV64I-NEXT:  .LBB214_4: # %atomicrmw.end
-; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
 ; RV64I-NEXT:    ld s0, 32(sp)
@@ -17047,42 +17262,48 @@ define i64 @atomicrmw_umin_i64_monotonic(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
 ; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    mv s0, a2
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a5, 4(a0)
+; RV32I-NEXT:    lw a4, 0(a0)
+; RV32I-NEXT:    mv s1, a2
 ; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a1, 4(a0)
-; RV32I-NEXT:    lw a2, 0(a0)
 ; RV32I-NEXT:    mv s3, sp
-; RV32I-NEXT:    bne a1, s0, .LBB215_3
-; RV32I-NEXT:    j .LBB215_4
+; RV32I-NEXT:    j .LBB215_2
 ; RV32I-NEXT:  .LBB215_1: # %atomicrmw.start
-; RV32I-NEXT:    sw a1, 4(sp)
-; RV32I-NEXT:    mv a0, s1
+; RV32I-NEXT:    # in Loop: Header=BB215_2 Depth=1
+; RV32I-NEXT:    sw a4, 0(sp)
+; RV32I-NEXT:    sw a5, 4(sp)
+; RV32I-NEXT:    mv a0, s0
 ; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    mv a5, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
-; RV32I-NEXT:    lw a1, 4(sp)
-; RV32I-NEXT:    lw a2, 0(sp)
+; RV32I-NEXT:    lw a5, 4(sp)
+; RV32I-NEXT:    lw a4, 0(sp)
 ; RV32I-NEXT:    bnez a0, .LBB215_7
-; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32I-NEXT:    beq a1, s0, .LBB215_4
-; RV32I-NEXT:  .LBB215_3: # %atomicrmw.start
-; RV32I-NEXT:    sltu a0, s0, a1
+; RV32I-NEXT:  .LBB215_2: # %atomicrmw.start
+; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32I-NEXT:    beq a5, s1, .LBB215_4
+; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB215_2 Depth=1
+; RV32I-NEXT:    sltu a0, s1, a5
 ; RV32I-NEXT:    j .LBB215_5
 ; RV32I-NEXT:  .LBB215_4:
-; RV32I-NEXT:    sltu a0, s2, a2
+; RV32I-NEXT:    sltu a0, s2, a4
 ; RV32I-NEXT:  .LBB215_5: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB215_2 Depth=1
 ; RV32I-NEXT:    xori a0, a0, 1
-; RV32I-NEXT:    sw a2, 0(sp)
-; RV32I-NEXT:    mv a3, a1
+; RV32I-NEXT:    mv a2, a4
+; RV32I-NEXT:    mv a3, a5
 ; RV32I-NEXT:    bnez a0, .LBB215_1
 ; RV32I-NEXT:  # %bb.6: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB215_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
-; RV32I-NEXT:    mv a3, s0
+; RV32I-NEXT:    mv a3, s1
 ; RV32I-NEXT:    j .LBB215_1
 ; RV32I-NEXT:  .LBB215_7: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a4
+; RV32I-NEXT:    mv a1, a5
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -17099,42 +17320,48 @@ define i64 @atomicrmw_umin_i64_monotonic(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    sw s1, 20(sp)
 ; RV32IA-NEXT:    sw s2, 16(sp)
 ; RV32IA-NEXT:    sw s3, 12(sp)
-; RV32IA-NEXT:    mv s0, a2
+; RV32IA-NEXT:    mv s0, a0
+; RV32IA-NEXT:    lw a5, 4(a0)
+; RV32IA-NEXT:    lw a4, 0(a0)
+; RV32IA-NEXT:    mv s1, a2
 ; RV32IA-NEXT:    mv s2, a1
-; RV32IA-NEXT:    mv s1, a0
-; RV32IA-NEXT:    lw a1, 4(a0)
-; RV32IA-NEXT:    lw a2, 0(a0)
 ; RV32IA-NEXT:    mv s3, sp
-; RV32IA-NEXT:    bne a1, s0, .LBB215_3
-; RV32IA-NEXT:    j .LBB215_4
+; RV32IA-NEXT:    j .LBB215_2
 ; RV32IA-NEXT:  .LBB215_1: # %atomicrmw.start
-; RV32IA-NEXT:    sw a1, 4(sp)
-; RV32IA-NEXT:    mv a0, s1
+; RV32IA-NEXT:    # in Loop: Header=BB215_2 Depth=1
+; RV32IA-NEXT:    sw a4, 0(sp)
+; RV32IA-NEXT:    sw a5, 4(sp)
+; RV32IA-NEXT:    mv a0, s0
 ; RV32IA-NEXT:    mv a1, s3
 ; RV32IA-NEXT:    mv a4, zero
 ; RV32IA-NEXT:    mv a5, zero
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
-; RV32IA-NEXT:    lw a1, 4(sp)
-; RV32IA-NEXT:    lw a2, 0(sp)
+; RV32IA-NEXT:    lw a5, 4(sp)
+; RV32IA-NEXT:    lw a4, 0(sp)
 ; RV32IA-NEXT:    bnez a0, .LBB215_7
-; RV32IA-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32IA-NEXT:    beq a1, s0, .LBB215_4
-; RV32IA-NEXT:  .LBB215_3: # %atomicrmw.start
-; RV32IA-NEXT:    sltu a0, s0, a1
+; RV32IA-NEXT:  .LBB215_2: # %atomicrmw.start
+; RV32IA-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32IA-NEXT:    beq a5, s1, .LBB215_4
+; RV32IA-NEXT:  # %bb.3: # %atomicrmw.start
+; RV32IA-NEXT:    # in Loop: Header=BB215_2 Depth=1
+; RV32IA-NEXT:    sltu a0, s1, a5
 ; RV32IA-NEXT:    j .LBB215_5
 ; RV32IA-NEXT:  .LBB215_4:
-; RV32IA-NEXT:    sltu a0, s2, a2
+; RV32IA-NEXT:    sltu a0, s2, a4
 ; RV32IA-NEXT:  .LBB215_5: # %atomicrmw.start
+; RV32IA-NEXT:    # in Loop: Header=BB215_2 Depth=1
 ; RV32IA-NEXT:    xori a0, a0, 1
-; RV32IA-NEXT:    sw a2, 0(sp)
-; RV32IA-NEXT:    mv a3, a1
+; RV32IA-NEXT:    mv a2, a4
+; RV32IA-NEXT:    mv a3, a5
 ; RV32IA-NEXT:    bnez a0, .LBB215_1
 ; RV32IA-NEXT:  # %bb.6: # %atomicrmw.start
+; RV32IA-NEXT:    # in Loop: Header=BB215_2 Depth=1
 ; RV32IA-NEXT:    mv a2, s2
-; RV32IA-NEXT:    mv a3, s0
+; RV32IA-NEXT:    mv a3, s1
 ; RV32IA-NEXT:    j .LBB215_1
 ; RV32IA-NEXT:  .LBB215_7: # %atomicrmw.end
-; RV32IA-NEXT:    mv a0, a2
+; RV32IA-NEXT:    mv a0, a4
+; RV32IA-NEXT:    mv a1, a5
 ; RV32IA-NEXT:    lw s3, 12(sp)
 ; RV32IA-NEXT:    lw s2, 16(sp)
 ; RV32IA-NEXT:    lw s1, 20(sp)
@@ -17150,30 +17377,31 @@ define i64 @atomicrmw_umin_i64_monotonic(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    sd s0, 32(sp)
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
-; RV64I-NEXT:    mv s0, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    ld a2, 0(a0)
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    ld a3, 0(a0)
+; RV64I-NEXT:    mv s1, a1
 ; RV64I-NEXT:    addi s2, sp, 8
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    bltu s0, a2, .LBB215_3
+; RV64I-NEXT:    j .LBB215_2
 ; RV64I-NEXT:  .LBB215_1: # %atomicrmw.start
-; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    mv a0, s1
+; RV64I-NEXT:    # in Loop: Header=BB215_2 Depth=1
+; RV64I-NEXT:    sd a3, 8(sp)
+; RV64I-NEXT:    mv a0, s0
 ; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    mv a3, zero
 ; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    call __atomic_compare_exchange_8
-; RV64I-NEXT:    ld a2, 8(sp)
+; RV64I-NEXT:    ld a3, 8(sp)
 ; RV64I-NEXT:    bnez a0, .LBB215_4
-; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV64I-NEXT:    # in Loop: Header=BB215_1 Depth=1
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    bgeu s0, a2, .LBB215_1
-; RV64I-NEXT:  .LBB215_3: # %atomicrmw.start
-; RV64I-NEXT:    mv a2, s0
+; RV64I-NEXT:  .LBB215_2: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bgeu s1, a3, .LBB215_1
+; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB215_2 Depth=1
+; RV64I-NEXT:    mv a2, s1
 ; RV64I-NEXT:    j .LBB215_1
 ; RV64I-NEXT:  .LBB215_4: # %atomicrmw.end
-; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
 ; RV64I-NEXT:    ld s0, 32(sp)
@@ -17198,42 +17426,48 @@ define i64 @atomicrmw_umin_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
 ; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    mv s0, a2
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a5, 4(a0)
+; RV32I-NEXT:    lw a4, 0(a0)
+; RV32I-NEXT:    mv s1, a2
 ; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a1, 4(a0)
-; RV32I-NEXT:    lw a2, 0(a0)
 ; RV32I-NEXT:    mv s3, sp
-; RV32I-NEXT:    bne a1, s0, .LBB216_3
-; RV32I-NEXT:    j .LBB216_4
+; RV32I-NEXT:    j .LBB216_2
 ; RV32I-NEXT:  .LBB216_1: # %atomicrmw.start
-; RV32I-NEXT:    sw a1, 4(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    # in Loop: Header=BB216_2 Depth=1
+; RV32I-NEXT:    sw a4, 0(sp)
+; RV32I-NEXT:    sw a5, 4(sp)
 ; RV32I-NEXT:    addi a4, zero, 2
 ; RV32I-NEXT:    addi a5, zero, 2
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
-; RV32I-NEXT:    lw a1, 4(sp)
-; RV32I-NEXT:    lw a2, 0(sp)
+; RV32I-NEXT:    lw a5, 4(sp)
+; RV32I-NEXT:    lw a4, 0(sp)
 ; RV32I-NEXT:    bnez a0, .LBB216_7
-; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32I-NEXT:    beq a1, s0, .LBB216_4
-; RV32I-NEXT:  .LBB216_3: # %atomicrmw.start
-; RV32I-NEXT:    sltu a0, s0, a1
+; RV32I-NEXT:  .LBB216_2: # %atomicrmw.start
+; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32I-NEXT:    beq a5, s1, .LBB216_4
+; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB216_2 Depth=1
+; RV32I-NEXT:    sltu a0, s1, a5
 ; RV32I-NEXT:    j .LBB216_5
 ; RV32I-NEXT:  .LBB216_4:
-; RV32I-NEXT:    sltu a0, s2, a2
+; RV32I-NEXT:    sltu a0, s2, a4
 ; RV32I-NEXT:  .LBB216_5: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB216_2 Depth=1
 ; RV32I-NEXT:    xori a0, a0, 1
-; RV32I-NEXT:    sw a2, 0(sp)
-; RV32I-NEXT:    mv a3, a1
+; RV32I-NEXT:    mv a2, a4
+; RV32I-NEXT:    mv a3, a5
 ; RV32I-NEXT:    bnez a0, .LBB216_1
 ; RV32I-NEXT:  # %bb.6: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB216_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
-; RV32I-NEXT:    mv a3, s0
+; RV32I-NEXT:    mv a3, s1
 ; RV32I-NEXT:    j .LBB216_1
 ; RV32I-NEXT:  .LBB216_7: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a4
+; RV32I-NEXT:    mv a1, a5
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -17250,42 +17484,48 @@ define i64 @atomicrmw_umin_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    sw s1, 20(sp)
 ; RV32IA-NEXT:    sw s2, 16(sp)
 ; RV32IA-NEXT:    sw s3, 12(sp)
-; RV32IA-NEXT:    mv s0, a2
+; RV32IA-NEXT:    mv s0, a0
+; RV32IA-NEXT:    lw a5, 4(a0)
+; RV32IA-NEXT:    lw a4, 0(a0)
+; RV32IA-NEXT:    mv s1, a2
 ; RV32IA-NEXT:    mv s2, a1
-; RV32IA-NEXT:    mv s1, a0
-; RV32IA-NEXT:    lw a1, 4(a0)
-; RV32IA-NEXT:    lw a2, 0(a0)
 ; RV32IA-NEXT:    mv s3, sp
-; RV32IA-NEXT:    bne a1, s0, .LBB216_3
-; RV32IA-NEXT:    j .LBB216_4
+; RV32IA-NEXT:    j .LBB216_2
 ; RV32IA-NEXT:  .LBB216_1: # %atomicrmw.start
-; RV32IA-NEXT:    sw a1, 4(sp)
-; RV32IA-NEXT:    mv a0, s1
-; RV32IA-NEXT:    mv a1, s3
+; RV32IA-NEXT:    # in Loop: Header=BB216_2 Depth=1
+; RV32IA-NEXT:    sw a4, 0(sp)
+; RV32IA-NEXT:    sw a5, 4(sp)
 ; RV32IA-NEXT:    addi a4, zero, 2
 ; RV32IA-NEXT:    addi a5, zero, 2
+; RV32IA-NEXT:    mv a0, s0
+; RV32IA-NEXT:    mv a1, s3
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
-; RV32IA-NEXT:    lw a1, 4(sp)
-; RV32IA-NEXT:    lw a2, 0(sp)
+; RV32IA-NEXT:    lw a5, 4(sp)
+; RV32IA-NEXT:    lw a4, 0(sp)
 ; RV32IA-NEXT:    bnez a0, .LBB216_7
-; RV32IA-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32IA-NEXT:    beq a1, s0, .LBB216_4
-; RV32IA-NEXT:  .LBB216_3: # %atomicrmw.start
-; RV32IA-NEXT:    sltu a0, s0, a1
+; RV32IA-NEXT:  .LBB216_2: # %atomicrmw.start
+; RV32IA-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32IA-NEXT:    beq a5, s1, .LBB216_4
+; RV32IA-NEXT:  # %bb.3: # %atomicrmw.start
+; RV32IA-NEXT:    # in Loop: Header=BB216_2 Depth=1
+; RV32IA-NEXT:    sltu a0, s1, a5
 ; RV32IA-NEXT:    j .LBB216_5
 ; RV32IA-NEXT:  .LBB216_4:
-; RV32IA-NEXT:    sltu a0, s2, a2
+; RV32IA-NEXT:    sltu a0, s2, a4
 ; RV32IA-NEXT:  .LBB216_5: # %atomicrmw.start
+; RV32IA-NEXT:    # in Loop: Header=BB216_2 Depth=1
 ; RV32IA-NEXT:    xori a0, a0, 1
-; RV32IA-NEXT:    sw a2, 0(sp)
-; RV32IA-NEXT:    mv a3, a1
+; RV32IA-NEXT:    mv a2, a4
+; RV32IA-NEXT:    mv a3, a5
 ; RV32IA-NEXT:    bnez a0, .LBB216_1
 ; RV32IA-NEXT:  # %bb.6: # %atomicrmw.start
+; RV32IA-NEXT:    # in Loop: Header=BB216_2 Depth=1
 ; RV32IA-NEXT:    mv a2, s2
-; RV32IA-NEXT:    mv a3, s0
+; RV32IA-NEXT:    mv a3, s1
 ; RV32IA-NEXT:    j .LBB216_1
 ; RV32IA-NEXT:  .LBB216_7: # %atomicrmw.end
-; RV32IA-NEXT:    mv a0, a2
+; RV32IA-NEXT:    mv a0, a4
+; RV32IA-NEXT:    mv a1, a5
 ; RV32IA-NEXT:    lw s3, 12(sp)
 ; RV32IA-NEXT:    lw s2, 16(sp)
 ; RV32IA-NEXT:    lw s1, 20(sp)
@@ -17301,30 +17541,31 @@ define i64 @atomicrmw_umin_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    sd s0, 32(sp)
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
-; RV64I-NEXT:    mv s0, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    ld a2, 0(a0)
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    ld a3, 0(a0)
+; RV64I-NEXT:    mv s1, a1
 ; RV64I-NEXT:    addi s2, sp, 8
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    bltu s0, a2, .LBB216_3
+; RV64I-NEXT:    j .LBB216_2
 ; RV64I-NEXT:  .LBB216_1: # %atomicrmw.start
-; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s2
+; RV64I-NEXT:    # in Loop: Header=BB216_2 Depth=1
+; RV64I-NEXT:    sd a3, 8(sp)
 ; RV64I-NEXT:    addi a3, zero, 2
 ; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    call __atomic_compare_exchange_8
-; RV64I-NEXT:    ld a2, 8(sp)
+; RV64I-NEXT:    ld a3, 8(sp)
 ; RV64I-NEXT:    bnez a0, .LBB216_4
-; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV64I-NEXT:    # in Loop: Header=BB216_1 Depth=1
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    bgeu s0, a2, .LBB216_1
-; RV64I-NEXT:  .LBB216_3: # %atomicrmw.start
-; RV64I-NEXT:    mv a2, s0
+; RV64I-NEXT:  .LBB216_2: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bgeu s1, a3, .LBB216_1
+; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB216_2 Depth=1
+; RV64I-NEXT:    mv a2, s1
 ; RV64I-NEXT:    j .LBB216_1
 ; RV64I-NEXT:  .LBB216_4: # %atomicrmw.end
-; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
 ; RV64I-NEXT:    ld s0, 32(sp)
@@ -17349,42 +17590,48 @@ define i64 @atomicrmw_umin_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
 ; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    mv s0, a2
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a5, 4(a0)
+; RV32I-NEXT:    lw a4, 0(a0)
+; RV32I-NEXT:    mv s1, a2
 ; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a1, 4(a0)
-; RV32I-NEXT:    lw a2, 0(a0)
 ; RV32I-NEXT:    mv s3, sp
-; RV32I-NEXT:    bne a1, s0, .LBB217_3
-; RV32I-NEXT:    j .LBB217_4
+; RV32I-NEXT:    j .LBB217_2
 ; RV32I-NEXT:  .LBB217_1: # %atomicrmw.start
-; RV32I-NEXT:    sw a1, 4(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    # in Loop: Header=BB217_2 Depth=1
+; RV32I-NEXT:    sw a4, 0(sp)
+; RV32I-NEXT:    sw a5, 4(sp)
 ; RV32I-NEXT:    addi a4, zero, 3
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    mv a5, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
-; RV32I-NEXT:    lw a1, 4(sp)
-; RV32I-NEXT:    lw a2, 0(sp)
+; RV32I-NEXT:    lw a5, 4(sp)
+; RV32I-NEXT:    lw a4, 0(sp)
 ; RV32I-NEXT:    bnez a0, .LBB217_7
-; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32I-NEXT:    beq a1, s0, .LBB217_4
-; RV32I-NEXT:  .LBB217_3: # %atomicrmw.start
-; RV32I-NEXT:    sltu a0, s0, a1
+; RV32I-NEXT:  .LBB217_2: # %atomicrmw.start
+; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32I-NEXT:    beq a5, s1, .LBB217_4
+; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB217_2 Depth=1
+; RV32I-NEXT:    sltu a0, s1, a5
 ; RV32I-NEXT:    j .LBB217_5
 ; RV32I-NEXT:  .LBB217_4:
-; RV32I-NEXT:    sltu a0, s2, a2
+; RV32I-NEXT:    sltu a0, s2, a4
 ; RV32I-NEXT:  .LBB217_5: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB217_2 Depth=1
 ; RV32I-NEXT:    xori a0, a0, 1
-; RV32I-NEXT:    sw a2, 0(sp)
-; RV32I-NEXT:    mv a3, a1
+; RV32I-NEXT:    mv a2, a4
+; RV32I-NEXT:    mv a3, a5
 ; RV32I-NEXT:    bnez a0, .LBB217_1
 ; RV32I-NEXT:  # %bb.6: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB217_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
-; RV32I-NEXT:    mv a3, s0
+; RV32I-NEXT:    mv a3, s1
 ; RV32I-NEXT:    j .LBB217_1
 ; RV32I-NEXT:  .LBB217_7: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a4
+; RV32I-NEXT:    mv a1, a5
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -17401,42 +17648,48 @@ define i64 @atomicrmw_umin_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    sw s1, 20(sp)
 ; RV32IA-NEXT:    sw s2, 16(sp)
 ; RV32IA-NEXT:    sw s3, 12(sp)
-; RV32IA-NEXT:    mv s0, a2
+; RV32IA-NEXT:    mv s0, a0
+; RV32IA-NEXT:    lw a5, 4(a0)
+; RV32IA-NEXT:    lw a4, 0(a0)
+; RV32IA-NEXT:    mv s1, a2
 ; RV32IA-NEXT:    mv s2, a1
-; RV32IA-NEXT:    mv s1, a0
-; RV32IA-NEXT:    lw a1, 4(a0)
-; RV32IA-NEXT:    lw a2, 0(a0)
 ; RV32IA-NEXT:    mv s3, sp
-; RV32IA-NEXT:    bne a1, s0, .LBB217_3
-; RV32IA-NEXT:    j .LBB217_4
+; RV32IA-NEXT:    j .LBB217_2
 ; RV32IA-NEXT:  .LBB217_1: # %atomicrmw.start
-; RV32IA-NEXT:    sw a1, 4(sp)
-; RV32IA-NEXT:    mv a0, s1
-; RV32IA-NEXT:    mv a1, s3
+; RV32IA-NEXT:    # in Loop: Header=BB217_2 Depth=1
+; RV32IA-NEXT:    sw a4, 0(sp)
+; RV32IA-NEXT:    sw a5, 4(sp)
 ; RV32IA-NEXT:    addi a4, zero, 3
+; RV32IA-NEXT:    mv a0, s0
+; RV32IA-NEXT:    mv a1, s3
 ; RV32IA-NEXT:    mv a5, zero
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
-; RV32IA-NEXT:    lw a1, 4(sp)
-; RV32IA-NEXT:    lw a2, 0(sp)
+; RV32IA-NEXT:    lw a5, 4(sp)
+; RV32IA-NEXT:    lw a4, 0(sp)
 ; RV32IA-NEXT:    bnez a0, .LBB217_7
-; RV32IA-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32IA-NEXT:    beq a1, s0, .LBB217_4
-; RV32IA-NEXT:  .LBB217_3: # %atomicrmw.start
-; RV32IA-NEXT:    sltu a0, s0, a1
+; RV32IA-NEXT:  .LBB217_2: # %atomicrmw.start
+; RV32IA-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32IA-NEXT:    beq a5, s1, .LBB217_4
+; RV32IA-NEXT:  # %bb.3: # %atomicrmw.start
+; RV32IA-NEXT:    # in Loop: Header=BB217_2 Depth=1
+; RV32IA-NEXT:    sltu a0, s1, a5
 ; RV32IA-NEXT:    j .LBB217_5
 ; RV32IA-NEXT:  .LBB217_4:
-; RV32IA-NEXT:    sltu a0, s2, a2
+; RV32IA-NEXT:    sltu a0, s2, a4
 ; RV32IA-NEXT:  .LBB217_5: # %atomicrmw.start
+; RV32IA-NEXT:    # in Loop: Header=BB217_2 Depth=1
 ; RV32IA-NEXT:    xori a0, a0, 1
-; RV32IA-NEXT:    sw a2, 0(sp)
-; RV32IA-NEXT:    mv a3, a1
+; RV32IA-NEXT:    mv a2, a4
+; RV32IA-NEXT:    mv a3, a5
 ; RV32IA-NEXT:    bnez a0, .LBB217_1
 ; RV32IA-NEXT:  # %bb.6: # %atomicrmw.start
+; RV32IA-NEXT:    # in Loop: Header=BB217_2 Depth=1
 ; RV32IA-NEXT:    mv a2, s2
-; RV32IA-NEXT:    mv a3, s0
+; RV32IA-NEXT:    mv a3, s1
 ; RV32IA-NEXT:    j .LBB217_1
 ; RV32IA-NEXT:  .LBB217_7: # %atomicrmw.end
-; RV32IA-NEXT:    mv a0, a2
+; RV32IA-NEXT:    mv a0, a4
+; RV32IA-NEXT:    mv a1, a5
 ; RV32IA-NEXT:    lw s3, 12(sp)
 ; RV32IA-NEXT:    lw s2, 16(sp)
 ; RV32IA-NEXT:    lw s1, 20(sp)
@@ -17452,30 +17705,31 @@ define i64 @atomicrmw_umin_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    sd s0, 32(sp)
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
-; RV64I-NEXT:    mv s0, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    ld a2, 0(a0)
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    ld a3, 0(a0)
+; RV64I-NEXT:    mv s1, a1
 ; RV64I-NEXT:    addi s2, sp, 8
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    bltu s0, a2, .LBB217_3
+; RV64I-NEXT:    j .LBB217_2
 ; RV64I-NEXT:  .LBB217_1: # %atomicrmw.start
-; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s2
+; RV64I-NEXT:    # in Loop: Header=BB217_2 Depth=1
+; RV64I-NEXT:    sd a3, 8(sp)
 ; RV64I-NEXT:    addi a3, zero, 3
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    call __atomic_compare_exchange_8
-; RV64I-NEXT:    ld a2, 8(sp)
+; RV64I-NEXT:    ld a3, 8(sp)
 ; RV64I-NEXT:    bnez a0, .LBB217_4
-; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV64I-NEXT:    # in Loop: Header=BB217_1 Depth=1
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    bgeu s0, a2, .LBB217_1
-; RV64I-NEXT:  .LBB217_3: # %atomicrmw.start
-; RV64I-NEXT:    mv a2, s0
+; RV64I-NEXT:  .LBB217_2: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bgeu s1, a3, .LBB217_1
+; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB217_2 Depth=1
+; RV64I-NEXT:    mv a2, s1
 ; RV64I-NEXT:    j .LBB217_1
 ; RV64I-NEXT:  .LBB217_4: # %atomicrmw.end
-; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
 ; RV64I-NEXT:    ld s0, 32(sp)
@@ -17500,42 +17754,48 @@ define i64 @atomicrmw_umin_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
 ; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    mv s0, a2
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a5, 4(a0)
+; RV32I-NEXT:    lw a4, 0(a0)
+; RV32I-NEXT:    mv s1, a2
 ; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a1, 4(a0)
-; RV32I-NEXT:    lw a2, 0(a0)
 ; RV32I-NEXT:    mv s3, sp
-; RV32I-NEXT:    bne a1, s0, .LBB218_3
-; RV32I-NEXT:    j .LBB218_4
+; RV32I-NEXT:    j .LBB218_2
 ; RV32I-NEXT:  .LBB218_1: # %atomicrmw.start
-; RV32I-NEXT:    sw a1, 4(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    # in Loop: Header=BB218_2 Depth=1
+; RV32I-NEXT:    sw a4, 0(sp)
+; RV32I-NEXT:    sw a5, 4(sp)
 ; RV32I-NEXT:    addi a4, zero, 4
 ; RV32I-NEXT:    addi a5, zero, 2
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
-; RV32I-NEXT:    lw a1, 4(sp)
-; RV32I-NEXT:    lw a2, 0(sp)
+; RV32I-NEXT:    lw a5, 4(sp)
+; RV32I-NEXT:    lw a4, 0(sp)
 ; RV32I-NEXT:    bnez a0, .LBB218_7
-; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32I-NEXT:    beq a1, s0, .LBB218_4
-; RV32I-NEXT:  .LBB218_3: # %atomicrmw.start
-; RV32I-NEXT:    sltu a0, s0, a1
+; RV32I-NEXT:  .LBB218_2: # %atomicrmw.start
+; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32I-NEXT:    beq a5, s1, .LBB218_4
+; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB218_2 Depth=1
+; RV32I-NEXT:    sltu a0, s1, a5
 ; RV32I-NEXT:    j .LBB218_5
 ; RV32I-NEXT:  .LBB218_4:
-; RV32I-NEXT:    sltu a0, s2, a2
+; RV32I-NEXT:    sltu a0, s2, a4
 ; RV32I-NEXT:  .LBB218_5: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB218_2 Depth=1
 ; RV32I-NEXT:    xori a0, a0, 1
-; RV32I-NEXT:    sw a2, 0(sp)
-; RV32I-NEXT:    mv a3, a1
+; RV32I-NEXT:    mv a2, a4
+; RV32I-NEXT:    mv a3, a5
 ; RV32I-NEXT:    bnez a0, .LBB218_1
 ; RV32I-NEXT:  # %bb.6: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB218_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
-; RV32I-NEXT:    mv a3, s0
+; RV32I-NEXT:    mv a3, s1
 ; RV32I-NEXT:    j .LBB218_1
 ; RV32I-NEXT:  .LBB218_7: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a4
+; RV32I-NEXT:    mv a1, a5
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -17552,42 +17812,48 @@ define i64 @atomicrmw_umin_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    sw s1, 20(sp)
 ; RV32IA-NEXT:    sw s2, 16(sp)
 ; RV32IA-NEXT:    sw s3, 12(sp)
-; RV32IA-NEXT:    mv s0, a2
+; RV32IA-NEXT:    mv s0, a0
+; RV32IA-NEXT:    lw a5, 4(a0)
+; RV32IA-NEXT:    lw a4, 0(a0)
+; RV32IA-NEXT:    mv s1, a2
 ; RV32IA-NEXT:    mv s2, a1
-; RV32IA-NEXT:    mv s1, a0
-; RV32IA-NEXT:    lw a1, 4(a0)
-; RV32IA-NEXT:    lw a2, 0(a0)
 ; RV32IA-NEXT:    mv s3, sp
-; RV32IA-NEXT:    bne a1, s0, .LBB218_3
-; RV32IA-NEXT:    j .LBB218_4
+; RV32IA-NEXT:    j .LBB218_2
 ; RV32IA-NEXT:  .LBB218_1: # %atomicrmw.start
-; RV32IA-NEXT:    sw a1, 4(sp)
-; RV32IA-NEXT:    mv a0, s1
-; RV32IA-NEXT:    mv a1, s3
+; RV32IA-NEXT:    # in Loop: Header=BB218_2 Depth=1
+; RV32IA-NEXT:    sw a4, 0(sp)
+; RV32IA-NEXT:    sw a5, 4(sp)
 ; RV32IA-NEXT:    addi a4, zero, 4
 ; RV32IA-NEXT:    addi a5, zero, 2
+; RV32IA-NEXT:    mv a0, s0
+; RV32IA-NEXT:    mv a1, s3
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
-; RV32IA-NEXT:    lw a1, 4(sp)
-; RV32IA-NEXT:    lw a2, 0(sp)
+; RV32IA-NEXT:    lw a5, 4(sp)
+; RV32IA-NEXT:    lw a4, 0(sp)
 ; RV32IA-NEXT:    bnez a0, .LBB218_7
-; RV32IA-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32IA-NEXT:    beq a1, s0, .LBB218_4
-; RV32IA-NEXT:  .LBB218_3: # %atomicrmw.start
-; RV32IA-NEXT:    sltu a0, s0, a1
+; RV32IA-NEXT:  .LBB218_2: # %atomicrmw.start
+; RV32IA-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32IA-NEXT:    beq a5, s1, .LBB218_4
+; RV32IA-NEXT:  # %bb.3: # %atomicrmw.start
+; RV32IA-NEXT:    # in Loop: Header=BB218_2 Depth=1
+; RV32IA-NEXT:    sltu a0, s1, a5
 ; RV32IA-NEXT:    j .LBB218_5
 ; RV32IA-NEXT:  .LBB218_4:
-; RV32IA-NEXT:    sltu a0, s2, a2
+; RV32IA-NEXT:    sltu a0, s2, a4
 ; RV32IA-NEXT:  .LBB218_5: # %atomicrmw.start
+; RV32IA-NEXT:    # in Loop: Header=BB218_2 Depth=1
 ; RV32IA-NEXT:    xori a0, a0, 1
-; RV32IA-NEXT:    sw a2, 0(sp)
-; RV32IA-NEXT:    mv a3, a1
+; RV32IA-NEXT:    mv a2, a4
+; RV32IA-NEXT:    mv a3, a5
 ; RV32IA-NEXT:    bnez a0, .LBB218_1
 ; RV32IA-NEXT:  # %bb.6: # %atomicrmw.start
+; RV32IA-NEXT:    # in Loop: Header=BB218_2 Depth=1
 ; RV32IA-NEXT:    mv a2, s2
-; RV32IA-NEXT:    mv a3, s0
+; RV32IA-NEXT:    mv a3, s1
 ; RV32IA-NEXT:    j .LBB218_1
 ; RV32IA-NEXT:  .LBB218_7: # %atomicrmw.end
-; RV32IA-NEXT:    mv a0, a2
+; RV32IA-NEXT:    mv a0, a4
+; RV32IA-NEXT:    mv a1, a5
 ; RV32IA-NEXT:    lw s3, 12(sp)
 ; RV32IA-NEXT:    lw s2, 16(sp)
 ; RV32IA-NEXT:    lw s1, 20(sp)
@@ -17603,30 +17869,31 @@ define i64 @atomicrmw_umin_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    sd s0, 32(sp)
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
-; RV64I-NEXT:    mv s0, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    ld a2, 0(a0)
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    ld a3, 0(a0)
+; RV64I-NEXT:    mv s1, a1
 ; RV64I-NEXT:    addi s2, sp, 8
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    bltu s0, a2, .LBB218_3
+; RV64I-NEXT:    j .LBB218_2
 ; RV64I-NEXT:  .LBB218_1: # %atomicrmw.start
-; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s2
+; RV64I-NEXT:    # in Loop: Header=BB218_2 Depth=1
+; RV64I-NEXT:    sd a3, 8(sp)
 ; RV64I-NEXT:    addi a3, zero, 4
 ; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    call __atomic_compare_exchange_8
-; RV64I-NEXT:    ld a2, 8(sp)
+; RV64I-NEXT:    ld a3, 8(sp)
 ; RV64I-NEXT:    bnez a0, .LBB218_4
-; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV64I-NEXT:    # in Loop: Header=BB218_1 Depth=1
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    bgeu s0, a2, .LBB218_1
-; RV64I-NEXT:  .LBB218_3: # %atomicrmw.start
-; RV64I-NEXT:    mv a2, s0
+; RV64I-NEXT:  .LBB218_2: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bgeu s1, a3, .LBB218_1
+; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB218_2 Depth=1
+; RV64I-NEXT:    mv a2, s1
 ; RV64I-NEXT:    j .LBB218_1
 ; RV64I-NEXT:  .LBB218_4: # %atomicrmw.end
-; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
 ; RV64I-NEXT:    ld s0, 32(sp)
@@ -17651,42 +17918,48 @@ define i64 @atomicrmw_umin_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
 ; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    mv s0, a2
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a5, 4(a0)
+; RV32I-NEXT:    lw a4, 0(a0)
+; RV32I-NEXT:    mv s1, a2
 ; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a1, 4(a0)
-; RV32I-NEXT:    lw a2, 0(a0)
 ; RV32I-NEXT:    mv s3, sp
-; RV32I-NEXT:    bne a1, s0, .LBB219_3
-; RV32I-NEXT:    j .LBB219_4
+; RV32I-NEXT:    j .LBB219_2
 ; RV32I-NEXT:  .LBB219_1: # %atomicrmw.start
-; RV32I-NEXT:    sw a1, 4(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    # in Loop: Header=BB219_2 Depth=1
+; RV32I-NEXT:    sw a4, 0(sp)
+; RV32I-NEXT:    sw a5, 4(sp)
 ; RV32I-NEXT:    addi a4, zero, 5
 ; RV32I-NEXT:    addi a5, zero, 5
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
-; RV32I-NEXT:    lw a1, 4(sp)
-; RV32I-NEXT:    lw a2, 0(sp)
+; RV32I-NEXT:    lw a5, 4(sp)
+; RV32I-NEXT:    lw a4, 0(sp)
 ; RV32I-NEXT:    bnez a0, .LBB219_7
-; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32I-NEXT:    beq a1, s0, .LBB219_4
-; RV32I-NEXT:  .LBB219_3: # %atomicrmw.start
-; RV32I-NEXT:    sltu a0, s0, a1
+; RV32I-NEXT:  .LBB219_2: # %atomicrmw.start
+; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32I-NEXT:    beq a5, s1, .LBB219_4
+; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB219_2 Depth=1
+; RV32I-NEXT:    sltu a0, s1, a5
 ; RV32I-NEXT:    j .LBB219_5
 ; RV32I-NEXT:  .LBB219_4:
-; RV32I-NEXT:    sltu a0, s2, a2
+; RV32I-NEXT:    sltu a0, s2, a4
 ; RV32I-NEXT:  .LBB219_5: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB219_2 Depth=1
 ; RV32I-NEXT:    xori a0, a0, 1
-; RV32I-NEXT:    sw a2, 0(sp)
-; RV32I-NEXT:    mv a3, a1
+; RV32I-NEXT:    mv a2, a4
+; RV32I-NEXT:    mv a3, a5
 ; RV32I-NEXT:    bnez a0, .LBB219_1
 ; RV32I-NEXT:  # %bb.6: # %atomicrmw.start
+; RV32I-NEXT:    # in Loop: Header=BB219_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
-; RV32I-NEXT:    mv a3, s0
+; RV32I-NEXT:    mv a3, s1
 ; RV32I-NEXT:    j .LBB219_1
 ; RV32I-NEXT:  .LBB219_7: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a4
+; RV32I-NEXT:    mv a1, a5
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -17703,42 +17976,48 @@ define i64 @atomicrmw_umin_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    sw s1, 20(sp)
 ; RV32IA-NEXT:    sw s2, 16(sp)
 ; RV32IA-NEXT:    sw s3, 12(sp)
-; RV32IA-NEXT:    mv s0, a2
+; RV32IA-NEXT:    mv s0, a0
+; RV32IA-NEXT:    lw a5, 4(a0)
+; RV32IA-NEXT:    lw a4, 0(a0)
+; RV32IA-NEXT:    mv s1, a2
 ; RV32IA-NEXT:    mv s2, a1
-; RV32IA-NEXT:    mv s1, a0
-; RV32IA-NEXT:    lw a1, 4(a0)
-; RV32IA-NEXT:    lw a2, 0(a0)
 ; RV32IA-NEXT:    mv s3, sp
-; RV32IA-NEXT:    bne a1, s0, .LBB219_3
-; RV32IA-NEXT:    j .LBB219_4
+; RV32IA-NEXT:    j .LBB219_2
 ; RV32IA-NEXT:  .LBB219_1: # %atomicrmw.start
-; RV32IA-NEXT:    sw a1, 4(sp)
-; RV32IA-NEXT:    mv a0, s1
-; RV32IA-NEXT:    mv a1, s3
+; RV32IA-NEXT:    # in Loop: Header=BB219_2 Depth=1
+; RV32IA-NEXT:    sw a4, 0(sp)
+; RV32IA-NEXT:    sw a5, 4(sp)
 ; RV32IA-NEXT:    addi a4, zero, 5
 ; RV32IA-NEXT:    addi a5, zero, 5
+; RV32IA-NEXT:    mv a0, s0
+; RV32IA-NEXT:    mv a1, s3
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
-; RV32IA-NEXT:    lw a1, 4(sp)
-; RV32IA-NEXT:    lw a2, 0(sp)
+; RV32IA-NEXT:    lw a5, 4(sp)
+; RV32IA-NEXT:    lw a4, 0(sp)
 ; RV32IA-NEXT:    bnez a0, .LBB219_7
-; RV32IA-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32IA-NEXT:    beq a1, s0, .LBB219_4
-; RV32IA-NEXT:  .LBB219_3: # %atomicrmw.start
-; RV32IA-NEXT:    sltu a0, s0, a1
+; RV32IA-NEXT:  .LBB219_2: # %atomicrmw.start
+; RV32IA-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32IA-NEXT:    beq a5, s1, .LBB219_4
+; RV32IA-NEXT:  # %bb.3: # %atomicrmw.start
+; RV32IA-NEXT:    # in Loop: Header=BB219_2 Depth=1
+; RV32IA-NEXT:    sltu a0, s1, a5
 ; RV32IA-NEXT:    j .LBB219_5
 ; RV32IA-NEXT:  .LBB219_4:
-; RV32IA-NEXT:    sltu a0, s2, a2
+; RV32IA-NEXT:    sltu a0, s2, a4
 ; RV32IA-NEXT:  .LBB219_5: # %atomicrmw.start
+; RV32IA-NEXT:    # in Loop: Header=BB219_2 Depth=1
 ; RV32IA-NEXT:    xori a0, a0, 1
-; RV32IA-NEXT:    sw a2, 0(sp)
-; RV32IA-NEXT:    mv a3, a1
+; RV32IA-NEXT:    mv a2, a4
+; RV32IA-NEXT:    mv a3, a5
 ; RV32IA-NEXT:    bnez a0, .LBB219_1
 ; RV32IA-NEXT:  # %bb.6: # %atomicrmw.start
+; RV32IA-NEXT:    # in Loop: Header=BB219_2 Depth=1
 ; RV32IA-NEXT:    mv a2, s2
-; RV32IA-NEXT:    mv a3, s0
+; RV32IA-NEXT:    mv a3, s1
 ; RV32IA-NEXT:    j .LBB219_1
 ; RV32IA-NEXT:  .LBB219_7: # %atomicrmw.end
-; RV32IA-NEXT:    mv a0, a2
+; RV32IA-NEXT:    mv a0, a4
+; RV32IA-NEXT:    mv a1, a5
 ; RV32IA-NEXT:    lw s3, 12(sp)
 ; RV32IA-NEXT:    lw s2, 16(sp)
 ; RV32IA-NEXT:    lw s1, 20(sp)
@@ -17754,30 +18033,31 @@ define i64 @atomicrmw_umin_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    sd s0, 32(sp)
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
-; RV64I-NEXT:    mv s0, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    ld a2, 0(a0)
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    ld a3, 0(a0)
+; RV64I-NEXT:    mv s1, a1
 ; RV64I-NEXT:    addi s2, sp, 8
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    bltu s0, a2, .LBB219_3
+; RV64I-NEXT:    j .LBB219_2
 ; RV64I-NEXT:  .LBB219_1: # %atomicrmw.start
-; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s2
+; RV64I-NEXT:    # in Loop: Header=BB219_2 Depth=1
+; RV64I-NEXT:    sd a3, 8(sp)
 ; RV64I-NEXT:    addi a3, zero, 5
 ; RV64I-NEXT:    addi a4, zero, 5
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    call __atomic_compare_exchange_8
-; RV64I-NEXT:    ld a2, 8(sp)
+; RV64I-NEXT:    ld a3, 8(sp)
 ; RV64I-NEXT:    bnez a0, .LBB219_4
-; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV64I-NEXT:    # in Loop: Header=BB219_1 Depth=1
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    bgeu s0, a2, .LBB219_1
-; RV64I-NEXT:  .LBB219_3: # %atomicrmw.start
-; RV64I-NEXT:    mv a2, s0
+; RV64I-NEXT:  .LBB219_2: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bgeu s1, a3, .LBB219_1
+; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB219_2 Depth=1
+; RV64I-NEXT:    mv a2, s1
 ; RV64I-NEXT:    j .LBB219_1
 ; RV64I-NEXT:  .LBB219_4: # %atomicrmw.end
-; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
 ; RV64I-NEXT:    ld s0, 32(sp)
diff --git a/llvm/test/CodeGen/RISCV/bare-select.ll b/llvm/test/CodeGen/RISCV/bare-select.ll
index 1b8f2f63e..cf8fe9674 100644
--- a/llvm/test/CodeGen/RISCV/bare-select.ll
+++ b/llvm/test/CodeGen/RISCV/bare-select.ll
@@ -5,12 +5,12 @@
 define i32 @bare_select(i1 %a, i32 %b, i32 %c) nounwind {
 ; RV32I-LABEL: bare_select:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    andi a0, a0, 1
-; RV32I-NEXT:    bnez a0, .LBB0_2
+; RV32I-NEXT:    andi a3, a0, 1
+; RV32I-NEXT:    mv a0, a1
+; RV32I-NEXT:    bnez a3, .LBB0_2
 ; RV32I-NEXT:  # %bb.1:
-; RV32I-NEXT:    mv a1, a2
+; RV32I-NEXT:    mv a0, a2
 ; RV32I-NEXT:  .LBB0_2:
-; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    ret
   %1 = select i1 %a, i32 %b, i32 %c
   ret i32 %1
@@ -19,12 +19,12 @@ define i32 @bare_select(i1 %a, i32 %b, i32 %c) nounwind {
 define float @bare_select_float(i1 %a, float %b, float %c) nounwind {
 ; RV32I-LABEL: bare_select_float:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    andi a0, a0, 1
-; RV32I-NEXT:    bnez a0, .LBB1_2
+; RV32I-NEXT:    andi a3, a0, 1
+; RV32I-NEXT:    mv a0, a1
+; RV32I-NEXT:    bnez a3, .LBB1_2
 ; RV32I-NEXT:  # %bb.1:
-; RV32I-NEXT:    mv a1, a2
+; RV32I-NEXT:    mv a0, a2
 ; RV32I-NEXT:  .LBB1_2:
-; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    ret
   %1 = select i1 %a, float %b, float %c
   ret float %1
diff --git a/llvm/test/CodeGen/RISCV/blockaddress.ll b/llvm/test/CodeGen/RISCV/blockaddress.ll
index 5b7f81876..d70f31480 100644
--- a/llvm/test/CodeGen/RISCV/blockaddress.ll
+++ b/llvm/test/CodeGen/RISCV/blockaddress.ll
@@ -9,11 +9,11 @@ define void @test_blockaddress() nounwind {
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    lui a0, %hi(.Ltmp0)
-; RV32I-NEXT:    addi a0, a0, %lo(.Ltmp0)
-; RV32I-NEXT:    lui a1, %hi(addr)
-; RV32I-NEXT:    sw a0, %lo(addr)(a1)
-; RV32I-NEXT:    lw a0, %lo(addr)(a1)
+; RV32I-NEXT:    lui a0, %hi(addr)
+; RV32I-NEXT:    lui a1, %hi(.Ltmp0)
+; RV32I-NEXT:    addi a1, a1, %lo(.Ltmp0)
+; RV32I-NEXT:    sw a1, %lo(addr)(a0)
+; RV32I-NEXT:    lw a0, %lo(addr)(a0)
 ; RV32I-NEXT:    jr a0
 ; RV32I-NEXT:  .Ltmp0: # Block address taken
 ; RV32I-NEXT:  .LBB0_1: # %block
diff --git a/llvm/test/CodeGen/RISCV/bswap-ctlz-cttz-ctpop.ll b/llvm/test/CodeGen/RISCV/bswap-ctlz-cttz-ctpop.ll
index 0a29f24a2..d896d9fe0 100644
--- a/llvm/test/CodeGen/RISCV/bswap-ctlz-cttz-ctpop.ll
+++ b/llvm/test/CodeGen/RISCV/bswap-ctlz-cttz-ctpop.ll
@@ -29,10 +29,10 @@ define i16 @test_bswap_i16(i16 %a) nounwind {
 define i32 @test_bswap_i32(i32 %a) nounwind {
 ; RV32I-LABEL: test_bswap_i32:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    lui a1, 16
-; RV32I-NEXT:    addi a1, a1, -256
-; RV32I-NEXT:    srli a2, a0, 8
-; RV32I-NEXT:    and a1, a2, a1
+; RV32I-NEXT:    srli a1, a0, 8
+; RV32I-NEXT:    lui a2, 16
+; RV32I-NEXT:    addi a2, a2, -256
+; RV32I-NEXT:    and a1, a1, a2
 ; RV32I-NEXT:    srli a2, a0, 24
 ; RV32I-NEXT:    or a1, a1, a2
 ; RV32I-NEXT:    slli a2, a0, 8
@@ -49,9 +49,9 @@ define i32 @test_bswap_i32(i32 %a) nounwind {
 define i64 @test_bswap_i64(i64 %a) nounwind {
 ; RV32I-LABEL: test_bswap_i64:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    lui a2, 16
-; RV32I-NEXT:    addi a3, a2, -256
 ; RV32I-NEXT:    srli a2, a1, 8
+; RV32I-NEXT:    lui a3, 16
+; RV32I-NEXT:    addi a3, a3, -256
 ; RV32I-NEXT:    and a2, a2, a3
 ; RV32I-NEXT:    srli a4, a1, 24
 ; RV32I-NEXT:    or a2, a2, a4
@@ -87,10 +87,10 @@ define i8 @test_cttz_i8(i8 %a) nounwind {
 ; RV32I-NEXT:    addi a1, a0, -1
 ; RV32I-NEXT:    not a0, a0
 ; RV32I-NEXT:    and a0, a0, a1
-; RV32I-NEXT:    lui a1, 349525
-; RV32I-NEXT:    addi a1, a1, 1365
-; RV32I-NEXT:    srli a2, a0, 1
-; RV32I-NEXT:    and a1, a2, a1
+; RV32I-NEXT:    srli a1, a0, 1
+; RV32I-NEXT:    lui a2, 349525
+; RV32I-NEXT:    addi a2, a2, 1365
+; RV32I-NEXT:    and a1, a1, a2
 ; RV32I-NEXT:    sub a0, a0, a1
 ; RV32I-NEXT:    lui a1, 209715
 ; RV32I-NEXT:    addi a1, a1, 819
@@ -131,10 +131,10 @@ define i16 @test_cttz_i16(i16 %a) nounwind {
 ; RV32I-NEXT:    addi a1, a0, -1
 ; RV32I-NEXT:    not a0, a0
 ; RV32I-NEXT:    and a0, a0, a1
-; RV32I-NEXT:    lui a1, 349525
-; RV32I-NEXT:    addi a1, a1, 1365
-; RV32I-NEXT:    srli a2, a0, 1
-; RV32I-NEXT:    and a1, a2, a1
+; RV32I-NEXT:    srli a1, a0, 1
+; RV32I-NEXT:    lui a2, 349525
+; RV32I-NEXT:    addi a2, a2, 1365
+; RV32I-NEXT:    and a1, a1, a2
 ; RV32I-NEXT:    sub a0, a0, a1
 ; RV32I-NEXT:    lui a1, 209715
 ; RV32I-NEXT:    addi a1, a1, 819
@@ -172,10 +172,10 @@ define i32 @test_cttz_i32(i32 %a) nounwind {
 ; RV32I-NEXT:    addi a1, a0, -1
 ; RV32I-NEXT:    not a0, a0
 ; RV32I-NEXT:    and a0, a0, a1
-; RV32I-NEXT:    lui a1, 349525
-; RV32I-NEXT:    addi a1, a1, 1365
-; RV32I-NEXT:    srli a2, a0, 1
-; RV32I-NEXT:    and a1, a2, a1
+; RV32I-NEXT:    srli a1, a0, 1
+; RV32I-NEXT:    lui a2, 349525
+; RV32I-NEXT:    addi a2, a2, 1365
+; RV32I-NEXT:    and a1, a1, a2
 ; RV32I-NEXT:    sub a0, a0, a1
 ; RV32I-NEXT:    lui a1, 209715
 ; RV32I-NEXT:    addi a1, a1, 819
@@ -220,11 +220,11 @@ define i32 @test_ctlz_i32(i32 %a) nounwind {
 ; RV32I-NEXT:    or a0, a0, a1
 ; RV32I-NEXT:    srli a1, a0, 16
 ; RV32I-NEXT:    or a0, a0, a1
-; RV32I-NEXT:    lui a1, 349525
-; RV32I-NEXT:    addi a1, a1, 1365
 ; RV32I-NEXT:    not a0, a0
-; RV32I-NEXT:    srli a2, a0, 1
-; RV32I-NEXT:    and a1, a2, a1
+; RV32I-NEXT:    srli a1, a0, 1
+; RV32I-NEXT:    lui a2, 349525
+; RV32I-NEXT:    addi a2, a2, 1365
+; RV32I-NEXT:    and a1, a1, a2
 ; RV32I-NEXT:    sub a0, a0, a1
 ; RV32I-NEXT:    lui a1, 209715
 ; RV32I-NEXT:    addi a1, a1, 819
@@ -269,9 +269,9 @@ define i64 @test_cttz_i64(i64 %a) nounwind {
 ; RV32I-NEXT:    addi a0, a0, -1
 ; RV32I-NEXT:    not a1, s4
 ; RV32I-NEXT:    and a0, a1, a0
-; RV32I-NEXT:    lui a1, 349525
-; RV32I-NEXT:    addi s5, a1, 1365
 ; RV32I-NEXT:    srli a1, a0, 1
+; RV32I-NEXT:    lui a2, 349525
+; RV32I-NEXT:    addi s5, a2, 1365
 ; RV32I-NEXT:    and a1, a1, s5
 ; RV32I-NEXT:    sub a0, a0, a1
 ; RV32I-NEXT:    lui a1, 209715
@@ -282,12 +282,12 @@ define i64 @test_cttz_i64(i64 %a) nounwind {
 ; RV32I-NEXT:    add a0, a1, a0
 ; RV32I-NEXT:    srli a1, a0, 4
 ; RV32I-NEXT:    add a0, a0, a1
-; RV32I-NEXT:    lui a1, 4112
-; RV32I-NEXT:    addi s6, a1, 257
 ; RV32I-NEXT:    lui a1, 61681
-; RV32I-NEXT:    addi s1, a1, -241
-; RV32I-NEXT:    and a0, a0, s1
-; RV32I-NEXT:    mv a1, s6
+; RV32I-NEXT:    addi s6, a1, -241
+; RV32I-NEXT:    and a0, a0, s6
+; RV32I-NEXT:    lui a1, 4112
+; RV32I-NEXT:    addi s1, a1, 257
+; RV32I-NEXT:    mv a1, s1
 ; RV32I-NEXT:    call __mulsi3
 ; RV32I-NEXT:    mv s2, a0
 ; RV32I-NEXT:    addi a0, s3, -1
@@ -302,8 +302,8 @@ define i64 @test_cttz_i64(i64 %a) nounwind {
 ; RV32I-NEXT:    add a0, a1, a0
 ; RV32I-NEXT:    srli a1, a0, 4
 ; RV32I-NEXT:    add a0, a0, a1
-; RV32I-NEXT:    and a0, a0, s1
-; RV32I-NEXT:    mv a1, s6
+; RV32I-NEXT:    and a0, a0, s6
+; RV32I-NEXT:    mv a1, s1
 ; RV32I-NEXT:    call __mulsi3
 ; RV32I-NEXT:    bnez s4, .LBB7_2
 ; RV32I-NEXT:  # %bb.1:
@@ -336,10 +336,10 @@ define i8 @test_cttz_i8_zero_undef(i8 %a) nounwind {
 ; RV32I-NEXT:    addi a1, a0, -1
 ; RV32I-NEXT:    not a0, a0
 ; RV32I-NEXT:    and a0, a0, a1
-; RV32I-NEXT:    lui a1, 349525
-; RV32I-NEXT:    addi a1, a1, 1365
-; RV32I-NEXT:    srli a2, a0, 1
-; RV32I-NEXT:    and a1, a2, a1
+; RV32I-NEXT:    srli a1, a0, 1
+; RV32I-NEXT:    lui a2, 349525
+; RV32I-NEXT:    addi a2, a2, 1365
+; RV32I-NEXT:    and a1, a1, a2
 ; RV32I-NEXT:    sub a0, a0, a1
 ; RV32I-NEXT:    lui a1, 209715
 ; RV32I-NEXT:    addi a1, a1, 819
@@ -371,10 +371,10 @@ define i16 @test_cttz_i16_zero_undef(i16 %a) nounwind {
 ; RV32I-NEXT:    addi a1, a0, -1
 ; RV32I-NEXT:    not a0, a0
 ; RV32I-NEXT:    and a0, a0, a1
-; RV32I-NEXT:    lui a1, 349525
-; RV32I-NEXT:    addi a1, a1, 1365
-; RV32I-NEXT:    srli a2, a0, 1
-; RV32I-NEXT:    and a1, a2, a1
+; RV32I-NEXT:    srli a1, a0, 1
+; RV32I-NEXT:    lui a2, 349525
+; RV32I-NEXT:    addi a2, a2, 1365
+; RV32I-NEXT:    and a1, a1, a2
 ; RV32I-NEXT:    sub a0, a0, a1
 ; RV32I-NEXT:    lui a1, 209715
 ; RV32I-NEXT:    addi a1, a1, 819
@@ -406,10 +406,10 @@ define i32 @test_cttz_i32_zero_undef(i32 %a) nounwind {
 ; RV32I-NEXT:    addi a1, a0, -1
 ; RV32I-NEXT:    not a0, a0
 ; RV32I-NEXT:    and a0, a0, a1
-; RV32I-NEXT:    lui a1, 349525
-; RV32I-NEXT:    addi a1, a1, 1365
-; RV32I-NEXT:    srli a2, a0, 1
-; RV32I-NEXT:    and a1, a2, a1
+; RV32I-NEXT:    srli a1, a0, 1
+; RV32I-NEXT:    lui a2, 349525
+; RV32I-NEXT:    addi a2, a2, 1365
+; RV32I-NEXT:    and a1, a1, a2
 ; RV32I-NEXT:    sub a0, a0, a1
 ; RV32I-NEXT:    lui a1, 209715
 ; RV32I-NEXT:    addi a1, a1, 819
@@ -450,9 +450,9 @@ define i64 @test_cttz_i64_zero_undef(i64 %a) nounwind {
 ; RV32I-NEXT:    addi a0, a0, -1
 ; RV32I-NEXT:    not a1, s4
 ; RV32I-NEXT:    and a0, a1, a0
-; RV32I-NEXT:    lui a1, 349525
-; RV32I-NEXT:    addi s5, a1, 1365
 ; RV32I-NEXT:    srli a1, a0, 1
+; RV32I-NEXT:    lui a2, 349525
+; RV32I-NEXT:    addi s5, a2, 1365
 ; RV32I-NEXT:    and a1, a1, s5
 ; RV32I-NEXT:    sub a0, a0, a1
 ; RV32I-NEXT:    lui a1, 209715
@@ -463,12 +463,12 @@ define i64 @test_cttz_i64_zero_undef(i64 %a) nounwind {
 ; RV32I-NEXT:    add a0, a1, a0
 ; RV32I-NEXT:    srli a1, a0, 4
 ; RV32I-NEXT:    add a0, a0, a1
-; RV32I-NEXT:    lui a1, 4112
-; RV32I-NEXT:    addi s6, a1, 257
 ; RV32I-NEXT:    lui a1, 61681
-; RV32I-NEXT:    addi s1, a1, -241
-; RV32I-NEXT:    and a0, a0, s1
-; RV32I-NEXT:    mv a1, s6
+; RV32I-NEXT:    addi s6, a1, -241
+; RV32I-NEXT:    and a0, a0, s6
+; RV32I-NEXT:    lui a1, 4112
+; RV32I-NEXT:    addi s1, a1, 257
+; RV32I-NEXT:    mv a1, s1
 ; RV32I-NEXT:    call __mulsi3
 ; RV32I-NEXT:    mv s2, a0
 ; RV32I-NEXT:    addi a0, s3, -1
@@ -483,8 +483,8 @@ define i64 @test_cttz_i64_zero_undef(i64 %a) nounwind {
 ; RV32I-NEXT:    add a0, a1, a0
 ; RV32I-NEXT:    srli a1, a0, 4
 ; RV32I-NEXT:    add a0, a0, a1
-; RV32I-NEXT:    and a0, a0, s1
-; RV32I-NEXT:    mv a1, s6
+; RV32I-NEXT:    and a0, a0, s6
+; RV32I-NEXT:    mv a1, s1
 ; RV32I-NEXT:    call __mulsi3
 ; RV32I-NEXT:    bnez s4, .LBB11_2
 ; RV32I-NEXT:  # %bb.1:
@@ -514,10 +514,10 @@ define i32 @test_ctpop_i32(i32 %a) nounwind {
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    lui a1, 349525
-; RV32I-NEXT:    addi a1, a1, 1365
-; RV32I-NEXT:    srli a2, a0, 1
-; RV32I-NEXT:    and a1, a2, a1
+; RV32I-NEXT:    srli a1, a0, 1
+; RV32I-NEXT:    lui a2, 349525
+; RV32I-NEXT:    addi a2, a2, 1365
+; RV32I-NEXT:    and a1, a1, a2
 ; RV32I-NEXT:    sub a0, a0, a1
 ; RV32I-NEXT:    lui a1, 209715
 ; RV32I-NEXT:    addi a1, a1, 819
diff --git a/llvm/test/CodeGen/RISCV/callee-saved-fpr32s.ll b/llvm/test/CodeGen/RISCV/callee-saved-fpr32s.ll
index ac1f4dafe..56d3ff04d 100644
--- a/llvm/test/CodeGen/RISCV/callee-saved-fpr32s.ll
+++ b/llvm/test/CodeGen/RISCV/callee-saved-fpr32s.ll
@@ -23,8 +23,8 @@ define void @callee() nounwind {
 ; ILP32-LP64-LABEL: callee:
 ; ILP32-LP64:       # %bb.0:
 ; ILP32-LP64-NEXT:    lui a0, %hi(var)
-; ILP32-LP64-NEXT:    addi a1, a0, %lo(var)
 ; ILP32-LP64-NEXT:    flw ft0, %lo(var)(a0)
+; ILP32-LP64-NEXT:    addi a1, a0, %lo(var)
 ; ILP32-LP64-NEXT:    flw ft1, 4(a1)
 ; ILP32-LP64-NEXT:    flw ft2, 8(a1)
 ; ILP32-LP64-NEXT:    flw ft3, 12(a1)
@@ -52,14 +52,14 @@ define void @callee() nounwind {
 ; ILP32-LP64-NEXT:    flw fs5, 100(a1)
 ; ILP32-LP64-NEXT:    flw fs6, 104(a1)
 ; ILP32-LP64-NEXT:    flw fs7, 108(a1)
-; ILP32-LP64-NEXT:    flw fs8, 112(a1)
-; ILP32-LP64-NEXT:    flw fs9, 116(a1)
-; ILP32-LP64-NEXT:    flw fs10, 120(a1)
-; ILP32-LP64-NEXT:    flw fs11, 124(a1)
-; ILP32-LP64-NEXT:    fsw fs11, 124(a1)
-; ILP32-LP64-NEXT:    fsw fs10, 120(a1)
-; ILP32-LP64-NEXT:    fsw fs9, 116(a1)
-; ILP32-LP64-NEXT:    fsw fs8, 112(a1)
+; ILP32-LP64-NEXT:    flw fs8, 124(a1)
+; ILP32-LP64-NEXT:    flw fs9, 120(a1)
+; ILP32-LP64-NEXT:    flw fs10, 116(a1)
+; ILP32-LP64-NEXT:    flw fs11, 112(a1)
+; ILP32-LP64-NEXT:    fsw fs8, 124(a1)
+; ILP32-LP64-NEXT:    fsw fs9, 120(a1)
+; ILP32-LP64-NEXT:    fsw fs10, 116(a1)
+; ILP32-LP64-NEXT:    fsw fs11, 112(a1)
 ; ILP32-LP64-NEXT:    fsw fs7, 108(a1)
 ; ILP32-LP64-NEXT:    fsw fs6, 104(a1)
 ; ILP32-LP64-NEXT:    fsw fs5, 100(a1)
@@ -106,6 +106,7 @@ define void @callee() nounwind {
 ; ILP32F-LP64F-NEXT:    fsw fs10, 4(sp)
 ; ILP32F-LP64F-NEXT:    fsw fs11, 0(sp)
 ; ILP32F-LP64F-NEXT:    lui a0, %hi(var)
+; ILP32F-LP64F-NEXT:    flw ft0, %lo(var)(a0)
 ; ILP32F-LP64F-NEXT:    addi a1, a0, %lo(var)
 ;
 ; ILP32D-LP64D-LABEL: callee:
@@ -124,6 +125,7 @@ define void @callee() nounwind {
 ; ILP32D-LP64D-NEXT:    fsd fs10, 8(sp)
 ; ILP32D-LP64D-NEXT:    fsd fs11, 0(sp)
 ; ILP32D-LP64D-NEXT:    lui a0, %hi(var)
+; ILP32D-LP64D-NEXT:    flw ft0, %lo(var)(a0)
 ; ILP32D-LP64D-NEXT:    addi a1, a0, %lo(var)
   %val = load [32 x float], [32 x float]* @var
   store volatile [32 x float] %val, [32 x float]* @var
diff --git a/llvm/test/CodeGen/RISCV/callee-saved-fpr64s.ll b/llvm/test/CodeGen/RISCV/callee-saved-fpr64s.ll
index cc51b10fe..f95bc4573 100644
--- a/llvm/test/CodeGen/RISCV/callee-saved-fpr64s.ll
+++ b/llvm/test/CodeGen/RISCV/callee-saved-fpr64s.ll
@@ -19,8 +19,8 @@ define void @callee() nounwind {
 ; ILP32-LP64-LABEL: callee:
 ; ILP32-LP64:       # %bb.0:
 ; ILP32-LP64-NEXT:    lui a0, %hi(var)
-; ILP32-LP64-NEXT:    addi a1, a0, %lo(var)
 ; ILP32-LP64-NEXT:    fld ft0, %lo(var)(a0)
+; ILP32-LP64-NEXT:    addi a1, a0, %lo(var)
 ; ILP32-LP64-NEXT:    fld ft1, 8(a1)
 ; ILP32-LP64-NEXT:    fld ft2, 16(a1)
 ; ILP32-LP64-NEXT:    fld ft3, 24(a1)
@@ -48,14 +48,14 @@ define void @callee() nounwind {
 ; ILP32-LP64-NEXT:    fld fs5, 200(a1)
 ; ILP32-LP64-NEXT:    fld fs6, 208(a1)
 ; ILP32-LP64-NEXT:    fld fs7, 216(a1)
-; ILP32-LP64-NEXT:    fld fs8, 224(a1)
-; ILP32-LP64-NEXT:    fld fs9, 232(a1)
-; ILP32-LP64-NEXT:    fld fs10, 240(a1)
-; ILP32-LP64-NEXT:    fld fs11, 248(a1)
-; ILP32-LP64-NEXT:    fsd fs11, 248(a1)
-; ILP32-LP64-NEXT:    fsd fs10, 240(a1)
-; ILP32-LP64-NEXT:    fsd fs9, 232(a1)
-; ILP32-LP64-NEXT:    fsd fs8, 224(a1)
+; ILP32-LP64-NEXT:    fld fs8, 248(a1)
+; ILP32-LP64-NEXT:    fld fs9, 240(a1)
+; ILP32-LP64-NEXT:    fld fs10, 232(a1)
+; ILP32-LP64-NEXT:    fld fs11, 224(a1)
+; ILP32-LP64-NEXT:    fsd fs8, 248(a1)
+; ILP32-LP64-NEXT:    fsd fs9, 240(a1)
+; ILP32-LP64-NEXT:    fsd fs10, 232(a1)
+; ILP32-LP64-NEXT:    fsd fs11, 224(a1)
 ; ILP32-LP64-NEXT:    fsd fs7, 216(a1)
 ; ILP32-LP64-NEXT:    fsd fs6, 208(a1)
 ; ILP32-LP64-NEXT:    fsd fs5, 200(a1)
@@ -102,6 +102,7 @@ define void @callee() nounwind {
 ; ILP32D-LP64D-NEXT:    fsd fs10, 8(sp)
 ; ILP32D-LP64D-NEXT:    fsd fs11, 0(sp)
 ; ILP32D-LP64D-NEXT:    lui a0, %hi(var)
+; ILP32D-LP64D-NEXT:    fld ft0, %lo(var)(a0)
 ; ILP32D-LP64D-NEXT:    addi a1, a0, %lo(var)
   %val = load [32 x double], [32 x double]* @var
   store volatile [32 x double] %val, [32 x double]* @var
diff --git a/llvm/test/CodeGen/RISCV/callee-saved-gprs.ll b/llvm/test/CodeGen/RISCV/callee-saved-gprs.ll
index 9bc7c55a9..eb3a4468b 100644
--- a/llvm/test/CodeGen/RISCV/callee-saved-gprs.ll
+++ b/llvm/test/CodeGen/RISCV/callee-saved-gprs.ll
@@ -10,11 +10,11 @@
 ; RUN:   | FileCheck %s -check-prefix=RV32I-WITH-FP
 ; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \
 ; RUN:   | FileCheck %s -check-prefix=RV64I
-; RUN: llc -mtriple=riscv64 -mattr=+f -target-abi ilp32f -verify-machineinstrs < %s \
+; RUN: llc -mtriple=riscv64 -mattr=+f -target-abi lp64f -verify-machineinstrs < %s \
 ; RUN:   | FileCheck %s -check-prefix=RV64I
-; RUN: llc -mtriple=riscv64 -mattr=+d -target-abi ilp32f -verify-machineinstrs < %s \
+; RUN: llc -mtriple=riscv64 -mattr=+d -target-abi lp64f -verify-machineinstrs < %s \
 ; RUN:   | FileCheck %s -check-prefix=RV64I
-; RUN: llc -mtriple=riscv64 -mattr=+d -target-abi ilp32d -verify-machineinstrs < %s \
+; RUN: llc -mtriple=riscv64 -mattr=+d -target-abi lp64d -verify-machineinstrs < %s \
 ; RUN:   | FileCheck %s -check-prefix=RV64I
 ; RUN: llc -mtriple=riscv64 -verify-machineinstrs -frame-pointer=all < %s \
 ; RUN:   | FileCheck %s -check-prefix=RV64I-WITH-FP
@@ -28,20 +28,23 @@ define void @callee() nounwind {
 ; RV32I-LABEL: callee:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -80
-; RV32I-NEXT:    sw s0, 76(sp)
-; RV32I-NEXT:    sw s1, 72(sp)
-; RV32I-NEXT:    sw s2, 68(sp)
-; RV32I-NEXT:    sw s3, 64(sp)
-; RV32I-NEXT:    sw s4, 60(sp)
-; RV32I-NEXT:    sw s5, 56(sp)
-; RV32I-NEXT:    sw s6, 52(sp)
-; RV32I-NEXT:    sw s7, 48(sp)
-; RV32I-NEXT:    sw s8, 44(sp)
-; RV32I-NEXT:    sw s9, 40(sp)
-; RV32I-NEXT:    sw s10, 36(sp)
-; RV32I-NEXT:    sw s11, 32(sp)
+; RV32I-NEXT:    sw ra, 76(sp)
+; RV32I-NEXT:    sw s0, 72(sp)
+; RV32I-NEXT:    sw s1, 68(sp)
+; RV32I-NEXT:    sw s2, 64(sp)
+; RV32I-NEXT:    sw s3, 60(sp)
+; RV32I-NEXT:    sw s4, 56(sp)
+; RV32I-NEXT:    sw s5, 52(sp)
+; RV32I-NEXT:    sw s6, 48(sp)
+; RV32I-NEXT:    sw s7, 44(sp)
+; RV32I-NEXT:    sw s8, 40(sp)
+; RV32I-NEXT:    sw s9, 36(sp)
+; RV32I-NEXT:    sw s10, 32(sp)
+; RV32I-NEXT:    sw s11, 28(sp)
 ; RV32I-NEXT:    lui a0, %hi(var)
-; RV32I-NEXT:    addi a1, a0, %lo(var)
+; RV32I-NEXT:    lw a1, %lo(var)(a0)
+; RV32I-NEXT:    sw a1, 24(sp)
+; RV32I-NEXT:    addi a2, a0, %lo(var)
 ;
 ; RV32I-WITH-FP-LABEL: callee:
 ; RV32I-WITH-FP:       # %bb.0:
@@ -61,25 +64,30 @@ define void @callee() nounwind {
 ; RV32I-WITH-FP-NEXT:    sw s11, 28(sp)
 ; RV32I-WITH-FP-NEXT:    addi s0, sp, 80
 ; RV32I-WITH-FP-NEXT:    lui a0, %hi(var)
-; RV32I-WITH-FP-NEXT:    addi a1, a0, %lo(var)
+; RV32I-WITH-FP-NEXT:    lw a1, %lo(var)(a0)
+; RV32I-WITH-FP-NEXT:    sw a1, -56(s0)
+; RV32I-WITH-FP-NEXT:    addi a2, a0, %lo(var)
 ;
 ; RV64I-LABEL: callee:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -144
-; RV64I-NEXT:    sd s0, 136(sp)
-; RV64I-NEXT:    sd s1, 128(sp)
-; RV64I-NEXT:    sd s2, 120(sp)
-; RV64I-NEXT:    sd s3, 112(sp)
-; RV64I-NEXT:    sd s4, 104(sp)
-; RV64I-NEXT:    sd s5, 96(sp)
-; RV64I-NEXT:    sd s6, 88(sp)
-; RV64I-NEXT:    sd s7, 80(sp)
-; RV64I-NEXT:    sd s8, 72(sp)
-; RV64I-NEXT:    sd s9, 64(sp)
-; RV64I-NEXT:    sd s10, 56(sp)
-; RV64I-NEXT:    sd s11, 48(sp)
+; RV64I-NEXT:    sd ra, 136(sp)
+; RV64I-NEXT:    sd s0, 128(sp)
+; RV64I-NEXT:    sd s1, 120(sp)
+; RV64I-NEXT:    sd s2, 112(sp)
+; RV64I-NEXT:    sd s3, 104(sp)
+; RV64I-NEXT:    sd s4, 96(sp)
+; RV64I-NEXT:    sd s5, 88(sp)
+; RV64I-NEXT:    sd s6, 80(sp)
+; RV64I-NEXT:    sd s7, 72(sp)
+; RV64I-NEXT:    sd s8, 64(sp)
+; RV64I-NEXT:    sd s9, 56(sp)
+; RV64I-NEXT:    sd s10, 48(sp)
+; RV64I-NEXT:    sd s11, 40(sp)
 ; RV64I-NEXT:    lui a0, %hi(var)
-; RV64I-NEXT:    addi a1, a0, %lo(var)
+; RV64I-NEXT:    lw a1, %lo(var)(a0)
+; RV64I-NEXT:    sd a1, 32(sp)
+; RV64I-NEXT:    addi a2, a0, %lo(var)
 ;
 ; RV64I-WITH-FP-LABEL: callee:
 ; RV64I-WITH-FP:       # %bb.0:
@@ -99,7 +107,9 @@ define void @callee() nounwind {
 ; RV64I-WITH-FP-NEXT:    sd s11, 56(sp)
 ; RV64I-WITH-FP-NEXT:    addi s0, sp, 160
 ; RV64I-WITH-FP-NEXT:    lui a0, %hi(var)
-; RV64I-WITH-FP-NEXT:    addi a1, a0, %lo(var)
+; RV64I-WITH-FP-NEXT:    lw a1, %lo(var)(a0)
+; RV64I-WITH-FP-NEXT:    sd a1, -112(s0)
+; RV64I-WITH-FP-NEXT:    addi a2, a0, %lo(var)
   %val = load [32 x i32], [32 x i32]* @var
   store volatile [32 x i32] %val, [32 x i32]* @var
   ret void
@@ -111,36 +121,41 @@ define void @callee() nounwind {
 define void @caller() nounwind {
 ; RV32I-LABEL: caller:
 ; RV32I:         lui a0, %hi(var)
-; RV32I-NEXT:    addi s1, a0, %lo(var)
+; RV32I-NEXT:    lw a1, %lo(var)(a0)
+; RV32I-NEXT:    sw a1, 88(sp)
+; RV32I-NEXT:    addi s0, a0, %lo(var)
+
 ; RV32I:         sw a0, 8(sp)
-; RV32I-NEXT:    lw s2, 84(s1)
-; RV32I-NEXT:    lw s3, 88(s1)
-; RV32I-NEXT:    lw s4, 92(s1)
-; RV32I-NEXT:    lw s5, 96(s1)
-; RV32I-NEXT:    lw s6, 100(s1)
-; RV32I-NEXT:    lw s7, 104(s1)
-; RV32I-NEXT:    lw s8, 108(s1)
-; RV32I-NEXT:    lw s9, 112(s1)
-; RV32I-NEXT:    lw s10, 116(s1)
-; RV32I-NEXT:    lw s11, 120(s1)
-; RV32I-NEXT:    lw s0, 124(s1)
+; RV32I-NEXT:    lw s2, 84(s0)
+; RV32I-NEXT:    lw s3, 88(s0)
+; RV32I-NEXT:    lw s4, 92(s0)
+; RV32I-NEXT:    lw s5, 96(s0)
+; RV32I-NEXT:    lw s6, 100(s0)
+; RV32I-NEXT:    lw s7, 104(s0)
+; RV32I-NEXT:    lw s8, 108(s0)
+; RV32I-NEXT:    lw s9, 112(s0)
+; RV32I-NEXT:    lw s10, 116(s0)
+; RV32I-NEXT:    lw s11, 120(s0)
+; RV32I-NEXT:    lw s1, 124(s0)
 ; RV32I-NEXT:    call callee
-; RV32I-NEXT:    sw s0, 124(s1)
-; RV32I-NEXT:    sw s11, 120(s1)
-; RV32I-NEXT:    sw s10, 116(s1)
-; RV32I-NEXT:    sw s9, 112(s1)
-; RV32I-NEXT:    sw s8, 108(s1)
-; RV32I-NEXT:    sw s7, 104(s1)
-; RV32I-NEXT:    sw s6, 100(s1)
-; RV32I-NEXT:    sw s5, 96(s1)
-; RV32I-NEXT:    sw s4, 92(s1)
-; RV32I-NEXT:    sw s3, 88(s1)
-; RV32I-NEXT:    sw s2, 84(s1)
+; RV32I-NEXT:    sw s1, 124(s0)
+; RV32I-NEXT:    sw s11, 120(s0)
+; RV32I-NEXT:    sw s10, 116(s0)
+; RV32I-NEXT:    sw s9, 112(s0)
+; RV32I-NEXT:    sw s8, 108(s0)
+; RV32I-NEXT:    sw s7, 104(s0)
+; RV32I-NEXT:    sw s6, 100(s0)
+; RV32I-NEXT:    sw s5, 96(s0)
+; RV32I-NEXT:    sw s4, 92(s0)
+; RV32I-NEXT:    sw s3, 88(s0)
+; RV32I-NEXT:    sw s2, 84(s0)
 ; RV32I-NEXT:    lw a0, 8(sp)
 ;
 ; RV32I-WITH-FP-LABEL: caller:
 ; RV32I-WITH-FP:         addi s0, sp, 144
 ; RV32I-WITH-FP-NEXT:    lui a0, %hi(var)
+; RV32I-WITH-FP-NEXT:    lw a1, %lo(var)(a0)
+; RV32I-WITH-FP-NEXT:    sw a1, -56(s0)
 ; RV32I-WITH-FP-NEXT:    addi s1, a0, %lo(var)
 ; RV32I-WITH-FP:         sw a0, -140(s0)
 ; RV32I-WITH-FP-NEXT:    lw s5, 88(s1)
@@ -168,36 +183,40 @@ define void @caller() nounwind {
 ;
 ; RV64I-LABEL: caller:
 ; RV64I:         lui a0, %hi(var)
-; RV64I-NEXT:    addi s1, a0, %lo(var)
+; RV64I-NEXT:    lw a1, %lo(var)(a0)
+; RV64I-NEXT:    sd a1, 160(sp)
+; RV64I-NEXT:    addi s0, a0, %lo(var)
 ; RV64I:         sd a0, 0(sp)
-; RV64I-NEXT:    lw s2, 84(s1)
-; RV64I-NEXT:    lw s3, 88(s1)
-; RV64I-NEXT:    lw s4, 92(s1)
-; RV64I-NEXT:    lw s5, 96(s1)
-; RV64I-NEXT:    lw s6, 100(s1)
-; RV64I-NEXT:    lw s7, 104(s1)
-; RV64I-NEXT:    lw s8, 108(s1)
-; RV64I-NEXT:    lw s9, 112(s1)
-; RV64I-NEXT:    lw s10, 116(s1)
-; RV64I-NEXT:    lw s11, 120(s1)
-; RV64I-NEXT:    lw s0, 124(s1)
+; RV64I-NEXT:    lw s2, 84(s0)
+; RV64I-NEXT:    lw s3, 88(s0)
+; RV64I-NEXT:    lw s4, 92(s0)
+; RV64I-NEXT:    lw s5, 96(s0)
+; RV64I-NEXT:    lw s6, 100(s0)
+; RV64I-NEXT:    lw s7, 104(s0)
+; RV64I-NEXT:    lw s8, 108(s0)
+; RV64I-NEXT:    lw s9, 112(s0)
+; RV64I-NEXT:    lw s10, 116(s0)
+; RV64I-NEXT:    lw s11, 120(s0)
+; RV64I-NEXT:    lw s1, 124(s0)
 ; RV64I-NEXT:    call callee
-; RV64I-NEXT:    sw s0, 124(s1)
-; RV64I-NEXT:    sw s11, 120(s1)
-; RV64I-NEXT:    sw s10, 116(s1)
-; RV64I-NEXT:    sw s9, 112(s1)
-; RV64I-NEXT:    sw s8, 108(s1)
-; RV64I-NEXT:    sw s7, 104(s1)
-; RV64I-NEXT:    sw s6, 100(s1)
-; RV64I-NEXT:    sw s5, 96(s1)
-; RV64I-NEXT:    sw s4, 92(s1)
-; RV64I-NEXT:    sw s3, 88(s1)
-; RV64I-NEXT:    sw s2, 84(s1)
+; RV64I-NEXT:    sw s1, 124(s0)
+; RV64I-NEXT:    sw s11, 120(s0)
+; RV64I-NEXT:    sw s10, 116(s0)
+; RV64I-NEXT:    sw s9, 112(s0)
+; RV64I-NEXT:    sw s8, 108(s0)
+; RV64I-NEXT:    sw s7, 104(s0)
+; RV64I-NEXT:    sw s6, 100(s0)
+; RV64I-NEXT:    sw s5, 96(s0)
+; RV64I-NEXT:    sw s4, 92(s0)
+; RV64I-NEXT:    sw s3, 88(s0)
+; RV64I-NEXT:    sw s2, 84(s0)
 ; RV64I-NEXT:    ld a0, 0(sp)
 ;
 ; RV64I-WITH-FP-LABEL: caller:
 ; RV64I-WITH-FP:         addi s0, sp, 288
 ; RV64I-WITH-FP-NEXT:    lui a0, %hi(var)
+; RV64I-WITH-FP-NEXT:    lw a1, %lo(var)(a0)
+; RV64I-WITH-FP-NEXT:    sd a1, -112(s0)
 ; RV64I-WITH-FP-NEXT:    addi s1, a0, %lo(var)
 ; RV64I-WITH-FP:         sd a0, -280(s0)
 ; RV64I-WITH-FP-NEXT:    lw s5, 88(s1)
diff --git a/llvm/test/CodeGen/RISCV/calling-conv-ilp32-ilp32f-common.ll b/llvm/test/CodeGen/RISCV/calling-conv-ilp32-ilp32f-common.ll
index c93edcaec..24788e110 100644
--- a/llvm/test/CodeGen/RISCV/calling-conv-ilp32-ilp32f-common.ll
+++ b/llvm/test/CodeGen/RISCV/calling-conv-ilp32-ilp32f-common.ll
@@ -61,8 +61,8 @@ define i32 @caller_double_in_regs() nounwind {
 ; RV32I-FPELIM-NEXT:    addi sp, sp, -16
 ; RV32I-FPELIM-NEXT:    sw ra, 12(sp)
 ; RV32I-FPELIM-NEXT:    addi a0, zero, 1
-; RV32I-FPELIM-NEXT:    mv a1, zero
 ; RV32I-FPELIM-NEXT:    lui a2, 262144
+; RV32I-FPELIM-NEXT:    mv a1, zero
 ; RV32I-FPELIM-NEXT:    call callee_double_in_regs
 ; RV32I-FPELIM-NEXT:    lw ra, 12(sp)
 ; RV32I-FPELIM-NEXT:    addi sp, sp, 16
@@ -75,8 +75,8 @@ define i32 @caller_double_in_regs() nounwind {
 ; RV32I-WITHFP-NEXT:    sw s0, 8(sp)
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 16
 ; RV32I-WITHFP-NEXT:    addi a0, zero, 1
-; RV32I-WITHFP-NEXT:    mv a1, zero
 ; RV32I-WITHFP-NEXT:    lui a2, 262144
+; RV32I-WITHFP-NEXT:    mv a1, zero
 ; RV32I-WITHFP-NEXT:    call callee_double_in_regs
 ; RV32I-WITHFP-NEXT:    lw s0, 8(sp)
 ; RV32I-WITHFP-NEXT:    lw ra, 12(sp)
@@ -94,14 +94,14 @@ define i32 @callee_aligned_stack(i32 %a, i32 %b, fp128 %c, i32 %d, i32 %e, i64 %
 ; RV32I-FPELIM-LABEL: callee_aligned_stack:
 ; RV32I-FPELIM:       # %bb.0:
 ; RV32I-FPELIM-NEXT:    lw a0, 0(a2)
-; RV32I-FPELIM-NEXT:    add a0, a0, a7
-; RV32I-FPELIM-NEXT:    lw a1, 0(sp)
-; RV32I-FPELIM-NEXT:    add a0, a0, a1
-; RV32I-FPELIM-NEXT:    lw a1, 8(sp)
-; RV32I-FPELIM-NEXT:    add a0, a0, a1
-; RV32I-FPELIM-NEXT:    lw a1, 16(sp)
-; RV32I-FPELIM-NEXT:    add a0, a0, a1
 ; RV32I-FPELIM-NEXT:    lw a1, 20(sp)
+; RV32I-FPELIM-NEXT:    lw a2, 0(sp)
+; RV32I-FPELIM-NEXT:    lw a3, 8(sp)
+; RV32I-FPELIM-NEXT:    lw a4, 16(sp)
+; RV32I-FPELIM-NEXT:    add a0, a0, a7
+; RV32I-FPELIM-NEXT:    add a0, a0, a2
+; RV32I-FPELIM-NEXT:    add a0, a0, a3
+; RV32I-FPELIM-NEXT:    add a0, a0, a4
 ; RV32I-FPELIM-NEXT:    add a0, a0, a1
 ; RV32I-FPELIM-NEXT:    ret
 ;
@@ -112,14 +112,14 @@ define i32 @callee_aligned_stack(i32 %a, i32 %b, fp128 %c, i32 %d, i32 %e, i64 %
 ; RV32I-WITHFP-NEXT:    sw s0, 8(sp)
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 16
 ; RV32I-WITHFP-NEXT:    lw a0, 0(a2)
-; RV32I-WITHFP-NEXT:    add a0, a0, a7
-; RV32I-WITHFP-NEXT:    lw a1, 0(s0)
-; RV32I-WITHFP-NEXT:    add a0, a0, a1
-; RV32I-WITHFP-NEXT:    lw a1, 8(s0)
-; RV32I-WITHFP-NEXT:    add a0, a0, a1
-; RV32I-WITHFP-NEXT:    lw a1, 16(s0)
-; RV32I-WITHFP-NEXT:    add a0, a0, a1
 ; RV32I-WITHFP-NEXT:    lw a1, 20(s0)
+; RV32I-WITHFP-NEXT:    lw a2, 0(s0)
+; RV32I-WITHFP-NEXT:    lw a3, 8(s0)
+; RV32I-WITHFP-NEXT:    lw a4, 16(s0)
+; RV32I-WITHFP-NEXT:    add a0, a0, a7
+; RV32I-WITHFP-NEXT:    add a0, a0, a2
+; RV32I-WITHFP-NEXT:    add a0, a0, a3
+; RV32I-WITHFP-NEXT:    add a0, a0, a4
 ; RV32I-WITHFP-NEXT:    add a0, a0, a1
 ; RV32I-WITHFP-NEXT:    lw s0, 8(sp)
 ; RV32I-WITHFP-NEXT:    lw ra, 12(sp)
@@ -169,8 +169,7 @@ define void @caller_aligned_stack() nounwind {
 ; RV32I-FPELIM-NEXT:    addi a0, a0, -328
 ; RV32I-FPELIM-NEXT:    sw a0, 36(sp)
 ; RV32I-FPELIM-NEXT:    lui a0, 335544
-; RV32I-FPELIM-NEXT:    addi a0, a0, 1311
-; RV32I-FPELIM-NEXT:    sw a0, 32(sp)
+; RV32I-FPELIM-NEXT:    addi t0, a0, 1311
 ; RV32I-FPELIM-NEXT:    lui a0, 688509
 ; RV32I-FPELIM-NEXT:    addi a5, a0, -2048
 ; RV32I-FPELIM-NEXT:    addi a2, sp, 32
@@ -180,6 +179,7 @@ define void @caller_aligned_stack() nounwind {
 ; RV32I-FPELIM-NEXT:    addi a4, zero, 13
 ; RV32I-FPELIM-NEXT:    addi a6, zero, 4
 ; RV32I-FPELIM-NEXT:    addi a7, zero, 14
+; RV32I-FPELIM-NEXT:    sw t0, 32(sp)
 ; RV32I-FPELIM-NEXT:    call callee_aligned_stack
 ; RV32I-FPELIM-NEXT:    lw ra, 60(sp)
 ; RV32I-FPELIM-NEXT:    addi sp, sp, 64
@@ -215,8 +215,7 @@ define void @caller_aligned_stack() nounwind {
 ; RV32I-WITHFP-NEXT:    addi a0, a0, -328
 ; RV32I-WITHFP-NEXT:    sw a0, -28(s0)
 ; RV32I-WITHFP-NEXT:    lui a0, 335544
-; RV32I-WITHFP-NEXT:    addi a0, a0, 1311
-; RV32I-WITHFP-NEXT:    sw a0, -32(s0)
+; RV32I-WITHFP-NEXT:    addi t0, a0, 1311
 ; RV32I-WITHFP-NEXT:    lui a0, 688509
 ; RV32I-WITHFP-NEXT:    addi a5, a0, -2048
 ; RV32I-WITHFP-NEXT:    addi a2, s0, -32
@@ -226,6 +225,7 @@ define void @caller_aligned_stack() nounwind {
 ; RV32I-WITHFP-NEXT:    addi a4, zero, 13
 ; RV32I-WITHFP-NEXT:    addi a6, zero, 4
 ; RV32I-WITHFP-NEXT:    addi a7, zero, 14
+; RV32I-WITHFP-NEXT:    sw t0, -32(s0)
 ; RV32I-WITHFP-NEXT:    call callee_aligned_stack
 ; RV32I-WITHFP-NEXT:    lw s0, 56(sp)
 ; RV32I-WITHFP-NEXT:    lw ra, 60(sp)
@@ -241,8 +241,8 @@ define void @caller_aligned_stack() nounwind {
 define double @callee_small_scalar_ret() nounwind {
 ; RV32I-FPELIM-LABEL: callee_small_scalar_ret:
 ; RV32I-FPELIM:       # %bb.0:
-; RV32I-FPELIM-NEXT:    mv a0, zero
 ; RV32I-FPELIM-NEXT:    lui a1, 261888
+; RV32I-FPELIM-NEXT:    mv a0, zero
 ; RV32I-FPELIM-NEXT:    ret
 ;
 ; RV32I-WITHFP-LABEL: callee_small_scalar_ret:
@@ -251,8 +251,8 @@ define double @callee_small_scalar_ret() nounwind {
 ; RV32I-WITHFP-NEXT:    sw ra, 12(sp)
 ; RV32I-WITHFP-NEXT:    sw s0, 8(sp)
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 16
-; RV32I-WITHFP-NEXT:    mv a0, zero
 ; RV32I-WITHFP-NEXT:    lui a1, 261888
+; RV32I-WITHFP-NEXT:    mv a0, zero
 ; RV32I-WITHFP-NEXT:    lw s0, 8(sp)
 ; RV32I-WITHFP-NEXT:    lw ra, 12(sp)
 ; RV32I-WITHFP-NEXT:    addi sp, sp, 16
diff --git a/llvm/test/CodeGen/RISCV/calling-conv-ilp32-ilp32f-ilp32d-common.ll b/llvm/test/CodeGen/RISCV/calling-conv-ilp32-ilp32f-ilp32d-common.ll
index 10b0d5462..e86a8c740 100644
--- a/llvm/test/CodeGen/RISCV/calling-conv-ilp32-ilp32f-ilp32d-common.ll
+++ b/llvm/test/CodeGen/RISCV/calling-conv-ilp32-ilp32f-ilp32d-common.ll
@@ -82,22 +82,22 @@ define i32 @caller_i64_in_regs() nounwind {
 define i32 @callee_many_scalars(i8 %a, i16 %b, i32 %c, i64 %d, i32 %e, i32 %f, i64 %g, i32 %h) nounwind {
 ; RV32I-FPELIM-LABEL: callee_many_scalars:
 ; RV32I-FPELIM:       # %bb.0:
-; RV32I-FPELIM-NEXT:    lw t0, 0(sp)
-; RV32I-FPELIM-NEXT:    xor a4, a4, t0
-; RV32I-FPELIM-NEXT:    xor a3, a3, a7
-; RV32I-FPELIM-NEXT:    or a3, a3, a4
-; RV32I-FPELIM-NEXT:    lui a4, 16
-; RV32I-FPELIM-NEXT:    addi a4, a4, -1
-; RV32I-FPELIM-NEXT:    and a1, a1, a4
-; RV32I-FPELIM-NEXT:    andi a0, a0, 255
-; RV32I-FPELIM-NEXT:    add a0, a0, a1
+; RV32I-FPELIM-NEXT:    lw t0, 4(sp)
+; RV32I-FPELIM-NEXT:    lw t1, 0(sp)
+; RV32I-FPELIM-NEXT:    andi t2, a0, 255
+; RV32I-FPELIM-NEXT:    lui a0, 16
+; RV32I-FPELIM-NEXT:    addi a0, a0, -1
+; RV32I-FPELIM-NEXT:    and a0, a1, a0
+; RV32I-FPELIM-NEXT:    add a0, t2, a0
 ; RV32I-FPELIM-NEXT:    add a0, a0, a2
-; RV32I-FPELIM-NEXT:    seqz a1, a3
+; RV32I-FPELIM-NEXT:    xor a1, a4, t1
+; RV32I-FPELIM-NEXT:    xor a2, a3, a7
+; RV32I-FPELIM-NEXT:    or a1, a2, a1
+; RV32I-FPELIM-NEXT:    seqz a1, a1
 ; RV32I-FPELIM-NEXT:    add a0, a1, a0
 ; RV32I-FPELIM-NEXT:    add a0, a0, a5
 ; RV32I-FPELIM-NEXT:    add a0, a0, a6
-; RV32I-FPELIM-NEXT:    lw a1, 4(sp)
-; RV32I-FPELIM-NEXT:    add a0, a0, a1
+; RV32I-FPELIM-NEXT:    add a0, a0, t0
 ; RV32I-FPELIM-NEXT:    ret
 ;
 ; RV32I-WITHFP-LABEL: callee_many_scalars:
@@ -106,22 +106,22 @@ define i32 @callee_many_scalars(i8 %a, i16 %b, i32 %c, i64 %d, i32 %e, i32 %f, i
 ; RV32I-WITHFP-NEXT:    sw ra, 12(sp)
 ; RV32I-WITHFP-NEXT:    sw s0, 8(sp)
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 16
-; RV32I-WITHFP-NEXT:    lw t0, 0(s0)
-; RV32I-WITHFP-NEXT:    xor a4, a4, t0
-; RV32I-WITHFP-NEXT:    xor a3, a3, a7
-; RV32I-WITHFP-NEXT:    or a3, a3, a4
-; RV32I-WITHFP-NEXT:    lui a4, 16
-; RV32I-WITHFP-NEXT:    addi a4, a4, -1
-; RV32I-WITHFP-NEXT:    and a1, a1, a4
-; RV32I-WITHFP-NEXT:    andi a0, a0, 255
-; RV32I-WITHFP-NEXT:    add a0, a0, a1
+; RV32I-WITHFP-NEXT:    lw t0, 4(s0)
+; RV32I-WITHFP-NEXT:    lw t1, 0(s0)
+; RV32I-WITHFP-NEXT:    andi t2, a0, 255
+; RV32I-WITHFP-NEXT:    lui a0, 16
+; RV32I-WITHFP-NEXT:    addi a0, a0, -1
+; RV32I-WITHFP-NEXT:    and a0, a1, a0
+; RV32I-WITHFP-NEXT:    add a0, t2, a0
 ; RV32I-WITHFP-NEXT:    add a0, a0, a2
-; RV32I-WITHFP-NEXT:    seqz a1, a3
+; RV32I-WITHFP-NEXT:    xor a1, a4, t1
+; RV32I-WITHFP-NEXT:    xor a2, a3, a7
+; RV32I-WITHFP-NEXT:    or a1, a2, a1
+; RV32I-WITHFP-NEXT:    seqz a1, a1
 ; RV32I-WITHFP-NEXT:    add a0, a1, a0
 ; RV32I-WITHFP-NEXT:    add a0, a0, a5
 ; RV32I-WITHFP-NEXT:    add a0, a0, a6
-; RV32I-WITHFP-NEXT:    lw a1, 4(s0)
-; RV32I-WITHFP-NEXT:    add a0, a0, a1
+; RV32I-WITHFP-NEXT:    add a0, a0, t0
 ; RV32I-WITHFP-NEXT:    lw s0, 8(sp)
 ; RV32I-WITHFP-NEXT:    lw ra, 12(sp)
 ; RV32I-WITHFP-NEXT:    addi sp, sp, 16
@@ -146,15 +146,15 @@ define i32 @caller_many_scalars() nounwind {
 ; RV32I-FPELIM-NEXT:    sw ra, 12(sp)
 ; RV32I-FPELIM-NEXT:    addi a0, zero, 8
 ; RV32I-FPELIM-NEXT:    sw a0, 4(sp)
-; RV32I-FPELIM-NEXT:    sw zero, 0(sp)
 ; RV32I-FPELIM-NEXT:    addi a0, zero, 1
 ; RV32I-FPELIM-NEXT:    addi a1, zero, 2
 ; RV32I-FPELIM-NEXT:    addi a2, zero, 3
 ; RV32I-FPELIM-NEXT:    addi a3, zero, 4
-; RV32I-FPELIM-NEXT:    mv a4, zero
 ; RV32I-FPELIM-NEXT:    addi a5, zero, 5
 ; RV32I-FPELIM-NEXT:    addi a6, zero, 6
 ; RV32I-FPELIM-NEXT:    addi a7, zero, 7
+; RV32I-FPELIM-NEXT:    sw zero, 0(sp)
+; RV32I-FPELIM-NEXT:    mv a4, zero
 ; RV32I-FPELIM-NEXT:    call callee_many_scalars
 ; RV32I-FPELIM-NEXT:    lw ra, 12(sp)
 ; RV32I-FPELIM-NEXT:    addi sp, sp, 16
@@ -168,15 +168,15 @@ define i32 @caller_many_scalars() nounwind {
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 16
 ; RV32I-WITHFP-NEXT:    addi a0, zero, 8
 ; RV32I-WITHFP-NEXT:    sw a0, 4(sp)
-; RV32I-WITHFP-NEXT:    sw zero, 0(sp)
 ; RV32I-WITHFP-NEXT:    addi a0, zero, 1
 ; RV32I-WITHFP-NEXT:    addi a1, zero, 2
 ; RV32I-WITHFP-NEXT:    addi a2, zero, 3
 ; RV32I-WITHFP-NEXT:    addi a3, zero, 4
-; RV32I-WITHFP-NEXT:    mv a4, zero
 ; RV32I-WITHFP-NEXT:    addi a5, zero, 5
 ; RV32I-WITHFP-NEXT:    addi a6, zero, 6
 ; RV32I-WITHFP-NEXT:    addi a7, zero, 7
+; RV32I-WITHFP-NEXT:    sw zero, 0(sp)
+; RV32I-WITHFP-NEXT:    mv a4, zero
 ; RV32I-WITHFP-NEXT:    call callee_many_scalars
 ; RV32I-WITHFP-NEXT:    lw s0, 8(sp)
 ; RV32I-WITHFP-NEXT:    lw ra, 12(sp)
@@ -192,20 +192,20 @@ define i32 @caller_many_scalars() nounwind {
 define i32 @callee_large_scalars(i128 %a, fp128 %b) nounwind {
 ; RV32I-FPELIM-LABEL: callee_large_scalars:
 ; RV32I-FPELIM:       # %bb.0:
-; RV32I-FPELIM-NEXT:    lw a2, 12(a1)
-; RV32I-FPELIM-NEXT:    lw a3, 12(a0)
-; RV32I-FPELIM-NEXT:    xor a2, a3, a2
-; RV32I-FPELIM-NEXT:    lw a3, 4(a1)
-; RV32I-FPELIM-NEXT:    lw a4, 4(a0)
-; RV32I-FPELIM-NEXT:    xor a3, a4, a3
+; RV32I-FPELIM-NEXT:    lw a6, 0(a1)
+; RV32I-FPELIM-NEXT:    lw a7, 0(a0)
+; RV32I-FPELIM-NEXT:    lw a4, 4(a1)
+; RV32I-FPELIM-NEXT:    lw a5, 12(a1)
+; RV32I-FPELIM-NEXT:    lw a2, 12(a0)
+; RV32I-FPELIM-NEXT:    lw a3, 4(a0)
+; RV32I-FPELIM-NEXT:    lw a1, 8(a1)
+; RV32I-FPELIM-NEXT:    lw a0, 8(a0)
+; RV32I-FPELIM-NEXT:    xor a2, a2, a5
+; RV32I-FPELIM-NEXT:    xor a3, a3, a4
 ; RV32I-FPELIM-NEXT:    or a2, a3, a2
-; RV32I-FPELIM-NEXT:    lw a3, 8(a1)
-; RV32I-FPELIM-NEXT:    lw a4, 8(a0)
-; RV32I-FPELIM-NEXT:    xor a3, a4, a3
-; RV32I-FPELIM-NEXT:    lw a1, 0(a1)
-; RV32I-FPELIM-NEXT:    lw a0, 0(a0)
 ; RV32I-FPELIM-NEXT:    xor a0, a0, a1
-; RV32I-FPELIM-NEXT:    or a0, a0, a3
+; RV32I-FPELIM-NEXT:    xor a1, a7, a6
+; RV32I-FPELIM-NEXT:    or a0, a1, a0
 ; RV32I-FPELIM-NEXT:    or a0, a0, a2
 ; RV32I-FPELIM-NEXT:    seqz a0, a0
 ; RV32I-FPELIM-NEXT:    ret
@@ -216,20 +216,20 @@ define i32 @callee_large_scalars(i128 %a, fp128 %b) nounwind {
 ; RV32I-WITHFP-NEXT:    sw ra, 12(sp)
 ; RV32I-WITHFP-NEXT:    sw s0, 8(sp)
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 16
-; RV32I-WITHFP-NEXT:    lw a2, 12(a1)
-; RV32I-WITHFP-NEXT:    lw a3, 12(a0)
-; RV32I-WITHFP-NEXT:    xor a2, a3, a2
-; RV32I-WITHFP-NEXT:    lw a3, 4(a1)
-; RV32I-WITHFP-NEXT:    lw a4, 4(a0)
-; RV32I-WITHFP-NEXT:    xor a3, a4, a3
+; RV32I-WITHFP-NEXT:    lw a6, 0(a1)
+; RV32I-WITHFP-NEXT:    lw a7, 0(a0)
+; RV32I-WITHFP-NEXT:    lw a4, 4(a1)
+; RV32I-WITHFP-NEXT:    lw a5, 12(a1)
+; RV32I-WITHFP-NEXT:    lw a2, 12(a0)
+; RV32I-WITHFP-NEXT:    lw a3, 4(a0)
+; RV32I-WITHFP-NEXT:    lw a1, 8(a1)
+; RV32I-WITHFP-NEXT:    lw a0, 8(a0)
+; RV32I-WITHFP-NEXT:    xor a2, a2, a5
+; RV32I-WITHFP-NEXT:    xor a3, a3, a4
 ; RV32I-WITHFP-NEXT:    or a2, a3, a2
-; RV32I-WITHFP-NEXT:    lw a3, 8(a1)
-; RV32I-WITHFP-NEXT:    lw a4, 8(a0)
-; RV32I-WITHFP-NEXT:    xor a3, a4, a3
-; RV32I-WITHFP-NEXT:    lw a1, 0(a1)
-; RV32I-WITHFP-NEXT:    lw a0, 0(a0)
 ; RV32I-WITHFP-NEXT:    xor a0, a0, a1
-; RV32I-WITHFP-NEXT:    or a0, a0, a3
+; RV32I-WITHFP-NEXT:    xor a1, a7, a6
+; RV32I-WITHFP-NEXT:    or a0, a1, a0
 ; RV32I-WITHFP-NEXT:    or a0, a0, a2
 ; RV32I-WITHFP-NEXT:    seqz a0, a0
 ; RV32I-WITHFP-NEXT:    lw s0, 8(sp)
@@ -255,10 +255,10 @@ define i32 @caller_large_scalars() nounwind {
 ; RV32I-FPELIM-NEXT:    sw zero, 36(sp)
 ; RV32I-FPELIM-NEXT:    sw zero, 32(sp)
 ; RV32I-FPELIM-NEXT:    sw zero, 28(sp)
-; RV32I-FPELIM-NEXT:    addi a0, zero, 1
-; RV32I-FPELIM-NEXT:    sw a0, 24(sp)
+; RV32I-FPELIM-NEXT:    addi a2, zero, 1
 ; RV32I-FPELIM-NEXT:    addi a0, sp, 24
 ; RV32I-FPELIM-NEXT:    mv a1, sp
+; RV32I-FPELIM-NEXT:    sw a2, 24(sp)
 ; RV32I-FPELIM-NEXT:    call callee_large_scalars
 ; RV32I-FPELIM-NEXT:    lw ra, 44(sp)
 ; RV32I-FPELIM-NEXT:    addi sp, sp, 48
@@ -278,10 +278,10 @@ define i32 @caller_large_scalars() nounwind {
 ; RV32I-WITHFP-NEXT:    sw zero, -12(s0)
 ; RV32I-WITHFP-NEXT:    sw zero, -16(s0)
 ; RV32I-WITHFP-NEXT:    sw zero, -20(s0)
-; RV32I-WITHFP-NEXT:    addi a0, zero, 1
-; RV32I-WITHFP-NEXT:    sw a0, -24(s0)
+; RV32I-WITHFP-NEXT:    addi a2, zero, 1
 ; RV32I-WITHFP-NEXT:    addi a0, s0, -24
 ; RV32I-WITHFP-NEXT:    addi a1, s0, -48
+; RV32I-WITHFP-NEXT:    sw a2, -24(s0)
 ; RV32I-WITHFP-NEXT:    call callee_large_scalars
 ; RV32I-WITHFP-NEXT:    lw s0, 40(sp)
 ; RV32I-WITHFP-NEXT:    lw ra, 44(sp)
@@ -299,20 +299,20 @@ define i32 @callee_large_scalars_exhausted_regs(i32 %a, i32 %b, i32 %c, i32 %d,
 ; RV32I-FPELIM-LABEL: callee_large_scalars_exhausted_regs:
 ; RV32I-FPELIM:       # %bb.0:
 ; RV32I-FPELIM-NEXT:    lw a0, 4(sp)
-; RV32I-FPELIM-NEXT:    lw a1, 12(a0)
-; RV32I-FPELIM-NEXT:    lw a2, 12(a7)
-; RV32I-FPELIM-NEXT:    xor a1, a2, a1
-; RV32I-FPELIM-NEXT:    lw a2, 4(a0)
-; RV32I-FPELIM-NEXT:    lw a3, 4(a7)
-; RV32I-FPELIM-NEXT:    xor a2, a3, a2
-; RV32I-FPELIM-NEXT:    or a1, a2, a1
-; RV32I-FPELIM-NEXT:    lw a2, 8(a0)
-; RV32I-FPELIM-NEXT:    lw a3, 8(a7)
-; RV32I-FPELIM-NEXT:    xor a2, a3, a2
-; RV32I-FPELIM-NEXT:    lw a0, 0(a0)
-; RV32I-FPELIM-NEXT:    lw a3, 0(a7)
-; RV32I-FPELIM-NEXT:    xor a0, a3, a0
-; RV32I-FPELIM-NEXT:    or a0, a0, a2
+; RV32I-FPELIM-NEXT:    lw a6, 0(a0)
+; RV32I-FPELIM-NEXT:    lw t0, 0(a7)
+; RV32I-FPELIM-NEXT:    lw a3, 4(a0)
+; RV32I-FPELIM-NEXT:    lw a4, 12(a0)
+; RV32I-FPELIM-NEXT:    lw a5, 12(a7)
+; RV32I-FPELIM-NEXT:    lw a1, 4(a7)
+; RV32I-FPELIM-NEXT:    lw a0, 8(a0)
+; RV32I-FPELIM-NEXT:    lw a2, 8(a7)
+; RV32I-FPELIM-NEXT:    xor a4, a5, a4
+; RV32I-FPELIM-NEXT:    xor a1, a1, a3
+; RV32I-FPELIM-NEXT:    or a1, a1, a4
+; RV32I-FPELIM-NEXT:    xor a0, a2, a0
+; RV32I-FPELIM-NEXT:    xor a2, t0, a6
+; RV32I-FPELIM-NEXT:    or a0, a2, a0
 ; RV32I-FPELIM-NEXT:    or a0, a0, a1
 ; RV32I-FPELIM-NEXT:    seqz a0, a0
 ; RV32I-FPELIM-NEXT:    ret
@@ -324,20 +324,20 @@ define i32 @callee_large_scalars_exhausted_regs(i32 %a, i32 %b, i32 %c, i32 %d,
 ; RV32I-WITHFP-NEXT:    sw s0, 8(sp)
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 16
 ; RV32I-WITHFP-NEXT:    lw a0, 4(s0)
-; RV32I-WITHFP-NEXT:    lw a1, 12(a0)
-; RV32I-WITHFP-NEXT:    lw a2, 12(a7)
-; RV32I-WITHFP-NEXT:    xor a1, a2, a1
-; RV32I-WITHFP-NEXT:    lw a2, 4(a0)
-; RV32I-WITHFP-NEXT:    lw a3, 4(a7)
-; RV32I-WITHFP-NEXT:    xor a2, a3, a2
-; RV32I-WITHFP-NEXT:    or a1, a2, a1
-; RV32I-WITHFP-NEXT:    lw a2, 8(a0)
-; RV32I-WITHFP-NEXT:    lw a3, 8(a7)
-; RV32I-WITHFP-NEXT:    xor a2, a3, a2
-; RV32I-WITHFP-NEXT:    lw a0, 0(a0)
-; RV32I-WITHFP-NEXT:    lw a3, 0(a7)
-; RV32I-WITHFP-NEXT:    xor a0, a3, a0
-; RV32I-WITHFP-NEXT:    or a0, a0, a2
+; RV32I-WITHFP-NEXT:    lw a6, 0(a0)
+; RV32I-WITHFP-NEXT:    lw t0, 0(a7)
+; RV32I-WITHFP-NEXT:    lw a3, 4(a0)
+; RV32I-WITHFP-NEXT:    lw a4, 12(a0)
+; RV32I-WITHFP-NEXT:    lw a5, 12(a7)
+; RV32I-WITHFP-NEXT:    lw a1, 4(a7)
+; RV32I-WITHFP-NEXT:    lw a0, 8(a0)
+; RV32I-WITHFP-NEXT:    lw a2, 8(a7)
+; RV32I-WITHFP-NEXT:    xor a4, a5, a4
+; RV32I-WITHFP-NEXT:    xor a1, a1, a3
+; RV32I-WITHFP-NEXT:    or a1, a1, a4
+; RV32I-WITHFP-NEXT:    xor a0, a2, a0
+; RV32I-WITHFP-NEXT:    xor a2, t0, a6
+; RV32I-WITHFP-NEXT:    or a0, a2, a0
 ; RV32I-WITHFP-NEXT:    or a0, a0, a1
 ; RV32I-WITHFP-NEXT:    seqz a0, a0
 ; RV32I-WITHFP-NEXT:    lw s0, 8(sp)
@@ -367,8 +367,7 @@ define i32 @caller_large_scalars_exhausted_regs() nounwind {
 ; RV32I-FPELIM-NEXT:    sw zero, 52(sp)
 ; RV32I-FPELIM-NEXT:    sw zero, 48(sp)
 ; RV32I-FPELIM-NEXT:    sw zero, 44(sp)
-; RV32I-FPELIM-NEXT:    addi a0, zero, 8
-; RV32I-FPELIM-NEXT:    sw a0, 40(sp)
+; RV32I-FPELIM-NEXT:    addi t0, zero, 8
 ; RV32I-FPELIM-NEXT:    addi a7, sp, 40
 ; RV32I-FPELIM-NEXT:    addi a0, zero, 1
 ; RV32I-FPELIM-NEXT:    addi a1, zero, 2
@@ -377,6 +376,7 @@ define i32 @caller_large_scalars_exhausted_regs() nounwind {
 ; RV32I-FPELIM-NEXT:    addi a4, zero, 5
 ; RV32I-FPELIM-NEXT:    addi a5, zero, 6
 ; RV32I-FPELIM-NEXT:    addi a6, zero, 7
+; RV32I-FPELIM-NEXT:    sw t0, 40(sp)
 ; RV32I-FPELIM-NEXT:    call callee_large_scalars_exhausted_regs
 ; RV32I-FPELIM-NEXT:    lw ra, 60(sp)
 ; RV32I-FPELIM-NEXT:    addi sp, sp, 64
@@ -400,8 +400,7 @@ define i32 @caller_large_scalars_exhausted_regs() nounwind {
 ; RV32I-WITHFP-NEXT:    sw zero, -12(s0)
 ; RV32I-WITHFP-NEXT:    sw zero, -16(s0)
 ; RV32I-WITHFP-NEXT:    sw zero, -20(s0)
-; RV32I-WITHFP-NEXT:    addi a0, zero, 8
-; RV32I-WITHFP-NEXT:    sw a0, -24(s0)
+; RV32I-WITHFP-NEXT:    addi t0, zero, 8
 ; RV32I-WITHFP-NEXT:    addi a7, s0, -24
 ; RV32I-WITHFP-NEXT:    addi a0, zero, 1
 ; RV32I-WITHFP-NEXT:    addi a1, zero, 2
@@ -410,6 +409,7 @@ define i32 @caller_large_scalars_exhausted_regs() nounwind {
 ; RV32I-WITHFP-NEXT:    addi a4, zero, 5
 ; RV32I-WITHFP-NEXT:    addi a5, zero, 6
 ; RV32I-WITHFP-NEXT:    addi a6, zero, 7
+; RV32I-WITHFP-NEXT:    sw t0, -24(s0)
 ; RV32I-WITHFP-NEXT:    call callee_large_scalars_exhausted_regs
 ; RV32I-WITHFP-NEXT:    lw s0, 56(sp)
 ; RV32I-WITHFP-NEXT:    lw ra, 60(sp)
@@ -524,9 +524,9 @@ define i32 @caller_small_coerced_struct() nounwind {
 define i32 @callee_large_struct(%struct.large* byval align 4 %a) nounwind {
 ; RV32I-FPELIM-LABEL: callee_large_struct:
 ; RV32I-FPELIM:       # %bb.0:
-; RV32I-FPELIM-NEXT:    lw a1, 12(a0)
-; RV32I-FPELIM-NEXT:    lw a0, 0(a0)
-; RV32I-FPELIM-NEXT:    add a0, a0, a1
+; RV32I-FPELIM-NEXT:    lw a1, 0(a0)
+; RV32I-FPELIM-NEXT:    lw a0, 12(a0)
+; RV32I-FPELIM-NEXT:    add a0, a1, a0
 ; RV32I-FPELIM-NEXT:    ret
 ;
 ; RV32I-WITHFP-LABEL: callee_large_struct:
@@ -535,9 +535,9 @@ define i32 @callee_large_struct(%struct.large* byval align 4 %a) nounwind {
 ; RV32I-WITHFP-NEXT:    sw ra, 12(sp)
 ; RV32I-WITHFP-NEXT:    sw s0, 8(sp)
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 16
-; RV32I-WITHFP-NEXT:    lw a1, 12(a0)
-; RV32I-WITHFP-NEXT:    lw a0, 0(a0)
-; RV32I-WITHFP-NEXT:    add a0, a0, a1
+; RV32I-WITHFP-NEXT:    lw a1, 0(a0)
+; RV32I-WITHFP-NEXT:    lw a0, 12(a0)
+; RV32I-WITHFP-NEXT:    add a0, a1, a0
 ; RV32I-WITHFP-NEXT:    lw s0, 8(sp)
 ; RV32I-WITHFP-NEXT:    lw ra, 12(sp)
 ; RV32I-WITHFP-NEXT:    addi sp, sp, 16
@@ -557,16 +557,16 @@ define i32 @caller_large_struct() nounwind {
 ; RV32I-FPELIM-NEXT:    sw ra, 44(sp)
 ; RV32I-FPELIM-NEXT:    addi a0, zero, 1
 ; RV32I-FPELIM-NEXT:    sw a0, 24(sp)
+; RV32I-FPELIM-NEXT:    addi a1, zero, 2
+; RV32I-FPELIM-NEXT:    sw a1, 28(sp)
+; RV32I-FPELIM-NEXT:    addi a2, zero, 3
+; RV32I-FPELIM-NEXT:    sw a2, 32(sp)
+; RV32I-FPELIM-NEXT:    addi a3, zero, 4
+; RV32I-FPELIM-NEXT:    sw a3, 36(sp)
 ; RV32I-FPELIM-NEXT:    sw a0, 8(sp)
-; RV32I-FPELIM-NEXT:    addi a0, zero, 2
-; RV32I-FPELIM-NEXT:    sw a0, 28(sp)
-; RV32I-FPELIM-NEXT:    sw a0, 12(sp)
-; RV32I-FPELIM-NEXT:    addi a0, zero, 3
-; RV32I-FPELIM-NEXT:    sw a0, 32(sp)
-; RV32I-FPELIM-NEXT:    sw a0, 16(sp)
-; RV32I-FPELIM-NEXT:    addi a0, zero, 4
-; RV32I-FPELIM-NEXT:    sw a0, 36(sp)
-; RV32I-FPELIM-NEXT:    sw a0, 20(sp)
+; RV32I-FPELIM-NEXT:    sw a1, 12(sp)
+; RV32I-FPELIM-NEXT:    sw a2, 16(sp)
+; RV32I-FPELIM-NEXT:    sw a3, 20(sp)
 ; RV32I-FPELIM-NEXT:    addi a0, sp, 8
 ; RV32I-FPELIM-NEXT:    call callee_large_struct
 ; RV32I-FPELIM-NEXT:    lw ra, 44(sp)
@@ -581,16 +581,16 @@ define i32 @caller_large_struct() nounwind {
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 48
 ; RV32I-WITHFP-NEXT:    addi a0, zero, 1
 ; RV32I-WITHFP-NEXT:    sw a0, -24(s0)
+; RV32I-WITHFP-NEXT:    addi a1, zero, 2
+; RV32I-WITHFP-NEXT:    sw a1, -20(s0)
+; RV32I-WITHFP-NEXT:    addi a2, zero, 3
+; RV32I-WITHFP-NEXT:    sw a2, -16(s0)
+; RV32I-WITHFP-NEXT:    addi a3, zero, 4
+; RV32I-WITHFP-NEXT:    sw a3, -12(s0)
 ; RV32I-WITHFP-NEXT:    sw a0, -40(s0)
-; RV32I-WITHFP-NEXT:    addi a0, zero, 2
-; RV32I-WITHFP-NEXT:    sw a0, -20(s0)
-; RV32I-WITHFP-NEXT:    sw a0, -36(s0)
-; RV32I-WITHFP-NEXT:    addi a0, zero, 3
-; RV32I-WITHFP-NEXT:    sw a0, -16(s0)
-; RV32I-WITHFP-NEXT:    sw a0, -32(s0)
-; RV32I-WITHFP-NEXT:    addi a0, zero, 4
-; RV32I-WITHFP-NEXT:    sw a0, -12(s0)
-; RV32I-WITHFP-NEXT:    sw a0, -28(s0)
+; RV32I-WITHFP-NEXT:    sw a1, -36(s0)
+; RV32I-WITHFP-NEXT:    sw a2, -32(s0)
+; RV32I-WITHFP-NEXT:    sw a3, -28(s0)
 ; RV32I-WITHFP-NEXT:    addi a0, s0, -40
 ; RV32I-WITHFP-NEXT:    call callee_large_struct
 ; RV32I-WITHFP-NEXT:    lw s0, 40(sp)
@@ -619,14 +619,14 @@ define i32 @callee_aligned_stack(i32 %a, i32 %b, fp128 %c, i32 %d, i32 %e, i64 %
 ; RV32I-FPELIM-LABEL: callee_aligned_stack:
 ; RV32I-FPELIM:       # %bb.0:
 ; RV32I-FPELIM-NEXT:    lw a0, 0(a2)
-; RV32I-FPELIM-NEXT:    add a0, a0, a7
-; RV32I-FPELIM-NEXT:    lw a1, 0(sp)
-; RV32I-FPELIM-NEXT:    add a0, a0, a1
-; RV32I-FPELIM-NEXT:    lw a1, 8(sp)
-; RV32I-FPELIM-NEXT:    add a0, a0, a1
-; RV32I-FPELIM-NEXT:    lw a1, 16(sp)
-; RV32I-FPELIM-NEXT:    add a0, a0, a1
 ; RV32I-FPELIM-NEXT:    lw a1, 20(sp)
+; RV32I-FPELIM-NEXT:    lw a2, 0(sp)
+; RV32I-FPELIM-NEXT:    lw a3, 8(sp)
+; RV32I-FPELIM-NEXT:    lw a4, 16(sp)
+; RV32I-FPELIM-NEXT:    add a0, a0, a7
+; RV32I-FPELIM-NEXT:    add a0, a0, a2
+; RV32I-FPELIM-NEXT:    add a0, a0, a3
+; RV32I-FPELIM-NEXT:    add a0, a0, a4
 ; RV32I-FPELIM-NEXT:    add a0, a0, a1
 ; RV32I-FPELIM-NEXT:    ret
 ;
@@ -637,14 +637,14 @@ define i32 @callee_aligned_stack(i32 %a, i32 %b, fp128 %c, i32 %d, i32 %e, i64 %
 ; RV32I-WITHFP-NEXT:    sw s0, 8(sp)
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 16
 ; RV32I-WITHFP-NEXT:    lw a0, 0(a2)
-; RV32I-WITHFP-NEXT:    add a0, a0, a7
-; RV32I-WITHFP-NEXT:    lw a1, 0(s0)
-; RV32I-WITHFP-NEXT:    add a0, a0, a1
-; RV32I-WITHFP-NEXT:    lw a1, 8(s0)
-; RV32I-WITHFP-NEXT:    add a0, a0, a1
-; RV32I-WITHFP-NEXT:    lw a1, 16(s0)
-; RV32I-WITHFP-NEXT:    add a0, a0, a1
 ; RV32I-WITHFP-NEXT:    lw a1, 20(s0)
+; RV32I-WITHFP-NEXT:    lw a2, 0(s0)
+; RV32I-WITHFP-NEXT:    lw a3, 8(s0)
+; RV32I-WITHFP-NEXT:    lw a4, 16(s0)
+; RV32I-WITHFP-NEXT:    add a0, a0, a7
+; RV32I-WITHFP-NEXT:    add a0, a0, a2
+; RV32I-WITHFP-NEXT:    add a0, a0, a3
+; RV32I-WITHFP-NEXT:    add a0, a0, a4
 ; RV32I-WITHFP-NEXT:    add a0, a0, a1
 ; RV32I-WITHFP-NEXT:    lw s0, 8(sp)
 ; RV32I-WITHFP-NEXT:    lw ra, 12(sp)
@@ -690,8 +690,7 @@ define void @caller_aligned_stack() nounwind {
 ; RV32I-FPELIM-NEXT:    addi a0, a0, -328
 ; RV32I-FPELIM-NEXT:    sw a0, 36(sp)
 ; RV32I-FPELIM-NEXT:    lui a0, 335544
-; RV32I-FPELIM-NEXT:    addi a0, a0, 1311
-; RV32I-FPELIM-NEXT:    sw a0, 32(sp)
+; RV32I-FPELIM-NEXT:    addi t0, a0, 1311
 ; RV32I-FPELIM-NEXT:    lui a0, 688509
 ; RV32I-FPELIM-NEXT:    addi a5, a0, -2048
 ; RV32I-FPELIM-NEXT:    addi a2, sp, 32
@@ -701,6 +700,7 @@ define void @caller_aligned_stack() nounwind {
 ; RV32I-FPELIM-NEXT:    addi a4, zero, 13
 ; RV32I-FPELIM-NEXT:    addi a6, zero, 4
 ; RV32I-FPELIM-NEXT:    addi a7, zero, 14
+; RV32I-FPELIM-NEXT:    sw t0, 32(sp)
 ; RV32I-FPELIM-NEXT:    call callee_aligned_stack
 ; RV32I-FPELIM-NEXT:    lw ra, 60(sp)
 ; RV32I-FPELIM-NEXT:    addi sp, sp, 64
@@ -733,8 +733,7 @@ define void @caller_aligned_stack() nounwind {
 ; RV32I-WITHFP-NEXT:    addi a0, a0, -328
 ; RV32I-WITHFP-NEXT:    sw a0, -28(s0)
 ; RV32I-WITHFP-NEXT:    lui a0, 335544
-; RV32I-WITHFP-NEXT:    addi a0, a0, 1311
-; RV32I-WITHFP-NEXT:    sw a0, -32(s0)
+; RV32I-WITHFP-NEXT:    addi t0, a0, 1311
 ; RV32I-WITHFP-NEXT:    lui a0, 688509
 ; RV32I-WITHFP-NEXT:    addi a5, a0, -2048
 ; RV32I-WITHFP-NEXT:    addi a2, s0, -32
@@ -744,6 +743,7 @@ define void @caller_aligned_stack() nounwind {
 ; RV32I-WITHFP-NEXT:    addi a4, zero, 13
 ; RV32I-WITHFP-NEXT:    addi a6, zero, 4
 ; RV32I-WITHFP-NEXT:    addi a7, zero, 14
+; RV32I-WITHFP-NEXT:    sw t0, -32(s0)
 ; RV32I-WITHFP-NEXT:    call callee_aligned_stack
 ; RV32I-WITHFP-NEXT:    lw s0, 56(sp)
 ; RV32I-WITHFP-NEXT:    lw ra, 60(sp)
@@ -787,17 +787,15 @@ define i32 @caller_small_scalar_ret() nounwind {
 ; RV32I-FPELIM:       # %bb.0:
 ; RV32I-FPELIM-NEXT:    addi sp, sp, -16
 ; RV32I-FPELIM-NEXT:    sw ra, 12(sp)
-; RV32I-FPELIM-NEXT:    sw s0, 8(sp)
-; RV32I-FPELIM-NEXT:    lui a0, 56
-; RV32I-FPELIM-NEXT:    addi s0, a0, 580
 ; RV32I-FPELIM-NEXT:    call callee_small_scalar_ret
-; RV32I-FPELIM-NEXT:    xor a1, a1, s0
+; RV32I-FPELIM-NEXT:    lui a2, 56
+; RV32I-FPELIM-NEXT:    addi a2, a2, 580
+; RV32I-FPELIM-NEXT:    xor a1, a1, a2
 ; RV32I-FPELIM-NEXT:    lui a2, 200614
 ; RV32I-FPELIM-NEXT:    addi a2, a2, 647
 ; RV32I-FPELIM-NEXT:    xor a0, a0, a2
 ; RV32I-FPELIM-NEXT:    or a0, a0, a1
 ; RV32I-FPELIM-NEXT:    seqz a0, a0
-; RV32I-FPELIM-NEXT:    lw s0, 8(sp)
 ; RV32I-FPELIM-NEXT:    lw ra, 12(sp)
 ; RV32I-FPELIM-NEXT:    addi sp, sp, 16
 ; RV32I-FPELIM-NEXT:    ret
@@ -807,18 +805,16 @@ define i32 @caller_small_scalar_ret() nounwind {
 ; RV32I-WITHFP-NEXT:    addi sp, sp, -16
 ; RV32I-WITHFP-NEXT:    sw ra, 12(sp)
 ; RV32I-WITHFP-NEXT:    sw s0, 8(sp)
-; RV32I-WITHFP-NEXT:    sw s1, 4(sp)
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 16
-; RV32I-WITHFP-NEXT:    lui a0, 56
-; RV32I-WITHFP-NEXT:    addi s1, a0, 580
 ; RV32I-WITHFP-NEXT:    call callee_small_scalar_ret
-; RV32I-WITHFP-NEXT:    xor a1, a1, s1
+; RV32I-WITHFP-NEXT:    lui a2, 56
+; RV32I-WITHFP-NEXT:    addi a2, a2, 580
+; RV32I-WITHFP-NEXT:    xor a1, a1, a2
 ; RV32I-WITHFP-NEXT:    lui a2, 200614
 ; RV32I-WITHFP-NEXT:    addi a2, a2, 647
 ; RV32I-WITHFP-NEXT:    xor a0, a0, a2
 ; RV32I-WITHFP-NEXT:    or a0, a0, a1
 ; RV32I-WITHFP-NEXT:    seqz a0, a0
-; RV32I-WITHFP-NEXT:    lw s1, 4(sp)
 ; RV32I-WITHFP-NEXT:    lw s0, 8(sp)
 ; RV32I-WITHFP-NEXT:    lw ra, 12(sp)
 ; RV32I-WITHFP-NEXT:    addi sp, sp, 16
@@ -946,14 +942,14 @@ define void @caller_large_scalar_ret() nounwind {
 define void @callee_large_struct_ret(%struct.large* noalias sret %agg.result) nounwind {
 ; RV32I-FPELIM-LABEL: callee_large_struct_ret:
 ; RV32I-FPELIM:       # %bb.0:
-; RV32I-FPELIM-NEXT:    addi a1, zero, 4
-; RV32I-FPELIM-NEXT:    sw a1, 12(a0)
-; RV32I-FPELIM-NEXT:    addi a1, zero, 3
-; RV32I-FPELIM-NEXT:    sw a1, 8(a0)
-; RV32I-FPELIM-NEXT:    addi a1, zero, 2
-; RV32I-FPELIM-NEXT:    sw a1, 4(a0)
 ; RV32I-FPELIM-NEXT:    addi a1, zero, 1
 ; RV32I-FPELIM-NEXT:    sw a1, 0(a0)
+; RV32I-FPELIM-NEXT:    addi a1, zero, 2
+; RV32I-FPELIM-NEXT:    sw a1, 4(a0)
+; RV32I-FPELIM-NEXT:    addi a1, zero, 3
+; RV32I-FPELIM-NEXT:    sw a1, 8(a0)
+; RV32I-FPELIM-NEXT:    addi a1, zero, 4
+; RV32I-FPELIM-NEXT:    sw a1, 12(a0)
 ; RV32I-FPELIM-NEXT:    ret
 ;
 ; RV32I-WITHFP-LABEL: callee_large_struct_ret:
@@ -962,14 +958,14 @@ define void @callee_large_struct_ret(%struct.large* noalias sret %agg.result) no
 ; RV32I-WITHFP-NEXT:    sw ra, 12(sp)
 ; RV32I-WITHFP-NEXT:    sw s0, 8(sp)
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 16
-; RV32I-WITHFP-NEXT:    addi a1, zero, 4
-; RV32I-WITHFP-NEXT:    sw a1, 12(a0)
-; RV32I-WITHFP-NEXT:    addi a1, zero, 3
-; RV32I-WITHFP-NEXT:    sw a1, 8(a0)
-; RV32I-WITHFP-NEXT:    addi a1, zero, 2
-; RV32I-WITHFP-NEXT:    sw a1, 4(a0)
 ; RV32I-WITHFP-NEXT:    addi a1, zero, 1
 ; RV32I-WITHFP-NEXT:    sw a1, 0(a0)
+; RV32I-WITHFP-NEXT:    addi a1, zero, 2
+; RV32I-WITHFP-NEXT:    sw a1, 4(a0)
+; RV32I-WITHFP-NEXT:    addi a1, zero, 3
+; RV32I-WITHFP-NEXT:    sw a1, 8(a0)
+; RV32I-WITHFP-NEXT:    addi a1, zero, 4
+; RV32I-WITHFP-NEXT:    sw a1, 12(a0)
 ; RV32I-WITHFP-NEXT:    lw s0, 8(sp)
 ; RV32I-WITHFP-NEXT:    lw ra, 12(sp)
 ; RV32I-WITHFP-NEXT:    addi sp, sp, 16
@@ -992,9 +988,9 @@ define i32 @caller_large_struct_ret() nounwind {
 ; RV32I-FPELIM-NEXT:    sw ra, 28(sp)
 ; RV32I-FPELIM-NEXT:    addi a0, sp, 8
 ; RV32I-FPELIM-NEXT:    call callee_large_struct_ret
-; RV32I-FPELIM-NEXT:    lw a0, 20(sp)
-; RV32I-FPELIM-NEXT:    lw a1, 8(sp)
-; RV32I-FPELIM-NEXT:    add a0, a1, a0
+; RV32I-FPELIM-NEXT:    lw a0, 8(sp)
+; RV32I-FPELIM-NEXT:    lw a1, 20(sp)
+; RV32I-FPELIM-NEXT:    add a0, a0, a1
 ; RV32I-FPELIM-NEXT:    lw ra, 28(sp)
 ; RV32I-FPELIM-NEXT:    addi sp, sp, 32
 ; RV32I-FPELIM-NEXT:    ret
@@ -1007,9 +1003,9 @@ define i32 @caller_large_struct_ret() nounwind {
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 32
 ; RV32I-WITHFP-NEXT:    addi a0, s0, -24
 ; RV32I-WITHFP-NEXT:    call callee_large_struct_ret
-; RV32I-WITHFP-NEXT:    lw a0, -12(s0)
-; RV32I-WITHFP-NEXT:    lw a1, -24(s0)
-; RV32I-WITHFP-NEXT:    add a0, a1, a0
+; RV32I-WITHFP-NEXT:    lw a0, -24(s0)
+; RV32I-WITHFP-NEXT:    lw a1, -12(s0)
+; RV32I-WITHFP-NEXT:    add a0, a0, a1
 ; RV32I-WITHFP-NEXT:    lw s0, 24(sp)
 ; RV32I-WITHFP-NEXT:    lw ra, 28(sp)
 ; RV32I-WITHFP-NEXT:    addi sp, sp, 32
diff --git a/llvm/test/CodeGen/RISCV/calling-conv-ilp32.ll b/llvm/test/CodeGen/RISCV/calling-conv-ilp32.ll
index ddedab591..8ff3de799 100644
--- a/llvm/test/CodeGen/RISCV/calling-conv-ilp32.ll
+++ b/llvm/test/CodeGen/RISCV/calling-conv-ilp32.ll
@@ -107,15 +107,15 @@ define i32 @caller_float_on_stack() nounwind {
 ; RV32I-FPELIM:       # %bb.0:
 ; RV32I-FPELIM-NEXT:    addi sp, sp, -16
 ; RV32I-FPELIM-NEXT:    sw ra, 12(sp)
-; RV32I-FPELIM-NEXT:    lui a0, 264704
-; RV32I-FPELIM-NEXT:    sw a0, 0(sp)
+; RV32I-FPELIM-NEXT:    lui a1, 264704
 ; RV32I-FPELIM-NEXT:    addi a0, zero, 1
-; RV32I-FPELIM-NEXT:    mv a1, zero
 ; RV32I-FPELIM-NEXT:    addi a2, zero, 2
-; RV32I-FPELIM-NEXT:    mv a3, zero
 ; RV32I-FPELIM-NEXT:    addi a4, zero, 3
-; RV32I-FPELIM-NEXT:    mv a5, zero
 ; RV32I-FPELIM-NEXT:    addi a6, zero, 4
+; RV32I-FPELIM-NEXT:    sw a1, 0(sp)
+; RV32I-FPELIM-NEXT:    mv a1, zero
+; RV32I-FPELIM-NEXT:    mv a3, zero
+; RV32I-FPELIM-NEXT:    mv a5, zero
 ; RV32I-FPELIM-NEXT:    mv a7, zero
 ; RV32I-FPELIM-NEXT:    call callee_float_on_stack
 ; RV32I-FPELIM-NEXT:    lw ra, 12(sp)
@@ -128,15 +128,15 @@ define i32 @caller_float_on_stack() nounwind {
 ; RV32I-WITHFP-NEXT:    sw ra, 12(sp)
 ; RV32I-WITHFP-NEXT:    sw s0, 8(sp)
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 16
-; RV32I-WITHFP-NEXT:    lui a0, 264704
-; RV32I-WITHFP-NEXT:    sw a0, 0(sp)
+; RV32I-WITHFP-NEXT:    lui a1, 264704
 ; RV32I-WITHFP-NEXT:    addi a0, zero, 1
-; RV32I-WITHFP-NEXT:    mv a1, zero
 ; RV32I-WITHFP-NEXT:    addi a2, zero, 2
-; RV32I-WITHFP-NEXT:    mv a3, zero
 ; RV32I-WITHFP-NEXT:    addi a4, zero, 3
-; RV32I-WITHFP-NEXT:    mv a5, zero
 ; RV32I-WITHFP-NEXT:    addi a6, zero, 4
+; RV32I-WITHFP-NEXT:    sw a1, 0(sp)
+; RV32I-WITHFP-NEXT:    mv a1, zero
+; RV32I-WITHFP-NEXT:    mv a3, zero
+; RV32I-WITHFP-NEXT:    mv a5, zero
 ; RV32I-WITHFP-NEXT:    mv a7, zero
 ; RV32I-WITHFP-NEXT:    call callee_float_on_stack
 ; RV32I-WITHFP-NEXT:    lw s0, 8(sp)
diff --git a/llvm/test/CodeGen/RISCV/calling-conv-ilp32d.ll b/llvm/test/CodeGen/RISCV/calling-conv-ilp32d.ll
index 37dfa7bf9..5ad002f5c 100644
--- a/llvm/test/CodeGen/RISCV/calling-conv-ilp32d.ll
+++ b/llvm/test/CodeGen/RISCV/calling-conv-ilp32d.ll
@@ -38,9 +38,9 @@ define i32 @caller_double_in_fpr() nounwind {
 define i32 @callee_double_in_fpr_exhausted_gprs(i64 %a, i64 %b, i64 %c, i64 %d, i32 %e, double %f) nounwind {
 ; RV32-ILP32D-LABEL: callee_double_in_fpr_exhausted_gprs:
 ; RV32-ILP32D:       # %bb.0:
-; RV32-ILP32D-NEXT:    fcvt.w.d a0, fa0, rtz
-; RV32-ILP32D-NEXT:    lw a1, 0(sp)
-; RV32-ILP32D-NEXT:    add a0, a1, a0
+; RV32-ILP32D-NEXT:    lw a0, 0(sp)
+; RV32-ILP32D-NEXT:    fcvt.w.d a1, fa0, rtz
+; RV32-ILP32D-NEXT:    add a0, a0, a1
 ; RV32-ILP32D-NEXT:    ret
   %f_fptosi = fptosi double %f to i32
   %1 = add i32 %e, %f_fptosi
@@ -52,18 +52,18 @@ define i32 @caller_double_in_fpr_exhausted_gprs() nounwind {
 ; RV32-ILP32D:       # %bb.0:
 ; RV32-ILP32D-NEXT:    addi sp, sp, -16
 ; RV32-ILP32D-NEXT:    sw ra, 12(sp)
-; RV32-ILP32D-NEXT:    addi a0, zero, 5
-; RV32-ILP32D-NEXT:    sw a0, 0(sp)
+; RV32-ILP32D-NEXT:    addi a1, zero, 5
 ; RV32-ILP32D-NEXT:    lui a0, %hi(.LCPI3_0)
 ; RV32-ILP32D-NEXT:    addi a0, a0, %lo(.LCPI3_0)
 ; RV32-ILP32D-NEXT:    fld fa0, 0(a0)
 ; RV32-ILP32D-NEXT:    addi a0, zero, 1
-; RV32-ILP32D-NEXT:    mv a1, zero
 ; RV32-ILP32D-NEXT:    addi a2, zero, 2
-; RV32-ILP32D-NEXT:    mv a3, zero
 ; RV32-ILP32D-NEXT:    addi a4, zero, 3
-; RV32-ILP32D-NEXT:    mv a5, zero
 ; RV32-ILP32D-NEXT:    addi a6, zero, 4
+; RV32-ILP32D-NEXT:    sw a1, 0(sp)
+; RV32-ILP32D-NEXT:    mv a1, zero
+; RV32-ILP32D-NEXT:    mv a3, zero
+; RV32-ILP32D-NEXT:    mv a5, zero
 ; RV32-ILP32D-NEXT:    mv a7, zero
 ; RV32-ILP32D-NEXT:    call callee_double_in_fpr_exhausted_gprs
 ; RV32-ILP32D-NEXT:    lw ra, 12(sp)
@@ -82,9 +82,9 @@ define i32 @callee_double_in_gpr_exhausted_fprs(double %a, double %b, double %c,
 ; RV32-ILP32D-NEXT:    sw a0, 8(sp)
 ; RV32-ILP32D-NEXT:    sw a1, 12(sp)
 ; RV32-ILP32D-NEXT:    fld ft0, 8(sp)
-; RV32-ILP32D-NEXT:    fcvt.w.d a0, ft0, rtz
-; RV32-ILP32D-NEXT:    fcvt.w.d a1, fa7, rtz
-; RV32-ILP32D-NEXT:    add a0, a1, a0
+; RV32-ILP32D-NEXT:    fcvt.w.d a0, fa7, rtz
+; RV32-ILP32D-NEXT:    fcvt.w.d a1, ft0, rtz
+; RV32-ILP32D-NEXT:    add a0, a0, a1
 ; RV32-ILP32D-NEXT:    addi sp, sp, 16
 ; RV32-ILP32D-NEXT:    ret
   %h_fptosi = fptosi double %h to i32
@@ -100,21 +100,21 @@ define i32 @caller_double_in_gpr_exhausted_fprs() nounwind {
 ; RV32-ILP32D-NEXT:    sw ra, 12(sp)
 ; RV32-ILP32D-NEXT:    lui a0, %hi(.LCPI5_0)
 ; RV32-ILP32D-NEXT:    addi a0, a0, %lo(.LCPI5_0)
-; RV32-ILP32D-NEXT:    lui a1, %hi(.LCPI5_1)
-; RV32-ILP32D-NEXT:    addi a1, a1, %lo(.LCPI5_1)
-; RV32-ILP32D-NEXT:    lui a2, %hi(.LCPI5_2)
-; RV32-ILP32D-NEXT:    addi a2, a2, %lo(.LCPI5_2)
-; RV32-ILP32D-NEXT:    lui a3, %hi(.LCPI5_3)
-; RV32-ILP32D-NEXT:    addi a3, a3, %lo(.LCPI5_3)
-; RV32-ILP32D-NEXT:    lui a4, %hi(.LCPI5_4)
-; RV32-ILP32D-NEXT:    addi a4, a4, %lo(.LCPI5_4)
-; RV32-ILP32D-NEXT:    lui a5, %hi(.LCPI5_5)
-; RV32-ILP32D-NEXT:    addi a5, a5, %lo(.LCPI5_5)
-; RV32-ILP32D-NEXT:    fld fa0, 0(a5)
-; RV32-ILP32D-NEXT:    fld fa1, 0(a4)
-; RV32-ILP32D-NEXT:    fld fa2, 0(a3)
-; RV32-ILP32D-NEXT:    fld fa3, 0(a2)
-; RV32-ILP32D-NEXT:    fld fa4, 0(a1)
+; RV32-ILP32D-NEXT:    fld fa0, 0(a0)
+; RV32-ILP32D-NEXT:    lui a0, %hi(.LCPI5_1)
+; RV32-ILP32D-NEXT:    addi a0, a0, %lo(.LCPI5_1)
+; RV32-ILP32D-NEXT:    fld fa1, 0(a0)
+; RV32-ILP32D-NEXT:    lui a0, %hi(.LCPI5_2)
+; RV32-ILP32D-NEXT:    addi a0, a0, %lo(.LCPI5_2)
+; RV32-ILP32D-NEXT:    fld fa2, 0(a0)
+; RV32-ILP32D-NEXT:    lui a0, %hi(.LCPI5_3)
+; RV32-ILP32D-NEXT:    addi a0, a0, %lo(.LCPI5_3)
+; RV32-ILP32D-NEXT:    fld fa3, 0(a0)
+; RV32-ILP32D-NEXT:    lui a0, %hi(.LCPI5_4)
+; RV32-ILP32D-NEXT:    addi a0, a0, %lo(.LCPI5_4)
+; RV32-ILP32D-NEXT:    fld fa4, 0(a0)
+; RV32-ILP32D-NEXT:    lui a0, %hi(.LCPI5_5)
+; RV32-ILP32D-NEXT:    addi a0, a0, %lo(.LCPI5_5)
 ; RV32-ILP32D-NEXT:    fld fa5, 0(a0)
 ; RV32-ILP32D-NEXT:    lui a0, %hi(.LCPI5_6)
 ; RV32-ILP32D-NEXT:    addi a0, a0, %lo(.LCPI5_6)
@@ -122,8 +122,8 @@ define i32 @caller_double_in_gpr_exhausted_fprs() nounwind {
 ; RV32-ILP32D-NEXT:    lui a0, %hi(.LCPI5_7)
 ; RV32-ILP32D-NEXT:    addi a0, a0, %lo(.LCPI5_7)
 ; RV32-ILP32D-NEXT:    fld fa7, 0(a0)
-; RV32-ILP32D-NEXT:    mv a0, zero
 ; RV32-ILP32D-NEXT:    lui a1, 262688
+; RV32-ILP32D-NEXT:    mv a0, zero
 ; RV32-ILP32D-NEXT:    call callee_double_in_gpr_exhausted_fprs
 ; RV32-ILP32D-NEXT:    lw ra, 12(sp)
 ; RV32-ILP32D-NEXT:    addi sp, sp, 16
@@ -157,39 +157,39 @@ define i32 @caller_double_in_gpr_and_stack_almost_exhausted_gprs_fprs() nounwind
 ; RV32-ILP32D:       # %bb.0:
 ; RV32-ILP32D-NEXT:    addi sp, sp, -16
 ; RV32-ILP32D-NEXT:    sw ra, 12(sp)
-; RV32-ILP32D-NEXT:    lui a0, 262816
-; RV32-ILP32D-NEXT:    sw a0, 0(sp)
+; RV32-ILP32D-NEXT:    lui a1, 262816
 ; RV32-ILP32D-NEXT:    lui a0, %hi(.LCPI7_0)
-; RV32-ILP32D-NEXT:    addi a6, a0, %lo(.LCPI7_0)
-; RV32-ILP32D-NEXT:    lui a1, %hi(.LCPI7_1)
-; RV32-ILP32D-NEXT:    addi a1, a1, %lo(.LCPI7_1)
-; RV32-ILP32D-NEXT:    lui a2, %hi(.LCPI7_2)
-; RV32-ILP32D-NEXT:    addi a2, a2, %lo(.LCPI7_2)
-; RV32-ILP32D-NEXT:    lui a3, %hi(.LCPI7_3)
-; RV32-ILP32D-NEXT:    addi a3, a3, %lo(.LCPI7_3)
-; RV32-ILP32D-NEXT:    lui a4, %hi(.LCPI7_4)
-; RV32-ILP32D-NEXT:    addi a4, a4, %lo(.LCPI7_4)
-; RV32-ILP32D-NEXT:    lui a5, %hi(.LCPI7_5)
-; RV32-ILP32D-NEXT:    addi a5, a5, %lo(.LCPI7_5)
+; RV32-ILP32D-NEXT:    addi a0, a0, %lo(.LCPI7_0)
+; RV32-ILP32D-NEXT:    fld fa0, 0(a0)
+; RV32-ILP32D-NEXT:    lui a0, %hi(.LCPI7_1)
+; RV32-ILP32D-NEXT:    addi a0, a0, %lo(.LCPI7_1)
+; RV32-ILP32D-NEXT:    fld fa1, 0(a0)
+; RV32-ILP32D-NEXT:    lui a0, %hi(.LCPI7_2)
+; RV32-ILP32D-NEXT:    addi a0, a0, %lo(.LCPI7_2)
+; RV32-ILP32D-NEXT:    fld fa2, 0(a0)
+; RV32-ILP32D-NEXT:    lui a0, %hi(.LCPI7_3)
+; RV32-ILP32D-NEXT:    addi a0, a0, %lo(.LCPI7_3)
+; RV32-ILP32D-NEXT:    fld fa3, 0(a0)
+; RV32-ILP32D-NEXT:    lui a0, %hi(.LCPI7_4)
+; RV32-ILP32D-NEXT:    addi a0, a0, %lo(.LCPI7_4)
+; RV32-ILP32D-NEXT:    fld fa4, 0(a0)
+; RV32-ILP32D-NEXT:    lui a0, %hi(.LCPI7_5)
+; RV32-ILP32D-NEXT:    addi a0, a0, %lo(.LCPI7_5)
+; RV32-ILP32D-NEXT:    fld fa5, 0(a0)
 ; RV32-ILP32D-NEXT:    lui a0, %hi(.LCPI7_6)
 ; RV32-ILP32D-NEXT:    addi a0, a0, %lo(.LCPI7_6)
-; RV32-ILP32D-NEXT:    fld fa0, 0(a0)
-; RV32-ILP32D-NEXT:    fld fa1, 0(a5)
-; RV32-ILP32D-NEXT:    fld fa2, 0(a4)
-; RV32-ILP32D-NEXT:    fld fa3, 0(a3)
-; RV32-ILP32D-NEXT:    fld fa4, 0(a2)
-; RV32-ILP32D-NEXT:    fld fa5, 0(a1)
-; RV32-ILP32D-NEXT:    fld fa6, 0(a6)
+; RV32-ILP32D-NEXT:    fld fa6, 0(a0)
 ; RV32-ILP32D-NEXT:    lui a0, %hi(.LCPI7_7)
 ; RV32-ILP32D-NEXT:    addi a0, a0, %lo(.LCPI7_7)
 ; RV32-ILP32D-NEXT:    fld fa7, 0(a0)
 ; RV32-ILP32D-NEXT:    addi a0, zero, 1
-; RV32-ILP32D-NEXT:    mv a1, zero
 ; RV32-ILP32D-NEXT:    addi a2, zero, 3
-; RV32-ILP32D-NEXT:    mv a3, zero
 ; RV32-ILP32D-NEXT:    addi a4, zero, 5
-; RV32-ILP32D-NEXT:    mv a5, zero
 ; RV32-ILP32D-NEXT:    addi a6, zero, 7
+; RV32-ILP32D-NEXT:    sw a1, 0(sp)
+; RV32-ILP32D-NEXT:    mv a1, zero
+; RV32-ILP32D-NEXT:    mv a3, zero
+; RV32-ILP32D-NEXT:    mv a5, zero
 ; RV32-ILP32D-NEXT:    mv a7, zero
 ; RV32-ILP32D-NEXT:    call callee_double_in_gpr_and_stack_almost_exhausted_gprs_fprs
 ; RV32-ILP32D-NEXT:    lw ra, 12(sp)
@@ -223,38 +223,38 @@ define i32 @caller_double_on_stack_exhausted_gprs_fprs() nounwind {
 ; RV32-ILP32D-NEXT:    sw ra, 12(sp)
 ; RV32-ILP32D-NEXT:    lui a0, 262816
 ; RV32-ILP32D-NEXT:    sw a0, 4(sp)
-; RV32-ILP32D-NEXT:    sw zero, 0(sp)
 ; RV32-ILP32D-NEXT:    lui a0, %hi(.LCPI9_0)
-; RV32-ILP32D-NEXT:    addi a6, a0, %lo(.LCPI9_0)
-; RV32-ILP32D-NEXT:    lui a1, %hi(.LCPI9_1)
-; RV32-ILP32D-NEXT:    addi a1, a1, %lo(.LCPI9_1)
-; RV32-ILP32D-NEXT:    lui a2, %hi(.LCPI9_2)
-; RV32-ILP32D-NEXT:    addi a2, a2, %lo(.LCPI9_2)
-; RV32-ILP32D-NEXT:    lui a3, %hi(.LCPI9_3)
-; RV32-ILP32D-NEXT:    addi a3, a3, %lo(.LCPI9_3)
-; RV32-ILP32D-NEXT:    lui a4, %hi(.LCPI9_4)
-; RV32-ILP32D-NEXT:    addi a4, a4, %lo(.LCPI9_4)
-; RV32-ILP32D-NEXT:    lui a5, %hi(.LCPI9_5)
-; RV32-ILP32D-NEXT:    addi a5, a5, %lo(.LCPI9_5)
+; RV32-ILP32D-NEXT:    addi a0, a0, %lo(.LCPI9_0)
+; RV32-ILP32D-NEXT:    fld fa0, 0(a0)
+; RV32-ILP32D-NEXT:    lui a0, %hi(.LCPI9_1)
+; RV32-ILP32D-NEXT:    addi a0, a0, %lo(.LCPI9_1)
+; RV32-ILP32D-NEXT:    fld fa1, 0(a0)
+; RV32-ILP32D-NEXT:    lui a0, %hi(.LCPI9_2)
+; RV32-ILP32D-NEXT:    addi a0, a0, %lo(.LCPI9_2)
+; RV32-ILP32D-NEXT:    fld fa2, 0(a0)
+; RV32-ILP32D-NEXT:    lui a0, %hi(.LCPI9_3)
+; RV32-ILP32D-NEXT:    addi a0, a0, %lo(.LCPI9_3)
+; RV32-ILP32D-NEXT:    fld fa3, 0(a0)
+; RV32-ILP32D-NEXT:    lui a0, %hi(.LCPI9_4)
+; RV32-ILP32D-NEXT:    addi a0, a0, %lo(.LCPI9_4)
+; RV32-ILP32D-NEXT:    fld fa4, 0(a0)
+; RV32-ILP32D-NEXT:    lui a0, %hi(.LCPI9_5)
+; RV32-ILP32D-NEXT:    addi a0, a0, %lo(.LCPI9_5)
+; RV32-ILP32D-NEXT:    fld fa5, 0(a0)
 ; RV32-ILP32D-NEXT:    lui a0, %hi(.LCPI9_6)
 ; RV32-ILP32D-NEXT:    addi a0, a0, %lo(.LCPI9_6)
-; RV32-ILP32D-NEXT:    fld fa0, 0(a0)
-; RV32-ILP32D-NEXT:    fld fa1, 0(a5)
-; RV32-ILP32D-NEXT:    fld fa2, 0(a4)
-; RV32-ILP32D-NEXT:    fld fa3, 0(a3)
-; RV32-ILP32D-NEXT:    fld fa4, 0(a2)
-; RV32-ILP32D-NEXT:    fld fa5, 0(a1)
-; RV32-ILP32D-NEXT:    fld fa6, 0(a6)
+; RV32-ILP32D-NEXT:    fld fa6, 0(a0)
 ; RV32-ILP32D-NEXT:    lui a0, %hi(.LCPI9_7)
 ; RV32-ILP32D-NEXT:    addi a0, a0, %lo(.LCPI9_7)
 ; RV32-ILP32D-NEXT:    fld fa7, 0(a0)
 ; RV32-ILP32D-NEXT:    addi a0, zero, 1
-; RV32-ILP32D-NEXT:    mv a1, zero
 ; RV32-ILP32D-NEXT:    addi a2, zero, 3
-; RV32-ILP32D-NEXT:    mv a3, zero
 ; RV32-ILP32D-NEXT:    addi a4, zero, 5
-; RV32-ILP32D-NEXT:    mv a5, zero
 ; RV32-ILP32D-NEXT:    addi a6, zero, 7
+; RV32-ILP32D-NEXT:    sw zero, 0(sp)
+; RV32-ILP32D-NEXT:    mv a1, zero
+; RV32-ILP32D-NEXT:    mv a3, zero
+; RV32-ILP32D-NEXT:    mv a5, zero
 ; RV32-ILP32D-NEXT:    mv a7, zero
 ; RV32-ILP32D-NEXT:    call callee_double_on_stack_exhausted_gprs_fprs
 ; RV32-ILP32D-NEXT:    lw ra, 12(sp)
diff --git a/llvm/test/CodeGen/RISCV/calling-conv-ilp32f-ilp32d-common.ll b/llvm/test/CodeGen/RISCV/calling-conv-ilp32f-ilp32d-common.ll
index 5f3042dfe..70b25059f 100644
--- a/llvm/test/CodeGen/RISCV/calling-conv-ilp32f-ilp32d-common.ll
+++ b/llvm/test/CodeGen/RISCV/calling-conv-ilp32f-ilp32d-common.ll
@@ -41,9 +41,9 @@ define i32 @caller_float_in_fpr() nounwind {
 define i32 @callee_float_in_fpr_exhausted_gprs(i64 %a, i64 %b, i64 %c, i64 %d, i32 %e, float %f) nounwind {
 ; RV32-ILP32FD-LABEL: callee_float_in_fpr_exhausted_gprs:
 ; RV32-ILP32FD:       # %bb.0:
-; RV32-ILP32FD-NEXT:    fcvt.w.s a0, fa0, rtz
-; RV32-ILP32FD-NEXT:    lw a1, 0(sp)
-; RV32-ILP32FD-NEXT:    add a0, a1, a0
+; RV32-ILP32FD-NEXT:    lw a0, 0(sp)
+; RV32-ILP32FD-NEXT:    fcvt.w.s a1, fa0, rtz
+; RV32-ILP32FD-NEXT:    add a0, a0, a1
 ; RV32-ILP32FD-NEXT:    ret
   %f_fptosi = fptosi float %f to i32
   %1 = add i32 %e, %f_fptosi
@@ -55,18 +55,18 @@ define i32 @caller_float_in_fpr_exhausted_gprs() nounwind {
 ; RV32-ILP32FD:       # %bb.0:
 ; RV32-ILP32FD-NEXT:    addi sp, sp, -16
 ; RV32-ILP32FD-NEXT:    sw ra, 12(sp)
-; RV32-ILP32FD-NEXT:    addi a0, zero, 5
-; RV32-ILP32FD-NEXT:    sw a0, 0(sp)
+; RV32-ILP32FD-NEXT:    addi a1, zero, 5
 ; RV32-ILP32FD-NEXT:    lui a0, %hi(.LCPI3_0)
 ; RV32-ILP32FD-NEXT:    addi a0, a0, %lo(.LCPI3_0)
 ; RV32-ILP32FD-NEXT:    flw fa0, 0(a0)
 ; RV32-ILP32FD-NEXT:    addi a0, zero, 1
-; RV32-ILP32FD-NEXT:    mv a1, zero
 ; RV32-ILP32FD-NEXT:    addi a2, zero, 2
-; RV32-ILP32FD-NEXT:    mv a3, zero
 ; RV32-ILP32FD-NEXT:    addi a4, zero, 3
-; RV32-ILP32FD-NEXT:    mv a5, zero
 ; RV32-ILP32FD-NEXT:    addi a6, zero, 4
+; RV32-ILP32FD-NEXT:    sw a1, 0(sp)
+; RV32-ILP32FD-NEXT:    mv a1, zero
+; RV32-ILP32FD-NEXT:    mv a3, zero
+; RV32-ILP32FD-NEXT:    mv a5, zero
 ; RV32-ILP32FD-NEXT:    mv a7, zero
 ; RV32-ILP32FD-NEXT:    call callee_float_in_fpr_exhausted_gprs
 ; RV32-ILP32FD-NEXT:    lw ra, 12(sp)
@@ -81,10 +81,10 @@ define i32 @caller_float_in_fpr_exhausted_gprs() nounwind {
 define i32 @callee_float_in_gpr_exhausted_fprs(float %a, float %b, float %c, float %d, float %e, float %f, float %g, float %h, float %i) nounwind {
 ; RV32-ILP32FD-LABEL: callee_float_in_gpr_exhausted_fprs:
 ; RV32-ILP32FD:       # %bb.0:
-; RV32-ILP32FD-NEXT:    fcvt.w.s a1, fa7, rtz
 ; RV32-ILP32FD-NEXT:    fmv.w.x ft0, a0
-; RV32-ILP32FD-NEXT:    fcvt.w.s a0, ft0, rtz
-; RV32-ILP32FD-NEXT:    add a0, a1, a0
+; RV32-ILP32FD-NEXT:    fcvt.w.s a0, fa7, rtz
+; RV32-ILP32FD-NEXT:    fcvt.w.s a1, ft0, rtz
+; RV32-ILP32FD-NEXT:    add a0, a0, a1
 ; RV32-ILP32FD-NEXT:    ret
   %h_fptosi = fptosi float %h to i32
   %i_fptosi = fptosi float %i to i32
@@ -99,21 +99,21 @@ define i32 @caller_float_in_gpr_exhausted_fprs() nounwind {
 ; RV32-ILP32FD-NEXT:    sw ra, 12(sp)
 ; RV32-ILP32FD-NEXT:    lui a0, %hi(.LCPI5_0)
 ; RV32-ILP32FD-NEXT:    addi a0, a0, %lo(.LCPI5_0)
-; RV32-ILP32FD-NEXT:    lui a1, %hi(.LCPI5_1)
-; RV32-ILP32FD-NEXT:    addi a1, a1, %lo(.LCPI5_1)
-; RV32-ILP32FD-NEXT:    lui a2, %hi(.LCPI5_2)
-; RV32-ILP32FD-NEXT:    addi a2, a2, %lo(.LCPI5_2)
-; RV32-ILP32FD-NEXT:    lui a3, %hi(.LCPI5_3)
-; RV32-ILP32FD-NEXT:    addi a3, a3, %lo(.LCPI5_3)
-; RV32-ILP32FD-NEXT:    lui a4, %hi(.LCPI5_4)
-; RV32-ILP32FD-NEXT:    addi a4, a4, %lo(.LCPI5_4)
-; RV32-ILP32FD-NEXT:    lui a5, %hi(.LCPI5_5)
-; RV32-ILP32FD-NEXT:    addi a5, a5, %lo(.LCPI5_5)
-; RV32-ILP32FD-NEXT:    flw fa0, 0(a5)
-; RV32-ILP32FD-NEXT:    flw fa1, 0(a4)
-; RV32-ILP32FD-NEXT:    flw fa2, 0(a3)
-; RV32-ILP32FD-NEXT:    flw fa3, 0(a2)
-; RV32-ILP32FD-NEXT:    flw fa4, 0(a1)
+; RV32-ILP32FD-NEXT:    flw fa0, 0(a0)
+; RV32-ILP32FD-NEXT:    lui a0, %hi(.LCPI5_1)
+; RV32-ILP32FD-NEXT:    addi a0, a0, %lo(.LCPI5_1)
+; RV32-ILP32FD-NEXT:    flw fa1, 0(a0)
+; RV32-ILP32FD-NEXT:    lui a0, %hi(.LCPI5_2)
+; RV32-ILP32FD-NEXT:    addi a0, a0, %lo(.LCPI5_2)
+; RV32-ILP32FD-NEXT:    flw fa2, 0(a0)
+; RV32-ILP32FD-NEXT:    lui a0, %hi(.LCPI5_3)
+; RV32-ILP32FD-NEXT:    addi a0, a0, %lo(.LCPI5_3)
+; RV32-ILP32FD-NEXT:    flw fa3, 0(a0)
+; RV32-ILP32FD-NEXT:    lui a0, %hi(.LCPI5_4)
+; RV32-ILP32FD-NEXT:    addi a0, a0, %lo(.LCPI5_4)
+; RV32-ILP32FD-NEXT:    flw fa4, 0(a0)
+; RV32-ILP32FD-NEXT:    lui a0, %hi(.LCPI5_5)
+; RV32-ILP32FD-NEXT:    addi a0, a0, %lo(.LCPI5_5)
 ; RV32-ILP32FD-NEXT:    flw fa5, 0(a0)
 ; RV32-ILP32FD-NEXT:    lui a0, %hi(.LCPI5_6)
 ; RV32-ILP32FD-NEXT:    addi a0, a0, %lo(.LCPI5_6)
@@ -151,39 +151,39 @@ define i32 @caller_float_on_stack_exhausted_gprs_fprs() nounwind {
 ; RV32-ILP32FD:       # %bb.0:
 ; RV32-ILP32FD-NEXT:    addi sp, sp, -16
 ; RV32-ILP32FD-NEXT:    sw ra, 12(sp)
-; RV32-ILP32FD-NEXT:    lui a0, 267520
-; RV32-ILP32FD-NEXT:    sw a0, 0(sp)
+; RV32-ILP32FD-NEXT:    lui a1, 267520
 ; RV32-ILP32FD-NEXT:    lui a0, %hi(.LCPI7_0)
-; RV32-ILP32FD-NEXT:    addi a6, a0, %lo(.LCPI7_0)
-; RV32-ILP32FD-NEXT:    lui a1, %hi(.LCPI7_1)
-; RV32-ILP32FD-NEXT:    addi a1, a1, %lo(.LCPI7_1)
-; RV32-ILP32FD-NEXT:    lui a2, %hi(.LCPI7_2)
-; RV32-ILP32FD-NEXT:    addi a2, a2, %lo(.LCPI7_2)
-; RV32-ILP32FD-NEXT:    lui a3, %hi(.LCPI7_3)
-; RV32-ILP32FD-NEXT:    addi a3, a3, %lo(.LCPI7_3)
-; RV32-ILP32FD-NEXT:    lui a4, %hi(.LCPI7_4)
-; RV32-ILP32FD-NEXT:    addi a4, a4, %lo(.LCPI7_4)
-; RV32-ILP32FD-NEXT:    lui a5, %hi(.LCPI7_5)
-; RV32-ILP32FD-NEXT:    addi a5, a5, %lo(.LCPI7_5)
+; RV32-ILP32FD-NEXT:    addi a0, a0, %lo(.LCPI7_0)
+; RV32-ILP32FD-NEXT:    flw fa0, 0(a0)
+; RV32-ILP32FD-NEXT:    lui a0, %hi(.LCPI7_1)
+; RV32-ILP32FD-NEXT:    addi a0, a0, %lo(.LCPI7_1)
+; RV32-ILP32FD-NEXT:    flw fa1, 0(a0)
+; RV32-ILP32FD-NEXT:    lui a0, %hi(.LCPI7_2)
+; RV32-ILP32FD-NEXT:    addi a0, a0, %lo(.LCPI7_2)
+; RV32-ILP32FD-NEXT:    flw fa2, 0(a0)
+; RV32-ILP32FD-NEXT:    lui a0, %hi(.LCPI7_3)
+; RV32-ILP32FD-NEXT:    addi a0, a0, %lo(.LCPI7_3)
+; RV32-ILP32FD-NEXT:    flw fa3, 0(a0)
+; RV32-ILP32FD-NEXT:    lui a0, %hi(.LCPI7_4)
+; RV32-ILP32FD-NEXT:    addi a0, a0, %lo(.LCPI7_4)
+; RV32-ILP32FD-NEXT:    flw fa4, 0(a0)
+; RV32-ILP32FD-NEXT:    lui a0, %hi(.LCPI7_5)
+; RV32-ILP32FD-NEXT:    addi a0, a0, %lo(.LCPI7_5)
+; RV32-ILP32FD-NEXT:    flw fa5, 0(a0)
 ; RV32-ILP32FD-NEXT:    lui a0, %hi(.LCPI7_6)
 ; RV32-ILP32FD-NEXT:    addi a0, a0, %lo(.LCPI7_6)
-; RV32-ILP32FD-NEXT:    flw fa0, 0(a0)
-; RV32-ILP32FD-NEXT:    flw fa1, 0(a5)
-; RV32-ILP32FD-NEXT:    flw fa2, 0(a4)
-; RV32-ILP32FD-NEXT:    flw fa3, 0(a3)
-; RV32-ILP32FD-NEXT:    flw fa4, 0(a2)
-; RV32-ILP32FD-NEXT:    flw fa5, 0(a1)
-; RV32-ILP32FD-NEXT:    flw fa6, 0(a6)
+; RV32-ILP32FD-NEXT:    flw fa6, 0(a0)
 ; RV32-ILP32FD-NEXT:    lui a0, %hi(.LCPI7_7)
 ; RV32-ILP32FD-NEXT:    addi a0, a0, %lo(.LCPI7_7)
 ; RV32-ILP32FD-NEXT:    flw fa7, 0(a0)
 ; RV32-ILP32FD-NEXT:    addi a0, zero, 1
-; RV32-ILP32FD-NEXT:    mv a1, zero
 ; RV32-ILP32FD-NEXT:    addi a2, zero, 3
-; RV32-ILP32FD-NEXT:    mv a3, zero
 ; RV32-ILP32FD-NEXT:    addi a4, zero, 5
-; RV32-ILP32FD-NEXT:    mv a5, zero
 ; RV32-ILP32FD-NEXT:    addi a6, zero, 7
+; RV32-ILP32FD-NEXT:    sw a1, 0(sp)
+; RV32-ILP32FD-NEXT:    mv a1, zero
+; RV32-ILP32FD-NEXT:    mv a3, zero
+; RV32-ILP32FD-NEXT:    mv a5, zero
 ; RV32-ILP32FD-NEXT:    mv a7, zero
 ; RV32-ILP32FD-NEXT:    call callee_float_on_stack_exhausted_gprs_fprs
 ; RV32-ILP32FD-NEXT:    lw ra, 12(sp)
diff --git a/llvm/test/CodeGen/RISCV/calling-conv-lp64-lp64f-lp64d-common.ll b/llvm/test/CodeGen/RISCV/calling-conv-lp64-lp64f-lp64d-common.ll
index fb27c3ae3..95cf39e79 100644
--- a/llvm/test/CodeGen/RISCV/calling-conv-lp64-lp64f-lp64d-common.ll
+++ b/llvm/test/CodeGen/RISCV/calling-conv-lp64-lp64f-lp64d-common.ll
@@ -48,22 +48,22 @@ define i64 @caller_i128_in_regs() nounwind {
 define i32 @callee_many_scalars(i8 %a, i16 %b, i32 %c, i128 %d, i32 %e, i32 %f, i128 %g, i32 %h) nounwind {
 ; RV64I-LABEL: callee_many_scalars:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    ld t0, 0(sp)
-; RV64I-NEXT:    xor a4, a4, t0
-; RV64I-NEXT:    xor a3, a3, a7
-; RV64I-NEXT:    or a3, a3, a4
-; RV64I-NEXT:    lui a4, 16
-; RV64I-NEXT:    addiw a4, a4, -1
-; RV64I-NEXT:    and a1, a1, a4
-; RV64I-NEXT:    andi a0, a0, 255
-; RV64I-NEXT:    add a0, a0, a1
+; RV64I-NEXT:    lw t0, 8(sp)
+; RV64I-NEXT:    ld t1, 0(sp)
+; RV64I-NEXT:    andi t2, a0, 255
+; RV64I-NEXT:    lui a0, 16
+; RV64I-NEXT:    addiw a0, a0, -1
+; RV64I-NEXT:    and a0, a1, a0
+; RV64I-NEXT:    add a0, t2, a0
 ; RV64I-NEXT:    add a0, a0, a2
-; RV64I-NEXT:    seqz a1, a3
+; RV64I-NEXT:    xor a1, a4, t1
+; RV64I-NEXT:    xor a2, a3, a7
+; RV64I-NEXT:    or a1, a2, a1
+; RV64I-NEXT:    seqz a1, a1
 ; RV64I-NEXT:    add a0, a1, a0
 ; RV64I-NEXT:    add a0, a0, a5
 ; RV64I-NEXT:    add a0, a0, a6
-; RV64I-NEXT:    lw a1, 8(sp)
-; RV64I-NEXT:    add a0, a0, a1
+; RV64I-NEXT:    addw a0, a0, t0
 ; RV64I-NEXT:    ret
   %a_ext = zext i8 %a to i32
   %b_ext = zext i16 %b to i32
@@ -85,15 +85,15 @@ define i32 @caller_many_scalars() nounwind {
 ; RV64I-NEXT:    sd ra, 24(sp)
 ; RV64I-NEXT:    addi a0, zero, 8
 ; RV64I-NEXT:    sd a0, 8(sp)
-; RV64I-NEXT:    sd zero, 0(sp)
 ; RV64I-NEXT:    addi a0, zero, 1
 ; RV64I-NEXT:    addi a1, zero, 2
 ; RV64I-NEXT:    addi a2, zero, 3
 ; RV64I-NEXT:    addi a3, zero, 4
-; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    addi a5, zero, 5
 ; RV64I-NEXT:    addi a6, zero, 6
 ; RV64I-NEXT:    addi a7, zero, 7
+; RV64I-NEXT:    sd zero, 0(sp)
+; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    call callee_many_scalars
 ; RV64I-NEXT:    ld ra, 24(sp)
 ; RV64I-NEXT:    addi sp, sp, 32
@@ -107,20 +107,20 @@ define i32 @caller_many_scalars() nounwind {
 define i64 @callee_large_scalars(i256 %a, i256 %b) nounwind {
 ; RV64I-LABEL: callee_large_scalars:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    ld a2, 24(a1)
-; RV64I-NEXT:    ld a3, 24(a0)
-; RV64I-NEXT:    xor a2, a3, a2
-; RV64I-NEXT:    ld a3, 8(a1)
-; RV64I-NEXT:    ld a4, 8(a0)
-; RV64I-NEXT:    xor a3, a4, a3
+; RV64I-NEXT:    ld a6, 0(a1)
+; RV64I-NEXT:    ld a7, 0(a0)
+; RV64I-NEXT:    ld a4, 8(a1)
+; RV64I-NEXT:    ld a5, 24(a1)
+; RV64I-NEXT:    ld a2, 24(a0)
+; RV64I-NEXT:    ld a3, 8(a0)
+; RV64I-NEXT:    ld a1, 16(a1)
+; RV64I-NEXT:    ld a0, 16(a0)
+; RV64I-NEXT:    xor a2, a2, a5
+; RV64I-NEXT:    xor a3, a3, a4
 ; RV64I-NEXT:    or a2, a3, a2
-; RV64I-NEXT:    ld a3, 16(a1)
-; RV64I-NEXT:    ld a4, 16(a0)
-; RV64I-NEXT:    xor a3, a4, a3
-; RV64I-NEXT:    ld a1, 0(a1)
-; RV64I-NEXT:    ld a0, 0(a0)
 ; RV64I-NEXT:    xor a0, a0, a1
-; RV64I-NEXT:    or a0, a0, a3
+; RV64I-NEXT:    xor a1, a7, a6
+; RV64I-NEXT:    or a0, a1, a0
 ; RV64I-NEXT:    or a0, a0, a2
 ; RV64I-NEXT:    seqz a0, a0
 ; RV64I-NEXT:    ret
@@ -134,18 +134,18 @@ define i64 @caller_large_scalars() nounwind {
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -80
 ; RV64I-NEXT:    sd ra, 72(sp)
-; RV64I-NEXT:    addi a0, zero, 2
-; RV64I-NEXT:    sd a0, 0(sp)
 ; RV64I-NEXT:    sd zero, 24(sp)
 ; RV64I-NEXT:    sd zero, 16(sp)
 ; RV64I-NEXT:    sd zero, 8(sp)
+; RV64I-NEXT:    addi a0, zero, 2
+; RV64I-NEXT:    sd a0, 0(sp)
 ; RV64I-NEXT:    sd zero, 56(sp)
 ; RV64I-NEXT:    sd zero, 48(sp)
 ; RV64I-NEXT:    sd zero, 40(sp)
-; RV64I-NEXT:    addi a0, zero, 1
-; RV64I-NEXT:    sd a0, 32(sp)
+; RV64I-NEXT:    addi a2, zero, 1
 ; RV64I-NEXT:    addi a0, sp, 32
 ; RV64I-NEXT:    mv a1, sp
+; RV64I-NEXT:    sd a2, 32(sp)
 ; RV64I-NEXT:    call callee_large_scalars
 ; RV64I-NEXT:    ld ra, 72(sp)
 ; RV64I-NEXT:    addi sp, sp, 80
@@ -162,20 +162,20 @@ define i64 @callee_large_scalars_exhausted_regs(i64 %a, i64 %b, i64 %c, i64 %d,
 ; RV64I-LABEL: callee_large_scalars_exhausted_regs:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    ld a0, 8(sp)
-; RV64I-NEXT:    ld a1, 24(a0)
-; RV64I-NEXT:    ld a2, 24(a7)
-; RV64I-NEXT:    xor a1, a2, a1
-; RV64I-NEXT:    ld a2, 8(a0)
-; RV64I-NEXT:    ld a3, 8(a7)
-; RV64I-NEXT:    xor a2, a3, a2
-; RV64I-NEXT:    or a1, a2, a1
-; RV64I-NEXT:    ld a2, 16(a0)
-; RV64I-NEXT:    ld a3, 16(a7)
-; RV64I-NEXT:    xor a2, a3, a2
-; RV64I-NEXT:    ld a0, 0(a0)
-; RV64I-NEXT:    ld a3, 0(a7)
-; RV64I-NEXT:    xor a0, a3, a0
-; RV64I-NEXT:    or a0, a0, a2
+; RV64I-NEXT:    ld a6, 0(a0)
+; RV64I-NEXT:    ld t0, 0(a7)
+; RV64I-NEXT:    ld a3, 8(a0)
+; RV64I-NEXT:    ld a4, 24(a0)
+; RV64I-NEXT:    ld a5, 24(a7)
+; RV64I-NEXT:    ld a1, 8(a7)
+; RV64I-NEXT:    ld a0, 16(a0)
+; RV64I-NEXT:    ld a2, 16(a7)
+; RV64I-NEXT:    xor a4, a5, a4
+; RV64I-NEXT:    xor a1, a1, a3
+; RV64I-NEXT:    or a1, a1, a4
+; RV64I-NEXT:    xor a0, a2, a0
+; RV64I-NEXT:    xor a2, t0, a6
+; RV64I-NEXT:    or a0, a2, a0
 ; RV64I-NEXT:    or a0, a0, a1
 ; RV64I-NEXT:    seqz a0, a0
 ; RV64I-NEXT:    ret
@@ -193,16 +193,15 @@ define i64 @caller_large_scalars_exhausted_regs() nounwind {
 ; RV64I-NEXT:    sd a0, 8(sp)
 ; RV64I-NEXT:    addi a0, zero, 9
 ; RV64I-NEXT:    sd a0, 0(sp)
-; RV64I-NEXT:    addi a0, zero, 10
-; RV64I-NEXT:    sd a0, 16(sp)
 ; RV64I-NEXT:    sd zero, 40(sp)
 ; RV64I-NEXT:    sd zero, 32(sp)
 ; RV64I-NEXT:    sd zero, 24(sp)
+; RV64I-NEXT:    addi a0, zero, 10
+; RV64I-NEXT:    sd a0, 16(sp)
 ; RV64I-NEXT:    sd zero, 72(sp)
 ; RV64I-NEXT:    sd zero, 64(sp)
 ; RV64I-NEXT:    sd zero, 56(sp)
-; RV64I-NEXT:    addi a0, zero, 8
-; RV64I-NEXT:    sd a0, 48(sp)
+; RV64I-NEXT:    addi t0, zero, 8
 ; RV64I-NEXT:    addi a7, sp, 48
 ; RV64I-NEXT:    addi a0, zero, 1
 ; RV64I-NEXT:    addi a1, zero, 2
@@ -211,6 +210,7 @@ define i64 @caller_large_scalars_exhausted_regs() nounwind {
 ; RV64I-NEXT:    addi a4, zero, 5
 ; RV64I-NEXT:    addi a5, zero, 6
 ; RV64I-NEXT:    addi a6, zero, 7
+; RV64I-NEXT:    sd t0, 48(sp)
 ; RV64I-NEXT:    call callee_large_scalars_exhausted_regs
 ; RV64I-NEXT:    ld ra, 88(sp)
 ; RV64I-NEXT:    addi sp, sp, 96
@@ -277,9 +277,9 @@ define i64 @caller_small_coerced_struct() nounwind {
 define i64 @callee_large_struct(%struct.large* byval align 8 %a) nounwind {
 ; RV64I-LABEL: callee_large_struct:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    ld a1, 24(a0)
-; RV64I-NEXT:    ld a0, 0(a0)
-; RV64I-NEXT:    add a0, a0, a1
+; RV64I-NEXT:    ld a1, 0(a0)
+; RV64I-NEXT:    ld a0, 24(a0)
+; RV64I-NEXT:    add a0, a1, a0
 ; RV64I-NEXT:    ret
   %1 = getelementptr inbounds %struct.large, %struct.large* %a, i64 0, i32 0
   %2 = getelementptr inbounds %struct.large, %struct.large* %a, i64 0, i32 3
@@ -296,16 +296,16 @@ define i64 @caller_large_struct() nounwind {
 ; RV64I-NEXT:    sd ra, 72(sp)
 ; RV64I-NEXT:    addi a0, zero, 1
 ; RV64I-NEXT:    sd a0, 40(sp)
+; RV64I-NEXT:    addi a1, zero, 2
+; RV64I-NEXT:    sd a1, 48(sp)
+; RV64I-NEXT:    addi a2, zero, 3
+; RV64I-NEXT:    sd a2, 56(sp)
+; RV64I-NEXT:    addi a3, zero, 4
+; RV64I-NEXT:    sd a3, 64(sp)
 ; RV64I-NEXT:    sd a0, 8(sp)
-; RV64I-NEXT:    addi a0, zero, 2
-; RV64I-NEXT:    sd a0, 48(sp)
-; RV64I-NEXT:    sd a0, 16(sp)
-; RV64I-NEXT:    addi a0, zero, 3
-; RV64I-NEXT:    sd a0, 56(sp)
-; RV64I-NEXT:    sd a0, 24(sp)
-; RV64I-NEXT:    addi a0, zero, 4
-; RV64I-NEXT:    sd a0, 64(sp)
-; RV64I-NEXT:    sd a0, 32(sp)
+; RV64I-NEXT:    sd a1, 16(sp)
+; RV64I-NEXT:    sd a2, 24(sp)
+; RV64I-NEXT:    sd a3, 32(sp)
 ; RV64I-NEXT:    addi a0, sp, 8
 ; RV64I-NEXT:    call callee_large_struct
 ; RV64I-NEXT:    ld ra, 72(sp)
@@ -332,15 +332,15 @@ define i64 @callee_aligned_stack(i64 %a, i64 %b, i64 %c, i64 %d, i64 %e, i128 %f
 ; should only be 8-byte aligned
 ; RV64I-LABEL: callee_aligned_stack:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    add a0, a5, a7
+; RV64I-NEXT:    ld a0, 40(sp)
 ; RV64I-NEXT:    ld a1, 0(sp)
-; RV64I-NEXT:    add a0, a0, a1
-; RV64I-NEXT:    ld a1, 16(sp)
-; RV64I-NEXT:    add a0, a0, a1
-; RV64I-NEXT:    ld a1, 32(sp)
-; RV64I-NEXT:    add a0, a0, a1
-; RV64I-NEXT:    ld a1, 40(sp)
-; RV64I-NEXT:    add a0, a0, a1
+; RV64I-NEXT:    ld a2, 16(sp)
+; RV64I-NEXT:    ld a3, 32(sp)
+; RV64I-NEXT:    add a4, a5, a7
+; RV64I-NEXT:    add a1, a4, a1
+; RV64I-NEXT:    add a1, a1, a2
+; RV64I-NEXT:    add a1, a1, a3
+; RV64I-NEXT:    add a0, a1, a0
 ; RV64I-NEXT:    ret
   %f_trunc = trunc i128 %f to i64
   %1 = add i64 %f_trunc, %g
@@ -366,19 +366,19 @@ define void @caller_aligned_stack() nounwind {
 ; RV64I-NEXT:    sd a0, 40(sp)
 ; RV64I-NEXT:    addi a0, zero, 10
 ; RV64I-NEXT:    sd a0, 32(sp)
+; RV64I-NEXT:    sd zero, 24(sp)
 ; RV64I-NEXT:    addi a0, zero, 9
 ; RV64I-NEXT:    sd a0, 16(sp)
-; RV64I-NEXT:    addi a0, zero, 8
-; RV64I-NEXT:    sd a0, 0(sp)
-; RV64I-NEXT:    sd zero, 24(sp)
+; RV64I-NEXT:    addi a6, zero, 8
 ; RV64I-NEXT:    addi a0, zero, 1
 ; RV64I-NEXT:    addi a1, zero, 2
 ; RV64I-NEXT:    addi a2, zero, 3
 ; RV64I-NEXT:    addi a3, zero, 4
 ; RV64I-NEXT:    addi a4, zero, 5
 ; RV64I-NEXT:    addi a5, zero, 6
-; RV64I-NEXT:    mv a6, zero
 ; RV64I-NEXT:    addi a7, zero, 7
+; RV64I-NEXT:    sd a6, 0(sp)
+; RV64I-NEXT:    mv a6, zero
 ; RV64I-NEXT:    call callee_aligned_stack
 ; RV64I-NEXT:    ld ra, 56(sp)
 ; RV64I-NEXT:    addi sp, sp, 64
@@ -482,18 +482,18 @@ define void @caller_large_scalar_ret() nounwind {
 define void @callee_large_struct_ret(%struct.large* noalias sret %agg.result) nounwind {
 ; RV64I-LABEL: callee_large_struct_ret:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    addi a1, zero, 4
-; RV64I-NEXT:    sw a1, 24(a0)
-; RV64I-NEXT:    addi a1, zero, 3
-; RV64I-NEXT:    sw a1, 16(a0)
-; RV64I-NEXT:    addi a1, zero, 2
-; RV64I-NEXT:    sw a1, 8(a0)
-; RV64I-NEXT:    sw zero, 28(a0)
-; RV64I-NEXT:    sw zero, 20(a0)
-; RV64I-NEXT:    sw zero, 12(a0)
 ; RV64I-NEXT:    sw zero, 4(a0)
 ; RV64I-NEXT:    addi a1, zero, 1
 ; RV64I-NEXT:    sw a1, 0(a0)
+; RV64I-NEXT:    sw zero, 12(a0)
+; RV64I-NEXT:    addi a1, zero, 2
+; RV64I-NEXT:    sw a1, 8(a0)
+; RV64I-NEXT:    sw zero, 20(a0)
+; RV64I-NEXT:    addi a1, zero, 3
+; RV64I-NEXT:    sw a1, 16(a0)
+; RV64I-NEXT:    sw zero, 28(a0)
+; RV64I-NEXT:    addi a1, zero, 4
+; RV64I-NEXT:    sw a1, 24(a0)
 ; RV64I-NEXT:    ret
   %a = getelementptr inbounds %struct.large, %struct.large* %agg.result, i64 0, i32 0
   store i64 1, i64* %a, align 4
@@ -513,9 +513,9 @@ define i64 @caller_large_struct_ret() nounwind {
 ; RV64I-NEXT:    sd ra, 40(sp)
 ; RV64I-NEXT:    addi a0, sp, 8
 ; RV64I-NEXT:    call callee_large_struct_ret
-; RV64I-NEXT:    ld a0, 32(sp)
-; RV64I-NEXT:    ld a1, 8(sp)
-; RV64I-NEXT:    add a0, a1, a0
+; RV64I-NEXT:    ld a0, 8(sp)
+; RV64I-NEXT:    ld a1, 32(sp)
+; RV64I-NEXT:    add a0, a0, a1
 ; RV64I-NEXT:    ld ra, 40(sp)
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/calling-conv-lp64.ll b/llvm/test/CodeGen/RISCV/calling-conv-lp64.ll
index faa68185a..3135fcd16 100644
--- a/llvm/test/CodeGen/RISCV/calling-conv-lp64.ll
+++ b/llvm/test/CodeGen/RISCV/calling-conv-lp64.ll
@@ -21,8 +21,7 @@ define i64 @callee_float_in_regs(i64 %a, float %b) nounwind {
 ; RV64I-FPELIM-NEXT:    sd ra, 8(sp)
 ; RV64I-FPELIM-NEXT:    sd s0, 0(sp)
 ; RV64I-FPELIM-NEXT:    mv s0, a0
-; RV64I-FPELIM-NEXT:    slli a0, a1, 32
-; RV64I-FPELIM-NEXT:    srli a0, a0, 32
+; RV64I-FPELIM-NEXT:    mv a0, a1
 ; RV64I-FPELIM-NEXT:    call __fixsfdi
 ; RV64I-FPELIM-NEXT:    add a0, s0, a0
 ; RV64I-FPELIM-NEXT:    ld s0, 0(sp)
@@ -38,8 +37,7 @@ define i64 @callee_float_in_regs(i64 %a, float %b) nounwind {
 ; RV64I-WITHFP-NEXT:    sd s1, 8(sp)
 ; RV64I-WITHFP-NEXT:    addi s0, sp, 32
 ; RV64I-WITHFP-NEXT:    mv s1, a0
-; RV64I-WITHFP-NEXT:    slli a0, a1, 32
-; RV64I-WITHFP-NEXT:    srli a0, a0, 32
+; RV64I-WITHFP-NEXT:    mv a0, a1
 ; RV64I-WITHFP-NEXT:    call __fixsfdi
 ; RV64I-WITHFP-NEXT:    add a0, s1, a0
 ; RV64I-WITHFP-NEXT:    ld s1, 8(sp)
@@ -110,15 +108,15 @@ define i64 @caller_float_on_stack() nounwind {
 ; RV64I-FPELIM:       # %bb.0:
 ; RV64I-FPELIM-NEXT:    addi sp, sp, -16
 ; RV64I-FPELIM-NEXT:    sd ra, 8(sp)
-; RV64I-FPELIM-NEXT:    lui a0, 264704
-; RV64I-FPELIM-NEXT:    sd a0, 0(sp)
+; RV64I-FPELIM-NEXT:    lui a1, 264704
 ; RV64I-FPELIM-NEXT:    addi a0, zero, 1
-; RV64I-FPELIM-NEXT:    mv a1, zero
 ; RV64I-FPELIM-NEXT:    addi a2, zero, 2
-; RV64I-FPELIM-NEXT:    mv a3, zero
 ; RV64I-FPELIM-NEXT:    addi a4, zero, 3
-; RV64I-FPELIM-NEXT:    mv a5, zero
 ; RV64I-FPELIM-NEXT:    addi a6, zero, 4
+; RV64I-FPELIM-NEXT:    sd a1, 0(sp)
+; RV64I-FPELIM-NEXT:    mv a1, zero
+; RV64I-FPELIM-NEXT:    mv a3, zero
+; RV64I-FPELIM-NEXT:    mv a5, zero
 ; RV64I-FPELIM-NEXT:    mv a7, zero
 ; RV64I-FPELIM-NEXT:    call callee_float_on_stack
 ; RV64I-FPELIM-NEXT:    ld ra, 8(sp)
@@ -131,15 +129,15 @@ define i64 @caller_float_on_stack() nounwind {
 ; RV64I-WITHFP-NEXT:    sd ra, 24(sp)
 ; RV64I-WITHFP-NEXT:    sd s0, 16(sp)
 ; RV64I-WITHFP-NEXT:    addi s0, sp, 32
-; RV64I-WITHFP-NEXT:    lui a0, 264704
-; RV64I-WITHFP-NEXT:    sd a0, 0(sp)
+; RV64I-WITHFP-NEXT:    lui a1, 264704
 ; RV64I-WITHFP-NEXT:    addi a0, zero, 1
-; RV64I-WITHFP-NEXT:    mv a1, zero
 ; RV64I-WITHFP-NEXT:    addi a2, zero, 2
-; RV64I-WITHFP-NEXT:    mv a3, zero
 ; RV64I-WITHFP-NEXT:    addi a4, zero, 3
-; RV64I-WITHFP-NEXT:    mv a5, zero
 ; RV64I-WITHFP-NEXT:    addi a6, zero, 4
+; RV64I-WITHFP-NEXT:    sd a1, 0(sp)
+; RV64I-WITHFP-NEXT:    mv a1, zero
+; RV64I-WITHFP-NEXT:    mv a3, zero
+; RV64I-WITHFP-NEXT:    mv a5, zero
 ; RV64I-WITHFP-NEXT:    mv a7, zero
 ; RV64I-WITHFP-NEXT:    call callee_float_on_stack
 ; RV64I-WITHFP-NEXT:    ld s0, 16(sp)
diff --git a/llvm/test/CodeGen/RISCV/calling-conv-rv32f-ilp32.ll b/llvm/test/CodeGen/RISCV/calling-conv-rv32f-ilp32.ll
index 9a68f0006..bc86428d0 100644
--- a/llvm/test/CodeGen/RISCV/calling-conv-rv32f-ilp32.ll
+++ b/llvm/test/CodeGen/RISCV/calling-conv-rv32f-ilp32.ll
@@ -33,15 +33,15 @@ define float @caller_onstack_f32_noop(float %a) nounwind {
 ; RV32IF-NEXT:    addi sp, sp, -16
 ; RV32IF-NEXT:    sw ra, 12(sp)
 ; RV32IF-NEXT:    sw a0, 4(sp)
-; RV32IF-NEXT:    lui a0, 264704
-; RV32IF-NEXT:    sw a0, 0(sp)
+; RV32IF-NEXT:    lui a1, 264704
 ; RV32IF-NEXT:    addi a0, zero, 1
-; RV32IF-NEXT:    mv a1, zero
 ; RV32IF-NEXT:    addi a2, zero, 2
-; RV32IF-NEXT:    mv a3, zero
 ; RV32IF-NEXT:    addi a4, zero, 3
-; RV32IF-NEXT:    mv a5, zero
 ; RV32IF-NEXT:    addi a6, zero, 4
+; RV32IF-NEXT:    sw a1, 0(sp)
+; RV32IF-NEXT:    mv a1, zero
+; RV32IF-NEXT:    mv a3, zero
+; RV32IF-NEXT:    mv a5, zero
 ; RV32IF-NEXT:    mv a7, zero
 ; RV32IF-NEXT:    call onstack_f32_noop
 ; RV32IF-NEXT:    lw ra, 12(sp)
@@ -56,19 +56,19 @@ define float @caller_onstack_f32_fadd(float %a, float %b) nounwind {
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
 ; RV32IF-NEXT:    sw ra, 12(sp)
-; RV32IF-NEXT:    fmv.w.x ft0, a0
-; RV32IF-NEXT:    fmv.w.x ft1, a1
-; RV32IF-NEXT:    fsub.s ft2, ft1, ft0
-; RV32IF-NEXT:    fsw ft2, 4(sp)
-; RV32IF-NEXT:    fadd.s ft0, ft0, ft1
-; RV32IF-NEXT:    fsw ft0, 0(sp)
+; RV32IF-NEXT:    fmv.w.x ft0, a1
+; RV32IF-NEXT:    fmv.w.x ft1, a0
+; RV32IF-NEXT:    fadd.s ft2, ft1, ft0
+; RV32IF-NEXT:    fsub.s ft0, ft0, ft1
+; RV32IF-NEXT:    fsw ft0, 4(sp)
 ; RV32IF-NEXT:    addi a0, zero, 1
-; RV32IF-NEXT:    mv a1, zero
 ; RV32IF-NEXT:    addi a2, zero, 2
-; RV32IF-NEXT:    mv a3, zero
 ; RV32IF-NEXT:    addi a4, zero, 3
-; RV32IF-NEXT:    mv a5, zero
 ; RV32IF-NEXT:    addi a6, zero, 4
+; RV32IF-NEXT:    fsw ft2, 0(sp)
+; RV32IF-NEXT:    mv a1, zero
+; RV32IF-NEXT:    mv a3, zero
+; RV32IF-NEXT:    mv a5, zero
 ; RV32IF-NEXT:    mv a7, zero
 ; RV32IF-NEXT:    call onstack_f32_noop
 ; RV32IF-NEXT:    lw ra, 12(sp)
diff --git a/llvm/test/CodeGen/RISCV/codemodel-lowering.ll b/llvm/test/CodeGen/RISCV/codemodel-lowering.ll
index 0706a6129..6430a221e 100644
--- a/llvm/test/CodeGen/RISCV/codemodel-lowering.ll
+++ b/llvm/test/CodeGen/RISCV/codemodel-lowering.ll
@@ -61,9 +61,9 @@ define signext i32 @lower_blockaddress_displ(i32 signext %w) nounwind {
 ; RV32I-SMALL-NEXT:    sw ra, 12(sp)
 ; RV32I-SMALL-NEXT:    lui a1, %hi(.Ltmp0)
 ; RV32I-SMALL-NEXT:    addi a1, a1, %lo(.Ltmp0)
+; RV32I-SMALL-NEXT:    addi a2, zero, 101
 ; RV32I-SMALL-NEXT:    sw a1, 8(sp)
-; RV32I-SMALL-NEXT:    addi a1, zero, 101
-; RV32I-SMALL-NEXT:    blt a0, a1, .LBB2_3
+; RV32I-SMALL-NEXT:    blt a0, a2, .LBB2_3
 ; RV32I-SMALL-NEXT:  # %bb.1: # %if.then
 ; RV32I-SMALL-NEXT:    lw a0, 8(sp)
 ; RV32I-SMALL-NEXT:    jr a0
@@ -86,9 +86,9 @@ define signext i32 @lower_blockaddress_displ(i32 signext %w) nounwind {
 ; RV32I-MEDIUM-NEXT:    # Label of block must be emitted
 ; RV32I-MEDIUM-NEXT:    auipc a1, %pcrel_hi(.Ltmp0)
 ; RV32I-MEDIUM-NEXT:    addi a1, a1, %pcrel_lo(.LBB2_5)
+; RV32I-MEDIUM-NEXT:    addi a2, zero, 101
 ; RV32I-MEDIUM-NEXT:    sw a1, 8(sp)
-; RV32I-MEDIUM-NEXT:    addi a1, zero, 101
-; RV32I-MEDIUM-NEXT:    blt a0, a1, .LBB2_3
+; RV32I-MEDIUM-NEXT:    blt a0, a2, .LBB2_3
 ; RV32I-MEDIUM-NEXT:  # %bb.1: # %if.then
 ; RV32I-MEDIUM-NEXT:    lw a0, 8(sp)
 ; RV32I-MEDIUM-NEXT:    jr a0
@@ -131,11 +131,11 @@ indirectgoto:
 define float @lower_constantpool(float %a) nounwind {
 ; RV32I-SMALL-LABEL: lower_constantpool:
 ; RV32I-SMALL:       # %bb.0:
-; RV32I-SMALL-NEXT:    fmv.w.x ft0, a0
-; RV32I-SMALL-NEXT:    lui a0, %hi(.LCPI3_0)
-; RV32I-SMALL-NEXT:    addi a0, a0, %lo(.LCPI3_0)
-; RV32I-SMALL-NEXT:    flw ft1, 0(a0)
-; RV32I-SMALL-NEXT:    fadd.s ft0, ft0, ft1
+; RV32I-SMALL-NEXT:    lui a1, %hi(.LCPI3_0)
+; RV32I-SMALL-NEXT:    addi a1, a1, %lo(.LCPI3_0)
+; RV32I-SMALL-NEXT:    flw ft0, 0(a1)
+; RV32I-SMALL-NEXT:    fmv.w.x ft1, a0
+; RV32I-SMALL-NEXT:    fadd.s ft0, ft1, ft0
 ; RV32I-SMALL-NEXT:    fmv.x.w a0, ft0
 ; RV32I-SMALL-NEXT:    ret
 ;
diff --git a/llvm/test/CodeGen/RISCV/compress-inline-asm.ll b/llvm/test/CodeGen/RISCV/compress-inline-asm.ll
index 9b84bdeaf..fca4d6d1c 100644
--- a/llvm/test/CodeGen/RISCV/compress-inline-asm.ll
+++ b/llvm/test/CodeGen/RISCV/compress-inline-asm.ll
@@ -1,5 +1,5 @@
 ; RUN: llc -mtriple=riscv32 -mattr=+c  -filetype=obj < %s\
-; RUN: | llvm-objdump -triple=riscv32 -mattr=+c -d -riscv-no-aliases -\
+; RUN: | llvm-objdump -triple=riscv32 -mattr=+c -d -M no-aliases -\
 ; RUN: | FileCheck -check-prefix=CHECK %s
 
 @ext = external global i32
diff --git a/llvm/test/CodeGen/RISCV/compress.ll b/llvm/test/CodeGen/RISCV/compress.ll
index 2f0d4e9f2..b58adb848 100644
--- a/llvm/test/CodeGen/RISCV/compress.ll
+++ b/llvm/test/CodeGen/RISCV/compress.ll
@@ -1,6 +1,6 @@
 ; RUN: llc -mtriple=riscv32 -mattr=+c -filetype=obj \
 ; RUN:   -disable-block-placement < %s \
-; RUN:   | llvm-objdump -d -triple=riscv32 -mattr=+c -riscv-no-aliases - \
+; RUN:   | llvm-objdump -d -triple=riscv32 -mattr=+c -M no-aliases - \
 ; RUN:   | FileCheck -check-prefix=RV32IC %s
 
 ; This acts as a sanity check for the codegen instruction compression path,
@@ -19,10 +19,10 @@
 
 define i32 @simple_arith(i32 %a, i32 %b) nounwind {
 ; RV32IC-LABEL: simple_arith:
-; RV32IC:         c.srai a1, 9
-; RV32IC-NEXT:    addi a2, a0, 1
+; RV32IC:         addi a2, a0, 1
 ; RV32IC-NEXT:    c.andi a2, 11
 ; RV32IC-NEXT:    c.slli a2, 7
+; RV32IC-NEXT:    c.srai a1, 9
 ; RV32IC-NEXT:    c.add a1, a2
 ; RV32IC-NEXT:    sub a0, a1, a0
 ; RV32IC-NEXT:    c.jr ra
diff --git a/llvm/test/CodeGen/RISCV/copysign-casts.ll b/llvm/test/CodeGen/RISCV/copysign-casts.ll
new file mode 100644
index 000000000..acd64c203
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/copysign-casts.ll
@@ -0,0 +1,110 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefix=RV32I
+; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefix=RV64I
+; RUN: llc -mtriple=riscv32 -verify-machineinstrs -mattr=+f \
+; RUN:   -target-abi ilp32f < %s | FileCheck %s -check-prefix=RV32IF
+; RUN: llc -mtriple=riscv32 -verify-machineinstrs -mattr=+f -mattr=+d \
+; RUN:   -target-abi ilp32d < %s | FileCheck %s -check-prefix=RV32IFD
+; RUN: llc -mtriple=riscv64 -verify-machineinstrs -mattr=+f -mattr=+d \
+; RUN:   -target-abi lp64d < %s | FileCheck %s -check-prefix=RV64IFD
+
+; Test fcopysign scenarios where the sign argument is casted to the type of the
+; magnitude argument. Those casts can be folded away by the DAGCombiner.
+
+declare double @llvm.copysign.f64(double, double)
+declare float @llvm.copysign.f32(float, float)
+
+define double @fold_promote(double %a, float %b) nounwind {
+; RV32I-LABEL: fold_promote:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lui a3, 524288
+; RV32I-NEXT:    and a2, a2, a3
+; RV32I-NEXT:    addi a3, a3, -1
+; RV32I-NEXT:    and a1, a1, a3
+; RV32I-NEXT:    or a1, a1, a2
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: fold_promote:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi a2, zero, -1
+; RV64I-NEXT:    slli a2, a2, 63
+; RV64I-NEXT:    addi a2, a2, -1
+; RV64I-NEXT:    and a0, a0, a2
+; RV64I-NEXT:    addi a2, zero, 1
+; RV64I-NEXT:    slli a2, a2, 31
+; RV64I-NEXT:    and a1, a1, a2
+; RV64I-NEXT:    slli a1, a1, 32
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV32IF-LABEL: fold_promote:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    fmv.x.w a2, fa0
+; RV32IF-NEXT:    lui a3, 524288
+; RV32IF-NEXT:    and a2, a2, a3
+; RV32IF-NEXT:    addi a3, a3, -1
+; RV32IF-NEXT:    and a1, a1, a3
+; RV32IF-NEXT:    or a1, a1, a2
+; RV32IF-NEXT:    ret
+;
+; RV32IFD-LABEL: fold_promote:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    fcvt.d.s ft0, fa1
+; RV32IFD-NEXT:    fsgnj.d fa0, fa0, ft0
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: fold_promote:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fcvt.d.s ft0, fa1
+; RV64IFD-NEXT:    fsgnj.d fa0, fa0, ft0
+; RV64IFD-NEXT:    ret
+  %c = fpext float %b to double
+  %t = call double @llvm.copysign.f64(double %a, double %c)
+  ret double %t
+}
+
+define float @fold_demote(float %a, double %b) nounwind {
+; RV32I-LABEL: fold_demote:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lui a1, 524288
+; RV32I-NEXT:    and a2, a2, a1
+; RV32I-NEXT:    addi a1, a1, -1
+; RV32I-NEXT:    and a0, a0, a1
+; RV32I-NEXT:    or a0, a0, a2
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: fold_demote:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    lui a2, 524288
+; RV64I-NEXT:    addiw a2, a2, -1
+; RV64I-NEXT:    and a0, a0, a2
+; RV64I-NEXT:    addi a2, zero, -1
+; RV64I-NEXT:    slli a2, a2, 63
+; RV64I-NEXT:    and a1, a1, a2
+; RV64I-NEXT:    srli a1, a1, 32
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV32IF-LABEL: fold_demote:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    fmv.w.x ft0, a1
+; RV32IF-NEXT:    fsgnj.s fa0, fa0, ft0
+; RV32IF-NEXT:    ret
+;
+; RV32IFD-LABEL: fold_demote:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    fcvt.s.d ft0, fa1
+; RV32IFD-NEXT:    fsgnj.s fa0, fa0, ft0
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: fold_demote:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fcvt.s.d ft0, fa1
+; RV64IFD-NEXT:    fsgnj.s fa0, fa0, ft0
+; RV64IFD-NEXT:    ret
+  %c = fptrunc double %b to float
+  %t = call float @llvm.copysign.f32(float %a, float %c)
+  ret float %t
+}
diff --git a/llvm/test/CodeGen/RISCV/disjoint.ll b/llvm/test/CodeGen/RISCV/disjoint.ll
new file mode 100644
index 000000000..5977c6113
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/disjoint.ll
@@ -0,0 +1,26 @@
+; REQUIRES: asserts
+; RUN: llc -mtriple=riscv32 -debug-only=machine-scheduler < %s \
+; RUN:   -o /dev/null 2>&1 | FileCheck %s
+; RUN: llc -mtriple=riscv64 -debug-only=machine-scheduler < %s \
+; RUN:   -o /dev/null 2>&1 | FileCheck %s
+
+; This test exercises the areMemAccessesTriviallyDisjoint hook.
+; Test that the two stores are disjoint memory accesses. If the corresponding
+; store machine instructions don't depend on each other, the second store should
+; not appear in the successors list of the first one and the first one should
+; not appear on the predecessors list of the second one.
+define i32 @test_disjoint(i32* %P, i32 %v) {
+entry:
+; CHECK: ********** MI Scheduling **********
+; CHECK-LABEL: test_disjoint:%bb.0
+; CHECK:SU(2):   SW %1:gpr, %0:gpr, 12 :: (store 4 into %ir.arrayidx)
+; CHECK-NOT: Successors:
+; CHECK:SU(3):   SW %1:gpr, %0:gpr, 8 :: (store 4 into %ir.arrayidx1)
+; CHECK: Predecessors:
+; CHECK-NOT:    SU(2): Ord  Latency=0 Memory
+  %arrayidx = getelementptr inbounds i32, i32* %P, i32 3
+  store i32 %v, i32* %arrayidx
+  %arrayidx1 = getelementptr inbounds i32, i32* %P, i32 2
+  store i32 %v, i32* %arrayidx1
+  ret i32 %v
+}
diff --git a/llvm/test/CodeGen/RISCV/div.ll b/llvm/test/CodeGen/RISCV/div.ll
index 1fd0084f3..e3504b927 100644
--- a/llvm/test/CodeGen/RISCV/div.ll
+++ b/llvm/test/CodeGen/RISCV/div.ll
@@ -457,8 +457,8 @@ define i64 @sdiv64_sext_operands(i32 %a, i32 %b) nounwind {
 ;
 ; RV64IM-LABEL: sdiv64_sext_operands:
 ; RV64IM:       # %bb.0:
-; RV64IM-NEXT:    sext.w a1, a1
 ; RV64IM-NEXT:    sext.w a0, a0
+; RV64IM-NEXT:    sext.w a1, a1
 ; RV64IM-NEXT:    div a0, a0, a1
 ; RV64IM-NEXT:    ret
   %1 = sext i32 %a to i64
diff --git a/llvm/test/CodeGen/RISCV/double-arith.ll b/llvm/test/CodeGen/RISCV/double-arith.ll
index 2b696210a..ad68dca15 100644
--- a/llvm/test/CodeGen/RISCV/double-arith.ll
+++ b/llvm/test/CodeGen/RISCV/double-arith.ll
@@ -473,13 +473,13 @@ define double @fmsub_d(double %a, double %b, double %c) nounwind {
 ;
 ; RV64IFD-LABEL: fmsub_d:
 ; RV64IFD:       # %bb.0:
-; RV64IFD-NEXT:    fmv.d.x ft0, a2
-; RV64IFD-NEXT:    lui a2, %hi(.LCPI15_0)
-; RV64IFD-NEXT:    addi a2, a2, %lo(.LCPI15_0)
-; RV64IFD-NEXT:    fld ft1, 0(a2)
-; RV64IFD-NEXT:    fadd.d ft0, ft0, ft1
+; RV64IFD-NEXT:    lui a3, %hi(.LCPI15_0)
+; RV64IFD-NEXT:    addi a3, a3, %lo(.LCPI15_0)
+; RV64IFD-NEXT:    fld ft0, 0(a3)
 ; RV64IFD-NEXT:    fmv.d.x ft1, a1
 ; RV64IFD-NEXT:    fmv.d.x ft2, a0
+; RV64IFD-NEXT:    fmv.d.x ft3, a2
+; RV64IFD-NEXT:    fadd.d ft0, ft3, ft0
 ; RV64IFD-NEXT:    fmsub.d ft0, ft2, ft1, ft0
 ; RV64IFD-NEXT:    fmv.x.d a0, ft0
 ; RV64IFD-NEXT:    ret
@@ -496,18 +496,18 @@ define double @fnmadd_d(double %a, double %b, double %c) nounwind {
 ; RV32IFD-NEXT:    sw a2, 8(sp)
 ; RV32IFD-NEXT:    sw a3, 12(sp)
 ; RV32IFD-NEXT:    fld ft0, 8(sp)
-; RV32IFD-NEXT:    sw a0, 8(sp)
-; RV32IFD-NEXT:    sw a1, 12(sp)
-; RV32IFD-NEXT:    fld ft1, 8(sp)
 ; RV32IFD-NEXT:    sw a4, 8(sp)
 ; RV32IFD-NEXT:    sw a5, 12(sp)
+; RV32IFD-NEXT:    fld ft1, 8(sp)
+; RV32IFD-NEXT:    sw a0, 8(sp)
+; RV32IFD-NEXT:    sw a1, 12(sp)
 ; RV32IFD-NEXT:    fld ft2, 8(sp)
 ; RV32IFD-NEXT:    lui a0, %hi(.LCPI16_0)
 ; RV32IFD-NEXT:    addi a0, a0, %lo(.LCPI16_0)
 ; RV32IFD-NEXT:    fld ft3, 0(a0)
 ; RV32IFD-NEXT:    fadd.d ft2, ft2, ft3
 ; RV32IFD-NEXT:    fadd.d ft1, ft1, ft3
-; RV32IFD-NEXT:    fnmadd.d ft0, ft1, ft0, ft2
+; RV32IFD-NEXT:    fnmadd.d ft0, ft2, ft0, ft1
 ; RV32IFD-NEXT:    fsd ft0, 8(sp)
 ; RV32IFD-NEXT:    lw a0, 8(sp)
 ; RV32IFD-NEXT:    lw a1, 12(sp)
@@ -516,15 +516,15 @@ define double @fnmadd_d(double %a, double %b, double %c) nounwind {
 ;
 ; RV64IFD-LABEL: fnmadd_d:
 ; RV64IFD:       # %bb.0:
-; RV64IFD-NEXT:    fmv.d.x ft0, a2
-; RV64IFD-NEXT:    lui a2, %hi(.LCPI16_0)
-; RV64IFD-NEXT:    addi a2, a2, %lo(.LCPI16_0)
-; RV64IFD-NEXT:    fld ft1, 0(a2)
-; RV64IFD-NEXT:    fadd.d ft0, ft0, ft1
-; RV64IFD-NEXT:    fmv.d.x ft2, a0
-; RV64IFD-NEXT:    fadd.d ft1, ft2, ft1
-; RV64IFD-NEXT:    fmv.d.x ft2, a1
-; RV64IFD-NEXT:    fnmadd.d ft0, ft1, ft2, ft0
+; RV64IFD-NEXT:    lui a3, %hi(.LCPI16_0)
+; RV64IFD-NEXT:    addi a3, a3, %lo(.LCPI16_0)
+; RV64IFD-NEXT:    fld ft0, 0(a3)
+; RV64IFD-NEXT:    fmv.d.x ft1, a1
+; RV64IFD-NEXT:    fmv.d.x ft2, a2
+; RV64IFD-NEXT:    fmv.d.x ft3, a0
+; RV64IFD-NEXT:    fadd.d ft3, ft3, ft0
+; RV64IFD-NEXT:    fadd.d ft0, ft2, ft0
+; RV64IFD-NEXT:    fnmadd.d ft0, ft3, ft1, ft0
 ; RV64IFD-NEXT:    fmv.x.d a0, ft0
 ; RV64IFD-NEXT:    ret
   %a_ = fadd double 0.0, %a
@@ -561,13 +561,13 @@ define double @fnmsub_d(double %a, double %b, double %c) nounwind {
 ;
 ; RV64IFD-LABEL: fnmsub_d:
 ; RV64IFD:       # %bb.0:
-; RV64IFD-NEXT:    fmv.d.x ft0, a0
-; RV64IFD-NEXT:    lui a0, %hi(.LCPI17_0)
-; RV64IFD-NEXT:    addi a0, a0, %lo(.LCPI17_0)
-; RV64IFD-NEXT:    fld ft1, 0(a0)
-; RV64IFD-NEXT:    fadd.d ft0, ft0, ft1
+; RV64IFD-NEXT:    lui a3, %hi(.LCPI17_0)
+; RV64IFD-NEXT:    addi a3, a3, %lo(.LCPI17_0)
+; RV64IFD-NEXT:    fld ft0, 0(a3)
 ; RV64IFD-NEXT:    fmv.d.x ft1, a2
 ; RV64IFD-NEXT:    fmv.d.x ft2, a1
+; RV64IFD-NEXT:    fmv.d.x ft3, a0
+; RV64IFD-NEXT:    fadd.d ft0, ft3, ft0
 ; RV64IFD-NEXT:    fnmsub.d ft0, ft0, ft2, ft1
 ; RV64IFD-NEXT:    fmv.x.d a0, ft0
 ; RV64IFD-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/double-bitmanip-dagcombines.ll b/llvm/test/CodeGen/RISCV/double-bitmanip-dagcombines.ll
index 626a239e2..f80e140ca 100644
--- a/llvm/test/CodeGen/RISCV/double-bitmanip-dagcombines.ll
+++ b/llvm/test/CodeGen/RISCV/double-bitmanip-dagcombines.ll
@@ -118,9 +118,9 @@ define double @fcopysign_fneg(double %a, double %b) nounwind {
 ;
 ; RV64I-LABEL: fcopysign_fneg:
 ; RV64I:       # %bb.0:
+; RV64I-NEXT:    not a1, a1
 ; RV64I-NEXT:    addi a2, zero, -1
 ; RV64I-NEXT:    slli a2, a2, 63
-; RV64I-NEXT:    not a1, a1
 ; RV64I-NEXT:    and a1, a1, a2
 ; RV64I-NEXT:    addi a2, a2, -1
 ; RV64I-NEXT:    and a0, a0, a2
diff --git a/llvm/test/CodeGen/RISCV/double-br-fcmp.ll b/llvm/test/CodeGen/RISCV/double-br-fcmp.ll
index 1c8b867bd..356f632a3 100644
--- a/llvm/test/CodeGen/RISCV/double-br-fcmp.ll
+++ b/llvm/test/CodeGen/RISCV/double-br-fcmp.ll
@@ -385,11 +385,11 @@ define void @br_fcmp_ord(double %a, double %b) nounwind {
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -16
 ; RV64IFD-NEXT:    sd ra, 8(sp)
-; RV64IFD-NEXT:    fmv.d.x ft0, a1
-; RV64IFD-NEXT:    feq.d a1, ft0, ft0
 ; RV64IFD-NEXT:    fmv.d.x ft0, a0
-; RV64IFD-NEXT:    feq.d a0, ft0, ft0
-; RV64IFD-NEXT:    and a0, a0, a1
+; RV64IFD-NEXT:    fmv.d.x ft1, a1
+; RV64IFD-NEXT:    feq.d a0, ft1, ft1
+; RV64IFD-NEXT:    feq.d a1, ft0, ft0
+; RV64IFD-NEXT:    and a0, a1, a0
 ; RV64IFD-NEXT:    bnez a0, .LBB8_2
 ; RV64IFD-NEXT:  # %bb.1: # %if.else
 ; RV64IFD-NEXT:    ld ra, 8(sp)
@@ -712,11 +712,11 @@ define void @br_fcmp_uno(double %a, double %b) nounwind {
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -16
 ; RV64IFD-NEXT:    sd ra, 8(sp)
-; RV64IFD-NEXT:    fmv.d.x ft0, a1
-; RV64IFD-NEXT:    feq.d a1, ft0, ft0
 ; RV64IFD-NEXT:    fmv.d.x ft0, a0
-; RV64IFD-NEXT:    feq.d a0, ft0, ft0
-; RV64IFD-NEXT:    and a0, a0, a1
+; RV64IFD-NEXT:    fmv.d.x ft1, a1
+; RV64IFD-NEXT:    feq.d a0, ft1, ft1
+; RV64IFD-NEXT:    feq.d a1, ft0, ft0
+; RV64IFD-NEXT:    and a0, a1, a0
 ; RV64IFD-NEXT:    seqz a0, a0
 ; RV64IFD-NEXT:    bnez a0, .LBB15_2
 ; RV64IFD-NEXT:  # %bb.1: # %if.else
diff --git a/llvm/test/CodeGen/RISCV/double-calling-conv.ll b/llvm/test/CodeGen/RISCV/double-calling-conv.ll
index f3cb0feb5..837cb819e 100644
--- a/llvm/test/CodeGen/RISCV/double-calling-conv.ll
+++ b/llvm/test/CodeGen/RISCV/double-calling-conv.ll
@@ -76,16 +76,16 @@ define double @caller_double_split_reg_stack() nounwind {
 ; RV32IFD-NEXT:    addi sp, sp, -16
 ; RV32IFD-NEXT:    sw ra, 12(sp)
 ; RV32IFD-NEXT:    lui a0, 262510
-; RV32IFD-NEXT:    addi a0, a0, 327
-; RV32IFD-NEXT:    sw a0, 0(sp)
+; RV32IFD-NEXT:    addi a2, a0, 327
 ; RV32IFD-NEXT:    lui a0, 262446
 ; RV32IFD-NEXT:    addi a6, a0, 327
 ; RV32IFD-NEXT:    lui a0, 713032
 ; RV32IFD-NEXT:    addi a5, a0, -1311
 ; RV32IFD-NEXT:    addi a0, zero, 1
 ; RV32IFD-NEXT:    addi a1, zero, 2
-; RV32IFD-NEXT:    mv a2, zero
 ; RV32IFD-NEXT:    addi a3, zero, 3
+; RV32IFD-NEXT:    sw a2, 0(sp)
+; RV32IFD-NEXT:    mv a2, zero
 ; RV32IFD-NEXT:    mv a4, zero
 ; RV32IFD-NEXT:    mv a7, a5
 ; RV32IFD-NEXT:    call callee_double_split_reg_stack
@@ -120,20 +120,20 @@ define double @caller_double_stack() nounwind {
 ; RV32IFD-NEXT:    lui a0, 262510
 ; RV32IFD-NEXT:    addi a0, a0, 327
 ; RV32IFD-NEXT:    sw a0, 4(sp)
+; RV32IFD-NEXT:    lui a0, 713032
+; RV32IFD-NEXT:    addi a1, a0, -1311
+; RV32IFD-NEXT:    sw a1, 0(sp)
 ; RV32IFD-NEXT:    lui a0, 262574
 ; RV32IFD-NEXT:    addi a0, a0, 327
 ; RV32IFD-NEXT:    sw a0, 12(sp)
-; RV32IFD-NEXT:    lui a0, 713032
-; RV32IFD-NEXT:    addi a0, a0, -1311
-; RV32IFD-NEXT:    sw a0, 0(sp)
-; RV32IFD-NEXT:    sw a0, 8(sp)
 ; RV32IFD-NEXT:    addi a0, zero, 1
-; RV32IFD-NEXT:    mv a1, zero
 ; RV32IFD-NEXT:    addi a2, zero, 2
-; RV32IFD-NEXT:    mv a3, zero
 ; RV32IFD-NEXT:    addi a4, zero, 3
-; RV32IFD-NEXT:    mv a5, zero
 ; RV32IFD-NEXT:    addi a6, zero, 4
+; RV32IFD-NEXT:    sw a1, 8(sp)
+; RV32IFD-NEXT:    mv a1, zero
+; RV32IFD-NEXT:    mv a3, zero
+; RV32IFD-NEXT:    mv a5, zero
 ; RV32IFD-NEXT:    mv a7, zero
 ; RV32IFD-NEXT:    call callee_double_stack
 ; RV32IFD-NEXT:    lw ra, 28(sp)
diff --git a/llvm/test/CodeGen/RISCV/double-convert.ll b/llvm/test/CodeGen/RISCV/double-convert.ll
index 1d30fdadf..f7abdea68 100644
--- a/llvm/test/CodeGen/RISCV/double-convert.ll
+++ b/llvm/test/CodeGen/RISCV/double-convert.ll
@@ -257,9 +257,9 @@ define double @fmv_d_x(i64 %a, i64 %b) nounwind {
 ;
 ; RV64IFD-LABEL: fmv_d_x:
 ; RV64IFD:       # %bb.0:
-; RV64IFD-NEXT:    fmv.d.x ft0, a1
-; RV64IFD-NEXT:    fmv.d.x ft1, a0
-; RV64IFD-NEXT:    fadd.d ft0, ft1, ft0
+; RV64IFD-NEXT:    fmv.d.x ft0, a0
+; RV64IFD-NEXT:    fmv.d.x ft1, a1
+; RV64IFD-NEXT:    fadd.d ft0, ft0, ft1
 ; RV64IFD-NEXT:    fmv.x.d a0, ft0
 ; RV64IFD-NEXT:    ret
   %1 = bitcast i64 %a to double
diff --git a/llvm/test/CodeGen/RISCV/double-fcmp.ll b/llvm/test/CodeGen/RISCV/double-fcmp.ll
index 0046b59d4..28875226a 100644
--- a/llvm/test/CodeGen/RISCV/double-fcmp.ll
+++ b/llvm/test/CodeGen/RISCV/double-fcmp.ll
@@ -197,11 +197,11 @@ define i32 @fcmp_ord(double %a, double %b) nounwind {
 ;
 ; RV64IFD-LABEL: fcmp_ord:
 ; RV64IFD:       # %bb.0:
-; RV64IFD-NEXT:    fmv.d.x ft0, a1
-; RV64IFD-NEXT:    feq.d a1, ft0, ft0
 ; RV64IFD-NEXT:    fmv.d.x ft0, a0
-; RV64IFD-NEXT:    feq.d a0, ft0, ft0
-; RV64IFD-NEXT:    and a0, a0, a1
+; RV64IFD-NEXT:    fmv.d.x ft1, a1
+; RV64IFD-NEXT:    feq.d a0, ft1, ft1
+; RV64IFD-NEXT:    feq.d a1, ft0, ft0
+; RV64IFD-NEXT:    and a0, a1, a0
 ; RV64IFD-NEXT:    ret
   %1 = fcmp ord double %a, %b
   %2 = zext i1 %1 to i32
@@ -397,11 +397,11 @@ define i32 @fcmp_uno(double %a, double %b) nounwind {
 ;
 ; RV64IFD-LABEL: fcmp_uno:
 ; RV64IFD:       # %bb.0:
-; RV64IFD-NEXT:    fmv.d.x ft0, a1
-; RV64IFD-NEXT:    feq.d a1, ft0, ft0
 ; RV64IFD-NEXT:    fmv.d.x ft0, a0
-; RV64IFD-NEXT:    feq.d a0, ft0, ft0
-; RV64IFD-NEXT:    and a0, a0, a1
+; RV64IFD-NEXT:    fmv.d.x ft1, a1
+; RV64IFD-NEXT:    feq.d a0, ft1, ft1
+; RV64IFD-NEXT:    feq.d a1, ft0, ft0
+; RV64IFD-NEXT:    and a0, a1, a0
 ; RV64IFD-NEXT:    seqz a0, a0
 ; RV64IFD-NEXT:    ret
   %1 = fcmp uno double %a, %b
diff --git a/llvm/test/CodeGen/RISCV/double-imm.ll b/llvm/test/CodeGen/RISCV/double-imm.ll
index 7f8356440..3cd397969 100644
--- a/llvm/test/CodeGen/RISCV/double-imm.ll
+++ b/llvm/test/CodeGen/RISCV/double-imm.ll
@@ -49,11 +49,11 @@ define double @double_imm_op(double %a) nounwind {
 ;
 ; RV64IFD-LABEL: double_imm_op:
 ; RV64IFD:       # %bb.0:
-; RV64IFD-NEXT:    fmv.d.x ft0, a0
-; RV64IFD-NEXT:    lui a0, %hi(.LCPI1_0)
-; RV64IFD-NEXT:    addi a0, a0, %lo(.LCPI1_0)
-; RV64IFD-NEXT:    fld ft1, 0(a0)
-; RV64IFD-NEXT:    fadd.d ft0, ft0, ft1
+; RV64IFD-NEXT:    lui a1, %hi(.LCPI1_0)
+; RV64IFD-NEXT:    addi a1, a1, %lo(.LCPI1_0)
+; RV64IFD-NEXT:    fld ft0, 0(a1)
+; RV64IFD-NEXT:    fmv.d.x ft1, a0
+; RV64IFD-NEXT:    fadd.d ft0, ft1, ft0
 ; RV64IFD-NEXT:    fmv.x.d a0, ft0
 ; RV64IFD-NEXT:    ret
   %1 = fadd double %a, 1.0
diff --git a/llvm/test/CodeGen/RISCV/double-intrinsics.ll b/llvm/test/CodeGen/RISCV/double-intrinsics.ll
index 29327452f..b8873883c 100644
--- a/llvm/test/CodeGen/RISCV/double-intrinsics.ll
+++ b/llvm/test/CodeGen/RISCV/double-intrinsics.ll
@@ -111,28 +111,24 @@ define double @sincos_f64(double %a) nounwind {
 ; RV32IFD-NEXT:    sw ra, 28(sp)
 ; RV32IFD-NEXT:    sw s0, 24(sp)
 ; RV32IFD-NEXT:    sw s1, 20(sp)
-; RV32IFD-NEXT:    sw s2, 16(sp)
-; RV32IFD-NEXT:    sw s3, 12(sp)
 ; RV32IFD-NEXT:    mv s0, a1
 ; RV32IFD-NEXT:    mv s1, a0
 ; RV32IFD-NEXT:    call sin
-; RV32IFD-NEXT:    mv s2, a0
-; RV32IFD-NEXT:    mv s3, a1
+; RV32IFD-NEXT:    sw a0, 8(sp)
+; RV32IFD-NEXT:    sw a1, 12(sp)
+; RV32IFD-NEXT:    fld ft0, 8(sp)
+; RV32IFD-NEXT:    fsd ft0, 0(sp)
 ; RV32IFD-NEXT:    mv a0, s1
 ; RV32IFD-NEXT:    mv a1, s0
 ; RV32IFD-NEXT:    call cos
-; RV32IFD-NEXT:    sw a0, 0(sp)
-; RV32IFD-NEXT:    sw a1, 4(sp)
-; RV32IFD-NEXT:    fld ft0, 0(sp)
-; RV32IFD-NEXT:    sw s2, 0(sp)
-; RV32IFD-NEXT:    sw s3, 4(sp)
+; RV32IFD-NEXT:    sw a0, 8(sp)
+; RV32IFD-NEXT:    sw a1, 12(sp)
+; RV32IFD-NEXT:    fld ft0, 8(sp)
 ; RV32IFD-NEXT:    fld ft1, 0(sp)
 ; RV32IFD-NEXT:    fadd.d ft0, ft1, ft0
-; RV32IFD-NEXT:    fsd ft0, 0(sp)
-; RV32IFD-NEXT:    lw a0, 0(sp)
-; RV32IFD-NEXT:    lw a1, 4(sp)
-; RV32IFD-NEXT:    lw s3, 12(sp)
-; RV32IFD-NEXT:    lw s2, 16(sp)
+; RV32IFD-NEXT:    fsd ft0, 8(sp)
+; RV32IFD-NEXT:    lw a0, 8(sp)
+; RV32IFD-NEXT:    lw a1, 12(sp)
 ; RV32IFD-NEXT:    lw s1, 20(sp)
 ; RV32IFD-NEXT:    lw s0, 24(sp)
 ; RV32IFD-NEXT:    lw ra, 28(sp)
@@ -144,17 +140,16 @@ define double @sincos_f64(double %a) nounwind {
 ; RV64IFD-NEXT:    addi sp, sp, -32
 ; RV64IFD-NEXT:    sd ra, 24(sp)
 ; RV64IFD-NEXT:    sd s0, 16(sp)
-; RV64IFD-NEXT:    sd s1, 8(sp)
 ; RV64IFD-NEXT:    mv s0, a0
 ; RV64IFD-NEXT:    call sin
-; RV64IFD-NEXT:    mv s1, a0
+; RV64IFD-NEXT:    fmv.d.x ft0, a0
+; RV64IFD-NEXT:    fsd ft0, 8(sp)
 ; RV64IFD-NEXT:    mv a0, s0
 ; RV64IFD-NEXT:    call cos
 ; RV64IFD-NEXT:    fmv.d.x ft0, a0
-; RV64IFD-NEXT:    fmv.d.x ft1, s1
+; RV64IFD-NEXT:    fld ft1, 8(sp)
 ; RV64IFD-NEXT:    fadd.d ft0, ft1, ft0
 ; RV64IFD-NEXT:    fmv.x.d a0, ft0
-; RV64IFD-NEXT:    ld s1, 8(sp)
 ; RV64IFD-NEXT:    ld s0, 16(sp)
 ; RV64IFD-NEXT:    ld ra, 24(sp)
 ; RV64IFD-NEXT:    addi sp, sp, 32
@@ -350,17 +345,17 @@ define double @fmuladd_f64(double %a, double %b, double %c) nounwind {
 ; RV32IFD-LABEL: fmuladd_f64:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
+; RV32IFD-NEXT:    sw a4, 8(sp)
+; RV32IFD-NEXT:    sw a5, 12(sp)
+; RV32IFD-NEXT:    fld ft0, 8(sp)
 ; RV32IFD-NEXT:    sw a2, 8(sp)
 ; RV32IFD-NEXT:    sw a3, 12(sp)
-; RV32IFD-NEXT:    fld ft0, 8(sp)
+; RV32IFD-NEXT:    fld ft1, 8(sp)
 ; RV32IFD-NEXT:    sw a0, 8(sp)
 ; RV32IFD-NEXT:    sw a1, 12(sp)
-; RV32IFD-NEXT:    fld ft1, 8(sp)
-; RV32IFD-NEXT:    fmul.d ft0, ft1, ft0
-; RV32IFD-NEXT:    sw a4, 8(sp)
-; RV32IFD-NEXT:    sw a5, 12(sp)
-; RV32IFD-NEXT:    fld ft1, 8(sp)
-; RV32IFD-NEXT:    fadd.d ft0, ft0, ft1
+; RV32IFD-NEXT:    fld ft2, 8(sp)
+; RV32IFD-NEXT:    fmul.d ft1, ft2, ft1
+; RV32IFD-NEXT:    fadd.d ft0, ft1, ft0
 ; RV32IFD-NEXT:    fsd ft0, 8(sp)
 ; RV32IFD-NEXT:    lw a0, 8(sp)
 ; RV32IFD-NEXT:    lw a1, 12(sp)
@@ -369,11 +364,11 @@ define double @fmuladd_f64(double %a, double %b, double %c) nounwind {
 ;
 ; RV64IFD-LABEL: fmuladd_f64:
 ; RV64IFD:       # %bb.0:
-; RV64IFD-NEXT:    fmv.d.x ft0, a1
-; RV64IFD-NEXT:    fmv.d.x ft1, a0
-; RV64IFD-NEXT:    fmul.d ft0, ft1, ft0
-; RV64IFD-NEXT:    fmv.d.x ft1, a2
-; RV64IFD-NEXT:    fadd.d ft0, ft0, ft1
+; RV64IFD-NEXT:    fmv.d.x ft0, a2
+; RV64IFD-NEXT:    fmv.d.x ft1, a1
+; RV64IFD-NEXT:    fmv.d.x ft2, a0
+; RV64IFD-NEXT:    fmul.d ft1, ft2, ft1
+; RV64IFD-NEXT:    fadd.d ft0, ft1, ft0
 ; RV64IFD-NEXT:    fmv.x.d a0, ft0
 ; RV64IFD-NEXT:    ret
   %1 = call double @llvm.fmuladd.f64(double %a, double %b, double %c)
diff --git a/llvm/test/CodeGen/RISCV/double-mem.ll b/llvm/test/CodeGen/RISCV/double-mem.ll
index 76be198ff..ad6a3afc1 100644
--- a/llvm/test/CodeGen/RISCV/double-mem.ll
+++ b/llvm/test/CodeGen/RISCV/double-mem.ll
@@ -8,9 +8,9 @@ define double @fld(double *%a) nounwind {
 ; RV32IFD-LABEL: fld:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    fld ft0, 24(a0)
-; RV32IFD-NEXT:    fld ft1, 0(a0)
-; RV32IFD-NEXT:    fadd.d ft0, ft1, ft0
+; RV32IFD-NEXT:    fld ft0, 0(a0)
+; RV32IFD-NEXT:    fld ft1, 24(a0)
+; RV32IFD-NEXT:    fadd.d ft0, ft0, ft1
 ; RV32IFD-NEXT:    fsd ft0, 8(sp)
 ; RV32IFD-NEXT:    lw a0, 8(sp)
 ; RV32IFD-NEXT:    lw a1, 12(sp)
@@ -19,9 +19,9 @@ define double @fld(double *%a) nounwind {
 ;
 ; RV64IFD-LABEL: fld:
 ; RV64IFD:       # %bb.0:
-; RV64IFD-NEXT:    fld ft0, 24(a0)
-; RV64IFD-NEXT:    fld ft1, 0(a0)
-; RV64IFD-NEXT:    fadd.d ft0, ft1, ft0
+; RV64IFD-NEXT:    fld ft0, 0(a0)
+; RV64IFD-NEXT:    fld ft1, 24(a0)
+; RV64IFD-NEXT:    fadd.d ft0, ft0, ft1
 ; RV64IFD-NEXT:    fmv.x.d a0, ft0
 ; RV64IFD-NEXT:    ret
   %1 = load double, double* %a
@@ -44,8 +44,8 @@ define void @fsd(double *%a, double %b, double %c) nounwind {
 ; RV32IFD-NEXT:    sw a2, 12(sp)
 ; RV32IFD-NEXT:    fld ft1, 8(sp)
 ; RV32IFD-NEXT:    fadd.d ft0, ft1, ft0
-; RV32IFD-NEXT:    fsd ft0, 64(a0)
 ; RV32IFD-NEXT:    fsd ft0, 0(a0)
+; RV32IFD-NEXT:    fsd ft0, 64(a0)
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
@@ -54,8 +54,8 @@ define void @fsd(double *%a, double %b, double %c) nounwind {
 ; RV64IFD-NEXT:    fmv.d.x ft0, a2
 ; RV64IFD-NEXT:    fmv.d.x ft1, a1
 ; RV64IFD-NEXT:    fadd.d ft0, ft1, ft0
-; RV64IFD-NEXT:    fsd ft0, 64(a0)
 ; RV64IFD-NEXT:    fsd ft0, 0(a0)
+; RV64IFD-NEXT:    fsd ft0, 64(a0)
 ; RV64IFD-NEXT:    ret
 ; Use %b and %c in an FP op to ensure floating point registers are used, even
 ; for the soft float ABI
@@ -100,10 +100,10 @@ define double @fld_fsd_global(double %a, double %b) nounwind {
 ; RV64IFD-NEXT:    lui a0, %hi(G)
 ; RV64IFD-NEXT:    fld ft1, %lo(G)(a0)
 ; RV64IFD-NEXT:    fsd ft0, %lo(G)(a0)
-; RV64IFD-NEXT:    addi a0, a0, %lo(G)
-; RV64IFD-NEXT:    fld ft1, 72(a0)
-; RV64IFD-NEXT:    fsd ft0, 72(a0)
+; RV64IFD-NEXT:    addi a1, a0, %lo(G)
+; RV64IFD-NEXT:    fld ft1, 72(a1)
 ; RV64IFD-NEXT:    fmv.x.d a0, ft0
+; RV64IFD-NEXT:    fsd ft0, 72(a1)
 ; RV64IFD-NEXT:    ret
 ; Use %a and %b in an FP op to ensure floating point registers are used, even
 ; for the soft float ABI
@@ -136,14 +136,14 @@ define double @fld_fsd_constant(double %a) nounwind {
 ;
 ; RV64IFD-LABEL: fld_fsd_constant:
 ; RV64IFD:       # %bb.0:
-; RV64IFD-NEXT:    fmv.d.x ft0, a0
-; RV64IFD-NEXT:    lui a0, 56
-; RV64IFD-NEXT:    addiw a0, a0, -1353
-; RV64IFD-NEXT:    slli a0, a0, 14
-; RV64IFD-NEXT:    fld ft1, -273(a0)
-; RV64IFD-NEXT:    fadd.d ft0, ft0, ft1
-; RV64IFD-NEXT:    fsd ft0, -273(a0)
+; RV64IFD-NEXT:    lui a1, 56
+; RV64IFD-NEXT:    addiw a1, a1, -1353
+; RV64IFD-NEXT:    slli a1, a1, 14
+; RV64IFD-NEXT:    fld ft0, -273(a1)
+; RV64IFD-NEXT:    fmv.d.x ft1, a0
+; RV64IFD-NEXT:    fadd.d ft0, ft1, ft0
 ; RV64IFD-NEXT:    fmv.x.d a0, ft0
+; RV64IFD-NEXT:    fsd ft0, -273(a1)
 ; RV64IFD-NEXT:    ret
   %1 = inttoptr i32 3735928559 to double*
   %2 = load volatile double, double* %1
@@ -159,22 +159,18 @@ define double @fld_stack(double %a) nounwind {
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -32
 ; RV32IFD-NEXT:    sw ra, 28(sp)
-; RV32IFD-NEXT:    sw s0, 24(sp)
-; RV32IFD-NEXT:    sw s1, 20(sp)
-; RV32IFD-NEXT:    mv s0, a1
-; RV32IFD-NEXT:    mv s1, a0
-; RV32IFD-NEXT:    addi a0, sp, 8
-; RV32IFD-NEXT:    call notdead
-; RV32IFD-NEXT:    sw s1, 0(sp)
-; RV32IFD-NEXT:    sw s0, 4(sp)
-; RV32IFD-NEXT:    fld ft0, 0(sp)
-; RV32IFD-NEXT:    fld ft1, 8(sp)
-; RV32IFD-NEXT:    fadd.d ft0, ft1, ft0
+; RV32IFD-NEXT:    sw a0, 8(sp)
+; RV32IFD-NEXT:    sw a1, 12(sp)
+; RV32IFD-NEXT:    fld ft0, 8(sp)
 ; RV32IFD-NEXT:    fsd ft0, 0(sp)
-; RV32IFD-NEXT:    lw a0, 0(sp)
-; RV32IFD-NEXT:    lw a1, 4(sp)
-; RV32IFD-NEXT:    lw s1, 20(sp)
-; RV32IFD-NEXT:    lw s0, 24(sp)
+; RV32IFD-NEXT:    addi a0, sp, 16
+; RV32IFD-NEXT:    call notdead
+; RV32IFD-NEXT:    fld ft0, 16(sp)
+; RV32IFD-NEXT:    fld ft1, 0(sp)
+; RV32IFD-NEXT:    fadd.d ft0, ft0, ft1
+; RV32IFD-NEXT:    fsd ft0, 8(sp)
+; RV32IFD-NEXT:    lw a0, 8(sp)
+; RV32IFD-NEXT:    lw a1, 12(sp)
 ; RV32IFD-NEXT:    lw ra, 28(sp)
 ; RV32IFD-NEXT:    addi sp, sp, 32
 ; RV32IFD-NEXT:    ret
@@ -183,15 +179,14 @@ define double @fld_stack(double %a) nounwind {
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -32
 ; RV64IFD-NEXT:    sd ra, 24(sp)
-; RV64IFD-NEXT:    sd s0, 16(sp)
-; RV64IFD-NEXT:    mv s0, a0
-; RV64IFD-NEXT:    addi a0, sp, 8
+; RV64IFD-NEXT:    fmv.d.x ft0, a0
+; RV64IFD-NEXT:    fsd ft0, 8(sp)
+; RV64IFD-NEXT:    addi a0, sp, 16
 ; RV64IFD-NEXT:    call notdead
-; RV64IFD-NEXT:    fmv.d.x ft0, s0
+; RV64IFD-NEXT:    fld ft0, 16(sp)
 ; RV64IFD-NEXT:    fld ft1, 8(sp)
-; RV64IFD-NEXT:    fadd.d ft0, ft1, ft0
+; RV64IFD-NEXT:    fadd.d ft0, ft0, ft1
 ; RV64IFD-NEXT:    fmv.x.d a0, ft0
-; RV64IFD-NEXT:    ld s0, 16(sp)
 ; RV64IFD-NEXT:    ld ra, 24(sp)
 ; RV64IFD-NEXT:    addi sp, sp, 32
 ; RV64IFD-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/double-previous-failure.ll b/llvm/test/CodeGen/RISCV/double-previous-failure.ll
index 168c6c2e6..eb58ba4c6 100644
--- a/llvm/test/CodeGen/RISCV/double-previous-failure.ll
+++ b/llvm/test/CodeGen/RISCV/double-previous-failure.ll
@@ -17,8 +17,8 @@ define i32 @main() nounwind {
 ; RV32IFD:       # %bb.0: # %entry
 ; RV32IFD-NEXT:    addi sp, sp, -16
 ; RV32IFD-NEXT:    sw ra, 12(sp)
-; RV32IFD-NEXT:    mv a0, zero
 ; RV32IFD-NEXT:    lui a1, 262144
+; RV32IFD-NEXT:    mv a0, zero
 ; RV32IFD-NEXT:    call test
 ; RV32IFD-NEXT:    sw a0, 0(sp)
 ; RV32IFD-NEXT:    sw a1, 4(sp)
diff --git a/llvm/test/CodeGen/RISCV/double-select-fcmp.ll b/llvm/test/CodeGen/RISCV/double-select-fcmp.ll
index 321adce3d..12789c053 100644
--- a/llvm/test/CodeGen/RISCV/double-select-fcmp.ll
+++ b/llvm/test/CodeGen/RISCV/double-select-fcmp.ll
@@ -298,23 +298,23 @@ define double @select_fcmp_ueq(double %a, double %b) nounwind {
 ; RV32IFD-LABEL: select_fcmp_ueq:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    sw a0, 8(sp)
-; RV32IFD-NEXT:    sw a1, 12(sp)
-; RV32IFD-NEXT:    fld ft0, 8(sp)
 ; RV32IFD-NEXT:    sw a2, 8(sp)
 ; RV32IFD-NEXT:    sw a3, 12(sp)
+; RV32IFD-NEXT:    fld ft0, 8(sp)
+; RV32IFD-NEXT:    sw a0, 8(sp)
+; RV32IFD-NEXT:    sw a1, 12(sp)
 ; RV32IFD-NEXT:    fld ft1, 8(sp)
-; RV32IFD-NEXT:    feq.d a0, ft1, ft1
+; RV32IFD-NEXT:    feq.d a0, ft1, ft0
 ; RV32IFD-NEXT:    feq.d a1, ft0, ft0
-; RV32IFD-NEXT:    and a0, a1, a0
-; RV32IFD-NEXT:    seqz a0, a0
-; RV32IFD-NEXT:    feq.d a1, ft0, ft1
-; RV32IFD-NEXT:    or a0, a1, a0
+; RV32IFD-NEXT:    feq.d a2, ft1, ft1
+; RV32IFD-NEXT:    and a1, a2, a1
+; RV32IFD-NEXT:    seqz a1, a1
+; RV32IFD-NEXT:    or a0, a0, a1
 ; RV32IFD-NEXT:    bnez a0, .LBB8_2
 ; RV32IFD-NEXT:  # %bb.1:
-; RV32IFD-NEXT:    fmv.d ft0, ft1
+; RV32IFD-NEXT:    fmv.d ft1, ft0
 ; RV32IFD-NEXT:  .LBB8_2:
-; RV32IFD-NEXT:    fsd ft0, 8(sp)
+; RV32IFD-NEXT:    fsd ft1, 8(sp)
 ; RV32IFD-NEXT:    lw a0, 8(sp)
 ; RV32IFD-NEXT:    lw a1, 12(sp)
 ; RV32IFD-NEXT:    addi sp, sp, 16
@@ -322,14 +322,14 @@ define double @select_fcmp_ueq(double %a, double %b) nounwind {
 ;
 ; RV64IFD-LABEL: select_fcmp_ueq:
 ; RV64IFD:       # %bb.0:
-; RV64IFD-NEXT:    fmv.d.x ft0, a0
 ; RV64IFD-NEXT:    fmv.d.x ft1, a1
-; RV64IFD-NEXT:    feq.d a0, ft1, ft1
-; RV64IFD-NEXT:    feq.d a1, ft0, ft0
-; RV64IFD-NEXT:    and a0, a1, a0
-; RV64IFD-NEXT:    seqz a0, a0
-; RV64IFD-NEXT:    feq.d a1, ft0, ft1
-; RV64IFD-NEXT:    or a0, a1, a0
+; RV64IFD-NEXT:    fmv.d.x ft0, a0
+; RV64IFD-NEXT:    feq.d a0, ft0, ft1
+; RV64IFD-NEXT:    feq.d a1, ft1, ft1
+; RV64IFD-NEXT:    feq.d a2, ft0, ft0
+; RV64IFD-NEXT:    and a1, a2, a1
+; RV64IFD-NEXT:    seqz a1, a1
+; RV64IFD-NEXT:    or a0, a0, a1
 ; RV64IFD-NEXT:    bnez a0, .LBB8_2
 ; RV64IFD-NEXT:  # %bb.1:
 ; RV64IFD-NEXT:    fmv.d ft0, ft1
@@ -604,12 +604,12 @@ define i32 @i32_select_fcmp_oeq(double %a, double %b, i32 %c, i32 %d) nounwind {
 ; RV32IFD-NEXT:    sw a0, 8(sp)
 ; RV32IFD-NEXT:    sw a1, 12(sp)
 ; RV32IFD-NEXT:    fld ft1, 8(sp)
-; RV32IFD-NEXT:    feq.d a0, ft1, ft0
-; RV32IFD-NEXT:    bnez a0, .LBB16_2
+; RV32IFD-NEXT:    feq.d a1, ft1, ft0
+; RV32IFD-NEXT:    mv a0, a4
+; RV32IFD-NEXT:    bnez a1, .LBB16_2
 ; RV32IFD-NEXT:  # %bb.1:
-; RV32IFD-NEXT:    mv a4, a5
+; RV32IFD-NEXT:    mv a0, a5
 ; RV32IFD-NEXT:  .LBB16_2:
-; RV32IFD-NEXT:    mv a0, a4
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
@@ -617,12 +617,12 @@ define i32 @i32_select_fcmp_oeq(double %a, double %b, i32 %c, i32 %d) nounwind {
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    fmv.d.x ft0, a1
 ; RV64IFD-NEXT:    fmv.d.x ft1, a0
-; RV64IFD-NEXT:    feq.d a0, ft1, ft0
-; RV64IFD-NEXT:    bnez a0, .LBB16_2
+; RV64IFD-NEXT:    feq.d a1, ft1, ft0
+; RV64IFD-NEXT:    mv a0, a2
+; RV64IFD-NEXT:    bnez a1, .LBB16_2
 ; RV64IFD-NEXT:  # %bb.1:
-; RV64IFD-NEXT:    mv a2, a3
+; RV64IFD-NEXT:    mv a0, a3
 ; RV64IFD-NEXT:  .LBB16_2:
-; RV64IFD-NEXT:    mv a0, a2
 ; RV64IFD-NEXT:    ret
   %1 = fcmp oeq double %a, %b
   %2 = select i1 %1, i32 %c, i32 %d
diff --git a/llvm/test/CodeGen/RISCV/double-stack-spill-restore.ll b/llvm/test/CodeGen/RISCV/double-stack-spill-restore.ll
index 28a9e12da..a595cd8d9 100644
--- a/llvm/test/CodeGen/RISCV/double-stack-spill-restore.ll
+++ b/llvm/test/CodeGen/RISCV/double-stack-spill-restore.ll
@@ -37,10 +37,10 @@ define double @func(double %d, i32 %n) nounwind {
 ; RV64IFD:       # %bb.0: # %entry
 ; RV64IFD-NEXT:    addi sp, sp, -16
 ; RV64IFD-NEXT:    sd ra, 8(sp)
+; RV64IFD-NEXT:    slli a2, a1, 32
+; RV64IFD-NEXT:    srli a2, a2, 32
 ; RV64IFD-NEXT:    fmv.d.x ft0, a0
-; RV64IFD-NEXT:    slli a0, a1, 32
-; RV64IFD-NEXT:    srli a0, a0, 32
-; RV64IFD-NEXT:    beqz a0, .LBB0_2
+; RV64IFD-NEXT:    beqz a2, .LBB0_2
 ; RV64IFD-NEXT:  # %bb.1: # %if.else
 ; RV64IFD-NEXT:    addi a1, a1, -1
 ; RV64IFD-NEXT:    fmv.x.d a0, ft0
diff --git a/llvm/test/CodeGen/RISCV/exception-pointer-register.ll b/llvm/test/CodeGen/RISCV/exception-pointer-register.ll
index b341268ff..fafd506bc 100644
--- a/llvm/test/CodeGen/RISCV/exception-pointer-register.ll
+++ b/llvm/test/CodeGen/RISCV/exception-pointer-register.ll
@@ -40,11 +40,7 @@ define void @caller(i1* %p) personality i8* bitcast (i32 (...)* @__gxx_personali
 ; RV32I-NEXT:    lw s1, 4(sp)
 ; RV32I-NEXT:    lw s0, 8(sp)
 ; RV32I-NEXT:    lw ra, 12(sp)
-; RV32I-NEXT:    .cfi_restore ra
-; RV32I-NEXT:    .cfi_restore s0
-; RV32I-NEXT:    .cfi_restore s1
 ; RV32I-NEXT:    addi sp, sp, 16
-; RV32I-NEXT:    .cfi_def_cfa_offset 0
 ; RV32I-NEXT:    ret
 ; RV32I-NEXT:  .LBB0_4: # %lpad
 ; RV32I-NEXT:  .Ltmp4:
@@ -81,11 +77,7 @@ define void @caller(i1* %p) personality i8* bitcast (i32 (...)* @__gxx_personali
 ; RV64I-NEXT:    ld s1, 8(sp)
 ; RV64I-NEXT:    ld s0, 16(sp)
 ; RV64I-NEXT:    ld ra, 24(sp)
-; RV64I-NEXT:    .cfi_restore ra
-; RV64I-NEXT:    .cfi_restore s0
-; RV64I-NEXT:    .cfi_restore s1
 ; RV64I-NEXT:    addi sp, sp, 32
-; RV64I-NEXT:    .cfi_def_cfa_offset 0
 ; RV64I-NEXT:    ret
 ; RV64I-NEXT:  .LBB0_4: # %lpad
 ; RV64I-NEXT:  .Ltmp4:
@@ -119,12 +111,10 @@ end2:
 define internal void @callee(i1* %p) {
 ; RV32I-LABEL: callee:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    .cfi_def_cfa_offset 0
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: callee:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    .cfi_def_cfa_offset 0
 ; RV64I-NEXT:    ret
   ret void
 }
diff --git a/llvm/test/CodeGen/RISCV/fastcc-float.ll b/llvm/test/CodeGen/RISCV/fastcc-float.ll
new file mode 100644
index 000000000..a70c26bf6
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/fastcc-float.ll
@@ -0,0 +1,71 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+f,+d -verify-machineinstrs < %s \
+; RUN:     | FileCheck %s
+
+define fastcc float @callee(<32 x float> %A) nounwind {
+; CHECK-LABEL: callee:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fmv.x.w a0, fa0
+; CHECK-NEXT:    ret
+	%B = extractelement <32 x float> %A, i32 0
+	ret float %B
+}
+
+; With the fastcc, arguments will be passed by fa0-fa7 and ft0-ft11.
+; The rest will be pushed on the stack.
+define float @caller(<32 x float> %A) nounwind {
+; CHECK-LABEL: caller:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -64
+; CHECK-NEXT:    sw ra, 60(sp)
+; CHECK-NEXT:    flw fa0, 0(a0)
+; CHECK-NEXT:    flw fa1, 4(a0)
+; CHECK-NEXT:    flw fa2, 8(a0)
+; CHECK-NEXT:    flw fa3, 12(a0)
+; CHECK-NEXT:    flw fa4, 16(a0)
+; CHECK-NEXT:    flw fa5, 20(a0)
+; CHECK-NEXT:    flw fa6, 24(a0)
+; CHECK-NEXT:    flw fa7, 28(a0)
+; CHECK-NEXT:    flw ft0, 32(a0)
+; CHECK-NEXT:    flw ft1, 36(a0)
+; CHECK-NEXT:    flw ft2, 40(a0)
+; CHECK-NEXT:    flw ft3, 44(a0)
+; CHECK-NEXT:    flw ft4, 48(a0)
+; CHECK-NEXT:    flw ft5, 52(a0)
+; CHECK-NEXT:    flw ft6, 56(a0)
+; CHECK-NEXT:    flw ft7, 60(a0)
+; CHECK-NEXT:    flw ft8, 64(a0)
+; CHECK-NEXT:    flw ft9, 68(a0)
+; CHECK-NEXT:    flw ft10, 72(a0)
+; CHECK-NEXT:    flw ft11, 76(a0)
+; CHECK-NEXT:    flw fs0, 80(a0)
+; CHECK-NEXT:    flw fs1, 84(a0)
+; CHECK-NEXT:    flw fs2, 88(a0)
+; CHECK-NEXT:    flw fs3, 92(a0)
+; CHECK-NEXT:    flw fs4, 96(a0)
+; CHECK-NEXT:    flw fs5, 100(a0)
+; CHECK-NEXT:    flw fs6, 104(a0)
+; CHECK-NEXT:    flw fs7, 108(a0)
+; CHECK-NEXT:    flw fs8, 112(a0)
+; CHECK-NEXT:    flw fs9, 116(a0)
+; CHECK-NEXT:    flw fs10, 120(a0)
+; CHECK-NEXT:    flw fs11, 124(a0)
+; CHECK-NEXT:    fsw fs11, 44(sp)
+; CHECK-NEXT:    fsw fs10, 40(sp)
+; CHECK-NEXT:    fsw fs9, 36(sp)
+; CHECK-NEXT:    fsw fs8, 32(sp)
+; CHECK-NEXT:    fsw fs7, 28(sp)
+; CHECK-NEXT:    fsw fs6, 24(sp)
+; CHECK-NEXT:    fsw fs5, 20(sp)
+; CHECK-NEXT:    fsw fs4, 16(sp)
+; CHECK-NEXT:    fsw fs3, 12(sp)
+; CHECK-NEXT:    fsw fs2, 8(sp)
+; CHECK-NEXT:    fsw fs1, 4(sp)
+; CHECK-NEXT:    fsw fs0, 0(sp)
+; CHECK-NEXT:    call callee
+; CHECK-NEXT:    lw ra, 60(sp)
+; CHECK-NEXT:    addi sp, sp, 64
+; CHECK-NEXT:    ret
+	%C = call fastcc float @callee(<32 x float> %A)
+	ret float %C
+}
diff --git a/llvm/test/CodeGen/RISCV/fastcc-int.ll b/llvm/test/CodeGen/RISCV/fastcc-int.ll
new file mode 100644
index 000000000..a48639d66
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/fastcc-int.ll
@@ -0,0 +1,85 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV32 %s
+; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64 %s
+
+define fastcc i32 @callee(<16 x i32> %A) nounwind {
+; RV32-LABEL: callee:
+; RV32:       # %bb.0:
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: callee:
+; RV64:       # %bb.0:
+; RV64-NEXT:    ret
+	%B = extractelement <16 x i32> %A, i32 0
+	ret i32 %B
+}
+
+; With the fastcc, arguments will be passed by a0-a7 and t2-t6.
+; The rest will be pushed on the stack.
+define i32 @caller(<16 x i32> %A) nounwind {
+; RV32-LABEL: caller:
+; RV32:       # %bb.0:
+; RV32-NEXT:    addi sp, sp, -32
+; RV32-NEXT:    sw ra, 28(sp)
+; RV32-NEXT:    sw s0, 24(sp)
+; RV32-NEXT:    lw t0, 0(a0)
+; RV32-NEXT:    lw a1, 4(a0)
+; RV32-NEXT:    lw a2, 8(a0)
+; RV32-NEXT:    lw a3, 12(a0)
+; RV32-NEXT:    lw a4, 16(a0)
+; RV32-NEXT:    lw a5, 20(a0)
+; RV32-NEXT:    lw a6, 24(a0)
+; RV32-NEXT:    lw a7, 28(a0)
+; RV32-NEXT:    lw t2, 32(a0)
+; RV32-NEXT:    lw t3, 36(a0)
+; RV32-NEXT:    lw t4, 40(a0)
+; RV32-NEXT:    lw t5, 44(a0)
+; RV32-NEXT:    lw t6, 48(a0)
+; RV32-NEXT:    lw t1, 52(a0)
+; RV32-NEXT:    lw s0, 56(a0)
+; RV32-NEXT:    lw a0, 60(a0)
+; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw s0, 4(sp)
+; RV32-NEXT:    sw t1, 0(sp)
+; RV32-NEXT:    mv a0, t0
+; RV32-NEXT:    call callee
+; RV32-NEXT:    lw s0, 24(sp)
+; RV32-NEXT:    lw ra, 28(sp)
+; RV32-NEXT:    addi sp, sp, 32
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: caller:
+; RV64:       # %bb.0:
+; RV64-NEXT:    addi sp, sp, -48
+; RV64-NEXT:    sd ra, 40(sp)
+; RV64-NEXT:    sd s0, 32(sp)
+; RV64-NEXT:    ld t0, 0(a0)
+; RV64-NEXT:    ld a1, 8(a0)
+; RV64-NEXT:    ld a2, 16(a0)
+; RV64-NEXT:    ld a3, 24(a0)
+; RV64-NEXT:    ld a4, 32(a0)
+; RV64-NEXT:    ld a5, 40(a0)
+; RV64-NEXT:    ld a6, 48(a0)
+; RV64-NEXT:    ld a7, 56(a0)
+; RV64-NEXT:    ld t2, 64(a0)
+; RV64-NEXT:    ld t3, 72(a0)
+; RV64-NEXT:    ld t4, 80(a0)
+; RV64-NEXT:    ld t5, 88(a0)
+; RV64-NEXT:    ld t6, 96(a0)
+; RV64-NEXT:    ld t1, 104(a0)
+; RV64-NEXT:    ld s0, 112(a0)
+; RV64-NEXT:    ld a0, 120(a0)
+; RV64-NEXT:    sd a0, 16(sp)
+; RV64-NEXT:    sd s0, 8(sp)
+; RV64-NEXT:    sd t1, 0(sp)
+; RV64-NEXT:    mv a0, t0
+; RV64-NEXT:    call callee
+; RV64-NEXT:    ld s0, 32(sp)
+; RV64-NEXT:    ld ra, 40(sp)
+; RV64-NEXT:    addi sp, sp, 48
+; RV64-NEXT:    ret
+	%C = call fastcc i32 @callee(<16 x i32> %A)
+	ret i32 %C
+}
diff --git a/llvm/test/CodeGen/RISCV/fixups-relax-diff.ll b/llvm/test/CodeGen/RISCV/fixups-relax-diff.ll
index e52c960b9..0fa7998da 100644
--- a/llvm/test/CodeGen/RISCV/fixups-relax-diff.ll
+++ b/llvm/test/CodeGen/RISCV/fixups-relax-diff.ll
@@ -14,7 +14,7 @@ entry:
   store i32 0, i32* %retval, align 4
   ; RELAX: R_RISCV_ADD64 b
   ; RELAX: R_RISCV_SUB64 a
-  ; NORELAX-NOT: R_RISCV
+  ; NORELAX-NOT: R_RISCV_ADD
   call void asm sideeffect "a:\0Ab:\0A.dword b-a", ""()
   ret i32 0
 }
diff --git a/llvm/test/CodeGen/RISCV/float-arith.ll b/llvm/test/CodeGen/RISCV/float-arith.ll
index a668b7e49..5244a69a6 100644
--- a/llvm/test/CodeGen/RISCV/float-arith.ll
+++ b/llvm/test/CodeGen/RISCV/float-arith.ll
@@ -339,26 +339,26 @@ define float @fmadd_s(float %a, float %b, float %c) nounwind {
 define float @fmsub_s(float %a, float %b, float %c) nounwind {
 ; RV32IF-LABEL: fmsub_s:
 ; RV32IF:       # %bb.0:
-; RV32IF-NEXT:    fmv.w.x ft0, a2
-; RV32IF-NEXT:    lui a2, %hi(.LCPI15_0)
-; RV32IF-NEXT:    addi a2, a2, %lo(.LCPI15_0)
-; RV32IF-NEXT:    flw ft1, 0(a2)
-; RV32IF-NEXT:    fadd.s ft0, ft0, ft1
+; RV32IF-NEXT:    lui a3, %hi(.LCPI15_0)
+; RV32IF-NEXT:    addi a3, a3, %lo(.LCPI15_0)
+; RV32IF-NEXT:    flw ft0, 0(a3)
 ; RV32IF-NEXT:    fmv.w.x ft1, a1
 ; RV32IF-NEXT:    fmv.w.x ft2, a0
+; RV32IF-NEXT:    fmv.w.x ft3, a2
+; RV32IF-NEXT:    fadd.s ft0, ft3, ft0
 ; RV32IF-NEXT:    fmsub.s ft0, ft2, ft1, ft0
 ; RV32IF-NEXT:    fmv.x.w a0, ft0
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: fmsub_s:
 ; RV64IF:       # %bb.0:
-; RV64IF-NEXT:    fmv.w.x ft0, a2
-; RV64IF-NEXT:    lui a2, %hi(.LCPI15_0)
-; RV64IF-NEXT:    addi a2, a2, %lo(.LCPI15_0)
-; RV64IF-NEXT:    flw ft1, 0(a2)
-; RV64IF-NEXT:    fadd.s ft0, ft0, ft1
+; RV64IF-NEXT:    lui a3, %hi(.LCPI15_0)
+; RV64IF-NEXT:    addi a3, a3, %lo(.LCPI15_0)
+; RV64IF-NEXT:    flw ft0, 0(a3)
 ; RV64IF-NEXT:    fmv.w.x ft1, a1
 ; RV64IF-NEXT:    fmv.w.x ft2, a0
+; RV64IF-NEXT:    fmv.w.x ft3, a2
+; RV64IF-NEXT:    fadd.s ft0, ft3, ft0
 ; RV64IF-NEXT:    fmsub.s ft0, ft2, ft1, ft0
 ; RV64IF-NEXT:    fmv.x.w a0, ft0
 ; RV64IF-NEXT:    ret
@@ -371,29 +371,29 @@ define float @fmsub_s(float %a, float %b, float %c) nounwind {
 define float @fnmadd_s(float %a, float %b, float %c) nounwind {
 ; RV32IF-LABEL: fnmadd_s:
 ; RV32IF:       # %bb.0:
-; RV32IF-NEXT:    fmv.w.x ft0, a2
-; RV32IF-NEXT:    lui a2, %hi(.LCPI16_0)
-; RV32IF-NEXT:    addi a2, a2, %lo(.LCPI16_0)
-; RV32IF-NEXT:    flw ft1, 0(a2)
-; RV32IF-NEXT:    fadd.s ft0, ft0, ft1
-; RV32IF-NEXT:    fmv.w.x ft2, a0
-; RV32IF-NEXT:    fadd.s ft1, ft2, ft1
-; RV32IF-NEXT:    fmv.w.x ft2, a1
-; RV32IF-NEXT:    fnmadd.s ft0, ft1, ft2, ft0
+; RV32IF-NEXT:    lui a3, %hi(.LCPI16_0)
+; RV32IF-NEXT:    addi a3, a3, %lo(.LCPI16_0)
+; RV32IF-NEXT:    flw ft0, 0(a3)
+; RV32IF-NEXT:    fmv.w.x ft1, a1
+; RV32IF-NEXT:    fmv.w.x ft2, a2
+; RV32IF-NEXT:    fmv.w.x ft3, a0
+; RV32IF-NEXT:    fadd.s ft3, ft3, ft0
+; RV32IF-NEXT:    fadd.s ft0, ft2, ft0
+; RV32IF-NEXT:    fnmadd.s ft0, ft3, ft1, ft0
 ; RV32IF-NEXT:    fmv.x.w a0, ft0
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: fnmadd_s:
 ; RV64IF:       # %bb.0:
-; RV64IF-NEXT:    fmv.w.x ft0, a2
-; RV64IF-NEXT:    lui a2, %hi(.LCPI16_0)
-; RV64IF-NEXT:    addi a2, a2, %lo(.LCPI16_0)
-; RV64IF-NEXT:    flw ft1, 0(a2)
-; RV64IF-NEXT:    fadd.s ft0, ft0, ft1
-; RV64IF-NEXT:    fmv.w.x ft2, a0
-; RV64IF-NEXT:    fadd.s ft1, ft2, ft1
-; RV64IF-NEXT:    fmv.w.x ft2, a1
-; RV64IF-NEXT:    fnmadd.s ft0, ft1, ft2, ft0
+; RV64IF-NEXT:    lui a3, %hi(.LCPI16_0)
+; RV64IF-NEXT:    addi a3, a3, %lo(.LCPI16_0)
+; RV64IF-NEXT:    flw ft0, 0(a3)
+; RV64IF-NEXT:    fmv.w.x ft1, a1
+; RV64IF-NEXT:    fmv.w.x ft2, a2
+; RV64IF-NEXT:    fmv.w.x ft3, a0
+; RV64IF-NEXT:    fadd.s ft3, ft3, ft0
+; RV64IF-NEXT:    fadd.s ft0, ft2, ft0
+; RV64IF-NEXT:    fnmadd.s ft0, ft3, ft1, ft0
 ; RV64IF-NEXT:    fmv.x.w a0, ft0
 ; RV64IF-NEXT:    ret
   %a_ = fadd float 0.0, %a
@@ -407,26 +407,26 @@ define float @fnmadd_s(float %a, float %b, float %c) nounwind {
 define float @fnmsub_s(float %a, float %b, float %c) nounwind {
 ; RV32IF-LABEL: fnmsub_s:
 ; RV32IF:       # %bb.0:
-; RV32IF-NEXT:    fmv.w.x ft0, a0
-; RV32IF-NEXT:    lui a0, %hi(.LCPI17_0)
-; RV32IF-NEXT:    addi a0, a0, %lo(.LCPI17_0)
-; RV32IF-NEXT:    flw ft1, 0(a0)
-; RV32IF-NEXT:    fadd.s ft0, ft0, ft1
+; RV32IF-NEXT:    lui a3, %hi(.LCPI17_0)
+; RV32IF-NEXT:    addi a3, a3, %lo(.LCPI17_0)
+; RV32IF-NEXT:    flw ft0, 0(a3)
 ; RV32IF-NEXT:    fmv.w.x ft1, a2
 ; RV32IF-NEXT:    fmv.w.x ft2, a1
+; RV32IF-NEXT:    fmv.w.x ft3, a0
+; RV32IF-NEXT:    fadd.s ft0, ft3, ft0
 ; RV32IF-NEXT:    fnmsub.s ft0, ft0, ft2, ft1
 ; RV32IF-NEXT:    fmv.x.w a0, ft0
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: fnmsub_s:
 ; RV64IF:       # %bb.0:
-; RV64IF-NEXT:    fmv.w.x ft0, a0
-; RV64IF-NEXT:    lui a0, %hi(.LCPI17_0)
-; RV64IF-NEXT:    addi a0, a0, %lo(.LCPI17_0)
-; RV64IF-NEXT:    flw ft1, 0(a0)
-; RV64IF-NEXT:    fadd.s ft0, ft0, ft1
+; RV64IF-NEXT:    lui a3, %hi(.LCPI17_0)
+; RV64IF-NEXT:    addi a3, a3, %lo(.LCPI17_0)
+; RV64IF-NEXT:    flw ft0, 0(a3)
 ; RV64IF-NEXT:    fmv.w.x ft1, a2
 ; RV64IF-NEXT:    fmv.w.x ft2, a1
+; RV64IF-NEXT:    fmv.w.x ft3, a0
+; RV64IF-NEXT:    fadd.s ft0, ft3, ft0
 ; RV64IF-NEXT:    fnmsub.s ft0, ft0, ft2, ft1
 ; RV64IF-NEXT:    fmv.x.w a0, ft0
 ; RV64IF-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/float-br-fcmp.ll b/llvm/test/CodeGen/RISCV/float-br-fcmp.ll
index 425da7383..a1e5b32ee 100644
--- a/llvm/test/CodeGen/RISCV/float-br-fcmp.ll
+++ b/llvm/test/CodeGen/RISCV/float-br-fcmp.ll
@@ -337,11 +337,11 @@ define void @br_fcmp_ord(float %a, float %b) nounwind {
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
 ; RV32IF-NEXT:    sw ra, 12(sp)
-; RV32IF-NEXT:    fmv.w.x ft0, a1
-; RV32IF-NEXT:    feq.s a1, ft0, ft0
 ; RV32IF-NEXT:    fmv.w.x ft0, a0
-; RV32IF-NEXT:    feq.s a0, ft0, ft0
-; RV32IF-NEXT:    and a0, a0, a1
+; RV32IF-NEXT:    fmv.w.x ft1, a1
+; RV32IF-NEXT:    feq.s a0, ft1, ft1
+; RV32IF-NEXT:    feq.s a1, ft0, ft0
+; RV32IF-NEXT:    and a0, a1, a0
 ; RV32IF-NEXT:    bnez a0, .LBB8_2
 ; RV32IF-NEXT:  # %bb.1: # %if.else
 ; RV32IF-NEXT:    lw ra, 12(sp)
@@ -354,11 +354,11 @@ define void @br_fcmp_ord(float %a, float %b) nounwind {
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
 ; RV64IF-NEXT:    sd ra, 8(sp)
-; RV64IF-NEXT:    fmv.w.x ft0, a1
-; RV64IF-NEXT:    feq.s a1, ft0, ft0
 ; RV64IF-NEXT:    fmv.w.x ft0, a0
-; RV64IF-NEXT:    feq.s a0, ft0, ft0
-; RV64IF-NEXT:    and a0, a0, a1
+; RV64IF-NEXT:    fmv.w.x ft1, a1
+; RV64IF-NEXT:    feq.s a0, ft1, ft1
+; RV64IF-NEXT:    feq.s a1, ft0, ft0
+; RV64IF-NEXT:    and a0, a1, a0
 ; RV64IF-NEXT:    bnez a0, .LBB8_2
 ; RV64IF-NEXT:  # %bb.1: # %if.else
 ; RV64IF-NEXT:    ld ra, 8(sp)
@@ -635,11 +635,11 @@ define void @br_fcmp_uno(float %a, float %b) nounwind {
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
 ; RV32IF-NEXT:    sw ra, 12(sp)
-; RV32IF-NEXT:    fmv.w.x ft0, a1
-; RV32IF-NEXT:    feq.s a1, ft0, ft0
 ; RV32IF-NEXT:    fmv.w.x ft0, a0
-; RV32IF-NEXT:    feq.s a0, ft0, ft0
-; RV32IF-NEXT:    and a0, a0, a1
+; RV32IF-NEXT:    fmv.w.x ft1, a1
+; RV32IF-NEXT:    feq.s a0, ft1, ft1
+; RV32IF-NEXT:    feq.s a1, ft0, ft0
+; RV32IF-NEXT:    and a0, a1, a0
 ; RV32IF-NEXT:    seqz a0, a0
 ; RV32IF-NEXT:    bnez a0, .LBB15_2
 ; RV32IF-NEXT:  # %bb.1: # %if.else
@@ -653,11 +653,11 @@ define void @br_fcmp_uno(float %a, float %b) nounwind {
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
 ; RV64IF-NEXT:    sd ra, 8(sp)
-; RV64IF-NEXT:    fmv.w.x ft0, a1
-; RV64IF-NEXT:    feq.s a1, ft0, ft0
 ; RV64IF-NEXT:    fmv.w.x ft0, a0
-; RV64IF-NEXT:    feq.s a0, ft0, ft0
-; RV64IF-NEXT:    and a0, a0, a1
+; RV64IF-NEXT:    fmv.w.x ft1, a1
+; RV64IF-NEXT:    feq.s a0, ft1, ft1
+; RV64IF-NEXT:    feq.s a1, ft0, ft0
+; RV64IF-NEXT:    and a0, a1, a0
 ; RV64IF-NEXT:    seqz a0, a0
 ; RV64IF-NEXT:    bnez a0, .LBB15_2
 ; RV64IF-NEXT:  # %bb.1: # %if.else
diff --git a/llvm/test/CodeGen/RISCV/float-convert.ll b/llvm/test/CodeGen/RISCV/float-convert.ll
index 296e9d468..35a71c4ee 100644
--- a/llvm/test/CodeGen/RISCV/float-convert.ll
+++ b/llvm/test/CodeGen/RISCV/float-convert.ll
@@ -97,17 +97,17 @@ define float @fcvt_s_wu(i32 %a) nounwind {
 define float @fmv_w_x(i32 %a, i32 %b) nounwind {
 ; RV32IF-LABEL: fmv_w_x:
 ; RV32IF:       # %bb.0:
-; RV32IF-NEXT:    fmv.w.x ft0, a1
-; RV32IF-NEXT:    fmv.w.x ft1, a0
-; RV32IF-NEXT:    fadd.s ft0, ft1, ft0
+; RV32IF-NEXT:    fmv.w.x ft0, a0
+; RV32IF-NEXT:    fmv.w.x ft1, a1
+; RV32IF-NEXT:    fadd.s ft0, ft0, ft1
 ; RV32IF-NEXT:    fmv.x.w a0, ft0
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: fmv_w_x:
 ; RV64IF:       # %bb.0:
-; RV64IF-NEXT:    fmv.w.x ft0, a1
-; RV64IF-NEXT:    fmv.w.x ft1, a0
-; RV64IF-NEXT:    fadd.s ft0, ft1, ft0
+; RV64IF-NEXT:    fmv.w.x ft0, a0
+; RV64IF-NEXT:    fmv.w.x ft1, a1
+; RV64IF-NEXT:    fadd.s ft0, ft0, ft1
 ; RV64IF-NEXT:    fmv.x.w a0, ft0
 ; RV64IF-NEXT:    ret
 ; Ensure fmv.w.x is generated even for a soft float calling convention
diff --git a/llvm/test/CodeGen/RISCV/float-fcmp.ll b/llvm/test/CodeGen/RISCV/float-fcmp.ll
index 43a849bee..3a44020a2 100644
--- a/llvm/test/CodeGen/RISCV/float-fcmp.ll
+++ b/llvm/test/CodeGen/RISCV/float-fcmp.ll
@@ -146,20 +146,20 @@ define i32 @fcmp_one(float %a, float %b) nounwind {
 define i32 @fcmp_ord(float %a, float %b) nounwind {
 ; RV32IF-LABEL: fcmp_ord:
 ; RV32IF:       # %bb.0:
-; RV32IF-NEXT:    fmv.w.x ft0, a1
-; RV32IF-NEXT:    feq.s a1, ft0, ft0
 ; RV32IF-NEXT:    fmv.w.x ft0, a0
-; RV32IF-NEXT:    feq.s a0, ft0, ft0
-; RV32IF-NEXT:    and a0, a0, a1
+; RV32IF-NEXT:    fmv.w.x ft1, a1
+; RV32IF-NEXT:    feq.s a0, ft1, ft1
+; RV32IF-NEXT:    feq.s a1, ft0, ft0
+; RV32IF-NEXT:    and a0, a1, a0
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: fcmp_ord:
 ; RV64IF:       # %bb.0:
-; RV64IF-NEXT:    fmv.w.x ft0, a1
-; RV64IF-NEXT:    feq.s a1, ft0, ft0
 ; RV64IF-NEXT:    fmv.w.x ft0, a0
-; RV64IF-NEXT:    feq.s a0, ft0, ft0
-; RV64IF-NEXT:    and a0, a0, a1
+; RV64IF-NEXT:    fmv.w.x ft1, a1
+; RV64IF-NEXT:    feq.s a0, ft1, ft1
+; RV64IF-NEXT:    feq.s a1, ft0, ft0
+; RV64IF-NEXT:    and a0, a1, a0
 ; RV64IF-NEXT:    ret
   %1 = fcmp ord float %a, %b
   %2 = zext i1 %1 to i32
@@ -303,21 +303,21 @@ define i32 @fcmp_une(float %a, float %b) nounwind {
 define i32 @fcmp_uno(float %a, float %b) nounwind {
 ; RV32IF-LABEL: fcmp_uno:
 ; RV32IF:       # %bb.0:
-; RV32IF-NEXT:    fmv.w.x ft0, a1
-; RV32IF-NEXT:    feq.s a1, ft0, ft0
 ; RV32IF-NEXT:    fmv.w.x ft0, a0
-; RV32IF-NEXT:    feq.s a0, ft0, ft0
-; RV32IF-NEXT:    and a0, a0, a1
+; RV32IF-NEXT:    fmv.w.x ft1, a1
+; RV32IF-NEXT:    feq.s a0, ft1, ft1
+; RV32IF-NEXT:    feq.s a1, ft0, ft0
+; RV32IF-NEXT:    and a0, a1, a0
 ; RV32IF-NEXT:    seqz a0, a0
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: fcmp_uno:
 ; RV64IF:       # %bb.0:
-; RV64IF-NEXT:    fmv.w.x ft0, a1
-; RV64IF-NEXT:    feq.s a1, ft0, ft0
 ; RV64IF-NEXT:    fmv.w.x ft0, a0
-; RV64IF-NEXT:    feq.s a0, ft0, ft0
-; RV64IF-NEXT:    and a0, a0, a1
+; RV64IF-NEXT:    fmv.w.x ft1, a1
+; RV64IF-NEXT:    feq.s a0, ft1, ft1
+; RV64IF-NEXT:    feq.s a1, ft0, ft0
+; RV64IF-NEXT:    and a0, a1, a0
 ; RV64IF-NEXT:    seqz a0, a0
 ; RV64IF-NEXT:    ret
   %1 = fcmp uno float %a, %b
diff --git a/llvm/test/CodeGen/RISCV/float-imm.ll b/llvm/test/CodeGen/RISCV/float-imm.ll
index 09d9c3c93..e0ec464ff 100644
--- a/llvm/test/CodeGen/RISCV/float-imm.ll
+++ b/llvm/test/CodeGen/RISCV/float-imm.ll
@@ -26,21 +26,21 @@ define float @float_imm_op(float %a) nounwind {
 ; TODO: addi should be folded in to the flw
 ; RV32IF-LABEL: float_imm_op:
 ; RV32IF:       # %bb.0:
-; RV32IF-NEXT:    fmv.w.x ft0, a0
-; RV32IF-NEXT:    lui a0, %hi(.LCPI1_0)
-; RV32IF-NEXT:    addi a0, a0, %lo(.LCPI1_0)
-; RV32IF-NEXT:    flw ft1, 0(a0)
-; RV32IF-NEXT:    fadd.s ft0, ft0, ft1
+; RV32IF-NEXT:    lui a1, %hi(.LCPI1_0)
+; RV32IF-NEXT:    addi a1, a1, %lo(.LCPI1_0)
+; RV32IF-NEXT:    flw ft0, 0(a1)
+; RV32IF-NEXT:    fmv.w.x ft1, a0
+; RV32IF-NEXT:    fadd.s ft0, ft1, ft0
 ; RV32IF-NEXT:    fmv.x.w a0, ft0
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: float_imm_op:
 ; RV64IF:       # %bb.0:
-; RV64IF-NEXT:    fmv.w.x ft0, a0
-; RV64IF-NEXT:    lui a0, %hi(.LCPI1_0)
-; RV64IF-NEXT:    addi a0, a0, %lo(.LCPI1_0)
-; RV64IF-NEXT:    flw ft1, 0(a0)
-; RV64IF-NEXT:    fadd.s ft0, ft0, ft1
+; RV64IF-NEXT:    lui a1, %hi(.LCPI1_0)
+; RV64IF-NEXT:    addi a1, a1, %lo(.LCPI1_0)
+; RV64IF-NEXT:    flw ft0, 0(a1)
+; RV64IF-NEXT:    fmv.w.x ft1, a0
+; RV64IF-NEXT:    fadd.s ft0, ft1, ft0
 ; RV64IF-NEXT:    fmv.x.w a0, ft0
 ; RV64IF-NEXT:    ret
   %1 = fadd float %a, 1.0
diff --git a/llvm/test/CodeGen/RISCV/float-intrinsics.ll b/llvm/test/CodeGen/RISCV/float-intrinsics.ll
index 7ac2df6e8..68bb95b63 100644
--- a/llvm/test/CodeGen/RISCV/float-intrinsics.ll
+++ b/llvm/test/CodeGen/RISCV/float-intrinsics.ll
@@ -108,17 +108,16 @@ define float @sincos_f32(float %a) nounwind {
 ; RV32IF-NEXT:    addi sp, sp, -16
 ; RV32IF-NEXT:    sw ra, 12(sp)
 ; RV32IF-NEXT:    sw s0, 8(sp)
-; RV32IF-NEXT:    sw s1, 4(sp)
 ; RV32IF-NEXT:    mv s0, a0
 ; RV32IF-NEXT:    call sinf
-; RV32IF-NEXT:    mv s1, a0
+; RV32IF-NEXT:    fmv.w.x ft0, a0
+; RV32IF-NEXT:    fsw ft0, 4(sp)
 ; RV32IF-NEXT:    mv a0, s0
 ; RV32IF-NEXT:    call cosf
 ; RV32IF-NEXT:    fmv.w.x ft0, a0
-; RV32IF-NEXT:    fmv.w.x ft1, s1
+; RV32IF-NEXT:    flw ft1, 4(sp)
 ; RV32IF-NEXT:    fadd.s ft0, ft1, ft0
 ; RV32IF-NEXT:    fmv.x.w a0, ft0
-; RV32IF-NEXT:    lw s1, 4(sp)
 ; RV32IF-NEXT:    lw s0, 8(sp)
 ; RV32IF-NEXT:    lw ra, 12(sp)
 ; RV32IF-NEXT:    addi sp, sp, 16
@@ -129,17 +128,16 @@ define float @sincos_f32(float %a) nounwind {
 ; RV64IF-NEXT:    addi sp, sp, -32
 ; RV64IF-NEXT:    sd ra, 24(sp)
 ; RV64IF-NEXT:    sd s0, 16(sp)
-; RV64IF-NEXT:    sd s1, 8(sp)
 ; RV64IF-NEXT:    mv s0, a0
 ; RV64IF-NEXT:    call sinf
-; RV64IF-NEXT:    mv s1, a0
+; RV64IF-NEXT:    fmv.w.x ft0, a0
+; RV64IF-NEXT:    fsw ft0, 12(sp)
 ; RV64IF-NEXT:    mv a0, s0
 ; RV64IF-NEXT:    call cosf
 ; RV64IF-NEXT:    fmv.w.x ft0, a0
-; RV64IF-NEXT:    fmv.w.x ft1, s1
+; RV64IF-NEXT:    flw ft1, 12(sp)
 ; RV64IF-NEXT:    fadd.s ft0, ft1, ft0
 ; RV64IF-NEXT:    fmv.x.w a0, ft0
-; RV64IF-NEXT:    ld s1, 8(sp)
 ; RV64IF-NEXT:    ld s0, 16(sp)
 ; RV64IF-NEXT:    ld ra, 24(sp)
 ; RV64IF-NEXT:    addi sp, sp, 32
@@ -324,21 +322,21 @@ define float @fmuladd_f32(float %a, float %b, float %c) nounwind {
 ; Use of fmadd depends on TargetLowering::isFMAFasterthanFMulAndFAdd
 ; RV32IF-LABEL: fmuladd_f32:
 ; RV32IF:       # %bb.0:
-; RV32IF-NEXT:    fmv.w.x ft0, a1
-; RV32IF-NEXT:    fmv.w.x ft1, a0
-; RV32IF-NEXT:    fmul.s ft0, ft1, ft0
-; RV32IF-NEXT:    fmv.w.x ft1, a2
-; RV32IF-NEXT:    fadd.s ft0, ft0, ft1
+; RV32IF-NEXT:    fmv.w.x ft0, a2
+; RV32IF-NEXT:    fmv.w.x ft1, a1
+; RV32IF-NEXT:    fmv.w.x ft2, a0
+; RV32IF-NEXT:    fmul.s ft1, ft2, ft1
+; RV32IF-NEXT:    fadd.s ft0, ft1, ft0
 ; RV32IF-NEXT:    fmv.x.w a0, ft0
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: fmuladd_f32:
 ; RV64IF:       # %bb.0:
-; RV64IF-NEXT:    fmv.w.x ft0, a1
-; RV64IF-NEXT:    fmv.w.x ft1, a0
-; RV64IF-NEXT:    fmul.s ft0, ft1, ft0
-; RV64IF-NEXT:    fmv.w.x ft1, a2
-; RV64IF-NEXT:    fadd.s ft0, ft0, ft1
+; RV64IF-NEXT:    fmv.w.x ft0, a2
+; RV64IF-NEXT:    fmv.w.x ft1, a1
+; RV64IF-NEXT:    fmv.w.x ft2, a0
+; RV64IF-NEXT:    fmul.s ft1, ft2, ft1
+; RV64IF-NEXT:    fadd.s ft0, ft1, ft0
 ; RV64IF-NEXT:    fmv.x.w a0, ft0
 ; RV64IF-NEXT:    ret
   %1 = call float @llvm.fmuladd.f32(float %a, float %b, float %c)
diff --git a/llvm/test/CodeGen/RISCV/float-mem.ll b/llvm/test/CodeGen/RISCV/float-mem.ll
index 2d1a7a7f9..c989235e2 100644
--- a/llvm/test/CodeGen/RISCV/float-mem.ll
+++ b/llvm/test/CodeGen/RISCV/float-mem.ll
@@ -7,17 +7,17 @@
 define float @flw(float *%a) nounwind {
 ; RV32IF-LABEL: flw:
 ; RV32IF:       # %bb.0:
-; RV32IF-NEXT:    flw ft0, 12(a0)
-; RV32IF-NEXT:    flw ft1, 0(a0)
-; RV32IF-NEXT:    fadd.s ft0, ft1, ft0
+; RV32IF-NEXT:    flw ft0, 0(a0)
+; RV32IF-NEXT:    flw ft1, 12(a0)
+; RV32IF-NEXT:    fadd.s ft0, ft0, ft1
 ; RV32IF-NEXT:    fmv.x.w a0, ft0
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: flw:
 ; RV64IF:       # %bb.0:
-; RV64IF-NEXT:    flw ft0, 12(a0)
-; RV64IF-NEXT:    flw ft1, 0(a0)
-; RV64IF-NEXT:    fadd.s ft0, ft1, ft0
+; RV64IF-NEXT:    flw ft0, 0(a0)
+; RV64IF-NEXT:    flw ft1, 12(a0)
+; RV64IF-NEXT:    fadd.s ft0, ft0, ft1
 ; RV64IF-NEXT:    fmv.x.w a0, ft0
 ; RV64IF-NEXT:    ret
   %1 = load float, float* %a
@@ -37,8 +37,8 @@ define void @fsw(float *%a, float %b, float %c) nounwind {
 ; RV32IF-NEXT:    fmv.w.x ft0, a2
 ; RV32IF-NEXT:    fmv.w.x ft1, a1
 ; RV32IF-NEXT:    fadd.s ft0, ft1, ft0
-; RV32IF-NEXT:    fsw ft0, 32(a0)
 ; RV32IF-NEXT:    fsw ft0, 0(a0)
+; RV32IF-NEXT:    fsw ft0, 32(a0)
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: fsw:
@@ -46,8 +46,8 @@ define void @fsw(float *%a, float %b, float %c) nounwind {
 ; RV64IF-NEXT:    fmv.w.x ft0, a2
 ; RV64IF-NEXT:    fmv.w.x ft1, a1
 ; RV64IF-NEXT:    fadd.s ft0, ft1, ft0
-; RV64IF-NEXT:    fsw ft0, 32(a0)
 ; RV64IF-NEXT:    fsw ft0, 0(a0)
+; RV64IF-NEXT:    fsw ft0, 32(a0)
 ; RV64IF-NEXT:    ret
   %1 = fadd float %b, %c
   store float %1, float* %a
@@ -70,10 +70,10 @@ define float @flw_fsw_global(float %a, float %b) nounwind {
 ; RV32IF-NEXT:    lui a0, %hi(G)
 ; RV32IF-NEXT:    flw ft1, %lo(G)(a0)
 ; RV32IF-NEXT:    fsw ft0, %lo(G)(a0)
-; RV32IF-NEXT:    addi a0, a0, %lo(G)
-; RV32IF-NEXT:    flw ft1, 36(a0)
-; RV32IF-NEXT:    fsw ft0, 36(a0)
+; RV32IF-NEXT:    addi a1, a0, %lo(G)
+; RV32IF-NEXT:    flw ft1, 36(a1)
 ; RV32IF-NEXT:    fmv.x.w a0, ft0
+; RV32IF-NEXT:    fsw ft0, 36(a1)
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: flw_fsw_global:
@@ -84,10 +84,10 @@ define float @flw_fsw_global(float %a, float %b) nounwind {
 ; RV64IF-NEXT:    lui a0, %hi(G)
 ; RV64IF-NEXT:    flw ft1, %lo(G)(a0)
 ; RV64IF-NEXT:    fsw ft0, %lo(G)(a0)
-; RV64IF-NEXT:    addi a0, a0, %lo(G)
-; RV64IF-NEXT:    flw ft1, 36(a0)
-; RV64IF-NEXT:    fsw ft0, 36(a0)
+; RV64IF-NEXT:    addi a1, a0, %lo(G)
+; RV64IF-NEXT:    flw ft1, 36(a1)
 ; RV64IF-NEXT:    fmv.x.w a0, ft0
+; RV64IF-NEXT:    fsw ft0, 36(a1)
 ; RV64IF-NEXT:    ret
   %1 = fadd float %a, %b
   %2 = load volatile float, float* @G
@@ -102,24 +102,24 @@ define float @flw_fsw_global(float %a, float %b) nounwind {
 define float @flw_fsw_constant(float %a) nounwind {
 ; RV32IF-LABEL: flw_fsw_constant:
 ; RV32IF:       # %bb.0:
-; RV32IF-NEXT:    fmv.w.x ft0, a0
-; RV32IF-NEXT:    lui a0, 912092
-; RV32IF-NEXT:    flw ft1, -273(a0)
-; RV32IF-NEXT:    fadd.s ft0, ft0, ft1
-; RV32IF-NEXT:    fsw ft0, -273(a0)
+; RV32IF-NEXT:    lui a1, 912092
+; RV32IF-NEXT:    flw ft0, -273(a1)
+; RV32IF-NEXT:    fmv.w.x ft1, a0
+; RV32IF-NEXT:    fadd.s ft0, ft1, ft0
 ; RV32IF-NEXT:    fmv.x.w a0, ft0
+; RV32IF-NEXT:    fsw ft0, -273(a1)
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: flw_fsw_constant:
 ; RV64IF:       # %bb.0:
-; RV64IF-NEXT:    fmv.w.x ft0, a0
-; RV64IF-NEXT:    lui a0, 56
-; RV64IF-NEXT:    addiw a0, a0, -1353
-; RV64IF-NEXT:    slli a0, a0, 14
-; RV64IF-NEXT:    flw ft1, -273(a0)
-; RV64IF-NEXT:    fadd.s ft0, ft0, ft1
-; RV64IF-NEXT:    fsw ft0, -273(a0)
+; RV64IF-NEXT:    lui a1, 56
+; RV64IF-NEXT:    addiw a1, a1, -1353
+; RV64IF-NEXT:    slli a1, a1, 14
+; RV64IF-NEXT:    flw ft0, -273(a1)
+; RV64IF-NEXT:    fmv.w.x ft1, a0
+; RV64IF-NEXT:    fadd.s ft0, ft1, ft0
 ; RV64IF-NEXT:    fmv.x.w a0, ft0
+; RV64IF-NEXT:    fsw ft0, -273(a1)
 ; RV64IF-NEXT:    ret
   %1 = inttoptr i32 3735928559 to float*
   %2 = load volatile float, float* %1
@@ -135,34 +135,32 @@ define float @flw_stack(float %a) nounwind {
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
 ; RV32IF-NEXT:    sw ra, 12(sp)
-; RV32IF-NEXT:    sw s0, 8(sp)
-; RV32IF-NEXT:    mv s0, a0
-; RV32IF-NEXT:    addi a0, sp, 4
+; RV32IF-NEXT:    fmv.w.x ft0, a0
+; RV32IF-NEXT:    fsw ft0, 4(sp)
+; RV32IF-NEXT:    addi a0, sp, 8
 ; RV32IF-NEXT:    call notdead
-; RV32IF-NEXT:    fmv.w.x ft0, s0
+; RV32IF-NEXT:    flw ft0, 8(sp)
 ; RV32IF-NEXT:    flw ft1, 4(sp)
-; RV32IF-NEXT:    fadd.s ft0, ft1, ft0
+; RV32IF-NEXT:    fadd.s ft0, ft0, ft1
 ; RV32IF-NEXT:    fmv.x.w a0, ft0
-; RV32IF-NEXT:    lw s0, 8(sp)
 ; RV32IF-NEXT:    lw ra, 12(sp)
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: flw_stack:
 ; RV64IF:       # %bb.0:
-; RV64IF-NEXT:    addi sp, sp, -32
-; RV64IF-NEXT:    sd ra, 24(sp)
-; RV64IF-NEXT:    sd s0, 16(sp)
-; RV64IF-NEXT:    mv s0, a0
-; RV64IF-NEXT:    addi a0, sp, 12
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    sd ra, 8(sp)
+; RV64IF-NEXT:    fmv.w.x ft0, a0
+; RV64IF-NEXT:    fsw ft0, 0(sp)
+; RV64IF-NEXT:    addi a0, sp, 4
 ; RV64IF-NEXT:    call notdead
-; RV64IF-NEXT:    fmv.w.x ft0, s0
-; RV64IF-NEXT:    flw ft1, 12(sp)
-; RV64IF-NEXT:    fadd.s ft0, ft1, ft0
+; RV64IF-NEXT:    flw ft0, 4(sp)
+; RV64IF-NEXT:    flw ft1, 0(sp)
+; RV64IF-NEXT:    fadd.s ft0, ft0, ft1
 ; RV64IF-NEXT:    fmv.x.w a0, ft0
-; RV64IF-NEXT:    ld s0, 16(sp)
-; RV64IF-NEXT:    ld ra, 24(sp)
-; RV64IF-NEXT:    addi sp, sp, 32
+; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
   %1 = alloca float, align 4
   %2 = bitcast float* %1 to i8*
diff --git a/llvm/test/CodeGen/RISCV/float-select-fcmp.ll b/llvm/test/CodeGen/RISCV/float-select-fcmp.ll
index ec81c53e1..142e4f8e8 100644
--- a/llvm/test/CodeGen/RISCV/float-select-fcmp.ll
+++ b/llvm/test/CodeGen/RISCV/float-select-fcmp.ll
@@ -240,14 +240,14 @@ define float @select_fcmp_ord(float %a, float %b) nounwind {
 define float @select_fcmp_ueq(float %a, float %b) nounwind {
 ; RV32IF-LABEL: select_fcmp_ueq:
 ; RV32IF:       # %bb.0:
-; RV32IF-NEXT:    fmv.w.x ft0, a0
 ; RV32IF-NEXT:    fmv.w.x ft1, a1
-; RV32IF-NEXT:    feq.s a0, ft1, ft1
-; RV32IF-NEXT:    feq.s a1, ft0, ft0
-; RV32IF-NEXT:    and a0, a1, a0
-; RV32IF-NEXT:    seqz a0, a0
-; RV32IF-NEXT:    feq.s a1, ft0, ft1
-; RV32IF-NEXT:    or a0, a1, a0
+; RV32IF-NEXT:    fmv.w.x ft0, a0
+; RV32IF-NEXT:    feq.s a0, ft0, ft1
+; RV32IF-NEXT:    feq.s a1, ft1, ft1
+; RV32IF-NEXT:    feq.s a2, ft0, ft0
+; RV32IF-NEXT:    and a1, a2, a1
+; RV32IF-NEXT:    seqz a1, a1
+; RV32IF-NEXT:    or a0, a0, a1
 ; RV32IF-NEXT:    bnez a0, .LBB8_2
 ; RV32IF-NEXT:  # %bb.1:
 ; RV32IF-NEXT:    fmv.s ft0, ft1
@@ -257,14 +257,14 @@ define float @select_fcmp_ueq(float %a, float %b) nounwind {
 ;
 ; RV64IF-LABEL: select_fcmp_ueq:
 ; RV64IF:       # %bb.0:
-; RV64IF-NEXT:    fmv.w.x ft0, a0
 ; RV64IF-NEXT:    fmv.w.x ft1, a1
-; RV64IF-NEXT:    feq.s a0, ft1, ft1
-; RV64IF-NEXT:    feq.s a1, ft0, ft0
-; RV64IF-NEXT:    and a0, a1, a0
-; RV64IF-NEXT:    seqz a0, a0
-; RV64IF-NEXT:    feq.s a1, ft0, ft1
-; RV64IF-NEXT:    or a0, a1, a0
+; RV64IF-NEXT:    fmv.w.x ft0, a0
+; RV64IF-NEXT:    feq.s a0, ft0, ft1
+; RV64IF-NEXT:    feq.s a1, ft1, ft1
+; RV64IF-NEXT:    feq.s a2, ft0, ft0
+; RV64IF-NEXT:    and a1, a2, a1
+; RV64IF-NEXT:    seqz a1, a1
+; RV64IF-NEXT:    or a0, a0, a1
 ; RV64IF-NEXT:    bnez a0, .LBB8_2
 ; RV64IF-NEXT:  # %bb.1:
 ; RV64IF-NEXT:    fmv.s ft0, ft1
@@ -486,24 +486,24 @@ define i32 @i32_select_fcmp_oeq(float %a, float %b, i32 %c, i32 %d) nounwind {
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    fmv.w.x ft0, a1
 ; RV32IF-NEXT:    fmv.w.x ft1, a0
-; RV32IF-NEXT:    feq.s a0, ft1, ft0
-; RV32IF-NEXT:    bnez a0, .LBB16_2
+; RV32IF-NEXT:    feq.s a1, ft1, ft0
+; RV32IF-NEXT:    mv a0, a2
+; RV32IF-NEXT:    bnez a1, .LBB16_2
 ; RV32IF-NEXT:  # %bb.1:
-; RV32IF-NEXT:    mv a2, a3
+; RV32IF-NEXT:    mv a0, a3
 ; RV32IF-NEXT:  .LBB16_2:
-; RV32IF-NEXT:    mv a0, a2
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: i32_select_fcmp_oeq:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fmv.w.x ft0, a1
 ; RV64IF-NEXT:    fmv.w.x ft1, a0
-; RV64IF-NEXT:    feq.s a0, ft1, ft0
-; RV64IF-NEXT:    bnez a0, .LBB16_2
+; RV64IF-NEXT:    feq.s a1, ft1, ft0
+; RV64IF-NEXT:    mv a0, a2
+; RV64IF-NEXT:    bnez a1, .LBB16_2
 ; RV64IF-NEXT:  # %bb.1:
-; RV64IF-NEXT:    mv a2, a3
+; RV64IF-NEXT:    mv a0, a3
 ; RV64IF-NEXT:  .LBB16_2:
-; RV64IF-NEXT:    mv a0, a2
 ; RV64IF-NEXT:    ret
   %1 = fcmp oeq float %a, %b
   %2 = select i1 %1, i32 %c, i32 %d
diff --git a/llvm/test/CodeGen/RISCV/fp128.ll b/llvm/test/CodeGen/RISCV/fp128.ll
index a928d69fe..91b170291 100644
--- a/llvm/test/CodeGen/RISCV/fp128.ll
+++ b/llvm/test/CodeGen/RISCV/fp128.ll
@@ -13,28 +13,28 @@ define i32 @test_load_and_cmp() nounwind {
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -48
 ; RV32I-NEXT:    sw ra, 44(sp)
-; RV32I-NEXT:    lui a0, %hi(y)
-; RV32I-NEXT:    lw a1, %lo(y)(a0)
-; RV32I-NEXT:    sw a1, 8(sp)
-; RV32I-NEXT:    lui a1, %hi(x)
-; RV32I-NEXT:    lw a2, %lo(x)(a1)
-; RV32I-NEXT:    sw a2, 24(sp)
-; RV32I-NEXT:    addi a0, a0, %lo(y)
-; RV32I-NEXT:    lw a2, 12(a0)
-; RV32I-NEXT:    sw a2, 20(sp)
-; RV32I-NEXT:    lw a2, 8(a0)
-; RV32I-NEXT:    sw a2, 16(sp)
-; RV32I-NEXT:    lw a0, 4(a0)
-; RV32I-NEXT:    sw a0, 12(sp)
-; RV32I-NEXT:    addi a0, a1, %lo(x)
-; RV32I-NEXT:    lw a1, 12(a0)
+; RV32I-NEXT:    lui a0, %hi(x)
+; RV32I-NEXT:    addi a1, a0, %lo(x)
+; RV32I-NEXT:    lw a6, 4(a1)
+; RV32I-NEXT:    lw a7, 8(a1)
+; RV32I-NEXT:    lw a1, 12(a1)
+; RV32I-NEXT:    lw a0, %lo(x)(a0)
+; RV32I-NEXT:    lui a4, %hi(y)
+; RV32I-NEXT:    addi a5, a4, %lo(y)
+; RV32I-NEXT:    lw a2, 4(a5)
+; RV32I-NEXT:    lw a3, 8(a5)
+; RV32I-NEXT:    lw a5, 12(a5)
+; RV32I-NEXT:    lw a4, %lo(y)(a4)
+; RV32I-NEXT:    sw a4, 8(sp)
+; RV32I-NEXT:    sw a0, 24(sp)
+; RV32I-NEXT:    sw a5, 20(sp)
+; RV32I-NEXT:    sw a3, 16(sp)
+; RV32I-NEXT:    sw a2, 12(sp)
 ; RV32I-NEXT:    sw a1, 36(sp)
-; RV32I-NEXT:    lw a1, 8(a0)
-; RV32I-NEXT:    sw a1, 32(sp)
-; RV32I-NEXT:    lw a0, 4(a0)
-; RV32I-NEXT:    sw a0, 28(sp)
+; RV32I-NEXT:    sw a7, 32(sp)
 ; RV32I-NEXT:    addi a0, sp, 24
 ; RV32I-NEXT:    addi a1, sp, 8
+; RV32I-NEXT:    sw a6, 28(sp)
 ; RV32I-NEXT:    call __netf2
 ; RV32I-NEXT:    snez a0, a0
 ; RV32I-NEXT:    lw ra, 44(sp)
@@ -52,39 +52,39 @@ define i32 @test_add_and_fptosi() nounwind {
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -80
 ; RV32I-NEXT:    sw ra, 76(sp)
-; RV32I-NEXT:    lui a0, %hi(y)
-; RV32I-NEXT:    lw a1, %lo(y)(a0)
-; RV32I-NEXT:    sw a1, 24(sp)
-; RV32I-NEXT:    lui a1, %hi(x)
-; RV32I-NEXT:    lw a2, %lo(x)(a1)
-; RV32I-NEXT:    sw a2, 40(sp)
-; RV32I-NEXT:    addi a0, a0, %lo(y)
-; RV32I-NEXT:    lw a2, 12(a0)
-; RV32I-NEXT:    sw a2, 36(sp)
-; RV32I-NEXT:    lw a2, 8(a0)
+; RV32I-NEXT:    lui a0, %hi(x)
+; RV32I-NEXT:    addi a1, a0, %lo(x)
+; RV32I-NEXT:    lw a6, 4(a1)
+; RV32I-NEXT:    lw a7, 8(a1)
+; RV32I-NEXT:    lw a1, 12(a1)
+; RV32I-NEXT:    lw a0, %lo(x)(a0)
+; RV32I-NEXT:    lui a4, %hi(y)
+; RV32I-NEXT:    addi a5, a4, %lo(y)
+; RV32I-NEXT:    lw a3, 4(a5)
+; RV32I-NEXT:    lw a2, 8(a5)
+; RV32I-NEXT:    lw a5, 12(a5)
+; RV32I-NEXT:    lw a4, %lo(y)(a4)
+; RV32I-NEXT:    sw a4, 24(sp)
+; RV32I-NEXT:    sw a0, 40(sp)
+; RV32I-NEXT:    sw a5, 36(sp)
 ; RV32I-NEXT:    sw a2, 32(sp)
-; RV32I-NEXT:    lw a0, 4(a0)
-; RV32I-NEXT:    sw a0, 28(sp)
-; RV32I-NEXT:    addi a0, a1, %lo(x)
-; RV32I-NEXT:    lw a1, 12(a0)
+; RV32I-NEXT:    sw a3, 28(sp)
 ; RV32I-NEXT:    sw a1, 52(sp)
-; RV32I-NEXT:    lw a1, 8(a0)
-; RV32I-NEXT:    sw a1, 48(sp)
-; RV32I-NEXT:    lw a0, 4(a0)
-; RV32I-NEXT:    sw a0, 44(sp)
+; RV32I-NEXT:    sw a7, 48(sp)
 ; RV32I-NEXT:    addi a0, sp, 56
 ; RV32I-NEXT:    addi a1, sp, 40
 ; RV32I-NEXT:    addi a2, sp, 24
+; RV32I-NEXT:    sw a6, 44(sp)
 ; RV32I-NEXT:    call __addtf3
-; RV32I-NEXT:    lw a0, 68(sp)
-; RV32I-NEXT:    sw a0, 20(sp)
-; RV32I-NEXT:    lw a0, 64(sp)
-; RV32I-NEXT:    sw a0, 16(sp)
+; RV32I-NEXT:    lw a1, 56(sp)
 ; RV32I-NEXT:    lw a0, 60(sp)
+; RV32I-NEXT:    lw a2, 64(sp)
+; RV32I-NEXT:    lw a3, 68(sp)
+; RV32I-NEXT:    sw a3, 20(sp)
+; RV32I-NEXT:    sw a2, 16(sp)
 ; RV32I-NEXT:    sw a0, 12(sp)
-; RV32I-NEXT:    lw a0, 56(sp)
-; RV32I-NEXT:    sw a0, 8(sp)
 ; RV32I-NEXT:    addi a0, sp, 8
+; RV32I-NEXT:    sw a1, 8(sp)
 ; RV32I-NEXT:    call __fixtfsi
 ; RV32I-NEXT:    lw ra, 76(sp)
 ; RV32I-NEXT:    addi sp, sp, 80
diff --git a/llvm/test/CodeGen/RISCV/fp16-promote.ll b/llvm/test/CodeGen/RISCV/fp16-promote.ll
new file mode 100644
index 000000000..c99fc1245
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/fp16-promote.ll
@@ -0,0 +1,142 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr +d -target-abi ilp32d < %s | FileCheck %s
+
+define void @test_load_store(half* %p, half* %q) nounwind {
+; CHECK-LABEL: test_load_store:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lh a0, 0(a0)
+; CHECK-NEXT:    sh a0, 0(a1)
+; CHECK-NEXT:    ret
+  %a = load half, half* %p
+  store half %a, half* %q
+  ret void
+}
+
+define float @test_fpextend_float(half* %p) nounwind {
+; CHECK-LABEL: test_fpextend_float:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    sw ra, 12(sp)
+; CHECK-NEXT:    lhu a0, 0(a0)
+; CHECK-NEXT:    call __gnu_h2f_ieee
+; CHECK-NEXT:    lw ra, 12(sp)
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+  %a = load half, half* %p
+  %r = fpext half %a to float
+  ret float %r
+}
+
+define double @test_fpextend_double(half* %p) nounwind {
+; CHECK-LABEL: test_fpextend_double:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    sw ra, 12(sp)
+; CHECK-NEXT:    lhu a0, 0(a0)
+; CHECK-NEXT:    call __gnu_h2f_ieee
+; CHECK-NEXT:    fcvt.d.s fa0, fa0
+; CHECK-NEXT:    lw ra, 12(sp)
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+  %a = load half, half* %p
+  %r = fpext half %a to double
+  ret double %r
+}
+
+define void @test_fptrunc_float(float %f, half* %p) nounwind {
+; CHECK-LABEL: test_fptrunc_float:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    sw ra, 12(sp)
+; CHECK-NEXT:    sw s0, 8(sp)
+; CHECK-NEXT:    mv s0, a0
+; CHECK-NEXT:    call __gnu_f2h_ieee
+; CHECK-NEXT:    sh a0, 0(s0)
+; CHECK-NEXT:    lw s0, 8(sp)
+; CHECK-NEXT:    lw ra, 12(sp)
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+  %a = fptrunc float %f to half
+  store half %a, half* %p
+  ret void
+}
+
+define void @test_fptrunc_double(double %d, half* %p) nounwind {
+; CHECK-LABEL: test_fptrunc_double:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    sw ra, 12(sp)
+; CHECK-NEXT:    sw s0, 8(sp)
+; CHECK-NEXT:    mv s0, a0
+; CHECK-NEXT:    call __truncdfhf2
+; CHECK-NEXT:    sh a0, 0(s0)
+; CHECK-NEXT:    lw s0, 8(sp)
+; CHECK-NEXT:    lw ra, 12(sp)
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+  %a = fptrunc double %d to half
+  store half %a, half* %p
+  ret void
+}
+
+define void @test_fadd(half* %p, half* %q) nounwind {
+; CHECK-LABEL: test_fadd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -32
+; CHECK-NEXT:    sw ra, 28(sp)
+; CHECK-NEXT:    sw s0, 24(sp)
+; CHECK-NEXT:    sw s1, 20(sp)
+; CHECK-NEXT:    fsd fs0, 8(sp)
+; CHECK-NEXT:    mv s0, a1
+; CHECK-NEXT:    mv s1, a0
+; CHECK-NEXT:    lhu a0, 0(a0)
+; CHECK-NEXT:    call __gnu_h2f_ieee
+; CHECK-NEXT:    fmv.s fs0, fa0
+; CHECK-NEXT:    lhu a0, 0(s0)
+; CHECK-NEXT:    call __gnu_h2f_ieee
+; CHECK-NEXT:    fadd.s fa0, fs0, fa0
+; CHECK-NEXT:    call __gnu_f2h_ieee
+; CHECK-NEXT:    sh a0, 0(s1)
+; CHECK-NEXT:    fld fs0, 8(sp)
+; CHECK-NEXT:    lw s1, 20(sp)
+; CHECK-NEXT:    lw s0, 24(sp)
+; CHECK-NEXT:    lw ra, 28(sp)
+; CHECK-NEXT:    addi sp, sp, 32
+; CHECK-NEXT:    ret
+  %a = load half, half* %p
+  %b = load half, half* %q
+  %r = fadd half %a, %b
+  store half %r, half* %p
+  ret void
+}
+
+define void @test_fmul(half* %p, half* %q) nounwind {
+; CHECK-LABEL: test_fmul:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -32
+; CHECK-NEXT:    sw ra, 28(sp)
+; CHECK-NEXT:    sw s0, 24(sp)
+; CHECK-NEXT:    sw s1, 20(sp)
+; CHECK-NEXT:    fsd fs0, 8(sp)
+; CHECK-NEXT:    mv s0, a1
+; CHECK-NEXT:    mv s1, a0
+; CHECK-NEXT:    lhu a0, 0(a0)
+; CHECK-NEXT:    call __gnu_h2f_ieee
+; CHECK-NEXT:    fmv.s fs0, fa0
+; CHECK-NEXT:    lhu a0, 0(s0)
+; CHECK-NEXT:    call __gnu_h2f_ieee
+; CHECK-NEXT:    fmul.s fa0, fs0, fa0
+; CHECK-NEXT:    call __gnu_f2h_ieee
+; CHECK-NEXT:    sh a0, 0(s1)
+; CHECK-NEXT:    fld fs0, 8(sp)
+; CHECK-NEXT:    lw s1, 20(sp)
+; CHECK-NEXT:    lw s0, 24(sp)
+; CHECK-NEXT:    lw ra, 28(sp)
+; CHECK-NEXT:    addi sp, sp, 32
+; CHECK-NEXT:    ret
+  %a = load half, half* %p
+  %b = load half, half* %q
+  %r = fmul half %a, %b
+  store half %r, half* %p
+  ret void
+}
diff --git a/llvm/test/CodeGen/RISCV/frame-info.ll b/llvm/test/CodeGen/RISCV/frame-info.ll
index 9022fc395..0b1c1bde0 100644
--- a/llvm/test/CodeGen/RISCV/frame-info.ll
+++ b/llvm/test/CodeGen/RISCV/frame-info.ll
@@ -1,9 +1,54 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=riscv32 < %s | FileCheck -check-prefix=RV32 %s
 ; RUN: llc -mtriple=riscv64 < %s | FileCheck -check-prefix=RV64 %s
+; RUN: llc -mtriple=riscv32 -frame-pointer=all -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV32-WITHFP %s
+; RUN: llc -mtriple=riscv64 -frame-pointer=all -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64-WITHFP %s
 
-define void @foo(i32 signext %size) {
-; RV32-LABEL: foo:
+define void @trivial() {
+; RV32-LABEL: trivial:
+; RV32:       # %bb.0:
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: trivial:
+; RV64:       # %bb.0:
+; RV64-NEXT:    ret
+;
+; RV32-WITHFP-LABEL: trivial:
+; RV32-WITHFP:       # %bb.0:
+; RV32-WITHFP-NEXT:    addi sp, sp, -16
+; RV32-WITHFP-NEXT:    .cfi_def_cfa_offset 16
+; RV32-WITHFP-NEXT:    sw ra, 12(sp)
+; RV32-WITHFP-NEXT:    sw s0, 8(sp)
+; RV32-WITHFP-NEXT:    .cfi_offset ra, -4
+; RV32-WITHFP-NEXT:    .cfi_offset s0, -8
+; RV32-WITHFP-NEXT:    addi s0, sp, 16
+; RV32-WITHFP-NEXT:    .cfi_def_cfa s0, 0
+; RV32-WITHFP-NEXT:    lw s0, 8(sp)
+; RV32-WITHFP-NEXT:    lw ra, 12(sp)
+; RV32-WITHFP-NEXT:    addi sp, sp, 16
+; RV32-WITHFP-NEXT:    ret
+;
+; RV64-WITHFP-LABEL: trivial:
+; RV64-WITHFP:       # %bb.0:
+; RV64-WITHFP-NEXT:    addi sp, sp, -16
+; RV64-WITHFP-NEXT:    .cfi_def_cfa_offset 16
+; RV64-WITHFP-NEXT:    sd ra, 8(sp)
+; RV64-WITHFP-NEXT:    sd s0, 0(sp)
+; RV64-WITHFP-NEXT:    .cfi_offset ra, -8
+; RV64-WITHFP-NEXT:    .cfi_offset s0, -16
+; RV64-WITHFP-NEXT:    addi s0, sp, 16
+; RV64-WITHFP-NEXT:    .cfi_def_cfa s0, 0
+; RV64-WITHFP-NEXT:    ld s0, 0(sp)
+; RV64-WITHFP-NEXT:    ld ra, 8(sp)
+; RV64-WITHFP-NEXT:    addi sp, sp, 16
+; RV64-WITHFP-NEXT:    ret
+  ret void
+}
+
+define void @stack_alloc(i32 signext %size) {
+; RV32-LABEL: stack_alloc:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
@@ -17,18 +62,14 @@ define void @foo(i32 signext %size) {
 ; RV32-NEXT:    andi a0, a0, -16
 ; RV32-NEXT:    sub a0, sp, a0
 ; RV32-NEXT:    mv sp, a0
-; RV32-NEXT:    call bar
+; RV32-NEXT:    call callee_with_args
 ; RV32-NEXT:    addi sp, s0, -16
 ; RV32-NEXT:    lw s0, 8(sp)
-; RV32-NEXT:    .cfi_def_cfa sp, 16
 ; RV32-NEXT:    lw ra, 12(sp)
-; RV32-NEXT:    .cfi_restore ra
-; RV32-NEXT:    .cfi_restore s0
 ; RV32-NEXT:    addi sp, sp, 16
-; RV32-NEXT:    .cfi_def_cfa_offset 0
 ; RV32-NEXT:    ret
 ;
-; RV64-LABEL: foo:
+; RV64-LABEL: stack_alloc:
 ; RV64:       # %bb.0: # %entry
 ; RV64-NEXT:    addi sp, sp, -16
 ; RV64-NEXT:    .cfi_def_cfa_offset 16
@@ -38,29 +79,167 @@ define void @foo(i32 signext %size) {
 ; RV64-NEXT:    .cfi_offset s0, -16
 ; RV64-NEXT:    addi s0, sp, 16
 ; RV64-NEXT:    .cfi_def_cfa s0, 0
-; RV64-NEXT:    addi a1, zero, 1
-; RV64-NEXT:    slli a1, a1, 33
-; RV64-NEXT:    addi a1, a1, -16
 ; RV64-NEXT:    slli a0, a0, 32
 ; RV64-NEXT:    srli a0, a0, 32
 ; RV64-NEXT:    addi a0, a0, 15
+; RV64-NEXT:    addi a1, zero, 1
+; RV64-NEXT:    slli a1, a1, 33
+; RV64-NEXT:    addi a1, a1, -16
 ; RV64-NEXT:    and a0, a0, a1
 ; RV64-NEXT:    sub a0, sp, a0
 ; RV64-NEXT:    mv sp, a0
-; RV64-NEXT:    call bar
+; RV64-NEXT:    call callee_with_args
 ; RV64-NEXT:    addi sp, s0, -16
 ; RV64-NEXT:    ld s0, 0(sp)
-; RV64-NEXT:    .cfi_def_cfa sp, 16
 ; RV64-NEXT:    ld ra, 8(sp)
-; RV64-NEXT:    .cfi_restore ra
-; RV64-NEXT:    .cfi_restore s0
 ; RV64-NEXT:    addi sp, sp, 16
-; RV64-NEXT:    .cfi_def_cfa_offset 0
 ; RV64-NEXT:    ret
+;
+; RV32-WITHFP-LABEL: stack_alloc:
+; RV32-WITHFP:       # %bb.0: # %entry
+; RV32-WITHFP-NEXT:    addi sp, sp, -16
+; RV32-WITHFP-NEXT:    .cfi_def_cfa_offset 16
+; RV32-WITHFP-NEXT:    sw ra, 12(sp)
+; RV32-WITHFP-NEXT:    sw s0, 8(sp)
+; RV32-WITHFP-NEXT:    .cfi_offset ra, -4
+; RV32-WITHFP-NEXT:    .cfi_offset s0, -8
+; RV32-WITHFP-NEXT:    addi s0, sp, 16
+; RV32-WITHFP-NEXT:    .cfi_def_cfa s0, 0
+; RV32-WITHFP-NEXT:    addi a0, a0, 15
+; RV32-WITHFP-NEXT:    andi a0, a0, -16
+; RV32-WITHFP-NEXT:    sub a0, sp, a0
+; RV32-WITHFP-NEXT:    mv sp, a0
+; RV32-WITHFP-NEXT:    call callee_with_args
+; RV32-WITHFP-NEXT:    addi sp, s0, -16
+; RV32-WITHFP-NEXT:    lw s0, 8(sp)
+; RV32-WITHFP-NEXT:    lw ra, 12(sp)
+; RV32-WITHFP-NEXT:    addi sp, sp, 16
+; RV32-WITHFP-NEXT:    ret
+;
+; RV64-WITHFP-LABEL: stack_alloc:
+; RV64-WITHFP:       # %bb.0: # %entry
+; RV64-WITHFP-NEXT:    addi sp, sp, -16
+; RV64-WITHFP-NEXT:    .cfi_def_cfa_offset 16
+; RV64-WITHFP-NEXT:    sd ra, 8(sp)
+; RV64-WITHFP-NEXT:    sd s0, 0(sp)
+; RV64-WITHFP-NEXT:    .cfi_offset ra, -8
+; RV64-WITHFP-NEXT:    .cfi_offset s0, -16
+; RV64-WITHFP-NEXT:    addi s0, sp, 16
+; RV64-WITHFP-NEXT:    .cfi_def_cfa s0, 0
+; RV64-WITHFP-NEXT:    slli a0, a0, 32
+; RV64-WITHFP-NEXT:    srli a0, a0, 32
+; RV64-WITHFP-NEXT:    addi a0, a0, 15
+; RV64-WITHFP-NEXT:    addi a1, zero, 1
+; RV64-WITHFP-NEXT:    slli a1, a1, 33
+; RV64-WITHFP-NEXT:    addi a1, a1, -16
+; RV64-WITHFP-NEXT:    and a0, a0, a1
+; RV64-WITHFP-NEXT:    sub a0, sp, a0
+; RV64-WITHFP-NEXT:    mv sp, a0
+; RV64-WITHFP-NEXT:    call callee_with_args
+; RV64-WITHFP-NEXT:    addi sp, s0, -16
+; RV64-WITHFP-NEXT:    ld s0, 0(sp)
+; RV64-WITHFP-NEXT:    ld ra, 8(sp)
+; RV64-WITHFP-NEXT:    addi sp, sp, 16
+; RV64-WITHFP-NEXT:    ret
 entry:
   %0 = alloca i8, i32 %size, align 16
-  call void @bar(i8* nonnull %0) #2
+  call void @callee_with_args(i8* nonnull %0)
+  ret void
+}
+
+define void @branch_and_tail_call(i1 %a) {
+; RV32-LABEL: branch_and_tail_call:
+; RV32:       # %bb.0:
+; RV32-NEXT:    addi sp, sp, -16
+; RV32-NEXT:    .cfi_def_cfa_offset 16
+; RV32-NEXT:    sw ra, 12(sp)
+; RV32-NEXT:    .cfi_offset ra, -4
+; RV32-NEXT:    andi a0, a0, 1
+; RV32-NEXT:    beqz a0, .LBB2_2
+; RV32-NEXT:  # %bb.1: # %blue_pill
+; RV32-NEXT:    lw ra, 12(sp)
+; RV32-NEXT:    addi sp, sp, 16
+; RV32-NEXT:    tail callee1
+; RV32-NEXT:  .LBB2_2: # %red_pill
+; RV32-NEXT:    call callee2
+; RV32-NEXT:    lw ra, 12(sp)
+; RV32-NEXT:    addi sp, sp, 16
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: branch_and_tail_call:
+; RV64:       # %bb.0:
+; RV64-NEXT:    addi sp, sp, -16
+; RV64-NEXT:    .cfi_def_cfa_offset 16
+; RV64-NEXT:    sd ra, 8(sp)
+; RV64-NEXT:    .cfi_offset ra, -8
+; RV64-NEXT:    andi a0, a0, 1
+; RV64-NEXT:    beqz a0, .LBB2_2
+; RV64-NEXT:  # %bb.1: # %blue_pill
+; RV64-NEXT:    ld ra, 8(sp)
+; RV64-NEXT:    addi sp, sp, 16
+; RV64-NEXT:    tail callee1
+; RV64-NEXT:  .LBB2_2: # %red_pill
+; RV64-NEXT:    call callee2
+; RV64-NEXT:    ld ra, 8(sp)
+; RV64-NEXT:    addi sp, sp, 16
+; RV64-NEXT:    ret
+;
+; RV32-WITHFP-LABEL: branch_and_tail_call:
+; RV32-WITHFP:       # %bb.0:
+; RV32-WITHFP-NEXT:    addi sp, sp, -16
+; RV32-WITHFP-NEXT:    .cfi_def_cfa_offset 16
+; RV32-WITHFP-NEXT:    sw ra, 12(sp)
+; RV32-WITHFP-NEXT:    sw s0, 8(sp)
+; RV32-WITHFP-NEXT:    .cfi_offset ra, -4
+; RV32-WITHFP-NEXT:    .cfi_offset s0, -8
+; RV32-WITHFP-NEXT:    addi s0, sp, 16
+; RV32-WITHFP-NEXT:    .cfi_def_cfa s0, 0
+; RV32-WITHFP-NEXT:    andi a0, a0, 1
+; RV32-WITHFP-NEXT:    beqz a0, .LBB2_2
+; RV32-WITHFP-NEXT:  # %bb.1: # %blue_pill
+; RV32-WITHFP-NEXT:    lw s0, 8(sp)
+; RV32-WITHFP-NEXT:    lw ra, 12(sp)
+; RV32-WITHFP-NEXT:    addi sp, sp, 16
+; RV32-WITHFP-NEXT:    tail callee1
+; RV32-WITHFP-NEXT:  .LBB2_2: # %red_pill
+; RV32-WITHFP-NEXT:    call callee2
+; RV32-WITHFP-NEXT:    lw s0, 8(sp)
+; RV32-WITHFP-NEXT:    lw ra, 12(sp)
+; RV32-WITHFP-NEXT:    addi sp, sp, 16
+; RV32-WITHFP-NEXT:    ret
+;
+; RV64-WITHFP-LABEL: branch_and_tail_call:
+; RV64-WITHFP:       # %bb.0:
+; RV64-WITHFP-NEXT:    addi sp, sp, -16
+; RV64-WITHFP-NEXT:    .cfi_def_cfa_offset 16
+; RV64-WITHFP-NEXT:    sd ra, 8(sp)
+; RV64-WITHFP-NEXT:    sd s0, 0(sp)
+; RV64-WITHFP-NEXT:    .cfi_offset ra, -8
+; RV64-WITHFP-NEXT:    .cfi_offset s0, -16
+; RV64-WITHFP-NEXT:    addi s0, sp, 16
+; RV64-WITHFP-NEXT:    .cfi_def_cfa s0, 0
+; RV64-WITHFP-NEXT:    andi a0, a0, 1
+; RV64-WITHFP-NEXT:    beqz a0, .LBB2_2
+; RV64-WITHFP-NEXT:  # %bb.1: # %blue_pill
+; RV64-WITHFP-NEXT:    ld s0, 0(sp)
+; RV64-WITHFP-NEXT:    ld ra, 8(sp)
+; RV64-WITHFP-NEXT:    addi sp, sp, 16
+; RV64-WITHFP-NEXT:    tail callee1
+; RV64-WITHFP-NEXT:  .LBB2_2: # %red_pill
+; RV64-WITHFP-NEXT:    call callee2
+; RV64-WITHFP-NEXT:    ld s0, 0(sp)
+; RV64-WITHFP-NEXT:    ld ra, 8(sp)
+; RV64-WITHFP-NEXT:    addi sp, sp, 16
+; RV64-WITHFP-NEXT:    ret
+  br i1 %a, label %blue_pill, label %red_pill
+blue_pill:
+  tail call void @callee1()
+  ret void
+red_pill:
+  call void @callee2()
   ret void
 }
 
-declare void @bar(i8*)
+declare void @callee1()
+declare void @callee2()
+declare void @callee_with_args(i8*)
diff --git a/llvm/test/CodeGen/RISCV/frame.ll b/llvm/test/CodeGen/RISCV/frame.ll
index e8a833f37..6f6be73e7 100644
--- a/llvm/test/CodeGen/RISCV/frame.ll
+++ b/llvm/test/CodeGen/RISCV/frame.ll
@@ -45,7 +45,7 @@ define i32 @test() nounwind {
   %1 = bitcast %struct.key_t* %key to i8*
   call void @llvm.memset.p0i8.i64(i8* align 4 %1, i8 0, i64 20, i1 false)
   %2 = getelementptr inbounds %struct.key_t, %struct.key_t* %key, i64 0, i32 1, i64 0
-  call void @test1(i8* %2) #3
+  call void @test1(i8* %2)
   ret i32 0
 }
 
diff --git a/llvm/test/CodeGen/RISCV/get-register-invalid.ll b/llvm/test/CodeGen/RISCV/get-register-invalid.ll
new file mode 100644
index 000000000..ee8ec44ce
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/get-register-invalid.ll
@@ -0,0 +1,12 @@
+; RUN: not llc < %s -mtriple=riscv32 2>&1 | FileCheck %s
+
+define i32 @get_invalid_reg() nounwind {
+entry:
+; CHECK: Invalid register name "notareg".
+  %reg = call i32 @llvm.read_register.i32(metadata !0)
+  ret i32 %reg
+}
+
+declare i32 @llvm.read_register.i32(metadata) nounwind
+
+!0 = !{!"notareg\00"}
diff --git a/llvm/test/CodeGen/RISCV/get-register-noreserve.ll b/llvm/test/CodeGen/RISCV/get-register-noreserve.ll
new file mode 100644
index 000000000..b5b8e8ed4
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/get-register-noreserve.ll
@@ -0,0 +1,38 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=riscv32 | FileCheck %s
+
+define i32 @get_stack() nounwind {
+; CHECK-LABEL: get_stack:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    mv a0, sp
+; CHECK-NEXT:    ret
+entry:
+  %sp = call i32 @llvm.read_register.i32(metadata !0)
+  ret i32 %sp
+}
+
+define void @set_stack(i32 %val) nounwind {
+; CHECK-LABEL: set_stack:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    mv sp, a0
+; CHECK-NEXT:    ret
+entry:
+  call void @llvm.write_register.i32(metadata !0, i32 %val)
+  ret void
+}
+
+define i32 @get_tp_arch_name() nounwind {
+; CHECK-LABEL: get_tp_arch_name:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    mv a0, tp
+; CHECK-NEXT:    ret
+entry:
+  %sp = call i32 @llvm.read_register.i32(metadata !1)
+  ret i32 %sp
+}
+
+declare i32 @llvm.read_register.i32(metadata) nounwind
+declare void @llvm.write_register.i32(metadata, i32) nounwind
+
+!0 = !{!"sp\00"}
+!1 = !{!"x4\00"}
diff --git a/llvm/test/CodeGen/RISCV/get-register-reserve.ll b/llvm/test/CodeGen/RISCV/get-register-reserve.ll
new file mode 100644
index 000000000..7549b4dd3
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/get-register-reserve.ll
@@ -0,0 +1,34 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: not llc < %s -mtriple=riscv32 -mattr +reserve-x8 2>&1 \
+; RUN:   | FileCheck -check-prefix=NO-RESERVE-A1 %s
+; RUN: not llc < %s -mtriple=riscv32 -mattr +reserve-x11 2>&1 \
+; RUN:   | FileCheck -check-prefix=NO-RESERVE-FP %s
+; RUN: llc < %s -mtriple=riscv32 -mattr +reserve-x8 -mattr +reserve-x11 \
+; RUN:   | FileCheck -check-prefix=RESERVE %s
+
+define i32 @get_reg_a1() nounwind {
+; NO-RESERVE-A1: Trying to obtain non-reserved register "a1".
+; RESERVE-LABEL: get_reg_a1:
+; RESERVE:       # %bb.0: # %entry
+; RESERVE-NEXT:    mv a0, a1
+; RESERVE-NEXT:    ret
+entry:
+  %a1 = call i32 @llvm.read_register.i32(metadata !0)
+  ret i32 %a1
+}
+
+define i32 @get_reg_fp() nounwind {
+; NO-RESERVE-FP: Trying to obtain non-reserved register "fp".
+; RESERVE-LABEL: get_reg_fp:
+; RESERVE:       # %bb.0: # %entry
+; RESERVE-NEXT:    mv a0, s0
+; RESERVE-NEXT:    ret
+entry:
+  %fp = call i32 @llvm.read_register.i32(metadata !1)
+  ret i32 %fp
+}
+
+declare i32 @llvm.read_register.i32(metadata) nounwind
+
+!0 = !{!"a1\00"}
+!1 = !{!"fp\00"}
diff --git a/llvm/test/CodeGen/RISCV/get-setcc-result-type.ll b/llvm/test/CodeGen/RISCV/get-setcc-result-type.ll
index fe326b383..21923f076 100644
--- a/llvm/test/CodeGen/RISCV/get-setcc-result-type.ll
+++ b/llvm/test/CodeGen/RISCV/get-setcc-result-type.ll
@@ -6,21 +6,21 @@ define void @getSetCCResultType(<4 x i32>* %p, <4 x i32>* %q) nounwind {
 ; RV32I-LABEL: getSetCCResultType:
 ; RV32I:       # %bb.0: # %entry
 ; RV32I-NEXT:    lw a1, 12(a0)
+; RV32I-NEXT:    lw a2, 8(a0)
+; RV32I-NEXT:    lw a3, 4(a0)
+; RV32I-NEXT:    lw a4, 0(a0)
 ; RV32I-NEXT:    seqz a1, a1
+; RV32I-NEXT:    seqz a2, a2
+; RV32I-NEXT:    seqz a3, a3
+; RV32I-NEXT:    seqz a4, a4
+; RV32I-NEXT:    neg a4, a4
+; RV32I-NEXT:    neg a3, a3
+; RV32I-NEXT:    neg a2, a2
 ; RV32I-NEXT:    neg a1, a1
 ; RV32I-NEXT:    sw a1, 12(a0)
-; RV32I-NEXT:    lw a1, 8(a0)
-; RV32I-NEXT:    seqz a1, a1
-; RV32I-NEXT:    neg a1, a1
-; RV32I-NEXT:    sw a1, 8(a0)
-; RV32I-NEXT:    lw a1, 4(a0)
-; RV32I-NEXT:    seqz a1, a1
-; RV32I-NEXT:    neg a1, a1
-; RV32I-NEXT:    sw a1, 4(a0)
-; RV32I-NEXT:    lw a1, 0(a0)
-; RV32I-NEXT:    seqz a1, a1
-; RV32I-NEXT:    neg a1, a1
-; RV32I-NEXT:    sw a1, 0(a0)
+; RV32I-NEXT:    sw a2, 8(a0)
+; RV32I-NEXT:    sw a3, 4(a0)
+; RV32I-NEXT:    sw a4, 0(a0)
 ; RV32I-NEXT:    ret
 entry:
   %0 = load <4 x i32>, <4 x i32>* %p, align 16
diff --git a/llvm/test/CodeGen/RISCV/hoist-global-addr-base.ll b/llvm/test/CodeGen/RISCV/hoist-global-addr-base.ll
index b00873ce4..df8b31170 100644
--- a/llvm/test/CodeGen/RISCV/hoist-global-addr-base.ll
+++ b/llvm/test/CodeGen/RISCV/hoist-global-addr-base.ll
@@ -12,10 +12,10 @@ define dso_local void @multiple_stores() local_unnamed_addr nounwind {
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    lui a0, %hi(s)
 ; CHECK-NEXT:    addi a0, a0, %lo(s)
-; CHECK-NEXT:    addi a1, zero, 20
-; CHECK-NEXT:    sw a1, 164(a0)
 ; CHECK-NEXT:    addi a1, zero, 10
 ; CHECK-NEXT:    sw a1, 160(a0)
+; CHECK-NEXT:    addi a1, zero, 20
+; CHECK-NEXT:    sw a1, 164(a0)
 ; CHECK-NEXT:    ret
 entry:
   store i32 10, i32* getelementptr inbounds (%struct.S, %struct.S* @s, i32 0, i32 1), align 4
diff --git a/llvm/test/CodeGen/RISCV/imm-cse.ll b/llvm/test/CodeGen/RISCV/imm-cse.ll
index 1af27f087..a1ddcd644 100644
--- a/llvm/test/CodeGen/RISCV/imm-cse.ll
+++ b/llvm/test/CodeGen/RISCV/imm-cse.ll
@@ -10,19 +10,19 @@
 define void @imm32_cse() nounwind {
 ; RV32I-LABEL: imm32_cse:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    lui a0, 1
-; RV32I-NEXT:    addi a0, a0, 1
-; RV32I-NEXT:    lui a1, %hi(src)
-; RV32I-NEXT:    lw a2, %lo(src)(a1)
-; RV32I-NEXT:    add a2, a2, a0
-; RV32I-NEXT:    lui a3, %hi(dst)
-; RV32I-NEXT:    sw a2, %lo(dst)(a3)
-; RV32I-NEXT:    lw a2, %lo(src)(a1)
-; RV32I-NEXT:    add a2, a2, a0
+; RV32I-NEXT:    lui a0, %hi(src)
+; RV32I-NEXT:    lw a1, %lo(src)(a0)
+; RV32I-NEXT:    lui a2, 1
 ; RV32I-NEXT:    addi a2, a2, 1
-; RV32I-NEXT:    sw a2, %lo(dst)(a3)
-; RV32I-NEXT:    lw a1, %lo(src)(a1)
-; RV32I-NEXT:    add a0, a1, a0
+; RV32I-NEXT:    add a1, a1, a2
+; RV32I-NEXT:    lui a3, %hi(dst)
+; RV32I-NEXT:    sw a1, %lo(dst)(a3)
+; RV32I-NEXT:    lw a1, %lo(src)(a0)
+; RV32I-NEXT:    add a1, a1, a2
+; RV32I-NEXT:    addi a1, a1, 1
+; RV32I-NEXT:    sw a1, %lo(dst)(a3)
+; RV32I-NEXT:    lw a0, %lo(src)(a0)
+; RV32I-NEXT:    add a0, a0, a2
 ; RV32I-NEXT:    addi a0, a0, 2
 ; RV32I-NEXT:    sw a0, %lo(dst)(a3)
 ; RV32I-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/imm.ll b/llvm/test/CodeGen/RISCV/imm.ll
index b733b5921..ce254f2b1 100644
--- a/llvm/test/CodeGen/RISCV/imm.ll
+++ b/llvm/test/CodeGen/RISCV/imm.ll
@@ -142,8 +142,8 @@ define i64 @imm64_2() nounwind {
 define i64 @imm64_3() nounwind {
 ; RV32I-LABEL: imm64_3:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    mv a0, zero
 ; RV32I-NEXT:    addi a1, zero, 1
+; RV32I-NEXT:    mv a0, zero
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: imm64_3:
@@ -157,8 +157,8 @@ define i64 @imm64_3() nounwind {
 define i64 @imm64_4() nounwind {
 ; RV32I-LABEL: imm64_4:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    mv a0, zero
 ; RV32I-NEXT:    lui a1, 524288
+; RV32I-NEXT:    mv a0, zero
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: imm64_4:
@@ -172,8 +172,8 @@ define i64 @imm64_4() nounwind {
 define i64 @imm64_5() nounwind {
 ; RV32I-LABEL: imm64_5:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    mv a0, zero
 ; RV32I-NEXT:    lui a1, 524288
+; RV32I-NEXT:    mv a0, zero
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: imm64_5:
@@ -249,7 +249,7 @@ define i64 @imm64_9() nounwind {
 ; RV32I-LABEL: imm64_9:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi a0, zero, -1
-; RV32I-NEXT:    addi  a1, zero, -1
+; RV32I-NEXT:    addi a1, zero, -1
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: imm64_9:
diff --git a/llvm/test/CodeGen/RISCV/indirectbr.ll b/llvm/test/CodeGen/RISCV/indirectbr.ll
index e734de3c8..01201e818 100644
--- a/llvm/test/CodeGen/RISCV/indirectbr.ll
+++ b/llvm/test/CodeGen/RISCV/indirectbr.ll
@@ -8,7 +8,7 @@ define i32 @indirectbr(i8* %target) nounwind {
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp)
 ; RV32I-NEXT:    jr a0
-; RV32I-NEXT:  .LBB0_1:
+; RV32I-NEXT:  .LBB0_1: # %test_label
 ; RV32I-NEXT:    mv a0, zero
 ; RV32I-NEXT:    lw ra, 12(sp)
 ; RV32I-NEXT:    addi sp, sp, 16
@@ -26,7 +26,7 @@ define i32 @indirectbr_with_offset(i8* %a) nounwind {
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp)
 ; RV32I-NEXT:    jr 1380(a0)
-; RV32I-NEXT:  .LBB1_1:
+; RV32I-NEXT:  .LBB1_1: # %test_label
 ; RV32I-NEXT:    mv a0, zero
 ; RV32I-NEXT:    lw ra, 12(sp)
 ; RV32I-NEXT:    addi sp, sp, 16
diff --git a/llvm/test/CodeGen/RISCV/inline-asm-abi-names.ll b/llvm/test/CodeGen/RISCV/inline-asm-abi-names.ll
new file mode 100644
index 000000000..4d85e3ea0
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/inline-asm-abi-names.ll
@@ -0,0 +1,1590 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV32I %s
+; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64I %s
+
+; These test that we can use both the architectural names (x*) and the ABI names
+; (a*, s*, t* etc) to refer to registers in inline asm constraint lists. In each
+; case, the named register should be used for the source register of the `addi`.
+; It is very likely that `a0` will be chosen as the designation register, but
+; this is left to the compiler to choose.
+;
+; The inline assembly will, by default, contain the ABI names for the registers.
+;
+; Parenthesised registers in comments are the other aliases for this register.
+
+; NOTE: This test has to pass in 0 to the inline asm, because that's the only
+; value `x0` (`zero`) can take.
+define i32 @explicit_register_x0() nounwind {
+; RV32I-LABEL: explicit_register_x0:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    addi a0, zero, 0
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: explicit_register_x0:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    addi a0, zero, 0
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ret
+  %1 = tail call i32 asm "addi $0, $1, 0", "=r,{x0}"(i32 0)
+  ret i32 %1
+}
+
+; NOTE: This test has to pass in 0 to the inline asm, because that's the only
+; value that `zero` (`x0`) can take.
+define i32 @explicit_register_zero() nounwind {
+; RV32I-LABEL: explicit_register_zero:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    addi a0, zero, 0
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: explicit_register_zero:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    addi a0, zero, 0
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ret
+  %1 = tail call i32 asm "addi $0, $1, 0", "=r,{zero}"(i32 0)
+  ret i32 %1
+}
+
+; NOTE: This test uses `x1` (`ra`) as an input, so it should be saved.
+define i32 @explicit_register_x1(i32 %a) nounwind {
+; RV32I-LABEL: explicit_register_x1:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    mv ra, a0
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    addi a0, ra, 0
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: explicit_register_x1:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    mv ra, a0
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    addi a0, ra, 0
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = tail call i32 asm "addi $0, $1, 0", "=r,{x1}"(i32 %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `ra` (`x1`) as an input, so it should be saved.
+define i32 @explicit_register_ra(i32 %a) nounwind {
+; RV32I-LABEL: explicit_register_ra:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    mv ra, a0
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    addi a0, ra, 0
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: explicit_register_ra:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    mv ra, a0
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    addi a0, ra, 0
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = tail call i32 asm "addi $0, $1, 0", "=r,{ra}"(i32 %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_x2(i32 %a) nounwind {
+; RV32I-LABEL: explicit_register_x2:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    mv sp, a0
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    addi a0, sp, 0
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: explicit_register_x2:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    mv sp, a0
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    addi a0, sp, 0
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ret
+  %1 = tail call i32 asm "addi $0, $1, 0", "=r,{x2}"(i32 %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_sp(i32 %a) nounwind {
+; RV32I-LABEL: explicit_register_sp:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    mv sp, a0
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    addi a0, sp, 0
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: explicit_register_sp:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    mv sp, a0
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    addi a0, sp, 0
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ret
+  %1 = tail call i32 asm "addi $0, $1, 0", "=r,{sp}"(i32 %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `x3` (`gp`) as an input, so it should be saved.
+define i32 @explicit_register_x3(i32 %a) nounwind {
+; RV32I-LABEL: explicit_register_x3:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw gp, 12(sp)
+; RV32I-NEXT:    mv gp, a0
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    addi a0, gp, 0
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    lw gp, 12(sp)
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: explicit_register_x3:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd gp, 8(sp)
+; RV64I-NEXT:    mv gp, a0
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    addi a0, gp, 0
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ld gp, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = tail call i32 asm "addi $0, $1, 0", "=r,{x3}"(i32 %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `gp` (`x3`) as an input, so it should be saved.
+define i32 @explicit_register_gp(i32 %a) nounwind {
+; RV32I-LABEL: explicit_register_gp:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw gp, 12(sp)
+; RV32I-NEXT:    mv gp, a0
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    addi a0, gp, 0
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    lw gp, 12(sp)
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: explicit_register_gp:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd gp, 8(sp)
+; RV64I-NEXT:    mv gp, a0
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    addi a0, gp, 0
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ld gp, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = tail call i32 asm "addi $0, $1, 0", "=r,{gp}"(i32 %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `x4` (`tp`) as an input, so it should be saved.
+define i32 @explicit_register_x4(i32 %a) nounwind {
+; RV32I-LABEL: explicit_register_x4:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw tp, 12(sp)
+; RV32I-NEXT:    mv tp, a0
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    addi a0, tp, 0
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    lw tp, 12(sp)
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: explicit_register_x4:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd tp, 8(sp)
+; RV64I-NEXT:    mv tp, a0
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    addi a0, tp, 0
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ld tp, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = tail call i32 asm "addi $0, $1, 0", "=r,{x4}"(i32 %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `tp` (`x4`) as an input, so it should be saved.
+define i32 @explicit_register_tp(i32 %a) nounwind {
+; RV32I-LABEL: explicit_register_tp:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw tp, 12(sp)
+; RV32I-NEXT:    mv tp, a0
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    addi a0, tp, 0
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    lw tp, 12(sp)
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: explicit_register_tp:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd tp, 8(sp)
+; RV64I-NEXT:    mv tp, a0
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    addi a0, tp, 0
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ld tp, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = tail call i32 asm "addi $0, $1, 0", "=r,{tp}"(i32 %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_x5(i32 %a) nounwind {
+; RV32I-LABEL: explicit_register_x5:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    mv t0, a0
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    addi a0, t0, 0
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: explicit_register_x5:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    mv t0, a0
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    addi a0, t0, 0
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ret
+  %1 = tail call i32 asm "addi $0, $1, 0", "=r,{x5}"(i32 %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_t0(i32 %a) nounwind {
+; RV32I-LABEL: explicit_register_t0:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    mv t0, a0
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    addi a0, t0, 0
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: explicit_register_t0:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    mv t0, a0
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    addi a0, t0, 0
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ret
+  %1 = tail call i32 asm "addi $0, $1, 0", "=r,{t0}"(i32 %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_x6(i32 %a) nounwind {
+; RV32I-LABEL: explicit_register_x6:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    mv t1, a0
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    addi a0, t1, 0
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: explicit_register_x6:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    mv t1, a0
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    addi a0, t1, 0
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ret
+  %1 = tail call i32 asm "addi $0, $1, 0", "=r,{x6}"(i32 %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_t1(i32 %a) nounwind {
+; RV32I-LABEL: explicit_register_t1:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    mv t1, a0
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    addi a0, t1, 0
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: explicit_register_t1:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    mv t1, a0
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    addi a0, t1, 0
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ret
+  %1 = tail call i32 asm "addi $0, $1, 0", "=r,{t1}"(i32 %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_x7(i32 %a) nounwind {
+; RV32I-LABEL: explicit_register_x7:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    mv t2, a0
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    addi a0, t2, 0
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: explicit_register_x7:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    mv t2, a0
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    addi a0, t2, 0
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ret
+  %1 = tail call i32 asm "addi $0, $1, 0", "=r,{x7}"(i32 %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_t2(i32 %a) nounwind {
+; RV32I-LABEL: explicit_register_t2:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    mv t2, a0
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    addi a0, t2, 0
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: explicit_register_t2:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    mv t2, a0
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    addi a0, t2, 0
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ret
+  %1 = tail call i32 asm "addi $0, $1, 0", "=r,{t2}"(i32 %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `x8` (`s0`, `fp`) as an input, so it should be saved.
+define i32 @explicit_register_x8(i32 %a) nounwind {
+; RV32I-LABEL: explicit_register_x8:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw s0, 12(sp)
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    addi a0, s0, 0
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    lw s0, 12(sp)
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: explicit_register_x8:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd s0, 8(sp)
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    addi a0, s0, 0
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ld s0, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = tail call i32 asm "addi $0, $1, 0", "=r,{x8}"(i32 %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `s0` (`x8`, `fp`) as an input, so it should be saved.
+define i32 @explicit_register_s0(i32 %a) nounwind {
+; RV32I-LABEL: explicit_register_s0:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw s0, 12(sp)
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    addi a0, s0, 0
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    lw s0, 12(sp)
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: explicit_register_s0:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd s0, 8(sp)
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    addi a0, s0, 0
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ld s0, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = tail call i32 asm "addi $0, $1, 0", "=r,{s0}"(i32 %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `fp` (`x8`, `s0`) as an input, so it should be saved.
+define i32 @explicit_register_fp(i32 %a) nounwind {
+; RV32I-LABEL: explicit_register_fp:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw s0, 12(sp)
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    addi a0, s0, 0
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    lw s0, 12(sp)
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: explicit_register_fp:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd s0, 8(sp)
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    addi a0, s0, 0
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ld s0, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = tail call i32 asm "addi $0, $1, 0", "=r,{fp}"(i32 %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `x9` (`s1`) as an input, so it should be saved.
+define i32 @explicit_register_x9(i32 %a) nounwind {
+; RV32I-LABEL: explicit_register_x9:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw s1, 12(sp)
+; RV32I-NEXT:    mv s1, a0
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    addi a0, s1, 0
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    lw s1, 12(sp)
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: explicit_register_x9:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd s1, 8(sp)
+; RV64I-NEXT:    mv s1, a0
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    addi a0, s1, 0
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ld s1, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = tail call i32 asm "addi $0, $1, 0", "=r,{x9}"(i32 %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `s1` (`x9`) as an input, so it should be saved.
+define i32 @explicit_register_s1(i32 %a) nounwind {
+; RV32I-LABEL: explicit_register_s1:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw s1, 12(sp)
+; RV32I-NEXT:    mv s1, a0
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    addi a0, s1, 0
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    lw s1, 12(sp)
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: explicit_register_s1:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd s1, 8(sp)
+; RV64I-NEXT:    mv s1, a0
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    addi a0, s1, 0
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ld s1, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = tail call i32 asm "addi $0, $1, 0", "=r,{s1}"(i32 %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_x10(i32 %a) nounwind {
+; RV32I-LABEL: explicit_register_x10:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    addi a0, a0, 0
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: explicit_register_x10:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    addi a0, a0, 0
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ret
+  %1 = tail call i32 asm "addi $0, $1, 0", "=r,{x10}"(i32 %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_a0(i32 %a) nounwind {
+; RV32I-LABEL: explicit_register_a0:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    addi a0, a0, 0
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: explicit_register_a0:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    addi a0, a0, 0
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ret
+  %1 = tail call i32 asm "addi $0, $1, 0", "=r,{a0}"(i32 %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_x11(i32 %a) nounwind {
+; RV32I-LABEL: explicit_register_x11:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    mv a1, a0
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    addi a0, a1, 0
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: explicit_register_x11:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    addi a0, a1, 0
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ret
+  %1 = tail call i32 asm "addi $0, $1, 0", "=r,{x11}"(i32 %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_a1(i32 %a) nounwind {
+; RV32I-LABEL: explicit_register_a1:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    mv a1, a0
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    addi a0, a1, 0
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: explicit_register_a1:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    addi a0, a1, 0
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ret
+  %1 = tail call i32 asm "addi $0, $1, 0", "=r,{a1}"(i32 %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_x12(i32 %a) nounwind {
+; RV32I-LABEL: explicit_register_x12:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    mv a2, a0
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    addi a0, a2, 0
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: explicit_register_x12:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    addi a0, a2, 0
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ret
+  %1 = tail call i32 asm "addi $0, $1, 0", "=r,{x12}"(i32 %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_a2(i32 %a) nounwind {
+; RV32I-LABEL: explicit_register_a2:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    mv a2, a0
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    addi a0, a2, 0
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: explicit_register_a2:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    addi a0, a2, 0
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ret
+  %1 = tail call i32 asm "addi $0, $1, 0", "=r,{a2}"(i32 %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_x13(i32 %a) nounwind {
+; RV32I-LABEL: explicit_register_x13:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    mv a3, a0
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    addi a0, a3, 0
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: explicit_register_x13:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    mv a3, a0
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    addi a0, a3, 0
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ret
+  %1 = tail call i32 asm "addi $0, $1, 0", "=r,{x13}"(i32 %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_a3(i32 %a) nounwind {
+; RV32I-LABEL: explicit_register_a3:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    mv a3, a0
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    addi a0, a3, 0
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: explicit_register_a3:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    mv a3, a0
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    addi a0, a3, 0
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ret
+  %1 = tail call i32 asm "addi $0, $1, 0", "=r,{a3}"(i32 %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_x14(i32 %a) nounwind {
+; RV32I-LABEL: explicit_register_x14:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    mv a4, a0
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    addi a0, a4, 0
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: explicit_register_x14:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    mv a4, a0
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    addi a0, a4, 0
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ret
+  %1 = tail call i32 asm "addi $0, $1, 0", "=r,{x14}"(i32 %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_a4(i32 %a) nounwind {
+; RV32I-LABEL: explicit_register_a4:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    mv a4, a0
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    addi a0, a4, 0
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: explicit_register_a4:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    mv a4, a0
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    addi a0, a4, 0
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ret
+  %1 = tail call i32 asm "addi $0, $1, 0", "=r,{a4}"(i32 %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_x15(i32 %a) nounwind {
+; RV32I-LABEL: explicit_register_x15:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    mv a5, a0
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    addi a0, a5, 0
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: explicit_register_x15:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    mv a5, a0
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    addi a0, a5, 0
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ret
+  %1 = tail call i32 asm "addi $0, $1, 0", "=r,{x15}"(i32 %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_a5(i32 %a) nounwind {
+; RV32I-LABEL: explicit_register_a5:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    mv a5, a0
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    addi a0, a5, 0
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: explicit_register_a5:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    mv a5, a0
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    addi a0, a5, 0
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ret
+  %1 = tail call i32 asm "addi $0, $1, 0", "=r,{a5}"(i32 %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_x16(i32 %a) nounwind {
+; RV32I-LABEL: explicit_register_x16:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    mv a6, a0
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    addi a0, a6, 0
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: explicit_register_x16:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    mv a6, a0
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    addi a0, a6, 0
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ret
+  %1 = tail call i32 asm "addi $0, $1, 0", "=r,{x16}"(i32 %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_a6(i32 %a) nounwind {
+; RV32I-LABEL: explicit_register_a6:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    mv a6, a0
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    addi a0, a6, 0
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: explicit_register_a6:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    mv a6, a0
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    addi a0, a6, 0
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ret
+  %1 = tail call i32 asm "addi $0, $1, 0", "=r,{a6}"(i32 %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_x17(i32 %a) nounwind {
+; RV32I-LABEL: explicit_register_x17:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    mv a7, a0
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    addi a0, a7, 0
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: explicit_register_x17:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    mv a7, a0
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    addi a0, a7, 0
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ret
+  %1 = tail call i32 asm "addi $0, $1, 0", "=r,{x17}"(i32 %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_a7(i32 %a) nounwind {
+; RV32I-LABEL: explicit_register_a7:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    mv a7, a0
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    addi a0, a7, 0
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: explicit_register_a7:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    mv a7, a0
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    addi a0, a7, 0
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ret
+  %1 = tail call i32 asm "addi $0, $1, 0", "=r,{a7}"(i32 %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `x18` (`s2`) as an input, so it should be saved.
+define i32 @explicit_register_x18(i32 %a) nounwind {
+; RV32I-LABEL: explicit_register_x18:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw s2, 12(sp)
+; RV32I-NEXT:    mv s2, a0
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    addi a0, s2, 0
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    lw s2, 12(sp)
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: explicit_register_x18:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd s2, 8(sp)
+; RV64I-NEXT:    mv s2, a0
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    addi a0, s2, 0
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ld s2, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = tail call i32 asm "addi $0, $1, 0", "=r,{x18}"(i32 %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `s2` (`x18`) as an input, so it should be saved.
+define i32 @explicit_register_s2(i32 %a) nounwind {
+; RV32I-LABEL: explicit_register_s2:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw s2, 12(sp)
+; RV32I-NEXT:    mv s2, a0
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    addi a0, s2, 0
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    lw s2, 12(sp)
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: explicit_register_s2:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd s2, 8(sp)
+; RV64I-NEXT:    mv s2, a0
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    addi a0, s2, 0
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ld s2, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = tail call i32 asm "addi $0, $1, 0", "=r,{s2}"(i32 %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `x19` (`s3`) as an input, so it should be saved.
+define i32 @explicit_register_x19(i32 %a) nounwind {
+; RV32I-LABEL: explicit_register_x19:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw s3, 12(sp)
+; RV32I-NEXT:    mv s3, a0
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    addi a0, s3, 0
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    lw s3, 12(sp)
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: explicit_register_x19:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd s3, 8(sp)
+; RV64I-NEXT:    mv s3, a0
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    addi a0, s3, 0
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ld s3, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = tail call i32 asm "addi $0, $1, 0", "=r,{x19}"(i32 %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `s3` (`x19`) as an input, so it should be saved.
+define i32 @explicit_register_s3(i32 %a) nounwind {
+; RV32I-LABEL: explicit_register_s3:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw s3, 12(sp)
+; RV32I-NEXT:    mv s3, a0
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    addi a0, s3, 0
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    lw s3, 12(sp)
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: explicit_register_s3:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd s3, 8(sp)
+; RV64I-NEXT:    mv s3, a0
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    addi a0, s3, 0
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ld s3, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = tail call i32 asm "addi $0, $1, 0", "=r,{s3}"(i32 %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `x20` (`s4`) as an input, so it should be saved.
+define i32 @explicit_register_x20(i32 %a) nounwind {
+; RV32I-LABEL: explicit_register_x20:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw s4, 12(sp)
+; RV32I-NEXT:    mv s4, a0
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    addi a0, s4, 0
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    lw s4, 12(sp)
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: explicit_register_x20:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd s4, 8(sp)
+; RV64I-NEXT:    mv s4, a0
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    addi a0, s4, 0
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ld s4, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = tail call i32 asm "addi $0, $1, 0", "=r,{x20}"(i32 %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `s4` (`x20`) as an input, so it should be saved.
+define i32 @explicit_register_s4(i32 %a) nounwind {
+; RV32I-LABEL: explicit_register_s4:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw s4, 12(sp)
+; RV32I-NEXT:    mv s4, a0
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    addi a0, s4, 0
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    lw s4, 12(sp)
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: explicit_register_s4:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd s4, 8(sp)
+; RV64I-NEXT:    mv s4, a0
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    addi a0, s4, 0
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ld s4, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = tail call i32 asm "addi $0, $1, 0", "=r,{s4}"(i32 %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `x21` (`s5`) as an input, so it should be saved.
+define i32 @explicit_register_x21(i32 %a) nounwind {
+; RV32I-LABEL: explicit_register_x21:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw s5, 12(sp)
+; RV32I-NEXT:    mv s5, a0
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    addi a0, s5, 0
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    lw s5, 12(sp)
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: explicit_register_x21:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd s5, 8(sp)
+; RV64I-NEXT:    mv s5, a0
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    addi a0, s5, 0
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ld s5, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = tail call i32 asm "addi $0, $1, 0", "=r,{x21}"(i32 %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `s5` (`x21`) as an input, so it should be saved.
+define i32 @explicit_register_s5(i32 %a) nounwind {
+; RV32I-LABEL: explicit_register_s5:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw s5, 12(sp)
+; RV32I-NEXT:    mv s5, a0
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    addi a0, s5, 0
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    lw s5, 12(sp)
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: explicit_register_s5:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd s5, 8(sp)
+; RV64I-NEXT:    mv s5, a0
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    addi a0, s5, 0
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ld s5, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = tail call i32 asm "addi $0, $1, 0", "=r,{s5}"(i32 %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `x22` (`s6`) as an input, so it should be saved.
+define i32 @explicit_register_x22(i32 %a) nounwind {
+; RV32I-LABEL: explicit_register_x22:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw s6, 12(sp)
+; RV32I-NEXT:    mv s6, a0
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    addi a0, s6, 0
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    lw s6, 12(sp)
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: explicit_register_x22:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd s6, 8(sp)
+; RV64I-NEXT:    mv s6, a0
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    addi a0, s6, 0
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ld s6, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = tail call i32 asm "addi $0, $1, 0", "=r,{x22}"(i32 %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `s6` (`x22`) as an input, so it should be saved.
+define i32 @explicit_register_s6(i32 %a) nounwind {
+; RV32I-LABEL: explicit_register_s6:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw s6, 12(sp)
+; RV32I-NEXT:    mv s6, a0
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    addi a0, s6, 0
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    lw s6, 12(sp)
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: explicit_register_s6:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd s6, 8(sp)
+; RV64I-NEXT:    mv s6, a0
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    addi a0, s6, 0
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ld s6, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = tail call i32 asm "addi $0, $1, 0", "=r,{s6}"(i32 %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `x23` (`s7`) as an input, so it should be saved.
+define i32 @explicit_register_x23(i32 %a) nounwind {
+; RV32I-LABEL: explicit_register_x23:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw s7, 12(sp)
+; RV32I-NEXT:    mv s7, a0
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    addi a0, s7, 0
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    lw s7, 12(sp)
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: explicit_register_x23:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd s7, 8(sp)
+; RV64I-NEXT:    mv s7, a0
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    addi a0, s7, 0
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ld s7, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = tail call i32 asm "addi $0, $1, 0", "=r,{x23}"(i32 %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `s7` (`x23`) as an input, so it should be saved.
+define i32 @explicit_register_s7(i32 %a) nounwind {
+; RV32I-LABEL: explicit_register_s7:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw s7, 12(sp)
+; RV32I-NEXT:    mv s7, a0
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    addi a0, s7, 0
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    lw s7, 12(sp)
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: explicit_register_s7:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd s7, 8(sp)
+; RV64I-NEXT:    mv s7, a0
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    addi a0, s7, 0
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ld s7, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = tail call i32 asm "addi $0, $1, 0", "=r,{s7}"(i32 %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `x24` (`s8`) as an input, so it should be saved.
+define i32 @explicit_register_x24(i32 %a) nounwind {
+; RV32I-LABEL: explicit_register_x24:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw s8, 12(sp)
+; RV32I-NEXT:    mv s8, a0
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    addi a0, s8, 0
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    lw s8, 12(sp)
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: explicit_register_x24:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd s8, 8(sp)
+; RV64I-NEXT:    mv s8, a0
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    addi a0, s8, 0
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ld s8, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = tail call i32 asm "addi $0, $1, 0", "=r,{x24}"(i32 %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `s8` (`x24`) as an input, so it should be saved.
+define i32 @explicit_register_s8(i32 %a) nounwind {
+; RV32I-LABEL: explicit_register_s8:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw s8, 12(sp)
+; RV32I-NEXT:    mv s8, a0
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    addi a0, s8, 0
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    lw s8, 12(sp)
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: explicit_register_s8:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd s8, 8(sp)
+; RV64I-NEXT:    mv s8, a0
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    addi a0, s8, 0
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ld s8, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = tail call i32 asm "addi $0, $1, 0", "=r,{s8}"(i32 %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `x25` (`s9`) as an input, so it should be saved.
+define i32 @explicit_register_x25(i32 %a) nounwind {
+; RV32I-LABEL: explicit_register_x25:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw s9, 12(sp)
+; RV32I-NEXT:    mv s9, a0
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    addi a0, s9, 0
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    lw s9, 12(sp)
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: explicit_register_x25:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd s9, 8(sp)
+; RV64I-NEXT:    mv s9, a0
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    addi a0, s9, 0
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ld s9, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = tail call i32 asm "addi $0, $1, 0", "=r,{x25}"(i32 %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `s9` (`x25`) as an input, so it should be saved.
+define i32 @explicit_register_s9(i32 %a) nounwind {
+; RV32I-LABEL: explicit_register_s9:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw s9, 12(sp)
+; RV32I-NEXT:    mv s9, a0
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    addi a0, s9, 0
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    lw s9, 12(sp)
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: explicit_register_s9:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd s9, 8(sp)
+; RV64I-NEXT:    mv s9, a0
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    addi a0, s9, 0
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ld s9, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = tail call i32 asm "addi $0, $1, 0", "=r,{s9}"(i32 %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `x26` (`s10`) as an input, so it should be saved.
+define i32 @explicit_register_x26(i32 %a) nounwind {
+; RV32I-LABEL: explicit_register_x26:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw s10, 12(sp)
+; RV32I-NEXT:    mv s10, a0
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    addi a0, s10, 0
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    lw s10, 12(sp)
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: explicit_register_x26:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd s10, 8(sp)
+; RV64I-NEXT:    mv s10, a0
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    addi a0, s10, 0
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ld s10, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = tail call i32 asm "addi $0, $1, 0", "=r,{x26}"(i32 %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `s10` (`x28`) as an input, so it should be saved.
+define i32 @explicit_register_s10(i32 %a) nounwind {
+; RV32I-LABEL: explicit_register_s10:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw s10, 12(sp)
+; RV32I-NEXT:    mv s10, a0
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    addi a0, s10, 0
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    lw s10, 12(sp)
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: explicit_register_s10:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd s10, 8(sp)
+; RV64I-NEXT:    mv s10, a0
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    addi a0, s10, 0
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ld s10, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = tail call i32 asm "addi $0, $1, 0", "=r,{s10}"(i32 %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `x27` (`s11`) as an input, so it should be saved.
+define i32 @explicit_register_x27(i32 %a) nounwind {
+; RV32I-LABEL: explicit_register_x27:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw s11, 12(sp)
+; RV32I-NEXT:    mv s11, a0
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    addi a0, s11, 0
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    lw s11, 12(sp)
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: explicit_register_x27:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd s11, 8(sp)
+; RV64I-NEXT:    mv s11, a0
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    addi a0, s11, 0
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ld s11, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = tail call i32 asm "addi $0, $1, 0", "=r,{x27}"(i32 %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `s11` (`x27`) as an input, so it should be saved.
+define i32 @explicit_register_s11(i32 %a) nounwind {
+; RV32I-LABEL: explicit_register_s11:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw s11, 12(sp)
+; RV32I-NEXT:    mv s11, a0
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    addi a0, s11, 0
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    lw s11, 12(sp)
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: explicit_register_s11:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd s11, 8(sp)
+; RV64I-NEXT:    mv s11, a0
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    addi a0, s11, 0
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ld s11, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = tail call i32 asm "addi $0, $1, 0", "=r,{s11}"(i32 %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_x28(i32 %a) nounwind {
+; RV32I-LABEL: explicit_register_x28:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    mv t3, a0
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    addi a0, t3, 0
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: explicit_register_x28:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    mv t3, a0
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    addi a0, t3, 0
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ret
+  %1 = tail call i32 asm "addi $0, $1, 0", "=r,{x28}"(i32 %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_t3(i32 %a) nounwind {
+; RV32I-LABEL: explicit_register_t3:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    mv t3, a0
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    addi a0, t3, 0
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: explicit_register_t3:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    mv t3, a0
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    addi a0, t3, 0
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ret
+  %1 = tail call i32 asm "addi $0, $1, 0", "=r,{t3}"(i32 %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_x29(i32 %a) nounwind {
+; RV32I-LABEL: explicit_register_x29:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    mv t4, a0
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    addi a0, t4, 0
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: explicit_register_x29:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    mv t4, a0
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    addi a0, t4, 0
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ret
+  %1 = tail call i32 asm "addi $0, $1, 0", "=r,{x29}"(i32 %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_t4(i32 %a) nounwind {
+; RV32I-LABEL: explicit_register_t4:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    mv t4, a0
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    addi a0, t4, 0
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: explicit_register_t4:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    mv t4, a0
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    addi a0, t4, 0
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ret
+  %1 = tail call i32 asm "addi $0, $1, 0", "=r,{t4}"(i32 %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_x30(i32 %a) nounwind {
+; RV32I-LABEL: explicit_register_x30:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    mv t5, a0
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    addi a0, t5, 0
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: explicit_register_x30:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    mv t5, a0
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    addi a0, t5, 0
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ret
+  %1 = tail call i32 asm "addi $0, $1, 0", "=r,{x30}"(i32 %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_t5(i32 %a) nounwind {
+; RV32I-LABEL: explicit_register_t5:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    mv t5, a0
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    addi a0, t5, 0
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: explicit_register_t5:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    mv t5, a0
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    addi a0, t5, 0
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ret
+  %1 = tail call i32 asm "addi $0, $1, 0", "=r,{t5}"(i32 %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_x31(i32 %a) nounwind {
+; RV32I-LABEL: explicit_register_x31:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    mv t6, a0
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    addi a0, t6, 0
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: explicit_register_x31:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    mv t6, a0
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    addi a0, t6, 0
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ret
+  %1 = tail call i32 asm "addi $0, $1, 0", "=r,{x31}"(i32 %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_t6(i32 %a) nounwind {
+; RV32I-LABEL: explicit_register_t6:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    mv t6, a0
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    addi a0, t6, 0
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: explicit_register_t6:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    mv t6, a0
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    addi a0, t6, 0
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ret
+  %1 = tail call i32 asm "addi $0, $1, 0", "=r,{t6}"(i32 %a)
+  ret i32 %1
+}
diff --git a/llvm/test/CodeGen/RISCV/inline-asm-clobbers.ll b/llvm/test/CodeGen/RISCV/inline-asm-clobbers.ll
new file mode 100644
index 000000000..605a28011
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/inline-asm-clobbers.ll
@@ -0,0 +1,61 @@
+; RUN: llc -mtriple=riscv32 -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV32I %s
+; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64I %s
+; RUN: llc -mtriple=riscv32 -mattr=+f -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV32I %s
+; RUN: llc -mtriple=riscv64 -mattr=+f -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64I %s
+; RUN: llc -mtriple=riscv32 -mattr=+d -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV32I %s
+; RUN: llc -mtriple=riscv64 -mattr=+d -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64I %s
+; RUN: llc -mtriple=riscv32 -mattr=+f -target-abi ilp32f -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV32IF %s
+; RUN: llc -mtriple=riscv64 -mattr=+f -target-abi lp64f -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64IF %s
+; RUN: llc -mtriple=riscv32 -mattr=+d -target-abi ilp32d -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV32ID %s
+; RUN: llc -mtriple=riscv64 -mattr=+d -target-abi lp64d -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64ID %s
+
+
+define void @testcase() nounwind {
+; RV32I-LABEL: testcase:
+; RV32I:      sw s1, {{[0-9]+}}(sp)
+; RV32I-NEXT: sw s2, {{[0-9]+}}(sp)
+; RV32I-NOT:  fsw fs0, {{[0-9]+}}(sp)
+; RV32I-NOT:  fsd fs0, {{[0-9]+}}(sp)
+;
+; RV64I-LABEL: testcase:
+; RV64I:      sd s1, {{[0-9]+}}(sp)
+; RV64I-NEXT: sd s2, {{[0-9]+}}(sp)
+; RV64I-NOT:  fsw fs0, {{[0-9]+}}(sp)
+; RV64I-NOT:  fsd fs0, {{[0-9]+}}(sp)
+;
+; RV32IF-LABEL: testcase:
+; RV32IF:      sw s1, {{[0-9]+}}(sp)
+; RV32IF-NEXT: sw s2, {{[0-9]+}}(sp)
+; RV32IF-NEXT: fsw fs0, {{[0-9]+}}(sp)
+; RV32IF-NEXT: fsw fs1, {{[0-9]+}}(sp)
+;
+; RV64IF-LABEL: testcase:
+; RV64IF:      sd s1, {{[0-9]+}}(sp)
+; RV64IF-NEXT: sd s2, {{[0-9]+}}(sp)
+; RV64IF-NEXT: fsw fs0, {{[0-9]+}}(sp)
+; RV64IF-NEXT: fsw fs1, {{[0-9]+}}(sp)
+;
+; RV32ID-LABEL: testcase:
+; RV32ID:      sw s1, {{[0-9]+}}(sp)
+; RV32ID-NEXT: sw s2, {{[0-9]+}}(sp)
+; RV32ID-NEXT: fsd fs0, {{[0-9]+}}(sp)
+; RV32ID-NEXT: fsd fs1, {{[0-9]+}}(sp)
+;
+; RV64ID-LABEL: testcase:
+; RV64ID:      sd s1, {{[0-9]+}}(sp)
+; RV64ID-NEXT: sd s2, {{[0-9]+}}(sp)
+; RV64ID-NEXT: fsd fs0, {{[0-9]+}}(sp)
+; RV64ID-NEXT: fsd fs1, {{[0-9]+}}(sp)
+  tail call void asm sideeffect "", "~{f8},~{f9},~{x9},~{x18}"()
+  ret void
+}
diff --git a/llvm/test/CodeGen/RISCV/inline-asm-d-abi-names.ll b/llvm/test/CodeGen/RISCV/inline-asm-d-abi-names.ll
new file mode 100644
index 000000000..bdce94828
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/inline-asm-d-abi-names.ll
@@ -0,0 +1,1509 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+f,+d -target-abi ilp32d -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV32IFD %s
+; RUN: llc -mtriple=riscv64 -mattr=+f,+d -target-abi lp64d -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64IFD %s
+
+; These test that we can use both the architectural names (x*) and the ABI names
+; (a*, s*, t* etc) to refer to registers in inline asm constraint lists. In each
+; case, the named register should be used for the source register of the `addi`.
+; It is very likely that `a0` will be chosen as the designation register, but
+; this is left to the compiler to choose.
+;
+; The inline assembly will, by default, contain the ABI names for the registers.
+;
+; Parenthesised registers in comments are the other aliases for this register.
+
+
+define i32 @explicit_register_f0(double %a) nounwind {
+; RV32IFD-LABEL: explicit_register_f0:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    fmv.d ft0, fa0
+; RV32IFD-NEXT:    #APP
+; RV32IFD-NEXT:    fcvt.w.d a0, ft0
+; RV32IFD-NEXT:    #NO_APP
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: explicit_register_f0:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d ft0, fa0
+; RV64IFD-NEXT:    #APP
+; RV64IFD-NEXT:    fcvt.w.d a0, ft0
+; RV64IFD-NEXT:    #NO_APP
+; RV64IFD-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{f0}"(double %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_ft0(double %a) nounwind {
+; RV32IFD-LABEL: explicit_register_ft0:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    fmv.d ft0, fa0
+; RV32IFD-NEXT:    #APP
+; RV32IFD-NEXT:    fcvt.w.d a0, ft0
+; RV32IFD-NEXT:    #NO_APP
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: explicit_register_ft0:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d ft0, fa0
+; RV64IFD-NEXT:    #APP
+; RV64IFD-NEXT:    fcvt.w.d a0, ft0
+; RV64IFD-NEXT:    #NO_APP
+; RV64IFD-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{ft0}"(double %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_f1(double %a) nounwind {
+; RV32IFD-LABEL: explicit_register_f1:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    fmv.d ft1, fa0
+; RV32IFD-NEXT:    #APP
+; RV32IFD-NEXT:    fcvt.w.d a0, ft1
+; RV32IFD-NEXT:    #NO_APP
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: explicit_register_f1:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d ft1, fa0
+; RV64IFD-NEXT:    #APP
+; RV64IFD-NEXT:    fcvt.w.d a0, ft1
+; RV64IFD-NEXT:    #NO_APP
+; RV64IFD-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{f1}"(double %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_ft1(double %a) nounwind {
+; RV32IFD-LABEL: explicit_register_ft1:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    fmv.d ft1, fa0
+; RV32IFD-NEXT:    #APP
+; RV32IFD-NEXT:    fcvt.w.d a0, ft1
+; RV32IFD-NEXT:    #NO_APP
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: explicit_register_ft1:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d ft1, fa0
+; RV64IFD-NEXT:    #APP
+; RV64IFD-NEXT:    fcvt.w.d a0, ft1
+; RV64IFD-NEXT:    #NO_APP
+; RV64IFD-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{ft1}"(double %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_f2(double %a) nounwind {
+; RV32IFD-LABEL: explicit_register_f2:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    fmv.d ft2, fa0
+; RV32IFD-NEXT:    #APP
+; RV32IFD-NEXT:    fcvt.w.d a0, ft2
+; RV32IFD-NEXT:    #NO_APP
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: explicit_register_f2:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d ft2, fa0
+; RV64IFD-NEXT:    #APP
+; RV64IFD-NEXT:    fcvt.w.d a0, ft2
+; RV64IFD-NEXT:    #NO_APP
+; RV64IFD-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{f2}"(double %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_ft2(double %a) nounwind {
+; RV32IFD-LABEL: explicit_register_ft2:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    fmv.d ft2, fa0
+; RV32IFD-NEXT:    #APP
+; RV32IFD-NEXT:    fcvt.w.d a0, ft2
+; RV32IFD-NEXT:    #NO_APP
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: explicit_register_ft2:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d ft2, fa0
+; RV64IFD-NEXT:    #APP
+; RV64IFD-NEXT:    fcvt.w.d a0, ft2
+; RV64IFD-NEXT:    #NO_APP
+; RV64IFD-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{ft2}"(double %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_f3(double %a) nounwind {
+; RV32IFD-LABEL: explicit_register_f3:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    fmv.d ft3, fa0
+; RV32IFD-NEXT:    #APP
+; RV32IFD-NEXT:    fcvt.w.d a0, ft3
+; RV32IFD-NEXT:    #NO_APP
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: explicit_register_f3:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d ft3, fa0
+; RV64IFD-NEXT:    #APP
+; RV64IFD-NEXT:    fcvt.w.d a0, ft3
+; RV64IFD-NEXT:    #NO_APP
+; RV64IFD-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{f3}"(double %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_ft3(double %a) nounwind {
+; RV32IFD-LABEL: explicit_register_ft3:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    fmv.d ft3, fa0
+; RV32IFD-NEXT:    #APP
+; RV32IFD-NEXT:    fcvt.w.d a0, ft3
+; RV32IFD-NEXT:    #NO_APP
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: explicit_register_ft3:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d ft3, fa0
+; RV64IFD-NEXT:    #APP
+; RV64IFD-NEXT:    fcvt.w.d a0, ft3
+; RV64IFD-NEXT:    #NO_APP
+; RV64IFD-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{ft3}"(double %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_f4(double %a) nounwind {
+; RV32IFD-LABEL: explicit_register_f4:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    fmv.d ft4, fa0
+; RV32IFD-NEXT:    #APP
+; RV32IFD-NEXT:    fcvt.w.d a0, ft4
+; RV32IFD-NEXT:    #NO_APP
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: explicit_register_f4:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d ft4, fa0
+; RV64IFD-NEXT:    #APP
+; RV64IFD-NEXT:    fcvt.w.d a0, ft4
+; RV64IFD-NEXT:    #NO_APP
+; RV64IFD-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{f4}"(double %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_ft4(double %a) nounwind {
+; RV32IFD-LABEL: explicit_register_ft4:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    fmv.d ft4, fa0
+; RV32IFD-NEXT:    #APP
+; RV32IFD-NEXT:    fcvt.w.d a0, ft4
+; RV32IFD-NEXT:    #NO_APP
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: explicit_register_ft4:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d ft4, fa0
+; RV64IFD-NEXT:    #APP
+; RV64IFD-NEXT:    fcvt.w.d a0, ft4
+; RV64IFD-NEXT:    #NO_APP
+; RV64IFD-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{ft4}"(double %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_f5(double %a) nounwind {
+; RV32IFD-LABEL: explicit_register_f5:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    fmv.d ft5, fa0
+; RV32IFD-NEXT:    #APP
+; RV32IFD-NEXT:    fcvt.w.d a0, ft5
+; RV32IFD-NEXT:    #NO_APP
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: explicit_register_f5:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d ft5, fa0
+; RV64IFD-NEXT:    #APP
+; RV64IFD-NEXT:    fcvt.w.d a0, ft5
+; RV64IFD-NEXT:    #NO_APP
+; RV64IFD-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{f5}"(double %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_ft5(double %a) nounwind {
+; RV32IFD-LABEL: explicit_register_ft5:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    fmv.d ft5, fa0
+; RV32IFD-NEXT:    #APP
+; RV32IFD-NEXT:    fcvt.w.d a0, ft5
+; RV32IFD-NEXT:    #NO_APP
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: explicit_register_ft5:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d ft5, fa0
+; RV64IFD-NEXT:    #APP
+; RV64IFD-NEXT:    fcvt.w.d a0, ft5
+; RV64IFD-NEXT:    #NO_APP
+; RV64IFD-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{ft5}"(double %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_f6(double %a) nounwind {
+; RV32IFD-LABEL: explicit_register_f6:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    fmv.d ft6, fa0
+; RV32IFD-NEXT:    #APP
+; RV32IFD-NEXT:    fcvt.w.d a0, ft6
+; RV32IFD-NEXT:    #NO_APP
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: explicit_register_f6:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d ft6, fa0
+; RV64IFD-NEXT:    #APP
+; RV64IFD-NEXT:    fcvt.w.d a0, ft6
+; RV64IFD-NEXT:    #NO_APP
+; RV64IFD-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{f6}"(double %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_ft6(double %a) nounwind {
+; RV32IFD-LABEL: explicit_register_ft6:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    fmv.d ft6, fa0
+; RV32IFD-NEXT:    #APP
+; RV32IFD-NEXT:    fcvt.w.d a0, ft6
+; RV32IFD-NEXT:    #NO_APP
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: explicit_register_ft6:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d ft6, fa0
+; RV64IFD-NEXT:    #APP
+; RV64IFD-NEXT:    fcvt.w.d a0, ft6
+; RV64IFD-NEXT:    #NO_APP
+; RV64IFD-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{ft6}"(double %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_f7(double %a) nounwind {
+; RV32IFD-LABEL: explicit_register_f7:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    fmv.d ft7, fa0
+; RV32IFD-NEXT:    #APP
+; RV32IFD-NEXT:    fcvt.w.d a0, ft7
+; RV32IFD-NEXT:    #NO_APP
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: explicit_register_f7:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d ft7, fa0
+; RV64IFD-NEXT:    #APP
+; RV64IFD-NEXT:    fcvt.w.d a0, ft7
+; RV64IFD-NEXT:    #NO_APP
+; RV64IFD-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{f7}"(double %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_ft7(double %a) nounwind {
+; RV32IFD-LABEL: explicit_register_ft7:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    fmv.d ft7, fa0
+; RV32IFD-NEXT:    #APP
+; RV32IFD-NEXT:    fcvt.w.d a0, ft7
+; RV32IFD-NEXT:    #NO_APP
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: explicit_register_ft7:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d ft7, fa0
+; RV64IFD-NEXT:    #APP
+; RV64IFD-NEXT:    fcvt.w.d a0, ft7
+; RV64IFD-NEXT:    #NO_APP
+; RV64IFD-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{ft7}"(double %a)
+  ret i32 %1
+}
+
+
+; NOTE: This test uses `f8` (`fs0`) as an input, so it should be saved.
+define i32 @explicit_register_f8(double %a) nounwind {
+; RV32IFD-LABEL: explicit_register_f8:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi sp, sp, -16
+; RV32IFD-NEXT:    fsd fs0, 8(sp)
+; RV32IFD-NEXT:    fmv.d fs0, fa0
+; RV32IFD-NEXT:    #APP
+; RV32IFD-NEXT:    fcvt.w.d a0, fs0
+; RV32IFD-NEXT:    #NO_APP
+; RV32IFD-NEXT:    fld fs0, 8(sp)
+; RV32IFD-NEXT:    addi sp, sp, 16
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: explicit_register_f8:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    fsd fs0, 8(sp)
+; RV64IFD-NEXT:    fmv.d fs0, fa0
+; RV64IFD-NEXT:    #APP
+; RV64IFD-NEXT:    fcvt.w.d a0, fs0
+; RV64IFD-NEXT:    #NO_APP
+; RV64IFD-NEXT:    fld fs0, 8(sp)
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{f8}"(double %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `fs0` (`f8`) as an input, so it should be saved.
+define i32 @explicit_register_fs0(double %a) nounwind {
+; RV32IFD-LABEL: explicit_register_fs0:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi sp, sp, -16
+; RV32IFD-NEXT:    fsd fs0, 8(sp)
+; RV32IFD-NEXT:    fmv.d fs0, fa0
+; RV32IFD-NEXT:    #APP
+; RV32IFD-NEXT:    fcvt.w.d a0, fs0
+; RV32IFD-NEXT:    #NO_APP
+; RV32IFD-NEXT:    fld fs0, 8(sp)
+; RV32IFD-NEXT:    addi sp, sp, 16
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: explicit_register_fs0:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    fsd fs0, 8(sp)
+; RV64IFD-NEXT:    fmv.d fs0, fa0
+; RV64IFD-NEXT:    #APP
+; RV64IFD-NEXT:    fcvt.w.d a0, fs0
+; RV64IFD-NEXT:    #NO_APP
+; RV64IFD-NEXT:    fld fs0, 8(sp)
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{fs0}"(double %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `f9` (`fs1`) as an input, so it should be saved.
+define i32 @explicit_register_f9(double %a) nounwind {
+; RV32IFD-LABEL: explicit_register_f9:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi sp, sp, -16
+; RV32IFD-NEXT:    fsd fs1, 8(sp)
+; RV32IFD-NEXT:    fmv.d fs1, fa0
+; RV32IFD-NEXT:    #APP
+; RV32IFD-NEXT:    fcvt.w.d a0, fs1
+; RV32IFD-NEXT:    #NO_APP
+; RV32IFD-NEXT:    fld fs1, 8(sp)
+; RV32IFD-NEXT:    addi sp, sp, 16
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: explicit_register_f9:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    fsd fs1, 8(sp)
+; RV64IFD-NEXT:    fmv.d fs1, fa0
+; RV64IFD-NEXT:    #APP
+; RV64IFD-NEXT:    fcvt.w.d a0, fs1
+; RV64IFD-NEXT:    #NO_APP
+; RV64IFD-NEXT:    fld fs1, 8(sp)
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{f9}"(double %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `fs1` (`f9`) as an input, so it should be saved.
+define i32 @explicit_register_fs1(double %a) nounwind {
+; RV32IFD-LABEL: explicit_register_fs1:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi sp, sp, -16
+; RV32IFD-NEXT:    fsd fs1, 8(sp)
+; RV32IFD-NEXT:    fmv.d fs1, fa0
+; RV32IFD-NEXT:    #APP
+; RV32IFD-NEXT:    fcvt.w.d a0, fs1
+; RV32IFD-NEXT:    #NO_APP
+; RV32IFD-NEXT:    fld fs1, 8(sp)
+; RV32IFD-NEXT:    addi sp, sp, 16
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: explicit_register_fs1:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    fsd fs1, 8(sp)
+; RV64IFD-NEXT:    fmv.d fs1, fa0
+; RV64IFD-NEXT:    #APP
+; RV64IFD-NEXT:    fcvt.w.d a0, fs1
+; RV64IFD-NEXT:    #NO_APP
+; RV64IFD-NEXT:    fld fs1, 8(sp)
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{fs1}"(double %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_f10(double %a) nounwind {
+; RV32IFD-LABEL: explicit_register_f10:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    #APP
+; RV32IFD-NEXT:    fcvt.w.d a0, fa0
+; RV32IFD-NEXT:    #NO_APP
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: explicit_register_f10:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    #APP
+; RV64IFD-NEXT:    fcvt.w.d a0, fa0
+; RV64IFD-NEXT:    #NO_APP
+; RV64IFD-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{f10}"(double %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_fa0(double %a) nounwind {
+; RV32IFD-LABEL: explicit_register_fa0:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    #APP
+; RV32IFD-NEXT:    fcvt.w.d a0, fa0
+; RV32IFD-NEXT:    #NO_APP
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: explicit_register_fa0:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    #APP
+; RV64IFD-NEXT:    fcvt.w.d a0, fa0
+; RV64IFD-NEXT:    #NO_APP
+; RV64IFD-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{fa0}"(double %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_f11(double %a) nounwind {
+; RV32IFD-LABEL: explicit_register_f11:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    fmv.d fa1, fa0
+; RV32IFD-NEXT:    #APP
+; RV32IFD-NEXT:    fcvt.w.d a0, fa1
+; RV32IFD-NEXT:    #NO_APP
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: explicit_register_f11:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d fa1, fa0
+; RV64IFD-NEXT:    #APP
+; RV64IFD-NEXT:    fcvt.w.d a0, fa1
+; RV64IFD-NEXT:    #NO_APP
+; RV64IFD-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{f11}"(double %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_fa1(double %a) nounwind {
+; RV32IFD-LABEL: explicit_register_fa1:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    fmv.d fa1, fa0
+; RV32IFD-NEXT:    #APP
+; RV32IFD-NEXT:    fcvt.w.d a0, fa1
+; RV32IFD-NEXT:    #NO_APP
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: explicit_register_fa1:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d fa1, fa0
+; RV64IFD-NEXT:    #APP
+; RV64IFD-NEXT:    fcvt.w.d a0, fa1
+; RV64IFD-NEXT:    #NO_APP
+; RV64IFD-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{fa1}"(double %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_f12(double %a) nounwind {
+; RV32IFD-LABEL: explicit_register_f12:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    fmv.d fa2, fa0
+; RV32IFD-NEXT:    #APP
+; RV32IFD-NEXT:    fcvt.w.d a0, fa2
+; RV32IFD-NEXT:    #NO_APP
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: explicit_register_f12:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d fa2, fa0
+; RV64IFD-NEXT:    #APP
+; RV64IFD-NEXT:    fcvt.w.d a0, fa2
+; RV64IFD-NEXT:    #NO_APP
+; RV64IFD-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{f12}"(double %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_fa2(double %a) nounwind {
+; RV32IFD-LABEL: explicit_register_fa2:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    fmv.d fa2, fa0
+; RV32IFD-NEXT:    #APP
+; RV32IFD-NEXT:    fcvt.w.d a0, fa2
+; RV32IFD-NEXT:    #NO_APP
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: explicit_register_fa2:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d fa2, fa0
+; RV64IFD-NEXT:    #APP
+; RV64IFD-NEXT:    fcvt.w.d a0, fa2
+; RV64IFD-NEXT:    #NO_APP
+; RV64IFD-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{fa2}"(double %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_f13(double %a) nounwind {
+; RV32IFD-LABEL: explicit_register_f13:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    fmv.d fa3, fa0
+; RV32IFD-NEXT:    #APP
+; RV32IFD-NEXT:    fcvt.w.d a0, fa3
+; RV32IFD-NEXT:    #NO_APP
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: explicit_register_f13:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d fa3, fa0
+; RV64IFD-NEXT:    #APP
+; RV64IFD-NEXT:    fcvt.w.d a0, fa3
+; RV64IFD-NEXT:    #NO_APP
+; RV64IFD-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{f13}"(double %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_fa3(double %a) nounwind {
+; RV32IFD-LABEL: explicit_register_fa3:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    fmv.d fa3, fa0
+; RV32IFD-NEXT:    #APP
+; RV32IFD-NEXT:    fcvt.w.d a0, fa3
+; RV32IFD-NEXT:    #NO_APP
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: explicit_register_fa3:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d fa3, fa0
+; RV64IFD-NEXT:    #APP
+; RV64IFD-NEXT:    fcvt.w.d a0, fa3
+; RV64IFD-NEXT:    #NO_APP
+; RV64IFD-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{fa3}"(double %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_f14(double %a) nounwind {
+; RV32IFD-LABEL: explicit_register_f14:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    fmv.d fa4, fa0
+; RV32IFD-NEXT:    #APP
+; RV32IFD-NEXT:    fcvt.w.d a0, fa4
+; RV32IFD-NEXT:    #NO_APP
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: explicit_register_f14:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d fa4, fa0
+; RV64IFD-NEXT:    #APP
+; RV64IFD-NEXT:    fcvt.w.d a0, fa4
+; RV64IFD-NEXT:    #NO_APP
+; RV64IFD-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{f14}"(double %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_fa4(double %a) nounwind {
+; RV32IFD-LABEL: explicit_register_fa4:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    fmv.d fa4, fa0
+; RV32IFD-NEXT:    #APP
+; RV32IFD-NEXT:    fcvt.w.d a0, fa4
+; RV32IFD-NEXT:    #NO_APP
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: explicit_register_fa4:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d fa4, fa0
+; RV64IFD-NEXT:    #APP
+; RV64IFD-NEXT:    fcvt.w.d a0, fa4
+; RV64IFD-NEXT:    #NO_APP
+; RV64IFD-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{fa4}"(double %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_f15(double %a) nounwind {
+; RV32IFD-LABEL: explicit_register_f15:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    fmv.d fa5, fa0
+; RV32IFD-NEXT:    #APP
+; RV32IFD-NEXT:    fcvt.w.d a0, fa5
+; RV32IFD-NEXT:    #NO_APP
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: explicit_register_f15:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d fa5, fa0
+; RV64IFD-NEXT:    #APP
+; RV64IFD-NEXT:    fcvt.w.d a0, fa5
+; RV64IFD-NEXT:    #NO_APP
+; RV64IFD-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{f15}"(double %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_fa5(double %a) nounwind {
+; RV32IFD-LABEL: explicit_register_fa5:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    fmv.d fa5, fa0
+; RV32IFD-NEXT:    #APP
+; RV32IFD-NEXT:    fcvt.w.d a0, fa5
+; RV32IFD-NEXT:    #NO_APP
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: explicit_register_fa5:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d fa5, fa0
+; RV64IFD-NEXT:    #APP
+; RV64IFD-NEXT:    fcvt.w.d a0, fa5
+; RV64IFD-NEXT:    #NO_APP
+; RV64IFD-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{fa5}"(double %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_f16(double %a) nounwind {
+; RV32IFD-LABEL: explicit_register_f16:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    fmv.d fa6, fa0
+; RV32IFD-NEXT:    #APP
+; RV32IFD-NEXT:    fcvt.w.d a0, fa6
+; RV32IFD-NEXT:    #NO_APP
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: explicit_register_f16:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d fa6, fa0
+; RV64IFD-NEXT:    #APP
+; RV64IFD-NEXT:    fcvt.w.d a0, fa6
+; RV64IFD-NEXT:    #NO_APP
+; RV64IFD-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{f16}"(double %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_fa6(double %a) nounwind {
+; RV32IFD-LABEL: explicit_register_fa6:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    fmv.d fa6, fa0
+; RV32IFD-NEXT:    #APP
+; RV32IFD-NEXT:    fcvt.w.d a0, fa6
+; RV32IFD-NEXT:    #NO_APP
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: explicit_register_fa6:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d fa6, fa0
+; RV64IFD-NEXT:    #APP
+; RV64IFD-NEXT:    fcvt.w.d a0, fa6
+; RV64IFD-NEXT:    #NO_APP
+; RV64IFD-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{fa6}"(double %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_f17(double %a) nounwind {
+; RV32IFD-LABEL: explicit_register_f17:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    fmv.d fa7, fa0
+; RV32IFD-NEXT:    #APP
+; RV32IFD-NEXT:    fcvt.w.d a0, fa7
+; RV32IFD-NEXT:    #NO_APP
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: explicit_register_f17:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d fa7, fa0
+; RV64IFD-NEXT:    #APP
+; RV64IFD-NEXT:    fcvt.w.d a0, fa7
+; RV64IFD-NEXT:    #NO_APP
+; RV64IFD-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{f17}"(double %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_fa7(double %a) nounwind {
+; RV32IFD-LABEL: explicit_register_fa7:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    fmv.d fa7, fa0
+; RV32IFD-NEXT:    #APP
+; RV32IFD-NEXT:    fcvt.w.d a0, fa7
+; RV32IFD-NEXT:    #NO_APP
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: explicit_register_fa7:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d fa7, fa0
+; RV64IFD-NEXT:    #APP
+; RV64IFD-NEXT:    fcvt.w.d a0, fa7
+; RV64IFD-NEXT:    #NO_APP
+; RV64IFD-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{fa7}"(double %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `f18` (`fs2`) as an input, so it should be saved.
+define i32 @explicit_register_f18(double %a) nounwind {
+; RV32IFD-LABEL: explicit_register_f18:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi sp, sp, -16
+; RV32IFD-NEXT:    fsd fs2, 8(sp)
+; RV32IFD-NEXT:    fmv.d fs2, fa0
+; RV32IFD-NEXT:    #APP
+; RV32IFD-NEXT:    fcvt.w.d a0, fs2
+; RV32IFD-NEXT:    #NO_APP
+; RV32IFD-NEXT:    fld fs2, 8(sp)
+; RV32IFD-NEXT:    addi sp, sp, 16
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: explicit_register_f18:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    fsd fs2, 8(sp)
+; RV64IFD-NEXT:    fmv.d fs2, fa0
+; RV64IFD-NEXT:    #APP
+; RV64IFD-NEXT:    fcvt.w.d a0, fs2
+; RV64IFD-NEXT:    #NO_APP
+; RV64IFD-NEXT:    fld fs2, 8(sp)
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{f18}"(double %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `fs2` (`f18`) as an input, so it should be saved.
+define i32 @explicit_register_fs2(double %a) nounwind {
+; RV32IFD-LABEL: explicit_register_fs2:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi sp, sp, -16
+; RV32IFD-NEXT:    fsd fs2, 8(sp)
+; RV32IFD-NEXT:    fmv.d fs2, fa0
+; RV32IFD-NEXT:    #APP
+; RV32IFD-NEXT:    fcvt.w.d a0, fs2
+; RV32IFD-NEXT:    #NO_APP
+; RV32IFD-NEXT:    fld fs2, 8(sp)
+; RV32IFD-NEXT:    addi sp, sp, 16
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: explicit_register_fs2:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    fsd fs2, 8(sp)
+; RV64IFD-NEXT:    fmv.d fs2, fa0
+; RV64IFD-NEXT:    #APP
+; RV64IFD-NEXT:    fcvt.w.d a0, fs2
+; RV64IFD-NEXT:    #NO_APP
+; RV64IFD-NEXT:    fld fs2, 8(sp)
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{fs2}"(double %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `f19` (`fs3`) as an input, so it should be saved.
+define i32 @explicit_register_f19(double %a) nounwind {
+; RV32IFD-LABEL: explicit_register_f19:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi sp, sp, -16
+; RV32IFD-NEXT:    fsd fs3, 8(sp)
+; RV32IFD-NEXT:    fmv.d fs3, fa0
+; RV32IFD-NEXT:    #APP
+; RV32IFD-NEXT:    fcvt.w.d a0, fs3
+; RV32IFD-NEXT:    #NO_APP
+; RV32IFD-NEXT:    fld fs3, 8(sp)
+; RV32IFD-NEXT:    addi sp, sp, 16
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: explicit_register_f19:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    fsd fs3, 8(sp)
+; RV64IFD-NEXT:    fmv.d fs3, fa0
+; RV64IFD-NEXT:    #APP
+; RV64IFD-NEXT:    fcvt.w.d a0, fs3
+; RV64IFD-NEXT:    #NO_APP
+; RV64IFD-NEXT:    fld fs3, 8(sp)
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{f19}"(double %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `fs3` (`f19`) as an input, so it should be saved.
+define i32 @explicit_register_fs3(double %a) nounwind {
+; RV32IFD-LABEL: explicit_register_fs3:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi sp, sp, -16
+; RV32IFD-NEXT:    fsd fs3, 8(sp)
+; RV32IFD-NEXT:    fmv.d fs3, fa0
+; RV32IFD-NEXT:    #APP
+; RV32IFD-NEXT:    fcvt.w.d a0, fs3
+; RV32IFD-NEXT:    #NO_APP
+; RV32IFD-NEXT:    fld fs3, 8(sp)
+; RV32IFD-NEXT:    addi sp, sp, 16
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: explicit_register_fs3:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    fsd fs3, 8(sp)
+; RV64IFD-NEXT:    fmv.d fs3, fa0
+; RV64IFD-NEXT:    #APP
+; RV64IFD-NEXT:    fcvt.w.d a0, fs3
+; RV64IFD-NEXT:    #NO_APP
+; RV64IFD-NEXT:    fld fs3, 8(sp)
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{fs3}"(double %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `f20` (`fs4`) as an input, so it should be saved.
+define i32 @explicit_register_f20(double %a) nounwind {
+; RV32IFD-LABEL: explicit_register_f20:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi sp, sp, -16
+; RV32IFD-NEXT:    fsd fs4, 8(sp)
+; RV32IFD-NEXT:    fmv.d fs4, fa0
+; RV32IFD-NEXT:    #APP
+; RV32IFD-NEXT:    fcvt.w.d a0, fs4
+; RV32IFD-NEXT:    #NO_APP
+; RV32IFD-NEXT:    fld fs4, 8(sp)
+; RV32IFD-NEXT:    addi sp, sp, 16
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: explicit_register_f20:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    fsd fs4, 8(sp)
+; RV64IFD-NEXT:    fmv.d fs4, fa0
+; RV64IFD-NEXT:    #APP
+; RV64IFD-NEXT:    fcvt.w.d a0, fs4
+; RV64IFD-NEXT:    #NO_APP
+; RV64IFD-NEXT:    fld fs4, 8(sp)
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{f20}"(double %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `fs4` (`f20`) as an input, so it should be saved.
+define i32 @explicit_register_fs4(double %a) nounwind {
+; RV32IFD-LABEL: explicit_register_fs4:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi sp, sp, -16
+; RV32IFD-NEXT:    fsd fs4, 8(sp)
+; RV32IFD-NEXT:    fmv.d fs4, fa0
+; RV32IFD-NEXT:    #APP
+; RV32IFD-NEXT:    fcvt.w.d a0, fs4
+; RV32IFD-NEXT:    #NO_APP
+; RV32IFD-NEXT:    fld fs4, 8(sp)
+; RV32IFD-NEXT:    addi sp, sp, 16
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: explicit_register_fs4:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    fsd fs4, 8(sp)
+; RV64IFD-NEXT:    fmv.d fs4, fa0
+; RV64IFD-NEXT:    #APP
+; RV64IFD-NEXT:    fcvt.w.d a0, fs4
+; RV64IFD-NEXT:    #NO_APP
+; RV64IFD-NEXT:    fld fs4, 8(sp)
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{fs4}"(double %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `f21` (`fs5`) as an input, so it should be saved.
+define i32 @explicit_register_f21(double %a) nounwind {
+; RV32IFD-LABEL: explicit_register_f21:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi sp, sp, -16
+; RV32IFD-NEXT:    fsd fs5, 8(sp)
+; RV32IFD-NEXT:    fmv.d fs5, fa0
+; RV32IFD-NEXT:    #APP
+; RV32IFD-NEXT:    fcvt.w.d a0, fs5
+; RV32IFD-NEXT:    #NO_APP
+; RV32IFD-NEXT:    fld fs5, 8(sp)
+; RV32IFD-NEXT:    addi sp, sp, 16
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: explicit_register_f21:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    fsd fs5, 8(sp)
+; RV64IFD-NEXT:    fmv.d fs5, fa0
+; RV64IFD-NEXT:    #APP
+; RV64IFD-NEXT:    fcvt.w.d a0, fs5
+; RV64IFD-NEXT:    #NO_APP
+; RV64IFD-NEXT:    fld fs5, 8(sp)
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{f21}"(double %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `fs5` (`f21`) as an input, so it should be saved.
+define i32 @explicit_register_fs5(double %a) nounwind {
+; RV32IFD-LABEL: explicit_register_fs5:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi sp, sp, -16
+; RV32IFD-NEXT:    fsd fs5, 8(sp)
+; RV32IFD-NEXT:    fmv.d fs5, fa0
+; RV32IFD-NEXT:    #APP
+; RV32IFD-NEXT:    fcvt.w.d a0, fs5
+; RV32IFD-NEXT:    #NO_APP
+; RV32IFD-NEXT:    fld fs5, 8(sp)
+; RV32IFD-NEXT:    addi sp, sp, 16
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: explicit_register_fs5:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    fsd fs5, 8(sp)
+; RV64IFD-NEXT:    fmv.d fs5, fa0
+; RV64IFD-NEXT:    #APP
+; RV64IFD-NEXT:    fcvt.w.d a0, fs5
+; RV64IFD-NEXT:    #NO_APP
+; RV64IFD-NEXT:    fld fs5, 8(sp)
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{fs5}"(double %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `f22` (`fs6`) as an input, so it should be saved.
+define i32 @explicit_register_f22(double %a) nounwind {
+; RV32IFD-LABEL: explicit_register_f22:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi sp, sp, -16
+; RV32IFD-NEXT:    fsd fs6, 8(sp)
+; RV32IFD-NEXT:    fmv.d fs6, fa0
+; RV32IFD-NEXT:    #APP
+; RV32IFD-NEXT:    fcvt.w.d a0, fs6
+; RV32IFD-NEXT:    #NO_APP
+; RV32IFD-NEXT:    fld fs6, 8(sp)
+; RV32IFD-NEXT:    addi sp, sp, 16
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: explicit_register_f22:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    fsd fs6, 8(sp)
+; RV64IFD-NEXT:    fmv.d fs6, fa0
+; RV64IFD-NEXT:    #APP
+; RV64IFD-NEXT:    fcvt.w.d a0, fs6
+; RV64IFD-NEXT:    #NO_APP
+; RV64IFD-NEXT:    fld fs6, 8(sp)
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{f22}"(double %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `fs6` (`f22`) as an input, so it should be saved.
+define i32 @explicit_register_fs6(double %a) nounwind {
+; RV32IFD-LABEL: explicit_register_fs6:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi sp, sp, -16
+; RV32IFD-NEXT:    fsd fs6, 8(sp)
+; RV32IFD-NEXT:    fmv.d fs6, fa0
+; RV32IFD-NEXT:    #APP
+; RV32IFD-NEXT:    fcvt.w.d a0, fs6
+; RV32IFD-NEXT:    #NO_APP
+; RV32IFD-NEXT:    fld fs6, 8(sp)
+; RV32IFD-NEXT:    addi sp, sp, 16
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: explicit_register_fs6:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    fsd fs6, 8(sp)
+; RV64IFD-NEXT:    fmv.d fs6, fa0
+; RV64IFD-NEXT:    #APP
+; RV64IFD-NEXT:    fcvt.w.d a0, fs6
+; RV64IFD-NEXT:    #NO_APP
+; RV64IFD-NEXT:    fld fs6, 8(sp)
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{fs6}"(double %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `f23` (`fs7`) as an input, so it should be saved.
+define i32 @explicit_register_f23(double %a) nounwind {
+; RV32IFD-LABEL: explicit_register_f23:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi sp, sp, -16
+; RV32IFD-NEXT:    fsd fs7, 8(sp)
+; RV32IFD-NEXT:    fmv.d fs7, fa0
+; RV32IFD-NEXT:    #APP
+; RV32IFD-NEXT:    fcvt.w.d a0, fs7
+; RV32IFD-NEXT:    #NO_APP
+; RV32IFD-NEXT:    fld fs7, 8(sp)
+; RV32IFD-NEXT:    addi sp, sp, 16
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: explicit_register_f23:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    fsd fs7, 8(sp)
+; RV64IFD-NEXT:    fmv.d fs7, fa0
+; RV64IFD-NEXT:    #APP
+; RV64IFD-NEXT:    fcvt.w.d a0, fs7
+; RV64IFD-NEXT:    #NO_APP
+; RV64IFD-NEXT:    fld fs7, 8(sp)
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{f23}"(double %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `fs7` (`f23`) as an input, so it should be saved.
+define i32 @explicit_register_fs7(double %a) nounwind {
+; RV32IFD-LABEL: explicit_register_fs7:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi sp, sp, -16
+; RV32IFD-NEXT:    fsd fs7, 8(sp)
+; RV32IFD-NEXT:    fmv.d fs7, fa0
+; RV32IFD-NEXT:    #APP
+; RV32IFD-NEXT:    fcvt.w.d a0, fs7
+; RV32IFD-NEXT:    #NO_APP
+; RV32IFD-NEXT:    fld fs7, 8(sp)
+; RV32IFD-NEXT:    addi sp, sp, 16
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: explicit_register_fs7:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    fsd fs7, 8(sp)
+; RV64IFD-NEXT:    fmv.d fs7, fa0
+; RV64IFD-NEXT:    #APP
+; RV64IFD-NEXT:    fcvt.w.d a0, fs7
+; RV64IFD-NEXT:    #NO_APP
+; RV64IFD-NEXT:    fld fs7, 8(sp)
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{fs7}"(double %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `f24` (`fs8`) as an input, so it should be saved.
+define i32 @explicit_register_f24(double %a) nounwind {
+; RV32IFD-LABEL: explicit_register_f24:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi sp, sp, -16
+; RV32IFD-NEXT:    fsd fs8, 8(sp)
+; RV32IFD-NEXT:    fmv.d fs8, fa0
+; RV32IFD-NEXT:    #APP
+; RV32IFD-NEXT:    fcvt.w.d a0, fs8
+; RV32IFD-NEXT:    #NO_APP
+; RV32IFD-NEXT:    fld fs8, 8(sp)
+; RV32IFD-NEXT:    addi sp, sp, 16
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: explicit_register_f24:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    fsd fs8, 8(sp)
+; RV64IFD-NEXT:    fmv.d fs8, fa0
+; RV64IFD-NEXT:    #APP
+; RV64IFD-NEXT:    fcvt.w.d a0, fs8
+; RV64IFD-NEXT:    #NO_APP
+; RV64IFD-NEXT:    fld fs8, 8(sp)
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{f24}"(double %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `fs8` (`f24`) as an input, so it should be saved.
+define i32 @explicit_register_fs8(double %a) nounwind {
+; RV32IFD-LABEL: explicit_register_fs8:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi sp, sp, -16
+; RV32IFD-NEXT:    fsd fs8, 8(sp)
+; RV32IFD-NEXT:    fmv.d fs8, fa0
+; RV32IFD-NEXT:    #APP
+; RV32IFD-NEXT:    fcvt.w.d a0, fs8
+; RV32IFD-NEXT:    #NO_APP
+; RV32IFD-NEXT:    fld fs8, 8(sp)
+; RV32IFD-NEXT:    addi sp, sp, 16
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: explicit_register_fs8:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    fsd fs8, 8(sp)
+; RV64IFD-NEXT:    fmv.d fs8, fa0
+; RV64IFD-NEXT:    #APP
+; RV64IFD-NEXT:    fcvt.w.d a0, fs8
+; RV64IFD-NEXT:    #NO_APP
+; RV64IFD-NEXT:    fld fs8, 8(sp)
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{fs8}"(double %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `f25` (`fs9`) as an input, so it should be saved.
+define i32 @explicit_register_f25(double %a) nounwind {
+; RV32IFD-LABEL: explicit_register_f25:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi sp, sp, -16
+; RV32IFD-NEXT:    fsd fs9, 8(sp)
+; RV32IFD-NEXT:    fmv.d fs9, fa0
+; RV32IFD-NEXT:    #APP
+; RV32IFD-NEXT:    fcvt.w.d a0, fs9
+; RV32IFD-NEXT:    #NO_APP
+; RV32IFD-NEXT:    fld fs9, 8(sp)
+; RV32IFD-NEXT:    addi sp, sp, 16
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: explicit_register_f25:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    fsd fs9, 8(sp)
+; RV64IFD-NEXT:    fmv.d fs9, fa0
+; RV64IFD-NEXT:    #APP
+; RV64IFD-NEXT:    fcvt.w.d a0, fs9
+; RV64IFD-NEXT:    #NO_APP
+; RV64IFD-NEXT:    fld fs9, 8(sp)
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{f25}"(double %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `fs9` (`f25`) as an input, so it should be saved.
+define i32 @explicit_register_fs9(double %a) nounwind {
+; RV32IFD-LABEL: explicit_register_fs9:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi sp, sp, -16
+; RV32IFD-NEXT:    fsd fs9, 8(sp)
+; RV32IFD-NEXT:    fmv.d fs9, fa0
+; RV32IFD-NEXT:    #APP
+; RV32IFD-NEXT:    fcvt.w.d a0, fs9
+; RV32IFD-NEXT:    #NO_APP
+; RV32IFD-NEXT:    fld fs9, 8(sp)
+; RV32IFD-NEXT:    addi sp, sp, 16
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: explicit_register_fs9:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    fsd fs9, 8(sp)
+; RV64IFD-NEXT:    fmv.d fs9, fa0
+; RV64IFD-NEXT:    #APP
+; RV64IFD-NEXT:    fcvt.w.d a0, fs9
+; RV64IFD-NEXT:    #NO_APP
+; RV64IFD-NEXT:    fld fs9, 8(sp)
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{fs9}"(double %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `f26` (`fs10`) as an input, so it should be saved.
+define i32 @explicit_register_f26(double %a) nounwind {
+; RV32IFD-LABEL: explicit_register_f26:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi sp, sp, -16
+; RV32IFD-NEXT:    fsd fs10, 8(sp)
+; RV32IFD-NEXT:    fmv.d fs10, fa0
+; RV32IFD-NEXT:    #APP
+; RV32IFD-NEXT:    fcvt.w.d a0, fs10
+; RV32IFD-NEXT:    #NO_APP
+; RV32IFD-NEXT:    fld fs10, 8(sp)
+; RV32IFD-NEXT:    addi sp, sp, 16
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: explicit_register_f26:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    fsd fs10, 8(sp)
+; RV64IFD-NEXT:    fmv.d fs10, fa0
+; RV64IFD-NEXT:    #APP
+; RV64IFD-NEXT:    fcvt.w.d a0, fs10
+; RV64IFD-NEXT:    #NO_APP
+; RV64IFD-NEXT:    fld fs10, 8(sp)
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{f26}"(double %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `fs10` (`f26`) as an input, so it should be saved.
+define i32 @explicit_register_fs10(double %a) nounwind {
+; RV32IFD-LABEL: explicit_register_fs10:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi sp, sp, -16
+; RV32IFD-NEXT:    fsd fs10, 8(sp)
+; RV32IFD-NEXT:    fmv.d fs10, fa0
+; RV32IFD-NEXT:    #APP
+; RV32IFD-NEXT:    fcvt.w.d a0, fs10
+; RV32IFD-NEXT:    #NO_APP
+; RV32IFD-NEXT:    fld fs10, 8(sp)
+; RV32IFD-NEXT:    addi sp, sp, 16
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: explicit_register_fs10:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    fsd fs10, 8(sp)
+; RV64IFD-NEXT:    fmv.d fs10, fa0
+; RV64IFD-NEXT:    #APP
+; RV64IFD-NEXT:    fcvt.w.d a0, fs10
+; RV64IFD-NEXT:    #NO_APP
+; RV64IFD-NEXT:    fld fs10, 8(sp)
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{fs10}"(double %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `f27` (`fs11`) as an input, so it should be saved.
+define i32 @explicit_register_f27(double %a) nounwind {
+; RV32IFD-LABEL: explicit_register_f27:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi sp, sp, -16
+; RV32IFD-NEXT:    fsd fs11, 8(sp)
+; RV32IFD-NEXT:    fmv.d fs11, fa0
+; RV32IFD-NEXT:    #APP
+; RV32IFD-NEXT:    fcvt.w.d a0, fs11
+; RV32IFD-NEXT:    #NO_APP
+; RV32IFD-NEXT:    fld fs11, 8(sp)
+; RV32IFD-NEXT:    addi sp, sp, 16
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: explicit_register_f27:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    fsd fs11, 8(sp)
+; RV64IFD-NEXT:    fmv.d fs11, fa0
+; RV64IFD-NEXT:    #APP
+; RV64IFD-NEXT:    fcvt.w.d a0, fs11
+; RV64IFD-NEXT:    #NO_APP
+; RV64IFD-NEXT:    fld fs11, 8(sp)
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{f27}"(double %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `fs11` (`f27`) as an input, so it should be saved.
+define i32 @explicit_register_fs11(double %a) nounwind {
+; RV32IFD-LABEL: explicit_register_fs11:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi sp, sp, -16
+; RV32IFD-NEXT:    fsd fs11, 8(sp)
+; RV32IFD-NEXT:    fmv.d fs11, fa0
+; RV32IFD-NEXT:    #APP
+; RV32IFD-NEXT:    fcvt.w.d a0, fs11
+; RV32IFD-NEXT:    #NO_APP
+; RV32IFD-NEXT:    fld fs11, 8(sp)
+; RV32IFD-NEXT:    addi sp, sp, 16
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: explicit_register_fs11:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    fsd fs11, 8(sp)
+; RV64IFD-NEXT:    fmv.d fs11, fa0
+; RV64IFD-NEXT:    #APP
+; RV64IFD-NEXT:    fcvt.w.d a0, fs11
+; RV64IFD-NEXT:    #NO_APP
+; RV64IFD-NEXT:    fld fs11, 8(sp)
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{fs11}"(double %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_f28(double %a) nounwind {
+; RV32IFD-LABEL: explicit_register_f28:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    fmv.d ft8, fa0
+; RV32IFD-NEXT:    #APP
+; RV32IFD-NEXT:    fcvt.w.d a0, ft8
+; RV32IFD-NEXT:    #NO_APP
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: explicit_register_f28:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d ft8, fa0
+; RV64IFD-NEXT:    #APP
+; RV64IFD-NEXT:    fcvt.w.d a0, ft8
+; RV64IFD-NEXT:    #NO_APP
+; RV64IFD-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{f28}"(double %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_ft8(double %a) nounwind {
+; RV32IFD-LABEL: explicit_register_ft8:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    fmv.d ft8, fa0
+; RV32IFD-NEXT:    #APP
+; RV32IFD-NEXT:    fcvt.w.d a0, ft8
+; RV32IFD-NEXT:    #NO_APP
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: explicit_register_ft8:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d ft8, fa0
+; RV64IFD-NEXT:    #APP
+; RV64IFD-NEXT:    fcvt.w.d a0, ft8
+; RV64IFD-NEXT:    #NO_APP
+; RV64IFD-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{ft8}"(double %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_f29(double %a) nounwind {
+; RV32IFD-LABEL: explicit_register_f29:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    fmv.d ft9, fa0
+; RV32IFD-NEXT:    #APP
+; RV32IFD-NEXT:    fcvt.w.d a0, ft9
+; RV32IFD-NEXT:    #NO_APP
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: explicit_register_f29:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d ft9, fa0
+; RV64IFD-NEXT:    #APP
+; RV64IFD-NEXT:    fcvt.w.d a0, ft9
+; RV64IFD-NEXT:    #NO_APP
+; RV64IFD-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{f29}"(double %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_ft9(double %a) nounwind {
+; RV32IFD-LABEL: explicit_register_ft9:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    fmv.d ft9, fa0
+; RV32IFD-NEXT:    #APP
+; RV32IFD-NEXT:    fcvt.w.d a0, ft9
+; RV32IFD-NEXT:    #NO_APP
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: explicit_register_ft9:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d ft9, fa0
+; RV64IFD-NEXT:    #APP
+; RV64IFD-NEXT:    fcvt.w.d a0, ft9
+; RV64IFD-NEXT:    #NO_APP
+; RV64IFD-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{ft9}"(double %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_f30(double %a) nounwind {
+; RV32IFD-LABEL: explicit_register_f30:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    fmv.d ft10, fa0
+; RV32IFD-NEXT:    #APP
+; RV32IFD-NEXT:    fcvt.w.d a0, ft10
+; RV32IFD-NEXT:    #NO_APP
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: explicit_register_f30:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d ft10, fa0
+; RV64IFD-NEXT:    #APP
+; RV64IFD-NEXT:    fcvt.w.d a0, ft10
+; RV64IFD-NEXT:    #NO_APP
+; RV64IFD-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{f30}"(double %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_ft10(double %a) nounwind {
+; RV32IFD-LABEL: explicit_register_ft10:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    fmv.d ft10, fa0
+; RV32IFD-NEXT:    #APP
+; RV32IFD-NEXT:    fcvt.w.d a0, ft10
+; RV32IFD-NEXT:    #NO_APP
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: explicit_register_ft10:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d ft10, fa0
+; RV64IFD-NEXT:    #APP
+; RV64IFD-NEXT:    fcvt.w.d a0, ft10
+; RV64IFD-NEXT:    #NO_APP
+; RV64IFD-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{ft10}"(double %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_f31(double %a) nounwind {
+; RV32IFD-LABEL: explicit_register_f31:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    fmv.d ft11, fa0
+; RV32IFD-NEXT:    #APP
+; RV32IFD-NEXT:    fcvt.w.d a0, ft11
+; RV32IFD-NEXT:    #NO_APP
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: explicit_register_f31:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d ft11, fa0
+; RV64IFD-NEXT:    #APP
+; RV64IFD-NEXT:    fcvt.w.d a0, ft11
+; RV64IFD-NEXT:    #NO_APP
+; RV64IFD-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{f31}"(double %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_ft11(double %a) nounwind {
+; RV32IFD-LABEL: explicit_register_ft11:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    fmv.d ft11, fa0
+; RV32IFD-NEXT:    #APP
+; RV32IFD-NEXT:    fcvt.w.d a0, ft11
+; RV32IFD-NEXT:    #NO_APP
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: explicit_register_ft11:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d ft11, fa0
+; RV64IFD-NEXT:    #APP
+; RV64IFD-NEXT:    fcvt.w.d a0, ft11
+; RV64IFD-NEXT:    #NO_APP
+; RV64IFD-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{ft11}"(double %a)
+  ret i32 %1
+}
diff --git a/llvm/test/CodeGen/RISCV/inline-asm-d-constraint-f.ll b/llvm/test/CodeGen/RISCV/inline-asm-d-constraint-f.ll
index 251c06343..37ce0f89e 100644
--- a/llvm/test/CodeGen/RISCV/inline-asm-d-constraint-f.ll
+++ b/llvm/test/CodeGen/RISCV/inline-asm-d-constraint-f.ll
@@ -26,11 +26,11 @@ define double @constraint_f_double(double %a) nounwind {
 ;
 ; RV64F-LABEL: constraint_f_double:
 ; RV64F:       # %bb.0:
-; RV64F-NEXT:    fmv.d.x ft0, a0
-; RV64F-NEXT:    lui a0, %hi(gd)
-; RV64F-NEXT:    fld ft1, %lo(gd)(a0)
+; RV64F-NEXT:    lui a1, %hi(gd)
+; RV64F-NEXT:    fld ft0, %lo(gd)(a1)
+; RV64F-NEXT:    fmv.d.x ft1, a0
 ; RV64F-NEXT:    #APP
-; RV64F-NEXT:    fadd.d ft0, ft0, ft1
+; RV64F-NEXT:    fadd.d ft0, ft1, ft0
 ; RV64F-NEXT:    #NO_APP
 ; RV64F-NEXT:    fmv.x.d a0, ft0
 ; RV64F-NEXT:    ret
@@ -38,3 +38,36 @@ define double @constraint_f_double(double %a) nounwind {
   %2 = tail call double asm "fadd.d $0, $1, $2", "=f,f,f"(double %a, double %1)
   ret double %2
 }
+
+define double @constraint_f_double_abi_name(double %a) nounwind {
+; RV32F-LABEL: constraint_f_double_abi_name:
+; RV32F:       # %bb.0:
+; RV32F-NEXT:    addi sp, sp, -16
+; RV32F-NEXT:    sw a0, 8(sp)
+; RV32F-NEXT:    sw a1, 12(sp)
+; RV32F-NEXT:    fld fa1, 8(sp)
+; RV32F-NEXT:    lui a0, %hi(gd)
+; RV32F-NEXT:    fld fs0, %lo(gd)(a0)
+; RV32F-NEXT:    #APP
+; RV32F-NEXT:    fadd.d ft0, fa1, fs0
+; RV32F-NEXT:    #NO_APP
+; RV32F-NEXT:    fsd ft0, 8(sp)
+; RV32F-NEXT:    lw a0, 8(sp)
+; RV32F-NEXT:    lw a1, 12(sp)
+; RV32F-NEXT:    addi sp, sp, 16
+; RV32F-NEXT:    ret
+;
+; RV64F-LABEL: constraint_f_double_abi_name:
+; RV64F:       # %bb.0:
+; RV64F-NEXT:    lui a1, %hi(gd)
+; RV64F-NEXT:    fld fs0, %lo(gd)(a1)
+; RV64F-NEXT:    fmv.d.x fa1, a0
+; RV64F-NEXT:    #APP
+; RV64F-NEXT:    fadd.d ft0, fa1, fs0
+; RV64F-NEXT:    #NO_APP
+; RV64F-NEXT:    fmv.x.d a0, ft0
+; RV64F-NEXT:    ret
+  %1 = load double, double* @gd
+  %2 = tail call double asm "fadd.d $0, $1, $2", "={ft0},{fa1},{fs0}"(double %a, double %1)
+  ret double %2
+}
diff --git a/llvm/test/CodeGen/RISCV/inline-asm-f-abi-names.ll b/llvm/test/CodeGen/RISCV/inline-asm-f-abi-names.ll
new file mode 100644
index 000000000..8eddc3ebf
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/inline-asm-f-abi-names.ll
@@ -0,0 +1,1509 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+f -target-abi ilp32f -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV32IF %s
+; RUN: llc -mtriple=riscv64 -mattr=+f -target-abi lp64f -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64IF %s
+
+; These test that we can use both the architectural names (x*) and the ABI names
+; (a*, s*, t* etc) to refer to registers in inline asm constraint lists. In each
+; case, the named register should be used for the source register of the `addi`.
+; It is very likely that `a0` will be chosen as the designation register, but
+; this is left to the compiler to choose.
+;
+; The inline assembly will, by default, contain the ABI names for the registers.
+;
+; Parenthesised registers in comments are the other aliases for this register.
+
+
+define i32 @explicit_register_f0(float %a) nounwind {
+; RV32IF-LABEL: explicit_register_f0:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    fmv.s ft0, fa0
+; RV32IF-NEXT:    #APP
+; RV32IF-NEXT:    fcvt.w.s a0, ft0
+; RV32IF-NEXT:    #NO_APP
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: explicit_register_f0:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.s ft0, fa0
+; RV64IF-NEXT:    #APP
+; RV64IF-NEXT:    fcvt.w.s a0, ft0
+; RV64IF-NEXT:    #NO_APP
+; RV64IF-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{f0}"(float %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_ft0(float %a) nounwind {
+; RV32IF-LABEL: explicit_register_ft0:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    fmv.s ft0, fa0
+; RV32IF-NEXT:    #APP
+; RV32IF-NEXT:    fcvt.w.s a0, ft0
+; RV32IF-NEXT:    #NO_APP
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: explicit_register_ft0:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.s ft0, fa0
+; RV64IF-NEXT:    #APP
+; RV64IF-NEXT:    fcvt.w.s a0, ft0
+; RV64IF-NEXT:    #NO_APP
+; RV64IF-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{ft0}"(float %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_f1(float %a) nounwind {
+; RV32IF-LABEL: explicit_register_f1:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    fmv.s ft1, fa0
+; RV32IF-NEXT:    #APP
+; RV32IF-NEXT:    fcvt.w.s a0, ft1
+; RV32IF-NEXT:    #NO_APP
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: explicit_register_f1:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.s ft1, fa0
+; RV64IF-NEXT:    #APP
+; RV64IF-NEXT:    fcvt.w.s a0, ft1
+; RV64IF-NEXT:    #NO_APP
+; RV64IF-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{f1}"(float %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_ft1(float %a) nounwind {
+; RV32IF-LABEL: explicit_register_ft1:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    fmv.s ft1, fa0
+; RV32IF-NEXT:    #APP
+; RV32IF-NEXT:    fcvt.w.s a0, ft1
+; RV32IF-NEXT:    #NO_APP
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: explicit_register_ft1:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.s ft1, fa0
+; RV64IF-NEXT:    #APP
+; RV64IF-NEXT:    fcvt.w.s a0, ft1
+; RV64IF-NEXT:    #NO_APP
+; RV64IF-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{ft1}"(float %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_f2(float %a) nounwind {
+; RV32IF-LABEL: explicit_register_f2:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    fmv.s ft2, fa0
+; RV32IF-NEXT:    #APP
+; RV32IF-NEXT:    fcvt.w.s a0, ft2
+; RV32IF-NEXT:    #NO_APP
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: explicit_register_f2:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.s ft2, fa0
+; RV64IF-NEXT:    #APP
+; RV64IF-NEXT:    fcvt.w.s a0, ft2
+; RV64IF-NEXT:    #NO_APP
+; RV64IF-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{f2}"(float %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_ft2(float %a) nounwind {
+; RV32IF-LABEL: explicit_register_ft2:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    fmv.s ft2, fa0
+; RV32IF-NEXT:    #APP
+; RV32IF-NEXT:    fcvt.w.s a0, ft2
+; RV32IF-NEXT:    #NO_APP
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: explicit_register_ft2:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.s ft2, fa0
+; RV64IF-NEXT:    #APP
+; RV64IF-NEXT:    fcvt.w.s a0, ft2
+; RV64IF-NEXT:    #NO_APP
+; RV64IF-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{ft2}"(float %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_f3(float %a) nounwind {
+; RV32IF-LABEL: explicit_register_f3:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    fmv.s ft3, fa0
+; RV32IF-NEXT:    #APP
+; RV32IF-NEXT:    fcvt.w.s a0, ft3
+; RV32IF-NEXT:    #NO_APP
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: explicit_register_f3:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.s ft3, fa0
+; RV64IF-NEXT:    #APP
+; RV64IF-NEXT:    fcvt.w.s a0, ft3
+; RV64IF-NEXT:    #NO_APP
+; RV64IF-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{f3}"(float %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_ft3(float %a) nounwind {
+; RV32IF-LABEL: explicit_register_ft3:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    fmv.s ft3, fa0
+; RV32IF-NEXT:    #APP
+; RV32IF-NEXT:    fcvt.w.s a0, ft3
+; RV32IF-NEXT:    #NO_APP
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: explicit_register_ft3:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.s ft3, fa0
+; RV64IF-NEXT:    #APP
+; RV64IF-NEXT:    fcvt.w.s a0, ft3
+; RV64IF-NEXT:    #NO_APP
+; RV64IF-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{ft3}"(float %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_f4(float %a) nounwind {
+; RV32IF-LABEL: explicit_register_f4:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    fmv.s ft4, fa0
+; RV32IF-NEXT:    #APP
+; RV32IF-NEXT:    fcvt.w.s a0, ft4
+; RV32IF-NEXT:    #NO_APP
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: explicit_register_f4:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.s ft4, fa0
+; RV64IF-NEXT:    #APP
+; RV64IF-NEXT:    fcvt.w.s a0, ft4
+; RV64IF-NEXT:    #NO_APP
+; RV64IF-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{f4}"(float %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_ft4(float %a) nounwind {
+; RV32IF-LABEL: explicit_register_ft4:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    fmv.s ft4, fa0
+; RV32IF-NEXT:    #APP
+; RV32IF-NEXT:    fcvt.w.s a0, ft4
+; RV32IF-NEXT:    #NO_APP
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: explicit_register_ft4:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.s ft4, fa0
+; RV64IF-NEXT:    #APP
+; RV64IF-NEXT:    fcvt.w.s a0, ft4
+; RV64IF-NEXT:    #NO_APP
+; RV64IF-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{ft4}"(float %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_f5(float %a) nounwind {
+; RV32IF-LABEL: explicit_register_f5:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    fmv.s ft5, fa0
+; RV32IF-NEXT:    #APP
+; RV32IF-NEXT:    fcvt.w.s a0, ft5
+; RV32IF-NEXT:    #NO_APP
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: explicit_register_f5:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.s ft5, fa0
+; RV64IF-NEXT:    #APP
+; RV64IF-NEXT:    fcvt.w.s a0, ft5
+; RV64IF-NEXT:    #NO_APP
+; RV64IF-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{f5}"(float %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_ft5(float %a) nounwind {
+; RV32IF-LABEL: explicit_register_ft5:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    fmv.s ft5, fa0
+; RV32IF-NEXT:    #APP
+; RV32IF-NEXT:    fcvt.w.s a0, ft5
+; RV32IF-NEXT:    #NO_APP
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: explicit_register_ft5:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.s ft5, fa0
+; RV64IF-NEXT:    #APP
+; RV64IF-NEXT:    fcvt.w.s a0, ft5
+; RV64IF-NEXT:    #NO_APP
+; RV64IF-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{ft5}"(float %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_f6(float %a) nounwind {
+; RV32IF-LABEL: explicit_register_f6:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    fmv.s ft6, fa0
+; RV32IF-NEXT:    #APP
+; RV32IF-NEXT:    fcvt.w.s a0, ft6
+; RV32IF-NEXT:    #NO_APP
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: explicit_register_f6:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.s ft6, fa0
+; RV64IF-NEXT:    #APP
+; RV64IF-NEXT:    fcvt.w.s a0, ft6
+; RV64IF-NEXT:    #NO_APP
+; RV64IF-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{f6}"(float %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_ft6(float %a) nounwind {
+; RV32IF-LABEL: explicit_register_ft6:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    fmv.s ft6, fa0
+; RV32IF-NEXT:    #APP
+; RV32IF-NEXT:    fcvt.w.s a0, ft6
+; RV32IF-NEXT:    #NO_APP
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: explicit_register_ft6:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.s ft6, fa0
+; RV64IF-NEXT:    #APP
+; RV64IF-NEXT:    fcvt.w.s a0, ft6
+; RV64IF-NEXT:    #NO_APP
+; RV64IF-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{ft6}"(float %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_f7(float %a) nounwind {
+; RV32IF-LABEL: explicit_register_f7:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    fmv.s ft7, fa0
+; RV32IF-NEXT:    #APP
+; RV32IF-NEXT:    fcvt.w.s a0, ft7
+; RV32IF-NEXT:    #NO_APP
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: explicit_register_f7:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.s ft7, fa0
+; RV64IF-NEXT:    #APP
+; RV64IF-NEXT:    fcvt.w.s a0, ft7
+; RV64IF-NEXT:    #NO_APP
+; RV64IF-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{f7}"(float %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_ft7(float %a) nounwind {
+; RV32IF-LABEL: explicit_register_ft7:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    fmv.s ft7, fa0
+; RV32IF-NEXT:    #APP
+; RV32IF-NEXT:    fcvt.w.s a0, ft7
+; RV32IF-NEXT:    #NO_APP
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: explicit_register_ft7:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.s ft7, fa0
+; RV64IF-NEXT:    #APP
+; RV64IF-NEXT:    fcvt.w.s a0, ft7
+; RV64IF-NEXT:    #NO_APP
+; RV64IF-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{ft7}"(float %a)
+  ret i32 %1
+}
+
+
+; NOTE: This test uses `f8` (`fs0`) as an input, so it should be saved.
+define i32 @explicit_register_f8(float %a) nounwind {
+; RV32IF-LABEL: explicit_register_f8:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    addi sp, sp, -16
+; RV32IF-NEXT:    fsw fs0, 12(sp)
+; RV32IF-NEXT:    fmv.s fs0, fa0
+; RV32IF-NEXT:    #APP
+; RV32IF-NEXT:    fcvt.w.s a0, fs0
+; RV32IF-NEXT:    #NO_APP
+; RV32IF-NEXT:    flw fs0, 12(sp)
+; RV32IF-NEXT:    addi sp, sp, 16
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: explicit_register_f8:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    fsw fs0, 12(sp)
+; RV64IF-NEXT:    fmv.s fs0, fa0
+; RV64IF-NEXT:    #APP
+; RV64IF-NEXT:    fcvt.w.s a0, fs0
+; RV64IF-NEXT:    #NO_APP
+; RV64IF-NEXT:    flw fs0, 12(sp)
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{f8}"(float %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `fs0` (`f8`) as an input, so it should be saved.
+define i32 @explicit_register_fs0(float %a) nounwind {
+; RV32IF-LABEL: explicit_register_fs0:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    addi sp, sp, -16
+; RV32IF-NEXT:    fsw fs0, 12(sp)
+; RV32IF-NEXT:    fmv.s fs0, fa0
+; RV32IF-NEXT:    #APP
+; RV32IF-NEXT:    fcvt.w.s a0, fs0
+; RV32IF-NEXT:    #NO_APP
+; RV32IF-NEXT:    flw fs0, 12(sp)
+; RV32IF-NEXT:    addi sp, sp, 16
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: explicit_register_fs0:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    fsw fs0, 12(sp)
+; RV64IF-NEXT:    fmv.s fs0, fa0
+; RV64IF-NEXT:    #APP
+; RV64IF-NEXT:    fcvt.w.s a0, fs0
+; RV64IF-NEXT:    #NO_APP
+; RV64IF-NEXT:    flw fs0, 12(sp)
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{fs0}"(float %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `f9` (`fs1`) as an input, so it should be saved.
+define i32 @explicit_register_f9(float %a) nounwind {
+; RV32IF-LABEL: explicit_register_f9:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    addi sp, sp, -16
+; RV32IF-NEXT:    fsw fs1, 12(sp)
+; RV32IF-NEXT:    fmv.s fs1, fa0
+; RV32IF-NEXT:    #APP
+; RV32IF-NEXT:    fcvt.w.s a0, fs1
+; RV32IF-NEXT:    #NO_APP
+; RV32IF-NEXT:    flw fs1, 12(sp)
+; RV32IF-NEXT:    addi sp, sp, 16
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: explicit_register_f9:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    fsw fs1, 12(sp)
+; RV64IF-NEXT:    fmv.s fs1, fa0
+; RV64IF-NEXT:    #APP
+; RV64IF-NEXT:    fcvt.w.s a0, fs1
+; RV64IF-NEXT:    #NO_APP
+; RV64IF-NEXT:    flw fs1, 12(sp)
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{f9}"(float %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `fs1` (`f9`) as an input, so it should be saved.
+define i32 @explicit_register_fs1(float %a) nounwind {
+; RV32IF-LABEL: explicit_register_fs1:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    addi sp, sp, -16
+; RV32IF-NEXT:    fsw fs1, 12(sp)
+; RV32IF-NEXT:    fmv.s fs1, fa0
+; RV32IF-NEXT:    #APP
+; RV32IF-NEXT:    fcvt.w.s a0, fs1
+; RV32IF-NEXT:    #NO_APP
+; RV32IF-NEXT:    flw fs1, 12(sp)
+; RV32IF-NEXT:    addi sp, sp, 16
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: explicit_register_fs1:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    fsw fs1, 12(sp)
+; RV64IF-NEXT:    fmv.s fs1, fa0
+; RV64IF-NEXT:    #APP
+; RV64IF-NEXT:    fcvt.w.s a0, fs1
+; RV64IF-NEXT:    #NO_APP
+; RV64IF-NEXT:    flw fs1, 12(sp)
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{fs1}"(float %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_f10(float %a) nounwind {
+; RV32IF-LABEL: explicit_register_f10:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    #APP
+; RV32IF-NEXT:    fcvt.w.s a0, fa0
+; RV32IF-NEXT:    #NO_APP
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: explicit_register_f10:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    #APP
+; RV64IF-NEXT:    fcvt.w.s a0, fa0
+; RV64IF-NEXT:    #NO_APP
+; RV64IF-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{f10}"(float %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_fa0(float %a) nounwind {
+; RV32IF-LABEL: explicit_register_fa0:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    #APP
+; RV32IF-NEXT:    fcvt.w.s a0, fa0
+; RV32IF-NEXT:    #NO_APP
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: explicit_register_fa0:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    #APP
+; RV64IF-NEXT:    fcvt.w.s a0, fa0
+; RV64IF-NEXT:    #NO_APP
+; RV64IF-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{fa0}"(float %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_f11(float %a) nounwind {
+; RV32IF-LABEL: explicit_register_f11:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    fmv.s fa1, fa0
+; RV32IF-NEXT:    #APP
+; RV32IF-NEXT:    fcvt.w.s a0, fa1
+; RV32IF-NEXT:    #NO_APP
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: explicit_register_f11:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.s fa1, fa0
+; RV64IF-NEXT:    #APP
+; RV64IF-NEXT:    fcvt.w.s a0, fa1
+; RV64IF-NEXT:    #NO_APP
+; RV64IF-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{f11}"(float %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_fa1(float %a) nounwind {
+; RV32IF-LABEL: explicit_register_fa1:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    fmv.s fa1, fa0
+; RV32IF-NEXT:    #APP
+; RV32IF-NEXT:    fcvt.w.s a0, fa1
+; RV32IF-NEXT:    #NO_APP
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: explicit_register_fa1:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.s fa1, fa0
+; RV64IF-NEXT:    #APP
+; RV64IF-NEXT:    fcvt.w.s a0, fa1
+; RV64IF-NEXT:    #NO_APP
+; RV64IF-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{fa1}"(float %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_f12(float %a) nounwind {
+; RV32IF-LABEL: explicit_register_f12:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    fmv.s fa2, fa0
+; RV32IF-NEXT:    #APP
+; RV32IF-NEXT:    fcvt.w.s a0, fa2
+; RV32IF-NEXT:    #NO_APP
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: explicit_register_f12:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.s fa2, fa0
+; RV64IF-NEXT:    #APP
+; RV64IF-NEXT:    fcvt.w.s a0, fa2
+; RV64IF-NEXT:    #NO_APP
+; RV64IF-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{f12}"(float %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_fa2(float %a) nounwind {
+; RV32IF-LABEL: explicit_register_fa2:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    fmv.s fa2, fa0
+; RV32IF-NEXT:    #APP
+; RV32IF-NEXT:    fcvt.w.s a0, fa2
+; RV32IF-NEXT:    #NO_APP
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: explicit_register_fa2:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.s fa2, fa0
+; RV64IF-NEXT:    #APP
+; RV64IF-NEXT:    fcvt.w.s a0, fa2
+; RV64IF-NEXT:    #NO_APP
+; RV64IF-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{fa2}"(float %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_f13(float %a) nounwind {
+; RV32IF-LABEL: explicit_register_f13:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    fmv.s fa3, fa0
+; RV32IF-NEXT:    #APP
+; RV32IF-NEXT:    fcvt.w.s a0, fa3
+; RV32IF-NEXT:    #NO_APP
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: explicit_register_f13:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.s fa3, fa0
+; RV64IF-NEXT:    #APP
+; RV64IF-NEXT:    fcvt.w.s a0, fa3
+; RV64IF-NEXT:    #NO_APP
+; RV64IF-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{f13}"(float %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_fa3(float %a) nounwind {
+; RV32IF-LABEL: explicit_register_fa3:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    fmv.s fa3, fa0
+; RV32IF-NEXT:    #APP
+; RV32IF-NEXT:    fcvt.w.s a0, fa3
+; RV32IF-NEXT:    #NO_APP
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: explicit_register_fa3:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.s fa3, fa0
+; RV64IF-NEXT:    #APP
+; RV64IF-NEXT:    fcvt.w.s a0, fa3
+; RV64IF-NEXT:    #NO_APP
+; RV64IF-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{fa3}"(float %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_f14(float %a) nounwind {
+; RV32IF-LABEL: explicit_register_f14:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    fmv.s fa4, fa0
+; RV32IF-NEXT:    #APP
+; RV32IF-NEXT:    fcvt.w.s a0, fa4
+; RV32IF-NEXT:    #NO_APP
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: explicit_register_f14:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.s fa4, fa0
+; RV64IF-NEXT:    #APP
+; RV64IF-NEXT:    fcvt.w.s a0, fa4
+; RV64IF-NEXT:    #NO_APP
+; RV64IF-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{f14}"(float %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_fa4(float %a) nounwind {
+; RV32IF-LABEL: explicit_register_fa4:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    fmv.s fa4, fa0
+; RV32IF-NEXT:    #APP
+; RV32IF-NEXT:    fcvt.w.s a0, fa4
+; RV32IF-NEXT:    #NO_APP
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: explicit_register_fa4:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.s fa4, fa0
+; RV64IF-NEXT:    #APP
+; RV64IF-NEXT:    fcvt.w.s a0, fa4
+; RV64IF-NEXT:    #NO_APP
+; RV64IF-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{fa4}"(float %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_f15(float %a) nounwind {
+; RV32IF-LABEL: explicit_register_f15:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    fmv.s fa5, fa0
+; RV32IF-NEXT:    #APP
+; RV32IF-NEXT:    fcvt.w.s a0, fa5
+; RV32IF-NEXT:    #NO_APP
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: explicit_register_f15:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.s fa5, fa0
+; RV64IF-NEXT:    #APP
+; RV64IF-NEXT:    fcvt.w.s a0, fa5
+; RV64IF-NEXT:    #NO_APP
+; RV64IF-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{f15}"(float %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_fa5(float %a) nounwind {
+; RV32IF-LABEL: explicit_register_fa5:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    fmv.s fa5, fa0
+; RV32IF-NEXT:    #APP
+; RV32IF-NEXT:    fcvt.w.s a0, fa5
+; RV32IF-NEXT:    #NO_APP
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: explicit_register_fa5:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.s fa5, fa0
+; RV64IF-NEXT:    #APP
+; RV64IF-NEXT:    fcvt.w.s a0, fa5
+; RV64IF-NEXT:    #NO_APP
+; RV64IF-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{fa5}"(float %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_f16(float %a) nounwind {
+; RV32IF-LABEL: explicit_register_f16:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    fmv.s fa6, fa0
+; RV32IF-NEXT:    #APP
+; RV32IF-NEXT:    fcvt.w.s a0, fa6
+; RV32IF-NEXT:    #NO_APP
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: explicit_register_f16:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.s fa6, fa0
+; RV64IF-NEXT:    #APP
+; RV64IF-NEXT:    fcvt.w.s a0, fa6
+; RV64IF-NEXT:    #NO_APP
+; RV64IF-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{f16}"(float %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_fa6(float %a) nounwind {
+; RV32IF-LABEL: explicit_register_fa6:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    fmv.s fa6, fa0
+; RV32IF-NEXT:    #APP
+; RV32IF-NEXT:    fcvt.w.s a0, fa6
+; RV32IF-NEXT:    #NO_APP
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: explicit_register_fa6:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.s fa6, fa0
+; RV64IF-NEXT:    #APP
+; RV64IF-NEXT:    fcvt.w.s a0, fa6
+; RV64IF-NEXT:    #NO_APP
+; RV64IF-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{fa6}"(float %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_f17(float %a) nounwind {
+; RV32IF-LABEL: explicit_register_f17:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    fmv.s fa7, fa0
+; RV32IF-NEXT:    #APP
+; RV32IF-NEXT:    fcvt.w.s a0, fa7
+; RV32IF-NEXT:    #NO_APP
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: explicit_register_f17:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.s fa7, fa0
+; RV64IF-NEXT:    #APP
+; RV64IF-NEXT:    fcvt.w.s a0, fa7
+; RV64IF-NEXT:    #NO_APP
+; RV64IF-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{f17}"(float %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_fa7(float %a) nounwind {
+; RV32IF-LABEL: explicit_register_fa7:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    fmv.s fa7, fa0
+; RV32IF-NEXT:    #APP
+; RV32IF-NEXT:    fcvt.w.s a0, fa7
+; RV32IF-NEXT:    #NO_APP
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: explicit_register_fa7:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.s fa7, fa0
+; RV64IF-NEXT:    #APP
+; RV64IF-NEXT:    fcvt.w.s a0, fa7
+; RV64IF-NEXT:    #NO_APP
+; RV64IF-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{fa7}"(float %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `f18` (`fs2`) as an input, so it should be saved.
+define i32 @explicit_register_f18(float %a) nounwind {
+; RV32IF-LABEL: explicit_register_f18:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    addi sp, sp, -16
+; RV32IF-NEXT:    fsw fs2, 12(sp)
+; RV32IF-NEXT:    fmv.s fs2, fa0
+; RV32IF-NEXT:    #APP
+; RV32IF-NEXT:    fcvt.w.s a0, fs2
+; RV32IF-NEXT:    #NO_APP
+; RV32IF-NEXT:    flw fs2, 12(sp)
+; RV32IF-NEXT:    addi sp, sp, 16
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: explicit_register_f18:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    fsw fs2, 12(sp)
+; RV64IF-NEXT:    fmv.s fs2, fa0
+; RV64IF-NEXT:    #APP
+; RV64IF-NEXT:    fcvt.w.s a0, fs2
+; RV64IF-NEXT:    #NO_APP
+; RV64IF-NEXT:    flw fs2, 12(sp)
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{f18}"(float %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `fs2` (`f18`) as an input, so it should be saved.
+define i32 @explicit_register_fs2(float %a) nounwind {
+; RV32IF-LABEL: explicit_register_fs2:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    addi sp, sp, -16
+; RV32IF-NEXT:    fsw fs2, 12(sp)
+; RV32IF-NEXT:    fmv.s fs2, fa0
+; RV32IF-NEXT:    #APP
+; RV32IF-NEXT:    fcvt.w.s a0, fs2
+; RV32IF-NEXT:    #NO_APP
+; RV32IF-NEXT:    flw fs2, 12(sp)
+; RV32IF-NEXT:    addi sp, sp, 16
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: explicit_register_fs2:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    fsw fs2, 12(sp)
+; RV64IF-NEXT:    fmv.s fs2, fa0
+; RV64IF-NEXT:    #APP
+; RV64IF-NEXT:    fcvt.w.s a0, fs2
+; RV64IF-NEXT:    #NO_APP
+; RV64IF-NEXT:    flw fs2, 12(sp)
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{fs2}"(float %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `f19` (`fs3`) as an input, so it should be saved.
+define i32 @explicit_register_f19(float %a) nounwind {
+; RV32IF-LABEL: explicit_register_f19:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    addi sp, sp, -16
+; RV32IF-NEXT:    fsw fs3, 12(sp)
+; RV32IF-NEXT:    fmv.s fs3, fa0
+; RV32IF-NEXT:    #APP
+; RV32IF-NEXT:    fcvt.w.s a0, fs3
+; RV32IF-NEXT:    #NO_APP
+; RV32IF-NEXT:    flw fs3, 12(sp)
+; RV32IF-NEXT:    addi sp, sp, 16
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: explicit_register_f19:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    fsw fs3, 12(sp)
+; RV64IF-NEXT:    fmv.s fs3, fa0
+; RV64IF-NEXT:    #APP
+; RV64IF-NEXT:    fcvt.w.s a0, fs3
+; RV64IF-NEXT:    #NO_APP
+; RV64IF-NEXT:    flw fs3, 12(sp)
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{f19}"(float %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `fs3` (`f19`) as an input, so it should be saved.
+define i32 @explicit_register_fs3(float %a) nounwind {
+; RV32IF-LABEL: explicit_register_fs3:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    addi sp, sp, -16
+; RV32IF-NEXT:    fsw fs3, 12(sp)
+; RV32IF-NEXT:    fmv.s fs3, fa0
+; RV32IF-NEXT:    #APP
+; RV32IF-NEXT:    fcvt.w.s a0, fs3
+; RV32IF-NEXT:    #NO_APP
+; RV32IF-NEXT:    flw fs3, 12(sp)
+; RV32IF-NEXT:    addi sp, sp, 16
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: explicit_register_fs3:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    fsw fs3, 12(sp)
+; RV64IF-NEXT:    fmv.s fs3, fa0
+; RV64IF-NEXT:    #APP
+; RV64IF-NEXT:    fcvt.w.s a0, fs3
+; RV64IF-NEXT:    #NO_APP
+; RV64IF-NEXT:    flw fs3, 12(sp)
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{fs3}"(float %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `f20` (`fs4`) as an input, so it should be saved.
+define i32 @explicit_register_f20(float %a) nounwind {
+; RV32IF-LABEL: explicit_register_f20:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    addi sp, sp, -16
+; RV32IF-NEXT:    fsw fs4, 12(sp)
+; RV32IF-NEXT:    fmv.s fs4, fa0
+; RV32IF-NEXT:    #APP
+; RV32IF-NEXT:    fcvt.w.s a0, fs4
+; RV32IF-NEXT:    #NO_APP
+; RV32IF-NEXT:    flw fs4, 12(sp)
+; RV32IF-NEXT:    addi sp, sp, 16
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: explicit_register_f20:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    fsw fs4, 12(sp)
+; RV64IF-NEXT:    fmv.s fs4, fa0
+; RV64IF-NEXT:    #APP
+; RV64IF-NEXT:    fcvt.w.s a0, fs4
+; RV64IF-NEXT:    #NO_APP
+; RV64IF-NEXT:    flw fs4, 12(sp)
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{f20}"(float %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `fs4` (`f20`) as an input, so it should be saved.
+define i32 @explicit_register_fs4(float %a) nounwind {
+; RV32IF-LABEL: explicit_register_fs4:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    addi sp, sp, -16
+; RV32IF-NEXT:    fsw fs4, 12(sp)
+; RV32IF-NEXT:    fmv.s fs4, fa0
+; RV32IF-NEXT:    #APP
+; RV32IF-NEXT:    fcvt.w.s a0, fs4
+; RV32IF-NEXT:    #NO_APP
+; RV32IF-NEXT:    flw fs4, 12(sp)
+; RV32IF-NEXT:    addi sp, sp, 16
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: explicit_register_fs4:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    fsw fs4, 12(sp)
+; RV64IF-NEXT:    fmv.s fs4, fa0
+; RV64IF-NEXT:    #APP
+; RV64IF-NEXT:    fcvt.w.s a0, fs4
+; RV64IF-NEXT:    #NO_APP
+; RV64IF-NEXT:    flw fs4, 12(sp)
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{fs4}"(float %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `f21` (`fs5`) as an input, so it should be saved.
+define i32 @explicit_register_f21(float %a) nounwind {
+; RV32IF-LABEL: explicit_register_f21:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    addi sp, sp, -16
+; RV32IF-NEXT:    fsw fs5, 12(sp)
+; RV32IF-NEXT:    fmv.s fs5, fa0
+; RV32IF-NEXT:    #APP
+; RV32IF-NEXT:    fcvt.w.s a0, fs5
+; RV32IF-NEXT:    #NO_APP
+; RV32IF-NEXT:    flw fs5, 12(sp)
+; RV32IF-NEXT:    addi sp, sp, 16
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: explicit_register_f21:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    fsw fs5, 12(sp)
+; RV64IF-NEXT:    fmv.s fs5, fa0
+; RV64IF-NEXT:    #APP
+; RV64IF-NEXT:    fcvt.w.s a0, fs5
+; RV64IF-NEXT:    #NO_APP
+; RV64IF-NEXT:    flw fs5, 12(sp)
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{f21}"(float %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `fs5` (`f21`) as an input, so it should be saved.
+define i32 @explicit_register_fs5(float %a) nounwind {
+; RV32IF-LABEL: explicit_register_fs5:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    addi sp, sp, -16
+; RV32IF-NEXT:    fsw fs5, 12(sp)
+; RV32IF-NEXT:    fmv.s fs5, fa0
+; RV32IF-NEXT:    #APP
+; RV32IF-NEXT:    fcvt.w.s a0, fs5
+; RV32IF-NEXT:    #NO_APP
+; RV32IF-NEXT:    flw fs5, 12(sp)
+; RV32IF-NEXT:    addi sp, sp, 16
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: explicit_register_fs5:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    fsw fs5, 12(sp)
+; RV64IF-NEXT:    fmv.s fs5, fa0
+; RV64IF-NEXT:    #APP
+; RV64IF-NEXT:    fcvt.w.s a0, fs5
+; RV64IF-NEXT:    #NO_APP
+; RV64IF-NEXT:    flw fs5, 12(sp)
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{fs5}"(float %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `f22` (`fs6`) as an input, so it should be saved.
+define i32 @explicit_register_f22(float %a) nounwind {
+; RV32IF-LABEL: explicit_register_f22:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    addi sp, sp, -16
+; RV32IF-NEXT:    fsw fs6, 12(sp)
+; RV32IF-NEXT:    fmv.s fs6, fa0
+; RV32IF-NEXT:    #APP
+; RV32IF-NEXT:    fcvt.w.s a0, fs6
+; RV32IF-NEXT:    #NO_APP
+; RV32IF-NEXT:    flw fs6, 12(sp)
+; RV32IF-NEXT:    addi sp, sp, 16
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: explicit_register_f22:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    fsw fs6, 12(sp)
+; RV64IF-NEXT:    fmv.s fs6, fa0
+; RV64IF-NEXT:    #APP
+; RV64IF-NEXT:    fcvt.w.s a0, fs6
+; RV64IF-NEXT:    #NO_APP
+; RV64IF-NEXT:    flw fs6, 12(sp)
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{f22}"(float %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `fs6` (`f22`) as an input, so it should be saved.
+define i32 @explicit_register_fs6(float %a) nounwind {
+; RV32IF-LABEL: explicit_register_fs6:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    addi sp, sp, -16
+; RV32IF-NEXT:    fsw fs6, 12(sp)
+; RV32IF-NEXT:    fmv.s fs6, fa0
+; RV32IF-NEXT:    #APP
+; RV32IF-NEXT:    fcvt.w.s a0, fs6
+; RV32IF-NEXT:    #NO_APP
+; RV32IF-NEXT:    flw fs6, 12(sp)
+; RV32IF-NEXT:    addi sp, sp, 16
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: explicit_register_fs6:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    fsw fs6, 12(sp)
+; RV64IF-NEXT:    fmv.s fs6, fa0
+; RV64IF-NEXT:    #APP
+; RV64IF-NEXT:    fcvt.w.s a0, fs6
+; RV64IF-NEXT:    #NO_APP
+; RV64IF-NEXT:    flw fs6, 12(sp)
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{fs6}"(float %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `f23` (`fs7`) as an input, so it should be saved.
+define i32 @explicit_register_f23(float %a) nounwind {
+; RV32IF-LABEL: explicit_register_f23:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    addi sp, sp, -16
+; RV32IF-NEXT:    fsw fs7, 12(sp)
+; RV32IF-NEXT:    fmv.s fs7, fa0
+; RV32IF-NEXT:    #APP
+; RV32IF-NEXT:    fcvt.w.s a0, fs7
+; RV32IF-NEXT:    #NO_APP
+; RV32IF-NEXT:    flw fs7, 12(sp)
+; RV32IF-NEXT:    addi sp, sp, 16
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: explicit_register_f23:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    fsw fs7, 12(sp)
+; RV64IF-NEXT:    fmv.s fs7, fa0
+; RV64IF-NEXT:    #APP
+; RV64IF-NEXT:    fcvt.w.s a0, fs7
+; RV64IF-NEXT:    #NO_APP
+; RV64IF-NEXT:    flw fs7, 12(sp)
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{f23}"(float %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `fs7` (`f23`) as an input, so it should be saved.
+define i32 @explicit_register_fs7(float %a) nounwind {
+; RV32IF-LABEL: explicit_register_fs7:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    addi sp, sp, -16
+; RV32IF-NEXT:    fsw fs7, 12(sp)
+; RV32IF-NEXT:    fmv.s fs7, fa0
+; RV32IF-NEXT:    #APP
+; RV32IF-NEXT:    fcvt.w.s a0, fs7
+; RV32IF-NEXT:    #NO_APP
+; RV32IF-NEXT:    flw fs7, 12(sp)
+; RV32IF-NEXT:    addi sp, sp, 16
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: explicit_register_fs7:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    fsw fs7, 12(sp)
+; RV64IF-NEXT:    fmv.s fs7, fa0
+; RV64IF-NEXT:    #APP
+; RV64IF-NEXT:    fcvt.w.s a0, fs7
+; RV64IF-NEXT:    #NO_APP
+; RV64IF-NEXT:    flw fs7, 12(sp)
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{fs7}"(float %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `f24` (`fs8`) as an input, so it should be saved.
+define i32 @explicit_register_f24(float %a) nounwind {
+; RV32IF-LABEL: explicit_register_f24:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    addi sp, sp, -16
+; RV32IF-NEXT:    fsw fs8, 12(sp)
+; RV32IF-NEXT:    fmv.s fs8, fa0
+; RV32IF-NEXT:    #APP
+; RV32IF-NEXT:    fcvt.w.s a0, fs8
+; RV32IF-NEXT:    #NO_APP
+; RV32IF-NEXT:    flw fs8, 12(sp)
+; RV32IF-NEXT:    addi sp, sp, 16
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: explicit_register_f24:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    fsw fs8, 12(sp)
+; RV64IF-NEXT:    fmv.s fs8, fa0
+; RV64IF-NEXT:    #APP
+; RV64IF-NEXT:    fcvt.w.s a0, fs8
+; RV64IF-NEXT:    #NO_APP
+; RV64IF-NEXT:    flw fs8, 12(sp)
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{f24}"(float %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `fs8` (`f24`) as an input, so it should be saved.
+define i32 @explicit_register_fs8(float %a) nounwind {
+; RV32IF-LABEL: explicit_register_fs8:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    addi sp, sp, -16
+; RV32IF-NEXT:    fsw fs8, 12(sp)
+; RV32IF-NEXT:    fmv.s fs8, fa0
+; RV32IF-NEXT:    #APP
+; RV32IF-NEXT:    fcvt.w.s a0, fs8
+; RV32IF-NEXT:    #NO_APP
+; RV32IF-NEXT:    flw fs8, 12(sp)
+; RV32IF-NEXT:    addi sp, sp, 16
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: explicit_register_fs8:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    fsw fs8, 12(sp)
+; RV64IF-NEXT:    fmv.s fs8, fa0
+; RV64IF-NEXT:    #APP
+; RV64IF-NEXT:    fcvt.w.s a0, fs8
+; RV64IF-NEXT:    #NO_APP
+; RV64IF-NEXT:    flw fs8, 12(sp)
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{fs8}"(float %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `f25` (`fs9`) as an input, so it should be saved.
+define i32 @explicit_register_f25(float %a) nounwind {
+; RV32IF-LABEL: explicit_register_f25:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    addi sp, sp, -16
+; RV32IF-NEXT:    fsw fs9, 12(sp)
+; RV32IF-NEXT:    fmv.s fs9, fa0
+; RV32IF-NEXT:    #APP
+; RV32IF-NEXT:    fcvt.w.s a0, fs9
+; RV32IF-NEXT:    #NO_APP
+; RV32IF-NEXT:    flw fs9, 12(sp)
+; RV32IF-NEXT:    addi sp, sp, 16
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: explicit_register_f25:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    fsw fs9, 12(sp)
+; RV64IF-NEXT:    fmv.s fs9, fa0
+; RV64IF-NEXT:    #APP
+; RV64IF-NEXT:    fcvt.w.s a0, fs9
+; RV64IF-NEXT:    #NO_APP
+; RV64IF-NEXT:    flw fs9, 12(sp)
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{f25}"(float %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `fs9` (`f25`) as an input, so it should be saved.
+define i32 @explicit_register_fs9(float %a) nounwind {
+; RV32IF-LABEL: explicit_register_fs9:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    addi sp, sp, -16
+; RV32IF-NEXT:    fsw fs9, 12(sp)
+; RV32IF-NEXT:    fmv.s fs9, fa0
+; RV32IF-NEXT:    #APP
+; RV32IF-NEXT:    fcvt.w.s a0, fs9
+; RV32IF-NEXT:    #NO_APP
+; RV32IF-NEXT:    flw fs9, 12(sp)
+; RV32IF-NEXT:    addi sp, sp, 16
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: explicit_register_fs9:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    fsw fs9, 12(sp)
+; RV64IF-NEXT:    fmv.s fs9, fa0
+; RV64IF-NEXT:    #APP
+; RV64IF-NEXT:    fcvt.w.s a0, fs9
+; RV64IF-NEXT:    #NO_APP
+; RV64IF-NEXT:    flw fs9, 12(sp)
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{fs9}"(float %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `f26` (`fs10`) as an input, so it should be saved.
+define i32 @explicit_register_f26(float %a) nounwind {
+; RV32IF-LABEL: explicit_register_f26:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    addi sp, sp, -16
+; RV32IF-NEXT:    fsw fs10, 12(sp)
+; RV32IF-NEXT:    fmv.s fs10, fa0
+; RV32IF-NEXT:    #APP
+; RV32IF-NEXT:    fcvt.w.s a0, fs10
+; RV32IF-NEXT:    #NO_APP
+; RV32IF-NEXT:    flw fs10, 12(sp)
+; RV32IF-NEXT:    addi sp, sp, 16
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: explicit_register_f26:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    fsw fs10, 12(sp)
+; RV64IF-NEXT:    fmv.s fs10, fa0
+; RV64IF-NEXT:    #APP
+; RV64IF-NEXT:    fcvt.w.s a0, fs10
+; RV64IF-NEXT:    #NO_APP
+; RV64IF-NEXT:    flw fs10, 12(sp)
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{f26}"(float %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `fs10` (`f26`) as an input, so it should be saved.
+define i32 @explicit_register_fs10(float %a) nounwind {
+; RV32IF-LABEL: explicit_register_fs10:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    addi sp, sp, -16
+; RV32IF-NEXT:    fsw fs10, 12(sp)
+; RV32IF-NEXT:    fmv.s fs10, fa0
+; RV32IF-NEXT:    #APP
+; RV32IF-NEXT:    fcvt.w.s a0, fs10
+; RV32IF-NEXT:    #NO_APP
+; RV32IF-NEXT:    flw fs10, 12(sp)
+; RV32IF-NEXT:    addi sp, sp, 16
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: explicit_register_fs10:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    fsw fs10, 12(sp)
+; RV64IF-NEXT:    fmv.s fs10, fa0
+; RV64IF-NEXT:    #APP
+; RV64IF-NEXT:    fcvt.w.s a0, fs10
+; RV64IF-NEXT:    #NO_APP
+; RV64IF-NEXT:    flw fs10, 12(sp)
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{fs10}"(float %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `f27` (`fs11`) as an input, so it should be saved.
+define i32 @explicit_register_f27(float %a) nounwind {
+; RV32IF-LABEL: explicit_register_f27:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    addi sp, sp, -16
+; RV32IF-NEXT:    fsw fs11, 12(sp)
+; RV32IF-NEXT:    fmv.s fs11, fa0
+; RV32IF-NEXT:    #APP
+; RV32IF-NEXT:    fcvt.w.s a0, fs11
+; RV32IF-NEXT:    #NO_APP
+; RV32IF-NEXT:    flw fs11, 12(sp)
+; RV32IF-NEXT:    addi sp, sp, 16
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: explicit_register_f27:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    fsw fs11, 12(sp)
+; RV64IF-NEXT:    fmv.s fs11, fa0
+; RV64IF-NEXT:    #APP
+; RV64IF-NEXT:    fcvt.w.s a0, fs11
+; RV64IF-NEXT:    #NO_APP
+; RV64IF-NEXT:    flw fs11, 12(sp)
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{f27}"(float %a)
+  ret i32 %1
+}
+
+; NOTE: This test uses `fs11` (`f27`) as an input, so it should be saved.
+define i32 @explicit_register_fs11(float %a) nounwind {
+; RV32IF-LABEL: explicit_register_fs11:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    addi sp, sp, -16
+; RV32IF-NEXT:    fsw fs11, 12(sp)
+; RV32IF-NEXT:    fmv.s fs11, fa0
+; RV32IF-NEXT:    #APP
+; RV32IF-NEXT:    fcvt.w.s a0, fs11
+; RV32IF-NEXT:    #NO_APP
+; RV32IF-NEXT:    flw fs11, 12(sp)
+; RV32IF-NEXT:    addi sp, sp, 16
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: explicit_register_fs11:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    fsw fs11, 12(sp)
+; RV64IF-NEXT:    fmv.s fs11, fa0
+; RV64IF-NEXT:    #APP
+; RV64IF-NEXT:    fcvt.w.s a0, fs11
+; RV64IF-NEXT:    #NO_APP
+; RV64IF-NEXT:    flw fs11, 12(sp)
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{fs11}"(float %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_f28(float %a) nounwind {
+; RV32IF-LABEL: explicit_register_f28:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    fmv.s ft8, fa0
+; RV32IF-NEXT:    #APP
+; RV32IF-NEXT:    fcvt.w.s a0, ft8
+; RV32IF-NEXT:    #NO_APP
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: explicit_register_f28:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.s ft8, fa0
+; RV64IF-NEXT:    #APP
+; RV64IF-NEXT:    fcvt.w.s a0, ft8
+; RV64IF-NEXT:    #NO_APP
+; RV64IF-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{f28}"(float %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_ft8(float %a) nounwind {
+; RV32IF-LABEL: explicit_register_ft8:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    fmv.s ft8, fa0
+; RV32IF-NEXT:    #APP
+; RV32IF-NEXT:    fcvt.w.s a0, ft8
+; RV32IF-NEXT:    #NO_APP
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: explicit_register_ft8:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.s ft8, fa0
+; RV64IF-NEXT:    #APP
+; RV64IF-NEXT:    fcvt.w.s a0, ft8
+; RV64IF-NEXT:    #NO_APP
+; RV64IF-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{ft8}"(float %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_f29(float %a) nounwind {
+; RV32IF-LABEL: explicit_register_f29:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    fmv.s ft9, fa0
+; RV32IF-NEXT:    #APP
+; RV32IF-NEXT:    fcvt.w.s a0, ft9
+; RV32IF-NEXT:    #NO_APP
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: explicit_register_f29:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.s ft9, fa0
+; RV64IF-NEXT:    #APP
+; RV64IF-NEXT:    fcvt.w.s a0, ft9
+; RV64IF-NEXT:    #NO_APP
+; RV64IF-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{f29}"(float %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_ft9(float %a) nounwind {
+; RV32IF-LABEL: explicit_register_ft9:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    fmv.s ft9, fa0
+; RV32IF-NEXT:    #APP
+; RV32IF-NEXT:    fcvt.w.s a0, ft9
+; RV32IF-NEXT:    #NO_APP
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: explicit_register_ft9:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.s ft9, fa0
+; RV64IF-NEXT:    #APP
+; RV64IF-NEXT:    fcvt.w.s a0, ft9
+; RV64IF-NEXT:    #NO_APP
+; RV64IF-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{ft9}"(float %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_f30(float %a) nounwind {
+; RV32IF-LABEL: explicit_register_f30:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    fmv.s ft10, fa0
+; RV32IF-NEXT:    #APP
+; RV32IF-NEXT:    fcvt.w.s a0, ft10
+; RV32IF-NEXT:    #NO_APP
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: explicit_register_f30:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.s ft10, fa0
+; RV64IF-NEXT:    #APP
+; RV64IF-NEXT:    fcvt.w.s a0, ft10
+; RV64IF-NEXT:    #NO_APP
+; RV64IF-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{f30}"(float %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_ft10(float %a) nounwind {
+; RV32IF-LABEL: explicit_register_ft10:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    fmv.s ft10, fa0
+; RV32IF-NEXT:    #APP
+; RV32IF-NEXT:    fcvt.w.s a0, ft10
+; RV32IF-NEXT:    #NO_APP
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: explicit_register_ft10:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.s ft10, fa0
+; RV64IF-NEXT:    #APP
+; RV64IF-NEXT:    fcvt.w.s a0, ft10
+; RV64IF-NEXT:    #NO_APP
+; RV64IF-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{ft10}"(float %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_f31(float %a) nounwind {
+; RV32IF-LABEL: explicit_register_f31:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    fmv.s ft11, fa0
+; RV32IF-NEXT:    #APP
+; RV32IF-NEXT:    fcvt.w.s a0, ft11
+; RV32IF-NEXT:    #NO_APP
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: explicit_register_f31:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.s ft11, fa0
+; RV64IF-NEXT:    #APP
+; RV64IF-NEXT:    fcvt.w.s a0, ft11
+; RV64IF-NEXT:    #NO_APP
+; RV64IF-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{f31}"(float %a)
+  ret i32 %1
+}
+
+define i32 @explicit_register_ft11(float %a) nounwind {
+; RV32IF-LABEL: explicit_register_ft11:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    fmv.s ft11, fa0
+; RV32IF-NEXT:    #APP
+; RV32IF-NEXT:    fcvt.w.s a0, ft11
+; RV32IF-NEXT:    #NO_APP
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: explicit_register_ft11:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.s ft11, fa0
+; RV64IF-NEXT:    #APP
+; RV64IF-NEXT:    fcvt.w.s a0, ft11
+; RV64IF-NEXT:    #NO_APP
+; RV64IF-NEXT:    ret
+  %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{ft11}"(float %a)
+  ret i32 %1
+}
diff --git a/llvm/test/CodeGen/RISCV/inline-asm-f-constraint-f.ll b/llvm/test/CodeGen/RISCV/inline-asm-f-constraint-f.ll
index e342789ca..25bce6295 100644
--- a/llvm/test/CodeGen/RISCV/inline-asm-f-constraint-f.ll
+++ b/llvm/test/CodeGen/RISCV/inline-asm-f-constraint-f.ll
@@ -9,22 +9,22 @@
 define float @constraint_f_float(float %a) nounwind {
 ; RV32F-LABEL: constraint_f_float:
 ; RV32F:       # %bb.0:
-; RV32F-NEXT:    fmv.w.x ft0, a0
-; RV32F-NEXT:    lui a0, %hi(gf)
-; RV32F-NEXT:    flw ft1, %lo(gf)(a0)
+; RV32F-NEXT:    lui a1, %hi(gf)
+; RV32F-NEXT:    flw ft0, %lo(gf)(a1)
+; RV32F-NEXT:    fmv.w.x ft1, a0
 ; RV32F-NEXT:    #APP
-; RV32F-NEXT:    fadd.s ft0, ft0, ft1
+; RV32F-NEXT:    fadd.s ft0, ft1, ft0
 ; RV32F-NEXT:    #NO_APP
 ; RV32F-NEXT:    fmv.x.w a0, ft0
 ; RV32F-NEXT:    ret
 ;
 ; RV64F-LABEL: constraint_f_float:
 ; RV64F:       # %bb.0:
-; RV64F-NEXT:    fmv.w.x ft0, a0
-; RV64F-NEXT:    lui a0, %hi(gf)
-; RV64F-NEXT:    flw ft1, %lo(gf)(a0)
+; RV64F-NEXT:    lui a1, %hi(gf)
+; RV64F-NEXT:    flw ft0, %lo(gf)(a1)
+; RV64F-NEXT:    fmv.w.x ft1, a0
 ; RV64F-NEXT:    #APP
-; RV64F-NEXT:    fadd.s ft0, ft0, ft1
+; RV64F-NEXT:    fadd.s ft0, ft1, ft0
 ; RV64F-NEXT:    #NO_APP
 ; RV64F-NEXT:    fmv.x.w a0, ft0
 ; RV64F-NEXT:    ret
@@ -32,3 +32,30 @@ define float @constraint_f_float(float %a) nounwind {
   %2 = tail call float asm "fadd.s $0, $1, $2", "=f,f,f"(float %a, float %1)
   ret float %2
 }
+
+define float @constraint_f_float_abi_name(float %a) nounwind {
+; RV32F-LABEL: constraint_f_float_abi_name:
+; RV32F:       # %bb.0:
+; RV32F-NEXT:    lui a1, %hi(gf)
+; RV32F-NEXT:    flw fs0, %lo(gf)(a1)
+; RV32F-NEXT:    fmv.w.x fa0, a0
+; RV32F-NEXT:    #APP
+; RV32F-NEXT:    fadd.s ft0, fa0, fs0
+; RV32F-NEXT:    #NO_APP
+; RV32F-NEXT:    fmv.x.w a0, ft0
+; RV32F-NEXT:    ret
+;
+; RV64F-LABEL: constraint_f_float_abi_name:
+; RV64F:       # %bb.0:
+; RV64F-NEXT:    lui a1, %hi(gf)
+; RV64F-NEXT:    flw fs0, %lo(gf)(a1)
+; RV64F-NEXT:    fmv.w.x fa0, a0
+; RV64F-NEXT:    #APP
+; RV64F-NEXT:    fadd.s ft0, fa0, fs0
+; RV64F-NEXT:    #NO_APP
+; RV64F-NEXT:    fmv.x.w a0, ft0
+; RV64F-NEXT:    ret
+  %1 = load float, float* @gf
+  %2 = tail call float asm "fadd.s $0, $1, $2", "={ft0},{fa0},{fs0}"(float %a, float %1)
+  ret float %2
+}
diff --git a/llvm/test/CodeGen/RISCV/inline-asm.ll b/llvm/test/CodeGen/RISCV/inline-asm.ll
index 394033602..a16f1272f 100644
--- a/llvm/test/CodeGen/RISCV/inline-asm.ll
+++ b/llvm/test/CodeGen/RISCV/inline-asm.ll
@@ -78,7 +78,7 @@ define i32 @constraint_m2(i32* %a) nounwind {
 ; RV64I-NEXT:    lw a0, 0(a0)
 ; RV64I-NEXT:    #NO_APP
 ; RV64I-NEXT:    ret
-  %1 = tail call i32 asm "lw $0, $1", "=r,*m"(i32* %a) nounwind
+  %1 = tail call i32 asm "lw $0, $1", "=r,*m"(i32* %a)
   ret i32 %1
 }
 
@@ -182,6 +182,13 @@ define i32 @modifier_z_zero(i32 %a) nounwind {
 ; RV32I-NEXT:    add a0, a0, zero
 ; RV32I-NEXT:    #NO_APP
 ; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: modifier_z_zero:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    add a0, a0, zero
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ret
   %1 = tail call i32 asm "add $0, $1, ${2:z}", "=r,r,r"(i32 %a, i32 0)
   ret i32 %1
 }
@@ -194,6 +201,14 @@ define i32 @modifier_z_nonzero(i32 %a) nounwind {
 ; RV32I-NEXT:    add a0, a0, a1
 ; RV32I-NEXT:    #NO_APP
 ; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: modifier_z_nonzero:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi a1, zero, 1
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    add a0, a0, a1
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ret
   %1 = tail call i32 asm "add $0, $1, ${2:z}", "=r,r,r"(i32 %a, i32 1)
   ret i32 %1
 }
@@ -205,6 +220,13 @@ define i32 @modifier_i_imm(i32 %a) nounwind {
 ; RV32I-NEXT:    addi a0, a0, 1
 ; RV32I-NEXT:    #NO_APP
 ; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: modifier_i_imm:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    addi a0, a0, 1
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ret
   %1 = tail call i32 asm "add${2:i} $0, $1, $2", "=r,r,ri"(i32 %a, i32 1)
   ret i32 %1
 }
@@ -216,8 +238,57 @@ define i32 @modifier_i_reg(i32 %a, i32 %b) nounwind {
 ; RV32I-NEXT:    add a0, a0, a1
 ; RV32I-NEXT:    #NO_APP
 ; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: modifier_i_reg:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    add a0, a0, a1
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ret
   %1 = tail call i32 asm "add${2:i} $0, $1, $2", "=r,r,ri"(i32 %a, i32 %b)
   ret i32 %1
 }
 
-; TODO: expend tests for more complex constraints, out of range immediates etc
+define void @operand_global() nounwind {
+; RV32I-LABEL: operand_global:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    .8byte gi
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: operand_global:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    .8byte gi
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ret
+  tail call void asm sideeffect ".8byte $0", "i"(i32* @gi)
+  ret void
+}
+
+define void @operand_block_address() nounwind {
+; RV32I-LABEL: operand_block_address:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    j .Ltmp0
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:  .Ltmp0: # Block address taken
+; RV32I-NEXT:  # %bb.1: # %bb
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: operand_block_address:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    j .Ltmp0
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:  .Ltmp0: # Block address taken
+; RV64I-NEXT:  # %bb.1: # %bb
+; RV64I-NEXT:    ret
+  call void asm sideeffect "j $0", "i"(i8* blockaddress(@operand_block_address, %bb))
+  br label %bb
+bb:
+  ret void
+}
+
+; TODO: expand tests for more complex constraints, out of range immediates etc
diff --git a/llvm/test/CodeGen/RISCV/interrupt-attr-callee.ll b/llvm/test/CodeGen/RISCV/interrupt-attr-callee.ll
new file mode 100644
index 000000000..457b8667e
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/interrupt-attr-callee.ll
@@ -0,0 +1,70 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple riscv32-unknown-elf -o - %s \
+; RUN: 2>&1 | FileCheck %s -check-prefix CHECK-RV32
+; RUN: llc -mtriple riscv32-unknown-elf -mattr=+f -o - %s \
+; RUN: 2>&1 | FileCheck %s -check-prefix CHECK-RV32-F
+; RUN: llc -mtriple riscv32-unknown-elf -mattr=+f,+d -o - %s \
+; RUN: 2>&1 | FileCheck %s -check-prefix CHECK-RV32-FD
+;
+; The test case check that the function call in an interrupt handler will use
+; the correct CallPreservedMask as normal function. So only callee saved
+; registers could live through the function call.
+
+define dso_local void @handler() nounwind {
+; CHECK-RV32-LABEL: handler:
+; CHECK-RV32:       # %bb.0: # %entry
+; CHECK-RV32-NEXT:    addi sp, sp, -16
+; CHECK-RV32-NEXT:    sw ra, 12(sp)
+; CHECK-RV32-NEXT:    sw s0, 8(sp)
+; CHECK-RV32-NEXT:    lui a0, 2
+; CHECK-RV32-NEXT:    addi a0, a0, 4
+; CHECK-RV32-NEXT:    call read
+; CHECK-RV32-NEXT:    mv s0, a0
+; CHECK-RV32-NEXT:    call callee
+; CHECK-RV32-NEXT:    mv a0, s0
+; CHECK-RV32-NEXT:    lw s0, 8(sp)
+; CHECK-RV32-NEXT:    lw ra, 12(sp)
+; CHECK-RV32-NEXT:    addi sp, sp, 16
+; CHECK-RV32-NEXT:    tail write
+;
+; CHECK-RV32-F-LABEL: handler:
+; CHECK-RV32-F:       # %bb.0: # %entry
+; CHECK-RV32-F-NEXT:    addi sp, sp, -16
+; CHECK-RV32-F-NEXT:    sw ra, 12(sp)
+; CHECK-RV32-F-NEXT:    sw s0, 8(sp)
+; CHECK-RV32-F-NEXT:    lui a0, 2
+; CHECK-RV32-F-NEXT:    addi a0, a0, 4
+; CHECK-RV32-F-NEXT:    call read
+; CHECK-RV32-F-NEXT:    mv s0, a0
+; CHECK-RV32-F-NEXT:    call callee
+; CHECK-RV32-F-NEXT:    mv a0, s0
+; CHECK-RV32-F-NEXT:    lw s0, 8(sp)
+; CHECK-RV32-F-NEXT:    lw ra, 12(sp)
+; CHECK-RV32-F-NEXT:    addi sp, sp, 16
+; CHECK-RV32-F-NEXT:    tail write
+;
+; CHECK-RV32-FD-LABEL: handler:
+; CHECK-RV32-FD:       # %bb.0: # %entry
+; CHECK-RV32-FD-NEXT:    addi sp, sp, -16
+; CHECK-RV32-FD-NEXT:    sw ra, 12(sp)
+; CHECK-RV32-FD-NEXT:    sw s0, 8(sp)
+; CHECK-RV32-FD-NEXT:    lui a0, 2
+; CHECK-RV32-FD-NEXT:    addi a0, a0, 4
+; CHECK-RV32-FD-NEXT:    call read
+; CHECK-RV32-FD-NEXT:    mv s0, a0
+; CHECK-RV32-FD-NEXT:    call callee
+; CHECK-RV32-FD-NEXT:    mv a0, s0
+; CHECK-RV32-FD-NEXT:    lw s0, 8(sp)
+; CHECK-RV32-FD-NEXT:    lw ra, 12(sp)
+; CHECK-RV32-FD-NEXT:    addi sp, sp, 16
+; CHECK-RV32-FD-NEXT:    tail write
+entry:
+  %call = tail call i32 @read(i32 8196)
+  tail call void bitcast (void (...)* @callee to void ()*)()
+  tail call void @write(i32 %call)
+  ret void
+}
+
+declare i32 @read(i32)
+declare void @callee(...)
+declare void @write(i32)
diff --git a/llvm/test/CodeGen/RISCV/interrupt-attr-nocall.ll b/llvm/test/CodeGen/RISCV/interrupt-attr-nocall.ll
index 3e14b7511..c36a01ca3 100644
--- a/llvm/test/CodeGen/RISCV/interrupt-attr-nocall.ll
+++ b/llvm/test/CodeGen/RISCV/interrupt-attr-nocall.ll
@@ -2,9 +2,9 @@
 ; RUN: llc -mtriple riscv32-unknown-elf -o - %s \
 ; RUN: 2>&1 | FileCheck %s -check-prefix CHECK-RV32
 ; RUN: llc -mtriple riscv32-unknown-elf -mattr=+f -o - %s \
-; RUN: 2>&1 | FileCheck %s -check-prefix CHECK-RV32-F
+; RUN: 2>&1 | FileCheck %s -check-prefix CHECK-RV32IF
 ; RUN: llc -mtriple riscv32-unknown-elf -mattr=+f,+d -o - %s \
-; RUN: 2>&1 | FileCheck %s -check-prefix CHECK-RV32-FD
+; RUN: 2>&1 | FileCheck %s -check-prefix CHECK-RV32IFD
 ;
 ; TODO: Add RV64 tests when we can lower global addresses.
 
@@ -40,6 +40,39 @@ define void @foo_i32() nounwind #0 {
 ; CHECK-RV32-NEXT:    addi sp, sp, 16
 ; CHECK-RV32-NEXT:    mret
 ;
+; CHECK-RV32IF-LABEL: foo_i32:
+; CHECK-RV32IF:       # %bb.0:
+; CHECK-RV32IF-NEXT:    addi sp, sp, -16
+; CHECK-RV32IF-NEXT:    sw a0, 12(sp)
+; CHECK-RV32IF-NEXT:    sw a1, 8(sp)
+; CHECK-RV32IF-NEXT:    lui a0, %hi(a)
+; CHECK-RV32IF-NEXT:    lw a0, %lo(a)(a0)
+; CHECK-RV32IF-NEXT:    lui a1, %hi(b)
+; CHECK-RV32IF-NEXT:    lw a1, %lo(b)(a1)
+; CHECK-RV32IF-NEXT:    add a0, a1, a0
+; CHECK-RV32IF-NEXT:    lui a1, %hi(c)
+; CHECK-RV32IF-NEXT:    sw a0, %lo(c)(a1)
+; CHECK-RV32IF-NEXT:    lw a1, 8(sp)
+; CHECK-RV32IF-NEXT:    lw a0, 12(sp)
+; CHECK-RV32IF-NEXT:    addi sp, sp, 16
+; CHECK-RV32IF-NEXT:    mret
+;
+; CHECK-RV32IFD-LABEL: foo_i32:
+; CHECK-RV32IFD:       # %bb.0:
+; CHECK-RV32IFD-NEXT:    addi sp, sp, -16
+; CHECK-RV32IFD-NEXT:    sw a0, 12(sp)
+; CHECK-RV32IFD-NEXT:    sw a1, 8(sp)
+; CHECK-RV32IFD-NEXT:    lui a0, %hi(a)
+; CHECK-RV32IFD-NEXT:    lw a0, %lo(a)(a0)
+; CHECK-RV32IFD-NEXT:    lui a1, %hi(b)
+; CHECK-RV32IFD-NEXT:    lw a1, %lo(b)(a1)
+; CHECK-RV32IFD-NEXT:    add a0, a1, a0
+; CHECK-RV32IFD-NEXT:    lui a1, %hi(c)
+; CHECK-RV32IFD-NEXT:    sw a0, %lo(c)(a1)
+; CHECK-RV32IFD-NEXT:    lw a1, 8(sp)
+; CHECK-RV32IFD-NEXT:    lw a0, 12(sp)
+; CHECK-RV32IFD-NEXT:    addi sp, sp, 16
+; CHECK-RV32IFD-NEXT:    mret
   %1 = load i32, i32* @a
   %2 = load i32, i32* @b
   %add = add nsw i32 %2, %1
@@ -74,6 +107,49 @@ define void @foo_fp_i32() nounwind #1 {
 ; CHECK-RV32-NEXT:    addi sp, sp, 16
 ; CHECK-RV32-NEXT:    mret
 ;
+; CHECK-RV32IF-LABEL: foo_fp_i32:
+; CHECK-RV32IF:       # %bb.0:
+; CHECK-RV32IF-NEXT:    addi sp, sp, -16
+; CHECK-RV32IF-NEXT:    sw ra, 12(sp)
+; CHECK-RV32IF-NEXT:    sw s0, 8(sp)
+; CHECK-RV32IF-NEXT:    sw a0, 4(sp)
+; CHECK-RV32IF-NEXT:    sw a1, 0(sp)
+; CHECK-RV32IF-NEXT:    addi s0, sp, 16
+; CHECK-RV32IF-NEXT:    lui a0, %hi(a)
+; CHECK-RV32IF-NEXT:    lw a0, %lo(a)(a0)
+; CHECK-RV32IF-NEXT:    lui a1, %hi(b)
+; CHECK-RV32IF-NEXT:    lw a1, %lo(b)(a1)
+; CHECK-RV32IF-NEXT:    add a0, a1, a0
+; CHECK-RV32IF-NEXT:    lui a1, %hi(c)
+; CHECK-RV32IF-NEXT:    sw a0, %lo(c)(a1)
+; CHECK-RV32IF-NEXT:    lw a1, 0(sp)
+; CHECK-RV32IF-NEXT:    lw a0, 4(sp)
+; CHECK-RV32IF-NEXT:    lw s0, 8(sp)
+; CHECK-RV32IF-NEXT:    lw ra, 12(sp)
+; CHECK-RV32IF-NEXT:    addi sp, sp, 16
+; CHECK-RV32IF-NEXT:    mret
+;
+; CHECK-RV32IFD-LABEL: foo_fp_i32:
+; CHECK-RV32IFD:       # %bb.0:
+; CHECK-RV32IFD-NEXT:    addi sp, sp, -16
+; CHECK-RV32IFD-NEXT:    sw ra, 12(sp)
+; CHECK-RV32IFD-NEXT:    sw s0, 8(sp)
+; CHECK-RV32IFD-NEXT:    sw a0, 4(sp)
+; CHECK-RV32IFD-NEXT:    sw a1, 0(sp)
+; CHECK-RV32IFD-NEXT:    addi s0, sp, 16
+; CHECK-RV32IFD-NEXT:    lui a0, %hi(a)
+; CHECK-RV32IFD-NEXT:    lw a0, %lo(a)(a0)
+; CHECK-RV32IFD-NEXT:    lui a1, %hi(b)
+; CHECK-RV32IFD-NEXT:    lw a1, %lo(b)(a1)
+; CHECK-RV32IFD-NEXT:    add a0, a1, a0
+; CHECK-RV32IFD-NEXT:    lui a1, %hi(c)
+; CHECK-RV32IFD-NEXT:    sw a0, %lo(c)(a1)
+; CHECK-RV32IFD-NEXT:    lw a1, 0(sp)
+; CHECK-RV32IFD-NEXT:    lw a0, 4(sp)
+; CHECK-RV32IFD-NEXT:    lw s0, 8(sp)
+; CHECK-RV32IFD-NEXT:    lw ra, 12(sp)
+; CHECK-RV32IFD-NEXT:    addi sp, sp, 16
+; CHECK-RV32IFD-NEXT:    mret
   %1 = load i32, i32* @a
   %2 = load i32, i32* @b
   %add = add nsw i32 %2, %1
@@ -86,25 +162,88 @@ define void @foo_fp_i32() nounwind #1 {
 @d = external global float
 
 define void @foo_float() nounwind #0 {
-; CHECK-RV32-F-LABEL: foo_float:
-; CHECK-RV32-F:       # %bb.0:
-; CHECK-RV32-F-NEXT:    addi sp, sp, -16
-; CHECK-RV32-F-NEXT:    sw a0, 12(sp)
-; CHECK-RV32-F-NEXT:    fsw ft0, 8(sp)
-; CHECK-RV32-F-NEXT:    fsw ft1, 4(sp)
-; CHECK-RV32-F-NEXT:    lui a0, %hi(f)
-; CHECK-RV32-F-NEXT:    flw ft0, %lo(f)(a0)
-; CHECK-RV32-F-NEXT:    lui a0, %hi(e)
-; CHECK-RV32-F-NEXT:    flw ft1, %lo(e)(a0)
-; CHECK-RV32-F-NEXT:    fadd.s ft0, ft1, ft0
-; CHECK-RV32-F-NEXT:    lui a0, %hi(d)
-; CHECK-RV32-F-NEXT:    fsw ft0, %lo(d)(a0)
-; CHECK-RV32-F-NEXT:    flw ft1, 4(sp)
-; CHECK-RV32-F-NEXT:    flw ft0, 8(sp)
-; CHECK-RV32-F-NEXT:    lw a0, 12(sp)
-; CHECK-RV32-F-NEXT:    addi sp, sp, 16
-; CHECK-RV32-F-NEXT:    mret
+; CHECK-RV32-LABEL: foo_float:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    addi sp, sp, -64
+; CHECK-RV32-NEXT:    sw ra, 60(sp)
+; CHECK-RV32-NEXT:    sw t0, 56(sp)
+; CHECK-RV32-NEXT:    sw t1, 52(sp)
+; CHECK-RV32-NEXT:    sw t2, 48(sp)
+; CHECK-RV32-NEXT:    sw a0, 44(sp)
+; CHECK-RV32-NEXT:    sw a1, 40(sp)
+; CHECK-RV32-NEXT:    sw a2, 36(sp)
+; CHECK-RV32-NEXT:    sw a3, 32(sp)
+; CHECK-RV32-NEXT:    sw a4, 28(sp)
+; CHECK-RV32-NEXT:    sw a5, 24(sp)
+; CHECK-RV32-NEXT:    sw a6, 20(sp)
+; CHECK-RV32-NEXT:    sw a7, 16(sp)
+; CHECK-RV32-NEXT:    sw t3, 12(sp)
+; CHECK-RV32-NEXT:    sw t4, 8(sp)
+; CHECK-RV32-NEXT:    sw t5, 4(sp)
+; CHECK-RV32-NEXT:    sw t6, 0(sp)
+; CHECK-RV32-NEXT:    lui a0, %hi(e)
+; CHECK-RV32-NEXT:    lw a0, %lo(e)(a0)
+; CHECK-RV32-NEXT:    lui a1, %hi(f)
+; CHECK-RV32-NEXT:    lw a1, %lo(f)(a1)
+; CHECK-RV32-NEXT:    call __addsf3
+; CHECK-RV32-NEXT:    lui a1, %hi(d)
+; CHECK-RV32-NEXT:    sw a0, %lo(d)(a1)
+; CHECK-RV32-NEXT:    lw t6, 0(sp)
+; CHECK-RV32-NEXT:    lw t5, 4(sp)
+; CHECK-RV32-NEXT:    lw t4, 8(sp)
+; CHECK-RV32-NEXT:    lw t3, 12(sp)
+; CHECK-RV32-NEXT:    lw a7, 16(sp)
+; CHECK-RV32-NEXT:    lw a6, 20(sp)
+; CHECK-RV32-NEXT:    lw a5, 24(sp)
+; CHECK-RV32-NEXT:    lw a4, 28(sp)
+; CHECK-RV32-NEXT:    lw a3, 32(sp)
+; CHECK-RV32-NEXT:    lw a2, 36(sp)
+; CHECK-RV32-NEXT:    lw a1, 40(sp)
+; CHECK-RV32-NEXT:    lw a0, 44(sp)
+; CHECK-RV32-NEXT:    lw t2, 48(sp)
+; CHECK-RV32-NEXT:    lw t1, 52(sp)
+; CHECK-RV32-NEXT:    lw t0, 56(sp)
+; CHECK-RV32-NEXT:    lw ra, 60(sp)
+; CHECK-RV32-NEXT:    addi sp, sp, 64
+; CHECK-RV32-NEXT:    mret
 ;
+; CHECK-RV32IF-LABEL: foo_float:
+; CHECK-RV32IF:       # %bb.0:
+; CHECK-RV32IF-NEXT:    addi sp, sp, -16
+; CHECK-RV32IF-NEXT:    sw a0, 12(sp)
+; CHECK-RV32IF-NEXT:    fsw ft0, 8(sp)
+; CHECK-RV32IF-NEXT:    fsw ft1, 4(sp)
+; CHECK-RV32IF-NEXT:    lui a0, %hi(e)
+; CHECK-RV32IF-NEXT:    flw ft0, %lo(e)(a0)
+; CHECK-RV32IF-NEXT:    lui a0, %hi(f)
+; CHECK-RV32IF-NEXT:    flw ft1, %lo(f)(a0)
+; CHECK-RV32IF-NEXT:    fadd.s ft0, ft0, ft1
+; CHECK-RV32IF-NEXT:    lui a0, %hi(d)
+; CHECK-RV32IF-NEXT:    fsw ft0, %lo(d)(a0)
+; CHECK-RV32IF-NEXT:    flw ft1, 4(sp)
+; CHECK-RV32IF-NEXT:    flw ft0, 8(sp)
+; CHECK-RV32IF-NEXT:    lw a0, 12(sp)
+; CHECK-RV32IF-NEXT:    addi sp, sp, 16
+; CHECK-RV32IF-NEXT:    mret
+;
+; CHECK-RV32IFD-LABEL: foo_float:
+; CHECK-RV32IFD:       # %bb.0:
+; CHECK-RV32IFD-NEXT:    addi sp, sp, -32
+; CHECK-RV32IFD-NEXT:    sw a0, 28(sp)
+; CHECK-RV32IFD-NEXT:    fsd ft0, 16(sp)
+; CHECK-RV32IFD-NEXT:    fsd ft1, 8(sp)
+; CHECK-RV32IFD-NEXT:    lui a0, %hi(e)
+; CHECK-RV32IFD-NEXT:    flw ft0, %lo(e)(a0)
+; CHECK-RV32IFD-NEXT:    lui a0, %hi(f)
+; CHECK-RV32IFD-NEXT:    flw ft1, %lo(f)(a0)
+; CHECK-RV32IFD-NEXT:    fadd.s ft0, ft0, ft1
+; CHECK-RV32IFD-NEXT:    lui a0, %hi(d)
+; CHECK-RV32IFD-NEXT:    fsw ft0, %lo(d)(a0)
+; CHECK-RV32IFD-NEXT:    fld ft1, 8(sp)
+; CHECK-RV32IFD-NEXT:    fld ft0, 16(sp)
+; CHECK-RV32IFD-NEXT:    lw a0, 28(sp)
+; CHECK-RV32IFD-NEXT:    addi sp, sp, 32
+; CHECK-RV32IFD-NEXT:    mret
   %1 = load float, float* @e
   %2 = load float, float* @f
   %add = fadd float %1, %2
@@ -116,30 +255,101 @@ define void @foo_float() nounwind #0 {
 ; Additionally check frame pointer and return address are properly saved.
 ;
 define void @foo_fp_float() nounwind #1 {
-; CHECK-RV32-F-LABEL: foo_fp_float:
-; CHECK-RV32-F:       # %bb.0:
-; CHECK-RV32-F-NEXT:    addi sp, sp, -32
-; CHECK-RV32-F-NEXT:    sw ra, 28(sp)
-; CHECK-RV32-F-NEXT:    sw s0, 24(sp)
-; CHECK-RV32-F-NEXT:    sw a0, 20(sp)
-; CHECK-RV32-F-NEXT:    fsw ft0, 16(sp)
-; CHECK-RV32-F-NEXT:    fsw ft1, 12(sp)
-; CHECK-RV32-F-NEXT:    addi s0, sp, 32
-; CHECK-RV32-F-NEXT:    lui a0, %hi(f)
-; CHECK-RV32-F-NEXT:    flw ft0, %lo(f)(a0)
-; CHECK-RV32-F-NEXT:    lui a0, %hi(e)
-; CHECK-RV32-F-NEXT:    flw ft1, %lo(e)(a0)
-; CHECK-RV32-F-NEXT:    fadd.s ft0, ft1, ft0
-; CHECK-RV32-F-NEXT:    lui a0, %hi(d)
-; CHECK-RV32-F-NEXT:    fsw ft0, %lo(d)(a0)
-; CHECK-RV32-F-NEXT:    flw ft1, 12(sp)
-; CHECK-RV32-F-NEXT:    flw ft0, 16(sp)
-; CHECK-RV32-F-NEXT:    lw a0, 20(sp)
-; CHECK-RV32-F-NEXT:    lw s0, 24(sp)
-; CHECK-RV32-F-NEXT:    lw ra, 28(sp)
-; CHECK-RV32-F-NEXT:    addi sp, sp, 32
-; CHECK-RV32-F-NEXT:    mret
+; CHECK-RV32-LABEL: foo_fp_float:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    addi sp, sp, -80
+; CHECK-RV32-NEXT:    sw ra, 76(sp)
+; CHECK-RV32-NEXT:    sw t0, 72(sp)
+; CHECK-RV32-NEXT:    sw t1, 68(sp)
+; CHECK-RV32-NEXT:    sw t2, 64(sp)
+; CHECK-RV32-NEXT:    sw s0, 60(sp)
+; CHECK-RV32-NEXT:    sw a0, 56(sp)
+; CHECK-RV32-NEXT:    sw a1, 52(sp)
+; CHECK-RV32-NEXT:    sw a2, 48(sp)
+; CHECK-RV32-NEXT:    sw a3, 44(sp)
+; CHECK-RV32-NEXT:    sw a4, 40(sp)
+; CHECK-RV32-NEXT:    sw a5, 36(sp)
+; CHECK-RV32-NEXT:    sw a6, 32(sp)
+; CHECK-RV32-NEXT:    sw a7, 28(sp)
+; CHECK-RV32-NEXT:    sw t3, 24(sp)
+; CHECK-RV32-NEXT:    sw t4, 20(sp)
+; CHECK-RV32-NEXT:    sw t5, 16(sp)
+; CHECK-RV32-NEXT:    sw t6, 12(sp)
+; CHECK-RV32-NEXT:    addi s0, sp, 80
+; CHECK-RV32-NEXT:    lui a0, %hi(e)
+; CHECK-RV32-NEXT:    lw a0, %lo(e)(a0)
+; CHECK-RV32-NEXT:    lui a1, %hi(f)
+; CHECK-RV32-NEXT:    lw a1, %lo(f)(a1)
+; CHECK-RV32-NEXT:    call __addsf3
+; CHECK-RV32-NEXT:    lui a1, %hi(d)
+; CHECK-RV32-NEXT:    sw a0, %lo(d)(a1)
+; CHECK-RV32-NEXT:    lw t6, 12(sp)
+; CHECK-RV32-NEXT:    lw t5, 16(sp)
+; CHECK-RV32-NEXT:    lw t4, 20(sp)
+; CHECK-RV32-NEXT:    lw t3, 24(sp)
+; CHECK-RV32-NEXT:    lw a7, 28(sp)
+; CHECK-RV32-NEXT:    lw a6, 32(sp)
+; CHECK-RV32-NEXT:    lw a5, 36(sp)
+; CHECK-RV32-NEXT:    lw a4, 40(sp)
+; CHECK-RV32-NEXT:    lw a3, 44(sp)
+; CHECK-RV32-NEXT:    lw a2, 48(sp)
+; CHECK-RV32-NEXT:    lw a1, 52(sp)
+; CHECK-RV32-NEXT:    lw a0, 56(sp)
+; CHECK-RV32-NEXT:    lw s0, 60(sp)
+; CHECK-RV32-NEXT:    lw t2, 64(sp)
+; CHECK-RV32-NEXT:    lw t1, 68(sp)
+; CHECK-RV32-NEXT:    lw t0, 72(sp)
+; CHECK-RV32-NEXT:    lw ra, 76(sp)
+; CHECK-RV32-NEXT:    addi sp, sp, 80
+; CHECK-RV32-NEXT:    mret
 ;
+; CHECK-RV32IF-LABEL: foo_fp_float:
+; CHECK-RV32IF:       # %bb.0:
+; CHECK-RV32IF-NEXT:    addi sp, sp, -32
+; CHECK-RV32IF-NEXT:    sw ra, 28(sp)
+; CHECK-RV32IF-NEXT:    sw s0, 24(sp)
+; CHECK-RV32IF-NEXT:    sw a0, 20(sp)
+; CHECK-RV32IF-NEXT:    fsw ft0, 16(sp)
+; CHECK-RV32IF-NEXT:    fsw ft1, 12(sp)
+; CHECK-RV32IF-NEXT:    addi s0, sp, 32
+; CHECK-RV32IF-NEXT:    lui a0, %hi(e)
+; CHECK-RV32IF-NEXT:    flw ft0, %lo(e)(a0)
+; CHECK-RV32IF-NEXT:    lui a0, %hi(f)
+; CHECK-RV32IF-NEXT:    flw ft1, %lo(f)(a0)
+; CHECK-RV32IF-NEXT:    fadd.s ft0, ft0, ft1
+; CHECK-RV32IF-NEXT:    lui a0, %hi(d)
+; CHECK-RV32IF-NEXT:    fsw ft0, %lo(d)(a0)
+; CHECK-RV32IF-NEXT:    flw ft1, 12(sp)
+; CHECK-RV32IF-NEXT:    flw ft0, 16(sp)
+; CHECK-RV32IF-NEXT:    lw a0, 20(sp)
+; CHECK-RV32IF-NEXT:    lw s0, 24(sp)
+; CHECK-RV32IF-NEXT:    lw ra, 28(sp)
+; CHECK-RV32IF-NEXT:    addi sp, sp, 32
+; CHECK-RV32IF-NEXT:    mret
+;
+; CHECK-RV32IFD-LABEL: foo_fp_float:
+; CHECK-RV32IFD:       # %bb.0:
+; CHECK-RV32IFD-NEXT:    addi sp, sp, -32
+; CHECK-RV32IFD-NEXT:    sw ra, 28(sp)
+; CHECK-RV32IFD-NEXT:    sw s0, 24(sp)
+; CHECK-RV32IFD-NEXT:    sw a0, 20(sp)
+; CHECK-RV32IFD-NEXT:    fsd ft0, 8(sp)
+; CHECK-RV32IFD-NEXT:    fsd ft1, 0(sp)
+; CHECK-RV32IFD-NEXT:    addi s0, sp, 32
+; CHECK-RV32IFD-NEXT:    lui a0, %hi(e)
+; CHECK-RV32IFD-NEXT:    flw ft0, %lo(e)(a0)
+; CHECK-RV32IFD-NEXT:    lui a0, %hi(f)
+; CHECK-RV32IFD-NEXT:    flw ft1, %lo(f)(a0)
+; CHECK-RV32IFD-NEXT:    fadd.s ft0, ft0, ft1
+; CHECK-RV32IFD-NEXT:    lui a0, %hi(d)
+; CHECK-RV32IFD-NEXT:    fsw ft0, %lo(d)(a0)
+; CHECK-RV32IFD-NEXT:    fld ft1, 0(sp)
+; CHECK-RV32IFD-NEXT:    fld ft0, 8(sp)
+; CHECK-RV32IFD-NEXT:    lw a0, 20(sp)
+; CHECK-RV32IFD-NEXT:    lw s0, 24(sp)
+; CHECK-RV32IFD-NEXT:    lw ra, 28(sp)
+; CHECK-RV32IFD-NEXT:    addi sp, sp, 32
+; CHECK-RV32IFD-NEXT:    mret
   %1 = load float, float* @e
   %2 = load float, float* @f
   %add = fadd float %1, %2
@@ -152,25 +362,190 @@ define void @foo_fp_float() nounwind #1 {
 @g = external global double
 
 define void @foo_double() nounwind #0 {
-; CHECK-RV32-FD-LABEL: foo_double:
-; CHECK-RV32-FD:       # %bb.0:
-; CHECK-RV32-FD-NEXT:    addi sp, sp, -32
-; CHECK-RV32-FD-NEXT:    sw a0, 28(sp)
-; CHECK-RV32-FD-NEXT:    fsd ft0, 16(sp)
-; CHECK-RV32-FD-NEXT:    fsd ft1, 8(sp)
-; CHECK-RV32-FD-NEXT:    lui a0, %hi(i)
-; CHECK-RV32-FD-NEXT:    fld ft0, %lo(i)(a0)
-; CHECK-RV32-FD-NEXT:    lui a0, %hi(h)
-; CHECK-RV32-FD-NEXT:    fld ft1, %lo(h)(a0)
-; CHECK-RV32-FD-NEXT:    fadd.d ft0, ft1, ft0
-; CHECK-RV32-FD-NEXT:    lui a0, %hi(g)
-; CHECK-RV32-FD-NEXT:    fsd ft0, %lo(g)(a0)
-; CHECK-RV32-FD-NEXT:    fld ft1, 8(sp)
-; CHECK-RV32-FD-NEXT:    fld ft0, 16(sp)
-; CHECK-RV32-FD-NEXT:    lw a0, 28(sp)
-; CHECK-RV32-FD-NEXT:    addi sp, sp, 32
-; CHECK-RV32-FD-NEXT:    mret
+; CHECK-RV32-LABEL: foo_double:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    addi sp, sp, -64
+; CHECK-RV32-NEXT:    sw ra, 60(sp)
+; CHECK-RV32-NEXT:    sw t0, 56(sp)
+; CHECK-RV32-NEXT:    sw t1, 52(sp)
+; CHECK-RV32-NEXT:    sw t2, 48(sp)
+; CHECK-RV32-NEXT:    sw a0, 44(sp)
+; CHECK-RV32-NEXT:    sw a1, 40(sp)
+; CHECK-RV32-NEXT:    sw a2, 36(sp)
+; CHECK-RV32-NEXT:    sw a3, 32(sp)
+; CHECK-RV32-NEXT:    sw a4, 28(sp)
+; CHECK-RV32-NEXT:    sw a5, 24(sp)
+; CHECK-RV32-NEXT:    sw a6, 20(sp)
+; CHECK-RV32-NEXT:    sw a7, 16(sp)
+; CHECK-RV32-NEXT:    sw t3, 12(sp)
+; CHECK-RV32-NEXT:    sw t4, 8(sp)
+; CHECK-RV32-NEXT:    sw t5, 4(sp)
+; CHECK-RV32-NEXT:    sw t6, 0(sp)
+; CHECK-RV32-NEXT:    lui a1, %hi(h)
+; CHECK-RV32-NEXT:    lw a0, %lo(h)(a1)
+; CHECK-RV32-NEXT:    addi a1, a1, %lo(h)
+; CHECK-RV32-NEXT:    lw a1, 4(a1)
+; CHECK-RV32-NEXT:    lui a3, %hi(i)
+; CHECK-RV32-NEXT:    lw a2, %lo(i)(a3)
+; CHECK-RV32-NEXT:    addi a3, a3, %lo(i)
+; CHECK-RV32-NEXT:    lw a3, 4(a3)
+; CHECK-RV32-NEXT:    call __adddf3
+; CHECK-RV32-NEXT:    lui a2, %hi(g)
+; CHECK-RV32-NEXT:    addi a3, a2, %lo(g)
+; CHECK-RV32-NEXT:    sw a1, 4(a3)
+; CHECK-RV32-NEXT:    sw a0, %lo(g)(a2)
+; CHECK-RV32-NEXT:    lw t6, 0(sp)
+; CHECK-RV32-NEXT:    lw t5, 4(sp)
+; CHECK-RV32-NEXT:    lw t4, 8(sp)
+; CHECK-RV32-NEXT:    lw t3, 12(sp)
+; CHECK-RV32-NEXT:    lw a7, 16(sp)
+; CHECK-RV32-NEXT:    lw a6, 20(sp)
+; CHECK-RV32-NEXT:    lw a5, 24(sp)
+; CHECK-RV32-NEXT:    lw a4, 28(sp)
+; CHECK-RV32-NEXT:    lw a3, 32(sp)
+; CHECK-RV32-NEXT:    lw a2, 36(sp)
+; CHECK-RV32-NEXT:    lw a1, 40(sp)
+; CHECK-RV32-NEXT:    lw a0, 44(sp)
+; CHECK-RV32-NEXT:    lw t2, 48(sp)
+; CHECK-RV32-NEXT:    lw t1, 52(sp)
+; CHECK-RV32-NEXT:    lw t0, 56(sp)
+; CHECK-RV32-NEXT:    lw ra, 60(sp)
+; CHECK-RV32-NEXT:    addi sp, sp, 64
+; CHECK-RV32-NEXT:    mret
+;
+; CHECK-RV32IF-LABEL: foo_double:
+; CHECK-RV32IF:       # %bb.0:
+; CHECK-RV32IF-NEXT:    addi sp, sp, -192
+; CHECK-RV32IF-NEXT:    sw ra, 188(sp)
+; CHECK-RV32IF-NEXT:    sw t0, 184(sp)
+; CHECK-RV32IF-NEXT:    sw t1, 180(sp)
+; CHECK-RV32IF-NEXT:    sw t2, 176(sp)
+; CHECK-RV32IF-NEXT:    sw a0, 172(sp)
+; CHECK-RV32IF-NEXT:    sw a1, 168(sp)
+; CHECK-RV32IF-NEXT:    sw a2, 164(sp)
+; CHECK-RV32IF-NEXT:    sw a3, 160(sp)
+; CHECK-RV32IF-NEXT:    sw a4, 156(sp)
+; CHECK-RV32IF-NEXT:    sw a5, 152(sp)
+; CHECK-RV32IF-NEXT:    sw a6, 148(sp)
+; CHECK-RV32IF-NEXT:    sw a7, 144(sp)
+; CHECK-RV32IF-NEXT:    sw t3, 140(sp)
+; CHECK-RV32IF-NEXT:    sw t4, 136(sp)
+; CHECK-RV32IF-NEXT:    sw t5, 132(sp)
+; CHECK-RV32IF-NEXT:    sw t6, 128(sp)
+; CHECK-RV32IF-NEXT:    fsw ft0, 124(sp)
+; CHECK-RV32IF-NEXT:    fsw ft1, 120(sp)
+; CHECK-RV32IF-NEXT:    fsw ft2, 116(sp)
+; CHECK-RV32IF-NEXT:    fsw ft3, 112(sp)
+; CHECK-RV32IF-NEXT:    fsw ft4, 108(sp)
+; CHECK-RV32IF-NEXT:    fsw ft5, 104(sp)
+; CHECK-RV32IF-NEXT:    fsw ft6, 100(sp)
+; CHECK-RV32IF-NEXT:    fsw ft7, 96(sp)
+; CHECK-RV32IF-NEXT:    fsw fa0, 92(sp)
+; CHECK-RV32IF-NEXT:    fsw fa1, 88(sp)
+; CHECK-RV32IF-NEXT:    fsw fa2, 84(sp)
+; CHECK-RV32IF-NEXT:    fsw fa3, 80(sp)
+; CHECK-RV32IF-NEXT:    fsw fa4, 76(sp)
+; CHECK-RV32IF-NEXT:    fsw fa5, 72(sp)
+; CHECK-RV32IF-NEXT:    fsw fa6, 68(sp)
+; CHECK-RV32IF-NEXT:    fsw fa7, 64(sp)
+; CHECK-RV32IF-NEXT:    fsw ft8, 60(sp)
+; CHECK-RV32IF-NEXT:    fsw ft9, 56(sp)
+; CHECK-RV32IF-NEXT:    fsw ft10, 52(sp)
+; CHECK-RV32IF-NEXT:    fsw ft11, 48(sp)
+; CHECK-RV32IF-NEXT:    fsw fs0, 44(sp)
+; CHECK-RV32IF-NEXT:    fsw fs1, 40(sp)
+; CHECK-RV32IF-NEXT:    fsw fs2, 36(sp)
+; CHECK-RV32IF-NEXT:    fsw fs3, 32(sp)
+; CHECK-RV32IF-NEXT:    fsw fs4, 28(sp)
+; CHECK-RV32IF-NEXT:    fsw fs5, 24(sp)
+; CHECK-RV32IF-NEXT:    fsw fs6, 20(sp)
+; CHECK-RV32IF-NEXT:    fsw fs7, 16(sp)
+; CHECK-RV32IF-NEXT:    fsw fs8, 12(sp)
+; CHECK-RV32IF-NEXT:    fsw fs9, 8(sp)
+; CHECK-RV32IF-NEXT:    fsw fs10, 4(sp)
+; CHECK-RV32IF-NEXT:    fsw fs11, 0(sp)
+; CHECK-RV32IF-NEXT:    lui a1, %hi(h)
+; CHECK-RV32IF-NEXT:    lw a0, %lo(h)(a1)
+; CHECK-RV32IF-NEXT:    addi a1, a1, %lo(h)
+; CHECK-RV32IF-NEXT:    lw a1, 4(a1)
+; CHECK-RV32IF-NEXT:    lui a3, %hi(i)
+; CHECK-RV32IF-NEXT:    lw a2, %lo(i)(a3)
+; CHECK-RV32IF-NEXT:    addi a3, a3, %lo(i)
+; CHECK-RV32IF-NEXT:    lw a3, 4(a3)
+; CHECK-RV32IF-NEXT:    call __adddf3
+; CHECK-RV32IF-NEXT:    lui a2, %hi(g)
+; CHECK-RV32IF-NEXT:    addi a3, a2, %lo(g)
+; CHECK-RV32IF-NEXT:    sw a1, 4(a3)
+; CHECK-RV32IF-NEXT:    sw a0, %lo(g)(a2)
+; CHECK-RV32IF-NEXT:    flw fs11, 0(sp)
+; CHECK-RV32IF-NEXT:    flw fs10, 4(sp)
+; CHECK-RV32IF-NEXT:    flw fs9, 8(sp)
+; CHECK-RV32IF-NEXT:    flw fs8, 12(sp)
+; CHECK-RV32IF-NEXT:    flw fs7, 16(sp)
+; CHECK-RV32IF-NEXT:    flw fs6, 20(sp)
+; CHECK-RV32IF-NEXT:    flw fs5, 24(sp)
+; CHECK-RV32IF-NEXT:    flw fs4, 28(sp)
+; CHECK-RV32IF-NEXT:    flw fs3, 32(sp)
+; CHECK-RV32IF-NEXT:    flw fs2, 36(sp)
+; CHECK-RV32IF-NEXT:    flw fs1, 40(sp)
+; CHECK-RV32IF-NEXT:    flw fs0, 44(sp)
+; CHECK-RV32IF-NEXT:    flw ft11, 48(sp)
+; CHECK-RV32IF-NEXT:    flw ft10, 52(sp)
+; CHECK-RV32IF-NEXT:    flw ft9, 56(sp)
+; CHECK-RV32IF-NEXT:    flw ft8, 60(sp)
+; CHECK-RV32IF-NEXT:    flw fa7, 64(sp)
+; CHECK-RV32IF-NEXT:    flw fa6, 68(sp)
+; CHECK-RV32IF-NEXT:    flw fa5, 72(sp)
+; CHECK-RV32IF-NEXT:    flw fa4, 76(sp)
+; CHECK-RV32IF-NEXT:    flw fa3, 80(sp)
+; CHECK-RV32IF-NEXT:    flw fa2, 84(sp)
+; CHECK-RV32IF-NEXT:    flw fa1, 88(sp)
+; CHECK-RV32IF-NEXT:    flw fa0, 92(sp)
+; CHECK-RV32IF-NEXT:    flw ft7, 96(sp)
+; CHECK-RV32IF-NEXT:    flw ft6, 100(sp)
+; CHECK-RV32IF-NEXT:    flw ft5, 104(sp)
+; CHECK-RV32IF-NEXT:    flw ft4, 108(sp)
+; CHECK-RV32IF-NEXT:    flw ft3, 112(sp)
+; CHECK-RV32IF-NEXT:    flw ft2, 116(sp)
+; CHECK-RV32IF-NEXT:    flw ft1, 120(sp)
+; CHECK-RV32IF-NEXT:    flw ft0, 124(sp)
+; CHECK-RV32IF-NEXT:    lw t6, 128(sp)
+; CHECK-RV32IF-NEXT:    lw t5, 132(sp)
+; CHECK-RV32IF-NEXT:    lw t4, 136(sp)
+; CHECK-RV32IF-NEXT:    lw t3, 140(sp)
+; CHECK-RV32IF-NEXT:    lw a7, 144(sp)
+; CHECK-RV32IF-NEXT:    lw a6, 148(sp)
+; CHECK-RV32IF-NEXT:    lw a5, 152(sp)
+; CHECK-RV32IF-NEXT:    lw a4, 156(sp)
+; CHECK-RV32IF-NEXT:    lw a3, 160(sp)
+; CHECK-RV32IF-NEXT:    lw a2, 164(sp)
+; CHECK-RV32IF-NEXT:    lw a1, 168(sp)
+; CHECK-RV32IF-NEXT:    lw a0, 172(sp)
+; CHECK-RV32IF-NEXT:    lw t2, 176(sp)
+; CHECK-RV32IF-NEXT:    lw t1, 180(sp)
+; CHECK-RV32IF-NEXT:    lw t0, 184(sp)
+; CHECK-RV32IF-NEXT:    lw ra, 188(sp)
+; CHECK-RV32IF-NEXT:    addi sp, sp, 192
+; CHECK-RV32IF-NEXT:    mret
 ;
+; CHECK-RV32IFD-LABEL: foo_double:
+; CHECK-RV32IFD:       # %bb.0:
+; CHECK-RV32IFD-NEXT:    addi sp, sp, -32
+; CHECK-RV32IFD-NEXT:    sw a0, 28(sp)
+; CHECK-RV32IFD-NEXT:    fsd ft0, 16(sp)
+; CHECK-RV32IFD-NEXT:    fsd ft1, 8(sp)
+; CHECK-RV32IFD-NEXT:    lui a0, %hi(h)
+; CHECK-RV32IFD-NEXT:    fld ft0, %lo(h)(a0)
+; CHECK-RV32IFD-NEXT:    lui a0, %hi(i)
+; CHECK-RV32IFD-NEXT:    fld ft1, %lo(i)(a0)
+; CHECK-RV32IFD-NEXT:    fadd.d ft0, ft0, ft1
+; CHECK-RV32IFD-NEXT:    lui a0, %hi(g)
+; CHECK-RV32IFD-NEXT:    fsd ft0, %lo(g)(a0)
+; CHECK-RV32IFD-NEXT:    fld ft1, 8(sp)
+; CHECK-RV32IFD-NEXT:    fld ft0, 16(sp)
+; CHECK-RV32IFD-NEXT:    lw a0, 28(sp)
+; CHECK-RV32IFD-NEXT:    addi sp, sp, 32
+; CHECK-RV32IFD-NEXT:    mret
   %1 = load double, double* @h
   %2 = load double, double* @i
   %add = fadd double %1, %2
@@ -182,30 +557,201 @@ define void @foo_double() nounwind #0 {
 ; Additionally check frame pointer and return address are properly saved.
 ;
 define void @foo_fp_double() nounwind #1 {
-; CHECK-RV32-FD-LABEL: foo_fp_double:
-; CHECK-RV32-FD:       # %bb.0:
-; CHECK-RV32-FD-NEXT:    addi sp, sp, -32
-; CHECK-RV32-FD-NEXT:    sw ra, 28(sp)
-; CHECK-RV32-FD-NEXT:    sw s0, 24(sp)
-; CHECK-RV32-FD-NEXT:    sw a0, 20(sp)
-; CHECK-RV32-FD-NEXT:    fsd ft0, 8(sp)
-; CHECK-RV32-FD-NEXT:    fsd ft1, 0(sp)
-; CHECK-RV32-FD-NEXT:    addi s0, sp, 32
-; CHECK-RV32-FD-NEXT:    lui a0, %hi(i)
-; CHECK-RV32-FD-NEXT:    fld ft0, %lo(i)(a0)
-; CHECK-RV32-FD-NEXT:    lui a0, %hi(h)
-; CHECK-RV32-FD-NEXT:    fld ft1, %lo(h)(a0)
-; CHECK-RV32-FD-NEXT:    fadd.d ft0, ft1, ft0
-; CHECK-RV32-FD-NEXT:    lui a0, %hi(g)
-; CHECK-RV32-FD-NEXT:    fsd ft0, %lo(g)(a0)
-; CHECK-RV32-FD-NEXT:    fld ft1, 0(sp)
-; CHECK-RV32-FD-NEXT:    fld ft0, 8(sp)
-; CHECK-RV32-FD-NEXT:    lw a0, 20(sp)
-; CHECK-RV32-FD-NEXT:    lw s0, 24(sp)
-; CHECK-RV32-FD-NEXT:    lw ra, 28(sp)
-; CHECK-RV32-FD-NEXT:    addi sp, sp, 32
-; CHECK-RV32-FD-NEXT:    mret
+; CHECK-RV32-LABEL: foo_fp_double:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    addi sp, sp, -80
+; CHECK-RV32-NEXT:    sw ra, 76(sp)
+; CHECK-RV32-NEXT:    sw t0, 72(sp)
+; CHECK-RV32-NEXT:    sw t1, 68(sp)
+; CHECK-RV32-NEXT:    sw t2, 64(sp)
+; CHECK-RV32-NEXT:    sw s0, 60(sp)
+; CHECK-RV32-NEXT:    sw a0, 56(sp)
+; CHECK-RV32-NEXT:    sw a1, 52(sp)
+; CHECK-RV32-NEXT:    sw a2, 48(sp)
+; CHECK-RV32-NEXT:    sw a3, 44(sp)
+; CHECK-RV32-NEXT:    sw a4, 40(sp)
+; CHECK-RV32-NEXT:    sw a5, 36(sp)
+; CHECK-RV32-NEXT:    sw a6, 32(sp)
+; CHECK-RV32-NEXT:    sw a7, 28(sp)
+; CHECK-RV32-NEXT:    sw t3, 24(sp)
+; CHECK-RV32-NEXT:    sw t4, 20(sp)
+; CHECK-RV32-NEXT:    sw t5, 16(sp)
+; CHECK-RV32-NEXT:    sw t6, 12(sp)
+; CHECK-RV32-NEXT:    addi s0, sp, 80
+; CHECK-RV32-NEXT:    lui a1, %hi(h)
+; CHECK-RV32-NEXT:    lw a0, %lo(h)(a1)
+; CHECK-RV32-NEXT:    addi a1, a1, %lo(h)
+; CHECK-RV32-NEXT:    lw a1, 4(a1)
+; CHECK-RV32-NEXT:    lui a3, %hi(i)
+; CHECK-RV32-NEXT:    lw a2, %lo(i)(a3)
+; CHECK-RV32-NEXT:    addi a3, a3, %lo(i)
+; CHECK-RV32-NEXT:    lw a3, 4(a3)
+; CHECK-RV32-NEXT:    call __adddf3
+; CHECK-RV32-NEXT:    lui a2, %hi(g)
+; CHECK-RV32-NEXT:    addi a3, a2, %lo(g)
+; CHECK-RV32-NEXT:    sw a1, 4(a3)
+; CHECK-RV32-NEXT:    sw a0, %lo(g)(a2)
+; CHECK-RV32-NEXT:    lw t6, 12(sp)
+; CHECK-RV32-NEXT:    lw t5, 16(sp)
+; CHECK-RV32-NEXT:    lw t4, 20(sp)
+; CHECK-RV32-NEXT:    lw t3, 24(sp)
+; CHECK-RV32-NEXT:    lw a7, 28(sp)
+; CHECK-RV32-NEXT:    lw a6, 32(sp)
+; CHECK-RV32-NEXT:    lw a5, 36(sp)
+; CHECK-RV32-NEXT:    lw a4, 40(sp)
+; CHECK-RV32-NEXT:    lw a3, 44(sp)
+; CHECK-RV32-NEXT:    lw a2, 48(sp)
+; CHECK-RV32-NEXT:    lw a1, 52(sp)
+; CHECK-RV32-NEXT:    lw a0, 56(sp)
+; CHECK-RV32-NEXT:    lw s0, 60(sp)
+; CHECK-RV32-NEXT:    lw t2, 64(sp)
+; CHECK-RV32-NEXT:    lw t1, 68(sp)
+; CHECK-RV32-NEXT:    lw t0, 72(sp)
+; CHECK-RV32-NEXT:    lw ra, 76(sp)
+; CHECK-RV32-NEXT:    addi sp, sp, 80
+; CHECK-RV32-NEXT:    mret
+;
+; CHECK-RV32IF-LABEL: foo_fp_double:
+; CHECK-RV32IF:       # %bb.0:
+; CHECK-RV32IF-NEXT:    addi sp, sp, -208
+; CHECK-RV32IF-NEXT:    sw ra, 204(sp)
+; CHECK-RV32IF-NEXT:    sw t0, 200(sp)
+; CHECK-RV32IF-NEXT:    sw t1, 196(sp)
+; CHECK-RV32IF-NEXT:    sw t2, 192(sp)
+; CHECK-RV32IF-NEXT:    sw s0, 188(sp)
+; CHECK-RV32IF-NEXT:    sw a0, 184(sp)
+; CHECK-RV32IF-NEXT:    sw a1, 180(sp)
+; CHECK-RV32IF-NEXT:    sw a2, 176(sp)
+; CHECK-RV32IF-NEXT:    sw a3, 172(sp)
+; CHECK-RV32IF-NEXT:    sw a4, 168(sp)
+; CHECK-RV32IF-NEXT:    sw a5, 164(sp)
+; CHECK-RV32IF-NEXT:    sw a6, 160(sp)
+; CHECK-RV32IF-NEXT:    sw a7, 156(sp)
+; CHECK-RV32IF-NEXT:    sw t3, 152(sp)
+; CHECK-RV32IF-NEXT:    sw t4, 148(sp)
+; CHECK-RV32IF-NEXT:    sw t5, 144(sp)
+; CHECK-RV32IF-NEXT:    sw t6, 140(sp)
+; CHECK-RV32IF-NEXT:    fsw ft0, 136(sp)
+; CHECK-RV32IF-NEXT:    fsw ft1, 132(sp)
+; CHECK-RV32IF-NEXT:    fsw ft2, 128(sp)
+; CHECK-RV32IF-NEXT:    fsw ft3, 124(sp)
+; CHECK-RV32IF-NEXT:    fsw ft4, 120(sp)
+; CHECK-RV32IF-NEXT:    fsw ft5, 116(sp)
+; CHECK-RV32IF-NEXT:    fsw ft6, 112(sp)
+; CHECK-RV32IF-NEXT:    fsw ft7, 108(sp)
+; CHECK-RV32IF-NEXT:    fsw fa0, 104(sp)
+; CHECK-RV32IF-NEXT:    fsw fa1, 100(sp)
+; CHECK-RV32IF-NEXT:    fsw fa2, 96(sp)
+; CHECK-RV32IF-NEXT:    fsw fa3, 92(sp)
+; CHECK-RV32IF-NEXT:    fsw fa4, 88(sp)
+; CHECK-RV32IF-NEXT:    fsw fa5, 84(sp)
+; CHECK-RV32IF-NEXT:    fsw fa6, 80(sp)
+; CHECK-RV32IF-NEXT:    fsw fa7, 76(sp)
+; CHECK-RV32IF-NEXT:    fsw ft8, 72(sp)
+; CHECK-RV32IF-NEXT:    fsw ft9, 68(sp)
+; CHECK-RV32IF-NEXT:    fsw ft10, 64(sp)
+; CHECK-RV32IF-NEXT:    fsw ft11, 60(sp)
+; CHECK-RV32IF-NEXT:    fsw fs0, 56(sp)
+; CHECK-RV32IF-NEXT:    fsw fs1, 52(sp)
+; CHECK-RV32IF-NEXT:    fsw fs2, 48(sp)
+; CHECK-RV32IF-NEXT:    fsw fs3, 44(sp)
+; CHECK-RV32IF-NEXT:    fsw fs4, 40(sp)
+; CHECK-RV32IF-NEXT:    fsw fs5, 36(sp)
+; CHECK-RV32IF-NEXT:    fsw fs6, 32(sp)
+; CHECK-RV32IF-NEXT:    fsw fs7, 28(sp)
+; CHECK-RV32IF-NEXT:    fsw fs8, 24(sp)
+; CHECK-RV32IF-NEXT:    fsw fs9, 20(sp)
+; CHECK-RV32IF-NEXT:    fsw fs10, 16(sp)
+; CHECK-RV32IF-NEXT:    fsw fs11, 12(sp)
+; CHECK-RV32IF-NEXT:    addi s0, sp, 208
+; CHECK-RV32IF-NEXT:    lui a1, %hi(h)
+; CHECK-RV32IF-NEXT:    lw a0, %lo(h)(a1)
+; CHECK-RV32IF-NEXT:    addi a1, a1, %lo(h)
+; CHECK-RV32IF-NEXT:    lw a1, 4(a1)
+; CHECK-RV32IF-NEXT:    lui a3, %hi(i)
+; CHECK-RV32IF-NEXT:    lw a2, %lo(i)(a3)
+; CHECK-RV32IF-NEXT:    addi a3, a3, %lo(i)
+; CHECK-RV32IF-NEXT:    lw a3, 4(a3)
+; CHECK-RV32IF-NEXT:    call __adddf3
+; CHECK-RV32IF-NEXT:    lui a2, %hi(g)
+; CHECK-RV32IF-NEXT:    addi a3, a2, %lo(g)
+; CHECK-RV32IF-NEXT:    sw a1, 4(a3)
+; CHECK-RV32IF-NEXT:    sw a0, %lo(g)(a2)
+; CHECK-RV32IF-NEXT:    flw fs11, 12(sp)
+; CHECK-RV32IF-NEXT:    flw fs10, 16(sp)
+; CHECK-RV32IF-NEXT:    flw fs9, 20(sp)
+; CHECK-RV32IF-NEXT:    flw fs8, 24(sp)
+; CHECK-RV32IF-NEXT:    flw fs7, 28(sp)
+; CHECK-RV32IF-NEXT:    flw fs6, 32(sp)
+; CHECK-RV32IF-NEXT:    flw fs5, 36(sp)
+; CHECK-RV32IF-NEXT:    flw fs4, 40(sp)
+; CHECK-RV32IF-NEXT:    flw fs3, 44(sp)
+; CHECK-RV32IF-NEXT:    flw fs2, 48(sp)
+; CHECK-RV32IF-NEXT:    flw fs1, 52(sp)
+; CHECK-RV32IF-NEXT:    flw fs0, 56(sp)
+; CHECK-RV32IF-NEXT:    flw ft11, 60(sp)
+; CHECK-RV32IF-NEXT:    flw ft10, 64(sp)
+; CHECK-RV32IF-NEXT:    flw ft9, 68(sp)
+; CHECK-RV32IF-NEXT:    flw ft8, 72(sp)
+; CHECK-RV32IF-NEXT:    flw fa7, 76(sp)
+; CHECK-RV32IF-NEXT:    flw fa6, 80(sp)
+; CHECK-RV32IF-NEXT:    flw fa5, 84(sp)
+; CHECK-RV32IF-NEXT:    flw fa4, 88(sp)
+; CHECK-RV32IF-NEXT:    flw fa3, 92(sp)
+; CHECK-RV32IF-NEXT:    flw fa2, 96(sp)
+; CHECK-RV32IF-NEXT:    flw fa1, 100(sp)
+; CHECK-RV32IF-NEXT:    flw fa0, 104(sp)
+; CHECK-RV32IF-NEXT:    flw ft7, 108(sp)
+; CHECK-RV32IF-NEXT:    flw ft6, 112(sp)
+; CHECK-RV32IF-NEXT:    flw ft5, 116(sp)
+; CHECK-RV32IF-NEXT:    flw ft4, 120(sp)
+; CHECK-RV32IF-NEXT:    flw ft3, 124(sp)
+; CHECK-RV32IF-NEXT:    flw ft2, 128(sp)
+; CHECK-RV32IF-NEXT:    flw ft1, 132(sp)
+; CHECK-RV32IF-NEXT:    flw ft0, 136(sp)
+; CHECK-RV32IF-NEXT:    lw t6, 140(sp)
+; CHECK-RV32IF-NEXT:    lw t5, 144(sp)
+; CHECK-RV32IF-NEXT:    lw t4, 148(sp)
+; CHECK-RV32IF-NEXT:    lw t3, 152(sp)
+; CHECK-RV32IF-NEXT:    lw a7, 156(sp)
+; CHECK-RV32IF-NEXT:    lw a6, 160(sp)
+; CHECK-RV32IF-NEXT:    lw a5, 164(sp)
+; CHECK-RV32IF-NEXT:    lw a4, 168(sp)
+; CHECK-RV32IF-NEXT:    lw a3, 172(sp)
+; CHECK-RV32IF-NEXT:    lw a2, 176(sp)
+; CHECK-RV32IF-NEXT:    lw a1, 180(sp)
+; CHECK-RV32IF-NEXT:    lw a0, 184(sp)
+; CHECK-RV32IF-NEXT:    lw s0, 188(sp)
+; CHECK-RV32IF-NEXT:    lw t2, 192(sp)
+; CHECK-RV32IF-NEXT:    lw t1, 196(sp)
+; CHECK-RV32IF-NEXT:    lw t0, 200(sp)
+; CHECK-RV32IF-NEXT:    lw ra, 204(sp)
+; CHECK-RV32IF-NEXT:    addi sp, sp, 208
+; CHECK-RV32IF-NEXT:    mret
 ;
+; CHECK-RV32IFD-LABEL: foo_fp_double:
+; CHECK-RV32IFD:       # %bb.0:
+; CHECK-RV32IFD-NEXT:    addi sp, sp, -32
+; CHECK-RV32IFD-NEXT:    sw ra, 28(sp)
+; CHECK-RV32IFD-NEXT:    sw s0, 24(sp)
+; CHECK-RV32IFD-NEXT:    sw a0, 20(sp)
+; CHECK-RV32IFD-NEXT:    fsd ft0, 8(sp)
+; CHECK-RV32IFD-NEXT:    fsd ft1, 0(sp)
+; CHECK-RV32IFD-NEXT:    addi s0, sp, 32
+; CHECK-RV32IFD-NEXT:    lui a0, %hi(h)
+; CHECK-RV32IFD-NEXT:    fld ft0, %lo(h)(a0)
+; CHECK-RV32IFD-NEXT:    lui a0, %hi(i)
+; CHECK-RV32IFD-NEXT:    fld ft1, %lo(i)(a0)
+; CHECK-RV32IFD-NEXT:    fadd.d ft0, ft0, ft1
+; CHECK-RV32IFD-NEXT:    lui a0, %hi(g)
+; CHECK-RV32IFD-NEXT:    fsd ft0, %lo(g)(a0)
+; CHECK-RV32IFD-NEXT:    fld ft1, 0(sp)
+; CHECK-RV32IFD-NEXT:    fld ft0, 8(sp)
+; CHECK-RV32IFD-NEXT:    lw a0, 20(sp)
+; CHECK-RV32IFD-NEXT:    lw s0, 24(sp)
+; CHECK-RV32IFD-NEXT:    lw ra, 28(sp)
+; CHECK-RV32IFD-NEXT:    addi sp, sp, 32
+; CHECK-RV32IFD-NEXT:    mret
   %1 = load double, double* @h
   %2 = load double, double* @i
   %add = fadd double %1, %2
@@ -214,4 +760,4 @@ define void @foo_fp_double() nounwind #1 {
 }
 
 attributes #0 = { "interrupt"="machine" }
-attributes #1 = { "interrupt"="machine" "no-frame-pointer-elim"="true" }
+attributes #1 = { "interrupt"="machine" "frame-pointer"="all" }
diff --git a/llvm/test/CodeGen/RISCV/interrupt-attr.ll b/llvm/test/CodeGen/RISCV/interrupt-attr.ll
index f7831edac..040521a8e 100644
--- a/llvm/test/CodeGen/RISCV/interrupt-attr.ll
+++ b/llvm/test/CodeGen/RISCV/interrupt-attr.ll
@@ -1063,4 +1063,4 @@ define void @foo_fp_with_call() #3 {
 attributes #0 = { nounwind "interrupt"="user" }
 attributes #1 = { nounwind "interrupt"="supervisor" }
 attributes #2 = { nounwind  "interrupt"="machine" }
-attributes #3 = { nounwind "interrupt"="machine" "no-frame-pointer-elim"="true" }
+attributes #3 = { nounwind "interrupt"="machine" "frame-pointer"="all" }
diff --git a/llvm/test/CodeGen/RISCV/intrinsics/trap.ll b/llvm/test/CodeGen/RISCV/intrinsics/trap.ll
new file mode 100644
index 000000000..e85073518
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/intrinsics/trap.ll
@@ -0,0 +1,38 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV32I %s
+; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64I %s
+
+; Verify that we lower @llvm.trap() and @llvm.debugtrap() correctly.
+
+declare void @llvm.trap()
+declare void @llvm.debugtrap()
+
+define void @test_trap() nounwind {
+; RV32I-LABEL: test_trap:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    unimp
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: test_trap:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    unimp
+; RV64I-NEXT:    ret
+  tail call void @llvm.trap()
+  ret void
+}
+
+define void @test_debugtrap() nounwind {
+; RV32I-LABEL: test_debugtrap:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    ebreak
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: test_debugtrap:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    ebreak
+; RV64I-NEXT:    ret
+  tail call void @llvm.debugtrap()
+  ret void
+}
diff --git a/llvm/test/CodeGen/RISCV/large-stack.ll b/llvm/test/CodeGen/RISCV/large-stack.ll
index f41497122..7acf0f407 100644
--- a/llvm/test/CodeGen/RISCV/large-stack.ll
+++ b/llvm/test/CodeGen/RISCV/large-stack.ll
@@ -6,12 +6,13 @@
 
 ; TODO: the quality of the generated code is poor
 
-define void @test() nounwind {
+define void @test() {
 ; RV32I-FPELIM-LABEL: test:
 ; RV32I-FPELIM:       # %bb.0:
 ; RV32I-FPELIM-NEXT:    lui a0, 74565
 ; RV32I-FPELIM-NEXT:    addi a0, a0, 1664
 ; RV32I-FPELIM-NEXT:    sub sp, sp, a0
+; RV32I-FPELIM-NEXT:    .cfi_def_cfa_offset 305419904
 ; RV32I-FPELIM-NEXT:    lui a0, 74565
 ; RV32I-FPELIM-NEXT:    addi a0, a0, 1664
 ; RV32I-FPELIM-NEXT:    add sp, sp, a0
@@ -19,31 +20,23 @@ define void @test() nounwind {
 ;
 ; RV32I-WITHFP-LABEL: test:
 ; RV32I-WITHFP:       # %bb.0:
+; RV32I-WITHFP-NEXT:    addi sp, sp, -2032
+; RV32I-WITHFP-NEXT:    .cfi_def_cfa_offset 2032
+; RV32I-WITHFP-NEXT:    sw ra, 2028(sp)
+; RV32I-WITHFP-NEXT:    sw s0, 2024(sp)
+; RV32I-WITHFP-NEXT:    .cfi_offset ra, -4
+; RV32I-WITHFP-NEXT:    .cfi_offset s0, -8
+; RV32I-WITHFP-NEXT:    addi s0, sp, 2032
+; RV32I-WITHFP-NEXT:    .cfi_def_cfa s0, 0
 ; RV32I-WITHFP-NEXT:    lui a0, 74565
-; RV32I-WITHFP-NEXT:    addi a0, a0, 1680
+; RV32I-WITHFP-NEXT:    addi a0, a0, -352
 ; RV32I-WITHFP-NEXT:    sub sp, sp, a0
 ; RV32I-WITHFP-NEXT:    lui a0, 74565
-; RV32I-WITHFP-NEXT:    addi a0, a0, 1676
-; RV32I-WITHFP-NEXT:    add a0, sp, a0
-; RV32I-WITHFP-NEXT:    sw ra, 0(a0)
-; RV32I-WITHFP-NEXT:    lui a0, 74565
-; RV32I-WITHFP-NEXT:    addi a0, a0, 1672
-; RV32I-WITHFP-NEXT:    add a0, sp, a0
-; RV32I-WITHFP-NEXT:    sw s0, 0(a0)
-; RV32I-WITHFP-NEXT:    lui a0, 74565
-; RV32I-WITHFP-NEXT:    addi a0, a0, 1680
-; RV32I-WITHFP-NEXT:    add s0, sp, a0
-; RV32I-WITHFP-NEXT:    lui a0, 74565
-; RV32I-WITHFP-NEXT:    addi a0, a0, 1672
-; RV32I-WITHFP-NEXT:    add a0, sp, a0
-; RV32I-WITHFP-NEXT:    lw s0, 0(a0)
-; RV32I-WITHFP-NEXT:    lui a0, 74565
-; RV32I-WITHFP-NEXT:    addi a0, a0, 1676
-; RV32I-WITHFP-NEXT:    add a0, sp, a0
-; RV32I-WITHFP-NEXT:    lw ra, 0(a0)
-; RV32I-WITHFP-NEXT:    lui a0, 74565
-; RV32I-WITHFP-NEXT:    addi a0, a0, 1680
+; RV32I-WITHFP-NEXT:    addi a0, a0, -352
 ; RV32I-WITHFP-NEXT:    add sp, sp, a0
+; RV32I-WITHFP-NEXT:    lw s0, 2024(sp)
+; RV32I-WITHFP-NEXT:    lw ra, 2028(sp)
+; RV32I-WITHFP-NEXT:    addi sp, sp, 2032
 ; RV32I-WITHFP-NEXT:    ret
   %tmp = alloca [ 305419896 x i8 ] , align 4
   ret void
@@ -52,20 +45,19 @@ define void @test() nounwind {
 ; This test case artificially produces register pressure which should force
 ; use of the emergency spill slot.
 
-define void @test_emergency_spill_slot(i32 %a) nounwind {
+define void @test_emergency_spill_slot(i32 %a) {
 ; RV32I-FPELIM-LABEL: test_emergency_spill_slot:
 ; RV32I-FPELIM:       # %bb.0:
-; RV32I-FPELIM-NEXT:    lui a1, 98
-; RV32I-FPELIM-NEXT:    addi a1, a1, -1392
+; RV32I-FPELIM-NEXT:    addi sp, sp, -2032
+; RV32I-FPELIM-NEXT:    .cfi_def_cfa_offset 2032
+; RV32I-FPELIM-NEXT:    sw s0, 2028(sp)
+; RV32I-FPELIM-NEXT:    sw s1, 2024(sp)
+; RV32I-FPELIM-NEXT:    .cfi_offset s0, -4
+; RV32I-FPELIM-NEXT:    .cfi_offset s1, -8
+; RV32I-FPELIM-NEXT:    lui a1, 97
+; RV32I-FPELIM-NEXT:    addi a1, a1, 672
 ; RV32I-FPELIM-NEXT:    sub sp, sp, a1
-; RV32I-FPELIM-NEXT:    lui a1, 98
-; RV32I-FPELIM-NEXT:    addi a1, a1, -1396
-; RV32I-FPELIM-NEXT:    add a1, sp, a1
-; RV32I-FPELIM-NEXT:    sw s0, 0(a1)
-; RV32I-FPELIM-NEXT:    lui a1, 98
-; RV32I-FPELIM-NEXT:    addi a1, a1, -1400
-; RV32I-FPELIM-NEXT:    add a1, sp, a1
-; RV32I-FPELIM-NEXT:    sw s1, 0(a1)
+; RV32I-FPELIM-NEXT:    .cfi_def_cfa_offset 400016
 ; RV32I-FPELIM-NEXT:    lui a1, 78
 ; RV32I-FPELIM-NEXT:    addi a1, a1, 512
 ; RV32I-FPELIM-NEXT:    addi a2, sp, 8
@@ -77,43 +69,31 @@ define void @test_emergency_spill_slot(i32 %a) nounwind {
 ; RV32I-FPELIM-NEXT:    #APP
 ; RV32I-FPELIM-NEXT:    nop
 ; RV32I-FPELIM-NEXT:    #NO_APP
-; RV32I-FPELIM-NEXT:    lui a0, 98
-; RV32I-FPELIM-NEXT:    addi a0, a0, -1400
-; RV32I-FPELIM-NEXT:    add a0, sp, a0
-; RV32I-FPELIM-NEXT:    lw s1, 0(a0)
-; RV32I-FPELIM-NEXT:    lui a0, 98
-; RV32I-FPELIM-NEXT:    addi a0, a0, -1396
-; RV32I-FPELIM-NEXT:    add a0, sp, a0
-; RV32I-FPELIM-NEXT:    lw s0, 0(a0)
-; RV32I-FPELIM-NEXT:    lui a0, 98
-; RV32I-FPELIM-NEXT:    addi a0, a0, -1392
+; RV32I-FPELIM-NEXT:    lui a0, 97
+; RV32I-FPELIM-NEXT:    addi a0, a0, 672
 ; RV32I-FPELIM-NEXT:    add sp, sp, a0
+; RV32I-FPELIM-NEXT:    lw s1, 2024(sp)
+; RV32I-FPELIM-NEXT:    lw s0, 2028(sp)
+; RV32I-FPELIM-NEXT:    addi sp, sp, 2032
 ; RV32I-FPELIM-NEXT:    ret
 ;
 ; RV32I-WITHFP-LABEL: test_emergency_spill_slot:
 ; RV32I-WITHFP:       # %bb.0:
-; RV32I-WITHFP-NEXT:    lui a1, 98
-; RV32I-WITHFP-NEXT:    addi a1, a1, -1376
+; RV32I-WITHFP-NEXT:    addi sp, sp, -2032
+; RV32I-WITHFP-NEXT:    .cfi_def_cfa_offset 2032
+; RV32I-WITHFP-NEXT:    sw ra, 2028(sp)
+; RV32I-WITHFP-NEXT:    sw s0, 2024(sp)
+; RV32I-WITHFP-NEXT:    sw s1, 2020(sp)
+; RV32I-WITHFP-NEXT:    sw s2, 2016(sp)
+; RV32I-WITHFP-NEXT:    .cfi_offset ra, -4
+; RV32I-WITHFP-NEXT:    .cfi_offset s0, -8
+; RV32I-WITHFP-NEXT:    .cfi_offset s1, -12
+; RV32I-WITHFP-NEXT:    .cfi_offset s2, -16
+; RV32I-WITHFP-NEXT:    addi s0, sp, 2032
+; RV32I-WITHFP-NEXT:    .cfi_def_cfa s0, 0
+; RV32I-WITHFP-NEXT:    lui a1, 97
+; RV32I-WITHFP-NEXT:    addi a1, a1, 688
 ; RV32I-WITHFP-NEXT:    sub sp, sp, a1
-; RV32I-WITHFP-NEXT:    lui a1, 98
-; RV32I-WITHFP-NEXT:    addi a1, a1, -1380
-; RV32I-WITHFP-NEXT:    add a1, sp, a1
-; RV32I-WITHFP-NEXT:    sw ra, 0(a1)
-; RV32I-WITHFP-NEXT:    lui a1, 98
-; RV32I-WITHFP-NEXT:    addi a1, a1, -1384
-; RV32I-WITHFP-NEXT:    add a1, sp, a1
-; RV32I-WITHFP-NEXT:    sw s0, 0(a1)
-; RV32I-WITHFP-NEXT:    lui a1, 98
-; RV32I-WITHFP-NEXT:    addi a1, a1, -1388
-; RV32I-WITHFP-NEXT:    add a1, sp, a1
-; RV32I-WITHFP-NEXT:    sw s1, 0(a1)
-; RV32I-WITHFP-NEXT:    lui a1, 98
-; RV32I-WITHFP-NEXT:    addi a1, a1, -1392
-; RV32I-WITHFP-NEXT:    add a1, sp, a1
-; RV32I-WITHFP-NEXT:    sw s2, 0(a1)
-; RV32I-WITHFP-NEXT:    lui a1, 98
-; RV32I-WITHFP-NEXT:    addi a1, a1, -1376
-; RV32I-WITHFP-NEXT:    add s0, sp, a1
 ; RV32I-WITHFP-NEXT:    lui a1, 78
 ; RV32I-WITHFP-NEXT:    addi a1, a1, 512
 ; RV32I-WITHFP-NEXT:    lui a2, 1048478
@@ -128,25 +108,14 @@ define void @test_emergency_spill_slot(i32 %a) nounwind {
 ; RV32I-WITHFP-NEXT:    #APP
 ; RV32I-WITHFP-NEXT:    nop
 ; RV32I-WITHFP-NEXT:    #NO_APP
-; RV32I-WITHFP-NEXT:    lui a0, 98
-; RV32I-WITHFP-NEXT:    addi a0, a0, -1392
-; RV32I-WITHFP-NEXT:    add a0, sp, a0
-; RV32I-WITHFP-NEXT:    lw s2, 0(a0)
-; RV32I-WITHFP-NEXT:    lui a0, 98
-; RV32I-WITHFP-NEXT:    addi a0, a0, -1388
-; RV32I-WITHFP-NEXT:    add a0, sp, a0
-; RV32I-WITHFP-NEXT:    lw s1, 0(a0)
-; RV32I-WITHFP-NEXT:    lui a0, 98
-; RV32I-WITHFP-NEXT:    addi a0, a0, -1384
-; RV32I-WITHFP-NEXT:    add a0, sp, a0
-; RV32I-WITHFP-NEXT:    lw s0, 0(a0)
-; RV32I-WITHFP-NEXT:    lui a0, 98
-; RV32I-WITHFP-NEXT:    addi a0, a0, -1380
-; RV32I-WITHFP-NEXT:    add a0, sp, a0
-; RV32I-WITHFP-NEXT:    lw ra, 0(a0)
-; RV32I-WITHFP-NEXT:    lui a0, 98
-; RV32I-WITHFP-NEXT:    addi a0, a0, -1376
+; RV32I-WITHFP-NEXT:    lui a0, 97
+; RV32I-WITHFP-NEXT:    addi a0, a0, 688
 ; RV32I-WITHFP-NEXT:    add sp, sp, a0
+; RV32I-WITHFP-NEXT:    lw s2, 2016(sp)
+; RV32I-WITHFP-NEXT:    lw s1, 2020(sp)
+; RV32I-WITHFP-NEXT:    lw s0, 2024(sp)
+; RV32I-WITHFP-NEXT:    lw ra, 2028(sp)
+; RV32I-WITHFP-NEXT:    addi sp, sp, 2032
 ; RV32I-WITHFP-NEXT:    ret
   %data = alloca [ 100000 x i32 ] , align 4
   %ptr = getelementptr inbounds [100000 x i32], [100000 x i32]* %data, i32 0, i32 80000
diff --git a/llvm/test/CodeGen/RISCV/legalize-fneg.ll b/llvm/test/CodeGen/RISCV/legalize-fneg.ll
index d2d0b9ea6..80f0a988a 100644
--- a/llvm/test/CodeGen/RISCV/legalize-fneg.ll
+++ b/llvm/test/CodeGen/RISCV/legalize-fneg.ll
@@ -15,9 +15,9 @@ define void @test1(float* %a, float* %b) nounwind {
 ;
 ; RV64-LABEL: test1:
 ; RV64:       # %bb.0: # %entry
+; RV64-NEXT:    lw a1, 0(a1)
 ; RV64-NEXT:    addi a2, zero, 1
 ; RV64-NEXT:    slli a2, a2, 31
-; RV64-NEXT:    lw a1, 0(a1)
 ; RV64-NEXT:    xor a1, a1, a2
 ; RV64-NEXT:    sw a1, 0(a0)
 ; RV64-NEXT:    ret
@@ -33,17 +33,17 @@ define void @test2(double* %a, double* %b) nounwind {
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    lw a2, 4(a1)
 ; RV32-NEXT:    lw a1, 0(a1)
+; RV32-NEXT:    lui a3, 524288
+; RV32-NEXT:    xor a2, a2, a3
 ; RV32-NEXT:    sw a1, 0(a0)
-; RV32-NEXT:    lui a1, 524288
-; RV32-NEXT:    xor a1, a2, a1
-; RV32-NEXT:    sw a1, 4(a0)
+; RV32-NEXT:    sw a2, 4(a0)
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: test2:
 ; RV64:       # %bb.0: # %entry
+; RV64-NEXT:    ld a1, 0(a1)
 ; RV64-NEXT:    addi a2, zero, -1
 ; RV64-NEXT:    slli a2, a2, 63
-; RV64-NEXT:    ld a1, 0(a1)
 ; RV64-NEXT:    xor a1, a1, a2
 ; RV64-NEXT:    sd a1, 0(a0)
 ; RV64-NEXT:    ret
@@ -57,27 +57,27 @@ entry:
 define void @test3(fp128* %a, fp128* %b) nounwind {
 ; RV32-LABEL: test3:
 ; RV32:       # %bb.0: # %entry
-; RV32-NEXT:    lw a2, 12(a1)
-; RV32-NEXT:    lw a3, 4(a1)
-; RV32-NEXT:    lw a4, 0(a1)
-; RV32-NEXT:    lw a1, 8(a1)
-; RV32-NEXT:    sw a1, 8(a0)
-; RV32-NEXT:    sw a4, 0(a0)
-; RV32-NEXT:    sw a3, 4(a0)
-; RV32-NEXT:    lui a1, 524288
-; RV32-NEXT:    xor a1, a2, a1
-; RV32-NEXT:    sw a1, 12(a0)
+; RV32-NEXT:    lw a2, 4(a1)
+; RV32-NEXT:    lw a3, 12(a1)
+; RV32-NEXT:    lw a4, 8(a1)
+; RV32-NEXT:    lw a1, 0(a1)
+; RV32-NEXT:    lui a5, 524288
+; RV32-NEXT:    xor a3, a3, a5
+; RV32-NEXT:    sw a4, 8(a0)
+; RV32-NEXT:    sw a1, 0(a0)
+; RV32-NEXT:    sw a2, 4(a0)
+; RV32-NEXT:    sw a3, 12(a0)
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: test3:
 ; RV64:       # %bb.0: # %entry
 ; RV64-NEXT:    ld a2, 8(a1)
 ; RV64-NEXT:    ld a1, 0(a1)
+; RV64-NEXT:    addi a3, zero, -1
+; RV64-NEXT:    slli a3, a3, 63
+; RV64-NEXT:    xor a2, a2, a3
 ; RV64-NEXT:    sd a1, 0(a0)
-; RV64-NEXT:    addi a1, zero, -1
-; RV64-NEXT:    slli a1, a1, 63
-; RV64-NEXT:    xor a1, a2, a1
-; RV64-NEXT:    sd a1, 8(a0)
+; RV64-NEXT:    sd a2, 8(a0)
 ; RV64-NEXT:    ret
 entry:
   %0 = load fp128, fp128* %b
diff --git a/llvm/test/CodeGen/RISCV/lsr-legaladdimm.ll b/llvm/test/CodeGen/RISCV/lsr-legaladdimm.ll
index 2f4c821d8..dbe9eb9f0 100644
--- a/llvm/test/CodeGen/RISCV/lsr-legaladdimm.ll
+++ b/llvm/test/CodeGen/RISCV/lsr-legaladdimm.ll
@@ -11,21 +11,21 @@
 define i32 @main() nounwind {
 ; RV32I-LABEL: main:
 ; RV32I:       # %bb.0: # %entry
-; RV32I-NEXT:    lui a0, %hi(b)
-; RV32I-NEXT:    addi a0, a0, %lo(b)
-; RV32I-NEXT:    lui a1, %hi(a)
-; RV32I-NEXT:    addi a1, a1, %lo(a)
-; RV32I-NEXT:    lui a2, 1
-; RV32I-NEXT:    mv a3, zero
+; RV32I-NEXT:    mv a0, zero
+; RV32I-NEXT:    lui a1, %hi(b)
+; RV32I-NEXT:    addi a1, a1, %lo(b)
+; RV32I-NEXT:    lui a2, %hi(a)
+; RV32I-NEXT:    addi a2, a2, %lo(a)
+; RV32I-NEXT:    lui a3, 1
 ; RV32I-NEXT:  .LBB0_1: # %for.body
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    addi a4, a3, -2048
-; RV32I-NEXT:    sw a4, 0(a1)
+; RV32I-NEXT:    addi a4, a0, -2048
+; RV32I-NEXT:    sw a4, 0(a2)
+; RV32I-NEXT:    sw a0, 0(a1)
+; RV32I-NEXT:    addi a0, a0, 1
 ; RV32I-NEXT:    addi a1, a1, 4
-; RV32I-NEXT:    sw a3, 0(a0)
-; RV32I-NEXT:    addi a0, a0, 4
-; RV32I-NEXT:    addi a3, a3, 1
-; RV32I-NEXT:    bne a3, a2, .LBB0_1
+; RV32I-NEXT:    addi a2, a2, 4
+; RV32I-NEXT:    bne a0, a3, .LBB0_1
 ; RV32I-NEXT:  # %bb.2: # %for.end
 ; RV32I-NEXT:    mv a0, zero
 ; RV32I-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/mem.ll b/llvm/test/CodeGen/RISCV/mem.ll
index f873e8428..5a2eeee41 100644
--- a/llvm/test/CodeGen/RISCV/mem.ll
+++ b/llvm/test/CodeGen/RISCV/mem.ll
@@ -7,8 +7,9 @@
 define i32 @lb(i8 *%a) nounwind {
 ; RV32I-LABEL: lb:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    lb a1, 0(a0)
-; RV32I-NEXT:    lb a0, 1(a0)
+; RV32I-NEXT:    lb a1, 1(a0)
+; RV32I-NEXT:    lb a0, 0(a0)
+; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    ret
   %1 = getelementptr i8, i8* %a, i32 1
   %2 = load i8, i8* %1
@@ -21,8 +22,9 @@ define i32 @lb(i8 *%a) nounwind {
 define i32 @lh(i16 *%a) nounwind {
 ; RV32I-LABEL: lh:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    lh a1, 0(a0)
-; RV32I-NEXT:    lh a0, 4(a0)
+; RV32I-NEXT:    lh a1, 4(a0)
+; RV32I-NEXT:    lh a0, 0(a0)
+; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    ret
   %1 = getelementptr i16, i16* %a, i32 2
   %2 = load i16, i16* %1
@@ -35,8 +37,9 @@ define i32 @lh(i16 *%a) nounwind {
 define i32 @lw(i32 *%a) nounwind {
 ; RV32I-LABEL: lw:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    lw a1, 0(a0)
-; RV32I-NEXT:    lw a0, 12(a0)
+; RV32I-NEXT:    lw a1, 12(a0)
+; RV32I-NEXT:    lw a0, 0(a0)
+; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    ret
   %1 = getelementptr i32, i32* %a, i32 3
   %2 = load i32, i32* %1
@@ -47,9 +50,9 @@ define i32 @lw(i32 *%a) nounwind {
 define i32 @lbu(i8 *%a) nounwind {
 ; RV32I-LABEL: lbu:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    lbu a1, 0(a0)
-; RV32I-NEXT:    lbu a0, 4(a0)
-; RV32I-NEXT:    add a0, a0, a1
+; RV32I-NEXT:    lbu a1, 4(a0)
+; RV32I-NEXT:    lbu a0, 0(a0)
+; RV32I-NEXT:    add a0, a1, a0
 ; RV32I-NEXT:    ret
   %1 = getelementptr i8, i8* %a, i32 4
   %2 = load i8, i8* %1
@@ -63,9 +66,9 @@ define i32 @lbu(i8 *%a) nounwind {
 define i32 @lhu(i16 *%a) nounwind {
 ; RV32I-LABEL: lhu:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    lhu a1, 0(a0)
-; RV32I-NEXT:    lhu a0, 10(a0)
-; RV32I-NEXT:    add a0, a0, a1
+; RV32I-NEXT:    lhu a1, 10(a0)
+; RV32I-NEXT:    lhu a0, 0(a0)
+; RV32I-NEXT:    add a0, a1, a0
 ; RV32I-NEXT:    ret
   %1 = getelementptr i16, i16* %a, i32 5
   %2 = load i16, i16* %1
@@ -81,8 +84,8 @@ define i32 @lhu(i16 *%a) nounwind {
 define void @sb(i8 *%a, i8 %b) nounwind {
 ; RV32I-LABEL: sb:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    sb a1, 6(a0)
 ; RV32I-NEXT:    sb a1, 0(a0)
+; RV32I-NEXT:    sb a1, 6(a0)
 ; RV32I-NEXT:    ret
   store i8 %b, i8* %a
   %1 = getelementptr i8, i8* %a, i32 6
@@ -93,8 +96,8 @@ define void @sb(i8 *%a, i8 %b) nounwind {
 define void @sh(i16 *%a, i16 %b) nounwind {
 ; RV32I-LABEL: sh:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    sh a1, 14(a0)
 ; RV32I-NEXT:    sh a1, 0(a0)
+; RV32I-NEXT:    sh a1, 14(a0)
 ; RV32I-NEXT:    ret
   store i16 %b, i16* %a
   %1 = getelementptr i16, i16* %a, i32 7
@@ -105,8 +108,8 @@ define void @sh(i16 *%a, i16 %b) nounwind {
 define void @sw(i32 *%a, i32 %b) nounwind {
 ; RV32I-LABEL: sw:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    sw a1, 32(a0)
 ; RV32I-NEXT:    sw a1, 0(a0)
+; RV32I-NEXT:    sw a1, 32(a0)
 ; RV32I-NEXT:    ret
   store i32 %b, i32* %a
   %1 = getelementptr i32, i32* %a, i32 8
@@ -118,10 +121,10 @@ define void @sw(i32 *%a, i32 %b) nounwind {
 define i32 @load_sext_zext_anyext_i1(i1 *%a) nounwind {
 ; RV32I-LABEL: load_sext_zext_anyext_i1:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    lb a1, 0(a0)
 ; RV32I-NEXT:    lbu a1, 1(a0)
-; RV32I-NEXT:    lbu a0, 2(a0)
-; RV32I-NEXT:    sub a0, a0, a1
+; RV32I-NEXT:    lbu a2, 2(a0)
+; RV32I-NEXT:    lb a0, 0(a0)
+; RV32I-NEXT:    sub a0, a2, a1
 ; RV32I-NEXT:    ret
   ; sextload i1
   %1 = getelementptr i1, i1* %a, i32 1
@@ -140,10 +143,10 @@ define i32 @load_sext_zext_anyext_i1(i1 *%a) nounwind {
 define i16 @load_sext_zext_anyext_i1_i16(i1 *%a) nounwind {
 ; RV32I-LABEL: load_sext_zext_anyext_i1_i16:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    lb a1, 0(a0)
 ; RV32I-NEXT:    lbu a1, 1(a0)
-; RV32I-NEXT:    lbu a0, 2(a0)
-; RV32I-NEXT:    sub a0, a0, a1
+; RV32I-NEXT:    lbu a2, 2(a0)
+; RV32I-NEXT:    lb a0, 0(a0)
+; RV32I-NEXT:    sub a0, a2, a1
 ; RV32I-NEXT:    ret
   ; sextload i1
   %1 = getelementptr i1, i1* %a, i32 1
diff --git a/llvm/test/CodeGen/RISCV/mem64.ll b/llvm/test/CodeGen/RISCV/mem64.ll
index 32669e8ae..522786b86 100644
--- a/llvm/test/CodeGen/RISCV/mem64.ll
+++ b/llvm/test/CodeGen/RISCV/mem64.ll
@@ -7,8 +7,9 @@
 define i64 @lb(i8 *%a) nounwind {
 ; RV64I-LABEL: lb:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    lb a1, 0(a0)
-; RV64I-NEXT:    lb a0, 1(a0)
+; RV64I-NEXT:    lb a1, 1(a0)
+; RV64I-NEXT:    lb a0, 0(a0)
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ret
   %1 = getelementptr i8, i8* %a, i32 1
   %2 = load i8, i8* %1
@@ -21,8 +22,9 @@ define i64 @lb(i8 *%a) nounwind {
 define i64 @lh(i16 *%a) nounwind {
 ; RV64I-LABEL: lh:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    lh a1, 0(a0)
-; RV64I-NEXT:    lh a0, 4(a0)
+; RV64I-NEXT:    lh a1, 4(a0)
+; RV64I-NEXT:    lh a0, 0(a0)
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ret
   %1 = getelementptr i16, i16* %a, i32 2
   %2 = load i16, i16* %1
@@ -35,8 +37,9 @@ define i64 @lh(i16 *%a) nounwind {
 define i64 @lw(i32 *%a) nounwind {
 ; RV64I-LABEL: lw:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    lw a1, 0(a0)
-; RV64I-NEXT:    lw a0, 12(a0)
+; RV64I-NEXT:    lw a1, 12(a0)
+; RV64I-NEXT:    lw a0, 0(a0)
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ret
   %1 = getelementptr i32, i32* %a, i32 3
   %2 = load i32, i32* %1
@@ -49,9 +52,9 @@ define i64 @lw(i32 *%a) nounwind {
 define i64 @lbu(i8 *%a) nounwind {
 ; RV64I-LABEL: lbu:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    lbu a1, 0(a0)
-; RV64I-NEXT:    lbu a0, 4(a0)
-; RV64I-NEXT:    add a0, a0, a1
+; RV64I-NEXT:    lbu a1, 4(a0)
+; RV64I-NEXT:    lbu a0, 0(a0)
+; RV64I-NEXT:    add a0, a1, a0
 ; RV64I-NEXT:    ret
   %1 = getelementptr i8, i8* %a, i32 4
   %2 = load i8, i8* %1
@@ -65,9 +68,9 @@ define i64 @lbu(i8 *%a) nounwind {
 define i64 @lhu(i16 *%a) nounwind {
 ; RV64I-LABEL: lhu:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    lhu a1, 0(a0)
-; RV64I-NEXT:    lhu a0, 10(a0)
-; RV64I-NEXT:    add a0, a0, a1
+; RV64I-NEXT:    lhu a1, 10(a0)
+; RV64I-NEXT:    lhu a0, 0(a0)
+; RV64I-NEXT:    add a0, a1, a0
 ; RV64I-NEXT:    ret
   %1 = getelementptr i16, i16* %a, i32 5
   %2 = load i16, i16* %1
@@ -81,9 +84,9 @@ define i64 @lhu(i16 *%a) nounwind {
 define i64 @lwu(i32 *%a) nounwind {
 ; RV64I-LABEL: lwu:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    lwu a1, 0(a0)
-; RV64I-NEXT:    lwu a0, 24(a0)
-; RV64I-NEXT:    add a0, a0, a1
+; RV64I-NEXT:    lwu a1, 24(a0)
+; RV64I-NEXT:    lwu a0, 0(a0)
+; RV64I-NEXT:    add a0, a1, a0
 ; RV64I-NEXT:    ret
   %1 = getelementptr i32, i32* %a, i32 6
   %2 = load i32, i32* %1
@@ -99,8 +102,8 @@ define i64 @lwu(i32 *%a) nounwind {
 define void @sb(i8 *%a, i8 %b) nounwind {
 ; RV64I-LABEL: sb:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sb a1, 7(a0)
 ; RV64I-NEXT:    sb a1, 0(a0)
+; RV64I-NEXT:    sb a1, 7(a0)
 ; RV64I-NEXT:    ret
   store i8 %b, i8* %a
   %1 = getelementptr i8, i8* %a, i32 7
@@ -111,8 +114,8 @@ define void @sb(i8 *%a, i8 %b) nounwind {
 define void @sh(i16 *%a, i16 %b) nounwind {
 ; RV64I-LABEL: sh:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sh a1, 16(a0)
 ; RV64I-NEXT:    sh a1, 0(a0)
+; RV64I-NEXT:    sh a1, 16(a0)
 ; RV64I-NEXT:    ret
   store i16 %b, i16* %a
   %1 = getelementptr i16, i16* %a, i32 8
@@ -123,8 +126,8 @@ define void @sh(i16 *%a, i16 %b) nounwind {
 define void @sw(i32 *%a, i32 %b) nounwind {
 ; RV64I-LABEL: sw:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sw a1, 36(a0)
 ; RV64I-NEXT:    sw a1, 0(a0)
+; RV64I-NEXT:    sw a1, 36(a0)
 ; RV64I-NEXT:    ret
   store i32 %b, i32* %a
   %1 = getelementptr i32, i32* %a, i32 9
@@ -137,8 +140,9 @@ define void @sw(i32 *%a, i32 %b) nounwind {
 define i64 @ld(i64 *%a) nounwind {
 ; RV64I-LABEL: ld:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    ld a1, 0(a0)
-; RV64I-NEXT:    ld a0, 80(a0)
+; RV64I-NEXT:    ld a1, 80(a0)
+; RV64I-NEXT:    ld a0, 0(a0)
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ret
   %1 = getelementptr i64, i64* %a, i32 10
   %2 = load i64, i64* %1
@@ -149,8 +153,8 @@ define i64 @ld(i64 *%a) nounwind {
 define void @sd(i64 *%a, i64 %b) nounwind {
 ; RV64I-LABEL: sd:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sd a1, 88(a0)
 ; RV64I-NEXT:    sd a1, 0(a0)
+; RV64I-NEXT:    sd a1, 88(a0)
 ; RV64I-NEXT:    ret
   store i64 %b, i64* %a
   %1 = getelementptr i64, i64* %a, i32 11
@@ -162,10 +166,10 @@ define void @sd(i64 *%a, i64 %b) nounwind {
 define i64 @load_sext_zext_anyext_i1(i1 *%a) nounwind {
 ; RV64I-LABEL: load_sext_zext_anyext_i1:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    lb a1, 0(a0)
 ; RV64I-NEXT:    lbu a1, 1(a0)
-; RV64I-NEXT:    lbu a0, 2(a0)
-; RV64I-NEXT:    sub a0, a0, a1
+; RV64I-NEXT:    lbu a2, 2(a0)
+; RV64I-NEXT:    lb a0, 0(a0)
+; RV64I-NEXT:    sub a0, a2, a1
 ; RV64I-NEXT:    ret
   ; sextload i1
   %1 = getelementptr i1, i1* %a, i32 1
@@ -184,10 +188,10 @@ define i64 @load_sext_zext_anyext_i1(i1 *%a) nounwind {
 define i16 @load_sext_zext_anyext_i1_i16(i1 *%a) nounwind {
 ; RV64I-LABEL: load_sext_zext_anyext_i1_i16:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    lb a1, 0(a0)
 ; RV64I-NEXT:    lbu a1, 1(a0)
-; RV64I-NEXT:    lbu a0, 2(a0)
-; RV64I-NEXT:    sub a0, a0, a1
+; RV64I-NEXT:    lbu a2, 2(a0)
+; RV64I-NEXT:    lb a0, 0(a0)
+; RV64I-NEXT:    sub a0, a2, a1
 ; RV64I-NEXT:    ret
   ; sextload i1
   %1 = getelementptr i1, i1* %a, i32 1
diff --git a/llvm/test/CodeGen/RISCV/mir-target-flags.ll b/llvm/test/CodeGen/RISCV/mir-target-flags.ll
new file mode 100644
index 000000000..f41fb77db
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/mir-target-flags.ll
@@ -0,0 +1,74 @@
+; RUN: llc -mtriple=riscv32 --code-model=small \
+; RUN:    -stop-after riscv-expand-pseudo %s -o %t.mir
+; RUN: llc -mtriple=riscv32 -run-pass none %t.mir -o - | \
+; RUN:   FileCheck %s -check-prefix=RV32-SMALL
+;
+; RUN: llc -mtriple=riscv32 --code-model=medium --relocation-model=pic \
+; RUN:   -stop-after riscv-expand-pseudo %s -o %t.mir
+; RUN: llc -mtriple=riscv32 -run-pass none %t.mir -o - | \
+; RUN:   FileCheck %s -check-prefix=RV32-MED
+
+; This tests the RISC-V-specific serialization and deserialization of
+; `target-flags(...)`
+
+@g_e = external global i32
+@g_i = internal global i32 0
+@t_un = external thread_local global i32
+@t_ld = external thread_local(localdynamic) global i32
+@t_ie = external thread_local(initialexec) global i32
+@t_le = external thread_local(localexec) global i32
+
+declare i32 @callee(i32) nounwind
+
+define i32 @caller(i32 %a) nounwind {
+; RV32-SMALL-LABEL: name: caller
+; RV32-SMALL:      target-flags(riscv-hi) @g_e
+; RV32-SMALL-NEXT: target-flags(riscv-lo) @g_e
+; RV32-SMALL-NEXT: target-flags(riscv-hi) @g_i
+; RV32-SMALL-NEXT: target-flags(riscv-lo) @g_i
+; RV32-SMALL:      target-flags(riscv-tls-got-hi) @t_un
+; RV32-SMALL-NEXT: target-flags(riscv-pcrel-lo) %bb.1
+; RV32-SMALL:      target-flags(riscv-tls-got-hi) @t_ld
+; RV32-SMALL-NEXT: target-flags(riscv-pcrel-lo) %bb.2
+; RV32-SMALL:      target-flags(riscv-tls-got-hi) @t_ie
+; RV32-SMALL-NEXT: target-flags(riscv-pcrel-lo) %bb.3
+; RV32-SMALL:      target-flags(riscv-tprel-hi) @t_le
+; RV32-SMALL-NEXT: target-flags(riscv-tprel-add) @t_le
+; RV32-SMALL-NEXT: target-flags(riscv-tprel-lo) @t_le
+; RV32-SMALL:      target-flags(riscv-call) @callee
+;
+; RV32-MED-LABEL: name: caller
+; RV32-MED:      target-flags(riscv-got-hi) @g_e
+; RV32-MED-NEXT: target-flags(riscv-pcrel-lo) %bb.1
+; RV32-MED:      target-flags(riscv-pcrel-hi) @g_i
+; RV32-MED-NEXT: target-flags(riscv-pcrel-lo) %bb.2
+; RV32-MED:      target-flags(riscv-tls-gd-hi) @t_un
+; RV32-MED-NEXT: target-flags(riscv-pcrel-lo) %bb.3
+; RV32-MED-NEXT: target-flags(riscv-plt) &__tls_get_addr
+; RV32-MED:      target-flags(riscv-tls-gd-hi) @t_ld
+; RV32-MED-NEXT: target-flags(riscv-pcrel-lo) %bb.4
+; RV32-MED-NEXT: target-flags(riscv-plt) &__tls_get_addr
+; RV32-MED:      target-flags(riscv-tls-got-hi) @t_ie
+; RV32-MED-NEXT: target-flags(riscv-pcrel-lo) %bb.5
+; RV32-MED:      target-flags(riscv-tprel-hi) @t_le
+; RV32-MED-NEXT: target-flags(riscv-tprel-add) @t_le
+; RV32-MED-NEXT: target-flags(riscv-tprel-lo) @t_le
+; RV32-MED:      target-flags(riscv-plt) @callee
+;
+  %b = load i32, i32* @g_e
+  %c = load i32, i32* @g_i
+  %d = load i32, i32* @t_un
+  %e = load i32, i32* @t_ld
+  %f = load i32, i32* @t_ie
+  %g = load i32, i32* @t_le
+  %sum = bitcast i32 0 to i32
+  %sum.a = add i32 %sum, %a
+  %sum.b = add i32 %sum.a, %b
+  %sum.c = add i32 %sum.b, %c
+  %sum.d = add i32 %sum.c, %d
+  %sum.e = add i32 %sum.d, %e
+  %sum.f = add i32 %sum.e, %f
+  %sum.g = add i32 %sum.f, %g
+  %retval = call i32 @callee(i32 %sum.g)
+  ret i32 %retval
+}
diff --git a/llvm/test/CodeGen/RISCV/module-target-abi.ll b/llvm/test/CodeGen/RISCV/module-target-abi.ll
new file mode 100644
index 000000000..b2930a06a
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/module-target-abi.ll
@@ -0,0 +1,24 @@
+; RUN: llc -mtriple=riscv32 < %s 2>&1 \
+; RUN:   | FileCheck -check-prefix=DEFAULT %s
+; RUN: llc -mtriple=riscv32 -target-abi ilp32 < %s 2>&1 \
+; RUN:   | FileCheck -check-prefix=RV32IF-ILP32 %s
+; RUN: not llc -mtriple=riscv32 -target-abi ilp32f < %s 2>&1 \
+; RUN:   | FileCheck -check-prefix=RV32IF-ILP32F %s
+; RUN: llc -mtriple=riscv32 -filetype=obj < %s | llvm-readelf -h - | FileCheck -check-prefixes=FLAGS %s
+
+; RV32IF-ILP32F: -target-abi option != target-abi module flag
+
+; FLAGS: Flags: 0x0
+
+define float @foo(i32 %a) nounwind #0 {
+; DEFAULT: # %bb.0:
+; DEFAULT: fmv.x.w a0, ft0
+; RV32IF-ILP32: # %bb.0:
+; RV32IF-ILP32: fmv.x.w a0, ft0
+  %conv = sitofp i32 %a to float
+  ret float %conv
+}
+
+attributes #0 = { "target-features"="+f"}
+!llvm.module.flags = !{!0}
+!0 = !{i32 1, !"target-abi", !"ilp32"}
diff --git a/llvm/test/CodeGen/RISCV/module-target-abi2.ll b/llvm/test/CodeGen/RISCV/module-target-abi2.ll
new file mode 100644
index 000000000..f07f2770a
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/module-target-abi2.ll
@@ -0,0 +1,27 @@
+; RUN: llc -mtriple=riscv32 < %s 2>&1 \
+; RUN:   | FileCheck -check-prefix=DEFAULT %s
+; RUN: not llc -mtriple=riscv32 -target-abi ilp32 < %s 2>&1 \
+; RUN:   | FileCheck -check-prefix=RV32IF-ILP32 %s
+; RUN: llc -mtriple=riscv32 -target-abi ilp32f < %s 2>&1 \
+; RUN:   | FileCheck -check-prefix=RV32IF-ILP32F %s
+; RUN: llc -mtriple=riscv32 -filetype=obj < %s | llvm-readelf -h - | FileCheck -check-prefixes=FLAGS %s
+
+; RV32IF-ILP32: -target-abi option != target-abi module flag
+
+; FLAGS: Flags: 0x0
+; // this should be "Flags :0x2, single-float ABI", it will be fixed later.
+
+define float @foo(i32 %a) nounwind #0 {
+; DEFAULT: # %bb.0:
+; DEFAULT-NEXT: fcvt.s.w fa0, a0
+; DEFAULT-NEXT: ret
+; RV32IF-ILP32F: # %bb.0:
+; RV32IF-ILP32F: fcvt.s.w fa0, a0
+; RV32IF-ILP32F: ret
+  %conv = sitofp i32 %a to float
+  ret float %conv
+}
+
+attributes #0 = { "target-features"="+f"}
+!llvm.module.flags = !{!0}
+!0 = !{i32 1, !"target-abi", !"ilp32f"}
diff --git a/llvm/test/CodeGen/RISCV/mul.ll b/llvm/test/CodeGen/RISCV/mul.ll
index 9bf95bece..2448580d3 100644
--- a/llvm/test/CodeGen/RISCV/mul.ll
+++ b/llvm/test/CodeGen/RISCV/mul.ll
@@ -247,8 +247,8 @@ define i32 @mulhs(i32 %a, i32 %b) nounwind {
 ;
 ; RV64IM-LABEL: mulhs:
 ; RV64IM:       # %bb.0:
-; RV64IM-NEXT:    sext.w a1, a1
 ; RV64IM-NEXT:    sext.w a0, a0
+; RV64IM-NEXT:    sext.w a1, a1
 ; RV64IM-NEXT:    mul a0, a0, a1
 ; RV64IM-NEXT:    srli a0, a0, 32
 ; RV64IM-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/option-norvc.ll b/llvm/test/CodeGen/RISCV/option-norvc.ll
index 381fb3773..db96a1106 100644
--- a/llvm/test/CodeGen/RISCV/option-norvc.ll
+++ b/llvm/test/CodeGen/RISCV/option-norvc.ll
@@ -1,5 +1,5 @@
 ; RUN: llc -mtriple=riscv32 -mattr=+c  -filetype=obj < %s\
-; RUN: | llvm-objdump -triple=riscv32 -mattr=+c -d -riscv-no-aliases -\
+; RUN: | llvm-objdump -triple=riscv32 -mattr=+c -d -M no-aliases -\
 ; RUN: | FileCheck -check-prefix=CHECK %s
 
 ; This test demonstrates that .option norvc has no effect on codegen when
diff --git a/llvm/test/CodeGen/RISCV/option-rvc.ll b/llvm/test/CodeGen/RISCV/option-rvc.ll
index a0f09c052..9d7f19400 100644
--- a/llvm/test/CodeGen/RISCV/option-rvc.ll
+++ b/llvm/test/CodeGen/RISCV/option-rvc.ll
@@ -1,5 +1,5 @@
 ; RUN: llc -mtriple=riscv32 -filetype=obj < %s\
-; RUN: | llvm-objdump -triple=riscv32 -mattr=+c -d -riscv-no-aliases -\
+; RUN: | llvm-objdump -triple=riscv32 -mattr=+c -d -M no-aliases -\
 ; RUN: | FileCheck -check-prefix=CHECK %s
 
 ; This test demonstrates that .option norvc has no effect on codegen when
diff --git a/llvm/test/CodeGen/RISCV/remat.ll b/llvm/test/CodeGen/RISCV/remat.ll
index 41cb90ba3..b3be74cd7 100644
--- a/llvm/test/CodeGen/RISCV/remat.ll
+++ b/llvm/test/CodeGen/RISCV/remat.ll
@@ -37,78 +37,77 @@ define i32 @test() nounwind {
 ; RV32I-NEXT:    sw s9, 20(sp)
 ; RV32I-NEXT:    sw s10, 16(sp)
 ; RV32I-NEXT:    sw s11, 12(sp)
-; RV32I-NEXT:    lui s9, %hi(a)
-; RV32I-NEXT:    lw a0, %lo(a)(s9)
+; RV32I-NEXT:    lui s6, %hi(a)
+; RV32I-NEXT:    lw a0, %lo(a)(s6)
 ; RV32I-NEXT:    beqz a0, .LBB0_11
 ; RV32I-NEXT:  # %bb.1: # %for.body.preheader
 ; RV32I-NEXT:    lui s2, %hi(l)
 ; RV32I-NEXT:    lui s3, %hi(k)
 ; RV32I-NEXT:    lui s4, %hi(j)
-; RV32I-NEXT:    lui s6, %hi(i)
-; RV32I-NEXT:    lui s5, %hi(h)
-; RV32I-NEXT:    lui s7, %hi(g)
-; RV32I-NEXT:    lui s8, %hi(f)
-; RV32I-NEXT:    lui s1, %hi(e)
-; RV32I-NEXT:    lui s0, %hi(d)
+; RV32I-NEXT:    lui s5, %hi(i)
+; RV32I-NEXT:    lui s1, %hi(d)
+; RV32I-NEXT:    lui s0, %hi(e)
+; RV32I-NEXT:    lui s7, %hi(f)
+; RV32I-NEXT:    lui s8, %hi(g)
+; RV32I-NEXT:    lui s9, %hi(h)
 ; RV32I-NEXT:    lui s10, %hi(c)
 ; RV32I-NEXT:    lui s11, %hi(b)
-; RV32I-NEXT:    lw a1, %lo(l)(s2)
-; RV32I-NEXT:    bnez a1, .LBB0_4
-; RV32I-NEXT:    j .LBB0_5
+; RV32I-NEXT:    j .LBB0_3
 ; RV32I-NEXT:  .LBB0_2: # %for.inc
-; RV32I-NEXT:    # in Loop: Header=BB0_5 Depth=1
-; RV32I-NEXT:    lw a0, %lo(a)(s9)
+; RV32I-NEXT:    # in Loop: Header=BB0_3 Depth=1
+; RV32I-NEXT:    lw a0, %lo(a)(s6)
 ; RV32I-NEXT:    addi a0, a0, -1
-; RV32I-NEXT:    sw a0, %lo(a)(s9)
+; RV32I-NEXT:    sw a0, %lo(a)(s6)
 ; RV32I-NEXT:    beqz a0, .LBB0_11
-; RV32I-NEXT:  # %bb.3: # %for.body
-; RV32I-NEXT:    # in Loop: Header=BB0_5 Depth=1
+; RV32I-NEXT:  .LBB0_3: # %for.body
+; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV32I-NEXT:    lw a1, %lo(l)(s2)
 ; RV32I-NEXT:    beqz a1, .LBB0_5
-; RV32I-NEXT:  .LBB0_4: # %if.then
-; RV32I-NEXT:    lw a4, %lo(e)(s1)
-; RV32I-NEXT:    lw a3, %lo(d)(s0)
-; RV32I-NEXT:    lw a2, %lo(c)(s10)
+; RV32I-NEXT:  # %bb.4: # %if.then
+; RV32I-NEXT:    # in Loop: Header=BB0_3 Depth=1
 ; RV32I-NEXT:    lw a1, %lo(b)(s11)
+; RV32I-NEXT:    lw a2, %lo(c)(s10)
+; RV32I-NEXT:    lw a3, %lo(d)(s1)
+; RV32I-NEXT:    lw a4, %lo(e)(s0)
 ; RV32I-NEXT:    addi a5, zero, 32
 ; RV32I-NEXT:    call foo
 ; RV32I-NEXT:  .LBB0_5: # %if.end
-; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32I-NEXT:    # in Loop: Header=BB0_3 Depth=1
 ; RV32I-NEXT:    lw a0, %lo(k)(s3)
 ; RV32I-NEXT:    beqz a0, .LBB0_7
 ; RV32I-NEXT:  # %bb.6: # %if.then3
-; RV32I-NEXT:    # in Loop: Header=BB0_5 Depth=1
-; RV32I-NEXT:    lw a4, %lo(f)(s8)
-; RV32I-NEXT:    lw a3, %lo(e)(s1)
-; RV32I-NEXT:    lw a2, %lo(d)(s0)
-; RV32I-NEXT:    lw a1, %lo(c)(s10)
+; RV32I-NEXT:    # in Loop: Header=BB0_3 Depth=1
 ; RV32I-NEXT:    lw a0, %lo(b)(s11)
+; RV32I-NEXT:    lw a1, %lo(c)(s10)
+; RV32I-NEXT:    lw a2, %lo(d)(s1)
+; RV32I-NEXT:    lw a3, %lo(e)(s0)
+; RV32I-NEXT:    lw a4, %lo(f)(s7)
 ; RV32I-NEXT:    addi a5, zero, 64
 ; RV32I-NEXT:    call foo
 ; RV32I-NEXT:  .LBB0_7: # %if.end5
-; RV32I-NEXT:    # in Loop: Header=BB0_5 Depth=1
+; RV32I-NEXT:    # in Loop: Header=BB0_3 Depth=1
 ; RV32I-NEXT:    lw a0, %lo(j)(s4)
 ; RV32I-NEXT:    beqz a0, .LBB0_9
 ; RV32I-NEXT:  # %bb.8: # %if.then7
-; RV32I-NEXT:    # in Loop: Header=BB0_5 Depth=1
-; RV32I-NEXT:    lw a4, %lo(g)(s7)
-; RV32I-NEXT:    lw a3, %lo(f)(s8)
-; RV32I-NEXT:    lw a2, %lo(e)(s1)
-; RV32I-NEXT:    lw a1, %lo(d)(s0)
+; RV32I-NEXT:    # in Loop: Header=BB0_3 Depth=1
 ; RV32I-NEXT:    lw a0, %lo(c)(s10)
+; RV32I-NEXT:    lw a1, %lo(d)(s1)
+; RV32I-NEXT:    lw a2, %lo(e)(s0)
+; RV32I-NEXT:    lw a3, %lo(f)(s7)
+; RV32I-NEXT:    lw a4, %lo(g)(s8)
 ; RV32I-NEXT:    addi a5, zero, 32
 ; RV32I-NEXT:    call foo
 ; RV32I-NEXT:  .LBB0_9: # %if.end9
-; RV32I-NEXT:    # in Loop: Header=BB0_5 Depth=1
-; RV32I-NEXT:    lw a0, %lo(i)(s6)
+; RV32I-NEXT:    # in Loop: Header=BB0_3 Depth=1
+; RV32I-NEXT:    lw a0, %lo(i)(s5)
 ; RV32I-NEXT:    beqz a0, .LBB0_2
 ; RV32I-NEXT:  # %bb.10: # %if.then11
-; RV32I-NEXT:    # in Loop: Header=BB0_5 Depth=1
-; RV32I-NEXT:    lw a4, %lo(h)(s5)
-; RV32I-NEXT:    lw a3, %lo(g)(s7)
-; RV32I-NEXT:    lw a2, %lo(f)(s8)
-; RV32I-NEXT:    lw a1, %lo(e)(s1)
-; RV32I-NEXT:    lw a0, %lo(d)(s0)
+; RV32I-NEXT:    # in Loop: Header=BB0_3 Depth=1
+; RV32I-NEXT:    lw a0, %lo(d)(s1)
+; RV32I-NEXT:    lw a1, %lo(e)(s0)
+; RV32I-NEXT:    lw a2, %lo(f)(s7)
+; RV32I-NEXT:    lw a3, %lo(g)(s8)
+; RV32I-NEXT:    lw a4, %lo(h)(s9)
 ; RV32I-NEXT:    addi a5, zero, 32
 ; RV32I-NEXT:    call foo
 ; RV32I-NEXT:    j .LBB0_2
@@ -145,7 +144,7 @@ if.then:                                          ; preds = %for.body
   %3 = load i32, i32* @c, align 4
   %4 = load i32, i32* @d, align 4
   %5 = load i32, i32* @e, align 4
-  %call = tail call i32 @foo(i32 %0, i32 %2, i32 %3, i32 %4, i32 %5, i32 32) #3
+  %call = tail call i32 @foo(i32 %0, i32 %2, i32 %3, i32 %4, i32 %5, i32 32)
   br label %if.end
 
 if.end:                                           ; preds = %for.body, %if.then
@@ -159,7 +158,7 @@ if.then3:                                         ; preds = %if.end
   %9 = load i32, i32* @d, align 4
   %10 = load i32, i32* @e, align 4
   %11 = load i32, i32* @f, align 4
-  %call4 = tail call i32 @foo(i32 %7, i32 %8, i32 %9, i32 %10, i32 %11, i32 64) #3
+  %call4 = tail call i32 @foo(i32 %7, i32 %8, i32 %9, i32 %10, i32 %11, i32 64)
   br label %if.end5
 
 if.end5:                                          ; preds = %if.end, %if.then3
@@ -173,7 +172,7 @@ if.then7:                                         ; preds = %if.end5
   %15 = load i32, i32* @e, align 4
   %16 = load i32, i32* @f, align 4
   %17 = load i32, i32* @g, align 4
-  %call8 = tail call i32 @foo(i32 %13, i32 %14, i32 %15, i32 %16, i32 %17, i32 32) #3
+  %call8 = tail call i32 @foo(i32 %13, i32 %14, i32 %15, i32 %16, i32 %17, i32 32)
   br label %if.end9
 
 if.end9:                                          ; preds = %if.end5, %if.then7
@@ -187,7 +186,7 @@ if.then11:                                        ; preds = %if.end9
   %21 = load i32, i32* @f, align 4
   %22 = load i32, i32* @g, align 4
   %23 = load i32, i32* @h, align 4
-  %call12 = tail call i32 @foo(i32 %19, i32 %20, i32 %21, i32 %22, i32 %23, i32 32) #3
+  %call12 = tail call i32 @foo(i32 %19, i32 %20, i32 %21, i32 %22, i32 %23, i32 32)
   br label %for.inc
 
 for.inc:                                          ; preds = %if.end9, %if.then11
diff --git a/llvm/test/CodeGen/RISCV/reserved-reg-errors.ll b/llvm/test/CodeGen/RISCV/reserved-reg-errors.ll
new file mode 100644
index 000000000..f10ef9ab3
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/reserved-reg-errors.ll
@@ -0,0 +1,36 @@
+; RUN: not llc -mtriple=riscv32 -mattr=+reserve-x1 < %s 2>&1 | FileCheck %s -check-prefix=X1
+; RUN: not llc -mtriple=riscv64 -mattr=+reserve-x1 < %s 2>&1 | FileCheck %s -check-prefix=X1
+; RUN: not llc -mtriple=riscv32 -mattr=+reserve-x2 < %s 2>&1 | FileCheck %s -check-prefix=X2
+; RUN: not llc -mtriple=riscv64 -mattr=+reserve-x2 < %s 2>&1 | FileCheck %s -check-prefix=X2
+; RUN: not llc -mtriple=riscv32 -mattr=+reserve-x8 < %s 2>&1 | FileCheck %s -check-prefix=X8
+; RUN: not llc -mtriple=riscv64 -mattr=+reserve-x8 < %s 2>&1 | FileCheck %s -check-prefix=X8
+; RUN: not llc -mtriple=riscv32 -mattr=+reserve-x10 < %s 2>&1 | FileCheck %s -check-prefix=X10
+; RUN: not llc -mtriple=riscv64 -mattr=+reserve-x10 < %s 2>&1 | FileCheck %s -check-prefix=X10
+; RUN: llc -mtriple=riscv32 -mattr=+reserve-x11 < %s 2>&1 | FileCheck %s -check-prefix=X11
+; RUN: llc -mtriple=riscv32 -mattr=+reserve-x11 < %s 2>&1 | FileCheck %s -check-prefix=X11
+; RUN: llc -mtriple=riscv32 <%s
+; RUN: llc -mtriple=riscv64 <%s
+
+; This tests combinations when we would expect an error to be produced because
+; a reserved register is required by the default ABI. The final test checks no
+; errors are produced when no registers are reserved.
+
+define i32 @caller(i32 %a) #0 {
+; X1: in function caller {{.*}} Return address register required, but has been reserved.
+; X2: in function caller {{.*}} Stack pointer required, but has been reserved.
+; X8: in function caller {{.*}} Frame pointer required, but has been reserved.
+; X10: in function caller {{.*}} Argument register required, but has been reserved.
+; X10: in function caller {{.*}} Return value register required, but has been reserved.
+  %call = call i32 @callee(i32 0)
+  ret i32 %call
+}
+
+declare i32 @callee(i32 %a)
+
+define void @clobber() {
+; X11: warning: inline asm clobber list contains reserved registers: X11
+  call void asm sideeffect "nop", "~{x11}"()
+  ret void
+}
+
+attributes #0 = { "frame-pointer"="all" }
diff --git a/llvm/test/CodeGen/RISCV/reserved-regs.ll b/llvm/test/CodeGen/RISCV/reserved-regs.ll
new file mode 100644
index 000000000..a4d09904a
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/reserved-regs.ll
@@ -0,0 +1,130 @@
+; RUN: llc -mtriple=riscv32 -mattr=+reserve-x3 -verify-machineinstrs < %s | FileCheck %s -check-prefix=X3
+; RUN: llc -mtriple=riscv64 -mattr=+reserve-x3 -verify-machineinstrs < %s | FileCheck %s -check-prefix=X3
+; RUN: llc -mtriple=riscv32 -mattr=+reserve-x4 -verify-machineinstrs < %s | FileCheck %s -check-prefix=X4
+; RUN: llc -mtriple=riscv64 -mattr=+reserve-x4 -verify-machineinstrs < %s | FileCheck %s -check-prefix=X4
+; RUN: llc -mtriple=riscv32 -mattr=+reserve-x5 -verify-machineinstrs < %s | FileCheck %s -check-prefix=X5
+; RUN: llc -mtriple=riscv64 -mattr=+reserve-x5 -verify-machineinstrs < %s | FileCheck %s -check-prefix=X5
+; RUN: llc -mtriple=riscv32 -mattr=+reserve-x6 -verify-machineinstrs < %s | FileCheck %s -check-prefix=X6
+; RUN: llc -mtriple=riscv64 -mattr=+reserve-x6 -verify-machineinstrs < %s | FileCheck %s -check-prefix=X6
+; RUN: llc -mtriple=riscv32 -mattr=+reserve-x7 -verify-machineinstrs < %s | FileCheck %s -check-prefix=X7
+; RUN: llc -mtriple=riscv64 -mattr=+reserve-x7 -verify-machineinstrs < %s | FileCheck %s -check-prefix=X7
+; RUN: llc -mtriple=riscv32 -mattr=+reserve-x8 -verify-machineinstrs < %s | FileCheck %s -check-prefix=X8
+; RUN: llc -mtriple=riscv64 -mattr=+reserve-x8 -verify-machineinstrs < %s | FileCheck %s -check-prefix=X8
+; RUN: llc -mtriple=riscv32 -mattr=+reserve-x9 -verify-machineinstrs < %s | FileCheck %s -check-prefix=X9
+; RUN: llc -mtriple=riscv64 -mattr=+reserve-x9 -verify-machineinstrs < %s | FileCheck %s -check-prefix=X9
+; RUN: llc -mtriple=riscv32 -mattr=+reserve-x10 -verify-machineinstrs < %s | FileCheck %s -check-prefix=X10
+; RUN: llc -mtriple=riscv64 -mattr=+reserve-x10 -verify-machineinstrs < %s | FileCheck %s -check-prefix=X10
+; RUN: llc -mtriple=riscv32 -mattr=+reserve-x11 -verify-machineinstrs < %s | FileCheck %s -check-prefix=X11
+; RUN: llc -mtriple=riscv64 -mattr=+reserve-x11 -verify-machineinstrs < %s | FileCheck %s -check-prefix=X11
+; RUN: llc -mtriple=riscv32 -mattr=+reserve-x12 -verify-machineinstrs < %s | FileCheck %s -check-prefix=X12
+; RUN: llc -mtriple=riscv64 -mattr=+reserve-x12 -verify-machineinstrs < %s | FileCheck %s -check-prefix=X12
+; RUN: llc -mtriple=riscv32 -mattr=+reserve-x13 -verify-machineinstrs < %s | FileCheck %s -check-prefix=X13
+; RUN: llc -mtriple=riscv64 -mattr=+reserve-x13 -verify-machineinstrs < %s | FileCheck %s -check-prefix=X13
+; RUN: llc -mtriple=riscv32 -mattr=+reserve-x14 -verify-machineinstrs < %s | FileCheck %s -check-prefix=X14
+; RUN: llc -mtriple=riscv64 -mattr=+reserve-x14 -verify-machineinstrs < %s | FileCheck %s -check-prefix=X14
+; RUN: llc -mtriple=riscv32 -mattr=+reserve-x15 -verify-machineinstrs < %s | FileCheck %s -check-prefix=X15
+; RUN: llc -mtriple=riscv64 -mattr=+reserve-x15 -verify-machineinstrs < %s | FileCheck %s -check-prefix=X15
+; RUN: llc -mtriple=riscv32 -mattr=+reserve-x16 -verify-machineinstrs < %s | FileCheck %s -check-prefix=X16
+; RUN: llc -mtriple=riscv64 -mattr=+reserve-x16 -verify-machineinstrs < %s | FileCheck %s -check-prefix=X16
+; RUN: llc -mtriple=riscv32 -mattr=+reserve-x17 -verify-machineinstrs < %s | FileCheck %s -check-prefix=X17
+; RUN: llc -mtriple=riscv64 -mattr=+reserve-x17 -verify-machineinstrs < %s | FileCheck %s -check-prefix=X17
+; RUN: llc -mtriple=riscv32 -mattr=+reserve-x18 -verify-machineinstrs < %s | FileCheck %s -check-prefix=X18
+; RUN: llc -mtriple=riscv64 -mattr=+reserve-x18 -verify-machineinstrs < %s | FileCheck %s -check-prefix=X18
+; RUN: llc -mtriple=riscv32 -mattr=+reserve-x19 -verify-machineinstrs < %s | FileCheck %s -check-prefix=X19
+; RUN: llc -mtriple=riscv64 -mattr=+reserve-x19 -verify-machineinstrs < %s | FileCheck %s -check-prefix=X19
+; RUN: llc -mtriple=riscv32 -mattr=+reserve-x20 -verify-machineinstrs < %s | FileCheck %s -check-prefix=X20
+; RUN: llc -mtriple=riscv64 -mattr=+reserve-x20 -verify-machineinstrs < %s | FileCheck %s -check-prefix=X20
+; RUN: llc -mtriple=riscv32 -mattr=+reserve-x21 -verify-machineinstrs < %s | FileCheck %s -check-prefix=X21
+; RUN: llc -mtriple=riscv64 -mattr=+reserve-x21 -verify-machineinstrs < %s | FileCheck %s -check-prefix=X21
+; RUN: llc -mtriple=riscv32 -mattr=+reserve-x22 -verify-machineinstrs < %s | FileCheck %s -check-prefix=X22
+; RUN: llc -mtriple=riscv64 -mattr=+reserve-x22 -verify-machineinstrs < %s | FileCheck %s -check-prefix=X22
+; RUN: llc -mtriple=riscv32 -mattr=+reserve-x23 -verify-machineinstrs < %s | FileCheck %s -check-prefix=X23
+; RUN: llc -mtriple=riscv64 -mattr=+reserve-x23 -verify-machineinstrs < %s | FileCheck %s -check-prefix=X23
+; RUN: llc -mtriple=riscv32 -mattr=+reserve-x24 -verify-machineinstrs < %s | FileCheck %s -check-prefix=X24
+; RUN: llc -mtriple=riscv64 -mattr=+reserve-x24 -verify-machineinstrs < %s | FileCheck %s -check-prefix=X24
+; RUN: llc -mtriple=riscv32 -mattr=+reserve-x25 -verify-machineinstrs < %s | FileCheck %s -check-prefix=X25
+; RUN: llc -mtriple=riscv64 -mattr=+reserve-x25 -verify-machineinstrs < %s | FileCheck %s -check-prefix=X25
+; RUN: llc -mtriple=riscv32 -mattr=+reserve-x26 -verify-machineinstrs < %s | FileCheck %s -check-prefix=X26
+; RUN: llc -mtriple=riscv64 -mattr=+reserve-x26 -verify-machineinstrs < %s | FileCheck %s -check-prefix=X26
+; RUN: llc -mtriple=riscv32 -mattr=+reserve-x27 -verify-machineinstrs < %s | FileCheck %s -check-prefix=X27
+; RUN: llc -mtriple=riscv64 -mattr=+reserve-x27 -verify-machineinstrs < %s | FileCheck %s -check-prefix=X27
+; RUN: llc -mtriple=riscv32 -mattr=+reserve-x28 -verify-machineinstrs < %s | FileCheck %s -check-prefix=X28
+; RUN: llc -mtriple=riscv64 -mattr=+reserve-x28 -verify-machineinstrs < %s | FileCheck %s -check-prefix=X28
+; RUN: llc -mtriple=riscv32 -mattr=+reserve-x29 -verify-machineinstrs < %s | FileCheck %s -check-prefix=X29
+; RUN: llc -mtriple=riscv64 -mattr=+reserve-x29 -verify-machineinstrs < %s | FileCheck %s -check-prefix=X29
+; RUN: llc -mtriple=riscv32 -mattr=+reserve-x30 -verify-machineinstrs < %s | FileCheck %s -check-prefix=X30
+; RUN: llc -mtriple=riscv64 -mattr=+reserve-x30 -verify-machineinstrs < %s | FileCheck %s -check-prefix=X30
+; RUN: llc -mtriple=riscv32 -mattr=+reserve-x31 -verify-machineinstrs < %s | FileCheck %s -check-prefix=X31
+; RUN: llc -mtriple=riscv64 -mattr=+reserve-x31 -verify-machineinstrs < %s | FileCheck %s -check-prefix=X31
+
+; This program is free to use all registers, but needs a stack pointer for
+; spill values, so do not test for reserving the stack pointer.
+
+; Used to exhaust all registers
+@var = global [32 x i64] zeroinitializer
+
+define void @foo() {
+  %1 = load volatile [32 x i64], [32 x i64]* @var
+  store volatile [32 x i64] %1, [32 x i64]* @var
+
+; X3-NOT: lw gp,
+; X3-NOT: ld gp,
+; X4-NOT: lw tp,
+; X4-NOT: ld tp,
+; X5-NOT: lw t0,
+; X5-NOT: ld t0,
+; X6-NOT: lw t1,
+; X6-NOT: ld t1,
+; X7-NOT: lw t2,
+; X7-NOT: ld t2,
+; X8-NOT: lw s0,
+; X8-NOT: ld s0,
+; X9-NOT: lw s1,
+; X9-NOT: ld s1,
+; X10-NOT: lw a0,
+; X10-NOT: ld a0,
+; X11-NOT: lw a1,
+; X11-NOT: ld a1,
+; X12-NOT: lw a2,
+; X12-NOT: ld a2,
+; X13-NOT: lw a3,
+; X13-NOT: ld a3,
+; X14-NOT: lw a4,
+; X14-NOT: ld a4,
+; X15-NOT: lw a5,
+; X15-NOT: ld a5,
+; X16-NOT: lw a6,
+; X16-NOT: ld a6,
+; X17-NOT: lw a7,
+; X17-NOT: ld a7,
+; X18-NOT: lw s2,
+; X18-NOT: ld s2,
+; X19-NOT: lw s3,
+; X19-NOT: ld s3,
+; X20-NOT: lw s4,
+; X20-NOT: ld s4,
+; X21-NOT: lw s5,
+; X21-NOT: ld s5,
+; X22-NOT: lw s6,
+; X22-NOT: ld s6,
+; X23-NOT: lw s7,
+; X23-NOT: ld s7,
+; X24-NOT: lw s8,
+; X24-NOT: ld s8,
+; X25-NOT: lw s9,
+; X25-NOT: ld s9,
+; X26-NOT: lw s10,
+; X26-NOT: ld s10,
+; X27-NOT: lw s11,
+; X27-NOT: ld s11,
+; X28-NOT: lw t3,
+; X28-NOT: ld t3,
+; X29-NOT: lw t4,
+; X29-NOT: ld t4,
+; X30-NOT: lw t5,
+; X30-NOT: ld t5,
+; X31-NOT: lw t6,
+; X31-NOT: ld t6,
+
+  ret void
+}
diff --git a/llvm/test/CodeGen/RISCV/rv32i-rv64i-float-double.ll b/llvm/test/CodeGen/RISCV/rv32i-rv64i-float-double.ll
index 7bb4e28fa..11e7568eb 100644
--- a/llvm/test/CodeGen/RISCV/rv32i-rv64i-float-double.ll
+++ b/llvm/test/CodeGen/RISCV/rv32i-rv64i-float-double.ll
@@ -31,11 +31,7 @@ define float @float_test(float %a, float %b) nounwind {
 ; RV64IF-NEXT:    addi sp, sp, -16
 ; RV64IF-NEXT:    sd ra, 8(sp)
 ; RV64IF-NEXT:    sd s0, 0(sp)
-; RV64IF-NEXT:    slli a0, a0, 32
-; RV64IF-NEXT:    srli a0, a0, 32
-; RV64IF-NEXT:    slli a1, a1, 32
-; RV64IF-NEXT:    srli s0, a1, 32
-; RV64IF-NEXT:    mv a1, s0
+; RV64IF-NEXT:    mv s0, a1
 ; RV64IF-NEXT:    call __addsf3
 ; RV64IF-NEXT:    mv a1, s0
 ; RV64IF-NEXT:    call __divsf3
diff --git a/llvm/test/CodeGen/RISCV/rv64-large-stack.ll b/llvm/test/CodeGen/RISCV/rv64-large-stack.ll
new file mode 100644
index 000000000..dbe19cc60
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rv64-large-stack.ll
@@ -0,0 +1,34 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s | FileCheck %s
+;
+; The test case check that RV64 could handle the stack adjustment offset exceed
+; 32-bit.
+
+define void @foo() nounwind {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addi sp, sp, -2032
+; CHECK-NEXT:    sd ra, 2024(sp)
+; CHECK-NEXT:    lui a0, 95
+; CHECK-NEXT:    addiw a0, a0, 1505
+; CHECK-NEXT:    slli a0, a0, 13
+; CHECK-NEXT:    addi a0, a0, -2000
+; CHECK-NEXT:    sub sp, sp, a0
+; CHECK-NEXT:    addi a0, sp, 16
+; CHECK-NEXT:    call baz
+; CHECK-NEXT:    lui a0, 95
+; CHECK-NEXT:    addiw a0, a0, 1505
+; CHECK-NEXT:    slli a0, a0, 13
+; CHECK-NEXT:    addi a0, a0, -2000
+; CHECK-NEXT:    add sp, sp, a0
+; CHECK-NEXT:    ld ra, 2024(sp)
+; CHECK-NEXT:    addi sp, sp, 2032
+; CHECK-NEXT:    ret
+entry:
+  %w = alloca [100000000 x { fp128, fp128 }], align 16
+  %arraydecay = getelementptr inbounds [100000000 x { fp128, fp128 }], [100000000 x { fp128, fp128 }]* %w, i64 0, i64 0
+  call void @baz({ fp128, fp128 }* nonnull %arraydecay)
+  ret void
+}
+
+declare void @baz({ fp128, fp128 }*)
diff --git a/llvm/test/CodeGen/RISCV/rv64f-float-convert.ll b/llvm/test/CodeGen/RISCV/rv64f-float-convert.ll
index 5f8952085..13e47242d 100644
--- a/llvm/test/CodeGen/RISCV/rv64f-float-convert.ll
+++ b/llvm/test/CodeGen/RISCV/rv64f-float-convert.ll
@@ -113,9 +113,9 @@ define zeroext i32 @bcvt_f32_to_zext_i32(float %a, float %b) nounwind {
 define float @bcvt_i64_to_f32_via_i32(i64 %a, i64 %b) nounwind {
 ; RV64IF-LABEL: bcvt_i64_to_f32_via_i32:
 ; RV64IF:       # %bb.0:
-; RV64IF-NEXT:    fmv.w.x ft0, a1
-; RV64IF-NEXT:    fmv.w.x ft1, a0
-; RV64IF-NEXT:    fadd.s ft0, ft1, ft0
+; RV64IF-NEXT:    fmv.w.x ft0, a0
+; RV64IF-NEXT:    fmv.w.x ft1, a1
+; RV64IF-NEXT:    fadd.s ft0, ft0, ft1
 ; RV64IF-NEXT:    fmv.x.w a0, ft0
 ; RV64IF-NEXT:    ret
   %1 = trunc i64 %a to i32
diff --git a/llvm/test/CodeGen/RISCV/rv64i-complex-float.ll b/llvm/test/CodeGen/RISCV/rv64i-complex-float.ll
new file mode 100644
index 000000000..a545a13b9
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rv64i-complex-float.ll
@@ -0,0 +1,53 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s | FileCheck %s
+
+; The complex floating value will be returned by a single register for LP64 ABI.
+; The test case check that the real part returned by __addsf3 will be
+; cleared upper bits by shifts to avoid corrupting the imaginary part.
+
+define i64 @complex_float_add(i64 %a.coerce, i64 %b.coerce) nounwind {
+; CHECK-LABEL: complex_float_add:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addi sp, sp, -32
+; CHECK-NEXT:    sd ra, 24(sp)
+; CHECK-NEXT:    sd s0, 16(sp)
+; CHECK-NEXT:    sd s1, 8(sp)
+; CHECK-NEXT:    sd s2, 0(sp)
+; CHECK-NEXT:    srli s2, a0, 32
+; CHECK-NEXT:    srli s1, a1, 32
+; CHECK-NEXT:    call __addsf3
+; CHECK-NEXT:    mv s0, a0
+; CHECK-NEXT:    mv a0, s2
+; CHECK-NEXT:    mv a1, s1
+; CHECK-NEXT:    call __addsf3
+; CHECK-NEXT:    slli a0, a0, 32
+; CHECK-NEXT:    slli a1, s0, 32
+; CHECK-NEXT:    srli a1, a1, 32
+; CHECK-NEXT:    or a0, a0, a1
+; CHECK-NEXT:    ld s2, 0(sp)
+; CHECK-NEXT:    ld s1, 8(sp)
+; CHECK-NEXT:    ld s0, 16(sp)
+; CHECK-NEXT:    ld ra, 24(sp)
+; CHECK-NEXT:    addi sp, sp, 32
+; CHECK-NEXT:    ret
+entry:
+  %a.sroa.0.0.extract.trunc = trunc i64 %a.coerce to i32
+  %0 = bitcast i32 %a.sroa.0.0.extract.trunc to float
+  %a.sroa.2.0.extract.shift = lshr i64 %a.coerce, 32
+  %a.sroa.2.0.extract.trunc = trunc i64 %a.sroa.2.0.extract.shift to i32
+  %1 = bitcast i32 %a.sroa.2.0.extract.trunc to float
+  %b.sroa.0.0.extract.trunc = trunc i64 %b.coerce to i32
+  %2 = bitcast i32 %b.sroa.0.0.extract.trunc to float
+  %b.sroa.2.0.extract.shift = lshr i64 %b.coerce, 32
+  %b.sroa.2.0.extract.trunc = trunc i64 %b.sroa.2.0.extract.shift to i32
+  %3 = bitcast i32 %b.sroa.2.0.extract.trunc to float
+  %add.r = fadd float %0, %2
+  %add.i = fadd float %1, %3
+  %4 = bitcast float %add.r to i32
+  %5 = bitcast float %add.i to i32
+  %retval.sroa.2.0.insert.ext = zext i32 %5 to i64
+  %retval.sroa.2.0.insert.shift = shl nuw i64 %retval.sroa.2.0.insert.ext, 32
+  %retval.sroa.0.0.insert.ext = zext i32 %4 to i64
+  %retval.sroa.0.0.insert.insert = or i64 %retval.sroa.2.0.insert.shift, %retval.sroa.0.0.insert.ext
+  ret i64 %retval.sroa.0.0.insert.insert
+}
diff --git a/llvm/test/CodeGen/RISCV/rv64i-exhaustive-w-insts.ll b/llvm/test/CodeGen/RISCV/rv64i-exhaustive-w-insts.ll
index b7f513d6d..b14ed5031 100644
--- a/llvm/test/CodeGen/RISCV/rv64i-exhaustive-w-insts.ll
+++ b/llvm/test/CodeGen/RISCV/rv64i-exhaustive-w-insts.ll
@@ -11,7 +11,7 @@
 define i32 @aext_addw_aext_aext(i32 %a, i32 %b) nounwind {
 ; RV64I-LABEL: aext_addw_aext_aext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    add a0, a0, a1
+; RV64I-NEXT:    addw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = add i32 %a, %b
   ret i32 %1
@@ -20,7 +20,7 @@ define i32 @aext_addw_aext_aext(i32 %a, i32 %b) nounwind {
 define i32 @aext_addw_aext_sext(i32 %a, i32 signext %b) nounwind {
 ; RV64I-LABEL: aext_addw_aext_sext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    add a0, a0, a1
+; RV64I-NEXT:    addw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = add i32 %a, %b
   ret i32 %1
@@ -29,7 +29,7 @@ define i32 @aext_addw_aext_sext(i32 %a, i32 signext %b) nounwind {
 define i32 @aext_addw_aext_zext(i32 %a, i32 zeroext %b) nounwind {
 ; RV64I-LABEL: aext_addw_aext_zext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    add a0, a0, a1
+; RV64I-NEXT:    addw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = add i32 %a, %b
   ret i32 %1
@@ -38,7 +38,7 @@ define i32 @aext_addw_aext_zext(i32 %a, i32 zeroext %b) nounwind {
 define i32 @aext_addw_sext_aext(i32 signext %a, i32 %b) nounwind {
 ; RV64I-LABEL: aext_addw_sext_aext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    add a0, a0, a1
+; RV64I-NEXT:    addw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = add i32 %a, %b
   ret i32 %1
@@ -47,7 +47,7 @@ define i32 @aext_addw_sext_aext(i32 signext %a, i32 %b) nounwind {
 define i32 @aext_addw_sext_sext(i32 signext %a, i32 signext %b) nounwind {
 ; RV64I-LABEL: aext_addw_sext_sext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    add a0, a0, a1
+; RV64I-NEXT:    addw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = add i32 %a, %b
   ret i32 %1
@@ -56,7 +56,7 @@ define i32 @aext_addw_sext_sext(i32 signext %a, i32 signext %b) nounwind {
 define i32 @aext_addw_sext_zext(i32 signext %a, i32 zeroext %b) nounwind {
 ; RV64I-LABEL: aext_addw_sext_zext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    add a0, a0, a1
+; RV64I-NEXT:    addw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = add i32 %a, %b
   ret i32 %1
@@ -65,7 +65,7 @@ define i32 @aext_addw_sext_zext(i32 signext %a, i32 zeroext %b) nounwind {
 define i32 @aext_addw_zext_aext(i32 zeroext %a, i32 %b) nounwind {
 ; RV64I-LABEL: aext_addw_zext_aext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    add a0, a0, a1
+; RV64I-NEXT:    addw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = add i32 %a, %b
   ret i32 %1
@@ -74,7 +74,7 @@ define i32 @aext_addw_zext_aext(i32 zeroext %a, i32 %b) nounwind {
 define i32 @aext_addw_zext_sext(i32 zeroext %a, i32 signext %b) nounwind {
 ; RV64I-LABEL: aext_addw_zext_sext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    add a0, a0, a1
+; RV64I-NEXT:    addw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = add i32 %a, %b
   ret i32 %1
@@ -83,7 +83,7 @@ define i32 @aext_addw_zext_sext(i32 zeroext %a, i32 signext %b) nounwind {
 define i32 @aext_addw_zext_zext(i32 zeroext %a, i32 zeroext %b) nounwind {
 ; RV64I-LABEL: aext_addw_zext_zext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    add a0, a0, a1
+; RV64I-NEXT:    addw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = add i32 %a, %b
   ret i32 %1
@@ -279,7 +279,7 @@ define zeroext i32 @zext_addw_zext_zext(i32 zeroext %a, i32 zeroext %b) nounwind
 define i32 @aext_subw_aext_aext(i32 %a, i32 %b) nounwind {
 ; RV64I-LABEL: aext_subw_aext_aext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sub a0, a0, a1
+; RV64I-NEXT:    subw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = sub i32 %a, %b
   ret i32 %1
@@ -288,7 +288,7 @@ define i32 @aext_subw_aext_aext(i32 %a, i32 %b) nounwind {
 define i32 @aext_subw_aext_sext(i32 %a, i32 signext %b) nounwind {
 ; RV64I-LABEL: aext_subw_aext_sext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sub a0, a0, a1
+; RV64I-NEXT:    subw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = sub i32 %a, %b
   ret i32 %1
@@ -297,7 +297,7 @@ define i32 @aext_subw_aext_sext(i32 %a, i32 signext %b) nounwind {
 define i32 @aext_subw_aext_zext(i32 %a, i32 zeroext %b) nounwind {
 ; RV64I-LABEL: aext_subw_aext_zext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sub a0, a0, a1
+; RV64I-NEXT:    subw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = sub i32 %a, %b
   ret i32 %1
@@ -306,7 +306,7 @@ define i32 @aext_subw_aext_zext(i32 %a, i32 zeroext %b) nounwind {
 define i32 @aext_subw_sext_aext(i32 signext %a, i32 %b) nounwind {
 ; RV64I-LABEL: aext_subw_sext_aext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sub a0, a0, a1
+; RV64I-NEXT:    subw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = sub i32 %a, %b
   ret i32 %1
@@ -315,7 +315,7 @@ define i32 @aext_subw_sext_aext(i32 signext %a, i32 %b) nounwind {
 define i32 @aext_subw_sext_sext(i32 signext %a, i32 signext %b) nounwind {
 ; RV64I-LABEL: aext_subw_sext_sext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sub a0, a0, a1
+; RV64I-NEXT:    subw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = sub i32 %a, %b
   ret i32 %1
@@ -324,7 +324,7 @@ define i32 @aext_subw_sext_sext(i32 signext %a, i32 signext %b) nounwind {
 define i32 @aext_subw_sext_zext(i32 signext %a, i32 zeroext %b) nounwind {
 ; RV64I-LABEL: aext_subw_sext_zext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sub a0, a0, a1
+; RV64I-NEXT:    subw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = sub i32 %a, %b
   ret i32 %1
@@ -333,7 +333,7 @@ define i32 @aext_subw_sext_zext(i32 signext %a, i32 zeroext %b) nounwind {
 define i32 @aext_subw_zext_aext(i32 zeroext %a, i32 %b) nounwind {
 ; RV64I-LABEL: aext_subw_zext_aext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sub a0, a0, a1
+; RV64I-NEXT:    subw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = sub i32 %a, %b
   ret i32 %1
@@ -342,7 +342,7 @@ define i32 @aext_subw_zext_aext(i32 zeroext %a, i32 %b) nounwind {
 define i32 @aext_subw_zext_sext(i32 zeroext %a, i32 signext %b) nounwind {
 ; RV64I-LABEL: aext_subw_zext_sext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sub a0, a0, a1
+; RV64I-NEXT:    subw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = sub i32 %a, %b
   ret i32 %1
@@ -351,7 +351,7 @@ define i32 @aext_subw_zext_sext(i32 zeroext %a, i32 signext %b) nounwind {
 define i32 @aext_subw_zext_zext(i32 zeroext %a, i32 zeroext %b) nounwind {
 ; RV64I-LABEL: aext_subw_zext_zext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sub a0, a0, a1
+; RV64I-NEXT:    subw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = sub i32 %a, %b
   ret i32 %1
diff --git a/llvm/test/CodeGen/RISCV/rv64i-single-softfloat.ll b/llvm/test/CodeGen/RISCV/rv64i-single-softfloat.ll
new file mode 100644
index 000000000..e486a48cb
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rv64i-single-softfloat.ll
@@ -0,0 +1,712 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64I %s
+
+; The test cases check that the single float arguments won't be extended
+; when passing to softfloat functions.
+; RISCV backend using shouldExtendTypeInLibCall target hook to suppress
+; the extension generation.
+
+define float @fadd_s(float %a, float %b) nounwind {
+; RV64I-LABEL: fadd_s:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    call __addsf3
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = fadd float %a, %b
+  ret float %1
+}
+
+define float @fsub_s(float %a, float %b) nounwind {
+; RV64I-LABEL: fsub_s:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    call __subsf3
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = fsub float %a, %b
+  ret float %1
+}
+
+define float @fmul_s(float %a, float %b) nounwind {
+; RV64I-LABEL: fmul_s:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    call __mulsf3
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = fmul float %a, %b
+  ret float %1
+}
+
+define float @fdiv_s(float %a, float %b) nounwind {
+; RV64I-LABEL: fdiv_s:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    call __divsf3
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = fdiv float %a, %b
+  ret float %1
+}
+
+define i32 @feq_s(float %a, float %b) nounwind {
+; RV64I-LABEL: feq_s:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    call __eqsf2
+; RV64I-NEXT:    seqz a0, a0
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = fcmp oeq float %a, %b
+  %2 = zext i1 %1 to i32
+  ret i32 %2
+}
+
+define i32 @flt_s(float %a, float %b) nounwind {
+; RV64I-LABEL: flt_s:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    call __ltsf2
+; RV64I-NEXT:    sext.w a0, a0
+; RV64I-NEXT:    slti a0, a0, 0
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = fcmp olt float %a, %b
+  %2 = zext i1 %1 to i32
+  ret i32 %2
+}
+
+define i32 @fle_s(float %a, float %b) nounwind {
+; RV64I-LABEL: fle_s:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    call __lesf2
+; RV64I-NEXT:    sext.w a0, a0
+; RV64I-NEXT:    slti a0, a0, 1
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = fcmp ole float %a, %b
+  %2 = zext i1 %1 to i32
+  ret i32 %2
+}
+
+define i32 @fcmp_ogt(float %a, float %b) nounwind {
+; RV64I-LABEL: fcmp_ogt:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    call __gtsf2
+; RV64I-NEXT:    sext.w a0, a0
+; RV64I-NEXT:    sgtz a0, a0
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = fcmp ogt float %a, %b
+  %2 = zext i1 %1 to i32
+  ret i32 %2
+}
+
+define i32 @fcmp_oge(float %a, float %b) nounwind {
+; RV64I-LABEL: fcmp_oge:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    call __gesf2
+; RV64I-NEXT:    sext.w a0, a0
+; RV64I-NEXT:    addi a1, zero, -1
+; RV64I-NEXT:    slt a0, a1, a0
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = fcmp oge float %a, %b
+  %2 = zext i1 %1 to i32
+  ret i32 %2
+}
+
+define i32 @fcmp_ord(float %a, float %b) nounwind {
+; RV64I-LABEL: fcmp_ord:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    call __unordsf2
+; RV64I-NEXT:    seqz a0, a0
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = fcmp ord float %a, %b
+  %2 = zext i1 %1 to i32
+  ret i32 %2
+}
+
+define i32 @fcmp_une(float %a, float %b) nounwind {
+; RV64I-LABEL: fcmp_une:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    call __nesf2
+; RV64I-NEXT:    snez a0, a0
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = fcmp une float %a, %b
+  %2 = zext i1 %1 to i32
+  ret i32 %2
+}
+
+define i32 @fcvt_w_s(float %a) nounwind {
+; RV64I-LABEL: fcvt_w_s:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    call __fixsfdi
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = fptosi float %a to i32
+  ret i32 %1
+}
+
+define i32 @fcvt_wu_s(float %a) nounwind {
+; RV64I-LABEL: fcvt_wu_s:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    call __fixunssfdi
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = fptoui float %a to i32
+  ret i32 %1
+}
+
+define float @fcvt_s_w(i32 %a) nounwind {
+; RV64I-LABEL: fcvt_s_w:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sext.w a0, a0
+; RV64I-NEXT:    call __floatsisf
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = sitofp i32 %a to float
+  ret float %1
+}
+
+define float @fcvt_s_wu(i32 %a) nounwind {
+; RV64I-LABEL: fcvt_s_wu:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    slli a0, a0, 32
+; RV64I-NEXT:    srli a0, a0, 32
+; RV64I-NEXT:    call __floatunsisf
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = uitofp i32 %a to float
+  ret float %1
+}
+
+define i64 @fcvt_l_s(float %a) nounwind {
+; RV64I-LABEL: fcvt_l_s:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    call __fixsfdi
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = fptosi float %a to i64
+  ret i64 %1
+}
+
+define i64 @fcvt_lu_s(float %a) nounwind {
+; RV64I-LABEL: fcvt_lu_s:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    call __fixunssfdi
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = fptoui float %a to i64
+  ret i64 %1
+}
+
+define float @fcvt_s_l(i64 %a) nounwind {
+; RV64I-LABEL: fcvt_s_l:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    call __floatdisf
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = sitofp i64 %a to float
+  ret float %1
+}
+
+define float @fcvt_s_lu(i64 %a) nounwind {
+; RV64I-LABEL: fcvt_s_lu:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    call __floatundisf
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = uitofp i64 %a to float
+  ret float %1
+}
+
+declare float @llvm.sqrt.f32(float)
+
+define float @fsqrt_s(float %a) nounwind {
+; RV64I-LABEL: fsqrt_s:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    call sqrtf
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call float @llvm.sqrt.f32(float %a)
+  ret float %1
+}
+
+declare float @llvm.copysign.f32(float, float)
+
+define float @fsgnj_s(float %a, float %b) nounwind {
+; RV64I-LABEL: fsgnj_s:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    lui a2, 524288
+; RV64I-NEXT:    and a1, a1, a2
+; RV64I-NEXT:    addiw a2, a2, -1
+; RV64I-NEXT:    and a0, a0, a2
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    ret
+  %1 = call float @llvm.copysign.f32(float %a, float %b)
+  ret float %1
+}
+
+declare float @llvm.minnum.f32(float, float)
+
+define float @fmin_s(float %a, float %b) nounwind {
+; RV64I-LABEL: fmin_s:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    call fminf
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call float @llvm.minnum.f32(float %a, float %b)
+  ret float %1
+}
+
+declare float @llvm.maxnum.f32(float, float)
+
+define float @fmax_s(float %a, float %b) nounwind {
+; RV64I-LABEL: fmax_s:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    call fmaxf
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call float @llvm.maxnum.f32(float %a, float %b)
+  ret float %1
+}
+
+
+declare float @llvm.fma.f32(float, float, float)
+
+define float @fmadd_s(float %a, float %b, float %c) nounwind {
+; RV64I-LABEL: fmadd_s:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    call fmaf
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call float @llvm.fma.f32(float %a, float %b, float %c)
+  ret float %1
+}
+
+define float @fmsub_s(float %a, float %b, float %c) nounwind {
+; RV64I-LABEL: fmsub_s:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -32
+; RV64I-NEXT:    sd ra, 24(sp)
+; RV64I-NEXT:    sd s0, 16(sp)
+; RV64I-NEXT:    sd s1, 8(sp)
+; RV64I-NEXT:    mv s0, a1
+; RV64I-NEXT:    mv s1, a0
+; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    mv a1, zero
+; RV64I-NEXT:    call __addsf3
+; RV64I-NEXT:    lui a1, 524288
+; RV64I-NEXT:    xor a2, a0, a1
+; RV64I-NEXT:    mv a0, s1
+; RV64I-NEXT:    mv a1, s0
+; RV64I-NEXT:    call fmaf
+; RV64I-NEXT:    ld s1, 8(sp)
+; RV64I-NEXT:    ld s0, 16(sp)
+; RV64I-NEXT:    ld ra, 24(sp)
+; RV64I-NEXT:    addi sp, sp, 32
+; RV64I-NEXT:    ret
+  %c_ = fadd float 0.0, %c ; avoid negation using xor
+  %negc = fsub float -0.0, %c_
+  %1 = call float @llvm.fma.f32(float %a, float %b, float %negc)
+  ret float %1
+}
+
+define float @fnmadd_s(float %a, float %b, float %c) nounwind {
+; RV64I-LABEL: fnmadd_s:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -32
+; RV64I-NEXT:    sd ra, 24(sp)
+; RV64I-NEXT:    sd s0, 16(sp)
+; RV64I-NEXT:    sd s1, 8(sp)
+; RV64I-NEXT:    sd s2, 0(sp)
+; RV64I-NEXT:    mv s0, a2
+; RV64I-NEXT:    mv s2, a1
+; RV64I-NEXT:    mv a1, zero
+; RV64I-NEXT:    call __addsf3
+; RV64I-NEXT:    mv s1, a0
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, zero
+; RV64I-NEXT:    call __addsf3
+; RV64I-NEXT:    lui a2, 524288
+; RV64I-NEXT:    xor a1, s1, a2
+; RV64I-NEXT:    xor a2, a0, a2
+; RV64I-NEXT:    mv a0, a1
+; RV64I-NEXT:    mv a1, s2
+; RV64I-NEXT:    call fmaf
+; RV64I-NEXT:    ld s2, 0(sp)
+; RV64I-NEXT:    ld s1, 8(sp)
+; RV64I-NEXT:    ld s0, 16(sp)
+; RV64I-NEXT:    ld ra, 24(sp)
+; RV64I-NEXT:    addi sp, sp, 32
+; RV64I-NEXT:    ret
+  %a_ = fadd float 0.0, %a
+  %c_ = fadd float 0.0, %c
+  %nega = fsub float -0.0, %a_
+  %negc = fsub float -0.0, %c_
+  %1 = call float @llvm.fma.f32(float %nega, float %b, float %negc)
+  ret float %1
+}
+
+define float @fnmsub_s(float %a, float %b, float %c) nounwind {
+; RV64I-LABEL: fnmsub_s:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -32
+; RV64I-NEXT:    sd ra, 24(sp)
+; RV64I-NEXT:    sd s0, 16(sp)
+; RV64I-NEXT:    sd s1, 8(sp)
+; RV64I-NEXT:    mv s0, a2
+; RV64I-NEXT:    mv s1, a1
+; RV64I-NEXT:    mv a1, zero
+; RV64I-NEXT:    call __addsf3
+; RV64I-NEXT:    lui a1, 524288
+; RV64I-NEXT:    xor a0, a0, a1
+; RV64I-NEXT:    mv a1, s1
+; RV64I-NEXT:    mv a2, s0
+; RV64I-NEXT:    call fmaf
+; RV64I-NEXT:    ld s1, 8(sp)
+; RV64I-NEXT:    ld s0, 16(sp)
+; RV64I-NEXT:    ld ra, 24(sp)
+; RV64I-NEXT:    addi sp, sp, 32
+; RV64I-NEXT:    ret
+  %a_ = fadd float 0.0, %a
+  %nega = fsub float -0.0, %a_
+  %1 = call float @llvm.fma.f32(float %nega, float %b, float %c)
+  ret float %1
+}
+
+declare float @llvm.ceil.f32(float)
+
+define float @fceil_s(float %a) nounwind {
+; RV64I-LABEL: fceil_s:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    call ceilf
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call float @llvm.ceil.f32(float %a)
+  ret float %1
+}
+
+declare float @llvm.cos.f32(float)
+
+define float @fcos_s(float %a) nounwind {
+; RV64I-LABEL: fcos_s:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    call cosf
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call float @llvm.cos.f32(float %a)
+  ret float %1
+}
+
+declare float @llvm.sin.f32(float)
+
+define float @fsin_s(float %a) nounwind {
+; RV64I-LABEL: fsin_s:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    call sinf
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call float @llvm.sin.f32(float %a)
+  ret float %1
+}
+
+declare float @llvm.exp.f32(float)
+
+define float @fexp_s(float %a) nounwind {
+; RV64I-LABEL: fexp_s:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    call expf
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call float @llvm.exp.f32(float %a)
+  ret float %1
+}
+
+declare float @llvm.exp2.f32(float)
+
+define float @fexp2_s(float %a) nounwind {
+; RV64I-LABEL: fexp2_s:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    call exp2f
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call float @llvm.exp2.f32(float %a)
+  ret float %1
+}
+
+declare float @llvm.floor.f32(float)
+
+define float @ffloor_s(float %a) nounwind {
+; RV64I-LABEL: ffloor_s:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    call floorf
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call float @llvm.floor.f32(float %a)
+  ret float %1
+}
+
+declare float @llvm.flog.f32(float)
+
+define float @fflog_s(float %a) nounwind {
+; RV64I-LABEL: fflog_s:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    call llvm.flog.f32
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call float @llvm.flog.f32(float %a)
+  ret float %1
+}
+
+declare float @llvm.flog2.f32(float)
+
+define float @fflog2_s(float %a) nounwind {
+; RV64I-LABEL: fflog2_s:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    call llvm.flog2.f32
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call float @llvm.flog2.f32(float %a)
+  ret float %1
+}
+
+declare float @llvm.flog10.f32(float)
+
+define float @fflog10_s(float %a) nounwind {
+; RV64I-LABEL: fflog10_s:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    call llvm.flog10.f32
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call float @llvm.flog10.f32(float %a)
+  ret float %1
+}
+
+declare float @llvm.fnearbyint.f32(float)
+
+define float @fnearbyint_s(float %a) nounwind {
+; RV64I-LABEL: fnearbyint_s:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    call llvm.fnearbyint.f32
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call float @llvm.fnearbyint.f32(float %a)
+  ret float %1
+}
+
+declare float @llvm.round.f32(float)
+
+define float @fround_s(float %a) nounwind {
+; RV64I-LABEL: fround_s:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    call roundf
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call float @llvm.round.f32(float %a)
+  ret float %1
+}
+
+declare float @llvm.fpround.f32(float)
+
+define float @fpround_s(float %a) nounwind {
+; RV64I-LABEL: fpround_s:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    call llvm.fpround.f32
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call float @llvm.fpround.f32(float %a)
+  ret float %1
+}
+
+declare float @llvm.rint.f32(float)
+
+define float @frint_s(float %a) nounwind {
+; RV64I-LABEL: frint_s:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    call rintf
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call float @llvm.rint.f32(float %a)
+  ret float %1
+}
+
+declare float @llvm.rem.f32(float)
+
+define float @frem_s(float %a) nounwind {
+; RV64I-LABEL: frem_s:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    call llvm.rem.f32
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call float @llvm.rem.f32(float %a)
+  ret float %1
+}
+
+declare float @llvm.pow.f32(float %Val, float %power)
+
+define float @fpow_s(float %a, float %b) nounwind {
+; RV64I-LABEL: fpow_s:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    call powf
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call float @llvm.pow.f32(float %a, float %b)
+  ret float %1
+}
+
+declare float @llvm.powi.f32(float %Val, i32 %power)
+
+define float @fpowi_s(float %a, i32 %b) nounwind {
+; RV64I-LABEL: fpowi_s:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    slli a1, a1, 32
+; RV64I-NEXT:    srli a1, a1, 32
+; RV64I-NEXT:    call __powisf2
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call float @llvm.powi.f32(float %a, i32 %b)
+  ret float %1
+}
+
+define double @fp_ext(float %a) nounwind {
+; RV64I-LABEL: fp_ext:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    call __extendsfdf2
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %conv = fpext float %a to double
+  ret double %conv
+}
+
+define float @fp_trunc(double %a) nounwind {
+; RV64I-LABEL: fp_trunc:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    call __truncdfsf2
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %conv = fptrunc double %a to float
+  ret float %conv
+}
diff --git a/llvm/test/CodeGen/RISCV/rv64i-w-insts-legalization.ll b/llvm/test/CodeGen/RISCV/rv64i-w-insts-legalization.ll
new file mode 100644
index 000000000..c4a4de768
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rv64i-w-insts-legalization.ll
@@ -0,0 +1,97 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv64 -mattr=+m -verify-machineinstrs < %s | FileCheck %s
+
+define signext i32 @addw(i32 signext %s, i32 signext %n, i32 signext %k) nounwind {
+; CHECK-LABEL: addw:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    bge a0, a1, .LBB0_2
+; CHECK-NEXT:  # %bb.1: # %for.body.preheader
+; CHECK-NEXT:    not a2, a0
+; CHECK-NEXT:    add a2, a2, a1
+; CHECK-NEXT:    addi a3, a0, 1
+; CHECK-NEXT:    mul a3, a2, a3
+; CHECK-NEXT:    slli a2, a2, 32
+; CHECK-NEXT:    srli a2, a2, 32
+; CHECK-NEXT:    sub a1, a1, a0
+; CHECK-NEXT:    addi a1, a1, -2
+; CHECK-NEXT:    slli a1, a1, 32
+; CHECK-NEXT:    srli a1, a1, 32
+; CHECK-NEXT:    mul a1, a2, a1
+; CHECK-NEXT:    srli a1, a1, 1
+; CHECK-NEXT:    add a0, a3, a0
+; CHECK-NEXT:    addw a0, a0, a1
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB0_2:
+; CHECK-NEXT:    mv a0, zero
+; CHECK-NEXT:    ret
+entry:
+  %cmp6 = icmp slt i32 %s, %n
+  br i1 %cmp6, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:                               ; preds = %entry
+  %0 = xor i32 %s, -1
+  %1 = add i32 %0, %n
+  %2 = add i32 %s, 1
+  %3 = mul i32 %1, %2
+  %4 = zext i32 %1 to i33
+  %5 = add i32 %n, -2
+  %6 = sub i32 %5, %s
+  %7 = zext i32 %6 to i33
+  %8 = mul i33 %4, %7
+  %9 = lshr i33 %8, 1
+  %10 = trunc i33 %9 to i32
+  %11 = add i32 %3, %s
+  %12 = add i32 %11, %10
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.body.preheader, %entry
+  %sum.0.lcssa = phi i32 [ 0, %entry ], [ %12, %for.body.preheader ]
+  ret i32 %sum.0.lcssa
+}
+
+define signext i32 @subw(i32 signext %s, i32 signext %n, i32 signext %k) nounwind {
+; CHECK-LABEL: subw:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    bge a0, a1, .LBB1_2
+; CHECK-NEXT:  # %bb.1: # %for.body.preheader
+; CHECK-NEXT:    not a2, a0
+; CHECK-NEXT:    add a3, a2, a1
+; CHECK-NEXT:    mul a2, a3, a2
+; CHECK-NEXT:    slli a3, a3, 32
+; CHECK-NEXT:    srli a3, a3, 32
+; CHECK-NEXT:    sub a1, a1, a0
+; CHECK-NEXT:    addi a1, a1, -2
+; CHECK-NEXT:    slli a1, a1, 32
+; CHECK-NEXT:    srli a1, a1, 32
+; CHECK-NEXT:    mul a1, a3, a1
+; CHECK-NEXT:    srli a1, a1, 1
+; CHECK-NEXT:    sub a0, a2, a0
+; CHECK-NEXT:    subw a0, a0, a1
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB1_2:
+; CHECK-NEXT:    mv a0, zero
+; CHECK-NEXT:    ret
+entry:
+  %cmp6 = icmp slt i32 %s, %n
+  br i1 %cmp6, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:                               ; preds = %entry
+  %0 = xor i32 %s, -1
+  %1 = add i32 %0, %n
+  %2 = xor i32 %s, -1
+  %3 = mul i32 %1, %2
+  %4 = zext i32 %1 to i33
+  %5 = add i32 %n, -2
+  %6 = sub i32 %5, %s
+  %7 = zext i32 %6 to i33
+  %8 = mul i33 %4, %7
+  %9 = lshr i33 %8, 1
+  %10 = trunc i33 %9 to i32
+  %11 = sub i32 %3, %s
+  %12 = sub i32 %11, %10
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.body.preheader, %entry
+  %sum.0.lcssa = phi i32 [ 0, %entry ], [ %12, %for.body.preheader ]
+  ret i32 %sum.0.lcssa
+}
diff --git a/llvm/test/CodeGen/RISCV/rv64m-exhaustive-w-insts.ll b/llvm/test/CodeGen/RISCV/rv64m-exhaustive-w-insts.ll
index 980c58139..f980c1885 100644
--- a/llvm/test/CodeGen/RISCV/rv64m-exhaustive-w-insts.ll
+++ b/llvm/test/CodeGen/RISCV/rv64m-exhaustive-w-insts.ll
@@ -9,7 +9,7 @@
 define i32 @aext_mulw_aext_aext(i32 %a, i32 %b) nounwind {
 ; RV64IM-LABEL: aext_mulw_aext_aext:
 ; RV64IM:       # %bb.0:
-; RV64IM-NEXT:    mul a0, a0, a1
+; RV64IM-NEXT:    mulw a0, a0, a1
 ; RV64IM-NEXT:    ret
   %1 = mul i32 %a, %b
   ret i32 %1
@@ -18,7 +18,7 @@ define i32 @aext_mulw_aext_aext(i32 %a, i32 %b) nounwind {
 define i32 @aext_mulw_aext_sext(i32 %a, i32 signext %b) nounwind {
 ; RV64IM-LABEL: aext_mulw_aext_sext:
 ; RV64IM:       # %bb.0:
-; RV64IM-NEXT:    mul a0, a0, a1
+; RV64IM-NEXT:    mulw a0, a0, a1
 ; RV64IM-NEXT:    ret
   %1 = mul i32 %a, %b
   ret i32 %1
@@ -27,7 +27,7 @@ define i32 @aext_mulw_aext_sext(i32 %a, i32 signext %b) nounwind {
 define i32 @aext_mulw_aext_zext(i32 %a, i32 zeroext %b) nounwind {
 ; RV64IM-LABEL: aext_mulw_aext_zext:
 ; RV64IM:       # %bb.0:
-; RV64IM-NEXT:    mul a0, a0, a1
+; RV64IM-NEXT:    mulw a0, a0, a1
 ; RV64IM-NEXT:    ret
   %1 = mul i32 %a, %b
   ret i32 %1
@@ -36,7 +36,7 @@ define i32 @aext_mulw_aext_zext(i32 %a, i32 zeroext %b) nounwind {
 define i32 @aext_mulw_sext_aext(i32 signext %a, i32 %b) nounwind {
 ; RV64IM-LABEL: aext_mulw_sext_aext:
 ; RV64IM:       # %bb.0:
-; RV64IM-NEXT:    mul a0, a0, a1
+; RV64IM-NEXT:    mulw a0, a0, a1
 ; RV64IM-NEXT:    ret
   %1 = mul i32 %a, %b
   ret i32 %1
@@ -45,7 +45,7 @@ define i32 @aext_mulw_sext_aext(i32 signext %a, i32 %b) nounwind {
 define i32 @aext_mulw_sext_sext(i32 signext %a, i32 signext %b) nounwind {
 ; RV64IM-LABEL: aext_mulw_sext_sext:
 ; RV64IM:       # %bb.0:
-; RV64IM-NEXT:    mul a0, a0, a1
+; RV64IM-NEXT:    mulw a0, a0, a1
 ; RV64IM-NEXT:    ret
   %1 = mul i32 %a, %b
   ret i32 %1
@@ -54,7 +54,7 @@ define i32 @aext_mulw_sext_sext(i32 signext %a, i32 signext %b) nounwind {
 define i32 @aext_mulw_sext_zext(i32 signext %a, i32 zeroext %b) nounwind {
 ; RV64IM-LABEL: aext_mulw_sext_zext:
 ; RV64IM:       # %bb.0:
-; RV64IM-NEXT:    mul a0, a0, a1
+; RV64IM-NEXT:    mulw a0, a0, a1
 ; RV64IM-NEXT:    ret
   %1 = mul i32 %a, %b
   ret i32 %1
@@ -63,7 +63,7 @@ define i32 @aext_mulw_sext_zext(i32 signext %a, i32 zeroext %b) nounwind {
 define i32 @aext_mulw_zext_aext(i32 zeroext %a, i32 %b) nounwind {
 ; RV64IM-LABEL: aext_mulw_zext_aext:
 ; RV64IM:       # %bb.0:
-; RV64IM-NEXT:    mul a0, a0, a1
+; RV64IM-NEXT:    mulw a0, a0, a1
 ; RV64IM-NEXT:    ret
   %1 = mul i32 %a, %b
   ret i32 %1
@@ -72,7 +72,7 @@ define i32 @aext_mulw_zext_aext(i32 zeroext %a, i32 %b) nounwind {
 define i32 @aext_mulw_zext_sext(i32 zeroext %a, i32 signext %b) nounwind {
 ; RV64IM-LABEL: aext_mulw_zext_sext:
 ; RV64IM:       # %bb.0:
-; RV64IM-NEXT:    mul a0, a0, a1
+; RV64IM-NEXT:    mulw a0, a0, a1
 ; RV64IM-NEXT:    ret
   %1 = mul i32 %a, %b
   ret i32 %1
@@ -81,7 +81,7 @@ define i32 @aext_mulw_zext_sext(i32 zeroext %a, i32 signext %b) nounwind {
 define i32 @aext_mulw_zext_zext(i32 zeroext %a, i32 zeroext %b) nounwind {
 ; RV64IM-LABEL: aext_mulw_zext_zext:
 ; RV64IM:       # %bb.0:
-; RV64IM-NEXT:    mul a0, a0, a1
+; RV64IM-NEXT:    mulw a0, a0, a1
 ; RV64IM-NEXT:    ret
   %1 = mul i32 %a, %b
   ret i32 %1
diff --git a/llvm/test/CodeGen/RISCV/rv64m-w-insts-legalization.ll b/llvm/test/CodeGen/RISCV/rv64m-w-insts-legalization.ll
new file mode 100644
index 000000000..8af41d260
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rv64m-w-insts-legalization.ll
@@ -0,0 +1,34 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv64 -mattr=+m -verify-machineinstrs < %s | FileCheck %s
+
+define signext i32 @mulw(i32 signext %s, i32 signext %n, i32 signext %k) nounwind {
+; CHECK-LABEL: mulw:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addi a2, zero, 1
+; CHECK-NEXT:    bge a0, a1, .LBB0_3
+; CHECK-NEXT:  # %bb.1: # %for.body.preheader
+; CHECK-NEXT:    addi a2, zero, 1
+; CHECK-NEXT:  .LBB0_2: # %for.body
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    mulw a2, a0, a2
+; CHECK-NEXT:    addiw a0, a0, 1
+; CHECK-NEXT:    blt a0, a1, .LBB0_2
+; CHECK-NEXT:  .LBB0_3: # %for.cond.cleanup
+; CHECK-NEXT:    mv a0, a2
+; CHECK-NEXT:    ret
+entry:
+  %cmp6 = icmp slt i32 %s, %n
+  br i1 %cmp6, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.body, %entry
+  %sum.0.lcssa = phi i32 [ 1, %entry ], [ %mul, %for.body ]
+  ret i32 %sum.0.lcssa
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.08 = phi i32 [ %inc, %for.body ], [ %s, %entry ]
+  %sum.07 = phi i32 [ %mul, %for.body ], [ 1, %entry ]
+  %mul = mul nsw i32 %i.08, %sum.07
+  %inc = add nsw i32 %i.08, 1
+  %cmp = icmp slt i32 %inc, %n
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+}
diff --git a/llvm/test/CodeGen/RISCV/select-optimize-multiple.ll b/llvm/test/CodeGen/RISCV/select-optimize-multiple.ll
index 27ca82e79..d38f3d577 100644
--- a/llvm/test/CodeGen/RISCV/select-optimize-multiple.ll
+++ b/llvm/test/CodeGen/RISCV/select-optimize-multiple.ll
@@ -40,27 +40,41 @@ define i128 @cmovcc128(i64 signext %a, i128 %b, i128 %c) nounwind {
 ; RV32I-LABEL: cmovcc128:
 ; RV32I:       # %bb.0: # %entry
 ; RV32I-NEXT:    xori a1, a1, 123
-; RV32I-NEXT:    or a1, a1, a2
-; RV32I-NEXT:    beqz a1, .LBB1_2
+; RV32I-NEXT:    or a2, a1, a2
+; RV32I-NEXT:    mv a1, a3
+; RV32I-NEXT:    beqz a2, .LBB1_2
 ; RV32I-NEXT:  # %bb.1: # %entry
+; RV32I-NEXT:    mv a1, a4
+; RV32I-NEXT:  .LBB1_2: # %entry
+; RV32I-NEXT:    lw a6, 0(a1)
+; RV32I-NEXT:    beqz a2, .LBB1_6
+; RV32I-NEXT:  # %bb.3: # %entry
 ; RV32I-NEXT:    addi a1, a4, 4
-; RV32I-NEXT:    addi a2, a4, 8
-; RV32I-NEXT:    addi a5, a4, 12
-; RV32I-NEXT:    mv a3, a4
-; RV32I-NEXT:    j .LBB1_3
-; RV32I-NEXT:  .LBB1_2:
+; RV32I-NEXT:    lw a5, 0(a1)
+; RV32I-NEXT:    bnez a2, .LBB1_7
+; RV32I-NEXT:  .LBB1_4:
+; RV32I-NEXT:    addi a1, a3, 8
+; RV32I-NEXT:    lw a1, 0(a1)
+; RV32I-NEXT:    bnez a2, .LBB1_8
+; RV32I-NEXT:  .LBB1_5:
+; RV32I-NEXT:    addi a2, a3, 12
+; RV32I-NEXT:    j .LBB1_9
+; RV32I-NEXT:  .LBB1_6:
 ; RV32I-NEXT:    addi a1, a3, 4
-; RV32I-NEXT:    addi a2, a3, 8
-; RV32I-NEXT:    addi a5, a3, 12
-; RV32I-NEXT:  .LBB1_3: # %entry
-; RV32I-NEXT:    lw a4, 0(a5)
-; RV32I-NEXT:    sw a4, 12(a0)
-; RV32I-NEXT:    lw a2, 0(a2)
-; RV32I-NEXT:    sw a2, 8(a0)
+; RV32I-NEXT:    lw a5, 0(a1)
+; RV32I-NEXT:    beqz a2, .LBB1_4
+; RV32I-NEXT:  .LBB1_7: # %entry
+; RV32I-NEXT:    addi a1, a4, 8
 ; RV32I-NEXT:    lw a1, 0(a1)
-; RV32I-NEXT:    sw a1, 4(a0)
-; RV32I-NEXT:    lw a1, 0(a3)
-; RV32I-NEXT:    sw a1, 0(a0)
+; RV32I-NEXT:    beqz a2, .LBB1_5
+; RV32I-NEXT:  .LBB1_8: # %entry
+; RV32I-NEXT:    addi a2, a4, 12
+; RV32I-NEXT:  .LBB1_9: # %entry
+; RV32I-NEXT:    lw a2, 0(a2)
+; RV32I-NEXT:    sw a2, 12(a0)
+; RV32I-NEXT:    sw a1, 8(a0)
+; RV32I-NEXT:    sw a5, 4(a0)
+; RV32I-NEXT:    sw a6, 0(a0)
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: cmovcc128:
@@ -83,24 +97,24 @@ entry:
 define i64 @cmov64(i1 %a, i64 %b, i64 %c) nounwind {
 ; RV32I-LABEL: cmov64:
 ; RV32I:       # %bb.0: # %entry
-; RV32I-NEXT:    andi a0, a0, 1
-; RV32I-NEXT:    bnez a0, .LBB2_2
+; RV32I-NEXT:    andi a5, a0, 1
+; RV32I-NEXT:    mv a0, a1
+; RV32I-NEXT:    bnez a5, .LBB2_2
 ; RV32I-NEXT:  # %bb.1: # %entry
-; RV32I-NEXT:    mv a1, a3
+; RV32I-NEXT:    mv a0, a3
 ; RV32I-NEXT:    mv a2, a4
 ; RV32I-NEXT:  .LBB2_2: # %entry
-; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    mv a1, a2
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: cmov64:
 ; RV64I:       # %bb.0: # %entry
-; RV64I-NEXT:    andi a0, a0, 1
-; RV64I-NEXT:    bnez a0, .LBB2_2
+; RV64I-NEXT:    andi a3, a0, 1
+; RV64I-NEXT:    mv a0, a1
+; RV64I-NEXT:    bnez a3, .LBB2_2
 ; RV64I-NEXT:  # %bb.1: # %entry
-; RV64I-NEXT:    mv a1, a2
+; RV64I-NEXT:    mv a0, a2
 ; RV64I-NEXT:  .LBB2_2: # %entry
-; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ret
 entry:
   %cond = select i1 %a, i64 %b, i64 %c
@@ -110,38 +124,52 @@ entry:
 define i128 @cmov128(i1 %a, i128 %b, i128 %c) nounwind {
 ; RV32I-LABEL: cmov128:
 ; RV32I:       # %bb.0: # %entry
-; RV32I-NEXT:    andi a1, a1, 1
-; RV32I-NEXT:    bnez a1, .LBB3_2
+; RV32I-NEXT:    andi a4, a1, 1
+; RV32I-NEXT:    mv a1, a2
+; RV32I-NEXT:    bnez a4, .LBB3_2
 ; RV32I-NEXT:  # %bb.1: # %entry
+; RV32I-NEXT:    mv a1, a3
+; RV32I-NEXT:  .LBB3_2: # %entry
+; RV32I-NEXT:    lw a6, 0(a1)
+; RV32I-NEXT:    bnez a4, .LBB3_6
+; RV32I-NEXT:  # %bb.3: # %entry
 ; RV32I-NEXT:    addi a1, a3, 4
-; RV32I-NEXT:    addi a4, a3, 8
-; RV32I-NEXT:    addi a5, a3, 12
-; RV32I-NEXT:    mv a2, a3
-; RV32I-NEXT:    j .LBB3_3
-; RV32I-NEXT:  .LBB3_2:
+; RV32I-NEXT:    lw a5, 0(a1)
+; RV32I-NEXT:    beqz a4, .LBB3_7
+; RV32I-NEXT:  .LBB3_4:
+; RV32I-NEXT:    addi a1, a2, 8
+; RV32I-NEXT:    lw a1, 0(a1)
+; RV32I-NEXT:    beqz a4, .LBB3_8
+; RV32I-NEXT:  .LBB3_5:
+; RV32I-NEXT:    addi a2, a2, 12
+; RV32I-NEXT:    j .LBB3_9
+; RV32I-NEXT:  .LBB3_6:
 ; RV32I-NEXT:    addi a1, a2, 4
-; RV32I-NEXT:    addi a4, a2, 8
-; RV32I-NEXT:    addi a5, a2, 12
-; RV32I-NEXT:  .LBB3_3: # %entry
-; RV32I-NEXT:    lw a3, 0(a5)
-; RV32I-NEXT:    sw a3, 12(a0)
-; RV32I-NEXT:    lw a3, 0(a4)
-; RV32I-NEXT:    sw a3, 8(a0)
+; RV32I-NEXT:    lw a5, 0(a1)
+; RV32I-NEXT:    bnez a4, .LBB3_4
+; RV32I-NEXT:  .LBB3_7: # %entry
+; RV32I-NEXT:    addi a1, a3, 8
 ; RV32I-NEXT:    lw a1, 0(a1)
-; RV32I-NEXT:    sw a1, 4(a0)
-; RV32I-NEXT:    lw a1, 0(a2)
-; RV32I-NEXT:    sw a1, 0(a0)
+; RV32I-NEXT:    bnez a4, .LBB3_5
+; RV32I-NEXT:  .LBB3_8: # %entry
+; RV32I-NEXT:    addi a2, a3, 12
+; RV32I-NEXT:  .LBB3_9: # %entry
+; RV32I-NEXT:    lw a2, 0(a2)
+; RV32I-NEXT:    sw a2, 12(a0)
+; RV32I-NEXT:    sw a1, 8(a0)
+; RV32I-NEXT:    sw a5, 4(a0)
+; RV32I-NEXT:    sw a6, 0(a0)
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: cmov128:
 ; RV64I:       # %bb.0: # %entry
-; RV64I-NEXT:    andi a0, a0, 1
-; RV64I-NEXT:    bnez a0, .LBB3_2
+; RV64I-NEXT:    andi a5, a0, 1
+; RV64I-NEXT:    mv a0, a1
+; RV64I-NEXT:    bnez a5, .LBB3_2
 ; RV64I-NEXT:  # %bb.1: # %entry
-; RV64I-NEXT:    mv a1, a3
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    mv a2, a4
 ; RV64I-NEXT:  .LBB3_2: # %entry
-; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    mv a1, a2
 ; RV64I-NEXT:    ret
 entry:
@@ -257,7 +285,7 @@ define i32 @cmovccdep(i32 signext %a, i32 %b, i32 %c, i32 %d) nounwind {
 ; RV64I-NEXT:    mv a2, a1
 ; RV64I-NEXT:    bne a0, a4, .LBB6_4
 ; RV64I-NEXT:  .LBB6_2: # %entry
-; RV64I-NEXT:    add a0, a1, a2
+; RV64I-NEXT:    addw a0, a1, a2
 ; RV64I-NEXT:    ret
 ; RV64I-NEXT:  .LBB6_3: # %entry
 ; RV64I-NEXT:    mv a1, a2
@@ -265,7 +293,7 @@ define i32 @cmovccdep(i32 signext %a, i32 %b, i32 %c, i32 %d) nounwind {
 ; RV64I-NEXT:    beq a0, a4, .LBB6_2
 ; RV64I-NEXT:  .LBB6_4: # %entry
 ; RV64I-NEXT:    mv a2, a3
-; RV64I-NEXT:    add a0, a1, a2
+; RV64I-NEXT:    addw a0, a1, a2
 ; RV64I-NEXT:    ret
 entry:
   %cmp = icmp eq i32 %a, 123
@@ -280,40 +308,38 @@ entry:
 define i32 @cmovdiffcc(i1 %a, i1 %b, i32 %c, i32 %d, i32 %e, i32 %f) nounwind {
 ; RV32I-LABEL: cmovdiffcc:
 ; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    andi a0, a0, 1
 ; RV32I-NEXT:    andi a1, a1, 1
-; RV32I-NEXT:    beqz a1, .LBB7_3
+; RV32I-NEXT:    beqz a0, .LBB7_3
 ; RV32I-NEXT:  # %bb.1: # %entry
-; RV32I-NEXT:    andi a0, a0, 1
-; RV32I-NEXT:    beqz a0, .LBB7_4
+; RV32I-NEXT:    beqz a1, .LBB7_4
 ; RV32I-NEXT:  .LBB7_2: # %entry
 ; RV32I-NEXT:    add a0, a2, a4
 ; RV32I-NEXT:    ret
 ; RV32I-NEXT:  .LBB7_3: # %entry
-; RV32I-NEXT:    mv a4, a5
-; RV32I-NEXT:    andi a0, a0, 1
-; RV32I-NEXT:    bnez a0, .LBB7_2
-; RV32I-NEXT:  .LBB7_4: # %entry
 ; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    bnez a1, .LBB7_2
+; RV32I-NEXT:  .LBB7_4: # %entry
+; RV32I-NEXT:    mv a4, a5
 ; RV32I-NEXT:    add a0, a2, a4
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: cmovdiffcc:
 ; RV64I:       # %bb.0: # %entry
+; RV64I-NEXT:    andi a0, a0, 1
 ; RV64I-NEXT:    andi a1, a1, 1
-; RV64I-NEXT:    beqz a1, .LBB7_3
+; RV64I-NEXT:    beqz a0, .LBB7_3
 ; RV64I-NEXT:  # %bb.1: # %entry
-; RV64I-NEXT:    andi a0, a0, 1
-; RV64I-NEXT:    beqz a0, .LBB7_4
+; RV64I-NEXT:    beqz a1, .LBB7_4
 ; RV64I-NEXT:  .LBB7_2: # %entry
-; RV64I-NEXT:    add a0, a2, a4
+; RV64I-NEXT:    addw a0, a2, a4
 ; RV64I-NEXT:    ret
 ; RV64I-NEXT:  .LBB7_3: # %entry
-; RV64I-NEXT:    mv a4, a5
-; RV64I-NEXT:    andi a0, a0, 1
-; RV64I-NEXT:    bnez a0, .LBB7_2
-; RV64I-NEXT:  .LBB7_4: # %entry
 ; RV64I-NEXT:    mv a2, a3
-; RV64I-NEXT:    add a0, a2, a4
+; RV64I-NEXT:    bnez a1, .LBB7_2
+; RV64I-NEXT:  .LBB7_4: # %entry
+; RV64I-NEXT:    mv a4, a5
+; RV64I-NEXT:    addw a0, a2, a4
 ; RV64I-NEXT:    ret
 entry:
   %cond1 = select i1 %a, i32 %c, i32 %d
diff --git a/llvm/test/CodeGen/RISCV/setcc-logic.ll b/llvm/test/CodeGen/RISCV/setcc-logic.ll
index 72f797db1..0c29bf505 100644
--- a/llvm/test/CodeGen/RISCV/setcc-logic.ll
+++ b/llvm/test/CodeGen/RISCV/setcc-logic.ll
@@ -7,17 +7,17 @@
 define i1 @and_icmp_eq(i32 %a, i32 %b, i32 %c, i32 %d) nounwind {
 ; RV32I-LABEL: and_icmp_eq:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    xor a2, a2, a3
 ; RV32I-NEXT:    xor a0, a0, a1
-; RV32I-NEXT:    or a0, a0, a2
+; RV32I-NEXT:    xor a1, a2, a3
+; RV32I-NEXT:    or a0, a0, a1
 ; RV32I-NEXT:    seqz a0, a0
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: and_icmp_eq:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    xor a2, a2, a3
 ; RV64I-NEXT:    xor a0, a0, a1
-; RV64I-NEXT:    or a0, a0, a2
+; RV64I-NEXT:    xor a1, a2, a3
+; RV64I-NEXT:    or a0, a0, a1
 ; RV64I-NEXT:    slli a0, a0, 32
 ; RV64I-NEXT:    srli a0, a0, 32
 ; RV64I-NEXT:    seqz a0, a0
@@ -31,17 +31,17 @@ define i1 @and_icmp_eq(i32 %a, i32 %b, i32 %c, i32 %d) nounwind {
 define i1 @or_icmp_ne(i32 %a, i32 %b, i32 %c, i32 %d) nounwind {
 ; RV32I-LABEL: or_icmp_ne:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    xor a2, a2, a3
 ; RV32I-NEXT:    xor a0, a0, a1
-; RV32I-NEXT:    or a0, a0, a2
+; RV32I-NEXT:    xor a1, a2, a3
+; RV32I-NEXT:    or a0, a0, a1
 ; RV32I-NEXT:    snez a0, a0
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: or_icmp_ne:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    xor a2, a2, a3
 ; RV64I-NEXT:    xor a0, a0, a1
-; RV64I-NEXT:    or a0, a0, a2
+; RV64I-NEXT:    xor a1, a2, a3
+; RV64I-NEXT:    or a0, a0, a1
 ; RV64I-NEXT:    slli a0, a0, 32
 ; RV64I-NEXT:    srli a0, a0, 32
 ; RV64I-NEXT:    snez a0, a0
@@ -102,22 +102,22 @@ define i1 @and_icmps_const_1bit_diff(i32 %x) nounwind {
 define i1 @and_icmps_const_not1bit_diff(i32 %x) nounwind {
 ; RV32I-LABEL: and_icmps_const_not1bit_diff:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    xori a1, a0, 92
+; RV32I-NEXT:    xori a1, a0, 44
 ; RV32I-NEXT:    snez a1, a1
-; RV32I-NEXT:    xori a0, a0, 44
+; RV32I-NEXT:    xori a0, a0, 92
 ; RV32I-NEXT:    snez a0, a0
-; RV32I-NEXT:    and a0, a0, a1
+; RV32I-NEXT:    and a0, a1, a0
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: and_icmps_const_not1bit_diff:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    slli a0, a0, 32
 ; RV64I-NEXT:    srli a0, a0, 32
-; RV64I-NEXT:    xori a1, a0, 92
+; RV64I-NEXT:    xori a1, a0, 44
 ; RV64I-NEXT:    snez a1, a1
-; RV64I-NEXT:    xori a0, a0, 44
+; RV64I-NEXT:    xori a0, a0, 92
 ; RV64I-NEXT:    snez a0, a0
-; RV64I-NEXT:    and a0, a0, a1
+; RV64I-NEXT:    and a0, a1, a0
 ; RV64I-NEXT:    ret
   %a = icmp ne i32 %x, 44
   %b = icmp ne i32 %x, 92
diff --git a/llvm/test/CodeGen/RISCV/shifts.ll b/llvm/test/CodeGen/RISCV/shifts.ll
index 80cf13614..6e1575d9d 100644
--- a/llvm/test/CodeGen/RISCV/shifts.ll
+++ b/llvm/test/CodeGen/RISCV/shifts.ll
@@ -17,11 +17,11 @@ define i64 @lshr64(i64 %a, i64 %b) nounwind {
 ; RV32I-NEXT:    mv a1, zero
 ; RV32I-NEXT:    ret
 ; RV32I-NEXT:  .LBB0_2:
+; RV32I-NEXT:    srl a0, a0, a2
 ; RV32I-NEXT:    addi a3, zero, 31
 ; RV32I-NEXT:    sub a3, a3, a2
 ; RV32I-NEXT:    slli a4, a1, 1
 ; RV32I-NEXT:    sll a3, a4, a3
-; RV32I-NEXT:    srl a0, a0, a2
 ; RV32I-NEXT:    or a0, a0, a3
 ; RV32I-NEXT:    srl a1, a1, a2
 ; RV32I-NEXT:    ret
@@ -62,11 +62,11 @@ define i64 @ashr64(i64 %a, i64 %b) nounwind {
 ; RV32I-NEXT:    srai a1, a1, 31
 ; RV32I-NEXT:    ret
 ; RV32I-NEXT:  .LBB2_2:
+; RV32I-NEXT:    srl a0, a0, a2
 ; RV32I-NEXT:    addi a3, zero, 31
 ; RV32I-NEXT:    sub a3, a3, a2
 ; RV32I-NEXT:    slli a4, a1, 1
 ; RV32I-NEXT:    sll a3, a4, a3
-; RV32I-NEXT:    srl a0, a0, a2
 ; RV32I-NEXT:    or a0, a0, a3
 ; RV32I-NEXT:    sra a1, a1, a2
 ; RV32I-NEXT:    ret
@@ -107,11 +107,11 @@ define i64 @shl64(i64 %a, i64 %b) nounwind {
 ; RV32I-NEXT:    mv a0, zero
 ; RV32I-NEXT:    ret
 ; RV32I-NEXT:  .LBB4_2:
+; RV32I-NEXT:    sll a1, a1, a2
 ; RV32I-NEXT:    addi a3, zero, 31
 ; RV32I-NEXT:    sub a3, a3, a2
 ; RV32I-NEXT:    srli a4, a0, 1
 ; RV32I-NEXT:    srl a3, a4, a3
-; RV32I-NEXT:    sll a1, a1, a2
 ; RV32I-NEXT:    or a1, a1, a3
 ; RV32I-NEXT:    sll a0, a0, a2
 ; RV32I-NEXT:    ret
@@ -148,27 +148,27 @@ define i128 @lshr128(i128 %a, i128 %b) nounwind {
 ; RV32I-NEXT:    addi sp, sp, -48
 ; RV32I-NEXT:    sw ra, 44(sp)
 ; RV32I-NEXT:    sw s0, 40(sp)
-; RV32I-NEXT:    mv s0, a0
-; RV32I-NEXT:    lw a0, 12(a1)
-; RV32I-NEXT:    sw a0, 20(sp)
-; RV32I-NEXT:    lw a0, 8(a1)
-; RV32I-NEXT:    sw a0, 16(sp)
-; RV32I-NEXT:    lw a0, 4(a1)
-; RV32I-NEXT:    sw a0, 12(sp)
-; RV32I-NEXT:    lw a0, 0(a1)
-; RV32I-NEXT:    sw a0, 8(sp)
 ; RV32I-NEXT:    lw a2, 0(a2)
+; RV32I-NEXT:    lw a3, 0(a1)
+; RV32I-NEXT:    lw a4, 4(a1)
+; RV32I-NEXT:    lw a5, 8(a1)
+; RV32I-NEXT:    lw a1, 12(a1)
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    sw a1, 20(sp)
+; RV32I-NEXT:    sw a5, 16(sp)
+; RV32I-NEXT:    sw a4, 12(sp)
 ; RV32I-NEXT:    addi a0, sp, 24
 ; RV32I-NEXT:    addi a1, sp, 8
+; RV32I-NEXT:    sw a3, 8(sp)
 ; RV32I-NEXT:    call __lshrti3
 ; RV32I-NEXT:    lw a0, 36(sp)
+; RV32I-NEXT:    lw a1, 32(sp)
+; RV32I-NEXT:    lw a2, 28(sp)
+; RV32I-NEXT:    lw a3, 24(sp)
 ; RV32I-NEXT:    sw a0, 12(s0)
-; RV32I-NEXT:    lw a0, 32(sp)
-; RV32I-NEXT:    sw a0, 8(s0)
-; RV32I-NEXT:    lw a0, 28(sp)
-; RV32I-NEXT:    sw a0, 4(s0)
-; RV32I-NEXT:    lw a0, 24(sp)
-; RV32I-NEXT:    sw a0, 0(s0)
+; RV32I-NEXT:    sw a1, 8(s0)
+; RV32I-NEXT:    sw a2, 4(s0)
+; RV32I-NEXT:    sw a3, 0(s0)
 ; RV32I-NEXT:    lw s0, 40(sp)
 ; RV32I-NEXT:    lw ra, 44(sp)
 ; RV32I-NEXT:    addi sp, sp, 48
@@ -183,11 +183,11 @@ define i128 @lshr128(i128 %a, i128 %b) nounwind {
 ; RV64I-NEXT:    mv a1, zero
 ; RV64I-NEXT:    ret
 ; RV64I-NEXT:  .LBB6_2:
+; RV64I-NEXT:    srl a0, a0, a2
 ; RV64I-NEXT:    addi a3, zero, 63
 ; RV64I-NEXT:    sub a3, a3, a2
 ; RV64I-NEXT:    slli a4, a1, 1
 ; RV64I-NEXT:    sll a3, a4, a3
-; RV64I-NEXT:    srl a0, a0, a2
 ; RV64I-NEXT:    or a0, a0, a3
 ; RV64I-NEXT:    srl a1, a1, a2
 ; RV64I-NEXT:    ret
@@ -201,27 +201,27 @@ define i128 @ashr128(i128 %a, i128 %b) nounwind {
 ; RV32I-NEXT:    addi sp, sp, -48
 ; RV32I-NEXT:    sw ra, 44(sp)
 ; RV32I-NEXT:    sw s0, 40(sp)
-; RV32I-NEXT:    mv s0, a0
-; RV32I-NEXT:    lw a0, 12(a1)
-; RV32I-NEXT:    sw a0, 20(sp)
-; RV32I-NEXT:    lw a0, 8(a1)
-; RV32I-NEXT:    sw a0, 16(sp)
-; RV32I-NEXT:    lw a0, 4(a1)
-; RV32I-NEXT:    sw a0, 12(sp)
-; RV32I-NEXT:    lw a0, 0(a1)
-; RV32I-NEXT:    sw a0, 8(sp)
 ; RV32I-NEXT:    lw a2, 0(a2)
+; RV32I-NEXT:    lw a3, 0(a1)
+; RV32I-NEXT:    lw a4, 4(a1)
+; RV32I-NEXT:    lw a5, 8(a1)
+; RV32I-NEXT:    lw a1, 12(a1)
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    sw a1, 20(sp)
+; RV32I-NEXT:    sw a5, 16(sp)
+; RV32I-NEXT:    sw a4, 12(sp)
 ; RV32I-NEXT:    addi a0, sp, 24
 ; RV32I-NEXT:    addi a1, sp, 8
+; RV32I-NEXT:    sw a3, 8(sp)
 ; RV32I-NEXT:    call __ashrti3
 ; RV32I-NEXT:    lw a0, 36(sp)
+; RV32I-NEXT:    lw a1, 32(sp)
+; RV32I-NEXT:    lw a2, 28(sp)
+; RV32I-NEXT:    lw a3, 24(sp)
 ; RV32I-NEXT:    sw a0, 12(s0)
-; RV32I-NEXT:    lw a0, 32(sp)
-; RV32I-NEXT:    sw a0, 8(s0)
-; RV32I-NEXT:    lw a0, 28(sp)
-; RV32I-NEXT:    sw a0, 4(s0)
-; RV32I-NEXT:    lw a0, 24(sp)
-; RV32I-NEXT:    sw a0, 0(s0)
+; RV32I-NEXT:    sw a1, 8(s0)
+; RV32I-NEXT:    sw a2, 4(s0)
+; RV32I-NEXT:    sw a3, 0(s0)
 ; RV32I-NEXT:    lw s0, 40(sp)
 ; RV32I-NEXT:    lw ra, 44(sp)
 ; RV32I-NEXT:    addi sp, sp, 48
@@ -236,11 +236,11 @@ define i128 @ashr128(i128 %a, i128 %b) nounwind {
 ; RV64I-NEXT:    srai a1, a1, 63
 ; RV64I-NEXT:    ret
 ; RV64I-NEXT:  .LBB7_2:
+; RV64I-NEXT:    srl a0, a0, a2
 ; RV64I-NEXT:    addi a3, zero, 63
 ; RV64I-NEXT:    sub a3, a3, a2
 ; RV64I-NEXT:    slli a4, a1, 1
 ; RV64I-NEXT:    sll a3, a4, a3
-; RV64I-NEXT:    srl a0, a0, a2
 ; RV64I-NEXT:    or a0, a0, a3
 ; RV64I-NEXT:    sra a1, a1, a2
 ; RV64I-NEXT:    ret
@@ -254,27 +254,27 @@ define i128 @shl128(i128 %a, i128 %b) nounwind {
 ; RV32I-NEXT:    addi sp, sp, -48
 ; RV32I-NEXT:    sw ra, 44(sp)
 ; RV32I-NEXT:    sw s0, 40(sp)
-; RV32I-NEXT:    mv s0, a0
-; RV32I-NEXT:    lw a0, 12(a1)
-; RV32I-NEXT:    sw a0, 20(sp)
-; RV32I-NEXT:    lw a0, 8(a1)
-; RV32I-NEXT:    sw a0, 16(sp)
-; RV32I-NEXT:    lw a0, 4(a1)
-; RV32I-NEXT:    sw a0, 12(sp)
-; RV32I-NEXT:    lw a0, 0(a1)
-; RV32I-NEXT:    sw a0, 8(sp)
 ; RV32I-NEXT:    lw a2, 0(a2)
+; RV32I-NEXT:    lw a3, 0(a1)
+; RV32I-NEXT:    lw a4, 4(a1)
+; RV32I-NEXT:    lw a5, 8(a1)
+; RV32I-NEXT:    lw a1, 12(a1)
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    sw a1, 20(sp)
+; RV32I-NEXT:    sw a5, 16(sp)
+; RV32I-NEXT:    sw a4, 12(sp)
 ; RV32I-NEXT:    addi a0, sp, 24
 ; RV32I-NEXT:    addi a1, sp, 8
+; RV32I-NEXT:    sw a3, 8(sp)
 ; RV32I-NEXT:    call __ashlti3
 ; RV32I-NEXT:    lw a0, 36(sp)
+; RV32I-NEXT:    lw a1, 32(sp)
+; RV32I-NEXT:    lw a2, 28(sp)
+; RV32I-NEXT:    lw a3, 24(sp)
 ; RV32I-NEXT:    sw a0, 12(s0)
-; RV32I-NEXT:    lw a0, 32(sp)
-; RV32I-NEXT:    sw a0, 8(s0)
-; RV32I-NEXT:    lw a0, 28(sp)
-; RV32I-NEXT:    sw a0, 4(s0)
-; RV32I-NEXT:    lw a0, 24(sp)
-; RV32I-NEXT:    sw a0, 0(s0)
+; RV32I-NEXT:    sw a1, 8(s0)
+; RV32I-NEXT:    sw a2, 4(s0)
+; RV32I-NEXT:    sw a3, 0(s0)
 ; RV32I-NEXT:    lw s0, 40(sp)
 ; RV32I-NEXT:    lw ra, 44(sp)
 ; RV32I-NEXT:    addi sp, sp, 48
@@ -289,11 +289,11 @@ define i128 @shl128(i128 %a, i128 %b) nounwind {
 ; RV64I-NEXT:    mv a0, zero
 ; RV64I-NEXT:    ret
 ; RV64I-NEXT:  .LBB8_2:
+; RV64I-NEXT:    sll a1, a1, a2
 ; RV64I-NEXT:    addi a3, zero, 63
 ; RV64I-NEXT:    sub a3, a3, a2
 ; RV64I-NEXT:    srli a4, a0, 1
 ; RV64I-NEXT:    srl a3, a4, a3
-; RV64I-NEXT:    sll a1, a1, a2
 ; RV64I-NEXT:    or a1, a1, a3
 ; RV64I-NEXT:    sll a0, a0, a2
 ; RV64I-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/shrinkwrap.ll b/llvm/test/CodeGen/RISCV/shrinkwrap.ll
new file mode 100644
index 000000000..88ff585a7
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/shrinkwrap.ll
@@ -0,0 +1,97 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple riscv32 < %s | FileCheck %s -check-prefix=RV32I-NOSW
+; RUN: llc -mtriple riscv32 -enable-shrink-wrap < %s | FileCheck %s -check-prefix=RV32I-SW
+
+
+declare void @abort()
+
+define void @eliminate_restore(i32 %n) nounwind {
+; RV32I-NOSW-LABEL: eliminate_restore:
+; RV32I-NOSW:       # %bb.0:
+; RV32I-NOSW-NEXT:    addi sp, sp, -16
+; RV32I-NOSW-NEXT:    sw ra, 12(sp)
+; RV32I-NOSW-NEXT:    addi a1, zero, 32
+; RV32I-NOSW-NEXT:    bgeu a1, a0, .LBB0_2
+; RV32I-NOSW-NEXT:  # %bb.1: # %if.end
+; RV32I-NOSW-NEXT:    lw ra, 12(sp)
+; RV32I-NOSW-NEXT:    addi sp, sp, 16
+; RV32I-NOSW-NEXT:    ret
+; RV32I-NOSW-NEXT:  .LBB0_2: # %if.then
+; RV32I-NOSW-NEXT:    call abort
+;
+; RV32I-SW-LABEL: eliminate_restore:
+; RV32I-SW:       # %bb.0:
+; RV32I-SW-NEXT:    addi a1, zero, 32
+; RV32I-SW-NEXT:    bgeu a1, a0, .LBB0_2
+; RV32I-SW-NEXT:  # %bb.1: # %if.end
+; RV32I-SW-NEXT:    ret
+; RV32I-SW-NEXT:  .LBB0_2: # %if.then
+; RV32I-SW-NEXT:    addi sp, sp, -16
+; RV32I-SW-NEXT:    sw ra, 12(sp)
+; RV32I-SW-NEXT:    call abort
+  %cmp = icmp ule i32 %n, 32
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  call void @abort()
+  unreachable
+
+if.end:
+  ret void
+}
+
+declare void @notdead(i8*)
+
+define void @conditional_alloca(i32 %n) nounwind {
+; RV32I-NOSW-LABEL: conditional_alloca:
+; RV32I-NOSW:       # %bb.0:
+; RV32I-NOSW-NEXT:    addi sp, sp, -16
+; RV32I-NOSW-NEXT:    sw ra, 12(sp)
+; RV32I-NOSW-NEXT:    sw s0, 8(sp)
+; RV32I-NOSW-NEXT:    addi s0, sp, 16
+; RV32I-NOSW-NEXT:    addi a1, zero, 32
+; RV32I-NOSW-NEXT:    bltu a1, a0, .LBB1_2
+; RV32I-NOSW-NEXT:  # %bb.1: # %if.then
+; RV32I-NOSW-NEXT:    addi a0, a0, 15
+; RV32I-NOSW-NEXT:    andi a0, a0, -16
+; RV32I-NOSW-NEXT:    sub a0, sp, a0
+; RV32I-NOSW-NEXT:    mv sp, a0
+; RV32I-NOSW-NEXT:    call notdead
+; RV32I-NOSW-NEXT:  .LBB1_2: # %if.end
+; RV32I-NOSW-NEXT:    addi sp, s0, -16
+; RV32I-NOSW-NEXT:    lw s0, 8(sp)
+; RV32I-NOSW-NEXT:    lw ra, 12(sp)
+; RV32I-NOSW-NEXT:    addi sp, sp, 16
+; RV32I-NOSW-NEXT:    ret
+;
+; RV32I-SW-LABEL: conditional_alloca:
+; RV32I-SW:       # %bb.0:
+; RV32I-SW-NEXT:    addi a1, zero, 32
+; RV32I-SW-NEXT:    bltu a1, a0, .LBB1_2
+; RV32I-SW-NEXT:  # %bb.1: # %if.then
+; RV32I-SW-NEXT:    addi sp, sp, -16
+; RV32I-SW-NEXT:    sw ra, 12(sp)
+; RV32I-SW-NEXT:    sw s0, 8(sp)
+; RV32I-SW-NEXT:    addi s0, sp, 16
+; RV32I-SW-NEXT:    addi a0, a0, 15
+; RV32I-SW-NEXT:    andi a0, a0, -16
+; RV32I-SW-NEXT:    sub a0, sp, a0
+; RV32I-SW-NEXT:    mv sp, a0
+; RV32I-SW-NEXT:    call notdead
+; RV32I-SW-NEXT:    addi sp, s0, -16
+; RV32I-SW-NEXT:    lw s0, 8(sp)
+; RV32I-SW-NEXT:    lw ra, 12(sp)
+; RV32I-SW-NEXT:    addi sp, sp, 16
+; RV32I-SW-NEXT:  .LBB1_2: # %if.end
+; RV32I-SW-NEXT:    ret
+  %cmp = icmp ule i32 %n, 32
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  %addr = alloca i8, i32 %n
+  call void @notdead(i8* %addr)
+  br label %if.end
+
+if.end:
+  ret void
+}
diff --git a/llvm/test/CodeGen/RISCV/split-offsets.ll b/llvm/test/CodeGen/RISCV/split-offsets.ll
index 731f6cacd..2d943ae71 100644
--- a/llvm/test/CodeGen/RISCV/split-offsets.ll
+++ b/llvm/test/CodeGen/RISCV/split-offsets.ll
@@ -11,34 +11,32 @@
 define void @test1([65536 x i32]** %sp, [65536 x i32]* %t, i32 %n) {
 ; RV32I-LABEL: test1:
 ; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    lw a0, 0(a0)
 ; RV32I-NEXT:    lui a2, 20
 ; RV32I-NEXT:    addi a2, a2, -1920
-; RV32I-NEXT:    lw a0, 0(a0)
+; RV32I-NEXT:    add a1, a1, a2
 ; RV32I-NEXT:    add a0, a0, a2
+; RV32I-NEXT:    addi a2, zero, 2
+; RV32I-NEXT:    sw a2, 0(a0)
 ; RV32I-NEXT:    addi a3, zero, 1
 ; RV32I-NEXT:    sw a3, 4(a0)
-; RV32I-NEXT:    addi a4, zero, 2
-; RV32I-NEXT:    sw a4, 0(a0)
-; RV32I-NEXT:    add a0, a1, a2
-; RV32I-NEXT:    sw a4, 4(a0)
-; RV32I-NEXT:    sw a3, 0(a0)
-; RV32I-NEXT:    .cfi_def_cfa_offset 0
+; RV32I-NEXT:    sw a3, 0(a1)
+; RV32I-NEXT:    sw a2, 4(a1)
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: test1:
 ; RV64I:       # %bb.0: # %entry
+; RV64I-NEXT:    ld a0, 0(a0)
 ; RV64I-NEXT:    lui a2, 20
 ; RV64I-NEXT:    addiw a2, a2, -1920
-; RV64I-NEXT:    ld a0, 0(a0)
+; RV64I-NEXT:    add a1, a1, a2
 ; RV64I-NEXT:    add a0, a0, a2
+; RV64I-NEXT:    addi a2, zero, 2
+; RV64I-NEXT:    sw a2, 0(a0)
 ; RV64I-NEXT:    addi a3, zero, 1
 ; RV64I-NEXT:    sw a3, 4(a0)
-; RV64I-NEXT:    addi a4, zero, 2
-; RV64I-NEXT:    sw a4, 0(a0)
-; RV64I-NEXT:    add a0, a1, a2
-; RV64I-NEXT:    sw a4, 4(a0)
-; RV64I-NEXT:    sw a3, 0(a0)
-; RV64I-NEXT:    .cfi_def_cfa_offset 0
+; RV64I-NEXT:    sw a3, 0(a1)
+; RV64I-NEXT:    sw a2, 4(a1)
 ; RV64I-NEXT:    ret
 entry:
   %s = load [65536 x i32]*, [65536 x i32]** %sp
@@ -57,49 +55,47 @@ entry:
 define void @test2([65536 x i32]** %sp, [65536 x i32]* %t, i32 %n) {
 ; RV32I-LABEL: test2:
 ; RV32I:       # %bb.0: # %entry
-; RV32I-NEXT:    lui a3, 20
-; RV32I-NEXT:    addi a3, a3, -1920
-; RV32I-NEXT:    lw a0, 0(a0)
-; RV32I-NEXT:    add a0, a0, a3
-; RV32I-NEXT:    add a1, a1, a3
 ; RV32I-NEXT:    mv a3, zero
+; RV32I-NEXT:    lw a4, 0(a0)
+; RV32I-NEXT:    lui a0, 20
+; RV32I-NEXT:    addi a5, a0, -1920
+; RV32I-NEXT:    add a0, a1, a5
+; RV32I-NEXT:    add a1, a4, a5
 ; RV32I-NEXT:    bge a3, a2, .LBB1_2
 ; RV32I-NEXT:  .LBB1_1: # %while_body
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    sw a3, 4(a0)
 ; RV32I-NEXT:    addi a4, a3, 1
-; RV32I-NEXT:    sw a4, 0(a0)
-; RV32I-NEXT:    sw a3, 4(a1)
 ; RV32I-NEXT:    sw a4, 0(a1)
+; RV32I-NEXT:    sw a3, 4(a1)
+; RV32I-NEXT:    sw a4, 0(a0)
+; RV32I-NEXT:    sw a3, 4(a0)
 ; RV32I-NEXT:    mv a3, a4
 ; RV32I-NEXT:    blt a3, a2, .LBB1_1
 ; RV32I-NEXT:  .LBB1_2: # %while_end
-; RV32I-NEXT:    .cfi_def_cfa_offset 0
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: test2:
 ; RV64I:       # %bb.0: # %entry
-; RV64I-NEXT:    lui a3, 20
-; RV64I-NEXT:    addiw a3, a3, -1920
-; RV64I-NEXT:    ld a0, 0(a0)
-; RV64I-NEXT:    add a0, a0, a3
-; RV64I-NEXT:    add a1, a1, a3
-; RV64I-NEXT:    sext.w a2, a2
 ; RV64I-NEXT:    mv a3, zero
+; RV64I-NEXT:    ld a4, 0(a0)
+; RV64I-NEXT:    lui a0, 20
+; RV64I-NEXT:    addiw a5, a0, -1920
+; RV64I-NEXT:    add a0, a1, a5
+; RV64I-NEXT:    add a1, a4, a5
+; RV64I-NEXT:    sext.w a2, a2
 ; RV64I-NEXT:    sext.w a4, a3
 ; RV64I-NEXT:    bge a4, a2, .LBB1_2
 ; RV64I-NEXT:  .LBB1_1: # %while_body
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    sw a3, 4(a0)
 ; RV64I-NEXT:    addi a4, a3, 1
-; RV64I-NEXT:    sw a4, 0(a0)
-; RV64I-NEXT:    sw a3, 4(a1)
 ; RV64I-NEXT:    sw a4, 0(a1)
+; RV64I-NEXT:    sw a3, 4(a1)
+; RV64I-NEXT:    sw a4, 0(a0)
+; RV64I-NEXT:    sw a3, 4(a0)
 ; RV64I-NEXT:    mv a3, a4
 ; RV64I-NEXT:    sext.w a4, a3
 ; RV64I-NEXT:    blt a4, a2, .LBB1_1
 ; RV64I-NEXT:  .LBB1_2: # %while_end
-; RV64I-NEXT:    .cfi_def_cfa_offset 0
 ; RV64I-NEXT:    ret
 entry:
   %s = load [65536 x i32]*, [65536 x i32]** %sp
diff --git a/llvm/test/CodeGen/RISCV/split-sp-adjust.ll b/llvm/test/CodeGen/RISCV/split-sp-adjust.ll
new file mode 100644
index 000000000..49f8e7010
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/split-sp-adjust.ll
@@ -0,0 +1,45 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefix=RV32I
+
+; The stack size is 2048 and the SP adjustment will be split.
+define i32 @SplitSP() nounwind {
+; RV32I-LABEL: SplitSP:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    addi sp, sp, -2032
+; RV32I-NEXT:    sw ra, 2028(sp)
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    addi a0, sp, 16
+; RV32I-NEXT:    call foo
+; RV32I-NEXT:    mv a0, zero
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    lw ra, 2028(sp)
+; RV32I-NEXT:    addi sp, sp, 2032
+; RV32I-NEXT:    ret
+entry:
+  %xx = alloca [2028 x i8], align 1
+  %0 = getelementptr inbounds [2028 x i8], [2028 x i8]* %xx, i32 0, i32 0
+  %call = call i32 @foo(i8* nonnull %0)
+  ret i32 0
+}
+
+; The stack size is 2032 and the SP adjustment will not be split.
+define i32 @NoSplitSP() nounwind {
+; RV32I-LABEL: NoSplitSP:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    addi sp, sp, -2032
+; RV32I-NEXT:    sw ra, 2028(sp)
+; RV32I-NEXT:    addi a0, sp, 4
+; RV32I-NEXT:    call foo
+; RV32I-NEXT:    mv a0, zero
+; RV32I-NEXT:    lw ra, 2028(sp)
+; RV32I-NEXT:    addi sp, sp, 2032
+; RV32I-NEXT:    ret
+entry:
+  %xx = alloca [2024 x i8], align 1
+  %0 = getelementptr inbounds [2024 x i8], [2024 x i8]* %xx, i32 0, i32 0
+  %call = call i32 @foo(i8* nonnull %0)
+  ret i32 0
+}
+
+declare i32 @foo(i8*)
diff --git a/llvm/test/CodeGen/RISCV/srem-lkk.ll b/llvm/test/CodeGen/RISCV/srem-lkk.ll
new file mode 100644
index 000000000..477320f2e
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/srem-lkk.ll
@@ -0,0 +1,503 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefixes=CHECK,RV32I %s
+; RUN: llc -mtriple=riscv32 -mattr=+m -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefixes=CHECK,RV32IM %s
+; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefixes=CHECK,RV64I %s
+; RUN: llc -mtriple=riscv64 -mattr=+m -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefixes=CHECK,RV64IM %s
+
+define i32 @fold_srem_positive_odd(i32 %x) nounwind {
+; RV32I-LABEL: fold_srem_positive_odd:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    addi a1, zero, 95
+; RV32I-NEXT:    call __modsi3
+; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV32IM-LABEL: fold_srem_positive_odd:
+; RV32IM:       # %bb.0:
+; RV32IM-NEXT:    lui a1, 706409
+; RV32IM-NEXT:    addi a1, a1, 389
+; RV32IM-NEXT:    mulh a1, a0, a1
+; RV32IM-NEXT:    add a1, a1, a0
+; RV32IM-NEXT:    srli a2, a1, 31
+; RV32IM-NEXT:    srai a1, a1, 6
+; RV32IM-NEXT:    add a1, a1, a2
+; RV32IM-NEXT:    addi a2, zero, 95
+; RV32IM-NEXT:    mul a1, a1, a2
+; RV32IM-NEXT:    sub a0, a0, a1
+; RV32IM-NEXT:    ret
+;
+; RV64I-LABEL: fold_srem_positive_odd:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sext.w a0, a0
+; RV64I-NEXT:    addi a1, zero, 95
+; RV64I-NEXT:    call __moddi3
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IM-LABEL: fold_srem_positive_odd:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    sext.w a0, a0
+; RV64IM-NEXT:    lui a1, 1045903
+; RV64IM-NEXT:    addiw a1, a1, -733
+; RV64IM-NEXT:    slli a1, a1, 15
+; RV64IM-NEXT:    addi a1, a1, 1035
+; RV64IM-NEXT:    slli a1, a1, 12
+; RV64IM-NEXT:    addi a1, a1, -905
+; RV64IM-NEXT:    slli a1, a1, 12
+; RV64IM-NEXT:    addi a1, a1, -1767
+; RV64IM-NEXT:    mulh a1, a0, a1
+; RV64IM-NEXT:    add a1, a1, a0
+; RV64IM-NEXT:    srli a2, a1, 63
+; RV64IM-NEXT:    srai a1, a1, 6
+; RV64IM-NEXT:    add a1, a1, a2
+; RV64IM-NEXT:    addi a2, zero, 95
+; RV64IM-NEXT:    mul a1, a1, a2
+; RV64IM-NEXT:    sub a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = srem i32 %x, 95
+  ret i32 %1
+}
+
+
+define i32 @fold_srem_positive_even(i32 %x) nounwind {
+; RV32I-LABEL: fold_srem_positive_even:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    addi a1, zero, 1060
+; RV32I-NEXT:    call __modsi3
+; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV32IM-LABEL: fold_srem_positive_even:
+; RV32IM:       # %bb.0:
+; RV32IM-NEXT:    lui a1, 253241
+; RV32IM-NEXT:    addi a1, a1, -15
+; RV32IM-NEXT:    mulh a1, a0, a1
+; RV32IM-NEXT:    srli a2, a1, 31
+; RV32IM-NEXT:    srai a1, a1, 8
+; RV32IM-NEXT:    add a1, a1, a2
+; RV32IM-NEXT:    addi a2, zero, 1060
+; RV32IM-NEXT:    mul a1, a1, a2
+; RV32IM-NEXT:    sub a0, a0, a1
+; RV32IM-NEXT:    ret
+;
+; RV64I-LABEL: fold_srem_positive_even:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sext.w a0, a0
+; RV64I-NEXT:    addi a1, zero, 1060
+; RV64I-NEXT:    call __moddi3
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IM-LABEL: fold_srem_positive_even:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    sext.w a0, a0
+; RV64IM-NEXT:    lui a1, 506482
+; RV64IM-NEXT:    addiw a1, a1, -31
+; RV64IM-NEXT:    slli a1, a1, 13
+; RV64IM-NEXT:    addi a1, a1, 711
+; RV64IM-NEXT:    slli a1, a1, 19
+; RV64IM-NEXT:    addi a1, a1, 1979
+; RV64IM-NEXT:    mulh a1, a0, a1
+; RV64IM-NEXT:    srli a2, a1, 63
+; RV64IM-NEXT:    srai a1, a1, 9
+; RV64IM-NEXT:    add a1, a1, a2
+; RV64IM-NEXT:    addi a2, zero, 1060
+; RV64IM-NEXT:    mul a1, a1, a2
+; RV64IM-NEXT:    sub a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = srem i32 %x, 1060
+  ret i32 %1
+}
+
+
+define i32 @fold_srem_negative_odd(i32 %x) nounwind {
+; RV32I-LABEL: fold_srem_negative_odd:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    addi a1, zero, -723
+; RV32I-NEXT:    call __modsi3
+; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV32IM-LABEL: fold_srem_negative_odd:
+; RV32IM:       # %bb.0:
+; RV32IM-NEXT:    lui a1, 677296
+; RV32IM-NEXT:    addi a1, a1, -91
+; RV32IM-NEXT:    mulh a1, a0, a1
+; RV32IM-NEXT:    srli a2, a1, 31
+; RV32IM-NEXT:    srai a1, a1, 8
+; RV32IM-NEXT:    add a1, a1, a2
+; RV32IM-NEXT:    addi a2, zero, -723
+; RV32IM-NEXT:    mul a1, a1, a2
+; RV32IM-NEXT:    sub a0, a0, a1
+; RV32IM-NEXT:    ret
+;
+; RV64I-LABEL: fold_srem_negative_odd:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sext.w a0, a0
+; RV64I-NEXT:    addi a1, zero, -723
+; RV64I-NEXT:    call __moddi3
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IM-LABEL: fold_srem_negative_odd:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    sext.w a0, a0
+; RV64IM-NEXT:    lui a1, 4781
+; RV64IM-NEXT:    addiw a1, a1, 2045
+; RV64IM-NEXT:    slli a1, a1, 13
+; RV64IM-NEXT:    addi a1, a1, 1371
+; RV64IM-NEXT:    slli a1, a1, 13
+; RV64IM-NEXT:    addi a1, a1, -11
+; RV64IM-NEXT:    slli a1, a1, 12
+; RV64IM-NEXT:    addi a1, a1, -1355
+; RV64IM-NEXT:    mulh a1, a0, a1
+; RV64IM-NEXT:    sub a1, a1, a0
+; RV64IM-NEXT:    srli a2, a1, 63
+; RV64IM-NEXT:    srai a1, a1, 9
+; RV64IM-NEXT:    add a1, a1, a2
+; RV64IM-NEXT:    addi a2, zero, -723
+; RV64IM-NEXT:    mul a1, a1, a2
+; RV64IM-NEXT:    sub a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = srem i32 %x, -723
+  ret i32 %1
+}
+
+
+define i32 @fold_srem_negative_even(i32 %x) nounwind {
+; RV32I-LABEL: fold_srem_negative_even:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    lui a1, 1048570
+; RV32I-NEXT:    addi a1, a1, 1595
+; RV32I-NEXT:    call __modsi3
+; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV32IM-LABEL: fold_srem_negative_even:
+; RV32IM:       # %bb.0:
+; RV32IM-NEXT:    lui a1, 1036895
+; RV32IM-NEXT:    addi a1, a1, 999
+; RV32IM-NEXT:    mulh a1, a0, a1
+; RV32IM-NEXT:    srli a2, a1, 31
+; RV32IM-NEXT:    srai a1, a1, 8
+; RV32IM-NEXT:    add a1, a1, a2
+; RV32IM-NEXT:    lui a2, 1048570
+; RV32IM-NEXT:    addi a2, a2, 1595
+; RV32IM-NEXT:    mul a1, a1, a2
+; RV32IM-NEXT:    sub a0, a0, a1
+; RV32IM-NEXT:    ret
+;
+; RV64I-LABEL: fold_srem_negative_even:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sext.w a0, a0
+; RV64I-NEXT:    lui a1, 1048570
+; RV64I-NEXT:    addiw a1, a1, 1595
+; RV64I-NEXT:    call __moddi3
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IM-LABEL: fold_srem_negative_even:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    sext.w a0, a0
+; RV64IM-NEXT:    lui a1, 1036895
+; RV64IM-NEXT:    addiw a1, a1, 999
+; RV64IM-NEXT:    slli a1, a1, 12
+; RV64IM-NEXT:    addi a1, a1, 11
+; RV64IM-NEXT:    slli a1, a1, 12
+; RV64IM-NEXT:    addi a1, a1, -523
+; RV64IM-NEXT:    slli a1, a1, 12
+; RV64IM-NEXT:    addi a1, a1, -481
+; RV64IM-NEXT:    mulh a1, a0, a1
+; RV64IM-NEXT:    srli a2, a1, 63
+; RV64IM-NEXT:    srai a1, a1, 12
+; RV64IM-NEXT:    add a1, a1, a2
+; RV64IM-NEXT:    lui a2, 1048570
+; RV64IM-NEXT:    addiw a2, a2, 1595
+; RV64IM-NEXT:    mul a1, a1, a2
+; RV64IM-NEXT:    sub a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = srem i32 %x, -22981
+  ret i32 %1
+}
+
+
+; Don't fold if we can combine srem with sdiv.
+define i32 @combine_srem_sdiv(i32 %x) nounwind {
+; RV32I-LABEL: combine_srem_sdiv:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw s0, 8(sp)
+; RV32I-NEXT:    sw s1, 4(sp)
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    addi a1, zero, 95
+; RV32I-NEXT:    call __modsi3
+; RV32I-NEXT:    mv s1, a0
+; RV32I-NEXT:    addi a1, zero, 95
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    call __divsi3
+; RV32I-NEXT:    add a0, s1, a0
+; RV32I-NEXT:    lw s1, 4(sp)
+; RV32I-NEXT:    lw s0, 8(sp)
+; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV32IM-LABEL: combine_srem_sdiv:
+; RV32IM:       # %bb.0:
+; RV32IM-NEXT:    lui a1, 706409
+; RV32IM-NEXT:    addi a1, a1, 389
+; RV32IM-NEXT:    mulh a1, a0, a1
+; RV32IM-NEXT:    add a1, a1, a0
+; RV32IM-NEXT:    srli a2, a1, 31
+; RV32IM-NEXT:    srai a1, a1, 6
+; RV32IM-NEXT:    add a1, a1, a2
+; RV32IM-NEXT:    addi a2, zero, 95
+; RV32IM-NEXT:    mul a2, a1, a2
+; RV32IM-NEXT:    sub a0, a0, a2
+; RV32IM-NEXT:    add a0, a0, a1
+; RV32IM-NEXT:    ret
+;
+; RV64I-LABEL: combine_srem_sdiv:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -32
+; RV64I-NEXT:    sd ra, 24(sp)
+; RV64I-NEXT:    sd s0, 16(sp)
+; RV64I-NEXT:    sd s1, 8(sp)
+; RV64I-NEXT:    sext.w s0, a0
+; RV64I-NEXT:    addi a1, zero, 95
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    call __moddi3
+; RV64I-NEXT:    mv s1, a0
+; RV64I-NEXT:    addi a1, zero, 95
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    call __divdi3
+; RV64I-NEXT:    addw a0, s1, a0
+; RV64I-NEXT:    ld s1, 8(sp)
+; RV64I-NEXT:    ld s0, 16(sp)
+; RV64I-NEXT:    ld ra, 24(sp)
+; RV64I-NEXT:    addi sp, sp, 32
+; RV64I-NEXT:    ret
+;
+; RV64IM-LABEL: combine_srem_sdiv:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    sext.w a1, a0
+; RV64IM-NEXT:    lui a2, 1045903
+; RV64IM-NEXT:    addiw a2, a2, -733
+; RV64IM-NEXT:    slli a2, a2, 15
+; RV64IM-NEXT:    addi a2, a2, 1035
+; RV64IM-NEXT:    slli a2, a2, 12
+; RV64IM-NEXT:    addi a2, a2, -905
+; RV64IM-NEXT:    slli a2, a2, 12
+; RV64IM-NEXT:    addi a2, a2, -1767
+; RV64IM-NEXT:    mulh a2, a1, a2
+; RV64IM-NEXT:    add a1, a2, a1
+; RV64IM-NEXT:    srli a2, a1, 63
+; RV64IM-NEXT:    srai a1, a1, 6
+; RV64IM-NEXT:    add a1, a1, a2
+; RV64IM-NEXT:    addi a2, zero, 95
+; RV64IM-NEXT:    mul a2, a1, a2
+; RV64IM-NEXT:    sub a0, a0, a2
+; RV64IM-NEXT:    addw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = srem i32 %x, 95
+  %2 = sdiv i32 %x, 95
+  %3 = add i32 %1, %2
+  ret i32 %3
+}
+
+; Don't fold for divisors that are a power of two.
+define i32 @dont_fold_srem_power_of_two(i32 %x) nounwind {
+; RV32I-LABEL: dont_fold_srem_power_of_two:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    srai a1, a0, 31
+; RV32I-NEXT:    srli a1, a1, 26
+; RV32I-NEXT:    add a1, a0, a1
+; RV32I-NEXT:    andi a1, a1, -64
+; RV32I-NEXT:    sub a0, a0, a1
+; RV32I-NEXT:    ret
+;
+; RV32IM-LABEL: dont_fold_srem_power_of_two:
+; RV32IM:       # %bb.0:
+; RV32IM-NEXT:    srai a1, a0, 31
+; RV32IM-NEXT:    srli a1, a1, 26
+; RV32IM-NEXT:    add a1, a0, a1
+; RV32IM-NEXT:    andi a1, a1, -64
+; RV32IM-NEXT:    sub a0, a0, a1
+; RV32IM-NEXT:    ret
+;
+; RV64I-LABEL: dont_fold_srem_power_of_two:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    sext.w a1, a0
+; RV64I-NEXT:    srli a1, a1, 57
+; RV64I-NEXT:    andi a1, a1, 63
+; RV64I-NEXT:    add a1, a0, a1
+; RV64I-NEXT:    addi a2, zero, 1
+; RV64I-NEXT:    slli a2, a2, 32
+; RV64I-NEXT:    addi a2, a2, -64
+; RV64I-NEXT:    and a1, a1, a2
+; RV64I-NEXT:    subw a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64IM-LABEL: dont_fold_srem_power_of_two:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    sext.w a1, a0
+; RV64IM-NEXT:    srli a1, a1, 57
+; RV64IM-NEXT:    andi a1, a1, 63
+; RV64IM-NEXT:    add a1, a0, a1
+; RV64IM-NEXT:    addi a2, zero, 1
+; RV64IM-NEXT:    slli a2, a2, 32
+; RV64IM-NEXT:    addi a2, a2, -64
+; RV64IM-NEXT:    and a1, a1, a2
+; RV64IM-NEXT:    subw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = srem i32 %x, 64
+  ret i32 %1
+}
+
+; Don't fold if the divisor is one.
+define i32 @dont_fold_srem_one(i32 %x) nounwind {
+; CHECK-LABEL: dont_fold_srem_one:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mv a0, zero
+; CHECK-NEXT:    ret
+  %1 = srem i32 %x, 1
+  ret i32 %1
+}
+
+; Don't fold if the divisor is 2^31.
+define i32 @dont_fold_srem_i32_smax(i32 %x) nounwind {
+; RV32I-LABEL: dont_fold_srem_i32_smax:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    srai a1, a0, 31
+; RV32I-NEXT:    srli a1, a1, 1
+; RV32I-NEXT:    add a1, a0, a1
+; RV32I-NEXT:    lui a2, 524288
+; RV32I-NEXT:    and a1, a1, a2
+; RV32I-NEXT:    add a0, a0, a1
+; RV32I-NEXT:    ret
+;
+; RV32IM-LABEL: dont_fold_srem_i32_smax:
+; RV32IM:       # %bb.0:
+; RV32IM-NEXT:    srai a1, a0, 31
+; RV32IM-NEXT:    srli a1, a1, 1
+; RV32IM-NEXT:    add a1, a0, a1
+; RV32IM-NEXT:    lui a2, 524288
+; RV32IM-NEXT:    and a1, a1, a2
+; RV32IM-NEXT:    add a0, a0, a1
+; RV32IM-NEXT:    ret
+;
+; RV64I-LABEL: dont_fold_srem_i32_smax:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    sext.w a1, a0
+; RV64I-NEXT:    srli a1, a1, 32
+; RV64I-NEXT:    lui a2, 524288
+; RV64I-NEXT:    addiw a2, a2, -1
+; RV64I-NEXT:    and a1, a1, a2
+; RV64I-NEXT:    add a1, a0, a1
+; RV64I-NEXT:    addi a2, zero, 1
+; RV64I-NEXT:    slli a2, a2, 31
+; RV64I-NEXT:    and a1, a1, a2
+; RV64I-NEXT:    addw a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64IM-LABEL: dont_fold_srem_i32_smax:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    sext.w a1, a0
+; RV64IM-NEXT:    srli a1, a1, 32
+; RV64IM-NEXT:    lui a2, 524288
+; RV64IM-NEXT:    addiw a2, a2, -1
+; RV64IM-NEXT:    and a1, a1, a2
+; RV64IM-NEXT:    add a1, a0, a1
+; RV64IM-NEXT:    addi a2, zero, 1
+; RV64IM-NEXT:    slli a2, a2, 31
+; RV64IM-NEXT:    and a1, a1, a2
+; RV64IM-NEXT:    addw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = srem i32 %x, 2147483648
+  ret i32 %1
+}
+
+; Don't fold i64 srem
+define i64 @dont_fold_srem_i64(i64 %x) nounwind {
+; RV32I-LABEL: dont_fold_srem_i64:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    addi a2, zero, 98
+; RV32I-NEXT:    mv a3, zero
+; RV32I-NEXT:    call __moddi3
+; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV32IM-LABEL: dont_fold_srem_i64:
+; RV32IM:       # %bb.0:
+; RV32IM-NEXT:    addi sp, sp, -16
+; RV32IM-NEXT:    sw ra, 12(sp)
+; RV32IM-NEXT:    addi a2, zero, 98
+; RV32IM-NEXT:    mv a3, zero
+; RV32IM-NEXT:    call __moddi3
+; RV32IM-NEXT:    lw ra, 12(sp)
+; RV32IM-NEXT:    addi sp, sp, 16
+; RV32IM-NEXT:    ret
+;
+; RV64I-LABEL: dont_fold_srem_i64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a1, zero, 98
+; RV64I-NEXT:    call __moddi3
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IM-LABEL: dont_fold_srem_i64:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    lui a1, 2675
+; RV64IM-NEXT:    addiw a1, a1, -251
+; RV64IM-NEXT:    slli a1, a1, 13
+; RV64IM-NEXT:    addi a1, a1, 1839
+; RV64IM-NEXT:    slli a1, a1, 13
+; RV64IM-NEXT:    addi a1, a1, 167
+; RV64IM-NEXT:    slli a1, a1, 13
+; RV64IM-NEXT:    addi a1, a1, 1505
+; RV64IM-NEXT:    mulh a1, a0, a1
+; RV64IM-NEXT:    srli a2, a1, 63
+; RV64IM-NEXT:    srai a1, a1, 5
+; RV64IM-NEXT:    add a1, a1, a2
+; RV64IM-NEXT:    addi a2, zero, 98
+; RV64IM-NEXT:    mul a1, a1, a2
+; RV64IM-NEXT:    sub a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = srem i64 %x, 98
+  ret i64 %1
+}
diff --git a/llvm/test/CodeGen/RISCV/srem-vector-lkk.ll b/llvm/test/CodeGen/RISCV/srem-vector-lkk.ll
new file mode 100644
index 000000000..c262e6aa2
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/srem-vector-lkk.ll
@@ -0,0 +1,1432 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefixes=CHECK,RV32I %s
+; RUN: llc -mtriple=riscv32 -mattr=+m -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefixes=CHECK,RV32IM %s
+; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefixes=CHECK,RV64I %s
+; RUN: llc -mtriple=riscv64 -mattr=+m -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefixes=CHECK,RV64IM %s
+
+define <4 x i16> @fold_srem_vec_1(<4 x i16> %x) nounwind {
+; RV32I-LABEL: fold_srem_vec_1:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -32
+; RV32I-NEXT:    sw ra, 28(sp)
+; RV32I-NEXT:    sw s0, 24(sp)
+; RV32I-NEXT:    sw s1, 20(sp)
+; RV32I-NEXT:    sw s2, 16(sp)
+; RV32I-NEXT:    sw s3, 12(sp)
+; RV32I-NEXT:    sw s4, 8(sp)
+; RV32I-NEXT:    sw s5, 4(sp)
+; RV32I-NEXT:    lh s2, 12(a1)
+; RV32I-NEXT:    lh s3, 8(a1)
+; RV32I-NEXT:    lh s0, 4(a1)
+; RV32I-NEXT:    lh a2, 0(a1)
+; RV32I-NEXT:    mv s1, a0
+; RV32I-NEXT:    addi a1, zero, 95
+; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    call __modsi3
+; RV32I-NEXT:    mv s4, a0
+; RV32I-NEXT:    addi a1, zero, -124
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    call __modsi3
+; RV32I-NEXT:    mv s5, a0
+; RV32I-NEXT:    addi a1, zero, 98
+; RV32I-NEXT:    mv a0, s3
+; RV32I-NEXT:    call __modsi3
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    addi a1, zero, -1003
+; RV32I-NEXT:    mv a0, s2
+; RV32I-NEXT:    call __modsi3
+; RV32I-NEXT:    sh a0, 6(s1)
+; RV32I-NEXT:    sh s0, 4(s1)
+; RV32I-NEXT:    sh s5, 2(s1)
+; RV32I-NEXT:    sh s4, 0(s1)
+; RV32I-NEXT:    lw s5, 4(sp)
+; RV32I-NEXT:    lw s4, 8(sp)
+; RV32I-NEXT:    lw s3, 12(sp)
+; RV32I-NEXT:    lw s2, 16(sp)
+; RV32I-NEXT:    lw s1, 20(sp)
+; RV32I-NEXT:    lw s0, 24(sp)
+; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    addi sp, sp, 32
+; RV32I-NEXT:    ret
+;
+; RV32IM-LABEL: fold_srem_vec_1:
+; RV32IM:       # %bb.0:
+; RV32IM-NEXT:    lh a6, 12(a1)
+; RV32IM-NEXT:    lh a3, 8(a1)
+; RV32IM-NEXT:    lh a4, 0(a1)
+; RV32IM-NEXT:    lh a1, 4(a1)
+; RV32IM-NEXT:    lui a5, 706409
+; RV32IM-NEXT:    addi a5, a5, 389
+; RV32IM-NEXT:    mulh a5, a4, a5
+; RV32IM-NEXT:    add a5, a5, a4
+; RV32IM-NEXT:    srli a2, a5, 31
+; RV32IM-NEXT:    srli a5, a5, 6
+; RV32IM-NEXT:    add a2, a5, a2
+; RV32IM-NEXT:    addi a5, zero, 95
+; RV32IM-NEXT:    mul a2, a2, a5
+; RV32IM-NEXT:    sub a2, a4, a2
+; RV32IM-NEXT:    lui a4, 507375
+; RV32IM-NEXT:    addi a4, a4, 1981
+; RV32IM-NEXT:    mulh a4, a1, a4
+; RV32IM-NEXT:    sub a4, a4, a1
+; RV32IM-NEXT:    srli a5, a4, 31
+; RV32IM-NEXT:    srli a4, a4, 6
+; RV32IM-NEXT:    add a4, a4, a5
+; RV32IM-NEXT:    addi a5, zero, -124
+; RV32IM-NEXT:    mul a4, a4, a5
+; RV32IM-NEXT:    sub a1, a1, a4
+; RV32IM-NEXT:    lui a4, 342392
+; RV32IM-NEXT:    addi a4, a4, 669
+; RV32IM-NEXT:    mulh a4, a3, a4
+; RV32IM-NEXT:    srli a5, a4, 31
+; RV32IM-NEXT:    srli a4, a4, 5
+; RV32IM-NEXT:    add a4, a4, a5
+; RV32IM-NEXT:    addi a5, zero, 98
+; RV32IM-NEXT:    mul a4, a4, a5
+; RV32IM-NEXT:    sub a3, a3, a4
+; RV32IM-NEXT:    lui a4, 780943
+; RV32IM-NEXT:    addi a4, a4, 1809
+; RV32IM-NEXT:    mulh a4, a6, a4
+; RV32IM-NEXT:    srli a5, a4, 31
+; RV32IM-NEXT:    srli a4, a4, 8
+; RV32IM-NEXT:    add a4, a4, a5
+; RV32IM-NEXT:    addi a5, zero, -1003
+; RV32IM-NEXT:    mul a4, a4, a5
+; RV32IM-NEXT:    sub a4, a6, a4
+; RV32IM-NEXT:    sh a4, 6(a0)
+; RV32IM-NEXT:    sh a3, 4(a0)
+; RV32IM-NEXT:    sh a1, 2(a0)
+; RV32IM-NEXT:    sh a2, 0(a0)
+; RV32IM-NEXT:    ret
+;
+; RV64I-LABEL: fold_srem_vec_1:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -64
+; RV64I-NEXT:    sd ra, 56(sp)
+; RV64I-NEXT:    sd s0, 48(sp)
+; RV64I-NEXT:    sd s1, 40(sp)
+; RV64I-NEXT:    sd s2, 32(sp)
+; RV64I-NEXT:    sd s3, 24(sp)
+; RV64I-NEXT:    sd s4, 16(sp)
+; RV64I-NEXT:    sd s5, 8(sp)
+; RV64I-NEXT:    lh s2, 24(a1)
+; RV64I-NEXT:    lh s3, 16(a1)
+; RV64I-NEXT:    lh s0, 8(a1)
+; RV64I-NEXT:    lh a2, 0(a1)
+; RV64I-NEXT:    mv s1, a0
+; RV64I-NEXT:    addi a1, zero, 95
+; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    call __moddi3
+; RV64I-NEXT:    mv s4, a0
+; RV64I-NEXT:    addi a1, zero, -124
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    call __moddi3
+; RV64I-NEXT:    mv s5, a0
+; RV64I-NEXT:    addi a1, zero, 98
+; RV64I-NEXT:    mv a0, s3
+; RV64I-NEXT:    call __moddi3
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    addi a1, zero, -1003
+; RV64I-NEXT:    mv a0, s2
+; RV64I-NEXT:    call __moddi3
+; RV64I-NEXT:    sh a0, 6(s1)
+; RV64I-NEXT:    sh s0, 4(s1)
+; RV64I-NEXT:    sh s5, 2(s1)
+; RV64I-NEXT:    sh s4, 0(s1)
+; RV64I-NEXT:    ld s5, 8(sp)
+; RV64I-NEXT:    ld s4, 16(sp)
+; RV64I-NEXT:    ld s3, 24(sp)
+; RV64I-NEXT:    ld s2, 32(sp)
+; RV64I-NEXT:    ld s1, 40(sp)
+; RV64I-NEXT:    ld s0, 48(sp)
+; RV64I-NEXT:    ld ra, 56(sp)
+; RV64I-NEXT:    addi sp, sp, 64
+; RV64I-NEXT:    ret
+;
+; RV64IM-LABEL: fold_srem_vec_1:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    lh a6, 24(a1)
+; RV64IM-NEXT:    lh a3, 16(a1)
+; RV64IM-NEXT:    lh a4, 8(a1)
+; RV64IM-NEXT:    lh a1, 0(a1)
+; RV64IM-NEXT:    lui a5, 1045903
+; RV64IM-NEXT:    addiw a5, a5, -733
+; RV64IM-NEXT:    slli a5, a5, 15
+; RV64IM-NEXT:    addi a5, a5, 1035
+; RV64IM-NEXT:    slli a5, a5, 12
+; RV64IM-NEXT:    addi a5, a5, -905
+; RV64IM-NEXT:    slli a5, a5, 12
+; RV64IM-NEXT:    addi a5, a5, -1767
+; RV64IM-NEXT:    mulh a5, a1, a5
+; RV64IM-NEXT:    add a5, a5, a1
+; RV64IM-NEXT:    srli a2, a5, 63
+; RV64IM-NEXT:    srli a5, a5, 6
+; RV64IM-NEXT:    add a2, a5, a2
+; RV64IM-NEXT:    addi a5, zero, 95
+; RV64IM-NEXT:    mul a2, a2, a5
+; RV64IM-NEXT:    sub a1, a1, a2
+; RV64IM-NEXT:    lui a2, 248
+; RV64IM-NEXT:    addiw a2, a2, -1057
+; RV64IM-NEXT:    slli a2, a2, 15
+; RV64IM-NEXT:    addi a2, a2, -1057
+; RV64IM-NEXT:    slli a2, a2, 15
+; RV64IM-NEXT:    addi a2, a2, -1057
+; RV64IM-NEXT:    slli a2, a2, 13
+; RV64IM-NEXT:    addi a2, a2, -265
+; RV64IM-NEXT:    mulh a2, a4, a2
+; RV64IM-NEXT:    sub a2, a2, a4
+; RV64IM-NEXT:    srli a5, a2, 63
+; RV64IM-NEXT:    srli a2, a2, 6
+; RV64IM-NEXT:    add a2, a2, a5
+; RV64IM-NEXT:    addi a5, zero, -124
+; RV64IM-NEXT:    mul a2, a2, a5
+; RV64IM-NEXT:    sub a2, a4, a2
+; RV64IM-NEXT:    lui a4, 2675
+; RV64IM-NEXT:    addiw a4, a4, -251
+; RV64IM-NEXT:    slli a4, a4, 13
+; RV64IM-NEXT:    addi a4, a4, 1839
+; RV64IM-NEXT:    slli a4, a4, 13
+; RV64IM-NEXT:    addi a4, a4, 167
+; RV64IM-NEXT:    slli a4, a4, 13
+; RV64IM-NEXT:    addi a4, a4, 1505
+; RV64IM-NEXT:    mulh a4, a3, a4
+; RV64IM-NEXT:    srli a5, a4, 63
+; RV64IM-NEXT:    srli a4, a4, 5
+; RV64IM-NEXT:    add a4, a4, a5
+; RV64IM-NEXT:    addi a5, zero, 98
+; RV64IM-NEXT:    mul a4, a4, a5
+; RV64IM-NEXT:    sub a3, a3, a4
+; RV64IM-NEXT:    lui a4, 1040212
+; RV64IM-NEXT:    addiw a4, a4, 1977
+; RV64IM-NEXT:    slli a4, a4, 12
+; RV64IM-NEXT:    addi a4, a4, -1907
+; RV64IM-NEXT:    slli a4, a4, 12
+; RV64IM-NEXT:    addi a4, a4, -453
+; RV64IM-NEXT:    slli a4, a4, 12
+; RV64IM-NEXT:    addi a4, a4, -1213
+; RV64IM-NEXT:    mulh a4, a6, a4
+; RV64IM-NEXT:    srli a5, a4, 63
+; RV64IM-NEXT:    srli a4, a4, 7
+; RV64IM-NEXT:    add a4, a4, a5
+; RV64IM-NEXT:    addi a5, zero, -1003
+; RV64IM-NEXT:    mul a4, a4, a5
+; RV64IM-NEXT:    sub a4, a6, a4
+; RV64IM-NEXT:    sh a4, 6(a0)
+; RV64IM-NEXT:    sh a3, 4(a0)
+; RV64IM-NEXT:    sh a2, 2(a0)
+; RV64IM-NEXT:    sh a1, 0(a0)
+; RV64IM-NEXT:    ret
+  %1 = srem <4 x i16> %x, <i16 95, i16 -124, i16 98, i16 -1003>
+  ret <4 x i16> %1
+}
+
+define <4 x i16> @fold_srem_vec_2(<4 x i16> %x) nounwind {
+; RV32I-LABEL: fold_srem_vec_2:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -32
+; RV32I-NEXT:    sw ra, 28(sp)
+; RV32I-NEXT:    sw s0, 24(sp)
+; RV32I-NEXT:    sw s1, 20(sp)
+; RV32I-NEXT:    sw s2, 16(sp)
+; RV32I-NEXT:    sw s3, 12(sp)
+; RV32I-NEXT:    sw s4, 8(sp)
+; RV32I-NEXT:    sw s5, 4(sp)
+; RV32I-NEXT:    lh s2, 12(a1)
+; RV32I-NEXT:    lh s3, 8(a1)
+; RV32I-NEXT:    lh s0, 4(a1)
+; RV32I-NEXT:    lh a2, 0(a1)
+; RV32I-NEXT:    mv s1, a0
+; RV32I-NEXT:    addi a1, zero, 95
+; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    call __modsi3
+; RV32I-NEXT:    mv s4, a0
+; RV32I-NEXT:    addi a1, zero, 95
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    call __modsi3
+; RV32I-NEXT:    mv s5, a0
+; RV32I-NEXT:    addi a1, zero, 95
+; RV32I-NEXT:    mv a0, s3
+; RV32I-NEXT:    call __modsi3
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    addi a1, zero, 95
+; RV32I-NEXT:    mv a0, s2
+; RV32I-NEXT:    call __modsi3
+; RV32I-NEXT:    sh a0, 6(s1)
+; RV32I-NEXT:    sh s0, 4(s1)
+; RV32I-NEXT:    sh s5, 2(s1)
+; RV32I-NEXT:    sh s4, 0(s1)
+; RV32I-NEXT:    lw s5, 4(sp)
+; RV32I-NEXT:    lw s4, 8(sp)
+; RV32I-NEXT:    lw s3, 12(sp)
+; RV32I-NEXT:    lw s2, 16(sp)
+; RV32I-NEXT:    lw s1, 20(sp)
+; RV32I-NEXT:    lw s0, 24(sp)
+; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    addi sp, sp, 32
+; RV32I-NEXT:    ret
+;
+; RV32IM-LABEL: fold_srem_vec_2:
+; RV32IM:       # %bb.0:
+; RV32IM-NEXT:    lh a6, 12(a1)
+; RV32IM-NEXT:    lh a3, 8(a1)
+; RV32IM-NEXT:    lh a4, 0(a1)
+; RV32IM-NEXT:    lh a1, 4(a1)
+; RV32IM-NEXT:    lui a5, 706409
+; RV32IM-NEXT:    addi a5, a5, 389
+; RV32IM-NEXT:    mulh a2, a4, a5
+; RV32IM-NEXT:    add a2, a2, a4
+; RV32IM-NEXT:    srli a7, a2, 31
+; RV32IM-NEXT:    srli a2, a2, 6
+; RV32IM-NEXT:    add a2, a2, a7
+; RV32IM-NEXT:    addi a7, zero, 95
+; RV32IM-NEXT:    mul a2, a2, a7
+; RV32IM-NEXT:    sub t0, a4, a2
+; RV32IM-NEXT:    mulh a4, a1, a5
+; RV32IM-NEXT:    add a4, a4, a1
+; RV32IM-NEXT:    srli a2, a4, 31
+; RV32IM-NEXT:    srli a4, a4, 6
+; RV32IM-NEXT:    add a2, a4, a2
+; RV32IM-NEXT:    mul a2, a2, a7
+; RV32IM-NEXT:    sub a1, a1, a2
+; RV32IM-NEXT:    mulh a2, a3, a5
+; RV32IM-NEXT:    add a2, a2, a3
+; RV32IM-NEXT:    srli a4, a2, 31
+; RV32IM-NEXT:    srli a2, a2, 6
+; RV32IM-NEXT:    add a2, a2, a4
+; RV32IM-NEXT:    mul a2, a2, a7
+; RV32IM-NEXT:    sub a2, a3, a2
+; RV32IM-NEXT:    mulh a3, a6, a5
+; RV32IM-NEXT:    add a3, a3, a6
+; RV32IM-NEXT:    srli a4, a3, 31
+; RV32IM-NEXT:    srli a3, a3, 6
+; RV32IM-NEXT:    add a3, a3, a4
+; RV32IM-NEXT:    mul a3, a3, a7
+; RV32IM-NEXT:    sub a3, a6, a3
+; RV32IM-NEXT:    sh a3, 6(a0)
+; RV32IM-NEXT:    sh a2, 4(a0)
+; RV32IM-NEXT:    sh a1, 2(a0)
+; RV32IM-NEXT:    sh t0, 0(a0)
+; RV32IM-NEXT:    ret
+;
+; RV64I-LABEL: fold_srem_vec_2:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -64
+; RV64I-NEXT:    sd ra, 56(sp)
+; RV64I-NEXT:    sd s0, 48(sp)
+; RV64I-NEXT:    sd s1, 40(sp)
+; RV64I-NEXT:    sd s2, 32(sp)
+; RV64I-NEXT:    sd s3, 24(sp)
+; RV64I-NEXT:    sd s4, 16(sp)
+; RV64I-NEXT:    sd s5, 8(sp)
+; RV64I-NEXT:    lh s2, 24(a1)
+; RV64I-NEXT:    lh s3, 16(a1)
+; RV64I-NEXT:    lh s0, 8(a1)
+; RV64I-NEXT:    lh a2, 0(a1)
+; RV64I-NEXT:    mv s1, a0
+; RV64I-NEXT:    addi a1, zero, 95
+; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    call __moddi3
+; RV64I-NEXT:    mv s4, a0
+; RV64I-NEXT:    addi a1, zero, 95
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    call __moddi3
+; RV64I-NEXT:    mv s5, a0
+; RV64I-NEXT:    addi a1, zero, 95
+; RV64I-NEXT:    mv a0, s3
+; RV64I-NEXT:    call __moddi3
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    addi a1, zero, 95
+; RV64I-NEXT:    mv a0, s2
+; RV64I-NEXT:    call __moddi3
+; RV64I-NEXT:    sh a0, 6(s1)
+; RV64I-NEXT:    sh s0, 4(s1)
+; RV64I-NEXT:    sh s5, 2(s1)
+; RV64I-NEXT:    sh s4, 0(s1)
+; RV64I-NEXT:    ld s5, 8(sp)
+; RV64I-NEXT:    ld s4, 16(sp)
+; RV64I-NEXT:    ld s3, 24(sp)
+; RV64I-NEXT:    ld s2, 32(sp)
+; RV64I-NEXT:    ld s1, 40(sp)
+; RV64I-NEXT:    ld s0, 48(sp)
+; RV64I-NEXT:    ld ra, 56(sp)
+; RV64I-NEXT:    addi sp, sp, 64
+; RV64I-NEXT:    ret
+;
+; RV64IM-LABEL: fold_srem_vec_2:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    lh a6, 24(a1)
+; RV64IM-NEXT:    lh a7, 16(a1)
+; RV64IM-NEXT:    lh a4, 8(a1)
+; RV64IM-NEXT:    lh a1, 0(a1)
+; RV64IM-NEXT:    lui a5, 1045903
+; RV64IM-NEXT:    addiw a5, a5, -733
+; RV64IM-NEXT:    slli a5, a5, 15
+; RV64IM-NEXT:    addi a5, a5, 1035
+; RV64IM-NEXT:    slli a5, a5, 12
+; RV64IM-NEXT:    addi a5, a5, -905
+; RV64IM-NEXT:    slli a5, a5, 12
+; RV64IM-NEXT:    addi a5, a5, -1767
+; RV64IM-NEXT:    mulh a2, a1, a5
+; RV64IM-NEXT:    add a2, a2, a1
+; RV64IM-NEXT:    srli a3, a2, 63
+; RV64IM-NEXT:    srli a2, a2, 6
+; RV64IM-NEXT:    add a2, a2, a3
+; RV64IM-NEXT:    addi a3, zero, 95
+; RV64IM-NEXT:    mul a2, a2, a3
+; RV64IM-NEXT:    sub t0, a1, a2
+; RV64IM-NEXT:    mulh a2, a4, a5
+; RV64IM-NEXT:    add a2, a2, a4
+; RV64IM-NEXT:    srli a1, a2, 63
+; RV64IM-NEXT:    srli a2, a2, 6
+; RV64IM-NEXT:    add a1, a2, a1
+; RV64IM-NEXT:    mul a1, a1, a3
+; RV64IM-NEXT:    sub a1, a4, a1
+; RV64IM-NEXT:    mulh a2, a7, a5
+; RV64IM-NEXT:    add a2, a2, a7
+; RV64IM-NEXT:    srli a4, a2, 63
+; RV64IM-NEXT:    srli a2, a2, 6
+; RV64IM-NEXT:    add a2, a2, a4
+; RV64IM-NEXT:    mul a2, a2, a3
+; RV64IM-NEXT:    sub a2, a7, a2
+; RV64IM-NEXT:    mulh a4, a6, a5
+; RV64IM-NEXT:    add a4, a4, a6
+; RV64IM-NEXT:    srli a5, a4, 63
+; RV64IM-NEXT:    srli a4, a4, 6
+; RV64IM-NEXT:    add a4, a4, a5
+; RV64IM-NEXT:    mul a3, a4, a3
+; RV64IM-NEXT:    sub a3, a6, a3
+; RV64IM-NEXT:    sh a3, 6(a0)
+; RV64IM-NEXT:    sh a2, 4(a0)
+; RV64IM-NEXT:    sh a1, 2(a0)
+; RV64IM-NEXT:    sh t0, 0(a0)
+; RV64IM-NEXT:    ret
+  %1 = srem <4 x i16> %x, <i16 95, i16 95, i16 95, i16 95>
+  ret <4 x i16> %1
+}
+
+
+; Don't fold if we can combine srem with sdiv.
+define <4 x i16> @combine_srem_sdiv(<4 x i16> %x) nounwind {
+; RV32I-LABEL: combine_srem_sdiv:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -48
+; RV32I-NEXT:    sw ra, 44(sp)
+; RV32I-NEXT:    sw s0, 40(sp)
+; RV32I-NEXT:    sw s1, 36(sp)
+; RV32I-NEXT:    sw s2, 32(sp)
+; RV32I-NEXT:    sw s3, 28(sp)
+; RV32I-NEXT:    sw s4, 24(sp)
+; RV32I-NEXT:    sw s5, 20(sp)
+; RV32I-NEXT:    sw s6, 16(sp)
+; RV32I-NEXT:    sw s7, 12(sp)
+; RV32I-NEXT:    sw s8, 8(sp)
+; RV32I-NEXT:    sw s9, 4(sp)
+; RV32I-NEXT:    lh s2, 0(a1)
+; RV32I-NEXT:    lh s3, 4(a1)
+; RV32I-NEXT:    lh s4, 8(a1)
+; RV32I-NEXT:    lh s1, 12(a1)
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    addi a1, zero, 95
+; RV32I-NEXT:    mv a0, s1
+; RV32I-NEXT:    call __modsi3
+; RV32I-NEXT:    mv s5, a0
+; RV32I-NEXT:    addi a1, zero, 95
+; RV32I-NEXT:    mv a0, s4
+; RV32I-NEXT:    call __modsi3
+; RV32I-NEXT:    mv s6, a0
+; RV32I-NEXT:    addi a1, zero, 95
+; RV32I-NEXT:    mv a0, s3
+; RV32I-NEXT:    call __modsi3
+; RV32I-NEXT:    mv s7, a0
+; RV32I-NEXT:    addi a1, zero, 95
+; RV32I-NEXT:    mv a0, s2
+; RV32I-NEXT:    call __modsi3
+; RV32I-NEXT:    mv s8, a0
+; RV32I-NEXT:    addi a1, zero, 95
+; RV32I-NEXT:    mv a0, s1
+; RV32I-NEXT:    call __divsi3
+; RV32I-NEXT:    mv s9, a0
+; RV32I-NEXT:    addi a1, zero, 95
+; RV32I-NEXT:    mv a0, s4
+; RV32I-NEXT:    call __divsi3
+; RV32I-NEXT:    mv s4, a0
+; RV32I-NEXT:    addi a1, zero, 95
+; RV32I-NEXT:    mv a0, s3
+; RV32I-NEXT:    call __divsi3
+; RV32I-NEXT:    mv s1, a0
+; RV32I-NEXT:    addi a1, zero, 95
+; RV32I-NEXT:    mv a0, s2
+; RV32I-NEXT:    call __divsi3
+; RV32I-NEXT:    add a0, s8, a0
+; RV32I-NEXT:    add a1, s7, s1
+; RV32I-NEXT:    add a2, s6, s4
+; RV32I-NEXT:    add a3, s5, s9
+; RV32I-NEXT:    sh a3, 6(s0)
+; RV32I-NEXT:    sh a2, 4(s0)
+; RV32I-NEXT:    sh a1, 2(s0)
+; RV32I-NEXT:    sh a0, 0(s0)
+; RV32I-NEXT:    lw s9, 4(sp)
+; RV32I-NEXT:    lw s8, 8(sp)
+; RV32I-NEXT:    lw s7, 12(sp)
+; RV32I-NEXT:    lw s6, 16(sp)
+; RV32I-NEXT:    lw s5, 20(sp)
+; RV32I-NEXT:    lw s4, 24(sp)
+; RV32I-NEXT:    lw s3, 28(sp)
+; RV32I-NEXT:    lw s2, 32(sp)
+; RV32I-NEXT:    lw s1, 36(sp)
+; RV32I-NEXT:    lw s0, 40(sp)
+; RV32I-NEXT:    lw ra, 44(sp)
+; RV32I-NEXT:    addi sp, sp, 48
+; RV32I-NEXT:    ret
+;
+; RV32IM-LABEL: combine_srem_sdiv:
+; RV32IM:       # %bb.0:
+; RV32IM-NEXT:    lh a6, 0(a1)
+; RV32IM-NEXT:    lh a3, 4(a1)
+; RV32IM-NEXT:    lh a4, 12(a1)
+; RV32IM-NEXT:    lh a1, 8(a1)
+; RV32IM-NEXT:    lui a5, 706409
+; RV32IM-NEXT:    addi a5, a5, 389
+; RV32IM-NEXT:    mulh a2, a4, a5
+; RV32IM-NEXT:    add a2, a2, a4
+; RV32IM-NEXT:    srli a7, a2, 31
+; RV32IM-NEXT:    srai a2, a2, 6
+; RV32IM-NEXT:    add t0, a2, a7
+; RV32IM-NEXT:    addi a7, zero, 95
+; RV32IM-NEXT:    mul a2, t0, a7
+; RV32IM-NEXT:    sub t1, a4, a2
+; RV32IM-NEXT:    mulh a4, a1, a5
+; RV32IM-NEXT:    add a4, a4, a1
+; RV32IM-NEXT:    srli a2, a4, 31
+; RV32IM-NEXT:    srai a4, a4, 6
+; RV32IM-NEXT:    add a2, a4, a2
+; RV32IM-NEXT:    mul a4, a2, a7
+; RV32IM-NEXT:    sub t2, a1, a4
+; RV32IM-NEXT:    mulh a4, a3, a5
+; RV32IM-NEXT:    add a4, a4, a3
+; RV32IM-NEXT:    srli a1, a4, 31
+; RV32IM-NEXT:    srai a4, a4, 6
+; RV32IM-NEXT:    add a1, a4, a1
+; RV32IM-NEXT:    mul a4, a1, a7
+; RV32IM-NEXT:    sub a3, a3, a4
+; RV32IM-NEXT:    mulh a4, a6, a5
+; RV32IM-NEXT:    add a4, a4, a6
+; RV32IM-NEXT:    srli a5, a4, 31
+; RV32IM-NEXT:    srai a4, a4, 6
+; RV32IM-NEXT:    add a4, a4, a5
+; RV32IM-NEXT:    mul a5, a4, a7
+; RV32IM-NEXT:    sub a5, a6, a5
+; RV32IM-NEXT:    add a4, a5, a4
+; RV32IM-NEXT:    add a1, a3, a1
+; RV32IM-NEXT:    add a2, t2, a2
+; RV32IM-NEXT:    add a3, t1, t0
+; RV32IM-NEXT:    sh a3, 6(a0)
+; RV32IM-NEXT:    sh a2, 4(a0)
+; RV32IM-NEXT:    sh a1, 2(a0)
+; RV32IM-NEXT:    sh a4, 0(a0)
+; RV32IM-NEXT:    ret
+;
+; RV64I-LABEL: combine_srem_sdiv:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -96
+; RV64I-NEXT:    sd ra, 88(sp)
+; RV64I-NEXT:    sd s0, 80(sp)
+; RV64I-NEXT:    sd s1, 72(sp)
+; RV64I-NEXT:    sd s2, 64(sp)
+; RV64I-NEXT:    sd s3, 56(sp)
+; RV64I-NEXT:    sd s4, 48(sp)
+; RV64I-NEXT:    sd s5, 40(sp)
+; RV64I-NEXT:    sd s6, 32(sp)
+; RV64I-NEXT:    sd s7, 24(sp)
+; RV64I-NEXT:    sd s8, 16(sp)
+; RV64I-NEXT:    sd s9, 8(sp)
+; RV64I-NEXT:    lh s2, 0(a1)
+; RV64I-NEXT:    lh s3, 8(a1)
+; RV64I-NEXT:    lh s4, 16(a1)
+; RV64I-NEXT:    lh s1, 24(a1)
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    addi a1, zero, 95
+; RV64I-NEXT:    mv a0, s1
+; RV64I-NEXT:    call __moddi3
+; RV64I-NEXT:    mv s5, a0
+; RV64I-NEXT:    addi a1, zero, 95
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    call __moddi3
+; RV64I-NEXT:    mv s6, a0
+; RV64I-NEXT:    addi a1, zero, 95
+; RV64I-NEXT:    mv a0, s3
+; RV64I-NEXT:    call __moddi3
+; RV64I-NEXT:    mv s7, a0
+; RV64I-NEXT:    addi a1, zero, 95
+; RV64I-NEXT:    mv a0, s2
+; RV64I-NEXT:    call __moddi3
+; RV64I-NEXT:    mv s8, a0
+; RV64I-NEXT:    addi a1, zero, 95
+; RV64I-NEXT:    mv a0, s1
+; RV64I-NEXT:    call __divdi3
+; RV64I-NEXT:    mv s9, a0
+; RV64I-NEXT:    addi a1, zero, 95
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    call __divdi3
+; RV64I-NEXT:    mv s4, a0
+; RV64I-NEXT:    addi a1, zero, 95
+; RV64I-NEXT:    mv a0, s3
+; RV64I-NEXT:    call __divdi3
+; RV64I-NEXT:    mv s1, a0
+; RV64I-NEXT:    addi a1, zero, 95
+; RV64I-NEXT:    mv a0, s2
+; RV64I-NEXT:    call __divdi3
+; RV64I-NEXT:    add a0, s8, a0
+; RV64I-NEXT:    add a1, s7, s1
+; RV64I-NEXT:    add a2, s6, s4
+; RV64I-NEXT:    add a3, s5, s9
+; RV64I-NEXT:    sh a3, 6(s0)
+; RV64I-NEXT:    sh a2, 4(s0)
+; RV64I-NEXT:    sh a1, 2(s0)
+; RV64I-NEXT:    sh a0, 0(s0)
+; RV64I-NEXT:    ld s9, 8(sp)
+; RV64I-NEXT:    ld s8, 16(sp)
+; RV64I-NEXT:    ld s7, 24(sp)
+; RV64I-NEXT:    ld s6, 32(sp)
+; RV64I-NEXT:    ld s5, 40(sp)
+; RV64I-NEXT:    ld s4, 48(sp)
+; RV64I-NEXT:    ld s3, 56(sp)
+; RV64I-NEXT:    ld s2, 64(sp)
+; RV64I-NEXT:    ld s1, 72(sp)
+; RV64I-NEXT:    ld s0, 80(sp)
+; RV64I-NEXT:    ld ra, 88(sp)
+; RV64I-NEXT:    addi sp, sp, 96
+; RV64I-NEXT:    ret
+;
+; RV64IM-LABEL: combine_srem_sdiv:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    lh a6, 0(a1)
+; RV64IM-NEXT:    lh a7, 8(a1)
+; RV64IM-NEXT:    lh a4, 16(a1)
+; RV64IM-NEXT:    lh a1, 24(a1)
+; RV64IM-NEXT:    lui a5, 1045903
+; RV64IM-NEXT:    addiw a5, a5, -733
+; RV64IM-NEXT:    slli a5, a5, 15
+; RV64IM-NEXT:    addi a5, a5, 1035
+; RV64IM-NEXT:    slli a5, a5, 12
+; RV64IM-NEXT:    addi a5, a5, -905
+; RV64IM-NEXT:    slli a5, a5, 12
+; RV64IM-NEXT:    addi a5, a5, -1767
+; RV64IM-NEXT:    mulh a2, a1, a5
+; RV64IM-NEXT:    add a2, a2, a1
+; RV64IM-NEXT:    srli a3, a2, 63
+; RV64IM-NEXT:    srai a2, a2, 6
+; RV64IM-NEXT:    add t3, a2, a3
+; RV64IM-NEXT:    addi t0, zero, 95
+; RV64IM-NEXT:    mul a3, t3, t0
+; RV64IM-NEXT:    sub t1, a1, a3
+; RV64IM-NEXT:    mulh a3, a4, a5
+; RV64IM-NEXT:    add a3, a3, a4
+; RV64IM-NEXT:    srli a1, a3, 63
+; RV64IM-NEXT:    srai a3, a3, 6
+; RV64IM-NEXT:    add a1, a3, a1
+; RV64IM-NEXT:    mul a3, a1, t0
+; RV64IM-NEXT:    sub t2, a4, a3
+; RV64IM-NEXT:    mulh a4, a7, a5
+; RV64IM-NEXT:    add a4, a4, a7
+; RV64IM-NEXT:    srli a3, a4, 63
+; RV64IM-NEXT:    srai a4, a4, 6
+; RV64IM-NEXT:    add a3, a4, a3
+; RV64IM-NEXT:    mul a4, a3, t0
+; RV64IM-NEXT:    sub a4, a7, a4
+; RV64IM-NEXT:    mulh a5, a6, a5
+; RV64IM-NEXT:    add a5, a5, a6
+; RV64IM-NEXT:    srli a2, a5, 63
+; RV64IM-NEXT:    srai a5, a5, 6
+; RV64IM-NEXT:    add a2, a5, a2
+; RV64IM-NEXT:    mul a5, a2, t0
+; RV64IM-NEXT:    sub a5, a6, a5
+; RV64IM-NEXT:    add a2, a5, a2
+; RV64IM-NEXT:    add a3, a4, a3
+; RV64IM-NEXT:    add a1, t2, a1
+; RV64IM-NEXT:    add a4, t1, t3
+; RV64IM-NEXT:    sh a4, 6(a0)
+; RV64IM-NEXT:    sh a1, 4(a0)
+; RV64IM-NEXT:    sh a3, 2(a0)
+; RV64IM-NEXT:    sh a2, 0(a0)
+; RV64IM-NEXT:    ret
+  %1 = srem <4 x i16> %x, <i16 95, i16 95, i16 95, i16 95>
+  %2 = sdiv <4 x i16> %x, <i16 95, i16 95, i16 95, i16 95>
+  %3 = add <4 x i16> %1, %2
+  ret <4 x i16> %3
+}
+
+; Don't fold for divisors that are a power of two.
+define <4 x i16> @dont_fold_srem_power_of_two(<4 x i16> %x) nounwind {
+; RV32I-LABEL: dont_fold_srem_power_of_two:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -32
+; RV32I-NEXT:    sw ra, 28(sp)
+; RV32I-NEXT:    sw s0, 24(sp)
+; RV32I-NEXT:    sw s1, 20(sp)
+; RV32I-NEXT:    sw s2, 16(sp)
+; RV32I-NEXT:    sw s3, 12(sp)
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lh a2, 0(a1)
+; RV32I-NEXT:    lh a0, 12(a1)
+; RV32I-NEXT:    lh a3, 8(a1)
+; RV32I-NEXT:    lh a1, 4(a1)
+; RV32I-NEXT:    srai a4, a2, 31
+; RV32I-NEXT:    srli a4, a4, 26
+; RV32I-NEXT:    add a4, a2, a4
+; RV32I-NEXT:    lui a6, 16
+; RV32I-NEXT:    addi a5, a6, -64
+; RV32I-NEXT:    and a4, a4, a5
+; RV32I-NEXT:    sub s2, a2, a4
+; RV32I-NEXT:    srai a2, a1, 31
+; RV32I-NEXT:    srli a2, a2, 27
+; RV32I-NEXT:    add a2, a1, a2
+; RV32I-NEXT:    addi a4, a6, -32
+; RV32I-NEXT:    and a2, a2, a4
+; RV32I-NEXT:    sub s3, a1, a2
+; RV32I-NEXT:    srai a1, a3, 31
+; RV32I-NEXT:    srli a1, a1, 29
+; RV32I-NEXT:    add a1, a3, a1
+; RV32I-NEXT:    addi a2, a6, -8
+; RV32I-NEXT:    and a1, a1, a2
+; RV32I-NEXT:    sub s1, a3, a1
+; RV32I-NEXT:    addi a1, zero, 95
+; RV32I-NEXT:    call __modsi3
+; RV32I-NEXT:    sh a0, 6(s0)
+; RV32I-NEXT:    sh s1, 4(s0)
+; RV32I-NEXT:    sh s3, 2(s0)
+; RV32I-NEXT:    sh s2, 0(s0)
+; RV32I-NEXT:    lw s3, 12(sp)
+; RV32I-NEXT:    lw s2, 16(sp)
+; RV32I-NEXT:    lw s1, 20(sp)
+; RV32I-NEXT:    lw s0, 24(sp)
+; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    addi sp, sp, 32
+; RV32I-NEXT:    ret
+;
+; RV32IM-LABEL: dont_fold_srem_power_of_two:
+; RV32IM:       # %bb.0:
+; RV32IM-NEXT:    lh a6, 8(a1)
+; RV32IM-NEXT:    lh a3, 4(a1)
+; RV32IM-NEXT:    lh a4, 12(a1)
+; RV32IM-NEXT:    lh a1, 0(a1)
+; RV32IM-NEXT:    lui a5, 706409
+; RV32IM-NEXT:    addi a5, a5, 389
+; RV32IM-NEXT:    mulh a5, a4, a5
+; RV32IM-NEXT:    add a5, a5, a4
+; RV32IM-NEXT:    srli a2, a5, 31
+; RV32IM-NEXT:    srli a5, a5, 6
+; RV32IM-NEXT:    add a2, a5, a2
+; RV32IM-NEXT:    addi a5, zero, 95
+; RV32IM-NEXT:    mul a2, a2, a5
+; RV32IM-NEXT:    sub a7, a4, a2
+; RV32IM-NEXT:    srai a4, a1, 31
+; RV32IM-NEXT:    srli a4, a4, 26
+; RV32IM-NEXT:    add a4, a1, a4
+; RV32IM-NEXT:    lui a5, 16
+; RV32IM-NEXT:    addi a2, a5, -64
+; RV32IM-NEXT:    and a2, a4, a2
+; RV32IM-NEXT:    sub a1, a1, a2
+; RV32IM-NEXT:    srai a2, a3, 31
+; RV32IM-NEXT:    srli a2, a2, 27
+; RV32IM-NEXT:    add a2, a3, a2
+; RV32IM-NEXT:    addi a4, a5, -32
+; RV32IM-NEXT:    and a2, a2, a4
+; RV32IM-NEXT:    sub a2, a3, a2
+; RV32IM-NEXT:    srai a3, a6, 31
+; RV32IM-NEXT:    srli a3, a3, 29
+; RV32IM-NEXT:    add a3, a6, a3
+; RV32IM-NEXT:    addi a4, a5, -8
+; RV32IM-NEXT:    and a3, a3, a4
+; RV32IM-NEXT:    sub a3, a6, a3
+; RV32IM-NEXT:    sh a3, 4(a0)
+; RV32IM-NEXT:    sh a2, 2(a0)
+; RV32IM-NEXT:    sh a1, 0(a0)
+; RV32IM-NEXT:    sh a7, 6(a0)
+; RV32IM-NEXT:    ret
+;
+; RV64I-LABEL: dont_fold_srem_power_of_two:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s0, 32(sp)
+; RV64I-NEXT:    sd s1, 24(sp)
+; RV64I-NEXT:    sd s2, 16(sp)
+; RV64I-NEXT:    sd s3, 8(sp)
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    lh a2, 0(a1)
+; RV64I-NEXT:    lh a0, 24(a1)
+; RV64I-NEXT:    lh a3, 16(a1)
+; RV64I-NEXT:    lh a1, 8(a1)
+; RV64I-NEXT:    srai a4, a2, 63
+; RV64I-NEXT:    srli a4, a4, 58
+; RV64I-NEXT:    add a4, a2, a4
+; RV64I-NEXT:    lui a6, 16
+; RV64I-NEXT:    addiw a5, a6, -64
+; RV64I-NEXT:    and a4, a4, a5
+; RV64I-NEXT:    sub s2, a2, a4
+; RV64I-NEXT:    srai a2, a1, 63
+; RV64I-NEXT:    srli a2, a2, 59
+; RV64I-NEXT:    add a2, a1, a2
+; RV64I-NEXT:    addiw a4, a6, -32
+; RV64I-NEXT:    and a2, a2, a4
+; RV64I-NEXT:    sub s3, a1, a2
+; RV64I-NEXT:    srai a1, a3, 63
+; RV64I-NEXT:    srli a1, a1, 61
+; RV64I-NEXT:    add a1, a3, a1
+; RV64I-NEXT:    addiw a2, a6, -8
+; RV64I-NEXT:    and a1, a1, a2
+; RV64I-NEXT:    sub s1, a3, a1
+; RV64I-NEXT:    addi a1, zero, 95
+; RV64I-NEXT:    call __moddi3
+; RV64I-NEXT:    sh a0, 6(s0)
+; RV64I-NEXT:    sh s1, 4(s0)
+; RV64I-NEXT:    sh s3, 2(s0)
+; RV64I-NEXT:    sh s2, 0(s0)
+; RV64I-NEXT:    ld s3, 8(sp)
+; RV64I-NEXT:    ld s2, 16(sp)
+; RV64I-NEXT:    ld s1, 24(sp)
+; RV64I-NEXT:    ld s0, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IM-LABEL: dont_fold_srem_power_of_two:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    lh a6, 16(a1)
+; RV64IM-NEXT:    lh a3, 8(a1)
+; RV64IM-NEXT:    lh a4, 0(a1)
+; RV64IM-NEXT:    lh a1, 24(a1)
+; RV64IM-NEXT:    lui a5, 1045903
+; RV64IM-NEXT:    addiw a5, a5, -733
+; RV64IM-NEXT:    slli a5, a5, 15
+; RV64IM-NEXT:    addi a5, a5, 1035
+; RV64IM-NEXT:    slli a5, a5, 12
+; RV64IM-NEXT:    addi a5, a5, -905
+; RV64IM-NEXT:    slli a5, a5, 12
+; RV64IM-NEXT:    addi a5, a5, -1767
+; RV64IM-NEXT:    mulh a5, a1, a5
+; RV64IM-NEXT:    add a5, a5, a1
+; RV64IM-NEXT:    srli a2, a5, 63
+; RV64IM-NEXT:    srli a5, a5, 6
+; RV64IM-NEXT:    add a2, a5, a2
+; RV64IM-NEXT:    addi a5, zero, 95
+; RV64IM-NEXT:    mul a2, a2, a5
+; RV64IM-NEXT:    sub a7, a1, a2
+; RV64IM-NEXT:    srai a2, a4, 63
+; RV64IM-NEXT:    srli a2, a2, 58
+; RV64IM-NEXT:    add a2, a4, a2
+; RV64IM-NEXT:    lui a5, 16
+; RV64IM-NEXT:    addiw a1, a5, -64
+; RV64IM-NEXT:    and a1, a2, a1
+; RV64IM-NEXT:    sub a1, a4, a1
+; RV64IM-NEXT:    srai a2, a3, 63
+; RV64IM-NEXT:    srli a2, a2, 59
+; RV64IM-NEXT:    add a2, a3, a2
+; RV64IM-NEXT:    addiw a4, a5, -32
+; RV64IM-NEXT:    and a2, a2, a4
+; RV64IM-NEXT:    sub a2, a3, a2
+; RV64IM-NEXT:    srai a3, a6, 63
+; RV64IM-NEXT:    srli a3, a3, 61
+; RV64IM-NEXT:    add a3, a6, a3
+; RV64IM-NEXT:    addiw a4, a5, -8
+; RV64IM-NEXT:    and a3, a3, a4
+; RV64IM-NEXT:    sub a3, a6, a3
+; RV64IM-NEXT:    sh a3, 4(a0)
+; RV64IM-NEXT:    sh a2, 2(a0)
+; RV64IM-NEXT:    sh a1, 0(a0)
+; RV64IM-NEXT:    sh a7, 6(a0)
+; RV64IM-NEXT:    ret
+  %1 = srem <4 x i16> %x, <i16 64, i16 32, i16 8, i16 95>
+  ret <4 x i16> %1
+}
+
+; Don't fold if the divisor is one.
+define <4 x i16> @dont_fold_srem_one(<4 x i16> %x) nounwind {
+; RV32I-LABEL: dont_fold_srem_one:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -32
+; RV32I-NEXT:    sw ra, 28(sp)
+; RV32I-NEXT:    sw s0, 24(sp)
+; RV32I-NEXT:    sw s1, 20(sp)
+; RV32I-NEXT:    sw s2, 16(sp)
+; RV32I-NEXT:    sw s3, 12(sp)
+; RV32I-NEXT:    lh s2, 12(a1)
+; RV32I-NEXT:    lh s1, 8(a1)
+; RV32I-NEXT:    lh a2, 4(a1)
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    addi a1, zero, 654
+; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    call __modsi3
+; RV32I-NEXT:    mv s3, a0
+; RV32I-NEXT:    addi a1, zero, 23
+; RV32I-NEXT:    mv a0, s1
+; RV32I-NEXT:    call __modsi3
+; RV32I-NEXT:    mv s1, a0
+; RV32I-NEXT:    lui a0, 1
+; RV32I-NEXT:    addi a1, a0, 1327
+; RV32I-NEXT:    mv a0, s2
+; RV32I-NEXT:    call __modsi3
+; RV32I-NEXT:    sh zero, 0(s0)
+; RV32I-NEXT:    sh a0, 6(s0)
+; RV32I-NEXT:    sh s1, 4(s0)
+; RV32I-NEXT:    sh s3, 2(s0)
+; RV32I-NEXT:    lw s3, 12(sp)
+; RV32I-NEXT:    lw s2, 16(sp)
+; RV32I-NEXT:    lw s1, 20(sp)
+; RV32I-NEXT:    lw s0, 24(sp)
+; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    addi sp, sp, 32
+; RV32I-NEXT:    ret
+;
+; RV32IM-LABEL: dont_fold_srem_one:
+; RV32IM:       # %bb.0:
+; RV32IM-NEXT:    lh a2, 12(a1)
+; RV32IM-NEXT:    lh a3, 4(a1)
+; RV32IM-NEXT:    lh a1, 8(a1)
+; RV32IM-NEXT:    lui a4, 820904
+; RV32IM-NEXT:    addi a4, a4, -1903
+; RV32IM-NEXT:    mulh a4, a3, a4
+; RV32IM-NEXT:    add a4, a4, a3
+; RV32IM-NEXT:    srli a5, a4, 31
+; RV32IM-NEXT:    srli a4, a4, 9
+; RV32IM-NEXT:    add a4, a4, a5
+; RV32IM-NEXT:    addi a5, zero, 654
+; RV32IM-NEXT:    mul a4, a4, a5
+; RV32IM-NEXT:    sub a3, a3, a4
+; RV32IM-NEXT:    lui a4, 729444
+; RV32IM-NEXT:    addi a4, a4, 713
+; RV32IM-NEXT:    mulh a4, a1, a4
+; RV32IM-NEXT:    add a4, a4, a1
+; RV32IM-NEXT:    srli a5, a4, 31
+; RV32IM-NEXT:    srli a4, a4, 4
+; RV32IM-NEXT:    add a4, a4, a5
+; RV32IM-NEXT:    addi a5, zero, 23
+; RV32IM-NEXT:    mul a4, a4, a5
+; RV32IM-NEXT:    sub a1, a1, a4
+; RV32IM-NEXT:    lui a4, 395996
+; RV32IM-NEXT:    addi a4, a4, -2009
+; RV32IM-NEXT:    mulh a4, a2, a4
+; RV32IM-NEXT:    srli a5, a4, 31
+; RV32IM-NEXT:    srli a4, a4, 11
+; RV32IM-NEXT:    add a4, a4, a5
+; RV32IM-NEXT:    lui a5, 1
+; RV32IM-NEXT:    addi a5, a5, 1327
+; RV32IM-NEXT:    mul a4, a4, a5
+; RV32IM-NEXT:    sub a2, a2, a4
+; RV32IM-NEXT:    sh zero, 0(a0)
+; RV32IM-NEXT:    sh a2, 6(a0)
+; RV32IM-NEXT:    sh a1, 4(a0)
+; RV32IM-NEXT:    sh a3, 2(a0)
+; RV32IM-NEXT:    ret
+;
+; RV64I-LABEL: dont_fold_srem_one:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s0, 32(sp)
+; RV64I-NEXT:    sd s1, 24(sp)
+; RV64I-NEXT:    sd s2, 16(sp)
+; RV64I-NEXT:    sd s3, 8(sp)
+; RV64I-NEXT:    lh s2, 24(a1)
+; RV64I-NEXT:    lh s1, 16(a1)
+; RV64I-NEXT:    lh a2, 8(a1)
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    addi a1, zero, 654
+; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    call __moddi3
+; RV64I-NEXT:    mv s3, a0
+; RV64I-NEXT:    addi a1, zero, 23
+; RV64I-NEXT:    mv a0, s1
+; RV64I-NEXT:    call __moddi3
+; RV64I-NEXT:    mv s1, a0
+; RV64I-NEXT:    lui a0, 1
+; RV64I-NEXT:    addiw a1, a0, 1327
+; RV64I-NEXT:    mv a0, s2
+; RV64I-NEXT:    call __moddi3
+; RV64I-NEXT:    sh zero, 0(s0)
+; RV64I-NEXT:    sh a0, 6(s0)
+; RV64I-NEXT:    sh s1, 4(s0)
+; RV64I-NEXT:    sh s3, 2(s0)
+; RV64I-NEXT:    ld s3, 8(sp)
+; RV64I-NEXT:    ld s2, 16(sp)
+; RV64I-NEXT:    ld s1, 24(sp)
+; RV64I-NEXT:    ld s0, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IM-LABEL: dont_fold_srem_one:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    lh a2, 24(a1)
+; RV64IM-NEXT:    lh a3, 8(a1)
+; RV64IM-NEXT:    lh a1, 16(a1)
+; RV64IM-NEXT:    lui a4, 1043590
+; RV64IM-NEXT:    addiw a4, a4, -1781
+; RV64IM-NEXT:    slli a4, a4, 13
+; RV64IM-NEXT:    addi a4, a4, 1069
+; RV64IM-NEXT:    slli a4, a4, 12
+; RV64IM-NEXT:    addi a4, a4, -1959
+; RV64IM-NEXT:    slli a4, a4, 13
+; RV64IM-NEXT:    addi a4, a4, 357
+; RV64IM-NEXT:    mulh a4, a1, a4
+; RV64IM-NEXT:    add a4, a4, a1
+; RV64IM-NEXT:    srli a5, a4, 63
+; RV64IM-NEXT:    srli a4, a4, 4
+; RV64IM-NEXT:    add a4, a4, a5
+; RV64IM-NEXT:    addi a5, zero, 23
+; RV64IM-NEXT:    mul a4, a4, a5
+; RV64IM-NEXT:    sub a1, a1, a4
+; RV64IM-NEXT:    lui a4, 6413
+; RV64IM-NEXT:    addiw a4, a4, 1265
+; RV64IM-NEXT:    slli a4, a4, 13
+; RV64IM-NEXT:    addi a4, a4, 1027
+; RV64IM-NEXT:    slli a4, a4, 13
+; RV64IM-NEXT:    addi a4, a4, 1077
+; RV64IM-NEXT:    slli a4, a4, 12
+; RV64IM-NEXT:    addi a4, a4, 965
+; RV64IM-NEXT:    mulh a4, a3, a4
+; RV64IM-NEXT:    srli a5, a4, 63
+; RV64IM-NEXT:    srli a4, a4, 8
+; RV64IM-NEXT:    add a4, a4, a5
+; RV64IM-NEXT:    addi a5, zero, 654
+; RV64IM-NEXT:    mul a4, a4, a5
+; RV64IM-NEXT:    sub a3, a3, a4
+; RV64IM-NEXT:    lui a4, 12375
+; RV64IM-NEXT:    addiw a4, a4, -575
+; RV64IM-NEXT:    slli a4, a4, 12
+; RV64IM-NEXT:    addi a4, a4, 883
+; RV64IM-NEXT:    slli a4, a4, 13
+; RV64IM-NEXT:    addi a4, a4, -431
+; RV64IM-NEXT:    slli a4, a4, 12
+; RV64IM-NEXT:    addi a4, a4, 1959
+; RV64IM-NEXT:    mulh a4, a2, a4
+; RV64IM-NEXT:    srli a5, a4, 63
+; RV64IM-NEXT:    srli a4, a4, 11
+; RV64IM-NEXT:    add a4, a4, a5
+; RV64IM-NEXT:    lui a5, 1
+; RV64IM-NEXT:    addiw a5, a5, 1327
+; RV64IM-NEXT:    mul a4, a4, a5
+; RV64IM-NEXT:    sub a2, a2, a4
+; RV64IM-NEXT:    sh zero, 0(a0)
+; RV64IM-NEXT:    sh a2, 6(a0)
+; RV64IM-NEXT:    sh a3, 2(a0)
+; RV64IM-NEXT:    sh a1, 4(a0)
+; RV64IM-NEXT:    ret
+  %1 = srem <4 x i16> %x, <i16 1, i16 654, i16 23, i16 5423>
+  ret <4 x i16> %1
+}
+
+; Don't fold if the divisor is 2^15.
+define <4 x i16> @dont_fold_urem_i16_smax(<4 x i16> %x) nounwind {
+; RV32I-LABEL: dont_fold_urem_i16_smax:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -32
+; RV32I-NEXT:    sw ra, 28(sp)
+; RV32I-NEXT:    sw s0, 24(sp)
+; RV32I-NEXT:    sw s1, 20(sp)
+; RV32I-NEXT:    sw s2, 16(sp)
+; RV32I-NEXT:    sw s3, 12(sp)
+; RV32I-NEXT:    lh a2, 4(a1)
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lh s2, 12(a1)
+; RV32I-NEXT:    lh a0, 8(a1)
+; RV32I-NEXT:    slli a1, a2, 16
+; RV32I-NEXT:    srai a1, a1, 31
+; RV32I-NEXT:    srli a1, a1, 17
+; RV32I-NEXT:    add a1, a2, a1
+; RV32I-NEXT:    lui a3, 8
+; RV32I-NEXT:    and a1, a1, a3
+; RV32I-NEXT:    sub s3, a2, a1
+; RV32I-NEXT:    addi a1, zero, 23
+; RV32I-NEXT:    call __modsi3
+; RV32I-NEXT:    mv s1, a0
+; RV32I-NEXT:    lui a0, 1
+; RV32I-NEXT:    addi a1, a0, 1327
+; RV32I-NEXT:    mv a0, s2
+; RV32I-NEXT:    call __modsi3
+; RV32I-NEXT:    sh zero, 0(s0)
+; RV32I-NEXT:    sh a0, 6(s0)
+; RV32I-NEXT:    sh s1, 4(s0)
+; RV32I-NEXT:    sh s3, 2(s0)
+; RV32I-NEXT:    lw s3, 12(sp)
+; RV32I-NEXT:    lw s2, 16(sp)
+; RV32I-NEXT:    lw s1, 20(sp)
+; RV32I-NEXT:    lw s0, 24(sp)
+; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    addi sp, sp, 32
+; RV32I-NEXT:    ret
+;
+; RV32IM-LABEL: dont_fold_urem_i16_smax:
+; RV32IM:       # %bb.0:
+; RV32IM-NEXT:    lh a2, 4(a1)
+; RV32IM-NEXT:    slli a6, a2, 16
+; RV32IM-NEXT:    lh a4, 8(a1)
+; RV32IM-NEXT:    lh a1, 12(a1)
+; RV32IM-NEXT:    lui a5, 729444
+; RV32IM-NEXT:    addi a5, a5, 713
+; RV32IM-NEXT:    mulh a5, a4, a5
+; RV32IM-NEXT:    add a5, a5, a4
+; RV32IM-NEXT:    srli a3, a5, 31
+; RV32IM-NEXT:    srli a5, a5, 4
+; RV32IM-NEXT:    add a3, a5, a3
+; RV32IM-NEXT:    addi a5, zero, 23
+; RV32IM-NEXT:    mul a3, a3, a5
+; RV32IM-NEXT:    sub a3, a4, a3
+; RV32IM-NEXT:    lui a4, 395996
+; RV32IM-NEXT:    addi a4, a4, -2009
+; RV32IM-NEXT:    mulh a4, a1, a4
+; RV32IM-NEXT:    srli a5, a4, 31
+; RV32IM-NEXT:    srli a4, a4, 11
+; RV32IM-NEXT:    add a4, a4, a5
+; RV32IM-NEXT:    lui a5, 1
+; RV32IM-NEXT:    addi a5, a5, 1327
+; RV32IM-NEXT:    mul a4, a4, a5
+; RV32IM-NEXT:    sub a1, a1, a4
+; RV32IM-NEXT:    srai a4, a6, 31
+; RV32IM-NEXT:    srli a4, a4, 17
+; RV32IM-NEXT:    add a4, a2, a4
+; RV32IM-NEXT:    lui a5, 8
+; RV32IM-NEXT:    and a4, a4, a5
+; RV32IM-NEXT:    sub a2, a2, a4
+; RV32IM-NEXT:    sh zero, 0(a0)
+; RV32IM-NEXT:    sh a1, 6(a0)
+; RV32IM-NEXT:    sh a3, 4(a0)
+; RV32IM-NEXT:    sh a2, 2(a0)
+; RV32IM-NEXT:    ret
+;
+; RV64I-LABEL: dont_fold_urem_i16_smax:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s0, 32(sp)
+; RV64I-NEXT:    sd s1, 24(sp)
+; RV64I-NEXT:    sd s2, 16(sp)
+; RV64I-NEXT:    sd s3, 8(sp)
+; RV64I-NEXT:    lh a2, 8(a1)
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    lh s2, 24(a1)
+; RV64I-NEXT:    lh a0, 16(a1)
+; RV64I-NEXT:    slli a1, a2, 48
+; RV64I-NEXT:    srai a1, a1, 63
+; RV64I-NEXT:    srli a1, a1, 49
+; RV64I-NEXT:    add a1, a2, a1
+; RV64I-NEXT:    lui a3, 8
+; RV64I-NEXT:    and a1, a1, a3
+; RV64I-NEXT:    sub s3, a2, a1
+; RV64I-NEXT:    addi a1, zero, 23
+; RV64I-NEXT:    call __moddi3
+; RV64I-NEXT:    mv s1, a0
+; RV64I-NEXT:    lui a0, 1
+; RV64I-NEXT:    addiw a1, a0, 1327
+; RV64I-NEXT:    mv a0, s2
+; RV64I-NEXT:    call __moddi3
+; RV64I-NEXT:    sh zero, 0(s0)
+; RV64I-NEXT:    sh a0, 6(s0)
+; RV64I-NEXT:    sh s1, 4(s0)
+; RV64I-NEXT:    sh s3, 2(s0)
+; RV64I-NEXT:    ld s3, 8(sp)
+; RV64I-NEXT:    ld s2, 16(sp)
+; RV64I-NEXT:    ld s1, 24(sp)
+; RV64I-NEXT:    ld s0, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IM-LABEL: dont_fold_urem_i16_smax:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    lh a2, 8(a1)
+; RV64IM-NEXT:    slli a6, a2, 48
+; RV64IM-NEXT:    lh a4, 24(a1)
+; RV64IM-NEXT:    lh a1, 16(a1)
+; RV64IM-NEXT:    lui a5, 1043590
+; RV64IM-NEXT:    addiw a5, a5, -1781
+; RV64IM-NEXT:    slli a5, a5, 13
+; RV64IM-NEXT:    addi a5, a5, 1069
+; RV64IM-NEXT:    slli a5, a5, 12
+; RV64IM-NEXT:    addi a5, a5, -1959
+; RV64IM-NEXT:    slli a5, a5, 13
+; RV64IM-NEXT:    addi a5, a5, 357
+; RV64IM-NEXT:    mulh a5, a1, a5
+; RV64IM-NEXT:    add a5, a5, a1
+; RV64IM-NEXT:    srli a3, a5, 63
+; RV64IM-NEXT:    srli a5, a5, 4
+; RV64IM-NEXT:    add a3, a5, a3
+; RV64IM-NEXT:    addi a5, zero, 23
+; RV64IM-NEXT:    mul a3, a3, a5
+; RV64IM-NEXT:    sub a1, a1, a3
+; RV64IM-NEXT:    lui a3, 12375
+; RV64IM-NEXT:    addiw a3, a3, -575
+; RV64IM-NEXT:    slli a3, a3, 12
+; RV64IM-NEXT:    addi a3, a3, 883
+; RV64IM-NEXT:    slli a3, a3, 13
+; RV64IM-NEXT:    addi a3, a3, -431
+; RV64IM-NEXT:    slli a3, a3, 12
+; RV64IM-NEXT:    addi a3, a3, 1959
+; RV64IM-NEXT:    mulh a3, a4, a3
+; RV64IM-NEXT:    srli a5, a3, 63
+; RV64IM-NEXT:    srli a3, a3, 11
+; RV64IM-NEXT:    add a3, a3, a5
+; RV64IM-NEXT:    lui a5, 1
+; RV64IM-NEXT:    addiw a5, a5, 1327
+; RV64IM-NEXT:    mul a3, a3, a5
+; RV64IM-NEXT:    sub a3, a4, a3
+; RV64IM-NEXT:    srai a4, a6, 63
+; RV64IM-NEXT:    srli a4, a4, 49
+; RV64IM-NEXT:    add a4, a2, a4
+; RV64IM-NEXT:    lui a5, 8
+; RV64IM-NEXT:    and a4, a4, a5
+; RV64IM-NEXT:    sub a2, a2, a4
+; RV64IM-NEXT:    sh zero, 0(a0)
+; RV64IM-NEXT:    sh a2, 2(a0)
+; RV64IM-NEXT:    sh a3, 6(a0)
+; RV64IM-NEXT:    sh a1, 4(a0)
+; RV64IM-NEXT:    ret
+  %1 = srem <4 x i16> %x, <i16 1, i16 32768, i16 23, i16 5423>
+  ret <4 x i16> %1
+}
+
+; Don't fold i64 srem.
+define <4 x i64> @dont_fold_srem_i64(<4 x i64> %x) nounwind {
+; RV32I-LABEL: dont_fold_srem_i64:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -48
+; RV32I-NEXT:    sw ra, 44(sp)
+; RV32I-NEXT:    sw s0, 40(sp)
+; RV32I-NEXT:    sw s1, 36(sp)
+; RV32I-NEXT:    sw s2, 32(sp)
+; RV32I-NEXT:    sw s3, 28(sp)
+; RV32I-NEXT:    sw s4, 24(sp)
+; RV32I-NEXT:    sw s5, 20(sp)
+; RV32I-NEXT:    sw s6, 16(sp)
+; RV32I-NEXT:    sw s7, 12(sp)
+; RV32I-NEXT:    sw s8, 8(sp)
+; RV32I-NEXT:    sw s9, 4(sp)
+; RV32I-NEXT:    lw s2, 24(a1)
+; RV32I-NEXT:    lw s3, 28(a1)
+; RV32I-NEXT:    lw s4, 16(a1)
+; RV32I-NEXT:    lw s5, 20(a1)
+; RV32I-NEXT:    lw s6, 8(a1)
+; RV32I-NEXT:    lw s1, 12(a1)
+; RV32I-NEXT:    lw a3, 0(a1)
+; RV32I-NEXT:    lw a1, 4(a1)
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    addi a2, zero, 1
+; RV32I-NEXT:    mv a0, a3
+; RV32I-NEXT:    mv a3, zero
+; RV32I-NEXT:    call __moddi3
+; RV32I-NEXT:    mv s7, a0
+; RV32I-NEXT:    mv s8, a1
+; RV32I-NEXT:    addi a2, zero, 654
+; RV32I-NEXT:    mv a0, s6
+; RV32I-NEXT:    mv a1, s1
+; RV32I-NEXT:    mv a3, zero
+; RV32I-NEXT:    call __moddi3
+; RV32I-NEXT:    mv s6, a0
+; RV32I-NEXT:    mv s9, a1
+; RV32I-NEXT:    addi a2, zero, 23
+; RV32I-NEXT:    mv a0, s4
+; RV32I-NEXT:    mv a1, s5
+; RV32I-NEXT:    mv a3, zero
+; RV32I-NEXT:    call __moddi3
+; RV32I-NEXT:    mv s4, a0
+; RV32I-NEXT:    mv s1, a1
+; RV32I-NEXT:    lui a0, 1
+; RV32I-NEXT:    addi a2, a0, 1327
+; RV32I-NEXT:    mv a0, s2
+; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    mv a3, zero
+; RV32I-NEXT:    call __moddi3
+; RV32I-NEXT:    sw a1, 28(s0)
+; RV32I-NEXT:    sw a0, 24(s0)
+; RV32I-NEXT:    sw s1, 20(s0)
+; RV32I-NEXT:    sw s4, 16(s0)
+; RV32I-NEXT:    sw s9, 12(s0)
+; RV32I-NEXT:    sw s6, 8(s0)
+; RV32I-NEXT:    sw s8, 4(s0)
+; RV32I-NEXT:    sw s7, 0(s0)
+; RV32I-NEXT:    lw s9, 4(sp)
+; RV32I-NEXT:    lw s8, 8(sp)
+; RV32I-NEXT:    lw s7, 12(sp)
+; RV32I-NEXT:    lw s6, 16(sp)
+; RV32I-NEXT:    lw s5, 20(sp)
+; RV32I-NEXT:    lw s4, 24(sp)
+; RV32I-NEXT:    lw s3, 28(sp)
+; RV32I-NEXT:    lw s2, 32(sp)
+; RV32I-NEXT:    lw s1, 36(sp)
+; RV32I-NEXT:    lw s0, 40(sp)
+; RV32I-NEXT:    lw ra, 44(sp)
+; RV32I-NEXT:    addi sp, sp, 48
+; RV32I-NEXT:    ret
+;
+; RV32IM-LABEL: dont_fold_srem_i64:
+; RV32IM:       # %bb.0:
+; RV32IM-NEXT:    addi sp, sp, -48
+; RV32IM-NEXT:    sw ra, 44(sp)
+; RV32IM-NEXT:    sw s0, 40(sp)
+; RV32IM-NEXT:    sw s1, 36(sp)
+; RV32IM-NEXT:    sw s2, 32(sp)
+; RV32IM-NEXT:    sw s3, 28(sp)
+; RV32IM-NEXT:    sw s4, 24(sp)
+; RV32IM-NEXT:    sw s5, 20(sp)
+; RV32IM-NEXT:    sw s6, 16(sp)
+; RV32IM-NEXT:    sw s7, 12(sp)
+; RV32IM-NEXT:    sw s8, 8(sp)
+; RV32IM-NEXT:    sw s9, 4(sp)
+; RV32IM-NEXT:    lw s2, 24(a1)
+; RV32IM-NEXT:    lw s3, 28(a1)
+; RV32IM-NEXT:    lw s4, 16(a1)
+; RV32IM-NEXT:    lw s5, 20(a1)
+; RV32IM-NEXT:    lw s6, 8(a1)
+; RV32IM-NEXT:    lw s1, 12(a1)
+; RV32IM-NEXT:    lw a3, 0(a1)
+; RV32IM-NEXT:    lw a1, 4(a1)
+; RV32IM-NEXT:    mv s0, a0
+; RV32IM-NEXT:    addi a2, zero, 1
+; RV32IM-NEXT:    mv a0, a3
+; RV32IM-NEXT:    mv a3, zero
+; RV32IM-NEXT:    call __moddi3
+; RV32IM-NEXT:    mv s7, a0
+; RV32IM-NEXT:    mv s8, a1
+; RV32IM-NEXT:    addi a2, zero, 654
+; RV32IM-NEXT:    mv a0, s6
+; RV32IM-NEXT:    mv a1, s1
+; RV32IM-NEXT:    mv a3, zero
+; RV32IM-NEXT:    call __moddi3
+; RV32IM-NEXT:    mv s6, a0
+; RV32IM-NEXT:    mv s9, a1
+; RV32IM-NEXT:    addi a2, zero, 23
+; RV32IM-NEXT:    mv a0, s4
+; RV32IM-NEXT:    mv a1, s5
+; RV32IM-NEXT:    mv a3, zero
+; RV32IM-NEXT:    call __moddi3
+; RV32IM-NEXT:    mv s4, a0
+; RV32IM-NEXT:    mv s1, a1
+; RV32IM-NEXT:    lui a0, 1
+; RV32IM-NEXT:    addi a2, a0, 1327
+; RV32IM-NEXT:    mv a0, s2
+; RV32IM-NEXT:    mv a1, s3
+; RV32IM-NEXT:    mv a3, zero
+; RV32IM-NEXT:    call __moddi3
+; RV32IM-NEXT:    sw a1, 28(s0)
+; RV32IM-NEXT:    sw a0, 24(s0)
+; RV32IM-NEXT:    sw s1, 20(s0)
+; RV32IM-NEXT:    sw s4, 16(s0)
+; RV32IM-NEXT:    sw s9, 12(s0)
+; RV32IM-NEXT:    sw s6, 8(s0)
+; RV32IM-NEXT:    sw s8, 4(s0)
+; RV32IM-NEXT:    sw s7, 0(s0)
+; RV32IM-NEXT:    lw s9, 4(sp)
+; RV32IM-NEXT:    lw s8, 8(sp)
+; RV32IM-NEXT:    lw s7, 12(sp)
+; RV32IM-NEXT:    lw s6, 16(sp)
+; RV32IM-NEXT:    lw s5, 20(sp)
+; RV32IM-NEXT:    lw s4, 24(sp)
+; RV32IM-NEXT:    lw s3, 28(sp)
+; RV32IM-NEXT:    lw s2, 32(sp)
+; RV32IM-NEXT:    lw s1, 36(sp)
+; RV32IM-NEXT:    lw s0, 40(sp)
+; RV32IM-NEXT:    lw ra, 44(sp)
+; RV32IM-NEXT:    addi sp, sp, 48
+; RV32IM-NEXT:    ret
+;
+; RV64I-LABEL: dont_fold_srem_i64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s0, 32(sp)
+; RV64I-NEXT:    sd s1, 24(sp)
+; RV64I-NEXT:    sd s2, 16(sp)
+; RV64I-NEXT:    sd s3, 8(sp)
+; RV64I-NEXT:    ld s2, 24(a1)
+; RV64I-NEXT:    ld s1, 16(a1)
+; RV64I-NEXT:    ld a2, 8(a1)
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    addi a1, zero, 654
+; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    call __moddi3
+; RV64I-NEXT:    mv s3, a0
+; RV64I-NEXT:    addi a1, zero, 23
+; RV64I-NEXT:    mv a0, s1
+; RV64I-NEXT:    call __moddi3
+; RV64I-NEXT:    mv s1, a0
+; RV64I-NEXT:    lui a0, 1
+; RV64I-NEXT:    addiw a1, a0, 1327
+; RV64I-NEXT:    mv a0, s2
+; RV64I-NEXT:    call __moddi3
+; RV64I-NEXT:    sd zero, 0(s0)
+; RV64I-NEXT:    sd a0, 24(s0)
+; RV64I-NEXT:    sd s1, 16(s0)
+; RV64I-NEXT:    sd s3, 8(s0)
+; RV64I-NEXT:    ld s3, 8(sp)
+; RV64I-NEXT:    ld s2, 16(sp)
+; RV64I-NEXT:    ld s1, 24(sp)
+; RV64I-NEXT:    ld s0, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IM-LABEL: dont_fold_srem_i64:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    ld a2, 24(a1)
+; RV64IM-NEXT:    ld a3, 8(a1)
+; RV64IM-NEXT:    ld a1, 16(a1)
+; RV64IM-NEXT:    lui a4, 1043590
+; RV64IM-NEXT:    addiw a4, a4, -1781
+; RV64IM-NEXT:    slli a4, a4, 13
+; RV64IM-NEXT:    addi a4, a4, 1069
+; RV64IM-NEXT:    slli a4, a4, 12
+; RV64IM-NEXT:    addi a4, a4, -1959
+; RV64IM-NEXT:    slli a4, a4, 13
+; RV64IM-NEXT:    addi a4, a4, 357
+; RV64IM-NEXT:    mulh a4, a1, a4
+; RV64IM-NEXT:    add a4, a4, a1
+; RV64IM-NEXT:    srli a5, a4, 63
+; RV64IM-NEXT:    srai a4, a4, 4
+; RV64IM-NEXT:    add a4, a4, a5
+; RV64IM-NEXT:    addi a5, zero, 23
+; RV64IM-NEXT:    mul a4, a4, a5
+; RV64IM-NEXT:    sub a1, a1, a4
+; RV64IM-NEXT:    lui a4, 6413
+; RV64IM-NEXT:    addiw a4, a4, 1265
+; RV64IM-NEXT:    slli a4, a4, 13
+; RV64IM-NEXT:    addi a4, a4, 1027
+; RV64IM-NEXT:    slli a4, a4, 13
+; RV64IM-NEXT:    addi a4, a4, 1077
+; RV64IM-NEXT:    slli a4, a4, 12
+; RV64IM-NEXT:    addi a4, a4, 965
+; RV64IM-NEXT:    mulh a4, a3, a4
+; RV64IM-NEXT:    srli a5, a4, 63
+; RV64IM-NEXT:    srai a4, a4, 8
+; RV64IM-NEXT:    add a4, a4, a5
+; RV64IM-NEXT:    addi a5, zero, 654
+; RV64IM-NEXT:    mul a4, a4, a5
+; RV64IM-NEXT:    sub a3, a3, a4
+; RV64IM-NEXT:    lui a4, 12375
+; RV64IM-NEXT:    addiw a4, a4, -575
+; RV64IM-NEXT:    slli a4, a4, 12
+; RV64IM-NEXT:    addi a4, a4, 883
+; RV64IM-NEXT:    slli a4, a4, 13
+; RV64IM-NEXT:    addi a4, a4, -431
+; RV64IM-NEXT:    slli a4, a4, 12
+; RV64IM-NEXT:    addi a4, a4, 1959
+; RV64IM-NEXT:    mulh a4, a2, a4
+; RV64IM-NEXT:    srli a5, a4, 63
+; RV64IM-NEXT:    srai a4, a4, 11
+; RV64IM-NEXT:    add a4, a4, a5
+; RV64IM-NEXT:    lui a5, 1
+; RV64IM-NEXT:    addiw a5, a5, 1327
+; RV64IM-NEXT:    mul a4, a4, a5
+; RV64IM-NEXT:    sub a2, a2, a4
+; RV64IM-NEXT:    sd zero, 0(a0)
+; RV64IM-NEXT:    sd a2, 24(a0)
+; RV64IM-NEXT:    sd a3, 8(a0)
+; RV64IM-NEXT:    sd a1, 16(a0)
+; RV64IM-NEXT:    ret
+  %1 = srem <4 x i64> %x, <i64 1, i64 654, i64 23, i64 5423>
+  ret <4 x i64> %1
+}
diff --git a/llvm/test/CodeGen/RISCV/stack-realignment-with-variable-sized-objects.ll b/llvm/test/CodeGen/RISCV/stack-realignment-with-variable-sized-objects.ll
new file mode 100644
index 000000000..293eab03b
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/stack-realignment-with-variable-sized-objects.ll
@@ -0,0 +1,72 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefix=RV32I
+; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefix=RV64I
+
+declare void @callee(i8*, i32*)
+
+define void @caller(i32 %n) {
+; RV32I-LABEL: caller:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -128
+; RV32I-NEXT:    .cfi_def_cfa_offset 128
+; RV32I-NEXT:    sw ra, 124(sp)
+; RV32I-NEXT:    sw s0, 120(sp)
+; RV32I-NEXT:    sw s1, 116(sp)
+; RV32I-NEXT:    .cfi_offset ra, -4
+; RV32I-NEXT:    .cfi_offset s0, -8
+; RV32I-NEXT:    .cfi_offset s1, -12
+; RV32I-NEXT:    addi s0, sp, 128
+; RV32I-NEXT:    .cfi_def_cfa s0, 0
+; RV32I-NEXT:    andi sp, sp, -64
+; RV32I-NEXT:    mv s1, sp
+; RV32I-NEXT:    addi a0, a0, 15
+; RV32I-NEXT:    andi a0, a0, -16
+; RV32I-NEXT:    sub a0, sp, a0
+; RV32I-NEXT:    mv sp, a0
+; RV32I-NEXT:    addi a1, s1, 64
+; RV32I-NEXT:    call callee
+; RV32I-NEXT:    addi sp, s0, -128
+; RV32I-NEXT:    lw s1, 116(sp)
+; RV32I-NEXT:    lw s0, 120(sp)
+; RV32I-NEXT:    lw ra, 124(sp)
+; RV32I-NEXT:    addi sp, sp, 128
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: caller:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -128
+; RV64I-NEXT:    .cfi_def_cfa_offset 128
+; RV64I-NEXT:    sd ra, 120(sp)
+; RV64I-NEXT:    sd s0, 112(sp)
+; RV64I-NEXT:    sd s1, 104(sp)
+; RV64I-NEXT:    .cfi_offset ra, -8
+; RV64I-NEXT:    .cfi_offset s0, -16
+; RV64I-NEXT:    .cfi_offset s1, -24
+; RV64I-NEXT:    addi s0, sp, 128
+; RV64I-NEXT:    .cfi_def_cfa s0, 0
+; RV64I-NEXT:    andi sp, sp, -64
+; RV64I-NEXT:    mv s1, sp
+; RV64I-NEXT:    slli a0, a0, 32
+; RV64I-NEXT:    srli a0, a0, 32
+; RV64I-NEXT:    addi a0, a0, 15
+; RV64I-NEXT:    addi a1, zero, 1
+; RV64I-NEXT:    slli a1, a1, 33
+; RV64I-NEXT:    addi a1, a1, -16
+; RV64I-NEXT:    and a0, a0, a1
+; RV64I-NEXT:    sub a0, sp, a0
+; RV64I-NEXT:    mv sp, a0
+; RV64I-NEXT:    addi a1, s1, 64
+; RV64I-NEXT:    call callee
+; RV64I-NEXT:    addi sp, s0, -128
+; RV64I-NEXT:    ld s1, 104(sp)
+; RV64I-NEXT:    ld s0, 112(sp)
+; RV64I-NEXT:    ld ra, 120(sp)
+; RV64I-NEXT:    addi sp, sp, 128
+; RV64I-NEXT:    ret
+  %1 = alloca i8, i32 %n
+  %2 = alloca i32, align 64
+  call void @callee(i8* %1, i32 *%2)
+  ret void
+}
diff --git a/llvm/test/CodeGen/RISCV/stack-realignment.ll b/llvm/test/CodeGen/RISCV/stack-realignment.ll
index 252a099d0..c15e6e14b 100644
--- a/llvm/test/CodeGen/RISCV/stack-realignment.ll
+++ b/llvm/test/CodeGen/RISCV/stack-realignment.ll
@@ -309,20 +309,11 @@ define void @caller_no_realign512() nounwind "no-realign-stack" {
 define void @caller1024() nounwind {
 ; RV32I-LABEL: caller1024:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    lui a0, 1
-; RV32I-NEXT:    addi a0, a0, -1024
-; RV32I-NEXT:    sub sp, sp, a0
-; RV32I-NEXT:    lui a0, 1
-; RV32I-NEXT:    addi a0, a0, -1028
-; RV32I-NEXT:    add a0, sp, a0
-; RV32I-NEXT:    sw ra, 0(a0)
-; RV32I-NEXT:    lui a0, 1
-; RV32I-NEXT:    addi a0, a0, -1032
-; RV32I-NEXT:    add a0, sp, a0
-; RV32I-NEXT:    sw s0, 0(a0)
-; RV32I-NEXT:    lui a0, 1
-; RV32I-NEXT:    addi a0, a0, -1024
-; RV32I-NEXT:    add s0, sp, a0
+; RV32I-NEXT:    addi sp, sp, -2032
+; RV32I-NEXT:    sw ra, 2028(sp)
+; RV32I-NEXT:    sw s0, 2024(sp)
+; RV32I-NEXT:    addi s0, sp, 2032
+; RV32I-NEXT:    addi sp, sp, -1040
 ; RV32I-NEXT:    andi sp, sp, -1024
 ; RV32I-NEXT:    lui a0, 1
 ; RV32I-NEXT:    addi a0, a0, -2048
@@ -332,55 +323,32 @@ define void @caller1024() nounwind {
 ; RV32I-NEXT:    lui a0, 1
 ; RV32I-NEXT:    addi a0, a0, -1024
 ; RV32I-NEXT:    sub sp, s0, a0
-; RV32I-NEXT:    lui a0, 1
-; RV32I-NEXT:    addi a0, a0, -1032
-; RV32I-NEXT:    add a0, sp, a0
-; RV32I-NEXT:    lw s0, 0(a0)
-; RV32I-NEXT:    lui a0, 1
-; RV32I-NEXT:    addi a0, a0, -1028
-; RV32I-NEXT:    add a0, sp, a0
-; RV32I-NEXT:    lw ra, 0(a0)
-; RV32I-NEXT:    lui a0, 1
-; RV32I-NEXT:    addi a0, a0, -1024
-; RV32I-NEXT:    add sp, sp, a0
+; RV32I-NEXT:    addi sp, sp, 1040
+; RV32I-NEXT:    lw s0, 2024(sp)
+; RV32I-NEXT:    lw ra, 2028(sp)
+; RV32I-NEXT:    addi sp, sp, 2032
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: caller1024:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    lui a0, 1
-; RV64I-NEXT:    addi a0, a0, -1024
-; RV64I-NEXT:    sub sp, sp, a0
-; RV64I-NEXT:    lui a0, 1
-; RV64I-NEXT:    addi a0, a0, -1032
-; RV64I-NEXT:    add a0, sp, a0
-; RV64I-NEXT:    sd ra, 0(a0)
-; RV64I-NEXT:    lui a0, 1
-; RV64I-NEXT:    addi a0, a0, -1040
-; RV64I-NEXT:    add a0, sp, a0
-; RV64I-NEXT:    sd s0, 0(a0)
-; RV64I-NEXT:    lui a0, 1
-; RV64I-NEXT:    addi a0, a0, -1024
-; RV64I-NEXT:    add s0, sp, a0
+; RV64I-NEXT:    addi sp, sp, -2032
+; RV64I-NEXT:    sd ra, 2024(sp)
+; RV64I-NEXT:    sd s0, 2016(sp)
+; RV64I-NEXT:    addi s0, sp, 2032
+; RV64I-NEXT:    addi sp, sp, -1040
 ; RV64I-NEXT:    andi sp, sp, -1024
 ; RV64I-NEXT:    lui a0, 1
-; RV64I-NEXT:    addi a0, a0, -2048
+; RV64I-NEXT:    addiw a0, a0, -2048
 ; RV64I-NEXT:    add a0, sp, a0
 ; RV64I-NEXT:    mv a0, a0
 ; RV64I-NEXT:    call callee
 ; RV64I-NEXT:    lui a0, 1
-; RV64I-NEXT:    addi a0, a0, -1024
+; RV64I-NEXT:    addiw a0, a0, -1024
 ; RV64I-NEXT:    sub sp, s0, a0
-; RV64I-NEXT:    lui a0, 1
-; RV64I-NEXT:    addi a0, a0, -1040
-; RV64I-NEXT:    add a0, sp, a0
-; RV64I-NEXT:    ld s0, 0(a0)
-; RV64I-NEXT:    lui a0, 1
-; RV64I-NEXT:    addi a0, a0, -1032
-; RV64I-NEXT:    add a0, sp, a0
-; RV64I-NEXT:    ld ra, 0(a0)
-; RV64I-NEXT:    lui a0, 1
-; RV64I-NEXT:    addi a0, a0, -1024
-; RV64I-NEXT:    add sp, sp, a0
+; RV64I-NEXT:    addi sp, sp, 1040
+; RV64I-NEXT:    ld s0, 2016(sp)
+; RV64I-NEXT:    ld ra, 2024(sp)
+; RV64I-NEXT:    addi sp, sp, 2032
 ; RV64I-NEXT:    ret
   %1 = alloca i8, align 1024
   call void @callee(i8* %1)
@@ -415,23 +383,15 @@ define void @caller_no_realign1024() nounwind "no-realign-stack" {
 define void @caller2048() nounwind {
 ; RV32I-LABEL: caller2048:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    lui a0, 2
-; RV32I-NEXT:    addi a0, a0, -2048
-; RV32I-NEXT:    sub sp, sp, a0
+; RV32I-NEXT:    addi sp, sp, -2032
+; RV32I-NEXT:    sw ra, 2028(sp)
+; RV32I-NEXT:    sw s0, 2024(sp)
+; RV32I-NEXT:    addi s0, sp, 2032
 ; RV32I-NEXT:    lui a0, 1
-; RV32I-NEXT:    addi a0, a0, 2044
-; RV32I-NEXT:    add a0, sp, a0
-; RV32I-NEXT:    sw ra, 0(a0)
-; RV32I-NEXT:    lui a0, 1
-; RV32I-NEXT:    addi a0, a0, 2040
-; RV32I-NEXT:    add a0, sp, a0
-; RV32I-NEXT:    sw s0, 0(a0)
-; RV32I-NEXT:    lui a0, 2
-; RV32I-NEXT:    addi a0, a0, -2048
-; RV32I-NEXT:    add s0, sp, a0
+; RV32I-NEXT:    addi a0, a0, 16
+; RV32I-NEXT:    sub sp, sp, a0
 ; RV32I-NEXT:    andi sp, sp, -2048
 ; RV32I-NEXT:    lui a0, 1
-; RV32I-NEXT:    mv a0, a0
 ; RV32I-NEXT:    add a0, sp, a0
 ; RV32I-NEXT:    mv a0, a0
 ; RV32I-NEXT:    call callee
@@ -439,54 +399,36 @@ define void @caller2048() nounwind {
 ; RV32I-NEXT:    addi a0, a0, -2048
 ; RV32I-NEXT:    sub sp, s0, a0
 ; RV32I-NEXT:    lui a0, 1
-; RV32I-NEXT:    addi a0, a0, 2040
-; RV32I-NEXT:    add a0, sp, a0
-; RV32I-NEXT:    lw s0, 0(a0)
-; RV32I-NEXT:    lui a0, 1
-; RV32I-NEXT:    addi a0, a0, 2044
-; RV32I-NEXT:    add a0, sp, a0
-; RV32I-NEXT:    lw ra, 0(a0)
-; RV32I-NEXT:    lui a0, 2
-; RV32I-NEXT:    addi a0, a0, -2048
+; RV32I-NEXT:    addi a0, a0, 16
 ; RV32I-NEXT:    add sp, sp, a0
+; RV32I-NEXT:    lw s0, 2024(sp)
+; RV32I-NEXT:    lw ra, 2028(sp)
+; RV32I-NEXT:    addi sp, sp, 2032
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: caller2048:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    lui a0, 2
-; RV64I-NEXT:    addi a0, a0, -2048
-; RV64I-NEXT:    sub sp, sp, a0
-; RV64I-NEXT:    lui a0, 1
-; RV64I-NEXT:    addi a0, a0, 2040
-; RV64I-NEXT:    add a0, sp, a0
-; RV64I-NEXT:    sd ra, 0(a0)
+; RV64I-NEXT:    addi sp, sp, -2032
+; RV64I-NEXT:    sd ra, 2024(sp)
+; RV64I-NEXT:    sd s0, 2016(sp)
+; RV64I-NEXT:    addi s0, sp, 2032
 ; RV64I-NEXT:    lui a0, 1
-; RV64I-NEXT:    addi a0, a0, 2032
-; RV64I-NEXT:    add a0, sp, a0
-; RV64I-NEXT:    sd s0, 0(a0)
-; RV64I-NEXT:    lui a0, 2
-; RV64I-NEXT:    addi a0, a0, -2048
-; RV64I-NEXT:    add s0, sp, a0
+; RV64I-NEXT:    addiw a0, a0, 16
+; RV64I-NEXT:    sub sp, sp, a0
 ; RV64I-NEXT:    andi sp, sp, -2048
 ; RV64I-NEXT:    lui a0, 1
-; RV64I-NEXT:    mv a0, a0
 ; RV64I-NEXT:    add a0, sp, a0
 ; RV64I-NEXT:    mv a0, a0
 ; RV64I-NEXT:    call callee
 ; RV64I-NEXT:    lui a0, 2
-; RV64I-NEXT:    addi a0, a0, -2048
+; RV64I-NEXT:    addiw a0, a0, -2048
 ; RV64I-NEXT:    sub sp, s0, a0
 ; RV64I-NEXT:    lui a0, 1
-; RV64I-NEXT:    addi a0, a0, 2032
-; RV64I-NEXT:    add a0, sp, a0
-; RV64I-NEXT:    ld s0, 0(a0)
-; RV64I-NEXT:    lui a0, 1
-; RV64I-NEXT:    addi a0, a0, 2040
-; RV64I-NEXT:    add a0, sp, a0
-; RV64I-NEXT:    ld ra, 0(a0)
-; RV64I-NEXT:    lui a0, 2
-; RV64I-NEXT:    addi a0, a0, -2048
+; RV64I-NEXT:    addiw a0, a0, 16
 ; RV64I-NEXT:    add sp, sp, a0
+; RV64I-NEXT:    ld s0, 2016(sp)
+; RV64I-NEXT:    ld ra, 2024(sp)
+; RV64I-NEXT:    addi sp, sp, 2032
 ; RV64I-NEXT:    ret
   %1 = alloca i8, align 2048
   call void @callee(i8* %1)
@@ -521,80 +463,52 @@ define void @caller_no_realign2048() nounwind "no-realign-stack" {
 define void @caller4096() nounwind {
 ; RV32I-LABEL: caller4096:
 ; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -2032
+; RV32I-NEXT:    sw ra, 2028(sp)
+; RV32I-NEXT:    sw s0, 2024(sp)
+; RV32I-NEXT:    addi s0, sp, 2032
 ; RV32I-NEXT:    lui a0, 3
-; RV32I-NEXT:    mv a0, a0
+; RV32I-NEXT:    addi a0, a0, -2032
 ; RV32I-NEXT:    sub sp, sp, a0
-; RV32I-NEXT:    lui a0, 3
-; RV32I-NEXT:    addi a0, a0, -4
-; RV32I-NEXT:    add a0, sp, a0
-; RV32I-NEXT:    sw ra, 0(a0)
-; RV32I-NEXT:    lui a0, 3
-; RV32I-NEXT:    addi a0, a0, -8
-; RV32I-NEXT:    add a0, sp, a0
-; RV32I-NEXT:    sw s0, 0(a0)
-; RV32I-NEXT:    lui a0, 3
-; RV32I-NEXT:    mv a0, a0
-; RV32I-NEXT:    add s0, sp, a0
 ; RV32I-NEXT:    srli a0, sp, 12
 ; RV32I-NEXT:    slli sp, a0, 12
 ; RV32I-NEXT:    lui a0, 2
-; RV32I-NEXT:    mv a0, a0
 ; RV32I-NEXT:    add a0, sp, a0
 ; RV32I-NEXT:    mv a0, a0
 ; RV32I-NEXT:    call callee
 ; RV32I-NEXT:    lui a0, 3
-; RV32I-NEXT:    mv a0, a0
 ; RV32I-NEXT:    sub sp, s0, a0
 ; RV32I-NEXT:    lui a0, 3
-; RV32I-NEXT:    addi a0, a0, -8
-; RV32I-NEXT:    add a0, sp, a0
-; RV32I-NEXT:    lw s0, 0(a0)
-; RV32I-NEXT:    lui a0, 3
-; RV32I-NEXT:    addi a0, a0, -4
-; RV32I-NEXT:    add a0, sp, a0
-; RV32I-NEXT:    lw ra, 0(a0)
-; RV32I-NEXT:    lui a0, 3
-; RV32I-NEXT:    mv a0, a0
+; RV32I-NEXT:    addi a0, a0, -2032
 ; RV32I-NEXT:    add sp, sp, a0
+; RV32I-NEXT:    lw s0, 2024(sp)
+; RV32I-NEXT:    lw ra, 2028(sp)
+; RV32I-NEXT:    addi sp, sp, 2032
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: caller4096:
 ; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -2032
+; RV64I-NEXT:    sd ra, 2024(sp)
+; RV64I-NEXT:    sd s0, 2016(sp)
+; RV64I-NEXT:    addi s0, sp, 2032
 ; RV64I-NEXT:    lui a0, 3
-; RV64I-NEXT:    mv a0, a0
+; RV64I-NEXT:    addiw a0, a0, -2032
 ; RV64I-NEXT:    sub sp, sp, a0
-; RV64I-NEXT:    lui a0, 3
-; RV64I-NEXT:    addi a0, a0, -8
-; RV64I-NEXT:    add a0, sp, a0
-; RV64I-NEXT:    sd ra, 0(a0)
-; RV64I-NEXT:    lui a0, 3
-; RV64I-NEXT:    addi a0, a0, -16
-; RV64I-NEXT:    add a0, sp, a0
-; RV64I-NEXT:    sd s0, 0(a0)
-; RV64I-NEXT:    lui a0, 3
-; RV64I-NEXT:    mv a0, a0
-; RV64I-NEXT:    add s0, sp, a0
 ; RV64I-NEXT:    srli a0, sp, 12
 ; RV64I-NEXT:    slli sp, a0, 12
 ; RV64I-NEXT:    lui a0, 2
-; RV64I-NEXT:    mv a0, a0
 ; RV64I-NEXT:    add a0, sp, a0
 ; RV64I-NEXT:    mv a0, a0
 ; RV64I-NEXT:    call callee
 ; RV64I-NEXT:    lui a0, 3
-; RV64I-NEXT:    mv a0, a0
 ; RV64I-NEXT:    sub sp, s0, a0
 ; RV64I-NEXT:    lui a0, 3
-; RV64I-NEXT:    addi a0, a0, -16
-; RV64I-NEXT:    add a0, sp, a0
-; RV64I-NEXT:    ld s0, 0(a0)
-; RV64I-NEXT:    lui a0, 3
-; RV64I-NEXT:    addi a0, a0, -8
-; RV64I-NEXT:    add a0, sp, a0
-; RV64I-NEXT:    ld ra, 0(a0)
-; RV64I-NEXT:    lui a0, 3
-; RV64I-NEXT:    mv a0, a0
+; RV64I-NEXT:    addiw a0, a0, -2032
 ; RV64I-NEXT:    add sp, sp, a0
+; RV64I-NEXT:    ld s0, 2016(sp)
+; RV64I-NEXT:    ld ra, 2024(sp)
+; RV64I-NEXT:    addi sp, sp, 2032
 ; RV64I-NEXT:    ret
   %1 = alloca i8, align 4096
   call void @callee(i8* %1)
diff --git a/llvm/test/CodeGen/RISCV/subtarget-features-std-ext.ll b/llvm/test/CodeGen/RISCV/subtarget-features-std-ext.ll
new file mode 100644
index 000000000..613a983f9
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/subtarget-features-std-ext.ll
@@ -0,0 +1,20 @@
+; RUN: llc -mtriple=riscv32 -target-abi ilp32 < %s 2>&1 \
+; RUN:   | FileCheck -check-prefix=RV32IF-ILP32 %s
+; RUN: llc -mtriple=riscv32 -target-abi ilp32f < %s 2>&1 \
+; RUN:   | FileCheck -check-prefix=RV32IF-ILP32F %s
+; RUN: llc -mtriple=riscv32 -mattr=-f -target-abi ilp32f <%s 2>&1 \
+; RUN:   | FileCheck -check-prefix=RV32I-ILP32F-FAILED %s
+
+; RV32I-ILP32F-FAILED: Hard-float 'f' ABI can't be used for a target that doesn't support the F instruction set extension
+
+
+define float @foo(i32 %a) nounwind #0 {
+; RV32IF-ILP32: fcvt.s.w  ft0, a0
+; RV32IF-ILP32-NEXT: fmv.x.w a0, ft0
+; RV32IF-ILP32F: fcvt.s.w fa0, a0
+; RV32IF-ILP32F-NEXT: ret
+  %conv = sitofp i32 %a to float
+  ret float %conv
+}
+
+attributes #0 = { "target-features"="+f"}
diff --git a/llvm/test/CodeGen/RISCV/tail-calls.ll b/llvm/test/CodeGen/RISCV/tail-calls.ll
index dea1521b0..14af16ed3 100644
--- a/llvm/test/CodeGen/RISCV/tail-calls.ll
+++ b/llvm/test/CodeGen/RISCV/tail-calls.ll
@@ -23,6 +23,17 @@ entry:
   ret void
 }
 
+; Perform tail call optimization for external symbol.
+@dest_pgso = global [2 x i8] zeroinitializer
+define void @caller_extern_pgso(i8* %src) !prof !14 {
+entry:
+; CHECK: caller_extern_pgso
+; CHECK-NOT: call memcpy
+; CHECK: tail memcpy
+  tail call void @llvm.memcpy.p0i8.p0i8.i32(i8* getelementptr inbounds ([2 x i8], [2 x i8]* @dest_pgso, i32 0, i32 0), i8* %src, i32 7, i1 false)
+  ret void
+}
+
 ; Perform indirect tail call optimization (for function pointer call).
 declare void @callee_indirect1()
 declare void @callee_indirect2()
@@ -47,14 +58,14 @@ entry:
   ret void
 }
 
-; Do not tail call optimize functions with varargs.
+; Do not tail call optimize functions with varargs passed by stack.
 declare i32 @callee_varargs(i32, ...)
 define void @caller_varargs(i32 %a, i32 %b) nounwind {
 ; CHECK-LABEL: caller_varargs
 ; CHECK-NOT: tail callee_varargs
 ; CHECK: call callee_varargs
 entry:
-  %call = tail call i32 (i32, ...) @callee_varargs(i32 %a, i32 %b, i32 %b, i32 %a)
+  %call = tail call i32 (i32, ...) @callee_varargs(i32 %a, i32 %b, i32 %b, i32 %a, i32 %a, i32 %b, i32 %b, i32 %a, i32 %a)
   ret void
 }
 
@@ -146,3 +157,30 @@ entry:
   tail call void @callee_nostruct()
   ret void
 }
+
+; Do not tail call optimize if disabled.
+define i32 @disable_tail_calls(i32 %i) nounwind "disable-tail-calls"="true" {
+; CHECK-LABEL: disable_tail_calls:
+; CHECK-NOT: tail callee_nostruct
+; CHECK: call callee_tail
+entry:
+  %rv = tail call i32 @callee_tail(i32 %i)
+  ret i32 %rv
+}
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 1, !"ProfileSummary", !1}
+!1 = !{!2, !3, !4, !5, !6, !7, !8, !9}
+!2 = !{!"ProfileFormat", !"InstrProf"}
+!3 = !{!"TotalCount", i64 10000}
+!4 = !{!"MaxCount", i64 10}
+!5 = !{!"MaxInternalCount", i64 1}
+!6 = !{!"MaxFunctionCount", i64 1000}
+!7 = !{!"NumCounts", i64 3}
+!8 = !{!"NumFunctions", i64 3}
+!9 = !{!"DetailedSummary", !10}
+!10 = !{!11, !12, !13}
+!11 = !{i32 10000, i64 100, i32 1}
+!12 = !{i32 999000, i64 100, i32 1}
+!13 = !{i32 999999, i64 1, i32 2}
+!14 = !{!"function_entry_count", i64 0}
diff --git a/llvm/test/CodeGen/RISCV/tls-models.ll b/llvm/test/CodeGen/RISCV/tls-models.ll
index a2015b086..25a2f71be 100644
--- a/llvm/test/CodeGen/RISCV/tls-models.ll
+++ b/llvm/test/CodeGen/RISCV/tls-models.ll
@@ -3,16 +3,17 @@
 ; RUN:     | FileCheck -check-prefix=RV32-PIC %s
 ; RUN: llc -mtriple=riscv64 -relocation-model=pic < %s \
 ; RUN:     | FileCheck -check-prefix=RV64-PIC %s
-; RUN: llc -mtriple=riscv32 < %s | FileCheck -check-prefix=NOPIC %s
-; RUN: llc -mtriple=riscv64 < %s | FileCheck -check-prefix=NOPIC %s
+; RUN: llc -mtriple=riscv32 < %s | FileCheck -check-prefix=RV32-NOPIC %s
+; RUN: llc -mtriple=riscv64 < %s | FileCheck -check-prefix=RV64-NOPIC %s
 
 ; Check that TLS symbols are lowered correctly based on the specified
-; model.
+; model. Make sure they're external to avoid them all being optimised to Local
+; Exec for the executable.
 
-@unspecified = thread_local global i32 42
-@ld = thread_local(localdynamic) global i32 42
-@ie = thread_local(initialexec) global i32 42
-@le = thread_local(localexec) global i32 42
+@unspecified = external thread_local global i32
+@ld = external thread_local(localdynamic) global i32
+@ie = external thread_local(initialexec) global i32
+@le = external thread_local(localexec) global i32
 
 
 ; No model specified
@@ -44,12 +45,23 @@ define i32* @f1() nounwind {
 ; RV64-PIC-NEXT:    addi sp, sp, 16
 ; RV64-PIC-NEXT:    ret
 ;
-; NOPIC-LABEL: f1:
-; NOPIC:       # %bb.0: # %entry
-; NOPIC-NEXT:    lui a0, %tprel_hi(unspecified)
-; NOPIC-NEXT:    add a0, a0, tp, %tprel_add(unspecified)
-; NOPIC-NEXT:    addi a0, a0, %tprel_lo(unspecified)
-; NOPIC-NEXT:    ret
+; RV32-NOPIC-LABEL: f1:
+; RV32-NOPIC:       # %bb.0: # %entry
+; RV32-NOPIC-NEXT:  .LBB0_1: # %entry
+; RV32-NOPIC-NEXT:    # Label of block must be emitted
+; RV32-NOPIC-NEXT:    auipc a0, %tls_ie_pcrel_hi(unspecified)
+; RV32-NOPIC-NEXT:    lw a0, %pcrel_lo(.LBB0_1)(a0)
+; RV32-NOPIC-NEXT:    add a0, a0, tp
+; RV32-NOPIC-NEXT:    ret
+;
+; RV64-NOPIC-LABEL: f1:
+; RV64-NOPIC:       # %bb.0: # %entry
+; RV64-NOPIC-NEXT:  .LBB0_1: # %entry
+; RV64-NOPIC-NEXT:    # Label of block must be emitted
+; RV64-NOPIC-NEXT:    auipc a0, %tls_ie_pcrel_hi(unspecified)
+; RV64-NOPIC-NEXT:    ld a0, %pcrel_lo(.LBB0_1)(a0)
+; RV64-NOPIC-NEXT:    add a0, a0, tp
+; RV64-NOPIC-NEXT:    ret
 entry:
   ret i32* @unspecified
 }
@@ -84,12 +96,23 @@ define i32* @f2() nounwind {
 ; RV64-PIC-NEXT:    addi sp, sp, 16
 ; RV64-PIC-NEXT:    ret
 ;
-; NOPIC-LABEL: f2:
-; NOPIC:       # %bb.0: # %entry
-; NOPIC-NEXT:    lui a0, %tprel_hi(ld)
-; NOPIC-NEXT:    add a0, a0, tp, %tprel_add(ld)
-; NOPIC-NEXT:    addi a0, a0, %tprel_lo(ld)
-; NOPIC-NEXT:    ret
+; RV32-NOPIC-LABEL: f2:
+; RV32-NOPIC:       # %bb.0: # %entry
+; RV32-NOPIC-NEXT:  .LBB1_1: # %entry
+; RV32-NOPIC-NEXT:    # Label of block must be emitted
+; RV32-NOPIC-NEXT:    auipc a0, %tls_ie_pcrel_hi(ld)
+; RV32-NOPIC-NEXT:    lw a0, %pcrel_lo(.LBB1_1)(a0)
+; RV32-NOPIC-NEXT:    add a0, a0, tp
+; RV32-NOPIC-NEXT:    ret
+;
+; RV64-NOPIC-LABEL: f2:
+; RV64-NOPIC:       # %bb.0: # %entry
+; RV64-NOPIC-NEXT:  .LBB1_1: # %entry
+; RV64-NOPIC-NEXT:    # Label of block must be emitted
+; RV64-NOPIC-NEXT:    auipc a0, %tls_ie_pcrel_hi(ld)
+; RV64-NOPIC-NEXT:    ld a0, %pcrel_lo(.LBB1_1)(a0)
+; RV64-NOPIC-NEXT:    add a0, a0, tp
+; RV64-NOPIC-NEXT:    ret
 entry:
   ret i32* @ld
 }
@@ -116,12 +139,23 @@ define i32* @f3() nounwind {
 ; RV64-PIC-NEXT:    add a0, a0, tp
 ; RV64-PIC-NEXT:    ret
 ;
-; NOPIC-LABEL: f3:
-; NOPIC:       # %bb.0: # %entry
-; NOPIC-NEXT:    lui a0, %tprel_hi(ie)
-; NOPIC-NEXT:    add a0, a0, tp, %tprel_add(ie)
-; NOPIC-NEXT:    addi a0, a0, %tprel_lo(ie)
-; NOPIC-NEXT:    ret
+; RV32-NOPIC-LABEL: f3:
+; RV32-NOPIC:       # %bb.0: # %entry
+; RV32-NOPIC-NEXT:  .LBB2_1: # %entry
+; RV32-NOPIC-NEXT:    # Label of block must be emitted
+; RV32-NOPIC-NEXT:    auipc a0, %tls_ie_pcrel_hi(ie)
+; RV32-NOPIC-NEXT:    lw a0, %pcrel_lo(.LBB2_1)(a0)
+; RV32-NOPIC-NEXT:    add a0, a0, tp
+; RV32-NOPIC-NEXT:    ret
+;
+; RV64-NOPIC-LABEL: f3:
+; RV64-NOPIC:       # %bb.0: # %entry
+; RV64-NOPIC-NEXT:  .LBB2_1: # %entry
+; RV64-NOPIC-NEXT:    # Label of block must be emitted
+; RV64-NOPIC-NEXT:    auipc a0, %tls_ie_pcrel_hi(ie)
+; RV64-NOPIC-NEXT:    ld a0, %pcrel_lo(.LBB2_1)(a0)
+; RV64-NOPIC-NEXT:    add a0, a0, tp
+; RV64-NOPIC-NEXT:    ret
 entry:
   ret i32* @ie
 }
@@ -144,12 +178,19 @@ define i32* @f4() nounwind {
 ; RV64-PIC-NEXT:    addi a0, a0, %tprel_lo(le)
 ; RV64-PIC-NEXT:    ret
 ;
-; NOPIC-LABEL: f4:
-; NOPIC:       # %bb.0: # %entry
-; NOPIC-NEXT:    lui a0, %tprel_hi(le)
-; NOPIC-NEXT:    add a0, a0, tp, %tprel_add(le)
-; NOPIC-NEXT:    addi a0, a0, %tprel_lo(le)
-; NOPIC-NEXT:    ret
+; RV32-NOPIC-LABEL: f4:
+; RV32-NOPIC:       # %bb.0: # %entry
+; RV32-NOPIC-NEXT:    lui a0, %tprel_hi(le)
+; RV32-NOPIC-NEXT:    add a0, a0, tp, %tprel_add(le)
+; RV32-NOPIC-NEXT:    addi a0, a0, %tprel_lo(le)
+; RV32-NOPIC-NEXT:    ret
+;
+; RV64-NOPIC-LABEL: f4:
+; RV64-NOPIC:       # %bb.0: # %entry
+; RV64-NOPIC-NEXT:    lui a0, %tprel_hi(le)
+; RV64-NOPIC-NEXT:    add a0, a0, tp, %tprel_add(le)
+; RV64-NOPIC-NEXT:    addi a0, a0, %tprel_lo(le)
+; RV64-NOPIC-NEXT:    ret
 entry:
   ret i32* @le
 }
diff --git a/llvm/test/CodeGen/RISCV/umulo-128-legalisation-lowering.ll b/llvm/test/CodeGen/RISCV/umulo-128-legalisation-lowering.ll
index e0ac6483a..30346a134 100644
--- a/llvm/test/CodeGen/RISCV/umulo-128-legalisation-lowering.ll
+++ b/llvm/test/CodeGen/RISCV/umulo-128-legalisation-lowering.ll
@@ -4,111 +4,113 @@
 define { i128, i8 } @muloti_test(i128 %l, i128 %r) #0 {
 ; RISCV32-LABEL: muloti_test:
 ; RISCV32:       # %bb.0: # %start
-; RISCV32-NEXT:    addi sp, sp, -80
-; RISCV32-NEXT:    sw ra, 76(sp)
-; RISCV32-NEXT:    sw s0, 72(sp)
-; RISCV32-NEXT:    sw s1, 68(sp)
-; RISCV32-NEXT:    sw s2, 64(sp)
-; RISCV32-NEXT:    sw s3, 60(sp)
-; RISCV32-NEXT:    sw s4, 56(sp)
-; RISCV32-NEXT:    sw s5, 52(sp)
-; RISCV32-NEXT:    sw s6, 48(sp)
-; RISCV32-NEXT:    mv s1, a2
-; RISCV32-NEXT:    mv s0, a1
-; RISCV32-NEXT:    mv s2, a0
-; RISCV32-NEXT:    sw zero, 12(sp)
-; RISCV32-NEXT:    sw zero, 8(sp)
-; RISCV32-NEXT:    sw zero, 28(sp)
-; RISCV32-NEXT:    sw zero, 24(sp)
-; RISCV32-NEXT:    lw s3, 4(a2)
-; RISCV32-NEXT:    sw s3, 4(sp)
-; RISCV32-NEXT:    lw s5, 0(a2)
-; RISCV32-NEXT:    sw s5, 0(sp)
-; RISCV32-NEXT:    lw s4, 4(a1)
-; RISCV32-NEXT:    sw s4, 20(sp)
-; RISCV32-NEXT:    lw s6, 0(a1)
-; RISCV32-NEXT:    sw s6, 16(sp)
-; RISCV32-NEXT:    addi a0, sp, 32
-; RISCV32-NEXT:    addi a1, sp, 16
-; RISCV32-NEXT:    mv a2, sp
+; RISCV32-NEXT:    addi sp, sp, -96
+; RISCV32-NEXT:    sw ra, 92(sp)
+; RISCV32-NEXT:    sw s0, 88(sp)
+; RISCV32-NEXT:    sw s1, 84(sp)
+; RISCV32-NEXT:    sw s2, 80(sp)
+; RISCV32-NEXT:    sw s3, 76(sp)
+; RISCV32-NEXT:    sw s4, 72(sp)
+; RISCV32-NEXT:    sw s5, 68(sp)
+; RISCV32-NEXT:    sw s6, 64(sp)
+; RISCV32-NEXT:    sw s7, 60(sp)
+; RISCV32-NEXT:    sw s8, 56(sp)
+; RISCV32-NEXT:    lw s2, 12(a1)
+; RISCV32-NEXT:    lw s6, 8(a1)
+; RISCV32-NEXT:    lw s3, 12(a2)
+; RISCV32-NEXT:    lw s7, 8(a2)
+; RISCV32-NEXT:    lw s0, 0(a1)
+; RISCV32-NEXT:    lw s8, 4(a1)
+; RISCV32-NEXT:    lw s1, 0(a2)
+; RISCV32-NEXT:    lw s5, 4(a2)
+; RISCV32-NEXT:    mv s4, a0
+; RISCV32-NEXT:    sw zero, 20(sp)
+; RISCV32-NEXT:    sw zero, 16(sp)
+; RISCV32-NEXT:    sw zero, 36(sp)
+; RISCV32-NEXT:    sw zero, 32(sp)
+; RISCV32-NEXT:    sw s5, 12(sp)
+; RISCV32-NEXT:    sw s1, 8(sp)
+; RISCV32-NEXT:    sw s8, 28(sp)
+; RISCV32-NEXT:    addi a0, sp, 40
+; RISCV32-NEXT:    addi a1, sp, 24
+; RISCV32-NEXT:    addi a2, sp, 8
+; RISCV32-NEXT:    sw s0, 24(sp)
 ; RISCV32-NEXT:    call __multi3
-; RISCV32-NEXT:    lw a0, 12(s0)
-; RISCV32-NEXT:    lw a1, 8(s0)
-; RISCV32-NEXT:    mul a2, s3, a1
-; RISCV32-NEXT:    mul a3, a0, s5
-; RISCV32-NEXT:    add a4, a3, a2
-; RISCV32-NEXT:    lw a2, 12(s1)
-; RISCV32-NEXT:    lw a3, 8(s1)
-; RISCV32-NEXT:    mul a5, s4, a3
-; RISCV32-NEXT:    mul s1, a2, s6
-; RISCV32-NEXT:    add a5, s1, a5
-; RISCV32-NEXT:    mul s1, a3, s6
-; RISCV32-NEXT:    mul s0, a1, s5
-; RISCV32-NEXT:    add s1, s0, s1
-; RISCV32-NEXT:    sltu s0, s1, s0
-; RISCV32-NEXT:    mulhu a6, a3, s6
-; RISCV32-NEXT:    add t1, a6, a5
-; RISCV32-NEXT:    mulhu t2, a1, s5
-; RISCV32-NEXT:    add t3, t2, a4
-; RISCV32-NEXT:    add a5, t3, t1
-; RISCV32-NEXT:    add a5, a5, s0
-; RISCV32-NEXT:    lw s0, 44(sp)
-; RISCV32-NEXT:    add a5, s0, a5
-; RISCV32-NEXT:    lw a4, 40(sp)
-; RISCV32-NEXT:    add a7, a4, s1
-; RISCV32-NEXT:    sltu t0, a7, a4
-; RISCV32-NEXT:    add a5, a5, t0
-; RISCV32-NEXT:    beq a5, s0, .LBB0_2
+; RISCV32-NEXT:    mul a0, s8, s7
+; RISCV32-NEXT:    mul a1, s3, s0
+; RISCV32-NEXT:    add a0, a1, a0
+; RISCV32-NEXT:    mulhu a5, s7, s0
+; RISCV32-NEXT:    add a0, a5, a0
+; RISCV32-NEXT:    mul a1, s5, s6
+; RISCV32-NEXT:    mul a2, s2, s1
+; RISCV32-NEXT:    add a1, a2, a1
+; RISCV32-NEXT:    mulhu t0, s6, s1
+; RISCV32-NEXT:    add t1, t0, a1
+; RISCV32-NEXT:    add a6, t1, a0
+; RISCV32-NEXT:    mul a1, s7, s0
+; RISCV32-NEXT:    mul a3, s6, s1
+; RISCV32-NEXT:    add a4, a3, a1
+; RISCV32-NEXT:    lw a1, 52(sp)
+; RISCV32-NEXT:    lw a2, 48(sp)
+; RISCV32-NEXT:    sltu a3, a4, a3
+; RISCV32-NEXT:    add a3, a6, a3
+; RISCV32-NEXT:    add a3, a1, a3
+; RISCV32-NEXT:    add a6, a2, a4
+; RISCV32-NEXT:    sltu a2, a6, a2
+; RISCV32-NEXT:    add a7, a3, a2
+; RISCV32-NEXT:    beq a7, a1, .LBB0_2
 ; RISCV32-NEXT:  # %bb.1: # %start
-; RISCV32-NEXT:    sltu t0, a5, s0
+; RISCV32-NEXT:    sltu a2, a7, a1
 ; RISCV32-NEXT:  .LBB0_2: # %start
-; RISCV32-NEXT:    snez a4, s3
-; RISCV32-NEXT:    snez s1, a0
-; RISCV32-NEXT:    and a4, s1, a4
-; RISCV32-NEXT:    snez s1, s4
-; RISCV32-NEXT:    snez s0, a2
-; RISCV32-NEXT:    and s1, s0, s1
-; RISCV32-NEXT:    mulhu s0, a2, s6
-; RISCV32-NEXT:    snez s0, s0
-; RISCV32-NEXT:    or s1, s1, s0
-; RISCV32-NEXT:    mulhu s0, a0, s5
-; RISCV32-NEXT:    snez s0, s0
-; RISCV32-NEXT:    or a4, a4, s0
-; RISCV32-NEXT:    sltu t2, t3, t2
-; RISCV32-NEXT:    mulhu s0, s3, a1
-; RISCV32-NEXT:    snez s0, s0
-; RISCV32-NEXT:    or t3, a4, s0
-; RISCV32-NEXT:    sltu s0, t1, a6
-; RISCV32-NEXT:    mulhu a4, s4, a3
+; RISCV32-NEXT:    sltu a0, a0, a5
+; RISCV32-NEXT:    snez a1, s8
+; RISCV32-NEXT:    snez a3, s3
+; RISCV32-NEXT:    and a1, a3, a1
+; RISCV32-NEXT:    mulhu a3, s3, s0
+; RISCV32-NEXT:    snez a3, a3
+; RISCV32-NEXT:    or a1, a1, a3
+; RISCV32-NEXT:    mulhu a3, s8, s7
+; RISCV32-NEXT:    snez a3, a3
+; RISCV32-NEXT:    or a1, a1, a3
+; RISCV32-NEXT:    or a0, a1, a0
+; RISCV32-NEXT:    sltu a1, t1, t0
+; RISCV32-NEXT:    snez a3, s5
+; RISCV32-NEXT:    snez a4, s2
+; RISCV32-NEXT:    and a3, a4, a3
+; RISCV32-NEXT:    mulhu a4, s2, s1
+; RISCV32-NEXT:    snez a4, a4
+; RISCV32-NEXT:    or a3, a3, a4
+; RISCV32-NEXT:    mulhu a4, s5, s6
+; RISCV32-NEXT:    snez a4, a4
+; RISCV32-NEXT:    or a3, a3, a4
+; RISCV32-NEXT:    or a1, a3, a1
+; RISCV32-NEXT:    or a3, s7, s3
+; RISCV32-NEXT:    snez a3, a3
+; RISCV32-NEXT:    or a4, s6, s2
 ; RISCV32-NEXT:    snez a4, a4
-; RISCV32-NEXT:    or a4, s1, a4
-; RISCV32-NEXT:    lw s1, 36(sp)
-; RISCV32-NEXT:    sw s1, 4(s2)
-; RISCV32-NEXT:    lw s1, 32(sp)
-; RISCV32-NEXT:    sw s1, 0(s2)
-; RISCV32-NEXT:    sw a7, 8(s2)
-; RISCV32-NEXT:    sw a5, 12(s2)
-; RISCV32-NEXT:    or a4, a4, s0
-; RISCV32-NEXT:    or a5, t3, t2
+; RISCV32-NEXT:    and a3, a4, a3
+; RISCV32-NEXT:    or a1, a3, a1
 ; RISCV32-NEXT:    or a0, a1, a0
-; RISCV32-NEXT:    or a1, a3, a2
-; RISCV32-NEXT:    snez a1, a1
-; RISCV32-NEXT:    snez a0, a0
-; RISCV32-NEXT:    and a0, a0, a1
-; RISCV32-NEXT:    or a0, a0, a5
-; RISCV32-NEXT:    or a0, a0, a4
-; RISCV32-NEXT:    or a0, a0, t0
+; RISCV32-NEXT:    lw a1, 44(sp)
+; RISCV32-NEXT:    lw a3, 40(sp)
+; RISCV32-NEXT:    or a0, a0, a2
 ; RISCV32-NEXT:    andi a0, a0, 1
-; RISCV32-NEXT:    sb a0, 16(s2)
-; RISCV32-NEXT:    lw s6, 48(sp)
-; RISCV32-NEXT:    lw s5, 52(sp)
-; RISCV32-NEXT:    lw s4, 56(sp)
-; RISCV32-NEXT:    lw s3, 60(sp)
-; RISCV32-NEXT:    lw s2, 64(sp)
-; RISCV32-NEXT:    lw s1, 68(sp)
-; RISCV32-NEXT:    lw s0, 72(sp)
-; RISCV32-NEXT:    lw ra, 76(sp)
-; RISCV32-NEXT:    addi sp, sp, 80
+; RISCV32-NEXT:    sw a1, 4(s4)
+; RISCV32-NEXT:    sw a3, 0(s4)
+; RISCV32-NEXT:    sw a6, 8(s4)
+; RISCV32-NEXT:    sw a7, 12(s4)
+; RISCV32-NEXT:    sb a0, 16(s4)
+; RISCV32-NEXT:    lw s8, 56(sp)
+; RISCV32-NEXT:    lw s7, 60(sp)
+; RISCV32-NEXT:    lw s6, 64(sp)
+; RISCV32-NEXT:    lw s5, 68(sp)
+; RISCV32-NEXT:    lw s4, 72(sp)
+; RISCV32-NEXT:    lw s3, 76(sp)
+; RISCV32-NEXT:    lw s2, 80(sp)
+; RISCV32-NEXT:    lw s1, 84(sp)
+; RISCV32-NEXT:    lw s0, 88(sp)
+; RISCV32-NEXT:    lw ra, 92(sp)
+; RISCV32-NEXT:    addi sp, sp, 96
 ; RISCV32-NEXT:    ret
 start:
   %0 = tail call { i128, i1 } @llvm.umul.with.overflow.i128(i128 %l, i128 %r) #2
diff --git a/llvm/test/CodeGen/RISCV/urem-lkk.ll b/llvm/test/CodeGen/RISCV/urem-lkk.ll
new file mode 100644
index 000000000..5286ad507
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/urem-lkk.ll
@@ -0,0 +1,300 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefixes=CHECK,RV32I %s
+; RUN: llc -mtriple=riscv32 -mattr=+m -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefixes=CHECK,RV32IM %s
+; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefixes=CHECK,RV64I %s
+; RUN: llc -mtriple=riscv64 -mattr=+m -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefixes=CHECK,RV64IM %s
+
+define i32 @fold_urem_positive_odd(i32 %x) nounwind {
+; RV32I-LABEL: fold_urem_positive_odd:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    addi a1, zero, 95
+; RV32I-NEXT:    call __umodsi3
+; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV32IM-LABEL: fold_urem_positive_odd:
+; RV32IM:       # %bb.0:
+; RV32IM-NEXT:    lui a1, 364242
+; RV32IM-NEXT:    addi a1, a1, 777
+; RV32IM-NEXT:    mulhu a1, a0, a1
+; RV32IM-NEXT:    sub a2, a0, a1
+; RV32IM-NEXT:    srli a2, a2, 1
+; RV32IM-NEXT:    add a1, a2, a1
+; RV32IM-NEXT:    srli a1, a1, 6
+; RV32IM-NEXT:    addi a2, zero, 95
+; RV32IM-NEXT:    mul a1, a1, a2
+; RV32IM-NEXT:    sub a0, a0, a1
+; RV32IM-NEXT:    ret
+;
+; RV64I-LABEL: fold_urem_positive_odd:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    slli a0, a0, 32
+; RV64I-NEXT:    srli a0, a0, 32
+; RV64I-NEXT:    addi a1, zero, 95
+; RV64I-NEXT:    call __umoddi3
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IM-LABEL: fold_urem_positive_odd:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    slli a0, a0, 32
+; RV64IM-NEXT:    srli a0, a0, 32
+; RV64IM-NEXT:    lui a1, 1423
+; RV64IM-NEXT:    addiw a1, a1, -733
+; RV64IM-NEXT:    slli a1, a1, 15
+; RV64IM-NEXT:    addi a1, a1, 1035
+; RV64IM-NEXT:    slli a1, a1, 13
+; RV64IM-NEXT:    addi a1, a1, -1811
+; RV64IM-NEXT:    slli a1, a1, 12
+; RV64IM-NEXT:    addi a1, a1, 561
+; RV64IM-NEXT:    mulhu a1, a0, a1
+; RV64IM-NEXT:    sub a2, a0, a1
+; RV64IM-NEXT:    srli a2, a2, 1
+; RV64IM-NEXT:    add a1, a2, a1
+; RV64IM-NEXT:    srli a1, a1, 6
+; RV64IM-NEXT:    addi a2, zero, 95
+; RV64IM-NEXT:    mul a1, a1, a2
+; RV64IM-NEXT:    sub a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = urem i32 %x, 95
+  ret i32 %1
+}
+
+
+define i32 @fold_urem_positive_even(i32 %x) nounwind {
+; RV32I-LABEL: fold_urem_positive_even:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    addi a1, zero, 1060
+; RV32I-NEXT:    call __umodsi3
+; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV32IM-LABEL: fold_urem_positive_even:
+; RV32IM:       # %bb.0:
+; RV32IM-NEXT:    lui a1, 1012964
+; RV32IM-NEXT:    addi a1, a1, -61
+; RV32IM-NEXT:    mulhu a1, a0, a1
+; RV32IM-NEXT:    srli a1, a1, 10
+; RV32IM-NEXT:    addi a2, zero, 1060
+; RV32IM-NEXT:    mul a1, a1, a2
+; RV32IM-NEXT:    sub a0, a0, a1
+; RV32IM-NEXT:    ret
+;
+; RV64I-LABEL: fold_urem_positive_even:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    slli a0, a0, 32
+; RV64I-NEXT:    srli a0, a0, 32
+; RV64I-NEXT:    addi a1, zero, 1060
+; RV64I-NEXT:    call __umoddi3
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IM-LABEL: fold_urem_positive_even:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    slli a0, a0, 32
+; RV64IM-NEXT:    srli a0, a0, 32
+; RV64IM-NEXT:    lui a1, 1048020
+; RV64IM-NEXT:    addiw a1, a1, -1793
+; RV64IM-NEXT:    slli a1, a1, 12
+; RV64IM-NEXT:    addi a1, a1, 139
+; RV64IM-NEXT:    slli a1, a1, 14
+; RV64IM-NEXT:    addi a1, a1, 1793
+; RV64IM-NEXT:    slli a1, a1, 12
+; RV64IM-NEXT:    addi a1, a1, -139
+; RV64IM-NEXT:    mulhu a1, a0, a1
+; RV64IM-NEXT:    srli a1, a1, 10
+; RV64IM-NEXT:    addi a2, zero, 1060
+; RV64IM-NEXT:    mul a1, a1, a2
+; RV64IM-NEXT:    sub a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = urem i32 %x, 1060
+  ret i32 %1
+}
+
+
+; Don't fold if we can combine urem with udiv.
+define i32 @combine_urem_udiv(i32 %x) nounwind {
+; RV32I-LABEL: combine_urem_udiv:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw s0, 8(sp)
+; RV32I-NEXT:    sw s1, 4(sp)
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    addi a1, zero, 95
+; RV32I-NEXT:    call __umodsi3
+; RV32I-NEXT:    mv s1, a0
+; RV32I-NEXT:    addi a1, zero, 95
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    call __udivsi3
+; RV32I-NEXT:    add a0, s1, a0
+; RV32I-NEXT:    lw s1, 4(sp)
+; RV32I-NEXT:    lw s0, 8(sp)
+; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV32IM-LABEL: combine_urem_udiv:
+; RV32IM:       # %bb.0:
+; RV32IM-NEXT:    lui a1, 364242
+; RV32IM-NEXT:    addi a1, a1, 777
+; RV32IM-NEXT:    mulhu a1, a0, a1
+; RV32IM-NEXT:    sub a2, a0, a1
+; RV32IM-NEXT:    srli a2, a2, 1
+; RV32IM-NEXT:    add a1, a2, a1
+; RV32IM-NEXT:    srli a1, a1, 6
+; RV32IM-NEXT:    addi a2, zero, 95
+; RV32IM-NEXT:    mul a2, a1, a2
+; RV32IM-NEXT:    sub a0, a0, a2
+; RV32IM-NEXT:    add a0, a0, a1
+; RV32IM-NEXT:    ret
+;
+; RV64I-LABEL: combine_urem_udiv:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -32
+; RV64I-NEXT:    sd ra, 24(sp)
+; RV64I-NEXT:    sd s0, 16(sp)
+; RV64I-NEXT:    sd s1, 8(sp)
+; RV64I-NEXT:    slli a0, a0, 32
+; RV64I-NEXT:    srli s0, a0, 32
+; RV64I-NEXT:    addi a1, zero, 95
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    call __umoddi3
+; RV64I-NEXT:    mv s1, a0
+; RV64I-NEXT:    addi a1, zero, 95
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    call __udivdi3
+; RV64I-NEXT:    add a0, s1, a0
+; RV64I-NEXT:    ld s1, 8(sp)
+; RV64I-NEXT:    ld s0, 16(sp)
+; RV64I-NEXT:    ld ra, 24(sp)
+; RV64I-NEXT:    addi sp, sp, 32
+; RV64I-NEXT:    ret
+;
+; RV64IM-LABEL: combine_urem_udiv:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    slli a0, a0, 32
+; RV64IM-NEXT:    srli a0, a0, 32
+; RV64IM-NEXT:    lui a1, 1423
+; RV64IM-NEXT:    addiw a1, a1, -733
+; RV64IM-NEXT:    slli a1, a1, 15
+; RV64IM-NEXT:    addi a1, a1, 1035
+; RV64IM-NEXT:    slli a1, a1, 13
+; RV64IM-NEXT:    addi a1, a1, -1811
+; RV64IM-NEXT:    slli a1, a1, 12
+; RV64IM-NEXT:    addi a1, a1, 561
+; RV64IM-NEXT:    mulhu a1, a0, a1
+; RV64IM-NEXT:    sub a2, a0, a1
+; RV64IM-NEXT:    srli a2, a2, 1
+; RV64IM-NEXT:    add a1, a2, a1
+; RV64IM-NEXT:    srli a1, a1, 6
+; RV64IM-NEXT:    addi a2, zero, 95
+; RV64IM-NEXT:    mul a2, a1, a2
+; RV64IM-NEXT:    sub a0, a0, a2
+; RV64IM-NEXT:    add a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = urem i32 %x, 95
+  %2 = udiv i32 %x, 95
+  %3 = add i32 %1, %2
+  ret i32 %3
+}
+
+; Don't fold for divisors that are a power of two.
+define i32 @dont_fold_urem_power_of_two(i32 %x) nounwind {
+; CHECK-LABEL: dont_fold_urem_power_of_two:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    andi a0, a0, 63
+; CHECK-NEXT:    ret
+  %1 = urem i32 %x, 64
+  ret i32 %1
+}
+
+; Don't fold if the divisor is one.
+define i32 @dont_fold_urem_one(i32 %x) nounwind {
+; CHECK-LABEL: dont_fold_urem_one:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mv a0, zero
+; CHECK-NEXT:    ret
+  %1 = urem i32 %x, 1
+  ret i32 %1
+}
+
+; Don't fold if the divisor is 2^32.
+define i32 @dont_fold_urem_i32_umax(i32 %x) nounwind {
+; CHECK-LABEL: dont_fold_urem_i32_umax:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret
+  %1 = urem i32 %x, 4294967296
+  ret i32 %1
+}
+
+; Don't fold i64 urem
+define i64 @dont_fold_urem_i64(i64 %x) nounwind {
+; RV32I-LABEL: dont_fold_urem_i64:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    addi a2, zero, 98
+; RV32I-NEXT:    mv a3, zero
+; RV32I-NEXT:    call __umoddi3
+; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV32IM-LABEL: dont_fold_urem_i64:
+; RV32IM:       # %bb.0:
+; RV32IM-NEXT:    addi sp, sp, -16
+; RV32IM-NEXT:    sw ra, 12(sp)
+; RV32IM-NEXT:    addi a2, zero, 98
+; RV32IM-NEXT:    mv a3, zero
+; RV32IM-NEXT:    call __umoddi3
+; RV32IM-NEXT:    lw ra, 12(sp)
+; RV32IM-NEXT:    addi sp, sp, 16
+; RV32IM-NEXT:    ret
+;
+; RV64I-LABEL: dont_fold_urem_i64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a1, zero, 98
+; RV64I-NEXT:    call __umoddi3
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IM-LABEL: dont_fold_urem_i64:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    srli a1, a0, 1
+; RV64IM-NEXT:    lui a2, 2675
+; RV64IM-NEXT:    addiw a2, a2, -251
+; RV64IM-NEXT:    slli a2, a2, 13
+; RV64IM-NEXT:    addi a2, a2, 1839
+; RV64IM-NEXT:    slli a2, a2, 13
+; RV64IM-NEXT:    addi a2, a2, 167
+; RV64IM-NEXT:    slli a2, a2, 13
+; RV64IM-NEXT:    addi a2, a2, 1505
+; RV64IM-NEXT:    mulhu a1, a1, a2
+; RV64IM-NEXT:    srli a1, a1, 4
+; RV64IM-NEXT:    addi a2, zero, 98
+; RV64IM-NEXT:    mul a1, a1, a2
+; RV64IM-NEXT:    sub a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = urem i64 %x, 98
+  ret i64 %1
+}
diff --git a/llvm/test/CodeGen/RISCV/urem-vector-lkk.ll b/llvm/test/CodeGen/RISCV/urem-vector-lkk.ll
new file mode 100644
index 000000000..0ae2575da
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/urem-vector-lkk.ll
@@ -0,0 +1,1187 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefixes=CHECK,RV32I %s
+; RUN: llc -mtriple=riscv32 -mattr=+m -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefixes=CHECK,RV32IM %s
+; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefixes=CHECK,RV64I %s
+; RUN: llc -mtriple=riscv64 -mattr=+m -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefixes=CHECK,RV64IM %s
+
+
+define <4 x i16> @fold_urem_vec_1(<4 x i16> %x) nounwind {
+; RV32I-LABEL: fold_urem_vec_1:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -32
+; RV32I-NEXT:    sw ra, 28(sp)
+; RV32I-NEXT:    sw s0, 24(sp)
+; RV32I-NEXT:    sw s1, 20(sp)
+; RV32I-NEXT:    sw s2, 16(sp)
+; RV32I-NEXT:    sw s3, 12(sp)
+; RV32I-NEXT:    sw s4, 8(sp)
+; RV32I-NEXT:    sw s5, 4(sp)
+; RV32I-NEXT:    lhu s2, 12(a1)
+; RV32I-NEXT:    lhu s3, 8(a1)
+; RV32I-NEXT:    lhu s0, 4(a1)
+; RV32I-NEXT:    lhu a2, 0(a1)
+; RV32I-NEXT:    mv s1, a0
+; RV32I-NEXT:    addi a1, zero, 95
+; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    call __umodsi3
+; RV32I-NEXT:    mv s4, a0
+; RV32I-NEXT:    addi a1, zero, 124
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    call __umodsi3
+; RV32I-NEXT:    mv s5, a0
+; RV32I-NEXT:    addi a1, zero, 98
+; RV32I-NEXT:    mv a0, s3
+; RV32I-NEXT:    call __umodsi3
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    addi a1, zero, 1003
+; RV32I-NEXT:    mv a0, s2
+; RV32I-NEXT:    call __umodsi3
+; RV32I-NEXT:    sh a0, 6(s1)
+; RV32I-NEXT:    sh s0, 4(s1)
+; RV32I-NEXT:    sh s5, 2(s1)
+; RV32I-NEXT:    sh s4, 0(s1)
+; RV32I-NEXT:    lw s5, 4(sp)
+; RV32I-NEXT:    lw s4, 8(sp)
+; RV32I-NEXT:    lw s3, 12(sp)
+; RV32I-NEXT:    lw s2, 16(sp)
+; RV32I-NEXT:    lw s1, 20(sp)
+; RV32I-NEXT:    lw s0, 24(sp)
+; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    addi sp, sp, 32
+; RV32I-NEXT:    ret
+;
+; RV32IM-LABEL: fold_urem_vec_1:
+; RV32IM:       # %bb.0:
+; RV32IM-NEXT:    lhu a6, 12(a1)
+; RV32IM-NEXT:    lhu a3, 8(a1)
+; RV32IM-NEXT:    lhu a4, 0(a1)
+; RV32IM-NEXT:    lhu a1, 4(a1)
+; RV32IM-NEXT:    lui a5, 364242
+; RV32IM-NEXT:    addi a5, a5, 777
+; RV32IM-NEXT:    mulhu a5, a4, a5
+; RV32IM-NEXT:    sub a2, a4, a5
+; RV32IM-NEXT:    srli a2, a2, 1
+; RV32IM-NEXT:    add a2, a2, a5
+; RV32IM-NEXT:    srli a2, a2, 6
+; RV32IM-NEXT:    addi a5, zero, 95
+; RV32IM-NEXT:    mul a2, a2, a5
+; RV32IM-NEXT:    sub a2, a4, a2
+; RV32IM-NEXT:    srli a4, a1, 2
+; RV32IM-NEXT:    lui a5, 135300
+; RV32IM-NEXT:    addi a5, a5, 529
+; RV32IM-NEXT:    mulhu a4, a4, a5
+; RV32IM-NEXT:    srli a4, a4, 2
+; RV32IM-NEXT:    addi a5, zero, 124
+; RV32IM-NEXT:    mul a4, a4, a5
+; RV32IM-NEXT:    sub a1, a1, a4
+; RV32IM-NEXT:    lui a4, 342392
+; RV32IM-NEXT:    addi a4, a4, 669
+; RV32IM-NEXT:    mulhu a4, a3, a4
+; RV32IM-NEXT:    srli a4, a4, 5
+; RV32IM-NEXT:    addi a5, zero, 98
+; RV32IM-NEXT:    mul a4, a4, a5
+; RV32IM-NEXT:    sub a3, a3, a4
+; RV32IM-NEXT:    lui a4, 267633
+; RV32IM-NEXT:    addi a4, a4, -1809
+; RV32IM-NEXT:    mulhu a4, a6, a4
+; RV32IM-NEXT:    srli a4, a4, 8
+; RV32IM-NEXT:    addi a5, zero, 1003
+; RV32IM-NEXT:    mul a4, a4, a5
+; RV32IM-NEXT:    sub a4, a6, a4
+; RV32IM-NEXT:    sh a4, 6(a0)
+; RV32IM-NEXT:    sh a3, 4(a0)
+; RV32IM-NEXT:    sh a1, 2(a0)
+; RV32IM-NEXT:    sh a2, 0(a0)
+; RV32IM-NEXT:    ret
+;
+; RV64I-LABEL: fold_urem_vec_1:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -64
+; RV64I-NEXT:    sd ra, 56(sp)
+; RV64I-NEXT:    sd s0, 48(sp)
+; RV64I-NEXT:    sd s1, 40(sp)
+; RV64I-NEXT:    sd s2, 32(sp)
+; RV64I-NEXT:    sd s3, 24(sp)
+; RV64I-NEXT:    sd s4, 16(sp)
+; RV64I-NEXT:    sd s5, 8(sp)
+; RV64I-NEXT:    lhu s2, 24(a1)
+; RV64I-NEXT:    lhu s3, 16(a1)
+; RV64I-NEXT:    lhu s0, 8(a1)
+; RV64I-NEXT:    lhu a2, 0(a1)
+; RV64I-NEXT:    mv s1, a0
+; RV64I-NEXT:    addi a1, zero, 95
+; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    call __umoddi3
+; RV64I-NEXT:    mv s4, a0
+; RV64I-NEXT:    addi a1, zero, 124
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    call __umoddi3
+; RV64I-NEXT:    mv s5, a0
+; RV64I-NEXT:    addi a1, zero, 98
+; RV64I-NEXT:    mv a0, s3
+; RV64I-NEXT:    call __umoddi3
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    addi a1, zero, 1003
+; RV64I-NEXT:    mv a0, s2
+; RV64I-NEXT:    call __umoddi3
+; RV64I-NEXT:    sh a0, 6(s1)
+; RV64I-NEXT:    sh s0, 4(s1)
+; RV64I-NEXT:    sh s5, 2(s1)
+; RV64I-NEXT:    sh s4, 0(s1)
+; RV64I-NEXT:    ld s5, 8(sp)
+; RV64I-NEXT:    ld s4, 16(sp)
+; RV64I-NEXT:    ld s3, 24(sp)
+; RV64I-NEXT:    ld s2, 32(sp)
+; RV64I-NEXT:    ld s1, 40(sp)
+; RV64I-NEXT:    ld s0, 48(sp)
+; RV64I-NEXT:    ld ra, 56(sp)
+; RV64I-NEXT:    addi sp, sp, 64
+; RV64I-NEXT:    ret
+;
+; RV64IM-LABEL: fold_urem_vec_1:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    lhu a6, 24(a1)
+; RV64IM-NEXT:    lhu a3, 16(a1)
+; RV64IM-NEXT:    lhu a4, 8(a1)
+; RV64IM-NEXT:    lhu a1, 0(a1)
+; RV64IM-NEXT:    lui a5, 1423
+; RV64IM-NEXT:    addiw a5, a5, -733
+; RV64IM-NEXT:    slli a5, a5, 15
+; RV64IM-NEXT:    addi a5, a5, 1035
+; RV64IM-NEXT:    slli a5, a5, 13
+; RV64IM-NEXT:    addi a5, a5, -1811
+; RV64IM-NEXT:    slli a5, a5, 12
+; RV64IM-NEXT:    addi a5, a5, 561
+; RV64IM-NEXT:    mulhu a5, a1, a5
+; RV64IM-NEXT:    sub a2, a1, a5
+; RV64IM-NEXT:    srli a2, a2, 1
+; RV64IM-NEXT:    add a2, a2, a5
+; RV64IM-NEXT:    srli a2, a2, 6
+; RV64IM-NEXT:    addi a5, zero, 95
+; RV64IM-NEXT:    mul a2, a2, a5
+; RV64IM-NEXT:    sub a1, a1, a2
+; RV64IM-NEXT:    srli a2, a4, 2
+; RV64IM-NEXT:    lui a5, 264
+; RV64IM-NEXT:    addiw a5, a5, 1057
+; RV64IM-NEXT:    slli a5, a5, 15
+; RV64IM-NEXT:    addi a5, a5, 1057
+; RV64IM-NEXT:    slli a5, a5, 15
+; RV64IM-NEXT:    addi a5, a5, 1057
+; RV64IM-NEXT:    slli a5, a5, 12
+; RV64IM-NEXT:    addi a5, a5, 133
+; RV64IM-NEXT:    mulhu a2, a2, a5
+; RV64IM-NEXT:    srli a2, a2, 3
+; RV64IM-NEXT:    addi a5, zero, 124
+; RV64IM-NEXT:    mul a2, a2, a5
+; RV64IM-NEXT:    sub a2, a4, a2
+; RV64IM-NEXT:    srli a4, a3, 1
+; RV64IM-NEXT:    lui a5, 2675
+; RV64IM-NEXT:    addiw a5, a5, -251
+; RV64IM-NEXT:    slli a5, a5, 13
+; RV64IM-NEXT:    addi a5, a5, 1839
+; RV64IM-NEXT:    slli a5, a5, 13
+; RV64IM-NEXT:    addi a5, a5, 167
+; RV64IM-NEXT:    slli a5, a5, 13
+; RV64IM-NEXT:    addi a5, a5, 1505
+; RV64IM-NEXT:    mulhu a4, a4, a5
+; RV64IM-NEXT:    srli a4, a4, 4
+; RV64IM-NEXT:    addi a5, zero, 98
+; RV64IM-NEXT:    mul a4, a4, a5
+; RV64IM-NEXT:    sub a3, a3, a4
+; RV64IM-NEXT:    lui a4, 8364
+; RV64IM-NEXT:    addiw a4, a4, -1977
+; RV64IM-NEXT:    slli a4, a4, 12
+; RV64IM-NEXT:    addi a4, a4, 1907
+; RV64IM-NEXT:    slli a4, a4, 12
+; RV64IM-NEXT:    addi a4, a4, 453
+; RV64IM-NEXT:    slli a4, a4, 12
+; RV64IM-NEXT:    addi a4, a4, 1213
+; RV64IM-NEXT:    mulhu a4, a6, a4
+; RV64IM-NEXT:    srli a4, a4, 7
+; RV64IM-NEXT:    addi a5, zero, 1003
+; RV64IM-NEXT:    mul a4, a4, a5
+; RV64IM-NEXT:    sub a4, a6, a4
+; RV64IM-NEXT:    sh a4, 6(a0)
+; RV64IM-NEXT:    sh a3, 4(a0)
+; RV64IM-NEXT:    sh a2, 2(a0)
+; RV64IM-NEXT:    sh a1, 0(a0)
+; RV64IM-NEXT:    ret
+  %1 = urem <4 x i16> %x, <i16 95, i16 124, i16 98, i16 1003>
+  ret <4 x i16> %1
+}
+
+define <4 x i16> @fold_urem_vec_2(<4 x i16> %x) nounwind {
+; RV32I-LABEL: fold_urem_vec_2:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -32
+; RV32I-NEXT:    sw ra, 28(sp)
+; RV32I-NEXT:    sw s0, 24(sp)
+; RV32I-NEXT:    sw s1, 20(sp)
+; RV32I-NEXT:    sw s2, 16(sp)
+; RV32I-NEXT:    sw s3, 12(sp)
+; RV32I-NEXT:    sw s4, 8(sp)
+; RV32I-NEXT:    sw s5, 4(sp)
+; RV32I-NEXT:    lhu s2, 12(a1)
+; RV32I-NEXT:    lhu s3, 8(a1)
+; RV32I-NEXT:    lhu s0, 4(a1)
+; RV32I-NEXT:    lhu a2, 0(a1)
+; RV32I-NEXT:    mv s1, a0
+; RV32I-NEXT:    addi a1, zero, 95
+; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    call __umodsi3
+; RV32I-NEXT:    mv s4, a0
+; RV32I-NEXT:    addi a1, zero, 95
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    call __umodsi3
+; RV32I-NEXT:    mv s5, a0
+; RV32I-NEXT:    addi a1, zero, 95
+; RV32I-NEXT:    mv a0, s3
+; RV32I-NEXT:    call __umodsi3
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    addi a1, zero, 95
+; RV32I-NEXT:    mv a0, s2
+; RV32I-NEXT:    call __umodsi3
+; RV32I-NEXT:    sh a0, 6(s1)
+; RV32I-NEXT:    sh s0, 4(s1)
+; RV32I-NEXT:    sh s5, 2(s1)
+; RV32I-NEXT:    sh s4, 0(s1)
+; RV32I-NEXT:    lw s5, 4(sp)
+; RV32I-NEXT:    lw s4, 8(sp)
+; RV32I-NEXT:    lw s3, 12(sp)
+; RV32I-NEXT:    lw s2, 16(sp)
+; RV32I-NEXT:    lw s1, 20(sp)
+; RV32I-NEXT:    lw s0, 24(sp)
+; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    addi sp, sp, 32
+; RV32I-NEXT:    ret
+;
+; RV32IM-LABEL: fold_urem_vec_2:
+; RV32IM:       # %bb.0:
+; RV32IM-NEXT:    lhu a6, 12(a1)
+; RV32IM-NEXT:    lhu a7, 8(a1)
+; RV32IM-NEXT:    lhu a4, 0(a1)
+; RV32IM-NEXT:    lhu a1, 4(a1)
+; RV32IM-NEXT:    lui a5, 364242
+; RV32IM-NEXT:    addi a5, a5, 777
+; RV32IM-NEXT:    mulhu a2, a4, a5
+; RV32IM-NEXT:    sub a3, a4, a2
+; RV32IM-NEXT:    srli a3, a3, 1
+; RV32IM-NEXT:    add a2, a3, a2
+; RV32IM-NEXT:    srli a2, a2, 6
+; RV32IM-NEXT:    addi a3, zero, 95
+; RV32IM-NEXT:    mul a2, a2, a3
+; RV32IM-NEXT:    sub t0, a4, a2
+; RV32IM-NEXT:    mulhu a4, a1, a5
+; RV32IM-NEXT:    sub a2, a1, a4
+; RV32IM-NEXT:    srli a2, a2, 1
+; RV32IM-NEXT:    add a2, a2, a4
+; RV32IM-NEXT:    srli a2, a2, 6
+; RV32IM-NEXT:    mul a2, a2, a3
+; RV32IM-NEXT:    sub a1, a1, a2
+; RV32IM-NEXT:    mulhu a2, a7, a5
+; RV32IM-NEXT:    sub a4, a7, a2
+; RV32IM-NEXT:    srli a4, a4, 1
+; RV32IM-NEXT:    add a2, a4, a2
+; RV32IM-NEXT:    srli a2, a2, 6
+; RV32IM-NEXT:    mul a2, a2, a3
+; RV32IM-NEXT:    sub a2, a7, a2
+; RV32IM-NEXT:    mulhu a4, a6, a5
+; RV32IM-NEXT:    sub a5, a6, a4
+; RV32IM-NEXT:    srli a5, a5, 1
+; RV32IM-NEXT:    add a4, a5, a4
+; RV32IM-NEXT:    srli a4, a4, 6
+; RV32IM-NEXT:    mul a3, a4, a3
+; RV32IM-NEXT:    sub a3, a6, a3
+; RV32IM-NEXT:    sh a3, 6(a0)
+; RV32IM-NEXT:    sh a2, 4(a0)
+; RV32IM-NEXT:    sh a1, 2(a0)
+; RV32IM-NEXT:    sh t0, 0(a0)
+; RV32IM-NEXT:    ret
+;
+; RV64I-LABEL: fold_urem_vec_2:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -64
+; RV64I-NEXT:    sd ra, 56(sp)
+; RV64I-NEXT:    sd s0, 48(sp)
+; RV64I-NEXT:    sd s1, 40(sp)
+; RV64I-NEXT:    sd s2, 32(sp)
+; RV64I-NEXT:    sd s3, 24(sp)
+; RV64I-NEXT:    sd s4, 16(sp)
+; RV64I-NEXT:    sd s5, 8(sp)
+; RV64I-NEXT:    lhu s2, 24(a1)
+; RV64I-NEXT:    lhu s3, 16(a1)
+; RV64I-NEXT:    lhu s0, 8(a1)
+; RV64I-NEXT:    lhu a2, 0(a1)
+; RV64I-NEXT:    mv s1, a0
+; RV64I-NEXT:    addi a1, zero, 95
+; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    call __umoddi3
+; RV64I-NEXT:    mv s4, a0
+; RV64I-NEXT:    addi a1, zero, 95
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    call __umoddi3
+; RV64I-NEXT:    mv s5, a0
+; RV64I-NEXT:    addi a1, zero, 95
+; RV64I-NEXT:    mv a0, s3
+; RV64I-NEXT:    call __umoddi3
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    addi a1, zero, 95
+; RV64I-NEXT:    mv a0, s2
+; RV64I-NEXT:    call __umoddi3
+; RV64I-NEXT:    sh a0, 6(s1)
+; RV64I-NEXT:    sh s0, 4(s1)
+; RV64I-NEXT:    sh s5, 2(s1)
+; RV64I-NEXT:    sh s4, 0(s1)
+; RV64I-NEXT:    ld s5, 8(sp)
+; RV64I-NEXT:    ld s4, 16(sp)
+; RV64I-NEXT:    ld s3, 24(sp)
+; RV64I-NEXT:    ld s2, 32(sp)
+; RV64I-NEXT:    ld s1, 40(sp)
+; RV64I-NEXT:    ld s0, 48(sp)
+; RV64I-NEXT:    ld ra, 56(sp)
+; RV64I-NEXT:    addi sp, sp, 64
+; RV64I-NEXT:    ret
+;
+; RV64IM-LABEL: fold_urem_vec_2:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    lhu a6, 24(a1)
+; RV64IM-NEXT:    lhu a7, 16(a1)
+; RV64IM-NEXT:    lhu a4, 8(a1)
+; RV64IM-NEXT:    lhu a1, 0(a1)
+; RV64IM-NEXT:    lui a5, 1423
+; RV64IM-NEXT:    addiw a5, a5, -733
+; RV64IM-NEXT:    slli a5, a5, 15
+; RV64IM-NEXT:    addi a5, a5, 1035
+; RV64IM-NEXT:    slli a5, a5, 13
+; RV64IM-NEXT:    addi a5, a5, -1811
+; RV64IM-NEXT:    slli a5, a5, 12
+; RV64IM-NEXT:    addi a5, a5, 561
+; RV64IM-NEXT:    mulhu a2, a1, a5
+; RV64IM-NEXT:    sub a3, a1, a2
+; RV64IM-NEXT:    srli a3, a3, 1
+; RV64IM-NEXT:    add a2, a3, a2
+; RV64IM-NEXT:    srli a2, a2, 6
+; RV64IM-NEXT:    addi a3, zero, 95
+; RV64IM-NEXT:    mul a2, a2, a3
+; RV64IM-NEXT:    sub t0, a1, a2
+; RV64IM-NEXT:    mulhu a2, a4, a5
+; RV64IM-NEXT:    sub a1, a4, a2
+; RV64IM-NEXT:    srli a1, a1, 1
+; RV64IM-NEXT:    add a1, a1, a2
+; RV64IM-NEXT:    srli a1, a1, 6
+; RV64IM-NEXT:    mul a1, a1, a3
+; RV64IM-NEXT:    sub a1, a4, a1
+; RV64IM-NEXT:    mulhu a2, a7, a5
+; RV64IM-NEXT:    sub a4, a7, a2
+; RV64IM-NEXT:    srli a4, a4, 1
+; RV64IM-NEXT:    add a2, a4, a2
+; RV64IM-NEXT:    srli a2, a2, 6
+; RV64IM-NEXT:    mul a2, a2, a3
+; RV64IM-NEXT:    sub a2, a7, a2
+; RV64IM-NEXT:    mulhu a4, a6, a5
+; RV64IM-NEXT:    sub a5, a6, a4
+; RV64IM-NEXT:    srli a5, a5, 1
+; RV64IM-NEXT:    add a4, a5, a4
+; RV64IM-NEXT:    srli a4, a4, 6
+; RV64IM-NEXT:    mul a3, a4, a3
+; RV64IM-NEXT:    sub a3, a6, a3
+; RV64IM-NEXT:    sh a3, 6(a0)
+; RV64IM-NEXT:    sh a2, 4(a0)
+; RV64IM-NEXT:    sh a1, 2(a0)
+; RV64IM-NEXT:    sh t0, 0(a0)
+; RV64IM-NEXT:    ret
+  %1 = urem <4 x i16> %x, <i16 95, i16 95, i16 95, i16 95>
+  ret <4 x i16> %1
+}
+
+
+; Don't fold if we can combine urem with udiv.
+define <4 x i16> @combine_urem_udiv(<4 x i16> %x) nounwind {
+; RV32I-LABEL: combine_urem_udiv:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -48
+; RV32I-NEXT:    sw ra, 44(sp)
+; RV32I-NEXT:    sw s0, 40(sp)
+; RV32I-NEXT:    sw s1, 36(sp)
+; RV32I-NEXT:    sw s2, 32(sp)
+; RV32I-NEXT:    sw s3, 28(sp)
+; RV32I-NEXT:    sw s4, 24(sp)
+; RV32I-NEXT:    sw s5, 20(sp)
+; RV32I-NEXT:    sw s6, 16(sp)
+; RV32I-NEXT:    sw s7, 12(sp)
+; RV32I-NEXT:    sw s8, 8(sp)
+; RV32I-NEXT:    sw s9, 4(sp)
+; RV32I-NEXT:    lhu s2, 0(a1)
+; RV32I-NEXT:    lhu s3, 4(a1)
+; RV32I-NEXT:    lhu s4, 8(a1)
+; RV32I-NEXT:    lhu s1, 12(a1)
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    addi a1, zero, 95
+; RV32I-NEXT:    mv a0, s1
+; RV32I-NEXT:    call __umodsi3
+; RV32I-NEXT:    mv s5, a0
+; RV32I-NEXT:    addi a1, zero, 95
+; RV32I-NEXT:    mv a0, s4
+; RV32I-NEXT:    call __umodsi3
+; RV32I-NEXT:    mv s6, a0
+; RV32I-NEXT:    addi a1, zero, 95
+; RV32I-NEXT:    mv a0, s3
+; RV32I-NEXT:    call __umodsi3
+; RV32I-NEXT:    mv s7, a0
+; RV32I-NEXT:    addi a1, zero, 95
+; RV32I-NEXT:    mv a0, s2
+; RV32I-NEXT:    call __umodsi3
+; RV32I-NEXT:    mv s8, a0
+; RV32I-NEXT:    addi a1, zero, 95
+; RV32I-NEXT:    mv a0, s1
+; RV32I-NEXT:    call __udivsi3
+; RV32I-NEXT:    mv s9, a0
+; RV32I-NEXT:    addi a1, zero, 95
+; RV32I-NEXT:    mv a0, s4
+; RV32I-NEXT:    call __udivsi3
+; RV32I-NEXT:    mv s4, a0
+; RV32I-NEXT:    addi a1, zero, 95
+; RV32I-NEXT:    mv a0, s3
+; RV32I-NEXT:    call __udivsi3
+; RV32I-NEXT:    mv s1, a0
+; RV32I-NEXT:    addi a1, zero, 95
+; RV32I-NEXT:    mv a0, s2
+; RV32I-NEXT:    call __udivsi3
+; RV32I-NEXT:    add a0, s8, a0
+; RV32I-NEXT:    add a1, s7, s1
+; RV32I-NEXT:    add a2, s6, s4
+; RV32I-NEXT:    add a3, s5, s9
+; RV32I-NEXT:    sh a3, 6(s0)
+; RV32I-NEXT:    sh a2, 4(s0)
+; RV32I-NEXT:    sh a1, 2(s0)
+; RV32I-NEXT:    sh a0, 0(s0)
+; RV32I-NEXT:    lw s9, 4(sp)
+; RV32I-NEXT:    lw s8, 8(sp)
+; RV32I-NEXT:    lw s7, 12(sp)
+; RV32I-NEXT:    lw s6, 16(sp)
+; RV32I-NEXT:    lw s5, 20(sp)
+; RV32I-NEXT:    lw s4, 24(sp)
+; RV32I-NEXT:    lw s3, 28(sp)
+; RV32I-NEXT:    lw s2, 32(sp)
+; RV32I-NEXT:    lw s1, 36(sp)
+; RV32I-NEXT:    lw s0, 40(sp)
+; RV32I-NEXT:    lw ra, 44(sp)
+; RV32I-NEXT:    addi sp, sp, 48
+; RV32I-NEXT:    ret
+;
+; RV32IM-LABEL: combine_urem_udiv:
+; RV32IM:       # %bb.0:
+; RV32IM-NEXT:    lhu a6, 0(a1)
+; RV32IM-NEXT:    lhu a7, 4(a1)
+; RV32IM-NEXT:    lhu a4, 12(a1)
+; RV32IM-NEXT:    lhu a1, 8(a1)
+; RV32IM-NEXT:    lui a5, 364242
+; RV32IM-NEXT:    addi a5, a5, 777
+; RV32IM-NEXT:    mulhu a2, a4, a5
+; RV32IM-NEXT:    sub a3, a4, a2
+; RV32IM-NEXT:    srli a3, a3, 1
+; RV32IM-NEXT:    add a2, a3, a2
+; RV32IM-NEXT:    srli t3, a2, 6
+; RV32IM-NEXT:    addi t0, zero, 95
+; RV32IM-NEXT:    mul a3, t3, t0
+; RV32IM-NEXT:    sub t1, a4, a3
+; RV32IM-NEXT:    mulhu a4, a1, a5
+; RV32IM-NEXT:    sub a3, a1, a4
+; RV32IM-NEXT:    srli a3, a3, 1
+; RV32IM-NEXT:    add a3, a3, a4
+; RV32IM-NEXT:    srli a3, a3, 6
+; RV32IM-NEXT:    mul a4, a3, t0
+; RV32IM-NEXT:    sub t2, a1, a4
+; RV32IM-NEXT:    mulhu a4, a7, a5
+; RV32IM-NEXT:    sub a1, a7, a4
+; RV32IM-NEXT:    srli a1, a1, 1
+; RV32IM-NEXT:    add a1, a1, a4
+; RV32IM-NEXT:    srli a1, a1, 6
+; RV32IM-NEXT:    mul a4, a1, t0
+; RV32IM-NEXT:    sub a4, a7, a4
+; RV32IM-NEXT:    mulhu a5, a6, a5
+; RV32IM-NEXT:    sub a2, a6, a5
+; RV32IM-NEXT:    srli a2, a2, 1
+; RV32IM-NEXT:    add a2, a2, a5
+; RV32IM-NEXT:    srli a2, a2, 6
+; RV32IM-NEXT:    mul a5, a2, t0
+; RV32IM-NEXT:    sub a5, a6, a5
+; RV32IM-NEXT:    add a2, a5, a2
+; RV32IM-NEXT:    add a1, a4, a1
+; RV32IM-NEXT:    add a3, t2, a3
+; RV32IM-NEXT:    add a4, t1, t3
+; RV32IM-NEXT:    sh a4, 6(a0)
+; RV32IM-NEXT:    sh a3, 4(a0)
+; RV32IM-NEXT:    sh a1, 2(a0)
+; RV32IM-NEXT:    sh a2, 0(a0)
+; RV32IM-NEXT:    ret
+;
+; RV64I-LABEL: combine_urem_udiv:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -96
+; RV64I-NEXT:    sd ra, 88(sp)
+; RV64I-NEXT:    sd s0, 80(sp)
+; RV64I-NEXT:    sd s1, 72(sp)
+; RV64I-NEXT:    sd s2, 64(sp)
+; RV64I-NEXT:    sd s3, 56(sp)
+; RV64I-NEXT:    sd s4, 48(sp)
+; RV64I-NEXT:    sd s5, 40(sp)
+; RV64I-NEXT:    sd s6, 32(sp)
+; RV64I-NEXT:    sd s7, 24(sp)
+; RV64I-NEXT:    sd s8, 16(sp)
+; RV64I-NEXT:    sd s9, 8(sp)
+; RV64I-NEXT:    lhu s2, 0(a1)
+; RV64I-NEXT:    lhu s3, 8(a1)
+; RV64I-NEXT:    lhu s4, 16(a1)
+; RV64I-NEXT:    lhu s1, 24(a1)
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    addi a1, zero, 95
+; RV64I-NEXT:    mv a0, s1
+; RV64I-NEXT:    call __umoddi3
+; RV64I-NEXT:    mv s5, a0
+; RV64I-NEXT:    addi a1, zero, 95
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    call __umoddi3
+; RV64I-NEXT:    mv s6, a0
+; RV64I-NEXT:    addi a1, zero, 95
+; RV64I-NEXT:    mv a0, s3
+; RV64I-NEXT:    call __umoddi3
+; RV64I-NEXT:    mv s7, a0
+; RV64I-NEXT:    addi a1, zero, 95
+; RV64I-NEXT:    mv a0, s2
+; RV64I-NEXT:    call __umoddi3
+; RV64I-NEXT:    mv s8, a0
+; RV64I-NEXT:    addi a1, zero, 95
+; RV64I-NEXT:    mv a0, s1
+; RV64I-NEXT:    call __udivdi3
+; RV64I-NEXT:    mv s9, a0
+; RV64I-NEXT:    addi a1, zero, 95
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    call __udivdi3
+; RV64I-NEXT:    mv s4, a0
+; RV64I-NEXT:    addi a1, zero, 95
+; RV64I-NEXT:    mv a0, s3
+; RV64I-NEXT:    call __udivdi3
+; RV64I-NEXT:    mv s1, a0
+; RV64I-NEXT:    addi a1, zero, 95
+; RV64I-NEXT:    mv a0, s2
+; RV64I-NEXT:    call __udivdi3
+; RV64I-NEXT:    add a0, s8, a0
+; RV64I-NEXT:    add a1, s7, s1
+; RV64I-NEXT:    add a2, s6, s4
+; RV64I-NEXT:    add a3, s5, s9
+; RV64I-NEXT:    sh a3, 6(s0)
+; RV64I-NEXT:    sh a2, 4(s0)
+; RV64I-NEXT:    sh a1, 2(s0)
+; RV64I-NEXT:    sh a0, 0(s0)
+; RV64I-NEXT:    ld s9, 8(sp)
+; RV64I-NEXT:    ld s8, 16(sp)
+; RV64I-NEXT:    ld s7, 24(sp)
+; RV64I-NEXT:    ld s6, 32(sp)
+; RV64I-NEXT:    ld s5, 40(sp)
+; RV64I-NEXT:    ld s4, 48(sp)
+; RV64I-NEXT:    ld s3, 56(sp)
+; RV64I-NEXT:    ld s2, 64(sp)
+; RV64I-NEXT:    ld s1, 72(sp)
+; RV64I-NEXT:    ld s0, 80(sp)
+; RV64I-NEXT:    ld ra, 88(sp)
+; RV64I-NEXT:    addi sp, sp, 96
+; RV64I-NEXT:    ret
+;
+; RV64IM-LABEL: combine_urem_udiv:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    lhu a6, 0(a1)
+; RV64IM-NEXT:    lhu a7, 8(a1)
+; RV64IM-NEXT:    lhu a4, 16(a1)
+; RV64IM-NEXT:    lhu a1, 24(a1)
+; RV64IM-NEXT:    lui a5, 1423
+; RV64IM-NEXT:    addiw a5, a5, -733
+; RV64IM-NEXT:    slli a5, a5, 15
+; RV64IM-NEXT:    addi a5, a5, 1035
+; RV64IM-NEXT:    slli a5, a5, 13
+; RV64IM-NEXT:    addi a5, a5, -1811
+; RV64IM-NEXT:    slli a5, a5, 12
+; RV64IM-NEXT:    addi a5, a5, 561
+; RV64IM-NEXT:    mulhu a2, a1, a5
+; RV64IM-NEXT:    sub a3, a1, a2
+; RV64IM-NEXT:    srli a3, a3, 1
+; RV64IM-NEXT:    add a2, a3, a2
+; RV64IM-NEXT:    srli t3, a2, 6
+; RV64IM-NEXT:    addi t0, zero, 95
+; RV64IM-NEXT:    mul a3, t3, t0
+; RV64IM-NEXT:    sub t1, a1, a3
+; RV64IM-NEXT:    mulhu a3, a4, a5
+; RV64IM-NEXT:    sub a1, a4, a3
+; RV64IM-NEXT:    srli a1, a1, 1
+; RV64IM-NEXT:    add a1, a1, a3
+; RV64IM-NEXT:    srli a1, a1, 6
+; RV64IM-NEXT:    mul a3, a1, t0
+; RV64IM-NEXT:    sub t2, a4, a3
+; RV64IM-NEXT:    mulhu a4, a7, a5
+; RV64IM-NEXT:    sub a3, a7, a4
+; RV64IM-NEXT:    srli a3, a3, 1
+; RV64IM-NEXT:    add a3, a3, a4
+; RV64IM-NEXT:    srli a3, a3, 6
+; RV64IM-NEXT:    mul a4, a3, t0
+; RV64IM-NEXT:    sub a4, a7, a4
+; RV64IM-NEXT:    mulhu a5, a6, a5
+; RV64IM-NEXT:    sub a2, a6, a5
+; RV64IM-NEXT:    srli a2, a2, 1
+; RV64IM-NEXT:    add a2, a2, a5
+; RV64IM-NEXT:    srli a2, a2, 6
+; RV64IM-NEXT:    mul a5, a2, t0
+; RV64IM-NEXT:    sub a5, a6, a5
+; RV64IM-NEXT:    add a2, a5, a2
+; RV64IM-NEXT:    add a3, a4, a3
+; RV64IM-NEXT:    add a1, t2, a1
+; RV64IM-NEXT:    add a4, t1, t3
+; RV64IM-NEXT:    sh a4, 6(a0)
+; RV64IM-NEXT:    sh a1, 4(a0)
+; RV64IM-NEXT:    sh a3, 2(a0)
+; RV64IM-NEXT:    sh a2, 0(a0)
+; RV64IM-NEXT:    ret
+  %1 = urem <4 x i16> %x, <i16 95, i16 95, i16 95, i16 95>
+  %2 = udiv <4 x i16> %x, <i16 95, i16 95, i16 95, i16 95>
+  %3 = add <4 x i16> %1, %2
+  ret <4 x i16> %3
+}
+
+; Don't fold for divisors that are a power of two.
+define <4 x i16> @dont_fold_urem_power_of_two(<4 x i16> %x) nounwind {
+; RV32I-LABEL: dont_fold_urem_power_of_two:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -32
+; RV32I-NEXT:    sw ra, 28(sp)
+; RV32I-NEXT:    sw s0, 24(sp)
+; RV32I-NEXT:    sw s1, 20(sp)
+; RV32I-NEXT:    sw s2, 16(sp)
+; RV32I-NEXT:    sw s3, 12(sp)
+; RV32I-NEXT:    lhu s2, 8(a1)
+; RV32I-NEXT:    lhu s3, 4(a1)
+; RV32I-NEXT:    lhu s1, 0(a1)
+; RV32I-NEXT:    lhu a2, 12(a1)
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    addi a1, zero, 95
+; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    call __umodsi3
+; RV32I-NEXT:    andi a1, s1, 63
+; RV32I-NEXT:    andi a2, s3, 31
+; RV32I-NEXT:    andi a3, s2, 7
+; RV32I-NEXT:    sh a0, 6(s0)
+; RV32I-NEXT:    sh a3, 4(s0)
+; RV32I-NEXT:    sh a2, 2(s0)
+; RV32I-NEXT:    sh a1, 0(s0)
+; RV32I-NEXT:    lw s3, 12(sp)
+; RV32I-NEXT:    lw s2, 16(sp)
+; RV32I-NEXT:    lw s1, 20(sp)
+; RV32I-NEXT:    lw s0, 24(sp)
+; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    addi sp, sp, 32
+; RV32I-NEXT:    ret
+;
+; RV32IM-LABEL: dont_fold_urem_power_of_two:
+; RV32IM:       # %bb.0:
+; RV32IM-NEXT:    lhu a6, 8(a1)
+; RV32IM-NEXT:    lhu a3, 4(a1)
+; RV32IM-NEXT:    lhu a4, 12(a1)
+; RV32IM-NEXT:    lhu a1, 0(a1)
+; RV32IM-NEXT:    lui a5, 364242
+; RV32IM-NEXT:    addi a5, a5, 777
+; RV32IM-NEXT:    mulhu a5, a4, a5
+; RV32IM-NEXT:    sub a2, a4, a5
+; RV32IM-NEXT:    srli a2, a2, 1
+; RV32IM-NEXT:    add a2, a2, a5
+; RV32IM-NEXT:    srli a2, a2, 6
+; RV32IM-NEXT:    addi a5, zero, 95
+; RV32IM-NEXT:    mul a2, a2, a5
+; RV32IM-NEXT:    sub a2, a4, a2
+; RV32IM-NEXT:    andi a1, a1, 63
+; RV32IM-NEXT:    andi a3, a3, 31
+; RV32IM-NEXT:    andi a4, a6, 7
+; RV32IM-NEXT:    sh a4, 4(a0)
+; RV32IM-NEXT:    sh a3, 2(a0)
+; RV32IM-NEXT:    sh a1, 0(a0)
+; RV32IM-NEXT:    sh a2, 6(a0)
+; RV32IM-NEXT:    ret
+;
+; RV64I-LABEL: dont_fold_urem_power_of_two:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s0, 32(sp)
+; RV64I-NEXT:    sd s1, 24(sp)
+; RV64I-NEXT:    sd s2, 16(sp)
+; RV64I-NEXT:    sd s3, 8(sp)
+; RV64I-NEXT:    lhu s2, 16(a1)
+; RV64I-NEXT:    lhu s3, 8(a1)
+; RV64I-NEXT:    lhu s1, 0(a1)
+; RV64I-NEXT:    lhu a2, 24(a1)
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    addi a1, zero, 95
+; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    call __umoddi3
+; RV64I-NEXT:    andi a1, s1, 63
+; RV64I-NEXT:    andi a2, s3, 31
+; RV64I-NEXT:    andi a3, s2, 7
+; RV64I-NEXT:    sh a0, 6(s0)
+; RV64I-NEXT:    sh a3, 4(s0)
+; RV64I-NEXT:    sh a2, 2(s0)
+; RV64I-NEXT:    sh a1, 0(s0)
+; RV64I-NEXT:    ld s3, 8(sp)
+; RV64I-NEXT:    ld s2, 16(sp)
+; RV64I-NEXT:    ld s1, 24(sp)
+; RV64I-NEXT:    ld s0, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IM-LABEL: dont_fold_urem_power_of_two:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    lhu a6, 16(a1)
+; RV64IM-NEXT:    lhu a3, 8(a1)
+; RV64IM-NEXT:    lhu a4, 0(a1)
+; RV64IM-NEXT:    lhu a1, 24(a1)
+; RV64IM-NEXT:    lui a5, 1423
+; RV64IM-NEXT:    addiw a5, a5, -733
+; RV64IM-NEXT:    slli a5, a5, 15
+; RV64IM-NEXT:    addi a5, a5, 1035
+; RV64IM-NEXT:    slli a5, a5, 13
+; RV64IM-NEXT:    addi a5, a5, -1811
+; RV64IM-NEXT:    slli a5, a5, 12
+; RV64IM-NEXT:    addi a5, a5, 561
+; RV64IM-NEXT:    mulhu a5, a1, a5
+; RV64IM-NEXT:    sub a2, a1, a5
+; RV64IM-NEXT:    srli a2, a2, 1
+; RV64IM-NEXT:    add a2, a2, a5
+; RV64IM-NEXT:    srli a2, a2, 6
+; RV64IM-NEXT:    addi a5, zero, 95
+; RV64IM-NEXT:    mul a2, a2, a5
+; RV64IM-NEXT:    sub a1, a1, a2
+; RV64IM-NEXT:    andi a2, a4, 63
+; RV64IM-NEXT:    andi a3, a3, 31
+; RV64IM-NEXT:    andi a4, a6, 7
+; RV64IM-NEXT:    sh a4, 4(a0)
+; RV64IM-NEXT:    sh a3, 2(a0)
+; RV64IM-NEXT:    sh a2, 0(a0)
+; RV64IM-NEXT:    sh a1, 6(a0)
+; RV64IM-NEXT:    ret
+  %1 = urem <4 x i16> %x, <i16 64, i16 32, i16 8, i16 95>
+  ret <4 x i16> %1
+}
+
+; Don't fold if the divisor is one.
+define <4 x i16> @dont_fold_urem_one(<4 x i16> %x) nounwind {
+; RV32I-LABEL: dont_fold_urem_one:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -32
+; RV32I-NEXT:    sw ra, 28(sp)
+; RV32I-NEXT:    sw s0, 24(sp)
+; RV32I-NEXT:    sw s1, 20(sp)
+; RV32I-NEXT:    sw s2, 16(sp)
+; RV32I-NEXT:    sw s3, 12(sp)
+; RV32I-NEXT:    lhu s2, 12(a1)
+; RV32I-NEXT:    lhu s1, 8(a1)
+; RV32I-NEXT:    lhu a2, 4(a1)
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    addi a1, zero, 654
+; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    call __umodsi3
+; RV32I-NEXT:    mv s3, a0
+; RV32I-NEXT:    addi a1, zero, 23
+; RV32I-NEXT:    mv a0, s1
+; RV32I-NEXT:    call __umodsi3
+; RV32I-NEXT:    mv s1, a0
+; RV32I-NEXT:    lui a0, 1
+; RV32I-NEXT:    addi a1, a0, 1327
+; RV32I-NEXT:    mv a0, s2
+; RV32I-NEXT:    call __umodsi3
+; RV32I-NEXT:    sh zero, 0(s0)
+; RV32I-NEXT:    sh a0, 6(s0)
+; RV32I-NEXT:    sh s1, 4(s0)
+; RV32I-NEXT:    sh s3, 2(s0)
+; RV32I-NEXT:    lw s3, 12(sp)
+; RV32I-NEXT:    lw s2, 16(sp)
+; RV32I-NEXT:    lw s1, 20(sp)
+; RV32I-NEXT:    lw s0, 24(sp)
+; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    addi sp, sp, 32
+; RV32I-NEXT:    ret
+;
+; RV32IM-LABEL: dont_fold_urem_one:
+; RV32IM:       # %bb.0:
+; RV32IM-NEXT:    lhu a2, 4(a1)
+; RV32IM-NEXT:    lhu a3, 12(a1)
+; RV32IM-NEXT:    lhu a1, 8(a1)
+; RV32IM-NEXT:    srli a4, a2, 1
+; RV32IM-NEXT:    lui a5, 820904
+; RV32IM-NEXT:    addi a5, a5, -1903
+; RV32IM-NEXT:    mulhu a4, a4, a5
+; RV32IM-NEXT:    srli a4, a4, 8
+; RV32IM-NEXT:    addi a5, zero, 654
+; RV32IM-NEXT:    mul a4, a4, a5
+; RV32IM-NEXT:    sub a2, a2, a4
+; RV32IM-NEXT:    lui a4, 729444
+; RV32IM-NEXT:    addi a4, a4, 713
+; RV32IM-NEXT:    mulhu a4, a1, a4
+; RV32IM-NEXT:    srli a4, a4, 4
+; RV32IM-NEXT:    addi a5, zero, 23
+; RV32IM-NEXT:    mul a4, a4, a5
+; RV32IM-NEXT:    sub a1, a1, a4
+; RV32IM-NEXT:    lui a4, 395996
+; RV32IM-NEXT:    addi a4, a4, -2009
+; RV32IM-NEXT:    mulhu a4, a3, a4
+; RV32IM-NEXT:    srli a4, a4, 11
+; RV32IM-NEXT:    lui a5, 1
+; RV32IM-NEXT:    addi a5, a5, 1327
+; RV32IM-NEXT:    mul a4, a4, a5
+; RV32IM-NEXT:    sub a3, a3, a4
+; RV32IM-NEXT:    sh zero, 0(a0)
+; RV32IM-NEXT:    sh a3, 6(a0)
+; RV32IM-NEXT:    sh a1, 4(a0)
+; RV32IM-NEXT:    sh a2, 2(a0)
+; RV32IM-NEXT:    ret
+;
+; RV64I-LABEL: dont_fold_urem_one:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s0, 32(sp)
+; RV64I-NEXT:    sd s1, 24(sp)
+; RV64I-NEXT:    sd s2, 16(sp)
+; RV64I-NEXT:    sd s3, 8(sp)
+; RV64I-NEXT:    lhu s2, 24(a1)
+; RV64I-NEXT:    lhu s1, 16(a1)
+; RV64I-NEXT:    lhu a2, 8(a1)
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    addi a1, zero, 654
+; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    call __umoddi3
+; RV64I-NEXT:    mv s3, a0
+; RV64I-NEXT:    addi a1, zero, 23
+; RV64I-NEXT:    mv a0, s1
+; RV64I-NEXT:    call __umoddi3
+; RV64I-NEXT:    mv s1, a0
+; RV64I-NEXT:    lui a0, 1
+; RV64I-NEXT:    addiw a1, a0, 1327
+; RV64I-NEXT:    mv a0, s2
+; RV64I-NEXT:    call __umoddi3
+; RV64I-NEXT:    sh zero, 0(s0)
+; RV64I-NEXT:    sh a0, 6(s0)
+; RV64I-NEXT:    sh s1, 4(s0)
+; RV64I-NEXT:    sh s3, 2(s0)
+; RV64I-NEXT:    ld s3, 8(sp)
+; RV64I-NEXT:    ld s2, 16(sp)
+; RV64I-NEXT:    ld s1, 24(sp)
+; RV64I-NEXT:    ld s0, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IM-LABEL: dont_fold_urem_one:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    lhu a2, 24(a1)
+; RV64IM-NEXT:    lhu a3, 8(a1)
+; RV64IM-NEXT:    lhu a1, 16(a1)
+; RV64IM-NEXT:    lui a4, 3206
+; RV64IM-NEXT:    addiw a4, a4, -1781
+; RV64IM-NEXT:    slli a4, a4, 13
+; RV64IM-NEXT:    addi a4, a4, 1069
+; RV64IM-NEXT:    slli a4, a4, 12
+; RV64IM-NEXT:    addi a4, a4, -1959
+; RV64IM-NEXT:    slli a4, a4, 14
+; RV64IM-NEXT:    addi a4, a4, 713
+; RV64IM-NEXT:    mulhu a4, a1, a4
+; RV64IM-NEXT:    sub a5, a1, a4
+; RV64IM-NEXT:    srli a5, a5, 1
+; RV64IM-NEXT:    add a4, a5, a4
+; RV64IM-NEXT:    srli a4, a4, 4
+; RV64IM-NEXT:    addi a5, zero, 23
+; RV64IM-NEXT:    mul a4, a4, a5
+; RV64IM-NEXT:    sub a1, a1, a4
+; RV64IM-NEXT:    srli a4, a3, 1
+; RV64IM-NEXT:    lui a5, 6413
+; RV64IM-NEXT:    addiw a5, a5, 1265
+; RV64IM-NEXT:    slli a5, a5, 13
+; RV64IM-NEXT:    addi a5, a5, 1027
+; RV64IM-NEXT:    slli a5, a5, 13
+; RV64IM-NEXT:    addi a5, a5, 1077
+; RV64IM-NEXT:    slli a5, a5, 12
+; RV64IM-NEXT:    addi a5, a5, 965
+; RV64IM-NEXT:    mulhu a4, a4, a5
+; RV64IM-NEXT:    srli a4, a4, 7
+; RV64IM-NEXT:    addi a5, zero, 654
+; RV64IM-NEXT:    mul a4, a4, a5
+; RV64IM-NEXT:    sub a3, a3, a4
+; RV64IM-NEXT:    lui a4, 1044567
+; RV64IM-NEXT:    addiw a4, a4, -575
+; RV64IM-NEXT:    slli a4, a4, 12
+; RV64IM-NEXT:    addi a4, a4, 883
+; RV64IM-NEXT:    slli a4, a4, 14
+; RV64IM-NEXT:    addi a4, a4, -861
+; RV64IM-NEXT:    slli a4, a4, 12
+; RV64IM-NEXT:    addi a4, a4, -179
+; RV64IM-NEXT:    mulhu a4, a2, a4
+; RV64IM-NEXT:    srli a4, a4, 12
+; RV64IM-NEXT:    lui a5, 1
+; RV64IM-NEXT:    addiw a5, a5, 1327
+; RV64IM-NEXT:    mul a4, a4, a5
+; RV64IM-NEXT:    sub a2, a2, a4
+; RV64IM-NEXT:    sh zero, 0(a0)
+; RV64IM-NEXT:    sh a2, 6(a0)
+; RV64IM-NEXT:    sh a3, 2(a0)
+; RV64IM-NEXT:    sh a1, 4(a0)
+; RV64IM-NEXT:    ret
+  %1 = urem <4 x i16> %x, <i16 1, i16 654, i16 23, i16 5423>
+  ret <4 x i16> %1
+}
+
+; Don't fold if the divisor is 2^16.
+define <4 x i16> @dont_fold_urem_i16_smax(<4 x i16> %x) nounwind {
+; CHECK-LABEL: dont_fold_urem_i16_smax:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret
+  %1 = urem <4 x i16> %x, <i16 1, i16 65536, i16 23, i16 5423>
+  ret <4 x i16> %1
+}
+
+; Don't fold i64 urem.
+define <4 x i64> @dont_fold_urem_i64(<4 x i64> %x) nounwind {
+; RV32I-LABEL: dont_fold_urem_i64:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -48
+; RV32I-NEXT:    sw ra, 44(sp)
+; RV32I-NEXT:    sw s0, 40(sp)
+; RV32I-NEXT:    sw s1, 36(sp)
+; RV32I-NEXT:    sw s2, 32(sp)
+; RV32I-NEXT:    sw s3, 28(sp)
+; RV32I-NEXT:    sw s4, 24(sp)
+; RV32I-NEXT:    sw s5, 20(sp)
+; RV32I-NEXT:    sw s6, 16(sp)
+; RV32I-NEXT:    sw s7, 12(sp)
+; RV32I-NEXT:    sw s8, 8(sp)
+; RV32I-NEXT:    sw s9, 4(sp)
+; RV32I-NEXT:    lw s2, 24(a1)
+; RV32I-NEXT:    lw s3, 28(a1)
+; RV32I-NEXT:    lw s4, 16(a1)
+; RV32I-NEXT:    lw s5, 20(a1)
+; RV32I-NEXT:    lw s6, 8(a1)
+; RV32I-NEXT:    lw s1, 12(a1)
+; RV32I-NEXT:    lw a3, 0(a1)
+; RV32I-NEXT:    lw a1, 4(a1)
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    addi a2, zero, 1
+; RV32I-NEXT:    mv a0, a3
+; RV32I-NEXT:    mv a3, zero
+; RV32I-NEXT:    call __umoddi3
+; RV32I-NEXT:    mv s7, a0
+; RV32I-NEXT:    mv s8, a1
+; RV32I-NEXT:    addi a2, zero, 654
+; RV32I-NEXT:    mv a0, s6
+; RV32I-NEXT:    mv a1, s1
+; RV32I-NEXT:    mv a3, zero
+; RV32I-NEXT:    call __umoddi3
+; RV32I-NEXT:    mv s6, a0
+; RV32I-NEXT:    mv s9, a1
+; RV32I-NEXT:    addi a2, zero, 23
+; RV32I-NEXT:    mv a0, s4
+; RV32I-NEXT:    mv a1, s5
+; RV32I-NEXT:    mv a3, zero
+; RV32I-NEXT:    call __umoddi3
+; RV32I-NEXT:    mv s4, a0
+; RV32I-NEXT:    mv s1, a1
+; RV32I-NEXT:    lui a0, 1
+; RV32I-NEXT:    addi a2, a0, 1327
+; RV32I-NEXT:    mv a0, s2
+; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    mv a3, zero
+; RV32I-NEXT:    call __umoddi3
+; RV32I-NEXT:    sw a1, 28(s0)
+; RV32I-NEXT:    sw a0, 24(s0)
+; RV32I-NEXT:    sw s1, 20(s0)
+; RV32I-NEXT:    sw s4, 16(s0)
+; RV32I-NEXT:    sw s9, 12(s0)
+; RV32I-NEXT:    sw s6, 8(s0)
+; RV32I-NEXT:    sw s8, 4(s0)
+; RV32I-NEXT:    sw s7, 0(s0)
+; RV32I-NEXT:    lw s9, 4(sp)
+; RV32I-NEXT:    lw s8, 8(sp)
+; RV32I-NEXT:    lw s7, 12(sp)
+; RV32I-NEXT:    lw s6, 16(sp)
+; RV32I-NEXT:    lw s5, 20(sp)
+; RV32I-NEXT:    lw s4, 24(sp)
+; RV32I-NEXT:    lw s3, 28(sp)
+; RV32I-NEXT:    lw s2, 32(sp)
+; RV32I-NEXT:    lw s1, 36(sp)
+; RV32I-NEXT:    lw s0, 40(sp)
+; RV32I-NEXT:    lw ra, 44(sp)
+; RV32I-NEXT:    addi sp, sp, 48
+; RV32I-NEXT:    ret
+;
+; RV32IM-LABEL: dont_fold_urem_i64:
+; RV32IM:       # %bb.0:
+; RV32IM-NEXT:    addi sp, sp, -48
+; RV32IM-NEXT:    sw ra, 44(sp)
+; RV32IM-NEXT:    sw s0, 40(sp)
+; RV32IM-NEXT:    sw s1, 36(sp)
+; RV32IM-NEXT:    sw s2, 32(sp)
+; RV32IM-NEXT:    sw s3, 28(sp)
+; RV32IM-NEXT:    sw s4, 24(sp)
+; RV32IM-NEXT:    sw s5, 20(sp)
+; RV32IM-NEXT:    sw s6, 16(sp)
+; RV32IM-NEXT:    sw s7, 12(sp)
+; RV32IM-NEXT:    sw s8, 8(sp)
+; RV32IM-NEXT:    sw s9, 4(sp)
+; RV32IM-NEXT:    lw s2, 24(a1)
+; RV32IM-NEXT:    lw s3, 28(a1)
+; RV32IM-NEXT:    lw s4, 16(a1)
+; RV32IM-NEXT:    lw s5, 20(a1)
+; RV32IM-NEXT:    lw s6, 8(a1)
+; RV32IM-NEXT:    lw s1, 12(a1)
+; RV32IM-NEXT:    lw a3, 0(a1)
+; RV32IM-NEXT:    lw a1, 4(a1)
+; RV32IM-NEXT:    mv s0, a0
+; RV32IM-NEXT:    addi a2, zero, 1
+; RV32IM-NEXT:    mv a0, a3
+; RV32IM-NEXT:    mv a3, zero
+; RV32IM-NEXT:    call __umoddi3
+; RV32IM-NEXT:    mv s7, a0
+; RV32IM-NEXT:    mv s8, a1
+; RV32IM-NEXT:    addi a2, zero, 654
+; RV32IM-NEXT:    mv a0, s6
+; RV32IM-NEXT:    mv a1, s1
+; RV32IM-NEXT:    mv a3, zero
+; RV32IM-NEXT:    call __umoddi3
+; RV32IM-NEXT:    mv s6, a0
+; RV32IM-NEXT:    mv s9, a1
+; RV32IM-NEXT:    addi a2, zero, 23
+; RV32IM-NEXT:    mv a0, s4
+; RV32IM-NEXT:    mv a1, s5
+; RV32IM-NEXT:    mv a3, zero
+; RV32IM-NEXT:    call __umoddi3
+; RV32IM-NEXT:    mv s4, a0
+; RV32IM-NEXT:    mv s1, a1
+; RV32IM-NEXT:    lui a0, 1
+; RV32IM-NEXT:    addi a2, a0, 1327
+; RV32IM-NEXT:    mv a0, s2
+; RV32IM-NEXT:    mv a1, s3
+; RV32IM-NEXT:    mv a3, zero
+; RV32IM-NEXT:    call __umoddi3
+; RV32IM-NEXT:    sw a1, 28(s0)
+; RV32IM-NEXT:    sw a0, 24(s0)
+; RV32IM-NEXT:    sw s1, 20(s0)
+; RV32IM-NEXT:    sw s4, 16(s0)
+; RV32IM-NEXT:    sw s9, 12(s0)
+; RV32IM-NEXT:    sw s6, 8(s0)
+; RV32IM-NEXT:    sw s8, 4(s0)
+; RV32IM-NEXT:    sw s7, 0(s0)
+; RV32IM-NEXT:    lw s9, 4(sp)
+; RV32IM-NEXT:    lw s8, 8(sp)
+; RV32IM-NEXT:    lw s7, 12(sp)
+; RV32IM-NEXT:    lw s6, 16(sp)
+; RV32IM-NEXT:    lw s5, 20(sp)
+; RV32IM-NEXT:    lw s4, 24(sp)
+; RV32IM-NEXT:    lw s3, 28(sp)
+; RV32IM-NEXT:    lw s2, 32(sp)
+; RV32IM-NEXT:    lw s1, 36(sp)
+; RV32IM-NEXT:    lw s0, 40(sp)
+; RV32IM-NEXT:    lw ra, 44(sp)
+; RV32IM-NEXT:    addi sp, sp, 48
+; RV32IM-NEXT:    ret
+;
+; RV64I-LABEL: dont_fold_urem_i64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s0, 32(sp)
+; RV64I-NEXT:    sd s1, 24(sp)
+; RV64I-NEXT:    sd s2, 16(sp)
+; RV64I-NEXT:    sd s3, 8(sp)
+; RV64I-NEXT:    ld s2, 24(a1)
+; RV64I-NEXT:    ld s1, 16(a1)
+; RV64I-NEXT:    ld a2, 8(a1)
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    addi a1, zero, 654
+; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    call __umoddi3
+; RV64I-NEXT:    mv s3, a0
+; RV64I-NEXT:    addi a1, zero, 23
+; RV64I-NEXT:    mv a0, s1
+; RV64I-NEXT:    call __umoddi3
+; RV64I-NEXT:    mv s1, a0
+; RV64I-NEXT:    lui a0, 1
+; RV64I-NEXT:    addiw a1, a0, 1327
+; RV64I-NEXT:    mv a0, s2
+; RV64I-NEXT:    call __umoddi3
+; RV64I-NEXT:    sd zero, 0(s0)
+; RV64I-NEXT:    sd a0, 24(s0)
+; RV64I-NEXT:    sd s1, 16(s0)
+; RV64I-NEXT:    sd s3, 8(s0)
+; RV64I-NEXT:    ld s3, 8(sp)
+; RV64I-NEXT:    ld s2, 16(sp)
+; RV64I-NEXT:    ld s1, 24(sp)
+; RV64I-NEXT:    ld s0, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IM-LABEL: dont_fold_urem_i64:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    ld a2, 24(a1)
+; RV64IM-NEXT:    ld a3, 8(a1)
+; RV64IM-NEXT:    ld a1, 16(a1)
+; RV64IM-NEXT:    lui a4, 3206
+; RV64IM-NEXT:    addiw a4, a4, -1781
+; RV64IM-NEXT:    slli a4, a4, 13
+; RV64IM-NEXT:    addi a4, a4, 1069
+; RV64IM-NEXT:    slli a4, a4, 12
+; RV64IM-NEXT:    addi a4, a4, -1959
+; RV64IM-NEXT:    slli a4, a4, 14
+; RV64IM-NEXT:    addi a4, a4, 713
+; RV64IM-NEXT:    mulhu a4, a1, a4
+; RV64IM-NEXT:    sub a5, a1, a4
+; RV64IM-NEXT:    srli a5, a5, 1
+; RV64IM-NEXT:    add a4, a5, a4
+; RV64IM-NEXT:    srli a4, a4, 4
+; RV64IM-NEXT:    addi a5, zero, 23
+; RV64IM-NEXT:    mul a4, a4, a5
+; RV64IM-NEXT:    sub a1, a1, a4
+; RV64IM-NEXT:    srli a4, a3, 1
+; RV64IM-NEXT:    lui a5, 6413
+; RV64IM-NEXT:    addiw a5, a5, 1265
+; RV64IM-NEXT:    slli a5, a5, 13
+; RV64IM-NEXT:    addi a5, a5, 1027
+; RV64IM-NEXT:    slli a5, a5, 13
+; RV64IM-NEXT:    addi a5, a5, 1077
+; RV64IM-NEXT:    slli a5, a5, 12
+; RV64IM-NEXT:    addi a5, a5, 965
+; RV64IM-NEXT:    mulhu a4, a4, a5
+; RV64IM-NEXT:    srli a4, a4, 7
+; RV64IM-NEXT:    addi a5, zero, 654
+; RV64IM-NEXT:    mul a4, a4, a5
+; RV64IM-NEXT:    sub a3, a3, a4
+; RV64IM-NEXT:    lui a4, 1044567
+; RV64IM-NEXT:    addiw a4, a4, -575
+; RV64IM-NEXT:    slli a4, a4, 12
+; RV64IM-NEXT:    addi a4, a4, 883
+; RV64IM-NEXT:    slli a4, a4, 14
+; RV64IM-NEXT:    addi a4, a4, -861
+; RV64IM-NEXT:    slli a4, a4, 12
+; RV64IM-NEXT:    addi a4, a4, -179
+; RV64IM-NEXT:    mulhu a4, a2, a4
+; RV64IM-NEXT:    srli a4, a4, 12
+; RV64IM-NEXT:    lui a5, 1
+; RV64IM-NEXT:    addiw a5, a5, 1327
+; RV64IM-NEXT:    mul a4, a4, a5
+; RV64IM-NEXT:    sub a2, a2, a4
+; RV64IM-NEXT:    sd zero, 0(a0)
+; RV64IM-NEXT:    sd a2, 24(a0)
+; RV64IM-NEXT:    sd a3, 8(a0)
+; RV64IM-NEXT:    sd a1, 16(a0)
+; RV64IM-NEXT:    ret
+  %1 = urem <4 x i64> %x, <i64 1, i64 654, i64 23, i64 5423>
+  ret <4 x i64> %1
+}
diff --git a/llvm/test/CodeGen/RISCV/vararg.ll b/llvm/test/CodeGen/RISCV/vararg.ll
index 34132825c..afa766930 100644
--- a/llvm/test/CodeGen/RISCV/vararg.ll
+++ b/llvm/test/CodeGen/RISCV/vararg.ll
@@ -25,6 +25,8 @@
 ; The same vararg calling convention is used for ilp32/ilp32f/ilp32d and for
 ; lp64/lp64f/lp64d. Different CHECK lines are required for RV32D due to slight
 ; codegen differences due to the way the f64 load operations are lowered.
+; The nounwind attribute is omitted for some of the tests, to check that CFI
+; directives are correctly generated.
 
 declare void @llvm.va_start(i8*)
 declare void @llvm.va_end(i8*)
@@ -35,10 +37,11 @@ declare void @notdead(i8*)
 ; support for aggregate types, we test simple cases here to ensure they are
 ; lowered correctly
 
-define i32 @va1(i8* %fmt, ...) nounwind {
+define i32 @va1(i8* %fmt, ...) {
 ; ILP32-ILP32F-FPELIM-LABEL: va1:
 ; ILP32-ILP32F-FPELIM:       # %bb.0:
 ; ILP32-ILP32F-FPELIM-NEXT:    addi sp, sp, -48
+; ILP32-ILP32F-FPELIM-NEXT:    .cfi_def_cfa_offset 48
 ; ILP32-ILP32F-FPELIM-NEXT:    mv a0, a1
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a7, 44(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a6, 40(sp)
@@ -46,18 +49,22 @@ define i32 @va1(i8* %fmt, ...) nounwind {
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a4, 32(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a3, 28(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a2, 24(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    sw a1, 20(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    addi a1, sp, 24
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a1, 12(sp)
-; ILP32-ILP32F-FPELIM-NEXT:    sw a0, 20(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    addi sp, sp, 48
 ; ILP32-ILP32F-FPELIM-NEXT:    ret
 ;
 ; ILP32-ILP32F-WITHFP-LABEL: va1:
 ; ILP32-ILP32F-WITHFP:       # %bb.0:
 ; ILP32-ILP32F-WITHFP-NEXT:    addi sp, sp, -48
+; ILP32-ILP32F-WITHFP-NEXT:    .cfi_def_cfa_offset 48
 ; ILP32-ILP32F-WITHFP-NEXT:    sw ra, 12(sp)
 ; ILP32-ILP32F-WITHFP-NEXT:    sw s0, 8(sp)
+; ILP32-ILP32F-WITHFP-NEXT:    .cfi_offset ra, -36
+; ILP32-ILP32F-WITHFP-NEXT:    .cfi_offset s0, -40
 ; ILP32-ILP32F-WITHFP-NEXT:    addi s0, sp, 16
+; ILP32-ILP32F-WITHFP-NEXT:    .cfi_def_cfa s0, 0
 ; ILP32-ILP32F-WITHFP-NEXT:    mv a0, a1
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a7, 28(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a6, 24(s0)
@@ -65,9 +72,9 @@ define i32 @va1(i8* %fmt, ...) nounwind {
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a4, 16(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a3, 12(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a2, 8(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    sw a1, 4(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    addi a1, s0, 8
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a1, -12(s0)
-; ILP32-ILP32F-WITHFP-NEXT:    sw a0, 4(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    lw s0, 8(sp)
 ; ILP32-ILP32F-WITHFP-NEXT:    lw ra, 12(sp)
 ; ILP32-ILP32F-WITHFP-NEXT:    addi sp, sp, 48
@@ -76,6 +83,7 @@ define i32 @va1(i8* %fmt, ...) nounwind {
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-LABEL: va1:
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM:       # %bb.0:
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi sp, sp, -48
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    .cfi_def_cfa_offset 48
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    mv a0, a1
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a7, 44(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a6, 40(sp)
@@ -83,15 +91,16 @@ define i32 @va1(i8* %fmt, ...) nounwind {
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a4, 32(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a3, 28(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a2, 24(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a1, 20(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a1, sp, 24
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a1, 12(sp)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a0, 20(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi sp, sp, 48
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    ret
 ;
 ; LP64-LP64F-LP64D-FPELIM-LABEL: va1:
 ; LP64-LP64F-LP64D-FPELIM:       # %bb.0:
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi sp, sp, -80
+; LP64-LP64F-LP64D-FPELIM-NEXT:    .cfi_def_cfa_offset 80
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a1, 24(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a7, 72(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a6, 64(sp)
@@ -109,9 +118,13 @@ define i32 @va1(i8* %fmt, ...) nounwind {
 ; LP64-LP64F-LP64D-WITHFP-LABEL: va1:
 ; LP64-LP64F-LP64D-WITHFP:       # %bb.0:
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi sp, sp, -96
+; LP64-LP64F-LP64D-WITHFP-NEXT:    .cfi_def_cfa_offset 96
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd ra, 24(sp)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd s0, 16(sp)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    .cfi_offset ra, -72
+; LP64-LP64F-LP64D-WITHFP-NEXT:    .cfi_offset s0, -80
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi s0, sp, 32
+; LP64-LP64F-LP64D-WITHFP-NEXT:    .cfi_def_cfa s0, 0
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a1, 8(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a7, 56(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a6, 48(s0)
@@ -150,9 +163,9 @@ define i32 @va1_va_arg(i8* %fmt, ...) nounwind {
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a4, 32(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a3, 28(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a2, 24(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    sw a1, 20(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    addi a1, sp, 24
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a1, 12(sp)
-; ILP32-ILP32F-FPELIM-NEXT:    sw a0, 20(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    addi sp, sp, 48
 ; ILP32-ILP32F-FPELIM-NEXT:    ret
 ;
@@ -169,9 +182,9 @@ define i32 @va1_va_arg(i8* %fmt, ...) nounwind {
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a4, 16(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a3, 12(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a2, 8(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    sw a1, 4(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    addi a1, s0, 8
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a1, -12(s0)
-; ILP32-ILP32F-WITHFP-NEXT:    sw a0, 4(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    lw s0, 8(sp)
 ; ILP32-ILP32F-WITHFP-NEXT:    lw ra, 12(sp)
 ; ILP32-ILP32F-WITHFP-NEXT:    addi sp, sp, 48
@@ -187,9 +200,9 @@ define i32 @va1_va_arg(i8* %fmt, ...) nounwind {
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a4, 32(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a3, 28(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a2, 24(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a1, 20(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a1, sp, 24
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a1, 12(sp)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a0, 20(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi sp, sp, 48
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    ret
 ;
@@ -203,10 +216,10 @@ define i32 @va1_va_arg(i8* %fmt, ...) nounwind {
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a4, 48(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a3, 40(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a2, 32(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a1, 24(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a1, sp, 24
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a1, a1, 8
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a1, 8(sp)
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a0, 24(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi sp, sp, 80
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    ret
 ;
@@ -223,10 +236,10 @@ define i32 @va1_va_arg(i8* %fmt, ...) nounwind {
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a4, 32(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a3, 24(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a2, 16(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a1, 8(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a1, s0, 8
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a1, a1, 8
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a1, -24(s0)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a0, 8(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    ld s0, 16(sp)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    ld ra, 24(sp)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi sp, sp, 96
@@ -256,9 +269,9 @@ define i32 @va1_va_arg_alloca(i8* %fmt, ...) nounwind {
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a4, 16(s0)
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a3, 12(s0)
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a2, 8(s0)
+; ILP32-ILP32F-FPELIM-NEXT:    sw a1, 4(s0)
 ; ILP32-ILP32F-FPELIM-NEXT:    addi a0, s0, 8
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a0, -16(s0)
-; ILP32-ILP32F-FPELIM-NEXT:    sw a1, 4(s0)
 ; ILP32-ILP32F-FPELIM-NEXT:    addi a0, a1, 15
 ; ILP32-ILP32F-FPELIM-NEXT:    andi a0, a0, -16
 ; ILP32-ILP32F-FPELIM-NEXT:    sub a0, sp, a0
@@ -286,9 +299,9 @@ define i32 @va1_va_arg_alloca(i8* %fmt, ...) nounwind {
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a4, 16(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a3, 12(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a2, 8(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    sw a1, 4(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    addi a0, s0, 8
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a0, -16(s0)
-; ILP32-ILP32F-WITHFP-NEXT:    sw a1, 4(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    addi a0, a1, 15
 ; ILP32-ILP32F-WITHFP-NEXT:    andi a0, a0, -16
 ; ILP32-ILP32F-WITHFP-NEXT:    sub a0, sp, a0
@@ -316,9 +329,9 @@ define i32 @va1_va_arg_alloca(i8* %fmt, ...) nounwind {
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a4, 16(s0)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a3, 12(s0)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a2, 8(s0)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a1, 4(s0)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a0, s0, 8
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a0, -16(s0)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a1, 4(s0)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a0, a1, 15
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    andi a0, a0, -16
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sub a0, sp, a0
@@ -346,17 +359,17 @@ define i32 @va1_va_arg_alloca(i8* %fmt, ...) nounwind {
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a4, 32(s0)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a3, 24(s0)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a2, 16(s0)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a1, 8(s0)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a0, s0, 8
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a0, a0, 8
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a0, -32(s0)
-; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a0, zero, 1
-; LP64-LP64F-LP64D-FPELIM-NEXT:    slli a0, a0, 33
-; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a0, a0, -16
-; LP64-LP64F-LP64D-FPELIM-NEXT:    slli a1, a1, 32
-; LP64-LP64F-LP64D-FPELIM-NEXT:    srli a1, a1, 32
-; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a1, a1, 15
-; LP64-LP64F-LP64D-FPELIM-NEXT:    and a0, a1, a0
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd s1, 8(s0)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    slli a0, a1, 32
+; LP64-LP64F-LP64D-FPELIM-NEXT:    srli a0, a0, 32
+; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a0, a0, 15
+; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a1, zero, 1
+; LP64-LP64F-LP64D-FPELIM-NEXT:    slli a1, a1, 33
+; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a1, a1, -16
+; LP64-LP64F-LP64D-FPELIM-NEXT:    and a0, a0, a1
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sub a0, sp, a0
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    mv sp, a0
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    call notdead
@@ -382,17 +395,17 @@ define i32 @va1_va_arg_alloca(i8* %fmt, ...) nounwind {
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a4, 32(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a3, 24(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a2, 16(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a1, 8(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a0, s0, 8
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a0, a0, 8
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a0, -32(s0)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a0, zero, 1
-; LP64-LP64F-LP64D-WITHFP-NEXT:    slli a0, a0, 33
-; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a0, a0, -16
-; LP64-LP64F-LP64D-WITHFP-NEXT:    slli a1, a1, 32
-; LP64-LP64F-LP64D-WITHFP-NEXT:    srli a1, a1, 32
-; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a1, a1, 15
-; LP64-LP64F-LP64D-WITHFP-NEXT:    and a0, a1, a0
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd s1, 8(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    slli a0, a1, 32
+; LP64-LP64F-LP64D-WITHFP-NEXT:    srli a0, a0, 32
+; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a0, a0, 15
+; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a1, zero, 1
+; LP64-LP64F-LP64D-WITHFP-NEXT:    slli a1, a1, 33
+; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a1, a1, -16
+; LP64-LP64F-LP64D-WITHFP-NEXT:    and a0, a0, a1
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sub a0, sp, a0
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    mv sp, a0
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    call notdead
@@ -419,9 +432,9 @@ define void @va1_caller() nounwind {
 ; ILP32-ILP32F-FPELIM:       # %bb.0:
 ; ILP32-ILP32F-FPELIM-NEXT:    addi sp, sp, -16
 ; ILP32-ILP32F-FPELIM-NEXT:    sw ra, 12(sp)
-; ILP32-ILP32F-FPELIM-NEXT:    mv a2, zero
 ; ILP32-ILP32F-FPELIM-NEXT:    lui a3, 261888
 ; ILP32-ILP32F-FPELIM-NEXT:    addi a4, zero, 2
+; ILP32-ILP32F-FPELIM-NEXT:    mv a2, zero
 ; ILP32-ILP32F-FPELIM-NEXT:    call va1
 ; ILP32-ILP32F-FPELIM-NEXT:    lw ra, 12(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    addi sp, sp, 16
@@ -433,9 +446,9 @@ define void @va1_caller() nounwind {
 ; ILP32-ILP32F-WITHFP-NEXT:    sw ra, 12(sp)
 ; ILP32-ILP32F-WITHFP-NEXT:    sw s0, 8(sp)
 ; ILP32-ILP32F-WITHFP-NEXT:    addi s0, sp, 16
-; ILP32-ILP32F-WITHFP-NEXT:    mv a2, zero
 ; ILP32-ILP32F-WITHFP-NEXT:    lui a3, 261888
 ; ILP32-ILP32F-WITHFP-NEXT:    addi a4, zero, 2
+; ILP32-ILP32F-WITHFP-NEXT:    mv a2, zero
 ; ILP32-ILP32F-WITHFP-NEXT:    call va1
 ; ILP32-ILP32F-WITHFP-NEXT:    lw s0, 8(sp)
 ; ILP32-ILP32F-WITHFP-NEXT:    lw ra, 12(sp)
@@ -446,9 +459,9 @@ define void @va1_caller() nounwind {
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM:       # %bb.0:
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi sp, sp, -16
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw ra, 12(sp)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    mv a2, zero
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lui a3, 261888
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a4, zero, 2
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    mv a2, zero
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    call va1
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lw ra, 12(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi sp, sp, 16
@@ -498,10 +511,10 @@ define i64 @va2(i8 *%fmt, ...) nounwind {
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a3, 28(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a2, 24(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a1, 20(sp)
-; ILP32-ILP32F-FPELIM-NEXT:    addi a0, sp, 35
-; ILP32-ILP32F-FPELIM-NEXT:    sw a0, 12(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    addi a0, sp, 27
 ; ILP32-ILP32F-FPELIM-NEXT:    andi a1, a0, -8
+; ILP32-ILP32F-FPELIM-NEXT:    addi a0, sp, 35
+; ILP32-ILP32F-FPELIM-NEXT:    sw a0, 12(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    lw a0, 0(a1)
 ; ILP32-ILP32F-FPELIM-NEXT:    ori a1, a1, 4
 ; ILP32-ILP32F-FPELIM-NEXT:    lw a1, 0(a1)
@@ -521,10 +534,10 @@ define i64 @va2(i8 *%fmt, ...) nounwind {
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a3, 12(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a2, 8(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a1, 4(s0)
-; ILP32-ILP32F-WITHFP-NEXT:    addi a0, s0, 19
-; ILP32-ILP32F-WITHFP-NEXT:    sw a0, -12(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    addi a0, s0, 11
 ; ILP32-ILP32F-WITHFP-NEXT:    andi a1, a0, -8
+; ILP32-ILP32F-WITHFP-NEXT:    addi a0, s0, 19
+; ILP32-ILP32F-WITHFP-NEXT:    sw a0, -12(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    lw a0, 0(a1)
 ; ILP32-ILP32F-WITHFP-NEXT:    ori a1, a1, 4
 ; ILP32-ILP32F-WITHFP-NEXT:    lw a1, 0(a1)
@@ -543,10 +556,10 @@ define i64 @va2(i8 *%fmt, ...) nounwind {
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a3, 28(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a2, 24(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a1, 20(sp)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a0, sp, 35
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a0, 12(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a0, sp, 27
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    andi a1, a0, -8
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a0, sp, 35
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a0, 12(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lw a0, 0(a1)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    ori a1, a1, 4
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lw a1, 0(a1)
@@ -556,8 +569,6 @@ define i64 @va2(i8 *%fmt, ...) nounwind {
 ; LP64-LP64F-LP64D-FPELIM-LABEL: va2:
 ; LP64-LP64F-LP64D-FPELIM:       # %bb.0:
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi sp, sp, -80
-; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a0, sp, 24
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a0, 8(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a7, 72(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a6, 64(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a5, 56(sp)
@@ -565,6 +576,8 @@ define i64 @va2(i8 *%fmt, ...) nounwind {
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a3, 40(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a2, 32(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a1, 24(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a0, sp, 24
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a0, 8(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    lw a0, 8(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a0, a0, 7
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    slli a1, a0, 32
@@ -585,8 +598,6 @@ define i64 @va2(i8 *%fmt, ...) nounwind {
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd ra, 24(sp)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd s0, 16(sp)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi s0, sp, 32
-; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a0, s0, 8
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a0, -24(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a7, 56(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a6, 48(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a5, 40(s0)
@@ -594,6 +605,8 @@ define i64 @va2(i8 *%fmt, ...) nounwind {
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a3, 24(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a2, 16(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a1, 8(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a0, s0, 8
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a0, -24(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    lw a0, -24(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a0, a0, 7
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    slli a1, a0, 32
@@ -705,10 +718,10 @@ define i64 @va2_va_arg(i8 *%fmt, ...) nounwind {
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a4, 48(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a3, 40(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a2, 32(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a1, 24(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a1, sp, 24
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a1, a1, 8
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a1, 8(sp)
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a0, 24(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi sp, sp, 80
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    ret
 ;
@@ -725,10 +738,10 @@ define i64 @va2_va_arg(i8 *%fmt, ...) nounwind {
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a4, 32(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a3, 24(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a2, 16(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a1, 8(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a1, s0, 8
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a1, a1, 8
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a1, -24(s0)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a0, 8(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    ld s0, 16(sp)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    ld ra, 24(sp)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi sp, sp, 96
@@ -747,8 +760,8 @@ define void @va2_caller() nounwind {
 ; ILP32-ILP32F-FPELIM:       # %bb.0:
 ; ILP32-ILP32F-FPELIM-NEXT:    addi sp, sp, -16
 ; ILP32-ILP32F-FPELIM-NEXT:    sw ra, 12(sp)
-; ILP32-ILP32F-FPELIM-NEXT:    mv a2, zero
 ; ILP32-ILP32F-FPELIM-NEXT:    lui a3, 261888
+; ILP32-ILP32F-FPELIM-NEXT:    mv a2, zero
 ; ILP32-ILP32F-FPELIM-NEXT:    call va2
 ; ILP32-ILP32F-FPELIM-NEXT:    lw ra, 12(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    addi sp, sp, 16
@@ -760,8 +773,8 @@ define void @va2_caller() nounwind {
 ; ILP32-ILP32F-WITHFP-NEXT:    sw ra, 12(sp)
 ; ILP32-ILP32F-WITHFP-NEXT:    sw s0, 8(sp)
 ; ILP32-ILP32F-WITHFP-NEXT:    addi s0, sp, 16
-; ILP32-ILP32F-WITHFP-NEXT:    mv a2, zero
 ; ILP32-ILP32F-WITHFP-NEXT:    lui a3, 261888
+; ILP32-ILP32F-WITHFP-NEXT:    mv a2, zero
 ; ILP32-ILP32F-WITHFP-NEXT:    call va2
 ; ILP32-ILP32F-WITHFP-NEXT:    lw s0, 8(sp)
 ; ILP32-ILP32F-WITHFP-NEXT:    lw ra, 12(sp)
@@ -772,8 +785,8 @@ define void @va2_caller() nounwind {
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM:       # %bb.0:
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi sp, sp, -16
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw ra, 12(sp)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    mv a2, zero
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lui a3, 261888
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    mv a2, zero
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    call va2
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lw ra, 12(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi sp, sp, 16
@@ -819,16 +832,16 @@ define i64 @va3(i32 %a, i64 %b, ...) nounwind {
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a5, 20(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a4, 16(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a3, 12(sp)
-; ILP32-ILP32F-FPELIM-NEXT:    addi a0, sp, 27
-; ILP32-ILP32F-FPELIM-NEXT:    sw a0, 4(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    addi a0, sp, 19
 ; ILP32-ILP32F-FPELIM-NEXT:    andi a0, a0, -8
-; ILP32-ILP32F-FPELIM-NEXT:    ori a3, a0, 4
-; ILP32-ILP32F-FPELIM-NEXT:    lw a3, 0(a3)
-; ILP32-ILP32F-FPELIM-NEXT:    add a2, a2, a3
-; ILP32-ILP32F-FPELIM-NEXT:    lw a0, 0(a0)
-; ILP32-ILP32F-FPELIM-NEXT:    add a0, a1, a0
+; ILP32-ILP32F-FPELIM-NEXT:    addi a3, sp, 27
+; ILP32-ILP32F-FPELIM-NEXT:    sw a3, 4(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    lw a3, 0(a0)
+; ILP32-ILP32F-FPELIM-NEXT:    ori a0, a0, 4
+; ILP32-ILP32F-FPELIM-NEXT:    lw a4, 0(a0)
+; ILP32-ILP32F-FPELIM-NEXT:    add a0, a1, a3
 ; ILP32-ILP32F-FPELIM-NEXT:    sltu a1, a0, a1
+; ILP32-ILP32F-FPELIM-NEXT:    add a2, a2, a4
 ; ILP32-ILP32F-FPELIM-NEXT:    add a1, a2, a1
 ; ILP32-ILP32F-FPELIM-NEXT:    addi sp, sp, 32
 ; ILP32-ILP32F-FPELIM-NEXT:    ret
@@ -844,16 +857,16 @@ define i64 @va3(i32 %a, i64 %b, ...) nounwind {
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a5, 12(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a4, 8(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a3, 4(s0)
-; ILP32-ILP32F-WITHFP-NEXT:    addi a0, s0, 19
-; ILP32-ILP32F-WITHFP-NEXT:    sw a0, -12(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    addi a0, s0, 11
 ; ILP32-ILP32F-WITHFP-NEXT:    andi a0, a0, -8
-; ILP32-ILP32F-WITHFP-NEXT:    ori a3, a0, 4
-; ILP32-ILP32F-WITHFP-NEXT:    lw a3, 0(a3)
-; ILP32-ILP32F-WITHFP-NEXT:    add a2, a2, a3
-; ILP32-ILP32F-WITHFP-NEXT:    lw a0, 0(a0)
-; ILP32-ILP32F-WITHFP-NEXT:    add a0, a1, a0
+; ILP32-ILP32F-WITHFP-NEXT:    addi a3, s0, 19
+; ILP32-ILP32F-WITHFP-NEXT:    sw a3, -12(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    lw a3, 0(a0)
+; ILP32-ILP32F-WITHFP-NEXT:    ori a0, a0, 4
+; ILP32-ILP32F-WITHFP-NEXT:    lw a4, 0(a0)
+; ILP32-ILP32F-WITHFP-NEXT:    add a0, a1, a3
 ; ILP32-ILP32F-WITHFP-NEXT:    sltu a1, a0, a1
+; ILP32-ILP32F-WITHFP-NEXT:    add a2, a2, a4
 ; ILP32-ILP32F-WITHFP-NEXT:    add a1, a2, a1
 ; ILP32-ILP32F-WITHFP-NEXT:    lw s0, 16(sp)
 ; ILP32-ILP32F-WITHFP-NEXT:    lw ra, 20(sp)
@@ -868,16 +881,16 @@ define i64 @va3(i32 %a, i64 %b, ...) nounwind {
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a5, 20(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a4, 16(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a3, 12(sp)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a0, sp, 27
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a0, 4(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a0, sp, 19
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    andi a0, a0, -8
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    ori a3, a0, 4
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lw a3, 0(a3)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    add a2, a2, a3
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lw a0, 0(a0)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    add a0, a1, a0
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a3, sp, 27
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a3, 4(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lw a3, 0(a0)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    ori a0, a0, 4
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lw a4, 0(a0)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    add a0, a1, a3
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sltu a1, a0, a1
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    add a2, a2, a4
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    add a1, a2, a1
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi sp, sp, 32
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    ret
@@ -885,15 +898,15 @@ define i64 @va3(i32 %a, i64 %b, ...) nounwind {
 ; LP64-LP64F-LP64D-FPELIM-LABEL: va3:
 ; LP64-LP64F-LP64D-FPELIM:       # %bb.0:
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi sp, sp, -64
-; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a0, sp, 16
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a0, 8(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a7, 56(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a6, 48(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a5, 40(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a4, 32(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a3, 24(sp)
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a2, 16(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a0, sp, 16
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a0, 8(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    lw a0, 8(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a2, 16(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a0, a0, 7
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    slli a2, a0, 32
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    srli a2, a2, 32
@@ -914,15 +927,15 @@ define i64 @va3(i32 %a, i64 %b, ...) nounwind {
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd ra, 24(sp)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd s0, 16(sp)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi s0, sp, 32
-; LP64-LP64F-LP64D-WITHFP-NEXT:    mv a0, s0
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a0, -24(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a7, 40(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a6, 32(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a5, 24(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a4, 16(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a3, 8(s0)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a2, 0(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    mv a0, s0
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a0, -24(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    lw a0, -24(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a2, 0(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a0, a0, 7
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    slli a2, a0, 32
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    srli a2, a2, 32
@@ -973,9 +986,9 @@ define i64 @va3_va_arg(i32 %a, i64 %b, ...) nounwind {
 ; ILP32-ILP32F-FPELIM-NEXT:    addi a4, a3, 4
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a4, 4(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    lw a3, 0(a3)
-; ILP32-ILP32F-FPELIM-NEXT:    add a2, a2, a3
 ; ILP32-ILP32F-FPELIM-NEXT:    add a0, a1, a0
 ; ILP32-ILP32F-FPELIM-NEXT:    sltu a1, a0, a1
+; ILP32-ILP32F-FPELIM-NEXT:    add a2, a2, a3
 ; ILP32-ILP32F-FPELIM-NEXT:    add a1, a2, a1
 ; ILP32-ILP32F-FPELIM-NEXT:    addi sp, sp, 32
 ; ILP32-ILP32F-FPELIM-NEXT:    ret
@@ -999,9 +1012,9 @@ define i64 @va3_va_arg(i32 %a, i64 %b, ...) nounwind {
 ; ILP32-ILP32F-WITHFP-NEXT:    addi a4, a3, 4
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a4, -12(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    lw a3, 0(a3)
-; ILP32-ILP32F-WITHFP-NEXT:    add a2, a2, a3
 ; ILP32-ILP32F-WITHFP-NEXT:    add a0, a1, a0
 ; ILP32-ILP32F-WITHFP-NEXT:    sltu a1, a0, a1
+; ILP32-ILP32F-WITHFP-NEXT:    add a2, a2, a3
 ; ILP32-ILP32F-WITHFP-NEXT:    add a1, a2, a1
 ; ILP32-ILP32F-WITHFP-NEXT:    lw s0, 16(sp)
 ; ILP32-ILP32F-WITHFP-NEXT:    lw ra, 20(sp)
@@ -1023,9 +1036,9 @@ define i64 @va3_va_arg(i32 %a, i64 %b, ...) nounwind {
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    fld ft0, 0(a0)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    fsd ft0, 8(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lw a0, 12(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lw a3, 8(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    add a2, a2, a0
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lw a0, 8(sp)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    add a0, a1, a0
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    add a0, a1, a3
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sltu a1, a0, a1
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    add a1, a2, a1
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi sp, sp, 48
@@ -1039,11 +1052,11 @@ define i64 @va3_va_arg(i32 %a, i64 %b, ...) nounwind {
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a5, 40(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a4, 32(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a3, 24(sp)
-; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a0, sp, 16
-; LP64-LP64F-LP64D-FPELIM-NEXT:    ori a0, a0, 8
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a0, 8(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a2, 16(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a0, sp, 16
+; LP64-LP64F-LP64D-FPELIM-NEXT:    ori a3, a0, 8
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    add a0, a1, a2
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a3, 8(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi sp, sp, 64
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    ret
 ;
@@ -1058,11 +1071,11 @@ define i64 @va3_va_arg(i32 %a, i64 %b, ...) nounwind {
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a5, 24(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a4, 16(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a3, 8(s0)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    mv a0, s0
-; LP64-LP64F-LP64D-WITHFP-NEXT:    ori a0, a0, 8
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a0, -24(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a2, 0(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    mv a0, s0
+; LP64-LP64F-LP64D-WITHFP-NEXT:    ori a3, a0, 8
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    add a0, a1, a2
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a3, -24(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    ld s0, 16(sp)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    ld ra, 24(sp)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi sp, sp, 80
@@ -1084,9 +1097,9 @@ define void @va3_caller() nounwind {
 ; ILP32-ILP32F-FPELIM-NEXT:    sw ra, 12(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    addi a0, zero, 2
 ; ILP32-ILP32F-FPELIM-NEXT:    addi a1, zero, 1111
+; ILP32-ILP32F-FPELIM-NEXT:    lui a5, 262144
 ; ILP32-ILP32F-FPELIM-NEXT:    mv a2, zero
 ; ILP32-ILP32F-FPELIM-NEXT:    mv a4, zero
-; ILP32-ILP32F-FPELIM-NEXT:    lui a5, 262144
 ; ILP32-ILP32F-FPELIM-NEXT:    call va3
 ; ILP32-ILP32F-FPELIM-NEXT:    lw ra, 12(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    addi sp, sp, 16
@@ -1100,9 +1113,9 @@ define void @va3_caller() nounwind {
 ; ILP32-ILP32F-WITHFP-NEXT:    addi s0, sp, 16
 ; ILP32-ILP32F-WITHFP-NEXT:    addi a0, zero, 2
 ; ILP32-ILP32F-WITHFP-NEXT:    addi a1, zero, 1111
+; ILP32-ILP32F-WITHFP-NEXT:    lui a5, 262144
 ; ILP32-ILP32F-WITHFP-NEXT:    mv a2, zero
 ; ILP32-ILP32F-WITHFP-NEXT:    mv a4, zero
-; ILP32-ILP32F-WITHFP-NEXT:    lui a5, 262144
 ; ILP32-ILP32F-WITHFP-NEXT:    call va3
 ; ILP32-ILP32F-WITHFP-NEXT:    lw s0, 8(sp)
 ; ILP32-ILP32F-WITHFP-NEXT:    lw ra, 12(sp)
@@ -1115,9 +1128,9 @@ define void @va3_caller() nounwind {
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw ra, 12(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a0, zero, 2
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a1, zero, 1111
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lui a5, 262144
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    mv a2, zero
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    mv a4, zero
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lui a5, 262144
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    call va3
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lw ra, 12(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi sp, sp, 16
@@ -1190,9 +1203,9 @@ define i32 @va4_va_copy(i32 %argno, ...) nounwind {
 ; ILP32-ILP32F-FPELIM-NEXT:    andi a0, a0, -4
 ; ILP32-ILP32F-FPELIM-NEXT:    addi a3, a0, 4
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a3, 4(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    lw a0, 0(a0)
 ; ILP32-ILP32F-FPELIM-NEXT:    add a1, a1, s0
 ; ILP32-ILP32F-FPELIM-NEXT:    add a1, a1, a2
-; ILP32-ILP32F-FPELIM-NEXT:    lw a0, 0(a0)
 ; ILP32-ILP32F-FPELIM-NEXT:    add a0, a1, a0
 ; ILP32-ILP32F-FPELIM-NEXT:    lw s0, 8(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    lw ra, 12(sp)
@@ -1233,9 +1246,9 @@ define i32 @va4_va_copy(i32 %argno, ...) nounwind {
 ; ILP32-ILP32F-WITHFP-NEXT:    andi a0, a0, -4
 ; ILP32-ILP32F-WITHFP-NEXT:    addi a3, a0, 4
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a3, -16(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    lw a0, 0(a0)
 ; ILP32-ILP32F-WITHFP-NEXT:    add a1, a1, s1
 ; ILP32-ILP32F-WITHFP-NEXT:    add a1, a1, a2
-; ILP32-ILP32F-WITHFP-NEXT:    lw a0, 0(a0)
 ; ILP32-ILP32F-WITHFP-NEXT:    add a0, a1, a0
 ; ILP32-ILP32F-WITHFP-NEXT:    lw s1, 20(sp)
 ; ILP32-ILP32F-WITHFP-NEXT:    lw s0, 24(sp)
@@ -1275,9 +1288,9 @@ define i32 @va4_va_copy(i32 %argno, ...) nounwind {
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    andi a0, a0, -4
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a3, a0, 4
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a3, 4(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lw a0, 0(a0)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    add a1, a1, s0
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    add a1, a1, a2
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lw a0, 0(a0)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    add a0, a1, a0
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lw s0, 8(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lw ra, 12(sp)
@@ -1317,10 +1330,10 @@ define i32 @va4_va_copy(i32 %argno, ...) nounwind {
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    andi a0, a0, -4
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a3, a0, 8
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a3, 8(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    ld a0, 0(a0)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    add a1, a1, s0
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    add a1, a1, a2
-; LP64-LP64F-LP64D-FPELIM-NEXT:    ld a0, 0(a0)
-; LP64-LP64F-LP64D-FPELIM-NEXT:    add a0, a1, a0
+; LP64-LP64F-LP64D-FPELIM-NEXT:    addw a0, a1, a0
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    ld s0, 16(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    ld ra, 24(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi sp, sp, 96
@@ -1361,10 +1374,10 @@ define i32 @va4_va_copy(i32 %argno, ...) nounwind {
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    andi a0, a0, -4
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a3, a0, 8
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a3, -32(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    ld a0, 0(a0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    add a1, a1, s1
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    add a1, a1, a2
-; LP64-LP64F-LP64D-WITHFP-NEXT:    ld a0, 0(a0)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    add a0, a1, a0
+; LP64-LP64F-LP64D-WITHFP-NEXT:    addw a0, a1, a0
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    ld s1, 24(sp)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    ld s0, 32(sp)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    ld ra, 40(sp)
@@ -1425,8 +1438,7 @@ define void @va5_aligned_stack_caller() nounwind {
 ; ILP32-ILP32F-FPELIM-NEXT:    addi a0, a0, -328
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a0, 36(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    lui a0, 335544
-; ILP32-ILP32F-FPELIM-NEXT:    addi a0, a0, 1311
-; ILP32-ILP32F-FPELIM-NEXT:    sw a0, 32(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    addi a5, a0, 1311
 ; ILP32-ILP32F-FPELIM-NEXT:    lui a0, 688509
 ; ILP32-ILP32F-FPELIM-NEXT:    addi a6, a0, -2048
 ; ILP32-ILP32F-FPELIM-NEXT:    addi a2, sp, 32
@@ -1435,6 +1447,7 @@ define void @va5_aligned_stack_caller() nounwind {
 ; ILP32-ILP32F-FPELIM-NEXT:    addi a3, zero, 12
 ; ILP32-ILP32F-FPELIM-NEXT:    addi a4, zero, 13
 ; ILP32-ILP32F-FPELIM-NEXT:    addi a7, zero, 4
+; ILP32-ILP32F-FPELIM-NEXT:    sw a5, 32(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    call va5_aligned_stack_callee
 ; ILP32-ILP32F-FPELIM-NEXT:    lw ra, 60(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    addi sp, sp, 64
@@ -1470,8 +1483,7 @@ define void @va5_aligned_stack_caller() nounwind {
 ; ILP32-ILP32F-WITHFP-NEXT:    addi a0, a0, -328
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a0, -28(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    lui a0, 335544
-; ILP32-ILP32F-WITHFP-NEXT:    addi a0, a0, 1311
-; ILP32-ILP32F-WITHFP-NEXT:    sw a0, -32(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    addi a5, a0, 1311
 ; ILP32-ILP32F-WITHFP-NEXT:    lui a0, 688509
 ; ILP32-ILP32F-WITHFP-NEXT:    addi a6, a0, -2048
 ; ILP32-ILP32F-WITHFP-NEXT:    addi a2, s0, -32
@@ -1480,6 +1492,7 @@ define void @va5_aligned_stack_caller() nounwind {
 ; ILP32-ILP32F-WITHFP-NEXT:    addi a3, zero, 12
 ; ILP32-ILP32F-WITHFP-NEXT:    addi a4, zero, 13
 ; ILP32-ILP32F-WITHFP-NEXT:    addi a7, zero, 4
+; ILP32-ILP32F-WITHFP-NEXT:    sw a5, -32(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    call va5_aligned_stack_callee
 ; ILP32-ILP32F-WITHFP-NEXT:    lw s0, 56(sp)
 ; ILP32-ILP32F-WITHFP-NEXT:    lw ra, 60(sp)
@@ -1514,8 +1527,7 @@ define void @va5_aligned_stack_caller() nounwind {
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a0, a0, -328
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a0, 36(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lui a0, 335544
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a0, a0, 1311
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a0, 32(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a5, a0, 1311
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lui a0, 688509
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a6, a0, -2048
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a2, sp, 32
@@ -1524,6 +1536,7 @@ define void @va5_aligned_stack_caller() nounwind {
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a3, zero, 12
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a4, zero, 13
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a7, zero, 4
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a5, 32(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    call va5_aligned_stack_callee
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lw ra, 60(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi sp, sp, 64
@@ -1546,33 +1559,33 @@ define void @va5_aligned_stack_caller() nounwind {
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    slli a0, a0, 14
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a0, a0, 655
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    slli a0, a0, 12
-; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a0, a0, 1475
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a0, 0(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    addi t0, a0, 1475
+; LP64-LP64F-LP64D-FPELIM-NEXT:    lui a0, 1192
+; LP64-LP64F-LP64D-FPELIM-NEXT:    addiw a0, a0, 381
+; LP64-LP64F-LP64D-FPELIM-NEXT:    slli a0, a0, 12
+; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a6, a0, -2048
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    lui a0, 1048248
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addiw a0, a0, 1311
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    slli a0, a0, 12
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a0, a0, -1147
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    slli a0, a0, 13
-; LP64-LP64F-LP64D-FPELIM-NEXT:    lui a1, 512
-; LP64-LP64F-LP64D-FPELIM-NEXT:    addiw a1, a1, 73
-; LP64-LP64F-LP64D-FPELIM-NEXT:    slli a1, a1, 15
-; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a1, a1, -1311
-; LP64-LP64F-LP64D-FPELIM-NEXT:    slli a1, a1, 12
-; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a1, a1, 1147
-; LP64-LP64F-LP64D-FPELIM-NEXT:    slli a1, a1, 14
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a0, a0, 983
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    slli a0, a0, 15
-; LP64-LP64F-LP64D-FPELIM-NEXT:    lui a2, 1192
-; LP64-LP64F-LP64D-FPELIM-NEXT:    addiw a2, a2, 381
-; LP64-LP64F-LP64D-FPELIM-NEXT:    slli a2, a2, 12
-; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a6, a2, -2048
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a2, a0, 1311
-; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a3, a1, -1967
+; LP64-LP64F-LP64D-FPELIM-NEXT:    lui a0, 512
+; LP64-LP64F-LP64D-FPELIM-NEXT:    addiw a0, a0, 73
+; LP64-LP64F-LP64D-FPELIM-NEXT:    slli a0, a0, 15
+; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a0, a0, -1311
+; LP64-LP64F-LP64D-FPELIM-NEXT:    slli a0, a0, 12
+; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a0, a0, 1147
+; LP64-LP64F-LP64D-FPELIM-NEXT:    slli a0, a0, 14
+; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a3, a0, -1967
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a0, zero, 1
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a1, zero, 11
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a4, zero, 12
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a5, zero, 13
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a7, zero, 14
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd t0, 0(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    call va5_aligned_stack_callee
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    ld ra, 40(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi sp, sp, 48
@@ -1597,33 +1610,33 @@ define void @va5_aligned_stack_caller() nounwind {
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    slli a0, a0, 14
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a0, a0, 655
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    slli a0, a0, 12
-; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a0, a0, 1475
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a0, 0(sp)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    addi t0, a0, 1475
+; LP64-LP64F-LP64D-WITHFP-NEXT:    lui a0, 1192
+; LP64-LP64F-LP64D-WITHFP-NEXT:    addiw a0, a0, 381
+; LP64-LP64F-LP64D-WITHFP-NEXT:    slli a0, a0, 12
+; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a6, a0, -2048
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    lui a0, 1048248
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addiw a0, a0, 1311
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    slli a0, a0, 12
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a0, a0, -1147
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    slli a0, a0, 13
-; LP64-LP64F-LP64D-WITHFP-NEXT:    lui a1, 512
-; LP64-LP64F-LP64D-WITHFP-NEXT:    addiw a1, a1, 73
-; LP64-LP64F-LP64D-WITHFP-NEXT:    slli a1, a1, 15
-; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a1, a1, -1311
-; LP64-LP64F-LP64D-WITHFP-NEXT:    slli a1, a1, 12
-; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a1, a1, 1147
-; LP64-LP64F-LP64D-WITHFP-NEXT:    slli a1, a1, 14
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a0, a0, 983
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    slli a0, a0, 15
-; LP64-LP64F-LP64D-WITHFP-NEXT:    lui a2, 1192
-; LP64-LP64F-LP64D-WITHFP-NEXT:    addiw a2, a2, 381
-; LP64-LP64F-LP64D-WITHFP-NEXT:    slli a2, a2, 12
-; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a6, a2, -2048
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a2, a0, 1311
-; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a3, a1, -1967
+; LP64-LP64F-LP64D-WITHFP-NEXT:    lui a0, 512
+; LP64-LP64F-LP64D-WITHFP-NEXT:    addiw a0, a0, 73
+; LP64-LP64F-LP64D-WITHFP-NEXT:    slli a0, a0, 15
+; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a0, a0, -1311
+; LP64-LP64F-LP64D-WITHFP-NEXT:    slli a0, a0, 12
+; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a0, a0, 1147
+; LP64-LP64F-LP64D-WITHFP-NEXT:    slli a0, a0, 14
+; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a3, a0, -1967
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a0, zero, 1
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a1, zero, 11
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a4, zero, 12
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a5, zero, 13
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a7, zero, 14
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd t0, 0(sp)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    call va5_aligned_stack_callee
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    ld s0, 32(sp)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    ld ra, 40(sp)
@@ -1650,9 +1663,9 @@ define i32 @va6_no_fixed_args(...) nounwind {
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a3, 28(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a2, 24(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a1, 20(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    sw a0, 16(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    addi a1, sp, 20
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a1, 12(sp)
-; ILP32-ILP32F-FPELIM-NEXT:    sw a0, 16(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    addi sp, sp, 48
 ; ILP32-ILP32F-FPELIM-NEXT:    ret
 ;
@@ -1669,9 +1682,9 @@ define i32 @va6_no_fixed_args(...) nounwind {
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a3, 12(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a2, 8(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a1, 4(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    sw a0, 0(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    addi a1, s0, 4
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a1, -12(s0)
-; ILP32-ILP32F-WITHFP-NEXT:    sw a0, 0(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    lw s0, 8(sp)
 ; ILP32-ILP32F-WITHFP-NEXT:    lw ra, 12(sp)
 ; ILP32-ILP32F-WITHFP-NEXT:    addi sp, sp, 48
@@ -1687,9 +1700,9 @@ define i32 @va6_no_fixed_args(...) nounwind {
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a3, 28(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a2, 24(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a1, 20(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a0, 16(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a1, sp, 20
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a1, 12(sp)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a0, 16(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi sp, sp, 48
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    ret
 ;
@@ -1703,10 +1716,10 @@ define i32 @va6_no_fixed_args(...) nounwind {
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a3, 40(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a2, 32(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a1, 24(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a0, 16(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a1, sp, 16
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    ori a1, a1, 8
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a1, 8(sp)
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a0, 16(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi sp, sp, 80
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    ret
 ;
@@ -1723,10 +1736,10 @@ define i32 @va6_no_fixed_args(...) nounwind {
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a3, 24(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a2, 16(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a1, 8(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a0, 0(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    mv a1, s0
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    ori a1, a1, 8
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a1, -24(s0)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a0, 0(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    ld s0, 16(sp)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    ld ra, 24(sp)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi sp, sp, 96
@@ -1738,3 +1751,226 @@ define i32 @va6_no_fixed_args(...) nounwind {
   call void @llvm.va_end(i8* %1)
   ret i32 %2
 }
+
+; TODO: improve constant materialization of stack addresses
+
+define i32 @va_large_stack(i8* %fmt, ...) {
+; ILP32-ILP32F-FPELIM-LABEL: va_large_stack:
+; ILP32-ILP32F-FPELIM:       # %bb.0:
+; ILP32-ILP32F-FPELIM-NEXT:    lui a0, 24414
+; ILP32-ILP32F-FPELIM-NEXT:    addi a0, a0, 304
+; ILP32-ILP32F-FPELIM-NEXT:    sub sp, sp, a0
+; ILP32-ILP32F-FPELIM-NEXT:    .cfi_def_cfa_offset 100000048
+; ILP32-ILP32F-FPELIM-NEXT:    mv a0, a1
+; ILP32-ILP32F-FPELIM-NEXT:    lui t0, 24414
+; ILP32-ILP32F-FPELIM-NEXT:    addi t0, t0, 300
+; ILP32-ILP32F-FPELIM-NEXT:    add t0, sp, t0
+; ILP32-ILP32F-FPELIM-NEXT:    sw a7, 0(t0)
+; ILP32-ILP32F-FPELIM-NEXT:    lui a7, 24414
+; ILP32-ILP32F-FPELIM-NEXT:    addi a7, a7, 296
+; ILP32-ILP32F-FPELIM-NEXT:    add a7, sp, a7
+; ILP32-ILP32F-FPELIM-NEXT:    sw a6, 0(a7)
+; ILP32-ILP32F-FPELIM-NEXT:    lui a6, 24414
+; ILP32-ILP32F-FPELIM-NEXT:    addi a6, a6, 292
+; ILP32-ILP32F-FPELIM-NEXT:    add a6, sp, a6
+; ILP32-ILP32F-FPELIM-NEXT:    sw a5, 0(a6)
+; ILP32-ILP32F-FPELIM-NEXT:    lui a5, 24414
+; ILP32-ILP32F-FPELIM-NEXT:    addi a5, a5, 288
+; ILP32-ILP32F-FPELIM-NEXT:    add a5, sp, a5
+; ILP32-ILP32F-FPELIM-NEXT:    sw a4, 0(a5)
+; ILP32-ILP32F-FPELIM-NEXT:    lui a4, 24414
+; ILP32-ILP32F-FPELIM-NEXT:    addi a4, a4, 284
+; ILP32-ILP32F-FPELIM-NEXT:    add a4, sp, a4
+; ILP32-ILP32F-FPELIM-NEXT:    sw a3, 0(a4)
+; ILP32-ILP32F-FPELIM-NEXT:    lui a3, 24414
+; ILP32-ILP32F-FPELIM-NEXT:    addi a3, a3, 280
+; ILP32-ILP32F-FPELIM-NEXT:    add a3, sp, a3
+; ILP32-ILP32F-FPELIM-NEXT:    sw a2, 0(a3)
+; ILP32-ILP32F-FPELIM-NEXT:    lui a2, 24414
+; ILP32-ILP32F-FPELIM-NEXT:    addi a2, a2, 276
+; ILP32-ILP32F-FPELIM-NEXT:    add a2, sp, a2
+; ILP32-ILP32F-FPELIM-NEXT:    sw a1, 0(a2)
+; ILP32-ILP32F-FPELIM-NEXT:    lui a1, 24414
+; ILP32-ILP32F-FPELIM-NEXT:    addi a1, a1, 280
+; ILP32-ILP32F-FPELIM-NEXT:    add a1, sp, a1
+; ILP32-ILP32F-FPELIM-NEXT:    mv a1, a1
+; ILP32-ILP32F-FPELIM-NEXT:    sw a1, 12(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    lui a1, 24414
+; ILP32-ILP32F-FPELIM-NEXT:    addi a1, a1, 304
+; ILP32-ILP32F-FPELIM-NEXT:    add sp, sp, a1
+; ILP32-ILP32F-FPELIM-NEXT:    ret
+;
+; ILP32-ILP32F-WITHFP-LABEL: va_large_stack:
+; ILP32-ILP32F-WITHFP:       # %bb.0:
+; ILP32-ILP32F-WITHFP-NEXT:    addi sp, sp, -2032
+; ILP32-ILP32F-WITHFP-NEXT:    .cfi_def_cfa_offset 2032
+; ILP32-ILP32F-WITHFP-NEXT:    sw ra, 1996(sp)
+; ILP32-ILP32F-WITHFP-NEXT:    sw s0, 1992(sp)
+; ILP32-ILP32F-WITHFP-NEXT:    .cfi_offset ra, -36
+; ILP32-ILP32F-WITHFP-NEXT:    .cfi_offset s0, -40
+; ILP32-ILP32F-WITHFP-NEXT:    addi s0, sp, 2000
+; ILP32-ILP32F-WITHFP-NEXT:    .cfi_def_cfa s0, 0
+; ILP32-ILP32F-WITHFP-NEXT:    lui a0, 24414
+; ILP32-ILP32F-WITHFP-NEXT:    addi a0, a0, -1728
+; ILP32-ILP32F-WITHFP-NEXT:    sub sp, sp, a0
+; ILP32-ILP32F-WITHFP-NEXT:    mv a0, a1
+; ILP32-ILP32F-WITHFP-NEXT:    sw a7, 28(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    sw a6, 24(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    sw a5, 20(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    sw a4, 16(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    sw a3, 12(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    sw a2, 8(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    sw a1, 4(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    addi a1, s0, 8
+; ILP32-ILP32F-WITHFP-NEXT:    lui a2, 1024162
+; ILP32-ILP32F-WITHFP-NEXT:    addi a2, a2, -272
+; ILP32-ILP32F-WITHFP-NEXT:    add a2, s0, a2
+; ILP32-ILP32F-WITHFP-NEXT:    sw a1, 0(a2)
+; ILP32-ILP32F-WITHFP-NEXT:    lui a1, 24414
+; ILP32-ILP32F-WITHFP-NEXT:    addi a1, a1, -1728
+; ILP32-ILP32F-WITHFP-NEXT:    add sp, sp, a1
+; ILP32-ILP32F-WITHFP-NEXT:    lw s0, 1992(sp)
+; ILP32-ILP32F-WITHFP-NEXT:    lw ra, 1996(sp)
+; ILP32-ILP32F-WITHFP-NEXT:    addi sp, sp, 2032
+; ILP32-ILP32F-WITHFP-NEXT:    ret
+;
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-LABEL: va_large_stack:
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM:       # %bb.0:
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lui a0, 24414
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a0, a0, 304
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sub sp, sp, a0
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    .cfi_def_cfa_offset 100000048
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    mv a0, a1
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lui t0, 24414
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi t0, t0, 300
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    add t0, sp, t0
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a7, 0(t0)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lui a7, 24414
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a7, a7, 296
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    add a7, sp, a7
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a6, 0(a7)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lui a6, 24414
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a6, a6, 292
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    add a6, sp, a6
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a5, 0(a6)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lui a5, 24414
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a5, a5, 288
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    add a5, sp, a5
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a4, 0(a5)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lui a4, 24414
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a4, a4, 284
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    add a4, sp, a4
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a3, 0(a4)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lui a3, 24414
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a3, a3, 280
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    add a3, sp, a3
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a2, 0(a3)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lui a2, 24414
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a2, a2, 276
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    add a2, sp, a2
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a1, 0(a2)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lui a1, 24414
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a1, a1, 280
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    add a1, sp, a1
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    mv a1, a1
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a1, 12(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lui a1, 24414
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a1, a1, 304
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    add sp, sp, a1
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    ret
+;
+; LP64-LP64F-LP64D-FPELIM-LABEL: va_large_stack:
+; LP64-LP64F-LP64D-FPELIM:       # %bb.0:
+; LP64-LP64F-LP64D-FPELIM-NEXT:    lui a0, 24414
+; LP64-LP64F-LP64D-FPELIM-NEXT:    addiw a0, a0, 336
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sub sp, sp, a0
+; LP64-LP64F-LP64D-FPELIM-NEXT:    .cfi_def_cfa_offset 100000080
+; LP64-LP64F-LP64D-FPELIM-NEXT:    lui a0, 24414
+; LP64-LP64F-LP64D-FPELIM-NEXT:    addiw a0, a0, 280
+; LP64-LP64F-LP64D-FPELIM-NEXT:    add a0, sp, a0
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a1, 0(a0)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    lui a0, 24414
+; LP64-LP64F-LP64D-FPELIM-NEXT:    addiw a0, a0, 328
+; LP64-LP64F-LP64D-FPELIM-NEXT:    add a0, sp, a0
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a7, 0(a0)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    lui a0, 24414
+; LP64-LP64F-LP64D-FPELIM-NEXT:    addiw a0, a0, 320
+; LP64-LP64F-LP64D-FPELIM-NEXT:    add a0, sp, a0
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a6, 0(a0)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    lui a0, 24414
+; LP64-LP64F-LP64D-FPELIM-NEXT:    addiw a0, a0, 312
+; LP64-LP64F-LP64D-FPELIM-NEXT:    add a0, sp, a0
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a5, 0(a0)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    lui a0, 24414
+; LP64-LP64F-LP64D-FPELIM-NEXT:    addiw a0, a0, 304
+; LP64-LP64F-LP64D-FPELIM-NEXT:    add a0, sp, a0
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a4, 0(a0)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    lui a0, 24414
+; LP64-LP64F-LP64D-FPELIM-NEXT:    addiw a0, a0, 296
+; LP64-LP64F-LP64D-FPELIM-NEXT:    add a0, sp, a0
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a3, 0(a0)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    lui a0, 24414
+; LP64-LP64F-LP64D-FPELIM-NEXT:    addiw a0, a0, 288
+; LP64-LP64F-LP64D-FPELIM-NEXT:    add a0, sp, a0
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a2, 0(a0)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    lui a0, 24414
+; LP64-LP64F-LP64D-FPELIM-NEXT:    addiw a0, a0, 280
+; LP64-LP64F-LP64D-FPELIM-NEXT:    add a0, sp, a0
+; LP64-LP64F-LP64D-FPELIM-NEXT:    mv a0, a0
+; LP64-LP64F-LP64D-FPELIM-NEXT:    ori a0, a0, 4
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a0, 8(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    lui a0, 24414
+; LP64-LP64F-LP64D-FPELIM-NEXT:    addiw a0, a0, 280
+; LP64-LP64F-LP64D-FPELIM-NEXT:    add a0, sp, a0
+; LP64-LP64F-LP64D-FPELIM-NEXT:    lw a0, 0(a0)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    lui a1, 24414
+; LP64-LP64F-LP64D-FPELIM-NEXT:    addiw a1, a1, 336
+; LP64-LP64F-LP64D-FPELIM-NEXT:    add sp, sp, a1
+; LP64-LP64F-LP64D-FPELIM-NEXT:    ret
+;
+; LP64-LP64F-LP64D-WITHFP-LABEL: va_large_stack:
+; LP64-LP64F-LP64D-WITHFP:       # %bb.0:
+; LP64-LP64F-LP64D-WITHFP-NEXT:    addi sp, sp, -2032
+; LP64-LP64F-LP64D-WITHFP-NEXT:    .cfi_def_cfa_offset 2032
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd ra, 1960(sp)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd s0, 1952(sp)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    .cfi_offset ra, -72
+; LP64-LP64F-LP64D-WITHFP-NEXT:    .cfi_offset s0, -80
+; LP64-LP64F-LP64D-WITHFP-NEXT:    addi s0, sp, 1968
+; LP64-LP64F-LP64D-WITHFP-NEXT:    .cfi_def_cfa s0, 0
+; LP64-LP64F-LP64D-WITHFP-NEXT:    lui a0, 24414
+; LP64-LP64F-LP64D-WITHFP-NEXT:    addiw a0, a0, -1680
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sub sp, sp, a0
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a1, 8(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a7, 56(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a6, 48(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a5, 40(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a4, 32(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a3, 24(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a2, 16(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a0, s0, 8
+; LP64-LP64F-LP64D-WITHFP-NEXT:    ori a0, a0, 4
+; LP64-LP64F-LP64D-WITHFP-NEXT:    lui a1, 1024162
+; LP64-LP64F-LP64D-WITHFP-NEXT:    addiw a1, a1, -288
+; LP64-LP64F-LP64D-WITHFP-NEXT:    add a1, s0, a1
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a0, 0(a1)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    lw a0, 8(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    lui a1, 24414
+; LP64-LP64F-LP64D-WITHFP-NEXT:    addiw a1, a1, -1680
+; LP64-LP64F-LP64D-WITHFP-NEXT:    add sp, sp, a1
+; LP64-LP64F-LP64D-WITHFP-NEXT:    ld s0, 1952(sp)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    ld ra, 1960(sp)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    addi sp, sp, 2032
+; LP64-LP64F-LP64D-WITHFP-NEXT:    ret
+  %large = alloca [ 100000000 x i8 ]
+  %va = alloca i8*, align 4
+  %1 = bitcast i8** %va to i8*
+  call void @llvm.va_start(i8* %1)
+  %argp.cur = load i8*, i8** %va, align 4
+  %argp.next = getelementptr inbounds i8, i8* %argp.cur, i32 4
+  store i8* %argp.next, i8** %va, align 4
+  %2 = bitcast i8* %argp.cur to i32*
+  %3 = load i32, i32* %2, align 4
+  call void @llvm.va_end(i8* %1)
+  ret i32 %3
+}
diff --git a/llvm/test/CodeGen/RISCV/zext-with-load-is-free.ll b/llvm/test/CodeGen/RISCV/zext-with-load-is-free.ll
index 12c779699..7c4bbcdaa 100644
--- a/llvm/test/CodeGen/RISCV/zext-with-load-is-free.ll
+++ b/llvm/test/CodeGen/RISCV/zext-with-load-is-free.ll
@@ -45,10 +45,10 @@ define i32 @test_zext_i16() nounwind {
 ; RV32I-LABEL: test_zext_i16:
 ; RV32I:       # %bb.0: # %entry
 ; RV32I-NEXT:    lui a0, %hi(shorts)
-; RV32I-NEXT:    lui a1, 16
-; RV32I-NEXT:    addi a1, a1, -120
-; RV32I-NEXT:    lhu a2, %lo(shorts)(a0)
-; RV32I-NEXT:    bne a2, a1, .LBB1_3
+; RV32I-NEXT:    lhu a1, %lo(shorts)(a0)
+; RV32I-NEXT:    lui a2, 16
+; RV32I-NEXT:    addi a2, a2, -120
+; RV32I-NEXT:    bne a1, a2, .LBB1_3
 ; RV32I-NEXT:  # %bb.1: # %entry
 ; RV32I-NEXT:    addi a0, a0, %lo(shorts)
 ; RV32I-NEXT:    lhu a0, 2(a0)
diff --git a/llvm/test/CodeGen/SPARC/64atomics.ll b/llvm/test/CodeGen/SPARC/64atomics.ll
new file mode 100644
index 000000000..89175b624
--- /dev/null
+++ b/llvm/test/CodeGen/SPARC/64atomics.ll
@@ -0,0 +1,60 @@
+; RUN: llc < %s -march=sparcv9 -verify-machineinstrs | FileCheck %s
+
+; CHECK-LABEL: test_atomic_i64
+; CHECK:       ldx [%o0]
+; CHECK:       membar
+; CHECK:       ldx [%o1]
+; CHECK:       membar
+; CHECK:       membar
+; CHECK:       stx {{.+}}, [%o2]
+define i64 @test_atomic_i64(i64* %ptr1, i64* %ptr2, i64* %ptr3) {
+entry:
+  %0 = load atomic i64, i64* %ptr1 acquire, align 8
+  %1 = load atomic i64, i64* %ptr2 acquire, align 8
+  %2 = add i64 %0, %1
+  store atomic i64 %2, i64* %ptr3 release, align 8
+  ret i64 %2
+}
+
+; CHECK-LABEL: test_cmpxchg_i64
+; CHECK:       mov 123, [[R:%[gilo][0-7]]]
+; CHECK:       casx [%o1], %o0, [[R]]
+
+define i64 @test_cmpxchg_i64(i64 %a, i64* %ptr) {
+entry:
+  %pair = cmpxchg i64* %ptr, i64 %a, i64 123 monotonic monotonic
+  %b = extractvalue { i64, i1 } %pair, 0
+  ret i64 %b
+}
+
+; CHECK-LABEL: test_swap_i64
+; CHECK:       casx [%o1],
+
+define i64 @test_swap_i64(i64 %a, i64* %ptr) {
+entry:
+  %b = atomicrmw xchg i64* %ptr, i64 42 monotonic
+  ret i64 %b
+}
+
+; CHECK-LABEL: test_load_sub_64
+; CHECK: membar
+; CHECK: sub
+; CHECK: casx [%o0]
+; CHECK: membar
+define zeroext i64 @test_load_sub_64(i64* %p, i64 zeroext %v) {
+entry:
+  %0 = atomicrmw sub i64* %p, i64 %v seq_cst
+  ret i64 %0
+}
+
+; CHECK-LABEL: test_load_max_64
+; CHECK: membar
+; CHECK: cmp
+; CHECK: movg %xcc
+; CHECK: casx [%o0]
+; CHECK: membar
+define zeroext i64 @test_load_max_64(i64* %p, i64 zeroext %v) {
+entry:
+  %0 = atomicrmw max i64* %p, i64 %v seq_cst
+  ret i64 %0
+}
diff --git a/llvm/test/CodeGen/SPARC/atomics.ll b/llvm/test/CodeGen/SPARC/atomics.ll
index 17aa227ec..c6ecbaf91 100644
--- a/llvm/test/CodeGen/SPARC/atomics.ll
+++ b/llvm/test/CodeGen/SPARC/atomics.ll
@@ -1,3 +1,4 @@
+; RUN: llc < %s -march=sparc -mcpu=v9 -verify-machineinstrs | FileCheck %s
 ; RUN: llc < %s -march=sparcv9 -verify-machineinstrs | FileCheck %s
 
 ; CHECK-LABEL: test_atomic_i8
@@ -48,22 +49,6 @@ entry:
   ret i32 %2
 }
 
-; CHECK-LABEL: test_atomic_i64
-; CHECK:       ldx [%o0]
-; CHECK:       membar
-; CHECK:       ldx [%o1]
-; CHECK:       membar
-; CHECK:       membar
-; CHECK:       stx {{.+}}, [%o2]
-define i64 @test_atomic_i64(i64* %ptr1, i64* %ptr2, i64* %ptr3) {
-entry:
-  %0 = load atomic i64, i64* %ptr1 acquire, align 8
-  %1 = load atomic i64, i64* %ptr2 acquire, align 8
-  %2 = add i64 %0, %1
-  store atomic i64 %2, i64* %ptr3 release, align 8
-  ret i64 %2
-}
-
 ;; TODO: the "move %icc" and related instructions are totally
 ;; redundant here. There's something weird happening in optimization
 ;; of the success value of cmpxchg.
@@ -159,17 +144,6 @@ entry:
   ret i32 %b
 }
 
-; CHECK-LABEL: test_cmpxchg_i64
-; CHECK:       mov 123, [[R:%[gilo][0-7]]]
-; CHECK:       casx [%o1], %o0, [[R]]
-
-define i64 @test_cmpxchg_i64(i64 %a, i64* %ptr) {
-entry:
-  %pair = cmpxchg i64* %ptr, i64 %a, i64 123 monotonic monotonic
-  %b = extractvalue { i64, i1 } %pair, 0
-  ret i64 %b
-}
-
 ; CHECK-LABEL: test_swap_i8
 ; CHECK:       mov 42, [[R:%[gilo][0-7]]]
 ; CHECK:       cas
@@ -200,15 +174,6 @@ entry:
   ret i32 %b
 }
 
-; CHECK-LABEL: test_swap_i64
-; CHECK:       casx [%o1],
-
-define i64 @test_swap_i64(i64 %a, i64* %ptr) {
-entry:
-  %b = atomicrmw xchg i64* %ptr, i64 42 monotonic
-  ret i64 %b
-}
-
 ; CHECK-LABEL: test_load_sub_i8
 ; CHECK: membar
 ; CHECK: .L{{.*}}:
@@ -246,17 +211,6 @@ entry:
   ret i32 %0
 }
 
-; CHECK-LABEL: test_load_sub_64
-; CHECK: membar
-; CHECK: sub
-; CHECK: casx [%o0]
-; CHECK: membar
-define zeroext i64 @test_load_sub_64(i64* %p, i64 zeroext %v) {
-entry:
-  %0 = atomicrmw sub i64* %p, i64 %v seq_cst
-  ret i64 %0
-}
-
 ; CHECK-LABEL: test_load_xor_32
 ; CHECK: membar
 ; CHECK: xor
@@ -292,18 +246,6 @@ entry:
   ret i32 %0
 }
 
-; CHECK-LABEL: test_load_max_64
-; CHECK: membar
-; CHECK: cmp
-; CHECK: movg %xcc
-; CHECK: casx [%o0]
-; CHECK: membar
-define zeroext i64 @test_load_max_64(i64* %p, i64 zeroext %v) {
-entry:
-  %0 = atomicrmw max i64* %p, i64 %v seq_cst
-  ret i64 %0
-}
-
 ; CHECK-LABEL: test_load_umin_32
 ; CHECK: membar
 ; CHECK: cmp
diff --git a/llvm/test/CodeGen/SPARC/exception.ll b/llvm/test/CodeGen/SPARC/exception.ll
index f11232834..42a9c1fdd 100644
--- a/llvm/test/CodeGen/SPARC/exception.ll
+++ b/llvm/test/CodeGen/SPARC/exception.ll
@@ -151,7 +151,7 @@ declare i32 @puts(i8* nocapture readonly) #1
 
 declare i32 @__gxx_personality_v0(i32, i64, i8*, i8*)
 
-attributes #0 = { "no-frame-pointer-elim-non-leaf"="false" }
+attributes #0 = { "frame-pointer"="none" }
 attributes #1 = { nounwind }
 attributes #2 = { noreturn }
 attributes #3 = { nounwind readnone }
diff --git a/llvm/test/CodeGen/SPARC/fp128.ll b/llvm/test/CodeGen/SPARC/fp128.ll
index 83912e0f2..1f5d2db66 100644
--- a/llvm/test/CodeGen/SPARC/fp128.ll
+++ b/llvm/test/CodeGen/SPARC/fp128.ll
@@ -1,8 +1,17 @@
 ; RUN: llc < %s -march=sparc -mattr=hard-quad-float | FileCheck %s --check-prefix=CHECK --check-prefix=HARD --check-prefix=BE
 ; RUN: llc < %s -march=sparcel -mattr=hard-quad-float | FileCheck %s --check-prefix=CHECK --check-prefix=HARD --check-prefix=EL
-; RUN: llc < %s -march=sparc -mattr=-hard-quad-float | FileCheck %s --check-prefix=CHECK --check-prefix=SOFT --check-prefix=BE
+; RUN: llc < %s -march=sparc -mattr=-hard-quad-float -verify-machineinstrs | FileCheck %s --check-prefix=CHECK --check-prefix=SOFT --check-prefix=BE
 ; RUN: llc < %s -march=sparcel -mattr=-hard-quad-float | FileCheck %s --check-prefix=CHECK --check-prefix=SOFT --check-prefix=EL
 
+; XFAIL: *
+; This test currently fails with expensive checks enabled, for more details see
+; https://bugs.llvm.org/show_bug.cgi?id=44091.
+; *** Bad machine code: Expected a register operand. ***
+; - function:    f128_compare
+; - basic block: %bb.0 entry (0x63f4028)
+; - instruction: CMPrr killed %21:intregs, 0, implicit-def $icc
+; - operand 1:   0
+; NB: When this is fixed the verifier should not be run by default in the CL above.
 
 ; CHECK-LABEL: f128_ops:
 ; CHECK:      ldd
diff --git a/llvm/test/CodeGen/SPARC/inlineasm.ll b/llvm/test/CodeGen/SPARC/inlineasm.ll
index 814845459..ca68a5a23 100644
--- a/llvm/test/CodeGen/SPARC/inlineasm.ll
+++ b/llvm/test/CodeGen/SPARC/inlineasm.ll
@@ -80,7 +80,7 @@ entry:
   tail call void asm sideeffect "std $0, [$1]", "r,r,~{memory}"(i64 %conv, i64* %out)
   ret i32 %o
 }
-attributes #0 = { "no-frame-pointer-elim"="true" }
+attributes #0 = { "frame-pointer"="all" }
 
 ;; Ensures that tied in and out gets allocated properly.
 ; CHECK-LABEL: test_i64_inout:
diff --git a/llvm/test/CodeGen/SystemZ/Large/branch-01.ll b/llvm/test/CodeGen/SystemZ/Large/branch-01.ll
index ed505471c..e0a45b38c 100644
--- a/llvm/test/CodeGen/SystemZ/Large/branch-01.ll
+++ b/llvm/test/CodeGen/SystemZ/Large/branch-01.ll
@@ -11882,9 +11882,9 @@ declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture writeonly, i8* nocapture r
 ; Function Attrs: nounwind
 declare dso_local fastcc void @func_62(%1* noalias nocapture, i64) unnamed_addr #0
 
-attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="z13" "target-features"="+transactional-execution,+vector" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="z13" "target-features"="+transactional-execution,+vector" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { argmemonly nounwind }
-attributes #2 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="z13" "target-features"="+transactional-execution,+vector" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="z13" "target-features"="+transactional-execution,+vector" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #3 = { nounwind }
 
 !llvm.ident = !{!0}
diff --git a/llvm/test/CodeGen/SystemZ/anyregcc-novec.ll b/llvm/test/CodeGen/SystemZ/anyregcc-novec.ll
index 72bf00c22..b37cd220f 100644
--- a/llvm/test/CodeGen/SystemZ/anyregcc-novec.ll
+++ b/llvm/test/CodeGen/SystemZ/anyregcc-novec.ll
@@ -5,13 +5,14 @@ define anyregcc void @anyregcc1() {
 entry:
 ;CHECK-LABEL: anyregcc1
 ;CHECK: stmg %r2, %r15, 16(%r15)
-;CHECK: std %f0,
+;CHECK: aghi %r15, -96
+;CHECK: std %f0, 224(%r15)
 ;CHECK: std %f1,
-;CHECK: std %f2,
+;CHECK: std %f2, 232(%r15)
 ;CHECK: std %f3,
-;CHECK: std %f4,
+;CHECK: std %f4, 240(%r15)
 ;CHECK: std %f5,
-;CHECK: std %f6,
+;CHECK: std %f6, 248(%r15)
 ;CHECK: std %f7,
 ;CHECK: std %f8,
 ;CHECK: std %f9,
@@ -21,6 +22,14 @@ entry:
 ;CHECK: std %f13,
 ;CHECK: std %f14,
 ;CHECK: std %f15,
+;CHECK: .cfi_offset %f0, -32
+;CHECK: .cfi_offset %f2, -24
+;CHECK: .cfi_offset %f4, -16
+;CHECK: .cfi_offset %f6, -8
+;CHECK: ld %f0, 224(%r15)
+;CHECK: ld %f2, 232(%r15)
+;CHECK: ld %f4, 240(%r15)
+;CHECK: ld %f6, 248(%r15)
   call void asm sideeffect "", "~{r0},~{r1},~{r2},~{r3},~{r4},~{r5},~{r6},~{r7},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{f0},~{f1},~{f2},~{f3},~{f4},~{f5},~{f6},~{f7},~{f8},~{f9},~{f10},~{f11},~{f12},~{f13},~{f14},~{f15}"() nounwind
   ret void
 }
diff --git a/llvm/test/CodeGen/SystemZ/call-05.ll b/llvm/test/CodeGen/SystemZ/call-05.ll
index 15704531d..b958c0b6d 100644
--- a/llvm/test/CodeGen/SystemZ/call-05.ll
+++ b/llvm/test/CodeGen/SystemZ/call-05.ll
@@ -451,8 +451,9 @@ b:
 define void @f25(float %val1, float %val2) {
 ; CHECK-LABEL: f25:
 ; CHECK: cebr %f0, %f2
-; CHECK: bor %r1
+; CHECK: jo
 ; CHECK: br %r14
+; CHECK: br %r1
   %fun_a = load volatile void() *, void()** @fun_a;
   %cond = fcmp uno float %val1, %val2;
   br i1 %cond, label %a, label %b;
diff --git a/llvm/test/CodeGen/SystemZ/cond-move-01.ll b/llvm/test/CodeGen/SystemZ/cond-move-01.ll
index fad5012be..c6b0cd539 100644
--- a/llvm/test/CodeGen/SystemZ/cond-move-01.ll
+++ b/llvm/test/CodeGen/SystemZ/cond-move-01.ll
@@ -7,7 +7,7 @@
 ; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 -verify-machineinstrs | FileCheck %s
 ;
 ; And again in the presence of the select instructions.
-; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=arch13 -verify-machineinstrs | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z15 -verify-machineinstrs | FileCheck %s
 
 ; Test LOCR.
 define i32 @f1(i32 %a, i32 %b, i32 %limit) {
diff --git a/llvm/test/CodeGen/SystemZ/cond-move-02.ll b/llvm/test/CodeGen/SystemZ/cond-move-02.ll
index c8ab53312..362d7cc38 100644
--- a/llvm/test/CodeGen/SystemZ/cond-move-02.ll
+++ b/llvm/test/CodeGen/SystemZ/cond-move-02.ll
@@ -4,7 +4,7 @@
 ;
 ; Run the test again to make sure it still works the same even
 ; in the presence of the select instructions.
-; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=arch13 -verify-machineinstrs | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z15 -verify-machineinstrs | FileCheck %s
 
 
 define i32 @f1(i32 %x) {
diff --git a/llvm/test/CodeGen/SystemZ/cond-move-03.ll b/llvm/test/CodeGen/SystemZ/cond-move-03.ll
index 0f4d080d6..f668f1ac4 100644
--- a/llvm/test/CodeGen/SystemZ/cond-move-03.ll
+++ b/llvm/test/CodeGen/SystemZ/cond-move-03.ll
@@ -6,7 +6,7 @@
 ;
 ; Run the test again to make sure it still works the same even
 ; in the presence of the select instructions.
-; RUN: llc < %s -verify-machineinstrs -mtriple=s390x-linux-gnu -mcpu=arch13 \
+; RUN: llc < %s -verify-machineinstrs -mtriple=s390x-linux-gnu -mcpu=z15 \
 ; RUN:   -no-integrated-as | FileCheck %s
 
 define void @f1(i32 %limit) {
diff --git a/llvm/test/CodeGen/SystemZ/cond-move-06.ll b/llvm/test/CodeGen/SystemZ/cond-move-06.ll
index 3c133d916..bcedb3ae8 100644
--- a/llvm/test/CodeGen/SystemZ/cond-move-06.ll
+++ b/llvm/test/CodeGen/SystemZ/cond-move-06.ll
@@ -1,6 +1,6 @@
 ; Test SELR and SELGR.
 ;
-; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=arch13 -verify-machineinstrs | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z15 -verify-machineinstrs | FileCheck %s
 
 ; Test SELR.
 define i32 @f1(i32 %limit, i32 %a, i32 %b) {
diff --git a/llvm/test/CodeGen/SystemZ/cond-move-07.ll b/llvm/test/CodeGen/SystemZ/cond-move-07.ll
index 87123b533..2bee8919b 100644
--- a/llvm/test/CodeGen/SystemZ/cond-move-07.ll
+++ b/llvm/test/CodeGen/SystemZ/cond-move-07.ll
@@ -1,7 +1,7 @@
 ; Test SELFHR.
 ; See comments in asm-18.ll about testing high-word operations.
 ;
-; RUN: llc < %s -verify-machineinstrs -mtriple=s390x-linux-gnu -mcpu=arch13 \
+; RUN: llc < %s -verify-machineinstrs -mtriple=s390x-linux-gnu -mcpu=z15 \
 ; RUN:   -no-integrated-as | FileCheck %s
 
 define void @f1(i32 %limit) {
diff --git a/llvm/test/CodeGen/SystemZ/ctpop-02.ll b/llvm/test/CodeGen/SystemZ/ctpop-02.ll
index 5b9d41f9a..a2bddac8c 100644
--- a/llvm/test/CodeGen/SystemZ/ctpop-02.ll
+++ b/llvm/test/CodeGen/SystemZ/ctpop-02.ll
@@ -1,6 +1,6 @@
-; Test population-count instruction on arch13
+; Test population-count instruction on z15
 ;
-; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=arch13 | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z15 | FileCheck %s
 
 declare i32 @llvm.ctpop.i32(i32 %a)
 declare i64 @llvm.ctpop.i64(i64 %a)
diff --git a/llvm/test/CodeGen/SystemZ/fentry-insertion.ll b/llvm/test/CodeGen/SystemZ/fentry-insertion.ll
new file mode 100644
index 000000000..63632d582
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/fentry-insertion.ll
@@ -0,0 +1,29 @@
+; RUN: llc %s -mtriple=s390x-linux-gnu -mcpu=z10 -o - -verify-machineinstrs \
+; RUN:   | FileCheck %s
+
+define void @test1() #0 {
+entry:
+  ret void
+
+; CHECK-LABEL: @test1
+; CHECK: brasl %r0, __fentry__
+; CHECK-NOT: mcount
+; CHECK: br %r14
+}
+
+define void @test2() #1 {
+entry:
+  br label %bb1
+bb1:
+  call void @address_taken(i64 ptrtoint (i8* blockaddress(@test2, %bb1) to i64), i32 512)
+  ret void
+
+; CHECK-LABEL: @test2
+; CHECK: brasl %r0, __fentry__
+; CHECK-NOT: mcount
+; CHECK: br %r14
+}
+
+declare void @address_taken(i64, i32) local_unnamed_addr
+attributes #0 = { "fentry-call"="true" }
+attributes #1 = { inlinehint minsize noredzone nounwind optsize sspstrong "fentry-call"="true" }
diff --git a/llvm/test/CodeGen/SystemZ/fp-abs-02.ll b/llvm/test/CodeGen/SystemZ/fp-abs-02.ll
index 2c70ca7dc..48be04980 100644
--- a/llvm/test/CodeGen/SystemZ/fp-abs-02.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-abs-02.ll
@@ -10,7 +10,7 @@ define float @f1(float %f) {
 ; CHECK: lndfr %f0, %f0
 ; CHECK: br %r14
   %abs = call float @llvm.fabs.f32(float %f)
-  %res = fsub float -0.0, %abs
+  %res = fneg float %abs
   ret float %res
 }
 
@@ -21,7 +21,7 @@ define double @f2(double %f) {
 ; CHECK: lndfr %f0, %f0
 ; CHECK: br %r14
   %abs = call double @llvm.fabs.f64(double %f)
-  %res = fsub double -0.0, %abs
+  %res = fneg double %abs
   ret double %res
 }
 
@@ -36,7 +36,7 @@ define void @f3(fp128 *%ptr, fp128 *%ptr2) {
 ; CHECK: br %r14
   %orig = load fp128, fp128 *%ptr
   %abs = call fp128 @llvm.fabs.f128(fp128 %orig)
-  %negabs = fsub fp128 0xL00000000000000008000000000000000, %abs
+  %negabs = fneg fp128 %abs
   %op2 = load fp128, fp128 *%ptr2
   %res = fdiv fp128 %negabs, %op2
   store fp128 %res, fp128 *%ptr
diff --git a/llvm/test/CodeGen/SystemZ/fp-abs-04.ll b/llvm/test/CodeGen/SystemZ/fp-abs-04.ll
index 5676429d4..6ec22292f 100644
--- a/llvm/test/CodeGen/SystemZ/fp-abs-04.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-abs-04.ll
@@ -9,7 +9,7 @@ define float @f1(float %f) {
 ; CHECK: lndfr %f0, %f0
 ; CHECK: br %r14
   %abs = call float @llvm.fabs.f32(float %f)
-  %res = fsub float -0.0, %abs
+  %res = fneg float %abs
   ret float %res
 }
 
@@ -20,7 +20,7 @@ define double @f2(double %f) {
 ; CHECK: lndfr %f0, %f0
 ; CHECK: br %r14
   %abs = call double @llvm.fabs.f64(double %f)
-  %res = fsub double -0.0, %abs
+  %res = fneg double %abs
   ret double %res
 }
 
@@ -38,7 +38,7 @@ define void @f3(fp128 *%ptr, fp128 *%ptr2) {
 ; CHECK: br %r14
   %orig = load fp128, fp128 *%ptr
   %abs = call fp128 @llvm.fabs.f128(fp128 %orig)
-  %negabs = fsub fp128 0xL00000000000000008000000000000000, %abs
+  %negabs = fneg fp128 %abs
   %op2 = load fp128, fp128 *%ptr2
   %res = fdiv fp128 %negabs, %op2
   store fp128 %res, fp128 *%ptr
diff --git a/llvm/test/CodeGen/SystemZ/fp-cmp-04.ll b/llvm/test/CodeGen/SystemZ/fp-cmp-04.ll
index 1e5e59eb6..23ff1b7da 100644
--- a/llvm/test/CodeGen/SystemZ/fp-cmp-04.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-cmp-04.ll
@@ -128,7 +128,7 @@ define float @f7(float %dummy, float %a, float *%dest) {
 ; CHECK: br %r14
 entry:
   %abs = call float @llvm.fabs.f32(float %a)
-  %res = fsub float -0.0, %abs
+  %res = fneg float %abs
   %cmp = fcmp olt float %res, 0.0
   br i1 %cmp, label %exit, label %store
 
@@ -147,7 +147,7 @@ define float @f8(float %dummy, float %a, float *%dest) {
 ; CHECK-NEXT: bler %r14
 ; CHECK: br %r14
 entry:
-  %res = fsub float -0.0, %a
+  %res = fneg float %a
   %cmp = fcmp ole float %res, 0.0
   br i1 %cmp, label %exit, label %store
 
@@ -375,7 +375,7 @@ define float @f18(float %dummy, float %a, float *%dest) {
 ; CHECK: br %r14
 entry:
   %abs = call float @llvm.fabs.f32(float %a)
-  %res = fsub float -0.0, %abs
+  %res = fneg float %abs
   %cmp = fcmp ogt float %abs, 0.0
   br i1 %cmp, label %exit, label %store
 
@@ -394,7 +394,7 @@ define float @f19(float %dummy, float %a, float *%dest) {
 ; CHECK-NEXT: bler %r14
 ; CHECK: br %r14
 entry:
-  %res = fsub float -0.0, %a
+  %res = fneg float %a
   %cmp = fcmp oge float %a, 0.0
   br i1 %cmp, label %exit, label %store
 
diff --git a/llvm/test/CodeGen/SystemZ/fp-cmp-05.ll b/llvm/test/CodeGen/SystemZ/fp-cmp-05.ll
index d25c8e78c..a8580db41 100644
--- a/llvm/test/CodeGen/SystemZ/fp-cmp-05.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-cmp-05.ll
@@ -11,7 +11,7 @@ define float @f1(float %a, float %b, float %f) {
 ; CHECK-LABEL: f1:
 ; CHECK: ltebr
 ; CHECK-NEXT: ber %r14
-  %neg = fsub float -0.0, %f
+  %neg = fneg float %f
   %cond = fcmp oeq float %neg, 0.0
   %res = select i1 %cond, float %a, float %b
   ret float %res
@@ -22,7 +22,7 @@ define double @f2(double %a, double %b, double %f) {
 ; CHECK-LABEL: f2:
 ; CHECK: ltdbr
 ; CHECK-NEXT: ber %r14
-  %neg = fsub double -0.0, %f
+  %neg = fneg double %f
   %cond = fcmp oeq double %neg, 0.0
   %res = select i1 %cond, double %a, double %b
   ret double %res
@@ -36,7 +36,7 @@ define float @f3(float %a, float %b, float %f) {
 ; CHECK: lpebr
 ; CHECK-NEXT: ber %r14
   %abs = call float @llvm.fabs.f32(float %f)
-  %neg = fsub float -0.0, %abs
+  %neg = fneg float %abs
   %cond = fcmp oeq float %neg, 0.0
   %res = select i1 %cond, float %a, float %b
   ret float %res
@@ -49,7 +49,7 @@ define double @f4(double %a, double %b, double %f) {
 ; CHECK: lpdbr
 ; CHECK-NEXT: ber %r14
   %abs = call double @llvm.fabs.f64(double %f)
-  %neg = fsub double -0.0, %abs
+  %neg = fneg double %abs
   %cond = fcmp oeq double %neg, 0.0
   %res = select i1 %cond, double %a, double %b
   ret double %res
diff --git a/llvm/test/CodeGen/SystemZ/fp-libcall.ll b/llvm/test/CodeGen/SystemZ/fp-libcall.ll
index 75250b811..2df25aaf8 100644
--- a/llvm/test/CodeGen/SystemZ/fp-libcall.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-libcall.ll
@@ -233,6 +233,68 @@ define fp128 @f33(fp128 %x, fp128 %y) {
   ret fp128 %tmp
 }
 
+; Verify that "nnan" minnum/maxnum calls are transformed to
+; compare+select sequences instead of libcalls.
+define float @f34(float %x, float %y) {
+; CHECK-LABEL: f34:
+; CHECK: cebr %f0, %f2
+; CHECK: blr %r14
+; CHECK: ler %f0, %f2
+; CHECK: br %r14
+  %tmp = call nnan float @llvm.minnum.f32(float %x, float %y)
+  ret float %tmp
+}
+
+define double @f35(double %x, double %y) {
+; CHECK-LABEL: f35:
+; CHECK: cdbr %f0, %f2
+; CHECK: blr %r14
+; CHECK: ldr %f0, %f2
+; CHECK: br %r14
+  %tmp = call nnan double @llvm.minnum.f64(double %x, double %y)
+  ret double %tmp
+}
+
+define fp128 @f36(fp128 %x, fp128 %y) {
+; CHECK-LABEL: f36:
+; CHECK: cxbr
+; CHECK: jl
+; CHECK: lxr
+; CHECK: br %r14
+  %tmp = call nnan fp128 @llvm.minnum.f128(fp128 %x, fp128 %y)
+  ret fp128 %tmp
+}
+
+define float @f37(float %x, float %y) {
+; CHECK-LABEL: f37:
+; CHECK: cebr %f0, %f2
+; CHECK: bhr %r14
+; CHECK: ler %f0, %f2
+; CHECK: br %r14
+  %tmp = call nnan float @llvm.maxnum.f32(float %x, float %y)
+  ret float %tmp
+}
+
+define double @f38(double %x, double %y) {
+; CHECK-LABEL: f38:
+; CHECK: cdbr %f0, %f2
+; CHECK: bhr %r14
+; CHECK: ldr %f0, %f2
+; CHECK: br %r14
+  %tmp = call nnan double @llvm.maxnum.f64(double %x, double %y)
+  ret double %tmp
+}
+
+define fp128 @f39(fp128 %x, fp128 %y) {
+; CHECK-LABEL: f39:
+; CHECK: cxbr
+; CHECK: jh
+; CHECK: lxr
+; CHECK: br %r14
+  %tmp = call nnan fp128 @llvm.maxnum.f128(fp128 %x, fp128 %y)
+  ret fp128 %tmp
+}
+
 declare float @llvm.powi.f32(float, i32)
 declare double @llvm.powi.f64(double, i32)
 declare fp128 @llvm.powi.f128(fp128, i32)
diff --git a/llvm/test/CodeGen/SystemZ/fp-mul-08.ll b/llvm/test/CodeGen/SystemZ/fp-mul-08.ll
index 83f198b58..2aeaefc78 100644
--- a/llvm/test/CodeGen/SystemZ/fp-mul-08.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-mul-08.ll
@@ -11,7 +11,7 @@ define float @f1(float %f1, float %f2, float %acc) {
 ; CHECK-SCALAR: ler %f0, %f4
 ; CHECK-VECTOR: wfmssb %f0, %f0, %f2, %f4
 ; CHECK: br %r14
-  %negacc = fsub float -0.0, %acc
+  %negacc = fneg float %acc
   %res = call float @llvm.fma.f32 (float %f1, float %f2, float %negacc)
   ret float %res
 }
@@ -23,7 +23,7 @@ define float @f2(float %f1, float *%ptr, float %acc) {
 ; CHECK-VECTOR: ldr %f0, %f2
 ; CHECK: br %r14
   %f2 = load float, float *%ptr
-  %negacc = fsub float -0.0, %acc
+  %negacc = fneg float %acc
   %res = call float @llvm.fma.f32 (float %f1, float %f2, float %negacc)
   ret float %res
 }
@@ -36,7 +36,7 @@ define float @f3(float %f1, float *%base, float %acc) {
 ; CHECK: br %r14
   %ptr = getelementptr float, float *%base, i64 1023
   %f2 = load float, float *%ptr
-  %negacc = fsub float -0.0, %acc
+  %negacc = fneg float %acc
   %res = call float @llvm.fma.f32 (float %f1, float %f2, float %negacc)
   ret float %res
 }
@@ -53,7 +53,7 @@ define float @f4(float %f1, float *%base, float %acc) {
 ; CHECK: br %r14
   %ptr = getelementptr float, float *%base, i64 1024
   %f2 = load float, float *%ptr
-  %negacc = fsub float -0.0, %acc
+  %negacc = fneg float %acc
   %res = call float @llvm.fma.f32 (float %f1, float %f2, float %negacc)
   ret float %res
 }
@@ -70,7 +70,7 @@ define float @f5(float %f1, float *%base, float %acc) {
 ; CHECK: br %r14
   %ptr = getelementptr float, float *%base, i64 -1
   %f2 = load float, float *%ptr
-  %negacc = fsub float -0.0, %acc
+  %negacc = fneg float %acc
   %res = call float @llvm.fma.f32 (float %f1, float %f2, float %negacc)
   ret float %res
 }
@@ -84,7 +84,7 @@ define float @f6(float %f1, float *%base, i64 %index, float %acc) {
 ; CHECK: br %r14
   %ptr = getelementptr float, float *%base, i64 %index
   %f2 = load float, float *%ptr
-  %negacc = fsub float -0.0, %acc
+  %negacc = fneg float %acc
   %res = call float @llvm.fma.f32 (float %f1, float %f2, float %negacc)
   ret float %res
 }
@@ -99,7 +99,7 @@ define float @f7(float %f1, float *%base, i64 %index, float %acc) {
   %index2 = add i64 %index, 1023
   %ptr = getelementptr float, float *%base, i64 %index2
   %f2 = load float, float *%ptr
-  %negacc = fsub float -0.0, %acc
+  %negacc = fneg float %acc
   %res = call float @llvm.fma.f32 (float %f1, float %f2, float %negacc)
   ret float %res
 }
@@ -115,7 +115,7 @@ define float @f8(float %f1, float *%base, i64 %index, float %acc) {
   %index2 = add i64 %index, 1024
   %ptr = getelementptr float, float *%base, i64 %index2
   %f2 = load float, float *%ptr
-  %negacc = fsub float -0.0, %acc
+  %negacc = fneg float %acc
   %res = call float @llvm.fma.f32 (float %f1, float %f2, float %negacc)
   ret float %res
 }
diff --git a/llvm/test/CodeGen/SystemZ/fp-mul-09.ll b/llvm/test/CodeGen/SystemZ/fp-mul-09.ll
index c66ab2f7c..2ad593517 100644
--- a/llvm/test/CodeGen/SystemZ/fp-mul-09.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-mul-09.ll
@@ -11,7 +11,7 @@ define double @f1(double %f1, double %f2, double %acc) {
 ; CHECK-SCALAR: ldr %f0, %f4
 ; CHECK-VECTOR: wfmsdb %f0, %f0, %f2, %f4
 ; CHECK: br %r14
-  %negacc = fsub double -0.0, %acc
+  %negacc = fneg double %acc
   %res = call double @llvm.fma.f64 (double %f1, double %f2, double %negacc)
   ret double %res
 }
@@ -22,7 +22,7 @@ define double @f2(double %f1, double *%ptr, double %acc) {
 ; CHECK: ldr %f0, %f2
 ; CHECK: br %r14
   %f2 = load double, double *%ptr
-  %negacc = fsub double -0.0, %acc
+  %negacc = fneg double %acc
   %res = call double @llvm.fma.f64 (double %f1, double %f2, double %negacc)
   ret double %res
 }
@@ -34,7 +34,7 @@ define double @f3(double %f1, double *%base, double %acc) {
 ; CHECK: br %r14
   %ptr = getelementptr double, double *%base, i64 511
   %f2 = load double, double *%ptr
-  %negacc = fsub double -0.0, %acc
+  %negacc = fneg double %acc
   %res = call double @llvm.fma.f64 (double %f1, double %f2, double %negacc)
   ret double %res
 }
@@ -50,7 +50,7 @@ define double @f4(double %f1, double *%base, double %acc) {
 ; CHECK: br %r14
   %ptr = getelementptr double, double *%base, i64 512
   %f2 = load double, double *%ptr
-  %negacc = fsub double -0.0, %acc
+  %negacc = fneg double %acc
   %res = call double @llvm.fma.f64 (double %f1, double %f2, double %negacc)
   ret double %res
 }
@@ -66,7 +66,7 @@ define double @f5(double %f1, double *%base, double %acc) {
 ; CHECK: br %r14
   %ptr = getelementptr double, double *%base, i64 -1
   %f2 = load double, double *%ptr
-  %negacc = fsub double -0.0, %acc
+  %negacc = fneg double %acc
   %res = call double @llvm.fma.f64 (double %f1, double %f2, double %negacc)
   ret double %res
 }
@@ -79,7 +79,7 @@ define double @f6(double %f1, double *%base, i64 %index, double %acc) {
 ; CHECK: br %r14
   %ptr = getelementptr double, double *%base, i64 %index
   %f2 = load double, double *%ptr
-  %negacc = fsub double -0.0, %acc
+  %negacc = fneg double %acc
   %res = call double @llvm.fma.f64 (double %f1, double %f2, double %negacc)
   ret double %res
 }
@@ -93,7 +93,7 @@ define double @f7(double %f1, double *%base, i64 %index, double %acc) {
   %index2 = add i64 %index, 511
   %ptr = getelementptr double, double *%base, i64 %index2
   %f2 = load double, double *%ptr
-  %negacc = fsub double -0.0, %acc
+  %negacc = fneg double %acc
   %res = call double @llvm.fma.f64 (double %f1, double %f2, double %negacc)
   ret double %res
 }
@@ -108,7 +108,7 @@ define double @f8(double %f1, double *%base, i64 %index, double %acc) {
   %index2 = add i64 %index, 512
   %ptr = getelementptr double, double *%base, i64 %index2
   %f2 = load double, double *%ptr
-  %negacc = fsub double -0.0, %acc
+  %negacc = fneg double %acc
   %res = call double @llvm.fma.f64 (double %f1, double %f2, double %negacc)
   ret double %res
 }
diff --git a/llvm/test/CodeGen/SystemZ/fp-mul-10.ll b/llvm/test/CodeGen/SystemZ/fp-mul-10.ll
index c23a6a202..669ccbacf 100644
--- a/llvm/test/CodeGen/SystemZ/fp-mul-10.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-mul-10.ll
@@ -8,7 +8,7 @@ define double @f1(double %f1, double %f2, double %acc) {
 ; CHECK: wfnmadb %f0, %f0, %f2, %f4
 ; CHECK: br %r14
   %res = call double @llvm.fma.f64 (double %f1, double %f2, double %acc)
-  %negres = fsub double -0.0, %res
+  %negres = fneg double %res
   ret double %negres
 }
 
@@ -16,9 +16,9 @@ define double @f2(double %f1, double %f2, double %acc) {
 ; CHECK-LABEL: f2:
 ; CHECK: wfnmsdb %f0, %f0, %f2, %f4
 ; CHECK: br %r14
-  %negacc = fsub double -0.0, %acc
+  %negacc = fneg double %acc
   %res = call double @llvm.fma.f64 (double %f1, double %f2, double %negacc)
-  %negres = fsub double -0.0, %res
+  %negres = fneg double %res
   ret double %negres
 }
 
@@ -27,7 +27,7 @@ define float @f3(float %f1, float %f2, float %acc) {
 ; CHECK: wfnmasb %f0, %f0, %f2, %f4
 ; CHECK: br %r14
   %res = call float @llvm.fma.f32 (float %f1, float %f2, float %acc)
-  %negres = fsub float -0.0, %res
+  %negres = fneg float %res
   ret float %negres
 }
 
@@ -35,9 +35,9 @@ define float @f4(float %f1, float %f2, float %acc) {
 ; CHECK-LABEL: f4:
 ; CHECK: wfnmssb %f0, %f0, %f2, %f4
 ; CHECK: br %r14
-  %negacc = fsub float -0.0, %acc
+  %negacc = fneg float %acc
   %res = call float @llvm.fma.f32 (float %f1, float %f2, float %negacc)
-  %negres = fsub float -0.0, %res
+  %negres = fneg float %res
   ret float %negres
 }
 
diff --git a/llvm/test/CodeGen/SystemZ/fp-mul-13.ll b/llvm/test/CodeGen/SystemZ/fp-mul-13.ll
new file mode 100644
index 000000000..4475195cc
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/fp-mul-13.ll
@@ -0,0 +1,16 @@
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+declare fp128 @llvm.fma.f128(fp128 %f1, fp128 %f2, fp128 %f3)
+
+define void @f1(fp128 *%ptr1, fp128 *%ptr2, fp128 *%ptr3, fp128 *%dst) {
+; CHECK-LABEL: f1:
+; CHECK: brasl %r14, fmal
+; CHECK: br %r14
+  %f1 = load fp128, fp128 *%ptr1
+  %f2 = load fp128, fp128 *%ptr2
+  %f3 = load fp128, fp128 *%ptr3
+  %res = call fp128 @llvm.fma.f128 (fp128 %f1, fp128 %f2, fp128 %f3)
+  store fp128 %res, fp128 *%dst
+  ret void
+}
+
diff --git a/llvm/test/CodeGen/SystemZ/fp-neg-01.ll b/llvm/test/CodeGen/SystemZ/fp-neg-01.ll
index 679dd7188..ff3a98231 100644
--- a/llvm/test/CodeGen/SystemZ/fp-neg-01.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-neg-01.ll
@@ -8,7 +8,7 @@ define float @f1(float %f) {
 ; CHECK-LABEL: f1:
 ; CHECK: lcdfr %f0, %f0
 ; CHECK: br %r14
-  %res = fsub float -0.0, %f
+  %res = fneg float %f
   ret float %res
 }
 
@@ -17,7 +17,7 @@ define double @f2(double %f) {
 ; CHECK-LABEL: f2:
 ; CHECK: lcdfr %f0, %f0
 ; CHECK: br %r14
-  %res = fsub double -0.0, %f
+  %res = fneg double %f
   ret double %res
 }
 
@@ -31,7 +31,7 @@ define void @f3(fp128 *%ptr, fp128 *%ptr2) {
 ; CHECK: br %r14
   %orig = load fp128, fp128 *%ptr
   %negzero = fpext float -0.0 to fp128
-  %neg = fsub fp128 0xL00000000000000008000000000000000, %orig
+  %neg = fneg fp128 %orig
   %op2 = load fp128, fp128 *%ptr2
   %res = fdiv fp128 %neg, %op2
   store fp128 %res, fp128 *%ptr
diff --git a/llvm/test/CodeGen/SystemZ/fp-neg-02.ll b/llvm/test/CodeGen/SystemZ/fp-neg-02.ll
index a721b837d..70ff19c80 100644
--- a/llvm/test/CodeGen/SystemZ/fp-neg-02.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-neg-02.ll
@@ -7,7 +7,7 @@ define float @f1(float %f) {
 ; CHECK-LABEL: f1:
 ; CHECK: lcdfr %f0, %f0
 ; CHECK: br %r14
-  %res = fsub float -0.0, %f
+  %res = fneg float %f
   ret float %res
 }
 
@@ -16,7 +16,7 @@ define double @f2(double %f) {
 ; CHECK-LABEL: f2:
 ; CHECK: lcdfr %f0, %f0
 ; CHECK: br %r14
-  %res = fsub double -0.0, %f
+  %res = fneg double %f
   ret double %res
 }
 
@@ -33,7 +33,7 @@ define void @f3(fp128 *%ptr, fp128 *%ptr2) {
 ; CHECK: br %r14
   %orig = load fp128, fp128 *%ptr
   %negzero = fpext float -0.0 to fp128
-  %neg = fsub fp128 0xL00000000000000008000000000000000, %orig
+  %neg = fneg fp128 %orig
   %op2 = load fp128, fp128 *%ptr2
   %res = fdiv fp128 %neg, %op2
   store fp128 %res, fp128 *%ptr
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-add-01.ll b/llvm/test/CodeGen/SystemZ/fp-strict-add-01.ll
index 57f75cc87..d95ab9331 100644
--- a/llvm/test/CodeGen/SystemZ/fp-strict-add-01.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-add-01.ll
@@ -8,19 +8,19 @@ declare float @foo()
 declare float @llvm.experimental.constrained.fadd.f32(float, float, metadata, metadata)
 
 ; Check register addition.
-define float @f1(float %f1, float %f2) {
+define float @f1(float %f1, float %f2) #0 {
 ; CHECK-LABEL: f1:
 ; CHECK: aebr %f0, %f2
 ; CHECK: br %r14
   %res = call float @llvm.experimental.constrained.fadd.f32(
                         float %f1, float %f2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
 
 ; Check the low end of the AEB range.
-define float @f2(float %f1, float *%ptr) {
+define float @f2(float %f1, float *%ptr) #0 {
 ; CHECK-LABEL: f2:
 ; CHECK: aeb %f0, 0(%r2)
 ; CHECK: br %r14
@@ -28,12 +28,12 @@ define float @f2(float %f1, float *%ptr) {
   %res = call float @llvm.experimental.constrained.fadd.f32(
                         float %f1, float %f2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
 
 ; Check the high end of the aligned AEB range.
-define float @f3(float %f1, float *%base) {
+define float @f3(float %f1, float *%base) #0 {
 ; CHECK-LABEL: f3:
 ; CHECK: aeb %f0, 4092(%r2)
 ; CHECK: br %r14
@@ -42,13 +42,13 @@ define float @f3(float %f1, float *%base) {
   %res = call float @llvm.experimental.constrained.fadd.f32(
                         float %f1, float %f2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
 
 ; Check the next word up, which needs separate address logic.
 ; Other sequences besides this one would be OK.
-define float @f4(float %f1, float *%base) {
+define float @f4(float %f1, float *%base) #0 {
 ; CHECK-LABEL: f4:
 ; CHECK: aghi %r2, 4096
 ; CHECK: aeb %f0, 0(%r2)
@@ -58,12 +58,12 @@ define float @f4(float %f1, float *%base) {
   %res = call float @llvm.experimental.constrained.fadd.f32(
                         float %f1, float %f2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
 
 ; Check negative displacements, which also need separate address logic.
-define float @f5(float %f1, float *%base) {
+define float @f5(float %f1, float *%base) #0 {
 ; CHECK-LABEL: f5:
 ; CHECK: aghi %r2, -4
 ; CHECK: aeb %f0, 0(%r2)
@@ -73,12 +73,12 @@ define float @f5(float %f1, float *%base) {
   %res = call float @llvm.experimental.constrained.fadd.f32(
                         float %f1, float %f2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
 
 ; Check that AEB allows indices.
-define float @f6(float %f1, float *%base, i64 %index) {
+define float @f6(float %f1, float *%base, i64 %index) #0 {
 ; CHECK-LABEL: f6:
 ; CHECK: sllg %r1, %r3, 2
 ; CHECK: aeb %f0, 400(%r1,%r2)
@@ -89,12 +89,12 @@ define float @f6(float %f1, float *%base, i64 %index) {
   %res = call float @llvm.experimental.constrained.fadd.f32(
                         float %f1, float %f2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
 
 ; Check that additions of spilled values can use AEB rather than AEBR.
-define float @f7(float *%ptr0) {
+define float @f7(float *%ptr0) #0 {
 ; CHECK-LABEL: f7:
 ; CHECK: brasl %r14, foo@PLT
 ; CHECK-SCALAR: aeb %f0, 16{{[04]}}(%r15)
@@ -122,52 +122,54 @@ define float @f7(float *%ptr0) {
   %val9 = load float, float *%ptr9
   %val10 = load float, float *%ptr10
 
-  %ret = call float @foo()
+  %ret = call float @foo() #0
 
   %add0 = call float @llvm.experimental.constrained.fadd.f32(
                         float %ret, float %val0,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %add1 = call float @llvm.experimental.constrained.fadd.f32(
                         float %add0, float %val1,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %add2 = call float @llvm.experimental.constrained.fadd.f32(
                         float %add1, float %val2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %add3 = call float @llvm.experimental.constrained.fadd.f32(
                         float %add2, float %val3,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %add4 = call float @llvm.experimental.constrained.fadd.f32(
                         float %add3, float %val4,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %add5 = call float @llvm.experimental.constrained.fadd.f32(
                         float %add4, float %val5,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %add6 = call float @llvm.experimental.constrained.fadd.f32(
                         float %add5, float %val6,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %add7 = call float @llvm.experimental.constrained.fadd.f32(
                         float %add6, float %val7,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %add8 = call float @llvm.experimental.constrained.fadd.f32(
                         float %add7, float %val8,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %add9 = call float @llvm.experimental.constrained.fadd.f32(
                         float %add8, float %val9,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %add10 = call float @llvm.experimental.constrained.fadd.f32(
                         float %add9, float %val10,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
 
   ret float %add10
 }
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-add-02.ll b/llvm/test/CodeGen/SystemZ/fp-strict-add-02.ll
index 739290969..2693aef7c 100644
--- a/llvm/test/CodeGen/SystemZ/fp-strict-add-02.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-add-02.ll
@@ -7,19 +7,19 @@ declare double @foo()
 declare double @llvm.experimental.constrained.fadd.f64(double, double, metadata, metadata)
 
 ; Check register addition.
-define double @f1(double %f1, double %f2) {
+define double @f1(double %f1, double %f2) #0 {
 ; CHECK-LABEL: f1:
 ; CHECK: adbr %f0, %f2
 ; CHECK: br %r14
   %res = call double @llvm.experimental.constrained.fadd.f64(
                         double %f1, double %f2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
 ; Check the low end of the ADB range.
-define double @f2(double %f1, double *%ptr) {
+define double @f2(double %f1, double *%ptr) #0 {
 ; CHECK-LABEL: f2:
 ; CHECK: adb %f0, 0(%r2)
 ; CHECK: br %r14
@@ -27,12 +27,12 @@ define double @f2(double %f1, double *%ptr) {
   %res = call double @llvm.experimental.constrained.fadd.f64(
                         double %f1, double %f2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
 ; Check the high end of the aligned ADB range.
-define double @f3(double %f1, double *%base) {
+define double @f3(double %f1, double *%base) #0 {
 ; CHECK-LABEL: f3:
 ; CHECK: adb %f0, 4088(%r2)
 ; CHECK: br %r14
@@ -41,13 +41,13 @@ define double @f3(double %f1, double *%base) {
   %res = call double @llvm.experimental.constrained.fadd.f64(
                         double %f1, double %f2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
 ; Check the next doubleword up, which needs separate address logic.
 ; Other sequences besides this one would be OK.
-define double @f4(double %f1, double *%base) {
+define double @f4(double %f1, double *%base) #0 {
 ; CHECK-LABEL: f4:
 ; CHECK: aghi %r2, 4096
 ; CHECK: adb %f0, 0(%r2)
@@ -57,12 +57,12 @@ define double @f4(double %f1, double *%base) {
   %res = call double @llvm.experimental.constrained.fadd.f64(
                         double %f1, double %f2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
 ; Check negative displacements, which also need separate address logic.
-define double @f5(double %f1, double *%base) {
+define double @f5(double %f1, double *%base) #0 {
 ; CHECK-LABEL: f5:
 ; CHECK: aghi %r2, -8
 ; CHECK: adb %f0, 0(%r2)
@@ -72,12 +72,12 @@ define double @f5(double %f1, double *%base) {
   %res = call double @llvm.experimental.constrained.fadd.f64(
                         double %f1, double %f2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
 ; Check that ADB allows indices.
-define double @f6(double %f1, double *%base, i64 %index) {
+define double @f6(double %f1, double *%base, i64 %index) #0 {
 ; CHECK-LABEL: f6:
 ; CHECK: sllg %r1, %r3, 3
 ; CHECK: adb %f0, 800(%r1,%r2)
@@ -88,12 +88,12 @@ define double @f6(double %f1, double *%base, i64 %index) {
   %res = call double @llvm.experimental.constrained.fadd.f64(
                         double %f1, double %f2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
 ; Check that additions of spilled values can use ADB rather than ADBR.
-define double @f7(double *%ptr0) {
+define double @f7(double *%ptr0) #0 {
 ; CHECK-LABEL: f7:
 ; CHECK: brasl %r14, foo@PLT
 ; CHECK-SCALAR: adb %f0, 160(%r15)
@@ -121,52 +121,54 @@ define double @f7(double *%ptr0) {
   %val9 = load double, double *%ptr9
   %val10 = load double, double *%ptr10
 
-  %ret = call double @foo()
+  %ret = call double @foo() #0
 
   %add0 = call double @llvm.experimental.constrained.fadd.f64(
                         double %ret, double %val0,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %add1 = call double @llvm.experimental.constrained.fadd.f64(
                         double %add0, double %val1,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %add2 = call double @llvm.experimental.constrained.fadd.f64(
                         double %add1, double %val2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %add3 = call double @llvm.experimental.constrained.fadd.f64(
                         double %add2, double %val3,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %add4 = call double @llvm.experimental.constrained.fadd.f64(
                         double %add3, double %val4,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %add5 = call double @llvm.experimental.constrained.fadd.f64(
                         double %add4, double %val5,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %add6 = call double @llvm.experimental.constrained.fadd.f64(
                         double %add5, double %val6,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %add7 = call double @llvm.experimental.constrained.fadd.f64(
                         double %add6, double %val7,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %add8 = call double @llvm.experimental.constrained.fadd.f64(
                         double %add7, double %val8,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %add9 = call double @llvm.experimental.constrained.fadd.f64(
                         double %add8, double %val9,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %add10 = call double @llvm.experimental.constrained.fadd.f64(
                         double %add9, double %val10,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
 
   ret double %add10
 }
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-add-03.ll b/llvm/test/CodeGen/SystemZ/fp-strict-add-03.ll
index d2535c9f0..13710a1f0 100644
--- a/llvm/test/CodeGen/SystemZ/fp-strict-add-03.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-add-03.ll
@@ -3,9 +3,10 @@
 ; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
 
 declare fp128 @llvm.experimental.constrained.fadd.f128(fp128, fp128, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.fpext.f128.f32(float, metadata)
 
 ; There is no memory form of 128-bit addition.
-define void @f1(fp128 *%ptr, float %f2) {
+define void @f1(fp128 *%ptr, float %f2) strictfp {
 ; CHECK-LABEL: f1:
 ; CHECK-DAG: lxebr %f0, %f0
 ; CHECK-DAG: ld %f1, 0(%r2)
@@ -15,11 +16,14 @@ define void @f1(fp128 *%ptr, float %f2) {
 ; CHECK: std %f2, 8(%r2)
 ; CHECK: br %r14
   %f1 = load fp128, fp128 *%ptr
-  %f2x = fpext float %f2 to fp128
+  %f2x = call fp128 @llvm.experimental.constrained.fpext.f128.f32(float %f2,
+                                               metadata !"fpexcept.strict") #0
   %sum = call fp128 @llvm.experimental.constrained.fadd.f128(
                         fp128 %f1, fp128 %f2x,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   store fp128 %sum, fp128 *%ptr
   ret void
 }
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-add-04.ll b/llvm/test/CodeGen/SystemZ/fp-strict-add-04.ll
index d4ec5fc68..98a3454a8 100644
--- a/llvm/test/CodeGen/SystemZ/fp-strict-add-04.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-add-04.ll
@@ -4,7 +4,7 @@
 
 declare fp128 @llvm.experimental.constrained.fadd.f128(fp128, fp128, metadata, metadata)
 
-define void @f1(fp128 *%ptr1, fp128 *%ptr2) {
+define void @f1(fp128 *%ptr1, fp128 *%ptr2) strictfp {
 ; CHECK-LABEL: f1:
 ; CHECK-DAG: vl [[REG1:%v[0-9]+]], 0(%r2)
 ; CHECK-DAG: vl [[REG2:%v[0-9]+]], 0(%r3)
@@ -16,7 +16,7 @@ define void @f1(fp128 *%ptr1, fp128 *%ptr2) {
   %sum = call fp128 @llvm.experimental.constrained.fadd.f128(
                         fp128 %f1, fp128 %f2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") strictfp
   store fp128 %sum, fp128 *%ptr1
   ret void
 }
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-alias.ll b/llvm/test/CodeGen/SystemZ/fp-strict-alias.ll
index fe27b61c2..5091c3dc9 100644
--- a/llvm/test/CodeGen/SystemZ/fp-strict-alias.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-alias.ll
@@ -5,6 +5,7 @@
 declare float @llvm.experimental.constrained.sqrt.f32(float, metadata, metadata)
 declare float @llvm.sqrt.f32(float)
 declare void @llvm.s390.sfpc(i32)
+declare void @bar()
 
 ; The basic assumption of all following tests is that on z13, we never
 ; want to see two square root instructions directly in a row, so the
@@ -30,7 +31,7 @@ define void @f1(float %f1, float %f2, float *%ptr1, float *%ptr2) {
   ret void
 }
 
-define void @f2(float %f1, float %f2, float *%ptr1, float *%ptr2) {
+define void @f2(float %f1, float %f2, float *%ptr1, float *%ptr2) #0 {
 ; CHECK-LABEL: f2:
 ; CHECK: sqebr
 ; CHECK: ste
@@ -41,11 +42,11 @@ define void @f2(float %f1, float %f2, float *%ptr1, float *%ptr2) {
   %sqrt1 = call float @llvm.experimental.constrained.sqrt.f32(
                         float %f1,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.ignore")
+                        metadata !"fpexcept.ignore") #0
   %sqrt2 = call float @llvm.experimental.constrained.sqrt.f32(
                         float %f2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.ignore")
+                        metadata !"fpexcept.ignore") #0
 
   store float %sqrt1, float *%ptr1
   store float %sqrt2, float *%ptr2
@@ -53,7 +54,7 @@ define void @f2(float %f1, float %f2, float *%ptr1, float *%ptr2) {
   ret void
 }
 
-define void @f3(float %f1, float %f2, float *%ptr1, float *%ptr2) {
+define void @f3(float %f1, float %f2, float *%ptr1, float *%ptr2) #0 {
 ; CHECK-LABEL: f3:
 ; CHECK: sqebr
 ; CHECK: ste
@@ -64,11 +65,34 @@ define void @f3(float %f1, float %f2, float *%ptr1, float *%ptr2) {
   %sqrt1 = call float @llvm.experimental.constrained.sqrt.f32(
                         float %f1,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.maytrap") #0
   %sqrt2 = call float @llvm.experimental.constrained.sqrt.f32(
                         float %f2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.maytrap") #0
+
+  store float %sqrt1, float *%ptr1
+  store float %sqrt2, float *%ptr2
+
+  ret void
+}
+
+define void @f4(float %f1, float %f2, float *%ptr1, float *%ptr2) #0 {
+; CHECK-LABEL: f4:
+; CHECK: sqebr
+; CHECK: ste
+; CHECK: sqebr
+; CHECK: ste
+; CHECK: br %r14
+
+  %sqrt1 = call float @llvm.experimental.constrained.sqrt.f32(
+                        float %f1,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
+  %sqrt2 = call float @llvm.experimental.constrained.sqrt.f32(
+                        float %f2,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
 
   store float %sqrt1, float *%ptr1
   store float %sqrt2, float *%ptr2
@@ -78,11 +102,11 @@ define void @f3(float %f1, float %f2, float *%ptr1, float *%ptr2) {
 
 
 ; We can move a non-strict FP operation or a fpexcept.ignore
-; operation even across a volatile store, but not a fpexcept.strict
-; operation.
+; operation even across a volatile store, but not a fpexcept.maytrap
+; or fpexcept.strict operation.
 
-define void @f4(float %f1, float %f2, float *%ptr1, float *%ptr2) {
-; CHECK-LABEL: f4:
+define void @f5(float %f1, float %f2, float *%ptr1, float *%ptr2) {
+; CHECK-LABEL: f5:
 ; CHECK: sqebr
 ; CHECK: ste
 ; CHECK: sqebr
@@ -98,8 +122,8 @@ define void @f4(float %f1, float %f2, float *%ptr1, float *%ptr2) {
   ret void
 }
 
-define void @f5(float %f1, float %f2, float *%ptr1, float *%ptr2) {
-; CHECK-LABEL: f5:
+define void @f6(float %f1, float %f2, float *%ptr1, float *%ptr2) #0 {
+; CHECK-LABEL: f6:
 ; CHECK: sqebr
 ; CHECK: ste
 ; CHECK: sqebr
@@ -109,11 +133,11 @@ define void @f5(float %f1, float %f2, float *%ptr1, float *%ptr2) {
   %sqrt1 = call float @llvm.experimental.constrained.sqrt.f32(
                         float %f1,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.ignore")
+                        metadata !"fpexcept.ignore") #0
   %sqrt2 = call float @llvm.experimental.constrained.sqrt.f32(
                         float %f2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.ignore")
+                        metadata !"fpexcept.ignore") #0
 
   store volatile float %sqrt1, float *%ptr1
   store volatile float %sqrt2, float *%ptr2
@@ -121,8 +145,8 @@ define void @f5(float %f1, float %f2, float *%ptr1, float *%ptr2) {
   ret void
 }
 
-define void @f6(float %f1, float %f2, float *%ptr1, float *%ptr2) {
-; CHECK-LABEL: f6:
+define void @f7(float %f1, float %f2, float *%ptr1, float *%ptr2) #0 {
+; CHECK-LABEL: f7:
 ; CHECK: sqebr
 ; CHECK: sqebr
 ; CHECK: ste
@@ -132,11 +156,34 @@ define void @f6(float %f1, float %f2, float *%ptr1, float *%ptr2) {
   %sqrt1 = call float @llvm.experimental.constrained.sqrt.f32(
                         float %f1,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.maytrap") #0
   %sqrt2 = call float @llvm.experimental.constrained.sqrt.f32(
                         float %f2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.maytrap") #0
+
+  store volatile float %sqrt1, float *%ptr1
+  store volatile float %sqrt2, float *%ptr2
+
+  ret void
+}
+
+define void @f8(float %f1, float %f2, float *%ptr1, float *%ptr2) #0 {
+; CHECK-LABEL: f8:
+; CHECK: sqebr
+; CHECK: sqebr
+; CHECK: ste
+; CHECK: ste
+; CHECK: br %r14
+
+  %sqrt1 = call float @llvm.experimental.constrained.sqrt.f32(
+                        float %f1,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
+  %sqrt2 = call float @llvm.experimental.constrained.sqrt.f32(
+                        float %f2,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
 
   store volatile float %sqrt1, float *%ptr1
   store volatile float %sqrt2, float *%ptr2
@@ -147,8 +194,8 @@ define void @f6(float %f1, float %f2, float *%ptr1, float *%ptr2) {
 
 ; No variant of FP operations can be scheduled across a SPFC.
 
-define void @f7(float %f1, float %f2, float *%ptr1, float *%ptr2) {
-; CHECK-LABEL: f7:
+define void @f9(float %f1, float %f2, float *%ptr1, float *%ptr2) {
+; CHECK-LABEL: f9:
 ; CHECK: sqebr
 ; CHECK: sqebr
 ; CHECK: ste
@@ -166,8 +213,8 @@ define void @f7(float %f1, float %f2, float *%ptr1, float *%ptr2) {
   ret void
 }
 
-define void @f8(float %f1, float %f2, float *%ptr1, float *%ptr2) {
-; CHECK-LABEL: f8:
+define void @f10(float %f1, float %f2, float *%ptr1, float *%ptr2) #0 {
+; CHECK-LABEL: f10:
 ; CHECK: sqebr
 ; CHECK: sqebr
 ; CHECK: ste
@@ -177,13 +224,13 @@ define void @f8(float %f1, float %f2, float *%ptr1, float *%ptr2) {
   %sqrt1 = call float @llvm.experimental.constrained.sqrt.f32(
                         float %f1,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.ignore")
+                        metadata !"fpexcept.ignore") #0
   %sqrt2 = call float @llvm.experimental.constrained.sqrt.f32(
                         float %f2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.ignore")
+                        metadata !"fpexcept.ignore") #0
 
-  call void @llvm.s390.sfpc(i32 0)
+  call void @llvm.s390.sfpc(i32 0) #0
 
   store float %sqrt1, float *%ptr1
   store float %sqrt2, float *%ptr2
@@ -191,8 +238,8 @@ define void @f8(float %f1, float %f2, float *%ptr1, float *%ptr2) {
   ret void
 }
 
-define void @f9(float %f1, float %f2, float *%ptr1, float *%ptr2) {
-; CHECK-LABEL: f9:
+define void @f11(float %f1, float %f2, float *%ptr1, float *%ptr2) #0 {
+; CHECK-LABEL: f11:
 ; CHECK: sqebr
 ; CHECK: sqebr
 ; CHECK: ste
@@ -202,13 +249,13 @@ define void @f9(float %f1, float %f2, float *%ptr1, float *%ptr2) {
   %sqrt1 = call float @llvm.experimental.constrained.sqrt.f32(
                         float %f1,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.maytrap") #0
   %sqrt2 = call float @llvm.experimental.constrained.sqrt.f32(
                         float %f2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.maytrap") #0
 
-  call void @llvm.s390.sfpc(i32 0)
+  call void @llvm.s390.sfpc(i32 0) #0
 
   store float %sqrt1, float *%ptr1
   store float %sqrt2, float *%ptr2
@@ -216,3 +263,138 @@ define void @f9(float %f1, float %f2, float *%ptr1, float *%ptr2) {
   ret void
 }
 
+define void @f12(float %f1, float %f2, float *%ptr1, float *%ptr2) #0 {
+; CHECK-LABEL: f12:
+; CHECK: sqebr
+; CHECK: sqebr
+; CHECK: ste
+; CHECK: ste
+; CHECK: br %r14
+
+  %sqrt1 = call float @llvm.experimental.constrained.sqrt.f32(
+                        float %f1,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
+  %sqrt2 = call float @llvm.experimental.constrained.sqrt.f32(
+                        float %f2,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
+
+  call void @llvm.s390.sfpc(i32 0) #0
+
+  store float %sqrt1, float *%ptr1
+  store float %sqrt2, float *%ptr2
+
+  ret void
+}
+
+; If the result of any FP operation is unused, it can be removed
+; -- except for fpexcept.strict operations.
+
+define void @f13(float %f1) {
+; CHECK-LABEL: f13:
+; CHECK-NOT: sqeb
+; CHECK: br %r14
+
+  %sqrt = call float @llvm.sqrt.f32(float %f1)
+
+  ret void
+}
+
+define void @f14(float %f1) #0 {
+; CHECK-LABEL: f14:
+; CHECK-NOT: sqeb
+; CHECK: br %r14
+
+  %sqrt = call float @llvm.experimental.constrained.sqrt.f32(
+                        float %f1,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.ignore") #0
+
+  ret void
+}
+
+define void @f15(float %f1) #0 {
+; CHECK-LABEL: f15:
+; CHECK-NOT: sqeb
+; CHECK: br %r14
+
+  %sqrt = call float @llvm.experimental.constrained.sqrt.f32(
+                        float %f1,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.maytrap") #0
+
+  ret void
+}
+
+define void @f16(float %f1) #0 {
+; CHECK-LABEL: f16:
+; CHECK: sqebr
+; CHECK: br %r14
+
+  %sqrt = call float @llvm.experimental.constrained.sqrt.f32(
+                        float %f1,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
+
+  ret void
+}
+
+
+; Verify that constrained intrinsics and memory operations get their
+; chains linked up properly.
+
+define void @f17(float %in, float* %out) #0 {
+; CHECK-LABEL: f17:
+; CHECK: sqebr
+; CHECK: ste
+; CHECK: jg bar
+  %sqrt = call float @llvm.sqrt.f32(float %in)
+  store float %sqrt, float* %out, align 4
+  tail call void @bar() #0
+  ret void
+}
+
+define void @f18(float %in, float* %out) #0 {
+; CHECK-LABEL: f18:
+; CHECK: sqebr
+; CHECK: ste
+; CHECK: jg bar
+  %sqrt = call float @llvm.experimental.constrained.sqrt.f32(
+                        float %in,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.ignore") #0
+  store float %sqrt, float* %out, align 4
+  tail call void @bar() #0
+  ret void
+}
+
+define void @f19(float %in, float* %out) #0 {
+; CHECK-LABEL: f19:
+; CHECK: sqebr
+; CHECK: ste
+; CHECK: jg bar
+  %sqrt = call float @llvm.experimental.constrained.sqrt.f32(
+                        float %in,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.maytrap") #0
+  store float %sqrt, float* %out, align 4
+  tail call void @bar() #0
+  ret void
+}
+
+define void @f20(float %in, float* %out) #0 {
+; CHECK-LABEL: f20:
+; CHECK: sqebr
+; CHECK: ste
+; CHECK: jg bar
+  %sqrt = call float @llvm.experimental.constrained.sqrt.f32(
+                        float %in,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
+  store float %sqrt, float* %out, align 4
+  tail call void @bar() #0
+  ret void
+}
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-cmp-01.ll b/llvm/test/CodeGen/SystemZ/fp-strict-cmp-01.ll
new file mode 100644
index 000000000..3a815294a
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-cmp-01.ll
@@ -0,0 +1,435 @@
+; Test 32-bit floating-point strict comparison.  The tests assume a z10
+; implementation of select, using conditional branches rather than LOCGR.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z10 \
+; RUN:   | FileCheck -check-prefix=CHECK -check-prefix=CHECK-SCALAR %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z14 \
+; RUN:   | FileCheck -check-prefix=CHECK -check-prefix=CHECK-VECTOR %s
+
+declare float @foo()
+
+; Check comparison with registers.
+define i64 @f1(i64 %a, i64 %b, float %f1, float %f2) #0 {
+; CHECK-LABEL: f1:
+; CHECK: cebr %f0, %f2
+; CHECK-SCALAR-NEXT: ber %r14
+; CHECK-SCALAR: lgr %r2, %r3
+; CHECK-VECTOR-NEXT: locgrne %r2, %r3
+; CHECK: br %r14
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f32(
+                                               float %f1, float %f2,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+; Check the low end of the CEB range.
+define i64 @f2(i64 %a, i64 %b, float %f1, float *%ptr) #0 {
+; CHECK-LABEL: f2:
+; CHECK: ceb %f0, 0(%r4)
+; CHECK-SCALAR-NEXT: ber %r14
+; CHECK-SCALAR: lgr %r2, %r3
+; CHECK-VECTOR-NEXT: locgrne %r2, %r3
+; CHECK: br %r14
+  %f2 = load float, float *%ptr
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f32(
+                                               float %f1, float %f2,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+; Check the high end of the aligned CEB range.
+define i64 @f3(i64 %a, i64 %b, float %f1, float *%base) #0 {
+; CHECK-LABEL: f3:
+; CHECK: ceb %f0, 4092(%r4)
+; CHECK-SCALAR-NEXT: ber %r14
+; CHECK-SCALAR: lgr %r2, %r3
+; CHECK-VECTOR-NEXT: locgrne %r2, %r3
+; CHECK: br %r14
+  %ptr = getelementptr float, float *%base, i64 1023
+  %f2 = load float, float *%ptr
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f32(
+                                               float %f1, float %f2,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+; Check the next word up, which needs separate address logic.
+; Other sequences besides this one would be OK.
+define i64 @f4(i64 %a, i64 %b, float %f1, float *%base) #0 {
+; CHECK-LABEL: f4:
+; CHECK: aghi %r4, 4096
+; CHECK: ceb %f0, 0(%r4)
+; CHECK-SCALAR-NEXT: ber %r14
+; CHECK-SCALAR: lgr %r2, %r3
+; CHECK-VECTOR-NEXT: locgrne %r2, %r3
+; CHECK: br %r14
+  %ptr = getelementptr float, float *%base, i64 1024
+  %f2 = load float, float *%ptr
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f32(
+                                               float %f1, float %f2,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+; Check negative displacements, which also need separate address logic.
+define i64 @f5(i64 %a, i64 %b, float %f1, float *%base) #0 {
+; CHECK-LABEL: f5:
+; CHECK: aghi %r4, -4
+; CHECK: ceb %f0, 0(%r4)
+; CHECK-SCALAR-NEXT: ber %r14
+; CHECK-SCALAR: lgr %r2, %r3
+; CHECK-VECTOR-NEXT: locgrne %r2, %r3
+; CHECK: br %r14
+  %ptr = getelementptr float, float *%base, i64 -1
+  %f2 = load float, float *%ptr
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f32(
+                                               float %f1, float %f2,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+; Check that CEB allows indices.
+define i64 @f6(i64 %a, i64 %b, float %f1, float *%base, i64 %index) #0 {
+; CHECK-LABEL: f6:
+; CHECK: sllg %r1, %r5, 2
+; CHECK: ceb %f0, 400(%r1,%r4)
+; CHECK-SCALAR-NEXT: ber %r14
+; CHECK-SCALAR: lgr %r2, %r3
+; CHECK-VECTOR-NEXT: locgrne %r2, %r3
+; CHECK: br %r14
+  %ptr1 = getelementptr float, float *%base, i64 %index
+  %ptr2 = getelementptr float, float *%ptr1, i64 100
+  %f2 = load float, float *%ptr2
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f32(
+                                               float %f1, float %f2,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+; Check that comparisons of spilled values can use CEB rather than CEBR.
+define float @f7(float *%ptr0) #0 {
+; CHECK-LABEL: f7:
+; CHECK: brasl %r14, foo@PLT
+; CHECK-SCALAR: ceb {{%f[0-9]+}}, 16{{[04]}}(%r15)
+; CHECK: br %r14
+  %ptr1 = getelementptr float, float *%ptr0, i64 2
+  %ptr2 = getelementptr float, float *%ptr0, i64 4
+  %ptr3 = getelementptr float, float *%ptr0, i64 6
+  %ptr4 = getelementptr float, float *%ptr0, i64 8
+  %ptr5 = getelementptr float, float *%ptr0, i64 10
+  %ptr6 = getelementptr float, float *%ptr0, i64 12
+  %ptr7 = getelementptr float, float *%ptr0, i64 14
+  %ptr8 = getelementptr float, float *%ptr0, i64 16
+  %ptr9 = getelementptr float, float *%ptr0, i64 18
+  %ptr10 = getelementptr float, float *%ptr0, i64 20
+
+  %val0 = load float, float *%ptr0
+  %val1 = load float, float *%ptr1
+  %val2 = load float, float *%ptr2
+  %val3 = load float, float *%ptr3
+  %val4 = load float, float *%ptr4
+  %val5 = load float, float *%ptr5
+  %val6 = load float, float *%ptr6
+  %val7 = load float, float *%ptr7
+  %val8 = load float, float *%ptr8
+  %val9 = load float, float *%ptr9
+  %val10 = load float, float *%ptr10
+
+  %ret = call float @foo() #0
+
+  %cmp0 = call i1 @llvm.experimental.constrained.fcmp.f32(
+                                               float %ret, float %val0,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %cmp1 = call i1 @llvm.experimental.constrained.fcmp.f32(
+                                               float %ret, float %val1,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %cmp2 = call i1 @llvm.experimental.constrained.fcmp.f32(
+                                               float %ret, float %val2,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %cmp3 = call i1 @llvm.experimental.constrained.fcmp.f32(
+                                               float %ret, float %val3,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %cmp4 = call i1 @llvm.experimental.constrained.fcmp.f32(
+                                               float %ret, float %val4,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %cmp5 = call i1 @llvm.experimental.constrained.fcmp.f32(
+                                               float %ret, float %val5,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %cmp6 = call i1 @llvm.experimental.constrained.fcmp.f32(
+                                               float %ret, float %val6,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %cmp7 = call i1 @llvm.experimental.constrained.fcmp.f32(
+                                               float %ret, float %val7,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %cmp8 = call i1 @llvm.experimental.constrained.fcmp.f32(
+                                               float %ret, float %val8,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %cmp9 = call i1 @llvm.experimental.constrained.fcmp.f32(
+                                               float %ret, float %val9,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %cmp10 = call i1 @llvm.experimental.constrained.fcmp.f32(
+                                               float %ret, float %val10,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+
+  %sel0 = select i1 %cmp0, float %ret, float 0.0
+  %sel1 = select i1 %cmp1, float %sel0, float 1.0
+  %sel2 = select i1 %cmp2, float %sel1, float 2.0
+  %sel3 = select i1 %cmp3, float %sel2, float 3.0
+  %sel4 = select i1 %cmp4, float %sel3, float 4.0
+  %sel5 = select i1 %cmp5, float %sel4, float 5.0
+  %sel6 = select i1 %cmp6, float %sel5, float 6.0
+  %sel7 = select i1 %cmp7, float %sel6, float 7.0
+  %sel8 = select i1 %cmp8, float %sel7, float 8.0
+  %sel9 = select i1 %cmp9, float %sel8, float 9.0
+  %sel10 = select i1 %cmp10, float %sel9, float 10.0
+
+  ret float %sel10
+}
+
+; Check comparison with zero.
+define i64 @f8(i64 %a, i64 %b, float %f) #0 {
+; CHECK-LABEL: f8:
+; CHECK: ltebr %f0, %f0
+; CHECK-SCALAR-NEXT: ber %r14
+; CHECK-SCALAR: lgr %r2, %r3
+; CHECK-VECTOR-NEXT: locgrne %r2, %r3
+; CHECK: br %r14
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f32(
+                                               float %f, float 0.0,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+; Check the comparison can be reversed if that allows CEB to be used,
+; first with oeq.
+define i64 @f9(i64 %a, i64 %b, float %f2, float *%ptr) #0 {
+; CHECK-LABEL: f9:
+; CHECK: ceb %f0, 0(%r4)
+; CHECK-SCALAR-NEXT: ber %r14
+; CHECK-SCALAR: lgr %r2, %r3
+; CHECK-VECTOR-NEXT: locgrne %r2, %r3
+; CHECK: br %r14
+  %f1 = load float, float *%ptr
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f32(
+                                               float %f1, float %f2,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+; ...then one.
+define i64 @f10(i64 %a, i64 %b, float %f2, float *%ptr) #0 {
+; CHECK-LABEL: f10:
+; CHECK: ceb %f0, 0(%r4)
+; CHECK-SCALAR-NEXT: blhr %r14
+; CHECK-SCALAR: lgr %r2, %r3
+; CHECK-VECTOR-NEXT: locgrnlh %r2, %r3
+; CHECK: br %r14
+  %f1 = load float, float *%ptr
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f32(
+                                               float %f1, float %f2,
+                                               metadata !"one",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+; ...then olt.
+define i64 @f11(i64 %a, i64 %b, float %f2, float *%ptr) #0 {
+; CHECK-LABEL: f11:
+; CHECK: ceb %f0, 0(%r4)
+; CHECK-SCALAR-NEXT: bhr %r14
+; CHECK-SCALAR: lgr %r2, %r3
+; CHECK-VECTOR-NEXT: locgrnh %r2, %r3
+; CHECK: br %r14
+  %f1 = load float, float *%ptr
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f32(
+                                               float %f1, float %f2,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+; ...then ole.
+define i64 @f12(i64 %a, i64 %b, float %f2, float *%ptr) #0 {
+; CHECK-LABEL: f12:
+; CHECK: ceb %f0, 0(%r4)
+; CHECK-SCALAR-NEXT: bher %r14
+; CHECK-SCALAR: lgr %r2, %r3
+; CHECK-VECTOR-NEXT: locgrnhe %r2, %r3
+; CHECK: br %r14
+  %f1 = load float, float *%ptr
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f32(
+                                               float %f1, float %f2,
+                                               metadata !"ole",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+; ...then oge.
+define i64 @f13(i64 %a, i64 %b, float %f2, float *%ptr) #0 {
+; CHECK-LABEL: f13:
+; CHECK: ceb %f0, 0(%r4)
+; CHECK-SCALAR-NEXT: bler %r14
+; CHECK-SCALAR: lgr %r2, %r3
+; CHECK-VECTOR-NEXT: locgrnle %r2, %r3
+; CHECK: br %r14
+  %f1 = load float, float *%ptr
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f32(
+                                               float %f1, float %f2,
+                                               metadata !"oge",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+; ...then ogt.
+define i64 @f14(i64 %a, i64 %b, float %f2, float *%ptr) #0 {
+; CHECK-LABEL: f14:
+; CHECK: ceb %f0, 0(%r4)
+; CHECK-SCALAR-NEXT: blr %r14
+; CHECK-SCALAR: lgr %r2, %r3
+; CHECK-VECTOR-NEXT: locgrnl %r2, %r3
+; CHECK: br %r14
+  %f1 = load float, float *%ptr
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f32(
+                                               float %f1, float %f2,
+                                               metadata !"ogt",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+; ...then ueq.
+define i64 @f15(i64 %a, i64 %b, float %f2, float *%ptr) #0 {
+; CHECK-LABEL: f15:
+; CHECK: ceb %f0, 0(%r4)
+; CHECK-SCALAR-NEXT: bnlhr %r14
+; CHECK-SCALAR: lgr %r2, %r3
+; CHECK-VECTOR-NEXT: locgrlh %r2, %r3
+; CHECK: br %r14
+  %f1 = load float, float *%ptr
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f32(
+                                               float %f1, float %f2,
+                                               metadata !"ueq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+; ...then une.
+define i64 @f16(i64 %a, i64 %b, float %f2, float *%ptr) #0 {
+; CHECK-LABEL: f16:
+; CHECK: ceb %f0, 0(%r4)
+; CHECK-SCALAR-NEXT: bner %r14
+; CHECK-SCALAR: lgr %r2, %r3
+; CHECK-VECTOR-NEXT: locgre %r2, %r3
+; CHECK: br %r14
+  %f1 = load float, float *%ptr
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f32(
+                                               float %f1, float %f2,
+                                               metadata !"une",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+; ...then ult.
+define i64 @f17(i64 %a, i64 %b, float %f2, float *%ptr) #0 {
+; CHECK-LABEL: f17:
+; CHECK: ceb %f0, 0(%r4)
+; CHECK-SCALAR-NEXT: bnler %r14
+; CHECK-SCALAR: lgr %r2, %r3
+; CHECK-VECTOR-NEXT: locgrle %r2, %r3
+; CHECK: br %r14
+  %f1 = load float, float *%ptr
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f32(
+                                               float %f1, float %f2,
+                                               metadata !"ult",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+; ...then ule.
+define i64 @f18(i64 %a, i64 %b, float %f2, float *%ptr) #0 {
+; CHECK-LABEL: f18:
+; CHECK: ceb %f0, 0(%r4)
+; CHECK-SCALAR-NEXT: bnlr %r14
+; CHECK-SCALAR: lgr %r2, %r3
+; CHECK-VECTOR-NEXT: locgrl %r2, %r3
+; CHECK: br %r14
+  %f1 = load float, float *%ptr
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f32(
+                                               float %f1, float %f2,
+                                               metadata !"ule",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+; ...then uge.
+define i64 @f19(i64 %a, i64 %b, float %f2, float *%ptr) #0 {
+; CHECK-LABEL: f19:
+; CHECK: ceb %f0, 0(%r4)
+; CHECK-SCALAR-NEXT: bnhr %r14
+; CHECK-SCALAR: lgr %r2, %r3
+; CHECK-VECTOR-NEXT: locgrh %r2, %r3
+; CHECK: br %r14
+  %f1 = load float, float *%ptr
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f32(
+                                               float %f1, float %f2,
+                                               metadata !"uge",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+; ...then ugt.
+define i64 @f20(i64 %a, i64 %b, float %f2, float *%ptr) #0 {
+; CHECK-LABEL: f20:
+; CHECK: ceb %f0, 0(%r4)
+; CHECK-SCALAR-NEXT: bnher %r14
+; CHECK-SCALAR: lgr %r2, %r3
+; CHECK-VECTOR-NEXT: locgrhe %r2, %r3
+; CHECK: br %r14
+  %f1 = load float, float *%ptr
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f32(
+                                               float %f1, float %f2,
+                                               metadata !"ugt",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+attributes #0 = { strictfp }
+
+declare i1 @llvm.experimental.constrained.fcmp.f32(float, float, metadata, metadata)
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-cmp-02.ll b/llvm/test/CodeGen/SystemZ/fp-strict-cmp-02.ll
new file mode 100644
index 000000000..12a39951f
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-cmp-02.ll
@@ -0,0 +1,249 @@
+; Test 64-bit floating-point strict comparison.  The tests assume a z10
+; implementation of select, using conditional branches rather than LOCGR.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z10 \
+; RUN:   | FileCheck -check-prefix=CHECK -check-prefix=CHECK-SCALAR %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 -verify-machineinstrs\
+; RUN:   | FileCheck -check-prefix=CHECK -check-prefix=CHECK-VECTOR %s
+
+declare double @foo()
+
+; Check comparison with registers.
+define i64 @f1(i64 %a, i64 %b, double %f1, double %f2) #0 {
+; CHECK-LABEL: f1:
+; CHECK: cdbr %f0, %f2
+; CHECK-SCALAR-NEXT: ber %r14
+; CHECK-SCALAR: lgr %r2, %r3
+; CHECK-VECTOR-NEXT: locgrne %r2, %r3
+; CHECK: br %r14
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f64(
+                                               double %f1, double %f2,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+; Check the low end of the CDB range.
+define i64 @f2(i64 %a, i64 %b, double %f1, double *%ptr) #0 {
+; CHECK-LABEL: f2:
+; CHECK: cdb %f0, 0(%r4)
+; CHECK-SCALAR-NEXT: ber %r14
+; CHECK-SCALAR: lgr %r2, %r3
+; CHECK-VECTOR-NEXT: locgrne %r2, %r3
+; CHECK: br %r14
+  %f2 = load double, double *%ptr
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f64(
+                                               double %f1, double %f2,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+; Check the high end of the aligned CDB range.
+define i64 @f3(i64 %a, i64 %b, double %f1, double *%base) #0 {
+; CHECK-LABEL: f3:
+; CHECK: cdb %f0, 4088(%r4)
+; CHECK-SCALAR-NEXT: ber %r14
+; CHECK-SCALAR: lgr %r2, %r3
+; CHECK-VECTOR-NEXT: locgrne %r2, %r3
+; CHECK: br %r14
+  %ptr = getelementptr double, double *%base, i64 511
+  %f2 = load double, double *%ptr
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f64(
+                                               double %f1, double %f2,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+; Check the next doubleword up, which needs separate address logic.
+; Other sequences besides this one would be OK.
+define i64 @f4(i64 %a, i64 %b, double %f1, double *%base) #0 {
+; CHECK-LABEL: f4:
+; CHECK: aghi %r4, 4096
+; CHECK: cdb %f0, 0(%r4)
+; CHECK-SCALAR-NEXT: ber %r14
+; CHECK-SCALAR: lgr %r2, %r3
+; CHECK-VECTOR-NEXT: locgrne %r2, %r3
+; CHECK: br %r14
+  %ptr = getelementptr double, double *%base, i64 512
+  %f2 = load double, double *%ptr
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f64(
+                                               double %f1, double %f2,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+; Check negative displacements, which also need separate address logic.
+define i64 @f5(i64 %a, i64 %b, double %f1, double *%base) #0 {
+; CHECK-LABEL: f5:
+; CHECK: aghi %r4, -8
+; CHECK: cdb %f0, 0(%r4)
+; CHECK-SCALAR-NEXT: ber %r14
+; CHECK-SCALAR: lgr %r2, %r3
+; CHECK-VECTOR-NEXT: locgrne %r2, %r3
+; CHECK: br %r14
+  %ptr = getelementptr double, double *%base, i64 -1
+  %f2 = load double, double *%ptr
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f64(
+                                               double %f1, double %f2,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+; Check that CDB allows indices.
+define i64 @f6(i64 %a, i64 %b, double %f1, double *%base, i64 %index) #0 {
+; CHECK-LABEL: f6:
+; CHECK: sllg %r1, %r5, 3
+; CHECK: cdb %f0, 800(%r1,%r4)
+; CHECK-SCALAR-NEXT: ber %r14
+; CHECK-SCALAR: lgr %r2, %r3
+; CHECK-VECTOR-NEXT: locgrne %r2, %r3
+; CHECK: br %r14
+  %ptr1 = getelementptr double, double *%base, i64 %index
+  %ptr2 = getelementptr double, double *%ptr1, i64 100
+  %f2 = load double, double *%ptr2
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f64(
+                                               double %f1, double %f2,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+; Check that comparisons of spilled values can use CDB rather than CDBR.
+define double @f7(double *%ptr0) #0 {
+; CHECK-LABEL: f7:
+; CHECK: brasl %r14, foo@PLT
+; CHECK-SCALAR: cdb {{%f[0-9]+}}, 160(%r15)
+; CHECK: br %r14
+  %ptr1 = getelementptr double, double *%ptr0, i64 2
+  %ptr2 = getelementptr double, double *%ptr0, i64 4
+  %ptr3 = getelementptr double, double *%ptr0, i64 6
+  %ptr4 = getelementptr double, double *%ptr0, i64 8
+  %ptr5 = getelementptr double, double *%ptr0, i64 10
+  %ptr6 = getelementptr double, double *%ptr0, i64 12
+  %ptr7 = getelementptr double, double *%ptr0, i64 14
+  %ptr8 = getelementptr double, double *%ptr0, i64 16
+  %ptr9 = getelementptr double, double *%ptr0, i64 18
+  %ptr10 = getelementptr double, double *%ptr0, i64 20
+
+  %val0 = load double, double *%ptr0
+  %val1 = load double, double *%ptr1
+  %val2 = load double, double *%ptr2
+  %val3 = load double, double *%ptr3
+  %val4 = load double, double *%ptr4
+  %val5 = load double, double *%ptr5
+  %val6 = load double, double *%ptr6
+  %val7 = load double, double *%ptr7
+  %val8 = load double, double *%ptr8
+  %val9 = load double, double *%ptr9
+  %val10 = load double, double *%ptr10
+
+  %ret = call double @foo() #0
+
+  %cmp0 = call i1 @llvm.experimental.constrained.fcmp.f64(
+                                               double %ret, double %val0,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %cmp1 = call i1 @llvm.experimental.constrained.fcmp.f64(
+                                               double %ret, double %val1,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %cmp2 = call i1 @llvm.experimental.constrained.fcmp.f64(
+                                               double %ret, double %val2,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %cmp3 = call i1 @llvm.experimental.constrained.fcmp.f64(
+                                               double %ret, double %val3,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %cmp4 = call i1 @llvm.experimental.constrained.fcmp.f64(
+                                               double %ret, double %val4,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %cmp5 = call i1 @llvm.experimental.constrained.fcmp.f64(
+                                               double %ret, double %val5,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %cmp6 = call i1 @llvm.experimental.constrained.fcmp.f64(
+                                               double %ret, double %val6,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %cmp7 = call i1 @llvm.experimental.constrained.fcmp.f64(
+                                               double %ret, double %val7,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %cmp8 = call i1 @llvm.experimental.constrained.fcmp.f64(
+                                               double %ret, double %val8,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %cmp9 = call i1 @llvm.experimental.constrained.fcmp.f64(
+                                               double %ret, double %val9,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %cmp10 = call i1 @llvm.experimental.constrained.fcmp.f64(
+                                               double %ret, double %val10,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+
+  %sel0 = select i1 %cmp0, double %ret, double 0.0
+  %sel1 = select i1 %cmp1, double %sel0, double 1.0
+  %sel2 = select i1 %cmp2, double %sel1, double 2.0
+  %sel3 = select i1 %cmp3, double %sel2, double 3.0
+  %sel4 = select i1 %cmp4, double %sel3, double 4.0
+  %sel5 = select i1 %cmp5, double %sel4, double 5.0
+  %sel6 = select i1 %cmp6, double %sel5, double 6.0
+  %sel7 = select i1 %cmp7, double %sel6, double 7.0
+  %sel8 = select i1 %cmp8, double %sel7, double 8.0
+  %sel9 = select i1 %cmp9, double %sel8, double 9.0
+  %sel10 = select i1 %cmp10, double %sel9, double 10.0
+
+  ret double %sel10
+}
+
+; Check comparison with zero.
+define i64 @f8(i64 %a, i64 %b, double %f) #0 {
+; CHECK-LABEL: f8:
+; CHECK-SCALAR: ltdbr %f0, %f0
+; CHECK-SCALAR-NEXT: ber %r14
+; CHECK-SCALAR: lgr %r2, %r3
+; CHECK-VECTOR: ltdbr %f0, %f0
+; CHECK-VECTOR-NEXT: locgrne %r2, %r3
+; CHECK: br %r14
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f64(
+                                               double %f, double 0.0,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+; Check the comparison can be reversed if that allows CDB to be used,
+define i64 @f9(i64 %a, i64 %b, double %f2, double *%ptr) #0 {
+; CHECK-LABEL: f9:
+; CHECK: cdb %f0, 0(%r4)
+; CHECK-SCALAR-NEXT: blr %r14
+; CHECK-SCALAR: lgr %r2, %r3
+; CHECK-VECTOR-NEXT: locgrnl %r2, %r3
+; CHECK: br %r14
+  %f1 = load double, double *%ptr
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f64(
+                                               double %f1, double %f2,
+                                               metadata !"ogt",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+attributes #0 = { strictfp }
+
+declare i1 @llvm.experimental.constrained.fcmp.f64(double, double, metadata, metadata)
+
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-cmp-03.ll b/llvm/test/CodeGen/SystemZ/fp-strict-cmp-03.ll
new file mode 100644
index 000000000..ae94572eb
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-cmp-03.ll
@@ -0,0 +1,47 @@
+; Test 128-bit floating-point strict comparison.  The tests assume a z10
+; implementation of select, using conditional branches rather than LOCGR.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z10 | FileCheck %s
+
+; There is no memory form of 128-bit comparison.
+define i64 @f1(i64 %a, i64 %b, fp128 *%ptr, float %f2) #0 {
+; CHECK-LABEL: f1:
+; CHECK-DAG: lxebr %f0, %f0
+; CHECK-DAG: ld %f1, 0(%r4)
+; CHECK-DAG: ld %f3, 8(%r4)
+; CHECK: cxbr %f1, %f0
+; CHECK-NEXT: ber %r14
+; CHECK: lgr %r2, %r3
+; CHECK: br %r14
+  %f2x = fpext float %f2 to fp128
+  %f1 = load fp128, fp128 *%ptr
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f128(
+                                               fp128 %f1, fp128 %f2x,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+; Check comparison with zero.
+define i64 @f2(i64 %a, i64 %b, fp128 *%ptr) #0 {
+; CHECK-LABEL: f2:
+; CHECK: ld %f0, 0(%r4)
+; CHECK: ld %f2, 8(%r4)
+; CHECK: ltxbr %f0, %f0
+; CHECK-NEXT: ber %r14
+; CHECK: lgr %r2, %r3
+; CHECK: br %r14
+  %f = load fp128, fp128 *%ptr
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f128(
+                                               fp128 %f, fp128 0xL00000000000000000000000000000000,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+attributes #0 = { strictfp }
+
+declare i1 @llvm.experimental.constrained.fcmp.f128(fp128, fp128, metadata, metadata)
+
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-cmp-04.ll b/llvm/test/CodeGen/SystemZ/fp-strict-cmp-04.ll
new file mode 100644
index 000000000..b62532ed6
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-cmp-04.ll
@@ -0,0 +1,524 @@
+; Test that floating-point strict compares are omitted if CC already has the
+; right value.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z10 \
+; RUN:   -enable-misched=0 -no-integrated-as | FileCheck %s
+;
+; We need -enable-misched=0 to make sure f12 and following routines really
+; test the compare elimination pass.
+
+
+declare float @llvm.fabs.f32(float %f)
+
+; Test addition followed by EQ, which can use the CC result of the addition.
+define float @f1(float %a, float %b, float *%dest) #0 {
+; CHECK-LABEL: f1:
+; CHECK: aebr %f0, %f2
+; CHECK-NEXT: ber %r14
+; CHECK: br %r14
+entry:
+  %res = call float @llvm.experimental.constrained.fadd.f32(
+                        float %a, float %b,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f32(
+                                               float %res, float 0.0,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  br i1 %cmp, label %exit, label %store
+
+store:
+  store float %b, float *%dest
+  br label %exit
+
+exit:
+  ret float %res
+}
+
+; ...and again with LT.
+define float @f2(float %a, float %b, float *%dest) #0 {
+; CHECK-LABEL: f2:
+; CHECK: aebr %f0, %f2
+; CHECK-NEXT: blr %r14
+; CHECK: br %r14
+entry:
+  %res = call float @llvm.experimental.constrained.fadd.f32(
+                        float %a, float %b,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f32(
+                                               float %res, float 0.0,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  br i1 %cmp, label %exit, label %store
+
+store:
+  store float %b, float *%dest
+  br label %exit
+
+exit:
+  ret float %res
+}
+
+; ...and again with GT.
+define float @f3(float %a, float %b, float *%dest) #0 {
+; CHECK-LABEL: f3:
+; CHECK: aebr %f0, %f2
+; CHECK-NEXT: bhr %r14
+; CHECK: br %r14
+entry:
+  %res = call float @llvm.experimental.constrained.fadd.f32(
+                        float %a, float %b,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f32(
+                                               float %res, float 0.0,
+                                               metadata !"ogt",
+                                               metadata !"fpexcept.strict") #0
+  br i1 %cmp, label %exit, label %store
+
+store:
+  store float %b, float *%dest
+  br label %exit
+
+exit:
+  ret float %res
+}
+
+; ...and again with UEQ.
+define float @f4(float %a, float %b, float *%dest) #0 {
+; CHECK-LABEL: f4:
+; CHECK: aebr %f0, %f2
+; CHECK-NEXT: bnlhr %r14
+; CHECK: br %r14
+entry:
+  %res = call float @llvm.experimental.constrained.fadd.f32(
+                        float %a, float %b,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f32(
+                                               float %res, float 0.0,
+                                               metadata !"ueq",
+                                               metadata !"fpexcept.strict") #0
+  br i1 %cmp, label %exit, label %store
+
+store:
+  store float %b, float *%dest
+  br label %exit
+
+exit:
+  ret float %res
+}
+
+; Subtraction also provides a zero-based CC value.
+define float @f5(float %a, float %b, float *%dest) {
+; CHECK-LABEL: f5:
+; CHECK: seb %f0, 0(%r2)
+; CHECK-NEXT: bnher %r14
+; CHECK: br %r14
+entry:
+  %cur = load float, float *%dest
+  %res = call float @llvm.experimental.constrained.fsub.f32(
+                        float %a, float %cur,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f32(
+                                               float %res, float 0.0,
+                                               metadata !"ult",
+                                               metadata !"fpexcept.strict") #0
+  br i1 %cmp, label %exit, label %store
+
+store:
+  store float %b, float *%dest
+  br label %exit
+
+exit:
+  ret float %res
+}
+
+; Test the result of LOAD POSITIVE.  We cannot omit the LTEBR.
+define float @f6(float %dummy, float %a, float *%dest) #0 {
+; CHECK-LABEL: f6:
+; CHECK: lpdfr %f0, %f2
+; CHECK-NEXT: ltebr %f0, %f0
+; CHECK-NEXT: bhr %r14
+; CHECK: br %r14
+entry:
+  %res = call float @llvm.fabs.f32(float %a)
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f32(
+                                               float %res, float 0.0,
+                                               metadata !"ogt",
+                                               metadata !"fpexcept.strict") #0
+  br i1 %cmp, label %exit, label %store
+
+store:
+  store float %res, float *%dest
+  br label %exit
+
+exit:
+  ret float %res
+}
+
+; Test the result of LOAD NEGATIVE.  We cannot omit the LTEBR.
+define float @f7(float %dummy, float %a, float *%dest) #0 {
+; CHECK-LABEL: f7:
+; CHECK: lndfr %f0, %f2
+; CHECK-NEXT: ltebr %f0, %f0
+; CHECK-NEXT: blr %r14
+; CHECK: br %r14
+entry:
+  %abs = call float @llvm.fabs.f32(float %a)
+  %res = fneg float %abs
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f32(
+                                               float %res, float 0.0,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  br i1 %cmp, label %exit, label %store
+
+store:
+  store float %res, float *%dest
+  br label %exit
+
+exit:
+  ret float %res
+}
+
+; Test the result of LOAD COMPLEMENT.  We cannot omit the LTEBR.
+define float @f8(float %dummy, float %a, float *%dest) #0 {
+; CHECK-LABEL: f8:
+; CHECK: lcdfr %f0, %f2
+; CHECK-NEXT: ltebr %f0, %f0
+; CHECK-NEXT: bler %r14
+; CHECK: br %r14
+entry:
+  %res = fneg float %a
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f32(
+                                               float %res, float 0.0,
+                                               metadata !"ole",
+                                               metadata !"fpexcept.strict") #0
+  br i1 %cmp, label %exit, label %store
+
+store:
+  store float %res, float *%dest
+  br label %exit
+
+exit:
+  ret float %res
+}
+
+; Multiplication (for example) does not modify CC.
+define float @f9(float %a, float %b, float *%dest) #0 {
+; CHECK-LABEL: f9:
+; CHECK: meebr %f0, %f2
+; CHECK-NEXT: ltebr %f0, %f0
+; CHECK-NEXT: blhr %r14
+; CHECK: br %r14
+entry:
+  %res = call float @llvm.experimental.constrained.fmul.f32(
+                        float %a, float %b,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f32(
+                                               float %res, float 0.0,
+                                               metadata !"one",
+                                               metadata !"fpexcept.strict") #0
+  br i1 %cmp, label %exit, label %store
+
+store:
+  store float %b, float *%dest
+  br label %exit
+
+exit:
+  ret float %res
+}
+
+; Test a combination involving a CC-setting instruction followed by
+; a non-CC-setting instruction.
+define float @f10(float %a, float %b, float %c, float *%dest) #0 {
+; CHECK-LABEL: f10:
+; CHECK: aebr %f0, %f2
+; CHECK-NEXT: debr %f0, %f4
+; CHECK-NEXT: ltebr %f0, %f0
+; CHECK-NEXT: bner %r14
+; CHECK: br %r14
+entry:
+  %add = call float @llvm.experimental.constrained.fadd.f32(
+                        float %a, float %b,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
+  %res = call float @llvm.experimental.constrained.fdiv.f32(
+                        float %add, float %c,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f32(
+                                               float %res, float 0.0,
+                                               metadata !"une",
+                                               metadata !"fpexcept.strict") #0
+  br i1 %cmp, label %exit, label %store
+
+store:
+  store float %b, float *%dest
+  br label %exit
+
+exit:
+  ret float %res
+}
+
+; Test a case where CC is set based on a different register from the
+; compare input.
+define float @f11(float %a, float %b, float %c, float *%dest1, float *%dest2) #0 {
+; CHECK-LABEL: f11:
+; CHECK: aebr %f0, %f2
+; CHECK-NEXT: sebr %f4, %f0
+; CHECK-DAG: ste %f4, 0(%r2)
+; CHECK-DAG: ltebr %f0, %f0
+; CHECK-NEXT: ber %r14
+; CHECK: br %r14
+entry:
+  %add = call float @llvm.experimental.constrained.fadd.f32(
+                        float %a, float %b,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
+  %sub = call float @llvm.experimental.constrained.fsub.f32(
+                        float %c, float %add,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
+  store float %sub, float *%dest1
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f32(
+                                               float %add, float 0.0,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  br i1 %cmp, label %exit, label %store
+
+store:
+  store float %sub, float *%dest2
+  br label %exit
+
+exit:
+  ret float %add
+}
+
+; Test that LER gets converted to LTEBR where useful.
+define float @f12(float %dummy, float %val) #0 {
+; CHECK-LABEL: f12:
+; CHECK: ltebr %f0, %f2
+; CHECK-NEXT: #APP
+; CHECK-NEXT: blah %f0
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: blr %r14
+; CHECK: br %r14
+entry:
+  %ret = call float asm "blah $1", "=f,{f0}"(float %val)
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f32(
+                                               float %val, float 0.0,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  br i1 %cmp, label %exit, label %store
+
+store:
+  call void asm sideeffect "blah", ""()
+  br label %exit
+
+exit:
+  ret float %ret
+}
+
+; Test that LDR gets converted to LTDBR where useful.
+define double @f13(double %dummy, double %val) #0 {
+; CHECK-LABEL: f13:
+; CHECK: ltdbr %f0, %f2
+; CHECK-NEXT: #APP
+; CHECK-NEXT: blah %f0
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: blr %r14
+; CHECK: br %r14
+entry:
+  %ret = call double asm "blah $1", "=f,{f0}"(double %val)
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f64(
+                                               double %val, double 0.0,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  br i1 %cmp, label %exit, label %store
+
+store:
+  call void asm sideeffect "blah", ""()
+  br label %exit
+
+exit:
+  ret double %ret
+}
+
+; Test that LXR gets converted to LTXBR where useful.
+define void @f14(fp128 *%ptr1, fp128 *%ptr2) #0 {
+; CHECK-LABEL: f14:
+; CHECK: ltxbr
+; CHECK-NEXT: dxbr
+; CHECK-NEXT: std
+; CHECK-NEXT: std
+; CHECK-NEXT: mxbr
+; CHECK-NEXT: std
+; CHECK-NEXT: std
+; CHECK-NEXT: blr %r14
+; CHECK: br %r14
+entry:
+  %val1 = load fp128, fp128 *%ptr1
+  %val2 = load fp128, fp128 *%ptr2
+  %div = fdiv fp128 %val1, %val2
+  store fp128 %div, fp128 *%ptr1
+  %mul = fmul fp128 %val1, %val2
+  store fp128 %mul, fp128 *%ptr2
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f128(
+                                               fp128 %val1, fp128 0xL00000000000000000000000000000000,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  br i1 %cmp, label %exit, label %store
+
+store:
+  call void asm sideeffect "blah", ""()
+  br label %exit
+
+exit:
+  ret void
+}
+
+; Test a case where it is the source rather than destination of LER that
+; we need.
+define float @f15(float %val, float %dummy) #0 {
+; CHECK-LABEL: f15:
+; CHECK: ltebr %f2, %f0
+; CHECK-NEXT: #APP
+; CHECK-NEXT: blah %f2
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: blr %r14
+; CHECK: br %r14
+entry:
+  %ret = call float asm "blah $1", "=f,{f2}"(float %val)
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f32(
+                                               float %val, float 0.0,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  br i1 %cmp, label %exit, label %store
+
+store:
+  call void asm sideeffect "blah", ""()
+  br label %exit
+
+exit:
+  ret float %ret
+}
+
+; Test a case where it is the source rather than destination of LDR that
+; we need.
+define double @f16(double %val, double %dummy) #0 {
+; CHECK-LABEL: f16:
+; CHECK: ltdbr %f2, %f0
+; CHECK-NEXT: #APP
+; CHECK-NEXT: blah %f2
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: blr %r14
+; CHECK: br %r14
+entry:
+  %ret = call double asm "blah $1", "=f,{f2}"(double %val)
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f64(
+                                               double %val, double 0.0,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  br i1 %cmp, label %exit, label %store
+
+store:
+  call void asm sideeffect "blah", ""()
+  br label %exit
+
+exit:
+  ret double %ret
+}
+
+; Repeat f2 with a comparison against -0.
+define float @f17(float %a, float %b, float *%dest) #0 {
+; CHECK-LABEL: f17:
+; CHECK: aebr %f0, %f2
+; CHECK-NEXT: blr %r14
+; CHECK: br %r14
+entry:
+  %res = call float @llvm.experimental.constrained.fadd.f32(
+                        float %a, float %b,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f32(
+                                               float %res, float -0.0,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  br i1 %cmp, label %exit, label %store
+
+store:
+  store float %b, float *%dest
+  br label %exit
+
+exit:
+  ret float %res
+}
+
+; Verify that we cannot omit the compare if there may be an intervening
+; change to the exception flags.
+define float @f18(float %a, float %b, float *%dest) #0 {
+; CHECK-LABEL: f18:
+; CHECK: aebr %f0, %f2
+; CHECK: ltebr %f0, %f0
+; CHECK-NEXT: ber %r14
+; CHECK: br %r14
+entry:
+  %res = call float @llvm.experimental.constrained.fadd.f32(
+                        float %a, float %b,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
+  call void asm sideeffect "blah", ""()
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f32(
+                                               float %res, float 0.0,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  br i1 %cmp, label %exit, label %store
+
+store:
+  store float %b, float *%dest
+  br label %exit
+
+exit:
+  ret float %res
+}
+
+; Verify that we cannot convert LER to LTEBR and omit the compare if
+; there may be an intervening change to the exception flags.
+define float @f19(float %dummy, float %val) #0 {
+; CHECK-LABEL: f19:
+; CHECK: ler %f0, %f2
+; CHECK-NEXT: #APP
+; CHECK-NEXT: blah %f0
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: ltebr %f2, %f2
+; CHECK-NEXT: blr %r14
+; CHECK: br %r14
+entry:
+  %ret = call float asm sideeffect "blah $1", "=f,{f0}"(float %val)
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f32(
+                                               float %val, float 0.0,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  br i1 %cmp, label %exit, label %store
+
+store:
+  call void asm sideeffect "blah", ""()
+  br label %exit
+
+exit:
+  ret float %ret
+}
+
+attributes #0 = { strictfp }
+
+declare float @llvm.experimental.constrained.fadd.f32(float, float, metadata, metadata)
+declare float @llvm.experimental.constrained.fsub.f32(float, float, metadata, metadata)
+declare float @llvm.experimental.constrained.fmul.f32(float, float, metadata, metadata)
+declare float @llvm.experimental.constrained.fdiv.f32(float, float, metadata, metadata)
+declare i1 @llvm.experimental.constrained.fcmp.f32(float, float, metadata, metadata)
+declare i1 @llvm.experimental.constrained.fcmp.f64(double, double, metadata, metadata)
+declare i1 @llvm.experimental.constrained.fcmp.f128(fp128, fp128, metadata, metadata)
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-cmp-05.ll b/llvm/test/CodeGen/SystemZ/fp-strict-cmp-05.ll
new file mode 100644
index 000000000..590705a55
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-cmp-05.ll
@@ -0,0 +1,103 @@
+; Test that floating-point instructions that set cc are *not* used to
+; eliminate *strict* compares for load complement, load negative and load
+; positive
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z10 | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Load complement (sign-bit flipped).
+; Test f32
+define float @f1(float %a, float %b, float %f) #0 {
+; CHECK-LABEL: f1:
+; CHECK: ltebr
+; CHECK-NEXT: ber %r14
+  %neg = fneg float %f
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f32(
+                                               float %neg, float 0.0,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, float %a, float %b
+  ret float %res
+}
+
+; Test f64
+define double @f2(double %a, double %b, double %f) #0 {
+; CHECK-LABEL: f2:
+; CHECK: ltdbr
+; CHECK-NEXT: ber %r14
+  %neg = fneg double %f
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f64(
+                                               double %neg, double 0.0,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, double %a, double %b
+  ret double %res
+}
+
+; Negation of floating-point absolute.
+; Test f32
+declare float @llvm.fabs.f32(float %f)
+define float @f3(float %a, float %b, float %f) #0 {
+; CHECK-LABEL: f3:
+; CHECK: ltebr
+; CHECK-NEXT: ber %r14
+  %abs = call float @llvm.fabs.f32(float %f)
+  %neg = fneg float %abs
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f32(
+                                               float %neg, float 0.0,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, float %a, float %b
+  ret float %res
+}
+
+; Test f64
+declare double @llvm.fabs.f64(double %f)
+define double @f4(double %a, double %b, double %f) #0 {
+; CHECK-LABEL: f4:
+; CHECK: ltdbr
+; CHECK-NEXT: ber %r14
+  %abs = call double @llvm.fabs.f64(double %f)
+  %neg = fneg double %abs
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f64(
+                                               double %neg, double 0.0,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, double %a, double %b
+  ret double %res
+}
+
+; Absolute floating-point value.
+; Test f32
+define float @f5(float %a, float %b, float %f) #0 {
+; CHECK-LABEL: f5:
+; CHECK: ltebr
+; CHECK-NEXT: ber %r14
+  %abs = call float @llvm.fabs.f32(float %f)
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f32(
+                                               float %abs, float 0.0,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, float %a, float %b
+  ret float %res
+}
+
+; Test f64
+define double @f6(double %a, double %b, double %f) #0 {
+; CHECK-LABEL: f6:
+; CHECK: ltdbr
+; CHECK-NEXT: ber %r14
+  %abs = call double @llvm.fabs.f64(double %f)
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f64(
+                                               double %abs, double 0.0,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, double %a, double %b
+  ret double %res
+}
+
+attributes #0 = { strictfp }
+
+declare i1 @llvm.experimental.constrained.fcmp.f32(float, float, metadata, metadata)
+declare i1 @llvm.experimental.constrained.fcmp.f64(double, double, metadata, metadata)
+
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-cmp-06.ll b/llvm/test/CodeGen/SystemZ/fp-strict-cmp-06.ll
new file mode 100644
index 000000000..5192ec6d1
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-cmp-06.ll
@@ -0,0 +1,44 @@
+; Test f128 strict comparisons on z14.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z14 | FileCheck %s
+
+; There is no memory form of 128-bit comparison.
+define i64 @f1(i64 %a, i64 %b, fp128 *%ptr1, fp128 *%ptr2) #0 {
+; CHECK-LABEL: f1:
+; CHECK-DAG: vl [[REG1:%v[0-9]+]], 0(%r4)
+; CHECK-DAG: vl [[REG2:%v[0-9]+]], 0(%r5)
+; CHECK: wfcxb [[REG1]], [[REG2]]
+; CHECK-NEXT: locgrne %r2, %r3
+; CHECK: br %r14
+  %f1 = load fp128, fp128 *%ptr1
+  %f2 = load fp128, fp128 *%ptr2
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f128(
+                                               fp128 %f1, fp128 %f2,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+; Check comparison with zero -- it is not worthwhile to copy to
+; FP pairs just so we can use LTXBR, so simply load up a zero.
+define i64 @f2(i64 %a, i64 %b, fp128 *%ptr) #0 {
+; CHECK-LABEL: f2:
+; CHECK-DAG: vl [[REG1:%v[0-9]+]], 0(%r4)
+; CHECK-DAG: vzero [[REG2:%v[0-9]+]]
+; CHECK: wfcxb [[REG1]], [[REG2]]
+; CHECK-NEXT: locgrne %r2, %r3
+; CHECK: br %r14
+  %f = load fp128, fp128 *%ptr
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f128(
+                                               fp128 %f, fp128 0xL00000000000000000000000000000000,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+attributes #0 = { strictfp }
+
+declare i1 @llvm.experimental.constrained.fcmp.f128(fp128, fp128, metadata, metadata)
+
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-cmps-01.ll b/llvm/test/CodeGen/SystemZ/fp-strict-cmps-01.ll
new file mode 100644
index 000000000..fa51e53a4
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-cmps-01.ll
@@ -0,0 +1,436 @@
+; Test 32-bit floating-point signaling comparison.  The tests assume a z10
+; implementation of select, using conditional branches rather than LOCGR.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z10 \
+; RUN:   | FileCheck -check-prefix=CHECK -check-prefix=CHECK-SCALAR %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z14 \
+; RUN:   | FileCheck -check-prefix=CHECK -check-prefix=CHECK-VECTOR %s
+
+declare float @foo()
+
+; Check comparison with registers.
+define i64 @f1(i64 %a, i64 %b, float %f1, float %f2) #0 {
+; CHECK-LABEL: f1:
+; CHECK: kebr %f0, %f2
+; CHECK-SCALAR-NEXT: ber %r14
+; CHECK-SCALAR: lgr %r2, %r3
+; CHECK-VECTOR-NEXT: locgrne %r2, %r3
+; CHECK: br %r14
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f32(
+                                               float %f1, float %f2,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+; Check the low end of the KEB range.
+define i64 @f2(i64 %a, i64 %b, float %f1, float *%ptr) #0 {
+; CHECK-LABEL: f2:
+; CHECK: keb %f0, 0(%r4)
+; CHECK-SCALAR-NEXT: ber %r14
+; CHECK-SCALAR: lgr %r2, %r3
+; CHECK-VECTOR-NEXT: locgrne %r2, %r3
+; CHECK: br %r14
+  %f2 = load float, float *%ptr
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f32(
+                                               float %f1, float %f2,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+; Check the high end of the aligned KEB range.
+define i64 @f3(i64 %a, i64 %b, float %f1, float *%base) #0 {
+; CHECK-LABEL: f3:
+; CHECK: keb %f0, 4092(%r4)
+; CHECK-SCALAR-NEXT: ber %r14
+; CHECK-SCALAR: lgr %r2, %r3
+; CHECK-VECTOR-NEXT: locgrne %r2, %r3
+; CHECK: br %r14
+  %ptr = getelementptr float, float *%base, i64 1023
+  %f2 = load float, float *%ptr
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f32(
+                                               float %f1, float %f2,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+; Check the next word up, which needs separate address logic.
+; Other sequences besides this one would be OK.
+define i64 @f4(i64 %a, i64 %b, float %f1, float *%base) #0 {
+; CHECK-LABEL: f4:
+; CHECK: aghi %r4, 4096
+; CHECK: keb %f0, 0(%r4)
+; CHECK-SCALAR-NEXT: ber %r14
+; CHECK-SCALAR: lgr %r2, %r3
+; CHECK-VECTOR-NEXT: locgrne %r2, %r3
+; CHECK: br %r14
+  %ptr = getelementptr float, float *%base, i64 1024
+  %f2 = load float, float *%ptr
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f32(
+                                               float %f1, float %f2,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+; Check negative displacements, which also need separate address logic.
+define i64 @f5(i64 %a, i64 %b, float %f1, float *%base) #0 {
+; CHECK-LABEL: f5:
+; CHECK: aghi %r4, -4
+; CHECK: keb %f0, 0(%r4)
+; CHECK-SCALAR-NEXT: ber %r14
+; CHECK-SCALAR: lgr %r2, %r3
+; CHECK-VECTOR-NEXT: locgrne %r2, %r3
+; CHECK: br %r14
+  %ptr = getelementptr float, float *%base, i64 -1
+  %f2 = load float, float *%ptr
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f32(
+                                               float %f1, float %f2,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+; Check that KEB allows indices.
+define i64 @f6(i64 %a, i64 %b, float %f1, float *%base, i64 %index) #0 {
+; CHECK-LABEL: f6:
+; CHECK: sllg %r1, %r5, 2
+; CHECK: keb %f0, 400(%r1,%r4)
+; CHECK-SCALAR-NEXT: ber %r14
+; CHECK-SCALAR: lgr %r2, %r3
+; CHECK-VECTOR-NEXT: locgrne %r2, %r3
+; CHECK: br %r14
+  %ptr1 = getelementptr float, float *%base, i64 %index
+  %ptr2 = getelementptr float, float *%ptr1, i64 100
+  %f2 = load float, float *%ptr2
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f32(
+                                               float %f1, float %f2,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+; Check that comparisons of spilled values can use KEB rather than KEBR.
+define float @f7(float *%ptr0) #0 {
+; CHECK-LABEL: f7:
+; CHECK: brasl %r14, foo@PLT
+; CHECK-SCALAR: keb {{%f[0-9]+}}, 16{{[04]}}(%r15)
+; CHECK: br %r14
+  %ptr1 = getelementptr float, float *%ptr0, i64 2
+  %ptr2 = getelementptr float, float *%ptr0, i64 4
+  %ptr3 = getelementptr float, float *%ptr0, i64 6
+  %ptr4 = getelementptr float, float *%ptr0, i64 8
+  %ptr5 = getelementptr float, float *%ptr0, i64 10
+  %ptr6 = getelementptr float, float *%ptr0, i64 12
+  %ptr7 = getelementptr float, float *%ptr0, i64 14
+  %ptr8 = getelementptr float, float *%ptr0, i64 16
+  %ptr9 = getelementptr float, float *%ptr0, i64 18
+  %ptr10 = getelementptr float, float *%ptr0, i64 20
+
+  %val0 = load float, float *%ptr0
+  %val1 = load float, float *%ptr1
+  %val2 = load float, float *%ptr2
+  %val3 = load float, float *%ptr3
+  %val4 = load float, float *%ptr4
+  %val5 = load float, float *%ptr5
+  %val6 = load float, float *%ptr6
+  %val7 = load float, float *%ptr7
+  %val8 = load float, float *%ptr8
+  %val9 = load float, float *%ptr9
+  %val10 = load float, float *%ptr10
+
+  %ret = call float @foo() #0
+
+  %cmp0 = call i1 @llvm.experimental.constrained.fcmps.f32(
+                                               float %ret, float %val0,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %cmp1 = call i1 @llvm.experimental.constrained.fcmps.f32(
+                                               float %ret, float %val1,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %cmp2 = call i1 @llvm.experimental.constrained.fcmps.f32(
+                                               float %ret, float %val2,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %cmp3 = call i1 @llvm.experimental.constrained.fcmps.f32(
+                                               float %ret, float %val3,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %cmp4 = call i1 @llvm.experimental.constrained.fcmps.f32(
+                                               float %ret, float %val4,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %cmp5 = call i1 @llvm.experimental.constrained.fcmps.f32(
+                                               float %ret, float %val5,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %cmp6 = call i1 @llvm.experimental.constrained.fcmps.f32(
+                                               float %ret, float %val6,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %cmp7 = call i1 @llvm.experimental.constrained.fcmps.f32(
+                                               float %ret, float %val7,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %cmp8 = call i1 @llvm.experimental.constrained.fcmps.f32(
+                                               float %ret, float %val8,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %cmp9 = call i1 @llvm.experimental.constrained.fcmps.f32(
+                                               float %ret, float %val9,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %cmp10 = call i1 @llvm.experimental.constrained.fcmps.f32(
+                                               float %ret, float %val10,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+
+  %sel0 = select i1 %cmp0, float %ret, float 0.0
+  %sel1 = select i1 %cmp1, float %sel0, float 1.0
+  %sel2 = select i1 %cmp2, float %sel1, float 2.0
+  %sel3 = select i1 %cmp3, float %sel2, float 3.0
+  %sel4 = select i1 %cmp4, float %sel3, float 4.0
+  %sel5 = select i1 %cmp5, float %sel4, float 5.0
+  %sel6 = select i1 %cmp6, float %sel5, float 6.0
+  %sel7 = select i1 %cmp7, float %sel6, float 7.0
+  %sel8 = select i1 %cmp8, float %sel7, float 8.0
+  %sel9 = select i1 %cmp9, float %sel8, float 9.0
+  %sel10 = select i1 %cmp10, float %sel9, float 10.0
+
+  ret float %sel10
+}
+
+; Check comparison with zero - cannot use LOAD AND TEST.
+define i64 @f8(i64 %a, i64 %b, float %f) #0 {
+; CHECK-LABEL: f8:
+; CHECK: lzer [[REG:%f[0-9]+]]
+; CHECK-NEXT: kebr %f0, [[REG]]
+; CHECK-SCALAR-NEXT: ber %r14
+; CHECK-SCALAR: lgr %r2, %r3
+; CHECK-VECTOR-NEXT: locgrne %r2, %r3
+; CHECK: br %r14
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f32(
+                                               float %f, float 0.0,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+; Check the comparison can be reversed if that allows KEB to be used,
+; first with oeq.
+define i64 @f9(i64 %a, i64 %b, float %f2, float *%ptr) #0 {
+; CHECK-LABEL: f9:
+; CHECK: keb %f0, 0(%r4)
+; CHECK-SCALAR-NEXT: ber %r14
+; CHECK-SCALAR: lgr %r2, %r3
+; CHECK-VECTOR-NEXT: locgrne %r2, %r3
+; CHECK: br %r14
+  %f1 = load float, float *%ptr
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f32(
+                                               float %f1, float %f2,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+; ...then one.
+define i64 @f10(i64 %a, i64 %b, float %f2, float *%ptr) #0 {
+; CHECK-LABEL: f10:
+; CHECK: keb %f0, 0(%r4)
+; CHECK-SCALAR-NEXT: blhr %r14
+; CHECK-SCALAR: lgr %r2, %r3
+; CHECK-VECTOR-NEXT: locgrnlh %r2, %r3
+; CHECK: br %r14
+  %f1 = load float, float *%ptr
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f32(
+                                               float %f1, float %f2,
+                                               metadata !"one",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+; ...then olt.
+define i64 @f11(i64 %a, i64 %b, float %f2, float *%ptr) #0 {
+; CHECK-LABEL: f11:
+; CHECK: keb %f0, 0(%r4)
+; CHECK-SCALAR-NEXT: bhr %r14
+; CHECK-SCALAR: lgr %r2, %r3
+; CHECK-VECTOR-NEXT: locgrnh %r2, %r3
+; CHECK: br %r14
+  %f1 = load float, float *%ptr
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f32(
+                                               float %f1, float %f2,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+; ...then ole.
+define i64 @f12(i64 %a, i64 %b, float %f2, float *%ptr) #0 {
+; CHECK-LABEL: f12:
+; CHECK: keb %f0, 0(%r4)
+; CHECK-SCALAR-NEXT: bher %r14
+; CHECK-SCALAR: lgr %r2, %r3
+; CHECK-VECTOR-NEXT: locgrnhe %r2, %r3
+; CHECK: br %r14
+  %f1 = load float, float *%ptr
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f32(
+                                               float %f1, float %f2,
+                                               metadata !"ole",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+; ...then oge.
+define i64 @f13(i64 %a, i64 %b, float %f2, float *%ptr) #0 {
+; CHECK-LABEL: f13:
+; CHECK: keb %f0, 0(%r4)
+; CHECK-SCALAR-NEXT: bler %r14
+; CHECK-SCALAR: lgr %r2, %r3
+; CHECK-VECTOR-NEXT: locgrnle %r2, %r3
+; CHECK: br %r14
+  %f1 = load float, float *%ptr
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f32(
+                                               float %f1, float %f2,
+                                               metadata !"oge",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+; ...then ogt.
+define i64 @f14(i64 %a, i64 %b, float %f2, float *%ptr) #0 {
+; CHECK-LABEL: f14:
+; CHECK: keb %f0, 0(%r4)
+; CHECK-SCALAR-NEXT: blr %r14
+; CHECK-SCALAR: lgr %r2, %r3
+; CHECK-VECTOR-NEXT: locgrnl %r2, %r3
+; CHECK: br %r14
+  %f1 = load float, float *%ptr
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f32(
+                                               float %f1, float %f2,
+                                               metadata !"ogt",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+; ...then ueq.
+define i64 @f15(i64 %a, i64 %b, float %f2, float *%ptr) #0 {
+; CHECK-LABEL: f15:
+; CHECK: keb %f0, 0(%r4)
+; CHECK-SCALAR-NEXT: bnlhr %r14
+; CHECK-SCALAR: lgr %r2, %r3
+; CHECK-VECTOR-NEXT: locgrlh %r2, %r3
+; CHECK: br %r14
+  %f1 = load float, float *%ptr
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f32(
+                                               float %f1, float %f2,
+                                               metadata !"ueq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+; ...then une.
+define i64 @f16(i64 %a, i64 %b, float %f2, float *%ptr) #0 {
+; CHECK-LABEL: f16:
+; CHECK: keb %f0, 0(%r4)
+; CHECK-SCALAR-NEXT: bner %r14
+; CHECK-SCALAR: lgr %r2, %r3
+; CHECK-VECTOR-NEXT: locgre %r2, %r3
+; CHECK: br %r14
+  %f1 = load float, float *%ptr
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f32(
+                                               float %f1, float %f2,
+                                               metadata !"une",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+; ...then ult.
+define i64 @f17(i64 %a, i64 %b, float %f2, float *%ptr) #0 {
+; CHECK-LABEL: f17:
+; CHECK: keb %f0, 0(%r4)
+; CHECK-SCALAR-NEXT: bnler %r14
+; CHECK-SCALAR: lgr %r2, %r3
+; CHECK-VECTOR-NEXT: locgrle %r2, %r3
+; CHECK: br %r14
+  %f1 = load float, float *%ptr
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f32(
+                                               float %f1, float %f2,
+                                               metadata !"ult",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+; ...then ule.
+define i64 @f18(i64 %a, i64 %b, float %f2, float *%ptr) #0 {
+; CHECK-LABEL: f18:
+; CHECK: keb %f0, 0(%r4)
+; CHECK-SCALAR-NEXT: bnlr %r14
+; CHECK-SCALAR: lgr %r2, %r3
+; CHECK-VECTOR-NEXT: locgrl %r2, %r3
+; CHECK: br %r14
+  %f1 = load float, float *%ptr
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f32(
+                                               float %f1, float %f2,
+                                               metadata !"ule",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+; ...then uge.
+define i64 @f19(i64 %a, i64 %b, float %f2, float *%ptr) #0 {
+; CHECK-LABEL: f19:
+; CHECK: keb %f0, 0(%r4)
+; CHECK-SCALAR-NEXT: bnhr %r14
+; CHECK-SCALAR: lgr %r2, %r3
+; CHECK-VECTOR-NEXT: locgrh %r2, %r3
+; CHECK: br %r14
+  %f1 = load float, float *%ptr
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f32(
+                                               float %f1, float %f2,
+                                               metadata !"uge",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+; ...then ugt.
+define i64 @f20(i64 %a, i64 %b, float %f2, float *%ptr) #0 {
+; CHECK-LABEL: f20:
+; CHECK: keb %f0, 0(%r4)
+; CHECK-SCALAR-NEXT: bnher %r14
+; CHECK-SCALAR: lgr %r2, %r3
+; CHECK-VECTOR-NEXT: locgrhe %r2, %r3
+; CHECK: br %r14
+  %f1 = load float, float *%ptr
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f32(
+                                               float %f1, float %f2,
+                                               metadata !"ugt",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+attributes #0 = { strictfp }
+
+declare i1 @llvm.experimental.constrained.fcmps.f32(float, float, metadata, metadata)
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-cmps-02.ll b/llvm/test/CodeGen/SystemZ/fp-strict-cmps-02.ll
new file mode 100644
index 000000000..4d4b66b4f
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-cmps-02.ll
@@ -0,0 +1,249 @@
+; Test 64-bit floating-point signaling comparison.  The tests assume a z10
+; implementation of select, using conditional branches rather than LOCGR.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z10 \
+; RUN:   | FileCheck -check-prefix=CHECK -check-prefix=CHECK-SCALAR %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 -verify-machineinstrs\
+; RUN:   | FileCheck -check-prefix=CHECK -check-prefix=CHECK-VECTOR %s
+
+declare double @foo()
+
+; Check comparison with registers.
+define i64 @f1(i64 %a, i64 %b, double %f1, double %f2) #0 {
+; CHECK-LABEL: f1:
+; CHECK: kdbr %f0, %f2
+; CHECK-SCALAR-NEXT: ber %r14
+; CHECK-SCALAR: lgr %r2, %r3
+; CHECK-VECTOR-NEXT: locgrne %r2, %r3
+; CHECK: br %r14
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f64(
+                                               double %f1, double %f2,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+; Check the low end of the KDB range.
+define i64 @f2(i64 %a, i64 %b, double %f1, double *%ptr) #0 {
+; CHECK-LABEL: f2:
+; CHECK: kdb %f0, 0(%r4)
+; CHECK-SCALAR-NEXT: ber %r14
+; CHECK-SCALAR: lgr %r2, %r3
+; CHECK-VECTOR-NEXT: locgrne %r2, %r3
+; CHECK: br %r14
+  %f2 = load double, double *%ptr
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f64(
+                                               double %f1, double %f2,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+; Check the high end of the aligned KDB range.
+define i64 @f3(i64 %a, i64 %b, double %f1, double *%base) #0 {
+; CHECK-LABEL: f3:
+; CHECK: kdb %f0, 4088(%r4)
+; CHECK-SCALAR-NEXT: ber %r14
+; CHECK-SCALAR: lgr %r2, %r3
+; CHECK-VECTOR-NEXT: locgrne %r2, %r3
+; CHECK: br %r14
+  %ptr = getelementptr double, double *%base, i64 511
+  %f2 = load double, double *%ptr
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f64(
+                                               double %f1, double %f2,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+; Check the next doubleword up, which needs separate address logic.
+; Other sequences besides this one would be OK.
+define i64 @f4(i64 %a, i64 %b, double %f1, double *%base) #0 {
+; CHECK-LABEL: f4:
+; CHECK: aghi %r4, 4096
+; CHECK: kdb %f0, 0(%r4)
+; CHECK-SCALAR-NEXT: ber %r14
+; CHECK-SCALAR: lgr %r2, %r3
+; CHECK-VECTOR-NEXT: locgrne %r2, %r3
+; CHECK: br %r14
+  %ptr = getelementptr double, double *%base, i64 512
+  %f2 = load double, double *%ptr
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f64(
+                                               double %f1, double %f2,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+; Check negative displacements, which also need separate address logic.
+define i64 @f5(i64 %a, i64 %b, double %f1, double *%base) #0 {
+; CHECK-LABEL: f5:
+; CHECK: aghi %r4, -8
+; CHECK: kdb %f0, 0(%r4)
+; CHECK-SCALAR-NEXT: ber %r14
+; CHECK-SCALAR: lgr %r2, %r3
+; CHECK-VECTOR-NEXT: locgrne %r2, %r3
+; CHECK: br %r14
+  %ptr = getelementptr double, double *%base, i64 -1
+  %f2 = load double, double *%ptr
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f64(
+                                               double %f1, double %f2,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+; Check that KDB allows indices.
+define i64 @f6(i64 %a, i64 %b, double %f1, double *%base, i64 %index) #0 {
+; CHECK-LABEL: f6:
+; CHECK: sllg %r1, %r5, 3
+; CHECK: kdb %f0, 800(%r1,%r4)
+; CHECK-SCALAR-NEXT: ber %r14
+; CHECK-SCALAR: lgr %r2, %r3
+; CHECK-VECTOR-NEXT: locgrne %r2, %r3
+; CHECK: br %r14
+  %ptr1 = getelementptr double, double *%base, i64 %index
+  %ptr2 = getelementptr double, double *%ptr1, i64 100
+  %f2 = load double, double *%ptr2
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f64(
+                                               double %f1, double %f2,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+; Check that comparisons of spilled values can use KDB rather than KDBR.
+define double @f7(double *%ptr0) #0 {
+; CHECK-LABEL: f7:
+; CHECK: brasl %r14, foo@PLT
+; CHECK-SCALAR: kdb {{%f[0-9]+}}, 160(%r15)
+; CHECK: br %r14
+  %ptr1 = getelementptr double, double *%ptr0, i64 2
+  %ptr2 = getelementptr double, double *%ptr0, i64 4
+  %ptr3 = getelementptr double, double *%ptr0, i64 6
+  %ptr4 = getelementptr double, double *%ptr0, i64 8
+  %ptr5 = getelementptr double, double *%ptr0, i64 10
+  %ptr6 = getelementptr double, double *%ptr0, i64 12
+  %ptr7 = getelementptr double, double *%ptr0, i64 14
+  %ptr8 = getelementptr double, double *%ptr0, i64 16
+  %ptr9 = getelementptr double, double *%ptr0, i64 18
+  %ptr10 = getelementptr double, double *%ptr0, i64 20
+
+  %val0 = load double, double *%ptr0
+  %val1 = load double, double *%ptr1
+  %val2 = load double, double *%ptr2
+  %val3 = load double, double *%ptr3
+  %val4 = load double, double *%ptr4
+  %val5 = load double, double *%ptr5
+  %val6 = load double, double *%ptr6
+  %val7 = load double, double *%ptr7
+  %val8 = load double, double *%ptr8
+  %val9 = load double, double *%ptr9
+  %val10 = load double, double *%ptr10
+
+  %ret = call double @foo() #0
+
+  %cmp0 = call i1 @llvm.experimental.constrained.fcmps.f64(
+                                               double %ret, double %val0,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %cmp1 = call i1 @llvm.experimental.constrained.fcmps.f64(
+                                               double %ret, double %val1,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %cmp2 = call i1 @llvm.experimental.constrained.fcmps.f64(
+                                               double %ret, double %val2,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %cmp3 = call i1 @llvm.experimental.constrained.fcmps.f64(
+                                               double %ret, double %val3,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %cmp4 = call i1 @llvm.experimental.constrained.fcmps.f64(
+                                               double %ret, double %val4,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %cmp5 = call i1 @llvm.experimental.constrained.fcmps.f64(
+                                               double %ret, double %val5,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %cmp6 = call i1 @llvm.experimental.constrained.fcmps.f64(
+                                               double %ret, double %val6,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %cmp7 = call i1 @llvm.experimental.constrained.fcmps.f64(
+                                               double %ret, double %val7,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %cmp8 = call i1 @llvm.experimental.constrained.fcmps.f64(
+                                               double %ret, double %val8,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %cmp9 = call i1 @llvm.experimental.constrained.fcmps.f64(
+                                               double %ret, double %val9,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %cmp10 = call i1 @llvm.experimental.constrained.fcmps.f64(
+                                               double %ret, double %val10,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+
+  %sel0 = select i1 %cmp0, double %ret, double 0.0
+  %sel1 = select i1 %cmp1, double %sel0, double 1.0
+  %sel2 = select i1 %cmp2, double %sel1, double 2.0
+  %sel3 = select i1 %cmp3, double %sel2, double 3.0
+  %sel4 = select i1 %cmp4, double %sel3, double 4.0
+  %sel5 = select i1 %cmp5, double %sel4, double 5.0
+  %sel6 = select i1 %cmp6, double %sel5, double 6.0
+  %sel7 = select i1 %cmp7, double %sel6, double 7.0
+  %sel8 = select i1 %cmp8, double %sel7, double 8.0
+  %sel9 = select i1 %cmp9, double %sel8, double 9.0
+  %sel10 = select i1 %cmp10, double %sel9, double 10.0
+
+  ret double %sel10
+}
+
+; Check comparison with zero - cannot use LOAD AND TEST.
+define i64 @f8(i64 %a, i64 %b, double %f) #0 {
+; CHECK-LABEL: f8:
+; CHECK: lzdr [[REG:%f[0-9]+]]
+; CHECK-NEXT: kdbr %f0, [[REG]]
+; CHECK-SCALAR-NEXT: ber %r14
+; CHECK-SCALAR: lgr %r2, %r3
+; CHECK-VECTOR-NEXT: locgrne %r2, %r3
+; CHECK: br %r14
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f64(
+                                               double %f, double 0.0,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+; Check the comparison can be reversed if that allows KDB to be used,
+define i64 @f9(i64 %a, i64 %b, double %f2, double *%ptr) #0 {
+; CHECK-LABEL: f9:
+; CHECK: kdb %f0, 0(%r4)
+; CHECK-SCALAR-NEXT: blr %r14
+; CHECK-SCALAR: lgr %r2, %r3
+; CHECK-VECTOR-NEXT: locgrnl %r2, %r3
+; CHECK: br %r14
+  %f1 = load double, double *%ptr
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f64(
+                                               double %f1, double %f2,
+                                               metadata !"ogt",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+attributes #0 = { strictfp }
+
+declare i1 @llvm.experimental.constrained.fcmps.f64(double, double, metadata, metadata)
+
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-cmps-03.ll b/llvm/test/CodeGen/SystemZ/fp-strict-cmps-03.ll
new file mode 100644
index 000000000..4c571b619
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-cmps-03.ll
@@ -0,0 +1,48 @@
+; Test 128-bit floating-point signaling comparison.  The tests assume a z10
+; implementation of select, using conditional branches rather than LOCGR.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z10 | FileCheck %s
+
+; There is no memory form of 128-bit comparison.
+define i64 @f1(i64 %a, i64 %b, fp128 *%ptr, float %f2) #0 {
+; CHECK-LABEL: f1:
+; CHECK-DAG: lxebr %f0, %f0
+; CHECK-DAG: ld %f1, 0(%r4)
+; CHECK-DAG: ld %f3, 8(%r4)
+; CHECK: kxbr %f1, %f0
+; CHECK-NEXT: ber %r14
+; CHECK: lgr %r2, %r3
+; CHECK: br %r14
+  %f2x = fpext float %f2 to fp128
+  %f1 = load fp128, fp128 *%ptr
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f128(
+                                               fp128 %f1, fp128 %f2x,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+; Check comparison with zero - cannot use LOAD AND TEST.
+define i64 @f2(i64 %a, i64 %b, fp128 *%ptr) #0 {
+; CHECK-LABEL: f2:
+; CHECK-DAG: ld %f0, 0(%r4)
+; CHECK-DAG: ld %f2, 8(%r4)
+; CHECK-DAG: lzxr [[REG:%f[0-9]+]]
+; CHECK-NEXT: kxbr %f0, [[REG]]
+; CHECK-NEXT: ber %r14
+; CHECK: lgr %r2, %r3
+; CHECK: br %r14
+  %f = load fp128, fp128 *%ptr
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f128(
+                                               fp128 %f, fp128 0xL00000000000000000000000000000000,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+attributes #0 = { strictfp }
+
+declare i1 @llvm.experimental.constrained.fcmps.f128(fp128, fp128, metadata, metadata)
+
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-cmps-04.ll b/llvm/test/CodeGen/SystemZ/fp-strict-cmps-04.ll
new file mode 100644
index 000000000..47c0d5caa
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-cmps-04.ll
@@ -0,0 +1,148 @@
+; Verify that floating-point strict signaling compares cannot be omitted
+; even if CC already has the right value.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z10 \
+; RUN:   -enable-misched=0 -no-integrated-as | FileCheck %s
+;
+; We need -enable-misched=0 to make sure f12 and following routines really
+; test the compare elimination pass.
+
+
+declare float @llvm.fabs.f32(float %f)
+
+; Test addition followed by EQ, which could use the CC result of the addition.
+define float @f1(float %a, float %b, float *%dest) #0 {
+; CHECK-LABEL: f1:
+; CHECK-DAG: aebr %f0, %f2
+; CHECK-DAG: lzer [[REG:%f[0-9]+]]
+; CHECK-NEXT: kebr %f0, [[REG]]
+; CHECK-NEXT: ber %r14
+; CHECK: br %r14
+entry:
+  %res = call float @llvm.experimental.constrained.fadd.f32(
+                        float %a, float %b,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
+  %cmp = call i1 @llvm.experimental.constrained.fcmps.f32(
+                                               float %res, float 0.0,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  br i1 %cmp, label %exit, label %store
+
+store:
+  store float %b, float *%dest
+  br label %exit
+
+exit:
+  ret float %res
+}
+
+; Test the result of LOAD POSITIVE.
+define float @f6(float %dummy, float %a, float *%dest) #0 {
+; CHECK-LABEL: f6:
+; CHECK-DAG: lpdfr %f0, %f2
+; CHECK-DAG: lzer [[REG:%f[0-9]+]]
+; CHECK-NEXT: kebr %f0, [[REG]]
+; CHECK-NEXT: bhr %r14
+; CHECK: br %r14
+entry:
+  %res = call float @llvm.fabs.f32(float %a)
+  %cmp = call i1 @llvm.experimental.constrained.fcmps.f32(
+                                               float %res, float 0.0,
+                                               metadata !"ogt",
+                                               metadata !"fpexcept.strict") #0
+  br i1 %cmp, label %exit, label %store
+
+store:
+  store float %res, float *%dest
+  br label %exit
+
+exit:
+  ret float %res
+}
+
+; Test the result of LOAD NEGATIVE.
+define float @f7(float %dummy, float %a, float *%dest) #0 {
+; CHECK-LABEL: f7:
+; CHECK-DAG: lndfr %f0, %f2
+; CHECK-DAG: lzer [[REG:%f[0-9]+]]
+; CHECK-NEXT: kebr %f0, [[REG]]
+; CHECK-NEXT: blr %r14
+; CHECK: br %r14
+entry:
+  %abs = call float @llvm.fabs.f32(float %a)
+  %res = fneg float %abs
+  %cmp = call i1 @llvm.experimental.constrained.fcmps.f32(
+                                               float %res, float 0.0,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  br i1 %cmp, label %exit, label %store
+
+store:
+  store float %res, float *%dest
+  br label %exit
+
+exit:
+  ret float %res
+}
+
+; Test the result of LOAD COMPLEMENT.
+define float @f8(float %dummy, float %a, float *%dest) #0 {
+; CHECK-LABEL: f8:
+; CHECK-DAG: lcdfr %f0, %f2
+; CHECK-DAG: lzer [[REG:%f[0-9]+]]
+; CHECK-NEXT: kebr %f0, [[REG]]
+; CHECK-NEXT: bler %r14
+; CHECK: br %r14
+entry:
+  %res = fneg float %a
+  %cmp = call i1 @llvm.experimental.constrained.fcmps.f32(
+                                               float %res, float 0.0,
+                                               metadata !"ole",
+                                               metadata !"fpexcept.strict") #0
+  br i1 %cmp, label %exit, label %store
+
+store:
+  store float %res, float *%dest
+  br label %exit
+
+exit:
+  ret float %res
+}
+
+; Test that LER does not get converted to LTEBR.
+define float @f12(float %dummy, float %val) #0 {
+; CHECK-LABEL: f12:
+; CHECK: ler %f0, %f2
+; CHECK-NEXT: #APP
+; CHECK-NEXT: blah %f0
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: lzer [[REG:%f[0-9]+]]
+; CHECK-NEXT: kebr %f2, [[REG]]
+; CHECK-NEXT: blr %r14
+; CHECK: br %r14
+entry:
+  %ret = call float asm "blah $1", "=f,{f0}"(float %val)
+  %cmp = call i1 @llvm.experimental.constrained.fcmps.f32(
+                                               float %val, float 0.0,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  br i1 %cmp, label %exit, label %store
+
+store:
+  call void asm sideeffect "blah", ""()
+  br label %exit
+
+exit:
+  ret float %ret
+}
+
+attributes #0 = { strictfp }
+
+declare float @llvm.experimental.constrained.fadd.f32(float, float, metadata, metadata)
+declare float @llvm.experimental.constrained.fsub.f32(float, float, metadata, metadata)
+declare float @llvm.experimental.constrained.fmul.f32(float, float, metadata, metadata)
+declare float @llvm.experimental.constrained.fdiv.f32(float, float, metadata, metadata)
+declare i1 @llvm.experimental.constrained.fcmps.f32(float, float, metadata, metadata)
+declare i1 @llvm.experimental.constrained.fcmps.f64(double, double, metadata, metadata)
+declare i1 @llvm.experimental.constrained.fcmps.f128(fp128, fp128, metadata, metadata)
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-cmps-05.ll b/llvm/test/CodeGen/SystemZ/fp-strict-cmps-05.ll
new file mode 100644
index 000000000..ac677b296
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-cmps-05.ll
@@ -0,0 +1,103 @@
+; Test that floating-point instructions that set cc are *not* used to
+; eliminate *strict* signaling compares for load complement, load negative
+; and load positive
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z10 | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Load complement (sign-bit flipped).
+; Test f32
+define float @f1(float %a, float %b, float %f) #0 {
+; CHECK-LABEL: f1:
+; CHECK: kebr
+; CHECK-NEXT: ber %r14
+  %neg = fneg float %f
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f32(
+                                               float %neg, float 0.0,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, float %a, float %b
+  ret float %res
+}
+
+; Test f64
+define double @f2(double %a, double %b, double %f) #0 {
+; CHECK-LABEL: f2:
+; CHECK: kdbr
+; CHECK-NEXT: ber %r14
+  %neg = fneg double %f
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f64(
+                                               double %neg, double 0.0,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, double %a, double %b
+  ret double %res
+}
+
+; Negation of floating-point absolute.
+; Test f32
+declare float @llvm.fabs.f32(float %f)
+define float @f3(float %a, float %b, float %f) #0 {
+; CHECK-LABEL: f3:
+; CHECK: kebr
+; CHECK-NEXT: ber %r14
+  %abs = call float @llvm.fabs.f32(float %f)
+  %neg = fneg float %abs
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f32(
+                                               float %neg, float 0.0,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, float %a, float %b
+  ret float %res
+}
+
+; Test f64
+declare double @llvm.fabs.f64(double %f)
+define double @f4(double %a, double %b, double %f) #0 {
+; CHECK-LABEL: f4:
+; CHECK: kdbr
+; CHECK-NEXT: ber %r14
+  %abs = call double @llvm.fabs.f64(double %f)
+  %neg = fneg double %abs
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f64(
+                                               double %neg, double 0.0,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, double %a, double %b
+  ret double %res
+}
+
+; Absolute floating-point value.
+; Test f32
+define float @f5(float %a, float %b, float %f) #0 {
+; CHECK-LABEL: f5:
+; CHECK: kebr
+; CHECK-NEXT: ber %r14
+  %abs = call float @llvm.fabs.f32(float %f)
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f32(
+                                               float %abs, float 0.0,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, float %a, float %b
+  ret float %res
+}
+
+; Test f64
+define double @f6(double %a, double %b, double %f) #0 {
+; CHECK-LABEL: f6:
+; CHECK: kdbr
+; CHECK-NEXT: ber %r14
+  %abs = call double @llvm.fabs.f64(double %f)
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f64(
+                                               double %abs, double 0.0,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, double %a, double %b
+  ret double %res
+}
+
+attributes #0 = { strictfp }
+
+declare i1 @llvm.experimental.constrained.fcmps.f32(float, float, metadata, metadata)
+declare i1 @llvm.experimental.constrained.fcmps.f64(double, double, metadata, metadata)
+
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-cmps-06.ll b/llvm/test/CodeGen/SystemZ/fp-strict-cmps-06.ll
new file mode 100644
index 000000000..37c18d836
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-cmps-06.ll
@@ -0,0 +1,44 @@
+; Test f128 signaling comparisons on z14.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z14 | FileCheck %s
+
+; There is no memory form of 128-bit comparison.
+define i64 @f1(i64 %a, i64 %b, fp128 *%ptr1, fp128 *%ptr2) #0 {
+; CHECK-LABEL: f1:
+; CHECK-DAG: vl [[REG1:%v[0-9]+]], 0(%r4)
+; CHECK-DAG: vl [[REG2:%v[0-9]+]], 0(%r5)
+; CHECK: wfkxb [[REG1]], [[REG2]]
+; CHECK-NEXT: locgrne %r2, %r3
+; CHECK: br %r14
+  %f1 = load fp128, fp128 *%ptr1
+  %f2 = load fp128, fp128 *%ptr2
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f128(
+                                               fp128 %f1, fp128 %f2,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+; Check comparison with zero -- it is not worthwhile to copy to
+; FP pairs just so we can use LTXBR, so simply load up a zero.
+define i64 @f2(i64 %a, i64 %b, fp128 *%ptr) #0 {
+; CHECK-LABEL: f2:
+; CHECK-DAG: vl [[REG1:%v[0-9]+]], 0(%r4)
+; CHECK-DAG: vzero [[REG2:%v[0-9]+]]
+; CHECK: wfkxb [[REG1]], [[REG2]]
+; CHECK-NEXT: locgrne %r2, %r3
+; CHECK: br %r14
+  %f = load fp128, fp128 *%ptr
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f128(
+                                               fp128 %f, fp128 0xL00000000000000000000000000000000,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+attributes #0 = { strictfp }
+
+declare i1 @llvm.experimental.constrained.fcmps.f128(fp128, fp128, metadata, metadata)
+
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-conv-01.ll b/llvm/test/CodeGen/SystemZ/fp-strict-conv-01.ll
index b20ab7180..45dc51ea5 100644
--- a/llvm/test/CodeGen/SystemZ/fp-strict-conv-01.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-conv-01.ll
@@ -13,7 +13,7 @@ declare float @llvm.experimental.constrained.fadd.f32(float, float, metadata, me
 declare double @llvm.experimental.constrained.fadd.f64(double, double, metadata, metadata)
 
 ; Test f64->f32.
-define float @f1(double %d1, double %d2) {
+define float @f1(double %d1, double %d2) #0 {
 ; CHECK-LABEL: f1:
 ; CHECK-SCALAR: ledbr %f0, %f2
 ; CHECK-VECTOR: ledbra %f0, 0, %f2, 0
@@ -21,12 +21,12 @@ define float @f1(double %d1, double %d2) {
   %res = call float @llvm.experimental.constrained.fptrunc.f32.f64(
                                                double %d2,
                                                metadata !"round.dynamic",
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
   ret float %res
 }
 
 ; Test f128->f32.
-define float @f2(fp128 *%ptr) {
+define float @f2(fp128 *%ptr) #0 {
 ; CHECK-LABEL: f2:
 ; CHECK: lexbr %f0, %f0
 ; CHECK: br %r14
@@ -34,13 +34,13 @@ define float @f2(fp128 *%ptr) {
   %res = call float @llvm.experimental.constrained.fptrunc.f32.f128(
                                                fp128 %val,
                                                metadata !"round.dynamic",
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
   ret float %res
 }
 
 ; Make sure that we don't use %f0 as the destination of LEXBR when %f2
 ; is still live.
-define void @f3(float *%dst, fp128 *%ptr, float %d1, float %d2) {
+define void @f3(float *%dst, fp128 *%ptr, float %d1, float %d2) #0 {
 ; CHECK-LABEL: f3:
 ; CHECK: lexbr %f1, %f1
 ; CHECK: aebr %f1, %f2
@@ -50,17 +50,17 @@ define void @f3(float *%dst, fp128 *%ptr, float %d1, float %d2) {
   %conv = call float @llvm.experimental.constrained.fptrunc.f32.f128(
                                                fp128 %val,
                                                metadata !"round.dynamic",
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
   %res = call float @llvm.experimental.constrained.fadd.f32(
                         float %conv, float %d2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   store float %res, float *%dst
   ret void
 }
 
 ; Test f128->f64.
-define double @f4(fp128 *%ptr) {
+define double @f4(fp128 *%ptr) #0 {
 ; CHECK-LABEL: f4:
 ; CHECK: ldxbr %f0, %f0
 ; CHECK: br %r14
@@ -68,12 +68,12 @@ define double @f4(fp128 *%ptr) {
   %res = call double @llvm.experimental.constrained.fptrunc.f64.f128(
                                                fp128 %val,
                                                metadata !"round.dynamic",
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
   ret double %res
 }
 
 ; Like f3, but for f128->f64.
-define void @f5(double *%dst, fp128 *%ptr, double %d1, double %d2) {
+define void @f5(double *%dst, fp128 *%ptr, double %d1, double %d2) #0 {
 ; CHECK-LABEL: f5:
 ; CHECK: ldxbr %f1, %f1
 ; CHECK-SCALAR: adbr %f1, %f2
@@ -85,11 +85,13 @@ define void @f5(double *%dst, fp128 *%ptr, double %d1, double %d2) {
   %conv = call double @llvm.experimental.constrained.fptrunc.f64.f128(
                                                fp128 %val,
                                                metadata !"round.dynamic",
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
   %res = call double @llvm.experimental.constrained.fadd.f64(
                         double %conv, double %d2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   store double %res, double *%dst
   ret void
 }
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-conv-02.ll b/llvm/test/CodeGen/SystemZ/fp-strict-conv-02.ll
index 0f24b91e2..4cada62d0 100644
--- a/llvm/test/CodeGen/SystemZ/fp-strict-conv-02.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-conv-02.ll
@@ -5,41 +5,41 @@
 declare double @llvm.experimental.constrained.fpext.f64.f32(float, metadata)
 
 ; Check register extension.
-define double @f1(float %val) {
+define double @f1(float %val) #0 {
 ; CHECK-LABEL: f1:
 ; CHECK: ldebr %f0, %f0
 ; CHECK: br %r14
   %res = call double @llvm.experimental.constrained.fpext.f64.f32(float %val,
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
   ret double %res
 }
 
 ; Check the low end of the LDEB range.
-define double @f2(float *%ptr) {
+define double @f2(float *%ptr) #0 {
 ; CHECK-LABEL: f2:
 ; CHECK: ldeb %f0, 0(%r2)
 ; CHECK: br %r14
   %val = load float, float *%ptr
   %res = call double @llvm.experimental.constrained.fpext.f64.f32(float %val,
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
   ret double %res
 }
 
 ; Check the high end of the aligned LDEB range.
-define double @f3(float *%base) {
+define double @f3(float *%base) #0 {
 ; CHECK-LABEL: f3:
 ; CHECK: ldeb %f0, 4092(%r2)
 ; CHECK: br %r14
   %ptr = getelementptr float, float *%base, i64 1023
   %val = load float, float *%ptr
   %res = call double @llvm.experimental.constrained.fpext.f64.f32(float %val,
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
   ret double %res
 }
 
 ; Check the next word up, which needs separate address logic.
 ; Other sequences besides this one would be OK.
-define double @f4(float *%base) {
+define double @f4(float *%base) #0 {
 ; CHECK-LABEL: f4:
 ; CHECK: aghi %r2, 4096
 ; CHECK: ldeb %f0, 0(%r2)
@@ -47,12 +47,12 @@ define double @f4(float *%base) {
   %ptr = getelementptr float, float *%base, i64 1024
   %val = load float, float *%ptr
   %res = call double @llvm.experimental.constrained.fpext.f64.f32(float %val,
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
   ret double %res
 }
 
 ; Check negative displacements, which also need separate address logic.
-define double @f5(float *%base) {
+define double @f5(float *%base) #0 {
 ; CHECK-LABEL: f5:
 ; CHECK: aghi %r2, -4
 ; CHECK: ldeb %f0, 0(%r2)
@@ -60,12 +60,12 @@ define double @f5(float *%base) {
   %ptr = getelementptr float, float *%base, i64 -1
   %val = load float, float *%ptr
   %res = call double @llvm.experimental.constrained.fpext.f64.f32(float %val,
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
   ret double %res
 }
 
 ; Check that LDEB allows indices.
-define double @f6(float *%base, i64 %index) {
+define double @f6(float *%base, i64 %index) #0 {
 ; CHECK-LABEL: f6:
 ; CHECK: sllg %r1, %r3, 2
 ; CHECK: ldeb %f0, 400(%r1,%r2)
@@ -74,7 +74,8 @@ define double @f6(float *%base, i64 %index) {
   %ptr2 = getelementptr float, float *%ptr1, i64 100
   %val = load float, float *%ptr2
   %res = call double @llvm.experimental.constrained.fpext.f64.f32(float %val,
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
   ret double %res
 }
 
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-conv-03.ll b/llvm/test/CodeGen/SystemZ/fp-strict-conv-03.ll
index b3fbac975..7a8a7a88e 100644
--- a/llvm/test/CodeGen/SystemZ/fp-strict-conv-03.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-conv-03.ll
@@ -5,20 +5,20 @@
 declare fp128 @llvm.experimental.constrained.fpext.f128.f32(float, metadata)
 
 ; Check register extension.
-define void @f1(fp128 *%dst, float %val) {
+define void @f1(fp128 *%dst, float %val) #0 {
 ; CHECK-LABEL: f1:
 ; CHECK: lxebr %f0, %f0
 ; CHECK: std %f0, 0(%r2)
 ; CHECK: std %f2, 8(%r2)
 ; CHECK: br %r14
   %res = call fp128 @llvm.experimental.constrained.fpext.f128.f32(float %val,
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
   store fp128 %res, fp128 *%dst
   ret void
 }
 
 ; Check the low end of the LXEB range.
-define void @f2(fp128 *%dst, float *%ptr) {
+define void @f2(fp128 *%dst, float *%ptr) #0 {
 ; CHECK-LABEL: f2:
 ; CHECK: lxeb %f0, 0(%r3)
 ; CHECK: std %f0, 0(%r2)
@@ -26,13 +26,13 @@ define void @f2(fp128 *%dst, float *%ptr) {
 ; CHECK: br %r14
   %val = load float, float *%ptr
   %res = call fp128 @llvm.experimental.constrained.fpext.f128.f32(float %val,
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
   store fp128 %res, fp128 *%dst
   ret void
 }
 
 ; Check the high end of the aligned LXEB range.
-define void @f3(fp128 *%dst, float *%base) {
+define void @f3(fp128 *%dst, float *%base) #0 {
 ; CHECK-LABEL: f3:
 ; CHECK: lxeb %f0, 4092(%r3)
 ; CHECK: std %f0, 0(%r2)
@@ -41,14 +41,14 @@ define void @f3(fp128 *%dst, float *%base) {
   %ptr = getelementptr float, float *%base, i64 1023
   %val = load float, float *%ptr
   %res = call fp128 @llvm.experimental.constrained.fpext.f128.f32(float %val,
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
   store fp128 %res, fp128 *%dst
   ret void
 }
 
 ; Check the next word up, which needs separate address logic.
 ; Other sequences besides this one would be OK.
-define void @f4(fp128 *%dst, float *%base) {
+define void @f4(fp128 *%dst, float *%base) #0 {
 ; CHECK-LABEL: f4:
 ; CHECK: aghi %r3, 4096
 ; CHECK: lxeb %f0, 0(%r3)
@@ -58,13 +58,13 @@ define void @f4(fp128 *%dst, float *%base) {
   %ptr = getelementptr float, float *%base, i64 1024
   %val = load float, float *%ptr
   %res = call fp128 @llvm.experimental.constrained.fpext.f128.f32(float %val,
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
   store fp128 %res, fp128 *%dst
   ret void
 }
 
 ; Check negative displacements, which also need separate address logic.
-define void @f5(fp128 *%dst, float *%base) {
+define void @f5(fp128 *%dst, float *%base) #0 {
 ; CHECK-LABEL: f5:
 ; CHECK: aghi %r3, -4
 ; CHECK: lxeb %f0, 0(%r3)
@@ -74,13 +74,13 @@ define void @f5(fp128 *%dst, float *%base) {
   %ptr = getelementptr float, float *%base, i64 -1
   %val = load float, float *%ptr
   %res = call fp128 @llvm.experimental.constrained.fpext.f128.f32(float %val,
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
   store fp128 %res, fp128 *%dst
   ret void
 }
 
 ; Check that LXEB allows indices.
-define void @f6(fp128 *%dst, float *%base, i64 %index) {
+define void @f6(fp128 *%dst, float *%base, i64 %index) #0 {
 ; CHECK-LABEL: f6:
 ; CHECK: sllg %r1, %r4, 2
 ; CHECK: lxeb %f0, 400(%r1,%r3)
@@ -91,8 +91,9 @@ define void @f6(fp128 *%dst, float *%base, i64 %index) {
   %ptr2 = getelementptr float, float *%ptr1, i64 100
   %val = load float, float *%ptr2
   %res = call fp128 @llvm.experimental.constrained.fpext.f128.f32(float %val,
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
   store fp128 %res, fp128 *%dst
   ret void
 }
 
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-conv-04.ll b/llvm/test/CodeGen/SystemZ/fp-strict-conv-04.ll
index 657cdcdfd..4ddfe1031 100644
--- a/llvm/test/CodeGen/SystemZ/fp-strict-conv-04.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-conv-04.ll
@@ -5,20 +5,20 @@
 declare fp128 @llvm.experimental.constrained.fpext.f128.f64(double, metadata)
 
 ; Check register extension.
-define void @f1(fp128 *%dst, double %val) {
+define void @f1(fp128 *%dst, double %val) #0 {
 ; CHECK-LABEL: f1:
 ; CHECK: lxdbr %f0, %f0
 ; CHECK: std %f0, 0(%r2)
 ; CHECK: std %f2, 8(%r2)
 ; CHECK: br %r14
   %res = call fp128 @llvm.experimental.constrained.fpext.f128.f64(double %val,
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
   store fp128 %res, fp128 *%dst
   ret void
 }
 
 ; Check the low end of the LXDB range.
-define void @f2(fp128 *%dst, double *%ptr) {
+define void @f2(fp128 *%dst, double *%ptr) #0 {
 ; CHECK-LABEL: f2:
 ; CHECK: lxdb %f0, 0(%r3)
 ; CHECK: std %f0, 0(%r2)
@@ -26,13 +26,13 @@ define void @f2(fp128 *%dst, double *%ptr) {
 ; CHECK: br %r14
   %val = load double, double *%ptr
   %res = call fp128 @llvm.experimental.constrained.fpext.f128.f64(double %val,
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
   store fp128 %res, fp128 *%dst
   ret void
 }
 
 ; Check the high end of the aligned LXDB range.
-define void @f3(fp128 *%dst, double *%base) {
+define void @f3(fp128 *%dst, double *%base) #0 {
 ; CHECK-LABEL: f3:
 ; CHECK: lxdb %f0, 4088(%r3)
 ; CHECK: std %f0, 0(%r2)
@@ -41,14 +41,14 @@ define void @f3(fp128 *%dst, double *%base) {
   %ptr = getelementptr double, double *%base, i64 511
   %val = load double, double *%ptr
   %res = call fp128 @llvm.experimental.constrained.fpext.f128.f64(double %val,
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
   store fp128 %res, fp128 *%dst
   ret void
 }
 
 ; Check the next doubleword up, which needs separate address logic.
 ; Other sequences besides this one would be OK.
-define void @f4(fp128 *%dst, double *%base) {
+define void @f4(fp128 *%dst, double *%base) #0 {
 ; CHECK-LABEL: f4:
 ; CHECK: aghi %r3, 4096
 ; CHECK: lxdb %f0, 0(%r3)
@@ -58,13 +58,13 @@ define void @f4(fp128 *%dst, double *%base) {
   %ptr = getelementptr double, double *%base, i64 512
   %val = load double, double *%ptr
   %res = call fp128 @llvm.experimental.constrained.fpext.f128.f64(double %val,
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
   store fp128 %res, fp128 *%dst
   ret void
 }
 
 ; Check negative displacements, which also need separate address logic.
-define void @f5(fp128 *%dst, double *%base) {
+define void @f5(fp128 *%dst, double *%base) #0 {
 ; CHECK-LABEL: f5:
 ; CHECK: aghi %r3, -8
 ; CHECK: lxdb %f0, 0(%r3)
@@ -74,13 +74,13 @@ define void @f5(fp128 *%dst, double *%base) {
   %ptr = getelementptr double, double *%base, i64 -1
   %val = load double, double *%ptr
   %res = call fp128 @llvm.experimental.constrained.fpext.f128.f64(double %val,
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
   store fp128 %res, fp128 *%dst
   ret void
 }
 
 ; Check that LXDB allows indices.
-define void @f6(fp128 *%dst, double *%base, i64 %index) {
+define void @f6(fp128 *%dst, double *%base, i64 %index) #0 {
 ; CHECK-LABEL: f6:
 ; CHECK: sllg %r1, %r4, 3
 ; CHECK: lxdb %f0, 800(%r1,%r3)
@@ -91,8 +91,9 @@ define void @f6(fp128 *%dst, double *%base, i64 %index) {
   %ptr2 = getelementptr double, double *%ptr1, i64 100
   %val = load double, double *%ptr2
   %res = call fp128 @llvm.experimental.constrained.fpext.f128.f64(double %val,
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
   store fp128 %res, fp128 *%dst
   ret void
 }
 
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-conv-05.ll b/llvm/test/CodeGen/SystemZ/fp-strict-conv-05.ll
new file mode 100644
index 000000000..ab3e5a72e
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-conv-05.ll
@@ -0,0 +1,45 @@
+; Test strict conversions of signed i32s to floating-point values.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+declare float @llvm.experimental.constrained.sitofp.f32.i32(i32, metadata, metadata)
+declare double @llvm.experimental.constrained.sitofp.f64.i32(i32, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.sitofp.f128.i32(i32, metadata, metadata)
+
+; Check i32->f32.
+define float @f1(i32 %i) #0 {
+; CHECK-LABEL: f1:
+; CHECK: cefbr %f0, %r2
+; CHECK: br %r14
+  %conv = call float @llvm.experimental.constrained.sitofp.f32.i32(i32 %i,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret float %conv
+}
+
+; Check i32->f64.
+define double @f2(i32 %i) #0 {
+; CHECK-LABEL: f2:
+; CHECK: cdfbr %f0, %r2
+; CHECK: br %r14
+  %conv = call double @llvm.experimental.constrained.sitofp.f64.i32(i32 %i,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret double %conv
+}
+
+; Check i32->f128.
+define void @f3(i32 %i, fp128 *%dst) #0 {
+; CHECK-LABEL: f3:
+; CHECK: cxfbr %f0, %r2
+; CHECK: std %f0, 0(%r3)
+; CHECK: std %f2, 8(%r3)
+; CHECK: br %r14
+  %conv = call fp128 @llvm.experimental.constrained.sitofp.f128.i32(i32 %i,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  store fp128 %conv, fp128 *%dst
+  ret void
+}
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-conv-06.ll b/llvm/test/CodeGen/SystemZ/fp-strict-conv-06.ll
new file mode 100644
index 000000000..58970ba1e
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-conv-06.ll
@@ -0,0 +1,49 @@
+; Test strict conversions of unsigned i32s to floating-point values (z10 only).
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z10 | FileCheck %s
+
+declare float @llvm.experimental.constrained.uitofp.f32.i32(i32, metadata, metadata)
+declare double @llvm.experimental.constrained.uitofp.f64.i32(i32, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.uitofp.f128.i32(i32, metadata, metadata)
+
+; Check i32->f32.  There is no native instruction, so we must promote
+; to i64 first.
+define float @f1(i32 %i) #0 {
+; CHECK-LABEL: f1:
+; CHECK: llgfr [[REGISTER:%r[0-5]]], %r2
+; CHECK: cegbr %f0, [[REGISTER]]
+; CHECK: br %r14
+  %conv = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 %i,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret float %conv
+}
+
+; Check i32->f64.
+define double @f2(i32 %i) #0 {
+; CHECK-LABEL: f2:
+; CHECK: llgfr [[REGISTER:%r[0-5]]], %r2
+; CHECK: cdgbr %f0, [[REGISTER]]
+; CHECK: br %r14
+  %conv = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 %i,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret double %conv
+}
+
+; Check i32->f128.
+define void @f3(i32 %i, fp128 *%dst) #0 {
+; CHECK-LABEL: f3:
+; CHECK: llgfr [[REGISTER:%r[0-5]]], %r2
+; CHECK: cxgbr %f0, [[REGISTER]]
+; CHECK: std %f0, 0(%r3)
+; CHECK: std %f2, 8(%r3)
+; CHECK: br %r14
+  %conv = call fp128 @llvm.experimental.constrained.uitofp.f128.i32(i32 %i,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  store fp128 %conv, fp128 *%dst
+  ret void
+}
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-conv-07.ll b/llvm/test/CodeGen/SystemZ/fp-strict-conv-07.ll
new file mode 100644
index 000000000..b1111237c
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-conv-07.ll
@@ -0,0 +1,45 @@
+; Test strict conversions of signed i64s to floating-point values.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+declare float @llvm.experimental.constrained.sitofp.f32.i64(i64, metadata, metadata)
+declare double @llvm.experimental.constrained.sitofp.f64.i64(i64, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.sitofp.f128.i64(i64, metadata, metadata)
+
+; Test i64->f32.
+define float @f1(i64 %i) #0 {
+; CHECK-LABEL: f1:
+; CHECK: cegbr %f0, %r2
+; CHECK: br %r14
+  %conv = call float @llvm.experimental.constrained.sitofp.f32.i64(i64 %i,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret float %conv
+}
+
+; Test i64->f64.
+define double @f2(i64 %i) #0 {
+; CHECK-LABEL: f2:
+; CHECK: cdgbr %f0, %r2
+; CHECK: br %r14
+  %conv = call double @llvm.experimental.constrained.sitofp.f64.i64(i64 %i,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret double %conv
+}
+
+; Test i64->f128.
+define void @f3(i64 %i, fp128 *%dst) #0 {
+; CHECK-LABEL: f3:
+; CHECK: cxgbr %f0, %r2
+; CHECK: std %f0, 0(%r3)
+; CHECK: std %f2, 8(%r3)
+; CHECK: br %r14
+  %conv = call fp128 @llvm.experimental.constrained.sitofp.f128.i64(i64 %i,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  store fp128 %conv, fp128 *%dst
+  ret void
+}
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-conv-08.ll b/llvm/test/CodeGen/SystemZ/fp-strict-conv-08.ll
new file mode 100644
index 000000000..a151c15e3
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-conv-08.ll
@@ -0,0 +1,47 @@
+; Test strict conversions of unsigned i64s to floating-point values (z10 only).
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z10 | FileCheck %s
+
+declare float @llvm.experimental.constrained.uitofp.f32.i64(i64, metadata, metadata)
+declare double @llvm.experimental.constrained.uitofp.f64.i64(i64, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.uitofp.f128.i64(i64, metadata, metadata)
+
+; Test i64->f32.  There's no native support for unsigned i64-to-fp conversions,
+; but we should be able to implement them using signed i64-to-fp conversions.
+define float @f1(i64 %i) #0 {
+; CHECK-LABEL: f1:
+; CHECK: cegbr
+; CHECK: aebr
+; CHECK: br %r14
+  %conv = call float @llvm.experimental.constrained.uitofp.f32.i64(i64 %i,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret float %conv
+}
+
+; Test i64->f64.
+define double @f2(i64 %i) #0 {
+; CHECK-LABEL: f2:
+; CHECK: ldgr
+; CHECK: adbr
+; CHECK: br %r14
+  %conv = call double @llvm.experimental.constrained.uitofp.f64.i64(i64 %i,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret double %conv
+}
+
+; Test i64->f128.
+define void @f3(i64 %i, fp128 *%dst) #0 {
+; CHECK-LABEL: f3:
+; CHECK: cxgbr
+; CHECK: axbr
+; CHECK: br %r14
+  %conv = call fp128 @llvm.experimental.constrained.uitofp.f128.i64(i64 %i,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  store fp128 %conv, fp128 *%dst
+  ret void
+}
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-conv-09.ll b/llvm/test/CodeGen/SystemZ/fp-strict-conv-09.ll
new file mode 100644
index 000000000..abdb865d4
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-conv-09.ll
@@ -0,0 +1,42 @@
+; Test strict conversion of floating-point values to signed i32s.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+declare i32 @llvm.experimental.constrained.fptosi.i32.f32(float, metadata)
+declare i32 @llvm.experimental.constrained.fptosi.i32.f64(double, metadata)
+declare i32 @llvm.experimental.constrained.fptosi.i32.f128(fp128, metadata)
+
+; Test f32->i32.
+define i32 @f1(float %f) #0 {
+; CHECK-LABEL: f1:
+; CHECK: cfebr %r2, 5, %f0
+; CHECK: br %r14
+  %conv = call i32 @llvm.experimental.constrained.fptosi.i32.f32(float %f,
+                                               metadata !"fpexcept.strict") #0
+  ret i32 %conv
+}
+
+; Test f64->i32.
+define i32 @f2(double %f) #0 {
+; CHECK-LABEL: f2:
+; CHECK: cfdbr %r2, 5, %f0
+; CHECK: br %r14
+  %conv = call i32 @llvm.experimental.constrained.fptosi.i32.f64(double %f,
+                                               metadata !"fpexcept.strict") #0
+  ret i32 %conv
+}
+
+; Test f128->i32.
+define i32 @f3(fp128 *%src) #0 {
+; CHECK-LABEL: f3:
+; CHECK: ld %f0, 0(%r2)
+; CHECK: ld %f2, 8(%r2)
+; CHECK: cfxbr %r2, 5, %f0
+; CHECK: br %r14
+  %f = load fp128, fp128 *%src
+  %conv = call i32 @llvm.experimental.constrained.fptosi.i32.f128(fp128 %f,
+                                               metadata !"fpexcept.strict") #0
+  ret i32 %conv
+}
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-conv-10.ll b/llvm/test/CodeGen/SystemZ/fp-strict-conv-10.ll
new file mode 100644
index 000000000..5327eccdc
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-conv-10.ll
@@ -0,0 +1,91 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; Test strict conversion of floating-point values to unsigned i32s (z10 only).
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z10 | FileCheck %s
+
+; z10 doesn't have native support for unsigned fp-to-i32 conversions;
+; they were added in z196 as the Convert to Logical family of instructions.
+; Promoting to i64 doesn't generate an inexact condition for values that are
+; outside the i32 range but in the i64 range, so use the default expansion.
+; Note that the strict expansion sequence must be used.
+
+declare i32 @llvm.experimental.constrained.fptoui.i32.f32(float, metadata)
+declare i32 @llvm.experimental.constrained.fptoui.i32.f64(double, metadata)
+declare i32 @llvm.experimental.constrained.fptoui.i32.f128(fp128, metadata)
+
+; Test f32->i32.
+define i32 @f1(float %f) #0 {
+; CHECK-LABEL: f1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    larl %r1, .LCPI0_0
+; CHECK-NEXT:    le %f1, 0(%r1)
+; CHECK-NEXT:    kebr %f0, %f1
+; CHECK-NEXT:    jnl .LBB0_2
+; CHECK-NEXT:  # %bb.1:
+; CHECK-NEXT:    lhi %r0, 0
+; CHECK-NEXT:    lzer %f1
+; CHECK-NEXT:    j .LBB0_3
+; CHECK-NEXT:  .LBB0_2:
+; CHECK-NEXT:    llilh %r0, 32768
+; CHECK-NEXT:  .LBB0_3:
+; CHECK-NEXT:    sebr %f0, %f1
+; CHECK-NEXT:    cfebr %r2, 5, %f0
+; CHECK-NEXT:    xr %r2, %r0
+; CHECK-NEXT:    br %r14
+  %conv = call i32 @llvm.experimental.constrained.fptoui.i32.f32(float %f,
+                                               metadata !"fpexcept.strict") #0
+  ret i32 %conv
+}
+
+; Test f64->i32.
+define i32 @f2(double %f) #0 {
+; CHECK-LABEL: f2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    larl %r1, .LCPI1_0
+; CHECK-NEXT:    ldeb %f1, 0(%r1)
+; CHECK-NEXT:    kdbr %f0, %f1
+; CHECK-NEXT:    jnl .LBB1_2
+; CHECK-NEXT:  # %bb.1:
+; CHECK-NEXT:    lhi %r0, 0
+; CHECK-NEXT:    lzdr %f1
+; CHECK-NEXT:    j .LBB1_3
+; CHECK-NEXT:  .LBB1_2:
+; CHECK-NEXT:    llilh %r0, 32768
+; CHECK-NEXT:  .LBB1_3:
+; CHECK-NEXT:    sdbr %f0, %f1
+; CHECK-NEXT:    cfdbr %r2, 5, %f0
+; CHECK-NEXT:    xr %r2, %r0
+; CHECK-NEXT:    br %r14
+  %conv = call i32 @llvm.experimental.constrained.fptoui.i32.f64(double %f,
+                                               metadata !"fpexcept.strict") #0
+  ret i32 %conv
+}
+
+; Test f128->i32.
+define i32 @f3(fp128 *%src) #0 {
+; CHECK-LABEL: f3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld %f0, 0(%r2)
+; CHECK-NEXT:    ld %f2, 8(%r2)
+; CHECK-NEXT:    larl %r1, .LCPI2_0
+; CHECK-NEXT:    lxeb %f1, 0(%r1)
+; CHECK-NEXT:    kxbr %f0, %f1
+; CHECK-NEXT:    jnl .LBB2_2
+; CHECK-NEXT:  # %bb.1:
+; CHECK-NEXT:    lhi %r0, 0
+; CHECK-NEXT:    lzxr %f1
+; CHECK-NEXT:    j .LBB2_3
+; CHECK-NEXT:  .LBB2_2:
+; CHECK-NEXT:    llilh %r0, 32768
+; CHECK-NEXT:  .LBB2_3:
+; CHECK-NEXT:    sxbr %f0, %f1
+; CHECK-NEXT:    cfxbr %r2, 5, %f0
+; CHECK-NEXT:    xr %r2, %r0
+; CHECK-NEXT:    br %r14
+  %f = load fp128, fp128 *%src
+  %conv = call i32 @llvm.experimental.constrained.fptoui.i32.f128(fp128 %f,
+                                               metadata !"fpexcept.strict") #0
+  ret i32 %conv
+}
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-conv-11.ll b/llvm/test/CodeGen/SystemZ/fp-strict-conv-11.ll
new file mode 100644
index 000000000..d9cc33700
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-conv-11.ll
@@ -0,0 +1,42 @@
+; Test strict conversion of floating-point values to signed i64s.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+declare i64 @llvm.experimental.constrained.fptosi.i64.f32(float, metadata)
+declare i64 @llvm.experimental.constrained.fptosi.i64.f64(double, metadata)
+declare i64 @llvm.experimental.constrained.fptosi.i64.f128(fp128, metadata)
+
+; Test f32->i64.
+define i64 @f1(float %f) #0 {
+; CHECK-LABEL: f1:
+; CHECK: cgebr %r2, 5, %f0
+; CHECK: br %r14
+  %conv = call i64 @llvm.experimental.constrained.fptosi.i64.f32(float %f,
+                                               metadata !"fpexcept.strict") #0
+  ret i64 %conv
+}
+
+; Test f64->i64.
+define i64 @f2(double %f) #0 {
+; CHECK-LABEL: f2:
+; CHECK: cgdbr %r2, 5, %f0
+; CHECK: br %r14
+  %conv = call i64 @llvm.experimental.constrained.fptosi.i64.f64(double %f,
+                                               metadata !"fpexcept.strict") #0
+  ret i64 %conv
+}
+
+; Test f128->i64.
+define i64 @f3(fp128 *%src) #0 {
+; CHECK-LABEL: f3:
+; CHECK: ld %f0, 0(%r2)
+; CHECK: ld %f2, 8(%r2)
+; CHECK: cgxbr %r2, 5, %f0
+; CHECK: br %r14
+  %f = load fp128, fp128 *%src
+  %conv = call i64 @llvm.experimental.constrained.fptosi.i64.f128(fp128 %f,
+                                               metadata !"fpexcept.strict") #0
+  ret i64 %conv
+}
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-conv-12.ll b/llvm/test/CodeGen/SystemZ/fp-strict-conv-12.ll
new file mode 100644
index 000000000..e7ed6af33
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-conv-12.ll
@@ -0,0 +1,90 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; Test strict conversion of floating-point values to unsigned i64s (z10 only).
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z10 | FileCheck %s
+
+; z10 doesn't have native support for unsigned fp-to-i64 conversions;
+; they were added in z196 as the Convert to Logical family of instructions.
+; Convert via signed i64s instead.
+; Note that the strict expansion sequence must be used.
+
+declare i64 @llvm.experimental.constrained.fptoui.i64.f32(float, metadata)
+declare i64 @llvm.experimental.constrained.fptoui.i64.f64(double, metadata)
+declare i64 @llvm.experimental.constrained.fptoui.i64.f128(fp128, metadata)
+
+; Test f32->i64.
+define i64 @f1(float %f) #0 {
+; CHECK-LABEL: f1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    larl %r1, .LCPI0_0
+; CHECK-NEXT:    le %f1, 0(%r1)
+; CHECK-NEXT:    kebr %f0, %f1
+; CHECK-NEXT:    jnl .LBB0_2
+; CHECK-NEXT:  # %bb.1:
+; CHECK-NEXT:    lghi %r0, 0
+; CHECK-NEXT:    lzer %f1
+; CHECK-NEXT:    j .LBB0_3
+; CHECK-NEXT:  .LBB0_2:
+; CHECK-NEXT:    llihh %r0, 32768
+; CHECK-NEXT:  .LBB0_3:
+; CHECK-NEXT:    sebr %f0, %f1
+; CHECK-NEXT:    cgebr %r2, 5, %f0
+; CHECK-NEXT:    xgr %r2, %r0
+; CHECK-NEXT:    br %r14
+  %conv = call i64 @llvm.experimental.constrained.fptoui.i64.f32(float %f,
+                                               metadata !"fpexcept.strict") #0
+  ret i64 %conv
+}
+
+; Test f64->i64.
+define i64 @f2(double %f) #0 {
+; CHECK-LABEL: f2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    larl %r1, .LCPI1_0
+; CHECK-NEXT:    ldeb %f1, 0(%r1)
+; CHECK-NEXT:    kdbr %f0, %f1
+; CHECK-NEXT:    jnl .LBB1_2
+; CHECK-NEXT:  # %bb.1:
+; CHECK-NEXT:    lghi %r0, 0
+; CHECK-NEXT:    lzdr %f1
+; CHECK-NEXT:    j .LBB1_3
+; CHECK-NEXT:  .LBB1_2:
+; CHECK-NEXT:    llihh %r0, 32768
+; CHECK-NEXT:  .LBB1_3:
+; CHECK-NEXT:    sdbr %f0, %f1
+; CHECK-NEXT:    cgdbr %r2, 5, %f0
+; CHECK-NEXT:    xgr %r2, %r0
+; CHECK-NEXT:    br %r14
+  %conv = call i64 @llvm.experimental.constrained.fptoui.i64.f64(double %f,
+                                               metadata !"fpexcept.strict") #0
+  ret i64 %conv
+}
+
+; Test f128->i64.
+define i64 @f3(fp128 *%src) #0 {
+; CHECK-LABEL: f3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld %f0, 0(%r2)
+; CHECK-NEXT:    ld %f2, 8(%r2)
+; CHECK-NEXT:    larl %r1, .LCPI2_0
+; CHECK-NEXT:    lxeb %f1, 0(%r1)
+; CHECK-NEXT:    kxbr %f0, %f1
+; CHECK-NEXT:    jnl .LBB2_2
+; CHECK-NEXT:  # %bb.1:
+; CHECK-NEXT:    lghi %r0, 0
+; CHECK-NEXT:    lzxr %f1
+; CHECK-NEXT:    j .LBB2_3
+; CHECK-NEXT:  .LBB2_2:
+; CHECK-NEXT:    llihh %r0, 32768
+; CHECK-NEXT:  .LBB2_3:
+; CHECK-NEXT:    sxbr %f0, %f1
+; CHECK-NEXT:    cgxbr %r2, 5, %f0
+; CHECK-NEXT:    xgr %r2, %r0
+; CHECK-NEXT:    br %r14
+  %f = load fp128, fp128 *%src
+  %conv = call i64 @llvm.experimental.constrained.fptoui.i64.f128(fp128 %f,
+                                               metadata !"fpexcept.strict") #0
+  ret i64 %conv
+}
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-conv-13.ll b/llvm/test/CodeGen/SystemZ/fp-strict-conv-13.ll
new file mode 100644
index 000000000..de87c8872
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-conv-13.ll
@@ -0,0 +1,86 @@
+; Test strict conversions of unsigned integers to floating-point values
+; (z196 and above).
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z196 | FileCheck %s
+
+declare float @llvm.experimental.constrained.uitofp.f32.i32(i32, metadata, metadata)
+declare double @llvm.experimental.constrained.uitofp.f64.i32(i32, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.uitofp.f128.i32(i32, metadata, metadata)
+
+declare float @llvm.experimental.constrained.uitofp.f32.i64(i64, metadata, metadata)
+declare double @llvm.experimental.constrained.uitofp.f64.i64(i64, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.uitofp.f128.i64(i64, metadata, metadata)
+
+; Check i32->f32.
+define float @f1(i32 %i) #0 {
+; CHECK-LABEL: f1:
+; CHECK: celfbr %f0, 0, %r2, 0
+; CHECK: br %r14
+  %conv = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 %i,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret float %conv
+}
+
+; Check i32->f64.
+define double @f2(i32 %i) #0 {
+; CHECK-LABEL: f2:
+; CHECK: cdlfbr %f0, 0, %r2, 0
+; CHECK: br %r14
+  %conv = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 %i,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret double %conv
+}
+
+; Check i32->f128.
+define void @f3(i32 %i, fp128 *%dst) #0 {
+; CHECK-LABEL: f3:
+; CHECK: cxlfbr %f0, 0, %r2, 0
+; CHECK-DAG: std %f0, 0(%r3)
+; CHECK-DAG: std %f2, 8(%r3)
+; CHECK: br %r14
+  %conv = call fp128 @llvm.experimental.constrained.uitofp.f128.i32(i32 %i,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  store fp128 %conv, fp128 *%dst
+  ret void
+}
+
+; Check i64->f32.
+define float @f4(i64 %i) #0 {
+; CHECK-LABEL: f4:
+; CHECK: celgbr %f0, 0, %r2, 0
+; CHECK: br %r14
+  %conv = call float @llvm.experimental.constrained.uitofp.f32.i64(i64 %i,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret float %conv
+}
+
+; Check i64->f64.
+define double @f5(i64 %i) #0 {
+; CHECK-LABEL: f5:
+; CHECK: cdlgbr %f0, 0, %r2, 0
+; CHECK: br %r14
+  %conv = call double @llvm.experimental.constrained.uitofp.f64.i64(i64 %i,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret double %conv
+}
+
+; Check i64->f128.
+define void @f6(i64 %i, fp128 *%dst) #0 {
+; CHECK-LABEL: f6:
+; CHECK: cxlgbr %f0, 0, %r2, 0
+; CHECK-DAG: std %f0, 0(%r3)
+; CHECK-DAG: std %f2, 8(%r3)
+; CHECK: br %r14
+  %conv = call fp128 @llvm.experimental.constrained.uitofp.f128.i64(i64 %i,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  store fp128 %conv, fp128 *%dst
+  ret void
+}
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-conv-14.ll b/llvm/test/CodeGen/SystemZ/fp-strict-conv-14.ll
new file mode 100644
index 000000000..83478bab7
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-conv-14.ll
@@ -0,0 +1,79 @@
+; Test strict conversion of floating-point values to unsigned integers.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z196 | FileCheck %s
+
+declare i32 @llvm.experimental.constrained.fptoui.i32.f32(float, metadata)
+declare i32 @llvm.experimental.constrained.fptoui.i32.f64(double, metadata)
+declare i32 @llvm.experimental.constrained.fptoui.i32.f128(fp128, metadata)
+
+declare i64 @llvm.experimental.constrained.fptoui.i64.f32(float, metadata)
+declare i64 @llvm.experimental.constrained.fptoui.i64.f64(double, metadata)
+declare i64 @llvm.experimental.constrained.fptoui.i64.f128(fp128, metadata)
+
+; Test f32->i32.
+define i32 @f1(float %f) #0 {
+; CHECK-LABEL: f1:
+; CHECK: clfebr %r2, 5, %f0, 0
+; CHECK: br %r14
+  %conv = call i32 @llvm.experimental.constrained.fptoui.i32.f32(float %f,
+                                               metadata !"fpexcept.strict") #0
+  ret i32 %conv
+}
+
+; Test f64->i32.
+define i32 @f2(double %f) #0 {
+; CHECK-LABEL: f2:
+; CHECK: clfdbr %r2, 5, %f0, 0
+; CHECK: br %r14
+  %conv = call i32 @llvm.experimental.constrained.fptoui.i32.f64(double %f,
+                                               metadata !"fpexcept.strict") #0
+  ret i32 %conv
+}
+
+; Test f128->i32.
+define i32 @f3(fp128 *%src) #0 {
+; CHECK-LABEL: f3:
+; CHECK-DAG: ld %f0, 0(%r2)
+; CHECK-DAG: ld %f2, 8(%r2)
+; CHECK: clfxbr %r2, 5, %f0, 0
+; CHECK: br %r14
+  %f = load fp128, fp128 *%src
+  %conv = call i32 @llvm.experimental.constrained.fptoui.i32.f128(fp128 %f,
+                                               metadata !"fpexcept.strict") #0
+  ret i32 %conv
+}
+
+; Test f32->i64.
+define i64 @f4(float %f) #0 {
+; CHECK-LABEL: f4:
+; CHECK: clgebr %r2, 5, %f0, 0
+; CHECK: br %r14
+  %conv = call i64 @llvm.experimental.constrained.fptoui.i64.f32(float %f,
+                                               metadata !"fpexcept.strict") #0
+  ret i64 %conv
+}
+
+; Test f64->i64.
+define i64 @f5(double %f) #0 {
+; CHECK-LABEL: f5:
+; CHECK: clgdbr %r2, 5, %f0, 0
+; CHECK: br %r14
+  %conv = call i64 @llvm.experimental.constrained.fptoui.i64.f64(double %f,
+                                               metadata !"fpexcept.strict") #0
+  ret i64 %conv
+}
+
+; Test f128->i64.
+define i64 @f6(fp128 *%src) #0 {
+; CHECK-LABEL: f6:
+; CHECK-DAG: ld %f0, 0(%r2)
+; CHECK-DAG: ld %f2, 8(%r2)
+; CHECK: clgxbr %r2, 5, %f0, 0
+; CHECK: br %r14
+  %f = load fp128, fp128 *%src
+  %conv = call i64 @llvm.experimental.constrained.fptoui.i64.f128(fp128 %f,
+                                               metadata !"fpexcept.strict") #0
+  ret i64 %conv
+}
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-conv-15.ll b/llvm/test/CodeGen/SystemZ/fp-strict-conv-15.ll
index 64a82c32f..9b080c14e 100644
--- a/llvm/test/CodeGen/SystemZ/fp-strict-conv-15.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-conv-15.ll
@@ -9,7 +9,7 @@ declare fp128 @llvm.experimental.constrained.fpext.f128.f32(float, metadata)
 declare fp128 @llvm.experimental.constrained.fpext.f128.f64(double, metadata)
 
 ; Test f128->f64.
-define double @f1(fp128 *%ptr) {
+define double @f1(fp128 *%ptr) #0 {
 ; CHECK-LABEL: f1:
 ; CHECK: vl [[REG:%v[0-9]+]], 0(%r2)
 ; CHECK: wflrx %f0, [[REG]], 0, 0
@@ -18,12 +18,12 @@ define double @f1(fp128 *%ptr) {
   %res = call double @llvm.experimental.constrained.fptrunc.f64.f128(
                                                fp128 %val,
                                                metadata !"round.dynamic",
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
   ret double %res
 }
 
 ; Test f128->f32.
-define float @f2(fp128 *%ptr) {
+define float @f2(fp128 *%ptr) #0 {
 ; CHECK-LABEL: f2:
 ; CHECK: vl [[REG:%v[0-9]+]], 0(%r2)
 ; CHECK: wflrx %f0, [[REG]], 0, 3
@@ -33,32 +33,33 @@ define float @f2(fp128 *%ptr) {
   %res = call float @llvm.experimental.constrained.fptrunc.f32.f128(
                                                fp128 %val,
                                                metadata !"round.dynamic",
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
   ret float %res
 }
 
 ; Test f64->f128.
-define void @f3(fp128 *%dst, double %val) {
+define void @f3(fp128 *%dst, double %val) #0 {
 ; CHECK-LABEL: f3:
 ; CHECK: wflld [[RES:%v[0-9]+]], %f0
 ; CHECK: vst [[RES]], 0(%r2)
 ; CHECK: br %r14
   %res = call fp128 @llvm.experimental.constrained.fpext.f128.f64(double %val,
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
   store fp128 %res, fp128 *%dst
   ret void
 }
 
 ; Test f32->f128.
-define void @f4(fp128 *%dst, float %val) {
+define void @f4(fp128 *%dst, float %val) #0 {
 ; CHECK-LABEL: f4:
 ; CHECK: ldebr %f0, %f0
 ; CHECK: wflld [[RES:%v[0-9]+]], %f0
 ; CHECK: vst [[RES]], 0(%r2)
 ; CHECK: br %r14
   %res = call fp128 @llvm.experimental.constrained.fpext.f128.f32(float %val,
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
   store fp128 %res, fp128 *%dst
   ret void
 }
 
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-conv-16.ll b/llvm/test/CodeGen/SystemZ/fp-strict-conv-16.ll
new file mode 100644
index 000000000..e7523534e
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-conv-16.ll
@@ -0,0 +1,125 @@
+; Test f128 floating-point strict conversion to/from integers on z14.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z14 | FileCheck %s
+
+declare fp128 @llvm.experimental.constrained.sitofp.f128.i32(i32, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.sitofp.f128.i64(i64, metadata, metadata)
+
+declare fp128 @llvm.experimental.constrained.uitofp.f128.i32(i32, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.uitofp.f128.i64(i64, metadata, metadata)
+
+declare i32 @llvm.experimental.constrained.fptosi.i32.f128(fp128, metadata)
+declare i64 @llvm.experimental.constrained.fptosi.i64.f128(fp128, metadata)
+
+declare i32 @llvm.experimental.constrained.fptoui.i32.f128(fp128, metadata)
+declare i64 @llvm.experimental.constrained.fptoui.i64.f128(fp128, metadata)
+
+; Test signed i32->f128.
+define void @f1(i32 %i, fp128 *%dst) #0 {
+; CHECK-LABEL: f1:
+; CHECK: cxfbr %f0, %r2
+; CHECK: vmrhg %v0, %v0, %v2
+; CHECK: vst %v0, 0(%r3)
+; CHECK: br %r14
+  %conv = call fp128 @llvm.experimental.constrained.sitofp.f128.i32(i32 %i,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  store fp128 %conv, fp128 *%dst
+  ret void
+}
+
+; Test signed i64->f128.
+define void @f2(i64 %i, fp128 *%dst) #0 {
+; CHECK-LABEL: f2:
+; CHECK: cxgbr %f0, %r2
+; CHECK: vmrhg %v0, %v0, %v2
+; CHECK: vst %v0, 0(%r3)
+; CHECK: br %r14
+  %conv = call fp128 @llvm.experimental.constrained.sitofp.f128.i64(i64 %i,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  store fp128 %conv, fp128 *%dst
+  ret void
+}
+
+; Test unsigned i32->f128.
+define void @f3(i32 %i, fp128 *%dst) #0 {
+; CHECK-LABEL: f3:
+; CHECK: cxlfbr %f0, 0, %r2, 0
+; CHECK: vmrhg %v0, %v0, %v2
+; CHECK: vst %v0, 0(%r3)
+; CHECK: br %r14
+  %conv = call fp128 @llvm.experimental.constrained.uitofp.f128.i32(i32 %i,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  store fp128 %conv, fp128 *%dst
+  ret void
+}
+
+; Test unsigned i64->f128.
+define void @f4(i64 %i, fp128 *%dst) #0 {
+; CHECK-LABEL: f4:
+; CHECK: cxlgbr %f0, 0, %r2, 0
+; CHECK: vmrhg %v0, %v0, %v2
+; CHECK: vst %v0, 0(%r3)
+; CHECK: br %r14
+  %conv = call fp128 @llvm.experimental.constrained.uitofp.f128.i64(i64 %i,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  store fp128 %conv, fp128 *%dst
+  ret void
+}
+
+; Test signed f128->i32.
+define i32 @f5(fp128 *%src) #0 {
+; CHECK-LABEL: f5:
+; CHECK: vl %v0, 0(%r2)
+; CHECK: vrepg %v2, %v0, 1
+; CHECK: cfxbr %r2, 5, %f0
+; CHECK: br %r14
+  %f = load fp128, fp128 *%src
+  %conv = call i32 @llvm.experimental.constrained.fptosi.i32.f128(fp128 %f,
+                                               metadata !"fpexcept.strict") #0
+  ret i32 %conv
+}
+
+; Test signed f128->i64.
+define i64 @f6(fp128 *%src) #0 {
+; CHECK-LABEL: f6:
+; CHECK: vl %v0, 0(%r2)
+; CHECK: vrepg %v2, %v0, 1
+; CHECK: cgxbr %r2, 5, %f0
+; CHECK: br %r14
+  %f = load fp128, fp128 *%src
+  %conv = call i64 @llvm.experimental.constrained.fptosi.i64.f128(fp128 %f,
+                                               metadata !"fpexcept.strict") #0
+  ret i64 %conv
+}
+
+; Test unsigned f128->i32.
+define i32 @f7(fp128 *%src) #0 {
+; CHECK-LABEL: f7:
+; CHECK: vl %v0, 0(%r2)
+; CHECK: vrepg %v2, %v0, 1
+; CHECK: clfxbr %r2, 5, %f0, 0
+; CHECK: br %r14
+  %f = load fp128, fp128 *%src
+  %conv = call i32 @llvm.experimental.constrained.fptoui.i32.f128(fp128 %f,
+                                               metadata !"fpexcept.strict") #0
+  ret i32 %conv
+}
+
+; Test unsigned f128->i64.
+define i64 @f8(fp128 *%src) #0 {
+; CHECK-LABEL: f8:
+; CHECK: vl %v0, 0(%r2)
+; CHECK: vrepg %v2, %v0, 1
+; CHECK: clgxbr %r2, 5, %f0, 0
+; CHECK: br %r14
+  %f = load fp128, fp128 *%src
+  %conv = call i64 @llvm.experimental.constrained.fptoui.i64.f128(fp128 %f,
+                                               metadata !"fpexcept.strict") #0
+  ret i64 %conv
+}
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-div-01.ll b/llvm/test/CodeGen/SystemZ/fp-strict-div-01.ll
index 7f9a4fce1..27cb70cd0 100644
--- a/llvm/test/CodeGen/SystemZ/fp-strict-div-01.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-div-01.ll
@@ -8,19 +8,19 @@ declare float @foo()
 declare float @llvm.experimental.constrained.fdiv.f32(float, float, metadata, metadata)
 
 ; Check register division.
-define float @f1(float %f1, float %f2) {
+define float @f1(float %f1, float %f2) #0 {
 ; CHECK-LABEL: f1:
 ; CHECK: debr %f0, %f2
 ; CHECK: br %r14
   %res = call float @llvm.experimental.constrained.fdiv.f32(
                         float %f1, float %f2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
 
 ; Check the low end of the DEB range.
-define float @f2(float %f1, float *%ptr) {
+define float @f2(float %f1, float *%ptr) #0 {
 ; CHECK-LABEL: f2:
 ; CHECK: deb %f0, 0(%r2)
 ; CHECK: br %r14
@@ -28,12 +28,12 @@ define float @f2(float %f1, float *%ptr) {
   %res = call float @llvm.experimental.constrained.fdiv.f32(
                         float %f1, float %f2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
 
 ; Check the high end of the aligned DEB range.
-define float @f3(float %f1, float *%base) {
+define float @f3(float %f1, float *%base) #0 {
 ; CHECK-LABEL: f3:
 ; CHECK: deb %f0, 4092(%r2)
 ; CHECK: br %r14
@@ -42,13 +42,13 @@ define float @f3(float %f1, float *%base) {
   %res = call float @llvm.experimental.constrained.fdiv.f32(
                         float %f1, float %f2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
 
 ; Check the next word up, which needs separate address logic.
 ; Other sequences besides this one would be OK.
-define float @f4(float %f1, float *%base) {
+define float @f4(float %f1, float *%base) #0 {
 ; CHECK-LABEL: f4:
 ; CHECK: aghi %r2, 4096
 ; CHECK: deb %f0, 0(%r2)
@@ -58,12 +58,12 @@ define float @f4(float %f1, float *%base) {
   %res = call float @llvm.experimental.constrained.fdiv.f32(
                         float %f1, float %f2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
 
 ; Check negative displacements, which also need separate address logic.
-define float @f5(float %f1, float *%base) {
+define float @f5(float %f1, float *%base) #0 {
 ; CHECK-LABEL: f5:
 ; CHECK: aghi %r2, -4
 ; CHECK: deb %f0, 0(%r2)
@@ -73,12 +73,12 @@ define float @f5(float %f1, float *%base) {
   %res = call float @llvm.experimental.constrained.fdiv.f32(
                         float %f1, float %f2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
 
 ; Check that DEB allows indices.
-define float @f6(float %f1, float *%base, i64 %index) {
+define float @f6(float %f1, float *%base, i64 %index) #0 {
 ; CHECK-LABEL: f6:
 ; CHECK: sllg %r1, %r3, 2
 ; CHECK: deb %f0, 400(%r1,%r2)
@@ -89,12 +89,12 @@ define float @f6(float %f1, float *%base, i64 %index) {
   %res = call float @llvm.experimental.constrained.fdiv.f32(
                         float %f1, float %f2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
 
 ; Check that divisions of spilled values can use DEB rather than DEBR.
-define float @f7(float *%ptr0) {
+define float @f7(float *%ptr0) #0 {
 ; CHECK-LABEL: f7:
 ; CHECK: brasl %r14, foo@PLT
 ; CHECK-SCALAR: deb %f0, 16{{[04]}}(%r15)
@@ -122,52 +122,54 @@ define float @f7(float *%ptr0) {
   %val9 = load float, float *%ptr9
   %val10 = load float, float *%ptr10
 
-  %ret = call float @foo()
+  %ret = call float @foo() #0
 
   %div0 = call float @llvm.experimental.constrained.fdiv.f32(
                         float %ret, float %val0,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %div1 = call float @llvm.experimental.constrained.fdiv.f32(
                         float %div0, float %val1,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %div2 = call float @llvm.experimental.constrained.fdiv.f32(
                         float %div1, float %val2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %div3 = call float @llvm.experimental.constrained.fdiv.f32(
                         float %div2, float %val3,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %div4 = call float @llvm.experimental.constrained.fdiv.f32(
                         float %div3, float %val4,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %div5 = call float @llvm.experimental.constrained.fdiv.f32(
                         float %div4, float %val5,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %div6 = call float @llvm.experimental.constrained.fdiv.f32(
                         float %div5, float %val6,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %div7 = call float @llvm.experimental.constrained.fdiv.f32(
                         float %div6, float %val7,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %div8 = call float @llvm.experimental.constrained.fdiv.f32(
                         float %div7, float %val8,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %div9 = call float @llvm.experimental.constrained.fdiv.f32(
                         float %div8, float %val9,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %div10 = call float @llvm.experimental.constrained.fdiv.f32(
                         float %div9, float %val10,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
 
   ret float %div10
 }
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-div-02.ll b/llvm/test/CodeGen/SystemZ/fp-strict-div-02.ll
index 850af2172..d8fc2bd74 100644
--- a/llvm/test/CodeGen/SystemZ/fp-strict-div-02.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-div-02.ll
@@ -8,19 +8,19 @@ declare double @foo()
 declare double @llvm.experimental.constrained.fdiv.f64(double, double, metadata, metadata)
 
 ; Check register division.
-define double @f1(double %f1, double %f2) {
+define double @f1(double %f1, double %f2) #0 {
 ; CHECK-LABEL: f1:
 ; CHECK: ddbr %f0, %f2
 ; CHECK: br %r14
   %res = call double @llvm.experimental.constrained.fdiv.f64(
                         double %f1, double %f2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
 ; Check the low end of the DDB range.
-define double @f2(double %f1, double *%ptr) {
+define double @f2(double %f1, double *%ptr) #0 {
 ; CHECK-LABEL: f2:
 ; CHECK: ddb %f0, 0(%r2)
 ; CHECK: br %r14
@@ -28,12 +28,12 @@ define double @f2(double %f1, double *%ptr) {
   %res = call double @llvm.experimental.constrained.fdiv.f64(
                         double %f1, double %f2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
 ; Check the high end of the aligned DDB range.
-define double @f3(double %f1, double *%base) {
+define double @f3(double %f1, double *%base) #0 {
 ; CHECK-LABEL: f3:
 ; CHECK: ddb %f0, 4088(%r2)
 ; CHECK: br %r14
@@ -42,13 +42,13 @@ define double @f3(double %f1, double *%base) {
   %res = call double @llvm.experimental.constrained.fdiv.f64(
                         double %f1, double %f2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
 ; Check the next doubleword up, which needs separate address logic.
 ; Other sequences besides this one would be OK.
-define double @f4(double %f1, double *%base) {
+define double @f4(double %f1, double *%base) #0 {
 ; CHECK-LABEL: f4:
 ; CHECK: aghi %r2, 4096
 ; CHECK: ddb %f0, 0(%r2)
@@ -58,12 +58,12 @@ define double @f4(double %f1, double *%base) {
   %res = call double @llvm.experimental.constrained.fdiv.f64(
                         double %f1, double %f2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
 ; Check negative displacements, which also need separate address logic.
-define double @f5(double %f1, double *%base) {
+define double @f5(double %f1, double *%base) #0 {
 ; CHECK-LABEL: f5:
 ; CHECK: aghi %r2, -8
 ; CHECK: ddb %f0, 0(%r2)
@@ -73,12 +73,12 @@ define double @f5(double %f1, double *%base) {
   %res = call double @llvm.experimental.constrained.fdiv.f64(
                         double %f1, double %f2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
 ; Check that DDB allows indices.
-define double @f6(double %f1, double *%base, i64 %index) {
+define double @f6(double %f1, double *%base, i64 %index) #0 {
 ; CHECK-LABEL: f6:
 ; CHECK: sllg %r1, %r3, 3
 ; CHECK: ddb %f0, 800(%r1,%r2)
@@ -89,12 +89,12 @@ define double @f6(double %f1, double *%base, i64 %index) {
   %res = call double @llvm.experimental.constrained.fdiv.f64(
                         double %f1, double %f2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
 ; Check that divisions of spilled values can use DDB rather than DDBR.
-define double @f7(double *%ptr0) {
+define double @f7(double *%ptr0) #0 {
 ; CHECK-LABEL: f7:
 ; CHECK: brasl %r14, foo@PLT
 ; CHECK-SCALAR: ddb %f0, 160(%r15)
@@ -122,52 +122,54 @@ define double @f7(double *%ptr0) {
   %val9 = load double, double *%ptr9
   %val10 = load double, double *%ptr10
 
-  %ret = call double @foo()
+  %ret = call double @foo() #0
 
   %div0 = call double @llvm.experimental.constrained.fdiv.f64(
                         double %ret, double %val0,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %div1 = call double @llvm.experimental.constrained.fdiv.f64(
                         double %div0, double %val1,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %div2 = call double @llvm.experimental.constrained.fdiv.f64(
                         double %div1, double %val2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %div3 = call double @llvm.experimental.constrained.fdiv.f64(
                         double %div2, double %val3,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %div4 = call double @llvm.experimental.constrained.fdiv.f64(
                         double %div3, double %val4,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %div5 = call double @llvm.experimental.constrained.fdiv.f64(
                         double %div4, double %val5,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %div6 = call double @llvm.experimental.constrained.fdiv.f64(
                         double %div5, double %val6,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %div7 = call double @llvm.experimental.constrained.fdiv.f64(
                         double %div6, double %val7,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %div8 = call double @llvm.experimental.constrained.fdiv.f64(
                         double %div7, double %val8,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %div9 = call double @llvm.experimental.constrained.fdiv.f64(
                         double %div8, double %val9,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %div10 = call double @llvm.experimental.constrained.fdiv.f64(
                         double %div9, double %val10,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
 
   ret double %div10
 }
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-div-03.ll b/llvm/test/CodeGen/SystemZ/fp-strict-div-03.ll
index 860f91d8e..a53512eeb 100644
--- a/llvm/test/CodeGen/SystemZ/fp-strict-div-03.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-div-03.ll
@@ -3,9 +3,10 @@
 ; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
 
 declare fp128 @llvm.experimental.constrained.fdiv.f128(fp128, fp128, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.fpext.f128.f32(float, metadata)
 
 ; There is no memory form of 128-bit division.
-define void @f1(fp128 *%ptr, float %f2) {
+define void @f1(fp128 *%ptr, float %f2) strictfp {
 ; CHECK-LABEL: f1:
 ; CHECK-DAG: lxebr %f0, %f0
 ; CHECK-DAG: ld %f1, 0(%r2)
@@ -15,11 +16,14 @@ define void @f1(fp128 *%ptr, float %f2) {
 ; CHECK: std %f3, 8(%r2)
 ; CHECK: br %r14
   %f1 = load fp128, fp128 *%ptr
-  %f2x = fpext float %f2 to fp128
+  %f2x = call fp128 @llvm.experimental.constrained.fpext.f128.f32(float %f2,
+                                               metadata !"fpexcept.strict") #0
   %sum = call fp128 @llvm.experimental.constrained.fdiv.f128(
                         fp128 %f1, fp128 %f2x,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   store fp128 %sum, fp128 *%ptr
   ret void
 }
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-div-04.ll b/llvm/test/CodeGen/SystemZ/fp-strict-div-04.ll
index e3f117515..a43eebb5f 100644
--- a/llvm/test/CodeGen/SystemZ/fp-strict-div-04.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-div-04.ll
@@ -4,7 +4,7 @@
 
 declare fp128 @llvm.experimental.constrained.fdiv.f128(fp128, fp128, metadata, metadata)
 
-define void @f1(fp128 *%ptr1, fp128 *%ptr2) {
+define void @f1(fp128 *%ptr1, fp128 *%ptr2) strictfp {
 ; CHECK-LABEL: f1:
 ; CHECK-DAG: vl [[REG1:%v[0-9]+]], 0(%r2)
 ; CHECK-DAG: vl [[REG2:%v[0-9]+]], 0(%r3)
@@ -16,7 +16,7 @@ define void @f1(fp128 *%ptr1, fp128 *%ptr2) {
   %sum = call fp128 @llvm.experimental.constrained.fdiv.f128(
                         fp128 %f1, fp128 %f2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") strictfp
   store fp128 %sum, fp128 *%ptr1
   ret void
 }
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-mul-01.ll b/llvm/test/CodeGen/SystemZ/fp-strict-mul-01.ll
index 3e07091bf..623ef9007 100644
--- a/llvm/test/CodeGen/SystemZ/fp-strict-mul-01.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-mul-01.ll
@@ -8,19 +8,19 @@ declare float @foo()
 declare float @llvm.experimental.constrained.fmul.f32(float, float, metadata, metadata)
 
 ; Check register multiplication.
-define float @f1(float %f1, float %f2) {
+define float @f1(float %f1, float %f2) #0 {
 ; CHECK-LABEL: f1:
 ; CHECK: meebr %f0, %f2
 ; CHECK: br %r14
   %res = call float @llvm.experimental.constrained.fmul.f32(
                         float %f1, float %f2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
 
 ; Check the low end of the MEEB range.
-define float @f2(float %f1, float *%ptr) {
+define float @f2(float %f1, float *%ptr) #0 {
 ; CHECK-LABEL: f2:
 ; CHECK: meeb %f0, 0(%r2)
 ; CHECK: br %r14
@@ -28,12 +28,12 @@ define float @f2(float %f1, float *%ptr) {
   %res = call float @llvm.experimental.constrained.fmul.f32(
                         float %f1, float %f2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
 
 ; Check the high end of the aligned MEEB range.
-define float @f3(float %f1, float *%base) {
+define float @f3(float %f1, float *%base) #0 {
 ; CHECK-LABEL: f3:
 ; CHECK: meeb %f0, 4092(%r2)
 ; CHECK: br %r14
@@ -42,13 +42,13 @@ define float @f3(float %f1, float *%base) {
   %res = call float @llvm.experimental.constrained.fmul.f32(
                         float %f1, float %f2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
 
 ; Check the next word up, which needs separate address logic.
 ; Other sequences besides this one would be OK.
-define float @f4(float %f1, float *%base) {
+define float @f4(float %f1, float *%base) #0 {
 ; CHECK-LABEL: f4:
 ; CHECK: aghi %r2, 4096
 ; CHECK: meeb %f0, 0(%r2)
@@ -58,12 +58,12 @@ define float @f4(float %f1, float *%base) {
   %res = call float @llvm.experimental.constrained.fmul.f32(
                         float %f1, float %f2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
 
 ; Check negative displacements, which also need separate address logic.
-define float @f5(float %f1, float *%base) {
+define float @f5(float %f1, float *%base) #0 {
 ; CHECK-LABEL: f5:
 ; CHECK: aghi %r2, -4
 ; CHECK: meeb %f0, 0(%r2)
@@ -73,12 +73,12 @@ define float @f5(float %f1, float *%base) {
   %res = call float @llvm.experimental.constrained.fmul.f32(
                         float %f1, float %f2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
 
 ; Check that MEEB allows indices.
-define float @f6(float %f1, float *%base, i64 %index) {
+define float @f6(float %f1, float *%base, i64 %index) #0 {
 ; CHECK-LABEL: f6:
 ; CHECK: sllg %r1, %r3, 2
 ; CHECK: meeb %f0, 400(%r1,%r2)
@@ -89,12 +89,12 @@ define float @f6(float %f1, float *%base, i64 %index) {
   %res = call float @llvm.experimental.constrained.fmul.f32(
                         float %f1, float %f2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
 
 ; Check that multiplications of spilled values can use MEEB rather than MEEBR.
-define float @f7(float *%ptr0) {
+define float @f7(float *%ptr0) #0 {
 ; CHECK-LABEL: f7:
 ; CHECK: brasl %r14, foo@PLT
 ; CHECK-SCALAR: meeb %f0, 16{{[04]}}(%r15)
@@ -122,52 +122,54 @@ define float @f7(float *%ptr0) {
   %val9 = load float, float *%ptr9
   %val10 = load float, float *%ptr10
 
-  %ret = call float @foo()
+  %ret = call float @foo() #0
 
   %mul0 = call float @llvm.experimental.constrained.fmul.f32(
                         float %ret, float %val0,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %mul1 = call float @llvm.experimental.constrained.fmul.f32(
                         float %mul0, float %val1,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %mul2 = call float @llvm.experimental.constrained.fmul.f32(
                         float %mul1, float %val2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %mul3 = call float @llvm.experimental.constrained.fmul.f32(
                         float %mul2, float %val3,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %mul4 = call float @llvm.experimental.constrained.fmul.f32(
                         float %mul3, float %val4,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %mul5 = call float @llvm.experimental.constrained.fmul.f32(
                         float %mul4, float %val5,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %mul6 = call float @llvm.experimental.constrained.fmul.f32(
                         float %mul5, float %val6,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %mul7 = call float @llvm.experimental.constrained.fmul.f32(
                         float %mul6, float %val7,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %mul8 = call float @llvm.experimental.constrained.fmul.f32(
                         float %mul7, float %val8,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %mul9 = call float @llvm.experimental.constrained.fmul.f32(
                         float %mul8, float %val9,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %mul10 = call float @llvm.experimental.constrained.fmul.f32(
                         float %mul9, float %val10,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
 
   ret float %mul10
 }
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-mul-02.ll b/llvm/test/CodeGen/SystemZ/fp-strict-mul-02.ll
index 6f080f6e4..a5cd0b8c0 100644
--- a/llvm/test/CodeGen/SystemZ/fp-strict-mul-02.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-mul-02.ll
@@ -1,93 +1,115 @@
 ; Test strict multiplication of two f32s, producing an f64 result.
-; FIXME: we do not have a strict version of fpext yet
 ;
 ; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
 
 declare float @foo()
 declare double @llvm.experimental.constrained.fmul.f64(double, double, metadata, metadata)
+declare float @llvm.experimental.constrained.fadd.f32(float, float, metadata, metadata)
+declare float @llvm.experimental.constrained.fptrunc.f32.f64(double, metadata, metadata)
+declare double @llvm.experimental.constrained.fpext.f64.f32(float, metadata)
 
 ; Check register multiplication.
-define double @f1(float %f1, float %f2) {
+define double @f1(float %f1, float %f2) #0 {
 ; CHECK-LABEL: f1:
 ; CHECK: mdebr %f0, %f2
 ; CHECK: br %r14
-  %f1x = fpext float %f1 to double
-  %f2x = fpext float %f2 to double
+  %f1x = call double @llvm.experimental.constrained.fpext.f64.f32(
+                        float %f1,
+                        metadata !"fpexcept.strict") #0
+  %f2x = call double @llvm.experimental.constrained.fpext.f64.f32(
+                        float %f2,
+                        metadata !"fpexcept.strict") #0
   %res = call double @llvm.experimental.constrained.fmul.f64(
                         double %f1x, double %f2x,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
 ; Check the low end of the MDEB range.
-define double @f2(float %f1, float *%ptr) {
+define double @f2(float %f1, float *%ptr) #0 {
 ; CHECK-LABEL: f2:
 ; CHECK: mdeb %f0, 0(%r2)
 ; CHECK: br %r14
   %f2 = load float, float *%ptr
-  %f1x = fpext float %f1 to double
-  %f2x = fpext float %f2 to double
+  %f1x = call double @llvm.experimental.constrained.fpext.f64.f32(
+                        float %f1,
+                        metadata !"fpexcept.strict") #0
+  %f2x = call double @llvm.experimental.constrained.fpext.f64.f32(
+                        float %f2,
+                        metadata !"fpexcept.strict") #0
   %res = call double @llvm.experimental.constrained.fmul.f64(
                         double %f1x, double %f2x,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
 ; Check the high end of the aligned MDEB range.
-define double @f3(float %f1, float *%base) {
+define double @f3(float %f1, float *%base) #0 {
 ; CHECK-LABEL: f3:
 ; CHECK: mdeb %f0, 4092(%r2)
 ; CHECK: br %r14
   %ptr = getelementptr float, float *%base, i64 1023
   %f2 = load float, float *%ptr
-  %f1x = fpext float %f1 to double
-  %f2x = fpext float %f2 to double
+  %f1x = call double @llvm.experimental.constrained.fpext.f64.f32(
+                        float %f1,
+                        metadata !"fpexcept.strict") #0
+  %f2x = call double @llvm.experimental.constrained.fpext.f64.f32(
+                        float %f2,
+                        metadata !"fpexcept.strict") #0
   %res = call double @llvm.experimental.constrained.fmul.f64(
                         double %f1x, double %f2x,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
 ; Check the next word up, which needs separate address logic.
 ; Other sequences besides this one would be OK.
-define double @f4(float %f1, float *%base) {
+define double @f4(float %f1, float *%base) #0 {
 ; CHECK-LABEL: f4:
 ; CHECK: aghi %r2, 4096
 ; CHECK: mdeb %f0, 0(%r2)
 ; CHECK: br %r14
   %ptr = getelementptr float, float *%base, i64 1024
   %f2 = load float, float *%ptr
-  %f1x = fpext float %f1 to double
-  %f2x = fpext float %f2 to double
+  %f1x = call double @llvm.experimental.constrained.fpext.f64.f32(
+                        float %f1,
+                        metadata !"fpexcept.strict") #0
+  %f2x = call double @llvm.experimental.constrained.fpext.f64.f32(
+                        float %f2,
+                        metadata !"fpexcept.strict") #0
   %res = call double @llvm.experimental.constrained.fmul.f64(
                         double %f1x, double %f2x,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
 ; Check negative displacements, which also need separate address logic.
-define double @f5(float %f1, float *%base) {
+define double @f5(float %f1, float *%base) #0 {
 ; CHECK-LABEL: f5:
 ; CHECK: aghi %r2, -4
 ; CHECK: mdeb %f0, 0(%r2)
 ; CHECK: br %r14
   %ptr = getelementptr float, float *%base, i64 -1
   %f2 = load float, float *%ptr
-  %f1x = fpext float %f1 to double
-  %f2x = fpext float %f2 to double
+  %f1x = call double @llvm.experimental.constrained.fpext.f64.f32(
+                        float %f1,
+                        metadata !"fpexcept.strict") #0
+  %f2x = call double @llvm.experimental.constrained.fpext.f64.f32(
+                        float %f2,
+                        metadata !"fpexcept.strict") #0
   %res = call double @llvm.experimental.constrained.fmul.f64(
                         double %f1x, double %f2x,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
 ; Check that MDEB allows indices.
-define double @f6(float %f1, float *%base, i64 %index) {
+define double @f6(float %f1, float *%base, i64 %index) #0 {
 ; CHECK-LABEL: f6:
 ; CHECK: sllg %r1, %r3, 2
 ; CHECK: mdeb %f0, 400(%r1,%r2)
@@ -95,17 +117,21 @@ define double @f6(float %f1, float *%base, i64 %index) {
   %ptr1 = getelementptr float, float *%base, i64 %index
   %ptr2 = getelementptr float, float *%ptr1, i64 100
   %f2 = load float, float *%ptr2
-  %f1x = fpext float %f1 to double
-  %f2x = fpext float %f2 to double
+  %f1x = call double @llvm.experimental.constrained.fpext.f64.f32(
+                        float %f1,
+                        metadata !"fpexcept.strict") #0
+  %f2x = call double @llvm.experimental.constrained.fpext.f64.f32(
+                        float %f2,
+                        metadata !"fpexcept.strict") #0
   %res = call double @llvm.experimental.constrained.fmul.f64(
                         double %f1x, double %f2x,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
 ; Check that multiplications of spilled values can use MDEB rather than MDEBR.
-define float @f7(float *%ptr0) {
+define float @f7(float *%ptr0) #0 {
 ; CHECK-LABEL: f7:
 ; CHECK: brasl %r14, foo@PLT
 ; CHECK: mdeb %f0, 16{{[04]}}(%r15)
@@ -133,17 +159,50 @@ define float @f7(float *%ptr0) {
   %val9 = load float, float *%ptr9
   %val10 = load float, float *%ptr10
 
-  %frob0 = fadd float %val0, %val0
-  %frob1 = fadd float %val1, %val1
-  %frob2 = fadd float %val2, %val2
-  %frob3 = fadd float %val3, %val3
-  %frob4 = fadd float %val4, %val4
-  %frob5 = fadd float %val5, %val5
-  %frob6 = fadd float %val6, %val6
-  %frob7 = fadd float %val7, %val7
-  %frob8 = fadd float %val8, %val8
-  %frob9 = fadd float %val9, %val9
-  %frob10 = fadd float %val9, %val10
+  %frob0 = call float @llvm.experimental.constrained.fadd.f32(
+                        float %val0, float %val0,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
+  %frob1 = call float @llvm.experimental.constrained.fadd.f32(
+                        float %val1, float %val1,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
+  %frob2 = call float @llvm.experimental.constrained.fadd.f32(
+                        float %val2, float %val2,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
+  %frob3 = call float @llvm.experimental.constrained.fadd.f32(
+                        float %val3, float %val3,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
+  %frob4 = call float @llvm.experimental.constrained.fadd.f32(
+                        float %val4, float %val4,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
+  %frob5 = call float @llvm.experimental.constrained.fadd.f32(
+                        float %val5, float %val5,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
+  %frob6 = call float @llvm.experimental.constrained.fadd.f32(
+                        float %val6, float %val6,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
+  %frob7 = call float @llvm.experimental.constrained.fadd.f32(
+                        float %val7, float %val7,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
+  %frob8 = call float @llvm.experimental.constrained.fadd.f32(
+                        float %val8, float %val8,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
+  %frob9 = call float @llvm.experimental.constrained.fadd.f32(
+                        float %val9, float %val9,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
+  %frob10 = call float @llvm.experimental.constrained.fadd.f32(
+                        float %val10, float %val10,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
 
   store float %frob0, float *%ptr0
   store float %frob1, float *%ptr1
@@ -157,127 +216,199 @@ define float @f7(float *%ptr0) {
   store float %frob9, float *%ptr9
   store float %frob10, float *%ptr10
 
-  %ret = call float @foo()
+  %ret = call float @foo() #0
 
-  %accext0 = fpext float %ret to double
-  %ext0 = fpext float %frob0 to double
+  %accext0 = call double @llvm.experimental.constrained.fpext.f64.f32(
+                        float %ret,
+                        metadata !"fpexcept.strict") #0
+  %ext0 = call double @llvm.experimental.constrained.fpext.f64.f32(
+                        float %frob0,
+                        metadata !"fpexcept.strict") #0
   %mul0 = call double @llvm.experimental.constrained.fmul.f64(
                         double %accext0, double %ext0,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %extra0 = call double @llvm.experimental.constrained.fmul.f64(
                         double %mul0, double 1.01,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
-  %trunc0 = fptrunc double %extra0 to float
+                        metadata !"fpexcept.strict") #0
+  %trunc0 = call float @llvm.experimental.constrained.fptrunc.f32.f64(
+                        double %extra0,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
 
-  %accext1 = fpext float %trunc0 to double
-  %ext1 = fpext float %frob1 to double
+  %accext1 = call double @llvm.experimental.constrained.fpext.f64.f32(
+                        float %trunc0,
+                        metadata !"fpexcept.strict") #0
+  %ext1 = call double @llvm.experimental.constrained.fpext.f64.f32(
+                        float %frob1,
+                        metadata !"fpexcept.strict") #0
   %mul1 = call double @llvm.experimental.constrained.fmul.f64(
                         double %accext1, double %ext1,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %extra1 = call double @llvm.experimental.constrained.fmul.f64(
                         double %mul1, double 1.11,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
-  %trunc1 = fptrunc double %extra1 to float
+                        metadata !"fpexcept.strict") #0
+  %trunc1 = call float @llvm.experimental.constrained.fptrunc.f32.f64(
+                        double %extra1,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
 
-  %accext2 = fpext float %trunc1 to double
-  %ext2 = fpext float %frob2 to double
+  %accext2 = call double @llvm.experimental.constrained.fpext.f64.f32(
+                        float %trunc1,
+                        metadata !"fpexcept.strict") #0
+  %ext2 = call double @llvm.experimental.constrained.fpext.f64.f32(
+                        float %frob2,
+                        metadata !"fpexcept.strict") #0
   %mul2 = call double @llvm.experimental.constrained.fmul.f64(
                         double %accext2, double %ext2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %extra2 = call double @llvm.experimental.constrained.fmul.f64(
                         double %mul2, double 1.21,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
-  %trunc2 = fptrunc double %extra2 to float
+                        metadata !"fpexcept.strict") #0
+  %trunc2 = call float @llvm.experimental.constrained.fptrunc.f32.f64(
+                        double %extra2,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
 
-  %accext3 = fpext float %trunc2 to double
-  %ext3 = fpext float %frob3 to double
+  %accext3 = call double @llvm.experimental.constrained.fpext.f64.f32(
+                        float %trunc2,
+                        metadata !"fpexcept.strict") #0
+  %ext3 = call double @llvm.experimental.constrained.fpext.f64.f32(
+                        float %frob3,
+                        metadata !"fpexcept.strict") #0
   %mul3 = call double @llvm.experimental.constrained.fmul.f64(
                         double %accext3, double %ext3,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %extra3 = call double @llvm.experimental.constrained.fmul.f64(
                         double %mul3, double 1.31,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
-  %trunc3 = fptrunc double %extra3 to float
+                        metadata !"fpexcept.strict") #0
+  %trunc3 = call float @llvm.experimental.constrained.fptrunc.f32.f64(
+                        double %extra3,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
 
-  %accext4 = fpext float %trunc3 to double
-  %ext4 = fpext float %frob4 to double
+  %accext4 = call double @llvm.experimental.constrained.fpext.f64.f32(
+                        float %trunc3,
+                        metadata !"fpexcept.strict") #0
+  %ext4 = call double @llvm.experimental.constrained.fpext.f64.f32(
+                        float %frob4,
+                        metadata !"fpexcept.strict") #0
   %mul4 = call double @llvm.experimental.constrained.fmul.f64(
                         double %accext4, double %ext4,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %extra4 = call double @llvm.experimental.constrained.fmul.f64(
                         double %mul4, double 1.41,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
-  %trunc4 = fptrunc double %extra4 to float
+                        metadata !"fpexcept.strict") #0
+  %trunc4 = call float @llvm.experimental.constrained.fptrunc.f32.f64(
+                        double %extra4,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
 
-  %accext5 = fpext float %trunc4 to double
-  %ext5 = fpext float %frob5 to double
+  %accext5 = call double @llvm.experimental.constrained.fpext.f64.f32(
+                        float %trunc4,
+                        metadata !"fpexcept.strict") #0
+  %ext5 = call double @llvm.experimental.constrained.fpext.f64.f32(
+                        float %frob5,
+                        metadata !"fpexcept.strict") #0
   %mul5 = call double @llvm.experimental.constrained.fmul.f64(
                         double %accext5, double %ext5,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %extra5 = call double @llvm.experimental.constrained.fmul.f64(
                         double %mul5, double 1.51,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
-  %trunc5 = fptrunc double %extra5 to float
+                        metadata !"fpexcept.strict") #0
+  %trunc5 = call float @llvm.experimental.constrained.fptrunc.f32.f64(
+                        double %extra5,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
 
-  %accext6 = fpext float %trunc5 to double
-  %ext6 = fpext float %frob6 to double
+  %accext6 = call double @llvm.experimental.constrained.fpext.f64.f32(
+                        float %trunc5,
+                        metadata !"fpexcept.strict") #0
+  %ext6 = call double @llvm.experimental.constrained.fpext.f64.f32(
+                        float %frob6,
+                        metadata !"fpexcept.strict") #0
   %mul6 = call double @llvm.experimental.constrained.fmul.f64(
                         double %accext6, double %ext6,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %extra6 = call double @llvm.experimental.constrained.fmul.f64(
                         double %mul6, double 1.61,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
-  %trunc6 = fptrunc double %extra6 to float
+                        metadata !"fpexcept.strict") #0
+  %trunc6 = call float @llvm.experimental.constrained.fptrunc.f32.f64(
+                        double %extra6,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
 
-  %accext7 = fpext float %trunc6 to double
-  %ext7 = fpext float %frob7 to double
+  %accext7 = call double @llvm.experimental.constrained.fpext.f64.f32(
+                        float %trunc6,
+                        metadata !"fpexcept.strict") #0
+  %ext7 = call double @llvm.experimental.constrained.fpext.f64.f32(
+                        float %frob7,
+                        metadata !"fpexcept.strict") #0
   %mul7 = call double @llvm.experimental.constrained.fmul.f64(
                         double %accext7, double %ext7,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %extra7 = call double @llvm.experimental.constrained.fmul.f64(
                         double %mul7, double 1.71,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
-  %trunc7 = fptrunc double %extra7 to float
+                        metadata !"fpexcept.strict") #0
+  %trunc7 = call float @llvm.experimental.constrained.fptrunc.f32.f64(
+                        double %extra7,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
 
-  %accext8 = fpext float %trunc7 to double
-  %ext8 = fpext float %frob8 to double
+  %accext8 = call double @llvm.experimental.constrained.fpext.f64.f32(
+                        float %trunc7,
+                        metadata !"fpexcept.strict") #0
+  %ext8 = call double @llvm.experimental.constrained.fpext.f64.f32(
+                        float %frob8,
+                        metadata !"fpexcept.strict") #0
   %mul8 = call double @llvm.experimental.constrained.fmul.f64(
                         double %accext8, double %ext8,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %extra8 = call double @llvm.experimental.constrained.fmul.f64(
                         double %mul8, double 1.81,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
-  %trunc8 = fptrunc double %extra8 to float
+                        metadata !"fpexcept.strict") #0
+  %trunc8 = call float @llvm.experimental.constrained.fptrunc.f32.f64(
+                        double %extra8,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
 
-  %accext9 = fpext float %trunc8 to double
-  %ext9 = fpext float %frob9 to double
+  %accext9 = call double @llvm.experimental.constrained.fpext.f64.f32(
+                        float %trunc8,
+                        metadata !"fpexcept.strict") #0
+  %ext9 = call double @llvm.experimental.constrained.fpext.f64.f32(
+                        float %frob9,
+                        metadata !"fpexcept.strict") #0
   %mul9 = call double @llvm.experimental.constrained.fmul.f64(
                         double %accext9, double %ext9,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %extra9 = call double @llvm.experimental.constrained.fmul.f64(
                         double %mul9, double 1.91,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
-  %trunc9 = fptrunc double %extra9 to float
+                        metadata !"fpexcept.strict") #0
+  %trunc9 = call float @llvm.experimental.constrained.fptrunc.f32.f64(
+                        double %extra9,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
 
   ret float %trunc9
 }
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-mul-03.ll b/llvm/test/CodeGen/SystemZ/fp-strict-mul-03.ll
index 736bd2e50..edfc5d46b 100644
--- a/llvm/test/CodeGen/SystemZ/fp-strict-mul-03.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-mul-03.ll
@@ -8,19 +8,19 @@ declare double @foo()
 declare double @llvm.experimental.constrained.fmul.f64(double, double, metadata, metadata)
 
 ; Check register multiplication.
-define double @f1(double %f1, double %f2) {
+define double @f1(double %f1, double %f2) #0 {
 ; CHECK-LABEL: f1:
 ; CHECK: mdbr %f0, %f2
 ; CHECK: br %r14
   %res = call double @llvm.experimental.constrained.fmul.f64(
                         double %f1, double %f2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
 ; Check the low end of the MDB range.
-define double @f2(double %f1, double *%ptr) {
+define double @f2(double %f1, double *%ptr) #0 {
 ; CHECK-LABEL: f2:
 ; CHECK: mdb %f0, 0(%r2)
 ; CHECK: br %r14
@@ -28,12 +28,12 @@ define double @f2(double %f1, double *%ptr) {
   %res = call double @llvm.experimental.constrained.fmul.f64(
                         double %f1, double %f2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
 ; Check the high end of the aligned MDB range.
-define double @f3(double %f1, double *%base) {
+define double @f3(double %f1, double *%base) #0 {
 ; CHECK-LABEL: f3:
 ; CHECK: mdb %f0, 4088(%r2)
 ; CHECK: br %r14
@@ -42,13 +42,13 @@ define double @f3(double %f1, double *%base) {
   %res = call double @llvm.experimental.constrained.fmul.f64(
                         double %f1, double %f2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
 ; Check the next doubleword up, which needs separate address logic.
 ; Other sequences besides this one would be OK.
-define double @f4(double %f1, double *%base) {
+define double @f4(double %f1, double *%base) #0 {
 ; CHECK-LABEL: f4:
 ; CHECK: aghi %r2, 4096
 ; CHECK: mdb %f0, 0(%r2)
@@ -58,12 +58,12 @@ define double @f4(double %f1, double *%base) {
   %res = call double @llvm.experimental.constrained.fmul.f64(
                         double %f1, double %f2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
 ; Check negative displacements, which also need separate address logic.
-define double @f5(double %f1, double *%base) {
+define double @f5(double %f1, double *%base) #0 {
 ; CHECK-LABEL: f5:
 ; CHECK: aghi %r2, -8
 ; CHECK: mdb %f0, 0(%r2)
@@ -73,12 +73,12 @@ define double @f5(double %f1, double *%base) {
   %res = call double @llvm.experimental.constrained.fmul.f64(
                         double %f1, double %f2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
 ; Check that MDB allows indices.
-define double @f6(double %f1, double *%base, i64 %index) {
+define double @f6(double %f1, double *%base, i64 %index) #0 {
 ; CHECK-LABEL: f6:
 ; CHECK: sllg %r1, %r3, 3
 ; CHECK: mdb %f0, 800(%r1,%r2)
@@ -89,12 +89,12 @@ define double @f6(double %f1, double *%base, i64 %index) {
   %res = call double @llvm.experimental.constrained.fmul.f64(
                         double %f1, double %f2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
 ; Check that multiplications of spilled values can use MDB rather than MDBR.
-define double @f7(double *%ptr0) {
+define double @f7(double *%ptr0) #0 {
 ; CHECK-LABEL: f7:
 ; CHECK: brasl %r14, foo@PLT
 ; CHECK-SCALAR: mdb %f0, 160(%r15)
@@ -122,52 +122,54 @@ define double @f7(double *%ptr0) {
   %val9 = load double, double *%ptr9
   %val10 = load double, double *%ptr10
 
-  %ret = call double @foo()
+  %ret = call double @foo() #0
 
   %mul0 = call double @llvm.experimental.constrained.fmul.f64(
                         double %ret, double %val0,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %mul1 = call double @llvm.experimental.constrained.fmul.f64(
                         double %mul0, double %val1,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %mul2 = call double @llvm.experimental.constrained.fmul.f64(
                         double %mul1, double %val2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %mul3 = call double @llvm.experimental.constrained.fmul.f64(
                         double %mul2, double %val3,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %mul4 = call double @llvm.experimental.constrained.fmul.f64(
                         double %mul3, double %val4,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %mul5 = call double @llvm.experimental.constrained.fmul.f64(
                         double %mul4, double %val5,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %mul6 = call double @llvm.experimental.constrained.fmul.f64(
                         double %mul5, double %val6,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %mul7 = call double @llvm.experimental.constrained.fmul.f64(
                         double %mul6, double %val7,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %mul8 = call double @llvm.experimental.constrained.fmul.f64(
                         double %mul7, double %val8,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %mul9 = call double @llvm.experimental.constrained.fmul.f64(
                         double %mul8, double %val9,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %mul10 = call double @llvm.experimental.constrained.fmul.f64(
                         double %mul9, double %val10,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
 
   ret double %mul10
 }
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-mul-04.ll b/llvm/test/CodeGen/SystemZ/fp-strict-mul-04.ll
index a61303079..fe41a6506 100644
--- a/llvm/test/CodeGen/SystemZ/fp-strict-mul-04.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-mul-04.ll
@@ -1,51 +1,61 @@
 ; Test strict multiplication of two f64s, producing an f128 result.
-; FIXME: we do not have a strict version of fpext yet
 ;
 ; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
 
 declare fp128 @llvm.experimental.constrained.fmul.f128(fp128, fp128, metadata, metadata)
+declare double @llvm.experimental.constrained.fadd.f64(double, double, metadata, metadata)
+declare double @llvm.experimental.constrained.fptrunc.f64.f128(fp128, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.fpext.f128.f64(double, metadata)
 
 declare double @foo()
 
 ; Check register multiplication.  "mxdbr %f0, %f2" is not valid from LLVM's
 ; point of view, because %f2 is the low register of the FP128 %f0.  Pass the
 ; multiplier in %f4 instead.
-define void @f1(double %f1, double %dummy, double %f2, fp128 *%dst) {
+define void @f1(double %f1, double %dummy, double %f2, fp128 *%dst) #0 {
 ; CHECK-LABEL: f1:
 ; CHECK: mxdbr %f0, %f4
 ; CHECK: std %f0, 0(%r2)
 ; CHECK: std %f2, 8(%r2)
 ; CHECK: br %r14
-  %f1x = fpext double %f1 to fp128
-  %f2x = fpext double %f2 to fp128
+  %f1x = call fp128 @llvm.experimental.constrained.fpext.f128.f64(
+                        double %f1,
+                        metadata !"fpexcept.strict") #0
+  %f2x = call fp128 @llvm.experimental.constrained.fpext.f128.f64(
+                        double %f2,
+                        metadata !"fpexcept.strict") #0
   %res = call fp128 @llvm.experimental.constrained.fmul.f128(
                         fp128 %f1x, fp128 %f2x,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   store fp128 %res, fp128 *%dst
   ret void
 }
 
 ; Check the low end of the MXDB range.
-define void @f2(double %f1, double *%ptr, fp128 *%dst) {
+define void @f2(double %f1, double *%ptr, fp128 *%dst) #0 {
 ; CHECK-LABEL: f2:
 ; CHECK: mxdb %f0, 0(%r2)
 ; CHECK: std %f0, 0(%r3)
 ; CHECK: std %f2, 8(%r3)
 ; CHECK: br %r14
   %f2 = load double, double *%ptr
-  %f1x = fpext double %f1 to fp128
-  %f2x = fpext double %f2 to fp128
+  %f1x = call fp128 @llvm.experimental.constrained.fpext.f128.f64(
+                        double %f1,
+                        metadata !"fpexcept.strict") #0
+  %f2x = call fp128 @llvm.experimental.constrained.fpext.f128.f64(
+                        double %f2,
+                        metadata !"fpexcept.strict") #0
   %res = call fp128 @llvm.experimental.constrained.fmul.f128(
                         fp128 %f1x, fp128 %f2x,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   store fp128 %res, fp128 *%dst
   ret void
 }
 
 ; Check the high end of the aligned MXDB range.
-define void @f3(double %f1, double *%base, fp128 *%dst) {
+define void @f3(double %f1, double *%base, fp128 *%dst) #0 {
 ; CHECK-LABEL: f3:
 ; CHECK: mxdb %f0, 4088(%r2)
 ; CHECK: std %f0, 0(%r3)
@@ -53,19 +63,23 @@ define void @f3(double %f1, double *%base, fp128 *%dst) {
 ; CHECK: br %r14
   %ptr = getelementptr double, double *%base, i64 511
   %f2 = load double, double *%ptr
-  %f1x = fpext double %f1 to fp128
-  %f2x = fpext double %f2 to fp128
+  %f1x = call fp128 @llvm.experimental.constrained.fpext.f128.f64(
+                        double %f1,
+                        metadata !"fpexcept.strict") #0
+  %f2x = call fp128 @llvm.experimental.constrained.fpext.f128.f64(
+                        double %f2,
+                        metadata !"fpexcept.strict") #0
   %res = call fp128 @llvm.experimental.constrained.fmul.f128(
                         fp128 %f1x, fp128 %f2x,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   store fp128 %res, fp128 *%dst
   ret void
 }
 
 ; Check the next doubleword up, which needs separate address logic.
 ; Other sequences besides this one would be OK.
-define void @f4(double %f1, double *%base, fp128 *%dst) {
+define void @f4(double %f1, double *%base, fp128 *%dst) #0 {
 ; CHECK-LABEL: f4:
 ; CHECK: aghi %r2, 4096
 ; CHECK: mxdb %f0, 0(%r2)
@@ -74,18 +88,22 @@ define void @f4(double %f1, double *%base, fp128 *%dst) {
 ; CHECK: br %r14
   %ptr = getelementptr double, double *%base, i64 512
   %f2 = load double, double *%ptr
-  %f1x = fpext double %f1 to fp128
-  %f2x = fpext double %f2 to fp128
+  %f1x = call fp128 @llvm.experimental.constrained.fpext.f128.f64(
+                        double %f1,
+                        metadata !"fpexcept.strict") #0
+  %f2x = call fp128 @llvm.experimental.constrained.fpext.f128.f64(
+                        double %f2,
+                        metadata !"fpexcept.strict") #0
   %res = call fp128 @llvm.experimental.constrained.fmul.f128(
                         fp128 %f1x, fp128 %f2x,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   store fp128 %res, fp128 *%dst
   ret void
 }
 
 ; Check negative displacements, which also need separate address logic.
-define void @f5(double %f1, double *%base, fp128 *%dst) {
+define void @f5(double %f1, double *%base, fp128 *%dst) #0 {
 ; CHECK-LABEL: f5:
 ; CHECK: aghi %r2, -8
 ; CHECK: mxdb %f0, 0(%r2)
@@ -94,18 +112,22 @@ define void @f5(double %f1, double *%base, fp128 *%dst) {
 ; CHECK: br %r14
   %ptr = getelementptr double, double *%base, i64 -1
   %f2 = load double, double *%ptr
-  %f1x = fpext double %f1 to fp128
-  %f2x = fpext double %f2 to fp128
+  %f1x = call fp128 @llvm.experimental.constrained.fpext.f128.f64(
+                        double %f1,
+                        metadata !"fpexcept.strict") #0
+  %f2x = call fp128 @llvm.experimental.constrained.fpext.f128.f64(
+                        double %f2,
+                        metadata !"fpexcept.strict") #0
   %res = call fp128 @llvm.experimental.constrained.fmul.f128(
                         fp128 %f1x, fp128 %f2x,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   store fp128 %res, fp128 *%dst
   ret void
 }
 
 ; Check that MXDB allows indices.
-define void @f6(double %f1, double *%base, i64 %index, fp128 *%dst) {
+define void @f6(double %f1, double *%base, i64 %index, fp128 *%dst) #0 {
 ; CHECK-LABEL: f6:
 ; CHECK: sllg %r1, %r3, 3
 ; CHECK: mxdb %f0, 800(%r1,%r2)
@@ -115,18 +137,22 @@ define void @f6(double %f1, double *%base, i64 %index, fp128 *%dst) {
   %ptr1 = getelementptr double, double *%base, i64 %index
   %ptr2 = getelementptr double, double *%ptr1, i64 100
   %f2 = load double, double *%ptr2
-  %f1x = fpext double %f1 to fp128
-  %f2x = fpext double %f2 to fp128
+  %f1x = call fp128 @llvm.experimental.constrained.fpext.f128.f64(
+                        double %f1,
+                        metadata !"fpexcept.strict") #0
+  %f2x = call fp128 @llvm.experimental.constrained.fpext.f128.f64(
+                        double %f2,
+                        metadata !"fpexcept.strict") #0
   %res = call fp128 @llvm.experimental.constrained.fmul.f128(
                         fp128 %f1x, fp128 %f2x,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   store fp128 %res, fp128 *%dst
   ret void
 }
 
 ; Check that multiplications of spilled values can use MXDB rather than MXDBR.
-define double @f7(double *%ptr0) {
+define double @f7(double *%ptr0) #0 {
 ; CHECK-LABEL: f7:
 ; CHECK: brasl %r14, foo@PLT
 ; CHECK: mxdb %f0, 160(%r15)
@@ -154,17 +180,50 @@ define double @f7(double *%ptr0) {
   %val9 = load double, double *%ptr9
   %val10 = load double, double *%ptr10
 
-  %frob0 = fadd double %val0, %val0
-  %frob1 = fadd double %val1, %val1
-  %frob2 = fadd double %val2, %val2
-  %frob3 = fadd double %val3, %val3
-  %frob4 = fadd double %val4, %val4
-  %frob5 = fadd double %val5, %val5
-  %frob6 = fadd double %val6, %val6
-  %frob7 = fadd double %val7, %val7
-  %frob8 = fadd double %val8, %val8
-  %frob9 = fadd double %val9, %val9
-  %frob10 = fadd double %val9, %val10
+  %frob0 = call double @llvm.experimental.constrained.fadd.f64(
+                        double %val0, double %val0,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
+  %frob1 = call double @llvm.experimental.constrained.fadd.f64(
+                        double %val1, double %val1,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
+  %frob2 = call double @llvm.experimental.constrained.fadd.f64(
+                        double %val2, double %val2,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
+  %frob3 = call double @llvm.experimental.constrained.fadd.f64(
+                        double %val3, double %val3,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
+  %frob4 = call double @llvm.experimental.constrained.fadd.f64(
+                        double %val4, double %val4,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
+  %frob5 = call double @llvm.experimental.constrained.fadd.f64(
+                        double %val5, double %val5,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
+  %frob6 = call double @llvm.experimental.constrained.fadd.f64(
+                        double %val6, double %val6,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
+  %frob7 = call double @llvm.experimental.constrained.fadd.f64(
+                        double %val7, double %val7,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
+  %frob8 = call double @llvm.experimental.constrained.fadd.f64(
+                        double %val8, double %val8,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
+  %frob9 = call double @llvm.experimental.constrained.fadd.f64(
+                        double %val9, double %val9,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
+  %frob10 = call double @llvm.experimental.constrained.fadd.f64(
+                        double %val10, double %val10,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
 
   store double %frob0, double *%ptr0
   store double %frob1, double *%ptr1
@@ -178,137 +237,199 @@ define double @f7(double *%ptr0) {
   store double %frob9, double *%ptr9
   store double %frob10, double *%ptr10
 
-  %ret = call double @foo()
+  %ret = call double @foo() #0
 
-  %accext0 = fpext double %ret to fp128
-  %ext0 = fpext double %frob0 to fp128
+  %accext0 = call fp128 @llvm.experimental.constrained.fpext.f128.f64(
+                        double %ret,
+                        metadata !"fpexcept.strict") #0
+  %ext0 = call fp128 @llvm.experimental.constrained.fpext.f128.f64(
+                        double %frob0,
+                        metadata !"fpexcept.strict") #0
   %mul0 = call fp128 @llvm.experimental.constrained.fmul.f128(
                         fp128 %accext0, fp128 %ext0,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
-  %const0 = fpext double 1.01 to fp128
+                        metadata !"fpexcept.strict") #0
   %extra0 = call fp128 @llvm.experimental.constrained.fmul.f128(
-                        fp128 %mul0, fp128 %const0,
+                        fp128 %mul0, fp128 0xL00000000000000003fff000001000000,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
-  %trunc0 = fptrunc fp128 %extra0 to double
+                        metadata !"fpexcept.strict") #0
+  %trunc0 = call double @llvm.experimental.constrained.fptrunc.f64.f128(
+                        fp128 %extra0,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
 
-  %accext1 = fpext double %trunc0 to fp128
-  %ext1 = fpext double %frob1 to fp128
+  %accext1 = call fp128 @llvm.experimental.constrained.fpext.f128.f64(
+                        double %trunc0,
+                        metadata !"fpexcept.strict") #0
+  %ext1 = call fp128 @llvm.experimental.constrained.fpext.f128.f64(
+                        double %frob1,
+                        metadata !"fpexcept.strict") #0
   %mul1 = call fp128 @llvm.experimental.constrained.fmul.f128(
                         fp128 %accext1, fp128 %ext1,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
-  %const1 = fpext double 1.11 to fp128
+                        metadata !"fpexcept.strict") #0
   %extra1 = call fp128 @llvm.experimental.constrained.fmul.f128(
-                        fp128 %mul1, fp128 %const1,
+                        fp128 %mul1, fp128 0xL00000000000000003fff000002000000,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
+  %trunc1 = call double @llvm.experimental.constrained.fptrunc.f64.f128(
+                        fp128 %extra1,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
-  %trunc1 = fptrunc fp128 %extra1 to double
+                        metadata !"fpexcept.strict") #0
 
-  %accext2 = fpext double %trunc1 to fp128
-  %ext2 = fpext double %frob2 to fp128
+  %accext2 = call fp128 @llvm.experimental.constrained.fpext.f128.f64(
+                        double %trunc1,
+                        metadata !"fpexcept.strict") #0
+  %ext2 = call fp128 @llvm.experimental.constrained.fpext.f128.f64(
+                        double %frob2,
+                        metadata !"fpexcept.strict") #0
   %mul2 = call fp128 @llvm.experimental.constrained.fmul.f128(
                         fp128 %accext2, fp128 %ext2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
-  %const2 = fpext double 1.21 to fp128
+                        metadata !"fpexcept.strict") #0
   %extra2 = call fp128 @llvm.experimental.constrained.fmul.f128(
-                        fp128 %mul2, fp128 %const2,
+                        fp128 %mul2, fp128 0xL00000000000000003fff000003000000,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
+  %trunc2 = call double @llvm.experimental.constrained.fptrunc.f64.f128(
+                        fp128 %extra2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
-  %trunc2 = fptrunc fp128 %extra2 to double
+                        metadata !"fpexcept.strict") #0
 
-  %accext3 = fpext double %trunc2 to fp128
-  %ext3 = fpext double %frob3 to fp128
+  %accext3 = call fp128 @llvm.experimental.constrained.fpext.f128.f64(
+                        double %trunc2,
+                        metadata !"fpexcept.strict") #0
+  %ext3 = call fp128 @llvm.experimental.constrained.fpext.f128.f64(
+                        double %frob3,
+                        metadata !"fpexcept.strict") #0
   %mul3 = call fp128 @llvm.experimental.constrained.fmul.f128(
                         fp128 %accext3, fp128 %ext3,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
-  %const3 = fpext double 1.31 to fp128
+                        metadata !"fpexcept.strict") #0
   %extra3 = call fp128 @llvm.experimental.constrained.fmul.f128(
-                        fp128 %mul3, fp128 %const3,
+                        fp128 %mul3, fp128 0xL00000000000000003fff000004000000,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
-  %trunc3 = fptrunc fp128 %extra3 to double
+                        metadata !"fpexcept.strict") #0
+  %trunc3 = call double @llvm.experimental.constrained.fptrunc.f64.f128(
+                        fp128 %extra3,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
 
-  %accext4 = fpext double %trunc3 to fp128
-  %ext4 = fpext double %frob4 to fp128
+  %accext4 = call fp128 @llvm.experimental.constrained.fpext.f128.f64(
+                        double %trunc3,
+                        metadata !"fpexcept.strict") #0
+  %ext4 = call fp128 @llvm.experimental.constrained.fpext.f128.f64(
+                        double %frob4,
+                        metadata !"fpexcept.strict") #0
   %mul4 = call fp128 @llvm.experimental.constrained.fmul.f128(
                         fp128 %accext4, fp128 %ext4,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
-  %const4 = fpext double 1.41 to fp128
+                        metadata !"fpexcept.strict") #0
   %extra4 = call fp128 @llvm.experimental.constrained.fmul.f128(
-                        fp128 %mul4, fp128 %const4,
+                        fp128 %mul4, fp128 0xL00000000000000003fff000005000000,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
+  %trunc4 = call double @llvm.experimental.constrained.fptrunc.f64.f128(
+                        fp128 %extra4,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
-  %trunc4 = fptrunc fp128 %extra4 to double
+                        metadata !"fpexcept.strict") #0
 
-  %accext5 = fpext double %trunc4 to fp128
-  %ext5 = fpext double %frob5 to fp128
+  %accext5 = call fp128 @llvm.experimental.constrained.fpext.f128.f64(
+                        double %trunc4,
+                        metadata !"fpexcept.strict") #0
+  %ext5 = call fp128 @llvm.experimental.constrained.fpext.f128.f64(
+                        double %frob5,
+                        metadata !"fpexcept.strict") #0
   %mul5 = call fp128 @llvm.experimental.constrained.fmul.f128(
                         fp128 %accext5, fp128 %ext5,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
-  %const5 = fpext double 1.51 to fp128
+                        metadata !"fpexcept.strict") #0
   %extra5 = call fp128 @llvm.experimental.constrained.fmul.f128(
-                        fp128 %mul5, fp128 %const5,
+                        fp128 %mul5, fp128 0xL00000000000000003fff000006000000,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
+  %trunc5 = call double @llvm.experimental.constrained.fptrunc.f64.f128(
+                        fp128 %extra5,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
-  %trunc5 = fptrunc fp128 %extra5 to double
+                        metadata !"fpexcept.strict") #0
 
-  %accext6 = fpext double %trunc5 to fp128
-  %ext6 = fpext double %frob6 to fp128
+  %accext6 = call fp128 @llvm.experimental.constrained.fpext.f128.f64(
+                        double %trunc5,
+                        metadata !"fpexcept.strict") #0
+  %ext6 = call fp128 @llvm.experimental.constrained.fpext.f128.f64(
+                        double %frob6,
+                        metadata !"fpexcept.strict") #0
   %mul6 = call fp128 @llvm.experimental.constrained.fmul.f128(
                         fp128 %accext6, fp128 %ext6,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
-  %const6 = fpext double 1.61 to fp128
+                        metadata !"fpexcept.strict") #0
   %extra6 = call fp128 @llvm.experimental.constrained.fmul.f128(
-                        fp128 %mul6, fp128 %const6,
+                        fp128 %mul6, fp128 0xL00000000000000003fff000007000000,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
-  %trunc6 = fptrunc fp128 %extra6 to double
+                        metadata !"fpexcept.strict") #0
+  %trunc6 = call double @llvm.experimental.constrained.fptrunc.f64.f128(
+                        fp128 %extra6,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
 
-  %accext7 = fpext double %trunc6 to fp128
-  %ext7 = fpext double %frob7 to fp128
+  %accext7 = call fp128 @llvm.experimental.constrained.fpext.f128.f64(
+                        double %trunc6,
+                        metadata !"fpexcept.strict") #0
+  %ext7 = call fp128 @llvm.experimental.constrained.fpext.f128.f64(
+                        double %frob7,
+                        metadata !"fpexcept.strict") #0
   %mul7 = call fp128 @llvm.experimental.constrained.fmul.f128(
                         fp128 %accext7, fp128 %ext7,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
-  %const7 = fpext double 1.71 to fp128
+                        metadata !"fpexcept.strict") #0
   %extra7 = call fp128 @llvm.experimental.constrained.fmul.f128(
-                        fp128 %mul7, fp128 %const7,
+                        fp128 %mul7, fp128 0xL00000000000000003fff000008000000,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
+  %trunc7 = call double @llvm.experimental.constrained.fptrunc.f64.f128(
+                        fp128 %extra7,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
-  %trunc7 = fptrunc fp128 %extra7 to double
+                        metadata !"fpexcept.strict") #0
 
-  %accext8 = fpext double %trunc7 to fp128
-  %ext8 = fpext double %frob8 to fp128
+  %accext8 = call fp128 @llvm.experimental.constrained.fpext.f128.f64(
+                        double %trunc7,
+                        metadata !"fpexcept.strict") #0
+  %ext8 = call fp128 @llvm.experimental.constrained.fpext.f128.f64(
+                        double %frob8,
+                        metadata !"fpexcept.strict") #0
   %mul8 = call fp128 @llvm.experimental.constrained.fmul.f128(
                         fp128 %accext8, fp128 %ext8,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
-  %const8 = fpext double 1.81 to fp128
+                        metadata !"fpexcept.strict") #0
   %extra8 = call fp128 @llvm.experimental.constrained.fmul.f128(
-                        fp128 %mul8, fp128 %const8,
+                        fp128 %mul8, fp128 0xL00000000000000003fff000009000000,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
+  %trunc8 = call double @llvm.experimental.constrained.fptrunc.f64.f128(
+                        fp128 %extra8,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
-  %trunc8 = fptrunc fp128 %extra8 to double
+                        metadata !"fpexcept.strict") #0
 
-  %accext9 = fpext double %trunc8 to fp128
-  %ext9 = fpext double %frob9 to fp128
+  %accext9 = call fp128 @llvm.experimental.constrained.fpext.f128.f64(
+                        double %trunc8,
+                        metadata !"fpexcept.strict") #0
+  %ext9 = call fp128 @llvm.experimental.constrained.fpext.f128.f64(
+                        double %frob9,
+                        metadata !"fpexcept.strict") #0
   %mul9 = call fp128 @llvm.experimental.constrained.fmul.f128(
                         fp128 %accext9, fp128 %ext9,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
-  %const9 = fpext double 1.91 to fp128
+                        metadata !"fpexcept.strict") #0
   %extra9 = call fp128 @llvm.experimental.constrained.fmul.f128(
-                        fp128 %mul9, fp128 %const9,
+                        fp128 %mul9, fp128 0xL00000000000000003fff00000a000000,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
-  %trunc9 = fptrunc fp128 %extra9 to double
+                        metadata !"fpexcept.strict") #0
+  %trunc9 = call double @llvm.experimental.constrained.fptrunc.f64.f128(
+                        fp128 %extra9,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
 
   ret double %trunc9
 }
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-mul-05.ll b/llvm/test/CodeGen/SystemZ/fp-strict-mul-05.ll
index 32bab44bf..422566c86 100644
--- a/llvm/test/CodeGen/SystemZ/fp-strict-mul-05.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-mul-05.ll
@@ -3,9 +3,10 @@
 ; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
 
 declare fp128 @llvm.experimental.constrained.fmul.f128(fp128, fp128, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.fpext.f128.f32(float, metadata)
 
 ; There is no memory form of 128-bit multiplication.
-define void @f1(fp128 *%ptr, float %f2) {
+define void @f1(fp128 *%ptr, float %f2) strictfp {
 ; CHECK-LABEL: f1:
 ; CHECK-DAG: lxebr %f0, %f0
 ; CHECK-DAG: ld %f1, 0(%r2)
@@ -15,11 +16,14 @@ define void @f1(fp128 *%ptr, float %f2) {
 ; CHECK: std %f2, 8(%r2)
 ; CHECK: br %r14
   %f1 = load fp128, fp128 *%ptr
-  %f2x = fpext float %f2 to fp128
+  %f2x = call fp128 @llvm.experimental.constrained.fpext.f128.f32(float %f2,
+                                               metadata !"fpexcept.strict") #0
   %diff = call fp128 @llvm.experimental.constrained.fmul.f128(
                         fp128 %f1, fp128 %f2x,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   store fp128 %diff, fp128 *%ptr
   ret void
 }
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-mul-06.ll b/llvm/test/CodeGen/SystemZ/fp-strict-mul-06.ll
index 0de99aeed..0f6405459 100644
--- a/llvm/test/CodeGen/SystemZ/fp-strict-mul-06.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-mul-06.ll
@@ -5,7 +5,7 @@
 
 declare float @llvm.experimental.constrained.fma.f32(float, float, float, metadata, metadata)
 
-define float @f1(float %f1, float %f2, float %acc) {
+define float @f1(float %f1, float %f2, float %acc) #0 {
 ; CHECK-LABEL: f1:
 ; CHECK-SCALAR: maebr %f4, %f0, %f2
 ; CHECK-SCALAR: ler %f0, %f4
@@ -14,11 +14,11 @@ define float @f1(float %f1, float %f2, float %acc) {
   %res = call float @llvm.experimental.constrained.fma.f32 (
                         float %f1, float %f2, float %acc,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
 
-define float @f2(float %f1, float *%ptr, float %acc) {
+define float @f2(float %f1, float *%ptr, float %acc) #0 {
 ; CHECK-LABEL: f2:
 ; CHECK: maeb %f2, %f0, 0(%r2)
 ; CHECK-SCALAR: ler %f0, %f2
@@ -28,11 +28,11 @@ define float @f2(float %f1, float *%ptr, float %acc) {
   %res = call float @llvm.experimental.constrained.fma.f32 (
                         float %f1, float %f2, float %acc,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
 
-define float @f3(float %f1, float *%base, float %acc) {
+define float @f3(float %f1, float *%base, float %acc) #0 {
 ; CHECK-LABEL: f3:
 ; CHECK: maeb %f2, %f0, 4092(%r2)
 ; CHECK-SCALAR: ler %f0, %f2
@@ -43,11 +43,11 @@ define float @f3(float %f1, float *%base, float %acc) {
   %res = call float @llvm.experimental.constrained.fma.f32 (
                         float %f1, float %f2, float %acc,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
 
-define float @f4(float %f1, float *%base, float %acc) {
+define float @f4(float %f1, float *%base, float %acc) #0 {
 ; The important thing here is that we don't generate an out-of-range
 ; displacement.  Other sequences besides this one would be OK.
 ;
@@ -62,11 +62,11 @@ define float @f4(float %f1, float *%base, float %acc) {
   %res = call float @llvm.experimental.constrained.fma.f32 (
                         float %f1, float %f2, float %acc,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
 
-define float @f5(float %f1, float *%base, float %acc) {
+define float @f5(float %f1, float *%base, float %acc) #0 {
 ; Here too the important thing is that we don't generate an out-of-range
 ; displacement.  Other sequences besides this one would be OK.
 ;
@@ -81,11 +81,11 @@ define float @f5(float %f1, float *%base, float %acc) {
   %res = call float @llvm.experimental.constrained.fma.f32 (
                         float %f1, float %f2, float %acc,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
 
-define float @f6(float %f1, float *%base, i64 %index, float %acc) {
+define float @f6(float %f1, float *%base, i64 %index, float %acc) #0 {
 ; CHECK-LABEL: f6:
 ; CHECK: sllg %r1, %r3, 2
 ; CHECK: maeb %f2, %f0, 0(%r1,%r2)
@@ -97,11 +97,11 @@ define float @f6(float %f1, float *%base, i64 %index, float %acc) {
   %res = call float @llvm.experimental.constrained.fma.f32 (
                         float %f1, float %f2, float %acc,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
 
-define float @f7(float %f1, float *%base, i64 %index, float %acc) {
+define float @f7(float %f1, float *%base, i64 %index, float %acc) #0 {
 ; CHECK-LABEL: f7:
 ; CHECK: sllg %r1, %r3, 2
 ; CHECK: maeb %f2, %f0, 4092({{%r1,%r2|%r2,%r1}})
@@ -114,11 +114,11 @@ define float @f7(float %f1, float *%base, i64 %index, float %acc) {
   %res = call float @llvm.experimental.constrained.fma.f32 (
                         float %f1, float %f2, float %acc,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
 
-define float @f8(float %f1, float *%base, i64 %index, float %acc) {
+define float @f8(float %f1, float *%base, i64 %index, float %acc) #0 {
 ; CHECK-LABEL: f8:
 ; CHECK: sllg %r1, %r3, 2
 ; CHECK: lay %r1, 4096({{%r1,%r2|%r2,%r1}})
@@ -132,6 +132,8 @@ define float @f8(float %f1, float *%base, i64 %index, float %acc) {
   %res = call float @llvm.experimental.constrained.fma.f32 (
                         float %f1, float %f2, float %acc,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-mul-07.ll b/llvm/test/CodeGen/SystemZ/fp-strict-mul-07.ll
index b088aae16..d929fbba0 100644
--- a/llvm/test/CodeGen/SystemZ/fp-strict-mul-07.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-mul-07.ll
@@ -5,7 +5,7 @@
 
 declare double @llvm.experimental.constrained.fma.f64(double %f1, double %f2, double %f3, metadata, metadata)
 
-define double @f1(double %f1, double %f2, double %acc) {
+define double @f1(double %f1, double %f2, double %acc) #0 {
 ; CHECK-LABEL: f1:
 ; CHECK-SCALAR: madbr %f4, %f0, %f2
 ; CHECK-SCALAR: ldr %f0, %f4
@@ -14,11 +14,11 @@ define double @f1(double %f1, double %f2, double %acc) {
   %res = call double @llvm.experimental.constrained.fma.f64 (
                         double %f1, double %f2, double %acc,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
-define double @f2(double %f1, double *%ptr, double %acc) {
+define double @f2(double %f1, double *%ptr, double %acc) #0 {
 ; CHECK-LABEL: f2:
 ; CHECK: madb %f2, %f0, 0(%r2)
 ; CHECK: ldr %f0, %f2
@@ -27,11 +27,11 @@ define double @f2(double %f1, double *%ptr, double %acc) {
   %res = call double @llvm.experimental.constrained.fma.f64 (
                         double %f1, double %f2, double %acc,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
-define double @f3(double %f1, double *%base, double %acc) {
+define double @f3(double %f1, double *%base, double %acc) #0 {
 ; CHECK-LABEL: f3:
 ; CHECK: madb %f2, %f0, 4088(%r2)
 ; CHECK: ldr %f0, %f2
@@ -41,11 +41,11 @@ define double @f3(double %f1, double *%base, double %acc) {
   %res = call double @llvm.experimental.constrained.fma.f64 (
                         double %f1, double %f2, double %acc,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
-define double @f4(double %f1, double *%base, double %acc) {
+define double @f4(double %f1, double *%base, double %acc) #0 {
 ; The important thing here is that we don't generate an out-of-range
 ; displacement.  Other sequences besides this one would be OK.
 ;
@@ -59,11 +59,11 @@ define double @f4(double %f1, double *%base, double %acc) {
   %res = call double @llvm.experimental.constrained.fma.f64 (
                         double %f1, double %f2, double %acc,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
-define double @f5(double %f1, double *%base, double %acc) {
+define double @f5(double %f1, double *%base, double %acc) #0 {
 ; Here too the important thing is that we don't generate an out-of-range
 ; displacement.  Other sequences besides this one would be OK.
 ;
@@ -77,11 +77,11 @@ define double @f5(double %f1, double *%base, double %acc) {
   %res = call double @llvm.experimental.constrained.fma.f64 (
                         double %f1, double %f2, double %acc,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
-define double @f6(double %f1, double *%base, i64 %index, double %acc) {
+define double @f6(double %f1, double *%base, i64 %index, double %acc) #0 {
 ; CHECK-LABEL: f6:
 ; CHECK: sllg %r1, %r3, 3
 ; CHECK: madb %f2, %f0, 0(%r1,%r2)
@@ -92,11 +92,11 @@ define double @f6(double %f1, double *%base, i64 %index, double %acc) {
   %res = call double @llvm.experimental.constrained.fma.f64 (
                         double %f1, double %f2, double %acc,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
-define double @f7(double %f1, double *%base, i64 %index, double %acc) {
+define double @f7(double %f1, double *%base, i64 %index, double %acc) #0 {
 ; CHECK-LABEL: f7:
 ; CHECK: sllg %r1, %r3, 3
 ; CHECK: madb %f2, %f0, 4088({{%r1,%r2|%r2,%r1}})
@@ -108,11 +108,11 @@ define double @f7(double %f1, double *%base, i64 %index, double %acc) {
   %res = call double @llvm.experimental.constrained.fma.f64 (
                         double %f1, double %f2, double %acc,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
-define double @f8(double %f1, double *%base, i64 %index, double %acc) {
+define double @f8(double %f1, double *%base, i64 %index, double %acc) #0 {
 ; CHECK-LABEL: f8:
 ; CHECK: sllg %r1, %r3, 3
 ; CHECK: lay %r1, 4096({{%r1,%r2|%r2,%r1}})
@@ -125,6 +125,8 @@ define double @f8(double %f1, double *%base, i64 %index, double %acc) {
   %res = call double @llvm.experimental.constrained.fma.f64 (
                         double %f1, double %f2, double %acc,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-mul-08.ll b/llvm/test/CodeGen/SystemZ/fp-strict-mul-08.ll
index ea3b7a1db..a92191142 100644
--- a/llvm/test/CodeGen/SystemZ/fp-strict-mul-08.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-mul-08.ll
@@ -5,36 +5,36 @@
 
 declare float @llvm.experimental.constrained.fma.f32(float %f1, float %f2, float %f3, metadata, metadata)
 
-define float @f1(float %f1, float %f2, float %acc) {
+define float @f1(float %f1, float %f2, float %acc) #0 {
 ; CHECK-LABEL: f1:
 ; CHECK-SCALAR: msebr %f4, %f0, %f2
 ; CHECK-SCALAR: ler %f0, %f4
 ; CHECK-VECTOR: wfmssb %f0, %f0, %f2, %f4
 ; CHECK: br %r14
-  %negacc = fsub float -0.0, %acc
+  %negacc = fneg float %acc
   %res = call float @llvm.experimental.constrained.fma.f32 (
                         float %f1, float %f2, float %negacc,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
 
-define float @f2(float %f1, float *%ptr, float %acc) {
+define float @f2(float %f1, float *%ptr, float %acc) #0 {
 ; CHECK-LABEL: f2:
 ; CHECK: mseb %f2, %f0, 0(%r2)
 ; CHECK-SCALAR: ler %f0, %f2
 ; CHECK-VECTOR: ldr %f0, %f2
 ; CHECK: br %r14
   %f2 = load float, float *%ptr
-  %negacc = fsub float -0.0, %acc
+  %negacc = fneg float %acc
   %res = call float @llvm.experimental.constrained.fma.f32 (
                         float %f1, float %f2, float %negacc,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
 
-define float @f3(float %f1, float *%base, float %acc) {
+define float @f3(float %f1, float *%base, float %acc) #0 {
 ; CHECK-LABEL: f3:
 ; CHECK: mseb %f2, %f0, 4092(%r2)
 ; CHECK-SCALAR: ler %f0, %f2
@@ -42,15 +42,15 @@ define float @f3(float %f1, float *%base, float %acc) {
 ; CHECK: br %r14
   %ptr = getelementptr float, float *%base, i64 1023
   %f2 = load float, float *%ptr
-  %negacc = fsub float -0.0, %acc
+  %negacc = fneg float %acc
   %res = call float @llvm.experimental.constrained.fma.f32 (
                         float %f1, float %f2, float %negacc,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
 
-define float @f4(float %f1, float *%base, float %acc) {
+define float @f4(float %f1, float *%base, float %acc) #0 {
 ; The important thing here is that we don't generate an out-of-range
 ; displacement.  Other sequences besides this one would be OK.
 ;
@@ -62,15 +62,15 @@ define float @f4(float %f1, float *%base, float %acc) {
 ; CHECK: br %r14
   %ptr = getelementptr float, float *%base, i64 1024
   %f2 = load float, float *%ptr
-  %negacc = fsub float -0.0, %acc
+  %negacc = fneg float %acc
   %res = call float @llvm.experimental.constrained.fma.f32 (
                         float %f1, float %f2, float %negacc,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
 
-define float @f5(float %f1, float *%base, float %acc) {
+define float @f5(float %f1, float *%base, float %acc) #0 {
 ; Here too the important thing is that we don't generate an out-of-range
 ; displacement.  Other sequences besides this one would be OK.
 ;
@@ -82,15 +82,15 @@ define float @f5(float %f1, float *%base, float %acc) {
 ; CHECK: br %r14
   %ptr = getelementptr float, float *%base, i64 -1
   %f2 = load float, float *%ptr
-  %negacc = fsub float -0.0, %acc
+  %negacc = fneg float %acc
   %res = call float @llvm.experimental.constrained.fma.f32 (
                         float %f1, float %f2, float %negacc,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
 
-define float @f6(float %f1, float *%base, i64 %index, float %acc) {
+define float @f6(float %f1, float *%base, i64 %index, float %acc) #0 {
 ; CHECK-LABEL: f6:
 ; CHECK: sllg %r1, %r3, 2
 ; CHECK: mseb %f2, %f0, 0(%r1,%r2)
@@ -99,15 +99,15 @@ define float @f6(float %f1, float *%base, i64 %index, float %acc) {
 ; CHECK: br %r14
   %ptr = getelementptr float, float *%base, i64 %index
   %f2 = load float, float *%ptr
-  %negacc = fsub float -0.0, %acc
+  %negacc = fneg float %acc
   %res = call float @llvm.experimental.constrained.fma.f32 (
                         float %f1, float %f2, float %negacc,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
 
-define float @f7(float %f1, float *%base, i64 %index, float %acc) {
+define float @f7(float %f1, float *%base, i64 %index, float %acc) #0 {
 ; CHECK-LABEL: f7:
 ; CHECK: sllg %r1, %r3, 2
 ; CHECK: mseb %f2, %f0, 4092({{%r1,%r2|%r2,%r1}})
@@ -117,15 +117,15 @@ define float @f7(float %f1, float *%base, i64 %index, float %acc) {
   %index2 = add i64 %index, 1023
   %ptr = getelementptr float, float *%base, i64 %index2
   %f2 = load float, float *%ptr
-  %negacc = fsub float -0.0, %acc
+  %negacc = fneg float %acc
   %res = call float @llvm.experimental.constrained.fma.f32 (
                         float %f1, float %f2, float %negacc,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
 
-define float @f8(float %f1, float *%base, i64 %index, float %acc) {
+define float @f8(float %f1, float *%base, i64 %index, float %acc) #0 {
 ; CHECK-LABEL: f8:
 ; CHECK: sllg %r1, %r3, 2
 ; CHECK: lay %r1, 4096({{%r1,%r2|%r2,%r1}})
@@ -136,10 +136,12 @@ define float @f8(float %f1, float *%base, i64 %index, float %acc) {
   %index2 = add i64 %index, 1024
   %ptr = getelementptr float, float *%base, i64 %index2
   %f2 = load float, float *%ptr
-  %negacc = fsub float -0.0, %acc
+  %negacc = fneg float %acc
   %res = call float @llvm.experimental.constrained.fma.f32 (
                         float %f1, float %f2, float %negacc,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-mul-09.ll b/llvm/test/CodeGen/SystemZ/fp-strict-mul-09.ll
index e8f6eeb41..ffa7ea823 100644
--- a/llvm/test/CodeGen/SystemZ/fp-strict-mul-09.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-mul-09.ll
@@ -5,50 +5,50 @@
 
 declare double @llvm.experimental.constrained.fma.f64(double %f1, double %f2, double %f3, metadata, metadata)
 
-define double @f1(double %f1, double %f2, double %acc) {
+define double @f1(double %f1, double %f2, double %acc) #0 {
 ; CHECK-LABEL: f1:
 ; CHECK-SCALAR: msdbr %f4, %f0, %f2
 ; CHECK-SCALAR: ldr %f0, %f4
 ; CHECK-VECTOR: wfmsdb %f0, %f0, %f2, %f4
 ; CHECK: br %r14
-  %negacc = fsub double -0.0, %acc
+  %negacc = fneg double %acc
   %res = call double @llvm.experimental.constrained.fma.f64 (
                         double %f1, double %f2, double %negacc,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
-define double @f2(double %f1, double *%ptr, double %acc) {
+define double @f2(double %f1, double *%ptr, double %acc) #0 {
 ; CHECK-LABEL: f2:
 ; CHECK: msdb %f2, %f0, 0(%r2)
 ; CHECK: ldr %f0, %f2
 ; CHECK: br %r14
   %f2 = load double, double *%ptr
-  %negacc = fsub double -0.0, %acc
+  %negacc = fneg double %acc
   %res = call double @llvm.experimental.constrained.fma.f64 (
                         double %f1, double %f2, double %negacc,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
-define double @f3(double %f1, double *%base, double %acc) {
+define double @f3(double %f1, double *%base, double %acc) #0 {
 ; CHECK-LABEL: f3:
 ; CHECK: msdb %f2, %f0, 4088(%r2)
 ; CHECK: ldr %f0, %f2
 ; CHECK: br %r14
   %ptr = getelementptr double, double *%base, i64 511
   %f2 = load double, double *%ptr
-  %negacc = fsub double -0.0, %acc
+  %negacc = fneg double %acc
   %res = call double @llvm.experimental.constrained.fma.f64 (
                         double %f1, double %f2, double %negacc,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
-define double @f4(double %f1, double *%base, double %acc) {
+define double @f4(double %f1, double *%base, double %acc) #0 {
 ; The important thing here is that we don't generate an out-of-range
 ; displacement.  Other sequences besides this one would be OK.
 ;
@@ -59,15 +59,15 @@ define double @f4(double %f1, double *%base, double %acc) {
 ; CHECK: br %r14
   %ptr = getelementptr double, double *%base, i64 512
   %f2 = load double, double *%ptr
-  %negacc = fsub double -0.0, %acc
+  %negacc = fneg double %acc
   %res = call double @llvm.experimental.constrained.fma.f64 (
                         double %f1, double %f2, double %negacc,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
-define double @f5(double %f1, double *%base, double %acc) {
+define double @f5(double %f1, double *%base, double %acc) #0 {
 ; Here too the important thing is that we don't generate an out-of-range
 ; displacement.  Other sequences besides this one would be OK.
 ;
@@ -78,15 +78,15 @@ define double @f5(double %f1, double *%base, double %acc) {
 ; CHECK: br %r14
   %ptr = getelementptr double, double *%base, i64 -1
   %f2 = load double, double *%ptr
-  %negacc = fsub double -0.0, %acc
+  %negacc = fneg double %acc
   %res = call double @llvm.experimental.constrained.fma.f64 (
                         double %f1, double %f2, double %negacc,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
-define double @f6(double %f1, double *%base, i64 %index, double %acc) {
+define double @f6(double %f1, double *%base, i64 %index, double %acc) #0 {
 ; CHECK-LABEL: f6:
 ; CHECK: sllg %r1, %r3, 3
 ; CHECK: msdb %f2, %f0, 0(%r1,%r2)
@@ -94,15 +94,15 @@ define double @f6(double %f1, double *%base, i64 %index, double %acc) {
 ; CHECK: br %r14
   %ptr = getelementptr double, double *%base, i64 %index
   %f2 = load double, double *%ptr
-  %negacc = fsub double -0.0, %acc
+  %negacc = fneg double %acc
   %res = call double @llvm.experimental.constrained.fma.f64 (
                         double %f1, double %f2, double %negacc,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
-define double @f7(double %f1, double *%base, i64 %index, double %acc) {
+define double @f7(double %f1, double *%base, i64 %index, double %acc) #0 {
 ; CHECK-LABEL: f7:
 ; CHECK: sllg %r1, %r3, 3
 ; CHECK: msdb %f2, %f0, 4088({{%r1,%r2|%r2,%r1}})
@@ -111,15 +111,15 @@ define double @f7(double %f1, double *%base, i64 %index, double %acc) {
   %index2 = add i64 %index, 511
   %ptr = getelementptr double, double *%base, i64 %index2
   %f2 = load double, double *%ptr
-  %negacc = fsub double -0.0, %acc
+  %negacc = fneg double %acc
   %res = call double @llvm.experimental.constrained.fma.f64 (
                         double %f1, double %f2, double %negacc,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
-define double @f8(double %f1, double *%base, i64 %index, double %acc) {
+define double @f8(double %f1, double *%base, i64 %index, double %acc) #0 {
 ; CHECK-LABEL: f8:
 ; CHECK: sllg %r1, %r3, 3
 ; CHECK: lay %r1, 4096({{%r1,%r2|%r2,%r1}})
@@ -129,10 +129,12 @@ define double @f8(double %f1, double *%base, i64 %index, double %acc) {
   %index2 = add i64 %index, 512
   %ptr = getelementptr double, double *%base, i64 %index2
   %f2 = load double, double *%ptr
-  %negacc = fsub double -0.0, %acc
+  %negacc = fneg double %acc
   %res = call double @llvm.experimental.constrained.fma.f64 (
                         double %f1, double %f2, double %negacc,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-mul-10.ll b/llvm/test/CodeGen/SystemZ/fp-strict-mul-10.ll
index dc0a4bbcc..f9af9017b 100644
--- a/llvm/test/CodeGen/SystemZ/fp-strict-mul-10.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-mul-10.ll
@@ -3,53 +3,54 @@
 declare double @llvm.experimental.constrained.fma.f64(double %f1, double %f2, double %f3, metadata, metadata)
 declare float @llvm.experimental.constrained.fma.f32(float %f1, float %f2, float %f3, metadata, metadata)
 
-define double @f1(double %f1, double %f2, double %acc) {
+define double @f1(double %f1, double %f2, double %acc) #0 {
 ; CHECK-LABEL: f1:
 ; CHECK: wfnmadb %f0, %f0, %f2, %f4
 ; CHECK: br %r14
   %res = call double @llvm.experimental.constrained.fma.f64 (
                         double %f1, double %f2, double %acc,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
-  %negres = fsub double -0.0, %res
+                        metadata !"fpexcept.strict") #0
+  %negres = fneg double %res
   ret double %negres
 }
 
-define double @f2(double %f1, double %f2, double %acc) {
+define double @f2(double %f1, double %f2, double %acc) #0 {
 ; CHECK-LABEL: f2:
 ; CHECK: wfnmsdb %f0, %f0, %f2, %f4
 ; CHECK: br %r14
-  %negacc = fsub double -0.0, %acc
+  %negacc = fneg double %acc
   %res = call double @llvm.experimental.constrained.fma.f64 (
                         double %f1, double %f2, double %negacc,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
-  %negres = fsub double -0.0, %res
+                        metadata !"fpexcept.strict") #0
+  %negres = fneg double %res
   ret double %negres
 }
 
-define float @f3(float %f1, float %f2, float %acc) {
+define float @f3(float %f1, float %f2, float %acc) #0 {
 ; CHECK-LABEL: f3:
 ; CHECK: wfnmasb %f0, %f0, %f2, %f4
 ; CHECK: br %r14
   %res = call float @llvm.experimental.constrained.fma.f32 (
                         float %f1, float %f2, float %acc,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
-  %negres = fsub float -0.0, %res
+                        metadata !"fpexcept.strict") #0
+  %negres = fneg float %res
   ret float %negres
 }
 
-define float @f4(float %f1, float %f2, float %acc) {
+define float @f4(float %f1, float %f2, float %acc) #0 {
 ; CHECK-LABEL: f4:
 ; CHECK: wfnmssb %f0, %f0, %f2, %f4
 ; CHECK: br %r14
-  %negacc = fsub float -0.0, %acc
+  %negacc = fneg float %acc
   %res = call float @llvm.experimental.constrained.fma.f32 (
                         float %f1, float %f2, float %negacc,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
-  %negres = fsub float -0.0, %res
+                        metadata !"fpexcept.strict") #0
+  %negres = fneg float %res
   ret float %negres
 }
 
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-mul-11.ll b/llvm/test/CodeGen/SystemZ/fp-strict-mul-11.ll
index a8133ad2e..6ad75c302 100644
--- a/llvm/test/CodeGen/SystemZ/fp-strict-mul-11.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-mul-11.ll
@@ -3,8 +3,9 @@
 ; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z14 | FileCheck %s
 
 declare fp128 @llvm.experimental.constrained.fmul.f128(fp128, fp128, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.fpext.f128.f64(double, metadata)
 
-define void @f1(fp128 *%ptr1, fp128 *%ptr2) {
+define void @f1(fp128 *%ptr1, fp128 *%ptr2) #0 {
 ; CHECK-LABEL: f1:
 ; CHECK-DAG: vl [[REG1:%v[0-9]+]], 0(%r2)
 ; CHECK-DAG: vl [[REG2:%v[0-9]+]], 0(%r3)
@@ -16,25 +17,28 @@ define void @f1(fp128 *%ptr1, fp128 *%ptr2) {
   %sum = call fp128 @llvm.experimental.constrained.fmul.f128(
                         fp128 %f1, fp128 %f2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   store fp128 %sum, fp128 *%ptr1
   ret void
 }
 
-define void @f2(double %f1, double %f2, fp128 *%dst) {
+define void @f2(double %f1, double %f2, fp128 *%dst) #0 {
 ; CHECK-LABEL: f2:
 ; CHECK-DAG: wflld [[REG1:%v[0-9]+]], %f0
 ; CHECK-DAG: wflld [[REG2:%v[0-9]+]], %f2
 ; CHECK: wfmxb [[RES:%v[0-9]+]], [[REG1]], [[REG2]]
 ; CHECK: vst [[RES]], 0(%r2)
 ; CHECK: br %r14
-  %f1x = fpext double %f1 to fp128
-  %f2x = fpext double %f2 to fp128
+  %f1x = call fp128 @llvm.experimental.constrained.fpext.f128.f64(double %f1,
+                                               metadata !"fpexcept.strict") #0
+  %f2x = call fp128 @llvm.experimental.constrained.fpext.f128.f64(double %f2,
+                                               metadata !"fpexcept.strict") #0
   %res = call fp128 @llvm.experimental.constrained.fmul.f128(
                         fp128 %f1x, fp128 %f2x,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   store fp128 %res, fp128 *%dst
   ret void
 }
 
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-mul-12.ll b/llvm/test/CodeGen/SystemZ/fp-strict-mul-12.ll
new file mode 100644
index 000000000..3af5efb6c
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-mul-12.ll
@@ -0,0 +1,86 @@
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z14 | FileCheck %s
+
+declare fp128 @llvm.experimental.constrained.fma.f128(fp128 %f1, fp128 %f2, fp128 %f3, metadata, metadata)
+
+define void @f1(fp128 *%ptr1, fp128 *%ptr2, fp128 *%ptr3, fp128 *%dst) #0 {
+; CHECK-LABEL: f1:
+; CHECK-DAG: vl [[REG1:%v[0-9]+]], 0(%r2)
+; CHECK-DAG: vl [[REG2:%v[0-9]+]], 0(%r3)
+; CHECK-DAG: vl [[REG3:%v[0-9]+]], 0(%r4)
+; CHECK: wfmaxb [[RES:%v[0-9]+]], [[REG1]], [[REG2]], [[REG3]]
+; CHECK: vst [[RES]], 0(%r5)
+; CHECK: br %r14
+  %f1 = load fp128, fp128 *%ptr1
+  %f2 = load fp128, fp128 *%ptr2
+  %f3 = load fp128, fp128 *%ptr3
+  %res = call fp128 @llvm.experimental.constrained.fma.f128 (
+                        fp128 %f1, fp128 %f2, fp128 %f3,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
+  store fp128 %res, fp128 *%dst
+  ret void
+}
+
+define void @f2(fp128 *%ptr1, fp128 *%ptr2, fp128 *%ptr3, fp128 *%dst) #0 {
+; CHECK-LABEL: f2:
+; CHECK-DAG: vl [[REG1:%v[0-9]+]], 0(%r2)
+; CHECK-DAG: vl [[REG2:%v[0-9]+]], 0(%r3)
+; CHECK-DAG: vl [[REG3:%v[0-9]+]], 0(%r4)
+; CHECK: wfmsxb [[RES:%v[0-9]+]], [[REG1]], [[REG2]], [[REG3]]
+; CHECK: vst [[RES]], 0(%r5)
+; CHECK: br %r14
+  %f1 = load fp128, fp128 *%ptr1
+  %f2 = load fp128, fp128 *%ptr2
+  %f3 = load fp128, fp128 *%ptr3
+  %neg = fsub fp128 0xL00000000000000008000000000000000, %f3
+  %res = call fp128 @llvm.experimental.constrained.fma.f128 (
+                        fp128 %f1, fp128 %f2, fp128 %neg,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
+  store fp128 %res, fp128 *%dst
+  ret void
+}
+
+define void @f3(fp128 *%ptr1, fp128 *%ptr2, fp128 *%ptr3, fp128 *%dst) #0 {
+; CHECK-LABEL: f3:
+; CHECK-DAG: vl [[REG1:%v[0-9]+]], 0(%r2)
+; CHECK-DAG: vl [[REG2:%v[0-9]+]], 0(%r3)
+; CHECK-DAG: vl [[REG3:%v[0-9]+]], 0(%r4)
+; CHECK: wfnmaxb [[RES:%v[0-9]+]], [[REG1]], [[REG2]], [[REG3]]
+; CHECK: vst [[RES]], 0(%r5)
+; CHECK: br %r14
+  %f1 = load fp128, fp128 *%ptr1
+  %f2 = load fp128, fp128 *%ptr2
+  %f3 = load fp128, fp128 *%ptr3
+  %res = call fp128 @llvm.experimental.constrained.fma.f128 (
+                        fp128 %f1, fp128 %f2, fp128 %f3,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
+  %negres = fsub fp128 0xL00000000000000008000000000000000, %res
+  store fp128 %negres, fp128 *%dst
+  ret void
+}
+
+define void @f4(fp128 *%ptr1, fp128 *%ptr2, fp128 *%ptr3, fp128 *%dst) #0 {
+; CHECK-LABEL: f4:
+; CHECK-DAG: vl [[REG1:%v[0-9]+]], 0(%r2)
+; CHECK-DAG: vl [[REG2:%v[0-9]+]], 0(%r3)
+; CHECK-DAG: vl [[REG3:%v[0-9]+]], 0(%r4)
+; CHECK: wfnmsxb [[RES:%v[0-9]+]], [[REG1]], [[REG2]], [[REG3]]
+; CHECK: vst [[RES]], 0(%r5)
+; CHECK: br %r14
+  %f1 = load fp128, fp128 *%ptr1
+  %f2 = load fp128, fp128 *%ptr2
+  %f3 = load fp128, fp128 *%ptr3
+  %neg = fsub fp128 0xL00000000000000008000000000000000, %f3
+  %res = call fp128 @llvm.experimental.constrained.fma.f128 (
+                        fp128 %f1, fp128 %f2, fp128 %neg,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
+  %negres = fsub fp128 0xL00000000000000008000000000000000, %res
+  store fp128 %negres, fp128 *%dst
+  ret void
+}
+
+attributes #0 = { strictfp }
+
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-mul-13.ll b/llvm/test/CodeGen/SystemZ/fp-strict-mul-13.ll
new file mode 100644
index 000000000..32f609bb2
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-mul-13.ll
@@ -0,0 +1,21 @@
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+declare fp128 @llvm.experimental.constrained.fma.f128(fp128 %f1, fp128 %f2, fp128 %f3, metadata, metadata)
+
+define void @f1(fp128 *%ptr1, fp128 *%ptr2, fp128 *%ptr3, fp128 *%dst) #0 {
+; CHECK-LABEL: f1:
+; CHECK: brasl %r14, fmal
+; CHECK: br %r14
+  %f1 = load fp128, fp128 *%ptr1
+  %f2 = load fp128, fp128 *%ptr2
+  %f3 = load fp128, fp128 *%ptr3
+  %res = call fp128 @llvm.experimental.constrained.fma.f128 (
+                        fp128 %f1, fp128 %f2, fp128 %f3,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
+  store fp128 %res, fp128 *%dst
+  ret void
+}
+
+attributes #0 = { strictfp }
+
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-round-01.ll b/llvm/test/CodeGen/SystemZ/fp-strict-round-01.ll
index dfd2c07dc..5b7005499 100644
--- a/llvm/test/CodeGen/SystemZ/fp-strict-round-01.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-round-01.ll
@@ -4,33 +4,33 @@
 
 ; Test rint for f32.
 declare float @llvm.experimental.constrained.rint.f32(float, metadata, metadata)
-define float @f1(float %f) {
+define float @f1(float %f) #0 {
 ; CHECK-LABEL: f1:
 ; CHECK: fiebr %f0, 0, %f0
 ; CHECK: br %r14
   %res = call float @llvm.experimental.constrained.rint.f32(
                         float %f,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
 
 ; Test rint for f64.
 declare double @llvm.experimental.constrained.rint.f64(double, metadata, metadata)
-define double @f2(double %f) {
+define double @f2(double %f) #0 {
 ; CHECK-LABEL: f2:
 ; CHECK: fidbr %f0, 0, %f0
 ; CHECK: br %r14
   %res = call double @llvm.experimental.constrained.rint.f64(
                         double %f,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
 ; Test rint for f128.
 declare fp128 @llvm.experimental.constrained.rint.f128(fp128, metadata, metadata)
-define void @f3(fp128 *%ptr) {
+define void @f3(fp128 *%ptr) #0 {
 ; CHECK-LABEL: f3:
 ; CHECK: fixbr %f0, 0, %f0
 ; CHECK: br %r14
@@ -38,40 +38,40 @@ define void @f3(fp128 *%ptr) {
   %res = call fp128 @llvm.experimental.constrained.rint.f128(
                         fp128 %src,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   store fp128 %res, fp128 *%ptr
   ret void
 }
 
 ; Test nearbyint for f32.
 declare float @llvm.experimental.constrained.nearbyint.f32(float, metadata, metadata)
-define float @f4(float %f) {
+define float @f4(float %f) #0 {
 ; CHECK-LABEL: f4:
 ; CHECK: brasl %r14, nearbyintf@PLT
 ; CHECK: br %r14
   %res = call float @llvm.experimental.constrained.nearbyint.f32(
                         float %f,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
 
 ; Test nearbyint for f64.
 declare double @llvm.experimental.constrained.nearbyint.f64(double, metadata, metadata)
-define double @f5(double %f) {
+define double @f5(double %f) #0 {
 ; CHECK-LABEL: f5:
 ; CHECK: brasl %r14, nearbyint@PLT
 ; CHECK: br %r14
   %res = call double @llvm.experimental.constrained.nearbyint.f64(
                         double %f,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
 ; Test nearbyint for f128.
 declare fp128 @llvm.experimental.constrained.nearbyint.f128(fp128, metadata, metadata)
-define void @f6(fp128 *%ptr) {
+define void @f6(fp128 *%ptr) #0 {
 ; CHECK-LABEL: f6:
 ; CHECK: brasl %r14, nearbyintl@PLT
 ; CHECK: br %r14
@@ -79,172 +79,161 @@ define void @f6(fp128 *%ptr) {
   %res = call fp128 @llvm.experimental.constrained.nearbyint.f128(
                         fp128 %src,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   store fp128 %res, fp128 *%ptr
   ret void
 }
 
 ; Test floor for f32.
-declare float @llvm.experimental.constrained.floor.f32(float, metadata, metadata)
-define float @f7(float %f) {
+declare float @llvm.experimental.constrained.floor.f32(float, metadata)
+define float @f7(float %f) #0 {
 ; CHECK-LABEL: f7:
 ; CHECK: brasl %r14, floorf@PLT
 ; CHECK: br %r14
   %res = call float @llvm.experimental.constrained.floor.f32(
                         float %f,
-                        metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
 
 ; Test floor for f64.
-declare double @llvm.experimental.constrained.floor.f64(double, metadata, metadata)
-define double @f8(double %f) {
+declare double @llvm.experimental.constrained.floor.f64(double, metadata)
+define double @f8(double %f) #0 {
 ; CHECK-LABEL: f8:
 ; CHECK: brasl %r14, floor@PLT
 ; CHECK: br %r14
   %res = call double @llvm.experimental.constrained.floor.f64(
                         double %f,
-                        metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
 ; Test floor for f128.
-declare fp128 @llvm.experimental.constrained.floor.f128(fp128, metadata, metadata)
-define void @f9(fp128 *%ptr) {
+declare fp128 @llvm.experimental.constrained.floor.f128(fp128, metadata)
+define void @f9(fp128 *%ptr) #0 {
 ; CHECK-LABEL: f9:
 ; CHECK: brasl %r14, floorl@PLT
 ; CHECK: br %r14
   %src = load fp128, fp128 *%ptr
   %res = call fp128 @llvm.experimental.constrained.floor.f128(
                         fp128 %src,
-                        metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   store fp128 %res, fp128 *%ptr
   ret void
 }
 
 ; Test ceil for f32.
-declare float @llvm.experimental.constrained.ceil.f32(float, metadata, metadata)
-define float @f10(float %f) {
+declare float @llvm.experimental.constrained.ceil.f32(float, metadata)
+define float @f10(float %f) #0 {
 ; CHECK-LABEL: f10:
 ; CHECK: brasl %r14, ceilf@PLT
 ; CHECK: br %r14
   %res = call float @llvm.experimental.constrained.ceil.f32(
                         float %f,
-                        metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
 
 ; Test ceil for f64.
-declare double @llvm.experimental.constrained.ceil.f64(double, metadata, metadata)
-define double @f11(double %f) {
+declare double @llvm.experimental.constrained.ceil.f64(double, metadata)
+define double @f11(double %f) #0 {
 ; CHECK-LABEL: f11:
 ; CHECK: brasl %r14, ceil@PLT
 ; CHECK: br %r14
   %res = call double @llvm.experimental.constrained.ceil.f64(
                         double %f,
-                        metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
 ; Test ceil for f128.
-declare fp128 @llvm.experimental.constrained.ceil.f128(fp128, metadata, metadata)
-define void @f12(fp128 *%ptr) {
+declare fp128 @llvm.experimental.constrained.ceil.f128(fp128, metadata)
+define void @f12(fp128 *%ptr) #0 {
 ; CHECK-LABEL: f12:
 ; CHECK: brasl %r14, ceill@PLT
 ; CHECK: br %r14
   %src = load fp128, fp128 *%ptr
   %res = call fp128 @llvm.experimental.constrained.ceil.f128(
                         fp128 %src,
-                        metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   store fp128 %res, fp128 *%ptr
   ret void
 }
 
 ; Test trunc for f32.
-declare float @llvm.experimental.constrained.trunc.f32(float, metadata, metadata)
-define float @f13(float %f) {
+declare float @llvm.experimental.constrained.trunc.f32(float, metadata)
+define float @f13(float %f) #0 {
 ; CHECK-LABEL: f13:
 ; CHECK: brasl %r14, truncf@PLT
 ; CHECK: br %r14
   %res = call float @llvm.experimental.constrained.trunc.f32(
                         float %f,
-                        metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
 
 ; Test trunc for f64.
-declare double @llvm.experimental.constrained.trunc.f64(double, metadata, metadata)
-define double @f14(double %f) {
+declare double @llvm.experimental.constrained.trunc.f64(double, metadata)
+define double @f14(double %f) #0 {
 ; CHECK-LABEL: f14:
 ; CHECK: brasl %r14, trunc@PLT
 ; CHECK: br %r14
   %res = call double @llvm.experimental.constrained.trunc.f64(
                         double %f,
-                        metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
 ; Test trunc for f128.
-declare fp128 @llvm.experimental.constrained.trunc.f128(fp128, metadata, metadata)
-define void @f15(fp128 *%ptr) {
+declare fp128 @llvm.experimental.constrained.trunc.f128(fp128, metadata)
+define void @f15(fp128 *%ptr) #0 {
 ; CHECK-LABEL: f15:
 ; CHECK: brasl %r14, truncl@PLT
 ; CHECK: br %r14
   %src = load fp128, fp128 *%ptr
   %res = call fp128 @llvm.experimental.constrained.trunc.f128(
                         fp128 %src,
-                        metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   store fp128 %res, fp128 *%ptr
   ret void
 }
 
 ; Test round for f32.
-declare float @llvm.experimental.constrained.round.f32(float, metadata, metadata)
-define float @f16(float %f) {
+declare float @llvm.experimental.constrained.round.f32(float, metadata)
+define float @f16(float %f) #0 {
 ; CHECK-LABEL: f16:
 ; CHECK: brasl %r14, roundf@PLT
 ; CHECK: br %r14
   %res = call float @llvm.experimental.constrained.round.f32(
                         float %f,
-                        metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
 
 ; Test round for f64.
-declare double @llvm.experimental.constrained.round.f64(double, metadata, metadata)
-define double @f17(double %f) {
+declare double @llvm.experimental.constrained.round.f64(double, metadata)
+define double @f17(double %f) #0 {
 ; CHECK-LABEL: f17:
 ; CHECK: brasl %r14, round@PLT
 ; CHECK: br %r14
   %res = call double @llvm.experimental.constrained.round.f64(
                         double %f,
-                        metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
 ; Test round for f128.
-declare fp128 @llvm.experimental.constrained.round.f128(fp128, metadata, metadata)
-define void @f18(fp128 *%ptr) {
+declare fp128 @llvm.experimental.constrained.round.f128(fp128, metadata)
+define void @f18(fp128 *%ptr) #0 {
 ; CHECK-LABEL: f18:
 ; CHECK: brasl %r14, roundl@PLT
 ; CHECK: br %r14
   %src = load fp128, fp128 *%ptr
   %res = call fp128 @llvm.experimental.constrained.round.f128(
                         fp128 %src,
-                        metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   store fp128 %res, fp128 *%ptr
   ret void
 }
 
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-round-02.ll b/llvm/test/CodeGen/SystemZ/fp-strict-round-02.ll
index fe1be5769..c51e87bf5 100644
--- a/llvm/test/CodeGen/SystemZ/fp-strict-round-02.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-round-02.ll
@@ -7,20 +7,20 @@
 
 ; Test rint for f32.
 declare float @llvm.experimental.constrained.rint.f32(float, metadata, metadata)
-define float @f1(float %f) {
+define float @f1(float %f) #0 {
 ; CHECK-LABEL: f1:
 ; CHECK: fiebr %f0, 0, %f0
 ; CHECK: br %r14
   %res = call float @llvm.experimental.constrained.rint.f32(
                         float %f,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
 
 ; Test rint for f64.
 declare double @llvm.experimental.constrained.rint.f64(double, metadata, metadata)
-define double @f2(double %f) {
+define double @f2(double %f) #0 {
 ; CHECK-LABEL: f2:
 ; CHECK-SCALAR: fidbr %f0, 0, %f0
 ; CHECK-VECTOR: fidbra %f0, 0, %f0, 0
@@ -28,13 +28,13 @@ define double @f2(double %f) {
   %res = call double @llvm.experimental.constrained.rint.f64(
                         double %f,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
 ; Test rint for f128.
 declare fp128 @llvm.experimental.constrained.rint.f128(fp128, metadata, metadata)
-define void @f3(fp128 *%ptr) {
+define void @f3(fp128 *%ptr) #0 {
 ; CHECK-LABEL: f3:
 ; CHECK: fixbr %f0, 0, %f0
 ; CHECK: br %r14
@@ -42,40 +42,40 @@ define void @f3(fp128 *%ptr) {
   %res = call fp128 @llvm.experimental.constrained.rint.f128(
                         fp128 %src,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   store fp128 %res, fp128 *%ptr
   ret void
 }
 
 ; Test nearbyint for f32.
 declare float @llvm.experimental.constrained.nearbyint.f32(float, metadata, metadata)
-define float @f4(float %f) {
+define float @f4(float %f) #0 {
 ; CHECK-LABEL: f4:
 ; CHECK: fiebra %f0, 0, %f0, 4
 ; CHECK: br %r14
   %res = call float @llvm.experimental.constrained.nearbyint.f32(
                         float %f,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
 
 ; Test nearbyint for f64.
 declare double @llvm.experimental.constrained.nearbyint.f64(double, metadata, metadata)
-define double @f5(double %f) {
+define double @f5(double %f) #0 {
 ; CHECK-LABEL: f5:
 ; CHECK: fidbra %f0, 0, %f0, 4
 ; CHECK: br %r14
   %res = call double @llvm.experimental.constrained.nearbyint.f64(
                         double %f,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
 ; Test nearbyint for f128.
 declare fp128 @llvm.experimental.constrained.nearbyint.f128(fp128, metadata, metadata)
-define void @f6(fp128 *%ptr) {
+define void @f6(fp128 *%ptr) #0 {
 ; CHECK-LABEL: f6:
 ; CHECK: fixbra %f0, 0, %f0, 4
 ; CHECK: br %r14
@@ -83,172 +83,161 @@ define void @f6(fp128 *%ptr) {
   %res = call fp128 @llvm.experimental.constrained.nearbyint.f128(
                         fp128 %src,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   store fp128 %res, fp128 *%ptr
   ret void
 }
 
 ; Test floor for f32.
-declare float @llvm.experimental.constrained.floor.f32(float, metadata, metadata)
-define float @f7(float %f) {
+declare float @llvm.experimental.constrained.floor.f32(float, metadata)
+define float @f7(float %f) #0 {
 ; CHECK-LABEL: f7:
 ; CHECK: fiebra %f0, 7, %f0, 4
 ; CHECK: br %r14
   %res = call float @llvm.experimental.constrained.floor.f32(
                         float %f,
-                        metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
 
 ; Test floor for f64.
-declare double @llvm.experimental.constrained.floor.f64(double, metadata, metadata)
-define double @f8(double %f) {
+declare double @llvm.experimental.constrained.floor.f64(double, metadata)
+define double @f8(double %f) #0 {
 ; CHECK-LABEL: f8:
 ; CHECK: fidbra %f0, 7, %f0, 4
 ; CHECK: br %r14
   %res = call double @llvm.experimental.constrained.floor.f64(
                         double %f,
-                        metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
 ; Test floor for f128.
-declare fp128 @llvm.experimental.constrained.floor.f128(fp128, metadata, metadata)
-define void @f9(fp128 *%ptr) {
+declare fp128 @llvm.experimental.constrained.floor.f128(fp128, metadata)
+define void @f9(fp128 *%ptr) #0 {
 ; CHECK-LABEL: f9:
 ; CHECK: fixbra %f0, 7, %f0, 4
 ; CHECK: br %r14
   %src = load fp128, fp128 *%ptr
   %res = call fp128 @llvm.experimental.constrained.floor.f128(
                         fp128 %src,
-                        metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   store fp128 %res, fp128 *%ptr
   ret void
 }
 
 ; Test ceil for f32.
-declare float @llvm.experimental.constrained.ceil.f32(float, metadata, metadata)
-define float @f10(float %f) {
+declare float @llvm.experimental.constrained.ceil.f32(float, metadata)
+define float @f10(float %f) #0 {
 ; CHECK-LABEL: f10:
 ; CHECK: fiebra %f0, 6, %f0, 4
 ; CHECK: br %r14
   %res = call float @llvm.experimental.constrained.ceil.f32(
                         float %f,
-                        metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
 
 ; Test ceil for f64.
-declare double @llvm.experimental.constrained.ceil.f64(double, metadata, metadata)
-define double @f11(double %f) {
+declare double @llvm.experimental.constrained.ceil.f64(double, metadata)
+define double @f11(double %f) #0 {
 ; CHECK-LABEL: f11:
 ; CHECK: fidbra %f0, 6, %f0, 4
 ; CHECK: br %r14
   %res = call double @llvm.experimental.constrained.ceil.f64(
                         double %f,
-                        metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
 ; Test ceil for f128.
-declare fp128 @llvm.experimental.constrained.ceil.f128(fp128, metadata, metadata)
-define void @f12(fp128 *%ptr) {
+declare fp128 @llvm.experimental.constrained.ceil.f128(fp128, metadata)
+define void @f12(fp128 *%ptr) #0 {
 ; CHECK-LABEL: f12:
 ; CHECK: fixbra %f0, 6, %f0, 4
 ; CHECK: br %r14
   %src = load fp128, fp128 *%ptr
   %res = call fp128 @llvm.experimental.constrained.ceil.f128(
                         fp128 %src,
-                        metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   store fp128 %res, fp128 *%ptr
   ret void
 }
 
 ; Test trunc for f32.
-declare float @llvm.experimental.constrained.trunc.f32(float, metadata, metadata)
-define float @f13(float %f) {
+declare float @llvm.experimental.constrained.trunc.f32(float, metadata)
+define float @f13(float %f) #0 {
 ; CHECK-LABEL: f13:
 ; CHECK: fiebra %f0, 5, %f0, 4
 ; CHECK: br %r14
   %res = call float @llvm.experimental.constrained.trunc.f32(
                         float %f,
-                        metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
 
 ; Test trunc for f64.
-declare double @llvm.experimental.constrained.trunc.f64(double, metadata, metadata)
-define double @f14(double %f) {
+declare double @llvm.experimental.constrained.trunc.f64(double, metadata)
+define double @f14(double %f) #0 {
 ; CHECK-LABEL: f14:
 ; CHECK: fidbra %f0, 5, %f0, 4
 ; CHECK: br %r14
   %res = call double @llvm.experimental.constrained.trunc.f64(
                         double %f,
-                        metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
 ; Test trunc for f128.
-declare fp128 @llvm.experimental.constrained.trunc.f128(fp128, metadata, metadata)
-define void @f15(fp128 *%ptr) {
+declare fp128 @llvm.experimental.constrained.trunc.f128(fp128, metadata)
+define void @f15(fp128 *%ptr) #0 {
 ; CHECK-LABEL: f15:
 ; CHECK: fixbra %f0, 5, %f0, 4
 ; CHECK: br %r14
   %src = load fp128, fp128 *%ptr
   %res = call fp128 @llvm.experimental.constrained.trunc.f128(
                         fp128 %src,
-                        metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   store fp128 %res, fp128 *%ptr
   ret void
 }
 
 ; Test round for f32.
-declare float @llvm.experimental.constrained.round.f32(float, metadata, metadata)
-define float @f16(float %f) {
+declare float @llvm.experimental.constrained.round.f32(float, metadata)
+define float @f16(float %f) #0 {
 ; CHECK-LABEL: f16:
 ; CHECK: fiebra %f0, 1, %f0, 4
 ; CHECK: br %r14
   %res = call float @llvm.experimental.constrained.round.f32(
                         float %f,
-                        metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
 
 ; Test round for f64.
-declare double @llvm.experimental.constrained.round.f64(double, metadata, metadata)
-define double @f17(double %f) {
+declare double @llvm.experimental.constrained.round.f64(double, metadata)
+define double @f17(double %f) #0 {
 ; CHECK-LABEL: f17:
 ; CHECK: fidbra %f0, 1, %f0, 4
 ; CHECK: br %r14
   %res = call double @llvm.experimental.constrained.round.f64(
                         double %f,
-                        metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
 ; Test round for f128.
-declare fp128 @llvm.experimental.constrained.round.f128(fp128, metadata, metadata)
-define void @f18(fp128 *%ptr) {
+declare fp128 @llvm.experimental.constrained.round.f128(fp128, metadata)
+define void @f18(fp128 *%ptr) #0 {
 ; CHECK-LABEL: f18:
 ; CHECK: fixbra %f0, 1, %f0, 4
 ; CHECK: br %r14
   %src = load fp128, fp128 *%ptr
   %res = call fp128 @llvm.experimental.constrained.round.f128(
                         fp128 %src,
-                        metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   store fp128 %res, fp128 *%ptr
   ret void
 }
 
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-round-03.ll b/llvm/test/CodeGen/SystemZ/fp-strict-round-03.ll
index e1f634b9a..4b452ceb1 100644
--- a/llvm/test/CodeGen/SystemZ/fp-strict-round-03.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-round-03.ll
@@ -4,33 +4,33 @@
 
 ; Test rint for f32.
 declare float @llvm.experimental.constrained.rint.f32(float, metadata, metadata)
-define float @f1(float %f) {
+define float @f1(float %f) #0 {
 ; CHECK-LABEL: f1:
 ; CHECK: fiebra %f0, 0, %f0, 0
 ; CHECK: br %r14
   %res = call float @llvm.experimental.constrained.rint.f32(
                         float %f,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
 
 ; Test rint for f64.
 declare double @llvm.experimental.constrained.rint.f64(double, metadata, metadata)
-define double @f2(double %f) {
+define double @f2(double %f) #0 {
 ; CHECK-LABEL: f2:
 ; CHECK: fidbra %f0, 0, %f0, 0
 ; CHECK: br %r14
   %res = call double @llvm.experimental.constrained.rint.f64(
                         double %f,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
 ; Test rint for f128.
 declare fp128 @llvm.experimental.constrained.rint.f128(fp128, metadata, metadata)
-define void @f3(fp128 *%ptr) {
+define void @f3(fp128 *%ptr) #0 {
 ; CHECK-LABEL: f3:
 ; CHECK: vl [[REG:%v[0-9]+]], 0(%r2)
 ; CHECK: wfixb [[RES:%v[0-9]+]], [[REG]], 0, 0
@@ -40,40 +40,40 @@ define void @f3(fp128 *%ptr) {
   %res = call fp128 @llvm.experimental.constrained.rint.f128(
                         fp128 %src,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   store fp128 %res, fp128 *%ptr
   ret void
 }
 
 ; Test nearbyint for f32.
 declare float @llvm.experimental.constrained.nearbyint.f32(float, metadata, metadata)
-define float @f4(float %f) {
+define float @f4(float %f) #0 {
 ; CHECK-LABEL: f4:
 ; CHECK: fiebra %f0, 0, %f0, 4
 ; CHECK: br %r14
   %res = call float @llvm.experimental.constrained.nearbyint.f32(
                         float %f,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
 
 ; Test nearbyint for f64.
 declare double @llvm.experimental.constrained.nearbyint.f64(double, metadata, metadata)
-define double @f5(double %f) {
+define double @f5(double %f) #0 {
 ; CHECK-LABEL: f5:
 ; CHECK: fidbra %f0, 0, %f0, 4
 ; CHECK: br %r14
   %res = call double @llvm.experimental.constrained.nearbyint.f64(
                         double %f,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
 ; Test nearbyint for f128.
 declare fp128 @llvm.experimental.constrained.nearbyint.f128(fp128, metadata, metadata)
-define void @f6(fp128 *%ptr) {
+define void @f6(fp128 *%ptr) #0 {
 ; CHECK-LABEL: f6:
 ; CHECK: vl [[REG:%v[0-9]+]], 0(%r2)
 ; CHECK: wfixb [[RES:%v[0-9]+]], [[REG]], 4, 0
@@ -83,40 +83,38 @@ define void @f6(fp128 *%ptr) {
   %res = call fp128 @llvm.experimental.constrained.nearbyint.f128(
                         fp128 %src,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   store fp128 %res, fp128 *%ptr
   ret void
 }
 
 ; Test floor for f32.
-declare float @llvm.experimental.constrained.floor.f32(float, metadata, metadata)
-define float @f7(float %f) {
+declare float @llvm.experimental.constrained.floor.f32(float, metadata)
+define float @f7(float %f) #0 {
 ; CHECK-LABEL: f7:
 ; CHECK: fiebra %f0, 7, %f0, 4
 ; CHECK: br %r14
   %res = call float @llvm.experimental.constrained.floor.f32(
                         float %f,
-                        metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
 
 ; Test floor for f64.
-declare double @llvm.experimental.constrained.floor.f64(double, metadata, metadata)
-define double @f8(double %f) {
+declare double @llvm.experimental.constrained.floor.f64(double, metadata)
+define double @f8(double %f) #0 {
 ; CHECK-LABEL: f8:
 ; CHECK: fidbra %f0, 7, %f0, 4
 ; CHECK: br %r14
   %res = call double @llvm.experimental.constrained.floor.f64(
                         double %f,
-                        metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
 ; Test floor for f128.
-declare fp128 @llvm.experimental.constrained.floor.f128(fp128, metadata, metadata)
-define void @f9(fp128 *%ptr) {
+declare fp128 @llvm.experimental.constrained.floor.f128(fp128, metadata)
+define void @f9(fp128 *%ptr) #0 {
 ; CHECK-LABEL: f9:
 ; CHECK: vl [[REG:%v[0-9]+]], 0(%r2)
 ; CHECK: wfixb [[RES:%v[0-9]+]], [[REG]], 4, 7
@@ -125,41 +123,38 @@ define void @f9(fp128 *%ptr) {
   %src = load fp128, fp128 *%ptr
   %res = call fp128 @llvm.experimental.constrained.floor.f128(
                         fp128 %src,
-                        metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   store fp128 %res, fp128 *%ptr
   ret void
 }
 
 ; Test ceil for f32.
-declare float @llvm.experimental.constrained.ceil.f32(float, metadata, metadata)
-define float @f10(float %f) {
+declare float @llvm.experimental.constrained.ceil.f32(float, metadata)
+define float @f10(float %f) #0 {
 ; CHECK-LABEL: f10:
 ; CHECK: fiebra %f0, 6, %f0, 4
 ; CHECK: br %r14
   %res = call float @llvm.experimental.constrained.ceil.f32(
                         float %f,
-                        metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
 
 ; Test ceil for f64.
-declare double @llvm.experimental.constrained.ceil.f64(double, metadata, metadata)
-define double @f11(double %f) {
+declare double @llvm.experimental.constrained.ceil.f64(double, metadata)
+define double @f11(double %f) #0 {
 ; CHECK-LABEL: f11:
 ; CHECK: fidbra %f0, 6, %f0, 4
 ; CHECK: br %r14
   %res = call double @llvm.experimental.constrained.ceil.f64(
                         double %f,
-                        metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
 ; Test ceil for f128.
-declare fp128 @llvm.experimental.constrained.ceil.f128(fp128, metadata, metadata)
-define void @f12(fp128 *%ptr) {
+declare fp128 @llvm.experimental.constrained.ceil.f128(fp128, metadata)
+define void @f12(fp128 *%ptr) #0 {
 ; CHECK-LABEL: f12:
 ; CHECK: vl [[REG:%v[0-9]+]], 0(%r2)
 ; CHECK: wfixb [[RES:%v[0-9]+]], [[REG]], 4, 6
@@ -168,41 +163,38 @@ define void @f12(fp128 *%ptr) {
   %src = load fp128, fp128 *%ptr
   %res = call fp128 @llvm.experimental.constrained.ceil.f128(
                         fp128 %src,
-                        metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   store fp128 %res, fp128 *%ptr
   ret void
 }
 
 ; Test trunc for f32.
-declare float @llvm.experimental.constrained.trunc.f32(float, metadata, metadata)
-define float @f13(float %f) {
+declare float @llvm.experimental.constrained.trunc.f32(float, metadata)
+define float @f13(float %f) #0 {
 ; CHECK-LABEL: f13:
 ; CHECK: fiebra %f0, 5, %f0, 4
 ; CHECK: br %r14
   %res = call float @llvm.experimental.constrained.trunc.f32(
                         float %f,
-                        metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
 
 ; Test trunc for f64.
-declare double @llvm.experimental.constrained.trunc.f64(double, metadata, metadata)
-define double @f14(double %f) {
+declare double @llvm.experimental.constrained.trunc.f64(double, metadata)
+define double @f14(double %f) #0 {
 ; CHECK-LABEL: f14:
 ; CHECK: fidbra %f0, 5, %f0, 4
 ; CHECK: br %r14
   %res = call double @llvm.experimental.constrained.trunc.f64(
                         double %f,
-                        metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
 ; Test trunc for f128.
-declare fp128 @llvm.experimental.constrained.trunc.f128(fp128, metadata, metadata)
-define void @f15(fp128 *%ptr) {
+declare fp128 @llvm.experimental.constrained.trunc.f128(fp128, metadata)
+define void @f15(fp128 *%ptr) #0 {
 ; CHECK-LABEL: f15:
 ; CHECK: vl [[REG:%v[0-9]+]], 0(%r2)
 ; CHECK: wfixb [[RES:%v[0-9]+]], [[REG]], 4, 5
@@ -211,41 +203,38 @@ define void @f15(fp128 *%ptr) {
   %src = load fp128, fp128 *%ptr
   %res = call fp128 @llvm.experimental.constrained.trunc.f128(
                         fp128 %src,
-                        metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   store fp128 %res, fp128 *%ptr
   ret void
 }
 
 ; Test round for f32.
-declare float @llvm.experimental.constrained.round.f32(float, metadata, metadata)
-define float @f16(float %f) {
+declare float @llvm.experimental.constrained.round.f32(float, metadata)
+define float @f16(float %f) #0 {
 ; CHECK-LABEL: f16:
 ; CHECK: fiebra %f0, 1, %f0, 4
 ; CHECK: br %r14
   %res = call float @llvm.experimental.constrained.round.f32(
                         float %f,
-                        metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
 
 ; Test round for f64.
-declare double @llvm.experimental.constrained.round.f64(double, metadata, metadata)
-define double @f17(double %f) {
+declare double @llvm.experimental.constrained.round.f64(double, metadata)
+define double @f17(double %f) #0 {
 ; CHECK-LABEL: f17:
 ; CHECK: fidbra %f0, 1, %f0, 4
 ; CHECK: br %r14
   %res = call double @llvm.experimental.constrained.round.f64(
                         double %f,
-                        metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
 ; Test round for f128.
-declare fp128 @llvm.experimental.constrained.round.f128(fp128, metadata, metadata)
-define void @f18(fp128 *%ptr) {
+declare fp128 @llvm.experimental.constrained.round.f128(fp128, metadata)
+define void @f18(fp128 *%ptr) #0 {
 ; CHECK-LABEL: f18:
 ; CHECK: vl [[REG:%v[0-9]+]], 0(%r2)
 ; CHECK: wfixb [[RES:%v[0-9]+]], [[REG]], 4, 1
@@ -254,9 +243,9 @@ define void @f18(fp128 *%ptr) {
   %src = load fp128, fp128 *%ptr
   %res = call fp128 @llvm.experimental.constrained.round.f128(
                         fp128 %src,
-                        metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   store fp128 %res, fp128 *%ptr
   ret void
 }
 
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-sqrt-01.ll b/llvm/test/CodeGen/SystemZ/fp-strict-sqrt-01.ll
index 158308c49..cff839506 100644
--- a/llvm/test/CodeGen/SystemZ/fp-strict-sqrt-01.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-sqrt-01.ll
@@ -7,19 +7,19 @@
 declare float @llvm.experimental.constrained.sqrt.f32(float, metadata, metadata)
 
 ; Check register square root.
-define float @f1(float %val) {
+define float @f1(float %val) #0 {
 ; CHECK-LABEL: f1:
 ; CHECK: sqebr %f0, %f0
 ; CHECK: br %r14
   %res = call float @llvm.experimental.constrained.sqrt.f32(
                         float %val,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
 
 ; Check the low end of the SQEB range.
-define float @f2(float *%ptr) {
+define float @f2(float *%ptr) #0 {
 ; CHECK-LABEL: f2:
 ; CHECK: sqeb %f0, 0(%r2)
 ; CHECK: br %r14
@@ -27,12 +27,12 @@ define float @f2(float *%ptr) {
   %res = call float @llvm.experimental.constrained.sqrt.f32(
                         float %val,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
 
 ; Check the high end of the aligned SQEB range.
-define float @f3(float *%base) {
+define float @f3(float *%base) #0 {
 ; CHECK-LABEL: f3:
 ; CHECK: sqeb %f0, 4092(%r2)
 ; CHECK: br %r14
@@ -41,13 +41,13 @@ define float @f3(float *%base) {
   %res = call float @llvm.experimental.constrained.sqrt.f32(
                         float %val,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
 
 ; Check the next word up, which needs separate address logic.
 ; Other sequences besides this one would be OK.
-define float @f4(float *%base) {
+define float @f4(float *%base) #0 {
 ; CHECK-LABEL: f4:
 ; CHECK: aghi %r2, 4096
 ; CHECK: sqeb %f0, 0(%r2)
@@ -57,12 +57,12 @@ define float @f4(float *%base) {
   %res = call float @llvm.experimental.constrained.sqrt.f32(
                         float %val,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
 
 ; Check negative displacements, which also need separate address logic.
-define float @f5(float *%base) {
+define float @f5(float *%base) #0 {
 ; CHECK-LABEL: f5:
 ; CHECK: aghi %r2, -4
 ; CHECK: sqeb %f0, 0(%r2)
@@ -72,12 +72,12 @@ define float @f5(float *%base) {
   %res = call float @llvm.experimental.constrained.sqrt.f32(
                         float %val,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
 
 ; Check that SQEB allows indices.
-define float @f6(float *%base, i64 %index) {
+define float @f6(float *%base, i64 %index) #0 {
 ; CHECK-LABEL: f6:
 ; CHECK: sllg %r1, %r3, 2
 ; CHECK: sqeb %f0, 400(%r1,%r2)
@@ -88,7 +88,8 @@ define float @f6(float *%base, i64 %index) {
   %res = call float @llvm.experimental.constrained.sqrt.f32(
                         float %val,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
 
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-sqrt-02.ll b/llvm/test/CodeGen/SystemZ/fp-strict-sqrt-02.ll
index 4e90939d2..791c39301 100644
--- a/llvm/test/CodeGen/SystemZ/fp-strict-sqrt-02.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-sqrt-02.ll
@@ -7,19 +7,19 @@
 declare double @llvm.experimental.constrained.sqrt.f64(double, metadata, metadata)
 
 ; Check register square root.
-define double @f1(double %val) {
+define double @f1(double %val) #0 {
 ; CHECK-LABEL: f1:
 ; CHECK: sqdbr %f0, %f0
 ; CHECK: br %r14
   %res = call double @llvm.experimental.constrained.sqrt.f64(
                         double %val,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
 ; Check the low end of the SQDB range.
-define double @f2(double *%ptr) {
+define double @f2(double *%ptr) #0 {
 ; CHECK-LABEL: f2:
 ; CHECK: sqdb %f0, 0(%r2)
 ; CHECK: br %r14
@@ -27,12 +27,12 @@ define double @f2(double *%ptr) {
   %res = call double @llvm.experimental.constrained.sqrt.f64(
                         double %val,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
 ; Check the high end of the aligned SQDB range.
-define double @f3(double *%base) {
+define double @f3(double *%base) #0 {
 ; CHECK-LABEL: f3:
 ; CHECK: sqdb %f0, 4088(%r2)
 ; CHECK: br %r14
@@ -41,13 +41,13 @@ define double @f3(double *%base) {
   %res = call double @llvm.experimental.constrained.sqrt.f64(
                         double %val,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
 ; Check the next doubleword up, which needs separate address logic.
 ; Other sequences besides this one would be OK.
-define double @f4(double *%base) {
+define double @f4(double *%base) #0 {
 ; CHECK-LABEL: f4:
 ; CHECK: aghi %r2, 4096
 ; CHECK: sqdb %f0, 0(%r2)
@@ -57,12 +57,12 @@ define double @f4(double *%base) {
   %res = call double @llvm.experimental.constrained.sqrt.f64(
                         double %val,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
 ; Check negative displacements, which also need separate address logic.
-define double @f5(double *%base) {
+define double @f5(double *%base) #0 {
 ; CHECK-LABEL: f5:
 ; CHECK: aghi %r2, -8
 ; CHECK: sqdb %f0, 0(%r2)
@@ -72,12 +72,12 @@ define double @f5(double *%base) {
   %res = call double @llvm.experimental.constrained.sqrt.f64(
                         double %val,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
 ; Check that SQDB allows indices.
-define double @f6(double *%base, i64 %index) {
+define double @f6(double *%base, i64 %index) #0 {
 ; CHECK-LABEL: f6:
 ; CHECK: sllg %r1, %r3, 3
 ; CHECK: sqdb %f0, 800(%r1,%r2)
@@ -88,7 +88,8 @@ define double @f6(double *%base, i64 %index) {
   %res = call double @llvm.experimental.constrained.sqrt.f64(
                         double %val,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-sqrt-03.ll b/llvm/test/CodeGen/SystemZ/fp-strict-sqrt-03.ll
index 9bcceb74f..0f2f27293 100644
--- a/llvm/test/CodeGen/SystemZ/fp-strict-sqrt-03.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-sqrt-03.ll
@@ -5,7 +5,7 @@
 declare fp128 @llvm.experimental.constrained.sqrt.f128(fp128, metadata, metadata)
 
 ; There's no memory form of SQXBR.
-define void @f1(fp128 *%ptr) {
+define void @f1(fp128 *%ptr) strictfp {
 ; CHECK-LABEL: f1:
 ; CHECK: ld %f0, 0(%r2)
 ; CHECK: ld %f2, 8(%r2)
@@ -17,7 +17,7 @@ define void @f1(fp128 *%ptr) {
   %sqrt = call fp128 @llvm.experimental.constrained.sqrt.f128(
                         fp128 %orig,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") strictfp
   store fp128 %sqrt, fp128 *%ptr
   ret void
 }
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-sqrt-04.ll b/llvm/test/CodeGen/SystemZ/fp-strict-sqrt-04.ll
index f24f958bc..0667aeb28 100644
--- a/llvm/test/CodeGen/SystemZ/fp-strict-sqrt-04.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-sqrt-04.ll
@@ -4,7 +4,7 @@
 
 declare fp128 @llvm.experimental.constrained.sqrt.f128(fp128, metadata, metadata)
 
-define void @f1(fp128 *%ptr) {
+define void @f1(fp128 *%ptr) strictfp {
 ; CHECK-LABEL: f1:
 ; CHECK-DAG: vl [[REG:%v[0-9]+]], 0(%r2)
 ; CHECK: wfsqxb [[RES:%v[0-9]+]], [[REG]]
@@ -14,7 +14,7 @@ define void @f1(fp128 *%ptr) {
   %res = call fp128 @llvm.experimental.constrained.sqrt.f128(
                         fp128 %f,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") strictfp
   store fp128 %res, fp128 *%ptr
   ret void
 }
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-sub-01.ll b/llvm/test/CodeGen/SystemZ/fp-strict-sub-01.ll
index a4c485a2f..82156e485 100644
--- a/llvm/test/CodeGen/SystemZ/fp-strict-sub-01.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-sub-01.ll
@@ -8,19 +8,19 @@ declare float @foo()
 declare float @llvm.experimental.constrained.fsub.f32(float, float, metadata, metadata)
 
 ; Check register subtraction.
-define float @f1(float %f1, float %f2) {
+define float @f1(float %f1, float %f2) #0 {
 ; CHECK-LABEL: f1:
 ; CHECK: sebr %f0, %f2
 ; CHECK: br %r14
   %res = call float @llvm.experimental.constrained.fsub.f32(
                         float %f1, float %f2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
 
 ; Check the low end of the SEB range.
-define float @f2(float %f1, float *%ptr) {
+define float @f2(float %f1, float *%ptr) #0 {
 ; CHECK-LABEL: f2:
 ; CHECK: seb %f0, 0(%r2)
 ; CHECK: br %r14
@@ -28,12 +28,12 @@ define float @f2(float %f1, float *%ptr) {
   %res = call float @llvm.experimental.constrained.fsub.f32(
                         float %f1, float %f2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
 
 ; Check the high end of the aligned SEB range.
-define float @f3(float %f1, float *%base) {
+define float @f3(float %f1, float *%base) #0 {
 ; CHECK-LABEL: f3:
 ; CHECK: seb %f0, 4092(%r2)
 ; CHECK: br %r14
@@ -42,13 +42,13 @@ define float @f3(float %f1, float *%base) {
   %res = call float @llvm.experimental.constrained.fsub.f32(
                         float %f1, float %f2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
 
 ; Check the next word up, which needs separate address logic.
 ; Other sequences besides this one would be OK.
-define float @f4(float %f1, float *%base) {
+define float @f4(float %f1, float *%base) #0 {
 ; CHECK-LABEL: f4:
 ; CHECK: aghi %r2, 4096
 ; CHECK: seb %f0, 0(%r2)
@@ -58,12 +58,12 @@ define float @f4(float %f1, float *%base) {
   %res = call float @llvm.experimental.constrained.fsub.f32(
                         float %f1, float %f2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
 
 ; Check negative displacements, which also need separate address logic.
-define float @f5(float %f1, float *%base) {
+define float @f5(float %f1, float *%base) #0 {
 ; CHECK-LABEL: f5:
 ; CHECK: aghi %r2, -4
 ; CHECK: seb %f0, 0(%r2)
@@ -73,12 +73,12 @@ define float @f5(float %f1, float *%base) {
   %res = call float @llvm.experimental.constrained.fsub.f32(
                         float %f1, float %f2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
 
 ; Check that SEB allows indices.
-define float @f6(float %f1, float *%base, i64 %index) {
+define float @f6(float %f1, float *%base, i64 %index) #0 {
 ; CHECK-LABEL: f6:
 ; CHECK: sllg %r1, %r3, 2
 ; CHECK: seb %f0, 400(%r1,%r2)
@@ -89,12 +89,12 @@ define float @f6(float %f1, float *%base, i64 %index) {
   %res = call float @llvm.experimental.constrained.fsub.f32(
                         float %f1, float %f2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
 
 ; Check that subtractions of spilled values can use SEB rather than SEBR.
-define float @f7(float *%ptr0) {
+define float @f7(float *%ptr0) #0 {
 ; CHECK-LABEL: f7:
 ; CHECK: brasl %r14, foo@PLT
 ; CHECK-SCALAR: seb %f0, 16{{[04]}}(%r15)
@@ -122,52 +122,54 @@ define float @f7(float *%ptr0) {
   %val9 = load float, float *%ptr9
   %val10 = load float, float *%ptr10
 
-  %ret = call float @foo()
+  %ret = call float @foo() #0
 
   %sub0 = call float @llvm.experimental.constrained.fsub.f32(
                         float %ret, float %val0,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %sub1 = call float @llvm.experimental.constrained.fsub.f32(
                         float %sub0, float %val1,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %sub2 = call float @llvm.experimental.constrained.fsub.f32(
                         float %sub1, float %val2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %sub3 = call float @llvm.experimental.constrained.fsub.f32(
                         float %sub2, float %val3,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %sub4 = call float @llvm.experimental.constrained.fsub.f32(
                         float %sub3, float %val4,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %sub5 = call float @llvm.experimental.constrained.fsub.f32(
                         float %sub4, float %val5,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %sub6 = call float @llvm.experimental.constrained.fsub.f32(
                         float %sub5, float %val6,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %sub7 = call float @llvm.experimental.constrained.fsub.f32(
                         float %sub6, float %val7,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %sub8 = call float @llvm.experimental.constrained.fsub.f32(
                         float %sub7, float %val8,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %sub9 = call float @llvm.experimental.constrained.fsub.f32(
                         float %sub8, float %val9,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %sub10 = call float @llvm.experimental.constrained.fsub.f32(
                         float %sub9, float %val10,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
 
   ret float %sub10
 }
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-sub-02.ll b/llvm/test/CodeGen/SystemZ/fp-strict-sub-02.ll
index 0d3cdd351..6184d88d8 100644
--- a/llvm/test/CodeGen/SystemZ/fp-strict-sub-02.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-sub-02.ll
@@ -8,19 +8,19 @@ declare double @foo()
 declare double @llvm.experimental.constrained.fsub.f64(double, double, metadata, metadata)
 
 ; Check register subtraction.
-define double @f1(double %f1, double %f2) {
+define double @f1(double %f1, double %f2) #0 {
 ; CHECK-LABEL: f1:
 ; CHECK: sdbr %f0, %f2
 ; CHECK: br %r14
   %res = call double @llvm.experimental.constrained.fsub.f64(
                         double %f1, double %f2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
 ; Check the low end of the SDB range.
-define double @f2(double %f1, double *%ptr) {
+define double @f2(double %f1, double *%ptr) #0 {
 ; CHECK-LABEL: f2:
 ; CHECK: sdb %f0, 0(%r2)
 ; CHECK: br %r14
@@ -28,12 +28,12 @@ define double @f2(double %f1, double *%ptr) {
   %res = call double @llvm.experimental.constrained.fsub.f64(
                         double %f1, double %f2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
 ; Check the high end of the aligned SDB range.
-define double @f3(double %f1, double *%base) {
+define double @f3(double %f1, double *%base) #0 {
 ; CHECK-LABEL: f3:
 ; CHECK: sdb %f0, 4088(%r2)
 ; CHECK: br %r14
@@ -42,13 +42,13 @@ define double @f3(double %f1, double *%base) {
   %res = call double @llvm.experimental.constrained.fsub.f64(
                         double %f1, double %f2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
 ; Check the next doubleword up, which needs separate address logic.
 ; Other sequences besides this one would be OK.
-define double @f4(double %f1, double *%base) {
+define double @f4(double %f1, double *%base) #0 {
 ; CHECK-LABEL: f4:
 ; CHECK: aghi %r2, 4096
 ; CHECK: sdb %f0, 0(%r2)
@@ -58,12 +58,12 @@ define double @f4(double %f1, double *%base) {
   %res = call double @llvm.experimental.constrained.fsub.f64(
                         double %f1, double %f2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
 ; Check negative displacements, which also need separate address logic.
-define double @f5(double %f1, double *%base) {
+define double @f5(double %f1, double *%base) #0 {
 ; CHECK-LABEL: f5:
 ; CHECK: aghi %r2, -8
 ; CHECK: sdb %f0, 0(%r2)
@@ -73,12 +73,12 @@ define double @f5(double %f1, double *%base) {
   %res = call double @llvm.experimental.constrained.fsub.f64(
                         double %f1, double %f2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
 ; Check that SDB allows indices.
-define double @f6(double %f1, double *%base, i64 %index) {
+define double @f6(double %f1, double *%base, i64 %index) #0 {
 ; CHECK-LABEL: f6:
 ; CHECK: sllg %r1, %r3, 3
 ; CHECK: sdb %f0, 800(%r1,%r2)
@@ -89,12 +89,12 @@ define double @f6(double %f1, double *%base, i64 %index) {
   %res = call double @llvm.experimental.constrained.fsub.f64(
                         double %f1, double %f2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
 ; Check that subtractions of spilled values can use SDB rather than SDBR.
-define double @f7(double *%ptr0) {
+define double @f7(double *%ptr0) #0 {
 ; CHECK-LABEL: f7:
 ; CHECK: brasl %r14, foo@PLT
 ; CHECK-SCALAR: sdb %f0, 16{{[04]}}(%r15)
@@ -122,52 +122,54 @@ define double @f7(double *%ptr0) {
   %val9 = load double, double *%ptr9
   %val10 = load double, double *%ptr10
 
-  %ret = call double @foo()
+  %ret = call double @foo() #0
 
   %sub0 = call double @llvm.experimental.constrained.fsub.f64(
                         double %ret, double %val0,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %sub1 = call double @llvm.experimental.constrained.fsub.f64(
                         double %sub0, double %val1,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %sub2 = call double @llvm.experimental.constrained.fsub.f64(
                         double %sub1, double %val2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %sub3 = call double @llvm.experimental.constrained.fsub.f64(
                         double %sub2, double %val3,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %sub4 = call double @llvm.experimental.constrained.fsub.f64(
                         double %sub3, double %val4,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %sub5 = call double @llvm.experimental.constrained.fsub.f64(
                         double %sub4, double %val5,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %sub6 = call double @llvm.experimental.constrained.fsub.f64(
                         double %sub5, double %val6,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %sub7 = call double @llvm.experimental.constrained.fsub.f64(
                         double %sub6, double %val7,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %sub8 = call double @llvm.experimental.constrained.fsub.f64(
                         double %sub7, double %val8,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %sub9 = call double @llvm.experimental.constrained.fsub.f64(
                         double %sub8, double %val9,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   %sub10 = call double @llvm.experimental.constrained.fsub.f64(
                         double %sub9, double %val10,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
 
   ret double %sub10
 }
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-sub-03.ll b/llvm/test/CodeGen/SystemZ/fp-strict-sub-03.ll
index 63bb2fdd5..d5ab4caf0 100644
--- a/llvm/test/CodeGen/SystemZ/fp-strict-sub-03.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-sub-03.ll
@@ -3,9 +3,10 @@
 ; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
 
 declare fp128 @llvm.experimental.constrained.fsub.f128(fp128, fp128, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.fpext.f128.f32(float, metadata)
 
 ; There is no memory form of 128-bit subtraction.
-define void @f1(fp128 *%ptr, float %f2) {
+define void @f1(fp128 *%ptr, float %f2) strictfp {
 ; CHECK-LABEL: f1:
 ; CHECK-DAG: lxebr %f0, %f0
 ; CHECK-DAG: ld %f1, 0(%r2)
@@ -15,11 +16,14 @@ define void @f1(fp128 *%ptr, float %f2) {
 ; CHECK: std %f3, 8(%r2)
 ; CHECK: br %r14
   %f1 = load fp128, fp128 *%ptr
-  %f2x = fpext float %f2 to fp128
+  %f2x = call fp128 @llvm.experimental.constrained.fpext.f128.f32(float %f2,
+                                               metadata !"fpexcept.strict") #0
   %sum = call fp128 @llvm.experimental.constrained.fsub.f128(
                         fp128 %f1, fp128 %f2x,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   store fp128 %sum, fp128 *%ptr
   ret void
 }
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-sub-04.ll b/llvm/test/CodeGen/SystemZ/fp-strict-sub-04.ll
index 1e8326847..0eaf5f3af 100644
--- a/llvm/test/CodeGen/SystemZ/fp-strict-sub-04.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-sub-04.ll
@@ -4,7 +4,7 @@
 
 declare fp128 @llvm.experimental.constrained.fsub.f128(fp128, fp128, metadata, metadata)
 
-define void @f1(fp128 *%ptr1, fp128 *%ptr2) {
+define void @f1(fp128 *%ptr1, fp128 *%ptr2) strictfp {
 ; CHECK-LABEL: f1:
 ; CHECK-DAG: vl [[REG1:%v[0-9]+]], 0(%r2)
 ; CHECK-DAG: vl [[REG2:%v[0-9]+]], 0(%r3)
@@ -16,7 +16,7 @@ define void @f1(fp128 *%ptr1, fp128 *%ptr2) {
   %sum = call fp128 @llvm.experimental.constrained.fsub.f128(
                         fp128 %f1, fp128 %f2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") strictfp
   store fp128 %sum, fp128 *%ptr1
   ret void
 }
diff --git a/llvm/test/CodeGen/SystemZ/frame-02.ll b/llvm/test/CodeGen/SystemZ/frame-02.ll
index 411a0826d..d7160483b 100644
--- a/llvm/test/CodeGen/SystemZ/frame-02.ll
+++ b/llvm/test/CodeGen/SystemZ/frame-02.ll
@@ -4,20 +4,20 @@
 
 ; This function should require all FPRs, but no other spill slots.
 ; We need to save and restore 8 of the 16 FPRs, so the frame size
-; should be exactly 160 + 8 * 8 = 224.  The CFA offset is 160
-; (the caller-allocated part of the frame) + 224.
+; should be exactly 8 * 8 = 64.  The CFA offset is 160
+; (the caller-allocated part of the frame) + 64.
 define void @f1(float *%ptr) {
 ; CHECK-LABEL: f1:
-; CHECK: aghi %r15, -224
-; CHECK: .cfi_def_cfa_offset 384
-; CHECK: std %f8, 216(%r15)
-; CHECK: std %f9, 208(%r15)
-; CHECK: std %f10, 200(%r15)
-; CHECK: std %f11, 192(%r15)
-; CHECK: std %f12, 184(%r15)
-; CHECK: std %f13, 176(%r15)
-; CHECK: std %f14, 168(%r15)
-; CHECK: std %f15, 160(%r15)
+; CHECK: aghi %r15, -64
+; CHECK: .cfi_def_cfa_offset 224
+; CHECK: std %f8, 56(%r15)
+; CHECK: std %f9, 48(%r15)
+; CHECK: std %f10, 40(%r15)
+; CHECK: std %f11, 32(%r15)
+; CHECK: std %f12, 24(%r15)
+; CHECK: std %f13, 16(%r15)
+; CHECK: std %f14, 8(%r15)
+; CHECK: std %f15, 0(%r15)
 ; CHECK: .cfi_offset %f8, -168
 ; CHECK: .cfi_offset %f9, -176
 ; CHECK: .cfi_offset %f10, -184
@@ -27,15 +27,15 @@ define void @f1(float *%ptr) {
 ; CHECK: .cfi_offset %f14, -216
 ; CHECK: .cfi_offset %f15, -224
 ; ...main function body...
-; CHECK: ld %f8, 216(%r15)
-; CHECK: ld %f9, 208(%r15)
-; CHECK: ld %f10, 200(%r15)
-; CHECK: ld %f11, 192(%r15)
-; CHECK: ld %f12, 184(%r15)
-; CHECK: ld %f13, 176(%r15)
-; CHECK: ld %f14, 168(%r15)
-; CHECK: ld %f15, 160(%r15)
-; CHECK: aghi %r15, 224
+; CHECK: ld %f8, 56(%r15)
+; CHECK: ld %f9, 48(%r15)
+; CHECK: ld %f10, 40(%r15)
+; CHECK: ld %f11, 32(%r15)
+; CHECK: ld %f12, 24(%r15)
+; CHECK: ld %f13, 16(%r15)
+; CHECK: ld %f14, 8(%r15)
+; CHECK: ld %f15, 0(%r15)
+; CHECK: aghi %r15, 64
 ; CHECK: br %r14
   %l0 = load volatile float, float *%ptr
   %l1 = load volatile float, float *%ptr
@@ -92,15 +92,15 @@ define void @f1(float *%ptr) {
 ; so %f15 is the one that gets dropped.
 define void @f2(float *%ptr) {
 ; CHECK-LABEL: f2:
-; CHECK: aghi %r15, -216
-; CHECK: .cfi_def_cfa_offset 376
-; CHECK: std %f8, 208(%r15)
-; CHECK: std %f9, 200(%r15)
-; CHECK: std %f10, 192(%r15)
-; CHECK: std %f11, 184(%r15)
-; CHECK: std %f12, 176(%r15)
-; CHECK: std %f13, 168(%r15)
-; CHECK: std %f14, 160(%r15)
+; CHECK: aghi %r15, -56
+; CHECK: .cfi_def_cfa_offset 216
+; CHECK: std %f8, 48(%r15)
+; CHECK: std %f9, 40(%r15)
+; CHECK: std %f10, 32(%r15)
+; CHECK: std %f11, 24(%r15)
+; CHECK: std %f12, 16(%r15)
+; CHECK: std %f13, 8(%r15)
+; CHECK: std %f14, 0(%r15)
 ; CHECK: .cfi_offset %f8, -168
 ; CHECK: .cfi_offset %f9, -176
 ; CHECK: .cfi_offset %f10, -184
@@ -110,14 +110,14 @@ define void @f2(float *%ptr) {
 ; CHECK: .cfi_offset %f14, -216
 ; CHECK-NOT: %f15
 ; ...main function body...
-; CHECK: ld %f8, 208(%r15)
-; CHECK: ld %f9, 200(%r15)
-; CHECK: ld %f10, 192(%r15)
-; CHECK: ld %f11, 184(%r15)
-; CHECK: ld %f12, 176(%r15)
-; CHECK: ld %f13, 168(%r15)
-; CHECK: ld %f14, 160(%r15)
-; CHECK: aghi %r15, 216
+; CHECK: ld %f8, 48(%r15)
+; CHECK: ld %f9, 40(%r15)
+; CHECK: ld %f10, 32(%r15)
+; CHECK: ld %f11, 24(%r15)
+; CHECK: ld %f12, 16(%r15)
+; CHECK: ld %f13, 8(%r15)
+; CHECK: ld %f14, 0(%r15)
+; CHECK: aghi %r15, 56
 ; CHECK: br %r14
   %l0 = load volatile float, float *%ptr
   %l1 = load volatile float, float *%ptr
@@ -170,9 +170,9 @@ define void @f2(float *%ptr) {
 ; Like f1, but should require only one call-saved FPR.
 define void @f3(float *%ptr) {
 ; CHECK-LABEL: f3:
-; CHECK: aghi %r15, -168
-; CHECK: .cfi_def_cfa_offset 328
-; CHECK: std %f8, 160(%r15)
+; CHECK: aghi %r15, -8
+; CHECK: .cfi_def_cfa_offset 168
+; CHECK: std %f8, 0(%r15)
 ; CHECK: .cfi_offset %f8, -168
 ; CHECK-NOT: %f9
 ; CHECK-NOT: %f10
@@ -182,8 +182,8 @@ define void @f3(float *%ptr) {
 ; CHECK-NOT: %f14
 ; CHECK-NOT: %f15
 ; ...main function body...
-; CHECK: ld %f8, 160(%r15)
-; CHECK: aghi %r15, 168
+; CHECK: ld %f8, 0(%r15)
+; CHECK: aghi %r15, 8
 ; CHECK: br %r14
   %l0 = load volatile float, float *%ptr
   %l1 = load volatile float, float *%ptr
diff --git a/llvm/test/CodeGen/SystemZ/frame-03.ll b/llvm/test/CodeGen/SystemZ/frame-03.ll
index 28826150e..ce2e02444 100644
--- a/llvm/test/CodeGen/SystemZ/frame-03.ll
+++ b/llvm/test/CodeGen/SystemZ/frame-03.ll
@@ -6,20 +6,20 @@
 
 ; This function should require all FPRs, but no other spill slots.
 ; We need to save and restore 8 of the 16 FPRs, so the frame size
-; should be exactly 160 + 8 * 8 = 224.  The CFA offset is 160
-; (the caller-allocated part of the frame) + 224.
+; should be exactly 8 * 8 = 64.  The CFA offset is 160
+; (the caller-allocated part of the frame) + 64.
 define void @f1(double *%ptr) {
 ; CHECK-LABEL: f1:
-; CHECK: aghi %r15, -224
-; CHECK: .cfi_def_cfa_offset 384
-; CHECK: std %f8, 216(%r15)
-; CHECK: std %f9, 208(%r15)
-; CHECK: std %f10, 200(%r15)
-; CHECK: std %f11, 192(%r15)
-; CHECK: std %f12, 184(%r15)
-; CHECK: std %f13, 176(%r15)
-; CHECK: std %f14, 168(%r15)
-; CHECK: std %f15, 160(%r15)
+; CHECK: aghi %r15, -64
+; CHECK: .cfi_def_cfa_offset 224
+; CHECK: std %f8, 56(%r15)
+; CHECK: std %f9, 48(%r15)
+; CHECK: std %f10, 40(%r15)
+; CHECK: std %f11, 32(%r15)
+; CHECK: std %f12, 24(%r15)
+; CHECK: std %f13, 16(%r15)
+; CHECK: std %f14, 8(%r15)
+; CHECK: std %f15, 0(%r15)
 ; CHECK: .cfi_offset %f8, -168
 ; CHECK: .cfi_offset %f9, -176
 ; CHECK: .cfi_offset %f10, -184
@@ -29,15 +29,15 @@ define void @f1(double *%ptr) {
 ; CHECK: .cfi_offset %f14, -216
 ; CHECK: .cfi_offset %f15, -224
 ; ...main function body...
-; CHECK: ld %f8, 216(%r15)
-; CHECK: ld %f9, 208(%r15)
-; CHECK: ld %f10, 200(%r15)
-; CHECK: ld %f11, 192(%r15)
-; CHECK: ld %f12, 184(%r15)
-; CHECK: ld %f13, 176(%r15)
-; CHECK: ld %f14, 168(%r15)
-; CHECK: ld %f15, 160(%r15)
-; CHECK: aghi %r15, 224
+; CHECK: ld %f8, 56(%r15)
+; CHECK: ld %f9, 48(%r15)
+; CHECK: ld %f10, 40(%r15)
+; CHECK: ld %f11, 32(%r15)
+; CHECK: ld %f12, 24(%r15)
+; CHECK: ld %f13, 16(%r15)
+; CHECK: ld %f14, 8(%r15)
+; CHECK: ld %f15, 0(%r15)
+; CHECK: aghi %r15, 64
 ; CHECK: br %r14
   %l0 = load volatile double, double *%ptr
   %l1 = load volatile double, double *%ptr
@@ -94,15 +94,15 @@ define void @f1(double *%ptr) {
 ; so %f15 is the one that gets dropped.
 define void @f2(double *%ptr) {
 ; CHECK-LABEL: f2:
-; CHECK: aghi %r15, -216
-; CHECK: .cfi_def_cfa_offset 376
-; CHECK: std %f8, 208(%r15)
-; CHECK: std %f9, 200(%r15)
-; CHECK: std %f10, 192(%r15)
-; CHECK: std %f11, 184(%r15)
-; CHECK: std %f12, 176(%r15)
-; CHECK: std %f13, 168(%r15)
-; CHECK: std %f14, 160(%r15)
+; CHECK: aghi %r15, -56
+; CHECK: .cfi_def_cfa_offset 216
+; CHECK: std %f8, 48(%r15)
+; CHECK: std %f9, 40(%r15)
+; CHECK: std %f10, 32(%r15)
+; CHECK: std %f11, 24(%r15)
+; CHECK: std %f12, 16(%r15)
+; CHECK: std %f13, 8(%r15)
+; CHECK: std %f14, 0(%r15)
 ; CHECK: .cfi_offset %f8, -168
 ; CHECK: .cfi_offset %f9, -176
 ; CHECK: .cfi_offset %f10, -184
@@ -112,14 +112,14 @@ define void @f2(double *%ptr) {
 ; CHECK: .cfi_offset %f14, -216
 ; CHECK-NOT: %f15
 ; ...main function body...
-; CHECK: ld %f8, 208(%r15)
-; CHECK: ld %f9, 200(%r15)
-; CHECK: ld %f10, 192(%r15)
-; CHECK: ld %f11, 184(%r15)
-; CHECK: ld %f12, 176(%r15)
-; CHECK: ld %f13, 168(%r15)
-; CHECK: ld %f14, 160(%r15)
-; CHECK: aghi %r15, 216
+; CHECK: ld %f8, 48(%r15)
+; CHECK: ld %f9, 40(%r15)
+; CHECK: ld %f10, 32(%r15)
+; CHECK: ld %f11, 24(%r15)
+; CHECK: ld %f12, 16(%r15)
+; CHECK: ld %f13, 8(%r15)
+; CHECK: ld %f14, 0(%r15)
+; CHECK: aghi %r15, 56
 ; CHECK: br %r14
   %l0 = load volatile double, double *%ptr
   %l1 = load volatile double, double *%ptr
@@ -172,9 +172,9 @@ define void @f2(double *%ptr) {
 ; Like f1, but should require only one call-saved FPR.
 define void @f3(double *%ptr) {
 ; CHECK-LABEL: f3:
-; CHECK: aghi %r15, -168
-; CHECK: .cfi_def_cfa_offset 328
-; CHECK: std %f8, 160(%r15)
+; CHECK: aghi %r15, -8
+; CHECK: .cfi_def_cfa_offset 168
+; CHECK: std %f8, 0(%r15)
 ; CHECK: .cfi_offset %f8, -168
 ; CHECK-NOT: %f9
 ; CHECK-NOT: %f10
@@ -184,8 +184,8 @@ define void @f3(double *%ptr) {
 ; CHECK-NOT: %f14
 ; CHECK-NOT: %f15
 ; ...main function body...
-; CHECK: ld %f8, 160(%r15)
-; CHECK: aghi %r15, 168
+; CHECK: ld %f8, 0(%r15)
+; CHECK: aghi %r15, 8
 ; CHECK: br %r14
   %l0 = load volatile double, double *%ptr
   %l1 = load volatile double, double *%ptr
diff --git a/llvm/test/CodeGen/SystemZ/frame-04.ll b/llvm/test/CodeGen/SystemZ/frame-04.ll
index 5e1b29f57..ae9c53a7d 100644
--- a/llvm/test/CodeGen/SystemZ/frame-04.ll
+++ b/llvm/test/CodeGen/SystemZ/frame-04.ll
@@ -5,20 +5,20 @@
 
 ; This function should require all FPRs, but no other spill slots.
 ; We need to save and restore 8 of the 16 FPRs, so the frame size
-; should be exactly 160 + 8 * 8 = 224.  The CFA offset is 160
-; (the caller-allocated part of the frame) + 224.
+; should be exactly 8 * 8 = 64.  The CFA offset is 160
+; (the caller-allocated part of the frame) + 64.
 define void @f1(fp128 *%ptr) {
 ; CHECK-LABEL: f1:
-; CHECK: aghi %r15, -224
-; CHECK: .cfi_def_cfa_offset 384
-; CHECK: std %f8, 216(%r15)
-; CHECK: std %f9, 208(%r15)
-; CHECK: std %f10, 200(%r15)
-; CHECK: std %f11, 192(%r15)
-; CHECK: std %f12, 184(%r15)
-; CHECK: std %f13, 176(%r15)
-; CHECK: std %f14, 168(%r15)
-; CHECK: std %f15, 160(%r15)
+; CHECK: aghi %r15, -64
+; CHECK: .cfi_def_cfa_offset 224
+; CHECK: std %f8, 56(%r15)
+; CHECK: std %f9, 48(%r15)
+; CHECK: std %f10, 40(%r15)
+; CHECK: std %f11, 32(%r15)
+; CHECK: std %f12, 24(%r15)
+; CHECK: std %f13, 16(%r15)
+; CHECK: std %f14, 8(%r15)
+; CHECK: std %f15, 0(%r15)
 ; CHECK: .cfi_offset %f8, -168
 ; CHECK: .cfi_offset %f9, -176
 ; CHECK: .cfi_offset %f10, -184
@@ -28,15 +28,15 @@ define void @f1(fp128 *%ptr) {
 ; CHECK: .cfi_offset %f14, -216
 ; CHECK: .cfi_offset %f15, -224
 ; ...main function body...
-; CHECK: ld %f8, 216(%r15)
-; CHECK: ld %f9, 208(%r15)
-; CHECK: ld %f10, 200(%r15)
-; CHECK: ld %f11, 192(%r15)
-; CHECK: ld %f12, 184(%r15)
-; CHECK: ld %f13, 176(%r15)
-; CHECK: ld %f14, 168(%r15)
-; CHECK: ld %f15, 160(%r15)
-; CHECK: aghi %r15, 224
+; CHECK: ld %f8, 56(%r15)
+; CHECK: ld %f9, 48(%r15)
+; CHECK: ld %f10, 40(%r15)
+; CHECK: ld %f11, 32(%r15)
+; CHECK: ld %f12, 24(%r15)
+; CHECK: ld %f13, 16(%r15)
+; CHECK: ld %f14, 8(%r15)
+; CHECK: ld %f15, 0(%r15)
+; CHECK: aghi %r15, 64
 ; CHECK: br %r14
   %l0 = load volatile fp128, fp128 *%ptr
   %l1 = load volatile fp128, fp128 *%ptr
@@ -69,14 +69,14 @@ define void @f1(fp128 *%ptr) {
 ; so %f13+%f15 is the pair that gets dropped.
 define void @f2(fp128 *%ptr) {
 ; CHECK-LABEL: f2:
-; CHECK: aghi %r15, -208
-; CHECK: .cfi_def_cfa_offset 368
-; CHECK: std %f8, 200(%r15)
-; CHECK: std %f9, 192(%r15)
-; CHECK: std %f10, 184(%r15)
-; CHECK: std %f11, 176(%r15)
-; CHECK: std %f12, 168(%r15)
-; CHECK: std %f14, 160(%r15)
+; CHECK: aghi %r15, -48
+; CHECK: .cfi_def_cfa_offset 208
+; CHECK: std %f8, 40(%r15)
+; CHECK: std %f9, 32(%r15)
+; CHECK: std %f10, 24(%r15)
+; CHECK: std %f11, 16(%r15)
+; CHECK: std %f12, 8(%r15)
+; CHECK: std %f14, 0(%r15)
 ; CHECK: .cfi_offset %f8, -168
 ; CHECK: .cfi_offset %f9, -176
 ; CHECK: .cfi_offset %f10, -184
@@ -86,13 +86,13 @@ define void @f2(fp128 *%ptr) {
 ; CHECK-NOT: %f13
 ; CHECK-NOT: %f15
 ; ...main function body...
-; CHECK: ld %f8, 200(%r15)
-; CHECK: ld %f9, 192(%r15)
-; CHECK: ld %f10, 184(%r15)
-; CHECK: ld %f11, 176(%r15)
-; CHECK: ld %f12, 168(%r15)
-; CHECK: ld %f14, 160(%r15)
-; CHECK: aghi %r15, 208
+; CHECK: ld %f8, 40(%r15)
+; CHECK: ld %f9, 32(%r15)
+; CHECK: ld %f10, 24(%r15)
+; CHECK: ld %f11, 16(%r15)
+; CHECK: ld %f12, 8(%r15)
+; CHECK: ld %f14, 0(%r15)
+; CHECK: aghi %r15, 48
 ; CHECK: br %r14
   %l0 = load volatile fp128, fp128 *%ptr
   %l1 = load volatile fp128, fp128 *%ptr
@@ -122,10 +122,10 @@ define void @f2(fp128 *%ptr) {
 ; numerical order so the pair should be %f8+%f10.
 define void @f3(fp128 *%ptr) {
 ; CHECK-LABEL: f3:
-; CHECK: aghi %r15, -176
-; CHECK: .cfi_def_cfa_offset 336
-; CHECK: std %f8, 168(%r15)
-; CHECK: std %f10, 160(%r15)
+; CHECK: aghi %r15, -16
+; CHECK: .cfi_def_cfa_offset 176
+; CHECK: std %f8, 8(%r15)
+; CHECK: std %f10, 0(%r15)
 ; CHECK: .cfi_offset %f8, -168
 ; CHECK: .cfi_offset %f10, -176
 ; CHECK-NOT: %f9
@@ -135,9 +135,9 @@ define void @f3(fp128 *%ptr) {
 ; CHECK-NOT: %f14
 ; CHECK-NOT: %f15
 ; ...main function body...
-; CHECK: ld %f8, 168(%r15)
-; CHECK: ld %f10, 160(%r15)
-; CHECK: aghi %r15, 176
+; CHECK: ld %f8, 8(%r15)
+; CHECK: ld %f10, 0(%r15)
+; CHECK: aghi %r15, 16
 ; CHECK: br %r14
   %l0 = load volatile fp128, fp128 *%ptr
   %l1 = load volatile fp128, fp128 *%ptr
diff --git a/llvm/test/CodeGen/SystemZ/frame-19.ll b/llvm/test/CodeGen/SystemZ/frame-19.ll
index 2a0693b9a..c6a266006 100644
--- a/llvm/test/CodeGen/SystemZ/frame-19.ll
+++ b/llvm/test/CodeGen/SystemZ/frame-19.ll
@@ -96,10 +96,11 @@ define void @f1(<16 x i8> *%ptr) {
   ret void
 }
 
-; Like f1, but no 16-byte slot should be needed.
+; Like f1, but no 16-byte slot should be needed, and no outgoing reg save
+; area of 160 bytes.
 define void @f2(<16 x i8> *%ptr) {
 ; CHECK-LABEL: f2:
-; CHECK: aghi %r15, -224
+; CHECK: aghi %r15, -64
 ; CHECK-DAG: std %f8,
 ; CHECK-DAG: std %f9,
 ; CHECK-DAG: std %f10,
@@ -118,7 +119,7 @@ define void @f2(<16 x i8> *%ptr) {
 ; CHECK-DAG: ld %f13,
 ; CHECK-DAG: ld %f14,
 ; CHECK-DAG: ld %f15,
-; CHECK: aghi %r15, 224
+; CHECK: aghi %r15, 64
 ; CHECK: br %r14
   %v0 = load volatile <16 x i8>, <16 x i8> *%ptr
   %v1 = load volatile <16 x i8>, <16 x i8> *%ptr
@@ -190,7 +191,7 @@ define void @f2(<16 x i8> *%ptr) {
 ; Like f2, but only %f8 should be saved.
 define void @f3(<16 x i8> *%ptr) {
 ; CHECK-LABEL: f3:
-; CHECK: aghi %r15, -168
+; CHECK: aghi %r15, -8
 ; CHECK-DAG: std %f8,
 ; CHECK-NOT: vst {{.*}}(%r15)
 ; CHECK-NOT: vl {{.*}}(%r15)
@@ -202,7 +203,7 @@ define void @f3(<16 x i8> *%ptr) {
 ; CHECK-NOT: %v14
 ; CHECK-NOT: %v15
 ; CHECK-DAG: ld %f8,
-; CHECK: aghi %r15, 168
+; CHECK: aghi %r15, 8
 ; CHECK: br %r14
   %v0 = load volatile <16 x i8>, <16 x i8> *%ptr
   %v1 = load volatile <16 x i8>, <16 x i8> *%ptr
diff --git a/llvm/test/CodeGen/SystemZ/frame-20.ll b/llvm/test/CodeGen/SystemZ/frame-20.ll
index 8d601c6f6..6653c8185 100644
--- a/llvm/test/CodeGen/SystemZ/frame-20.ll
+++ b/llvm/test/CodeGen/SystemZ/frame-20.ll
@@ -5,20 +5,20 @@
 
 ; This function should require all FPRs, but no other spill slots.
 ; We need to save and restore 8 of the 16 FPRs, so the frame size
-; should be exactly 160 + 8 * 8 = 224.  The CFA offset is 160
-; (the caller-allocated part of the frame) + 224.
+; should be exactly 8 * 8 = 64.  The CFA offset is 160
+; (the caller-allocated part of the frame) + 64.
 define void @f1(double *%ptr) {
 ; CHECK-LABEL: f1:
-; CHECK: aghi %r15, -224
-; CHECK: .cfi_def_cfa_offset 384
-; CHECK: std %f8, 216(%r15)
-; CHECK: std %f9, 208(%r15)
-; CHECK: std %f10, 200(%r15)
-; CHECK: std %f11, 192(%r15)
-; CHECK: std %f12, 184(%r15)
-; CHECK: std %f13, 176(%r15)
-; CHECK: std %f14, 168(%r15)
-; CHECK: std %f15, 160(%r15)
+; CHECK: aghi %r15, -64
+; CHECK: .cfi_def_cfa_offset 224
+; CHECK: std %f8, 56(%r15)
+; CHECK: std %f9, 48(%r15)
+; CHECK: std %f10, 40(%r15)
+; CHECK: std %f11, 32(%r15)
+; CHECK: std %f12, 24(%r15)
+; CHECK: std %f13, 16(%r15)
+; CHECK: std %f14, 8(%r15)
+; CHECK: std %f15, 0(%r15)
 ; CHECK: .cfi_offset %f8, -168
 ; CHECK: .cfi_offset %f9, -176
 ; CHECK: .cfi_offset %f10, -184
@@ -35,15 +35,15 @@ define void @f1(double *%ptr) {
 ; CHECK-DAG: vlrepg %v23, 0(%r2)
 ; CHECK-DAG: vlrepg %v24, 0(%r2)
 ; CHECK-DAG: vlrepg %v31, 0(%r2)
-; CHECK: ld %f8, 216(%r15)
-; CHECK: ld %f9, 208(%r15)
-; CHECK: ld %f10, 200(%r15)
-; CHECK: ld %f11, 192(%r15)
-; CHECK: ld %f12, 184(%r15)
-; CHECK: ld %f13, 176(%r15)
-; CHECK: ld %f14, 168(%r15)
-; CHECK: ld %f15, 160(%r15)
-; CHECK: aghi %r15, 224
+; CHECK: ld %f8, 56(%r15)
+; CHECK: ld %f9, 48(%r15)
+; CHECK: ld %f10, 40(%r15)
+; CHECK: ld %f11, 32(%r15)
+; CHECK: ld %f12, 24(%r15)
+; CHECK: ld %f13, 16(%r15)
+; CHECK: ld %f14, 8(%r15)
+; CHECK: ld %f15, 0(%r15)
+; CHECK: aghi %r15, 64
 ; CHECK: br %r14
   %l0 = load volatile double, double *%ptr
   %l1 = load volatile double, double *%ptr
@@ -148,15 +148,15 @@ define void @f1(double *%ptr) {
 ; so %f15 is the one that gets dropped.
 define void @f2(double *%ptr) {
 ; CHECK-LABEL: f2:
-; CHECK: aghi %r15, -216
-; CHECK: .cfi_def_cfa_offset 376
-; CHECK: std %f8, 208(%r15)
-; CHECK: std %f9, 200(%r15)
-; CHECK: std %f10, 192(%r15)
-; CHECK: std %f11, 184(%r15)
-; CHECK: std %f12, 176(%r15)
-; CHECK: std %f13, 168(%r15)
-; CHECK: std %f14, 160(%r15)
+; CHECK: aghi %r15, -56
+; CHECK: .cfi_def_cfa_offset 216
+; CHECK: std %f8, 48(%r15)
+; CHECK: std %f9, 40(%r15)
+; CHECK: std %f10, 32(%r15)
+; CHECK: std %f11, 24(%r15)
+; CHECK: std %f12, 16(%r15)
+; CHECK: std %f13, 8(%r15)
+; CHECK: std %f14, 0(%r15)
 ; CHECK: .cfi_offset %f8, -168
 ; CHECK: .cfi_offset %f9, -176
 ; CHECK: .cfi_offset %f10, -184
@@ -166,14 +166,14 @@ define void @f2(double *%ptr) {
 ; CHECK: .cfi_offset %f14, -216
 ; CHECK-NOT: %v15
 ; CHECK-NOT: %f15
-; CHECK: ld %f8, 208(%r15)
-; CHECK: ld %f9, 200(%r15)
-; CHECK: ld %f10, 192(%r15)
-; CHECK: ld %f11, 184(%r15)
-; CHECK: ld %f12, 176(%r15)
-; CHECK: ld %f13, 168(%r15)
-; CHECK: ld %f14, 160(%r15)
-; CHECK: aghi %r15, 216
+; CHECK: ld %f8, 48(%r15)
+; CHECK: ld %f9, 40(%r15)
+; CHECK: ld %f10, 32(%r15)
+; CHECK: ld %f11, 24(%r15)
+; CHECK: ld %f12, 16(%r15)
+; CHECK: ld %f13, 8(%r15)
+; CHECK: ld %f14, 0(%r15)
+; CHECK: aghi %r15, 56
 ; CHECK: br %r14
   %l0 = load volatile double, double *%ptr
   %l1 = load volatile double, double *%ptr
@@ -274,14 +274,14 @@ define void @f2(double *%ptr) {
 ; Like f1, but should require only one call-saved FPR.
 define void @f3(double *%ptr) {
 ; CHECK-LABEL: f3:
-; CHECK: aghi %r15, -168
-; CHECK: .cfi_def_cfa_offset 328
-; CHECK: std %f8, 160(%r15)
+; CHECK: aghi %r15, -8
+; CHECK: .cfi_def_cfa_offset 168
+; CHECK: std %f8, 0(%r15)
 ; CHECK: .cfi_offset %f8, -168
 ; CHECK-NOT: {{%[fv]9}}
 ; CHECK-NOT: {{%[fv]1[0-5]}}
-; CHECK: ld %f8, 160(%r15)
-; CHECK: aghi %r15, 168
+; CHECK: ld %f8, 0(%r15)
+; CHECK: aghi %r15, 8
 ; CHECK: br %r14
   %l0 = load volatile double, double *%ptr
   %l1 = load volatile double, double *%ptr
diff --git a/llvm/test/CodeGen/SystemZ/frame-22.ll b/llvm/test/CodeGen/SystemZ/frame-22.ll
new file mode 100644
index 000000000..fd4e3fec5
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/frame-22.ll
@@ -0,0 +1,87 @@
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+;
+; Test the packed stack layout.
+
+; Test spill/restore of an FPR and a GPR.
+define void @f1() #0 {
+; CHECK-LABEL: f1:
+; CHECK: stmg %r12, %r15, 128(%r15)
+; CHECK-NEXT: .cfi_offset %r12, -32
+; CHECK-NEXT: .cfi_offset %r15, -8
+; CHECK-NEXT: std %f8, 120(%r15)          # 8-byte Folded Spill
+; CHECK-NEXT: .cfi_offset %f8, -40
+; CHECK-NEXT: #APP
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: ld %f8, 120(%r15)          # 8-byte Folded Reload
+; CHECK-NEXT: lmg %r12, %r15, 128(%r15)
+; CHECK-NEXT: br %r14
+  call void asm sideeffect "", "~{f8},~{r12}"() nounwind
+  ret void
+}
+
+; Test spill/restore with anyregcc, including an FP argument register.
+define anyregcc void @f2() #0 {
+; CHECK-LABEL: f2:
+; CHECK: stmg %r3, %r15, 56(%r15)
+; CHECK-NEXT: .cfi_offset %r3, -104
+; CHECK-NEXT: .cfi_offset %r15, -8
+; CHECK-NEXT: std %f0, 48(%r15)          # 8-byte Folded Spill
+; CHECK-NEXT: std %f1, 40(%r15)          # 8-byte Folded Spill
+; CHECK-NEXT: .cfi_offset %f0, -112
+; CHECK-NEXT: .cfi_offset %f1, -120
+; CHECK-NEXT: #APP
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: ld %f0, 48(%r15)          # 8-byte Folded Reload
+; CHECK-NEXT: ld %f1, 40(%r15)          # 8-byte Folded Reload
+; CHECK-NEXT: lmg %r3, %r15, 56(%r15)
+; CHECK-NEXT: br %r14
+  call void asm sideeffect "", "~{f0},~{f1},~{r3}"() nounwind
+  ret void
+}
+
+; Test spill/restore in local area with incoming stack arguments.
+define i64 @f3(i64 %a, i64 %b, i64 %c, i64 %d, i64 %e, i64 %f,
+               double %A, double %B, double %C, double %D, double %E) #0 {
+; CHECK-LABEL: f3:
+; CHECK: std %f8, 152(%r15)          # 8-byte Folded Spill
+; CHECK-NEXT: .cfi_offset %f8, -8
+; CHECK-NEXT: ld %f0, 168(%r15)
+; CHECK-NEXT: cgdbr %r2, 5, %f0
+; CHECK-NEXT: ag %r2, 160(%r15)
+; CHECK-NEXT: #APP
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: ld %f8, 152(%r15)          # 8-byte Folded Reload
+; CHECK-NEXT: br %r14
+  call void asm sideeffect "", "~{f8}"() nounwind
+  %Ei = fptosi double %E to i64
+  %S = add i64 %f, %Ei
+  ret i64 %S
+}
+
+; Test spill/restore in local area with outgoing stack arguments.
+define i64 @f4() #0 {
+; CHECK-LABEL: f4:
+; CHECK: stmg    %r6, %r15, 80(%r15)
+; CHECK-NEXT: .cfi_offset %r6, -80
+; CHECK-NEXT: .cfi_offset %r14, -16
+; CHECK-NEXT: .cfi_offset %r15, -8
+; CHECK-NEXT: aghi    %r15, -104
+; CHECK-NEXT: .cfi_def_cfa_offset 264
+; CHECK-NEXT: std     %f8, 176(%r15)          # 8-byte Folded Spill
+; CHECK-NEXT: .cfi_offset %f8, -88
+; CHECK-NEXT: #APP
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: llihh   %r0, 16404
+; CHECK-NEXT: stg     %r0, 168(%r15)
+; CHECK: mvghi   160(%r15), 6
+; CHECK-NEXT: brasl   %r14, f3@PLT
+; CHECK-NEXT: ld      %f8, 176(%r15)          # 8-byte Folded Reload
+; CHECK-NEXT: lmg     %r6, %r15, 184(%r15)
+; CHECK-NEXT: br      %r14
+  call void asm sideeffect "", "~{f8}"() nounwind
+  %C = call i64 @f3 (i64 1, i64 2, i64 3, i64 4, i64 5, i64 6,
+                     double 1.0, double 2.0, double 3.0, double 4.0, double 5.0)
+  ret i64 %C
+}
+
+attributes #0 = { "packed-stack"="true" }
diff --git a/llvm/test/CodeGen/SystemZ/ghc-cc-01.ll b/llvm/test/CodeGen/SystemZ/ghc-cc-01.ll
new file mode 100644
index 000000000..dc8052d8f
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/ghc-cc-01.ll
@@ -0,0 +1,103 @@
+; Check that the GHC calling convention works (s390x)
+;
+; RUN: llc -mtriple=s390x-ibm-linux < %s | FileCheck %s
+
+@base  = external global i64 ; assigned to register: r7
+@sp    = external global i64 ; assigned to register: r8
+@hp    = external global i64 ; assigned to register: r10
+@r1    = external global i64 ; assigned to register: r11
+@r2    = external global i64 ; assigned to register: r12
+@r3    = external global i64 ; assigned to register: r13
+@r4    = external global i64 ; assigned to register: r6
+@r5    = external global i64 ; assigned to register: r2
+@r6    = external global i64 ; assigned to register: r3
+@r7    = external global i64 ; assigned to register: r4
+@r8    = external global i64 ; assigned to register: r5
+@splim = external global i64 ; assigned to register: r9
+
+@f1 = external global float  ; assigned to register: s8
+@f2 = external global float  ; assigned to register: s9
+@f3 = external global float  ; assigned to register: s10
+@f4 = external global float  ; assigned to register: s11
+@f5 = external global float  ; assigned to register: s0
+@f6 = external global float  ; assigned to register: s1
+
+@d1 = external global double ; assigned to register: d12
+@d2 = external global double ; assigned to register: d13
+@d3 = external global double ; assigned to register: d14
+@d4 = external global double ; assigned to register: d15
+@d5 = external global double ; assigned to register: d2
+@d6 = external global double ; assigned to register: d3
+
+define ghccc void @foo() nounwind {
+entry:
+  ; CHECK:      larl    {{%r[0-9]+}}, d6
+  ; CHECK-NEXT: ld      %f3, 0({{%r[0-9]+}})
+  ; CHECK-NEXT: larl    {{%r[0-9]+}}, d5
+  ; CHECK-NEXT: ld      %f2, 0({{%r[0-9]+}})
+  ; CHECK-NEXT: larl    {{%r[0-9]+}}, d4
+  ; CHECK-NEXT: ld      %f15, 0({{%r[0-9]+}})
+  ; CHECK-NEXT: larl    {{%r[0-9]+}}, d3
+  ; CHECK-NEXT: ld      %f14, 0({{%r[0-9]+}})
+  ; CHECK-NEXT: larl    {{%r[0-9]+}}, d2
+  ; CHECK-NEXT: ld      %f13, 0({{%r[0-9]+}})
+  ; CHECK-NEXT: larl    {{%r[0-9]+}}, d1
+  ; CHECK-NEXT: ld      %f12, 0({{%r[0-9]+}})
+  ; CHECK-NEXT: larl    {{%r[0-9]+}}, f6
+  ; CHECK-NEXT: le      %f1, 0({{%r[0-9]+}})
+  ; CHECK-NEXT: larl    {{%r[0-9]+}}, f5
+  ; CHECK-NEXT: le      %f0, 0({{%r[0-9]+}})
+  ; CHECK-NEXT: larl    {{%r[0-9]+}}, f4
+  ; CHECK-NEXT: le      %f11, 0({{%r[0-9]+}})
+  ; CHECK-NEXT: larl    {{%r[0-9]+}}, f3
+  ; CHECK-NEXT: le      %f10, 0({{%r[0-9]+}})
+  ; CHECK-NEXT: larl    {{%r[0-9]+}}, f2
+  ; CHECK-NEXT: le      %f9, 0({{%r[0-9]+}})
+  ; CHECK-NEXT: larl    {{%r[0-9]+}}, f1
+  ; CHECK-NEXT: le      %f8, 0({{%r[0-9]+}})
+  ; CHECK-NEXT: lgrl    %r9,  splim
+  ; CHECK-NEXT: lgrl    %r5,  r8
+  ; CHECK-NEXT: lgrl    %r4,  r7
+  ; CHECK-NEXT: lgrl    %r3,  r6
+  ; CHECK-NEXT: lgrl    %r2,  r5
+  ; CHECK-NEXT: lgrl    %r6,  r4
+  ; CHECK-NEXT: lgrl    %r13, r3
+  ; CHECK-NEXT: lgrl    %r12, r2
+  ; CHECK-NEXT: lgrl    %r11, r1
+  ; CHECK-NEXT: lgrl    %r10, hp
+  ; CHECK-NEXT: lgrl    %r8,  sp
+  ; CHECK-NEXT: lgrl    %r7,  base
+  %0  = load double, double* @d6
+  %1  = load double, double* @d5
+  %2  = load double, double* @d4
+  %3  = load double, double* @d3
+  %4  = load double, double* @d2
+  %5  = load double, double* @d1
+  %6  = load float, float* @f6
+  %7  = load float, float* @f5
+  %8  = load float, float* @f4
+  %9  = load float, float* @f3
+  %10 = load float, float* @f2
+  %11 = load float, float* @f1
+  %12 = load i64, i64* @splim
+  %13 = load i64, i64* @r8
+  %14 = load i64, i64* @r7
+  %15 = load i64, i64* @r6
+  %16 = load i64, i64* @r5
+  %17 = load i64, i64* @r4
+  %18 = load i64, i64* @r3
+  %19 = load i64, i64* @r2
+  %20 = load i64, i64* @r1
+  %21 = load i64, i64* @hp
+  %22 = load i64, i64* @sp
+  %23 = load i64, i64* @base
+  ; CHECK: brasl %r14, bar
+  tail call ghccc void @bar(i64 %23, i64 %22, i64 %21, i64 %20, i64 %19, i64 %18, i64 %17, i64 %16, i64 %15, i64 %14, i64 %13, i64 %12,
+                            float %11, float %10, float %9, float %8, float %7, float %6,
+                            double %5, double %4, double %3, double %2, double %1, double %0) nounwind
+  ret void
+}
+
+declare ghccc void @bar(i64, i64, i64, i64, i64, i64, i64, i64, i64, i64, i64, i64,
+                        float, float, float, float, float, float,
+                        double, double, double, double, double, double)
diff --git a/llvm/test/CodeGen/SystemZ/ghc-cc-02.ll b/llvm/test/CodeGen/SystemZ/ghc-cc-02.ll
new file mode 100644
index 000000000..1d13429d2
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/ghc-cc-02.ll
@@ -0,0 +1,14 @@
+; Check that the GHC calling convention works (s390x)
+; Check that no more than 12 integer arguments are passed
+;
+; RUN: not llc -mtriple=s390x-ibm-linux < %s 2>&1 | FileCheck %s
+
+define ghccc void @foo() nounwind {
+entry:
+  tail call ghccc void (...) @bar(i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13);
+  ret void
+}
+
+declare ghccc void @bar(...)
+
+; CHECK: LLVM ERROR: No registers left in GHC calling convention
diff --git a/llvm/test/CodeGen/SystemZ/ghc-cc-03.ll b/llvm/test/CodeGen/SystemZ/ghc-cc-03.ll
new file mode 100644
index 000000000..1db7a3ff3
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/ghc-cc-03.ll
@@ -0,0 +1,11 @@
+; Check that the GHC calling convention works (s390x)
+; In GHC calling convention the only allowed return type is void
+;
+; RUN: not llc -mtriple=s390x-ibm-linux < %s 2>&1 | FileCheck %s
+
+define ghccc i64 @foo() nounwind {
+entry:
+  ret i64 42
+}
+
+; CHECK: LLVM ERROR: GHC functions return void only
diff --git a/llvm/test/CodeGen/SystemZ/ghc-cc-04.ll b/llvm/test/CodeGen/SystemZ/ghc-cc-04.ll
new file mode 100644
index 000000000..0dbe54722
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/ghc-cc-04.ll
@@ -0,0 +1,16 @@
+; Check that the GHC calling convention works (s390x)
+; Thread local storage is not supported in GHC calling convention
+;
+; RUN: not llc -mtriple=s390x-ibm-linux < %s 2>&1 | FileCheck %s
+
+@x = thread_local global i32 0
+
+define ghccc void @foo() nounwind {
+entry:
+  call void @bar(i32 *@x)
+  ret void
+}
+
+declare void @bar(i32*)
+
+; CHECK: LLVM ERROR: In GHC calling convention TLS is not supported
diff --git a/llvm/test/CodeGen/SystemZ/ghc-cc-05.ll b/llvm/test/CodeGen/SystemZ/ghc-cc-05.ll
new file mode 100644
index 000000000..be2cc6780
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/ghc-cc-05.ll
@@ -0,0 +1,16 @@
+; Check that the GHC calling convention works (s390x)
+; Variable-sized stack allocations are not supported in GHC calling convention
+;
+; RUN: not llc -mtriple=s390x-ibm-linux < %s 2>&1 | FileCheck %s
+
+define ghccc void @foo() nounwind {
+entry:
+  %0 = call i8* @llvm.stacksave()
+  call void @llvm.stackrestore(i8* %0)
+  ret void
+}
+
+declare i8* @llvm.stacksave()
+declare void @llvm.stackrestore(i8*)
+
+; CHECK: LLVM ERROR: Variable-sized stack allocations are not supported in GHC calling convention
diff --git a/llvm/test/CodeGen/SystemZ/ghc-cc-06.ll b/llvm/test/CodeGen/SystemZ/ghc-cc-06.ll
new file mode 100644
index 000000000..04df248c2
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/ghc-cc-06.ll
@@ -0,0 +1,12 @@
+; Check that the GHC calling convention works (s390x)
+; At most 2048*sizeof(long)=16384 bytes of stack space may be used
+;
+; RUN: not llc -mtriple=s390x-ibm-linux < %s 2>&1 | FileCheck %s
+
+define ghccc void @foo() nounwind {
+entry:
+  alloca [16385 x i8], align 1
+  ret void
+}
+
+; CHECK: LLVM ERROR: Pre allocated stack space for GHC function is too small
diff --git a/llvm/test/CodeGen/SystemZ/ghc-cc-07.ll b/llvm/test/CodeGen/SystemZ/ghc-cc-07.ll
new file mode 100644
index 000000000..e9bb3b5e1
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/ghc-cc-07.ll
@@ -0,0 +1,12 @@
+; Check that the GHC calling convention works (s390x)
+; In GHC calling convention a frame pointer is not supported
+;
+; RUN: not llc -mtriple=s390x-ibm-linux < %s 2>&1 | FileCheck %s
+
+define ghccc void @foo(i64 %0) nounwind {
+entry:
+  alloca i64, i64 %0
+  ret void
+}
+
+; CHECK: LLVM ERROR: In GHC calling convention a frame pointer is not supported
diff --git a/llvm/test/CodeGen/SystemZ/int-cmp-44.ll b/llvm/test/CodeGen/SystemZ/int-cmp-44.ll
index 45bb32b9c..2321b7d16 100644
--- a/llvm/test/CodeGen/SystemZ/int-cmp-44.ll
+++ b/llvm/test/CodeGen/SystemZ/int-cmp-44.ll
@@ -6,15 +6,16 @@
 
 declare void @foo()
 
-; Addition provides enough for equality comparisons with zero.  First teest
-; the EQ case.
+; Addition provides enough for comparisons with zero if we know no
+; signed overflow happens, which is when the "nsw" flag is set.
+; First test the EQ case.
 define i32 @f1(i32 %a, i32 %b, i32 *%dest) {
 ; CHECK-LABEL: f1:
 ; CHECK: afi %r2, 1000000
 ; CHECK-NEXT: ber %r14
 ; CHECK: br %r14
 entry:
-  %res = add i32 %a, 1000000
+  %res = add nsw i32 %a, 1000000
   %cmp = icmp eq i32 %res, 0
   br i1 %cmp, label %exit, label %store
 
@@ -30,10 +31,10 @@ exit:
 define i32 @f2(i32 %a, i32 %b, i32 *%dest) {
 ; CHECK-LABEL: f2:
 ; CHECK: afi %r2, 1000000
-; CHECK-NEXT: bner %r14
+; CHECK-NEXT: blhr %r14
 ; CHECK: br %r14
 entry:
-  %res = add i32 %a, 1000000
+  %res = add nsw i32 %a, 1000000
   %cmp = icmp ne i32 %res, 0
   br i1 %cmp, label %exit, label %store
 
@@ -45,14 +46,13 @@ exit:
   ret i32 %res
 }
 
-; SLT requires a comparison.
+; ...and again with SLT.
 define i32 @f3(i32 %a, i32 %b, i32 *%dest) {
 ; CHECK-LABEL: f3:
 ; CHECK: afi %r2, 1000000
-; CHECK-NEXT: cibl %r2, 0, 0(%r14)
-; CHECK: br %r14
+; CHECK-NEXT: blr %r14
 entry:
-  %res = add i32 %a, 1000000
+  %res = add nsw i32 %a, 1000000
   %cmp = icmp slt i32 %res, 0
   br i1 %cmp, label %exit, label %store
 
@@ -64,14 +64,13 @@ exit:
   ret i32 %res
 }
 
-; ...SLE too.
+; ...and again with SLE.
 define i32 @f4(i32 %a, i32 %b, i32 *%dest) {
 ; CHECK-LABEL: f4:
 ; CHECK: afi %r2, 1000000
-; CHECK-NEXT: cible %r2, 0, 0(%r14)
-; CHECK: br %r14
+; CHECK-NEXT: bler %r14
 entry:
-  %res = add i32 %a, 1000000
+  %res = add nsw i32 %a, 1000000
   %cmp = icmp sle i32 %res, 0
   br i1 %cmp, label %exit, label %store
 
@@ -83,14 +82,13 @@ exit:
   ret i32 %res
 }
 
-; ...SGT too.
+; ...and again with SGT.
 define i32 @f5(i32 %a, i32 %b, i32 *%dest) {
 ; CHECK-LABEL: f5:
 ; CHECK: afi %r2, 1000000
-; CHECK-NEXT: cibh %r2, 0, 0(%r14)
-; CHECK: br %r14
+; CHECK-NEXT: bhr %r14
 entry:
-  %res = add i32 %a, 1000000
+  %res = add nsw i32 %a, 1000000
   %cmp = icmp sgt i32 %res, 0
   br i1 %cmp, label %exit, label %store
 
@@ -102,14 +100,13 @@ exit:
   ret i32 %res
 }
 
-; ...SGE too.
+; ...and again with SGE.
 define i32 @f6(i32 %a, i32 %b, i32 *%dest) {
 ; CHECK-LABEL: f6:
 ; CHECK: afi %r2, 1000000
-; CHECK-NEXT: cibhe %r2, 0, 0(%r14)
-; CHECK: br %r14
+; CHECK-NEXT: bher %r14
 entry:
-  %res = add i32 %a, 1000000
+  %res = add nsw i32 %a, 1000000
   %cmp = icmp sge i32 %res, 0
   br i1 %cmp, label %exit, label %store
 
@@ -121,7 +118,8 @@ exit:
   ret i32 %res
 }
 
-; Subtraction also provides enough for equality comparisons with zero.
+; Subtraction provides in addition also enough for equality comparisons with
+; zero even without "nsw".
 define i32 @f7(i32 %a, i32 %b, i32 *%dest) {
 ; CHECK-LABEL: f7:
 ; CHECK: s %r2, 0(%r4)
@@ -141,15 +139,14 @@ exit:
   ret i32 %res
 }
 
-; ...but not for ordered comparisons.
+; ...and again with SLT.
 define i32 @f8(i32 %a, i32 %b, i32 *%dest) {
 ; CHECK-LABEL: f8:
 ; CHECK: s %r2, 0(%r4)
-; CHECK-NEXT: cibl %r2, 0, 0(%r14)
-; CHECK: br %r14
+; CHECK-NEXT: blr %r14
 entry:
   %cur = load i32, i32 *%dest
-  %res = sub i32 %a, %cur
+  %res = sub nsw i32 %a, %cur
   %cmp = icmp slt i32 %res, 0
   br i1 %cmp, label %exit, label %store
 
@@ -445,10 +442,10 @@ define i32 @f23(i32 %a, i32 %b, i32 *%dest1, i32 *%dest2) {
 ; CHECK-LABEL: f23:
 ; CHECK: afi %r2, 1000000
 ; CHECK-NEXT: st %r2, 0(%r4)
-; CHECK-NEXT: bner %r14
+; CHECK-NEXT: blhr %r14
 ; CHECK: br %r14
 entry:
-  %res = add i32 %a, 1000000
+  %res = add nsw i32 %a, 1000000
   store i32 %res, i32 *%dest1
   %cmp = icmp ne i32 %res, 0
   br i1 %cmp, label %exit, label %store
@@ -491,10 +488,10 @@ define void @f25(i32 %a, i32 *%ptr) {
 ; CHECK-NEXT: #APP
 ; CHECK-NEXT: blah
 ; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: bner %r14
+; CHECK-NEXT: blhr %r14
 ; CHECK: br %r14
 entry:
-  %add = add i32 %a, 1000000
+  %add = add nsw i32 %a, 1000000
   call void asm sideeffect "blah", "r"(i32 %add)
   %cmp = icmp ne i32 %add, 0
   br i1 %cmp, label %exit, label %store
@@ -540,7 +537,7 @@ define i32 @f27(i32 %a, i32 %b, i32 *%dest1, i32 *%dest2) {
 ; CHECK-NEXT: cibe %r2, 0, 0(%r14)
 ; CHECK: br %r14
 entry:
-  %add = add i32 %a, 1000000
+  %add = add nsw i32 %a, 1000000
   %sub = sub i32 %b, %add
   store i32 %sub, i32 *%dest1
   %cmp = icmp eq i32 %add, 0
diff --git a/llvm/test/CodeGen/SystemZ/int-cmp-45.ll b/llvm/test/CodeGen/SystemZ/int-cmp-45.ll
index e07d37e57..296cf4497 100644
--- a/llvm/test/CodeGen/SystemZ/int-cmp-45.ll
+++ b/llvm/test/CodeGen/SystemZ/int-cmp-45.ll
@@ -3,14 +3,15 @@
 ;
 ; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z196 -no-integrated-as | FileCheck %s
 
-; Addition provides enough for equality comparisons with zero.  First teest
-; the EQ case with LOC.
+; Addition provides enough for comparisons with zero if we know no
+; signed overflow happens, which is when the "nsw" flag is set.
+; First test the EQ case with LOC.
 define i32 @f1(i32 %a, i32 %b, i32 *%cptr) {
 ; CHECK-LABEL: f1:
 ; CHECK: afi %r2, 1000000
 ; CHECK-NEXT: loce %r3, 0(%r4)
 ; CHECK: br %r14
-  %add = add i32 %a, 1000000
+  %add = add nsw i32 %a, 1000000
   %cmp = icmp eq i32 %add, 0
   %c = load i32, i32 *%cptr
   %arg = select i1 %cmp, i32 %c, i32 %b
@@ -24,7 +25,7 @@ define i32 @f2(i32 %a, i32 %b, i32 *%cptr) {
 ; CHECK: afi %r2, 1000000
 ; CHECK-NEXT: stoce %r3, 0(%r4)
 ; CHECK: br %r14
-  %add = add i32 %a, 1000000
+  %add = add nsw i32 %a, 1000000
   %cmp = icmp eq i32 %add, 0
   %c = load i32, i32 *%cptr
   %newval = select i1 %cmp, i32 %b, i32 %c
@@ -36,9 +37,9 @@ define i32 @f2(i32 %a, i32 %b, i32 *%cptr) {
 define i32 @f3(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: f3:
 ; CHECK: afi %r2, 1000000
-; CHECK-NEXT: locrne %r3, %r4
+; CHECK-NEXT: locrlh %r3, %r4
 ; CHECK: br %r14
-  %add = add i32 %a, 1000000
+  %add = add nsw i32 %a, 1000000
   %cmp = icmp eq i32 %add, 0
   %arg = select i1 %cmp, i32 %b, i32 %c
   call void asm sideeffect "blah $0", "{r3}"(i32 %arg)
@@ -49,9 +50,9 @@ define i32 @f3(i32 %a, i32 %b, i32 %c) {
 define i32 @f4(i32 %a, i32 %b, i32 *%cptr) {
 ; CHECK-LABEL: f4:
 ; CHECK: afi %r2, 1000000
-; CHECK-NEXT: locne %r3, 0(%r4)
+; CHECK-NEXT: loclh %r3, 0(%r4)
 ; CHECK: br %r14
-  %add = add i32 %a, 1000000
+  %add = add nsw i32 %a, 1000000
   %cmp = icmp eq i32 %add, 0
   %c = load i32, i32 *%cptr
   %arg = select i1 %cmp, i32 %b, i32 %c
@@ -63,9 +64,9 @@ define i32 @f4(i32 %a, i32 %b, i32 *%cptr) {
 define i32 @f5(i32 %a, i32 %b, i32 *%cptr) {
 ; CHECK-LABEL: f5:
 ; CHECK: afi %r2, 1000000
-; CHECK-NEXT: stocne %r3, 0(%r4)
+; CHECK-NEXT: stoclh %r3, 0(%r4)
 ; CHECK: br %r14
-  %add = add i32 %a, 1000000
+  %add = add nsw i32 %a, 1000000
   %cmp = icmp eq i32 %add, 0
   %c = load i32, i32 *%cptr
   %newval = select i1 %cmp, i32 %c, i32 %b
@@ -79,7 +80,7 @@ define i32 @f6(i32 %a, i32 %b, i32 %c) {
 ; CHECK: afi %r2, 1000000
 ; CHECK-NEXT: locre %r3, %r4
 ; CHECK: br %r14
-  %add = add i32 %a, 1000000
+  %add = add nsw i32 %a, 1000000
   %cmp = icmp ne i32 %add, 0
   %arg = select i1 %cmp, i32 %b, i32 %c
   call void asm sideeffect "blah $0", "{r3}"(i32 %arg)
@@ -92,7 +93,7 @@ define i32 @f7(i32 %a, i32 %b, i32 *%cptr) {
 ; CHECK: afi %r2, 1000000
 ; CHECK-NEXT: loce %r3, 0(%r4)
 ; CHECK: br %r14
-  %add = add i32 %a, 1000000
+  %add = add nsw i32 %a, 1000000
   %cmp = icmp ne i32 %add, 0
   %c = load i32, i32 *%cptr
   %arg = select i1 %cmp, i32 %b, i32 %c
@@ -106,7 +107,7 @@ define i32 @f8(i32 %a, i32 %b, i32 *%cptr) {
 ; CHECK: afi %r2, 1000000
 ; CHECK-NEXT: stoce %r3, 0(%r4)
 ; CHECK: br %r14
-  %add = add i32 %a, 1000000
+  %add = add nsw i32 %a, 1000000
   %cmp = icmp ne i32 %add, 0
   %c = load i32, i32 *%cptr
   %newval = select i1 %cmp, i32 %c, i32 %b
diff --git a/llvm/test/CodeGen/SystemZ/int-cmp-56.ll b/llvm/test/CodeGen/SystemZ/int-cmp-56.ll
new file mode 100644
index 000000000..5b55e4a81
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/int-cmp-56.ll
@@ -0,0 +1,163 @@
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z14 | FileCheck %s
+;
+; Check that signed comparisons against 0 are only eliminated if the "nsw"
+; flag is present on the defining add (with register) instruction. For an
+; equality comparison, add logical can be used.
+
+define i32 @fun0(i32 %arg, i32 %arg2, i32 %arg3) {
+; CHECK-LABEL: fun0:
+; CHECK: jle .LBB0_2{{$}}
+; CHECK: je .LBB0_4{{$}}
+
+bb:
+  %tmp2 = add nsw i32 %arg, %arg2
+  %tmp3 = icmp sgt i32 %tmp2, 0
+  br i1 %tmp3, label %bb3, label %bb1
+
+bb1:
+  %tmp4 = add nsw i32 %arg, %arg3
+  %tmp5 = icmp eq i32 %tmp4, 0
+  br i1 %tmp5, label %bb4, label %bb2
+
+bb2:
+  ret i32 0
+
+bb3:
+  ret i32 1
+
+bb4:
+  ret i32 2
+}
+
+; No "nsw" flag
+define i32 @fun1(i32 %arg, i32 %arg2, i32 %arg3) {
+; CHECK-LABEL: fun1:
+; CHECK: cijle
+; CHECK: alr
+; CHECK: jhe
+
+bb:
+  %tmp2 = add i32 %arg, %arg2
+  %tmp3 = icmp sgt i32 %tmp2, 0
+  br i1 %tmp3, label %bb3, label %bb1
+
+bb1:
+  %tmp4 = add i32 %arg, %arg3
+  %tmp5 = icmp eq i32 %tmp4, 0
+  br i1 %tmp5, label %bb4, label %bb2
+
+bb2:
+  ret i32 0
+
+bb3:
+  ret i32 1
+
+bb4:
+  ret i32 2
+}
+
+; "nuw" flag
+define i32 @fun2(i32 %arg, i32 %arg2, i32 %arg3) {
+; CHECK-LABEL: fun2:
+; CHECK: cijle
+; CHECK: alr
+; CHECK: jhe
+
+bb:
+  %tmp2 = add nuw i32 %arg, %arg2
+  %tmp3 = icmp sgt i32 %tmp2, 0
+  br i1 %tmp3, label %bb3, label %bb1
+
+bb1:
+  %tmp4 = add nuw i32 %arg, %arg3
+  %tmp5 = icmp eq i32 %tmp4, 0
+  br i1 %tmp5, label %bb4, label %bb2
+
+bb2:
+  ret i32 0
+
+bb3:
+  ret i32 1
+
+bb4:
+  ret i32 2
+}
+
+; Subtraction does not produce the value of zero in case of overflow, so
+; "nsw" is not needed for the equality check against zero.
+define i32 @fun3(i32 %arg, i32 %arg2, i32 %arg3) {
+; CHECK-LABEL: fun3:
+; CHECK: jle .LBB3_2{{$}}
+; CHECK: je .LBB3_4{{$}}
+
+bb:
+  %tmp2 = sub nsw i32 %arg, %arg2
+  %tmp3 = icmp sgt i32 %tmp2, 0
+  br i1 %tmp3, label %bb3, label %bb1
+
+bb1:
+  %tmp4 = sub nsw i32 %arg, %arg3
+  %tmp5 = icmp eq i32 %tmp4, 0
+  br i1 %tmp5, label %bb4, label %bb2
+
+bb2:
+  ret i32 0
+
+bb3:
+  ret i32 1
+
+bb4:
+  ret i32 2
+}
+
+; No "nsw" flag
+define i32 @fun4(i32 %arg, i32 %arg2, i32 %arg3) {
+; CHECK-LABEL: fun4:
+; CHECK: cijle
+; CHECK: je .LBB4_4{{$}}
+
+bb:
+  %tmp2 = sub i32 %arg, %arg2
+  %tmp3 = icmp sgt i32 %tmp2, 0
+  br i1 %tmp3, label %bb3, label %bb1
+
+bb1:
+  %tmp4 = sub i32 %arg, %arg3
+  %tmp5 = icmp eq i32 %tmp4, 0
+  br i1 %tmp5, label %bb4, label %bb2
+
+bb2:
+  ret i32 0
+
+bb3:
+  ret i32 1
+
+bb4:
+  ret i32 2
+}
+
+; "nuw" flag
+define i32 @fun5(i32 %arg, i32 %arg2, i32 %arg3) {
+; CHECK-LABEL: fun5:
+; CHECK: cijle
+; CHECK: je .LBB5_4{{$}}
+
+bb:
+  %tmp2 = sub nuw i32 %arg, %arg2
+  %tmp3 = icmp sgt i32 %tmp2, 0
+  br i1 %tmp3, label %bb3, label %bb1
+
+bb1:
+  %tmp4 = sub nuw i32 %arg, %arg3
+  %tmp5 = icmp eq i32 %tmp4, 0
+  br i1 %tmp5, label %bb4, label %bb2
+
+bb2:
+  ret i32 0
+
+bb3:
+  ret i32 1
+
+bb4:
+  ret i32 2
+}
diff --git a/llvm/test/CodeGen/SystemZ/int-cmp-57.ll b/llvm/test/CodeGen/SystemZ/int-cmp-57.ll
new file mode 100644
index 000000000..a49edc196
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/int-cmp-57.ll
@@ -0,0 +1,103 @@
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z14 -disable-cgp | FileCheck %s
+;
+; Check that signed comparisons against 0 are eliminated if the defining
+; instruction is an add with immediate.
+;
+; Addition of an immediate does not depend on the "nsw" flag, since the
+; result can be predicted in case of overflow. For example, if adding a
+; positive immediate gives overflow, the result must be negative.
+
+; Addition of a negative immediate gives a positive result in case of
+; overflow (except for the case of the minimum value which may also result in
+; a zero result).
+define i32 @fun0(i32 %arg) {
+; CHECK-LABEL: fun0:
+; CHECK: ahik
+; CHECK-NEXT: locre
+bb:
+  %tmp = add i32 %arg, -1
+  %tmp1 = icmp eq i32 %tmp, 0
+  %res = select i1 %tmp1, i32 %tmp, i32 %arg
+  ret i32 %res
+}
+
+define i32 @fun1(i32 %arg) {
+; CHECK-LABEL: fun1:
+; CHECK: ahik
+; CHECK-NEXT: locrnle
+bb:
+  %tmp = add i32 %arg, -1
+  %tmp1 = icmp sgt i32 %tmp, 0
+  %res = select i1 %tmp1, i32 %tmp, i32 %arg
+  ret i32 %res
+}
+
+define i32 @fun2(i32 %arg) {
+; CHECK-LABEL: fun2:
+; CHECK: ahik
+; CHECK-NEXT: locrl
+bb:
+  %tmp = add i32 %arg, -1
+  %tmp1 = icmp slt i32 %tmp, 0
+  %res = select i1 %tmp1, i32 %tmp, i32 %arg
+  ret i32 %res
+}
+
+; Addition of a positive immediate gives a negative result in case of overflow.
+define i32 @fun3(i32 %arg) {
+; CHECK-LABEL: fun3:
+; CHECK: ahik
+; CHECK-NEXT: locre
+bb:
+  %tmp = add i32 %arg, 1
+  %tmp1 = icmp eq i32 %tmp, 0
+  %res = select i1 %tmp1, i32 %tmp, i32 %arg
+  ret i32 %res
+}
+
+define i32 @fun4(i32 %arg) {
+; CHECK-LABEL: fun4:
+; CHECK: ahik
+; CHECK-NEXT: locrh
+bb:
+  %tmp = add i32 %arg, 1
+  %tmp1 = icmp sgt i32 %tmp, 0
+  %res = select i1 %tmp1, i32 %tmp, i32 %arg
+  ret i32 %res
+}
+
+define i32 @fun5(i32 %arg) {
+; CHECK-LABEL: fun5:
+; CHECK: ahik
+; CHECK-NEXT: locrnhe
+bb:
+  %tmp = add i32 %arg, 1
+  %tmp1 = icmp slt i32 %tmp, 0
+  %res = select i1 %tmp1, i32 %tmp, i32 %arg
+  ret i32 %res
+}
+
+; Addition of the minimum value gives a positive or zero result.
+define i32 @fun6(i32 %arg) {
+; CHECK-LABEL: fun6:
+; CHECK: afi
+; CHECK-NEXT: chi
+; CHECK-NEXT: locrlh
+bb:
+  %tmp = add i32 %arg, -2147483648
+  %tmp1 = icmp eq i32 %tmp, 0
+  %res = select i1 %tmp1, i32 %tmp, i32 %arg
+  ret i32 %res
+}
+
+define i32 @fun7(i32 %arg) {
+; CHECK-LABEL: fun7:
+; CHECK: afi
+; CHECK-NEXT: chi
+; CHECK-NEXT: locrle
+bb:
+  %tmp = add i32 %arg, -2147483648
+  %tmp1 = icmp sgt i32 %tmp, 0
+  %res = select i1 %tmp1, i32 %tmp, i32 %arg
+  ret i32 %res
+}
diff --git a/llvm/test/CodeGen/SystemZ/int-uadd-03.ll b/llvm/test/CodeGen/SystemZ/int-uadd-03.ll
index d57f8a844..b7b9883ec 100644
--- a/llvm/test/CodeGen/SystemZ/int-uadd-03.ll
+++ b/llvm/test/CodeGen/SystemZ/int-uadd-03.ll
@@ -199,7 +199,7 @@ define zeroext i1 @f10(i64 %src, i64 %index, i64 %a, i64 *%res) {
 define zeroext i1 @f11(i32 *%ptr0) {
 ; CHECK-LABEL: f11:
 ; CHECK: brasl %r14, foo@PLT
-; CHECK: algf %r2, 160(%r15)
+; CHECK: algf {{%r[0-9]+}}, 160(%r15)
 ; CHECK: br %r14
   %ptr1 = getelementptr i32, i32 *%ptr0, i64 2
   %ptr2 = getelementptr i32, i32 *%ptr0, i64 4
diff --git a/llvm/test/CodeGen/SystemZ/int-usub-03.ll b/llvm/test/CodeGen/SystemZ/int-usub-03.ll
index 4e5f99fce..5e0a94777 100644
--- a/llvm/test/CodeGen/SystemZ/int-usub-03.ll
+++ b/llvm/test/CodeGen/SystemZ/int-usub-03.ll
@@ -207,7 +207,7 @@ define zeroext i1 @f10(i64 %src, i64 %index, i64 %a, i64 *%res) {
 define zeroext i1 @f11(i32 *%ptr0) {
 ; CHECK-LABEL: f11:
 ; CHECK: brasl %r14, foo@PLT
-; CHECK: slgf %r2, 160(%r15)
+; CHECK: slgf {{%r[0-9]+}}, 160(%r15)
 ; CHECK: br %r14
   %ptr1 = getelementptr i32, i32 *%ptr0, i64 2
   %ptr2 = getelementptr i32, i32 *%ptr0, i64 4
diff --git a/llvm/test/CodeGen/SystemZ/ipra-04.ll b/llvm/test/CodeGen/SystemZ/ipra-04.ll
index 516d85952..55a849e18 100644
--- a/llvm/test/CodeGen/SystemZ/ipra-04.ll
+++ b/llvm/test/CodeGen/SystemZ/ipra-04.ll
@@ -31,4 +31,4 @@ bb:
   ret void
 }
 
-attributes #0 = { norecurse nounwind "no-frame-pointer-elim"="false" }
+attributes #0 = { norecurse nounwind "frame-pointer"="none" }
diff --git a/llvm/test/CodeGen/SystemZ/la-05.ll b/llvm/test/CodeGen/SystemZ/la-05.ll
new file mode 100644
index 000000000..27d7d91e6
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/la-05.ll
@@ -0,0 +1,31 @@
+; Test that a huge address offset is loaded into a register and then added
+; separately.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+@a = common dso_local global i32 0, align 4
+
+define i64 @f1() {
+; CHECK-LABEL: f1:
+; CHECK: llihl   %r0, 829
+; CHECK: oilf    %r0, 4294966308
+; CHECK: larl    %r2, a
+; CHECK: agr     %r2, %r0
+; CHECK: br      %r14
+  ret i64 add (i64 ptrtoint (i32* @a to i64), i64 3564822854692)
+}
+
+define signext i32 @f2() {
+; CHECK-LABEL: f2:
+; CHECK: llihl   %r0, 829
+; CHECK: oilf    %r0, 4294966308
+; CHECK: larl    %r1, a
+; CHECK: agr     %r1, %r0
+; CHECK: lgf     %r2, 0(%r1)
+; CHECK: br      %r14
+entry:
+  %0 = load i32, i32* inttoptr (i64 add (i64 ptrtoint (i32* @a to i64),
+                                i64 3564822854692) to i32*)
+  ret i32 %0
+}
+
diff --git a/llvm/test/CodeGen/SystemZ/mature-mc-support.ll b/llvm/test/CodeGen/SystemZ/mature-mc-support.ll
index a01716c27..bf82c363f 100644
--- a/llvm/test/CodeGen/SystemZ/mature-mc-support.ll
+++ b/llvm/test/CodeGen/SystemZ/mature-mc-support.ll
@@ -1,12 +1,10 @@
 ; Test that inline assembly is parsed by the MC layer when MC support is mature
 ; (even when the output is assembly).
-; FIXME: SystemZ doesn't use the integrated assembler by default so we only test
-; that -filetype=obj tries to parse the assembly.
 
-; SKIP: not llc -march=systemz < %s > /dev/null 2> %t1
-; SKIP: FileCheck %s < %t1
+; RUN: not llc -mtriple=s390x-linux-gnu < %s > /dev/null 2> %t1
+; RUN: FileCheck %s < %t1
 
-; RUN: not llc -march=systemz -filetype=obj < %s > /dev/null 2> %t2
+; RUN: not llc -mtriple=s390x-linux-gnu -filetype=obj < %s > /dev/null 2> %t2
 ; RUN: FileCheck %s < %t2
 
 
diff --git a/llvm/test/CodeGen/SystemZ/mnop-mcount-01.ll b/llvm/test/CodeGen/SystemZ/mnop-mcount-01.ll
new file mode 100644
index 000000000..99aff5a22
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/mnop-mcount-01.ll
@@ -0,0 +1,26 @@
+; RUN: llc %s -mtriple=s390x-linux-gnu -mcpu=z10 -o - -verify-machineinstrs \
+; RUN:   | FileCheck %s
+
+define void @test1() #0 {
+entry:
+  ret void
+
+; CHECK-LABEL: @test1
+; CHECK: brasl %r0, __fentry__
+; CHECK-NOT: brcl 0, .Ltmp0
+; CHECK: br %r14
+}
+
+define void @test2() #1 {
+entry:
+  ret void
+
+; CHECK-LABEL: @test2
+; CHECK-NOT: brasl %r0, __fentry__
+; CHECK: brcl 0, .Ltmp0
+; CHECK: br %r14
+}
+
+attributes #0 = { "fentry-call"="true" }
+attributes #1 = { "fentry-call"="true" "mnop-mcount" }
+
diff --git a/llvm/test/CodeGen/SystemZ/mnop-mcount-02.ll b/llvm/test/CodeGen/SystemZ/mnop-mcount-02.ll
new file mode 100644
index 000000000..4a3629111
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/mnop-mcount-02.ll
@@ -0,0 +1,10 @@
+; RUN: not llc %s -mtriple=s390x-linux-gnu -o - 2>&1 | FileCheck %s
+;
+; CHECK: LLVM ERROR: mnop-mcount only supported with fentry-call
+
+define void @test1() #0 {
+entry:
+  ret void
+}
+
+attributes #0 = { "instrument-function-entry-inlined"="mcount" "mnop-mcount" }
diff --git a/llvm/test/CodeGen/SystemZ/mrecord-mcount-01.ll b/llvm/test/CodeGen/SystemZ/mrecord-mcount-01.ll
new file mode 100644
index 000000000..20dd387fb
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/mrecord-mcount-01.ll
@@ -0,0 +1,32 @@
+; RUN: llc %s -mtriple=s390x-linux-gnu -mcpu=z10 -o - -verify-machineinstrs \
+; RUN:   | FileCheck %s
+
+define void @test1() #0 {
+entry:
+  ret void
+
+; CHECK-LABEL: test1:
+; CHECK: .section __mcount_loc,"a",@progbits
+; CHECK: .quad .Ltmp0
+; CHECK: .text
+; CHECK: .Ltmp0:
+; CHECK: brasl %r0, __fentry__@PLT
+; CHECK: br %r14
+}
+
+define void @test2() #1 {
+entry:
+  ret void
+
+; CHECK-LABEL: test2:
+; CHECK: .section __mcount_loc,"a",@progbits
+; CHECK: .quad .Ltmp1
+; CHECK: .text
+; CHECK: .Ltmp1:
+; CHECK: .Ltmp2:
+; CHECK: brcl 0, .Ltmp2
+; CHECK: br %r14
+}
+
+attributes #0 = { "fentry-call"="true" "mrecord-mcount" }
+attributes #1 = { "fentry-call"="true" "mnop-mcount" "mrecord-mcount" }
diff --git a/llvm/test/CodeGen/SystemZ/mrecord-mcount-02.ll b/llvm/test/CodeGen/SystemZ/mrecord-mcount-02.ll
new file mode 100644
index 000000000..9ce7cdd41
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/mrecord-mcount-02.ll
@@ -0,0 +1,10 @@
+; RUN: not llc %s -mtriple=s390x-linux-gnu -o - 2>&1 | FileCheck %s
+;
+; CHECK: LLVM ERROR: mrecord-mcount only supported with fentry-call
+
+define void @test1() #0 {
+entry:
+  ret void
+}
+
+attributes #0 = { "instrument-function-entry-inlined"="mcount" "mrecord-mcount" }
diff --git a/llvm/test/CodeGen/SystemZ/multiselect.ll b/llvm/test/CodeGen/SystemZ/multiselect.ll
index d2078ff29..672a302a5 100644
--- a/llvm/test/CodeGen/SystemZ/multiselect.ll
+++ b/llvm/test/CodeGen/SystemZ/multiselect.ll
@@ -1,10 +1,11 @@
 ; Test that multiple select statements using the same condition are expanded
-; into a single conditional branch.
+; into a single conditional branch when possible.
 ;
-; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -disable-block-placement | FileCheck %s
 
-define void @test(i32 signext %positive, double %base, double %offset, double* %rmin, double* %rmax) {
+define void @test0(i32 signext %positive, double %base, double %offset, double* %rmin, double* %rmax) {
 entry:
+; CHECK-LABEL: test0
 ; CHECK: cijlh %r2, 0,
 ; CHECK-NOT: cij
 ; CHECK-NOT: je
@@ -19,3 +20,51 @@ entry:
   ret void
 }
 
+; Two selects with an intervening instruction that doesn't clobber CC can
+; still be merged.
+define double @test1(i32 signext %positive, double %A, double %B, double %C) {
+entry:
+; CHECK-LABEL: test1
+; CHECK: cijhe {{.*}}LBB1_2
+; CHECK-NOT: cij
+; CHECK: br %r14
+
+  %tobool = icmp slt i32 %positive, 0
+  %s1  = select i1 %tobool, double %A, double %B
+  %mul = fmul double %A, %B
+  %s2  = select i1 %tobool, double %B, double %C
+  %add = fadd double %s1, %s2
+  %add2 = fadd double %add, %mul
+  ret double %add2
+}
+
+; Two selects with an intervening user of the first select can't be merged.
+define double @test2(i32 signext %positive, double %A, double %B) {
+entry:
+; CHECK-LABEL: test2
+; CHECK: cije {{.*}}LBB2_2
+; CHECK: cibe {{.*}}%r14
+; CHECK: br %r14
+
+  %tobool = icmp eq i32 %positive, 0
+  %s1  = select i1 %tobool, double %A, double %B
+  %add = fadd double %A, %s1
+  %s2  = select i1 %tobool, double %A, double %add
+  ret double %s2
+}
+
+; Two selects with different conditions can't be merged
+define double @test3(i32 signext %positive, double %A, double %B, double %C) {
+entry:
+; CHECK-LABEL: test3
+; CHECK: cijl {{.*}}LBB3_2
+; CHECK: cijl {{.*}}LBB3_4
+; CHECK: br %r14
+
+  %tobool = icmp slt i32 %positive, 0
+  %s1  = select i1 %tobool, double %A, double %B
+  %tobool2 = icmp slt i32 %positive, 2
+  %s2  = select i1 %tobool2, double %B, double %C
+  %add = fadd double %s1, %s2
+  ret double %add
+}
diff --git a/llvm/test/CodeGen/SystemZ/not-01.ll b/llvm/test/CodeGen/SystemZ/not-01.ll
index 3b9dbd131..62248d6a1 100644
--- a/llvm/test/CodeGen/SystemZ/not-01.ll
+++ b/llvm/test/CodeGen/SystemZ/not-01.ll
@@ -1,6 +1,6 @@
-; Combined logical operations involving complement on arch13
+; Combined logical operations involving complement on z15
 ;
-; RUN: llc -mcpu=arch13 < %s -mtriple=s390x-linux-gnu | FileCheck %s
+; RUN: llc -mcpu=z15 < %s -mtriple=s390x-linux-gnu | FileCheck %s
 
 ; And-with-complement 32-bit.
 define i32 @f1(i32 %dummy, i32 %a, i32 %b) {
@@ -124,3 +124,29 @@ define i64 @f12(i64 %a) {
   ret i64 %ret
 }
 
+; NXOR 32-bit (alternate match).
+define i32 @f13(i32 %a) {
+; CHECK-LABEL: f13:
+; CHECK: lhi [[REG:%r[0-5]]], -256
+; CHECK: nxrk %r2, %r2, [[REG]]
+; CHECK: br %r14
+  ; Use an opaque const so the pattern doesn't get optimized away early.
+  %const = bitcast i32 -256 to i32
+  %neg = xor i32 %a, -1
+  %ret = xor i32 %neg, %const
+  ret i32 %ret
+}
+
+; NXOR 64-bit (alternate match).
+define i64 @f14(i64 %a) {
+; CHECK-LABEL: f14:
+; CHECK: lghi [[REG:%r[0-5]]], -256
+; CHECK: nxgrk %r2, %r2, [[REG]]
+; CHECK: br %r14
+  ; Use an opaque const so the pattern doesn't get optimized away early.
+  %const = bitcast i64 -256 to i64
+  %neg = xor i64 %a, -1
+  %ret = xor i64 %neg, %const
+  ret i64 %ret
+}
+
diff --git a/llvm/test/CodeGen/SystemZ/stack-size-section.ll b/llvm/test/CodeGen/SystemZ/stack-size-section.ll
index 6aacc7051..d1d79531c 100644
--- a/llvm/test/CodeGen/SystemZ/stack-size-section.ll
+++ b/llvm/test/CodeGen/SystemZ/stack-size-section.ll
@@ -38,4 +38,4 @@ define void @dynalloc(i32 %N) #0 {
   ret void
 }
 
-attributes #0 = { "no-frame-pointer-elim"="true" }
+attributes #0 = { "frame-pointer"="all" }
diff --git a/llvm/test/CodeGen/SystemZ/stackmap-nops.ll b/llvm/test/CodeGen/SystemZ/stackmap-nops.ll
index 066d7f613..f92248e22 100644
--- a/llvm/test/CodeGen/SystemZ/stackmap-nops.ll
+++ b/llvm/test/CodeGen/SystemZ/stackmap-nops.ll
@@ -11,108 +11,121 @@ entry:
 ; CHECK:      bc 0, 0
 
 ; 6
-; CHECK:      brcl 0, [[LAB:.Ltmp[0-9]+]]
-; CHECK-NEXT: [[LAB]]:
+; CHECK:      .Ltmp
+; CHECK-NEXT: [[LAB:.Ltmp[0-9]+]]:
+; CHECK-NEXT: brcl 0, [[LAB]]
 
 ; 8
-; CHECK:      brcl 0, [[LAB:.Ltmp[0-9]+]]
-; CHECK-NEXT: [[LAB]]:
+; CHECK:      .Ltmp
+; CHECK-NEXT: [[LAB:.Ltmp[0-9]+]]:
+; CHECK-NEXT: brcl 0, [[LAB]]
 ; CHECK:      bcr 0, %r0
 
 ; 10
-; CHECK:      brcl 0, [[LAB:.Ltmp[0-9]+]]
-; CHECK-NEXT: [[LAB]]:
+; CHECK:      .Ltmp
+; CHECK-NEXT: [[LAB:.Ltmp[0-9]+]]:
+; CHECK-NEXT: brcl 0, [[LAB]]
 ; CHECK:      bc 0, 0
 
 ; 12
-; CHECK:      brcl 0, [[LAB:.Ltmp[0-9]+]]
-; CHECK-NEXT: [[LAB]]:
-; CHECK:      brcl 0, [[LAB:.Ltmp[0-9]+]]
-; CHECK-NEXT: [[LAB]]:
+; CHECK:      .Ltmp
+; CHECK-NEXT: [[LAB:.Ltmp[0-9]+]]:
+; CHECK-NEXT: brcl 0, [[LAB]]
+; CHECK:      [[LAB:.Ltmp[0-9]+]]:
+; CHECK-NEXT: brcl 0, [[LAB]]
 
 ; 14
-; CHECK:      brcl 0, [[LAB:.Ltmp[0-9]+]]
-; CHECK-NEXT: [[LAB]]:
-; CHECK:      brcl 0, [[LAB:.Ltmp[0-9]+]]
-; CHECK-NEXT: [[LAB]]:
+; CHECK:      .Ltmp
+; CHECK-NEXT: [[LAB:.Ltmp[0-9]+]]:
+; CHECK-NEXT: brcl 0, [[LAB]]
+; CHECK:      [[LAB:.Ltmp[0-9]+]]:
+; CHECK-NEXT: brcl 0, [[LAB]]
 ; CHECK:      bcr 0, %r0
 
 ; 16
-; CHECK:      brcl 0, [[LAB:.Ltmp[0-9]+]]
-; CHECK-NEXT: [[LAB]]:
-; CHECK:      brcl 0, [[LAB:.Ltmp[0-9]+]]
-; CHECK-NEXT: [[LAB]]:
+; CHECK:      .Ltmp
+; CHECK-NEXT: [[LAB:.Ltmp[0-9]+]]:
+; CHECK-NEXT: brcl 0, [[LAB]]
+; CHECK:      [[LAB:.Ltmp[0-9]+]]:
+; CHECK-NEXT: brcl 0, [[LAB]]
 ; CHECK:      bc 0, 0
 
 ; 18
-; CHECK:      brcl 0, [[LAB:.Ltmp[0-9]+]]
-; CHECK-NEXT: [[LAB]]:
-; CHECK:      brcl 0, [[LAB:.Ltmp[0-9]+]]
-; CHECK-NEXT: [[LAB]]:
-; CHECK:      brcl 0, [[LAB:.Ltmp[0-9]+]]
-; CHECK-NEXT: [[LAB]]:
+; CHECK:      .Ltmp
+; CHECK-NEXT: [[LAB:.Ltmp[0-9]+]]:
+; CHECK-NEXT: brcl 0, [[LAB]]
+; CHECK:      [[LAB:.Ltmp[0-9]+]]:
+; CHECK-NEXT: brcl 0, [[LAB]]
+; CHECK:      [[LAB:.Ltmp[0-9]+]]:
+; CHECK-NEXT: brcl 0, [[LAB]]
 
 ; 20
-; CHECK:      brcl 0, [[LAB:.Ltmp[0-9]+]]
-; CHECK-NEXT: [[LAB]]:
-; CHECK:      brcl 0, [[LAB:.Ltmp[0-9]+]]
-; CHECK-NEXT: [[LAB]]:
-; CHECK:      brcl 0, [[LAB:.Ltmp[0-9]+]]
-; CHECK-NEXT: [[LAB]]:
+; CHECK:      .Ltmp
+; CHECK-NEXT: [[LAB:.Ltmp[0-9]+]]:
+; CHECK-NEXT: brcl 0, [[LAB]]
+; CHECK:      [[LAB:.Ltmp[0-9]+]]:
+; CHECK-NEXT: brcl 0, [[LAB]]
+; CHECK:      [[LAB:.Ltmp[0-9]+]]:
+; CHECK-NEXT: brcl 0, [[LAB]]
 ; CHECK:      bcr 0, %r0
 
 ; 22
-; CHECK:      brcl 0, [[LAB:.Ltmp[0-9]+]]
-; CHECK-NEXT: [[LAB]]:
-; CHECK:      brcl 0, [[LAB:.Ltmp[0-9]+]]
-; CHECK-NEXT: [[LAB]]:
-; CHECK:      brcl 0, [[LAB:.Ltmp[0-9]+]]
-; CHECK-NEXT: [[LAB]]:
+; CHECK:      .Ltmp
+; CHECK-NEXT: [[LAB:.Ltmp[0-9]+]]:
+; CHECK-NEXT: brcl 0, [[LAB]]
+; CHECK:      [[LAB:.Ltmp[0-9]+]]:
+; CHECK-NEXT: brcl 0, [[LAB]]
+; CHECK:      [[LAB:.Ltmp[0-9]+]]:
+; CHECK-NEXT: brcl 0, [[LAB]]
 ; CHECK:      bc 0, 0
 
 ; 24
-; CHECK:      brcl 0, [[LAB:.Ltmp[0-9]+]]
-; CHECK-NEXT: [[LAB]]:
-; CHECK:      brcl 0, [[LAB:.Ltmp[0-9]+]]
-; CHECK-NEXT: [[LAB]]:
-; CHECK:      brcl 0, [[LAB:.Ltmp[0-9]+]]
-; CHECK-NEXT: [[LAB]]:
-; CHECK:      brcl 0, [[LAB:.Ltmp[0-9]+]]
-; CHECK-NEXT: [[LAB]]:
+; CHECK:      .Ltmp
+; CHECK-NEXT: [[LAB:.Ltmp[0-9]+]]:
+; CHECK-NEXT: brcl 0, [[LAB]]
+; CHECK:      [[LAB:.Ltmp[0-9]+]]:
+; CHECK-NEXT: brcl 0, [[LAB]]
+; CHECK:      [[LAB:.Ltmp[0-9]+]]:
+; CHECK-NEXT: brcl 0, [[LAB]]
+; CHECK:      [[LAB:.Ltmp[0-9]+]]:
+; CHECK-NEXT: brcl 0, [[LAB]]
 
 ; 26
-; CHECK:      brcl 0, [[LAB:.Ltmp[0-9]+]]
-; CHECK-NEXT: [[LAB]]:
-; CHECK:      brcl 0, [[LAB:.Ltmp[0-9]+]]
-; CHECK-NEXT: [[LAB]]:
-; CHECK:      brcl 0, [[LAB:.Ltmp[0-9]+]]
-; CHECK-NEXT: [[LAB]]:
-; CHECK:      brcl 0, [[LAB:.Ltmp[0-9]+]]
-; CHECK-NEXT: [[LAB]]:
+; CHECK:      .Ltmp
+; CHECK-NEXT: [[LAB:.Ltmp[0-9]+]]:
+; CHECK-NEXT: brcl 0, [[LAB]]
+; CHECK:      [[LAB:.Ltmp[0-9]+]]:
+; CHECK-NEXT: brcl 0, [[LAB]]
+; CHECK:      [[LAB:.Ltmp[0-9]+]]:
+; CHECK-NEXT: brcl 0, [[LAB]]
+; CHECK:      [[LAB:.Ltmp[0-9]+]]:
+; CHECK-NEXT: brcl 0, [[LAB]]
 ; CHECK:      bcr 0, %r0
 
 ; 28
-; CHECK:      brcl 0, [[LAB:.Ltmp[0-9]+]]
-; CHECK-NEXT: [[LAB]]:
-; CHECK:      brcl 0, [[LAB:.Ltmp[0-9]+]]
-; CHECK-NEXT: [[LAB]]:
-; CHECK:      brcl 0, [[LAB:.Ltmp[0-9]+]]
-; CHECK-NEXT: [[LAB]]:
-; CHECK:      brcl 0, [[LAB:.Ltmp[0-9]+]]
-; CHECK-NEXT: [[LAB]]:
+; CHECK:      .Ltmp
+; CHECK-NEXT: [[LAB:.Ltmp[0-9]+]]:
+; CHECK-NEXT: brcl 0, [[LAB]]
+; CHECK:      [[LAB:.Ltmp[0-9]+]]:
+; CHECK-NEXT: brcl 0, [[LAB]]
+; CHECK:      [[LAB:.Ltmp[0-9]+]]:
+; CHECK-NEXT: brcl 0, [[LAB]]
+; CHECK:      [[LAB:.Ltmp[0-9]+]]:
+; CHECK-NEXT: brcl 0, [[LAB]]
 ; CHECK:      bc 0, 0
 
 ; 30
-; CHECK:      brcl 0, [[LAB:.Ltmp[0-9]+]]
-; CHECK-NEXT: [[LAB]]:
-; CHECK:      brcl 0, [[LAB:.Ltmp[0-9]+]]
-; CHECK-NEXT: [[LAB]]:
-; CHECK:      brcl 0, [[LAB:.Ltmp[0-9]+]]
-; CHECK-NEXT: [[LAB]]:
-; CHECK:      brcl 0, [[LAB:.Ltmp[0-9]+]]
-; CHECK-NEXT: [[LAB]]:
-; CHECK:      brcl 0, [[LAB:.Ltmp[0-9]+]]
-; CHECK-NEXT: [[LAB]]:
+; CHECK:      .Ltmp
+; CHECK-NEXT: [[LAB:.Ltmp[0-9]+]]:
+; CHECK-NEXT: brcl 0, [[LAB]]
+; CHECK:      [[LAB:.Ltmp[0-9]+]]:
+; CHECK-NEXT: brcl 0, [[LAB]]
+; CHECK:      [[LAB:.Ltmp[0-9]+]]:
+; CHECK-NEXT: brcl 0, [[LAB]]
+; CHECK:      [[LAB:.Ltmp[0-9]+]]:
+; CHECK-NEXT: brcl 0, [[LAB]]
+; CHECK:      [[LAB:.Ltmp[0-9]+]]:
+; CHECK-NEXT: brcl 0, [[LAB]]
 
   tail call void (i64, i32, ...) @llvm.experimental.stackmap(i64  0, i32  0)
   tail call void (i64, i32, ...) @llvm.experimental.stackmap(i64  2, i32  2)
diff --git a/llvm/test/CodeGen/SystemZ/store_nonbytesized_vecs.ll b/llvm/test/CodeGen/SystemZ/store_nonbytesized_vecs.ll
index 839ab78b0..45434ed27 100644
--- a/llvm/test/CodeGen/SystemZ/store_nonbytesized_vecs.ll
+++ b/llvm/test/CodeGen/SystemZ/store_nonbytesized_vecs.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=s390x-linux-gnu -mcpu=z13 < %s  | FileCheck %s
 
 ; Store a <4 x i31> vector.
@@ -75,38 +76,38 @@ define void @fun2(<8 x i32> %src, <8 x i31>* %p)
 ; CHECK-NEXT:    stmg %r14, %r15, 112(%r15)
 ; CHECK-NEXT:    .cfi_offset %r14, -48
 ; CHECK-NEXT:    .cfi_offset %r15, -40
-; CHECK-DAG:     vlgvf [[REG11:%r[0-9]+]], %v26, 1
-; CHECK-DAG:     vlgvf [[REG12:%r[0-9]+]], %v26, 2
-; CHECK-DAG:     risbgn [[REG13:%r[0-9]+]], [[REG11]], 0, 129, 62
-; CHECK-DAG:     rosbg [[REG13]], [[REG12]], 2, 32, 31
-; CHECK-DAG:     vlgvf %r0, %v26, 3
-; CHECK-DAG:     rosbg [[REG13]], %r0, 33, 63, 0
-; CHECK-DAG:     stc %r0, 30(%r2)
-; CHECK-DAG:     srlk %r1, %r0, 8
-; CHECK-DAG:     vlgvf [[REG0:%r[0-9]+]], %v24, 1
-; CHECK-DAG:     vlgvf [[REG1:%r[0-9]+]], %v24, 0
-; CHECK-DAG:     sth %r1, 28(%r2)
-; CHECK-DAG:     vlgvf [[REG2:%r[0-9]+]], %v24, 2
-; CHECK-DAG:     risbgn [[REG3:%r[0-9]+]], [[REG0]], 0, 133, 58
-; CHECK-DAG:     rosbg [[REG3]], [[REG2]], 6, 36, 27
-; CHECK-DAG:     sllg [[REG4:%r[0-9]+]], [[REG1]], 25
-; CHECK-DAG:     rosbg [[REG4]], [[REG0]], 39, 63, 58
-; CHECK-DAG:     vlgvf [[REG5:%r[0-9]+]], %v24, 3
-; CHECK-DAG:     rosbg [[REG3]], [[REG5]], 37, 63, 60
-; CHECK-DAG:     sllg [[REG6:%r[0-9]+]], [[REG4]], 8
-; CHECK-DAG:     rosbg [[REG6]], [[REG3]], 56, 63, 8
-; CHECK-DAG:     stg [[REG6]], 0(%r2)
-; CHECK-DAG:     srlg [[REG7:%r[0-9]+]], [[REG13]], 24
-; CHECK-DAG:     st [[REG7]], 24(%r2)
-; CHECK-DAG:     vlgvf [[REG8:%r[0-9]+]], %v26, 0
-; CHECK-DAG:     risbgn [[REG10:%r[0-9]+]], [[REG5]], 0, 131, 60
-; CHECK-DAG:     rosbg [[REG10]], [[REG8]], 4, 34, 29
-; CHECK-DAG:     sllg [[REG9:%r[0-9]+]], [[REG3]], 8
-; CHECK-DAG:     rosbg [[REG10]], [[REG11]], 35, 63, 62
-; CHECK-DAG:     rosbg [[REG9]], [[REG10]], 56, 63, 8
-; CHECK-DAG:     stg [[REG9]], 8(%r2)
-; CHECK-DAG:     sllg %r0, [[REG10]], 8
-; CHECK-DAG:     rosbg %r0, [[REG13]], 56, 63, 8
+; CHECK-NEXT:    vlgvf %r0, %v26, 3
+; CHECK-NEXT:    vlgvf %r4, %v24, 1
+; CHECK-NEXT:    vlgvf %r3, %v24, 2
+; CHECK-NEXT:    srlk %r1, %r0, 8
+; CHECK-NEXT:    vlgvf %r5, %v24, 0
+; CHECK-NEXT:    sth %r1, 28(%r2)
+; CHECK-NEXT:    risbgn %r1, %r4, 0, 133, 58
+; CHECK-NEXT:    sllg %r5, %r5, 25
+; CHECK-NEXT:    stc %r0, 30(%r2)
+; CHECK-NEXT:    rosbg %r1, %r3, 6, 36, 27
+; CHECK-NEXT:    vlgvf %r3, %v24, 3
+; CHECK-NEXT:    rosbg %r5, %r4, 39, 63, 58
+; CHECK-NEXT:    sllg %r4, %r5, 8
+; CHECK-NEXT:    rosbg %r1, %r3, 37, 63, 60
+; CHECK-NEXT:    vlgvf %r5, %v26, 1
+; CHECK-NEXT:    rosbg %r4, %r1, 56, 63, 8
+; CHECK-NEXT:    stg %r4, 0(%r2)
+; CHECK-NEXT:    vlgvf %r4, %v26, 2
+; CHECK-NEXT:    risbgn %r14, %r5, 0, 129, 62
+; CHECK-NEXT:    risbgn %r3, %r3, 0, 131, 60
+; CHECK-NEXT:    rosbg %r14, %r4, 2, 32, 31
+; CHECK-NEXT:    rosbg %r14, %r0, 33, 63, 0
+; CHECK-NEXT:    srlg %r0, %r14, 24
+; CHECK-NEXT:    st %r0, 24(%r2)
+; CHECK-NEXT:    vlgvf %r0, %v26, 0
+; CHECK-NEXT:    rosbg %r3, %r0, 4, 34, 29
+; CHECK-NEXT:    sllg %r0, %r1, 8
+; CHECK-NEXT:    rosbg %r3, %r5, 35, 63, 62
+; CHECK-NEXT:    rosbg %r0, %r3, 56, 63, 8
+; CHECK-NEXT:    stg %r0, 8(%r2)
+; CHECK-NEXT:    sllg %r0, %r3, 8
+; CHECK-NEXT:    rosbg %r0, %r14, 56, 63, 8
 ; CHECK-NEXT:    stg %r0, 16(%r2)
 ; CHECK-NEXT:    lmg %r14, %r15, 112(%r15)
 ; CHECK-NEXT:    br %r14
@@ -120,15 +121,15 @@ define void @fun2(<8 x i32> %src, <8 x i31>* %p)
 define void @fun3(<3 x i31>* %src, <3 x i31>* %p)
 ; CHECK-LABEL: fun3:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    llgf %r1, 0(%r2)
 ; CHECK-NEXT:    llgf %r0, 3(%r2)
-; CHECK-NEXT:    sllg %r4, %r1, 62
+; CHECK-NEXT:    llgf %r1, 6(%r2)
+; CHECK-NEXT:    llgf %r2, 0(%r2)
+; CHECK-NEXT:    rosbg %r1, %r0, 0, 32, 31
+; CHECK-NEXT:    sllg %r4, %r2, 62
 ; CHECK-NEXT:    rosbg %r4, %r0, 0, 32, 31
-; CHECK-NEXT:    llgf %r0, 6(%r2)
-; CHECK-NEXT:    ogr %r0, %r4
-; CHECK-NEXT:    st %r0, 8(%r3)
 ; CHECK-NEXT:    srlg %r0, %r4, 32
-; CHECK-NEXT:    sllg %r1, %r1, 30
+; CHECK-NEXT:    st %r1, 8(%r3)
+; CHECK-NEXT:    sllg %r1, %r2, 30
 ; CHECK-NEXT:    lr %r1, %r0
 ; CHECK-NEXT:    nihh %r1, 8191
 ; CHECK-NEXT:    stg %r1, 0(%r3)
diff --git a/llvm/test/CodeGen/SystemZ/subregliveness-01.ll b/llvm/test/CodeGen/SystemZ/subregliveness-01.ll
index 5813fd309..15bb01493 100644
--- a/llvm/test/CodeGen/SystemZ/subregliveness-01.ll
+++ b/llvm/test/CodeGen/SystemZ/subregliveness-01.ll
@@ -32,7 +32,7 @@ bb4:                                              ; preds = %bb4, %bb
   br label %bb4
 }
 
-attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="z13" "target-features"="+transactional-execution,+vector" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="z13" "target-features"="+transactional-execution,+vector" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.ident = !{!0}
 
diff --git a/llvm/test/CodeGen/SystemZ/subregliveness-02.ll b/llvm/test/CodeGen/SystemZ/subregliveness-02.ll
index e0e1c197e..778b3b703 100644
--- a/llvm/test/CodeGen/SystemZ/subregliveness-02.ll
+++ b/llvm/test/CodeGen/SystemZ/subregliveness-02.ll
@@ -19,4 +19,4 @@ bb:
   unreachable
 }
 
-attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="z13" "target-features"="+transactional-execution,+vector" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="z13" "target-features"="+transactional-execution,+vector" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/SystemZ/subregliveness-03.ll b/llvm/test/CodeGen/SystemZ/subregliveness-03.ll
index a55e33089..5987bc398 100644
--- a/llvm/test/CodeGen/SystemZ/subregliveness-03.ll
+++ b/llvm/test/CodeGen/SystemZ/subregliveness-03.ll
@@ -46,6 +46,6 @@ bb15:                                             ; preds = %bb10, %bb9
 
 declare i8* @Perl_uvuni_to_utf8(i8*, i64) #0
 
-attributes #0 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="z13" "target-features"="+transactional-execution,+vector" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="z13" "target-features"="+transactional-execution,+vector" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="z13" "target-features"="+transactional-execution,+vector" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="z13" "target-features"="+transactional-execution,+vector" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind }
diff --git a/llvm/test/CodeGen/SystemZ/subregliveness-04.ll b/llvm/test/CodeGen/SystemZ/subregliveness-04.ll
index 11ecc9bd9..a7011885e 100644
--- a/llvm/test/CodeGen/SystemZ/subregliveness-04.ll
+++ b/llvm/test/CodeGen/SystemZ/subregliveness-04.ll
@@ -38,4 +38,4 @@ bb15:                                             ; preds = %bb14, %bb5
   br label %bb3
 }
 
-attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="z13" "target-features"="+transactional-execution,+vector" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="z13" "target-features"="+transactional-execution,+vector" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/SystemZ/subregliveness-05.ll b/llvm/test/CodeGen/SystemZ/subregliveness-05.ll
index 9b470cda1..a14385242 100644
--- a/llvm/test/CodeGen/SystemZ/subregliveness-05.ll
+++ b/llvm/test/CodeGen/SystemZ/subregliveness-05.ll
@@ -45,4 +45,4 @@ bb20:                                             ; preds = %bb12
   unreachable
 }
 
-attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="z10" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="z10" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/SystemZ/tail-call-mem-intrinsics.ll b/llvm/test/CodeGen/SystemZ/tail-call-mem-intrinsics.ll
index 0290d425e..463317583 100644
--- a/llvm/test/CodeGen/SystemZ/tail-call-mem-intrinsics.ll
+++ b/llvm/test/CodeGen/SystemZ/tail-call-mem-intrinsics.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=systemz < %s | FileCheck %s
+; RUN: llc -mtriple=s390x-linux-gnu < %s | FileCheck %s
 
 ; CHECK-LABEL: tail_memcpy:
 ; CHECK: jg memcpy
diff --git a/llvm/test/CodeGen/SystemZ/vec-abs-05.ll b/llvm/test/CodeGen/SystemZ/vec-abs-05.ll
index 63210f87b..800badd10 100644
--- a/llvm/test/CodeGen/SystemZ/vec-abs-05.ll
+++ b/llvm/test/CodeGen/SystemZ/vec-abs-05.ll
@@ -20,7 +20,7 @@ define <2 x double> @f2(<2 x double> %val) {
 ; CHECK: vflndb %v24, %v24
 ; CHECK: br %r14
   %abs = call <2 x double> @llvm.fabs.v2f64(<2 x double> %val)
-  %ret = fsub <2 x double> <double -0.0, double -0.0>, %abs
+  %ret = fneg <2 x double> %abs
   ret <2 x double> %ret
 }
 
@@ -41,6 +41,6 @@ define double @f4(<2 x double> %val) {
 ; CHECK: br %r14
   %scalar = extractelement <2 x double> %val, i32 0
   %abs = call double @llvm.fabs.f64(double %scalar)
-  %ret = fsub double -0.0, %abs
+  %ret = fneg double %abs
   ret double %ret
 }
diff --git a/llvm/test/CodeGen/SystemZ/vec-abs-06.ll b/llvm/test/CodeGen/SystemZ/vec-abs-06.ll
index 8eee1d9d2..79c93cc09 100644
--- a/llvm/test/CodeGen/SystemZ/vec-abs-06.ll
+++ b/llvm/test/CodeGen/SystemZ/vec-abs-06.ll
@@ -20,8 +20,7 @@ define <4 x float> @f2(<4 x float> %val) {
 ; CHECK: vflnsb %v24, %v24
 ; CHECK: br %r14
   %abs = call <4 x float> @llvm.fabs.v4f32(<4 x float> %val)
-  %ret = fsub <4 x float> <float -0.0, float -0.0,
-                           float -0.0, float -0.0>, %abs
+  %ret = fneg <4 x float> %abs
   ret <4 x float> %ret
 }
 
@@ -42,6 +41,6 @@ define float @f4(<4 x float> %val) {
 ; CHECK: br %r14
   %scalar = extractelement <4 x float> %val, i32 0
   %abs = call float @llvm.fabs.f32(float %scalar)
-  %ret = fsub float -0.0, %abs
+  %ret = fneg float %abs
   ret float %ret
 }
diff --git a/llvm/test/CodeGen/SystemZ/vec-bswap-01.ll b/llvm/test/CodeGen/SystemZ/vec-bswap-01.ll
index 813210895..4d5e57403 100644
--- a/llvm/test/CodeGen/SystemZ/vec-bswap-01.ll
+++ b/llvm/test/CodeGen/SystemZ/vec-bswap-01.ll
@@ -1,6 +1,6 @@
 ; Test loads of byte-swapped vector elements.
 ;
-; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=arch13 | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z15 | FileCheck %s
 
 declare <8 x i16> @llvm.bswap.v8i16(<8 x i16>)
 declare <4 x i32> @llvm.bswap.v4i32(<4 x i32>)
diff --git a/llvm/test/CodeGen/SystemZ/vec-bswap-02.ll b/llvm/test/CodeGen/SystemZ/vec-bswap-02.ll
index ff7facc09..b7e8a1c17 100644
--- a/llvm/test/CodeGen/SystemZ/vec-bswap-02.ll
+++ b/llvm/test/CodeGen/SystemZ/vec-bswap-02.ll
@@ -1,6 +1,6 @@
 ; Test stores of byte-swapped vector elements.
 ;
-; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=arch13 | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z15 | FileCheck %s
 
 declare <8 x i16> @llvm.bswap.v8i16(<8 x i16>)
 declare <4 x i32> @llvm.bswap.v4i32(<4 x i32>)
diff --git a/llvm/test/CodeGen/SystemZ/vec-bswap-03.ll b/llvm/test/CodeGen/SystemZ/vec-bswap-03.ll
index 9102c739b..b812308fc 100644
--- a/llvm/test/CodeGen/SystemZ/vec-bswap-03.ll
+++ b/llvm/test/CodeGen/SystemZ/vec-bswap-03.ll
@@ -1,6 +1,6 @@
 ; Test vector insertion of byte-swapped memory values.
 ;
-; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=arch13 | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z15 | FileCheck %s
 
 declare i16 @llvm.bswap.i16(i16)
 declare i32 @llvm.bswap.i32(i32)
diff --git a/llvm/test/CodeGen/SystemZ/vec-bswap-04.ll b/llvm/test/CodeGen/SystemZ/vec-bswap-04.ll
index 43787a48b..0ab521dbb 100644
--- a/llvm/test/CodeGen/SystemZ/vec-bswap-04.ll
+++ b/llvm/test/CodeGen/SystemZ/vec-bswap-04.ll
@@ -1,6 +1,6 @@
 ; Test vector extraction of byte-swapped value to memory.
 ;
-; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=arch13 | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z15 | FileCheck %s
 
 declare i16 @llvm.bswap.i16(i16)
 declare i32 @llvm.bswap.i32(i32)
diff --git a/llvm/test/CodeGen/SystemZ/vec-bswap-05.ll b/llvm/test/CodeGen/SystemZ/vec-bswap-05.ll
index 038a3f4f4..4718aadbd 100644
--- a/llvm/test/CodeGen/SystemZ/vec-bswap-05.ll
+++ b/llvm/test/CodeGen/SystemZ/vec-bswap-05.ll
@@ -1,6 +1,6 @@
 ; Test vector insertions of byte-swapped memory values into 0.
 ;
-; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=arch13 | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z15 | FileCheck %s
 
 declare i16 @llvm.bswap.i16(i16)
 declare i32 @llvm.bswap.i32(i32)
diff --git a/llvm/test/CodeGen/SystemZ/vec-bswap-06.ll b/llvm/test/CodeGen/SystemZ/vec-bswap-06.ll
index 0c78633ee..a67b5756c 100644
--- a/llvm/test/CodeGen/SystemZ/vec-bswap-06.ll
+++ b/llvm/test/CodeGen/SystemZ/vec-bswap-06.ll
@@ -1,6 +1,6 @@
 ; Test insertions of byte-swapped memory values into a nonzero index of an undef.
 ;
-; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=arch13 | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z15 | FileCheck %s
 
 declare i16 @llvm.bswap.i16(i16)
 declare i32 @llvm.bswap.i32(i32)
diff --git a/llvm/test/CodeGen/SystemZ/vec-bswap-07.ll b/llvm/test/CodeGen/SystemZ/vec-bswap-07.ll
index 0190184c1..d3f38a0a6 100644
--- a/llvm/test/CodeGen/SystemZ/vec-bswap-07.ll
+++ b/llvm/test/CodeGen/SystemZ/vec-bswap-07.ll
@@ -1,6 +1,6 @@
 ; Test replications of a byte-swapped scalar memory value.
 ;
-; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=arch13 | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z15 | FileCheck %s
 
 declare i16 @llvm.bswap.i16(i16)
 declare i32 @llvm.bswap.i32(i32)
diff --git a/llvm/test/CodeGen/SystemZ/vec-conv-03.ll b/llvm/test/CodeGen/SystemZ/vec-conv-03.ll
index 839887682..df1e3a4e0 100644
--- a/llvm/test/CodeGen/SystemZ/vec-conv-03.ll
+++ b/llvm/test/CodeGen/SystemZ/vec-conv-03.ll
@@ -1,6 +1,6 @@
-; Test conversions between integer and float elements on arch13.
+; Test conversions between integer and float elements on z15.
 ;
-; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=arch13 | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z15 | FileCheck %s
 
 ; Test conversion of f32s to signed i32s.
 define <4 x i32> @f1(<4 x float> %floats) {
diff --git a/llvm/test/CodeGen/SystemZ/vec-eswap-01.ll b/llvm/test/CodeGen/SystemZ/vec-eswap-01.ll
index 89b653924..0e93f609e 100644
--- a/llvm/test/CodeGen/SystemZ/vec-eswap-01.ll
+++ b/llvm/test/CodeGen/SystemZ/vec-eswap-01.ll
@@ -1,6 +1,6 @@
 ; Test loads of byte-swapped vector elements.
 ;
-; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=arch13 | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z15 | FileCheck %s
 
 ; Test v16i8 loads.
 define <16 x i8> @f1(<16 x i8> *%ptr) {
diff --git a/llvm/test/CodeGen/SystemZ/vec-eswap-02.ll b/llvm/test/CodeGen/SystemZ/vec-eswap-02.ll
index 1eab5a06f..07e072582 100644
--- a/llvm/test/CodeGen/SystemZ/vec-eswap-02.ll
+++ b/llvm/test/CodeGen/SystemZ/vec-eswap-02.ll
@@ -1,6 +1,6 @@
 ; Test stores of element-swapped vector elements.
 ;
-; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=arch13 | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z15 | FileCheck %s
 
 ; Test v16i8 stores.
 define void @f1(<16 x i8> %val, <16 x i8> *%ptr) {
diff --git a/llvm/test/CodeGen/SystemZ/vec-intrinsics-03.ll b/llvm/test/CodeGen/SystemZ/vec-intrinsics-03.ll
index d192ad894..fa144cc93 100644
--- a/llvm/test/CodeGen/SystemZ/vec-intrinsics-03.ll
+++ b/llvm/test/CodeGen/SystemZ/vec-intrinsics-03.ll
@@ -1,6 +1,6 @@
-; Test vector intrinsics added with arch13.
+; Test vector intrinsics added with z15.
 ;
-; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=arch13 | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z15 | FileCheck %s
 
 declare <16 x i8> @llvm.s390.vsld(<16 x i8>, <16 x i8>, i32)
 declare <16 x i8> @llvm.s390.vsrd(<16 x i8>, <16 x i8>, i32)
diff --git a/llvm/test/CodeGen/SystemZ/vec-max-05.ll b/llvm/test/CodeGen/SystemZ/vec-max-05.ll
index 1fe0db350..8ad5e0e9b 100644
--- a/llvm/test/CodeGen/SystemZ/vec-max-05.ll
+++ b/llvm/test/CodeGen/SystemZ/vec-max-05.ll
@@ -5,13 +5,18 @@
 declare double @fmax(double, double)
 declare double @llvm.maxnum.f64(double, double)
 declare <2 x double> @llvm.maxnum.v2f64(<2 x double>, <2 x double>)
+declare double @llvm.maximum.f64(double, double)
+declare <2 x double> @llvm.maximum.v2f64(<2 x double>, <2 x double>)
 
 declare float @fmaxf(float, float)
 declare float @llvm.maxnum.f32(float, float)
 declare <4 x float> @llvm.maxnum.v4f32(<4 x float>, <4 x float>)
+declare float @llvm.maximum.f32(float, float)
+declare <4 x float> @llvm.maximum.v4f32(<4 x float>, <4 x float>)
 
 declare fp128 @fmaxl(fp128, fp128)
 declare fp128 @llvm.maxnum.f128(fp128, fp128)
+declare fp128 @llvm.maximum.f128(fp128, fp128)
 
 ; Test the fmax library function.
 define double @f1(double %dummy, double %val1, double %val2) {
@@ -31,9 +36,18 @@ define double @f2(double %dummy, double %val1, double %val2) {
   ret double %ret
 }
 
-; Test a f64 constant compare/select resulting in maxnum.
-define double @f3(double %dummy, double %val) {
+; Test the f64 maximum intrinsic.
+define double @f3(double %dummy, double %val1, double %val2) {
 ; CHECK-LABEL: f3:
+; CHECK: wfmaxdb %f0, %f2, %f4, 1
+; CHECK: br %r14
+  %ret = call double @llvm.maximum.f64(double %val1, double %val2)
+  ret double %ret
+}
+
+; Test a f64 constant compare/select resulting in maxnum.
+define double @f4(double %dummy, double %val) {
+; CHECK-LABEL: f4:
 ; CHECK: lzdr [[REG:%f[0-9]+]]
 ; CHECK: wfmaxdb %f0, %f2, [[REG]], 4
 ; CHECK: br %r14
@@ -43,8 +57,8 @@ define double @f3(double %dummy, double %val) {
 }
 
 ; Test a f64 constant compare/select resulting in maximum.
-define double @f4(double %dummy, double %val) {
-; CHECK-LABEL: f4:
+define double @f5(double %dummy, double %val) {
+; CHECK-LABEL: f5:
 ; CHECK: lzdr [[REG:%f[0-9]+]]
 ; CHECK: wfmaxdb %f0, %f2, [[REG]], 1
 ; CHECK: br %r14
@@ -54,15 +68,25 @@ define double @f4(double %dummy, double %val) {
 }
 
 ; Test the v2f64 maxnum intrinsic.
-define <2 x double> @f5(<2 x double> %dummy, <2 x double> %val1,
+define <2 x double> @f6(<2 x double> %dummy, <2 x double> %val1,
                         <2 x double> %val2) {
-; CHECK-LABEL: f5:
+; CHECK-LABEL: f6:
 ; CHECK: vfmaxdb %v24, %v26, %v28, 4
 ; CHECK: br %r14
   %ret = call <2 x double> @llvm.maxnum.v2f64(<2 x double> %val1, <2 x double> %val2)
   ret <2 x double> %ret
 }
 
+; Test the v2f64 maximum intrinsic.
+define <2 x double> @f7(<2 x double> %dummy, <2 x double> %val1,
+                        <2 x double> %val2) {
+; CHECK-LABEL: f7:
+; CHECK: vfmaxdb %v24, %v26, %v28, 1
+; CHECK: br %r14
+  %ret = call <2 x double> @llvm.maximum.v2f64(<2 x double> %val1, <2 x double> %val2)
+  ret <2 x double> %ret
+}
+
 ; Test the fmaxf library function.
 define float @f11(float %dummy, float %val1, float %val2) {
 ; CHECK-LABEL: f11:
@@ -81,9 +105,18 @@ define float @f12(float %dummy, float %val1, float %val2) {
   ret float %ret
 }
 
-; Test a f32 constant compare/select resulting in maxnum.
-define float @f13(float %dummy, float %val) {
+; Test the f32 maximum intrinsic.
+define float @f13(float %dummy, float %val1, float %val2) {
 ; CHECK-LABEL: f13:
+; CHECK: wfmaxsb %f0, %f2, %f4, 1
+; CHECK: br %r14
+  %ret = call float @llvm.maximum.f32(float %val1, float %val2)
+  ret float %ret
+}
+
+; Test a f32 constant compare/select resulting in maxnum.
+define float @f14(float %dummy, float %val) {
+; CHECK-LABEL: f14:
 ; CHECK: lzer [[REG:%f[0-9]+]]
 ; CHECK: wfmaxsb %f0, %f2, [[REG]], 4
 ; CHECK: br %r14
@@ -93,8 +126,8 @@ define float @f13(float %dummy, float %val) {
 }
 
 ; Test a f32 constant compare/select resulting in maximum.
-define float @f14(float %dummy, float %val) {
-; CHECK-LABEL: f14:
+define float @f15(float %dummy, float %val) {
+; CHECK-LABEL: f15:
 ; CHECK: lzer [[REG:%f[0-9]+]]
 ; CHECK: wfmaxsb %f0, %f2, [[REG]], 1
 ; CHECK: br %r14
@@ -104,15 +137,25 @@ define float @f14(float %dummy, float %val) {
 }
 
 ; Test the v4f32 maxnum intrinsic.
-define <4 x float> @f15(<4 x float> %dummy, <4 x float> %val1,
+define <4 x float> @f16(<4 x float> %dummy, <4 x float> %val1,
                         <4 x float> %val2) {
-; CHECK-LABEL: f15:
+; CHECK-LABEL: f16:
 ; CHECK: vfmaxsb %v24, %v26, %v28, 4
 ; CHECK: br %r14
   %ret = call <4 x float> @llvm.maxnum.v4f32(<4 x float> %val1, <4 x float> %val2)
   ret <4 x float> %ret
 }
 
+; Test the v4f32 maximum intrinsic.
+define <4 x float> @f17(<4 x float> %dummy, <4 x float> %val1,
+                        <4 x float> %val2) {
+; CHECK-LABEL: f17:
+; CHECK: vfmaxsb %v24, %v26, %v28, 1
+; CHECK: br %r14
+  %ret = call <4 x float> @llvm.maximum.v4f32(<4 x float> %val1, <4 x float> %val2)
+  ret <4 x float> %ret
+}
+
 ; Test the fmaxl library function.
 define void @f21(fp128 *%ptr1, fp128 *%ptr2, fp128 *%dst) {
 ; CHECK-LABEL: f21:
@@ -143,10 +186,25 @@ define void @f22(fp128 *%ptr1, fp128 *%ptr2, fp128 *%dst) {
   ret void
 }
 
-; Test a f128 constant compare/select resulting in maxnum.
-define void @f23(fp128 *%ptr, fp128 *%dst) {
+; Test the f128 maximum intrinsic.
+define void @f23(fp128 *%ptr1, fp128 *%ptr2, fp128 *%dst) {
 ; CHECK-LABEL: f23:
 ; CHECK-DAG: vl [[REG1:%v[0-9]+]], 0(%r2)
+; CHECK-DAG: vl [[REG2:%v[0-9]+]], 0(%r3)
+; CHECK: wfmaxxb [[RES:%v[0-9]+]], [[REG1]], [[REG2]], 1
+; CHECK: vst [[RES]], 0(%r4)
+; CHECK: br %r14
+  %val1 = load fp128, fp128* %ptr1
+  %val2 = load fp128, fp128* %ptr2
+  %res = call fp128 @llvm.maximum.f128(fp128 %val1, fp128 %val2)
+  store fp128 %res, fp128* %dst
+  ret void
+}
+
+; Test a f128 constant compare/select resulting in maxnum.
+define void @f24(fp128 *%ptr, fp128 *%dst) {
+; CHECK-LABEL: f24:
+; CHECK-DAG: vl [[REG1:%v[0-9]+]], 0(%r2)
 ; CHECK-DAG: vzero [[REG2:%v[0-9]+]]
 ; CHECK: wfmaxxb [[RES:%v[0-9]+]], [[REG1]], [[REG2]], 4
 ; CHECK: vst [[RES]], 0(%r3)
@@ -159,8 +217,8 @@ define void @f23(fp128 *%ptr, fp128 *%dst) {
 }
 
 ; Test a f128 constant compare/select resulting in maximum.
-define void @f24(fp128 *%ptr, fp128 *%dst) {
-; CHECK-LABEL: f24:
+define void @f25(fp128 *%ptr, fp128 *%dst) {
+; CHECK-LABEL: f25:
 ; CHECK-DAG: vl [[REG1:%v[0-9]+]], 0(%r2)
 ; CHECK-DAG: vzero [[REG2:%v[0-9]+]]
 ; CHECK: wfmaxxb [[RES:%v[0-9]+]], [[REG1]], [[REG2]], 1
diff --git a/llvm/test/CodeGen/SystemZ/vec-min-05.ll b/llvm/test/CodeGen/SystemZ/vec-min-05.ll
index 6417e5ed7..9e1e81602 100644
--- a/llvm/test/CodeGen/SystemZ/vec-min-05.ll
+++ b/llvm/test/CodeGen/SystemZ/vec-min-05.ll
@@ -5,13 +5,18 @@
 declare double @fmin(double, double)
 declare double @llvm.minnum.f64(double, double)
 declare <2 x double> @llvm.minnum.v2f64(<2 x double>, <2 x double>)
+declare double @llvm.minimum.f64(double, double)
+declare <2 x double> @llvm.minimum.v2f64(<2 x double>, <2 x double>)
 
 declare float @fminf(float, float)
 declare float @llvm.minnum.f32(float, float)
 declare <4 x float> @llvm.minnum.v4f32(<4 x float>, <4 x float>)
+declare float @llvm.minimum.f32(float, float)
+declare <4 x float> @llvm.minimum.v4f32(<4 x float>, <4 x float>)
 
 declare fp128 @fminl(fp128, fp128)
 declare fp128 @llvm.minnum.f128(fp128, fp128)
+declare fp128 @llvm.minimum.f128(fp128, fp128)
 
 ; Test the fmin library function.
 define double @f1(double %dummy, double %val1, double %val2) {
@@ -31,9 +36,18 @@ define double @f2(double %dummy, double %val1, double %val2) {
   ret double %ret
 }
 
-; Test a f64 constant compare/select resulting in minnum.
-define double @f3(double %dummy, double %val) {
+; Test the f64 minimum intrinsic.
+define double @f3(double %dummy, double %val1, double %val2) {
 ; CHECK-LABEL: f3:
+; CHECK: wfmindb %f0, %f2, %f4, 1
+; CHECK: br %r14
+  %ret = call double @llvm.minimum.f64(double %val1, double %val2)
+  ret double %ret
+}
+
+; Test a f64 constant compare/select resulting in minnum.
+define double @f4(double %dummy, double %val) {
+; CHECK-LABEL: f4:
 ; CHECK: lzdr [[REG:%f[0-9]+]]
 ; CHECK: wfmindb %f0, %f2, [[REG]], 4
 ; CHECK: br %r14
@@ -43,8 +57,8 @@ define double @f3(double %dummy, double %val) {
 }
 
 ; Test a f64 constant compare/select resulting in minimum.
-define double @f4(double %dummy, double %val) {
-; CHECK-LABEL: f4:
+define double @f5(double %dummy, double %val) {
+; CHECK-LABEL: f5:
 ; CHECK: lzdr [[REG:%f[0-9]+]]
 ; CHECK: wfmindb %f0, %f2, [[REG]], 1
 ; CHECK: br %r14
@@ -54,15 +68,25 @@ define double @f4(double %dummy, double %val) {
 }
 
 ; Test the v2f64 minnum intrinsic.
-define <2 x double> @f5(<2 x double> %dummy, <2 x double> %val1,
+define <2 x double> @f6(<2 x double> %dummy, <2 x double> %val1,
                         <2 x double> %val2) {
-; CHECK-LABEL: f5:
+; CHECK-LABEL: f6:
 ; CHECK: vfmindb %v24, %v26, %v28, 4
 ; CHECK: br %r14
   %ret = call <2 x double> @llvm.minnum.v2f64(<2 x double> %val1, <2 x double> %val2)
   ret <2 x double> %ret
 }
 
+; Test the v2f64 minimum intrinsic.
+define <2 x double> @f7(<2 x double> %dummy, <2 x double> %val1,
+                        <2 x double> %val2) {
+; CHECK-LABEL: f7:
+; CHECK: vfmindb %v24, %v26, %v28, 1
+; CHECK: br %r14
+  %ret = call <2 x double> @llvm.minimum.v2f64(<2 x double> %val1, <2 x double> %val2)
+  ret <2 x double> %ret
+}
+
 ; Test the fminf library function.
 define float @f11(float %dummy, float %val1, float %val2) {
 ; CHECK-LABEL: f11:
@@ -81,9 +105,18 @@ define float @f12(float %dummy, float %val1, float %val2) {
   ret float %ret
 }
 
-; Test a f32 constant compare/select resulting in minnum.
-define float @f13(float %dummy, float %val) {
+; Test the f32 minimum intrinsic.
+define float @f13(float %dummy, float %val1, float %val2) {
 ; CHECK-LABEL: f13:
+; CHECK: wfminsb %f0, %f2, %f4, 1
+; CHECK: br %r14
+  %ret = call float @llvm.minimum.f32(float %val1, float %val2)
+  ret float %ret
+}
+
+; Test a f32 constant compare/select resulting in minnum.
+define float @f14(float %dummy, float %val) {
+; CHECK-LABEL: f14:
 ; CHECK: lzer [[REG:%f[0-9]+]]
 ; CHECK: wfminsb %f0, %f2, [[REG]], 4
 ; CHECK: br %r14
@@ -93,8 +126,8 @@ define float @f13(float %dummy, float %val) {
 }
 
 ; Test a f32 constant compare/select resulting in minimum.
-define float @f14(float %dummy, float %val) {
-; CHECK-LABEL: f14:
+define float @f15(float %dummy, float %val) {
+; CHECK-LABEL: f15:
 ; CHECK: lzer [[REG:%f[0-9]+]]
 ; CHECK: wfminsb %f0, %f2, [[REG]], 1
 ; CHECK: br %r14
@@ -104,15 +137,25 @@ define float @f14(float %dummy, float %val) {
 }
 
 ; Test the v4f32 minnum intrinsic.
-define <4 x float> @f15(<4 x float> %dummy, <4 x float> %val1,
+define <4 x float> @f16(<4 x float> %dummy, <4 x float> %val1,
                         <4 x float> %val2) {
-; CHECK-LABEL: f15:
+; CHECK-LABEL: f16:
 ; CHECK: vfminsb %v24, %v26, %v28, 4
 ; CHECK: br %r14
   %ret = call <4 x float> @llvm.minnum.v4f32(<4 x float> %val1, <4 x float> %val2)
   ret <4 x float> %ret
 }
 
+; Test the v4f32 minimum intrinsic.
+define <4 x float> @f17(<4 x float> %dummy, <4 x float> %val1,
+                        <4 x float> %val2) {
+; CHECK-LABEL: f17:
+; CHECK: vfminsb %v24, %v26, %v28, 1
+; CHECK: br %r14
+  %ret = call <4 x float> @llvm.minimum.v4f32(<4 x float> %val1, <4 x float> %val2)
+  ret <4 x float> %ret
+}
+
 ; Test the fminl library function.
 define void @f21(fp128 *%ptr1, fp128 *%ptr2, fp128 *%dst) {
 ; CHECK-LABEL: f21:
@@ -143,10 +186,25 @@ define void @f22(fp128 *%ptr1, fp128 *%ptr2, fp128 *%dst) {
   ret void
 }
 
-; Test a f128 constant compare/select resulting in minnum.
-define void @f23(fp128 *%ptr, fp128 *%dst) {
+; Test the f128 minimum intrinsic.
+define void @f23(fp128 *%ptr1, fp128 *%ptr2, fp128 *%dst) {
 ; CHECK-LABEL: f23:
 ; CHECK-DAG: vl [[REG1:%v[0-9]+]], 0(%r2)
+; CHECK-DAG: vl [[REG2:%v[0-9]+]], 0(%r3)
+; CHECK: wfminxb [[RES:%v[0-9]+]], [[REG1]], [[REG2]], 1
+; CHECK: vst [[RES]], 0(%r4)
+; CHECK: br %r14
+  %val1 = load fp128, fp128* %ptr1
+  %val2 = load fp128, fp128* %ptr2
+  %res = call fp128 @llvm.minimum.f128(fp128 %val1, fp128 %val2)
+  store fp128 %res, fp128* %dst
+  ret void
+}
+
+; Test a f128 constant compare/select resulting in minnum.
+define void @f24(fp128 *%ptr, fp128 *%dst) {
+; CHECK-LABEL: f24:
+; CHECK-DAG: vl [[REG1:%v[0-9]+]], 0(%r2)
 ; CHECK-DAG: vzero [[REG2:%v[0-9]+]]
 ; CHECK: wfminxb [[RES:%v[0-9]+]], [[REG1]], [[REG2]], 4
 ; CHECK: vst [[RES]], 0(%r3)
@@ -159,8 +217,8 @@ define void @f23(fp128 *%ptr, fp128 *%dst) {
 }
 
 ; Test a f128 constant compare/select resulting in minimum.
-define void @f24(fp128 *%ptr, fp128 *%dst) {
-; CHECK-LABEL: f24:
+define void @f25(fp128 *%ptr, fp128 *%dst) {
+; CHECK-LABEL: f25:
 ; CHECK-DAG: vl [[REG1:%v[0-9]+]], 0(%r2)
 ; CHECK-DAG: vzero [[REG2:%v[0-9]+]]
 ; CHECK: wfminxb [[RES:%v[0-9]+]], [[REG1]], [[REG2]], 1
diff --git a/llvm/test/CodeGen/SystemZ/vec-mul-02.ll b/llvm/test/CodeGen/SystemZ/vec-mul-02.ll
index 11a651e49..a4f6775d3 100644
--- a/llvm/test/CodeGen/SystemZ/vec-mul-02.ll
+++ b/llvm/test/CodeGen/SystemZ/vec-mul-02.ll
@@ -55,7 +55,7 @@ define <2 x double> @f5(<2 x double> %dummy, <2 x double> %val1,
 ; CHECK-LABEL: f5:
 ; CHECK: vfmsdb %v24, %v26, %v28, %v30
 ; CHECK: br %r14
-  %negval3 = fsub <2 x double> <double -0.0, double -0.0>, %val3
+  %negval3 = fneg <2 x double> %val3
   %ret = call <2 x double> @llvm.fma.v2f64 (<2 x double> %val1,
                                             <2 x double> %val2,
                                             <2 x double> %negval3)
diff --git a/llvm/test/CodeGen/SystemZ/vec-mul-04.ll b/llvm/test/CodeGen/SystemZ/vec-mul-04.ll
index d96f0b6a7..ad48d0446 100644
--- a/llvm/test/CodeGen/SystemZ/vec-mul-04.ll
+++ b/llvm/test/CodeGen/SystemZ/vec-mul-04.ll
@@ -22,8 +22,7 @@ define <4 x float> @f2(<4 x float> %dummy, <4 x float> %val1,
 ; CHECK-LABEL: f2:
 ; CHECK: vfmssb %v24, %v26, %v28, %v30
 ; CHECK: br %r14
-  %negval3 = fsub <4 x float> <float -0.0, float -0.0,
-                               float -0.0, float -0.0>, %val3
+  %negval3 = fneg <4 x float> %val3
   %ret = call <4 x float> @llvm.fma.v4f32 (<4 x float> %val1,
                                            <4 x float> %val2,
                                            <4 x float> %negval3)
diff --git a/llvm/test/CodeGen/SystemZ/vec-mul-05.ll b/llvm/test/CodeGen/SystemZ/vec-mul-05.ll
index 90a1f7a7e..5435239f2 100644
--- a/llvm/test/CodeGen/SystemZ/vec-mul-05.ll
+++ b/llvm/test/CodeGen/SystemZ/vec-mul-05.ll
@@ -14,7 +14,7 @@ define <2 x double> @f1(<2 x double> %dummy, <2 x double> %val1,
   %ret = call <2 x double> @llvm.fma.v2f64 (<2 x double> %val1,
                                             <2 x double> %val2,
                                             <2 x double> %val3)
-  %negret = fsub <2 x double> <double -0.0, double -0.0>, %ret
+  %negret = fneg <2 x double> %ret
   ret <2 x double> %negret
 }
 
@@ -24,11 +24,11 @@ define <2 x double> @f2(<2 x double> %dummy, <2 x double> %val1,
 ; CHECK-LABEL: f2:
 ; CHECK: vfnmsdb %v24, %v26, %v28, %v30
 ; CHECK: br %r14
-  %negval3 = fsub <2 x double> <double -0.0, double -0.0>, %val3
+  %negval3 = fneg <2 x double> %val3
   %ret = call <2 x double> @llvm.fma.v2f64 (<2 x double> %val1,
                                             <2 x double> %val2,
                                             <2 x double> %negval3)
-  %negret = fsub <2 x double> <double -0.0, double -0.0>, %ret
+  %negret = fneg <2 x double> %ret
   ret <2 x double> %negret
 }
 
@@ -41,8 +41,7 @@ define <4 x float> @f3(<4 x float> %dummy, <4 x float> %val1,
   %ret = call <4 x float> @llvm.fma.v4f32 (<4 x float> %val1,
                                            <4 x float> %val2,
                                            <4 x float> %val3)
-  %negret = fsub <4 x float> <float -0.0, float -0.0,
-                              float -0.0, float -0.0>, %ret
+  %negret = fneg <4 x float> %ret
   ret <4 x float> %negret
 }
 
@@ -52,12 +51,10 @@ define <4 x float> @f4(<4 x float> %dummy, <4 x float> %val1,
 ; CHECK-LABEL: f4:
 ; CHECK: vfnmssb %v24, %v26, %v28, %v30
 ; CHECK: br %r14
-  %negval3 = fsub <4 x float> <float -0.0, float -0.0,
-                               float -0.0, float -0.0>, %val3
+  %negval3 = fneg <4 x float> %val3
   %ret = call <4 x float> @llvm.fma.v4f32 (<4 x float> %val1,
                                            <4 x float> %val2,
                                            <4 x float> %negval3)
-  %negret = fsub <4 x float> <float -0.0, float -0.0,
-                               float -0.0, float -0.0>, %ret
+  %negret = fneg <4 x float> %ret
   ret <4 x float> %negret
 }
diff --git a/llvm/test/CodeGen/SystemZ/vec-neg-01.ll b/llvm/test/CodeGen/SystemZ/vec-neg-01.ll
index b1389ce4d..3c1f6e880 100644
--- a/llvm/test/CodeGen/SystemZ/vec-neg-01.ll
+++ b/llvm/test/CodeGen/SystemZ/vec-neg-01.ll
@@ -43,7 +43,7 @@ define <2 x double> @f5(<2 x double> %dummy, <2 x double> %val) {
 ; CHECK-LABEL: f5:
 ; CHECK: vflcdb %v24, %v26
 ; CHECK: br %r14
-  %ret = fsub <2 x double> <double -0.0, double -0.0>, %val
+  %ret = fneg <2 x double> %val
   ret <2 x double> %ret
 }
 
@@ -53,6 +53,6 @@ define double @f6(<2 x double> %val) {
 ; CHECK: wflcdb %f0, %v24
 ; CHECK: br %r14
   %scalar = extractelement <2 x double> %val, i32 0
-  %ret = fsub double -0.0, %scalar
+  %ret = fneg double %scalar
   ret double %ret
 }
diff --git a/llvm/test/CodeGen/SystemZ/vec-neg-02.ll b/llvm/test/CodeGen/SystemZ/vec-neg-02.ll
index 07ce03754..f9059dece 100644
--- a/llvm/test/CodeGen/SystemZ/vec-neg-02.ll
+++ b/llvm/test/CodeGen/SystemZ/vec-neg-02.ll
@@ -7,8 +7,7 @@ define <4 x float> @f1(<4 x float> %dummy, <4 x float> %val) {
 ; CHECK-LABEL: f1:
 ; CHECK: vflcsb %v24, %v26
 ; CHECK: br %r14
-  %ret = fsub <4 x float> <float -0.0, float -0.0,
-                           float -0.0, float -0.0>, %val
+  %ret = fneg <4 x float> %val
   ret <4 x float> %ret
 }
 
@@ -18,6 +17,6 @@ define float @f2(<4 x float> %val) {
 ; CHECK: wflcsb %f0, %v24
 ; CHECK: br %r14
   %scalar = extractelement <4 x float> %val, i32 0
-  %ret = fsub float -0.0, %scalar
+  %ret = fneg float %scalar
   ret float %ret
 }
diff --git a/llvm/test/CodeGen/SystemZ/vec-strict-add-01.ll b/llvm/test/CodeGen/SystemZ/vec-strict-add-01.ll
index d1270b9d2..1ac2e190d 100644
--- a/llvm/test/CodeGen/SystemZ/vec-strict-add-01.ll
+++ b/llvm/test/CodeGen/SystemZ/vec-strict-add-01.ll
@@ -7,19 +7,19 @@ declare <2 x double> @llvm.experimental.constrained.fadd.v2f64(<2 x double>, <2
 
 ; Test a v2f64 addition.
 define <2 x double> @f5(<2 x double> %dummy, <2 x double> %val1,
-                        <2 x double> %val2) {
+                        <2 x double> %val2) strictfp {
 ; CHECK-LABEL: f5:
 ; CHECK: vfadb %v24, %v26, %v28
 ; CHECK: br %r14
   %ret = call <2 x double> @llvm.experimental.constrained.fadd.v2f64(
                         <2 x double> %val1, <2 x double> %val2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") strictfp
   ret <2 x double> %ret
 }
 
 ; Test an f64 addition that uses vector registers.
-define double @f6(<2 x double> %val1, <2 x double> %val2) {
+define double @f6(<2 x double> %val1, <2 x double> %val2) strictfp {
 ; CHECK-LABEL: f6:
 ; CHECK: wfadb %f0, %v24, %v26
 ; CHECK: br %r14
@@ -28,6 +28,6 @@ define double @f6(<2 x double> %val1, <2 x double> %val2) {
   %ret = call double @llvm.experimental.constrained.fadd.f64(
                         double %scalar1, double %scalar2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") strictfp
   ret double %ret
 }
diff --git a/llvm/test/CodeGen/SystemZ/vec-strict-add-02.ll b/llvm/test/CodeGen/SystemZ/vec-strict-add-02.ll
index 4aee31aee..7cdd63831 100644
--- a/llvm/test/CodeGen/SystemZ/vec-strict-add-02.ll
+++ b/llvm/test/CodeGen/SystemZ/vec-strict-add-02.ll
@@ -7,19 +7,19 @@ declare <4 x float> @llvm.experimental.constrained.fadd.v4f32(<4 x float>, <4 x
 
 ; Test a v4f32 addition.
 define <4 x float> @f1(<4 x float> %dummy, <4 x float> %val1,
-                       <4 x float> %val2) {
+                       <4 x float> %val2) strictfp {
 ; CHECK-LABEL: f1:
 ; CHECK: vfasb %v24, %v26, %v28
 ; CHECK: br %r14
   %ret = call <4 x float> @llvm.experimental.constrained.fadd.v4f32(
                         <4 x float> %val1, <4 x float> %val2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") strictfp
   ret <4 x float> %ret
 }
 
 ; Test an f32 addition that uses vector registers.
-define float @f2(<4 x float> %val1, <4 x float> %val2) {
+define float @f2(<4 x float> %val1, <4 x float> %val2) strictfp {
 ; CHECK-LABEL: f2:
 ; CHECK: wfasb %f0, %v24, %v26
 ; CHECK: br %r14
@@ -28,6 +28,6 @@ define float @f2(<4 x float> %val1, <4 x float> %val2) {
   %ret = call float @llvm.experimental.constrained.fadd.f32(
                         float %scalar1, float %scalar2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") strictfp
   ret float %ret
 }
diff --git a/llvm/test/CodeGen/SystemZ/vec-strict-cmp-01.ll b/llvm/test/CodeGen/SystemZ/vec-strict-cmp-01.ll
new file mode 100644
index 000000000..dc8d0090d
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/vec-strict-cmp-01.ll
@@ -0,0 +1,560 @@
+; Test strict v4f32 comparisons.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test oeq.
+define <4 x i32> @f1(<4 x float> %val1, <4 x float> %val2) #0 {
+; CHECK-LABEL: f1:
+; CHECK-DAG: vmrhf [[HIGH0E:%v[0-9]+]], %v24, %v24
+; CHECK-DAG: vmrlf [[LOW0E:%v[0-9]+]], %v24, %v24
+; CHECK-DAG: vmrhf [[HIGH1E:%v[0-9]+]], %v26, %v26
+; CHECK-DAG: vmrlf [[LOW1E:%v[0-9]+]], %v26, %v26
+; CHECK-DAG: vldeb [[HIGH0D:%v[0-9]+]], [[HIGH0E]]
+; CHECK-DAG: vldeb [[HIGH1D:%v[0-9]+]], [[HIGH1E]]
+; CHECK-DAG: vldeb [[LOW0D:%v[0-9]+]], [[LOW0E]]
+; CHECK-DAG: vldeb [[LOW1D:%v[0-9]+]], [[LOW1E]]
+; CHECK-DAG: vfcedb [[HIGHRES:%v[0-9]+]], [[HIGH0D]], [[HIGH1D]]
+; CHECK-DAG: vfcedb [[LOWRES:%v[0-9]+]], [[LOW0D]], [[LOW1D]]
+; CHECK: vpkg %v24, [[HIGHRES]], [[LOWRES]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test one.
+define <4 x i32> @f2(<4 x float> %val1, <4 x float> %val2) #0 {
+; CHECK-LABEL: f2:
+; CHECK-DAG: vmrhf [[HIGH0E:%v[0-9]+]], %v24, %v24
+; CHECK-DAG: vmrlf [[LOW0E:%v[0-9]+]], %v24, %v24
+; CHECK-DAG: vmrhf [[HIGH1E:%v[0-9]+]], %v26, %v26
+; CHECK-DAG: vmrlf [[LOW1E:%v[0-9]+]], %v26, %v26
+; CHECK-DAG: vldeb [[HIGH0D:%v[0-9]+]], [[HIGH0E]]
+; CHECK-DAG: vldeb [[HIGH1D:%v[0-9]+]], [[HIGH1E]]
+; CHECK-DAG: vldeb [[LOW0D:%v[0-9]+]], [[LOW0E]]
+; CHECK-DAG: vldeb [[LOW1D:%v[0-9]+]], [[LOW1E]]
+; CHECK-DAG: vfchdb [[HIGHRES0:%v[0-9]+]], [[HIGH0D]], [[HIGH1D]]
+; CHECK-DAG: vfchdb [[LOWRES0:%v[0-9]+]], [[LOW0D]], [[LOW1D]]
+; CHECK-DAG: vfchdb [[HIGHRES1:%v[0-9]+]], [[HIGH1D]], [[HIGH0D]]
+; CHECK-DAG: vfchdb [[LOWRES1:%v[0-9]+]], [[LOW1D]], [[LOW0D]]
+; CHECK-DAG: vpkg [[RES0:%v[0-9]+]], [[HIGHRES0]], [[LOWRES0]]
+; CHECK-DAG: vpkg [[RES1:%v[0-9]+]], [[HIGHRES1]], [[LOWRES1]]
+; CHECK: vo %v24, [[RES1]], [[RES0]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"one",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test ogt.
+define <4 x i32> @f3(<4 x float> %val1, <4 x float> %val2) #0 {
+; CHECK-LABEL: f3:
+; CHECK-DAG: vmrhf [[HIGH0E:%v[0-9]+]], %v24, %v24
+; CHECK-DAG: vmrlf [[LOW0E:%v[0-9]+]], %v24, %v24
+; CHECK-DAG: vmrhf [[HIGH1E:%v[0-9]+]], %v26, %v26
+; CHECK-DAG: vmrlf [[LOW1E:%v[0-9]+]], %v26, %v26
+; CHECK-DAG: vldeb [[HIGH0D:%v[0-9]+]], [[HIGH0E]]
+; CHECK-DAG: vldeb [[HIGH1D:%v[0-9]+]], [[HIGH1E]]
+; CHECK-DAG: vldeb [[LOW0D:%v[0-9]+]], [[LOW0E]]
+; CHECK-DAG: vldeb [[LOW1D:%v[0-9]+]], [[LOW1E]]
+; CHECK-DAG: vfchdb [[HIGHRES:%v[0-9]+]], [[HIGH0D]], [[HIGH1D]]
+; CHECK-DAG: vfchdb [[LOWRES:%v[0-9]+]], [[LOW0D]], [[LOW1D]]
+; CHECK: vpkg %v24, [[HIGHRES]], [[LOWRES]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"ogt",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test oge.
+define <4 x i32> @f4(<4 x float> %val1, <4 x float> %val2) #0 {
+; CHECK-LABEL: f4:
+; CHECK-DAG: vmrhf [[HIGH0E:%v[0-9]+]], %v24, %v24
+; CHECK-DAG: vmrlf [[LOW0E:%v[0-9]+]], %v24, %v24
+; CHECK-DAG: vmrhf [[HIGH1E:%v[0-9]+]], %v26, %v26
+; CHECK-DAG: vmrlf [[LOW1E:%v[0-9]+]], %v26, %v26
+; CHECK-DAG: vldeb [[HIGH0D:%v[0-9]+]], [[HIGH0E]]
+; CHECK-DAG: vldeb [[HIGH1D:%v[0-9]+]], [[HIGH1E]]
+; CHECK-DAG: vldeb [[LOW0D:%v[0-9]+]], [[LOW0E]]
+; CHECK-DAG: vldeb [[LOW1D:%v[0-9]+]], [[LOW1E]]
+; CHECK-DAG: vfchedb [[HIGHRES:%v[0-9]+]], [[HIGH0D]], [[HIGH1D]]
+; CHECK-DAG: vfchedb [[LOWRES:%v[0-9]+]], [[LOW0D]], [[LOW1D]]
+; CHECK: vpkg %v24, [[HIGHRES]], [[LOWRES]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"oge",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test ole.
+define <4 x i32> @f5(<4 x float> %val1, <4 x float> %val2) #0 {
+; CHECK-LABEL: f5:
+; CHECK-DAG: vmrhf [[HIGH0E:%v[0-9]+]], %v24, %v24
+; CHECK-DAG: vmrlf [[LOW0E:%v[0-9]+]], %v24, %v24
+; CHECK-DAG: vmrhf [[HIGH1E:%v[0-9]+]], %v26, %v26
+; CHECK-DAG: vmrlf [[LOW1E:%v[0-9]+]], %v26, %v26
+; CHECK-DAG: vldeb [[HIGH0D:%v[0-9]+]], [[HIGH0E]]
+; CHECK-DAG: vldeb [[HIGH1D:%v[0-9]+]], [[HIGH1E]]
+; CHECK-DAG: vldeb [[LOW0D:%v[0-9]+]], [[LOW0E]]
+; CHECK-DAG: vldeb [[LOW1D:%v[0-9]+]], [[LOW1E]]
+; CHECK-DAG: vfchedb [[HIGHRES:%v[0-9]+]], [[HIGH1D]], [[HIGH0D]]
+; CHECK-DAG: vfchedb [[LOWRES:%v[0-9]+]], [[LOW1D]], [[LOW0D]]
+; CHECK: vpkg %v24, [[HIGHRES]], [[LOWRES]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"ole",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test olt.
+define <4 x i32> @f6(<4 x float> %val1, <4 x float> %val2) #0 {
+; CHECK-LABEL: f6:
+; CHECK-DAG: vmrhf [[HIGH0E:%v[0-9]+]], %v24, %v24
+; CHECK-DAG: vmrlf [[LOW0E:%v[0-9]+]], %v24, %v24
+; CHECK-DAG: vmrhf [[HIGH1E:%v[0-9]+]], %v26, %v26
+; CHECK-DAG: vmrlf [[LOW1E:%v[0-9]+]], %v26, %v26
+; CHECK-DAG: vldeb [[HIGH0D:%v[0-9]+]], [[HIGH0E]]
+; CHECK-DAG: vldeb [[HIGH1D:%v[0-9]+]], [[HIGH1E]]
+; CHECK-DAG: vldeb [[LOW0D:%v[0-9]+]], [[LOW0E]]
+; CHECK-DAG: vldeb [[LOW1D:%v[0-9]+]], [[LOW1E]]
+; CHECK-DAG: vfchdb [[HIGHRES:%v[0-9]+]], [[HIGH1D]], [[HIGH0D]]
+; CHECK-DAG: vfchdb [[LOWRES:%v[0-9]+]], [[LOW1D]], [[LOW0D]]
+; CHECK: vpkg %v24, [[HIGHRES]], [[LOWRES]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test ueq.
+define <4 x i32> @f7(<4 x float> %val1, <4 x float> %val2) #0 {
+; CHECK-LABEL: f7:
+; CHECK-DAG: vmrhf [[HIGH0E:%v[0-9]+]], %v24, %v24
+; CHECK-DAG: vmrlf [[LOW0E:%v[0-9]+]], %v24, %v24
+; CHECK-DAG: vmrhf [[HIGH1E:%v[0-9]+]], %v26, %v26
+; CHECK-DAG: vmrlf [[LOW1E:%v[0-9]+]], %v26, %v26
+; CHECK-DAG: vldeb [[HIGH0D:%v[0-9]+]], [[HIGH0E]]
+; CHECK-DAG: vldeb [[HIGH1D:%v[0-9]+]], [[HIGH1E]]
+; CHECK-DAG: vldeb [[LOW0D:%v[0-9]+]], [[LOW0E]]
+; CHECK-DAG: vldeb [[LOW1D:%v[0-9]+]], [[LOW1E]]
+; CHECK-DAG: vfchdb [[HIGHRES0:%v[0-9]+]], [[HIGH0D]], [[HIGH1D]]
+; CHECK-DAG: vfchdb [[LOWRES0:%v[0-9]+]], [[LOW0D]], [[LOW1D]]
+; CHECK-DAG: vfchdb [[HIGHRES1:%v[0-9]+]], [[HIGH1D]], [[HIGH0D]]
+; CHECK-DAG: vfchdb [[LOWRES1:%v[0-9]+]], [[LOW1D]], [[LOW0D]]
+; CHECK-DAG: vpkg [[RES0:%v[0-9]+]], [[HIGHRES0]], [[LOWRES0]]
+; CHECK-DAG: vpkg [[RES1:%v[0-9]+]], [[HIGHRES1]], [[LOWRES1]]
+; CHECK: vno %v24, [[RES1]], [[RES0]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"ueq",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test une.
+define <4 x i32> @f8(<4 x float> %val1, <4 x float> %val2) #0 {
+; CHECK-LABEL: f8:
+; CHECK-DAG: vmrhf [[HIGH0E:%v[0-9]+]], %v24, %v24
+; CHECK-DAG: vmrlf [[LOW0E:%v[0-9]+]], %v24, %v24
+; CHECK-DAG: vmrhf [[HIGH1E:%v[0-9]+]], %v26, %v26
+; CHECK-DAG: vmrlf [[LOW1E:%v[0-9]+]], %v26, %v26
+; CHECK-DAG: vldeb [[HIGH0D:%v[0-9]+]], [[HIGH0E]]
+; CHECK-DAG: vldeb [[HIGH1D:%v[0-9]+]], [[HIGH1E]]
+; CHECK-DAG: vldeb [[LOW0D:%v[0-9]+]], [[LOW0E]]
+; CHECK-DAG: vldeb [[LOW1D:%v[0-9]+]], [[LOW1E]]
+; CHECK-DAG: vfcedb [[HIGHRES:%v[0-9]+]], [[HIGH0D]], [[HIGH1D]]
+; CHECK-DAG: vfcedb [[LOWRES:%v[0-9]+]], [[LOW0D]], [[LOW1D]]
+; CHECK: vpkg [[RES:%v[0-9]+]], [[HIGHRES]], [[LOWRES]]
+; CHECK-NEXT: vno %v24, [[RES]], [[RES]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"une",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test ugt.
+define <4 x i32> @f9(<4 x float> %val1, <4 x float> %val2) #0 {
+; CHECK-LABEL: f9:
+; CHECK-DAG: vmrhf [[HIGH0E:%v[0-9]+]], %v24, %v24
+; CHECK-DAG: vmrlf [[LOW0E:%v[0-9]+]], %v24, %v24
+; CHECK-DAG: vmrhf [[HIGH1E:%v[0-9]+]], %v26, %v26
+; CHECK-DAG: vmrlf [[LOW1E:%v[0-9]+]], %v26, %v26
+; CHECK-DAG: vldeb [[HIGH0D:%v[0-9]+]], [[HIGH0E]]
+; CHECK-DAG: vldeb [[HIGH1D:%v[0-9]+]], [[HIGH1E]]
+; CHECK-DAG: vldeb [[LOW0D:%v[0-9]+]], [[LOW0E]]
+; CHECK-DAG: vldeb [[LOW1D:%v[0-9]+]], [[LOW1E]]
+; CHECK-DAG: vfchedb [[HIGHRES:%v[0-9]+]], [[HIGH1D]], [[HIGH0D]]
+; CHECK-DAG: vfchedb [[LOWRES:%v[0-9]+]], [[LOW1D]], [[LOW0D]]
+; CHECK: vpkg [[RES:%v[0-9]+]], [[HIGHRES]], [[LOWRES]]
+; CHECK-NEXT: vno %v24, [[RES]], [[RES]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"ugt",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test uge.
+define <4 x i32> @f10(<4 x float> %val1, <4 x float> %val2) #0 {
+; CHECK-LABEL: f10:
+; CHECK-DAG: vmrhf [[HIGH0E:%v[0-9]+]], %v24, %v24
+; CHECK-DAG: vmrlf [[LOW0E:%v[0-9]+]], %v24, %v24
+; CHECK-DAG: vmrhf [[HIGH1E:%v[0-9]+]], %v26, %v26
+; CHECK-DAG: vmrlf [[LOW1E:%v[0-9]+]], %v26, %v26
+; CHECK-DAG: vldeb [[HIGH0D:%v[0-9]+]], [[HIGH0E]]
+; CHECK-DAG: vldeb [[HIGH1D:%v[0-9]+]], [[HIGH1E]]
+; CHECK-DAG: vldeb [[LOW0D:%v[0-9]+]], [[LOW0E]]
+; CHECK-DAG: vldeb [[LOW1D:%v[0-9]+]], [[LOW1E]]
+; CHECK-DAG: vfchdb [[HIGHRES:%v[0-9]+]], [[HIGH1D]], [[HIGH0D]]
+; CHECK-DAG: vfchdb [[LOWRES:%v[0-9]+]], [[LOW1D]], [[LOW0D]]
+; CHECK: vpkg [[RES:%v[0-9]+]], [[HIGHRES]], [[LOWRES]]
+; CHECK-NEXT: vno %v24, [[RES]], [[RES]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"uge",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test ule.
+define <4 x i32> @f11(<4 x float> %val1, <4 x float> %val2) #0 {
+; CHECK-LABEL: f11:
+; CHECK-DAG: vmrhf [[HIGH0E:%v[0-9]+]], %v24, %v24
+; CHECK-DAG: vmrlf [[LOW0E:%v[0-9]+]], %v24, %v24
+; CHECK-DAG: vmrhf [[HIGH1E:%v[0-9]+]], %v26, %v26
+; CHECK-DAG: vmrlf [[LOW1E:%v[0-9]+]], %v26, %v26
+; CHECK-DAG: vldeb [[HIGH0D:%v[0-9]+]], [[HIGH0E]]
+; CHECK-DAG: vldeb [[HIGH1D:%v[0-9]+]], [[HIGH1E]]
+; CHECK-DAG: vldeb [[LOW0D:%v[0-9]+]], [[LOW0E]]
+; CHECK-DAG: vldeb [[LOW1D:%v[0-9]+]], [[LOW1E]]
+; CHECK-DAG: vfchdb [[HIGHRES:%v[0-9]+]], [[HIGH0D]], [[HIGH1D]]
+; CHECK-DAG: vfchdb [[LOWRES:%v[0-9]+]], [[LOW0D]], [[LOW1D]]
+; CHECK: vpkg [[RES:%v[0-9]+]], [[HIGHRES]], [[LOWRES]]
+; CHECK-NEXT: vno %v24, [[RES]], [[RES]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"ule",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test ult.
+define <4 x i32> @f12(<4 x float> %val1, <4 x float> %val2) #0 {
+; CHECK-LABEL: f12:
+; CHECK-DAG: vmrhf [[HIGH0E:%v[0-9]+]], %v24, %v24
+; CHECK-DAG: vmrlf [[LOW0E:%v[0-9]+]], %v24, %v24
+; CHECK-DAG: vmrhf [[HIGH1E:%v[0-9]+]], %v26, %v26
+; CHECK-DAG: vmrlf [[LOW1E:%v[0-9]+]], %v26, %v26
+; CHECK-DAG: vldeb [[HIGH0D:%v[0-9]+]], [[HIGH0E]]
+; CHECK-DAG: vldeb [[HIGH1D:%v[0-9]+]], [[HIGH1E]]
+; CHECK-DAG: vldeb [[LOW0D:%v[0-9]+]], [[LOW0E]]
+; CHECK-DAG: vldeb [[LOW1D:%v[0-9]+]], [[LOW1E]]
+; CHECK-DAG: vfchedb [[HIGHRES:%v[0-9]+]], [[HIGH0D]], [[HIGH1D]]
+; CHECK-DAG: vfchedb [[LOWRES:%v[0-9]+]], [[LOW0D]], [[LOW1D]]
+; CHECK: vpkg [[RES:%v[0-9]+]], [[HIGHRES]], [[LOWRES]]
+; CHECK-NEXT: vno %v24, [[RES]], [[RES]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"ult",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test ord.
+define <4 x i32> @f13(<4 x float> %val1, <4 x float> %val2) #0 {
+; CHECK-LABEL: f13:
+; CHECK-DAG: vmrhf [[HIGH0E:%v[0-9]+]], %v24, %v24
+; CHECK-DAG: vmrlf [[LOW0E:%v[0-9]+]], %v24, %v24
+; CHECK-DAG: vmrhf [[HIGH1E:%v[0-9]+]], %v26, %v26
+; CHECK-DAG: vmrlf [[LOW1E:%v[0-9]+]], %v26, %v26
+; CHECK-DAG: vldeb [[HIGH0D:%v[0-9]+]], [[HIGH0E]]
+; CHECK-DAG: vldeb [[HIGH1D:%v[0-9]+]], [[HIGH1E]]
+; CHECK-DAG: vldeb [[LOW0D:%v[0-9]+]], [[LOW0E]]
+; CHECK-DAG: vldeb [[LOW1D:%v[0-9]+]], [[LOW1E]]
+; CHECK-DAG: vfchedb [[HIGHRES0:%v[0-9]+]], [[HIGH0D]], [[HIGH1D]]
+; CHECK-DAG: vfchedb [[LOWRES0:%v[0-9]+]], [[LOW0D]], [[LOW1D]]
+; CHECK-DAG: vfchdb [[HIGHRES1:%v[0-9]+]], [[HIGH1D]], [[HIGH0D]]
+; CHECK-DAG: vfchdb [[LOWRES1:%v[0-9]+]], [[LOW1D]], [[LOW0D]]
+; CHECK-DAG: vpkg [[RES0:%v[0-9]+]], [[HIGHRES0]], [[LOWRES0]]
+; CHECK-DAG: vpkg [[RES1:%v[0-9]+]], [[HIGHRES1]], [[LOWRES1]]
+; CHECK: vo %v24, [[RES1]], [[RES0]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"ord",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test uno.
+define <4 x i32> @f14(<4 x float> %val1, <4 x float> %val2) #0 {
+; CHECK-LABEL: f14:
+; CHECK-DAG: vmrhf [[HIGH0E:%v[0-9]+]], %v24, %v24
+; CHECK-DAG: vmrlf [[LOW0E:%v[0-9]+]], %v24, %v24
+; CHECK-DAG: vmrhf [[HIGH1E:%v[0-9]+]], %v26, %v26
+; CHECK-DAG: vmrlf [[LOW1E:%v[0-9]+]], %v26, %v26
+; CHECK-DAG: vldeb [[HIGH0D:%v[0-9]+]], [[HIGH0E]]
+; CHECK-DAG: vldeb [[HIGH1D:%v[0-9]+]], [[HIGH1E]]
+; CHECK-DAG: vldeb [[LOW0D:%v[0-9]+]], [[LOW0E]]
+; CHECK-DAG: vldeb [[LOW1D:%v[0-9]+]], [[LOW1E]]
+; CHECK-DAG: vfchedb [[HIGHRES0:%v[0-9]+]], [[HIGH0D]], [[HIGH1D]]
+; CHECK-DAG: vfchedb [[LOWRES0:%v[0-9]+]], [[LOW0D]], [[LOW1D]]
+; CHECK-DAG: vfchdb [[HIGHRES1:%v[0-9]+]], [[HIGH1D]], [[HIGH0D]]
+; CHECK-DAG: vfchdb [[LOWRES1:%v[0-9]+]], [[LOW1D]], [[LOW0D]]
+; CHECK-DAG: vpkg [[RES0:%v[0-9]+]], [[HIGHRES0]], [[LOWRES0]]
+; CHECK-DAG: vpkg [[RES1:%v[0-9]+]], [[HIGHRES1]], [[LOWRES1]]
+; CHECK: vno %v24, [[RES1]], [[RES0]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"uno",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test oeq selects.
+define <4 x float> @f15(<4 x float> %val1, <4 x float> %val2,
+                        <4 x float> %val3, <4 x float> %val4) #0 {
+; CHECK-LABEL: f15:
+; CHECK: vpkg [[REG:%v[0-9]+]],
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <4 x i1> %cmp, <4 x float> %val3, <4 x float> %val4
+  ret <4 x float> %ret
+}
+
+; Test one selects.
+define <4 x float> @f16(<4 x float> %val1, <4 x float> %val2,
+                        <4 x float> %val3, <4 x float> %val4) #0 {
+; CHECK-LABEL: f16:
+; CHECK: vo [[REG:%v[0-9]+]],
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"one",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <4 x i1> %cmp, <4 x float> %val3, <4 x float> %val4
+  ret <4 x float> %ret
+}
+
+; Test ogt selects.
+define <4 x float> @f17(<4 x float> %val1, <4 x float> %val2,
+                        <4 x float> %val3, <4 x float> %val4) #0 {
+; CHECK-LABEL: f17:
+; CHECK: vpkg [[REG:%v[0-9]+]],
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"ogt",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <4 x i1> %cmp, <4 x float> %val3, <4 x float> %val4
+  ret <4 x float> %ret
+}
+
+; Test oge selects.
+define <4 x float> @f18(<4 x float> %val1, <4 x float> %val2,
+                        <4 x float> %val3, <4 x float> %val4) #0 {
+; CHECK-LABEL: f18:
+; CHECK: vpkg [[REG:%v[0-9]+]],
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"oge",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <4 x i1> %cmp, <4 x float> %val3, <4 x float> %val4
+  ret <4 x float> %ret
+}
+
+; Test ole selects.
+define <4 x float> @f19(<4 x float> %val1, <4 x float> %val2,
+                        <4 x float> %val3, <4 x float> %val4) #0 {
+; CHECK-LABEL: f19:
+; CHECK: vpkg [[REG:%v[0-9]+]],
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"ole",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <4 x i1> %cmp, <4 x float> %val3, <4 x float> %val4
+  ret <4 x float> %ret
+}
+
+; Test olt selects.
+define <4 x float> @f20(<4 x float> %val1, <4 x float> %val2,
+                        <4 x float> %val3, <4 x float> %val4) #0 {
+; CHECK-LABEL: f20:
+; CHECK: vpkg [[REG:%v[0-9]+]],
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <4 x i1> %cmp, <4 x float> %val3, <4 x float> %val4
+  ret <4 x float> %ret
+}
+
+; Test ueq selects.
+define <4 x float> @f21(<4 x float> %val1, <4 x float> %val2,
+                        <4 x float> %val3, <4 x float> %val4) #0 {
+; CHECK-LABEL: f21:
+; CHECK: vo [[REG:%v[0-9]+]],
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"ueq",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <4 x i1> %cmp, <4 x float> %val3, <4 x float> %val4
+  ret <4 x float> %ret
+}
+
+; Test une selects.
+define <4 x float> @f22(<4 x float> %val1, <4 x float> %val2,
+                        <4 x float> %val3, <4 x float> %val4) #0 {
+; CHECK-LABEL: f22:
+; CHECK: vpkg [[REG:%v[0-9]+]],
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"une",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <4 x i1> %cmp, <4 x float> %val3, <4 x float> %val4
+  ret <4 x float> %ret
+}
+
+; Test ugt selects.
+define <4 x float> @f23(<4 x float> %val1, <4 x float> %val2,
+                        <4 x float> %val3, <4 x float> %val4) #0 {
+; CHECK-LABEL: f23:
+; CHECK: vpkg [[REG:%v[0-9]+]],
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"ugt",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <4 x i1> %cmp, <4 x float> %val3, <4 x float> %val4
+  ret <4 x float> %ret
+}
+
+; Test uge selects.
+define <4 x float> @f24(<4 x float> %val1, <4 x float> %val2,
+                        <4 x float> %val3, <4 x float> %val4) #0 {
+; CHECK-LABEL: f24:
+; CHECK: vpkg [[REG:%v[0-9]+]],
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"uge",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <4 x i1> %cmp, <4 x float> %val3, <4 x float> %val4
+  ret <4 x float> %ret
+}
+
+; Test ule selects.
+define <4 x float> @f25(<4 x float> %val1, <4 x float> %val2,
+                        <4 x float> %val3, <4 x float> %val4) #0 {
+; CHECK-LABEL: f25:
+; CHECK: vpkg [[REG:%v[0-9]+]],
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"ule",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <4 x i1> %cmp, <4 x float> %val3, <4 x float> %val4
+  ret <4 x float> %ret
+}
+
+; Test ult selects.
+define <4 x float> @f26(<4 x float> %val1, <4 x float> %val2,
+                        <4 x float> %val3, <4 x float> %val4) #0 {
+; CHECK-LABEL: f26:
+; CHECK: vpkg [[REG:%v[0-9]+]],
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"ult",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <4 x i1> %cmp, <4 x float> %val3, <4 x float> %val4
+  ret <4 x float> %ret
+}
+
+; Test ord selects.
+define <4 x float> @f27(<4 x float> %val1, <4 x float> %val2,
+                        <4 x float> %val3, <4 x float> %val4) #0 {
+; CHECK-LABEL: f27:
+; CHECK: vo [[REG:%v[0-9]+]],
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"ord",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <4 x i1> %cmp, <4 x float> %val3, <4 x float> %val4
+  ret <4 x float> %ret
+}
+
+; Test uno selects.
+define <4 x float> @f28(<4 x float> %val1, <4 x float> %val2,
+                        <4 x float> %val3, <4 x float> %val4) #0 {
+; CHECK-LABEL: f28:
+; CHECK: vo [[REG:%v[0-9]+]],
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"uno",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <4 x i1> %cmp, <4 x float> %val3, <4 x float> %val4
+  ret <4 x float> %ret
+}
+
+attributes #0 = { strictfp }
+
+declare <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(<4 x float>, <4 x float>, metadata, metadata)
diff --git a/llvm/test/CodeGen/SystemZ/vec-strict-cmp-02.ll b/llvm/test/CodeGen/SystemZ/vec-strict-cmp-02.ll
new file mode 100644
index 000000000..2e8df56fd
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/vec-strict-cmp-02.ll
@@ -0,0 +1,442 @@
+; Test f64 and v2f64 strict comparisons.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test oeq.
+define <2 x i64> @f1(<2 x i64> %dummy, <2 x double> %val1, <2 x double> %val2) #0 {
+; CHECK-LABEL: f1:
+; CHECK: vfcedb %v24, %v26, %v28
+; CHECK-NEXT: br %r14
+  %cmp = call <2 x i1> @llvm.experimental.constrained.fcmp.v2f64(
+                                               <2 x double> %val1, <2 x double> %val2,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <2 x i1> %cmp to <2 x i64>
+  ret <2 x i64> %ret
+}
+
+; Test one.
+define <2 x i64> @f2(<2 x i64> %dummy, <2 x double> %val1, <2 x double> %val2) #0 {
+; CHECK-LABEL: f2:
+; CHECK-DAG: vfchdb [[REG1:%v[0-9]+]], %v28, %v26
+; CHECK-DAG: vfchdb [[REG2:%v[0-9]+]], %v26, %v28
+; CHECK: vo %v24, [[REG1]], [[REG2]]
+; CHECK-NEXT: br %r14
+  %cmp = call <2 x i1> @llvm.experimental.constrained.fcmp.v2f64(
+                                               <2 x double> %val1, <2 x double> %val2,
+                                               metadata !"one",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <2 x i1> %cmp to <2 x i64>
+  ret <2 x i64> %ret
+}
+
+; Test ogt.
+define <2 x i64> @f3(<2 x i64> %dummy, <2 x double> %val1, <2 x double> %val2) #0 {
+; CHECK-LABEL: f3:
+; CHECK: vfchdb %v24, %v26, %v28
+; CHECK-NEXT: br %r14
+  %cmp = call <2 x i1> @llvm.experimental.constrained.fcmp.v2f64(
+                                               <2 x double> %val1, <2 x double> %val2,
+                                               metadata !"ogt",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <2 x i1> %cmp to <2 x i64>
+  ret <2 x i64> %ret
+}
+
+; Test oge.
+define <2 x i64> @f4(<2 x i64> %dummy, <2 x double> %val1, <2 x double> %val2) #0 {
+; CHECK-LABEL: f4:
+; CHECK: vfchedb %v24, %v26, %v28
+; CHECK-NEXT: br %r14
+  %cmp = call <2 x i1> @llvm.experimental.constrained.fcmp.v2f64(
+                                               <2 x double> %val1, <2 x double> %val2,
+                                               metadata !"oge",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <2 x i1> %cmp to <2 x i64>
+  ret <2 x i64> %ret
+}
+
+; Test ole.
+define <2 x i64> @f5(<2 x i64> %dummy, <2 x double> %val1, <2 x double> %val2) #0 {
+; CHECK-LABEL: f5:
+; CHECK: vfchedb %v24, %v28, %v26
+; CHECK-NEXT: br %r14
+  %cmp = call <2 x i1> @llvm.experimental.constrained.fcmp.v2f64(
+                                               <2 x double> %val1, <2 x double> %val2,
+                                               metadata !"ole",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <2 x i1> %cmp to <2 x i64>
+  ret <2 x i64> %ret
+}
+
+; Test olt.
+define <2 x i64> @f6(<2 x i64> %dummy, <2 x double> %val1, <2 x double> %val2) #0 {
+; CHECK-LABEL: f6:
+; CHECK: vfchdb %v24, %v28, %v26
+; CHECK-NEXT: br %r14
+  %cmp = call <2 x i1> @llvm.experimental.constrained.fcmp.v2f64(
+                                               <2 x double> %val1, <2 x double> %val2,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <2 x i1> %cmp to <2 x i64>
+  ret <2 x i64> %ret
+}
+
+; Test ueq.
+define <2 x i64> @f7(<2 x i64> %dummy, <2 x double> %val1, <2 x double> %val2) #0 {
+; CHECK-LABEL: f7:
+; CHECK-DAG: vfchdb [[REG1:%v[0-9]+]], %v28, %v26
+; CHECK-DAG: vfchdb [[REG2:%v[0-9]+]], %v26, %v28
+; CHECK: vno %v24, [[REG1]], [[REG2]]
+; CHECK-NEXT: br %r14
+  %cmp = call <2 x i1> @llvm.experimental.constrained.fcmp.v2f64(
+                                               <2 x double> %val1, <2 x double> %val2,
+                                               metadata !"ueq",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <2 x i1> %cmp to <2 x i64>
+  ret <2 x i64> %ret
+}
+
+; Test une.
+define <2 x i64> @f8(<2 x i64> %dummy, <2 x double> %val1, <2 x double> %val2) #0 {
+; CHECK-LABEL: f8:
+; CHECK: vfcedb [[REG:%v[0-9]+]], %v26, %v28
+; CHECK-NEXT: vno %v24, [[REG]], [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <2 x i1> @llvm.experimental.constrained.fcmp.v2f64(
+                                               <2 x double> %val1, <2 x double> %val2,
+                                               metadata !"une",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <2 x i1> %cmp to <2 x i64>
+  ret <2 x i64> %ret
+}
+
+; Test ugt.
+define <2 x i64> @f9(<2 x i64> %dummy, <2 x double> %val1, <2 x double> %val2) #0 {
+; CHECK-LABEL: f9:
+; CHECK: vfchedb [[REG:%v[0-9]+]], %v28, %v26
+; CHECK-NEXT: vno %v24, [[REG]], [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <2 x i1> @llvm.experimental.constrained.fcmp.v2f64(
+                                               <2 x double> %val1, <2 x double> %val2,
+                                               metadata !"ugt",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <2 x i1> %cmp to <2 x i64>
+  ret <2 x i64> %ret
+}
+
+; Test uge.
+define <2 x i64> @f10(<2 x i64> %dummy, <2 x double> %val1,
+                      <2 x double> %val2) #0 {
+; CHECK-LABEL: f10:
+; CHECK: vfchdb [[REG:%v[0-9]+]], %v28, %v26
+; CHECK-NEXT: vno %v24, [[REG]], [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <2 x i1> @llvm.experimental.constrained.fcmp.v2f64(
+                                               <2 x double> %val1, <2 x double> %val2,
+                                               metadata !"uge",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <2 x i1> %cmp to <2 x i64>
+  ret <2 x i64> %ret
+}
+
+; Test ule.
+define <2 x i64> @f11(<2 x i64> %dummy, <2 x double> %val1,
+                      <2 x double> %val2) #0 {
+; CHECK-LABEL: f11:
+; CHECK: vfchdb [[REG:%v[0-9]+]], %v26, %v28
+; CHECK-NEXT: vno %v24, [[REG]], [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <2 x i1> @llvm.experimental.constrained.fcmp.v2f64(
+                                               <2 x double> %val1, <2 x double> %val2,
+                                               metadata !"ule",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <2 x i1> %cmp to <2 x i64>
+  ret <2 x i64> %ret
+}
+
+; Test ult.
+define <2 x i64> @f12(<2 x i64> %dummy, <2 x double> %val1,
+                      <2 x double> %val2) #0 {
+; CHECK-LABEL: f12:
+; CHECK: vfchedb [[REG:%v[0-9]+]], %v26, %v28
+; CHECK-NEXT: vno %v24, [[REG]], [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <2 x i1> @llvm.experimental.constrained.fcmp.v2f64(
+                                               <2 x double> %val1, <2 x double> %val2,
+                                               metadata !"ult",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <2 x i1> %cmp to <2 x i64>
+  ret <2 x i64> %ret
+}
+
+; Test ord.
+define <2 x i64> @f13(<2 x i64> %dummy, <2 x double> %val1,
+                      <2 x double> %val2) #0 {
+; CHECK-LABEL: f13:
+; CHECK-DAG: vfchdb [[REG1:%v[0-9]+]], %v28, %v26
+; CHECK-DAG: vfchedb [[REG2:%v[0-9]+]], %v26, %v28
+; CHECK: vo %v24, [[REG1]], [[REG2]]
+; CHECK-NEXT: br %r14
+  %cmp = call <2 x i1> @llvm.experimental.constrained.fcmp.v2f64(
+                                               <2 x double> %val1, <2 x double> %val2,
+                                               metadata !"ord",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <2 x i1> %cmp to <2 x i64>
+  ret <2 x i64> %ret
+}
+
+; Test uno.
+define <2 x i64> @f14(<2 x i64> %dummy, <2 x double> %val1,
+                      <2 x double> %val2) #0 {
+; CHECK-LABEL: f14:
+; CHECK-DAG: vfchdb [[REG1:%v[0-9]+]], %v28, %v26
+; CHECK-DAG: vfchedb [[REG2:%v[0-9]+]], %v26, %v28
+; CHECK: vno %v24, [[REG1]], [[REG2]]
+; CHECK-NEXT: br %r14
+  %cmp = call <2 x i1> @llvm.experimental.constrained.fcmp.v2f64(
+                                               <2 x double> %val1, <2 x double> %val2,
+                                               metadata !"uno",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <2 x i1> %cmp to <2 x i64>
+  ret <2 x i64> %ret
+}
+
+; Test oeq selects.
+define <2 x double> @f15(<2 x double> %val1, <2 x double> %val2,
+                         <2 x double> %val3, <2 x double> %val4) #0 {
+; CHECK-LABEL: f15:
+; CHECK: vfcedb [[REG:%v[0-9]+]], %v24, %v26
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <2 x i1> @llvm.experimental.constrained.fcmp.v2f64(
+                                               <2 x double> %val1, <2 x double> %val2,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <2 x i1> %cmp, <2 x double> %val3, <2 x double> %val4
+  ret <2 x double> %ret
+}
+
+; Test one selects.
+define <2 x double> @f16(<2 x double> %val1, <2 x double> %val2,
+                         <2 x double> %val3, <2 x double> %val4) #0 {
+; CHECK-LABEL: f16:
+; CHECK-DAG: vfchdb [[REG1:%v[0-9]+]], %v26, %v24
+; CHECK-DAG: vfchdb [[REG2:%v[0-9]+]], %v24, %v26
+; CHECK: vo [[REG:%v[0-9]+]], [[REG1]], [[REG2]]
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <2 x i1> @llvm.experimental.constrained.fcmp.v2f64(
+                                               <2 x double> %val1, <2 x double> %val2,
+                                               metadata !"one",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <2 x i1> %cmp, <2 x double> %val3, <2 x double> %val4
+  ret <2 x double> %ret
+}
+
+; Test ogt selects.
+define <2 x double> @f17(<2 x double> %val1, <2 x double> %val2,
+                         <2 x double> %val3, <2 x double> %val4) #0 {
+; CHECK-LABEL: f17:
+; CHECK: vfchdb [[REG:%v[0-9]+]], %v24, %v26
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <2 x i1> @llvm.experimental.constrained.fcmp.v2f64(
+                                               <2 x double> %val1, <2 x double> %val2,
+                                               metadata !"ogt",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <2 x i1> %cmp, <2 x double> %val3, <2 x double> %val4
+  ret <2 x double> %ret
+}
+
+; Test oge selects.
+define <2 x double> @f18(<2 x double> %val1, <2 x double> %val2,
+                         <2 x double> %val3, <2 x double> %val4) #0 {
+; CHECK-LABEL: f18:
+; CHECK: vfchedb [[REG:%v[0-9]+]], %v24, %v26
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <2 x i1> @llvm.experimental.constrained.fcmp.v2f64(
+                                               <2 x double> %val1, <2 x double> %val2,
+                                               metadata !"oge",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <2 x i1> %cmp, <2 x double> %val3, <2 x double> %val4
+  ret <2 x double> %ret
+}
+
+; Test ole selects.
+define <2 x double> @f19(<2 x double> %val1, <2 x double> %val2,
+                         <2 x double> %val3, <2 x double> %val4) #0 {
+; CHECK-LABEL: f19:
+; CHECK: vfchedb [[REG:%v[0-9]+]], %v26, %v24
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <2 x i1> @llvm.experimental.constrained.fcmp.v2f64(
+                                               <2 x double> %val1, <2 x double> %val2,
+                                               metadata !"ole",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <2 x i1> %cmp, <2 x double> %val3, <2 x double> %val4
+  ret <2 x double> %ret
+}
+
+; Test olt selects.
+define <2 x double> @f20(<2 x double> %val1, <2 x double> %val2,
+                         <2 x double> %val3, <2 x double> %val4) #0 {
+; CHECK-LABEL: f20:
+; CHECK: vfchdb [[REG:%v[0-9]+]], %v26, %v24
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <2 x i1> @llvm.experimental.constrained.fcmp.v2f64(
+                                               <2 x double> %val1, <2 x double> %val2,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <2 x i1> %cmp, <2 x double> %val3, <2 x double> %val4
+  ret <2 x double> %ret
+}
+
+; Test ueq selects.
+define <2 x double> @f21(<2 x double> %val1, <2 x double> %val2,
+                         <2 x double> %val3, <2 x double> %val4) #0 {
+; CHECK-LABEL: f21:
+; CHECK-DAG: vfchdb [[REG1:%v[0-9]+]], %v26, %v24
+; CHECK-DAG: vfchdb [[REG2:%v[0-9]+]], %v24, %v26
+; CHECK: vo [[REG:%v[0-9]+]], [[REG1]], [[REG2]]
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <2 x i1> @llvm.experimental.constrained.fcmp.v2f64(
+                                               <2 x double> %val1, <2 x double> %val2,
+                                               metadata !"ueq",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <2 x i1> %cmp, <2 x double> %val3, <2 x double> %val4
+  ret <2 x double> %ret
+}
+
+; Test une selects.
+define <2 x double> @f22(<2 x double> %val1, <2 x double> %val2,
+                         <2 x double> %val3, <2 x double> %val4) #0 {
+; CHECK-LABEL: f22:
+; CHECK: vfcedb [[REG:%v[0-9]+]], %v24, %v26
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <2 x i1> @llvm.experimental.constrained.fcmp.v2f64(
+                                               <2 x double> %val1, <2 x double> %val2,
+                                               metadata !"une",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <2 x i1> %cmp, <2 x double> %val3, <2 x double> %val4
+  ret <2 x double> %ret
+}
+
+; Test ugt selects.
+define <2 x double> @f23(<2 x double> %val1, <2 x double> %val2,
+                         <2 x double> %val3, <2 x double> %val4) #0 {
+; CHECK-LABEL: f23:
+; CHECK: vfchedb [[REG:%v[0-9]+]], %v26, %v24
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <2 x i1> @llvm.experimental.constrained.fcmp.v2f64(
+                                               <2 x double> %val1, <2 x double> %val2,
+                                               metadata !"ugt",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <2 x i1> %cmp, <2 x double> %val3, <2 x double> %val4
+  ret <2 x double> %ret
+}
+
+; Test uge selects.
+define <2 x double> @f24(<2 x double> %val1, <2 x double> %val2,
+                         <2 x double> %val3, <2 x double> %val4) #0 {
+; CHECK-LABEL: f24:
+; CHECK: vfchdb [[REG:%v[0-9]+]], %v26, %v24
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <2 x i1> @llvm.experimental.constrained.fcmp.v2f64(
+                                               <2 x double> %val1, <2 x double> %val2,
+                                               metadata !"uge",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <2 x i1> %cmp, <2 x double> %val3, <2 x double> %val4
+  ret <2 x double> %ret
+}
+
+; Test ule selects.
+define <2 x double> @f25(<2 x double> %val1, <2 x double> %val2,
+                         <2 x double> %val3, <2 x double> %val4) #0 {
+; CHECK-LABEL: f25:
+; CHECK: vfchdb [[REG:%v[0-9]+]], %v24, %v26
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <2 x i1> @llvm.experimental.constrained.fcmp.v2f64(
+                                               <2 x double> %val1, <2 x double> %val2,
+                                               metadata !"ule",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <2 x i1> %cmp, <2 x double> %val3, <2 x double> %val4
+  ret <2 x double> %ret
+}
+
+; Test ult selects.
+define <2 x double> @f26(<2 x double> %val1, <2 x double> %val2,
+                         <2 x double> %val3, <2 x double> %val4) #0 {
+; CHECK-LABEL: f26:
+; CHECK: vfchedb [[REG:%v[0-9]+]], %v24, %v26
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <2 x i1> @llvm.experimental.constrained.fcmp.v2f64(
+                                               <2 x double> %val1, <2 x double> %val2,
+                                               metadata !"ult",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <2 x i1> %cmp, <2 x double> %val3, <2 x double> %val4
+  ret <2 x double> %ret
+}
+
+; Test ord selects.
+define <2 x double> @f27(<2 x double> %val1, <2 x double> %val2,
+                         <2 x double> %val3, <2 x double> %val4) #0 {
+; CHECK-LABEL: f27:
+; CHECK-DAG: vfchdb [[REG1:%v[0-9]+]], %v26, %v24
+; CHECK-DAG: vfchedb [[REG2:%v[0-9]+]], %v24, %v26
+; CHECK: vo [[REG:%v[0-9]+]], [[REG1]], [[REG2]]
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <2 x i1> @llvm.experimental.constrained.fcmp.v2f64(
+                                               <2 x double> %val1, <2 x double> %val2,
+                                               metadata !"ord",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <2 x i1> %cmp, <2 x double> %val3, <2 x double> %val4
+  ret <2 x double> %ret
+}
+
+; Test uno selects.
+define <2 x double> @f28(<2 x double> %val1, <2 x double> %val2,
+                         <2 x double> %val3, <2 x double> %val4) #0 {
+; CHECK-LABEL: f28:
+; CHECK-DAG: vfchdb [[REG1:%v[0-9]+]], %v26, %v24
+; CHECK-DAG: vfchedb [[REG2:%v[0-9]+]], %v24, %v26
+; CHECK: vo [[REG:%v[0-9]+]], [[REG1]], [[REG2]]
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <2 x i1> @llvm.experimental.constrained.fcmp.v2f64(
+                                               <2 x double> %val1, <2 x double> %val2,
+                                               metadata !"uno",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <2 x i1> %cmp, <2 x double> %val3, <2 x double> %val4
+  ret <2 x double> %ret
+}
+
+; Test an f64 comparison that uses vector registers.
+define i64 @f29(i64 %a, i64 %b, double %f1, <2 x double> %vec) #0 {
+; CHECK-LABEL: f29:
+; CHECK: wfcdb %f0, %v24
+; CHECK-NEXT: locgrne %r2, %r3
+; CHECK: br %r14
+  %f2 = extractelement <2 x double> %vec, i32 0
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f64(
+                                               double %f1, double %f2,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+attributes #0 = { strictfp }
+
+declare <2 x i1> @llvm.experimental.constrained.fcmp.v2f64(<2 x double>, <2 x double>, metadata, metadata)
+declare i1 @llvm.experimental.constrained.fcmp.f64(double, double, metadata, metadata)
+
diff --git a/llvm/test/CodeGen/SystemZ/vec-strict-cmp-03.ll b/llvm/test/CodeGen/SystemZ/vec-strict-cmp-03.ll
new file mode 100644
index 000000000..a40720977
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/vec-strict-cmp-03.ll
@@ -0,0 +1,442 @@
+; Test strict f32 and v4f32 comparisons on z14.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z14 | FileCheck %s
+
+; Test oeq.
+define <4 x i32> @f1(<4 x i32> %dummy, <4 x float> %val1, <4 x float> %val2) #0 {
+; CHECK-LABEL: f1:
+; CHECK: vfcesb %v24, %v26, %v28
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test one.
+define <4 x i32> @f2(<4 x i32> %dummy, <4 x float> %val1, <4 x float> %val2) #0 {
+; CHECK-LABEL: f2:
+; CHECK-DAG: vfchsb [[REG1:%v[0-9]+]], %v28, %v26
+; CHECK-DAG: vfchsb [[REG2:%v[0-9]+]], %v26, %v28
+; CHECK: vo %v24, [[REG1]], [[REG2]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"one",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test ogt.
+define <4 x i32> @f3(<4 x i32> %dummy, <4 x float> %val1, <4 x float> %val2) #0 {
+; CHECK-LABEL: f3:
+; CHECK: vfchsb %v24, %v26, %v28
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"ogt",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test oge.
+define <4 x i32> @f4(<4 x i32> %dummy, <4 x float> %val1, <4 x float> %val2) #0 {
+; CHECK-LABEL: f4:
+; CHECK: vfchesb %v24, %v26, %v28
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"oge",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test ole.
+define <4 x i32> @f5(<4 x i32> %dummy, <4 x float> %val1, <4 x float> %val2) #0 {
+; CHECK-LABEL: f5:
+; CHECK: vfchesb %v24, %v28, %v26
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"ole",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test olt.
+define <4 x i32> @f6(<4 x i32> %dummy, <4 x float> %val1, <4 x float> %val2) #0 {
+; CHECK-LABEL: f6:
+; CHECK: vfchsb %v24, %v28, %v26
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test ueq.
+define <4 x i32> @f7(<4 x i32> %dummy, <4 x float> %val1, <4 x float> %val2) #0 {
+; CHECK-LABEL: f7:
+; CHECK-DAG: vfchsb [[REG1:%v[0-9]+]], %v28, %v26
+; CHECK-DAG: vfchsb [[REG2:%v[0-9]+]], %v26, %v28
+; CHECK: vno %v24, [[REG1]], [[REG2]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"ueq",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test une.
+define <4 x i32> @f8(<4 x i32> %dummy, <4 x float> %val1, <4 x float> %val2) #0 {
+; CHECK-LABEL: f8:
+; CHECK: vfcesb [[REG:%v[0-9]+]], %v26, %v28
+; CHECK-NEXT: vno %v24, [[REG]], [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"une",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test ugt.
+define <4 x i32> @f9(<4 x i32> %dummy, <4 x float> %val1, <4 x float> %val2) #0 {
+; CHECK-LABEL: f9:
+; CHECK: vfchesb [[REG:%v[0-9]+]], %v28, %v26
+; CHECK-NEXT: vno %v24, [[REG]], [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"ugt",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test uge.
+define <4 x i32> @f10(<4 x i32> %dummy, <4 x float> %val1,
+                      <4 x float> %val2) #0 {
+; CHECK-LABEL: f10:
+; CHECK: vfchsb [[REG:%v[0-9]+]], %v28, %v26
+; CHECK-NEXT: vno %v24, [[REG]], [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"uge",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test ule.
+define <4 x i32> @f11(<4 x i32> %dummy, <4 x float> %val1,
+                      <4 x float> %val2) #0 {
+; CHECK-LABEL: f11:
+; CHECK: vfchsb [[REG:%v[0-9]+]], %v26, %v28
+; CHECK-NEXT: vno %v24, [[REG]], [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"ule",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test ult.
+define <4 x i32> @f12(<4 x i32> %dummy, <4 x float> %val1,
+                      <4 x float> %val2) #0 {
+; CHECK-LABEL: f12:
+; CHECK: vfchesb [[REG:%v[0-9]+]], %v26, %v28
+; CHECK-NEXT: vno %v24, [[REG]], [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"ult",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test ord.
+define <4 x i32> @f13(<4 x i32> %dummy, <4 x float> %val1,
+                      <4 x float> %val2) #0 {
+; CHECK-LABEL: f13:
+; CHECK-DAG: vfchsb [[REG1:%v[0-9]+]], %v28, %v26
+; CHECK-DAG: vfchesb [[REG2:%v[0-9]+]], %v26, %v28
+; CHECK: vo %v24, [[REG1]], [[REG2]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"ord",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test uno.
+define <4 x i32> @f14(<4 x i32> %dummy, <4 x float> %val1,
+                      <4 x float> %val2) #0 {
+; CHECK-LABEL: f14:
+; CHECK-DAG: vfchsb [[REG1:%v[0-9]+]], %v28, %v26
+; CHECK-DAG: vfchesb [[REG2:%v[0-9]+]], %v26, %v28
+; CHECK: vno %v24, [[REG1]], [[REG2]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"uno",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test oeq selects.
+define <4 x float> @f15(<4 x float> %val1, <4 x float> %val2,
+                        <4 x float> %val3, <4 x float> %val4) #0 {
+; CHECK-LABEL: f15:
+; CHECK: vfcesb [[REG:%v[0-9]+]], %v24, %v26
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <4 x i1> %cmp, <4 x float> %val3, <4 x float> %val4
+  ret <4 x float> %ret
+}
+
+; Test one selects.
+define <4 x float> @f16(<4 x float> %val1, <4 x float> %val2,
+                        <4 x float> %val3, <4 x float> %val4) #0 {
+; CHECK-LABEL: f16:
+; CHECK-DAG: vfchsb [[REG1:%v[0-9]+]], %v26, %v24
+; CHECK-DAG: vfchsb [[REG2:%v[0-9]+]], %v24, %v26
+; CHECK: vo [[REG:%v[0-9]+]], [[REG1]], [[REG2]]
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"one",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <4 x i1> %cmp, <4 x float> %val3, <4 x float> %val4
+  ret <4 x float> %ret
+}
+
+; Test ogt selects.
+define <4 x float> @f17(<4 x float> %val1, <4 x float> %val2,
+                        <4 x float> %val3, <4 x float> %val4) #0 {
+; CHECK-LABEL: f17:
+; CHECK: vfchsb [[REG:%v[0-9]+]], %v24, %v26
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"ogt",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <4 x i1> %cmp, <4 x float> %val3, <4 x float> %val4
+  ret <4 x float> %ret
+}
+
+; Test oge selects.
+define <4 x float> @f18(<4 x float> %val1, <4 x float> %val2,
+                        <4 x float> %val3, <4 x float> %val4) #0 {
+; CHECK-LABEL: f18:
+; CHECK: vfchesb [[REG:%v[0-9]+]], %v24, %v26
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"oge",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <4 x i1> %cmp, <4 x float> %val3, <4 x float> %val4
+  ret <4 x float> %ret
+}
+
+; Test ole selects.
+define <4 x float> @f19(<4 x float> %val1, <4 x float> %val2,
+                        <4 x float> %val3, <4 x float> %val4) #0 {
+; CHECK-LABEL: f19:
+; CHECK: vfchesb [[REG:%v[0-9]+]], %v26, %v24
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"ole",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <4 x i1> %cmp, <4 x float> %val3, <4 x float> %val4
+  ret <4 x float> %ret
+}
+
+; Test olt selects.
+define <4 x float> @f20(<4 x float> %val1, <4 x float> %val2,
+                        <4 x float> %val3, <4 x float> %val4) #0 {
+; CHECK-LABEL: f20:
+; CHECK: vfchsb [[REG:%v[0-9]+]], %v26, %v24
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <4 x i1> %cmp, <4 x float> %val3, <4 x float> %val4
+  ret <4 x float> %ret
+}
+
+; Test ueq selects.
+define <4 x float> @f21(<4 x float> %val1, <4 x float> %val2,
+                        <4 x float> %val3, <4 x float> %val4) #0 {
+; CHECK-LABEL: f21:
+; CHECK-DAG: vfchsb [[REG1:%v[0-9]+]], %v26, %v24
+; CHECK-DAG: vfchsb [[REG2:%v[0-9]+]], %v24, %v26
+; CHECK: vo [[REG:%v[0-9]+]], [[REG1]], [[REG2]]
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"ueq",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <4 x i1> %cmp, <4 x float> %val3, <4 x float> %val4
+  ret <4 x float> %ret
+}
+
+; Test une selects.
+define <4 x float> @f22(<4 x float> %val1, <4 x float> %val2,
+                        <4 x float> %val3, <4 x float> %val4) #0 {
+; CHECK-LABEL: f22:
+; CHECK: vfcesb [[REG:%v[0-9]+]], %v24, %v26
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"une",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <4 x i1> %cmp, <4 x float> %val3, <4 x float> %val4
+  ret <4 x float> %ret
+}
+
+; Test ugt selects.
+define <4 x float> @f23(<4 x float> %val1, <4 x float> %val2,
+                        <4 x float> %val3, <4 x float> %val4) #0 {
+; CHECK-LABEL: f23:
+; CHECK: vfchesb [[REG:%v[0-9]+]], %v26, %v24
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"ugt",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <4 x i1> %cmp, <4 x float> %val3, <4 x float> %val4
+  ret <4 x float> %ret
+}
+
+; Test uge selects.
+define <4 x float> @f24(<4 x float> %val1, <4 x float> %val2,
+                        <4 x float> %val3, <4 x float> %val4) #0 {
+; CHECK-LABEL: f24:
+; CHECK: vfchsb [[REG:%v[0-9]+]], %v26, %v24
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"uge",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <4 x i1> %cmp, <4 x float> %val3, <4 x float> %val4
+  ret <4 x float> %ret
+}
+
+; Test ule selects.
+define <4 x float> @f25(<4 x float> %val1, <4 x float> %val2,
+                        <4 x float> %val3, <4 x float> %val4) #0 {
+; CHECK-LABEL: f25:
+; CHECK: vfchsb [[REG:%v[0-9]+]], %v24, %v26
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"ule",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <4 x i1> %cmp, <4 x float> %val3, <4 x float> %val4
+  ret <4 x float> %ret
+}
+
+; Test ult selects.
+define <4 x float> @f26(<4 x float> %val1, <4 x float> %val2,
+                        <4 x float> %val3, <4 x float> %val4) #0 {
+; CHECK-LABEL: f26:
+; CHECK: vfchesb [[REG:%v[0-9]+]], %v24, %v26
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"ult",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <4 x i1> %cmp, <4 x float> %val3, <4 x float> %val4
+  ret <4 x float> %ret
+}
+
+; Test ord selects.
+define <4 x float> @f27(<4 x float> %val1, <4 x float> %val2,
+                        <4 x float> %val3, <4 x float> %val4) #0 {
+; CHECK-LABEL: f27:
+; CHECK-DAG: vfchsb [[REG1:%v[0-9]+]], %v26, %v24
+; CHECK-DAG: vfchesb [[REG2:%v[0-9]+]], %v24, %v26
+; CHECK: vo [[REG:%v[0-9]+]], [[REG1]], [[REG2]]
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"ord",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <4 x i1> %cmp, <4 x float> %val3, <4 x float> %val4
+  ret <4 x float> %ret
+}
+
+; Test uno selects.
+define <4 x float> @f28(<4 x float> %val1, <4 x float> %val2,
+                        <4 x float> %val3, <4 x float> %val4) #0 {
+; CHECK-LABEL: f28:
+; CHECK-DAG: vfchsb [[REG1:%v[0-9]+]], %v26, %v24
+; CHECK-DAG: vfchesb [[REG2:%v[0-9]+]], %v24, %v26
+; CHECK: vo [[REG:%v[0-9]+]], [[REG1]], [[REG2]]
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"uno",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <4 x i1> %cmp, <4 x float> %val3, <4 x float> %val4
+  ret <4 x float> %ret
+}
+
+; Test an f32 comparison that uses vector registers.
+define i64 @f29(i64 %a, i64 %b, float %f1, <4 x float> %vec) #0 {
+; CHECK-LABEL: f29:
+; CHECK: wfcsb %f0, %v24
+; CHECK-NEXT: locgrne %r2, %r3
+; CHECK: br %r14
+  %f2 = extractelement <4 x float> %vec, i32 0
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f32(
+                                               float %f1, float %f2,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+attributes #0 = { strictfp }
+
+declare <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(<4 x float>, <4 x float>, metadata, metadata)
+declare i1 @llvm.experimental.constrained.fcmp.f32(float, float, metadata, metadata)
+
diff --git a/llvm/test/CodeGen/SystemZ/vec-strict-cmps-01.ll b/llvm/test/CodeGen/SystemZ/vec-strict-cmps-01.ll
new file mode 100644
index 000000000..e55f01c3f
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/vec-strict-cmps-01.ll
@@ -0,0 +1,442 @@
+; Test signaling f32 and v4f32 comparisons on z14.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z14 | FileCheck %s
+
+; Test oeq.
+define <4 x i32> @f1(<4 x i32> %dummy, <4 x float> %val1, <4 x float> %val2) #0 {
+; CHECK-LABEL: f1:
+; CHECK: vfkesb %v24, %v26, %v28
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmps.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test one.
+define <4 x i32> @f2(<4 x i32> %dummy, <4 x float> %val1, <4 x float> %val2) #0 {
+; CHECK-LABEL: f2:
+; CHECK-DAG: vfkhsb [[REG1:%v[0-9]+]], %v28, %v26
+; CHECK-DAG: vfkhsb [[REG2:%v[0-9]+]], %v26, %v28
+; CHECK: vo %v24, [[REG1]], [[REG2]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmps.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"one",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test ogt.
+define <4 x i32> @f3(<4 x i32> %dummy, <4 x float> %val1, <4 x float> %val2) #0 {
+; CHECK-LABEL: f3:
+; CHECK: vfkhsb %v24, %v26, %v28
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmps.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"ogt",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test oge.
+define <4 x i32> @f4(<4 x i32> %dummy, <4 x float> %val1, <4 x float> %val2) #0 {
+; CHECK-LABEL: f4:
+; CHECK: vfkhesb %v24, %v26, %v28
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmps.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"oge",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test ole.
+define <4 x i32> @f5(<4 x i32> %dummy, <4 x float> %val1, <4 x float> %val2) #0 {
+; CHECK-LABEL: f5:
+; CHECK: vfkhesb %v24, %v28, %v26
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmps.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"ole",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test olt.
+define <4 x i32> @f6(<4 x i32> %dummy, <4 x float> %val1, <4 x float> %val2) #0 {
+; CHECK-LABEL: f6:
+; CHECK: vfkhsb %v24, %v28, %v26
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmps.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test ueq.
+define <4 x i32> @f7(<4 x i32> %dummy, <4 x float> %val1, <4 x float> %val2) #0 {
+; CHECK-LABEL: f7:
+; CHECK-DAG: vfkhsb [[REG1:%v[0-9]+]], %v28, %v26
+; CHECK-DAG: vfkhsb [[REG2:%v[0-9]+]], %v26, %v28
+; CHECK: vno %v24, [[REG1]], [[REG2]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmps.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"ueq",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test une.
+define <4 x i32> @f8(<4 x i32> %dummy, <4 x float> %val1, <4 x float> %val2) #0 {
+; CHECK-LABEL: f8:
+; CHECK: vfkesb [[REG:%v[0-9]+]], %v26, %v28
+; CHECK-NEXT: vno %v24, [[REG]], [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmps.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"une",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test ugt.
+define <4 x i32> @f9(<4 x i32> %dummy, <4 x float> %val1, <4 x float> %val2) #0 {
+; CHECK-LABEL: f9:
+; CHECK: vfkhesb [[REG:%v[0-9]+]], %v28, %v26
+; CHECK-NEXT: vno %v24, [[REG]], [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmps.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"ugt",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test uge.
+define <4 x i32> @f10(<4 x i32> %dummy, <4 x float> %val1,
+                      <4 x float> %val2) #0 {
+; CHECK-LABEL: f10:
+; CHECK: vfkhsb [[REG:%v[0-9]+]], %v28, %v26
+; CHECK-NEXT: vno %v24, [[REG]], [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmps.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"uge",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test ule.
+define <4 x i32> @f11(<4 x i32> %dummy, <4 x float> %val1,
+                      <4 x float> %val2) #0 {
+; CHECK-LABEL: f11:
+; CHECK: vfkhsb [[REG:%v[0-9]+]], %v26, %v28
+; CHECK-NEXT: vno %v24, [[REG]], [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmps.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"ule",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test ult.
+define <4 x i32> @f12(<4 x i32> %dummy, <4 x float> %val1,
+                      <4 x float> %val2) #0 {
+; CHECK-LABEL: f12:
+; CHECK: vfkhesb [[REG:%v[0-9]+]], %v26, %v28
+; CHECK-NEXT: vno %v24, [[REG]], [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmps.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"ult",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test ord.
+define <4 x i32> @f13(<4 x i32> %dummy, <4 x float> %val1,
+                      <4 x float> %val2) #0 {
+; CHECK-LABEL: f13:
+; CHECK-DAG: vfkhsb [[REG1:%v[0-9]+]], %v28, %v26
+; CHECK-DAG: vfkhesb [[REG2:%v[0-9]+]], %v26, %v28
+; CHECK: vo %v24, [[REG1]], [[REG2]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmps.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"ord",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test uno.
+define <4 x i32> @f14(<4 x i32> %dummy, <4 x float> %val1,
+                      <4 x float> %val2) #0 {
+; CHECK-LABEL: f14:
+; CHECK-DAG: vfkhsb [[REG1:%v[0-9]+]], %v28, %v26
+; CHECK-DAG: vfkhesb [[REG2:%v[0-9]+]], %v26, %v28
+; CHECK: vno %v24, [[REG1]], [[REG2]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmps.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"uno",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test oeq selects.
+define <4 x float> @f15(<4 x float> %val1, <4 x float> %val2,
+                        <4 x float> %val3, <4 x float> %val4) #0 {
+; CHECK-LABEL: f15:
+; CHECK: vfkesb [[REG:%v[0-9]+]], %v24, %v26
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmps.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <4 x i1> %cmp, <4 x float> %val3, <4 x float> %val4
+  ret <4 x float> %ret
+}
+
+; Test one selects.
+define <4 x float> @f16(<4 x float> %val1, <4 x float> %val2,
+                        <4 x float> %val3, <4 x float> %val4) #0 {
+; CHECK-LABEL: f16:
+; CHECK-DAG: vfkhsb [[REG1:%v[0-9]+]], %v26, %v24
+; CHECK-DAG: vfkhsb [[REG2:%v[0-9]+]], %v24, %v26
+; CHECK: vo [[REG:%v[0-9]+]], [[REG1]], [[REG2]]
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmps.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"one",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <4 x i1> %cmp, <4 x float> %val3, <4 x float> %val4
+  ret <4 x float> %ret
+}
+
+; Test ogt selects.
+define <4 x float> @f17(<4 x float> %val1, <4 x float> %val2,
+                        <4 x float> %val3, <4 x float> %val4) #0 {
+; CHECK-LABEL: f17:
+; CHECK: vfkhsb [[REG:%v[0-9]+]], %v24, %v26
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmps.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"ogt",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <4 x i1> %cmp, <4 x float> %val3, <4 x float> %val4
+  ret <4 x float> %ret
+}
+
+; Test oge selects.
+define <4 x float> @f18(<4 x float> %val1, <4 x float> %val2,
+                        <4 x float> %val3, <4 x float> %val4) #0 {
+; CHECK-LABEL: f18:
+; CHECK: vfkhesb [[REG:%v[0-9]+]], %v24, %v26
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmps.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"oge",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <4 x i1> %cmp, <4 x float> %val3, <4 x float> %val4
+  ret <4 x float> %ret
+}
+
+; Test ole selects.
+define <4 x float> @f19(<4 x float> %val1, <4 x float> %val2,
+                        <4 x float> %val3, <4 x float> %val4) #0 {
+; CHECK-LABEL: f19:
+; CHECK: vfkhesb [[REG:%v[0-9]+]], %v26, %v24
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmps.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"ole",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <4 x i1> %cmp, <4 x float> %val3, <4 x float> %val4
+  ret <4 x float> %ret
+}
+
+; Test olt selects.
+define <4 x float> @f20(<4 x float> %val1, <4 x float> %val2,
+                        <4 x float> %val3, <4 x float> %val4) #0 {
+; CHECK-LABEL: f20:
+; CHECK: vfkhsb [[REG:%v[0-9]+]], %v26, %v24
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmps.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <4 x i1> %cmp, <4 x float> %val3, <4 x float> %val4
+  ret <4 x float> %ret
+}
+
+; Test ueq selects.
+define <4 x float> @f21(<4 x float> %val1, <4 x float> %val2,
+                        <4 x float> %val3, <4 x float> %val4) #0 {
+; CHECK-LABEL: f21:
+; CHECK-DAG: vfkhsb [[REG1:%v[0-9]+]], %v26, %v24
+; CHECK-DAG: vfkhsb [[REG2:%v[0-9]+]], %v24, %v26
+; CHECK: vo [[REG:%v[0-9]+]], [[REG1]], [[REG2]]
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmps.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"ueq",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <4 x i1> %cmp, <4 x float> %val3, <4 x float> %val4
+  ret <4 x float> %ret
+}
+
+; Test une selects.
+define <4 x float> @f22(<4 x float> %val1, <4 x float> %val2,
+                        <4 x float> %val3, <4 x float> %val4) #0 {
+; CHECK-LABEL: f22:
+; CHECK: vfkesb [[REG:%v[0-9]+]], %v24, %v26
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmps.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"une",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <4 x i1> %cmp, <4 x float> %val3, <4 x float> %val4
+  ret <4 x float> %ret
+}
+
+; Test ugt selects.
+define <4 x float> @f23(<4 x float> %val1, <4 x float> %val2,
+                        <4 x float> %val3, <4 x float> %val4) #0 {
+; CHECK-LABEL: f23:
+; CHECK: vfkhesb [[REG:%v[0-9]+]], %v26, %v24
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmps.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"ugt",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <4 x i1> %cmp, <4 x float> %val3, <4 x float> %val4
+  ret <4 x float> %ret
+}
+
+; Test uge selects.
+define <4 x float> @f24(<4 x float> %val1, <4 x float> %val2,
+                        <4 x float> %val3, <4 x float> %val4) #0 {
+; CHECK-LABEL: f24:
+; CHECK: vfkhsb [[REG:%v[0-9]+]], %v26, %v24
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmps.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"uge",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <4 x i1> %cmp, <4 x float> %val3, <4 x float> %val4
+  ret <4 x float> %ret
+}
+
+; Test ule selects.
+define <4 x float> @f25(<4 x float> %val1, <4 x float> %val2,
+                        <4 x float> %val3, <4 x float> %val4) #0 {
+; CHECK-LABEL: f25:
+; CHECK: vfkhsb [[REG:%v[0-9]+]], %v24, %v26
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmps.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"ule",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <4 x i1> %cmp, <4 x float> %val3, <4 x float> %val4
+  ret <4 x float> %ret
+}
+
+; Test ult selects.
+define <4 x float> @f26(<4 x float> %val1, <4 x float> %val2,
+                        <4 x float> %val3, <4 x float> %val4) #0 {
+; CHECK-LABEL: f26:
+; CHECK: vfkhesb [[REG:%v[0-9]+]], %v24, %v26
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmps.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"ult",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <4 x i1> %cmp, <4 x float> %val3, <4 x float> %val4
+  ret <4 x float> %ret
+}
+
+; Test ord selects.
+define <4 x float> @f27(<4 x float> %val1, <4 x float> %val2,
+                        <4 x float> %val3, <4 x float> %val4) #0 {
+; CHECK-LABEL: f27:
+; CHECK-DAG: vfkhsb [[REG1:%v[0-9]+]], %v26, %v24
+; CHECK-DAG: vfkhesb [[REG2:%v[0-9]+]], %v24, %v26
+; CHECK: vo [[REG:%v[0-9]+]], [[REG1]], [[REG2]]
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmps.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"ord",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <4 x i1> %cmp, <4 x float> %val3, <4 x float> %val4
+  ret <4 x float> %ret
+}
+
+; Test uno selects.
+define <4 x float> @f28(<4 x float> %val1, <4 x float> %val2,
+                        <4 x float> %val3, <4 x float> %val4) #0 {
+; CHECK-LABEL: f28:
+; CHECK-DAG: vfkhsb [[REG1:%v[0-9]+]], %v26, %v24
+; CHECK-DAG: vfkhesb [[REG2:%v[0-9]+]], %v24, %v26
+; CHECK: vo [[REG:%v[0-9]+]], [[REG1]], [[REG2]]
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmps.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"uno",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <4 x i1> %cmp, <4 x float> %val3, <4 x float> %val4
+  ret <4 x float> %ret
+}
+
+; Test an f32 comparison that uses vector registers.
+define i64 @f29(i64 %a, i64 %b, float %f1, <4 x float> %vec) #0 {
+; CHECK-LABEL: f29:
+; CHECK: wfksb %f0, %v24
+; CHECK-NEXT: locgrne %r2, %r3
+; CHECK: br %r14
+  %f2 = extractelement <4 x float> %vec, i32 0
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f32(
+                                               float %f1, float %f2,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+attributes #0 = { strictfp }
+
+declare <4 x i1> @llvm.experimental.constrained.fcmps.v4f32(<4 x float>, <4 x float>, metadata, metadata)
+declare i1 @llvm.experimental.constrained.fcmps.f32(float, float, metadata, metadata)
+
diff --git a/llvm/test/CodeGen/SystemZ/vec-strict-cmps-02.ll b/llvm/test/CodeGen/SystemZ/vec-strict-cmps-02.ll
new file mode 100644
index 000000000..0a09fdf25
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/vec-strict-cmps-02.ll
@@ -0,0 +1,442 @@
+; Test f64 and v2f64 signaling comparisons on z14.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z14 | FileCheck %s
+
+; Test oeq.
+define <2 x i64> @f1(<2 x i64> %dummy, <2 x double> %val1, <2 x double> %val2) #0 {
+; CHECK-LABEL: f1:
+; CHECK: vfkedb %v24, %v26, %v28
+; CHECK-NEXT: br %r14
+  %cmp = call <2 x i1> @llvm.experimental.constrained.fcmps.v2f64(
+                                               <2 x double> %val1, <2 x double> %val2,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <2 x i1> %cmp to <2 x i64>
+  ret <2 x i64> %ret
+}
+
+; Test one.
+define <2 x i64> @f2(<2 x i64> %dummy, <2 x double> %val1, <2 x double> %val2) #0 {
+; CHECK-LABEL: f2:
+; CHECK-DAG: vfkhdb [[REG1:%v[0-9]+]], %v28, %v26
+; CHECK-DAG: vfkhdb [[REG2:%v[0-9]+]], %v26, %v28
+; CHECK: vo %v24, [[REG1]], [[REG2]]
+; CHECK-NEXT: br %r14
+  %cmp = call <2 x i1> @llvm.experimental.constrained.fcmps.v2f64(
+                                               <2 x double> %val1, <2 x double> %val2,
+                                               metadata !"one",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <2 x i1> %cmp to <2 x i64>
+  ret <2 x i64> %ret
+}
+
+; Test ogt.
+define <2 x i64> @f3(<2 x i64> %dummy, <2 x double> %val1, <2 x double> %val2) #0 {
+; CHECK-LABEL: f3:
+; CHECK: vfkhdb %v24, %v26, %v28
+; CHECK-NEXT: br %r14
+  %cmp = call <2 x i1> @llvm.experimental.constrained.fcmps.v2f64(
+                                               <2 x double> %val1, <2 x double> %val2,
+                                               metadata !"ogt",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <2 x i1> %cmp to <2 x i64>
+  ret <2 x i64> %ret
+}
+
+; Test oge.
+define <2 x i64> @f4(<2 x i64> %dummy, <2 x double> %val1, <2 x double> %val2) #0 {
+; CHECK-LABEL: f4:
+; CHECK: vfkhedb %v24, %v26, %v28
+; CHECK-NEXT: br %r14
+  %cmp = call <2 x i1> @llvm.experimental.constrained.fcmps.v2f64(
+                                               <2 x double> %val1, <2 x double> %val2,
+                                               metadata !"oge",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <2 x i1> %cmp to <2 x i64>
+  ret <2 x i64> %ret
+}
+
+; Test ole.
+define <2 x i64> @f5(<2 x i64> %dummy, <2 x double> %val1, <2 x double> %val2) #0 {
+; CHECK-LABEL: f5:
+; CHECK: vfkhedb %v24, %v28, %v26
+; CHECK-NEXT: br %r14
+  %cmp = call <2 x i1> @llvm.experimental.constrained.fcmps.v2f64(
+                                               <2 x double> %val1, <2 x double> %val2,
+                                               metadata !"ole",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <2 x i1> %cmp to <2 x i64>
+  ret <2 x i64> %ret
+}
+
+; Test olt.
+define <2 x i64> @f6(<2 x i64> %dummy, <2 x double> %val1, <2 x double> %val2) #0 {
+; CHECK-LABEL: f6:
+; CHECK: vfkhdb %v24, %v28, %v26
+; CHECK-NEXT: br %r14
+  %cmp = call <2 x i1> @llvm.experimental.constrained.fcmps.v2f64(
+                                               <2 x double> %val1, <2 x double> %val2,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <2 x i1> %cmp to <2 x i64>
+  ret <2 x i64> %ret
+}
+
+; Test ueq.
+define <2 x i64> @f7(<2 x i64> %dummy, <2 x double> %val1, <2 x double> %val2) #0 {
+; CHECK-LABEL: f7:
+; CHECK-DAG: vfkhdb [[REG1:%v[0-9]+]], %v28, %v26
+; CHECK-DAG: vfkhdb [[REG2:%v[0-9]+]], %v26, %v28
+; CHECK: vno %v24, [[REG1]], [[REG2]]
+; CHECK-NEXT: br %r14
+  %cmp = call <2 x i1> @llvm.experimental.constrained.fcmps.v2f64(
+                                               <2 x double> %val1, <2 x double> %val2,
+                                               metadata !"ueq",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <2 x i1> %cmp to <2 x i64>
+  ret <2 x i64> %ret
+}
+
+; Test une.
+define <2 x i64> @f8(<2 x i64> %dummy, <2 x double> %val1, <2 x double> %val2) #0 {
+; CHECK-LABEL: f8:
+; CHECK: vfkedb [[REG:%v[0-9]+]], %v26, %v28
+; CHECK-NEXT: vno %v24, [[REG]], [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <2 x i1> @llvm.experimental.constrained.fcmps.v2f64(
+                                               <2 x double> %val1, <2 x double> %val2,
+                                               metadata !"une",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <2 x i1> %cmp to <2 x i64>
+  ret <2 x i64> %ret
+}
+
+; Test ugt.
+define <2 x i64> @f9(<2 x i64> %dummy, <2 x double> %val1, <2 x double> %val2) #0 {
+; CHECK-LABEL: f9:
+; CHECK: vfkhedb [[REG:%v[0-9]+]], %v28, %v26
+; CHECK-NEXT: vno %v24, [[REG]], [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <2 x i1> @llvm.experimental.constrained.fcmps.v2f64(
+                                               <2 x double> %val1, <2 x double> %val2,
+                                               metadata !"ugt",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <2 x i1> %cmp to <2 x i64>
+  ret <2 x i64> %ret
+}
+
+; Test uge.
+define <2 x i64> @f10(<2 x i64> %dummy, <2 x double> %val1,
+                      <2 x double> %val2) #0 {
+; CHECK-LABEL: f10:
+; CHECK: vfkhdb [[REG:%v[0-9]+]], %v28, %v26
+; CHECK-NEXT: vno %v24, [[REG]], [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <2 x i1> @llvm.experimental.constrained.fcmps.v2f64(
+                                               <2 x double> %val1, <2 x double> %val2,
+                                               metadata !"uge",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <2 x i1> %cmp to <2 x i64>
+  ret <2 x i64> %ret
+}
+
+; Test ule.
+define <2 x i64> @f11(<2 x i64> %dummy, <2 x double> %val1,
+                      <2 x double> %val2) #0 {
+; CHECK-LABEL: f11:
+; CHECK: vfkhdb [[REG:%v[0-9]+]], %v26, %v28
+; CHECK-NEXT: vno %v24, [[REG]], [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <2 x i1> @llvm.experimental.constrained.fcmps.v2f64(
+                                               <2 x double> %val1, <2 x double> %val2,
+                                               metadata !"ule",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <2 x i1> %cmp to <2 x i64>
+  ret <2 x i64> %ret
+}
+
+; Test ult.
+define <2 x i64> @f12(<2 x i64> %dummy, <2 x double> %val1,
+                      <2 x double> %val2) #0 {
+; CHECK-LABEL: f12:
+; CHECK: vfkhedb [[REG:%v[0-9]+]], %v26, %v28
+; CHECK-NEXT: vno %v24, [[REG]], [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <2 x i1> @llvm.experimental.constrained.fcmps.v2f64(
+                                               <2 x double> %val1, <2 x double> %val2,
+                                               metadata !"ult",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <2 x i1> %cmp to <2 x i64>
+  ret <2 x i64> %ret
+}
+
+; Test ord.
+define <2 x i64> @f13(<2 x i64> %dummy, <2 x double> %val1,
+                      <2 x double> %val2) #0 {
+; CHECK-LABEL: f13:
+; CHECK-DAG: vfkhdb [[REG1:%v[0-9]+]], %v28, %v26
+; CHECK-DAG: vfkhedb [[REG2:%v[0-9]+]], %v26, %v28
+; CHECK: vo %v24, [[REG1]], [[REG2]]
+; CHECK-NEXT: br %r14
+  %cmp = call <2 x i1> @llvm.experimental.constrained.fcmps.v2f64(
+                                               <2 x double> %val1, <2 x double> %val2,
+                                               metadata !"ord",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <2 x i1> %cmp to <2 x i64>
+  ret <2 x i64> %ret
+}
+
+; Test uno.
+define <2 x i64> @f14(<2 x i64> %dummy, <2 x double> %val1,
+                      <2 x double> %val2) #0 {
+; CHECK-LABEL: f14:
+; CHECK-DAG: vfkhdb [[REG1:%v[0-9]+]], %v28, %v26
+; CHECK-DAG: vfkhedb [[REG2:%v[0-9]+]], %v26, %v28
+; CHECK: vno %v24, [[REG1]], [[REG2]]
+; CHECK-NEXT: br %r14
+  %cmp = call <2 x i1> @llvm.experimental.constrained.fcmps.v2f64(
+                                               <2 x double> %val1, <2 x double> %val2,
+                                               metadata !"uno",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <2 x i1> %cmp to <2 x i64>
+  ret <2 x i64> %ret
+}
+
+; Test oeq selects.
+define <2 x double> @f15(<2 x double> %val1, <2 x double> %val2,
+                         <2 x double> %val3, <2 x double> %val4) #0 {
+; CHECK-LABEL: f15:
+; CHECK: vfkedb [[REG:%v[0-9]+]], %v24, %v26
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <2 x i1> @llvm.experimental.constrained.fcmps.v2f64(
+                                               <2 x double> %val1, <2 x double> %val2,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <2 x i1> %cmp, <2 x double> %val3, <2 x double> %val4
+  ret <2 x double> %ret
+}
+
+; Test one selects.
+define <2 x double> @f16(<2 x double> %val1, <2 x double> %val2,
+                         <2 x double> %val3, <2 x double> %val4) #0 {
+; CHECK-LABEL: f16:
+; CHECK-DAG: vfkhdb [[REG1:%v[0-9]+]], %v26, %v24
+; CHECK-DAG: vfkhdb [[REG2:%v[0-9]+]], %v24, %v26
+; CHECK: vo [[REG:%v[0-9]+]], [[REG1]], [[REG2]]
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <2 x i1> @llvm.experimental.constrained.fcmps.v2f64(
+                                               <2 x double> %val1, <2 x double> %val2,
+                                               metadata !"one",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <2 x i1> %cmp, <2 x double> %val3, <2 x double> %val4
+  ret <2 x double> %ret
+}
+
+; Test ogt selects.
+define <2 x double> @f17(<2 x double> %val1, <2 x double> %val2,
+                         <2 x double> %val3, <2 x double> %val4) #0 {
+; CHECK-LABEL: f17:
+; CHECK: vfkhdb [[REG:%v[0-9]+]], %v24, %v26
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <2 x i1> @llvm.experimental.constrained.fcmps.v2f64(
+                                               <2 x double> %val1, <2 x double> %val2,
+                                               metadata !"ogt",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <2 x i1> %cmp, <2 x double> %val3, <2 x double> %val4
+  ret <2 x double> %ret
+}
+
+; Test oge selects.
+define <2 x double> @f18(<2 x double> %val1, <2 x double> %val2,
+                         <2 x double> %val3, <2 x double> %val4) #0 {
+; CHECK-LABEL: f18:
+; CHECK: vfkhedb [[REG:%v[0-9]+]], %v24, %v26
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <2 x i1> @llvm.experimental.constrained.fcmps.v2f64(
+                                               <2 x double> %val1, <2 x double> %val2,
+                                               metadata !"oge",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <2 x i1> %cmp, <2 x double> %val3, <2 x double> %val4
+  ret <2 x double> %ret
+}
+
+; Test ole selects.
+define <2 x double> @f19(<2 x double> %val1, <2 x double> %val2,
+                         <2 x double> %val3, <2 x double> %val4) #0 {
+; CHECK-LABEL: f19:
+; CHECK: vfkhedb [[REG:%v[0-9]+]], %v26, %v24
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <2 x i1> @llvm.experimental.constrained.fcmps.v2f64(
+                                               <2 x double> %val1, <2 x double> %val2,
+                                               metadata !"ole",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <2 x i1> %cmp, <2 x double> %val3, <2 x double> %val4
+  ret <2 x double> %ret
+}
+
+; Test olt selects.
+define <2 x double> @f20(<2 x double> %val1, <2 x double> %val2,
+                         <2 x double> %val3, <2 x double> %val4) #0 {
+; CHECK-LABEL: f20:
+; CHECK: vfkhdb [[REG:%v[0-9]+]], %v26, %v24
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <2 x i1> @llvm.experimental.constrained.fcmps.v2f64(
+                                               <2 x double> %val1, <2 x double> %val2,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <2 x i1> %cmp, <2 x double> %val3, <2 x double> %val4
+  ret <2 x double> %ret
+}
+
+; Test ueq selects.
+define <2 x double> @f21(<2 x double> %val1, <2 x double> %val2,
+                         <2 x double> %val3, <2 x double> %val4) #0 {
+; CHECK-LABEL: f21:
+; CHECK-DAG: vfkhdb [[REG1:%v[0-9]+]], %v26, %v24
+; CHECK-DAG: vfkhdb [[REG2:%v[0-9]+]], %v24, %v26
+; CHECK: vo [[REG:%v[0-9]+]], [[REG1]], [[REG2]]
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <2 x i1> @llvm.experimental.constrained.fcmps.v2f64(
+                                               <2 x double> %val1, <2 x double> %val2,
+                                               metadata !"ueq",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <2 x i1> %cmp, <2 x double> %val3, <2 x double> %val4
+  ret <2 x double> %ret
+}
+
+; Test une selects.
+define <2 x double> @f22(<2 x double> %val1, <2 x double> %val2,
+                         <2 x double> %val3, <2 x double> %val4) #0 {
+; CHECK-LABEL: f22:
+; CHECK: vfkedb [[REG:%v[0-9]+]], %v24, %v26
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <2 x i1> @llvm.experimental.constrained.fcmps.v2f64(
+                                               <2 x double> %val1, <2 x double> %val2,
+                                               metadata !"une",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <2 x i1> %cmp, <2 x double> %val3, <2 x double> %val4
+  ret <2 x double> %ret
+}
+
+; Test ugt selects.
+define <2 x double> @f23(<2 x double> %val1, <2 x double> %val2,
+                         <2 x double> %val3, <2 x double> %val4) #0 {
+; CHECK-LABEL: f23:
+; CHECK: vfkhedb [[REG:%v[0-9]+]], %v26, %v24
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <2 x i1> @llvm.experimental.constrained.fcmps.v2f64(
+                                               <2 x double> %val1, <2 x double> %val2,
+                                               metadata !"ugt",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <2 x i1> %cmp, <2 x double> %val3, <2 x double> %val4
+  ret <2 x double> %ret
+}
+
+; Test uge selects.
+define <2 x double> @f24(<2 x double> %val1, <2 x double> %val2,
+                         <2 x double> %val3, <2 x double> %val4) #0 {
+; CHECK-LABEL: f24:
+; CHECK: vfkhdb [[REG:%v[0-9]+]], %v26, %v24
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <2 x i1> @llvm.experimental.constrained.fcmps.v2f64(
+                                               <2 x double> %val1, <2 x double> %val2,
+                                               metadata !"uge",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <2 x i1> %cmp, <2 x double> %val3, <2 x double> %val4
+  ret <2 x double> %ret
+}
+
+; Test ule selects.
+define <2 x double> @f25(<2 x double> %val1, <2 x double> %val2,
+                         <2 x double> %val3, <2 x double> %val4) #0 {
+; CHECK-LABEL: f25:
+; CHECK: vfkhdb [[REG:%v[0-9]+]], %v24, %v26
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <2 x i1> @llvm.experimental.constrained.fcmps.v2f64(
+                                               <2 x double> %val1, <2 x double> %val2,
+                                               metadata !"ule",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <2 x i1> %cmp, <2 x double> %val3, <2 x double> %val4
+  ret <2 x double> %ret
+}
+
+; Test ult selects.
+define <2 x double> @f26(<2 x double> %val1, <2 x double> %val2,
+                         <2 x double> %val3, <2 x double> %val4) #0 {
+; CHECK-LABEL: f26:
+; CHECK: vfkhedb [[REG:%v[0-9]+]], %v24, %v26
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <2 x i1> @llvm.experimental.constrained.fcmps.v2f64(
+                                               <2 x double> %val1, <2 x double> %val2,
+                                               metadata !"ult",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <2 x i1> %cmp, <2 x double> %val3, <2 x double> %val4
+  ret <2 x double> %ret
+}
+
+; Test ord selects.
+define <2 x double> @f27(<2 x double> %val1, <2 x double> %val2,
+                         <2 x double> %val3, <2 x double> %val4) #0 {
+; CHECK-LABEL: f27:
+; CHECK-DAG: vfkhdb [[REG1:%v[0-9]+]], %v26, %v24
+; CHECK-DAG: vfkhedb [[REG2:%v[0-9]+]], %v24, %v26
+; CHECK: vo [[REG:%v[0-9]+]], [[REG1]], [[REG2]]
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <2 x i1> @llvm.experimental.constrained.fcmps.v2f64(
+                                               <2 x double> %val1, <2 x double> %val2,
+                                               metadata !"ord",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <2 x i1> %cmp, <2 x double> %val3, <2 x double> %val4
+  ret <2 x double> %ret
+}
+
+; Test uno selects.
+define <2 x double> @f28(<2 x double> %val1, <2 x double> %val2,
+                         <2 x double> %val3, <2 x double> %val4) #0 {
+; CHECK-LABEL: f28:
+; CHECK-DAG: vfkhdb [[REG1:%v[0-9]+]], %v26, %v24
+; CHECK-DAG: vfkhedb [[REG2:%v[0-9]+]], %v24, %v26
+; CHECK: vo [[REG:%v[0-9]+]], [[REG1]], [[REG2]]
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = call <2 x i1> @llvm.experimental.constrained.fcmps.v2f64(
+                                               <2 x double> %val1, <2 x double> %val2,
+                                               metadata !"uno",
+                                               metadata !"fpexcept.strict") #0
+  %ret = select <2 x i1> %cmp, <2 x double> %val3, <2 x double> %val4
+  ret <2 x double> %ret
+}
+
+; Test an f64 comparison that uses vector registers.
+define i64 @f29(i64 %a, i64 %b, double %f1, <2 x double> %vec) #0 {
+; CHECK-LABEL: f29:
+; CHECK: wfkdb %f0, %v24
+; CHECK-NEXT: locgrne %r2, %r3
+; CHECK: br %r14
+  %f2 = extractelement <2 x double> %vec, i32 0
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f64(
+                                               double %f1, double %f2,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+attributes #0 = { strictfp }
+
+declare <2 x i1> @llvm.experimental.constrained.fcmps.v2f64(<2 x double>, <2 x double>, metadata, metadata)
+declare i1 @llvm.experimental.constrained.fcmps.f64(double, double, metadata, metadata)
+
diff --git a/llvm/test/CodeGen/SystemZ/vec-strict-cmps-03.ll b/llvm/test/CodeGen/SystemZ/vec-strict-cmps-03.ll
new file mode 100644
index 000000000..ca4d0768b
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/vec-strict-cmps-03.ll
@@ -0,0 +1,56 @@
+; Test signaling vector floating-point comparisons on z13.
+; Note that these must be scalarized as we do not have native instructions.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test v4f32.
+define <4 x i32> @f1(<4 x float> %val1, <4 x float> %val2) #0 {
+; CHECK-LABEL: f1:
+; CHECK: kebr
+; CHECK: kebr
+; CHECK: kebr
+; CHECK: kebr
+; CHECK: br %r14
+  %cmp = call <4 x i1> @llvm.experimental.constrained.fcmps.v4f32(
+                                               <4 x float> %val1, <4 x float> %val2,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test v2f64.
+define <2 x i64> @f2(<2 x i64> %dummy, <2 x double> %val1, <2 x double> %val2) #0 {
+; CHECK-LABEL: f2:
+; CHECK: {{kdbr|wfkdb}}
+; CHECK: {{kdbr|wfkdb}}
+; CHECK: br %r14
+  %cmp = call <2 x i1> @llvm.experimental.constrained.fcmps.v2f64(
+                                               <2 x double> %val1, <2 x double> %val2,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %ret = sext <2 x i1> %cmp to <2 x i64>
+  ret <2 x i64> %ret
+}
+
+; Test an f64 comparison that uses vector registers.
+define i64 @f3(i64 %a, i64 %b, double %f1, <2 x double> %vec) #0 {
+; CHECK-LABEL: f3:
+; CHECK: wfkdb %f0, %v24
+; CHECK-NEXT: locgrne %r2, %r3
+; CHECK: br %r14
+  %f2 = extractelement <2 x double> %vec, i32 0
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f64(
+                                               double %f1, double %f2,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+attributes #0 = { strictfp }
+
+declare <4 x i1> @llvm.experimental.constrained.fcmps.v4f32(<4 x float>, <4 x float>, metadata, metadata)
+declare <2 x i1> @llvm.experimental.constrained.fcmps.v2f64(<2 x double>, <2 x double>, metadata, metadata)
+declare i1 @llvm.experimental.constrained.fcmps.f64(double, double, metadata, metadata)
+
diff --git a/llvm/test/CodeGen/SystemZ/vec-strict-conv-01.ll b/llvm/test/CodeGen/SystemZ/vec-strict-conv-01.ll
new file mode 100644
index 000000000..3b56b27a4
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/vec-strict-conv-01.ll
@@ -0,0 +1,130 @@
+; Test strict conversions between integer and float elements.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+declare <2 x i64> @llvm.experimental.constrained.fptoui.v2i64.v2f64(<2 x double>, metadata)
+declare <2 x i64> @llvm.experimental.constrained.fptosi.v2i64.v2f64(<2 x double>, metadata)
+declare <2 x double> @llvm.experimental.constrained.uitofp.v2f64.v2i64(<2 x i64>, metadata, metadata)
+declare <2 x double> @llvm.experimental.constrained.sitofp.v2f64.v2i64(<2 x i64>, metadata, metadata)
+
+declare <2 x i32> @llvm.experimental.constrained.fptoui.v2i32.v2f64(<2 x double>, metadata)
+declare <2 x i32> @llvm.experimental.constrained.fptosi.v2i32.v2f64(<2 x double>, metadata)
+declare <2 x double> @llvm.experimental.constrained.uitofp.v2f64.v2i32(<2 x i32>, metadata, metadata)
+declare <2 x double> @llvm.experimental.constrained.sitofp.v2f64.v2i32(<2 x i32>, metadata, metadata)
+
+declare <2 x i64> @llvm.experimental.constrained.fptoui.v2i64.v2f32(<2 x float>, metadata)
+declare <2 x i64> @llvm.experimental.constrained.fptosi.v2i64.v2f32(<2 x float>, metadata)
+declare <2 x float> @llvm.experimental.constrained.uitofp.v2f32.v2i64(<2 x i64>, metadata, metadata)
+declare <2 x float> @llvm.experimental.constrained.sitofp.v2f32.v2i64(<2 x i64>, metadata, metadata)
+
+; Test conversion of f64s to signed i64s.
+define <2 x i64> @f1(<2 x double> %doubles) #0 {
+; CHECK-LABEL: f1:
+; CHECK: vcgdb %v24, %v24, 0, 5
+; CHECK: br %r14
+  %dwords = call <2 x i64> @llvm.experimental.constrained.fptosi.v2i64.v2f64(<2 x double> %doubles,
+                                               metadata !"fpexcept.strict") #0
+  ret <2 x i64> %dwords
+}
+
+; Test conversion of f64s to unsigned i64s.
+define <2 x i64> @f2(<2 x double> %doubles) #0 {
+; CHECK-LABEL: f2:
+; CHECK: vclgdb %v24, %v24, 0, 5
+; CHECK: br %r14
+  %dwords = call <2 x i64> @llvm.experimental.constrained.fptoui.v2i64.v2f64(<2 x double> %doubles,
+                                               metadata !"fpexcept.strict") #0
+  ret <2 x i64> %dwords
+}
+
+; Test conversion of signed i64s to f64s.
+define <2 x double> @f3(<2 x i64> %dwords) #0 {
+; CHECK-LABEL: f3:
+; CHECK: vcdgb %v24, %v24, 0, 0
+; CHECK: br %r14
+  %doubles = call <2 x double> @llvm.experimental.constrained.sitofp.v2f64.v2i64(<2 x i64> %dwords,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret <2 x double> %doubles
+}
+
+; Test conversion of unsigned i64s to f64s.
+define <2 x double> @f4(<2 x i64> %dwords) #0 {
+; CHECK-LABEL: f4:
+; CHECK: vcdlgb %v24, %v24, 0, 0
+; CHECK: br %r14
+  %doubles = call <2 x double> @llvm.experimental.constrained.uitofp.v2f64.v2i64(<2 x i64> %dwords,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret <2 x double> %doubles
+}
+
+; Test conversion of f64s to signed i32s, which must compile.
+define void @f5(<2 x double> %doubles, <2 x i32> *%ptr) #0 {
+  %words = call <2 x i32> @llvm.experimental.constrained.fptosi.v2i32.v2f64(<2 x double> %doubles,
+                                               metadata !"fpexcept.strict") #0
+  store <2 x i32> %words, <2 x i32> *%ptr
+  ret void
+}
+
+; Test conversion of f64s to unsigned i32s, which must compile.
+define void @f6(<2 x double> %doubles, <2 x i32> *%ptr) #0 {
+  %words = call <2 x i32> @llvm.experimental.constrained.fptoui.v2i32.v2f64(<2 x double> %doubles,
+                                               metadata !"fpexcept.strict") #0
+  store <2 x i32> %words, <2 x i32> *%ptr
+  ret void
+}
+
+; Test conversion of signed i32s to f64s, which must compile.
+define <2 x double> @f7(<2 x i32> *%ptr) #0 {
+  %words = load <2 x i32>, <2 x i32> *%ptr
+  %doubles = call <2 x double> @llvm.experimental.constrained.sitofp.v2f64.v2i32(<2 x i32> %words,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret <2 x double> %doubles
+}
+
+; Test conversion of unsigned i32s to f64s, which must compile.
+define <2 x double> @f8(<2 x i32> *%ptr) #0 {
+  %words = load <2 x i32>, <2 x i32> *%ptr
+  %doubles = call <2 x double> @llvm.experimental.constrained.uitofp.v2f64.v2i32(<2 x i32> %words,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret <2 x double> %doubles
+}
+
+; Test conversion of f32s to signed i64s, which must compile.
+define <2 x i64> @f9(<2 x float> *%ptr) #0 {
+  %floats = load <2 x float>, <2 x float> *%ptr
+  %dwords = call <2 x i64> @llvm.experimental.constrained.fptosi.v2i64.v2f32(<2 x float> %floats,
+                                               metadata !"fpexcept.strict") #0
+  ret <2 x i64> %dwords
+}
+
+; Test conversion of f32s to unsigned i64s, which must compile.
+define <2 x i64> @f10(<2 x float> *%ptr) #0 {
+  %floats = load <2 x float>, <2 x float> *%ptr
+  %dwords = call <2 x i64> @llvm.experimental.constrained.fptoui.v2i64.v2f32(<2 x float> %floats,
+                                               metadata !"fpexcept.strict") #0
+  ret <2 x i64> %dwords
+}
+
+; Test conversion of signed i64s to f32, which must compile.
+define void @f11(<2 x i64> %dwords, <2 x float> *%ptr) #0 {
+  %floats = call <2 x float> @llvm.experimental.constrained.sitofp.v2f32.v2i64(<2 x i64> %dwords,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  store <2 x float> %floats, <2 x float> *%ptr
+  ret void
+}
+
+; Test conversion of unsigned i64s to f32, which must compile.
+define void @f12(<2 x i64> %dwords, <2 x float> *%ptr) #0 {
+  %floats = call <2 x float> @llvm.experimental.constrained.uitofp.v2f32.v2i64(<2 x i64> %dwords,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  store <2 x float> %floats, <2 x float> *%ptr
+  ret void
+}
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/SystemZ/vec-strict-conv-02.ll b/llvm/test/CodeGen/SystemZ/vec-strict-conv-02.ll
new file mode 100644
index 000000000..d4590a57d
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/vec-strict-conv-02.ll
@@ -0,0 +1,61 @@
+; Test conversions between different-sized float elements.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+declare <2 x float> @llvm.experimental.constrained.fptrunc.v2f32.v2f64(<2 x double>, metadata, metadata)
+declare float @llvm.experimental.constrained.fptrunc.f32.f64(double, metadata, metadata)
+
+declare <2 x double> @llvm.experimental.constrained.fpext.v2f64.v2f32(<2 x float>, metadata)
+declare double @llvm.experimental.constrained.fpext.f64.f32(float, metadata)
+
+; Test cases where both elements of a v2f64 are converted to f32s.
+define void @f1(<2 x double> %val, <2 x float> *%ptr) {
+; CHECK-LABEL: f1:
+; CHECK: vledb {{%v[0-9]+}}, %v24, 0, 0
+; CHECK: br %r14
+  %res = call <2 x float> @llvm.experimental.constrained.fptrunc.v2f32.v2f64(
+                                               <2 x double> %val,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  store <2 x float> %res, <2 x float> *%ptr
+  ret void
+}
+
+; Test conversion of an f64 in a vector register to an f32.
+define float @f2(<2 x double> %vec) #0 {
+; CHECK-LABEL: f2:
+; CHECK: wledb %f0, %v24, 0, 0
+; CHECK: br %r14
+  %scalar = extractelement <2 x double> %vec, i32 0
+  %ret = call float @llvm.experimental.constrained.fptrunc.f32.f64(
+                                               double %scalar,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret float %ret
+}
+
+; Test cases where even elements of a v4f32 are converted to f64s.
+define <2 x double> @f3(<4 x float> %vec) {
+; CHECK-LABEL: f3:
+; CHECK: vldeb %v24, {{%v[0-9]+}}
+; CHECK: br %r14
+  %shuffle = shufflevector <4 x float> %vec, <4 x float> undef, <2 x i32> <i32 0, i32 2>
+  %res = call <2 x double> @llvm.experimental.constrained.fpext.v2f64.v2f32(
+                                               <2 x float> %shuffle,
+                                               metadata !"fpexcept.strict") #0
+  ret <2 x double> %res
+}
+
+; Test conversion of an f32 in a vector register to an f64.
+define double @f4(<4 x float> %vec) {
+; CHECK-LABEL: f4:
+; CHECK: wldeb %f0, %v24
+; CHECK: br %r14
+  %scalar = extractelement <4 x float> %vec, i32 0
+  %ret = call double @llvm.experimental.constrained.fpext.f64.f32(
+                                               float %scalar,
+                                               metadata !"fpexcept.strict") #0
+  ret double %ret
+}
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/SystemZ/vec-strict-conv-03.ll b/llvm/test/CodeGen/SystemZ/vec-strict-conv-03.ll
new file mode 100644
index 000000000..fd3825f04
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/vec-strict-conv-03.ll
@@ -0,0 +1,52 @@
+; Test strict conversions between integer and float elements on z15.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z15 | FileCheck %s
+
+declare <4 x i32> @llvm.experimental.constrained.fptoui.v4i32.v4f32(<4 x float>, metadata)
+declare <4 x i32> @llvm.experimental.constrained.fptosi.v4i32.v4f32(<4 x float>, metadata)
+declare <4 x float> @llvm.experimental.constrained.uitofp.v4f32.v4i32(<4 x i32>, metadata, metadata)
+declare <4 x float> @llvm.experimental.constrained.sitofp.v4f32.v4i32(<4 x i32>, metadata, metadata)
+
+; Test conversion of f32s to signed i32s.
+define <4 x i32> @f1(<4 x float> %floats) #0 {
+; CHECK-LABEL: f1:
+; CHECK: vcfeb %v24, %v24, 0, 5
+; CHECK: br %r14
+  %words = call <4 x i32> @llvm.experimental.constrained.fptosi.v4i32.v4f32(<4 x float> %floats,
+                                               metadata !"fpexcept.strict") #0
+  ret <4 x i32> %words
+}
+
+; Test conversion of f32s to unsigned i32s.
+define <4 x i32> @f2(<4 x float> %floats) #0 {
+; CHECK-LABEL: f2:
+; CHECK: vclfeb %v24, %v24, 0, 5
+; CHECK: br %r14
+  %words = call <4 x i32> @llvm.experimental.constrained.fptoui.v4i32.v4f32(<4 x float> %floats,
+                                               metadata !"fpexcept.strict") #0
+  ret <4 x i32> %words
+}
+
+; Test conversion of signed i32s to f32s.
+define <4 x float> @f3(<4 x i32> %dwords) #0 {
+; CHECK-LABEL: f3:
+; CHECK: vcefb %v24, %v24, 0, 0
+; CHECK: br %r14
+  %floats = call <4 x float> @llvm.experimental.constrained.sitofp.v4f32.v4i32(<4 x i32> %dwords,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret <4 x float> %floats
+}
+
+; Test conversion of unsigned i32s to f32s.
+define <4 x float> @f4(<4 x i32> %dwords) #0 {
+; CHECK-LABEL: f4:
+; CHECK: vcelfb %v24, %v24, 0, 0
+; CHECK: br %r14
+  %floats = call <4 x float> @llvm.experimental.constrained.uitofp.v4f32.v4i32(<4 x i32> %dwords,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret <4 x float> %floats
+}
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/SystemZ/vec-strict-div-01.ll b/llvm/test/CodeGen/SystemZ/vec-strict-div-01.ll
index ec5477625..bb15c0d40 100644
--- a/llvm/test/CodeGen/SystemZ/vec-strict-div-01.ll
+++ b/llvm/test/CodeGen/SystemZ/vec-strict-div-01.ll
@@ -7,19 +7,19 @@ declare <2 x double> @llvm.experimental.constrained.fdiv.v2f64(<2 x double>, <2
 
 ; Test a v2f64 division.
 define <2 x double> @f5(<2 x double> %dummy, <2 x double> %val1,
-                        <2 x double> %val2) {
+                        <2 x double> %val2) #0 {
 ; CHECK-LABEL: f5:
 ; CHECK: vfddb %v24, %v26, %v28
 ; CHECK: br %r14
   %ret = call <2 x double> @llvm.experimental.constrained.fdiv.v2f64(
                         <2 x double> %val1, <2 x double> %val2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret <2 x double> %ret
 }
 
 ; Test an f64 division that uses vector registers.
-define double @f6(<2 x double> %val1, <2 x double> %val2) {
+define double @f6(<2 x double> %val1, <2 x double> %val2) #0 {
 ; CHECK-LABEL: f6:
 ; CHECK: wfddb %f0, %v24, %v26
 ; CHECK: br %r14
@@ -28,6 +28,8 @@ define double @f6(<2 x double> %val1, <2 x double> %val2) {
   %ret = call double @llvm.experimental.constrained.fdiv.f64(
                         double %scalar1, double %scalar2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %ret
 }
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/SystemZ/vec-strict-div-02.ll b/llvm/test/CodeGen/SystemZ/vec-strict-div-02.ll
index 0fce46295..b791d67b3 100644
--- a/llvm/test/CodeGen/SystemZ/vec-strict-div-02.ll
+++ b/llvm/test/CodeGen/SystemZ/vec-strict-div-02.ll
@@ -7,19 +7,19 @@ declare <4 x float> @llvm.experimental.constrained.fdiv.v4f32(<4 x float>, <4 x
 
 ; Test a v4f32 division.
 define <4 x float> @f1(<4 x float> %dummy, <4 x float> %val1,
-                       <4 x float> %val2) {
+                       <4 x float> %val2) #0 {
 ; CHECK-LABEL: f1:
 ; CHECK: vfdsb %v24, %v26, %v28
 ; CHECK: br %r14
   %ret = call <4 x float> @llvm.experimental.constrained.fdiv.v4f32(
                         <4 x float> %val1, <4 x float> %val2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret <4 x float> %ret
 }
 
 ; Test an f32 division that uses vector registers.
-define float @f2(<4 x float> %val1, <4 x float> %val2) {
+define float @f2(<4 x float> %val1, <4 x float> %val2) #0 {
 ; CHECK-LABEL: f2:
 ; CHECK: wfdsb %f0, %v24, %v26
 ; CHECK: br %r14
@@ -28,6 +28,8 @@ define float @f2(<4 x float> %val1, <4 x float> %val2) {
   %ret = call float @llvm.experimental.constrained.fdiv.f32(
                         float %scalar1, float %scalar2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %ret
 }
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/SystemZ/vec-strict-max-01.ll b/llvm/test/CodeGen/SystemZ/vec-strict-max-01.ll
index 82e7c32c0..a5a312e80 100644
--- a/llvm/test/CodeGen/SystemZ/vec-strict-max-01.ll
+++ b/llvm/test/CodeGen/SystemZ/vec-strict-max-01.ll
@@ -2,66 +2,67 @@
 ;
 ; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z14 | FileCheck %s
 
-declare double @llvm.experimental.constrained.maxnum.f64(double, double, metadata, metadata)
-declare <2 x double> @llvm.experimental.constrained.maxnum.v2f64(<2 x double>, <2 x double>, metadata, metadata)
+declare double @llvm.experimental.constrained.maxnum.f64(double, double, metadata)
+declare <2 x double> @llvm.experimental.constrained.maxnum.v2f64(<2 x double>, <2 x double>, metadata)
+declare double @llvm.experimental.constrained.maximum.f64(double, double, metadata)
+declare <2 x double> @llvm.experimental.constrained.maximum.v2f64(<2 x double>, <2 x double>, metadata)
 
-declare float @llvm.experimental.constrained.maxnum.f32(float, float, metadata, metadata)
-declare <4 x float> @llvm.experimental.constrained.maxnum.v4f32(<4 x float>, <4 x float>, metadata, metadata)
+declare float @llvm.experimental.constrained.maxnum.f32(float, float, metadata)
+declare <4 x float> @llvm.experimental.constrained.maxnum.v4f32(<4 x float>, <4 x float>, metadata)
+declare float @llvm.experimental.constrained.maximum.f32(float, float, metadata)
+declare <4 x float> @llvm.experimental.constrained.maximum.v4f32(<4 x float>, <4 x float>, metadata)
 
-declare fp128 @llvm.experimental.constrained.maxnum.f128(fp128, fp128, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.maxnum.f128(fp128, fp128, metadata)
+declare fp128 @llvm.experimental.constrained.maximum.f128(fp128, fp128, metadata)
 
 ; Test the f64 maxnum intrinsic.
-define double @f1(double %dummy, double %val1, double %val2) {
+define double @f1(double %dummy, double %val1, double %val2) #0 {
 ; CHECK-LABEL: f1:
 ; CHECK: wfmaxdb %f0, %f2, %f4, 4
 ; CHECK: br %r14
   %ret = call double @llvm.experimental.constrained.maxnum.f64(
                         double %val1, double %val2,
-                        metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %ret
 }
 
 ; Test the v2f64 maxnum intrinsic.
 define <2 x double> @f2(<2 x double> %dummy, <2 x double> %val1,
-                        <2 x double> %val2) {
+                        <2 x double> %val2) #0 {
 ; CHECK-LABEL: f2:
 ; CHECK: vfmaxdb %v24, %v26, %v28, 4
 ; CHECK: br %r14
   %ret = call <2 x double> @llvm.experimental.constrained.maxnum.v2f64(
                         <2 x double> %val1, <2 x double> %val2,
-                        metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret <2 x double> %ret
 }
 
 ; Test the f32 maxnum intrinsic.
-define float @f3(float %dummy, float %val1, float %val2) {
+define float @f3(float %dummy, float %val1, float %val2) #0 {
 ; CHECK-LABEL: f3:
 ; CHECK: wfmaxsb %f0, %f2, %f4, 4
 ; CHECK: br %r14
   %ret = call float @llvm.experimental.constrained.maxnum.f32(
                         float %val1, float %val2,
-                        metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %ret
 }
 
 ; Test the v4f32 maxnum intrinsic.
 define <4 x float> @f4(<4 x float> %dummy, <4 x float> %val1,
-                       <4 x float> %val2) {
+                       <4 x float> %val2) #0 {
 ; CHECK-LABEL: f4:
 ; CHECK: vfmaxsb %v24, %v26, %v28, 4
 ; CHECK: br %r14
   %ret = call <4 x float> @llvm.experimental.constrained.maxnum.v4f32(
                         <4 x float> %val1, <4 x float> %val2,
-                        metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret <4 x float> %ret
 }
 
 ; Test the f128 maxnum intrinsic.
-define void @f5(fp128 *%ptr1, fp128 *%ptr2, fp128 *%dst) {
+define void @f5(fp128 *%ptr1, fp128 *%ptr2, fp128 *%dst) #0 {
 ; CHECK-LABEL: f5:
 ; CHECK-DAG: vl [[REG1:%v[0-9]+]], 0(%r2)
 ; CHECK-DAG: vl [[REG2:%v[0-9]+]], 0(%r3)
@@ -72,9 +73,72 @@ define void @f5(fp128 *%ptr1, fp128 *%ptr2, fp128 *%dst) {
   %val2 = load fp128, fp128* %ptr2
   %res = call fp128 @llvm.experimental.constrained.maxnum.f128(
                         fp128 %val1, fp128 %val2,
-                        metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   store fp128 %res, fp128* %dst
   ret void
 }
 
+; Test the f64 maximum intrinsic.
+define double @f11(double %dummy, double %val1, double %val2) #0 {
+; CHECK-LABEL: f11:
+; CHECK: wfmaxdb %f0, %f2, %f4, 1
+; CHECK: br %r14
+  %ret = call double @llvm.experimental.constrained.maximum.f64(
+                        double %val1, double %val2,
+                        metadata !"fpexcept.strict") #0
+  ret double %ret
+}
+
+; Test the v2f64 maximum intrinsic.
+define <2 x double> @f12(<2 x double> %dummy, <2 x double> %val1,
+                         <2 x double> %val2) #0 {
+; CHECK-LABEL: f12:
+; CHECK: vfmaxdb %v24, %v26, %v28, 1
+; CHECK: br %r14
+  %ret = call <2 x double> @llvm.experimental.constrained.maximum.v2f64(
+                        <2 x double> %val1, <2 x double> %val2,
+                        metadata !"fpexcept.strict") #0
+  ret <2 x double> %ret
+}
+
+; Test the f32 maximum intrinsic.
+define float @f13(float %dummy, float %val1, float %val2) #0 {
+; CHECK-LABEL: f13:
+; CHECK: wfmaxsb %f0, %f2, %f4, 1
+; CHECK: br %r14
+  %ret = call float @llvm.experimental.constrained.maximum.f32(
+                        float %val1, float %val2,
+                        metadata !"fpexcept.strict") #0
+  ret float %ret
+}
+
+; Test the v4f32 maximum intrinsic.
+define <4 x float> @f14(<4 x float> %dummy, <4 x float> %val1,
+                        <4 x float> %val2) #0 {
+; CHECK-LABEL: f14:
+; CHECK: vfmaxsb %v24, %v26, %v28, 1
+; CHECK: br %r14
+  %ret = call <4 x float> @llvm.experimental.constrained.maximum.v4f32(
+                        <4 x float> %val1, <4 x float> %val2,
+                        metadata !"fpexcept.strict") #0
+  ret <4 x float> %ret
+}
+
+; Test the f128 maximum intrinsic.
+define void @f15(fp128 *%ptr1, fp128 *%ptr2, fp128 *%dst) #0 {
+; CHECK-LABEL: f15:
+; CHECK-DAG: vl [[REG1:%v[0-9]+]], 0(%r2)
+; CHECK-DAG: vl [[REG2:%v[0-9]+]], 0(%r3)
+; CHECK: wfmaxxb [[RES:%v[0-9]+]], [[REG1]], [[REG2]], 1
+; CHECK: vst [[RES]], 0(%r4)
+; CHECK: br %r14
+  %val1 = load fp128, fp128* %ptr1
+  %val2 = load fp128, fp128* %ptr2
+  %res = call fp128 @llvm.experimental.constrained.maximum.f128(
+                        fp128 %val1, fp128 %val2,
+                        metadata !"fpexcept.strict") #0
+  store fp128 %res, fp128* %dst
+  ret void
+}
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/SystemZ/vec-strict-min-01.ll b/llvm/test/CodeGen/SystemZ/vec-strict-min-01.ll
index 641b9c334..5b66b1ec0 100644
--- a/llvm/test/CodeGen/SystemZ/vec-strict-min-01.ll
+++ b/llvm/test/CodeGen/SystemZ/vec-strict-min-01.ll
@@ -2,66 +2,67 @@
 ;
 ; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z14 | FileCheck %s
 
-declare double @llvm.experimental.constrained.minnum.f64(double, double, metadata, metadata)
-declare <2 x double> @llvm.experimental.constrained.minnum.v2f64(<2 x double>, <2 x double>, metadata, metadata)
+declare double @llvm.experimental.constrained.minnum.f64(double, double, metadata)
+declare <2 x double> @llvm.experimental.constrained.minnum.v2f64(<2 x double>, <2 x double>, metadata)
+declare double @llvm.experimental.constrained.minimum.f64(double, double, metadata)
+declare <2 x double> @llvm.experimental.constrained.minimum.v2f64(<2 x double>, <2 x double>, metadata)
 
-declare float @llvm.experimental.constrained.minnum.f32(float, float, metadata, metadata)
-declare <4 x float> @llvm.experimental.constrained.minnum.v4f32(<4 x float>, <4 x float>, metadata, metadata)
+declare float @llvm.experimental.constrained.minnum.f32(float, float, metadata)
+declare <4 x float> @llvm.experimental.constrained.minnum.v4f32(<4 x float>, <4 x float>, metadata)
+declare float @llvm.experimental.constrained.minimum.f32(float, float, metadata)
+declare <4 x float> @llvm.experimental.constrained.minimum.v4f32(<4 x float>, <4 x float>, metadata)
 
-declare fp128 @llvm.experimental.constrained.minnum.f128(fp128, fp128, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.minnum.f128(fp128, fp128, metadata)
+declare fp128 @llvm.experimental.constrained.minimum.f128(fp128, fp128, metadata)
 
 ; Test the f64 minnum intrinsic.
-define double @f1(double %dummy, double %val1, double %val2) {
+define double @f1(double %dummy, double %val1, double %val2) #0 {
 ; CHECK-LABEL: f1:
 ; CHECK: wfmindb %f0, %f2, %f4, 4
 ; CHECK: br %r14
   %ret = call double @llvm.experimental.constrained.minnum.f64(
                         double %val1, double %val2,
-                        metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %ret
 }
 
 ; Test the v2f64 minnum intrinsic.
 define <2 x double> @f2(<2 x double> %dummy, <2 x double> %val1,
-                        <2 x double> %val2) {
+                        <2 x double> %val2) #0 {
 ; CHECK-LABEL: f2:
 ; CHECK: vfmindb %v24, %v26, %v28, 4
 ; CHECK: br %r14
   %ret = call <2 x double> @llvm.experimental.constrained.minnum.v2f64(
                         <2 x double> %val1, <2 x double> %val2,
-                        metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret <2 x double> %ret
 }
 
 ; Test the f32 minnum intrinsic.
-define float @f3(float %dummy, float %val1, float %val2) {
+define float @f3(float %dummy, float %val1, float %val2) #0 {
 ; CHECK-LABEL: f3:
 ; CHECK: wfminsb %f0, %f2, %f4, 4
 ; CHECK: br %r14
   %ret = call float @llvm.experimental.constrained.minnum.f32(
                         float %val1, float %val2,
-                        metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %ret
 }
 
 ; Test the v4f32 minnum intrinsic.
 define <4 x float> @f4(<4 x float> %dummy, <4 x float> %val1,
-                       <4 x float> %val2) {
+                       <4 x float> %val2) #0 {
 ; CHECK-LABEL: f4:
 ; CHECK: vfminsb %v24, %v26, %v28, 4
 ; CHECK: br %r14
   %ret = call <4 x float> @llvm.experimental.constrained.minnum.v4f32(
                         <4 x float> %val1, <4 x float> %val2,
-                        metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret <4 x float> %ret
 }
 
 ; Test the f128 minnum intrinsic.
-define void @f5(fp128 *%ptr1, fp128 *%ptr2, fp128 *%dst) {
+define void @f5(fp128 *%ptr1, fp128 *%ptr2, fp128 *%dst) #0 {
 ; CHECK-LABEL: f5:
 ; CHECK-DAG: vl [[REG1:%v[0-9]+]], 0(%r2)
 ; CHECK-DAG: vl [[REG2:%v[0-9]+]], 0(%r3)
@@ -72,9 +73,72 @@ define void @f5(fp128 *%ptr1, fp128 *%ptr2, fp128 *%dst) {
   %val2 = load fp128, fp128* %ptr2
   %res = call fp128 @llvm.experimental.constrained.minnum.f128(
                         fp128 %val1, fp128 %val2,
-                        metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   store fp128 %res, fp128* %dst
   ret void
 }
 
+; Test the f64 minimum intrinsic.
+define double @f11(double %dummy, double %val1, double %val2) #0 {
+; CHECK-LABEL: f11:
+; CHECK: wfmindb %f0, %f2, %f4, 1
+; CHECK: br %r14
+  %ret = call double @llvm.experimental.constrained.minimum.f64(
+                        double %val1, double %val2,
+                        metadata !"fpexcept.strict") #0
+  ret double %ret
+}
+
+; Test the v2f64 minimum intrinsic.
+define <2 x double> @f12(<2 x double> %dummy, <2 x double> %val1,
+                         <2 x double> %val2) #0 {
+; CHECK-LABEL: f12:
+; CHECK: vfmindb %v24, %v26, %v28, 1
+; CHECK: br %r14
+  %ret = call <2 x double> @llvm.experimental.constrained.minimum.v2f64(
+                        <2 x double> %val1, <2 x double> %val2,
+                        metadata !"fpexcept.strict") #0
+  ret <2 x double> %ret
+}
+
+; Test the f32 minimum intrinsic.
+define float @f13(float %dummy, float %val1, float %val2) #0 {
+; CHECK-LABEL: f13:
+; CHECK: wfminsb %f0, %f2, %f4, 1
+; CHECK: br %r14
+  %ret = call float @llvm.experimental.constrained.minimum.f32(
+                        float %val1, float %val2,
+                        metadata !"fpexcept.strict") #0
+  ret float %ret
+}
+
+; Test the v4f32 minimum intrinsic.
+define <4 x float> @f14(<4 x float> %dummy, <4 x float> %val1,
+                        <4 x float> %val2) #0 {
+; CHECK-LABEL: f14:
+; CHECK: vfminsb %v24, %v26, %v28, 1
+; CHECK: br %r14
+  %ret = call <4 x float> @llvm.experimental.constrained.minimum.v4f32(
+                        <4 x float> %val1, <4 x float> %val2,
+                        metadata !"fpexcept.strict") #0
+  ret <4 x float> %ret
+}
+
+; Test the f128 minimum intrinsic.
+define void @f15(fp128 *%ptr1, fp128 *%ptr2, fp128 *%dst) #0 {
+; CHECK-LABEL: f15:
+; CHECK-DAG: vl [[REG1:%v[0-9]+]], 0(%r2)
+; CHECK-DAG: vl [[REG2:%v[0-9]+]], 0(%r3)
+; CHECK: wfminxb [[RES:%v[0-9]+]], [[REG1]], [[REG2]], 1
+; CHECK: vst [[RES]], 0(%r4)
+; CHECK: br %r14
+  %val1 = load fp128, fp128* %ptr1
+  %val2 = load fp128, fp128* %ptr2
+  %res = call fp128 @llvm.experimental.constrained.minimum.f128(
+                        fp128 %val1, fp128 %val2,
+                        metadata !"fpexcept.strict") #0
+  store fp128 %res, fp128* %dst
+  ret void
+}
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/SystemZ/vec-strict-mul-01.ll b/llvm/test/CodeGen/SystemZ/vec-strict-mul-01.ll
index 0bb10188b..52c40f4cb 100644
--- a/llvm/test/CodeGen/SystemZ/vec-strict-mul-01.ll
+++ b/llvm/test/CodeGen/SystemZ/vec-strict-mul-01.ll
@@ -7,19 +7,19 @@ declare <2 x double> @llvm.experimental.constrained.fmul.v2f64(<2 x double>, <2
 
 ; Test a v2f64 multiplication.
 define <2 x double> @f5(<2 x double> %dummy, <2 x double> %val1,
-                        <2 x double> %val2) {
+                        <2 x double> %val2) #0 {
 ; CHECK-LABEL: f5:
 ; CHECK: vfmdb %v24, %v26, %v28
 ; CHECK: br %r14
   %ret = call <2 x double> @llvm.experimental.constrained.fmul.v2f64(
                         <2 x double> %val1, <2 x double> %val2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret <2 x double> %ret
 }
 
 ; Test an f64 multiplication that uses vector registers.
-define double @f6(<2 x double> %val1, <2 x double> %val2) {
+define double @f6(<2 x double> %val1, <2 x double> %val2) #0 {
 ; CHECK-LABEL: f6:
 ; CHECK: wfmdb %f0, %v24, %v26
 ; CHECK: br %r14
@@ -28,6 +28,8 @@ define double @f6(<2 x double> %val1, <2 x double> %val2) {
   %ret = call double @llvm.experimental.constrained.fmul.f64(
                         double %scalar1, double %scalar2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %ret
 }
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/SystemZ/vec-strict-mul-02.ll b/llvm/test/CodeGen/SystemZ/vec-strict-mul-02.ll
index 61447aa9b..d2341c323 100644
--- a/llvm/test/CodeGen/SystemZ/vec-strict-mul-02.ll
+++ b/llvm/test/CodeGen/SystemZ/vec-strict-mul-02.ll
@@ -6,7 +6,7 @@ declare <2 x double> @llvm.experimental.constrained.fma.v2f64(<2 x double>, <2 x
 
 ; Test a v2f64 multiply-and-add.
 define <2 x double> @f4(<2 x double> %dummy, <2 x double> %val1,
-                        <2 x double> %val2, <2 x double> %val3) {
+                        <2 x double> %val2, <2 x double> %val3) #0 {
 ; CHECK-LABEL: f4:
 ; CHECK: vfmadb %v24, %v26, %v28, %v30
 ; CHECK: br %r14
@@ -15,22 +15,24 @@ define <2 x double> @f4(<2 x double> %dummy, <2 x double> %val1,
                         <2 x double> %val2,
                         <2 x double> %val3,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret <2 x double> %ret
 }
 
 ; Test a v2f64 multiply-and-subtract.
 define <2 x double> @f5(<2 x double> %dummy, <2 x double> %val1,
-                        <2 x double> %val2, <2 x double> %val3) {
+                        <2 x double> %val2, <2 x double> %val3) #0 {
 ; CHECK-LABEL: f5:
 ; CHECK: vfmsdb %v24, %v26, %v28, %v30
 ; CHECK: br %r14
-  %negval3 = fsub <2 x double> <double -0.0, double -0.0>, %val3
+  %negval3 = fneg <2 x double> %val3
   %ret = call <2 x double> @llvm.experimental.constrained.fma.v2f64 (
                         <2 x double> %val1,
                         <2 x double> %val2,
                         <2 x double> %negval3,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret <2 x double> %ret
 }
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/SystemZ/vec-strict-mul-03.ll b/llvm/test/CodeGen/SystemZ/vec-strict-mul-03.ll
index a61d55913..a05ed27d4 100644
--- a/llvm/test/CodeGen/SystemZ/vec-strict-mul-03.ll
+++ b/llvm/test/CodeGen/SystemZ/vec-strict-mul-03.ll
@@ -7,19 +7,19 @@ declare <4 x float> @llvm.experimental.constrained.fmul.v4f32(<4 x float>, <4 x
 
 ; Test a v4f32 multiplication.
 define <4 x float> @f1(<4 x float> %dummy, <4 x float> %val1,
-                       <4 x float> %val2) {
+                       <4 x float> %val2) #0 {
 ; CHECK-LABEL: f1:
 ; CHECK: vfmsb %v24, %v26, %v28
 ; CHECK: br %r14
   %ret = call <4 x float> @llvm.experimental.constrained.fmul.v4f32(
                         <4 x float> %val1, <4 x float> %val2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret <4 x float> %ret
 }
 
 ; Test an f32 multiplication that uses vector registers.
-define float @f2(<4 x float> %val1, <4 x float> %val2) {
+define float @f2(<4 x float> %val1, <4 x float> %val2) #0 {
 ; CHECK-LABEL: f2:
 ; CHECK: wfmsb %f0, %v24, %v26
 ; CHECK: br %r14
@@ -28,6 +28,8 @@ define float @f2(<4 x float> %val1, <4 x float> %val2) {
   %ret = call float @llvm.experimental.constrained.fmul.f32(
                         float %scalar1, float %scalar2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %ret
 }
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/SystemZ/vec-strict-mul-04.ll b/llvm/test/CodeGen/SystemZ/vec-strict-mul-04.ll
index e24c38959..43656b7da 100644
--- a/llvm/test/CodeGen/SystemZ/vec-strict-mul-04.ll
+++ b/llvm/test/CodeGen/SystemZ/vec-strict-mul-04.ll
@@ -6,7 +6,7 @@ declare <4 x float> @llvm.experimental.constrained.fma.v4f32(<4 x float>, <4 x f
 
 ; Test a v4f32 multiply-and-add.
 define <4 x float> @f1(<4 x float> %dummy, <4 x float> %val1,
-                       <4 x float> %val2, <4 x float> %val3) {
+                       <4 x float> %val2, <4 x float> %val3) #0 {
 ; CHECK-LABEL: f1:
 ; CHECK: vfmasb %v24, %v26, %v28, %v30
 ; CHECK: br %r14
@@ -15,23 +15,24 @@ define <4 x float> @f1(<4 x float> %dummy, <4 x float> %val1,
                         <4 x float> %val2,
                         <4 x float> %val3,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret <4 x float> %ret
 }
 
 ; Test a v4f32 multiply-and-subtract.
 define <4 x float> @f2(<4 x float> %dummy, <4 x float> %val1,
-                       <4 x float> %val2, <4 x float> %val3) {
+                       <4 x float> %val2, <4 x float> %val3) #0 {
 ; CHECK-LABEL: f2:
 ; CHECK: vfmssb %v24, %v26, %v28, %v30
 ; CHECK: br %r14
-  %negval3 = fsub <4 x float> <float -0.0, float -0.0,
-                               float -0.0, float -0.0>, %val3
+  %negval3 = fneg <4 x float> %val3
   %ret = call <4 x float> @llvm.experimental.constrained.fma.v4f32 (
                         <4 x float> %val1,
                         <4 x float> %val2,
                         <4 x float> %negval3,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret <4 x float> %ret
 }
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/SystemZ/vec-strict-mul-05.ll b/llvm/test/CodeGen/SystemZ/vec-strict-mul-05.ll
index 9fdefc505..a367cbe0f 100644
--- a/llvm/test/CodeGen/SystemZ/vec-strict-mul-05.ll
+++ b/llvm/test/CodeGen/SystemZ/vec-strict-mul-05.ll
@@ -7,7 +7,7 @@ declare <4 x float> @llvm.experimental.constrained.fma.v4f32(<4 x float>, <4 x f
 
 ; Test a v2f64 negative multiply-and-add.
 define <2 x double> @f1(<2 x double> %dummy, <2 x double> %val1,
-                        <2 x double> %val2, <2 x double> %val3) {
+                        <2 x double> %val2, <2 x double> %val3) #0 {
 ; CHECK-LABEL: f1:
 ; CHECK: vfnmadb %v24, %v26, %v28, %v30
 ; CHECK: br %r14
@@ -16,31 +16,31 @@ define <2 x double> @f1(<2 x double> %dummy, <2 x double> %val1,
                         <2 x double> %val2,
                         <2 x double> %val3,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
-  %negret = fsub <2 x double> <double -0.0, double -0.0>, %ret
+                        metadata !"fpexcept.strict") #0
+  %negret = fneg <2 x double> %ret
   ret <2 x double> %negret
 }
 
 ; Test a v2f64 negative multiply-and-subtract.
 define <2 x double> @f2(<2 x double> %dummy, <2 x double> %val1,
-                        <2 x double> %val2, <2 x double> %val3) {
+                        <2 x double> %val2, <2 x double> %val3) #0 {
 ; CHECK-LABEL: f2:
 ; CHECK: vfnmsdb %v24, %v26, %v28, %v30
 ; CHECK: br %r14
-  %negval3 = fsub <2 x double> <double -0.0, double -0.0>, %val3
+  %negval3 = fneg <2 x double> %val3
   %ret = call <2 x double> @llvm.experimental.constrained.fma.v2f64 (
                         <2 x double> %val1,
                         <2 x double> %val2,
                         <2 x double> %negval3,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
-  %negret = fsub <2 x double> <double -0.0, double -0.0>, %ret
+                        metadata !"fpexcept.strict") #0
+  %negret = fneg <2 x double> %ret
   ret <2 x double> %negret
 }
 
 ; Test a v4f32 negative multiply-and-add.
 define <4 x float> @f3(<4 x float> %dummy, <4 x float> %val1,
-                       <4 x float> %val2, <4 x float> %val3) {
+                       <4 x float> %val2, <4 x float> %val3) #0 {
 ; CHECK-LABEL: f3:
 ; CHECK: vfnmasb %v24, %v26, %v28, %v30
 ; CHECK: br %r14
@@ -49,27 +49,26 @@ define <4 x float> @f3(<4 x float> %dummy, <4 x float> %val1,
                         <4 x float> %val2,
                         <4 x float> %val3,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
-  %negret = fsub <4 x float> <float -0.0, float -0.0,
-                              float -0.0, float -0.0>, %ret
+                        metadata !"fpexcept.strict") #0
+  %negret = fneg <4 x float> %ret
   ret <4 x float> %negret
 }
 
 ; Test a v4f32 negative multiply-and-subtract.
 define <4 x float> @f4(<4 x float> %dummy, <4 x float> %val1,
-                       <4 x float> %val2, <4 x float> %val3) {
+                       <4 x float> %val2, <4 x float> %val3) #0 {
 ; CHECK-LABEL: f4:
 ; CHECK: vfnmssb %v24, %v26, %v28, %v30
 ; CHECK: br %r14
-  %negval3 = fsub <4 x float> <float -0.0, float -0.0,
-                               float -0.0, float -0.0>, %val3
+  %negval3 = fneg <4 x float> %val3
   %ret = call <4 x float> @llvm.experimental.constrained.fma.v4f32 (
                         <4 x float> %val1,
                         <4 x float> %val2,
                         <4 x float> %negval3,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
-  %negret = fsub <4 x float> <float -0.0, float -0.0,
-                               float -0.0, float -0.0>, %ret
+                        metadata !"fpexcept.strict") #0
+  %negret = fneg <4 x float> %ret
   ret <4 x float> %negret
 }
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/SystemZ/vec-strict-round-01.ll b/llvm/test/CodeGen/SystemZ/vec-strict-round-01.ll
index e86357bb1..b82cb8082 100644
--- a/llvm/test/CodeGen/SystemZ/vec-strict-round-01.ll
+++ b/llvm/test/CodeGen/SystemZ/vec-strict-round-01.ll
@@ -4,84 +4,80 @@
 
 declare double @llvm.experimental.constrained.rint.f64(double, metadata, metadata)
 declare double @llvm.experimental.constrained.nearbyint.f64(double, metadata, metadata)
-declare double @llvm.experimental.constrained.floor.f64(double, metadata, metadata)
-declare double @llvm.experimental.constrained.ceil.f64(double, metadata, metadata)
-declare double @llvm.experimental.constrained.trunc.f64(double, metadata, metadata)
-declare double @llvm.experimental.constrained.round.f64(double, metadata, metadata)
+declare double @llvm.experimental.constrained.floor.f64(double, metadata)
+declare double @llvm.experimental.constrained.ceil.f64(double, metadata)
+declare double @llvm.experimental.constrained.trunc.f64(double, metadata)
+declare double @llvm.experimental.constrained.round.f64(double, metadata)
 declare <2 x double> @llvm.experimental.constrained.rint.v2f64(<2 x double>, metadata, metadata)
 declare <2 x double> @llvm.experimental.constrained.nearbyint.v2f64(<2 x double>, metadata, metadata)
-declare <2 x double> @llvm.experimental.constrained.floor.v2f64(<2 x double>, metadata, metadata)
-declare <2 x double> @llvm.experimental.constrained.ceil.v2f64(<2 x double>, metadata, metadata)
-declare <2 x double> @llvm.experimental.constrained.trunc.v2f64(<2 x double>, metadata, metadata)
-declare <2 x double> @llvm.experimental.constrained.round.v2f64(<2 x double>, metadata, metadata)
+declare <2 x double> @llvm.experimental.constrained.floor.v2f64(<2 x double>, metadata)
+declare <2 x double> @llvm.experimental.constrained.ceil.v2f64(<2 x double>, metadata)
+declare <2 x double> @llvm.experimental.constrained.trunc.v2f64(<2 x double>, metadata)
+declare <2 x double> @llvm.experimental.constrained.round.v2f64(<2 x double>, metadata)
 
-define <2 x double> @f1(<2 x double> %val) {
+define <2 x double> @f1(<2 x double> %val) #0 {
 ; CHECK-LABEL: f1:
 ; CHECK: vfidb %v24, %v24, 0, 0
 ; CHECK: br %r14
   %res = call <2 x double> @llvm.experimental.constrained.rint.v2f64(
                         <2 x double> %val,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret <2 x double> %res
 }
 
-define <2 x double> @f2(<2 x double> %val) {
+define <2 x double> @f2(<2 x double> %val) #0 {
 ; CHECK-LABEL: f2:
 ; CHECK: vfidb %v24, %v24, 4, 0
 ; CHECK: br %r14
   %res = call <2 x double> @llvm.experimental.constrained.nearbyint.v2f64(
                         <2 x double> %val,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret <2 x double> %res
 }
 
-define <2 x double> @f3(<2 x double> %val) {
+define <2 x double> @f3(<2 x double> %val) #0 {
 ; CHECK-LABEL: f3:
 ; CHECK: vfidb %v24, %v24, 4, 7
 ; CHECK: br %r14
   %res = call <2 x double> @llvm.experimental.constrained.floor.v2f64(
                         <2 x double> %val,
-                        metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret <2 x double> %res
 }
 
-define <2 x double> @f4(<2 x double> %val) {
+define <2 x double> @f4(<2 x double> %val) #0 {
 ; CHECK-LABEL: f4:
 ; CHECK: vfidb %v24, %v24, 4, 6
 ; CHECK: br %r14
   %res = call <2 x double> @llvm.experimental.constrained.ceil.v2f64(
                         <2 x double> %val,
-                        metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret <2 x double> %res
 }
 
-define <2 x double> @f5(<2 x double> %val) {
+define <2 x double> @f5(<2 x double> %val) #0 {
 ; CHECK-LABEL: f5:
 ; CHECK: vfidb %v24, %v24, 4, 5
 ; CHECK: br %r14
   %res = call <2 x double> @llvm.experimental.constrained.trunc.v2f64(
                         <2 x double> %val,
-                        metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret <2 x double> %res
 }
 
-define <2 x double> @f6(<2 x double> %val) {
+define <2 x double> @f6(<2 x double> %val) #0 {
 ; CHECK-LABEL: f6:
 ; CHECK: vfidb %v24, %v24, 4, 1
 ; CHECK: br %r14
   %res = call <2 x double> @llvm.experimental.constrained.round.v2f64(
                         <2 x double> %val,
-                        metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret <2 x double> %res
 }
 
-define double @f7(<2 x double> %val) {
+define double @f7(<2 x double> %val) #0 {
 ; CHECK-LABEL: f7:
 ; CHECK: wfidb %f0, %v24, 0, 0
 ; CHECK: br %r14
@@ -89,11 +85,11 @@ define double @f7(<2 x double> %val) {
   %res = call double @llvm.experimental.constrained.rint.f64(
                         double %scalar,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
-define double @f8(<2 x double> %val) {
+define double @f8(<2 x double> %val) #0 {
 ; CHECK-LABEL: f8:
 ; CHECK: wfidb %f0, %v24, 4, 0
 ; CHECK: br %r14
@@ -101,55 +97,53 @@ define double @f8(<2 x double> %val) {
   %res = call double @llvm.experimental.constrained.nearbyint.f64(
                         double %scalar,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
-define double @f9(<2 x double> %val) {
+define double @f9(<2 x double> %val) #0 {
 ; CHECK-LABEL: f9:
 ; CHECK: wfidb %f0, %v24, 4, 7
 ; CHECK: br %r14
   %scalar = extractelement <2 x double> %val, i32 0
   %res = call double @llvm.experimental.constrained.floor.f64(
                         double %scalar,
-                        metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
 
-define double @f10(<2 x double> %val) {
+define double @f10(<2 x double> %val) #0 {
 ; CHECK-LABEL: f10:
 ; CHECK: wfidb %f0, %v24, 4, 6
 ; CHECK: br %r14
   %scalar = extractelement <2 x double> %val, i32 0
   %res = call double @llvm.experimental.constrained.ceil.f64(
                         double %scalar,
-                        metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
-define double @f11(<2 x double> %val) {
+define double @f11(<2 x double> %val) #0 {
 ; CHECK-LABEL: f11:
 ; CHECK: wfidb %f0, %v24, 4, 5
 ; CHECK: br %r14
   %scalar = extractelement <2 x double> %val, i32 0
   %res = call double @llvm.experimental.constrained.trunc.f64(
                         double %scalar,
-                        metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
 
-define double @f12(<2 x double> %val) {
+define double @f12(<2 x double> %val) #0 {
 ; CHECK-LABEL: f12:
 ; CHECK: wfidb %f0, %v24, 4, 1
 ; CHECK: br %r14
   %scalar = extractelement <2 x double> %val, i32 0
   %res = call double @llvm.experimental.constrained.round.f64(
                         double %scalar,
-                        metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %res
 }
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/SystemZ/vec-strict-round-02.ll b/llvm/test/CodeGen/SystemZ/vec-strict-round-02.ll
index 2ee9b10cc..701dd5b23 100644
--- a/llvm/test/CodeGen/SystemZ/vec-strict-round-02.ll
+++ b/llvm/test/CodeGen/SystemZ/vec-strict-round-02.ll
@@ -4,84 +4,80 @@
 
 declare float @llvm.experimental.constrained.rint.f32(float, metadata, metadata)
 declare float @llvm.experimental.constrained.nearbyint.f32(float, metadata, metadata)
-declare float @llvm.experimental.constrained.floor.f32(float, metadata, metadata)
-declare float @llvm.experimental.constrained.ceil.f32(float, metadata, metadata)
-declare float @llvm.experimental.constrained.trunc.f32(float, metadata, metadata)
-declare float @llvm.experimental.constrained.round.f32(float, metadata, metadata)
+declare float @llvm.experimental.constrained.floor.f32(float, metadata)
+declare float @llvm.experimental.constrained.ceil.f32(float, metadata)
+declare float @llvm.experimental.constrained.trunc.f32(float, metadata)
+declare float @llvm.experimental.constrained.round.f32(float, metadata)
 declare <4 x float> @llvm.experimental.constrained.rint.v4f32(<4 x float>, metadata, metadata)
 declare <4 x float> @llvm.experimental.constrained.nearbyint.v4f32(<4 x float>, metadata, metadata)
-declare <4 x float> @llvm.experimental.constrained.floor.v4f32(<4 x float>, metadata, metadata)
-declare <4 x float> @llvm.experimental.constrained.ceil.v4f32(<4 x float>, metadata, metadata)
-declare <4 x float> @llvm.experimental.constrained.trunc.v4f32(<4 x float>, metadata, metadata)
-declare <4 x float> @llvm.experimental.constrained.round.v4f32(<4 x float>, metadata, metadata)
+declare <4 x float> @llvm.experimental.constrained.floor.v4f32(<4 x float>, metadata)
+declare <4 x float> @llvm.experimental.constrained.ceil.v4f32(<4 x float>, metadata)
+declare <4 x float> @llvm.experimental.constrained.trunc.v4f32(<4 x float>, metadata)
+declare <4 x float> @llvm.experimental.constrained.round.v4f32(<4 x float>, metadata)
 
-define <4 x float> @f1(<4 x float> %val) {
+define <4 x float> @f1(<4 x float> %val) #0 {
 ; CHECK-LABEL: f1:
 ; CHECK: vfisb %v24, %v24, 0, 0
 ; CHECK: br %r14
   %res = call <4 x float> @llvm.experimental.constrained.rint.v4f32(
                         <4 x float> %val,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret <4 x float> %res
 }
 
-define <4 x float> @f2(<4 x float> %val) {
+define <4 x float> @f2(<4 x float> %val) #0 {
 ; CHECK-LABEL: f2:
 ; CHECK: vfisb %v24, %v24, 4, 0
 ; CHECK: br %r14
   %res = call <4 x float> @llvm.experimental.constrained.nearbyint.v4f32(
                         <4 x float> %val,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret <4 x float> %res
 }
 
-define <4 x float> @f3(<4 x float> %val) {
+define <4 x float> @f3(<4 x float> %val) #0 {
 ; CHECK-LABEL: f3:
 ; CHECK: vfisb %v24, %v24, 4, 7
 ; CHECK: br %r14
   %res = call <4 x float> @llvm.experimental.constrained.floor.v4f32(
                         <4 x float> %val,
-                        metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret <4 x float> %res
 }
 
-define <4 x float> @f4(<4 x float> %val) {
+define <4 x float> @f4(<4 x float> %val) #0 {
 ; CHECK-LABEL: f4:
 ; CHECK: vfisb %v24, %v24, 4, 6
 ; CHECK: br %r14
   %res = call <4 x float> @llvm.experimental.constrained.ceil.v4f32(
                         <4 x float> %val,
-                        metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret <4 x float> %res
 }
 
-define <4 x float> @f5(<4 x float> %val) {
+define <4 x float> @f5(<4 x float> %val) #0 {
 ; CHECK-LABEL: f5:
 ; CHECK: vfisb %v24, %v24, 4, 5
 ; CHECK: br %r14
   %res = call <4 x float> @llvm.experimental.constrained.trunc.v4f32(
                         <4 x float> %val,
-                        metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret <4 x float> %res
 }
 
-define <4 x float> @f6(<4 x float> %val) {
+define <4 x float> @f6(<4 x float> %val) #0 {
 ; CHECK-LABEL: f6:
 ; CHECK: vfisb %v24, %v24, 4, 1
 ; CHECK: br %r14
   %res = call <4 x float> @llvm.experimental.constrained.round.v4f32(
                         <4 x float> %val,
-                        metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret <4 x float> %res
 }
 
-define float @f7(<4 x float> %val) {
+define float @f7(<4 x float> %val) #0 {
 ; CHECK-LABEL: f7:
 ; CHECK: wfisb %f0, %v24, 0, 0
 ; CHECK: br %r14
@@ -89,11 +85,11 @@ define float @f7(<4 x float> %val) {
   %res = call float @llvm.experimental.constrained.rint.f32(
                         float %scalar,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
 
-define float @f8(<4 x float> %val) {
+define float @f8(<4 x float> %val) #0 {
 ; CHECK-LABEL: f8:
 ; CHECK: wfisb %f0, %v24, 4, 0
 ; CHECK: br %r14
@@ -101,54 +97,52 @@ define float @f8(<4 x float> %val) {
   %res = call float @llvm.experimental.constrained.nearbyint.f32(
                         float %scalar,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
 
-define float @f9(<4 x float> %val) {
+define float @f9(<4 x float> %val) #0 {
 ; CHECK-LABEL: f9:
 ; CHECK: wfisb %f0, %v24, 4, 7
 ; CHECK: br %r14
   %scalar = extractelement <4 x float> %val, i32 0
   %res = call float @llvm.experimental.constrained.floor.f32(
                         float %scalar,
-                        metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
 
-define float @f10(<4 x float> %val) {
+define float @f10(<4 x float> %val) #0 {
 ; CHECK-LABEL: f10:
 ; CHECK: wfisb %f0, %v24, 4, 6
 ; CHECK: br %r14
   %scalar = extractelement <4 x float> %val, i32 0
   %res = call float @llvm.experimental.constrained.ceil.f32(
                         float %scalar,
-                        metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
 
-define float @f11(<4 x float> %val) {
+define float @f11(<4 x float> %val) #0 {
 ; CHECK-LABEL: f11:
 ; CHECK: wfisb %f0, %v24, 4, 5
 ; CHECK: br %r14
   %scalar = extractelement <4 x float> %val, i32 0
   %res = call float @llvm.experimental.constrained.trunc.f32(
                         float %scalar,
-                        metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
 
-define float @f12(<4 x float> %val) {
+define float @f12(<4 x float> %val) #0 {
 ; CHECK-LABEL: f12:
 ; CHECK: wfisb %f0, %v24, 4, 1
 ; CHECK: br %r14
   %scalar = extractelement <4 x float> %val, i32 0
   %res = call float @llvm.experimental.constrained.round.f32(
                         float %scalar,
-                        metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %res
 }
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/SystemZ/vec-strict-sqrt-01.ll b/llvm/test/CodeGen/SystemZ/vec-strict-sqrt-01.ll
index f59558f11..f7fee09af 100644
--- a/llvm/test/CodeGen/SystemZ/vec-strict-sqrt-01.ll
+++ b/llvm/test/CodeGen/SystemZ/vec-strict-sqrt-01.ll
@@ -5,18 +5,18 @@
 declare double @llvm.experimental.constrained.sqrt.f64(double, metadata, metadata)
 declare <2 x double> @llvm.experimental.constrained.sqrt.v2f64(<2 x double>, metadata, metadata)
 
-define <2 x double> @f1(<2 x double> %val) {
+define <2 x double> @f1(<2 x double> %val) #0 {
 ; CHECK-LABEL: f1:
 ; CHECK: vfsqdb %v24, %v24
 ; CHECK: br %r14
   %ret = call <2 x double> @llvm.experimental.constrained.sqrt.v2f64(
                         <2 x double> %val,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret <2 x double> %ret
 }
 
-define double @f2(<2 x double> %val) {
+define double @f2(<2 x double> %val) #0 {
 ; CHECK-LABEL: f2:
 ; CHECK: wfsqdb %f0, %v24
 ; CHECK: br %r14
@@ -24,6 +24,8 @@ define double @f2(<2 x double> %val) {
   %ret = call double @llvm.experimental.constrained.sqrt.f64(
                         double %scalar,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %ret
 }
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/SystemZ/vec-strict-sqrt-02.ll b/llvm/test/CodeGen/SystemZ/vec-strict-sqrt-02.ll
index 591c2c48c..8c60bd3f6 100644
--- a/llvm/test/CodeGen/SystemZ/vec-strict-sqrt-02.ll
+++ b/llvm/test/CodeGen/SystemZ/vec-strict-sqrt-02.ll
@@ -5,18 +5,18 @@
 declare float @llvm.experimental.constrained.sqrt.f32(float, metadata, metadata)
 declare <4 x float> @llvm.experimental.constrained.sqrt.v4f32(<4 x float>, metadata, metadata)
 
-define <4 x float> @f1(<4 x float> %val) {
+define <4 x float> @f1(<4 x float> %val) #0 {
 ; CHECK-LABEL: f1:
 ; CHECK: vfsqsb %v24, %v24
 ; CHECK: br %r14
   %ret = call <4 x float> @llvm.experimental.constrained.sqrt.v4f32(
                         <4 x float> %val,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret <4 x float> %ret
 }
 
-define float @f2(<4 x float> %val) {
+define float @f2(<4 x float> %val) #0 {
 ; CHECK-LABEL: f2:
 ; CHECK: wfsqsb %f0, %v24
 ; CHECK: br %r14
@@ -24,6 +24,8 @@ define float @f2(<4 x float> %val) {
   %ret = call float @llvm.experimental.constrained.sqrt.f32(
                         float %scalar,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %ret
 }
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/SystemZ/vec-strict-sub-01.ll b/llvm/test/CodeGen/SystemZ/vec-strict-sub-01.ll
index 8564d2275..a379613c3 100644
--- a/llvm/test/CodeGen/SystemZ/vec-strict-sub-01.ll
+++ b/llvm/test/CodeGen/SystemZ/vec-strict-sub-01.ll
@@ -7,19 +7,19 @@ declare <2 x double> @llvm.experimental.constrained.fsub.v2f64(<2 x double>, <2
 
 ; Test a v2f64 subtraction.
 define <2 x double> @f6(<2 x double> %dummy, <2 x double> %val1,
-                        <2 x double> %val2) {
+                        <2 x double> %val2) #0 {
 ; CHECK-LABEL: f6:
 ; CHECK: vfsdb %v24, %v26, %v28
 ; CHECK: br %r14
   %ret = call <2 x double> @llvm.experimental.constrained.fsub.v2f64(
                         <2 x double> %val1, <2 x double> %val2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret <2 x double> %ret
 }
 
 ; Test an f64 subtraction that uses vector registers.
-define double @f7(<2 x double> %val1, <2 x double> %val2) {
+define double @f7(<2 x double> %val1, <2 x double> %val2) #0 {
 ; CHECK-LABEL: f7:
 ; CHECK: wfsdb %f0, %v24, %v26
 ; CHECK: br %r14
@@ -28,7 +28,8 @@ define double @f7(<2 x double> %val1, <2 x double> %val2) {
   %ret = call double @llvm.experimental.constrained.fsub.f64(
                         double %scalar1, double %scalar2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret double %ret
 }
 
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/SystemZ/vec-strict-sub-02.ll b/llvm/test/CodeGen/SystemZ/vec-strict-sub-02.ll
index 1843678d2..fc93e6a09 100644
--- a/llvm/test/CodeGen/SystemZ/vec-strict-sub-02.ll
+++ b/llvm/test/CodeGen/SystemZ/vec-strict-sub-02.ll
@@ -7,19 +7,19 @@ declare <4 x float> @llvm.experimental.constrained.fsub.v4f32(<4 x float>, <4 x
 
 ; Test a v4f32 subtraction.
 define <4 x float> @f6(<4 x float> %dummy, <4 x float> %val1,
-                       <4 x float> %val2) {
+                       <4 x float> %val2) #0 {
 ; CHECK-LABEL: f6:
 ; CHECK: vfssb %v24, %v26, %v28
 ; CHECK: br %r14
   %ret = call <4 x float> @llvm.experimental.constrained.fsub.v4f32(
                         <4 x float> %val1, <4 x float> %val2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret <4 x float> %ret
 }
 
 ; Test an f32 subtraction that uses vector registers.
-define float @f7(<4 x float> %val1, <4 x float> %val2) {
+define float @f7(<4 x float> %val1, <4 x float> %val2) #0 {
 ; CHECK-LABEL: f7:
 ; CHECK: wfssb %f0, %v24, %v26
 ; CHECK: br %r14
@@ -28,6 +28,8 @@ define float @f7(<4 x float> %val1, <4 x float> %val2) {
   %ret = call float @llvm.experimental.constrained.fsub.f32(
                         float %scalar1, float %scalar2,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret float %ret
 }
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/SystemZ/vector-constrained-fp-intrinsics.ll b/llvm/test/CodeGen/SystemZ/vector-constrained-fp-intrinsics.ll
index 8ab4c6db2..7cea2ff8e 100644
--- a/llvm/test/CodeGen/SystemZ/vector-constrained-fp-intrinsics.ll
+++ b/llvm/test/CodeGen/SystemZ/vector-constrained-fp-intrinsics.ll
@@ -2,7 +2,7 @@
 ; RUN: llc -O3 -mtriple=s390x-linux-gnu  < %s | FileCheck --check-prefix=S390X %s
 ; RUN: llc -O3 -mtriple=s390x-linux-gnu -mcpu=z13 < %s | FileCheck --check-prefix=SZ13 %s
 
-define <1 x float> @constrained_vector_fdiv_v1f32() {
+define <1 x float> @constrained_vector_fdiv_v1f32() #0 {
 ; S390X-LABEL: constrained_vector_fdiv_v1f32:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    larl %r1, .LCPI0_0
@@ -23,11 +23,11 @@ entry:
            <1 x float> <float 1.000000e+00>,
            <1 x float> <float 1.000000e+01>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #0
   ret <1 x float> %div
 }
 
-define <2 x double> @constrained_vector_fdiv_v2f64() {
+define <2 x double> @constrained_vector_fdiv_v2f64() #0 {
 ; S390X-LABEL: constrained_vector_fdiv_v2f64:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    larl %r1, .LCPI1_0
@@ -53,11 +53,11 @@ entry:
            <2 x double> <double 1.000000e+00, double 2.000000e+00>,
            <2 x double> <double 1.000000e+01, double 1.000000e+01>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #0
   ret <2 x double> %div
 }
 
-define <3 x float> @constrained_vector_fdiv_v3f32() {
+define <3 x float> @constrained_vector_fdiv_v3f32() #0 {
 ; S390X-LABEL: constrained_vector_fdiv_v3f32:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    larl %r1, .LCPI2_0
@@ -93,27 +93,25 @@ entry:
            <3 x float> <float 1.000000e+00, float 2.000000e+00, float 3.000000e+00>,
            <3 x float> <float 1.000000e+01, float 1.000000e+01, float 1.000000e+01>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #0
   ret <3 x float> %div
 }
 
-define void @constrained_vector_fdiv_v3f64(<3 x double>* %a) {
+define void @constrained_vector_fdiv_v3f64(<3 x double>* %a) #0 {
 ; S390X-LABEL: constrained_vector_fdiv_v3f64:
 ; S390X:       # %bb.0: # %entry
-; S390X-NEXT:    ld %f0, 16(%r2)
-; S390X-NEXT:    ld %f1, 8(%r2)
-; S390X-NEXT:    larl %r1, .LCPI3_0
-; S390X-NEXT:    ldeb %f2, 0(%r1)
 ; S390X-NEXT:    larl %r1, .LCPI3_1
-; S390X-NEXT:    ldeb %f3, 0(%r1)
+; S390X-NEXT:    ldeb %f0, 0(%r1)
 ; S390X-NEXT:    larl %r1, .LCPI3_2
-; S390X-NEXT:    ldeb %f4, 0(%r1)
-; S390X-NEXT:    ddbr %f3, %f1
-; S390X-NEXT:    ddb %f2, 0(%r2)
-; S390X-NEXT:    ddbr %f4, %f0
-; S390X-NEXT:    std %f4, 16(%r2)
-; S390X-NEXT:    std %f3, 8(%r2)
-; S390X-NEXT:    std %f2, 0(%r2)
+; S390X-NEXT:    ldeb %f1, 0(%r1)
+; S390X-NEXT:    larl %r1, .LCPI3_0
+; S390X-NEXT:    ldeb %f2, 0(%r1)
+; S390X-NEXT:    ddb %f1, 0(%r2)
+; S390X-NEXT:    ddb %f0, 8(%r2)
+; S390X-NEXT:    ddb %f2, 16(%r2)
+; S390X-NEXT:    std %f1, 0(%r2)
+; S390X-NEXT:    std %f0, 8(%r2)
+; S390X-NEXT:    std %f2, 16(%r2)
 ; S390X-NEXT:    br %r14
 ;
 ; SZ13-LABEL: constrained_vector_fdiv_v3f64:
@@ -134,12 +132,12 @@ entry:
            <3 x double> <double 1.000000e+00, double 2.000000e+00, double 3.000000e+00>,
            <3 x double> %b,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #0
   store <3 x double> %div, <3 x double>* %a
   ret void
 }
 
-define <4 x double> @constrained_vector_fdiv_v4f64() {
+define <4 x double> @constrained_vector_fdiv_v4f64() #0 {
 ; S390X-LABEL: constrained_vector_fdiv_v4f64:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    larl %r1, .LCPI4_0
@@ -176,11 +174,11 @@ entry:
            <4 x double> <double 1.000000e+01, double 1.000000e+01,
                          double 1.000000e+01, double 1.000000e+01>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #0
   ret <4 x double> %div
 }
 
-define <1 x float> @constrained_vector_frem_v1f32() {
+define <1 x float> @constrained_vector_frem_v1f32() #0 {
 ; S390X-LABEL: constrained_vector_frem_v1f32:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r14, %r15, 112(%r15)
@@ -216,11 +214,11 @@ entry:
            <1 x float> <float 1.000000e+00>,
            <1 x float> <float 1.000000e+01>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #0
   ret <1 x float> %rem
 }
 
-define <2 x double> @constrained_vector_frem_v2f64() {
+define <2 x double> @constrained_vector_frem_v2f64() #0 {
 ; S390X-LABEL: constrained_vector_frem_v2f64:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r14, %r15, 112(%r15)
@@ -244,8 +242,7 @@ define <2 x double> @constrained_vector_frem_v2f64() {
 ; S390X-NEXT:    ldr %f0, %f1
 ; S390X-NEXT:    ldr %f2, %f8
 ; S390X-NEXT:    brasl %r14, fmod@PLT
-; S390X-NEXT:    ldr %f2, %f0
-; S390X-NEXT:    ldr %f0, %f9
+; S390X-NEXT:    ldr %f2, %f9
 ; S390X-NEXT:    ld %f8, 168(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f9, 160(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    lmg %r14, %r15, 288(%r15)
@@ -281,11 +278,11 @@ entry:
            <2 x double> <double 1.000000e+00, double 2.000000e+00>,
            <2 x double> <double 1.000000e+01, double 1.000000e+01>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #0
   ret <2 x double> %rem
 }
 
-define <3 x float> @constrained_vector_frem_v3f32() {
+define <3 x float> @constrained_vector_frem_v3f32() #0 {
 ; S390X-LABEL: constrained_vector_frem_v3f32:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r14, %r15, 112(%r15)
@@ -317,9 +314,8 @@ define <3 x float> @constrained_vector_frem_v3f32() {
 ; S390X-NEXT:    ler %f0, %f1
 ; S390X-NEXT:    ler %f2, %f8
 ; S390X-NEXT:    brasl %r14, fmodf@PLT
-; S390X-NEXT:    ler %f4, %f0
-; S390X-NEXT:    ler %f0, %f9
 ; S390X-NEXT:    ler %f2, %f10
+; S390X-NEXT:    ler %f4, %f9
 ; S390X-NEXT:    ld %f8, 176(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f9, 168(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f10, 160(%r15) # 8-byte Folded Reload
@@ -365,11 +361,11 @@ entry:
            <3 x float> <float 1.000000e+00, float 2.000000e+00, float 3.000000e+00>,
            <3 x float> <float 1.000000e+01, float 1.000000e+01, float 1.000000e+01>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #0
   ret <3 x float> %rem
 }
 
-define void @constrained_vector_frem_v3f64(<3 x double>* %a) {
+define void @constrained_vector_frem_v3f64(<3 x double>* %a) #0 {
 ; S390X-LABEL: constrained_vector_frem_v3f64:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r13, %r15, 104(%r15)
@@ -385,8 +381,8 @@ define void @constrained_vector_frem_v3f64(<3 x double>* %a) {
 ; S390X-NEXT:    .cfi_offset %f9, -176
 ; S390X-NEXT:    .cfi_offset %f10, -184
 ; S390X-NEXT:    lgr %r13, %r2
-; S390X-NEXT:    ld %f8, 16(%r2)
-; S390X-NEXT:    ld %f2, 0(%r2)
+; S390X-NEXT:    ld %f8, 0(%r2)
+; S390X-NEXT:    ld %f2, 16(%r2)
 ; S390X-NEXT:    larl %r1, .LCPI8_0
 ; S390X-NEXT:    ldeb %f0, 0(%r1)
 ; S390X-NEXT:    ld %f9, 8(%r2)
@@ -403,9 +399,9 @@ define void @constrained_vector_frem_v3f64(<3 x double>* %a) {
 ; S390X-NEXT:    ldr %f0, %f1
 ; S390X-NEXT:    ldr %f2, %f8
 ; S390X-NEXT:    brasl %r14, fmod@PLT
-; S390X-NEXT:    std %f0, 16(%r13)
+; S390X-NEXT:    std %f0, 0(%r13)
 ; S390X-NEXT:    std %f9, 8(%r13)
-; S390X-NEXT:    std %f10, 0(%r13)
+; S390X-NEXT:    std %f10, 16(%r13)
 ; S390X-NEXT:    ld %f8, 176(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f9, 168(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f10, 160(%r15) # 8-byte Folded Reload
@@ -456,12 +452,12 @@ entry:
            <3 x double> <double 1.000000e+00, double 2.000000e+00, double 3.000000e+00>,
            <3 x double> %b,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #0
   store <3 x double> %rem, <3 x double>* %a
   ret void
 }
 
-define <4 x double> @constrained_vector_frem_v4f64() {
+define <4 x double> @constrained_vector_frem_v4f64() #0 {
 ; S390X-LABEL: constrained_vector_frem_v4f64:
 ; S390X:       # %bb.0:
 ; S390X-NEXT:    stmg %r14, %r15, 112(%r15)
@@ -501,10 +497,9 @@ define <4 x double> @constrained_vector_frem_v4f64() {
 ; S390X-NEXT:    ldr %f0, %f1
 ; S390X-NEXT:    ldr %f2, %f8
 ; S390X-NEXT:    brasl %r14, fmod@PLT
-; S390X-NEXT:    ldr %f6, %f0
-; S390X-NEXT:    ldr %f0, %f9
-; S390X-NEXT:    ldr %f2, %f10
-; S390X-NEXT:    ldr %f4, %f11
+; S390X-NEXT:    ldr %f2, %f11
+; S390X-NEXT:    ldr %f4, %f10
+; S390X-NEXT:    ldr %f6, %f9
 ; S390X-NEXT:    ld %f8, 184(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f9, 176(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f10, 168(%r15) # 8-byte Folded Reload
@@ -558,11 +553,11 @@ define <4 x double> @constrained_vector_frem_v4f64() {
            <4 x double> <double 1.000000e+01, double 1.000000e+01,
                          double 1.000000e+01, double 1.000000e+01>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #0
   ret <4 x double> %rem
 }
 
-define <1 x float> @constrained_vector_fmul_v1f32() {
+define <1 x float> @constrained_vector_fmul_v1f32() #0 {
 ; S390X-LABEL: constrained_vector_fmul_v1f32:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    larl %r1, .LCPI10_0
@@ -583,11 +578,11 @@ entry:
            <1 x float> <float 0x7FF0000000000000>,
            <1 x float> <float 2.000000e+00>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #0
   ret <1 x float> %mul
 }
 
-define <2 x double> @constrained_vector_fmul_v2f64() {
+define <2 x double> @constrained_vector_fmul_v2f64() #0 {
 ; S390X-LABEL: constrained_vector_fmul_v2f64:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    larl %r1, .LCPI11_0
@@ -613,11 +608,11 @@ entry:
            <2 x double> <double 0x7FEFFFFFFFFFFFFF, double 0x7FEFFFFFFFFFFFFF>,
            <2 x double> <double 2.000000e+00, double 3.000000e+00>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #0
   ret <2 x double> %mul
 }
 
-define <3 x float> @constrained_vector_fmul_v3f32() {
+define <3 x float> @constrained_vector_fmul_v3f32() #0 {
 ; S390X-LABEL: constrained_vector_fmul_v3f32:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    larl %r1, .LCPI12_0
@@ -652,24 +647,23 @@ entry:
                         float 0x7FF0000000000000>,
            <3 x float> <float 1.000000e+00, float 1.000000e+01, float 1.000000e+02>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #0
   ret <3 x float> %mul
 }
 
-define void @constrained_vector_fmul_v3f64(<3 x double>* %a) {
+define void @constrained_vector_fmul_v3f64(<3 x double>* %a) #0 {
 ; S390X-LABEL: constrained_vector_fmul_v3f64:
 ; S390X:       # %bb.0: # %entry
-; S390X-NEXT:    ld %f0, 8(%r2)
 ; S390X-NEXT:    larl %r1, .LCPI13_0
-; S390X-NEXT:    ld %f1, 0(%r1)
-; S390X-NEXT:    ld %f2, 16(%r2)
-; S390X-NEXT:    mdbr %f0, %f1
-; S390X-NEXT:    ldr %f3, %f1
-; S390X-NEXT:    mdb %f3, 0(%r2)
-; S390X-NEXT:    mdbr %f2, %f1
-; S390X-NEXT:    std %f2, 16(%r2)
-; S390X-NEXT:    std %f0, 8(%r2)
-; S390X-NEXT:    std %f3, 0(%r2)
+; S390X-NEXT:    ld %f0, 0(%r1)
+; S390X-NEXT:    ldr %f1, %f0
+; S390X-NEXT:    ldr %f2, %f0
+; S390X-NEXT:    mdb %f0, 0(%r2)
+; S390X-NEXT:    mdb %f2, 8(%r2)
+; S390X-NEXT:    mdb %f1, 16(%r2)
+; S390X-NEXT:    std %f0, 0(%r2)
+; S390X-NEXT:    std %f2, 8(%r2)
+; S390X-NEXT:    std %f1, 16(%r2)
 ; S390X-NEXT:    br %r14
 ;
 ; SZ13-LABEL: constrained_vector_fmul_v3f64:
@@ -681,8 +675,8 @@ define void @constrained_vector_fmul_v3f64(<3 x double>* %a) {
 ; SZ13-NEXT:    vl %v2, 0(%r1), 3
 ; SZ13-NEXT:    mdb %f1, 16(%r2)
 ; SZ13-NEXT:    vfmdb %v0, %v2, %v0
-; SZ13-NEXT:    std %f1, 16(%r2)
 ; SZ13-NEXT:    vst %v0, 0(%r2), 4
+; SZ13-NEXT:    std %f1, 16(%r2)
 ; SZ13-NEXT:    br %r14
 entry:
   %b = load <3 x double>, <3 x double>* %a
@@ -691,12 +685,12 @@ entry:
                         double 0x7FEFFFFFFFFFFFFF>,
            <3 x double> %b,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #0
   store <3 x double> %mul, <3 x double>* %a
   ret void
 }
 
-define <4 x double> @constrained_vector_fmul_v4f64() {
+define <4 x double> @constrained_vector_fmul_v4f64() #0 {
 ; S390X-LABEL: constrained_vector_fmul_v4f64:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    larl %r1, .LCPI14_0
@@ -733,11 +727,11 @@ entry:
            <4 x double> <double 2.000000e+00, double 3.000000e+00,
                          double 4.000000e+00, double 5.000000e+00>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #0
   ret <4 x double> %mul
 }
 
-define <1 x float> @constrained_vector_fadd_v1f32() {
+define <1 x float> @constrained_vector_fadd_v1f32() #0 {
 ; S390X-LABEL: constrained_vector_fadd_v1f32:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    larl %r1, .LCPI15_0
@@ -758,11 +752,11 @@ entry:
            <1 x float> <float 0x7FF0000000000000>,
            <1 x float> <float 1.0>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #0
   ret <1 x float> %add
 }
 
-define <2 x double> @constrained_vector_fadd_v2f64() {
+define <2 x double> @constrained_vector_fadd_v2f64() #0 {
 ; S390X-LABEL: constrained_vector_fadd_v2f64:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    larl %r1, .LCPI16_0
@@ -788,11 +782,11 @@ entry:
            <2 x double> <double 0x7FEFFFFFFFFFFFFF, double 0x7FEFFFFFFFFFFFFF>,
            <2 x double> <double 1.000000e+00, double 1.000000e-01>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #0
   ret <2 x double> %add
 }
 
-define <3 x float> @constrained_vector_fadd_v3f32() {
+define <3 x float> @constrained_vector_fadd_v3f32() #0 {
 ; S390X-LABEL: constrained_vector_fadd_v3f32:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    larl %r1, .LCPI17_0
@@ -825,24 +819,23 @@ entry:
                         float 0xFFFFFFFFE0000000>,
            <3 x float> <float 2.0, float 1.0, float 0.0>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #0
   ret <3 x float> %add
 }
 
-define void @constrained_vector_fadd_v3f64(<3 x double>* %a) {
+define void @constrained_vector_fadd_v3f64(<3 x double>* %a) #0 {
 ; S390X-LABEL: constrained_vector_fadd_v3f64:
 ; S390X:       # %bb.0: # %entry
-; S390X-NEXT:    ld %f0, 8(%r2)
 ; S390X-NEXT:    larl %r1, .LCPI18_0
-; S390X-NEXT:    ld %f1, 0(%r1)
-; S390X-NEXT:    ld %f2, 16(%r2)
-; S390X-NEXT:    adbr %f0, %f1
-; S390X-NEXT:    ldr %f3, %f1
-; S390X-NEXT:    adb %f3, 0(%r2)
-; S390X-NEXT:    adbr %f2, %f1
-; S390X-NEXT:    std %f2, 16(%r2)
-; S390X-NEXT:    std %f0, 8(%r2)
-; S390X-NEXT:    std %f3, 0(%r2)
+; S390X-NEXT:    ld %f0, 0(%r1)
+; S390X-NEXT:    ldr %f1, %f0
+; S390X-NEXT:    ldr %f2, %f0
+; S390X-NEXT:    adb %f0, 0(%r2)
+; S390X-NEXT:    adb %f2, 8(%r2)
+; S390X-NEXT:    adb %f1, 16(%r2)
+; S390X-NEXT:    std %f0, 0(%r2)
+; S390X-NEXT:    std %f2, 8(%r2)
+; S390X-NEXT:    std %f1, 16(%r2)
 ; S390X-NEXT:    br %r14
 ;
 ; SZ13-LABEL: constrained_vector_fadd_v3f64:
@@ -854,8 +847,8 @@ define void @constrained_vector_fadd_v3f64(<3 x double>* %a) {
 ; SZ13-NEXT:    vl %v2, 0(%r1), 3
 ; SZ13-NEXT:    adb %f1, 16(%r2)
 ; SZ13-NEXT:    vfadb %v0, %v2, %v0
-; SZ13-NEXT:    std %f1, 16(%r2)
 ; SZ13-NEXT:    vst %v0, 0(%r2), 4
+; SZ13-NEXT:    std %f1, 16(%r2)
 ; SZ13-NEXT:    br %r14
 entry:
   %b = load <3 x double>, <3 x double>* %a
@@ -864,12 +857,12 @@ entry:
                          double 0x7FEFFFFFFFFFFFFF>,
            <3 x double> %b,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #0
   store <3 x double> %add, <3 x double>* %a
   ret void
 }
 
-define <4 x double> @constrained_vector_fadd_v4f64() {
+define <4 x double> @constrained_vector_fadd_v4f64() #0 {
 ; S390X-LABEL: constrained_vector_fadd_v4f64:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    larl %r1, .LCPI19_0
@@ -906,11 +899,11 @@ entry:
            <4 x double> <double 1.000000e+00, double 1.000000e-01,
                          double 2.000000e+00, double 2.000000e-01>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #0
   ret <4 x double> %add
 }
 
-define <1 x float> @constrained_vector_fsub_v1f32() {
+define <1 x float> @constrained_vector_fsub_v1f32() #0 {
 ; S390X-LABEL: constrained_vector_fsub_v1f32:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    larl %r1, .LCPI20_0
@@ -931,11 +924,11 @@ entry:
            <1 x float> <float 0x7FF0000000000000>,
            <1 x float> <float 1.000000e+00>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #0
   ret <1 x float> %sub
 }
 
-define <2 x double> @constrained_vector_fsub_v2f64() {
+define <2 x double> @constrained_vector_fsub_v2f64() #0 {
 ; S390X-LABEL: constrained_vector_fsub_v2f64:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    larl %r1, .LCPI21_0
@@ -960,11 +953,11 @@ entry:
            <2 x double> <double 0xFFEFFFFFFFFFFFFF, double 0xFFEFFFFFFFFFFFFF>,
            <2 x double> <double 1.000000e+00, double 1.000000e-01>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #0
   ret <2 x double> %sub
 }
 
-define <3 x float> @constrained_vector_fsub_v3f32() {
+define <3 x float> @constrained_vector_fsub_v3f32() #0 {
 ; S390X-LABEL: constrained_vector_fsub_v3f32:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    larl %r1, .LCPI22_0
@@ -1000,25 +993,23 @@ entry:
                         float 0xFFFFFFFFE0000000>,
            <3 x float> <float 2.0, float 1.0, float 0.0>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #0
   ret <3 x float> %sub
 }
 
-define void @constrained_vector_fsub_v3f64(<3 x double>* %a) {
+define void @constrained_vector_fsub_v3f64(<3 x double>* %a) #0 {
 ; S390X-LABEL: constrained_vector_fsub_v3f64:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    larl %r1, .LCPI23_0
 ; S390X-NEXT:    ld %f0, 0(%r1)
-; S390X-NEXT:    ld %f1, 8(%r2)
-; S390X-NEXT:    ld %f2, 16(%r2)
-; S390X-NEXT:    ldr %f3, %f0
-; S390X-NEXT:    sdb %f3, 0(%r2)
-; S390X-NEXT:    ldr %f4, %f0
-; S390X-NEXT:    sdbr %f4, %f1
-; S390X-NEXT:    sdbr %f0, %f2
-; S390X-NEXT:    std %f0, 16(%r2)
-; S390X-NEXT:    std %f4, 8(%r2)
-; S390X-NEXT:    std %f3, 0(%r2)
+; S390X-NEXT:    ldr %f1, %f0
+; S390X-NEXT:    ldr %f2, %f0
+; S390X-NEXT:    sdb %f0, 0(%r2)
+; S390X-NEXT:    sdb %f2, 8(%r2)
+; S390X-NEXT:    sdb %f1, 16(%r2)
+; S390X-NEXT:    std %f0, 0(%r2)
+; S390X-NEXT:    std %f2, 8(%r2)
+; S390X-NEXT:    std %f1, 16(%r2)
 ; S390X-NEXT:    br %r14
 ;
 ; SZ13-LABEL: constrained_vector_fsub_v3f64:
@@ -1028,8 +1019,8 @@ define void @constrained_vector_fsub_v3f64(<3 x double>* %a) {
 ; SZ13-NEXT:    sdb %f2, 16(%r2)
 ; SZ13-NEXT:    vgmg %v1, 12, 10
 ; SZ13-NEXT:    vfsdb %v0, %v1, %v0
-; SZ13-NEXT:    std %f2, 16(%r2)
 ; SZ13-NEXT:    vst %v0, 0(%r2), 4
+; SZ13-NEXT:    std %f2, 16(%r2)
 ; SZ13-NEXT:    br %r14
 entry:
   %b = load <3 x double>, <3 x double>* %a
@@ -1038,12 +1029,12 @@ entry:
                          double 0xFFEFFFFFFFFFFFFF>,
            <3 x double> %b,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #0
   store <3 x double> %sub, <3 x double>* %a
   ret void
 }
 
-define <4 x double> @constrained_vector_fsub_v4f64() {
+define <4 x double> @constrained_vector_fsub_v4f64() #0 {
 ; S390X-LABEL: constrained_vector_fsub_v4f64:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    larl %r1, .LCPI24_0
@@ -1080,11 +1071,11 @@ entry:
            <4 x double> <double 1.000000e+00, double 1.000000e-01,
                          double 2.000000e+00, double 2.000000e-01>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #0
   ret <4 x double> %sub
 }
 
-define <1 x float> @constrained_vector_sqrt_v1f32() {
+define <1 x float> @constrained_vector_sqrt_v1f32() #0 {
 ; S390X-LABEL: constrained_vector_sqrt_v1f32:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    larl %r1, .LCPI25_0
@@ -1101,11 +1092,11 @@ entry:
   %sqrt = call <1 x float> @llvm.experimental.constrained.sqrt.v1f32(
                               <1 x float> <float 42.0>,
                               metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #0
   ret <1 x float> %sqrt
 }
 
-define <2 x double> @constrained_vector_sqrt_v2f64() {
+define <2 x double> @constrained_vector_sqrt_v2f64() #0 {
 ; S390X-LABEL: constrained_vector_sqrt_v2f64:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    larl %r1, .LCPI26_0
@@ -1125,11 +1116,11 @@ entry:
   %sqrt = call <2 x double> @llvm.experimental.constrained.sqrt.v2f64(
                               <2 x double> <double 42.0, double 42.1>,
                               metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #0
   ret <2 x double> %sqrt
 }
 
-define <3 x float> @constrained_vector_sqrt_v3f32() {
+define <3 x float> @constrained_vector_sqrt_v3f32() #0 {
 ; S390X-LABEL: constrained_vector_sqrt_v3f32:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    larl %r1, .LCPI27_0
@@ -1156,21 +1147,19 @@ entry:
   %sqrt = call <3 x float> @llvm.experimental.constrained.sqrt.v3f32(
                               <3 x float> <float 42.0, float 43.0, float 44.0>,
                               metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #0
   ret <3 x float> %sqrt
 }
 
-define void @constrained_vector_sqrt_v3f64(<3 x double>* %a) {
+define void @constrained_vector_sqrt_v3f64(<3 x double>* %a) #0 {
 ; S390X-LABEL: constrained_vector_sqrt_v3f64:
 ; S390X:       # %bb.0: # %entry
-; S390X-NEXT:    ld %f0, 8(%r2)
-; S390X-NEXT:    ld %f1, 16(%r2)
-; S390X-NEXT:    sqdb %f2, 0(%r2)
-; S390X-NEXT:    sqdbr %f0, %f0
-; S390X-NEXT:    sqdbr %f1, %f1
-; S390X-NEXT:    std %f1, 16(%r2)
-; S390X-NEXT:    std %f0, 8(%r2)
-; S390X-NEXT:    std %f2, 0(%r2)
+; S390X-NEXT:    sqdb %f0, 0(%r2)
+; S390X-NEXT:    sqdb %f1, 8(%r2)
+; S390X-NEXT:    sqdb %f2, 16(%r2)
+; S390X-NEXT:    std %f0, 0(%r2)
+; S390X-NEXT:    std %f1, 8(%r2)
+; S390X-NEXT:    std %f2, 16(%r2)
 ; S390X-NEXT:    br %r14
 ;
 ; SZ13-LABEL: constrained_vector_sqrt_v3f64:
@@ -1186,12 +1175,12 @@ entry:
   %sqrt = call <3 x double> @llvm.experimental.constrained.sqrt.v3f64(
                           <3 x double> %b,
                           metadata !"round.dynamic",
-                          metadata !"fpexcept.strict")
+                          metadata !"fpexcept.strict") #0
   store <3 x double> %sqrt, <3 x double>* %a
   ret void
 }
 
-define <4 x double> @constrained_vector_sqrt_v4f64() {
+define <4 x double> @constrained_vector_sqrt_v4f64() #0 {
 ; S390X-LABEL: constrained_vector_sqrt_v4f64:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    larl %r1, .LCPI29_0
@@ -1219,11 +1208,11 @@ define <4 x double> @constrained_vector_sqrt_v4f64() {
                               <4 x double> <double 42.0, double 42.1,
                                             double 42.2, double 42.3>,
                               metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #0
   ret <4 x double> %sqrt
 }
 
-define <1 x float> @constrained_vector_pow_v1f32() {
+define <1 x float> @constrained_vector_pow_v1f32() #0 {
 ; S390X-LABEL: constrained_vector_pow_v1f32:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r14, %r15, 112(%r15)
@@ -1260,11 +1249,11 @@ entry:
                              <1 x float> <float 42.0>,
                              <1 x float> <float 3.0>,
                              metadata !"round.dynamic",
-                             metadata !"fpexcept.strict")
+                             metadata !"fpexcept.strict") #0
   ret <1 x float> %pow
 }
 
-define <2 x double> @constrained_vector_pow_v2f64() {
+define <2 x double> @constrained_vector_pow_v2f64() #0 {
 ; S390X-LABEL: constrained_vector_pow_v2f64:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r14, %r15, 112(%r15)
@@ -1288,8 +1277,7 @@ define <2 x double> @constrained_vector_pow_v2f64() {
 ; S390X-NEXT:    ldr %f0, %f1
 ; S390X-NEXT:    ldr %f2, %f8
 ; S390X-NEXT:    brasl %r14, pow@PLT
-; S390X-NEXT:    ldr %f2, %f0
-; S390X-NEXT:    ldr %f0, %f9
+; S390X-NEXT:    ldr %f2, %f9
 ; S390X-NEXT:    ld %f8, 168(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f9, 160(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    lmg %r14, %r15, 288(%r15)
@@ -1327,11 +1315,11 @@ entry:
                              <2 x double> <double 42.1, double 42.2>,
                              <2 x double> <double 3.0, double 3.0>,
                              metadata !"round.dynamic",
-                             metadata !"fpexcept.strict")
+                             metadata !"fpexcept.strict") #0
   ret <2 x double> %pow
 }
 
-define <3 x float> @constrained_vector_pow_v3f32() {
+define <3 x float> @constrained_vector_pow_v3f32() #0 {
 ; S390X-LABEL: constrained_vector_pow_v3f32:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r14, %r15, 112(%r15)
@@ -1363,9 +1351,8 @@ define <3 x float> @constrained_vector_pow_v3f32() {
 ; S390X-NEXT:    ler %f0, %f1
 ; S390X-NEXT:    ler %f2, %f8
 ; S390X-NEXT:    brasl %r14, powf@PLT
-; S390X-NEXT:    ler %f4, %f0
-; S390X-NEXT:    ler %f0, %f9
 ; S390X-NEXT:    ler %f2, %f10
+; S390X-NEXT:    ler %f4, %f9
 ; S390X-NEXT:    ld %f8, 176(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f9, 168(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f10, 160(%r15) # 8-byte Folded Reload
@@ -1413,11 +1400,11 @@ entry:
                              <3 x float> <float 42.0, float 43.0, float 44.0>,
                              <3 x float> <float 3.0, float 3.0, float 3.0>,
                              metadata !"round.dynamic",
-                             metadata !"fpexcept.strict")
+                             metadata !"fpexcept.strict") #0
   ret <3 x float> %pow
 }
 
-define void @constrained_vector_pow_v3f64(<3 x double>* %a) {
+define void @constrained_vector_pow_v3f64(<3 x double>* %a) #0 {
 ; S390X-LABEL: constrained_vector_pow_v3f64:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r13, %r15, 104(%r15)
@@ -1435,8 +1422,8 @@ define void @constrained_vector_pow_v3f64(<3 x double>* %a) {
 ; S390X-NEXT:    .cfi_offset %f10, -184
 ; S390X-NEXT:    .cfi_offset %f11, -192
 ; S390X-NEXT:    lgr %r13, %r2
-; S390X-NEXT:    ld %f8, 16(%r2)
-; S390X-NEXT:    ld %f0, 0(%r2)
+; S390X-NEXT:    ld %f8, 0(%r2)
+; S390X-NEXT:    ld %f0, 16(%r2)
 ; S390X-NEXT:    larl %r1, .LCPI33_0
 ; S390X-NEXT:    ldeb %f9, 0(%r1)
 ; S390X-NEXT:    ld %f10, 8(%r2)
@@ -1450,9 +1437,9 @@ define void @constrained_vector_pow_v3f64(<3 x double>* %a) {
 ; S390X-NEXT:    ldr %f0, %f8
 ; S390X-NEXT:    ldr %f2, %f9
 ; S390X-NEXT:    brasl %r14, pow@PLT
-; S390X-NEXT:    std %f0, 16(%r13)
+; S390X-NEXT:    std %f0, 0(%r13)
 ; S390X-NEXT:    std %f10, 8(%r13)
-; S390X-NEXT:    std %f11, 0(%r13)
+; S390X-NEXT:    std %f11, 16(%r13)
 ; S390X-NEXT:    ld %f8, 184(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f9, 176(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f10, 168(%r15) # 8-byte Folded Reload
@@ -1508,12 +1495,12 @@ entry:
                           <3 x double> %b,
                           <3 x double> <double 3.0, double 3.0, double 3.0>,
                           metadata !"round.dynamic",
-                          metadata !"fpexcept.strict")
+                          metadata !"fpexcept.strict") #0
   store <3 x double> %pow, <3 x double>* %a
   ret void
 }
 
-define <4 x double> @constrained_vector_pow_v4f64() {
+define <4 x double> @constrained_vector_pow_v4f64() #0 {
 ; S390X-LABEL: constrained_vector_pow_v4f64:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r14, %r15, 112(%r15)
@@ -1553,10 +1540,9 @@ define <4 x double> @constrained_vector_pow_v4f64() {
 ; S390X-NEXT:    ldr %f0, %f1
 ; S390X-NEXT:    ldr %f2, %f8
 ; S390X-NEXT:    brasl %r14, pow@PLT
-; S390X-NEXT:    ldr %f6, %f0
-; S390X-NEXT:    ldr %f0, %f9
-; S390X-NEXT:    ldr %f2, %f10
-; S390X-NEXT:    ldr %f4, %f11
+; S390X-NEXT:    ldr %f2, %f11
+; S390X-NEXT:    ldr %f4, %f10
+; S390X-NEXT:    ldr %f6, %f9
 ; S390X-NEXT:    ld %f8, 184(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f9, 176(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f10, 168(%r15) # 8-byte Folded Reload
@@ -1613,11 +1599,11 @@ entry:
                              <4 x double> <double 3.0, double 3.0,
                                            double 3.0, double 3.0>,
                              metadata !"round.dynamic",
-                             metadata !"fpexcept.strict")
+                             metadata !"fpexcept.strict") #0
   ret <4 x double> %pow
 }
 
-define <1 x float> @constrained_vector_powi_v1f32() {
+define <1 x float> @constrained_vector_powi_v1f32() #0 {
 ; S390X-LABEL: constrained_vector_powi_v1f32:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r14, %r15, 112(%r15)
@@ -1652,11 +1638,11 @@ entry:
                               <1 x float> <float 42.0>,
                               i32 3,
                               metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #0
   ret <1 x float> %powi
 }
 
-define <2 x double> @constrained_vector_powi_v2f64() {
+define <2 x double> @constrained_vector_powi_v2f64() #0 {
 ; S390X-LABEL: constrained_vector_powi_v2f64:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r14, %r15, 112(%r15)
@@ -1676,8 +1662,7 @@ define <2 x double> @constrained_vector_powi_v2f64() {
 ; S390X-NEXT:    lghi %r2, 3
 ; S390X-NEXT:    ldr %f0, %f1
 ; S390X-NEXT:    brasl %r14, __powidf2@PLT
-; S390X-NEXT:    ldr %f2, %f0
-; S390X-NEXT:    ldr %f0, %f8
+; S390X-NEXT:    ldr %f2, %f8
 ; S390X-NEXT:    ld %f8, 160(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    lmg %r14, %r15, 280(%r15)
 ; S390X-NEXT:    br %r14
@@ -1709,11 +1694,11 @@ entry:
                               <2 x double> <double 42.1, double 42.2>,
                               i32 3,
                               metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #0
   ret <2 x double> %powi
 }
 
-define <3 x float> @constrained_vector_powi_v3f32() {
+define <3 x float> @constrained_vector_powi_v3f32() #0 {
 ; S390X-LABEL: constrained_vector_powi_v3f32:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r14, %r15, 112(%r15)
@@ -1741,9 +1726,8 @@ define <3 x float> @constrained_vector_powi_v3f32() {
 ; S390X-NEXT:    lghi %r2, 3
 ; S390X-NEXT:    ler %f0, %f1
 ; S390X-NEXT:    brasl %r14, __powisf2@PLT
-; S390X-NEXT:    ler %f4, %f0
-; S390X-NEXT:    ler %f0, %f8
 ; S390X-NEXT:    ler %f2, %f9
+; S390X-NEXT:    ler %f4, %f8
 ; S390X-NEXT:    ld %f8, 168(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f9, 160(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    lmg %r14, %r15, 288(%r15)
@@ -1785,11 +1769,11 @@ entry:
                               <3 x float> <float 42.0, float 43.0, float 44.0>,
                               i32 3,
                               metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #0
   ret <3 x float> %powi
 }
 
-define void @constrained_vector_powi_v3f64(<3 x double>* %a) {
+define void @constrained_vector_powi_v3f64(<3 x double>* %a) #0 {
 ; S390X-LABEL: constrained_vector_powi_v3f64:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r13, %r15, 104(%r15)
@@ -1803,12 +1787,12 @@ define void @constrained_vector_powi_v3f64(<3 x double>* %a) {
 ; S390X-NEXT:    .cfi_offset %f8, -168
 ; S390X-NEXT:    .cfi_offset %f9, -176
 ; S390X-NEXT:    larl %r1, .LCPI38_0
-; S390X-NEXT:    ld %f0, 0(%r1)
+; S390X-NEXT:    ldeb %f0, 0(%r1)
 ; S390X-NEXT:    lgr %r13, %r2
 ; S390X-NEXT:    lghi %r2, 3
 ; S390X-NEXT:    brasl %r14, __powidf2@PLT
 ; S390X-NEXT:    larl %r1, .LCPI38_1
-; S390X-NEXT:    ldeb %f1, 0(%r1)
+; S390X-NEXT:    ld %f1, 0(%r1)
 ; S390X-NEXT:    ldr %f8, %f0
 ; S390X-NEXT:    lghi %r2, 3
 ; S390X-NEXT:    ldr %f0, %f1
@@ -1819,9 +1803,9 @@ define void @constrained_vector_powi_v3f64(<3 x double>* %a) {
 ; S390X-NEXT:    lghi %r2, 3
 ; S390X-NEXT:    ldr %f0, %f1
 ; S390X-NEXT:    brasl %r14, __powidf2@PLT
-; S390X-NEXT:    std %f0, 8(%r13)
-; S390X-NEXT:    std %f9, 0(%r13)
-; S390X-NEXT:    std %f8, 16(%r13)
+; S390X-NEXT:    std %f0, 16(%r13)
+; S390X-NEXT:    std %f9, 8(%r13)
+; S390X-NEXT:    std %f8, 0(%r13)
 ; S390X-NEXT:    ld %f8, 168(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f9, 160(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    lmg %r13, %r15, 280(%r15)
@@ -1865,12 +1849,12 @@ entry:
                           <3 x double> <double 42.0, double 42.1, double 42.2>,
                           i32 3,
                           metadata !"round.dynamic",
-                          metadata !"fpexcept.strict")
+                          metadata !"fpexcept.strict") #0
   store <3 x double> %powi, <3 x double>* %a
   ret void
 }
 
-define <4 x double> @constrained_vector_powi_v4f64() {
+define <4 x double> @constrained_vector_powi_v4f64() #0 {
 ; S390X-LABEL: constrained_vector_powi_v4f64:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r14, %r15, 112(%r15)
@@ -1906,10 +1890,9 @@ define <4 x double> @constrained_vector_powi_v4f64() {
 ; S390X-NEXT:    lghi %r2, 3
 ; S390X-NEXT:    ldr %f0, %f1
 ; S390X-NEXT:    brasl %r14, __powidf2@PLT
-; S390X-NEXT:    ldr %f6, %f0
-; S390X-NEXT:    ldr %f0, %f8
-; S390X-NEXT:    ldr %f2, %f9
-; S390X-NEXT:    ldr %f4, %f10
+; S390X-NEXT:    ldr %f2, %f10
+; S390X-NEXT:    ldr %f4, %f9
+; S390X-NEXT:    ldr %f6, %f8
 ; S390X-NEXT:    ld %f8, 176(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f9, 168(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f10, 160(%r15) # 8-byte Folded Reload
@@ -1959,11 +1942,11 @@ entry:
                                             double 42.3, double 42.4>,
                               i32 3,
                               metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #0
   ret <4 x double> %powi
 }
 
-define <1 x float> @constrained_vector_sin_v1f32() {
+define <1 x float> @constrained_vector_sin_v1f32() #0 {
 ; S390X-LABEL: constrained_vector_sin_v1f32:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r14, %r15, 112(%r15)
@@ -1995,11 +1978,11 @@ entry:
   %sin = call <1 x float> @llvm.experimental.constrained.sin.v1f32(
                              <1 x float> <float 42.0>,
                              metadata !"round.dynamic",
-                             metadata !"fpexcept.strict")
+                             metadata !"fpexcept.strict") #0
   ret <1 x float> %sin
 }
 
-define <2 x double> @constrained_vector_sin_v2f64() {
+define <2 x double> @constrained_vector_sin_v2f64() #0 {
 ; S390X-LABEL: constrained_vector_sin_v2f64:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r14, %r15, 112(%r15)
@@ -2010,15 +1993,14 @@ define <2 x double> @constrained_vector_sin_v2f64() {
 ; S390X-NEXT:    std %f8, 160(%r15) # 8-byte Folded Spill
 ; S390X-NEXT:    .cfi_offset %f8, -168
 ; S390X-NEXT:    larl %r1, .LCPI41_0
-; S390X-NEXT:    ldeb %f0, 0(%r1)
+; S390X-NEXT:    ld %f0, 0(%r1)
 ; S390X-NEXT:    brasl %r14, sin@PLT
 ; S390X-NEXT:    larl %r1, .LCPI41_1
-; S390X-NEXT:    ld %f1, 0(%r1)
+; S390X-NEXT:    ldeb %f1, 0(%r1)
 ; S390X-NEXT:    ldr %f8, %f0
 ; S390X-NEXT:    ldr %f0, %f1
 ; S390X-NEXT:    brasl %r14, sin@PLT
-; S390X-NEXT:    ldr %f2, %f0
-; S390X-NEXT:    ldr %f0, %f8
+; S390X-NEXT:    ldr %f2, %f8
 ; S390X-NEXT:    ld %f8, 160(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    lmg %r14, %r15, 280(%r15)
 ; S390X-NEXT:    br %r14
@@ -2047,11 +2029,11 @@ entry:
   %sin = call <2 x double> @llvm.experimental.constrained.sin.v2f64(
                              <2 x double> <double 42.0, double 42.1>,
                              metadata !"round.dynamic",
-                             metadata !"fpexcept.strict")
+                             metadata !"fpexcept.strict") #0
   ret <2 x double> %sin
 }
 
-define <3 x float> @constrained_vector_sin_v3f32() {
+define <3 x float> @constrained_vector_sin_v3f32() #0 {
 ; S390X-LABEL: constrained_vector_sin_v3f32:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r14, %r15, 112(%r15)
@@ -2076,9 +2058,8 @@ define <3 x float> @constrained_vector_sin_v3f32() {
 ; S390X-NEXT:    ler %f9, %f0
 ; S390X-NEXT:    ler %f0, %f1
 ; S390X-NEXT:    brasl %r14, sinf@PLT
-; S390X-NEXT:    ler %f4, %f0
-; S390X-NEXT:    ler %f0, %f8
 ; S390X-NEXT:    ler %f2, %f9
+; S390X-NEXT:    ler %f4, %f8
 ; S390X-NEXT:    ld %f8, 168(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f9, 160(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    lmg %r14, %r15, 288(%r15)
@@ -2116,11 +2097,11 @@ entry:
   %sin = call <3 x float> @llvm.experimental.constrained.sin.v3f32(
                               <3 x float> <float 42.0, float 43.0, float 44.0>,
                               metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #0
   ret <3 x float> %sin
 }
 
-define void @constrained_vector_sin_v3f64(<3 x double>* %a) {
+define void @constrained_vector_sin_v3f64(<3 x double>* %a) #0 {
 ; S390X-LABEL: constrained_vector_sin_v3f64:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r13, %r15, 104(%r15)
@@ -2136,8 +2117,8 @@ define void @constrained_vector_sin_v3f64(<3 x double>* %a) {
 ; S390X-NEXT:    .cfi_offset %f9, -176
 ; S390X-NEXT:    .cfi_offset %f10, -184
 ; S390X-NEXT:    lgr %r13, %r2
-; S390X-NEXT:    ld %f8, 16(%r2)
-; S390X-NEXT:    ld %f0, 0(%r2)
+; S390X-NEXT:    ld %f8, 0(%r2)
+; S390X-NEXT:    ld %f0, 16(%r2)
 ; S390X-NEXT:    ld %f9, 8(%r2)
 ; S390X-NEXT:    brasl %r14, sin@PLT
 ; S390X-NEXT:    ldr %f10, %f0
@@ -2146,9 +2127,9 @@ define void @constrained_vector_sin_v3f64(<3 x double>* %a) {
 ; S390X-NEXT:    ldr %f9, %f0
 ; S390X-NEXT:    ldr %f0, %f8
 ; S390X-NEXT:    brasl %r14, sin@PLT
-; S390X-NEXT:    std %f0, 16(%r13)
+; S390X-NEXT:    std %f0, 0(%r13)
 ; S390X-NEXT:    std %f9, 8(%r13)
-; S390X-NEXT:    std %f10, 0(%r13)
+; S390X-NEXT:    std %f10, 16(%r13)
 ; S390X-NEXT:    ld %f8, 176(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f9, 168(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f10, 160(%r15) # 8-byte Folded Reload
@@ -2194,12 +2175,12 @@ entry:
   %sin = call <3 x double> @llvm.experimental.constrained.sin.v3f64(
                           <3 x double> %b,
                           metadata !"round.dynamic",
-                          metadata !"fpexcept.strict")
+                          metadata !"fpexcept.strict") #0
   store <3 x double> %sin, <3 x double>* %a
   ret void
 }
 
-define <4 x double> @constrained_vector_sin_v4f64() {
+define <4 x double> @constrained_vector_sin_v4f64() #0 {
 ; S390X-LABEL: constrained_vector_sin_v4f64:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r14, %r15, 112(%r15)
@@ -2214,7 +2195,7 @@ define <4 x double> @constrained_vector_sin_v4f64() {
 ; S390X-NEXT:    .cfi_offset %f9, -176
 ; S390X-NEXT:    .cfi_offset %f10, -184
 ; S390X-NEXT:    larl %r1, .LCPI44_0
-; S390X-NEXT:    ldeb %f0, 0(%r1)
+; S390X-NEXT:    ld %f0, 0(%r1)
 ; S390X-NEXT:    brasl %r14, sin@PLT
 ; S390X-NEXT:    larl %r1, .LCPI44_1
 ; S390X-NEXT:    ld %f1, 0(%r1)
@@ -2227,14 +2208,13 @@ define <4 x double> @constrained_vector_sin_v4f64() {
 ; S390X-NEXT:    ldr %f0, %f1
 ; S390X-NEXT:    brasl %r14, sin@PLT
 ; S390X-NEXT:    larl %r1, .LCPI44_3
-; S390X-NEXT:    ld %f1, 0(%r1)
+; S390X-NEXT:    ldeb %f1, 0(%r1)
 ; S390X-NEXT:    ldr %f10, %f0
 ; S390X-NEXT:    ldr %f0, %f1
 ; S390X-NEXT:    brasl %r14, sin@PLT
-; S390X-NEXT:    ldr %f6, %f0
-; S390X-NEXT:    ldr %f0, %f8
-; S390X-NEXT:    ldr %f2, %f9
-; S390X-NEXT:    ldr %f4, %f10
+; S390X-NEXT:    ldr %f2, %f10
+; S390X-NEXT:    ldr %f4, %f9
+; S390X-NEXT:    ldr %f6, %f8
 ; S390X-NEXT:    ld %f8, 176(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f9, 168(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f10, 160(%r15) # 8-byte Folded Reload
@@ -2279,11 +2259,11 @@ entry:
                              <4 x double> <double 42.0, double 42.1,
                                            double 42.2, double 42.3>,
                              metadata !"round.dynamic",
-                             metadata !"fpexcept.strict")
+                             metadata !"fpexcept.strict") #0
   ret <4 x double> %sin
 }
 
-define <1 x float> @constrained_vector_cos_v1f32() {
+define <1 x float> @constrained_vector_cos_v1f32() #0 {
 ; S390X-LABEL: constrained_vector_cos_v1f32:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r14, %r15, 112(%r15)
@@ -2315,11 +2295,11 @@ entry:
   %cos = call <1 x float> @llvm.experimental.constrained.cos.v1f32(
                              <1 x float> <float 42.0>,
                              metadata !"round.dynamic",
-                             metadata !"fpexcept.strict")
+                             metadata !"fpexcept.strict") #0
   ret <1 x float> %cos
 }
 
-define <2 x double> @constrained_vector_cos_v2f64() {
+define <2 x double> @constrained_vector_cos_v2f64() #0 {
 ; S390X-LABEL: constrained_vector_cos_v2f64:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r14, %r15, 112(%r15)
@@ -2330,15 +2310,14 @@ define <2 x double> @constrained_vector_cos_v2f64() {
 ; S390X-NEXT:    std %f8, 160(%r15) # 8-byte Folded Spill
 ; S390X-NEXT:    .cfi_offset %f8, -168
 ; S390X-NEXT:    larl %r1, .LCPI46_0
-; S390X-NEXT:    ldeb %f0, 0(%r1)
+; S390X-NEXT:    ld %f0, 0(%r1)
 ; S390X-NEXT:    brasl %r14, cos@PLT
 ; S390X-NEXT:    larl %r1, .LCPI46_1
-; S390X-NEXT:    ld %f1, 0(%r1)
+; S390X-NEXT:    ldeb %f1, 0(%r1)
 ; S390X-NEXT:    ldr %f8, %f0
 ; S390X-NEXT:    ldr %f0, %f1
 ; S390X-NEXT:    brasl %r14, cos@PLT
-; S390X-NEXT:    ldr %f2, %f0
-; S390X-NEXT:    ldr %f0, %f8
+; S390X-NEXT:    ldr %f2, %f8
 ; S390X-NEXT:    ld %f8, 160(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    lmg %r14, %r15, 280(%r15)
 ; S390X-NEXT:    br %r14
@@ -2367,11 +2346,11 @@ entry:
   %cos = call <2 x double> @llvm.experimental.constrained.cos.v2f64(
                              <2 x double> <double 42.0, double 42.1>,
                              metadata !"round.dynamic",
-                             metadata !"fpexcept.strict")
+                             metadata !"fpexcept.strict") #0
   ret <2 x double> %cos
 }
 
-define <3 x float> @constrained_vector_cos_v3f32() {
+define <3 x float> @constrained_vector_cos_v3f32() #0 {
 ; S390X-LABEL: constrained_vector_cos_v3f32:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r14, %r15, 112(%r15)
@@ -2396,9 +2375,8 @@ define <3 x float> @constrained_vector_cos_v3f32() {
 ; S390X-NEXT:    ler %f9, %f0
 ; S390X-NEXT:    ler %f0, %f1
 ; S390X-NEXT:    brasl %r14, cosf@PLT
-; S390X-NEXT:    ler %f4, %f0
-; S390X-NEXT:    ler %f0, %f8
 ; S390X-NEXT:    ler %f2, %f9
+; S390X-NEXT:    ler %f4, %f8
 ; S390X-NEXT:    ld %f8, 168(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f9, 160(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    lmg %r14, %r15, 288(%r15)
@@ -2436,11 +2414,11 @@ entry:
   %cos = call <3 x float> @llvm.experimental.constrained.cos.v3f32(
                               <3 x float> <float 42.0, float 43.0, float 44.0>,
                               metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #0
   ret <3 x float> %cos
 }
 
-define void @constrained_vector_cos_v3f64(<3 x double>* %a) {
+define void @constrained_vector_cos_v3f64(<3 x double>* %a) #0 {
 ; S390X-LABEL: constrained_vector_cos_v3f64:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r13, %r15, 104(%r15)
@@ -2456,8 +2434,8 @@ define void @constrained_vector_cos_v3f64(<3 x double>* %a) {
 ; S390X-NEXT:    .cfi_offset %f9, -176
 ; S390X-NEXT:    .cfi_offset %f10, -184
 ; S390X-NEXT:    lgr %r13, %r2
-; S390X-NEXT:    ld %f8, 16(%r2)
-; S390X-NEXT:    ld %f0, 0(%r2)
+; S390X-NEXT:    ld %f8, 0(%r2)
+; S390X-NEXT:    ld %f0, 16(%r2)
 ; S390X-NEXT:    ld %f9, 8(%r2)
 ; S390X-NEXT:    brasl %r14, cos@PLT
 ; S390X-NEXT:    ldr %f10, %f0
@@ -2466,9 +2444,9 @@ define void @constrained_vector_cos_v3f64(<3 x double>* %a) {
 ; S390X-NEXT:    ldr %f9, %f0
 ; S390X-NEXT:    ldr %f0, %f8
 ; S390X-NEXT:    brasl %r14, cos@PLT
-; S390X-NEXT:    std %f0, 16(%r13)
+; S390X-NEXT:    std %f0, 0(%r13)
 ; S390X-NEXT:    std %f9, 8(%r13)
-; S390X-NEXT:    std %f10, 0(%r13)
+; S390X-NEXT:    std %f10, 16(%r13)
 ; S390X-NEXT:    ld %f8, 176(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f9, 168(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f10, 160(%r15) # 8-byte Folded Reload
@@ -2514,12 +2492,12 @@ entry:
   %cos = call <3 x double> @llvm.experimental.constrained.cos.v3f64(
                           <3 x double> %b,
                           metadata !"round.dynamic",
-                          metadata !"fpexcept.strict")
+                          metadata !"fpexcept.strict") #0
   store <3 x double> %cos, <3 x double>* %a
   ret void
 }
 
-define <4 x double> @constrained_vector_cos_v4f64() {
+define <4 x double> @constrained_vector_cos_v4f64() #0 {
 ; S390X-LABEL: constrained_vector_cos_v4f64:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r14, %r15, 112(%r15)
@@ -2534,7 +2512,7 @@ define <4 x double> @constrained_vector_cos_v4f64() {
 ; S390X-NEXT:    .cfi_offset %f9, -176
 ; S390X-NEXT:    .cfi_offset %f10, -184
 ; S390X-NEXT:    larl %r1, .LCPI49_0
-; S390X-NEXT:    ldeb %f0, 0(%r1)
+; S390X-NEXT:    ld %f0, 0(%r1)
 ; S390X-NEXT:    brasl %r14, cos@PLT
 ; S390X-NEXT:    larl %r1, .LCPI49_1
 ; S390X-NEXT:    ld %f1, 0(%r1)
@@ -2547,14 +2525,13 @@ define <4 x double> @constrained_vector_cos_v4f64() {
 ; S390X-NEXT:    ldr %f0, %f1
 ; S390X-NEXT:    brasl %r14, cos@PLT
 ; S390X-NEXT:    larl %r1, .LCPI49_3
-; S390X-NEXT:    ld %f1, 0(%r1)
+; S390X-NEXT:    ldeb %f1, 0(%r1)
 ; S390X-NEXT:    ldr %f10, %f0
 ; S390X-NEXT:    ldr %f0, %f1
 ; S390X-NEXT:    brasl %r14, cos@PLT
-; S390X-NEXT:    ldr %f6, %f0
-; S390X-NEXT:    ldr %f0, %f8
-; S390X-NEXT:    ldr %f2, %f9
-; S390X-NEXT:    ldr %f4, %f10
+; S390X-NEXT:    ldr %f2, %f10
+; S390X-NEXT:    ldr %f4, %f9
+; S390X-NEXT:    ldr %f6, %f8
 ; S390X-NEXT:    ld %f8, 176(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f9, 168(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f10, 160(%r15) # 8-byte Folded Reload
@@ -2599,11 +2576,11 @@ entry:
                              <4 x double> <double 42.0, double 42.1,
                                            double 42.2, double 42.3>,
                              metadata !"round.dynamic",
-                             metadata !"fpexcept.strict")
+                             metadata !"fpexcept.strict") #0
   ret <4 x double> %cos
 }
 
-define <1 x float> @constrained_vector_exp_v1f32() {
+define <1 x float> @constrained_vector_exp_v1f32() #0 {
 ; S390X-LABEL: constrained_vector_exp_v1f32:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r14, %r15, 112(%r15)
@@ -2635,11 +2612,11 @@ entry:
   %exp = call <1 x float> @llvm.experimental.constrained.exp.v1f32(
                              <1 x float> <float 42.0>,
                              metadata !"round.dynamic",
-                             metadata !"fpexcept.strict")
+                             metadata !"fpexcept.strict") #0
   ret <1 x float> %exp
 }
 
-define <2 x double> @constrained_vector_exp_v2f64() {
+define <2 x double> @constrained_vector_exp_v2f64() #0 {
 ; S390X-LABEL: constrained_vector_exp_v2f64:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r14, %r15, 112(%r15)
@@ -2650,15 +2627,14 @@ define <2 x double> @constrained_vector_exp_v2f64() {
 ; S390X-NEXT:    std %f8, 160(%r15) # 8-byte Folded Spill
 ; S390X-NEXT:    .cfi_offset %f8, -168
 ; S390X-NEXT:    larl %r1, .LCPI51_0
-; S390X-NEXT:    ldeb %f0, 0(%r1)
+; S390X-NEXT:    ld %f0, 0(%r1)
 ; S390X-NEXT:    brasl %r14, exp@PLT
 ; S390X-NEXT:    larl %r1, .LCPI51_1
-; S390X-NEXT:    ld %f1, 0(%r1)
+; S390X-NEXT:    ldeb %f1, 0(%r1)
 ; S390X-NEXT:    ldr %f8, %f0
 ; S390X-NEXT:    ldr %f0, %f1
 ; S390X-NEXT:    brasl %r14, exp@PLT
-; S390X-NEXT:    ldr %f2, %f0
-; S390X-NEXT:    ldr %f0, %f8
+; S390X-NEXT:    ldr %f2, %f8
 ; S390X-NEXT:    ld %f8, 160(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    lmg %r14, %r15, 280(%r15)
 ; S390X-NEXT:    br %r14
@@ -2687,11 +2663,11 @@ entry:
   %exp = call <2 x double> @llvm.experimental.constrained.exp.v2f64(
                              <2 x double> <double 42.0, double 42.1>,
                              metadata !"round.dynamic",
-                             metadata !"fpexcept.strict")
+                             metadata !"fpexcept.strict") #0
   ret <2 x double> %exp
 }
 
-define <3 x float> @constrained_vector_exp_v3f32() {
+define <3 x float> @constrained_vector_exp_v3f32() #0 {
 ; S390X-LABEL: constrained_vector_exp_v3f32:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r14, %r15, 112(%r15)
@@ -2716,9 +2692,8 @@ define <3 x float> @constrained_vector_exp_v3f32() {
 ; S390X-NEXT:    ler %f9, %f0
 ; S390X-NEXT:    ler %f0, %f1
 ; S390X-NEXT:    brasl %r14, expf@PLT
-; S390X-NEXT:    ler %f4, %f0
-; S390X-NEXT:    ler %f0, %f8
 ; S390X-NEXT:    ler %f2, %f9
+; S390X-NEXT:    ler %f4, %f8
 ; S390X-NEXT:    ld %f8, 168(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f9, 160(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    lmg %r14, %r15, 288(%r15)
@@ -2756,11 +2731,11 @@ entry:
   %exp = call <3 x float> @llvm.experimental.constrained.exp.v3f32(
                               <3 x float> <float 42.0, float 43.0, float 44.0>,
                               metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #0
   ret <3 x float> %exp
 }
 
-define void @constrained_vector_exp_v3f64(<3 x double>* %a) {
+define void @constrained_vector_exp_v3f64(<3 x double>* %a) #0 {
 ; S390X-LABEL: constrained_vector_exp_v3f64:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r13, %r15, 104(%r15)
@@ -2776,8 +2751,8 @@ define void @constrained_vector_exp_v3f64(<3 x double>* %a) {
 ; S390X-NEXT:    .cfi_offset %f9, -176
 ; S390X-NEXT:    .cfi_offset %f10, -184
 ; S390X-NEXT:    lgr %r13, %r2
-; S390X-NEXT:    ld %f8, 16(%r2)
-; S390X-NEXT:    ld %f0, 0(%r2)
+; S390X-NEXT:    ld %f8, 0(%r2)
+; S390X-NEXT:    ld %f0, 16(%r2)
 ; S390X-NEXT:    ld %f9, 8(%r2)
 ; S390X-NEXT:    brasl %r14, exp@PLT
 ; S390X-NEXT:    ldr %f10, %f0
@@ -2786,9 +2761,9 @@ define void @constrained_vector_exp_v3f64(<3 x double>* %a) {
 ; S390X-NEXT:    ldr %f9, %f0
 ; S390X-NEXT:    ldr %f0, %f8
 ; S390X-NEXT:    brasl %r14, exp@PLT
-; S390X-NEXT:    std %f0, 16(%r13)
+; S390X-NEXT:    std %f0, 0(%r13)
 ; S390X-NEXT:    std %f9, 8(%r13)
-; S390X-NEXT:    std %f10, 0(%r13)
+; S390X-NEXT:    std %f10, 16(%r13)
 ; S390X-NEXT:    ld %f8, 176(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f9, 168(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f10, 160(%r15) # 8-byte Folded Reload
@@ -2834,12 +2809,12 @@ entry:
   %exp = call <3 x double> @llvm.experimental.constrained.exp.v3f64(
                           <3 x double> %b,
                           metadata !"round.dynamic",
-                          metadata !"fpexcept.strict")
+                          metadata !"fpexcept.strict") #0
   store <3 x double> %exp, <3 x double>* %a
   ret void
 }
 
-define <4 x double> @constrained_vector_exp_v4f64() {
+define <4 x double> @constrained_vector_exp_v4f64() #0 {
 ; S390X-LABEL: constrained_vector_exp_v4f64:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r14, %r15, 112(%r15)
@@ -2854,7 +2829,7 @@ define <4 x double> @constrained_vector_exp_v4f64() {
 ; S390X-NEXT:    .cfi_offset %f9, -176
 ; S390X-NEXT:    .cfi_offset %f10, -184
 ; S390X-NEXT:    larl %r1, .LCPI54_0
-; S390X-NEXT:    ldeb %f0, 0(%r1)
+; S390X-NEXT:    ld %f0, 0(%r1)
 ; S390X-NEXT:    brasl %r14, exp@PLT
 ; S390X-NEXT:    larl %r1, .LCPI54_1
 ; S390X-NEXT:    ld %f1, 0(%r1)
@@ -2867,14 +2842,13 @@ define <4 x double> @constrained_vector_exp_v4f64() {
 ; S390X-NEXT:    ldr %f0, %f1
 ; S390X-NEXT:    brasl %r14, exp@PLT
 ; S390X-NEXT:    larl %r1, .LCPI54_3
-; S390X-NEXT:    ld %f1, 0(%r1)
+; S390X-NEXT:    ldeb %f1, 0(%r1)
 ; S390X-NEXT:    ldr %f10, %f0
 ; S390X-NEXT:    ldr %f0, %f1
 ; S390X-NEXT:    brasl %r14, exp@PLT
-; S390X-NEXT:    ldr %f6, %f0
-; S390X-NEXT:    ldr %f0, %f8
-; S390X-NEXT:    ldr %f2, %f9
-; S390X-NEXT:    ldr %f4, %f10
+; S390X-NEXT:    ldr %f2, %f10
+; S390X-NEXT:    ldr %f4, %f9
+; S390X-NEXT:    ldr %f6, %f8
 ; S390X-NEXT:    ld %f8, 176(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f9, 168(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f10, 160(%r15) # 8-byte Folded Reload
@@ -2919,11 +2893,11 @@ entry:
                              <4 x double> <double 42.0, double 42.1,
                                            double 42.2, double 42.3>,
                              metadata !"round.dynamic",
-                             metadata !"fpexcept.strict")
+                             metadata !"fpexcept.strict") #0
   ret <4 x double> %exp
 }
 
-define <1 x float> @constrained_vector_exp2_v1f32() {
+define <1 x float> @constrained_vector_exp2_v1f32() #0 {
 ; S390X-LABEL: constrained_vector_exp2_v1f32:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r14, %r15, 112(%r15)
@@ -2955,11 +2929,11 @@ entry:
   %exp2 = call <1 x float> @llvm.experimental.constrained.exp2.v1f32(
                              <1 x float> <float 42.0>,
                              metadata !"round.dynamic",
-                             metadata !"fpexcept.strict")
+                             metadata !"fpexcept.strict") #0
   ret <1 x float> %exp2
 }
 
-define <2 x double> @constrained_vector_exp2_v2f64() {
+define <2 x double> @constrained_vector_exp2_v2f64() #0 {
 ; S390X-LABEL: constrained_vector_exp2_v2f64:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r14, %r15, 112(%r15)
@@ -2970,15 +2944,14 @@ define <2 x double> @constrained_vector_exp2_v2f64() {
 ; S390X-NEXT:    std %f8, 160(%r15) # 8-byte Folded Spill
 ; S390X-NEXT:    .cfi_offset %f8, -168
 ; S390X-NEXT:    larl %r1, .LCPI56_0
-; S390X-NEXT:    ld %f0, 0(%r1)
+; S390X-NEXT:    ldeb %f0, 0(%r1)
 ; S390X-NEXT:    brasl %r14, exp2@PLT
 ; S390X-NEXT:    larl %r1, .LCPI56_1
-; S390X-NEXT:    ldeb %f1, 0(%r1)
+; S390X-NEXT:    ld %f1, 0(%r1)
 ; S390X-NEXT:    ldr %f8, %f0
 ; S390X-NEXT:    ldr %f0, %f1
 ; S390X-NEXT:    brasl %r14, exp2@PLT
-; S390X-NEXT:    ldr %f2, %f0
-; S390X-NEXT:    ldr %f0, %f8
+; S390X-NEXT:    ldr %f2, %f8
 ; S390X-NEXT:    ld %f8, 160(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    lmg %r14, %r15, 280(%r15)
 ; S390X-NEXT:    br %r14
@@ -3007,11 +2980,11 @@ entry:
   %exp2 = call <2 x double> @llvm.experimental.constrained.exp2.v2f64(
                               <2 x double> <double 42.1, double 42.0>,
                               metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #0
   ret <2 x double> %exp2
 }
 
-define <3 x float> @constrained_vector_exp2_v3f32() {
+define <3 x float> @constrained_vector_exp2_v3f32() #0 {
 ; S390X-LABEL: constrained_vector_exp2_v3f32:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r14, %r15, 112(%r15)
@@ -3036,9 +3009,8 @@ define <3 x float> @constrained_vector_exp2_v3f32() {
 ; S390X-NEXT:    ler %f9, %f0
 ; S390X-NEXT:    ler %f0, %f1
 ; S390X-NEXT:    brasl %r14, exp2f@PLT
-; S390X-NEXT:    ler %f4, %f0
-; S390X-NEXT:    ler %f0, %f8
 ; S390X-NEXT:    ler %f2, %f9
+; S390X-NEXT:    ler %f4, %f8
 ; S390X-NEXT:    ld %f8, 168(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f9, 160(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    lmg %r14, %r15, 288(%r15)
@@ -3076,11 +3048,11 @@ entry:
   %exp2 = call <3 x float> @llvm.experimental.constrained.exp2.v3f32(
                               <3 x float> <float 42.0, float 43.0, float 44.0>,
                               metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #0
   ret <3 x float> %exp2
 }
 
-define void @constrained_vector_exp2_v3f64(<3 x double>* %a) {
+define void @constrained_vector_exp2_v3f64(<3 x double>* %a) #0 {
 ; S390X-LABEL: constrained_vector_exp2_v3f64:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r13, %r15, 104(%r15)
@@ -3096,8 +3068,8 @@ define void @constrained_vector_exp2_v3f64(<3 x double>* %a) {
 ; S390X-NEXT:    .cfi_offset %f9, -176
 ; S390X-NEXT:    .cfi_offset %f10, -184
 ; S390X-NEXT:    lgr %r13, %r2
-; S390X-NEXT:    ld %f8, 16(%r2)
-; S390X-NEXT:    ld %f0, 0(%r2)
+; S390X-NEXT:    ld %f8, 0(%r2)
+; S390X-NEXT:    ld %f0, 16(%r2)
 ; S390X-NEXT:    ld %f9, 8(%r2)
 ; S390X-NEXT:    brasl %r14, exp2@PLT
 ; S390X-NEXT:    ldr %f10, %f0
@@ -3106,9 +3078,9 @@ define void @constrained_vector_exp2_v3f64(<3 x double>* %a) {
 ; S390X-NEXT:    ldr %f9, %f0
 ; S390X-NEXT:    ldr %f0, %f8
 ; S390X-NEXT:    brasl %r14, exp2@PLT
-; S390X-NEXT:    std %f0, 16(%r13)
+; S390X-NEXT:    std %f0, 0(%r13)
 ; S390X-NEXT:    std %f9, 8(%r13)
-; S390X-NEXT:    std %f10, 0(%r13)
+; S390X-NEXT:    std %f10, 16(%r13)
 ; S390X-NEXT:    ld %f8, 176(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f9, 168(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f10, 160(%r15) # 8-byte Folded Reload
@@ -3154,12 +3126,12 @@ entry:
   %exp2 = call <3 x double> @llvm.experimental.constrained.exp2.v3f64(
                           <3 x double> %b,
                           metadata !"round.dynamic",
-                          metadata !"fpexcept.strict")
+                          metadata !"fpexcept.strict") #0
   store <3 x double> %exp2, <3 x double>* %a
   ret void
 }
 
-define <4 x double> @constrained_vector_exp2_v4f64() {
+define <4 x double> @constrained_vector_exp2_v4f64() #0 {
 ; S390X-LABEL: constrained_vector_exp2_v4f64:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r14, %r15, 112(%r15)
@@ -3191,10 +3163,9 @@ define <4 x double> @constrained_vector_exp2_v4f64() {
 ; S390X-NEXT:    ldr %f10, %f0
 ; S390X-NEXT:    ldr %f0, %f1
 ; S390X-NEXT:    brasl %r14, exp2@PLT
-; S390X-NEXT:    ldr %f6, %f0
-; S390X-NEXT:    ldr %f0, %f8
-; S390X-NEXT:    ldr %f2, %f9
-; S390X-NEXT:    ldr %f4, %f10
+; S390X-NEXT:    ldr %f2, %f10
+; S390X-NEXT:    ldr %f4, %f9
+; S390X-NEXT:    ldr %f6, %f8
 ; S390X-NEXT:    ld %f8, 176(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f9, 168(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f10, 160(%r15) # 8-byte Folded Reload
@@ -3239,11 +3210,11 @@ entry:
                               <4 x double> <double 42.1, double 42.2,
                                             double 42.3, double 42.4>,
                               metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #0
   ret <4 x double> %exp2
 }
 
-define <1 x float> @constrained_vector_log_v1f32() {
+define <1 x float> @constrained_vector_log_v1f32() #0 {
 ; S390X-LABEL: constrained_vector_log_v1f32:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r14, %r15, 112(%r15)
@@ -3275,11 +3246,11 @@ entry:
   %log = call <1 x float> @llvm.experimental.constrained.log.v1f32(
                              <1 x float> <float 42.0>,
                              metadata !"round.dynamic",
-                             metadata !"fpexcept.strict")
+                             metadata !"fpexcept.strict") #0
   ret <1 x float> %log
 }
 
-define <2 x double> @constrained_vector_log_v2f64() {
+define <2 x double> @constrained_vector_log_v2f64() #0 {
 ; S390X-LABEL: constrained_vector_log_v2f64:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r14, %r15, 112(%r15)
@@ -3290,15 +3261,14 @@ define <2 x double> @constrained_vector_log_v2f64() {
 ; S390X-NEXT:    std %f8, 160(%r15) # 8-byte Folded Spill
 ; S390X-NEXT:    .cfi_offset %f8, -168
 ; S390X-NEXT:    larl %r1, .LCPI61_0
-; S390X-NEXT:    ldeb %f0, 0(%r1)
+; S390X-NEXT:    ld %f0, 0(%r1)
 ; S390X-NEXT:    brasl %r14, log@PLT
 ; S390X-NEXT:    larl %r1, .LCPI61_1
-; S390X-NEXT:    ld %f1, 0(%r1)
+; S390X-NEXT:    ldeb %f1, 0(%r1)
 ; S390X-NEXT:    ldr %f8, %f0
 ; S390X-NEXT:    ldr %f0, %f1
 ; S390X-NEXT:    brasl %r14, log@PLT
-; S390X-NEXT:    ldr %f2, %f0
-; S390X-NEXT:    ldr %f0, %f8
+; S390X-NEXT:    ldr %f2, %f8
 ; S390X-NEXT:    ld %f8, 160(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    lmg %r14, %r15, 280(%r15)
 ; S390X-NEXT:    br %r14
@@ -3327,11 +3297,11 @@ entry:
   %log = call <2 x double> @llvm.experimental.constrained.log.v2f64(
                              <2 x double> <double 42.0, double 42.1>,
                              metadata !"round.dynamic",
-                             metadata !"fpexcept.strict")
+                             metadata !"fpexcept.strict") #0
   ret <2 x double> %log
 }
 
-define <3 x float> @constrained_vector_log_v3f32() {
+define <3 x float> @constrained_vector_log_v3f32() #0 {
 ; S390X-LABEL: constrained_vector_log_v3f32:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r14, %r15, 112(%r15)
@@ -3356,9 +3326,8 @@ define <3 x float> @constrained_vector_log_v3f32() {
 ; S390X-NEXT:    ler %f9, %f0
 ; S390X-NEXT:    ler %f0, %f1
 ; S390X-NEXT:    brasl %r14, logf@PLT
-; S390X-NEXT:    ler %f4, %f0
-; S390X-NEXT:    ler %f0, %f8
 ; S390X-NEXT:    ler %f2, %f9
+; S390X-NEXT:    ler %f4, %f8
 ; S390X-NEXT:    ld %f8, 168(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f9, 160(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    lmg %r14, %r15, 288(%r15)
@@ -3396,11 +3365,11 @@ entry:
   %log = call <3 x float> @llvm.experimental.constrained.log.v3f32(
                               <3 x float> <float 42.0, float 43.0, float 44.0>,
                               metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #0
   ret <3 x float> %log
 }
 
-define void @constrained_vector_log_v3f64(<3 x double>* %a) {
+define void @constrained_vector_log_v3f64(<3 x double>* %a) #0 {
 ; S390X-LABEL: constrained_vector_log_v3f64:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r13, %r15, 104(%r15)
@@ -3416,8 +3385,8 @@ define void @constrained_vector_log_v3f64(<3 x double>* %a) {
 ; S390X-NEXT:    .cfi_offset %f9, -176
 ; S390X-NEXT:    .cfi_offset %f10, -184
 ; S390X-NEXT:    lgr %r13, %r2
-; S390X-NEXT:    ld %f8, 16(%r2)
-; S390X-NEXT:    ld %f0, 0(%r2)
+; S390X-NEXT:    ld %f8, 0(%r2)
+; S390X-NEXT:    ld %f0, 16(%r2)
 ; S390X-NEXT:    ld %f9, 8(%r2)
 ; S390X-NEXT:    brasl %r14, log@PLT
 ; S390X-NEXT:    ldr %f10, %f0
@@ -3426,9 +3395,9 @@ define void @constrained_vector_log_v3f64(<3 x double>* %a) {
 ; S390X-NEXT:    ldr %f9, %f0
 ; S390X-NEXT:    ldr %f0, %f8
 ; S390X-NEXT:    brasl %r14, log@PLT
-; S390X-NEXT:    std %f0, 16(%r13)
+; S390X-NEXT:    std %f0, 0(%r13)
 ; S390X-NEXT:    std %f9, 8(%r13)
-; S390X-NEXT:    std %f10, 0(%r13)
+; S390X-NEXT:    std %f10, 16(%r13)
 ; S390X-NEXT:    ld %f8, 176(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f9, 168(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f10, 160(%r15) # 8-byte Folded Reload
@@ -3474,12 +3443,12 @@ entry:
   %log = call <3 x double> @llvm.experimental.constrained.log.v3f64(
                           <3 x double> %b,
                           metadata !"round.dynamic",
-                          metadata !"fpexcept.strict")
+                          metadata !"fpexcept.strict") #0
   store <3 x double> %log, <3 x double>* %a
   ret void
 }
 
-define <4 x double> @constrained_vector_log_v4f64() {
+define <4 x double> @constrained_vector_log_v4f64() #0 {
 ; S390X-LABEL: constrained_vector_log_v4f64:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r14, %r15, 112(%r15)
@@ -3494,7 +3463,7 @@ define <4 x double> @constrained_vector_log_v4f64() {
 ; S390X-NEXT:    .cfi_offset %f9, -176
 ; S390X-NEXT:    .cfi_offset %f10, -184
 ; S390X-NEXT:    larl %r1, .LCPI64_0
-; S390X-NEXT:    ldeb %f0, 0(%r1)
+; S390X-NEXT:    ld %f0, 0(%r1)
 ; S390X-NEXT:    brasl %r14, log@PLT
 ; S390X-NEXT:    larl %r1, .LCPI64_1
 ; S390X-NEXT:    ld %f1, 0(%r1)
@@ -3507,14 +3476,13 @@ define <4 x double> @constrained_vector_log_v4f64() {
 ; S390X-NEXT:    ldr %f0, %f1
 ; S390X-NEXT:    brasl %r14, log@PLT
 ; S390X-NEXT:    larl %r1, .LCPI64_3
-; S390X-NEXT:    ld %f1, 0(%r1)
+; S390X-NEXT:    ldeb %f1, 0(%r1)
 ; S390X-NEXT:    ldr %f10, %f0
 ; S390X-NEXT:    ldr %f0, %f1
 ; S390X-NEXT:    brasl %r14, log@PLT
-; S390X-NEXT:    ldr %f6, %f0
-; S390X-NEXT:    ldr %f0, %f8
-; S390X-NEXT:    ldr %f2, %f9
-; S390X-NEXT:    ldr %f4, %f10
+; S390X-NEXT:    ldr %f2, %f10
+; S390X-NEXT:    ldr %f4, %f9
+; S390X-NEXT:    ldr %f6, %f8
 ; S390X-NEXT:    ld %f8, 176(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f9, 168(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f10, 160(%r15) # 8-byte Folded Reload
@@ -3559,11 +3527,11 @@ entry:
                              <4 x double> <double 42.0, double 42.1,
                                            double 42.2, double 42.3>,
                              metadata !"round.dynamic",
-                             metadata !"fpexcept.strict")
+                             metadata !"fpexcept.strict") #0
   ret <4 x double> %log
 }
 
-define <1 x float> @constrained_vector_log10_v1f32() {
+define <1 x float> @constrained_vector_log10_v1f32() #0 {
 ; S390X-LABEL: constrained_vector_log10_v1f32:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r14, %r15, 112(%r15)
@@ -3595,11 +3563,11 @@ entry:
   %log10 = call <1 x float> @llvm.experimental.constrained.log10.v1f32(
                              <1 x float> <float 42.0>,
                              metadata !"round.dynamic",
-                             metadata !"fpexcept.strict")
+                             metadata !"fpexcept.strict") #0
   ret <1 x float> %log10
 }
 
-define <2 x double> @constrained_vector_log10_v2f64() {
+define <2 x double> @constrained_vector_log10_v2f64() #0 {
 ; S390X-LABEL: constrained_vector_log10_v2f64:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r14, %r15, 112(%r15)
@@ -3610,15 +3578,14 @@ define <2 x double> @constrained_vector_log10_v2f64() {
 ; S390X-NEXT:    std %f8, 160(%r15) # 8-byte Folded Spill
 ; S390X-NEXT:    .cfi_offset %f8, -168
 ; S390X-NEXT:    larl %r1, .LCPI66_0
-; S390X-NEXT:    ldeb %f0, 0(%r1)
+; S390X-NEXT:    ld %f0, 0(%r1)
 ; S390X-NEXT:    brasl %r14, log10@PLT
 ; S390X-NEXT:    larl %r1, .LCPI66_1
-; S390X-NEXT:    ld %f1, 0(%r1)
+; S390X-NEXT:    ldeb %f1, 0(%r1)
 ; S390X-NEXT:    ldr %f8, %f0
 ; S390X-NEXT:    ldr %f0, %f1
 ; S390X-NEXT:    brasl %r14, log10@PLT
-; S390X-NEXT:    ldr %f2, %f0
-; S390X-NEXT:    ldr %f0, %f8
+; S390X-NEXT:    ldr %f2, %f8
 ; S390X-NEXT:    ld %f8, 160(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    lmg %r14, %r15, 280(%r15)
 ; S390X-NEXT:    br %r14
@@ -3647,11 +3614,11 @@ entry:
   %log10 = call <2 x double> @llvm.experimental.constrained.log10.v2f64(
                                <2 x double> <double 42.0, double 42.1>,
                                metadata !"round.dynamic",
-                               metadata !"fpexcept.strict")
+                               metadata !"fpexcept.strict") #0
   ret <2 x double> %log10
 }
 
-define <3 x float> @constrained_vector_log10_v3f32() {
+define <3 x float> @constrained_vector_log10_v3f32() #0 {
 ; S390X-LABEL: constrained_vector_log10_v3f32:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r14, %r15, 112(%r15)
@@ -3676,9 +3643,8 @@ define <3 x float> @constrained_vector_log10_v3f32() {
 ; S390X-NEXT:    ler %f9, %f0
 ; S390X-NEXT:    ler %f0, %f1
 ; S390X-NEXT:    brasl %r14, log10f@PLT
-; S390X-NEXT:    ler %f4, %f0
-; S390X-NEXT:    ler %f0, %f8
 ; S390X-NEXT:    ler %f2, %f9
+; S390X-NEXT:    ler %f4, %f8
 ; S390X-NEXT:    ld %f8, 168(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f9, 160(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    lmg %r14, %r15, 288(%r15)
@@ -3716,11 +3682,11 @@ entry:
   %log10 = call <3 x float> @llvm.experimental.constrained.log10.v3f32(
                               <3 x float> <float 42.0, float 43.0, float 44.0>,
                               metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #0
   ret <3 x float> %log10
 }
 
-define void @constrained_vector_log10_v3f64(<3 x double>* %a) {
+define void @constrained_vector_log10_v3f64(<3 x double>* %a) #0 {
 ; S390X-LABEL: constrained_vector_log10_v3f64:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r13, %r15, 104(%r15)
@@ -3736,8 +3702,8 @@ define void @constrained_vector_log10_v3f64(<3 x double>* %a) {
 ; S390X-NEXT:    .cfi_offset %f9, -176
 ; S390X-NEXT:    .cfi_offset %f10, -184
 ; S390X-NEXT:    lgr %r13, %r2
-; S390X-NEXT:    ld %f8, 16(%r2)
-; S390X-NEXT:    ld %f0, 0(%r2)
+; S390X-NEXT:    ld %f8, 0(%r2)
+; S390X-NEXT:    ld %f0, 16(%r2)
 ; S390X-NEXT:    ld %f9, 8(%r2)
 ; S390X-NEXT:    brasl %r14, log10@PLT
 ; S390X-NEXT:    ldr %f10, %f0
@@ -3746,9 +3712,9 @@ define void @constrained_vector_log10_v3f64(<3 x double>* %a) {
 ; S390X-NEXT:    ldr %f9, %f0
 ; S390X-NEXT:    ldr %f0, %f8
 ; S390X-NEXT:    brasl %r14, log10@PLT
-; S390X-NEXT:    std %f0, 16(%r13)
+; S390X-NEXT:    std %f0, 0(%r13)
 ; S390X-NEXT:    std %f9, 8(%r13)
-; S390X-NEXT:    std %f10, 0(%r13)
+; S390X-NEXT:    std %f10, 16(%r13)
 ; S390X-NEXT:    ld %f8, 176(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f9, 168(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f10, 160(%r15) # 8-byte Folded Reload
@@ -3794,12 +3760,12 @@ entry:
   %log10 = call <3 x double> @llvm.experimental.constrained.log10.v3f64(
                           <3 x double> %b,
                           metadata !"round.dynamic",
-                          metadata !"fpexcept.strict")
+                          metadata !"fpexcept.strict") #0
   store <3 x double> %log10, <3 x double>* %a
   ret void
 }
 
-define <4 x double> @constrained_vector_log10_v4f64() {
+define <4 x double> @constrained_vector_log10_v4f64() #0 {
 ; S390X-LABEL: constrained_vector_log10_v4f64:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r14, %r15, 112(%r15)
@@ -3814,7 +3780,7 @@ define <4 x double> @constrained_vector_log10_v4f64() {
 ; S390X-NEXT:    .cfi_offset %f9, -176
 ; S390X-NEXT:    .cfi_offset %f10, -184
 ; S390X-NEXT:    larl %r1, .LCPI69_0
-; S390X-NEXT:    ldeb %f0, 0(%r1)
+; S390X-NEXT:    ld %f0, 0(%r1)
 ; S390X-NEXT:    brasl %r14, log10@PLT
 ; S390X-NEXT:    larl %r1, .LCPI69_1
 ; S390X-NEXT:    ld %f1, 0(%r1)
@@ -3827,14 +3793,13 @@ define <4 x double> @constrained_vector_log10_v4f64() {
 ; S390X-NEXT:    ldr %f0, %f1
 ; S390X-NEXT:    brasl %r14, log10@PLT
 ; S390X-NEXT:    larl %r1, .LCPI69_3
-; S390X-NEXT:    ld %f1, 0(%r1)
+; S390X-NEXT:    ldeb %f1, 0(%r1)
 ; S390X-NEXT:    ldr %f10, %f0
 ; S390X-NEXT:    ldr %f0, %f1
 ; S390X-NEXT:    brasl %r14, log10@PLT
-; S390X-NEXT:    ldr %f6, %f0
-; S390X-NEXT:    ldr %f0, %f8
-; S390X-NEXT:    ldr %f2, %f9
-; S390X-NEXT:    ldr %f4, %f10
+; S390X-NEXT:    ldr %f2, %f10
+; S390X-NEXT:    ldr %f4, %f9
+; S390X-NEXT:    ldr %f6, %f8
 ; S390X-NEXT:    ld %f8, 176(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f9, 168(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f10, 160(%r15) # 8-byte Folded Reload
@@ -3879,11 +3844,11 @@ entry:
                                <4 x double> <double 42.0, double 42.1,
                                              double 42.2, double 42.3>,
                                metadata !"round.dynamic",
-                               metadata !"fpexcept.strict")
+                               metadata !"fpexcept.strict") #0
   ret <4 x double> %log10
 }
 
-define <1 x float> @constrained_vector_log2_v1f32() {
+define <1 x float> @constrained_vector_log2_v1f32() #0 {
 ; S390X-LABEL: constrained_vector_log2_v1f32:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r14, %r15, 112(%r15)
@@ -3915,11 +3880,11 @@ entry:
   %log2 = call <1 x float> @llvm.experimental.constrained.log2.v1f32(
                              <1 x float> <float 42.0>,
                              metadata !"round.dynamic",
-                             metadata !"fpexcept.strict")
+                             metadata !"fpexcept.strict") #0
   ret <1 x float> %log2
 }
 
-define <2 x double> @constrained_vector_log2_v2f64() {
+define <2 x double> @constrained_vector_log2_v2f64() #0 {
 ; S390X-LABEL: constrained_vector_log2_v2f64:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r14, %r15, 112(%r15)
@@ -3930,15 +3895,14 @@ define <2 x double> @constrained_vector_log2_v2f64() {
 ; S390X-NEXT:    std %f8, 160(%r15) # 8-byte Folded Spill
 ; S390X-NEXT:    .cfi_offset %f8, -168
 ; S390X-NEXT:    larl %r1, .LCPI71_0
-; S390X-NEXT:    ldeb %f0, 0(%r1)
+; S390X-NEXT:    ld %f0, 0(%r1)
 ; S390X-NEXT:    brasl %r14, log2@PLT
 ; S390X-NEXT:    larl %r1, .LCPI71_1
-; S390X-NEXT:    ld %f1, 0(%r1)
+; S390X-NEXT:    ldeb %f1, 0(%r1)
 ; S390X-NEXT:    ldr %f8, %f0
 ; S390X-NEXT:    ldr %f0, %f1
 ; S390X-NEXT:    brasl %r14, log2@PLT
-; S390X-NEXT:    ldr %f2, %f0
-; S390X-NEXT:    ldr %f0, %f8
+; S390X-NEXT:    ldr %f2, %f8
 ; S390X-NEXT:    ld %f8, 160(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    lmg %r14, %r15, 280(%r15)
 ; S390X-NEXT:    br %r14
@@ -3967,11 +3931,11 @@ entry:
   %log2 = call <2 x double> @llvm.experimental.constrained.log2.v2f64(
                               <2 x double> <double 42.0, double 42.1>,
                               metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #0
   ret <2 x double> %log2
 }
 
-define <3 x float> @constrained_vector_log2_v3f32() {
+define <3 x float> @constrained_vector_log2_v3f32() #0 {
 ; S390X-LABEL: constrained_vector_log2_v3f32:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r14, %r15, 112(%r15)
@@ -3996,9 +3960,8 @@ define <3 x float> @constrained_vector_log2_v3f32() {
 ; S390X-NEXT:    ler %f9, %f0
 ; S390X-NEXT:    ler %f0, %f1
 ; S390X-NEXT:    brasl %r14, log2f@PLT
-; S390X-NEXT:    ler %f4, %f0
-; S390X-NEXT:    ler %f0, %f8
 ; S390X-NEXT:    ler %f2, %f9
+; S390X-NEXT:    ler %f4, %f8
 ; S390X-NEXT:    ld %f8, 168(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f9, 160(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    lmg %r14, %r15, 288(%r15)
@@ -4036,11 +3999,11 @@ entry:
   %log2 = call <3 x float> @llvm.experimental.constrained.log2.v3f32(
                               <3 x float> <float 42.0, float 43.0, float 44.0>,
                               metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #0
   ret <3 x float> %log2
 }
 
-define void @constrained_vector_log2_v3f64(<3 x double>* %a) {
+define void @constrained_vector_log2_v3f64(<3 x double>* %a) #0 {
 ; S390X-LABEL: constrained_vector_log2_v3f64:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r13, %r15, 104(%r15)
@@ -4056,8 +4019,8 @@ define void @constrained_vector_log2_v3f64(<3 x double>* %a) {
 ; S390X-NEXT:    .cfi_offset %f9, -176
 ; S390X-NEXT:    .cfi_offset %f10, -184
 ; S390X-NEXT:    lgr %r13, %r2
-; S390X-NEXT:    ld %f8, 16(%r2)
-; S390X-NEXT:    ld %f0, 0(%r2)
+; S390X-NEXT:    ld %f8, 0(%r2)
+; S390X-NEXT:    ld %f0, 16(%r2)
 ; S390X-NEXT:    ld %f9, 8(%r2)
 ; S390X-NEXT:    brasl %r14, log2@PLT
 ; S390X-NEXT:    ldr %f10, %f0
@@ -4066,9 +4029,9 @@ define void @constrained_vector_log2_v3f64(<3 x double>* %a) {
 ; S390X-NEXT:    ldr %f9, %f0
 ; S390X-NEXT:    ldr %f0, %f8
 ; S390X-NEXT:    brasl %r14, log2@PLT
-; S390X-NEXT:    std %f0, 16(%r13)
+; S390X-NEXT:    std %f0, 0(%r13)
 ; S390X-NEXT:    std %f9, 8(%r13)
-; S390X-NEXT:    std %f10, 0(%r13)
+; S390X-NEXT:    std %f10, 16(%r13)
 ; S390X-NEXT:    ld %f8, 176(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f9, 168(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f10, 160(%r15) # 8-byte Folded Reload
@@ -4114,12 +4077,12 @@ entry:
   %log2 = call <3 x double> @llvm.experimental.constrained.log2.v3f64(
                           <3 x double> %b,
                           metadata !"round.dynamic",
-                          metadata !"fpexcept.strict")
+                          metadata !"fpexcept.strict") #0
   store <3 x double> %log2, <3 x double>* %a
   ret void
 }
 
-define <4 x double> @constrained_vector_log2_v4f64() {
+define <4 x double> @constrained_vector_log2_v4f64() #0 {
 ; S390X-LABEL: constrained_vector_log2_v4f64:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r14, %r15, 112(%r15)
@@ -4134,7 +4097,7 @@ define <4 x double> @constrained_vector_log2_v4f64() {
 ; S390X-NEXT:    .cfi_offset %f9, -176
 ; S390X-NEXT:    .cfi_offset %f10, -184
 ; S390X-NEXT:    larl %r1, .LCPI74_0
-; S390X-NEXT:    ldeb %f0, 0(%r1)
+; S390X-NEXT:    ld %f0, 0(%r1)
 ; S390X-NEXT:    brasl %r14, log2@PLT
 ; S390X-NEXT:    larl %r1, .LCPI74_1
 ; S390X-NEXT:    ld %f1, 0(%r1)
@@ -4147,14 +4110,13 @@ define <4 x double> @constrained_vector_log2_v4f64() {
 ; S390X-NEXT:    ldr %f0, %f1
 ; S390X-NEXT:    brasl %r14, log2@PLT
 ; S390X-NEXT:    larl %r1, .LCPI74_3
-; S390X-NEXT:    ld %f1, 0(%r1)
+; S390X-NEXT:    ldeb %f1, 0(%r1)
 ; S390X-NEXT:    ldr %f10, %f0
 ; S390X-NEXT:    ldr %f0, %f1
 ; S390X-NEXT:    brasl %r14, log2@PLT
-; S390X-NEXT:    ldr %f6, %f0
-; S390X-NEXT:    ldr %f0, %f8
-; S390X-NEXT:    ldr %f2, %f9
-; S390X-NEXT:    ldr %f4, %f10
+; S390X-NEXT:    ldr %f2, %f10
+; S390X-NEXT:    ldr %f4, %f9
+; S390X-NEXT:    ldr %f6, %f8
 ; S390X-NEXT:    ld %f8, 176(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f9, 168(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f10, 160(%r15) # 8-byte Folded Reload
@@ -4199,11 +4161,11 @@ entry:
                               <4 x double> <double 42.0, double 42.1,
                                             double 42.2, double 42.3>,
                               metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #0
   ret <4 x double> %log2
 }
 
-define <1 x float> @constrained_vector_rint_v1f32() {
+define <1 x float> @constrained_vector_rint_v1f32() #0 {
 ; S390X-LABEL: constrained_vector_rint_v1f32:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    larl %r1, .LCPI75_0
@@ -4222,11 +4184,11 @@ entry:
   %rint = call <1 x float> @llvm.experimental.constrained.rint.v1f32(
                              <1 x float> <float 42.0>,
                              metadata !"round.dynamic",
-                             metadata !"fpexcept.strict")
+                             metadata !"fpexcept.strict") #0
   ret <1 x float> %rint
 }
 
-define <2 x double> @constrained_vector_rint_v2f64() {
+define <2 x double> @constrained_vector_rint_v2f64() #0 {
 ; S390X-LABEL: constrained_vector_rint_v2f64:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    larl %r1, .LCPI76_0
@@ -4247,11 +4209,11 @@ entry:
   %rint = call <2 x double> @llvm.experimental.constrained.rint.v2f64(
                         <2 x double> <double 42.1, double 42.0>,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret <2 x double> %rint
 }
 
-define <3 x float> @constrained_vector_rint_v3f32() {
+define <3 x float> @constrained_vector_rint_v3f32() #0 {
 ; S390X-LABEL: constrained_vector_rint_v3f32:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    larl %r1, .LCPI77_0
@@ -4284,44 +4246,44 @@ define <3 x float> @constrained_vector_rint_v3f32() {
   %rint = call <3 x float> @llvm.experimental.constrained.rint.v3f32(
                               <3 x float> <float 42.0, float 43.0, float 44.0>,
                               metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #0
   ret <3 x float> %rint
 }
 
-define void @constrained_vector_rint_v3f64(<3 x double>* %a) {
+define void @constrained_vector_rint_v3f64(<3 x double>* %a) #0 {
 ; S390X-LABEL: constrained_vector_rint_v3f64:
 ; S390X:       # %bb.0: # %entry
-; S390X-NEXT:    ld %f0, 0(%r2)
+; S390X-NEXT:    ld %f0, 16(%r2)
 ; S390X-NEXT:    ld %f1, 8(%r2)
-; S390X-NEXT:    ld %f2, 16(%r2)
+; S390X-NEXT:    ld %f2, 0(%r2)
 ; S390X-NEXT:    fidbr %f0, 0, %f0
 ; S390X-NEXT:    fidbr %f1, 0, %f1
 ; S390X-NEXT:    fidbr %f2, 0, %f2
-; S390X-NEXT:    std %f2, 16(%r2)
+; S390X-NEXT:    std %f2, 0(%r2)
 ; S390X-NEXT:    std %f1, 8(%r2)
-; S390X-NEXT:    std %f0, 0(%r2)
+; S390X-NEXT:    std %f0, 16(%r2)
 ; S390X-NEXT:    br %r14
 ;
 ; SZ13-LABEL: constrained_vector_rint_v3f64:
 ; SZ13:       # %bb.0: # %entry
-; SZ13-NEXT:    vl %v1, 0(%r2), 4
-; SZ13-NEXT:    ld %f0, 16(%r2)
-; SZ13-NEXT:    vfidb %v1, %v1, 0, 0
-; SZ13-NEXT:    fidbra %f0, 0, %f0, 0
-; SZ13-NEXT:    std %f0, 16(%r2)
-; SZ13-NEXT:    vst %v1, 0(%r2), 4
+; SZ13-NEXT:    vl %v0, 0(%r2), 4
+; SZ13-NEXT:    ld %f1, 16(%r2)
+; SZ13-NEXT:    vfidb %v0, %v0, 0, 0
+; SZ13-NEXT:    fidbra %f1, 0, %f1, 0
+; SZ13-NEXT:    vst %v0, 0(%r2), 4
+; SZ13-NEXT:    std %f1, 16(%r2)
 ; SZ13-NEXT:    br %r14
 entry:
   %b = load <3 x double>, <3 x double>* %a
   %rint = call <3 x double> @llvm.experimental.constrained.rint.v3f64(
                           <3 x double> %b,
                           metadata !"round.dynamic",
-                          metadata !"fpexcept.strict")
+                          metadata !"fpexcept.strict") #0
   store <3 x double> %rint, <3 x double>* %a
   ret void
 }
 
-define <4 x double> @constrained_vector_rint_v4f64() {
+define <4 x double> @constrained_vector_rint_v4f64() #0 {
 ; S390X-LABEL: constrained_vector_rint_v4f64:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    larl %r1, .LCPI79_0
@@ -4352,11 +4314,11 @@ entry:
                         <4 x double> <double 42.1, double 42.2,
                                       double 42.3, double 42.4>,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret <4 x double> %rint
 }
 
-define <1 x float> @constrained_vector_nearbyint_v1f32() {
+define <1 x float> @constrained_vector_nearbyint_v1f32() #0 {
 ; S390X-LABEL: constrained_vector_nearbyint_v1f32:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r14, %r15, 112(%r15)
@@ -4381,11 +4343,11 @@ entry:
   %nearby = call <1 x float> @llvm.experimental.constrained.nearbyint.v1f32(
                                <1 x float> <float 42.0>,
                                metadata !"round.dynamic",
-                               metadata !"fpexcept.strict")
+                               metadata !"fpexcept.strict") #0
   ret <1 x float> %nearby
 }
 
-define <2 x double> @constrained_vector_nearbyint_v2f64() {
+define <2 x double> @constrained_vector_nearbyint_v2f64() #0 {
 ; S390X-LABEL: constrained_vector_nearbyint_v2f64:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r14, %r15, 112(%r15)
@@ -4396,15 +4358,14 @@ define <2 x double> @constrained_vector_nearbyint_v2f64() {
 ; S390X-NEXT:    std %f8, 160(%r15) # 8-byte Folded Spill
 ; S390X-NEXT:    .cfi_offset %f8, -168
 ; S390X-NEXT:    larl %r1, .LCPI81_0
-; S390X-NEXT:    ld %f0, 0(%r1)
+; S390X-NEXT:    ldeb %f0, 0(%r1)
 ; S390X-NEXT:    brasl %r14, nearbyint@PLT
 ; S390X-NEXT:    larl %r1, .LCPI81_1
-; S390X-NEXT:    ldeb %f1, 0(%r1)
+; S390X-NEXT:    ld %f1, 0(%r1)
 ; S390X-NEXT:    ldr %f8, %f0
 ; S390X-NEXT:    ldr %f0, %f1
 ; S390X-NEXT:    brasl %r14, nearbyint@PLT
-; S390X-NEXT:    ldr %f2, %f0
-; S390X-NEXT:    ldr %f0, %f8
+; S390X-NEXT:    ldr %f2, %f8
 ; S390X-NEXT:    ld %f8, 160(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    lmg %r14, %r15, 280(%r15)
 ; S390X-NEXT:    br %r14
@@ -4419,11 +4380,11 @@ entry:
   %nearby = call <2 x double> @llvm.experimental.constrained.nearbyint.v2f64(
                                 <2 x double> <double 42.1, double 42.0>,
                                 metadata !"round.dynamic",
-                                metadata !"fpexcept.strict")
+                                metadata !"fpexcept.strict") #0
   ret <2 x double> %nearby
 }
 
-define <3 x float> @constrained_vector_nearbyint_v3f32() {
+define <3 x float> @constrained_vector_nearbyint_v3f32() #0 {
 ; S390X-LABEL: constrained_vector_nearbyint_v3f32:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r14, %r15, 112(%r15)
@@ -4448,9 +4409,8 @@ define <3 x float> @constrained_vector_nearbyint_v3f32() {
 ; S390X-NEXT:    ler %f9, %f0
 ; S390X-NEXT:    ler %f0, %f1
 ; S390X-NEXT:    brasl %r14, nearbyintf@PLT
-; S390X-NEXT:    ler %f4, %f0
-; S390X-NEXT:    ler %f0, %f8
 ; S390X-NEXT:    ler %f2, %f9
+; S390X-NEXT:    ler %f4, %f8
 ; S390X-NEXT:    ld %f8, 168(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f9, 160(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    lmg %r14, %r15, 288(%r15)
@@ -4475,11 +4435,11 @@ entry:
   %nearby = call <3 x float> @llvm.experimental.constrained.nearbyint.v3f32(
                               <3 x float> <float 42.0, float 43.0, float 44.0>,
                               metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #0
   ret <3 x float> %nearby
 }
 
-define void @constrained_vector_nearbyint_v3f64(<3 x double>* %a) {
+define void @constrained_vector_nearbyint_v3f64(<3 x double>* %a) #0 {
 ; S390X-LABEL: constrained_vector_nearbyint_v3f64:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r13, %r15, 104(%r15)
@@ -4495,8 +4455,8 @@ define void @constrained_vector_nearbyint_v3f64(<3 x double>* %a) {
 ; S390X-NEXT:    .cfi_offset %f9, -176
 ; S390X-NEXT:    .cfi_offset %f10, -184
 ; S390X-NEXT:    lgr %r13, %r2
-; S390X-NEXT:    ld %f8, 16(%r2)
-; S390X-NEXT:    ld %f0, 0(%r2)
+; S390X-NEXT:    ld %f8, 0(%r2)
+; S390X-NEXT:    ld %f0, 16(%r2)
 ; S390X-NEXT:    ld %f9, 8(%r2)
 ; S390X-NEXT:    brasl %r14, nearbyint@PLT
 ; S390X-NEXT:    ldr %f10, %f0
@@ -4505,9 +4465,9 @@ define void @constrained_vector_nearbyint_v3f64(<3 x double>* %a) {
 ; S390X-NEXT:    ldr %f9, %f0
 ; S390X-NEXT:    ldr %f0, %f8
 ; S390X-NEXT:    brasl %r14, nearbyint@PLT
-; S390X-NEXT:    std %f0, 16(%r13)
+; S390X-NEXT:    std %f0, 0(%r13)
 ; S390X-NEXT:    std %f9, 8(%r13)
-; S390X-NEXT:    std %f10, 0(%r13)
+; S390X-NEXT:    std %f10, 16(%r13)
 ; S390X-NEXT:    ld %f8, 176(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f9, 168(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f10, 160(%r15) # 8-byte Folded Reload
@@ -4516,24 +4476,24 @@ define void @constrained_vector_nearbyint_v3f64(<3 x double>* %a) {
 ;
 ; SZ13-LABEL: constrained_vector_nearbyint_v3f64:
 ; SZ13:       # %bb.0: # %entry
-; SZ13-NEXT:    vl %v1, 0(%r2), 4
-; SZ13-NEXT:    ld %f0, 16(%r2)
-; SZ13-NEXT:    vfidb %v1, %v1, 4, 0
-; SZ13-NEXT:    fidbra %f0, 0, %f0, 4
-; SZ13-NEXT:    std %f0, 16(%r2)
-; SZ13-NEXT:    vst %v1, 0(%r2), 4
+; SZ13-NEXT:    vl %v0, 0(%r2), 4
+; SZ13-NEXT:    ld %f1, 16(%r2)
+; SZ13-NEXT:    vfidb %v0, %v0, 4, 0
+; SZ13-NEXT:    fidbra %f1, 0, %f1, 4
+; SZ13-NEXT:    vst %v0, 0(%r2), 4
+; SZ13-NEXT:    std %f1, 16(%r2)
 ; SZ13-NEXT:    br %r14
 entry:
   %b = load <3 x double>, <3 x double>* %a
   %nearby = call <3 x double> @llvm.experimental.constrained.nearbyint.v3f64(
                           <3 x double> %b,
                           metadata !"round.dynamic",
-                          metadata !"fpexcept.strict")
+                          metadata !"fpexcept.strict") #0
   store <3 x double> %nearby, <3 x double>* %a
   ret void
 }
 
-define <4 x double> @constrained_vector_nearbyint_v4f64() {
+define <4 x double> @constrained_vector_nearbyint_v4f64() #0 {
 ; S390X-LABEL: constrained_vector_nearbyint_v4f64:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r14, %r15, 112(%r15)
@@ -4565,10 +4525,9 @@ define <4 x double> @constrained_vector_nearbyint_v4f64() {
 ; S390X-NEXT:    ldr %f10, %f0
 ; S390X-NEXT:    ldr %f0, %f1
 ; S390X-NEXT:    brasl %r14, nearbyint@PLT
-; S390X-NEXT:    ldr %f6, %f0
-; S390X-NEXT:    ldr %f0, %f8
-; S390X-NEXT:    ldr %f2, %f9
-; S390X-NEXT:    ldr %f4, %f10
+; S390X-NEXT:    ldr %f2, %f10
+; S390X-NEXT:    ldr %f4, %f9
+; S390X-NEXT:    ldr %f6, %f8
 ; S390X-NEXT:    ld %f8, 176(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f9, 168(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f10, 160(%r15) # 8-byte Folded Reload
@@ -4589,11 +4548,11 @@ entry:
                                 <4 x double> <double 42.1, double 42.2,
                                               double 42.3, double 42.4>,
                                 metadata !"round.dynamic",
-                                metadata !"fpexcept.strict")
+                                metadata !"fpexcept.strict") #0
   ret <4 x double> %nearby
 }
 
-define <1 x float> @constrained_vector_maxnum_v1f32() {
+define <1 x float> @constrained_vector_maxnum_v1f32() #0 {
 ; S390X-LABEL: constrained_vector_maxnum_v1f32:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r14, %r15, 112(%r15)
@@ -4628,12 +4587,11 @@ define <1 x float> @constrained_vector_maxnum_v1f32() {
 entry:
   %max = call <1 x float> @llvm.experimental.constrained.maxnum.v1f32(
                                <1 x float> <float 42.0>, <1 x float> <float 41.0>,
-                               metadata !"round.dynamic",
-                               metadata !"fpexcept.strict")
+                               metadata !"fpexcept.strict") #0
   ret <1 x float> %max
 }
 
-define <2 x double> @constrained_vector_maxnum_v2f64() {
+define <2 x double> @constrained_vector_maxnum_v2f64() #0 {
 ; S390X-LABEL: constrained_vector_maxnum_v2f64:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r14, %r15, 112(%r15)
@@ -4655,8 +4613,7 @@ define <2 x double> @constrained_vector_maxnum_v2f64() {
 ; S390X-NEXT:    ldr %f8, %f0
 ; S390X-NEXT:    ldr %f0, %f1
 ; S390X-NEXT:    brasl %r14, fmax@PLT
-; S390X-NEXT:    ldr %f2, %f0
-; S390X-NEXT:    ldr %f0, %f8
+; S390X-NEXT:    ldr %f2, %f8
 ; S390X-NEXT:    ld %f8, 160(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    lmg %r14, %r15, 280(%r15)
 ; S390X-NEXT:    br %r14
@@ -4689,12 +4646,11 @@ entry:
   %max = call <2 x double> @llvm.experimental.constrained.maxnum.v2f64(
                                 <2 x double> <double 43.0, double 42.0>,
                                 <2 x double> <double 41.0, double 40.0>,
-                                metadata !"round.dynamic",
-                                metadata !"fpexcept.strict")
+                                metadata !"fpexcept.strict") #0
   ret <2 x double> %max
 }
 
-define <3 x float> @constrained_vector_maxnum_v3f32() {
+define <3 x float> @constrained_vector_maxnum_v3f32() #0 {
 ; S390X-LABEL: constrained_vector_maxnum_v3f32:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r14, %r15, 112(%r15)
@@ -4709,10 +4665,10 @@ define <3 x float> @constrained_vector_maxnum_v3f32() {
 ; S390X-NEXT:    .cfi_offset %f9, -176
 ; S390X-NEXT:    .cfi_offset %f10, -184
 ; S390X-NEXT:    larl %r1, .LCPI87_0
-; S390X-NEXT:    le %f8, 0(%r1)
+; S390X-NEXT:    le %f0, 0(%r1)
 ; S390X-NEXT:    larl %r1, .LCPI87_1
-; S390X-NEXT:    le %f2, 0(%r1)
-; S390X-NEXT:    ler %f0, %f8
+; S390X-NEXT:    le %f8, 0(%r1)
+; S390X-NEXT:    ler %f2, %f8
 ; S390X-NEXT:    brasl %r14, fmaxf@PLT
 ; S390X-NEXT:    larl %r1, .LCPI87_2
 ; S390X-NEXT:    le %f1, 0(%r1)
@@ -4722,14 +4678,12 @@ define <3 x float> @constrained_vector_maxnum_v3f32() {
 ; S390X-NEXT:    ler %f0, %f1
 ; S390X-NEXT:    brasl %r14, fmaxf@PLT
 ; S390X-NEXT:    larl %r1, .LCPI87_4
-; S390X-NEXT:    le %f1, 0(%r1)
+; S390X-NEXT:    le %f2, 0(%r1)
 ; S390X-NEXT:    ler %f10, %f0
-; S390X-NEXT:    ler %f0, %f1
-; S390X-NEXT:    ler %f2, %f8
+; S390X-NEXT:    ler %f0, %f8
 ; S390X-NEXT:    brasl %r14, fmaxf@PLT
-; S390X-NEXT:    ler %f4, %f0
-; S390X-NEXT:    ler %f0, %f9
 ; S390X-NEXT:    ler %f2, %f10
+; S390X-NEXT:    ler %f4, %f9
 ; S390X-NEXT:    ld %f8, 176(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f9, 168(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f10, 160(%r15) # 8-byte Folded Reload
@@ -4777,12 +4731,11 @@ entry:
   %max = call <3 x float> @llvm.experimental.constrained.maxnum.v3f32(
                               <3 x float> <float 43.0, float 44.0, float 45.0>,
                               <3 x float> <float 41.0, float 42.0, float 43.0>,
-                              metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #0
   ret <3 x float> %max
 }
 
-define void @constrained_vector_log10_maxnum_v3f64(<3 x double>* %a) {
+define void @constrained_vector_log10_maxnum_v3f64(<3 x double>* %a) #0 {
 ; S390X-LABEL: constrained_vector_log10_maxnum_v3f64:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r13, %r15, 104(%r15)
@@ -4798,8 +4751,8 @@ define void @constrained_vector_log10_maxnum_v3f64(<3 x double>* %a) {
 ; S390X-NEXT:    .cfi_offset %f9, -176
 ; S390X-NEXT:    .cfi_offset %f10, -184
 ; S390X-NEXT:    lgr %r13, %r2
-; S390X-NEXT:    ld %f8, 16(%r2)
-; S390X-NEXT:    ld %f0, 0(%r2)
+; S390X-NEXT:    ld %f8, 0(%r2)
+; S390X-NEXT:    ld %f0, 16(%r2)
 ; S390X-NEXT:    larl %r1, .LCPI88_0
 ; S390X-NEXT:    ldeb %f2, 0(%r1)
 ; S390X-NEXT:    ld %f9, 8(%r2)
@@ -4814,9 +4767,9 @@ define void @constrained_vector_log10_maxnum_v3f64(<3 x double>* %a) {
 ; S390X-NEXT:    ldr %f9, %f0
 ; S390X-NEXT:    ldr %f0, %f8
 ; S390X-NEXT:    brasl %r14, fmax@PLT
-; S390X-NEXT:    std %f0, 16(%r13)
+; S390X-NEXT:    std %f0, 0(%r13)
 ; S390X-NEXT:    std %f9, 8(%r13)
-; S390X-NEXT:    std %f10, 0(%r13)
+; S390X-NEXT:    std %f10, 16(%r13)
 ; S390X-NEXT:    ld %f8, 176(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f9, 168(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f10, 160(%r15) # 8-byte Folded Reload
@@ -4868,13 +4821,12 @@ entry:
   %max = call <3 x double> @llvm.experimental.constrained.maxnum.v3f64(
                           <3 x double> %b,
                           <3 x double> <double 40.0, double 41.0, double 42.0>,
-                          metadata !"round.dynamic",
-                          metadata !"fpexcept.strict")
+                          metadata !"fpexcept.strict") #0
   store <3 x double> %max, <3 x double>* %a
   ret void
 }
 
-define <4 x double> @constrained_vector_maxnum_v4f64() {
+define <4 x double> @constrained_vector_maxnum_v4f64() #0 {
 ; S390X-LABEL: constrained_vector_maxnum_v4f64:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r14, %r15, 112(%r15)
@@ -4914,10 +4866,9 @@ define <4 x double> @constrained_vector_maxnum_v4f64() {
 ; S390X-NEXT:    ldr %f10, %f0
 ; S390X-NEXT:    ldr %f0, %f1
 ; S390X-NEXT:    brasl %r14, fmax@PLT
-; S390X-NEXT:    ldr %f6, %f0
-; S390X-NEXT:    ldr %f0, %f8
-; S390X-NEXT:    ldr %f2, %f9
-; S390X-NEXT:    ldr %f4, %f10
+; S390X-NEXT:    ldr %f2, %f10
+; S390X-NEXT:    ldr %f4, %f9
+; S390X-NEXT:    ldr %f6, %f8
 ; S390X-NEXT:    ld %f8, 176(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f9, 168(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f10, 160(%r15) # 8-byte Folded Reload
@@ -4971,12 +4922,11 @@ entry:
                                               double 46.0, double 47.0>,
                                 <4 x double> <double 40.0, double 41.0,
                                               double 42.0, double 43.0>,
-                                metadata !"round.dynamic",
-                                metadata !"fpexcept.strict")
+                                metadata !"fpexcept.strict") #0
   ret <4 x double> %max
 }
 
-define <1 x float> @constrained_vector_minnum_v1f32() {
+define <1 x float> @constrained_vector_minnum_v1f32() #0 {
 ; S390X-LABEL: constrained_vector_minnum_v1f32:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r14, %r15, 112(%r15)
@@ -5011,12 +4961,11 @@ define <1 x float> @constrained_vector_minnum_v1f32() {
  entry:
   %min = call <1 x float> @llvm.experimental.constrained.minnum.v1f32(
                                <1 x float> <float 42.0>, <1 x float> <float 41.0>,
-                               metadata !"round.dynamic",
-                               metadata !"fpexcept.strict")
+                               metadata !"fpexcept.strict") #0
   ret <1 x float> %min
 }
 
-define <2 x double> @constrained_vector_minnum_v2f64() {
+define <2 x double> @constrained_vector_minnum_v2f64() #0 {
 ; S390X-LABEL: constrained_vector_minnum_v2f64:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r14, %r15, 112(%r15)
@@ -5038,8 +4987,7 @@ define <2 x double> @constrained_vector_minnum_v2f64() {
 ; S390X-NEXT:    ldr %f8, %f0
 ; S390X-NEXT:    ldr %f0, %f1
 ; S390X-NEXT:    brasl %r14, fmin@PLT
-; S390X-NEXT:    ldr %f2, %f0
-; S390X-NEXT:    ldr %f0, %f8
+; S390X-NEXT:    ldr %f2, %f8
 ; S390X-NEXT:    ld %f8, 160(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    lmg %r14, %r15, 280(%r15)
 ; S390X-NEXT:    br %r14
@@ -5072,12 +5020,11 @@ entry:
   %min = call <2 x double> @llvm.experimental.constrained.minnum.v2f64(
                                 <2 x double> <double 43.0, double 42.0>,
                                 <2 x double> <double 41.0, double 40.0>,
-                                metadata !"round.dynamic",
-                                metadata !"fpexcept.strict")
+                                metadata !"fpexcept.strict") #0
   ret <2 x double> %min
 }
 
-define <3 x float> @constrained_vector_minnum_v3f32() {
+define <3 x float> @constrained_vector_minnum_v3f32() #0 {
 ; S390X-LABEL: constrained_vector_minnum_v3f32:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r14, %r15, 112(%r15)
@@ -5092,10 +5039,10 @@ define <3 x float> @constrained_vector_minnum_v3f32() {
 ; S390X-NEXT:    .cfi_offset %f9, -176
 ; S390X-NEXT:    .cfi_offset %f10, -184
 ; S390X-NEXT:    larl %r1, .LCPI92_0
-; S390X-NEXT:    le %f8, 0(%r1)
+; S390X-NEXT:    le %f0, 0(%r1)
 ; S390X-NEXT:    larl %r1, .LCPI92_1
-; S390X-NEXT:    le %f2, 0(%r1)
-; S390X-NEXT:    ler %f0, %f8
+; S390X-NEXT:    le %f8, 0(%r1)
+; S390X-NEXT:    ler %f2, %f8
 ; S390X-NEXT:    brasl %r14, fminf@PLT
 ; S390X-NEXT:    larl %r1, .LCPI92_2
 ; S390X-NEXT:    le %f1, 0(%r1)
@@ -5105,14 +5052,12 @@ define <3 x float> @constrained_vector_minnum_v3f32() {
 ; S390X-NEXT:    ler %f0, %f1
 ; S390X-NEXT:    brasl %r14, fminf@PLT
 ; S390X-NEXT:    larl %r1, .LCPI92_4
-; S390X-NEXT:    le %f1, 0(%r1)
+; S390X-NEXT:    le %f2, 0(%r1)
 ; S390X-NEXT:    ler %f10, %f0
-; S390X-NEXT:    ler %f0, %f1
-; S390X-NEXT:    ler %f2, %f8
+; S390X-NEXT:    ler %f0, %f8
 ; S390X-NEXT:    brasl %r14, fminf@PLT
-; S390X-NEXT:    ler %f4, %f0
-; S390X-NEXT:    ler %f0, %f9
 ; S390X-NEXT:    ler %f2, %f10
+; S390X-NEXT:    ler %f4, %f9
 ; S390X-NEXT:    ld %f8, 176(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f9, 168(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f10, 160(%r15) # 8-byte Folded Reload
@@ -5160,12 +5105,11 @@ entry:
   %min = call <3 x float> @llvm.experimental.constrained.minnum.v3f32(
                               <3 x float> <float 43.0, float 44.0, float 45.0>,
                               <3 x float> <float 41.0, float 42.0, float 43.0>,
-                              metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #0
   ret <3 x float> %min
 }
 
-define void @constrained_vector_minnum_v3f64(<3 x double>* %a) {
+define void @constrained_vector_minnum_v3f64(<3 x double>* %a) #0 {
 ; S390X-LABEL: constrained_vector_minnum_v3f64:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r13, %r15, 104(%r15)
@@ -5183,8 +5127,8 @@ define void @constrained_vector_minnum_v3f64(<3 x double>* %a) {
 ; S390X-NEXT:    .cfi_offset %f10, -184
 ; S390X-NEXT:    .cfi_offset %f11, -192
 ; S390X-NEXT:    lgr %r13, %r2
-; S390X-NEXT:    ld %f8, 16(%r2)
-; S390X-NEXT:    ld %f0, 0(%r2)
+; S390X-NEXT:    ld %f8, 0(%r2)
+; S390X-NEXT:    ld %f0, 16(%r2)
 ; S390X-NEXT:    larl %r1, .LCPI93_0
 ; S390X-NEXT:    ldeb %f9, 0(%r1)
 ; S390X-NEXT:    ld %f10, 8(%r2)
@@ -5198,9 +5142,9 @@ define void @constrained_vector_minnum_v3f64(<3 x double>* %a) {
 ; S390X-NEXT:    ldr %f0, %f8
 ; S390X-NEXT:    ldr %f2, %f9
 ; S390X-NEXT:    brasl %r14, fmin@PLT
-; S390X-NEXT:    std %f0, 16(%r13)
+; S390X-NEXT:    std %f0, 0(%r13)
 ; S390X-NEXT:    std %f10, 8(%r13)
-; S390X-NEXT:    std %f11, 0(%r13)
+; S390X-NEXT:    std %f11, 16(%r13)
 ; S390X-NEXT:    ld %f8, 184(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f9, 176(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f10, 168(%r15) # 8-byte Folded Reload
@@ -5255,13 +5199,12 @@ entry:
  %min = call <3 x double> @llvm.experimental.constrained.minnum.v3f64(
                           <3 x double> %b,
                           <3 x double> <double 3.0, double 3.0, double 3.0>,
-                          metadata !"round.dynamic",
-                          metadata !"fpexcept.strict")
+                          metadata !"fpexcept.strict") #0
   store <3 x double> %min, <3 x double>* %a
   ret void
 }
 
-define <4 x double> @constrained_vector_minnum_v4f64() {
+define <4 x double> @constrained_vector_minnum_v4f64() #0 {
 ; S390X-LABEL: constrained_vector_minnum_v4f64:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r14, %r15, 112(%r15)
@@ -5301,10 +5244,9 @@ define <4 x double> @constrained_vector_minnum_v4f64() {
 ; S390X-NEXT:    ldr %f10, %f0
 ; S390X-NEXT:    ldr %f0, %f1
 ; S390X-NEXT:    brasl %r14, fmin@PLT
-; S390X-NEXT:    ldr %f6, %f0
-; S390X-NEXT:    ldr %f0, %f8
-; S390X-NEXT:    ldr %f2, %f9
-; S390X-NEXT:    ldr %f4, %f10
+; S390X-NEXT:    ldr %f2, %f10
+; S390X-NEXT:    ldr %f4, %f9
+; S390X-NEXT:    ldr %f6, %f8
 ; S390X-NEXT:    ld %f8, 176(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f9, 168(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f10, 160(%r15) # 8-byte Folded Reload
@@ -5358,12 +5300,11 @@ entry:
                                               double 46.0, double 47.0>,
                                 <4 x double> <double 40.0, double 41.0,
                                               double 42.0, double 43.0>,
-                                metadata !"round.dynamic",
-                                metadata !"fpexcept.strict")
+                                metadata !"fpexcept.strict") #0
   ret <4 x double> %min
 }
 
-define <1 x float> @constrained_vector_fptrunc_v1f64() {
+define <1 x float> @constrained_vector_fptrunc_v1f64() #0 {
 ; S390X-LABEL: constrained_vector_fptrunc_v1f64:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    larl %r1, .LCPI95_0
@@ -5381,11 +5322,11 @@ entry:
   %result = call <1 x float> @llvm.experimental.constrained.fptrunc.v1f32.v1f64(
                                 <1 x double><double 42.1>,
                                 metadata !"round.dynamic",
-                                metadata !"fpexcept.strict")
+                                metadata !"fpexcept.strict") #0
   ret <1 x float> %result
 }
 
-define <2 x float> @constrained_vector_fptrunc_v2f64() {
+define <2 x float> @constrained_vector_fptrunc_v2f64() #0 {
 ; S390X-LABEL: constrained_vector_fptrunc_v2f64:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    larl %r1, .LCPI96_0
@@ -5411,11 +5352,11 @@ entry:
   %result = call <2 x float> @llvm.experimental.constrained.fptrunc.v2f32.v2f64(
                                 <2 x double><double 42.1, double 42.2>,
                                 metadata !"round.dynamic",
-                                metadata !"fpexcept.strict")
+                                metadata !"fpexcept.strict") #0
   ret <2 x float> %result
 }
 
-define void @constrained_vector_fptrunc_v3f64(<3 x double>* %src, <3 x float>* %dest) {
+define void @constrained_vector_fptrunc_v3f64(<3 x double>* %src, <3 x float>* %dest) #0 {
 ; S390X-LABEL: constrained_vector_fptrunc_v3f64:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    ld %f0, 0(%r2)
@@ -5436,13 +5377,12 @@ define void @constrained_vector_fptrunc_v3f64(<3 x double>* %src, <3 x float>* %
 ; SZ13-LABEL: constrained_vector_fptrunc_v3f64:
 ; SZ13:       # %bb.0: # %entry
 ; SZ13-NEXT:    vl %v1, 0(%r2), 4
-; SZ13-NEXT:    ledbra %f2, 0, %f1, 0
-; SZ13-NEXT:    vrepg %v1, %v1, 1
+; SZ13-NEXT:    vledb %v1, %v1, 0, 0
+; SZ13-NEXT:    larl %r1, .LCPI97_0
 ; SZ13-NEXT:    ld %f0, 16(%r2)
-; SZ13-NEXT:    ledbra %f1, 0, %f1, 0
+; SZ13-NEXT:    vl %v2, 0(%r1), 3
+; SZ13-NEXT:    vperm %v1, %v1, %v0, %v2
 ; SZ13-NEXT:    ledbra %f0, 0, %f0, 0
-; SZ13-NEXT:    vmrhf %v1, %v2, %v1
-; SZ13-NEXT:    vmrhg %v1, %v1, %v1
 ; SZ13-NEXT:    ste %f0, 8(%r3)
 ; SZ13-NEXT:    vsteg %v1, 0(%r3), 0
 ; SZ13-NEXT:    br %r14
@@ -5451,12 +5391,12 @@ entry:
   %result = call <3 x float> @llvm.experimental.constrained.fptrunc.v3f32.v3f64(
                                 <3 x double> %b,
                                 metadata !"round.dynamic",
-                                metadata !"fpexcept.strict")
+                                metadata !"fpexcept.strict") #0
   store <3 x float> %result, <3 x float>* %dest
   ret void
 }
 
-define <4 x float> @constrained_vector_fptrunc_v4f64() {
+define <4 x float> @constrained_vector_fptrunc_v4f64() #0 {
 ; S390X-LABEL: constrained_vector_fptrunc_v4f64:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    larl %r1, .LCPI98_0
@@ -5496,11 +5436,11 @@ entry:
                                 <4 x double><double 42.1, double 42.2,
                                              double 42.3, double 42.4>,
                                 metadata !"round.dynamic",
-                                metadata !"fpexcept.strict")
+                                metadata !"fpexcept.strict") #0
   ret <4 x float> %result
 }
 
-define <1 x double> @constrained_vector_fpext_v1f32() {
+define <1 x double> @constrained_vector_fpext_v1f32() #0 {
 ; S390X-LABEL: constrained_vector_fpext_v1f32:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    larl %r1, .LCPI99_0
@@ -5516,11 +5456,11 @@ define <1 x double> @constrained_vector_fpext_v1f32() {
 entry:
   %result = call <1 x double> @llvm.experimental.constrained.fpext.v1f64.v1f32(
                                 <1 x float><float 42.0>,
-                                metadata !"fpexcept.strict")
+                                metadata !"fpexcept.strict") #0
   ret <1 x double> %result
 }
 
-define <2 x double> @constrained_vector_fpext_v2f32() {
+define <2 x double> @constrained_vector_fpext_v2f32() #0 {
 ; S390X-LABEL: constrained_vector_fpext_v2f32:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    larl %r1, .LCPI100_0
@@ -5540,11 +5480,11 @@ define <2 x double> @constrained_vector_fpext_v2f32() {
 entry:
   %result = call <2 x double> @llvm.experimental.constrained.fpext.v2f64.v2f32(
                                 <2 x float><float 42.0, float 43.0>,
-                                metadata !"fpexcept.strict")
+                                metadata !"fpexcept.strict") #0
   ret <2 x double> %result
 }
 
-define void @constrained_vector_fpext_v3f64(<3 x float>* %src, <3 x double>* %dest) {
+define void @constrained_vector_fpext_v3f64(<3 x float>* %src, <3 x double>* %dest) #0 {
 ; S390X-LABEL: constrained_vector_fpext_v3f64:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    lg %r0, 0(%r2)
@@ -5563,25 +5503,23 @@ define void @constrained_vector_fpext_v3f64(<3 x float>* %src, <3 x double>* %de
 ; SZ13-LABEL: constrained_vector_fpext_v3f64:
 ; SZ13:       # %bb.0: # %entry
 ; SZ13-NEXT:    vl %v0, 0(%r2), 4
-; SZ13-NEXT:    vrepf %v2, %v0, 1
-; SZ13-NEXT:    ldebr %f1, %f0
-; SZ13-NEXT:    ldebr %f2, %f2
-; SZ13-NEXT:    vrepf %v0, %v0, 2
-; SZ13-NEXT:    ldebr %f0, %f0
-; SZ13-NEXT:    vmrhg %v1, %v1, %v2
-; SZ13-NEXT:    std %f0, 16(%r3)
+; SZ13-NEXT:    vrepf %v1, %v0, 1
+; SZ13-NEXT:    vldeb %v0, %v0
+; SZ13-NEXT:    ldebr %f1, %f1
+; SZ13-NEXT:    vmrhg %v1, %v0, %v1
+; SZ13-NEXT:    vsteg %v0, 16(%r3), 1
 ; SZ13-NEXT:    vst %v1, 0(%r3), 4
 ; SZ13-NEXT:    br %r14
 entry:
   %b = load <3 x float>, <3 x float>* %src
   %result = call <3 x double> @llvm.experimental.constrained.fpext.v3f64.v3f32(
                               <3 x float> %b,
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #0
   store <3 x double> %result, <3 x double>* %dest
   ret void
 }
 
-define <4 x double> @constrained_vector_fpext_v4f32() {
+define <4 x double> @constrained_vector_fpext_v4f32() #0 {
 ; S390X-LABEL: constrained_vector_fpext_v4f32:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    larl %r1, .LCPI102_0
@@ -5611,11 +5549,11 @@ entry:
   %result = call <4 x double> @llvm.experimental.constrained.fpext.v4f64.v4f32(
                                 <4 x float><float 42.0, float 43.0,
                                             float 44.0, float 45.0>,
-                                metadata !"fpexcept.strict")
+                                metadata !"fpexcept.strict") #0
   ret <4 x double> %result
 }
 
-define <1 x float> @constrained_vector_ceil_v1f32() {
+define <1 x float> @constrained_vector_ceil_v1f32() #0 {
 ; S390X-LABEL: constrained_vector_ceil_v1f32:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r14, %r15, 112(%r15)
@@ -5638,12 +5576,11 @@ define <1 x float> @constrained_vector_ceil_v1f32() {
 entry:
   %ceil = call <1 x float> @llvm.experimental.constrained.ceil.v1f32(
                                <1 x float> <float 1.5>,
-                               metadata !"round.dynamic",
-                               metadata !"fpexcept.strict")
+                               metadata !"fpexcept.strict") #0
   ret <1 x float> %ceil
 }
 
-define <2 x double> @constrained_vector_ceil_v2f64() {
+define <2 x double> @constrained_vector_ceil_v2f64() #0 {
 ; S390X-LABEL: constrained_vector_ceil_v2f64:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r14, %r15, 112(%r15)
@@ -5661,8 +5598,7 @@ define <2 x double> @constrained_vector_ceil_v2f64() {
 ; S390X-NEXT:    ldr %f8, %f0
 ; S390X-NEXT:    ldr %f0, %f1
 ; S390X-NEXT:    brasl %r14, ceil@PLT
-; S390X-NEXT:    ldr %f2, %f0
-; S390X-NEXT:    ldr %f0, %f8
+; S390X-NEXT:    ldr %f2, %f8
 ; S390X-NEXT:    ld %f8, 160(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    lmg %r14, %r15, 280(%r15)
 ; S390X-NEXT:    br %r14
@@ -5676,12 +5612,11 @@ define <2 x double> @constrained_vector_ceil_v2f64() {
 entry:
   %ceil = call <2 x double> @llvm.experimental.constrained.ceil.v2f64(
                                 <2 x double> <double 1.1, double 1.9>,
-                                metadata !"round.dynamic",
-                                metadata !"fpexcept.strict")
+                                metadata !"fpexcept.strict") #0
   ret <2 x double> %ceil
 }
 
-define <3 x float> @constrained_vector_ceil_v3f32() {
+define <3 x float> @constrained_vector_ceil_v3f32() #0 {
 ; S390X-LABEL: constrained_vector_ceil_v3f32:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r14, %r15, 112(%r15)
@@ -5706,9 +5641,8 @@ define <3 x float> @constrained_vector_ceil_v3f32() {
 ; S390X-NEXT:    ler %f9, %f0
 ; S390X-NEXT:    ler %f0, %f1
 ; S390X-NEXT:    brasl %r14, ceilf@PLT
-; S390X-NEXT:    ler %f4, %f0
-; S390X-NEXT:    ler %f0, %f8
 ; S390X-NEXT:    ler %f2, %f9
+; S390X-NEXT:    ler %f4, %f8
 ; S390X-NEXT:    ld %f8, 168(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f9, 160(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    lmg %r14, %r15, 288(%r15)
@@ -5731,12 +5665,11 @@ define <3 x float> @constrained_vector_ceil_v3f32() {
 entry:
   %ceil = call <3 x float> @llvm.experimental.constrained.ceil.v3f32(
                               <3 x float> <float 1.5, float 2.5, float 3.5>,
-                              metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #0
   ret <3 x float> %ceil
 }
 
-define void @constrained_vector_ceil_v3f64(<3 x double>* %a) {
+define void @constrained_vector_ceil_v3f64(<3 x double>* %a) #0 {
 ; S390X-LABEL: constrained_vector_ceil_v3f64:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r13, %r15, 104(%r15)
@@ -5752,8 +5685,8 @@ define void @constrained_vector_ceil_v3f64(<3 x double>* %a) {
 ; S390X-NEXT:    .cfi_offset %f9, -176
 ; S390X-NEXT:    .cfi_offset %f10, -184
 ; S390X-NEXT:    lgr %r13, %r2
-; S390X-NEXT:    ld %f8, 16(%r2)
-; S390X-NEXT:    ld %f0, 0(%r2)
+; S390X-NEXT:    ld %f8, 0(%r2)
+; S390X-NEXT:    ld %f0, 16(%r2)
 ; S390X-NEXT:    ld %f9, 8(%r2)
 ; S390X-NEXT:    brasl %r14, ceil@PLT
 ; S390X-NEXT:    ldr %f10, %f0
@@ -5762,9 +5695,9 @@ define void @constrained_vector_ceil_v3f64(<3 x double>* %a) {
 ; S390X-NEXT:    ldr %f9, %f0
 ; S390X-NEXT:    ldr %f0, %f8
 ; S390X-NEXT:    brasl %r14, ceil@PLT
-; S390X-NEXT:    std %f0, 16(%r13)
+; S390X-NEXT:    std %f0, 0(%r13)
 ; S390X-NEXT:    std %f9, 8(%r13)
-; S390X-NEXT:    std %f10, 0(%r13)
+; S390X-NEXT:    std %f10, 16(%r13)
 ; S390X-NEXT:    ld %f8, 176(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f9, 168(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f10, 160(%r15) # 8-byte Folded Reload
@@ -5773,24 +5706,23 @@ define void @constrained_vector_ceil_v3f64(<3 x double>* %a) {
 ;
 ; SZ13-LABEL: constrained_vector_ceil_v3f64:
 ; SZ13:       # %bb.0: # %entry
-; SZ13-NEXT:    vl %v1, 0(%r2), 4
-; SZ13-NEXT:    ld %f0, 16(%r2)
-; SZ13-NEXT:    vfidb %v1, %v1, 4, 6
-; SZ13-NEXT:    fidbra %f0, 6, %f0, 4
-; SZ13-NEXT:    std %f0, 16(%r2)
-; SZ13-NEXT:    vst %v1, 0(%r2), 4
+; SZ13-NEXT:    vl %v0, 0(%r2), 4
+; SZ13-NEXT:    ld %f1, 16(%r2)
+; SZ13-NEXT:    vfidb %v0, %v0, 4, 6
+; SZ13-NEXT:    fidbra %f1, 6, %f1, 4
+; SZ13-NEXT:    vst %v0, 0(%r2), 4
+; SZ13-NEXT:    std %f1, 16(%r2)
 ; SZ13-NEXT:    br %r14
 entry:
   %b = load <3 x double>, <3 x double>* %a
   %ceil = call <3 x double> @llvm.experimental.constrained.ceil.v3f64(
                           <3 x double> %b,
-                          metadata !"round.dynamic",
-                          metadata !"fpexcept.strict")
+                          metadata !"fpexcept.strict") #0
   store <3 x double> %ceil, <3 x double>* %a
   ret void
 }
 
-define <1 x float> @constrained_vector_floor_v1f32() {
+define <1 x float> @constrained_vector_floor_v1f32() #0 {
 ; S390X-LABEL: constrained_vector_floor_v1f32:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r14, %r15, 112(%r15)
@@ -5813,13 +5745,12 @@ define <1 x float> @constrained_vector_floor_v1f32() {
 entry:
   %floor = call <1 x float> @llvm.experimental.constrained.floor.v1f32(
                                <1 x float> <float 1.5>,
-                               metadata !"round.dynamic",
-                               metadata !"fpexcept.strict")
+                               metadata !"fpexcept.strict") #0
   ret <1 x float> %floor
 }
 
 
-define <2 x double> @constrained_vector_floor_v2f64() {
+define <2 x double> @constrained_vector_floor_v2f64() #0 {
 ; S390X-LABEL: constrained_vector_floor_v2f64:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r14, %r15, 112(%r15)
@@ -5837,8 +5768,7 @@ define <2 x double> @constrained_vector_floor_v2f64() {
 ; S390X-NEXT:    ldr %f8, %f0
 ; S390X-NEXT:    ldr %f0, %f1
 ; S390X-NEXT:    brasl %r14, floor@PLT
-; S390X-NEXT:    ldr %f2, %f0
-; S390X-NEXT:    ldr %f0, %f8
+; S390X-NEXT:    ldr %f2, %f8
 ; S390X-NEXT:    ld %f8, 160(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    lmg %r14, %r15, 280(%r15)
 ; S390X-NEXT:    br %r14
@@ -5852,12 +5782,11 @@ define <2 x double> @constrained_vector_floor_v2f64() {
 entry:
   %floor = call <2 x double> @llvm.experimental.constrained.floor.v2f64(
                                 <2 x double> <double 1.1, double 1.9>,
-                                metadata !"round.dynamic",
-                                metadata !"fpexcept.strict")
+                                metadata !"fpexcept.strict") #0
   ret <2 x double> %floor
 }
 
-define <3 x float> @constrained_vector_floor_v3f32() {
+define <3 x float> @constrained_vector_floor_v3f32() #0 {
 ; S390X-LABEL: constrained_vector_floor_v3f32:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r14, %r15, 112(%r15)
@@ -5882,9 +5811,8 @@ define <3 x float> @constrained_vector_floor_v3f32() {
 ; S390X-NEXT:    ler %f9, %f0
 ; S390X-NEXT:    ler %f0, %f1
 ; S390X-NEXT:    brasl %r14, floorf@PLT
-; S390X-NEXT:    ler %f4, %f0
-; S390X-NEXT:    ler %f0, %f8
 ; S390X-NEXT:    ler %f2, %f9
+; S390X-NEXT:    ler %f4, %f8
 ; S390X-NEXT:    ld %f8, 168(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f9, 160(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    lmg %r14, %r15, 288(%r15)
@@ -5907,12 +5835,11 @@ define <3 x float> @constrained_vector_floor_v3f32() {
 entry:
   %floor = call <3 x float> @llvm.experimental.constrained.floor.v3f32(
                               <3 x float> <float 1.5, float 2.5, float 3.5>,
-                              metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #0
   ret <3 x float> %floor
 }
 
-define void @constrained_vector_floor_v3f64(<3 x double>* %a) {
+define void @constrained_vector_floor_v3f64(<3 x double>* %a) #0 {
 ; S390X-LABEL: constrained_vector_floor_v3f64:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r13, %r15, 104(%r15)
@@ -5928,8 +5855,8 @@ define void @constrained_vector_floor_v3f64(<3 x double>* %a) {
 ; S390X-NEXT:    .cfi_offset %f9, -176
 ; S390X-NEXT:    .cfi_offset %f10, -184
 ; S390X-NEXT:    lgr %r13, %r2
-; S390X-NEXT:    ld %f8, 16(%r2)
-; S390X-NEXT:    ld %f0, 0(%r2)
+; S390X-NEXT:    ld %f8, 0(%r2)
+; S390X-NEXT:    ld %f0, 16(%r2)
 ; S390X-NEXT:    ld %f9, 8(%r2)
 ; S390X-NEXT:    brasl %r14, floor@PLT
 ; S390X-NEXT:    ldr %f10, %f0
@@ -5938,9 +5865,9 @@ define void @constrained_vector_floor_v3f64(<3 x double>* %a) {
 ; S390X-NEXT:    ldr %f9, %f0
 ; S390X-NEXT:    ldr %f0, %f8
 ; S390X-NEXT:    brasl %r14, floor@PLT
-; S390X-NEXT:    std %f0, 16(%r13)
+; S390X-NEXT:    std %f0, 0(%r13)
 ; S390X-NEXT:    std %f9, 8(%r13)
-; S390X-NEXT:    std %f10, 0(%r13)
+; S390X-NEXT:    std %f10, 16(%r13)
 ; S390X-NEXT:    ld %f8, 176(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f9, 168(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f10, 160(%r15) # 8-byte Folded Reload
@@ -5949,24 +5876,23 @@ define void @constrained_vector_floor_v3f64(<3 x double>* %a) {
 ;
 ; SZ13-LABEL: constrained_vector_floor_v3f64:
 ; SZ13:       # %bb.0: # %entry
-; SZ13-NEXT:    vl %v1, 0(%r2), 4
-; SZ13-NEXT:    ld %f0, 16(%r2)
-; SZ13-NEXT:    vfidb %v1, %v1, 4, 7
-; SZ13-NEXT:    fidbra %f0, 7, %f0, 4
-; SZ13-NEXT:    std %f0, 16(%r2)
-; SZ13-NEXT:    vst %v1, 0(%r2), 4
+; SZ13-NEXT:    vl %v0, 0(%r2), 4
+; SZ13-NEXT:    ld %f1, 16(%r2)
+; SZ13-NEXT:    vfidb %v0, %v0, 4, 7
+; SZ13-NEXT:    fidbra %f1, 7, %f1, 4
+; SZ13-NEXT:    vst %v0, 0(%r2), 4
+; SZ13-NEXT:    std %f1, 16(%r2)
 ; SZ13-NEXT:    br %r14
 entry:
   %b = load <3 x double>, <3 x double>* %a
   %floor = call <3 x double> @llvm.experimental.constrained.floor.v3f64(
                           <3 x double> %b,
-                          metadata !"round.dynamic",
-                          metadata !"fpexcept.strict")
+                          metadata !"fpexcept.strict") #0
   store <3 x double> %floor, <3 x double>* %a
   ret void
 }
 
-define <1 x float> @constrained_vector_round_v1f32() {
+define <1 x float> @constrained_vector_round_v1f32() #0 {
 ; S390X-LABEL: constrained_vector_round_v1f32:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r14, %r15, 112(%r15)
@@ -5989,12 +5915,11 @@ define <1 x float> @constrained_vector_round_v1f32() {
 entry:
   %round = call <1 x float> @llvm.experimental.constrained.round.v1f32(
                                <1 x float> <float 1.5>,
-                               metadata !"round.dynamic",
-                               metadata !"fpexcept.strict")
+                               metadata !"fpexcept.strict") #0
   ret <1 x float> %round
 }
 
-define <2 x double> @constrained_vector_round_v2f64() {
+define <2 x double> @constrained_vector_round_v2f64() #0 {
 ; S390X-LABEL: constrained_vector_round_v2f64:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r14, %r15, 112(%r15)
@@ -6012,8 +5937,7 @@ define <2 x double> @constrained_vector_round_v2f64() {
 ; S390X-NEXT:    ldr %f8, %f0
 ; S390X-NEXT:    ldr %f0, %f1
 ; S390X-NEXT:    brasl %r14, round@PLT
-; S390X-NEXT:    ldr %f2, %f0
-; S390X-NEXT:    ldr %f0, %f8
+; S390X-NEXT:    ldr %f2, %f8
 ; S390X-NEXT:    ld %f8, 160(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    lmg %r14, %r15, 280(%r15)
 ; S390X-NEXT:    br %r14
@@ -6027,12 +5951,11 @@ define <2 x double> @constrained_vector_round_v2f64() {
 entry:
   %round = call <2 x double> @llvm.experimental.constrained.round.v2f64(
                                 <2 x double> <double 1.1, double 1.9>,
-                                metadata !"round.dynamic",
-                                metadata !"fpexcept.strict")
+                                metadata !"fpexcept.strict") #0
   ret <2 x double> %round
 }
 
-define <3 x float> @constrained_vector_round_v3f32() {
+define <3 x float> @constrained_vector_round_v3f32() #0 {
 ; S390X-LABEL: constrained_vector_round_v3f32:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r14, %r15, 112(%r15)
@@ -6057,9 +5980,8 @@ define <3 x float> @constrained_vector_round_v3f32() {
 ; S390X-NEXT:    ler %f9, %f0
 ; S390X-NEXT:    ler %f0, %f1
 ; S390X-NEXT:    brasl %r14, roundf@PLT
-; S390X-NEXT:    ler %f4, %f0
-; S390X-NEXT:    ler %f0, %f8
 ; S390X-NEXT:    ler %f2, %f9
+; S390X-NEXT:    ler %f4, %f8
 ; S390X-NEXT:    ld %f8, 168(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f9, 160(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    lmg %r14, %r15, 288(%r15)
@@ -6082,13 +6004,12 @@ define <3 x float> @constrained_vector_round_v3f32() {
 entry:
   %round = call <3 x float> @llvm.experimental.constrained.round.v3f32(
                               <3 x float> <float 1.5, float 2.5, float 3.5>,
-                              metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #0
   ret <3 x float> %round
 }
 
 
-define void @constrained_vector_round_v3f64(<3 x double>* %a) {
+define void @constrained_vector_round_v3f64(<3 x double>* %a) #0 {
 ; S390X-LABEL: constrained_vector_round_v3f64:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r13, %r15, 104(%r15)
@@ -6104,8 +6025,8 @@ define void @constrained_vector_round_v3f64(<3 x double>* %a) {
 ; S390X-NEXT:    .cfi_offset %f9, -176
 ; S390X-NEXT:    .cfi_offset %f10, -184
 ; S390X-NEXT:    lgr %r13, %r2
-; S390X-NEXT:    ld %f8, 16(%r2)
-; S390X-NEXT:    ld %f0, 0(%r2)
+; S390X-NEXT:    ld %f8, 0(%r2)
+; S390X-NEXT:    ld %f0, 16(%r2)
 ; S390X-NEXT:    ld %f9, 8(%r2)
 ; S390X-NEXT:    brasl %r14, round@PLT
 ; S390X-NEXT:    ldr %f10, %f0
@@ -6114,9 +6035,9 @@ define void @constrained_vector_round_v3f64(<3 x double>* %a) {
 ; S390X-NEXT:    ldr %f9, %f0
 ; S390X-NEXT:    ldr %f0, %f8
 ; S390X-NEXT:    brasl %r14, round@PLT
-; S390X-NEXT:    std %f0, 16(%r13)
+; S390X-NEXT:    std %f0, 0(%r13)
 ; S390X-NEXT:    std %f9, 8(%r13)
-; S390X-NEXT:    std %f10, 0(%r13)
+; S390X-NEXT:    std %f10, 16(%r13)
 ; S390X-NEXT:    ld %f8, 176(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f9, 168(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f10, 160(%r15) # 8-byte Folded Reload
@@ -6125,24 +6046,23 @@ define void @constrained_vector_round_v3f64(<3 x double>* %a) {
 ;
 ; SZ13-LABEL: constrained_vector_round_v3f64:
 ; SZ13:       # %bb.0: # %entry
-; SZ13-NEXT:    vl %v1, 0(%r2), 4
-; SZ13-NEXT:    ld %f0, 16(%r2)
-; SZ13-NEXT:    vfidb %v1, %v1, 4, 1
-; SZ13-NEXT:    fidbra %f0, 1, %f0, 4
-; SZ13-NEXT:    std %f0, 16(%r2)
-; SZ13-NEXT:    vst %v1, 0(%r2), 4
+; SZ13-NEXT:    vl %v0, 0(%r2), 4
+; SZ13-NEXT:    ld %f1, 16(%r2)
+; SZ13-NEXT:    vfidb %v0, %v0, 4, 1
+; SZ13-NEXT:    fidbra %f1, 1, %f1, 4
+; SZ13-NEXT:    vst %v0, 0(%r2), 4
+; SZ13-NEXT:    std %f1, 16(%r2)
 ; SZ13-NEXT:    br %r14
 entry:
   %b = load <3 x double>, <3 x double>* %a
   %round = call <3 x double> @llvm.experimental.constrained.round.v3f64(
                           <3 x double> %b,
-                          metadata !"round.dynamic",
-                          metadata !"fpexcept.strict")
+                          metadata !"fpexcept.strict") #0
   store <3 x double> %round, <3 x double>* %a
   ret void
 }
 
-define <1 x float> @constrained_vector_trunc_v1f32() {
+define <1 x float> @constrained_vector_trunc_v1f32() #0 {
 ; S390X-LABEL: constrained_vector_trunc_v1f32:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r14, %r15, 112(%r15)
@@ -6165,12 +6085,11 @@ define <1 x float> @constrained_vector_trunc_v1f32() {
 entry:
   %trunc = call <1 x float> @llvm.experimental.constrained.trunc.v1f32(
                                <1 x float> <float 1.5>,
-                               metadata !"round.dynamic",
-                               metadata !"fpexcept.strict")
+                               metadata !"fpexcept.strict") #0
   ret <1 x float> %trunc
 }
 
-define <2 x double> @constrained_vector_trunc_v2f64() {
+define <2 x double> @constrained_vector_trunc_v2f64() #0 {
 ; S390X-LABEL: constrained_vector_trunc_v2f64:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r14, %r15, 112(%r15)
@@ -6188,8 +6107,7 @@ define <2 x double> @constrained_vector_trunc_v2f64() {
 ; S390X-NEXT:    ldr %f8, %f0
 ; S390X-NEXT:    ldr %f0, %f1
 ; S390X-NEXT:    brasl %r14, trunc@PLT
-; S390X-NEXT:    ldr %f2, %f0
-; S390X-NEXT:    ldr %f0, %f8
+; S390X-NEXT:    ldr %f2, %f8
 ; S390X-NEXT:    ld %f8, 160(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    lmg %r14, %r15, 280(%r15)
 ; S390X-NEXT:    br %r14
@@ -6203,12 +6121,11 @@ define <2 x double> @constrained_vector_trunc_v2f64() {
 entry:
   %trunc = call <2 x double> @llvm.experimental.constrained.trunc.v2f64(
                                 <2 x double> <double 1.1, double 1.9>,
-                                metadata !"round.dynamic",
-                                metadata !"fpexcept.strict")
+                                metadata !"fpexcept.strict") #0
   ret <2 x double> %trunc
 }
 
-define <3 x float> @constrained_vector_trunc_v3f32() {
+define <3 x float> @constrained_vector_trunc_v3f32() #0 {
 ; S390X-LABEL: constrained_vector_trunc_v3f32:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r14, %r15, 112(%r15)
@@ -6233,9 +6150,8 @@ define <3 x float> @constrained_vector_trunc_v3f32() {
 ; S390X-NEXT:    ler %f9, %f0
 ; S390X-NEXT:    ler %f0, %f1
 ; S390X-NEXT:    brasl %r14, truncf@PLT
-; S390X-NEXT:    ler %f4, %f0
-; S390X-NEXT:    ler %f0, %f8
 ; S390X-NEXT:    ler %f2, %f9
+; S390X-NEXT:    ler %f4, %f8
 ; S390X-NEXT:    ld %f8, 168(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f9, 160(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    lmg %r14, %r15, 288(%r15)
@@ -6258,12 +6174,11 @@ define <3 x float> @constrained_vector_trunc_v3f32() {
 entry:
   %trunc = call <3 x float> @llvm.experimental.constrained.trunc.v3f32(
                               <3 x float> <float 1.5, float 2.5, float 3.5>,
-                              metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #0
   ret <3 x float> %trunc
 }
 
-define void @constrained_vector_trunc_v3f64(<3 x double>* %a) {
+define void @constrained_vector_trunc_v3f64(<3 x double>* %a) #0 {
 ; S390X-LABEL: constrained_vector_trunc_v3f64:
 ; S390X:       # %bb.0: # %entry
 ; S390X-NEXT:    stmg %r13, %r15, 104(%r15)
@@ -6279,8 +6194,8 @@ define void @constrained_vector_trunc_v3f64(<3 x double>* %a) {
 ; S390X-NEXT:    .cfi_offset %f9, -176
 ; S390X-NEXT:    .cfi_offset %f10, -184
 ; S390X-NEXT:    lgr %r13, %r2
-; S390X-NEXT:    ld %f8, 16(%r2)
-; S390X-NEXT:    ld %f0, 0(%r2)
+; S390X-NEXT:    ld %f8, 0(%r2)
+; S390X-NEXT:    ld %f0, 16(%r2)
 ; S390X-NEXT:    ld %f9, 8(%r2)
 ; S390X-NEXT:    brasl %r14, trunc@PLT
 ; S390X-NEXT:    ldr %f10, %f0
@@ -6289,9 +6204,9 @@ define void @constrained_vector_trunc_v3f64(<3 x double>* %a) {
 ; S390X-NEXT:    ldr %f9, %f0
 ; S390X-NEXT:    ldr %f0, %f8
 ; S390X-NEXT:    brasl %r14, trunc@PLT
-; S390X-NEXT:    std %f0, 16(%r13)
+; S390X-NEXT:    std %f0, 0(%r13)
 ; S390X-NEXT:    std %f9, 8(%r13)
-; S390X-NEXT:    std %f10, 0(%r13)
+; S390X-NEXT:    std %f10, 16(%r13)
 ; S390X-NEXT:    ld %f8, 176(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f9, 168(%r15) # 8-byte Folded Reload
 ; S390X-NEXT:    ld %f10, 160(%r15) # 8-byte Folded Reload
@@ -6300,23 +6215,24 @@ define void @constrained_vector_trunc_v3f64(<3 x double>* %a) {
 ;
 ; SZ13-LABEL: constrained_vector_trunc_v3f64:
 ; SZ13:       # %bb.0: # %entry
-; SZ13-NEXT:    vl %v1, 0(%r2), 4
-; SZ13-NEXT:    ld %f0, 16(%r2)
-; SZ13-NEXT:    vfidb %v1, %v1, 4, 5
-; SZ13-NEXT:    fidbra %f0, 5, %f0, 4
-; SZ13-NEXT:    std %f0, 16(%r2)
-; SZ13-NEXT:    vst %v1, 0(%r2), 4
+; SZ13-NEXT:    vl %v0, 0(%r2), 4
+; SZ13-NEXT:    ld %f1, 16(%r2)
+; SZ13-NEXT:    vfidb %v0, %v0, 4, 5
+; SZ13-NEXT:    fidbra %f1, 5, %f1, 4
+; SZ13-NEXT:    vst %v0, 0(%r2), 4
+; SZ13-NEXT:    std %f1, 16(%r2)
 ; SZ13-NEXT:    br %r14
 entry:
   %b = load <3 x double>, <3 x double>* %a
   %trunc = call <3 x double> @llvm.experimental.constrained.trunc.v3f64(
                           <3 x double> %b,
-                          metadata !"round.dynamic",
-                          metadata !"fpexcept.strict")
+                          metadata !"fpexcept.strict") #0
   store <3 x double> %trunc, <3 x double>* %a
   ret void
 }
 
+attributes #0 = { strictfp }
+
 declare <2 x double> @llvm.experimental.constrained.fadd.v2f64(<2 x double>, <2 x double>, metadata, metadata)
 declare <2 x double> @llvm.experimental.constrained.fsub.v2f64(<2 x double>, <2 x double>, metadata, metadata)
 declare <2 x double> @llvm.experimental.constrained.fmul.v2f64(<2 x double>, <2 x double>, metadata, metadata)
@@ -6334,14 +6250,14 @@ declare <2 x double> @llvm.experimental.constrained.log10.v2f64(<2 x double>, me
 declare <2 x double> @llvm.experimental.constrained.log2.v2f64(<2 x double>, metadata, metadata)
 declare <2 x double> @llvm.experimental.constrained.rint.v2f64(<2 x double>, metadata, metadata)
 declare <2 x double> @llvm.experimental.constrained.nearbyint.v2f64(<2 x double>, metadata, metadata)
-declare <2 x double> @llvm.experimental.constrained.maxnum.v2f64(<2 x double>, <2 x double>, metadata, metadata)
-declare <2 x double> @llvm.experimental.constrained.minnum.v2f64(<2 x double>, <2 x double>, metadata, metadata)
+declare <2 x double> @llvm.experimental.constrained.maxnum.v2f64(<2 x double>, <2 x double>, metadata)
+declare <2 x double> @llvm.experimental.constrained.minnum.v2f64(<2 x double>, <2 x double>, metadata)
 declare <2 x float> @llvm.experimental.constrained.fptrunc.v2f32.v2f64(<2 x double>, metadata, metadata)
 declare <2 x double> @llvm.experimental.constrained.fpext.v2f64.v2f32(<2 x float>, metadata)
-declare <2 x double> @llvm.experimental.constrained.ceil.v2f64(<2 x double>, metadata, metadata)
-declare <2 x double> @llvm.experimental.constrained.floor.v2f64(<2 x double>, metadata, metadata)
-declare <2 x double> @llvm.experimental.constrained.round.v2f64(<2 x double>, metadata, metadata)
-declare <2 x double> @llvm.experimental.constrained.trunc.v2f64(<2 x double>, metadata, metadata)
+declare <2 x double> @llvm.experimental.constrained.ceil.v2f64(<2 x double>, metadata)
+declare <2 x double> @llvm.experimental.constrained.floor.v2f64(<2 x double>, metadata)
+declare <2 x double> @llvm.experimental.constrained.round.v2f64(<2 x double>, metadata)
+declare <2 x double> @llvm.experimental.constrained.trunc.v2f64(<2 x double>, metadata)
 
 declare <1 x float> @llvm.experimental.constrained.fadd.v1f32(<1 x float>, <1 x float>, metadata, metadata)
 declare <1 x float> @llvm.experimental.constrained.fsub.v1f32(<1 x float>, <1 x float>, metadata, metadata)
@@ -6360,14 +6276,14 @@ declare <1 x float> @llvm.experimental.constrained.log10.v1f32(<1 x float>, meta
 declare <1 x float> @llvm.experimental.constrained.log2.v1f32(<1 x float>, metadata, metadata)
 declare <1 x float> @llvm.experimental.constrained.rint.v1f32(<1 x float>, metadata, metadata)
 declare <1 x float> @llvm.experimental.constrained.nearbyint.v1f32(<1 x float>, metadata, metadata)
-declare <1 x float> @llvm.experimental.constrained.maxnum.v1f32(<1 x float>, <1 x float>, metadata, metadata)
-declare <1 x float> @llvm.experimental.constrained.minnum.v1f32(<1 x float>, <1 x float>, metadata, metadata)
+declare <1 x float> @llvm.experimental.constrained.maxnum.v1f32(<1 x float>, <1 x float>, metadata)
+declare <1 x float> @llvm.experimental.constrained.minnum.v1f32(<1 x float>, <1 x float>, metadata)
 declare <1 x float> @llvm.experimental.constrained.fptrunc.v1f32.v1f64(<1 x double>, metadata, metadata)
 declare <1 x double> @llvm.experimental.constrained.fpext.v1f64.v1f32(<1 x float>, metadata)
-declare <1 x float> @llvm.experimental.constrained.ceil.v1f32(<1 x float>, metadata, metadata)
-declare <1 x float> @llvm.experimental.constrained.floor.v1f32(<1 x float>, metadata, metadata)
-declare <1 x float> @llvm.experimental.constrained.round.v1f32(<1 x float>, metadata, metadata)
-declare <1 x float> @llvm.experimental.constrained.trunc.v1f32(<1 x float>, metadata, metadata)
+declare <1 x float> @llvm.experimental.constrained.ceil.v1f32(<1 x float>, metadata)
+declare <1 x float> @llvm.experimental.constrained.floor.v1f32(<1 x float>, metadata)
+declare <1 x float> @llvm.experimental.constrained.round.v1f32(<1 x float>, metadata)
+declare <1 x float> @llvm.experimental.constrained.trunc.v1f32(<1 x float>, metadata)
 
 declare <3 x float> @llvm.experimental.constrained.fadd.v3f32(<3 x float>, <3 x float>, metadata, metadata)
 declare <3 x double> @llvm.experimental.constrained.fadd.v3f64(<3 x double>, <3 x double>, metadata, metadata)
@@ -6403,20 +6319,20 @@ declare <3 x float> @llvm.experimental.constrained.rint.v3f32(<3 x float>, metad
 declare <3 x double> @llvm.experimental.constrained.rint.v3f64(<3 x double>, metadata, metadata)
 declare <3 x float> @llvm.experimental.constrained.nearbyint.v3f32(<3 x float>, metadata, metadata)
 declare <3 x double> @llvm.experimental.constrained.nearbyint.v3f64(<3 x double>, metadata, metadata)
-declare <3 x float> @llvm.experimental.constrained.maxnum.v3f32(<3 x float>, <3 x float>, metadata, metadata)
-declare <3 x double> @llvm.experimental.constrained.maxnum.v3f64(<3 x double>, <3 x double>, metadata, metadata)
-declare <3 x float> @llvm.experimental.constrained.minnum.v3f32(<3 x float>, <3 x float>, metadata, metadata)
-declare <3 x double> @llvm.experimental.constrained.minnum.v3f64(<3 x double>, <3 x double>, metadata, metadata)
+declare <3 x float> @llvm.experimental.constrained.maxnum.v3f32(<3 x float>, <3 x float>, metadata)
+declare <3 x double> @llvm.experimental.constrained.maxnum.v3f64(<3 x double>, <3 x double>, metadata)
+declare <3 x float> @llvm.experimental.constrained.minnum.v3f32(<3 x float>, <3 x float>, metadata)
+declare <3 x double> @llvm.experimental.constrained.minnum.v3f64(<3 x double>, <3 x double>, metadata)
 declare <3 x float> @llvm.experimental.constrained.fptrunc.v3f32.v3f64(<3 x double>, metadata, metadata)
 declare <3 x double> @llvm.experimental.constrained.fpext.v3f64.v3f32(<3 x float>, metadata)
-declare <3 x float> @llvm.experimental.constrained.ceil.v3f32(<3 x float>, metadata, metadata)
-declare <3 x double> @llvm.experimental.constrained.ceil.v3f64(<3 x double>, metadata, metadata)
-declare <3 x float> @llvm.experimental.constrained.floor.v3f32(<3 x float>, metadata, metadata)
-declare <3 x double> @llvm.experimental.constrained.floor.v3f64(<3 x double>, metadata, metadata)
-declare <3 x float> @llvm.experimental.constrained.round.v3f32(<3 x float>, metadata, metadata)
-declare <3 x double> @llvm.experimental.constrained.round.v3f64(<3 x double>, metadata, metadata)
-declare <3 x float> @llvm.experimental.constrained.trunc.v3f32(<3 x float>, metadata, metadata)
-declare <3 x double> @llvm.experimental.constrained.trunc.v3f64(<3 x double>, metadata, metadata)
+declare <3 x float> @llvm.experimental.constrained.ceil.v3f32(<3 x float>, metadata)
+declare <3 x double> @llvm.experimental.constrained.ceil.v3f64(<3 x double>, metadata)
+declare <3 x float> @llvm.experimental.constrained.floor.v3f32(<3 x float>, metadata)
+declare <3 x double> @llvm.experimental.constrained.floor.v3f64(<3 x double>, metadata)
+declare <3 x float> @llvm.experimental.constrained.round.v3f32(<3 x float>, metadata)
+declare <3 x double> @llvm.experimental.constrained.round.v3f64(<3 x double>, metadata)
+declare <3 x float> @llvm.experimental.constrained.trunc.v3f32(<3 x float>, metadata)
+declare <3 x double> @llvm.experimental.constrained.trunc.v3f64(<3 x double>, metadata)
 
 declare <4 x double> @llvm.experimental.constrained.fadd.v4f64(<4 x double>, <4 x double>, metadata, metadata)
 declare <4 x double> @llvm.experimental.constrained.fsub.v4f64(<4 x double>, <4 x double>, metadata, metadata)
@@ -6435,11 +6351,11 @@ declare <4 x double> @llvm.experimental.constrained.log10.v4f64(<4 x double>, me
 declare <4 x double> @llvm.experimental.constrained.log2.v4f64(<4 x double>, metadata, metadata)
 declare <4 x double> @llvm.experimental.constrained.rint.v4f64(<4 x double>, metadata, metadata)
 declare <4 x double> @llvm.experimental.constrained.nearbyint.v4f64(<4 x double>, metadata, metadata)
-declare <4 x double> @llvm.experimental.constrained.maxnum.v4f64(<4 x double>, <4 x double>, metadata, metadata)
-declare <4 x double> @llvm.experimental.constrained.minnum.v4f64(<4 x double>, <4 x double>, metadata, metadata)
+declare <4 x double> @llvm.experimental.constrained.maxnum.v4f64(<4 x double>, <4 x double>, metadata)
+declare <4 x double> @llvm.experimental.constrained.minnum.v4f64(<4 x double>, <4 x double>, metadata)
 declare <4 x float> @llvm.experimental.constrained.fptrunc.v4f32.v4f64(<4 x double>, metadata, metadata)
 declare <4 x double> @llvm.experimental.constrained.fpext.v4f64.v4f32(<4 x float>, metadata)
-declare <4 x double> @llvm.experimental.constrained.ceil.v4f64(<4 x double>, metadata, metadata)
-declare <4 x double> @llvm.experimental.constrained.floor.v4f64(<4 x double>, metadata, metadata)
-declare <4 x double> @llvm.experimental.constrained.round.v4f64(<4 x double>, metadata, metadata)
-declare <4 x double> @llvm.experimental.constrained.trunc.v4f64(<4 x double>, metadata, metadata)
+declare <4 x double> @llvm.experimental.constrained.ceil.v4f64(<4 x double>, metadata)
+declare <4 x double> @llvm.experimental.constrained.floor.v4f64(<4 x double>, metadata)
+declare <4 x double> @llvm.experimental.constrained.round.v4f64(<4 x double>, metadata)
+declare <4 x double> @llvm.experimental.constrained.trunc.v4f64(<4 x double>, metadata)
diff --git a/llvm/test/CodeGen/Thumb/PR17309.ll b/llvm/test/CodeGen/Thumb/PR17309.ll
index 886953742..9815f899a 100644
--- a/llvm/test/CodeGen/Thumb/PR17309.ll
+++ b/llvm/test/CodeGen/Thumb/PR17309.ll
@@ -51,7 +51,7 @@ declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture) #1
 declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture) #1
 
 
-attributes #0 = { nounwind optsize "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind optsize "less-precise-fpmad"="false" "frame-pointer"="non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind }
-attributes #2 = { optsize "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { optsize "less-precise-fpmad"="false" "frame-pointer"="non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #3 = { nounwind optsize }
diff --git a/llvm/test/CodeGen/Thumb/PR35481.ll b/llvm/test/CodeGen/Thumb/PR35481.ll
index 99137c63d..ebbdb93bb 100644
--- a/llvm/test/CodeGen/Thumb/PR35481.ll
+++ b/llvm/test/CodeGen/Thumb/PR35481.ll
@@ -21,4 +21,4 @@ declare <4 x i32> @g(i32, i32, i32, i32) local_unnamed_addr
 
 declare i32 @h(i32) local_unnamed_addr
 
-attributes #0 = { "disable-tail-calls"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" }
+attributes #0 = { "disable-tail-calls"="false" "frame-pointer"="all" }
diff --git a/llvm/test/CodeGen/Thumb/callee_save.ll b/llvm/test/CodeGen/Thumb/callee_save.ll
index 20140e429..7d019cd4b 100644
--- a/llvm/test/CodeGen/Thumb/callee_save.ll
+++ b/llvm/test/CodeGen/Thumb/callee_save.ll
@@ -110,7 +110,7 @@ entry:
 
 ; All callee-saved registers clobbered, and frame pointer is requested. r7 now
 ; cannot be used while saving/restoring the high regs.
-define void @four_high_four_low_frame_ptr() "no-frame-pointer-elim"="true" {
+define void @four_high_four_low_frame_ptr() "frame-pointer"="all" {
 ; CHECK-LABEL: four_high_four_low_frame_ptr:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r4, r5, r6, r7, lr}
@@ -139,7 +139,7 @@ entry:
 ; All callee-saved registers clobbered, frame pointer is requested and
 ; llvm.returnaddress used. r7 and lr now cannot be used while saving/restoring
 ; the high regs.
-define void @four_high_four_low_frame_ptr_ret_addr() "no-frame-pointer-elim"="true" {
+define void @four_high_four_low_frame_ptr_ret_addr() "frame-pointer"="all" {
 ; CHECK-LABEL: four_high_four_low_frame_ptr_ret_addr:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r4, r5, r6, r7, lr}
@@ -231,7 +231,7 @@ entry:
 ; llvm.returnaddress called. This leaves us with 3 low registers available (r4,
 ; r5, r6), with which to save 4 high registers, so we have to use two pushes
 ; and pops.
-define <4 x i32> @all_of_the_above(i32 %a, i32 %b, i32 %c, i32 %d) "no-frame-pointer-elim"="true" {
+define <4 x i32> @all_of_the_above(i32 %a, i32 %b, i32 %c, i32 %d) "frame-pointer"="all" {
 ; CHECK-LABEL: all_of_the_above:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r4, r5, r6, r7, lr}
diff --git a/llvm/test/CodeGen/Thumb/cmp-and-fold.ll b/llvm/test/CodeGen/Thumb/cmp-and-fold.ll
new file mode 100644
index 000000000..44c873a5f
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb/cmp-and-fold.ll
@@ -0,0 +1,208 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv6-eabi %s -o - | FileCheck %s
+
+define void @test1(i32 %x, void ()* %f)  {
+; CHECK-LABEL: test1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    lsls r0, r0, #2
+; CHECK-NEXT:    cmp r0, #68
+; CHECK-NEXT:    beq .LBB0_2
+; CHECK-NEXT:  @ %bb.1: @ %if.then
+; CHECK-NEXT:    blx r1
+; CHECK-NEXT:  .LBB0_2: @ %if.end
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %a = and i32 %x, 1073741823
+  %cmp = icmp eq i32 %a, 17
+  br i1 %cmp, label %if.end, label %if.then
+
+if.then:
+  tail call void %f()
+  br label %if.end
+
+if.end:
+  ret void
+}
+
+define void @test2(i32 %x, void ()* %f)  {
+; CHECK-LABEL: test2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    lsls r0, r0, #8
+; CHECK-NEXT:    bhi .LBB1_2
+; CHECK-NEXT:  @ %bb.1: @ %if.then
+; CHECK-NEXT:    blx r1
+; CHECK-NEXT:  .LBB1_2: @ %if.end
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %a = shl i32 %x, 7
+  %cmp = icmp ugt i32 %a, 2147483648
+  br i1 %cmp, label %if.end, label %if.then
+
+if.then:
+  tail call void %f()
+  br label %if.end
+
+if.end:
+  ret void
+}
+
+define void @test3(i32 %x, void ()* %f)  {
+; CHECK-LABEL: test3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    lsls r0, r0, #3
+; CHECK-NEXT:    bhi .LBB2_2
+; CHECK-NEXT:  @ %bb.1: @ %if.then
+; CHECK-NEXT:    blx r1
+; CHECK-NEXT:  .LBB2_2: @ %if.end
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %a = and i32 %x, 1073741823
+  %cmp = icmp ugt i32 %a, 536870912
+  br i1 %cmp, label %if.end, label %if.then
+
+if.then:
+  tail call void %f()
+  br label %if.end
+
+if.end:
+  ret void
+}
+
+define void @test4(i32 %x, void ()* %f)  {
+; CHECK-LABEL: test4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    uxtb r0, r0
+; CHECK-NEXT:    cmp r0, #17
+; CHECK-NEXT:    beq .LBB3_2
+; CHECK-NEXT:  @ %bb.1: @ %if.then
+; CHECK-NEXT:    blx r1
+; CHECK-NEXT:  .LBB3_2: @ %if.end
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %a = and i32 %x, 255
+  %cmp = icmp eq i32 %a, 17
+  br i1 %cmp, label %if.end, label %if.then
+
+if.then:
+  tail call void %f()
+  br label %if.end
+
+if.end:
+  ret void
+}
+
+define void @test5(i32 %x, void ()* %f)  {
+; CHECK-LABEL: test5:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    uxth r0, r0
+; CHECK-NEXT:    cmp r0, #17
+; CHECK-NEXT:    beq .LBB4_2
+; CHECK-NEXT:  @ %bb.1: @ %if.then
+; CHECK-NEXT:    blx r1
+; CHECK-NEXT:  .LBB4_2: @ %if.end
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %a = and i32 %x, 65535
+  %cmp = icmp eq i32 %a, 17
+  br i1 %cmp, label %if.end, label %if.then
+
+if.then:
+  tail call void %f()
+  br label %if.end
+
+if.end:
+  ret void
+}
+
+define void @test6(i32 %x, void ()* %f)  {
+; CHECK-LABEL: test6:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    movs r2, #32
+; CHECK-NEXT:    ands r2, r0
+; CHECK-NEXT:    cmp r2, #17
+; CHECK-NEXT:    beq .LBB5_2
+; CHECK-NEXT:  @ %bb.1: @ %if.then
+; CHECK-NEXT:    blx r1
+; CHECK-NEXT:  .LBB5_2: @ %if.end
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %a = and i32 %x, 32
+  %cmp = icmp eq i32 %a, 17
+  br i1 %cmp, label %if.end, label %if.then
+
+if.then:
+  tail call void %f()
+  br label %if.end
+
+if.end:
+  ret void
+}
+
+define void @test7(i32 %x, void ()* %f)  {
+; CHECK-LABEL: test7:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    ldr r2, .LCPI6_0
+; CHECK-NEXT:    ands r2, r0
+; CHECK-NEXT:    cmp r2, #17
+; CHECK-NEXT:    beq .LBB6_2
+; CHECK-NEXT:  @ %bb.1: @ %if.then
+; CHECK-NEXT:    blx r1
+; CHECK-NEXT:  .LBB6_2: @ %if.end
+; CHECK-NEXT:    pop {r7, pc}
+; CHECK-NEXT:    .p2align 2
+; CHECK-NEXT:  @ %bb.3:
+; CHECK-NEXT:  .LCPI6_0:
+; CHECK-NEXT:    .long 1023 @ 0x3ff
+entry:
+  %a = and i32 %x, 1023
+  %cmp = icmp eq i32 %a, 17
+  br i1 %cmp, label %if.end, label %if.then
+
+if.then:
+  tail call void %f()
+  br label %if.end
+
+if.end:
+  ret void
+}
+
+define void @test8(i32 %x, void ()* %f)  {
+; CHECK-LABEL: test8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    movs r2, #129
+; CHECK-NEXT:    lsls r2, r2, #23
+; CHECK-NEXT:    lsls r0, r0, #22
+; CHECK-NEXT:    cmp r0, r2
+; CHECK-NEXT:    beq .LBB7_2
+; CHECK-NEXT:  @ %bb.1: @ %if.then
+; CHECK-NEXT:    blx r1
+; CHECK-NEXT:  .LBB7_2: @ %if.end
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %a = and i32 %x, 1023
+  %cmp = icmp eq i32 %a, 258
+  br i1 %cmp, label %if.end, label %if.then
+
+if.then:
+  tail call void %f()
+  br label %if.end
+
+if.end:
+  ret void
+}
diff --git a/llvm/test/CodeGen/Thumb/fastcc.ll b/llvm/test/CodeGen/Thumb/fastcc.ll
index 1a012465f..be356d8cb 100644
--- a/llvm/test/CodeGen/Thumb/fastcc.ll
+++ b/llvm/test/CodeGen/Thumb/fastcc.ll
@@ -29,7 +29,7 @@ for.body193:                                      ; preds = %for.body193, %for.e
   br label %for.body193
 }
 
-attributes #0 = { optsize "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { optsize "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.ident = !{!0}
 
diff --git a/llvm/test/CodeGen/Thumb/large-stack.ll b/llvm/test/CodeGen/Thumb/large-stack.ll
index 923d2c06d..1242b571f 100644
--- a/llvm/test/CodeGen/Thumb/large-stack.ll
+++ b/llvm/test/CodeGen/Thumb/large-stack.ll
@@ -28,7 +28,7 @@ define void @test100() {
 }
 
 ; Largest stack for which three tADDspi/tSUBspis are enough
-define void @test100_nofpelim() "no-frame-pointer-elim"="true" {
+define void @test100_nofpelim() "frame-pointer"="all" {
 ; CHECK-LABEL: test100_nofpelim:
 ; CHECK: sub sp, #508
 ; CHECK: sub sp, #508
@@ -52,7 +52,7 @@ define void @test2() {
 }
 
 ; Smallest stack for which we use a constant pool
-define void @test2_nofpelim() "no-frame-pointer-elim"="true" {
+define void @test2_nofpelim() "frame-pointer"="all" {
 ; CHECK-LABEL: test2_nofpelim:
 ; CHECK: ldr [[TEMP:r[0-7]]],
 ; CHECK: add sp, [[TEMP]]
@@ -79,7 +79,7 @@ define i32 @test3() {
     ret i32 %tmp1
 }
 
-define i32 @test3_nofpelim() "no-frame-pointer-elim"="true" {
+define i32 @test3_nofpelim() "frame-pointer"="all" {
 ; CHECK-LABEL: test3_nofpelim:
 ; CHECK: ldr [[TEMP:r[0-7]]],
 ; CHECK: add sp, [[TEMP]]
diff --git a/llvm/test/CodeGen/Thumb/ldm-merge-call.ll b/llvm/test/CodeGen/Thumb/ldm-merge-call.ll
index eb7852934..c329f872c 100644
--- a/llvm/test/CodeGen/Thumb/ldm-merge-call.ll
+++ b/llvm/test/CodeGen/Thumb/ldm-merge-call.ll
@@ -19,6 +19,6 @@ entry:
 ; Function Attrs: optsize
 declare i32 @bar(i32, i32, i32, i32) #1
 
-attributes #0 = { nounwind optsize "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { optsize "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind optsize "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { optsize "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind optsize }
diff --git a/llvm/test/CodeGen/Thumb/rev.ll b/llvm/test/CodeGen/Thumb/rev.ll
index 3e947022e..9ac65bf5c 100644
--- a/llvm/test/CodeGen/Thumb/rev.ll
+++ b/llvm/test/CodeGen/Thumb/rev.ll
@@ -1,8 +1,11 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=thumb-eabi -mattr=+v6 %s -o - | FileCheck %s
 
 define i32 @test1(i32 %X) nounwind {
-; CHECK: test1
-; CHECK: rev16 r0, r0
+; CHECK-LABEL: test1:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    rev16 r0, r0
+; CHECK-NEXT:    bx lr
         %tmp1 = lshr i32 %X, 8
         %X15 = bitcast i32 %X to i32
         %tmp4 = shl i32 %X15, 8
@@ -17,8 +20,10 @@ define i32 @test1(i32 %X) nounwind {
 }
 
 define i32 @test2(i32 %X) nounwind {
-; CHECK: test2
-; CHECK: revsh r0, r0
+; CHECK-LABEL: test2:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    revsh r0, r0
+; CHECK-NEXT:    bx lr
         %tmp1 = lshr i32 %X, 8
         %tmp1.upgrd.1 = trunc i32 %tmp1 to i16
         %tmp3 = trunc i32 %X to i16
@@ -31,9 +36,11 @@ define i32 @test2(i32 %X) nounwind {
 
 ; rdar://9147637
 define i32 @test3(i16 zeroext %a) nounwind {
-entry:
 ; CHECK-LABEL: test3:
-; CHECK: revsh r0, r0
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    revsh r0, r0
+; CHECK-NEXT:    bx lr
+entry:
   %0 = tail call i16 @llvm.bswap.i16(i16 %a)
   %1 = sext i16 %0 to i32
   ret i32 %1
@@ -42,9 +49,11 @@ entry:
 declare i16 @llvm.bswap.i16(i16) nounwind readnone
 
 define i32 @test4(i16 zeroext %a) nounwind {
-entry:
 ; CHECK-LABEL: test4:
-; CHECK: revsh r0, r0
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    revsh r0, r0
+; CHECK-NEXT:    bx lr
+entry:
   %conv = zext i16 %a to i32
   %shr9 = lshr i16 %a, 8
   %conv2 = zext i16 %shr9 to i32
diff --git a/llvm/test/CodeGen/Thumb/scheduler-clone-cpsr-def.ll b/llvm/test/CodeGen/Thumb/scheduler-clone-cpsr-def.ll
new file mode 100644
index 000000000..31e54c43c
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb/scheduler-clone-cpsr-def.ll
@@ -0,0 +1,41 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv6-linux-gnueabi < %s | FileCheck %s
+
+; After various DAGCombine optimizations, we end up with an sbcs with
+; multiple uses of the cpsr def, and we therefore clone the subs/sbcs.
+; Make sure this doesn't crash.
+;
+; The output here might change at some point in the future, and no
+; longer clone the operations; if that happens, there probably isn't any
+; straightforward way to fix the test.
+define i64 @f(i64 %x2, i32 %z) {
+; CHECK-LABEL: f:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    .save {r4, r5, r7, lr}
+; CHECK-NEXT:    push {r4, r5, r7, lr}
+; CHECK-NEXT:    movs r2, #0
+; CHECK-NEXT:    subs r3, r0, #1
+; CHECK-NEXT:    mov r3, r1
+; CHECK-NEXT:    sbcs r3, r2
+; CHECK-NEXT:    mov r3, r2
+; CHECK-NEXT:    adcs r3, r2
+; CHECK-NEXT:    movs r4, #30
+; CHECK-NEXT:    subs r5, r0, #1
+; CHECK-NEXT:    mov r5, r1
+; CHECK-NEXT:    sbcs r5, r2
+; CHECK-NEXT:    adcs r4, r2
+; CHECK-NEXT:    lsls r2, r1, #1
+; CHECK-NEXT:    lsls r2, r4
+; CHECK-NEXT:    movs r4, #1
+; CHECK-NEXT:    eors r4, r3
+; CHECK-NEXT:    lsrs r0, r4
+; CHECK-NEXT:    orrs r0, r2
+; CHECK-NEXT:    lsrs r1, r4
+; CHECK-NEXT:    pop {r4, r5, r7, pc}
+  %x3 = add nsw i64 %x2, -1
+  %x8 = icmp ne i64 %x2, 0
+  %x9 = xor i1 %x8, true
+  %x10 = zext i1 %x9 to i64
+  %x11 = lshr i64 %x2, %x10
+  ret i64 %x11
+}
diff --git a/llvm/test/CodeGen/Thumb/stack_guard_remat.ll b/llvm/test/CodeGen/Thumb/stack_guard_remat.ll
index 294c6a6bd..dcf85d262 100644
--- a/llvm/test/CodeGen/Thumb/stack_guard_remat.ll
+++ b/llvm/test/CodeGen/Thumb/stack_guard_remat.ll
@@ -2,25 +2,40 @@
 ; RUN: llc < %s -mtriple=thumb-apple-darwin -relocation-model=static -no-integrated-as | FileCheck %s -check-prefix=NO-PIC  -check-prefix=STATIC
 ; RUN: llc < %s -mtriple=thumb-apple-darwin -relocation-model=dynamic-no-pic -no-integrated-as | FileCheck %s  -check-prefix=NO-PIC -check-prefix=DYNAMIC-NO-PIC
 
-;PIC:   foo2
-;PIC:   ldr [[R0:r[0-9]+]], [[LABEL0:LCPI[0-9_]+]]
-;PIC: [[LABEL1:LPC[0-9_]+]]:
-;PIC:   add [[R0]], pc
-;PIC:   ldr [[R1:r[0-9]+]], {{\[}}[[R0]]{{\]}}
-;PIC:   ldr [[R1:r[0-9]+]], {{\[}}[[R1]]{{\]}}
-
-;PIC:      [[LABEL0]]:
+;PIC:        foo2
+;PIC:        ldr [[SAVED_GUARD:r[0-9]+]], [[GUARD_STACK_OFFSET:LCPI[0-9_]+]]
+;PIC-NEXT:   add [[SAVED_GUARD]], sp
+;PIC-NEXT:   ldr [[SAVED_GUARD]], {{\[}}[[SAVED_GUARD]]{{\]}}
+;PIC-NEXT:   ldr [[ORIGINAL_GUARD:r[0-9]+]], [[ORIGINAL_GUARD_LABEL:LCPI[0-9_]+]]
+;PIC-NEXT: [[LABEL1:LPC[0-9_]+]]:
+;PIC-NEXT:   add [[ORIGINAL_GUARD]], pc
+;PIC-NEXT:   ldr [[ORIGINAL_GUARD]], {{\[}}[[ORIGINAL_GUARD]]{{\]}}
+;PIC-NEXT:   ldr [[ORIGINAL_GUARD]], {{\[}}[[ORIGINAL_GUARD]]{{\]}}
+;PIC-NEXT:   subs {{r[0-9]+}}, [[ORIGINAL_GUARD]], [[SAVED_GUARD]]
+
+;PIC:      [[GUARD_STACK_OFFSET]]:
+;PIC-NEXT:   .long 1028
+;PIC:      [[ORIGINAL_GUARD_LABEL]]:
 ;PIC-NEXT:   .long L___stack_chk_guard$non_lazy_ptr-([[LABEL1]]+4)
 
 ;NO-PIC:   foo2
-;NO-PIC:   ldr [[R0:r[0-9]+]], [[LABEL0:LCPI[0-9_]+]]
+;NO-PIC:                ldr [[SAVED_GUARD:r[0-9]+]], [[GUARD_STACK_OFFSET:LCPI[0-9_]+]]
+;NO-PIC-NEXT:           add [[SAVED_GUARD]], sp
+;NO-PIC-NEXT:           ldr [[SAVED_GUARD]], {{\[}}[[SAVED_GUARD]]{{\]}}
+;NO-PIC-NEXT:           ldr [[ORIGINAL_GUARD:r[0-9]+]], [[ORIGINAL_GUARD_LABEL:LCPI[0-9_]+]]
 ;NO-PIC-NOT: LPC
-;NO-PIC:   ldr {{r[0-9]+}}, {{\[}}[[R0]]{{\]}}
+;NO-PIC-NEXT:           ldr [[ORIGINAL_GUARD]], {{\[}}[[ORIGINAL_GUARD]]{{\]}}
+;DYNAMIC-NO-PIC-NEXT:   ldr [[ORIGINAL_GUARD]], {{\[}}[[ORIGINAL_GUARD]]{{\]}}
+;NO-PIC-NEXT:           subs {{r[0-9]+}}, [[ORIGINAL_GUARD]], [[SAVED_GUARD]]
 
-;STATIC:      [[LABEL0]]:
+;STATIC:      [[GUARD_STACK_OFFSET]]:
+;STATIC-NEXT:   .long 1028
+;STATIC:      [[ORIGINAL_GUARD_LABEL]]:
 ;STATIC-NEXT:   .long ___stack_chk_guard
 
-;DYNAMIC-NO-PIC:      [[LABEL0]]:
+;DYNAMIC-NO-PIC:      [[GUARD_STACK_OFFSET]]:
+;DYNAMIC-NO-PIC-NEXT:   .long 1028
+;DYNAMIC-NO-PIC:      [[ORIGINAL_GUARD_LABEL]]:
 ;DYNAMIC-NO-PIC-NEXT:   .long L___stack_chk_guard$non_lazy_ptr
 
 ; Function Attrs: nounwind ssp
@@ -43,4 +58,4 @@ declare void @foo3(i32*)
 ; Function Attrs: nounwind
 declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture)
 
-attributes #0 = { nounwind ssp "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind ssp "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/Thumb/stm-merge.ll b/llvm/test/CodeGen/Thumb/stm-merge.ll
index 5492ad8f5..2eaf8ff2d 100644
--- a/llvm/test/CodeGen/Thumb/stm-merge.ll
+++ b/llvm/test/CodeGen/Thumb/stm-merge.ll
@@ -38,4 +38,4 @@ for.end8:                                         ; preds = %for.body5
   ret void
 }
 
-attributes #0 = { nounwind optsize "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind optsize "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/Thumb/stm-scavenging.ll b/llvm/test/CodeGen/Thumb/stm-scavenging.ll
index 3ed5763f2..84a0d505a 100644
--- a/llvm/test/CodeGen/Thumb/stm-scavenging.ll
+++ b/llvm/test/CodeGen/Thumb/stm-scavenging.ll
@@ -7,7 +7,7 @@ target triple = "thumbv6---gnueabi"
 ; CHECK: .setfp  r7, sp
 ; CHECK: stm r3!, {r0, r1, r2}
 ; CHECK: bl throws_1
-define void @use_stm(i32 %a, i32 %b, i32 %c, i32* %d) local_unnamed_addr noreturn "no-frame-pointer-elim"="true" {
+define void @use_stm(i32 %a, i32 %b, i32 %c, i32* %d) local_unnamed_addr noreturn "frame-pointer"="all" {
 entry:
   %arrayidx = getelementptr inbounds i32, i32* %d, i32 2
   store i32 %a, i32* %arrayidx, align 4
@@ -29,7 +29,7 @@ entry:
 ; CHECK: str r1,
 ; CHECK: str r2,
 ; CHECK: bl throws_2
-define void @no_stm(i32 %a, i32 %b, i32 %c, i32* %d) local_unnamed_addr noreturn "no-frame-pointer-elim"="true" {
+define void @no_stm(i32 %a, i32 %b, i32 %c, i32* %d) local_unnamed_addr noreturn "frame-pointer"="all" {
 entry:
   %arrayidx = getelementptr inbounds i32, i32* %d, i32 2
   store i32 %a, i32* %arrayidx, align 4
diff --git a/llvm/test/CodeGen/Thumb2/2009-07-21-ISelBug.ll b/llvm/test/CodeGen/Thumb2/2009-07-21-ISelBug.ll
index 1b3cb9920..1d5d03866 100644
--- a/llvm/test/CodeGen/Thumb2/2009-07-21-ISelBug.ll
+++ b/llvm/test/CodeGen/Thumb2/2009-07-21-ISelBug.ll
@@ -3,7 +3,7 @@
 
 @"\01LC" = external constant [36 x i8], align 1		; <[36 x i8]*> [#uses=1]
 
-define i32 @t(i32, ...) nounwind "no-frame-pointer-elim"="true" {
+define i32 @t(i32, ...) nounwind "frame-pointer"="all" {
 entry:
 ; CHECK-LABEL: t:
 ; CHECK: add r7, sp, #12
diff --git a/llvm/test/CodeGen/Thumb2/2010-02-11-phi-cycle.ll b/llvm/test/CodeGen/Thumb2/2010-02-11-phi-cycle.ll
index fff83c546..bf3a0f803 100644
--- a/llvm/test/CodeGen/Thumb2/2010-02-11-phi-cycle.ll
+++ b/llvm/test/CodeGen/Thumb2/2010-02-11-phi-cycle.ll
@@ -31,9 +31,14 @@ return:                                           ; preds = %bb, %entry
 
 define i32 @test_dead_cycle(i32 %n) nounwind {
 ; CHECK-LABEL: test_dead_cycle:
+; CHECK: subs
+; also check for duplicate induction variables (radar 7645034)
+; CHECK: subs r{{.*}}, #1
+; CHECK-NOT: subs r{{.*}}, #1
 ; CHECK: bl
 ; CHECK-NOT: mov
 ; CHECK: bl
+; CHECK: pop
 entry:
   %0 = icmp eq i32 %n, 1                          ; <i1> [#uses=1]
   br i1 %0, label %return, label %bb.nph
@@ -58,10 +63,6 @@ bb1:                                              ; preds = %bb
   br label %bb2
 
 bb2:                                              ; preds = %bb1, %bb
-; also check for duplicate induction variables (radar 7645034)
-; CHECK: subs r{{.*}}, #1
-; CHECK-NOT: subs r{{.*}}, #1
-; CHECK: pop
   %u.0 = phi i64 [ %ins, %bb1 ], [ %u.17, %bb ]   ; <i64> [#uses=2]
   %indvar.next = add i32 %indvar, 1               ; <i32> [#uses=2]
   %exitcond = icmp eq i32 %indvar.next, %tmp      ; <i1> [#uses=1]
diff --git a/llvm/test/CodeGen/Thumb2/2010-03-15-AsmCCClobber.ll b/llvm/test/CodeGen/Thumb2/2010-03-15-AsmCCClobber.ll
index de45f72c8..b28f4542c 100644
--- a/llvm/test/CodeGen/Thumb2/2010-03-15-AsmCCClobber.ll
+++ b/llvm/test/CodeGen/Thumb2/2010-03-15-AsmCCClobber.ll
@@ -15,9 +15,11 @@ target datalayout = "e-p:32:32:32-i1:8:32-i8:8:32-i16:16:32-i32:32:32-i64:32:32-
 
 ; Make sure the cmp is not scheduled before the InlineAsm that clobbers cc.
 ; CHECK: bl _f2
-; CHECK: cmp {{r[0-9]+}}, #0
-; CHECK-NOT: cmp
-; CHECK: InlineAsm Start
+; CHECK: clz {{r[0-9]+}}
+; CHECK-DAG: lsrs    {{r[0-9]+}}
+; CHECK-DAG: lsls    {{r[0-9]+}}
+; CHECK-NEXT: orr.w   {{r[0-9]+}}
+; CHECK-NEXT: InlineAsm Start
 define void @test(%s1* %this, i32 %format, i32 %w, i32 %h, i32 %levels, i32* %s, i8* %data, i32* nocapture %rowbytes, void (i8*, i8*)* %release, i8* %info) nounwind {
 entry:
   %tmp1 = getelementptr inbounds %s1, %s1* %this, i32 0, i32 0, i32 0, i32 1, i32 0, i32 0
diff --git a/llvm/test/CodeGen/Thumb2/2010-11-22-EpilogueBug.ll b/llvm/test/CodeGen/Thumb2/2010-11-22-EpilogueBug.ll
index 0372bd7be..9610cf7ab 100644
--- a/llvm/test/CodeGen/Thumb2/2010-11-22-EpilogueBug.ll
+++ b/llvm/test/CodeGen/Thumb2/2010-11-22-EpilogueBug.ll
@@ -5,7 +5,7 @@
 
 declare void @bar() nounwind optsize
 
-define void @foo() nounwind optsize "no-frame-pointer-elim"="true" {
+define void @foo() nounwind optsize "frame-pointer"="all" {
 ; CHECK-LABEL: foo:
 ; CHECK: push
 ; CHECK: add r7, sp
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/basic-tail-pred.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/basic-tail-pred.ll
new file mode 100644
index 000000000..257d950c6
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/basic-tail-pred.ll
@@ -0,0 +1,337 @@
+; RUN: opt -mtriple=thumbv8.1m.main -mve-tail-predication -disable-mve-tail-predication=false -mattr=+mve,+lob %s -S -o - | FileCheck %s
+
+; CHECK-LABEL: mul_v16i8
+; CHECK: vector.body:
+; CHECK: %index = phi i32
+; CHECK: [[ELEMS:%[^ ]+]] = phi i32 [ %N, %vector.ph ], [ [[REMAINING:%[^ ]+]], %vector.body ]
+; CHECK: [[VCTP:%[^ ]+]] = call <16 x i1> @llvm.arm.mve.vctp8(i32 [[ELEMS]])
+; CHECK: [[REMAINING]] = sub i32 [[ELEMS]], 16
+; CHECK: [[LD0:%[^ ]+]] = tail call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* {{.*}}, i32 4, <16 x i1> [[VCTP]], <16 x i8> undef)
+; CHECK: [[LD1:%[^ ]+]] = tail call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* {{.*}}, i32 4, <16 x i1> [[VCTP]], <16 x i8> undef)
+; CHECK: tail call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> {{.*}}, <16 x i8>* {{.*}}, i32 4, <16 x i1> [[VCTP]])
+define dso_local arm_aapcs_vfpcc void @mul_v16i8(i8* noalias nocapture readonly %a, i8* noalias nocapture readonly %b, i8* noalias nocapture %c, i32 %N) {
+entry:
+  %cmp8 = icmp eq i32 %N, 0
+  %tmp8 = add i32 %N, 15
+  %tmp9 = lshr i32 %tmp8, 4
+  %tmp10 = shl nuw i32 %tmp9, 4
+  %tmp11 = add i32 %tmp10, -16
+  %tmp12 = lshr i32 %tmp11, 4
+  %tmp13 = add nuw nsw i32 %tmp12, 1
+  br i1 %cmp8, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert10 = insertelement <16 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat11 = shufflevector <16 x i32> %broadcast.splatinsert10, <16 x i32> undef, <16 x i32> zeroinitializer
+  call void @llvm.set.loop.iterations.i32(i32 %tmp13)
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %tmp14 = phi i32 [ %tmp13, %vector.ph ], [ %tmp15, %vector.body ]
+  %broadcast.splatinsert = insertelement <16 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <16 x i32> %broadcast.splatinsert, <16 x i32> undef, <16 x i32> zeroinitializer
+  %induction = add <16 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %tmp = getelementptr inbounds i8, i8* %a, i32 %index
+  %tmp1 = icmp ule <16 x i32> %induction, %broadcast.splat11
+  %tmp2 = bitcast i8* %tmp to <16 x i8>*
+  %wide.masked.load = tail call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %tmp2, i32 4, <16 x i1> %tmp1, <16 x i8> undef)
+  %tmp3 = getelementptr inbounds i8, i8* %b, i32 %index
+  %tmp4 = bitcast i8* %tmp3 to <16 x i8>*
+  %wide.masked.load2 = tail call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %tmp4, i32 4, <16 x i1> %tmp1, <16 x i8> undef)
+  %mul = mul nsw <16 x i8> %wide.masked.load2, %wide.masked.load
+  %tmp6 = getelementptr inbounds i8, i8* %c, i32 %index
+  %tmp7 = bitcast i8* %tmp6 to <16 x i8>*
+  tail call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %mul, <16 x i8>* %tmp7, i32 4, <16 x i1> %tmp1)
+  %index.next = add i32 %index, 16
+  %tmp15 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %tmp14, i32 1)
+  %tmp16 = icmp ne i32 %tmp15, 0
+  br i1 %tmp16, label %vector.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %vector.body, %entry
+  ret void
+}
+
+; CHECK-LABEL: mul_v8i16
+; CHECK: vector.body:
+; CHECK: %index = phi i32
+; CHECK: [[ELEMS:%[^ ]+]] = phi i32 [ %N, %vector.ph ], [ [[REMAINING:%[^ ]+]], %vector.body ]
+; CHECK: [[VCTP:%[^ ]+]] = call <8 x i1> @llvm.arm.mve.vctp16(i32 [[ELEMS]])
+; CHECK: [[REMAINING]] = sub i32 [[ELEMS]], 8
+; CHECK: [[LD0:%[^ ]+]] = tail call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* {{.*}}, i32 4, <8 x i1> [[VCTP]], <8 x i16> undef)
+; CHECK: [[LD1:%[^ ]+]] = tail call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* {{.*}}, i32 4, <8 x i1> [[VCTP]], <8 x i16> undef)
+; CHECK: tail call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> {{.*}}, <8 x i16>* {{.*}}, i32 4, <8 x i1> [[VCTP]])
+define dso_local arm_aapcs_vfpcc void @mul_v8i16(i16* noalias nocapture readonly %a, i16* noalias nocapture readonly %b, i16* noalias nocapture %c, i32 %N) {
+entry:
+  %cmp8 = icmp eq i32 %N, 0
+  %tmp8 = add i32 %N, 7
+  %tmp9 = lshr i32 %tmp8, 3
+  %tmp10 = shl nuw i32 %tmp9, 3
+  %tmp11 = add i32 %tmp10, -8
+  %tmp12 = lshr i32 %tmp11, 3
+  %tmp13 = add nuw nsw i32 %tmp12, 1
+  br i1 %cmp8, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert10 = insertelement <8 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat11 = shufflevector <8 x i32> %broadcast.splatinsert10, <8 x i32> undef, <8 x i32> zeroinitializer
+  call void @llvm.set.loop.iterations.i32(i32 %tmp13)
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %tmp14 = phi i32 [ %tmp13, %vector.ph ], [ %tmp15, %vector.body ]
+  %broadcast.splatinsert = insertelement <8 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <8 x i32> %broadcast.splatinsert, <8 x i32> undef, <8 x i32> zeroinitializer
+  %induction = add <8 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %tmp = getelementptr inbounds i16, i16* %a, i32 %index
+  %tmp1 = icmp ule <8 x i32> %induction, %broadcast.splat11
+  %tmp2 = bitcast i16* %tmp to <8 x i16>*
+  %wide.masked.load = tail call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %tmp2, i32 4, <8 x i1> %tmp1, <8 x i16> undef)
+  %tmp3 = getelementptr inbounds i16, i16* %b, i32 %index
+  %tmp4 = bitcast i16* %tmp3 to <8 x i16>*
+  %wide.masked.load2 = tail call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %tmp4, i32 4, <8 x i1> %tmp1, <8 x i16> undef)
+  %mul = mul nsw <8 x i16> %wide.masked.load2, %wide.masked.load
+  %tmp6 = getelementptr inbounds i16, i16* %c, i32 %index
+  %tmp7 = bitcast i16* %tmp6 to <8 x i16>*
+  tail call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %mul, <8 x i16>* %tmp7, i32 4, <8 x i1> %tmp1)
+  %index.next = add i32 %index, 8
+  %tmp15 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %tmp14, i32 1)
+  %tmp16 = icmp ne i32 %tmp15, 0
+  br i1 %tmp16, label %vector.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %vector.body, %entry
+  ret void
+}
+
+; CHECK-LABEL: mul_v4i32
+; CHECK: vector.body:
+; CHECK: [[ELEMS:%[^ ]+]] = phi i32 [ %N, %vector.ph ], [ [[REMAINING:%[^ ]+]], %vector.body ]
+; CHECK: [[VCTP:%[^ ]+]] = call <4 x i1> @llvm.arm.mve.vctp32(i32 [[ELEMS]])
+; CHECK: [[REMAINING]] = sub i32 [[ELEMS]], 4
+; CHECK: [[LD0:%[^ ]+]] = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* {{.*}}, i32 4, <4 x i1> [[VCTP]], <4 x i32> undef)
+; CHECK: [[LD1:%[^ ]+]] = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* {{.*}}, i32 4, <4 x i1> [[VCTP]], <4 x i32> undef)
+; CHECK: tail call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> {{.*}}, <4 x i32>* {{.*}}, i32 4, <4 x i1> [[VCTP]])
+define dso_local arm_aapcs_vfpcc void @mul_v4i32(i32* noalias nocapture readonly %a, i32* noalias nocapture readonly %b, i32* noalias nocapture %c, i32 %N) {
+entry:
+  %cmp8 = icmp eq i32 %N, 0
+  %tmp8 = add i32 %N, 3
+  %tmp9 = lshr i32 %tmp8, 2
+  %tmp10 = shl nuw i32 %tmp9, 2
+  %tmp11 = add i32 %tmp10, -4
+  %tmp12 = lshr i32 %tmp11, 2
+  %tmp13 = add nuw nsw i32 %tmp12, 1
+  br i1 %cmp8, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert10 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat11 = shufflevector <4 x i32> %broadcast.splatinsert10, <4 x i32> undef, <4 x i32> zeroinitializer
+  call void @llvm.set.loop.iterations.i32(i32 %tmp13)
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %tmp14 = phi i32 [ %tmp13, %vector.ph ], [ %tmp15, %vector.body ]
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+  %tmp = getelementptr inbounds i32, i32* %a, i32 %index
+  %tmp1 = icmp ule <4 x i32> %induction, %broadcast.splat11
+  %tmp2 = bitcast i32* %tmp to <4 x i32>*
+  %wide.masked.load = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp2, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
+  %tmp3 = getelementptr inbounds i32, i32* %b, i32 %index
+  %tmp4 = bitcast i32* %tmp3 to <4 x i32>*
+  %wide.masked.load2 = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp4, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
+  %mul = mul nsw <4 x i32> %wide.masked.load2, %wide.masked.load
+  %tmp6 = getelementptr inbounds i32, i32* %c, i32 %index
+  %tmp7 = bitcast i32* %tmp6 to <4 x i32>*
+  tail call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %mul, <4 x i32>* %tmp7, i32 4, <4 x i1> %tmp1)
+  %index.next = add i32 %index, 4
+  %tmp15 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %tmp14, i32 1)
+  %tmp16 = icmp ne i32 %tmp15, 0
+  br i1 %tmp16, label %vector.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %vector.body, %entry
+  ret void
+}
+
+; CHECK-LABEL: split_vector
+; CHECK: vector.body:
+; CHECK: %index = phi i32
+; CHECK: [[ELEMS:%[^ ]+]] = phi i32 [ %N, %vector.ph ], [ [[REMAINING:%[^ ]+]], %vector.body ]
+; CHECK: [[VCTP:%[^ ]+]] = call <4 x i1> @llvm.arm.mve.vctp32(i32 [[ELEMS]])
+; CHECK: [[REMAINING]] = sub i32 [[ELEMS]], 4
+; CHECK: [[LD0:%[^ ]+]] = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* {{.*}}, i32 4, <4 x i1> [[VCTP]], <4 x i32> undef)
+; CHECK: [[LD1:%[^ ]+]] = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* {{.*}}, i32 4, <4 x i1> [[VCTP]], <4 x i32> undef)
+; CHECK: tail call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> {{.*}}, <4 x i32>* {{.*}}, i32 4, <4 x i1> [[VCTP]])
+define dso_local arm_aapcs_vfpcc void @split_vector(i32* noalias nocapture readonly %a, i32* noalias nocapture readonly %b, i32* noalias nocapture %c, i32 %N) {
+entry:
+  %cmp8 = icmp eq i32 %N, 0
+  %tmp8 = add i32 %N, 3
+  %tmp9 = lshr i32 %tmp8, 2
+  %tmp10 = shl nuw i32 %tmp9, 2
+  %tmp11 = add i32 %tmp10, -4
+  %tmp12 = lshr i32 %tmp11, 2
+  %tmp13 = add nuw nsw i32 %tmp12, 1
+  br i1 %cmp8, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert10 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat11 = shufflevector <4 x i32> %broadcast.splatinsert10, <4 x i32> undef, <4 x i32> zeroinitializer
+  call void @llvm.set.loop.iterations.i32(i32 %tmp13)
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %tmp14 = phi i32 [ %tmp13, %vector.ph ], [ %tmp15, %vector.body ]
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+  %tmp = getelementptr inbounds i32, i32* %a, i32 %index
+  %tmp1 = icmp ule <4 x i32> %induction, %broadcast.splat11
+  %tmp2 = bitcast i32* %tmp to <4 x i32>*
+  %wide.masked.load = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp2, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
+  %extract.1.low = shufflevector <4 x i32> %wide.masked.load, <4 x i32> undef, < 2 x i32> < i32 0, i32 2>
+  %extract.1.high = shufflevector <4 x i32> %wide.masked.load, <4 x i32> undef, < 2 x i32> < i32 1, i32 3>
+  %tmp3 = getelementptr inbounds i32, i32* %b, i32 %index
+  %tmp4 = bitcast i32* %tmp3 to <4 x i32>*
+  %wide.masked.load2 = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp4, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
+  %extract.2.low = shufflevector <4 x i32> %wide.masked.load2, <4 x i32> undef, < 2 x i32> < i32 0, i32 2>
+  %extract.2.high = shufflevector <4 x i32> %wide.masked.load2, <4 x i32> undef, < 2 x i32> < i32 1, i32 3>
+  %mul = mul nsw <2 x i32> %extract.1.low, %extract.2.low
+  %sub = sub nsw <2 x i32> %extract.1.high, %extract.2.high
+  %combine = shufflevector <2 x i32> %mul, <2 x i32> %sub, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %tmp6 = getelementptr inbounds i32, i32* %c, i32 %index
+  %tmp7 = bitcast i32* %tmp6 to <4 x i32>*
+  tail call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %combine, <4 x i32>* %tmp7, i32 4, <4 x i1> %tmp1)
+  %index.next = add i32 %index, 4
+  %tmp15 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %tmp14, i32 1)
+  %tmp16 = icmp ne i32 %tmp15, 0
+  br i1 %tmp16, label %vector.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %vector.body, %entry
+  ret void
+}
+
+; One of the loads now uses ult predicate.
+; CHECK-LABEL: mismatch_load_pred
+; CHECK: [[ELEMS:%[^ ]+]] = phi i32 [ %N, %vector.ph ], [ [[REMAINING:%[^ ]+]], %vector.body ]
+; CHECK: [[VCTP:%[^ ]+]] = call <4 x i1> @llvm.arm.mve.vctp32(i32 [[ELEMS]])
+; CHECK: [[REMAINING]] = sub i32 [[ELEMS]], 4
+; CHECK: [[LD0:%[^ ]+]] = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* {{.*}}, i32 4, <4 x i1> [[VCTP]], <4 x i32> undef)
+; CHECK: [[LD1:%[^ ]+]] = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* {{.*}}, i32 4, <4 x i1> %wrong, <4 x i32> undef)
+; CHECK: tail call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> {{.*}}, <4 x i32>* {{.*}}, i32 4, <4 x i1> [[VCTP]])
+define dso_local arm_aapcs_vfpcc void @mismatch_load_pred(i32* noalias nocapture readonly %a, i32* noalias nocapture readonly %b, i32* noalias nocapture %c, i32 %N) {
+entry:
+  %cmp8 = icmp eq i32 %N, 0
+  %tmp8 = add i32 %N, 3
+  %tmp9 = lshr i32 %tmp8, 2
+  %tmp10 = shl nuw i32 %tmp9, 2
+  %tmp11 = add i32 %tmp10, -4
+  %tmp12 = lshr i32 %tmp11, 2
+  %tmp13 = add nuw nsw i32 %tmp12, 1
+  br i1 %cmp8, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert10 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat11 = shufflevector <4 x i32> %broadcast.splatinsert10, <4 x i32> undef, <4 x i32> zeroinitializer
+  call void @llvm.set.loop.iterations.i32(i32 %tmp13)
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %tmp14 = phi i32 [ %tmp13, %vector.ph ], [ %tmp15, %vector.body ]
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+  %tmp = getelementptr inbounds i32, i32* %a, i32 %index
+  %tmp1 = icmp ule <4 x i32> %induction, %broadcast.splat11
+  %wrong = icmp ult <4 x i32> %induction, %broadcast.splat11
+  %tmp2 = bitcast i32* %tmp to <4 x i32>*
+  %wide.masked.load = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp2, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
+  %tmp3 = getelementptr inbounds i32, i32* %b, i32 %index
+  %tmp4 = bitcast i32* %tmp3 to <4 x i32>*
+  %wide.masked.load12 = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp4, i32 4, <4 x i1> %wrong, <4 x i32> undef)
+  %tmp5 = mul nsw <4 x i32> %wide.masked.load12, %wide.masked.load
+  %tmp6 = getelementptr inbounds i32, i32* %c, i32 %index
+  %tmp7 = bitcast i32* %tmp6 to <4 x i32>*
+  tail call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %tmp5, <4 x i32>* %tmp7, i32 4, <4 x i1> %tmp1)
+  %index.next = add i32 %index, 4
+  %tmp15 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %tmp14, i32 1)
+  %tmp16 = icmp ne i32 %tmp15, 0
+  br i1 %tmp16, label %vector.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %vector.body, %entry
+  ret void
+}
+
+; The store now uses ult predicate.
+; CHECK-LABEL: mismatch_store_pred
+; CHECK: %index = phi i32
+; CHECK: [[ELEMS:%[^ ]+]] = phi i32 [ %N, %vector.ph ], [ [[REMAINING:%[^ ]+]], %vector.body ]
+; CHECK: [[VCTP:%[^ ]+]] = call <4 x i1> @llvm.arm.mve.vctp32(i32 [[ELEMS]])
+; CHECK: [[REMAINING]] = sub i32 [[ELEMS]], 4
+; CHECK: [[LD0:%[^ ]+]] = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* {{.*}}, i32 4, <4 x i1> [[VCTP]], <4 x i32> undef)
+; CHECK: [[LD1:%[^ ]+]] = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* {{.*}}, i32 4, <4 x i1> [[VCTP]], <4 x i32> undef)
+; CHECK: tail call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> {{.*}}, <4 x i32>* {{.*}}, i32 4, <4 x i1> %wrong)
+define dso_local arm_aapcs_vfpcc void @mismatch_store_pred(i32* noalias nocapture readonly %a, i32* noalias nocapture readonly %b, i32* noalias nocapture %c, i32 %N) {
+entry:
+  %cmp8 = icmp eq i32 %N, 0
+  %tmp8 = add i32 %N, 3
+  %tmp9 = lshr i32 %tmp8, 2
+  %tmp10 = shl nuw i32 %tmp9, 2
+  %tmp11 = add i32 %tmp10, -4
+  %tmp12 = lshr i32 %tmp11, 2
+  %tmp13 = add nuw nsw i32 %tmp12, 1
+  br i1 %cmp8, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert10 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat11 = shufflevector <4 x i32> %broadcast.splatinsert10, <4 x i32> undef, <4 x i32> zeroinitializer
+  call void @llvm.set.loop.iterations.i32(i32 %tmp13)
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %tmp14 = phi i32 [ %tmp13, %vector.ph ], [ %tmp15, %vector.body ]
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+  %tmp = getelementptr inbounds i32, i32* %a, i32 %index
+  %tmp1 = icmp ule <4 x i32> %induction, %broadcast.splat11
+  %wrong = icmp ult <4 x i32> %induction, %broadcast.splat11
+  %tmp2 = bitcast i32* %tmp to <4 x i32>*
+  %wide.masked.load = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp2, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
+  %tmp3 = getelementptr inbounds i32, i32* %b, i32 %index
+  %tmp4 = bitcast i32* %tmp3 to <4 x i32>*
+  %wide.masked.load12 = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp4, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
+  %tmp5 = mul nsw <4 x i32> %wide.masked.load12, %wide.masked.load
+  %tmp6 = getelementptr inbounds i32, i32* %c, i32 %index
+  %tmp7 = bitcast i32* %tmp6 to <4 x i32>*
+  tail call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %tmp5, <4 x i32>* %tmp7, i32 4, <4 x i1> %wrong)
+  %index.next = add i32 %index, 4
+  %tmp15 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %tmp14, i32 1)
+  %tmp16 = icmp ne i32 %tmp15, 0
+  br i1 %tmp16, label %vector.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %vector.body, %entry
+  ret void
+}
+
+declare <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>*, i32 immarg, <16 x i1>, <16 x i8>)
+declare void @llvm.masked.store.v16i8.p0v16i8(<16 x i8>, <16 x i8>*, i32 immarg, <16 x i1>)
+declare <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>*, i32 immarg, <8 x i1>, <8 x i16>)
+declare void @llvm.masked.store.v8i16.p0v8i16(<8 x i16>, <8 x i16>*, i32 immarg, <8 x i1>)
+declare <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>*, i32 immarg, <4 x i1>, <4 x i32>)
+declare void @llvm.masked.store.v2i64.p0v2i64(<2 x i64>, <2 x i64>*, i32 immarg, <2 x i1>)
+declare <2 x i64> @llvm.masked.load.v2i64.p0v2i64(<2 x i64>*, i32 immarg, <2 x i1>, <2 x i64>)
+declare void @llvm.masked.store.v4i32.p0v4i32(<4 x i32>, <4 x i32>*, i32 immarg, <4 x i1>)
+declare void @llvm.set.loop.iterations.i32(i32)
+declare i32 @llvm.loop.decrement.reg.i32.i32.i32(i32, i32)
+
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/branch-targets.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/branch-targets.ll
new file mode 100644
index 000000000..a0f13f3af
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/branch-targets.ll
@@ -0,0 +1,512 @@
+; RUN: llc -mtriple=thumbv8.1m.main -O0 -mattr=+lob -disable-arm-loloops=false -stop-before=arm-low-overhead-loops %s -o - | FileCheck %s --check-prefix=CHECK-MID
+; RUN: llc -mtriple=thumbv8.1m.main -O0 -mattr=+lob -disable-arm-loloops=false -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK-END
+
+; Test that the branch targets are correct after isel, even though the loop
+; will sometimes be reverted anyway.
+
+; CHECK-MID: name: check_loop_dec_brcond_combine
+; CHECK-MID: bb.2.for.body:
+; CHECK-MID:   renamable $lr = t2LoopDec killed renamable $lr, 1
+; CHECK-MID:   t2LoopEnd killed renamable $lr, %bb.3
+; CHECK-MID: bb.3.for.header:
+; CHECK-MID:   tB %bb.2
+
+; CHECK-END: .LBB0_1:
+; CHECK-END:   b .LBB0_3
+; CHECK-END: .LBB0_2:
+; CHECK-END:   subs.w lr, lr, #1
+; CHECK-END:   bne .LBB0_3
+; CHECK-END:   b .LBB0_4
+; CHECK-END: .LBB0_3:
+; CHECK-END:   b .LBB0_2
+define void @check_loop_dec_brcond_combine(i32* nocapture %a, i32* nocapture readonly %b, i32* nocapture readonly %c, i32 %N) {
+entry:
+  call void @llvm.set.loop.iterations.i32(i32 %N)
+  br label %for.body.preheader
+  
+for.body.preheader:
+  %scevgep = getelementptr i32, i32* %a, i32 -1
+  %scevgep4 = getelementptr i32, i32* %c, i32 -1
+  %scevgep8 = getelementptr i32, i32* %b, i32 -1
+  br label %for.header
+ 
+for.body:
+  %scevgep11 = getelementptr i32, i32* %lsr.iv9, i32 1
+  %ld1 = load i32, i32* %scevgep11, align 4
+  %scevgep7 = getelementptr i32, i32* %lsr.iv5, i32 1
+  %ld2 = load i32, i32* %scevgep7, align 4
+  %mul = mul nsw i32 %ld2, %ld1
+  %scevgep3 = getelementptr i32, i32* %lsr.iv1, i32 1
+  store i32 %mul, i32* %scevgep3, align 4
+  %scevgep2 = getelementptr i32, i32* %lsr.iv1, i32 1
+  %scevgep6 = getelementptr i32, i32* %lsr.iv5, i32 1
+  %scevgep10 = getelementptr i32, i32* %lsr.iv9, i32 1
+  %count.next = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %count, i32 1)
+  %cmp = icmp ne i32 %count.next, 0
+  br i1 %cmp, label %for.header, label %for.cond.cleanup
+
+for.header:
+  %lsr.iv9 = phi i32* [ %scevgep8, %for.body.preheader ], [ %scevgep10, %for.body ]
+  %lsr.iv5 = phi i32* [ %scevgep4, %for.body.preheader ], [ %scevgep6, %for.body ]
+  %lsr.iv1 = phi i32* [ %scevgep, %for.body.preheader ], [ %scevgep2, %for.body ]
+  %count = phi i32 [ %N, %for.body.preheader ], [ %count.next, %for.body ]
+  br label %for.body
+
+for.cond.cleanup:
+  ret void
+}
+
+; CHECK-MID: name: check_loop_dec_ugt_brcond_combine
+; CHECK-MID: bb.2.for.body:
+; CHECK-MID:   renamable $lr = t2LoopDec killed renamable $lr, 1
+; CHECK-MID:   t2LoopEnd killed renamable $lr, %bb.3
+; CHECK-MID: bb.3.for.header:
+; CHECK-MID:   tB %bb.2
+define void @check_loop_dec_ugt_brcond_combine(i32* nocapture %a, i32* nocapture readonly %b, i32* nocapture readonly %c, i32 %N) {
+entry:
+  call void @llvm.set.loop.iterations.i32(i32 %N)
+  br label %for.body.preheader
+  
+for.body.preheader:
+  %scevgep = getelementptr i32, i32* %a, i32 -1
+  %scevgep4 = getelementptr i32, i32* %c, i32 -1
+  %scevgep8 = getelementptr i32, i32* %b, i32 -1
+  br label %for.header
+ 
+for.body:
+  %scevgep11 = getelementptr i32, i32* %lsr.iv9, i32 1
+  %ld1 = load i32, i32* %scevgep11, align 4
+  %scevgep7 = getelementptr i32, i32* %lsr.iv5, i32 1
+  %ld2 = load i32, i32* %scevgep7, align 4
+  %mul = mul nsw i32 %ld2, %ld1
+  %scevgep3 = getelementptr i32, i32* %lsr.iv1, i32 1
+  store i32 %mul, i32* %scevgep3, align 4
+  %scevgep2 = getelementptr i32, i32* %lsr.iv1, i32 1
+  %scevgep6 = getelementptr i32, i32* %lsr.iv5, i32 1
+  %scevgep10 = getelementptr i32, i32* %lsr.iv9, i32 1
+  %count.next = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %count, i32 1)
+  %cmp = icmp ugt i32 %count.next, 0
+  br i1 %cmp, label %for.header, label %for.cond.cleanup
+
+for.header:
+  %lsr.iv9 = phi i32* [ %scevgep8, %for.body.preheader ], [ %scevgep10, %for.body ]
+  %lsr.iv5 = phi i32* [ %scevgep4, %for.body.preheader ], [ %scevgep6, %for.body ]
+  %lsr.iv1 = phi i32* [ %scevgep, %for.body.preheader ], [ %scevgep2, %for.body ]
+  %count = phi i32 [ %N, %for.body.preheader ], [ %count.next, %for.body ]
+  br label %for.body
+
+for.cond.cleanup:
+  ret void
+}
+
+; CHECK-MID: name: check_loop_dec_ult_brcond_combine
+; CHECK-MID: bb.2.for.body:
+; CHECK-MID:   renamable $lr = t2LoopDec killed renamable $lr, 1
+; CHECK-MID:   t2LoopEnd killed renamable $lr, %bb.3
+; CHECK-MID: bb.3.for.header:
+; CHECK-MID:   tB %bb.2
+define void @check_loop_dec_ult_brcond_combine(i32* nocapture %a, i32* nocapture readonly %b, i32* nocapture readonly %c, i32 %N) {
+entry:
+  call void @llvm.set.loop.iterations.i32(i32 %N)
+  br label %for.body.preheader
+  
+for.body.preheader:
+  %scevgep = getelementptr i32, i32* %a, i32 -1
+  %scevgep4 = getelementptr i32, i32* %c, i32 -1
+  %scevgep8 = getelementptr i32, i32* %b, i32 -1
+  br label %for.header
+ 
+for.body:
+  %scevgep11 = getelementptr i32, i32* %lsr.iv9, i32 1
+  %ld1 = load i32, i32* %scevgep11, align 4
+  %scevgep7 = getelementptr i32, i32* %lsr.iv5, i32 1
+  %ld2 = load i32, i32* %scevgep7, align 4
+  %mul = mul nsw i32 %ld2, %ld1
+  %scevgep3 = getelementptr i32, i32* %lsr.iv1, i32 1
+  store i32 %mul, i32* %scevgep3, align 4
+  %scevgep2 = getelementptr i32, i32* %lsr.iv1, i32 1
+  %scevgep6 = getelementptr i32, i32* %lsr.iv5, i32 1
+  %scevgep10 = getelementptr i32, i32* %lsr.iv9, i32 1
+  %count.next = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %count, i32 1)
+  %cmp = icmp ult i32 %count.next, 1
+  br i1 %cmp, label %for.cond.cleanup, label %for.header
+
+for.header:
+  %lsr.iv9 = phi i32* [ %scevgep8, %for.body.preheader ], [ %scevgep10, %for.body ]
+  %lsr.iv5 = phi i32* [ %scevgep4, %for.body.preheader ], [ %scevgep6, %for.body ]
+  %lsr.iv1 = phi i32* [ %scevgep, %for.body.preheader ], [ %scevgep2, %for.body ]
+  %count = phi i32 [ %N, %for.body.preheader ], [ %count.next, %for.body ]
+  br label %for.body
+
+for.cond.cleanup:
+  ret void
+}
+
+; CHECK-MID: name: check_loop_dec_ult_xor_brcond_combine
+; CHECK-MIO: bb.2.for.body:
+; CHECK-MID:   t2LoopEnd killed renamable $lr, %bb.3
+; CHECK-MID:   tB %bb.4, 14
+; CHECk-MID: bb.3.for.header:
+; CHECK-MID:   tB %bb.2
+define void @check_loop_dec_ult_xor_brcond_combine(i32* nocapture %a, i32* nocapture readonly %b, i32* nocapture readonly %c, i32 %N) {
+entry:
+  call void @llvm.set.loop.iterations.i32(i32 %N)
+  br label %for.body.preheader
+  
+for.body.preheader:
+  %scevgep = getelementptr i32, i32* %a, i32 -1
+  %scevgep4 = getelementptr i32, i32* %c, i32 -1
+  %scevgep8 = getelementptr i32, i32* %b, i32 -1
+  br label %for.header
+ 
+for.body:
+  %scevgep11 = getelementptr i32, i32* %lsr.iv9, i32 1
+  %ld1 = load i32, i32* %scevgep11, align 4
+  %scevgep7 = getelementptr i32, i32* %lsr.iv5, i32 1
+  %ld2 = load i32, i32* %scevgep7, align 4
+  %mul = mul nsw i32 %ld2, %ld1
+  %scevgep3 = getelementptr i32, i32* %lsr.iv1, i32 1
+  store i32 %mul, i32* %scevgep3, align 4
+  %scevgep2 = getelementptr i32, i32* %lsr.iv1, i32 1
+  %scevgep6 = getelementptr i32, i32* %lsr.iv5, i32 1
+  %scevgep10 = getelementptr i32, i32* %lsr.iv9, i32 1
+  %count.next = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %count, i32 1)
+  %cmp = icmp ult i32 %count.next, 1
+  %negate = xor i1 %cmp, 1
+  br i1 %negate, label %for.header, label %for.cond.cleanup
+
+for.header:
+  %lsr.iv9 = phi i32* [ %scevgep8, %for.body.preheader ], [ %scevgep10, %for.body ]
+  %lsr.iv5 = phi i32* [ %scevgep4, %for.body.preheader ], [ %scevgep6, %for.body ]
+  %lsr.iv1 = phi i32* [ %scevgep, %for.body.preheader ], [ %scevgep2, %for.body ]
+  %count = phi i32 [ %N, %for.body.preheader ], [ %count.next, %for.body ]
+  br label %for.body
+
+for.cond.cleanup:
+  ret void
+}
+
+; CHECK-MID: name: check_loop_dec_sgt_brcond_combine
+; CHECK-MIO: bb.2.for.body:
+; CHECK-MID:   t2LoopEnd killed renamable $lr, %bb.3
+; CHECK-MID:   tB %bb.4, 14
+; CHECk-MID: bb.3.for.header:
+; CHECK-MID:   tB %bb.2
+define void @check_loop_dec_sgt_brcond_combine(i32* nocapture %a, i32* nocapture readonly %b, i32* nocapture readonly %c, i32 %N) {
+entry:
+  call void @llvm.set.loop.iterations.i32(i32 %N)
+  br label %for.body.preheader
+  
+for.body.preheader:
+  %scevgep = getelementptr i32, i32* %a, i32 -1
+  %scevgep4 = getelementptr i32, i32* %c, i32 -1
+  %scevgep8 = getelementptr i32, i32* %b, i32 -1
+  br label %for.header
+ 
+for.body:
+  %scevgep11 = getelementptr i32, i32* %lsr.iv9, i32 1
+  %ld1 = load i32, i32* %scevgep11, align 4
+  %scevgep7 = getelementptr i32, i32* %lsr.iv5, i32 1
+  %ld2 = load i32, i32* %scevgep7, align 4
+  %mul = mul nsw i32 %ld2, %ld1
+  %scevgep3 = getelementptr i32, i32* %lsr.iv1, i32 1
+  store i32 %mul, i32* %scevgep3, align 4
+  %scevgep2 = getelementptr i32, i32* %lsr.iv1, i32 1
+  %scevgep6 = getelementptr i32, i32* %lsr.iv5, i32 1
+  %scevgep10 = getelementptr i32, i32* %lsr.iv9, i32 1
+  %count.next = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %count, i32 1)
+  %cmp = icmp sgt i32 %count.next, 0
+  br i1 %cmp, label %for.header, label %for.cond.cleanup
+
+for.header:
+  %lsr.iv9 = phi i32* [ %scevgep8, %for.body.preheader ], [ %scevgep10, %for.body ]
+  %lsr.iv5 = phi i32* [ %scevgep4, %for.body.preheader ], [ %scevgep6, %for.body ]
+  %lsr.iv1 = phi i32* [ %scevgep, %for.body.preheader ], [ %scevgep2, %for.body ]
+  %count = phi i32 [ %N, %for.body.preheader ], [ %count.next, %for.body ]
+  br label %for.body
+
+for.cond.cleanup:
+  ret void
+}
+
+; CHECK-MID: name: check_loop_dec_sge_brcond_combine
+; CHECK-MIO: bb.2.for.body:
+; CHECK-MID:   t2LoopEnd killed renamable $lr, %bb.3
+; CHECK-MID:   tB %bb.4, 14
+; CHECk-MID: bb.3.for.header:
+; CHECK-MID:   tB %bb.2
+define void @check_loop_dec_sge_brcond_combine(i32* nocapture %a, i32* nocapture readonly %b, i32* nocapture readonly %c, i32 %N) {
+entry:
+  call void @llvm.set.loop.iterations.i32(i32 %N)
+  br label %for.body.preheader
+  
+for.body.preheader:
+  %scevgep = getelementptr i32, i32* %a, i32 -1
+  %scevgep4 = getelementptr i32, i32* %c, i32 -1
+  %scevgep8 = getelementptr i32, i32* %b, i32 -1
+  br label %for.header
+ 
+for.body:
+  %scevgep11 = getelementptr i32, i32* %lsr.iv9, i32 1
+  %ld1 = load i32, i32* %scevgep11, align 4
+  %scevgep7 = getelementptr i32, i32* %lsr.iv5, i32 1
+  %ld2 = load i32, i32* %scevgep7, align 4
+  %mul = mul nsw i32 %ld2, %ld1
+  %scevgep3 = getelementptr i32, i32* %lsr.iv1, i32 1
+  store i32 %mul, i32* %scevgep3, align 4
+  %scevgep2 = getelementptr i32, i32* %lsr.iv1, i32 1
+  %scevgep6 = getelementptr i32, i32* %lsr.iv5, i32 1
+  %scevgep10 = getelementptr i32, i32* %lsr.iv9, i32 1
+  %count.next = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %count, i32 1)
+  %cmp = icmp sge i32 %count.next, 1
+  br i1 %cmp, label %for.header, label %for.cond.cleanup
+
+for.header:
+  %lsr.iv9 = phi i32* [ %scevgep8, %for.body.preheader ], [ %scevgep10, %for.body ]
+  %lsr.iv5 = phi i32* [ %scevgep4, %for.body.preheader ], [ %scevgep6, %for.body ]
+  %lsr.iv1 = phi i32* [ %scevgep, %for.body.preheader ], [ %scevgep2, %for.body ]
+  %count = phi i32 [ %N, %for.body.preheader ], [ %count.next, %for.body ]
+  br label %for.body
+
+for.cond.cleanup:
+  ret void
+}
+
+; CHECK-MID: name: check_loop_dec_sge_xor_brcond_combine
+; CHECK-MIO: bb.2.for.body:
+; CHECK-MID:   t2LoopEnd killed renamable $lr, %bb.3
+; CHECK-MID:   tB %bb.4, 14
+; CHECk-MID: bb.3.for.header:
+; CHECK-MID:   tB %bb.2
+define void @check_loop_dec_sge_xor_brcond_combine(i32* nocapture %a, i32* nocapture readonly %b, i32* nocapture readonly %c, i32 %N) {
+entry:
+  call void @llvm.set.loop.iterations.i32(i32 %N)
+  br label %for.body.preheader
+  
+for.body.preheader:
+  %scevgep = getelementptr i32, i32* %a, i32 -1
+  %scevgep4 = getelementptr i32, i32* %c, i32 -1
+  %scevgep8 = getelementptr i32, i32* %b, i32 -1
+  br label %for.header
+ 
+for.body:
+  %scevgep11 = getelementptr i32, i32* %lsr.iv9, i32 1
+  %ld1 = load i32, i32* %scevgep11, align 4
+  %scevgep7 = getelementptr i32, i32* %lsr.iv5, i32 1
+  %ld2 = load i32, i32* %scevgep7, align 4
+  %mul = mul nsw i32 %ld2, %ld1
+  %scevgep3 = getelementptr i32, i32* %lsr.iv1, i32 1
+  store i32 %mul, i32* %scevgep3, align 4
+  %scevgep2 = getelementptr i32, i32* %lsr.iv1, i32 1
+  %scevgep6 = getelementptr i32, i32* %lsr.iv5, i32 1
+  %scevgep10 = getelementptr i32, i32* %lsr.iv9, i32 1
+  %count.next = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %count, i32 1)
+  %cmp = icmp sge i32 %count.next, 1
+  %negated = xor i1 %cmp, 1
+  br i1 %negated, label %for.cond.cleanup, label %for.header
+
+for.header:
+  %lsr.iv9 = phi i32* [ %scevgep8, %for.body.preheader ], [ %scevgep10, %for.body ]
+  %lsr.iv5 = phi i32* [ %scevgep4, %for.body.preheader ], [ %scevgep6, %for.body ]
+  %lsr.iv1 = phi i32* [ %scevgep, %for.body.preheader ], [ %scevgep2, %for.body ]
+  %count = phi i32 [ %N, %for.body.preheader ], [ %count.next, %for.body ]
+  br label %for.body
+
+for.cond.cleanup:
+  ret void
+}
+
+; CHECK-MID: name: check_loop_dec_uge_brcond_combine
+; CHECK-MIO: bb.2.for.body:
+; CHECK-MID:   t2LoopEnd killed renamable $lr, %bb.3
+; CHECK-MID:   tB %bb.4, 14
+; CHECk-MID: bb.3.for.header:
+; CHECK-MID:   tB %bb.2
+define void @check_loop_dec_uge_brcond_combine(i32* nocapture %a, i32* nocapture readonly %b, i32* nocapture readonly %c, i32 %N) {
+entry:
+  call void @llvm.set.loop.iterations.i32(i32 %N)
+  br label %for.body.preheader
+  
+for.body.preheader:
+  %scevgep = getelementptr i32, i32* %a, i32 -1
+  %scevgep4 = getelementptr i32, i32* %c, i32 -1
+  %scevgep8 = getelementptr i32, i32* %b, i32 -1
+  br label %for.header
+ 
+for.body:
+  %scevgep11 = getelementptr i32, i32* %lsr.iv9, i32 1
+  %ld1 = load i32, i32* %scevgep11, align 4
+  %scevgep7 = getelementptr i32, i32* %lsr.iv5, i32 1
+  %ld2 = load i32, i32* %scevgep7, align 4
+  %mul = mul nsw i32 %ld2, %ld1
+  %scevgep3 = getelementptr i32, i32* %lsr.iv1, i32 1
+  store i32 %mul, i32* %scevgep3, align 4
+  %scevgep2 = getelementptr i32, i32* %lsr.iv1, i32 1
+  %scevgep6 = getelementptr i32, i32* %lsr.iv5, i32 1
+  %scevgep10 = getelementptr i32, i32* %lsr.iv9, i32 1
+  %count.next = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %count, i32 1)
+  %cmp = icmp uge i32 %count.next, 1
+  br i1 %cmp, label %for.header, label %for.cond.cleanup
+
+for.header:
+  %lsr.iv9 = phi i32* [ %scevgep8, %for.body.preheader ], [ %scevgep10, %for.body ]
+  %lsr.iv5 = phi i32* [ %scevgep4, %for.body.preheader ], [ %scevgep6, %for.body ]
+  %lsr.iv1 = phi i32* [ %scevgep, %for.body.preheader ], [ %scevgep2, %for.body ]
+  %count = phi i32 [ %N, %for.body.preheader ], [ %count.next, %for.body ]
+  br label %for.body
+
+for.cond.cleanup:
+  ret void
+}
+
+; CHECK-MID: name: check_loop_dec_uge_xor_brcond_combine
+; CHECK-MIO: bb.2.for.body:
+; CHECK-MID:   t2LoopEnd killed renamable $lr, %bb.3
+; CHECK-MID:   tB %bb.4, 14
+; CHECk-MID: bb.3.for.header:
+; CHECK-MID:   tB %bb.2
+define void @check_loop_dec_uge_xor_brcond_combine(i32* nocapture %a, i32* nocapture readonly %b, i32* nocapture readonly %c, i32 %N) {
+entry:
+  call void @llvm.set.loop.iterations.i32(i32 %N)
+  br label %for.body.preheader
+  
+for.body.preheader:
+  %scevgep = getelementptr i32, i32* %a, i32 -1
+  %scevgep4 = getelementptr i32, i32* %c, i32 -1
+  %scevgep8 = getelementptr i32, i32* %b, i32 -1
+  br label %for.header
+ 
+for.body:
+  %scevgep11 = getelementptr i32, i32* %lsr.iv9, i32 1
+  %ld1 = load i32, i32* %scevgep11, align 4
+  %scevgep7 = getelementptr i32, i32* %lsr.iv5, i32 1
+  %ld2 = load i32, i32* %scevgep7, align 4
+  %mul = mul nsw i32 %ld2, %ld1
+  %scevgep3 = getelementptr i32, i32* %lsr.iv1, i32 1
+  store i32 %mul, i32* %scevgep3, align 4
+  %scevgep2 = getelementptr i32, i32* %lsr.iv1, i32 1
+  %scevgep6 = getelementptr i32, i32* %lsr.iv5, i32 1
+  %scevgep10 = getelementptr i32, i32* %lsr.iv9, i32 1
+  %count.next = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %count, i32 1)
+  %cmp = icmp uge i32 %count.next, 1
+  %negated = xor i1 %cmp, 1
+  br i1 %negated, label %for.cond.cleanup, label %for.header
+
+for.header:
+  %lsr.iv9 = phi i32* [ %scevgep8, %for.body.preheader ], [ %scevgep10, %for.body ]
+  %lsr.iv5 = phi i32* [ %scevgep4, %for.body.preheader ], [ %scevgep6, %for.body ]
+  %lsr.iv1 = phi i32* [ %scevgep, %for.body.preheader ], [ %scevgep2, %for.body ]
+  %count = phi i32 [ %N, %for.body.preheader ], [ %count.next, %for.body ]
+  br label %for.body
+
+for.cond.cleanup:
+  ret void
+}
+
+; CHECK-MID: check_negated_xor_wls
+; CHECK-MID: t2WhileLoopStart killed renamable $r2, %bb.3
+; CHECK-MID: tB %bb.1
+; CHECK-MID: bb.1.while.body.preheader:
+; CHECK-MID:   $lr = t2LoopDec killed renamable $lr, 1
+; CHECK-MID:   t2LoopEnd killed renamable $lr, %bb.2
+; CHECk-MID:   tB %bb.3
+; CHECK-MID: bb.3.while.end:
+define void @check_negated_xor_wls(i16* nocapture %a, i16* nocapture readonly %b, i32 %N) {
+entry:
+  %wls = call i1 @llvm.test.set.loop.iterations.i32(i32 %N)
+  %xor = xor i1 %wls, 1
+  br i1 %xor, label %while.end, label %while.body.preheader
+  
+while.body.preheader:
+  br label %while.body
+  
+while.body:
+  %a.addr.06 = phi i16* [ %incdec.ptr1, %while.body ], [ %a, %while.body.preheader ]
+  %b.addr.05 = phi i16* [ %incdec.ptr, %while.body ], [ %b, %while.body.preheader ]
+  %count = phi i32 [ %N, %while.body.preheader ], [ %count.next, %while.body ]
+  %incdec.ptr = getelementptr inbounds i16, i16* %b.addr.05, i32 1
+  %ld.b = load i16, i16* %b.addr.05, align 2
+  %incdec.ptr1 = getelementptr inbounds i16, i16* %a.addr.06, i32 1
+  store i16 %ld.b, i16* %a.addr.06, align 2
+  %count.next = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %count, i32 1)
+  %cmp = icmp ne i32 %count.next, 0
+  br i1 %cmp, label %while.body, label %while.end
+  
+while.end:
+  ret void
+}
+
+; CHECK-MID: check_negated_cmp_wls
+; CHECK-MID: t2WhileLoopStart killed renamable $r2, %bb.3
+; CHECK-MID: tB %bb.1
+; CHECK-MID: bb.1.while.body.preheader:
+; CHECK-MID:   $lr = t2LoopDec killed renamable $lr, 1
+; CHECK-MID:   t2LoopEnd killed renamable $lr, %bb.2
+; CHECk-MID:   tB %bb.3
+; CHECK-MID: bb.3.while.end:
+define void @check_negated_cmp_wls(i16* nocapture %a, i16* nocapture readonly %b, i32 %N) {
+entry:
+  %wls = call i1 @llvm.test.set.loop.iterations.i32(i32 %N)
+  %cmp = icmp ne i1 %wls, 1
+  br i1 %cmp, label %while.end, label %while.body.preheader
+  
+while.body.preheader:
+  br label %while.body
+  
+while.body:
+  %a.addr.06 = phi i16* [ %incdec.ptr1, %while.body ], [ %a, %while.body.preheader ]
+  %b.addr.05 = phi i16* [ %incdec.ptr, %while.body ], [ %b, %while.body.preheader ]
+  %count = phi i32 [ %N, %while.body.preheader ], [ %count.next, %while.body ]
+  %incdec.ptr = getelementptr inbounds i16, i16* %b.addr.05, i32 1
+  %ld.b = load i16, i16* %b.addr.05, align 2
+  %incdec.ptr1 = getelementptr inbounds i16, i16* %a.addr.06, i32 1
+  store i16 %ld.b, i16* %a.addr.06, align 2
+  %count.next = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %count, i32 1)
+  %cmp.1 = icmp ne i32 %count.next, 0
+  br i1 %cmp.1, label %while.body, label %while.end
+  
+while.end:
+  ret void
+}
+
+; CHECK-MID: check_negated_reordered_wls
+; CHECK-MID: bb.1.while.body.preheader:
+; CHECK-MID:   tB %bb.2
+; CHECK-MID: bb.2.while.body:
+; CHECK-MID:   t2LoopDec killed renamable $lr, 1
+; CHECK-MID:   t2LoopEnd killed renamable $lr, %bb.2
+; CHECK-MID:   tB %bb.4
+; CHECK-MID: bb.3.while:
+; CHECK-MID:   t2WhileLoopStart {{.*}}, %bb.4
+; CHECK-MID: bb.4.while.end
+define void @check_negated_reordered_wls(i16* nocapture %a, i16* nocapture readonly %b, i32 %N) {
+entry:
+  br label %while
+  
+while.body.preheader:
+  br label %while.body
+  
+while.body:
+  %a.addr.06 = phi i16* [ %incdec.ptr1, %while.body ], [ %a, %while.body.preheader ]
+  %b.addr.05 = phi i16* [ %incdec.ptr, %while.body ], [ %b, %while.body.preheader ]
+  %count = phi i32 [ %N, %while.body.preheader ], [ %count.next, %while.body ]
+  %incdec.ptr = getelementptr inbounds i16, i16* %b.addr.05, i32 1
+  %ld.b = load i16, i16* %b.addr.05, align 2
+  %incdec.ptr1 = getelementptr inbounds i16, i16* %a.addr.06, i32 1
+  store i16 %ld.b, i16* %a.addr.06, align 2
+  %count.next = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %count, i32 1)
+  %cmp = icmp ne i32 %count.next, 0
+  br i1 %cmp, label %while.body, label %while.end
+
+while:
+  %wls = call i1 @llvm.test.set.loop.iterations.i32(i32 %N)
+  %xor = xor i1 %wls, 1
+  br i1 %xor, label %while.end, label %while.body.preheader
+
+while.end:
+  ret void
+}
+
+declare void @llvm.set.loop.iterations.i32(i32)
+declare i1 @llvm.test.set.loop.iterations.i32(i32)
+declare i32 @llvm.loop.decrement.reg.i32.i32.i32(i32, i32)
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/cond-vector-reduce-mve-codegen.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/cond-vector-reduce-mve-codegen.ll
new file mode 100644
index 000000000..9bebd5dc5
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/cond-vector-reduce-mve-codegen.ll
@@ -0,0 +1,520 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve -disable-mve-tail-predication=false --verify-machineinstrs %s -o - | FileCheck %s
+
+define dso_local i32 @vpsel_mul_reduce_add(i32* noalias nocapture readonly %a, i32* noalias nocapture readonly %b, i32* noalias nocapture readonly %c, i32 %N) {
+; CHECK-LABEL: vpsel_mul_reduce_add:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    cmp r3, #0
+; CHECK-NEXT:    itt eq
+; CHECK-NEXT:    moveq r0, #0
+; CHECK-NEXT:    bxeq lr
+; CHECK-NEXT:    push {r4, lr}
+; CHECK-NEXT:    sub sp, #4
+; CHECK-NEXT:    adds r4, r3, #3
+; CHECK-NEXT:    vmov.i32 q1, #0x0
+; CHECK-NEXT:    bic r4, r4, #3
+; CHECK-NEXT:    sub.w r12, r4, #4
+; CHECK-NEXT:    movs r4, #1
+; CHECK-NEXT:    add.w lr, r4, r12, lsr #2
+; CHECK-NEXT:    mov.w r12, #0
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB0_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vctp.32 r3
+; CHECK-NEXT:    mov r4, r3
+; CHECK-NEXT:    and r3, r12, #15
+; CHECK-NEXT:    vstr p0, [sp] @ 4-byte Spill
+; CHECK-NEXT:    vdup.32 q3, r3
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    vpstt
+; CHECK-NEXT:    vldrwt.u32 q1, [r2], #16
+; CHECK-NEXT:    vldrwt.u32 q2, [r1], #16
+; CHECK-NEXT:    vcmp.i32 eq, q3, zr
+; CHECK-NEXT:    vpsel q1, q2, q1
+; CHECK-NEXT:    vldr p0, [sp] @ 4-byte Reload
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrwt.u32 q2, [r0], #16
+; CHECK-NEXT:    vmul.i32 q1, q1, q2
+; CHECK-NEXT:    add.w r12, r12, #4
+; CHECK-NEXT:    subs r3, r4, #4
+; CHECK-NEXT:    vadd.i32 q1, q1, q0
+; CHECK-NEXT:    le lr, .LBB0_1
+; CHECK-NEXT:  @ %bb.2: @ %middle.block
+; CHECK-NEXT:    vctp.32 r4
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    vaddv.u32 r0, q0
+; CHECK-NEXT:    add sp, #4
+; CHECK-NEXT:    pop {r4, pc}
+entry:
+  %cmp8 = icmp eq i32 %N, 0
+  br i1 %cmp8, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %n.rnd.up = add i32 %N, 3
+  %n.vec = and i32 %n.rnd.up, -4
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert11 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat12 = shufflevector <4 x i32> %broadcast.splatinsert11, <4 x i32> undef, <4 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %vec.phi = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ %add, %vector.body ]
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+  %tmp = getelementptr inbounds i32, i32* %a, i32 %index
+  %tmp1 = icmp ule <4 x i32> %induction, %broadcast.splat12
+  %tmp2 = bitcast i32* %tmp to <4 x i32>*
+  %wide.masked.load.a = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp2, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
+  %tmp3 = getelementptr inbounds i32, i32* %b, i32 %index
+  %tmp4 = bitcast i32* %tmp3 to <4 x i32>*
+  %wide.masked.load.b = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp4, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
+  %tmp5 = getelementptr inbounds i32, i32* %c, i32 %index
+  %tmp6 = bitcast i32* %tmp5 to <4 x i32>*
+  %wide.masked.load.c = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp6, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
+  %rem = urem i32 %index, 16
+  %rem.broadcast.splatinsert = insertelement <4 x i32> undef, i32 %rem, i32 0
+  %rem.broadcast.splat = shufflevector <4 x i32> %rem.broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %cmp = icmp eq <4 x i32> %rem.broadcast.splat, <i32 0, i32 0, i32 0, i32 0>
+  %wide.masked.load = select <4 x i1> %cmp, <4 x i32> %wide.masked.load.b, <4 x i32> %wide.masked.load.c
+  %mul = mul nsw <4 x i32> %wide.masked.load, %wide.masked.load.a
+  %add = add nsw <4 x i32> %mul, %vec.phi
+  %index.next = add i32 %index, 4
+  %tmp7 = icmp eq i32 %index.next, %n.vec
+  br i1 %tmp7, label %middle.block, label %vector.body
+
+middle.block:                                     ; preds = %vector.body
+  %tmp8 = select <4 x i1> %tmp1, <4 x i32> %add, <4 x i32> %vec.phi
+  %tmp9 = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %tmp8)
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %middle.block, %entry
+  %res.0.lcssa = phi i32 [ 0, %entry ], [ %tmp9, %middle.block ]
+  ret i32 %res.0.lcssa
+}
+
+define dso_local i32 @vpsel_mul_reduce_add_2(i32* noalias nocapture readonly %a, i32* noalias nocapture readonly %b,
+; CHECK-LABEL: vpsel_mul_reduce_add_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    push {r4, r5, r7, lr}
+; CHECK-NEXT:    sub sp, #4
+; CHECK-NEXT:    ldr r5, [sp, #20]
+; CHECK-NEXT:    cmp r5, #0
+; CHECK-NEXT:    beq .LBB1_4
+; CHECK-NEXT:  @ %bb.1: @ %vector.ph
+; CHECK-NEXT:    adds r4, r5, #3
+; CHECK-NEXT:    vmov.i32 q1, #0x0
+; CHECK-NEXT:    bic r4, r4, #3
+; CHECK-NEXT:    sub.w r12, r4, #4
+; CHECK-NEXT:    movs r4, #1
+; CHECK-NEXT:    add.w lr, r4, r12, lsr #2
+; CHECK-NEXT:    mov.w r12, #0
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB1_2: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vctp.32 r5
+; CHECK-NEXT:    mov r4, r5
+; CHECK-NEXT:    and r5, r12, #15
+; CHECK-NEXT:    vstr p0, [sp] @ 4-byte Spill
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    vpstt
+; CHECK-NEXT:    vldrwt.u32 q1, [r3], #16
+; CHECK-NEXT:    vldrwt.u32 q2, [r2], #16
+; CHECK-NEXT:    vdup.32 q3, r5
+; CHECK-NEXT:    vsub.i32 q1, q2, q1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrwt.u32 q2, [r1], #16
+; CHECK-NEXT:    vcmp.i32 eq, q3, zr
+; CHECK-NEXT:    vpsel q1, q1, q2
+; CHECK-NEXT:    vldr p0, [sp] @ 4-byte Reload
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrwt.u32 q2, [r0], #16
+; CHECK-NEXT:    vmul.i32 q1, q1, q2
+; CHECK-NEXT:    add.w r12, r12, #4
+; CHECK-NEXT:    subs r5, r4, #4
+; CHECK-NEXT:    vadd.i32 q1, q1, q0
+; CHECK-NEXT:    le lr, .LBB1_2
+; CHECK-NEXT:  @ %bb.3: @ %middle.block
+; CHECK-NEXT:    vctp.32 r4
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    vaddv.u32 r0, q0
+; CHECK-NEXT:    add sp, #4
+; CHECK-NEXT:    pop {r4, r5, r7, pc}
+; CHECK-NEXT:  .LBB1_4:
+; CHECK-NEXT:    movs r0, #0
+; CHECK-NEXT:    add sp, #4
+; CHECK-NEXT:    pop {r4, r5, r7, pc}
+                                         i32* noalias nocapture readonly %c, i32* noalias nocapture readonly %d, i32 %N) {
+entry:
+  %cmp8 = icmp eq i32 %N, 0
+  br i1 %cmp8, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %n.rnd.up = add i32 %N, 3
+  %n.vec = and i32 %n.rnd.up, -4
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert11 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat12 = shufflevector <4 x i32> %broadcast.splatinsert11, <4 x i32> undef, <4 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %vec.phi = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ %add, %vector.body ]
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+  %tmp = getelementptr inbounds i32, i32* %a, i32 %index
+  %tmp1 = icmp ule <4 x i32> %induction, %broadcast.splat12
+  %tmp2 = bitcast i32* %tmp to <4 x i32>*
+  %wide.masked.load.a = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp2, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
+  %tmp3 = getelementptr inbounds i32, i32* %b, i32 %index
+  %tmp4 = bitcast i32* %tmp3 to <4 x i32>*
+  %wide.masked.load.b = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp4, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
+  %tmp5 = getelementptr inbounds i32, i32* %c, i32 %index
+  %tmp6 = bitcast i32* %tmp5 to <4 x i32>*
+  %wide.masked.load.c = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp6, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
+  %tmp7 = getelementptr inbounds i32, i32* %d, i32 %index
+  %tmp8 = bitcast i32* %tmp7 to <4 x i32>*
+  %wide.masked.load.d = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp8, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
+  %sub = sub <4 x i32> %wide.masked.load.c, %wide.masked.load.d
+  %rem = urem i32 %index, 16
+  %rem.broadcast.splatinsert = insertelement <4 x i32> undef, i32 %rem, i32 0
+  %rem.broadcast.splat = shufflevector <4 x i32> %rem.broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %cmp = icmp eq <4 x i32> %rem.broadcast.splat, <i32 0, i32 0, i32 0, i32 0>
+  %sel = select <4 x i1> %cmp, <4 x i32> %sub, <4 x i32> %wide.masked.load.b
+  %mul = mul  <4 x i32> %sel, %wide.masked.load.a
+  %add = add  <4 x i32> %mul, %vec.phi
+  %index.next = add i32 %index, 4
+  %cmp.exit = icmp eq i32 %index.next, %n.vec
+  br i1 %cmp.exit, label %middle.block, label %vector.body
+
+middle.block:                                     ; preds = %vector.body
+  %acc = select <4 x i1> %tmp1, <4 x i32> %add, <4 x i32> %vec.phi
+  %reduce = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %acc)
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %middle.block, %entry
+  %res.0.lcssa = phi i32 [ 0, %entry ], [ %reduce, %middle.block ]
+  ret i32 %res.0.lcssa
+}
+
+define dso_local i32 @and_mul_reduce_add(i32* noalias nocapture readonly %a, i32* noalias nocapture readonly %b,
+; CHECK-LABEL: and_mul_reduce_add:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    push {r4, r5, r7, lr}
+; CHECK-NEXT:    ldr r5, [sp, #16]
+; CHECK-NEXT:    cbz r5, .LBB2_4
+; CHECK-NEXT:  @ %bb.1: @ %vector.ph
+; CHECK-NEXT:    vmov.i32 q1, #0x0
+; CHECK-NEXT:    dlstp.32 lr, r5
+; CHECK-NEXT:  .LBB2_2: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    vldrw.u32 q1, [r1], #16
+; CHECK-NEXT:    vldrw.u32 q2, [r0], #16
+; CHECK-NEXT:    mov r12, r5
+; CHECK-NEXT:    vsub.i32 q1, q2, q1
+; CHECK-NEXT:    subs r5, #4
+; CHECK-NEXT:    vcmp.i32 eq, q1, zr
+; CHECK-NEXT:    vpstt
+; CHECK-NEXT:    vldrwt.u32 q1, [r3], #16
+; CHECK-NEXT:    vldrwt.u32 q2, [r2], #16
+; CHECK-NEXT:    vmul.i32 q1, q2, q1
+; CHECK-NEXT:    vadd.i32 q1, q1, q0
+; CHECK-NEXT:    letp lr, .LBB2_2
+; CHECK-NEXT:  @ %bb.3: @ %middle.block
+; CHECK-NEXT:    vctp.32 r12
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    vaddv.u32 r0, q0
+; CHECK-NEXT:    pop {r4, r5, r7, pc}
+; CHECK-NEXT:  .LBB2_4:
+; CHECK-NEXT:    movs r0, #0
+; CHECK-NEXT:    pop {r4, r5, r7, pc}
+                                         i32* noalias nocapture readonly %c, i32* noalias nocapture readonly %d, i32 %N) {
+entry:
+  %cmp8 = icmp eq i32 %N, 0
+  br i1 %cmp8, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %n.rnd.up = add i32 %N, 3
+  %n.vec = and i32 %n.rnd.up, -4
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert11 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat12 = shufflevector <4 x i32> %broadcast.splatinsert11, <4 x i32> undef, <4 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %vec.phi = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ %add, %vector.body ]
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+  %tmp = getelementptr inbounds i32, i32* %a, i32 %index
+  %tmp1 = icmp ule <4 x i32> %induction, %broadcast.splat12
+  %tmp2 = bitcast i32* %tmp to <4 x i32>*
+  %wide.masked.load.a = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp2, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
+  %tmp3 = getelementptr inbounds i32, i32* %b, i32 %index
+  %tmp4 = bitcast i32* %tmp3 to <4 x i32>*
+  %wide.masked.load.b = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp4, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
+  %sub = sub <4 x i32> %wide.masked.load.a, %wide.masked.load.b
+  %cmp = icmp eq <4 x i32> %sub, <i32 0, i32 0, i32 0, i32 0>
+  %mask = and <4 x i1> %cmp, %tmp1
+  %tmp5 = getelementptr inbounds i32, i32* %c, i32 %index
+  %tmp6 = bitcast i32* %tmp5 to <4 x i32>*
+  %wide.masked.load.c = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp6, i32 4, <4 x i1> %mask, <4 x i32> undef)
+  %tmp7 = getelementptr inbounds i32, i32* %d, i32 %index
+  %tmp8 = bitcast i32* %tmp7 to <4 x i32>*
+  %wide.masked.load.d = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp8, i32 4, <4 x i1> %mask, <4 x i32> undef)
+  %mul = mul  <4 x i32> %wide.masked.load.c, %wide.masked.load.d
+  %add = add  <4 x i32> %mul, %vec.phi
+  %index.next = add i32 %index, 4
+  %cmp.exit = icmp eq i32 %index.next, %n.vec
+  br i1 %cmp.exit, label %middle.block, label %vector.body
+
+middle.block:                                     ; preds = %vector.body
+  %acc = select <4 x i1> %tmp1, <4 x i32> %add, <4 x i32> %vec.phi
+  %reduce = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %acc)
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %middle.block, %entry
+  %res.0.lcssa = phi i32 [ 0, %entry ], [ %reduce, %middle.block ]
+  ret i32 %res.0.lcssa
+}
+
+define dso_local i32 @or_mul_reduce_add(i32* noalias nocapture readonly %a, i32* noalias nocapture readonly %b, i32* noalias nocapture readonly %c, i32* noalias nocapture readonly %d, i32 %N) {
+; CHECK-LABEL: or_mul_reduce_add:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    push {r4, r5, r7, lr}
+; CHECK-NEXT:    sub sp, #4
+; CHECK-NEXT:    ldr r5, [sp, #20]
+; CHECK-NEXT:    cmp r5, #0
+; CHECK-NEXT:    beq .LBB3_4
+; CHECK-NEXT:  @ %bb.1: @ %vector.ph
+; CHECK-NEXT:    adds r4, r5, #3
+; CHECK-NEXT:    vmov.i32 q1, #0x0
+; CHECK-NEXT:    bic r4, r4, #3
+; CHECK-NEXT:    sub.w r12, r4, #4
+; CHECK-NEXT:    movs r4, #1
+; CHECK-NEXT:    add.w lr, r4, r12, lsr #2
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB3_2: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vctp.32 r5
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    vstr p0, [sp] @ 4-byte Spill
+; CHECK-NEXT:    mov r12, r5
+; CHECK-NEXT:    vpstt
+; CHECK-NEXT:    vldrwt.u32 q1, [r1], #16
+; CHECK-NEXT:    vldrwt.u32 q2, [r0], #16
+; CHECK-NEXT:    vsub.i32 q1, q2, q1
+; CHECK-NEXT:    vcmp.i32 eq, q1, zr
+; CHECK-NEXT:    vmrs r4, p0
+; CHECK-NEXT:    vldr p0, [sp] @ 4-byte Reload
+; CHECK-NEXT:    vmrs r5, p0
+; CHECK-NEXT:    orrs r4, r5
+; CHECK-NEXT:    sub.w r5, r12, #4
+; CHECK-NEXT:    vmsr p0, r4
+; CHECK-NEXT:    vpstt
+; CHECK-NEXT:    vldrwt.u32 q1, [r3], #16
+; CHECK-NEXT:    vldrwt.u32 q2, [r2], #16
+; CHECK-NEXT:    vmul.i32 q1, q2, q1
+; CHECK-NEXT:    vadd.i32 q1, q1, q0
+; CHECK-NEXT:    le lr, .LBB3_2
+; CHECK-NEXT:  @ %bb.3: @ %middle.block
+; CHECK-NEXT:    vctp.32 r12
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    vaddv.u32 r0, q0
+; CHECK-NEXT:    add sp, #4
+; CHECK-NEXT:    pop {r4, r5, r7, pc}
+; CHECK-NEXT:  .LBB3_4:
+; CHECK-NEXT:    movs r0, #0
+; CHECK-NEXT:    add sp, #4
+; CHECK-NEXT:    pop {r4, r5, r7, pc}
+entry:
+  %cmp8 = icmp eq i32 %N, 0
+  br i1 %cmp8, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %n.rnd.up = add i32 %N, 3
+  %n.vec = and i32 %n.rnd.up, -4
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert11 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat12 = shufflevector <4 x i32> %broadcast.splatinsert11, <4 x i32> undef, <4 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %vec.phi = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ %add, %vector.body ]
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+  %tmp = getelementptr inbounds i32, i32* %a, i32 %index
+  %tmp1 = icmp ule <4 x i32> %induction, %broadcast.splat12
+  %tmp2 = bitcast i32* %tmp to <4 x i32>*
+  %wide.masked.load.a = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp2, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
+  %tmp3 = getelementptr inbounds i32, i32* %b, i32 %index
+  %tmp4 = bitcast i32* %tmp3 to <4 x i32>*
+  %wide.masked.load.b = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp4, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
+  %sub = sub <4 x i32> %wide.masked.load.a, %wide.masked.load.b
+  %cmp = icmp eq <4 x i32> %sub, <i32 0, i32 0, i32 0, i32 0>
+  %mask = or <4 x i1> %cmp, %tmp1
+  %tmp5 = getelementptr inbounds i32, i32* %c, i32 %index
+  %tmp6 = bitcast i32* %tmp5 to <4 x i32>*
+  %wide.masked.load.c = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp6, i32 4, <4 x i1> %mask, <4 x i32> undef)
+  %tmp7 = getelementptr inbounds i32, i32* %d, i32 %index
+  %tmp8 = bitcast i32* %tmp7 to <4 x i32>*
+  %wide.masked.load.d = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp8, i32 4, <4 x i1> %mask, <4 x i32> undef)
+  %mul = mul  <4 x i32> %wide.masked.load.c, %wide.masked.load.d
+  %add = add  <4 x i32> %mul, %vec.phi
+  %index.next = add i32 %index, 4
+  %cmp.exit = icmp eq i32 %index.next, %n.vec
+  br i1 %cmp.exit, label %middle.block, label %vector.body
+
+middle.block:                                     ; preds = %vector.body
+  %acc = select <4 x i1> %tmp1, <4 x i32> %add, <4 x i32> %vec.phi
+  %reduce = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %acc)
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %middle.block, %entry
+  %res.0.lcssa = phi i32 [ 0, %entry ], [ %reduce, %middle.block ]
+  ret i32 %res.0.lcssa
+}
+
+define dso_local void @continue_on_zero(i32* noalias nocapture %arg, i32* noalias nocapture readonly %arg1, i32 %arg2) {
+; CHECK-LABEL: continue_on_zero:
+; CHECK:       @ %bb.0: @ %bb
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    it eq
+; CHECK-NEXT:    popeq {r7, pc}
+; CHECK-NEXT:    mov r3, r0
+; CHECK-NEXT:    dlstp.32 lr, r2
+; CHECK-NEXT:  .LBB4_1: @ %bb9
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrw.u32 q0, [r1], #16
+; CHECK-NEXT:    vcmp.i32 ne, q0, zr
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrwt.u32 q1, [r3], #16
+; CHECK-NEXT:    vmul.i32 q0, q1, q0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrwt.32 q0, [r0]
+; CHECK-NEXT:    mov r0, r3
+; CHECK-NEXT:    letp lr, .LBB4_1
+; CHECK-NEXT:  @ %bb.2: @ %bb27
+; CHECK-NEXT:    pop {r7, pc}
+bb:
+  %tmp = icmp eq i32 %arg2, 0
+  br i1 %tmp, label %bb27, label %bb3
+
+bb3:                                              ; preds = %bb
+  %tmp4 = add i32 %arg2, 3
+  %tmp5 = and i32 %tmp4, -4
+  %tmp6 = add i32 %arg2, -1
+  %tmp7 = insertelement <4 x i32> undef, i32 %tmp6, i32 0
+  %tmp8 = shufflevector <4 x i32> %tmp7, <4 x i32> undef, <4 x i32> zeroinitializer
+  br label %bb9
+
+bb9:                                              ; preds = %bb9, %bb3
+  %tmp10 = phi i32 [ 0, %bb3 ], [ %tmp25, %bb9 ]
+  %tmp11 = insertelement <4 x i32> undef, i32 %tmp10, i32 0
+  %tmp12 = shufflevector <4 x i32> %tmp11, <4 x i32> undef, <4 x i32> zeroinitializer
+  %tmp13 = add <4 x i32> %tmp12, <i32 0, i32 1, i32 2, i32 3>
+  %tmp14 = getelementptr inbounds i32, i32* %arg1, i32 %tmp10
+  %tmp15 = icmp ule <4 x i32> %tmp13, %tmp8
+  %tmp16 = bitcast i32* %tmp14 to <4 x i32>*
+  %tmp17 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp16, i32 4, <4 x i1> %tmp15, <4 x i32> undef)
+  %tmp18 = icmp ne <4 x i32> %tmp17, zeroinitializer
+  %tmp19 = getelementptr inbounds i32, i32* %arg, i32 %tmp10
+  %tmp20 = and <4 x i1> %tmp18, %tmp15
+  %tmp21 = bitcast i32* %tmp19 to <4 x i32>*
+  %tmp22 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp21, i32 4, <4 x i1> %tmp20, <4 x i32> undef)
+  %tmp23 = mul nsw <4 x i32> %tmp22, %tmp17
+  %tmp24 = bitcast i32* %tmp19 to <4 x i32>*
+  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %tmp23, <4 x i32>* %tmp24, i32 4, <4 x i1> %tmp20)
+  %tmp25 = add i32 %tmp10, 4
+  %tmp26 = icmp eq i32 %tmp25, %tmp5
+  br i1 %tmp26, label %bb27, label %bb9
+
+bb27:                                             ; preds = %bb9, %bb
+  ret void
+}
+
+define dso_local arm_aapcs_vfpcc void @range_test(i32* noalias nocapture %arg, i32* noalias nocapture readonly %arg1, i32 %arg2, i32 %arg3) {
+; CHECK-LABEL: range_test:
+; CHECK:       @ %bb.0: @ %bb
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    cmp r3, #0
+; CHECK-NEXT:    it eq
+; CHECK-NEXT:    popeq {r7, pc}
+; CHECK-NEXT:    add.w r12, r3, #3
+; CHECK-NEXT:    mov.w lr, #1
+; CHECK-NEXT:    bic r12, r12, #3
+; CHECK-NEXT:    sub.w r12, r12, #4
+; CHECK-NEXT:    add.w lr, lr, r12, lsr #2
+; CHECK-NEXT:    mov r12, r0
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB5_1: @ %bb12
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vctp.32 r3
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrwt.u32 q0, [r12], #16
+; CHECK-NEXT:    vpttt.i32 ne, q0, zr
+; CHECK-NEXT:    vcmpt.s32 le, q0, r2
+; CHECK-NEXT:    vctpt.32 r3
+; CHECK-NEXT:    vldrwt.u32 q1, [r1], #16
+; CHECK-NEXT:    subs r3, #4
+; CHECK-NEXT:    vmul.i32 q0, q1, q0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrwt.32 q0, [r0]
+; CHECK-NEXT:    mov r0, r12
+; CHECK-NEXT:    le lr, .LBB5_1
+; CHECK-NEXT:  @ %bb.2: @ %bb32
+; CHECK-NEXT:    pop {r7, pc}
+bb:
+  %tmp = icmp eq i32 %arg3, 0
+  br i1 %tmp, label %bb32, label %bb4
+
+bb4:                                              ; preds = %bb
+  %tmp5 = add i32 %arg3, 3
+  %tmp6 = and i32 %tmp5, -4
+  %tmp7 = add i32 %arg3, -1
+  %tmp8 = insertelement <4 x i32> undef, i32 %tmp7, i32 0
+  %tmp9 = shufflevector <4 x i32> %tmp8, <4 x i32> undef, <4 x i32> zeroinitializer
+  %tmp10 = insertelement <4 x i32> undef, i32 %arg2, i32 0
+  %tmp11 = shufflevector <4 x i32> %tmp10, <4 x i32> undef, <4 x i32> zeroinitializer
+  br label %bb12
+
+bb12:                                             ; preds = %bb12, %bb4
+  %tmp13 = phi i32 [ 0, %bb4 ], [ %tmp30, %bb12 ]
+  %tmp14 = insertelement <4 x i32> undef, i32 %tmp13, i32 0
+  %tmp15 = shufflevector <4 x i32> %tmp14, <4 x i32> undef, <4 x i32> zeroinitializer
+  %tmp16 = add <4 x i32> %tmp15, <i32 0, i32 1, i32 2, i32 3>
+  %tmp17 = getelementptr inbounds i32, i32* %arg, i32 %tmp13
+  %tmp18 = icmp ule <4 x i32> %tmp16, %tmp9
+  %tmp19 = bitcast i32* %tmp17 to <4 x i32>*
+  %tmp20 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp19, i32 4, <4 x i1> %tmp18, <4 x i32> undef)
+  %tmp21 = icmp ne <4 x i32> %tmp20, zeroinitializer
+  %tmp22 = icmp sle <4 x i32> %tmp20, %tmp11
+  %tmp23 = getelementptr inbounds i32, i32* %arg1, i32 %tmp13
+  %tmp24 = and <4 x i1> %tmp22, %tmp21
+  %tmp25 = and <4 x i1> %tmp24, %tmp18
+  %tmp26 = bitcast i32* %tmp23 to <4 x i32>*
+  %tmp27 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp26, i32 4, <4 x i1> %tmp25, <4 x i32> undef)
+  %tmp28 = mul nsw <4 x i32> %tmp27, %tmp20
+  %tmp29 = bitcast i32* %tmp17 to <4 x i32>*
+  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %tmp28, <4 x i32>* %tmp29, i32 4, <4 x i1> %tmp25)
+  %tmp30 = add i32 %tmp13, 4
+  %tmp31 = icmp eq i32 %tmp30, %tmp6
+  br i1 %tmp31, label %bb32, label %bb12
+
+bb32:                                             ; preds = %bb12, %bb
+  ret void
+}
+
+; Function Attrs: argmemonly nounwind readonly willreturn
+declare <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>*, i32 immarg, <4 x i1>, <4 x i32>)
+declare void @llvm.masked.store.v4i32.p0v4i32(<4 x i32>, <4 x i32>*, i32, <4 x i1>)
+
+; Function Attrs: nounwind readnone willreturn
+declare i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32>)
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/fast-fp-loops.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/fast-fp-loops.ll
new file mode 100644
index 000000000..e25e0298e
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/fast-fp-loops.ll
@@ -0,0 +1,592 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve.fp,+fp-armv8d16sp,+fp16,+fullfp16 -disable-mve-tail-predication=false %s -o - | FileCheck %s
+
+define arm_aapcs_vfpcc void @fast_float_mul(float* nocapture %a, float* nocapture readonly %b, float* nocapture readonly %c, i32 %N) {
+; CHECK-LABEL: fast_float_mul:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    push {r4, r5, r6, r7, lr}
+; CHECK-NEXT:    cmp r3, #0
+; CHECK-NEXT:    beq.w .LBB0_11
+; CHECK-NEXT:  @ %bb.1: @ %vector.memcheck
+; CHECK-NEXT:    add.w r4, r0, r3, lsl #2
+; CHECK-NEXT:    add.w r5, r2, r3, lsl #2
+; CHECK-NEXT:    cmp r4, r2
+; CHECK-NEXT:    mov.w r12, #1
+; CHECK-NEXT:    cset lr, hi
+; CHECK-NEXT:    cmp r5, r0
+; CHECK-NEXT:    cset r6, hi
+; CHECK-NEXT:    cmp r4, r1
+; CHECK-NEXT:    add.w r5, r1, r3, lsl #2
+; CHECK-NEXT:    cset r4, hi
+; CHECK-NEXT:    cmp r5, r0
+; CHECK-NEXT:    cset r5, hi
+; CHECK-NEXT:    ands r5, r4
+; CHECK-NEXT:    lsls r5, r5, #31
+; CHECK-NEXT:    itt eq
+; CHECK-NEXT:    andeq.w r6, r6, lr
+; CHECK-NEXT:    lslseq.w r6, r6, #31
+; CHECK-NEXT:    beq .LBB0_4
+; CHECK-NEXT:  @ %bb.2: @ %for.body.preheader
+; CHECK-NEXT:    subs r6, r3, #1
+; CHECK-NEXT:    and r7, r3, #3
+; CHECK-NEXT:    cmp r6, #3
+; CHECK-NEXT:    bhs .LBB0_6
+; CHECK-NEXT:  @ %bb.3:
+; CHECK-NEXT:    mov.w r12, #0
+; CHECK-NEXT:    b .LBB0_8
+; CHECK-NEXT:  .LBB0_4: @ %vector.ph
+; CHECK-NEXT:    dlstp.32 lr, r3
+; CHECK-NEXT:  .LBB0_5: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrw.u32 q0, [r1], #16
+; CHECK-NEXT:    vldrw.u32 q1, [r2], #16
+; CHECK-NEXT:    vmul.f32 q0, q1, q0
+; CHECK-NEXT:    vstrw.32 q0, [r0], #16
+; CHECK-NEXT:    letp lr, .LBB0_5
+; CHECK-NEXT:    b .LBB0_11
+; CHECK-NEXT:  .LBB0_6: @ %for.body.preheader.new
+; CHECK-NEXT:    bic r3, r3, #3
+; CHECK-NEXT:    subs r3, #4
+; CHECK-NEXT:    add.w lr, r12, r3, lsr #2
+; CHECK-NEXT:    movs r3, #0
+; CHECK-NEXT:    mov.w r12, #0
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB0_7: @ %for.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    adds r4, r1, r3
+; CHECK-NEXT:    adds r5, r2, r3
+; CHECK-NEXT:    adds r6, r0, r3
+; CHECK-NEXT:    adds r3, #16
+; CHECK-NEXT:    vldr s0, [r4]
+; CHECK-NEXT:    add.w r12, r12, #4
+; CHECK-NEXT:    vldr s2, [r5]
+; CHECK-NEXT:    vmul.f32 s0, s2, s0
+; CHECK-NEXT:    vstr s0, [r6]
+; CHECK-NEXT:    vldr s0, [r4, #4]
+; CHECK-NEXT:    vldr s2, [r5, #4]
+; CHECK-NEXT:    vmul.f32 s0, s2, s0
+; CHECK-NEXT:    vstr s0, [r6, #4]
+; CHECK-NEXT:    vldr s0, [r4, #8]
+; CHECK-NEXT:    vldr s2, [r5, #8]
+; CHECK-NEXT:    vmul.f32 s0, s2, s0
+; CHECK-NEXT:    vstr s0, [r6, #8]
+; CHECK-NEXT:    vldr s0, [r4, #12]
+; CHECK-NEXT:    vldr s2, [r5, #12]
+; CHECK-NEXT:    vmul.f32 s0, s2, s0
+; CHECK-NEXT:    vstr s0, [r6, #12]
+; CHECK-NEXT:    le lr, .LBB0_7
+; CHECK-NEXT:  .LBB0_8: @ %for.cond.cleanup.loopexit.unr-lcssa
+; CHECK-NEXT:    wls lr, r7, .LBB0_11
+; CHECK-NEXT:  @ %bb.9: @ %for.body.epil.preheader
+; CHECK-NEXT:    mvn r3, #3
+; CHECK-NEXT:    mov lr, r7
+; CHECK-NEXT:    add.w r3, r3, r12, lsl #2
+; CHECK-NEXT:    add r1, r3
+; CHECK-NEXT:    add r2, r3
+; CHECK-NEXT:    add r0, r3
+; CHECK-NEXT:  .LBB0_10: @ %for.body.epil
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldr s0, [r1, #4]
+; CHECK-NEXT:    adds r1, #4
+; CHECK-NEXT:    vldr s2, [r2, #4]
+; CHECK-NEXT:    adds r2, #4
+; CHECK-NEXT:    vmul.f32 s0, s2, s0
+; CHECK-NEXT:    vstr s0, [r0, #4]
+; CHECK-NEXT:    adds r0, #4
+; CHECK-NEXT:    le lr, .LBB0_10
+; CHECK-NEXT:  .LBB0_11: @ %for.cond.cleanup
+; CHECK-NEXT:    pop {r4, r5, r6, r7, pc}
+entry:
+  %cmp8 = icmp eq i32 %N, 0
+  br i1 %cmp8, label %for.cond.cleanup, label %vector.memcheck
+
+vector.memcheck:                                  ; preds = %entry
+  %scevgep = getelementptr float, float* %a, i32 %N
+  %scevgep13 = getelementptr float, float* %b, i32 %N
+  %scevgep16 = getelementptr float, float* %c, i32 %N
+  %bound0 = icmp ugt float* %scevgep13, %a
+  %bound1 = icmp ugt float* %scevgep, %b
+  %found.conflict = and i1 %bound0, %bound1
+  %bound018 = icmp ugt float* %scevgep16, %a
+  %bound119 = icmp ugt float* %scevgep, %c
+  %found.conflict20 = and i1 %bound018, %bound119
+  %conflict.rdx = or i1 %found.conflict, %found.conflict20
+  br i1 %conflict.rdx, label %for.body.preheader, label %vector.ph
+
+for.body.preheader:                               ; preds = %vector.memcheck
+  %0 = add i32 %N, -1
+  %xtraiter = and i32 %N, 3
+  %1 = icmp ult i32 %0, 3
+  br i1 %1, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body.preheader.new
+
+for.body.preheader.new:                           ; preds = %for.body.preheader
+  %unroll_iter = sub i32 %N, %xtraiter
+  br label %for.body
+
+vector.ph:                                        ; preds = %vector.memcheck
+  %n.rnd.up = add i32 %N, 3
+  %n.vec = and i32 %n.rnd.up, -4
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert21 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat22 = shufflevector <4 x i32> %broadcast.splatinsert21, <4 x i32> undef, <4 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+  %2 = getelementptr inbounds float, float* %b, i32 %index
+  %3 = icmp ule <4 x i32> %induction, %broadcast.splat22
+  %4 = bitcast float* %2 to <4 x float>*
+  %wide.masked.load = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* %4, i32 4, <4 x i1> %3, <4 x float> undef)
+  %5 = getelementptr inbounds float, float* %c, i32 %index
+  %6 = bitcast float* %5 to <4 x float>*
+  %wide.masked.load23 = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* %6, i32 4, <4 x i1> %3, <4 x float> undef)
+  %7 = fmul fast <4 x float> %wide.masked.load23, %wide.masked.load
+  %8 = getelementptr inbounds float, float* %a, i32 %index
+  %9 = bitcast float* %8 to <4 x float>*
+  call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %7, <4 x float>* %9, i32 4, <4 x i1> %3)
+  %index.next = add i32 %index, 4
+  %10 = icmp eq i32 %index.next, %n.vec
+  br i1 %10, label %for.cond.cleanup, label %vector.body
+
+for.cond.cleanup.loopexit.unr-lcssa:              ; preds = %for.body, %for.body.preheader
+  %i.09.unr = phi i32 [ 0, %for.body.preheader ], [ %inc.3, %for.body ]
+  %lcmp.mod = icmp eq i32 %xtraiter, 0
+  br i1 %lcmp.mod, label %for.cond.cleanup, label %for.body.epil
+
+for.body.epil:                                    ; preds = %for.cond.cleanup.loopexit.unr-lcssa, %for.body.epil
+  %i.09.epil = phi i32 [ %inc.epil, %for.body.epil ], [ %i.09.unr, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %epil.iter = phi i32 [ %epil.iter.sub, %for.body.epil ], [ %xtraiter, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %arrayidx.epil = getelementptr inbounds float, float* %b, i32 %i.09.epil
+  %11 = load float, float* %arrayidx.epil, align 4
+  %arrayidx1.epil = getelementptr inbounds float, float* %c, i32 %i.09.epil
+  %12 = load float, float* %arrayidx1.epil, align 4
+  %mul.epil = fmul fast float %12, %11
+  %arrayidx2.epil = getelementptr inbounds float, float* %a, i32 %i.09.epil
+  store float %mul.epil, float* %arrayidx2.epil, align 4
+  %inc.epil = add nuw i32 %i.09.epil, 1
+  %epil.iter.sub = add i32 %epil.iter, -1
+  %epil.iter.cmp = icmp eq i32 %epil.iter.sub, 0
+  br i1 %epil.iter.cmp, label %for.cond.cleanup, label %for.body.epil
+
+for.cond.cleanup:                                 ; preds = %vector.body, %for.cond.cleanup.loopexit.unr-lcssa, %for.body.epil, %entry
+  ret void
+
+for.body:                                         ; preds = %for.body, %for.body.preheader.new
+  %i.09 = phi i32 [ 0, %for.body.preheader.new ], [ %inc.3, %for.body ]
+  %niter = phi i32 [ %unroll_iter, %for.body.preheader.new ], [ %niter.nsub.3, %for.body ]
+  %arrayidx = getelementptr inbounds float, float* %b, i32 %i.09
+  %13 = load float, float* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds float, float* %c, i32 %i.09
+  %14 = load float, float* %arrayidx1, align 4
+  %mul = fmul fast float %14, %13
+  %arrayidx2 = getelementptr inbounds float, float* %a, i32 %i.09
+  store float %mul, float* %arrayidx2, align 4
+  %inc = or i32 %i.09, 1
+  %arrayidx.1 = getelementptr inbounds float, float* %b, i32 %inc
+  %15 = load float, float* %arrayidx.1, align 4
+  %arrayidx1.1 = getelementptr inbounds float, float* %c, i32 %inc
+  %16 = load float, float* %arrayidx1.1, align 4
+  %mul.1 = fmul fast float %16, %15
+  %arrayidx2.1 = getelementptr inbounds float, float* %a, i32 %inc
+  store float %mul.1, float* %arrayidx2.1, align 4
+  %inc.1 = or i32 %i.09, 2
+  %arrayidx.2 = getelementptr inbounds float, float* %b, i32 %inc.1
+  %17 = load float, float* %arrayidx.2, align 4
+  %arrayidx1.2 = getelementptr inbounds float, float* %c, i32 %inc.1
+  %18 = load float, float* %arrayidx1.2, align 4
+  %mul.2 = fmul fast float %18, %17
+  %arrayidx2.2 = getelementptr inbounds float, float* %a, i32 %inc.1
+  store float %mul.2, float* %arrayidx2.2, align 4
+  %inc.2 = or i32 %i.09, 3
+  %arrayidx.3 = getelementptr inbounds float, float* %b, i32 %inc.2
+  %19 = load float, float* %arrayidx.3, align 4
+  %arrayidx1.3 = getelementptr inbounds float, float* %c, i32 %inc.2
+  %20 = load float, float* %arrayidx1.3, align 4
+  %mul.3 = fmul fast float %20, %19
+  %arrayidx2.3 = getelementptr inbounds float, float* %a, i32 %inc.2
+  store float %mul.3, float* %arrayidx2.3, align 4
+  %inc.3 = add nuw i32 %i.09, 4
+  %niter.nsub.3 = add i32 %niter, -4
+  %niter.ncmp.3 = icmp eq i32 %niter.nsub.3, 0
+  br i1 %niter.ncmp.3, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body
+}
+
+define arm_aapcs_vfpcc float @fast_float_mac(float* nocapture readonly %b, float* nocapture readonly %c, i32 %N) {
+; CHECK-LABEL: fast_float_mac:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    cbz r2, .LBB1_4
+; CHECK-NEXT:  @ %bb.1: @ %vector.ph
+; CHECK-NEXT:    adds r3, r2, #3
+; CHECK-NEXT:    vmov.i32 q0, #0x0
+; CHECK-NEXT:    bic r3, r3, #3
+; CHECK-NEXT:    sub.w r12, r3, #4
+; CHECK-NEXT:    movs r3, #1
+; CHECK-NEXT:    add.w lr, r3, r12, lsr #2
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB1_2: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vctp.32 r2
+; CHECK-NEXT:    mov r3, r2
+; CHECK-NEXT:    subs r2, #4
+; CHECK-NEXT:    vpstt
+; CHECK-NEXT:    vldrwt.u32 q2, [r0], #16
+; CHECK-NEXT:    vldrwt.u32 q3, [r1], #16
+; CHECK-NEXT:    vmov q1, q0
+; CHECK-NEXT:    vfma.f32 q0, q3, q2
+; CHECK-NEXT:    le lr, .LBB1_2
+; CHECK-NEXT:  @ %bb.3: @ %middle.block
+; CHECK-NEXT:    vctp.32 r3
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    vmov.f32 s4, s2
+; CHECK-NEXT:    vmov.f32 s5, s3
+; CHECK-NEXT:    vadd.f32 q0, q0, q1
+; CHECK-NEXT:    vmov.32 r0, q0[1]
+; CHECK-NEXT:    vdup.32 q1, r0
+; CHECK-NEXT:    vadd.f32 q0, q0, q1
+; CHECK-NEXT:    @ kill: def $s0 killed $s0 killed $q0
+; CHECK-NEXT:    pop {r7, pc}
+; CHECK-NEXT:  .LBB1_4:
+; CHECK-NEXT:    vldr s0, .LCPI1_0
+; CHECK-NEXT:    @ kill: def $s0 killed $s0 killed $q0
+; CHECK-NEXT:    pop {r7, pc}
+; CHECK-NEXT:    .p2align 2
+; CHECK-NEXT:  @ %bb.5:
+; CHECK-NEXT:  .LCPI1_0:
+; CHECK-NEXT:    .long 0 @ float 0
+entry:
+  %cmp8 = icmp eq i32 %N, 0
+  br i1 %cmp8, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %n.rnd.up = add i32 %N, 3
+  %n.vec = and i32 %n.rnd.up, -4
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert11 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat12 = shufflevector <4 x i32> %broadcast.splatinsert11, <4 x i32> undef, <4 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %vec.phi = phi <4 x float> [ zeroinitializer, %vector.ph ], [ %6, %vector.body ]
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+  %0 = getelementptr inbounds float, float* %b, i32 %index
+  %1 = icmp ule <4 x i32> %induction, %broadcast.splat12
+  %2 = bitcast float* %0 to <4 x float>*
+  %wide.masked.load = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* %2, i32 4, <4 x i1> %1, <4 x float> undef)
+  %3 = getelementptr inbounds float, float* %c, i32 %index
+  %4 = bitcast float* %3 to <4 x float>*
+  %wide.masked.load13 = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* %4, i32 4, <4 x i1> %1, <4 x float> undef)
+  %5 = fmul fast <4 x float> %wide.masked.load13, %wide.masked.load
+  %6 = fadd fast <4 x float> %5, %vec.phi
+  %index.next = add i32 %index, 4
+  %7 = icmp eq i32 %index.next, %n.vec
+  br i1 %7, label %middle.block, label %vector.body
+
+middle.block:                                     ; preds = %vector.body
+  %8 = select <4 x i1> %1, <4 x float> %6, <4 x float> %vec.phi
+  %rdx.shuf = shufflevector <4 x float> %8, <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+  %bin.rdx = fadd fast <4 x float> %8, %rdx.shuf
+  %rdx.shuf14 = shufflevector <4 x float> %bin.rdx, <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+  %bin.rdx15 = fadd fast <4 x float> %bin.rdx, %rdx.shuf14
+  %9 = extractelement <4 x float> %bin.rdx15, i32 0
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %middle.block, %entry
+  %a.0.lcssa = phi float [ 0.000000e+00, %entry ], [ %9, %middle.block ]
+  ret float %a.0.lcssa
+}
+
+define arm_aapcs_vfpcc float @fast_float_half_mac(half* nocapture readonly %b, half* nocapture readonly %c, i32 %N) {
+; CHECK-LABEL: fast_float_half_mac:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    push {r4, r5, r7, lr}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    sub sp, #32
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    beq.w .LBB2_22
+; CHECK-NEXT:  @ %bb.1: @ %vector.ph
+; CHECK-NEXT:    adds r3, r2, #3
+; CHECK-NEXT:    vmov.i32 q5, #0x0
+; CHECK-NEXT:    bic r3, r3, #3
+; CHECK-NEXT:    sub.w r12, r3, #4
+; CHECK-NEXT:    movs r3, #1
+; CHECK-NEXT:    add.w lr, r3, r12, lsr #2
+; CHECK-NEXT:    sub.w r12, r2, #1
+; CHECK-NEXT:    adr r2, .LCPI2_1
+; CHECK-NEXT:    movs r3, #0
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vdup.32 q1, r12
+; CHECK-NEXT:    vdup.32 q2, r12
+; CHECK-NEXT:    vstrw.32 q0, [sp] @ 16-byte Spill
+; CHECK-NEXT:    b .LBB2_4
+; CHECK-NEXT:  .LBB2_2: @ %cond.load24
+; CHECK-NEXT:    @ in Loop: Header=BB2_4 Depth=1
+; CHECK-NEXT:    vmovx.f16 s0, s28
+; CHECK-NEXT:    vmov r4, s28
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vmov.16 q6[0], r4
+; CHECK-NEXT:    vldr.16 s0, [r1, #6]
+; CHECK-NEXT:    vmov.16 q6[1], r2
+; CHECK-NEXT:    vmov r2, s29
+; CHECK-NEXT:    vmov.16 q6[2], r2
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vmov.16 q6[3], r2
+; CHECK-NEXT:  .LBB2_3: @ %else25
+; CHECK-NEXT:    @ in Loop: Header=BB2_4 Depth=1
+; CHECK-NEXT:    vmul.f16 q5, q6, q5
+; CHECK-NEXT:    sub.w lr, lr, #1
+; CHECK-NEXT:    vmovx.f16 s2, s21
+; CHECK-NEXT:    vmovx.f16 s0, s20
+; CHECK-NEXT:    vcvtb.f32.f16 s27, s2
+; CHECK-NEXT:    adds r0, #8
+; CHECK-NEXT:    vcvtb.f32.f16 s26, s21
+; CHECK-NEXT:    adds r1, #8
+; CHECK-NEXT:    vcvtb.f32.f16 s25, s0
+; CHECK-NEXT:    adds r3, #4
+; CHECK-NEXT:    vcvtb.f32.f16 s24, s20
+; CHECK-NEXT:    vadd.f32 q5, q3, q6
+; CHECK-NEXT:    cmp.w lr, #0
+; CHECK-NEXT:    bne .LBB2_4
+; CHECK-NEXT:    b .LBB2_21
+; CHECK-NEXT:  .LBB2_4: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrw.u32 q0, [sp] @ 16-byte Reload
+; CHECK-NEXT:    vmov q3, q5
+; CHECK-NEXT:    @ implicit-def: $q6
+; CHECK-NEXT:    vadd.i32 q4, q0, r3
+; CHECK-NEXT:    vcmp.u32 cs, q1, q4
+; CHECK-NEXT:    vmrs r4, p0
+; CHECK-NEXT:    and r2, r4, #1
+; CHECK-NEXT:    rsbs r5, r2, #0
+; CHECK-NEXT:    movs r2, #0
+; CHECK-NEXT:    bfi r2, r5, #0, #1
+; CHECK-NEXT:    ubfx r5, r4, #4, #1
+; CHECK-NEXT:    rsbs r5, r5, #0
+; CHECK-NEXT:    bfi r2, r5, #1, #1
+; CHECK-NEXT:    ubfx r5, r4, #8, #1
+; CHECK-NEXT:    ubfx r4, r4, #12, #1
+; CHECK-NEXT:    rsbs r5, r5, #0
+; CHECK-NEXT:    bfi r2, r5, #2, #1
+; CHECK-NEXT:    rsbs r4, r4, #0
+; CHECK-NEXT:    bfi r2, r4, #3, #1
+; CHECK-NEXT:    lsls r4, r2, #31
+; CHECK-NEXT:    bne .LBB2_9
+; CHECK-NEXT:  @ %bb.5: @ %else
+; CHECK-NEXT:    @ in Loop: Header=BB2_4 Depth=1
+; CHECK-NEXT:    lsls r4, r2, #30
+; CHECK-NEXT:    bpl .LBB2_10
+; CHECK-NEXT:  .LBB2_6: @ %cond.load5
+; CHECK-NEXT:    @ in Loop: Header=BB2_4 Depth=1
+; CHECK-NEXT:    vldr.16 s20, [r0, #2]
+; CHECK-NEXT:    vmov r5, s24
+; CHECK-NEXT:    vmovx.f16 s24, s25
+; CHECK-NEXT:    vmov r4, s20
+; CHECK-NEXT:    vmov.16 q5[0], r5
+; CHECK-NEXT:    vmov.16 q5[1], r4
+; CHECK-NEXT:    vmov r4, s25
+; CHECK-NEXT:    vmov.16 q5[2], r4
+; CHECK-NEXT:    vmov r4, s24
+; CHECK-NEXT:    vmov.16 q5[3], r4
+; CHECK-NEXT:    lsls r4, r2, #29
+; CHECK-NEXT:    bmi .LBB2_11
+; CHECK-NEXT:  .LBB2_7: @ in Loop: Header=BB2_4 Depth=1
+; CHECK-NEXT:    vmov q6, q5
+; CHECK-NEXT:    lsls r2, r2, #28
+; CHECK-NEXT:    bmi .LBB2_12
+; CHECK-NEXT:  .LBB2_8: @ in Loop: Header=BB2_4 Depth=1
+; CHECK-NEXT:    vmov q5, q6
+; CHECK-NEXT:    b .LBB2_13
+; CHECK-NEXT:  .LBB2_9: @ %cond.load
+; CHECK-NEXT:    @ in Loop: Header=BB2_4 Depth=1
+; CHECK-NEXT:    vldr.16 s24, [r0]
+; CHECK-NEXT:    lsls r4, r2, #30
+; CHECK-NEXT:    bmi .LBB2_6
+; CHECK-NEXT:  .LBB2_10: @ in Loop: Header=BB2_4 Depth=1
+; CHECK-NEXT:    vmov q5, q6
+; CHECK-NEXT:    lsls r4, r2, #29
+; CHECK-NEXT:    bpl .LBB2_7
+; CHECK-NEXT:  .LBB2_11: @ %cond.load8
+; CHECK-NEXT:    @ in Loop: Header=BB2_4 Depth=1
+; CHECK-NEXT:    vmovx.f16 s24, s20
+; CHECK-NEXT:    vmov r4, s20
+; CHECK-NEXT:    vldr.16 s28, [r0, #4]
+; CHECK-NEXT:    vmov r5, s24
+; CHECK-NEXT:    vmov.16 q6[0], r4
+; CHECK-NEXT:    vmovx.f16 s20, s21
+; CHECK-NEXT:    vmov.16 q6[1], r5
+; CHECK-NEXT:    vmov r4, s28
+; CHECK-NEXT:    vmov.16 q6[2], r4
+; CHECK-NEXT:    vmov r4, s20
+; CHECK-NEXT:    vmov.16 q6[3], r4
+; CHECK-NEXT:    lsls r2, r2, #28
+; CHECK-NEXT:    bpl .LBB2_8
+; CHECK-NEXT:  .LBB2_12: @ %cond.load11
+; CHECK-NEXT:    @ in Loop: Header=BB2_4 Depth=1
+; CHECK-NEXT:    vmovx.f16 s20, s24
+; CHECK-NEXT:    vmov r4, s24
+; CHECK-NEXT:    vmov r2, s20
+; CHECK-NEXT:    vmov.16 q5[0], r4
+; CHECK-NEXT:    vmov.16 q5[1], r2
+; CHECK-NEXT:    vmov r2, s25
+; CHECK-NEXT:    vldr.16 s24, [r0, #6]
+; CHECK-NEXT:    vmov.16 q5[2], r2
+; CHECK-NEXT:    vmov r2, s24
+; CHECK-NEXT:    vmov.16 q5[3], r2
+; CHECK-NEXT:  .LBB2_13: @ %else12
+; CHECK-NEXT:    @ in Loop: Header=BB2_4 Depth=1
+; CHECK-NEXT:    vcmp.u32 cs, q2, q4
+; CHECK-NEXT:    @ implicit-def: $q7
+; CHECK-NEXT:    vmrs r4, p0
+; CHECK-NEXT:    and r2, r4, #1
+; CHECK-NEXT:    rsbs r5, r2, #0
+; CHECK-NEXT:    movs r2, #0
+; CHECK-NEXT:    bfi r2, r5, #0, #1
+; CHECK-NEXT:    ubfx r5, r4, #4, #1
+; CHECK-NEXT:    rsbs r5, r5, #0
+; CHECK-NEXT:    bfi r2, r5, #1, #1
+; CHECK-NEXT:    ubfx r5, r4, #8, #1
+; CHECK-NEXT:    ubfx r4, r4, #12, #1
+; CHECK-NEXT:    rsbs r5, r5, #0
+; CHECK-NEXT:    bfi r2, r5, #2, #1
+; CHECK-NEXT:    rsbs r4, r4, #0
+; CHECK-NEXT:    bfi r2, r4, #3, #1
+; CHECK-NEXT:    lsls r4, r2, #31
+; CHECK-NEXT:    bne .LBB2_17
+; CHECK-NEXT:  @ %bb.14: @ %else16
+; CHECK-NEXT:    @ in Loop: Header=BB2_4 Depth=1
+; CHECK-NEXT:    lsls r4, r2, #30
+; CHECK-NEXT:    bpl .LBB2_18
+; CHECK-NEXT:  .LBB2_15: @ %cond.load18
+; CHECK-NEXT:    @ in Loop: Header=BB2_4 Depth=1
+; CHECK-NEXT:    vldr.16 s24, [r1, #2]
+; CHECK-NEXT:    vmov r5, s28
+; CHECK-NEXT:    vmovx.f16 s28, s29
+; CHECK-NEXT:    vmov r4, s24
+; CHECK-NEXT:    vmov.16 q6[0], r5
+; CHECK-NEXT:    vmov.16 q6[1], r4
+; CHECK-NEXT:    vmov r4, s29
+; CHECK-NEXT:    vmov.16 q6[2], r4
+; CHECK-NEXT:    vmov r4, s28
+; CHECK-NEXT:    vmov.16 q6[3], r4
+; CHECK-NEXT:    lsls r4, r2, #29
+; CHECK-NEXT:    bmi .LBB2_19
+; CHECK-NEXT:  .LBB2_16: @ in Loop: Header=BB2_4 Depth=1
+; CHECK-NEXT:    vmov q7, q6
+; CHECK-NEXT:    lsls r2, r2, #28
+; CHECK-NEXT:    bmi.w .LBB2_2
+; CHECK-NEXT:    b .LBB2_20
+; CHECK-NEXT:  .LBB2_17: @ %cond.load15
+; CHECK-NEXT:    @ in Loop: Header=BB2_4 Depth=1
+; CHECK-NEXT:    vldr.16 s28, [r1]
+; CHECK-NEXT:    lsls r4, r2, #30
+; CHECK-NEXT:    bmi .LBB2_15
+; CHECK-NEXT:  .LBB2_18: @ in Loop: Header=BB2_4 Depth=1
+; CHECK-NEXT:    vmov q6, q7
+; CHECK-NEXT:    lsls r4, r2, #29
+; CHECK-NEXT:    bpl .LBB2_16
+; CHECK-NEXT:  .LBB2_19: @ %cond.load21
+; CHECK-NEXT:    @ in Loop: Header=BB2_4 Depth=1
+; CHECK-NEXT:    vmovx.f16 s28, s24
+; CHECK-NEXT:    vmov r4, s24
+; CHECK-NEXT:    vldr.16 s0, [r1, #4]
+; CHECK-NEXT:    vmov r5, s28
+; CHECK-NEXT:    vmov.16 q7[0], r4
+; CHECK-NEXT:    vmov r4, s0
+; CHECK-NEXT:    vmov.16 q7[1], r5
+; CHECK-NEXT:    vmovx.f16 s0, s25
+; CHECK-NEXT:    vmov.16 q7[2], r4
+; CHECK-NEXT:    vmov r4, s0
+; CHECK-NEXT:    vmov.16 q7[3], r4
+; CHECK-NEXT:    lsls r2, r2, #28
+; CHECK-NEXT:    bmi.w .LBB2_2
+; CHECK-NEXT:  .LBB2_20: @ in Loop: Header=BB2_4 Depth=1
+; CHECK-NEXT:    vmov q6, q7
+; CHECK-NEXT:    b .LBB2_3
+; CHECK-NEXT:  .LBB2_21: @ %middle.block
+; CHECK-NEXT:    vdup.32 q0, r12
+; CHECK-NEXT:    vcmp.u32 cs, q0, q4
+; CHECK-NEXT:    vpsel q0, q5, q3
+; CHECK-NEXT:    vmov.f32 s4, s2
+; CHECK-NEXT:    vmov.f32 s5, s3
+; CHECK-NEXT:    vadd.f32 q0, q0, q1
+; CHECK-NEXT:    vmov.32 r0, q0[1]
+; CHECK-NEXT:    vdup.32 q1, r0
+; CHECK-NEXT:    vadd.f32 q0, q0, q1
+; CHECK-NEXT:    b .LBB2_23
+; CHECK-NEXT:  .LBB2_22:
+; CHECK-NEXT:    vldr s0, .LCPI2_0
+; CHECK-NEXT:  .LBB2_23: @ %for.cond.cleanup
+; CHECK-NEXT:    @ kill: def $s0 killed $s0 killed $q0
+; CHECK-NEXT:    add sp, #32
+; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    pop {r4, r5, r7, pc}
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  @ %bb.24:
+; CHECK-NEXT:  .LCPI2_1:
+; CHECK-NEXT:    .long 0 @ 0x0
+; CHECK-NEXT:    .long 1 @ 0x1
+; CHECK-NEXT:    .long 2 @ 0x2
+; CHECK-NEXT:    .long 3 @ 0x3
+; CHECK-NEXT:  .LCPI2_0:
+; CHECK-NEXT:    .long 0 @ float 0
+entry:
+  %cmp8 = icmp eq i32 %N, 0
+  br i1 %cmp8, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %n.rnd.up = add i32 %N, 3
+  %n.vec = and i32 %n.rnd.up, -4
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert11 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat12 = shufflevector <4 x i32> %broadcast.splatinsert11, <4 x i32> undef, <4 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %vec.phi = phi <4 x float> [ zeroinitializer, %vector.ph ], [ %7, %vector.body ]
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+  %0 = getelementptr inbounds half, half* %b, i32 %index
+  %1 = icmp ule <4 x i32> %induction, %broadcast.splat12
+  %2 = bitcast half* %0 to <4 x half>*
+  %wide.masked.load = call <4 x half> @llvm.masked.load.v4f16.p0v4f16(<4 x half>* %2, i32 2, <4 x i1> %1, <4 x half> undef)
+  %3 = getelementptr inbounds half, half* %c, i32 %index
+  %4 = bitcast half* %3 to <4 x half>*
+  %wide.masked.load13 = call <4 x half> @llvm.masked.load.v4f16.p0v4f16(<4 x half>* %4, i32 2, <4 x i1> %1, <4 x half> undef)
+  %5 = fmul fast <4 x half> %wide.masked.load13, %wide.masked.load
+  %6 = fpext <4 x half> %5 to <4 x float>
+  %7 = fadd fast <4 x float> %vec.phi, %6
+  %index.next = add i32 %index, 4
+  %8 = icmp eq i32 %index.next, %n.vec
+  br i1 %8, label %middle.block, label %vector.body
+
+middle.block:                                     ; preds = %vector.body
+  %9 = select <4 x i1> %1, <4 x float> %7, <4 x float> %vec.phi
+  %rdx.shuf = shufflevector <4 x float> %9, <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+  %bin.rdx = fadd fast <4 x float> %9, %rdx.shuf
+  %rdx.shuf14 = shufflevector <4 x float> %bin.rdx, <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+  %bin.rdx15 = fadd fast <4 x float> %bin.rdx, %rdx.shuf14
+  %10 = extractelement <4 x float> %bin.rdx15, i32 0
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %middle.block, %entry
+  %a.0.lcssa = phi float [ 0.000000e+00, %entry ], [ %10, %middle.block ]
+  ret float %a.0.lcssa
+}
+
+; Function Attrs: argmemonly nounwind readonly willreturn
+declare <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>*, i32 immarg, <4 x i1>, <4 x float>)
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.masked.store.v4f32.p0v4f32(<4 x float>, <4 x float>*, i32 immarg, <4 x i1>)
+
+; Function Attrs: argmemonly nounwind readonly willreturn
+declare <4 x half> @llvm.masked.load.v4f16.p0v4f16(<4 x half>*, i32 immarg, <4 x i1>, <4 x half>)
+
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mve-float-loops.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mve-float-loops.ll
new file mode 100644
index 000000000..b15219179
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mve-float-loops.ll
@@ -0,0 +1,2008 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve.fp,+fp-armv8d16sp,+fp16,+fullfp16 %s -o - | FileCheck %s
+
+define arm_aapcs_vfpcc void @float_float_mul(float* nocapture readonly %a, float* nocapture readonly %b, float* nocapture %c, i32 %N) {
+; CHECK-LABEL: float_float_mul:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; CHECK-NEXT:    cmp r3, #0
+; CHECK-NEXT:    beq.w .LBB0_10
+; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
+; CHECK-NEXT:    cmp r3, #3
+; CHECK-NEXT:    bhi .LBB0_3
+; CHECK-NEXT:  @ %bb.2:
+; CHECK-NEXT:    mov.w r12, #0
+; CHECK-NEXT:    b .LBB0_4
+; CHECK-NEXT:  .LBB0_3: @ %vector.memcheck
+; CHECK-NEXT:    add.w r5, r2, r3, lsl #2
+; CHECK-NEXT:    add.w r6, r1, r3, lsl #2
+; CHECK-NEXT:    cmp r5, r1
+; CHECK-NEXT:    add.w r4, r0, r3, lsl #2
+; CHECK-NEXT:    cset r7, hi
+; CHECK-NEXT:    cmp r6, r2
+; CHECK-NEXT:    cset r6, hi
+; CHECK-NEXT:    cmp r5, r0
+; CHECK-NEXT:    cset r5, hi
+; CHECK-NEXT:    cmp r4, r2
+; CHECK-NEXT:    cset r4, hi
+; CHECK-NEXT:    mov.w r12, #0
+; CHECK-NEXT:    ands r5, r4
+; CHECK-NEXT:    lsls r5, r5, #31
+; CHECK-NEXT:    itt eq
+; CHECK-NEXT:    andeq r7, r6
+; CHECK-NEXT:    lslseq.w r7, r7, #31
+; CHECK-NEXT:    beq .LBB0_11
+; CHECK-NEXT:  .LBB0_4: @ %for.body.preheader22
+; CHECK-NEXT:    mvn.w r7, r12
+; CHECK-NEXT:    adds r5, r7, r3
+; CHECK-NEXT:    and lr, r3, #3
+; CHECK-NEXT:    wls lr, lr, .LBB0_7
+; CHECK-NEXT:  @ %bb.5: @ %for.body.prol.preheader
+; CHECK-NEXT:    mvn r4, #3
+; CHECK-NEXT:    add.w r7, r4, r12, lsl #2
+; CHECK-NEXT:    adds r4, r0, r7
+; CHECK-NEXT:    adds r6, r1, r7
+; CHECK-NEXT:    add r7, r2
+; CHECK-NEXT:  .LBB0_6: @ %for.body.prol
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldr s0, [r6, #4]
+; CHECK-NEXT:    adds r6, #4
+; CHECK-NEXT:    vldr s2, [r4, #4]
+; CHECK-NEXT:    adds r4, #4
+; CHECK-NEXT:    add.w r12, r12, #1
+; CHECK-NEXT:    vmul.f32 s0, s2, s0
+; CHECK-NEXT:    vstr s0, [r7, #4]
+; CHECK-NEXT:    adds r7, #4
+; CHECK-NEXT:    le lr, .LBB0_6
+; CHECK-NEXT:  .LBB0_7: @ %for.body.prol.loopexit
+; CHECK-NEXT:    cmp r5, #3
+; CHECK-NEXT:    blo .LBB0_10
+; CHECK-NEXT:  @ %bb.8: @ %for.body.preheader1
+; CHECK-NEXT:    sub.w lr, r3, r12
+; CHECK-NEXT:    sub.w r8, r0, #8
+; CHECK-NEXT:    sub.w r10, r1, #8
+; CHECK-NEXT:    sub.w r5, r2, #8
+; CHECK-NEXT:    subs r0, #4
+; CHECK-NEXT:    subs r1, #4
+; CHECK-NEXT:    subs r2, #4
+; CHECK-NEXT:    lsl.w r12, r12, #2
+; CHECK-NEXT:  .LBB0_9: @ %for.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    add.w r4, r0, r12
+; CHECK-NEXT:    add.w r6, r1, r12
+; CHECK-NEXT:    add.w r9, r2, r12
+; CHECK-NEXT:    add.w r7, r8, r12
+; CHECK-NEXT:    vldr s0, [r6, #4]
+; CHECK-NEXT:    add.w r3, r10, r12
+; CHECK-NEXT:    vldr s2, [r4, #4]
+; CHECK-NEXT:    add.w r11, r5, r12
+; CHECK-NEXT:    add.w r8, r8, #16
+; CHECK-NEXT:    add.w r10, r10, #16
+; CHECK-NEXT:    vmul.f32 s0, s2, s0
+; CHECK-NEXT:    adds r5, #16
+; CHECK-NEXT:    adds r0, #16
+; CHECK-NEXT:    adds r1, #16
+; CHECK-NEXT:    adds r2, #16
+; CHECK-NEXT:    subs.w lr, lr, #4
+; CHECK-NEXT:    vstr s0, [r9, #4]
+; CHECK-NEXT:    vldr s0, [r3, #12]
+; CHECK-NEXT:    vldr s2, [r7, #12]
+; CHECK-NEXT:    vmul.f32 s0, s2, s0
+; CHECK-NEXT:    vstr s0, [r11, #12]
+; CHECK-NEXT:    vldr s0, [r3, #16]
+; CHECK-NEXT:    vldr s2, [r7, #16]
+; CHECK-NEXT:    vmul.f32 s0, s2, s0
+; CHECK-NEXT:    vstr s0, [r11, #16]
+; CHECK-NEXT:    vldr s0, [r6, #16]
+; CHECK-NEXT:    vldr s2, [r4, #16]
+; CHECK-NEXT:    vmul.f32 s0, s2, s0
+; CHECK-NEXT:    vstr s0, [r9, #16]
+; CHECK-NEXT:    bne .LBB0_9
+; CHECK-NEXT:  .LBB0_10: @ %for.cond.cleanup
+; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
+; CHECK-NEXT:  .LBB0_11: @ %vector.ph
+; CHECK-NEXT:    bic r12, r3, #3
+; CHECK-NEXT:    movs r6, #1
+; CHECK-NEXT:    sub.w r7, r12, #4
+; CHECK-NEXT:    sub.w r4, r0, #16
+; CHECK-NEXT:    sub.w r5, r1, #16
+; CHECK-NEXT:    add.w lr, r6, r7, lsr #2
+; CHECK-NEXT:    sub.w r6, r2, #16
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB0_12: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrw.u32 q0, [r5, #16]!
+; CHECK-NEXT:    vldrw.u32 q1, [r4, #16]!
+; CHECK-NEXT:    vmul.f32 q0, q1, q0
+; CHECK-NEXT:    vstrb.8 q0, [r6, #16]!
+; CHECK-NEXT:    le lr, .LBB0_12
+; CHECK-NEXT:  @ %bb.13: @ %middle.block
+; CHECK-NEXT:    cmp r12, r3
+; CHECK-NEXT:    bne.w .LBB0_4
+; CHECK-NEXT:    b .LBB0_10
+entry:
+  %cmp8 = icmp eq i32 %N, 0
+  br i1 %cmp8, label %for.cond.cleanup, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  %min.iters.check = icmp ult i32 %N, 4
+  br i1 %min.iters.check, label %for.body.preheader22, label %vector.memcheck
+
+for.body.preheader22:                             ; preds = %middle.block, %vector.memcheck, %for.body.preheader
+  %i.09.ph = phi i32 [ 0, %vector.memcheck ], [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
+  %0 = xor i32 %i.09.ph, -1
+  %1 = add i32 %0, %N
+  %xtraiter = and i32 %N, 3
+  %lcmp.mod = icmp eq i32 %xtraiter, 0
+  br i1 %lcmp.mod, label %for.body.prol.loopexit, label %for.body.prol
+
+for.body.prol:                                    ; preds = %for.body.preheader22, %for.body.prol
+  %i.09.prol = phi i32 [ %inc.prol, %for.body.prol ], [ %i.09.ph, %for.body.preheader22 ]
+  %prol.iter = phi i32 [ %prol.iter.sub, %for.body.prol ], [ %xtraiter, %for.body.preheader22 ]
+  %arrayidx.prol = getelementptr inbounds float, float* %a, i32 %i.09.prol
+  %2 = load float, float* %arrayidx.prol, align 4
+  %arrayidx1.prol = getelementptr inbounds float, float* %b, i32 %i.09.prol
+  %3 = load float, float* %arrayidx1.prol, align 4
+  %mul.prol = fmul float %2, %3
+  %arrayidx2.prol = getelementptr inbounds float, float* %c, i32 %i.09.prol
+  store float %mul.prol, float* %arrayidx2.prol, align 4
+  %inc.prol = add nuw i32 %i.09.prol, 1
+  %prol.iter.sub = add i32 %prol.iter, -1
+  %prol.iter.cmp = icmp eq i32 %prol.iter.sub, 0
+  br i1 %prol.iter.cmp, label %for.body.prol.loopexit, label %for.body.prol
+
+for.body.prol.loopexit:                           ; preds = %for.body.prol, %for.body.preheader22
+  %i.09.unr = phi i32 [ %i.09.ph, %for.body.preheader22 ], [ %inc.prol, %for.body.prol ]
+  %4 = icmp ult i32 %1, 3
+  br i1 %4, label %for.cond.cleanup, label %for.body
+
+vector.memcheck:                                  ; preds = %for.body.preheader
+  %scevgep = getelementptr float, float* %c, i32 %N
+  %scevgep13 = getelementptr float, float* %a, i32 %N
+  %scevgep16 = getelementptr float, float* %b, i32 %N
+  %bound0 = icmp ugt float* %scevgep13, %c
+  %bound1 = icmp ugt float* %scevgep, %a
+  %found.conflict = and i1 %bound0, %bound1
+  %bound018 = icmp ugt float* %scevgep16, %c
+  %bound119 = icmp ugt float* %scevgep, %b
+  %found.conflict20 = and i1 %bound018, %bound119
+  %conflict.rdx = or i1 %found.conflict, %found.conflict20
+  br i1 %conflict.rdx, label %for.body.preheader22, label %vector.ph
+
+vector.ph:                                        ; preds = %vector.memcheck
+  %n.vec = and i32 %N, -4
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %5 = getelementptr inbounds float, float* %a, i32 %index
+  %6 = bitcast float* %5 to <4 x float>*
+  %wide.load = load <4 x float>, <4 x float>* %6, align 4
+  %7 = getelementptr inbounds float, float* %b, i32 %index
+  %8 = bitcast float* %7 to <4 x float>*
+  %wide.load21 = load <4 x float>, <4 x float>* %8, align 4
+  %9 = fmul <4 x float> %wide.load, %wide.load21
+  %10 = getelementptr inbounds float, float* %c, i32 %index
+  %11 = bitcast float* %10 to <4 x float>*
+  store <4 x float> %9, <4 x float>* %11, align 4
+  %index.next = add i32 %index, 4
+  %12 = icmp eq i32 %index.next, %n.vec
+  br i1 %12, label %middle.block, label %vector.body
+
+middle.block:                                     ; preds = %vector.body
+  %cmp.n = icmp eq i32 %n.vec, %N
+  br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader22
+
+for.cond.cleanup:                                 ; preds = %for.body.prol.loopexit, %for.body, %middle.block, %entry
+  ret void
+
+for.body:                                         ; preds = %for.body.prol.loopexit, %for.body
+  %i.09 = phi i32 [ %inc.3, %for.body ], [ %i.09.unr, %for.body.prol.loopexit ]
+  %arrayidx = getelementptr inbounds float, float* %a, i32 %i.09
+  %13 = load float, float* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds float, float* %b, i32 %i.09
+  %14 = load float, float* %arrayidx1, align 4
+  %mul = fmul float %13, %14
+  %arrayidx2 = getelementptr inbounds float, float* %c, i32 %i.09
+  store float %mul, float* %arrayidx2, align 4
+  %inc = add nuw i32 %i.09, 1
+  %arrayidx.1 = getelementptr inbounds float, float* %a, i32 %inc
+  %15 = load float, float* %arrayidx.1, align 4
+  %arrayidx1.1 = getelementptr inbounds float, float* %b, i32 %inc
+  %16 = load float, float* %arrayidx1.1, align 4
+  %mul.1 = fmul float %15, %16
+  %arrayidx2.1 = getelementptr inbounds float, float* %c, i32 %inc
+  store float %mul.1, float* %arrayidx2.1, align 4
+  %inc.1 = add nuw i32 %i.09, 2
+  %arrayidx.2 = getelementptr inbounds float, float* %a, i32 %inc.1
+  %17 = load float, float* %arrayidx.2, align 4
+  %arrayidx1.2 = getelementptr inbounds float, float* %b, i32 %inc.1
+  %18 = load float, float* %arrayidx1.2, align 4
+  %mul.2 = fmul float %17, %18
+  %arrayidx2.2 = getelementptr inbounds float, float* %c, i32 %inc.1
+  store float %mul.2, float* %arrayidx2.2, align 4
+  %inc.2 = add nuw i32 %i.09, 3
+  %arrayidx.3 = getelementptr inbounds float, float* %a, i32 %inc.2
+  %19 = load float, float* %arrayidx.3, align 4
+  %arrayidx1.3 = getelementptr inbounds float, float* %b, i32 %inc.2
+  %20 = load float, float* %arrayidx1.3, align 4
+  %mul.3 = fmul float %19, %20
+  %arrayidx2.3 = getelementptr inbounds float, float* %c, i32 %inc.2
+  store float %mul.3, float* %arrayidx2.3, align 4
+  %inc.3 = add nuw i32 %i.09, 4
+  %exitcond.3 = icmp eq i32 %inc.3, %N
+  br i1 %exitcond.3, label %for.cond.cleanup, label %for.body
+}
+
+define arm_aapcs_vfpcc void @float_float_add(float* nocapture readonly %a, float* nocapture readonly %b, float* nocapture %c, i32 %N) {
+; CHECK-LABEL: float_float_add:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; CHECK-NEXT:    cmp r3, #0
+; CHECK-NEXT:    beq.w .LBB1_10
+; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
+; CHECK-NEXT:    cmp r3, #3
+; CHECK-NEXT:    bhi .LBB1_3
+; CHECK-NEXT:  @ %bb.2:
+; CHECK-NEXT:    mov.w r12, #0
+; CHECK-NEXT:    b .LBB1_4
+; CHECK-NEXT:  .LBB1_3: @ %vector.memcheck
+; CHECK-NEXT:    add.w r5, r2, r3, lsl #2
+; CHECK-NEXT:    add.w r6, r1, r3, lsl #2
+; CHECK-NEXT:    cmp r5, r1
+; CHECK-NEXT:    add.w r4, r0, r3, lsl #2
+; CHECK-NEXT:    cset r7, hi
+; CHECK-NEXT:    cmp r6, r2
+; CHECK-NEXT:    cset r6, hi
+; CHECK-NEXT:    cmp r5, r0
+; CHECK-NEXT:    cset r5, hi
+; CHECK-NEXT:    cmp r4, r2
+; CHECK-NEXT:    cset r4, hi
+; CHECK-NEXT:    mov.w r12, #0
+; CHECK-NEXT:    ands r5, r4
+; CHECK-NEXT:    lsls r5, r5, #31
+; CHECK-NEXT:    itt eq
+; CHECK-NEXT:    andeq r7, r6
+; CHECK-NEXT:    lslseq.w r7, r7, #31
+; CHECK-NEXT:    beq .LBB1_11
+; CHECK-NEXT:  .LBB1_4: @ %for.body.preheader22
+; CHECK-NEXT:    mvn.w r7, r12
+; CHECK-NEXT:    adds r5, r7, r3
+; CHECK-NEXT:    and lr, r3, #3
+; CHECK-NEXT:    wls lr, lr, .LBB1_7
+; CHECK-NEXT:  @ %bb.5: @ %for.body.prol.preheader
+; CHECK-NEXT:    mvn r4, #3
+; CHECK-NEXT:    add.w r7, r4, r12, lsl #2
+; CHECK-NEXT:    adds r4, r0, r7
+; CHECK-NEXT:    adds r6, r1, r7
+; CHECK-NEXT:    add r7, r2
+; CHECK-NEXT:  .LBB1_6: @ %for.body.prol
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldr s0, [r6, #4]
+; CHECK-NEXT:    adds r6, #4
+; CHECK-NEXT:    vldr s2, [r4, #4]
+; CHECK-NEXT:    adds r4, #4
+; CHECK-NEXT:    add.w r12, r12, #1
+; CHECK-NEXT:    vadd.f32 s0, s2, s0
+; CHECK-NEXT:    vstr s0, [r7, #4]
+; CHECK-NEXT:    adds r7, #4
+; CHECK-NEXT:    le lr, .LBB1_6
+; CHECK-NEXT:  .LBB1_7: @ %for.body.prol.loopexit
+; CHECK-NEXT:    cmp r5, #3
+; CHECK-NEXT:    blo .LBB1_10
+; CHECK-NEXT:  @ %bb.8: @ %for.body.preheader1
+; CHECK-NEXT:    sub.w lr, r3, r12
+; CHECK-NEXT:    sub.w r8, r0, #8
+; CHECK-NEXT:    sub.w r10, r1, #8
+; CHECK-NEXT:    sub.w r5, r2, #8
+; CHECK-NEXT:    subs r0, #4
+; CHECK-NEXT:    subs r1, #4
+; CHECK-NEXT:    subs r2, #4
+; CHECK-NEXT:    lsl.w r12, r12, #2
+; CHECK-NEXT:  .LBB1_9: @ %for.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    add.w r4, r0, r12
+; CHECK-NEXT:    add.w r6, r1, r12
+; CHECK-NEXT:    add.w r9, r2, r12
+; CHECK-NEXT:    add.w r7, r8, r12
+; CHECK-NEXT:    vldr s0, [r6, #4]
+; CHECK-NEXT:    add.w r3, r10, r12
+; CHECK-NEXT:    vldr s2, [r4, #4]
+; CHECK-NEXT:    add.w r11, r5, r12
+; CHECK-NEXT:    add.w r8, r8, #16
+; CHECK-NEXT:    add.w r10, r10, #16
+; CHECK-NEXT:    vadd.f32 s0, s2, s0
+; CHECK-NEXT:    adds r5, #16
+; CHECK-NEXT:    adds r0, #16
+; CHECK-NEXT:    adds r1, #16
+; CHECK-NEXT:    adds r2, #16
+; CHECK-NEXT:    subs.w lr, lr, #4
+; CHECK-NEXT:    vstr s0, [r9, #4]
+; CHECK-NEXT:    vldr s0, [r3, #12]
+; CHECK-NEXT:    vldr s2, [r7, #12]
+; CHECK-NEXT:    vadd.f32 s0, s2, s0
+; CHECK-NEXT:    vstr s0, [r11, #12]
+; CHECK-NEXT:    vldr s0, [r3, #16]
+; CHECK-NEXT:    vldr s2, [r7, #16]
+; CHECK-NEXT:    vadd.f32 s0, s2, s0
+; CHECK-NEXT:    vstr s0, [r11, #16]
+; CHECK-NEXT:    vldr s0, [r6, #16]
+; CHECK-NEXT:    vldr s2, [r4, #16]
+; CHECK-NEXT:    vadd.f32 s0, s2, s0
+; CHECK-NEXT:    vstr s0, [r9, #16]
+; CHECK-NEXT:    bne .LBB1_9
+; CHECK-NEXT:  .LBB1_10: @ %for.cond.cleanup
+; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
+; CHECK-NEXT:  .LBB1_11: @ %vector.ph
+; CHECK-NEXT:    bic r12, r3, #3
+; CHECK-NEXT:    movs r6, #1
+; CHECK-NEXT:    sub.w r7, r12, #4
+; CHECK-NEXT:    sub.w r4, r0, #16
+; CHECK-NEXT:    sub.w r5, r1, #16
+; CHECK-NEXT:    add.w lr, r6, r7, lsr #2
+; CHECK-NEXT:    sub.w r6, r2, #16
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB1_12: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrw.u32 q0, [r5, #16]!
+; CHECK-NEXT:    vldrw.u32 q1, [r4, #16]!
+; CHECK-NEXT:    vadd.f32 q0, q1, q0
+; CHECK-NEXT:    vstrb.8 q0, [r6, #16]!
+; CHECK-NEXT:    le lr, .LBB1_12
+; CHECK-NEXT:  @ %bb.13: @ %middle.block
+; CHECK-NEXT:    cmp r12, r3
+; CHECK-NEXT:    bne.w .LBB1_4
+; CHECK-NEXT:    b .LBB1_10
+entry:
+  %cmp8 = icmp eq i32 %N, 0
+  br i1 %cmp8, label %for.cond.cleanup, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  %min.iters.check = icmp ult i32 %N, 4
+  br i1 %min.iters.check, label %for.body.preheader22, label %vector.memcheck
+
+for.body.preheader22:                             ; preds = %middle.block, %vector.memcheck, %for.body.preheader
+  %i.09.ph = phi i32 [ 0, %vector.memcheck ], [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
+  %0 = xor i32 %i.09.ph, -1
+  %1 = add i32 %0, %N
+  %xtraiter = and i32 %N, 3
+  %lcmp.mod = icmp eq i32 %xtraiter, 0
+  br i1 %lcmp.mod, label %for.body.prol.loopexit, label %for.body.prol
+
+for.body.prol:                                    ; preds = %for.body.preheader22, %for.body.prol
+  %i.09.prol = phi i32 [ %inc.prol, %for.body.prol ], [ %i.09.ph, %for.body.preheader22 ]
+  %prol.iter = phi i32 [ %prol.iter.sub, %for.body.prol ], [ %xtraiter, %for.body.preheader22 ]
+  %arrayidx.prol = getelementptr inbounds float, float* %a, i32 %i.09.prol
+  %2 = load float, float* %arrayidx.prol, align 4
+  %arrayidx1.prol = getelementptr inbounds float, float* %b, i32 %i.09.prol
+  %3 = load float, float* %arrayidx1.prol, align 4
+  %add.prol = fadd float %2, %3
+  %arrayidx2.prol = getelementptr inbounds float, float* %c, i32 %i.09.prol
+  store float %add.prol, float* %arrayidx2.prol, align 4
+  %inc.prol = add nuw i32 %i.09.prol, 1
+  %prol.iter.sub = add i32 %prol.iter, -1
+  %prol.iter.cmp = icmp eq i32 %prol.iter.sub, 0
+  br i1 %prol.iter.cmp, label %for.body.prol.loopexit, label %for.body.prol
+
+for.body.prol.loopexit:                           ; preds = %for.body.prol, %for.body.preheader22
+  %i.09.unr = phi i32 [ %i.09.ph, %for.body.preheader22 ], [ %inc.prol, %for.body.prol ]
+  %4 = icmp ult i32 %1, 3
+  br i1 %4, label %for.cond.cleanup, label %for.body
+
+vector.memcheck:                                  ; preds = %for.body.preheader
+  %scevgep = getelementptr float, float* %c, i32 %N
+  %scevgep13 = getelementptr float, float* %a, i32 %N
+  %scevgep16 = getelementptr float, float* %b, i32 %N
+  %bound0 = icmp ugt float* %scevgep13, %c
+  %bound1 = icmp ugt float* %scevgep, %a
+  %found.conflict = and i1 %bound0, %bound1
+  %bound018 = icmp ugt float* %scevgep16, %c
+  %bound119 = icmp ugt float* %scevgep, %b
+  %found.conflict20 = and i1 %bound018, %bound119
+  %conflict.rdx = or i1 %found.conflict, %found.conflict20
+  br i1 %conflict.rdx, label %for.body.preheader22, label %vector.ph
+
+vector.ph:                                        ; preds = %vector.memcheck
+  %n.vec = and i32 %N, -4
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %5 = getelementptr inbounds float, float* %a, i32 %index
+  %6 = bitcast float* %5 to <4 x float>*
+  %wide.load = load <4 x float>, <4 x float>* %6, align 4
+  %7 = getelementptr inbounds float, float* %b, i32 %index
+  %8 = bitcast float* %7 to <4 x float>*
+  %wide.load21 = load <4 x float>, <4 x float>* %8, align 4
+  %9 = fadd <4 x float> %wide.load, %wide.load21
+  %10 = getelementptr inbounds float, float* %c, i32 %index
+  %11 = bitcast float* %10 to <4 x float>*
+  store <4 x float> %9, <4 x float>* %11, align 4
+  %index.next = add i32 %index, 4
+  %12 = icmp eq i32 %index.next, %n.vec
+  br i1 %12, label %middle.block, label %vector.body
+
+middle.block:                                     ; preds = %vector.body
+  %cmp.n = icmp eq i32 %n.vec, %N
+  br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader22
+
+for.cond.cleanup:                                 ; preds = %for.body.prol.loopexit, %for.body, %middle.block, %entry
+  ret void
+
+for.body:                                         ; preds = %for.body.prol.loopexit, %for.body
+  %i.09 = phi i32 [ %inc.3, %for.body ], [ %i.09.unr, %for.body.prol.loopexit ]
+  %arrayidx = getelementptr inbounds float, float* %a, i32 %i.09
+  %13 = load float, float* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds float, float* %b, i32 %i.09
+  %14 = load float, float* %arrayidx1, align 4
+  %add = fadd float %13, %14
+  %arrayidx2 = getelementptr inbounds float, float* %c, i32 %i.09
+  store float %add, float* %arrayidx2, align 4
+  %inc = add nuw i32 %i.09, 1
+  %arrayidx.1 = getelementptr inbounds float, float* %a, i32 %inc
+  %15 = load float, float* %arrayidx.1, align 4
+  %arrayidx1.1 = getelementptr inbounds float, float* %b, i32 %inc
+  %16 = load float, float* %arrayidx1.1, align 4
+  %add.1 = fadd float %15, %16
+  %arrayidx2.1 = getelementptr inbounds float, float* %c, i32 %inc
+  store float %add.1, float* %arrayidx2.1, align 4
+  %inc.1 = add nuw i32 %i.09, 2
+  %arrayidx.2 = getelementptr inbounds float, float* %a, i32 %inc.1
+  %17 = load float, float* %arrayidx.2, align 4
+  %arrayidx1.2 = getelementptr inbounds float, float* %b, i32 %inc.1
+  %18 = load float, float* %arrayidx1.2, align 4
+  %add.2 = fadd float %17, %18
+  %arrayidx2.2 = getelementptr inbounds float, float* %c, i32 %inc.1
+  store float %add.2, float* %arrayidx2.2, align 4
+  %inc.2 = add nuw i32 %i.09, 3
+  %arrayidx.3 = getelementptr inbounds float, float* %a, i32 %inc.2
+  %19 = load float, float* %arrayidx.3, align 4
+  %arrayidx1.3 = getelementptr inbounds float, float* %b, i32 %inc.2
+  %20 = load float, float* %arrayidx1.3, align 4
+  %add.3 = fadd float %19, %20
+  %arrayidx2.3 = getelementptr inbounds float, float* %c, i32 %inc.2
+  store float %add.3, float* %arrayidx2.3, align 4
+  %inc.3 = add nuw i32 %i.09, 4
+  %exitcond.3 = icmp eq i32 %inc.3, %N
+  br i1 %exitcond.3, label %for.cond.cleanup, label %for.body
+}
+
+define arm_aapcs_vfpcc void @float_float_sub(float* nocapture readonly %a, float* nocapture readonly %b, float* nocapture %c, i32 %N) {
+; CHECK-LABEL: float_float_sub:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; CHECK-NEXT:    cmp r3, #0
+; CHECK-NEXT:    beq.w .LBB2_10
+; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
+; CHECK-NEXT:    cmp r3, #3
+; CHECK-NEXT:    bhi .LBB2_3
+; CHECK-NEXT:  @ %bb.2:
+; CHECK-NEXT:    mov.w r12, #0
+; CHECK-NEXT:    b .LBB2_4
+; CHECK-NEXT:  .LBB2_3: @ %vector.memcheck
+; CHECK-NEXT:    add.w r5, r2, r3, lsl #2
+; CHECK-NEXT:    add.w r6, r1, r3, lsl #2
+; CHECK-NEXT:    cmp r5, r1
+; CHECK-NEXT:    add.w r4, r0, r3, lsl #2
+; CHECK-NEXT:    cset r7, hi
+; CHECK-NEXT:    cmp r6, r2
+; CHECK-NEXT:    cset r6, hi
+; CHECK-NEXT:    cmp r5, r0
+; CHECK-NEXT:    cset r5, hi
+; CHECK-NEXT:    cmp r4, r2
+; CHECK-NEXT:    cset r4, hi
+; CHECK-NEXT:    mov.w r12, #0
+; CHECK-NEXT:    ands r5, r4
+; CHECK-NEXT:    lsls r5, r5, #31
+; CHECK-NEXT:    itt eq
+; CHECK-NEXT:    andeq r7, r6
+; CHECK-NEXT:    lslseq.w r7, r7, #31
+; CHECK-NEXT:    beq .LBB2_11
+; CHECK-NEXT:  .LBB2_4: @ %for.body.preheader22
+; CHECK-NEXT:    mvn.w r7, r12
+; CHECK-NEXT:    adds r5, r7, r3
+; CHECK-NEXT:    and lr, r3, #3
+; CHECK-NEXT:    wls lr, lr, .LBB2_7
+; CHECK-NEXT:  @ %bb.5: @ %for.body.prol.preheader
+; CHECK-NEXT:    mvn r4, #3
+; CHECK-NEXT:    add.w r7, r4, r12, lsl #2
+; CHECK-NEXT:    adds r4, r0, r7
+; CHECK-NEXT:    adds r6, r1, r7
+; CHECK-NEXT:    add r7, r2
+; CHECK-NEXT:  .LBB2_6: @ %for.body.prol
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldr s0, [r6, #4]
+; CHECK-NEXT:    adds r6, #4
+; CHECK-NEXT:    vldr s2, [r4, #4]
+; CHECK-NEXT:    adds r4, #4
+; CHECK-NEXT:    add.w r12, r12, #1
+; CHECK-NEXT:    vsub.f32 s0, s2, s0
+; CHECK-NEXT:    vstr s0, [r7, #4]
+; CHECK-NEXT:    adds r7, #4
+; CHECK-NEXT:    le lr, .LBB2_6
+; CHECK-NEXT:  .LBB2_7: @ %for.body.prol.loopexit
+; CHECK-NEXT:    cmp r5, #3
+; CHECK-NEXT:    blo .LBB2_10
+; CHECK-NEXT:  @ %bb.8: @ %for.body.preheader1
+; CHECK-NEXT:    sub.w lr, r3, r12
+; CHECK-NEXT:    sub.w r8, r0, #8
+; CHECK-NEXT:    sub.w r10, r1, #8
+; CHECK-NEXT:    sub.w r5, r2, #8
+; CHECK-NEXT:    subs r0, #4
+; CHECK-NEXT:    subs r1, #4
+; CHECK-NEXT:    subs r2, #4
+; CHECK-NEXT:    lsl.w r12, r12, #2
+; CHECK-NEXT:  .LBB2_9: @ %for.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    add.w r4, r0, r12
+; CHECK-NEXT:    add.w r6, r1, r12
+; CHECK-NEXT:    add.w r9, r2, r12
+; CHECK-NEXT:    add.w r7, r8, r12
+; CHECK-NEXT:    vldr s0, [r6, #4]
+; CHECK-NEXT:    add.w r3, r10, r12
+; CHECK-NEXT:    vldr s2, [r4, #4]
+; CHECK-NEXT:    add.w r11, r5, r12
+; CHECK-NEXT:    add.w r8, r8, #16
+; CHECK-NEXT:    add.w r10, r10, #16
+; CHECK-NEXT:    vsub.f32 s0, s2, s0
+; CHECK-NEXT:    adds r5, #16
+; CHECK-NEXT:    adds r0, #16
+; CHECK-NEXT:    adds r1, #16
+; CHECK-NEXT:    adds r2, #16
+; CHECK-NEXT:    subs.w lr, lr, #4
+; CHECK-NEXT:    vstr s0, [r9, #4]
+; CHECK-NEXT:    vldr s0, [r3, #12]
+; CHECK-NEXT:    vldr s2, [r7, #12]
+; CHECK-NEXT:    vsub.f32 s0, s2, s0
+; CHECK-NEXT:    vstr s0, [r11, #12]
+; CHECK-NEXT:    vldr s0, [r3, #16]
+; CHECK-NEXT:    vldr s2, [r7, #16]
+; CHECK-NEXT:    vsub.f32 s0, s2, s0
+; CHECK-NEXT:    vstr s0, [r11, #16]
+; CHECK-NEXT:    vldr s0, [r6, #16]
+; CHECK-NEXT:    vldr s2, [r4, #16]
+; CHECK-NEXT:    vsub.f32 s0, s2, s0
+; CHECK-NEXT:    vstr s0, [r9, #16]
+; CHECK-NEXT:    bne .LBB2_9
+; CHECK-NEXT:  .LBB2_10: @ %for.cond.cleanup
+; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
+; CHECK-NEXT:  .LBB2_11: @ %vector.ph
+; CHECK-NEXT:    bic r12, r3, #3
+; CHECK-NEXT:    movs r6, #1
+; CHECK-NEXT:    sub.w r7, r12, #4
+; CHECK-NEXT:    sub.w r4, r0, #16
+; CHECK-NEXT:    sub.w r5, r1, #16
+; CHECK-NEXT:    add.w lr, r6, r7, lsr #2
+; CHECK-NEXT:    sub.w r6, r2, #16
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB2_12: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrw.u32 q0, [r5, #16]!
+; CHECK-NEXT:    vldrw.u32 q1, [r4, #16]!
+; CHECK-NEXT:    vsub.f32 q0, q1, q0
+; CHECK-NEXT:    vstrb.8 q0, [r6, #16]!
+; CHECK-NEXT:    le lr, .LBB2_12
+; CHECK-NEXT:  @ %bb.13: @ %middle.block
+; CHECK-NEXT:    cmp r12, r3
+; CHECK-NEXT:    bne.w .LBB2_4
+; CHECK-NEXT:    b .LBB2_10
+entry:
+  %cmp8 = icmp eq i32 %N, 0
+  br i1 %cmp8, label %for.cond.cleanup, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  %min.iters.check = icmp ult i32 %N, 4
+  br i1 %min.iters.check, label %for.body.preheader22, label %vector.memcheck
+
+for.body.preheader22:                             ; preds = %middle.block, %vector.memcheck, %for.body.preheader
+  %i.09.ph = phi i32 [ 0, %vector.memcheck ], [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
+  %0 = xor i32 %i.09.ph, -1
+  %1 = add i32 %0, %N
+  %xtraiter = and i32 %N, 3
+  %lcmp.mod = icmp eq i32 %xtraiter, 0
+  br i1 %lcmp.mod, label %for.body.prol.loopexit, label %for.body.prol
+
+for.body.prol:                                    ; preds = %for.body.preheader22, %for.body.prol
+  %i.09.prol = phi i32 [ %inc.prol, %for.body.prol ], [ %i.09.ph, %for.body.preheader22 ]
+  %prol.iter = phi i32 [ %prol.iter.sub, %for.body.prol ], [ %xtraiter, %for.body.preheader22 ]
+  %arrayidx.prol = getelementptr inbounds float, float* %a, i32 %i.09.prol
+  %2 = load float, float* %arrayidx.prol, align 4
+  %arrayidx1.prol = getelementptr inbounds float, float* %b, i32 %i.09.prol
+  %3 = load float, float* %arrayidx1.prol, align 4
+  %sub.prol = fsub float %2, %3
+  %arrayidx2.prol = getelementptr inbounds float, float* %c, i32 %i.09.prol
+  store float %sub.prol, float* %arrayidx2.prol, align 4
+  %inc.prol = add nuw i32 %i.09.prol, 1
+  %prol.iter.sub = add i32 %prol.iter, -1
+  %prol.iter.cmp = icmp eq i32 %prol.iter.sub, 0
+  br i1 %prol.iter.cmp, label %for.body.prol.loopexit, label %for.body.prol
+
+for.body.prol.loopexit:                           ; preds = %for.body.prol, %for.body.preheader22
+  %i.09.unr = phi i32 [ %i.09.ph, %for.body.preheader22 ], [ %inc.prol, %for.body.prol ]
+  %4 = icmp ult i32 %1, 3
+  br i1 %4, label %for.cond.cleanup, label %for.body
+
+vector.memcheck:                                  ; preds = %for.body.preheader
+  %scevgep = getelementptr float, float* %c, i32 %N
+  %scevgep13 = getelementptr float, float* %a, i32 %N
+  %scevgep16 = getelementptr float, float* %b, i32 %N
+  %bound0 = icmp ugt float* %scevgep13, %c
+  %bound1 = icmp ugt float* %scevgep, %a
+  %found.conflict = and i1 %bound0, %bound1
+  %bound018 = icmp ugt float* %scevgep16, %c
+  %bound119 = icmp ugt float* %scevgep, %b
+  %found.conflict20 = and i1 %bound018, %bound119
+  %conflict.rdx = or i1 %found.conflict, %found.conflict20
+  br i1 %conflict.rdx, label %for.body.preheader22, label %vector.ph
+
+vector.ph:                                        ; preds = %vector.memcheck
+  %n.vec = and i32 %N, -4
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %5 = getelementptr inbounds float, float* %a, i32 %index
+  %6 = bitcast float* %5 to <4 x float>*
+  %wide.load = load <4 x float>, <4 x float>* %6, align 4
+  %7 = getelementptr inbounds float, float* %b, i32 %index
+  %8 = bitcast float* %7 to <4 x float>*
+  %wide.load21 = load <4 x float>, <4 x float>* %8, align 4
+  %9 = fsub <4 x float> %wide.load, %wide.load21
+  %10 = getelementptr inbounds float, float* %c, i32 %index
+  %11 = bitcast float* %10 to <4 x float>*
+  store <4 x float> %9, <4 x float>* %11, align 4
+  %index.next = add i32 %index, 4
+  %12 = icmp eq i32 %index.next, %n.vec
+  br i1 %12, label %middle.block, label %vector.body
+
+middle.block:                                     ; preds = %vector.body
+  %cmp.n = icmp eq i32 %n.vec, %N
+  br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader22
+
+for.cond.cleanup:                                 ; preds = %for.body.prol.loopexit, %for.body, %middle.block, %entry
+  ret void
+
+for.body:                                         ; preds = %for.body.prol.loopexit, %for.body
+  %i.09 = phi i32 [ %inc.3, %for.body ], [ %i.09.unr, %for.body.prol.loopexit ]
+  %arrayidx = getelementptr inbounds float, float* %a, i32 %i.09
+  %13 = load float, float* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds float, float* %b, i32 %i.09
+  %14 = load float, float* %arrayidx1, align 4
+  %sub = fsub float %13, %14
+  %arrayidx2 = getelementptr inbounds float, float* %c, i32 %i.09
+  store float %sub, float* %arrayidx2, align 4
+  %inc = add nuw i32 %i.09, 1
+  %arrayidx.1 = getelementptr inbounds float, float* %a, i32 %inc
+  %15 = load float, float* %arrayidx.1, align 4
+  %arrayidx1.1 = getelementptr inbounds float, float* %b, i32 %inc
+  %16 = load float, float* %arrayidx1.1, align 4
+  %sub.1 = fsub float %15, %16
+  %arrayidx2.1 = getelementptr inbounds float, float* %c, i32 %inc
+  store float %sub.1, float* %arrayidx2.1, align 4
+  %inc.1 = add nuw i32 %i.09, 2
+  %arrayidx.2 = getelementptr inbounds float, float* %a, i32 %inc.1
+  %17 = load float, float* %arrayidx.2, align 4
+  %arrayidx1.2 = getelementptr inbounds float, float* %b, i32 %inc.1
+  %18 = load float, float* %arrayidx1.2, align 4
+  %sub.2 = fsub float %17, %18
+  %arrayidx2.2 = getelementptr inbounds float, float* %c, i32 %inc.1
+  store float %sub.2, float* %arrayidx2.2, align 4
+  %inc.2 = add nuw i32 %i.09, 3
+  %arrayidx.3 = getelementptr inbounds float, float* %a, i32 %inc.2
+  %19 = load float, float* %arrayidx.3, align 4
+  %arrayidx1.3 = getelementptr inbounds float, float* %b, i32 %inc.2
+  %20 = load float, float* %arrayidx1.3, align 4
+  %sub.3 = fsub float %19, %20
+  %arrayidx2.3 = getelementptr inbounds float, float* %c, i32 %inc.2
+  store float %sub.3, float* %arrayidx2.3, align 4
+  %inc.3 = add nuw i32 %i.09, 4
+  %exitcond.3 = icmp eq i32 %inc.3, %N
+  br i1 %exitcond.3, label %for.cond.cleanup, label %for.body
+}
+
+define arm_aapcs_vfpcc void @float_int_mul(float* nocapture readonly %a, i32* nocapture readonly %b, float* nocapture %c, i32 %N) {
+; CHECK-LABEL: float_int_mul:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; CHECK-NEXT:    cmp r3, #0
+; CHECK-NEXT:    beq.w .LBB3_13
+; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
+; CHECK-NEXT:    cmp r3, #3
+; CHECK-NEXT:    bls .LBB3_6
+; CHECK-NEXT:  @ %bb.2: @ %vector.memcheck
+; CHECK-NEXT:    add.w r7, r0, r3, lsl #2
+; CHECK-NEXT:    cmp r7, r2
+; CHECK-NEXT:    itt hi
+; CHECK-NEXT:    addhi.w r7, r2, r3, lsl #2
+; CHECK-NEXT:    cmphi r7, r0
+; CHECK-NEXT:    bhi .LBB3_6
+; CHECK-NEXT:  @ %bb.3: @ %vector.ph
+; CHECK-NEXT:    bic r12, r3, #3
+; CHECK-NEXT:    movs r6, #1
+; CHECK-NEXT:    sub.w r7, r12, #4
+; CHECK-NEXT:    sub.w r4, r0, #16
+; CHECK-NEXT:    sub.w r5, r1, #16
+; CHECK-NEXT:    add.w lr, r6, r7, lsr #2
+; CHECK-NEXT:    sub.w r6, r2, #16
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB3_4: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrw.u32 q0, [r5, #16]!
+; CHECK-NEXT:    vldrw.u32 q1, [r4, #16]!
+; CHECK-NEXT:    vcvt.f32.s32 q0, q0
+; CHECK-NEXT:    vmul.f32 q0, q1, q0
+; CHECK-NEXT:    vstrb.8 q0, [r6, #16]!
+; CHECK-NEXT:    le lr, .LBB3_4
+; CHECK-NEXT:  @ %bb.5: @ %middle.block
+; CHECK-NEXT:    cmp r12, r3
+; CHECK-NEXT:    bne .LBB3_7
+; CHECK-NEXT:    b .LBB3_13
+; CHECK-NEXT:  .LBB3_6:
+; CHECK-NEXT:    mov.w r12, #0
+; CHECK-NEXT:  .LBB3_7: @ %for.body.preheader16
+; CHECK-NEXT:    mvn.w r6, r12
+; CHECK-NEXT:    adds r5, r6, r3
+; CHECK-NEXT:    and lr, r3, #3
+; CHECK-NEXT:    wls lr, lr, .LBB3_10
+; CHECK-NEXT:  @ %bb.8: @ %for.body.prol.preheader
+; CHECK-NEXT:    mvn r4, #3
+; CHECK-NEXT:    add.w r6, r4, r12, lsl #2
+; CHECK-NEXT:    adds r4, r0, r6
+; CHECK-NEXT:    add r6, r2
+; CHECK-NEXT:  .LBB3_9: @ %for.body.prol
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    add.w r7, r1, r12, lsl #2
+; CHECK-NEXT:    add.w r12, r12, #1
+; CHECK-NEXT:    vldr s0, [r7]
+; CHECK-NEXT:    vcvt.f32.s32 s0, s0
+; CHECK-NEXT:    vldr s2, [r4, #4]
+; CHECK-NEXT:    adds r4, #4
+; CHECK-NEXT:    vmul.f32 s0, s2, s0
+; CHECK-NEXT:    vstr s0, [r6, #4]
+; CHECK-NEXT:    adds r6, #4
+; CHECK-NEXT:    le lr, .LBB3_9
+; CHECK-NEXT:  .LBB3_10: @ %for.body.prol.loopexit
+; CHECK-NEXT:    cmp r5, #3
+; CHECK-NEXT:    blo .LBB3_13
+; CHECK-NEXT:  @ %bb.11: @ %for.body.preheader1
+; CHECK-NEXT:    sub.w lr, r3, r12
+; CHECK-NEXT:    sub.w r10, r0, #8
+; CHECK-NEXT:    sub.w r4, r1, #8
+; CHECK-NEXT:    sub.w r5, r2, #8
+; CHECK-NEXT:    subs r0, #4
+; CHECK-NEXT:    subs r1, #4
+; CHECK-NEXT:    subs r2, #4
+; CHECK-NEXT:    lsl.w r12, r12, #2
+; CHECK-NEXT:  .LBB3_12: @ %for.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    add.w r3, r1, r12
+; CHECK-NEXT:    add.w r11, r0, r12
+; CHECK-NEXT:    add.w r9, r2, r12
+; CHECK-NEXT:    add.w r6, r4, r12
+; CHECK-NEXT:    vldr s0, [r3, #4]
+; CHECK-NEXT:    add.w r7, r10, r12
+; CHECK-NEXT:    add.w r8, r5, r12
+; CHECK-NEXT:    add.w r10, r10, #16
+; CHECK-NEXT:    vcvt.f32.s32 s0, s0
+; CHECK-NEXT:    vldr s2, [r11, #4]
+; CHECK-NEXT:    adds r4, #16
+; CHECK-NEXT:    adds r5, #16
+; CHECK-NEXT:    adds r0, #16
+; CHECK-NEXT:    adds r1, #16
+; CHECK-NEXT:    adds r2, #16
+; CHECK-NEXT:    subs.w lr, lr, #4
+; CHECK-NEXT:    vmul.f32 s0, s2, s0
+; CHECK-NEXT:    vstr s0, [r9, #4]
+; CHECK-NEXT:    vldr s0, [r6, #12]
+; CHECK-NEXT:    vcvt.f32.s32 s0, s0
+; CHECK-NEXT:    vldr s2, [r7, #12]
+; CHECK-NEXT:    vmul.f32 s0, s2, s0
+; CHECK-NEXT:    vstr s0, [r8, #12]
+; CHECK-NEXT:    vldr s2, [r6, #16]
+; CHECK-NEXT:    vldr s0, [r7, #16]
+; CHECK-NEXT:    vcvt.f32.s32 s2, s2
+; CHECK-NEXT:    vmul.f32 s0, s0, s2
+; CHECK-NEXT:    vstr s0, [r8, #16]
+; CHECK-NEXT:    vldr s0, [r3, #16]
+; CHECK-NEXT:    vcvt.f32.s32 s0, s0
+; CHECK-NEXT:    vldr s2, [r11, #16]
+; CHECK-NEXT:    vmul.f32 s0, s2, s0
+; CHECK-NEXT:    vstr s0, [r9, #16]
+; CHECK-NEXT:    bne .LBB3_12
+; CHECK-NEXT:  .LBB3_13: @ %for.cond.cleanup
+; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
+entry:
+  %cmp8 = icmp eq i32 %N, 0
+  br i1 %cmp8, label %for.cond.cleanup, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  %min.iters.check = icmp ult i32 %N, 4
+  br i1 %min.iters.check, label %for.body.preheader16, label %vector.memcheck
+
+for.body.preheader16:                             ; preds = %middle.block, %vector.memcheck, %for.body.preheader
+  %i.09.ph = phi i32 [ 0, %vector.memcheck ], [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
+  %0 = xor i32 %i.09.ph, -1
+  %1 = add i32 %0, %N
+  %xtraiter = and i32 %N, 3
+  %lcmp.mod = icmp eq i32 %xtraiter, 0
+  br i1 %lcmp.mod, label %for.body.prol.loopexit, label %for.body.prol
+
+for.body.prol:                                    ; preds = %for.body.preheader16, %for.body.prol
+  %i.09.prol = phi i32 [ %inc.prol, %for.body.prol ], [ %i.09.ph, %for.body.preheader16 ]
+  %prol.iter = phi i32 [ %prol.iter.sub, %for.body.prol ], [ %xtraiter, %for.body.preheader16 ]
+  %arrayidx.prol = getelementptr inbounds float, float* %a, i32 %i.09.prol
+  %2 = load float, float* %arrayidx.prol, align 4
+  %arrayidx1.prol = getelementptr inbounds i32, i32* %b, i32 %i.09.prol
+  %3 = load i32, i32* %arrayidx1.prol, align 4
+  %conv.prol = sitofp i32 %3 to float
+  %mul.prol = fmul float %2, %conv.prol
+  %arrayidx2.prol = getelementptr inbounds float, float* %c, i32 %i.09.prol
+  store float %mul.prol, float* %arrayidx2.prol, align 4
+  %inc.prol = add nuw i32 %i.09.prol, 1
+  %prol.iter.sub = add i32 %prol.iter, -1
+  %prol.iter.cmp = icmp eq i32 %prol.iter.sub, 0
+  br i1 %prol.iter.cmp, label %for.body.prol.loopexit, label %for.body.prol
+
+for.body.prol.loopexit:                           ; preds = %for.body.prol, %for.body.preheader16
+  %i.09.unr = phi i32 [ %i.09.ph, %for.body.preheader16 ], [ %inc.prol, %for.body.prol ]
+  %4 = icmp ult i32 %1, 3
+  br i1 %4, label %for.cond.cleanup, label %for.body
+
+vector.memcheck:                                  ; preds = %for.body.preheader
+  %scevgep = getelementptr float, float* %c, i32 %N
+  %scevgep13 = getelementptr float, float* %a, i32 %N
+  %bound0 = icmp ugt float* %scevgep13, %c
+  %bound1 = icmp ugt float* %scevgep, %a
+  %found.conflict = and i1 %bound0, %bound1
+  br i1 %found.conflict, label %for.body.preheader16, label %vector.ph
+
+vector.ph:                                        ; preds = %vector.memcheck
+  %n.vec = and i32 %N, -4
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %5 = getelementptr inbounds float, float* %a, i32 %index
+  %6 = bitcast float* %5 to <4 x float>*
+  %wide.load = load <4 x float>, <4 x float>* %6, align 4
+  %7 = getelementptr inbounds i32, i32* %b, i32 %index
+  %8 = bitcast i32* %7 to <4 x i32>*
+  %wide.load15 = load <4 x i32>, <4 x i32>* %8, align 4
+  %9 = sitofp <4 x i32> %wide.load15 to <4 x float>
+  %10 = fmul <4 x float> %wide.load, %9
+  %11 = getelementptr inbounds float, float* %c, i32 %index
+  %12 = bitcast float* %11 to <4 x float>*
+  store <4 x float> %10, <4 x float>* %12, align 4
+  %index.next = add i32 %index, 4
+  %13 = icmp eq i32 %index.next, %n.vec
+  br i1 %13, label %middle.block, label %vector.body
+
+middle.block:                                     ; preds = %vector.body
+  %cmp.n = icmp eq i32 %n.vec, %N
+  br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader16
+
+for.cond.cleanup:                                 ; preds = %for.body.prol.loopexit, %for.body, %middle.block, %entry
+  ret void
+
+for.body:                                         ; preds = %for.body.prol.loopexit, %for.body
+  %i.09 = phi i32 [ %inc.3, %for.body ], [ %i.09.unr, %for.body.prol.loopexit ]
+  %arrayidx = getelementptr inbounds float, float* %a, i32 %i.09
+  %14 = load float, float* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds i32, i32* %b, i32 %i.09
+  %15 = load i32, i32* %arrayidx1, align 4
+  %conv = sitofp i32 %15 to float
+  %mul = fmul float %14, %conv
+  %arrayidx2 = getelementptr inbounds float, float* %c, i32 %i.09
+  store float %mul, float* %arrayidx2, align 4
+  %inc = add nuw i32 %i.09, 1
+  %arrayidx.1 = getelementptr inbounds float, float* %a, i32 %inc
+  %16 = load float, float* %arrayidx.1, align 4
+  %arrayidx1.1 = getelementptr inbounds i32, i32* %b, i32 %inc
+  %17 = load i32, i32* %arrayidx1.1, align 4
+  %conv.1 = sitofp i32 %17 to float
+  %mul.1 = fmul float %16, %conv.1
+  %arrayidx2.1 = getelementptr inbounds float, float* %c, i32 %inc
+  store float %mul.1, float* %arrayidx2.1, align 4
+  %inc.1 = add nuw i32 %i.09, 2
+  %arrayidx.2 = getelementptr inbounds float, float* %a, i32 %inc.1
+  %18 = load float, float* %arrayidx.2, align 4
+  %arrayidx1.2 = getelementptr inbounds i32, i32* %b, i32 %inc.1
+  %19 = load i32, i32* %arrayidx1.2, align 4
+  %conv.2 = sitofp i32 %19 to float
+  %mul.2 = fmul float %18, %conv.2
+  %arrayidx2.2 = getelementptr inbounds float, float* %c, i32 %inc.1
+  store float %mul.2, float* %arrayidx2.2, align 4
+  %inc.2 = add nuw i32 %i.09, 3
+  %arrayidx.3 = getelementptr inbounds float, float* %a, i32 %inc.2
+  %20 = load float, float* %arrayidx.3, align 4
+  %arrayidx1.3 = getelementptr inbounds i32, i32* %b, i32 %inc.2
+  %21 = load i32, i32* %arrayidx1.3, align 4
+  %conv.3 = sitofp i32 %21 to float
+  %mul.3 = fmul float %20, %conv.3
+  %arrayidx2.3 = getelementptr inbounds float, float* %c, i32 %inc.2
+  store float %mul.3, float* %arrayidx2.3, align 4
+  %inc.3 = add nuw i32 %i.09, 4
+  %exitcond.3 = icmp eq i32 %inc.3, %N
+  br i1 %exitcond.3, label %for.cond.cleanup, label %for.body
+}
+
+define arm_aapcs_vfpcc void @float_int_int_mul(i32* nocapture readonly %a, i32* nocapture readonly %b, float* nocapture %c, i32 %N) {
+; CHECK-LABEL: float_int_int_mul:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    push {r4, r5, r6, lr}
+; CHECK-NEXT:    cmp r3, #0
+; CHECK-NEXT:    beq .LBB4_8
+; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
+; CHECK-NEXT:    cmp r3, #3
+; CHECK-NEXT:    bhi .LBB4_3
+; CHECK-NEXT:  @ %bb.2:
+; CHECK-NEXT:    mov.w r12, #0
+; CHECK-NEXT:    b .LBB4_6
+; CHECK-NEXT:  .LBB4_3: @ %vector.ph
+; CHECK-NEXT:    bic r12, r3, #3
+; CHECK-NEXT:    movs r5, #1
+; CHECK-NEXT:    sub.w r6, r12, #4
+; CHECK-NEXT:    sub.w r4, r0, #16
+; CHECK-NEXT:    add.w lr, r5, r6, lsr #2
+; CHECK-NEXT:    sub.w r5, r1, #16
+; CHECK-NEXT:    sub.w r6, r2, #16
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB4_4: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrw.u32 q0, [r4, #16]!
+; CHECK-NEXT:    vldrw.u32 q1, [r5, #16]!
+; CHECK-NEXT:    vmul.i32 q0, q1, q0
+; CHECK-NEXT:    vcvt.f32.s32 q0, q0
+; CHECK-NEXT:    vstrb.8 q0, [r6, #16]!
+; CHECK-NEXT:    le lr, .LBB4_4
+; CHECK-NEXT:  @ %bb.5: @ %middle.block
+; CHECK-NEXT:    cmp r12, r3
+; CHECK-NEXT:    it eq
+; CHECK-NEXT:    popeq {r4, r5, r6, pc}
+; CHECK-NEXT:  .LBB4_6: @ %for.body.preheader11
+; CHECK-NEXT:    sub.w lr, r3, r12
+; CHECK-NEXT:    mvn r3, #3
+; CHECK-NEXT:    add.w r3, r3, r12, lsl #2
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:    add r0, r3
+; CHECK-NEXT:    add r1, r3
+; CHECK-NEXT:    add r2, r3
+; CHECK-NEXT:  .LBB4_7: @ %for.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    ldr r3, [r0, #4]!
+; CHECK-NEXT:    ldr r6, [r1, #4]!
+; CHECK-NEXT:    muls r3, r6, r3
+; CHECK-NEXT:    vmov s0, r3
+; CHECK-NEXT:    vcvt.f32.s32 s0, s0
+; CHECK-NEXT:    vstr s0, [r2, #4]
+; CHECK-NEXT:    adds r2, #4
+; CHECK-NEXT:    le lr, .LBB4_7
+; CHECK-NEXT:  .LBB4_8: @ %for.cond.cleanup
+; CHECK-NEXT:    pop {r4, r5, r6, pc}
+entry:
+  %cmp8 = icmp eq i32 %N, 0
+  br i1 %cmp8, label %for.cond.cleanup, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  %min.iters.check = icmp ult i32 %N, 4
+  br i1 %min.iters.check, label %for.body.preheader11, label %vector.ph
+
+for.body.preheader11:                             ; preds = %middle.block, %for.body.preheader
+  %i.09.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
+  br label %for.body
+
+vector.ph:                                        ; preds = %for.body.preheader
+  %n.vec = and i32 %N, -4
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %0 = getelementptr inbounds i32, i32* %a, i32 %index
+  %1 = bitcast i32* %0 to <4 x i32>*
+  %wide.load = load <4 x i32>, <4 x i32>* %1, align 4
+  %2 = getelementptr inbounds i32, i32* %b, i32 %index
+  %3 = bitcast i32* %2 to <4 x i32>*
+  %wide.load10 = load <4 x i32>, <4 x i32>* %3, align 4
+  %4 = mul nsw <4 x i32> %wide.load10, %wide.load
+  %5 = sitofp <4 x i32> %4 to <4 x float>
+  %6 = getelementptr inbounds float, float* %c, i32 %index
+  %7 = bitcast float* %6 to <4 x float>*
+  store <4 x float> %5, <4 x float>* %7, align 4
+  %index.next = add i32 %index, 4
+  %8 = icmp eq i32 %index.next, %n.vec
+  br i1 %8, label %middle.block, label %vector.body
+
+middle.block:                                     ; preds = %vector.body
+  %cmp.n = icmp eq i32 %n.vec, %N
+  br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader11
+
+for.cond.cleanup:                                 ; preds = %for.body, %middle.block, %entry
+  ret void
+
+for.body:                                         ; preds = %for.body.preheader11, %for.body
+  %i.09 = phi i32 [ %inc, %for.body ], [ %i.09.ph, %for.body.preheader11 ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i32 %i.09
+  %9 = load i32, i32* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds i32, i32* %b, i32 %i.09
+  %10 = load i32, i32* %arrayidx1, align 4
+  %mul = mul nsw i32 %10, %9
+  %conv = sitofp i32 %mul to float
+  %arrayidx2 = getelementptr inbounds float, float* %c, i32 %i.09
+  store float %conv, float* %arrayidx2, align 4
+  %inc = add nuw i32 %i.09, 1
+  %exitcond = icmp eq i32 %inc, %N
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+define arm_aapcs_vfpcc void @half_half_mul(half* nocapture readonly %a, half* nocapture readonly %b, float* nocapture %c, i32 %N) {
+; CHECK-LABEL: half_half_mul:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, lr}
+; CHECK-NEXT:    cmp r3, #0
+; CHECK-NEXT:    beq .LBB5_8
+; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
+; CHECK-NEXT:    cmp r3, #3
+; CHECK-NEXT:    bhi .LBB5_3
+; CHECK-NEXT:  @ %bb.2:
+; CHECK-NEXT:    mov.w r12, #0
+; CHECK-NEXT:    b .LBB5_6
+; CHECK-NEXT:  .LBB5_3: @ %vector.ph
+; CHECK-NEXT:    bic r12, r3, #3
+; CHECK-NEXT:    movs r5, #1
+; CHECK-NEXT:    sub.w r6, r12, #4
+; CHECK-NEXT:    sub.w r4, r0, #8
+; CHECK-NEXT:    add.w lr, r5, r6, lsr #2
+; CHECK-NEXT:    sub.w r5, r1, #8
+; CHECK-NEXT:    sub.w r6, r2, #16
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB5_4: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    ldr r8, [r5, #8]!
+; CHECK-NEXT:    ldr r7, [r4, #8]!
+; CHECK-NEXT:    vmov.32 q1[0], r8
+; CHECK-NEXT:    vmov.32 q0[0], r7
+; CHECK-NEXT:    ldr r7, [r5, #4]
+; CHECK-NEXT:    ldr.w r8, [r4, #4]
+; CHECK-NEXT:    vmov.32 q1[1], r7
+; CHECK-NEXT:    vmov.32 q0[1], r8
+; CHECK-NEXT:    vmul.f16 q0, q0, q1
+; CHECK-NEXT:    vmovx.f16 s6, s1
+; CHECK-NEXT:    vmovx.f16 s4, s0
+; CHECK-NEXT:    vcvtb.f32.f16 s11, s6
+; CHECK-NEXT:    vcvtb.f32.f16 s10, s1
+; CHECK-NEXT:    vcvtb.f32.f16 s9, s4
+; CHECK-NEXT:    vcvtb.f32.f16 s8, s0
+; CHECK-NEXT:    vstrb.8 q2, [r6, #16]!
+; CHECK-NEXT:    le lr, .LBB5_4
+; CHECK-NEXT:  @ %bb.5: @ %middle.block
+; CHECK-NEXT:    cmp r12, r3
+; CHECK-NEXT:    beq .LBB5_8
+; CHECK-NEXT:  .LBB5_6: @ %for.body.preheader11
+; CHECK-NEXT:    sub.w lr, r3, r12
+; CHECK-NEXT:    sub.w r3, r12, #1
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:    add.w r0, r0, r3, lsl #1
+; CHECK-NEXT:    add.w r1, r1, r3, lsl #1
+; CHECK-NEXT:    add.w r2, r2, r3, lsl #2
+; CHECK-NEXT:  .LBB5_7: @ %for.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldr.16 s0, [r1, #2]
+; CHECK-NEXT:    vldr.16 s2, [r0, #2]
+; CHECK-NEXT:    adds r0, #2
+; CHECK-NEXT:    adds r1, #2
+; CHECK-NEXT:    vmul.f16 s0, s2, s0
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-NEXT:    vstr s0, [r2, #4]
+; CHECK-NEXT:    adds r2, #4
+; CHECK-NEXT:    le lr, .LBB5_7
+; CHECK-NEXT:  .LBB5_8: @ %for.cond.cleanup
+; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, pc}
+entry:
+  %cmp8 = icmp eq i32 %N, 0
+  br i1 %cmp8, label %for.cond.cleanup, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  %min.iters.check = icmp ult i32 %N, 4
+  br i1 %min.iters.check, label %for.body.preheader11, label %vector.ph
+
+for.body.preheader11:                             ; preds = %middle.block, %for.body.preheader
+  %i.09.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
+  br label %for.body
+
+vector.ph:                                        ; preds = %for.body.preheader
+  %n.vec = and i32 %N, -4
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %0 = getelementptr inbounds half, half* %a, i32 %index
+  %1 = bitcast half* %0 to <4 x half>*
+  %wide.load = load <4 x half>, <4 x half>* %1, align 2
+  %2 = getelementptr inbounds half, half* %b, i32 %index
+  %3 = bitcast half* %2 to <4 x half>*
+  %wide.load10 = load <4 x half>, <4 x half>* %3, align 2
+  %4 = fmul <4 x half> %wide.load, %wide.load10
+  %5 = fpext <4 x half> %4 to <4 x float>
+  %6 = getelementptr inbounds float, float* %c, i32 %index
+  %7 = bitcast float* %6 to <4 x float>*
+  store <4 x float> %5, <4 x float>* %7, align 4
+  %index.next = add i32 %index, 4
+  %8 = icmp eq i32 %index.next, %n.vec
+  br i1 %8, label %middle.block, label %vector.body
+
+middle.block:                                     ; preds = %vector.body
+  %cmp.n = icmp eq i32 %n.vec, %N
+  br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader11
+
+for.cond.cleanup:                                 ; preds = %for.body, %middle.block, %entry
+  ret void
+
+for.body:                                         ; preds = %for.body.preheader11, %for.body
+  %i.09 = phi i32 [ %inc, %for.body ], [ %i.09.ph, %for.body.preheader11 ]
+  %arrayidx = getelementptr inbounds half, half* %a, i32 %i.09
+  %9 = load half, half* %arrayidx, align 2
+  %arrayidx1 = getelementptr inbounds half, half* %b, i32 %i.09
+  %10 = load half, half* %arrayidx1, align 2
+  %mul = fmul half %9, %10
+  %conv = fpext half %mul to float
+  %arrayidx2 = getelementptr inbounds float, float* %c, i32 %i.09
+  store float %conv, float* %arrayidx2, align 4
+  %inc = add nuw i32 %i.09, 1
+  %exitcond = icmp eq i32 %inc, %N
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+define arm_aapcs_vfpcc void @half_half_add(half* nocapture readonly %a, half* nocapture readonly %b, float* nocapture %c, i32 %N) {
+; CHECK-LABEL: half_half_add:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, lr}
+; CHECK-NEXT:    cmp r3, #0
+; CHECK-NEXT:    beq .LBB6_8
+; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
+; CHECK-NEXT:    cmp r3, #3
+; CHECK-NEXT:    bhi .LBB6_3
+; CHECK-NEXT:  @ %bb.2:
+; CHECK-NEXT:    mov.w r12, #0
+; CHECK-NEXT:    b .LBB6_6
+; CHECK-NEXT:  .LBB6_3: @ %vector.ph
+; CHECK-NEXT:    bic r12, r3, #3
+; CHECK-NEXT:    movs r5, #1
+; CHECK-NEXT:    sub.w r6, r12, #4
+; CHECK-NEXT:    sub.w r4, r0, #8
+; CHECK-NEXT:    add.w lr, r5, r6, lsr #2
+; CHECK-NEXT:    sub.w r5, r1, #8
+; CHECK-NEXT:    sub.w r6, r2, #16
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB6_4: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    ldr r8, [r5, #8]!
+; CHECK-NEXT:    ldr r7, [r4, #8]!
+; CHECK-NEXT:    vmov.32 q1[0], r8
+; CHECK-NEXT:    vmov.32 q0[0], r7
+; CHECK-NEXT:    ldr r7, [r5, #4]
+; CHECK-NEXT:    ldr.w r8, [r4, #4]
+; CHECK-NEXT:    vmov.32 q1[1], r7
+; CHECK-NEXT:    vmov.32 q0[1], r8
+; CHECK-NEXT:    vadd.f16 q0, q0, q1
+; CHECK-NEXT:    vmovx.f16 s6, s1
+; CHECK-NEXT:    vmovx.f16 s4, s0
+; CHECK-NEXT:    vcvtb.f32.f16 s11, s6
+; CHECK-NEXT:    vcvtb.f32.f16 s10, s1
+; CHECK-NEXT:    vcvtb.f32.f16 s9, s4
+; CHECK-NEXT:    vcvtb.f32.f16 s8, s0
+; CHECK-NEXT:    vstrb.8 q2, [r6, #16]!
+; CHECK-NEXT:    le lr, .LBB6_4
+; CHECK-NEXT:  @ %bb.5: @ %middle.block
+; CHECK-NEXT:    cmp r12, r3
+; CHECK-NEXT:    beq .LBB6_8
+; CHECK-NEXT:  .LBB6_6: @ %for.body.preheader11
+; CHECK-NEXT:    sub.w lr, r3, r12
+; CHECK-NEXT:    sub.w r3, r12, #1
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:    add.w r0, r0, r3, lsl #1
+; CHECK-NEXT:    add.w r1, r1, r3, lsl #1
+; CHECK-NEXT:    add.w r2, r2, r3, lsl #2
+; CHECK-NEXT:  .LBB6_7: @ %for.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldr.16 s0, [r1, #2]
+; CHECK-NEXT:    vldr.16 s2, [r0, #2]
+; CHECK-NEXT:    adds r0, #2
+; CHECK-NEXT:    adds r1, #2
+; CHECK-NEXT:    vadd.f16 s0, s2, s0
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-NEXT:    vstr s0, [r2, #4]
+; CHECK-NEXT:    adds r2, #4
+; CHECK-NEXT:    le lr, .LBB6_7
+; CHECK-NEXT:  .LBB6_8: @ %for.cond.cleanup
+; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, pc}
+entry:
+  %cmp8 = icmp eq i32 %N, 0
+  br i1 %cmp8, label %for.cond.cleanup, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  %min.iters.check = icmp ult i32 %N, 4
+  br i1 %min.iters.check, label %for.body.preheader11, label %vector.ph
+
+for.body.preheader11:                             ; preds = %middle.block, %for.body.preheader
+  %i.09.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
+  br label %for.body
+
+vector.ph:                                        ; preds = %for.body.preheader
+  %n.vec = and i32 %N, -4
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %0 = getelementptr inbounds half, half* %a, i32 %index
+  %1 = bitcast half* %0 to <4 x half>*
+  %wide.load = load <4 x half>, <4 x half>* %1, align 2
+  %2 = getelementptr inbounds half, half* %b, i32 %index
+  %3 = bitcast half* %2 to <4 x half>*
+  %wide.load10 = load <4 x half>, <4 x half>* %3, align 2
+  %4 = fadd <4 x half> %wide.load, %wide.load10
+  %5 = fpext <4 x half> %4 to <4 x float>
+  %6 = getelementptr inbounds float, float* %c, i32 %index
+  %7 = bitcast float* %6 to <4 x float>*
+  store <4 x float> %5, <4 x float>* %7, align 4
+  %index.next = add i32 %index, 4
+  %8 = icmp eq i32 %index.next, %n.vec
+  br i1 %8, label %middle.block, label %vector.body
+
+middle.block:                                     ; preds = %vector.body
+  %cmp.n = icmp eq i32 %n.vec, %N
+  br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader11
+
+for.cond.cleanup:                                 ; preds = %for.body, %middle.block, %entry
+  ret void
+
+for.body:                                         ; preds = %for.body.preheader11, %for.body
+  %i.09 = phi i32 [ %inc, %for.body ], [ %i.09.ph, %for.body.preheader11 ]
+  %arrayidx = getelementptr inbounds half, half* %a, i32 %i.09
+  %9 = load half, half* %arrayidx, align 2
+  %arrayidx1 = getelementptr inbounds half, half* %b, i32 %i.09
+  %10 = load half, half* %arrayidx1, align 2
+  %add = fadd half %9, %10
+  %conv = fpext half %add to float
+  %arrayidx2 = getelementptr inbounds float, float* %c, i32 %i.09
+  store float %conv, float* %arrayidx2, align 4
+  %inc = add nuw i32 %i.09, 1
+  %exitcond = icmp eq i32 %inc, %N
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+define arm_aapcs_vfpcc void @half_half_sub(half* nocapture readonly %a, half* nocapture readonly %b, float* nocapture %c, i32 %N) {
+; CHECK-LABEL: half_half_sub:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, lr}
+; CHECK-NEXT:    cmp r3, #0
+; CHECK-NEXT:    beq .LBB7_8
+; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
+; CHECK-NEXT:    cmp r3, #3
+; CHECK-NEXT:    bhi .LBB7_3
+; CHECK-NEXT:  @ %bb.2:
+; CHECK-NEXT:    mov.w r12, #0
+; CHECK-NEXT:    b .LBB7_6
+; CHECK-NEXT:  .LBB7_3: @ %vector.ph
+; CHECK-NEXT:    bic r12, r3, #3
+; CHECK-NEXT:    movs r5, #1
+; CHECK-NEXT:    sub.w r6, r12, #4
+; CHECK-NEXT:    sub.w r4, r0, #8
+; CHECK-NEXT:    add.w lr, r5, r6, lsr #2
+; CHECK-NEXT:    sub.w r5, r1, #8
+; CHECK-NEXT:    sub.w r6, r2, #16
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB7_4: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    ldr r8, [r5, #8]!
+; CHECK-NEXT:    ldr r7, [r4, #8]!
+; CHECK-NEXT:    vmov.32 q1[0], r8
+; CHECK-NEXT:    vmov.32 q0[0], r7
+; CHECK-NEXT:    ldr r7, [r5, #4]
+; CHECK-NEXT:    ldr.w r8, [r4, #4]
+; CHECK-NEXT:    vmov.32 q1[1], r7
+; CHECK-NEXT:    vmov.32 q0[1], r8
+; CHECK-NEXT:    vsub.f16 q0, q0, q1
+; CHECK-NEXT:    vmovx.f16 s6, s1
+; CHECK-NEXT:    vmovx.f16 s4, s0
+; CHECK-NEXT:    vcvtb.f32.f16 s11, s6
+; CHECK-NEXT:    vcvtb.f32.f16 s10, s1
+; CHECK-NEXT:    vcvtb.f32.f16 s9, s4
+; CHECK-NEXT:    vcvtb.f32.f16 s8, s0
+; CHECK-NEXT:    vstrb.8 q2, [r6, #16]!
+; CHECK-NEXT:    le lr, .LBB7_4
+; CHECK-NEXT:  @ %bb.5: @ %middle.block
+; CHECK-NEXT:    cmp r12, r3
+; CHECK-NEXT:    beq .LBB7_8
+; CHECK-NEXT:  .LBB7_6: @ %for.body.preheader11
+; CHECK-NEXT:    sub.w lr, r3, r12
+; CHECK-NEXT:    sub.w r3, r12, #1
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:    add.w r0, r0, r3, lsl #1
+; CHECK-NEXT:    add.w r1, r1, r3, lsl #1
+; CHECK-NEXT:    add.w r2, r2, r3, lsl #2
+; CHECK-NEXT:  .LBB7_7: @ %for.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldr.16 s0, [r1, #2]
+; CHECK-NEXT:    vldr.16 s2, [r0, #2]
+; CHECK-NEXT:    adds r0, #2
+; CHECK-NEXT:    adds r1, #2
+; CHECK-NEXT:    vsub.f16 s0, s2, s0
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-NEXT:    vstr s0, [r2, #4]
+; CHECK-NEXT:    adds r2, #4
+; CHECK-NEXT:    le lr, .LBB7_7
+; CHECK-NEXT:  .LBB7_8: @ %for.cond.cleanup
+; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, pc}
+entry:
+  %cmp8 = icmp eq i32 %N, 0
+  br i1 %cmp8, label %for.cond.cleanup, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  %min.iters.check = icmp ult i32 %N, 4
+  br i1 %min.iters.check, label %for.body.preheader11, label %vector.ph
+
+for.body.preheader11:                             ; preds = %middle.block, %for.body.preheader
+  %i.09.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
+  br label %for.body
+
+vector.ph:                                        ; preds = %for.body.preheader
+  %n.vec = and i32 %N, -4
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %0 = getelementptr inbounds half, half* %a, i32 %index
+  %1 = bitcast half* %0 to <4 x half>*
+  %wide.load = load <4 x half>, <4 x half>* %1, align 2
+  %2 = getelementptr inbounds half, half* %b, i32 %index
+  %3 = bitcast half* %2 to <4 x half>*
+  %wide.load10 = load <4 x half>, <4 x half>* %3, align 2
+  %4 = fsub <4 x half> %wide.load, %wide.load10
+  %5 = fpext <4 x half> %4 to <4 x float>
+  %6 = getelementptr inbounds float, float* %c, i32 %index
+  %7 = bitcast float* %6 to <4 x float>*
+  store <4 x float> %5, <4 x float>* %7, align 4
+  %index.next = add i32 %index, 4
+  %8 = icmp eq i32 %index.next, %n.vec
+  br i1 %8, label %middle.block, label %vector.body
+
+middle.block:                                     ; preds = %vector.body
+  %cmp.n = icmp eq i32 %n.vec, %N
+  br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader11
+
+for.cond.cleanup:                                 ; preds = %for.body, %middle.block, %entry
+  ret void
+
+for.body:                                         ; preds = %for.body.preheader11, %for.body
+  %i.09 = phi i32 [ %inc, %for.body ], [ %i.09.ph, %for.body.preheader11 ]
+  %arrayidx = getelementptr inbounds half, half* %a, i32 %i.09
+  %9 = load half, half* %arrayidx, align 2
+  %arrayidx1 = getelementptr inbounds half, half* %b, i32 %i.09
+  %10 = load half, half* %arrayidx1, align 2
+  %sub = fsub half %9, %10
+  %conv = fpext half %sub to float
+  %arrayidx2 = getelementptr inbounds float, float* %c, i32 %i.09
+  store float %conv, float* %arrayidx2, align 4
+  %inc = add nuw i32 %i.09, 1
+  %exitcond = icmp eq i32 %inc, %N
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+define arm_aapcs_vfpcc void @half_short_mul(half* nocapture readonly %a, i16* nocapture readonly %b, float* nocapture %c, i32 %N) {
+; CHECK-LABEL: half_short_mul:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, lr}
+; CHECK-NEXT:    cmp r3, #0
+; CHECK-NEXT:    beq .LBB8_8
+; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
+; CHECK-NEXT:    cmp r3, #3
+; CHECK-NEXT:    bhi .LBB8_3
+; CHECK-NEXT:  @ %bb.2:
+; CHECK-NEXT:    mov.w r12, #0
+; CHECK-NEXT:    b .LBB8_6
+; CHECK-NEXT:  .LBB8_3: @ %vector.ph
+; CHECK-NEXT:    bic r12, r3, #3
+; CHECK-NEXT:    movs r5, #1
+; CHECK-NEXT:    sub.w r6, r12, #4
+; CHECK-NEXT:    sub.w r4, r0, #8
+; CHECK-NEXT:    add.w lr, r5, r6, lsr #2
+; CHECK-NEXT:    sub.w r5, r1, #8
+; CHECK-NEXT:    sub.w r6, r2, #16
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB8_4: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    ldr r8, [r4, #8]!
+; CHECK-NEXT:    vldrh.u32 q0, [r5, #8]!
+; CHECK-NEXT:    vmov r7, s0
+; CHECK-NEXT:    vmov.16 q1[0], r7
+; CHECK-NEXT:    vmov r7, s1
+; CHECK-NEXT:    vmov.16 q1[1], r7
+; CHECK-NEXT:    vmov r7, s2
+; CHECK-NEXT:    vmov.16 q1[2], r7
+; CHECK-NEXT:    vmov r7, s3
+; CHECK-NEXT:    vmov.16 q1[3], r7
+; CHECK-NEXT:    ldr r7, [r4, #4]
+; CHECK-NEXT:    vcvt.f16.s16 q0, q1
+; CHECK-NEXT:    vmov.32 q1[0], r8
+; CHECK-NEXT:    vmov.32 q1[1], r7
+; CHECK-NEXT:    vmul.f16 q0, q1, q0
+; CHECK-NEXT:    vmovx.f16 s6, s1
+; CHECK-NEXT:    vmovx.f16 s4, s0
+; CHECK-NEXT:    vcvtb.f32.f16 s11, s6
+; CHECK-NEXT:    vcvtb.f32.f16 s10, s1
+; CHECK-NEXT:    vcvtb.f32.f16 s9, s4
+; CHECK-NEXT:    vcvtb.f32.f16 s8, s0
+; CHECK-NEXT:    vstrb.8 q2, [r6, #16]!
+; CHECK-NEXT:    le lr, .LBB8_4
+; CHECK-NEXT:  @ %bb.5: @ %middle.block
+; CHECK-NEXT:    cmp r12, r3
+; CHECK-NEXT:    beq .LBB8_8
+; CHECK-NEXT:  .LBB8_6: @ %for.body.preheader13
+; CHECK-NEXT:    sub.w lr, r3, r12
+; CHECK-NEXT:    sub.w r3, r12, #1
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:    add.w r0, r0, r3, lsl #1
+; CHECK-NEXT:    add.w r1, r1, r3, lsl #1
+; CHECK-NEXT:    add.w r2, r2, r3, lsl #2
+; CHECK-NEXT:  .LBB8_7: @ %for.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    ldrsh r3, [r1, #2]!
+; CHECK-NEXT:    vldr.16 s0, [r0, #2]
+; CHECK-NEXT:    adds r0, #2
+; CHECK-NEXT:    vmov s2, r3
+; CHECK-NEXT:    vcvt.f16.s32 s2, s2
+; CHECK-NEXT:    vmul.f16 s0, s0, s2
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-NEXT:    vstr s0, [r2, #4]
+; CHECK-NEXT:    adds r2, #4
+; CHECK-NEXT:    le lr, .LBB8_7
+; CHECK-NEXT:  .LBB8_8: @ %for.cond.cleanup
+; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, pc}
+entry:
+  %cmp10 = icmp eq i32 %N, 0
+  br i1 %cmp10, label %for.cond.cleanup, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  %min.iters.check = icmp ult i32 %N, 4
+  br i1 %min.iters.check, label %for.body.preheader13, label %vector.ph
+
+for.body.preheader13:                             ; preds = %middle.block, %for.body.preheader
+  %i.011.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
+  br label %for.body
+
+vector.ph:                                        ; preds = %for.body.preheader
+  %n.vec = and i32 %N, -4
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %0 = getelementptr inbounds half, half* %a, i32 %index
+  %1 = bitcast half* %0 to <4 x half>*
+  %wide.load = load <4 x half>, <4 x half>* %1, align 2
+  %2 = getelementptr inbounds i16, i16* %b, i32 %index
+  %3 = bitcast i16* %2 to <4 x i16>*
+  %wide.load12 = load <4 x i16>, <4 x i16>* %3, align 2
+  %4 = sitofp <4 x i16> %wide.load12 to <4 x half>
+  %5 = fmul <4 x half> %wide.load, %4
+  %6 = fpext <4 x half> %5 to <4 x float>
+  %7 = getelementptr inbounds float, float* %c, i32 %index
+  %8 = bitcast float* %7 to <4 x float>*
+  store <4 x float> %6, <4 x float>* %8, align 4
+  %index.next = add i32 %index, 4
+  %9 = icmp eq i32 %index.next, %n.vec
+  br i1 %9, label %middle.block, label %vector.body
+
+middle.block:                                     ; preds = %vector.body
+  %cmp.n = icmp eq i32 %n.vec, %N
+  br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader13
+
+for.cond.cleanup:                                 ; preds = %for.body, %middle.block, %entry
+  ret void
+
+for.body:                                         ; preds = %for.body.preheader13, %for.body
+  %i.011 = phi i32 [ %inc, %for.body ], [ %i.011.ph, %for.body.preheader13 ]
+  %arrayidx = getelementptr inbounds half, half* %a, i32 %i.011
+  %10 = load half, half* %arrayidx, align 2
+  %arrayidx1 = getelementptr inbounds i16, i16* %b, i32 %i.011
+  %11 = load i16, i16* %arrayidx1, align 2
+  %conv2 = sitofp i16 %11 to half
+  %mul = fmul half %10, %conv2
+  %conv3 = fpext half %mul to float
+  %arrayidx4 = getelementptr inbounds float, float* %c, i32 %i.011
+  store float %conv3, float* %arrayidx4, align 4
+  %inc = add nuw i32 %i.011, 1
+  %exitcond = icmp eq i32 %inc, %N
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+define arm_aapcs_vfpcc float @half_half_mac(half* nocapture readonly %a, half* nocapture readonly %b, i32 %N) {
+; CHECK-LABEL: half_half_mac:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    push {r4, lr}
+; CHECK-NEXT:    cbz r2, .LBB9_3
+; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
+; CHECK-NEXT:    subs r3, r2, #1
+; CHECK-NEXT:    and r4, r2, #3
+; CHECK-NEXT:    cmp r3, #3
+; CHECK-NEXT:    bhs .LBB9_4
+; CHECK-NEXT:  @ %bb.2:
+; CHECK-NEXT:    vldr s0, .LCPI9_0
+; CHECK-NEXT:    mov.w r12, #0
+; CHECK-NEXT:    b .LBB9_6
+; CHECK-NEXT:  .LBB9_3:
+; CHECK-NEXT:    vldr s0, .LCPI9_0
+; CHECK-NEXT:    b .LBB9_9
+; CHECK-NEXT:  .LBB9_4: @ %for.body.preheader.new
+; CHECK-NEXT:    bic r2, r2, #3
+; CHECK-NEXT:    movs r3, #1
+; CHECK-NEXT:    subs r2, #4
+; CHECK-NEXT:    vldr s0, .LCPI9_0
+; CHECK-NEXT:    mov.w r12, #0
+; CHECK-NEXT:    add.w lr, r3, r2, lsr #2
+; CHECK-NEXT:    sub.w r3, r0, #8
+; CHECK-NEXT:    sub.w r2, r1, #8
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB9_5: @ %for.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldr.16 s2, [r2, #14]
+; CHECK-NEXT:    vldr.16 s4, [r3, #14]
+; CHECK-NEXT:    vldr.16 s6, [r3, #12]
+; CHECK-NEXT:    vldr.16 s8, [r3, #10]
+; CHECK-NEXT:    vmul.f16 s2, s4, s2
+; CHECK-NEXT:    vldr.16 s4, [r2, #12]
+; CHECK-NEXT:    vldr.16 s10, [r3, #8]
+; CHECK-NEXT:    adds r3, #8
+; CHECK-NEXT:    vmul.f16 s4, s6, s4
+; CHECK-NEXT:    vldr.16 s6, [r2, #10]
+; CHECK-NEXT:    add.w r12, r12, #4
+; CHECK-NEXT:    vmul.f16 s6, s8, s6
+; CHECK-NEXT:    vldr.16 s8, [r2, #8]
+; CHECK-NEXT:    adds r2, #8
+; CHECK-NEXT:    vmul.f16 s8, s10, s8
+; CHECK-NEXT:    vcvtb.f32.f16 s8, s8
+; CHECK-NEXT:    vcvtb.f32.f16 s6, s6
+; CHECK-NEXT:    vadd.f32 s0, s0, s8
+; CHECK-NEXT:    vcvtb.f32.f16 s4, s4
+; CHECK-NEXT:    vcvtb.f32.f16 s2, s2
+; CHECK-NEXT:    vadd.f32 s0, s0, s6
+; CHECK-NEXT:    vadd.f32 s0, s0, s4
+; CHECK-NEXT:    vadd.f32 s0, s0, s2
+; CHECK-NEXT:    le lr, .LBB9_5
+; CHECK-NEXT:  .LBB9_6: @ %for.cond.cleanup.loopexit.unr-lcssa
+; CHECK-NEXT:    wls lr, r4, .LBB9_9
+; CHECK-NEXT:  @ %bb.7: @ %for.body.epil.preheader
+; CHECK-NEXT:    mvn r2, #1
+; CHECK-NEXT:    mov lr, r4
+; CHECK-NEXT:    add.w r2, r2, r12, lsl #1
+; CHECK-NEXT:    add r0, r2
+; CHECK-NEXT:    add r1, r2
+; CHECK-NEXT:  .LBB9_8: @ %for.body.epil
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldr.16 s2, [r1, #2]
+; CHECK-NEXT:    vldr.16 s4, [r0, #2]
+; CHECK-NEXT:    adds r0, #2
+; CHECK-NEXT:    adds r1, #2
+; CHECK-NEXT:    vmul.f16 s2, s4, s2
+; CHECK-NEXT:    vcvtb.f32.f16 s2, s2
+; CHECK-NEXT:    vadd.f32 s0, s0, s2
+; CHECK-NEXT:    le lr, .LBB9_8
+; CHECK-NEXT:  .LBB9_9: @ %for.cond.cleanup
+; CHECK-NEXT:    pop {r4, pc}
+; CHECK-NEXT:    .p2align 2
+; CHECK-NEXT:  @ %bb.10:
+; CHECK-NEXT:  .LCPI9_0:
+; CHECK-NEXT:    .long 0 @ float 0
+entry:
+  %cmp8 = icmp eq i32 %N, 0
+  br i1 %cmp8, label %for.cond.cleanup, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  %0 = add i32 %N, -1
+  %xtraiter = and i32 %N, 3
+  %1 = icmp ult i32 %0, 3
+  br i1 %1, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body.preheader.new
+
+for.body.preheader.new:                           ; preds = %for.body.preheader
+  %unroll_iter = sub i32 %N, %xtraiter
+  br label %for.body
+
+for.cond.cleanup.loopexit.unr-lcssa:              ; preds = %for.body, %for.body.preheader
+  %add.lcssa.ph = phi float [ undef, %for.body.preheader ], [ %add.3, %for.body ]
+  %i.010.unr = phi i32 [ 0, %for.body.preheader ], [ %inc.3, %for.body ]
+  %res.09.unr = phi float [ 0.000000e+00, %for.body.preheader ], [ %add.3, %for.body ]
+  %lcmp.mod = icmp eq i32 %xtraiter, 0
+  br i1 %lcmp.mod, label %for.cond.cleanup, label %for.body.epil
+
+for.body.epil:                                    ; preds = %for.cond.cleanup.loopexit.unr-lcssa, %for.body.epil
+  %i.010.epil = phi i32 [ %inc.epil, %for.body.epil ], [ %i.010.unr, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %res.09.epil = phi float [ %add.epil, %for.body.epil ], [ %res.09.unr, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %epil.iter = phi i32 [ %epil.iter.sub, %for.body.epil ], [ %xtraiter, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %arrayidx.epil = getelementptr inbounds half, half* %a, i32 %i.010.epil
+  %2 = load half, half* %arrayidx.epil, align 2
+  %arrayidx1.epil = getelementptr inbounds half, half* %b, i32 %i.010.epil
+  %3 = load half, half* %arrayidx1.epil, align 2
+  %mul.epil = fmul half %2, %3
+  %conv.epil = fpext half %mul.epil to float
+  %add.epil = fadd float %res.09.epil, %conv.epil
+  %inc.epil = add nuw i32 %i.010.epil, 1
+  %epil.iter.sub = add i32 %epil.iter, -1
+  %epil.iter.cmp = icmp eq i32 %epil.iter.sub, 0
+  br i1 %epil.iter.cmp, label %for.cond.cleanup, label %for.body.epil
+
+for.cond.cleanup:                                 ; preds = %for.cond.cleanup.loopexit.unr-lcssa, %for.body.epil, %entry
+  %res.0.lcssa = phi float [ 0.000000e+00, %entry ], [ %add.lcssa.ph, %for.cond.cleanup.loopexit.unr-lcssa ], [ %add.epil, %for.body.epil ]
+  ret float %res.0.lcssa
+
+for.body:                                         ; preds = %for.body, %for.body.preheader.new
+  %i.010 = phi i32 [ 0, %for.body.preheader.new ], [ %inc.3, %for.body ]
+  %res.09 = phi float [ 0.000000e+00, %for.body.preheader.new ], [ %add.3, %for.body ]
+  %niter = phi i32 [ %unroll_iter, %for.body.preheader.new ], [ %niter.nsub.3, %for.body ]
+  %arrayidx = getelementptr inbounds half, half* %a, i32 %i.010
+  %4 = load half, half* %arrayidx, align 2
+  %arrayidx1 = getelementptr inbounds half, half* %b, i32 %i.010
+  %5 = load half, half* %arrayidx1, align 2
+  %mul = fmul half %4, %5
+  %conv = fpext half %mul to float
+  %add = fadd float %res.09, %conv
+  %inc = or i32 %i.010, 1
+  %arrayidx.1 = getelementptr inbounds half, half* %a, i32 %inc
+  %6 = load half, half* %arrayidx.1, align 2
+  %arrayidx1.1 = getelementptr inbounds half, half* %b, i32 %inc
+  %7 = load half, half* %arrayidx1.1, align 2
+  %mul.1 = fmul half %6, %7
+  %conv.1 = fpext half %mul.1 to float
+  %add.1 = fadd float %add, %conv.1
+  %inc.1 = or i32 %i.010, 2
+  %arrayidx.2 = getelementptr inbounds half, half* %a, i32 %inc.1
+  %8 = load half, half* %arrayidx.2, align 2
+  %arrayidx1.2 = getelementptr inbounds half, half* %b, i32 %inc.1
+  %9 = load half, half* %arrayidx1.2, align 2
+  %mul.2 = fmul half %8, %9
+  %conv.2 = fpext half %mul.2 to float
+  %add.2 = fadd float %add.1, %conv.2
+  %inc.2 = or i32 %i.010, 3
+  %arrayidx.3 = getelementptr inbounds half, half* %a, i32 %inc.2
+  %10 = load half, half* %arrayidx.3, align 2
+  %arrayidx1.3 = getelementptr inbounds half, half* %b, i32 %inc.2
+  %11 = load half, half* %arrayidx1.3, align 2
+  %mul.3 = fmul half %10, %11
+  %conv.3 = fpext half %mul.3 to float
+  %add.3 = fadd float %add.2, %conv.3
+  %inc.3 = add nuw i32 %i.010, 4
+  %niter.nsub.3 = add i32 %niter, -4
+  %niter.ncmp.3 = icmp eq i32 %niter.nsub.3, 0
+  br i1 %niter.ncmp.3, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body
+}
+
+define arm_aapcs_vfpcc float @half_half_acc(half* nocapture readonly %a, half* nocapture readonly %b, i32 %N) {
+; CHECK-LABEL: half_half_acc:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    push {r4, lr}
+; CHECK-NEXT:    cbz r2, .LBB10_3
+; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
+; CHECK-NEXT:    subs r3, r2, #1
+; CHECK-NEXT:    and r4, r2, #3
+; CHECK-NEXT:    cmp r3, #3
+; CHECK-NEXT:    bhs .LBB10_4
+; CHECK-NEXT:  @ %bb.2:
+; CHECK-NEXT:    vldr s0, .LCPI10_0
+; CHECK-NEXT:    mov.w r12, #0
+; CHECK-NEXT:    b .LBB10_6
+; CHECK-NEXT:  .LBB10_3:
+; CHECK-NEXT:    vldr s0, .LCPI10_0
+; CHECK-NEXT:    b .LBB10_9
+; CHECK-NEXT:  .LBB10_4: @ %for.body.preheader.new
+; CHECK-NEXT:    bic r2, r2, #3
+; CHECK-NEXT:    movs r3, #1
+; CHECK-NEXT:    subs r2, #4
+; CHECK-NEXT:    vldr s0, .LCPI10_0
+; CHECK-NEXT:    mov.w r12, #0
+; CHECK-NEXT:    add.w lr, r3, r2, lsr #2
+; CHECK-NEXT:    sub.w r3, r0, #8
+; CHECK-NEXT:    sub.w r2, r1, #8
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB10_5: @ %for.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldr.16 s2, [r2, #14]
+; CHECK-NEXT:    vldr.16 s4, [r3, #14]
+; CHECK-NEXT:    vldr.16 s6, [r3, #12]
+; CHECK-NEXT:    vldr.16 s8, [r3, #10]
+; CHECK-NEXT:    vadd.f16 s2, s4, s2
+; CHECK-NEXT:    vldr.16 s4, [r2, #12]
+; CHECK-NEXT:    vldr.16 s10, [r3, #8]
+; CHECK-NEXT:    adds r3, #8
+; CHECK-NEXT:    vadd.f16 s4, s6, s4
+; CHECK-NEXT:    vldr.16 s6, [r2, #10]
+; CHECK-NEXT:    add.w r12, r12, #4
+; CHECK-NEXT:    vadd.f16 s6, s8, s6
+; CHECK-NEXT:    vldr.16 s8, [r2, #8]
+; CHECK-NEXT:    adds r2, #8
+; CHECK-NEXT:    vadd.f16 s8, s10, s8
+; CHECK-NEXT:    vcvtb.f32.f16 s8, s8
+; CHECK-NEXT:    vcvtb.f32.f16 s6, s6
+; CHECK-NEXT:    vadd.f32 s0, s0, s8
+; CHECK-NEXT:    vcvtb.f32.f16 s4, s4
+; CHECK-NEXT:    vcvtb.f32.f16 s2, s2
+; CHECK-NEXT:    vadd.f32 s0, s0, s6
+; CHECK-NEXT:    vadd.f32 s0, s0, s4
+; CHECK-NEXT:    vadd.f32 s0, s0, s2
+; CHECK-NEXT:    le lr, .LBB10_5
+; CHECK-NEXT:  .LBB10_6: @ %for.cond.cleanup.loopexit.unr-lcssa
+; CHECK-NEXT:    wls lr, r4, .LBB10_9
+; CHECK-NEXT:  @ %bb.7: @ %for.body.epil.preheader
+; CHECK-NEXT:    mvn r2, #1
+; CHECK-NEXT:    mov lr, r4
+; CHECK-NEXT:    add.w r2, r2, r12, lsl #1
+; CHECK-NEXT:    add r0, r2
+; CHECK-NEXT:    add r1, r2
+; CHECK-NEXT:  .LBB10_8: @ %for.body.epil
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldr.16 s2, [r1, #2]
+; CHECK-NEXT:    vldr.16 s4, [r0, #2]
+; CHECK-NEXT:    adds r0, #2
+; CHECK-NEXT:    adds r1, #2
+; CHECK-NEXT:    vadd.f16 s2, s4, s2
+; CHECK-NEXT:    vcvtb.f32.f16 s2, s2
+; CHECK-NEXT:    vadd.f32 s0, s0, s2
+; CHECK-NEXT:    le lr, .LBB10_8
+; CHECK-NEXT:  .LBB10_9: @ %for.cond.cleanup
+; CHECK-NEXT:    pop {r4, pc}
+; CHECK-NEXT:    .p2align 2
+; CHECK-NEXT:  @ %bb.10:
+; CHECK-NEXT:  .LCPI10_0:
+; CHECK-NEXT:    .long 0 @ float 0
+entry:
+  %cmp9 = icmp eq i32 %N, 0
+  br i1 %cmp9, label %for.cond.cleanup, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  %0 = add i32 %N, -1
+  %xtraiter = and i32 %N, 3
+  %1 = icmp ult i32 %0, 3
+  br i1 %1, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body.preheader.new
+
+for.body.preheader.new:                           ; preds = %for.body.preheader
+  %unroll_iter = sub i32 %N, %xtraiter
+  br label %for.body
+
+for.cond.cleanup.loopexit.unr-lcssa:              ; preds = %for.body, %for.body.preheader
+  %add2.lcssa.ph = phi float [ undef, %for.body.preheader ], [ %add2.3, %for.body ]
+  %i.011.unr = phi i32 [ 0, %for.body.preheader ], [ %inc.3, %for.body ]
+  %res.010.unr = phi float [ 0.000000e+00, %for.body.preheader ], [ %add2.3, %for.body ]
+  %lcmp.mod = icmp eq i32 %xtraiter, 0
+  br i1 %lcmp.mod, label %for.cond.cleanup, label %for.body.epil
+
+for.body.epil:                                    ; preds = %for.cond.cleanup.loopexit.unr-lcssa, %for.body.epil
+  %i.011.epil = phi i32 [ %inc.epil, %for.body.epil ], [ %i.011.unr, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %res.010.epil = phi float [ %add2.epil, %for.body.epil ], [ %res.010.unr, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %epil.iter = phi i32 [ %epil.iter.sub, %for.body.epil ], [ %xtraiter, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %arrayidx.epil = getelementptr inbounds half, half* %a, i32 %i.011.epil
+  %2 = load half, half* %arrayidx.epil, align 2
+  %arrayidx1.epil = getelementptr inbounds half, half* %b, i32 %i.011.epil
+  %3 = load half, half* %arrayidx1.epil, align 2
+  %add.epil = fadd half %2, %3
+  %conv.epil = fpext half %add.epil to float
+  %add2.epil = fadd float %res.010.epil, %conv.epil
+  %inc.epil = add nuw i32 %i.011.epil, 1
+  %epil.iter.sub = add i32 %epil.iter, -1
+  %epil.iter.cmp = icmp eq i32 %epil.iter.sub, 0
+  br i1 %epil.iter.cmp, label %for.cond.cleanup, label %for.body.epil
+
+for.cond.cleanup:                                 ; preds = %for.cond.cleanup.loopexit.unr-lcssa, %for.body.epil, %entry
+  %res.0.lcssa = phi float [ 0.000000e+00, %entry ], [ %add2.lcssa.ph, %for.cond.cleanup.loopexit.unr-lcssa ], [ %add2.epil, %for.body.epil ]
+  ret float %res.0.lcssa
+
+for.body:                                         ; preds = %for.body, %for.body.preheader.new
+  %i.011 = phi i32 [ 0, %for.body.preheader.new ], [ %inc.3, %for.body ]
+  %res.010 = phi float [ 0.000000e+00, %for.body.preheader.new ], [ %add2.3, %for.body ]
+  %niter = phi i32 [ %unroll_iter, %for.body.preheader.new ], [ %niter.nsub.3, %for.body ]
+  %arrayidx = getelementptr inbounds half, half* %a, i32 %i.011
+  %4 = load half, half* %arrayidx, align 2
+  %arrayidx1 = getelementptr inbounds half, half* %b, i32 %i.011
+  %5 = load half, half* %arrayidx1, align 2
+  %add = fadd half %4, %5
+  %conv = fpext half %add to float
+  %add2 = fadd float %res.010, %conv
+  %inc = or i32 %i.011, 1
+  %arrayidx.1 = getelementptr inbounds half, half* %a, i32 %inc
+  %6 = load half, half* %arrayidx.1, align 2
+  %arrayidx1.1 = getelementptr inbounds half, half* %b, i32 %inc
+  %7 = load half, half* %arrayidx1.1, align 2
+  %add.1 = fadd half %6, %7
+  %conv.1 = fpext half %add.1 to float
+  %add2.1 = fadd float %add2, %conv.1
+  %inc.1 = or i32 %i.011, 2
+  %arrayidx.2 = getelementptr inbounds half, half* %a, i32 %inc.1
+  %8 = load half, half* %arrayidx.2, align 2
+  %arrayidx1.2 = getelementptr inbounds half, half* %b, i32 %inc.1
+  %9 = load half, half* %arrayidx1.2, align 2
+  %add.2 = fadd half %8, %9
+  %conv.2 = fpext half %add.2 to float
+  %add2.2 = fadd float %add2.1, %conv.2
+  %inc.2 = or i32 %i.011, 3
+  %arrayidx.3 = getelementptr inbounds half, half* %a, i32 %inc.2
+  %10 = load half, half* %arrayidx.3, align 2
+  %arrayidx1.3 = getelementptr inbounds half, half* %b, i32 %inc.2
+  %11 = load half, half* %arrayidx1.3, align 2
+  %add.3 = fadd half %10, %11
+  %conv.3 = fpext half %add.3 to float
+  %add2.3 = fadd float %add2.2, %conv.3
+  %inc.3 = add nuw i32 %i.011, 4
+  %niter.nsub.3 = add i32 %niter, -4
+  %niter.ncmp.3 = icmp eq i32 %niter.nsub.3, 0
+  br i1 %niter.ncmp.3, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body
+}
+
+define arm_aapcs_vfpcc float @half_short_mac(half* nocapture readonly %a, i16* nocapture readonly %b, i32 %N) {
+; CHECK-LABEL: half_short_mac:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    push {r4, r5, r6, r7, lr}
+; CHECK-NEXT:    cbz r2, .LBB11_3
+; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
+; CHECK-NEXT:    subs r3, r2, #1
+; CHECK-NEXT:    and r7, r2, #3
+; CHECK-NEXT:    cmp r3, #3
+; CHECK-NEXT:    bhs .LBB11_4
+; CHECK-NEXT:  @ %bb.2:
+; CHECK-NEXT:    vldr s0, .LCPI11_0
+; CHECK-NEXT:    mov.w r12, #0
+; CHECK-NEXT:    b .LBB11_6
+; CHECK-NEXT:  .LBB11_3:
+; CHECK-NEXT:    vldr s0, .LCPI11_0
+; CHECK-NEXT:    b .LBB11_9
+; CHECK-NEXT:  .LBB11_4: @ %for.body.preheader.new
+; CHECK-NEXT:    bic r2, r2, #3
+; CHECK-NEXT:    movs r3, #1
+; CHECK-NEXT:    subs r2, #4
+; CHECK-NEXT:    vldr s0, .LCPI11_0
+; CHECK-NEXT:    mov.w r12, #0
+; CHECK-NEXT:    add.w lr, r3, r2, lsr #2
+; CHECK-NEXT:    movs r3, #0
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB11_5: @ %for.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    adds r2, r1, r3
+; CHECK-NEXT:    adds r6, r0, r3
+; CHECK-NEXT:    vldr.16 s2, [r6, #6]
+; CHECK-NEXT:    add.w r12, r12, #4
+; CHECK-NEXT:    ldrsh.w r4, [r2, #2]
+; CHECK-NEXT:    ldrsh.w r5, [r2, #4]
+; CHECK-NEXT:    ldrsh.w r2, [r2, #6]
+; CHECK-NEXT:    vmov s8, r4
+; CHECK-NEXT:    vmov s6, r5
+; CHECK-NEXT:    vmov s4, r2
+; CHECK-NEXT:    vcvt.f16.s32 s4, s4
+; CHECK-NEXT:    vmul.f16 s2, s2, s4
+; CHECK-NEXT:    vldr.16 s4, [r6, #4]
+; CHECK-NEXT:    vcvt.f16.s32 s6, s6
+; CHECK-NEXT:    vmul.f16 s4, s4, s6
+; CHECK-NEXT:    vldr.16 s6, [r6, #2]
+; CHECK-NEXT:    vcvt.f16.s32 s8, s8
+; CHECK-NEXT:    ldrsh r2, [r1, r3]
+; CHECK-NEXT:    vmul.f16 s6, s6, s8
+; CHECK-NEXT:    vldr.16 s8, [r6]
+; CHECK-NEXT:    adds r3, #8
+; CHECK-NEXT:    vmov s10, r2
+; CHECK-NEXT:    vcvt.f16.s32 s10, s10
+; CHECK-NEXT:    vmul.f16 s8, s8, s10
+; CHECK-NEXT:    vcvtb.f32.f16 s8, s8
+; CHECK-NEXT:    vcvtb.f32.f16 s6, s6
+; CHECK-NEXT:    vadd.f32 s0, s0, s8
+; CHECK-NEXT:    vcvtb.f32.f16 s4, s4
+; CHECK-NEXT:    vcvtb.f32.f16 s2, s2
+; CHECK-NEXT:    vadd.f32 s0, s0, s6
+; CHECK-NEXT:    vadd.f32 s0, s0, s4
+; CHECK-NEXT:    vadd.f32 s0, s0, s2
+; CHECK-NEXT:    le lr, .LBB11_5
+; CHECK-NEXT:  .LBB11_6: @ %for.cond.cleanup.loopexit.unr-lcssa
+; CHECK-NEXT:    wls lr, r7, .LBB11_9
+; CHECK-NEXT:  @ %bb.7: @ %for.body.epil.preheader
+; CHECK-NEXT:    mvn r3, #1
+; CHECK-NEXT:    add.w r2, r3, r12, lsl #1
+; CHECK-NEXT:    add r0, r2
+; CHECK-NEXT:    add r1, r2
+; CHECK-NEXT:    mov lr, r7
+; CHECK-NEXT:  .LBB11_8: @ %for.body.epil
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    ldrsh r2, [r1, #2]!
+; CHECK-NEXT:    vldr.16 s2, [r0, #2]
+; CHECK-NEXT:    adds r0, #2
+; CHECK-NEXT:    vmov s4, r2
+; CHECK-NEXT:    vcvt.f16.s32 s4, s4
+; CHECK-NEXT:    vmul.f16 s2, s2, s4
+; CHECK-NEXT:    vcvtb.f32.f16 s2, s2
+; CHECK-NEXT:    vadd.f32 s0, s0, s2
+; CHECK-NEXT:    le lr, .LBB11_8
+; CHECK-NEXT:  .LBB11_9: @ %for.cond.cleanup
+; CHECK-NEXT:    pop {r4, r5, r6, r7, pc}
+; CHECK-NEXT:    .p2align 2
+; CHECK-NEXT:  @ %bb.10:
+; CHECK-NEXT:  .LCPI11_0:
+; CHECK-NEXT:    .long 0 @ float 0
+entry:
+  %cmp10 = icmp eq i32 %N, 0
+  br i1 %cmp10, label %for.cond.cleanup, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  %0 = add i32 %N, -1
+  %xtraiter = and i32 %N, 3
+  %1 = icmp ult i32 %0, 3
+  br i1 %1, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body.preheader.new
+
+for.body.preheader.new:                           ; preds = %for.body.preheader
+  %unroll_iter = sub i32 %N, %xtraiter
+  br label %for.body
+
+for.cond.cleanup.loopexit.unr-lcssa:              ; preds = %for.body, %for.body.preheader
+  %add.lcssa.ph = phi float [ undef, %for.body.preheader ], [ %add.3, %for.body ]
+  %i.012.unr = phi i32 [ 0, %for.body.preheader ], [ %inc.3, %for.body ]
+  %res.011.unr = phi float [ 0.000000e+00, %for.body.preheader ], [ %add.3, %for.body ]
+  %lcmp.mod = icmp eq i32 %xtraiter, 0
+  br i1 %lcmp.mod, label %for.cond.cleanup, label %for.body.epil
+
+for.body.epil:                                    ; preds = %for.cond.cleanup.loopexit.unr-lcssa, %for.body.epil
+  %i.012.epil = phi i32 [ %inc.epil, %for.body.epil ], [ %i.012.unr, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %res.011.epil = phi float [ %add.epil, %for.body.epil ], [ %res.011.unr, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %epil.iter = phi i32 [ %epil.iter.sub, %for.body.epil ], [ %xtraiter, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %arrayidx.epil = getelementptr inbounds half, half* %a, i32 %i.012.epil
+  %2 = load half, half* %arrayidx.epil, align 2
+  %arrayidx1.epil = getelementptr inbounds i16, i16* %b, i32 %i.012.epil
+  %3 = load i16, i16* %arrayidx1.epil, align 2
+  %conv2.epil = sitofp i16 %3 to half
+  %mul.epil = fmul half %2, %conv2.epil
+  %conv3.epil = fpext half %mul.epil to float
+  %add.epil = fadd float %res.011.epil, %conv3.epil
+  %inc.epil = add nuw i32 %i.012.epil, 1
+  %epil.iter.sub = add i32 %epil.iter, -1
+  %epil.iter.cmp = icmp eq i32 %epil.iter.sub, 0
+  br i1 %epil.iter.cmp, label %for.cond.cleanup, label %for.body.epil
+
+for.cond.cleanup:                                 ; preds = %for.cond.cleanup.loopexit.unr-lcssa, %for.body.epil, %entry
+  %res.0.lcssa = phi float [ 0.000000e+00, %entry ], [ %add.lcssa.ph, %for.cond.cleanup.loopexit.unr-lcssa ], [ %add.epil, %for.body.epil ]
+  ret float %res.0.lcssa
+
+for.body:                                         ; preds = %for.body, %for.body.preheader.new
+  %i.012 = phi i32 [ 0, %for.body.preheader.new ], [ %inc.3, %for.body ]
+  %res.011 = phi float [ 0.000000e+00, %for.body.preheader.new ], [ %add.3, %for.body ]
+  %niter = phi i32 [ %unroll_iter, %for.body.preheader.new ], [ %niter.nsub.3, %for.body ]
+  %arrayidx = getelementptr inbounds half, half* %a, i32 %i.012
+  %4 = load half, half* %arrayidx, align 2
+  %arrayidx1 = getelementptr inbounds i16, i16* %b, i32 %i.012
+  %5 = load i16, i16* %arrayidx1, align 2
+  %conv2 = sitofp i16 %5 to half
+  %mul = fmul half %4, %conv2
+  %conv3 = fpext half %mul to float
+  %add = fadd float %res.011, %conv3
+  %inc = or i32 %i.012, 1
+  %arrayidx.1 = getelementptr inbounds half, half* %a, i32 %inc
+  %6 = load half, half* %arrayidx.1, align 2
+  %arrayidx1.1 = getelementptr inbounds i16, i16* %b, i32 %inc
+  %7 = load i16, i16* %arrayidx1.1, align 2
+  %conv2.1 = sitofp i16 %7 to half
+  %mul.1 = fmul half %6, %conv2.1
+  %conv3.1 = fpext half %mul.1 to float
+  %add.1 = fadd float %add, %conv3.1
+  %inc.1 = or i32 %i.012, 2
+  %arrayidx.2 = getelementptr inbounds half, half* %a, i32 %inc.1
+  %8 = load half, half* %arrayidx.2, align 2
+  %arrayidx1.2 = getelementptr inbounds i16, i16* %b, i32 %inc.1
+  %9 = load i16, i16* %arrayidx1.2, align 2
+  %conv2.2 = sitofp i16 %9 to half
+  %mul.2 = fmul half %8, %conv2.2
+  %conv3.2 = fpext half %mul.2 to float
+  %add.2 = fadd float %add.1, %conv3.2
+  %inc.2 = or i32 %i.012, 3
+  %arrayidx.3 = getelementptr inbounds half, half* %a, i32 %inc.2
+  %10 = load half, half* %arrayidx.3, align 2
+  %arrayidx1.3 = getelementptr inbounds i16, i16* %b, i32 %inc.2
+  %11 = load i16, i16* %arrayidx1.3, align 2
+  %conv2.3 = sitofp i16 %11 to half
+  %mul.3 = fmul half %10, %conv2.3
+  %conv3.3 = fpext half %mul.3 to float
+  %add.3 = fadd float %add.2, %conv3.3
+  %inc.3 = add nuw i32 %i.012, 4
+  %niter.nsub.3 = add i32 %niter, -4
+  %niter.ncmp.3 = icmp eq i32 %niter.nsub.3, 0
+  br i1 %niter.ncmp.3, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body
+}
+
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mve-tail-data-types.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mve-tail-data-types.ll
new file mode 100644
index 000000000..f9c83b74b
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mve-tail-data-types.ll
@@ -0,0 +1,1230 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve -disable-mve-tail-predication=false %s -o - | FileCheck %s
+
+define arm_aapcs_vfpcc i32 @test_acc_scalar_char(i8 zeroext %a, i8* nocapture readonly %b, i32 %N) {
+; CHECK-LABEL: test_acc_scalar_char:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    itt eq
+; CHECK-NEXT:    moveq r0, #0
+; CHECK-NEXT:    bxeq lr
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    adds r3, r2, #3
+; CHECK-NEXT:    vmov.i32 q0, #0x0
+; CHECK-NEXT:    bic r3, r3, #3
+; CHECK-NEXT:    sub.w r12, r3, #4
+; CHECK-NEXT:    movs r3, #1
+; CHECK-NEXT:    add.w lr, r3, r12, lsr #2
+; CHECK-NEXT:    movs r3, #0
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB0_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    mov r12, r2
+; CHECK-NEXT:    adds r2, r1, r3
+; CHECK-NEXT:    vctp.32 r12
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrbt.u32 q2, [r2]
+; CHECK-NEXT:    adds r3, #4
+; CHECK-NEXT:    sub.w r2, r12, #4
+; CHECK-NEXT:    vmov q1, q0
+; CHECK-NEXT:    vmla.u32 q0, q2, r0
+; CHECK-NEXT:    le lr, .LBB0_1
+; CHECK-NEXT:  @ %bb.2: @ %middle.block
+; CHECK-NEXT:    vctp.32 r12
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    vaddv.u32 r0, q0
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %cmp7 = icmp eq i32 %N, 0
+  br i1 %cmp7, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %conv = zext i8 %a to i32
+  %n.rnd.up = add i32 %N, 3
+  %n.vec = and i32 %n.rnd.up, -4
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert10 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat11 = shufflevector <4 x i32> %broadcast.splatinsert10, <4 x i32> undef, <4 x i32> zeroinitializer
+  %broadcast.splatinsert12 = insertelement <4 x i32> undef, i32 %conv, i32 0
+  %broadcast.splat13 = shufflevector <4 x i32> %broadcast.splatinsert12, <4 x i32> undef, <4 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %vec.phi = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ %5, %vector.body ]
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+  %0 = getelementptr inbounds i8, i8* %b, i32 %index
+  %1 = icmp ule <4 x i32> %induction, %broadcast.splat11
+  %2 = bitcast i8* %0 to <4 x i8>*
+  %wide.masked.load = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %2, i32 1, <4 x i1> %1, <4 x i8> undef)
+  %3 = zext <4 x i8> %wide.masked.load to <4 x i32>
+  %4 = mul nuw nsw <4 x i32> %broadcast.splat13, %3
+  %5 = add nuw nsw <4 x i32> %4, %vec.phi
+  %index.next = add i32 %index, 4
+  %6 = icmp eq i32 %index.next, %n.vec
+  br i1 %6, label %middle.block, label %vector.body
+
+middle.block:                                     ; preds = %vector.body
+  %7 = select <4 x i1> %1, <4 x i32> %5, <4 x i32> %vec.phi
+  %8 = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %7)
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %middle.block, %entry
+  %res.0.lcssa = phi i32 [ 0, %entry ], [ %8, %middle.block ]
+  ret i32 %res.0.lcssa
+}
+
+define arm_aapcs_vfpcc i32 @test_acc_scalar_short(i16 signext %a, i16* nocapture readonly %b, i32 %N) {
+; CHECK-LABEL: test_acc_scalar_short:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    itt eq
+; CHECK-NEXT:    moveq r0, #0
+; CHECK-NEXT:    bxeq lr
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    vmov.i32 q0, #0x0
+; CHECK-NEXT:    dlstp.32 lr, r2
+; CHECK-NEXT:  .LBB1_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    mov r3, r2
+; CHECK-NEXT:    subs r2, #4
+; CHECK-NEXT:    vldrh.s32 q2, [r1], #8
+; CHECK-NEXT:    vmov q1, q0
+; CHECK-NEXT:    vmla.u32 q0, q2, r0
+; CHECK-NEXT:    letp lr, .LBB1_1
+; CHECK-NEXT:  @ %bb.2: @ %middle.block
+; CHECK-NEXT:    vctp.32 r3
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    vaddv.u32 r0, q0
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %cmp7 = icmp eq i32 %N, 0
+  br i1 %cmp7, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %conv = sext i16 %a to i32
+  %n.rnd.up = add i32 %N, 3
+  %n.vec = and i32 %n.rnd.up, -4
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert10 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat11 = shufflevector <4 x i32> %broadcast.splatinsert10, <4 x i32> undef, <4 x i32> zeroinitializer
+  %broadcast.splatinsert12 = insertelement <4 x i32> undef, i32 %conv, i32 0
+  %broadcast.splat13 = shufflevector <4 x i32> %broadcast.splatinsert12, <4 x i32> undef, <4 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %vec.phi = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ %5, %vector.body ]
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+  %0 = getelementptr inbounds i16, i16* %b, i32 %index
+  %1 = icmp ule <4 x i32> %induction, %broadcast.splat11
+  %2 = bitcast i16* %0 to <4 x i16>*
+  %wide.masked.load = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %2, i32 2, <4 x i1> %1, <4 x i16> undef)
+  %3 = sext <4 x i16> %wide.masked.load to <4 x i32>
+  %4 = mul nsw <4 x i32> %broadcast.splat13, %3
+  %5 = add nsw <4 x i32> %4, %vec.phi
+  %index.next = add i32 %index, 4
+  %6 = icmp eq i32 %index.next, %n.vec
+  br i1 %6, label %middle.block, label %vector.body
+
+middle.block:                                     ; preds = %vector.body
+  %7 = select <4 x i1> %1, <4 x i32> %5, <4 x i32> %vec.phi
+  %8 = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %7)
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %middle.block, %entry
+  %res.0.lcssa = phi i32 [ 0, %entry ], [ %8, %middle.block ]
+  ret i32 %res.0.lcssa
+}
+
+define arm_aapcs_vfpcc i32 @test_acc_scalar_uchar(i8 zeroext %a, i8* nocapture readonly %b, i32 %N) {
+; CHECK-LABEL: test_acc_scalar_uchar:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    itt eq
+; CHECK-NEXT:    moveq r0, #0
+; CHECK-NEXT:    bxeq lr
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    adds r3, r2, #3
+; CHECK-NEXT:    vmov.i32 q0, #0x0
+; CHECK-NEXT:    bic r3, r3, #3
+; CHECK-NEXT:    sub.w r12, r3, #4
+; CHECK-NEXT:    movs r3, #1
+; CHECK-NEXT:    add.w lr, r3, r12, lsr #2
+; CHECK-NEXT:    movs r3, #0
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB2_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    mov r12, r2
+; CHECK-NEXT:    adds r2, r1, r3
+; CHECK-NEXT:    vctp.32 r12
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrbt.u32 q2, [r2]
+; CHECK-NEXT:    adds r3, #4
+; CHECK-NEXT:    sub.w r2, r12, #4
+; CHECK-NEXT:    vmov q1, q0
+; CHECK-NEXT:    vmla.u32 q0, q2, r0
+; CHECK-NEXT:    le lr, .LBB2_1
+; CHECK-NEXT:  @ %bb.2: @ %middle.block
+; CHECK-NEXT:    vctp.32 r12
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    vaddv.u32 r0, q0
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %cmp7 = icmp eq i32 %N, 0
+  br i1 %cmp7, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %conv = zext i8 %a to i32
+  %n.rnd.up = add i32 %N, 3
+  %n.vec = and i32 %n.rnd.up, -4
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert10 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat11 = shufflevector <4 x i32> %broadcast.splatinsert10, <4 x i32> undef, <4 x i32> zeroinitializer
+  %broadcast.splatinsert12 = insertelement <4 x i32> undef, i32 %conv, i32 0
+  %broadcast.splat13 = shufflevector <4 x i32> %broadcast.splatinsert12, <4 x i32> undef, <4 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %vec.phi = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ %5, %vector.body ]
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+  %0 = getelementptr inbounds i8, i8* %b, i32 %index
+  %1 = icmp ule <4 x i32> %induction, %broadcast.splat11
+  %2 = bitcast i8* %0 to <4 x i8>*
+  %wide.masked.load = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %2, i32 1, <4 x i1> %1, <4 x i8> undef)
+  %3 = zext <4 x i8> %wide.masked.load to <4 x i32>
+  %4 = mul nuw nsw <4 x i32> %broadcast.splat13, %3
+  %5 = add nuw nsw <4 x i32> %4, %vec.phi
+  %index.next = add i32 %index, 4
+  %6 = icmp eq i32 %index.next, %n.vec
+  br i1 %6, label %middle.block, label %vector.body
+
+middle.block:                                     ; preds = %vector.body
+  %7 = select <4 x i1> %1, <4 x i32> %5, <4 x i32> %vec.phi
+  %8 = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %7)
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %middle.block, %entry
+  %res.0.lcssa = phi i32 [ 0, %entry ], [ %8, %middle.block ]
+  ret i32 %res.0.lcssa
+}
+
+define arm_aapcs_vfpcc i32 @test_acc_scalar_ushort(i16 signext %a, i16* nocapture readonly %b, i32 %N) {
+; CHECK-LABEL: test_acc_scalar_ushort:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    itt eq
+; CHECK-NEXT:    moveq r0, #0
+; CHECK-NEXT:    bxeq lr
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    vmov.i32 q0, #0x0
+; CHECK-NEXT:    dlstp.32 lr, r2
+; CHECK-NEXT:  .LBB3_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    mov r3, r2
+; CHECK-NEXT:    subs r2, #4
+; CHECK-NEXT:    vldrh.u32 q2, [r1], #8
+; CHECK-NEXT:    vmov q1, q0
+; CHECK-NEXT:    vmla.u32 q0, q2, r0
+; CHECK-NEXT:    letp lr, .LBB3_1
+; CHECK-NEXT:  @ %bb.2: @ %middle.block
+; CHECK-NEXT:    vctp.32 r3
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    vaddv.u32 r0, q0
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %cmp7 = icmp eq i32 %N, 0
+  br i1 %cmp7, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %conv = sext i16 %a to i32
+  %n.rnd.up = add i32 %N, 3
+  %n.vec = and i32 %n.rnd.up, -4
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert10 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat11 = shufflevector <4 x i32> %broadcast.splatinsert10, <4 x i32> undef, <4 x i32> zeroinitializer
+  %broadcast.splatinsert12 = insertelement <4 x i32> undef, i32 %conv, i32 0
+  %broadcast.splat13 = shufflevector <4 x i32> %broadcast.splatinsert12, <4 x i32> undef, <4 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %vec.phi = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ %5, %vector.body ]
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+  %0 = getelementptr inbounds i16, i16* %b, i32 %index
+  %1 = icmp ule <4 x i32> %induction, %broadcast.splat11
+  %2 = bitcast i16* %0 to <4 x i16>*
+  %wide.masked.load = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %2, i32 2, <4 x i1> %1, <4 x i16> undef)
+  %3 = zext <4 x i16> %wide.masked.load to <4 x i32>
+  %4 = mul nsw <4 x i32> %broadcast.splat13, %3
+  %5 = add nsw <4 x i32> %4, %vec.phi
+  %index.next = add i32 %index, 4
+  %6 = icmp eq i32 %index.next, %n.vec
+  br i1 %6, label %middle.block, label %vector.body
+
+middle.block:                                     ; preds = %vector.body
+  %7 = select <4 x i1> %1, <4 x i32> %5, <4 x i32> %vec.phi
+  %8 = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %7)
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %middle.block, %entry
+  %res.0.lcssa = phi i32 [ 0, %entry ], [ %8, %middle.block ]
+  ret i32 %res.0.lcssa
+}
+
+define arm_aapcs_vfpcc i32 @test_acc_scalar_int(i32 %a, i32* nocapture readonly %b, i32 %N) {
+; CHECK-LABEL: test_acc_scalar_int:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    itt eq
+; CHECK-NEXT:    moveq r0, #0
+; CHECK-NEXT:    bxeq lr
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    vmov.i32 q0, #0x0
+; CHECK-NEXT:    dlstp.32 lr, r2
+; CHECK-NEXT:  .LBB4_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    mov r3, r2
+; CHECK-NEXT:    subs r2, #4
+; CHECK-NEXT:    vldrw.u32 q2, [r1], #16
+; CHECK-NEXT:    vmov q1, q0
+; CHECK-NEXT:    vmla.u32 q0, q2, r0
+; CHECK-NEXT:    letp lr, .LBB4_1
+; CHECK-NEXT:  @ %bb.2: @ %middle.block
+; CHECK-NEXT:    vctp.32 r3
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    vaddv.u32 r0, q0
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %cmp6 = icmp eq i32 %N, 0
+  br i1 %cmp6, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %n.rnd.up = add i32 %N, 3
+  %n.vec = and i32 %n.rnd.up, -4
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert9 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat10 = shufflevector <4 x i32> %broadcast.splatinsert9, <4 x i32> undef, <4 x i32> zeroinitializer
+  %broadcast.splatinsert11 = insertelement <4 x i32> undef, i32 %a, i32 0
+  %broadcast.splat12 = shufflevector <4 x i32> %broadcast.splatinsert11, <4 x i32> undef, <4 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %vec.phi = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ %4, %vector.body ]
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+  %0 = getelementptr inbounds i32, i32* %b, i32 %index
+  %1 = icmp ule <4 x i32> %induction, %broadcast.splat10
+  %2 = bitcast i32* %0 to <4 x i32>*
+  %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %2, i32 4, <4 x i1> %1, <4 x i32> undef)
+  %3 = mul nsw <4 x i32> %wide.masked.load, %broadcast.splat12
+  %4 = add nsw <4 x i32> %3, %vec.phi
+  %index.next = add i32 %index, 4
+  %5 = icmp eq i32 %index.next, %n.vec
+  br i1 %5, label %middle.block, label %vector.body
+
+middle.block:                                     ; preds = %vector.body
+  %6 = select <4 x i1> %1, <4 x i32> %4, <4 x i32> %vec.phi
+  %7 = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %6)
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %middle.block, %entry
+  %res.0.lcssa = phi i32 [ 0, %entry ], [ %7, %middle.block ]
+  ret i32 %res.0.lcssa
+}
+
+define arm_aapcs_vfpcc void @test_vec_mul_scalar_add_char(i8* nocapture readonly %a, i8* nocapture readonly %b, i8 zeroext %c, i32* nocapture %res, i32 %N) {
+; CHECK-LABEL: test_vec_mul_scalar_add_char:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, r9, lr}
+; CHECK-NEXT:    ldr.w r12, [sp, #28]
+; CHECK-NEXT:    cmp.w r12, #0
+; CHECK-NEXT:    beq.w .LBB5_12
+; CHECK-NEXT:  @ %bb.1: @ %for.body.lr.ph
+; CHECK-NEXT:    add.w r5, r3, r12, lsl #2
+; CHECK-NEXT:    add.w r6, r1, r12
+; CHECK-NEXT:    cmp r5, r1
+; CHECK-NEXT:    add.w r4, r0, r12
+; CHECK-NEXT:    cset r7, hi
+; CHECK-NEXT:    cmp r6, r3
+; CHECK-NEXT:    cset r6, hi
+; CHECK-NEXT:    cmp r5, r0
+; CHECK-NEXT:    cset r5, hi
+; CHECK-NEXT:    cmp r4, r3
+; CHECK-NEXT:    cset r4, hi
+; CHECK-NEXT:    ands r5, r4
+; CHECK-NEXT:    lsls r5, r5, #31
+; CHECK-NEXT:    itt eq
+; CHECK-NEXT:    andeq r7, r6
+; CHECK-NEXT:    lslseq.w r7, r7, #31
+; CHECK-NEXT:    beq .LBB5_4
+; CHECK-NEXT:  @ %bb.2: @ %for.body.preheader
+; CHECK-NEXT:    sub.w r4, r12, #1
+; CHECK-NEXT:    and lr, r12, #3
+; CHECK-NEXT:    cmp r4, #3
+; CHECK-NEXT:    bhs .LBB5_6
+; CHECK-NEXT:  @ %bb.3:
+; CHECK-NEXT:    movs r7, #0
+; CHECK-NEXT:    b .LBB5_9
+; CHECK-NEXT:  .LBB5_4: @ %vector.ph
+; CHECK-NEXT:    movs r4, #0
+; CHECK-NEXT:    dlstp.32 lr, r12
+; CHECK-NEXT:  .LBB5_5: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    adds r5, r0, r4
+; CHECK-NEXT:    vldrb.u32 q0, [r5]
+; CHECK-NEXT:    adds r5, r1, r4
+; CHECK-NEXT:    vldrb.u32 q1, [r5]
+; CHECK-NEXT:    vmul.i32 q0, q1, q0
+; CHECK-NEXT:    adds r4, #4
+; CHECK-NEXT:    sub.w r12, r12, #4
+; CHECK-NEXT:    vadd.i32 q0, q0, r2
+; CHECK-NEXT:    vstrw.32 q0, [r3], #16
+; CHECK-NEXT:    letp lr, .LBB5_5
+; CHECK-NEXT:    b .LBB5_12
+; CHECK-NEXT:  .LBB5_6: @ %for.body.preheader.new
+; CHECK-NEXT:    sub.w r12, lr, r12
+; CHECK-NEXT:    subs r4, r1, #3
+; CHECK-NEXT:    subs r5, r0, #3
+; CHECK-NEXT:    sub.w r7, r3, #16
+; CHECK-NEXT:    mov.w r9, #0
+; CHECK-NEXT:  .LBB5_7: @ %for.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    ldrb.w r8, [r5, #3]
+; CHECK-NEXT:    sub.w r9, r9, #4
+; CHECK-NEXT:    ldrb r6, [r4, #3]
+; CHECK-NEXT:    cmp r12, r9
+; CHECK-NEXT:    smlabb r6, r6, r8, r2
+; CHECK-NEXT:    str r6, [r7, #16]!
+; CHECK-NEXT:    ldrb r8, [r5, #4]!
+; CHECK-NEXT:    ldrb r6, [r4, #4]!
+; CHECK-NEXT:    smlabb r6, r6, r8, r2
+; CHECK-NEXT:    str r6, [r7, #4]
+; CHECK-NEXT:    ldrb.w r8, [r5, #1]
+; CHECK-NEXT:    ldrb r6, [r4, #1]
+; CHECK-NEXT:    smlabb r6, r6, r8, r2
+; CHECK-NEXT:    str r6, [r7, #8]
+; CHECK-NEXT:    ldrb.w r8, [r5, #2]
+; CHECK-NEXT:    ldrb r6, [r4, #2]
+; CHECK-NEXT:    smlabb r6, r6, r8, r2
+; CHECK-NEXT:    str r6, [r7, #12]
+; CHECK-NEXT:    bne .LBB5_7
+; CHECK-NEXT:  @ %bb.8: @ %for.cond.cleanup.loopexit.unr-lcssa.loopexit
+; CHECK-NEXT:    rsb.w r7, r9, #0
+; CHECK-NEXT:  .LBB5_9: @ %for.cond.cleanup.loopexit.unr-lcssa
+; CHECK-NEXT:    wls lr, lr, .LBB5_12
+; CHECK-NEXT:  @ %bb.10: @ %for.body.epil.preheader
+; CHECK-NEXT:    subs r7, #1
+; CHECK-NEXT:    add r0, r7
+; CHECK-NEXT:    add r1, r7
+; CHECK-NEXT:    add.w r3, r3, r7, lsl #2
+; CHECK-NEXT:  .LBB5_11: @ %for.body.epil
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    ldrb r7, [r0, #1]!
+; CHECK-NEXT:    ldrb r6, [r1, #1]!
+; CHECK-NEXT:    smlabb r7, r6, r7, r2
+; CHECK-NEXT:    str r7, [r3, #4]!
+; CHECK-NEXT:    le lr, .LBB5_11
+; CHECK-NEXT:  .LBB5_12: @ %for.cond.cleanup
+; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, pc}
+entry:
+  %res12 = bitcast i32* %res to i8*
+  %cmp10 = icmp eq i32 %N, 0
+  br i1 %cmp10, label %for.cond.cleanup, label %for.body.lr.ph
+
+for.body.lr.ph:                                   ; preds = %entry
+  %conv3 = zext i8 %c to i32
+  %scevgep = getelementptr i32, i32* %res, i32 %N
+  %scevgep13 = bitcast i32* %scevgep to i8*
+  %scevgep14 = getelementptr i8, i8* %a, i32 %N
+  %scevgep15 = getelementptr i8, i8* %b, i32 %N
+  %bound0 = icmp ugt i8* %scevgep14, %res12
+  %bound1 = icmp ugt i8* %scevgep13, %a
+  %found.conflict = and i1 %bound0, %bound1
+  %bound016 = icmp ugt i8* %scevgep15, %res12
+  %bound117 = icmp ugt i8* %scevgep13, %b
+  %found.conflict18 = and i1 %bound016, %bound117
+  %conflict.rdx = or i1 %found.conflict, %found.conflict18
+  br i1 %conflict.rdx, label %for.body.preheader, label %vector.ph
+
+for.body.preheader:                               ; preds = %for.body.lr.ph
+  %0 = add i32 %N, -1
+  %xtraiter = and i32 %N, 3
+  %1 = icmp ult i32 %0, 3
+  br i1 %1, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body.preheader.new
+
+for.body.preheader.new:                           ; preds = %for.body.preheader
+  %unroll_iter = sub i32 %N, %xtraiter
+  br label %for.body
+
+vector.ph:                                        ; preds = %for.body.lr.ph
+  %n.rnd.up = add i32 %N, 3
+  %n.vec = and i32 %n.rnd.up, -4
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert19 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat20 = shufflevector <4 x i32> %broadcast.splatinsert19, <4 x i32> undef, <4 x i32> zeroinitializer
+  %broadcast.splatinsert22 = insertelement <4 x i32> undef, i32 %conv3, i32 0
+  %broadcast.splat23 = shufflevector <4 x i32> %broadcast.splatinsert22, <4 x i32> undef, <4 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+  %2 = getelementptr inbounds i8, i8* %a, i32 %index
+  %3 = icmp ule <4 x i32> %induction, %broadcast.splat20
+  %4 = bitcast i8* %2 to <4 x i8>*
+  %wide.masked.load = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %4, i32 1, <4 x i1> %3, <4 x i8> undef)
+  %5 = zext <4 x i8> %wide.masked.load to <4 x i32>
+  %6 = getelementptr inbounds i8, i8* %b, i32 %index
+  %7 = bitcast i8* %6 to <4 x i8>*
+  %wide.masked.load21 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %7, i32 1, <4 x i1> %3, <4 x i8> undef)
+  %8 = zext <4 x i8> %wide.masked.load21 to <4 x i32>
+  %9 = mul nuw nsw <4 x i32> %8, %5
+  %10 = add nuw nsw <4 x i32> %9, %broadcast.splat23
+  %11 = getelementptr inbounds i32, i32* %res, i32 %index
+  %12 = bitcast i32* %11 to <4 x i32>*
+  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %10, <4 x i32>* %12, i32 4, <4 x i1> %3)
+  %index.next = add i32 %index, 4
+  %13 = icmp eq i32 %index.next, %n.vec
+  br i1 %13, label %for.cond.cleanup, label %vector.body
+
+for.cond.cleanup.loopexit.unr-lcssa:              ; preds = %for.body, %for.body.preheader
+  %i.011.unr = phi i32 [ 0, %for.body.preheader ], [ %inc.3, %for.body ]
+  %lcmp.mod = icmp eq i32 %xtraiter, 0
+  br i1 %lcmp.mod, label %for.cond.cleanup, label %for.body.epil
+
+for.body.epil:                                    ; preds = %for.cond.cleanup.loopexit.unr-lcssa, %for.body.epil
+  %i.011.epil = phi i32 [ %inc.epil, %for.body.epil ], [ %i.011.unr, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %epil.iter = phi i32 [ %epil.iter.sub, %for.body.epil ], [ %xtraiter, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %arrayidx.epil = getelementptr inbounds i8, i8* %a, i32 %i.011.epil
+  %14 = load i8, i8* %arrayidx.epil, align 1
+  %conv.epil = zext i8 %14 to i32
+  %arrayidx1.epil = getelementptr inbounds i8, i8* %b, i32 %i.011.epil
+  %15 = load i8, i8* %arrayidx1.epil, align 1
+  %conv2.epil = zext i8 %15 to i32
+  %mul.epil = mul nuw nsw i32 %conv2.epil, %conv.epil
+  %add.epil = add nuw nsw i32 %mul.epil, %conv3
+  %arrayidx4.epil = getelementptr inbounds i32, i32* %res, i32 %i.011.epil
+  store i32 %add.epil, i32* %arrayidx4.epil, align 4
+  %inc.epil = add nuw i32 %i.011.epil, 1
+  %epil.iter.sub = add i32 %epil.iter, -1
+  %epil.iter.cmp = icmp eq i32 %epil.iter.sub, 0
+  br i1 %epil.iter.cmp, label %for.cond.cleanup, label %for.body.epil
+
+for.cond.cleanup:                                 ; preds = %vector.body, %for.cond.cleanup.loopexit.unr-lcssa, %for.body.epil, %entry
+  ret void
+
+for.body:                                         ; preds = %for.body, %for.body.preheader.new
+  %i.011 = phi i32 [ 0, %for.body.preheader.new ], [ %inc.3, %for.body ]
+  %niter = phi i32 [ %unroll_iter, %for.body.preheader.new ], [ %niter.nsub.3, %for.body ]
+  %arrayidx = getelementptr inbounds i8, i8* %a, i32 %i.011
+  %16 = load i8, i8* %arrayidx, align 1
+  %conv = zext i8 %16 to i32
+  %arrayidx1 = getelementptr inbounds i8, i8* %b, i32 %i.011
+  %17 = load i8, i8* %arrayidx1, align 1
+  %conv2 = zext i8 %17 to i32
+  %mul = mul nuw nsw i32 %conv2, %conv
+  %add = add nuw nsw i32 %mul, %conv3
+  %arrayidx4 = getelementptr inbounds i32, i32* %res, i32 %i.011
+  store i32 %add, i32* %arrayidx4, align 4
+  %inc = or i32 %i.011, 1
+  %arrayidx.1 = getelementptr inbounds i8, i8* %a, i32 %inc
+  %18 = load i8, i8* %arrayidx.1, align 1
+  %conv.1 = zext i8 %18 to i32
+  %arrayidx1.1 = getelementptr inbounds i8, i8* %b, i32 %inc
+  %19 = load i8, i8* %arrayidx1.1, align 1
+  %conv2.1 = zext i8 %19 to i32
+  %mul.1 = mul nuw nsw i32 %conv2.1, %conv.1
+  %add.1 = add nuw nsw i32 %mul.1, %conv3
+  %arrayidx4.1 = getelementptr inbounds i32, i32* %res, i32 %inc
+  store i32 %add.1, i32* %arrayidx4.1, align 4
+  %inc.1 = or i32 %i.011, 2
+  %arrayidx.2 = getelementptr inbounds i8, i8* %a, i32 %inc.1
+  %20 = load i8, i8* %arrayidx.2, align 1
+  %conv.2 = zext i8 %20 to i32
+  %arrayidx1.2 = getelementptr inbounds i8, i8* %b, i32 %inc.1
+  %21 = load i8, i8* %arrayidx1.2, align 1
+  %conv2.2 = zext i8 %21 to i32
+  %mul.2 = mul nuw nsw i32 %conv2.2, %conv.2
+  %add.2 = add nuw nsw i32 %mul.2, %conv3
+  %arrayidx4.2 = getelementptr inbounds i32, i32* %res, i32 %inc.1
+  store i32 %add.2, i32* %arrayidx4.2, align 4
+  %inc.2 = or i32 %i.011, 3
+  %arrayidx.3 = getelementptr inbounds i8, i8* %a, i32 %inc.2
+  %22 = load i8, i8* %arrayidx.3, align 1
+  %conv.3 = zext i8 %22 to i32
+  %arrayidx1.3 = getelementptr inbounds i8, i8* %b, i32 %inc.2
+  %23 = load i8, i8* %arrayidx1.3, align 1
+  %conv2.3 = zext i8 %23 to i32
+  %mul.3 = mul nuw nsw i32 %conv2.3, %conv.3
+  %add.3 = add nuw nsw i32 %mul.3, %conv3
+  %arrayidx4.3 = getelementptr inbounds i32, i32* %res, i32 %inc.2
+  store i32 %add.3, i32* %arrayidx4.3, align 4
+  %inc.3 = add nuw i32 %i.011, 4
+  %niter.nsub.3 = add i32 %niter, -4
+  %niter.ncmp.3 = icmp eq i32 %niter.nsub.3, 0
+  br i1 %niter.ncmp.3, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body
+}
+
+define arm_aapcs_vfpcc void @test_vec_mul_scalar_add_short(i16* nocapture readonly %a, i16* nocapture readonly %b, i16 signext %c, i32* nocapture %res, i32 %N) {
+; CHECK-LABEL: test_vec_mul_scalar_add_short:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    push {r4, lr}
+; CHECK-NEXT:    ldr.w r12, [sp, #8]
+; CHECK-NEXT:    cmp.w r12, #0
+; CHECK-NEXT:    it eq
+; CHECK-NEXT:    popeq {r4, pc}
+; CHECK-NEXT:    dlstp.32 lr, r12
+; CHECK-NEXT:  .LBB6_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrh.s32 q0, [r0], #8
+; CHECK-NEXT:    vldrh.s32 q1, [r1], #8
+; CHECK-NEXT:    vmul.i32 q0, q1, q0
+; CHECK-NEXT:    sub.w r12, r12, #4
+; CHECK-NEXT:    vadd.i32 q0, q0, r2
+; CHECK-NEXT:    vstrw.32 q0, [r3], #16
+; CHECK-NEXT:    letp lr, .LBB6_1
+; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
+; CHECK-NEXT:    pop {r4, pc}
+entry:
+  %cmp10 = icmp eq i32 %N, 0
+  br i1 %cmp10, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %conv3 = sext i16 %c to i32
+  %n.rnd.up = add i32 %N, 3
+  %n.vec = and i32 %n.rnd.up, -4
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert12 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat13 = shufflevector <4 x i32> %broadcast.splatinsert12, <4 x i32> undef, <4 x i32> zeroinitializer
+  %broadcast.splatinsert15 = insertelement <4 x i32> undef, i32 %conv3, i32 0
+  %broadcast.splat16 = shufflevector <4 x i32> %broadcast.splatinsert15, <4 x i32> undef, <4 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+  %0 = getelementptr inbounds i16, i16* %a, i32 %index
+  %1 = icmp ule <4 x i32> %induction, %broadcast.splat13
+  %2 = bitcast i16* %0 to <4 x i16>*
+  %wide.masked.load = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %2, i32 2, <4 x i1> %1, <4 x i16> undef)
+  %3 = sext <4 x i16> %wide.masked.load to <4 x i32>
+  %4 = getelementptr inbounds i16, i16* %b, i32 %index
+  %5 = bitcast i16* %4 to <4 x i16>*
+  %wide.masked.load14 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %5, i32 2, <4 x i1> %1, <4 x i16> undef)
+  %6 = sext <4 x i16> %wide.masked.load14 to <4 x i32>
+  %7 = mul nsw <4 x i32> %6, %3
+  %8 = add nsw <4 x i32> %7, %broadcast.splat16
+  %9 = getelementptr inbounds i32, i32* %res, i32 %index
+  %10 = bitcast i32* %9 to <4 x i32>*
+  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %8, <4 x i32>* %10, i32 4, <4 x i1> %1)
+  %index.next = add i32 %index, 4
+  %11 = icmp eq i32 %index.next, %n.vec
+  br i1 %11, label %for.cond.cleanup, label %vector.body
+
+for.cond.cleanup:                                 ; preds = %vector.body, %entry
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vec_mul_scalar_add_uchar(i8* nocapture readonly %a, i8* nocapture readonly %b, i8 zeroext %c, i32* nocapture %res, i32 %N) {
+; CHECK-LABEL: test_vec_mul_scalar_add_uchar:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, r9, lr}
+; CHECK-NEXT:    ldr.w r12, [sp, #28]
+; CHECK-NEXT:    cmp.w r12, #0
+; CHECK-NEXT:    beq.w .LBB7_12
+; CHECK-NEXT:  @ %bb.1: @ %for.body.lr.ph
+; CHECK-NEXT:    add.w r5, r3, r12, lsl #2
+; CHECK-NEXT:    add.w r6, r1, r12
+; CHECK-NEXT:    cmp r5, r1
+; CHECK-NEXT:    add.w r4, r0, r12
+; CHECK-NEXT:    cset r7, hi
+; CHECK-NEXT:    cmp r6, r3
+; CHECK-NEXT:    cset r6, hi
+; CHECK-NEXT:    cmp r5, r0
+; CHECK-NEXT:    cset r5, hi
+; CHECK-NEXT:    cmp r4, r3
+; CHECK-NEXT:    cset r4, hi
+; CHECK-NEXT:    ands r5, r4
+; CHECK-NEXT:    lsls r5, r5, #31
+; CHECK-NEXT:    itt eq
+; CHECK-NEXT:    andeq r7, r6
+; CHECK-NEXT:    lslseq.w r7, r7, #31
+; CHECK-NEXT:    beq .LBB7_4
+; CHECK-NEXT:  @ %bb.2: @ %for.body.preheader
+; CHECK-NEXT:    sub.w r4, r12, #1
+; CHECK-NEXT:    and lr, r12, #3
+; CHECK-NEXT:    cmp r4, #3
+; CHECK-NEXT:    bhs .LBB7_6
+; CHECK-NEXT:  @ %bb.3:
+; CHECK-NEXT:    movs r7, #0
+; CHECK-NEXT:    b .LBB7_9
+; CHECK-NEXT:  .LBB7_4: @ %vector.ph
+; CHECK-NEXT:    movs r4, #0
+; CHECK-NEXT:    dlstp.32 lr, r12
+; CHECK-NEXT:  .LBB7_5: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    adds r5, r0, r4
+; CHECK-NEXT:    vldrb.u32 q0, [r5]
+; CHECK-NEXT:    adds r5, r1, r4
+; CHECK-NEXT:    vldrb.u32 q1, [r5]
+; CHECK-NEXT:    vmul.i32 q0, q1, q0
+; CHECK-NEXT:    adds r4, #4
+; CHECK-NEXT:    sub.w r12, r12, #4
+; CHECK-NEXT:    vadd.i32 q0, q0, r2
+; CHECK-NEXT:    vstrw.32 q0, [r3], #16
+; CHECK-NEXT:    letp lr, .LBB7_5
+; CHECK-NEXT:    b .LBB7_12
+; CHECK-NEXT:  .LBB7_6: @ %for.body.preheader.new
+; CHECK-NEXT:    sub.w r12, lr, r12
+; CHECK-NEXT:    subs r4, r1, #3
+; CHECK-NEXT:    subs r5, r0, #3
+; CHECK-NEXT:    sub.w r7, r3, #16
+; CHECK-NEXT:    mov.w r9, #0
+; CHECK-NEXT:  .LBB7_7: @ %for.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    ldrb.w r8, [r5, #3]
+; CHECK-NEXT:    sub.w r9, r9, #4
+; CHECK-NEXT:    ldrb r6, [r4, #3]
+; CHECK-NEXT:    cmp r12, r9
+; CHECK-NEXT:    smlabb r6, r6, r8, r2
+; CHECK-NEXT:    str r6, [r7, #16]!
+; CHECK-NEXT:    ldrb r8, [r5, #4]!
+; CHECK-NEXT:    ldrb r6, [r4, #4]!
+; CHECK-NEXT:    smlabb r6, r6, r8, r2
+; CHECK-NEXT:    str r6, [r7, #4]
+; CHECK-NEXT:    ldrb.w r8, [r5, #1]
+; CHECK-NEXT:    ldrb r6, [r4, #1]
+; CHECK-NEXT:    smlabb r6, r6, r8, r2
+; CHECK-NEXT:    str r6, [r7, #8]
+; CHECK-NEXT:    ldrb.w r8, [r5, #2]
+; CHECK-NEXT:    ldrb r6, [r4, #2]
+; CHECK-NEXT:    smlabb r6, r6, r8, r2
+; CHECK-NEXT:    str r6, [r7, #12]
+; CHECK-NEXT:    bne .LBB7_7
+; CHECK-NEXT:  @ %bb.8: @ %for.cond.cleanup.loopexit.unr-lcssa.loopexit
+; CHECK-NEXT:    rsb.w r7, r9, #0
+; CHECK-NEXT:  .LBB7_9: @ %for.cond.cleanup.loopexit.unr-lcssa
+; CHECK-NEXT:    wls lr, lr, .LBB7_12
+; CHECK-NEXT:  @ %bb.10: @ %for.body.epil.preheader
+; CHECK-NEXT:    subs r7, #1
+; CHECK-NEXT:    add r0, r7
+; CHECK-NEXT:    add r1, r7
+; CHECK-NEXT:    add.w r3, r3, r7, lsl #2
+; CHECK-NEXT:  .LBB7_11: @ %for.body.epil
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    ldrb r7, [r0, #1]!
+; CHECK-NEXT:    ldrb r6, [r1, #1]!
+; CHECK-NEXT:    smlabb r7, r6, r7, r2
+; CHECK-NEXT:    str r7, [r3, #4]!
+; CHECK-NEXT:    le lr, .LBB7_11
+; CHECK-NEXT:  .LBB7_12: @ %for.cond.cleanup
+; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, pc}
+entry:
+  %res12 = bitcast i32* %res to i8*
+  %cmp10 = icmp eq i32 %N, 0
+  br i1 %cmp10, label %for.cond.cleanup, label %for.body.lr.ph
+
+for.body.lr.ph:                                   ; preds = %entry
+  %conv3 = zext i8 %c to i32
+  %scevgep = getelementptr i32, i32* %res, i32 %N
+  %scevgep13 = bitcast i32* %scevgep to i8*
+  %scevgep14 = getelementptr i8, i8* %a, i32 %N
+  %scevgep15 = getelementptr i8, i8* %b, i32 %N
+  %bound0 = icmp ugt i8* %scevgep14, %res12
+  %bound1 = icmp ugt i8* %scevgep13, %a
+  %found.conflict = and i1 %bound0, %bound1
+  %bound016 = icmp ugt i8* %scevgep15, %res12
+  %bound117 = icmp ugt i8* %scevgep13, %b
+  %found.conflict18 = and i1 %bound016, %bound117
+  %conflict.rdx = or i1 %found.conflict, %found.conflict18
+  br i1 %conflict.rdx, label %for.body.preheader, label %vector.ph
+
+for.body.preheader:                               ; preds = %for.body.lr.ph
+  %0 = add i32 %N, -1
+  %xtraiter = and i32 %N, 3
+  %1 = icmp ult i32 %0, 3
+  br i1 %1, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body.preheader.new
+
+for.body.preheader.new:                           ; preds = %for.body.preheader
+  %unroll_iter = sub i32 %N, %xtraiter
+  br label %for.body
+
+vector.ph:                                        ; preds = %for.body.lr.ph
+  %n.rnd.up = add i32 %N, 3
+  %n.vec = and i32 %n.rnd.up, -4
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert19 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat20 = shufflevector <4 x i32> %broadcast.splatinsert19, <4 x i32> undef, <4 x i32> zeroinitializer
+  %broadcast.splatinsert22 = insertelement <4 x i32> undef, i32 %conv3, i32 0
+  %broadcast.splat23 = shufflevector <4 x i32> %broadcast.splatinsert22, <4 x i32> undef, <4 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+  %2 = getelementptr inbounds i8, i8* %a, i32 %index
+  %3 = icmp ule <4 x i32> %induction, %broadcast.splat20
+  %4 = bitcast i8* %2 to <4 x i8>*
+  %wide.masked.load = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %4, i32 1, <4 x i1> %3, <4 x i8> undef)
+  %5 = zext <4 x i8> %wide.masked.load to <4 x i32>
+  %6 = getelementptr inbounds i8, i8* %b, i32 %index
+  %7 = bitcast i8* %6 to <4 x i8>*
+  %wide.masked.load21 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %7, i32 1, <4 x i1> %3, <4 x i8> undef)
+  %8 = zext <4 x i8> %wide.masked.load21 to <4 x i32>
+  %9 = mul nuw nsw <4 x i32> %8, %5
+  %10 = add nuw nsw <4 x i32> %9, %broadcast.splat23
+  %11 = getelementptr inbounds i32, i32* %res, i32 %index
+  %12 = bitcast i32* %11 to <4 x i32>*
+  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %10, <4 x i32>* %12, i32 4, <4 x i1> %3)
+  %index.next = add i32 %index, 4
+  %13 = icmp eq i32 %index.next, %n.vec
+  br i1 %13, label %for.cond.cleanup, label %vector.body
+
+for.cond.cleanup.loopexit.unr-lcssa:              ; preds = %for.body, %for.body.preheader
+  %i.011.unr = phi i32 [ 0, %for.body.preheader ], [ %inc.3, %for.body ]
+  %lcmp.mod = icmp eq i32 %xtraiter, 0
+  br i1 %lcmp.mod, label %for.cond.cleanup, label %for.body.epil
+
+for.body.epil:                                    ; preds = %for.cond.cleanup.loopexit.unr-lcssa, %for.body.epil
+  %i.011.epil = phi i32 [ %inc.epil, %for.body.epil ], [ %i.011.unr, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %epil.iter = phi i32 [ %epil.iter.sub, %for.body.epil ], [ %xtraiter, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %arrayidx.epil = getelementptr inbounds i8, i8* %a, i32 %i.011.epil
+  %14 = load i8, i8* %arrayidx.epil, align 1
+  %conv.epil = zext i8 %14 to i32
+  %arrayidx1.epil = getelementptr inbounds i8, i8* %b, i32 %i.011.epil
+  %15 = load i8, i8* %arrayidx1.epil, align 1
+  %conv2.epil = zext i8 %15 to i32
+  %mul.epil = mul nuw nsw i32 %conv2.epil, %conv.epil
+  %add.epil = add nuw nsw i32 %mul.epil, %conv3
+  %arrayidx4.epil = getelementptr inbounds i32, i32* %res, i32 %i.011.epil
+  store i32 %add.epil, i32* %arrayidx4.epil, align 4
+  %inc.epil = add nuw i32 %i.011.epil, 1
+  %epil.iter.sub = add i32 %epil.iter, -1
+  %epil.iter.cmp = icmp eq i32 %epil.iter.sub, 0
+  br i1 %epil.iter.cmp, label %for.cond.cleanup, label %for.body.epil
+
+for.cond.cleanup:                                 ; preds = %vector.body, %for.cond.cleanup.loopexit.unr-lcssa, %for.body.epil, %entry
+  ret void
+
+for.body:                                         ; preds = %for.body, %for.body.preheader.new
+  %i.011 = phi i32 [ 0, %for.body.preheader.new ], [ %inc.3, %for.body ]
+  %niter = phi i32 [ %unroll_iter, %for.body.preheader.new ], [ %niter.nsub.3, %for.body ]
+  %arrayidx = getelementptr inbounds i8, i8* %a, i32 %i.011
+  %16 = load i8, i8* %arrayidx, align 1
+  %conv = zext i8 %16 to i32
+  %arrayidx1 = getelementptr inbounds i8, i8* %b, i32 %i.011
+  %17 = load i8, i8* %arrayidx1, align 1
+  %conv2 = zext i8 %17 to i32
+  %mul = mul nuw nsw i32 %conv2, %conv
+  %add = add nuw nsw i32 %mul, %conv3
+  %arrayidx4 = getelementptr inbounds i32, i32* %res, i32 %i.011
+  store i32 %add, i32* %arrayidx4, align 4
+  %inc = or i32 %i.011, 1
+  %arrayidx.1 = getelementptr inbounds i8, i8* %a, i32 %inc
+  %18 = load i8, i8* %arrayidx.1, align 1
+  %conv.1 = zext i8 %18 to i32
+  %arrayidx1.1 = getelementptr inbounds i8, i8* %b, i32 %inc
+  %19 = load i8, i8* %arrayidx1.1, align 1
+  %conv2.1 = zext i8 %19 to i32
+  %mul.1 = mul nuw nsw i32 %conv2.1, %conv.1
+  %add.1 = add nuw nsw i32 %mul.1, %conv3
+  %arrayidx4.1 = getelementptr inbounds i32, i32* %res, i32 %inc
+  store i32 %add.1, i32* %arrayidx4.1, align 4
+  %inc.1 = or i32 %i.011, 2
+  %arrayidx.2 = getelementptr inbounds i8, i8* %a, i32 %inc.1
+  %20 = load i8, i8* %arrayidx.2, align 1
+  %conv.2 = zext i8 %20 to i32
+  %arrayidx1.2 = getelementptr inbounds i8, i8* %b, i32 %inc.1
+  %21 = load i8, i8* %arrayidx1.2, align 1
+  %conv2.2 = zext i8 %21 to i32
+  %mul.2 = mul nuw nsw i32 %conv2.2, %conv.2
+  %add.2 = add nuw nsw i32 %mul.2, %conv3
+  %arrayidx4.2 = getelementptr inbounds i32, i32* %res, i32 %inc.1
+  store i32 %add.2, i32* %arrayidx4.2, align 4
+  %inc.2 = or i32 %i.011, 3
+  %arrayidx.3 = getelementptr inbounds i8, i8* %a, i32 %inc.2
+  %22 = load i8, i8* %arrayidx.3, align 1
+  %conv.3 = zext i8 %22 to i32
+  %arrayidx1.3 = getelementptr inbounds i8, i8* %b, i32 %inc.2
+  %23 = load i8, i8* %arrayidx1.3, align 1
+  %conv2.3 = zext i8 %23 to i32
+  %mul.3 = mul nuw nsw i32 %conv2.3, %conv.3
+  %add.3 = add nuw nsw i32 %mul.3, %conv3
+  %arrayidx4.3 = getelementptr inbounds i32, i32* %res, i32 %inc.2
+  store i32 %add.3, i32* %arrayidx4.3, align 4
+  %inc.3 = add nuw i32 %i.011, 4
+  %niter.nsub.3 = add i32 %niter, -4
+  %niter.ncmp.3 = icmp eq i32 %niter.nsub.3, 0
+  br i1 %niter.ncmp.3, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body
+}
+
+define arm_aapcs_vfpcc void @test_vec_mul_scalar_add_ushort(i16* nocapture readonly %a, i16* nocapture readonly %b, i16 signext %c, i32* nocapture %res, i32 %N) {
+; CHECK-LABEL: test_vec_mul_scalar_add_ushort:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    push {r4, lr}
+; CHECK-NEXT:    ldr.w r12, [sp, #8]
+; CHECK-NEXT:    cmp.w r12, #0
+; CHECK-NEXT:    it eq
+; CHECK-NEXT:    popeq {r4, pc}
+; CHECK-NEXT:    dlstp.32 lr, r12
+; CHECK-NEXT:  .LBB8_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrh.u32 q0, [r0], #8
+; CHECK-NEXT:    vldrh.u32 q1, [r1], #8
+; CHECK-NEXT:    vmul.i32 q0, q1, q0
+; CHECK-NEXT:    sub.w r12, r12, #4
+; CHECK-NEXT:    vadd.i32 q0, q0, r2
+; CHECK-NEXT:    vstrw.32 q0, [r3], #16
+; CHECK-NEXT:    letp lr, .LBB8_1
+; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
+; CHECK-NEXT:    pop {r4, pc}
+entry:
+  %cmp10 = icmp eq i32 %N, 0
+  br i1 %cmp10, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %conv3 = sext i16 %c to i32
+  %n.rnd.up = add i32 %N, 3
+  %n.vec = and i32 %n.rnd.up, -4
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert12 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat13 = shufflevector <4 x i32> %broadcast.splatinsert12, <4 x i32> undef, <4 x i32> zeroinitializer
+  %broadcast.splatinsert15 = insertelement <4 x i32> undef, i32 %conv3, i32 0
+  %broadcast.splat16 = shufflevector <4 x i32> %broadcast.splatinsert15, <4 x i32> undef, <4 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+  %0 = getelementptr inbounds i16, i16* %a, i32 %index
+  %1 = icmp ule <4 x i32> %induction, %broadcast.splat13
+  %2 = bitcast i16* %0 to <4 x i16>*
+  %wide.masked.load = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %2, i32 2, <4 x i1> %1, <4 x i16> undef)
+  %3 = zext <4 x i16> %wide.masked.load to <4 x i32>
+  %4 = getelementptr inbounds i16, i16* %b, i32 %index
+  %5 = bitcast i16* %4 to <4 x i16>*
+  %wide.masked.load14 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %5, i32 2, <4 x i1> %1, <4 x i16> undef)
+  %6 = zext <4 x i16> %wide.masked.load14 to <4 x i32>
+  %7 = mul nuw nsw <4 x i32> %6, %3
+  %8 = add nsw <4 x i32> %7, %broadcast.splat16
+  %9 = getelementptr inbounds i32, i32* %res, i32 %index
+  %10 = bitcast i32* %9 to <4 x i32>*
+  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %8, <4 x i32>* %10, i32 4, <4 x i1> %1)
+  %index.next = add i32 %index, 4
+  %11 = icmp eq i32 %index.next, %n.vec
+  br i1 %11, label %for.cond.cleanup, label %vector.body
+
+for.cond.cleanup:                                 ; preds = %vector.body, %entry
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vec_mul_scalar_add_int(i32* nocapture readonly %a, i32* nocapture readonly %b, i32 %c, i32* nocapture %res, i32 %N) {
+; CHECK-LABEL: test_vec_mul_scalar_add_int:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, r9, r10, lr}
+; CHECK-NEXT:    ldr.w r12, [sp, #32]
+; CHECK-NEXT:    cmp.w r12, #0
+; CHECK-NEXT:    beq.w .LBB9_11
+; CHECK-NEXT:  @ %bb.1: @ %vector.memcheck
+; CHECK-NEXT:    add.w r4, r3, r12, lsl #2
+; CHECK-NEXT:    add.w r5, r1, r12, lsl #2
+; CHECK-NEXT:    cmp r4, r1
+; CHECK-NEXT:    add.w r6, r0, r12, lsl #2
+; CHECK-NEXT:    cset r7, hi
+; CHECK-NEXT:    cmp r5, r3
+; CHECK-NEXT:    cset r5, hi
+; CHECK-NEXT:    cmp r4, r0
+; CHECK-NEXT:    cset r4, hi
+; CHECK-NEXT:    cmp r6, r3
+; CHECK-NEXT:    cset r6, hi
+; CHECK-NEXT:    mov.w lr, #1
+; CHECK-NEXT:    ands r6, r4
+; CHECK-NEXT:    lsls r6, r6, #31
+; CHECK-NEXT:    itt eq
+; CHECK-NEXT:    andeq.w r4, r5, r7
+; CHECK-NEXT:    lslseq.w r4, r4, #31
+; CHECK-NEXT:    beq .LBB9_4
+; CHECK-NEXT:  @ %bb.2: @ %for.body.preheader
+; CHECK-NEXT:    sub.w r4, r12, #1
+; CHECK-NEXT:    and r10, r12, #3
+; CHECK-NEXT:    cmp r4, #3
+; CHECK-NEXT:    bhs .LBB9_6
+; CHECK-NEXT:  @ %bb.3:
+; CHECK-NEXT:    mov.w r12, #0
+; CHECK-NEXT:    b .LBB9_8
+; CHECK-NEXT:  .LBB9_4: @ %vector.ph
+; CHECK-NEXT:    dlstp.32 lr, r12
+; CHECK-NEXT:  .LBB9_5: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrw.u32 q0, [r0], #16
+; CHECK-NEXT:    vldrw.u32 q1, [r1], #16
+; CHECK-NEXT:    vmul.i32 q0, q1, q0
+; CHECK-NEXT:    sub.w r12, r12, #4
+; CHECK-NEXT:    vadd.i32 q0, q0, r2
+; CHECK-NEXT:    vstrw.32 q0, [r3], #16
+; CHECK-NEXT:    letp lr, .LBB9_5
+; CHECK-NEXT:    b .LBB9_11
+; CHECK-NEXT:  .LBB9_6: @ %for.body.preheader.new
+; CHECK-NEXT:    bic r7, r12, #3
+; CHECK-NEXT:    movs r4, #0
+; CHECK-NEXT:    subs r7, #4
+; CHECK-NEXT:    mov.w r12, #0
+; CHECK-NEXT:    add.w lr, lr, r7, lsr #2
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB9_7: @ %for.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    ldr r5, [r0, r4]
+; CHECK-NEXT:    add.w r9, r0, r4
+; CHECK-NEXT:    ldr r6, [r1, r4]
+; CHECK-NEXT:    adds r7, r1, r4
+; CHECK-NEXT:    add.w r12, r12, #4
+; CHECK-NEXT:    mla r5, r6, r5, r2
+; CHECK-NEXT:    str r5, [r3, r4]
+; CHECK-NEXT:    ldr.w r8, [r9, #4]
+; CHECK-NEXT:    ldr r6, [r7, #4]
+; CHECK-NEXT:    mla r8, r6, r8, r2
+; CHECK-NEXT:    adds r6, r3, r4
+; CHECK-NEXT:    adds r4, #16
+; CHECK-NEXT:    str.w r8, [r6, #4]
+; CHECK-NEXT:    ldr.w r8, [r9, #8]
+; CHECK-NEXT:    ldr r5, [r7, #8]
+; CHECK-NEXT:    mla r5, r5, r8, r2
+; CHECK-NEXT:    str r5, [r6, #8]
+; CHECK-NEXT:    ldr.w r5, [r9, #12]
+; CHECK-NEXT:    ldr r7, [r7, #12]
+; CHECK-NEXT:    mla r5, r7, r5, r2
+; CHECK-NEXT:    str r5, [r6, #12]
+; CHECK-NEXT:    le lr, .LBB9_7
+; CHECK-NEXT:  .LBB9_8: @ %for.cond.cleanup.loopexit.unr-lcssa
+; CHECK-NEXT:    wls lr, r10, .LBB9_11
+; CHECK-NEXT:  @ %bb.9: @ %for.body.epil.preheader
+; CHECK-NEXT:    mvn r7, #3
+; CHECK-NEXT:    mov lr, r10
+; CHECK-NEXT:    add.w r7, r7, r12, lsl #2
+; CHECK-NEXT:    add r0, r7
+; CHECK-NEXT:    add r1, r7
+; CHECK-NEXT:    add r3, r7
+; CHECK-NEXT:  .LBB9_10: @ %for.body.epil
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    ldr r7, [r0, #4]!
+; CHECK-NEXT:    ldr r6, [r1, #4]!
+; CHECK-NEXT:    mla r7, r6, r7, r2
+; CHECK-NEXT:    str r7, [r3, #4]!
+; CHECK-NEXT:    le lr, .LBB9_10
+; CHECK-NEXT:  .LBB9_11: @ %for.cond.cleanup
+; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, pc}
+entry:
+  %cmp8 = icmp eq i32 %N, 0
+  br i1 %cmp8, label %for.cond.cleanup, label %vector.memcheck
+
+vector.memcheck:                                  ; preds = %entry
+  %scevgep = getelementptr i32, i32* %res, i32 %N
+  %scevgep13 = getelementptr i32, i32* %a, i32 %N
+  %scevgep16 = getelementptr i32, i32* %b, i32 %N
+  %bound0 = icmp ugt i32* %scevgep13, %res
+  %bound1 = icmp ugt i32* %scevgep, %a
+  %found.conflict = and i1 %bound0, %bound1
+  %bound018 = icmp ugt i32* %scevgep16, %res
+  %bound119 = icmp ugt i32* %scevgep, %b
+  %found.conflict20 = and i1 %bound018, %bound119
+  %conflict.rdx = or i1 %found.conflict, %found.conflict20
+  br i1 %conflict.rdx, label %for.body.preheader, label %vector.ph
+
+for.body.preheader:                               ; preds = %vector.memcheck
+  %0 = add i32 %N, -1
+  %xtraiter = and i32 %N, 3
+  %1 = icmp ult i32 %0, 3
+  br i1 %1, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body.preheader.new
+
+for.body.preheader.new:                           ; preds = %for.body.preheader
+  %unroll_iter = sub i32 %N, %xtraiter
+  br label %for.body
+
+vector.ph:                                        ; preds = %vector.memcheck
+  %n.rnd.up = add i32 %N, 3
+  %n.vec = and i32 %n.rnd.up, -4
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert21 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat22 = shufflevector <4 x i32> %broadcast.splatinsert21, <4 x i32> undef, <4 x i32> zeroinitializer
+  %broadcast.splatinsert24 = insertelement <4 x i32> undef, i32 %c, i32 0
+  %broadcast.splat25 = shufflevector <4 x i32> %broadcast.splatinsert24, <4 x i32> undef, <4 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+  %2 = getelementptr inbounds i32, i32* %a, i32 %index
+  %3 = icmp ule <4 x i32> %induction, %broadcast.splat22
+  %4 = bitcast i32* %2 to <4 x i32>*
+  %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %4, i32 4, <4 x i1> %3, <4 x i32> undef)
+  %5 = getelementptr inbounds i32, i32* %b, i32 %index
+  %6 = bitcast i32* %5 to <4 x i32>*
+  %wide.masked.load23 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %6, i32 4, <4 x i1> %3, <4 x i32> undef)
+  %7 = mul nsw <4 x i32> %wide.masked.load23, %wide.masked.load
+  %8 = add nsw <4 x i32> %7, %broadcast.splat25
+  %9 = getelementptr inbounds i32, i32* %res, i32 %index
+  %10 = bitcast i32* %9 to <4 x i32>*
+  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %8, <4 x i32>* %10, i32 4, <4 x i1> %3)
+  %index.next = add i32 %index, 4
+  %11 = icmp eq i32 %index.next, %n.vec
+  br i1 %11, label %for.cond.cleanup, label %vector.body
+
+for.cond.cleanup.loopexit.unr-lcssa:              ; preds = %for.body, %for.body.preheader
+  %i.09.unr = phi i32 [ 0, %for.body.preheader ], [ %inc.3, %for.body ]
+  %lcmp.mod = icmp eq i32 %xtraiter, 0
+  br i1 %lcmp.mod, label %for.cond.cleanup, label %for.body.epil
+
+for.body.epil:                                    ; preds = %for.cond.cleanup.loopexit.unr-lcssa, %for.body.epil
+  %i.09.epil = phi i32 [ %inc.epil, %for.body.epil ], [ %i.09.unr, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %epil.iter = phi i32 [ %epil.iter.sub, %for.body.epil ], [ %xtraiter, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %arrayidx.epil = getelementptr inbounds i32, i32* %a, i32 %i.09.epil
+  %12 = load i32, i32* %arrayidx.epil, align 4
+  %arrayidx1.epil = getelementptr inbounds i32, i32* %b, i32 %i.09.epil
+  %13 = load i32, i32* %arrayidx1.epil, align 4
+  %mul.epil = mul nsw i32 %13, %12
+  %add.epil = add nsw i32 %mul.epil, %c
+  %arrayidx2.epil = getelementptr inbounds i32, i32* %res, i32 %i.09.epil
+  store i32 %add.epil, i32* %arrayidx2.epil, align 4
+  %inc.epil = add nuw i32 %i.09.epil, 1
+  %epil.iter.sub = add i32 %epil.iter, -1
+  %epil.iter.cmp = icmp eq i32 %epil.iter.sub, 0
+  br i1 %epil.iter.cmp, label %for.cond.cleanup, label %for.body.epil
+
+for.cond.cleanup:                                 ; preds = %vector.body, %for.cond.cleanup.loopexit.unr-lcssa, %for.body.epil, %entry
+  ret void
+
+for.body:                                         ; preds = %for.body, %for.body.preheader.new
+  %i.09 = phi i32 [ 0, %for.body.preheader.new ], [ %inc.3, %for.body ]
+  %niter = phi i32 [ %unroll_iter, %for.body.preheader.new ], [ %niter.nsub.3, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i32 %i.09
+  %14 = load i32, i32* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds i32, i32* %b, i32 %i.09
+  %15 = load i32, i32* %arrayidx1, align 4
+  %mul = mul nsw i32 %15, %14
+  %add = add nsw i32 %mul, %c
+  %arrayidx2 = getelementptr inbounds i32, i32* %res, i32 %i.09
+  store i32 %add, i32* %arrayidx2, align 4
+  %inc = or i32 %i.09, 1
+  %arrayidx.1 = getelementptr inbounds i32, i32* %a, i32 %inc
+  %16 = load i32, i32* %arrayidx.1, align 4
+  %arrayidx1.1 = getelementptr inbounds i32, i32* %b, i32 %inc
+  %17 = load i32, i32* %arrayidx1.1, align 4
+  %mul.1 = mul nsw i32 %17, %16
+  %add.1 = add nsw i32 %mul.1, %c
+  %arrayidx2.1 = getelementptr inbounds i32, i32* %res, i32 %inc
+  store i32 %add.1, i32* %arrayidx2.1, align 4
+  %inc.1 = or i32 %i.09, 2
+  %arrayidx.2 = getelementptr inbounds i32, i32* %a, i32 %inc.1
+  %18 = load i32, i32* %arrayidx.2, align 4
+  %arrayidx1.2 = getelementptr inbounds i32, i32* %b, i32 %inc.1
+  %19 = load i32, i32* %arrayidx1.2, align 4
+  %mul.2 = mul nsw i32 %19, %18
+  %add.2 = add nsw i32 %mul.2, %c
+  %arrayidx2.2 = getelementptr inbounds i32, i32* %res, i32 %inc.1
+  store i32 %add.2, i32* %arrayidx2.2, align 4
+  %inc.2 = or i32 %i.09, 3
+  %arrayidx.3 = getelementptr inbounds i32, i32* %a, i32 %inc.2
+  %20 = load i32, i32* %arrayidx.3, align 4
+  %arrayidx1.3 = getelementptr inbounds i32, i32* %b, i32 %inc.2
+  %21 = load i32, i32* %arrayidx1.3, align 4
+  %mul.3 = mul nsw i32 %21, %20
+  %add.3 = add nsw i32 %mul.3, %c
+  %arrayidx2.3 = getelementptr inbounds i32, i32* %res, i32 %inc.2
+  store i32 %add.3, i32* %arrayidx2.3, align 4
+  %inc.3 = add nuw i32 %i.09, 4
+  %niter.nsub.3 = add i32 %niter, -4
+  %niter.ncmp.3 = icmp eq i32 %niter.nsub.3, 0
+  br i1 %niter.ncmp.3, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body
+}
+
+define dso_local arm_aapcs_vfpcc void @test_v8i8_to_v8i16(i16* noalias nocapture %a, i8* nocapture readonly %b, i8* nocapture readonly %c, i32 %N) {
+; CHECK-LABEL: test_v8i8_to_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    push {r4, lr}
+; CHECK-NEXT:    cmp r3, #0
+; CHECK-NEXT:    it eq
+; CHECK-NEXT:    popeq {r4, pc}
+; CHECK-NEXT:    mov.w r12, #0
+; CHECK-NEXT:    dlstp.16 lr, r3
+; CHECK-NEXT:  .LBB10_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    add.w r4, r1, r12
+; CHECK-NEXT:    vldrb.u16 q0, [r4]
+; CHECK-NEXT:    add.w r4, r2, r12
+; CHECK-NEXT:    add.w r12, r12, #8
+; CHECK-NEXT:    vldrb.u16 q1, [r4]
+; CHECK-NEXT:    vmul.i16 q0, q1, q0
+; CHECK-NEXT:    vstrh.16 q0, [r0], #16
+; CHECK-NEXT:    letp lr, .LBB10_1
+; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
+; CHECK-NEXT:    pop {r4, pc}
+entry:
+  %cmp10 = icmp eq i32 %N, 0
+  br i1 %cmp10, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %n.rnd.up = add i32 %N, 7
+  %n.vec = and i32 %n.rnd.up, -8
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert12 = insertelement <8 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat13 = shufflevector <8 x i32> %broadcast.splatinsert12, <8 x i32> undef, <8 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %broadcast.splatinsert = insertelement <8 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <8 x i32> %broadcast.splatinsert, <8 x i32> undef, <8 x i32> zeroinitializer
+  %induction = add <8 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %0 = getelementptr inbounds i8, i8* %b, i32 %index
+  %1 = icmp ule <8 x i32> %induction, %broadcast.splat13
+  %2 = bitcast i8* %0 to <8 x i8>*
+  %wide.masked.load = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %2, i32 1, <8 x i1> %1, <8 x i8> undef)
+  %3 = zext <8 x i8> %wide.masked.load to <8 x i16>
+  %4 = getelementptr inbounds i8, i8* %c, i32 %index
+  %5 = bitcast i8* %4 to <8 x i8>*
+  %wide.masked.load14 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %5, i32 1, <8 x i1> %1, <8 x i8> undef)
+  %6 = zext <8 x i8> %wide.masked.load14 to <8 x i16>
+  %7 = mul nuw <8 x i16> %6, %3
+  %8 = getelementptr inbounds i16, i16* %a, i32 %index
+  %9 = bitcast i16* %8 to <8 x i16>*
+  call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %7, <8 x i16>* %9, i32 2, <8 x i1> %1)
+  %index.next = add i32 %index, 8
+  %10 = icmp eq i32 %index.next, %n.vec
+  br i1 %10, label %for.cond.cleanup, label %vector.body
+
+for.cond.cleanup:                                 ; preds = %vector.body, %entry
+  ret void
+}
+
+declare <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>*, i32 immarg, <4 x i1>, <4 x i8>)
+declare <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>*, i32 immarg, <8 x i1>, <8 x i8>)
+declare <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>*, i32 immarg, <4 x i1>, <4 x i16>)
+declare <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>*, i32 immarg, <4 x i1>, <4 x i32>)
+declare void @llvm.masked.store.v8i16.p0v8i16(<8 x i16>, <8 x i16>*, i32 immarg, <8 x i1>)
+declare void @llvm.masked.store.v4i32.p0v4i32(<4 x i32>, <4 x i32>*, i32 immarg, <4 x i1>)
+declare i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32>)
+
+
+
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/nested.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/nested.ll
new file mode 100644
index 000000000..f67a59f74
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/nested.ll
@@ -0,0 +1,241 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -mtriple=armv8.1m.main -mattr=+mve -S -mve-tail-predication -disable-mve-tail-predication=false %s -o - | FileCheck %s
+
+define void @mat_vec_sext_i16(i16** nocapture readonly %A, i16* nocapture readonly %B, i32* noalias nocapture %C, i32 %N) {
+; CHECK-LABEL: @mat_vec_sext_i16(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP24:%.*]] = icmp eq i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP24]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_COND1_PREHEADER_US_PREHEADER:%.*]]
+; CHECK:       for.cond1.preheader.us.preheader:
+; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i32 [[N]], 3
+; CHECK-NEXT:    [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -4
+; CHECK-NEXT:    [[TMP:%.*]] = add i32 [[N_VEC]], -4
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i32 [[TMP]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = add nuw nsw i32 [[TMP1]], 1
+; CHECK-NEXT:    br label [[FOR_COND1_PREHEADER_US:%.*]]
+; CHECK:       for.cond1.preheader.us:
+; CHECK-NEXT:    [[I_025_US:%.*]] = phi i32 [ [[INC10_US:%.*]], [[MIDDLE_BLOCK:%.*]] ], [ 0, [[FOR_COND1_PREHEADER_US_PREHEADER]] ]
+; CHECK-NEXT:    [[ARRAYIDX_US:%.*]] = getelementptr inbounds i16*, i16** [[A:%.*]], i32 [[I_025_US]]
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16*, i16** [[ARRAYIDX_US]], align 4
+; CHECK-NEXT:    [[ARRAYIDX8_US:%.*]] = getelementptr inbounds i32, i32* [[C:%.*]], i32 [[I_025_US]]
+; CHECK-NEXT:    [[ARRAYIDX8_PROMOTED_US:%.*]] = load i32, i32* [[ARRAYIDX8_US]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <4 x i32> <i32 undef, i32 0, i32 0, i32 0>, i32 [[ARRAYIDX8_PROMOTED_US]], i32 0
+; CHECK-NEXT:    call void @llvm.set.loop.iterations.i32(i32 [[TMP2]])
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[FOR_COND1_PREHEADER_US]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ [[TMP4]], [[FOR_COND1_PREHEADER_US]] ], [ [[TMP14:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP5:%.*]] = phi i32 [ [[TMP2]], [[FOR_COND1_PREHEADER_US]] ], [ [[TMP15:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = phi i32 [ [[N]], [[FOR_COND1_PREHEADER_US]] ], [ [[TMP2:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i16, i16* [[TMP3]], i32 [[INDEX]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.vctp32(i32 [[TMP0]])
+; CHECK-NEXT:    [[TMP2]] = sub i32 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i16* [[TMP6]] to <4 x i16>*
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* [[TMP8]], i32 2, <4 x i1> [[TMP1]], <4 x i16> undef)
+; CHECK-NEXT:    [[TMP9:%.*]] = sext <4 x i16> [[WIDE_MASKED_LOAD]] to <4 x i32>
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i16, i16* [[B:%.*]], i32 [[INDEX]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16* [[TMP10]] to <4 x i16>*
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD30:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* [[TMP11]], i32 2, <4 x i1> [[TMP1]], <4 x i16> undef)
+; CHECK-NEXT:    [[TMP12:%.*]] = sext <4 x i16> [[WIDE_MASKED_LOAD30]] to <4 x i32>
+; CHECK-NEXT:    [[TMP13:%.*]] = mul nsw <4 x i32> [[TMP12]], [[TMP9]]
+; CHECK-NEXT:    [[TMP14]] = add nsw <4 x i32> [[TMP13]], [[VEC_PHI]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP15]] = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 [[TMP5]], i32 1)
+; CHECK-NEXT:    [[TMP16:%.*]] = icmp ne i32 [[TMP15]], 0
+; CHECK-NEXT:    br i1 [[TMP16]], label [[VECTOR_BODY]], label [[MIDDLE_BLOCK]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[TMP17:%.*]] = select <4 x i1> [[TMP1]], <4 x i32> [[TMP14]], <4 x i32> [[VEC_PHI]]
+; CHECK-NEXT:    [[TMP18:%.*]] = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> [[TMP17]])
+; CHECK-NEXT:    store i32 [[TMP18]], i32* [[ARRAYIDX8_US]], align 4
+; CHECK-NEXT:    [[INC10_US]] = add nuw i32 [[I_025_US]], 1
+; CHECK-NEXT:    [[EXITCOND27:%.*]] = icmp eq i32 [[INC10_US]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND27]], label [[FOR_COND_CLEANUP]], label [[FOR_COND1_PREHEADER_US]]
+; CHECK:       for.cond.cleanup:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cmp24 = icmp eq i32 %N, 0
+  br i1 %cmp24, label %for.cond.cleanup, label %for.cond1.preheader.us.preheader
+
+for.cond1.preheader.us.preheader:                 ; preds = %entry
+  %n.rnd.up = add i32 %N, 3
+  %n.vec = and i32 %n.rnd.up, -4
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert28 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat29 = shufflevector <4 x i32> %broadcast.splatinsert28, <4 x i32> undef, <4 x i32> zeroinitializer
+  %tmp = add i32 %n.vec, -4
+  %tmp1 = lshr i32 %tmp, 2
+  %tmp2 = add nuw nsw i32 %tmp1, 1
+  br label %for.cond1.preheader.us
+
+for.cond1.preheader.us:                           ; preds = %middle.block, %for.cond1.preheader.us.preheader
+  %i.025.us = phi i32 [ %inc10.us, %middle.block ], [ 0, %for.cond1.preheader.us.preheader ]
+  %arrayidx.us = getelementptr inbounds i16*, i16** %A, i32 %i.025.us
+  %tmp3 = load i16*, i16** %arrayidx.us, align 4
+  %arrayidx8.us = getelementptr inbounds i32, i32* %C, i32 %i.025.us
+  %arrayidx8.promoted.us = load i32, i32* %arrayidx8.us, align 4
+  %tmp4 = insertelement <4 x i32> <i32 undef, i32 0, i32 0, i32 0>, i32 %arrayidx8.promoted.us, i32 0
+  call void @llvm.set.loop.iterations.i32(i32 %tmp2)
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %for.cond1.preheader.us
+  %index = phi i32 [ 0, %for.cond1.preheader.us ], [ %index.next, %vector.body ]
+  %vec.phi = phi <4 x i32> [ %tmp4, %for.cond1.preheader.us ], [ %tmp14, %vector.body ]
+  %tmp5 = phi i32 [ %tmp2, %for.cond1.preheader.us ], [ %tmp15, %vector.body ]
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+  %tmp6 = getelementptr inbounds i16, i16* %tmp3, i32 %index
+  %tmp7 = icmp ule <4 x i32> %induction, %broadcast.splat29
+  %tmp8 = bitcast i16* %tmp6 to <4 x i16>*
+  %wide.masked.load = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %tmp8, i32 2, <4 x i1> %tmp7, <4 x i16> undef)
+  %tmp9 = sext <4 x i16> %wide.masked.load to <4 x i32>
+  %tmp10 = getelementptr inbounds i16, i16* %B, i32 %index
+  %tmp11 = bitcast i16* %tmp10 to <4 x i16>*
+  %wide.masked.load30 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %tmp11, i32 2, <4 x i1> %tmp7, <4 x i16> undef)
+  %tmp12 = sext <4 x i16> %wide.masked.load30 to <4 x i32>
+  %tmp13 = mul nsw <4 x i32> %tmp12, %tmp9
+  %tmp14 = add nsw <4 x i32> %tmp13, %vec.phi
+  %index.next = add i32 %index, 4
+  %tmp15 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %tmp5, i32 1)
+  %tmp16 = icmp ne i32 %tmp15, 0
+  br i1 %tmp16, label %vector.body, label %middle.block
+
+middle.block:                                     ; preds = %vector.body
+  %tmp17 = select <4 x i1> %tmp7, <4 x i32> %tmp14, <4 x i32> %vec.phi
+  %tmp18 = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %tmp17)
+  store i32 %tmp18, i32* %arrayidx8.us, align 4
+  %inc10.us = add nuw i32 %i.025.us, 1
+  %exitcond27 = icmp eq i32 %inc10.us, %N
+  br i1 %exitcond27, label %for.cond.cleanup, label %for.cond1.preheader.us
+
+for.cond.cleanup:                                 ; preds = %middle.block, %entry
+  ret void
+}
+
+define void @mat_vec_i32(i32** nocapture readonly %A, i32* nocapture readonly %B, i32* noalias nocapture %C, i32 %N) {
+; CHECK-LABEL: @mat_vec_i32(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP23:%.*]] = icmp eq i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP23]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_COND1_PREHEADER_US_PREHEADER:%.*]]
+; CHECK:       for.cond1.preheader.us.preheader:
+; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i32 [[N]], 3
+; CHECK-NEXT:    [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -4
+; CHECK-NEXT:    [[TMP:%.*]] = add i32 [[N_VEC]], -4
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i32 [[TMP]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = add nuw nsw i32 [[TMP1]], 1
+; CHECK-NEXT:    br label [[FOR_COND1_PREHEADER_US:%.*]]
+; CHECK:       for.cond1.preheader.us:
+; CHECK-NEXT:    [[I_024_US:%.*]] = phi i32 [ [[INC9_US:%.*]], [[MIDDLE_BLOCK:%.*]] ], [ 0, [[FOR_COND1_PREHEADER_US_PREHEADER]] ]
+; CHECK-NEXT:    [[ARRAYIDX_US:%.*]] = getelementptr inbounds i32*, i32** [[A:%.*]], i32 [[I_024_US]]
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32*, i32** [[ARRAYIDX_US]], align 4
+; CHECK-NEXT:    [[ARRAYIDX7_US:%.*]] = getelementptr inbounds i32, i32* [[C:%.*]], i32 [[I_024_US]]
+; CHECK-NEXT:    [[ARRAYIDX7_PROMOTED_US:%.*]] = load i32, i32* [[ARRAYIDX7_US]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <4 x i32> <i32 undef, i32 0, i32 0, i32 0>, i32 [[ARRAYIDX7_PROMOTED_US]], i32 0
+; CHECK-NEXT:    call void @llvm.set.loop.iterations.i32(i32 [[TMP2]])
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[FOR_COND1_PREHEADER_US]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ [[TMP4]], [[FOR_COND1_PREHEADER_US]] ], [ [[TMP12:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP5:%.*]] = phi i32 [ [[TMP2]], [[FOR_COND1_PREHEADER_US]] ], [ [[TMP13:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = phi i32 [ [[N]], [[FOR_COND1_PREHEADER_US]] ], [ [[TMP2:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[TMP3]], i32 [[INDEX]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.vctp32(i32 [[TMP0]])
+; CHECK-NEXT:    [[TMP2]] = sub i32 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i32* [[TMP6]] to <4 x i32>*
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* [[TMP8]], i32 4, <4 x i1> [[TMP1]], <4 x i32> undef)
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i32 [[INDEX]]
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i32* [[TMP9]] to <4 x i32>*
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD29:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* [[TMP10]], i32 4, <4 x i1> [[TMP1]], <4 x i32> undef)
+; CHECK-NEXT:    [[TMP11:%.*]] = mul nsw <4 x i32> [[WIDE_MASKED_LOAD29]], [[WIDE_MASKED_LOAD]]
+; CHECK-NEXT:    [[TMP12]] = add nsw <4 x i32> [[VEC_PHI]], [[TMP11]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP13]] = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 [[TMP5]], i32 1)
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp ne i32 [[TMP13]], 0
+; CHECK-NEXT:    br i1 [[TMP14]], label [[VECTOR_BODY]], label [[MIDDLE_BLOCK]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[TMP15:%.*]] = select <4 x i1> [[TMP1]], <4 x i32> [[TMP12]], <4 x i32> [[VEC_PHI]]
+; CHECK-NEXT:    [[TMP16:%.*]] = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> [[TMP15]])
+; CHECK-NEXT:    store i32 [[TMP16]], i32* [[ARRAYIDX7_US]], align 4
+; CHECK-NEXT:    [[INC9_US]] = add nuw i32 [[I_024_US]], 1
+; CHECK-NEXT:    [[EXITCOND26:%.*]] = icmp eq i32 [[INC9_US]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND26]], label [[FOR_COND_CLEANUP]], label [[FOR_COND1_PREHEADER_US]]
+; CHECK:       for.cond.cleanup:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cmp23 = icmp eq i32 %N, 0
+  br i1 %cmp23, label %for.cond.cleanup, label %for.cond1.preheader.us.preheader
+
+for.cond1.preheader.us.preheader:                 ; preds = %entry
+  %n.rnd.up = add i32 %N, 3
+  %n.vec = and i32 %n.rnd.up, -4
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert27 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat28 = shufflevector <4 x i32> %broadcast.splatinsert27, <4 x i32> undef, <4 x i32> zeroinitializer
+  %tmp = add i32 %n.vec, -4
+  %tmp1 = lshr i32 %tmp, 2
+  %tmp2 = add nuw nsw i32 %tmp1, 1
+  br label %for.cond1.preheader.us
+
+for.cond1.preheader.us:                           ; preds = %middle.block, %for.cond1.preheader.us.preheader
+  %i.024.us = phi i32 [ %inc9.us, %middle.block ], [ 0, %for.cond1.preheader.us.preheader ]
+  %arrayidx.us = getelementptr inbounds i32*, i32** %A, i32 %i.024.us
+  %tmp3 = load i32*, i32** %arrayidx.us, align 4
+  %arrayidx7.us = getelementptr inbounds i32, i32* %C, i32 %i.024.us
+  %arrayidx7.promoted.us = load i32, i32* %arrayidx7.us, align 4
+  %tmp4 = insertelement <4 x i32> <i32 undef, i32 0, i32 0, i32 0>, i32 %arrayidx7.promoted.us, i32 0
+  call void @llvm.set.loop.iterations.i32(i32 %tmp2)
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %for.cond1.preheader.us
+  %index = phi i32 [ 0, %for.cond1.preheader.us ], [ %index.next, %vector.body ]
+  %vec.phi = phi <4 x i32> [ %tmp4, %for.cond1.preheader.us ], [ %tmp12, %vector.body ]
+  %tmp5 = phi i32 [ %tmp2, %for.cond1.preheader.us ], [ %tmp13, %vector.body ]
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+  %tmp6 = getelementptr inbounds i32, i32* %tmp3, i32 %index
+  %tmp7 = icmp ule <4 x i32> %induction, %broadcast.splat28
+  %tmp8 = bitcast i32* %tmp6 to <4 x i32>*
+  %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp8, i32 4, <4 x i1> %tmp7, <4 x i32> undef)
+  %tmp9 = getelementptr inbounds i32, i32* %B, i32 %index
+  %tmp10 = bitcast i32* %tmp9 to <4 x i32>*
+  %wide.masked.load29 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp10, i32 4, <4 x i1> %tmp7, <4 x i32> undef)
+  %tmp11 = mul nsw <4 x i32> %wide.masked.load29, %wide.masked.load
+  %tmp12 = add nsw <4 x i32> %vec.phi, %tmp11
+  %index.next = add i32 %index, 4
+  %tmp13 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %tmp5, i32 1)
+  %tmp14 = icmp ne i32 %tmp13, 0
+  br i1 %tmp14, label %vector.body, label %middle.block
+
+middle.block:                                     ; preds = %vector.body
+  %tmp15 = select <4 x i1> %tmp7, <4 x i32> %tmp12, <4 x i32> %vec.phi
+  %tmp16 = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %tmp15)
+  store i32 %tmp16, i32* %arrayidx7.us, align 4
+  %inc9.us = add nuw i32 %i.024.us, 1
+  %exitcond26 = icmp eq i32 %inc9.us, %N
+  br i1 %exitcond26, label %for.cond.cleanup, label %for.cond1.preheader.us
+
+for.cond.cleanup:                                 ; preds = %middle.block, %entry
+  ret void
+}
+
+; Function Attrs: argmemonly nounwind readonly willreturn
+declare <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>*, i32 immarg, <4 x i1>, <4 x i32>) #0
+
+; Function Attrs: argmemonly nounwind readonly willreturn
+declare <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>*, i32 immarg, <4 x i1>, <4 x i16>) #0
+
+; Function Attrs: nounwind readnone willreturn
+declare i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32>) #1
+
+; Function Attrs: noduplicate nounwind
+declare void @llvm.set.loop.iterations.i32(i32) #2
+
+; Function Attrs: noduplicate nounwind
+declare i32 @llvm.loop.decrement.reg.i32.i32.i32(i32, i32) #2
+
+attributes #0 = { argmemonly nounwind readonly willreturn }
+attributes #1 = { nounwind readnone willreturn }
+attributes #2 = { noduplicate nounwind }
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/no-dec-le-simple.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/no-dec-le-simple.ll
new file mode 100644
index 000000000..78ddb35e7
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/no-dec-le-simple.ll
@@ -0,0 +1,158 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main %s -o - | FileCheck %s
+
+define void @cbz_exit(i32* %in, i32* %res) {
+; CHECK-LABEL: cbz_exit:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    subs r2, r0, #4
+; CHECK-NEXT:    mov.w r0, #-1
+; CHECK-NEXT:  .LBB0_1: @ %loop
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    ldr r3, [r2, #4]!
+; CHECK-NEXT:    adds r0, #1
+; CHECK-NEXT:    cbz r3, .LBB0_2
+; CHECK-NEXT:    le .LBB0_1
+; CHECK-NEXT:  .LBB0_2: @ %exit
+; CHECK-NEXT:    str r0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  br label %loop
+
+loop:
+  %offset = phi i32 [ 0, %entry ], [ %next, %loop ]
+  %ptr = getelementptr i32, i32* %in, i32 %offset
+  %val = load i32, i32* %ptr
+  %next = add i32 %offset, 1
+  %cmp = icmp eq i32 %val, 0
+  br i1 %cmp, label %exit, label %loop
+
+exit:
+  store i32 %offset, i32* %res
+  ret void
+}
+
+define void @cbnz_exit(i32* %in, i32* %res) {
+; CHECK-LABEL: cbnz_exit:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    subs r2, r0, #4
+; CHECK-NEXT:    mov.w r0, #-1
+; CHECK-NEXT:  .LBB1_1: @ %loop
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    ldr r3, [r2, #4]!
+; CHECK-NEXT:    adds r0, #1
+; CHECK-NEXT:    cbnz r3, .LBB1_2
+; CHECK-NEXT:    le .LBB1_1
+; CHECK-NEXT:  .LBB1_2: @ %exit
+; CHECK-NEXT:    str r0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  br label %loop
+
+loop:
+  %offset = phi i32 [ 0, %entry ], [ %next, %loop ]
+  %ptr = getelementptr i32, i32* %in, i32 %offset
+  %val = load i32, i32* %ptr
+  %next = add i32 %offset, 1
+  %cmp = icmp ne i32 %val, 0
+  br i1 %cmp, label %exit, label %loop
+
+exit:
+  store i32 %offset, i32* %res
+  ret void
+}
+
+define void @cbnz_exit_too_large(i32* %in, i32* %res) {
+; CHECK-LABEL: cbnz_exit_too_large:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    subs r2, r0, #4
+; CHECK-NEXT:    mov.w r0, #-1
+; CHECK-NEXT:  .LBB2_1: @ %loop
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    ldr r12, [r2, #4]!
+; CHECK-NEXT:    .zero 4090
+; CHECK-NEXT:    adds r0, #1
+; CHECK-NEXT:    cmp.w r12, #0
+; CHECK-NEXT:    beq.w .LBB2_1
+; CHECK-NEXT:  @ %bb.2: @ %exit
+; CHECK-NEXT:    str r0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  br label %loop
+
+loop:
+  %offset = phi i32 [ 0, %entry ], [ %next, %loop ]
+  %ptr = getelementptr i32, i32* %in, i32 %offset
+  %val = load i32, i32* %ptr
+  %next = add i32 %offset, 1
+  %cmp = icmp ne i32 %val, 0
+  %size = call i32 @llvm.arm.space(i32 4090, i32 undef)
+  br i1 %cmp, label %exit, label %loop
+
+exit:
+  store i32 %offset, i32* %res
+  ret void
+}
+
+define void @cbz_exit_minsize(i32* %in, i32* %res) #0 {
+; CHECK-LABEL: cbz_exit_minsize:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    movs r2, #0
+; CHECK-NEXT:  .LBB3_1: @ %loop
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    ldr.w r3, [r0, r2, lsl #2]
+; CHECK-NEXT:    adds r2, #1
+; CHECK-NEXT:    cmp r3, #0
+; CHECK-NEXT:    bne .LBB3_1
+; CHECK-NEXT:  @ %bb.2: @ %exit
+; CHECK-NEXT:    subs r0, r2, #1
+; CHECK-NEXT:    str r0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  br label %loop
+
+loop:
+  %offset = phi i32 [ 0, %entry ], [ %next, %loop ]
+  %ptr = getelementptr i32, i32* %in, i32 %offset
+  %val = load i32, i32* %ptr
+  %next = add i32 %offset, 1
+  %cmp = icmp eq i32 %val, 0
+  br i1 %cmp, label %exit, label %loop
+
+exit:
+  store i32 %offset, i32* %res
+  ret void
+}
+
+define void @cbnz_exit_minsize(i32* %in, i32* %res) #0 {
+; CHECK-LABEL: cbnz_exit_minsize:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    movs r2, #0
+; CHECK-NEXT:  .LBB4_1: @ %loop
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    ldr.w r3, [r0, r2, lsl #2]
+; CHECK-NEXT:    adds r2, #1
+; CHECK-NEXT:    cmp r3, #0
+; CHECK-NEXT:    beq .LBB4_1
+; CHECK-NEXT:  @ %bb.2: @ %exit
+; CHECK-NEXT:    subs r0, r2, #1
+; CHECK-NEXT:    str r0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  br label %loop
+
+loop:
+  %offset = phi i32 [ 0, %entry ], [ %next, %loop ]
+  %ptr = getelementptr i32, i32* %in, i32 %offset
+  %val = load i32, i32* %ptr
+  %next = add i32 %offset, 1
+  %cmp = icmp ne i32 %val, 0
+  br i1 %cmp, label %exit, label %loop
+
+exit:
+  store i32 %offset, i32* %res
+  ret void
+}
+
+attributes #0 = { minsize optsize }
+
+declare i32 @llvm.arm.space(i32 immarg, i32);
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-pred-narrow.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-pred-narrow.ll
new file mode 100644
index 000000000..8c1534be7
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-pred-narrow.ll
@@ -0,0 +1,54 @@
+; RUN: opt -mtriple=thumbv8.1m.main -mve-tail-predication -disable-mve-tail-predication=false -mattr=+mve,+lob %s -S -o - | FileCheck %s
+
+; TODO: We should be able to generate a vctp for the loads.
+; CHECK-LABEL: trunc_v4i32_v4i16
+; CHECK-NOT: vcpt
+define void @trunc_v4i32_v4i16(i32* readonly %a, i32* readonly %b, i16* %c, i32 %N) {
+entry:
+  %cmp8 = icmp eq i32 %N, 0
+  %tmp8 = add i32 %N, 3
+  %tmp9 = lshr i32 %tmp8, 2
+  %tmp10 = shl nuw i32 %tmp9, 2
+  %tmp11 = add i32 %tmp10, -4
+  %tmp12 = lshr i32 %tmp11, 2
+  %tmp13 = add nuw nsw i32 %tmp12, 1
+  br i1 %cmp8, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert10 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat11 = shufflevector <4 x i32> %broadcast.splatinsert10, <4 x i32> undef, <4 x i32> zeroinitializer
+  call void @llvm.set.loop.iterations.i32(i32 %tmp13)
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %tmp14 = phi i32 [ %tmp13, %vector.ph ], [ %tmp15, %vector.body ]
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+  %tmp = getelementptr inbounds i32, i32* %a, i32 %index
+  %tmp1 = icmp ule <4 x i32> %induction, %broadcast.splat11
+  %tmp2 = bitcast i32* %tmp to <4 x i32>*
+  %wide.masked.load = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp2, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
+  %tmp3 = getelementptr inbounds i32, i32* %b, i32 %index
+  %tmp4 = bitcast i32* %tmp3 to <4 x i32>*
+  %wide.masked.load2 = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp4, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
+  %mul = mul nsw <4 x i32> %wide.masked.load2, %wide.masked.load
+  %trunc = trunc <4 x i32> %mul to <4 x i16>
+  %tmp6 = getelementptr inbounds i16, i16* %c, i32 %index
+  %tmp7 = bitcast i16* %tmp6 to <4 x i16>*
+  tail call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %trunc, <4 x i16>* %tmp7, i32 4, <4 x i1> %tmp1)
+  %index.next = add i32 %index, 4
+  %tmp15 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %tmp14, i32 1)
+  %tmp16 = icmp ne i32 %tmp15, 0
+  br i1 %tmp16, label %vector.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %vector.body, %entry
+  ret void
+}
+
+declare <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>*, i32 immarg, <4 x i1>, <4 x i32>)
+declare void @llvm.masked.store.v4i16.p0v4i16(<4 x i16>, <4 x i16>*, i32 immarg, <4 x i1>)
+declare void @llvm.set.loop.iterations.i32(i32)
+declare i32 @llvm.loop.decrement.reg.i32.i32.i32(i32, i32)
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-pred-pattern-fail.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-pred-pattern-fail.ll
new file mode 100644
index 000000000..1926bbeea
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-pred-pattern-fail.ll
@@ -0,0 +1,505 @@
+; RUN: opt -mtriple=thumbv8.1m.main -mve-tail-predication -disable-mve-tail-predication=false -mattr=+mve,+lob %s -S -o - | FileCheck %s
+
+; The following functions should all fail to become tail-predicated.
+; CHECK-NOT: call i32 @llvm.arm.vctp
+
+; trip.count.minus.1 has been inserted into element 1, not 0.
+define dso_local arm_aapcs_vfpcc void @wrong_ph_insert_0(i32* noalias nocapture readonly %a, i32* noalias nocapture readonly %b, i32* noalias nocapture %c, i32 %N) {
+entry:
+  %cmp8 = icmp eq i32 %N, 0
+  %tmp8 = add i32 %N, 3
+  %tmp9 = lshr i32 %tmp8, 2
+  %tmp10 = shl nuw i32 %tmp9, 2
+  %tmp11 = add i32 %tmp10, -4
+  %tmp12 = lshr i32 %tmp11, 2
+  %tmp13 = add nuw nsw i32 %tmp12, 1
+  br i1 %cmp8, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert10 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 1
+  %broadcast.splat11 = shufflevector <4 x i32> %broadcast.splatinsert10, <4 x i32> undef, <4 x i32> zeroinitializer
+  call void @llvm.set.loop.iterations.i32(i32 %tmp13)
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %tmp14 = phi i32 [ %tmp13, %vector.ph ], [ %tmp15, %vector.body ]
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+  %tmp = getelementptr inbounds i32, i32* %a, i32 %index
+  %tmp1 = icmp ule <4 x i32> %induction, %broadcast.splat11
+  %tmp2 = bitcast i32* %tmp to <4 x i32>*
+  %wide.masked.load = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp2, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
+  %tmp3 = getelementptr inbounds i32, i32* %b, i32 %index
+  %tmp4 = bitcast i32* %tmp3 to <4 x i32>*
+  %wide.masked.load12 = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp4, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
+  %tmp5 = mul nsw <4 x i32> %wide.masked.load12, %wide.masked.load
+  %tmp6 = getelementptr inbounds i32, i32* %c, i32 %index
+  %tmp7 = bitcast i32* %tmp6 to <4 x i32>*
+  tail call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %tmp5, <4 x i32>* %tmp7, i32 4, <4 x i1> %tmp1)
+  %index.next = add i32 %index, 4
+  %tmp15 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %tmp14, i32 1)
+  %tmp16 = icmp ne i32 %tmp15, 0
+  br i1 %tmp16, label %vector.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %vector.body, %entry
+  ret void
+}
+
+; The insert isn't using an undef for operand 0.
+define dso_local arm_aapcs_vfpcc void @wrong_ph_insert_def(i32* noalias nocapture readonly %a, i32* noalias nocapture readonly %b, i32* noalias nocapture %c, i32 %N) {
+entry:
+  %cmp8 = icmp eq i32 %N, 0
+  %tmp8 = add i32 %N, 3
+  %tmp9 = lshr i32 %tmp8, 2
+  %tmp10 = shl nuw i32 %tmp9, 2
+  %tmp11 = add i32 %tmp10, -4
+  %tmp12 = lshr i32 %tmp11, 2
+  %tmp13 = add nuw nsw i32 %tmp12, 1
+  br i1 %cmp8, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert10 = insertelement <4 x i32> <i32 1, i32 1, i32 1, i32 1>, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat11 = shufflevector <4 x i32> %broadcast.splatinsert10, <4 x i32> undef, <4 x i32> zeroinitializer
+  call void @llvm.set.loop.iterations.i32(i32 %tmp13)
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %tmp14 = phi i32 [ %tmp13, %vector.ph ], [ %tmp15, %vector.body ]
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+  %tmp = getelementptr inbounds i32, i32* %a, i32 %index
+  %tmp1 = icmp ule <4 x i32> %induction, %broadcast.splat11
+  %tmp2 = bitcast i32* %tmp to <4 x i32>*
+  %wide.masked.load = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp2, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
+  %tmp3 = getelementptr inbounds i32, i32* %b, i32 %index
+  %tmp4 = bitcast i32* %tmp3 to <4 x i32>*
+  %wide.masked.load12 = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp4, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
+  %tmp5 = mul nsw <4 x i32> %wide.masked.load12, %wide.masked.load
+  %tmp6 = getelementptr inbounds i32, i32* %c, i32 %index
+  %tmp7 = bitcast i32* %tmp6 to <4 x i32>*
+  tail call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %tmp5, <4 x i32>* %tmp7, i32 4, <4 x i1> %tmp1)
+  %index.next = add i32 %index, 4
+  %tmp15 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %tmp14, i32 1)
+  %tmp16 = icmp ne i32 %tmp15, 0
+  br i1 %tmp16, label %vector.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %vector.body, %entry
+  ret void
+}
+
+; The shuffle uses a defined value for operand 1.
+define dso_local arm_aapcs_vfpcc void @wrong_ph_shuffle_1(i32* noalias nocapture readonly %a, i32* noalias nocapture readonly %b, i32* noalias nocapture %c, i32 %N) {
+entry:
+  %cmp8 = icmp eq i32 %N, 0
+  %tmp8 = add i32 %N, 3
+  %tmp9 = lshr i32 %tmp8, 2
+  %tmp10 = shl nuw i32 %tmp9, 2
+  %tmp11 = add i32 %tmp10, -4
+  %tmp12 = lshr i32 %tmp11, 2
+  %tmp13 = add nuw nsw i32 %tmp12, 1
+  br i1 %cmp8, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert10 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat11 = shufflevector <4 x i32> %broadcast.splatinsert10, <4 x i32> <i32 1, i32 1, i32 1, i32 1>, <4 x i32> zeroinitializer
+  call void @llvm.set.loop.iterations.i32(i32 %tmp13)
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %tmp14 = phi i32 [ %tmp13, %vector.ph ], [ %tmp15, %vector.body ]
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+  %tmp = getelementptr inbounds i32, i32* %a, i32 %index
+  %tmp1 = icmp ule <4 x i32> %induction, %broadcast.splat11
+  %tmp2 = bitcast i32* %tmp to <4 x i32>*
+  %wide.masked.load = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp2, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
+  %tmp3 = getelementptr inbounds i32, i32* %b, i32 %index
+  %tmp4 = bitcast i32* %tmp3 to <4 x i32>*
+  %wide.masked.load12 = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp4, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
+  %tmp5 = mul nsw <4 x i32> %wide.masked.load12, %wide.masked.load
+  %tmp6 = getelementptr inbounds i32, i32* %c, i32 %index
+  %tmp7 = bitcast i32* %tmp6 to <4 x i32>*
+  tail call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %tmp5, <4 x i32>* %tmp7, i32 4, <4 x i1> %tmp1)
+  %index.next = add i32 %index, 4
+  %tmp15 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %tmp14, i32 1)
+  %tmp16 = icmp ne i32 %tmp15, 0
+  br i1 %tmp16, label %vector.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %vector.body, %entry
+  ret void
+}
+
+; The shuffle uses a non zero value for operand 2.
+define dso_local arm_aapcs_vfpcc void @wrong_ph_shuffle_2(i32* noalias nocapture readonly %a, i32* noalias nocapture readonly %b, i32* noalias nocapture %c, i32 %N) {
+entry:
+  %cmp8 = icmp eq i32 %N, 0
+  %tmp8 = add i32 %N, 3
+  %tmp9 = lshr i32 %tmp8, 2
+  %tmp10 = shl nuw i32 %tmp9, 2
+  %tmp11 = add i32 %tmp10, -4
+  %tmp12 = lshr i32 %tmp11, 2
+  %tmp13 = add nuw nsw i32 %tmp12, 1
+  br i1 %cmp8, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert10 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat11 = shufflevector <4 x i32> %broadcast.splatinsert10, <4 x i32> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  call void @llvm.set.loop.iterations.i32(i32 %tmp13)
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %tmp14 = phi i32 [ %tmp13, %vector.ph ], [ %tmp15, %vector.body ]
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+  %tmp = getelementptr inbounds i32, i32* %a, i32 %index
+  %tmp1 = icmp ule <4 x i32> %induction, %broadcast.splat11
+  %tmp2 = bitcast i32* %tmp to <4 x i32>*
+  %wide.masked.load = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp2, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
+  %tmp3 = getelementptr inbounds i32, i32* %b, i32 %index
+  %tmp4 = bitcast i32* %tmp3 to <4 x i32>*
+  %wide.masked.load12 = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp4, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
+  %tmp5 = mul nsw <4 x i32> %wide.masked.load12, %wide.masked.load
+  %tmp6 = getelementptr inbounds i32, i32* %c, i32 %index
+  %tmp7 = bitcast i32* %tmp6 to <4 x i32>*
+  tail call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %tmp5, <4 x i32>* %tmp7, i32 4, <4 x i1> %tmp1)
+  %index.next = add i32 %index, 4
+  %tmp15 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %tmp14, i32 1)
+  %tmp16 = icmp ne i32 %tmp15, 0
+  br i1 %tmp16, label %vector.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %vector.body, %entry
+  ret void
+}
+
+; %N - 2
+define dso_local arm_aapcs_vfpcc void @trip_count_minus_2(i32* noalias nocapture readonly %a, i32* noalias nocapture readonly %b, i32* noalias nocapture %c, i32 %N) {
+entry:
+  %cmp8 = icmp eq i32 %N, 0
+  %tmp8 = add i32 %N, 3
+  %tmp9 = lshr i32 %tmp8, 2
+  %tmp10 = shl nuw i32 %tmp9, 2
+  %tmp11 = add i32 %tmp10, -4
+  %tmp12 = lshr i32 %tmp11, 2
+  %tmp13 = add nuw nsw i32 %tmp12, 1
+  br i1 %cmp8, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %trip.count.minus.2 = add i32 %N, -2
+  %broadcast.splatinsert10 = insertelement <4 x i32> undef, i32 %trip.count.minus.2, i32 1
+  %broadcast.splat11 = shufflevector <4 x i32> %broadcast.splatinsert10, <4 x i32> undef, <4 x i32> zeroinitializer
+  call void @llvm.set.loop.iterations.i32(i32 %tmp13)
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %tmp14 = phi i32 [ %tmp13, %vector.ph ], [ %tmp15, %vector.body ]
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+  %tmp = getelementptr inbounds i32, i32* %a, i32 %index
+  %tmp1 = icmp ule <4 x i32> %induction, %broadcast.splat11
+  %tmp2 = bitcast i32* %tmp to <4 x i32>*
+  %wide.masked.load = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp2, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
+  %tmp3 = getelementptr inbounds i32, i32* %b, i32 %index
+  %tmp4 = bitcast i32* %tmp3 to <4 x i32>*
+  %wide.masked.load12 = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp4, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
+  %tmp5 = mul nsw <4 x i32> %wide.masked.load12, %wide.masked.load
+  %tmp6 = getelementptr inbounds i32, i32* %c, i32 %index
+  %tmp7 = bitcast i32* %tmp6 to <4 x i32>*
+  tail call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %tmp5, <4 x i32>* %tmp7, i32 4, <4 x i1> %tmp1)
+  %index.next = add i32 %index, 4
+  %tmp15 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %tmp14, i32 1)
+  %tmp16 = icmp ne i32 %tmp15, 0
+  br i1 %tmp16, label %vector.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %vector.body, %entry
+  ret void
+}
+
+; index has been inserted at element 1, not 0.
+define dso_local arm_aapcs_vfpcc void @wrong_loop_insert(i32* noalias nocapture readonly %a, i32* noalias nocapture readonly %b, i32* noalias nocapture %c, i32 %N) {
+entry:
+  %cmp8 = icmp eq i32 %N, 0
+  %tmp8 = add i32 %N, 3
+  %tmp9 = lshr i32 %tmp8, 2
+  %tmp10 = shl nuw i32 %tmp9, 2
+  %tmp11 = add i32 %tmp10, -4
+  %tmp12 = lshr i32 %tmp11, 2
+  %tmp13 = add nuw nsw i32 %tmp12, 1
+  br i1 %cmp8, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert10 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat11 = shufflevector <4 x i32> %broadcast.splatinsert10, <4 x i32> undef, <4 x i32> zeroinitializer
+  call void @llvm.set.loop.iterations.i32(i32 %tmp13)
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %tmp14 = phi i32 [ %tmp13, %vector.ph ], [ %tmp15, %vector.body ]
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 1
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+  %tmp = getelementptr inbounds i32, i32* %a, i32 %index
+  %tmp1 = icmp ule <4 x i32> %induction, %broadcast.splat11
+  %tmp2 = bitcast i32* %tmp to <4 x i32>*
+  %wide.masked.load = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp2, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
+  %tmp3 = getelementptr inbounds i32, i32* %b, i32 %index
+  %tmp4 = bitcast i32* %tmp3 to <4 x i32>*
+  %wide.masked.load12 = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp4, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
+  %tmp5 = mul nsw <4 x i32> %wide.masked.load12, %wide.masked.load
+  %tmp6 = getelementptr inbounds i32, i32* %c, i32 %index
+  %tmp7 = bitcast i32* %tmp6 to <4 x i32>*
+  tail call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %tmp5, <4 x i32>* %tmp7, i32 4, <4 x i1> %tmp1)
+  %index.next = add i32 %index, 4
+  %tmp15 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %tmp14, i32 1)
+  %tmp16 = icmp ne i32 %tmp15, 0
+  br i1 %tmp16, label %vector.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %vector.body, %entry
+  ret void
+}
+
+define dso_local arm_aapcs_vfpcc void @wrong_loop_invalid_index_splat(i32* noalias nocapture readonly %a, i32* noalias nocapture readonly %b, i32* noalias nocapture %c, i32 %N) {
+entry:
+  %cmp8 = icmp eq i32 %N, 0
+  %tmp8 = add i32 %N, 3
+  %tmp9 = lshr i32 %tmp8, 2
+  %tmp10 = shl nuw i32 %tmp9, 2
+  %tmp11 = add i32 %tmp10, -4
+  %tmp12 = lshr i32 %tmp11, 2
+  %tmp13 = add nuw nsw i32 %tmp12, 1
+  br i1 %cmp8, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert10 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat11 = shufflevector <4 x i32> %broadcast.splatinsert10, <4 x i32> undef, <4 x i32> zeroinitializer
+  call void @llvm.set.loop.iterations.i32(i32 %tmp13)
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %tmp14 = phi i32 [ %tmp13, %vector.ph ], [ %tmp15, %vector.body ]
+  %incorrect = add i32 %index, 1
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %incorrect, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+  %tmp = getelementptr inbounds i32, i32* %a, i32 %index
+  %tmp1 = icmp ule <4 x i32> %induction, %broadcast.splat11
+  %tmp2 = bitcast i32* %tmp to <4 x i32>*
+  %wide.masked.load = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp2, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
+  %tmp3 = getelementptr inbounds i32, i32* %b, i32 %index
+  %tmp4 = bitcast i32* %tmp3 to <4 x i32>*
+  %wide.masked.load12 = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp4, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
+  %tmp5 = mul nsw <4 x i32> %wide.masked.load12, %wide.masked.load
+  %tmp6 = getelementptr inbounds i32, i32* %c, i32 %index
+  %tmp7 = bitcast i32* %tmp6 to <4 x i32>*
+  tail call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %tmp5, <4 x i32>* %tmp7, i32 4, <4 x i1> %tmp1)
+  %index.next = add i32 %index, 4
+  %tmp15 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %tmp14, i32 1)
+  %tmp16 = icmp ne i32 %tmp15, 0
+  br i1 %tmp16, label %vector.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %vector.body, %entry
+  ret void
+}
+
+; Now using ult, not ule for the vector icmp
+define dso_local arm_aapcs_vfpcc void @wrong_pred_opcode(i32* noalias nocapture readonly %a, i32* noalias nocapture readonly %b, i32* noalias nocapture %c, i32 %N) {
+entry:
+  %cmp8 = icmp eq i32 %N, 0
+  %tmp8 = add i32 %N, 3
+  %tmp9 = lshr i32 %tmp8, 2
+  %tmp10 = shl nuw i32 %tmp9, 2
+  %tmp11 = add i32 %tmp10, -4
+  %tmp12 = lshr i32 %tmp11, 2
+  %tmp13 = add nuw nsw i32 %tmp12, 1
+  br i1 %cmp8, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert10 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat11 = shufflevector <4 x i32> %broadcast.splatinsert10, <4 x i32> undef, <4 x i32> zeroinitializer
+  call void @llvm.set.loop.iterations.i32(i32 %tmp13)
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %tmp14 = phi i32 [ %tmp13, %vector.ph ], [ %tmp15, %vector.body ]
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+  %tmp = getelementptr inbounds i32, i32* %a, i32 %index
+  %tmp1 = icmp ult <4 x i32> %induction, %broadcast.splat11
+  %tmp2 = bitcast i32* %tmp to <4 x i32>*
+  %wide.masked.load = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp2, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
+  %tmp3 = getelementptr inbounds i32, i32* %b, i32 %index
+  %tmp4 = bitcast i32* %tmp3 to <4 x i32>*
+  %wide.masked.load12 = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp4, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
+  %tmp5 = mul nsw <4 x i32> %wide.masked.load12, %wide.masked.load
+  %tmp6 = getelementptr inbounds i32, i32* %c, i32 %index
+  %tmp7 = bitcast i32* %tmp6 to <4 x i32>*
+  tail call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %tmp5, <4 x i32>* %tmp7, i32 4, <4 x i1> %tmp1)
+  %index.next = add i32 %index, 4
+  %tmp15 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %tmp14, i32 1)
+  %tmp16 = icmp ne i32 %tmp15, 0
+  br i1 %tmp16, label %vector.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %vector.body, %entry
+  ret void
+}
+
+; The add in the body uses 1, 2, 3, 4
+define void @wrong_body_broadcast_splat(i32* noalias nocapture readonly %a, i32* noalias nocapture readonly %b, i32* noalias nocapture %c, i32 %N) {
+entry:
+  %cmp8 = icmp eq i32 %N, 0
+  %tmp8 = add i32 %N, 3
+  %tmp9 = lshr i32 %tmp8, 2
+  %tmp10 = shl nuw i32 %tmp9, 2
+  %tmp11 = add i32 %tmp10, -4
+  %tmp12 = lshr i32 %tmp11, 2
+  %tmp13 = add nuw nsw i32 %tmp12, 1
+  br i1 %cmp8, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert10 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat11 = shufflevector <4 x i32> %broadcast.splatinsert10, <4 x i32> undef, <4 x i32> zeroinitializer
+  call void @llvm.set.loop.iterations.i32(i32 %tmp13)
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %tmp14 = phi i32 [ %tmp13, %vector.ph ], [ %tmp15, %vector.body ]
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 1, i32 2, i32 3, i32 4>
+  %tmp = getelementptr inbounds i32, i32* %a, i32 %index
+  %tmp1 = icmp ule <4 x i32> %induction, %broadcast.splat11
+  %tmp2 = bitcast i32* %tmp to <4 x i32>*
+  %wide.masked.load = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp2, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
+  %tmp3 = getelementptr inbounds i32, i32* %b, i32 %index
+  %tmp4 = bitcast i32* %tmp3 to <4 x i32>*
+  %wide.masked.load12 = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp4, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
+  %tmp5 = mul nsw <4 x i32> %wide.masked.load12, %wide.masked.load
+  %tmp6 = getelementptr inbounds i32, i32* %c, i32 %index
+  %tmp7 = bitcast i32* %tmp6 to <4 x i32>*
+  tail call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %tmp5, <4 x i32>* %tmp7, i32 4, <4 x i1> %tmp1)
+  %index.next = add i32 %index, 4
+  %tmp15 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %tmp14, i32 1)
+  %tmp16 = icmp ne i32 %tmp15, 0
+  br i1 %tmp16, label %vector.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %vector.body, %entry
+  ret void
+}
+
+; Using a variable for the loop body broadcast.
+define void @wrong_body_broadcast_splat_2(i32* noalias nocapture readonly %a, i32* noalias nocapture readonly %b, i32* noalias nocapture %c, i32 %N, <4 x i32> %offsets) {
+entry:
+  %cmp8 = icmp eq i32 %N, 0
+  %tmp8 = add i32 %N, 3
+  %tmp9 = lshr i32 %tmp8, 2
+  %tmp10 = shl nuw i32 %tmp9, 2
+  %tmp11 = add i32 %tmp10, -4
+  %tmp12 = lshr i32 %tmp11, 2
+  %tmp13 = add nuw nsw i32 %tmp12, 1
+  br i1 %cmp8, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert10 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat11 = shufflevector <4 x i32> %broadcast.splatinsert10, <4 x i32> undef, <4 x i32> zeroinitializer
+  call void @llvm.set.loop.iterations.i32(i32 %tmp13)
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %tmp14 = phi i32 [ %tmp13, %vector.ph ], [ %tmp15, %vector.body ]
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, %offsets
+  %tmp = getelementptr inbounds i32, i32* %a, i32 %index
+  %tmp1 = icmp ule <4 x i32> %induction, %broadcast.splat11
+  %tmp2 = bitcast i32* %tmp to <4 x i32>*
+  %wide.masked.load = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp2, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
+  %tmp3 = getelementptr inbounds i32, i32* %b, i32 %index
+  %tmp4 = bitcast i32* %tmp3 to <4 x i32>*
+  %wide.masked.load12 = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp4, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
+  %tmp5 = mul nsw <4 x i32> %wide.masked.load12, %wide.masked.load
+  %tmp6 = getelementptr inbounds i32, i32* %c, i32 %index
+  %tmp7 = bitcast i32* %tmp6 to <4 x i32>*
+  tail call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %tmp5, <4 x i32>* %tmp7, i32 4, <4 x i1> %tmp1)
+  %index.next = add i32 %index, 4
+  %tmp15 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %tmp14, i32 1)
+  %tmp16 = icmp ne i32 %tmp15, 0
+  br i1 %tmp16, label %vector.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %vector.body, %entry
+  ret void
+}
+
+; adding 5, instead of 4, to index. 
+define void @wrong_index_add(i32* noalias nocapture readonly %a, i32* noalias nocapture readonly %b, i32* noalias nocapture %c, i32 %N) {
+entry:
+  %cmp8 = icmp eq i32 %N, 0
+  %tmp8 = add i32 %N, 3
+  %tmp9 = lshr i32 %tmp8, 2
+  %tmp10 = shl nuw i32 %tmp9, 2
+  %tmp11 = add i32 %tmp10, -4
+  %tmp12 = lshr i32 %tmp11, 2
+  %tmp13 = add nuw nsw i32 %tmp12, 1
+  br i1 %cmp8, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert10 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat11 = shufflevector <4 x i32> %broadcast.splatinsert10, <4 x i32> undef, <4 x i32> zeroinitializer
+  call void @llvm.set.loop.iterations.i32(i32 %tmp13)
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %tmp14 = phi i32 [ %tmp13, %vector.ph ], [ %tmp15, %vector.body ]
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+  %tmp = getelementptr inbounds i32, i32* %a, i32 %index
+  %tmp1 = icmp ule <4 x i32> %induction, %broadcast.splat11
+  %tmp2 = bitcast i32* %tmp to <4 x i32>*
+  %wide.masked.load = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp2, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
+  %tmp3 = getelementptr inbounds i32, i32* %b, i32 %index
+  %tmp4 = bitcast i32* %tmp3 to <4 x i32>*
+  %wide.masked.load12 = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp4, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
+  %tmp5 = mul nsw <4 x i32> %wide.masked.load12, %wide.masked.load
+  %tmp6 = getelementptr inbounds i32, i32* %c, i32 %index
+  %tmp7 = bitcast i32* %tmp6 to <4 x i32>*
+  tail call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %tmp5, <4 x i32>* %tmp7, i32 4, <4 x i1> %tmp1)
+  %index.next = add i32 %index, 5
+  %tmp15 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %tmp14, i32 1)
+  %tmp16 = icmp ne i32 %tmp15, 0
+  br i1 %tmp16, label %vector.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %vector.body, %entry
+  ret void
+}
+
+declare <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>*, i32 immarg, <4 x i1>, <4 x i32>) #1
+declare void @llvm.masked.store.v4i32.p0v4i32(<4 x i32>, <4 x i32>*, i32 immarg, <4 x i1>) #2
+declare void @llvm.set.loop.iterations.i32(i32) #3
+declare i32 @llvm.loop.decrement.reg.i32.i32.i32(i32, i32) #3
+
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-pred-widen.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-pred-widen.ll
new file mode 100644
index 000000000..330c6db24
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-pred-widen.ll
@@ -0,0 +1,173 @@
+; RUN: opt -mtriple=thumbv8.1m.main -mve-tail-predication -disable-mve-tail-predication=false -mattr=+mve,+lob %s -S -o - | FileCheck %s
+
+; CHECK-LABEL: expand_v8i16_v8i32
+; CHECK-NOT: call i32 @llvm.arm.mve.vctp
+define void @expand_v8i16_v8i32(i16* noalias nocapture readonly %a, i16* noalias nocapture readonly %b, i32* noalias nocapture %c, i32 %N) {
+entry:
+  %cmp8 = icmp eq i32 %N, 0
+  %tmp8 = add i32 %N, 7
+  %tmp9 = lshr i32 %tmp8, 3
+  %tmp10 = shl nuw i32 %tmp9, 3
+  %tmp11 = add i32 %tmp10, -8
+  %tmp12 = lshr i32 %tmp11, 3
+  %tmp13 = add nuw nsw i32 %tmp12, 1
+  br i1 %cmp8, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert10 = insertelement <8 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat11 = shufflevector <8 x i32> %broadcast.splatinsert10, <8 x i32> undef, <8 x i32> zeroinitializer
+  call void @llvm.set.loop.iterations.i32(i32 %tmp13)
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %tmp14 = phi i32 [ %tmp13, %vector.ph ], [ %tmp15, %vector.body ]
+  %broadcast.splatinsert = insertelement <8 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <8 x i32> %broadcast.splatinsert, <8 x i32> undef, <8 x i32> zeroinitializer
+  %induction = add <8 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %tmp = getelementptr inbounds i16, i16* %a, i32 %index
+  %tmp1 = icmp ule <8 x i32> %induction, %broadcast.splat11
+  %tmp2 = bitcast i16* %tmp to <8 x i16>*
+  %wide.masked.load = tail call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %tmp2, i32 4, <8 x i1> %tmp1, <8 x i16> undef)
+  %tmp3 = getelementptr inbounds i16, i16* %b, i32 %index
+  %tmp4 = bitcast i16* %tmp3 to <8 x i16>*
+  %wide.masked.load2 = tail call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %tmp4, i32 4, <8 x i1> %tmp1, <8 x i16> undef)
+  %expand.1 = zext <8 x i16> %wide.masked.load to <8 x i32>
+  %expand.2 = zext <8 x i16> %wide.masked.load2 to <8 x i32>
+  %mul = mul nsw <8 x i32> %expand.2, %expand.1
+  %tmp6 = getelementptr inbounds i32, i32* %c, i32 %index
+  %tmp7 = bitcast i32* %tmp6 to <8 x i32>*
+  tail call void @llvm.masked.store.v8i32.p0v8i32(<8 x i32> %mul, <8 x i32>* %tmp7, i32 4, <8 x i1> %tmp1)
+  %index.next = add i32 %index, 8
+  %tmp15 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %tmp14, i32 1)
+  %tmp16 = icmp ne i32 %tmp15, 0
+  br i1 %tmp16, label %vector.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %vector.body, %entry
+  ret void
+}
+
+; CHECK-LABEL: expand_v8i16_v4i32
+; CHECK: [[ELEMS:%[^ ]+]] = phi i32 [ %N, %vector.ph ], [ [[ELEMS_REM:%[^ ]+]], %vector.body ]
+; CHECK: [[VCTP:%[^ ]+]] = call <8 x i1> @llvm.arm.mve.vctp16(i32 [[ELEMS]])
+; CHECK: [[ELEMS_REM]] = sub i32 [[ELEMS]], 8
+; CHECK: tail call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* {{.*}}, i32 4, <8 x i1> [[VCTP]], <8 x i16> undef)
+; CHECK: %store.pred = icmp ule <4 x i32> %induction.store
+; CHECK: tail call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> {{.*}}, <4 x i32>* {{.*}}, i32 4, <4 x i1> %store.pred)
+; CHECK: tail call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> {{.*}}, <4 x i32>* {{.*}}, i32 4, <4 x i1> %store.pred)
+define void @expand_v8i16_v4i32(i16* readonly %a, i16* readonly %b, i32* %c, i32* %d, i32 %N) {
+entry:
+  %cmp8 = icmp eq i32 %N, 0
+  %tmp8 = add i32 %N, 7
+  %tmp9 = lshr i32 %tmp8, 3
+  %tmp10 = shl nuw i32 %tmp9, 3
+  %tmp11 = add i32 %tmp10, -8
+  %tmp12 = lshr i32 %tmp11, 3
+  %tmp13 = add nuw nsw i32 %tmp12, 1
+  br i1 %cmp8, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert10 = insertelement <8 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat11 = shufflevector <8 x i32> %broadcast.splatinsert10, <8 x i32> undef, <8 x i32> zeroinitializer
+  %broadcast.splatinsert10.store = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat11.store = shufflevector <4 x i32> %broadcast.splatinsert10.store, <4 x i32> undef, <4 x i32> zeroinitializer
+  call void @llvm.set.loop.iterations.i32(i32 %tmp13)
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %store.idx = phi i32 [ 0, %vector.ph ], [ %store.idx.next, %vector.body ]
+  %tmp14 = phi i32 [ %tmp13, %vector.ph ], [ %tmp15, %vector.body ]
+  %broadcast.splatinsert = insertelement <8 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <8 x i32> %broadcast.splatinsert, <8 x i32> undef, <8 x i32> zeroinitializer
+  %induction = add <8 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %tmp = getelementptr inbounds i16, i16* %a, i32 %index
+  %tmp1 = icmp ule <8 x i32> %induction, %broadcast.splat11
+  %tmp2 = bitcast i16* %tmp to <8 x i16>*
+  %wide.masked.load = tail call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %tmp2, i32 4, <8 x i1> %tmp1, <8 x i16> undef)
+  %tmp3 = getelementptr inbounds i16, i16* %b, i32 %index
+  %tmp4 = bitcast i16* %tmp3 to <8 x i16>*
+  %wide.masked.load2 = tail call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %tmp4, i32 4, <8 x i1> %tmp1, <8 x i16> undef)
+  %extract.2.low = shufflevector <8 x i16> %wide.masked.load2, <8 x i16> undef, < 4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %extract.2.high = shufflevector <8 x i16> %wide.masked.load2, <8 x i16> undef, < 4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %expand.1 = zext <4 x i16> %extract.2.low to <4 x i32>
+  %expand.2 = zext <4 x i16> %extract.2.high to <4 x i32>
+  %mul = mul nsw <4 x i32> %expand.2, %expand.1
+  %sub = mul nsw <4 x i32> %expand.1, %expand.2
+  %broadcast.splatinsert.store = insertelement <4 x i32> undef, i32 %store.idx, i32 0
+  %broadcast.splat.store = shufflevector <4 x i32> %broadcast.splatinsert.store, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction.store = add <4 x i32> %broadcast.splat.store, <i32 0, i32 1, i32 2, i32 3>
+  %store.pred = icmp ule <4 x i32> %induction.store, %broadcast.splat11.store
+  %tmp6 = getelementptr inbounds i32, i32* %c, i32 %store.idx
+  %tmp7 = bitcast i32* %tmp6 to <4 x i32>*
+  tail call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %mul, <4 x i32>* %tmp7, i32 4, <4 x i1> %store.pred)
+  %gep = getelementptr inbounds i32, i32* %d, i32 %store.idx
+  %cast.gep = bitcast i32* %gep to <4 x i32>*
+  tail call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %sub, <4 x i32>* %cast.gep, i32 4, <4 x i1> %store.pred)
+  %store.idx.next = add i32 %store.idx, 4
+  %index.next = add i32 %index, 8
+  %tmp15 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %tmp14, i32 1)
+  %tmp16 = icmp ne i32 %tmp15, 0
+  br i1 %tmp16, label %vector.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %vector.body, %entry
+  ret void
+}
+
+; CHECK-LABEL: expand_v4i32_v4i64
+; CHECK-NOT: call i32 @llvm.arm.mve.vctp
+define void @expand_v4i32_v4i64(i32* noalias nocapture readonly %a, i32* noalias nocapture readonly %b, i64* noalias nocapture %c, i32 %N) {
+entry:
+  %cmp8 = icmp eq i32 %N, 0
+  %tmp8 = add i32 %N, 3
+  %tmp9 = lshr i32 %tmp8, 2
+  %tmp10 = shl nuw i32 %tmp9, 2
+  %tmp11 = add i32 %tmp10, -4
+  %tmp12 = lshr i32 %tmp11, 2
+  %tmp13 = add nuw nsw i32 %tmp12, 1
+  br i1 %cmp8, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert10 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat11 = shufflevector <4 x i32> %broadcast.splatinsert10, <4 x i32> undef, <4 x i32> zeroinitializer
+  call void @llvm.set.loop.iterations.i32(i32 %tmp13)
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %tmp14 = phi i32 [ %tmp13, %vector.ph ], [ %tmp15, %vector.body ]
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+  %tmp = getelementptr inbounds i32, i32* %a, i32 %index
+  %tmp1 = icmp ule <4 x i32> %induction, %broadcast.splat11
+  %tmp2 = bitcast i32* %tmp to <4 x i32>*
+  %wide.masked.load = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp2, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
+  %tmp3 = getelementptr inbounds i32, i32* %b, i32 %index
+  %tmp4 = bitcast i32* %tmp3 to <4 x i32>*
+  %wide.masked.load2 = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp4, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
+  %expand.1 = zext <4 x i32> %wide.masked.load to <4 x i64>
+  %expand.2 = zext <4 x i32> %wide.masked.load2 to <4 x i64>
+  %mul = mul nsw <4 x i64> %expand.2, %expand.1
+  %tmp6 = getelementptr inbounds i64, i64* %c, i32 %index
+  %tmp7 = bitcast i64* %tmp6 to <4 x i64>*
+  tail call void @llvm.masked.store.v4i64.p0v4i64(<4 x i64> %mul, <4 x i64>* %tmp7, i32 4, <4 x i1> %tmp1)
+  %index.next = add i32 %index, 4
+  %tmp15 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %tmp14, i32 1)
+  %tmp16 = icmp ne i32 %tmp15, 0
+  br i1 %tmp16, label %vector.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %vector.body, %entry
+  ret void
+}
+
+declare <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>*, i32 immarg, <8 x i1>, <8 x i16>)
+declare void @llvm.masked.store.v8i32.p0v8i32(<8 x i32>, <8 x i32>*, i32 immarg, <8 x i1>)
+declare <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>*, i32 immarg, <4 x i1>, <4 x i32>)
+declare void @llvm.masked.store.v4i32.p0v4i32(<4 x i32>, <4 x i32>*, i32 immarg, <4 x i1>)
+declare void @llvm.masked.store.v4i64.p0v4i64(<4 x i64>, <4 x i64>*, i32 immarg, <4 x i1>)
+declare void @llvm.set.loop.iterations.i32(i32)
+declare i32 @llvm.loop.decrement.reg.i32.i32.i32(i32, i32)
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-reduce.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-reduce.ll
new file mode 100644
index 000000000..c7ed9ce67
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-reduce.ll
@@ -0,0 +1,118 @@
+; RUN: opt -mtriple=thumbv8.1m.main -mve-tail-predication -disable-mve-tail-predication=false -mattr=+mve %s -S -o - | FileCheck %s
+
+; CHECK-LABEL: reduction_i32
+; CHECK: phi i32 [ 0, %entry ]
+; CHECK: phi <8 x i16> [ zeroinitializer, %entry ]
+; CHECK: phi i32
+; CHECK: [[PHI:%[^ ]+]] = phi i32 [ %N, %entry ], [ [[ELEMS:%[^ ]+]], %vector.body ]
+; CHECK: [[VCTP:%[^ ]+]] = call <8 x i1> @llvm.arm.mve.vctp16(i32 [[PHI]])
+; CHECK: [[ELEMS]] = sub i32 [[PHI]], 8
+; CHECK: call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %tmp4, i32 4, <8 x i1> [[VCTP]], <8 x i16> undef)
+; CHECK: call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %tmp6, i32 4, <8 x i1> [[VCTP]], <8 x i16> undef)
+define i16 @reduction_i32(i16* nocapture readonly %A, i16* nocapture readonly %B, i32 %N) {
+entry:
+  %tmp = add i32 %N, -1
+  %n.rnd.up = add nuw nsw i32 %tmp, 8
+  %n.vec = and i32 %n.rnd.up, -8
+  %broadcast.splatinsert1 = insertelement <8 x i32> undef, i32 %tmp, i32 0
+  %broadcast.splat2 = shufflevector <8 x i32> %broadcast.splatinsert1, <8 x i32> undef, <8 x i32> zeroinitializer
+  %0 = add i32 %n.vec, -8
+  %1 = lshr i32 %0, 3
+  %2 = add nuw nsw i32 %1, 1
+  call void @llvm.set.loop.iterations.i32(i32 %2)
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %entry
+  %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]
+  %vec.phi = phi <8 x i16> [ zeroinitializer, %entry ], [ %tmp8, %vector.body ]
+  %3 = phi i32 [ %2, %entry ], [ %4, %vector.body ]
+  %broadcast.splatinsert = insertelement <8 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <8 x i32> %broadcast.splatinsert, <8 x i32> undef, <8 x i32> zeroinitializer
+  %induction = add <8 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %tmp2 = getelementptr inbounds i16, i16* %A, i32 %index
+  %tmp3 = icmp ule <8 x i32> %induction, %broadcast.splat2
+  %tmp4 = bitcast i16* %tmp2 to <8 x i16>*
+  %wide.masked.load = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %tmp4, i32 4, <8 x i1> %tmp3, <8 x i16> undef)
+  %tmp5 = getelementptr inbounds i16, i16* %B, i32 %index
+  %tmp6 = bitcast i16* %tmp5 to <8 x i16>*
+  %wide.masked.load3 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %tmp6, i32 4, <8 x i1> %tmp3, <8 x i16> undef)
+  %tmp7 = add <8 x i16> %wide.masked.load, %vec.phi
+  %tmp8 = add <8 x i16> %tmp7, %wide.masked.load3
+  %index.next = add nuw nsw i32 %index, 8
+  %4 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %3, i32 1)
+  %5 = icmp ne i32 %4, 0
+  br i1 %5, label %vector.body, label %middle.block
+
+middle.block:                                     ; preds = %vector.body
+  %vec.phi.lcssa = phi <8 x i16> [ %vec.phi, %vector.body ]
+  %.lcssa3 = phi <8 x i1> [ %tmp3, %vector.body ]
+  %.lcssa = phi <8 x i16> [ %tmp8, %vector.body ]
+  %tmp10 = select <8 x i1> %.lcssa3, <8 x i16> %.lcssa, <8 x i16> %vec.phi.lcssa
+  %rdx.shuf = shufflevector <8 x i16> %tmp10, <8 x i16> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+  %bin.rdx = add <8 x i16> %rdx.shuf, %tmp10
+  %rdx.shuf4 = shufflevector <8 x i16> %bin.rdx, <8 x i16> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %bin.rdx5 = add <8 x i16> %rdx.shuf4, %bin.rdx
+  %rdx.shuf6 = shufflevector <8 x i16> %bin.rdx5, <8 x i16> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %bin.rdx7 = add <8 x i16> %rdx.shuf6, %bin.rdx5
+  %tmp11 = extractelement <8 x i16> %bin.rdx7, i32 0
+  ret i16 %tmp11
+}
+
+; CHECK-LABEL: reduction_i32_with_scalar
+; CHECK: phi i32 [ 0, %entry ]
+; CHECK: phi <8 x i16> [ zeroinitializer, %entry ]
+; CHECK: phi i32
+; CHECK: [[PHI:%[^ ]+]] = phi i32 [ %N, %entry ], [ [[ELEMS:%[^ ]+]], %vector.body ]
+; CHECK: [[VCTP:%[^ ]+]] = call <8 x i1> @llvm.arm.mve.vctp16(i32 [[PHI]])
+; CHECK: [[ELEMS]] = sub i32 [[PHI]], 8
+; CHECK: call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %tmp4, i32 4, <8 x i1> [[VCTP]], <8 x i16> undef)
+define i16 @reduction_i32_with_scalar(i16* nocapture readonly %A, i16 %B, i32 %N) local_unnamed_addr {
+entry:
+  %tmp = add i32 %N, -1
+  %n.rnd.up = add nuw nsw i32 %tmp, 8
+  %n.vec = and i32 %n.rnd.up, -8
+  %broadcast.splatinsert1 = insertelement <8 x i32> undef, i32 %tmp, i32 0
+  %broadcast.splat2 = shufflevector <8 x i32> %broadcast.splatinsert1, <8 x i32> undef, <8 x i32> zeroinitializer
+  %broadcast.splatinsert3 = insertelement <8 x i16> undef, i16 %B, i32 0
+  %broadcast.splat4 = shufflevector <8 x i16> %broadcast.splatinsert3, <8 x i16> undef, <8 x i32> zeroinitializer
+  %0 = add i32 %n.vec, -8
+  %1 = lshr i32 %0, 3
+  %2 = add nuw nsw i32 %1, 1
+  call void @llvm.set.loop.iterations.i32(i32 %2)
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %entry
+  %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]
+  %vec.phi = phi <8 x i16> [ zeroinitializer, %entry ], [ %tmp6, %vector.body ]
+  %3 = phi i32 [ %2, %entry ], [ %4, %vector.body ]
+  %broadcast.splatinsert = insertelement <8 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <8 x i32> %broadcast.splatinsert, <8 x i32> undef, <8 x i32> zeroinitializer
+  %induction = add <8 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %tmp2 = getelementptr inbounds i16, i16* %A, i32 %index
+  %tmp3 = icmp ule <8 x i32> %induction, %broadcast.splat2
+  %tmp4 = bitcast i16* %tmp2 to <8 x i16>*
+  %wide.masked.load = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %tmp4, i32 4, <8 x i1> %tmp3, <8 x i16> undef)
+  %tmp5 = add <8 x i16> %vec.phi, %broadcast.splat4
+  %tmp6 = add <8 x i16> %tmp5, %wide.masked.load
+  %index.next = add nuw nsw i32 %index, 8
+  %4 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %3, i32 1)
+  %5 = icmp ne i32 %4, 0
+  br i1 %5, label %vector.body, label %middle.block
+
+middle.block:                                     ; preds = %vector.body
+  %tmp8 = select <8 x i1> %tmp3, <8 x i16> %tmp6, <8 x i16> %vec.phi
+  %rdx.shuf = shufflevector <8 x i16> %tmp8, <8 x i16> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+  %bin.rdx = add <8 x i16> %rdx.shuf, %tmp8
+  %rdx.shuf5 = shufflevector <8 x i16> %bin.rdx, <8 x i16> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %bin.rdx6 = add <8 x i16> %rdx.shuf5, %bin.rdx
+  %rdx.shuf7 = shufflevector <8 x i16> %bin.rdx6, <8 x i16> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %bin.rdx8 = add <8 x i16> %rdx.shuf7, %bin.rdx6
+  %tmp9 = extractelement <8 x i16> %bin.rdx8, i32 0
+  ret i16 %tmp9
+}
+
+declare <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>*, i32 immarg, <8 x i1>, <8 x i16>)
+declare void @llvm.set.loop.iterations.i32(i32)
+declare i32 @llvm.loop.decrement.reg.i32.i32.i32(i32, i32)
+
+
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vector-arith-codegen.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vector-arith-codegen.ll
new file mode 100644
index 000000000..bb2973935
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vector-arith-codegen.ll
@@ -0,0 +1,411 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=armv8.1m.main -mattr=+mve -disable-mve-tail-predication=false --verify-machineinstrs %s -o - | FileCheck %s
+
+define dso_local i32 @mul_reduce_add(i32* noalias nocapture readonly %a, i32* noalias nocapture readonly %b, i32 %N) {
+; CHECK-LABEL: mul_reduce_add:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    itt eq
+; CHECK-NEXT:    moveq r0, #0
+; CHECK-NEXT:    bxeq lr
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    vmov.i32 q1, #0x0
+; CHECK-NEXT:    dlstp.32 lr, r2
+; CHECK-NEXT:  .LBB0_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    vldrw.u32 q1, [r0], #16
+; CHECK-NEXT:    vldrw.u32 q2, [r1], #16
+; CHECK-NEXT:    mov r3, r2
+; CHECK-NEXT:    vmul.i32 q1, q2, q1
+; CHECK-NEXT:    subs r2, #4
+; CHECK-NEXT:    vadd.i32 q1, q1, q0
+; CHECK-NEXT:    letp lr, .LBB0_1
+; CHECK-NEXT:  @ %bb.2: @ %middle.block
+; CHECK-NEXT:    vctp.32 r3
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    vaddv.u32 r0, q0
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %cmp8 = icmp eq i32 %N, 0
+  br i1 %cmp8, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %n.rnd.up = add i32 %N, 3
+  %n.vec = and i32 %n.rnd.up, -4
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert11 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat12 = shufflevector <4 x i32> %broadcast.splatinsert11, <4 x i32> undef, <4 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %vec.phi = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ %6, %vector.body ]
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+  %0 = getelementptr inbounds i32, i32* %a, i32 %index
+  %1 = icmp ule <4 x i32> %induction, %broadcast.splat12
+  %2 = bitcast i32* %0 to <4 x i32>*
+  %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %2, i32 4, <4 x i1> %1, <4 x i32> undef)
+  %3 = getelementptr inbounds i32, i32* %b, i32 %index
+  %4 = bitcast i32* %3 to <4 x i32>*
+  %wide.masked.load13 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %4, i32 4, <4 x i1> %1, <4 x i32> undef)
+  %5 = mul nsw <4 x i32> %wide.masked.load13, %wide.masked.load
+  %6 = add nsw <4 x i32> %5, %vec.phi
+  %index.next = add i32 %index, 4
+  %7 = icmp eq i32 %index.next, %n.vec
+  br i1 %7, label %middle.block, label %vector.body
+
+middle.block:                                     ; preds = %vector.body
+  %8 = select <4 x i1> %1, <4 x i32> %6, <4 x i32> %vec.phi
+  %9 = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %8)
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %middle.block, %entry
+  %res.0.lcssa = phi i32 [ 0, %entry ], [ %9, %middle.block ]
+  ret i32 %res.0.lcssa
+}
+
+define dso_local i32 @mul_reduce_add_const(i32* noalias nocapture readonly %a, i32 %b, i32 %N) {
+; CHECK-LABEL: mul_reduce_add_const:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    itt eq
+; CHECK-NEXT:    moveq r0, #0
+; CHECK-NEXT:    bxeq lr
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    vmov.i32 q0, #0x0
+; CHECK-NEXT:    dlstp.32 lr, r2
+; CHECK-NEXT:  .LBB1_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    mov r1, r2
+; CHECK-NEXT:    vmov q1, q0
+; CHECK-NEXT:    vldrw.u32 q0, [r0], #16
+; CHECK-NEXT:    subs r2, #4
+; CHECK-NEXT:    vadd.i32 q0, q0, q1
+; CHECK-NEXT:    letp lr, .LBB1_1
+; CHECK-NEXT:  @ %bb.2: @ %middle.block
+; CHECK-NEXT:    vctp.32 r1
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    vaddv.u32 r0, q0
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %cmp6 = icmp eq i32 %N, 0
+  br i1 %cmp6, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %n.rnd.up = add i32 %N, 3
+  %n.vec = and i32 %n.rnd.up, -4
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert9 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat10 = shufflevector <4 x i32> %broadcast.splatinsert9, <4 x i32> undef, <4 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %vec.phi = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ %3, %vector.body ]
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+  %0 = getelementptr inbounds i32, i32* %a, i32 %index
+  %1 = icmp ule <4 x i32> %induction, %broadcast.splat10
+  %2 = bitcast i32* %0 to <4 x i32>*
+  %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %2, i32 4, <4 x i1> %1, <4 x i32> undef)
+  %3 = add nsw <4 x i32> %wide.masked.load, %vec.phi
+  %index.next = add i32 %index, 4
+  %4 = icmp eq i32 %index.next, %n.vec
+  br i1 %4, label %middle.block, label %vector.body
+
+middle.block:                                     ; preds = %vector.body
+  %5 = select <4 x i1> %1, <4 x i32> %3, <4 x i32> %vec.phi
+  %6 = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %5)
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %middle.block, %entry
+  %res.0.lcssa = phi i32 [ 0, %entry ], [ %6, %middle.block ]
+  ret i32 %res.0.lcssa
+}
+
+define dso_local i32 @add_reduce_add_const(i32* noalias nocapture readonly %a, i32 %b, i32 %N) {
+; CHECK-LABEL: add_reduce_add_const:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    itt eq
+; CHECK-NEXT:    moveq r0, #0
+; CHECK-NEXT:    bxeq lr
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    vmov.i32 q0, #0x0
+; CHECK-NEXT:    dlstp.32 lr, r2
+; CHECK-NEXT:  .LBB2_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    mov r1, r2
+; CHECK-NEXT:    vmov q1, q0
+; CHECK-NEXT:    vldrw.u32 q0, [r0], #16
+; CHECK-NEXT:    subs r2, #4
+; CHECK-NEXT:    vadd.i32 q0, q0, q1
+; CHECK-NEXT:    letp lr, .LBB2_1
+; CHECK-NEXT:  @ %bb.2: @ %middle.block
+; CHECK-NEXT:    vctp.32 r1
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    vaddv.u32 r0, q0
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %cmp6 = icmp eq i32 %N, 0
+  br i1 %cmp6, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %n.rnd.up = add i32 %N, 3
+  %n.vec = and i32 %n.rnd.up, -4
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert9 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat10 = shufflevector <4 x i32> %broadcast.splatinsert9, <4 x i32> undef, <4 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %vec.phi = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ %3, %vector.body ]
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+  %0 = getelementptr inbounds i32, i32* %a, i32 %index
+  %1 = icmp ule <4 x i32> %induction, %broadcast.splat10
+  %2 = bitcast i32* %0 to <4 x i32>*
+  %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %2, i32 4, <4 x i1> %1, <4 x i32> undef)
+  %3 = add nsw <4 x i32> %wide.masked.load, %vec.phi
+  %index.next = add i32 %index, 4
+  %4 = icmp eq i32 %index.next, %n.vec
+  br i1 %4, label %middle.block, label %vector.body
+
+middle.block:                                     ; preds = %vector.body
+  %5 = select <4 x i1> %1, <4 x i32> %3, <4 x i32> %vec.phi
+  %6 = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %5)
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %middle.block, %entry
+  %res.0.lcssa = phi i32 [ 0, %entry ], [ %6, %middle.block ]
+  ret i32 %res.0.lcssa
+}
+
+define dso_local void @vector_mul_const(i32* noalias nocapture %a, i32* noalias nocapture readonly %b, i32 %c, i32 %N) {
+; CHECK-LABEL: vector_mul_const:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    cmp r3, #0
+; CHECK-NEXT:    it eq
+; CHECK-NEXT:    popeq {r7, pc}
+; CHECK-NEXT:    dlstp.32 lr, r3
+; CHECK-NEXT:  .LBB3_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrw.u32 q0, [r1], #16
+; CHECK-NEXT:    vmul.i32 q0, q0, r2
+; CHECK-NEXT:    vstrw.32 q0, [r0], #16
+; CHECK-NEXT:    letp lr, .LBB3_1
+; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %cmp6 = icmp eq i32 %N, 0
+  br i1 %cmp6, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %n.rnd.up = add i32 %N, 3
+  %n.vec = and i32 %n.rnd.up, -4
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert8 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat9 = shufflevector <4 x i32> %broadcast.splatinsert8, <4 x i32> undef, <4 x i32> zeroinitializer
+  %broadcast.splatinsert10 = insertelement <4 x i32> undef, i32 %c, i32 0
+  %broadcast.splat11 = shufflevector <4 x i32> %broadcast.splatinsert10, <4 x i32> undef, <4 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+  %0 = getelementptr inbounds i32, i32* %b, i32 %index
+  %1 = icmp ule <4 x i32> %induction, %broadcast.splat9
+  %2 = bitcast i32* %0 to <4 x i32>*
+  %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %2, i32 4, <4 x i1> %1, <4 x i32> undef)
+  %3 = mul nsw <4 x i32> %wide.masked.load, %broadcast.splat11
+  %4 = getelementptr inbounds i32, i32* %a, i32 %index
+  %5 = bitcast i32* %4 to <4 x i32>*
+  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %3, <4 x i32>* %5, i32 4, <4 x i1> %1)
+  %index.next = add i32 %index, 4
+  %6 = icmp eq i32 %index.next, %n.vec
+  br i1 %6, label %for.cond.cleanup, label %vector.body
+
+for.cond.cleanup:                                 ; preds = %vector.body, %entry
+  ret void
+}
+
+define dso_local void @vector_add_const(i32* noalias nocapture %a, i32* noalias nocapture readonly %b, i32 %c, i32 %N) {
+; CHECK-LABEL: vector_add_const:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    cmp r3, #0
+; CHECK-NEXT:    it eq
+; CHECK-NEXT:    popeq {r7, pc}
+; CHECK-NEXT:    dlstp.32 lr, r3
+; CHECK-NEXT:  .LBB4_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrw.u32 q0, [r1], #16
+; CHECK-NEXT:    vadd.i32 q0, q0, r2
+; CHECK-NEXT:    vstrw.32 q0, [r0], #16
+; CHECK-NEXT:    letp lr, .LBB4_1
+; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %cmp6 = icmp eq i32 %N, 0
+  br i1 %cmp6, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %n.rnd.up = add i32 %N, 3
+  %n.vec = and i32 %n.rnd.up, -4
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert8 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat9 = shufflevector <4 x i32> %broadcast.splatinsert8, <4 x i32> undef, <4 x i32> zeroinitializer
+  %broadcast.splatinsert10 = insertelement <4 x i32> undef, i32 %c, i32 0
+  %broadcast.splat11 = shufflevector <4 x i32> %broadcast.splatinsert10, <4 x i32> undef, <4 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+  %0 = getelementptr inbounds i32, i32* %b, i32 %index
+  %1 = icmp ule <4 x i32> %induction, %broadcast.splat9
+  %2 = bitcast i32* %0 to <4 x i32>*
+  %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %2, i32 4, <4 x i1> %1, <4 x i32> undef)
+  %3 = add nsw <4 x i32> %wide.masked.load, %broadcast.splat11
+  %4 = getelementptr inbounds i32, i32* %a, i32 %index
+  %5 = bitcast i32* %4 to <4 x i32>*
+  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %3, <4 x i32>* %5, i32 4, <4 x i1> %1)
+  %index.next = add i32 %index, 4
+  %6 = icmp eq i32 %index.next, %n.vec
+  br i1 %6, label %for.cond.cleanup, label %vector.body
+
+for.cond.cleanup:                                 ; preds = %vector.body, %entry
+  ret void
+}
+
+define dso_local arm_aapcs_vfpcc void @vector_mul_vector_i8(i8* noalias nocapture %a, i8* noalias nocapture readonly %b, i8* noalias nocapture readonly %c, i32 %N) {
+; CHECK-LABEL: vector_mul_vector_i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    push {r4, lr}
+; CHECK-NEXT:    cmp r3, #0
+; CHECK-NEXT:    it eq
+; CHECK-NEXT:    popeq {r4, pc}
+; CHECK-NEXT:    mov.w r12, #0
+; CHECK-NEXT:    dlstp.8 lr, r3
+; CHECK-NEXT:  .LBB5_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    add.w r4, r1, r12
+; CHECK-NEXT:    vldrb.u8 q0, [r4]
+; CHECK-NEXT:    add.w r4, r2, r12
+; CHECK-NEXT:    vldrb.u8 q1, [r4]
+; CHECK-NEXT:    add.w r4, r0, r12
+; CHECK-NEXT:    add.w r12, r12, #16
+; CHECK-NEXT:    vmul.i8 q0, q1, q0
+; CHECK-NEXT:    vstrb.8 q0, [r4]
+; CHECK-NEXT:    letp lr, .LBB5_1
+; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
+; CHECK-NEXT:    pop {r4, pc}
+entry:
+  %cmp10 = icmp eq i32 %N, 0
+  br i1 %cmp10, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %n.rnd.up = add i32 %N, 15
+  %n.vec = and i32 %n.rnd.up, -16
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert12 = insertelement <16 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat13 = shufflevector <16 x i32> %broadcast.splatinsert12, <16 x i32> undef, <16 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %broadcast.splatinsert = insertelement <16 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <16 x i32> %broadcast.splatinsert, <16 x i32> undef, <16 x i32> zeroinitializer
+  %induction = add <16 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %0 = getelementptr inbounds i8, i8* %b, i32 %index
+  %1 = icmp ule <16 x i32> %induction, %broadcast.splat13
+  %2 = bitcast i8* %0 to <16 x i8>*
+  %wide.masked.load = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %2, i32 1, <16 x i1> %1, <16 x i8> undef)
+  %3 = getelementptr inbounds i8, i8* %c, i32 %index
+  %4 = bitcast i8* %3 to <16 x i8>*
+  %wide.masked.load14 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %4, i32 1, <16 x i1> %1, <16 x i8> undef)
+  %5 = mul <16 x i8> %wide.masked.load14, %wide.masked.load
+  %6 = getelementptr inbounds i8, i8* %a, i32 %index
+  %7 = bitcast i8* %6 to <16 x i8>*
+  call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %5, <16 x i8>* %7, i32 1, <16 x i1> %1)
+  %index.next = add i32 %index, 16
+  %8 = icmp eq i32 %index.next, %n.vec
+  br i1 %8, label %for.cond.cleanup, label %vector.body
+
+for.cond.cleanup:                                 ; preds = %vector.body, %entry
+  ret void
+}
+
+; Function Attrs: nofree norecurse nounwind
+define dso_local arm_aapcs_vfpcc void @vector_mul_vector_i16(i16* noalias nocapture %a, i16* noalias nocapture readonly %b, i16* noalias nocapture readonly %c, i32 %N) local_unnamed_addr #0 {
+; CHECK-LABEL: vector_mul_vector_i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    cmp r3, #0
+; CHECK-NEXT:    it eq
+; CHECK-NEXT:    popeq {r7, pc}
+; CHECK-NEXT:    dlstp.16 lr, r3
+; CHECK-NEXT:  .LBB6_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrh.u16 q0, [r1], #16
+; CHECK-NEXT:    vldrh.u16 q1, [r2], #16
+; CHECK-NEXT:    vmul.i16 q0, q1, q0
+; CHECK-NEXT:    vstrh.16 q0, [r0], #16
+; CHECK-NEXT:    letp lr, .LBB6_1
+; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %cmp10 = icmp eq i32 %N, 0
+  br i1 %cmp10, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %n.rnd.up = add i32 %N, 7
+  %n.vec = and i32 %n.rnd.up, -8
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert12 = insertelement <8 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat13 = shufflevector <8 x i32> %broadcast.splatinsert12, <8 x i32> undef, <8 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %broadcast.splatinsert = insertelement <8 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <8 x i32> %broadcast.splatinsert, <8 x i32> undef, <8 x i32> zeroinitializer
+  %induction = add <8 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %0 = getelementptr inbounds i16, i16* %b, i32 %index
+  %1 = icmp ule <8 x i32> %induction, %broadcast.splat13
+  %2 = bitcast i16* %0 to <8 x i16>*
+  %wide.masked.load = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %2, i32 2, <8 x i1> %1, <8 x i16> undef)
+  %3 = getelementptr inbounds i16, i16* %c, i32 %index
+  %4 = bitcast i16* %3 to <8 x i16>*
+  %wide.masked.load14 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %4, i32 2, <8 x i1> %1, <8 x i16> undef)
+  %5 = mul <8 x i16> %wide.masked.load14, %wide.masked.load
+  %6 = getelementptr inbounds i16, i16* %a, i32 %index
+  %7 = bitcast i16* %6 to <8 x i16>*
+  call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %5, <8 x i16>* %7, i32 2, <8 x i1> %1)
+  %index.next = add i32 %index, 8
+  %8 = icmp eq i32 %index.next, %n.vec
+  br i1 %8, label %for.cond.cleanup, label %vector.body
+
+for.cond.cleanup:                                 ; preds = %vector.body, %entry
+  ret void
+}
+
+declare <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>*, i32 immarg, <16 x i1>, <16 x i8>)
+declare <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>*, i32 immarg, <8 x i1>, <8 x i16>)
+declare <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>*, i32 immarg, <4 x i1>, <4 x i32>)
+declare void @llvm.masked.store.v16i8.p0v16i8(<16 x i8>, <16 x i8>*, i32 immarg, <16 x i1>)
+declare void @llvm.masked.store.v8i16.p0v8i16(<8 x i16>, <8 x i16>*, i32 immarg, <8 x i1>)
+declare void @llvm.masked.store.v4i32.p0v4i32(<4 x i32>, <4 x i32>*, i32 immarg, <4 x i1>)
+declare i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32>)
+
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vector-reduce-mve-tail.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vector-reduce-mve-tail.ll
new file mode 100644
index 000000000..38dc5ce54
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vector-reduce-mve-tail.ll
@@ -0,0 +1,75 @@
+
+; RUN: opt -mtriple=thumbv8.1m.main -mve-tail-predication -disable-mve-tail-predication=false -mattr=+mve %s -S -o - | FileCheck %s
+
+; CHECK-LABEL: vec_mul_reduce_add
+
+; CHECK: vector.body:
+; CHECK-NOT: phi i32 [ 0, %vector.ph ]
+; CHECK: [[ELTS:%[^ ]+]] = phi i32 [ %N, %vector.ph ], [ [[SUB:%[^ ]+]], %vector.body ]
+; CHECK: [[VCTP:%[^ ]+]] = call <4 x i1> @llvm.arm.mve.vctp32(i32 [[ELTS]])
+; CHECK: [[SUB]] = sub i32 [[ELTS]], 4
+; CHECK: call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* {{.*}}, i32 4, <4 x i1> [[VCTP]]
+; CHECK: call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* {{.*}}, i32 4, <4 x i1> [[VCTP]],
+
+; CHECK: middle.block:
+; CHECK: [[VCTP_CLONE:%[^ ]+]] = call <4 x i1> @llvm.arm.mve.vctp32(i32 [[ELTS]])
+; CHECK: [[VPSEL:%[^ ]+]] = select <4 x i1> [[VCTP_CLONE]],
+; CHECK: call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> [[VPSEL]])
+
+define i32 @vec_mul_reduce_add(i32* noalias nocapture readonly %a, i32* noalias nocapture readonly %b, i32 %N) {
+entry:
+  %cmp8 = icmp eq i32 %N, 0
+  %0 = add i32 %N, 3
+  %1 = lshr i32 %0, 2
+  %2 = shl nuw i32 %1, 2
+  %3 = add i32 %2, -4
+  %4 = lshr i32 %3, 2
+  %5 = add nuw nsw i32 %4, 1
+  br i1 %cmp8, label %for.cond.cleanup, label %vector.ph
+  
+vector.ph:                                        ; preds = %entry
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert11 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat12 = shufflevector <4 x i32> %broadcast.splatinsert11, <4 x i32> undef, <4 x i32> zeroinitializer
+  call void @llvm.set.loop.iterations.i32(i32 %5)
+  br label %vector.body
+  
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %lsr.iv2 = phi i32* [ %scevgep3, %vector.body ], [ %a, %vector.ph ]
+  %lsr.iv = phi i32* [ %scevgep, %vector.body ], [ %b, %vector.ph ]
+  %vec.phi = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ %9, %vector.body ]
+  %6 = phi i32 [ %5, %vector.ph ], [ %10, %vector.body ]
+  %lsr.iv24 = bitcast i32* %lsr.iv2 to <4 x i32>*
+  %lsr.iv1 = bitcast i32* %lsr.iv to <4 x i32>*
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+  %7 = icmp ule <4 x i32> %induction, %broadcast.splat12
+  %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %lsr.iv24, i32 4, <4 x i1> %7, <4 x i32> undef)
+  %wide.masked.load13 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %lsr.iv1, i32 4, <4 x i1> %7, <4 x i32> undef)
+  %8 = mul nsw <4 x i32> %wide.masked.load13, %wide.masked.load
+  %9 = add nsw <4 x i32> %8, %vec.phi
+  %index.next = add i32 %index, 4
+  %scevgep = getelementptr i32, i32* %lsr.iv, i32 4
+  %scevgep3 = getelementptr i32, i32* %lsr.iv2, i32 4
+  %10 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %6, i32 1)
+  %11 = icmp ne i32 %10, 0
+  br i1 %11, label %vector.body, label %middle.block
+  
+middle.block:                                     ; preds = %vector.body
+  %12 = icmp ule <4 x i32> %induction, %broadcast.splat12
+  %13 = select <4 x i1> %12, <4 x i32> %9, <4 x i32> %vec.phi
+  %14 = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %13)
+  br label %for.cond.cleanup
+  
+for.cond.cleanup:                                 ; preds = %middle.block, %entry
+  %res.0.lcssa = phi i32 [ 0, %entry ], [ %14, %middle.block ]
+  ret i32 %res.0.lcssa
+}
+  
+declare <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>*, i32 immarg, <4 x i1>, <4 x i32>)
+declare i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32>)
+declare void @llvm.set.loop.iterations.i32(i32)
+declare i32 @llvm.loop.decrement.reg.i32.i32.i32(i32, i32)
+  
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vector-unroll.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vector-unroll.ll
new file mode 100644
index 000000000..19d9c89da
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vector-unroll.ll
@@ -0,0 +1,118 @@
+; RUN: opt -mtriple=thumbv8.1m.main -mve-tail-predication -disable-mve-tail-predication=false -mattr=+mve,+lob %s -S -o - | FileCheck %s
+
+; TODO: The unrolled pattern is preventing the transform
+; CHECK-LABEL: mul_v16i8_unroll
+; CHECK-NOT: call i32 @llvm.arm.vcpt
+define void @mul_v16i8_unroll(i8* noalias nocapture readonly %a, i8* noalias nocapture readonly %b, i8* noalias nocapture %c, i32 %N) {
+entry:
+  %cmp8 = icmp eq i32 %N, 0
+  %tmp8 = add i32 %N, 15
+  %tmp9 = lshr i32 %tmp8, 4
+  %tmp10 = shl nuw i32 %tmp9, 4
+  %tmp11 = add i32 %tmp10, -16
+  %tmp12 = lshr i32 %tmp11, 4
+  %tmp13 = add nuw nsw i32 %tmp12, 1
+  br i1 %cmp8, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert10 = insertelement <16 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat11 = shufflevector <16 x i32> %broadcast.splatinsert10, <16 x i32> undef, <16 x i32> zeroinitializer
+  %xtraiter = and i32 %tmp13, 1
+  %0 = icmp ult i32 %tmp12, 1
+  br i1 %0, label %for.cond.cleanup.loopexit.unr-lcssa, label %vector.ph.new
+
+vector.ph.new:                                    ; preds = %vector.ph
+  call void @llvm.set.loop.iterations.i32(i32 %tmp13)
+  %unroll_iter = sub i32 %tmp13, %xtraiter
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph.new
+  %index = phi i32 [ 0, %vector.ph.new ], [ %index.next.1, %vector.body ]
+  %niter = phi i32 [ %unroll_iter, %vector.ph.new ], [ %niter.nsub.1, %vector.body ]
+  %broadcast.splatinsert = insertelement <16 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <16 x i32> %broadcast.splatinsert, <16 x i32> undef, <16 x i32> zeroinitializer
+  %induction = add <16 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %tmp = getelementptr inbounds i8, i8* %a, i32 %index
+  %tmp1 = icmp ule <16 x i32> %induction, %broadcast.splat11
+  %tmp2 = bitcast i8* %tmp to <16 x i8>*
+  %wide.masked.load = tail call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %tmp2, i32 4, <16 x i1> %tmp1, <16 x i8> undef)
+  %tmp3 = getelementptr inbounds i8, i8* %b, i32 %index
+  %tmp4 = bitcast i8* %tmp3 to <16 x i8>*
+  %wide.masked.load2 = tail call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %tmp4, i32 4, <16 x i1> %tmp1, <16 x i8> undef)
+  %mul = mul nsw <16 x i8> %wide.masked.load2, %wide.masked.load
+  %tmp6 = getelementptr inbounds i8, i8* %c, i32 %index
+  %tmp7 = bitcast i8* %tmp6 to <16 x i8>*
+  tail call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %mul, <16 x i8>* %tmp7, i32 4, <16 x i1> %tmp1)
+  %index.next = add nuw nsw i32 %index, 16
+  %niter.nsub = sub i32 %niter, 1
+  %broadcast.splatinsert.1 = insertelement <16 x i32> undef, i32 %index.next, i32 0
+  %broadcast.splat.1 = shufflevector <16 x i32> %broadcast.splatinsert.1, <16 x i32> undef, <16 x i32> zeroinitializer
+  %induction.1 = add <16 x i32> %broadcast.splat.1, <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %tmp.1 = getelementptr inbounds i8, i8* %a, i32 %index.next
+  %tmp1.1 = icmp ule <16 x i32> %induction.1, %broadcast.splat11
+  %tmp2.1 = bitcast i8* %tmp.1 to <16 x i8>*
+  %wide.masked.load.1 = tail call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %tmp2.1, i32 4, <16 x i1> %tmp1.1, <16 x i8> undef)
+  %tmp3.1 = getelementptr inbounds i8, i8* %b, i32 %index.next
+  %tmp4.1 = bitcast i8* %tmp3.1 to <16 x i8>*
+  %wide.masked.load2.1 = tail call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %tmp4.1, i32 4, <16 x i1> %tmp1.1, <16 x i8> undef)
+  %mul.1 = mul nsw <16 x i8> %wide.masked.load2.1, %wide.masked.load.1
+  %tmp6.1 = getelementptr inbounds i8, i8* %c, i32 %index.next
+  %tmp7.1 = bitcast i8* %tmp6.1 to <16 x i8>*
+  tail call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %mul.1, <16 x i8>* %tmp7.1, i32 4, <16 x i1> %tmp1.1)
+  %index.next.1 = add i32 %index.next, 16
+  %niter.nsub.1 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %niter.nsub, i32 1)
+  %niter.ncmp.1 = icmp ne i32 %niter.nsub.1, 0
+  br i1 %niter.ncmp.1, label %vector.body, label %for.cond.cleanup.loopexit.unr-lcssa.loopexit
+
+for.cond.cleanup.loopexit.unr-lcssa.loopexit:     ; preds = %vector.body
+  %index.unr.ph = phi i32 [ %index.next.1, %vector.body ]
+  %tmp14.unr.ph = phi i32 [ -2, %vector.body ]
+  br label %for.cond.cleanup.loopexit.unr-lcssa
+
+for.cond.cleanup.loopexit.unr-lcssa:              ; preds = %for.cond.cleanup.loopexit.unr-lcssa.loopexit, %vector.ph
+  %index.unr = phi i32 [ 0, %vector.ph ], [ %index.unr.ph, %for.cond.cleanup.loopexit.unr-lcssa.loopexit ]
+  %tmp14.unr = phi i32 [ %tmp13, %vector.ph ], [ %tmp14.unr.ph, %for.cond.cleanup.loopexit.unr-lcssa.loopexit ]
+  %lcmp.mod = icmp ne i32 %xtraiter, 0
+  br i1 %lcmp.mod, label %vector.body.epil.preheader, label %for.cond.cleanup.loopexit
+
+vector.body.epil.preheader:                       ; preds = %for.cond.cleanup.loopexit.unr-lcssa
+  br label %vector.body.epil
+
+vector.body.epil:                                 ; preds = %vector.body.epil.preheader
+  %index.epil = phi i32 [ %index.unr, %vector.body.epil.preheader ]
+  %tmp14.epil = phi i32 [ %tmp14.unr, %vector.body.epil.preheader ]
+  %broadcast.splatinsert.epil = insertelement <16 x i32> undef, i32 %index.epil, i32 0
+  %broadcast.splat.epil = shufflevector <16 x i32> %broadcast.splatinsert.epil, <16 x i32> undef, <16 x i32> zeroinitializer
+  %induction.epil = add <16 x i32> %broadcast.splat.epil, <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %tmp.epil = getelementptr inbounds i8, i8* %a, i32 %index.epil
+  %tmp1.epil = icmp ule <16 x i32> %induction.epil, %broadcast.splat11
+  %tmp2.epil = bitcast i8* %tmp.epil to <16 x i8>*
+  %wide.masked.load.epil = tail call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %tmp2.epil, i32 4, <16 x i1> %tmp1.epil, <16 x i8> undef)
+  %tmp3.epil = getelementptr inbounds i8, i8* %b, i32 %index.epil
+  %tmp4.epil = bitcast i8* %tmp3.epil to <16 x i8>*
+  %wide.masked.load2.epil = tail call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %tmp4.epil, i32 4, <16 x i1> %tmp1.epil, <16 x i8> undef)
+  %mul.epil = mul nsw <16 x i8> %wide.masked.load2.epil, %wide.masked.load.epil
+  %tmp6.epil = getelementptr inbounds i8, i8* %c, i32 %index.epil
+  %tmp7.epil = bitcast i8* %tmp6.epil to <16 x i8>*
+  tail call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %mul.epil, <16 x i8>* %tmp7.epil, i32 4, <16 x i1> %tmp1.epil)
+  %index.next.epil = add i32 %index.epil, 16
+  %tmp15.epil = add nuw nsw i32 %tmp14.epil, -1
+  %tmp16.epil = icmp ne i32 %tmp15.epil, 0
+  br label %for.cond.cleanup.loopexit.epilog-lcssa
+
+for.cond.cleanup.loopexit.epilog-lcssa:           ; preds = %vector.body.epil
+  br label %for.cond.cleanup.loopexit
+
+for.cond.cleanup.loopexit:                        ; preds = %for.cond.cleanup.loopexit.unr-lcssa, %for.cond.cleanup.loopexit.epilog-lcssa
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.cond.cleanup.loopexit, %entry
+  ret void
+}
+
+declare <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>*, i32 immarg, <16 x i1>, <16 x i8>) #1
+declare void @llvm.masked.store.v16i8.p0v16i8(<16 x i8>, <16 x i8>*, i32 immarg, <16 x i1>) #2
+declare void @llvm.set.loop.iterations.i32(i32) #3
+declare i32 @llvm.loop.decrement.reg.i32.i32.i32(i32, i32) #3
+
diff --git a/llvm/test/CodeGen/Thumb2/aligned-spill.ll b/llvm/test/CodeGen/Thumb2/aligned-spill.ll
index e3db73236..1e88f8212 100644
--- a/llvm/test/CodeGen/Thumb2/aligned-spill.ll
+++ b/llvm/test/CodeGen/Thumb2/aligned-spill.ll
@@ -11,7 +11,7 @@ target triple = "thumbv7-apple-ios"
 ; CHECK: push {r4, r7, lr}
 ; CHECK: bfc r4, #0, #3
 ; CHECK: mov sp, r4
-define void @f(double* nocapture %p) nounwind ssp "no-frame-pointer-elim"="true" {
+define void @f(double* nocapture %p) nounwind ssp "frame-pointer"="all" {
 entry:
   %0 = load double, double* %p, align 4
   tail call void asm sideeffect "", "~{d8},~{d9},~{d10},~{d11},~{d12},~{d13},~{d14},~{d15}"() nounwind
@@ -45,7 +45,7 @@ entry:
 declare void @g()
 
 ; Spill 7 d-registers.
-define void @f7(double* nocapture %p) nounwind ssp "no-frame-pointer-elim"="true" {
+define void @f7(double* nocapture %p) nounwind ssp "frame-pointer"="all" {
 entry:
   tail call void asm sideeffect "", "~{d8},~{d9},~{d10},~{d11},~{d12},~{d13},~{d14}"() nounwind
   ret void
@@ -69,7 +69,7 @@ entry:
 ; NEON: pop
 
 ; Spill 7 d-registers, leave a hole.
-define void @f3plus4(double* nocapture %p) nounwind ssp "no-frame-pointer-elim"="true" {
+define void @f3plus4(double* nocapture %p) nounwind ssp "frame-pointer"="all" {
 entry:
   tail call void asm sideeffect "", "~{d8},~{d9},~{d10},~{d12},~{d13},~{d14},~{d15}"() nounwind
   ret void
diff --git a/llvm/test/CodeGen/Thumb2/bug-subw.ll b/llvm/test/CodeGen/Thumb2/bug-subw.ll
new file mode 100644
index 000000000..a9ca37e75
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/bug-subw.ll
@@ -0,0 +1,74 @@
+; pr23772 - [ARM] r226200 can emit illegal thumb2 instruction: "sub sp, r12, #80"
+; RUN: llc -march=thumb -mcpu=cortex-m3 -O3 -filetype=asm -o - %s | FileCheck %s
+; CHECK-NOT: sub{{.*}} sp, r{{.*}}, #
+; CHECK:     .fnend
+; TODO: Missed optimization. The three instructions generated to subtract SP can be converged to a single one
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:32"
+target triple = "thumbv7m-unknown-unknown"
+%B = type {%B*}
+%R = type {i32}
+%U = type {%U*, i8, i8}
+%E = type {%B*, %U*}
+%X = type {i32, i8, i8}
+declare external [0 x i8]* @memalloc(i32, i32, i32)
+declare external void @memfree([0 x i8]*, i32, i32)
+define void @foo(%B* %pb$, %R* %pr$) nounwind {
+L.0:
+	%pb = alloca %B*
+	%pr = alloca %R*
+	store %B* %pb$, %B** %pb
+	store %R* %pr$, %R** %pr
+	%pe = alloca %E*
+	%0 = load %B*, %B** %pb
+	%1 = bitcast %B* %0 to %E*
+	store %E* %1, %E** %pe
+	%2 = load %R*, %R** %pr
+	%3 = getelementptr %R, %R* %2, i32 0, i32 0
+	%4 = load i32, i32* %3
+	switch i32 %4, label %L.1 [
+		i32 1, label %L.3
+	]
+L.3:
+	%px = alloca %X*
+	%5 = load %R*, %R** %pr
+	%6 = bitcast %R* %5 to %X*
+	store %X* %6, %X** %px
+	%7 = load %X*, %X** %px
+	%8 = getelementptr %X, %X* %7, i32 0, i32 0
+	%9 = load i32, i32* %8
+	%10 = icmp ne i32 %9, 0
+	br i1 %10, label %L.5, label %L.4
+L.5:
+	%pu = alloca %U*
+	%11 = call [0 x i8]* @memalloc(i32 8, i32 4, i32 0)
+	%12 = bitcast [0 x i8]* %11 to %U*
+	store %U* %12, %U** %pu
+	%13 = load %X*, %X** %px
+	%14 = getelementptr %X, %X* %13, i32 0, i32 1
+	%15 = load i8, i8* %14
+	%16 = load %U*, %U** %pu
+	%17 = getelementptr %U, %U* %16, i32 0, i32 1
+	store i8 %15, i8* %17
+	%18 = load %E*, %E** %pe
+	%19 = getelementptr %E, %E* %18, i32 0, i32 1
+	%20 = load %U*, %U** %19
+	%21 = load %U*, %U** %pu
+	%22 = getelementptr %U, %U* %21, i32 0, i32 0
+	store %U* %20, %U** %22
+	%23 = load %U*, %U** %pu
+	%24 = load %E*, %E** %pe
+	%25 = getelementptr %E, %E* %24, i32 0, i32 1
+	store %U* %23, %U** %25
+	br label %L.4
+L.4:
+	%26 = load %X*, %X** %px
+	%27 = bitcast %X* %26 to [0 x i8]*
+	call void @memfree([0 x i8]* %27, i32 8, i32 0)
+	br label %L.2
+L.1:
+	br label %L.2
+L.2:
+	br label %return
+return:
+	ret void
+}
diff --git a/llvm/test/CodeGen/Thumb2/cbnz.ll b/llvm/test/CodeGen/Thumb2/cbnz.ll
index e11c40386..51b15a2dd 100644
--- a/llvm/test/CodeGen/Thumb2/cbnz.ll
+++ b/llvm/test/CodeGen/Thumb2/cbnz.ll
@@ -5,7 +5,7 @@ declare void @y()
 
 define void @f(i32 %x, i32 %y) {
   ; CHECK-LABEL: f:
-  ; CHECK: cbnz
+  ; CHECK: cbz
   %p = icmp eq i32 %x, 0
   br i1 %p, label %t, label %f
 
@@ -26,7 +26,7 @@ t:
   call void @x()
   call void @x()
   call void @x()
-  ; CHECK: cbz
+  ; CHECK: bne
   %q = icmp eq i32 %y, 0
   br i1 %q, label %t2, label %f
 
diff --git a/llvm/test/CodeGen/Thumb2/constant-islands.ll b/llvm/test/CodeGen/Thumb2/constant-islands.ll
index 81a1f357e..ac323adfd 100644
--- a/llvm/test/CodeGen/Thumb2/constant-islands.ll
+++ b/llvm/test/CodeGen/Thumb2/constant-islands.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -mtriple=arm-apple-ios   -mcpu=cortex-a8 -O0 -filetype=obj -o %t.o
+; RUN: llc < %s -mtriple=arm-apple-ios   -mcpu=cortex-a8 -O0 -filetype=obj -verify-machine-dom-info -o %t.o
 ; RUN: llc < %s -mtriple=thumb-apple-ios -mcpu=cortex-a8 -O0 -filetype=obj -o %t.o
 ; RUN: llc < %s -mtriple=arm-apple-ios   -mcpu=cortex-a8 -O2 -filetype=obj -verify-machineinstrs -o %t.o
 ; RUN: llc < %s -mtriple=thumb-apple-ios -mcpu=cortex-a8 -O2 -filetype=obj -verify-machineinstrs -o %t.o
diff --git a/llvm/test/CodeGen/Thumb2/csel.ll b/llvm/test/CodeGen/Thumb2/csel.ll
new file mode 100644
index 000000000..4ff2fe512
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/csel.ll
@@ -0,0 +1,340 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi %s -verify-machineinstrs -o - | FileCheck %s
+
+define i32 @csinc_const_65(i32 %a) {
+; CHECK-LABEL: csinc_const_65:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    movs r1, #5
+; CHECK-NEXT:    cmp r0, #45
+; CHECK-NEXT:    cinc r0, r1, gt
+; CHECK-NEXT:    bx lr
+entry:
+  %cmp = icmp sgt i32 %a, 45
+  %spec.select = select i1 %cmp, i32 6, i32 5
+  ret i32 %spec.select
+}
+
+define i32 @csinc_const_56(i32 %a) {
+; CHECK-LABEL: csinc_const_56:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    movs r1, #5
+; CHECK-NEXT:    cmp r0, #45
+; CHECK-NEXT:    cinc r0, r1, le
+; CHECK-NEXT:    bx lr
+entry:
+  %cmp = icmp sgt i32 %a, 45
+  %spec.select = select i1 %cmp, i32 5, i32 6
+  ret i32 %spec.select
+}
+
+define i32 @csinc_const_zext(i32 %a) {
+; CHECK-LABEL: csinc_const_zext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    cmp r0, #45
+; CHECK-NEXT:    cset r0, gt
+; CHECK-NEXT:    bx lr
+entry:
+  %cmp = icmp sgt i32 %a, 45
+  %spec.select = zext i1 %cmp to i32
+  ret i32 %spec.select
+}
+
+define i32 @csinv_const_56(i32 %a) {
+; CHECK-LABEL: csinv_const_56:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    movs r1, #5
+; CHECK-NEXT:    cmp r0, #45
+; CHECK-NEXT:    cinv r0, r1, gt
+; CHECK-NEXT:    bx lr
+entry:
+  %cmp = icmp sgt i32 %a, 45
+  %spec.select = select i1 %cmp, i32 -6, i32 5
+  ret i32 %spec.select
+}
+
+define i32 @csinv_const_65(i32 %a) {
+; CHECK-LABEL: csinv_const_65:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    movs r1, #5
+; CHECK-NEXT:    cmp r0, #45
+; CHECK-NEXT:    cinv r0, r1, le
+; CHECK-NEXT:    bx lr
+entry:
+  %cmp = icmp sgt i32 %a, 45
+  %spec.select = select i1 %cmp, i32 5, i32 -6
+  ret i32 %spec.select
+}
+
+define i32 @csinv_const_sext(i32 %a) {
+; CHECK-LABEL: csinv_const_sext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    cmp r0, #45
+; CHECK-NEXT:    csetm r0, gt
+; CHECK-NEXT:    bx lr
+entry:
+  %cmp = icmp sgt i32 %a, 45
+  %spec.select = sext i1 %cmp to i32
+  ret i32 %spec.select
+}
+
+define i32 @csneg_const(i32 %a) {
+; CHECK-LABEL: csneg_const:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    movs r1, #1
+; CHECK-NEXT:    cmp r0, #45
+; CHECK-NEXT:    cneg r0, r1, le
+; CHECK-NEXT:    bx lr
+entry:
+  %cmp = icmp sgt i32 %a, 45
+  %spec.select = select i1 %cmp, i32 1, i32 -1
+  ret i32 %spec.select
+}
+
+define i32 @csneg_const_r(i32 %a) {
+; CHECK-LABEL: csneg_const_r:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    movs r1, #1
+; CHECK-NEXT:    cmp r0, #45
+; CHECK-NEXT:    cneg r0, r1, gt
+; CHECK-NEXT:    bx lr
+entry:
+  %cmp = icmp sgt i32 %a, 45
+  %spec.select = select i1 %cmp, i32 -1, i32 1
+  ret i32 %spec.select
+}
+
+define i32 @csel_var(i32 %a, i32 %b, i32 %c) {
+; CHECK-LABEL: csel_var:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    cmp r0, #45
+; CHECK-NEXT:    it le
+; CHECK-NEXT:    movle r1, r2
+; CHECK-NEXT:    mov r0, r1
+; CHECK-NEXT:    bx lr
+entry:
+  %cmp = icmp sgt i32 %a, 45
+  %spec.select = select i1 %cmp, i32 %b, i32 %c
+  ret i32 %spec.select
+}
+
+define i32 @csinc_var(i32 %a, i32 %b, i32 %c) {
+; CHECK-LABEL: csinc_var:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    cmp r0, #45
+; CHECK-NEXT:    csinc r0, r1, r2, gt
+; CHECK-NEXT:    bx lr
+entry:
+  %cmp = icmp sgt i32 %a, 45
+  %cplus1 = add nsw i32 %c, 1
+  %spec.select = select i1 %cmp, i32 %b, i32 %cplus1
+  ret i32 %spec.select
+}
+
+define i32 @csinc_swap_var(i32 %a, i32 %b, i32 %c) {
+; CHECK-LABEL: csinc_swap_var:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    cmp r0, #45
+; CHECK-NEXT:    csinc r0, r2, r1, le
+; CHECK-NEXT:    bx lr
+entry:
+  %cmp = icmp sgt i32 %a, 45
+  %bplus1 = add nsw i32 %b, 1
+  %spec.select = select i1 %cmp, i32 %bplus1, i32 %c
+  ret i32 %spec.select
+}
+
+define i32 @csinv_var(i32 %a, i32 %b, i32 %c) {
+; CHECK-LABEL: csinv_var:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    cmp r0, #45
+; CHECK-NEXT:    csinv r0, r1, r2, gt
+; CHECK-NEXT:    bx lr
+entry:
+  %cmp = icmp sgt i32 %a, 45
+  %cinv = xor i32 %c, -1
+  %spec.select = select i1 %cmp, i32 %b, i32 %cinv
+  ret i32 %spec.select
+}
+
+define i32 @csinv_swap_var(i32 %a, i32 %b, i32 %c) {
+; CHECK-LABEL: csinv_swap_var:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    cmp r0, #45
+; CHECK-NEXT:    csinv r0, r2, r1, le
+; CHECK-NEXT:    bx lr
+entry:
+  %cmp = icmp sgt i32 %a, 45
+  %binv = xor i32 %b, -1
+  %spec.select = select i1 %cmp, i32 %binv, i32 %c
+  ret i32 %spec.select
+}
+
+define i32 @csneg_var(i32 %a, i32 %b, i32 %c) {
+; CHECK-LABEL: csneg_var:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    cmp r0, #45
+; CHECK-NEXT:    csneg r0, r1, r2, gt
+; CHECK-NEXT:    bx lr
+entry:
+  %cmp = icmp sgt i32 %a, 45
+  %cneg = sub i32 0, %c
+  %spec.select = select i1 %cmp, i32 %b, i32 %cneg
+  ret i32 %spec.select
+}
+
+define i32 @csneg_swap_var_sgt(i32 %a, i32 %b, i32 %c) {
+; CHECK-LABEL: csneg_swap_var_sgt:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    cmp r0, #45
+; CHECK-NEXT:    csneg r0, r2, r1, le
+; CHECK-NEXT:    bx lr
+entry:
+  %cmp = icmp sgt i32 %a, 45
+  %bneg = sub i32 0, %b
+  %spec.select = select i1 %cmp, i32 %bneg, i32 %c
+  ret i32 %spec.select
+}
+
+define i32 @csneg_swap_var_sge(i32 %a, i32 %b, i32 %c) {
+; CHECK-LABEL: csneg_swap_var_sge:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    cmp r0, #44
+; CHECK-NEXT:    csneg r0, r2, r1, le
+; CHECK-NEXT:    bx lr
+entry:
+  %cmp = icmp sge i32 %a, 45
+  %bneg = sub i32 0, %b
+  %spec.select = select i1 %cmp, i32 %bneg, i32 %c
+  ret i32 %spec.select
+}
+
+define i32 @csneg_swap_var_sle(i32 %a, i32 %b, i32 %c) {
+; CHECK-LABEL: csneg_swap_var_sle:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    cmp r0, #46
+; CHECK-NEXT:    csneg r0, r2, r1, ge
+; CHECK-NEXT:    bx lr
+entry:
+  %cmp = icmp sle i32 %a, 45
+  %bneg = sub i32 0, %b
+  %spec.select = select i1 %cmp, i32 %bneg, i32 %c
+  ret i32 %spec.select
+}
+
+define i32 @csneg_swap_var_slt(i32 %a, i32 %b, i32 %c) {
+; CHECK-LABEL: csneg_swap_var_slt:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    cmp r0, #45
+; CHECK-NEXT:    csneg r0, r2, r1, ge
+; CHECK-NEXT:    bx lr
+entry:
+  %cmp = icmp slt i32 %a, 45
+  %bneg = sub i32 0, %b
+  %spec.select = select i1 %cmp, i32 %bneg, i32 %c
+  ret i32 %spec.select
+}
+
+define i32 @csneg_swap_var_ugt(i32 %a, i32 %b, i32 %c) {
+; CHECK-LABEL: csneg_swap_var_ugt:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    cmp r0, #45
+; CHECK-NEXT:    csneg r0, r2, r1, ls
+; CHECK-NEXT:    bx lr
+entry:
+  %cmp = icmp ugt i32 %a, 45
+  %bneg = sub i32 0, %b
+  %spec.select = select i1 %cmp, i32 %bneg, i32 %c
+  ret i32 %spec.select
+}
+
+define i32 @csneg_swap_var_uge(i32 %a, i32 %b, i32 %c) {
+; CHECK-LABEL: csneg_swap_var_uge:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    cmp r0, #44
+; CHECK-NEXT:    csneg r0, r2, r1, ls
+; CHECK-NEXT:    bx lr
+entry:
+  %cmp = icmp uge i32 %a, 45
+  %bneg = sub i32 0, %b
+  %spec.select = select i1 %cmp, i32 %bneg, i32 %c
+  ret i32 %spec.select
+}
+
+define i32 @csneg_swap_var_ule(i32 %a, i32 %b, i32 %c) {
+; CHECK-LABEL: csneg_swap_var_ule:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    cmp r0, #46
+; CHECK-NEXT:    csneg r0, r2, r1, hs
+; CHECK-NEXT:    bx lr
+entry:
+  %cmp = icmp ule i32 %a, 45
+  %bneg = sub i32 0, %b
+  %spec.select = select i1 %cmp, i32 %bneg, i32 %c
+  ret i32 %spec.select
+}
+
+define i32 @csneg_swap_var_ult(i32 %a, i32 %b, i32 %c) {
+; CHECK-LABEL: csneg_swap_var_ult:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    cmp r0, #45
+; CHECK-NEXT:    csneg r0, r2, r1, hs
+; CHECK-NEXT:    bx lr
+entry:
+  %cmp = icmp ult i32 %a, 45
+  %bneg = sub i32 0, %b
+  %spec.select = select i1 %cmp, i32 %bneg, i32 %c
+  ret i32 %spec.select
+}
+
+define i32 @csneg_swap_var_ne(i32 %a, i32 %b, i32 %c) {
+; CHECK-LABEL: csneg_swap_var_ne:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    cmp r0, #45
+; CHECK-NEXT:    csneg r0, r2, r1, ne
+; CHECK-NEXT:    bx lr
+entry:
+  %cmp = icmp eq i32 %a, 45
+  %bneg = sub i32 0, %b
+  %spec.select = select i1 %cmp, i32 %bneg, i32 %c
+  ret i32 %spec.select
+}
+
+define i32 @csneg_swap_var_eq(i32 %a, i32 %b, i32 %c) {
+; CHECK-LABEL: csneg_swap_var_eq:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    cmp r0, #45
+; CHECK-NEXT:    csneg r0, r2, r1, eq
+; CHECK-NEXT:    bx lr
+entry:
+  %cmp = icmp ne i32 %a, 45
+  %bneg = sub i32 0, %b
+  %spec.select = select i1 %cmp, i32 %bneg, i32 %c
+  ret i32 %spec.select
+}
+
+define i32 @csinc_inplace(i32 %a, i32 %b) {
+; CHECK-LABEL: csinc_inplace:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    cmp r1, #45
+; CHECK-NEXT:    cinc r0, r0, gt
+; CHECK-NEXT:    bx lr
+entry:
+  %cmp = icmp sgt i32 %b, 45
+  %inc = zext i1 %cmp to i32
+  %spec.select = add nsw i32 %inc, %a
+  ret i32 %spec.select
+}
+
+define i32 @csinv_inplace(i32 %a, i32 %b) {
+; CHECK-LABEL: csinv_inplace:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    cmp r1, #45
+; CHECK-NEXT:    cinv r0, r0, gt
+; CHECK-NEXT:    bx lr
+entry:
+  %cmp = icmp sgt i32 %b, 45
+  %sub = sext i1 %cmp to i32
+  %xor = xor i32 %sub, %a
+  ret i32 %xor
+}
+
diff --git a/llvm/test/CodeGen/Thumb2/emit-unwinding.ll b/llvm/test/CodeGen/Thumb2/emit-unwinding.ll
index b77bb9e6b..13d509acd 100644
--- a/llvm/test/CodeGen/Thumb2/emit-unwinding.ll
+++ b/llvm/test/CodeGen/Thumb2/emit-unwinding.ll
@@ -9,3 +9,18 @@ define void @foo1() {
 }
 
 declare void @foo2()
+
+; CHECK: _bar:
+; CHECK-NEXT: .cfi_startproc
+; CHECK-NEXT: @ %bb.0:
+; CHECK-NEXT: subw    sp, sp, #3800
+; CHECK-NEXT: .cfi_def_cfa_offset 3800
+; CHECK-NEXT: addw    sp, sp, #3800
+; CHECK-NEXT: bx      lr
+; CHECK-NEXT: .cfi_endproc
+
+define void @bar() {
+  %a1 = alloca [3800 x i8], align 4
+  %p = getelementptr inbounds [3800 x i8], [3800 x i8]* %a1, i32 0, i32 0
+  ret void
+}
diff --git a/llvm/test/CodeGen/Thumb2/float-cmp.ll b/llvm/test/CodeGen/Thumb2/float-cmp.ll
index 87d6ad365..73e0063a9 100644
--- a/llvm/test/CodeGen/Thumb2/float-cmp.ll
+++ b/llvm/test/CodeGen/Thumb2/float-cmp.ll
@@ -23,7 +23,7 @@ define i1 @cmp_f_oeq(float %a, float %b) {
 define i1 @cmp_f_ogt(float %a, float %b) {
 ; CHECK-LABEL: cmp_f_ogt:
 ; NONE: bl __aeabi_fcmpgt
-; HARD: vcmpe.f32
+; HARD: vcmp.f32
 ; HARD: movgt r0, #1
   %1 = fcmp ogt float %a, %b
   ret i1 %1
@@ -31,7 +31,7 @@ define i1 @cmp_f_ogt(float %a, float %b) {
 define i1 @cmp_f_oge(float %a, float %b) {
 ; CHECK-LABEL: cmp_f_oge:
 ; NONE: bl __aeabi_fcmpge
-; HARD: vcmpe.f32
+; HARD: vcmp.f32
 ; HARD: movge r0, #1
   %1 = fcmp oge float %a, %b
   ret i1 %1
@@ -39,7 +39,7 @@ define i1 @cmp_f_oge(float %a, float %b) {
 define i1 @cmp_f_olt(float %a, float %b) {
 ; CHECK-LABEL: cmp_f_olt:
 ; NONE: bl __aeabi_fcmplt
-; HARD: vcmpe.f32
+; HARD: vcmp.f32
 ; HARD: movmi r0, #1
   %1 = fcmp olt float %a, %b
   ret i1 %1
@@ -47,15 +47,15 @@ define i1 @cmp_f_olt(float %a, float %b) {
 define i1 @cmp_f_ole(float %a, float %b) {
 ; CHECK-LABEL: cmp_f_ole:
 ; NONE: bl __aeabi_fcmple
-; HARD: vcmpe.f32
+; HARD: vcmp.f32
 ; HARD: movls r0, #1
   %1 = fcmp ole float %a, %b
   ret i1 %1
 }
 define i1 @cmp_f_one(float %a, float %b) {
 ; CHECK-LABEL: cmp_f_one:
-; NONE: bl __aeabi_fcmpgt
-; NONE: bl __aeabi_fcmplt
+; NONE: bl __aeabi_fcmpeq
+; NONE: bl __aeabi_fcmpun
 ; HARD: vcmp.f32
 ; HARD: movmi r0, #1
 ; HARD: movgt r0, #1
@@ -65,7 +65,7 @@ define i1 @cmp_f_one(float %a, float %b) {
 define i1 @cmp_f_ord(float %a, float %b) {
 ; CHECK-LABEL: cmp_f_ord:
 ; NONE: bl __aeabi_fcmpun
-; HARD: vcmpe.f32
+; HARD: vcmp.f32
 ; HARD: movvc r0, #1
   %1 = fcmp ord float %a, %b
   ret i1 %1
@@ -85,7 +85,7 @@ define i1 @cmp_f_ugt(float %a, float %b) {
 ; NONE: bl __aeabi_fcmple
 ; NONE-NEXT: clz r0, r0
 ; NONE-NEXT: lsrs r0, r0, #5
-; HARD: vcmpe.f32
+; HARD: vcmp.f32
 ; HARD: movhi r0, #1
   %1 = fcmp ugt float %a, %b
   ret i1 %1
@@ -95,7 +95,7 @@ define i1 @cmp_f_uge(float %a, float %b) {
 ; NONE: bl __aeabi_fcmplt
 ; NONE-NEXT: clz r0, r0
 ; NONE-NEXT: lsrs r0, r0, #5
-; HARD: vcmpe.f32
+; HARD: vcmp.f32
 ; HARD: movpl r0, #1
   %1 = fcmp uge float %a, %b
   ret i1 %1
@@ -105,7 +105,7 @@ define i1 @cmp_f_ult(float %a, float %b) {
 ; NONE: bl __aeabi_fcmpge
 ; NONE-NEXT: clz r0, r0
 ; NONE-NEXT: lsrs r0, r0, #5
-; HARD: vcmpe.f32
+; HARD: vcmp.f32
 ; HARD: movlt r0, #1
   %1 = fcmp ult float %a, %b
   ret i1 %1
@@ -115,7 +115,7 @@ define i1 @cmp_f_ule(float %a, float %b) {
 ; NONE: bl __aeabi_fcmpgt
 ; NONE-NEXT: clz r0, r0
 ; NONE-NEXT: lsrs r0, r0, #5
-; HARD: vcmpe.f32
+; HARD: vcmp.f32
 ; HARD: movle r0, #1
   %1 = fcmp ule float %a, %b
   ret i1 %1
@@ -131,7 +131,7 @@ define i1 @cmp_f_une(float %a, float %b) {
 define i1 @cmp_f_uno(float %a, float %b) {
 ; CHECK-LABEL: cmp_f_uno:
 ; NONE: bl __aeabi_fcmpun
-; HARD: vcmpe.f32
+; HARD: vcmp.f32
 ; HARD: movvs r0, #1
   %1 = fcmp uno float %a, %b
   ret i1 %1
@@ -164,7 +164,7 @@ define i1 @cmp_d_ogt(double %a, double %b) {
 ; CHECK-LABEL: cmp_d_ogt:
 ; NONE: bl __aeabi_dcmpgt
 ; SP: bl __aeabi_dcmpgt
-; DP: vcmpe.f64
+; DP: vcmp.f64
 ; DP: movgt r0, #1
   %1 = fcmp ogt double %a, %b
   ret i1 %1
@@ -173,7 +173,7 @@ define i1 @cmp_d_oge(double %a, double %b) {
 ; CHECK-LABEL: cmp_d_oge:
 ; NONE: bl __aeabi_dcmpge
 ; SP: bl __aeabi_dcmpge
-; DP: vcmpe.f64
+; DP: vcmp.f64
 ; DP: movge r0, #1
   %1 = fcmp oge double %a, %b
   ret i1 %1
@@ -182,7 +182,7 @@ define i1 @cmp_d_olt(double %a, double %b) {
 ; CHECK-LABEL: cmp_d_olt:
 ; NONE: bl __aeabi_dcmplt
 ; SP: bl __aeabi_dcmplt
-; DP: vcmpe.f64
+; DP: vcmp.f64
 ; DP: movmi r0, #1
   %1 = fcmp olt double %a, %b
   ret i1 %1
@@ -191,17 +191,17 @@ define i1 @cmp_d_ole(double %a, double %b) {
 ; CHECK-LABEL: cmp_d_ole:
 ; NONE: bl __aeabi_dcmple
 ; SP: bl __aeabi_dcmple
-; DP: vcmpe.f64
+; DP: vcmp.f64
 ; DP: movls r0, #1
   %1 = fcmp ole double %a, %b
   ret i1 %1
 }
 define i1 @cmp_d_one(double %a, double %b) {
 ; CHECK-LABEL: cmp_d_one:
-; NONE: bl __aeabi_dcmpgt
-; NONE: bl __aeabi_dcmplt
-; SP: bl __aeabi_dcmpgt
-; SP: bl __aeabi_dcmplt
+; NONE: bl __aeabi_dcmpeq
+; NONE: bl __aeabi_dcmpun
+; SP: bl __aeabi_dcmpeq
+; SP: bl __aeabi_dcmpun
 ; DP: vcmp.f64
 ; DP: movmi r0, #1
 ; DP: movgt r0, #1
@@ -212,7 +212,7 @@ define i1 @cmp_d_ord(double %a, double %b) {
 ; CHECK-LABEL: cmp_d_ord:
 ; NONE: bl __aeabi_dcmpun
 ; SP: bl __aeabi_dcmpun
-; DP: vcmpe.f64
+; DP: vcmp.f64
 ; DP: movvc r0, #1
   %1 = fcmp ord double %a, %b
   ret i1 %1
@@ -221,7 +221,7 @@ define i1 @cmp_d_ugt(double %a, double %b) {
 ; CHECK-LABEL: cmp_d_ugt:
 ; NONE: bl __aeabi_dcmple
 ; SP: bl __aeabi_dcmple
-; DP: vcmpe.f64
+; DP: vcmp.f64
 ; DP: movhi r0, #1
   %1 = fcmp ugt double %a, %b
   ret i1 %1
@@ -231,7 +231,7 @@ define i1 @cmp_d_ult(double %a, double %b) {
 ; CHECK-LABEL: cmp_d_ult:
 ; NONE: bl __aeabi_dcmpge
 ; SP: bl __aeabi_dcmpge
-; DP: vcmpe.f64
+; DP: vcmp.f64
 ; DP: movlt r0, #1
   %1 = fcmp ult double %a, %b
   ret i1 %1
@@ -242,7 +242,7 @@ define i1 @cmp_d_uno(double %a, double %b) {
 ; CHECK-LABEL: cmp_d_uno:
 ; NONE: bl __aeabi_dcmpun
 ; SP: bl __aeabi_dcmpun
-; DP: vcmpe.f64
+; DP: vcmp.f64
 ; DP: movvs r0, #1
   %1 = fcmp uno double %a, %b
   ret i1 %1
@@ -271,7 +271,7 @@ define i1 @cmp_d_uge(double %a, double %b) {
 ; CHECK-LABEL: cmp_d_uge:
 ; NONE: bl __aeabi_dcmplt
 ; SP: bl __aeabi_dcmplt
-; DP: vcmpe.f64
+; DP: vcmp.f64
 ; DP: movpl r0, #1
   %1 = fcmp uge double %a, %b
   ret i1 %1
@@ -281,7 +281,7 @@ define i1 @cmp_d_ule(double %a, double %b) {
 ; CHECK-LABEL: cmp_d_ule:
 ; NONE: bl __aeabi_dcmpgt
 ; SP: bl __aeabi_dcmpgt
-; DP: vcmpe.f64
+; DP: vcmp.f64
 ; DP: movle r0, #1
   %1 = fcmp ule double %a, %b
   ret i1 %1
diff --git a/llvm/test/CodeGen/Thumb2/float-intrinsics-double.ll b/llvm/test/CodeGen/Thumb2/float-intrinsics-double.ll
index 05d303adb..acafde53a 100644
--- a/llvm/test/CodeGen/Thumb2/float-intrinsics-double.ll
+++ b/llvm/test/CodeGen/Thumb2/float-intrinsics-double.ll
@@ -201,7 +201,7 @@ define double @fmuladd_d(double %a, double %b, double %c) {
 ; SOFT: bl __aeabi_dadd
 ; VFP4: vmul.f64
 ; VFP4: vadd.f64
-; FP-ARMv8: vmla.f64
+; FP-ARMv8: vfma.f64
   %1 = call double @llvm.fmuladd.f64(double %a, double %b, double %c)
   ret double %1
 }
diff --git a/llvm/test/CodeGen/Thumb2/float-intrinsics-float.ll b/llvm/test/CodeGen/Thumb2/float-intrinsics-float.ll
index ec81164b4..1263ae15b 100644
--- a/llvm/test/CodeGen/Thumb2/float-intrinsics-float.ll
+++ b/llvm/test/CodeGen/Thumb2/float-intrinsics-float.ll
@@ -194,7 +194,7 @@ define float @fmuladd_f(float %a, float %b, float %c) {
 ; CHECK-LABEL: fmuladd_f:
 ; SOFT: bl __aeabi_fmul
 ; SOFT: bl __aeabi_fadd
-; VMLA: vmla.f32
+; VMLA: vfma.f32
 ; NO-VMLA: vmul.f32
 ; NO-VMLA: vadd.f32
   %1 = call float @llvm.fmuladd.f32(float %a, float %b, float %c)
diff --git a/llvm/test/CodeGen/Thumb2/float-ops.ll b/llvm/test/CodeGen/Thumb2/float-ops.ll
index cbd8ca557..cc3684c2e 100644
--- a/llvm/test/CodeGen/Thumb2/float-ops.ll
+++ b/llvm/test/CodeGen/Thumb2/float-ops.ll
@@ -130,7 +130,7 @@ define void @store_d(double* %a, double %b) {
 entry:
 ; CHECK-LABEL: store_d:
 ; NOREGS: strd r2, r3, [r0]
-; ONLYREGS: vstr d0, [r0]
+; ONLYREGS: strd r2, r3, [r0]
 ; HARD: vstr d0, [r0]
   store double %b, double* %a, align 8
   ret void
diff --git a/llvm/test/CodeGen/Thumb2/frame-pointer.ll b/llvm/test/CodeGen/Thumb2/frame-pointer.ll
index 832cebe32..ae3c1c8a5 100644
--- a/llvm/test/CodeGen/Thumb2/frame-pointer.ll
+++ b/llvm/test/CodeGen/Thumb2/frame-pointer.ll
@@ -14,7 +14,7 @@ define void @leaf() {
 
 ; Leaf function, frame pointer is requested but we don't need any stack frame,
 ; so don't create a frame pointer.
-define void @leaf_nofpelim() "no-frame-pointer-elim"="true" {
+define void @leaf_nofpelim() "frame-pointer"="all" {
 ; CHECK-LABEL: leaf_nofpelim:
 ; CHECK-NOT: push
 ; CHECK-NOT: sp
@@ -25,7 +25,7 @@ define void @leaf_nofpelim() "no-frame-pointer-elim"="true" {
 
 ; Leaf function, frame pointer is requested and we need a stack frame, so we
 ; need to use a frame pointer.
-define void @leaf_lowreg_nofpelim() "no-frame-pointer-elim"="true" {
+define void @leaf_lowreg_nofpelim() "frame-pointer"="all" {
 ; CHECK-LABEL: leaf_lowreg_nofpelim:
 ; CHECK: push {r4, r6, r7, lr}
 ; CHECK: add r7, sp, #8
@@ -38,7 +38,7 @@ define void @leaf_lowreg_nofpelim() "no-frame-pointer-elim"="true" {
 ; need to use a frame pointer. A high register is pushed to the stack, so we
 ; must use two push/pop instructions to ensure that fp and sp are adjacent on
 ; the stack.
-define void @leaf_highreg_nofpelim() "no-frame-pointer-elim"="true" {
+define void @leaf_highreg_nofpelim() "frame-pointer"="all" {
 ; CHECK-LABEL: leaf_highreg_nofpelim:
 ; CHECK: push {r6, r7, lr}
 ; CHECK: add r7, sp, #4
@@ -51,7 +51,7 @@ define void @leaf_highreg_nofpelim() "no-frame-pointer-elim"="true" {
 
 ; Leaf function, frame pointer requested for non-leaf functions only, so no
 ; need for a stack frame.
-define void @leaf_nononleaffpelim() "no-frame-pointer-elim-non-leaf" {
+define void @leaf_nononleaffpelim() "frame-pointer"="non-leaf" {
 ; CHECK-LABEL: leaf_nononleaffpelim:
 ; CHECK-NOT: push
 ; CHECK-NOT: sp
@@ -72,7 +72,7 @@ define void @call() {
 }
 
 ; Has a call, and frame pointer requested.
-define void @call_nofpelim() "no-frame-pointer-elim"="true" {
+define void @call_nofpelim() "frame-pointer"="all" {
 ; CHECK-LABEL: call_nofpelim:
 ; CHECK: push {r7, lr}
 ; CHECK: mov r7, sp
@@ -83,7 +83,7 @@ define void @call_nofpelim() "no-frame-pointer-elim"="true" {
 }
 
 ; Has a call, and frame pointer requested for non-leaf function.
-define void @call_nononleaffpelim() "no-frame-pointer-elim-non-leaf" {
+define void @call_nononleaffpelim() "frame-pointer"="non-leaf" {
 ; CHECK-LABEL: call_nononleaffpelim:
 ; CHECK: push {r7, lr}
 ; CHECK: mov r7, sp
@@ -107,7 +107,7 @@ define void @highreg() {
 
 ; Has a high register clobbered, frame pointer requested. We need to split the
 ; push into two, to ensure that r7 and sp are adjacent on the stack.
-define void @highreg_nofpelim() "no-frame-pointer-elim"="true" {
+define void @highreg_nofpelim() "frame-pointer"="all" {
 ; CHECK-LABEL: highreg_nofpelim:
 ; CHECK: push {[[DUMMYREG:r[0-9]+]], r7, lr}
 ; CHECK: add r7, sp, #4
@@ -137,7 +137,7 @@ define void @highreg_alloca(i32 %a) {
 
 ; Has a high register clobbered, frame required due to both variable-sized
 ; alloca and ABI. We do need to split the push/pop here.
-define void @highreg_alloca_nofpelim(i32 %a) "no-frame-pointer-elim"="true" {
+define void @highreg_alloca_nofpelim(i32 %a) "frame-pointer"="all" {
 ; CHECK-LABEL: highreg_alloca_nofpelim:
 ; CHECK: push {[[SOMEREGS:.*]], r7, lr}
 ; CHECK: add r7, sp, #{{[0-9]+}}
diff --git a/llvm/test/CodeGen/Thumb2/large-call.ll b/llvm/test/CodeGen/Thumb2/large-call.ll
index f6a5a60ba..9ec622ef6 100644
--- a/llvm/test/CodeGen/Thumb2/large-call.ll
+++ b/llvm/test/CodeGen/Thumb2/large-call.ll
@@ -9,7 +9,7 @@ target triple = "thumbv7-apple-ios0.0.0"
 ; CHECK: main
 ; CHECK: vmov.f64
 ; Adjust SP for the large call
-; CHECK: sub sp,
+; CHECK: subw sp, sp, #3720
 ; Store to call frame + #8
 ; CHECK: vstr{{.*\[}}sp, #8]
 ; Don't clobber that store until the call.
diff --git a/llvm/test/CodeGen/Thumb2/lsll0.ll b/llvm/test/CodeGen/Thumb2/lsll0.ll
new file mode 100644
index 000000000..58949cbbb
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/lsll0.ll
@@ -0,0 +1,47 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s
+
+define void @_Z4loopPxS_iS_i(i64* %d) {
+; CHECK-LABEL: _Z4loopPxS_iS_i:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vmov r2, s2
+; CHECK-NEXT:    vmov r1, s0
+; CHECK-NEXT:    sxth r2, r2
+; CHECK-NEXT:    rsbs r1, r1, #0
+; CHECK-NEXT:    rsbs r2, r2, #0
+; CHECK-NEXT:    sxth r1, r1
+; CHECK-NEXT:    sxth r2, r2
+; CHECK-NEXT:    asrs r3, r1, #31
+; CHECK-NEXT:    asr.w r12, r2, #31
+; CHECK-NEXT:    strd r1, r3, [r0]
+; CHECK-NEXT:    strd r2, r12, [r0, #8]
+; CHECK-NEXT:    bx lr
+entry:
+  %wide.load = load <2 x i64>, <2 x i64>* undef, align 8
+  %0 = trunc <2 x i64> %wide.load to <2 x i32>
+  %1 = shl <2 x i32> %0, <i32 16, i32 16>
+  %2 = ashr exact <2 x i32> %1, <i32 16, i32 16>
+  %3 = sub <2 x i32> %2, %0
+  %4 = and <2 x i32> %3, <i32 7, i32 7>
+  %5 = shl <2 x i32> %2, %4
+  %6 = extractelement <2 x i32> %5, i32 0
+  %7 = zext i32 %6 to i64
+  %8 = select i1 false, i64 %7, i64 undef
+  %9 = trunc i64 %8 to i16
+  %10 = sub i16 0, %9
+  %11 = sext i16 %10 to i64
+  %12 = getelementptr inbounds i64, i64* %d, i64 undef
+  store i64 %11, i64* %12, align 8
+  %13 = extractelement <2 x i32> %5, i32 1
+  %14 = zext i32 %13 to i64
+  %15 = select i1 false, i64 %14, i64 undef
+  %16 = trunc i64 %15 to i16
+  %17 = sub i16 0, %16
+  %18 = sext i16 %17 to i64
+  %19 = or i32 0, 1
+  %20 = sext i32 %19 to i64
+  %21 = getelementptr inbounds i64, i64* %d, i64 %20
+  store i64 %18, i64* %21, align 8
+  ret void
+}
diff --git a/llvm/test/CodeGen/Thumb2/mve-abs.ll b/llvm/test/CodeGen/Thumb2/mve-abs.ll
index 6e2100e2f..90c9206dd 100644
--- a/llvm/test/CodeGen/Thumb2/mve-abs.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-abs.ll
@@ -40,43 +40,36 @@ entry:
 define arm_aapcs_vfpcc <2 x i64> @abs_v2i64(<2 x i64> %s1) {
 ; CHECK-LABEL: abs_v2i64:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, r5, r6, r7, lr}
-; CHECK-NEXT:    push {r4, r5, r6, r7, lr}
-; CHECK-NEXT:    vmov r12, s2
-; CHECK-NEXT:    movs r0, #0
-; CHECK-NEXT:    vmov r3, s3
-; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    rsbs.w lr, r12, #0
-; CHECK-NEXT:    sbc.w r5, r0, r3
-; CHECK-NEXT:    cmp r3, #0
-; CHECK-NEXT:    mov r2, lr
-; CHECK-NEXT:    lsrl r2, r5, #32
-; CHECK-NEXT:    mov.w r5, #0
-; CHECK-NEXT:    it mi
-; CHECK-NEXT:    movmi r5, #1
-; CHECK-NEXT:    cmp r5, #0
-; CHECK-NEXT:    it eq
-; CHECK-NEXT:    moveq r2, r3
-; CHECK-NEXT:    vmov r3, s1
-; CHECK-NEXT:    rsbs r4, r1, #0
-; CHECK-NEXT:    mov r6, r4
-; CHECK-NEXT:    sbc.w r7, r0, r3
-; CHECK-NEXT:    cmp r3, #0
-; CHECK-NEXT:    lsrl r6, r7, #32
-; CHECK-NEXT:    it mi
-; CHECK-NEXT:    movmi r0, #1
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    vmov q1, q0
+; CHECK-NEXT:    mov.w r12, #0
+; CHECK-NEXT:    vmov lr, s4
+; CHECK-NEXT:    vmov r0, s5
+; CHECK-NEXT:    rsbs.w r3, lr, #0
+; CHECK-NEXT:    sbc.w r2, r12, r0
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    cset r1, mi
+; CHECK-NEXT:    ands r1, r1, #1
+; CHECK-NEXT:    itt eq
+; CHECK-NEXT:    moveq r2, r0
+; CHECK-NEXT:    moveq r3, lr
+; CHECK-NEXT:    vmov lr, s6
+; CHECK-NEXT:    vmov.32 q0[0], r3
+; CHECK-NEXT:    vmov r0, s7
+; CHECK-NEXT:    vmov.32 q0[1], r2
+; CHECK-NEXT:    rsbs.w r2, lr, #0
+; CHECK-NEXT:    sbc.w r3, r12, r0
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    ite eq
-; CHECK-NEXT:    moveq r6, r3
-; CHECK-NEXT:    movne r1, r4
-; CHECK-NEXT:    vmov.32 q0[0], r1
-; CHECK-NEXT:    cmp r5, #0
-; CHECK-NEXT:    vmov.32 q0[1], r6
+; CHECK-NEXT:    cset r1, mi
+; CHECK-NEXT:    ands r1, r1, #1
+; CHECK-NEXT:    it eq
+; CHECK-NEXT:    moveq r2, lr
+; CHECK-NEXT:    vmov.32 q0[2], r2
 ; CHECK-NEXT:    it eq
-; CHECK-NEXT:    moveq lr, r12
-; CHECK-NEXT:    vmov.32 q0[2], lr
-; CHECK-NEXT:    vmov.32 q0[3], r2
-; CHECK-NEXT:    pop {r4, r5, r6, r7, pc}
+; CHECK-NEXT:    moveq r3, r0
+; CHECK-NEXT:    vmov.32 q0[3], r3
+; CHECK-NEXT:    pop {r7, pc}
 entry:
   %0 = icmp slt <2 x i64> %s1, zeroinitializer
   %1 = sub nsw <2 x i64> zeroinitializer, %s1
diff --git a/llvm/test/CodeGen/Thumb2/mve-be.ll b/llvm/test/CodeGen/Thumb2/mve-be.ll
new file mode 100644
index 000000000..7f355396a
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-be.ll
@@ -0,0 +1,297 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-LE
+; RUN: llc -mtriple=thumbebv8.1m.main-arm-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-BE
+
+define void @load_load_add_store(<4 x i32> *%src1, <4 x i32> *%src2) {
+; CHECK-LABEL: load_load_add_store:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vadd.i32 q0, q1, q0
+; CHECK-NEXT:    vstrw.32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %l1 = load <4 x i32>, <4 x i32>* %src1, align 4
+  %l2 = load <4 x i32>, <4 x i32>* %src2, align 4
+  %a = add <4 x i32> %l1, %l2
+  store <4 x i32> %a, <4 x i32>* %src1, align 4
+  ret void
+}
+
+define void @load_load_add_store_align1(<4 x i32> *%src1, <4 x i32> *%src2) {
+; CHECK-LE-LABEL: load_load_add_store_align1:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldrb.u8 q0, [r1]
+; CHECK-LE-NEXT:    vldrb.u8 q1, [r0]
+; CHECK-LE-NEXT:    vadd.i32 q0, q1, q0
+; CHECK-LE-NEXT:    vstrb.8 q0, [r0]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: load_load_add_store_align1:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vldrb.u8 q0, [r1]
+; CHECK-BE-NEXT:    vldrb.u8 q1, [r0]
+; CHECK-BE-NEXT:    vrev32.8 q0, q0
+; CHECK-BE-NEXT:    vrev32.8 q1, q1
+; CHECK-BE-NEXT:    vadd.i32 q0, q1, q0
+; CHECK-BE-NEXT:    vrev32.8 q0, q0
+; CHECK-BE-NEXT:    vstrb.8 q0, [r0]
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %l1 = load <4 x i32>, <4 x i32>* %src1, align 1
+  %l2 = load <4 x i32>, <4 x i32>* %src2, align 1
+  %a = add <4 x i32> %l1, %l2
+  store <4 x i32> %a, <4 x i32>* %src1, align 1
+  ret void
+}
+
+define arm_aapcs_vfpcc void @load_arg_add_store(<4 x i32> *%src1, <4 x i32> %src2) {
+; CHECK-LE-LABEL: load_arg_add_store:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-LE-NEXT:    vadd.i32 q0, q1, q0
+; CHECK-LE-NEXT:    vstrw.32 q0, [r0]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: load_arg_add_store:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-BE-NEXT:    vadd.i32 q0, q0, q1
+; CHECK-BE-NEXT:    vstrw.32 q0, [r0]
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %l1 = load <4 x i32>, <4 x i32>* %src1, align 4
+  %a = add <4 x i32> %l1, %src2
+  store <4 x i32> %a, <4 x i32>* %src1, align 4
+  ret void
+}
+
+define <4 x i32> @add_soft(<4 x i32> %src1, <4 x i32> %src2) {
+; CHECK-LE-LABEL: add_soft:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vmov d1, r2, r3
+; CHECK-LE-NEXT:    vmov d0, r0, r1
+; CHECK-LE-NEXT:    mov r0, sp
+; CHECK-LE-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-LE-NEXT:    vadd.i32 q0, q0, q1
+; CHECK-LE-NEXT:    vmov r0, r1, d0
+; CHECK-LE-NEXT:    vmov r2, r3, d1
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: add_soft:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vmov d1, r3, r2
+; CHECK-BE-NEXT:    vmov d0, r1, r0
+; CHECK-BE-NEXT:    mov r0, sp
+; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-BE-NEXT:    vadd.i32 q0, q1, q0
+; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    vmov r1, r0, d2
+; CHECK-BE-NEXT:    vmov r3, r2, d3
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %0 = add <4 x i32> %src1, %src2
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @add_hard(<4 x i32> %src1, <4 x i32> %src2) {
+; CHECK-LE-LABEL: add_hard:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vadd.i32 q0, q0, q1
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: add_hard:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vrev64.32 q2, q1
+; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    vadd.i32 q1, q1, q2
+; CHECK-BE-NEXT:    vrev64.32 q0, q1
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %0 = add <4 x i32> %src1, %src2
+  ret <4 x i32> %0
+}
+
+define <4 x i32> @call_soft(<4 x i32> %src1, <4 x i32> %src2) {
+; CHECK-LE-LABEL: call_soft:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    .save {r7, lr}
+; CHECK-LE-NEXT:    push {r7, lr}
+; CHECK-LE-NEXT:    .pad #16
+; CHECK-LE-NEXT:    sub sp, #16
+; CHECK-LE-NEXT:    add.w r12, sp, #24
+; CHECK-LE-NEXT:    vldrw.u32 q0, [r12]
+; CHECK-LE-NEXT:    vstrw.32 q0, [sp]
+; CHECK-LE-NEXT:    vmov d1, r2, r3
+; CHECK-LE-NEXT:    vmov d0, r0, r1
+; CHECK-LE-NEXT:    vshr.u32 q0, q0, #1
+; CHECK-LE-NEXT:    vmov r0, r1, d0
+; CHECK-LE-NEXT:    vmov r2, r3, d1
+; CHECK-LE-NEXT:    bl add_soft
+; CHECK-LE-NEXT:    vmov d1, r2, r3
+; CHECK-LE-NEXT:    vmov d0, r0, r1
+; CHECK-LE-NEXT:    vshr.u32 q0, q0, #1
+; CHECK-LE-NEXT:    vmov r0, r1, d0
+; CHECK-LE-NEXT:    vmov r2, r3, d1
+; CHECK-LE-NEXT:    add sp, #16
+; CHECK-LE-NEXT:    pop {r7, pc}
+;
+; CHECK-BE-LABEL: call_soft:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    .save {r7, lr}
+; CHECK-BE-NEXT:    push {r7, lr}
+; CHECK-BE-NEXT:    .pad #16
+; CHECK-BE-NEXT:    sub sp, #16
+; CHECK-BE-NEXT:    add.w r12, sp, #24
+; CHECK-BE-NEXT:    vldrw.u32 q0, [r12]
+; CHECK-BE-NEXT:    vstrw.32 q0, [sp]
+; CHECK-BE-NEXT:    vmov d1, r3, r2
+; CHECK-BE-NEXT:    vmov d0, r1, r0
+; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    vshr.u32 q0, q1, #1
+; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    vmov r1, r0, d2
+; CHECK-BE-NEXT:    vmov r3, r2, d3
+; CHECK-BE-NEXT:    bl add_soft
+; CHECK-BE-NEXT:    vmov d1, r3, r2
+; CHECK-BE-NEXT:    vmov d0, r1, r0
+; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    vshr.u32 q0, q1, #1
+; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    vmov r1, r0, d2
+; CHECK-BE-NEXT:    vmov r3, r2, d3
+; CHECK-BE-NEXT:    add sp, #16
+; CHECK-BE-NEXT:    pop {r7, pc}
+entry:
+  %0 = lshr <4 x i32> %src1, <i32 1, i32 1, i32 1, i32 1>
+  %1 = call <4 x i32> @add_soft(<4 x i32> %0, <4 x i32> %src2)
+  %2 = lshr <4 x i32> %1, <i32 1, i32 1, i32 1, i32 1>
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @call_hard(<4 x i32> %src1, <4 x i32> %src2) {
+; CHECK-LE-LABEL: call_hard:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    .save {r7, lr}
+; CHECK-LE-NEXT:    push {r7, lr}
+; CHECK-LE-NEXT:    vshr.u32 q0, q0, #1
+; CHECK-LE-NEXT:    bl add_hard
+; CHECK-LE-NEXT:    vshr.u32 q0, q0, #1
+; CHECK-LE-NEXT:    pop {r7, pc}
+;
+; CHECK-BE-LABEL: call_hard:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    .save {r7, lr}
+; CHECK-BE-NEXT:    push {r7, lr}
+; CHECK-BE-NEXT:    vrev64.32 q2, q0
+; CHECK-BE-NEXT:    vshr.u32 q2, q2, #1
+; CHECK-BE-NEXT:    vrev64.32 q0, q2
+; CHECK-BE-NEXT:    bl add_hard
+; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    vshr.u32 q1, q1, #1
+; CHECK-BE-NEXT:    vrev64.32 q0, q1
+; CHECK-BE-NEXT:    pop {r7, pc}
+entry:
+  %0 = lshr <4 x i32> %src1, <i32 1, i32 1, i32 1, i32 1>
+  %1 = call arm_aapcs_vfpcc <4 x i32> @add_hard(<4 x i32> %0, <4 x i32> %src2)
+  %2 = lshr <4 x i32> %1, <i32 1, i32 1, i32 1, i32 1>
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @and_v4i32(<4 x i32> %src) {
+; CHECK-LE-LABEL: and_v4i32:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vmov.i32 q1, #0x1
+; CHECK-LE-NEXT:    vand q0, q0, q1
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: and_v4i32:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    vmov.i32 q0, #0x1
+; CHECK-BE-NEXT:    vand q1, q1, q0
+; CHECK-BE-NEXT:    vrev64.32 q0, q1
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %s1 = and <4 x i32> %src, <i32 1, i32 1, i32 1, i32 1>
+  %r = bitcast <4 x i32> %s1 to <16 x i8>
+  ret <16 x i8> %r
+}
+
+; Should be the same as and_v4i32 for LE
+define arm_aapcs_vfpcc <16 x i8> @and_v16i8_le(<4 x i32> %src) {
+; CHECK-LE-LABEL: and_v16i8_le:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vmov.i32 q1, #0x1
+; CHECK-LE-NEXT:    vand q0, q0, q1
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: and_v16i8_le:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vrev64.8 q1, q0
+; CHECK-BE-NEXT:    vmov.i32 q0, #0x1
+; CHECK-BE-NEXT:    vrev32.8 q0, q0
+; CHECK-BE-NEXT:    vand q1, q1, q0
+; CHECK-BE-NEXT:    vrev64.8 q0, q1
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %0 = bitcast <4 x i32> %src to <16 x i8>
+  %r = and <16 x i8> %0, <i8 1, i8 0, i8 0, i8 0, i8 1, i8 0, i8 0, i8 0, i8 1, i8 0, i8 0, i8 0, i8 1, i8 0, i8 0, i8 0>
+  ret <16 x i8> %r
+}
+
+; Should be the same (or at least equivalent) as and_v4i32 for BE
+define arm_aapcs_vfpcc <16 x i8> @and_v16i8_be(<4 x i32> %src) {
+; CHECK-LE-LABEL: and_v16i8_be:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vmov.i32 q1, #0x1000000
+; CHECK-LE-NEXT:    vand q0, q0, q1
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: and_v16i8_be:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vrev64.8 q1, q0
+; CHECK-BE-NEXT:    vmov.i32 q0, #0x1000000
+; CHECK-BE-NEXT:    vrev32.8 q0, q0
+; CHECK-BE-NEXT:    vand q1, q1, q0
+; CHECK-BE-NEXT:    vrev64.8 q0, q1
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %0 = bitcast <4 x i32> %src to <16 x i8>
+  %r = and <16 x i8> %0, <i8 0, i8 0, i8 0, i8 1, i8 0, i8 0, i8 0, i8 1, i8 0, i8 0, i8 0, i8 1, i8 0, i8 0, i8 0, i8 1>
+  ret <16 x i8> %r
+}
+
+; FIXME: This looks wrong
+define arm_aapcs_vfpcc <4 x i32> @test(i32* %data) {
+; CHECK-LE-LABEL: test:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldrw.u32 q1, [r0, #32]
+; CHECK-LE-NEXT:    vmov.i32 q0, #0x1
+; CHECK-LE-NEXT:    vadd.i32 q1, q1, q0
+; CHECK-LE-NEXT:    @APP
+; CHECK-LE-NEXT:    vmullb.s32 q0, q1, q1
+; CHECK-LE-NEXT:    @NO_APP
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: test:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vldrw.u32 q1, [r0, #32]
+; CHECK-BE-NEXT:    vmov.i32 q0, #0x1
+; CHECK-BE-NEXT:    vadd.i32 q0, q1, q0
+; CHECK-BE-NEXT:    vrev32.8 q0, q0
+; CHECK-BE-NEXT:    @APP
+; CHECK-BE-NEXT:    vmullb.s32 q1, q0, q0
+; CHECK-BE-NEXT:    @NO_APP
+; CHECK-BE-NEXT:    vrev64.8 q0, q1
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %add.ptr = getelementptr inbounds i32, i32* %data, i32 8
+  %0 = bitcast i32* %add.ptr to <4 x i32>*
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
+  %2 = add <4 x i32> %1, <i32 1, i32 1, i32 1, i32 1>
+  %3 = tail call <4 x i32> asm sideeffect "  VMULLB.s32 $0, $1, $1", "=&w,w"(<4 x i32> %2) #2
+  ret <4 x i32> %3
+}
diff --git a/llvm/test/CodeGen/Thumb2/mve-bitreverse.ll b/llvm/test/CodeGen/Thumb2/mve-bitreverse.ll
new file mode 100644
index 000000000..99f9506c3
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-bitreverse.ll
@@ -0,0 +1,52 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -verify-machineinstrs -mattr=+mve %s -o - | FileCheck %s
+
+define arm_aapcs_vfpcc <2 x i64> @brv_2i64_t(<2 x i64> %src){
+; CHECK-LABEL: brv_2i64_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vrev64.8 q1, q0
+; CHECK-NEXT:    movs r0, #8
+; CHECK-NEXT:    vbrsr.8 q0, q1, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %src)
+  ret <2 x i64> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @brv_4i32_t(<4 x i32> %src){
+; CHECK-LABEL: brv_4i32_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    movs r0, #32
+; CHECK-NEXT:    vbrsr.32 q0, q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %src)
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @brv_8i16_t(<8 x i16> %src){
+; CHECK-LABEL: brv_8i16_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    movs r0, #16
+; CHECK-NEXT:    vbrsr.16 q0, q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %src)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @brv_16i8_t(<16 x i8> %src){
+; CHECK-LABEL: brv_16i8_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    movs r0, #8
+; CHECK-NEXT:    vbrsr.8 q0, q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <16 x i8> @llvm.bitreverse.v16i8(<16 x i8> %src)
+  ret <16 x i8> %0
+}
+
+declare <2 x i64> @llvm.bitreverse.v2i64(<2 x i64>)
+declare <4 x i32> @llvm.bitreverse.v4i32(<4 x i32>)
+declare <8 x i16> @llvm.bitreverse.v8i16(<8 x i16>)
+declare <16 x i8> @llvm.bitreverse.v16i8(<16 x i8>)
diff --git a/llvm/test/CodeGen/Thumb2/mve-bswap.ll b/llvm/test/CodeGen/Thumb2/mve-bswap.ll
new file mode 100644
index 000000000..c7d3f844f
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-bswap.ll
@@ -0,0 +1,37 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -verify-machineinstrs -mattr=+mve %s -o - | FileCheck %s
+
+define arm_aapcs_vfpcc <4 x i32> @bswap_4i32_t(<4 x i32> %src){
+; CHECK-LABEL: bswap_4i32_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vrev32.8 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+    %0 = call <4 x i32> @llvm.bswap.v4i32(<4 x i32> %src)
+    ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @bswap_8i16_t(<8 x i16> %src){
+; CHECK-LABEL: bswap_8i16_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vrev16.8 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+    %0 = call <8 x i16> @llvm.bswap.v8i16(<8 x i16> %src)
+    ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <2 x i64> @bswap_2i64_t(<2 x i64> %src){
+; CHECK-LABEL: bswap_2i64_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vrev64.8 q1, q0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+    %0 = call <2 x i64> @llvm.bswap.v2i64(<2 x i64> %src)
+    ret <2 x i64> %0
+}
+
+declare <2 x i64> @llvm.bswap.v2i64(<2 x i64>)
+declare <4 x i32> @llvm.bswap.v4i32(<4 x i32>)
+declare <8 x i16> @llvm.bswap.v8i16(<8 x i16>)
diff --git a/llvm/test/CodeGen/Thumb2/mve-ctlz.ll b/llvm/test/CodeGen/Thumb2/mve-ctlz.ll
new file mode 100644
index 000000000..0645fadd2
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-ctlz.ll
@@ -0,0 +1,140 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -verify-machineinstrs -mattr=+mve %s -o - | FileCheck %s
+
+define arm_aapcs_vfpcc <2 x i64> @ctlz_2i64_0_t(<2 x i64> %src){
+; CHECK-LABEL: ctlz_2i64_0_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov r0, s3
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    cset r1, ne
+; CHECK-NEXT:    lsls r1, r1, #31
+; CHECK-NEXT:    vmov r1, s2
+; CHECK-NEXT:    clz r1, r1
+; CHECK-NEXT:    add.w r1, r1, #32
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    clzne r1, r0
+; CHECK-NEXT:    vmov r0, s1
+; CHECK-NEXT:    vmov s6, r1
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    cset r1, ne
+; CHECK-NEXT:    lsls r1, r1, #31
+; CHECK-NEXT:    vmov r1, s0
+; CHECK-NEXT:    clz r1, r1
+; CHECK-NEXT:    add.w r1, r1, #32
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    clzne r1, r0
+; CHECK-NEXT:    vmov s4, r1
+; CHECK-NEXT:    vldr s5, .LCPI0_0
+; CHECK-NEXT:    vmov.f32 s7, s5
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    .p2align 2
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI0_0:
+; CHECK-NEXT:    .long 0 @ float 0
+entry:
+  %0 = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> %src, i1 0)
+  ret <2 x i64> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @ctlz_4i32_0_t(<4 x i32> %src){
+; CHECK-LABEL: ctlz_4i32_0_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vclz.i32 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> %src, i1 0)
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @ctlz_8i16_0_t(<8 x i16> %src){
+; CHECK-LABEL: ctlz_8i16_0_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vclz.i16 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.ctlz.v8i16(<8 x i16> %src, i1 0)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @ctlz_16i8_0_t(<16 x i8> %src){
+; CHECK-LABEL: ctlz_16i8_0_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vclz.i8 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <16 x i8> @llvm.ctlz.v16i8(<16 x i8> %src, i1 0)
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <2 x i64> @ctlz_2i64_1_t(<2 x i64> %src){
+; CHECK-LABEL: ctlz_2i64_1_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov r0, s3
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    cset r1, ne
+; CHECK-NEXT:    lsls r1, r1, #31
+; CHECK-NEXT:    vmov r1, s2
+; CHECK-NEXT:    clz r1, r1
+; CHECK-NEXT:    add.w r1, r1, #32
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    clzne r1, r0
+; CHECK-NEXT:    vmov r0, s1
+; CHECK-NEXT:    vmov s6, r1
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    cset r1, ne
+; CHECK-NEXT:    lsls r1, r1, #31
+; CHECK-NEXT:    vmov r1, s0
+; CHECK-NEXT:    clz r1, r1
+; CHECK-NEXT:    add.w r1, r1, #32
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    clzne r1, r0
+; CHECK-NEXT:    vmov s4, r1
+; CHECK-NEXT:    vldr s5, .LCPI4_0
+; CHECK-NEXT:    vmov.f32 s7, s5
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    .p2align 2
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI4_0:
+; CHECK-NEXT:    .long 0 @ float 0
+entry:
+  %0 = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> %src, i1 1)
+  ret <2 x i64> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @ctlz_4i32_1_t(<4 x i32> %src){
+; CHECK-LABEL: ctlz_4i32_1_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vclz.i32 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> %src, i1 1)
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @ctlz_8i16_1_t(<8 x i16> %src){
+; CHECK-LABEL: ctlz_8i16_1_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vclz.i16 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.ctlz.v8i16(<8 x i16> %src, i1 1)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @ctlz_16i8_1_t(<16 x i8> %src){
+; CHECK-LABEL: ctlz_16i8_1_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vclz.i8 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <16 x i8> @llvm.ctlz.v16i8(<16 x i8> %src, i1 1)
+  ret <16 x i8> %0
+}
+
+
+declare <2 x i64> @llvm.ctlz.v2i64(<2 x i64>, i1)
+declare <4 x i32> @llvm.ctlz.v4i32(<4 x i32>, i1)
+declare <8 x i16> @llvm.ctlz.v8i16(<8 x i16>, i1)
+declare <16 x i8> @llvm.ctlz.v16i8(<16 x i8>, i1)
diff --git a/llvm/test/CodeGen/Thumb2/mve-ctpop.ll b/llvm/test/CodeGen/Thumb2/mve-ctpop.ll
new file mode 100644
index 000000000..d289d3712
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-ctpop.ll
@@ -0,0 +1,151 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; NOTE: Assertions have been autoenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve.fp -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK
+
+define arm_aapcs_vfpcc <2 x i64> @ctpop_2i64_t(<2 x i64> %src){
+; CHECK-LABEL: ctpop_2i64_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, lr}
+; CHECK-NEXT:    push {r4, lr}
+; CHECK-NEXT:    vmov r0, s3
+; CHECK-NEXT:    mov.w r1, #1431655765
+; CHECK-NEXT:    mov.w lr, #858993459
+; CHECK-NEXT:    mov.w r4, #16843009
+; CHECK-NEXT:    and.w r2, r1, r0, lsr #1
+; CHECK-NEXT:    subs r0, r0, r2
+; CHECK-NEXT:    and.w r3, lr, r0, lsr #2
+; CHECK-NEXT:    bic r0, r0, #-858993460
+; CHECK-NEXT:    add r0, r3
+; CHECK-NEXT:    vmov r3, s2
+; CHECK-NEXT:    add.w r0, r0, r0, lsr #4
+; CHECK-NEXT:    bic r12, r0, #-252645136
+; CHECK-NEXT:    and.w r0, r1, r3, lsr #1
+; CHECK-NEXT:    subs r0, r3, r0
+; CHECK-NEXT:    and.w r3, lr, r0, lsr #2
+; CHECK-NEXT:    bic r0, r0, #-858993460
+; CHECK-NEXT:    add r0, r3
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    add.w r0, r0, r0, lsr #4
+; CHECK-NEXT:    bic r0, r0, #-252645136
+; CHECK-NEXT:    muls r0, r4, r0
+; CHECK-NEXT:    lsrs r0, r0, #24
+; CHECK-NEXT:    and.w r2, r1, r3, lsr #1
+; CHECK-NEXT:    subs r2, r3, r2
+; CHECK-NEXT:    and.w r3, lr, r2, lsr #2
+; CHECK-NEXT:    bic r2, r2, #-858993460
+; CHECK-NEXT:    add r2, r3
+; CHECK-NEXT:    vmov r3, s1
+; CHECK-NEXT:    vldr s1, .LCPI0_0
+; CHECK-NEXT:    add.w r2, r2, r2, lsr #4
+; CHECK-NEXT:    bic r2, r2, #-252645136
+; CHECK-NEXT:    muls r2, r4, r2
+; CHECK-NEXT:    lsrs r2, r2, #24
+; CHECK-NEXT:    and.w r1, r1, r3, lsr #1
+; CHECK-NEXT:    subs r1, r3, r1
+; CHECK-NEXT:    and.w r3, lr, r1, lsr #2
+; CHECK-NEXT:    bic r1, r1, #-858993460
+; CHECK-NEXT:    add r1, r3
+; CHECK-NEXT:    mul r3, r12, r4
+; CHECK-NEXT:    add.w r1, r1, r1, lsr #4
+; CHECK-NEXT:    bic r1, r1, #-252645136
+; CHECK-NEXT:    muls r1, r4, r1
+; CHECK-NEXT:    add.w r0, r0, r3, lsr #24
+; CHECK-NEXT:    vmov s2, r0
+; CHECK-NEXT:    add.w r0, r2, r1, lsr #24
+; CHECK-NEXT:    vmov s0, r0
+; CHECK-NEXT:    vmov.f32 s3, s1
+; CHECK-NEXT:    pop {r4, pc}
+; CHECK-NEXT:    .p2align 2
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI0_0:
+; CHECK-NEXT:    .long 0 @ float 0
+entry:
+  %0 = call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %src)
+  ret <2 x i64> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @ctpop_4i32_t(<4 x i32> %src){
+; CHECK-LABEL: ctpop_4i32_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11}
+; CHECK-NEXT:    vmov.i8 q4, #0x55
+; CHECK-NEXT:    vshr.u32 q5, q0, #1
+; CHECK-NEXT:    vand q4, q5, q4
+; CHECK-NEXT:    vmov.i8 q3, #0x33
+; CHECK-NEXT:    vsub.i32 q0, q0, q4
+; CHECK-NEXT:    vmov.i8 q2, #0xf
+; CHECK-NEXT:    vshr.u32 q4, q0, #2
+; CHECK-NEXT:    vand q0, q0, q3
+; CHECK-NEXT:    vand q4, q4, q3
+; CHECK-NEXT:    vmov.i8 q1, #0x1
+; CHECK-NEXT:    vadd.i32 q0, q0, q4
+; CHECK-NEXT:    vshr.u32 q3, q0, #4
+; CHECK-NEXT:    vadd.i32 q0, q0, q3
+; CHECK-NEXT:    vand q0, q0, q2
+; CHECK-NEXT:    vmul.i32 q0, q0, q1
+; CHECK-NEXT:    vshr.u32 q0, q0, #24
+; CHECK-NEXT:    vpop {d8, d9, d10, d11}
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> %src)
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @ctpop_8i16_t(<8 x i16> %src){
+; CHECK-LABEL: ctpop_8i16_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11}
+; CHECK-NEXT:    vmov.i8 q4, #0x55
+; CHECK-NEXT:    vshr.u16 q5, q0, #1
+; CHECK-NEXT:    vand q4, q5, q4
+; CHECK-NEXT:    vmov.i8 q3, #0x33
+; CHECK-NEXT:    vsub.i16 q0, q0, q4
+; CHECK-NEXT:    vmov.i8 q2, #0xf
+; CHECK-NEXT:    vshr.u16 q4, q0, #2
+; CHECK-NEXT:    vand q0, q0, q3
+; CHECK-NEXT:    vand q4, q4, q3
+; CHECK-NEXT:    vmov.i8 q1, #0x1
+; CHECK-NEXT:    vadd.i16 q0, q0, q4
+; CHECK-NEXT:    vshr.u16 q3, q0, #4
+; CHECK-NEXT:    vadd.i16 q0, q0, q3
+; CHECK-NEXT:    vand q0, q0, q2
+; CHECK-NEXT:    vmul.i16 q0, q0, q1
+; CHECK-NEXT:    vshr.u16 q0, q0, #8
+; CHECK-NEXT:    vpop {d8, d9, d10, d11}
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.ctpop.v8i16(<8 x i16> %src)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @ctpop_16i8_t(<16 x i8> %src){
+; CHECK-LABEL: ctpop_16i8_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9}
+; CHECK-NEXT:    vpush {d8, d9}
+; CHECK-NEXT:    vmov.i8 q3, #0x55
+; CHECK-NEXT:    vshr.u8 q4, q0, #1
+; CHECK-NEXT:    vand q3, q4, q3
+; CHECK-NEXT:    vmov.i8 q2, #0x33
+; CHECK-NEXT:    vsub.i8 q0, q0, q3
+; CHECK-NEXT:    vmov.i8 q1, #0xf
+; CHECK-NEXT:    vshr.u8 q3, q0, #2
+; CHECK-NEXT:    vand q0, q0, q2
+; CHECK-NEXT:    vand q3, q3, q2
+; CHECK-NEXT:    vadd.i8 q0, q0, q3
+; CHECK-NEXT:    vshr.u8 q2, q0, #4
+; CHECK-NEXT:    vadd.i8 q0, q0, q2
+; CHECK-NEXT:    vand q0, q0, q1
+; CHECK-NEXT:    vpop {d8, d9}
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <16 x i8> @llvm.ctpop.v16i8(<16 x i8> %src)
+  ret <16 x i8> %0
+}
+
+declare <2 x i64> @llvm.ctpop.v2i64(<2 x i64>)
+declare <4 x i32> @llvm.ctpop.v4i32(<4 x i32>)
+declare <8 x i16> @llvm.ctpop.v8i16(<8 x i16>)
+declare <16 x i8> @llvm.ctpop.v16i8(<16 x i8>)
diff --git a/llvm/test/CodeGen/Thumb2/mve-cttz.ll b/llvm/test/CodeGen/Thumb2/mve-cttz.ll
new file mode 100644
index 000000000..d1034fb2a
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-cttz.ll
@@ -0,0 +1,160 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -verify-machineinstrs -mattr=+mve %s -o - | FileCheck %s
+
+define arm_aapcs_vfpcc <2 x i64> @cttz_2i64_0_t(<2 x i64> %src){
+; CHECK-LABEL: cttz_2i64_0_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    rbit r0, r0
+; CHECK-NEXT:    cset r1, ne
+; CHECK-NEXT:    lsls r1, r1, #31
+; CHECK-NEXT:    vmov r1, s3
+; CHECK-NEXT:    rbit r1, r1
+; CHECK-NEXT:    clz r1, r1
+; CHECK-NEXT:    add.w r1, r1, #32
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    clzne r1, r0
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vmov s6, r1
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    rbit r0, r0
+; CHECK-NEXT:    cset r1, ne
+; CHECK-NEXT:    lsls r1, r1, #31
+; CHECK-NEXT:    vmov r1, s1
+; CHECK-NEXT:    rbit r1, r1
+; CHECK-NEXT:    clz r1, r1
+; CHECK-NEXT:    add.w r1, r1, #32
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    clzne r1, r0
+; CHECK-NEXT:    vmov s4, r1
+; CHECK-NEXT:    vldr s5, .LCPI0_0
+; CHECK-NEXT:    vmov.f32 s7, s5
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    .p2align 2
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI0_0:
+; CHECK-NEXT:    .long 0 @ float 0
+entry:
+  %0 = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %src, i1 0)
+  ret <2 x i64> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cttz_4i32_0_t(<4 x i32> %src){
+; CHECK-LABEL: cttz_4i32_0_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    movs r0, #32
+; CHECK-NEXT:    vbrsr.32 q0, q0, r0
+; CHECK-NEXT:    vclz.i32 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %src, i1 0)
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @cttz_8i16_0_t(<8 x i16> %src){
+; CHECK-LABEL: cttz_8i16_0_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    movs r0, #16
+; CHECK-NEXT:    vbrsr.16 q0, q0, r0
+; CHECK-NEXT:    vclz.i16 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> %src, i1 0)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @cttz_16i8_0_t(<16 x i8> %src) {
+; CHECK-LABEL: cttz_16i8_0_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    movs r0, #8
+; CHECK-NEXT:    vbrsr.8 q0, q0, r0
+; CHECK-NEXT:    vclz.i8 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %src, i1 0)
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <2 x i64> @cttz_2i64_1_t(<2 x i64> %src){
+; CHECK-LABEL: cttz_2i64_1_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    rbit r0, r0
+; CHECK-NEXT:    cset r1, ne
+; CHECK-NEXT:    lsls r1, r1, #31
+; CHECK-NEXT:    vmov r1, s3
+; CHECK-NEXT:    rbit r1, r1
+; CHECK-NEXT:    clz r1, r1
+; CHECK-NEXT:    add.w r1, r1, #32
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    clzne r1, r0
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vmov s6, r1
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    rbit r0, r0
+; CHECK-NEXT:    cset r1, ne
+; CHECK-NEXT:    lsls r1, r1, #31
+; CHECK-NEXT:    vmov r1, s1
+; CHECK-NEXT:    rbit r1, r1
+; CHECK-NEXT:    clz r1, r1
+; CHECK-NEXT:    add.w r1, r1, #32
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    clzne r1, r0
+; CHECK-NEXT:    vmov s4, r1
+; CHECK-NEXT:    vldr s5, .LCPI4_0
+; CHECK-NEXT:    vmov.f32 s7, s5
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    .p2align 2
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI4_0:
+; CHECK-NEXT:    .long 0 @ float 0
+entry:
+  %0 = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %src, i1 1)
+  ret <2 x i64> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cttz_4i32_1_t(<4 x i32> %src){
+; CHECK-LABEL: cttz_4i32_1_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    movs r0, #32
+; CHECK-NEXT:    vbrsr.32 q0, q0, r0
+; CHECK-NEXT:    vclz.i32 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %src, i1 1)
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @cttz_8i16_1_t(<8 x i16> %src){
+; CHECK-LABEL: cttz_8i16_1_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    movs r0, #16
+; CHECK-NEXT:    vbrsr.16 q0, q0, r0
+; CHECK-NEXT:    vclz.i16 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> %src, i1 1)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @cttz_16i8_1_t(<16 x i8> %src) {
+; CHECK-LABEL: cttz_16i8_1_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    movs r0, #8
+; CHECK-NEXT:    vbrsr.8 q0, q0, r0
+; CHECK-NEXT:    vclz.i8 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %src, i1 1)
+  ret <16 x i8> %0
+}
+
+
+declare <2 x i64> @llvm.cttz.v2i64(<2 x i64>, i1)
+declare <4 x i32> @llvm.cttz.v4i32(<4 x i32>, i1)
+declare <8 x i16> @llvm.cttz.v8i16(<8 x i16>, i1)
+declare <16 x i8> @llvm.cttz.v16i8(<16 x i8>, i1)
diff --git a/llvm/test/CodeGen/Thumb2/mve-div-expand.ll b/llvm/test/CodeGen/Thumb2/mve-div-expand.ll
index e0dddcd27..fbadfcdfc 100644
--- a/llvm/test/CodeGen/Thumb2/mve-div-expand.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-div-expand.ll
@@ -755,14 +755,12 @@ define arm_aapcs_vfpcc <2 x i64> @udiv_i64(<2 x i64> %in1, <2 x i64> %in2) {
 ; CHECK-NEXT:    vmov r2, s18
 ; CHECK-NEXT:    vmov r3, s19
 ; CHECK-NEXT:    vmov.32 q4[0], r0
-; CHECK-NEXT:    lsrl r0, r1, #32
-; CHECK-NEXT:    vmov.32 q4[1], r0
+; CHECK-NEXT:    vmov.32 q4[1], r1
 ; CHECK-NEXT:    mov r0, r12
 ; CHECK-NEXT:    mov r1, lr
 ; CHECK-NEXT:    bl __aeabi_uldivmod
 ; CHECK-NEXT:    vmov.32 q4[2], r0
-; CHECK-NEXT:    lsrl r0, r1, #32
-; CHECK-NEXT:    vmov.32 q4[3], r0
+; CHECK-NEXT:    vmov.32 q4[3], r1
 ; CHECK-NEXT:    vmov q0, q4
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11}
 ; CHECK-NEXT:    pop {r7, pc}
@@ -790,14 +788,12 @@ define arm_aapcs_vfpcc <2 x i64> @sdiv_i64(<2 x i64> %in1, <2 x i64> %in2) {
 ; CHECK-NEXT:    vmov r2, s18
 ; CHECK-NEXT:    vmov r3, s19
 ; CHECK-NEXT:    vmov.32 q4[0], r0
-; CHECK-NEXT:    lsrl r0, r1, #32
-; CHECK-NEXT:    vmov.32 q4[1], r0
+; CHECK-NEXT:    vmov.32 q4[1], r1
 ; CHECK-NEXT:    mov r0, r12
 ; CHECK-NEXT:    mov r1, lr
 ; CHECK-NEXT:    bl __aeabi_ldivmod
 ; CHECK-NEXT:    vmov.32 q4[2], r0
-; CHECK-NEXT:    lsrl r0, r1, #32
-; CHECK-NEXT:    vmov.32 q4[3], r0
+; CHECK-NEXT:    vmov.32 q4[3], r1
 ; CHECK-NEXT:    vmov q0, q4
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11}
 ; CHECK-NEXT:    pop {r7, pc}
@@ -936,60 +932,36 @@ entry:
 define arm_aapcs_vfpcc <8 x half> @fdiv_f16(<8 x half> %in1, <8 x half> %in2) {
 ; CHECK-LABEL: fdiv_f16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov.u16 r0, q1[0]
-; CHECK-NEXT:    vmov.u16 r1, q1[1]
-; CHECK-NEXT:    vmov s8, r0
-; CHECK-NEXT:    vmov.u16 r0, q0[0]
-; CHECK-NEXT:    vmov s10, r0
-; CHECK-NEXT:    vmov.u16 r2, q0[1]
-; CHECK-NEXT:    vdiv.f16 s8, s10, s8
-; CHECK-NEXT:    vmov s10, r2
+; CHECK-NEXT:    vdiv.f16 s8, s0, s4
+; CHECK-NEXT:    vmovx.f16 s10, s0
 ; CHECK-NEXT:    vmov r0, s8
-; CHECK-NEXT:    vmov s8, r1
+; CHECK-NEXT:    vmovx.f16 s8, s4
 ; CHECK-NEXT:    vdiv.f16 s8, s10, s8
+; CHECK-NEXT:    vdiv.f16 s12, s1, s5
 ; CHECK-NEXT:    vmov r1, s8
 ; CHECK-NEXT:    vmov.16 q2[0], r0
-; CHECK-NEXT:    vmov.u16 r0, q1[2]
+; CHECK-NEXT:    vmov r0, s12
+; CHECK-NEXT:    vmovx.f16 s12, s5
+; CHECK-NEXT:    vmovx.f16 s14, s1
 ; CHECK-NEXT:    vmov.16 q2[1], r1
-; CHECK-NEXT:    vmov s12, r0
-; CHECK-NEXT:    vmov.u16 r0, q0[2]
-; CHECK-NEXT:    vmov s14, r0
 ; CHECK-NEXT:    vdiv.f16 s12, s14, s12
-; CHECK-NEXT:    vmov r0, s12
 ; CHECK-NEXT:    vmov.16 q2[2], r0
-; CHECK-NEXT:    vmov.u16 r0, q1[3]
-; CHECK-NEXT:    vmov s12, r0
-; CHECK-NEXT:    vmov.u16 r0, q0[3]
-; CHECK-NEXT:    vmov s14, r0
-; CHECK-NEXT:    vdiv.f16 s12, s14, s12
 ; CHECK-NEXT:    vmov r0, s12
+; CHECK-NEXT:    vdiv.f16 s12, s2, s6
 ; CHECK-NEXT:    vmov.16 q2[3], r0
-; CHECK-NEXT:    vmov.u16 r0, q1[4]
-; CHECK-NEXT:    vmov s12, r0
-; CHECK-NEXT:    vmov.u16 r0, q0[4]
-; CHECK-NEXT:    vmov s14, r0
-; CHECK-NEXT:    vdiv.f16 s12, s14, s12
 ; CHECK-NEXT:    vmov r0, s12
-; CHECK-NEXT:    vmov.16 q2[4], r0
-; CHECK-NEXT:    vmov.u16 r0, q1[5]
-; CHECK-NEXT:    vmov s12, r0
-; CHECK-NEXT:    vmov.u16 r0, q0[5]
-; CHECK-NEXT:    vmov s14, r0
+; CHECK-NEXT:    vmovx.f16 s12, s6
+; CHECK-NEXT:    vmovx.f16 s14, s2
 ; CHECK-NEXT:    vdiv.f16 s12, s14, s12
+; CHECK-NEXT:    vmov.16 q2[4], r0
 ; CHECK-NEXT:    vmov r0, s12
+; CHECK-NEXT:    vdiv.f16 s12, s3, s7
+; CHECK-NEXT:    vmovx.f16 s4, s7
+; CHECK-NEXT:    vmovx.f16 s0, s3
 ; CHECK-NEXT:    vmov.16 q2[5], r0
-; CHECK-NEXT:    vmov.u16 r0, q1[6]
-; CHECK-NEXT:    vmov s12, r0
-; CHECK-NEXT:    vmov.u16 r0, q0[6]
-; CHECK-NEXT:    vmov s14, r0
-; CHECK-NEXT:    vdiv.f16 s12, s14, s12
 ; CHECK-NEXT:    vmov r0, s12
-; CHECK-NEXT:    vmov.16 q2[6], r0
-; CHECK-NEXT:    vmov.u16 r0, q1[7]
-; CHECK-NEXT:    vmov s4, r0
-; CHECK-NEXT:    vmov.u16 r0, q0[7]
-; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vdiv.f16 s0, s0, s4
+; CHECK-NEXT:    vmov.16 q2[6], r0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vmov.16 q2[7], r0
 ; CHECK-NEXT:    vmov q0, q2
@@ -1009,25 +981,19 @@ define arm_aapcs_vfpcc <8 x half> @frem_f16(<8 x half> %in1, <8 x half> %in2) {
 ; CHECK-NEXT:    .pad #64
 ; CHECK-NEXT:    sub sp, #64
 ; CHECK-NEXT:    vmov q4, q0
-; CHECK-NEXT:    vmov.u16 r0, q1[0]
-; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vmov.u16 r0, q4[0]
-; CHECK-NEXT:    vmov s2, r0
 ; CHECK-NEXT:    vmov q5, q1
-; CHECK-NEXT:    vcvtb.f32.f16 s2, s2
-; CHECK-NEXT:    vstr s2, [sp, #56]
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s16
+; CHECK-NEXT:    vstr s0, [sp, #56]
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s20
 ; CHECK-NEXT:    vstr s0, [sp, #60]
 ; CHECK-NEXT:    ldrd r0, r1, [sp, #56]
 ; CHECK-NEXT:    bl fmodf
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vmov.u16 r0, q5[1]
+; CHECK-NEXT:    vmovx.f16 s2, s16
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vmov r4, s0
-; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vmov.u16 r0, q4[1]
-; CHECK-NEXT:    vmov s2, r0
 ; CHECK-NEXT:    vcvtb.f32.f16 s2, s2
+; CHECK-NEXT:    vmov r4, s0
+; CHECK-NEXT:    vmovx.f16 s0, s20
 ; CHECK-NEXT:    vstr s2, [sp, #48]
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
 ; CHECK-NEXT:    vstr s0, [sp, #52]
@@ -1037,85 +1003,67 @@ define arm_aapcs_vfpcc <8 x half> @frem_f16(<8 x half> %in1, <8 x half> %in2) {
 ; CHECK-NEXT:    vmov.16 q6[0], r4
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q6[1], r0
-; CHECK-NEXT:    vmov.u16 r0, q5[2]
-; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vmov.u16 r0, q4[2]
-; CHECK-NEXT:    vmov s2, r0
-; CHECK-NEXT:    vcvtb.f32.f16 s2, s2
-; CHECK-NEXT:    vstr s2, [sp, #40]
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s17
+; CHECK-NEXT:    vstr s0, [sp, #40]
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s21
 ; CHECK-NEXT:    vstr s0, [sp, #44]
+; CHECK-NEXT:    vmov.16 q6[1], r0
 ; CHECK-NEXT:    ldrd r0, r1, [sp, #40]
 ; CHECK-NEXT:    bl fmodf
 ; CHECK-NEXT:    vmov s0, r0
+; CHECK-NEXT:    vmovx.f16 s2, s17
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q6[2], r0
-; CHECK-NEXT:    vmov.u16 r0, q5[3]
-; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vmov.u16 r0, q4[3]
-; CHECK-NEXT:    vmov s2, r0
 ; CHECK-NEXT:    vcvtb.f32.f16 s2, s2
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vmovx.f16 s0, s21
 ; CHECK-NEXT:    vstr s2, [sp, #32]
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
 ; CHECK-NEXT:    vstr s0, [sp, #36]
+; CHECK-NEXT:    vmov.16 q6[2], r0
 ; CHECK-NEXT:    ldrd r0, r1, [sp, #32]
 ; CHECK-NEXT:    bl fmodf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q6[3], r0
-; CHECK-NEXT:    vmov.u16 r0, q5[4]
-; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vmov.u16 r0, q4[4]
-; CHECK-NEXT:    vmov s2, r0
-; CHECK-NEXT:    vcvtb.f32.f16 s2, s2
-; CHECK-NEXT:    vstr s2, [sp, #24]
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s18
+; CHECK-NEXT:    vstr s0, [sp, #24]
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s22
 ; CHECK-NEXT:    vstr s0, [sp, #28]
+; CHECK-NEXT:    vmov.16 q6[3], r0
 ; CHECK-NEXT:    ldrd r0, r1, [sp, #24]
 ; CHECK-NEXT:    bl fmodf
 ; CHECK-NEXT:    vmov s0, r0
+; CHECK-NEXT:    vmovx.f16 s2, s18
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q6[4], r0
-; CHECK-NEXT:    vmov.u16 r0, q5[5]
-; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vmov.u16 r0, q4[5]
-; CHECK-NEXT:    vmov s2, r0
 ; CHECK-NEXT:    vcvtb.f32.f16 s2, s2
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vmovx.f16 s0, s22
 ; CHECK-NEXT:    vstr s2, [sp, #16]
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
 ; CHECK-NEXT:    vstr s0, [sp, #20]
+; CHECK-NEXT:    vmov.16 q6[4], r0
 ; CHECK-NEXT:    ldrd r0, r1, [sp, #16]
 ; CHECK-NEXT:    bl fmodf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q6[5], r0
-; CHECK-NEXT:    vmov.u16 r0, q5[6]
-; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vmov.u16 r0, q4[6]
-; CHECK-NEXT:    vmov s2, r0
-; CHECK-NEXT:    vcvtb.f32.f16 s2, s2
-; CHECK-NEXT:    vstr s2, [sp, #8]
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s19
+; CHECK-NEXT:    vstr s0, [sp, #8]
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s23
 ; CHECK-NEXT:    vstr s0, [sp, #12]
+; CHECK-NEXT:    vmov.16 q6[5], r0
 ; CHECK-NEXT:    ldrd r0, r1, [sp, #8]
 ; CHECK-NEXT:    bl fmodf
 ; CHECK-NEXT:    vmov s0, r0
+; CHECK-NEXT:    vmovx.f16 s2, s19
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q6[6], r0
-; CHECK-NEXT:    vmov.u16 r0, q5[7]
-; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vmov.u16 r0, q4[7]
-; CHECK-NEXT:    vmov s2, r0
 ; CHECK-NEXT:    vcvtb.f32.f16 s2, s2
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vmovx.f16 s0, s23
 ; CHECK-NEXT:    vstr s2, [sp]
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
 ; CHECK-NEXT:    vstr s0, [sp, #4]
+; CHECK-NEXT:    vmov.16 q6[6], r0
 ; CHECK-NEXT:    ldrd r0, r1, [sp]
 ; CHECK-NEXT:    bl fmodf
 ; CHECK-NEXT:    vmov s0, r0
diff --git a/llvm/test/CodeGen/Thumb2/mve-extractelt.ll b/llvm/test/CodeGen/Thumb2/mve-extractelt.ll
new file mode 100644
index 000000000..e0f1e9468
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-extractelt.ll
@@ -0,0 +1,86 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s
+
+define arm_aapcs_vfpcc i32 @u8_explicit_extend(<16 x i8> %a) {
+; CHECK-LABEL: u8_explicit_extend:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.u8 r0, q0[10]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = extractelement <16 x i8> %a, i32 10
+  %1 = zext i8 %0 to i32
+  ret i32 %1
+}
+
+define arm_aapcs_vfpcc i32 @s8_explicit_extend(<16 x i8> %a) {
+; CHECK-LABEL: s8_explicit_extend:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.s8 r0, q0[10]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = extractelement <16 x i8> %a, i32 10
+  %1 = sext i8 %0 to i32
+  ret i32 %1
+}
+
+define arm_aapcs_vfpcc i8 @u8_extend_via_pcs(<16 x i8> %a) {
+; CHECK-LABEL: u8_extend_via_pcs:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.u8 r0, q0[10]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = extractelement <16 x i8> %a, i32 10
+  ret i8 %0
+}
+
+define arm_aapcs_vfpcc signext i8 @s8_extend_via_pcs(<16 x i8> %a) {
+; CHECK-LABEL: s8_extend_via_pcs:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.s8 r0, q0[10]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = extractelement <16 x i8> %a, i32 10
+  ret i8 %0
+}
+
+define arm_aapcs_vfpcc i32 @u16_explicit_extend(<8 x i16> %a) {
+; CHECK-LABEL: u16_explicit_extend:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.u16 r0, q0[5]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = extractelement <8 x i16> %a, i32 5
+  %1 = zext i16 %0 to i32
+  ret i32 %1
+}
+
+define arm_aapcs_vfpcc i32 @s16_explicit_extend(<8 x i16> %a) {
+; CHECK-LABEL: s16_explicit_extend:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.s16 r0, q0[5]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = extractelement <8 x i16> %a, i32 5
+  %1 = sext i16 %0 to i32
+  ret i32 %1
+}
+
+define arm_aapcs_vfpcc i16 @u16_extend_via_pcs(<8 x i16> %a) {
+; CHECK-LABEL: u16_extend_via_pcs:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.u16 r0, q0[5]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = extractelement <8 x i16> %a, i32 5
+  ret i16 %0
+}
+
+define arm_aapcs_vfpcc signext i16 @s16_extend_via_pcs(<8 x i16> %a) {
+; CHECK-LABEL: s16_extend_via_pcs:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.s16 r0, q0[5]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = extractelement <8 x i16> %a, i32 5
+  ret i16 %0
+}
diff --git a/llvm/test/CodeGen/Thumb2/mve-fmas.ll b/llvm/test/CodeGen/Thumb2/mve-fmas.ll
new file mode 100644
index 000000000..5b53bcf14
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-fmas.ll
@@ -0,0 +1,271 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi, -mattr=+mve.fp -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK-MVE-FP
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi, -mattr=+mve.fp -fp-contract=fast -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK-MVE-VMLA
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve,+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK-MVE
+
+define arm_aapcs_vfpcc <8 x half> @vfma16_v1(<8 x half> %src1, <8 x half> %src2, <8 x half> %src3) {
+; CHECK-MVE-FP-LABEL: vfma16_v1:
+; CHECK-MVE-FP:       @ %bb.0: @ %entry
+; CHECK-MVE-FP-NEXT:    vmul.f16 q1, q1, q2
+; CHECK-MVE-FP-NEXT:    vadd.f16 q0, q0, q1
+; CHECK-MVE-FP-NEXT:    bx lr
+;
+; CHECK-MVE-VMLA-LABEL: vfma16_v1:
+; CHECK-MVE-VMLA:       @ %bb.0: @ %entry
+; CHECK-MVE-VMLA-NEXT:    vfma.f16 q0, q1, q2
+; CHECK-MVE-VMLA-NEXT:    bx lr
+;
+; CHECK-MVE-LABEL: vfma16_v1:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9, d10}
+; CHECK-MVE-NEXT:    vpush {d8, d9, d10}
+; CHECK-MVE-NEXT:    vmovx.f16 s13, s0
+; CHECK-MVE-NEXT:    vmla.f16 s0, s4, s8
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s8
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s4
+; CHECK-MVE-NEXT:    vmov.f32 s16, s1
+; CHECK-MVE-NEXT:    vmla.f16 s13, s14, s12
+; CHECK-MVE-NEXT:    vmov r1, s0
+; CHECK-MVE-NEXT:    vmla.f16 s16, s5, s9
+; CHECK-MVE-NEXT:    vmov r0, s13
+; CHECK-MVE-NEXT:    vmov.16 q3[0], r1
+; CHECK-MVE-NEXT:    vmov.16 q3[1], r0
+; CHECK-MVE-NEXT:    vmov r0, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s9
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s5
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s1
+; CHECK-MVE-NEXT:    vmov.16 q3[2], r0
+; CHECK-MVE-NEXT:    vmla.f16 s20, s18, s16
+; CHECK-MVE-NEXT:    vmov.f32 s16, s2
+; CHECK-MVE-NEXT:    vmov r0, s20
+; CHECK-MVE-NEXT:    vmla.f16 s16, s6, s10
+; CHECK-MVE-NEXT:    vmov.16 q3[3], r0
+; CHECK-MVE-NEXT:    vmov r0, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s10
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s6
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s2
+; CHECK-MVE-NEXT:    vmov.16 q3[4], r0
+; CHECK-MVE-NEXT:    vmla.f16 s20, s18, s16
+; CHECK-MVE-NEXT:    vmov.f32 s16, s3
+; CHECK-MVE-NEXT:    vmov r0, s20
+; CHECK-MVE-NEXT:    vmla.f16 s16, s7, s11
+; CHECK-MVE-NEXT:    vmovx.f16 s8, s11
+; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q3[5], r0
+; CHECK-MVE-NEXT:    vmov r0, s16
+; CHECK-MVE-NEXT:    vmla.f16 s0, s4, s8
+; CHECK-MVE-NEXT:    vmov.16 q3[6], r0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q3
+; CHECK-MVE-NEXT:    vpop {d8, d9, d10}
+; CHECK-MVE-NEXT:    bx lr
+entry:
+  %0 = fmul <8 x half> %src2, %src3
+  %1 = fadd <8 x half> %src1, %0
+  ret <8 x half> %1
+}
+
+define arm_aapcs_vfpcc <8 x half> @vfma16_v2(<8 x half> %src1, <8 x half> %src2, <8 x half> %src3) {
+; CHECK-MVE-FP-LABEL: vfma16_v2:
+; CHECK-MVE-FP:       @ %bb.0: @ %entry
+; CHECK-MVE-FP-NEXT:    vmul.f16 q1, q1, q2
+; CHECK-MVE-FP-NEXT:    vadd.f16 q0, q1, q0
+; CHECK-MVE-FP-NEXT:    bx lr
+;
+; CHECK-MVE-VMLA-LABEL: vfma16_v2:
+; CHECK-MVE-VMLA:       @ %bb.0: @ %entry
+; CHECK-MVE-VMLA-NEXT:    vfma.f16 q0, q1, q2
+; CHECK-MVE-VMLA-NEXT:    bx lr
+;
+; CHECK-MVE-LABEL: vfma16_v2:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9, d10}
+; CHECK-MVE-NEXT:    vpush {d8, d9, d10}
+; CHECK-MVE-NEXT:    vmovx.f16 s13, s0
+; CHECK-MVE-NEXT:    vmla.f16 s0, s4, s8
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s8
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s4
+; CHECK-MVE-NEXT:    vmov.f32 s16, s1
+; CHECK-MVE-NEXT:    vmla.f16 s13, s14, s12
+; CHECK-MVE-NEXT:    vmov r1, s0
+; CHECK-MVE-NEXT:    vmla.f16 s16, s5, s9
+; CHECK-MVE-NEXT:    vmov r0, s13
+; CHECK-MVE-NEXT:    vmov.16 q3[0], r1
+; CHECK-MVE-NEXT:    vmov.16 q3[1], r0
+; CHECK-MVE-NEXT:    vmov r0, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s9
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s5
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s1
+; CHECK-MVE-NEXT:    vmov.16 q3[2], r0
+; CHECK-MVE-NEXT:    vmla.f16 s20, s18, s16
+; CHECK-MVE-NEXT:    vmov.f32 s16, s2
+; CHECK-MVE-NEXT:    vmov r0, s20
+; CHECK-MVE-NEXT:    vmla.f16 s16, s6, s10
+; CHECK-MVE-NEXT:    vmov.16 q3[3], r0
+; CHECK-MVE-NEXT:    vmov r0, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s10
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s6
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s2
+; CHECK-MVE-NEXT:    vmov.16 q3[4], r0
+; CHECK-MVE-NEXT:    vmla.f16 s20, s18, s16
+; CHECK-MVE-NEXT:    vmov.f32 s16, s3
+; CHECK-MVE-NEXT:    vmov r0, s20
+; CHECK-MVE-NEXT:    vmla.f16 s16, s7, s11
+; CHECK-MVE-NEXT:    vmovx.f16 s8, s11
+; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q3[5], r0
+; CHECK-MVE-NEXT:    vmov r0, s16
+; CHECK-MVE-NEXT:    vmla.f16 s0, s4, s8
+; CHECK-MVE-NEXT:    vmov.16 q3[6], r0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q3
+; CHECK-MVE-NEXT:    vpop {d8, d9, d10}
+; CHECK-MVE-NEXT:    bx lr
+entry:
+  %0 = fmul <8 x half> %src2, %src3
+  %1 = fadd <8 x half> %0, %src1
+  ret <8 x half> %1
+}
+
+define arm_aapcs_vfpcc <8 x half> @vfms16(<8 x half> %src1, <8 x half> %src2, <8 x half> %src3) {
+; CHECK-MVE-FP-LABEL: vfms16:
+; CHECK-MVE-FP:       @ %bb.0: @ %entry
+; CHECK-MVE-FP-NEXT:    vmul.f16 q1, q1, q2
+; CHECK-MVE-FP-NEXT:    vsub.f16 q0, q0, q1
+; CHECK-MVE-FP-NEXT:    bx lr
+;
+; CHECK-MVE-VMLA-LABEL: vfms16:
+; CHECK-MVE-VMLA:       @ %bb.0: @ %entry
+; CHECK-MVE-VMLA-NEXT:    vfms.f16 q0, q1, q2
+; CHECK-MVE-VMLA-NEXT:    bx lr
+;
+; CHECK-MVE-LABEL: vfms16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9, d10}
+; CHECK-MVE-NEXT:    vpush {d8, d9, d10}
+; CHECK-MVE-NEXT:    vmovx.f16 s13, s0
+; CHECK-MVE-NEXT:    vmls.f16 s0, s4, s8
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s8
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s4
+; CHECK-MVE-NEXT:    vmov.f32 s16, s1
+; CHECK-MVE-NEXT:    vmls.f16 s13, s14, s12
+; CHECK-MVE-NEXT:    vmov r1, s0
+; CHECK-MVE-NEXT:    vmls.f16 s16, s5, s9
+; CHECK-MVE-NEXT:    vmov r0, s13
+; CHECK-MVE-NEXT:    vmov.16 q3[0], r1
+; CHECK-MVE-NEXT:    vmov.16 q3[1], r0
+; CHECK-MVE-NEXT:    vmov r0, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s9
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s5
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s1
+; CHECK-MVE-NEXT:    vmov.16 q3[2], r0
+; CHECK-MVE-NEXT:    vmls.f16 s20, s18, s16
+; CHECK-MVE-NEXT:    vmov.f32 s16, s2
+; CHECK-MVE-NEXT:    vmov r0, s20
+; CHECK-MVE-NEXT:    vmls.f16 s16, s6, s10
+; CHECK-MVE-NEXT:    vmov.16 q3[3], r0
+; CHECK-MVE-NEXT:    vmov r0, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s10
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s6
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s2
+; CHECK-MVE-NEXT:    vmov.16 q3[4], r0
+; CHECK-MVE-NEXT:    vmls.f16 s20, s18, s16
+; CHECK-MVE-NEXT:    vmov.f32 s16, s3
+; CHECK-MVE-NEXT:    vmov r0, s20
+; CHECK-MVE-NEXT:    vmls.f16 s16, s7, s11
+; CHECK-MVE-NEXT:    vmovx.f16 s8, s11
+; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q3[5], r0
+; CHECK-MVE-NEXT:    vmov r0, s16
+; CHECK-MVE-NEXT:    vmls.f16 s0, s4, s8
+; CHECK-MVE-NEXT:    vmov.16 q3[6], r0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q3
+; CHECK-MVE-NEXT:    vpop {d8, d9, d10}
+; CHECK-MVE-NEXT:    bx lr
+entry:
+  %0 = fmul <8 x half> %src2, %src3
+  %1 = fsub <8 x half> %src1, %0
+  ret <8 x half> %1
+}
+
+define arm_aapcs_vfpcc <4 x float> @vfma32_v1(<4 x float> %src1, <4 x float> %src2, <4 x float> %src3) {
+; CHECK-MVE-FP-LABEL: vfma32_v1:
+; CHECK-MVE-FP:       @ %bb.0: @ %entry
+; CHECK-MVE-FP-NEXT:    vmul.f32 q1, q1, q2
+; CHECK-MVE-FP-NEXT:    vadd.f32 q0, q0, q1
+; CHECK-MVE-FP-NEXT:    bx lr
+;
+; CHECK-MVE-VMLA-LABEL: vfma32_v1:
+; CHECK-MVE-VMLA:       @ %bb.0: @ %entry
+; CHECK-MVE-VMLA-NEXT:    vfma.f32 q0, q1, q2
+; CHECK-MVE-VMLA-NEXT:    bx lr
+;
+; CHECK-MVE-LABEL: vfma32_v1:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vmla.f32 s3, s7, s11
+; CHECK-MVE-NEXT:    vmla.f32 s2, s6, s10
+; CHECK-MVE-NEXT:    vmla.f32 s1, s5, s9
+; CHECK-MVE-NEXT:    vmla.f32 s0, s4, s8
+; CHECK-MVE-NEXT:    bx lr
+entry:
+  %0 = fmul <4 x float> %src2, %src3
+  %1 = fadd <4 x float> %src1, %0
+  ret <4 x float> %1
+}
+
+define arm_aapcs_vfpcc <4 x float> @vfma32_v2(<4 x float> %src1, <4 x float> %src2, <4 x float> %src3) {
+; CHECK-MVE-FP-LABEL: vfma32_v2:
+; CHECK-MVE-FP:       @ %bb.0: @ %entry
+; CHECK-MVE-FP-NEXT:    vmul.f32 q1, q1, q2
+; CHECK-MVE-FP-NEXT:    vadd.f32 q0, q1, q0
+; CHECK-MVE-FP-NEXT:    bx lr
+;
+; CHECK-MVE-VMLA-LABEL: vfma32_v2:
+; CHECK-MVE-VMLA:       @ %bb.0: @ %entry
+; CHECK-MVE-VMLA-NEXT:    vfma.f32 q0, q1, q2
+; CHECK-MVE-VMLA-NEXT:    bx lr
+;
+; CHECK-MVE-LABEL: vfma32_v2:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vmla.f32 s3, s7, s11
+; CHECK-MVE-NEXT:    vmla.f32 s2, s6, s10
+; CHECK-MVE-NEXT:    vmla.f32 s1, s5, s9
+; CHECK-MVE-NEXT:    vmla.f32 s0, s4, s8
+; CHECK-MVE-NEXT:    bx lr
+entry:
+  %0 = fmul <4 x float> %src2, %src3
+  %1 = fadd <4 x float> %0, %src1
+  ret <4 x float> %1
+}
+
+define arm_aapcs_vfpcc <4 x float> @vfms32(<4 x float> %src1, <4 x float> %src2, <4 x float> %src3) {
+; CHECK-MVE-FP-LABEL: vfms32:
+; CHECK-MVE-FP:       @ %bb.0: @ %entry
+; CHECK-MVE-FP-NEXT:    vmul.f32 q1, q1, q2
+; CHECK-MVE-FP-NEXT:    vsub.f32 q0, q0, q1
+; CHECK-MVE-FP-NEXT:    bx lr
+;
+; CHECK-MVE-VMLA-LABEL: vfms32:
+; CHECK-MVE-VMLA:       @ %bb.0: @ %entry
+; CHECK-MVE-VMLA-NEXT:    vfms.f32 q0, q1, q2
+; CHECK-MVE-VMLA-NEXT:    bx lr
+;
+; CHECK-MVE-LABEL: vfms32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vmls.f32 s3, s7, s11
+; CHECK-MVE-NEXT:    vmls.f32 s2, s6, s10
+; CHECK-MVE-NEXT:    vmls.f32 s1, s5, s9
+; CHECK-MVE-NEXT:    vmls.f32 s0, s4, s8
+; CHECK-MVE-NEXT:    bx lr
+entry:
+  %0 = fmul <4 x float> %src2, %src3
+  %1 = fsub <4 x float> %src1, %0
+  ret <4 x float> %1
+}
diff --git a/llvm/test/CodeGen/Thumb2/mve-fmath.ll b/llvm/test/CodeGen/Thumb2/mve-fmath.ll
index 31956979d..af9a3ed91 100644
--- a/llvm/test/CodeGen/Thumb2/mve-fmath.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-fmath.ll
@@ -19,44 +19,32 @@ entry:
 define arm_aapcs_vfpcc <8 x half> @sqrt_float16_t(<8 x half> %src) {
 ; CHECK-LABEL: sqrt_float16_t:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov.u16 r0, q0[0]
-; CHECK-NEXT:    vmov.u16 r1, q0[1]
-; CHECK-NEXT:    vmov s4, r0
+; CHECK-NEXT:    vmovx.f16 s4, s0
+; CHECK-NEXT:    vsqrt.f16 s8, s1
 ; CHECK-NEXT:    vsqrt.f16 s4, s4
 ; CHECK-NEXT:    vmov r0, s4
-; CHECK-NEXT:    vmov s4, r1
-; CHECK-NEXT:    vsqrt.f16 s4, s4
+; CHECK-NEXT:    vsqrt.f16 s4, s0
 ; CHECK-NEXT:    vmov r1, s4
-; CHECK-NEXT:    vmov.16 q1[0], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[2]
-; CHECK-NEXT:    vmov.16 q1[1], r1
-; CHECK-NEXT:    vmov s8, r0
-; CHECK-NEXT:    vsqrt.f16 s8, s8
+; CHECK-NEXT:    vmovx.f16 s0, s3
+; CHECK-NEXT:    vmov.16 q1[0], r1
+; CHECK-NEXT:    vsqrt.f16 s0, s0
+; CHECK-NEXT:    vmov.16 q1[1], r0
 ; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    vmovx.f16 s8, s1
 ; CHECK-NEXT:    vmov.16 q1[2], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[3]
-; CHECK-NEXT:    vmov s8, r0
 ; CHECK-NEXT:    vsqrt.f16 s8, s8
 ; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    vsqrt.f16 s8, s2
 ; CHECK-NEXT:    vmov.16 q1[3], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[4]
-; CHECK-NEXT:    vmov s8, r0
-; CHECK-NEXT:    vsqrt.f16 s8, s8
 ; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    vmovx.f16 s8, s2
 ; CHECK-NEXT:    vmov.16 q1[4], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[5]
-; CHECK-NEXT:    vmov s8, r0
 ; CHECK-NEXT:    vsqrt.f16 s8, s8
 ; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    vsqrt.f16 s8, s3
 ; CHECK-NEXT:    vmov.16 q1[5], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[6]
-; CHECK-NEXT:    vmov s8, r0
-; CHECK-NEXT:    vsqrt.f16 s8, s8
 ; CHECK-NEXT:    vmov r0, s8
 ; CHECK-NEXT:    vmov.16 q1[6], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[7]
-; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vsqrt.f16 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vmov.16 q1[7], r0
 ; CHECK-NEXT:    vmov q0, q1
@@ -132,80 +120,68 @@ define arm_aapcs_vfpcc <8 x half> @cos_float16_t(<8 x half> %src) {
 ; CHECK-NEXT:    vpush {d8, d9, d10, d11}
 ; CHECK-NEXT:    .pad #32
 ; CHECK-NEXT:    sub sp, #32
-; CHECK-NEXT:    vmov.u16 r0, q0[1]
 ; CHECK-NEXT:    vmov q4, q0
-; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vstr s0, [sp, #24]
-; CHECK-NEXT:    ldr r0, [sp, #24]
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s16
+; CHECK-NEXT:    vstr s0, [sp, #28]
+; CHECK-NEXT:    ldr r0, [sp, #28]
 ; CHECK-NEXT:    bl cosf
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vmov.u16 r0, q4[0]
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r4, s0
-; CHECK-NEXT:    vmov s0, r0
+; CHECK-NEXT:    vmovx.f16 s0, s16
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vstr s0, [sp, #28]
-; CHECK-NEXT:    ldr r0, [sp, #28]
+; CHECK-NEXT:    vstr s0, [sp, #24]
+; CHECK-NEXT:    ldr r0, [sp, #24]
 ; CHECK-NEXT:    bl cosf
 ; CHECK-NEXT:    vmov s0, r0
+; CHECK-NEXT:    vmov.16 q5[0], r4
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q5[0], r0
-; CHECK-NEXT:    vmov.u16 r0, q4[2]
-; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vmov.16 q5[1], r4
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s17
 ; CHECK-NEXT:    vstr s0, [sp, #20]
+; CHECK-NEXT:    vmov.16 q5[1], r0
 ; CHECK-NEXT:    ldr r0, [sp, #20]
 ; CHECK-NEXT:    bl cosf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q5[2], r0
-; CHECK-NEXT:    vmov.u16 r0, q4[3]
-; CHECK-NEXT:    vmov s0, r0
+; CHECK-NEXT:    vmovx.f16 s0, s17
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-NEXT:    vmov.16 q5[2], r0
 ; CHECK-NEXT:    vstr s0, [sp, #16]
 ; CHECK-NEXT:    ldr r0, [sp, #16]
 ; CHECK-NEXT:    bl cosf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q5[3], r0
-; CHECK-NEXT:    vmov.u16 r0, q4[4]
-; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s18
 ; CHECK-NEXT:    vstr s0, [sp, #12]
+; CHECK-NEXT:    vmov.16 q5[3], r0
 ; CHECK-NEXT:    ldr r0, [sp, #12]
 ; CHECK-NEXT:    bl cosf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q5[4], r0
-; CHECK-NEXT:    vmov.u16 r0, q4[5]
-; CHECK-NEXT:    vmov s0, r0
+; CHECK-NEXT:    vmovx.f16 s0, s18
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-NEXT:    vmov.16 q5[4], r0
 ; CHECK-NEXT:    vstr s0, [sp, #8]
 ; CHECK-NEXT:    ldr r0, [sp, #8]
 ; CHECK-NEXT:    bl cosf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q5[5], r0
-; CHECK-NEXT:    vmov.u16 r0, q4[6]
-; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s19
 ; CHECK-NEXT:    vstr s0, [sp, #4]
+; CHECK-NEXT:    vmov.16 q5[5], r0
 ; CHECK-NEXT:    ldr r0, [sp, #4]
 ; CHECK-NEXT:    bl cosf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q5[6], r0
-; CHECK-NEXT:    vmov.u16 r0, q4[7]
-; CHECK-NEXT:    vmov s0, r0
+; CHECK-NEXT:    vmovx.f16 s0, s19
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-NEXT:    vmov.16 q5[6], r0
 ; CHECK-NEXT:    vstr s0, [sp]
 ; CHECK-NEXT:    ldr r0, [sp]
 ; CHECK-NEXT:    bl cosf
@@ -288,80 +264,68 @@ define arm_aapcs_vfpcc <8 x half> @sin_float16_t(<8 x half> %src) {
 ; CHECK-NEXT:    vpush {d8, d9, d10, d11}
 ; CHECK-NEXT:    .pad #32
 ; CHECK-NEXT:    sub sp, #32
-; CHECK-NEXT:    vmov.u16 r0, q0[1]
 ; CHECK-NEXT:    vmov q4, q0
-; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vstr s0, [sp, #24]
-; CHECK-NEXT:    ldr r0, [sp, #24]
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s16
+; CHECK-NEXT:    vstr s0, [sp, #28]
+; CHECK-NEXT:    ldr r0, [sp, #28]
 ; CHECK-NEXT:    bl sinf
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vmov.u16 r0, q4[0]
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r4, s0
-; CHECK-NEXT:    vmov s0, r0
+; CHECK-NEXT:    vmovx.f16 s0, s16
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vstr s0, [sp, #28]
-; CHECK-NEXT:    ldr r0, [sp, #28]
+; CHECK-NEXT:    vstr s0, [sp, #24]
+; CHECK-NEXT:    ldr r0, [sp, #24]
 ; CHECK-NEXT:    bl sinf
 ; CHECK-NEXT:    vmov s0, r0
+; CHECK-NEXT:    vmov.16 q5[0], r4
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q5[0], r0
-; CHECK-NEXT:    vmov.u16 r0, q4[2]
-; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vmov.16 q5[1], r4
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s17
 ; CHECK-NEXT:    vstr s0, [sp, #20]
+; CHECK-NEXT:    vmov.16 q5[1], r0
 ; CHECK-NEXT:    ldr r0, [sp, #20]
 ; CHECK-NEXT:    bl sinf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q5[2], r0
-; CHECK-NEXT:    vmov.u16 r0, q4[3]
-; CHECK-NEXT:    vmov s0, r0
+; CHECK-NEXT:    vmovx.f16 s0, s17
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-NEXT:    vmov.16 q5[2], r0
 ; CHECK-NEXT:    vstr s0, [sp, #16]
 ; CHECK-NEXT:    ldr r0, [sp, #16]
 ; CHECK-NEXT:    bl sinf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q5[3], r0
-; CHECK-NEXT:    vmov.u16 r0, q4[4]
-; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s18
 ; CHECK-NEXT:    vstr s0, [sp, #12]
+; CHECK-NEXT:    vmov.16 q5[3], r0
 ; CHECK-NEXT:    ldr r0, [sp, #12]
 ; CHECK-NEXT:    bl sinf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q5[4], r0
-; CHECK-NEXT:    vmov.u16 r0, q4[5]
-; CHECK-NEXT:    vmov s0, r0
+; CHECK-NEXT:    vmovx.f16 s0, s18
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-NEXT:    vmov.16 q5[4], r0
 ; CHECK-NEXT:    vstr s0, [sp, #8]
 ; CHECK-NEXT:    ldr r0, [sp, #8]
 ; CHECK-NEXT:    bl sinf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q5[5], r0
-; CHECK-NEXT:    vmov.u16 r0, q4[6]
-; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s19
 ; CHECK-NEXT:    vstr s0, [sp, #4]
+; CHECK-NEXT:    vmov.16 q5[5], r0
 ; CHECK-NEXT:    ldr r0, [sp, #4]
 ; CHECK-NEXT:    bl sinf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q5[6], r0
-; CHECK-NEXT:    vmov.u16 r0, q4[7]
-; CHECK-NEXT:    vmov s0, r0
+; CHECK-NEXT:    vmovx.f16 s0, s19
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-NEXT:    vmov.16 q5[6], r0
 ; CHECK-NEXT:    vstr s0, [sp]
 ; CHECK-NEXT:    ldr r0, [sp]
 ; CHECK-NEXT:    bl sinf
@@ -444,80 +408,68 @@ define arm_aapcs_vfpcc <8 x half> @exp_float16_t(<8 x half> %src) {
 ; CHECK-NEXT:    vpush {d8, d9, d10, d11}
 ; CHECK-NEXT:    .pad #32
 ; CHECK-NEXT:    sub sp, #32
-; CHECK-NEXT:    vmov.u16 r0, q0[1]
 ; CHECK-NEXT:    vmov q4, q0
-; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vstr s0, [sp, #24]
-; CHECK-NEXT:    ldr r0, [sp, #24]
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s16
+; CHECK-NEXT:    vstr s0, [sp, #28]
+; CHECK-NEXT:    ldr r0, [sp, #28]
 ; CHECK-NEXT:    bl expf
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vmov.u16 r0, q4[0]
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r4, s0
-; CHECK-NEXT:    vmov s0, r0
+; CHECK-NEXT:    vmovx.f16 s0, s16
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vstr s0, [sp, #28]
-; CHECK-NEXT:    ldr r0, [sp, #28]
+; CHECK-NEXT:    vstr s0, [sp, #24]
+; CHECK-NEXT:    ldr r0, [sp, #24]
 ; CHECK-NEXT:    bl expf
 ; CHECK-NEXT:    vmov s0, r0
+; CHECK-NEXT:    vmov.16 q5[0], r4
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q5[0], r0
-; CHECK-NEXT:    vmov.u16 r0, q4[2]
-; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vmov.16 q5[1], r4
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s17
 ; CHECK-NEXT:    vstr s0, [sp, #20]
+; CHECK-NEXT:    vmov.16 q5[1], r0
 ; CHECK-NEXT:    ldr r0, [sp, #20]
 ; CHECK-NEXT:    bl expf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q5[2], r0
-; CHECK-NEXT:    vmov.u16 r0, q4[3]
-; CHECK-NEXT:    vmov s0, r0
+; CHECK-NEXT:    vmovx.f16 s0, s17
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-NEXT:    vmov.16 q5[2], r0
 ; CHECK-NEXT:    vstr s0, [sp, #16]
 ; CHECK-NEXT:    ldr r0, [sp, #16]
 ; CHECK-NEXT:    bl expf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q5[3], r0
-; CHECK-NEXT:    vmov.u16 r0, q4[4]
-; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s18
 ; CHECK-NEXT:    vstr s0, [sp, #12]
+; CHECK-NEXT:    vmov.16 q5[3], r0
 ; CHECK-NEXT:    ldr r0, [sp, #12]
 ; CHECK-NEXT:    bl expf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q5[4], r0
-; CHECK-NEXT:    vmov.u16 r0, q4[5]
-; CHECK-NEXT:    vmov s0, r0
+; CHECK-NEXT:    vmovx.f16 s0, s18
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-NEXT:    vmov.16 q5[4], r0
 ; CHECK-NEXT:    vstr s0, [sp, #8]
 ; CHECK-NEXT:    ldr r0, [sp, #8]
 ; CHECK-NEXT:    bl expf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q5[5], r0
-; CHECK-NEXT:    vmov.u16 r0, q4[6]
-; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s19
 ; CHECK-NEXT:    vstr s0, [sp, #4]
+; CHECK-NEXT:    vmov.16 q5[5], r0
 ; CHECK-NEXT:    ldr r0, [sp, #4]
 ; CHECK-NEXT:    bl expf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q5[6], r0
-; CHECK-NEXT:    vmov.u16 r0, q4[7]
-; CHECK-NEXT:    vmov s0, r0
+; CHECK-NEXT:    vmovx.f16 s0, s19
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-NEXT:    vmov.16 q5[6], r0
 ; CHECK-NEXT:    vstr s0, [sp]
 ; CHECK-NEXT:    ldr r0, [sp]
 ; CHECK-NEXT:    bl expf
@@ -600,80 +552,68 @@ define arm_aapcs_vfpcc <8 x half> @exp2_float16_t(<8 x half> %src) {
 ; CHECK-NEXT:    vpush {d8, d9, d10, d11}
 ; CHECK-NEXT:    .pad #32
 ; CHECK-NEXT:    sub sp, #32
-; CHECK-NEXT:    vmov.u16 r0, q0[1]
 ; CHECK-NEXT:    vmov q4, q0
-; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vstr s0, [sp, #24]
-; CHECK-NEXT:    ldr r0, [sp, #24]
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s16
+; CHECK-NEXT:    vstr s0, [sp, #28]
+; CHECK-NEXT:    ldr r0, [sp, #28]
 ; CHECK-NEXT:    bl exp2f
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vmov.u16 r0, q4[0]
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r4, s0
-; CHECK-NEXT:    vmov s0, r0
+; CHECK-NEXT:    vmovx.f16 s0, s16
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vstr s0, [sp, #28]
-; CHECK-NEXT:    ldr r0, [sp, #28]
+; CHECK-NEXT:    vstr s0, [sp, #24]
+; CHECK-NEXT:    ldr r0, [sp, #24]
 ; CHECK-NEXT:    bl exp2f
 ; CHECK-NEXT:    vmov s0, r0
+; CHECK-NEXT:    vmov.16 q5[0], r4
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q5[0], r0
-; CHECK-NEXT:    vmov.u16 r0, q4[2]
-; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vmov.16 q5[1], r4
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s17
 ; CHECK-NEXT:    vstr s0, [sp, #20]
+; CHECK-NEXT:    vmov.16 q5[1], r0
 ; CHECK-NEXT:    ldr r0, [sp, #20]
 ; CHECK-NEXT:    bl exp2f
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q5[2], r0
-; CHECK-NEXT:    vmov.u16 r0, q4[3]
-; CHECK-NEXT:    vmov s0, r0
+; CHECK-NEXT:    vmovx.f16 s0, s17
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-NEXT:    vmov.16 q5[2], r0
 ; CHECK-NEXT:    vstr s0, [sp, #16]
 ; CHECK-NEXT:    ldr r0, [sp, #16]
 ; CHECK-NEXT:    bl exp2f
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q5[3], r0
-; CHECK-NEXT:    vmov.u16 r0, q4[4]
-; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s18
 ; CHECK-NEXT:    vstr s0, [sp, #12]
+; CHECK-NEXT:    vmov.16 q5[3], r0
 ; CHECK-NEXT:    ldr r0, [sp, #12]
 ; CHECK-NEXT:    bl exp2f
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q5[4], r0
-; CHECK-NEXT:    vmov.u16 r0, q4[5]
-; CHECK-NEXT:    vmov s0, r0
+; CHECK-NEXT:    vmovx.f16 s0, s18
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-NEXT:    vmov.16 q5[4], r0
 ; CHECK-NEXT:    vstr s0, [sp, #8]
 ; CHECK-NEXT:    ldr r0, [sp, #8]
 ; CHECK-NEXT:    bl exp2f
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q5[5], r0
-; CHECK-NEXT:    vmov.u16 r0, q4[6]
-; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s19
 ; CHECK-NEXT:    vstr s0, [sp, #4]
+; CHECK-NEXT:    vmov.16 q5[5], r0
 ; CHECK-NEXT:    ldr r0, [sp, #4]
 ; CHECK-NEXT:    bl exp2f
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q5[6], r0
-; CHECK-NEXT:    vmov.u16 r0, q4[7]
-; CHECK-NEXT:    vmov s0, r0
+; CHECK-NEXT:    vmovx.f16 s0, s19
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-NEXT:    vmov.16 q5[6], r0
 ; CHECK-NEXT:    vstr s0, [sp]
 ; CHECK-NEXT:    ldr r0, [sp]
 ; CHECK-NEXT:    bl exp2f
@@ -756,80 +696,68 @@ define arm_aapcs_vfpcc <8 x half> @log_float16_t(<8 x half> %src) {
 ; CHECK-NEXT:    vpush {d8, d9, d10, d11}
 ; CHECK-NEXT:    .pad #32
 ; CHECK-NEXT:    sub sp, #32
-; CHECK-NEXT:    vmov.u16 r0, q0[1]
 ; CHECK-NEXT:    vmov q4, q0
-; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vstr s0, [sp, #24]
-; CHECK-NEXT:    ldr r0, [sp, #24]
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s16
+; CHECK-NEXT:    vstr s0, [sp, #28]
+; CHECK-NEXT:    ldr r0, [sp, #28]
 ; CHECK-NEXT:    bl logf
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vmov.u16 r0, q4[0]
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r4, s0
-; CHECK-NEXT:    vmov s0, r0
+; CHECK-NEXT:    vmovx.f16 s0, s16
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vstr s0, [sp, #28]
-; CHECK-NEXT:    ldr r0, [sp, #28]
+; CHECK-NEXT:    vstr s0, [sp, #24]
+; CHECK-NEXT:    ldr r0, [sp, #24]
 ; CHECK-NEXT:    bl logf
 ; CHECK-NEXT:    vmov s0, r0
+; CHECK-NEXT:    vmov.16 q5[0], r4
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q5[0], r0
-; CHECK-NEXT:    vmov.u16 r0, q4[2]
-; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vmov.16 q5[1], r4
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s17
 ; CHECK-NEXT:    vstr s0, [sp, #20]
+; CHECK-NEXT:    vmov.16 q5[1], r0
 ; CHECK-NEXT:    ldr r0, [sp, #20]
 ; CHECK-NEXT:    bl logf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q5[2], r0
-; CHECK-NEXT:    vmov.u16 r0, q4[3]
-; CHECK-NEXT:    vmov s0, r0
+; CHECK-NEXT:    vmovx.f16 s0, s17
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-NEXT:    vmov.16 q5[2], r0
 ; CHECK-NEXT:    vstr s0, [sp, #16]
 ; CHECK-NEXT:    ldr r0, [sp, #16]
 ; CHECK-NEXT:    bl logf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q5[3], r0
-; CHECK-NEXT:    vmov.u16 r0, q4[4]
-; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s18
 ; CHECK-NEXT:    vstr s0, [sp, #12]
+; CHECK-NEXT:    vmov.16 q5[3], r0
 ; CHECK-NEXT:    ldr r0, [sp, #12]
 ; CHECK-NEXT:    bl logf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q5[4], r0
-; CHECK-NEXT:    vmov.u16 r0, q4[5]
-; CHECK-NEXT:    vmov s0, r0
+; CHECK-NEXT:    vmovx.f16 s0, s18
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-NEXT:    vmov.16 q5[4], r0
 ; CHECK-NEXT:    vstr s0, [sp, #8]
 ; CHECK-NEXT:    ldr r0, [sp, #8]
 ; CHECK-NEXT:    bl logf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q5[5], r0
-; CHECK-NEXT:    vmov.u16 r0, q4[6]
-; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s19
 ; CHECK-NEXT:    vstr s0, [sp, #4]
+; CHECK-NEXT:    vmov.16 q5[5], r0
 ; CHECK-NEXT:    ldr r0, [sp, #4]
 ; CHECK-NEXT:    bl logf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q5[6], r0
-; CHECK-NEXT:    vmov.u16 r0, q4[7]
-; CHECK-NEXT:    vmov s0, r0
+; CHECK-NEXT:    vmovx.f16 s0, s19
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-NEXT:    vmov.16 q5[6], r0
 ; CHECK-NEXT:    vstr s0, [sp]
 ; CHECK-NEXT:    ldr r0, [sp]
 ; CHECK-NEXT:    bl logf
@@ -912,80 +840,68 @@ define arm_aapcs_vfpcc <8 x half> @log2_float16_t(<8 x half> %src) {
 ; CHECK-NEXT:    vpush {d8, d9, d10, d11}
 ; CHECK-NEXT:    .pad #32
 ; CHECK-NEXT:    sub sp, #32
-; CHECK-NEXT:    vmov.u16 r0, q0[1]
 ; CHECK-NEXT:    vmov q4, q0
-; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vstr s0, [sp, #24]
-; CHECK-NEXT:    ldr r0, [sp, #24]
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s16
+; CHECK-NEXT:    vstr s0, [sp, #28]
+; CHECK-NEXT:    ldr r0, [sp, #28]
 ; CHECK-NEXT:    bl log2f
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vmov.u16 r0, q4[0]
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r4, s0
-; CHECK-NEXT:    vmov s0, r0
+; CHECK-NEXT:    vmovx.f16 s0, s16
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vstr s0, [sp, #28]
-; CHECK-NEXT:    ldr r0, [sp, #28]
+; CHECK-NEXT:    vstr s0, [sp, #24]
+; CHECK-NEXT:    ldr r0, [sp, #24]
 ; CHECK-NEXT:    bl log2f
 ; CHECK-NEXT:    vmov s0, r0
+; CHECK-NEXT:    vmov.16 q5[0], r4
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q5[0], r0
-; CHECK-NEXT:    vmov.u16 r0, q4[2]
-; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vmov.16 q5[1], r4
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s17
 ; CHECK-NEXT:    vstr s0, [sp, #20]
+; CHECK-NEXT:    vmov.16 q5[1], r0
 ; CHECK-NEXT:    ldr r0, [sp, #20]
 ; CHECK-NEXT:    bl log2f
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q5[2], r0
-; CHECK-NEXT:    vmov.u16 r0, q4[3]
-; CHECK-NEXT:    vmov s0, r0
+; CHECK-NEXT:    vmovx.f16 s0, s17
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-NEXT:    vmov.16 q5[2], r0
 ; CHECK-NEXT:    vstr s0, [sp, #16]
 ; CHECK-NEXT:    ldr r0, [sp, #16]
 ; CHECK-NEXT:    bl log2f
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q5[3], r0
-; CHECK-NEXT:    vmov.u16 r0, q4[4]
-; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s18
 ; CHECK-NEXT:    vstr s0, [sp, #12]
+; CHECK-NEXT:    vmov.16 q5[3], r0
 ; CHECK-NEXT:    ldr r0, [sp, #12]
 ; CHECK-NEXT:    bl log2f
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q5[4], r0
-; CHECK-NEXT:    vmov.u16 r0, q4[5]
-; CHECK-NEXT:    vmov s0, r0
+; CHECK-NEXT:    vmovx.f16 s0, s18
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-NEXT:    vmov.16 q5[4], r0
 ; CHECK-NEXT:    vstr s0, [sp, #8]
 ; CHECK-NEXT:    ldr r0, [sp, #8]
 ; CHECK-NEXT:    bl log2f
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q5[5], r0
-; CHECK-NEXT:    vmov.u16 r0, q4[6]
-; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s19
 ; CHECK-NEXT:    vstr s0, [sp, #4]
+; CHECK-NEXT:    vmov.16 q5[5], r0
 ; CHECK-NEXT:    ldr r0, [sp, #4]
 ; CHECK-NEXT:    bl log2f
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q5[6], r0
-; CHECK-NEXT:    vmov.u16 r0, q4[7]
-; CHECK-NEXT:    vmov s0, r0
+; CHECK-NEXT:    vmovx.f16 s0, s19
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-NEXT:    vmov.16 q5[6], r0
 ; CHECK-NEXT:    vstr s0, [sp]
 ; CHECK-NEXT:    ldr r0, [sp]
 ; CHECK-NEXT:    bl log2f
@@ -1068,80 +984,68 @@ define arm_aapcs_vfpcc <8 x half> @log10_float16_t(<8 x half> %src) {
 ; CHECK-NEXT:    vpush {d8, d9, d10, d11}
 ; CHECK-NEXT:    .pad #32
 ; CHECK-NEXT:    sub sp, #32
-; CHECK-NEXT:    vmov.u16 r0, q0[1]
 ; CHECK-NEXT:    vmov q4, q0
-; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vstr s0, [sp, #24]
-; CHECK-NEXT:    ldr r0, [sp, #24]
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s16
+; CHECK-NEXT:    vstr s0, [sp, #28]
+; CHECK-NEXT:    ldr r0, [sp, #28]
 ; CHECK-NEXT:    bl log10f
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vmov.u16 r0, q4[0]
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r4, s0
-; CHECK-NEXT:    vmov s0, r0
+; CHECK-NEXT:    vmovx.f16 s0, s16
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vstr s0, [sp, #28]
-; CHECK-NEXT:    ldr r0, [sp, #28]
+; CHECK-NEXT:    vstr s0, [sp, #24]
+; CHECK-NEXT:    ldr r0, [sp, #24]
 ; CHECK-NEXT:    bl log10f
 ; CHECK-NEXT:    vmov s0, r0
+; CHECK-NEXT:    vmov.16 q5[0], r4
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q5[0], r0
-; CHECK-NEXT:    vmov.u16 r0, q4[2]
-; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vmov.16 q5[1], r4
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s17
 ; CHECK-NEXT:    vstr s0, [sp, #20]
+; CHECK-NEXT:    vmov.16 q5[1], r0
 ; CHECK-NEXT:    ldr r0, [sp, #20]
 ; CHECK-NEXT:    bl log10f
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q5[2], r0
-; CHECK-NEXT:    vmov.u16 r0, q4[3]
-; CHECK-NEXT:    vmov s0, r0
+; CHECK-NEXT:    vmovx.f16 s0, s17
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-NEXT:    vmov.16 q5[2], r0
 ; CHECK-NEXT:    vstr s0, [sp, #16]
 ; CHECK-NEXT:    ldr r0, [sp, #16]
 ; CHECK-NEXT:    bl log10f
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q5[3], r0
-; CHECK-NEXT:    vmov.u16 r0, q4[4]
-; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s18
 ; CHECK-NEXT:    vstr s0, [sp, #12]
+; CHECK-NEXT:    vmov.16 q5[3], r0
 ; CHECK-NEXT:    ldr r0, [sp, #12]
 ; CHECK-NEXT:    bl log10f
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q5[4], r0
-; CHECK-NEXT:    vmov.u16 r0, q4[5]
-; CHECK-NEXT:    vmov s0, r0
+; CHECK-NEXT:    vmovx.f16 s0, s18
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-NEXT:    vmov.16 q5[4], r0
 ; CHECK-NEXT:    vstr s0, [sp, #8]
 ; CHECK-NEXT:    ldr r0, [sp, #8]
 ; CHECK-NEXT:    bl log10f
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q5[5], r0
-; CHECK-NEXT:    vmov.u16 r0, q4[6]
-; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s19
 ; CHECK-NEXT:    vstr s0, [sp, #4]
+; CHECK-NEXT:    vmov.16 q5[5], r0
 ; CHECK-NEXT:    ldr r0, [sp, #4]
 ; CHECK-NEXT:    bl log10f
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q5[6], r0
-; CHECK-NEXT:    vmov.u16 r0, q4[7]
-; CHECK-NEXT:    vmov s0, r0
+; CHECK-NEXT:    vmovx.f16 s0, s19
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-NEXT:    vmov.16 q5[6], r0
 ; CHECK-NEXT:    vstr s0, [sp]
 ; CHECK-NEXT:    ldr r0, [sp]
 ; CHECK-NEXT:    bl log10f
@@ -1235,25 +1139,19 @@ define arm_aapcs_vfpcc <8 x half> @pow_float16_t(<8 x half> %src1, <8 x half> %s
 ; CHECK-NEXT:    .pad #64
 ; CHECK-NEXT:    sub sp, #64
 ; CHECK-NEXT:    vmov q4, q0
-; CHECK-NEXT:    vmov.u16 r0, q1[0]
-; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vmov.u16 r0, q4[0]
-; CHECK-NEXT:    vmov s2, r0
 ; CHECK-NEXT:    vmov q5, q1
-; CHECK-NEXT:    vcvtb.f32.f16 s2, s2
-; CHECK-NEXT:    vstr s2, [sp, #56]
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s16
+; CHECK-NEXT:    vstr s0, [sp, #56]
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s20
 ; CHECK-NEXT:    vstr s0, [sp, #60]
 ; CHECK-NEXT:    ldrd r0, r1, [sp, #56]
 ; CHECK-NEXT:    bl powf
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vmov.u16 r0, q5[1]
+; CHECK-NEXT:    vmovx.f16 s2, s16
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vmov r4, s0
-; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vmov.u16 r0, q4[1]
-; CHECK-NEXT:    vmov s2, r0
 ; CHECK-NEXT:    vcvtb.f32.f16 s2, s2
+; CHECK-NEXT:    vmov r4, s0
+; CHECK-NEXT:    vmovx.f16 s0, s20
 ; CHECK-NEXT:    vstr s2, [sp, #48]
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
 ; CHECK-NEXT:    vstr s0, [sp, #52]
@@ -1263,85 +1161,67 @@ define arm_aapcs_vfpcc <8 x half> @pow_float16_t(<8 x half> %src1, <8 x half> %s
 ; CHECK-NEXT:    vmov.16 q6[0], r4
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q6[1], r0
-; CHECK-NEXT:    vmov.u16 r0, q5[2]
-; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vmov.u16 r0, q4[2]
-; CHECK-NEXT:    vmov s2, r0
-; CHECK-NEXT:    vcvtb.f32.f16 s2, s2
-; CHECK-NEXT:    vstr s2, [sp, #40]
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s17
+; CHECK-NEXT:    vstr s0, [sp, #40]
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s21
 ; CHECK-NEXT:    vstr s0, [sp, #44]
+; CHECK-NEXT:    vmov.16 q6[1], r0
 ; CHECK-NEXT:    ldrd r0, r1, [sp, #40]
 ; CHECK-NEXT:    bl powf
 ; CHECK-NEXT:    vmov s0, r0
+; CHECK-NEXT:    vmovx.f16 s2, s17
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q6[2], r0
-; CHECK-NEXT:    vmov.u16 r0, q5[3]
-; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vmov.u16 r0, q4[3]
-; CHECK-NEXT:    vmov s2, r0
 ; CHECK-NEXT:    vcvtb.f32.f16 s2, s2
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vmovx.f16 s0, s21
 ; CHECK-NEXT:    vstr s2, [sp, #32]
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
 ; CHECK-NEXT:    vstr s0, [sp, #36]
+; CHECK-NEXT:    vmov.16 q6[2], r0
 ; CHECK-NEXT:    ldrd r0, r1, [sp, #32]
 ; CHECK-NEXT:    bl powf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q6[3], r0
-; CHECK-NEXT:    vmov.u16 r0, q5[4]
-; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vmov.u16 r0, q4[4]
-; CHECK-NEXT:    vmov s2, r0
-; CHECK-NEXT:    vcvtb.f32.f16 s2, s2
-; CHECK-NEXT:    vstr s2, [sp, #24]
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s18
+; CHECK-NEXT:    vstr s0, [sp, #24]
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s22
 ; CHECK-NEXT:    vstr s0, [sp, #28]
+; CHECK-NEXT:    vmov.16 q6[3], r0
 ; CHECK-NEXT:    ldrd r0, r1, [sp, #24]
 ; CHECK-NEXT:    bl powf
 ; CHECK-NEXT:    vmov s0, r0
+; CHECK-NEXT:    vmovx.f16 s2, s18
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q6[4], r0
-; CHECK-NEXT:    vmov.u16 r0, q5[5]
-; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vmov.u16 r0, q4[5]
-; CHECK-NEXT:    vmov s2, r0
 ; CHECK-NEXT:    vcvtb.f32.f16 s2, s2
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vmovx.f16 s0, s22
 ; CHECK-NEXT:    vstr s2, [sp, #16]
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
 ; CHECK-NEXT:    vstr s0, [sp, #20]
+; CHECK-NEXT:    vmov.16 q6[4], r0
 ; CHECK-NEXT:    ldrd r0, r1, [sp, #16]
 ; CHECK-NEXT:    bl powf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q6[5], r0
-; CHECK-NEXT:    vmov.u16 r0, q5[6]
-; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vmov.u16 r0, q4[6]
-; CHECK-NEXT:    vmov s2, r0
-; CHECK-NEXT:    vcvtb.f32.f16 s2, s2
-; CHECK-NEXT:    vstr s2, [sp, #8]
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s19
+; CHECK-NEXT:    vstr s0, [sp, #8]
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s23
 ; CHECK-NEXT:    vstr s0, [sp, #12]
+; CHECK-NEXT:    vmov.16 q6[5], r0
 ; CHECK-NEXT:    ldrd r0, r1, [sp, #8]
 ; CHECK-NEXT:    bl powf
 ; CHECK-NEXT:    vmov s0, r0
+; CHECK-NEXT:    vmovx.f16 s2, s19
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q6[6], r0
-; CHECK-NEXT:    vmov.u16 r0, q5[7]
-; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vmov.u16 r0, q4[7]
-; CHECK-NEXT:    vmov s2, r0
 ; CHECK-NEXT:    vcvtb.f32.f16 s2, s2
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vmovx.f16 s0, s23
 ; CHECK-NEXT:    vstr s2, [sp]
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
 ; CHECK-NEXT:    vstr s0, [sp, #4]
+; CHECK-NEXT:    vmov.16 q6[6], r0
 ; CHECK-NEXT:    ldrd r0, r1, [sp]
 ; CHECK-NEXT:    bl powf
 ; CHECK-NEXT:    vmov s0, r0
@@ -1431,123 +1311,91 @@ define arm_aapcs_vfpcc <8 x half> @copysign_float16_t(<8 x half> %src1, <8 x hal
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .pad #32
 ; CHECK-NEXT:    sub sp, #32
-; CHECK-NEXT:    vmov.u16 r0, q1[1]
-; CHECK-NEXT:    vmov.u16 r1, q0[0]
-; CHECK-NEXT:    vmov s8, r0
-; CHECK-NEXT:    vmov.u16 r0, q1[0]
+; CHECK-NEXT:    vmovx.f16 s8, s4
+; CHECK-NEXT:    vstr.16 s4, [sp, #28]
 ; CHECK-NEXT:    vstr.16 s8, [sp, #24]
-; CHECK-NEXT:    vmov s8, r0
-; CHECK-NEXT:    vmov.u16 r0, q1[2]
-; CHECK-NEXT:    vstr.16 s8, [sp, #28]
-; CHECK-NEXT:    vmov s8, r0
-; CHECK-NEXT:    vmov.u16 r0, q1[3]
-; CHECK-NEXT:    vstr.16 s8, [sp, #20]
-; CHECK-NEXT:    vmov s8, r0
-; CHECK-NEXT:    vmov.u16 r0, q1[4]
+; CHECK-NEXT:    vmovx.f16 s8, s5
+; CHECK-NEXT:    vstr.16 s5, [sp, #20]
 ; CHECK-NEXT:    vstr.16 s8, [sp, #16]
-; CHECK-NEXT:    vmov s8, r0
-; CHECK-NEXT:    vmov.u16 r0, q1[5]
-; CHECK-NEXT:    vstr.16 s8, [sp, #12]
-; CHECK-NEXT:    vmov s8, r0
-; CHECK-NEXT:    vmov.u16 r0, q1[6]
+; CHECK-NEXT:    vmovx.f16 s8, s6
+; CHECK-NEXT:    vmovx.f16 s4, s7
+; CHECK-NEXT:    vstr.16 s6, [sp, #12]
 ; CHECK-NEXT:    vstr.16 s8, [sp, #8]
-; CHECK-NEXT:    vmov s8, r0
-; CHECK-NEXT:    vmov.u16 r0, q1[7]
-; CHECK-NEXT:    vmov s4, r0
-; CHECK-NEXT:    vstr.16 s8, [sp, #4]
+; CHECK-NEXT:    vstr.16 s7, [sp, #4]
 ; CHECK-NEXT:    vstr.16 s4, [sp]
-; CHECK-NEXT:    vmov.u16 r0, q0[1]
-; CHECK-NEXT:    vmov s4, r0
-; CHECK-NEXT:    ldrb.w r0, [sp, #25]
-; CHECK-NEXT:    vabs.f16 s4, s4
-; CHECK-NEXT:    ands r0, r0, #128
+; CHECK-NEXT:    ldrb.w r0, [sp, #29]
+; CHECK-NEXT:    vabs.f16 s4, s0
 ; CHECK-NEXT:    vneg.f16 s6, s4
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r0, #1
-; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    ldrb.w r1, [sp, #25]
+; CHECK-NEXT:    tst.w r0, #128
+; CHECK-NEXT:    vabs.f16 s8, s1
+; CHECK-NEXT:    cset r0, ne
+; CHECK-NEXT:    vneg.f16 s10, s8
+; CHECK-NEXT:    lsls r0, r0, #31
 ; CHECK-NEXT:    vseleq.f16 s4, s4, s6
+; CHECK-NEXT:    tst.w r1, #128
 ; CHECK-NEXT:    vmov r0, s4
-; CHECK-NEXT:    vmov s4, r1
-; CHECK-NEXT:    ldrb.w r1, [sp, #29]
+; CHECK-NEXT:    vmovx.f16 s4, s0
+; CHECK-NEXT:    cset r1, ne
 ; CHECK-NEXT:    vabs.f16 s4, s4
-; CHECK-NEXT:    ands r1, r1, #128
 ; CHECK-NEXT:    vneg.f16 s6, s4
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r1, #1
-; CHECK-NEXT:    cmp r1, #0
+; CHECK-NEXT:    lsls r1, r1, #31
 ; CHECK-NEXT:    vseleq.f16 s4, s4, s6
+; CHECK-NEXT:    vmovx.f16 s0, s3
 ; CHECK-NEXT:    vmov r1, s4
-; CHECK-NEXT:    vmov.16 q1[0], r1
-; CHECK-NEXT:    vmov.16 q1[1], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[2]
-; CHECK-NEXT:    vmov s8, r0
+; CHECK-NEXT:    vmov.16 q1[0], r0
 ; CHECK-NEXT:    ldrb.w r0, [sp, #21]
-; CHECK-NEXT:    vabs.f16 s8, s8
-; CHECK-NEXT:    ands r0, r0, #128
-; CHECK-NEXT:    vneg.f16 s10, s8
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r0, #1
-; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    vmov.16 q1[1], r1
+; CHECK-NEXT:    vabs.f16 s0, s0
+; CHECK-NEXT:    tst.w r0, #128
+; CHECK-NEXT:    cset r0, ne
+; CHECK-NEXT:    lsls r0, r0, #31
 ; CHECK-NEXT:    vseleq.f16 s8, s8, s10
 ; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    vmovx.f16 s8, s1
 ; CHECK-NEXT:    vmov.16 q1[2], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[3]
-; CHECK-NEXT:    vmov s8, r0
 ; CHECK-NEXT:    ldrb.w r0, [sp, #17]
 ; CHECK-NEXT:    vabs.f16 s8, s8
-; CHECK-NEXT:    ands r0, r0, #128
+; CHECK-NEXT:    tst.w r0, #128
 ; CHECK-NEXT:    vneg.f16 s10, s8
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r0, #1
-; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    cset r0, ne
+; CHECK-NEXT:    lsls r0, r0, #31
 ; CHECK-NEXT:    vseleq.f16 s8, s8, s10
 ; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    vabs.f16 s8, s2
 ; CHECK-NEXT:    vmov.16 q1[3], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[4]
-; CHECK-NEXT:    vmov s8, r0
 ; CHECK-NEXT:    ldrb.w r0, [sp, #13]
-; CHECK-NEXT:    vabs.f16 s8, s8
-; CHECK-NEXT:    ands r0, r0, #128
 ; CHECK-NEXT:    vneg.f16 s10, s8
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r0, #1
-; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    tst.w r0, #128
+; CHECK-NEXT:    cset r0, ne
+; CHECK-NEXT:    lsls r0, r0, #31
 ; CHECK-NEXT:    vseleq.f16 s8, s8, s10
 ; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    vmovx.f16 s8, s2
 ; CHECK-NEXT:    vmov.16 q1[4], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[5]
-; CHECK-NEXT:    vmov s8, r0
 ; CHECK-NEXT:    ldrb.w r0, [sp, #9]
 ; CHECK-NEXT:    vabs.f16 s8, s8
-; CHECK-NEXT:    ands r0, r0, #128
+; CHECK-NEXT:    vneg.f16 s2, s0
+; CHECK-NEXT:    tst.w r0, #128
 ; CHECK-NEXT:    vneg.f16 s10, s8
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r0, #1
-; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    cset r0, ne
+; CHECK-NEXT:    lsls r0, r0, #31
 ; CHECK-NEXT:    vseleq.f16 s8, s8, s10
 ; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    vabs.f16 s8, s3
 ; CHECK-NEXT:    vmov.16 q1[5], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[6]
-; CHECK-NEXT:    vmov s8, r0
 ; CHECK-NEXT:    ldrb.w r0, [sp, #5]
-; CHECK-NEXT:    vabs.f16 s8, s8
-; CHECK-NEXT:    ands r0, r0, #128
 ; CHECK-NEXT:    vneg.f16 s10, s8
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r0, #1
-; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    tst.w r0, #128
+; CHECK-NEXT:    cset r0, ne
+; CHECK-NEXT:    lsls r0, r0, #31
 ; CHECK-NEXT:    vseleq.f16 s8, s8, s10
 ; CHECK-NEXT:    vmov r0, s8
 ; CHECK-NEXT:    vmov.16 q1[6], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[7]
-; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    ldrb.w r0, [sp, #1]
-; CHECK-NEXT:    vabs.f16 s0, s0
-; CHECK-NEXT:    ands r0, r0, #128
-; CHECK-NEXT:    vneg.f16 s2, s0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r0, #1
-; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    tst.w r0, #128
+; CHECK-NEXT:    cset r0, ne
+; CHECK-NEXT:    lsls r0, r0, #31
 ; CHECK-NEXT:    vseleq.f16 s0, s0, s2
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vmov.16 q1[7], r0
diff --git a/llvm/test/CodeGen/Thumb2/mve-fp-negabs.ll b/llvm/test/CodeGen/Thumb2/mve-fp-negabs.ll
index d1fd4b267..9b295d311 100644
--- a/llvm/test/CodeGen/Thumb2/mve-fp-negabs.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-fp-negabs.ll
@@ -5,44 +5,32 @@
 define arm_aapcs_vfpcc <8 x half> @fneg_float16_t(<8 x half> %src) {
 ; CHECK-MVE-LABEL: fneg_float16_t:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[0]
-; CHECK-MVE-NEXT:    vmov.u16 r1, q0[1]
-; CHECK-MVE-NEXT:    vmov s4, r0
+; CHECK-MVE-NEXT:    vmovx.f16 s4, s0
+; CHECK-MVE-NEXT:    vneg.f16 s8, s1
 ; CHECK-MVE-NEXT:    vneg.f16 s4, s4
 ; CHECK-MVE-NEXT:    vmov r0, s4
-; CHECK-MVE-NEXT:    vmov s4, r1
-; CHECK-MVE-NEXT:    vneg.f16 s4, s4
+; CHECK-MVE-NEXT:    vneg.f16 s4, s0
 ; CHECK-MVE-NEXT:    vmov r1, s4
-; CHECK-MVE-NEXT:    vmov.16 q1[0], r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[2]
-; CHECK-MVE-NEXT:    vmov.16 q1[1], r1
-; CHECK-MVE-NEXT:    vmov s8, r0
-; CHECK-MVE-NEXT:    vneg.f16 s8, s8
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q1[0], r1
+; CHECK-MVE-NEXT:    vneg.f16 s0, s0
+; CHECK-MVE-NEXT:    vmov.16 q1[1], r0
 ; CHECK-MVE-NEXT:    vmov r0, s8
+; CHECK-MVE-NEXT:    vmovx.f16 s8, s1
 ; CHECK-MVE-NEXT:    vmov.16 q1[2], r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[3]
-; CHECK-MVE-NEXT:    vmov s8, r0
 ; CHECK-MVE-NEXT:    vneg.f16 s8, s8
 ; CHECK-MVE-NEXT:    vmov r0, s8
+; CHECK-MVE-NEXT:    vneg.f16 s8, s2
 ; CHECK-MVE-NEXT:    vmov.16 q1[3], r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[4]
-; CHECK-MVE-NEXT:    vmov s8, r0
-; CHECK-MVE-NEXT:    vneg.f16 s8, s8
 ; CHECK-MVE-NEXT:    vmov r0, s8
+; CHECK-MVE-NEXT:    vmovx.f16 s8, s2
 ; CHECK-MVE-NEXT:    vmov.16 q1[4], r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[5]
-; CHECK-MVE-NEXT:    vmov s8, r0
 ; CHECK-MVE-NEXT:    vneg.f16 s8, s8
 ; CHECK-MVE-NEXT:    vmov r0, s8
+; CHECK-MVE-NEXT:    vneg.f16 s8, s3
 ; CHECK-MVE-NEXT:    vmov.16 q1[5], r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[6]
-; CHECK-MVE-NEXT:    vmov s8, r0
-; CHECK-MVE-NEXT:    vneg.f16 s8, s8
 ; CHECK-MVE-NEXT:    vmov r0, s8
 ; CHECK-MVE-NEXT:    vmov.16 q1[6], r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[7]
-; CHECK-MVE-NEXT:    vmov s0, r0
-; CHECK-MVE-NEXT:    vneg.f16 s0, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q1[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q1
@@ -112,44 +100,32 @@ entry:
 define arm_aapcs_vfpcc <8 x half> @fabs_float16_t(<8 x half> %src) {
 ; CHECK-MVE-LABEL: fabs_float16_t:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[0]
-; CHECK-MVE-NEXT:    vmov.u16 r1, q0[1]
-; CHECK-MVE-NEXT:    vmov s4, r0
+; CHECK-MVE-NEXT:    vmovx.f16 s4, s0
+; CHECK-MVE-NEXT:    vabs.f16 s8, s1
 ; CHECK-MVE-NEXT:    vabs.f16 s4, s4
 ; CHECK-MVE-NEXT:    vmov r0, s4
-; CHECK-MVE-NEXT:    vmov s4, r1
-; CHECK-MVE-NEXT:    vabs.f16 s4, s4
+; CHECK-MVE-NEXT:    vabs.f16 s4, s0
 ; CHECK-MVE-NEXT:    vmov r1, s4
-; CHECK-MVE-NEXT:    vmov.16 q1[0], r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[2]
-; CHECK-MVE-NEXT:    vmov.16 q1[1], r1
-; CHECK-MVE-NEXT:    vmov s8, r0
-; CHECK-MVE-NEXT:    vabs.f16 s8, s8
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q1[0], r1
+; CHECK-MVE-NEXT:    vabs.f16 s0, s0
+; CHECK-MVE-NEXT:    vmov.16 q1[1], r0
 ; CHECK-MVE-NEXT:    vmov r0, s8
+; CHECK-MVE-NEXT:    vmovx.f16 s8, s1
 ; CHECK-MVE-NEXT:    vmov.16 q1[2], r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[3]
-; CHECK-MVE-NEXT:    vmov s8, r0
 ; CHECK-MVE-NEXT:    vabs.f16 s8, s8
 ; CHECK-MVE-NEXT:    vmov r0, s8
+; CHECK-MVE-NEXT:    vabs.f16 s8, s2
 ; CHECK-MVE-NEXT:    vmov.16 q1[3], r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[4]
-; CHECK-MVE-NEXT:    vmov s8, r0
-; CHECK-MVE-NEXT:    vabs.f16 s8, s8
 ; CHECK-MVE-NEXT:    vmov r0, s8
+; CHECK-MVE-NEXT:    vmovx.f16 s8, s2
 ; CHECK-MVE-NEXT:    vmov.16 q1[4], r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[5]
-; CHECK-MVE-NEXT:    vmov s8, r0
 ; CHECK-MVE-NEXT:    vabs.f16 s8, s8
 ; CHECK-MVE-NEXT:    vmov r0, s8
+; CHECK-MVE-NEXT:    vabs.f16 s8, s3
 ; CHECK-MVE-NEXT:    vmov.16 q1[5], r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[6]
-; CHECK-MVE-NEXT:    vmov s8, r0
-; CHECK-MVE-NEXT:    vabs.f16 s8, s8
 ; CHECK-MVE-NEXT:    vmov r0, s8
 ; CHECK-MVE-NEXT:    vmov.16 q1[6], r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[7]
-; CHECK-MVE-NEXT:    vmov s0, r0
-; CHECK-MVE-NEXT:    vabs.f16 s0, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q1[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q1
diff --git a/llvm/test/CodeGen/Thumb2/mve-frint.ll b/llvm/test/CodeGen/Thumb2/mve-frint.ll
index e1758d5ed..6d936a375 100644
--- a/llvm/test/CodeGen/Thumb2/mve-frint.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-frint.ll
@@ -24,44 +24,32 @@ entry:
 define arm_aapcs_vfpcc <8 x half> @fceil_float16_t(<8 x half> %src) {
 ; CHECK-MVE-LABEL: fceil_float16_t:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[0]
-; CHECK-MVE-NEXT:    vmov.u16 r1, q0[1]
-; CHECK-MVE-NEXT:    vmov s4, r0
+; CHECK-MVE-NEXT:    vmovx.f16 s4, s0
+; CHECK-MVE-NEXT:    vrintp.f16 s8, s1
 ; CHECK-MVE-NEXT:    vrintp.f16 s4, s4
 ; CHECK-MVE-NEXT:    vmov r0, s4
-; CHECK-MVE-NEXT:    vmov s4, r1
-; CHECK-MVE-NEXT:    vrintp.f16 s4, s4
+; CHECK-MVE-NEXT:    vrintp.f16 s4, s0
 ; CHECK-MVE-NEXT:    vmov r1, s4
-; CHECK-MVE-NEXT:    vmov.16 q1[0], r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[2]
-; CHECK-MVE-NEXT:    vmov.16 q1[1], r1
-; CHECK-MVE-NEXT:    vmov s8, r0
-; CHECK-MVE-NEXT:    vrintp.f16 s8, s8
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q1[0], r1
+; CHECK-MVE-NEXT:    vrintp.f16 s0, s0
+; CHECK-MVE-NEXT:    vmov.16 q1[1], r0
 ; CHECK-MVE-NEXT:    vmov r0, s8
+; CHECK-MVE-NEXT:    vmovx.f16 s8, s1
 ; CHECK-MVE-NEXT:    vmov.16 q1[2], r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[3]
-; CHECK-MVE-NEXT:    vmov s8, r0
 ; CHECK-MVE-NEXT:    vrintp.f16 s8, s8
 ; CHECK-MVE-NEXT:    vmov r0, s8
+; CHECK-MVE-NEXT:    vrintp.f16 s8, s2
 ; CHECK-MVE-NEXT:    vmov.16 q1[3], r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[4]
-; CHECK-MVE-NEXT:    vmov s8, r0
-; CHECK-MVE-NEXT:    vrintp.f16 s8, s8
 ; CHECK-MVE-NEXT:    vmov r0, s8
+; CHECK-MVE-NEXT:    vmovx.f16 s8, s2
 ; CHECK-MVE-NEXT:    vmov.16 q1[4], r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[5]
-; CHECK-MVE-NEXT:    vmov s8, r0
 ; CHECK-MVE-NEXT:    vrintp.f16 s8, s8
 ; CHECK-MVE-NEXT:    vmov r0, s8
+; CHECK-MVE-NEXT:    vrintp.f16 s8, s3
 ; CHECK-MVE-NEXT:    vmov.16 q1[5], r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[6]
-; CHECK-MVE-NEXT:    vmov s8, r0
-; CHECK-MVE-NEXT:    vrintp.f16 s8, s8
 ; CHECK-MVE-NEXT:    vmov r0, s8
 ; CHECK-MVE-NEXT:    vmov.16 q1[6], r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[7]
-; CHECK-MVE-NEXT:    vmov s0, r0
-; CHECK-MVE-NEXT:    vrintp.f16 s0, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q1[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q1
@@ -122,44 +110,32 @@ entry:
 define arm_aapcs_vfpcc <8 x half> @ftrunc_float16_t(<8 x half> %src) {
 ; CHECK-MVE-LABEL: ftrunc_float16_t:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[0]
-; CHECK-MVE-NEXT:    vmov.u16 r1, q0[1]
-; CHECK-MVE-NEXT:    vmov s4, r0
+; CHECK-MVE-NEXT:    vmovx.f16 s4, s0
+; CHECK-MVE-NEXT:    vrintz.f16 s8, s1
 ; CHECK-MVE-NEXT:    vrintz.f16 s4, s4
 ; CHECK-MVE-NEXT:    vmov r0, s4
-; CHECK-MVE-NEXT:    vmov s4, r1
-; CHECK-MVE-NEXT:    vrintz.f16 s4, s4
+; CHECK-MVE-NEXT:    vrintz.f16 s4, s0
 ; CHECK-MVE-NEXT:    vmov r1, s4
-; CHECK-MVE-NEXT:    vmov.16 q1[0], r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[2]
-; CHECK-MVE-NEXT:    vmov.16 q1[1], r1
-; CHECK-MVE-NEXT:    vmov s8, r0
-; CHECK-MVE-NEXT:    vrintz.f16 s8, s8
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q1[0], r1
+; CHECK-MVE-NEXT:    vrintz.f16 s0, s0
+; CHECK-MVE-NEXT:    vmov.16 q1[1], r0
 ; CHECK-MVE-NEXT:    vmov r0, s8
+; CHECK-MVE-NEXT:    vmovx.f16 s8, s1
 ; CHECK-MVE-NEXT:    vmov.16 q1[2], r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[3]
-; CHECK-MVE-NEXT:    vmov s8, r0
 ; CHECK-MVE-NEXT:    vrintz.f16 s8, s8
 ; CHECK-MVE-NEXT:    vmov r0, s8
+; CHECK-MVE-NEXT:    vrintz.f16 s8, s2
 ; CHECK-MVE-NEXT:    vmov.16 q1[3], r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[4]
-; CHECK-MVE-NEXT:    vmov s8, r0
-; CHECK-MVE-NEXT:    vrintz.f16 s8, s8
 ; CHECK-MVE-NEXT:    vmov r0, s8
+; CHECK-MVE-NEXT:    vmovx.f16 s8, s2
 ; CHECK-MVE-NEXT:    vmov.16 q1[4], r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[5]
-; CHECK-MVE-NEXT:    vmov s8, r0
 ; CHECK-MVE-NEXT:    vrintz.f16 s8, s8
 ; CHECK-MVE-NEXT:    vmov r0, s8
+; CHECK-MVE-NEXT:    vrintz.f16 s8, s3
 ; CHECK-MVE-NEXT:    vmov.16 q1[5], r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[6]
-; CHECK-MVE-NEXT:    vmov s8, r0
-; CHECK-MVE-NEXT:    vrintz.f16 s8, s8
 ; CHECK-MVE-NEXT:    vmov r0, s8
 ; CHECK-MVE-NEXT:    vmov.16 q1[6], r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[7]
-; CHECK-MVE-NEXT:    vmov s0, r0
-; CHECK-MVE-NEXT:    vrintz.f16 s0, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q1[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q1
@@ -220,44 +196,32 @@ entry:
 define arm_aapcs_vfpcc <8 x half> @frint_float16_t(<8 x half> %src) {
 ; CHECK-MVE-LABEL: frint_float16_t:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[0]
-; CHECK-MVE-NEXT:    vmov.u16 r1, q0[1]
-; CHECK-MVE-NEXT:    vmov s4, r0
+; CHECK-MVE-NEXT:    vmovx.f16 s4, s0
+; CHECK-MVE-NEXT:    vrintx.f16 s8, s1
 ; CHECK-MVE-NEXT:    vrintx.f16 s4, s4
 ; CHECK-MVE-NEXT:    vmov r0, s4
-; CHECK-MVE-NEXT:    vmov s4, r1
-; CHECK-MVE-NEXT:    vrintx.f16 s4, s4
+; CHECK-MVE-NEXT:    vrintx.f16 s4, s0
 ; CHECK-MVE-NEXT:    vmov r1, s4
-; CHECK-MVE-NEXT:    vmov.16 q1[0], r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[2]
-; CHECK-MVE-NEXT:    vmov.16 q1[1], r1
-; CHECK-MVE-NEXT:    vmov s8, r0
-; CHECK-MVE-NEXT:    vrintx.f16 s8, s8
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q1[0], r1
+; CHECK-MVE-NEXT:    vrintx.f16 s0, s0
+; CHECK-MVE-NEXT:    vmov.16 q1[1], r0
 ; CHECK-MVE-NEXT:    vmov r0, s8
+; CHECK-MVE-NEXT:    vmovx.f16 s8, s1
 ; CHECK-MVE-NEXT:    vmov.16 q1[2], r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[3]
-; CHECK-MVE-NEXT:    vmov s8, r0
 ; CHECK-MVE-NEXT:    vrintx.f16 s8, s8
 ; CHECK-MVE-NEXT:    vmov r0, s8
+; CHECK-MVE-NEXT:    vrintx.f16 s8, s2
 ; CHECK-MVE-NEXT:    vmov.16 q1[3], r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[4]
-; CHECK-MVE-NEXT:    vmov s8, r0
-; CHECK-MVE-NEXT:    vrintx.f16 s8, s8
 ; CHECK-MVE-NEXT:    vmov r0, s8
+; CHECK-MVE-NEXT:    vmovx.f16 s8, s2
 ; CHECK-MVE-NEXT:    vmov.16 q1[4], r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[5]
-; CHECK-MVE-NEXT:    vmov s8, r0
 ; CHECK-MVE-NEXT:    vrintx.f16 s8, s8
 ; CHECK-MVE-NEXT:    vmov r0, s8
+; CHECK-MVE-NEXT:    vrintx.f16 s8, s3
 ; CHECK-MVE-NEXT:    vmov.16 q1[5], r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[6]
-; CHECK-MVE-NEXT:    vmov s8, r0
-; CHECK-MVE-NEXT:    vrintx.f16 s8, s8
 ; CHECK-MVE-NEXT:    vmov r0, s8
 ; CHECK-MVE-NEXT:    vmov.16 q1[6], r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[7]
-; CHECK-MVE-NEXT:    vmov s0, r0
-; CHECK-MVE-NEXT:    vrintx.f16 s0, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q1[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q1
@@ -313,44 +277,32 @@ entry:
 define arm_aapcs_vfpcc <8 x half> @fnearbyint_float16_t(<8 x half> %src) {
 ; CHECK-LABEL: fnearbyint_float16_t:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov.u16 r0, q0[0]
-; CHECK-NEXT:    vmov.u16 r1, q0[1]
-; CHECK-NEXT:    vmov s4, r0
+; CHECK-NEXT:    vmovx.f16 s4, s0
+; CHECK-NEXT:    vrintr.f16 s8, s1
 ; CHECK-NEXT:    vrintr.f16 s4, s4
 ; CHECK-NEXT:    vmov r0, s4
-; CHECK-NEXT:    vmov s4, r1
-; CHECK-NEXT:    vrintr.f16 s4, s4
+; CHECK-NEXT:    vrintr.f16 s4, s0
 ; CHECK-NEXT:    vmov r1, s4
-; CHECK-NEXT:    vmov.16 q1[0], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[2]
-; CHECK-NEXT:    vmov.16 q1[1], r1
-; CHECK-NEXT:    vmov s8, r0
-; CHECK-NEXT:    vrintr.f16 s8, s8
+; CHECK-NEXT:    vmovx.f16 s0, s3
+; CHECK-NEXT:    vmov.16 q1[0], r1
+; CHECK-NEXT:    vrintr.f16 s0, s0
+; CHECK-NEXT:    vmov.16 q1[1], r0
 ; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    vmovx.f16 s8, s1
 ; CHECK-NEXT:    vmov.16 q1[2], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[3]
-; CHECK-NEXT:    vmov s8, r0
 ; CHECK-NEXT:    vrintr.f16 s8, s8
 ; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    vrintr.f16 s8, s2
 ; CHECK-NEXT:    vmov.16 q1[3], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[4]
-; CHECK-NEXT:    vmov s8, r0
-; CHECK-NEXT:    vrintr.f16 s8, s8
 ; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    vmovx.f16 s8, s2
 ; CHECK-NEXT:    vmov.16 q1[4], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[5]
-; CHECK-NEXT:    vmov s8, r0
 ; CHECK-NEXT:    vrintr.f16 s8, s8
 ; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    vrintr.f16 s8, s3
 ; CHECK-NEXT:    vmov.16 q1[5], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[6]
-; CHECK-NEXT:    vmov s8, r0
-; CHECK-NEXT:    vrintr.f16 s8, s8
 ; CHECK-NEXT:    vmov r0, s8
 ; CHECK-NEXT:    vmov.16 q1[6], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[7]
-; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vrintr.f16 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vmov.16 q1[7], r0
 ; CHECK-NEXT:    vmov q0, q1
@@ -406,44 +358,32 @@ entry:
 define arm_aapcs_vfpcc <8 x half> @ffloor_float16_t(<8 x half> %src) {
 ; CHECK-MVE-LABEL: ffloor_float16_t:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[0]
-; CHECK-MVE-NEXT:    vmov.u16 r1, q0[1]
-; CHECK-MVE-NEXT:    vmov s4, r0
+; CHECK-MVE-NEXT:    vmovx.f16 s4, s0
+; CHECK-MVE-NEXT:    vrintm.f16 s8, s1
 ; CHECK-MVE-NEXT:    vrintm.f16 s4, s4
 ; CHECK-MVE-NEXT:    vmov r0, s4
-; CHECK-MVE-NEXT:    vmov s4, r1
-; CHECK-MVE-NEXT:    vrintm.f16 s4, s4
+; CHECK-MVE-NEXT:    vrintm.f16 s4, s0
 ; CHECK-MVE-NEXT:    vmov r1, s4
-; CHECK-MVE-NEXT:    vmov.16 q1[0], r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[2]
-; CHECK-MVE-NEXT:    vmov.16 q1[1], r1
-; CHECK-MVE-NEXT:    vmov s8, r0
-; CHECK-MVE-NEXT:    vrintm.f16 s8, s8
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q1[0], r1
+; CHECK-MVE-NEXT:    vrintm.f16 s0, s0
+; CHECK-MVE-NEXT:    vmov.16 q1[1], r0
 ; CHECK-MVE-NEXT:    vmov r0, s8
+; CHECK-MVE-NEXT:    vmovx.f16 s8, s1
 ; CHECK-MVE-NEXT:    vmov.16 q1[2], r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[3]
-; CHECK-MVE-NEXT:    vmov s8, r0
 ; CHECK-MVE-NEXT:    vrintm.f16 s8, s8
 ; CHECK-MVE-NEXT:    vmov r0, s8
+; CHECK-MVE-NEXT:    vrintm.f16 s8, s2
 ; CHECK-MVE-NEXT:    vmov.16 q1[3], r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[4]
-; CHECK-MVE-NEXT:    vmov s8, r0
-; CHECK-MVE-NEXT:    vrintm.f16 s8, s8
 ; CHECK-MVE-NEXT:    vmov r0, s8
+; CHECK-MVE-NEXT:    vmovx.f16 s8, s2
 ; CHECK-MVE-NEXT:    vmov.16 q1[4], r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[5]
-; CHECK-MVE-NEXT:    vmov s8, r0
 ; CHECK-MVE-NEXT:    vrintm.f16 s8, s8
 ; CHECK-MVE-NEXT:    vmov r0, s8
+; CHECK-MVE-NEXT:    vrintm.f16 s8, s3
 ; CHECK-MVE-NEXT:    vmov.16 q1[5], r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[6]
-; CHECK-MVE-NEXT:    vmov s8, r0
-; CHECK-MVE-NEXT:    vrintm.f16 s8, s8
 ; CHECK-MVE-NEXT:    vmov r0, s8
 ; CHECK-MVE-NEXT:    vmov.16 q1[6], r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[7]
-; CHECK-MVE-NEXT:    vmov s0, r0
-; CHECK-MVE-NEXT:    vrintm.f16 s0, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q1[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q1
@@ -504,44 +444,32 @@ entry:
 define arm_aapcs_vfpcc <8 x half> @fround_float16_t(<8 x half> %src) {
 ; CHECK-MVE-LABEL: fround_float16_t:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[0]
-; CHECK-MVE-NEXT:    vmov.u16 r1, q0[1]
-; CHECK-MVE-NEXT:    vmov s4, r0
+; CHECK-MVE-NEXT:    vmovx.f16 s4, s0
+; CHECK-MVE-NEXT:    vrinta.f16 s8, s1
 ; CHECK-MVE-NEXT:    vrinta.f16 s4, s4
 ; CHECK-MVE-NEXT:    vmov r0, s4
-; CHECK-MVE-NEXT:    vmov s4, r1
-; CHECK-MVE-NEXT:    vrinta.f16 s4, s4
+; CHECK-MVE-NEXT:    vrinta.f16 s4, s0
 ; CHECK-MVE-NEXT:    vmov r1, s4
-; CHECK-MVE-NEXT:    vmov.16 q1[0], r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[2]
-; CHECK-MVE-NEXT:    vmov.16 q1[1], r1
-; CHECK-MVE-NEXT:    vmov s8, r0
-; CHECK-MVE-NEXT:    vrinta.f16 s8, s8
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q1[0], r1
+; CHECK-MVE-NEXT:    vrinta.f16 s0, s0
+; CHECK-MVE-NEXT:    vmov.16 q1[1], r0
 ; CHECK-MVE-NEXT:    vmov r0, s8
+; CHECK-MVE-NEXT:    vmovx.f16 s8, s1
 ; CHECK-MVE-NEXT:    vmov.16 q1[2], r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[3]
-; CHECK-MVE-NEXT:    vmov s8, r0
 ; CHECK-MVE-NEXT:    vrinta.f16 s8, s8
 ; CHECK-MVE-NEXT:    vmov r0, s8
+; CHECK-MVE-NEXT:    vrinta.f16 s8, s2
 ; CHECK-MVE-NEXT:    vmov.16 q1[3], r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[4]
-; CHECK-MVE-NEXT:    vmov s8, r0
-; CHECK-MVE-NEXT:    vrinta.f16 s8, s8
 ; CHECK-MVE-NEXT:    vmov r0, s8
+; CHECK-MVE-NEXT:    vmovx.f16 s8, s2
 ; CHECK-MVE-NEXT:    vmov.16 q1[4], r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[5]
-; CHECK-MVE-NEXT:    vmov s8, r0
 ; CHECK-MVE-NEXT:    vrinta.f16 s8, s8
 ; CHECK-MVE-NEXT:    vmov r0, s8
+; CHECK-MVE-NEXT:    vrinta.f16 s8, s3
 ; CHECK-MVE-NEXT:    vmov.16 q1[5], r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[6]
-; CHECK-MVE-NEXT:    vmov s8, r0
-; CHECK-MVE-NEXT:    vrinta.f16 s8, s8
 ; CHECK-MVE-NEXT:    vmov r0, s8
 ; CHECK-MVE-NEXT:    vmov.16 q1[6], r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[7]
-; CHECK-MVE-NEXT:    vmov s0, r0
-; CHECK-MVE-NEXT:    vrinta.f16 s0, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q1[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q1
diff --git a/llvm/test/CodeGen/Thumb2/mve-gather-ind16-scaled.ll b/llvm/test/CodeGen/Thumb2/mve-gather-ind16-scaled.ll
new file mode 100644
index 000000000..948f030a8
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-gather-ind16-scaled.ll
@@ -0,0 +1,271 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve.fp -enable-arm-maskedgatscat %s -o - | FileCheck %s
+
+define arm_aapcs_vfpcc <8 x i16> @scaled_v8i16_i16(i16* %base, <8 x i16>* %offptr) {
+; CHECK-LABEL: scaled_v8i16_i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q1, [r1]
+; CHECK-NEXT:    vldrh.u16 q0, [r0, q1, uxtw #1]
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <8 x i16>, <8 x i16>* %offptr, align 2
+  %offs.zext = zext <8 x i16> %offs to <8 x i32>
+  %ptrs = getelementptr inbounds i16, i16* %base, <8 x i32> %offs.zext
+  %gather = call <8 x i16> @llvm.masked.gather.v8i16.v8p0i16(<8 x i16*> %ptrs, i32 2, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i16> undef)
+  ret <8 x i16> %gather
+}
+
+define arm_aapcs_vfpcc <8 x half> @scaled_v8f16_i16(i16* %base, <8 x i16>* %offptr) {
+; CHECK-LABEL: scaled_v8f16_i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q1, [r1]
+; CHECK-NEXT:    vldrh.u16 q0, [r0, q1, uxtw #1]
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <8 x i16>, <8 x i16>* %offptr, align 2
+  %offs.zext = zext <8 x i16> %offs to <8 x i32>
+  %i16_ptrs = getelementptr inbounds i16, i16* %base, <8 x i32> %offs.zext
+  %ptrs = bitcast <8 x i16*> %i16_ptrs to <8 x half*>
+  %gather = call <8 x half> @llvm.masked.gather.v8f16.v8p0f16(<8 x half*> %ptrs, i32 2, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x half> undef)
+  ret <8 x half> %gather
+}
+
+define arm_aapcs_vfpcc <8 x half> @scaled_v8f16_half(half* %base, <8 x i16>* %offptr) {
+; CHECK-LABEL: scaled_v8f16_half:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q1, [r1]
+; CHECK-NEXT:    vldrh.u16 q0, [r0, q1, uxtw #1]
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <8 x i16>, <8 x i16>* %offptr, align 2
+  %offs.zext = zext <8 x i16> %offs to <8 x i32>
+  %ptrs = getelementptr inbounds half, half* %base, <8 x i32> %offs.zext
+  %gather = call <8 x half> @llvm.masked.gather.v8f16.v8p0f16(<8 x half*> %ptrs, i32 2, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x half> undef)
+  ret <8 x half> %gather
+}
+
+define arm_aapcs_vfpcc <8 x i16> @scaled_v8i16_sext(i16* %base, <8 x i16>* %offptr) {
+; CHECK-LABEL: scaled_v8i16_sext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, r5, r7, lr}
+; CHECK-NEXT:    push {r4, r5, r7, lr}
+; CHECK-NEXT:    vldrh.s32 q0, [r1]
+; CHECK-NEXT:    vldrh.s32 q1, [r1, #8]
+; CHECK-NEXT:    vshl.i32 q0, q0, #1
+; CHECK-NEXT:    vshl.i32 q1, q1, #1
+; CHECK-NEXT:    vadd.i32 q0, q0, r0
+; CHECK-NEXT:    vadd.i32 q1, q1, r0
+; CHECK-NEXT:    vmov r2, s2
+; CHECK-NEXT:    vmov r3, s3
+; CHECK-NEXT:    vmov r5, s1
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmov r1, s5
+; CHECK-NEXT:    vmov r4, s7
+; CHECK-NEXT:    ldrh.w r12, [r2]
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    ldrh.w lr, [r3]
+; CHECK-NEXT:    vmov r3, s6
+; CHECK-NEXT:    ldrh r5, [r5]
+; CHECK-NEXT:    ldrh r0, [r0]
+; CHECK-NEXT:    ldrh r1, [r1]
+; CHECK-NEXT:    ldrh r4, [r4]
+; CHECK-NEXT:    ldrh r2, [r2]
+; CHECK-NEXT:    ldrh r3, [r3]
+; CHECK-NEXT:    vmov.16 q0[0], r2
+; CHECK-NEXT:    vmov.16 q0[1], r5
+; CHECK-NEXT:    vmov.16 q0[2], r12
+; CHECK-NEXT:    vmov.16 q0[3], lr
+; CHECK-NEXT:    vmov.16 q0[4], r0
+; CHECK-NEXT:    vmov.16 q0[5], r1
+; CHECK-NEXT:    vmov.16 q0[6], r3
+; CHECK-NEXT:    vmov.16 q0[7], r4
+; CHECK-NEXT:    pop {r4, r5, r7, pc}
+entry:
+  %offs = load <8 x i16>, <8 x i16>* %offptr, align 2
+  %offs.sext = sext <8 x i16> %offs to <8 x i32>
+  %ptrs = getelementptr inbounds i16, i16* %base, <8 x i32> %offs.sext
+  %gather = call <8 x i16> @llvm.masked.gather.v8i16.v8p0i16(<8 x i16*> %ptrs, i32 2, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i16> undef)
+  ret <8 x i16> %gather
+}
+
+define arm_aapcs_vfpcc <8 x half> @scaled_v8f16_sext(i16* %base, <8 x i16>* %offptr) {
+; CHECK-LABEL: scaled_v8f16_sext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.s32 q0, [r1]
+; CHECK-NEXT:    vshl.i32 q0, q0, #1
+; CHECK-NEXT:    vadd.i32 q1, q0, r0
+; CHECK-NEXT:    vmov r2, s5
+; CHECK-NEXT:    vldr.16 s0, [r2]
+; CHECK-NEXT:    vmov r3, s4
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vldr.16 s0, [r3]
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    vmov.16 q0[0], r3
+; CHECK-NEXT:    vmov.16 q0[1], r2
+; CHECK-NEXT:    vmov r2, s6
+; CHECK-NEXT:    vldr.16 s8, [r2]
+; CHECK-NEXT:    vmov r2, s8
+; CHECK-NEXT:    vmov.16 q0[2], r2
+; CHECK-NEXT:    vmov r2, s7
+; CHECK-NEXT:    vldr.16 s4, [r2]
+; CHECK-NEXT:    vmov r2, s4
+; CHECK-NEXT:    vldrh.s32 q1, [r1, #8]
+; CHECK-NEXT:    vmov.16 q0[3], r2
+; CHECK-NEXT:    vshl.i32 q1, q1, #1
+; CHECK-NEXT:    vadd.i32 q1, q1, r0
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vldr.16 s8, [r0]
+; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    vmov.16 q0[4], r0
+; CHECK-NEXT:    vmov r0, s5
+; CHECK-NEXT:    vldr.16 s8, [r0]
+; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    vmov.16 q0[5], r0
+; CHECK-NEXT:    vmov r0, s6
+; CHECK-NEXT:    vldr.16 s8, [r0]
+; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    vmov.16 q0[6], r0
+; CHECK-NEXT:    vmov r0, s7
+; CHECK-NEXT:    vldr.16 s4, [r0]
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmov.16 q0[7], r0
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <8 x i16>, <8 x i16>* %offptr, align 2
+  %offs.sext = sext <8 x i16> %offs to <8 x i32>
+  %i16_ptrs = getelementptr inbounds i16, i16* %base, <8 x i32> %offs.sext
+  %ptrs = bitcast <8 x i16*> %i16_ptrs to <8 x half*>
+  %gather = call <8 x half> @llvm.masked.gather.v8f16.v8p0f16(<8 x half*> %ptrs, i32 2, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x half> undef)
+  ret <8 x half> %gather
+}
+
+define arm_aapcs_vfpcc <8 x i16> @unsigned_scaled_v8i16_i8(i16* %base, <8 x i8>* %offptr) {
+; CHECK-LABEL: unsigned_scaled_v8i16_i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u16 q1, [r1]
+; CHECK-NEXT:    vldrh.u16 q0, [r0, q1, uxtw #1]
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <8 x i8>, <8 x i8>* %offptr, align 1
+  %offs.zext = zext <8 x i8> %offs to <8 x i32>
+  %ptrs = getelementptr inbounds i16, i16* %base, <8 x i32> %offs.zext
+  %gather = call <8 x i16> @llvm.masked.gather.v8i16.v8p0i16(<8 x i16*> %ptrs, i32 2, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i16> undef)
+  ret <8 x i16> %gather
+}
+
+define arm_aapcs_vfpcc <8 x half> @unsigned_scaled_v8f16_i8(i16* %base, <8 x i8>* %offptr) {
+; CHECK-LABEL: unsigned_scaled_v8f16_i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u16 q1, [r1]
+; CHECK-NEXT:    vldrh.u16 q0, [r0, q1, uxtw #1]
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <8 x i8>, <8 x i8>* %offptr, align 1
+  %offs.zext = zext <8 x i8> %offs to <8 x i32>
+  %i16_ptrs = getelementptr inbounds i16, i16* %base, <8 x i32> %offs.zext
+  %ptrs = bitcast <8 x i16*> %i16_ptrs to <8 x half*>
+  %gather = call <8 x half> @llvm.masked.gather.v8f16.v8p0f16(<8 x half*> %ptrs, i32 2, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x half> undef)
+  ret <8 x half> %gather
+}
+
+define arm_aapcs_vfpcc <8 x i16> @scaled_v8i16_i16_passthru0t(i16* %base, <8 x i16>* %offptr) {
+; CHECK-LABEL: scaled_v8i16_i16_passthru0t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q1, [r1]
+; CHECK-NEXT:    vldrh.u16 q0, [r0, q1, uxtw #1]
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <8 x i16>, <8 x i16>* %offptr, align 2
+  %offs.zext = zext <8 x i16> %offs to <8 x i32>
+  %ptrs = getelementptr inbounds i16, i16* %base, <8 x i32> %offs.zext
+  %gather = call <8 x i16> @llvm.masked.gather.v8i16.v8p0i16(<8 x i16*> %ptrs, i32 2, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i16> zeroinitializer)
+  ret <8 x i16> %gather
+}
+
+define arm_aapcs_vfpcc <8 x i16> @scaled_v8i16_i16_passthru1t(i16* %base, <8 x i16>* %offptr) {
+; CHECK-LABEL: scaled_v8i16_i16_passthru1t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q1, [r1]
+; CHECK-NEXT:    vldrh.u16 q0, [r0, q1, uxtw #1]
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <8 x i16>, <8 x i16>* %offptr, align 2
+  %offs.zext = zext <8 x i16> %offs to <8 x i32>
+  %ptrs = getelementptr inbounds i16, i16* %base, <8 x i32> %offs.zext
+  %gather = call <8 x i16> @llvm.masked.gather.v8i16.v8p0i16(<8 x i16*> %ptrs, i32 2, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>)
+  ret <8 x i16> %gather
+}
+
+define arm_aapcs_vfpcc <8 x i16> @scaled_v8i16_i16_passthru1f(i16* %base, <8 x i16>* %offptr) {
+; CHECK-LABEL: scaled_v8i16_i16_passthru1f:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    movw r2, #65487
+; CHECK-NEXT:    vmov.i16 q0, #0x1
+; CHECK-NEXT:    vmsr p0, r2
+; CHECK-NEXT:    vldrh.u16 q1, [r1]
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrht.u16 q2, [r0, q1, uxtw #1]
+; CHECK-NEXT:    vpsel q0, q2, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <8 x i16>, <8 x i16>* %offptr, align 2
+  %offs.zext = zext <8 x i16> %offs to <8 x i32>
+  %ptrs = getelementptr inbounds i16, i16* %base, <8 x i32> %offs.zext
+  %gather = call <8 x i16> @llvm.masked.gather.v8i16.v8p0i16(<8 x i16*> %ptrs, i32 2, <8 x i1> <i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>)
+  ret <8 x i16> %gather
+}
+
+define arm_aapcs_vfpcc <8 x i16> @scaled_v8i16_i16_passthru0f(i16* %base, <8 x i16>* %offptr) {
+; CHECK-LABEL: scaled_v8i16_i16_passthru0f:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    movw r2, #65523
+; CHECK-NEXT:    vmsr p0, r2
+; CHECK-NEXT:    vldrh.u16 q1, [r1]
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrht.u16 q0, [r0, q1, uxtw #1]
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <8 x i16>, <8 x i16>* %offptr, align 2
+  %offs.zext = zext <8 x i16> %offs to <8 x i32>
+  %ptrs = getelementptr inbounds i16, i16* %base, <8 x i32> %offs.zext
+  %gather = call <8 x i16> @llvm.masked.gather.v8i16.v8p0i16(<8 x i16*> %ptrs, i32 2, <8 x i1> <i1 true, i1 false, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i16> <i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0>)
+  ret <8 x i16> %gather
+}
+
+define arm_aapcs_vfpcc <8 x i16> @scaled_v8i16_i16_passthru_icmp0(i16* %base, <8 x i16>* %offptr) {
+; CHECK-LABEL: scaled_v8i16_i16_passthru_icmp0:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q1, [r1]
+; CHECK-NEXT:    vpt.s16 gt, q1, zr
+; CHECK-NEXT:    vldrht.u16 q0, [r0, q1, uxtw #1]
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <8 x i16>, <8 x i16>* %offptr, align 2
+  %offs.zext = zext <8 x i16> %offs to <8 x i32>
+  %ptrs = getelementptr inbounds i16, i16* %base, <8 x i32> %offs.zext
+  %mask = icmp sgt <8 x i16> %offs, zeroinitializer
+  %gather = call <8 x i16> @llvm.masked.gather.v8i16.v8p0i16(<8 x i16*> %ptrs, i32 2, <8 x i1> %mask, <8 x i16> <i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0>)
+  ret <8 x i16> %gather
+}
+
+define arm_aapcs_vfpcc <8 x i16> @scaled_v8i16_i16_passthru_icmp1(i16* %base, <8 x i16>* %offptr) {
+; CHECK-LABEL: scaled_v8i16_i16_passthru_icmp1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.i16 q0, #0x1
+; CHECK-NEXT:    vldrh.u16 q1, [r1]
+; CHECK-NEXT:    vpt.s16 gt, q1, zr
+; CHECK-NEXT:    vldrht.u16 q2, [r0, q1, uxtw #1]
+; CHECK-NEXT:    vpsel q0, q2, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <8 x i16>, <8 x i16>* %offptr, align 2
+  %offs.zext = zext <8 x i16> %offs to <8 x i32>
+  %ptrs = getelementptr inbounds i16, i16* %base, <8 x i32> %offs.zext
+  %mask = icmp sgt <8 x i16> %offs, zeroinitializer
+  %gather = call <8 x i16> @llvm.masked.gather.v8i16.v8p0i16(<8 x i16*> %ptrs, i32 2, <8 x i1> %mask, <8 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>)
+  ret <8 x i16> %gather
+}
+
+declare <8 x i8> @llvm.masked.gather.v8i8.v8p0i8(<8 x i8*>, i32, <8 x i1>, <8 x i8>) #1
+declare <8 x i16> @llvm.masked.gather.v8i16.v8p0i16(<8 x i16*>, i32, <8 x i1>, <8 x i16>) #1
+declare <8 x half> @llvm.masked.gather.v8f16.v8p0f16(<8 x half*>, i32, <8 x i1>, <8 x half>) #1
diff --git a/llvm/test/CodeGen/Thumb2/mve-gather-ind16-unscaled.ll b/llvm/test/CodeGen/Thumb2/mve-gather-ind16-unscaled.ll
new file mode 100644
index 000000000..5e4e5a394
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-gather-ind16-unscaled.ll
@@ -0,0 +1,242 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve.fp -enable-arm-maskedgatscat %s -o - | FileCheck %s
+
+define arm_aapcs_vfpcc <8 x i16> @zext_unscaled_i8_i16(i8* %base, <8 x i16>* %offptr) {
+; CHECK-LABEL: zext_unscaled_i8_i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, r5, r7, lr}
+; CHECK-NEXT:    push {r4, r5, r7, lr}
+; CHECK-NEXT:    vldrh.u32 q0, [r1]
+; CHECK-NEXT:    vldrh.u32 q1, [r1, #8]
+; CHECK-NEXT:    vadd.i32 q0, q0, r0
+; CHECK-NEXT:    vadd.i32 q1, q1, r0
+; CHECK-NEXT:    vmov r2, s3
+; CHECK-NEXT:    vmov r3, s1
+; CHECK-NEXT:    vmov r5, s0
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmov r1, s5
+; CHECK-NEXT:    vmov r4, s7
+; CHECK-NEXT:    ldrb.w r12, [r2]
+; CHECK-NEXT:    vmov r2, s2
+; CHECK-NEXT:    ldrb.w lr, [r3]
+; CHECK-NEXT:    vmov r3, s6
+; CHECK-NEXT:    ldrb r5, [r5]
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.16 q0[0], r5
+; CHECK-NEXT:    ldrb r1, [r1]
+; CHECK-NEXT:    vmov.16 q0[1], lr
+; CHECK-NEXT:    ldrb r4, [r4]
+; CHECK-NEXT:    ldrb r2, [r2]
+; CHECK-NEXT:    ldrb r3, [r3]
+; CHECK-NEXT:    vmov.16 q0[2], r2
+; CHECK-NEXT:    vmov.16 q0[3], r12
+; CHECK-NEXT:    vmov.16 q0[4], r0
+; CHECK-NEXT:    vmov.16 q0[5], r1
+; CHECK-NEXT:    vmov.16 q0[6], r3
+; CHECK-NEXT:    vmov.16 q0[7], r4
+; CHECK-NEXT:    vmovlb.u8 q0, q0
+; CHECK-NEXT:    pop {r4, r5, r7, pc}
+entry:
+  %offs = load <8 x i16>, <8 x i16>* %offptr, align 2
+  %offs.zext = zext <8 x i16> %offs to <8 x i32>
+  %ptrs = getelementptr inbounds i8, i8* %base, <8 x i32> %offs.zext
+  %gather = call <8 x i8> @llvm.masked.gather.v8i8.v8p0i8(<8 x i8*> %ptrs, i32 1, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i8> undef)
+  %gather.zext = zext <8 x i8> %gather to <8 x i16>
+  ret <8 x i16> %gather.zext
+}
+
+define arm_aapcs_vfpcc <8 x i16> @sext_unscaled_i8_i16(i8* %base, <8 x i16>* %offptr) {
+; CHECK-LABEL: sext_unscaled_i8_i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, r5, r7, lr}
+; CHECK-NEXT:    push {r4, r5, r7, lr}
+; CHECK-NEXT:    vldrh.u32 q0, [r1]
+; CHECK-NEXT:    vldrh.u32 q1, [r1, #8]
+; CHECK-NEXT:    vadd.i32 q0, q0, r0
+; CHECK-NEXT:    vadd.i32 q1, q1, r0
+; CHECK-NEXT:    vmov r2, s3
+; CHECK-NEXT:    vmov r3, s1
+; CHECK-NEXT:    vmov r5, s0
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmov r1, s5
+; CHECK-NEXT:    vmov r4, s7
+; CHECK-NEXT:    ldrb.w r12, [r2]
+; CHECK-NEXT:    vmov r2, s2
+; CHECK-NEXT:    ldrb.w lr, [r3]
+; CHECK-NEXT:    vmov r3, s6
+; CHECK-NEXT:    ldrb r5, [r5]
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.16 q0[0], r5
+; CHECK-NEXT:    ldrb r1, [r1]
+; CHECK-NEXT:    vmov.16 q0[1], lr
+; CHECK-NEXT:    ldrb r4, [r4]
+; CHECK-NEXT:    ldrb r2, [r2]
+; CHECK-NEXT:    ldrb r3, [r3]
+; CHECK-NEXT:    vmov.16 q0[2], r2
+; CHECK-NEXT:    vmov.16 q0[3], r12
+; CHECK-NEXT:    vmov.16 q0[4], r0
+; CHECK-NEXT:    vmov.16 q0[5], r1
+; CHECK-NEXT:    vmov.16 q0[6], r3
+; CHECK-NEXT:    vmov.16 q0[7], r4
+; CHECK-NEXT:    vmovlb.s8 q0, q0
+; CHECK-NEXT:    pop {r4, r5, r7, pc}
+entry:
+  %offs = load <8 x i16>, <8 x i16>* %offptr, align 2
+  %offs.zext = zext <8 x i16> %offs to <8 x i32>
+  %ptrs = getelementptr inbounds i8, i8* %base, <8 x i32> %offs.zext
+  %gather = call <8 x i8> @llvm.masked.gather.v8i8.v8p0i8(<8 x i8*> %ptrs, i32 1, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i8> undef)
+  %gather.sext = sext <8 x i8> %gather to <8 x i16>
+  ret <8 x i16> %gather.sext
+}
+
+define arm_aapcs_vfpcc <8 x i16> @unscaled_i16_i16(i8* %base, <8 x i16>* %offptr) {
+; CHECK-LABEL: unscaled_i16_i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q1, [r1]
+; CHECK-NEXT:    vldrh.u16 q0, [r0, q1]
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <8 x i16>, <8 x i16>* %offptr, align 2
+  %offs.zext = zext <8 x i16> %offs to <8 x i32>
+  %byte_ptrs = getelementptr inbounds i8, i8* %base, <8 x i32> %offs.zext
+  %ptrs = bitcast <8 x i8*> %byte_ptrs to <8 x i16*>
+  %gather = call <8 x i16> @llvm.masked.gather.v8i16.v8p0i16(<8 x i16*> %ptrs, i32 2, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i16> undef)
+  ret <8 x i16> %gather
+}
+
+define arm_aapcs_vfpcc <8 x half> @unscaled_f16_i16(i8* %base, <8 x i16>* %offptr) {
+; CHECK-LABEL: unscaled_f16_i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q1, [r1]
+; CHECK-NEXT:    vldrh.u16 q0, [r0, q1]
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <8 x i16>, <8 x i16>* %offptr, align 2
+  %offs.zext = zext <8 x i16> %offs to <8 x i32>
+  %byte_ptrs = getelementptr inbounds i8, i8* %base, <8 x i32> %offs.zext
+  %ptrs = bitcast <8 x i8*> %byte_ptrs to <8 x half*>
+  %gather = call <8 x half> @llvm.masked.gather.v8f16.v8p0f16(<8 x half*> %ptrs, i32 2, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x half> undef)
+  ret <8 x half> %gather
+}
+
+define arm_aapcs_vfpcc <8 x i16> @zext_unsigned_unscaled_i8_i8(i8* %base, <8 x i8>* %offptr) {
+; CHECK-LABEL: zext_unsigned_unscaled_i8_i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, r5, r7, lr}
+; CHECK-NEXT:    push {r4, r5, r7, lr}
+; CHECK-NEXT:    vldrb.u32 q0, [r1]
+; CHECK-NEXT:    vldrb.u32 q1, [r1, #4]
+; CHECK-NEXT:    vadd.i32 q0, q0, r0
+; CHECK-NEXT:    vadd.i32 q1, q1, r0
+; CHECK-NEXT:    vmov r2, s3
+; CHECK-NEXT:    vmov r3, s1
+; CHECK-NEXT:    vmov r5, s0
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmov r1, s5
+; CHECK-NEXT:    vmov r4, s7
+; CHECK-NEXT:    ldrb.w r12, [r2]
+; CHECK-NEXT:    vmov r2, s2
+; CHECK-NEXT:    ldrb.w lr, [r3]
+; CHECK-NEXT:    vmov r3, s6
+; CHECK-NEXT:    ldrb r5, [r5]
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.16 q0[0], r5
+; CHECK-NEXT:    ldrb r1, [r1]
+; CHECK-NEXT:    vmov.16 q0[1], lr
+; CHECK-NEXT:    ldrb r4, [r4]
+; CHECK-NEXT:    ldrb r2, [r2]
+; CHECK-NEXT:    ldrb r3, [r3]
+; CHECK-NEXT:    vmov.16 q0[2], r2
+; CHECK-NEXT:    vmov.16 q0[3], r12
+; CHECK-NEXT:    vmov.16 q0[4], r0
+; CHECK-NEXT:    vmov.16 q0[5], r1
+; CHECK-NEXT:    vmov.16 q0[6], r3
+; CHECK-NEXT:    vmov.16 q0[7], r4
+; CHECK-NEXT:    vmovlb.u8 q0, q0
+; CHECK-NEXT:    pop {r4, r5, r7, pc}
+entry:
+  %offs = load <8 x i8>, <8 x i8>* %offptr, align 1
+  %offs.zext = zext <8 x i8> %offs to <8 x i32>
+  %ptrs = getelementptr inbounds i8, i8* %base, <8 x i32> %offs.zext
+  %gather = call <8 x i8> @llvm.masked.gather.v8i8.v8p0i8(<8 x i8*> %ptrs, i32 1, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i8> undef)
+  %gather.zext = zext <8 x i8> %gather to <8 x i16>
+  ret <8 x i16> %gather.zext
+}
+
+define arm_aapcs_vfpcc <8 x i16> @sext_unsigned_unscaled_i8_i8(i8* %base, <8 x i8>* %offptr) {
+; CHECK-LABEL: sext_unsigned_unscaled_i8_i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, r5, r7, lr}
+; CHECK-NEXT:    push {r4, r5, r7, lr}
+; CHECK-NEXT:    vldrb.u32 q0, [r1]
+; CHECK-NEXT:    vldrb.u32 q1, [r1, #4]
+; CHECK-NEXT:    vadd.i32 q0, q0, r0
+; CHECK-NEXT:    vadd.i32 q1, q1, r0
+; CHECK-NEXT:    vmov r2, s3
+; CHECK-NEXT:    vmov r3, s1
+; CHECK-NEXT:    vmov r5, s0
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmov r1, s5
+; CHECK-NEXT:    vmov r4, s7
+; CHECK-NEXT:    ldrb.w r12, [r2]
+; CHECK-NEXT:    vmov r2, s2
+; CHECK-NEXT:    ldrb.w lr, [r3]
+; CHECK-NEXT:    vmov r3, s6
+; CHECK-NEXT:    ldrb r5, [r5]
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.16 q0[0], r5
+; CHECK-NEXT:    ldrb r1, [r1]
+; CHECK-NEXT:    vmov.16 q0[1], lr
+; CHECK-NEXT:    ldrb r4, [r4]
+; CHECK-NEXT:    ldrb r2, [r2]
+; CHECK-NEXT:    ldrb r3, [r3]
+; CHECK-NEXT:    vmov.16 q0[2], r2
+; CHECK-NEXT:    vmov.16 q0[3], r12
+; CHECK-NEXT:    vmov.16 q0[4], r0
+; CHECK-NEXT:    vmov.16 q0[5], r1
+; CHECK-NEXT:    vmov.16 q0[6], r3
+; CHECK-NEXT:    vmov.16 q0[7], r4
+; CHECK-NEXT:    vmovlb.s8 q0, q0
+; CHECK-NEXT:    pop {r4, r5, r7, pc}
+entry:
+  %offs = load <8 x i8>, <8 x i8>* %offptr, align 1
+  %offs.zext = zext <8 x i8> %offs to <8 x i32>
+  %ptrs = getelementptr inbounds i8, i8* %base, <8 x i32> %offs.zext
+  %gather = call <8 x i8> @llvm.masked.gather.v8i8.v8p0i8(<8 x i8*> %ptrs, i32 1, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i8> undef)
+  %gather.sext = sext <8 x i8> %gather to <8 x i16>
+  ret <8 x i16> %gather.sext
+}
+
+define arm_aapcs_vfpcc <8 x i16> @unsigned_unscaled_i16_i8(i8* %base, <8 x i8>* %offptr) {
+; CHECK-LABEL: unsigned_unscaled_i16_i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u16 q1, [r1]
+; CHECK-NEXT:    vldrh.u16 q0, [r0, q1]
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <8 x i8>, <8 x i8>* %offptr, align 1
+  %offs.zext = zext <8 x i8> %offs to <8 x i32>
+  %byte_ptrs = getelementptr inbounds i8, i8* %base, <8 x i32> %offs.zext
+  %ptrs = bitcast <8 x i8*> %byte_ptrs to <8 x i16*>
+  %gather = call <8 x i16> @llvm.masked.gather.v8i16.v8p0i16(<8 x i16*> %ptrs, i32 2, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i16> undef)
+  ret <8 x i16> %gather
+}
+
+define arm_aapcs_vfpcc <8 x half> @unsigned_unscaled_f16_i8(i8* %base, <8 x i8>* %offptr) {
+; CHECK-LABEL: unsigned_unscaled_f16_i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u16 q1, [r1]
+; CHECK-NEXT:    vldrh.u16 q0, [r0, q1]
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <8 x i8>, <8 x i8>* %offptr, align 1
+  %offs.zext = zext <8 x i8> %offs to <8 x i32>
+  %byte_ptrs = getelementptr inbounds i8, i8* %base, <8 x i32> %offs.zext
+  %ptrs = bitcast <8 x i8*> %byte_ptrs to <8 x half*>
+  %gather = call <8 x half> @llvm.masked.gather.v8f16.v8p0f16(<8 x half*> %ptrs, i32 2, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x half> undef)
+  ret <8 x half> %gather
+}
+
+declare <8 x i8> @llvm.masked.gather.v8i8.v8p0i8(<8 x i8*>, i32, <8 x i1>, <8 x i8>) #1
+declare <8 x i16> @llvm.masked.gather.v8i16.v8p0i16(<8 x i16*>, i32, <8 x i1>, <8 x i16>) #1
+declare <8 x half> @llvm.masked.gather.v8f16.v8p0f16(<8 x half*>, i32, <8 x i1>, <8 x half>) #1
diff --git a/llvm/test/CodeGen/Thumb2/mve-gather-ind32-scaled.ll b/llvm/test/CodeGen/Thumb2/mve-gather-ind32-scaled.ll
new file mode 100644
index 000000000..1acba1a95
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-gather-ind32-scaled.ll
@@ -0,0 +1,441 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve.fp -enable-arm-maskedgatscat %s -o 2>/dev/null - | FileCheck %s
+
+define arm_aapcs_vfpcc <4 x i32> @zext_scaled_i16_i32(i16* %base, <4 x i32>* %offptr) {
+; CHECK-LABEL: zext_scaled_i16_i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    vshl.i32 q0, q0, #1
+; CHECK-NEXT:    vadd.i32 q0, q0, r0
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vmov r0, s1
+; CHECK-NEXT:    vmov r3, s2
+; CHECK-NEXT:    vmov r1, s3
+; CHECK-NEXT:    ldrh r2, [r2]
+; CHECK-NEXT:    ldrh r0, [r0]
+; CHECK-NEXT:    vmov.32 q0[0], r2
+; CHECK-NEXT:    ldrh r3, [r3]
+; CHECK-NEXT:    vmov.32 q0[1], r0
+; CHECK-NEXT:    ldrh r1, [r1]
+; CHECK-NEXT:    vmov.32 q0[2], r3
+; CHECK-NEXT:    vmov.32 q0[3], r1
+; CHECK-NEXT:    vmovlb.u16 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <4 x i32>, <4 x i32>* %offptr, align 4
+  %ptrs = getelementptr inbounds i16, i16* %base, <4 x i32> %offs
+  %gather = call <4 x i16> @llvm.masked.gather.v4i16.v4p0i16(<4 x i16*> %ptrs, i32 2, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i16> undef)
+  %gather.zext = zext <4 x i16> %gather to <4 x i32>
+  ret <4 x i32> %gather.zext
+}
+
+define arm_aapcs_vfpcc <4 x i32> @sext_scaled_i16_i32(i16* %base, <4 x i32>* %offptr) {
+; CHECK-LABEL: sext_scaled_i16_i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    vshl.i32 q0, q0, #1
+; CHECK-NEXT:    vadd.i32 q0, q0, r0
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vmov r0, s1
+; CHECK-NEXT:    vmov r3, s2
+; CHECK-NEXT:    vmov r1, s3
+; CHECK-NEXT:    ldrh r2, [r2]
+; CHECK-NEXT:    ldrh r0, [r0]
+; CHECK-NEXT:    vmov.32 q0[0], r2
+; CHECK-NEXT:    ldrh r3, [r3]
+; CHECK-NEXT:    vmov.32 q0[1], r0
+; CHECK-NEXT:    ldrh r1, [r1]
+; CHECK-NEXT:    vmov.32 q0[2], r3
+; CHECK-NEXT:    vmov.32 q0[3], r1
+; CHECK-NEXT:    vmovlb.s16 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <4 x i32>, <4 x i32>* %offptr, align 4
+  %ptrs = getelementptr inbounds i16, i16* %base, <4 x i32> %offs
+  %gather = call <4 x i16> @llvm.masked.gather.v4i16.v4p0i16(<4 x i16*> %ptrs, i32 2, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i16> undef)
+  %gather.sext = sext <4 x i16> %gather to <4 x i32>
+  ret <4 x i32> %gather.sext
+}
+
+define arm_aapcs_vfpcc <4 x i32> @scaled_i32_i32(i32* %base, <4 x i32>* %offptr) {
+; CHECK-LABEL: scaled_i32_i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q1, [r1]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, q1, uxtw #2]
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <4 x i32>, <4 x i32>* %offptr, align 4
+  %ptrs = getelementptr inbounds i32, i32* %base, <4 x i32> %offs
+  %gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %ptrs, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
+  ret <4 x i32> %gather
+}
+
+; TODO: scaled_f16_i32
+
+define arm_aapcs_vfpcc <4 x float> @scaled_f32_i32(i32* %base, <4 x i32>* %offptr) {
+; CHECK-LABEL: scaled_f32_i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q1, [r1]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, q1, uxtw #2]
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <4 x i32>, <4 x i32>* %offptr, align 4
+  %i32_ptrs = getelementptr inbounds i32, i32* %base, <4 x i32> %offs
+  %ptrs = bitcast <4 x i32*> %i32_ptrs to <4 x float*>
+  %gather = call <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*> %ptrs, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x float> undef)
+  ret <4 x float> %gather
+}
+
+define arm_aapcs_vfpcc <4 x i32> @unsigned_scaled_b_i32_i16(i32* %base, <4 x i16>* %offptr) {
+; CHECK-LABEL: unsigned_scaled_b_i32_i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u32 q1, [r1]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, q1, uxtw #2]
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <4 x i16>, <4 x i16>* %offptr, align 2
+  %offs.zext = zext <4 x i16> %offs to <4 x i32>
+  %ptrs = getelementptr inbounds i32, i32* %base, <4 x i32> %offs.zext
+  %gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %ptrs, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
+  ret <4 x i32> %gather
+}
+
+define arm_aapcs_vfpcc <4 x i32> @signed_scaled_i32_i16(i32* %base, <4 x i16>* %offptr) {
+; CHECK-LABEL: signed_scaled_i32_i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.s32 q1, [r1]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, q1, uxtw #2]
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <4 x i16>, <4 x i16>* %offptr, align 2
+  %offs.sext = sext <4 x i16> %offs to <4 x i32>
+  %ptrs = getelementptr inbounds i32, i32* %base, <4 x i32> %offs.sext
+  %gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %ptrs, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
+  ret <4 x i32> %gather
+}
+
+define arm_aapcs_vfpcc <4 x float> @a_unsigned_scaled_f32_i16(i32* %base, <4 x i16>* %offptr) {
+; CHECK-LABEL: a_unsigned_scaled_f32_i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u32 q1, [r1]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, q1, uxtw #2]
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <4 x i16>, <4 x i16>* %offptr, align 2
+  %offs.zext = zext <4 x i16> %offs to <4 x i32>
+  %i32_ptrs = getelementptr inbounds i32, i32* %base, <4 x i32> %offs.zext
+  %ptrs = bitcast <4 x i32*> %i32_ptrs to <4 x float*>
+  %gather = call <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*> %ptrs, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x float> undef)
+  ret <4 x float> %gather
+}
+
+define arm_aapcs_vfpcc <4 x float> @b_signed_scaled_f32_i16(i32* %base, <4 x i16>* %offptr) {
+; CHECK-LABEL: b_signed_scaled_f32_i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.s32 q1, [r1]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, q1, uxtw #2]
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <4 x i16>, <4 x i16>* %offptr, align 2
+  %offs.sext = sext <4 x i16> %offs to <4 x i32>
+  %i32_ptrs = getelementptr inbounds i32, i32* %base, <4 x i32> %offs.sext
+  %ptrs = bitcast <4 x i32*> %i32_ptrs to <4 x float*>
+  %gather = call <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*> %ptrs, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x float> undef)
+  ret <4 x float> %gather
+}
+
+define arm_aapcs_vfpcc <4 x i32> @zext_signed_scaled_i16_i16(i16* %base, <4 x i16>* %offptr) {
+; CHECK-LABEL: zext_signed_scaled_i16_i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.s32 q0, [r1]
+; CHECK-NEXT:    vshl.i32 q0, q0, #1
+; CHECK-NEXT:    vadd.i32 q0, q0, r0
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vmov r0, s1
+; CHECK-NEXT:    vmov r3, s2
+; CHECK-NEXT:    vmov r1, s3
+; CHECK-NEXT:    ldrh r2, [r2]
+; CHECK-NEXT:    ldrh r0, [r0]
+; CHECK-NEXT:    vmov.32 q0[0], r2
+; CHECK-NEXT:    ldrh r3, [r3]
+; CHECK-NEXT:    vmov.32 q0[1], r0
+; CHECK-NEXT:    ldrh r1, [r1]
+; CHECK-NEXT:    vmov.32 q0[2], r3
+; CHECK-NEXT:    vmov.32 q0[3], r1
+; CHECK-NEXT:    vmovlb.u16 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <4 x i16>, <4 x i16>* %offptr, align 2
+  %offs.sext = sext <4 x i16> %offs to <4 x i32>
+  %ptrs = getelementptr inbounds i16, i16* %base, <4 x i32> %offs.sext
+  %gather = call <4 x i16> @llvm.masked.gather.v4i16.v4p0i16(<4 x i16*> %ptrs, i32 2, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i16> undef)
+  %gather.zext = zext <4 x i16> %gather to <4 x i32>
+  ret <4 x i32> %gather.zext
+}
+
+define arm_aapcs_vfpcc <4 x i32> @sext_signed_scaled_i16_i16(i16* %base, <4 x i16>* %offptr) {
+; CHECK-LABEL: sext_signed_scaled_i16_i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.s32 q0, [r1]
+; CHECK-NEXT:    vshl.i32 q0, q0, #1
+; CHECK-NEXT:    vadd.i32 q0, q0, r0
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vmov r0, s1
+; CHECK-NEXT:    vmov r3, s2
+; CHECK-NEXT:    vmov r1, s3
+; CHECK-NEXT:    ldrh r2, [r2]
+; CHECK-NEXT:    ldrh r0, [r0]
+; CHECK-NEXT:    vmov.32 q0[0], r2
+; CHECK-NEXT:    ldrh r3, [r3]
+; CHECK-NEXT:    vmov.32 q0[1], r0
+; CHECK-NEXT:    ldrh r1, [r1]
+; CHECK-NEXT:    vmov.32 q0[2], r3
+; CHECK-NEXT:    vmov.32 q0[3], r1
+; CHECK-NEXT:    vmovlb.s16 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <4 x i16>, <4 x i16>* %offptr, align 2
+  %offs.sext = sext <4 x i16> %offs to <4 x i32>
+  %ptrs = getelementptr inbounds i16, i16* %base, <4 x i32> %offs.sext
+  %gather = call <4 x i16> @llvm.masked.gather.v4i16.v4p0i16(<4 x i16*> %ptrs, i32 2, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i16> undef)
+  %gather.sext = sext <4 x i16> %gather to <4 x i32>
+  ret <4 x i32> %gather.sext
+}
+
+define arm_aapcs_vfpcc <4 x i32> @zext_unsigned_scaled_i16_i16(i16* %base, <4 x i16>* %offptr) {
+; CHECK-LABEL: zext_unsigned_scaled_i16_i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u32 q0, [r1]
+; CHECK-NEXT:    vshl.i32 q0, q0, #1
+; CHECK-NEXT:    vadd.i32 q0, q0, r0
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vmov r0, s1
+; CHECK-NEXT:    vmov r3, s2
+; CHECK-NEXT:    vmov r1, s3
+; CHECK-NEXT:    ldrh r2, [r2]
+; CHECK-NEXT:    ldrh r0, [r0]
+; CHECK-NEXT:    vmov.32 q0[0], r2
+; CHECK-NEXT:    ldrh r3, [r3]
+; CHECK-NEXT:    vmov.32 q0[1], r0
+; CHECK-NEXT:    ldrh r1, [r1]
+; CHECK-NEXT:    vmov.32 q0[2], r3
+; CHECK-NEXT:    vmov.32 q0[3], r1
+; CHECK-NEXT:    vmovlb.u16 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <4 x i16>, <4 x i16>* %offptr, align 2
+  %offs.zext = zext <4 x i16> %offs to <4 x i32>
+  %ptrs = getelementptr inbounds i16, i16* %base, <4 x i32> %offs.zext
+  %gather = call <4 x i16> @llvm.masked.gather.v4i16.v4p0i16(<4 x i16*> %ptrs, i32 2, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i16> undef)
+  %gather.zext = zext <4 x i16> %gather to <4 x i32>
+  ret <4 x i32> %gather.zext
+}
+
+define arm_aapcs_vfpcc <4 x i32> @sext_unsigned_scaled_i16_i16(i16* %base, <4 x i16>* %offptr) {
+; CHECK-LABEL: sext_unsigned_scaled_i16_i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u32 q0, [r1]
+; CHECK-NEXT:    vshl.i32 q0, q0, #1
+; CHECK-NEXT:    vadd.i32 q0, q0, r0
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vmov r0, s1
+; CHECK-NEXT:    vmov r3, s2
+; CHECK-NEXT:    vmov r1, s3
+; CHECK-NEXT:    ldrh r2, [r2]
+; CHECK-NEXT:    ldrh r0, [r0]
+; CHECK-NEXT:    vmov.32 q0[0], r2
+; CHECK-NEXT:    ldrh r3, [r3]
+; CHECK-NEXT:    vmov.32 q0[1], r0
+; CHECK-NEXT:    ldrh r1, [r1]
+; CHECK-NEXT:    vmov.32 q0[2], r3
+; CHECK-NEXT:    vmov.32 q0[3], r1
+; CHECK-NEXT:    vmovlb.s16 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <4 x i16>, <4 x i16>* %offptr, align 2
+  %offs.zext = zext <4 x i16> %offs to <4 x i32>
+  %ptrs = getelementptr inbounds i16, i16* %base, <4 x i32> %offs.zext
+  %gather = call <4 x i16> @llvm.masked.gather.v4i16.v4p0i16(<4 x i16*> %ptrs, i32 2, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i16> undef)
+  %gather.sext = sext <4 x i16> %gather to <4 x i32>
+  ret <4 x i32> %gather.sext
+}
+
+define arm_aapcs_vfpcc <4 x i32> @unsigned_scaled_b_i32_i8(i32* %base, <4 x i8>* %offptr) {
+; CHECK-LABEL: unsigned_scaled_b_i32_i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u32 q1, [r1]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, q1, uxtw #2]
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <4 x i8>, <4 x i8>* %offptr, align 1
+  %offs.zext = zext <4 x i8> %offs to <4 x i32>
+  %ptrs = getelementptr inbounds i32, i32* %base, <4 x i32> %offs.zext
+  %gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %ptrs, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
+  ret <4 x i32> %gather
+}
+
+define arm_aapcs_vfpcc <4 x i32> @signed_scaled_i32_i8(i32* %base, <4 x i8>* %offptr) {
+; CHECK-LABEL: signed_scaled_i32_i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.s32 q1, [r1]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, q1, uxtw #2]
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <4 x i8>, <4 x i8>* %offptr, align 1
+  %offs.sext = sext <4 x i8> %offs to <4 x i32>
+  %ptrs = getelementptr inbounds i32, i32* %base, <4 x i32> %offs.sext
+  %gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %ptrs, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
+  ret <4 x i32> %gather
+}
+
+define arm_aapcs_vfpcc <4 x float> @a_unsigned_scaled_f32_i8(i32* %base, <4 x i8>* %offptr) {
+; CHECK-LABEL: a_unsigned_scaled_f32_i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u32 q1, [r1]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, q1, uxtw #2]
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <4 x i8>, <4 x i8>* %offptr, align 1
+  %offs.zext = zext <4 x i8> %offs to <4 x i32>
+  %i32_ptrs = getelementptr inbounds i32, i32* %base, <4 x i32> %offs.zext
+  %ptrs = bitcast <4 x i32*> %i32_ptrs to <4 x float*>
+  %gather = call <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*> %ptrs, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x float> undef)
+  ret <4 x float> %gather
+}
+
+define arm_aapcs_vfpcc <4 x float> @b_signed_scaled_f32_i8(i32* %base, <4 x i8>* %offptr) {
+; CHECK-LABEL: b_signed_scaled_f32_i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.s32 q1, [r1]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, q1, uxtw #2]
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <4 x i8>, <4 x i8>* %offptr, align 1
+  %offs.sext = sext <4 x i8> %offs to <4 x i32>
+  %i32_ptrs = getelementptr inbounds i32, i32* %base, <4 x i32> %offs.sext
+  %ptrs = bitcast <4 x i32*> %i32_ptrs to <4 x float*>
+  %gather = call <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*> %ptrs, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x float> undef)
+  ret <4 x float> %gather
+}
+
+define arm_aapcs_vfpcc <4 x i32> @zext_signed_scaled_i16_i8(i16* %base, <4 x i8>* %offptr) {
+; CHECK-LABEL: zext_signed_scaled_i16_i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.s32 q0, [r1]
+; CHECK-NEXT:    vshl.i32 q0, q0, #1
+; CHECK-NEXT:    vadd.i32 q0, q0, r0
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vmov r0, s1
+; CHECK-NEXT:    vmov r3, s2
+; CHECK-NEXT:    vmov r1, s3
+; CHECK-NEXT:    ldrh r2, [r2]
+; CHECK-NEXT:    ldrh r0, [r0]
+; CHECK-NEXT:    vmov.32 q0[0], r2
+; CHECK-NEXT:    ldrh r3, [r3]
+; CHECK-NEXT:    vmov.32 q0[1], r0
+; CHECK-NEXT:    ldrh r1, [r1]
+; CHECK-NEXT:    vmov.32 q0[2], r3
+; CHECK-NEXT:    vmov.32 q0[3], r1
+; CHECK-NEXT:    vmovlb.u16 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <4 x i8>, <4 x i8>* %offptr, align 1
+  %offs.sext = sext <4 x i8> %offs to <4 x i32>
+  %ptrs = getelementptr inbounds i16, i16* %base, <4 x i32> %offs.sext
+  %gather = call <4 x i16> @llvm.masked.gather.v4i16.v4p0i16(<4 x i16*> %ptrs, i32 2, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i16> undef)
+  %gather.zext = zext <4 x i16> %gather to <4 x i32>
+  ret <4 x i32> %gather.zext
+}
+
+define arm_aapcs_vfpcc <4 x i32> @sext_signed_scaled_i16_i8(i16* %base, <4 x i8>* %offptr) {
+; CHECK-LABEL: sext_signed_scaled_i16_i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.s32 q0, [r1]
+; CHECK-NEXT:    vshl.i32 q0, q0, #1
+; CHECK-NEXT:    vadd.i32 q0, q0, r0
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vmov r0, s1
+; CHECK-NEXT:    vmov r3, s2
+; CHECK-NEXT:    vmov r1, s3
+; CHECK-NEXT:    ldrh r2, [r2]
+; CHECK-NEXT:    ldrh r0, [r0]
+; CHECK-NEXT:    vmov.32 q0[0], r2
+; CHECK-NEXT:    ldrh r3, [r3]
+; CHECK-NEXT:    vmov.32 q0[1], r0
+; CHECK-NEXT:    ldrh r1, [r1]
+; CHECK-NEXT:    vmov.32 q0[2], r3
+; CHECK-NEXT:    vmov.32 q0[3], r1
+; CHECK-NEXT:    vmovlb.s16 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <4 x i8>, <4 x i8>* %offptr, align 1
+  %offs.sext = sext <4 x i8> %offs to <4 x i32>
+  %ptrs = getelementptr inbounds i16, i16* %base, <4 x i32> %offs.sext
+  %gather = call <4 x i16> @llvm.masked.gather.v4i16.v4p0i16(<4 x i16*> %ptrs, i32 2, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i16> undef)
+  %gather.sext = sext <4 x i16> %gather to <4 x i32>
+  ret <4 x i32> %gather.sext
+}
+
+define arm_aapcs_vfpcc <4 x i32> @zext_unsigned_scaled_i16_i8(i16* %base, <4 x i8>* %offptr) {
+; CHECK-LABEL: zext_unsigned_scaled_i16_i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u32 q0, [r1]
+; CHECK-NEXT:    vshl.i32 q0, q0, #1
+; CHECK-NEXT:    vadd.i32 q0, q0, r0
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vmov r0, s1
+; CHECK-NEXT:    vmov r3, s2
+; CHECK-NEXT:    vmov r1, s3
+; CHECK-NEXT:    ldrh r2, [r2]
+; CHECK-NEXT:    ldrh r0, [r0]
+; CHECK-NEXT:    vmov.32 q0[0], r2
+; CHECK-NEXT:    ldrh r3, [r3]
+; CHECK-NEXT:    vmov.32 q0[1], r0
+; CHECK-NEXT:    ldrh r1, [r1]
+; CHECK-NEXT:    vmov.32 q0[2], r3
+; CHECK-NEXT:    vmov.32 q0[3], r1
+; CHECK-NEXT:    vmovlb.u16 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <4 x i8>, <4 x i8>* %offptr, align 1
+  %offs.zext = zext <4 x i8> %offs to <4 x i32>
+  %ptrs = getelementptr inbounds i16, i16* %base, <4 x i32> %offs.zext
+  %gather = call <4 x i16> @llvm.masked.gather.v4i16.v4p0i16(<4 x i16*> %ptrs, i32 2, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i16> undef)
+  %gather.zext = zext <4 x i16> %gather to <4 x i32>
+  ret <4 x i32> %gather.zext
+}
+
+define arm_aapcs_vfpcc <4 x i32> @sext_unsigned_scaled_i16_i8(i16* %base, <4 x i8>* %offptr) {
+; CHECK-LABEL: sext_unsigned_scaled_i16_i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u32 q0, [r1]
+; CHECK-NEXT:    vshl.i32 q0, q0, #1
+; CHECK-NEXT:    vadd.i32 q0, q0, r0
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vmov r0, s1
+; CHECK-NEXT:    vmov r3, s2
+; CHECK-NEXT:    vmov r1, s3
+; CHECK-NEXT:    ldrh r2, [r2]
+; CHECK-NEXT:    ldrh r0, [r0]
+; CHECK-NEXT:    vmov.32 q0[0], r2
+; CHECK-NEXT:    ldrh r3, [r3]
+; CHECK-NEXT:    vmov.32 q0[1], r0
+; CHECK-NEXT:    ldrh r1, [r1]
+; CHECK-NEXT:    vmov.32 q0[2], r3
+; CHECK-NEXT:    vmov.32 q0[3], r1
+; CHECK-NEXT:    vmovlb.s16 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <4 x i8>, <4 x i8>* %offptr, align 1
+  %offs.zext = zext <4 x i8> %offs to <4 x i32>
+  %ptrs = getelementptr inbounds i16, i16* %base, <4 x i32> %offs.zext
+  %gather = call <4 x i16> @llvm.masked.gather.v4i16.v4p0i16(<4 x i16*> %ptrs, i32 2, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i16> undef)
+  %gather.sext = sext <4 x i16> %gather to <4 x i32>
+  ret <4 x i32> %gather.sext
+}
+
+declare <4 x i8>  @llvm.masked.gather.v4i8.v4p0i8(<4 x i8*>, i32, <4 x i1>, <4 x i8>)
+declare <4 x i16> @llvm.masked.gather.v4i16.v4p0i16(<4 x i16*>, i32, <4 x i1>, <4 x i16>)
+declare <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*>, i32, <4 x i1>, <4 x i32>)
+declare <4 x half> @llvm.masked.gather.v4f16.v4p0f16(<4 x half*>, i32, <4 x i1>, <4 x half>)
+declare <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*>, i32, <4 x i1>, <4 x float>)
diff --git a/llvm/test/CodeGen/Thumb2/mve-gather-ind32-unscaled.ll b/llvm/test/CodeGen/Thumb2/mve-gather-ind32-unscaled.ll
new file mode 100644
index 000000000..c862aa965
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-gather-ind32-unscaled.ll
@@ -0,0 +1,757 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve.fp -enable-arm-maskedgatscat %s -o 2>/dev/null - | FileCheck %s
+
+define arm_aapcs_vfpcc <4 x i32> @zext_unscaled_i8_i32(i8* %base, <4 x i32>* %offptr) {
+; CHECK-LABEL: zext_unscaled_i8_i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    vmov.i32 q1, #0xff
+; CHECK-NEXT:    vadd.i32 q0, q0, r0
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vmov r1, s1
+; CHECK-NEXT:    vmov r3, s2
+; CHECK-NEXT:    vmov r0, s3
+; CHECK-NEXT:    ldrb r2, [r2]
+; CHECK-NEXT:    ldrb r1, [r1]
+; CHECK-NEXT:    vmov.32 q0[0], r2
+; CHECK-NEXT:    ldrb r3, [r3]
+; CHECK-NEXT:    vmov.32 q0[1], r1
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.32 q0[2], r3
+; CHECK-NEXT:    vmov.32 q0[3], r0
+; CHECK-NEXT:    vand q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <4 x i32>, <4 x i32>* %offptr, align 4
+  %ptrs = getelementptr inbounds i8, i8* %base, <4 x i32> %offs
+  %gather = call <4 x i8> @llvm.masked.gather.v4i8.v4p0i8(<4 x i8*> %ptrs, i32 1, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i8> undef)
+  %gather.zext = zext <4 x i8> %gather to <4 x i32>
+  ret <4 x i32> %gather.zext
+}
+
+define arm_aapcs_vfpcc <4 x i32> @sext_unscaled_i8_i32(i8* %base, <4 x i32>* %offptr) {
+; CHECK-LABEL: sext_unscaled_i8_i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    vadd.i32 q0, q0, r0
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vmov r3, s1
+; CHECK-NEXT:    vmov r1, s2
+; CHECK-NEXT:    vmov r2, s3
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    ldrb r3, [r3]
+; CHECK-NEXT:    vmov.32 q0[0], r0
+; CHECK-NEXT:    ldrb r1, [r1]
+; CHECK-NEXT:    vmov.32 q0[1], r3
+; CHECK-NEXT:    ldrb r2, [r2]
+; CHECK-NEXT:    vmov.32 q0[2], r1
+; CHECK-NEXT:    vmov.32 q0[3], r2
+; CHECK-NEXT:    vmovlb.s8 q0, q0
+; CHECK-NEXT:    vmovlb.s16 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <4 x i32>, <4 x i32>* %offptr, align 4
+  %ptrs = getelementptr inbounds i8, i8* %base, <4 x i32> %offs
+  %gather = call <4 x i8> @llvm.masked.gather.v4i8.v4p0i8(<4 x i8*> %ptrs, i32 1, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i8> undef)
+  %gather.sext = sext <4 x i8> %gather to <4 x i32>
+  ret <4 x i32> %gather.sext
+}
+
+define arm_aapcs_vfpcc <4 x i32> @zext_unscaled_i16_i32(i8* %base, <4 x i32>* %offptr) {
+; CHECK-LABEL: zext_unscaled_i16_i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    vadd.i32 q0, q0, r0
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vmov r0, s1
+; CHECK-NEXT:    vmov r3, s2
+; CHECK-NEXT:    vmov r1, s3
+; CHECK-NEXT:    ldrh r2, [r2]
+; CHECK-NEXT:    ldrh r0, [r0]
+; CHECK-NEXT:    vmov.32 q0[0], r2
+; CHECK-NEXT:    ldrh r3, [r3]
+; CHECK-NEXT:    vmov.32 q0[1], r0
+; CHECK-NEXT:    ldrh r1, [r1]
+; CHECK-NEXT:    vmov.32 q0[2], r3
+; CHECK-NEXT:    vmov.32 q0[3], r1
+; CHECK-NEXT:    vmovlb.u16 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <4 x i32>, <4 x i32>* %offptr, align 4
+  %byte_ptrs = getelementptr inbounds i8, i8* %base, <4 x i32> %offs
+  %ptrs = bitcast <4 x i8*> %byte_ptrs to <4 x i16*>
+  %gather = call <4 x i16> @llvm.masked.gather.v4i16.v4p0i16(<4 x i16*> %ptrs, i32 2, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i16> undef)
+  %gather.zext = zext <4 x i16> %gather to <4 x i32>
+  ret <4 x i32> %gather.zext
+}
+
+define arm_aapcs_vfpcc <4 x i32> @sext_unscaled_i16_i32(i8* %base, <4 x i32>* %offptr) {
+; CHECK-LABEL: sext_unscaled_i16_i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    vadd.i32 q0, q0, r0
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vmov r0, s1
+; CHECK-NEXT:    vmov r3, s2
+; CHECK-NEXT:    vmov r1, s3
+; CHECK-NEXT:    ldrh r2, [r2]
+; CHECK-NEXT:    ldrh r0, [r0]
+; CHECK-NEXT:    vmov.32 q0[0], r2
+; CHECK-NEXT:    ldrh r3, [r3]
+; CHECK-NEXT:    vmov.32 q0[1], r0
+; CHECK-NEXT:    ldrh r1, [r1]
+; CHECK-NEXT:    vmov.32 q0[2], r3
+; CHECK-NEXT:    vmov.32 q0[3], r1
+; CHECK-NEXT:    vmovlb.s16 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <4 x i32>, <4 x i32>* %offptr, align 4
+  %byte_ptrs = getelementptr inbounds i8, i8* %base, <4 x i32> %offs
+  %ptrs = bitcast <4 x i8*> %byte_ptrs to <4 x i16*>
+  %gather = call <4 x i16> @llvm.masked.gather.v4i16.v4p0i16(<4 x i16*> %ptrs, i32 2, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i16> undef)
+  %gather.sext = sext <4 x i16> %gather to <4 x i32>
+  ret <4 x i32> %gather.sext
+}
+
+define arm_aapcs_vfpcc <4 x i32> @unscaled_i32_i32(i8* %base, <4 x i32>* %offptr) {
+; CHECK-LABEL: unscaled_i32_i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q1, [r1]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, q1]
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <4 x i32>, <4 x i32>* %offptr, align 4
+  %byte_ptrs = getelementptr inbounds i8, i8* %base, <4 x i32> %offs
+  %ptrs = bitcast <4 x i8*> %byte_ptrs to <4 x i32*>
+  %gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %ptrs, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
+  ret <4 x i32> %gather
+}
+
+define arm_aapcs_vfpcc <4 x float> @unscaled_f32_i32(i8* %base, <4 x i32>* %offptr) {
+; CHECK-LABEL: unscaled_f32_i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q1, [r1]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, q1]
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <4 x i32>, <4 x i32>* %offptr, align 4
+  %byte_ptrs = getelementptr inbounds i8, i8* %base, <4 x i32> %offs
+  %ptrs = bitcast <4 x i8*> %byte_ptrs to <4 x float*>
+  %gather = call <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*> %ptrs, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x float> undef)
+  ret <4 x float> %gather
+}
+
+define arm_aapcs_vfpcc <4 x i32> @unsigned_unscaled_b_i32_i16(i8* %base, <4 x i16>* %offptr) {
+; CHECK-LABEL: unsigned_unscaled_b_i32_i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u32 q1, [r1]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, q1]
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <4 x i16>, <4 x i16>* %offptr, align 2
+  %offs.zext = zext <4 x i16> %offs to <4 x i32>
+  %byte_ptrs = getelementptr inbounds i8, i8* %base, <4 x i32> %offs.zext
+  %ptrs = bitcast <4 x i8*> %byte_ptrs to <4 x i32*>
+  %gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %ptrs, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
+  ret <4 x i32> %gather
+}
+
+define arm_aapcs_vfpcc <4 x i32> @signed_unscaled_i32_i16(i8* %base, <4 x i16>* %offptr) {
+; CHECK-LABEL: signed_unscaled_i32_i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.s32 q1, [r1]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, q1]
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <4 x i16>, <4 x i16>* %offptr, align 2
+  %offs.sext = sext <4 x i16> %offs to <4 x i32>
+  %byte_ptrs = getelementptr inbounds i8, i8* %base, <4 x i32> %offs.sext
+  %ptrs = bitcast <4 x i8*> %byte_ptrs to <4 x i32*>
+  %gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %ptrs, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
+  ret <4 x i32> %gather
+}
+
+define arm_aapcs_vfpcc <4 x float> @a_unsigned_unscaled_f32_i16(i8* %base, <4 x i16>* %offptr) {
+; CHECK-LABEL: a_unsigned_unscaled_f32_i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u32 q1, [r1]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, q1]
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <4 x i16>, <4 x i16>* %offptr, align 2
+  %offs.zext = zext <4 x i16> %offs to <4 x i32>
+  %byte_ptrs = getelementptr inbounds i8, i8* %base, <4 x i32> %offs.zext
+  %ptrs = bitcast <4 x i8*> %byte_ptrs to <4 x float*>
+  %gather = call <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*> %ptrs, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x float> undef)
+  ret <4 x float> %gather
+}
+
+define arm_aapcs_vfpcc <4 x float> @b_signed_unscaled_f32_i16(i8* %base, <4 x i16>* %offptr) {
+; CHECK-LABEL: b_signed_unscaled_f32_i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.s32 q1, [r1]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, q1]
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <4 x i16>, <4 x i16>* %offptr, align 2
+  %offs.sext = sext <4 x i16> %offs to <4 x i32>
+  %byte_ptrs = getelementptr inbounds i8, i8* %base, <4 x i32> %offs.sext
+  %ptrs = bitcast <4 x i8*> %byte_ptrs to <4 x float*>
+  %gather = call <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*> %ptrs, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x float> undef)
+  ret <4 x float> %gather
+}
+
+define arm_aapcs_vfpcc <4 x i32> @zext_signed_unscaled_i16_i16(i8* %base, <4 x i16>* %offptr) {
+; CHECK-LABEL: zext_signed_unscaled_i16_i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.s32 q0, [r1]
+; CHECK-NEXT:    vadd.i32 q0, q0, r0
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vmov r0, s1
+; CHECK-NEXT:    vmov r3, s2
+; CHECK-NEXT:    vmov r1, s3
+; CHECK-NEXT:    ldrh r2, [r2]
+; CHECK-NEXT:    ldrh r0, [r0]
+; CHECK-NEXT:    vmov.32 q0[0], r2
+; CHECK-NEXT:    ldrh r3, [r3]
+; CHECK-NEXT:    vmov.32 q0[1], r0
+; CHECK-NEXT:    ldrh r1, [r1]
+; CHECK-NEXT:    vmov.32 q0[2], r3
+; CHECK-NEXT:    vmov.32 q0[3], r1
+; CHECK-NEXT:    vmovlb.u16 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <4 x i16>, <4 x i16>* %offptr, align 2
+  %offs.sext = sext <4 x i16> %offs to <4 x i32>
+  %byte_ptrs = getelementptr inbounds i8, i8* %base, <4 x i32> %offs.sext
+  %ptrs = bitcast <4 x i8*> %byte_ptrs to <4 x i16*>
+  %gather = call <4 x i16> @llvm.masked.gather.v4i16.v4p0i16(<4 x i16*> %ptrs, i32 2, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i16> undef)
+  %gather.zext = zext <4 x i16> %gather to <4 x i32>
+  ret <4 x i32> %gather.zext
+}
+
+define arm_aapcs_vfpcc <4 x i32> @sext_signed_unscaled_i16_i16(i8* %base, <4 x i16>* %offptr) {
+; CHECK-LABEL: sext_signed_unscaled_i16_i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.s32 q0, [r1]
+; CHECK-NEXT:    vadd.i32 q0, q0, r0
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vmov r0, s1
+; CHECK-NEXT:    vmov r3, s2
+; CHECK-NEXT:    vmov r1, s3
+; CHECK-NEXT:    ldrh r2, [r2]
+; CHECK-NEXT:    ldrh r0, [r0]
+; CHECK-NEXT:    vmov.32 q0[0], r2
+; CHECK-NEXT:    ldrh r3, [r3]
+; CHECK-NEXT:    vmov.32 q0[1], r0
+; CHECK-NEXT:    ldrh r1, [r1]
+; CHECK-NEXT:    vmov.32 q0[2], r3
+; CHECK-NEXT:    vmov.32 q0[3], r1
+; CHECK-NEXT:    vmovlb.s16 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <4 x i16>, <4 x i16>* %offptr, align 2
+  %offs.sext = sext <4 x i16> %offs to <4 x i32>
+  %byte_ptrs = getelementptr inbounds i8, i8* %base, <4 x i32> %offs.sext
+  %ptrs = bitcast <4 x i8*> %byte_ptrs to <4 x i16*>
+  %gather = call <4 x i16> @llvm.masked.gather.v4i16.v4p0i16(<4 x i16*> %ptrs, i32 2, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i16> undef)
+  %gather.sext = sext <4 x i16> %gather to <4 x i32>
+  ret <4 x i32> %gather.sext
+}
+
+define arm_aapcs_vfpcc <4 x i32> @zext_unsigned_unscaled_i16_i16(i8* %base, <4 x i16>* %offptr) {
+; CHECK-LABEL: zext_unsigned_unscaled_i16_i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u32 q0, [r1]
+; CHECK-NEXT:    vadd.i32 q0, q0, r0
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vmov r0, s1
+; CHECK-NEXT:    vmov r3, s2
+; CHECK-NEXT:    vmov r1, s3
+; CHECK-NEXT:    ldrh r2, [r2]
+; CHECK-NEXT:    ldrh r0, [r0]
+; CHECK-NEXT:    vmov.32 q0[0], r2
+; CHECK-NEXT:    ldrh r3, [r3]
+; CHECK-NEXT:    vmov.32 q0[1], r0
+; CHECK-NEXT:    ldrh r1, [r1]
+; CHECK-NEXT:    vmov.32 q0[2], r3
+; CHECK-NEXT:    vmov.32 q0[3], r1
+; CHECK-NEXT:    vmovlb.u16 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <4 x i16>, <4 x i16>* %offptr, align 2
+  %offs.zext = zext <4 x i16> %offs to <4 x i32>
+  %byte_ptrs = getelementptr inbounds i8, i8* %base, <4 x i32> %offs.zext
+  %ptrs = bitcast <4 x i8*> %byte_ptrs to <4 x i16*>
+  %gather = call <4 x i16> @llvm.masked.gather.v4i16.v4p0i16(<4 x i16*> %ptrs, i32 2, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i16> undef)
+  %gather.zext = zext <4 x i16> %gather to <4 x i32>
+  ret <4 x i32> %gather.zext
+}
+
+define arm_aapcs_vfpcc <4 x i32> @sext_unsigned_unscaled_i16_i16(i8* %base, <4 x i16>* %offptr) {
+; CHECK-LABEL: sext_unsigned_unscaled_i16_i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u32 q0, [r1]
+; CHECK-NEXT:    vadd.i32 q0, q0, r0
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vmov r0, s1
+; CHECK-NEXT:    vmov r3, s2
+; CHECK-NEXT:    vmov r1, s3
+; CHECK-NEXT:    ldrh r2, [r2]
+; CHECK-NEXT:    ldrh r0, [r0]
+; CHECK-NEXT:    vmov.32 q0[0], r2
+; CHECK-NEXT:    ldrh r3, [r3]
+; CHECK-NEXT:    vmov.32 q0[1], r0
+; CHECK-NEXT:    ldrh r1, [r1]
+; CHECK-NEXT:    vmov.32 q0[2], r3
+; CHECK-NEXT:    vmov.32 q0[3], r1
+; CHECK-NEXT:    vmovlb.s16 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <4 x i16>, <4 x i16>* %offptr, align 2
+  %offs.zext = zext <4 x i16> %offs to <4 x i32>
+  %byte_ptrs = getelementptr inbounds i8, i8* %base, <4 x i32> %offs.zext
+  %ptrs = bitcast <4 x i8*> %byte_ptrs to <4 x i16*>
+  %gather = call <4 x i16> @llvm.masked.gather.v4i16.v4p0i16(<4 x i16*> %ptrs, i32 2, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i16> undef)
+  %gather.sext = sext <4 x i16> %gather to <4 x i32>
+  ret <4 x i32> %gather.sext
+}
+
+define arm_aapcs_vfpcc <4 x i32> @zext_signed_unscaled_i8_i16(i8* %base, <4 x i16>* %offptr) {
+; CHECK-LABEL: zext_signed_unscaled_i8_i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.s32 q0, [r1]
+; CHECK-NEXT:    vmov.i32 q1, #0xff
+; CHECK-NEXT:    vadd.i32 q0, q0, r0
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vmov r1, s1
+; CHECK-NEXT:    vmov r3, s2
+; CHECK-NEXT:    vmov r0, s3
+; CHECK-NEXT:    ldrb r2, [r2]
+; CHECK-NEXT:    ldrb r1, [r1]
+; CHECK-NEXT:    vmov.32 q0[0], r2
+; CHECK-NEXT:    ldrb r3, [r3]
+; CHECK-NEXT:    vmov.32 q0[1], r1
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.32 q0[2], r3
+; CHECK-NEXT:    vmov.32 q0[3], r0
+; CHECK-NEXT:    vand q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <4 x i16>, <4 x i16>* %offptr, align 2
+  %offs.sext = sext <4 x i16> %offs to <4 x i32>
+  %ptrs = getelementptr inbounds i8, i8* %base, <4 x i32> %offs.sext
+  %gather = call <4 x i8> @llvm.masked.gather.v4i8.v4p0i8(<4 x i8*> %ptrs, i32 1, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i8> undef)
+  %gather.zext = zext <4 x i8> %gather to <4 x i32>
+  ret <4 x i32> %gather.zext
+}
+
+define arm_aapcs_vfpcc <4 x i32> @sext_signed_unscaled_i8_i16(i8* %base, <4 x i16>* %offptr) {
+; CHECK-LABEL: sext_signed_unscaled_i8_i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.s32 q0, [r1]
+; CHECK-NEXT:    vadd.i32 q0, q0, r0
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vmov r3, s1
+; CHECK-NEXT:    vmov r1, s2
+; CHECK-NEXT:    vmov r2, s3
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    ldrb r3, [r3]
+; CHECK-NEXT:    vmov.32 q0[0], r0
+; CHECK-NEXT:    ldrb r1, [r1]
+; CHECK-NEXT:    vmov.32 q0[1], r3
+; CHECK-NEXT:    ldrb r2, [r2]
+; CHECK-NEXT:    vmov.32 q0[2], r1
+; CHECK-NEXT:    vmov.32 q0[3], r2
+; CHECK-NEXT:    vmovlb.s8 q0, q0
+; CHECK-NEXT:    vmovlb.s16 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <4 x i16>, <4 x i16>* %offptr, align 2
+  %offs.sext = sext <4 x i16> %offs to <4 x i32>
+  %ptrs = getelementptr inbounds i8, i8* %base, <4 x i32> %offs.sext
+  %gather = call <4 x i8> @llvm.masked.gather.v4i8.v4p0i8(<4 x i8*> %ptrs, i32 1, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i8> undef)
+  %gather.sext = sext <4 x i8> %gather to <4 x i32>
+  ret <4 x i32> %gather.sext
+}
+
+define arm_aapcs_vfpcc <4 x i32> @zext_unsigned_unscaled_i8_i16(i8* %base, <4 x i16>* %offptr) {
+; CHECK-LABEL: zext_unsigned_unscaled_i8_i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u32 q0, [r1]
+; CHECK-NEXT:    vmov.i32 q1, #0xff
+; CHECK-NEXT:    vadd.i32 q0, q0, r0
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vmov r1, s1
+; CHECK-NEXT:    vmov r3, s2
+; CHECK-NEXT:    vmov r0, s3
+; CHECK-NEXT:    ldrb r2, [r2]
+; CHECK-NEXT:    ldrb r1, [r1]
+; CHECK-NEXT:    vmov.32 q0[0], r2
+; CHECK-NEXT:    ldrb r3, [r3]
+; CHECK-NEXT:    vmov.32 q0[1], r1
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.32 q0[2], r3
+; CHECK-NEXT:    vmov.32 q0[3], r0
+; CHECK-NEXT:    vand q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <4 x i16>, <4 x i16>* %offptr, align 2
+  %offs.zext = zext <4 x i16> %offs to <4 x i32>
+  %ptrs = getelementptr inbounds i8, i8* %base, <4 x i32> %offs.zext
+  %gather = call <4 x i8> @llvm.masked.gather.v4i8.v4p0i8(<4 x i8*> %ptrs, i32 1, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i8> undef)
+  %gather.zext = zext <4 x i8> %gather to <4 x i32>
+  ret <4 x i32> %gather.zext
+}
+
+define arm_aapcs_vfpcc <4 x i32> @sext_unsigned_unscaled_i8_i16(i8* %base, <4 x i16>* %offptr) {
+; CHECK-LABEL: sext_unsigned_unscaled_i8_i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u32 q0, [r1]
+; CHECK-NEXT:    vadd.i32 q0, q0, r0
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vmov r3, s1
+; CHECK-NEXT:    vmov r1, s2
+; CHECK-NEXT:    vmov r2, s3
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    ldrb r3, [r3]
+; CHECK-NEXT:    vmov.32 q0[0], r0
+; CHECK-NEXT:    ldrb r1, [r1]
+; CHECK-NEXT:    vmov.32 q0[1], r3
+; CHECK-NEXT:    ldrb r2, [r2]
+; CHECK-NEXT:    vmov.32 q0[2], r1
+; CHECK-NEXT:    vmov.32 q0[3], r2
+; CHECK-NEXT:    vmovlb.s8 q0, q0
+; CHECK-NEXT:    vmovlb.s16 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <4 x i16>, <4 x i16>* %offptr, align 2
+  %offs.zext = zext <4 x i16> %offs to <4 x i32>
+  %ptrs = getelementptr inbounds i8, i8* %base, <4 x i32> %offs.zext
+  %gather = call <4 x i8> @llvm.masked.gather.v4i8.v4p0i8(<4 x i8*> %ptrs, i32 1, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i8> undef)
+  %gather.sext = sext <4 x i8> %gather to <4 x i32>
+  ret <4 x i32> %gather.sext
+}
+
+define arm_aapcs_vfpcc <4 x i32> @unsigned_unscaled_b_i32_i8(i8* %base, <4 x i8>* %offptr) {
+; CHECK-LABEL: unsigned_unscaled_b_i32_i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u32 q1, [r1]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, q1]
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <4 x i8>, <4 x i8>* %offptr, align 1
+  %offs.zext = zext <4 x i8> %offs to <4 x i32>
+  %byte_ptrs = getelementptr inbounds i8, i8* %base, <4 x i32> %offs.zext
+  %ptrs = bitcast <4 x i8*> %byte_ptrs to <4 x i32*>
+  %gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %ptrs, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
+  ret <4 x i32> %gather
+}
+
+define arm_aapcs_vfpcc <4 x i32> @signed_unscaled_i32_i8(i8* %base, <4 x i8>* %offptr) {
+; CHECK-LABEL: signed_unscaled_i32_i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.s32 q1, [r1]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, q1]
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <4 x i8>, <4 x i8>* %offptr, align 1
+  %offs.sext = sext <4 x i8> %offs to <4 x i32>
+  %byte_ptrs = getelementptr inbounds i8, i8* %base, <4 x i32> %offs.sext
+  %ptrs = bitcast <4 x i8*> %byte_ptrs to <4 x i32*>
+  %gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %ptrs, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
+  ret <4 x i32> %gather
+}
+
+define arm_aapcs_vfpcc <4 x float> @a_unsigned_unscaled_f32_i8(i8* %base, <4 x i8>* %offptr) {
+; CHECK-LABEL: a_unsigned_unscaled_f32_i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u32 q1, [r1]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, q1]
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <4 x i8>, <4 x i8>* %offptr, align 1
+  %offs.zext = zext <4 x i8> %offs to <4 x i32>
+  %byte_ptrs = getelementptr inbounds i8, i8* %base, <4 x i32> %offs.zext
+  %ptrs = bitcast <4 x i8*> %byte_ptrs to <4 x float*>
+  %gather = call <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*> %ptrs, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x float> undef)
+  ret <4 x float> %gather
+}
+
+define arm_aapcs_vfpcc <4 x float> @b_signed_unscaled_f32_i8(i8* %base, <4 x i8>* %offptr) {
+; CHECK-LABEL: b_signed_unscaled_f32_i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.s32 q1, [r1]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, q1]
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <4 x i8>, <4 x i8>* %offptr, align 1
+  %offs.sext = sext <4 x i8> %offs to <4 x i32>
+  %byte_ptrs = getelementptr inbounds i8, i8* %base, <4 x i32> %offs.sext
+  %ptrs = bitcast <4 x i8*> %byte_ptrs to <4 x float*>
+  %gather = call <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*> %ptrs, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x float> undef)
+  ret <4 x float> %gather
+}
+
+define arm_aapcs_vfpcc <4 x i32> @zext_signed_unscaled_i16_i8(i8* %base, <4 x i8>* %offptr) {
+; CHECK-LABEL: zext_signed_unscaled_i16_i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.s32 q0, [r1]
+; CHECK-NEXT:    vadd.i32 q0, q0, r0
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vmov r0, s1
+; CHECK-NEXT:    vmov r3, s2
+; CHECK-NEXT:    vmov r1, s3
+; CHECK-NEXT:    ldrh r2, [r2]
+; CHECK-NEXT:    ldrh r0, [r0]
+; CHECK-NEXT:    vmov.32 q0[0], r2
+; CHECK-NEXT:    ldrh r3, [r3]
+; CHECK-NEXT:    vmov.32 q0[1], r0
+; CHECK-NEXT:    ldrh r1, [r1]
+; CHECK-NEXT:    vmov.32 q0[2], r3
+; CHECK-NEXT:    vmov.32 q0[3], r1
+; CHECK-NEXT:    vmovlb.u16 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <4 x i8>, <4 x i8>* %offptr, align 1
+  %offs.sext = sext <4 x i8> %offs to <4 x i32>
+  %byte_ptrs = getelementptr inbounds i8, i8* %base, <4 x i32> %offs.sext
+  %ptrs = bitcast <4 x i8*> %byte_ptrs to <4 x i16*>
+  %gather = call <4 x i16> @llvm.masked.gather.v4i16.v4p0i16(<4 x i16*> %ptrs, i32 2, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i16> undef)
+  %gather.zext = zext <4 x i16> %gather to <4 x i32>
+  ret <4 x i32> %gather.zext
+}
+
+define arm_aapcs_vfpcc <4 x i32> @sext_signed_unscaled_i16_i8(i8* %base, <4 x i8>* %offptr) {
+; CHECK-LABEL: sext_signed_unscaled_i16_i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.s32 q0, [r1]
+; CHECK-NEXT:    vadd.i32 q0, q0, r0
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vmov r0, s1
+; CHECK-NEXT:    vmov r3, s2
+; CHECK-NEXT:    vmov r1, s3
+; CHECK-NEXT:    ldrh r2, [r2]
+; CHECK-NEXT:    ldrh r0, [r0]
+; CHECK-NEXT:    vmov.32 q0[0], r2
+; CHECK-NEXT:    ldrh r3, [r3]
+; CHECK-NEXT:    vmov.32 q0[1], r0
+; CHECK-NEXT:    ldrh r1, [r1]
+; CHECK-NEXT:    vmov.32 q0[2], r3
+; CHECK-NEXT:    vmov.32 q0[3], r1
+; CHECK-NEXT:    vmovlb.s16 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <4 x i8>, <4 x i8>* %offptr, align 1
+  %offs.sext = sext <4 x i8> %offs to <4 x i32>
+  %byte_ptrs = getelementptr inbounds i8, i8* %base, <4 x i32> %offs.sext
+  %ptrs = bitcast <4 x i8*> %byte_ptrs to <4 x i16*>
+  %gather = call <4 x i16> @llvm.masked.gather.v4i16.v4p0i16(<4 x i16*> %ptrs, i32 2, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i16> undef)
+  %gather.sext = sext <4 x i16> %gather to <4 x i32>
+  ret <4 x i32> %gather.sext
+}
+
+define arm_aapcs_vfpcc <4 x i32> @zext_unsigned_unscaled_i16_i8(i8* %base, <4 x i8>* %offptr) {
+; CHECK-LABEL: zext_unsigned_unscaled_i16_i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u32 q0, [r1]
+; CHECK-NEXT:    vadd.i32 q0, q0, r0
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vmov r0, s1
+; CHECK-NEXT:    vmov r3, s2
+; CHECK-NEXT:    vmov r1, s3
+; CHECK-NEXT:    ldrh r2, [r2]
+; CHECK-NEXT:    ldrh r0, [r0]
+; CHECK-NEXT:    vmov.32 q0[0], r2
+; CHECK-NEXT:    ldrh r3, [r3]
+; CHECK-NEXT:    vmov.32 q0[1], r0
+; CHECK-NEXT:    ldrh r1, [r1]
+; CHECK-NEXT:    vmov.32 q0[2], r3
+; CHECK-NEXT:    vmov.32 q0[3], r1
+; CHECK-NEXT:    vmovlb.u16 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <4 x i8>, <4 x i8>* %offptr, align 1
+  %offs.zext = zext <4 x i8> %offs to <4 x i32>
+  %byte_ptrs = getelementptr inbounds i8, i8* %base, <4 x i32> %offs.zext
+  %ptrs = bitcast <4 x i8*> %byte_ptrs to <4 x i16*>
+  %gather = call <4 x i16> @llvm.masked.gather.v4i16.v4p0i16(<4 x i16*> %ptrs, i32 2, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i16> undef)
+  %gather.zext = zext <4 x i16> %gather to <4 x i32>
+  ret <4 x i32> %gather.zext
+}
+
+define arm_aapcs_vfpcc <4 x i32> @sext_unsigned_unscaled_i16_i8(i8* %base, <4 x i8>* %offptr) {
+; CHECK-LABEL: sext_unsigned_unscaled_i16_i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u32 q0, [r1]
+; CHECK-NEXT:    vadd.i32 q0, q0, r0
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vmov r0, s1
+; CHECK-NEXT:    vmov r3, s2
+; CHECK-NEXT:    vmov r1, s3
+; CHECK-NEXT:    ldrh r2, [r2]
+; CHECK-NEXT:    ldrh r0, [r0]
+; CHECK-NEXT:    vmov.32 q0[0], r2
+; CHECK-NEXT:    ldrh r3, [r3]
+; CHECK-NEXT:    vmov.32 q0[1], r0
+; CHECK-NEXT:    ldrh r1, [r1]
+; CHECK-NEXT:    vmov.32 q0[2], r3
+; CHECK-NEXT:    vmov.32 q0[3], r1
+; CHECK-NEXT:    vmovlb.s16 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <4 x i8>, <4 x i8>* %offptr, align 1
+  %offs.zext = zext <4 x i8> %offs to <4 x i32>
+  %byte_ptrs = getelementptr inbounds i8, i8* %base, <4 x i32> %offs.zext
+  %ptrs = bitcast <4 x i8*> %byte_ptrs to <4 x i16*>
+  %gather = call <4 x i16> @llvm.masked.gather.v4i16.v4p0i16(<4 x i16*> %ptrs, i32 2, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i16> undef)
+  %gather.sext = sext <4 x i16> %gather to <4 x i32>
+  ret <4 x i32> %gather.sext
+}
+
+define arm_aapcs_vfpcc <4 x i32> @zext_signed_unscaled_i8_i8(i8* %base, <4 x i8>* %offptr) {
+; CHECK-LABEL: zext_signed_unscaled_i8_i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.s32 q0, [r1]
+; CHECK-NEXT:    vmov.i32 q1, #0xff
+; CHECK-NEXT:    vadd.i32 q0, q0, r0
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vmov r1, s1
+; CHECK-NEXT:    vmov r3, s2
+; CHECK-NEXT:    vmov r0, s3
+; CHECK-NEXT:    ldrb r2, [r2]
+; CHECK-NEXT:    ldrb r1, [r1]
+; CHECK-NEXT:    vmov.32 q0[0], r2
+; CHECK-NEXT:    ldrb r3, [r3]
+; CHECK-NEXT:    vmov.32 q0[1], r1
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.32 q0[2], r3
+; CHECK-NEXT:    vmov.32 q0[3], r0
+; CHECK-NEXT:    vand q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <4 x i8>, <4 x i8>* %offptr, align 1
+  %offs.sext = sext <4 x i8> %offs to <4 x i32>
+  %ptrs = getelementptr inbounds i8, i8* %base, <4 x i32> %offs.sext
+  %gather = call <4 x i8> @llvm.masked.gather.v4i8.v4p0i8(<4 x i8*> %ptrs, i32 1, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i8> undef)
+  %gather.zext = zext <4 x i8> %gather to <4 x i32>
+  ret <4 x i32> %gather.zext
+}
+
+define arm_aapcs_vfpcc <4 x i32> @sext_signed_unscaled_i8_i8(i8* %base, <4 x i8>* %offptr) {
+; CHECK-LABEL: sext_signed_unscaled_i8_i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.s32 q0, [r1]
+; CHECK-NEXT:    vadd.i32 q0, q0, r0
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vmov r3, s1
+; CHECK-NEXT:    vmov r1, s2
+; CHECK-NEXT:    vmov r2, s3
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    ldrb r3, [r3]
+; CHECK-NEXT:    vmov.32 q0[0], r0
+; CHECK-NEXT:    ldrb r1, [r1]
+; CHECK-NEXT:    vmov.32 q0[1], r3
+; CHECK-NEXT:    ldrb r2, [r2]
+; CHECK-NEXT:    vmov.32 q0[2], r1
+; CHECK-NEXT:    vmov.32 q0[3], r2
+; CHECK-NEXT:    vmovlb.s8 q0, q0
+; CHECK-NEXT:    vmovlb.s16 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <4 x i8>, <4 x i8>* %offptr, align 1
+  %offs.sext = sext <4 x i8> %offs to <4 x i32>
+  %ptrs = getelementptr inbounds i8, i8* %base, <4 x i32> %offs.sext
+  %gather = call <4 x i8> @llvm.masked.gather.v4i8.v4p0i8(<4 x i8*> %ptrs, i32 1, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i8> undef)
+  %gather.sext = sext <4 x i8> %gather to <4 x i32>
+  ret <4 x i32> %gather.sext
+}
+
+define arm_aapcs_vfpcc <4 x i32> @zext_unsigned_unscaled_i8_i8(i8* %base, <4 x i8>* %offptr) {
+; CHECK-LABEL: zext_unsigned_unscaled_i8_i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u32 q0, [r1]
+; CHECK-NEXT:    vmov.i32 q1, #0xff
+; CHECK-NEXT:    vadd.i32 q0, q0, r0
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vmov r1, s1
+; CHECK-NEXT:    vmov r3, s2
+; CHECK-NEXT:    vmov r0, s3
+; CHECK-NEXT:    ldrb r2, [r2]
+; CHECK-NEXT:    ldrb r1, [r1]
+; CHECK-NEXT:    vmov.32 q0[0], r2
+; CHECK-NEXT:    ldrb r3, [r3]
+; CHECK-NEXT:    vmov.32 q0[1], r1
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.32 q0[2], r3
+; CHECK-NEXT:    vmov.32 q0[3], r0
+; CHECK-NEXT:    vand q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <4 x i8>, <4 x i8>* %offptr, align 1
+  %offs.zext = zext <4 x i8> %offs to <4 x i32>
+  %ptrs = getelementptr inbounds i8, i8* %base, <4 x i32> %offs.zext
+  %gather = call <4 x i8> @llvm.masked.gather.v4i8.v4p0i8(<4 x i8*> %ptrs, i32 1, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i8> undef)
+  %gather.zext = zext <4 x i8> %gather to <4 x i32>
+  ret <4 x i32> %gather.zext
+}
+
+define arm_aapcs_vfpcc <4 x i32> @sext_unsigned_unscaled_i8_i8(i8* %base, <4 x i8>* %offptr) {
+; CHECK-LABEL: sext_unsigned_unscaled_i8_i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u32 q0, [r1]
+; CHECK-NEXT:    vadd.i32 q0, q0, r0
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vmov r3, s1
+; CHECK-NEXT:    vmov r1, s2
+; CHECK-NEXT:    vmov r2, s3
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    ldrb r3, [r3]
+; CHECK-NEXT:    vmov.32 q0[0], r0
+; CHECK-NEXT:    ldrb r1, [r1]
+; CHECK-NEXT:    vmov.32 q0[1], r3
+; CHECK-NEXT:    ldrb r2, [r2]
+; CHECK-NEXT:    vmov.32 q0[2], r1
+; CHECK-NEXT:    vmov.32 q0[3], r2
+; CHECK-NEXT:    vmovlb.s8 q0, q0
+; CHECK-NEXT:    vmovlb.s16 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <4 x i8>, <4 x i8>* %offptr, align 1
+  %offs.zext = zext <4 x i8> %offs to <4 x i32>
+  %ptrs = getelementptr inbounds i8, i8* %base, <4 x i32> %offs.zext
+  %gather = call <4 x i8> @llvm.masked.gather.v4i8.v4p0i8(<4 x i8*> %ptrs, i32 1, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i8> undef)
+  %gather.sext = sext <4 x i8> %gather to <4 x i32>
+  ret <4 x i32> %gather.sext
+}
+
+; VLDRW.u32 Qd, [P, 4]
+define arm_aapcs_vfpcc <4 x i32> @qi4(<4 x i32*> %p) {
+; CHECK-LABEL: qi4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.i32 q1, #0x10
+; CHECK-NEXT:    vadd.i32 q0, q0, q1
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vmov r3, s1
+; CHECK-NEXT:    vmov r1, s2
+; CHECK-NEXT:    vmov r2, s3
+; CHECK-NEXT:    ldr r0, [r0]
+; CHECK-NEXT:    ldr r3, [r3]
+; CHECK-NEXT:    vmov.32 q0[0], r0
+; CHECK-NEXT:    ldr r1, [r1]
+; CHECK-NEXT:    vmov.32 q0[1], r3
+; CHECK-NEXT:    ldr r2, [r2]
+; CHECK-NEXT:    vmov.32 q0[2], r1
+; CHECK-NEXT:    vmov.32 q0[3], r2
+; CHECK-NEXT:    bx lr
+entry:
+  %g = getelementptr inbounds i32, <4 x i32*> %p, i32 4
+  %gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %g, i32 1, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
+  ret <4 x i32> %gather
+}
+
+declare <4 x i8>  @llvm.masked.gather.v4i8.v4p0i8(<4 x i8*>, i32, <4 x i1>, <4 x i8>)
+declare <4 x i16> @llvm.masked.gather.v4i16.v4p0i16(<4 x i16*>, i32, <4 x i1>, <4 x i16>)
+declare <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*>, i32, <4 x i1>, <4 x i32>)
+declare <4 x half> @llvm.masked.gather.v4f16.v4p0f16(<4 x half*>, i32, <4 x i1>, <4 x half>)
+declare <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*>, i32, <4 x i1>, <4 x float>)
diff --git a/llvm/test/CodeGen/Thumb2/mve-gather-ind8-unscaled.ll b/llvm/test/CodeGen/Thumb2/mve-gather-ind8-unscaled.ll
new file mode 100644
index 000000000..deb3be28d
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-gather-ind8-unscaled.ll
@@ -0,0 +1,370 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve.fp -enable-arm-maskedgatscat %s -o - | FileCheck %s
+
+define arm_aapcs_vfpcc <16 x i8> @unscaled_v16i8_i8(i8* %base, <16 x i8>* %offptr) {
+; CHECK-LABEL: unscaled_v16i8_i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u8 q1, [r1]
+; CHECK-NEXT:    vldrb.u8 q0, [r0, q1]
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <16 x i8>, <16 x i8>* %offptr, align 1
+  %offs.zext = zext <16 x i8> %offs to <16 x i32>
+  %ptrs = getelementptr inbounds i8, i8* %base, <16 x i32> %offs.zext
+  %gather = call <16 x i8> @llvm.masked.gather.v16i8.v16p0i8(<16 x i8*> %ptrs, i32 1, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i8> undef)
+  ret <16 x i8> %gather
+}
+
+define arm_aapcs_vfpcc <8 x i8> @unscaled_v8i8_i8(i8* %base, <8 x i8>* %offptr) {
+; CHECK-LABEL: unscaled_v8i8_i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, r5, r7, lr}
+; CHECK-NEXT:    push {r4, r5, r7, lr}
+; CHECK-NEXT:    vldrb.u32 q0, [r1]
+; CHECK-NEXT:    vldrb.u32 q1, [r1, #4]
+; CHECK-NEXT:    vadd.i32 q0, q0, r0
+; CHECK-NEXT:    vadd.i32 q1, q1, r0
+; CHECK-NEXT:    vmov r2, s2
+; CHECK-NEXT:    vmov r3, s3
+; CHECK-NEXT:    vmov r5, s1
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmov r1, s5
+; CHECK-NEXT:    vmov r4, s7
+; CHECK-NEXT:    ldrb.w r12, [r2]
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    ldrb.w lr, [r3]
+; CHECK-NEXT:    vmov r3, s6
+; CHECK-NEXT:    ldrb r5, [r5]
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    ldrb r1, [r1]
+; CHECK-NEXT:    ldrb r4, [r4]
+; CHECK-NEXT:    ldrb r2, [r2]
+; CHECK-NEXT:    ldrb r3, [r3]
+; CHECK-NEXT:    vmov.16 q0[0], r2
+; CHECK-NEXT:    vmov.16 q0[1], r5
+; CHECK-NEXT:    vmov.16 q0[2], r12
+; CHECK-NEXT:    vmov.16 q0[3], lr
+; CHECK-NEXT:    vmov.16 q0[4], r0
+; CHECK-NEXT:    vmov.16 q0[5], r1
+; CHECK-NEXT:    vmov.16 q0[6], r3
+; CHECK-NEXT:    vmov.16 q0[7], r4
+; CHECK-NEXT:    pop {r4, r5, r7, pc}
+entry:
+  %offs = load <8 x i8>, <8 x i8>* %offptr, align 1
+  %offs.zext = zext <8 x i8> %offs to <8 x i32>
+  %ptrs = getelementptr inbounds i8, i8* %base, <8 x i32> %offs.zext
+  %gather = call <8 x i8> @llvm.masked.gather.v8i8.v8p0i8(<8 x i8*> %ptrs, i32 1, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i8> undef)
+  ret <8 x i8> %gather
+}
+
+define arm_aapcs_vfpcc <2 x i8> @unscaled_v2i8_i8(i8* %base, <2 x i8>* %offptr) {
+; CHECK-LABEL: unscaled_v2i8_i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    ldrb r2, [r1]
+; CHECK-NEXT:    vmov.i32 q0, #0xff
+; CHECK-NEXT:    ldrb r1, [r1, #1]
+; CHECK-NEXT:    vmov.32 q1[0], r2
+; CHECK-NEXT:    vmov.32 q1[2], r1
+; CHECK-NEXT:    vand q0, q1, q0
+; CHECK-NEXT:    vmov r1, s0
+; CHECK-NEXT:    vmov r2, s2
+; CHECK-NEXT:    ldrb r1, [r0, r1]
+; CHECK-NEXT:    ldrb r0, [r0, r2]
+; CHECK-NEXT:    vmov.32 q0[0], r1
+; CHECK-NEXT:    vmov.32 q0[2], r0
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <2 x i8>, <2 x i8>* %offptr, align 1
+  %offs.zext = zext <2 x i8> %offs to <2 x i32>
+  %ptrs = getelementptr inbounds i8, i8* %base, <2 x i32> %offs.zext
+  %gather = call <2 x i8> @llvm.masked.gather.v2i8.v2p0i8(<2 x i8*> %ptrs, i32 1, <2 x i1> <i1 true, i1 true>, <2 x i8> undef)
+  ret <2 x i8> %gather
+}
+
+define arm_aapcs_vfpcc <16 x i8> @unscaled_v16i8_sext(i8* %base, <16 x i8>* %offptr) {
+; CHECK-LABEL: unscaled_v16i8_sext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, r5, r6, lr}
+; CHECK-NEXT:    push {r4, r5, r6, lr}
+; CHECK-NEXT:    vldrb.s32 q0, [r1, #8]
+; CHECK-NEXT:    vldrb.s32 q2, [r1, #4]
+; CHECK-NEXT:    vadd.i32 q1, q0, r0
+; CHECK-NEXT:    vldrb.s32 q0, [r1, #12]
+; CHECK-NEXT:    vmov r2, s6
+; CHECK-NEXT:    vadd.i32 q2, q2, r0
+; CHECK-NEXT:    vadd.i32 q0, q0, r0
+; CHECK-NEXT:    vmov r6, s4
+; CHECK-NEXT:    vmov r3, s2
+; CHECK-NEXT:    vmov r4, s3
+; CHECK-NEXT:    vmov r5, s7
+; CHECK-NEXT:    ldrb.w r12, [r2]
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    ldrb r6, [r6]
+; CHECK-NEXT:    ldrb r3, [r3]
+; CHECK-NEXT:    ldrb r4, [r4]
+; CHECK-NEXT:    ldrb r5, [r5]
+; CHECK-NEXT:    ldrb.w lr, [r2]
+; CHECK-NEXT:    vmov r2, s1
+; CHECK-NEXT:    vldrb.s32 q0, [r1]
+; CHECK-NEXT:    vadd.i32 q3, q0, r0
+; CHECK-NEXT:    vmov r0, s12
+; CHECK-NEXT:    ldrb r2, [r2]
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[0], r0
+; CHECK-NEXT:    vmov r0, s13
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[1], r0
+; CHECK-NEXT:    vmov r0, s14
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[2], r0
+; CHECK-NEXT:    vmov r0, s15
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[3], r0
+; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[4], r0
+; CHECK-NEXT:    vmov r0, s9
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[5], r0
+; CHECK-NEXT:    vmov r0, s10
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[6], r0
+; CHECK-NEXT:    vmov r0, s11
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[7], r0
+; CHECK-NEXT:    vmov r0, s5
+; CHECK-NEXT:    vmov.8 q0[8], r6
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[9], r0
+; CHECK-NEXT:    vmov.8 q0[10], r12
+; CHECK-NEXT:    vmov.8 q0[11], r5
+; CHECK-NEXT:    vmov.8 q0[12], lr
+; CHECK-NEXT:    vmov.8 q0[13], r2
+; CHECK-NEXT:    vmov.8 q0[14], r3
+; CHECK-NEXT:    vmov.8 q0[15], r4
+; CHECK-NEXT:    pop {r4, r5, r6, pc}
+entry:
+  %offs = load <16 x i8>, <16 x i8>* %offptr, align 1
+  %offs.sext = sext <16 x i8> %offs to <16 x i32>
+  %ptrs = getelementptr inbounds i8, i8* %base, <16 x i32> %offs.sext
+  %gather = call <16 x i8> @llvm.masked.gather.v16i8.v16p0i8(<16 x i8*> %ptrs, i32 1, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i8> undef)
+  ret <16 x i8> %gather
+}
+
+define arm_aapcs_vfpcc <16 x i8> @unscaled_v16i8_i16(i8* %base, <16 x i16>* %offptr) {
+; CHECK-LABEL: unscaled_v16i8_i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, r5, r6, lr}
+; CHECK-NEXT:    push {r4, r5, r6, lr}
+; CHECK-NEXT:    vldrh.s32 q0, [r1, #16]
+; CHECK-NEXT:    vldrh.s32 q2, [r1, #8]
+; CHECK-NEXT:    vadd.i32 q1, q0, r0
+; CHECK-NEXT:    vldrh.s32 q0, [r1, #24]
+; CHECK-NEXT:    vmov r2, s6
+; CHECK-NEXT:    vadd.i32 q2, q2, r0
+; CHECK-NEXT:    vadd.i32 q0, q0, r0
+; CHECK-NEXT:    vmov r6, s4
+; CHECK-NEXT:    vmov r3, s2
+; CHECK-NEXT:    vmov r4, s3
+; CHECK-NEXT:    vmov r5, s7
+; CHECK-NEXT:    ldrb.w r12, [r2]
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    ldrb r6, [r6]
+; CHECK-NEXT:    ldrb r3, [r3]
+; CHECK-NEXT:    ldrb r4, [r4]
+; CHECK-NEXT:    ldrb r5, [r5]
+; CHECK-NEXT:    ldrb.w lr, [r2]
+; CHECK-NEXT:    vmov r2, s1
+; CHECK-NEXT:    vldrh.s32 q0, [r1]
+; CHECK-NEXT:    vadd.i32 q3, q0, r0
+; CHECK-NEXT:    vmov r0, s12
+; CHECK-NEXT:    ldrb r2, [r2]
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[0], r0
+; CHECK-NEXT:    vmov r0, s13
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[1], r0
+; CHECK-NEXT:    vmov r0, s14
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[2], r0
+; CHECK-NEXT:    vmov r0, s15
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[3], r0
+; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[4], r0
+; CHECK-NEXT:    vmov r0, s9
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[5], r0
+; CHECK-NEXT:    vmov r0, s10
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[6], r0
+; CHECK-NEXT:    vmov r0, s11
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[7], r0
+; CHECK-NEXT:    vmov r0, s5
+; CHECK-NEXT:    vmov.8 q0[8], r6
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[9], r0
+; CHECK-NEXT:    vmov.8 q0[10], r12
+; CHECK-NEXT:    vmov.8 q0[11], r5
+; CHECK-NEXT:    vmov.8 q0[12], lr
+; CHECK-NEXT:    vmov.8 q0[13], r2
+; CHECK-NEXT:    vmov.8 q0[14], r3
+; CHECK-NEXT:    vmov.8 q0[15], r4
+; CHECK-NEXT:    pop {r4, r5, r6, pc}
+entry:
+  %offs = load <16 x i16>, <16 x i16>* %offptr, align 2
+  %offs.sext = sext <16 x i16> %offs to <16 x i32>
+  %ptrs = getelementptr inbounds i8, i8* %base, <16 x i32> %offs.sext
+  %gather = call <16 x i8> @llvm.masked.gather.v16i8.v16p0i8(<16 x i8*> %ptrs, i32 1, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i8> undef)
+  ret <16 x i8> %gather
+}
+
+define arm_aapcs_vfpcc <16 x i8> @unscaled_v16i8_scaled(i32* %base, <16 x i8>* %offptr) {
+; CHECK-LABEL: unscaled_v16i8_scaled:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, r5, r6, lr}
+; CHECK-NEXT:    push {r4, r5, r6, lr}
+; CHECK-NEXT:    vldrb.u32 q0, [r1, #8]
+; CHECK-NEXT:    vldrb.u32 q2, [r1, #4]
+; CHECK-NEXT:    vshl.i32 q0, q0, #2
+; CHECK-NEXT:    vshl.i32 q2, q2, #2
+; CHECK-NEXT:    vadd.i32 q1, q0, r0
+; CHECK-NEXT:    vldrb.u32 q0, [r1, #12]
+; CHECK-NEXT:    vmov r2, s6
+; CHECK-NEXT:    vadd.i32 q2, q2, r0
+; CHECK-NEXT:    vshl.i32 q0, q0, #2
+; CHECK-NEXT:    vmov r6, s4
+; CHECK-NEXT:    vadd.i32 q0, q0, r0
+; CHECK-NEXT:    vmov r5, s7
+; CHECK-NEXT:    vmov r3, s2
+; CHECK-NEXT:    vmov r4, s3
+; CHECK-NEXT:    ldrb.w r12, [r2]
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    ldrb r6, [r6]
+; CHECK-NEXT:    ldrb r5, [r5]
+; CHECK-NEXT:    ldrb r3, [r3]
+; CHECK-NEXT:    ldrb r4, [r4]
+; CHECK-NEXT:    ldrb.w lr, [r2]
+; CHECK-NEXT:    vmov r2, s1
+; CHECK-NEXT:    vldrb.u32 q0, [r1]
+; CHECK-NEXT:    vshl.i32 q0, q0, #2
+; CHECK-NEXT:    vadd.i32 q3, q0, r0
+; CHECK-NEXT:    vmov r0, s12
+; CHECK-NEXT:    ldrb r2, [r2]
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[0], r0
+; CHECK-NEXT:    vmov r0, s13
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[1], r0
+; CHECK-NEXT:    vmov r0, s14
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[2], r0
+; CHECK-NEXT:    vmov r0, s15
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[3], r0
+; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[4], r0
+; CHECK-NEXT:    vmov r0, s9
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[5], r0
+; CHECK-NEXT:    vmov r0, s10
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[6], r0
+; CHECK-NEXT:    vmov r0, s11
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[7], r0
+; CHECK-NEXT:    vmov r0, s5
+; CHECK-NEXT:    vmov.8 q0[8], r6
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[9], r0
+; CHECK-NEXT:    vmov.8 q0[10], r12
+; CHECK-NEXT:    vmov.8 q0[11], r5
+; CHECK-NEXT:    vmov.8 q0[12], lr
+; CHECK-NEXT:    vmov.8 q0[13], r2
+; CHECK-NEXT:    vmov.8 q0[14], r3
+; CHECK-NEXT:    vmov.8 q0[15], r4
+; CHECK-NEXT:    pop {r4, r5, r6, pc}
+entry:
+  %offs = load <16 x i8>, <16 x i8>* %offptr, align 4
+  %offs.zext = zext <16 x i8> %offs to <16 x i32>
+  %ptrs32 = getelementptr inbounds i32, i32* %base, <16 x i32> %offs.zext
+  %ptrs = bitcast <16 x i32*> %ptrs32 to <16 x i8*>
+  %gather = call <16 x i8> @llvm.masked.gather.v16i8.v16p0i8(<16 x i8*> %ptrs, i32 1, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i8> undef)
+  ret <16 x i8> %gather
+}
+
+define arm_aapcs_vfpcc <16 x i8> @unscaled_v16i8_i8_next(i8* %base, <16 x i32>* %offptr) {
+; CHECK-LABEL: unscaled_v16i8_i8_next:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, r5, r6, lr}
+; CHECK-NEXT:    push {r4, r5, r6, lr}
+; CHECK-NEXT:    vldrw.u32 q0, [r1, #32]
+; CHECK-NEXT:    vldrw.u32 q2, [r1, #16]
+; CHECK-NEXT:    vadd.i32 q1, q0, r0
+; CHECK-NEXT:    vldrw.u32 q0, [r1, #48]
+; CHECK-NEXT:    vmov r2, s6
+; CHECK-NEXT:    vadd.i32 q2, q2, r0
+; CHECK-NEXT:    vadd.i32 q0, q0, r0
+; CHECK-NEXT:    vmov r6, s4
+; CHECK-NEXT:    vmov r3, s2
+; CHECK-NEXT:    vmov r4, s3
+; CHECK-NEXT:    vmov r5, s7
+; CHECK-NEXT:    ldrb.w r12, [r2]
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    ldrb r6, [r6]
+; CHECK-NEXT:    ldrb r3, [r3]
+; CHECK-NEXT:    ldrb r4, [r4]
+; CHECK-NEXT:    ldrb r5, [r5]
+; CHECK-NEXT:    ldrb.w lr, [r2]
+; CHECK-NEXT:    vmov r2, s1
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    vadd.i32 q3, q0, r0
+; CHECK-NEXT:    vmov r0, s12
+; CHECK-NEXT:    ldrb r2, [r2]
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[0], r0
+; CHECK-NEXT:    vmov r0, s13
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[1], r0
+; CHECK-NEXT:    vmov r0, s14
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[2], r0
+; CHECK-NEXT:    vmov r0, s15
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[3], r0
+; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[4], r0
+; CHECK-NEXT:    vmov r0, s9
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[5], r0
+; CHECK-NEXT:    vmov r0, s10
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[6], r0
+; CHECK-NEXT:    vmov r0, s11
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[7], r0
+; CHECK-NEXT:    vmov r0, s5
+; CHECK-NEXT:    vmov.8 q0[8], r6
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[9], r0
+; CHECK-NEXT:    vmov.8 q0[10], r12
+; CHECK-NEXT:    vmov.8 q0[11], r5
+; CHECK-NEXT:    vmov.8 q0[12], lr
+; CHECK-NEXT:    vmov.8 q0[13], r2
+; CHECK-NEXT:    vmov.8 q0[14], r3
+; CHECK-NEXT:    vmov.8 q0[15], r4
+; CHECK-NEXT:    pop {r4, r5, r6, pc}
+entry:
+  %offs = load <16 x i32>, <16 x i32>* %offptr, align 4
+  %ptrs = getelementptr inbounds i8, i8* %base, <16 x i32> %offs
+  %gather = call <16 x i8> @llvm.masked.gather.v16i8.v16p0i8(<16 x i8*> %ptrs, i32 1, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i8> undef)
+  ret <16 x i8> %gather
+}
+
+declare <16 x i8> @llvm.masked.gather.v16i8.v16p0i8(<16 x i8*>, i32, <16 x i1>, <16 x i8>)
+declare <8 x i8> @llvm.masked.gather.v8i8.v8p0i8(<8 x i8*>, i32, <8 x i1>, <8 x i8>)
+declare <2 x i8> @llvm.masked.gather.v2i8.v2p0i8(<2 x i8*>, i32, <2 x i1>, <2 x i8>)
diff --git a/llvm/test/CodeGen/Thumb2/mve-gather-ptrs.ll b/llvm/test/CodeGen/Thumb2/mve-gather-ptrs.ll
new file mode 100644
index 000000000..75f18ea8f
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-gather-ptrs.ll
@@ -0,0 +1,863 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve.fp -enable-arm-maskedldst -enable-arm-maskedgatscat %s -o - | FileCheck %s
+
+; i32
+
+define arm_aapcs_vfpcc <2 x i32> @ptr_v2i32(<2 x i32*>* %offptr) {
+; CHECK-LABEL: ptr_v2i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    ldrd r1, r0, [r0]
+; CHECK-NEXT:    ldr r0, [r0]
+; CHECK-NEXT:    ldr r1, [r1]
+; CHECK-NEXT:    vmov.32 q0[0], r1
+; CHECK-NEXT:    vmov.32 q0[2], r0
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <2 x i32*>, <2 x i32*>* %offptr, align 4
+  %gather = call <2 x i32> @llvm.masked.gather.v2i32.v2p0i32(<2 x i32*> %offs, i32 4, <2 x i1> <i1 true, i1 true>, <2 x i32> undef)
+  ret <2 x i32> %gather
+}
+
+define arm_aapcs_vfpcc <4 x i32> @ptr_v4i32(<4 x i32*>* %offptr) {
+; CHECK-LABEL: ptr_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vldrw.u32 q0, [q1]
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <4 x i32*>, <4 x i32*>* %offptr, align 4
+  %gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %offs, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
+  ret <4 x i32> %gather
+}
+
+define arm_aapcs_vfpcc <8 x i32> @ptr_v8i32(<8 x i32*>* %offptr) {
+; CHECK-LABEL: ptr_v8i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, r5, r7, lr}
+; CHECK-NEXT:    push {r4, r5, r7, lr}
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #16]
+; CHECK-NEXT:    vmov r1, s2
+; CHECK-NEXT:    vmov r2, s3
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmov r5, s1
+; CHECK-NEXT:    vmov r4, s7
+; CHECK-NEXT:    ldr.w r12, [r1]
+; CHECK-NEXT:    vmov r1, s5
+; CHECK-NEXT:    ldr.w lr, [r2]
+; CHECK-NEXT:    vmov r2, s6
+; CHECK-NEXT:    ldr r3, [r3]
+; CHECK-NEXT:    ldr r0, [r0]
+; CHECK-NEXT:    ldr r5, [r5]
+; CHECK-NEXT:    vmov.32 q0[0], r3
+; CHECK-NEXT:    vmov.32 q1[0], r0
+; CHECK-NEXT:    ldr r4, [r4]
+; CHECK-NEXT:    vmov.32 q0[1], r5
+; CHECK-NEXT:    vmov.32 q0[2], r12
+; CHECK-NEXT:    vmov.32 q0[3], lr
+; CHECK-NEXT:    ldr r1, [r1]
+; CHECK-NEXT:    ldr r2, [r2]
+; CHECK-NEXT:    vmov.32 q1[1], r1
+; CHECK-NEXT:    vmov.32 q1[2], r2
+; CHECK-NEXT:    vmov.32 q1[3], r4
+; CHECK-NEXT:    pop {r4, r5, r7, pc}
+entry:
+  %offs = load <8 x i32*>, <8 x i32*>* %offptr, align 4
+  %gather = call <8 x i32> @llvm.masked.gather.v8i32.v8p0i32(<8 x i32*> %offs, i32 4, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i32> undef)
+  ret <8 x i32> %gather
+}
+
+define arm_aapcs_vfpcc <16 x i32> @ptr_v16i32(<16 x i32*>* %offptr) {
+; CHECK-LABEL: ptr_v16i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, r5, r6, lr}
+; CHECK-NEXT:    push {r4, r5, r6, lr}
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #32]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #48]
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vldrw.u32 q3, [r0, #16]
+; CHECK-NEXT:    vmov r1, s10
+; CHECK-NEXT:    vmov r5, s4
+; CHECK-NEXT:    vmov r2, s3
+; CHECK-NEXT:    vmov r0, s12
+; CHECK-NEXT:    vmov r6, s7
+; CHECK-NEXT:    vmov r4, s11
+; CHECK-NEXT:    ldr.w r12, [r1]
+; CHECK-NEXT:    vmov r1, s0
+; CHECK-NEXT:    ldr r5, [r5]
+; CHECK-NEXT:    ldr r2, [r2]
+; CHECK-NEXT:    ldr r0, [r0]
+; CHECK-NEXT:    ldr r6, [r6]
+; CHECK-NEXT:    ldr r4, [r4]
+; CHECK-NEXT:    ldr.w lr, [r1]
+; CHECK-NEXT:    vmov r1, s1
+; CHECK-NEXT:    ldr r3, [r1]
+; CHECK-NEXT:    vmov r1, s2
+; CHECK-NEXT:    vmov.32 q0[0], r5
+; CHECK-NEXT:    vmov r5, s5
+; CHECK-NEXT:    ldr r1, [r1]
+; CHECK-NEXT:    ldr r5, [r5]
+; CHECK-NEXT:    vmov.32 q0[1], r5
+; CHECK-NEXT:    vmov r5, s6
+; CHECK-NEXT:    vmov.32 q1[0], r0
+; CHECK-NEXT:    vmov r0, s13
+; CHECK-NEXT:    ldr r5, [r5]
+; CHECK-NEXT:    ldr r0, [r0]
+; CHECK-NEXT:    vmov.32 q0[2], r5
+; CHECK-NEXT:    vmov r5, s8
+; CHECK-NEXT:    vmov.32 q1[1], r0
+; CHECK-NEXT:    vmov r0, s14
+; CHECK-NEXT:    vmov.32 q0[3], r6
+; CHECK-NEXT:    ldr r5, [r5]
+; CHECK-NEXT:    ldr r0, [r0]
+; CHECK-NEXT:    vmov.32 q1[2], r0
+; CHECK-NEXT:    vmov r0, s15
+; CHECK-NEXT:    vmov.32 q3[0], lr
+; CHECK-NEXT:    vmov.32 q3[1], r3
+; CHECK-NEXT:    vmov.32 q3[2], r1
+; CHECK-NEXT:    vmov.32 q3[3], r2
+; CHECK-NEXT:    ldr r0, [r0]
+; CHECK-NEXT:    vmov.32 q1[3], r0
+; CHECK-NEXT:    vmov r0, s9
+; CHECK-NEXT:    vmov.32 q2[0], r5
+; CHECK-NEXT:    ldr r0, [r0]
+; CHECK-NEXT:    vmov.32 q2[1], r0
+; CHECK-NEXT:    vmov.32 q2[2], r12
+; CHECK-NEXT:    vmov.32 q2[3], r4
+; CHECK-NEXT:    pop {r4, r5, r6, pc}
+entry:
+  %offs = load <16 x i32*>, <16 x i32*>* %offptr, align 4
+  %gather = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> %offs, i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+  ret <16 x i32> %gather
+}
+
+; f32
+
+define arm_aapcs_vfpcc <2 x float> @ptr_v2f32(<2 x float*>* %offptr) {
+; CHECK-LABEL: ptr_v2f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    ldrd r1, r0, [r0]
+; CHECK-NEXT:    vldr s1, [r0]
+; CHECK-NEXT:    vldr s0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <2 x float*>, <2 x float*>* %offptr, align 4
+  %gather = call <2 x float> @llvm.masked.gather.v2f32.v2p0f32(<2 x float*> %offs, i32 4, <2 x i1> <i1 true, i1 true>, <2 x float> undef)
+  ret <2 x float> %gather
+}
+
+define arm_aapcs_vfpcc <4 x float> @ptr_v4f32(<4 x float*>* %offptr) {
+; CHECK-LABEL: ptr_v4f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vldrw.u32 q0, [q1]
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <4 x float*>, <4 x float*>* %offptr, align 4
+  %gather = call <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*> %offs, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x float> undef)
+  ret <4 x float> %gather
+}
+
+define arm_aapcs_vfpcc <8 x float> @ptr_v8f32(<8 x float*>* %offptr) {
+; CHECK-LABEL: ptr_v8f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #16]
+; CHECK-NEXT:    vmov r1, s7
+; CHECK-NEXT:    vmov r0, s11
+; CHECK-NEXT:    vldr s3, [r1]
+; CHECK-NEXT:    vmov r1, s6
+; CHECK-NEXT:    vldr s2, [r1]
+; CHECK-NEXT:    vmov r1, s5
+; CHECK-NEXT:    vldr s1, [r1]
+; CHECK-NEXT:    vmov r1, s4
+; CHECK-NEXT:    vldr s7, [r0]
+; CHECK-NEXT:    vmov r0, s10
+; CHECK-NEXT:    vldr s0, [r1]
+; CHECK-NEXT:    vldr s6, [r0]
+; CHECK-NEXT:    vmov r0, s9
+; CHECK-NEXT:    vldr s5, [r0]
+; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    vldr s4, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <8 x float*>, <8 x float*>* %offptr, align 4
+  %gather = call <8 x float> @llvm.masked.gather.v8f32.v8p0f32(<8 x float*> %offs, i32 4, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x float> undef)
+  ret <8 x float> %gather
+}
+
+; i16
+
+define arm_aapcs_vfpcc <8 x i16> @ptr_i16(<8 x i16*>* %offptr) {
+; CHECK-LABEL: ptr_i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, r5, r7, lr}
+; CHECK-NEXT:    push {r4, r5, r7, lr}
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #16]
+; CHECK-NEXT:    vmov r1, s2
+; CHECK-NEXT:    vmov r2, s3
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    vmov r5, s1
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmov r4, s7
+; CHECK-NEXT:    ldrh.w r12, [r1]
+; CHECK-NEXT:    vmov r1, s5
+; CHECK-NEXT:    ldrh.w lr, [r2]
+; CHECK-NEXT:    vmov r2, s6
+; CHECK-NEXT:    ldrh r3, [r3]
+; CHECK-NEXT:    ldrh r5, [r5]
+; CHECK-NEXT:    vmov.16 q0[0], r3
+; CHECK-NEXT:    ldrh r0, [r0]
+; CHECK-NEXT:    vmov.16 q0[1], r5
+; CHECK-NEXT:    ldrh r4, [r4]
+; CHECK-NEXT:    vmov.16 q0[2], r12
+; CHECK-NEXT:    vmov.16 q0[3], lr
+; CHECK-NEXT:    vmov.16 q0[4], r0
+; CHECK-NEXT:    ldrh r1, [r1]
+; CHECK-NEXT:    ldrh r2, [r2]
+; CHECK-NEXT:    vmov.16 q0[5], r1
+; CHECK-NEXT:    vmov.16 q0[6], r2
+; CHECK-NEXT:    vmov.16 q0[7], r4
+; CHECK-NEXT:    pop {r4, r5, r7, pc}
+entry:
+  %offs = load <8 x i16*>, <8 x i16*>* %offptr, align 4
+  %gather = call <8 x i16> @llvm.masked.gather.v8i16.v8p0i16(<8 x i16*> %offs, i32 2, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i16> undef)
+  ret <8 x i16> %gather
+}
+
+define arm_aapcs_vfpcc <2 x i32> @ptr_v2i16_sext(<2 x i16*>* %offptr) {
+; CHECK-LABEL: ptr_v2i16_sext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    ldrd r1, r0, [r0]
+; CHECK-NEXT:    ldrsh.w r0, [r0]
+; CHECK-NEXT:    ldrsh.w r1, [r1]
+; CHECK-NEXT:    asrs r2, r0, #31
+; CHECK-NEXT:    vmov.32 q0[0], r1
+; CHECK-NEXT:    asrs r1, r1, #31
+; CHECK-NEXT:    vmov.32 q0[1], r1
+; CHECK-NEXT:    vmov.32 q0[2], r0
+; CHECK-NEXT:    vmov.32 q0[3], r2
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <2 x i16*>, <2 x i16*>* %offptr, align 4
+  %gather = call <2 x i16> @llvm.masked.gather.v2i16.v2p0i16(<2 x i16*> %offs, i32 2, <2 x i1> <i1 true, i1 true>, <2 x i16> undef)
+  %ext = sext <2 x i16> %gather to <2 x i32>
+  ret <2 x i32> %ext
+}
+
+define arm_aapcs_vfpcc <2 x i32> @ptr_v2i16_zext(<2 x i16*>* %offptr) {
+; CHECK-LABEL: ptr_v2i16_zext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    ldrd r1, r0, [r0]
+; CHECK-NEXT:    adr r2, .LCPI9_0
+; CHECK-NEXT:    ldrh r0, [r0]
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    ldrh r1, [r1]
+; CHECK-NEXT:    vmov.32 q1[0], r1
+; CHECK-NEXT:    vmov.32 q1[2], r0
+; CHECK-NEXT:    vand q0, q1, q0
+; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI9_0:
+; CHECK-NEXT:    .long 65535 @ 0xffff
+; CHECK-NEXT:    .long 0 @ 0x0
+; CHECK-NEXT:    .long 65535 @ 0xffff
+; CHECK-NEXT:    .long 0 @ 0x0
+entry:
+  %offs = load <2 x i16*>, <2 x i16*>* %offptr, align 4
+  %gather = call <2 x i16> @llvm.masked.gather.v2i16.v2p0i16(<2 x i16*> %offs, i32 2, <2 x i1> <i1 true, i1 true>, <2 x i16> undef)
+  %ext = zext <2 x i16> %gather to <2 x i32>
+  ret <2 x i32> %ext
+}
+
+define arm_aapcs_vfpcc <4 x i32> @ptr_v4i16_sext(<4 x i16*>* %offptr) {
+; CHECK-LABEL: ptr_v4i16_sext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vmov r0, s1
+; CHECK-NEXT:    vmov r3, s2
+; CHECK-NEXT:    vmov r1, s3
+; CHECK-NEXT:    ldrh r2, [r2]
+; CHECK-NEXT:    ldrh r0, [r0]
+; CHECK-NEXT:    vmov.32 q0[0], r2
+; CHECK-NEXT:    ldrh r3, [r3]
+; CHECK-NEXT:    vmov.32 q0[1], r0
+; CHECK-NEXT:    ldrh r1, [r1]
+; CHECK-NEXT:    vmov.32 q0[2], r3
+; CHECK-NEXT:    vmov.32 q0[3], r1
+; CHECK-NEXT:    vmovlb.s16 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <4 x i16*>, <4 x i16*>* %offptr, align 4
+  %gather = call <4 x i16> @llvm.masked.gather.v4i16.v4p0i16(<4 x i16*> %offs, i32 2, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i16> undef)
+  %ext = sext <4 x i16> %gather to <4 x i32>
+  ret <4 x i32> %ext
+}
+
+define arm_aapcs_vfpcc <4 x i32> @ptr_v4i16_zext(<4 x i16*>* %offptr) {
+; CHECK-LABEL: ptr_v4i16_zext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vmov r0, s1
+; CHECK-NEXT:    vmov r3, s2
+; CHECK-NEXT:    vmov r1, s3
+; CHECK-NEXT:    ldrh r2, [r2]
+; CHECK-NEXT:    ldrh r0, [r0]
+; CHECK-NEXT:    vmov.32 q0[0], r2
+; CHECK-NEXT:    ldrh r3, [r3]
+; CHECK-NEXT:    vmov.32 q0[1], r0
+; CHECK-NEXT:    ldrh r1, [r1]
+; CHECK-NEXT:    vmov.32 q0[2], r3
+; CHECK-NEXT:    vmov.32 q0[3], r1
+; CHECK-NEXT:    vmovlb.u16 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <4 x i16*>, <4 x i16*>* %offptr, align 4
+  %gather = call <4 x i16> @llvm.masked.gather.v4i16.v4p0i16(<4 x i16*> %offs, i32 2, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i16> undef)
+  %ext = zext <4 x i16> %gather to <4 x i32>
+  ret <4 x i32> %ext
+}
+
+define arm_aapcs_vfpcc <8 x i32> @ptr_v8i16_sext(<8 x i16*>* %offptr) {
+; CHECK-LABEL: ptr_v8i16_sext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, r5, r7, lr}
+; CHECK-NEXT:    push {r4, r5, r7, lr}
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #16]
+; CHECK-NEXT:    vmov r1, s2
+; CHECK-NEXT:    vmov r2, s3
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmov r5, s1
+; CHECK-NEXT:    vmov r4, s7
+; CHECK-NEXT:    ldrh.w r12, [r1]
+; CHECK-NEXT:    vmov r1, s5
+; CHECK-NEXT:    ldrh.w lr, [r2]
+; CHECK-NEXT:    vmov r2, s6
+; CHECK-NEXT:    ldrh r3, [r3]
+; CHECK-NEXT:    ldrh r0, [r0]
+; CHECK-NEXT:    ldrh r5, [r5]
+; CHECK-NEXT:    vmov.32 q0[0], r3
+; CHECK-NEXT:    vmov.32 q1[0], r0
+; CHECK-NEXT:    ldrh r4, [r4]
+; CHECK-NEXT:    vmov.32 q0[1], r5
+; CHECK-NEXT:    vmov.32 q0[2], r12
+; CHECK-NEXT:    vmov.32 q0[3], lr
+; CHECK-NEXT:    vmovlb.s16 q0, q0
+; CHECK-NEXT:    ldrh r1, [r1]
+; CHECK-NEXT:    ldrh r2, [r2]
+; CHECK-NEXT:    vmov.32 q1[1], r1
+; CHECK-NEXT:    vmov.32 q1[2], r2
+; CHECK-NEXT:    vmov.32 q1[3], r4
+; CHECK-NEXT:    vmovlb.s16 q1, q1
+; CHECK-NEXT:    pop {r4, r5, r7, pc}
+entry:
+  %offs = load <8 x i16*>, <8 x i16*>* %offptr, align 4
+  %gather = call <8 x i16> @llvm.masked.gather.v8i16.v8p0i16(<8 x i16*> %offs, i32 2, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i16> undef)
+  %ext = sext <8 x i16> %gather to <8 x i32>
+  ret <8 x i32> %ext
+}
+
+define arm_aapcs_vfpcc <8 x i32> @ptr_v8i16_zext(<8 x i16*>* %offptr) {
+; CHECK-LABEL: ptr_v8i16_zext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, r5, r7, lr}
+; CHECK-NEXT:    push {r4, r5, r7, lr}
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #16]
+; CHECK-NEXT:    vmov r1, s2
+; CHECK-NEXT:    vmov r2, s3
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmov r5, s1
+; CHECK-NEXT:    vmov r4, s7
+; CHECK-NEXT:    ldrh.w r12, [r1]
+; CHECK-NEXT:    vmov r1, s5
+; CHECK-NEXT:    ldrh.w lr, [r2]
+; CHECK-NEXT:    vmov r2, s6
+; CHECK-NEXT:    ldrh r3, [r3]
+; CHECK-NEXT:    ldrh r0, [r0]
+; CHECK-NEXT:    ldrh r5, [r5]
+; CHECK-NEXT:    vmov.32 q0[0], r3
+; CHECK-NEXT:    vmov.32 q1[0], r0
+; CHECK-NEXT:    ldrh r4, [r4]
+; CHECK-NEXT:    vmov.32 q0[1], r5
+; CHECK-NEXT:    vmov.32 q0[2], r12
+; CHECK-NEXT:    vmov.32 q0[3], lr
+; CHECK-NEXT:    vmovlb.u16 q0, q0
+; CHECK-NEXT:    ldrh r1, [r1]
+; CHECK-NEXT:    ldrh r2, [r2]
+; CHECK-NEXT:    vmov.32 q1[1], r1
+; CHECK-NEXT:    vmov.32 q1[2], r2
+; CHECK-NEXT:    vmov.32 q1[3], r4
+; CHECK-NEXT:    vmovlb.u16 q1, q1
+; CHECK-NEXT:    pop {r4, r5, r7, pc}
+entry:
+  %offs = load <8 x i16*>, <8 x i16*>* %offptr, align 4
+  %gather = call <8 x i16> @llvm.masked.gather.v8i16.v8p0i16(<8 x i16*> %offs, i32 2, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i16> undef)
+  %ext = zext <8 x i16> %gather to <8 x i32>
+  ret <8 x i32> %ext
+}
+
+; f16
+
+define arm_aapcs_vfpcc <8 x half> @ptr_f16(<8 x half*>* %offptr) {
+; CHECK-LABEL: ptr_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vmov r1, s5
+; CHECK-NEXT:    vldr.16 s0, [r1]
+; CHECK-NEXT:    vmov r2, s4
+; CHECK-NEXT:    vmov r1, s0
+; CHECK-NEXT:    vldr.16 s0, [r2]
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vmov.16 q0[0], r2
+; CHECK-NEXT:    vmov.16 q0[1], r1
+; CHECK-NEXT:    vmov r1, s6
+; CHECK-NEXT:    vldr.16 s8, [r1]
+; CHECK-NEXT:    vmov r1, s8
+; CHECK-NEXT:    vmov.16 q0[2], r1
+; CHECK-NEXT:    vmov r1, s7
+; CHECK-NEXT:    vldr.16 s4, [r1]
+; CHECK-NEXT:    vmov r1, s4
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #16]
+; CHECK-NEXT:    vmov.16 q0[3], r1
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vldr.16 s8, [r0]
+; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    vmov.16 q0[4], r0
+; CHECK-NEXT:    vmov r0, s5
+; CHECK-NEXT:    vldr.16 s8, [r0]
+; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    vmov.16 q0[5], r0
+; CHECK-NEXT:    vmov r0, s6
+; CHECK-NEXT:    vldr.16 s8, [r0]
+; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    vmov.16 q0[6], r0
+; CHECK-NEXT:    vmov r0, s7
+; CHECK-NEXT:    vldr.16 s4, [r0]
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmov.16 q0[7], r0
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <8 x half*>, <8 x half*>* %offptr, align 4
+  %gather = call <8 x half> @llvm.masked.gather.v8f16.v8p0f16(<8 x half*> %offs, i32 2, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x half> undef)
+  ret <8 x half> %gather
+}
+
+; i8
+
+define arm_aapcs_vfpcc <16 x i8> @ptr_i8(<16 x i8*>* %offptr) {
+; CHECK-LABEL: ptr_i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, r5, r6, lr}
+; CHECK-NEXT:    push {r4, r5, r6, lr}
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #32]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #48]
+; CHECK-NEXT:    vldrw.u32 q2, [r0]
+; CHECK-NEXT:    vmov r1, s6
+; CHECK-NEXT:    vmov r5, s8
+; CHECK-NEXT:    vmov r2, s3
+; CHECK-NEXT:    vmov r6, s11
+; CHECK-NEXT:    vmov r4, s7
+; CHECK-NEXT:    ldrb.w r12, [r1]
+; CHECK-NEXT:    vmov r1, s0
+; CHECK-NEXT:    ldrb r5, [r5]
+; CHECK-NEXT:    ldrb r2, [r2]
+; CHECK-NEXT:    ldrb r6, [r6]
+; CHECK-NEXT:    ldrb r4, [r4]
+; CHECK-NEXT:    ldrb.w lr, [r1]
+; CHECK-NEXT:    vmov r1, s1
+; CHECK-NEXT:    ldrb r3, [r1]
+; CHECK-NEXT:    vmov r1, s2
+; CHECK-NEXT:    vmov.8 q0[0], r5
+; CHECK-NEXT:    vmov r5, s9
+; CHECK-NEXT:    ldrb r1, [r1]
+; CHECK-NEXT:    ldrb r5, [r5]
+; CHECK-NEXT:    vmov.8 q0[1], r5
+; CHECK-NEXT:    vmov r5, s10
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #16]
+; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    ldrb r5, [r5]
+; CHECK-NEXT:    vmov.8 q0[2], r5
+; CHECK-NEXT:    vmov r5, s4
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[3], r6
+; CHECK-NEXT:    vmov.8 q0[4], r0
+; CHECK-NEXT:    vmov r0, s9
+; CHECK-NEXT:    ldrb r5, [r5]
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[5], r0
+; CHECK-NEXT:    vmov r0, s10
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[6], r0
+; CHECK-NEXT:    vmov r0, s11
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[7], r0
+; CHECK-NEXT:    vmov r0, s5
+; CHECK-NEXT:    vmov.8 q0[8], r5
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[9], r0
+; CHECK-NEXT:    vmov.8 q0[10], r12
+; CHECK-NEXT:    vmov.8 q0[11], r4
+; CHECK-NEXT:    vmov.8 q0[12], lr
+; CHECK-NEXT:    vmov.8 q0[13], r3
+; CHECK-NEXT:    vmov.8 q0[14], r1
+; CHECK-NEXT:    vmov.8 q0[15], r2
+; CHECK-NEXT:    pop {r4, r5, r6, pc}
+entry:
+  %offs = load <16 x i8*>, <16 x i8*>* %offptr, align 4
+  %gather = call <16 x i8> @llvm.masked.gather.v16i8.v16p0i8(<16 x i8*> %offs, i32 2, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i8> undef)
+  ret <16 x i8> %gather
+}
+
+define arm_aapcs_vfpcc <8 x i16> @ptr_v8i8_sext16(<8 x i8*>* %offptr) {
+; CHECK-LABEL: ptr_v8i8_sext16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, r5, r7, lr}
+; CHECK-NEXT:    push {r4, r5, r7, lr}
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #16]
+; CHECK-NEXT:    vmov r2, s3
+; CHECK-NEXT:    vmov r1, s1
+; CHECK-NEXT:    vmov r5, s0
+; CHECK-NEXT:    vmov r3, s2
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmov r4, s7
+; CHECK-NEXT:    ldrb.w r12, [r2]
+; CHECK-NEXT:    vmov r2, s5
+; CHECK-NEXT:    ldrb.w lr, [r1]
+; CHECK-NEXT:    vmov r1, s6
+; CHECK-NEXT:    ldrb r5, [r5]
+; CHECK-NEXT:    ldrb r3, [r3]
+; CHECK-NEXT:    vmov.16 q0[0], r5
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.16 q0[1], lr
+; CHECK-NEXT:    ldrb r4, [r4]
+; CHECK-NEXT:    vmov.16 q0[2], r3
+; CHECK-NEXT:    vmov.16 q0[3], r12
+; CHECK-NEXT:    vmov.16 q0[4], r0
+; CHECK-NEXT:    ldrb r2, [r2]
+; CHECK-NEXT:    ldrb r1, [r1]
+; CHECK-NEXT:    vmov.16 q0[5], r2
+; CHECK-NEXT:    vmov.16 q0[6], r1
+; CHECK-NEXT:    vmov.16 q0[7], r4
+; CHECK-NEXT:    vmovlb.s8 q0, q0
+; CHECK-NEXT:    pop {r4, r5, r7, pc}
+entry:
+  %offs = load <8 x i8*>, <8 x i8*>* %offptr, align 4
+  %gather = call <8 x i8> @llvm.masked.gather.v8i8.v8p0i8(<8 x i8*> %offs, i32 1, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i8> undef)
+  %ext = sext <8 x i8> %gather to <8 x i16>
+  ret <8 x i16> %ext
+}
+
+define arm_aapcs_vfpcc <8 x i16> @ptr_v8i8_zext16(<8 x i8*>* %offptr) {
+; CHECK-LABEL: ptr_v8i8_zext16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, r5, r7, lr}
+; CHECK-NEXT:    push {r4, r5, r7, lr}
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #16]
+; CHECK-NEXT:    vmov r2, s3
+; CHECK-NEXT:    vmov r1, s1
+; CHECK-NEXT:    vmov r5, s0
+; CHECK-NEXT:    vmov r3, s2
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmov r4, s7
+; CHECK-NEXT:    ldrb.w r12, [r2]
+; CHECK-NEXT:    vmov r2, s5
+; CHECK-NEXT:    ldrb.w lr, [r1]
+; CHECK-NEXT:    vmov r1, s6
+; CHECK-NEXT:    ldrb r5, [r5]
+; CHECK-NEXT:    ldrb r3, [r3]
+; CHECK-NEXT:    vmov.16 q0[0], r5
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.16 q0[1], lr
+; CHECK-NEXT:    ldrb r4, [r4]
+; CHECK-NEXT:    vmov.16 q0[2], r3
+; CHECK-NEXT:    vmov.16 q0[3], r12
+; CHECK-NEXT:    vmov.16 q0[4], r0
+; CHECK-NEXT:    ldrb r2, [r2]
+; CHECK-NEXT:    ldrb r1, [r1]
+; CHECK-NEXT:    vmov.16 q0[5], r2
+; CHECK-NEXT:    vmov.16 q0[6], r1
+; CHECK-NEXT:    vmov.16 q0[7], r4
+; CHECK-NEXT:    vmovlb.u8 q0, q0
+; CHECK-NEXT:    pop {r4, r5, r7, pc}
+entry:
+  %offs = load <8 x i8*>, <8 x i8*>* %offptr, align 4
+  %gather = call <8 x i8> @llvm.masked.gather.v8i8.v8p0i8(<8 x i8*> %offs, i32 1, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i8> undef)
+  %ext = zext <8 x i8> %gather to <8 x i16>
+  ret <8 x i16> %ext
+}
+
+define arm_aapcs_vfpcc <4 x i32> @ptr_v4i8_sext32(<4 x i8*>* %offptr) {
+; CHECK-LABEL: ptr_v4i8_sext32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vmov r3, s1
+; CHECK-NEXT:    vmov r1, s2
+; CHECK-NEXT:    vmov r2, s3
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    ldrb r3, [r3]
+; CHECK-NEXT:    vmov.32 q0[0], r0
+; CHECK-NEXT:    ldrb r1, [r1]
+; CHECK-NEXT:    vmov.32 q0[1], r3
+; CHECK-NEXT:    ldrb r2, [r2]
+; CHECK-NEXT:    vmov.32 q0[2], r1
+; CHECK-NEXT:    vmov.32 q0[3], r2
+; CHECK-NEXT:    vmovlb.s8 q0, q0
+; CHECK-NEXT:    vmovlb.s16 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <4 x i8*>, <4 x i8*>* %offptr, align 4
+  %gather = call <4 x i8> @llvm.masked.gather.v4i8.v4p0i8(<4 x i8*> %offs, i32 1, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i8> undef)
+  %ext = sext <4 x i8> %gather to <4 x i32>
+  ret <4 x i32> %ext
+}
+
+define arm_aapcs_vfpcc <4 x i32> @ptr_v4i8_zext32(<4 x i8*>* %offptr) {
+; CHECK-LABEL: ptr_v4i8_zext32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vmov.i32 q1, #0xff
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vmov r1, s1
+; CHECK-NEXT:    vmov r3, s2
+; CHECK-NEXT:    vmov r0, s3
+; CHECK-NEXT:    ldrb r2, [r2]
+; CHECK-NEXT:    ldrb r1, [r1]
+; CHECK-NEXT:    vmov.32 q0[0], r2
+; CHECK-NEXT:    ldrb r3, [r3]
+; CHECK-NEXT:    vmov.32 q0[1], r1
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.32 q0[2], r3
+; CHECK-NEXT:    vmov.32 q0[3], r0
+; CHECK-NEXT:    vand q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <4 x i8*>, <4 x i8*>* %offptr, align 4
+  %gather = call <4 x i8> @llvm.masked.gather.v4i8.v4p0i8(<4 x i8*> %offs, i32 1, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i8> undef)
+  %ext = zext <4 x i8> %gather to <4 x i32>
+  ret <4 x i32> %ext
+}
+
+define arm_aapcs_vfpcc <8 x i32> @ptr_v8i8_sext32(<8 x i8*>* %offptr) {
+; CHECK-LABEL: ptr_v8i8_sext32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, r5, r7, lr}
+; CHECK-NEXT:    push {r4, r5, r7, lr}
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #16]
+; CHECK-NEXT:    vmov r1, s2
+; CHECK-NEXT:    vmov r2, s3
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmov r5, s1
+; CHECK-NEXT:    vmov r4, s7
+; CHECK-NEXT:    ldrb.w r12, [r1]
+; CHECK-NEXT:    vmov r1, s5
+; CHECK-NEXT:    ldrb.w lr, [r2]
+; CHECK-NEXT:    vmov r2, s6
+; CHECK-NEXT:    ldrb r3, [r3]
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    ldrb r5, [r5]
+; CHECK-NEXT:    vmov.32 q0[0], r3
+; CHECK-NEXT:    vmov.32 q1[0], r0
+; CHECK-NEXT:    ldrb r4, [r4]
+; CHECK-NEXT:    vmov.32 q0[1], r5
+; CHECK-NEXT:    vmov.32 q0[2], r12
+; CHECK-NEXT:    vmov.32 q0[3], lr
+; CHECK-NEXT:    vmovlb.s8 q0, q0
+; CHECK-NEXT:    vmovlb.s16 q0, q0
+; CHECK-NEXT:    ldrb r1, [r1]
+; CHECK-NEXT:    ldrb r2, [r2]
+; CHECK-NEXT:    vmov.32 q1[1], r1
+; CHECK-NEXT:    vmov.32 q1[2], r2
+; CHECK-NEXT:    vmov.32 q1[3], r4
+; CHECK-NEXT:    vmovlb.s8 q1, q1
+; CHECK-NEXT:    vmovlb.s16 q1, q1
+; CHECK-NEXT:    pop {r4, r5, r7, pc}
+entry:
+  %offs = load <8 x i8*>, <8 x i8*>* %offptr, align 4
+  %gather = call <8 x i8> @llvm.masked.gather.v8i8.v8p0i8(<8 x i8*> %offs, i32 1, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i8> undef)
+  %ext = sext <8 x i8> %gather to <8 x i32>
+  ret <8 x i32> %ext
+}
+
+define arm_aapcs_vfpcc <8 x i32> @ptr_v8i8_zext32(<8 x i8*>* %offptr) {
+; CHECK-LABEL: ptr_v8i8_zext32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, r5, r7, lr}
+; CHECK-NEXT:    push {r4, r5, r7, lr}
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #16]
+; CHECK-NEXT:    vmov r1, s2
+; CHECK-NEXT:    vmov r2, s3
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmov r4, s0
+; CHECK-NEXT:    vmov r3, s5
+; CHECK-NEXT:    vmov r5, s1
+; CHECK-NEXT:    ldrb.w r12, [r1]
+; CHECK-NEXT:    vmov r1, s6
+; CHECK-NEXT:    ldrb.w lr, [r2]
+; CHECK-NEXT:    vmov r2, s7
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.i32 q1, #0xff
+; CHECK-NEXT:    ldrb r4, [r4]
+; CHECK-NEXT:    ldrb r3, [r3]
+; CHECK-NEXT:    vmov.32 q2[0], r0
+; CHECK-NEXT:    ldrb r5, [r5]
+; CHECK-NEXT:    vmov.32 q0[0], r4
+; CHECK-NEXT:    vmov.32 q2[1], r3
+; CHECK-NEXT:    vmov.32 q0[1], r5
+; CHECK-NEXT:    vmov.32 q0[2], r12
+; CHECK-NEXT:    vmov.32 q0[3], lr
+; CHECK-NEXT:    vand q0, q0, q1
+; CHECK-NEXT:    ldrb r1, [r1]
+; CHECK-NEXT:    ldrb r2, [r2]
+; CHECK-NEXT:    vmov.32 q2[2], r1
+; CHECK-NEXT:    vmov.32 q2[3], r2
+; CHECK-NEXT:    vand q1, q2, q1
+; CHECK-NEXT:    pop {r4, r5, r7, pc}
+entry:
+  %offs = load <8 x i8*>, <8 x i8*>* %offptr, align 4
+  %gather = call <8 x i8> @llvm.masked.gather.v8i8.v8p0i8(<8 x i8*> %offs, i32 1, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i8> undef)
+  %ext = zext <8 x i8> %gather to <8 x i32>
+  ret <8 x i32> %ext
+}
+
+; loops
+
+define void @foo_ptr_p_int32_t(i32* %dest, i32** %src, i32 %n) {
+; CHECK-LABEL: foo_ptr_p_int32_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    bic r2, r2, #15
+; CHECK-NEXT:    cmp r2, #1
+; CHECK-NEXT:    it lt
+; CHECK-NEXT:    poplt {r7, pc}
+; CHECK-NEXT:    subs r2, #4
+; CHECK-NEXT:    movs r3, #1
+; CHECK-NEXT:    subs r1, #16
+; CHECK-NEXT:    add.w lr, r3, r2, lsr #2
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB22_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrw.u32 q0, [r1, #16]!
+; CHECK-NEXT:    vptt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrwt.u32 q1, [q0]
+; CHECK-NEXT:    vstrwt.32 q1, [r0], #16
+; CHECK-NEXT:    le lr, .LBB22_1
+; CHECK-NEXT:  @ %bb.2: @ %for.end
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %and = and i32 %n, -16
+  %cmp11 = icmp sgt i32 %and, 0
+  br i1 %cmp11, label %vector.body, label %for.end
+
+vector.body:                                      ; preds = %entry, %vector.body
+  %index = phi i32 [ %index.next, %vector.body ], [ 0, %entry ]
+  %0 = getelementptr inbounds i32*, i32** %src, i32 %index
+  %1 = bitcast i32** %0 to <4 x i32*>*
+  %wide.load = load <4 x i32*>, <4 x i32*>* %1, align 4
+  %2 = icmp ne <4 x i32*> %wide.load, zeroinitializer
+  %wide.masked.gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %wide.load, i32 4, <4 x i1> %2, <4 x i32> undef)
+  %3 = getelementptr inbounds i32, i32* %dest, i32 %index
+  %4 = bitcast i32* %3 to <4 x i32>*
+  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %wide.masked.gather, <4 x i32>* %4, i32 4, <4 x i1> %2)
+  %index.next = add i32 %index, 4
+  %5 = icmp eq i32 %index.next, %and
+  br i1 %5, label %for.end, label %vector.body
+
+for.end:                                          ; preds = %vector.body, %entry
+  ret void
+}
+
+define void @foo_ptr_p_float(float* %dest, float** %src, i32 %n) {
+; CHECK-LABEL: foo_ptr_p_float:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    bic r2, r2, #15
+; CHECK-NEXT:    cmp r2, #1
+; CHECK-NEXT:    it lt
+; CHECK-NEXT:    poplt {r7, pc}
+; CHECK-NEXT:    subs r2, #4
+; CHECK-NEXT:    movs r3, #1
+; CHECK-NEXT:    subs r1, #16
+; CHECK-NEXT:    add.w lr, r3, r2, lsr #2
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB23_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrw.u32 q0, [r1, #16]!
+; CHECK-NEXT:    vptt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrwt.u32 q1, [q0]
+; CHECK-NEXT:    vstrwt.32 q1, [r0], #16
+; CHECK-NEXT:    le lr, .LBB23_1
+; CHECK-NEXT:  @ %bb.2: @ %for.end
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %and = and i32 %n, -16
+  %cmp11 = icmp sgt i32 %and, 0
+  br i1 %cmp11, label %vector.body, label %for.end
+
+vector.body:                                      ; preds = %entry, %vector.body
+  %index = phi i32 [ %index.next, %vector.body ], [ 0, %entry ]
+  %0 = getelementptr inbounds float*, float** %src, i32 %index
+  %1 = bitcast float** %0 to <4 x float*>*
+  %wide.load = load <4 x float*>, <4 x float*>* %1, align 4
+  %2 = icmp ne <4 x float*> %wide.load, zeroinitializer
+  %3 = bitcast <4 x float*> %wide.load to <4 x i32*>
+  %wide.masked.gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %3, i32 4, <4 x i1> %2, <4 x i32> undef)
+  %4 = getelementptr inbounds float, float* %dest, i32 %index
+  %5 = bitcast float* %4 to <4 x i32>*
+  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %wide.masked.gather, <4 x i32>* %5, i32 4, <4 x i1> %2)
+  %index.next = add i32 %index, 4
+  %6 = icmp eq i32 %index.next, %and
+  br i1 %6, label %for.end, label %vector.body
+
+for.end:                                          ; preds = %vector.body, %entry
+  ret void
+}
+
+define arm_aapcs_vfpcc <4 x i32> @qi4(<4 x i32*> %p) {
+; CHECK-LABEL: qi4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.i32 q1, #0x10
+; CHECK-NEXT:    vadd.i32 q1, q0, q1
+; CHECK-NEXT:    vldrw.u32 q0, [q1]
+; CHECK-NEXT:    bx lr
+entry:
+  %g = getelementptr inbounds i32, <4 x i32*> %p, i32 4
+  %gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %g, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
+  ret <4 x i32> %gather
+}
+
+declare <2 x i32> @llvm.masked.gather.v2i32.v2p0i32(<2 x i32*>, i32, <2 x i1>, <2 x i32>)
+declare <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*>, i32, <4 x i1>, <4 x i32>)
+declare <8 x i32> @llvm.masked.gather.v8i32.v8p0i32(<8 x i32*>, i32, <8 x i1>, <8 x i32>)
+declare <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*>, i32, <16 x i1>, <16 x i32>)
+declare <2 x float> @llvm.masked.gather.v2f32.v2p0f32(<2 x float*>, i32, <2 x i1>, <2 x float>)
+declare <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*>, i32, <4 x i1>, <4 x float>)
+declare <8 x float> @llvm.masked.gather.v8f32.v8p0f32(<8 x float*>, i32, <8 x i1>, <8 x float>)
+declare <2 x i16> @llvm.masked.gather.v2i16.v2p0i16(<2 x i16*>, i32, <2 x i1>, <2 x i16>)
+declare <4 x i16> @llvm.masked.gather.v4i16.v4p0i16(<4 x i16*>, i32, <4 x i1>, <4 x i16>)
+declare <8 x i16> @llvm.masked.gather.v8i16.v8p0i16(<8 x i16*>, i32, <8 x i1>, <8 x i16>)
+declare <16 x i16> @llvm.masked.gather.v16i16.v16p0i16(<16 x i16*>, i32, <16 x i1>, <16 x i16>)
+declare <4 x half> @llvm.masked.gather.v4f16.v4p0f16(<4 x half*>, i32, <4 x i1>, <4 x half>)
+declare <8 x half> @llvm.masked.gather.v8f16.v8p0f16(<8 x half*>, i32, <8 x i1>, <8 x half>)
+declare <16 x half> @llvm.masked.gather.v16f16.v16p0f16(<16 x half*>, i32, <16 x i1>, <16 x half>)
+declare <4 x i8> @llvm.masked.gather.v4i8.v4p0i8(<4 x i8*>, i32, <4 x i1>, <4 x i8>)
+declare <8 x i8> @llvm.masked.gather.v8i8.v8p0i8(<8 x i8*>, i32, <8 x i1>, <8 x i8>)
+declare <16 x i8> @llvm.masked.gather.v16i8.v16p0i8(<16 x i8*>, i32, <16 x i1>, <16 x i8>)
+declare <32 x i8> @llvm.masked.gather.v32i8.v32p0i8(<32 x i8*>, i32, <32 x i1>, <32 x i8>)
+declare void @llvm.masked.store.v4i32.p0v4i32(<4 x i32>, <4 x i32>*, i32, <4 x i1>)
diff --git a/llvm/test/CodeGen/Thumb2/mve-gather-scatter-opt.ll b/llvm/test/CodeGen/Thumb2/mve-gather-scatter-opt.ll
new file mode 100644
index 000000000..a50bd2cc9
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-gather-scatter-opt.ll
@@ -0,0 +1,46 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve.fp %s -o 2>/dev/null - | FileCheck --check-prefix NOGATSCAT %s
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=-mve -enable-arm-maskedgatscat %s -o 2>/dev/null - | FileCheck --check-prefix NOMVE %s
+
+define arm_aapcs_vfpcc <4 x i32> @unscaled_i32_i32(i8* %base, <4 x i32>* %offptr) {
+; NOGATSCAT-LABEL: unscaled_i32_i32:
+; NOGATSCAT:       @ %bb.0: @ %entry
+; NOGATSCAT-NEXT:    vldrw.u32 q0, [r1]
+; NOGATSCAT-NEXT:    vadd.i32 q0, q0, r0
+; NOGATSCAT-NEXT:    vmov r0, s0
+; NOGATSCAT-NEXT:    vmov r3, s1
+; NOGATSCAT-NEXT:    vmov r1, s2
+; NOGATSCAT-NEXT:    vmov r2, s3
+; NOGATSCAT-NEXT:    ldr r0, [r0]
+; NOGATSCAT-NEXT:    ldr r3, [r3]
+; NOGATSCAT-NEXT:    vmov.32 q0[0], r0
+; NOGATSCAT-NEXT:    ldr r1, [r1]
+; NOGATSCAT-NEXT:    vmov.32 q0[1], r3
+; NOGATSCAT-NEXT:    ldr r2, [r2]
+; NOGATSCAT-NEXT:    vmov.32 q0[2], r1
+; NOGATSCAT-NEXT:    vmov.32 q0[3], r2
+; NOGATSCAT-NEXT:    bx lr
+;
+; NOMVE-LABEL: unscaled_i32_i32:
+; NOMVE:       @ %bb.0: @ %entry
+; NOMVE-NEXT:    .save {r4, lr}
+; NOMVE-NEXT:    push {r4, lr}
+; NOMVE-NEXT:    ldm.w r1, {r2, r3, lr}
+; NOMVE-NEXT:    ldr r4, [r1, #12]
+; NOMVE-NEXT:    ldr.w r12, [r0, r2]
+; NOMVE-NEXT:    ldr r1, [r0, r3]
+; NOMVE-NEXT:    ldr.w r2, [r0, lr]
+; NOMVE-NEXT:    ldr r3, [r0, r4]
+; NOMVE-NEXT:    mov r0, r12
+; NOMVE-NEXT:    pop {r4, pc}
+
+
+entry:
+  %offs = load <4 x i32>, <4 x i32>* %offptr, align 4
+  %byte_ptrs = getelementptr inbounds i8, i8* %base, <4 x i32> %offs
+  %ptrs = bitcast <4 x i8*> %byte_ptrs to <4 x i32*>
+  %gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %ptrs, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
+  ret <4 x i32> %gather
+}
+
+declare <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*>, i32, <4 x i1>, <4 x i32>)
diff --git a/llvm/test/CodeGen/Thumb2/mve-intrinsics/load-store.ll b/llvm/test/CodeGen/Thumb2/mve-intrinsics/load-store.ll
new file mode 100644
index 000000000..6b5a1dd2b
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-intrinsics/load-store.ll
@@ -0,0 +1,1208 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve.fp -verify-machineinstrs -o - %s | FileCheck %s
+
+define arm_aapcs_vfpcc <8 x half> @test_vld1q_f16(half* %base) {
+; CHECK-LABEL: test_vld1q_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast half* %base to <8 x half>*
+  %1 = load <8 x half>, <8 x half>* %0, align 2
+  ret <8 x half> %1
+}
+
+define arm_aapcs_vfpcc <4 x float> @test_vld1q_f32(float* %base) {
+; CHECK-LABEL: test_vld1q_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast float* %base to <4 x float>*
+  %1 = load <4 x float>, <4 x float>* %0, align 4
+  ret <4 x float> %1
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vld1q_s8(i8* %base) {
+; CHECK-LABEL: test_vld1q_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u8 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast i8* %base to <16 x i8>*
+  %1 = load <16 x i8>, <16 x i8>* %0, align 1
+  ret <16 x i8> %1
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vld1q_s16(i16* %base) {
+; CHECK-LABEL: test_vld1q_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast i16* %base to <8 x i16>*
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
+  ret <8 x i16> %1
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vld1q_s32(i32* %base) {
+; CHECK-LABEL: test_vld1q_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast i32* %base to <4 x i32>*
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
+  ret <4 x i32> %1
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vld1q_u8(i8* %base) {
+; CHECK-LABEL: test_vld1q_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u8 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast i8* %base to <16 x i8>*
+  %1 = load <16 x i8>, <16 x i8>* %0, align 1
+  ret <16 x i8> %1
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vld1q_u16(i16* %base) {
+; CHECK-LABEL: test_vld1q_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast i16* %base to <8 x i16>*
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
+  ret <8 x i16> %1
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vld1q_u32(i32* %base) {
+; CHECK-LABEL: test_vld1q_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast i32* %base to <4 x i32>*
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
+  ret <4 x i32> %1
+}
+
+define arm_aapcs_vfpcc <8 x half> @test_vld1q_z_f16(half* %base, i16 zeroext %p) {
+; CHECK-LABEL: test_vld1q_z_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrht.u16 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast half* %base to <8 x half>*
+  %1 = zext i16 %p to i32
+  %2 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %1)
+  %3 = call <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>* %0, i32 2, <8 x i1> %2, <8 x half> zeroinitializer)
+  ret <8 x half> %3
+}
+
+declare <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32)
+
+declare <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>*, i32 immarg, <8 x i1>, <8 x half>)
+
+define arm_aapcs_vfpcc <4 x float> @test_vld1q_z_f32(float* %base, i16 zeroext %p) {
+; CHECK-LABEL: test_vld1q_z_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrwt.u32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast float* %base to <4 x float>*
+  %1 = zext i16 %p to i32
+  %2 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %1)
+  %3 = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* %0, i32 4, <4 x i1> %2, <4 x float> zeroinitializer)
+  ret <4 x float> %3
+}
+
+declare <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32)
+
+declare <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>*, i32 immarg, <4 x i1>, <4 x float>)
+
+define arm_aapcs_vfpcc <16 x i8> @test_vld1q_z_s8(i8* %base, i16 zeroext %p) {
+; CHECK-LABEL: test_vld1q_z_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrbt.u8 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast i8* %base to <16 x i8>*
+  %1 = zext i16 %p to i32
+  %2 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %1)
+  %3 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %0, i32 1, <16 x i1> %2, <16 x i8> zeroinitializer)
+  ret <16 x i8> %3
+}
+
+declare <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32)
+
+declare <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>*, i32 immarg, <16 x i1>, <16 x i8>)
+
+define arm_aapcs_vfpcc <8 x i16> @test_vld1q_z_s16(i16* %base, i16 zeroext %p) {
+; CHECK-LABEL: test_vld1q_z_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrht.u16 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast i16* %base to <8 x i16>*
+  %1 = zext i16 %p to i32
+  %2 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %1)
+  %3 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %0, i32 2, <8 x i1> %2, <8 x i16> zeroinitializer)
+  ret <8 x i16> %3
+}
+
+declare <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>*, i32 immarg, <8 x i1>, <8 x i16>)
+
+define arm_aapcs_vfpcc <4 x i32> @test_vld1q_z_s32(i32* %base, i16 zeroext %p) {
+; CHECK-LABEL: test_vld1q_z_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrwt.u32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast i32* %base to <4 x i32>*
+  %1 = zext i16 %p to i32
+  %2 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %1)
+  %3 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %0, i32 4, <4 x i1> %2, <4 x i32> zeroinitializer)
+  ret <4 x i32> %3
+}
+
+declare <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>*, i32 immarg, <4 x i1>, <4 x i32>)
+
+define arm_aapcs_vfpcc <16 x i8> @test_vld1q_z_u8(i8* %base, i16 zeroext %p) {
+; CHECK-LABEL: test_vld1q_z_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrbt.u8 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast i8* %base to <16 x i8>*
+  %1 = zext i16 %p to i32
+  %2 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %1)
+  %3 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %0, i32 1, <16 x i1> %2, <16 x i8> zeroinitializer)
+  ret <16 x i8> %3
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vld1q_z_u16(i16* %base, i16 zeroext %p) {
+; CHECK-LABEL: test_vld1q_z_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrht.u16 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast i16* %base to <8 x i16>*
+  %1 = zext i16 %p to i32
+  %2 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %1)
+  %3 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %0, i32 2, <8 x i1> %2, <8 x i16> zeroinitializer)
+  ret <8 x i16> %3
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vld1q_z_u32(i32* %base, i16 zeroext %p) {
+; CHECK-LABEL: test_vld1q_z_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrwt.u32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast i32* %base to <4 x i32>*
+  %1 = zext i16 %p to i32
+  %2 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %1)
+  %3 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %0, i32 4, <4 x i1> %2, <4 x i32> zeroinitializer)
+  ret <4 x i32> %3
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vldrbq_s8(i8* %base) {
+; CHECK-LABEL: test_vldrbq_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u8 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast i8* %base to <16 x i8>*
+  %1 = load <16 x i8>, <16 x i8>* %0, align 1
+  ret <16 x i8> %1
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vldrbq_s16(i8* %base) {
+; CHECK-LABEL: test_vldrbq_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.s16 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast i8* %base to <8 x i8>*
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
+  %2 = sext <8 x i8> %1 to <8 x i16>
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vldrbq_s32(i8* %base) {
+; CHECK-LABEL: test_vldrbq_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.s32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast i8* %base to <4 x i8>*
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
+  %2 = sext <4 x i8> %1 to <4 x i32>
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vldrbq_u8(i8* %base) {
+; CHECK-LABEL: test_vldrbq_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u8 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast i8* %base to <16 x i8>*
+  %1 = load <16 x i8>, <16 x i8>* %0, align 1
+  ret <16 x i8> %1
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vldrbq_u16(i8* %base) {
+; CHECK-LABEL: test_vldrbq_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u16 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast i8* %base to <8 x i8>*
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
+  %2 = zext <8 x i8> %1 to <8 x i16>
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vldrbq_u32(i8* %base) {
+; CHECK-LABEL: test_vldrbq_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast i8* %base to <4 x i8>*
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
+  %2 = zext <4 x i8> %1 to <4 x i32>
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vldrbq_z_s8(i8* %base, i16 zeroext %p) {
+; CHECK-LABEL: test_vldrbq_z_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrbt.u8 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast i8* %base to <16 x i8>*
+  %1 = zext i16 %p to i32
+  %2 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %1)
+  %3 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %0, i32 1, <16 x i1> %2, <16 x i8> zeroinitializer)
+  ret <16 x i8> %3
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vldrbq_z_s16(i8* %base, i16 zeroext %p) {
+; CHECK-LABEL: test_vldrbq_z_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrbt.s16 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast i8* %base to <8 x i8>*
+  %1 = zext i16 %p to i32
+  %2 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %1)
+  %3 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %0, i32 1, <8 x i1> %2, <8 x i8> zeroinitializer)
+  %4 = sext <8 x i8> %3 to <8 x i16>
+  ret <8 x i16> %4
+}
+
+declare <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>*, i32 immarg, <8 x i1>, <8 x i8>)
+
+define arm_aapcs_vfpcc <4 x i32> @test_vldrbq_z_s32(i8* %base, i16 zeroext %p) {
+; CHECK-LABEL: test_vldrbq_z_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrbt.s32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast i8* %base to <4 x i8>*
+  %1 = zext i16 %p to i32
+  %2 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %1)
+  %3 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %0, i32 1, <4 x i1> %2, <4 x i8> zeroinitializer)
+  %4 = sext <4 x i8> %3 to <4 x i32>
+  ret <4 x i32> %4
+}
+
+declare <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>*, i32 immarg, <4 x i1>, <4 x i8>)
+
+define arm_aapcs_vfpcc <16 x i8> @test_vldrbq_z_u8(i8* %base, i16 zeroext %p) {
+; CHECK-LABEL: test_vldrbq_z_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrbt.u8 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast i8* %base to <16 x i8>*
+  %1 = zext i16 %p to i32
+  %2 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %1)
+  %3 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %0, i32 1, <16 x i1> %2, <16 x i8> zeroinitializer)
+  ret <16 x i8> %3
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vldrbq_z_u16(i8* %base, i16 zeroext %p) {
+; CHECK-LABEL: test_vldrbq_z_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrbt.u16 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast i8* %base to <8 x i8>*
+  %1 = zext i16 %p to i32
+  %2 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %1)
+  %3 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %0, i32 1, <8 x i1> %2, <8 x i8> zeroinitializer)
+  %4 = zext <8 x i8> %3 to <8 x i16>
+  ret <8 x i16> %4
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vldrbq_z_u32(i8* %base, i16 zeroext %p) {
+; CHECK-LABEL: test_vldrbq_z_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrbt.u32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast i8* %base to <4 x i8>*
+  %1 = zext i16 %p to i32
+  %2 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %1)
+  %3 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %0, i32 1, <4 x i1> %2, <4 x i8> zeroinitializer)
+  %4 = zext <4 x i8> %3 to <4 x i32>
+  ret <4 x i32> %4
+}
+
+define arm_aapcs_vfpcc <8 x half> @test_vldrhq_f16(half* %base) {
+; CHECK-LABEL: test_vldrhq_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast half* %base to <8 x half>*
+  %1 = load <8 x half>, <8 x half>* %0, align 2
+  ret <8 x half> %1
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vldrhq_s16(i16* %base) {
+; CHECK-LABEL: test_vldrhq_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast i16* %base to <8 x i16>*
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
+  ret <8 x i16> %1
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vldrhq_s32(i16* %base) {
+; CHECK-LABEL: test_vldrhq_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.s32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast i16* %base to <4 x i16>*
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
+  %2 = sext <4 x i16> %1 to <4 x i32>
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vldrhq_u16(i16* %base) {
+; CHECK-LABEL: test_vldrhq_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast i16* %base to <8 x i16>*
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
+  ret <8 x i16> %1
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vldrhq_u32(i16* %base) {
+; CHECK-LABEL: test_vldrhq_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast i16* %base to <4 x i16>*
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
+  %2 = zext <4 x i16> %1 to <4 x i32>
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <8 x half> @test_vldrhq_z_f16(half* %base, i16 zeroext %p) {
+; CHECK-LABEL: test_vldrhq_z_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrht.u16 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast half* %base to <8 x half>*
+  %1 = zext i16 %p to i32
+  %2 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %1)
+  %3 = call <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>* %0, i32 2, <8 x i1> %2, <8 x half> zeroinitializer)
+  ret <8 x half> %3
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vldrhq_z_s16(i16* %base, i16 zeroext %p) {
+; CHECK-LABEL: test_vldrhq_z_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrht.u16 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast i16* %base to <8 x i16>*
+  %1 = zext i16 %p to i32
+  %2 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %1)
+  %3 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %0, i32 2, <8 x i1> %2, <8 x i16> zeroinitializer)
+  ret <8 x i16> %3
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vldrhq_z_s32(i16* %base, i16 zeroext %p) {
+; CHECK-LABEL: test_vldrhq_z_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrht.s32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast i16* %base to <4 x i16>*
+  %1 = zext i16 %p to i32
+  %2 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %1)
+  %3 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %0, i32 2, <4 x i1> %2, <4 x i16> zeroinitializer)
+  %4 = sext <4 x i16> %3 to <4 x i32>
+  ret <4 x i32> %4
+}
+
+declare <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>*, i32 immarg, <4 x i1>, <4 x i16>)
+
+define arm_aapcs_vfpcc <8 x i16> @test_vldrhq_z_u16(i16* %base, i16 zeroext %p) {
+; CHECK-LABEL: test_vldrhq_z_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrht.u16 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast i16* %base to <8 x i16>*
+  %1 = zext i16 %p to i32
+  %2 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %1)
+  %3 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %0, i32 2, <8 x i1> %2, <8 x i16> zeroinitializer)
+  ret <8 x i16> %3
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vldrhq_z_u32(i16* %base, i16 zeroext %p) {
+; CHECK-LABEL: test_vldrhq_z_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrht.u32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast i16* %base to <4 x i16>*
+  %1 = zext i16 %p to i32
+  %2 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %1)
+  %3 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %0, i32 2, <4 x i1> %2, <4 x i16> zeroinitializer)
+  %4 = zext <4 x i16> %3 to <4 x i32>
+  ret <4 x i32> %4
+}
+
+define arm_aapcs_vfpcc <4 x float> @test_vldrwq_f32(float* %base) {
+; CHECK-LABEL: test_vldrwq_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast float* %base to <4 x float>*
+  %1 = load <4 x float>, <4 x float>* %0, align 4
+  ret <4 x float> %1
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vldrwq_s32(i32* %base) {
+; CHECK-LABEL: test_vldrwq_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast i32* %base to <4 x i32>*
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
+  ret <4 x i32> %1
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vldrwq_u32(i32* %base) {
+; CHECK-LABEL: test_vldrwq_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast i32* %base to <4 x i32>*
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
+  ret <4 x i32> %1
+}
+
+define arm_aapcs_vfpcc <4 x float> @test_vldrwq_z_f32(float* %base, i16 zeroext %p) {
+; CHECK-LABEL: test_vldrwq_z_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrwt.u32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast float* %base to <4 x float>*
+  %1 = zext i16 %p to i32
+  %2 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %1)
+  %3 = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* %0, i32 4, <4 x i1> %2, <4 x float> zeroinitializer)
+  ret <4 x float> %3
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vldrwq_z_s32(i32* %base, i16 zeroext %p) {
+; CHECK-LABEL: test_vldrwq_z_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrwt.u32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast i32* %base to <4 x i32>*
+  %1 = zext i16 %p to i32
+  %2 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %1)
+  %3 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %0, i32 4, <4 x i1> %2, <4 x i32> zeroinitializer)
+  ret <4 x i32> %3
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vldrwq_z_u32(i32* %base, i16 zeroext %p) {
+; CHECK-LABEL: test_vldrwq_z_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrwt.u32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast i32* %base to <4 x i32>*
+  %1 = zext i16 %p to i32
+  %2 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %1)
+  %3 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %0, i32 4, <4 x i1> %2, <4 x i32> zeroinitializer)
+  ret <4 x i32> %3
+}
+
+define arm_aapcs_vfpcc void @test_vst1q_f16(half* %base, <8 x half> %value) {
+; CHECK-LABEL: test_vst1q_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vstrh.16 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast half* %base to <8 x half>*
+  store <8 x half> %value, <8 x half>* %0, align 2
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vst1q_f32(float* %base, <4 x float> %value) {
+; CHECK-LABEL: test_vst1q_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vstrw.32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast float* %base to <4 x float>*
+  store <4 x float> %value, <4 x float>* %0, align 4
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vst1q_s8(i8* %base, <16 x i8> %value) {
+; CHECK-LABEL: test_vst1q_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vstrb.8 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast i8* %base to <16 x i8>*
+  store <16 x i8> %value, <16 x i8>* %0, align 1
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vst1q_s16(i16* %base, <8 x i16> %value) {
+; CHECK-LABEL: test_vst1q_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vstrh.16 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast i16* %base to <8 x i16>*
+  store <8 x i16> %value, <8 x i16>* %0, align 2
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vst1q_s32(i32* %base, <4 x i32> %value) {
+; CHECK-LABEL: test_vst1q_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vstrw.32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast i32* %base to <4 x i32>*
+  store <4 x i32> %value, <4 x i32>* %0, align 4
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vst1q_u8(i8* %base, <16 x i8> %value) {
+; CHECK-LABEL: test_vst1q_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vstrb.8 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast i8* %base to <16 x i8>*
+  store <16 x i8> %value, <16 x i8>* %0, align 1
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vst1q_u16(i16* %base, <8 x i16> %value) {
+; CHECK-LABEL: test_vst1q_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vstrh.16 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast i16* %base to <8 x i16>*
+  store <8 x i16> %value, <8 x i16>* %0, align 2
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vst1q_u32(i32* %base, <4 x i32> %value) {
+; CHECK-LABEL: test_vst1q_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vstrw.32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast i32* %base to <4 x i32>*
+  store <4 x i32> %value, <4 x i32>* %0, align 4
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vst1q_p_f16(half* %base, <8 x half> %value, i16 zeroext %p) {
+; CHECK-LABEL: test_vst1q_p_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrht.16 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast half* %base to <8 x half>*
+  %1 = zext i16 %p to i32
+  %2 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %1)
+  call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %value, <8 x half>* %0, i32 2, <8 x i1> %2)
+  ret void
+}
+
+declare void @llvm.masked.store.v8f16.p0v8f16(<8 x half>, <8 x half>*, i32 immarg, <8 x i1>)
+
+define arm_aapcs_vfpcc void @test_vst1q_p_f32(float* %base, <4 x float> %value, i16 zeroext %p) {
+; CHECK-LABEL: test_vst1q_p_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrwt.32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast float* %base to <4 x float>*
+  %1 = zext i16 %p to i32
+  %2 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %1)
+  call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %value, <4 x float>* %0, i32 4, <4 x i1> %2)
+  ret void
+}
+
+declare void @llvm.masked.store.v4f32.p0v4f32(<4 x float>, <4 x float>*, i32 immarg, <4 x i1>)
+
+define arm_aapcs_vfpcc void @test_vst1q_p_s8(i8* %base, <16 x i8> %value, i16 zeroext %p) {
+; CHECK-LABEL: test_vst1q_p_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrbt.8 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast i8* %base to <16 x i8>*
+  %1 = zext i16 %p to i32
+  %2 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %1)
+  call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %value, <16 x i8>* %0, i32 1, <16 x i1> %2)
+  ret void
+}
+
+declare void @llvm.masked.store.v16i8.p0v16i8(<16 x i8>, <16 x i8>*, i32 immarg, <16 x i1>)
+
+define arm_aapcs_vfpcc void @test_vst1q_p_s16(i16* %base, <8 x i16> %value, i16 zeroext %p) {
+; CHECK-LABEL: test_vst1q_p_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrht.16 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast i16* %base to <8 x i16>*
+  %1 = zext i16 %p to i32
+  %2 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %1)
+  call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %value, <8 x i16>* %0, i32 2, <8 x i1> %2)
+  ret void
+}
+
+declare void @llvm.masked.store.v8i16.p0v8i16(<8 x i16>, <8 x i16>*, i32 immarg, <8 x i1>)
+
+define arm_aapcs_vfpcc void @test_vst1q_p_s32(i32* %base, <4 x i32> %value, i16 zeroext %p) {
+; CHECK-LABEL: test_vst1q_p_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrwt.32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast i32* %base to <4 x i32>*
+  %1 = zext i16 %p to i32
+  %2 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %1)
+  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %value, <4 x i32>* %0, i32 4, <4 x i1> %2)
+  ret void
+}
+
+declare void @llvm.masked.store.v4i32.p0v4i32(<4 x i32>, <4 x i32>*, i32 immarg, <4 x i1>)
+
+define arm_aapcs_vfpcc void @test_vst1q_p_u8(i8* %base, <16 x i8> %value, i16 zeroext %p) {
+; CHECK-LABEL: test_vst1q_p_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrbt.8 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast i8* %base to <16 x i8>*
+  %1 = zext i16 %p to i32
+  %2 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %1)
+  call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %value, <16 x i8>* %0, i32 1, <16 x i1> %2)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vst1q_p_u16(i16* %base, <8 x i16> %value, i16 zeroext %p) {
+; CHECK-LABEL: test_vst1q_p_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrht.16 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast i16* %base to <8 x i16>*
+  %1 = zext i16 %p to i32
+  %2 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %1)
+  call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %value, <8 x i16>* %0, i32 2, <8 x i1> %2)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vst1q_p_u32(i32* %base, <4 x i32> %value, i16 zeroext %p) {
+; CHECK-LABEL: test_vst1q_p_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrwt.32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast i32* %base to <4 x i32>*
+  %1 = zext i16 %p to i32
+  %2 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %1)
+  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %value, <4 x i32>* %0, i32 4, <4 x i1> %2)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vstrbq_s8(i8* %base, <16 x i8> %value) {
+; CHECK-LABEL: test_vstrbq_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vstrb.8 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast i8* %base to <16 x i8>*
+  store <16 x i8> %value, <16 x i8>* %0, align 1
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vstrbq_s16(i8* %base, <8 x i16> %value) {
+; CHECK-LABEL: test_vstrbq_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vstrb.16 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = trunc <8 x i16> %value to <8 x i8>
+  %1 = bitcast i8* %base to <8 x i8>*
+  store <8 x i8> %0, <8 x i8>* %1, align 1
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vstrbq_s32(i8* %base, <4 x i32> %value) {
+; CHECK-LABEL: test_vstrbq_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vstrb.32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = trunc <4 x i32> %value to <4 x i8>
+  %1 = bitcast i8* %base to <4 x i8>*
+  store <4 x i8> %0, <4 x i8>* %1, align 1
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vstrbq_u8(i8* %base, <16 x i8> %value) {
+; CHECK-LABEL: test_vstrbq_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vstrb.8 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast i8* %base to <16 x i8>*
+  store <16 x i8> %value, <16 x i8>* %0, align 1
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vstrbq_u16(i8* %base, <8 x i16> %value) {
+; CHECK-LABEL: test_vstrbq_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vstrb.16 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = trunc <8 x i16> %value to <8 x i8>
+  %1 = bitcast i8* %base to <8 x i8>*
+  store <8 x i8> %0, <8 x i8>* %1, align 1
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vstrbq_u32(i8* %base, <4 x i32> %value) {
+; CHECK-LABEL: test_vstrbq_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vstrb.32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = trunc <4 x i32> %value to <4 x i8>
+  %1 = bitcast i8* %base to <4 x i8>*
+  store <4 x i8> %0, <4 x i8>* %1, align 1
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vstrbq_p_s8(i8* %base, <16 x i8> %value, i16 zeroext %p) {
+; CHECK-LABEL: test_vstrbq_p_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrbt.8 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast i8* %base to <16 x i8>*
+  %1 = zext i16 %p to i32
+  %2 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %1)
+  call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %value, <16 x i8>* %0, i32 1, <16 x i1> %2)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vstrbq_p_s16(i8* %base, <8 x i16> %value, i16 zeroext %p) {
+; CHECK-LABEL: test_vstrbq_p_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrbt.16 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = trunc <8 x i16> %value to <8 x i8>
+  %1 = bitcast i8* %base to <8 x i8>*
+  %2 = zext i16 %p to i32
+  %3 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %2)
+  call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %0, <8 x i8>* %1, i32 1, <8 x i1> %3)
+  ret void
+}
+
+declare void @llvm.masked.store.v8i8.p0v8i8(<8 x i8>, <8 x i8>*, i32 immarg, <8 x i1>)
+
+define arm_aapcs_vfpcc void @test_vstrbq_p_s32(i8* %base, <4 x i32> %value, i16 zeroext %p) {
+; CHECK-LABEL: test_vstrbq_p_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrbt.32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = trunc <4 x i32> %value to <4 x i8>
+  %1 = bitcast i8* %base to <4 x i8>*
+  %2 = zext i16 %p to i32
+  %3 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %2)
+  call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %0, <4 x i8>* %1, i32 1, <4 x i1> %3)
+  ret void
+}
+
+declare void @llvm.masked.store.v4i8.p0v4i8(<4 x i8>, <4 x i8>*, i32 immarg, <4 x i1>)
+
+define arm_aapcs_vfpcc void @test_vstrbq_p_u8(i8* %base, <16 x i8> %value, i16 zeroext %p) {
+; CHECK-LABEL: test_vstrbq_p_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrbt.8 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast i8* %base to <16 x i8>*
+  %1 = zext i16 %p to i32
+  %2 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %1)
+  call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %value, <16 x i8>* %0, i32 1, <16 x i1> %2)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vstrbq_p_u16(i8* %base, <8 x i16> %value, i16 zeroext %p) {
+; CHECK-LABEL: test_vstrbq_p_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrbt.16 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = trunc <8 x i16> %value to <8 x i8>
+  %1 = bitcast i8* %base to <8 x i8>*
+  %2 = zext i16 %p to i32
+  %3 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %2)
+  call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %0, <8 x i8>* %1, i32 1, <8 x i1> %3)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vstrbq_p_u32(i8* %base, <4 x i32> %value, i16 zeroext %p) {
+; CHECK-LABEL: test_vstrbq_p_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrbt.32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = trunc <4 x i32> %value to <4 x i8>
+  %1 = bitcast i8* %base to <4 x i8>*
+  %2 = zext i16 %p to i32
+  %3 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %2)
+  call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %0, <4 x i8>* %1, i32 1, <4 x i1> %3)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vstrhq_f16(half* %base, <8 x half> %value) {
+; CHECK-LABEL: test_vstrhq_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vstrh.16 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast half* %base to <8 x half>*
+  store <8 x half> %value, <8 x half>* %0, align 2
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vstrhq_s16(i16* %base, <8 x i16> %value) {
+; CHECK-LABEL: test_vstrhq_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vstrh.16 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast i16* %base to <8 x i16>*
+  store <8 x i16> %value, <8 x i16>* %0, align 2
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vstrhq_s32(i16* %base, <4 x i32> %value) {
+; CHECK-LABEL: test_vstrhq_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vstrh.32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = trunc <4 x i32> %value to <4 x i16>
+  %1 = bitcast i16* %base to <4 x i16>*
+  store <4 x i16> %0, <4 x i16>* %1, align 2
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vstrhq_u16(i16* %base, <8 x i16> %value) {
+; CHECK-LABEL: test_vstrhq_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vstrh.16 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast i16* %base to <8 x i16>*
+  store <8 x i16> %value, <8 x i16>* %0, align 2
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vstrhq_u32(i16* %base, <4 x i32> %value) {
+; CHECK-LABEL: test_vstrhq_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vstrh.32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = trunc <4 x i32> %value to <4 x i16>
+  %1 = bitcast i16* %base to <4 x i16>*
+  store <4 x i16> %0, <4 x i16>* %1, align 2
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vstrhq_p_f16(half* %base, <8 x half> %value, i16 zeroext %p) {
+; CHECK-LABEL: test_vstrhq_p_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrht.16 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast half* %base to <8 x half>*
+  %1 = zext i16 %p to i32
+  %2 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %1)
+  call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %value, <8 x half>* %0, i32 2, <8 x i1> %2)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vstrhq_p_s16(i16* %base, <8 x i16> %value, i16 zeroext %p) {
+; CHECK-LABEL: test_vstrhq_p_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrht.16 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast i16* %base to <8 x i16>*
+  %1 = zext i16 %p to i32
+  %2 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %1)
+  call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %value, <8 x i16>* %0, i32 2, <8 x i1> %2)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vstrhq_p_s32(i16* %base, <4 x i32> %value, i16 zeroext %p) {
+; CHECK-LABEL: test_vstrhq_p_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrht.32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = trunc <4 x i32> %value to <4 x i16>
+  %1 = bitcast i16* %base to <4 x i16>*
+  %2 = zext i16 %p to i32
+  %3 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %2)
+  call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %0, <4 x i16>* %1, i32 2, <4 x i1> %3)
+  ret void
+}
+
+declare void @llvm.masked.store.v4i16.p0v4i16(<4 x i16>, <4 x i16>*, i32 immarg, <4 x i1>)
+
+define arm_aapcs_vfpcc void @test_vstrhq_p_u16(i16* %base, <8 x i16> %value, i16 zeroext %p) {
+; CHECK-LABEL: test_vstrhq_p_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrht.16 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast i16* %base to <8 x i16>*
+  %1 = zext i16 %p to i32
+  %2 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %1)
+  call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %value, <8 x i16>* %0, i32 2, <8 x i1> %2)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vstrhq_p_u32(i16* %base, <4 x i32> %value, i16 zeroext %p) {
+; CHECK-LABEL: test_vstrhq_p_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrht.32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = trunc <4 x i32> %value to <4 x i16>
+  %1 = bitcast i16* %base to <4 x i16>*
+  %2 = zext i16 %p to i32
+  %3 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %2)
+  call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %0, <4 x i16>* %1, i32 2, <4 x i1> %3)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vstrwq_f32(float* %base, <4 x float> %value) {
+; CHECK-LABEL: test_vstrwq_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vstrw.32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast float* %base to <4 x float>*
+  store <4 x float> %value, <4 x float>* %0, align 4
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vstrwq_s32(i32* %base, <4 x i32> %value) {
+; CHECK-LABEL: test_vstrwq_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vstrw.32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast i32* %base to <4 x i32>*
+  store <4 x i32> %value, <4 x i32>* %0, align 4
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vstrwq_u32(i32* %base, <4 x i32> %value) {
+; CHECK-LABEL: test_vstrwq_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vstrw.32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast i32* %base to <4 x i32>*
+  store <4 x i32> %value, <4 x i32>* %0, align 4
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vstrwq_p_f32(float* %base, <4 x float> %value, i16 zeroext %p) {
+; CHECK-LABEL: test_vstrwq_p_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrwt.32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast float* %base to <4 x float>*
+  %1 = zext i16 %p to i32
+  %2 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %1)
+  call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %value, <4 x float>* %0, i32 4, <4 x i1> %2)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vstrwq_p_s32(i32* %base, <4 x i32> %value, i16 zeroext %p) {
+; CHECK-LABEL: test_vstrwq_p_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrwt.32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast i32* %base to <4 x i32>*
+  %1 = zext i16 %p to i32
+  %2 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %1)
+  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %value, <4 x i32>* %0, i32 4, <4 x i1> %2)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vstrwq_p_u32(i32* %base, <4 x i32> %value, i16 zeroext %p) {
+; CHECK-LABEL: test_vstrwq_p_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrwt.32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast i32* %base to <4 x i32>*
+  %1 = zext i16 %p to i32
+  %2 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %1)
+  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %value, <4 x i32>* %0, i32 4, <4 x i1> %2)
+  ret void
+}
diff --git a/llvm/test/CodeGen/Thumb2/mve-intrinsics/predicates.ll b/llvm/test/CodeGen/Thumb2/mve-intrinsics/predicates.ll
new file mode 100644
index 000000000..f5b541203
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-intrinsics/predicates.ll
@@ -0,0 +1,219 @@
+; RUN: opt -instcombine %s | llc -mtriple=thumbv8.1m.main -mattr=+mve.fp -verify-machineinstrs -o - | FileCheck %s
+
+declare <16 x i1> @llvm.arm.mve.vctp8(i32)
+declare <8 x i1> @llvm.arm.mve.vctp16(i32)
+declare <4 x i1> @llvm.arm.mve.vctp32(i32)
+declare <4 x i1> @llvm.arm.mve.vctp64(i32)
+
+declare i32 @llvm.arm.mve.pred.v2i.v4i1(<4 x i1>)
+declare i32 @llvm.arm.mve.pred.v2i.v8i1(<8 x i1>)
+declare i32 @llvm.arm.mve.pred.v2i.v16i1(<16 x i1>)
+
+declare <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32)
+declare <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32)
+declare <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32)
+
+define arm_aapcs_vfpcc zeroext i16 @test_vctp8q(i32 %a) {
+; CHECK-LABEL: test_vctp8q:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vmrs r0, p0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <16 x i1> @llvm.arm.mve.vctp8(i32 %a)
+  %1 = call i32 @llvm.arm.mve.pred.v2i.v16i1(<16 x i1> %0)
+  %2 = trunc i32 %1 to i16
+  ret i16 %2
+}
+
+define arm_aapcs_vfpcc zeroext i16 @test_vctp8q_m(i32 %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vctp8q_m:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vctpt.8 r0
+; CHECK-NEXT:    vmrs r0, p0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = call <16 x i1> @llvm.arm.mve.vctp8(i32 %a)
+  %3 = and <16 x i1> %1, %2
+  %4 = call i32 @llvm.arm.mve.pred.v2i.v16i1(<16 x i1> %3)
+  %5 = trunc i32 %4 to i16
+  ret i16 %5
+}
+
+define arm_aapcs_vfpcc zeroext i16 @test_vctp16q(i32 %a) {
+; CHECK-LABEL: test_vctp16q:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vmrs r0, p0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i1> @llvm.arm.mve.vctp16(i32 %a)
+  %1 = call i32 @llvm.arm.mve.pred.v2i.v8i1(<8 x i1> %0)
+  %2 = trunc i32 %1 to i16
+  ret i16 %2
+}
+
+define arm_aapcs_vfpcc zeroext i16 @test_vctp16q_m(i32 %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vctp16q_m:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vctpt.16 r0
+; CHECK-NEXT:    vmrs r0, p0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <8 x i1> @llvm.arm.mve.vctp16(i32 %a)
+  %3 = and <8 x i1> %1, %2
+  %4 = call i32 @llvm.arm.mve.pred.v2i.v8i1(<8 x i1> %3)
+  %5 = trunc i32 %4 to i16
+  ret i16 %5
+}
+
+define arm_aapcs_vfpcc zeroext i16 @test_vctp32q(i32 %a) {
+; CHECK-LABEL: test_vctp32q:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vmrs r0, p0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x i1> @llvm.arm.mve.vctp32(i32 %a)
+  %1 = call i32 @llvm.arm.mve.pred.v2i.v4i1(<4 x i1> %0)
+  %2 = trunc i32 %1 to i16
+  ret i16 %2
+}
+
+define arm_aapcs_vfpcc zeroext i16 @test_vctp32q_m(i32 %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vctp32q_m:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vctpt.32 r0
+; CHECK-NEXT:    vmrs r0, p0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x i1> @llvm.arm.mve.vctp32(i32 %a)
+  %3 = and <4 x i1> %1, %2
+  %4 = call i32 @llvm.arm.mve.pred.v2i.v4i1(<4 x i1> %3)
+  %5 = trunc i32 %4 to i16
+  ret i16 %5
+}
+
+define arm_aapcs_vfpcc zeroext i16 @test_vctp64q(i32 %a) {
+; CHECK-LABEL: test_vctp64q:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.64 r0
+; CHECK-NEXT:    vmrs r0, p0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x i1> @llvm.arm.mve.vctp64(i32 %a)
+  %1 = call i32 @llvm.arm.mve.pred.v2i.v4i1(<4 x i1> %0)
+  %2 = trunc i32 %1 to i16
+  ret i16 %2
+}
+
+define arm_aapcs_vfpcc zeroext i16 @test_vctp64q_m(i32 %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vctp64q_m:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vctpt.64 r0
+; CHECK-NEXT:    vmrs r0, p0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x i1> @llvm.arm.mve.vctp64(i32 %a)
+  %3 = and <4 x i1> %1, %2
+  %4 = call i32 @llvm.arm.mve.pred.v2i.v4i1(<4 x i1> %3)
+  %5 = trunc i32 %4 to i16
+  ret i16 %5
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vpselq_i8(<16 x i8> %a, <16 x i8> %b, i16 zeroext %p) #2 {
+; CHECK-LABEL: test_vpselq_i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = select <16 x i1> %1, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vpselq_i16(<8 x i16> %a, <8 x i16> %b, i16 zeroext %p) #2 {
+; CHECK-LABEL: test_vpselq_i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = select <8 x i1> %1, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <8 x half> @test_vpselq_f16(<8 x half> %a, <8 x half> %b, i16 zeroext %p) #2 {
+; CHECK-LABEL: test_vpselq_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = select <8 x i1> %1, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vpselq_i32(<4 x i32> %a, <4 x i32> %b, i16 zeroext %p) #2 {
+; CHECK-LABEL: test_vpselq_i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = select <4 x i1> %1, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <4 x float> @test_vpselq_f32(<4 x float> %a, <4 x float> %b, i16 zeroext %p) #2 {
+; CHECK-LABEL: test_vpselq_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = select <4 x i1> %1, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %2
+}
+
+define arm_aapcs_vfpcc <2 x i64> @test_vpselq_i64(<2 x i64> %a, <2 x i64> %b, i16 zeroext %p) #2 {
+; CHECK-LABEL: test_vpselq_i64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = bitcast <2 x i64> %a to <4 x i32>
+  %3 = bitcast <2 x i64> %b to <4 x i32>
+  %4 = select <4 x i1> %1, <4 x i32> %2, <4 x i32> %3
+  %5 = bitcast <4 x i32> %4 to <2 x i64>
+  ret <2 x i64> %5
+}
diff --git a/llvm/test/CodeGen/Thumb2/mve-intrinsics/scalar-shifts.ll b/llvm/test/CodeGen/Thumb2/mve-intrinsics/scalar-shifts.ll
new file mode 100644
index 000000000..14326a9ef
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-intrinsics/scalar-shifts.ll
@@ -0,0 +1,280 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve -verify-machineinstrs -o - %s | FileCheck %s
+
+define i64 @test_asrl(i64 %value, i32 %shift) {
+; CHECK-LABEL: test_asrl:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    asrl r0, r1, r2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = lshr i64 %value, 32
+  %1 = trunc i64 %0 to i32
+  %2 = trunc i64 %value to i32
+  %3 = call { i32, i32 } @llvm.arm.mve.asrl(i32 %2, i32 %1, i32 %shift)
+  %4 = extractvalue { i32, i32 } %3, 1
+  %5 = zext i32 %4 to i64
+  %6 = shl i64 %5, 32
+  %7 = extractvalue { i32, i32 } %3, 0
+  %8 = zext i32 %7 to i64
+  %9 = or i64 %6, %8
+  ret i64 %9
+}
+
+declare { i32, i32 } @llvm.arm.mve.asrl(i32, i32, i32)
+
+define i64 @test_lsll(i64 %value, i32 %shift) {
+; CHECK-LABEL: test_lsll:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    lsll r0, r1, r2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = lshr i64 %value, 32
+  %1 = trunc i64 %0 to i32
+  %2 = trunc i64 %value to i32
+  %3 = call { i32, i32 } @llvm.arm.mve.lsll(i32 %2, i32 %1, i32 %shift)
+  %4 = extractvalue { i32, i32 } %3, 1
+  %5 = zext i32 %4 to i64
+  %6 = shl i64 %5, 32
+  %7 = extractvalue { i32, i32 } %3, 0
+  %8 = zext i32 %7 to i64
+  %9 = or i64 %6, %8
+  ret i64 %9
+}
+
+declare { i32, i32 } @llvm.arm.mve.lsll(i32, i32, i32)
+
+define i32 @test_sqrshr(i32 %value, i32 %shift) {
+; CHECK-LABEL: test_sqrshr:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    sqrshr r0, r1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call i32 @llvm.arm.mve.sqrshr(i32 %value, i32 %shift)
+  ret i32 %0
+}
+
+declare i32 @llvm.arm.mve.sqrshr(i32, i32)
+
+define i64 @test_sqrshrl(i64 %value, i32 %shift) {
+; CHECK-LABEL: test_sqrshrl:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    sqrshrl r0, r1, #64, r2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = lshr i64 %value, 32
+  %1 = trunc i64 %0 to i32
+  %2 = trunc i64 %value to i32
+  %3 = call { i32, i32 } @llvm.arm.mve.sqrshrl(i32 %2, i32 %1, i32 %shift, i32 64)
+  %4 = extractvalue { i32, i32 } %3, 1
+  %5 = zext i32 %4 to i64
+  %6 = shl i64 %5, 32
+  %7 = extractvalue { i32, i32 } %3, 0
+  %8 = zext i32 %7 to i64
+  %9 = or i64 %6, %8
+  ret i64 %9
+}
+
+declare { i32, i32 } @llvm.arm.mve.sqrshrl(i32, i32, i32, i32)
+
+define i64 @test_sqrshrl_sat48(i64 %value, i32 %shift) {
+; CHECK-LABEL: test_sqrshrl_sat48:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    sqrshrl r0, r1, #48, r2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = lshr i64 %value, 32
+  %1 = trunc i64 %0 to i32
+  %2 = trunc i64 %value to i32
+  %3 = call { i32, i32 } @llvm.arm.mve.sqrshrl(i32 %2, i32 %1, i32 %shift, i32 48)
+  %4 = extractvalue { i32, i32 } %3, 1
+  %5 = zext i32 %4 to i64
+  %6 = shl i64 %5, 32
+  %7 = extractvalue { i32, i32 } %3, 0
+  %8 = zext i32 %7 to i64
+  %9 = or i64 %6, %8
+  ret i64 %9
+}
+
+define i32 @test_sqshl(i32 %value) {
+; CHECK-LABEL: test_sqshl:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    sqshl r0, #2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call i32 @llvm.arm.mve.sqshl(i32 %value, i32 2)
+  ret i32 %0
+}
+
+declare i32 @llvm.arm.mve.sqshl(i32, i32)
+
+define i64 @test_sqshll(i64 %value) {
+; CHECK-LABEL: test_sqshll:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    sqshll r0, r1, #17
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = lshr i64 %value, 32
+  %1 = trunc i64 %0 to i32
+  %2 = trunc i64 %value to i32
+  %3 = call { i32, i32 } @llvm.arm.mve.sqshll(i32 %2, i32 %1, i32 17)
+  %4 = extractvalue { i32, i32 } %3, 1
+  %5 = zext i32 %4 to i64
+  %6 = shl i64 %5, 32
+  %7 = extractvalue { i32, i32 } %3, 0
+  %8 = zext i32 %7 to i64
+  %9 = or i64 %6, %8
+  ret i64 %9
+}
+
+declare { i32, i32 } @llvm.arm.mve.sqshll(i32, i32, i32)
+
+define i32 @test_srshr(i32 %value) {
+; CHECK-LABEL: test_srshr:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    srshr r0, #6
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call i32 @llvm.arm.mve.srshr(i32 %value, i32 6)
+  ret i32 %0
+}
+
+declare i32 @llvm.arm.mve.srshr(i32, i32)
+
+define i64 @test_srshrl(i64 %value) {
+; CHECK-LABEL: test_srshrl:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    srshrl r0, r1, #26
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = lshr i64 %value, 32
+  %1 = trunc i64 %0 to i32
+  %2 = trunc i64 %value to i32
+  %3 = call { i32, i32 } @llvm.arm.mve.srshrl(i32 %2, i32 %1, i32 26)
+  %4 = extractvalue { i32, i32 } %3, 1
+  %5 = zext i32 %4 to i64
+  %6 = shl i64 %5, 32
+  %7 = extractvalue { i32, i32 } %3, 0
+  %8 = zext i32 %7 to i64
+  %9 = or i64 %6, %8
+  ret i64 %9
+}
+
+declare { i32, i32 } @llvm.arm.mve.srshrl(i32, i32, i32)
+
+define i32 @test_uqrshl(i32 %value, i32 %shift) {
+; CHECK-LABEL: test_uqrshl:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    uqrshl r0, r1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call i32 @llvm.arm.mve.uqrshl(i32 %value, i32 %shift)
+  ret i32 %0
+}
+
+declare i32 @llvm.arm.mve.uqrshl(i32, i32)
+
+define i64 @test_uqrshll(i64 %value, i32 %shift) {
+; CHECK-LABEL: test_uqrshll:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    uqrshll r0, r1, #64, r2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = lshr i64 %value, 32
+  %1 = trunc i64 %0 to i32
+  %2 = trunc i64 %value to i32
+  %3 = call { i32, i32 } @llvm.arm.mve.uqrshll(i32 %2, i32 %1, i32 %shift, i32 64)
+  %4 = extractvalue { i32, i32 } %3, 1
+  %5 = zext i32 %4 to i64
+  %6 = shl i64 %5, 32
+  %7 = extractvalue { i32, i32 } %3, 0
+  %8 = zext i32 %7 to i64
+  %9 = or i64 %6, %8
+  ret i64 %9
+}
+
+declare { i32, i32 } @llvm.arm.mve.uqrshll(i32, i32, i32, i32)
+
+define i64 @test_uqrshll_sat48(i64 %value, i32 %shift) {
+; CHECK-LABEL: test_uqrshll_sat48:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    uqrshll r0, r1, #48, r2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = lshr i64 %value, 32
+  %1 = trunc i64 %0 to i32
+  %2 = trunc i64 %value to i32
+  %3 = call { i32, i32 } @llvm.arm.mve.uqrshll(i32 %2, i32 %1, i32 %shift, i32 48)
+  %4 = extractvalue { i32, i32 } %3, 1
+  %5 = zext i32 %4 to i64
+  %6 = shl i64 %5, 32
+  %7 = extractvalue { i32, i32 } %3, 0
+  %8 = zext i32 %7 to i64
+  %9 = or i64 %6, %8
+  ret i64 %9
+}
+
+define i32 @test_uqshl(i32 %value) {
+; CHECK-LABEL: test_uqshl:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    uqshl r0, #21
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call i32 @llvm.arm.mve.uqshl(i32 %value, i32 21)
+  ret i32 %0
+}
+
+declare i32 @llvm.arm.mve.uqshl(i32, i32)
+
+define i64 @test_uqshll(i64 %value) {
+; CHECK-LABEL: test_uqshll:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    uqshll r0, r1, #16
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = lshr i64 %value, 32
+  %1 = trunc i64 %0 to i32
+  %2 = trunc i64 %value to i32
+  %3 = call { i32, i32 } @llvm.arm.mve.uqshll(i32 %2, i32 %1, i32 16)
+  %4 = extractvalue { i32, i32 } %3, 1
+  %5 = zext i32 %4 to i64
+  %6 = shl i64 %5, 32
+  %7 = extractvalue { i32, i32 } %3, 0
+  %8 = zext i32 %7 to i64
+  %9 = or i64 %6, %8
+  ret i64 %9
+}
+
+declare { i32, i32 } @llvm.arm.mve.uqshll(i32, i32, i32)
+
+define i32 @test_urshr(i32 %value) {
+; CHECK-LABEL: test_urshr:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    urshr r0, #22
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call i32 @llvm.arm.mve.urshr(i32 %value, i32 22)
+  ret i32 %0
+}
+
+declare i32 @llvm.arm.mve.urshr(i32, i32)
+
+define i64 @test_urshrl(i64 %value) {
+; CHECK-LABEL: test_urshrl:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    urshrl r0, r1, #6
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = lshr i64 %value, 32
+  %1 = trunc i64 %0 to i32
+  %2 = trunc i64 %value to i32
+  %3 = call { i32, i32 } @llvm.arm.mve.urshrl(i32 %2, i32 %1, i32 6)
+  %4 = extractvalue { i32, i32 } %3, 1
+  %5 = zext i32 %4 to i64
+  %6 = shl i64 %5, 32
+  %7 = extractvalue { i32, i32 } %3, 0
+  %8 = zext i32 %7 to i64
+  %9 = or i64 %6, %8
+  ret i64 %9
+}
+
+declare { i32, i32 } @llvm.arm.mve.urshrl(i32, i32, i32)
diff --git a/llvm/test/CodeGen/Thumb2/mve-intrinsics/scatter-gather.ll b/llvm/test/CodeGen/Thumb2/mve-intrinsics/scatter-gather.ll
new file mode 100644
index 000000000..7eac79094
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-intrinsics/scatter-gather.ll
@@ -0,0 +1,2018 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve.fp -verify-machineinstrs -o - %s | FileCheck %s
+
+define arm_aapcs_vfpcc <8 x i16> @test_vldrbq_gather_offset_s16(i8* %base, <8 x i16> %offset) {
+; CHECK-LABEL: test_vldrbq_gather_offset_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.s16 q1, [r0, q0]
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.arm.mve.vldr.gather.offset.v8i16.p0i8.v8i16(i8* %base, <8 x i16> %offset, i32 8, i32 0, i32 0)
+  ret <8 x i16> %0
+}
+
+declare <8 x i16> @llvm.arm.mve.vldr.gather.offset.v8i16.p0i8.v8i16(i8*, <8 x i16>, i32, i32, i32)
+
+define arm_aapcs_vfpcc <4 x i32> @test_vldrbq_gather_offset_s32(i8* %base, <4 x i32> %offset) {
+; CHECK-LABEL: test_vldrbq_gather_offset_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.s32 q1, [r0, q0]
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x i32> @llvm.arm.mve.vldr.gather.offset.v4i32.p0i8.v4i32(i8* %base, <4 x i32> %offset, i32 8, i32 0, i32 0)
+  ret <4 x i32> %0
+}
+
+declare <4 x i32> @llvm.arm.mve.vldr.gather.offset.v4i32.p0i8.v4i32(i8*, <4 x i32>, i32, i32, i32)
+
+define arm_aapcs_vfpcc <16 x i8> @test_vldrbq_gather_offset_s8(i8* %base, <16 x i8> %offset) {
+; CHECK-LABEL: test_vldrbq_gather_offset_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u8 q1, [r0, q0]
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <16 x i8> @llvm.arm.mve.vldr.gather.offset.v16i8.p0i8.v16i8(i8* %base, <16 x i8> %offset, i32 8, i32 0, i32 0)
+  ret <16 x i8> %0
+}
+
+declare <16 x i8> @llvm.arm.mve.vldr.gather.offset.v16i8.p0i8.v16i8(i8*, <16 x i8>, i32, i32, i32)
+
+define arm_aapcs_vfpcc <8 x i16> @test_vldrbq_gather_offset_u16(i8* %base, <8 x i16> %offset) {
+; CHECK-LABEL: test_vldrbq_gather_offset_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u16 q1, [r0, q0]
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.arm.mve.vldr.gather.offset.v8i16.p0i8.v8i16(i8* %base, <8 x i16> %offset, i32 8, i32 0, i32 1)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vldrbq_gather_offset_u32(i8* %base, <4 x i32> %offset) {
+; CHECK-LABEL: test_vldrbq_gather_offset_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u32 q1, [r0, q0]
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x i32> @llvm.arm.mve.vldr.gather.offset.v4i32.p0i8.v4i32(i8* %base, <4 x i32> %offset, i32 8, i32 0, i32 1)
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vldrbq_gather_offset_u8(i8* %base, <16 x i8> %offset) {
+; CHECK-LABEL: test_vldrbq_gather_offset_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u8 q1, [r0, q0]
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <16 x i8> @llvm.arm.mve.vldr.gather.offset.v16i8.p0i8.v16i8(i8* %base, <16 x i8> %offset, i32 8, i32 0, i32 1)
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vldrbq_gather_offset_z_s16(i8* %base, <8 x i16> %offset, i16 zeroext %p) {
+; CHECK-LABEL: test_vldrbq_gather_offset_z_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrbt.s16 q1, [r0, q0]
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <8 x i16> @llvm.arm.mve.vldr.gather.offset.predicated.v8i16.p0i8.v8i16.v8i1(i8* %base, <8 x i16> %offset, i32 8, i32 0, i32 0, <8 x i1> %1)
+  ret <8 x i16> %2
+}
+
+declare <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32)
+
+declare <8 x i16> @llvm.arm.mve.vldr.gather.offset.predicated.v8i16.p0i8.v8i16.v8i1(i8*, <8 x i16>, i32, i32, i32, <8 x i1>)
+
+define arm_aapcs_vfpcc <4 x i32> @test_vldrbq_gather_offset_z_s32(i8* %base, <4 x i32> %offset, i16 zeroext %p) {
+; CHECK-LABEL: test_vldrbq_gather_offset_z_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrbt.s32 q1, [r0, q0]
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x i32> @llvm.arm.mve.vldr.gather.offset.predicated.v4i32.p0i8.v4i32.v4i1(i8* %base, <4 x i32> %offset, i32 8, i32 0, i32 0, <4 x i1> %1)
+  ret <4 x i32> %2
+}
+
+declare <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32)
+
+declare <4 x i32> @llvm.arm.mve.vldr.gather.offset.predicated.v4i32.p0i8.v4i32.v4i1(i8*, <4 x i32>, i32, i32, i32, <4 x i1>)
+
+define arm_aapcs_vfpcc <16 x i8> @test_vldrbq_gather_offset_z_s8(i8* %base, <16 x i8> %offset, i16 zeroext %p) {
+; CHECK-LABEL: test_vldrbq_gather_offset_z_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrbt.u8 q1, [r0, q0]
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = call <16 x i8> @llvm.arm.mve.vldr.gather.offset.predicated.v16i8.p0i8.v16i8.v16i1(i8* %base, <16 x i8> %offset, i32 8, i32 0, i32 0, <16 x i1> %1)
+  ret <16 x i8> %2
+}
+
+declare <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32)
+
+declare <16 x i8> @llvm.arm.mve.vldr.gather.offset.predicated.v16i8.p0i8.v16i8.v16i1(i8*, <16 x i8>, i32, i32, i32, <16 x i1>)
+
+define arm_aapcs_vfpcc <8 x i16> @test_vldrbq_gather_offset_z_u16(i8* %base, <8 x i16> %offset, i16 zeroext %p) {
+; CHECK-LABEL: test_vldrbq_gather_offset_z_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrbt.u16 q1, [r0, q0]
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <8 x i16> @llvm.arm.mve.vldr.gather.offset.predicated.v8i16.p0i8.v8i16.v8i1(i8* %base, <8 x i16> %offset, i32 8, i32 0, i32 1, <8 x i1> %1)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vldrbq_gather_offset_z_u32(i8* %base, <4 x i32> %offset, i16 zeroext %p) {
+; CHECK-LABEL: test_vldrbq_gather_offset_z_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrbt.u32 q1, [r0, q0]
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x i32> @llvm.arm.mve.vldr.gather.offset.predicated.v4i32.p0i8.v4i32.v4i1(i8* %base, <4 x i32> %offset, i32 8, i32 0, i32 1, <4 x i1> %1)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vldrbq_gather_offset_z_u8(i8* %base, <16 x i8> %offset, i16 zeroext %p) {
+; CHECK-LABEL: test_vldrbq_gather_offset_z_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrbt.u8 q1, [r0, q0]
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = call <16 x i8> @llvm.arm.mve.vldr.gather.offset.predicated.v16i8.p0i8.v16i8.v16i1(i8* %base, <16 x i8> %offset, i32 8, i32 0, i32 1, <16 x i1> %1)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <2 x i64> @test_vldrdq_gather_base_s64(<2 x i64> %addr) {
+; CHECK-LABEL: test_vldrdq_gather_base_s64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrd.u64 q1, [q0, #616]
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <2 x i64> @llvm.arm.mve.vldr.gather.base.v2i64.v2i64(<2 x i64> %addr, i32 616)
+  ret <2 x i64> %0
+}
+
+declare <2 x i64> @llvm.arm.mve.vldr.gather.base.v2i64.v2i64(<2 x i64>, i32)
+
+define arm_aapcs_vfpcc <2 x i64> @test_vldrdq_gather_base_u64(<2 x i64> %addr) {
+; CHECK-LABEL: test_vldrdq_gather_base_u64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrd.u64 q1, [q0, #-336]
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <2 x i64> @llvm.arm.mve.vldr.gather.base.v2i64.v2i64(<2 x i64> %addr, i32 -336)
+  ret <2 x i64> %0
+}
+
+define arm_aapcs_vfpcc <2 x i64> @test_vldrdq_gather_base_wb_s64(<2 x i64>* %addr) {
+; CHECK-LABEL: test_vldrdq_gather_base_wb_s64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vldrd.u64 q1, [q0, #576]!
+; CHECK-NEXT:    vstrw.32 q1, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = load <2 x i64>, <2 x i64>* %addr, align 8
+  %1 = call { <2 x i64>, <2 x i64> } @llvm.arm.mve.vldr.gather.base.wb.v2i64.v2i64(<2 x i64> %0, i32 576)
+  %2 = extractvalue { <2 x i64>, <2 x i64> } %1, 1
+  store <2 x i64> %2, <2 x i64>* %addr, align 8
+  %3 = extractvalue { <2 x i64>, <2 x i64> } %1, 0
+  ret <2 x i64> %3
+}
+
+declare { <2 x i64>, <2 x i64> } @llvm.arm.mve.vldr.gather.base.wb.v2i64.v2i64(<2 x i64>, i32)
+
+define arm_aapcs_vfpcc <2 x i64> @test_vldrdq_gather_base_wb_u64(<2 x i64>* %addr) {
+; CHECK-LABEL: test_vldrdq_gather_base_wb_u64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vldrd.u64 q1, [q0, #-328]!
+; CHECK-NEXT:    vstrw.32 q1, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = load <2 x i64>, <2 x i64>* %addr, align 8
+  %1 = call { <2 x i64>, <2 x i64> } @llvm.arm.mve.vldr.gather.base.wb.v2i64.v2i64(<2 x i64> %0, i32 -328)
+  %2 = extractvalue { <2 x i64>, <2 x i64> } %1, 1
+  store <2 x i64> %2, <2 x i64>* %addr, align 8
+  %3 = extractvalue { <2 x i64>, <2 x i64> } %1, 0
+  ret <2 x i64> %3
+}
+
+define arm_aapcs_vfpcc <2 x i64> @test_vldrdq_gather_base_wb_z_s64(<2 x i64>* %addr, i16 zeroext %p) {
+; CHECK-LABEL: test_vldrdq_gather_base_wb_z_s64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrdt.u64 q1, [q0, #664]!
+; CHECK-NEXT:    vstrw.32 q1, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = load <2 x i64>, <2 x i64>* %addr, align 8
+  %1 = zext i16 %p to i32
+  %2 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %1)
+  %3 = call { <2 x i64>, <2 x i64> } @llvm.arm.mve.vldr.gather.base.wb.predicated.v2i64.v2i64.v4i1(<2 x i64> %0, i32 664, <4 x i1> %2)
+  %4 = extractvalue { <2 x i64>, <2 x i64> } %3, 1
+  store <2 x i64> %4, <2 x i64>* %addr, align 8
+  %5 = extractvalue { <2 x i64>, <2 x i64> } %3, 0
+  ret <2 x i64> %5
+}
+
+declare { <2 x i64>, <2 x i64> } @llvm.arm.mve.vldr.gather.base.wb.predicated.v2i64.v2i64.v4i1(<2 x i64>, i32, <4 x i1>)
+
+define arm_aapcs_vfpcc <2 x i64> @test_vldrdq_gather_base_wb_z_u64(<2 x i64>* %addr, i16 zeroext %p) {
+; CHECK-LABEL: test_vldrdq_gather_base_wb_z_u64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrdt.u64 q1, [q0, #656]!
+; CHECK-NEXT:    vstrw.32 q1, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = load <2 x i64>, <2 x i64>* %addr, align 8
+  %1 = zext i16 %p to i32
+  %2 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %1)
+  %3 = call { <2 x i64>, <2 x i64> } @llvm.arm.mve.vldr.gather.base.wb.predicated.v2i64.v2i64.v4i1(<2 x i64> %0, i32 656, <4 x i1> %2)
+  %4 = extractvalue { <2 x i64>, <2 x i64> } %3, 1
+  store <2 x i64> %4, <2 x i64>* %addr, align 8
+  %5 = extractvalue { <2 x i64>, <2 x i64> } %3, 0
+  ret <2 x i64> %5
+}
+
+define arm_aapcs_vfpcc <2 x i64> @test_vldrdq_gather_base_z_s64(<2 x i64> %addr, i16 zeroext %p) {
+; CHECK-LABEL: test_vldrdq_gather_base_z_s64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrdt.u64 q1, [q0, #888]
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <2 x i64> @llvm.arm.mve.vldr.gather.base.predicated.v2i64.v2i64.v4i1(<2 x i64> %addr, i32 888, <4 x i1> %1)
+  ret <2 x i64> %2
+}
+
+declare <2 x i64> @llvm.arm.mve.vldr.gather.base.predicated.v2i64.v2i64.v4i1(<2 x i64>, i32, <4 x i1>)
+
+define arm_aapcs_vfpcc <2 x i64> @test_vldrdq_gather_base_z_u64(<2 x i64> %addr, i16 zeroext %p) {
+; CHECK-LABEL: test_vldrdq_gather_base_z_u64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrdt.u64 q1, [q0, #-1000]
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <2 x i64> @llvm.arm.mve.vldr.gather.base.predicated.v2i64.v2i64.v4i1(<2 x i64> %addr, i32 -1000, <4 x i1> %1)
+  ret <2 x i64> %2
+}
+
+define arm_aapcs_vfpcc <2 x i64> @test_vldrdq_gather_offset_s64(i64* %base, <2 x i64> %offset) {
+; CHECK-LABEL: test_vldrdq_gather_offset_s64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrd.u64 q1, [r0, q0]
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <2 x i64> @llvm.arm.mve.vldr.gather.offset.v2i64.p0i64.v2i64(i64* %base, <2 x i64> %offset, i32 64, i32 0, i32 0)
+  ret <2 x i64> %0
+}
+
+declare <2 x i64> @llvm.arm.mve.vldr.gather.offset.v2i64.p0i64.v2i64(i64*, <2 x i64>, i32, i32, i32)
+
+define arm_aapcs_vfpcc <2 x i64> @test_vldrdq_gather_offset_u64(i64* %base, <2 x i64> %offset) {
+; CHECK-LABEL: test_vldrdq_gather_offset_u64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrd.u64 q1, [r0, q0]
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <2 x i64> @llvm.arm.mve.vldr.gather.offset.v2i64.p0i64.v2i64(i64* %base, <2 x i64> %offset, i32 64, i32 0, i32 1)
+  ret <2 x i64> %0
+}
+
+define arm_aapcs_vfpcc <2 x i64> @test_vldrdq_gather_offset_z_s64(i64* %base, <2 x i64> %offset, i16 zeroext %p) {
+; CHECK-LABEL: test_vldrdq_gather_offset_z_s64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrdt.u64 q1, [r0, q0]
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <2 x i64> @llvm.arm.mve.vldr.gather.offset.predicated.v2i64.p0i64.v2i64.v4i1(i64* %base, <2 x i64> %offset, i32 64, i32 0, i32 0, <4 x i1> %1)
+  ret <2 x i64> %2
+}
+
+declare <2 x i64> @llvm.arm.mve.vldr.gather.offset.predicated.v2i64.p0i64.v2i64.v4i1(i64*, <2 x i64>, i32, i32, i32, <4 x i1>)
+
+define arm_aapcs_vfpcc <2 x i64> @test_vldrdq_gather_offset_z_u64(i64* %base, <2 x i64> %offset, i16 zeroext %p) {
+; CHECK-LABEL: test_vldrdq_gather_offset_z_u64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrdt.u64 q1, [r0, q0]
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <2 x i64> @llvm.arm.mve.vldr.gather.offset.predicated.v2i64.p0i64.v2i64.v4i1(i64* %base, <2 x i64> %offset, i32 64, i32 0, i32 1, <4 x i1> %1)
+  ret <2 x i64> %2
+}
+
+define arm_aapcs_vfpcc <2 x i64> @test_vldrdq_gather_shifted_offset_s64(i64* %base, <2 x i64> %offset) {
+; CHECK-LABEL: test_vldrdq_gather_shifted_offset_s64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrd.u64 q1, [r0, q0, uxtw #3]
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <2 x i64> @llvm.arm.mve.vldr.gather.offset.v2i64.p0i64.v2i64(i64* %base, <2 x i64> %offset, i32 64, i32 3, i32 0)
+  ret <2 x i64> %0
+}
+
+define arm_aapcs_vfpcc <2 x i64> @test_vldrdq_gather_shifted_offset_u64(i64* %base, <2 x i64> %offset) {
+; CHECK-LABEL: test_vldrdq_gather_shifted_offset_u64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrd.u64 q1, [r0, q0, uxtw #3]
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <2 x i64> @llvm.arm.mve.vldr.gather.offset.v2i64.p0i64.v2i64(i64* %base, <2 x i64> %offset, i32 64, i32 3, i32 1)
+  ret <2 x i64> %0
+}
+
+define arm_aapcs_vfpcc <2 x i64> @test_vldrdq_gather_shifted_offset_z_s64(i64* %base, <2 x i64> %offset, i16 zeroext %p) {
+; CHECK-LABEL: test_vldrdq_gather_shifted_offset_z_s64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrdt.u64 q1, [r0, q0, uxtw #3]
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <2 x i64> @llvm.arm.mve.vldr.gather.offset.predicated.v2i64.p0i64.v2i64.v4i1(i64* %base, <2 x i64> %offset, i32 64, i32 3, i32 0, <4 x i1> %1)
+  ret <2 x i64> %2
+}
+
+define arm_aapcs_vfpcc <2 x i64> @test_vldrdq_gather_shifted_offset_z_u64(i64* %base, <2 x i64> %offset, i16 zeroext %p) {
+; CHECK-LABEL: test_vldrdq_gather_shifted_offset_z_u64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrdt.u64 q1, [r0, q0, uxtw #3]
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <2 x i64> @llvm.arm.mve.vldr.gather.offset.predicated.v2i64.p0i64.v2i64.v4i1(i64* %base, <2 x i64> %offset, i32 64, i32 3, i32 1, <4 x i1> %1)
+  ret <2 x i64> %2
+}
+
+define arm_aapcs_vfpcc <8 x half> @test_vldrhq_gather_offset_f16(half* %base, <8 x i16> %offset) {
+; CHECK-LABEL: test_vldrhq_gather_offset_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q1, [r0, q0]
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x half> @llvm.arm.mve.vldr.gather.offset.v8f16.p0f16.v8i16(half* %base, <8 x i16> %offset, i32 16, i32 0, i32 0)
+  ret <8 x half> %0
+}
+
+declare <8 x half> @llvm.arm.mve.vldr.gather.offset.v8f16.p0f16.v8i16(half*, <8 x i16>, i32, i32, i32)
+
+define arm_aapcs_vfpcc <8 x i16> @test_vldrhq_gather_offset_s16(i16* %base, <8 x i16> %offset) {
+; CHECK-LABEL: test_vldrhq_gather_offset_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q1, [r0, q0]
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.arm.mve.vldr.gather.offset.v8i16.p0i16.v8i16(i16* %base, <8 x i16> %offset, i32 16, i32 0, i32 0)
+  ret <8 x i16> %0
+}
+
+declare <8 x i16> @llvm.arm.mve.vldr.gather.offset.v8i16.p0i16.v8i16(i16*, <8 x i16>, i32, i32, i32)
+
+define arm_aapcs_vfpcc <4 x i32> @test_vldrhq_gather_offset_s32(i16* %base, <4 x i32> %offset) {
+; CHECK-LABEL: test_vldrhq_gather_offset_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.s32 q1, [r0, q0]
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x i32> @llvm.arm.mve.vldr.gather.offset.v4i32.p0i16.v4i32(i16* %base, <4 x i32> %offset, i32 16, i32 0, i32 0)
+  ret <4 x i32> %0
+}
+
+declare <4 x i32> @llvm.arm.mve.vldr.gather.offset.v4i32.p0i16.v4i32(i16*, <4 x i32>, i32, i32, i32)
+
+define arm_aapcs_vfpcc <8 x i16> @test_vldrhq_gather_offset_u16(i16* %base, <8 x i16> %offset) {
+; CHECK-LABEL: test_vldrhq_gather_offset_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q1, [r0, q0]
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.arm.mve.vldr.gather.offset.v8i16.p0i16.v8i16(i16* %base, <8 x i16> %offset, i32 16, i32 0, i32 1)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vldrhq_gather_offset_u32(i16* %base, <4 x i32> %offset) {
+; CHECK-LABEL: test_vldrhq_gather_offset_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u32 q1, [r0, q0]
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x i32> @llvm.arm.mve.vldr.gather.offset.v4i32.p0i16.v4i32(i16* %base, <4 x i32> %offset, i32 16, i32 0, i32 1)
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <8 x half> @test_vldrhq_gather_offset_z_f16(half* %base, <8 x i16> %offset, i16 zeroext %p) {
+; CHECK-LABEL: test_vldrhq_gather_offset_z_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrht.u16 q1, [r0, q0]
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <8 x half> @llvm.arm.mve.vldr.gather.offset.predicated.v8f16.p0f16.v8i16.v8i1(half* %base, <8 x i16> %offset, i32 16, i32 0, i32 0, <8 x i1> %1)
+  ret <8 x half> %2
+}
+
+declare <8 x half> @llvm.arm.mve.vldr.gather.offset.predicated.v8f16.p0f16.v8i16.v8i1(half*, <8 x i16>, i32, i32, i32, <8 x i1>)
+
+define arm_aapcs_vfpcc <8 x i16> @test_vldrhq_gather_offset_z_s16(i16* %base, <8 x i16> %offset, i16 zeroext %p) {
+; CHECK-LABEL: test_vldrhq_gather_offset_z_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrht.u16 q1, [r0, q0]
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <8 x i16> @llvm.arm.mve.vldr.gather.offset.predicated.v8i16.p0i16.v8i16.v8i1(i16* %base, <8 x i16> %offset, i32 16, i32 0, i32 0, <8 x i1> %1)
+  ret <8 x i16> %2
+}
+
+declare <8 x i16> @llvm.arm.mve.vldr.gather.offset.predicated.v8i16.p0i16.v8i16.v8i1(i16*, <8 x i16>, i32, i32, i32, <8 x i1>)
+
+define arm_aapcs_vfpcc <4 x i32> @test_vldrhq_gather_offset_z_s32(i16* %base, <4 x i32> %offset, i16 zeroext %p) {
+; CHECK-LABEL: test_vldrhq_gather_offset_z_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrht.s32 q1, [r0, q0]
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x i32> @llvm.arm.mve.vldr.gather.offset.predicated.v4i32.p0i16.v4i32.v4i1(i16* %base, <4 x i32> %offset, i32 16, i32 0, i32 0, <4 x i1> %1)
+  ret <4 x i32> %2
+}
+
+declare <4 x i32> @llvm.arm.mve.vldr.gather.offset.predicated.v4i32.p0i16.v4i32.v4i1(i16*, <4 x i32>, i32, i32, i32, <4 x i1>)
+
+define arm_aapcs_vfpcc <8 x i16> @test_vldrhq_gather_offset_z_u16(i16* %base, <8 x i16> %offset, i16 zeroext %p) {
+; CHECK-LABEL: test_vldrhq_gather_offset_z_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrht.u16 q1, [r0, q0]
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <8 x i16> @llvm.arm.mve.vldr.gather.offset.predicated.v8i16.p0i16.v8i16.v8i1(i16* %base, <8 x i16> %offset, i32 16, i32 0, i32 1, <8 x i1> %1)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vldrhq_gather_offset_z_u32(i16* %base, <4 x i32> %offset, i16 zeroext %p) {
+; CHECK-LABEL: test_vldrhq_gather_offset_z_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrht.u32 q1, [r0, q0]
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x i32> @llvm.arm.mve.vldr.gather.offset.predicated.v4i32.p0i16.v4i32.v4i1(i16* %base, <4 x i32> %offset, i32 16, i32 0, i32 1, <4 x i1> %1)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <8 x half> @test_vldrhq_gather_shifted_offset_f16(half* %base, <8 x i16> %offset) {
+; CHECK-LABEL: test_vldrhq_gather_shifted_offset_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q1, [r0, q0, uxtw #1]
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x half> @llvm.arm.mve.vldr.gather.offset.v8f16.p0f16.v8i16(half* %base, <8 x i16> %offset, i32 16, i32 1, i32 0)
+  ret <8 x half> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vldrhq_gather_shifted_offset_s16(i16* %base, <8 x i16> %offset) {
+; CHECK-LABEL: test_vldrhq_gather_shifted_offset_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q1, [r0, q0, uxtw #1]
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.arm.mve.vldr.gather.offset.v8i16.p0i16.v8i16(i16* %base, <8 x i16> %offset, i32 16, i32 1, i32 0)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vldrhq_gather_shifted_offset_s32(i16* %base, <4 x i32> %offset) {
+; CHECK-LABEL: test_vldrhq_gather_shifted_offset_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.s32 q1, [r0, q0, uxtw #1]
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x i32> @llvm.arm.mve.vldr.gather.offset.v4i32.p0i16.v4i32(i16* %base, <4 x i32> %offset, i32 16, i32 1, i32 0)
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vldrhq_gather_shifted_offset_u16(i16* %base, <8 x i16> %offset) {
+; CHECK-LABEL: test_vldrhq_gather_shifted_offset_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q1, [r0, q0, uxtw #1]
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.arm.mve.vldr.gather.offset.v8i16.p0i16.v8i16(i16* %base, <8 x i16> %offset, i32 16, i32 1, i32 1)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vldrhq_gather_shifted_offset_u32(i16* %base, <4 x i32> %offset) {
+; CHECK-LABEL: test_vldrhq_gather_shifted_offset_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u32 q1, [r0, q0, uxtw #1]
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x i32> @llvm.arm.mve.vldr.gather.offset.v4i32.p0i16.v4i32(i16* %base, <4 x i32> %offset, i32 16, i32 1, i32 1)
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <8 x half> @test_vldrhq_gather_shifted_offset_z_f16(half* %base, <8 x i16> %offset, i16 zeroext %p) {
+; CHECK-LABEL: test_vldrhq_gather_shifted_offset_z_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrht.u16 q1, [r0, q0, uxtw #1]
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <8 x half> @llvm.arm.mve.vldr.gather.offset.predicated.v8f16.p0f16.v8i16.v8i1(half* %base, <8 x i16> %offset, i32 16, i32 1, i32 0, <8 x i1> %1)
+  ret <8 x half> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vldrhq_gather_shifted_offset_z_s16(i16* %base, <8 x i16> %offset, i16 zeroext %p) {
+; CHECK-LABEL: test_vldrhq_gather_shifted_offset_z_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrht.u16 q1, [r0, q0, uxtw #1]
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <8 x i16> @llvm.arm.mve.vldr.gather.offset.predicated.v8i16.p0i16.v8i16.v8i1(i16* %base, <8 x i16> %offset, i32 16, i32 1, i32 0, <8 x i1> %1)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vldrhq_gather_shifted_offset_z_s32(i16* %base, <4 x i32> %offset, i16 zeroext %p) {
+; CHECK-LABEL: test_vldrhq_gather_shifted_offset_z_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrht.s32 q1, [r0, q0, uxtw #1]
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x i32> @llvm.arm.mve.vldr.gather.offset.predicated.v4i32.p0i16.v4i32.v4i1(i16* %base, <4 x i32> %offset, i32 16, i32 1, i32 0, <4 x i1> %1)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vldrhq_gather_shifted_offset_z_u16(i16* %base, <8 x i16> %offset, i16 zeroext %p) {
+; CHECK-LABEL: test_vldrhq_gather_shifted_offset_z_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrht.u16 q1, [r0, q0, uxtw #1]
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <8 x i16> @llvm.arm.mve.vldr.gather.offset.predicated.v8i16.p0i16.v8i16.v8i1(i16* %base, <8 x i16> %offset, i32 16, i32 1, i32 1, <8 x i1> %1)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vldrhq_gather_shifted_offset_z_u32(i16* %base, <4 x i32> %offset, i16 zeroext %p) {
+; CHECK-LABEL: test_vldrhq_gather_shifted_offset_z_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrht.u32 q1, [r0, q0, uxtw #1]
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x i32> @llvm.arm.mve.vldr.gather.offset.predicated.v4i32.p0i16.v4i32.v4i1(i16* %base, <4 x i32> %offset, i32 16, i32 1, i32 1, <4 x i1> %1)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <4 x float> @test_vldrwq_gather_base_f32(<4 x i32> %addr) {
+; CHECK-LABEL: test_vldrwq_gather_base_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q1, [q0, #12]
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x float> @llvm.arm.mve.vldr.gather.base.v4f32.v4i32(<4 x i32> %addr, i32 12)
+  ret <4 x float> %0
+}
+
+declare <4 x float> @llvm.arm.mve.vldr.gather.base.v4f32.v4i32(<4 x i32>, i32)
+
+define arm_aapcs_vfpcc <4 x i32> @test_vldrwq_gather_base_s32(<4 x i32> %addr) {
+; CHECK-LABEL: test_vldrwq_gather_base_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q1, [q0, #400]
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x i32> @llvm.arm.mve.vldr.gather.base.v4i32.v4i32(<4 x i32> %addr, i32 400)
+  ret <4 x i32> %0
+}
+
+declare <4 x i32> @llvm.arm.mve.vldr.gather.base.v4i32.v4i32(<4 x i32>, i32)
+
+define arm_aapcs_vfpcc <4 x i32> @test_vldrwq_gather_base_u32(<4 x i32> %addr) {
+; CHECK-LABEL: test_vldrwq_gather_base_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q1, [q0, #284]
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x i32> @llvm.arm.mve.vldr.gather.base.v4i32.v4i32(<4 x i32> %addr, i32 284)
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <4 x float> @test_vldrwq_gather_base_wb_f32(<4 x i32>* %addr) {
+; CHECK-LABEL: test_vldrwq_gather_base_wb_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vldrw.u32 q1, [q0, #-64]!
+; CHECK-NEXT:    vstrw.32 q1, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = load <4 x i32>, <4 x i32>* %addr, align 8
+  %1 = call { <4 x float>, <4 x i32> } @llvm.arm.mve.vldr.gather.base.wb.v4f32.v4i32(<4 x i32> %0, i32 -64)
+  %2 = extractvalue { <4 x float>, <4 x i32> } %1, 1
+  store <4 x i32> %2, <4 x i32>* %addr, align 8
+  %3 = extractvalue { <4 x float>, <4 x i32> } %1, 0
+  ret <4 x float> %3
+}
+
+declare { <4 x float>, <4 x i32> } @llvm.arm.mve.vldr.gather.base.wb.v4f32.v4i32(<4 x i32>, i32)
+
+define arm_aapcs_vfpcc <4 x i32> @test_vldrwq_gather_base_wb_s32(<4 x i32>* %addr) {
+; CHECK-LABEL: test_vldrwq_gather_base_wb_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vldrw.u32 q1, [q0, #80]!
+; CHECK-NEXT:    vstrw.32 q1, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = load <4 x i32>, <4 x i32>* %addr, align 8
+  %1 = call { <4 x i32>, <4 x i32> } @llvm.arm.mve.vldr.gather.base.wb.v4i32.v4i32(<4 x i32> %0, i32 80)
+  %2 = extractvalue { <4 x i32>, <4 x i32> } %1, 1
+  store <4 x i32> %2, <4 x i32>* %addr, align 8
+  %3 = extractvalue { <4 x i32>, <4 x i32> } %1, 0
+  ret <4 x i32> %3
+}
+
+declare { <4 x i32>, <4 x i32> } @llvm.arm.mve.vldr.gather.base.wb.v4i32.v4i32(<4 x i32>, i32)
+
+define arm_aapcs_vfpcc <4 x i32> @test_vldrwq_gather_base_wb_u32(<4 x i32>* %addr) {
+; CHECK-LABEL: test_vldrwq_gather_base_wb_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vldrw.u32 q1, [q0, #480]!
+; CHECK-NEXT:    vstrw.32 q1, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = load <4 x i32>, <4 x i32>* %addr, align 8
+  %1 = call { <4 x i32>, <4 x i32> } @llvm.arm.mve.vldr.gather.base.wb.v4i32.v4i32(<4 x i32> %0, i32 480)
+  %2 = extractvalue { <4 x i32>, <4 x i32> } %1, 1
+  store <4 x i32> %2, <4 x i32>* %addr, align 8
+  %3 = extractvalue { <4 x i32>, <4 x i32> } %1, 0
+  ret <4 x i32> %3
+}
+
+define arm_aapcs_vfpcc <4 x float> @test_vldrwq_gather_base_wb_z_f32(<4 x i32>* %addr, i16 zeroext %p) {
+; CHECK-LABEL: test_vldrwq_gather_base_wb_z_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrwt.u32 q1, [q0, #-352]!
+; CHECK-NEXT:    vstrw.32 q1, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = load <4 x i32>, <4 x i32>* %addr, align 8
+  %1 = zext i16 %p to i32
+  %2 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %1)
+  %3 = call { <4 x float>, <4 x i32> } @llvm.arm.mve.vldr.gather.base.wb.predicated.v4f32.v4i32.v4i1(<4 x i32> %0, i32 -352, <4 x i1> %2)
+  %4 = extractvalue { <4 x float>, <4 x i32> } %3, 1
+  store <4 x i32> %4, <4 x i32>* %addr, align 8
+  %5 = extractvalue { <4 x float>, <4 x i32> } %3, 0
+  ret <4 x float> %5
+}
+
+declare { <4 x float>, <4 x i32> } @llvm.arm.mve.vldr.gather.base.wb.predicated.v4f32.v4i32.v4i1(<4 x i32>, i32, <4 x i1>)
+
+define arm_aapcs_vfpcc <4 x i32> @test_vldrwq_gather_base_wb_z_s32(<4 x i32>* %addr, i16 zeroext %p) {
+; CHECK-LABEL: test_vldrwq_gather_base_wb_z_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrwt.u32 q1, [q0, #276]!
+; CHECK-NEXT:    vstrw.32 q1, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = load <4 x i32>, <4 x i32>* %addr, align 8
+  %1 = zext i16 %p to i32
+  %2 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %1)
+  %3 = call { <4 x i32>, <4 x i32> } @llvm.arm.mve.vldr.gather.base.wb.predicated.v4i32.v4i32.v4i1(<4 x i32> %0, i32 276, <4 x i1> %2)
+  %4 = extractvalue { <4 x i32>, <4 x i32> } %3, 1
+  store <4 x i32> %4, <4 x i32>* %addr, align 8
+  %5 = extractvalue { <4 x i32>, <4 x i32> } %3, 0
+  ret <4 x i32> %5
+}
+
+declare { <4 x i32>, <4 x i32> } @llvm.arm.mve.vldr.gather.base.wb.predicated.v4i32.v4i32.v4i1(<4 x i32>, i32, <4 x i1>)
+
+define arm_aapcs_vfpcc <4 x i32> @test_vldrwq_gather_base_wb_z_u32(<4 x i32>* %addr, i16 zeroext %p) {
+; CHECK-LABEL: test_vldrwq_gather_base_wb_z_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrwt.u32 q1, [q0, #88]!
+; CHECK-NEXT:    vstrw.32 q1, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = load <4 x i32>, <4 x i32>* %addr, align 8
+  %1 = zext i16 %p to i32
+  %2 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %1)
+  %3 = call { <4 x i32>, <4 x i32> } @llvm.arm.mve.vldr.gather.base.wb.predicated.v4i32.v4i32.v4i1(<4 x i32> %0, i32 88, <4 x i1> %2)
+  %4 = extractvalue { <4 x i32>, <4 x i32> } %3, 1
+  store <4 x i32> %4, <4 x i32>* %addr, align 8
+  %5 = extractvalue { <4 x i32>, <4 x i32> } %3, 0
+  ret <4 x i32> %5
+}
+
+define arm_aapcs_vfpcc <4 x float> @test_vldrwq_gather_base_z_f32(<4 x i32> %addr, i16 zeroext %p) {
+; CHECK-LABEL: test_vldrwq_gather_base_z_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrwt.u32 q1, [q0, #-300]
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x float> @llvm.arm.mve.vldr.gather.base.predicated.v4f32.v4i32.v4i1(<4 x i32> %addr, i32 -300, <4 x i1> %1)
+  ret <4 x float> %2
+}
+
+declare <4 x float> @llvm.arm.mve.vldr.gather.base.predicated.v4f32.v4i32.v4i1(<4 x i32>, i32, <4 x i1>)
+
+define arm_aapcs_vfpcc <4 x i32> @test_vldrwq_gather_base_z_s32(<4 x i32> %addr, i16 zeroext %p) {
+; CHECK-LABEL: test_vldrwq_gather_base_z_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrwt.u32 q1, [q0, #440]
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x i32> @llvm.arm.mve.vldr.gather.base.predicated.v4i32.v4i32.v4i1(<4 x i32> %addr, i32 440, <4 x i1> %1)
+  ret <4 x i32> %2
+}
+
+declare <4 x i32> @llvm.arm.mve.vldr.gather.base.predicated.v4i32.v4i32.v4i1(<4 x i32>, i32, <4 x i1>)
+
+define arm_aapcs_vfpcc <4 x i32> @test_vldrwq_gather_base_z_u32(<4 x i32> %addr, i16 zeroext %p) {
+; CHECK-LABEL: test_vldrwq_gather_base_z_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrwt.u32 q1, [q0, #300]
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x i32> @llvm.arm.mve.vldr.gather.base.predicated.v4i32.v4i32.v4i1(<4 x i32> %addr, i32 300, <4 x i1> %1)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <4 x float> @test_vldrwq_gather_offset_f32(float* %base, <4 x i32> %offset) {
+; CHECK-LABEL: test_vldrwq_gather_offset_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q1, [r0, q0]
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x float> @llvm.arm.mve.vldr.gather.offset.v4f32.p0f32.v4i32(float* %base, <4 x i32> %offset, i32 32, i32 0, i32 0)
+  ret <4 x float> %0
+}
+
+declare <4 x float> @llvm.arm.mve.vldr.gather.offset.v4f32.p0f32.v4i32(float*, <4 x i32>, i32, i32, i32)
+
+define arm_aapcs_vfpcc <4 x i32> @test_vldrwq_gather_offset_s32(i32* %base, <4 x i32> %offset) {
+; CHECK-LABEL: test_vldrwq_gather_offset_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q1, [r0, q0]
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x i32> @llvm.arm.mve.vldr.gather.offset.v4i32.p0i32.v4i32(i32* %base, <4 x i32> %offset, i32 32, i32 0, i32 0)
+  ret <4 x i32> %0
+}
+
+declare <4 x i32> @llvm.arm.mve.vldr.gather.offset.v4i32.p0i32.v4i32(i32*, <4 x i32>, i32, i32, i32)
+
+define arm_aapcs_vfpcc <4 x i32> @test_vldrwq_gather_offset_u32(i32* %base, <4 x i32> %offset) {
+; CHECK-LABEL: test_vldrwq_gather_offset_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q1, [r0, q0]
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x i32> @llvm.arm.mve.vldr.gather.offset.v4i32.p0i32.v4i32(i32* %base, <4 x i32> %offset, i32 32, i32 0, i32 1)
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <4 x float> @test_vldrwq_gather_offset_z_f32(float* %base, <4 x i32> %offset, i16 zeroext %p) {
+; CHECK-LABEL: test_vldrwq_gather_offset_z_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrwt.u32 q1, [r0, q0]
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x float> @llvm.arm.mve.vldr.gather.offset.predicated.v4f32.p0f32.v4i32.v4i1(float* %base, <4 x i32> %offset, i32 32, i32 0, i32 0, <4 x i1> %1)
+  ret <4 x float> %2
+}
+
+declare <4 x float> @llvm.arm.mve.vldr.gather.offset.predicated.v4f32.p0f32.v4i32.v4i1(float*, <4 x i32>, i32, i32, i32, <4 x i1>)
+
+define arm_aapcs_vfpcc <4 x i32> @test_vldrwq_gather_offset_z_s32(i32* %base, <4 x i32> %offset, i16 zeroext %p) {
+; CHECK-LABEL: test_vldrwq_gather_offset_z_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrwt.u32 q1, [r0, q0]
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x i32> @llvm.arm.mve.vldr.gather.offset.predicated.v4i32.p0i32.v4i32.v4i1(i32* %base, <4 x i32> %offset, i32 32, i32 0, i32 0, <4 x i1> %1)
+  ret <4 x i32> %2
+}
+
+declare <4 x i32> @llvm.arm.mve.vldr.gather.offset.predicated.v4i32.p0i32.v4i32.v4i1(i32*, <4 x i32>, i32, i32, i32, <4 x i1>)
+
+define arm_aapcs_vfpcc <4 x i32> @test_vldrwq_gather_offset_z_u32(i32* %base, <4 x i32> %offset, i16 zeroext %p) {
+; CHECK-LABEL: test_vldrwq_gather_offset_z_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrwt.u32 q1, [r0, q0]
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x i32> @llvm.arm.mve.vldr.gather.offset.predicated.v4i32.p0i32.v4i32.v4i1(i32* %base, <4 x i32> %offset, i32 32, i32 0, i32 1, <4 x i1> %1)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <4 x float> @test_vldrwq_gather_shifted_offset_f32(float* %base, <4 x i32> %offset) {
+; CHECK-LABEL: test_vldrwq_gather_shifted_offset_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q1, [r0, q0, uxtw #2]
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x float> @llvm.arm.mve.vldr.gather.offset.v4f32.p0f32.v4i32(float* %base, <4 x i32> %offset, i32 32, i32 2, i32 0)
+  ret <4 x float> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vldrwq_gather_shifted_offset_s32(i32* %base, <4 x i32> %offset) {
+; CHECK-LABEL: test_vldrwq_gather_shifted_offset_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q1, [r0, q0, uxtw #2]
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x i32> @llvm.arm.mve.vldr.gather.offset.v4i32.p0i32.v4i32(i32* %base, <4 x i32> %offset, i32 32, i32 2, i32 0)
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vldrwq_gather_shifted_offset_u32(i32* %base, <4 x i32> %offset) {
+; CHECK-LABEL: test_vldrwq_gather_shifted_offset_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q1, [r0, q0, uxtw #2]
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x i32> @llvm.arm.mve.vldr.gather.offset.v4i32.p0i32.v4i32(i32* %base, <4 x i32> %offset, i32 32, i32 2, i32 1)
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <4 x float> @test_vldrwq_gather_shifted_offset_z_f32(float* %base, <4 x i32> %offset, i16 zeroext %p) {
+; CHECK-LABEL: test_vldrwq_gather_shifted_offset_z_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrwt.u32 q1, [r0, q0, uxtw #2]
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x float> @llvm.arm.mve.vldr.gather.offset.predicated.v4f32.p0f32.v4i32.v4i1(float* %base, <4 x i32> %offset, i32 32, i32 2, i32 0, <4 x i1> %1)
+  ret <4 x float> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vldrwq_gather_shifted_offset_z_s32(i32* %base, <4 x i32> %offset, i16 zeroext %p) {
+; CHECK-LABEL: test_vldrwq_gather_shifted_offset_z_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrwt.u32 q1, [r0, q0, uxtw #2]
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x i32> @llvm.arm.mve.vldr.gather.offset.predicated.v4i32.p0i32.v4i32.v4i1(i32* %base, <4 x i32> %offset, i32 32, i32 2, i32 0, <4 x i1> %1)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vldrwq_gather_shifted_offset_z_u32(i32* %base, <4 x i32> %offset, i16 zeroext %p) {
+; CHECK-LABEL: test_vldrwq_gather_shifted_offset_z_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrwt.u32 q1, [r0, q0, uxtw #2]
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x i32> @llvm.arm.mve.vldr.gather.offset.predicated.v4i32.p0i32.v4i32.v4i1(i32* %base, <4 x i32> %offset, i32 32, i32 2, i32 1, <4 x i1> %1)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc void @test_vstrbq_scatter_offset_p_s16(i8* %base, <8 x i16> %offset, <8 x i16> %value, i16 zeroext %p) {
+; CHECK-LABEL: test_vstrbq_scatter_offset_p_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrbt.16 q1, [r0, q0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  call void @llvm.arm.mve.vstr.scatter.offset.predicated.p0i8.v8i16.v8i16.v8i1(i8* %base, <8 x i16> %offset, <8 x i16> %value, i32 8, i32 0, <8 x i1> %1)
+  ret void
+}
+
+declare void @llvm.arm.mve.vstr.scatter.offset.predicated.p0i8.v8i16.v8i16.v8i1(i8*, <8 x i16>, <8 x i16>, i32, i32, <8 x i1>)
+
+define arm_aapcs_vfpcc void @test_vstrbq_scatter_offset_p_s32(i8* %base, <4 x i32> %offset, <4 x i32> %value, i16 zeroext %p) {
+; CHECK-LABEL: test_vstrbq_scatter_offset_p_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrbt.32 q1, [r0, q0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  call void @llvm.arm.mve.vstr.scatter.offset.predicated.p0i8.v4i32.v4i32.v4i1(i8* %base, <4 x i32> %offset, <4 x i32> %value, i32 8, i32 0, <4 x i1> %1)
+  ret void
+}
+
+declare void @llvm.arm.mve.vstr.scatter.offset.predicated.p0i8.v4i32.v4i32.v4i1(i8*, <4 x i32>, <4 x i32>, i32, i32, <4 x i1>)
+
+define arm_aapcs_vfpcc void @test_vstrbq_scatter_offset_p_s8(i8* %base, <16 x i8> %offset, <16 x i8> %value, i16 zeroext %p) {
+; CHECK-LABEL: test_vstrbq_scatter_offset_p_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrbt.8 q1, [r0, q0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  call void @llvm.arm.mve.vstr.scatter.offset.predicated.p0i8.v16i8.v16i8.v16i1(i8* %base, <16 x i8> %offset, <16 x i8> %value, i32 8, i32 0, <16 x i1> %1)
+  ret void
+}
+
+declare void @llvm.arm.mve.vstr.scatter.offset.predicated.p0i8.v16i8.v16i8.v16i1(i8*, <16 x i8>, <16 x i8>, i32, i32, <16 x i1>)
+
+define arm_aapcs_vfpcc void @test_vstrbq_scatter_offset_p_u16(i8* %base, <8 x i16> %offset, <8 x i16> %value, i16 zeroext %p) {
+; CHECK-LABEL: test_vstrbq_scatter_offset_p_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrbt.16 q1, [r0, q0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  call void @llvm.arm.mve.vstr.scatter.offset.predicated.p0i8.v8i16.v8i16.v8i1(i8* %base, <8 x i16> %offset, <8 x i16> %value, i32 8, i32 0, <8 x i1> %1)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vstrbq_scatter_offset_p_u32(i8* %base, <4 x i32> %offset, <4 x i32> %value, i16 zeroext %p) {
+; CHECK-LABEL: test_vstrbq_scatter_offset_p_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrbt.32 q1, [r0, q0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  call void @llvm.arm.mve.vstr.scatter.offset.predicated.p0i8.v4i32.v4i32.v4i1(i8* %base, <4 x i32> %offset, <4 x i32> %value, i32 8, i32 0, <4 x i1> %1)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vstrbq_scatter_offset_p_u8(i8* %base, <16 x i8> %offset, <16 x i8> %value, i16 zeroext %p) {
+; CHECK-LABEL: test_vstrbq_scatter_offset_p_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrbt.8 q1, [r0, q0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  call void @llvm.arm.mve.vstr.scatter.offset.predicated.p0i8.v16i8.v16i8.v16i1(i8* %base, <16 x i8> %offset, <16 x i8> %value, i32 8, i32 0, <16 x i1> %1)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vstrbq_scatter_offset_s16(i8* %base, <8 x i16> %offset, <8 x i16> %value) {
+; CHECK-LABEL: test_vstrbq_scatter_offset_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vstrb.16 q1, [r0, q0]
+; CHECK-NEXT:    bx lr
+entry:
+  call void @llvm.arm.mve.vstr.scatter.offset.p0i8.v8i16.v8i16(i8* %base, <8 x i16> %offset, <8 x i16> %value, i32 8, i32 0)
+  ret void
+}
+
+declare void @llvm.arm.mve.vstr.scatter.offset.p0i8.v8i16.v8i16(i8*, <8 x i16>, <8 x i16>, i32, i32)
+
+define arm_aapcs_vfpcc void @test_vstrbq_scatter_offset_s32(i8* %base, <4 x i32> %offset, <4 x i32> %value) {
+; CHECK-LABEL: test_vstrbq_scatter_offset_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vstrb.32 q1, [r0, q0]
+; CHECK-NEXT:    bx lr
+entry:
+  call void @llvm.arm.mve.vstr.scatter.offset.p0i8.v4i32.v4i32(i8* %base, <4 x i32> %offset, <4 x i32> %value, i32 8, i32 0)
+  ret void
+}
+
+declare void @llvm.arm.mve.vstr.scatter.offset.p0i8.v4i32.v4i32(i8*, <4 x i32>, <4 x i32>, i32, i32)
+
+define arm_aapcs_vfpcc void @test_vstrbq_scatter_offset_s8(i8* %base, <16 x i8> %offset, <16 x i8> %value) {
+; CHECK-LABEL: test_vstrbq_scatter_offset_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vstrb.8 q1, [r0, q0]
+; CHECK-NEXT:    bx lr
+entry:
+  call void @llvm.arm.mve.vstr.scatter.offset.p0i8.v16i8.v16i8(i8* %base, <16 x i8> %offset, <16 x i8> %value, i32 8, i32 0)
+  ret void
+}
+
+declare void @llvm.arm.mve.vstr.scatter.offset.p0i8.v16i8.v16i8(i8*, <16 x i8>, <16 x i8>, i32, i32)
+
+define arm_aapcs_vfpcc void @test_vstrbq_scatter_offset_u16(i8* %base, <8 x i16> %offset, <8 x i16> %value) {
+; CHECK-LABEL: test_vstrbq_scatter_offset_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vstrb.16 q1, [r0, q0]
+; CHECK-NEXT:    bx lr
+entry:
+  call void @llvm.arm.mve.vstr.scatter.offset.p0i8.v8i16.v8i16(i8* %base, <8 x i16> %offset, <8 x i16> %value, i32 8, i32 0)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vstrbq_scatter_offset_u32(i8* %base, <4 x i32> %offset, <4 x i32> %value) {
+; CHECK-LABEL: test_vstrbq_scatter_offset_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vstrb.32 q1, [r0, q0]
+; CHECK-NEXT:    bx lr
+entry:
+  call void @llvm.arm.mve.vstr.scatter.offset.p0i8.v4i32.v4i32(i8* %base, <4 x i32> %offset, <4 x i32> %value, i32 8, i32 0)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vstrbq_scatter_offset_u8(i8* %base, <16 x i8> %offset, <16 x i8> %value) {
+; CHECK-LABEL: test_vstrbq_scatter_offset_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vstrb.8 q1, [r0, q0]
+; CHECK-NEXT:    bx lr
+entry:
+  call void @llvm.arm.mve.vstr.scatter.offset.p0i8.v16i8.v16i8(i8* %base, <16 x i8> %offset, <16 x i8> %value, i32 8, i32 0)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vstrdq_scatter_base_p_s64(<2 x i64> %addr, <2 x i64> %value, i16 zeroext %p) {
+; CHECK-LABEL: test_vstrdq_scatter_base_p_s64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrdt.64 q1, [q0, #888]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  call void @llvm.arm.mve.vstr.scatter.base.predicated.v2i64.v2i64.v4i1(<2 x i64> %addr, i32 888, <2 x i64> %value, <4 x i1> %1)
+  ret void
+}
+
+declare void @llvm.arm.mve.vstr.scatter.base.predicated.v2i64.v2i64.v4i1(<2 x i64>, i32, <2 x i64>, <4 x i1>)
+
+define arm_aapcs_vfpcc void @test_vstrdq_scatter_base_p_u64(<2 x i64> %addr, <2 x i64> %value, i16 zeroext %p) {
+; CHECK-LABEL: test_vstrdq_scatter_base_p_u64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrdt.64 q1, [q0, #264]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  call void @llvm.arm.mve.vstr.scatter.base.predicated.v2i64.v2i64.v4i1(<2 x i64> %addr, i32 264, <2 x i64> %value, <4 x i1> %1)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vstrdq_scatter_base_s64(<2 x i64> %addr, <2 x i64> %value) {
+; CHECK-LABEL: test_vstrdq_scatter_base_s64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vstrd.64 q1, [q0, #408]
+; CHECK-NEXT:    bx lr
+entry:
+  call void @llvm.arm.mve.vstr.scatter.base.v2i64.v2i64(<2 x i64> %addr, i32 408, <2 x i64> %value)
+  ret void
+}
+
+declare void @llvm.arm.mve.vstr.scatter.base.v2i64.v2i64(<2 x i64>, i32, <2 x i64>)
+
+define arm_aapcs_vfpcc void @test_vstrdq_scatter_base_u64(<2 x i64> %addr, <2 x i64> %value) {
+; CHECK-LABEL: test_vstrdq_scatter_base_u64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vstrd.64 q1, [q0, #-472]
+; CHECK-NEXT:    bx lr
+entry:
+  call void @llvm.arm.mve.vstr.scatter.base.v2i64.v2i64(<2 x i64> %addr, i32 -472, <2 x i64> %value)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vstrdq_scatter_base_wb_p_s64(<2 x i64>* %addr, <2 x i64> %value, i16 zeroext %p) {
+; CHECK-LABEL: test_vstrdq_scatter_base_wb_p_s64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrdt.64 q0, [q1, #248]!
+; CHECK-NEXT:    vstrw.32 q1, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = load <2 x i64>, <2 x i64>* %addr, align 8
+  %1 = zext i16 %p to i32
+  %2 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %1)
+  %3 = call <2 x i64> @llvm.arm.mve.vstr.scatter.base.wb.predicated.v2i64.v2i64.v4i1(<2 x i64> %0, i32 248, <2 x i64> %value, <4 x i1> %2)
+  store <2 x i64> %3, <2 x i64>* %addr, align 8
+  ret void
+}
+
+declare <2 x i64> @llvm.arm.mve.vstr.scatter.base.wb.predicated.v2i64.v2i64.v4i1(<2 x i64>, i32, <2 x i64>, <4 x i1>)
+
+define arm_aapcs_vfpcc void @test_vstrdq_scatter_base_wb_p_u64(<2 x i64>* %addr, <2 x i64> %value, i16 zeroext %p) {
+; CHECK-LABEL: test_vstrdq_scatter_base_wb_p_u64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrdt.64 q0, [q1, #136]!
+; CHECK-NEXT:    vstrw.32 q1, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = load <2 x i64>, <2 x i64>* %addr, align 8
+  %1 = zext i16 %p to i32
+  %2 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %1)
+  %3 = call <2 x i64> @llvm.arm.mve.vstr.scatter.base.wb.predicated.v2i64.v2i64.v4i1(<2 x i64> %0, i32 136, <2 x i64> %value, <4 x i1> %2)
+  store <2 x i64> %3, <2 x i64>* %addr, align 8
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vstrdq_scatter_base_wb_s64(<2 x i64>* %addr, <2 x i64> %value) {
+; CHECK-LABEL: test_vstrdq_scatter_base_wb_s64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vstrd.64 q0, [q1, #208]!
+; CHECK-NEXT:    vstrw.32 q1, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = load <2 x i64>, <2 x i64>* %addr, align 8
+  %1 = call <2 x i64> @llvm.arm.mve.vstr.scatter.base.wb.v2i64.v2i64(<2 x i64> %0, i32 208, <2 x i64> %value)
+  store <2 x i64> %1, <2 x i64>* %addr, align 8
+  ret void
+}
+
+declare <2 x i64> @llvm.arm.mve.vstr.scatter.base.wb.v2i64.v2i64(<2 x i64>, i32, <2 x i64>)
+
+define arm_aapcs_vfpcc void @test_vstrdq_scatter_base_wb_u64(<2 x i64>* %addr, <2 x i64> %value) {
+; CHECK-LABEL: test_vstrdq_scatter_base_wb_u64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vstrd.64 q0, [q1, #-168]!
+; CHECK-NEXT:    vstrw.32 q1, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = load <2 x i64>, <2 x i64>* %addr, align 8
+  %1 = call <2 x i64> @llvm.arm.mve.vstr.scatter.base.wb.v2i64.v2i64(<2 x i64> %0, i32 -168, <2 x i64> %value)
+  store <2 x i64> %1, <2 x i64>* %addr, align 8
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vstrdq_scatter_offset_p_s64(i64* %base, <2 x i64> %offset, <2 x i64> %value, i16 zeroext %p) {
+; CHECK-LABEL: test_vstrdq_scatter_offset_p_s64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrdt.64 q1, [r0, q0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  call void @llvm.arm.mve.vstr.scatter.offset.predicated.p0i64.v2i64.v2i64.v4i1(i64* %base, <2 x i64> %offset, <2 x i64> %value, i32 64, i32 0, <4 x i1> %1)
+  ret void
+}
+
+declare void @llvm.arm.mve.vstr.scatter.offset.predicated.p0i64.v2i64.v2i64.v4i1(i64*, <2 x i64>, <2 x i64>, i32, i32, <4 x i1>)
+
+define arm_aapcs_vfpcc void @test_vstrdq_scatter_offset_p_u64(i64* %base, <2 x i64> %offset, <2 x i64> %value, i16 zeroext %p) {
+; CHECK-LABEL: test_vstrdq_scatter_offset_p_u64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrdt.64 q1, [r0, q0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  call void @llvm.arm.mve.vstr.scatter.offset.predicated.p0i64.v2i64.v2i64.v4i1(i64* %base, <2 x i64> %offset, <2 x i64> %value, i32 64, i32 0, <4 x i1> %1)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vstrdq_scatter_offset_s64(i64* %base, <2 x i64> %offset, <2 x i64> %value) {
+; CHECK-LABEL: test_vstrdq_scatter_offset_s64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vstrd.64 q1, [r0, q0]
+; CHECK-NEXT:    bx lr
+entry:
+  call void @llvm.arm.mve.vstr.scatter.offset.p0i64.v2i64.v2i64(i64* %base, <2 x i64> %offset, <2 x i64> %value, i32 64, i32 0)
+  ret void
+}
+
+declare void @llvm.arm.mve.vstr.scatter.offset.p0i64.v2i64.v2i64(i64*, <2 x i64>, <2 x i64>, i32, i32)
+
+define arm_aapcs_vfpcc void @test_vstrdq_scatter_offset_u64(i64* %base, <2 x i64> %offset, <2 x i64> %value) {
+; CHECK-LABEL: test_vstrdq_scatter_offset_u64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vstrd.64 q1, [r0, q0]
+; CHECK-NEXT:    bx lr
+entry:
+  call void @llvm.arm.mve.vstr.scatter.offset.p0i64.v2i64.v2i64(i64* %base, <2 x i64> %offset, <2 x i64> %value, i32 64, i32 0)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vstrdq_scatter_shifted_offset_p_s64(i64* %base, <2 x i64> %offset, <2 x i64> %value, i16 zeroext %p) {
+; CHECK-LABEL: test_vstrdq_scatter_shifted_offset_p_s64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrdt.64 q1, [r0, q0, uxtw #3]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  call void @llvm.arm.mve.vstr.scatter.offset.predicated.p0i64.v2i64.v2i64.v4i1(i64* %base, <2 x i64> %offset, <2 x i64> %value, i32 64, i32 3, <4 x i1> %1)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vstrdq_scatter_shifted_offset_p_u64(i64* %base, <2 x i64> %offset, <2 x i64> %value, i16 zeroext %p) {
+; CHECK-LABEL: test_vstrdq_scatter_shifted_offset_p_u64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrdt.64 q1, [r0, q0, uxtw #3]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  call void @llvm.arm.mve.vstr.scatter.offset.predicated.p0i64.v2i64.v2i64.v4i1(i64* %base, <2 x i64> %offset, <2 x i64> %value, i32 64, i32 3, <4 x i1> %1)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vstrdq_scatter_shifted_offset_s64(i64* %base, <2 x i64> %offset, <2 x i64> %value) {
+; CHECK-LABEL: test_vstrdq_scatter_shifted_offset_s64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vstrd.64 q1, [r0, q0, uxtw #3]
+; CHECK-NEXT:    bx lr
+entry:
+  call void @llvm.arm.mve.vstr.scatter.offset.p0i64.v2i64.v2i64(i64* %base, <2 x i64> %offset, <2 x i64> %value, i32 64, i32 3)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vstrdq_scatter_shifted_offset_u64(i64* %base, <2 x i64> %offset, <2 x i64> %value) {
+; CHECK-LABEL: test_vstrdq_scatter_shifted_offset_u64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vstrd.64 q1, [r0, q0, uxtw #3]
+; CHECK-NEXT:    bx lr
+entry:
+  call void @llvm.arm.mve.vstr.scatter.offset.p0i64.v2i64.v2i64(i64* %base, <2 x i64> %offset, <2 x i64> %value, i32 64, i32 3)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vstrhq_scatter_offset_f16(half* %base, <8 x i16> %offset, <8 x half> %value) {
+; CHECK-LABEL: test_vstrhq_scatter_offset_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vstrh.16 q1, [r0, q0]
+; CHECK-NEXT:    bx lr
+entry:
+  call void @llvm.arm.mve.vstr.scatter.offset.p0f16.v8i16.v8f16(half* %base, <8 x i16> %offset, <8 x half> %value, i32 16, i32 0)
+  ret void
+}
+
+declare void @llvm.arm.mve.vstr.scatter.offset.p0f16.v8i16.v8f16(half*, <8 x i16>, <8 x half>, i32, i32)
+
+define arm_aapcs_vfpcc void @test_vstrhq_scatter_offset_p_f16(half* %base, <8 x i16> %offset, <8 x half> %value, i16 zeroext %p) {
+; CHECK-LABEL: test_vstrhq_scatter_offset_p_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrht.16 q1, [r0, q0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  call void @llvm.arm.mve.vstr.scatter.offset.predicated.p0f16.v8i16.v8f16.v8i1(half* %base, <8 x i16> %offset, <8 x half> %value, i32 16, i32 0, <8 x i1> %1)
+  ret void
+}
+
+declare void @llvm.arm.mve.vstr.scatter.offset.predicated.p0f16.v8i16.v8f16.v8i1(half*, <8 x i16>, <8 x half>, i32, i32, <8 x i1>)
+
+define arm_aapcs_vfpcc void @test_vstrhq_scatter_offset_p_s16(i16* %base, <8 x i16> %offset, <8 x i16> %value, i16 zeroext %p) {
+; CHECK-LABEL: test_vstrhq_scatter_offset_p_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrht.16 q1, [r0, q0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  call void @llvm.arm.mve.vstr.scatter.offset.predicated.p0i16.v8i16.v8i16.v8i1(i16* %base, <8 x i16> %offset, <8 x i16> %value, i32 16, i32 0, <8 x i1> %1)
+  ret void
+}
+
+declare void @llvm.arm.mve.vstr.scatter.offset.predicated.p0i16.v8i16.v8i16.v8i1(i16*, <8 x i16>, <8 x i16>, i32, i32, <8 x i1>)
+
+define arm_aapcs_vfpcc void @test_vstrhq_scatter_offset_p_s32(i16* %base, <4 x i32> %offset, <4 x i32> %value, i16 zeroext %p) {
+; CHECK-LABEL: test_vstrhq_scatter_offset_p_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrht.32 q1, [r0, q0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  call void @llvm.arm.mve.vstr.scatter.offset.predicated.p0i16.v4i32.v4i32.v4i1(i16* %base, <4 x i32> %offset, <4 x i32> %value, i32 16, i32 0, <4 x i1> %1)
+  ret void
+}
+
+declare void @llvm.arm.mve.vstr.scatter.offset.predicated.p0i16.v4i32.v4i32.v4i1(i16*, <4 x i32>, <4 x i32>, i32, i32, <4 x i1>)
+
+define arm_aapcs_vfpcc void @test_vstrhq_scatter_offset_p_u16(i16* %base, <8 x i16> %offset, <8 x i16> %value, i16 zeroext %p) {
+; CHECK-LABEL: test_vstrhq_scatter_offset_p_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrht.16 q1, [r0, q0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  call void @llvm.arm.mve.vstr.scatter.offset.predicated.p0i16.v8i16.v8i16.v8i1(i16* %base, <8 x i16> %offset, <8 x i16> %value, i32 16, i32 0, <8 x i1> %1)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vstrhq_scatter_offset_p_u32(i16* %base, <4 x i32> %offset, <4 x i32> %value, i16 zeroext %p) {
+; CHECK-LABEL: test_vstrhq_scatter_offset_p_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrht.32 q1, [r0, q0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  call void @llvm.arm.mve.vstr.scatter.offset.predicated.p0i16.v4i32.v4i32.v4i1(i16* %base, <4 x i32> %offset, <4 x i32> %value, i32 16, i32 0, <4 x i1> %1)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vstrhq_scatter_offset_s16(i16* %base, <8 x i16> %offset, <8 x i16> %value) {
+; CHECK-LABEL: test_vstrhq_scatter_offset_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vstrh.16 q1, [r0, q0]
+; CHECK-NEXT:    bx lr
+entry:
+  call void @llvm.arm.mve.vstr.scatter.offset.p0i16.v8i16.v8i16(i16* %base, <8 x i16> %offset, <8 x i16> %value, i32 16, i32 0)
+  ret void
+}
+
+declare void @llvm.arm.mve.vstr.scatter.offset.p0i16.v8i16.v8i16(i16*, <8 x i16>, <8 x i16>, i32, i32)
+
+define arm_aapcs_vfpcc void @test_vstrhq_scatter_offset_s32(i16* %base, <4 x i32> %offset, <4 x i32> %value) {
+; CHECK-LABEL: test_vstrhq_scatter_offset_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vstrh.32 q1, [r0, q0]
+; CHECK-NEXT:    bx lr
+entry:
+  call void @llvm.arm.mve.vstr.scatter.offset.p0i16.v4i32.v4i32(i16* %base, <4 x i32> %offset, <4 x i32> %value, i32 16, i32 0)
+  ret void
+}
+
+declare void @llvm.arm.mve.vstr.scatter.offset.p0i16.v4i32.v4i32(i16*, <4 x i32>, <4 x i32>, i32, i32)
+
+define arm_aapcs_vfpcc void @test_vstrhq_scatter_offset_u16(i16* %base, <8 x i16> %offset, <8 x i16> %value) {
+; CHECK-LABEL: test_vstrhq_scatter_offset_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vstrh.16 q1, [r0, q0]
+; CHECK-NEXT:    bx lr
+entry:
+  call void @llvm.arm.mve.vstr.scatter.offset.p0i16.v8i16.v8i16(i16* %base, <8 x i16> %offset, <8 x i16> %value, i32 16, i32 0)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vstrhq_scatter_offset_u32(i16* %base, <4 x i32> %offset, <4 x i32> %value) {
+; CHECK-LABEL: test_vstrhq_scatter_offset_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vstrh.32 q1, [r0, q0]
+; CHECK-NEXT:    bx lr
+entry:
+  call void @llvm.arm.mve.vstr.scatter.offset.p0i16.v4i32.v4i32(i16* %base, <4 x i32> %offset, <4 x i32> %value, i32 16, i32 0)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vstrhq_scatter_shifted_offset_f16(half* %base, <8 x i16> %offset, <8 x half> %value) {
+; CHECK-LABEL: test_vstrhq_scatter_shifted_offset_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vstrh.16 q1, [r0, q0, uxtw #1]
+; CHECK-NEXT:    bx lr
+entry:
+  call void @llvm.arm.mve.vstr.scatter.offset.p0f16.v8i16.v8f16(half* %base, <8 x i16> %offset, <8 x half> %value, i32 16, i32 1)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vstrhq_scatter_shifted_offset_p_f16(half* %base, <8 x i16> %offset, <8 x half> %value, i16 zeroext %p) {
+; CHECK-LABEL: test_vstrhq_scatter_shifted_offset_p_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrht.16 q1, [r0, q0, uxtw #1]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  call void @llvm.arm.mve.vstr.scatter.offset.predicated.p0f16.v8i16.v8f16.v8i1(half* %base, <8 x i16> %offset, <8 x half> %value, i32 16, i32 1, <8 x i1> %1)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vstrhq_scatter_shifted_offset_p_s16(i16* %base, <8 x i16> %offset, <8 x i16> %value, i16 zeroext %p) {
+; CHECK-LABEL: test_vstrhq_scatter_shifted_offset_p_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrht.16 q1, [r0, q0, uxtw #1]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  call void @llvm.arm.mve.vstr.scatter.offset.predicated.p0i16.v8i16.v8i16.v8i1(i16* %base, <8 x i16> %offset, <8 x i16> %value, i32 16, i32 1, <8 x i1> %1)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vstrhq_scatter_shifted_offset_p_s32(i16* %base, <4 x i32> %offset, <4 x i32> %value, i16 zeroext %p) {
+; CHECK-LABEL: test_vstrhq_scatter_shifted_offset_p_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrht.32 q1, [r0, q0, uxtw #1]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  call void @llvm.arm.mve.vstr.scatter.offset.predicated.p0i16.v4i32.v4i32.v4i1(i16* %base, <4 x i32> %offset, <4 x i32> %value, i32 16, i32 1, <4 x i1> %1)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vstrhq_scatter_shifted_offset_p_u16(i16* %base, <8 x i16> %offset, <8 x i16> %value, i16 zeroext %p) {
+; CHECK-LABEL: test_vstrhq_scatter_shifted_offset_p_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrht.16 q1, [r0, q0, uxtw #1]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  call void @llvm.arm.mve.vstr.scatter.offset.predicated.p0i16.v8i16.v8i16.v8i1(i16* %base, <8 x i16> %offset, <8 x i16> %value, i32 16, i32 1, <8 x i1> %1)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vstrhq_scatter_shifted_offset_p_u32(i16* %base, <4 x i32> %offset, <4 x i32> %value, i16 zeroext %p) {
+; CHECK-LABEL: test_vstrhq_scatter_shifted_offset_p_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrht.32 q1, [r0, q0, uxtw #1]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  call void @llvm.arm.mve.vstr.scatter.offset.predicated.p0i16.v4i32.v4i32.v4i1(i16* %base, <4 x i32> %offset, <4 x i32> %value, i32 16, i32 1, <4 x i1> %1)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vstrhq_scatter_shifted_offset_s16(i16* %base, <8 x i16> %offset, <8 x i16> %value) {
+; CHECK-LABEL: test_vstrhq_scatter_shifted_offset_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vstrh.16 q1, [r0, q0, uxtw #1]
+; CHECK-NEXT:    bx lr
+entry:
+  call void @llvm.arm.mve.vstr.scatter.offset.p0i16.v8i16.v8i16(i16* %base, <8 x i16> %offset, <8 x i16> %value, i32 16, i32 1)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vstrhq_scatter_shifted_offset_s32(i16* %base, <4 x i32> %offset, <4 x i32> %value) {
+; CHECK-LABEL: test_vstrhq_scatter_shifted_offset_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vstrh.32 q1, [r0, q0, uxtw #1]
+; CHECK-NEXT:    bx lr
+entry:
+  call void @llvm.arm.mve.vstr.scatter.offset.p0i16.v4i32.v4i32(i16* %base, <4 x i32> %offset, <4 x i32> %value, i32 16, i32 1)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vstrhq_scatter_shifted_offset_u16(i16* %base, <8 x i16> %offset, <8 x i16> %value) {
+; CHECK-LABEL: test_vstrhq_scatter_shifted_offset_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vstrh.16 q1, [r0, q0, uxtw #1]
+; CHECK-NEXT:    bx lr
+entry:
+  call void @llvm.arm.mve.vstr.scatter.offset.p0i16.v8i16.v8i16(i16* %base, <8 x i16> %offset, <8 x i16> %value, i32 16, i32 1)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vstrhq_scatter_shifted_offset_u32(i16* %base, <4 x i32> %offset, <4 x i32> %value) {
+; CHECK-LABEL: test_vstrhq_scatter_shifted_offset_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vstrh.32 q1, [r0, q0, uxtw #1]
+; CHECK-NEXT:    bx lr
+entry:
+  call void @llvm.arm.mve.vstr.scatter.offset.p0i16.v4i32.v4i32(i16* %base, <4 x i32> %offset, <4 x i32> %value, i32 16, i32 1)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vstrwq_scatter_base_f32(<4 x i32> %addr, <4 x float> %value) {
+; CHECK-LABEL: test_vstrwq_scatter_base_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vstrw.32 q1, [q0, #380]
+; CHECK-NEXT:    bx lr
+entry:
+  call void @llvm.arm.mve.vstr.scatter.base.v4i32.v4f32(<4 x i32> %addr, i32 380, <4 x float> %value)
+  ret void
+}
+
+declare void @llvm.arm.mve.vstr.scatter.base.v4i32.v4f32(<4 x i32>, i32, <4 x float>)
+
+define arm_aapcs_vfpcc void @test_vstrwq_scatter_base_p_f32(<4 x i32> %addr, <4 x float> %value, i16 zeroext %p) {
+; CHECK-LABEL: test_vstrwq_scatter_base_p_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrwt.32 q1, [q0, #-400]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  call void @llvm.arm.mve.vstr.scatter.base.predicated.v4i32.v4f32.v4i1(<4 x i32> %addr, i32 -400, <4 x float> %value, <4 x i1> %1)
+  ret void
+}
+
+declare void @llvm.arm.mve.vstr.scatter.base.predicated.v4i32.v4f32.v4i1(<4 x i32>, i32, <4 x float>, <4 x i1>)
+
+define arm_aapcs_vfpcc void @test_vstrwq_scatter_base_p_s32(<4 x i32> %addr, <4 x i32> %value, i16 zeroext %p) {
+; CHECK-LABEL: test_vstrwq_scatter_base_p_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrwt.32 q1, [q0, #48]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  call void @llvm.arm.mve.vstr.scatter.base.predicated.v4i32.v4i32.v4i1(<4 x i32> %addr, i32 48, <4 x i32> %value, <4 x i1> %1)
+  ret void
+}
+
+declare void @llvm.arm.mve.vstr.scatter.base.predicated.v4i32.v4i32.v4i1(<4 x i32>, i32, <4 x i32>, <4 x i1>)
+
+define arm_aapcs_vfpcc void @test_vstrwq_scatter_base_p_u32(<4 x i32> %addr, <4 x i32> %value, i16 zeroext %p) {
+; CHECK-LABEL: test_vstrwq_scatter_base_p_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrwt.32 q1, [q0, #-376]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  call void @llvm.arm.mve.vstr.scatter.base.predicated.v4i32.v4i32.v4i1(<4 x i32> %addr, i32 -376, <4 x i32> %value, <4 x i1> %1)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vstrwq_scatter_base_s32(<4 x i32> %addr, <4 x i32> %value) {
+; CHECK-LABEL: test_vstrwq_scatter_base_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vstrw.32 q1, [q0, #156]
+; CHECK-NEXT:    bx lr
+entry:
+  call void @llvm.arm.mve.vstr.scatter.base.v4i32.v4i32(<4 x i32> %addr, i32 156, <4 x i32> %value)
+  ret void
+}
+
+declare void @llvm.arm.mve.vstr.scatter.base.v4i32.v4i32(<4 x i32>, i32, <4 x i32>)
+
+define arm_aapcs_vfpcc void @test_vstrwq_scatter_base_u32(<4 x i32> %addr, <4 x i32> %value) {
+; CHECK-LABEL: test_vstrwq_scatter_base_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vstrw.32 q1, [q0, #212]
+; CHECK-NEXT:    bx lr
+entry:
+  call void @llvm.arm.mve.vstr.scatter.base.v4i32.v4i32(<4 x i32> %addr, i32 212, <4 x i32> %value)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vstrwq_scatter_base_wb_f32(<4 x i32>* %addr, <4 x float> %value) {
+; CHECK-LABEL: test_vstrwq_scatter_base_wb_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vstrw.32 q0, [q1, #-412]!
+; CHECK-NEXT:    vstrw.32 q1, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = load <4 x i32>, <4 x i32>* %addr, align 8
+  %1 = call <4 x i32> @llvm.arm.mve.vstr.scatter.base.wb.v4i32.v4f32(<4 x i32> %0, i32 -412, <4 x float> %value)
+  store <4 x i32> %1, <4 x i32>* %addr, align 8
+  ret void
+}
+
+declare <4 x i32> @llvm.arm.mve.vstr.scatter.base.wb.v4i32.v4f32(<4 x i32>, i32, <4 x float>)
+
+define arm_aapcs_vfpcc void @test_vstrwq_scatter_base_wb_p_f32(<4 x i32>* %addr, <4 x float> %value, i16 zeroext %p) {
+; CHECK-LABEL: test_vstrwq_scatter_base_wb_p_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrwt.32 q0, [q1, #236]!
+; CHECK-NEXT:    vstrw.32 q1, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = load <4 x i32>, <4 x i32>* %addr, align 8
+  %1 = zext i16 %p to i32
+  %2 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %1)
+  %3 = call <4 x i32> @llvm.arm.mve.vstr.scatter.base.wb.predicated.v4i32.v4f32.v4i1(<4 x i32> %0, i32 236, <4 x float> %value, <4 x i1> %2)
+  store <4 x i32> %3, <4 x i32>* %addr, align 8
+  ret void
+}
+
+declare <4 x i32> @llvm.arm.mve.vstr.scatter.base.wb.predicated.v4i32.v4f32.v4i1(<4 x i32>, i32, <4 x float>, <4 x i1>)
+
+define arm_aapcs_vfpcc void @test_vstrwq_scatter_base_wb_p_s32(<4 x i32>* %addr, <4 x i32> %value, i16 zeroext %p) {
+; CHECK-LABEL: test_vstrwq_scatter_base_wb_p_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrwt.32 q0, [q1, #328]!
+; CHECK-NEXT:    vstrw.32 q1, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = load <4 x i32>, <4 x i32>* %addr, align 8
+  %1 = zext i16 %p to i32
+  %2 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %1)
+  %3 = call <4 x i32> @llvm.arm.mve.vstr.scatter.base.wb.predicated.v4i32.v4i32.v4i1(<4 x i32> %0, i32 328, <4 x i32> %value, <4 x i1> %2)
+  store <4 x i32> %3, <4 x i32>* %addr, align 8
+  ret void
+}
+
+declare <4 x i32> @llvm.arm.mve.vstr.scatter.base.wb.predicated.v4i32.v4i32.v4i1(<4 x i32>, i32, <4 x i32>, <4 x i1>)
+
+define arm_aapcs_vfpcc void @test_vstrwq_scatter_base_wb_p_u32(<4 x i32>* %addr, <4 x i32> %value, i16 zeroext %p) {
+; CHECK-LABEL: test_vstrwq_scatter_base_wb_p_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrwt.32 q0, [q1, #412]!
+; CHECK-NEXT:    vstrw.32 q1, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = load <4 x i32>, <4 x i32>* %addr, align 8
+  %1 = zext i16 %p to i32
+  %2 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %1)
+  %3 = call <4 x i32> @llvm.arm.mve.vstr.scatter.base.wb.predicated.v4i32.v4i32.v4i1(<4 x i32> %0, i32 412, <4 x i32> %value, <4 x i1> %2)
+  store <4 x i32> %3, <4 x i32>* %addr, align 8
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vstrwq_scatter_base_wb_s32(<4 x i32>* %addr, <4 x i32> %value) {
+; CHECK-LABEL: test_vstrwq_scatter_base_wb_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vstrw.32 q0, [q1, #-152]!
+; CHECK-NEXT:    vstrw.32 q1, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = load <4 x i32>, <4 x i32>* %addr, align 8
+  %1 = call <4 x i32> @llvm.arm.mve.vstr.scatter.base.wb.v4i32.v4i32(<4 x i32> %0, i32 -152, <4 x i32> %value)
+  store <4 x i32> %1, <4 x i32>* %addr, align 8
+  ret void
+}
+
+declare <4 x i32> @llvm.arm.mve.vstr.scatter.base.wb.v4i32.v4i32(<4 x i32>, i32, <4 x i32>)
+
+define arm_aapcs_vfpcc void @test_vstrwq_scatter_base_wb_u32(<4 x i32>* %addr, <4 x i32> %value) {
+; CHECK-LABEL: test_vstrwq_scatter_base_wb_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vstrw.32 q0, [q1, #64]!
+; CHECK-NEXT:    vstrw.32 q1, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = load <4 x i32>, <4 x i32>* %addr, align 8
+  %1 = call <4 x i32> @llvm.arm.mve.vstr.scatter.base.wb.v4i32.v4i32(<4 x i32> %0, i32 64, <4 x i32> %value)
+  store <4 x i32> %1, <4 x i32>* %addr, align 8
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vstrwq_scatter_offset_f32(float* %base, <4 x i32> %offset, <4 x float> %value) {
+; CHECK-LABEL: test_vstrwq_scatter_offset_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vstrw.32 q1, [r0, q0]
+; CHECK-NEXT:    bx lr
+entry:
+  call void @llvm.arm.mve.vstr.scatter.offset.p0f32.v4i32.v4f32(float* %base, <4 x i32> %offset, <4 x float> %value, i32 32, i32 0)
+  ret void
+}
+
+declare void @llvm.arm.mve.vstr.scatter.offset.p0f32.v4i32.v4f32(float*, <4 x i32>, <4 x float>, i32, i32)
+
+define arm_aapcs_vfpcc void @test_vstrwq_scatter_offset_p_f32(float* %base, <4 x i32> %offset, <4 x float> %value, i16 zeroext %p) {
+; CHECK-LABEL: test_vstrwq_scatter_offset_p_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrwt.32 q1, [r0, q0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  call void @llvm.arm.mve.vstr.scatter.offset.predicated.p0f32.v4i32.v4f32.v4i1(float* %base, <4 x i32> %offset, <4 x float> %value, i32 32, i32 0, <4 x i1> %1)
+  ret void
+}
+
+declare void @llvm.arm.mve.vstr.scatter.offset.predicated.p0f32.v4i32.v4f32.v4i1(float*, <4 x i32>, <4 x float>, i32, i32, <4 x i1>)
+
+define arm_aapcs_vfpcc void @test_vstrwq_scatter_offset_p_s32(i32* %base, <4 x i32> %offset, <4 x i32> %value, i16 zeroext %p) {
+; CHECK-LABEL: test_vstrwq_scatter_offset_p_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrwt.32 q1, [r0, q0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  call void @llvm.arm.mve.vstr.scatter.offset.predicated.p0i32.v4i32.v4i32.v4i1(i32* %base, <4 x i32> %offset, <4 x i32> %value, i32 32, i32 0, <4 x i1> %1)
+  ret void
+}
+
+declare void @llvm.arm.mve.vstr.scatter.offset.predicated.p0i32.v4i32.v4i32.v4i1(i32*, <4 x i32>, <4 x i32>, i32, i32, <4 x i1>)
+
+define arm_aapcs_vfpcc void @test_vstrwq_scatter_offset_p_u32(i32* %base, <4 x i32> %offset, <4 x i32> %value, i16 zeroext %p) {
+; CHECK-LABEL: test_vstrwq_scatter_offset_p_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrwt.32 q1, [r0, q0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  call void @llvm.arm.mve.vstr.scatter.offset.predicated.p0i32.v4i32.v4i32.v4i1(i32* %base, <4 x i32> %offset, <4 x i32> %value, i32 32, i32 0, <4 x i1> %1)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vstrwq_scatter_offset_s32(i32* %base, <4 x i32> %offset, <4 x i32> %value) {
+; CHECK-LABEL: test_vstrwq_scatter_offset_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vstrw.32 q1, [r0, q0]
+; CHECK-NEXT:    bx lr
+entry:
+  call void @llvm.arm.mve.vstr.scatter.offset.p0i32.v4i32.v4i32(i32* %base, <4 x i32> %offset, <4 x i32> %value, i32 32, i32 0)
+  ret void
+}
+
+declare void @llvm.arm.mve.vstr.scatter.offset.p0i32.v4i32.v4i32(i32*, <4 x i32>, <4 x i32>, i32, i32)
+
+define arm_aapcs_vfpcc void @test_vstrwq_scatter_offset_u32(i32* %base, <4 x i32> %offset, <4 x i32> %value) {
+; CHECK-LABEL: test_vstrwq_scatter_offset_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vstrw.32 q1, [r0, q0]
+; CHECK-NEXT:    bx lr
+entry:
+  call void @llvm.arm.mve.vstr.scatter.offset.p0i32.v4i32.v4i32(i32* %base, <4 x i32> %offset, <4 x i32> %value, i32 32, i32 0)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vstrwq_scatter_shifted_offset_f32(float* %base, <4 x i32> %offset, <4 x float> %value) {
+; CHECK-LABEL: test_vstrwq_scatter_shifted_offset_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vstrw.32 q1, [r0, q0, uxtw #2]
+; CHECK-NEXT:    bx lr
+entry:
+  call void @llvm.arm.mve.vstr.scatter.offset.p0f32.v4i32.v4f32(float* %base, <4 x i32> %offset, <4 x float> %value, i32 32, i32 2)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vstrwq_scatter_shifted_offset_p_f32(float* %base, <4 x i32> %offset, <4 x float> %value, i16 zeroext %p) {
+; CHECK-LABEL: test_vstrwq_scatter_shifted_offset_p_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrwt.32 q1, [r0, q0, uxtw #2]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  call void @llvm.arm.mve.vstr.scatter.offset.predicated.p0f32.v4i32.v4f32.v4i1(float* %base, <4 x i32> %offset, <4 x float> %value, i32 32, i32 2, <4 x i1> %1)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vstrwq_scatter_shifted_offset_p_s32(i32* %base, <4 x i32> %offset, <4 x i32> %value, i16 zeroext %p) {
+; CHECK-LABEL: test_vstrwq_scatter_shifted_offset_p_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrwt.32 q1, [r0, q0, uxtw #2]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  call void @llvm.arm.mve.vstr.scatter.offset.predicated.p0i32.v4i32.v4i32.v4i1(i32* %base, <4 x i32> %offset, <4 x i32> %value, i32 32, i32 2, <4 x i1> %1)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vstrwq_scatter_shifted_offset_p_u32(i32* %base, <4 x i32> %offset, <4 x i32> %value, i16 zeroext %p) {
+; CHECK-LABEL: test_vstrwq_scatter_shifted_offset_p_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrwt.32 q1, [r0, q0, uxtw #2]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  call void @llvm.arm.mve.vstr.scatter.offset.predicated.p0i32.v4i32.v4i32.v4i1(i32* %base, <4 x i32> %offset, <4 x i32> %value, i32 32, i32 2, <4 x i1> %1)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vstrwq_scatter_shifted_offset_s32(i32* %base, <4 x i32> %offset, <4 x i32> %value) {
+; CHECK-LABEL: test_vstrwq_scatter_shifted_offset_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vstrw.32 q1, [r0, q0, uxtw #2]
+; CHECK-NEXT:    bx lr
+entry:
+  call void @llvm.arm.mve.vstr.scatter.offset.p0i32.v4i32.v4i32(i32* %base, <4 x i32> %offset, <4 x i32> %value, i32 32, i32 2)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vstrwq_scatter_shifted_offset_u32(i32* %base, <4 x i32> %offset, <4 x i32> %value) {
+; CHECK-LABEL: test_vstrwq_scatter_shifted_offset_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vstrw.32 q1, [r0, q0, uxtw #2]
+; CHECK-NEXT:    bx lr
+entry:
+  call void @llvm.arm.mve.vstr.scatter.offset.p0i32.v4i32.v4i32(i32* %base, <4 x i32> %offset, <4 x i32> %value, i32 32, i32 2)
+  ret void
+}
diff --git a/llvm/test/CodeGen/Thumb2/mve-intrinsics/vabavq.ll b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vabavq.ll
new file mode 100644
index 000000000..f7fb69e8c
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vabavq.ll
@@ -0,0 +1,158 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve.fp -verify-machineinstrs -o - %s | FileCheck %s
+
+declare <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32)
+declare <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32)
+declare <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32)
+
+declare i32 @llvm.arm.mve.vabav.v16i8(i32, i32, <16 x i8>, <16 x i8>)
+declare i32 @llvm.arm.mve.vabav.v8i16(i32, i32, <8 x i16>, <8 x i16>)
+declare i32 @llvm.arm.mve.vabav.v4i32(i32, i32, <4 x i32>, <4 x i32>)
+
+declare i32 @llvm.arm.mve.vabav.predicated.v16i8.v16i1(i32, i32, <16 x i8>, <16 x i8>, <16 x i1>)
+declare i32 @llvm.arm.mve.vabav.predicated.v8i16.v8i1(i32, i32, <8 x i16>, <8 x i16>, <8 x i1>)
+declare i32 @llvm.arm.mve.vabav.predicated.v4i32.v4i1(i32, i32, <4 x i32>, <4 x i32>, <4 x i1>)
+
+define arm_aapcs_vfpcc i32 @test_vabavq_s8(i32 %a, <16 x i8> %b, <16 x i8> %c) {
+; CHECK-LABEL: test_vabavq_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vabav.s8 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call i32 @llvm.arm.mve.vabav.v16i8(i32 0, i32 %a, <16 x i8> %b, <16 x i8> %c)
+  ret i32 %0
+}
+
+define arm_aapcs_vfpcc i32 @test_vabavq_s16(i32 %a, <8 x i16> %b, <8 x i16> %c) {
+; CHECK-LABEL: test_vabavq_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vabav.s16 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call i32 @llvm.arm.mve.vabav.v8i16(i32 0, i32 %a, <8 x i16> %b, <8 x i16> %c)
+  ret i32 %0
+}
+
+define arm_aapcs_vfpcc i32 @test_vabavq_s32(i32 %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LABEL: test_vabavq_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vabav.s32 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call i32 @llvm.arm.mve.vabav.v4i32(i32 0, i32 %a, <4 x i32> %b, <4 x i32> %c)
+  ret i32 %0
+}
+
+define arm_aapcs_vfpcc i32 @test_vabavq_u8(i32 %a, <16 x i8> %b, <16 x i8> %c) {
+; CHECK-LABEL: test_vabavq_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vabav.u8 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call i32 @llvm.arm.mve.vabav.v16i8(i32 1, i32 %a, <16 x i8> %b, <16 x i8> %c)
+  ret i32 %0
+}
+
+define arm_aapcs_vfpcc i32 @test_vabavq_u16(i32 %a, <8 x i16> %b, <8 x i16> %c) {
+; CHECK-LABEL: test_vabavq_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vabav.u16 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call i32 @llvm.arm.mve.vabav.v8i16(i32 1, i32 %a, <8 x i16> %b, <8 x i16> %c)
+  ret i32 %0
+}
+
+define arm_aapcs_vfpcc i32 @test_vabavq_u32(i32 %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LABEL: test_vabavq_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vabav.u32 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call i32 @llvm.arm.mve.vabav.v4i32(i32 1, i32 %a, <4 x i32> %b, <4 x i32> %c)
+  ret i32 %0
+}
+
+define arm_aapcs_vfpcc i32 @test_vabavq_p_s8(i32 %a, <16 x i8> %b, <16 x i8> %c, i16 zeroext %p) {
+; CHECK-LABEL: test_vabavq_p_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vabavt.s8 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = call i32 @llvm.arm.mve.vabav.predicated.v16i8.v16i1(i32 0, i32 %a, <16 x i8> %b, <16 x i8> %c, <16 x i1> %1)
+  ret i32 %2
+}
+
+define arm_aapcs_vfpcc i32 @test_vabavq_p_s16(i32 %a, <8 x i16> %b, <8 x i16> %c, i16 zeroext %p) {
+; CHECK-LABEL: test_vabavq_p_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vabavt.s16 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call i32 @llvm.arm.mve.vabav.predicated.v8i16.v8i1(i32 0, i32 %a, <8 x i16> %b, <8 x i16> %c, <8 x i1> %1)
+  ret i32 %2
+}
+
+define arm_aapcs_vfpcc i32 @test_vabavq_p_s32(i32 %a, <4 x i32> %b, <4 x i32> %c, i16 zeroext %p) {
+; CHECK-LABEL: test_vabavq_p_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vabavt.s32 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call i32 @llvm.arm.mve.vabav.predicated.v4i32.v4i1(i32 0, i32 %a, <4 x i32> %b, <4 x i32> %c, <4 x i1> %1)
+  ret i32 %2
+}
+
+define arm_aapcs_vfpcc i32 @test_vabavq_p_u8(i32 %a, <16 x i8> %b, <16 x i8> %c, i16 zeroext %p) {
+; CHECK-LABEL: test_vabavq_p_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vabavt.u8 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = call i32 @llvm.arm.mve.vabav.predicated.v16i8.v16i1(i32 1, i32 %a, <16 x i8> %b, <16 x i8> %c, <16 x i1> %1)
+  ret i32 %2
+}
+
+define arm_aapcs_vfpcc i32 @test_vabavq_p_u16(i32 %a, <8 x i16> %b, <8 x i16> %c, i16 zeroext %p) {
+; CHECK-LABEL: test_vabavq_p_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vabavt.u16 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call i32 @llvm.arm.mve.vabav.predicated.v8i16.v8i1(i32 1, i32 %a, <8 x i16> %b, <8 x i16> %c, <8 x i1> %1)
+  ret i32 %2
+}
+
+define arm_aapcs_vfpcc i32 @test_vabavq_p_u32(i32 %a, <4 x i32> %b, <4 x i32> %c, i16 zeroext %p) {
+; CHECK-LABEL: test_vabavq_p_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vabavt.u32 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call i32 @llvm.arm.mve.vabav.predicated.v4i32.v4i1(i32 1, i32 %a, <4 x i32> %b, <4 x i32> %c, <4 x i1> %1)
+  ret i32 %2
+}
diff --git a/llvm/test/CodeGen/Thumb2/mve-intrinsics/vabdq.ll b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vabdq.ll
new file mode 100644
index 000000000..961489613
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vabdq.ll
@@ -0,0 +1,139 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve.fp -verify-machineinstrs -o - %s | FileCheck %s
+
+define arm_aapcs_vfpcc <16 x i8> @test_vabdq_s8(<16 x i8> %a, <16 x i8> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vabdq_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vabd.s8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <16 x i8> @llvm.arm.mve.vabd.v16i8(<16 x i8> %a, <16 x i8> %b, i32 0)
+  ret <16 x i8> %0
+}
+
+declare <16 x i8> @llvm.arm.mve.vabd.v16i8(<16 x i8>, <16 x i8>, i32) #1
+
+define arm_aapcs_vfpcc <4 x i32> @test_vabdq_u32(<4 x i32> %a, <4 x i32> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vabdq_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vabd.u32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <4 x i32> @llvm.arm.mve.vabd.v4i32(<4 x i32> %a, <4 x i32> %b, i32 1)
+  ret <4 x i32> %0
+}
+
+declare <4 x i32> @llvm.arm.mve.vabd.v4i32(<4 x i32>, <4 x i32>, i32) #1
+
+define arm_aapcs_vfpcc <8 x half> @test_vabdq_f32(<8 x half> %a, <8 x half> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vabdq_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vabd.f16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <8 x half> @llvm.arm.mve.vabd.v8f16(<8 x half> %a, <8 x half> %b, i32 0)
+  ret <8 x half> %0
+}
+
+declare <8 x half> @llvm.arm.mve.vabd.v8f16(<8 x half>, <8 x half>, i32) #1
+
+define arm_aapcs_vfpcc <8 x i16> @test_vabdq_m_u16(<8 x i16> %inactive, <8 x i16> %a, <8 x i16> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vabdq_m_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vabdt.u16 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.abd.predicated.v8i16.v8i1(<8 x i16> %a, <8 x i16> %b, i32 1, <8 x i1> %1, <8 x i16> %inactive)
+  ret <8 x i16> %2
+}
+
+declare <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32) #1
+
+declare <8 x i16> @llvm.arm.mve.abd.predicated.v8i16.v8i1(<8 x i16>, <8 x i16>, i32, <8 x i1>, <8 x i16>) #1
+
+define arm_aapcs_vfpcc <16 x i8> @test_vabdq_m_s8(<16 x i8> %inactive, <16 x i8> %a, <16 x i8> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vabdq_m_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vabdt.s8 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <16 x i8> @llvm.arm.mve.abd.predicated.v16i8.v16i1(<16 x i8> %a, <16 x i8> %b, i32 0, <16 x i1> %1, <16 x i8> %inactive)
+  ret <16 x i8> %2
+}
+
+declare <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32) #1
+
+declare <16 x i8> @llvm.arm.mve.abd.predicated.v16i8.v16i1(<16 x i8>, <16 x i8>, i32, <16 x i1>, <16 x i8>) #1
+
+define arm_aapcs_vfpcc <4 x float> @test_vabdq_m_f32(<4 x float> %inactive, <4 x float> %a, <4 x float> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vabdq_m_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vabdt.f32 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call <4 x float> @llvm.arm.mve.abd.predicated.v4f32.v4i1(<4 x float> %a, <4 x float> %b, i32 0, <4 x i1> %1, <4 x float> %inactive)
+  ret <4 x float> %2
+}
+
+declare <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32) #1
+
+declare <4 x float> @llvm.arm.mve.abd.predicated.v4f32.v4i1(<4 x float>, <4 x float>, i32, <4 x i1>, <4 x float>) #1
+
+define arm_aapcs_vfpcc <8 x i16> @test_vabdq_x_u16(<8 x i16> %a, <8 x i16> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vabdq_x_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vabdt.u16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.abd.predicated.v8i16.v8i1(<8 x i16> %a, <8 x i16> %b, i32 1, <8 x i1> %1, <8 x i16> undef)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vabdq_x_u32(<4 x i32> %a, <4 x i32> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vabdq_x_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vabdt.u32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call <4 x i32> @llvm.arm.mve.abd.predicated.v4i32.v4i1(<4 x i32> %a, <4 x i32> %b, i32 1, <4 x i1> %1, <4 x i32> undef)
+  ret <4 x i32> %2
+}
+
+declare <4 x i32> @llvm.arm.mve.abd.predicated.v4i32.v4i1(<4 x i32>, <4 x i32>, i32, <4 x i1>, <4 x i32>) #1
+
+define arm_aapcs_vfpcc <8 x half> @test_vabdq_x_f16(<8 x half> %a, <8 x half> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vabdq_x_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vabdt.f16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <8 x half> @llvm.arm.mve.abd.predicated.v8f16.v8i1(<8 x half> %a, <8 x half> %b, i32 0, <8 x i1> %1, <8 x half> undef)
+  ret <8 x half> %2
+}
+
+declare <8 x half> @llvm.arm.mve.abd.predicated.v8f16.v8i1(<8 x half>, <8 x half>, i32, <8 x i1>, <8 x half>) #1
+
diff --git a/llvm/test/CodeGen/Thumb2/mve-intrinsics/vadc-multiple.ll b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vadc-multiple.ll
new file mode 100644
index 000000000..c0a095327
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vadc-multiple.ll
@@ -0,0 +1,87 @@
+; RUN: opt -instcombine -S %s | FileCheck --check-prefix=IR %s
+; RUN: opt -instcombine    %s | llc -mtriple=thumbv8.1m.main -mattr=+mve.fp -verify-machineinstrs -O3 -o - | FileCheck --check-prefix=ASM %s
+
+%struct.foo = type { [2 x <4 x i32>] }
+
+define arm_aapcs_vfpcc i32 @test_vadciq_multiple(%struct.foo %a, %struct.foo %b, i32 %carry) {
+entry:
+  %a.0 = extractvalue %struct.foo %a, 0, 0
+  %a.1 = extractvalue %struct.foo %a, 0, 1
+  %b.0 = extractvalue %struct.foo %b, 0, 0
+  %b.1 = extractvalue %struct.foo %b, 0, 1
+
+  %fpscr.in.0 = shl i32 %carry, 29
+  %outpair.0 = call { <4 x i32>, i32 } @llvm.arm.mve.vadc.v4i32(<4 x i32> %a.0, <4 x i32> %b.0, i32 %fpscr.in.0)
+  %fpscr.out.0 = extractvalue { <4 x i32>, i32 } %outpair.0, 1
+  %shifted.out.0 = lshr i32 %fpscr.out.0, 29
+  %carry.out.0 = and i32 1, %shifted.out.0
+  %fpscr.in.1 = shl i32 %carry.out.0, 29
+  %outpair.1 = call { <4 x i32>, i32 } @llvm.arm.mve.vadc.v4i32(<4 x i32> %a.1, <4 x i32> %b.1, i32 %fpscr.in.1)
+  %fpscr.out.1 = extractvalue { <4 x i32>, i32 } %outpair.1, 1
+  %shifted.out.1 = lshr i32 %fpscr.out.1, 29
+  %carry.out.1 = and i32 1, %shifted.out.1
+  ret i32 %carry.out.1
+}
+
+define arm_aapcs_vfpcc i32 @test_vadciq_pred_multiple(%struct.foo %a, %struct.foo %b, i32 %ipred, i32 %carry) {
+entry:
+  %a.0 = extractvalue %struct.foo %a, 0, 0
+  %a.1 = extractvalue %struct.foo %a, 0, 1
+  %b.0 = extractvalue %struct.foo %b, 0, 0
+  %b.1 = extractvalue %struct.foo %b, 0, 1
+
+  %vpred = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %ipred)
+  %fpscr.in.0 = shl i32 %carry, 29
+  %outpair.0 = call { <4 x i32>, i32 } @llvm.arm.mve.vadc.predicated.v4i32.v4i1(<4 x i32> undef, <4 x i32> %a.0, <4 x i32> %b.0, i32 %fpscr.in.0, <4 x i1> %vpred)
+  %fpscr.out.0 = extractvalue { <4 x i32>, i32 } %outpair.0, 1
+  %shifted.out.0 = lshr i32 %fpscr.out.0, 29
+  %carry.out.0 = and i32 1, %shifted.out.0
+  %fpscr.in.1 = shl i32 %carry.out.0, 29
+  %outpair.1 = call { <4 x i32>, i32 } @llvm.arm.mve.vadc.predicated.v4i32.v4i1(<4 x i32> undef, <4 x i32> %a.1, <4 x i32> %b.1, i32 %fpscr.in.1, <4 x i1> %vpred)
+  %fpscr.out.1 = extractvalue { <4 x i32>, i32 } %outpair.1, 1
+  %shifted.out.1 = lshr i32 %fpscr.out.1, 29
+  %carry.out.1 = and i32 1, %shifted.out.1
+  ret i32 %carry.out.1
+}
+
+declare { <4 x i32>, i32 } @llvm.arm.mve.vadc.v4i32(<4 x i32>, <4 x i32>, i32)
+declare { <4 x i32>, i32 } @llvm.arm.mve.vadc.predicated.v4i32.v4i1(<4 x i32>, <4 x i32>, <4 x i32>, i32, <4 x i1>)
+declare <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32)
+
+; Expect the transformation in between the two intrinsics, where the
+; fpscr-formatted output value is turned back into just the carry bit
+; at bit 0 and then back again for the next call, to be optimized away
+; completely in InstCombine, so that the FPSCR output from one
+; intrinsic is passed straight on to the next:
+
+; IR: %outpair.0 = call { <4 x i32>, i32 } @llvm.arm.mve.vadc.v4i32(<4 x i32> %a.0, <4 x i32> %b.0, i32 %fpscr.in.0)
+; IR: %fpscr.out.0 = extractvalue { <4 x i32>, i32 } %outpair.0, 1
+; IR: %outpair.1 = call { <4 x i32>, i32 } @llvm.arm.mve.vadc.v4i32(<4 x i32> %a.1, <4 x i32> %b.1, i32 %fpscr.out.0)
+
+; IR: %outpair.0 = call { <4 x i32>, i32 } @llvm.arm.mve.vadc.predicated.v4i32.v4i1(<4 x i32> undef, <4 x i32> %a.0, <4 x i32> %b.0, i32 %fpscr.in.0, <4 x i1> %vpred)
+; IR: %fpscr.out.0 = extractvalue { <4 x i32>, i32 } %outpair.0, 1
+; IR: %outpair.1 = call { <4 x i32>, i32 } @llvm.arm.mve.vadc.predicated.v4i32.v4i1(<4 x i32> undef, <4 x i32> %a.1, <4 x i32> %b.1, i32 %fpscr.out.0, <4 x i1> %vpred)
+
+; And this is the assembly language we expect at the end of it, with
+; the two vadc.i32 instructions right next to each other, and the
+; second one implicitly reusing the FPSCR written by the first.
+
+; ASM: test_vadciq_multiple:
+; ASM:      lsls r0, r0, #29
+; ASM-NEXT: vmsr fpscr_nzcvqc, r0
+; ASM-NEXT: vadc.i32 q0, q0, q2
+; ASM-NEXT: vadc.i32 q0, q1, q3
+; ASM-NEXT: vmrs r0, fpscr_nzcvqc
+; ASM-NEXT: ubfx r0, r0, #29, #1
+; ASM-NEXT: bx lr
+
+; ASM: test_vadciq_pred_multiple:
+; ASM: lsls r1, r1, #29
+; ASM-NEXT: vmsr p0, r0
+; ASM-NEXT: vmsr fpscr_nzcvqc, r1
+; ASM-NEXT: vpstt
+; ASM-NEXT: vadct.i32 q0, q0, q2
+; ASM-NEXT: vadct.i32 q0, q1, q3
+; ASM-NEXT: vmrs r0, fpscr_nzcvqc
+; ASM-NEXT: ubfx r0, r0, #29, #1
+; ASM-NEXT: bx lr
diff --git a/llvm/test/CodeGen/Thumb2/mve-intrinsics/vadc.ll b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vadc.ll
new file mode 100644
index 000000000..47801bc10
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vadc.ll
@@ -0,0 +1,98 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve.fp -verify-machineinstrs -o - %s | FileCheck %s
+
+define arm_aapcs_vfpcc <4 x i32> @test_vadciq_s32(<4 x i32> %a, <4 x i32> %b, i32* %carry_out) {
+; CHECK-LABEL: test_vadciq_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vadci.i32 q0, q0, q1
+; CHECK-NEXT:    vmrs r1, fpscr_nzcvqc
+; CHECK-NEXT:    ubfx r1, r1, #29, #1
+; CHECK-NEXT:    str r1, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call { <4 x i32>, i32 } @llvm.arm.mve.vadc.v4i32(<4 x i32> %a, <4 x i32> %b, i32 0)
+  %1 = extractvalue { <4 x i32>, i32 } %0, 1
+  %2 = lshr i32 %1, 29
+  %3 = and i32 %2, 1
+  store i32 %3, i32* %carry_out, align 4
+  %4 = extractvalue { <4 x i32>, i32 } %0, 0
+  ret <4 x i32> %4
+}
+
+declare { <4 x i32>, i32 } @llvm.arm.mve.vadc.v4i32(<4 x i32>, <4 x i32>, i32)
+
+define arm_aapcs_vfpcc <4 x i32> @test_vadcq_u32(<4 x i32> %a, <4 x i32> %b, i32* %carry) {
+; CHECK-LABEL: test_vadcq_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    ldr r1, [r0]
+; CHECK-NEXT:    lsls r1, r1, #29
+; CHECK-NEXT:    vmsr fpscr_nzcvqc, r1
+; CHECK-NEXT:    vadc.i32 q0, q0, q1
+; CHECK-NEXT:    vmrs r1, fpscr_nzcvqc
+; CHECK-NEXT:    ubfx r1, r1, #29, #1
+; CHECK-NEXT:    str r1, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = load i32, i32* %carry, align 4
+  %1 = shl i32 %0, 29
+  %2 = tail call { <4 x i32>, i32 } @llvm.arm.mve.vadc.v4i32(<4 x i32> %a, <4 x i32> %b, i32 %1)
+  %3 = extractvalue { <4 x i32>, i32 } %2, 1
+  %4 = lshr i32 %3, 29
+  %5 = and i32 %4, 1
+  store i32 %5, i32* %carry, align 4
+  %6 = extractvalue { <4 x i32>, i32 } %2, 0
+  ret <4 x i32> %6
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vadciq_m_u32(<4 x i32> %inactive, <4 x i32> %a, <4 x i32> %b, i32* %carry_out, i16 zeroext %p) {
+; CHECK-LABEL: test_vadciq_m_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vadcit.i32 q0, q1, q2
+; CHECK-NEXT:    vmrs r1, fpscr_nzcvqc
+; CHECK-NEXT:    ubfx r1, r1, #29, #1
+; CHECK-NEXT:    str r1, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call { <4 x i32>, i32 } @llvm.arm.mve.vadc.predicated.v4i32.v4i1(<4 x i32> %inactive, <4 x i32> %a, <4 x i32> %b, i32 0, <4 x i1> %1)
+  %3 = extractvalue { <4 x i32>, i32 } %2, 1
+  %4 = lshr i32 %3, 29
+  %5 = and i32 %4, 1
+  store i32 %5, i32* %carry_out, align 4
+  %6 = extractvalue { <4 x i32>, i32 } %2, 0
+  ret <4 x i32> %6
+}
+
+declare <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32)
+
+declare { <4 x i32>, i32 } @llvm.arm.mve.vadc.predicated.v4i32.v4i1(<4 x i32>, <4 x i32>, <4 x i32>, i32, <4 x i1>)
+
+define arm_aapcs_vfpcc <4 x i32> @test_vadcq_m_s32(<4 x i32> %inactive, <4 x i32> %a, <4 x i32> %b, i32* %carry, i16 zeroext %p) {
+; CHECK-LABEL: test_vadcq_m_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    ldr r2, [r0]
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    lsls r1, r2, #29
+; CHECK-NEXT:    vmsr fpscr_nzcvqc, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vadct.i32 q0, q1, q2
+; CHECK-NEXT:    vmrs r1, fpscr_nzcvqc
+; CHECK-NEXT:    ubfx r1, r1, #29, #1
+; CHECK-NEXT:    str r1, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = load i32, i32* %carry, align 4
+  %1 = shl i32 %0, 29
+  %2 = zext i16 %p to i32
+  %3 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %2)
+  %4 = tail call { <4 x i32>, i32 } @llvm.arm.mve.vadc.predicated.v4i32.v4i1(<4 x i32> %inactive, <4 x i32> %a, <4 x i32> %b, i32 %1, <4 x i1> %3)
+  %5 = extractvalue { <4 x i32>, i32 } %4, 1
+  %6 = lshr i32 %5, 29
+  %7 = and i32 %6, 1
+  store i32 %7, i32* %carry, align 4
+  %8 = extractvalue { <4 x i32>, i32 } %4, 0
+  ret <4 x i32> %8
+}
diff --git a/llvm/test/CodeGen/Thumb2/mve-intrinsics/vaddq.ll b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vaddq.ll
new file mode 100644
index 000000000..b7becf6dd
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vaddq.ll
@@ -0,0 +1,93 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve.fp -verify-machineinstrs -o - %s | FileCheck %s
+
+define arm_aapcs_vfpcc <4 x i32> @test_vaddq_u32(<4 x i32> %a, <4 x i32> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vaddq_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vadd.i32 q0, q1, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = add <4 x i32> %b, %a
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <8 x half> @test_vaddq_f16(<8 x half> %a, <8 x half> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vaddq_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vadd.f16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = fadd <8 x half> %a, %b
+  ret <8 x half> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vaddq_m_s8(<16 x i8> %inactive, <16 x i8> %a, <16 x i8> %b, i16 zeroext %p) local_unnamed_addr #1 {
+; CHECK-LABEL: test_vaddq_m_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vaddt.i8 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <16 x i8> @llvm.arm.mve.add.predicated.v16i8.v16i1(<16 x i8> %a, <16 x i8> %b, <16 x i1> %1, <16 x i8> %inactive)
+  ret <16 x i8> %2
+}
+
+declare <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32) #2
+
+declare <16 x i8> @llvm.arm.mve.add.predicated.v16i8.v16i1(<16 x i8>, <16 x i8>, <16 x i1>, <16 x i8>) #2
+
+define arm_aapcs_vfpcc <4 x float> @test_vaddq_m_f32(<4 x float> %inactive, <4 x float> %a, <4 x float> %b, i16 zeroext %p) local_unnamed_addr #1 {
+; CHECK-LABEL: test_vaddq_m_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vaddt.f32 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call <4 x float> @llvm.arm.mve.add.predicated.v4f32.v4i1(<4 x float> %a, <4 x float> %b, <4 x i1> %1, <4 x float> %inactive)
+  ret <4 x float> %2
+}
+
+declare <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32) #2
+
+declare <4 x float> @llvm.arm.mve.add.predicated.v4f32.v4i1(<4 x float>, <4 x float>, <4 x i1>, <4 x float>) #2
+
+define arm_aapcs_vfpcc <8 x i16> @test_vaddq_x_u16(<8 x i16> %a, <8 x i16> %b, i16 zeroext %p) local_unnamed_addr #1 {
+; CHECK-LABEL: test_vaddq_x_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vaddt.i16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.add.predicated.v8i16.v8i1(<8 x i16> %a, <8 x i16> %b, <8 x i1> %1, <8 x i16> undef)
+  ret <8 x i16> %2
+}
+
+declare <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32) #2
+
+declare <8 x i16> @llvm.arm.mve.add.predicated.v8i16.v8i1(<8 x i16>, <8 x i16>, <8 x i1>, <8 x i16>) #2
+
+define arm_aapcs_vfpcc <8 x half> @test_vaddq_x_f16(<8 x half> %a, <8 x half> %b, i16 zeroext %p) local_unnamed_addr #1 {
+; CHECK-LABEL: test_vaddq_x_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vaddt.f16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <8 x half> @llvm.arm.mve.add.predicated.v8f16.v8i1(<8 x half> %a, <8 x half> %b, <8 x i1> %1, <8 x half> undef)
+  ret <8 x half> %2
+}
+
+declare <8 x half> @llvm.arm.mve.add.predicated.v8f16.v8i1(<8 x half>, <8 x half>, <8 x i1>, <8 x half>) #2
+
diff --git a/llvm/test/CodeGen/Thumb2/mve-intrinsics/vandq.ll b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vandq.ll
new file mode 100644
index 000000000..651ac1723
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vandq.ll
@@ -0,0 +1,177 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve.fp -verify-machineinstrs -o - %s | FileCheck %s
+
+define arm_aapcs_vfpcc <16 x i8> @test_vandq_u8(<16 x i8> %a, <16 x i8> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vandq_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vand q0, q1, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = and <16 x i8> %b, %a
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vandq_s16(<8 x i16> %a, <8 x i16> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vandq_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vand q0, q1, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = and <8 x i16> %b, %a
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vandq_u32(<4 x i32> %a, <4 x i32> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vandq_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vand q0, q1, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = and <4 x i32> %b, %a
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <4 x float> @test_vandq_f32(<4 x float> %a, <4 x float> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vandq_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vand q0, q1, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast <4 x float> %a to <4 x i32>
+  %1 = bitcast <4 x float> %b to <4 x i32>
+  %2 = and <4 x i32> %1, %0
+  %3 = bitcast <4 x i32> %2 to <4 x float>
+  ret <4 x float> %3
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vandq_m_s8(<16 x i8> %inactive, <16 x i8> %a, <16 x i8> %b, i16 zeroext %p) local_unnamed_addr #1 {
+; CHECK-LABEL: test_vandq_m_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vandt q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <16 x i8> @llvm.arm.mve.and.predicated.v16i8.v16i1(<16 x i8> %a, <16 x i8> %b, <16 x i1> %1, <16 x i8> %inactive)
+  ret <16 x i8> %2
+}
+
+declare <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32) #2
+
+declare <16 x i8> @llvm.arm.mve.and.predicated.v16i8.v16i1(<16 x i8>, <16 x i8>, <16 x i1>, <16 x i8>) #2
+
+define arm_aapcs_vfpcc <8 x i16> @test_vandq_m_u16(<8 x i16> %inactive, <8 x i16> %a, <8 x i16> %b, i16 zeroext %p) local_unnamed_addr #1 {
+; CHECK-LABEL: test_vandq_m_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vandt q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.and.predicated.v8i16.v8i1(<8 x i16> %a, <8 x i16> %b, <8 x i1> %1, <8 x i16> %inactive)
+  ret <8 x i16> %2
+}
+
+declare <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32) #2
+
+declare <8 x i16> @llvm.arm.mve.and.predicated.v8i16.v8i1(<8 x i16>, <8 x i16>, <8 x i1>, <8 x i16>) #2
+
+define arm_aapcs_vfpcc <4 x i32> @test_vandq_m_s32(<4 x i32> %inactive, <4 x i32> %a, <4 x i32> %b, i16 zeroext %p) local_unnamed_addr #1 {
+; CHECK-LABEL: test_vandq_m_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vandt q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call <4 x i32> @llvm.arm.mve.and.predicated.v4i32.v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i1> %1, <4 x i32> %inactive)
+  ret <4 x i32> %2
+}
+
+declare <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32) #2
+
+declare <4 x i32> @llvm.arm.mve.and.predicated.v4i32.v4i1(<4 x i32>, <4 x i32>, <4 x i1>, <4 x i32>) #2
+
+define arm_aapcs_vfpcc <8 x half> @test_vandq_m_f16(<8 x half> %inactive, <8 x half> %a, <8 x half> %b, i16 zeroext %p) local_unnamed_addr #1 {
+; CHECK-LABEL: test_vandq_m_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vandt q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast <8 x half> %a to <8 x i16>
+  %1 = bitcast <8 x half> %b to <8 x i16>
+  %2 = zext i16 %p to i32
+  %3 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %2)
+  %4 = bitcast <8 x half> %inactive to <8 x i16>
+  %5 = tail call <8 x i16> @llvm.arm.mve.and.predicated.v8i16.v8i1(<8 x i16> %0, <8 x i16> %1, <8 x i1> %3, <8 x i16> %4)
+  %6 = bitcast <8 x i16> %5 to <8 x half>
+  ret <8 x half> %6
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vandq_x_u8(<16 x i8> %a, <16 x i8> %b, i16 zeroext %p) local_unnamed_addr #1 {
+; CHECK-LABEL: test_vandq_x_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vandt q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <16 x i8> @llvm.arm.mve.and.predicated.v16i8.v16i1(<16 x i8> %a, <16 x i8> %b, <16 x i1> %1, <16 x i8> undef)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vandq_x_s16(<8 x i16> %a, <8 x i16> %b, i16 zeroext %p) local_unnamed_addr #1 {
+; CHECK-LABEL: test_vandq_x_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vandt q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.and.predicated.v8i16.v8i1(<8 x i16> %a, <8 x i16> %b, <8 x i1> %1, <8 x i16> undef)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vandq_x_u32(<4 x i32> %a, <4 x i32> %b, i16 zeroext %p) local_unnamed_addr #1 {
+; CHECK-LABEL: test_vandq_x_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vandt q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call <4 x i32> @llvm.arm.mve.and.predicated.v4i32.v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i1> %1, <4 x i32> undef)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <4 x float> @test_vandq_m_f32(<4 x float> %a, <4 x float> %b, i16 zeroext %p) local_unnamed_addr #1 {
+; CHECK-LABEL: test_vandq_m_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vandt q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast <4 x float> %a to <4 x i32>
+  %1 = bitcast <4 x float> %b to <4 x i32>
+  %2 = zext i16 %p to i32
+  %3 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %2)
+  %4 = tail call <4 x i32> @llvm.arm.mve.and.predicated.v4i32.v4i1(<4 x i32> %0, <4 x i32> %1, <4 x i1> %3, <4 x i32> undef)
+  %5 = bitcast <4 x i32> %4 to <4 x float>
+  ret <4 x float> %5
+}
+
diff --git a/llvm/test/CodeGen/Thumb2/mve-intrinsics/vbicq.ll b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vbicq.ll
new file mode 100644
index 000000000..97b7ac9fc
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vbicq.ll
@@ -0,0 +1,181 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve.fp -verify-machineinstrs -o - %s | FileCheck %s
+
+define arm_aapcs_vfpcc <16 x i8> @test_vbicq_u8(<16 x i8> %a, <16 x i8> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vbicq_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vbic q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = xor <16 x i8> %b, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
+  %1 = and <16 x i8> %0, %a
+  ret <16 x i8> %1
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vbicq_s16(<8 x i16> %a, <8 x i16> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vbicq_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vbic q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = xor <8 x i16> %b, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>
+  %1 = and <8 x i16> %0, %a
+  ret <8 x i16> %1
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vbicq_u32(<4 x i32> %a, <4 x i32> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vbicq_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vbic q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = xor <4 x i32> %b, <i32 -1, i32 -1, i32 -1, i32 -1>
+  %1 = and <4 x i32> %0, %a
+  ret <4 x i32> %1
+}
+
+define arm_aapcs_vfpcc <4 x float> @test_vbicq_f32(<4 x float> %a, <4 x float> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vbicq_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vbic q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast <4 x float> %a to <4 x i32>
+  %1 = bitcast <4 x float> %b to <4 x i32>
+  %2 = xor <4 x i32> %1, <i32 -1, i32 -1, i32 -1, i32 -1>
+  %3 = and <4 x i32> %2, %0
+  %4 = bitcast <4 x i32> %3 to <4 x float>
+  ret <4 x float> %4
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vbicq_m_s8(<16 x i8> %inactive, <16 x i8> %a, <16 x i8> %b, i16 zeroext %p) local_unnamed_addr #1 {
+; CHECK-LABEL: test_vbicq_m_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vbict q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <16 x i8> @llvm.arm.mve.bic.predicated.v16i8.v16i1(<16 x i8> %a, <16 x i8> %b, <16 x i1> %1, <16 x i8> %inactive)
+  ret <16 x i8> %2
+}
+
+declare <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32) #2
+
+declare <16 x i8> @llvm.arm.mve.bic.predicated.v16i8.v16i1(<16 x i8>, <16 x i8>, <16 x i1>, <16 x i8>) #2
+
+define arm_aapcs_vfpcc <8 x i16> @test_vbicq_m_u16(<8 x i16> %inactive, <8 x i16> %a, <8 x i16> %b, i16 zeroext %p) local_unnamed_addr #1 {
+; CHECK-LABEL: test_vbicq_m_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vbict q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.bic.predicated.v8i16.v8i1(<8 x i16> %a, <8 x i16> %b, <8 x i1> %1, <8 x i16> %inactive)
+  ret <8 x i16> %2
+}
+
+declare <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32) #2
+
+declare <8 x i16> @llvm.arm.mve.bic.predicated.v8i16.v8i1(<8 x i16>, <8 x i16>, <8 x i1>, <8 x i16>) #2
+
+define arm_aapcs_vfpcc <4 x i32> @test_vbicq_m_s32(<4 x i32> %inactive, <4 x i32> %a, <4 x i32> %b, i16 zeroext %p) local_unnamed_addr #1 {
+; CHECK-LABEL: test_vbicq_m_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vbict q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call <4 x i32> @llvm.arm.mve.bic.predicated.v4i32.v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i1> %1, <4 x i32> %inactive)
+  ret <4 x i32> %2
+}
+
+declare <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32) #2
+
+declare <4 x i32> @llvm.arm.mve.bic.predicated.v4i32.v4i1(<4 x i32>, <4 x i32>, <4 x i1>, <4 x i32>) #2
+
+define arm_aapcs_vfpcc <8 x half> @test_vbicq_m_f16(<8 x half> %inactive, <8 x half> %a, <8 x half> %b, i16 zeroext %p) local_unnamed_addr #1 {
+; CHECK-LABEL: test_vbicq_m_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vbict q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast <8 x half> %a to <8 x i16>
+  %1 = bitcast <8 x half> %b to <8 x i16>
+  %2 = zext i16 %p to i32
+  %3 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %2)
+  %4 = bitcast <8 x half> %inactive to <8 x i16>
+  %5 = tail call <8 x i16> @llvm.arm.mve.bic.predicated.v8i16.v8i1(<8 x i16> %0, <8 x i16> %1, <8 x i1> %3, <8 x i16> %4)
+  %6 = bitcast <8 x i16> %5 to <8 x half>
+  ret <8 x half> %6
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vbicq_x_u8(<16 x i8> %a, <16 x i8> %b, i16 zeroext %p) local_unnamed_addr #1 {
+; CHECK-LABEL: test_vbicq_x_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vbict q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <16 x i8> @llvm.arm.mve.bic.predicated.v16i8.v16i1(<16 x i8> %a, <16 x i8> %b, <16 x i1> %1, <16 x i8> undef)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vbicq_x_s16(<8 x i16> %a, <8 x i16> %b, i16 zeroext %p) local_unnamed_addr #1 {
+; CHECK-LABEL: test_vbicq_x_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vbict q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.bic.predicated.v8i16.v8i1(<8 x i16> %a, <8 x i16> %b, <8 x i1> %1, <8 x i16> undef)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vbicq_x_u32(<4 x i32> %a, <4 x i32> %b, i16 zeroext %p) local_unnamed_addr #1 {
+; CHECK-LABEL: test_vbicq_x_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vbict q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call <4 x i32> @llvm.arm.mve.bic.predicated.v4i32.v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i1> %1, <4 x i32> undef)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <4 x float> @test_vbicq_m_f32(<4 x float> %a, <4 x float> %b, i16 zeroext %p) local_unnamed_addr #1 {
+; CHECK-LABEL: test_vbicq_m_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vbict q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast <4 x float> %a to <4 x i32>
+  %1 = bitcast <4 x float> %b to <4 x i32>
+  %2 = zext i16 %p to i32
+  %3 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %2)
+  %4 = tail call <4 x i32> @llvm.arm.mve.bic.predicated.v4i32.v4i1(<4 x i32> %0, <4 x i32> %1, <4 x i1> %3, <4 x i32> undef)
+  %5 = bitcast <4 x i32> %4 to <4 x float>
+  ret <4 x float> %5
+}
+
diff --git a/llvm/test/CodeGen/Thumb2/mve-intrinsics/vcaddq.ll b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vcaddq.ll
new file mode 100644
index 000000000..9bb24fc61
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vcaddq.ll
@@ -0,0 +1,870 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve.fp -verify-machineinstrs -o - %s | FileCheck %s
+
+declare <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32)
+declare <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32)
+declare <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32)
+
+declare <16 x i8> @llvm.arm.mve.vcaddq.v16i8(i32, i32, <16 x i8>, <16 x i8>)
+declare <4 x i32> @llvm.arm.mve.vcaddq.v4i32(i32, i32, <4 x i32>, <4 x i32>)
+declare <8 x i16> @llvm.arm.mve.vcaddq.v8i16(i32, i32, <8 x i16>, <8 x i16>)
+declare <8 x half> @llvm.arm.mve.vcaddq.v8f16(i32, i32, <8 x half>, <8 x half>)
+declare <4 x float> @llvm.arm.mve.vcaddq.v4f32(i32, i32, <4 x float>, <4 x float>)
+
+declare <16 x i8> @llvm.arm.mve.vcaddq.predicated.v16i8.v16i1(i32, i32, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i1>)
+declare <8 x i16> @llvm.arm.mve.vcaddq.predicated.v8i16.v8i1(i32, i32, <8 x i16>, <8 x i16>, <8 x i16>, <8 x i1>)
+declare <4 x i32> @llvm.arm.mve.vcaddq.predicated.v4i32.v4i1(i32, i32, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i1>)
+declare <8 x half> @llvm.arm.mve.vcaddq.predicated.v8f16.v8i1(i32, i32, <8 x half>, <8 x half>, <8 x half>, <8 x i1>)
+declare <4 x float> @llvm.arm.mve.vcaddq.predicated.v4f32.v4i1(i32, i32, <4 x float>, <4 x float>, <4 x float>, <4 x i1>)
+
+define arm_aapcs_vfpcc <16 x i8> @test_vcaddq_rot90_u8(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vcaddq_rot90_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcadd.i8 q0, q0, q1, #90
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <16 x i8> @llvm.arm.mve.vcaddq.v16i8(i32 1, i32 0, <16 x i8> %a, <16 x i8> %b)
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vcaddq_rot90_u16(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vcaddq_rot90_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcadd.i16 q0, q0, q1, #90
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.arm.mve.vcaddq.v8i16(i32 1, i32 0, <8 x i16> %a, <8 x i16> %b)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vcaddq_rot90_u32(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vcaddq_rot90_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcadd.i32 q2, q0, q1, #90
+; CHECK-NEXT:    vmov q0, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x i32> @llvm.arm.mve.vcaddq.v4i32(i32 1, i32 0, <4 x i32> %a, <4 x i32> %b)
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vcaddq_rot90_s8(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vcaddq_rot90_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcadd.i8 q0, q0, q1, #90
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <16 x i8> @llvm.arm.mve.vcaddq.v16i8(i32 1, i32 0, <16 x i8> %a, <16 x i8> %b)
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vcaddq_rot90_s16(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vcaddq_rot90_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcadd.i16 q0, q0, q1, #90
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.arm.mve.vcaddq.v8i16(i32 1, i32 0, <8 x i16> %a, <8 x i16> %b)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vcaddq_rot90_s32(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vcaddq_rot90_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcadd.i32 q2, q0, q1, #90
+; CHECK-NEXT:    vmov q0, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x i32> @llvm.arm.mve.vcaddq.v4i32(i32 1, i32 0, <4 x i32> %a, <4 x i32> %b)
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <8 x half> @test_vcaddq_rot90_f16(<8 x half> %a, <8 x half> %b) {
+; CHECK-LABEL: test_vcaddq_rot90_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcadd.f16 q0, q0, q1, #90
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x half> @llvm.arm.mve.vcaddq.v8f16(i32 1, i32 0, <8 x half> %a, <8 x half> %b)
+  ret <8 x half> %0
+}
+
+define arm_aapcs_vfpcc <4 x float> @test_vcaddq_rot90_f32(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: test_vcaddq_rot90_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcadd.f32 q2, q0, q1, #90
+; CHECK-NEXT:    vmov q0, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x float> @llvm.arm.mve.vcaddq.v4f32(i32 1, i32 0, <4 x float> %a, <4 x float> %b)
+  ret <4 x float> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vcaddq_rot270_u8(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vcaddq_rot270_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcadd.i8 q0, q0, q1, #270
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <16 x i8> @llvm.arm.mve.vcaddq.v16i8(i32 1, i32 1, <16 x i8> %a, <16 x i8> %b)
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vcaddq_rot270_u16(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vcaddq_rot270_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcadd.i16 q0, q0, q1, #270
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.arm.mve.vcaddq.v8i16(i32 1, i32 1, <8 x i16> %a, <8 x i16> %b)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vcaddq_rot270_u32(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vcaddq_rot270_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcadd.i32 q2, q0, q1, #270
+; CHECK-NEXT:    vmov q0, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x i32> @llvm.arm.mve.vcaddq.v4i32(i32 1, i32 1, <4 x i32> %a, <4 x i32> %b)
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vcaddq_rot270_s8(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vcaddq_rot270_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcadd.i8 q0, q0, q1, #270
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <16 x i8> @llvm.arm.mve.vcaddq.v16i8(i32 1, i32 1, <16 x i8> %a, <16 x i8> %b)
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vcaddq_rot270_s16(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vcaddq_rot270_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcadd.i16 q0, q0, q1, #270
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.arm.mve.vcaddq.v8i16(i32 1, i32 1, <8 x i16> %a, <8 x i16> %b)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vcaddq_rot270_s32(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vcaddq_rot270_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcadd.i32 q2, q0, q1, #270
+; CHECK-NEXT:    vmov q0, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x i32> @llvm.arm.mve.vcaddq.v4i32(i32 1, i32 1, <4 x i32> %a, <4 x i32> %b)
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <8 x half> @test_vcaddq_rot270_f16(<8 x half> %a, <8 x half> %b) {
+; CHECK-LABEL: test_vcaddq_rot270_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcadd.f16 q0, q0, q1, #270
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x half> @llvm.arm.mve.vcaddq.v8f16(i32 1, i32 1, <8 x half> %a, <8 x half> %b)
+  ret <8 x half> %0
+}
+
+define arm_aapcs_vfpcc <4 x float> @test_vcaddq_rot270_f32(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: test_vcaddq_rot270_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcadd.f32 q2, q0, q1, #270
+; CHECK-NEXT:    vmov q0, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x float> @llvm.arm.mve.vcaddq.v4f32(i32 1, i32 1, <4 x float> %a, <4 x float> %b)
+  ret <4 x float> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vcaddq_rot90_m_u8(<16 x i8> %inactive, <16 x i8> %a, <16 x i8> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vcaddq_rot90_m_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcaddt.i8 q0, q1, q2, #90
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = call <16 x i8> @llvm.arm.mve.vcaddq.predicated.v16i8.v16i1(i32 1, i32 0, <16 x i8> %inactive, <16 x i8> %a, <16 x i8> %b, <16 x i1> %1)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vcaddq_rot90_m_u16(<8 x i16> %inactive, <8 x i16> %a, <8 x i16> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vcaddq_rot90_m_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcaddt.i16 q0, q1, q2, #90
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <8 x i16> @llvm.arm.mve.vcaddq.predicated.v8i16.v8i1(i32 1, i32 0, <8 x i16> %inactive, <8 x i16> %a, <8 x i16> %b, <8 x i1> %1)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vcaddq_rot90_m_u32(<4 x i32> %inactive, <4 x i32> %a, <4 x i32> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vcaddq_rot90_m_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcaddt.i32 q0, q1, q2, #90
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x i32> @llvm.arm.mve.vcaddq.predicated.v4i32.v4i1(i32 1, i32 0, <4 x i32> %inactive, <4 x i32> %a, <4 x i32> %b, <4 x i1> %1)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vcaddq_rot90_m_s8(<16 x i8> %inactive, <16 x i8> %a, <16 x i8> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vcaddq_rot90_m_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcaddt.i8 q0, q1, q2, #90
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = call <16 x i8> @llvm.arm.mve.vcaddq.predicated.v16i8.v16i1(i32 1, i32 0, <16 x i8> %inactive, <16 x i8> %a, <16 x i8> %b, <16 x i1> %1)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vcaddq_rot90_m_s16(<8 x i16> %inactive, <8 x i16> %a, <8 x i16> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vcaddq_rot90_m_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcaddt.i16 q0, q1, q2, #90
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <8 x i16> @llvm.arm.mve.vcaddq.predicated.v8i16.v8i1(i32 1, i32 0, <8 x i16> %inactive, <8 x i16> %a, <8 x i16> %b, <8 x i1> %1)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vcaddq_rot90_m_s32(<4 x i32> %inactive, <4 x i32> %a, <4 x i32> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vcaddq_rot90_m_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcaddt.i32 q0, q1, q2, #90
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x i32> @llvm.arm.mve.vcaddq.predicated.v4i32.v4i1(i32 1, i32 0, <4 x i32> %inactive, <4 x i32> %a, <4 x i32> %b, <4 x i1> %1)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <8 x half> @test_vcaddq_rot90_m_f16(<8 x half> %inactive, <8 x half> %a, <8 x half> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vcaddq_rot90_m_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcaddt.f16 q0, q1, q2, #90
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <8 x half> @llvm.arm.mve.vcaddq.predicated.v8f16.v8i1(i32 1, i32 0, <8 x half> %inactive, <8 x half> %a, <8 x half> %b, <8 x i1> %1)
+  ret <8 x half> %2
+}
+
+define arm_aapcs_vfpcc <4 x float> @test_vcaddq_rot90_m_f32(<4 x float> %inactive, <4 x float> %a, <4 x float> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vcaddq_rot90_m_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcaddt.f32 q0, q1, q2, #90
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x float> @llvm.arm.mve.vcaddq.predicated.v4f32.v4i1(i32 1, i32 0, <4 x float> %inactive, <4 x float> %a, <4 x float> %b, <4 x i1> %1)
+  ret <4 x float> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vcaddq_rot270_m_u8(<16 x i8> %inactive, <16 x i8> %a, <16 x i8> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vcaddq_rot270_m_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcaddt.i8 q0, q1, q2, #270
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = call <16 x i8> @llvm.arm.mve.vcaddq.predicated.v16i8.v16i1(i32 1, i32 1, <16 x i8> %inactive, <16 x i8> %a, <16 x i8> %b, <16 x i1> %1)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vcaddq_rot270_m_u16(<8 x i16> %inactive, <8 x i16> %a, <8 x i16> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vcaddq_rot270_m_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcaddt.i16 q0, q1, q2, #270
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <8 x i16> @llvm.arm.mve.vcaddq.predicated.v8i16.v8i1(i32 1, i32 1, <8 x i16> %inactive, <8 x i16> %a, <8 x i16> %b, <8 x i1> %1)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vcaddq_rot270_m_u32(<4 x i32> %inactive, <4 x i32> %a, <4 x i32> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vcaddq_rot270_m_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcaddt.i32 q0, q1, q2, #270
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x i32> @llvm.arm.mve.vcaddq.predicated.v4i32.v4i1(i32 1, i32 1, <4 x i32> %inactive, <4 x i32> %a, <4 x i32> %b, <4 x i1> %1)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vcaddq_rot270_m_s8(<16 x i8> %inactive, <16 x i8> %a, <16 x i8> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vcaddq_rot270_m_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcaddt.i8 q0, q1, q2, #270
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = call <16 x i8> @llvm.arm.mve.vcaddq.predicated.v16i8.v16i1(i32 1, i32 1, <16 x i8> %inactive, <16 x i8> %a, <16 x i8> %b, <16 x i1> %1)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vcaddq_rot270_m_s16(<8 x i16> %inactive, <8 x i16> %a, <8 x i16> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vcaddq_rot270_m_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcaddt.i16 q0, q1, q2, #270
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <8 x i16> @llvm.arm.mve.vcaddq.predicated.v8i16.v8i1(i32 1, i32 1, <8 x i16> %inactive, <8 x i16> %a, <8 x i16> %b, <8 x i1> %1)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vcaddq_rot270_m_s32(<4 x i32> %inactive, <4 x i32> %a, <4 x i32> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vcaddq_rot270_m_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcaddt.i32 q0, q1, q2, #270
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x i32> @llvm.arm.mve.vcaddq.predicated.v4i32.v4i1(i32 1, i32 1, <4 x i32> %inactive, <4 x i32> %a, <4 x i32> %b, <4 x i1> %1)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <8 x half> @test_vcaddq_rot270_m_f16(<8 x half> %inactive, <8 x half> %a, <8 x half> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vcaddq_rot270_m_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcaddt.f16 q0, q1, q2, #270
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <8 x half> @llvm.arm.mve.vcaddq.predicated.v8f16.v8i1(i32 1, i32 1, <8 x half> %inactive, <8 x half> %a, <8 x half> %b, <8 x i1> %1)
+  ret <8 x half> %2
+}
+
+define arm_aapcs_vfpcc <4 x float> @test_vcaddq_rot270_m_f32(<4 x float> %inactive, <4 x float> %a, <4 x float> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vcaddq_rot270_m_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcaddt.f32 q0, q1, q2, #270
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x float> @llvm.arm.mve.vcaddq.predicated.v4f32.v4i1(i32 1, i32 1, <4 x float> %inactive, <4 x float> %a, <4 x float> %b, <4 x i1> %1)
+  ret <4 x float> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vcaddq_rot90_x_u8(<16 x i8> %a, <16 x i8> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vcaddq_rot90_x_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcaddt.i8 q0, q0, q1, #90
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = call <16 x i8> @llvm.arm.mve.vcaddq.predicated.v16i8.v16i1(i32 1, i32 0, <16 x i8> undef, <16 x i8> %a, <16 x i8> %b, <16 x i1> %1)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vcaddq_rot90_x_u16(<8 x i16> %a, <8 x i16> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vcaddq_rot90_x_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcaddt.i16 q0, q0, q1, #90
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <8 x i16> @llvm.arm.mve.vcaddq.predicated.v8i16.v8i1(i32 1, i32 0, <8 x i16> undef, <8 x i16> %a, <8 x i16> %b, <8 x i1> %1)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vcaddq_rot90_x_u32(<4 x i32> %a, <4 x i32> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vcaddq_rot90_x_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcaddt.i32 q2, q0, q1, #90
+; CHECK-NEXT:    vmov q0, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x i32> @llvm.arm.mve.vcaddq.predicated.v4i32.v4i1(i32 1, i32 0, <4 x i32> undef, <4 x i32> %a, <4 x i32> %b, <4 x i1> %1)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vcaddq_rot90_x_s8(<16 x i8> %a, <16 x i8> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vcaddq_rot90_x_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcaddt.i8 q0, q0, q1, #90
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = call <16 x i8> @llvm.arm.mve.vcaddq.predicated.v16i8.v16i1(i32 1, i32 0, <16 x i8> undef, <16 x i8> %a, <16 x i8> %b, <16 x i1> %1)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vcaddq_rot90_x_s16(<8 x i16> %a, <8 x i16> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vcaddq_rot90_x_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcaddt.i16 q0, q0, q1, #90
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <8 x i16> @llvm.arm.mve.vcaddq.predicated.v8i16.v8i1(i32 1, i32 0, <8 x i16> undef, <8 x i16> %a, <8 x i16> %b, <8 x i1> %1)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vcaddq_rot90_x_s32(<4 x i32> %a, <4 x i32> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vcaddq_rot90_x_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcaddt.i32 q2, q0, q1, #90
+; CHECK-NEXT:    vmov q0, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x i32> @llvm.arm.mve.vcaddq.predicated.v4i32.v4i1(i32 1, i32 0, <4 x i32> undef, <4 x i32> %a, <4 x i32> %b, <4 x i1> %1)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <8 x half> @test_vcaddq_rot90_x_f16(<8 x half> %a, <8 x half> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vcaddq_rot90_x_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcaddt.f16 q0, q0, q1, #90
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <8 x half> @llvm.arm.mve.vcaddq.predicated.v8f16.v8i1(i32 1, i32 0, <8 x half> undef, <8 x half> %a, <8 x half> %b, <8 x i1> %1)
+  ret <8 x half> %2
+}
+
+define arm_aapcs_vfpcc <4 x float> @test_vcaddq_rot90_x_f32(<4 x float> %a, <4 x float> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vcaddq_rot90_x_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcaddt.f32 q2, q0, q1, #90
+; CHECK-NEXT:    vmov q0, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x float> @llvm.arm.mve.vcaddq.predicated.v4f32.v4i1(i32 1, i32 0, <4 x float> undef, <4 x float> %a, <4 x float> %b, <4 x i1> %1)
+  ret <4 x float> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vcaddq_rot270_x_u8(<16 x i8> %a, <16 x i8> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vcaddq_rot270_x_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcaddt.i8 q0, q0, q1, #270
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = call <16 x i8> @llvm.arm.mve.vcaddq.predicated.v16i8.v16i1(i32 1, i32 1, <16 x i8> undef, <16 x i8> %a, <16 x i8> %b, <16 x i1> %1)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vcaddq_rot270_x_u16(<8 x i16> %a, <8 x i16> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vcaddq_rot270_x_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcaddt.i16 q0, q0, q1, #270
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <8 x i16> @llvm.arm.mve.vcaddq.predicated.v8i16.v8i1(i32 1, i32 1, <8 x i16> undef, <8 x i16> %a, <8 x i16> %b, <8 x i1> %1)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vcaddq_rot270_x_u32(<4 x i32> %a, <4 x i32> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vcaddq_rot270_x_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcaddt.i32 q2, q0, q1, #270
+; CHECK-NEXT:    vmov q0, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x i32> @llvm.arm.mve.vcaddq.predicated.v4i32.v4i1(i32 1, i32 1, <4 x i32> undef, <4 x i32> %a, <4 x i32> %b, <4 x i1> %1)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vcaddq_rot270_x_s8(<16 x i8> %a, <16 x i8> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vcaddq_rot270_x_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcaddt.i8 q0, q0, q1, #270
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = call <16 x i8> @llvm.arm.mve.vcaddq.predicated.v16i8.v16i1(i32 1, i32 1, <16 x i8> undef, <16 x i8> %a, <16 x i8> %b, <16 x i1> %1)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vcaddq_rot270_x_s16(<8 x i16> %a, <8 x i16> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vcaddq_rot270_x_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcaddt.i16 q0, q0, q1, #270
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <8 x i16> @llvm.arm.mve.vcaddq.predicated.v8i16.v8i1(i32 1, i32 1, <8 x i16> undef, <8 x i16> %a, <8 x i16> %b, <8 x i1> %1)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vcaddq_rot270_x_s32(<4 x i32> %a, <4 x i32> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vcaddq_rot270_x_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcaddt.i32 q2, q0, q1, #270
+; CHECK-NEXT:    vmov q0, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x i32> @llvm.arm.mve.vcaddq.predicated.v4i32.v4i1(i32 1, i32 1, <4 x i32> undef, <4 x i32> %a, <4 x i32> %b, <4 x i1> %1)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <8 x half> @test_vcaddq_rot270_x_f16(<8 x half> %a, <8 x half> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vcaddq_rot270_x_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcaddt.f16 q0, q0, q1, #270
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <8 x half> @llvm.arm.mve.vcaddq.predicated.v8f16.v8i1(i32 1, i32 1, <8 x half> undef, <8 x half> %a, <8 x half> %b, <8 x i1> %1)
+  ret <8 x half> %2
+}
+
+define arm_aapcs_vfpcc <4 x float> @test_vcaddq_rot270_x_f32(<4 x float> %a, <4 x float> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vcaddq_rot270_x_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcaddt.f32 q2, q0, q1, #270
+; CHECK-NEXT:    vmov q0, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x float> @llvm.arm.mve.vcaddq.predicated.v4f32.v4i1(i32 1, i32 1, <4 x float> undef, <4 x float> %a, <4 x float> %b, <4 x i1> %1)
+  ret <4 x float> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vhcaddq_rot90_s8(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vhcaddq_rot90_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vhcadd.s8 q0, q0, q1, #90
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <16 x i8> @llvm.arm.mve.vcaddq.v16i8(i32 0, i32 0, <16 x i8> %a, <16 x i8> %b)
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vhcaddq_rot90_s16(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vhcaddq_rot90_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vhcadd.s16 q0, q0, q1, #90
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.arm.mve.vcaddq.v8i16(i32 0, i32 0, <8 x i16> %a, <8 x i16> %b)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vhcaddq_rot90_s32(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vhcaddq_rot90_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vhcadd.s32 q2, q0, q1, #90
+; CHECK-NEXT:    vmov q0, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x i32> @llvm.arm.mve.vcaddq.v4i32(i32 0, i32 0, <4 x i32> %a, <4 x i32> %b)
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vhcaddq_rot270_s8(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vhcaddq_rot270_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vhcadd.s8 q0, q0, q1, #270
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <16 x i8> @llvm.arm.mve.vcaddq.v16i8(i32 0, i32 1, <16 x i8> %a, <16 x i8> %b)
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vhcaddq_rot270_s16(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vhcaddq_rot270_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vhcadd.s16 q0, q0, q1, #270
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.arm.mve.vcaddq.v8i16(i32 0, i32 1, <8 x i16> %a, <8 x i16> %b)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vhcaddq_rot270_s32(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vhcaddq_rot270_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vhcadd.s32 q2, q0, q1, #270
+; CHECK-NEXT:    vmov q0, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x i32> @llvm.arm.mve.vcaddq.v4i32(i32 0, i32 1, <4 x i32> %a, <4 x i32> %b)
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vhcaddq_rot90_x_s8(<16 x i8> %a, <16 x i8> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vhcaddq_rot90_x_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vhcaddt.s8 q0, q0, q1, #90
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = call <16 x i8> @llvm.arm.mve.vcaddq.predicated.v16i8.v16i1(i32 0, i32 0, <16 x i8> undef, <16 x i8> %a, <16 x i8> %b, <16 x i1> %1)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vhcaddq_rot90_x_s16(<8 x i16> %a, <8 x i16> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vhcaddq_rot90_x_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vhcaddt.s16 q0, q0, q1, #90
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <8 x i16> @llvm.arm.mve.vcaddq.predicated.v8i16.v8i1(i32 0, i32 0, <8 x i16> undef, <8 x i16> %a, <8 x i16> %b, <8 x i1> %1)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vhcaddq_rot90_x_s32(<4 x i32> %a, <4 x i32> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vhcaddq_rot90_x_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vhcaddt.s32 q2, q0, q1, #90
+; CHECK-NEXT:    vmov q0, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x i32> @llvm.arm.mve.vcaddq.predicated.v4i32.v4i1(i32 0, i32 0, <4 x i32> undef, <4 x i32> %a, <4 x i32> %b, <4 x i1> %1)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vhcaddq_rot270_x_s8(<16 x i8> %a, <16 x i8> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vhcaddq_rot270_x_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vhcaddt.s8 q0, q0, q1, #270
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = call <16 x i8> @llvm.arm.mve.vcaddq.predicated.v16i8.v16i1(i32 0, i32 1, <16 x i8> undef, <16 x i8> %a, <16 x i8> %b, <16 x i1> %1)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vhcaddq_rot270_x_s16(<8 x i16> %a, <8 x i16> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vhcaddq_rot270_x_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vhcaddt.s16 q0, q0, q1, #270
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <8 x i16> @llvm.arm.mve.vcaddq.predicated.v8i16.v8i1(i32 0, i32 1, <8 x i16> undef, <8 x i16> %a, <8 x i16> %b, <8 x i1> %1)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vhcaddq_rot270_x_s32(<4 x i32> %a, <4 x i32> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vhcaddq_rot270_x_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vhcaddt.s32 q2, q0, q1, #270
+; CHECK-NEXT:    vmov q0, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x i32> @llvm.arm.mve.vcaddq.predicated.v4i32.v4i1(i32 0, i32 1, <4 x i32> undef, <4 x i32> %a, <4 x i32> %b, <4 x i1> %1)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vhcaddq_rot90_m_s8(<16 x i8> %inactive, <16 x i8> %a, <16 x i8> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vhcaddq_rot90_m_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vhcaddt.s8 q0, q1, q2, #90
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = call <16 x i8> @llvm.arm.mve.vcaddq.predicated.v16i8.v16i1(i32 0, i32 0, <16 x i8> %inactive, <16 x i8> %a, <16 x i8> %b, <16 x i1> %1)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vhcaddq_rot90_m_s16(<8 x i16> %inactive, <8 x i16> %a, <8 x i16> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vhcaddq_rot90_m_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vhcaddt.s16 q0, q1, q2, #90
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <8 x i16> @llvm.arm.mve.vcaddq.predicated.v8i16.v8i1(i32 0, i32 0, <8 x i16> %inactive, <8 x i16> %a, <8 x i16> %b, <8 x i1> %1)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vhcaddq_rot90_m_s32(<4 x i32> %inactive, <4 x i32> %a, <4 x i32> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vhcaddq_rot90_m_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vhcaddt.s32 q0, q1, q2, #90
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x i32> @llvm.arm.mve.vcaddq.predicated.v4i32.v4i1(i32 0, i32 0, <4 x i32> %inactive, <4 x i32> %a, <4 x i32> %b, <4 x i1> %1)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vhcaddq_rot270_m_s8(<16 x i8> %inactive, <16 x i8> %a, <16 x i8> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vhcaddq_rot270_m_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vhcaddt.s8 q0, q1, q2, #270
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = call <16 x i8> @llvm.arm.mve.vcaddq.predicated.v16i8.v16i1(i32 0, i32 1, <16 x i8> %inactive, <16 x i8> %a, <16 x i8> %b, <16 x i1> %1)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vhcaddq_rot270_m_s16(<8 x i16> %inactive, <8 x i16> %a, <8 x i16> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vhcaddq_rot270_m_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vhcaddt.s16 q0, q1, q2, #270
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <8 x i16> @llvm.arm.mve.vcaddq.predicated.v8i16.v8i1(i32 0, i32 1, <8 x i16> %inactive, <8 x i16> %a, <8 x i16> %b, <8 x i1> %1)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vhcaddq_rot270_m_s32(<4 x i32> %inactive, <4 x i32> %a, <4 x i32> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vhcaddq_rot270_m_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vhcaddt.s32 q0, q1, q2, #270
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x i32> @llvm.arm.mve.vcaddq.predicated.v4i32.v4i1(i32 0, i32 1, <4 x i32> %inactive, <4 x i32> %a, <4 x i32> %b, <4 x i1> %1)
+  ret <4 x i32> %2
+}
diff --git a/llvm/test/CodeGen/Thumb2/mve-intrinsics/vcmlaq.ll b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vcmlaq.ll
new file mode 100644
index 000000000..139c992d4
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vcmlaq.ll
@@ -0,0 +1,205 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve.fp -verify-machineinstrs -o - %s | FileCheck %s
+
+declare <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32)
+declare <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32)
+
+declare <8 x half> @llvm.arm.mve.vcmlaq.v8f16(i32, <8 x half>, <8 x half>, <8 x half>)
+declare <4 x float> @llvm.arm.mve.vcmlaq.v4f32(i32, <4 x float>, <4 x float>, <4 x float>)
+
+declare <8 x half> @llvm.arm.mve.vcmlaq.predicated.v8f16.v8i1(i32, <8 x half>, <8 x half>, <8 x half>, <8 x i1>)
+declare <4 x float> @llvm.arm.mve.vcmlaq.predicated.v4f32.v4i1(i32, <4 x float>, <4 x float>, <4 x float>, <4 x i1>)
+
+
+define arm_aapcs_vfpcc <8 x half> @test_vcmlaq_f16(<8 x half> %a, <8 x half> %b, <8 x half> %c) {
+; CHECK-LABEL: test_vcmlaq_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmla.f16 q0, q1, q2, #0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x half> @llvm.arm.mve.vcmlaq.v8f16(i32 0, <8 x half> %a, <8 x half> %b, <8 x half> %c)
+  ret <8 x half> %0
+}
+
+define arm_aapcs_vfpcc <4 x float> @test_vcmlaq_f32(<4 x float> %a, <4 x float> %b, <4 x float> %c) {
+; CHECK-LABEL: test_vcmlaq_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmla.f32 q0, q1, q2, #0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x float> @llvm.arm.mve.vcmlaq.v4f32(i32 0, <4 x float> %a, <4 x float> %b, <4 x float> %c)
+  ret <4 x float> %0
+}
+
+
+define arm_aapcs_vfpcc <8 x half> @test_vcmlaq_rot90_f16(<8 x half> %a, <8 x half> %b, <8 x half> %c) {
+; CHECK-LABEL: test_vcmlaq_rot90_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmla.f16 q0, q1, q2, #90
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x half> @llvm.arm.mve.vcmlaq.v8f16(i32 1, <8 x half> %a, <8 x half> %b, <8 x half> %c)
+  ret <8 x half> %0
+}
+
+define arm_aapcs_vfpcc <4 x float> @test_vcmlaq_rot90_f32(<4 x float> %a, <4 x float> %b, <4 x float> %c) {
+; CHECK-LABEL: test_vcmlaq_rot90_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmla.f32 q0, q1, q2, #90
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x float> @llvm.arm.mve.vcmlaq.v4f32(i32 1, <4 x float> %a, <4 x float> %b, <4 x float> %c)
+  ret <4 x float> %0
+}
+
+define arm_aapcs_vfpcc <8 x half> @test_vcmlaq_rot180_f16(<8 x half> %a, <8 x half> %b, <8 x half> %c) {
+; CHECK-LABEL: test_vcmlaq_rot180_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmla.f16 q0, q1, q2, #180
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x half> @llvm.arm.mve.vcmlaq.v8f16(i32 2, <8 x half> %a, <8 x half> %b, <8 x half> %c)
+  ret <8 x half> %0
+}
+
+define arm_aapcs_vfpcc <4 x float> @test_vcmlaq_rot180_f32(<4 x float> %a, <4 x float> %b, <4 x float> %c) {
+; CHECK-LABEL: test_vcmlaq_rot180_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmla.f32 q0, q1, q2, #180
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x float> @llvm.arm.mve.vcmlaq.v4f32(i32 2, <4 x float> %a, <4 x float> %b, <4 x float> %c)
+  ret <4 x float> %0
+}
+
+define arm_aapcs_vfpcc <8 x half> @test_vcmlaq_rot270_f16(<8 x half> %a, <8 x half> %b, <8 x half> %c) {
+; CHECK-LABEL: test_vcmlaq_rot270_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmla.f16 q0, q1, q2, #270
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x half> @llvm.arm.mve.vcmlaq.v8f16(i32 3, <8 x half> %a, <8 x half> %b, <8 x half> %c)
+  ret <8 x half> %0
+}
+
+define arm_aapcs_vfpcc <4 x float> @test_vcmlaq_rot270_f32(<4 x float> %a, <4 x float> %b, <4 x float> %c) {
+; CHECK-LABEL: test_vcmlaq_rot270_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmla.f32 q0, q1, q2, #270
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x float> @llvm.arm.mve.vcmlaq.v4f32(i32 3, <4 x float> %a, <4 x float> %b, <4 x float> %c)
+  ret <4 x float> %0
+}
+
+define arm_aapcs_vfpcc <8 x half> @test_vcmlaq_m_f16(<8 x half> %a, <8 x half> %b, <8 x half> %c, i16 zeroext %p) {
+; CHECK-LABEL: test_vcmlaq_m_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmlat.f16 q0, q1, q2, #0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <8 x half> @llvm.arm.mve.vcmlaq.predicated.v8f16.v8i1(i32 0, <8 x half> %a, <8 x half> %b, <8 x half> %c, <8 x i1> %1)
+  ret <8 x half> %2
+}
+
+define arm_aapcs_vfpcc <4 x float> @test_vcmlaq_m_f32(<4 x float> %a, <4 x float> %b, <4 x float> %c, i16 zeroext %p) {
+; CHECK-LABEL: test_vcmlaq_m_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmlat.f32 q0, q1, q2, #0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x float> @llvm.arm.mve.vcmlaq.predicated.v4f32.v4i1(i32 0, <4 x float> %a, <4 x float> %b, <4 x float> %c, <4 x i1> %1)
+  ret <4 x float> %2
+}
+
+define arm_aapcs_vfpcc <8 x half> @test_vcmlaq_rot90_m_f16(<8 x half> %a, <8 x half> %b, <8 x half> %c, i16 zeroext %p) {
+; CHECK-LABEL: test_vcmlaq_rot90_m_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmlat.f16 q0, q1, q2, #90
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <8 x half> @llvm.arm.mve.vcmlaq.predicated.v8f16.v8i1(i32 1, <8 x half> %a, <8 x half> %b, <8 x half> %c, <8 x i1> %1)
+  ret <8 x half> %2
+}
+
+define arm_aapcs_vfpcc <4 x float> @test_vcmlaq_rot90_m_f32(<4 x float> %a, <4 x float> %b, <4 x float> %c, i16 zeroext %p) {
+; CHECK-LABEL: test_vcmlaq_rot90_m_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmlat.f32 q0, q1, q2, #90
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x float> @llvm.arm.mve.vcmlaq.predicated.v4f32.v4i1(i32 1, <4 x float> %a, <4 x float> %b, <4 x float> %c, <4 x i1> %1)
+  ret <4 x float> %2
+}
+
+define arm_aapcs_vfpcc <8 x half> @test_vcmlaq_rot180_m_f16(<8 x half> %a, <8 x half> %b, <8 x half> %c, i16 zeroext %p) {
+; CHECK-LABEL: test_vcmlaq_rot180_m_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmlat.f16 q0, q1, q2, #180
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <8 x half> @llvm.arm.mve.vcmlaq.predicated.v8f16.v8i1(i32 2, <8 x half> %a, <8 x half> %b, <8 x half> %c, <8 x i1> %1)
+  ret <8 x half> %2
+}
+
+define arm_aapcs_vfpcc <4 x float> @test_vcmlaq_rot180_m_f32(<4 x float> %a, <4 x float> %b, <4 x float> %c, i16 zeroext %p) {
+; CHECK-LABEL: test_vcmlaq_rot180_m_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmlat.f32 q0, q1, q2, #180
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x float> @llvm.arm.mve.vcmlaq.predicated.v4f32.v4i1(i32 2, <4 x float> %a, <4 x float> %b, <4 x float> %c, <4 x i1> %1)
+  ret <4 x float> %2
+}
+
+define arm_aapcs_vfpcc <8 x half> @test_vcmlaq_rot270_m_f16(<8 x half> %a, <8 x half> %b, <8 x half> %c, i16 zeroext %p) {
+; CHECK-LABEL: test_vcmlaq_rot270_m_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmlat.f16 q0, q1, q2, #270
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <8 x half> @llvm.arm.mve.vcmlaq.predicated.v8f16.v8i1(i32 3, <8 x half> %a, <8 x half> %b, <8 x half> %c, <8 x i1> %1)
+  ret <8 x half> %2
+}
+
+define arm_aapcs_vfpcc <4 x float> @test_vcmlaq_rot270_m_f32(<4 x float> %a, <4 x float> %b, <4 x float> %c, i16 zeroext %p) {
+; CHECK-LABEL: test_vcmlaq_rot270_m_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmlat.f32 q0, q1, q2, #270
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x float> @llvm.arm.mve.vcmlaq.predicated.v4f32.v4i1(i32 3, <4 x float> %a, <4 x float> %b, <4 x float> %c, <4 x i1> %1)
+  ret <4 x float> %2
+}
diff --git a/llvm/test/CodeGen/Thumb2/mve-intrinsics/vcmulq.ll b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vcmulq.ll
new file mode 100644
index 000000000..199711579
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vcmulq.ll
@@ -0,0 +1,323 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve.fp -verify-machineinstrs -o - %s | FileCheck %s
+
+declare <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32)
+declare <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32)
+
+declare <8 x half> @llvm.arm.mve.vcmulq.v8f16(i32, <8 x half>, <8 x half>)
+declare <4 x float> @llvm.arm.mve.vcmulq.v4f32(i32, <4 x float>, <4 x float>)
+
+declare <8 x half> @llvm.arm.mve.vcmulq.predicated.v8f16.v8i1(i32, <8 x half>, <8 x half>, <8 x half>, <8 x i1>)
+declare <4 x float> @llvm.arm.mve.vcmulq.predicated.v4f32.v4i1(i32, <4 x float>, <4 x float>, <4 x float>, <4 x i1>)
+
+define arm_aapcs_vfpcc <8 x half> @test_vcmulq_f16(<8 x half> %a, <8 x half> %b) {
+; CHECK-LABEL: test_vcmulq_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmul.f16 q0, q0, q1, #0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x half> @llvm.arm.mve.vcmulq.v8f16(i32 0, <8 x half> %a, <8 x half> %b)
+  ret <8 x half> %0
+}
+
+define arm_aapcs_vfpcc <4 x float> @test_vcmulq_f32(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: test_vcmulq_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmul.f32 q2, q0, q1, #0
+; CHECK-NEXT:    vmov q0, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x float> @llvm.arm.mve.vcmulq.v4f32(i32 0, <4 x float> %a, <4 x float> %b)
+  ret <4 x float> %0
+}
+
+define arm_aapcs_vfpcc <8 x half> @test_vcmulq_rot90_f16(<8 x half> %a, <8 x half> %b) {
+; CHECK-LABEL: test_vcmulq_rot90_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmul.f16 q0, q0, q1, #90
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x half> @llvm.arm.mve.vcmulq.v8f16(i32 1, <8 x half> %a, <8 x half> %b)
+  ret <8 x half> %0
+}
+
+define arm_aapcs_vfpcc <4 x float> @test_vcmulq_rot90_f32(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: test_vcmulq_rot90_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmul.f32 q2, q0, q1, #90
+; CHECK-NEXT:    vmov q0, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x float> @llvm.arm.mve.vcmulq.v4f32(i32 1, <4 x float> %a, <4 x float> %b)
+  ret <4 x float> %0
+}
+
+define arm_aapcs_vfpcc <8 x half> @test_vcmulq_rot180_f16(<8 x half> %a, <8 x half> %b) {
+; CHECK-LABEL: test_vcmulq_rot180_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmul.f16 q0, q0, q1, #180
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x half> @llvm.arm.mve.vcmulq.v8f16(i32 2, <8 x half> %a, <8 x half> %b)
+  ret <8 x half> %0
+}
+
+define arm_aapcs_vfpcc <4 x float> @test_vcmulq_rot180_f32(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: test_vcmulq_rot180_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmul.f32 q2, q0, q1, #180
+; CHECK-NEXT:    vmov q0, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x float> @llvm.arm.mve.vcmulq.v4f32(i32 2, <4 x float> %a, <4 x float> %b)
+  ret <4 x float> %0
+}
+
+define arm_aapcs_vfpcc <8 x half> @test_vcmulq_rot270_f16(<8 x half> %a, <8 x half> %b) {
+; CHECK-LABEL: test_vcmulq_rot270_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmul.f16 q0, q0, q1, #270
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x half> @llvm.arm.mve.vcmulq.v8f16(i32 3, <8 x half> %a, <8 x half> %b)
+  ret <8 x half> %0
+}
+
+define arm_aapcs_vfpcc <4 x float> @test_vcmulq_rot270_f32(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: test_vcmulq_rot270_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmul.f32 q2, q0, q1, #270
+; CHECK-NEXT:    vmov q0, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x float> @llvm.arm.mve.vcmulq.v4f32(i32 3, <4 x float> %a, <4 x float> %b)
+  ret <4 x float> %0
+}
+
+define arm_aapcs_vfpcc <8 x half> @test_vcmulq_m_f16(<8 x half> %inactive, <8 x half> %a, <8 x half> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vcmulq_m_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmult.f16 q0, q1, q2, #0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <8 x half> @llvm.arm.mve.vcmulq.predicated.v8f16.v8i1(i32 0, <8 x half> %inactive, <8 x half> %a, <8 x half> %b, <8 x i1> %1)
+  ret <8 x half> %2
+}
+
+define arm_aapcs_vfpcc <4 x float> @test_vcmulq_m_f32(<4 x float> %inactive, <4 x float> %a, <4 x float> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vcmulq_m_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmult.f32 q0, q1, q2, #0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x float> @llvm.arm.mve.vcmulq.predicated.v4f32.v4i1(i32 0, <4 x float> %inactive, <4 x float> %a, <4 x float> %b, <4 x i1> %1)
+  ret <4 x float> %2
+}
+
+define arm_aapcs_vfpcc <8 x half> @test_vcmulq_rot90_m_f16(<8 x half> %inactive, <8 x half> %a, <8 x half> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vcmulq_rot90_m_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmult.f16 q0, q1, q2, #90
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <8 x half> @llvm.arm.mve.vcmulq.predicated.v8f16.v8i1(i32 1, <8 x half> %inactive, <8 x half> %a, <8 x half> %b, <8 x i1> %1)
+  ret <8 x half> %2
+}
+
+define arm_aapcs_vfpcc <4 x float> @test_vcmulq_rot90_m_f32(<4 x float> %inactive, <4 x float> %a, <4 x float> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vcmulq_rot90_m_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmult.f32 q0, q1, q2, #90
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x float> @llvm.arm.mve.vcmulq.predicated.v4f32.v4i1(i32 1, <4 x float> %inactive, <4 x float> %a, <4 x float> %b, <4 x i1> %1)
+  ret <4 x float> %2
+}
+
+define arm_aapcs_vfpcc <8 x half> @test_vcmulq_rot180_m_f16(<8 x half> %inactive, <8 x half> %a, <8 x half> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vcmulq_rot180_m_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmult.f16 q0, q1, q2, #180
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <8 x half> @llvm.arm.mve.vcmulq.predicated.v8f16.v8i1(i32 2, <8 x half> %inactive, <8 x half> %a, <8 x half> %b, <8 x i1> %1)
+  ret <8 x half> %2
+}
+
+define arm_aapcs_vfpcc <4 x float> @test_vcmulq_rot180_m_f32(<4 x float> %inactive, <4 x float> %a, <4 x float> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vcmulq_rot180_m_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmult.f32 q0, q1, q2, #180
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x float> @llvm.arm.mve.vcmulq.predicated.v4f32.v4i1(i32 2, <4 x float> %inactive, <4 x float> %a, <4 x float> %b, <4 x i1> %1)
+  ret <4 x float> %2
+}
+
+define arm_aapcs_vfpcc <8 x half> @test_vcmulq_rot270_m_f16(<8 x half> %inactive, <8 x half> %a, <8 x half> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vcmulq_rot270_m_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmult.f16 q0, q1, q2, #270
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <8 x half> @llvm.arm.mve.vcmulq.predicated.v8f16.v8i1(i32 3, <8 x half> %inactive, <8 x half> %a, <8 x half> %b, <8 x i1> %1)
+  ret <8 x half> %2
+}
+
+define arm_aapcs_vfpcc <4 x float> @test_vcmulq_rot270_m_f32(<4 x float> %inactive, <4 x float> %a, <4 x float> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vcmulq_rot270_m_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmult.f32 q0, q1, q2, #270
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x float> @llvm.arm.mve.vcmulq.predicated.v4f32.v4i1(i32 3, <4 x float> %inactive, <4 x float> %a, <4 x float> %b, <4 x i1> %1)
+  ret <4 x float> %2
+}
+
+define arm_aapcs_vfpcc <8 x half> @test_vcmulq_x_f16(<8 x half> %a, <8 x half> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vcmulq_x_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmult.f16 q0, q0, q1, #0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <8 x half> @llvm.arm.mve.vcmulq.predicated.v8f16.v8i1(i32 0, <8 x half> undef, <8 x half> %a, <8 x half> %b, <8 x i1> %1)
+  ret <8 x half> %2
+}
+
+define arm_aapcs_vfpcc <4 x float> @test_vcmulq_x_f32(<4 x float> %a, <4 x float> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vcmulq_x_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmult.f32 q2, q0, q1, #0
+; CHECK-NEXT:    vmov q0, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x float> @llvm.arm.mve.vcmulq.predicated.v4f32.v4i1(i32 0, <4 x float> undef, <4 x float> %a, <4 x float> %b, <4 x i1> %1)
+  ret <4 x float> %2
+}
+
+define arm_aapcs_vfpcc <8 x half> @test_vcmulq_rot90_x_f16(<8 x half> %a, <8 x half> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vcmulq_rot90_x_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmult.f16 q0, q0, q1, #90
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <8 x half> @llvm.arm.mve.vcmulq.predicated.v8f16.v8i1(i32 1, <8 x half> undef, <8 x half> %a, <8 x half> %b, <8 x i1> %1)
+  ret <8 x half> %2
+}
+
+define arm_aapcs_vfpcc <4 x float> @test_vcmulq_rot90_x_f32(<4 x float> %a, <4 x float> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vcmulq_rot90_x_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmult.f32 q2, q0, q1, #90
+; CHECK-NEXT:    vmov q0, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x float> @llvm.arm.mve.vcmulq.predicated.v4f32.v4i1(i32 1, <4 x float> undef, <4 x float> %a, <4 x float> %b, <4 x i1> %1)
+  ret <4 x float> %2
+}
+
+define arm_aapcs_vfpcc <8 x half> @test_vcmulq_rot180_x_f16(<8 x half> %a, <8 x half> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vcmulq_rot180_x_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmult.f16 q0, q0, q1, #180
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <8 x half> @llvm.arm.mve.vcmulq.predicated.v8f16.v8i1(i32 2, <8 x half> undef, <8 x half> %a, <8 x half> %b, <8 x i1> %1)
+  ret <8 x half> %2
+}
+
+define arm_aapcs_vfpcc <4 x float> @test_vcmulq_rot180_x_f32(<4 x float> %a, <4 x float> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vcmulq_rot180_x_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmult.f32 q2, q0, q1, #180
+; CHECK-NEXT:    vmov q0, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x float> @llvm.arm.mve.vcmulq.predicated.v4f32.v4i1(i32 2, <4 x float> undef, <4 x float> %a, <4 x float> %b, <4 x i1> %1)
+  ret <4 x float> %2
+}
+
+define arm_aapcs_vfpcc <8 x half> @test_vcmulq_rot270_x_f16(<8 x half> %a, <8 x half> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vcmulq_rot270_x_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmult.f16 q0, q0, q1, #270
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <8 x half> @llvm.arm.mve.vcmulq.predicated.v8f16.v8i1(i32 3, <8 x half> undef, <8 x half> %a, <8 x half> %b, <8 x i1> %1)
+  ret <8 x half> %2
+}
+
+define arm_aapcs_vfpcc <4 x float> @test_vcmulq_rot270_x_f32(<4 x float> %a, <4 x float> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vcmulq_rot270_x_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmult.f32 q2, q0, q1, #270
+; CHECK-NEXT:    vmov q0, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x float> @llvm.arm.mve.vcmulq.predicated.v4f32.v4i1(i32 3, <4 x float> undef, <4 x float> %a, <4 x float> %b, <4 x i1> %1)
+  ret <4 x float> %2
+}
diff --git a/llvm/test/CodeGen/Thumb2/mve-intrinsics/vcvt.ll b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vcvt.ll
new file mode 100644
index 000000000..1e9cabf7c
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vcvt.ll
@@ -0,0 +1,56 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve.fp -verify-machineinstrs -o - %s | FileCheck %s
+
+define arm_aapcs_vfpcc <8 x half> @test_vcvttq_f16_f32(<8 x half> %a, <4 x float> %b) {
+; CHECK-LABEL: test_vcvttq_f16_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcvtt.f16.f32 q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <8 x half> @llvm.arm.mve.vcvt.narrow(<8 x half> %a, <4 x float> %b, i32 1)
+  ret <8 x half> %0
+}
+
+define arm_aapcs_vfpcc <8 x half> @test_vcvtbq_f16_f32(<8 x half> %a, <4 x float> %b) {
+; CHECK-LABEL: test_vcvtbq_f16_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcvtb.f16.f32 q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <8 x half> @llvm.arm.mve.vcvt.narrow(<8 x half> %a, <4 x float> %b, i32 0)
+  ret <8 x half> %0
+}
+
+declare <8 x half> @llvm.arm.mve.vcvt.narrow(<8 x half>, <4 x float>, i32)
+
+define arm_aapcs_vfpcc <8 x half> @test_vcvttq_m_f16_f32(<8 x half> %a, <4 x float> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vcvttq_m_f16_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcvttt.f16.f32 q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call <8 x half> @llvm.arm.mve.vcvt.narrow.predicated(<8 x half> %a, <4 x float> %b, i32 1, <4 x i1> %1)
+  ret <8 x half> %2
+}
+
+define arm_aapcs_vfpcc <8 x half> @test_vcvtbq_m_f16_f32(<8 x half> %a, <4 x float> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vcvtbq_m_f16_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcvtbt.f16.f32 q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call <8 x half> @llvm.arm.mve.vcvt.narrow.predicated(<8 x half> %a, <4 x float> %b, i32 0, <4 x i1> %1)
+  ret <8 x half> %2
+}
+
+declare <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32)
+
+declare <8 x half> @llvm.arm.mve.vcvt.narrow.predicated(<8 x half>, <4 x float>, i32, <4 x i1>)
diff --git a/llvm/test/CodeGen/Thumb2/mve-intrinsics/vector-shift-imm-dyadic.ll b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vector-shift-imm-dyadic.ll
new file mode 100644
index 000000000..6173cb552
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vector-shift-imm-dyadic.ll
@@ -0,0 +1,1270 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve.fp -verify-machineinstrs -o - %s | FileCheck %s
+
+define arm_aapcs_vfpcc <16 x i8> @test_vshrnbq_n_s16(<16 x i8> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vshrnbq_n_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vshrnb.i16 q0, q1, #3
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <16 x i8> @llvm.arm.mve.vshrn.v16i8.v8i16(<16 x i8> %a, <8 x i16> %b, i32 3, i32 0, i32 0, i32 0, i32 0, i32 0)
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vshrnbq_n_s32(<8 x i16> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vshrnbq_n_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vshrnb.i32 q0, q1, #9
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.arm.mve.vshrn.v8i16.v4i32(<8 x i16> %a, <4 x i32> %b, i32 9, i32 0, i32 0, i32 0, i32 0, i32 0)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vshrnbq_n_u16(<16 x i8> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vshrnbq_n_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vshrnb.i16 q0, q1, #1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <16 x i8> @llvm.arm.mve.vshrn.v16i8.v8i16(<16 x i8> %a, <8 x i16> %b, i32 1, i32 0, i32 0, i32 1, i32 1, i32 0)
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vshrnbq_n_u32(<8 x i16> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vshrnbq_n_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vshrnb.i32 q0, q1, #3
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.arm.mve.vshrn.v8i16.v4i32(<8 x i16> %a, <4 x i32> %b, i32 3, i32 0, i32 0, i32 1, i32 1, i32 0)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vshrntq_n_s16(<16 x i8> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vshrntq_n_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vshrnt.i16 q0, q1, #1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <16 x i8> @llvm.arm.mve.vshrn.v16i8.v8i16(<16 x i8> %a, <8 x i16> %b, i32 1, i32 0, i32 0, i32 0, i32 0, i32 1)
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vshrntq_n_s32(<8 x i16> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vshrntq_n_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vshrnt.i32 q0, q1, #10
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.arm.mve.vshrn.v8i16.v4i32(<8 x i16> %a, <4 x i32> %b, i32 10, i32 0, i32 0, i32 0, i32 0, i32 1)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vshrntq_n_u16(<16 x i8> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vshrntq_n_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vshrnt.i16 q0, q1, #6
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <16 x i8> @llvm.arm.mve.vshrn.v16i8.v8i16(<16 x i8> %a, <8 x i16> %b, i32 6, i32 0, i32 0, i32 1, i32 1, i32 1)
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vshrntq_n_u32(<8 x i16> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vshrntq_n_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vshrnt.i32 q0, q1, #10
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.arm.mve.vshrn.v8i16.v4i32(<8 x i16> %a, <4 x i32> %b, i32 10, i32 0, i32 0, i32 1, i32 1, i32 1)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vshrnbq_m_n_s16(<16 x i8> %a, <8 x i16> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vshrnbq_m_n_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshrnbt.i16 q0, q1, #4
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <16 x i8> @llvm.arm.mve.vshrn.predicated.v16i8.v8i16.v8i1(<16 x i8> %a, <8 x i16> %b, i32 4, i32 0, i32 0, i32 0, i32 0, i32 0, <8 x i1> %1)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vshrnbq_m_n_s32(<8 x i16> %a, <4 x i32> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vshrnbq_m_n_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshrnbt.i32 q0, q1, #13
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <8 x i16> @llvm.arm.mve.vshrn.predicated.v8i16.v4i32.v4i1(<8 x i16> %a, <4 x i32> %b, i32 13, i32 0, i32 0, i32 0, i32 0, i32 0, <4 x i1> %1)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vshrnbq_m_n_u16(<16 x i8> %a, <8 x i16> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vshrnbq_m_n_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshrnbt.i16 q0, q1, #7
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <16 x i8> @llvm.arm.mve.vshrn.predicated.v16i8.v8i16.v8i1(<16 x i8> %a, <8 x i16> %b, i32 7, i32 0, i32 0, i32 1, i32 1, i32 0, <8 x i1> %1)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vshrnbq_m_n_u32(<8 x i16> %a, <4 x i32> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vshrnbq_m_n_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshrnbt.i32 q0, q1, #15
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <8 x i16> @llvm.arm.mve.vshrn.predicated.v8i16.v4i32.v4i1(<8 x i16> %a, <4 x i32> %b, i32 15, i32 0, i32 0, i32 1, i32 1, i32 0, <4 x i1> %1)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vshrntq_m_n_s16(<16 x i8> %a, <8 x i16> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vshrntq_m_n_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshrntt.i16 q0, q1, #6
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <16 x i8> @llvm.arm.mve.vshrn.predicated.v16i8.v8i16.v8i1(<16 x i8> %a, <8 x i16> %b, i32 6, i32 0, i32 0, i32 0, i32 0, i32 1, <8 x i1> %1)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vshrntq_m_n_s32(<8 x i16> %a, <4 x i32> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vshrntq_m_n_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshrntt.i32 q0, q1, #13
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <8 x i16> @llvm.arm.mve.vshrn.predicated.v8i16.v4i32.v4i1(<8 x i16> %a, <4 x i32> %b, i32 13, i32 0, i32 0, i32 0, i32 0, i32 1, <4 x i1> %1)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vshrntq_m_n_u16(<16 x i8> %a, <8 x i16> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vshrntq_m_n_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshrntt.i16 q0, q1, #1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <16 x i8> @llvm.arm.mve.vshrn.predicated.v16i8.v8i16.v8i1(<16 x i8> %a, <8 x i16> %b, i32 1, i32 0, i32 0, i32 1, i32 1, i32 1, <8 x i1> %1)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vshrntq_m_n_u32(<8 x i16> %a, <4 x i32> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vshrntq_m_n_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshrntt.i32 q0, q1, #10
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <8 x i16> @llvm.arm.mve.vshrn.predicated.v8i16.v4i32.v4i1(<8 x i16> %a, <4 x i32> %b, i32 10, i32 0, i32 0, i32 1, i32 1, i32 1, <4 x i1> %1)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vrshrnbq_n_s16(<16 x i8> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vrshrnbq_n_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vrshrnb.i16 q0, q1, #5
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <16 x i8> @llvm.arm.mve.vshrn.v16i8.v8i16(<16 x i8> %a, <8 x i16> %b, i32 5, i32 0, i32 1, i32 0, i32 0, i32 0)
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vrshrnbq_n_s32(<8 x i16> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vrshrnbq_n_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vrshrnb.i32 q0, q1, #10
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.arm.mve.vshrn.v8i16.v4i32(<8 x i16> %a, <4 x i32> %b, i32 10, i32 0, i32 1, i32 0, i32 0, i32 0)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vrshrnbq_n_u16(<16 x i8> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vrshrnbq_n_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vrshrnb.i16 q0, q1, #2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <16 x i8> @llvm.arm.mve.vshrn.v16i8.v8i16(<16 x i8> %a, <8 x i16> %b, i32 2, i32 0, i32 1, i32 1, i32 1, i32 0)
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vrshrnbq_n_u32(<8 x i16> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vrshrnbq_n_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vrshrnb.i32 q0, q1, #12
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.arm.mve.vshrn.v8i16.v4i32(<8 x i16> %a, <4 x i32> %b, i32 12, i32 0, i32 1, i32 1, i32 1, i32 0)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vrshrntq_n_s16(<16 x i8> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vrshrntq_n_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vrshrnt.i16 q0, q1, #4
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <16 x i8> @llvm.arm.mve.vshrn.v16i8.v8i16(<16 x i8> %a, <8 x i16> %b, i32 4, i32 0, i32 1, i32 0, i32 0, i32 1)
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vrshrntq_n_s32(<8 x i16> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vrshrntq_n_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vrshrnt.i32 q0, q1, #11
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.arm.mve.vshrn.v8i16.v4i32(<8 x i16> %a, <4 x i32> %b, i32 11, i32 0, i32 1, i32 0, i32 0, i32 1)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vrshrntq_n_u16(<16 x i8> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vrshrntq_n_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vrshrnt.i16 q0, q1, #1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <16 x i8> @llvm.arm.mve.vshrn.v16i8.v8i16(<16 x i8> %a, <8 x i16> %b, i32 1, i32 0, i32 1, i32 1, i32 1, i32 1)
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vrshrntq_n_u32(<8 x i16> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vrshrntq_n_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vrshrnt.i32 q0, q1, #6
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.arm.mve.vshrn.v8i16.v4i32(<8 x i16> %a, <4 x i32> %b, i32 6, i32 0, i32 1, i32 1, i32 1, i32 1)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vrshrnbq_m_n_s16(<16 x i8> %a, <8 x i16> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vrshrnbq_m_n_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vrshrnbt.i16 q0, q1, #1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <16 x i8> @llvm.arm.mve.vshrn.predicated.v16i8.v8i16.v8i1(<16 x i8> %a, <8 x i16> %b, i32 1, i32 0, i32 1, i32 0, i32 0, i32 0, <8 x i1> %1)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vrshrnbq_m_n_s32(<8 x i16> %a, <4 x i32> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vrshrnbq_m_n_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vrshrnbt.i32 q0, q1, #14
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <8 x i16> @llvm.arm.mve.vshrn.predicated.v8i16.v4i32.v4i1(<8 x i16> %a, <4 x i32> %b, i32 14, i32 0, i32 1, i32 0, i32 0, i32 0, <4 x i1> %1)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vrshrnbq_m_n_u16(<16 x i8> %a, <8 x i16> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vrshrnbq_m_n_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vrshrnbt.i16 q0, q1, #2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <16 x i8> @llvm.arm.mve.vshrn.predicated.v16i8.v8i16.v8i1(<16 x i8> %a, <8 x i16> %b, i32 2, i32 0, i32 1, i32 1, i32 1, i32 0, <8 x i1> %1)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vrshrnbq_m_n_u32(<8 x i16> %a, <4 x i32> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vrshrnbq_m_n_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vrshrnbt.i32 q0, q1, #12
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <8 x i16> @llvm.arm.mve.vshrn.predicated.v8i16.v4i32.v4i1(<8 x i16> %a, <4 x i32> %b, i32 12, i32 0, i32 1, i32 1, i32 1, i32 0, <4 x i1> %1)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vrshrntq_m_n_s16(<16 x i8> %a, <8 x i16> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vrshrntq_m_n_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vrshrntt.i16 q0, q1, #4
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <16 x i8> @llvm.arm.mve.vshrn.predicated.v16i8.v8i16.v8i1(<16 x i8> %a, <8 x i16> %b, i32 4, i32 0, i32 1, i32 0, i32 0, i32 1, <8 x i1> %1)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vrshrntq_m_n_s32(<8 x i16> %a, <4 x i32> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vrshrntq_m_n_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vrshrntt.i32 q0, q1, #6
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <8 x i16> @llvm.arm.mve.vshrn.predicated.v8i16.v4i32.v4i1(<8 x i16> %a, <4 x i32> %b, i32 6, i32 0, i32 1, i32 0, i32 0, i32 1, <4 x i1> %1)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vrshrntq_m_n_u16(<16 x i8> %a, <8 x i16> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vrshrntq_m_n_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vrshrntt.i16 q0, q1, #6
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <16 x i8> @llvm.arm.mve.vshrn.predicated.v16i8.v8i16.v8i1(<16 x i8> %a, <8 x i16> %b, i32 6, i32 0, i32 1, i32 1, i32 1, i32 1, <8 x i1> %1)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vrshrntq_m_n_u32(<8 x i16> %a, <4 x i32> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vrshrntq_m_n_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vrshrntt.i32 q0, q1, #10
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <8 x i16> @llvm.arm.mve.vshrn.predicated.v8i16.v4i32.v4i1(<8 x i16> %a, <4 x i32> %b, i32 10, i32 0, i32 1, i32 1, i32 1, i32 1, <4 x i1> %1)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vqshrnbq_n_s16(<16 x i8> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vqshrnbq_n_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqshrnb.s16 q0, q1, #7
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <16 x i8> @llvm.arm.mve.vshrn.v16i8.v8i16(<16 x i8> %a, <8 x i16> %b, i32 7, i32 1, i32 0, i32 0, i32 0, i32 0)
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vqshrnbq_n_s32(<8 x i16> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vqshrnbq_n_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqshrnb.s32 q0, q1, #15
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.arm.mve.vshrn.v8i16.v4i32(<8 x i16> %a, <4 x i32> %b, i32 15, i32 1, i32 0, i32 0, i32 0, i32 0)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vqshrnbq_n_u16(<16 x i8> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vqshrnbq_n_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqshrnb.u16 q0, q1, #3
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <16 x i8> @llvm.arm.mve.vshrn.v16i8.v8i16(<16 x i8> %a, <8 x i16> %b, i32 3, i32 1, i32 0, i32 1, i32 1, i32 0)
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vqshrnbq_n_u32(<8 x i16> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vqshrnbq_n_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqshrnb.u32 q0, q1, #3
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.arm.mve.vshrn.v8i16.v4i32(<8 x i16> %a, <4 x i32> %b, i32 3, i32 1, i32 0, i32 1, i32 1, i32 0)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vqshrntq_n_s16(<16 x i8> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vqshrntq_n_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqshrnt.s16 q0, q1, #5
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <16 x i8> @llvm.arm.mve.vshrn.v16i8.v8i16(<16 x i8> %a, <8 x i16> %b, i32 5, i32 1, i32 0, i32 0, i32 0, i32 1)
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vqshrntq_n_s32(<8 x i16> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vqshrntq_n_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqshrnt.s32 q0, q1, #6
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.arm.mve.vshrn.v8i16.v4i32(<8 x i16> %a, <4 x i32> %b, i32 6, i32 1, i32 0, i32 0, i32 0, i32 1)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vqshrntq_n_u16(<16 x i8> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vqshrntq_n_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqshrnt.u16 q0, q1, #1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <16 x i8> @llvm.arm.mve.vshrn.v16i8.v8i16(<16 x i8> %a, <8 x i16> %b, i32 1, i32 1, i32 0, i32 1, i32 1, i32 1)
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vqshrntq_n_u32(<8 x i16> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vqshrntq_n_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqshrnt.u32 q0, q1, #15
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.arm.mve.vshrn.v8i16.v4i32(<8 x i16> %a, <4 x i32> %b, i32 15, i32 1, i32 0, i32 1, i32 1, i32 1)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vqshrnbq_m_n_s16(<16 x i8> %a, <8 x i16> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vqshrnbq_m_n_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqshrnbt.s16 q0, q1, #7
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <16 x i8> @llvm.arm.mve.vshrn.predicated.v16i8.v8i16.v8i1(<16 x i8> %a, <8 x i16> %b, i32 7, i32 1, i32 0, i32 0, i32 0, i32 0, <8 x i1> %1)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vqshrnbq_m_n_s32(<8 x i16> %a, <4 x i32> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vqshrnbq_m_n_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqshrnbt.s32 q0, q1, #1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <8 x i16> @llvm.arm.mve.vshrn.predicated.v8i16.v4i32.v4i1(<8 x i16> %a, <4 x i32> %b, i32 1, i32 1, i32 0, i32 0, i32 0, i32 0, <4 x i1> %1)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vqshrnbq_m_n_u16(<16 x i8> %a, <8 x i16> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vqshrnbq_m_n_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqshrnbt.u16 q0, q1, #1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <16 x i8> @llvm.arm.mve.vshrn.predicated.v16i8.v8i16.v8i1(<16 x i8> %a, <8 x i16> %b, i32 1, i32 1, i32 0, i32 1, i32 1, i32 0, <8 x i1> %1)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vqshrnbq_m_n_u32(<8 x i16> %a, <4 x i32> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vqshrnbq_m_n_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqshrnbt.u32 q0, q1, #8
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <8 x i16> @llvm.arm.mve.vshrn.predicated.v8i16.v4i32.v4i1(<8 x i16> %a, <4 x i32> %b, i32 8, i32 1, i32 0, i32 1, i32 1, i32 0, <4 x i1> %1)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vqshrntq_m_n_s16(<16 x i8> %a, <8 x i16> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vqshrntq_m_n_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqshrntt.s16 q0, q1, #1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <16 x i8> @llvm.arm.mve.vshrn.predicated.v16i8.v8i16.v8i1(<16 x i8> %a, <8 x i16> %b, i32 1, i32 1, i32 0, i32 0, i32 0, i32 1, <8 x i1> %1)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vqshrntq_m_n_s32(<8 x i16> %a, <4 x i32> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vqshrntq_m_n_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqshrntt.s32 q0, q1, #11
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <8 x i16> @llvm.arm.mve.vshrn.predicated.v8i16.v4i32.v4i1(<8 x i16> %a, <4 x i32> %b, i32 11, i32 1, i32 0, i32 0, i32 0, i32 1, <4 x i1> %1)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vqshrntq_m_n_u16(<16 x i8> %a, <8 x i16> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vqshrntq_m_n_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqshrntt.u16 q0, q1, #3
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <16 x i8> @llvm.arm.mve.vshrn.predicated.v16i8.v8i16.v8i1(<16 x i8> %a, <8 x i16> %b, i32 3, i32 1, i32 0, i32 1, i32 1, i32 1, <8 x i1> %1)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vqshrntq_m_n_u32(<8 x i16> %a, <4 x i32> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vqshrntq_m_n_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqshrntt.u32 q0, q1, #1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <8 x i16> @llvm.arm.mve.vshrn.predicated.v8i16.v4i32.v4i1(<8 x i16> %a, <4 x i32> %b, i32 1, i32 1, i32 0, i32 1, i32 1, i32 1, <4 x i1> %1)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vqshrunbq_n_s16(<16 x i8> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vqshrunbq_n_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqshrunb.s16 q0, q1, #5
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <16 x i8> @llvm.arm.mve.vshrn.v16i8.v8i16(<16 x i8> %a, <8 x i16> %b, i32 5, i32 1, i32 0, i32 1, i32 0, i32 0)
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vqshrunbq_n_s32(<8 x i16> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vqshrunbq_n_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqshrunb.s32 q0, q1, #13
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.arm.mve.vshrn.v8i16.v4i32(<8 x i16> %a, <4 x i32> %b, i32 13, i32 1, i32 0, i32 1, i32 0, i32 0)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vqshruntq_n_s16(<16 x i8> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vqshruntq_n_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqshrunt.s16 q0, q1, #2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <16 x i8> @llvm.arm.mve.vshrn.v16i8.v8i16(<16 x i8> %a, <8 x i16> %b, i32 2, i32 1, i32 0, i32 1, i32 0, i32 1)
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vqshruntq_n_s32(<8 x i16> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vqshruntq_n_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqshrunt.s32 q0, q1, #7
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.arm.mve.vshrn.v8i16.v4i32(<8 x i16> %a, <4 x i32> %b, i32 7, i32 1, i32 0, i32 1, i32 0, i32 1)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vqshrunbq_m_n_s16(<16 x i8> %a, <8 x i16> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vqshrunbq_m_n_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqshrunbt.s16 q0, q1, #7
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <16 x i8> @llvm.arm.mve.vshrn.predicated.v16i8.v8i16.v8i1(<16 x i8> %a, <8 x i16> %b, i32 7, i32 1, i32 0, i32 1, i32 0, i32 0, <8 x i1> %1)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vqshrunbq_m_n_s32(<8 x i16> %a, <4 x i32> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vqshrunbq_m_n_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqshrunbt.s32 q0, q1, #7
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <8 x i16> @llvm.arm.mve.vshrn.predicated.v8i16.v4i32.v4i1(<8 x i16> %a, <4 x i32> %b, i32 7, i32 1, i32 0, i32 1, i32 0, i32 0, <4 x i1> %1)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vqshruntq_m_n_s16(<16 x i8> %a, <8 x i16> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vqshruntq_m_n_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqshruntt.s16 q0, q1, #7
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <16 x i8> @llvm.arm.mve.vshrn.predicated.v16i8.v8i16.v8i1(<16 x i8> %a, <8 x i16> %b, i32 7, i32 1, i32 0, i32 1, i32 0, i32 1, <8 x i1> %1)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vqshruntq_m_n_s32(<8 x i16> %a, <4 x i32> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vqshruntq_m_n_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqshruntt.s32 q0, q1, #7
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <8 x i16> @llvm.arm.mve.vshrn.predicated.v8i16.v4i32.v4i1(<8 x i16> %a, <4 x i32> %b, i32 7, i32 1, i32 0, i32 1, i32 0, i32 1, <4 x i1> %1)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vqrshrnbq_n_s16(<16 x i8> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vqrshrnbq_n_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqrshrnb.s16 q0, q1, #5
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <16 x i8> @llvm.arm.mve.vshrn.v16i8.v8i16(<16 x i8> %a, <8 x i16> %b, i32 5, i32 1, i32 1, i32 0, i32 0, i32 0)
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vqrshrnbq_n_s32(<8 x i16> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vqrshrnbq_n_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqrshrnb.s32 q0, q1, #13
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.arm.mve.vshrn.v8i16.v4i32(<8 x i16> %a, <4 x i32> %b, i32 13, i32 1, i32 1, i32 0, i32 0, i32 0)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vqrshrnbq_n_u16(<16 x i8> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vqrshrnbq_n_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqrshrnb.u16 q0, q1, #7
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <16 x i8> @llvm.arm.mve.vshrn.v16i8.v8i16(<16 x i8> %a, <8 x i16> %b, i32 7, i32 1, i32 1, i32 1, i32 1, i32 0)
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vqrshrnbq_n_u32(<8 x i16> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vqrshrnbq_n_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqrshrnb.u32 q0, q1, #8
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.arm.mve.vshrn.v8i16.v4i32(<8 x i16> %a, <4 x i32> %b, i32 8, i32 1, i32 1, i32 1, i32 1, i32 0)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vqrshrntq_n_s16(<16 x i8> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vqrshrntq_n_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqrshrnt.s16 q0, q1, #7
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <16 x i8> @llvm.arm.mve.vshrn.v16i8.v8i16(<16 x i8> %a, <8 x i16> %b, i32 7, i32 1, i32 1, i32 0, i32 0, i32 1)
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vqrshrntq_n_s32(<8 x i16> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vqrshrntq_n_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqrshrnt.s32 q0, q1, #2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.arm.mve.vshrn.v8i16.v4i32(<8 x i16> %a, <4 x i32> %b, i32 2, i32 1, i32 1, i32 0, i32 0, i32 1)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vqrshrntq_n_u16(<16 x i8> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vqrshrntq_n_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqrshrnt.u16 q0, q1, #1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <16 x i8> @llvm.arm.mve.vshrn.v16i8.v8i16(<16 x i8> %a, <8 x i16> %b, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1)
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vqrshrntq_n_u32(<8 x i16> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vqrshrntq_n_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqrshrnt.u32 q0, q1, #11
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.arm.mve.vshrn.v8i16.v4i32(<8 x i16> %a, <4 x i32> %b, i32 11, i32 1, i32 1, i32 1, i32 1, i32 1)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vqrshrnbq_m_n_s16(<16 x i8> %a, <8 x i16> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vqrshrnbq_m_n_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqrshrnbt.s16 q0, q1, #2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <16 x i8> @llvm.arm.mve.vshrn.predicated.v16i8.v8i16.v8i1(<16 x i8> %a, <8 x i16> %b, i32 2, i32 1, i32 1, i32 0, i32 0, i32 0, <8 x i1> %1)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vqrshrnbq_m_n_s32(<8 x i16> %a, <4 x i32> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vqrshrnbq_m_n_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqrshrnbt.s32 q0, q1, #12
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <8 x i16> @llvm.arm.mve.vshrn.predicated.v8i16.v4i32.v4i1(<8 x i16> %a, <4 x i32> %b, i32 12, i32 1, i32 1, i32 0, i32 0, i32 0, <4 x i1> %1)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vqrshrnbq_m_n_u16(<16 x i8> %a, <8 x i16> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vqrshrnbq_m_n_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqrshrnbt.u16 q0, q1, #5
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <16 x i8> @llvm.arm.mve.vshrn.predicated.v16i8.v8i16.v8i1(<16 x i8> %a, <8 x i16> %b, i32 5, i32 1, i32 1, i32 1, i32 1, i32 0, <8 x i1> %1)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vqrshrnbq_m_n_u32(<8 x i16> %a, <4 x i32> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vqrshrnbq_m_n_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqrshrnbt.u32 q0, q1, #11
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <8 x i16> @llvm.arm.mve.vshrn.predicated.v8i16.v4i32.v4i1(<8 x i16> %a, <4 x i32> %b, i32 11, i32 1, i32 1, i32 1, i32 1, i32 0, <4 x i1> %1)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vqrshrntq_m_n_s16(<16 x i8> %a, <8 x i16> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vqrshrntq_m_n_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqrshrntt.s16 q0, q1, #4
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <16 x i8> @llvm.arm.mve.vshrn.predicated.v16i8.v8i16.v8i1(<16 x i8> %a, <8 x i16> %b, i32 4, i32 1, i32 1, i32 0, i32 0, i32 1, <8 x i1> %1)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vqrshrntq_m_n_s32(<8 x i16> %a, <4 x i32> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vqrshrntq_m_n_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqrshrntt.s32 q0, q1, #6
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <8 x i16> @llvm.arm.mve.vshrn.predicated.v8i16.v4i32.v4i1(<8 x i16> %a, <4 x i32> %b, i32 6, i32 1, i32 1, i32 0, i32 0, i32 1, <4 x i1> %1)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vqrshrntq_m_n_u16(<16 x i8> %a, <8 x i16> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vqrshrntq_m_n_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqrshrntt.u16 q0, q1, #7
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <16 x i8> @llvm.arm.mve.vshrn.predicated.v16i8.v8i16.v8i1(<16 x i8> %a, <8 x i16> %b, i32 7, i32 1, i32 1, i32 1, i32 1, i32 1, <8 x i1> %1)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vqrshrntq_m_n_u32(<8 x i16> %a, <4 x i32> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vqrshrntq_m_n_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqrshrntt.u32 q0, q1, #15
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <8 x i16> @llvm.arm.mve.vshrn.predicated.v8i16.v4i32.v4i1(<8 x i16> %a, <4 x i32> %b, i32 15, i32 1, i32 1, i32 1, i32 1, i32 1, <4 x i1> %1)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vqrshrunbq_n_s16(<16 x i8> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vqrshrunbq_n_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqshrunb.s16 q0, q1, #7
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <16 x i8> @llvm.arm.mve.vshrn.v16i8.v8i16(<16 x i8> %a, <8 x i16> %b, i32 7, i32 1, i32 0, i32 1, i32 0, i32 0)
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vqrshrunbq_n_s32(<8 x i16> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vqrshrunbq_n_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqshrunb.s32 q0, q1, #1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.arm.mve.vshrn.v8i16.v4i32(<8 x i16> %a, <4 x i32> %b, i32 1, i32 1, i32 0, i32 1, i32 0, i32 0)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vqrshruntq_n_s16(<16 x i8> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vqrshruntq_n_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqshrunt.s16 q0, q1, #1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <16 x i8> @llvm.arm.mve.vshrn.v16i8.v8i16(<16 x i8> %a, <8 x i16> %b, i32 1, i32 1, i32 0, i32 1, i32 0, i32 1)
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vqrshruntq_n_s32(<8 x i16> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vqrshruntq_n_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqshrunt.s32 q0, q1, #3
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.arm.mve.vshrn.v8i16.v4i32(<8 x i16> %a, <4 x i32> %b, i32 3, i32 1, i32 0, i32 1, i32 0, i32 1)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vqrshrunbq_m_n_s16(<16 x i8> %a, <8 x i16> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vqrshrunbq_m_n_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqshrunbt.s16 q0, q1, #4
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <16 x i8> @llvm.arm.mve.vshrn.predicated.v16i8.v8i16.v8i1(<16 x i8> %a, <8 x i16> %b, i32 4, i32 1, i32 0, i32 1, i32 0, i32 0, <8 x i1> %1)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vqrshrunbq_m_n_s32(<8 x i16> %a, <4 x i32> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vqrshrunbq_m_n_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqshrunbt.s32 q0, q1, #10
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <8 x i16> @llvm.arm.mve.vshrn.predicated.v8i16.v4i32.v4i1(<8 x i16> %a, <4 x i32> %b, i32 10, i32 1, i32 0, i32 1, i32 0, i32 0, <4 x i1> %1)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vqrshruntq_m_n_s16(<16 x i8> %a, <8 x i16> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vqrshruntq_m_n_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqshruntt.s16 q0, q1, #3
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <16 x i8> @llvm.arm.mve.vshrn.predicated.v16i8.v8i16.v8i1(<16 x i8> %a, <8 x i16> %b, i32 3, i32 1, i32 0, i32 1, i32 0, i32 1, <8 x i1> %1)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vqrshruntq_m_n_s32(<8 x i16> %a, <4 x i32> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vqrshruntq_m_n_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqshruntt.s32 q0, q1, #13
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <8 x i16> @llvm.arm.mve.vshrn.predicated.v8i16.v4i32.v4i1(<8 x i16> %a, <4 x i32> %b, i32 13, i32 1, i32 0, i32 1, i32 0, i32 1, <4 x i1> %1)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vsliq_n_s8(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vsliq_n_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vsli.8 q0, q1, #2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <16 x i8> @llvm.arm.mve.vsli.v16i8(<16 x i8> %a, <16 x i8> %b, i32 2)
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vsliq_n_s16(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vsliq_n_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vsli.16 q0, q1, #10
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.arm.mve.vsli.v8i16(<8 x i16> %a, <8 x i16> %b, i32 10)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vsliq_n_s32(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vsliq_n_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vsli.32 q0, q1, #1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x i32> @llvm.arm.mve.vsli.v4i32(<4 x i32> %a, <4 x i32> %b, i32 1)
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vsliq_n_u8(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vsliq_n_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vsli.8 q0, q1, #1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <16 x i8> @llvm.arm.mve.vsli.v16i8(<16 x i8> %a, <16 x i8> %b, i32 1)
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vsliq_n_u16(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vsliq_n_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vsli.16 q0, q1, #1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.arm.mve.vsli.v8i16(<8 x i16> %a, <8 x i16> %b, i32 1)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vsliq_n_u32(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vsliq_n_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vsli.32 q0, q1, #28
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x i32> @llvm.arm.mve.vsli.v4i32(<4 x i32> %a, <4 x i32> %b, i32 28)
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vsliq_m_n_s8(<16 x i8> %a, <16 x i8> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vsliq_m_n_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vslit.8 q0, q1, #4
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = call <16 x i8> @llvm.arm.mve.vsli.predicated.v16i8.v16i1(<16 x i8> %a, <16 x i8> %b, i32 4, <16 x i1> %1)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vsliq_m_n_s16(<8 x i16> %a, <8 x i16> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vsliq_m_n_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vslit.16 q0, q1, #1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <8 x i16> @llvm.arm.mve.vsli.predicated.v8i16.v8i1(<8 x i16> %a, <8 x i16> %b, i32 1, <8 x i1> %1)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vsliq_m_n_s32(<4 x i32> %a, <4 x i32> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vsliq_m_n_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vslit.32 q0, q1, #1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x i32> @llvm.arm.mve.vsli.predicated.v4i32.v4i1(<4 x i32> %a, <4 x i32> %b, i32 1, <4 x i1> %1)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vsliq_m_n_u8(<16 x i8> %a, <16 x i8> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vsliq_m_n_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vslit.8 q0, q1, #5
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = call <16 x i8> @llvm.arm.mve.vsli.predicated.v16i8.v16i1(<16 x i8> %a, <16 x i8> %b, i32 5, <16 x i1> %1)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vsliq_m_n_u16(<8 x i16> %a, <8 x i16> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vsliq_m_n_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vslit.16 q0, q1, #3
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <8 x i16> @llvm.arm.mve.vsli.predicated.v8i16.v8i1(<8 x i16> %a, <8 x i16> %b, i32 3, <8 x i1> %1)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vsliq_m_n_u32(<4 x i32> %a, <4 x i32> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vsliq_m_n_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vslit.32 q0, q1, #9
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x i32> @llvm.arm.mve.vsli.predicated.v4i32.v4i1(<4 x i32> %a, <4 x i32> %b, i32 9, <4 x i1> %1)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vsriq_n_s8(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vsriq_n_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vsri.8 q0, q1, #3
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <16 x i8> @llvm.arm.mve.vsri.v16i8(<16 x i8> %a, <16 x i8> %b, i32 3)
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vsriq_n_s16(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vsriq_n_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vsri.16 q0, q1, #2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.arm.mve.vsri.v8i16(<8 x i16> %a, <8 x i16> %b, i32 2)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vsriq_n_s32(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vsriq_n_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vsri.32 q0, q1, #28
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x i32> @llvm.arm.mve.vsri.v4i32(<4 x i32> %a, <4 x i32> %b, i32 28)
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vsriq_n_u8(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vsriq_n_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vsri.8 q0, q1, #3
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <16 x i8> @llvm.arm.mve.vsri.v16i8(<16 x i8> %a, <16 x i8> %b, i32 3)
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vsriq_n_u16(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vsriq_n_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vsri.16 q0, q1, #3
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.arm.mve.vsri.v8i16(<8 x i16> %a, <8 x i16> %b, i32 3)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vsriq_n_u32(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vsriq_n_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vsri.32 q0, q1, #26
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x i32> @llvm.arm.mve.vsri.v4i32(<4 x i32> %a, <4 x i32> %b, i32 26)
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vsriq_m_n_s8(<16 x i8> %a, <16 x i8> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vsriq_m_n_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vsrit.8 q0, q1, #4
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = call <16 x i8> @llvm.arm.mve.vsri.predicated.v16i8.v16i1(<16 x i8> %a, <16 x i8> %b, i32 4, <16 x i1> %1)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vsriq_m_n_s16(<8 x i16> %a, <8 x i16> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vsriq_m_n_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vsrit.16 q0, q1, #1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <8 x i16> @llvm.arm.mve.vsri.predicated.v8i16.v8i1(<8 x i16> %a, <8 x i16> %b, i32 1, <8 x i1> %1)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vsriq_m_n_s32(<4 x i32> %a, <4 x i32> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vsriq_m_n_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vsrit.32 q0, q1, #27
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x i32> @llvm.arm.mve.vsri.predicated.v4i32.v4i1(<4 x i32> %a, <4 x i32> %b, i32 27, <4 x i1> %1)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vsriq_m_n_u8(<16 x i8> %a, <16 x i8> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vsriq_m_n_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vsrit.8 q0, q1, #7
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = call <16 x i8> @llvm.arm.mve.vsri.predicated.v16i8.v16i1(<16 x i8> %a, <16 x i8> %b, i32 7, <16 x i1> %1)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vsriq_m_n_u16(<8 x i16> %a, <8 x i16> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vsriq_m_n_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vsrit.16 q0, q1, #9
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <8 x i16> @llvm.arm.mve.vsri.predicated.v8i16.v8i1(<8 x i16> %a, <8 x i16> %b, i32 9, <8 x i1> %1)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vsriq_m_n_u32(<4 x i32> %a, <4 x i32> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vsriq_m_n_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vsrit.32 q0, q1, #13
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x i32> @llvm.arm.mve.vsri.predicated.v4i32.v4i1(<4 x i32> %a, <4 x i32> %b, i32 13, <4 x i1> %1)
+  ret <4 x i32> %2
+}
+
+declare <16 x i8> @llvm.arm.mve.vshrn.v16i8.v8i16(<16 x i8>, <8 x i16>, i32, i32, i32, i32, i32, i32)
+declare <8 x i16> @llvm.arm.mve.vshrn.v8i16.v4i32(<8 x i16>, <4 x i32>, i32, i32, i32, i32, i32, i32)
+declare <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32)
+declare <16 x i8> @llvm.arm.mve.vshrn.predicated.v16i8.v8i16.v8i1(<16 x i8>, <8 x i16>, i32, i32, i32, i32, i32, i32, <8 x i1>)
+declare <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32)
+declare <8 x i16> @llvm.arm.mve.vshrn.predicated.v8i16.v4i32.v4i1(<8 x i16>, <4 x i32>, i32, i32, i32, i32, i32, i32, <4 x i1>)
+declare <16 x i8> @llvm.arm.mve.vsli.v16i8(<16 x i8>, <16 x i8>, i32)
+declare <8 x i16> @llvm.arm.mve.vsli.v8i16(<8 x i16>, <8 x i16>, i32)
+declare <4 x i32> @llvm.arm.mve.vsli.v4i32(<4 x i32>, <4 x i32>, i32)
+declare <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32)
+declare <16 x i8> @llvm.arm.mve.vsli.predicated.v16i8.v16i1(<16 x i8>, <16 x i8>, i32, <16 x i1>)
+declare <8 x i16> @llvm.arm.mve.vsli.predicated.v8i16.v8i1(<8 x i16>, <8 x i16>, i32, <8 x i1>)
+declare <4 x i32> @llvm.arm.mve.vsli.predicated.v4i32.v4i1(<4 x i32>, <4 x i32>, i32, <4 x i1>)
+declare <16 x i8> @llvm.arm.mve.vsri.v16i8(<16 x i8>, <16 x i8>, i32)
+declare <8 x i16> @llvm.arm.mve.vsri.v8i16(<8 x i16>, <8 x i16>, i32)
+declare <4 x i32> @llvm.arm.mve.vsri.v4i32(<4 x i32>, <4 x i32>, i32)
+declare <16 x i8> @llvm.arm.mve.vsri.predicated.v16i8.v16i1(<16 x i8>, <16 x i8>, i32, <16 x i1>)
+declare <8 x i16> @llvm.arm.mve.vsri.predicated.v8i16.v8i1(<8 x i16>, <8 x i16>, i32, <8 x i1>)
+declare <4 x i32> @llvm.arm.mve.vsri.predicated.v4i32.v4i1(<4 x i32>, <4 x i32>, i32, <4 x i1>)
diff --git a/llvm/test/CodeGen/Thumb2/mve-intrinsics/vector-shift-imm.ll b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vector-shift-imm.ll
new file mode 100644
index 000000000..dd4980f99
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vector-shift-imm.ll
@@ -0,0 +1,1476 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve -verify-machineinstrs -o - %s | FileCheck %s
+
+define arm_aapcs_vfpcc <16 x i8> @test_vshlq_n_s8(<16 x i8> %a) {
+; CHECK-LABEL: test_vshlq_n_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vshl.i8 q0, q0, #5
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = shl <16 x i8> %a, <i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5>
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vshlq_n_s16(<8 x i16> %a) {
+; CHECK-LABEL: test_vshlq_n_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vshl.i16 q0, q0, #5
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = shl <8 x i16> %a, <i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5>
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vshlq_n_s32(<4 x i32> %a) {
+; CHECK-LABEL: test_vshlq_n_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vshl.i32 q0, q0, #18
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = shl <4 x i32> %a, <i32 18, i32 18, i32 18, i32 18>
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vshrq_n_s8(<16 x i8> %a) {
+; CHECK-LABEL: test_vshrq_n_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vshr.s8 q0, q0, #4
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = ashr <16 x i8> %a, <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vshrq_n_s16(<8 x i16> %a) {
+; CHECK-LABEL: test_vshrq_n_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vshr.s16 q0, q0, #10
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = ashr <8 x i16> %a, <i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10>
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vshrq_n_s32(<4 x i32> %a) {
+; CHECK-LABEL: test_vshrq_n_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vshr.s32 q0, q0, #19
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = ashr <4 x i32> %a, <i32 19, i32 19, i32 19, i32 19>
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vshrq_n_u8(<16 x i8> %a) {
+; CHECK-LABEL: test_vshrq_n_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vshr.u8 q0, q0, #1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = lshr <16 x i8> %a, <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vshrq_n_u16(<8 x i16> %a) {
+; CHECK-LABEL: test_vshrq_n_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vshr.u16 q0, q0, #10
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = lshr <8 x i16> %a, <i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10>
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vshrq_n_u32(<4 x i32> %a) {
+; CHECK-LABEL: test_vshrq_n_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vshr.u32 q0, q0, #10
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = lshr <4 x i32> %a, <i32 10, i32 10, i32 10, i32 10>
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vshlq_m_n_s8(<16 x i8> %inactive, <16 x i8> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vshlq_m_n_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlt.i8 q0, q1, #6
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <16 x i8> @llvm.arm.mve.shl.imm.predicated.v16i8.v16i1(<16 x i8> %a, i32 6, <16 x i1> %1, <16 x i8> %inactive)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vshlq_m_n_s16(<8 x i16> %inactive, <8 x i16> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vshlq_m_n_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlt.i16 q0, q1, #13
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.shl.imm.predicated.v8i16.v8i1(<8 x i16> %a, i32 13, <8 x i1> %1, <8 x i16> %inactive)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vshlq_m_n_s32(<4 x i32> %inactive, <4 x i32> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vshlq_m_n_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlt.i32 q0, q1, #0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call <4 x i32> @llvm.arm.mve.shl.imm.predicated.v4i32.v4i1(<4 x i32> %a, i32 0, <4 x i1> %1, <4 x i32> %inactive)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vshrq_m_n_s8(<16 x i8> %inactive, <16 x i8> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vshrq_m_n_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshrt.s8 q0, q1, #2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <16 x i8> @llvm.arm.mve.shr.imm.predicated.v16i8.v16i1(<16 x i8> %a, i32 2, i32 0, <16 x i1> %1, <16 x i8> %inactive)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vshrq_m_n_s16(<8 x i16> %inactive, <8 x i16> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vshrq_m_n_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshrt.s16 q0, q1, #3
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.shr.imm.predicated.v8i16.v8i1(<8 x i16> %a, i32 3, i32 0, <8 x i1> %1, <8 x i16> %inactive)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vshrq_m_n_s32(<4 x i32> %inactive, <4 x i32> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vshrq_m_n_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshrt.s32 q0, q1, #13
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call <4 x i32> @llvm.arm.mve.shr.imm.predicated.v4i32.v4i1(<4 x i32> %a, i32 13, i32 0, <4 x i1> %1, <4 x i32> %inactive)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vshrq_m_n_u8(<16 x i8> %inactive, <16 x i8> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vshrq_m_n_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshrt.u8 q0, q1, #4
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <16 x i8> @llvm.arm.mve.shr.imm.predicated.v16i8.v16i1(<16 x i8> %a, i32 4, i32 1, <16 x i1> %1, <16 x i8> %inactive)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vshrq_m_n_u16(<8 x i16> %inactive, <8 x i16> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vshrq_m_n_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshrt.u16 q0, q1, #14
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.shr.imm.predicated.v8i16.v8i1(<8 x i16> %a, i32 14, i32 1, <8 x i1> %1, <8 x i16> %inactive)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vshrq_m_n_u32(<4 x i32> %inactive, <4 x i32> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vshrq_m_n_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshrt.u32 q0, q1, #21
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call <4 x i32> @llvm.arm.mve.shr.imm.predicated.v4i32.v4i1(<4 x i32> %a, i32 21, i32 1, <4 x i1> %1, <4 x i32> %inactive)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vshlq_x_n_s8(<16 x i8> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vshlq_x_n_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlt.i8 q0, q0, #1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <16 x i8> @llvm.arm.mve.shl.imm.predicated.v16i8.v16i1(<16 x i8> %a, i32 1, <16 x i1> %1, <16 x i8> undef)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vshlq_x_n_s16(<8 x i16> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vshlq_x_n_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlt.i16 q0, q0, #15
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.shl.imm.predicated.v8i16.v8i1(<8 x i16> %a, i32 15, <8 x i1> %1, <8 x i16> undef)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vshlq_x_n_s32(<4 x i32> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vshlq_x_n_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlt.i32 q0, q0, #13
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call <4 x i32> @llvm.arm.mve.shl.imm.predicated.v4i32.v4i1(<4 x i32> %a, i32 13, <4 x i1> %1, <4 x i32> undef)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vshlq_x_n_u8(<16 x i8> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vshlq_x_n_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlt.i8 q0, q0, #4
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <16 x i8> @llvm.arm.mve.shl.imm.predicated.v16i8.v16i1(<16 x i8> %a, i32 4, <16 x i1> %1, <16 x i8> undef)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vshlq_x_n_u16(<8 x i16> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vshlq_x_n_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlt.i16 q0, q0, #10
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.shl.imm.predicated.v8i16.v8i1(<8 x i16> %a, i32 10, <8 x i1> %1, <8 x i16> undef)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vshlq_x_n_u32(<4 x i32> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vshlq_x_n_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlt.i32 q0, q0, #30
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call <4 x i32> @llvm.arm.mve.shl.imm.predicated.v4i32.v4i1(<4 x i32> %a, i32 30, <4 x i1> %1, <4 x i32> undef)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vshrq_x_n_s8(<16 x i8> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vshrq_x_n_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshrt.s8 q0, q0, #4
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <16 x i8> @llvm.arm.mve.shr.imm.predicated.v16i8.v16i1(<16 x i8> %a, i32 4, i32 0, <16 x i1> %1, <16 x i8> undef)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vshrq_x_n_s16(<8 x i16> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vshrq_x_n_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshrt.s16 q0, q0, #10
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.shr.imm.predicated.v8i16.v8i1(<8 x i16> %a, i32 10, i32 0, <8 x i1> %1, <8 x i16> undef)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vshrq_x_n_s32(<4 x i32> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vshrq_x_n_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshrt.s32 q0, q0, #7
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call <4 x i32> @llvm.arm.mve.shr.imm.predicated.v4i32.v4i1(<4 x i32> %a, i32 7, i32 0, <4 x i1> %1, <4 x i32> undef)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vshrq_x_n_u8(<16 x i8> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vshrq_x_n_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshrt.u8 q0, q0, #7
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <16 x i8> @llvm.arm.mve.shr.imm.predicated.v16i8.v16i1(<16 x i8> %a, i32 7, i32 1, <16 x i1> %1, <16 x i8> undef)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vshrq_x_n_u16(<8 x i16> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vshrq_x_n_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshrt.u16 q0, q0, #7
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.shr.imm.predicated.v8i16.v8i1(<8 x i16> %a, i32 7, i32 1, <8 x i1> %1, <8 x i16> undef)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vshrq_x_n_u32(<4 x i32> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vshrq_x_n_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshrt.u32 q0, q0, #6
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call <4 x i32> @llvm.arm.mve.shr.imm.predicated.v4i32.v4i1(<4 x i32> %a, i32 6, i32 1, <4 x i1> %1, <4 x i32> undef)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vqshlq_n_s8(<16 x i8> %a) {
+; CHECK-LABEL: test_vqshlq_n_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqshl.s8 q0, q0, #3
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <16 x i8> @llvm.arm.mve.vqshl.imm.v16i8(<16 x i8> %a, i32 3, i32 0)
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vqshlq_n_s16(<8 x i16> %a) {
+; CHECK-LABEL: test_vqshlq_n_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqshl.s16 q0, q0, #4
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <8 x i16> @llvm.arm.mve.vqshl.imm.v8i16(<8 x i16> %a, i32 4, i32 0)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vqshlq_n_s32(<4 x i32> %a) {
+; CHECK-LABEL: test_vqshlq_n_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqshl.s32 q0, q0, #4
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <4 x i32> @llvm.arm.mve.vqshl.imm.v4i32(<4 x i32> %a, i32 4, i32 0)
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vqshlq_n_u8(<16 x i8> %a) {
+; CHECK-LABEL: test_vqshlq_n_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqshl.u8 q0, q0, #0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <16 x i8> @llvm.arm.mve.vqshl.imm.v16i8(<16 x i8> %a, i32 0, i32 1)
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vqshlq_n_u16(<8 x i16> %a) {
+; CHECK-LABEL: test_vqshlq_n_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqshl.u16 q0, q0, #13
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <8 x i16> @llvm.arm.mve.vqshl.imm.v8i16(<8 x i16> %a, i32 13, i32 1)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vqshlq_n_u32(<4 x i32> %a) {
+; CHECK-LABEL: test_vqshlq_n_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqshl.u32 q0, q0, #6
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <4 x i32> @llvm.arm.mve.vqshl.imm.v4i32(<4 x i32> %a, i32 6, i32 1)
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vqshluq_n_s8(<16 x i8> %a) {
+; CHECK-LABEL: test_vqshluq_n_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqshlu.s8 q0, q0, #5
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <16 x i8> @llvm.arm.mve.vqshlu.imm.v16i8(<16 x i8> %a, i32 5)
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vqshluq_n_s16(<8 x i16> %a) {
+; CHECK-LABEL: test_vqshluq_n_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqshlu.s16 q0, q0, #5
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <8 x i16> @llvm.arm.mve.vqshlu.imm.v8i16(<8 x i16> %a, i32 5)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vqshluq_n_s32(<4 x i32> %a) {
+; CHECK-LABEL: test_vqshluq_n_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqshlu.s32 q0, q0, #4
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <4 x i32> @llvm.arm.mve.vqshlu.imm.v4i32(<4 x i32> %a, i32 4)
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vrshrq_n_s8(<16 x i8> %a) {
+; CHECK-LABEL: test_vrshrq_n_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vrshr.s8 q0, q0, #4
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <16 x i8> @llvm.arm.mve.vrshr.imm.v16i8(<16 x i8> %a, i32 4, i32 0)
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vrshrq_n_s16(<8 x i16> %a) {
+; CHECK-LABEL: test_vrshrq_n_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vrshr.s16 q0, q0, #12
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <8 x i16> @llvm.arm.mve.vrshr.imm.v8i16(<8 x i16> %a, i32 12, i32 0)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vrshrq_n_s32(<4 x i32> %a) {
+; CHECK-LABEL: test_vrshrq_n_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vrshr.s32 q0, q0, #30
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <4 x i32> @llvm.arm.mve.vrshr.imm.v4i32(<4 x i32> %a, i32 30, i32 0)
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vrshrq_n_u8(<16 x i8> %a) {
+; CHECK-LABEL: test_vrshrq_n_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vrshr.u8 q0, q0, #1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <16 x i8> @llvm.arm.mve.vrshr.imm.v16i8(<16 x i8> %a, i32 1, i32 1)
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vrshrq_n_u16(<8 x i16> %a) {
+; CHECK-LABEL: test_vrshrq_n_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vrshr.u16 q0, q0, #15
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <8 x i16> @llvm.arm.mve.vrshr.imm.v8i16(<8 x i16> %a, i32 15, i32 1)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vrshrq_n_u32(<4 x i32> %a) {
+; CHECK-LABEL: test_vrshrq_n_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vrshr.u32 q0, q0, #20
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <4 x i32> @llvm.arm.mve.vrshr.imm.v4i32(<4 x i32> %a, i32 20, i32 1)
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vqshlq_m_n_s8(<16 x i8> %inactive, <16 x i8> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vqshlq_m_n_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqshlt.s8 q0, q1, #6
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <16 x i8> @llvm.arm.mve.vqshl.imm.predicated.v16i8.v16i1(<16 x i8> %a, i32 6, i32 0, <16 x i1> %1, <16 x i8> %inactive)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vqshlq_m_n_s16(<8 x i16> %inactive, <8 x i16> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vqshlq_m_n_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqshlt.s16 q0, q1, #13
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.vqshl.imm.predicated.v8i16.v8i1(<8 x i16> %a, i32 13, i32 0, <8 x i1> %1, <8 x i16> %inactive)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vqshlq_m_n_s32(<4 x i32> %inactive, <4 x i32> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vqshlq_m_n_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqshlt.s32 q0, q1, #14
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call <4 x i32> @llvm.arm.mve.vqshl.imm.predicated.v4i32.v4i1(<4 x i32> %a, i32 14, i32 0, <4 x i1> %1, <4 x i32> %inactive)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vqshlq_m_n_u8(<16 x i8> %inactive, <16 x i8> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vqshlq_m_n_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqshlt.u8 q0, q1, #4
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <16 x i8> @llvm.arm.mve.vqshl.imm.predicated.v16i8.v16i1(<16 x i8> %a, i32 4, i32 1, <16 x i1> %1, <16 x i8> %inactive)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vqshlq_m_n_u16(<8 x i16> %inactive, <8 x i16> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vqshlq_m_n_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqshlt.u16 q0, q1, #9
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.vqshl.imm.predicated.v8i16.v8i1(<8 x i16> %a, i32 9, i32 1, <8 x i1> %1, <8 x i16> %inactive)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vqshlq_m_n_u32(<4 x i32> %inactive, <4 x i32> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vqshlq_m_n_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqshlt.u32 q0, q1, #25
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call <4 x i32> @llvm.arm.mve.vqshl.imm.predicated.v4i32.v4i1(<4 x i32> %a, i32 25, i32 1, <4 x i1> %1, <4 x i32> %inactive)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vqshluq_m_n_s8(<16 x i8> %inactive, <16 x i8> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vqshluq_m_n_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqshlut.s8 q0, q1, #2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <16 x i8> @llvm.arm.mve.vqshlu.imm.predicated.v16i8.v16i1(<16 x i8> %a, i32 2, <16 x i1> %1, <16 x i8> %inactive)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vqshluq_m_n_s16(<8 x i16> %inactive, <8 x i16> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vqshluq_m_n_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqshlut.s16 q0, q1, #12
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.vqshlu.imm.predicated.v8i16.v8i1(<8 x i16> %a, i32 12, <8 x i1> %1, <8 x i16> %inactive)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vqshluq_m_n_s32(<4 x i32> %inactive, <4 x i32> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vqshluq_m_n_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqshlut.s32 q0, q1, #24
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call <4 x i32> @llvm.arm.mve.vqshlu.imm.predicated.v4i32.v4i1(<4 x i32> %a, i32 24, <4 x i1> %1, <4 x i32> %inactive)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vrshrq_m_n_s8(<16 x i8> %inactive, <16 x i8> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vrshrq_m_n_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vrshrt.s8 q0, q1, #2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <16 x i8> @llvm.arm.mve.vrshr.imm.predicated.v16i8.v16i1(<16 x i8> %a, i32 2, i32 0, <16 x i1> %1, <16 x i8> %inactive)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vrshrq_m_n_s16(<8 x i16> %inactive, <8 x i16> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vrshrq_m_n_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vrshrt.s16 q0, q1, #11
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.vrshr.imm.predicated.v8i16.v8i1(<8 x i16> %a, i32 11, i32 0, <8 x i1> %1, <8 x i16> %inactive)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vrshrq_m_n_s32(<4 x i32> %inactive, <4 x i32> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vrshrq_m_n_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vrshrt.s32 q0, q1, #24
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call <4 x i32> @llvm.arm.mve.vrshr.imm.predicated.v4i32.v4i1(<4 x i32> %a, i32 24, i32 0, <4 x i1> %1, <4 x i32> %inactive)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vrshrq_m_n_u8(<16 x i8> %inactive, <16 x i8> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vrshrq_m_n_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vrshrt.u8 q0, q1, #7
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <16 x i8> @llvm.arm.mve.vrshr.imm.predicated.v16i8.v16i1(<16 x i8> %a, i32 7, i32 1, <16 x i1> %1, <16 x i8> %inactive)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vrshrq_m_n_u16(<8 x i16> %inactive, <8 x i16> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vrshrq_m_n_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vrshrt.u16 q0, q1, #4
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.vrshr.imm.predicated.v8i16.v8i1(<8 x i16> %a, i32 4, i32 1, <8 x i1> %1, <8 x i16> %inactive)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vrshrq_m_n_u32(<4 x i32> %inactive, <4 x i32> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vrshrq_m_n_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vrshrt.u32 q0, q1, #27
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call <4 x i32> @llvm.arm.mve.vrshr.imm.predicated.v4i32.v4i1(<4 x i32> %a, i32 27, i32 1, <4 x i1> %1, <4 x i32> %inactive)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vrshrq_x_n_s8(<16 x i8> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vrshrq_x_n_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vrshrt.s8 q0, q0, #3
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <16 x i8> @llvm.arm.mve.vrshr.imm.predicated.v16i8.v16i1(<16 x i8> %a, i32 3, i32 0, <16 x i1> %1, <16 x i8> undef)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vrshrq_x_n_s16(<8 x i16> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vrshrq_x_n_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vrshrt.s16 q0, q0, #12
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.vrshr.imm.predicated.v8i16.v8i1(<8 x i16> %a, i32 12, i32 0, <8 x i1> %1, <8 x i16> undef)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vrshrq_x_n_s32(<4 x i32> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vrshrq_x_n_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vrshrt.s32 q0, q0, #20
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call <4 x i32> @llvm.arm.mve.vrshr.imm.predicated.v4i32.v4i1(<4 x i32> %a, i32 20, i32 0, <4 x i1> %1, <4 x i32> undef)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vrshrq_x_n_u8(<16 x i8> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vrshrq_x_n_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vrshrt.u8 q0, q0, #1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <16 x i8> @llvm.arm.mve.vrshr.imm.predicated.v16i8.v16i1(<16 x i8> %a, i32 1, i32 1, <16 x i1> %1, <16 x i8> undef)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vrshrq_x_n_u16(<8 x i16> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vrshrq_x_n_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vrshrt.u16 q0, q0, #13
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.vrshr.imm.predicated.v8i16.v8i1(<8 x i16> %a, i32 13, i32 1, <8 x i1> %1, <8 x i16> undef)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vrshrq_x_n_u32(<4 x i32> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vrshrq_x_n_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vrshrt.u32 q0, q0, #6
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call <4 x i32> @llvm.arm.mve.vrshr.imm.predicated.v4i32.v4i1(<4 x i32> %a, i32 6, i32 1, <4 x i1> %1, <4 x i32> undef)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vshllbq_n_s8(<16 x i8> %a) {
+; CHECK-LABEL: test_vshllbq_n_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vshllb.s8 q0, q0, #2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <8 x i16> @llvm.arm.mve.vshll.imm.v8i16.v16i8(<16 x i8> %a, i32 2, i32 0, i32 0)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vshllbq_n_s8_lanewidth(<16 x i8> %a) {
+; CHECK-LABEL: test_vshllbq_n_s8_lanewidth:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vshllb.s8 q0, q0, #8
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <8 x i16> @llvm.arm.mve.vshll.imm.v8i16.v16i8(<16 x i8> %a, i32 8, i32 0, i32 0)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vshllbq_n_s16(<8 x i16> %a) {
+; CHECK-LABEL: test_vshllbq_n_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vshllb.s16 q0, q0, #13
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <4 x i32> @llvm.arm.mve.vshll.imm.v4i32.v8i16(<8 x i16> %a, i32 13, i32 0, i32 0)
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vshllbq_n_s16_lanewidth(<8 x i16> %a) {
+; CHECK-LABEL: test_vshllbq_n_s16_lanewidth:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vshllb.s16 q0, q0, #16
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <4 x i32> @llvm.arm.mve.vshll.imm.v4i32.v8i16(<8 x i16> %a, i32 16, i32 0, i32 0)
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vshllbq_n_u8(<16 x i8> %a) {
+; CHECK-LABEL: test_vshllbq_n_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vshllb.u8 q0, q0, #5
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <8 x i16> @llvm.arm.mve.vshll.imm.v8i16.v16i8(<16 x i8> %a, i32 5, i32 1, i32 0)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vshllbq_n_u8_lanewidth(<16 x i8> %a) {
+; CHECK-LABEL: test_vshllbq_n_u8_lanewidth:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vshllb.u8 q0, q0, #8
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <8 x i16> @llvm.arm.mve.vshll.imm.v8i16.v16i8(<16 x i8> %a, i32 8, i32 1, i32 0)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vshllbq_n_u16(<8 x i16> %a) {
+; CHECK-LABEL: test_vshllbq_n_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vshllb.u16 q0, q0, #6
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <4 x i32> @llvm.arm.mve.vshll.imm.v4i32.v8i16(<8 x i16> %a, i32 6, i32 1, i32 0)
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vshllbq_n_u16_lanewidth(<8 x i16> %a) {
+; CHECK-LABEL: test_vshllbq_n_u16_lanewidth:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vshllb.u16 q0, q0, #16
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <4 x i32> @llvm.arm.mve.vshll.imm.v4i32.v8i16(<8 x i16> %a, i32 16, i32 1, i32 0)
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vshlltq_n_s8(<16 x i8> %a) {
+; CHECK-LABEL: test_vshlltq_n_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vshllt.s8 q0, q0, #7
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <8 x i16> @llvm.arm.mve.vshll.imm.v8i16.v16i8(<16 x i8> %a, i32 7, i32 0, i32 1)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vshlltq_n_s8_lanewidth(<16 x i8> %a) {
+; CHECK-LABEL: test_vshlltq_n_s8_lanewidth:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vshllt.s8 q0, q0, #8
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <8 x i16> @llvm.arm.mve.vshll.imm.v8i16.v16i8(<16 x i8> %a, i32 8, i32 0, i32 1)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vshlltq_n_s16(<8 x i16> %a) {
+; CHECK-LABEL: test_vshlltq_n_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vshllt.s16 q0, q0, #2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <4 x i32> @llvm.arm.mve.vshll.imm.v4i32.v8i16(<8 x i16> %a, i32 2, i32 0, i32 1)
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vshlltq_n_s16_lanewidth(<8 x i16> %a) {
+; CHECK-LABEL: test_vshlltq_n_s16_lanewidth:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vshllt.s16 q0, q0, #16
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <4 x i32> @llvm.arm.mve.vshll.imm.v4i32.v8i16(<8 x i16> %a, i32 16, i32 0, i32 1)
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vshlltq_n_u8(<16 x i8> %a) {
+; CHECK-LABEL: test_vshlltq_n_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vshllt.u8 q0, q0, #7
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <8 x i16> @llvm.arm.mve.vshll.imm.v8i16.v16i8(<16 x i8> %a, i32 7, i32 1, i32 1)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vshlltq_n_u8_lanewidth(<16 x i8> %a) {
+; CHECK-LABEL: test_vshlltq_n_u8_lanewidth:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vshllt.u8 q0, q0, #8
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <8 x i16> @llvm.arm.mve.vshll.imm.v8i16.v16i8(<16 x i8> %a, i32 8, i32 1, i32 1)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vshlltq_n_u16(<8 x i16> %a) {
+; CHECK-LABEL: test_vshlltq_n_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vshllt.u16 q0, q0, #14
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <4 x i32> @llvm.arm.mve.vshll.imm.v4i32.v8i16(<8 x i16> %a, i32 14, i32 1, i32 1)
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vshlltq_n_u16_lanewidth(<8 x i16> %a) {
+; CHECK-LABEL: test_vshlltq_n_u16_lanewidth:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vshllt.u16 q0, q0, #16
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <4 x i32> @llvm.arm.mve.vshll.imm.v4i32.v8i16(<8 x i16> %a, i32 16, i32 1, i32 1)
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vshllbq_m_n_s8(<8 x i16> %inactive, <16 x i8> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vshllbq_m_n_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshllbt.s8 q0, q1, #6
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.vshll.imm.predicated.v8i16.v16i8.v16i1(<16 x i8> %a, i32 6, i32 0, i32 0, <16 x i1> %1, <8 x i16> %inactive)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vshllbq_m_n_s8_lanewidth(<8 x i16> %inactive, <16 x i8> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vshllbq_m_n_s8_lanewidth:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshllbt.s8 q0, q1, #8
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.vshll.imm.predicated.v8i16.v16i8.v16i1(<16 x i8> %a, i32 8, i32 0, i32 0, <16 x i1> %1, <8 x i16> %inactive)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vshllbq_m_n_s16(<4 x i32> %inactive, <8 x i16> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vshllbq_m_n_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshllbt.s16 q0, q1, #10
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <4 x i32> @llvm.arm.mve.vshll.imm.predicated.v4i32.v8i16.v8i1(<8 x i16> %a, i32 10, i32 0, i32 0, <8 x i1> %1, <4 x i32> %inactive)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vshllbq_m_n_s16_lanewidth(<4 x i32> %inactive, <8 x i16> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vshllbq_m_n_s16_lanewidth:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshllbt.s16 q0, q1, #16
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <4 x i32> @llvm.arm.mve.vshll.imm.predicated.v4i32.v8i16.v8i1(<8 x i16> %a, i32 16, i32 0, i32 0, <8 x i1> %1, <4 x i32> %inactive)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vshllbq_m_n_u8(<8 x i16> %inactive, <16 x i8> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vshllbq_m_n_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshllbt.u8 q0, q1, #3
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.vshll.imm.predicated.v8i16.v16i8.v16i1(<16 x i8> %a, i32 3, i32 1, i32 0, <16 x i1> %1, <8 x i16> %inactive)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vshllbq_m_n_u8_lanewidth(<8 x i16> %inactive, <16 x i8> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vshllbq_m_n_u8_lanewidth:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshllbt.u8 q0, q1, #8
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.vshll.imm.predicated.v8i16.v16i8.v16i1(<16 x i8> %a, i32 8, i32 1, i32 0, <16 x i1> %1, <8 x i16> %inactive)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vshllbq_m_n_u16(<4 x i32> %inactive, <8 x i16> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vshllbq_m_n_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshllbt.u16 q0, q1, #14
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <4 x i32> @llvm.arm.mve.vshll.imm.predicated.v4i32.v8i16.v8i1(<8 x i16> %a, i32 14, i32 1, i32 0, <8 x i1> %1, <4 x i32> %inactive)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vshllbq_m_n_u16_lanewidth(<4 x i32> %inactive, <8 x i16> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vshllbq_m_n_u16_lanewidth:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshllbt.u16 q0, q1, #16
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <4 x i32> @llvm.arm.mve.vshll.imm.predicated.v4i32.v8i16.v8i1(<8 x i16> %a, i32 16, i32 1, i32 0, <8 x i1> %1, <4 x i32> %inactive)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vshlltq_m_n_s8(<8 x i16> %inactive, <16 x i8> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vshlltq_m_n_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlltt.s8 q0, q1, #4
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.vshll.imm.predicated.v8i16.v16i8.v16i1(<16 x i8> %a, i32 4, i32 0, i32 1, <16 x i1> %1, <8 x i16> %inactive)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vshlltq_m_n_s8_lanewidth(<8 x i16> %inactive, <16 x i8> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vshlltq_m_n_s8_lanewidth:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlltt.s8 q0, q1, #8
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.vshll.imm.predicated.v8i16.v16i8.v16i1(<16 x i8> %a, i32 8, i32 0, i32 1, <16 x i1> %1, <8 x i16> %inactive)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vshlltq_m_n_s16(<4 x i32> %inactive, <8 x i16> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vshlltq_m_n_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlltt.s16 q0, q1, #12
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <4 x i32> @llvm.arm.mve.vshll.imm.predicated.v4i32.v8i16.v8i1(<8 x i16> %a, i32 12, i32 0, i32 1, <8 x i1> %1, <4 x i32> %inactive)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vshlltq_m_n_s16_lanewidth(<4 x i32> %inactive, <8 x i16> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vshlltq_m_n_s16_lanewidth:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlltt.s16 q0, q1, #16
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <4 x i32> @llvm.arm.mve.vshll.imm.predicated.v4i32.v8i16.v8i1(<8 x i16> %a, i32 16, i32 0, i32 1, <8 x i1> %1, <4 x i32> %inactive)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vshlltq_m_n_u8(<8 x i16> %inactive, <16 x i8> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vshlltq_m_n_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlltt.u8 q0, q1, #2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.vshll.imm.predicated.v8i16.v16i8.v16i1(<16 x i8> %a, i32 2, i32 1, i32 1, <16 x i1> %1, <8 x i16> %inactive)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vshlltq_m_n_u8_lanewidth(<8 x i16> %inactive, <16 x i8> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vshlltq_m_n_u8_lanewidth:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlltt.u8 q0, q1, #8
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.vshll.imm.predicated.v8i16.v16i8.v16i1(<16 x i8> %a, i32 8, i32 1, i32 1, <16 x i1> %1, <8 x i16> %inactive)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vshlltq_m_n_u16(<4 x i32> %inactive, <8 x i16> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vshlltq_m_n_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlltt.u16 q0, q1, #9
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <4 x i32> @llvm.arm.mve.vshll.imm.predicated.v4i32.v8i16.v8i1(<8 x i16> %a, i32 9, i32 1, i32 1, <8 x i1> %1, <4 x i32> %inactive)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vshlltq_m_n_u16_lanewidth(<4 x i32> %inactive, <8 x i16> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vshlltq_m_n_u16_lanewidth:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlltt.u16 q0, q1, #16
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <4 x i32> @llvm.arm.mve.vshll.imm.predicated.v4i32.v8i16.v8i1(<8 x i16> %a, i32 16, i32 1, i32 1, <8 x i1> %1, <4 x i32> %inactive)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vshllbq_x_n_s8(<16 x i8> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vshllbq_x_n_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshllbt.s8 q0, q0, #1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.vshll.imm.predicated.v8i16.v16i8.v16i1(<16 x i8> %a, i32 1, i32 0, i32 0, <16 x i1> %1, <8 x i16> undef)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vshllbq_x_n_s8_lanewidth(<16 x i8> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vshllbq_x_n_s8_lanewidth:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshllbt.s8 q0, q0, #8
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.vshll.imm.predicated.v8i16.v16i8.v16i1(<16 x i8> %a, i32 8, i32 0, i32 0, <16 x i1> %1, <8 x i16> undef)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vshllbq_x_n_s16(<8 x i16> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vshllbq_x_n_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshllbt.s16 q0, q0, #10
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <4 x i32> @llvm.arm.mve.vshll.imm.predicated.v4i32.v8i16.v8i1(<8 x i16> %a, i32 10, i32 0, i32 0, <8 x i1> %1, <4 x i32> undef)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vshllbq_x_n_s16_lanewidth(<8 x i16> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vshllbq_x_n_s16_lanewidth:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshllbt.s16 q0, q0, #16
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <4 x i32> @llvm.arm.mve.vshll.imm.predicated.v4i32.v8i16.v8i1(<8 x i16> %a, i32 16, i32 0, i32 0, <8 x i1> %1, <4 x i32> undef)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vshllbq_x_n_u8(<16 x i8> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vshllbq_x_n_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshllbt.u8 q0, q0, #6
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.vshll.imm.predicated.v8i16.v16i8.v16i1(<16 x i8> %a, i32 6, i32 1, i32 0, <16 x i1> %1, <8 x i16> undef)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vshllbq_x_n_u8_lanewidth(<16 x i8> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vshllbq_x_n_u8_lanewidth:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshllbt.u8 q0, q0, #8
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.vshll.imm.predicated.v8i16.v16i8.v16i1(<16 x i8> %a, i32 8, i32 1, i32 0, <16 x i1> %1, <8 x i16> undef)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vshllbq_x_n_u16(<8 x i16> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vshllbq_x_n_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshllbt.u16 q0, q0, #10
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <4 x i32> @llvm.arm.mve.vshll.imm.predicated.v4i32.v8i16.v8i1(<8 x i16> %a, i32 10, i32 1, i32 0, <8 x i1> %1, <4 x i32> undef)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vshllbq_x_n_u16_lanewidth(<8 x i16> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vshllbq_x_n_u16_lanewidth:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshllbt.u16 q0, q0, #16
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <4 x i32> @llvm.arm.mve.vshll.imm.predicated.v4i32.v8i16.v8i1(<8 x i16> %a, i32 16, i32 1, i32 0, <8 x i1> %1, <4 x i32> undef)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vshlltq_x_n_s8(<16 x i8> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vshlltq_x_n_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlltt.s8 q0, q0, #2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.vshll.imm.predicated.v8i16.v16i8.v16i1(<16 x i8> %a, i32 2, i32 0, i32 1, <16 x i1> %1, <8 x i16> undef)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vshlltq_x_n_s8_lanewidth(<16 x i8> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vshlltq_x_n_s8_lanewidth:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlltt.s8 q0, q0, #8
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.vshll.imm.predicated.v8i16.v16i8.v16i1(<16 x i8> %a, i32 8, i32 0, i32 1, <16 x i1> %1, <8 x i16> undef)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vshlltq_x_n_s16(<8 x i16> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vshlltq_x_n_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlltt.s16 q0, q0, #6
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <4 x i32> @llvm.arm.mve.vshll.imm.predicated.v4i32.v8i16.v8i1(<8 x i16> %a, i32 6, i32 0, i32 1, <8 x i1> %1, <4 x i32> undef)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vshlltq_x_n_s16_lanewidth(<8 x i16> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vshlltq_x_n_s16_lanewidth:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlltt.s16 q0, q0, #16
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <4 x i32> @llvm.arm.mve.vshll.imm.predicated.v4i32.v8i16.v8i1(<8 x i16> %a, i32 16, i32 0, i32 1, <8 x i1> %1, <4 x i32> undef)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vshlltq_x_n_u8(<16 x i8> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vshlltq_x_n_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlltt.u8 q0, q0, #5
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.vshll.imm.predicated.v8i16.v16i8.v16i1(<16 x i8> %a, i32 5, i32 1, i32 1, <16 x i1> %1, <8 x i16> undef)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vshlltq_x_n_u8_lanewidth(<16 x i8> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vshlltq_x_n_u8_lanewidth:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlltt.u8 q0, q0, #8
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.vshll.imm.predicated.v8i16.v16i8.v16i1(<16 x i8> %a, i32 8, i32 1, i32 1, <16 x i1> %1, <8 x i16> undef)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vshlltq_x_n_u16(<8 x i16> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vshlltq_x_n_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlltt.u16 q0, q0, #3
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <4 x i32> @llvm.arm.mve.vshll.imm.predicated.v4i32.v8i16.v8i1(<8 x i16> %a, i32 3, i32 1, i32 1, <8 x i1> %1, <4 x i32> undef)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vshlltq_x_n_u16_lanewidth(<8 x i16> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vshlltq_x_n_u16_lanewidth:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlltt.u16 q0, q0, #16
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <4 x i32> @llvm.arm.mve.vshll.imm.predicated.v4i32.v8i16.v8i1(<8 x i16> %a, i32 16, i32 1, i32 1, <8 x i1> %1, <4 x i32> undef)
+  ret <4 x i32> %2
+}
+
+declare <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32)
+declare <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32)
+declare <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32)
+
+declare <16 x i8> @llvm.arm.mve.shl.imm.predicated.v16i8.v16i1(<16 x i8>, i32, <16 x i1>, <16 x i8>)
+declare <8 x i16> @llvm.arm.mve.shl.imm.predicated.v8i16.v8i1(<8 x i16>, i32, <8 x i1>, <8 x i16>)
+declare <4 x i32> @llvm.arm.mve.shl.imm.predicated.v4i32.v4i1(<4 x i32>, i32, <4 x i1>, <4 x i32>)
+
+declare <16 x i8> @llvm.arm.mve.shr.imm.predicated.v16i8.v16i1(<16 x i8>, i32, i32, <16 x i1>, <16 x i8>)
+declare <8 x i16> @llvm.arm.mve.shr.imm.predicated.v8i16.v8i1(<8 x i16>, i32, i32, <8 x i1>, <8 x i16>)
+declare <4 x i32> @llvm.arm.mve.shr.imm.predicated.v4i32.v4i1(<4 x i32>, i32, i32, <4 x i1>, <4 x i32>)
+
+declare <16 x i8> @llvm.arm.mve.vqshl.imm.v16i8(<16 x i8>, i32, i32)
+declare <8 x i16> @llvm.arm.mve.vqshl.imm.v8i16(<8 x i16>, i32, i32)
+declare <4 x i32> @llvm.arm.mve.vqshl.imm.v4i32(<4 x i32>, i32, i32)
+declare <16 x i8> @llvm.arm.mve.vqshl.imm.predicated.v16i8.v16i1(<16 x i8>, i32, i32, <16 x i1>, <16 x i8>)
+declare <8 x i16> @llvm.arm.mve.vqshl.imm.predicated.v8i16.v8i1(<8 x i16>, i32, i32, <8 x i1>, <8 x i16>)
+declare <4 x i32> @llvm.arm.mve.vqshl.imm.predicated.v4i32.v4i1(<4 x i32>, i32, i32, <4 x i1>, <4 x i32>)
+
+declare <16 x i8> @llvm.arm.mve.vqshlu.imm.v16i8(<16 x i8>, i32)
+declare <8 x i16> @llvm.arm.mve.vqshlu.imm.v8i16(<8 x i16>, i32)
+declare <4 x i32> @llvm.arm.mve.vqshlu.imm.v4i32(<4 x i32>, i32)
+declare <16 x i8> @llvm.arm.mve.vqshlu.imm.predicated.v16i8.v16i1(<16 x i8>, i32, <16 x i1>, <16 x i8>)
+declare <8 x i16> @llvm.arm.mve.vqshlu.imm.predicated.v8i16.v8i1(<8 x i16>, i32, <8 x i1>, <8 x i16>)
+declare <4 x i32> @llvm.arm.mve.vqshlu.imm.predicated.v4i32.v4i1(<4 x i32>, i32, <4 x i1>, <4 x i32>)
+
+declare <16 x i8> @llvm.arm.mve.vrshr.imm.v16i8(<16 x i8>, i32, i32)
+declare <8 x i16> @llvm.arm.mve.vrshr.imm.v8i16(<8 x i16>, i32, i32)
+declare <4 x i32> @llvm.arm.mve.vrshr.imm.v4i32(<4 x i32>, i32, i32)
+declare <16 x i8> @llvm.arm.mve.vrshr.imm.predicated.v16i8.v16i1(<16 x i8>, i32, i32, <16 x i1>, <16 x i8>)
+declare <8 x i16> @llvm.arm.mve.vrshr.imm.predicated.v8i16.v8i1(<8 x i16>, i32, i32, <8 x i1>, <8 x i16>)
+declare <4 x i32> @llvm.arm.mve.vrshr.imm.predicated.v4i32.v4i1(<4 x i32>, i32, i32, <4 x i1>, <4 x i32>)
+
+declare <8 x i16> @llvm.arm.mve.vshll.imm.v8i16.v16i8(<16 x i8>, i32, i32, i32)
+declare <4 x i32> @llvm.arm.mve.vshll.imm.v4i32.v8i16(<8 x i16>, i32, i32, i32)
+declare <8 x i16> @llvm.arm.mve.vshll.imm.predicated.v8i16.v16i8.v16i1(<16 x i8>, i32, i32, i32, <16 x i1>, <8 x i16>)
+declare <4 x i32> @llvm.arm.mve.vshll.imm.predicated.v4i32.v8i16.v8i1(<8 x i16>, i32, i32, i32, <8 x i1>, <4 x i32>)
diff --git a/llvm/test/CodeGen/Thumb2/mve-intrinsics/vector-shift-var.ll b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vector-shift-var.ll
new file mode 100644
index 000000000..5c3651374
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vector-shift-var.ll
@@ -0,0 +1,1338 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve -verify-machineinstrs -o - %s | FileCheck %s
+
+define arm_aapcs_vfpcc <16 x i8> @test_vshlq_s8(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vshlq_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vshl.s8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <16 x i8> @llvm.arm.mve.vshl.vector.v16i8.v16i8(<16 x i8> %a, <16 x i8> %b, i32 0, i32 0, i32 0)
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vshlq_s16(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vshlq_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vshl.s16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.arm.mve.vshl.vector.v8i16.v8i16(<8 x i16> %a, <8 x i16> %b, i32 0, i32 0, i32 0)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vshlq_s32(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vshlq_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vshl.s32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x i32> @llvm.arm.mve.vshl.vector.v4i32.v4i32(<4 x i32> %a, <4 x i32> %b, i32 0, i32 0, i32 0)
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vshlq_u8(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vshlq_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vshl.u8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <16 x i8> @llvm.arm.mve.vshl.vector.v16i8.v16i8(<16 x i8> %a, <16 x i8> %b, i32 0, i32 0, i32 1)
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vshlq_u16(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vshlq_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vshl.u16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.arm.mve.vshl.vector.v8i16.v8i16(<8 x i16> %a, <8 x i16> %b, i32 0, i32 0, i32 1)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vshlq_u32(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vshlq_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vshl.u32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x i32> @llvm.arm.mve.vshl.vector.v4i32.v4i32(<4 x i32> %a, <4 x i32> %b, i32 0, i32 0, i32 1)
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vshlq_r_s8(<16 x i8> %a, i32 %b) {
+; CHECK-LABEL: test_vshlq_r_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vshl.s8 q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <16 x i8> @llvm.arm.mve.vshl.scalar.v16i8(<16 x i8> %a, i32 %b, i32 0, i32 0, i32 0)
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vshlq_r_s16(<8 x i16> %a, i32 %b) {
+; CHECK-LABEL: test_vshlq_r_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vshl.s16 q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.arm.mve.vshl.scalar.v8i16(<8 x i16> %a, i32 %b, i32 0, i32 0, i32 0)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vshlq_r_s32(<4 x i32> %a, i32 %b) {
+; CHECK-LABEL: test_vshlq_r_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vshl.s32 q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x i32> @llvm.arm.mve.vshl.scalar.v4i32(<4 x i32> %a, i32 %b, i32 0, i32 0, i32 0)
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vshlq_r_u8(<16 x i8> %a, i32 %b) {
+; CHECK-LABEL: test_vshlq_r_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vshl.u8 q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <16 x i8> @llvm.arm.mve.vshl.scalar.v16i8(<16 x i8> %a, i32 %b, i32 0, i32 0, i32 1)
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vshlq_r_u16(<8 x i16> %a, i32 %b) {
+; CHECK-LABEL: test_vshlq_r_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vshl.u16 q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.arm.mve.vshl.scalar.v8i16(<8 x i16> %a, i32 %b, i32 0, i32 0, i32 1)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vshlq_r_u32(<4 x i32> %a, i32 %b) {
+; CHECK-LABEL: test_vshlq_r_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vshl.u32 q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x i32> @llvm.arm.mve.vshl.scalar.v4i32(<4 x i32> %a, i32 %b, i32 0, i32 0, i32 1)
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vqshlq_s8(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vqshlq_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqshl.s8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <16 x i8> @llvm.arm.mve.vshl.vector.v16i8.v16i8(<16 x i8> %a, <16 x i8> %b, i32 1, i32 0, i32 0)
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vqshlq_s16(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vqshlq_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqshl.s16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.arm.mve.vshl.vector.v8i16.v8i16(<8 x i16> %a, <8 x i16> %b, i32 1, i32 0, i32 0)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vqshlq_s32(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vqshlq_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqshl.s32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x i32> @llvm.arm.mve.vshl.vector.v4i32.v4i32(<4 x i32> %a, <4 x i32> %b, i32 1, i32 0, i32 0)
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vqshlq_u8(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vqshlq_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqshl.u8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <16 x i8> @llvm.arm.mve.vshl.vector.v16i8.v16i8(<16 x i8> %a, <16 x i8> %b, i32 1, i32 0, i32 1)
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vqshlq_u16(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vqshlq_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqshl.u16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.arm.mve.vshl.vector.v8i16.v8i16(<8 x i16> %a, <8 x i16> %b, i32 1, i32 0, i32 1)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vqshlq_u32(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vqshlq_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqshl.u32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x i32> @llvm.arm.mve.vshl.vector.v4i32.v4i32(<4 x i32> %a, <4 x i32> %b, i32 1, i32 0, i32 1)
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vqshlq_r_s8(<16 x i8> %a, i32 %b) {
+; CHECK-LABEL: test_vqshlq_r_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqshl.s8 q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <16 x i8> @llvm.arm.mve.vshl.scalar.v16i8(<16 x i8> %a, i32 %b, i32 1, i32 0, i32 0)
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vqshlq_r_s16(<8 x i16> %a, i32 %b) {
+; CHECK-LABEL: test_vqshlq_r_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqshl.s16 q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.arm.mve.vshl.scalar.v8i16(<8 x i16> %a, i32 %b, i32 1, i32 0, i32 0)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vqshlq_r_s32(<4 x i32> %a, i32 %b) {
+; CHECK-LABEL: test_vqshlq_r_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqshl.s32 q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x i32> @llvm.arm.mve.vshl.scalar.v4i32(<4 x i32> %a, i32 %b, i32 1, i32 0, i32 0)
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vqshlq_r_u8(<16 x i8> %a, i32 %b) {
+; CHECK-LABEL: test_vqshlq_r_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqshl.u8 q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <16 x i8> @llvm.arm.mve.vshl.scalar.v16i8(<16 x i8> %a, i32 %b, i32 1, i32 0, i32 1)
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vqshlq_r_u16(<8 x i16> %a, i32 %b) {
+; CHECK-LABEL: test_vqshlq_r_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqshl.u16 q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.arm.mve.vshl.scalar.v8i16(<8 x i16> %a, i32 %b, i32 1, i32 0, i32 1)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vqshlq_r_u32(<4 x i32> %a, i32 %b) {
+; CHECK-LABEL: test_vqshlq_r_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqshl.u32 q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x i32> @llvm.arm.mve.vshl.scalar.v4i32(<4 x i32> %a, i32 %b, i32 1, i32 0, i32 1)
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vrshlq_s8(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vrshlq_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vrshl.s8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <16 x i8> @llvm.arm.mve.vshl.vector.v16i8.v16i8(<16 x i8> %a, <16 x i8> %b, i32 0, i32 1, i32 0)
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vrshlq_s16(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vrshlq_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vrshl.s16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.arm.mve.vshl.vector.v8i16.v8i16(<8 x i16> %a, <8 x i16> %b, i32 0, i32 1, i32 0)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vrshlq_s32(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vrshlq_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vrshl.s32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x i32> @llvm.arm.mve.vshl.vector.v4i32.v4i32(<4 x i32> %a, <4 x i32> %b, i32 0, i32 1, i32 0)
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vrshlq_u8(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vrshlq_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vrshl.u8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <16 x i8> @llvm.arm.mve.vshl.vector.v16i8.v16i8(<16 x i8> %a, <16 x i8> %b, i32 0, i32 1, i32 1)
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vrshlq_u16(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vrshlq_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vrshl.u16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.arm.mve.vshl.vector.v8i16.v8i16(<8 x i16> %a, <8 x i16> %b, i32 0, i32 1, i32 1)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vrshlq_u32(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vrshlq_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vrshl.u32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x i32> @llvm.arm.mve.vshl.vector.v4i32.v4i32(<4 x i32> %a, <4 x i32> %b, i32 0, i32 1, i32 1)
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vrshlq_n_s8(<16 x i8> %a, i32 %b) {
+; CHECK-LABEL: test_vrshlq_n_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vrshl.s8 q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <16 x i8> @llvm.arm.mve.vshl.scalar.v16i8(<16 x i8> %a, i32 %b, i32 0, i32 1, i32 0)
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vrshlq_n_s16(<8 x i16> %a, i32 %b) {
+; CHECK-LABEL: test_vrshlq_n_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vrshl.s16 q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.arm.mve.vshl.scalar.v8i16(<8 x i16> %a, i32 %b, i32 0, i32 1, i32 0)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vrshlq_n_s32(<4 x i32> %a, i32 %b) {
+; CHECK-LABEL: test_vrshlq_n_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vrshl.s32 q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x i32> @llvm.arm.mve.vshl.scalar.v4i32(<4 x i32> %a, i32 %b, i32 0, i32 1, i32 0)
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vrshlq_n_u8(<16 x i8> %a, i32 %b) {
+; CHECK-LABEL: test_vrshlq_n_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vrshl.u8 q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <16 x i8> @llvm.arm.mve.vshl.scalar.v16i8(<16 x i8> %a, i32 %b, i32 0, i32 1, i32 1)
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vrshlq_n_u16(<8 x i16> %a, i32 %b) {
+; CHECK-LABEL: test_vrshlq_n_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vrshl.u16 q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.arm.mve.vshl.scalar.v8i16(<8 x i16> %a, i32 %b, i32 0, i32 1, i32 1)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vrshlq_n_u32(<4 x i32> %a, i32 %b) {
+; CHECK-LABEL: test_vrshlq_n_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vrshl.u32 q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x i32> @llvm.arm.mve.vshl.scalar.v4i32(<4 x i32> %a, i32 %b, i32 0, i32 1, i32 1)
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vqrshlq_s8(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vqrshlq_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqrshl.s8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <16 x i8> @llvm.arm.mve.vshl.vector.v16i8.v16i8(<16 x i8> %a, <16 x i8> %b, i32 1, i32 1, i32 0)
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vqrshlq_s16(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vqrshlq_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqrshl.s16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.arm.mve.vshl.vector.v8i16.v8i16(<8 x i16> %a, <8 x i16> %b, i32 1, i32 1, i32 0)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vqrshlq_s32(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vqrshlq_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqrshl.s32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x i32> @llvm.arm.mve.vshl.vector.v4i32.v4i32(<4 x i32> %a, <4 x i32> %b, i32 1, i32 1, i32 0)
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vqrshlq_u8(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vqrshlq_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqrshl.u8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <16 x i8> @llvm.arm.mve.vshl.vector.v16i8.v16i8(<16 x i8> %a, <16 x i8> %b, i32 1, i32 1, i32 1)
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vqrshlq_u16(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vqrshlq_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqrshl.u16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.arm.mve.vshl.vector.v8i16.v8i16(<8 x i16> %a, <8 x i16> %b, i32 1, i32 1, i32 1)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vqrshlq_u32(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vqrshlq_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqrshl.u32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x i32> @llvm.arm.mve.vshl.vector.v4i32.v4i32(<4 x i32> %a, <4 x i32> %b, i32 1, i32 1, i32 1)
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vqrshlq_n_s8(<16 x i8> %a, i32 %b) {
+; CHECK-LABEL: test_vqrshlq_n_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqrshl.s8 q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <16 x i8> @llvm.arm.mve.vshl.scalar.v16i8(<16 x i8> %a, i32 %b, i32 1, i32 1, i32 0)
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vqrshlq_n_s16(<8 x i16> %a, i32 %b) {
+; CHECK-LABEL: test_vqrshlq_n_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqrshl.s16 q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.arm.mve.vshl.scalar.v8i16(<8 x i16> %a, i32 %b, i32 1, i32 1, i32 0)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vqrshlq_n_s32(<4 x i32> %a, i32 %b) {
+; CHECK-LABEL: test_vqrshlq_n_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqrshl.s32 q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x i32> @llvm.arm.mve.vshl.scalar.v4i32(<4 x i32> %a, i32 %b, i32 1, i32 1, i32 0)
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vqrshlq_n_u8(<16 x i8> %a, i32 %b) {
+; CHECK-LABEL: test_vqrshlq_n_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqrshl.u8 q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <16 x i8> @llvm.arm.mve.vshl.scalar.v16i8(<16 x i8> %a, i32 %b, i32 1, i32 1, i32 1)
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vqrshlq_n_u16(<8 x i16> %a, i32 %b) {
+; CHECK-LABEL: test_vqrshlq_n_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqrshl.u16 q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.arm.mve.vshl.scalar.v8i16(<8 x i16> %a, i32 %b, i32 1, i32 1, i32 1)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vqrshlq_n_u32(<4 x i32> %a, i32 %b) {
+; CHECK-LABEL: test_vqrshlq_n_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqrshl.u32 q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x i32> @llvm.arm.mve.vshl.scalar.v4i32(<4 x i32> %a, i32 %b, i32 1, i32 1, i32 1)
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vshlq_m_s8(<16 x i8> %inactive, <16 x i8> %a, <16 x i8> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vshlq_m_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlt.s8 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = call <16 x i8> @llvm.arm.mve.vshl.vector.predicated.v16i8.v16i8.v16i1(<16 x i8> %a, <16 x i8> %b, i32 0, i32 0, i32 0, <16 x i1> %1, <16 x i8> %inactive)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vshlq_m_s16(<8 x i16> %inactive, <8 x i16> %a, <8 x i16> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vshlq_m_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlt.s16 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <8 x i16> @llvm.arm.mve.vshl.vector.predicated.v8i16.v8i16.v8i1(<8 x i16> %a, <8 x i16> %b, i32 0, i32 0, i32 0, <8 x i1> %1, <8 x i16> %inactive)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vshlq_m_s32(<4 x i32> %inactive, <4 x i32> %a, <4 x i32> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vshlq_m_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlt.s32 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x i32> @llvm.arm.mve.vshl.vector.predicated.v4i32.v4i32.v4i1(<4 x i32> %a, <4 x i32> %b, i32 0, i32 0, i32 0, <4 x i1> %1, <4 x i32> %inactive)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vshlq_m_u8(<16 x i8> %inactive, <16 x i8> %a, <16 x i8> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vshlq_m_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlt.u8 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = call <16 x i8> @llvm.arm.mve.vshl.vector.predicated.v16i8.v16i8.v16i1(<16 x i8> %a, <16 x i8> %b, i32 0, i32 0, i32 1, <16 x i1> %1, <16 x i8> %inactive)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vshlq_m_u16(<8 x i16> %inactive, <8 x i16> %a, <8 x i16> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vshlq_m_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlt.u16 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <8 x i16> @llvm.arm.mve.vshl.vector.predicated.v8i16.v8i16.v8i1(<8 x i16> %a, <8 x i16> %b, i32 0, i32 0, i32 1, <8 x i1> %1, <8 x i16> %inactive)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vshlq_m_u32(<4 x i32> %inactive, <4 x i32> %a, <4 x i32> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vshlq_m_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlt.u32 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x i32> @llvm.arm.mve.vshl.vector.predicated.v4i32.v4i32.v4i1(<4 x i32> %a, <4 x i32> %b, i32 0, i32 0, i32 1, <4 x i1> %1, <4 x i32> %inactive)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vshlq_x_s8(<16 x i8> %a, <16 x i8> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vshlq_x_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlt.s8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = call <16 x i8> @llvm.arm.mve.vshl.vector.predicated.v16i8.v16i8.v16i1(<16 x i8> %a, <16 x i8> %b, i32 0, i32 0, i32 0, <16 x i1> %1, <16 x i8> undef)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vshlq_x_s16(<8 x i16> %a, <8 x i16> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vshlq_x_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlt.s16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <8 x i16> @llvm.arm.mve.vshl.vector.predicated.v8i16.v8i16.v8i1(<8 x i16> %a, <8 x i16> %b, i32 0, i32 0, i32 0, <8 x i1> %1, <8 x i16> undef)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vshlq_x_s32(<4 x i32> %a, <4 x i32> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vshlq_x_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlt.s32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x i32> @llvm.arm.mve.vshl.vector.predicated.v4i32.v4i32.v4i1(<4 x i32> %a, <4 x i32> %b, i32 0, i32 0, i32 0, <4 x i1> %1, <4 x i32> undef)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vshlq_x_u8(<16 x i8> %a, <16 x i8> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vshlq_x_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlt.u8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = call <16 x i8> @llvm.arm.mve.vshl.vector.predicated.v16i8.v16i8.v16i1(<16 x i8> %a, <16 x i8> %b, i32 0, i32 0, i32 1, <16 x i1> %1, <16 x i8> undef)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vshlq_x_u16(<8 x i16> %a, <8 x i16> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vshlq_x_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlt.u16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <8 x i16> @llvm.arm.mve.vshl.vector.predicated.v8i16.v8i16.v8i1(<8 x i16> %a, <8 x i16> %b, i32 0, i32 0, i32 1, <8 x i1> %1, <8 x i16> undef)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vshlq_x_u32(<4 x i32> %a, <4 x i32> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vshlq_x_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlt.u32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x i32> @llvm.arm.mve.vshl.vector.predicated.v4i32.v4i32.v4i1(<4 x i32> %a, <4 x i32> %b, i32 0, i32 0, i32 1, <4 x i1> %1, <4 x i32> undef)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vshlq_m_r_s8(<16 x i8> %a, i32 %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vshlq_m_r_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlt.s8 q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = call <16 x i8> @llvm.arm.mve.vshl.scalar.predicated.v16i8.v16i1(<16 x i8> %a, i32 %b, i32 0, i32 0, i32 0, <16 x i1> %1)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vshlq_m_r_s16(<8 x i16> %a, i32 %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vshlq_m_r_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlt.s16 q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <8 x i16> @llvm.arm.mve.vshl.scalar.predicated.v8i16.v8i1(<8 x i16> %a, i32 %b, i32 0, i32 0, i32 0, <8 x i1> %1)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vshlq_m_r_s32(<4 x i32> %a, i32 %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vshlq_m_r_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlt.s32 q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x i32> @llvm.arm.mve.vshl.scalar.predicated.v4i32.v4i1(<4 x i32> %a, i32 %b, i32 0, i32 0, i32 0, <4 x i1> %1)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vshlq_m_r_u8(<16 x i8> %a, i32 %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vshlq_m_r_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlt.u8 q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = call <16 x i8> @llvm.arm.mve.vshl.scalar.predicated.v16i8.v16i1(<16 x i8> %a, i32 %b, i32 0, i32 0, i32 1, <16 x i1> %1)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vshlq_m_r_u16(<8 x i16> %a, i32 %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vshlq_m_r_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlt.u16 q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <8 x i16> @llvm.arm.mve.vshl.scalar.predicated.v8i16.v8i1(<8 x i16> %a, i32 %b, i32 0, i32 0, i32 1, <8 x i1> %1)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vshlq_m_r_u32(<4 x i32> %a, i32 %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vshlq_m_r_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlt.u32 q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x i32> @llvm.arm.mve.vshl.scalar.predicated.v4i32.v4i1(<4 x i32> %a, i32 %b, i32 0, i32 0, i32 1, <4 x i1> %1)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vqshlq_m_s8(<16 x i8> %inactive, <16 x i8> %a, <16 x i8> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vqshlq_m_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqshlt.s8 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = call <16 x i8> @llvm.arm.mve.vshl.vector.predicated.v16i8.v16i8.v16i1(<16 x i8> %a, <16 x i8> %b, i32 1, i32 0, i32 0, <16 x i1> %1, <16 x i8> %inactive)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vqshlq_m_s16(<8 x i16> %inactive, <8 x i16> %a, <8 x i16> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vqshlq_m_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqshlt.s16 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <8 x i16> @llvm.arm.mve.vshl.vector.predicated.v8i16.v8i16.v8i1(<8 x i16> %a, <8 x i16> %b, i32 1, i32 0, i32 0, <8 x i1> %1, <8 x i16> %inactive)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vqshlq_m_s32(<4 x i32> %inactive, <4 x i32> %a, <4 x i32> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vqshlq_m_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqshlt.s32 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x i32> @llvm.arm.mve.vshl.vector.predicated.v4i32.v4i32.v4i1(<4 x i32> %a, <4 x i32> %b, i32 1, i32 0, i32 0, <4 x i1> %1, <4 x i32> %inactive)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vqshlq_m_u8(<16 x i8> %inactive, <16 x i8> %a, <16 x i8> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vqshlq_m_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqshlt.u8 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = call <16 x i8> @llvm.arm.mve.vshl.vector.predicated.v16i8.v16i8.v16i1(<16 x i8> %a, <16 x i8> %b, i32 1, i32 0, i32 1, <16 x i1> %1, <16 x i8> %inactive)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vqshlq_m_u16(<8 x i16> %inactive, <8 x i16> %a, <8 x i16> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vqshlq_m_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqshlt.u16 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <8 x i16> @llvm.arm.mve.vshl.vector.predicated.v8i16.v8i16.v8i1(<8 x i16> %a, <8 x i16> %b, i32 1, i32 0, i32 1, <8 x i1> %1, <8 x i16> %inactive)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vqshlq_m_u32(<4 x i32> %inactive, <4 x i32> %a, <4 x i32> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vqshlq_m_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqshlt.u32 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x i32> @llvm.arm.mve.vshl.vector.predicated.v4i32.v4i32.v4i1(<4 x i32> %a, <4 x i32> %b, i32 1, i32 0, i32 1, <4 x i1> %1, <4 x i32> %inactive)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vqshlq_m_r_s8(<16 x i8> %a, i32 %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vqshlq_m_r_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqshlt.s8 q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = call <16 x i8> @llvm.arm.mve.vshl.scalar.predicated.v16i8.v16i1(<16 x i8> %a, i32 %b, i32 1, i32 0, i32 0, <16 x i1> %1)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vqshlq_m_r_s16(<8 x i16> %a, i32 %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vqshlq_m_r_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqshlt.s16 q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <8 x i16> @llvm.arm.mve.vshl.scalar.predicated.v8i16.v8i1(<8 x i16> %a, i32 %b, i32 1, i32 0, i32 0, <8 x i1> %1)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vqshlq_m_r_s32(<4 x i32> %a, i32 %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vqshlq_m_r_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqshlt.s32 q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x i32> @llvm.arm.mve.vshl.scalar.predicated.v4i32.v4i1(<4 x i32> %a, i32 %b, i32 1, i32 0, i32 0, <4 x i1> %1)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vqshlq_m_r_u8(<16 x i8> %a, i32 %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vqshlq_m_r_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqshlt.u8 q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = call <16 x i8> @llvm.arm.mve.vshl.scalar.predicated.v16i8.v16i1(<16 x i8> %a, i32 %b, i32 1, i32 0, i32 1, <16 x i1> %1)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vqshlq_m_r_u16(<8 x i16> %a, i32 %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vqshlq_m_r_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqshlt.u16 q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <8 x i16> @llvm.arm.mve.vshl.scalar.predicated.v8i16.v8i1(<8 x i16> %a, i32 %b, i32 1, i32 0, i32 1, <8 x i1> %1)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vqshlq_m_r_u32(<4 x i32> %a, i32 %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vqshlq_m_r_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqshlt.u32 q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x i32> @llvm.arm.mve.vshl.scalar.predicated.v4i32.v4i1(<4 x i32> %a, i32 %b, i32 1, i32 0, i32 1, <4 x i1> %1)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vrshlq_m_s8(<16 x i8> %inactive, <16 x i8> %a, <16 x i8> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vrshlq_m_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vrshlt.s8 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = call <16 x i8> @llvm.arm.mve.vshl.vector.predicated.v16i8.v16i8.v16i1(<16 x i8> %a, <16 x i8> %b, i32 0, i32 1, i32 0, <16 x i1> %1, <16 x i8> %inactive)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vrshlq_m_s16(<8 x i16> %inactive, <8 x i16> %a, <8 x i16> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vrshlq_m_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vrshlt.s16 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <8 x i16> @llvm.arm.mve.vshl.vector.predicated.v8i16.v8i16.v8i1(<8 x i16> %a, <8 x i16> %b, i32 0, i32 1, i32 0, <8 x i1> %1, <8 x i16> %inactive)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vrshlq_m_s32(<4 x i32> %inactive, <4 x i32> %a, <4 x i32> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vrshlq_m_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vrshlt.s32 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x i32> @llvm.arm.mve.vshl.vector.predicated.v4i32.v4i32.v4i1(<4 x i32> %a, <4 x i32> %b, i32 0, i32 1, i32 0, <4 x i1> %1, <4 x i32> %inactive)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vrshlq_m_u8(<16 x i8> %inactive, <16 x i8> %a, <16 x i8> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vrshlq_m_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vrshlt.u8 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = call <16 x i8> @llvm.arm.mve.vshl.vector.predicated.v16i8.v16i8.v16i1(<16 x i8> %a, <16 x i8> %b, i32 0, i32 1, i32 1, <16 x i1> %1, <16 x i8> %inactive)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vrshlq_m_u16(<8 x i16> %inactive, <8 x i16> %a, <8 x i16> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vrshlq_m_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vrshlt.u16 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <8 x i16> @llvm.arm.mve.vshl.vector.predicated.v8i16.v8i16.v8i1(<8 x i16> %a, <8 x i16> %b, i32 0, i32 1, i32 1, <8 x i1> %1, <8 x i16> %inactive)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vrshlq_m_u32(<4 x i32> %inactive, <4 x i32> %a, <4 x i32> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vrshlq_m_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vrshlt.u32 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x i32> @llvm.arm.mve.vshl.vector.predicated.v4i32.v4i32.v4i1(<4 x i32> %a, <4 x i32> %b, i32 0, i32 1, i32 1, <4 x i1> %1, <4 x i32> %inactive)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vrshlq_x_s8(<16 x i8> %a, <16 x i8> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vrshlq_x_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vrshlt.s8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = call <16 x i8> @llvm.arm.mve.vshl.vector.predicated.v16i8.v16i8.v16i1(<16 x i8> %a, <16 x i8> %b, i32 0, i32 1, i32 0, <16 x i1> %1, <16 x i8> undef)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vrshlq_x_s16(<8 x i16> %a, <8 x i16> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vrshlq_x_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vrshlt.s16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <8 x i16> @llvm.arm.mve.vshl.vector.predicated.v8i16.v8i16.v8i1(<8 x i16> %a, <8 x i16> %b, i32 0, i32 1, i32 0, <8 x i1> %1, <8 x i16> undef)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vrshlq_x_s32(<4 x i32> %a, <4 x i32> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vrshlq_x_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vrshlt.s32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x i32> @llvm.arm.mve.vshl.vector.predicated.v4i32.v4i32.v4i1(<4 x i32> %a, <4 x i32> %b, i32 0, i32 1, i32 0, <4 x i1> %1, <4 x i32> undef)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vrshlq_x_u8(<16 x i8> %a, <16 x i8> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vrshlq_x_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vrshlt.u8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = call <16 x i8> @llvm.arm.mve.vshl.vector.predicated.v16i8.v16i8.v16i1(<16 x i8> %a, <16 x i8> %b, i32 0, i32 1, i32 1, <16 x i1> %1, <16 x i8> undef)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vrshlq_x_u16(<8 x i16> %a, <8 x i16> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vrshlq_x_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vrshlt.u16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <8 x i16> @llvm.arm.mve.vshl.vector.predicated.v8i16.v8i16.v8i1(<8 x i16> %a, <8 x i16> %b, i32 0, i32 1, i32 1, <8 x i1> %1, <8 x i16> undef)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vrshlq_x_u32(<4 x i32> %a, <4 x i32> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vrshlq_x_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vrshlt.u32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x i32> @llvm.arm.mve.vshl.vector.predicated.v4i32.v4i32.v4i1(<4 x i32> %a, <4 x i32> %b, i32 0, i32 1, i32 1, <4 x i1> %1, <4 x i32> undef)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vrshlq_m_n_s8(<16 x i8> %a, i32 %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vrshlq_m_n_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vrshlt.s8 q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = call <16 x i8> @llvm.arm.mve.vshl.scalar.predicated.v16i8.v16i1(<16 x i8> %a, i32 %b, i32 0, i32 1, i32 0, <16 x i1> %1)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vrshlq_m_n_s16(<8 x i16> %a, i32 %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vrshlq_m_n_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vrshlt.s16 q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <8 x i16> @llvm.arm.mve.vshl.scalar.predicated.v8i16.v8i1(<8 x i16> %a, i32 %b, i32 0, i32 1, i32 0, <8 x i1> %1)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vrshlq_m_n_s32(<4 x i32> %a, i32 %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vrshlq_m_n_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vrshlt.s32 q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x i32> @llvm.arm.mve.vshl.scalar.predicated.v4i32.v4i1(<4 x i32> %a, i32 %b, i32 0, i32 1, i32 0, <4 x i1> %1)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vrshlq_m_n_u8(<16 x i8> %a, i32 %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vrshlq_m_n_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vrshlt.u8 q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = call <16 x i8> @llvm.arm.mve.vshl.scalar.predicated.v16i8.v16i1(<16 x i8> %a, i32 %b, i32 0, i32 1, i32 1, <16 x i1> %1)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vrshlq_m_n_u16(<8 x i16> %a, i32 %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vrshlq_m_n_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vrshlt.u16 q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <8 x i16> @llvm.arm.mve.vshl.scalar.predicated.v8i16.v8i1(<8 x i16> %a, i32 %b, i32 0, i32 1, i32 1, <8 x i1> %1)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vrshlq_m_n_u32(<4 x i32> %a, i32 %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vrshlq_m_n_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vrshlt.u32 q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x i32> @llvm.arm.mve.vshl.scalar.predicated.v4i32.v4i1(<4 x i32> %a, i32 %b, i32 0, i32 1, i32 1, <4 x i1> %1)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vqrshlq_m_s8(<16 x i8> %inactive, <16 x i8> %a, <16 x i8> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vqrshlq_m_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqrshlt.s8 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = call <16 x i8> @llvm.arm.mve.vshl.vector.predicated.v16i8.v16i8.v16i1(<16 x i8> %a, <16 x i8> %b, i32 1, i32 1, i32 0, <16 x i1> %1, <16 x i8> %inactive)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vqrshlq_m_s16(<8 x i16> %inactive, <8 x i16> %a, <8 x i16> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vqrshlq_m_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqrshlt.s16 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <8 x i16> @llvm.arm.mve.vshl.vector.predicated.v8i16.v8i16.v8i1(<8 x i16> %a, <8 x i16> %b, i32 1, i32 1, i32 0, <8 x i1> %1, <8 x i16> %inactive)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vqrshlq_m_s32(<4 x i32> %inactive, <4 x i32> %a, <4 x i32> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vqrshlq_m_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqrshlt.s32 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x i32> @llvm.arm.mve.vshl.vector.predicated.v4i32.v4i32.v4i1(<4 x i32> %a, <4 x i32> %b, i32 1, i32 1, i32 0, <4 x i1> %1, <4 x i32> %inactive)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vqrshlq_m_u8(<16 x i8> %inactive, <16 x i8> %a, <16 x i8> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vqrshlq_m_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqrshlt.u8 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = call <16 x i8> @llvm.arm.mve.vshl.vector.predicated.v16i8.v16i8.v16i1(<16 x i8> %a, <16 x i8> %b, i32 1, i32 1, i32 1, <16 x i1> %1, <16 x i8> %inactive)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vqrshlq_m_u16(<8 x i16> %inactive, <8 x i16> %a, <8 x i16> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vqrshlq_m_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqrshlt.u16 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <8 x i16> @llvm.arm.mve.vshl.vector.predicated.v8i16.v8i16.v8i1(<8 x i16> %a, <8 x i16> %b, i32 1, i32 1, i32 1, <8 x i1> %1, <8 x i16> %inactive)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vqrshlq_m_u32(<4 x i32> %inactive, <4 x i32> %a, <4 x i32> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vqrshlq_m_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqrshlt.u32 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x i32> @llvm.arm.mve.vshl.vector.predicated.v4i32.v4i32.v4i1(<4 x i32> %a, <4 x i32> %b, i32 1, i32 1, i32 1, <4 x i1> %1, <4 x i32> %inactive)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vqrshlq_m_n_s8(<16 x i8> %a, i32 %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vqrshlq_m_n_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqrshlt.s8 q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = call <16 x i8> @llvm.arm.mve.vshl.scalar.predicated.v16i8.v16i1(<16 x i8> %a, i32 %b, i32 1, i32 1, i32 0, <16 x i1> %1)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vqrshlq_m_n_s16(<8 x i16> %a, i32 %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vqrshlq_m_n_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqrshlt.s16 q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <8 x i16> @llvm.arm.mve.vshl.scalar.predicated.v8i16.v8i1(<8 x i16> %a, i32 %b, i32 1, i32 1, i32 0, <8 x i1> %1)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vqrshlq_m_n_s32(<4 x i32> %a, i32 %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vqrshlq_m_n_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqrshlt.s32 q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x i32> @llvm.arm.mve.vshl.scalar.predicated.v4i32.v4i1(<4 x i32> %a, i32 %b, i32 1, i32 1, i32 0, <4 x i1> %1)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vqrshlq_m_n_u8(<16 x i8> %a, i32 %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vqrshlq_m_n_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqrshlt.u8 q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = call <16 x i8> @llvm.arm.mve.vshl.scalar.predicated.v16i8.v16i1(<16 x i8> %a, i32 %b, i32 1, i32 1, i32 1, <16 x i1> %1)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vqrshlq_m_n_u16(<8 x i16> %a, i32 %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vqrshlq_m_n_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqrshlt.u16 q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <8 x i16> @llvm.arm.mve.vshl.scalar.predicated.v8i16.v8i1(<8 x i16> %a, i32 %b, i32 1, i32 1, i32 1, <8 x i1> %1)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vqrshlq_m_n_u32(<4 x i32> %a, i32 %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vqrshlq_m_n_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqrshlt.u32 q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call <4 x i32> @llvm.arm.mve.vshl.scalar.predicated.v4i32.v4i1(<4 x i32> %a, i32 %b, i32 1, i32 1, i32 1, <4 x i1> %1)
+  ret <4 x i32> %2
+}
+
+declare <16 x i8> @llvm.arm.mve.vshl.vector.v16i8.v16i8(<16 x i8>, <16 x i8>, i32, i32, i32)
+declare <8 x i16> @llvm.arm.mve.vshl.vector.v8i16.v8i16(<8 x i16>, <8 x i16>, i32, i32, i32)
+declare <4 x i32> @llvm.arm.mve.vshl.vector.v4i32.v4i32(<4 x i32>, <4 x i32>, i32, i32, i32)
+declare <16 x i8> @llvm.arm.mve.vshl.scalar.v16i8(<16 x i8>, i32, i32, i32, i32)
+declare <8 x i16> @llvm.arm.mve.vshl.scalar.v8i16(<8 x i16>, i32, i32, i32, i32)
+declare <4 x i32> @llvm.arm.mve.vshl.scalar.v4i32(<4 x i32>, i32, i32, i32, i32)
+declare <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32)
+declare <16 x i8> @llvm.arm.mve.vshl.vector.predicated.v16i8.v16i8.v16i1(<16 x i8>, <16 x i8>, i32, i32, i32, <16 x i1>, <16 x i8>)
+declare <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32)
+declare <8 x i16> @llvm.arm.mve.vshl.vector.predicated.v8i16.v8i16.v8i1(<8 x i16>, <8 x i16>, i32, i32, i32, <8 x i1>, <8 x i16>)
+declare <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32)
+declare <4 x i32> @llvm.arm.mve.vshl.vector.predicated.v4i32.v4i32.v4i1(<4 x i32>, <4 x i32>, i32, i32, i32, <4 x i1>, <4 x i32>)
+declare <16 x i8> @llvm.arm.mve.vshl.scalar.predicated.v16i8.v16i1(<16 x i8>, i32, i32, i32, i32, <16 x i1>)
+declare <8 x i16> @llvm.arm.mve.vshl.scalar.predicated.v8i16.v8i1(<8 x i16>, i32, i32, i32, i32, <8 x i1>)
+declare <4 x i32> @llvm.arm.mve.vshl.scalar.predicated.v4i32.v4i1(<4 x i32>, i32, i32, i32, i32, <4 x i1>)
diff --git a/llvm/test/CodeGen/Thumb2/mve-intrinsics/veorq.ll b/llvm/test/CodeGen/Thumb2/mve-intrinsics/veorq.ll
new file mode 100644
index 000000000..6f70cafb0
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-intrinsics/veorq.ll
@@ -0,0 +1,177 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve.fp -verify-machineinstrs -o - %s | FileCheck %s
+
+define arm_aapcs_vfpcc <16 x i8> @test_veorq_u8(<16 x i8> %a, <16 x i8> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_veorq_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    veor q0, q1, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = xor <16 x i8> %b, %a
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_veorq_s16(<8 x i16> %a, <8 x i16> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_veorq_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    veor q0, q1, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = xor <8 x i16> %b, %a
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_veorq_u32(<4 x i32> %a, <4 x i32> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_veorq_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    veor q0, q1, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = xor <4 x i32> %b, %a
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <4 x float> @test_veorq_f32(<4 x float> %a, <4 x float> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_veorq_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    veor q0, q1, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast <4 x float> %a to <4 x i32>
+  %1 = bitcast <4 x float> %b to <4 x i32>
+  %2 = xor <4 x i32> %1, %0
+  %3 = bitcast <4 x i32> %2 to <4 x float>
+  ret <4 x float> %3
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_veorq_m_s8(<16 x i8> %inactive, <16 x i8> %a, <16 x i8> %b, i16 zeroext %p) local_unnamed_addr #1 {
+; CHECK-LABEL: test_veorq_m_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    veort q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <16 x i8> @llvm.arm.mve.eor.predicated.v16i8.v16i1(<16 x i8> %a, <16 x i8> %b, <16 x i1> %1, <16 x i8> %inactive)
+  ret <16 x i8> %2
+}
+
+declare <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32) #2
+
+declare <16 x i8> @llvm.arm.mve.eor.predicated.v16i8.v16i1(<16 x i8>, <16 x i8>, <16 x i1>, <16 x i8>) #2
+
+define arm_aapcs_vfpcc <8 x i16> @test_veorq_m_u16(<8 x i16> %inactive, <8 x i16> %a, <8 x i16> %b, i16 zeroext %p) local_unnamed_addr #1 {
+; CHECK-LABEL: test_veorq_m_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    veort q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.eor.predicated.v8i16.v8i1(<8 x i16> %a, <8 x i16> %b, <8 x i1> %1, <8 x i16> %inactive)
+  ret <8 x i16> %2
+}
+
+declare <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32) #2
+
+declare <8 x i16> @llvm.arm.mve.eor.predicated.v8i16.v8i1(<8 x i16>, <8 x i16>, <8 x i1>, <8 x i16>) #2
+
+define arm_aapcs_vfpcc <4 x i32> @test_veorq_m_s32(<4 x i32> %inactive, <4 x i32> %a, <4 x i32> %b, i16 zeroext %p) local_unnamed_addr #1 {
+; CHECK-LABEL: test_veorq_m_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    veort q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call <4 x i32> @llvm.arm.mve.eor.predicated.v4i32.v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i1> %1, <4 x i32> %inactive)
+  ret <4 x i32> %2
+}
+
+declare <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32) #2
+
+declare <4 x i32> @llvm.arm.mve.eor.predicated.v4i32.v4i1(<4 x i32>, <4 x i32>, <4 x i1>, <4 x i32>) #2
+
+define arm_aapcs_vfpcc <8 x half> @test_veorq_m_f16(<8 x half> %inactive, <8 x half> %a, <8 x half> %b, i16 zeroext %p) local_unnamed_addr #1 {
+; CHECK-LABEL: test_veorq_m_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    veort q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast <8 x half> %a to <8 x i16>
+  %1 = bitcast <8 x half> %b to <8 x i16>
+  %2 = zext i16 %p to i32
+  %3 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %2)
+  %4 = bitcast <8 x half> %inactive to <8 x i16>
+  %5 = tail call <8 x i16> @llvm.arm.mve.eor.predicated.v8i16.v8i1(<8 x i16> %0, <8 x i16> %1, <8 x i1> %3, <8 x i16> %4)
+  %6 = bitcast <8 x i16> %5 to <8 x half>
+  ret <8 x half> %6
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_veorq_x_u8(<16 x i8> %a, <16 x i8> %b, i16 zeroext %p) local_unnamed_addr #1 {
+; CHECK-LABEL: test_veorq_x_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    veort q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <16 x i8> @llvm.arm.mve.eor.predicated.v16i8.v16i1(<16 x i8> %a, <16 x i8> %b, <16 x i1> %1, <16 x i8> undef)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_veorq_x_s16(<8 x i16> %a, <8 x i16> %b, i16 zeroext %p) local_unnamed_addr #1 {
+; CHECK-LABEL: test_veorq_x_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    veort q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.eor.predicated.v8i16.v8i1(<8 x i16> %a, <8 x i16> %b, <8 x i1> %1, <8 x i16> undef)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_veorq_x_u32(<4 x i32> %a, <4 x i32> %b, i16 zeroext %p) local_unnamed_addr #1 {
+; CHECK-LABEL: test_veorq_x_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    veort q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call <4 x i32> @llvm.arm.mve.eor.predicated.v4i32.v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i1> %1, <4 x i32> undef)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <4 x float> @test_veorq_m_f32(<4 x float> %a, <4 x float> %b, i16 zeroext %p) local_unnamed_addr #1 {
+; CHECK-LABEL: test_veorq_m_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    veort q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast <4 x float> %a to <4 x i32>
+  %1 = bitcast <4 x float> %b to <4 x i32>
+  %2 = zext i16 %p to i32
+  %3 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %2)
+  %4 = tail call <4 x i32> @llvm.arm.mve.eor.predicated.v4i32.v4i1(<4 x i32> %0, <4 x i32> %1, <4 x i1> %3, <4 x i32> undef)
+  %5 = bitcast <4 x i32> %4 to <4 x float>
+  ret <4 x float> %5
+}
+
diff --git a/llvm/test/CodeGen/Thumb2/mve-intrinsics/vhaddq.ll b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vhaddq.ll
new file mode 100644
index 000000000..9319dd1c4
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vhaddq.ll
@@ -0,0 +1,135 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve.fp -verify-machineinstrs -o - %s | FileCheck %s
+
+define arm_aapcs_vfpcc <16 x i8> @test_vhaddq_u8(<16 x i8> %a, <16 x i8> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vhaddq_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vhadd.u8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <16 x i8> @llvm.arm.mve.vhadd.v16i8(<16 x i8> %a, <16 x i8> %b, i32 1)
+  ret <16 x i8> %0
+}
+
+declare <16 x i8> @llvm.arm.mve.vhadd.v16i8(<16 x i8>, <16 x i8>, i32) #1
+
+define arm_aapcs_vfpcc <8 x i16> @test_vhaddq_s16(<8 x i16> %a, <8 x i16> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vhaddq_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vhadd.s16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <8 x i16> @llvm.arm.mve.vhadd.v8i16(<8 x i16> %a, <8 x i16> %b, i32 0)
+  ret <8 x i16> %0
+}
+
+declare <8 x i16> @llvm.arm.mve.vhadd.v8i16(<8 x i16>, <8 x i16>, i32) #1
+
+define arm_aapcs_vfpcc <4 x i32> @test_vhaddq_u32(<4 x i32> %a, <4 x i32> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vhaddq_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vhadd.u32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <4 x i32> @llvm.arm.mve.vhadd.v4i32(<4 x i32> %a, <4 x i32> %b, i32 1)
+  ret <4 x i32> %0
+}
+
+declare <4 x i32> @llvm.arm.mve.vhadd.v4i32(<4 x i32>, <4 x i32>, i32) #1
+
+define arm_aapcs_vfpcc <16 x i8> @test_vhaddq_m_s8(<16 x i8> %inactive, <16 x i8> %a, <16 x i8> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vhaddq_m_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vhaddt.s8 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <16 x i8> @llvm.arm.mve.hadd.predicated.v16i8.v16i1(<16 x i8> %a, <16 x i8> %b, i32 0, <16 x i1> %1, <16 x i8> %inactive)
+  ret <16 x i8> %2
+}
+
+declare <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32) #1
+
+declare <16 x i8> @llvm.arm.mve.hadd.predicated.v16i8.v16i1(<16 x i8>, <16 x i8>, i32, <16 x i1>, <16 x i8>) #1
+
+define arm_aapcs_vfpcc <8 x i16> @test_vhaddq_m_u16(<8 x i16> %inactive, <8 x i16> %a, <8 x i16> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vhaddq_m_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vhaddt.u16 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.hadd.predicated.v8i16.v8i1(<8 x i16> %a, <8 x i16> %b, i32 1, <8 x i1> %1, <8 x i16> %inactive)
+  ret <8 x i16> %2
+}
+
+declare <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32) #1
+
+declare <8 x i16> @llvm.arm.mve.hadd.predicated.v8i16.v8i1(<8 x i16>, <8 x i16>, i32, <8 x i1>, <8 x i16>) #1
+
+define arm_aapcs_vfpcc <4 x i32> @test_vhaddq_m_s32(<4 x i32> %inactive, <4 x i32> %a, <4 x i32> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vhaddq_m_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vhaddt.s32 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call <4 x i32> @llvm.arm.mve.hadd.predicated.v4i32.v4i1(<4 x i32> %a, <4 x i32> %b, i32 0, <4 x i1> %1, <4 x i32> %inactive)
+  ret <4 x i32> %2
+}
+
+declare <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32) #1
+
+declare <4 x i32> @llvm.arm.mve.hadd.predicated.v4i32.v4i1(<4 x i32>, <4 x i32>, i32, <4 x i1>, <4 x i32>) #1
+
+define arm_aapcs_vfpcc <16 x i8> @test_vhaddq_x_u8(<16 x i8> %a, <16 x i8> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vhaddq_x_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vhaddt.u8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <16 x i8> @llvm.arm.mve.hadd.predicated.v16i8.v16i1(<16 x i8> %a, <16 x i8> %b, i32 1, <16 x i1> %1, <16 x i8> undef)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vhaddq_x_s16(<8 x i16> %a, <8 x i16> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vhaddq_x_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vhaddt.s16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.hadd.predicated.v8i16.v8i1(<8 x i16> %a, <8 x i16> %b, i32 0, <8 x i1> %1, <8 x i16> undef)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vhaddq_x_u32(<4 x i32> %a, <4 x i32> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vhaddq_x_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vhaddt.u32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call <4 x i32> @llvm.arm.mve.hadd.predicated.v4i32.v4i1(<4 x i32> %a, <4 x i32> %b, i32 1, <4 x i1> %1, <4 x i32> undef)
+  ret <4 x i32> %2
+}
+
diff --git a/llvm/test/CodeGen/Thumb2/mve-intrinsics/vhsubq.ll b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vhsubq.ll
new file mode 100644
index 000000000..5bc8a2281
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vhsubq.ll
@@ -0,0 +1,92 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve.fp -verify-machineinstrs -o - %s | FileCheck %s
+
+define arm_aapcs_vfpcc <16 x i8> @test_vhsubq_u8(<16 x i8> %a, <16 x i8> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vhsubq_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vhsub.u8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <16 x i8> @llvm.arm.mve.vhsub.v16i8(<16 x i8> %a, <16 x i8> %b, i32 1)
+  ret <16 x i8> %0
+}
+
+declare <16 x i8> @llvm.arm.mve.vhsub.v16i8(<16 x i8>, <16 x i8>, i32) #1
+
+define arm_aapcs_vfpcc <8 x i16> @test_vhsubq_s16(<8 x i16> %a, <8 x i16> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vhsubq_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vhsub.s16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <8 x i16> @llvm.arm.mve.vhsub.v8i16(<8 x i16> %a, <8 x i16> %b, i32 0)
+  ret <8 x i16> %0
+}
+
+declare <8 x i16> @llvm.arm.mve.vhsub.v8i16(<8 x i16>, <8 x i16>, i32) #1
+
+define arm_aapcs_vfpcc <4 x i32> @test_vhsubq_u32(<4 x i32> %a, <4 x i32> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vhsubq_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vhsub.u32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <4 x i32> @llvm.arm.mve.vhsub.v4i32(<4 x i32> %a, <4 x i32> %b, i32 1)
+  ret <4 x i32> %0
+}
+
+declare <4 x i32> @llvm.arm.mve.vhsub.v4i32(<4 x i32>, <4 x i32>, i32) #1
+
+define arm_aapcs_vfpcc <16 x i8> @test_vhsubq_m_s8(<16 x i8> %inactive, <16 x i8> %a, <16 x i8> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vhsubq_m_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vhsubt.s8 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <16 x i8> @llvm.arm.mve.hsub.predicated.v16i8.v16i1(<16 x i8> %a, <16 x i8> %b, i32 0, <16 x i1> %1, <16 x i8> %inactive)
+  ret <16 x i8> %2
+}
+
+declare <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32) #1
+
+declare <16 x i8> @llvm.arm.mve.hsub.predicated.v16i8.v16i1(<16 x i8>, <16 x i8>, i32, <16 x i1>, <16 x i8>) #1
+
+define arm_aapcs_vfpcc <8 x i16> @test_vhsubq_m_u16(<8 x i16> %inactive, <8 x i16> %a, <8 x i16> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vhsubq_m_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vhsubt.u16 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.hsub.predicated.v8i16.v8i1(<8 x i16> %a, <8 x i16> %b, i32 1, <8 x i1> %1, <8 x i16> %inactive)
+  ret <8 x i16> %2
+}
+
+declare <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32) #1
+
+declare <8 x i16> @llvm.arm.mve.hsub.predicated.v8i16.v8i1(<8 x i16>, <8 x i16>, i32, <8 x i1>, <8 x i16>) #1
+
+define arm_aapcs_vfpcc <4 x i32> @test_vhsubq_m_s32(<4 x i32> %inactive, <4 x i32> %a, <4 x i32> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vhsubq_m_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vhsubt.s32 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call <4 x i32> @llvm.arm.mve.hsub.predicated.v4i32.v4i1(<4 x i32> %a, <4 x i32> %b, i32 0, <4 x i1> %1, <4 x i32> %inactive)
+  ret <4 x i32> %2
+}
+
+declare <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32) #1
+
+declare <4 x i32> @llvm.arm.mve.hsub.predicated.v4i32.v4i1(<4 x i32>, <4 x i32>, i32, <4 x i1>, <4 x i32>) #1
diff --git a/llvm/test/CodeGen/Thumb2/mve-intrinsics/vld24.ll b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vld24.ll
new file mode 100644
index 000000000..a8036a3ce
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vld24.ll
@@ -0,0 +1,109 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve.fp -verify-machineinstrs -o - %s | FileCheck %s
+
+%struct.float16x8x2_t = type { [2 x <8 x half>] }
+%struct.uint8x16x4_t = type { [4 x <16 x i8>] }
+%struct.uint32x4x2_t = type { [2 x <4 x i32>] }
+%struct.int8x16x4_t = type { [4 x <16 x i8>] }
+
+define arm_aapcs_vfpcc %struct.float16x8x2_t @test_vld2q_f16(half* %addr) {
+; CHECK-LABEL: test_vld2q_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vld20.16 {q0, q1}, [r0]
+; CHECK-NEXT:    vld21.16 {q0, q1}, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call { <8 x half>, <8 x half> } @llvm.arm.mve.vld2q.v8f16.p0f16(half* %addr)
+  %1 = extractvalue { <8 x half>, <8 x half> } %0, 0
+  %2 = insertvalue %struct.float16x8x2_t undef, <8 x half> %1, 0, 0
+  %3 = extractvalue { <8 x half>, <8 x half> } %0, 1
+  %4 = insertvalue %struct.float16x8x2_t %2, <8 x half> %3, 0, 1
+  ret %struct.float16x8x2_t %4
+}
+
+declare { <8 x half>, <8 x half> } @llvm.arm.mve.vld2q.v8f16.p0f16(half*)
+
+define arm_aapcs_vfpcc %struct.uint8x16x4_t @test_vld4q_u8(i8* %addr) {
+; CHECK-LABEL: test_vld4q_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vld40.8 {q0, q1, q2, q3}, [r0]
+; CHECK-NEXT:    vld41.8 {q0, q1, q2, q3}, [r0]
+; CHECK-NEXT:    vld42.8 {q0, q1, q2, q3}, [r0]
+; CHECK-NEXT:    vld43.8 {q0, q1, q2, q3}, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } @llvm.arm.mve.vld4q.v16i8.p0i8(i8* %addr)
+  %1 = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } %0, 0
+  %2 = insertvalue %struct.uint8x16x4_t undef, <16 x i8> %1, 0, 0
+  %3 = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } %0, 1
+  %4 = insertvalue %struct.uint8x16x4_t %2, <16 x i8> %3, 0, 1
+  %5 = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } %0, 2
+  %6 = insertvalue %struct.uint8x16x4_t %4, <16 x i8> %5, 0, 2
+  %7 = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } %0, 3
+  %8 = insertvalue %struct.uint8x16x4_t %6, <16 x i8> %7, 0, 3
+  ret %struct.uint8x16x4_t %8
+}
+
+declare { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } @llvm.arm.mve.vld4q.v16i8.p0i8(i8*)
+
+define arm_aapcs_vfpcc void @test_vst2q_u32(i32* %addr, %struct.uint32x4x2_t %value.coerce) {
+; CHECK-LABEL: test_vst2q_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    @ kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; CHECK-NEXT:    @ kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; CHECK-NEXT:    vst20.32 {q0, q1}, [r0]
+; CHECK-NEXT:    vst21.32 {q0, q1}, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %value.coerce.fca.0.0.extract = extractvalue %struct.uint32x4x2_t %value.coerce, 0, 0
+  %value.coerce.fca.0.1.extract = extractvalue %struct.uint32x4x2_t %value.coerce, 0, 1
+  tail call void @llvm.arm.mve.vst2q.p0i32.v4i32(i32* %addr, <4 x i32> %value.coerce.fca.0.0.extract, <4 x i32> %value.coerce.fca.0.1.extract, i32 0)
+  tail call void @llvm.arm.mve.vst2q.p0i32.v4i32(i32* %addr, <4 x i32> %value.coerce.fca.0.0.extract, <4 x i32> %value.coerce.fca.0.1.extract, i32 1)
+  ret void
+}
+
+declare void @llvm.arm.mve.vst2q.p0i32.v4i32(i32*, <4 x i32>, <4 x i32>, i32)
+
+define arm_aapcs_vfpcc void @test_vst2q_f16(half* %addr, %struct.float16x8x2_t %value.coerce) {
+; CHECK-LABEL: test_vst2q_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    @ kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; CHECK-NEXT:    @ kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; CHECK-NEXT:    vst20.16 {q0, q1}, [r0]
+; CHECK-NEXT:    vst21.16 {q0, q1}, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %value.coerce.fca.0.0.extract = extractvalue %struct.float16x8x2_t %value.coerce, 0, 0
+  %value.coerce.fca.0.1.extract = extractvalue %struct.float16x8x2_t %value.coerce, 0, 1
+  call void @llvm.arm.mve.vst2q.p0f16.v8f16(half* %addr, <8 x half> %value.coerce.fca.0.0.extract, <8 x half> %value.coerce.fca.0.1.extract, i32 0)
+  call void @llvm.arm.mve.vst2q.p0f16.v8f16(half* %addr, <8 x half> %value.coerce.fca.0.0.extract, <8 x half> %value.coerce.fca.0.1.extract, i32 1)
+  ret void
+}
+
+declare void @llvm.arm.mve.vst2q.p0f16.v8f16(half*, <8 x half>, <8 x half>, i32)
+
+define arm_aapcs_vfpcc void @test_vst4q_s8(i8* %addr, %struct.int8x16x4_t %value.coerce) {
+; CHECK-LABEL: test_vst4q_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    @ kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; CHECK-NEXT:    @ kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; CHECK-NEXT:    @ kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; CHECK-NEXT:    @ kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; CHECK-NEXT:    vst40.8 {q0, q1, q2, q3}, [r0]
+; CHECK-NEXT:    vst41.8 {q0, q1, q2, q3}, [r0]
+; CHECK-NEXT:    vst42.8 {q0, q1, q2, q3}, [r0]
+; CHECK-NEXT:    vst43.8 {q0, q1, q2, q3}, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %value.coerce.fca.0.0.extract = extractvalue %struct.int8x16x4_t %value.coerce, 0, 0
+  %value.coerce.fca.0.1.extract = extractvalue %struct.int8x16x4_t %value.coerce, 0, 1
+  %value.coerce.fca.0.2.extract = extractvalue %struct.int8x16x4_t %value.coerce, 0, 2
+  %value.coerce.fca.0.3.extract = extractvalue %struct.int8x16x4_t %value.coerce, 0, 3
+  tail call void @llvm.arm.mve.vst4q.p0i8.v16i8(i8* %addr, <16 x i8> %value.coerce.fca.0.0.extract, <16 x i8> %value.coerce.fca.0.1.extract, <16 x i8> %value.coerce.fca.0.2.extract, <16 x i8> %value.coerce.fca.0.3.extract, i32 0)
+  tail call void @llvm.arm.mve.vst4q.p0i8.v16i8(i8* %addr, <16 x i8> %value.coerce.fca.0.0.extract, <16 x i8> %value.coerce.fca.0.1.extract, <16 x i8> %value.coerce.fca.0.2.extract, <16 x i8> %value.coerce.fca.0.3.extract, i32 1)
+  tail call void @llvm.arm.mve.vst4q.p0i8.v16i8(i8* %addr, <16 x i8> %value.coerce.fca.0.0.extract, <16 x i8> %value.coerce.fca.0.1.extract, <16 x i8> %value.coerce.fca.0.2.extract, <16 x i8> %value.coerce.fca.0.3.extract, i32 2)
+  tail call void @llvm.arm.mve.vst4q.p0i8.v16i8(i8* %addr, <16 x i8> %value.coerce.fca.0.0.extract, <16 x i8> %value.coerce.fca.0.1.extract, <16 x i8> %value.coerce.fca.0.2.extract, <16 x i8> %value.coerce.fca.0.3.extract, i32 3)
+  ret void
+}
+
+declare void @llvm.arm.mve.vst4q.p0i8.v16i8(i8*, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, i32)
diff --git a/llvm/test/CodeGen/Thumb2/mve-intrinsics/vldr.ll b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vldr.ll
new file mode 100644
index 000000000..aa893abc0
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vldr.ll
@@ -0,0 +1,62 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve.fp -verify-machineinstrs -o - %s | FileCheck %s
+
+define arm_aapcs_vfpcc <4 x i32> @test_vldrwq_gather_base_wb_s32(<4 x i32>* %addr) {
+; CHECK-LABEL: test_vldrwq_gather_base_wb_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vldrw.u32 q1, [q0, #80]!
+; CHECK-NEXT:    vstrw.32 q1, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = load <4 x i32>, <4 x i32>* %addr, align 8
+  %1 = tail call { <4 x i32>, <4 x i32> } @llvm.arm.mve.vldr.gather.base.wb.v4i32.v4i32(<4 x i32> %0, i32 80)
+  %2 = extractvalue { <4 x i32>, <4 x i32> } %1, 1
+  store <4 x i32> %2, <4 x i32>* %addr, align 8
+  %3 = extractvalue { <4 x i32>, <4 x i32> } %1, 0
+  ret <4 x i32> %3
+}
+
+declare { <4 x i32>, <4 x i32> } @llvm.arm.mve.vldr.gather.base.wb.v4i32.v4i32(<4 x i32>, i32)
+
+define arm_aapcs_vfpcc <4 x float> @test_vldrwq_gather_base_wb_f32(<4 x i32>* %addr) {
+; CHECK-LABEL: test_vldrwq_gather_base_wb_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vldrw.u32 q1, [q0, #64]!
+; CHECK-NEXT:    vstrw.32 q1, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = load <4 x i32>, <4 x i32>* %addr, align 8
+  %1 = tail call { <4 x float>, <4 x i32> } @llvm.arm.mve.vldr.gather.base.wb.v4f32.v4i32(<4 x i32> %0, i32 64)
+  %2 = extractvalue { <4 x float>, <4 x i32> } %1, 1
+  store <4 x i32> %2, <4 x i32>* %addr, align 8
+  %3 = extractvalue { <4 x float>, <4 x i32> } %1, 0
+  ret <4 x float> %3
+}
+
+declare { <4 x float>, <4 x i32> } @llvm.arm.mve.vldr.gather.base.wb.v4f32.v4i32(<4 x i32>, i32)
+
+define arm_aapcs_vfpcc <2 x i64> @test_vldrdq_gather_base_wb_z_u64(<2 x i64>* %addr, i16 zeroext %p) {
+; CHECK-LABEL: test_vldrdq_gather_base_wb_z_u64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrdt.u64 q1, [q0, #656]!
+; CHECK-NEXT:    vstrw.32 q1, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = load <2 x i64>, <2 x i64>* %addr, align 8
+  %1 = zext i16 %p to i32
+  %2 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %1)
+  %3 = tail call { <2 x i64>, <2 x i64> } @llvm.arm.mve.vldr.gather.base.wb.predicated.v2i64.v2i64.v4i1(<2 x i64> %0, i32 656, <4 x i1> %2)
+  %4 = extractvalue { <2 x i64>, <2 x i64> } %3, 1
+  store <2 x i64> %4, <2 x i64>* %addr, align 8
+  %5 = extractvalue { <2 x i64>, <2 x i64> } %3, 0
+  ret <2 x i64> %5
+}
+
+declare <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32)
+
+declare { <2 x i64>, <2 x i64> } @llvm.arm.mve.vldr.gather.base.wb.predicated.v2i64.v2i64.v4i1(<2 x i64>, i32, <4 x i1>)
diff --git a/llvm/test/CodeGen/Thumb2/mve-intrinsics/vmaxnmq.ll b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vmaxnmq.ll
new file mode 100644
index 000000000..a8dca7c77
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vmaxnmq.ll
@@ -0,0 +1,91 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve.fp -verify-machineinstrs -o - %s | FileCheck %s
+
+define arm_aapcs_vfpcc <8 x half> @test_vmaxnmq_f16(<8 x half> %a, <8 x half> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vmaxnmq_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmaxnm.f16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <8 x half> @llvm.maxnum.v8f16(<8 x half> %a, <8 x half> %b)
+  ret <8 x half> %0
+}
+
+declare <8 x half> @llvm.maxnum.v8f16(<8 x half>, <8 x half>) #1
+
+define arm_aapcs_vfpcc <4 x float> @test_vmaxnmq_f32(<4 x float> %a, <4 x float> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vmaxnmq_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmaxnm.f32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <4 x float> @llvm.maxnum.v4f32(<4 x float> %a, <4 x float> %b)
+  ret <4 x float> %0
+}
+
+declare <4 x float> @llvm.maxnum.v4f32(<4 x float>, <4 x float>) #1
+
+define arm_aapcs_vfpcc <8 x half> @test_vmaxnmq_m_f16(<8 x half> %inactive, <8 x half> %a, <8 x half> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vmaxnmq_m_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmaxnmt.f16 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <8 x half> @llvm.arm.mve.max.predicated.v8f16.v8i1(<8 x half> %a, <8 x half> %b, i32 0, <8 x i1> %1, <8 x half> %inactive)
+  ret <8 x half> %2
+}
+
+declare <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32) #2
+
+declare <8 x half> @llvm.arm.mve.max.predicated.v8f16.v8i1(<8 x half>, <8 x half>, i32, <8 x i1>, <8 x half>) #2
+
+define arm_aapcs_vfpcc <4 x float> @test_vmaxnmq_m_f32(<4 x float> %inactive, <4 x float> %a, <4 x float> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vmaxnmq_m_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmaxnmt.f32 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call <4 x float> @llvm.arm.mve.max.predicated.v4f32.v4i1(<4 x float> %a, <4 x float> %b, i32 0, <4 x i1> %1, <4 x float> %inactive)
+  ret <4 x float> %2
+}
+
+declare <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32) #2
+
+declare <4 x float> @llvm.arm.mve.max.predicated.v4f32.v4i1(<4 x float>, <4 x float>, i32, <4 x i1>, <4 x float>) #2
+
+define arm_aapcs_vfpcc <8 x half> @test_vmaxnmq_x_f16(<8 x half> %a, <8 x half> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vmaxnmq_x_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmaxnmt.f16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <8 x half> @llvm.arm.mve.max.predicated.v8f16.v8i1(<8 x half> %a, <8 x half> %b, i32 0, <8 x i1> %1, <8 x half> undef)
+  ret <8 x half> %2
+}
+
+define arm_aapcs_vfpcc <4 x float> @test_vmaxnmq_x_f32(<4 x float> %a, <4 x float> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vmaxnmq_x_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmaxnmt.f32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call <4 x float> @llvm.arm.mve.max.predicated.v4f32.v4i1(<4 x float> %a, <4 x float> %b, i32 0, <4 x i1> %1, <4 x float> undef)
+  ret <4 x float> %2
+}
+
diff --git a/llvm/test/CodeGen/Thumb2/mve-intrinsics/vmaxq.ll b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vmaxq.ll
new file mode 100644
index 000000000..2036378b4
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vmaxq.ll
@@ -0,0 +1,132 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve.fp -verify-machineinstrs -o - %s | FileCheck %s
+
+define arm_aapcs_vfpcc <16 x i8> @test_vmaxq_s8(<16 x i8> %a, <16 x i8> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vmaxq_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmax.s8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = icmp slt <16 x i8> %a, %b
+  %1 = select <16 x i1> %0, <16 x i8> %b, <16 x i8> %a
+  ret <16 x i8> %1
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vmaxq_u16(<8 x i16> %a, <8 x i16> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vmaxq_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmax.u16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = icmp ult <8 x i16> %a, %b
+  %1 = select <8 x i1> %0, <8 x i16> %b, <8 x i16> %a
+  ret <8 x i16> %1
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vmaxq_s32(<4 x i32> %a, <4 x i32> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vmaxq_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmax.s32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = icmp slt <4 x i32> %a, %b
+  %1 = select <4 x i1> %0, <4 x i32> %b, <4 x i32> %a
+  ret <4 x i32> %1
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vmaxq_m_u8(<16 x i8> %inactive, <16 x i8> %a, <16 x i8> %b, i16 zeroext %p) local_unnamed_addr #1 {
+; CHECK-LABEL: test_vmaxq_m_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmaxt.u8 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <16 x i8> @llvm.arm.mve.max.predicated.v16i8.v16i1(<16 x i8> %a, <16 x i8> %b, i32 1, <16 x i1> %1, <16 x i8> %inactive)
+  ret <16 x i8> %2
+}
+
+declare <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32) #2
+
+declare <16 x i8> @llvm.arm.mve.max.predicated.v16i8.v16i1(<16 x i8>, <16 x i8>, i32, <16 x i1>, <16 x i8>) #2
+
+define arm_aapcs_vfpcc <8 x i16> @test_vmaxq_m_s16(<8 x i16> %inactive, <8 x i16> %a, <8 x i16> %b, i16 zeroext %p) local_unnamed_addr #1 {
+; CHECK-LABEL: test_vmaxq_m_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmaxt.s16 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.max.predicated.v8i16.v8i1(<8 x i16> %a, <8 x i16> %b, i32 0, <8 x i1> %1, <8 x i16> %inactive)
+  ret <8 x i16> %2
+}
+
+declare <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32) #2
+
+declare <8 x i16> @llvm.arm.mve.max.predicated.v8i16.v8i1(<8 x i16>, <8 x i16>, i32, <8 x i1>, <8 x i16>) #2
+
+define arm_aapcs_vfpcc <4 x i32> @test_vmaxq_m_u32(<4 x i32> %inactive, <4 x i32> %a, <4 x i32> %b, i16 zeroext %p) local_unnamed_addr #1 {
+; CHECK-LABEL: test_vmaxq_m_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmaxt.u32 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call <4 x i32> @llvm.arm.mve.max.predicated.v4i32.v4i1(<4 x i32> %a, <4 x i32> %b, i32 1, <4 x i1> %1, <4 x i32> %inactive)
+  ret <4 x i32> %2
+}
+
+declare <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32) #2
+
+declare <4 x i32> @llvm.arm.mve.max.predicated.v4i32.v4i1(<4 x i32>, <4 x i32>, i32, <4 x i1>, <4 x i32>) #2
+
+define arm_aapcs_vfpcc <16 x i8> @test_vmaxq_x_u8(<16 x i8> %a, <16 x i8> %b, i16 zeroext %p) local_unnamed_addr #1 {
+; CHECK-LABEL: test_vmaxq_x_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmaxt.u8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <16 x i8> @llvm.arm.mve.max.predicated.v16i8.v16i1(<16 x i8> %a, <16 x i8> %b, i32 1, <16 x i1> %1, <16 x i8> undef)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vmaxq_x_u16(<8 x i16> %a, <8 x i16> %b, i16 zeroext %p) local_unnamed_addr #1 {
+; CHECK-LABEL: test_vmaxq_x_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmaxt.s16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.max.predicated.v8i16.v8i1(<8 x i16> %a, <8 x i16> %b, i32 0, <8 x i1> %1, <8 x i16> undef)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vmaxq_x_s32(<4 x i32> %a, <4 x i32> %b, i16 zeroext %p) local_unnamed_addr #1 {
+; CHECK-LABEL: test_vmaxq_x_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmaxt.u32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call <4 x i32> @llvm.arm.mve.max.predicated.v4i32.v4i1(<4 x i32> %a, <4 x i32> %b, i32 1, <4 x i1> %1, <4 x i32> undef)
+  ret <4 x i32> %2
+}
+
diff --git a/llvm/test/CodeGen/Thumb2/mve-intrinsics/vminnmq.ll b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vminnmq.ll
new file mode 100644
index 000000000..d4853f7a2
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vminnmq.ll
@@ -0,0 +1,91 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve.fp -verify-machineinstrs -o - %s | FileCheck %s
+
+define arm_aapcs_vfpcc <8 x half> @test_vminnmq_f16(<8 x half> %a, <8 x half> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vminnmq_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vminnm.f16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <8 x half> @llvm.minnum.v8f16(<8 x half> %a, <8 x half> %b)
+  ret <8 x half> %0
+}
+
+declare <8 x half> @llvm.minnum.v8f16(<8 x half>, <8 x half>) #1
+
+define arm_aapcs_vfpcc <4 x float> @test_vminnmq_f32(<4 x float> %a, <4 x float> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vminnmq_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vminnm.f32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <4 x float> @llvm.minnum.v4f32(<4 x float> %a, <4 x float> %b)
+  ret <4 x float> %0
+}
+
+declare <4 x float> @llvm.minnum.v4f32(<4 x float>, <4 x float>) #1
+
+define arm_aapcs_vfpcc <8 x half> @test_vminnmq_m_f16(<8 x half> %inactive, <8 x half> %a, <8 x half> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vminnmq_m_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vminnmt.f16 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <8 x half> @llvm.arm.mve.min.predicated.v8f16.v8i1(<8 x half> %a, <8 x half> %b, i32 0, <8 x i1> %1, <8 x half> %inactive)
+  ret <8 x half> %2
+}
+
+declare <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32) #2
+
+declare <8 x half> @llvm.arm.mve.min.predicated.v8f16.v8i1(<8 x half>, <8 x half>, i32, <8 x i1>, <8 x half>) #2
+
+define arm_aapcs_vfpcc <4 x float> @test_vminnmq_m_f32(<4 x float> %inactive, <4 x float> %a, <4 x float> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vminnmq_m_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vminnmt.f32 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call <4 x float> @llvm.arm.mve.min.predicated.v4f32.v4i1(<4 x float> %a, <4 x float> %b, i32 0, <4 x i1> %1, <4 x float> %inactive)
+  ret <4 x float> %2
+}
+
+declare <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32) #2
+
+declare <4 x float> @llvm.arm.mve.min.predicated.v4f32.v4i1(<4 x float>, <4 x float>, i32, <4 x i1>, <4 x float>) #2
+
+define arm_aapcs_vfpcc <8 x half> @test_vminnmq_x_f16(<8 x half> %a, <8 x half> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vminnmq_x_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vminnmt.f16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <8 x half> @llvm.arm.mve.min.predicated.v8f16.v8i1(<8 x half> %a, <8 x half> %b, i32 0, <8 x i1> %1, <8 x half> undef)
+  ret <8 x half> %2
+}
+
+define arm_aapcs_vfpcc <4 x float> @test_vminnmq_x_f32(<4 x float> %a, <4 x float> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vminnmq_x_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vminnmt.f32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call <4 x float> @llvm.arm.mve.min.predicated.v4f32.v4i1(<4 x float> %a, <4 x float> %b, i32 0, <4 x i1> %1, <4 x float> undef)
+  ret <4 x float> %2
+}
+
diff --git a/llvm/test/CodeGen/Thumb2/mve-intrinsics/vminq.ll b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vminq.ll
new file mode 100644
index 000000000..9c93a7d23
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vminq.ll
@@ -0,0 +1,132 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve.fp -verify-machineinstrs -o - %s | FileCheck %s
+
+define arm_aapcs_vfpcc <16 x i8> @test_vminq_u8(<16 x i8> %a, <16 x i8> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vminq_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmin.u8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = icmp ugt <16 x i8> %a, %b
+  %1 = select <16 x i1> %0, <16 x i8> %b, <16 x i8> %a
+  ret <16 x i8> %1
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vminq_s16(<8 x i16> %a, <8 x i16> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vminq_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmin.s16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = icmp sgt <8 x i16> %a, %b
+  %1 = select <8 x i1> %0, <8 x i16> %b, <8 x i16> %a
+  ret <8 x i16> %1
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vminq_u32(<4 x i32> %a, <4 x i32> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vminq_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmin.u32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = icmp ugt <4 x i32> %a, %b
+  %1 = select <4 x i1> %0, <4 x i32> %b, <4 x i32> %a
+  ret <4 x i32> %1
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vminq_m_s8(<16 x i8> %inactive, <16 x i8> %a, <16 x i8> %b, i16 zeroext %p) local_unnamed_addr #1 {
+; CHECK-LABEL: test_vminq_m_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmint.s8 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <16 x i8> @llvm.arm.mve.min.predicated.v16i8.v16i1(<16 x i8> %a, <16 x i8> %b, i32 0, <16 x i1> %1, <16 x i8> %inactive)
+  ret <16 x i8> %2
+}
+
+declare <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32) #2
+
+declare <16 x i8> @llvm.arm.mve.min.predicated.v16i8.v16i1(<16 x i8>, <16 x i8>, i32, <16 x i1>, <16 x i8>) #2
+
+define arm_aapcs_vfpcc <8 x i16> @test_vminq_m_u16(<8 x i16> %inactive, <8 x i16> %a, <8 x i16> %b, i16 zeroext %p) local_unnamed_addr #1 {
+; CHECK-LABEL: test_vminq_m_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmint.u16 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.min.predicated.v8i16.v8i1(<8 x i16> %a, <8 x i16> %b, i32 1, <8 x i1> %1, <8 x i16> %inactive)
+  ret <8 x i16> %2
+}
+
+declare <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32) #2
+
+declare <8 x i16> @llvm.arm.mve.min.predicated.v8i16.v8i1(<8 x i16>, <8 x i16>, i32, <8 x i1>, <8 x i16>) #2
+
+define arm_aapcs_vfpcc <4 x i32> @test_vminq_m_s32(<4 x i32> %inactive, <4 x i32> %a, <4 x i32> %b, i16 zeroext %p) local_unnamed_addr #1 {
+; CHECK-LABEL: test_vminq_m_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmint.s32 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call <4 x i32> @llvm.arm.mve.min.predicated.v4i32.v4i1(<4 x i32> %a, <4 x i32> %b, i32 0, <4 x i1> %1, <4 x i32> %inactive)
+  ret <4 x i32> %2
+}
+
+declare <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32) #2
+
+declare <4 x i32> @llvm.arm.mve.min.predicated.v4i32.v4i1(<4 x i32>, <4 x i32>, i32, <4 x i1>, <4 x i32>) #2
+
+define arm_aapcs_vfpcc <16 x i8> @test_vminq_x_u8(<16 x i8> %a, <16 x i8> %b, i16 zeroext %p) local_unnamed_addr #1 {
+; CHECK-LABEL: test_vminq_x_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmint.u8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <16 x i8> @llvm.arm.mve.min.predicated.v16i8.v16i1(<16 x i8> %a, <16 x i8> %b, i32 1, <16 x i1> %1, <16 x i8> undef)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vminq_x_s16(<8 x i16> %a, <8 x i16> %b, i16 zeroext %p) local_unnamed_addr #1 {
+; CHECK-LABEL: test_vminq_x_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmint.s16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.min.predicated.v8i16.v8i1(<8 x i16> %a, <8 x i16> %b, i32 0, <8 x i1> %1, <8 x i16> undef)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vminq_x_s32(<4 x i32> %a, <4 x i32> %b, i16 zeroext %p) local_unnamed_addr #1 {
+; CHECK-LABEL: test_vminq_x_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmint.s32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call <4 x i32> @llvm.arm.mve.min.predicated.v4i32.v4i1(<4 x i32> %a, <4 x i32> %b, i32 0, <4 x i1> %1, <4 x i32> undef)
+  ret <4 x i32> %2
+}
+
diff --git a/llvm/test/CodeGen/Thumb2/mve-intrinsics/vminvq.ll b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vminvq.ll
new file mode 100644
index 000000000..a7c378020
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vminvq.ll
@@ -0,0 +1,36 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve.fp -verify-machineinstrs -o - %s | FileCheck %s
+
+define arm_aapcs_vfpcc i32 @test_vminvq_u32(i32 %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vminvq_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vminv.u32 r0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call i32 @llvm.arm.mve.minv.u.v4i32(i32 %a, <4 x i32> %b)
+  ret i32 %0
+}
+
+define arm_aapcs_vfpcc i32 @test_vmaxvq_u8(i32 %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vmaxvq_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmaxv.u8 r0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call i32 @llvm.arm.mve.maxv.u.v16i8(i32 %a, <16 x i8> %b)
+  ret i32 %0
+}
+
+define arm_aapcs_vfpcc i32 @test_vminvq_s16(i32 %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vminvq_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vminv.s16 r0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call i32 @llvm.arm.mve.minv.s.v8i16(i32 %a, <8 x i16> %b)
+  ret i32 %0
+}
+
+declare i32 @llvm.arm.mve.minv.u.v4i32(i32, <4 x i32>)
+declare i32 @llvm.arm.mve.maxv.u.v16i8(i32, <16 x i8>)
+declare i32 @llvm.arm.mve.minv.s.v8i16(i32, <8 x i16>)
diff --git a/llvm/test/CodeGen/Thumb2/mve-intrinsics/vmldav.ll b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vmldav.ll
new file mode 100644
index 000000000..ae8f24386
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vmldav.ll
@@ -0,0 +1,734 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve.fp -verify-machineinstrs -o - %s | FileCheck %s
+
+declare <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32)
+declare <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32)
+declare <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32)
+
+declare i32 @llvm.arm.mve.vmldava.v16i8(i32, i32, i32, i32, <16 x i8>, <16 x i8>)
+declare i32 @llvm.arm.mve.vmldava.v8i16(i32, i32, i32, i32, <8 x i16>, <8 x i16>)
+declare i32 @llvm.arm.mve.vmldava.v4i32(i32, i32, i32, i32, <4 x i32>, <4 x i32>)
+
+declare i32 @llvm.arm.mve.vmldava.predicated.v16i8.v16i1(i32, i32, i32, i32, <16 x i8>, <16 x i8>, <16 x i1>)
+declare i32 @llvm.arm.mve.vmldava.predicated.v8i16.v8i1(i32, i32, i32, i32, <8 x i16>, <8 x i16>, <8 x i1>)
+declare i32 @llvm.arm.mve.vmldava.predicated.v4i32.v4i1(i32, i32, i32, i32, <4 x i32>, <4 x i32>, <4 x i1>)
+
+define arm_aapcs_vfpcc i32 @test_vmladavaq_s8(i32 %a, <16 x i8> %b, <16 x i8> %c) {
+; CHECK-LABEL: test_vmladavaq_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmlava.s8 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call i32 @llvm.arm.mve.vmldava.v16i8(i32 0, i32 0, i32 0, i32 %a, <16 x i8> %b, <16 x i8> %c)
+  ret i32 %0
+}
+
+define arm_aapcs_vfpcc i32 @test_vmladavaq_s16(i32 %a, <8 x i16> %b, <8 x i16> %c) {
+; CHECK-LABEL: test_vmladavaq_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmlava.s16 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call i32 @llvm.arm.mve.vmldava.v8i16(i32 0, i32 0, i32 0, i32 %a, <8 x i16> %b, <8 x i16> %c)
+  ret i32 %0
+}
+
+define arm_aapcs_vfpcc i32 @test_vmladavaq_s32(i32 %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LABEL: test_vmladavaq_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmlava.s32 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call i32 @llvm.arm.mve.vmldava.v4i32(i32 0, i32 0, i32 0, i32 %a, <4 x i32> %b, <4 x i32> %c)
+  ret i32 %0
+}
+
+define arm_aapcs_vfpcc i32 @test_vmladavaq_u8(i32 %a, <16 x i8> %b, <16 x i8> %c) {
+; CHECK-LABEL: test_vmladavaq_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmlava.u8 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call i32 @llvm.arm.mve.vmldava.v16i8(i32 1, i32 0, i32 0, i32 %a, <16 x i8> %b, <16 x i8> %c)
+  ret i32 %0
+}
+
+define arm_aapcs_vfpcc i32 @test_vmladavaq_u16(i32 %a, <8 x i16> %b, <8 x i16> %c) {
+; CHECK-LABEL: test_vmladavaq_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmlava.u16 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call i32 @llvm.arm.mve.vmldava.v8i16(i32 1, i32 0, i32 0, i32 %a, <8 x i16> %b, <8 x i16> %c)
+  ret i32 %0
+}
+
+define arm_aapcs_vfpcc i32 @test_vmladavaq_u32(i32 %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LABEL: test_vmladavaq_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmlava.u32 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call i32 @llvm.arm.mve.vmldava.v4i32(i32 1, i32 0, i32 0, i32 %a, <4 x i32> %b, <4 x i32> %c)
+  ret i32 %0
+}
+
+define arm_aapcs_vfpcc i32 @test_vmladavaxq_s8(i32 %a, <16 x i8> %b, <16 x i8> %c) {
+; CHECK-LABEL: test_vmladavaxq_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmladavax.s8 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call i32 @llvm.arm.mve.vmldava.v16i8(i32 0, i32 0, i32 1, i32 %a, <16 x i8> %b, <16 x i8> %c)
+  ret i32 %0
+}
+
+define arm_aapcs_vfpcc i32 @test_vmladavaxq_s16(i32 %a, <8 x i16> %b, <8 x i16> %c) {
+; CHECK-LABEL: test_vmladavaxq_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmladavax.s16 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call i32 @llvm.arm.mve.vmldava.v8i16(i32 0, i32 0, i32 1, i32 %a, <8 x i16> %b, <8 x i16> %c)
+  ret i32 %0
+}
+
+define arm_aapcs_vfpcc i32 @test_vmladavaxq_s32(i32 %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LABEL: test_vmladavaxq_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmladavax.s32 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call i32 @llvm.arm.mve.vmldava.v4i32(i32 0, i32 0, i32 1, i32 %a, <4 x i32> %b, <4 x i32> %c)
+  ret i32 %0
+}
+
+define arm_aapcs_vfpcc i32 @test_vmlsdavaq_s8(i32 %a, <16 x i8> %b, <16 x i8> %c) {
+; CHECK-LABEL: test_vmlsdavaq_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmlsdava.s8 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call i32 @llvm.arm.mve.vmldava.v16i8(i32 0, i32 1, i32 0, i32 %a, <16 x i8> %b, <16 x i8> %c)
+  ret i32 %0
+}
+
+define arm_aapcs_vfpcc i32 @test_vmlsdavaq_s16(i32 %a, <8 x i16> %b, <8 x i16> %c) {
+; CHECK-LABEL: test_vmlsdavaq_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmlsdava.s16 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call i32 @llvm.arm.mve.vmldava.v8i16(i32 0, i32 1, i32 0, i32 %a, <8 x i16> %b, <8 x i16> %c)
+  ret i32 %0
+}
+
+define arm_aapcs_vfpcc i32 @test_vmlsdavaq_s32(i32 %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LABEL: test_vmlsdavaq_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmlsdava.s32 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call i32 @llvm.arm.mve.vmldava.v4i32(i32 0, i32 1, i32 0, i32 %a, <4 x i32> %b, <4 x i32> %c)
+  ret i32 %0
+}
+
+define arm_aapcs_vfpcc i32 @test_vmlsdavaxq_s8(i32 %a, <16 x i8> %b, <16 x i8> %c) {
+; CHECK-LABEL: test_vmlsdavaxq_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmlsdavax.s8 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call i32 @llvm.arm.mve.vmldava.v16i8(i32 0, i32 1, i32 1, i32 %a, <16 x i8> %b, <16 x i8> %c)
+  ret i32 %0
+}
+
+define arm_aapcs_vfpcc i32 @test_vmlsdavaxq_s16(i32 %a, <8 x i16> %b, <8 x i16> %c) {
+; CHECK-LABEL: test_vmlsdavaxq_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmlsdavax.s16 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call i32 @llvm.arm.mve.vmldava.v8i16(i32 0, i32 1, i32 1, i32 %a, <8 x i16> %b, <8 x i16> %c)
+  ret i32 %0
+}
+
+define arm_aapcs_vfpcc i32 @test_vmlsdavaxq_s32(i32 %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LABEL: test_vmlsdavaxq_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmlsdavax.s32 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call i32 @llvm.arm.mve.vmldava.v4i32(i32 0, i32 1, i32 1, i32 %a, <4 x i32> %b, <4 x i32> %c)
+  ret i32 %0
+}
+
+define arm_aapcs_vfpcc i32 @test_vmladavaq_p_s8(i32 %a, <16 x i8> %b, <16 x i8> %c, i16 zeroext %p) {
+; CHECK-LABEL: test_vmladavaq_p_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmlavat.s8 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = call i32 @llvm.arm.mve.vmldava.predicated.v16i8.v16i1(i32 0, i32 0, i32 0, i32 %a, <16 x i8> %b, <16 x i8> %c, <16 x i1> %1)
+  ret i32 %2
+}
+
+define arm_aapcs_vfpcc i32 @test_vmladavaq_p_s16(i32 %a, <8 x i16> %b, <8 x i16> %c, i16 zeroext %p) {
+; CHECK-LABEL: test_vmladavaq_p_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmlavat.s16 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call i32 @llvm.arm.mve.vmldava.predicated.v8i16.v8i1(i32 0, i32 0, i32 0, i32 %a, <8 x i16> %b, <8 x i16> %c, <8 x i1> %1)
+  ret i32 %2
+}
+
+define arm_aapcs_vfpcc i32 @test_vmladavaq_p_s32(i32 %a, <4 x i32> %b, <4 x i32> %c, i16 zeroext %p) {
+; CHECK-LABEL: test_vmladavaq_p_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmlavat.s32 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call i32 @llvm.arm.mve.vmldava.predicated.v4i32.v4i1(i32 0, i32 0, i32 0, i32 %a, <4 x i32> %b, <4 x i32> %c, <4 x i1> %1)
+  ret i32 %2
+}
+
+define arm_aapcs_vfpcc i32 @test_vmladavaq_p_u8(i32 %a, <16 x i8> %b, <16 x i8> %c, i16 zeroext %p) {
+; CHECK-LABEL: test_vmladavaq_p_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmlavat.u8 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = call i32 @llvm.arm.mve.vmldava.predicated.v16i8.v16i1(i32 1, i32 0, i32 0, i32 %a, <16 x i8> %b, <16 x i8> %c, <16 x i1> %1)
+  ret i32 %2
+}
+
+define arm_aapcs_vfpcc i32 @test_vmladavaq_p_u16(i32 %a, <8 x i16> %b, <8 x i16> %c, i16 zeroext %p) {
+; CHECK-LABEL: test_vmladavaq_p_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmlavat.u16 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call i32 @llvm.arm.mve.vmldava.predicated.v8i16.v8i1(i32 1, i32 0, i32 0, i32 %a, <8 x i16> %b, <8 x i16> %c, <8 x i1> %1)
+  ret i32 %2
+}
+
+define arm_aapcs_vfpcc i32 @test_vmladavaq_p_u32(i32 %a, <4 x i32> %b, <4 x i32> %c, i16 zeroext %p) {
+; CHECK-LABEL: test_vmladavaq_p_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmlavat.u32 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call i32 @llvm.arm.mve.vmldava.predicated.v4i32.v4i1(i32 1, i32 0, i32 0, i32 %a, <4 x i32> %b, <4 x i32> %c, <4 x i1> %1)
+  ret i32 %2
+}
+
+define arm_aapcs_vfpcc i32 @test_vmladavaxq_p_s8(i32 %a, <16 x i8> %b, <16 x i8> %c, i16 zeroext %p) {
+; CHECK-LABEL: test_vmladavaxq_p_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmladavaxt.s8 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = call i32 @llvm.arm.mve.vmldava.predicated.v16i8.v16i1(i32 0, i32 0, i32 1, i32 %a, <16 x i8> %b, <16 x i8> %c, <16 x i1> %1)
+  ret i32 %2
+}
+
+define arm_aapcs_vfpcc i32 @test_vmladavaxq_p_s16(i32 %a, <8 x i16> %b, <8 x i16> %c, i16 zeroext %p) {
+; CHECK-LABEL: test_vmladavaxq_p_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmladavaxt.s16 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call i32 @llvm.arm.mve.vmldava.predicated.v8i16.v8i1(i32 0, i32 0, i32 1, i32 %a, <8 x i16> %b, <8 x i16> %c, <8 x i1> %1)
+  ret i32 %2
+}
+
+define arm_aapcs_vfpcc i32 @test_vmladavaxq_p_s32(i32 %a, <4 x i32> %b, <4 x i32> %c, i16 zeroext %p) {
+; CHECK-LABEL: test_vmladavaxq_p_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmladavaxt.s32 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call i32 @llvm.arm.mve.vmldava.predicated.v4i32.v4i1(i32 0, i32 0, i32 1, i32 %a, <4 x i32> %b, <4 x i32> %c, <4 x i1> %1)
+  ret i32 %2
+}
+
+define arm_aapcs_vfpcc i32 @test_vmlsdavaq_p_s8(i32 %a, <16 x i8> %b, <16 x i8> %c, i16 zeroext %p) {
+; CHECK-LABEL: test_vmlsdavaq_p_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmlsdavat.s8 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = call i32 @llvm.arm.mve.vmldava.predicated.v16i8.v16i1(i32 0, i32 1, i32 0, i32 %a, <16 x i8> %b, <16 x i8> %c, <16 x i1> %1)
+  ret i32 %2
+}
+
+define arm_aapcs_vfpcc i32 @test_vmlsdavaq_p_s16(i32 %a, <8 x i16> %b, <8 x i16> %c, i16 zeroext %p) {
+; CHECK-LABEL: test_vmlsdavaq_p_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmlsdavat.s16 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call i32 @llvm.arm.mve.vmldava.predicated.v8i16.v8i1(i32 0, i32 1, i32 0, i32 %a, <8 x i16> %b, <8 x i16> %c, <8 x i1> %1)
+  ret i32 %2
+}
+
+define arm_aapcs_vfpcc i32 @test_vmlsdavaq_p_s32(i32 %a, <4 x i32> %b, <4 x i32> %c, i16 zeroext %p) {
+; CHECK-LABEL: test_vmlsdavaq_p_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmlsdavat.s32 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call i32 @llvm.arm.mve.vmldava.predicated.v4i32.v4i1(i32 0, i32 1, i32 0, i32 %a, <4 x i32> %b, <4 x i32> %c, <4 x i1> %1)
+  ret i32 %2
+}
+
+define arm_aapcs_vfpcc i32 @test_vmlsdavaxq_p_s8(i32 %a, <16 x i8> %b, <16 x i8> %c, i16 zeroext %p) {
+; CHECK-LABEL: test_vmlsdavaxq_p_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmlsdavaxt.s8 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = call i32 @llvm.arm.mve.vmldava.predicated.v16i8.v16i1(i32 0, i32 1, i32 1, i32 %a, <16 x i8> %b, <16 x i8> %c, <16 x i1> %1)
+  ret i32 %2
+}
+
+define arm_aapcs_vfpcc i32 @test_vmlsdavaxq_p_s16(i32 %a, <8 x i16> %b, <8 x i16> %c, i16 zeroext %p) {
+; CHECK-LABEL: test_vmlsdavaxq_p_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmlsdavaxt.s16 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call i32 @llvm.arm.mve.vmldava.predicated.v8i16.v8i1(i32 0, i32 1, i32 1, i32 %a, <8 x i16> %b, <8 x i16> %c, <8 x i1> %1)
+  ret i32 %2
+}
+
+define arm_aapcs_vfpcc i32 @test_vmlsdavaxq_p_s32(i32 %a, <4 x i32> %b, <4 x i32> %c, i16 zeroext %p) {
+; CHECK-LABEL: test_vmlsdavaxq_p_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmlsdavaxt.s32 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call i32 @llvm.arm.mve.vmldava.predicated.v4i32.v4i1(i32 0, i32 1, i32 1, i32 %a, <4 x i32> %b, <4 x i32> %c, <4 x i1> %1)
+  ret i32 %2
+}
+
+define arm_aapcs_vfpcc i32 @test_vmladavq_s8(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vmladavq_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmlav.s8 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call i32 @llvm.arm.mve.vmldava.v16i8(i32 0, i32 0, i32 0, i32 0, <16 x i8> %a, <16 x i8> %b)
+  ret i32 %0
+}
+
+define arm_aapcs_vfpcc i32 @test_vmladavq_s16(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vmladavq_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmlav.s16 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call i32 @llvm.arm.mve.vmldava.v8i16(i32 0, i32 0, i32 0, i32 0, <8 x i16> %a, <8 x i16> %b)
+  ret i32 %0
+}
+
+define arm_aapcs_vfpcc i32 @test_vmladavq_s32(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vmladavq_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmlav.s32 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call i32 @llvm.arm.mve.vmldava.v4i32(i32 0, i32 0, i32 0, i32 0, <4 x i32> %a, <4 x i32> %b)
+  ret i32 %0
+}
+
+define arm_aapcs_vfpcc i32 @test_vmladavq_u8(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vmladavq_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmlav.u8 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call i32 @llvm.arm.mve.vmldava.v16i8(i32 1, i32 0, i32 0, i32 0, <16 x i8> %a, <16 x i8> %b)
+  ret i32 %0
+}
+
+define arm_aapcs_vfpcc i32 @test_vmladavq_u16(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vmladavq_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmlav.u16 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call i32 @llvm.arm.mve.vmldava.v8i16(i32 1, i32 0, i32 0, i32 0, <8 x i16> %a, <8 x i16> %b)
+  ret i32 %0
+}
+
+define arm_aapcs_vfpcc i32 @test_vmladavq_u32(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vmladavq_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmlav.u32 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call i32 @llvm.arm.mve.vmldava.v4i32(i32 1, i32 0, i32 0, i32 0, <4 x i32> %a, <4 x i32> %b)
+  ret i32 %0
+}
+
+define arm_aapcs_vfpcc i32 @test_vmladavxq_s8(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vmladavxq_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmladavx.s8 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call i32 @llvm.arm.mve.vmldava.v16i8(i32 0, i32 0, i32 1, i32 0, <16 x i8> %a, <16 x i8> %b)
+  ret i32 %0
+}
+
+define arm_aapcs_vfpcc i32 @test_vmladavxq_s16(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vmladavxq_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmladavx.s16 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call i32 @llvm.arm.mve.vmldava.v8i16(i32 0, i32 0, i32 1, i32 0, <8 x i16> %a, <8 x i16> %b)
+  ret i32 %0
+}
+
+define arm_aapcs_vfpcc i32 @test_vmladavxq_s32(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vmladavxq_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmladavx.s32 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call i32 @llvm.arm.mve.vmldava.v4i32(i32 0, i32 0, i32 1, i32 0, <4 x i32> %a, <4 x i32> %b)
+  ret i32 %0
+}
+
+define arm_aapcs_vfpcc i32 @test_vmlsdavq_s8(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vmlsdavq_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmlsdav.s8 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call i32 @llvm.arm.mve.vmldava.v16i8(i32 0, i32 1, i32 0, i32 0, <16 x i8> %a, <16 x i8> %b)
+  ret i32 %0
+}
+
+define arm_aapcs_vfpcc i32 @test_vmlsdavq_s16(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vmlsdavq_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmlsdav.s16 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call i32 @llvm.arm.mve.vmldava.v8i16(i32 0, i32 1, i32 0, i32 0, <8 x i16> %a, <8 x i16> %b)
+  ret i32 %0
+}
+
+define arm_aapcs_vfpcc i32 @test_vmlsdavq_s32(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vmlsdavq_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmlsdav.s32 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call i32 @llvm.arm.mve.vmldava.v4i32(i32 0, i32 1, i32 0, i32 0, <4 x i32> %a, <4 x i32> %b)
+  ret i32 %0
+}
+
+define arm_aapcs_vfpcc i32 @test_vmlsdavxq_s8(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vmlsdavxq_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmlsdavx.s8 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call i32 @llvm.arm.mve.vmldava.v16i8(i32 0, i32 1, i32 1, i32 0, <16 x i8> %a, <16 x i8> %b)
+  ret i32 %0
+}
+
+define arm_aapcs_vfpcc i32 @test_vmlsdavxq_s16(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vmlsdavxq_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmlsdavx.s16 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call i32 @llvm.arm.mve.vmldava.v8i16(i32 0, i32 1, i32 1, i32 0, <8 x i16> %a, <8 x i16> %b)
+  ret i32 %0
+}
+
+define arm_aapcs_vfpcc i32 @test_vmlsdavxq_s32(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vmlsdavxq_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmlsdavx.s32 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call i32 @llvm.arm.mve.vmldava.v4i32(i32 0, i32 1, i32 1, i32 0, <4 x i32> %a, <4 x i32> %b)
+  ret i32 %0
+}
+
+define arm_aapcs_vfpcc i32 @test_vmladavq_p_s8(<16 x i8> %a, <16 x i8> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vmladavq_p_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmlavt.s8 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = call i32 @llvm.arm.mve.vmldava.predicated.v16i8.v16i1(i32 0, i32 0, i32 0, i32 0, <16 x i8> %a, <16 x i8> %b, <16 x i1> %1)
+  ret i32 %2
+}
+
+define arm_aapcs_vfpcc i32 @test_vmladavq_p_s16(<8 x i16> %a, <8 x i16> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vmladavq_p_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmlavt.s16 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call i32 @llvm.arm.mve.vmldava.predicated.v8i16.v8i1(i32 0, i32 0, i32 0, i32 0, <8 x i16> %a, <8 x i16> %b, <8 x i1> %1)
+  ret i32 %2
+}
+
+define arm_aapcs_vfpcc i32 @test_vmladavq_p_s32(<4 x i32> %a, <4 x i32> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vmladavq_p_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmlavt.s32 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call i32 @llvm.arm.mve.vmldava.predicated.v4i32.v4i1(i32 0, i32 0, i32 0, i32 0, <4 x i32> %a, <4 x i32> %b, <4 x i1> %1)
+  ret i32 %2
+}
+
+define arm_aapcs_vfpcc i32 @test_vmladavq_p_u8(<16 x i8> %a, <16 x i8> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vmladavq_p_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmlavt.u8 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = call i32 @llvm.arm.mve.vmldava.predicated.v16i8.v16i1(i32 1, i32 0, i32 0, i32 0, <16 x i8> %a, <16 x i8> %b, <16 x i1> %1)
+  ret i32 %2
+}
+
+define arm_aapcs_vfpcc i32 @test_vmladavq_p_u16(<8 x i16> %a, <8 x i16> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vmladavq_p_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmlavt.u16 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call i32 @llvm.arm.mve.vmldava.predicated.v8i16.v8i1(i32 1, i32 0, i32 0, i32 0, <8 x i16> %a, <8 x i16> %b, <8 x i1> %1)
+  ret i32 %2
+}
+
+define arm_aapcs_vfpcc i32 @test_vmladavq_p_u32(<4 x i32> %a, <4 x i32> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vmladavq_p_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmlavt.u32 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call i32 @llvm.arm.mve.vmldava.predicated.v4i32.v4i1(i32 1, i32 0, i32 0, i32 0, <4 x i32> %a, <4 x i32> %b, <4 x i1> %1)
+  ret i32 %2
+}
+
+define arm_aapcs_vfpcc i32 @test_vmladavxq_p_s8(<16 x i8> %a, <16 x i8> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vmladavxq_p_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmladavxt.s8 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = call i32 @llvm.arm.mve.vmldava.predicated.v16i8.v16i1(i32 0, i32 0, i32 1, i32 0, <16 x i8> %a, <16 x i8> %b, <16 x i1> %1)
+  ret i32 %2
+}
+
+define arm_aapcs_vfpcc i32 @test_vmladavxq_p_s16(<8 x i16> %a, <8 x i16> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vmladavxq_p_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmladavxt.s16 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call i32 @llvm.arm.mve.vmldava.predicated.v8i16.v8i1(i32 0, i32 0, i32 1, i32 0, <8 x i16> %a, <8 x i16> %b, <8 x i1> %1)
+  ret i32 %2
+}
+
+define arm_aapcs_vfpcc i32 @test_vmladavxq_p_s32(<4 x i32> %a, <4 x i32> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vmladavxq_p_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmladavxt.s32 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call i32 @llvm.arm.mve.vmldava.predicated.v4i32.v4i1(i32 0, i32 0, i32 1, i32 0, <4 x i32> %a, <4 x i32> %b, <4 x i1> %1)
+  ret i32 %2
+}
+
+define arm_aapcs_vfpcc i32 @test_vmlsdavq_p_s8(<16 x i8> %a, <16 x i8> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vmlsdavq_p_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmlsdavt.s8 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = call i32 @llvm.arm.mve.vmldava.predicated.v16i8.v16i1(i32 0, i32 1, i32 0, i32 0, <16 x i8> %a, <16 x i8> %b, <16 x i1> %1)
+  ret i32 %2
+}
+
+define arm_aapcs_vfpcc i32 @test_vmlsdavq_p_s16(<8 x i16> %a, <8 x i16> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vmlsdavq_p_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmlsdavt.s16 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call i32 @llvm.arm.mve.vmldava.predicated.v8i16.v8i1(i32 0, i32 1, i32 0, i32 0, <8 x i16> %a, <8 x i16> %b, <8 x i1> %1)
+  ret i32 %2
+}
+
+define arm_aapcs_vfpcc i32 @test_vmlsdavq_p_s32(<4 x i32> %a, <4 x i32> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vmlsdavq_p_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmlsdavt.s32 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call i32 @llvm.arm.mve.vmldava.predicated.v4i32.v4i1(i32 0, i32 1, i32 0, i32 0, <4 x i32> %a, <4 x i32> %b, <4 x i1> %1)
+  ret i32 %2
+}
+
+define arm_aapcs_vfpcc i32 @test_vmlsdavxq_p_s8(<16 x i8> %a, <16 x i8> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vmlsdavxq_p_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmlsdavxt.s8 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = call i32 @llvm.arm.mve.vmldava.predicated.v16i8.v16i1(i32 0, i32 1, i32 1, i32 0, <16 x i8> %a, <16 x i8> %b, <16 x i1> %1)
+  ret i32 %2
+}
+
+define arm_aapcs_vfpcc i32 @test_vmlsdavxq_p_s16(<8 x i16> %a, <8 x i16> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vmlsdavxq_p_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmlsdavxt.s16 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call i32 @llvm.arm.mve.vmldava.predicated.v8i16.v8i1(i32 0, i32 1, i32 1, i32 0, <8 x i16> %a, <8 x i16> %b, <8 x i1> %1)
+  ret i32 %2
+}
+
+define arm_aapcs_vfpcc i32 @test_vmlsdavxq_p_s32(<4 x i32> %a, <4 x i32> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vmlsdavxq_p_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmlsdavxt.s32 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call i32 @llvm.arm.mve.vmldava.predicated.v4i32.v4i1(i32 0, i32 1, i32 1, i32 0, <4 x i32> %a, <4 x i32> %b, <4 x i1> %1)
+  ret i32 %2
+}
diff --git a/llvm/test/CodeGen/Thumb2/mve-intrinsics/vmlldav.ll b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vmlldav.ll
new file mode 100644
index 000000000..9a215b369
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vmlldav.ll
@@ -0,0 +1,1183 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve.fp -verify-machineinstrs -o - %s | FileCheck %s
+
+declare <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32)
+declare <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32)
+
+declare { i32, i32 } @llvm.arm.mve.vmlldava.v8i16(i32, i32, i32, i32, i32, <8 x i16>, <8 x i16>)
+declare { i32, i32 } @llvm.arm.mve.vmlldava.v4i32(i32, i32, i32, i32, i32, <4 x i32>, <4 x i32>)
+declare { i32, i32 } @llvm.arm.mve.vrmlldavha.v4i32(i32, i32, i32, i32, i32, <4 x i32>, <4 x i32>)
+
+declare { i32, i32 } @llvm.arm.mve.vmlldava.predicated.v8i16.v8i1(i32, i32, i32, i32, i32, <8 x i16>, <8 x i16>, <8 x i1>)
+declare { i32, i32 } @llvm.arm.mve.vmlldava.predicated.v4i32.v4i1(i32, i32, i32, i32, i32, <4 x i32>, <4 x i32>, <4 x i1>)
+declare { i32, i32 } @llvm.arm.mve.vrmlldavha.predicated.v4i32.v4i1(i32, i32, i32, i32, i32, <4 x i32>, <4 x i32>, <4 x i1>)
+
+define arm_aapcs_vfpcc i64 @test_vmlaldavaq_s16(i64 %a, <8 x i16> %b, <8 x i16> %c) {
+; CHECK-LABEL: test_vmlaldavaq_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmlalva.s16 r0, r1, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = lshr i64 %a, 32
+  %1 = trunc i64 %0 to i32
+  %2 = trunc i64 %a to i32
+  %3 = call { i32, i32 } @llvm.arm.mve.vmlldava.v8i16(i32 0, i32 0, i32 0, i32 %2, i32 %1, <8 x i16> %b, <8 x i16> %c)
+  %4 = extractvalue { i32, i32 } %3, 1
+  %5 = zext i32 %4 to i64
+  %6 = shl i64 %5, 32
+  %7 = extractvalue { i32, i32 } %3, 0
+  %8 = zext i32 %7 to i64
+  %9 = or i64 %6, %8
+  ret i64 %9
+}
+
+define arm_aapcs_vfpcc i64 @test_vmlaldavaq_s32(i64 %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LABEL: test_vmlaldavaq_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmlalva.s32 r0, r1, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = lshr i64 %a, 32
+  %1 = trunc i64 %0 to i32
+  %2 = trunc i64 %a to i32
+  %3 = call { i32, i32 } @llvm.arm.mve.vmlldava.v4i32(i32 0, i32 0, i32 0, i32 %2, i32 %1, <4 x i32> %b, <4 x i32> %c)
+  %4 = extractvalue { i32, i32 } %3, 1
+  %5 = zext i32 %4 to i64
+  %6 = shl i64 %5, 32
+  %7 = extractvalue { i32, i32 } %3, 0
+  %8 = zext i32 %7 to i64
+  %9 = or i64 %6, %8
+  ret i64 %9
+}
+
+define arm_aapcs_vfpcc i64 @test_vmlaldavaq_u16(i64 %a, <8 x i16> %b, <8 x i16> %c) {
+; CHECK-LABEL: test_vmlaldavaq_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmlalva.u16 r0, r1, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = lshr i64 %a, 32
+  %1 = trunc i64 %0 to i32
+  %2 = trunc i64 %a to i32
+  %3 = call { i32, i32 } @llvm.arm.mve.vmlldava.v8i16(i32 1, i32 0, i32 0, i32 %2, i32 %1, <8 x i16> %b, <8 x i16> %c)
+  %4 = extractvalue { i32, i32 } %3, 1
+  %5 = zext i32 %4 to i64
+  %6 = shl i64 %5, 32
+  %7 = extractvalue { i32, i32 } %3, 0
+  %8 = zext i32 %7 to i64
+  %9 = or i64 %6, %8
+  ret i64 %9
+}
+
+define arm_aapcs_vfpcc i64 @test_vmlaldavaq_u32(i64 %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LABEL: test_vmlaldavaq_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmlalva.u32 r0, r1, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = lshr i64 %a, 32
+  %1 = trunc i64 %0 to i32
+  %2 = trunc i64 %a to i32
+  %3 = call { i32, i32 } @llvm.arm.mve.vmlldava.v4i32(i32 1, i32 0, i32 0, i32 %2, i32 %1, <4 x i32> %b, <4 x i32> %c)
+  %4 = extractvalue { i32, i32 } %3, 1
+  %5 = zext i32 %4 to i64
+  %6 = shl i64 %5, 32
+  %7 = extractvalue { i32, i32 } %3, 0
+  %8 = zext i32 %7 to i64
+  %9 = or i64 %6, %8
+  ret i64 %9
+}
+
+define arm_aapcs_vfpcc i64 @test_vmlaldavaxq_s16(i64 %a, <8 x i16> %b, <8 x i16> %c) {
+; CHECK-LABEL: test_vmlaldavaxq_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmlaldavax.s16 r0, r1, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = lshr i64 %a, 32
+  %1 = trunc i64 %0 to i32
+  %2 = trunc i64 %a to i32
+  %3 = call { i32, i32 } @llvm.arm.mve.vmlldava.v8i16(i32 0, i32 0, i32 1, i32 %2, i32 %1, <8 x i16> %b, <8 x i16> %c)
+  %4 = extractvalue { i32, i32 } %3, 1
+  %5 = zext i32 %4 to i64
+  %6 = shl i64 %5, 32
+  %7 = extractvalue { i32, i32 } %3, 0
+  %8 = zext i32 %7 to i64
+  %9 = or i64 %6, %8
+  ret i64 %9
+}
+
+define arm_aapcs_vfpcc i64 @test_vmlaldavaxq_s32(i64 %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LABEL: test_vmlaldavaxq_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmlaldavax.s32 r0, r1, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = lshr i64 %a, 32
+  %1 = trunc i64 %0 to i32
+  %2 = trunc i64 %a to i32
+  %3 = call { i32, i32 } @llvm.arm.mve.vmlldava.v4i32(i32 0, i32 0, i32 1, i32 %2, i32 %1, <4 x i32> %b, <4 x i32> %c)
+  %4 = extractvalue { i32, i32 } %3, 1
+  %5 = zext i32 %4 to i64
+  %6 = shl i64 %5, 32
+  %7 = extractvalue { i32, i32 } %3, 0
+  %8 = zext i32 %7 to i64
+  %9 = or i64 %6, %8
+  ret i64 %9
+}
+
+define arm_aapcs_vfpcc i64 @test_vmlsldavaq_s16(i64 %a, <8 x i16> %b, <8 x i16> %c) {
+; CHECK-LABEL: test_vmlsldavaq_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmlsldava.s16 r0, r1, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = lshr i64 %a, 32
+  %1 = trunc i64 %0 to i32
+  %2 = trunc i64 %a to i32
+  %3 = call { i32, i32 } @llvm.arm.mve.vmlldava.v8i16(i32 0, i32 1, i32 0, i32 %2, i32 %1, <8 x i16> %b, <8 x i16> %c)
+  %4 = extractvalue { i32, i32 } %3, 1
+  %5 = zext i32 %4 to i64
+  %6 = shl i64 %5, 32
+  %7 = extractvalue { i32, i32 } %3, 0
+  %8 = zext i32 %7 to i64
+  %9 = or i64 %6, %8
+  ret i64 %9
+}
+
+define arm_aapcs_vfpcc i64 @test_vmlsldavaq_s32(i64 %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LABEL: test_vmlsldavaq_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmlsldava.s32 r0, r1, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = lshr i64 %a, 32
+  %1 = trunc i64 %0 to i32
+  %2 = trunc i64 %a to i32
+  %3 = call { i32, i32 } @llvm.arm.mve.vmlldava.v4i32(i32 0, i32 1, i32 0, i32 %2, i32 %1, <4 x i32> %b, <4 x i32> %c)
+  %4 = extractvalue { i32, i32 } %3, 1
+  %5 = zext i32 %4 to i64
+  %6 = shl i64 %5, 32
+  %7 = extractvalue { i32, i32 } %3, 0
+  %8 = zext i32 %7 to i64
+  %9 = or i64 %6, %8
+  ret i64 %9
+}
+
+define arm_aapcs_vfpcc i64 @test_vmlsldaxvaq_s16(i64 %a, <8 x i16> %b, <8 x i16> %c) {
+; CHECK-LABEL: test_vmlsldaxvaq_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmlsldavax.s16 r0, r1, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = lshr i64 %a, 32
+  %1 = trunc i64 %0 to i32
+  %2 = trunc i64 %a to i32
+  %3 = call { i32, i32 } @llvm.arm.mve.vmlldava.v8i16(i32 0, i32 1, i32 1, i32 %2, i32 %1, <8 x i16> %b, <8 x i16> %c)
+  %4 = extractvalue { i32, i32 } %3, 1
+  %5 = zext i32 %4 to i64
+  %6 = shl i64 %5, 32
+  %7 = extractvalue { i32, i32 } %3, 0
+  %8 = zext i32 %7 to i64
+  %9 = or i64 %6, %8
+  ret i64 %9
+}
+
+define arm_aapcs_vfpcc i64 @test_vmlsldavaxq_s32(i64 %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LABEL: test_vmlsldavaxq_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmlsldavax.s32 r0, r1, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = lshr i64 %a, 32
+  %1 = trunc i64 %0 to i32
+  %2 = trunc i64 %a to i32
+  %3 = call { i32, i32 } @llvm.arm.mve.vmlldava.v4i32(i32 0, i32 1, i32 1, i32 %2, i32 %1, <4 x i32> %b, <4 x i32> %c)
+  %4 = extractvalue { i32, i32 } %3, 1
+  %5 = zext i32 %4 to i64
+  %6 = shl i64 %5, 32
+  %7 = extractvalue { i32, i32 } %3, 0
+  %8 = zext i32 %7 to i64
+  %9 = or i64 %6, %8
+  ret i64 %9
+}
+
+define arm_aapcs_vfpcc i64 @test_vrmlaldavhaq_s32(i64 %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LABEL: test_vrmlaldavhaq_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vrmlalvha.s32 r0, r1, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = lshr i64 %a, 32
+  %1 = trunc i64 %0 to i32
+  %2 = trunc i64 %a to i32
+  %3 = call { i32, i32 } @llvm.arm.mve.vrmlldavha.v4i32(i32 0, i32 0, i32 0, i32 %2, i32 %1, <4 x i32> %b, <4 x i32> %c)
+  %4 = extractvalue { i32, i32 } %3, 1
+  %5 = zext i32 %4 to i64
+  %6 = shl i64 %5, 32
+  %7 = extractvalue { i32, i32 } %3, 0
+  %8 = zext i32 %7 to i64
+  %9 = or i64 %6, %8
+  ret i64 %9
+}
+
+define arm_aapcs_vfpcc i64 @test_vrmlaldavhaq_u32(i64 %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LABEL: test_vrmlaldavhaq_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vrmlalvha.u32 r0, r1, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = lshr i64 %a, 32
+  %1 = trunc i64 %0 to i32
+  %2 = trunc i64 %a to i32
+  %3 = call { i32, i32 } @llvm.arm.mve.vrmlldavha.v4i32(i32 1, i32 0, i32 0, i32 %2, i32 %1, <4 x i32> %b, <4 x i32> %c)
+  %4 = extractvalue { i32, i32 } %3, 1
+  %5 = zext i32 %4 to i64
+  %6 = shl i64 %5, 32
+  %7 = extractvalue { i32, i32 } %3, 0
+  %8 = zext i32 %7 to i64
+  %9 = or i64 %6, %8
+  ret i64 %9
+}
+
+define arm_aapcs_vfpcc i64 @test_vrmlaldavhaxq_s32(i64 %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LABEL: test_vrmlaldavhaxq_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vrmlaldavhax.s32 r0, r1, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = lshr i64 %a, 32
+  %1 = trunc i64 %0 to i32
+  %2 = trunc i64 %a to i32
+  %3 = call { i32, i32 } @llvm.arm.mve.vrmlldavha.v4i32(i32 0, i32 0, i32 1, i32 %2, i32 %1, <4 x i32> %b, <4 x i32> %c)
+  %4 = extractvalue { i32, i32 } %3, 1
+  %5 = zext i32 %4 to i64
+  %6 = shl i64 %5, 32
+  %7 = extractvalue { i32, i32 } %3, 0
+  %8 = zext i32 %7 to i64
+  %9 = or i64 %6, %8
+  ret i64 %9
+}
+
+define arm_aapcs_vfpcc i64 @test_vrmlsldavhaq_s32(i64 %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LABEL: test_vrmlsldavhaq_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vrmlsldavha.s32 r0, r1, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = lshr i64 %a, 32
+  %1 = trunc i64 %0 to i32
+  %2 = trunc i64 %a to i32
+  %3 = call { i32, i32 } @llvm.arm.mve.vrmlldavha.v4i32(i32 0, i32 1, i32 0, i32 %2, i32 %1, <4 x i32> %b, <4 x i32> %c)
+  %4 = extractvalue { i32, i32 } %3, 1
+  %5 = zext i32 %4 to i64
+  %6 = shl i64 %5, 32
+  %7 = extractvalue { i32, i32 } %3, 0
+  %8 = zext i32 %7 to i64
+  %9 = or i64 %6, %8
+  ret i64 %9
+}
+
+define arm_aapcs_vfpcc i64 @test_vrmlsldavhaxq_s32(i64 %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LABEL: test_vrmlsldavhaxq_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vrmlsldavhax.s32 r0, r1, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = lshr i64 %a, 32
+  %1 = trunc i64 %0 to i32
+  %2 = trunc i64 %a to i32
+  %3 = call { i32, i32 } @llvm.arm.mve.vrmlldavha.v4i32(i32 0, i32 1, i32 1, i32 %2, i32 %1, <4 x i32> %b, <4 x i32> %c)
+  %4 = extractvalue { i32, i32 } %3, 1
+  %5 = zext i32 %4 to i64
+  %6 = shl i64 %5, 32
+  %7 = extractvalue { i32, i32 } %3, 0
+  %8 = zext i32 %7 to i64
+  %9 = or i64 %6, %8
+  ret i64 %9
+}
+
+define arm_aapcs_vfpcc i64 @test_vmlaldavaq_p_s16(i64 %a, <8 x i16> %b, <8 x i16> %c, i16 zeroext %p) {
+; CHECK-LABEL: test_vmlaldavaq_p_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r2
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmlalvat.s16 r0, r1, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = lshr i64 %a, 32
+  %1 = trunc i64 %0 to i32
+  %2 = trunc i64 %a to i32
+  %3 = zext i16 %p to i32
+  %4 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %3)
+  %5 = call { i32, i32 } @llvm.arm.mve.vmlldava.predicated.v8i16.v8i1(i32 0, i32 0, i32 0, i32 %2, i32 %1, <8 x i16> %b, <8 x i16> %c, <8 x i1> %4)
+  %6 = extractvalue { i32, i32 } %5, 1
+  %7 = zext i32 %6 to i64
+  %8 = shl i64 %7, 32
+  %9 = extractvalue { i32, i32 } %5, 0
+  %10 = zext i32 %9 to i64
+  %11 = or i64 %8, %10
+  ret i64 %11
+}
+
+define arm_aapcs_vfpcc i64 @test_vmlaldavaq_p_s32(i64 %a, <4 x i32> %b, <4 x i32> %c, i16 zeroext %p) {
+; CHECK-LABEL: test_vmlaldavaq_p_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r2
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmlalvat.s32 r0, r1, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = lshr i64 %a, 32
+  %1 = trunc i64 %0 to i32
+  %2 = trunc i64 %a to i32
+  %3 = zext i16 %p to i32
+  %4 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %3)
+  %5 = call { i32, i32 } @llvm.arm.mve.vmlldava.predicated.v4i32.v4i1(i32 0, i32 0, i32 0, i32 %2, i32 %1, <4 x i32> %b, <4 x i32> %c, <4 x i1> %4)
+  %6 = extractvalue { i32, i32 } %5, 1
+  %7 = zext i32 %6 to i64
+  %8 = shl i64 %7, 32
+  %9 = extractvalue { i32, i32 } %5, 0
+  %10 = zext i32 %9 to i64
+  %11 = or i64 %8, %10
+  ret i64 %11
+}
+
+define arm_aapcs_vfpcc i64 @test_vmlaldavaq_p_u16(i64 %a, <8 x i16> %b, <8 x i16> %c, i16 zeroext %p) {
+; CHECK-LABEL: test_vmlaldavaq_p_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r2
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmlalvat.u16 r0, r1, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = lshr i64 %a, 32
+  %1 = trunc i64 %0 to i32
+  %2 = trunc i64 %a to i32
+  %3 = zext i16 %p to i32
+  %4 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %3)
+  %5 = call { i32, i32 } @llvm.arm.mve.vmlldava.predicated.v8i16.v8i1(i32 1, i32 0, i32 0, i32 %2, i32 %1, <8 x i16> %b, <8 x i16> %c, <8 x i1> %4)
+  %6 = extractvalue { i32, i32 } %5, 1
+  %7 = zext i32 %6 to i64
+  %8 = shl i64 %7, 32
+  %9 = extractvalue { i32, i32 } %5, 0
+  %10 = zext i32 %9 to i64
+  %11 = or i64 %8, %10
+  ret i64 %11
+}
+
+define arm_aapcs_vfpcc i64 @test_vmlaldavaq_p_u32(i64 %a, <4 x i32> %b, <4 x i32> %c, i16 zeroext %p) {
+; CHECK-LABEL: test_vmlaldavaq_p_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r2
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmlalvat.u32 r0, r1, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = lshr i64 %a, 32
+  %1 = trunc i64 %0 to i32
+  %2 = trunc i64 %a to i32
+  %3 = zext i16 %p to i32
+  %4 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %3)
+  %5 = call { i32, i32 } @llvm.arm.mve.vmlldava.predicated.v4i32.v4i1(i32 1, i32 0, i32 0, i32 %2, i32 %1, <4 x i32> %b, <4 x i32> %c, <4 x i1> %4)
+  %6 = extractvalue { i32, i32 } %5, 1
+  %7 = zext i32 %6 to i64
+  %8 = shl i64 %7, 32
+  %9 = extractvalue { i32, i32 } %5, 0
+  %10 = zext i32 %9 to i64
+  %11 = or i64 %8, %10
+  ret i64 %11
+}
+
+define arm_aapcs_vfpcc i64 @test_vmlaldavaxq_p_s16(i64 %a, <8 x i16> %b, <8 x i16> %c, i16 zeroext %p) {
+; CHECK-LABEL: test_vmlaldavaxq_p_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r2
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmlaldavaxt.s16 r0, r1, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = lshr i64 %a, 32
+  %1 = trunc i64 %0 to i32
+  %2 = trunc i64 %a to i32
+  %3 = zext i16 %p to i32
+  %4 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %3)
+  %5 = call { i32, i32 } @llvm.arm.mve.vmlldava.predicated.v8i16.v8i1(i32 0, i32 0, i32 1, i32 %2, i32 %1, <8 x i16> %b, <8 x i16> %c, <8 x i1> %4)
+  %6 = extractvalue { i32, i32 } %5, 1
+  %7 = zext i32 %6 to i64
+  %8 = shl i64 %7, 32
+  %9 = extractvalue { i32, i32 } %5, 0
+  %10 = zext i32 %9 to i64
+  %11 = or i64 %8, %10
+  ret i64 %11
+}
+
+define arm_aapcs_vfpcc i64 @test_vmlaldavaxq_p_s32(i64 %a, <4 x i32> %b, <4 x i32> %c, i16 zeroext %p) {
+; CHECK-LABEL: test_vmlaldavaxq_p_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r2
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmlaldavaxt.s32 r0, r1, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = lshr i64 %a, 32
+  %1 = trunc i64 %0 to i32
+  %2 = trunc i64 %a to i32
+  %3 = zext i16 %p to i32
+  %4 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %3)
+  %5 = call { i32, i32 } @llvm.arm.mve.vmlldava.predicated.v4i32.v4i1(i32 0, i32 0, i32 1, i32 %2, i32 %1, <4 x i32> %b, <4 x i32> %c, <4 x i1> %4)
+  %6 = extractvalue { i32, i32 } %5, 1
+  %7 = zext i32 %6 to i64
+  %8 = shl i64 %7, 32
+  %9 = extractvalue { i32, i32 } %5, 0
+  %10 = zext i32 %9 to i64
+  %11 = or i64 %8, %10
+  ret i64 %11
+}
+
+define arm_aapcs_vfpcc i64 @test_vmlsldavaq_p_s16(i64 %a, <8 x i16> %b, <8 x i16> %c, i16 zeroext %p) {
+; CHECK-LABEL: test_vmlsldavaq_p_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r2
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmlsldavat.s16 r0, r1, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = lshr i64 %a, 32
+  %1 = trunc i64 %0 to i32
+  %2 = trunc i64 %a to i32
+  %3 = zext i16 %p to i32
+  %4 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %3)
+  %5 = call { i32, i32 } @llvm.arm.mve.vmlldava.predicated.v8i16.v8i1(i32 0, i32 1, i32 0, i32 %2, i32 %1, <8 x i16> %b, <8 x i16> %c, <8 x i1> %4)
+  %6 = extractvalue { i32, i32 } %5, 1
+  %7 = zext i32 %6 to i64
+  %8 = shl i64 %7, 32
+  %9 = extractvalue { i32, i32 } %5, 0
+  %10 = zext i32 %9 to i64
+  %11 = or i64 %8, %10
+  ret i64 %11
+}
+
+define arm_aapcs_vfpcc i64 @test_vmlsldavaq_p_s32(i64 %a, <4 x i32> %b, <4 x i32> %c, i16 zeroext %p) {
+; CHECK-LABEL: test_vmlsldavaq_p_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r2
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmlsldavat.s32 r0, r1, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = lshr i64 %a, 32
+  %1 = trunc i64 %0 to i32
+  %2 = trunc i64 %a to i32
+  %3 = zext i16 %p to i32
+  %4 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %3)
+  %5 = call { i32, i32 } @llvm.arm.mve.vmlldava.predicated.v4i32.v4i1(i32 0, i32 1, i32 0, i32 %2, i32 %1, <4 x i32> %b, <4 x i32> %c, <4 x i1> %4)
+  %6 = extractvalue { i32, i32 } %5, 1
+  %7 = zext i32 %6 to i64
+  %8 = shl i64 %7, 32
+  %9 = extractvalue { i32, i32 } %5, 0
+  %10 = zext i32 %9 to i64
+  %11 = or i64 %8, %10
+  ret i64 %11
+}
+
+define arm_aapcs_vfpcc i64 @test_vmlsldaxvaq_p_s16(i64 %a, <8 x i16> %b, <8 x i16> %c, i16 zeroext %p) {
+; CHECK-LABEL: test_vmlsldaxvaq_p_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r2
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmlsldavaxt.s16 r0, r1, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = lshr i64 %a, 32
+  %1 = trunc i64 %0 to i32
+  %2 = trunc i64 %a to i32
+  %3 = zext i16 %p to i32
+  %4 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %3)
+  %5 = call { i32, i32 } @llvm.arm.mve.vmlldava.predicated.v8i16.v8i1(i32 0, i32 1, i32 1, i32 %2, i32 %1, <8 x i16> %b, <8 x i16> %c, <8 x i1> %4)
+  %6 = extractvalue { i32, i32 } %5, 1
+  %7 = zext i32 %6 to i64
+  %8 = shl i64 %7, 32
+  %9 = extractvalue { i32, i32 } %5, 0
+  %10 = zext i32 %9 to i64
+  %11 = or i64 %8, %10
+  ret i64 %11
+}
+
+define arm_aapcs_vfpcc i64 @test_vmlsldavaxq_p_s32(i64 %a, <4 x i32> %b, <4 x i32> %c, i16 zeroext %p) {
+; CHECK-LABEL: test_vmlsldavaxq_p_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r2
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmlsldavaxt.s32 r0, r1, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = lshr i64 %a, 32
+  %1 = trunc i64 %0 to i32
+  %2 = trunc i64 %a to i32
+  %3 = zext i16 %p to i32
+  %4 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %3)
+  %5 = call { i32, i32 } @llvm.arm.mve.vmlldava.predicated.v4i32.v4i1(i32 0, i32 1, i32 1, i32 %2, i32 %1, <4 x i32> %b, <4 x i32> %c, <4 x i1> %4)
+  %6 = extractvalue { i32, i32 } %5, 1
+  %7 = zext i32 %6 to i64
+  %8 = shl i64 %7, 32
+  %9 = extractvalue { i32, i32 } %5, 0
+  %10 = zext i32 %9 to i64
+  %11 = or i64 %8, %10
+  ret i64 %11
+}
+
+define arm_aapcs_vfpcc i64 @test_vrmlaldavhaq_p_s32(i64 %a, <4 x i32> %b, <4 x i32> %c, i16 zeroext %p) {
+; CHECK-LABEL: test_vrmlaldavhaq_p_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r2
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vrmlalvhat.s32 r0, r1, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = lshr i64 %a, 32
+  %1 = trunc i64 %0 to i32
+  %2 = trunc i64 %a to i32
+  %3 = zext i16 %p to i32
+  %4 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %3)
+  %5 = call { i32, i32 } @llvm.arm.mve.vrmlldavha.predicated.v4i32.v4i1(i32 0, i32 0, i32 0, i32 %2, i32 %1, <4 x i32> %b, <4 x i32> %c, <4 x i1> %4)
+  %6 = extractvalue { i32, i32 } %5, 1
+  %7 = zext i32 %6 to i64
+  %8 = shl i64 %7, 32
+  %9 = extractvalue { i32, i32 } %5, 0
+  %10 = zext i32 %9 to i64
+  %11 = or i64 %8, %10
+  ret i64 %11
+}
+
+define arm_aapcs_vfpcc i64 @test_vrmlaldavhaq_p_u32(i64 %a, <4 x i32> %b, <4 x i32> %c, i16 zeroext %p) {
+; CHECK-LABEL: test_vrmlaldavhaq_p_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r2
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vrmlalvhat.u32 r0, r1, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = lshr i64 %a, 32
+  %1 = trunc i64 %0 to i32
+  %2 = trunc i64 %a to i32
+  %3 = zext i16 %p to i32
+  %4 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %3)
+  %5 = call { i32, i32 } @llvm.arm.mve.vrmlldavha.predicated.v4i32.v4i1(i32 1, i32 0, i32 0, i32 %2, i32 %1, <4 x i32> %b, <4 x i32> %c, <4 x i1> %4)
+  %6 = extractvalue { i32, i32 } %5, 1
+  %7 = zext i32 %6 to i64
+  %8 = shl i64 %7, 32
+  %9 = extractvalue { i32, i32 } %5, 0
+  %10 = zext i32 %9 to i64
+  %11 = or i64 %8, %10
+  ret i64 %11
+}
+
+define arm_aapcs_vfpcc i64 @test_vrmlaldavhaxq_p_s32(i64 %a, <4 x i32> %b, <4 x i32> %c, i16 zeroext %p) {
+; CHECK-LABEL: test_vrmlaldavhaxq_p_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r2
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vrmlaldavhaxt.s32 r0, r1, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = lshr i64 %a, 32
+  %1 = trunc i64 %0 to i32
+  %2 = trunc i64 %a to i32
+  %3 = zext i16 %p to i32
+  %4 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %3)
+  %5 = call { i32, i32 } @llvm.arm.mve.vrmlldavha.predicated.v4i32.v4i1(i32 0, i32 0, i32 1, i32 %2, i32 %1, <4 x i32> %b, <4 x i32> %c, <4 x i1> %4)
+  %6 = extractvalue { i32, i32 } %5, 1
+  %7 = zext i32 %6 to i64
+  %8 = shl i64 %7, 32
+  %9 = extractvalue { i32, i32 } %5, 0
+  %10 = zext i32 %9 to i64
+  %11 = or i64 %8, %10
+  ret i64 %11
+}
+
+define arm_aapcs_vfpcc i64 @test_vrmlsldavhaq_p_s32(i64 %a, <4 x i32> %b, <4 x i32> %c, i16 zeroext %p) {
+; CHECK-LABEL: test_vrmlsldavhaq_p_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r2
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vrmlsldavhat.s32 r0, r1, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = lshr i64 %a, 32
+  %1 = trunc i64 %0 to i32
+  %2 = trunc i64 %a to i32
+  %3 = zext i16 %p to i32
+  %4 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %3)
+  %5 = call { i32, i32 } @llvm.arm.mve.vrmlldavha.predicated.v4i32.v4i1(i32 0, i32 1, i32 0, i32 %2, i32 %1, <4 x i32> %b, <4 x i32> %c, <4 x i1> %4)
+  %6 = extractvalue { i32, i32 } %5, 1
+  %7 = zext i32 %6 to i64
+  %8 = shl i64 %7, 32
+  %9 = extractvalue { i32, i32 } %5, 0
+  %10 = zext i32 %9 to i64
+  %11 = or i64 %8, %10
+  ret i64 %11
+}
+
+define arm_aapcs_vfpcc i64 @test_vrmlsldavhaxq_p_s32(i64 %a, <4 x i32> %b, <4 x i32> %c, i16 zeroext %p) {
+; CHECK-LABEL: test_vrmlsldavhaxq_p_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r2
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vrmlsldavhaxt.s32 r0, r1, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = lshr i64 %a, 32
+  %1 = trunc i64 %0 to i32
+  %2 = trunc i64 %a to i32
+  %3 = zext i16 %p to i32
+  %4 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %3)
+  %5 = call { i32, i32 } @llvm.arm.mve.vrmlldavha.predicated.v4i32.v4i1(i32 0, i32 1, i32 1, i32 %2, i32 %1, <4 x i32> %b, <4 x i32> %c, <4 x i1> %4)
+  %6 = extractvalue { i32, i32 } %5, 1
+  %7 = zext i32 %6 to i64
+  %8 = shl i64 %7, 32
+  %9 = extractvalue { i32, i32 } %5, 0
+  %10 = zext i32 %9 to i64
+  %11 = or i64 %8, %10
+  ret i64 %11
+}
+
+define arm_aapcs_vfpcc i64 @test_vmlaldavq_s16(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vmlaldavq_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmlalv.s16 r0, r1, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call { i32, i32 } @llvm.arm.mve.vmlldava.v8i16(i32 0, i32 0, i32 0, i32 0, i32 0, <8 x i16> %a, <8 x i16> %b)
+  %1 = extractvalue { i32, i32 } %0, 1
+  %2 = zext i32 %1 to i64
+  %3 = shl i64 %2, 32
+  %4 = extractvalue { i32, i32 } %0, 0
+  %5 = zext i32 %4 to i64
+  %6 = or i64 %3, %5
+  ret i64 %6
+}
+
+define arm_aapcs_vfpcc i64 @test_vmlaldavq_s32(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vmlaldavq_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmlalv.s32 r0, r1, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call { i32, i32 } @llvm.arm.mve.vmlldava.v4i32(i32 0, i32 0, i32 0, i32 0, i32 0, <4 x i32> %a, <4 x i32> %b)
+  %1 = extractvalue { i32, i32 } %0, 1
+  %2 = zext i32 %1 to i64
+  %3 = shl i64 %2, 32
+  %4 = extractvalue { i32, i32 } %0, 0
+  %5 = zext i32 %4 to i64
+  %6 = or i64 %3, %5
+  ret i64 %6
+}
+
+define arm_aapcs_vfpcc i64 @test_vmlaldavq_u16(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vmlaldavq_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmlalv.u16 r0, r1, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call { i32, i32 } @llvm.arm.mve.vmlldava.v8i16(i32 1, i32 0, i32 0, i32 0, i32 0, <8 x i16> %a, <8 x i16> %b)
+  %1 = extractvalue { i32, i32 } %0, 1
+  %2 = zext i32 %1 to i64
+  %3 = shl i64 %2, 32
+  %4 = extractvalue { i32, i32 } %0, 0
+  %5 = zext i32 %4 to i64
+  %6 = or i64 %3, %5
+  ret i64 %6
+}
+
+define arm_aapcs_vfpcc i64 @test_vmlaldavq_u32(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vmlaldavq_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmlalv.u32 r0, r1, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call { i32, i32 } @llvm.arm.mve.vmlldava.v4i32(i32 1, i32 0, i32 0, i32 0, i32 0, <4 x i32> %a, <4 x i32> %b)
+  %1 = extractvalue { i32, i32 } %0, 1
+  %2 = zext i32 %1 to i64
+  %3 = shl i64 %2, 32
+  %4 = extractvalue { i32, i32 } %0, 0
+  %5 = zext i32 %4 to i64
+  %6 = or i64 %3, %5
+  ret i64 %6
+}
+
+define arm_aapcs_vfpcc i64 @test_vmlaldavxq_s16(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vmlaldavxq_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmlaldavx.s16 r0, r1, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call { i32, i32 } @llvm.arm.mve.vmlldava.v8i16(i32 0, i32 0, i32 1, i32 0, i32 0, <8 x i16> %a, <8 x i16> %b)
+  %1 = extractvalue { i32, i32 } %0, 1
+  %2 = zext i32 %1 to i64
+  %3 = shl i64 %2, 32
+  %4 = extractvalue { i32, i32 } %0, 0
+  %5 = zext i32 %4 to i64
+  %6 = or i64 %3, %5
+  ret i64 %6
+}
+
+define arm_aapcs_vfpcc i64 @test_vmlaldavxq_s32(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vmlaldavxq_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmlaldavx.s32 r0, r1, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call { i32, i32 } @llvm.arm.mve.vmlldava.v4i32(i32 0, i32 0, i32 1, i32 0, i32 0, <4 x i32> %a, <4 x i32> %b)
+  %1 = extractvalue { i32, i32 } %0, 1
+  %2 = zext i32 %1 to i64
+  %3 = shl i64 %2, 32
+  %4 = extractvalue { i32, i32 } %0, 0
+  %5 = zext i32 %4 to i64
+  %6 = or i64 %3, %5
+  ret i64 %6
+}
+
+define arm_aapcs_vfpcc i64 @test_vmlsldavq_s16(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vmlsldavq_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmlsldav.s16 r0, r1, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call { i32, i32 } @llvm.arm.mve.vmlldava.v8i16(i32 0, i32 1, i32 0, i32 0, i32 0, <8 x i16> %a, <8 x i16> %b)
+  %1 = extractvalue { i32, i32 } %0, 1
+  %2 = zext i32 %1 to i64
+  %3 = shl i64 %2, 32
+  %4 = extractvalue { i32, i32 } %0, 0
+  %5 = zext i32 %4 to i64
+  %6 = or i64 %3, %5
+  ret i64 %6
+}
+
+define arm_aapcs_vfpcc i64 @test_vmlsldavq_s32(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vmlsldavq_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmlsldav.s32 r0, r1, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call { i32, i32 } @llvm.arm.mve.vmlldava.v4i32(i32 0, i32 1, i32 0, i32 0, i32 0, <4 x i32> %a, <4 x i32> %b)
+  %1 = extractvalue { i32, i32 } %0, 1
+  %2 = zext i32 %1 to i64
+  %3 = shl i64 %2, 32
+  %4 = extractvalue { i32, i32 } %0, 0
+  %5 = zext i32 %4 to i64
+  %6 = or i64 %3, %5
+  ret i64 %6
+}
+
+define arm_aapcs_vfpcc i64 @test_vmlsldavxvq_s16(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vmlsldavxvq_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmlsldavx.s16 r0, r1, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call { i32, i32 } @llvm.arm.mve.vmlldava.v8i16(i32 0, i32 1, i32 1, i32 0, i32 0, <8 x i16> %a, <8 x i16> %b)
+  %1 = extractvalue { i32, i32 } %0, 1
+  %2 = zext i32 %1 to i64
+  %3 = shl i64 %2, 32
+  %4 = extractvalue { i32, i32 } %0, 0
+  %5 = zext i32 %4 to i64
+  %6 = or i64 %3, %5
+  ret i64 %6
+}
+
+define arm_aapcs_vfpcc i64 @test_vmlsldavxq_s32(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vmlsldavxq_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmlsldavx.s32 r0, r1, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call { i32, i32 } @llvm.arm.mve.vmlldava.v4i32(i32 0, i32 1, i32 1, i32 0, i32 0, <4 x i32> %a, <4 x i32> %b)
+  %1 = extractvalue { i32, i32 } %0, 1
+  %2 = zext i32 %1 to i64
+  %3 = shl i64 %2, 32
+  %4 = extractvalue { i32, i32 } %0, 0
+  %5 = zext i32 %4 to i64
+  %6 = or i64 %3, %5
+  ret i64 %6
+}
+
+define arm_aapcs_vfpcc i64 @test_vrmlaldavhq_s32(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vrmlaldavhq_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vrmlalvh.s32 r0, r1, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call { i32, i32 } @llvm.arm.mve.vrmlldavha.v4i32(i32 0, i32 0, i32 0, i32 0, i32 0, <4 x i32> %a, <4 x i32> %b)
+  %1 = extractvalue { i32, i32 } %0, 1
+  %2 = zext i32 %1 to i64
+  %3 = shl i64 %2, 32
+  %4 = extractvalue { i32, i32 } %0, 0
+  %5 = zext i32 %4 to i64
+  %6 = or i64 %3, %5
+  ret i64 %6
+}
+
+define arm_aapcs_vfpcc i64 @test_vrmlaldavhq_u32(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vrmlaldavhq_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vrmlalvh.u32 r0, r1, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call { i32, i32 } @llvm.arm.mve.vrmlldavha.v4i32(i32 1, i32 0, i32 0, i32 0, i32 0, <4 x i32> %a, <4 x i32> %b)
+  %1 = extractvalue { i32, i32 } %0, 1
+  %2 = zext i32 %1 to i64
+  %3 = shl i64 %2, 32
+  %4 = extractvalue { i32, i32 } %0, 0
+  %5 = zext i32 %4 to i64
+  %6 = or i64 %3, %5
+  ret i64 %6
+}
+
+define arm_aapcs_vfpcc i64 @test_vrmlaldavhxq_s32(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vrmlaldavhxq_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vrmlaldavhx.s32 r0, r1, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call { i32, i32 } @llvm.arm.mve.vrmlldavha.v4i32(i32 0, i32 0, i32 1, i32 0, i32 0, <4 x i32> %a, <4 x i32> %b)
+  %1 = extractvalue { i32, i32 } %0, 1
+  %2 = zext i32 %1 to i64
+  %3 = shl i64 %2, 32
+  %4 = extractvalue { i32, i32 } %0, 0
+  %5 = zext i32 %4 to i64
+  %6 = or i64 %3, %5
+  ret i64 %6
+}
+
+define arm_aapcs_vfpcc i64 @test_vrmlsldavhq_s32(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vrmlsldavhq_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vrmlsldavh.s32 r0, r1, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call { i32, i32 } @llvm.arm.mve.vrmlldavha.v4i32(i32 0, i32 1, i32 0, i32 0, i32 0, <4 x i32> %a, <4 x i32> %b)
+  %1 = extractvalue { i32, i32 } %0, 1
+  %2 = zext i32 %1 to i64
+  %3 = shl i64 %2, 32
+  %4 = extractvalue { i32, i32 } %0, 0
+  %5 = zext i32 %4 to i64
+  %6 = or i64 %3, %5
+  ret i64 %6
+}
+
+define arm_aapcs_vfpcc i64 @test_vrmlsldavhxq_s32(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vrmlsldavhxq_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vrmlsldavhx.s32 r0, r1, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call { i32, i32 } @llvm.arm.mve.vrmlldavha.v4i32(i32 0, i32 1, i32 1, i32 0, i32 0, <4 x i32> %a, <4 x i32> %b)
+  %1 = extractvalue { i32, i32 } %0, 1
+  %2 = zext i32 %1 to i64
+  %3 = shl i64 %2, 32
+  %4 = extractvalue { i32, i32 } %0, 0
+  %5 = zext i32 %4 to i64
+  %6 = or i64 %3, %5
+  ret i64 %6
+}
+
+define arm_aapcs_vfpcc i64 @test_vmlaldavq_p_s16(<8 x i16> %a, <8 x i16> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vmlaldavq_p_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmlalvt.s16 r0, r1, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call { i32, i32 } @llvm.arm.mve.vmlldava.predicated.v8i16.v8i1(i32 0, i32 0, i32 0, i32 0, i32 0, <8 x i16> %a, <8 x i16> %b, <8 x i1> %1)
+  %3 = extractvalue { i32, i32 } %2, 1
+  %4 = zext i32 %3 to i64
+  %5 = shl i64 %4, 32
+  %6 = extractvalue { i32, i32 } %2, 0
+  %7 = zext i32 %6 to i64
+  %8 = or i64 %5, %7
+  ret i64 %8
+}
+
+define arm_aapcs_vfpcc i64 @test_vmlaldavq_p_s32(<4 x i32> %a, <4 x i32> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vmlaldavq_p_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmlalvt.s32 r0, r1, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call { i32, i32 } @llvm.arm.mve.vmlldava.predicated.v4i32.v4i1(i32 0, i32 0, i32 0, i32 0, i32 0, <4 x i32> %a, <4 x i32> %b, <4 x i1> %1)
+  %3 = extractvalue { i32, i32 } %2, 1
+  %4 = zext i32 %3 to i64
+  %5 = shl i64 %4, 32
+  %6 = extractvalue { i32, i32 } %2, 0
+  %7 = zext i32 %6 to i64
+  %8 = or i64 %5, %7
+  ret i64 %8
+}
+
+define arm_aapcs_vfpcc i64 @test_vmlaldavq_p_u16(<8 x i16> %a, <8 x i16> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vmlaldavq_p_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmlalvt.u16 r0, r1, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call { i32, i32 } @llvm.arm.mve.vmlldava.predicated.v8i16.v8i1(i32 1, i32 0, i32 0, i32 0, i32 0, <8 x i16> %a, <8 x i16> %b, <8 x i1> %1)
+  %3 = extractvalue { i32, i32 } %2, 1
+  %4 = zext i32 %3 to i64
+  %5 = shl i64 %4, 32
+  %6 = extractvalue { i32, i32 } %2, 0
+  %7 = zext i32 %6 to i64
+  %8 = or i64 %5, %7
+  ret i64 %8
+}
+
+define arm_aapcs_vfpcc i64 @test_vmlaldavq_p_u32(<4 x i32> %a, <4 x i32> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vmlaldavq_p_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmlalvt.u32 r0, r1, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call { i32, i32 } @llvm.arm.mve.vmlldava.predicated.v4i32.v4i1(i32 1, i32 0, i32 0, i32 0, i32 0, <4 x i32> %a, <4 x i32> %b, <4 x i1> %1)
+  %3 = extractvalue { i32, i32 } %2, 1
+  %4 = zext i32 %3 to i64
+  %5 = shl i64 %4, 32
+  %6 = extractvalue { i32, i32 } %2, 0
+  %7 = zext i32 %6 to i64
+  %8 = or i64 %5, %7
+  ret i64 %8
+}
+
+define arm_aapcs_vfpcc i64 @test_vmlaldavxq_p_s16(<8 x i16> %a, <8 x i16> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vmlaldavxq_p_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmlaldavxt.s16 r0, r1, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call { i32, i32 } @llvm.arm.mve.vmlldava.predicated.v8i16.v8i1(i32 0, i32 0, i32 1, i32 0, i32 0, <8 x i16> %a, <8 x i16> %b, <8 x i1> %1)
+  %3 = extractvalue { i32, i32 } %2, 1
+  %4 = zext i32 %3 to i64
+  %5 = shl i64 %4, 32
+  %6 = extractvalue { i32, i32 } %2, 0
+  %7 = zext i32 %6 to i64
+  %8 = or i64 %5, %7
+  ret i64 %8
+}
+
+define arm_aapcs_vfpcc i64 @test_vmlaldavxq_p_s32(<4 x i32> %a, <4 x i32> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vmlaldavxq_p_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmlaldavxt.s32 r0, r1, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call { i32, i32 } @llvm.arm.mve.vmlldava.predicated.v4i32.v4i1(i32 0, i32 0, i32 1, i32 0, i32 0, <4 x i32> %a, <4 x i32> %b, <4 x i1> %1)
+  %3 = extractvalue { i32, i32 } %2, 1
+  %4 = zext i32 %3 to i64
+  %5 = shl i64 %4, 32
+  %6 = extractvalue { i32, i32 } %2, 0
+  %7 = zext i32 %6 to i64
+  %8 = or i64 %5, %7
+  ret i64 %8
+}
+
+define arm_aapcs_vfpcc i64 @test_vmlsldavq_p_s16(<8 x i16> %a, <8 x i16> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vmlsldavq_p_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmlsldavt.s16 r0, r1, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call { i32, i32 } @llvm.arm.mve.vmlldava.predicated.v8i16.v8i1(i32 0, i32 1, i32 0, i32 0, i32 0, <8 x i16> %a, <8 x i16> %b, <8 x i1> %1)
+  %3 = extractvalue { i32, i32 } %2, 1
+  %4 = zext i32 %3 to i64
+  %5 = shl i64 %4, 32
+  %6 = extractvalue { i32, i32 } %2, 0
+  %7 = zext i32 %6 to i64
+  %8 = or i64 %5, %7
+  ret i64 %8
+}
+
+define arm_aapcs_vfpcc i64 @test_vmlsldavq_p_s32(<4 x i32> %a, <4 x i32> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vmlsldavq_p_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmlsldavt.s32 r0, r1, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call { i32, i32 } @llvm.arm.mve.vmlldava.predicated.v4i32.v4i1(i32 0, i32 1, i32 0, i32 0, i32 0, <4 x i32> %a, <4 x i32> %b, <4 x i1> %1)
+  %3 = extractvalue { i32, i32 } %2, 1
+  %4 = zext i32 %3 to i64
+  %5 = shl i64 %4, 32
+  %6 = extractvalue { i32, i32 } %2, 0
+  %7 = zext i32 %6 to i64
+  %8 = or i64 %5, %7
+  ret i64 %8
+}
+
+define arm_aapcs_vfpcc i64 @test_vmlsldaxvq_p_s16(<8 x i16> %a, <8 x i16> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vmlsldaxvq_p_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmlsldavxt.s16 r0, r1, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call { i32, i32 } @llvm.arm.mve.vmlldava.predicated.v8i16.v8i1(i32 0, i32 1, i32 1, i32 0, i32 0, <8 x i16> %a, <8 x i16> %b, <8 x i1> %1)
+  %3 = extractvalue { i32, i32 } %2, 1
+  %4 = zext i32 %3 to i64
+  %5 = shl i64 %4, 32
+  %6 = extractvalue { i32, i32 } %2, 0
+  %7 = zext i32 %6 to i64
+  %8 = or i64 %5, %7
+  ret i64 %8
+}
+
+define arm_aapcs_vfpcc i64 @test_vmlsldavxq_p_s32(<4 x i32> %a, <4 x i32> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vmlsldavxq_p_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmlsldavxt.s32 r0, r1, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call { i32, i32 } @llvm.arm.mve.vmlldava.predicated.v4i32.v4i1(i32 0, i32 1, i32 1, i32 0, i32 0, <4 x i32> %a, <4 x i32> %b, <4 x i1> %1)
+  %3 = extractvalue { i32, i32 } %2, 1
+  %4 = zext i32 %3 to i64
+  %5 = shl i64 %4, 32
+  %6 = extractvalue { i32, i32 } %2, 0
+  %7 = zext i32 %6 to i64
+  %8 = or i64 %5, %7
+  ret i64 %8
+}
+
+define arm_aapcs_vfpcc i64 @test_vrmlaldavhq_p_s32(<4 x i32> %a, <4 x i32> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vrmlaldavhq_p_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vrmlalvht.s32 r0, r1, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call { i32, i32 } @llvm.arm.mve.vrmlldavha.predicated.v4i32.v4i1(i32 0, i32 0, i32 0, i32 0, i32 0, <4 x i32> %a, <4 x i32> %b, <4 x i1> %1)
+  %3 = extractvalue { i32, i32 } %2, 1
+  %4 = zext i32 %3 to i64
+  %5 = shl i64 %4, 32
+  %6 = extractvalue { i32, i32 } %2, 0
+  %7 = zext i32 %6 to i64
+  %8 = or i64 %5, %7
+  ret i64 %8
+}
+
+define arm_aapcs_vfpcc i64 @test_vrmlaldavhq_p_u32(<4 x i32> %a, <4 x i32> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vrmlaldavhq_p_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vrmlalvht.u32 r0, r1, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call { i32, i32 } @llvm.arm.mve.vrmlldavha.predicated.v4i32.v4i1(i32 1, i32 0, i32 0, i32 0, i32 0, <4 x i32> %a, <4 x i32> %b, <4 x i1> %1)
+  %3 = extractvalue { i32, i32 } %2, 1
+  %4 = zext i32 %3 to i64
+  %5 = shl i64 %4, 32
+  %6 = extractvalue { i32, i32 } %2, 0
+  %7 = zext i32 %6 to i64
+  %8 = or i64 %5, %7
+  ret i64 %8
+}
+
+define arm_aapcs_vfpcc i64 @test_vrmlaldavhxq_p_s32(<4 x i32> %a, <4 x i32> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vrmlaldavhxq_p_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vrmlaldavhxt.s32 r0, r1, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call { i32, i32 } @llvm.arm.mve.vrmlldavha.predicated.v4i32.v4i1(i32 0, i32 0, i32 1, i32 0, i32 0, <4 x i32> %a, <4 x i32> %b, <4 x i1> %1)
+  %3 = extractvalue { i32, i32 } %2, 1
+  %4 = zext i32 %3 to i64
+  %5 = shl i64 %4, 32
+  %6 = extractvalue { i32, i32 } %2, 0
+  %7 = zext i32 %6 to i64
+  %8 = or i64 %5, %7
+  ret i64 %8
+}
+
+define arm_aapcs_vfpcc i64 @test_vrmlsldavhq_p_s32(<4 x i32> %a, <4 x i32> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vrmlsldavhq_p_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vrmlsldavht.s32 r0, r1, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call { i32, i32 } @llvm.arm.mve.vrmlldavha.predicated.v4i32.v4i1(i32 0, i32 1, i32 0, i32 0, i32 0, <4 x i32> %a, <4 x i32> %b, <4 x i1> %1)
+  %3 = extractvalue { i32, i32 } %2, 1
+  %4 = zext i32 %3 to i64
+  %5 = shl i64 %4, 32
+  %6 = extractvalue { i32, i32 } %2, 0
+  %7 = zext i32 %6 to i64
+  %8 = or i64 %5, %7
+  ret i64 %8
+}
+
+define arm_aapcs_vfpcc i64 @test_vrmlsldavhxq_p_s32(<4 x i32> %a, <4 x i32> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vrmlsldavhxq_p_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vrmlsldavhxt.s32 r0, r1, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = call { i32, i32 } @llvm.arm.mve.vrmlldavha.predicated.v4i32.v4i1(i32 0, i32 1, i32 1, i32 0, i32 0, <4 x i32> %a, <4 x i32> %b, <4 x i1> %1)
+  %3 = extractvalue { i32, i32 } %2, 1
+  %4 = zext i32 %3 to i64
+  %5 = shl i64 %4, 32
+  %6 = extractvalue { i32, i32 } %2, 0
+  %7 = zext i32 %6 to i64
+  %8 = or i64 %5, %7
+  ret i64 %8
+}
diff --git a/llvm/test/CodeGen/Thumb2/mve-intrinsics/vmulhq.ll b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vmulhq.ll
new file mode 100644
index 000000000..aae9a2397
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vmulhq.ll
@@ -0,0 +1,135 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve.fp -verify-machineinstrs -o - %s | FileCheck %s
+
+define arm_aapcs_vfpcc <16 x i8> @test_vmulhq_u8(<16 x i8> %a, <16 x i8> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vmulhq_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmulh.u8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <16 x i8> @llvm.arm.mve.vmulh.v16i8(<16 x i8> %a, <16 x i8> %b, i32 1)
+  ret <16 x i8> %0
+}
+
+declare <16 x i8> @llvm.arm.mve.vmulh.v16i8(<16 x i8>, <16 x i8>, i32) #1
+
+define arm_aapcs_vfpcc <8 x i16> @test_vmulhq_s16(<8 x i16> %a, <8 x i16> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vmulhq_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmulh.s16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <8 x i16> @llvm.arm.mve.vmulh.v8i16(<8 x i16> %a, <8 x i16> %b, i32 0)
+  ret <8 x i16> %0
+}
+
+declare <8 x i16> @llvm.arm.mve.vmulh.v8i16(<8 x i16>, <8 x i16>, i32) #1
+
+define arm_aapcs_vfpcc <4 x i32> @test_vmulhq_u32(<4 x i32> %a, <4 x i32> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vmulhq_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmulh.u32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <4 x i32> @llvm.arm.mve.vmulh.v4i32(<4 x i32> %a, <4 x i32> %b, i32 1)
+  ret <4 x i32> %0
+}
+
+declare <4 x i32> @llvm.arm.mve.vmulh.v4i32(<4 x i32>, <4 x i32>, i32) #1
+
+define arm_aapcs_vfpcc <16 x i8> @test_vmulhq_m_s8(<16 x i8> %inactive, <16 x i8> %a, <16 x i8> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vmulhq_m_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmulht.s8 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <16 x i8> @llvm.arm.mve.mulh.predicated.v16i8.v16i1(<16 x i8> %a, <16 x i8> %b, i32 0, <16 x i1> %1, <16 x i8> %inactive)
+  ret <16 x i8> %2
+}
+
+declare <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32) #1
+
+declare <16 x i8> @llvm.arm.mve.mulh.predicated.v16i8.v16i1(<16 x i8>, <16 x i8>, i32, <16 x i1>, <16 x i8>) #1
+
+define arm_aapcs_vfpcc <8 x i16> @test_vmulhq_m_u16(<8 x i16> %inactive, <8 x i16> %a, <8 x i16> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vmulhq_m_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmulht.u16 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.mulh.predicated.v8i16.v8i1(<8 x i16> %a, <8 x i16> %b, i32 1, <8 x i1> %1, <8 x i16> %inactive)
+  ret <8 x i16> %2
+}
+
+declare <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32) #1
+
+declare <8 x i16> @llvm.arm.mve.mulh.predicated.v8i16.v8i1(<8 x i16>, <8 x i16>, i32, <8 x i1>, <8 x i16>) #1
+
+define arm_aapcs_vfpcc <4 x i32> @test_vmulhq_m_s32(<4 x i32> %inactive, <4 x i32> %a, <4 x i32> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vmulhq_m_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmulht.s32 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call <4 x i32> @llvm.arm.mve.mulh.predicated.v4i32.v4i1(<4 x i32> %a, <4 x i32> %b, i32 0, <4 x i1> %1, <4 x i32> %inactive)
+  ret <4 x i32> %2
+}
+
+declare <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32) #1
+
+declare <4 x i32> @llvm.arm.mve.mulh.predicated.v4i32.v4i1(<4 x i32>, <4 x i32>, i32, <4 x i1>, <4 x i32>) #1
+
+define arm_aapcs_vfpcc <16 x i8> @test_vmulhq_x_u8(<16 x i8> %a, <16 x i8> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vmulhq_x_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmulht.u8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <16 x i8> @llvm.arm.mve.mulh.predicated.v16i8.v16i1(<16 x i8> %a, <16 x i8> %b, i32 1, <16 x i1> %1, <16 x i8> undef)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vmulhq_x_s16(<8 x i16> %a, <8 x i16> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vmulhq_x_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmulht.s16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.mulh.predicated.v8i16.v8i1(<8 x i16> %a, <8 x i16> %b, i32 0, <8 x i1> %1, <8 x i16> undef)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vmulhq_x_u32(<4 x i32> %a, <4 x i32> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vmulhq_x_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmulht.u32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call <4 x i32> @llvm.arm.mve.mulh.predicated.v4i32.v4i1(<4 x i32> %a, <4 x i32> %b, i32 1, <4 x i1> %1, <4 x i32> undef)
+  ret <4 x i32> %2
+}
+
diff --git a/llvm/test/CodeGen/Thumb2/mve-intrinsics/vmullbq.ll b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vmullbq.ll
new file mode 100644
index 000000000..439146905
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vmullbq.ll
@@ -0,0 +1,181 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve.fp -verify-machineinstrs -o - %s | FileCheck %s
+
+define arm_aapcs_vfpcc <8 x i16> @test_vmullbq_int_u8(<16 x i8> %a, <16 x i8> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vmullbq_int_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmullb.u8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <8 x i16> @llvm.arm.mve.vmull.v8i16.v16i8(<16 x i8> %a, <16 x i8> %b, i32 1, i32 0)
+  ret <8 x i16> %0
+}
+
+declare <8 x i16> @llvm.arm.mve.vmull.v8i16.v16i8(<16 x i8>, <16 x i8>, i32, i32) #1
+
+define arm_aapcs_vfpcc <4 x i32> @test_vmullbq_int_s16(<8 x i16> %a, <8 x i16> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vmullbq_int_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmullb.s16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <4 x i32> @llvm.arm.mve.vmull.v4i32.v8i16(<8 x i16> %a, <8 x i16> %b, i32 0, i32 0)
+  ret <4 x i32> %0
+}
+
+declare <4 x i32> @llvm.arm.mve.vmull.v4i32.v8i16(<8 x i16>, <8 x i16>, i32, i32) #1
+
+define arm_aapcs_vfpcc <2 x i64> @test_vmullbq_int_u32(<4 x i32> %a, <4 x i32> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vmullbq_int_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmullb.u32 q2, q0, q1
+; CHECK-NEXT:    vmov q0, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <2 x i64> @llvm.arm.mve.vmull.v2i64.v4i32(<4 x i32> %a, <4 x i32> %b, i32 1, i32 0)
+  ret <2 x i64> %0
+}
+
+declare <2 x i64> @llvm.arm.mve.vmull.v2i64.v4i32(<4 x i32>, <4 x i32>, i32, i32) #1
+
+define arm_aapcs_vfpcc <4 x i32> @test_vmullbq_poly_p16(<8 x i16> %a, <8 x i16> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vmullbq_poly_p16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmullb.p16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <4 x i32> @llvm.arm.mve.vmull.poly.v4i32.v8i16(<8 x i16> %a, <8 x i16> %b, i32 0)
+  ret <4 x i32> %0
+}
+
+declare <4 x i32> @llvm.arm.mve.vmull.poly.v4i32.v8i16(<8 x i16>, <8 x i16>, i32) #1
+
+define arm_aapcs_vfpcc <8 x i16> @test_vmullbq_int_m_s8(<8 x i16> %inactive, <16 x i8> %a, <16 x i8> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vmullbq_int_m_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmullbt.s8 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.mull.int.predicated.v8i16.v16i8.v16i1(<16 x i8> %a, <16 x i8> %b, i32 0, i32 0, <16 x i1> %1, <8 x i16> %inactive)
+  ret <8 x i16> %2
+}
+
+declare <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32) #1
+
+declare <8 x i16> @llvm.arm.mve.mull.int.predicated.v8i16.v16i8.v16i1(<16 x i8>, <16 x i8>, i32, i32, <16 x i1>, <8 x i16>) #1
+
+define arm_aapcs_vfpcc <4 x i32> @test_vmullbq_int_m_u16(<4 x i32> %inactive, <8 x i16> %a, <8 x i16> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vmullbq_int_m_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmullbt.u16 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <4 x i32> @llvm.arm.mve.mull.int.predicated.v4i32.v8i16.v8i1(<8 x i16> %a, <8 x i16> %b, i32 1, i32 0, <8 x i1> %1, <4 x i32> %inactive)
+  ret <4 x i32> %2
+}
+
+declare <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32) #1
+
+declare <4 x i32> @llvm.arm.mve.mull.int.predicated.v4i32.v8i16.v8i1(<8 x i16>, <8 x i16>, i32, i32, <8 x i1>, <4 x i32>) #1
+
+define arm_aapcs_vfpcc <2 x i64> @test_vmullbq_int_m_s32(<2 x i64> %inactive, <4 x i32> %a, <4 x i32> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vmullbq_int_m_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmullbt.s32 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call <2 x i64> @llvm.arm.mve.mull.int.predicated.v2i64.v4i32.v4i1(<4 x i32> %a, <4 x i32> %b, i32 0, i32 0, <4 x i1> %1, <2 x i64> %inactive)
+  ret <2 x i64> %2
+}
+
+declare <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32) #1
+
+declare <2 x i64> @llvm.arm.mve.mull.int.predicated.v2i64.v4i32.v4i1(<4 x i32>, <4 x i32>, i32, i32, <4 x i1>, <2 x i64>) #1
+
+define arm_aapcs_vfpcc <8 x i16> @test_vmullbq_poly_m_p8(<8 x i16> %inactive, <16 x i8> %a, <16 x i8> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vmullbq_poly_m_p8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmullbt.p8 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.mull.poly.predicated.v8i16.v16i8.v16i1(<16 x i8> %a, <16 x i8> %b, i32 0, <16 x i1> %1, <8 x i16> %inactive)
+  ret <8 x i16> %2
+}
+
+declare <8 x i16> @llvm.arm.mve.mull.poly.predicated.v8i16.v16i8.v16i1(<16 x i8>, <16 x i8>, i32, <16 x i1>, <8 x i16>) #1
+
+define arm_aapcs_vfpcc <8 x i16> @test_vmullbq_int_x_u8(<16 x i8> %a, <16 x i8> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vmullbq_int_x_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmullbt.u8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.mull.int.predicated.v8i16.v16i8.v16i1(<16 x i8> %a, <16 x i8> %b, i32 1, i32 0, <16 x i1> %1, <8 x i16> undef)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vmullbq_int_x_s16(<8 x i16> %a, <8 x i16> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vmullbq_int_x_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmullbt.s16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <4 x i32> @llvm.arm.mve.mull.int.predicated.v4i32.v8i16.v8i1(<8 x i16> %a, <8 x i16> %b, i32 0, i32 0, <8 x i1> %1, <4 x i32> undef)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <2 x i64> @test_vmullbq_int_x_u32(<4 x i32> %a, <4 x i32> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vmullbq_int_x_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmullbt.u32 q2, q0, q1
+; CHECK-NEXT:    vmov q0, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call <2 x i64> @llvm.arm.mve.mull.int.predicated.v2i64.v4i32.v4i1(<4 x i32> %a, <4 x i32> %b, i32 1, i32 0, <4 x i1> %1, <2 x i64> undef)
+  ret <2 x i64> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vmullbq_poly_x_p16(<8 x i16> %a, <8 x i16> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vmullbq_poly_x_p16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmullbt.p16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <4 x i32> @llvm.arm.mve.mull.poly.predicated.v4i32.v8i16.v8i1(<8 x i16> %a, <8 x i16> %b, i32 0, <8 x i1> %1, <4 x i32> undef)
+  ret <4 x i32> %2
+}
+
+declare <4 x i32> @llvm.arm.mve.mull.poly.predicated.v4i32.v8i16.v8i1(<8 x i16>, <8 x i16>, i32, <8 x i1>, <4 x i32>) #1
+
diff --git a/llvm/test/CodeGen/Thumb2/mve-intrinsics/vmulltq.ll b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vmulltq.ll
new file mode 100644
index 000000000..39c1105f9
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vmulltq.ll
@@ -0,0 +1,179 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve.fp -verify-machineinstrs -o - %s | FileCheck %s
+
+define arm_aapcs_vfpcc <8 x i16> @test_vmulltq_int_u8(<16 x i8> %a, <16 x i8> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vmulltq_int_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmullt.u8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <8 x i16> @llvm.arm.mve.vmull.v8i16.v16i8(<16 x i8> %a, <16 x i8> %b, i32 1, i32 1)
+  ret <8 x i16> %0
+}
+
+declare <8 x i16> @llvm.arm.mve.vmull.v8i16.v16i8(<16 x i8>, <16 x i8>, i32, i32) #1
+
+define arm_aapcs_vfpcc <4 x i32> @test_vmulltq_int_s16(<8 x i16> %a, <8 x i16> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vmulltq_int_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmullt.s16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <4 x i32> @llvm.arm.mve.vmull.v4i32.v8i16(<8 x i16> %a, <8 x i16> %b, i32 0, i32 1)
+  ret <4 x i32> %0
+}
+
+declare <4 x i32> @llvm.arm.mve.vmull.v4i32.v8i16(<8 x i16>, <8 x i16>, i32, i32) #1
+
+define arm_aapcs_vfpcc <2 x i64> @test_vmulltq_int_u32(<4 x i32> %a, <4 x i32> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vmulltq_int_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmullt.u32 q2, q0, q1
+; CHECK-NEXT:    vmov q0, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <2 x i64> @llvm.arm.mve.vmull.v2i64.v4i32(<4 x i32> %a, <4 x i32> %b, i32 1, i32 1)
+  ret <2 x i64> %0
+}
+
+declare <2 x i64> @llvm.arm.mve.vmull.v2i64.v4i32(<4 x i32>, <4 x i32>, i32, i32) #1
+
+define arm_aapcs_vfpcc <4 x i32> @test_vmulltq_poly_p16(<8 x i16> %a, <8 x i16> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vmulltq_poly_p16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmullt.p16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <4 x i32> @llvm.arm.mve.vmull.poly.v4i32.v8i16(<8 x i16> %a, <8 x i16> %b, i32 1)
+  ret <4 x i32> %0
+}
+
+declare <4 x i32> @llvm.arm.mve.vmull.poly.v4i32.v8i16(<8 x i16>, <8 x i16>, i32) #1
+
+define arm_aapcs_vfpcc <8 x i16> @test_vmulltq_int_m_s8(<8 x i16> %inactive, <16 x i8> %a, <16 x i8> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vmulltq_int_m_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmulltt.s8 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.mull.int.predicated.v8i16.v16i8.v16i1(<16 x i8> %a, <16 x i8> %b, i32 0, i32 1, <16 x i1> %1, <8 x i16> %inactive)
+  ret <8 x i16> %2
+}
+
+declare <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32) #1
+
+declare <8 x i16> @llvm.arm.mve.mull.int.predicated.v8i16.v16i8.v16i1(<16 x i8>, <16 x i8>, i32, i32, <16 x i1>, <8 x i16>) #1
+
+define arm_aapcs_vfpcc <4 x i32> @test_vmulltq_int_m_u16(<4 x i32> %inactive, <8 x i16> %a, <8 x i16> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vmulltq_int_m_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmulltt.u16 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <4 x i32> @llvm.arm.mve.mull.int.predicated.v4i32.v8i16.v8i1(<8 x i16> %a, <8 x i16> %b, i32 1, i32 1, <8 x i1> %1, <4 x i32> %inactive)
+  ret <4 x i32> %2
+}
+
+declare <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32) #1
+
+declare <4 x i32> @llvm.arm.mve.mull.int.predicated.v4i32.v8i16.v8i1(<8 x i16>, <8 x i16>, i32, i32, <8 x i1>, <4 x i32>) #1
+
+define arm_aapcs_vfpcc <2 x i64> @test_vmulltq_int_m_s32(<2 x i64> %inactive, <4 x i32> %a, <4 x i32> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vmulltq_int_m_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmulltt.s32 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call <2 x i64> @llvm.arm.mve.mull.int.predicated.v2i64.v4i32.v4i1(<4 x i32> %a, <4 x i32> %b, i32 0, i32 1, <4 x i1> %1, <2 x i64> %inactive)
+  ret <2 x i64> %2
+}
+
+declare <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32) #1
+
+declare <2 x i64> @llvm.arm.mve.mull.int.predicated.v2i64.v4i32.v4i1(<4 x i32>, <4 x i32>, i32, i32, <4 x i1>, <2 x i64>) #1
+
+define arm_aapcs_vfpcc <8 x i16> @test_vmulltq_poly_m_p8(<8 x i16> %inactive, <16 x i8> %a, <16 x i8> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vmulltq_poly_m_p8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmulltt.p8 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.mull.poly.predicated.v8i16.v16i8.v16i1(<16 x i8> %a, <16 x i8> %b, i32 1, <16 x i1> %1, <8 x i16> %inactive)
+  ret <8 x i16> %2
+}
+
+declare <8 x i16> @llvm.arm.mve.mull.poly.predicated.v8i16.v16i8.v16i1(<16 x i8>, <16 x i8>, i32, <16 x i1>, <8 x i16>) #1
+
+define arm_aapcs_vfpcc <8 x i16> @test_vmulltq_int_x_u8(<16 x i8> %a, <16 x i8> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vmulltq_int_x_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmulltt.u8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.mull.int.predicated.v8i16.v16i8.v16i1(<16 x i8> %a, <16 x i8> %b, i32 1, i32 1, <16 x i1> %1, <8 x i16> undef)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vmulltq_int_x_s16(<8 x i16> %a, <8 x i16> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vmulltq_int_x_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmulltt.s16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <4 x i32> @llvm.arm.mve.mull.int.predicated.v4i32.v8i16.v8i1(<8 x i16> %a, <8 x i16> %b, i32 0, i32 1, <8 x i1> %1, <4 x i32> undef)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <2 x i64> @test_vmulltq_int_x_u32(<4 x i32> %a, <4 x i32> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vmulltq_int_x_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmulltt.u32 q2, q0, q1
+; CHECK-NEXT:    vmov q0, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call <2 x i64> @llvm.arm.mve.mull.int.predicated.v2i64.v4i32.v4i1(<4 x i32> %a, <4 x i32> %b, i32 1, i32 1, <4 x i1> %1, <2 x i64> undef)
+  ret <2 x i64> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vmulltq_poly_x_p8(<16 x i8> %a, <16 x i8> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vmulltq_poly_x_p8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmulltt.p8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.mull.poly.predicated.v8i16.v16i8.v16i1(<16 x i8> %a, <16 x i8> %b, i32 1, <16 x i1> %1, <8 x i16> undef)
+  ret <8 x i16> %2
+}
+
diff --git a/llvm/test/CodeGen/Thumb2/mve-intrinsics/vmulq.ll b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vmulq.ll
new file mode 100644
index 000000000..19ed08a77
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vmulq.ll
@@ -0,0 +1,171 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve.fp -verify-machineinstrs -o - %s | FileCheck %s
+
+define arm_aapcs_vfpcc <16 x i8> @test_vmulq_u8(<16 x i8> %a, <16 x i8> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vmulq_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmul.i8 q0, q1, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = mul <16 x i8> %b, %a
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vmulq_s16(<8 x i16> %a, <8 x i16> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vmulq_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmul.i16 q0, q1, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = mul <8 x i16> %b, %a
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vmulq_u32(<4 x i32> %a, <4 x i32> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vmulq_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmul.i32 q0, q1, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = mul <4 x i32> %b, %a
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <4 x float> @test_vmulq_f32(<4 x float> %a, <4 x float> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vmulq_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmul.f32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = fmul <4 x float> %a, %b
+  ret <4 x float> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vmulq_m_s8(<16 x i8> %inactive, <16 x i8> %a, <16 x i8> %b, i16 zeroext %p) local_unnamed_addr #1 {
+; CHECK-LABEL: test_vmulq_m_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmult.i8 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <16 x i8> @llvm.arm.mve.mul.predicated.v16i8.v16i1(<16 x i8> %a, <16 x i8> %b, <16 x i1> %1, <16 x i8> %inactive)
+  ret <16 x i8> %2
+}
+
+declare <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32) #2
+
+declare <16 x i8> @llvm.arm.mve.mul.predicated.v16i8.v16i1(<16 x i8>, <16 x i8>, <16 x i1>, <16 x i8>) #2
+
+define arm_aapcs_vfpcc <8 x i16> @test_vmulq_m_u16(<8 x i16> %inactive, <8 x i16> %a, <8 x i16> %b, i16 zeroext %p) local_unnamed_addr #1 {
+; CHECK-LABEL: test_vmulq_m_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmult.i16 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.mul.predicated.v8i16.v8i1(<8 x i16> %a, <8 x i16> %b, <8 x i1> %1, <8 x i16> %inactive)
+  ret <8 x i16> %2
+}
+
+declare <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32) #2
+
+declare <8 x i16> @llvm.arm.mve.mul.predicated.v8i16.v8i1(<8 x i16>, <8 x i16>, <8 x i1>, <8 x i16>) #2
+
+define arm_aapcs_vfpcc <4 x i32> @test_vmulq_m_s32(<4 x i32> %inactive, <4 x i32> %a, <4 x i32> %b, i16 zeroext %p) local_unnamed_addr #1 {
+; CHECK-LABEL: test_vmulq_m_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmult.i32 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call <4 x i32> @llvm.arm.mve.mul.predicated.v4i32.v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i1> %1, <4 x i32> %inactive)
+  ret <4 x i32> %2
+}
+
+declare <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32) #2
+
+declare <4 x i32> @llvm.arm.mve.mul.predicated.v4i32.v4i1(<4 x i32>, <4 x i32>, <4 x i1>, <4 x i32>) #2
+
+define arm_aapcs_vfpcc <8 x half> @test_vmulq_m_f16(<8 x half> %inactive, <8 x half> %a, <8 x half> %b, i16 zeroext %p) local_unnamed_addr #1 {
+; CHECK-LABEL: test_vmulq_m_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmult.f16 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <8 x half> @llvm.arm.mve.mul.predicated.v8f16.v8i1(<8 x half> %a, <8 x half> %b, <8 x i1> %1, <8 x half> %inactive)
+  ret <8 x half> %2
+}
+
+declare <8 x half> @llvm.arm.mve.mul.predicated.v8f16.v8i1(<8 x half>, <8 x half>, <8 x i1>, <8 x half>) #2
+
+define arm_aapcs_vfpcc <16 x i8> @test_vmulq_x_u8(<16 x i8> %a, <16 x i8> %b, i16 zeroext %p) local_unnamed_addr #1 {
+; CHECK-LABEL: test_vmulq_x_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmult.i8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <16 x i8> @llvm.arm.mve.mul.predicated.v16i8.v16i1(<16 x i8> %a, <16 x i8> %b, <16 x i1> %1, <16 x i8> undef)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vmulq_x_s16(<8 x i16> %a, <8 x i16> %b, i16 zeroext %p) local_unnamed_addr #1 {
+; CHECK-LABEL: test_vmulq_x_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmult.i16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.mul.predicated.v8i16.v8i1(<8 x i16> %a, <8 x i16> %b, <8 x i1> %1, <8 x i16> undef)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vmulq_x_u32(<4 x i32> %a, <4 x i32> %b, i16 zeroext %p) local_unnamed_addr #1 {
+; CHECK-LABEL: test_vmulq_x_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmult.i32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call <4 x i32> @llvm.arm.mve.mul.predicated.v4i32.v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i1> %1, <4 x i32> undef)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <4 x float> @test_vmulq_m_f32(<4 x float> %a, <4 x float> %b, i16 zeroext %p) local_unnamed_addr #1 {
+; CHECK-LABEL: test_vmulq_m_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmult.f32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call <4 x float> @llvm.arm.mve.mul.predicated.v4f32.v4i1(<4 x float> %a, <4 x float> %b, <4 x i1> %1, <4 x float> undef)
+  ret <4 x float> %2
+}
+
+declare <4 x float> @llvm.arm.mve.mul.predicated.v4f32.v4i1(<4 x float>, <4 x float>, <4 x i1>, <4 x float>) #2
+
diff --git a/llvm/test/CodeGen/Thumb2/mve-intrinsics/vornq.ll b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vornq.ll
new file mode 100644
index 000000000..be4ad54dd
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vornq.ll
@@ -0,0 +1,181 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve.fp -verify-machineinstrs -o - %s | FileCheck %s
+
+define arm_aapcs_vfpcc <16 x i8> @test_vornq_u8(<16 x i8> %a, <16 x i8> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vornq_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vorn q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = xor <16 x i8> %b, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
+  %1 = or <16 x i8> %0, %a
+  ret <16 x i8> %1
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vornq_s16(<8 x i16> %a, <8 x i16> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vornq_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vorn q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = xor <8 x i16> %b, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>
+  %1 = or <8 x i16> %0, %a
+  ret <8 x i16> %1
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vornq_u32(<4 x i32> %a, <4 x i32> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vornq_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vorn q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = xor <4 x i32> %b, <i32 -1, i32 -1, i32 -1, i32 -1>
+  %1 = or <4 x i32> %0, %a
+  ret <4 x i32> %1
+}
+
+define arm_aapcs_vfpcc <4 x float> @test_vornq_f32(<4 x float> %a, <4 x float> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vornq_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vorn q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast <4 x float> %a to <4 x i32>
+  %1 = bitcast <4 x float> %b to <4 x i32>
+  %2 = xor <4 x i32> %1, <i32 -1, i32 -1, i32 -1, i32 -1>
+  %3 = or <4 x i32> %2, %0
+  %4 = bitcast <4 x i32> %3 to <4 x float>
+  ret <4 x float> %4
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vornq_m_s8(<16 x i8> %inactive, <16 x i8> %a, <16 x i8> %b, i16 zeroext %p) local_unnamed_addr #1 {
+; CHECK-LABEL: test_vornq_m_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vornt q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <16 x i8> @llvm.arm.mve.orn.predicated.v16i8.v16i1(<16 x i8> %a, <16 x i8> %b, <16 x i1> %1, <16 x i8> %inactive)
+  ret <16 x i8> %2
+}
+
+declare <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32) #2
+
+declare <16 x i8> @llvm.arm.mve.orn.predicated.v16i8.v16i1(<16 x i8>, <16 x i8>, <16 x i1>, <16 x i8>) #2
+
+define arm_aapcs_vfpcc <8 x i16> @test_vornq_m_u16(<8 x i16> %inactive, <8 x i16> %a, <8 x i16> %b, i16 zeroext %p) local_unnamed_addr #1 {
+; CHECK-LABEL: test_vornq_m_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vornt q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.orn.predicated.v8i16.v8i1(<8 x i16> %a, <8 x i16> %b, <8 x i1> %1, <8 x i16> %inactive)
+  ret <8 x i16> %2
+}
+
+declare <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32) #2
+
+declare <8 x i16> @llvm.arm.mve.orn.predicated.v8i16.v8i1(<8 x i16>, <8 x i16>, <8 x i1>, <8 x i16>) #2
+
+define arm_aapcs_vfpcc <4 x i32> @test_vornq_m_s32(<4 x i32> %inactive, <4 x i32> %a, <4 x i32> %b, i16 zeroext %p) local_unnamed_addr #1 {
+; CHECK-LABEL: test_vornq_m_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vornt q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call <4 x i32> @llvm.arm.mve.orn.predicated.v4i32.v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i1> %1, <4 x i32> %inactive)
+  ret <4 x i32> %2
+}
+
+declare <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32) #2
+
+declare <4 x i32> @llvm.arm.mve.orn.predicated.v4i32.v4i1(<4 x i32>, <4 x i32>, <4 x i1>, <4 x i32>) #2
+
+define arm_aapcs_vfpcc <8 x half> @test_vornq_m_f16(<8 x half> %inactive, <8 x half> %a, <8 x half> %b, i16 zeroext %p) local_unnamed_addr #1 {
+; CHECK-LABEL: test_vornq_m_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vornt q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast <8 x half> %a to <8 x i16>
+  %1 = bitcast <8 x half> %b to <8 x i16>
+  %2 = zext i16 %p to i32
+  %3 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %2)
+  %4 = bitcast <8 x half> %inactive to <8 x i16>
+  %5 = tail call <8 x i16> @llvm.arm.mve.orn.predicated.v8i16.v8i1(<8 x i16> %0, <8 x i16> %1, <8 x i1> %3, <8 x i16> %4)
+  %6 = bitcast <8 x i16> %5 to <8 x half>
+  ret <8 x half> %6
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vornq_x_u8(<16 x i8> %a, <16 x i8> %b, i16 zeroext %p) local_unnamed_addr #1 {
+; CHECK-LABEL: test_vornq_x_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vornt q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <16 x i8> @llvm.arm.mve.orn.predicated.v16i8.v16i1(<16 x i8> %a, <16 x i8> %b, <16 x i1> %1, <16 x i8> undef)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vornq_x_s16(<8 x i16> %a, <8 x i16> %b, i16 zeroext %p) local_unnamed_addr #1 {
+; CHECK-LABEL: test_vornq_x_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vornt q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.orn.predicated.v8i16.v8i1(<8 x i16> %a, <8 x i16> %b, <8 x i1> %1, <8 x i16> undef)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vornq_x_u32(<4 x i32> %a, <4 x i32> %b, i16 zeroext %p) local_unnamed_addr #1 {
+; CHECK-LABEL: test_vornq_x_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vornt q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call <4 x i32> @llvm.arm.mve.orn.predicated.v4i32.v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i1> %1, <4 x i32> undef)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <4 x float> @test_vornq_m_f32(<4 x float> %a, <4 x float> %b, i16 zeroext %p) local_unnamed_addr #1 {
+; CHECK-LABEL: test_vornq_m_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vornt q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast <4 x float> %a to <4 x i32>
+  %1 = bitcast <4 x float> %b to <4 x i32>
+  %2 = zext i16 %p to i32
+  %3 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %2)
+  %4 = tail call <4 x i32> @llvm.arm.mve.orn.predicated.v4i32.v4i1(<4 x i32> %0, <4 x i32> %1, <4 x i1> %3, <4 x i32> undef)
+  %5 = bitcast <4 x i32> %4 to <4 x float>
+  ret <4 x float> %5
+}
+
diff --git a/llvm/test/CodeGen/Thumb2/mve-intrinsics/vorrq.ll b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vorrq.ll
new file mode 100644
index 000000000..168dde080
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vorrq.ll
@@ -0,0 +1,177 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve.fp -verify-machineinstrs -o - %s | FileCheck %s
+
+define arm_aapcs_vfpcc <16 x i8> @test_vorrq_u8(<16 x i8> %a, <16 x i8> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vorrq_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vorr q0, q1, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = or <16 x i8> %b, %a
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vorrq_s16(<8 x i16> %a, <8 x i16> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vorrq_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vorr q0, q1, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = or <8 x i16> %b, %a
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vorrq_u32(<4 x i32> %a, <4 x i32> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vorrq_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vorr q0, q1, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = or <4 x i32> %b, %a
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <4 x float> @test_vorrq_f32(<4 x float> %a, <4 x float> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vorrq_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vorr q0, q1, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast <4 x float> %a to <4 x i32>
+  %1 = bitcast <4 x float> %b to <4 x i32>
+  %2 = or <4 x i32> %1, %0
+  %3 = bitcast <4 x i32> %2 to <4 x float>
+  ret <4 x float> %3
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vorrq_m_s8(<16 x i8> %inactive, <16 x i8> %a, <16 x i8> %b, i16 zeroext %p) local_unnamed_addr #1 {
+; CHECK-LABEL: test_vorrq_m_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vorrt q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <16 x i8> @llvm.arm.mve.orr.predicated.v16i8.v16i1(<16 x i8> %a, <16 x i8> %b, <16 x i1> %1, <16 x i8> %inactive)
+  ret <16 x i8> %2
+}
+
+declare <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32) #2
+
+declare <16 x i8> @llvm.arm.mve.orr.predicated.v16i8.v16i1(<16 x i8>, <16 x i8>, <16 x i1>, <16 x i8>) #2
+
+define arm_aapcs_vfpcc <8 x i16> @test_vorrq_m_u16(<8 x i16> %inactive, <8 x i16> %a, <8 x i16> %b, i16 zeroext %p) local_unnamed_addr #1 {
+; CHECK-LABEL: test_vorrq_m_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vorrt q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.orr.predicated.v8i16.v8i1(<8 x i16> %a, <8 x i16> %b, <8 x i1> %1, <8 x i16> %inactive)
+  ret <8 x i16> %2
+}
+
+declare <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32) #2
+
+declare <8 x i16> @llvm.arm.mve.orr.predicated.v8i16.v8i1(<8 x i16>, <8 x i16>, <8 x i1>, <8 x i16>) #2
+
+define arm_aapcs_vfpcc <4 x i32> @test_vorrq_m_s32(<4 x i32> %inactive, <4 x i32> %a, <4 x i32> %b, i16 zeroext %p) local_unnamed_addr #1 {
+; CHECK-LABEL: test_vorrq_m_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vorrt q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call <4 x i32> @llvm.arm.mve.orr.predicated.v4i32.v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i1> %1, <4 x i32> %inactive)
+  ret <4 x i32> %2
+}
+
+declare <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32) #2
+
+declare <4 x i32> @llvm.arm.mve.orr.predicated.v4i32.v4i1(<4 x i32>, <4 x i32>, <4 x i1>, <4 x i32>) #2
+
+define arm_aapcs_vfpcc <8 x half> @test_vorrq_m_f16(<8 x half> %inactive, <8 x half> %a, <8 x half> %b, i16 zeroext %p) local_unnamed_addr #1 {
+; CHECK-LABEL: test_vorrq_m_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vorrt q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast <8 x half> %a to <8 x i16>
+  %1 = bitcast <8 x half> %b to <8 x i16>
+  %2 = zext i16 %p to i32
+  %3 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %2)
+  %4 = bitcast <8 x half> %inactive to <8 x i16>
+  %5 = tail call <8 x i16> @llvm.arm.mve.orr.predicated.v8i16.v8i1(<8 x i16> %0, <8 x i16> %1, <8 x i1> %3, <8 x i16> %4)
+  %6 = bitcast <8 x i16> %5 to <8 x half>
+  ret <8 x half> %6
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vorrq_x_u8(<16 x i8> %a, <16 x i8> %b, i16 zeroext %p) local_unnamed_addr #1 {
+; CHECK-LABEL: test_vorrq_x_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vorrt q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <16 x i8> @llvm.arm.mve.orr.predicated.v16i8.v16i1(<16 x i8> %a, <16 x i8> %b, <16 x i1> %1, <16 x i8> undef)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vorrq_x_s16(<8 x i16> %a, <8 x i16> %b, i16 zeroext %p) local_unnamed_addr #1 {
+; CHECK-LABEL: test_vorrq_x_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vorrt q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.orr.predicated.v8i16.v8i1(<8 x i16> %a, <8 x i16> %b, <8 x i1> %1, <8 x i16> undef)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vorrq_x_u32(<4 x i32> %a, <4 x i32> %b, i16 zeroext %p) local_unnamed_addr #1 {
+; CHECK-LABEL: test_vorrq_x_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vorrt q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call <4 x i32> @llvm.arm.mve.orr.predicated.v4i32.v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i1> %1, <4 x i32> undef)
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <4 x float> @test_vorrq_m_f32(<4 x float> %a, <4 x float> %b, i16 zeroext %p) local_unnamed_addr #1 {
+; CHECK-LABEL: test_vorrq_m_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vorrt q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast <4 x float> %a to <4 x i32>
+  %1 = bitcast <4 x float> %b to <4 x i32>
+  %2 = zext i16 %p to i32
+  %3 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %2)
+  %4 = tail call <4 x i32> @llvm.arm.mve.orr.predicated.v4i32.v4i1(<4 x i32> %0, <4 x i32> %1, <4 x i1> %3, <4 x i32> undef)
+  %5 = bitcast <4 x i32> %4 to <4 x float>
+  ret <4 x float> %5
+}
+
diff --git a/llvm/test/CodeGen/Thumb2/mve-intrinsics/vqaddq.ll b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vqaddq.ll
new file mode 100644
index 000000000..b05fded2b
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vqaddq.ll
@@ -0,0 +1,92 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve.fp -verify-machineinstrs -o - %s | FileCheck %s
+
+define arm_aapcs_vfpcc <16 x i8> @test_vqaddq_u8(<16 x i8> %a, <16 x i8> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vqaddq_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqadd.u8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> %a, <16 x i8> %b)
+  ret <16 x i8> %0
+}
+
+declare <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8>, <16 x i8>) #1
+
+define arm_aapcs_vfpcc <8 x i16> @test_vqaddq_s16(<8 x i16> %a, <8 x i16> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vqaddq_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqadd.s16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> %a, <8 x i16> %b)
+  ret <8 x i16> %0
+}
+
+declare <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16>, <8 x i16>) #1
+
+define arm_aapcs_vfpcc <4 x i32> @test_vqaddq_u32(<4 x i32> %a, <4 x i32> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vqaddq_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqadd.u32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> %a, <4 x i32> %b)
+  ret <4 x i32> %0
+}
+
+declare <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32>, <4 x i32>) #1
+
+define arm_aapcs_vfpcc <16 x i8> @test_vqaddq_m_s8(<16 x i8> %inactive, <16 x i8> %a, <16 x i8> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vqaddq_m_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqaddt.s8 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <16 x i8> @llvm.arm.mve.qadd.predicated.v16i8.v16i1(<16 x i8> %a, <16 x i8> %b, i32 0, <16 x i1> %1, <16 x i8> %inactive)
+  ret <16 x i8> %2
+}
+
+declare <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32) #2
+
+declare <16 x i8> @llvm.arm.mve.qadd.predicated.v16i8.v16i1(<16 x i8>, <16 x i8>, i32, <16 x i1>, <16 x i8>) #2
+
+define arm_aapcs_vfpcc <8 x i16> @test_vqaddq_m_u16(<8 x i16> %inactive, <8 x i16> %a, <8 x i16> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vqaddq_m_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqaddt.u16 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.qadd.predicated.v8i16.v8i1(<8 x i16> %a, <8 x i16> %b, i32 1, <8 x i1> %1, <8 x i16> %inactive)
+  ret <8 x i16> %2
+}
+
+declare <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32) #2
+
+declare <8 x i16> @llvm.arm.mve.qadd.predicated.v8i16.v8i1(<8 x i16>, <8 x i16>, i32, <8 x i1>, <8 x i16>) #2
+
+define arm_aapcs_vfpcc <4 x i32> @test_vqaddq_m_s32(<4 x i32> %inactive, <4 x i32> %a, <4 x i32> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vqaddq_m_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqaddt.s32 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call <4 x i32> @llvm.arm.mve.qadd.predicated.v4i32.v4i1(<4 x i32> %a, <4 x i32> %b, i32 0, <4 x i1> %1, <4 x i32> %inactive)
+  ret <4 x i32> %2
+}
+
+declare <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32) #2
+
+declare <4 x i32> @llvm.arm.mve.qadd.predicated.v4i32.v4i1(<4 x i32>, <4 x i32>, i32, <4 x i1>, <4 x i32>) #2
diff --git a/llvm/test/CodeGen/Thumb2/mve-intrinsics/vqdmulhq.ll b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vqdmulhq.ll
new file mode 100644
index 000000000..58e54a9d2
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vqdmulhq.ll
@@ -0,0 +1,92 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve.fp -verify-machineinstrs -o - %s | FileCheck %s
+
+define arm_aapcs_vfpcc <16 x i8> @test_vqdmulhq_s8(<16 x i8> %a, <16 x i8> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vqdmulhq_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqdmulh.s8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <16 x i8> @llvm.arm.mve.vqdmulh.v16i8(<16 x i8> %a, <16 x i8> %b)
+  ret <16 x i8> %0
+}
+
+declare <16 x i8> @llvm.arm.mve.vqdmulh.v16i8(<16 x i8>, <16 x i8>) #1
+
+define arm_aapcs_vfpcc <8 x i16> @test_vqdmulhq_s16(<8 x i16> %a, <8 x i16> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vqdmulhq_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqdmulh.s16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <8 x i16> @llvm.arm.mve.vqdmulh.v8i16(<8 x i16> %a, <8 x i16> %b)
+  ret <8 x i16> %0
+}
+
+declare <8 x i16> @llvm.arm.mve.vqdmulh.v8i16(<8 x i16>, <8 x i16>) #1
+
+define arm_aapcs_vfpcc <4 x i32> @test_vqdmulhq_s32(<4 x i32> %a, <4 x i32> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vqdmulhq_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqdmulh.s32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <4 x i32> @llvm.arm.mve.vqdmulh.v4i32(<4 x i32> %a, <4 x i32> %b)
+  ret <4 x i32> %0
+}
+
+declare <4 x i32> @llvm.arm.mve.vqdmulh.v4i32(<4 x i32>, <4 x i32>) #1
+
+define arm_aapcs_vfpcc <16 x i8> @test_vqdmulhq_m_s8(<16 x i8> %inactive, <16 x i8> %a, <16 x i8> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vqdmulhq_m_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqdmulht.s8 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <16 x i8> @llvm.arm.mve.qdmulh.predicated.v16i8.v16i1(<16 x i8> %a, <16 x i8> %b, <16 x i1> %1, <16 x i8> %inactive)
+  ret <16 x i8> %2
+}
+
+declare <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32) #1
+
+declare <16 x i8> @llvm.arm.mve.qdmulh.predicated.v16i8.v16i1(<16 x i8>, <16 x i8>, <16 x i1>, <16 x i8>) #1
+
+define arm_aapcs_vfpcc <8 x i16> @test_vqdmulhq_m_s16(<8 x i16> %inactive, <8 x i16> %a, <8 x i16> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vqdmulhq_m_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqdmulht.s16 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.qdmulh.predicated.v8i16.v8i1(<8 x i16> %a, <8 x i16> %b, <8 x i1> %1, <8 x i16> %inactive)
+  ret <8 x i16> %2
+}
+
+declare <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32) #1
+
+declare <8 x i16> @llvm.arm.mve.qdmulh.predicated.v8i16.v8i1(<8 x i16>, <8 x i16>, <8 x i1>, <8 x i16>) #1
+
+define arm_aapcs_vfpcc <4 x i32> @test_vqdmulhq_m_s32(<4 x i32> %inactive, <4 x i32> %a, <4 x i32> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vqdmulhq_m_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqdmulht.s32 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call <4 x i32> @llvm.arm.mve.qdmulh.predicated.v4i32.v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i1> %1, <4 x i32> %inactive)
+  ret <4 x i32> %2
+}
+
+declare <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32) #1
+
+declare <4 x i32> @llvm.arm.mve.qdmulh.predicated.v4i32.v4i1(<4 x i32>, <4 x i32>, <4 x i1>, <4 x i32>) #1
diff --git a/llvm/test/CodeGen/Thumb2/mve-intrinsics/vqrdmulhq.ll b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vqrdmulhq.ll
new file mode 100644
index 000000000..806fb7bc9
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vqrdmulhq.ll
@@ -0,0 +1,92 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve.fp -verify-machineinstrs -o - %s | FileCheck %s
+
+define arm_aapcs_vfpcc <16 x i8> @test_vqrdmulhq_s8(<16 x i8> %a, <16 x i8> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vqrdmulhq_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqrdmulh.s8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <16 x i8> @llvm.arm.mve.vqrdmulh.v16i8(<16 x i8> %a, <16 x i8> %b)
+  ret <16 x i8> %0
+}
+
+declare <16 x i8> @llvm.arm.mve.vqrdmulh.v16i8(<16 x i8>, <16 x i8>) #1
+
+define arm_aapcs_vfpcc <8 x i16> @test_vqrdmulhq_s16(<8 x i16> %a, <8 x i16> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vqrdmulhq_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqrdmulh.s16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <8 x i16> @llvm.arm.mve.vqrdmulh.v8i16(<8 x i16> %a, <8 x i16> %b)
+  ret <8 x i16> %0
+}
+
+declare <8 x i16> @llvm.arm.mve.vqrdmulh.v8i16(<8 x i16>, <8 x i16>) #1
+
+define arm_aapcs_vfpcc <4 x i32> @test_vqrdmulhq_s32(<4 x i32> %a, <4 x i32> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vqrdmulhq_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqrdmulh.s32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <4 x i32> @llvm.arm.mve.vqrdmulh.v4i32(<4 x i32> %a, <4 x i32> %b)
+  ret <4 x i32> %0
+}
+
+declare <4 x i32> @llvm.arm.mve.vqrdmulh.v4i32(<4 x i32>, <4 x i32>) #1
+
+define arm_aapcs_vfpcc <16 x i8> @test_vqrdmulhq_m_s8(<16 x i8> %inactive, <16 x i8> %a, <16 x i8> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vqrdmulhq_m_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqrdmulht.s8 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <16 x i8> @llvm.arm.mve.qrdmulh.predicated.v16i8.v16i1(<16 x i8> %a, <16 x i8> %b, <16 x i1> %1, <16 x i8> %inactive)
+  ret <16 x i8> %2
+}
+
+declare <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32) #1
+
+declare <16 x i8> @llvm.arm.mve.qrdmulh.predicated.v16i8.v16i1(<16 x i8>, <16 x i8>, <16 x i1>, <16 x i8>) #1
+
+define arm_aapcs_vfpcc <8 x i16> @test_vqrdmulhq_m_s16(<8 x i16> %inactive, <8 x i16> %a, <8 x i16> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vqrdmulhq_m_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqrdmulht.s16 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.qrdmulh.predicated.v8i16.v8i1(<8 x i16> %a, <8 x i16> %b, <8 x i1> %1, <8 x i16> %inactive)
+  ret <8 x i16> %2
+}
+
+declare <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32) #1
+
+declare <8 x i16> @llvm.arm.mve.qrdmulh.predicated.v8i16.v8i1(<8 x i16>, <8 x i16>, <8 x i1>, <8 x i16>) #1
+
+define arm_aapcs_vfpcc <4 x i32> @test_vqrdmulhq_m_s32(<4 x i32> %inactive, <4 x i32> %a, <4 x i32> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vqrdmulhq_m_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqrdmulht.s32 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call <4 x i32> @llvm.arm.mve.qrdmulh.predicated.v4i32.v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i1> %1, <4 x i32> %inactive)
+  ret <4 x i32> %2
+}
+
+declare <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32) #1
+
+declare <4 x i32> @llvm.arm.mve.qrdmulh.predicated.v4i32.v4i1(<4 x i32>, <4 x i32>, <4 x i1>, <4 x i32>) #1
diff --git a/llvm/test/CodeGen/Thumb2/mve-intrinsics/vqsubq.ll b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vqsubq.ll
new file mode 100644
index 000000000..37f251c83
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vqsubq.ll
@@ -0,0 +1,92 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve.fp -verify-machineinstrs -o - %s | FileCheck %s
+
+define arm_aapcs_vfpcc <16 x i8> @test_vqsubq_u8(<16 x i8> %a, <16 x i8> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vqsubq_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqsub.u8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> %a, <16 x i8> %b)
+  ret <16 x i8> %0
+}
+
+declare <16 x i8> @llvm.usub.sat.v16i8(<16 x i8>, <16 x i8>) #1
+
+define arm_aapcs_vfpcc <8 x i16> @test_vqsubq_s16(<8 x i16> %a, <8 x i16> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vqsubq_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqsub.s16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> %a, <8 x i16> %b)
+  ret <8 x i16> %0
+}
+
+declare <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16>, <8 x i16>) #1
+
+define arm_aapcs_vfpcc <4 x i32> @test_vqsubq_u32(<4 x i32> %a, <4 x i32> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vqsubq_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqsub.u32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> %a, <4 x i32> %b)
+  ret <4 x i32> %0
+}
+
+declare <4 x i32> @llvm.usub.sat.v4i32(<4 x i32>, <4 x i32>) #1
+
+define arm_aapcs_vfpcc <16 x i8> @test_vqsubq_m_s8(<16 x i8> %inactive, <16 x i8> %a, <16 x i8> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vqsubq_m_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqsubt.s8 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <16 x i8> @llvm.arm.mve.qsub.predicated.v16i8.v16i1(<16 x i8> %a, <16 x i8> %b, i32 0, <16 x i1> %1, <16 x i8> %inactive)
+  ret <16 x i8> %2
+}
+
+declare <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32) #2
+
+declare <16 x i8> @llvm.arm.mve.qsub.predicated.v16i8.v16i1(<16 x i8>, <16 x i8>, i32, <16 x i1>, <16 x i8>) #2
+
+define arm_aapcs_vfpcc <8 x i16> @test_vqsubq_m_u16(<8 x i16> %inactive, <8 x i16> %a, <8 x i16> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vqsubq_m_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqsubt.u16 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.qsub.predicated.v8i16.v8i1(<8 x i16> %a, <8 x i16> %b, i32 1, <8 x i1> %1, <8 x i16> %inactive)
+  ret <8 x i16> %2
+}
+
+declare <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32) #2
+
+declare <8 x i16> @llvm.arm.mve.qsub.predicated.v8i16.v8i1(<8 x i16>, <8 x i16>, i32, <8 x i1>, <8 x i16>) #2
+
+define arm_aapcs_vfpcc <4 x i32> @test_vqsubq_m_s32(<4 x i32> %inactive, <4 x i32> %a, <4 x i32> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vqsubq_m_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqsubt.s32 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call <4 x i32> @llvm.arm.mve.qsub.predicated.v4i32.v4i1(<4 x i32> %a, <4 x i32> %b, i32 0, <4 x i1> %1, <4 x i32> %inactive)
+  ret <4 x i32> %2
+}
+
+declare <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32) #2
+
+declare <4 x i32> @llvm.arm.mve.qsub.predicated.v4i32.v4i1(<4 x i32>, <4 x i32>, i32, <4 x i1>, <4 x i32>) #2
diff --git a/llvm/test/CodeGen/Thumb2/mve-intrinsics/vrhaddq.ll b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vrhaddq.ll
new file mode 100644
index 000000000..d7e39548e
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vrhaddq.ll
@@ -0,0 +1,135 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve.fp -verify-machineinstrs -o - %s | FileCheck %s
+
+define arm_aapcs_vfpcc <16 x i8> @test_vrhaddq_u8(<16 x i8> %a, <16 x i8> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vrhaddq_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vrhadd.u8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <16 x i8> @llvm.arm.mve.vrhadd.v16i8(<16 x i8> %a, <16 x i8> %b, i32 1)
+  ret <16 x i8> %0
+}
+
+declare <16 x i8> @llvm.arm.mve.vrhadd.v16i8(<16 x i8>, <16 x i8>, i32) #1
+
+define arm_aapcs_vfpcc <8 x i16> @test_vrhaddq_s16(<8 x i16> %a, <8 x i16> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vrhaddq_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vrhadd.s16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <8 x i16> @llvm.arm.mve.vrhadd.v8i16(<8 x i16> %a, <8 x i16> %b, i32 0)
+  ret <8 x i16> %0
+}
+
+declare <8 x i16> @llvm.arm.mve.vrhadd.v8i16(<8 x i16>, <8 x i16>, i32) #1
+
+define arm_aapcs_vfpcc <4 x i32> @test_vrhaddq_u32(<4 x i32> %a, <4 x i32> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vrhaddq_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vrhadd.u32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <4 x i32> @llvm.arm.mve.vrhadd.v4i32(<4 x i32> %a, <4 x i32> %b, i32 1)
+  ret <4 x i32> %0
+}
+
+declare <4 x i32> @llvm.arm.mve.vrhadd.v4i32(<4 x i32>, <4 x i32>, i32) #1
+
+define arm_aapcs_vfpcc <16 x i8> @test_vrhaddq_m_s8(<16 x i8> %inactive, <16 x i8> %a, <16 x i8> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vrhaddq_m_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vrhaddt.s8 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <16 x i8> @llvm.arm.mve.rhadd.predicated.v16i8.v16i1(<16 x i8> %a, <16 x i8> %b, i32 0, <16 x i1> %1, <16 x i8> %inactive)
+  ret <16 x i8> %2
+}
+
+declare <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32) #1
+
+declare <16 x i8> @llvm.arm.mve.rhadd.predicated.v16i8.v16i1(<16 x i8>, <16 x i8>, i32, <16 x i1>, <16 x i8>) #1
+
+define arm_aapcs_vfpcc <8 x i16> @test_vrhaddq_m_u16(<8 x i16> %inactive, <8 x i16> %a, <8 x i16> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vrhaddq_m_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vrhaddt.u16 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.rhadd.predicated.v8i16.v8i1(<8 x i16> %a, <8 x i16> %b, i32 1, <8 x i1> %1, <8 x i16> %inactive)
+  ret <8 x i16> %2
+}
+
+declare <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32) #1
+
+declare <8 x i16> @llvm.arm.mve.rhadd.predicated.v8i16.v8i1(<8 x i16>, <8 x i16>, i32, <8 x i1>, <8 x i16>) #1
+
+define arm_aapcs_vfpcc <4 x i32> @test_vrhaddq_m_s32(<4 x i32> %inactive, <4 x i32> %a, <4 x i32> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vrhaddq_m_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vrhaddt.s32 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call <4 x i32> @llvm.arm.mve.rhadd.predicated.v4i32.v4i1(<4 x i32> %a, <4 x i32> %b, i32 0, <4 x i1> %1, <4 x i32> %inactive)
+  ret <4 x i32> %2
+}
+
+declare <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32) #1
+
+declare <4 x i32> @llvm.arm.mve.rhadd.predicated.v4i32.v4i1(<4 x i32>, <4 x i32>, i32, <4 x i1>, <4 x i32>) #1
+
+define arm_aapcs_vfpcc <16 x i8> @test_vrhaddq_x_u8(<16 x i8> %a, <16 x i8> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vrhaddq_x_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vrhaddt.u8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <16 x i8> @llvm.arm.mve.rhadd.predicated.v16i8.v16i1(<16 x i8> %a, <16 x i8> %b, i32 1, <16 x i1> %1, <16 x i8> undef)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vrhaddq_x_u16(<8 x i16> %a, <8 x i16> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vrhaddq_x_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vrhaddt.u16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.rhadd.predicated.v8i16.v8i1(<8 x i16> %a, <8 x i16> %b, i32 1, <8 x i1> %1, <8 x i16> undef)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vrhaddq_x_u32(<4 x i32> %a, <4 x i32> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vrhaddq_x_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vrhaddt.u32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call <4 x i32> @llvm.arm.mve.rhadd.predicated.v4i32.v4i1(<4 x i32> %a, <4 x i32> %b, i32 1, <4 x i1> %1, <4 x i32> undef)
+  ret <4 x i32> %2
+}
+
diff --git a/llvm/test/CodeGen/Thumb2/mve-intrinsics/vrmulhq.ll b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vrmulhq.ll
new file mode 100644
index 000000000..e8ff21452
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vrmulhq.ll
@@ -0,0 +1,135 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve.fp -verify-machineinstrs -o - %s | FileCheck %s
+
+define arm_aapcs_vfpcc <16 x i8> @test_vrmulhq_u8(<16 x i8> %a, <16 x i8> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vrmulhq_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vrmulh.u8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <16 x i8> @llvm.arm.mve.vrmulh.v16i8(<16 x i8> %a, <16 x i8> %b, i32 1)
+  ret <16 x i8> %0
+}
+
+declare <16 x i8> @llvm.arm.mve.vrmulh.v16i8(<16 x i8>, <16 x i8>, i32) #1
+
+define arm_aapcs_vfpcc <8 x i16> @test_vrmulhq_s16(<8 x i16> %a, <8 x i16> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vrmulhq_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vrmulh.s16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <8 x i16> @llvm.arm.mve.vrmulh.v8i16(<8 x i16> %a, <8 x i16> %b, i32 0)
+  ret <8 x i16> %0
+}
+
+declare <8 x i16> @llvm.arm.mve.vrmulh.v8i16(<8 x i16>, <8 x i16>, i32) #1
+
+define arm_aapcs_vfpcc <4 x i32> @test_vrmulhq_u32(<4 x i32> %a, <4 x i32> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vrmulhq_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vrmulh.u32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <4 x i32> @llvm.arm.mve.vrmulh.v4i32(<4 x i32> %a, <4 x i32> %b, i32 1)
+  ret <4 x i32> %0
+}
+
+declare <4 x i32> @llvm.arm.mve.vrmulh.v4i32(<4 x i32>, <4 x i32>, i32) #1
+
+define arm_aapcs_vfpcc <16 x i8> @test_vrmulhq_m_s8(<16 x i8> %inactive, <16 x i8> %a, <16 x i8> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vrmulhq_m_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vrmulht.s8 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <16 x i8> @llvm.arm.mve.rmulh.predicated.v16i8.v16i1(<16 x i8> %a, <16 x i8> %b, i32 0, <16 x i1> %1, <16 x i8> %inactive)
+  ret <16 x i8> %2
+}
+
+declare <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32) #1
+
+declare <16 x i8> @llvm.arm.mve.rmulh.predicated.v16i8.v16i1(<16 x i8>, <16 x i8>, i32, <16 x i1>, <16 x i8>) #1
+
+define arm_aapcs_vfpcc <8 x i16> @test_vrmulhq_m_u16(<8 x i16> %inactive, <8 x i16> %a, <8 x i16> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vrmulhq_m_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vrmulht.u16 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.rmulh.predicated.v8i16.v8i1(<8 x i16> %a, <8 x i16> %b, i32 1, <8 x i1> %1, <8 x i16> %inactive)
+  ret <8 x i16> %2
+}
+
+declare <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32) #1
+
+declare <8 x i16> @llvm.arm.mve.rmulh.predicated.v8i16.v8i1(<8 x i16>, <8 x i16>, i32, <8 x i1>, <8 x i16>) #1
+
+define arm_aapcs_vfpcc <4 x i32> @test_vrmulhq_m_s32(<4 x i32> %inactive, <4 x i32> %a, <4 x i32> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vrmulhq_m_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vrmulht.s32 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call <4 x i32> @llvm.arm.mve.rmulh.predicated.v4i32.v4i1(<4 x i32> %a, <4 x i32> %b, i32 0, <4 x i1> %1, <4 x i32> %inactive)
+  ret <4 x i32> %2
+}
+
+declare <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32) #1
+
+declare <4 x i32> @llvm.arm.mve.rmulh.predicated.v4i32.v4i1(<4 x i32>, <4 x i32>, i32, <4 x i1>, <4 x i32>) #1
+
+define arm_aapcs_vfpcc <16 x i8> @test_vrmulhq_x_u8(<16 x i8> %a, <16 x i8> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vrmulhq_x_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vrmulht.u8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <16 x i8> @llvm.arm.mve.rmulh.predicated.v16i8.v16i1(<16 x i8> %a, <16 x i8> %b, i32 1, <16 x i1> %1, <16 x i8> undef)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vrmulhq_x_s16(<8 x i16> %a, <8 x i16> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vrmulhq_x_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vrmulht.s16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.rmulh.predicated.v8i16.v8i1(<8 x i16> %a, <8 x i16> %b, i32 0, <8 x i1> %1, <8 x i16> undef)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vrmulhq_m_u32(<4 x i32> %a, <4 x i32> %b, i16 zeroext %p) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vrmulhq_m_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vrmulht.u32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call <4 x i32> @llvm.arm.mve.rmulh.predicated.v4i32.v4i1(<4 x i32> %a, <4 x i32> %b, i32 1, <4 x i1> %1, <4 x i32> undef)
+  ret <4 x i32> %2
+}
+
diff --git a/llvm/test/CodeGen/Thumb2/mve-intrinsics/vsubq.ll b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vsubq.ll
new file mode 100644
index 000000000..2959e61e0
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vsubq.ll
@@ -0,0 +1,93 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve.fp -verify-machineinstrs -o - %s | FileCheck %s
+
+define arm_aapcs_vfpcc <4 x i32> @test_vsubq_u32(<4 x i32> %a, <4 x i32> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vsubq_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vsub.i32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = sub <4 x i32> %a, %b
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <8 x half> @test_vsubq_f16(<8 x half> %a, <8 x half> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: test_vsubq_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vsub.f16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = fsub <8 x half> %a, %b
+  ret <8 x half> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vsubq_m_s8(<16 x i8> %inactive, <16 x i8> %a, <16 x i8> %b, i16 zeroext %p) local_unnamed_addr #1 {
+; CHECK-LABEL: test_vsubq_m_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vsubt.i8 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call <16 x i8> @llvm.arm.mve.sub.predicated.v16i8.v16i1(<16 x i8> %a, <16 x i8> %b, <16 x i1> %1, <16 x i8> %inactive)
+  ret <16 x i8> %2
+}
+
+declare <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32) #2
+
+declare <16 x i8> @llvm.arm.mve.sub.predicated.v16i8.v16i1(<16 x i8>, <16 x i8>, <16 x i1>, <16 x i8>) #2
+
+define arm_aapcs_vfpcc <4 x float> @test_vsubq_m_f32(<4 x float> %inactive, <4 x float> %a, <4 x float> %b, i16 zeroext %p) local_unnamed_addr #1 {
+; CHECK-LABEL: test_vsubq_m_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vsubt.f32 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call <4 x float> @llvm.arm.mve.sub.predicated.v4f32.v4i1(<4 x float> %a, <4 x float> %b, <4 x i1> %1, <4 x float> %inactive)
+  ret <4 x float> %2
+}
+
+declare <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32) #2
+
+declare <4 x float> @llvm.arm.mve.sub.predicated.v4f32.v4i1(<4 x float>, <4 x float>, <4 x i1>, <4 x float>) #2
+
+define arm_aapcs_vfpcc <8 x i16> @test_vsubq_x_u16(<8 x i16> %a, <8 x i16> %b, i16 zeroext %p) local_unnamed_addr #1 {
+; CHECK-LABEL: test_vsubq_x_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vsubt.i16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <8 x i16> @llvm.arm.mve.sub.predicated.v8i16.v8i1(<8 x i16> %a, <8 x i16> %b, <8 x i1> %1, <8 x i16> undef)
+  ret <8 x i16> %2
+}
+
+declare <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32) #2
+
+declare <8 x i16> @llvm.arm.mve.sub.predicated.v8i16.v8i1(<8 x i16>, <8 x i16>, <8 x i1>, <8 x i16>) #2
+
+define arm_aapcs_vfpcc <8 x half> @test_vsubq_x_f16(<8 x half> %a, <8 x half> %b, i16 zeroext %p) local_unnamed_addr #1 {
+; CHECK-LABEL: test_vsubq_x_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vsubt.f16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call <8 x half> @llvm.arm.mve.sub.predicated.v8f16.v8i1(<8 x half> %a, <8 x half> %b, <8 x i1> %1, <8 x half> undef)
+  ret <8 x half> %2
+}
+
+declare <8 x half> @llvm.arm.mve.sub.predicated.v8f16.v8i1(<8 x half>, <8 x half>, <8 x i1>, <8 x half>) #2
+
diff --git a/llvm/test/CodeGen/Thumb2/mve-ldst-offset.ll b/llvm/test/CodeGen/Thumb2/mve-ldst-offset.ll
index 3ad88ff48..54befb26d 100644
--- a/llvm/test/CodeGen/Thumb2/mve-ldst-offset.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-ldst-offset.ll
@@ -1,8 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve %s -o - | FileCheck %s
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-LE
+; RUN: llc -mtriple=thumbebv8.1m.main-arm-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-BE
 
-define i8* @post_ldrwu32_4(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrwu32_4:
+define i8* @ldrwu32_4(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrwu32_4:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vldrw.u32 q0, [r0, #4]
 ; CHECK-NEXT:    vstrw.32 q0, [r1]
@@ -10,14 +11,14 @@ define i8* @post_ldrwu32_4(i8* %x, i8* %y) {
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 4
   %0 = bitcast i8* %z to <4 x i32>*
-  %1 = load <4 x i32>, <4 x i32>* %0, align 8
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
   %2 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %1, <4 x i32>* %2, align 8
+  store <4 x i32> %1, <4 x i32>* %2, align 4
   ret i8* %x
 }
 
-define i8* @post_ldrwu32_3(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrwu32_3:
+define i8* @ldrwu32_3(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrwu32_3:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    adds r2, r0, #3
 ; CHECK-NEXT:    vldrw.u32 q0, [r2]
@@ -26,14 +27,14 @@ define i8* @post_ldrwu32_3(i8* %x, i8* %y) {
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 3
   %0 = bitcast i8* %z to <4 x i32>*
-  %1 = load <4 x i32>, <4 x i32>* %0, align 8
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
   %2 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %1, <4 x i32>* %2, align 8
+  store <4 x i32> %1, <4 x i32>* %2, align 4
   ret i8* %x
 }
 
-define i8* @post_ldrwu32_m4(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrwu32_m4:
+define i8* @ldrwu32_m4(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrwu32_m4:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vldrw.u32 q0, [r0, #-4]
 ; CHECK-NEXT:    vstrw.32 q0, [r1]
@@ -41,30 +42,29 @@ define i8* @post_ldrwu32_m4(i8* %x, i8* %y) {
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 -4
   %0 = bitcast i8* %z to <4 x i32>*
-  %1 = load <4 x i32>, <4 x i32>* %0, align 8
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
   %2 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %1, <4 x i32>* %2, align 8
+  store <4 x i32> %1, <4 x i32>* %2, align 4
   ret i8* %x
 }
 
-define i8* @post_ldrwu32_508(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrwu32_508:
+define i8* @ldrwu32_508(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrwu32_508:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    add.w r2, r0, #508
-; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #508]
 ; CHECK-NEXT:    vstrw.32 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 508
   %0 = bitcast i8* %z to <4 x i32>*
-  %1 = load <4 x i32>, <4 x i32>* %0, align 8
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
   %2 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %1, <4 x i32>* %2, align 8
+  store <4 x i32> %1, <4 x i32>* %2, align 4
   ret i8* %x
 }
 
-define i8* @post_ldrwu32_512(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrwu32_512:
+define i8* @ldrwu32_512(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrwu32_512:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    add.w r2, r0, #512
 ; CHECK-NEXT:    vldrw.u32 q0, [r2]
@@ -73,30 +73,29 @@ define i8* @post_ldrwu32_512(i8* %x, i8* %y) {
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 512
   %0 = bitcast i8* %z to <4 x i32>*
-  %1 = load <4 x i32>, <4 x i32>* %0, align 8
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
   %2 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %1, <4 x i32>* %2, align 8
+  store <4 x i32> %1, <4 x i32>* %2, align 4
   ret i8* %x
 }
 
-define i8* @post_ldrwu32_m508(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrwu32_m508:
+define i8* @ldrwu32_m508(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrwu32_m508:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    sub.w r2, r0, #508
-; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #-508]
 ; CHECK-NEXT:    vstrw.32 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 -508
   %0 = bitcast i8* %z to <4 x i32>*
-  %1 = load <4 x i32>, <4 x i32>* %0, align 8
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
   %2 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %1, <4 x i32>* %2, align 8
+  store <4 x i32> %1, <4 x i32>* %2, align 4
   ret i8* %x
 }
 
-define i8* @post_ldrwu32_m512(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrwu32_m512:
+define i8* @ldrwu32_m512(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrwu32_m512:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    sub.w r2, r0, #512
 ; CHECK-NEXT:    vldrw.u32 q0, [r2]
@@ -105,15 +104,15 @@ define i8* @post_ldrwu32_m512(i8* %x, i8* %y) {
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 -512
   %0 = bitcast i8* %z to <4 x i32>*
-  %1 = load <4 x i32>, <4 x i32>* %0, align 8
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
   %2 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %1, <4 x i32>* %2, align 8
+  store <4 x i32> %1, <4 x i32>* %2, align 4
   ret i8* %x
 }
 
 
-define i8* @post_ldrhu32_4(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrhu32_4:
+define i8* @ldrhu32_4(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrhu32_4:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vldrh.u32 q0, [r0, #4]
 ; CHECK-NEXT:    vstrw.32 q0, [r1]
@@ -121,15 +120,15 @@ define i8* @post_ldrhu32_4(i8* %x, i8* %y) {
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 4
   %0 = bitcast i8* %z to <4 x i16>*
-  %1 = load <4 x i16>, <4 x i16>* %0, align 8
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
   %2 = zext <4 x i16> %1 to <4 x i32>
   %3 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %2, <4 x i32>* %3, align 8
+  store <4 x i32> %2, <4 x i32>* %3, align 4
   ret i8* %x
 }
 
-define i8* @post_ldrhu32_3(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrhu32_3:
+define i8* @ldrhu32_3(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrhu32_3:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    adds r2, r0, #3
 ; CHECK-NEXT:    vldrh.u32 q0, [r2]
@@ -138,49 +137,47 @@ define i8* @post_ldrhu32_3(i8* %x, i8* %y) {
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 3
   %0 = bitcast i8* %z to <4 x i16>*
-  %1 = load <4 x i16>, <4 x i16>* %0, align 8
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
   %2 = zext <4 x i16> %1 to <4 x i32>
   %3 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %2, <4 x i32>* %3, align 8
+  store <4 x i32> %2, <4 x i32>* %3, align 4
   ret i8* %x
 }
 
-define i8* @post_ldrhu32_2(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrhu32_2:
+define i8* @ldrhu32_2(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrhu32_2:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    adds r2, r0, #2
-; CHECK-NEXT:    vldrh.u32 q0, [r2]
+; CHECK-NEXT:    vldrh.u32 q0, [r0, #2]
 ; CHECK-NEXT:    vstrw.32 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 2
   %0 = bitcast i8* %z to <4 x i16>*
-  %1 = load <4 x i16>, <4 x i16>* %0, align 8
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
   %2 = zext <4 x i16> %1 to <4 x i32>
   %3 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %2, <4 x i32>* %3, align 8
+  store <4 x i32> %2, <4 x i32>* %3, align 4
   ret i8* %x
 }
 
-define i8* @post_ldrhu32_254(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrhu32_254:
+define i8* @ldrhu32_254(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrhu32_254:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    add.w r2, r0, #254
-; CHECK-NEXT:    vldrh.u32 q0, [r2]
+; CHECK-NEXT:    vldrh.u32 q0, [r0, #254]
 ; CHECK-NEXT:    vstrw.32 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 254
   %0 = bitcast i8* %z to <4 x i16>*
-  %1 = load <4 x i16>, <4 x i16>* %0, align 8
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
   %2 = zext <4 x i16> %1 to <4 x i32>
   %3 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %2, <4 x i32>* %3, align 8
+  store <4 x i32> %2, <4 x i32>* %3, align 4
   ret i8* %x
 }
 
-define i8* @post_ldrhu32_256(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrhu32_256:
+define i8* @ldrhu32_256(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrhu32_256:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    add.w r2, r0, #256
 ; CHECK-NEXT:    vldrh.u32 q0, [r2]
@@ -189,16 +186,49 @@ define i8* @post_ldrhu32_256(i8* %x, i8* %y) {
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 256
   %0 = bitcast i8* %z to <4 x i16>*
-  %1 = load <4 x i16>, <4 x i16>* %0, align 8
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
+  %2 = zext <4 x i16> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %x
+}
+
+define i8* @ldrhu32_m254(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrhu32_m254:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u32 q0, [r0, #-254]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -254
+  %0 = bitcast i8* %z to <4 x i16>*
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
   %2 = zext <4 x i16> %1 to <4 x i32>
   %3 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %2, <4 x i32>* %3, align 8
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %x
+}
+
+define i8* @ldrhu32_m256(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrhu32_m256:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    sub.w r2, r0, #256
+; CHECK-NEXT:    vldrh.u32 q0, [r2]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -256
+  %0 = bitcast i8* %z to <4 x i16>*
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
+  %2 = zext <4 x i16> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
   ret i8* %x
 }
 
 
-define i8* @post_ldrhs32_4(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrhs32_4:
+define i8* @ldrhs32_4(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrhs32_4:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vldrh.s32 q0, [r0, #4]
 ; CHECK-NEXT:    vstrw.32 q0, [r1]
@@ -206,15 +236,15 @@ define i8* @post_ldrhs32_4(i8* %x, i8* %y) {
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 4
   %0 = bitcast i8* %z to <4 x i16>*
-  %1 = load <4 x i16>, <4 x i16>* %0, align 8
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
   %2 = sext <4 x i16> %1 to <4 x i32>
   %3 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %2, <4 x i32>* %3, align 8
+  store <4 x i32> %2, <4 x i32>* %3, align 4
   ret i8* %x
 }
 
-define i8* @post_ldrhs32_3(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrhs32_3:
+define i8* @ldrhs32_3(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrhs32_3:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    adds r2, r0, #3
 ; CHECK-NEXT:    vldrh.s32 q0, [r2]
@@ -223,49 +253,47 @@ define i8* @post_ldrhs32_3(i8* %x, i8* %y) {
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 3
   %0 = bitcast i8* %z to <4 x i16>*
-  %1 = load <4 x i16>, <4 x i16>* %0, align 8
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
   %2 = sext <4 x i16> %1 to <4 x i32>
   %3 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %2, <4 x i32>* %3, align 8
+  store <4 x i32> %2, <4 x i32>* %3, align 4
   ret i8* %x
 }
 
-define i8* @post_ldrhs32_2(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrhs32_2:
+define i8* @ldrhs32_2(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrhs32_2:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    adds r2, r0, #2
-; CHECK-NEXT:    vldrh.s32 q0, [r2]
+; CHECK-NEXT:    vldrh.s32 q0, [r0, #2]
 ; CHECK-NEXT:    vstrw.32 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 2
   %0 = bitcast i8* %z to <4 x i16>*
-  %1 = load <4 x i16>, <4 x i16>* %0, align 8
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
   %2 = sext <4 x i16> %1 to <4 x i32>
   %3 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %2, <4 x i32>* %3, align 8
+  store <4 x i32> %2, <4 x i32>* %3, align 4
   ret i8* %x
 }
 
-define i8* @post_ldrhs32_254(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrhs32_254:
+define i8* @ldrhs32_254(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrhs32_254:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    add.w r2, r0, #254
-; CHECK-NEXT:    vldrh.s32 q0, [r2]
+; CHECK-NEXT:    vldrh.s32 q0, [r0, #254]
 ; CHECK-NEXT:    vstrw.32 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 254
   %0 = bitcast i8* %z to <4 x i16>*
-  %1 = load <4 x i16>, <4 x i16>* %0, align 8
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
   %2 = sext <4 x i16> %1 to <4 x i32>
   %3 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %2, <4 x i32>* %3, align 8
+  store <4 x i32> %2, <4 x i32>* %3, align 4
   ret i8* %x
 }
 
-define i8* @post_ldrhs32_256(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrhs32_256:
+define i8* @ldrhs32_256(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrhs32_256:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    add.w r2, r0, #256
 ; CHECK-NEXT:    vldrh.s32 q0, [r2]
@@ -274,96 +302,158 @@ define i8* @post_ldrhs32_256(i8* %x, i8* %y) {
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 256
   %0 = bitcast i8* %z to <4 x i16>*
-  %1 = load <4 x i16>, <4 x i16>* %0, align 8
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
   %2 = sext <4 x i16> %1 to <4 x i32>
   %3 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %2, <4 x i32>* %3, align 8
+  store <4 x i32> %2, <4 x i32>* %3, align 4
   ret i8* %x
 }
 
+define i8* @ldrhs32_m254(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrhs32_m254:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.s32 q0, [r0, #-254]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -254
+  %0 = bitcast i8* %z to <4 x i16>*
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
+  %2 = sext <4 x i16> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %x
+}
 
-define i8* @post_ldrhu16_4(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrhu16_4:
+define i8* @ldrhs32_m256(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrhs32_m256:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r0, #4]
+; CHECK-NEXT:    sub.w r2, r0, #256
+; CHECK-NEXT:    vldrh.s32 q0, [r2]
 ; CHECK-NEXT:    vstrw.32 q0, [r1]
 ; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -256
+  %0 = bitcast i8* %z to <4 x i16>*
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
+  %2 = sext <4 x i16> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %x
+}
+
+
+define i8* @ldrhu16_4(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrhu16_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r0, #4]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 4
   %0 = bitcast i8* %z to <8 x i16>*
-  %1 = load <8 x i16>, <8 x i16>* %0, align 8
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
   %2 = bitcast i8* %y to <8 x i16>*
-  store <8 x i16> %1, <8 x i16>* %2, align 8
+  store <8 x i16> %1, <8 x i16>* %2, align 2
   ret i8* %x
 }
 
-define i8* @post_ldrhu16_3(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrhu16_3:
+define i8* @ldrhu16_3(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrhu16_3:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    adds r2, r0, #3
-; CHECK-NEXT:    vldrw.u32 q0, [r2]
-; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 3
   %0 = bitcast i8* %z to <8 x i16>*
-  %1 = load <8 x i16>, <8 x i16>* %0, align 8
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
   %2 = bitcast i8* %y to <8 x i16>*
-  store <8 x i16> %1, <8 x i16>* %2, align 8
+  store <8 x i16> %1, <8 x i16>* %2, align 2
   ret i8* %x
 }
 
-define i8* @post_ldrhu16_2(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrhu16_2:
+define i8* @ldrhu16_2(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrhu16_2:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    adds r2, r0, #2
-; CHECK-NEXT:    vldrw.u32 q0, [r2]
-; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q0, [r0, #2]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 2
   %0 = bitcast i8* %z to <8 x i16>*
-  %1 = load <8 x i16>, <8 x i16>* %0, align 8
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
   %2 = bitcast i8* %y to <8 x i16>*
-  store <8 x i16> %1, <8 x i16>* %2, align 8
+  store <8 x i16> %1, <8 x i16>* %2, align 2
   ret i8* %x
 }
 
-define i8* @post_ldrhu16_254(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrhu16_254:
+define i8* @ldrhu16_254(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrhu16_254:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    add.w r2, r0, #254
-; CHECK-NEXT:    vldrw.u32 q0, [r2]
-; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q0, [r0, #254]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 254
   %0 = bitcast i8* %z to <8 x i16>*
-  %1 = load <8 x i16>, <8 x i16>* %0, align 8
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
   %2 = bitcast i8* %y to <8 x i16>*
-  store <8 x i16> %1, <8 x i16>* %2, align 8
+  store <8 x i16> %1, <8 x i16>* %2, align 2
   ret i8* %x
 }
 
-define i8* @post_ldrhu16_256(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrhu16_256:
+define i8* @ldrhu16_256(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrhu16_256:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    add.w r2, r0, #256
-; CHECK-NEXT:    vldrw.u32 q0, [r2]
-; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 256
   %0 = bitcast i8* %z to <8 x i16>*
-  %1 = load <8 x i16>, <8 x i16>* %0, align 8
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
   %2 = bitcast i8* %y to <8 x i16>*
-  store <8 x i16> %1, <8 x i16>* %2, align 8
+  store <8 x i16> %1, <8 x i16>* %2, align 2
+  ret i8* %x
+}
+
+define i8* @ldrhu16_m254(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrhu16_m254:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r0, #-254]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -254
+  %0 = bitcast i8* %z to <8 x i16>*
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
+  %2 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %1, <8 x i16>* %2, align 2
+  ret i8* %x
+}
+
+define i8* @ldrhu16_m256(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrhu16_m256:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    sub.w r2, r0, #256
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -256
+  %0 = bitcast i8* %z to <8 x i16>*
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
+  %2 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %1, <8 x i16>* %2, align 2
   ret i8* %x
 }
 
 
-define i8* @post_ldrbu32_4(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrbu32_4:
+define i8* @ldrbu32_4(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbu32_4:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vldrb.u32 q0, [r0, #4]
 ; CHECK-NEXT:    vstrw.32 q0, [r1]
@@ -371,49 +461,47 @@ define i8* @post_ldrbu32_4(i8* %x, i8* %y) {
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 4
   %0 = bitcast i8* %z to <4 x i8>*
-  %1 = load <4 x i8>, <4 x i8>* %0, align 8
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
   %2 = zext <4 x i8> %1 to <4 x i32>
   %3 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %2, <4 x i32>* %3, align 8
+  store <4 x i32> %2, <4 x i32>* %3, align 4
   ret i8* %x
 }
 
-define i8* @post_ldrbu32_3(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrbu32_3:
+define i8* @ldrbu32_3(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbu32_3:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    adds r2, r0, #3
-; CHECK-NEXT:    vldrb.u32 q0, [r2]
+; CHECK-NEXT:    vldrb.u32 q0, [r0, #3]
 ; CHECK-NEXT:    vstrw.32 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 3
   %0 = bitcast i8* %z to <4 x i8>*
-  %1 = load <4 x i8>, <4 x i8>* %0, align 8
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
   %2 = zext <4 x i8> %1 to <4 x i32>
   %3 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %2, <4 x i32>* %3, align 8
+  store <4 x i32> %2, <4 x i32>* %3, align 4
   ret i8* %x
 }
 
-define i8* @post_ldrbu32_127(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrbu32_127:
+define i8* @ldrbu32_127(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbu32_127:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    add.w r2, r0, #127
-; CHECK-NEXT:    vldrb.u32 q0, [r2]
+; CHECK-NEXT:    vldrb.u32 q0, [r0, #127]
 ; CHECK-NEXT:    vstrw.32 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 127
   %0 = bitcast i8* %z to <4 x i8>*
-  %1 = load <4 x i8>, <4 x i8>* %0, align 8
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
   %2 = zext <4 x i8> %1 to <4 x i32>
   %3 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %2, <4 x i32>* %3, align 8
+  store <4 x i32> %2, <4 x i32>* %3, align 4
   ret i8* %x
 }
 
-define i8* @post_ldrbu32_128(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrbu32_128:
+define i8* @ldrbu32_128(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbu32_128:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    add.w r2, r0, #128
 ; CHECK-NEXT:    vldrb.u32 q0, [r2]
@@ -422,16 +510,49 @@ define i8* @post_ldrbu32_128(i8* %x, i8* %y) {
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 128
   %0 = bitcast i8* %z to <4 x i8>*
-  %1 = load <4 x i8>, <4 x i8>* %0, align 8
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
   %2 = zext <4 x i8> %1 to <4 x i32>
   %3 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %2, <4 x i32>* %3, align 8
+  store <4 x i32> %2, <4 x i32>* %3, align 4
   ret i8* %x
 }
 
+define i8* @ldrbu32_m127(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbu32_m127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u32 q0, [r0, #-127]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -127
+  %0 = bitcast i8* %z to <4 x i8>*
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
+  %2 = zext <4 x i8> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %x
+}
 
-define i8* @post_ldrbs32_4(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrbs32_4:
+define i8* @ldrbu32_m128(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbu32_m128:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    sub.w r2, r0, #128
+; CHECK-NEXT:    vldrb.u32 q0, [r2]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -128
+  %0 = bitcast i8* %z to <4 x i8>*
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
+  %2 = zext <4 x i8> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %x
+}
+
+
+define i8* @ldrbs32_4(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbs32_4:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vldrb.s32 q0, [r0, #4]
 ; CHECK-NEXT:    vstrw.32 q0, [r1]
@@ -439,49 +560,47 @@ define i8* @post_ldrbs32_4(i8* %x, i8* %y) {
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 4
   %0 = bitcast i8* %z to <4 x i8>*
-  %1 = load <4 x i8>, <4 x i8>* %0, align 8
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
   %2 = sext <4 x i8> %1 to <4 x i32>
   %3 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %2, <4 x i32>* %3, align 8
+  store <4 x i32> %2, <4 x i32>* %3, align 4
   ret i8* %x
 }
 
-define i8* @post_ldrbs32_3(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrbs32_3:
+define i8* @ldrbs32_3(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbs32_3:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    adds r2, r0, #3
-; CHECK-NEXT:    vldrb.s32 q0, [r2]
+; CHECK-NEXT:    vldrb.s32 q0, [r0, #3]
 ; CHECK-NEXT:    vstrw.32 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 3
   %0 = bitcast i8* %z to <4 x i8>*
-  %1 = load <4 x i8>, <4 x i8>* %0, align 8
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
   %2 = sext <4 x i8> %1 to <4 x i32>
   %3 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %2, <4 x i32>* %3, align 8
+  store <4 x i32> %2, <4 x i32>* %3, align 4
   ret i8* %x
 }
 
-define i8* @post_ldrbs32_127(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrbs32_127:
+define i8* @ldrbs32_127(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbs32_127:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    add.w r2, r0, #127
-; CHECK-NEXT:    vldrb.s32 q0, [r2]
+; CHECK-NEXT:    vldrb.s32 q0, [r0, #127]
 ; CHECK-NEXT:    vstrw.32 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 127
   %0 = bitcast i8* %z to <4 x i8>*
-  %1 = load <4 x i8>, <4 x i8>* %0, align 8
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
   %2 = sext <4 x i8> %1 to <4 x i32>
   %3 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %2, <4 x i32>* %3, align 8
+  store <4 x i32> %2, <4 x i32>* %3, align 4
   ret i8* %x
 }
 
-define i8* @post_ldrbs32_128(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrbs32_128:
+define i8* @ldrbs32_128(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbs32_128:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    add.w r2, r0, #128
 ; CHECK-NEXT:    vldrb.s32 q0, [r2]
@@ -490,214 +609,340 @@ define i8* @post_ldrbs32_128(i8* %x, i8* %y) {
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 128
   %0 = bitcast i8* %z to <4 x i8>*
-  %1 = load <4 x i8>, <4 x i8>* %0, align 8
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
   %2 = sext <4 x i8> %1 to <4 x i32>
   %3 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %2, <4 x i32>* %3, align 8
+  store <4 x i32> %2, <4 x i32>* %3, align 4
   ret i8* %x
 }
 
+define i8* @ldrbs32_m127(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbs32_m127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.s32 q0, [r0, #-127]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -127
+  %0 = bitcast i8* %z to <4 x i8>*
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
+  %2 = sext <4 x i8> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %x
+}
 
-define i8* @post_ldrbu16_4(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrbu16_4:
+define i8* @ldrbs32_m128(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbs32_m128:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrb.u16 q0, [r0, #4]
+; CHECK-NEXT:    sub.w r2, r0, #128
+; CHECK-NEXT:    vldrb.s32 q0, [r2]
 ; CHECK-NEXT:    vstrw.32 q0, [r1]
 ; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -128
+  %0 = bitcast i8* %z to <4 x i8>*
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
+  %2 = sext <4 x i8> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %x
+}
+
+
+define i8* @ldrbu16_4(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbu16_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u16 q0, [r0, #4]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 4
   %0 = bitcast i8* %z to <8 x i8>*
-  %1 = load <8 x i8>, <8 x i8>* %0, align 8
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
   %2 = zext <8 x i8> %1 to <8 x i16>
   %3 = bitcast i8* %y to <8 x i16>*
-  store <8 x i16> %2, <8 x i16>* %3, align 8
+  store <8 x i16> %2, <8 x i16>* %3, align 2
   ret i8* %x
 }
 
-define i8* @post_ldrbu16_3(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrbu16_3:
+define i8* @ldrbu16_3(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbu16_3:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    adds r2, r0, #3
-; CHECK-NEXT:    vldrb.u16 q0, [r2]
-; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vldrb.u16 q0, [r0, #3]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 3
   %0 = bitcast i8* %z to <8 x i8>*
-  %1 = load <8 x i8>, <8 x i8>* %0, align 8
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
   %2 = zext <8 x i8> %1 to <8 x i16>
   %3 = bitcast i8* %y to <8 x i16>*
-  store <8 x i16> %2, <8 x i16>* %3, align 8
+  store <8 x i16> %2, <8 x i16>* %3, align 2
   ret i8* %x
 }
 
-define i8* @post_ldrbu16_127(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrbu16_127:
+define i8* @ldrbu16_127(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbu16_127:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    add.w r2, r0, #127
-; CHECK-NEXT:    vldrb.u16 q0, [r2]
-; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vldrb.u16 q0, [r0, #127]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 127
   %0 = bitcast i8* %z to <8 x i8>*
-  %1 = load <8 x i8>, <8 x i8>* %0, align 8
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
   %2 = zext <8 x i8> %1 to <8 x i16>
   %3 = bitcast i8* %y to <8 x i16>*
-  store <8 x i16> %2, <8 x i16>* %3, align 8
+  store <8 x i16> %2, <8 x i16>* %3, align 2
   ret i8* %x
 }
 
-define i8* @post_ldrbu16_128(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrbu16_128:
+define i8* @ldrbu16_128(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbu16_128:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    add.w r2, r0, #128
 ; CHECK-NEXT:    vldrb.u16 q0, [r2]
-; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 128
   %0 = bitcast i8* %z to <8 x i8>*
-  %1 = load <8 x i8>, <8 x i8>* %0, align 8
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
   %2 = zext <8 x i8> %1 to <8 x i16>
   %3 = bitcast i8* %y to <8 x i16>*
-  store <8 x i16> %2, <8 x i16>* %3, align 8
+  store <8 x i16> %2, <8 x i16>* %3, align 2
   ret i8* %x
 }
 
+define i8* @ldrbu16_m127(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbu16_m127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u16 q0, [r0, #-127]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -127
+  %0 = bitcast i8* %z to <8 x i8>*
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
+  %2 = zext <8 x i8> %1 to <8 x i16>
+  %3 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %2, <8 x i16>* %3, align 2
+  ret i8* %x
+}
 
-define i8* @post_ldrbs16_4(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrbs16_4:
+define i8* @ldrbu16_m128(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbu16_m128:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    sub.w r2, r0, #128
+; CHECK-NEXT:    vldrb.u16 q0, [r2]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -128
+  %0 = bitcast i8* %z to <8 x i8>*
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
+  %2 = zext <8 x i8> %1 to <8 x i16>
+  %3 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %2, <8 x i16>* %3, align 2
+  ret i8* %x
+}
+
+
+define i8* @ldrbs16_4(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbs16_4:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vldrb.s16 q0, [r0, #4]
-; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 4
   %0 = bitcast i8* %z to <8 x i8>*
-  %1 = load <8 x i8>, <8 x i8>* %0, align 8
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
   %2 = sext <8 x i8> %1 to <8 x i16>
   %3 = bitcast i8* %y to <8 x i16>*
-  store <8 x i16> %2, <8 x i16>* %3, align 8
+  store <8 x i16> %2, <8 x i16>* %3, align 2
   ret i8* %x
 }
 
-define i8* @post_ldrbs16_3(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrbs16_3:
+define i8* @ldrbs16_3(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbs16_3:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    adds r2, r0, #3
-; CHECK-NEXT:    vldrb.s16 q0, [r2]
-; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vldrb.s16 q0, [r0, #3]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 3
   %0 = bitcast i8* %z to <8 x i8>*
-  %1 = load <8 x i8>, <8 x i8>* %0, align 8
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
   %2 = sext <8 x i8> %1 to <8 x i16>
   %3 = bitcast i8* %y to <8 x i16>*
-  store <8 x i16> %2, <8 x i16>* %3, align 8
+  store <8 x i16> %2, <8 x i16>* %3, align 2
   ret i8* %x
 }
 
-define i8* @post_ldrbs16_127(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrbs16_127:
+define i8* @ldrbs16_127(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbs16_127:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    add.w r2, r0, #127
-; CHECK-NEXT:    vldrb.s16 q0, [r2]
-; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vldrb.s16 q0, [r0, #127]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 127
   %0 = bitcast i8* %z to <8 x i8>*
-  %1 = load <8 x i8>, <8 x i8>* %0, align 8
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
   %2 = sext <8 x i8> %1 to <8 x i16>
   %3 = bitcast i8* %y to <8 x i16>*
-  store <8 x i16> %2, <8 x i16>* %3, align 8
+  store <8 x i16> %2, <8 x i16>* %3, align 2
   ret i8* %x
 }
 
-define i8* @post_ldrbs16_128(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrbs16_128:
+define i8* @ldrbs16_128(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbs16_128:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    add.w r2, r0, #128
 ; CHECK-NEXT:    vldrb.s16 q0, [r2]
-; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 128
   %0 = bitcast i8* %z to <8 x i8>*
-  %1 = load <8 x i8>, <8 x i8>* %0, align 8
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
   %2 = sext <8 x i8> %1 to <8 x i16>
   %3 = bitcast i8* %y to <8 x i16>*
-  store <8 x i16> %2, <8 x i16>* %3, align 8
+  store <8 x i16> %2, <8 x i16>* %3, align 2
   ret i8* %x
 }
 
+define i8* @ldrbs16_m127(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbs16_m127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.s16 q0, [r0, #-127]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -127
+  %0 = bitcast i8* %z to <8 x i8>*
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
+  %2 = sext <8 x i8> %1 to <8 x i16>
+  %3 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %2, <8 x i16>* %3, align 2
+  ret i8* %x
+}
 
-define i8* @post_ldrbu8_4(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrbu8_4:
+define i8* @ldrbs16_m128(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbs16_m128:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r0, #4]
-; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    sub.w r2, r0, #128
+; CHECK-NEXT:    vldrb.s16 q0, [r2]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -128
+  %0 = bitcast i8* %z to <8 x i8>*
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
+  %2 = sext <8 x i8> %1 to <8 x i16>
+  %3 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %2, <8 x i16>* %3, align 2
+  ret i8* %x
+}
+
+
+define i8* @ldrbu8_4(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbu8_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u8 q0, [r0, #4]
+; CHECK-NEXT:    vstrb.8 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 4
   %0 = bitcast i8* %z to <16 x i8>*
-  %1 = load <16 x i8>, <16 x i8>* %0, align 8
+  %1 = load <16 x i8>, <16 x i8>* %0, align 1
   %2 = bitcast i8* %y to <16 x i8>*
-  store <16 x i8> %1, <16 x i8>* %2, align 8
+  store <16 x i8> %1, <16 x i8>* %2, align 1
   ret i8* %x
 }
 
-define i8* @post_ldrbu8_3(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrbu8_3:
+define i8* @ldrbu8_3(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbu8_3:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    adds r2, r0, #3
-; CHECK-NEXT:    vldrw.u32 q0, [r2]
-; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vldrb.u8 q0, [r0, #3]
+; CHECK-NEXT:    vstrb.8 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 3
   %0 = bitcast i8* %z to <16 x i8>*
-  %1 = load <16 x i8>, <16 x i8>* %0, align 8
+  %1 = load <16 x i8>, <16 x i8>* %0, align 1
   %2 = bitcast i8* %y to <16 x i8>*
-  store <16 x i8> %1, <16 x i8>* %2, align 8
+  store <16 x i8> %1, <16 x i8>* %2, align 1
   ret i8* %x
 }
 
-define i8* @post_ldrbu8_127(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrbu8_127:
+define i8* @ldrbu8_127(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbu8_127:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    add.w r2, r0, #127
-; CHECK-NEXT:    vldrw.u32 q0, [r2]
-; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vldrb.u8 q0, [r0, #127]
+; CHECK-NEXT:    vstrb.8 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 127
   %0 = bitcast i8* %z to <16 x i8>*
-  %1 = load <16 x i8>, <16 x i8>* %0, align 8
+  %1 = load <16 x i8>, <16 x i8>* %0, align 1
   %2 = bitcast i8* %y to <16 x i8>*
-  store <16 x i8> %1, <16 x i8>* %2, align 8
+  store <16 x i8> %1, <16 x i8>* %2, align 1
   ret i8* %x
 }
 
-define i8* @post_ldrbu8_128(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrbu8_128:
+define i8* @ldrbu8_128(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbu8_128:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r0, #128]
-; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    add.w r2, r0, #128
+; CHECK-NEXT:    vldrb.u8 q0, [r2]
+; CHECK-NEXT:    vstrb.8 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 128
   %0 = bitcast i8* %z to <16 x i8>*
-  %1 = load <16 x i8>, <16 x i8>* %0, align 8
+  %1 = load <16 x i8>, <16 x i8>* %0, align 1
+  %2 = bitcast i8* %y to <16 x i8>*
+  store <16 x i8> %1, <16 x i8>* %2, align 1
+  ret i8* %x
+}
+
+define i8* @ldrbu8_m127(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbu8_m127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u8 q0, [r0, #-127]
+; CHECK-NEXT:    vstrb.8 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -127
+  %0 = bitcast i8* %z to <16 x i8>*
+  %1 = load <16 x i8>, <16 x i8>* %0, align 1
   %2 = bitcast i8* %y to <16 x i8>*
-  store <16 x i8> %1, <16 x i8>* %2, align 8
+  store <16 x i8> %1, <16 x i8>* %2, align 1
   ret i8* %x
 }
 
-define i8* @post_ldrwf32_4(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrwf32_4:
+define i8* @ldrbu8_m128(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbu8_m128:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    sub.w r2, r0, #128
+; CHECK-NEXT:    vldrb.u8 q0, [r2]
+; CHECK-NEXT:    vstrb.8 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -128
+  %0 = bitcast i8* %z to <16 x i8>*
+  %1 = load <16 x i8>, <16 x i8>* %0, align 1
+  %2 = bitcast i8* %y to <16 x i8>*
+  store <16 x i8> %1, <16 x i8>* %2, align 1
+  ret i8* %x
+}
+
+
+define i8* @ldrwf32_4(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrwf32_4:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vldrw.u32 q0, [r0, #4]
 ; CHECK-NEXT:    vstrw.32 q0, [r1]
@@ -705,33 +950,165 @@ define i8* @post_ldrwf32_4(i8* %x, i8* %y) {
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 4
   %0 = bitcast i8* %z to <4 x float>*
-  %1 = load <4 x float>, <4 x float>* %0, align 8
+  %1 = load <4 x float>, <4 x float>* %0, align 4
   %2 = bitcast i8* %y to <4 x float>*
-  store <4 x float> %1, <4 x float>* %2, align 8
+  store <4 x float> %1, <4 x float>* %2, align 4
   ret i8* %x
 }
 
-define i8* @post_ldrwf16_4(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrwf16_4:
+define i8* @ldrwf16_4(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrwf16_4:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r0, #4]
-; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q0, [r0, #4]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 4
   %0 = bitcast i8* %z to <8 x half>*
-  %1 = load <8 x half>, <8 x half>* %0, align 8
+  %1 = load <8 x half>, <8 x half>* %0, align 2
   %2 = bitcast i8* %y to <8 x half>*
-  store <8 x half> %1, <8 x half>* %2, align 8
+  store <8 x half> %1, <8 x half>* %2, align 2
   ret i8* %x
 }
 
+define i8* @ldrwi32_align1(i8* %x, i8* %y) {
+; CHECK-LE-LABEL: ldrwi32_align1:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldrb.u8 q0, [r0, #3]
+; CHECK-LE-NEXT:    vstrw.32 q0, [r1]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: ldrwi32_align1:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vldrb.u8 q0, [r0, #3]
+; CHECK-BE-NEXT:    vrev32.8 q0, q0
+; CHECK-BE-NEXT:    vstrw.32 q0, [r1]
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 3
+  %0 = bitcast i8* %z to <4 x i32>*
+  %1 = load <4 x i32>, <4 x i32>* %0, align 1
+  %2 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %1, <4 x i32>* %2, align 4
+  ret i8* %x
+}
 
+define i8* @ldrhi16_align1(i8* %x, i8* %y) {
+; CHECK-LE-LABEL: ldrhi16_align1:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldrb.u8 q0, [r0, #3]
+; CHECK-LE-NEXT:    vstrh.16 q0, [r1]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: ldrhi16_align1:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vldrb.u8 q0, [r0, #3]
+; CHECK-BE-NEXT:    vrev16.8 q0, q0
+; CHECK-BE-NEXT:    vstrh.16 q0, [r1]
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 3
+  %0 = bitcast i8* %z to <8 x i16>*
+  %1 = load <8 x i16>, <8 x i16>* %0, align 1
+  %2 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %1, <8 x i16>* %2, align 2
+  ret i8* %x
+}
+
+define i8* @ldrhi32_align1(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrhi32_align1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .pad #8
+; CHECK-NEXT:    sub sp, #8
+; CHECK-NEXT:    ldr.w r3, [r0, #7]
+; CHECK-NEXT:    ldr.w r2, [r0, #3]
+; CHECK-NEXT:    strd r2, r3, [sp]
+; CHECK-NEXT:    mov r2, sp
+; CHECK-NEXT:    vldrh.s32 q0, [r2]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    add sp, #8
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 3
+  %0 = bitcast i8* %z to <4 x i16>*
+  %1 = load <4 x i16>, <4 x i16>* %0, align 1
+  %2 = bitcast i8* %y to <4 x i32>*
+  %3 = sext <4 x i16> %1 to <4 x i32>
+  store <4 x i32> %3, <4 x i32>* %2, align 4
+  ret i8* %x
+}
+
+define i8* @ldrf32_align1(i8* %x, i8* %y) {
+; CHECK-LE-LABEL: ldrf32_align1:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldrb.u8 q0, [r0, #3]
+; CHECK-LE-NEXT:    vstrw.32 q0, [r1]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: ldrf32_align1:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vldrb.u8 q0, [r0, #3]
+; CHECK-BE-NEXT:    vrev32.8 q0, q0
+; CHECK-BE-NEXT:    vstrw.32 q0, [r1]
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 3
+  %0 = bitcast i8* %z to <4 x float>*
+  %1 = load <4 x float>, <4 x float>* %0, align 1
+  %2 = bitcast i8* %y to <4 x float>*
+  store <4 x float> %1, <4 x float>* %2, align 4
+  ret i8* %x
+}
+
+define i8* @ldrf16_align1(i8* %x, i8* %y) {
+; CHECK-LE-LABEL: ldrf16_align1:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldrb.u8 q0, [r0, #3]
+; CHECK-LE-NEXT:    vstrh.16 q0, [r1]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: ldrf16_align1:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vldrb.u8 q0, [r0, #3]
+; CHECK-BE-NEXT:    vrev16.8 q0, q0
+; CHECK-BE-NEXT:    vstrh.16 q0, [r1]
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 3
+  %0 = bitcast i8* %z to <8 x half>*
+  %1 = load <8 x half>, <8 x half>* %0, align 1
+  %2 = bitcast i8* %y to <8 x half>*
+  store <8 x half> %1, <8 x half>* %2, align 2
+  ret i8* %x
+}
+
+define i8* @ldrh16_align8(i8* %x, i8* %y) {
+; CHECK-LE-LABEL: ldrh16_align8:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldrw.u32 q0, [r0, #4]
+; CHECK-LE-NEXT:    vstrh.16 q0, [r1]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: ldrh16_align8:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vldrh.u16 q0, [r0, #4]
+; CHECK-BE-NEXT:    vstrh.16 q0, [r1]
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 4
+  %0 = bitcast i8* %z to <8 x i16>*
+  %1 = load <8 x i16>, <8 x i16>* %0, align 8
+  %2 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %1, <8 x i16>* %2, align 2
+  ret i8* %x
+}
 
 
 
-define i8* @post_strw32_4(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strw32_4:
+
+
+define i8* @strw32_4(i8* %y, i8* %x) {
+; CHECK-LABEL: strw32_4:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vldrw.u32 q0, [r1]
 ; CHECK-NEXT:    vstrw.32 q0, [r0, #4]
@@ -739,14 +1116,14 @@ define i8* @post_strw32_4(i8* %y, i8* %x) {
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 4
   %0 = bitcast i8* %x to <4 x i32>*
-  %1 = load <4 x i32>, <4 x i32>* %0, align 8
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
   %2 = bitcast i8* %z to <4 x i32>*
-  store <4 x i32> %1, <4 x i32>* %2, align 8
+  store <4 x i32> %1, <4 x i32>* %2, align 4
   ret i8* %y
 }
 
-define i8* @post_strw32_3(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strw32_3:
+define i8* @strw32_3(i8* %y, i8* %x) {
+; CHECK-LABEL: strw32_3:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vldrw.u32 q0, [r1]
 ; CHECK-NEXT:    adds r1, r0, #3
@@ -755,14 +1132,14 @@ define i8* @post_strw32_3(i8* %y, i8* %x) {
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 3
   %0 = bitcast i8* %x to <4 x i32>*
-  %1 = load <4 x i32>, <4 x i32>* %0, align 8
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
   %2 = bitcast i8* %z to <4 x i32>*
-  store <4 x i32> %1, <4 x i32>* %2, align 8
+  store <4 x i32> %1, <4 x i32>* %2, align 4
   ret i8* %y
 }
 
-define i8* @post_strw32_m4(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strw32_m4:
+define i8* @strw32_m4(i8* %y, i8* %x) {
+; CHECK-LABEL: strw32_m4:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vldrw.u32 q0, [r1]
 ; CHECK-NEXT:    vstrw.32 q0, [r0, #-4]
@@ -770,30 +1147,29 @@ define i8* @post_strw32_m4(i8* %y, i8* %x) {
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 -4
   %0 = bitcast i8* %x to <4 x i32>*
-  %1 = load <4 x i32>, <4 x i32>* %0, align 8
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
   %2 = bitcast i8* %z to <4 x i32>*
-  store <4 x i32> %1, <4 x i32>* %2, align 8
+  store <4 x i32> %1, <4 x i32>* %2, align 4
   ret i8* %y
 }
 
-define i8* @post_strw32_508(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strw32_508:
+define i8* @strw32_508(i8* %y, i8* %x) {
+; CHECK-LABEL: strw32_508:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-NEXT:    add.w r1, r0, #508
-; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vstrw.32 q0, [r0, #508]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 508
   %0 = bitcast i8* %x to <4 x i32>*
-  %1 = load <4 x i32>, <4 x i32>* %0, align 8
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
   %2 = bitcast i8* %z to <4 x i32>*
-  store <4 x i32> %1, <4 x i32>* %2, align 8
+  store <4 x i32> %1, <4 x i32>* %2, align 4
   ret i8* %y
 }
 
-define i8* @post_strw32_512(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strw32_512:
+define i8* @strw32_512(i8* %y, i8* %x) {
+; CHECK-LABEL: strw32_512:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vldrw.u32 q0, [r1]
 ; CHECK-NEXT:    add.w r1, r0, #512
@@ -802,30 +1178,29 @@ define i8* @post_strw32_512(i8* %y, i8* %x) {
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 512
   %0 = bitcast i8* %x to <4 x i32>*
-  %1 = load <4 x i32>, <4 x i32>* %0, align 8
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
   %2 = bitcast i8* %z to <4 x i32>*
-  store <4 x i32> %1, <4 x i32>* %2, align 8
+  store <4 x i32> %1, <4 x i32>* %2, align 4
   ret i8* %y
 }
 
-define i8* @post_strw32_m508(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strw32_m508:
+define i8* @strw32_m508(i8* %y, i8* %x) {
+; CHECK-LABEL: strw32_m508:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-NEXT:    sub.w r1, r0, #508
-; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vstrw.32 q0, [r0, #-508]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 -508
   %0 = bitcast i8* %x to <4 x i32>*
-  %1 = load <4 x i32>, <4 x i32>* %0, align 8
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
   %2 = bitcast i8* %z to <4 x i32>*
-  store <4 x i32> %1, <4 x i32>* %2, align 8
+  store <4 x i32> %1, <4 x i32>* %2, align 4
   ret i8* %y
 }
 
-define i8* @post_strw32_m512(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strw32_m512:
+define i8* @strw32_m512(i8* %y, i8* %x) {
+; CHECK-LABEL: strw32_m512:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vldrw.u32 q0, [r1]
 ; CHECK-NEXT:    sub.w r1, r0, #512
@@ -834,15 +1209,15 @@ define i8* @post_strw32_m512(i8* %y, i8* %x) {
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 -512
   %0 = bitcast i8* %x to <4 x i32>*
-  %1 = load <4 x i32>, <4 x i32>* %0, align 8
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
   %2 = bitcast i8* %z to <4 x i32>*
-  store <4 x i32> %1, <4 x i32>* %2, align 8
+  store <4 x i32> %1, <4 x i32>* %2, align 4
   ret i8* %y
 }
 
 
-define i8* @post_strh32_4(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strh32_4:
+define i8* @strh32_4(i8* %y, i8* %x) {
+; CHECK-LABEL: strh32_4:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vldrh.u32 q0, [r1]
 ; CHECK-NEXT:    vstrh.32 q0, [r0, #4]
@@ -850,14 +1225,14 @@ define i8* @post_strh32_4(i8* %y, i8* %x) {
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 4
   %0 = bitcast i8* %x to <4 x i16>*
-  %1 = load <4 x i16>, <4 x i16>* %0, align 8
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
   %2 = bitcast i8* %z to <4 x i16>*
-  store <4 x i16> %1, <4 x i16>* %2, align 8
+  store <4 x i16> %1, <4 x i16>* %2, align 2
   ret i8* %y
 }
 
-define i8* @post_strh32_3(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strh32_3:
+define i8* @strh32_3(i8* %y, i8* %x) {
+; CHECK-LABEL: strh32_3:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vldrh.u32 q0, [r1]
 ; CHECK-NEXT:    adds r1, r0, #3
@@ -866,46 +1241,44 @@ define i8* @post_strh32_3(i8* %y, i8* %x) {
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 3
   %0 = bitcast i8* %x to <4 x i16>*
-  %1 = load <4 x i16>, <4 x i16>* %0, align 8
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
   %2 = bitcast i8* %z to <4 x i16>*
-  store <4 x i16> %1, <4 x i16>* %2, align 8
+  store <4 x i16> %1, <4 x i16>* %2, align 2
   ret i8* %y
 }
 
-define i8* @post_strh32_2(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strh32_2:
+define i8* @strh32_2(i8* %y, i8* %x) {
+; CHECK-LABEL: strh32_2:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vldrh.u32 q0, [r1]
-; CHECK-NEXT:    adds r1, r0, #2
-; CHECK-NEXT:    vstrh.32 q0, [r1]
+; CHECK-NEXT:    vstrh.32 q0, [r0, #2]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 2
   %0 = bitcast i8* %x to <4 x i16>*
-  %1 = load <4 x i16>, <4 x i16>* %0, align 8
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
   %2 = bitcast i8* %z to <4 x i16>*
-  store <4 x i16> %1, <4 x i16>* %2, align 8
+  store <4 x i16> %1, <4 x i16>* %2, align 2
   ret i8* %y
 }
 
-define i8* @post_strh32_254(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strh32_254:
+define i8* @strh32_254(i8* %y, i8* %x) {
+; CHECK-LABEL: strh32_254:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vldrh.u32 q0, [r1]
-; CHECK-NEXT:    add.w r1, r0, #254
-; CHECK-NEXT:    vstrh.32 q0, [r1]
+; CHECK-NEXT:    vstrh.32 q0, [r0, #254]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 254
   %0 = bitcast i8* %x to <4 x i16>*
-  %1 = load <4 x i16>, <4 x i16>* %0, align 8
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
   %2 = bitcast i8* %z to <4 x i16>*
-  store <4 x i16> %1, <4 x i16>* %2, align 8
+  store <4 x i16> %1, <4 x i16>* %2, align 2
   ret i8* %y
 }
 
-define i8* @post_strh32_256(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strh32_256:
+define i8* @strh32_256(i8* %y, i8* %x) {
+; CHECK-LABEL: strh32_256:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vldrh.u32 q0, [r1]
 ; CHECK-NEXT:    add.w r1, r0, #256
@@ -914,95 +1287,155 @@ define i8* @post_strh32_256(i8* %y, i8* %x) {
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 256
   %0 = bitcast i8* %x to <4 x i16>*
-  %1 = load <4 x i16>, <4 x i16>* %0, align 8
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
   %2 = bitcast i8* %z to <4 x i16>*
-  store <4 x i16> %1, <4 x i16>* %2, align 8
+  store <4 x i16> %1, <4 x i16>* %2, align 2
   ret i8* %y
 }
 
+define i8* @strh32_m254(i8* %y, i8* %x) {
+; CHECK-LABEL: strh32_m254:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u32 q0, [r1]
+; CHECK-NEXT:    vstrh.32 q0, [r0, #-254]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -254
+  %0 = bitcast i8* %x to <4 x i16>*
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
+  %2 = bitcast i8* %z to <4 x i16>*
+  store <4 x i16> %1, <4 x i16>* %2, align 2
+  ret i8* %y
+}
 
-define i8* @post_strh16_4(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strh16_4:
+define i8* @strh32_m256(i8* %y, i8* %x) {
+; CHECK-LABEL: strh32_m256:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-NEXT:    vstrw.32 q0, [r0, #4]
+; CHECK-NEXT:    vldrh.u32 q0, [r1]
+; CHECK-NEXT:    sub.w r1, r0, #256
+; CHECK-NEXT:    vstrh.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -256
+  %0 = bitcast i8* %x to <4 x i16>*
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
+  %2 = bitcast i8* %z to <4 x i16>*
+  store <4 x i16> %1, <4 x i16>* %2, align 2
+  ret i8* %y
+}
+
+
+define i8* @strh16_4(i8* %y, i8* %x) {
+; CHECK-LABEL: strh16_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-NEXT:    vstrh.16 q0, [r0, #4]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 4
   %0 = bitcast i8* %x to <8 x i16>*
-  %1 = load <8 x i16>, <8 x i16>* %0, align 8
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
   %2 = bitcast i8* %z to <8 x i16>*
-  store <8 x i16> %1, <8 x i16>* %2, align 8
+  store <8 x i16> %1, <8 x i16>* %2, align 2
   ret i8* %y
 }
 
-define i8* @post_strh16_3(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strh16_3:
+define i8* @strh16_3(i8* %y, i8* %x) {
+; CHECK-LABEL: strh16_3:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q0, [r1]
 ; CHECK-NEXT:    adds r1, r0, #3
-; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 3
   %0 = bitcast i8* %x to <8 x i16>*
-  %1 = load <8 x i16>, <8 x i16>* %0, align 8
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
   %2 = bitcast i8* %z to <8 x i16>*
-  store <8 x i16> %1, <8 x i16>* %2, align 8
+  store <8 x i16> %1, <8 x i16>* %2, align 2
   ret i8* %y
 }
 
-define i8* @post_strh16_2(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strh16_2:
+define i8* @strh16_2(i8* %y, i8* %x) {
+; CHECK-LABEL: strh16_2:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-NEXT:    adds r1, r0, #2
-; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-NEXT:    vstrh.16 q0, [r0, #2]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 2
   %0 = bitcast i8* %x to <8 x i16>*
-  %1 = load <8 x i16>, <8 x i16>* %0, align 8
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
   %2 = bitcast i8* %z to <8 x i16>*
-  store <8 x i16> %1, <8 x i16>* %2, align 8
+  store <8 x i16> %1, <8 x i16>* %2, align 2
   ret i8* %y
 }
 
-define i8* @post_strh16_254(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strh16_254:
+define i8* @strh16_254(i8* %y, i8* %x) {
+; CHECK-LABEL: strh16_254:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-NEXT:    add.w r1, r0, #254
-; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-NEXT:    vstrh.16 q0, [r0, #254]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 254
   %0 = bitcast i8* %x to <8 x i16>*
-  %1 = load <8 x i16>, <8 x i16>* %0, align 8
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
   %2 = bitcast i8* %z to <8 x i16>*
-  store <8 x i16> %1, <8 x i16>* %2, align 8
+  store <8 x i16> %1, <8 x i16>* %2, align 2
   ret i8* %y
 }
 
-define i8* @post_strh16_256(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strh16_256:
+define i8* @strh16_256(i8* %y, i8* %x) {
+; CHECK-LABEL: strh16_256:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q0, [r1]
 ; CHECK-NEXT:    add.w r1, r0, #256
-; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 256
   %0 = bitcast i8* %x to <8 x i16>*
-  %1 = load <8 x i16>, <8 x i16>* %0, align 8
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
   %2 = bitcast i8* %z to <8 x i16>*
-  store <8 x i16> %1, <8 x i16>* %2, align 8
+  store <8 x i16> %1, <8 x i16>* %2, align 2
   ret i8* %y
 }
 
+define i8* @strh16_m254(i8* %y, i8* %x) {
+; CHECK-LABEL: strh16_m254:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-NEXT:    vstrh.16 q0, [r0, #-254]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -254
+  %0 = bitcast i8* %x to <8 x i16>*
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
+  %2 = bitcast i8* %z to <8 x i16>*
+  store <8 x i16> %1, <8 x i16>* %2, align 2
+  ret i8* %y
+}
 
-define i8* @post_strb32_4(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strb32_4:
+define i8* @strh16_m256(i8* %y, i8* %x) {
+; CHECK-LABEL: strh16_m256:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-NEXT:    sub.w r1, r0, #256
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -256
+  %0 = bitcast i8* %x to <8 x i16>*
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
+  %2 = bitcast i8* %z to <8 x i16>*
+  store <8 x i16> %1, <8 x i16>* %2, align 2
+  ret i8* %y
+}
+
+
+define i8* @strb32_4(i8* %y, i8* %x) {
+; CHECK-LABEL: strb32_4:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vldrb.u32 q0, [r1]
 ; CHECK-NEXT:    vstrb.32 q0, [r0, #4]
@@ -1010,46 +1443,44 @@ define i8* @post_strb32_4(i8* %y, i8* %x) {
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 4
   %0 = bitcast i8* %x to <4 x i8>*
-  %1 = load <4 x i8>, <4 x i8>* %0, align 8
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
   %2 = bitcast i8* %z to <4 x i8>*
-  store <4 x i8> %1, <4 x i8>* %2, align 8
+  store <4 x i8> %1, <4 x i8>* %2, align 1
   ret i8* %y
 }
 
-define i8* @post_strb32_3(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strb32_3:
+define i8* @strb32_3(i8* %y, i8* %x) {
+; CHECK-LABEL: strb32_3:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vldrb.u32 q0, [r1]
-; CHECK-NEXT:    adds r1, r0, #3
-; CHECK-NEXT:    vstrb.32 q0, [r1]
+; CHECK-NEXT:    vstrb.32 q0, [r0, #3]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 3
   %0 = bitcast i8* %x to <4 x i8>*
-  %1 = load <4 x i8>, <4 x i8>* %0, align 8
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
   %2 = bitcast i8* %z to <4 x i8>*
-  store <4 x i8> %1, <4 x i8>* %2, align 8
+  store <4 x i8> %1, <4 x i8>* %2, align 1
   ret i8* %y
 }
 
-define i8* @post_strb32_127(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strb32_127:
+define i8* @strb32_127(i8* %y, i8* %x) {
+; CHECK-LABEL: strb32_127:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vldrb.u32 q0, [r1]
-; CHECK-NEXT:    add.w r1, r0, #127
-; CHECK-NEXT:    vstrb.32 q0, [r1]
+; CHECK-NEXT:    vstrb.32 q0, [r0, #127]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 127
   %0 = bitcast i8* %x to <4 x i8>*
-  %1 = load <4 x i8>, <4 x i8>* %0, align 8
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
   %2 = bitcast i8* %z to <4 x i8>*
-  store <4 x i8> %1, <4 x i8>* %2, align 8
+  store <4 x i8> %1, <4 x i8>* %2, align 1
   ret i8* %y
 }
 
-define i8* @post_strb32_128(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strb32_128:
+define i8* @strb32_128(i8* %y, i8* %x) {
+; CHECK-LABEL: strb32_128:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vldrb.u32 q0, [r1]
 ; CHECK-NEXT:    add.w r1, r0, #128
@@ -1058,15 +1489,46 @@ define i8* @post_strb32_128(i8* %y, i8* %x) {
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 128
   %0 = bitcast i8* %x to <4 x i8>*
-  %1 = load <4 x i8>, <4 x i8>* %0, align 8
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
+  %2 = bitcast i8* %z to <4 x i8>*
+  store <4 x i8> %1, <4 x i8>* %2, align 1
+  ret i8* %y
+}
+
+define i8* @strb32_m127(i8* %y, i8* %x) {
+; CHECK-LABEL: strb32_m127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u32 q0, [r1]
+; CHECK-NEXT:    vstrb.32 q0, [r0, #-127]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -127
+  %0 = bitcast i8* %x to <4 x i8>*
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
+  %2 = bitcast i8* %z to <4 x i8>*
+  store <4 x i8> %1, <4 x i8>* %2, align 1
+  ret i8* %y
+}
+
+define i8* @strb32_m128(i8* %y, i8* %x) {
+; CHECK-LABEL: strb32_m128:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u32 q0, [r1]
+; CHECK-NEXT:    sub.w r1, r0, #128
+; CHECK-NEXT:    vstrb.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -128
+  %0 = bitcast i8* %x to <4 x i8>*
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
   %2 = bitcast i8* %z to <4 x i8>*
-  store <4 x i8> %1, <4 x i8>* %2, align 8
+  store <4 x i8> %1, <4 x i8>* %2, align 1
   ret i8* %y
 }
 
 
-define i8* @post_strb16_4(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strb16_4:
+define i8* @strb16_4(i8* %y, i8* %x) {
+; CHECK-LABEL: strb16_4:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vldrb.u16 q0, [r1]
 ; CHECK-NEXT:    vstrb.16 q0, [r0, #4]
@@ -1074,46 +1536,44 @@ define i8* @post_strb16_4(i8* %y, i8* %x) {
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 4
   %0 = bitcast i8* %x to <8 x i8>*
-  %1 = load <8 x i8>, <8 x i8>* %0, align 8
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
   %2 = bitcast i8* %z to <8 x i8>*
-  store <8 x i8> %1, <8 x i8>* %2, align 8
+  store <8 x i8> %1, <8 x i8>* %2, align 1
   ret i8* %y
 }
 
-define i8* @post_strb16_3(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strb16_3:
+define i8* @strb16_3(i8* %y, i8* %x) {
+; CHECK-LABEL: strb16_3:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vldrb.u16 q0, [r1]
-; CHECK-NEXT:    adds r1, r0, #3
-; CHECK-NEXT:    vstrb.16 q0, [r1]
+; CHECK-NEXT:    vstrb.16 q0, [r0, #3]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 3
   %0 = bitcast i8* %x to <8 x i8>*
-  %1 = load <8 x i8>, <8 x i8>* %0, align 8
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
   %2 = bitcast i8* %z to <8 x i8>*
-  store <8 x i8> %1, <8 x i8>* %2, align 8
+  store <8 x i8> %1, <8 x i8>* %2, align 1
   ret i8* %y
 }
 
-define i8* @post_strb16_127(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strb16_127:
+define i8* @strb16_127(i8* %y, i8* %x) {
+; CHECK-LABEL: strb16_127:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vldrb.u16 q0, [r1]
-; CHECK-NEXT:    add.w r1, r0, #127
-; CHECK-NEXT:    vstrb.16 q0, [r1]
+; CHECK-NEXT:    vstrb.16 q0, [r0, #127]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 127
   %0 = bitcast i8* %x to <8 x i8>*
-  %1 = load <8 x i8>, <8 x i8>* %0, align 8
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
   %2 = bitcast i8* %z to <8 x i8>*
-  store <8 x i8> %1, <8 x i8>* %2, align 8
+  store <8 x i8> %1, <8 x i8>* %2, align 1
   ret i8* %y
 }
 
-define i8* @post_strb16_128(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strb16_128:
+define i8* @strb16_128(i8* %y, i8* %x) {
+; CHECK-LABEL: strb16_128:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vldrb.u16 q0, [r1]
 ; CHECK-NEXT:    add.w r1, r0, #128
@@ -1122,77 +1582,139 @@ define i8* @post_strb16_128(i8* %y, i8* %x) {
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 128
   %0 = bitcast i8* %x to <8 x i8>*
-  %1 = load <8 x i8>, <8 x i8>* %0, align 8
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
   %2 = bitcast i8* %z to <8 x i8>*
-  store <8 x i8> %1, <8 x i8>* %2, align 8
+  store <8 x i8> %1, <8 x i8>* %2, align 1
   ret i8* %y
 }
 
+define i8* @strb16_m127(i8* %y, i8* %x) {
+; CHECK-LABEL: strb16_m127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u16 q0, [r1]
+; CHECK-NEXT:    vstrb.16 q0, [r0, #-127]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -127
+  %0 = bitcast i8* %x to <8 x i8>*
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
+  %2 = bitcast i8* %z to <8 x i8>*
+  store <8 x i8> %1, <8 x i8>* %2, align 1
+  ret i8* %y
+}
 
-define i8* @post_strb8_4(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strb8_4:
+define i8* @strb16_m128(i8* %y, i8* %x) {
+; CHECK-LABEL: strb16_m128:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-NEXT:    vstrw.32 q0, [r0, #4]
+; CHECK-NEXT:    vldrb.u16 q0, [r1]
+; CHECK-NEXT:    sub.w r1, r0, #128
+; CHECK-NEXT:    vstrb.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -128
+  %0 = bitcast i8* %x to <8 x i8>*
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
+  %2 = bitcast i8* %z to <8 x i8>*
+  store <8 x i8> %1, <8 x i8>* %2, align 1
+  ret i8* %y
+}
+
+
+define i8* @strb8_4(i8* %y, i8* %x) {
+; CHECK-LABEL: strb8_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u8 q0, [r1]
+; CHECK-NEXT:    vstrb.8 q0, [r0, #4]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 4
   %0 = bitcast i8* %x to <16 x i8>*
-  %1 = load <16 x i8>, <16 x i8>* %0, align 8
+  %1 = load <16 x i8>, <16 x i8>* %0, align 1
   %2 = bitcast i8* %z to <16 x i8>*
-  store <16 x i8> %1, <16 x i8>* %2, align 8
+  store <16 x i8> %1, <16 x i8>* %2, align 1
   ret i8* %y
 }
 
-define i8* @post_strb8_3(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strb8_3:
+define i8* @strb8_3(i8* %y, i8* %x) {
+; CHECK-LABEL: strb8_3:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-NEXT:    adds r1, r0, #3
-; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vldrb.u8 q0, [r1]
+; CHECK-NEXT:    vstrb.8 q0, [r0, #3]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 3
   %0 = bitcast i8* %x to <16 x i8>*
-  %1 = load <16 x i8>, <16 x i8>* %0, align 8
+  %1 = load <16 x i8>, <16 x i8>* %0, align 1
   %2 = bitcast i8* %z to <16 x i8>*
-  store <16 x i8> %1, <16 x i8>* %2, align 8
+  store <16 x i8> %1, <16 x i8>* %2, align 1
   ret i8* %y
 }
 
-define i8* @post_strb8_127(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strb8_127:
+define i8* @strb8_127(i8* %y, i8* %x) {
+; CHECK-LABEL: strb8_127:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-NEXT:    add.w r1, r0, #127
-; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vldrb.u8 q0, [r1]
+; CHECK-NEXT:    vstrb.8 q0, [r0, #127]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 127
   %0 = bitcast i8* %x to <16 x i8>*
-  %1 = load <16 x i8>, <16 x i8>* %0, align 8
+  %1 = load <16 x i8>, <16 x i8>* %0, align 1
   %2 = bitcast i8* %z to <16 x i8>*
-  store <16 x i8> %1, <16 x i8>* %2, align 8
+  store <16 x i8> %1, <16 x i8>* %2, align 1
   ret i8* %y
 }
 
-define i8* @post_strb8_128(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strb8_128:
+define i8* @strb8_128(i8* %y, i8* %x) {
+; CHECK-LABEL: strb8_128:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-NEXT:    vstrw.32 q0, [r0, #128]
+; CHECK-NEXT:    vldrb.u8 q0, [r1]
+; CHECK-NEXT:    add.w r1, r0, #128
+; CHECK-NEXT:    vstrb.8 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 128
   %0 = bitcast i8* %x to <16 x i8>*
-  %1 = load <16 x i8>, <16 x i8>* %0, align 8
+  %1 = load <16 x i8>, <16 x i8>* %0, align 1
+  %2 = bitcast i8* %z to <16 x i8>*
+  store <16 x i8> %1, <16 x i8>* %2, align 1
+  ret i8* %y
+}
+
+define i8* @strb8_m127(i8* %y, i8* %x) {
+; CHECK-LABEL: strb8_m127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u8 q0, [r1]
+; CHECK-NEXT:    vstrb.8 q0, [r0, #-127]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -127
+  %0 = bitcast i8* %x to <16 x i8>*
+  %1 = load <16 x i8>, <16 x i8>* %0, align 1
+  %2 = bitcast i8* %z to <16 x i8>*
+  store <16 x i8> %1, <16 x i8>* %2, align 1
+  ret i8* %y
+}
+
+define i8* @strb8_m128(i8* %y, i8* %x) {
+; CHECK-LABEL: strb8_m128:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u8 q0, [r1]
+; CHECK-NEXT:    sub.w r1, r0, #128
+; CHECK-NEXT:    vstrb.8 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -128
+  %0 = bitcast i8* %x to <16 x i8>*
+  %1 = load <16 x i8>, <16 x i8>* %0, align 1
   %2 = bitcast i8* %z to <16 x i8>*
-  store <16 x i8> %1, <16 x i8>* %2, align 8
+  store <16 x i8> %1, <16 x i8>* %2, align 1
   ret i8* %y
 }
 
-define i8* @post_strf32_4(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strf32_4:
+
+define i8* @strf32_4(i8* %y, i8* %x) {
+; CHECK-LABEL: strf32_4:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vldrw.u32 q0, [r1]
 ; CHECK-NEXT:    vstrw.32 q0, [r0, #4]
@@ -1200,23 +1722,155 @@ define i8* @post_strf32_4(i8* %y, i8* %x) {
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 4
   %0 = bitcast i8* %x to <4 x float>*
-  %1 = load <4 x float>, <4 x float>* %0, align 8
+  %1 = load <4 x float>, <4 x float>* %0, align 4
   %2 = bitcast i8* %z to <4 x float>*
-  store <4 x float> %1, <4 x float>* %2, align 8
+  store <4 x float> %1, <4 x float>* %2, align 4
   ret i8* %y
 }
 
-define i8* @post_strf16_4(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strf16_4:
+define i8* @strf16_4(i8* %y, i8* %x) {
+; CHECK-LABEL: strf16_4:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-NEXT:    vstrw.32 q0, [r0, #4]
+; CHECK-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-NEXT:    vstrh.16 q0, [r0, #4]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 4
   %0 = bitcast i8* %x to <8 x half>*
-  %1 = load <8 x half>, <8 x half>* %0, align 8
+  %1 = load <8 x half>, <8 x half>* %0, align 2
+  %2 = bitcast i8* %z to <8 x half>*
+  store <8 x half> %1, <8 x half>* %2, align 2
+  ret i8* %y
+}
+
+define i8* @strwi32_align1(i8* %y, i8* %x) {
+; CHECK-LE-LABEL: strwi32_align1:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-LE-NEXT:    vstrb.8 q0, [r0, #3]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: strwi32_align1:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-BE-NEXT:    vrev32.8 q0, q0
+; CHECK-BE-NEXT:    vstrb.8 q0, [r0, #3]
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 3
+  %0 = bitcast i8* %x to <4 x i32>*
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
+  %2 = bitcast i8* %z to <4 x i32>*
+  store <4 x i32> %1, <4 x i32>* %2, align 1
+  ret i8* %y
+}
+
+define i8* @strhi16_align1(i8* %y, i8* %x) {
+; CHECK-LE-LABEL: strhi16_align1:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-LE-NEXT:    vstrb.8 q0, [r0, #3]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: strhi16_align1:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-BE-NEXT:    vrev16.8 q0, q0
+; CHECK-BE-NEXT:    vstrb.8 q0, [r0, #3]
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 3
+  %0 = bitcast i8* %x to <8 x i16>*
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
+  %2 = bitcast i8* %z to <8 x i16>*
+  store <8 x i16> %1, <8 x i16>* %2, align 1
+  ret i8* %y
+}
+
+define i8* @strhi32_align1(i8* %y, i8* %x) {
+; CHECK-LABEL: strhi32_align1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .pad #8
+; CHECK-NEXT:    sub sp, #8
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    mov r1, sp
+; CHECK-NEXT:    vstrh.32 q0, [r1]
+; CHECK-NEXT:    ldrd r1, r2, [sp]
+; CHECK-NEXT:    str.w r1, [r0, #3]
+; CHECK-NEXT:    str.w r2, [r0, #7]
+; CHECK-NEXT:    add sp, #8
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 3
+  %0 = bitcast i8* %x to <4 x i32>*
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
+  %2 = bitcast i8* %z to <4 x i16>*
+  %3 = trunc <4 x i32> %1 to <4 x i16>
+  store <4 x i16> %3, <4 x i16>* %2, align 1
+  ret i8* %y
+}
+
+define i8* @strf32_align1(i8* %y, i8* %x) {
+; CHECK-LE-LABEL: strf32_align1:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-LE-NEXT:    vstrb.8 q0, [r0, #3]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: strf32_align1:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-BE-NEXT:    vrev32.8 q0, q0
+; CHECK-BE-NEXT:    vstrb.8 q0, [r0, #3]
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 3
+  %0 = bitcast i8* %x to <4 x float>*
+  %1 = load <4 x float>, <4 x float>* %0, align 4
+  %2 = bitcast i8* %z to <4 x float>*
+  store <4 x float> %1, <4 x float>* %2, align 1
+  ret i8* %y
+}
+
+define i8* @strf16_align1(i8* %y, i8* %x) {
+; CHECK-LE-LABEL: strf16_align1:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-LE-NEXT:    vstrb.8 q0, [r0, #3]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: strf16_align1:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-BE-NEXT:    vrev16.8 q0, q0
+; CHECK-BE-NEXT:    vstrb.8 q0, [r0, #3]
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 3
+  %0 = bitcast i8* %x to <8 x half>*
+  %1 = load <8 x half>, <8 x half>* %0, align 2
   %2 = bitcast i8* %z to <8 x half>*
-  store <8 x half> %1, <8 x half>* %2, align 8
+  store <8 x half> %1, <8 x half>* %2, align 1
+  ret i8* %y
+}
+
+define i8* @strf16_align8(i8* %y, i8* %x) {
+; CHECK-LE-LABEL: strf16_align8:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-LE-NEXT:    vstrw.32 q0, [r0, #16]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: strf16_align8:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-BE-NEXT:    vstrh.16 q0, [r0, #16]
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 16
+  %0 = bitcast i8* %x to <8 x i16>*
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
+  %2 = bitcast i8* %z to <8 x i16>*
+  store <8 x i16> %1, <8 x i16>* %2, align 8
   ret i8* %y
 }
diff --git a/llvm/test/CodeGen/Thumb2/mve-ldst-postinc.ll b/llvm/test/CodeGen/Thumb2/mve-ldst-postinc.ll
index e979711f3..d84724a0b 100644
--- a/llvm/test/CodeGen/Thumb2/mve-ldst-postinc.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-ldst-postinc.ll
@@ -1,72 +1,76 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve %s -o - | FileCheck %s
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-LE
+; RUN: llc -mtriple=thumbebv8.1m.main-arm-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-BE
 
-define i8* @post_ldrwu32_4(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrwu32_4:
+define i8* @ldrwu32_4(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrwu32_4:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r0]
-; CHECK-NEXT:    adds r0, #4
+; CHECK-NEXT:    vldrw.u32 q0, [r0], #4
 ; CHECK-NEXT:    vstrw.32 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 4
   %0 = bitcast i8* %x to <4 x i32>*
-  %1 = load <4 x i32>, <4 x i32>* %0, align 8
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
   %2 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %1, <4 x i32>* %2, align 8
+  store <4 x i32> %1, <4 x i32>* %2, align 4
   ret i8* %z
 }
 
-define i8* @post_ldrwu32_3(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrwu32_3:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r0]
-; CHECK-NEXT:    adds r0, #3
-; CHECK-NEXT:    vstrw.32 q0, [r1]
-; CHECK-NEXT:    bx lr
+define i8* @ldrwu32_3(i8* %x, i8* %y) {
+; CHECK-LE-LABEL: ldrwu32_3:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldrb.u8 q0, [r0], #3
+; CHECK-LE-NEXT:    vstrw.32 q0, [r1]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: ldrwu32_3:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-BE-NEXT:    adds r0, #3
+; CHECK-BE-NEXT:    vstrw.32 q0, [r1]
+; CHECK-BE-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 3
   %0 = bitcast i8* %x to <4 x i32>*
-  %1 = load <4 x i32>, <4 x i32>* %0, align 8
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
   %2 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %1, <4 x i32>* %2, align 8
+  store <4 x i32> %1, <4 x i32>* %2, align 4
   ret i8* %z
 }
 
-define i8* @post_ldrwu32_m4(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrwu32_m4:
+define i8* @ldrwu32_m4(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrwu32_m4:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r0]
-; CHECK-NEXT:    subs r0, #4
+; CHECK-NEXT:    vldrw.u32 q0, [r0], #-4
 ; CHECK-NEXT:    vstrw.32 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 -4
   %0 = bitcast i8* %x to <4 x i32>*
-  %1 = load <4 x i32>, <4 x i32>* %0, align 8
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
   %2 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %1, <4 x i32>* %2, align 8
+  store <4 x i32> %1, <4 x i32>* %2, align 4
   ret i8* %z
 }
 
-define i8* @post_ldrwu32_508(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrwu32_508:
+define i8* @ldrwu32_508(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrwu32_508:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r0]
-; CHECK-NEXT:    add.w r0, r0, #508
+; CHECK-NEXT:    vldrw.u32 q0, [r0], #508
 ; CHECK-NEXT:    vstrw.32 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 508
   %0 = bitcast i8* %x to <4 x i32>*
-  %1 = load <4 x i32>, <4 x i32>* %0, align 8
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
   %2 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %1, <4 x i32>* %2, align 8
+  store <4 x i32> %1, <4 x i32>* %2, align 4
   ret i8* %z
 }
 
-define i8* @post_ldrwu32_512(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrwu32_512:
+define i8* @ldrwu32_512(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrwu32_512:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vldrw.u32 q0, [r0]
 ; CHECK-NEXT:    add.w r0, r0, #512
@@ -75,30 +79,29 @@ define i8* @post_ldrwu32_512(i8* %x, i8* %y) {
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 512
   %0 = bitcast i8* %x to <4 x i32>*
-  %1 = load <4 x i32>, <4 x i32>* %0, align 8
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
   %2 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %1, <4 x i32>* %2, align 8
+  store <4 x i32> %1, <4 x i32>* %2, align 4
   ret i8* %z
 }
 
-define i8* @post_ldrwu32_m508(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrwu32_m508:
+define i8* @ldrwu32_m508(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrwu32_m508:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r0]
-; CHECK-NEXT:    sub.w r0, r0, #508
+; CHECK-NEXT:    vldrw.u32 q0, [r0], #-508
 ; CHECK-NEXT:    vstrw.32 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 -508
   %0 = bitcast i8* %x to <4 x i32>*
-  %1 = load <4 x i32>, <4 x i32>* %0, align 8
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
   %2 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %1, <4 x i32>* %2, align 8
+  store <4 x i32> %1, <4 x i32>* %2, align 4
   ret i8* %z
 }
 
-define i8* @post_ldrwu32_m512(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrwu32_m512:
+define i8* @ldrwu32_m512(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrwu32_m512:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vldrw.u32 q0, [r0]
 ; CHECK-NEXT:    sub.w r0, r0, #512
@@ -107,32 +110,31 @@ define i8* @post_ldrwu32_m512(i8* %x, i8* %y) {
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 -512
   %0 = bitcast i8* %x to <4 x i32>*
-  %1 = load <4 x i32>, <4 x i32>* %0, align 8
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
   %2 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %1, <4 x i32>* %2, align 8
+  store <4 x i32> %1, <4 x i32>* %2, align 4
   ret i8* %z
 }
 
 
-define i8* @post_ldrhu32_4(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrhu32_4:
+define i8* @ldrhu32_4(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrhu32_4:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrh.u32 q0, [r0]
-; CHECK-NEXT:    adds r0, #4
+; CHECK-NEXT:    vldrh.u32 q0, [r0], #4
 ; CHECK-NEXT:    vstrw.32 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 4
   %0 = bitcast i8* %x to <4 x i16>*
-  %1 = load <4 x i16>, <4 x i16>* %0, align 8
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
   %2 = zext <4 x i16> %1 to <4 x i32>
   %3 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %2, <4 x i32>* %3, align 8
+  store <4 x i32> %2, <4 x i32>* %3, align 4
   ret i8* %z
 }
 
-define i8* @post_ldrhu32_3(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrhu32_3:
+define i8* @ldrhu32_3(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrhu32_3:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vldrh.u32 q0, [r0]
 ; CHECK-NEXT:    adds r0, #3
@@ -141,49 +143,47 @@ define i8* @post_ldrhu32_3(i8* %x, i8* %y) {
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 3
   %0 = bitcast i8* %x to <4 x i16>*
-  %1 = load <4 x i16>, <4 x i16>* %0, align 8
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
   %2 = zext <4 x i16> %1 to <4 x i32>
   %3 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %2, <4 x i32>* %3, align 8
+  store <4 x i32> %2, <4 x i32>* %3, align 4
   ret i8* %z
 }
 
-define i8* @post_ldrhu32_2(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrhu32_2:
+define i8* @ldrhu32_2(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrhu32_2:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrh.u32 q0, [r0]
-; CHECK-NEXT:    adds r0, #2
+; CHECK-NEXT:    vldrh.u32 q0, [r0], #2
 ; CHECK-NEXT:    vstrw.32 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 2
   %0 = bitcast i8* %x to <4 x i16>*
-  %1 = load <4 x i16>, <4 x i16>* %0, align 8
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
   %2 = zext <4 x i16> %1 to <4 x i32>
   %3 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %2, <4 x i32>* %3, align 8
+  store <4 x i32> %2, <4 x i32>* %3, align 4
   ret i8* %z
 }
 
-define i8* @post_ldrhu32_254(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrhu32_254:
+define i8* @ldrhu32_254(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrhu32_254:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrh.u32 q0, [r0]
-; CHECK-NEXT:    adds r0, #254
+; CHECK-NEXT:    vldrh.u32 q0, [r0], #254
 ; CHECK-NEXT:    vstrw.32 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 254
   %0 = bitcast i8* %x to <4 x i16>*
-  %1 = load <4 x i16>, <4 x i16>* %0, align 8
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
   %2 = zext <4 x i16> %1 to <4 x i32>
   %3 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %2, <4 x i32>* %3, align 8
+  store <4 x i32> %2, <4 x i32>* %3, align 4
   ret i8* %z
 }
 
-define i8* @post_ldrhu32_256(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrhu32_256:
+define i8* @ldrhu32_256(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrhu32_256:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vldrh.u32 q0, [r0]
 ; CHECK-NEXT:    add.w r0, r0, #256
@@ -192,33 +192,65 @@ define i8* @post_ldrhu32_256(i8* %x, i8* %y) {
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 256
   %0 = bitcast i8* %x to <4 x i16>*
-  %1 = load <4 x i16>, <4 x i16>* %0, align 8
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
   %2 = zext <4 x i16> %1 to <4 x i32>
   %3 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %2, <4 x i32>* %3, align 8
+  store <4 x i32> %2, <4 x i32>* %3, align 4
   ret i8* %z
 }
 
+define i8* @ldrhu32_m254(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrhu32_m254:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u32 q0, [r0], #-254
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -254
+  %0 = bitcast i8* %x to <4 x i16>*
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
+  %2 = zext <4 x i16> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
 
-define i8* @post_ldrhs32_4(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrhs32_4:
+define i8* @ldrhu32_m256(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrhu32_m256:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrh.s32 q0, [r0]
-; CHECK-NEXT:    adds r0, #4
+; CHECK-NEXT:    vldrh.u32 q0, [r0]
+; CHECK-NEXT:    sub.w r0, r0, #256
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -256
+  %0 = bitcast i8* %x to <4 x i16>*
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
+  %2 = zext <4 x i16> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
+
+
+define i8* @ldrhs32_4(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrhs32_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.s32 q0, [r0], #4
 ; CHECK-NEXT:    vstrw.32 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 4
   %0 = bitcast i8* %x to <4 x i16>*
-  %1 = load <4 x i16>, <4 x i16>* %0, align 8
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
   %2 = sext <4 x i16> %1 to <4 x i32>
   %3 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %2, <4 x i32>* %3, align 8
+  store <4 x i32> %2, <4 x i32>* %3, align 4
   ret i8* %z
 }
 
-define i8* @post_ldrhs32_3(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrhs32_3:
+define i8* @ldrhs32_3(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrhs32_3:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vldrh.s32 q0, [r0]
 ; CHECK-NEXT:    adds r0, #3
@@ -227,49 +259,47 @@ define i8* @post_ldrhs32_3(i8* %x, i8* %y) {
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 3
   %0 = bitcast i8* %x to <4 x i16>*
-  %1 = load <4 x i16>, <4 x i16>* %0, align 8
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
   %2 = sext <4 x i16> %1 to <4 x i32>
   %3 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %2, <4 x i32>* %3, align 8
+  store <4 x i32> %2, <4 x i32>* %3, align 4
   ret i8* %z
 }
 
-define i8* @post_ldrhs32_2(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrhs32_2:
+define i8* @ldrhs32_2(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrhs32_2:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrh.s32 q0, [r0]
-; CHECK-NEXT:    adds r0, #2
+; CHECK-NEXT:    vldrh.s32 q0, [r0], #2
 ; CHECK-NEXT:    vstrw.32 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 2
   %0 = bitcast i8* %x to <4 x i16>*
-  %1 = load <4 x i16>, <4 x i16>* %0, align 8
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
   %2 = sext <4 x i16> %1 to <4 x i32>
   %3 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %2, <4 x i32>* %3, align 8
+  store <4 x i32> %2, <4 x i32>* %3, align 4
   ret i8* %z
 }
 
-define i8* @post_ldrhs32_254(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrhs32_254:
+define i8* @ldrhs32_254(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrhs32_254:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrh.s32 q0, [r0]
-; CHECK-NEXT:    adds r0, #254
+; CHECK-NEXT:    vldrh.s32 q0, [r0], #254
 ; CHECK-NEXT:    vstrw.32 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 254
   %0 = bitcast i8* %x to <4 x i16>*
-  %1 = load <4 x i16>, <4 x i16>* %0, align 8
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
   %2 = sext <4 x i16> %1 to <4 x i32>
   %3 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %2, <4 x i32>* %3, align 8
+  store <4 x i32> %2, <4 x i32>* %3, align 4
   ret i8* %z
 }
 
-define i8* @post_ldrhs32_256(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrhs32_256:
+define i8* @ldrhs32_256(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrhs32_256:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vldrh.s32 q0, [r0]
 ; CHECK-NEXT:    add.w r0, r0, #256
@@ -278,148 +308,212 @@ define i8* @post_ldrhs32_256(i8* %x, i8* %y) {
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 256
   %0 = bitcast i8* %x to <4 x i16>*
-  %1 = load <4 x i16>, <4 x i16>* %0, align 8
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
   %2 = sext <4 x i16> %1 to <4 x i32>
   %3 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %2, <4 x i32>* %3, align 8
+  store <4 x i32> %2, <4 x i32>* %3, align 4
   ret i8* %z
 }
 
+define i8* @ldrhs32_m254(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrhs32_m254:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.s32 q0, [r0], #-254
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -254
+  %0 = bitcast i8* %x to <4 x i16>*
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
+  %2 = sext <4 x i16> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
 
-define i8* @post_ldrhu16_4(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrhu16_4:
+define i8* @ldrhs32_m256(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrhs32_m256:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r0]
-; CHECK-NEXT:    adds r0, #4
+; CHECK-NEXT:    vldrh.s32 q0, [r0]
+; CHECK-NEXT:    sub.w r0, r0, #256
 ; CHECK-NEXT:    vstrw.32 q0, [r1]
 ; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -256
+  %0 = bitcast i8* %x to <4 x i16>*
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
+  %2 = sext <4 x i16> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
+
+
+define i8* @ldrhu16_4(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrhu16_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r0], #4
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 4
   %0 = bitcast i8* %x to <8 x i16>*
-  %1 = load <8 x i16>, <8 x i16>* %0, align 8
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
   %2 = bitcast i8* %y to <8 x i16>*
-  store <8 x i16> %1, <8 x i16>* %2, align 8
+  store <8 x i16> %1, <8 x i16>* %2, align 2
   ret i8* %z
 }
 
-define i8* @post_ldrhu16_3(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrhu16_3:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r0]
-; CHECK-NEXT:    adds r0, #3
-; CHECK-NEXT:    vstrw.32 q0, [r1]
-; CHECK-NEXT:    bx lr
+define i8* @ldrhu16_3(i8* %x, i8* %y) {
+; CHECK-LE-LABEL: ldrhu16_3:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldrb.u8 q0, [r0], #3
+; CHECK-LE-NEXT:    vstrh.16 q0, [r1]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: ldrhu16_3:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vldrh.u16 q0, [r0]
+; CHECK-BE-NEXT:    adds r0, #3
+; CHECK-BE-NEXT:    vstrh.16 q0, [r1]
+; CHECK-BE-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 3
   %0 = bitcast i8* %x to <8 x i16>*
-  %1 = load <8 x i16>, <8 x i16>* %0, align 8
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
   %2 = bitcast i8* %y to <8 x i16>*
-  store <8 x i16> %1, <8 x i16>* %2, align 8
+  store <8 x i16> %1, <8 x i16>* %2, align 2
   ret i8* %z
 }
 
-define i8* @post_ldrhu16_2(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrhu16_2:
+define i8* @ldrhu16_2(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrhu16_2:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r0]
-; CHECK-NEXT:    adds r0, #2
-; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q0, [r0], #2
+; CHECK-NEXT:    vstrh.16 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 2
   %0 = bitcast i8* %x to <8 x i16>*
-  %1 = load <8 x i16>, <8 x i16>* %0, align 8
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
   %2 = bitcast i8* %y to <8 x i16>*
-  store <8 x i16> %1, <8 x i16>* %2, align 8
+  store <8 x i16> %1, <8 x i16>* %2, align 2
   ret i8* %z
 }
 
-define i8* @post_ldrhu16_254(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrhu16_254:
+define i8* @ldrhu16_254(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrhu16_254:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r0]
-; CHECK-NEXT:    adds r0, #254
-; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q0, [r0], #254
+; CHECK-NEXT:    vstrh.16 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 254
   %0 = bitcast i8* %x to <8 x i16>*
-  %1 = load <8 x i16>, <8 x i16>* %0, align 8
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
   %2 = bitcast i8* %y to <8 x i16>*
-  store <8 x i16> %1, <8 x i16>* %2, align 8
+  store <8 x i16> %1, <8 x i16>* %2, align 2
   ret i8* %z
 }
 
-define i8* @post_ldrhu16_256(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrhu16_256:
+define i8* @ldrhu16_256(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrhu16_256:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vldrh.u16 q0, [r0]
 ; CHECK-NEXT:    add.w r0, r0, #256
-; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 256
   %0 = bitcast i8* %x to <8 x i16>*
-  %1 = load <8 x i16>, <8 x i16>* %0, align 8
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
   %2 = bitcast i8* %y to <8 x i16>*
-  store <8 x i16> %1, <8 x i16>* %2, align 8
+  store <8 x i16> %1, <8 x i16>* %2, align 2
   ret i8* %z
 }
 
+define i8* @ldrhu16_m254(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrhu16_m254:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r0], #-254
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -254
+  %0 = bitcast i8* %x to <8 x i16>*
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
+  %2 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %1, <8 x i16>* %2, align 2
+  ret i8* %z
+}
 
-define i8* @post_ldrbu32_4(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrbu32_4:
+define i8* @ldrhu16_m256(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrhu16_m256:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrb.u32 q0, [r0]
-; CHECK-NEXT:    adds r0, #4
+; CHECK-NEXT:    vldrh.u16 q0, [r0]
+; CHECK-NEXT:    sub.w r0, r0, #256
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -256
+  %0 = bitcast i8* %x to <8 x i16>*
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
+  %2 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %1, <8 x i16>* %2, align 2
+  ret i8* %z
+}
+
+
+define i8* @ldrbu32_4(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbu32_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u32 q0, [r0], #4
 ; CHECK-NEXT:    vstrw.32 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 4
   %0 = bitcast i8* %x to <4 x i8>*
-  %1 = load <4 x i8>, <4 x i8>* %0, align 8
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
   %2 = zext <4 x i8> %1 to <4 x i32>
   %3 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %2, <4 x i32>* %3, align 8
+  store <4 x i32> %2, <4 x i32>* %3, align 4
   ret i8* %z
 }
 
-define i8* @post_ldrbu32_3(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrbu32_3:
+define i8* @ldrbu32_3(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbu32_3:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrb.u32 q0, [r0]
-; CHECK-NEXT:    adds r0, #3
+; CHECK-NEXT:    vldrb.u32 q0, [r0], #3
 ; CHECK-NEXT:    vstrw.32 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 3
   %0 = bitcast i8* %x to <4 x i8>*
-  %1 = load <4 x i8>, <4 x i8>* %0, align 8
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
   %2 = zext <4 x i8> %1 to <4 x i32>
   %3 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %2, <4 x i32>* %3, align 8
+  store <4 x i32> %2, <4 x i32>* %3, align 4
   ret i8* %z
 }
 
-define i8* @post_ldrbu32_127(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrbu32_127:
+define i8* @ldrbu32_127(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbu32_127:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrb.u32 q0, [r0]
-; CHECK-NEXT:    adds r0, #127
+; CHECK-NEXT:    vldrb.u32 q0, [r0], #127
 ; CHECK-NEXT:    vstrw.32 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 127
   %0 = bitcast i8* %x to <4 x i8>*
-  %1 = load <4 x i8>, <4 x i8>* %0, align 8
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
   %2 = zext <4 x i8> %1 to <4 x i32>
   %3 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %2, <4 x i32>* %3, align 8
+  store <4 x i32> %2, <4 x i32>* %3, align 4
   ret i8* %z
 }
 
-define i8* @post_ldrbu32_128(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrbu32_128:
+define i8* @ldrbu32_128(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbu32_128:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vldrb.u32 q0, [r0]
 ; CHECK-NEXT:    adds r0, #128
@@ -428,67 +522,97 @@ define i8* @post_ldrbu32_128(i8* %x, i8* %y) {
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 128
   %0 = bitcast i8* %x to <4 x i8>*
-  %1 = load <4 x i8>, <4 x i8>* %0, align 8
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
   %2 = zext <4 x i8> %1 to <4 x i32>
   %3 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %2, <4 x i32>* %3, align 8
+  store <4 x i32> %2, <4 x i32>* %3, align 4
   ret i8* %z
 }
 
+define i8* @ldrbu32_m127(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbu32_m127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u32 q0, [r0], #-127
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -127
+  %0 = bitcast i8* %x to <4 x i8>*
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
+  %2 = zext <4 x i8> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
 
-define i8* @post_ldrbs32_4(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrbs32_4:
+define i8* @ldrbu32_m128(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbu32_m128:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrb.s32 q0, [r0]
-; CHECK-NEXT:    adds r0, #4
+; CHECK-NEXT:    vldrb.u32 q0, [r0]
+; CHECK-NEXT:    subs r0, #128
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -128
+  %0 = bitcast i8* %x to <4 x i8>*
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
+  %2 = zext <4 x i8> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
+
+
+define i8* @ldrbs32_4(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbs32_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.s32 q0, [r0], #4
 ; CHECK-NEXT:    vstrw.32 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 4
   %0 = bitcast i8* %x to <4 x i8>*
-  %1 = load <4 x i8>, <4 x i8>* %0, align 8
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
   %2 = sext <4 x i8> %1 to <4 x i32>
   %3 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %2, <4 x i32>* %3, align 8
+  store <4 x i32> %2, <4 x i32>* %3, align 4
   ret i8* %z
 }
 
-define i8* @post_ldrbs32_3(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrbs32_3:
+define i8* @ldrbs32_3(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbs32_3:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrb.s32 q0, [r0]
-; CHECK-NEXT:    adds r0, #3
+; CHECK-NEXT:    vldrb.s32 q0, [r0], #3
 ; CHECK-NEXT:    vstrw.32 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 3
   %0 = bitcast i8* %x to <4 x i8>*
-  %1 = load <4 x i8>, <4 x i8>* %0, align 8
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
   %2 = sext <4 x i8> %1 to <4 x i32>
   %3 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %2, <4 x i32>* %3, align 8
+  store <4 x i32> %2, <4 x i32>* %3, align 4
   ret i8* %z
 }
 
-define i8* @post_ldrbs32_127(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrbs32_127:
+define i8* @ldrbs32_127(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbs32_127:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrb.s32 q0, [r0]
-; CHECK-NEXT:    adds r0, #127
+; CHECK-NEXT:    vldrb.s32 q0, [r0], #127
 ; CHECK-NEXT:    vstrw.32 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 127
   %0 = bitcast i8* %x to <4 x i8>*
-  %1 = load <4 x i8>, <4 x i8>* %0, align 8
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
   %2 = sext <4 x i8> %1 to <4 x i32>
   %3 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %2, <4 x i32>* %3, align 8
+  store <4 x i32> %2, <4 x i32>* %3, align 4
   ret i8* %z
 }
 
-define i8* @post_ldrbs32_128(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrbs32_128:
+define i8* @ldrbs32_128(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbs32_128:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vldrb.s32 q0, [r0]
 ; CHECK-NEXT:    adds r0, #128
@@ -497,318 +621,590 @@ define i8* @post_ldrbs32_128(i8* %x, i8* %y) {
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 128
   %0 = bitcast i8* %x to <4 x i8>*
-  %1 = load <4 x i8>, <4 x i8>* %0, align 8
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
   %2 = sext <4 x i8> %1 to <4 x i32>
   %3 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %2, <4 x i32>* %3, align 8
+  store <4 x i32> %2, <4 x i32>* %3, align 4
   ret i8* %z
 }
 
+define i8* @ldrbs32_m127(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbs32_m127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.s32 q0, [r0], #-127
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -127
+  %0 = bitcast i8* %x to <4 x i8>*
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
+  %2 = sext <4 x i8> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
 
-define i8* @post_ldrbu16_4(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrbu16_4:
+define i8* @ldrbs32_m128(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbs32_m128:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrb.u16 q0, [r0]
-; CHECK-NEXT:    adds r0, #4
+; CHECK-NEXT:    vldrb.s32 q0, [r0]
+; CHECK-NEXT:    subs r0, #128
 ; CHECK-NEXT:    vstrw.32 q0, [r1]
 ; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -128
+  %0 = bitcast i8* %x to <4 x i8>*
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
+  %2 = sext <4 x i8> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
+
+
+define i8* @ldrbu16_4(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbu16_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u16 q0, [r0], #4
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 4
   %0 = bitcast i8* %x to <8 x i8>*
-  %1 = load <8 x i8>, <8 x i8>* %0, align 8
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
   %2 = zext <8 x i8> %1 to <8 x i16>
   %3 = bitcast i8* %y to <8 x i16>*
-  store <8 x i16> %2, <8 x i16>* %3, align 8
+  store <8 x i16> %2, <8 x i16>* %3, align 2
   ret i8* %z
 }
 
-define i8* @post_ldrbu16_3(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrbu16_3:
+define i8* @ldrbu16_3(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbu16_3:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrb.u16 q0, [r0]
-; CHECK-NEXT:    adds r0, #3
-; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vldrb.u16 q0, [r0], #3
+; CHECK-NEXT:    vstrh.16 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 3
   %0 = bitcast i8* %x to <8 x i8>*
-  %1 = load <8 x i8>, <8 x i8>* %0, align 8
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
   %2 = zext <8 x i8> %1 to <8 x i16>
   %3 = bitcast i8* %y to <8 x i16>*
-  store <8 x i16> %2, <8 x i16>* %3, align 8
+  store <8 x i16> %2, <8 x i16>* %3, align 2
   ret i8* %z
 }
 
-define i8* @post_ldrbu16_127(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrbu16_127:
+define i8* @ldrbu16_127(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbu16_127:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrb.u16 q0, [r0]
-; CHECK-NEXT:    adds r0, #127
-; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vldrb.u16 q0, [r0], #127
+; CHECK-NEXT:    vstrh.16 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 127
   %0 = bitcast i8* %x to <8 x i8>*
-  %1 = load <8 x i8>, <8 x i8>* %0, align 8
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
   %2 = zext <8 x i8> %1 to <8 x i16>
   %3 = bitcast i8* %y to <8 x i16>*
-  store <8 x i16> %2, <8 x i16>* %3, align 8
+  store <8 x i16> %2, <8 x i16>* %3, align 2
   ret i8* %z
 }
 
-define i8* @post_ldrbu16_128(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrbu16_128:
+define i8* @ldrbu16_128(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbu16_128:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vldrb.u16 q0, [r0]
 ; CHECK-NEXT:    adds r0, #128
-; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 128
   %0 = bitcast i8* %x to <8 x i8>*
-  %1 = load <8 x i8>, <8 x i8>* %0, align 8
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
   %2 = zext <8 x i8> %1 to <8 x i16>
   %3 = bitcast i8* %y to <8 x i16>*
-  store <8 x i16> %2, <8 x i16>* %3, align 8
+  store <8 x i16> %2, <8 x i16>* %3, align 2
   ret i8* %z
 }
 
+define i8* @ldrbu16_m127(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbu16_m127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u16 q0, [r0], #-127
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -127
+  %0 = bitcast i8* %x to <8 x i8>*
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
+  %2 = zext <8 x i8> %1 to <8 x i16>
+  %3 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %2, <8 x i16>* %3, align 2
+  ret i8* %z
+}
 
-define i8* @post_ldrbs16_4(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrbs16_4:
+define i8* @ldrbu16_m128(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbu16_m128:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrb.s16 q0, [r0]
-; CHECK-NEXT:    adds r0, #4
-; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vldrb.u16 q0, [r0]
+; CHECK-NEXT:    subs r0, #128
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -128
+  %0 = bitcast i8* %x to <8 x i8>*
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
+  %2 = zext <8 x i8> %1 to <8 x i16>
+  %3 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %2, <8 x i16>* %3, align 2
+  ret i8* %z
+}
+
+
+define i8* @ldrbs16_4(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbs16_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.s16 q0, [r0], #4
+; CHECK-NEXT:    vstrh.16 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 4
   %0 = bitcast i8* %x to <8 x i8>*
-  %1 = load <8 x i8>, <8 x i8>* %0, align 8
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
   %2 = sext <8 x i8> %1 to <8 x i16>
   %3 = bitcast i8* %y to <8 x i16>*
-  store <8 x i16> %2, <8 x i16>* %3, align 8
+  store <8 x i16> %2, <8 x i16>* %3, align 2
   ret i8* %z
 }
 
-define i8* @post_ldrbs16_3(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrbs16_3:
+define i8* @ldrbs16_3(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbs16_3:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrb.s16 q0, [r0]
-; CHECK-NEXT:    adds r0, #3
-; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vldrb.s16 q0, [r0], #3
+; CHECK-NEXT:    vstrh.16 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 3
   %0 = bitcast i8* %x to <8 x i8>*
-  %1 = load <8 x i8>, <8 x i8>* %0, align 8
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
   %2 = sext <8 x i8> %1 to <8 x i16>
   %3 = bitcast i8* %y to <8 x i16>*
-  store <8 x i16> %2, <8 x i16>* %3, align 8
+  store <8 x i16> %2, <8 x i16>* %3, align 2
   ret i8* %z
 }
 
-define i8* @post_ldrbs16_127(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrbs16_127:
+define i8* @ldrbs16_127(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbs16_127:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrb.s16 q0, [r0]
-; CHECK-NEXT:    adds r0, #127
-; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vldrb.s16 q0, [r0], #127
+; CHECK-NEXT:    vstrh.16 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 127
   %0 = bitcast i8* %x to <8 x i8>*
-  %1 = load <8 x i8>, <8 x i8>* %0, align 8
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
   %2 = sext <8 x i8> %1 to <8 x i16>
   %3 = bitcast i8* %y to <8 x i16>*
-  store <8 x i16> %2, <8 x i16>* %3, align 8
+  store <8 x i16> %2, <8 x i16>* %3, align 2
   ret i8* %z
 }
 
-define i8* @post_ldrbs16_128(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrbs16_128:
+define i8* @ldrbs16_128(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbs16_128:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vldrb.s16 q0, [r0]
 ; CHECK-NEXT:    adds r0, #128
-; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 128
   %0 = bitcast i8* %x to <8 x i8>*
-  %1 = load <8 x i8>, <8 x i8>* %0, align 8
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
   %2 = sext <8 x i8> %1 to <8 x i16>
   %3 = bitcast i8* %y to <8 x i16>*
-  store <8 x i16> %2, <8 x i16>* %3, align 8
+  store <8 x i16> %2, <8 x i16>* %3, align 2
   ret i8* %z
 }
 
+define i8* @ldrbs16_m127(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbs16_m127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.s16 q0, [r0], #-127
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -127
+  %0 = bitcast i8* %x to <8 x i8>*
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
+  %2 = sext <8 x i8> %1 to <8 x i16>
+  %3 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %2, <8 x i16>* %3, align 2
+  ret i8* %z
+}
 
-define i8* @post_ldrbu8_4(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrbu8_4:
+define i8* @ldrbs16_m128(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbs16_m128:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r0]
-; CHECK-NEXT:    adds r0, #4
-; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vldrb.s16 q0, [r0]
+; CHECK-NEXT:    subs r0, #128
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -128
+  %0 = bitcast i8* %x to <8 x i8>*
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
+  %2 = sext <8 x i8> %1 to <8 x i16>
+  %3 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %2, <8 x i16>* %3, align 2
+  ret i8* %z
+}
+
+
+define i8* @ldrbu8_4(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbu8_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u8 q0, [r0], #4
+; CHECK-NEXT:    vstrb.8 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 4
   %0 = bitcast i8* %x to <16 x i8>*
-  %1 = load <16 x i8>, <16 x i8>* %0, align 8
+  %1 = load <16 x i8>, <16 x i8>* %0, align 1
   %2 = bitcast i8* %y to <16 x i8>*
-  store <16 x i8> %1, <16 x i8>* %2, align 8
+  store <16 x i8> %1, <16 x i8>* %2, align 1
   ret i8* %z
 }
 
-define i8* @post_ldrbu8_3(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrbu8_3:
+define i8* @ldrbu8_3(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbu8_3:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r0]
-; CHECK-NEXT:    adds r0, #3
-; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vldrb.u8 q0, [r0], #3
+; CHECK-NEXT:    vstrb.8 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 3
   %0 = bitcast i8* %x to <16 x i8>*
-  %1 = load <16 x i8>, <16 x i8>* %0, align 8
+  %1 = load <16 x i8>, <16 x i8>* %0, align 1
   %2 = bitcast i8* %y to <16 x i8>*
-  store <16 x i8> %1, <16 x i8>* %2, align 8
+  store <16 x i8> %1, <16 x i8>* %2, align 1
   ret i8* %z
 }
 
-define i8* @post_ldrbu8_127(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrbu8_127:
+define i8* @ldrbu8_127(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbu8_127:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r0]
-; CHECK-NEXT:    adds r0, #127
-; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vldrb.u8 q0, [r0], #127
+; CHECK-NEXT:    vstrb.8 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 127
   %0 = bitcast i8* %x to <16 x i8>*
-  %1 = load <16 x i8>, <16 x i8>* %0, align 8
+  %1 = load <16 x i8>, <16 x i8>* %0, align 1
   %2 = bitcast i8* %y to <16 x i8>*
-  store <16 x i8> %1, <16 x i8>* %2, align 8
+  store <16 x i8> %1, <16 x i8>* %2, align 1
   ret i8* %z
 }
 
-define i8* @post_ldrbu8_128(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrbu8_128:
+define i8* @ldrbu8_128(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbu8_128:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vldrb.u8 q0, [r0]
 ; CHECK-NEXT:    adds r0, #128
-; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vstrb.8 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 128
   %0 = bitcast i8* %x to <16 x i8>*
-  %1 = load <16 x i8>, <16 x i8>* %0, align 8
+  %1 = load <16 x i8>, <16 x i8>* %0, align 1
   %2 = bitcast i8* %y to <16 x i8>*
-  store <16 x i8> %1, <16 x i8>* %2, align 8
+  store <16 x i8> %1, <16 x i8>* %2, align 1
   ret i8* %z
 }
 
-define i8* @post_ldrwf32_4(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrwf32_4:
+define i8* @ldrbu8_m127(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbu8_m127:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r0]
-; CHECK-NEXT:    adds r0, #4
+; CHECK-NEXT:    vldrb.u8 q0, [r0], #-127
+; CHECK-NEXT:    vstrb.8 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -127
+  %0 = bitcast i8* %x to <16 x i8>*
+  %1 = load <16 x i8>, <16 x i8>* %0, align 1
+  %2 = bitcast i8* %y to <16 x i8>*
+  store <16 x i8> %1, <16 x i8>* %2, align 1
+  ret i8* %z
+}
+
+define i8* @ldrbu8_m128(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbu8_m128:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u8 q0, [r0]
+; CHECK-NEXT:    subs r0, #128
+; CHECK-NEXT:    vstrb.8 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -128
+  %0 = bitcast i8* %x to <16 x i8>*
+  %1 = load <16 x i8>, <16 x i8>* %0, align 1
+  %2 = bitcast i8* %y to <16 x i8>*
+  store <16 x i8> %1, <16 x i8>* %2, align 1
+  ret i8* %z
+}
+
+
+define i8* @ldrwf32_4(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrwf32_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r0], #4
 ; CHECK-NEXT:    vstrw.32 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 4
   %0 = bitcast i8* %x to <4 x float>*
-  %1 = load <4 x float>, <4 x float>* %0, align 8
+  %1 = load <4 x float>, <4 x float>* %0, align 4
   %2 = bitcast i8* %y to <4 x float>*
-  store <4 x float> %1, <4 x float>* %2, align 8
+  store <4 x float> %1, <4 x float>* %2, align 4
   ret i8* %z
 }
 
-define i8* @post_ldrwf16_4(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrwf16_4:
+define i8* @ldrwf16_4(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrwf16_4:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r0]
-; CHECK-NEXT:    adds r0, #4
-; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q0, [r0], #4
+; CHECK-NEXT:    vstrh.16 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 4
   %0 = bitcast i8* %x to <8 x half>*
-  %1 = load <8 x half>, <8 x half>* %0, align 8
+  %1 = load <8 x half>, <8 x half>* %0, align 2
   %2 = bitcast i8* %y to <8 x half>*
-  store <8 x half> %1, <8 x half>* %2, align 8
+  store <8 x half> %1, <8 x half>* %2, align 2
   ret i8* %z
 }
 
+define i8* @ldrwi32_align1(i8* %x, i8* %y) {
+; CHECK-LE-LABEL: ldrwi32_align1:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldrb.u8 q0, [r0], #3
+; CHECK-LE-NEXT:    vstrw.32 q0, [r1]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: ldrwi32_align1:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vldrb.u8 q0, [r0]
+; CHECK-BE-NEXT:    adds r0, #3
+; CHECK-BE-NEXT:    vrev32.8 q0, q0
+; CHECK-BE-NEXT:    vstrw.32 q0, [r1]
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 3
+  %0 = bitcast i8* %x to <4 x i32>*
+  %1 = load <4 x i32>, <4 x i32>* %0, align 1
+  %2 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %1, <4 x i32>* %2, align 4
+  ret i8* %z
+}
 
+define i8* @ldrhi16_align1(i8* %x, i8* %y) {
+; CHECK-LE-LABEL: ldrhi16_align1:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldrb.u8 q0, [r0], #3
+; CHECK-LE-NEXT:    vstrh.16 q0, [r1]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: ldrhi16_align1:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vldrb.u8 q0, [r0]
+; CHECK-BE-NEXT:    adds r0, #3
+; CHECK-BE-NEXT:    vrev16.8 q0, q0
+; CHECK-BE-NEXT:    vstrh.16 q0, [r1]
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 3
+  %0 = bitcast i8* %x to <8 x i16>*
+  %1 = load <8 x i16>, <8 x i16>* %0, align 1
+  %2 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %1, <8 x i16>* %2, align 2
+  ret i8* %z
+}
 
-
-
-define i8* @post_strw32_4(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strw32_4:
+define i8* @ldrhi32_align1(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrhi32_align1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-NEXT:    vstrw.32 q0, [r0]
-; CHECK-NEXT:    adds r0, #4
+; CHECK-NEXT:    .pad #8
+; CHECK-NEXT:    sub sp, #8
+; CHECK-NEXT:    ldr r3, [r0, #4]
+; CHECK-NEXT:    ldr r2, [r0]
+; CHECK-NEXT:    adds r0, #3
+; CHECK-NEXT:    strd r2, r3, [sp]
+; CHECK-NEXT:    mov r2, sp
+; CHECK-NEXT:    vldrh.s32 q0, [r2]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    add sp, #8
 ; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 3
+  %0 = bitcast i8* %x to <4 x i16>*
+  %1 = load <4 x i16>, <4 x i16>* %0, align 1
+  %2 = bitcast i8* %y to <4 x i32>*
+  %3 = sext <4 x i16> %1 to <4 x i32>
+  store <4 x i32> %3, <4 x i32>* %2, align 4
+  ret i8* %z
+}
+
+define i8* @ldrf32_align1(i8* %x, i8* %y) {
+; CHECK-LE-LABEL: ldrf32_align1:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldrb.u8 q0, [r0], #3
+; CHECK-LE-NEXT:    vstrw.32 q0, [r1]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: ldrf32_align1:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vldrb.u8 q0, [r0]
+; CHECK-BE-NEXT:    adds r0, #3
+; CHECK-BE-NEXT:    vrev32.8 q0, q0
+; CHECK-BE-NEXT:    vstrw.32 q0, [r1]
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 3
+  %0 = bitcast i8* %x to <4 x float>*
+  %1 = load <4 x float>, <4 x float>* %0, align 1
+  %2 = bitcast i8* %y to <4 x float>*
+  store <4 x float> %1, <4 x float>* %2, align 4
+  ret i8* %z
+}
+
+define i8* @ldrf16_align1(i8* %x, i8* %y) {
+; CHECK-LE-LABEL: ldrf16_align1:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldrb.u8 q0, [r0], #3
+; CHECK-LE-NEXT:    vstrh.16 q0, [r1]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: ldrf16_align1:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vldrb.u8 q0, [r0]
+; CHECK-BE-NEXT:    adds r0, #3
+; CHECK-BE-NEXT:    vrev16.8 q0, q0
+; CHECK-BE-NEXT:    vstrh.16 q0, [r1]
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 3
+  %0 = bitcast i8* %x to <8 x half>*
+  %1 = load <8 x half>, <8 x half>* %0, align 1
+  %2 = bitcast i8* %y to <8 x half>*
+  store <8 x half> %1, <8 x half>* %2, align 2
+  ret i8* %z
+}
+
+define i8* @ldrh16_align8(i8* %x, i8* %y) {
+; CHECK-LE-LABEL: ldrh16_align8:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldrw.u32 q0, [r0], #4
+; CHECK-LE-NEXT:    vstrh.16 q0, [r1]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: ldrh16_align8:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vldrh.u16 q0, [r0], #4
+; CHECK-BE-NEXT:    vstrh.16 q0, [r1]
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 4
+  %0 = bitcast i8* %x to <8 x i16>*
+  %1 = load <8 x i16>, <8 x i16>* %0, align 8
+  %2 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %1, <8 x i16>* %2, align 2
+  ret i8* %z
+}
+
+
+
+
+
+define i8* @strw32_4(i8* %y, i8* %x) {
+; CHECK-LE-LABEL: strw32_4:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-LE-NEXT:    vstrb.8 q0, [r0], #4
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: strw32_4:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-BE-NEXT:    vstrw.32 q0, [r0], #4
+; CHECK-BE-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 4
   %0 = bitcast i8* %x to <4 x i32>*
-  %1 = load <4 x i32>, <4 x i32>* %0, align 8
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
   %2 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %1, <4 x i32>* %2, align 8
+  store <4 x i32> %1, <4 x i32>* %2, align 4
   ret i8* %z
 }
 
-define i8* @post_strw32_3(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strw32_3:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-NEXT:    vstrw.32 q0, [r0]
-; CHECK-NEXT:    adds r0, #3
-; CHECK-NEXT:    bx lr
+define i8* @strw32_3(i8* %y, i8* %x) {
+; CHECK-LE-LABEL: strw32_3:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-LE-NEXT:    vstrb.8 q0, [r0], #3
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: strw32_3:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-BE-NEXT:    vstrw.32 q0, [r0]
+; CHECK-BE-NEXT:    adds r0, #3
+; CHECK-BE-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 3
   %0 = bitcast i8* %x to <4 x i32>*
-  %1 = load <4 x i32>, <4 x i32>* %0, align 8
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
   %2 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %1, <4 x i32>* %2, align 8
+  store <4 x i32> %1, <4 x i32>* %2, align 4
   ret i8* %z
 }
 
-define i8* @post_strw32_m4(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strw32_m4:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-NEXT:    vstrw.32 q0, [r0]
-; CHECK-NEXT:    subs r0, #4
-; CHECK-NEXT:    bx lr
+define i8* @strw32_m4(i8* %y, i8* %x) {
+; CHECK-LE-LABEL: strw32_m4:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-LE-NEXT:    vstrb.8 q0, [r0], #-4
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: strw32_m4:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-BE-NEXT:    vstrw.32 q0, [r0], #-4
+; CHECK-BE-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 -4
   %0 = bitcast i8* %x to <4 x i32>*
-  %1 = load <4 x i32>, <4 x i32>* %0, align 8
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
   %2 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %1, <4 x i32>* %2, align 8
+  store <4 x i32> %1, <4 x i32>* %2, align 4
   ret i8* %z
 }
 
-define i8* @post_strw32_508(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strw32_508:
+define i8* @strw32_508(i8* %y, i8* %x) {
+; CHECK-LABEL: strw32_508:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-NEXT:    vstrw.32 q0, [r0]
-; CHECK-NEXT:    add.w r0, r0, #508
+; CHECK-NEXT:    vstrw.32 q0, [r0], #508
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 508
   %0 = bitcast i8* %x to <4 x i32>*
-  %1 = load <4 x i32>, <4 x i32>* %0, align 8
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
   %2 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %1, <4 x i32>* %2, align 8
+  store <4 x i32> %1, <4 x i32>* %2, align 4
   ret i8* %z
 }
 
-define i8* @post_strw32_512(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strw32_512:
+define i8* @strw32_512(i8* %y, i8* %x) {
+; CHECK-LABEL: strw32_512:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vldrw.u32 q0, [r1]
 ; CHECK-NEXT:    vstrw.32 q0, [r0]
@@ -817,30 +1213,29 @@ define i8* @post_strw32_512(i8* %y, i8* %x) {
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 512
   %0 = bitcast i8* %x to <4 x i32>*
-  %1 = load <4 x i32>, <4 x i32>* %0, align 8
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
   %2 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %1, <4 x i32>* %2, align 8
+  store <4 x i32> %1, <4 x i32>* %2, align 4
   ret i8* %z
 }
 
-define i8* @post_strw32_m508(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strw32_m508:
+define i8* @strw32_m508(i8* %y, i8* %x) {
+; CHECK-LABEL: strw32_m508:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-NEXT:    vstrw.32 q0, [r0]
-; CHECK-NEXT:    sub.w r0, r0, #508
+; CHECK-NEXT:    vstrw.32 q0, [r0], #-508
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 -508
   %0 = bitcast i8* %x to <4 x i32>*
-  %1 = load <4 x i32>, <4 x i32>* %0, align 8
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
   %2 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %1, <4 x i32>* %2, align 8
+  store <4 x i32> %1, <4 x i32>* %2, align 4
   ret i8* %z
 }
 
-define i8* @post_strw32_m512(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strw32_m512:
+define i8* @strw32_m512(i8* %y, i8* %x) {
+; CHECK-LABEL: strw32_m512:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vldrw.u32 q0, [r1]
 ; CHECK-NEXT:    vstrw.32 q0, [r0]
@@ -849,31 +1244,30 @@ define i8* @post_strw32_m512(i8* %y, i8* %x) {
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 -512
   %0 = bitcast i8* %x to <4 x i32>*
-  %1 = load <4 x i32>, <4 x i32>* %0, align 8
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
   %2 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %1, <4 x i32>* %2, align 8
+  store <4 x i32> %1, <4 x i32>* %2, align 4
   ret i8* %z
 }
 
 
-define i8* @post_strh32_4(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strh32_4:
+define i8* @strh32_4(i8* %y, i8* %x) {
+; CHECK-LABEL: strh32_4:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vldrh.u32 q0, [r1]
-; CHECK-NEXT:    vstrh.32 q0, [r0]
-; CHECK-NEXT:    adds r0, #4
+; CHECK-NEXT:    vstrh.32 q0, [r0], #4
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 4
   %0 = bitcast i8* %x to <4 x i16>*
-  %1 = load <4 x i16>, <4 x i16>* %0, align 8
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
   %2 = bitcast i8* %y to <4 x i16>*
-  store <4 x i16> %1, <4 x i16>* %2, align 8
+  store <4 x i16> %1, <4 x i16>* %2, align 2
   ret i8* %z
 }
 
-define i8* @post_strh32_3(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strh32_3:
+define i8* @strh32_3(i8* %y, i8* %x) {
+; CHECK-LABEL: strh32_3:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vldrh.u32 q0, [r1]
 ; CHECK-NEXT:    vstrh.32 q0, [r0]
@@ -882,46 +1276,44 @@ define i8* @post_strh32_3(i8* %y, i8* %x) {
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 3
   %0 = bitcast i8* %x to <4 x i16>*
-  %1 = load <4 x i16>, <4 x i16>* %0, align 8
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
   %2 = bitcast i8* %y to <4 x i16>*
-  store <4 x i16> %1, <4 x i16>* %2, align 8
+  store <4 x i16> %1, <4 x i16>* %2, align 2
   ret i8* %z
 }
 
-define i8* @post_strh32_2(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strh32_2:
+define i8* @strh32_2(i8* %y, i8* %x) {
+; CHECK-LABEL: strh32_2:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vldrh.u32 q0, [r1]
-; CHECK-NEXT:    vstrh.32 q0, [r0]
-; CHECK-NEXT:    adds r0, #2
+; CHECK-NEXT:    vstrh.32 q0, [r0], #2
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 2
   %0 = bitcast i8* %x to <4 x i16>*
-  %1 = load <4 x i16>, <4 x i16>* %0, align 8
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
   %2 = bitcast i8* %y to <4 x i16>*
-  store <4 x i16> %1, <4 x i16>* %2, align 8
+  store <4 x i16> %1, <4 x i16>* %2, align 2
   ret i8* %z
 }
 
-define i8* @post_strh32_254(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strh32_254:
+define i8* @strh32_254(i8* %y, i8* %x) {
+; CHECK-LABEL: strh32_254:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vldrh.u32 q0, [r1]
-; CHECK-NEXT:    vstrh.32 q0, [r0]
-; CHECK-NEXT:    adds r0, #254
+; CHECK-NEXT:    vstrh.32 q0, [r0], #254
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 254
   %0 = bitcast i8* %x to <4 x i16>*
-  %1 = load <4 x i16>, <4 x i16>* %0, align 8
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
   %2 = bitcast i8* %y to <4 x i16>*
-  store <4 x i16> %1, <4 x i16>* %2, align 8
+  store <4 x i16> %1, <4 x i16>* %2, align 2
   ret i8* %z
 }
 
-define i8* @post_strh32_256(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strh32_256:
+define i8* @strh32_256(i8* %y, i8* %x) {
+; CHECK-LABEL: strh32_256:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vldrh.u32 q0, [r1]
 ; CHECK-NEXT:    vstrh.32 q0, [r0]
@@ -930,144 +1322,218 @@ define i8* @post_strh32_256(i8* %y, i8* %x) {
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 256
   %0 = bitcast i8* %x to <4 x i16>*
-  %1 = load <4 x i16>, <4 x i16>* %0, align 8
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
   %2 = bitcast i8* %y to <4 x i16>*
-  store <4 x i16> %1, <4 x i16>* %2, align 8
+  store <4 x i16> %1, <4 x i16>* %2, align 2
   ret i8* %z
 }
 
+define i8* @strh32_m254(i8* %y, i8* %x) {
+; CHECK-LABEL: strh32_m254:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u32 q0, [r1]
+; CHECK-NEXT:    vstrh.32 q0, [r0], #-254
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -254
+  %0 = bitcast i8* %x to <4 x i16>*
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
+  %2 = bitcast i8* %y to <4 x i16>*
+  store <4 x i16> %1, <4 x i16>* %2, align 2
+  ret i8* %z
+}
 
-define i8* @post_strh16_4(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strh16_4:
+define i8* @strh32_m256(i8* %y, i8* %x) {
+; CHECK-LABEL: strh32_m256:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-NEXT:    vstrw.32 q0, [r0]
-; CHECK-NEXT:    adds r0, #4
+; CHECK-NEXT:    vldrh.u32 q0, [r1]
+; CHECK-NEXT:    vstrh.32 q0, [r0]
+; CHECK-NEXT:    sub.w r0, r0, #256
 ; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -256
+  %0 = bitcast i8* %x to <4 x i16>*
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
+  %2 = bitcast i8* %y to <4 x i16>*
+  store <4 x i16> %1, <4 x i16>* %2, align 2
+  ret i8* %z
+}
+
+
+define i8* @strh16_4(i8* %y, i8* %x) {
+; CHECK-LE-LABEL: strh16_4:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-LE-NEXT:    vstrb.8 q0, [r0], #4
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: strh16_4:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-BE-NEXT:    vstrh.16 q0, [r0], #4
+; CHECK-BE-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 4
   %0 = bitcast i8* %x to <8 x i16>*
-  %1 = load <8 x i16>, <8 x i16>* %0, align 8
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
   %2 = bitcast i8* %y to <8 x i16>*
-  store <8 x i16> %1, <8 x i16>* %2, align 8
+  store <8 x i16> %1, <8 x i16>* %2, align 2
   ret i8* %z
 }
 
-define i8* @post_strh16_3(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strh16_3:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-NEXT:    vstrw.32 q0, [r0]
-; CHECK-NEXT:    adds r0, #3
-; CHECK-NEXT:    bx lr
+define i8* @strh16_3(i8* %y, i8* %x) {
+; CHECK-LE-LABEL: strh16_3:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-LE-NEXT:    vstrb.8 q0, [r0], #3
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: strh16_3:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-BE-NEXT:    vstrh.16 q0, [r0]
+; CHECK-BE-NEXT:    adds r0, #3
+; CHECK-BE-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 3
   %0 = bitcast i8* %x to <8 x i16>*
-  %1 = load <8 x i16>, <8 x i16>* %0, align 8
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
   %2 = bitcast i8* %y to <8 x i16>*
-  store <8 x i16> %1, <8 x i16>* %2, align 8
+  store <8 x i16> %1, <8 x i16>* %2, align 2
   ret i8* %z
 }
 
-define i8* @post_strh16_2(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strh16_2:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-NEXT:    vstrw.32 q0, [r0]
-; CHECK-NEXT:    adds r0, #2
-; CHECK-NEXT:    bx lr
+define i8* @strh16_2(i8* %y, i8* %x) {
+; CHECK-LE-LABEL: strh16_2:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-LE-NEXT:    vstrb.8 q0, [r0], #2
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: strh16_2:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-BE-NEXT:    vstrh.16 q0, [r0], #2
+; CHECK-BE-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 2
   %0 = bitcast i8* %x to <8 x i16>*
-  %1 = load <8 x i16>, <8 x i16>* %0, align 8
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
   %2 = bitcast i8* %y to <8 x i16>*
-  store <8 x i16> %1, <8 x i16>* %2, align 8
+  store <8 x i16> %1, <8 x i16>* %2, align 2
   ret i8* %z
 }
 
-define i8* @post_strh16_254(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strh16_254:
+define i8* @strh16_254(i8* %y, i8* %x) {
+; CHECK-LABEL: strh16_254:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-NEXT:    vstrw.32 q0, [r0]
-; CHECK-NEXT:    adds r0, #254
+; CHECK-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-NEXT:    vstrh.16 q0, [r0], #254
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 254
   %0 = bitcast i8* %x to <8 x i16>*
-  %1 = load <8 x i16>, <8 x i16>* %0, align 8
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
   %2 = bitcast i8* %y to <8 x i16>*
-  store <8 x i16> %1, <8 x i16>* %2, align 8
+  store <8 x i16> %1, <8 x i16>* %2, align 2
   ret i8* %z
 }
 
-define i8* @post_strh16_256(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strh16_256:
+define i8* @strh16_256(i8* %y, i8* %x) {
+; CHECK-LABEL: strh16_256:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-NEXT:    vstrw.32 q0, [r0]
+; CHECK-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-NEXT:    vstrh.16 q0, [r0]
 ; CHECK-NEXT:    add.w r0, r0, #256
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 256
   %0 = bitcast i8* %x to <8 x i16>*
-  %1 = load <8 x i16>, <8 x i16>* %0, align 8
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
   %2 = bitcast i8* %y to <8 x i16>*
-  store <8 x i16> %1, <8 x i16>* %2, align 8
+  store <8 x i16> %1, <8 x i16>* %2, align 2
+  ret i8* %z
+}
+
+define i8* @strh16_m254(i8* %y, i8* %x) {
+; CHECK-LABEL: strh16_m254:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-NEXT:    vstrh.16 q0, [r0], #-254
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -254
+  %0 = bitcast i8* %x to <8 x i16>*
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
+  %2 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %1, <8 x i16>* %2, align 2
+  ret i8* %z
+}
+
+define i8* @strh16_m256(i8* %y, i8* %x) {
+; CHECK-LABEL: strh16_m256:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-NEXT:    vstrh.16 q0, [r0]
+; CHECK-NEXT:    sub.w r0, r0, #256
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -256
+  %0 = bitcast i8* %x to <8 x i16>*
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
+  %2 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %1, <8 x i16>* %2, align 2
   ret i8* %z
 }
 
 
-define i8* @post_strb32_4(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strb32_4:
+define i8* @strb32_4(i8* %y, i8* %x) {
+; CHECK-LABEL: strb32_4:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vldrb.u32 q0, [r1]
-; CHECK-NEXT:    vstrb.32 q0, [r0]
-; CHECK-NEXT:    adds r0, #4
+; CHECK-NEXT:    vstrb.32 q0, [r0], #4
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 4
   %0 = bitcast i8* %x to <4 x i8>*
-  %1 = load <4 x i8>, <4 x i8>* %0, align 8
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
   %2 = bitcast i8* %y to <4 x i8>*
-  store <4 x i8> %1, <4 x i8>* %2, align 8
+  store <4 x i8> %1, <4 x i8>* %2, align 1
   ret i8* %z
 }
 
-define i8* @post_strb32_3(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strb32_3:
+define i8* @strb32_3(i8* %y, i8* %x) {
+; CHECK-LABEL: strb32_3:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vldrb.u32 q0, [r1]
-; CHECK-NEXT:    vstrb.32 q0, [r0]
-; CHECK-NEXT:    adds r0, #3
+; CHECK-NEXT:    vstrb.32 q0, [r0], #3
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 3
   %0 = bitcast i8* %x to <4 x i8>*
-  %1 = load <4 x i8>, <4 x i8>* %0, align 8
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
   %2 = bitcast i8* %y to <4 x i8>*
-  store <4 x i8> %1, <4 x i8>* %2, align 8
+  store <4 x i8> %1, <4 x i8>* %2, align 1
   ret i8* %z
 }
 
-define i8* @post_strb32_127(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strb32_127:
+define i8* @strb32_127(i8* %y, i8* %x) {
+; CHECK-LABEL: strb32_127:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vldrb.u32 q0, [r1]
-; CHECK-NEXT:    vstrb.32 q0, [r0]
-; CHECK-NEXT:    adds r0, #127
+; CHECK-NEXT:    vstrb.32 q0, [r0], #127
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 127
   %0 = bitcast i8* %x to <4 x i8>*
-  %1 = load <4 x i8>, <4 x i8>* %0, align 8
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
   %2 = bitcast i8* %y to <4 x i8>*
-  store <4 x i8> %1, <4 x i8>* %2, align 8
+  store <4 x i8> %1, <4 x i8>* %2, align 1
   ret i8* %z
 }
 
-define i8* @post_strb32_128(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strb32_128:
+define i8* @strb32_128(i8* %y, i8* %x) {
+; CHECK-LABEL: strb32_128:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vldrb.u32 q0, [r1]
 ; CHECK-NEXT:    vstrb.32 q0, [r0]
@@ -1076,63 +1542,91 @@ define i8* @post_strb32_128(i8* %y, i8* %x) {
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 128
   %0 = bitcast i8* %x to <4 x i8>*
-  %1 = load <4 x i8>, <4 x i8>* %0, align 8
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
   %2 = bitcast i8* %y to <4 x i8>*
-  store <4 x i8> %1, <4 x i8>* %2, align 8
+  store <4 x i8> %1, <4 x i8>* %2, align 1
   ret i8* %z
 }
 
+define i8* @strb32_m127(i8* %y, i8* %x) {
+; CHECK-LABEL: strb32_m127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u32 q0, [r1]
+; CHECK-NEXT:    vstrb.32 q0, [r0], #-127
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -127
+  %0 = bitcast i8* %x to <4 x i8>*
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
+  %2 = bitcast i8* %y to <4 x i8>*
+  store <4 x i8> %1, <4 x i8>* %2, align 1
+  ret i8* %z
+}
 
-define i8* @post_strb16_4(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strb16_4:
+define i8* @strb32_m128(i8* %y, i8* %x) {
+; CHECK-LABEL: strb32_m128:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u32 q0, [r1]
+; CHECK-NEXT:    vstrb.32 q0, [r0]
+; CHECK-NEXT:    subs r0, #128
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -128
+  %0 = bitcast i8* %x to <4 x i8>*
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
+  %2 = bitcast i8* %y to <4 x i8>*
+  store <4 x i8> %1, <4 x i8>* %2, align 1
+  ret i8* %z
+}
+
+
+define i8* @strb16_4(i8* %y, i8* %x) {
+; CHECK-LABEL: strb16_4:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vldrb.u16 q0, [r1]
-; CHECK-NEXT:    vstrb.16 q0, [r0]
-; CHECK-NEXT:    adds r0, #4
+; CHECK-NEXT:    vstrb.16 q0, [r0], #4
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 4
   %0 = bitcast i8* %x to <8 x i8>*
-  %1 = load <8 x i8>, <8 x i8>* %0, align 8
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
   %2 = bitcast i8* %y to <8 x i8>*
-  store <8 x i8> %1, <8 x i8>* %2, align 8
+  store <8 x i8> %1, <8 x i8>* %2, align 1
   ret i8* %z
 }
 
-define i8* @post_strb16_3(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strb16_3:
+define i8* @strb16_3(i8* %y, i8* %x) {
+; CHECK-LABEL: strb16_3:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vldrb.u16 q0, [r1]
-; CHECK-NEXT:    vstrb.16 q0, [r0]
-; CHECK-NEXT:    adds r0, #3
+; CHECK-NEXT:    vstrb.16 q0, [r0], #3
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 3
   %0 = bitcast i8* %x to <8 x i8>*
-  %1 = load <8 x i8>, <8 x i8>* %0, align 8
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
   %2 = bitcast i8* %y to <8 x i8>*
-  store <8 x i8> %1, <8 x i8>* %2, align 8
+  store <8 x i8> %1, <8 x i8>* %2, align 1
   ret i8* %z
 }
 
-define i8* @post_strb16_127(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strb16_127:
+define i8* @strb16_127(i8* %y, i8* %x) {
+; CHECK-LABEL: strb16_127:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vldrb.u16 q0, [r1]
-; CHECK-NEXT:    vstrb.16 q0, [r0]
-; CHECK-NEXT:    adds r0, #127
+; CHECK-NEXT:    vstrb.16 q0, [r0], #127
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 127
   %0 = bitcast i8* %x to <8 x i8>*
-  %1 = load <8 x i8>, <8 x i8>* %0, align 8
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
   %2 = bitcast i8* %y to <8 x i8>*
-  store <8 x i8> %1, <8 x i8>* %2, align 8
+  store <8 x i8> %1, <8 x i8>* %2, align 1
   ret i8* %z
 }
 
-define i8* @post_strb16_128(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strb16_128:
+define i8* @strb16_128(i8* %y, i8* %x) {
+; CHECK-LABEL: strb16_128:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vldrb.u16 q0, [r1]
 ; CHECK-NEXT:    vstrb.16 q0, [r0]
@@ -1141,105 +1635,312 @@ define i8* @post_strb16_128(i8* %y, i8* %x) {
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 128
   %0 = bitcast i8* %x to <8 x i8>*
-  %1 = load <8 x i8>, <8 x i8>* %0, align 8
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
+  %2 = bitcast i8* %y to <8 x i8>*
+  store <8 x i8> %1, <8 x i8>* %2, align 1
+  ret i8* %z
+}
+
+define i8* @strb16_m127(i8* %y, i8* %x) {
+; CHECK-LABEL: strb16_m127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u16 q0, [r1]
+; CHECK-NEXT:    vstrb.16 q0, [r0], #-127
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -127
+  %0 = bitcast i8* %x to <8 x i8>*
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
   %2 = bitcast i8* %y to <8 x i8>*
-  store <8 x i8> %1, <8 x i8>* %2, align 8
+  store <8 x i8> %1, <8 x i8>* %2, align 1
+  ret i8* %z
+}
+
+define i8* @strb16_m128(i8* %y, i8* %x) {
+; CHECK-LABEL: strb16_m128:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u16 q0, [r1]
+; CHECK-NEXT:    vstrb.16 q0, [r0]
+; CHECK-NEXT:    subs r0, #128
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -128
+  %0 = bitcast i8* %x to <8 x i8>*
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
+  %2 = bitcast i8* %y to <8 x i8>*
+  store <8 x i8> %1, <8 x i8>* %2, align 1
   ret i8* %z
 }
 
 
-define i8* @post_strb8_4(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strb8_4:
+define i8* @strb8_4(i8* %y, i8* %x) {
+; CHECK-LABEL: strb8_4:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-NEXT:    vstrw.32 q0, [r0]
-; CHECK-NEXT:    adds r0, #4
+; CHECK-NEXT:    vldrb.u8 q0, [r1]
+; CHECK-NEXT:    vstrb.8 q0, [r0], #4
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 4
   %0 = bitcast i8* %x to <16 x i8>*
-  %1 = load <16 x i8>, <16 x i8>* %0, align 8
+  %1 = load <16 x i8>, <16 x i8>* %0, align 1
   %2 = bitcast i8* %y to <16 x i8>*
-  store <16 x i8> %1, <16 x i8>* %2, align 8
+  store <16 x i8> %1, <16 x i8>* %2, align 1
   ret i8* %z
 }
 
-define i8* @post_strb8_3(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strb8_3:
+define i8* @strb8_3(i8* %y, i8* %x) {
+; CHECK-LABEL: strb8_3:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-NEXT:    vstrw.32 q0, [r0]
-; CHECK-NEXT:    adds r0, #3
+; CHECK-NEXT:    vldrb.u8 q0, [r1]
+; CHECK-NEXT:    vstrb.8 q0, [r0], #3
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 3
   %0 = bitcast i8* %x to <16 x i8>*
-  %1 = load <16 x i8>, <16 x i8>* %0, align 8
+  %1 = load <16 x i8>, <16 x i8>* %0, align 1
   %2 = bitcast i8* %y to <16 x i8>*
-  store <16 x i8> %1, <16 x i8>* %2, align 8
+  store <16 x i8> %1, <16 x i8>* %2, align 1
   ret i8* %z
 }
 
-define i8* @post_strb8_127(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strb8_127:
+define i8* @strb8_127(i8* %y, i8* %x) {
+; CHECK-LABEL: strb8_127:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-NEXT:    vstrw.32 q0, [r0]
-; CHECK-NEXT:    adds r0, #127
+; CHECK-NEXT:    vldrb.u8 q0, [r1]
+; CHECK-NEXT:    vstrb.8 q0, [r0], #127
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 127
   %0 = bitcast i8* %x to <16 x i8>*
-  %1 = load <16 x i8>, <16 x i8>* %0, align 8
+  %1 = load <16 x i8>, <16 x i8>* %0, align 1
   %2 = bitcast i8* %y to <16 x i8>*
-  store <16 x i8> %1, <16 x i8>* %2, align 8
+  store <16 x i8> %1, <16 x i8>* %2, align 1
   ret i8* %z
 }
 
-define i8* @post_strb8_128(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strb8_128:
+define i8* @strb8_128(i8* %y, i8* %x) {
+; CHECK-LABEL: strb8_128:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-NEXT:    vstrw.32 q0, [r0]
+; CHECK-NEXT:    vldrb.u8 q0, [r1]
+; CHECK-NEXT:    vstrb.8 q0, [r0]
 ; CHECK-NEXT:    adds r0, #128
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 128
   %0 = bitcast i8* %x to <16 x i8>*
-  %1 = load <16 x i8>, <16 x i8>* %0, align 8
+  %1 = load <16 x i8>, <16 x i8>* %0, align 1
   %2 = bitcast i8* %y to <16 x i8>*
-  store <16 x i8> %1, <16 x i8>* %2, align 8
+  store <16 x i8> %1, <16 x i8>* %2, align 1
   ret i8* %z
 }
 
-define i8* @post_strf32_4(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strf32_4:
+define i8* @strb8_m127(i8* %y, i8* %x) {
+; CHECK-LABEL: strb8_m127:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-NEXT:    vstrw.32 q0, [r0]
-; CHECK-NEXT:    adds r0, #4
+; CHECK-NEXT:    vldrb.u8 q0, [r1]
+; CHECK-NEXT:    vstrb.8 q0, [r0], #-127
 ; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -127
+  %0 = bitcast i8* %x to <16 x i8>*
+  %1 = load <16 x i8>, <16 x i8>* %0, align 1
+  %2 = bitcast i8* %y to <16 x i8>*
+  store <16 x i8> %1, <16 x i8>* %2, align 1
+  ret i8* %z
+}
+
+define i8* @strb8_m128(i8* %y, i8* %x) {
+; CHECK-LABEL: strb8_m128:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u8 q0, [r1]
+; CHECK-NEXT:    vstrb.8 q0, [r0]
+; CHECK-NEXT:    subs r0, #128
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -128
+  %0 = bitcast i8* %x to <16 x i8>*
+  %1 = load <16 x i8>, <16 x i8>* %0, align 1
+  %2 = bitcast i8* %y to <16 x i8>*
+  store <16 x i8> %1, <16 x i8>* %2, align 1
+  ret i8* %z
+}
+
+
+define i8* @strf32_4(i8* %y, i8* %x) {
+; CHECK-LE-LABEL: strf32_4:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-LE-NEXT:    vstrb.8 q0, [r0], #4
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: strf32_4:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-BE-NEXT:    vstrw.32 q0, [r0], #4
+; CHECK-BE-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 4
   %0 = bitcast i8* %x to <4 x float>*
-  %1 = load <4 x float>, <4 x float>* %0, align 8
+  %1 = load <4 x float>, <4 x float>* %0, align 4
   %2 = bitcast i8* %y to <4 x float>*
-  store <4 x float> %1, <4 x float>* %2, align 8
+  store <4 x float> %1, <4 x float>* %2, align 4
   ret i8* %z
 }
 
-define i8* @post_strf16_4(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strf16_4:
+define i8* @strf16_4(i8* %y, i8* %x) {
+; CHECK-LE-LABEL: strf16_4:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-LE-NEXT:    vstrb.8 q0, [r0], #4
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: strf16_4:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-BE-NEXT:    vstrh.16 q0, [r0], #4
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 4
+  %0 = bitcast i8* %x to <8 x half>*
+  %1 = load <8 x half>, <8 x half>* %0, align 2
+  %2 = bitcast i8* %y to <8 x half>*
+  store <8 x half> %1, <8 x half>* %2, align 2
+  ret i8* %z
+}
+
+define i8* @strwi32_align1(i8* %y, i8* %x) {
+; CHECK-LE-LABEL: strwi32_align1:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-LE-NEXT:    vstrb.8 q0, [r0], #3
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: strwi32_align1:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-BE-NEXT:    vrev32.8 q0, q0
+; CHECK-BE-NEXT:    vstrb.8 q0, [r0]
+; CHECK-BE-NEXT:    adds r0, #3
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 3
+  %0 = bitcast i8* %x to <4 x i32>*
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
+  %2 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %1, <4 x i32>* %2, align 1
+  ret i8* %z
+}
+
+define i8* @strhi16_align1(i8* %y, i8* %x) {
+; CHECK-LE-LABEL: strhi16_align1:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-LE-NEXT:    vstrb.8 q0, [r0], #3
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: strhi16_align1:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-BE-NEXT:    vrev16.8 q0, q0
+; CHECK-BE-NEXT:    vstrb.8 q0, [r0]
+; CHECK-BE-NEXT:    adds r0, #3
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 3
+  %0 = bitcast i8* %x to <8 x i16>*
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
+  %2 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %1, <8 x i16>* %2, align 1
+  ret i8* %z
+}
+
+define i8* @strhi32_align1(i8* %y, i8* %x) {
+; CHECK-LABEL: strhi32_align1:
 ; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .pad #8
+; CHECK-NEXT:    sub sp, #8
 ; CHECK-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-NEXT:    vstrw.32 q0, [r0]
-; CHECK-NEXT:    adds r0, #4
+; CHECK-NEXT:    mov r1, sp
+; CHECK-NEXT:    vstrh.32 q0, [r1]
+; CHECK-NEXT:    ldrd r1, r2, [sp]
+; CHECK-NEXT:    str r1, [r0]
+; CHECK-NEXT:    str r2, [r0, #4]
+; CHECK-NEXT:    adds r0, #3
+; CHECK-NEXT:    add sp, #8
 ; CHECK-NEXT:    bx lr
 entry:
-  %z = getelementptr inbounds i8, i8* %y, i32 4
+  %z = getelementptr inbounds i8, i8* %y, i32 3
+  %0 = bitcast i8* %x to <4 x i32>*
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
+  %2 = bitcast i8* %y to <4 x i16>*
+  %3 = trunc <4 x i32> %1 to <4 x i16>
+  store <4 x i16> %3, <4 x i16>* %2, align 1
+  ret i8* %z
+}
+
+define i8* @strf32_align1(i8* %y, i8* %x) {
+; CHECK-LE-LABEL: strf32_align1:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-LE-NEXT:    vstrb.8 q0, [r0], #3
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: strf32_align1:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-BE-NEXT:    vrev32.8 q0, q0
+; CHECK-BE-NEXT:    vstrb.8 q0, [r0]
+; CHECK-BE-NEXT:    adds r0, #3
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 3
+  %0 = bitcast i8* %x to <4 x float>*
+  %1 = load <4 x float>, <4 x float>* %0, align 4
+  %2 = bitcast i8* %y to <4 x float>*
+  store <4 x float> %1, <4 x float>* %2, align 1
+  ret i8* %z
+}
+
+define i8* @strf16_align1(i8* %y, i8* %x) {
+; CHECK-LE-LABEL: strf16_align1:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-LE-NEXT:    vstrb.8 q0, [r0], #3
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: strf16_align1:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-BE-NEXT:    vrev16.8 q0, q0
+; CHECK-BE-NEXT:    vstrb.8 q0, [r0]
+; CHECK-BE-NEXT:    adds r0, #3
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 3
   %0 = bitcast i8* %x to <8 x half>*
-  %1 = load <8 x half>, <8 x half>* %0, align 8
+  %1 = load <8 x half>, <8 x half>* %0, align 2
   %2 = bitcast i8* %y to <8 x half>*
-  store <8 x half> %1, <8 x half>* %2, align 8
+  store <8 x half> %1, <8 x half>* %2, align 1
+  ret i8* %z
+}
+
+define i8* @strf16_align8(i8* %y, i8* %x) {
+; CHECK-LE-LABEL: strf16_align8:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-LE-NEXT:    vstrb.8 q0, [r0], #16
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: strf16_align8:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-BE-NEXT:    vstrh.16 q0, [r0], #16
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 16
+  %0 = bitcast i8* %x to <8 x i16>*
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
+  %2 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %1, <8 x i16>* %2, align 8
   ret i8* %z
 }
diff --git a/llvm/test/CodeGen/Thumb2/mve-ldst-preinc.ll b/llvm/test/CodeGen/Thumb2/mve-ldst-preinc.ll
index c0ff5cfc8..bf732c628 100644
--- a/llvm/test/CodeGen/Thumb2/mve-ldst-preinc.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-ldst-preinc.ll
@@ -1,72 +1,76 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve %s -o - | FileCheck %s
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-LE
+; RUN: llc -mtriple=thumbebv8.1m.main-arm-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-BE
 
-define i8* @post_ldrwu32_4(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrwu32_4:
+define i8* @ldrwu32_4(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrwu32_4:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r0, #4]
-; CHECK-NEXT:    adds r0, #4
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #4]!
 ; CHECK-NEXT:    vstrw.32 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 4
   %0 = bitcast i8* %z to <4 x i32>*
-  %1 = load <4 x i32>, <4 x i32>* %0, align 8
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
   %2 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %1, <4 x i32>* %2, align 8
+  store <4 x i32> %1, <4 x i32>* %2, align 4
   ret i8* %z
 }
 
-define i8* @post_ldrwu32_3(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrwu32_3:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    adds r0, #3
-; CHECK-NEXT:    vldrw.u32 q0, [r0]
-; CHECK-NEXT:    vstrw.32 q0, [r1]
-; CHECK-NEXT:    bx lr
+define i8* @ldrwu32_3(i8* %x, i8* %y) {
+; CHECK-LE-LABEL: ldrwu32_3:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldrb.u8 q0, [r0, #3]!
+; CHECK-LE-NEXT:    vstrw.32 q0, [r1]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: ldrwu32_3:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    adds r0, #3
+; CHECK-BE-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-BE-NEXT:    vstrw.32 q0, [r1]
+; CHECK-BE-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 3
   %0 = bitcast i8* %z to <4 x i32>*
-  %1 = load <4 x i32>, <4 x i32>* %0, align 8
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
   %2 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %1, <4 x i32>* %2, align 8
+  store <4 x i32> %1, <4 x i32>* %2, align 4
   ret i8* %z
 }
 
-define i8* @post_ldrwu32_m4(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrwu32_m4:
+define i8* @ldrwu32_m4(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrwu32_m4:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r0, #-4]
-; CHECK-NEXT:    subs r0, #4
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #-4]!
 ; CHECK-NEXT:    vstrw.32 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 -4
   %0 = bitcast i8* %z to <4 x i32>*
-  %1 = load <4 x i32>, <4 x i32>* %0, align 8
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
   %2 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %1, <4 x i32>* %2, align 8
+  store <4 x i32> %1, <4 x i32>* %2, align 4
   ret i8* %z
 }
 
-define i8* @post_ldrwu32_508(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrwu32_508:
+define i8* @ldrwu32_508(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrwu32_508:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    add.w r0, r0, #508
-; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #508]!
 ; CHECK-NEXT:    vstrw.32 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 508
   %0 = bitcast i8* %z to <4 x i32>*
-  %1 = load <4 x i32>, <4 x i32>* %0, align 8
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
   %2 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %1, <4 x i32>* %2, align 8
+  store <4 x i32> %1, <4 x i32>* %2, align 4
   ret i8* %z
 }
 
-define i8* @post_ldrwu32_512(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrwu32_512:
+define i8* @ldrwu32_512(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrwu32_512:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    add.w r0, r0, #512
 ; CHECK-NEXT:    vldrw.u32 q0, [r0]
@@ -75,30 +79,29 @@ define i8* @post_ldrwu32_512(i8* %x, i8* %y) {
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 512
   %0 = bitcast i8* %z to <4 x i32>*
-  %1 = load <4 x i32>, <4 x i32>* %0, align 8
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
   %2 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %1, <4 x i32>* %2, align 8
+  store <4 x i32> %1, <4 x i32>* %2, align 4
   ret i8* %z
 }
 
-define i8* @post_ldrwu32_m508(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrwu32_m508:
+define i8* @ldrwu32_m508(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrwu32_m508:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    sub.w r0, r0, #508
-; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #-508]!
 ; CHECK-NEXT:    vstrw.32 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 -508
   %0 = bitcast i8* %z to <4 x i32>*
-  %1 = load <4 x i32>, <4 x i32>* %0, align 8
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
   %2 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %1, <4 x i32>* %2, align 8
+  store <4 x i32> %1, <4 x i32>* %2, align 4
   ret i8* %z
 }
 
-define i8* @post_ldrwu32_m512(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrwu32_m512:
+define i8* @ldrwu32_m512(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrwu32_m512:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    sub.w r0, r0, #512
 ; CHECK-NEXT:    vldrw.u32 q0, [r0]
@@ -107,32 +110,31 @@ define i8* @post_ldrwu32_m512(i8* %x, i8* %y) {
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 -512
   %0 = bitcast i8* %z to <4 x i32>*
-  %1 = load <4 x i32>, <4 x i32>* %0, align 8
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
   %2 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %1, <4 x i32>* %2, align 8
+  store <4 x i32> %1, <4 x i32>* %2, align 4
   ret i8* %z
 }
 
 
-define i8* @post_ldrhu32_4(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrhu32_4:
+define i8* @ldrhu32_4(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrhu32_4:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrh.u32 q0, [r0, #4]
-; CHECK-NEXT:    adds r0, #4
+; CHECK-NEXT:    vldrh.u32 q0, [r0, #4]!
 ; CHECK-NEXT:    vstrw.32 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 4
   %0 = bitcast i8* %z to <4 x i16>*
-  %1 = load <4 x i16>, <4 x i16>* %0, align 8
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
   %2 = zext <4 x i16> %1 to <4 x i32>
   %3 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %2, <4 x i32>* %3, align 8
+  store <4 x i32> %2, <4 x i32>* %3, align 4
   ret i8* %z
 }
 
-define i8* @post_ldrhu32_3(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrhu32_3:
+define i8* @ldrhu32_3(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrhu32_3:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    adds r0, #3
 ; CHECK-NEXT:    vldrh.u32 q0, [r0]
@@ -141,49 +143,47 @@ define i8* @post_ldrhu32_3(i8* %x, i8* %y) {
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 3
   %0 = bitcast i8* %z to <4 x i16>*
-  %1 = load <4 x i16>, <4 x i16>* %0, align 8
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
   %2 = zext <4 x i16> %1 to <4 x i32>
   %3 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %2, <4 x i32>* %3, align 8
+  store <4 x i32> %2, <4 x i32>* %3, align 4
   ret i8* %z
 }
 
-define i8* @post_ldrhu32_2(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrhu32_2:
+define i8* @ldrhu32_2(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrhu32_2:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    adds r0, #2
-; CHECK-NEXT:    vldrh.u32 q0, [r0]
+; CHECK-NEXT:    vldrh.u32 q0, [r0, #2]!
 ; CHECK-NEXT:    vstrw.32 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 2
   %0 = bitcast i8* %z to <4 x i16>*
-  %1 = load <4 x i16>, <4 x i16>* %0, align 8
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
   %2 = zext <4 x i16> %1 to <4 x i32>
   %3 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %2, <4 x i32>* %3, align 8
+  store <4 x i32> %2, <4 x i32>* %3, align 4
   ret i8* %z
 }
 
-define i8* @post_ldrhu32_254(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrhu32_254:
+define i8* @ldrhu32_254(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrhu32_254:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    adds r0, #254
-; CHECK-NEXT:    vldrh.u32 q0, [r0]
+; CHECK-NEXT:    vldrh.u32 q0, [r0, #254]!
 ; CHECK-NEXT:    vstrw.32 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 254
   %0 = bitcast i8* %z to <4 x i16>*
-  %1 = load <4 x i16>, <4 x i16>* %0, align 8
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
   %2 = zext <4 x i16> %1 to <4 x i32>
   %3 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %2, <4 x i32>* %3, align 8
+  store <4 x i32> %2, <4 x i32>* %3, align 4
   ret i8* %z
 }
 
-define i8* @post_ldrhu32_256(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrhu32_256:
+define i8* @ldrhu32_256(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrhu32_256:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    add.w r0, r0, #256
 ; CHECK-NEXT:    vldrh.u32 q0, [r0]
@@ -192,33 +192,65 @@ define i8* @post_ldrhu32_256(i8* %x, i8* %y) {
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 256
   %0 = bitcast i8* %z to <4 x i16>*
-  %1 = load <4 x i16>, <4 x i16>* %0, align 8
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
   %2 = zext <4 x i16> %1 to <4 x i32>
   %3 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %2, <4 x i32>* %3, align 8
+  store <4 x i32> %2, <4 x i32>* %3, align 4
   ret i8* %z
 }
 
+define i8* @ldrhu32_m254(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrhu32_m254:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u32 q0, [r0, #-254]!
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -254
+  %0 = bitcast i8* %z to <4 x i16>*
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
+  %2 = zext <4 x i16> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
 
-define i8* @post_ldrhs32_4(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrhs32_4:
+define i8* @ldrhu32_m256(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrhu32_m256:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrh.s32 q0, [r0, #4]
-; CHECK-NEXT:    adds r0, #4
+; CHECK-NEXT:    sub.w r0, r0, #256
+; CHECK-NEXT:    vldrh.u32 q0, [r0]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -256
+  %0 = bitcast i8* %z to <4 x i16>*
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
+  %2 = zext <4 x i16> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
+
+
+define i8* @ldrhs32_4(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrhs32_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.s32 q0, [r0, #4]!
 ; CHECK-NEXT:    vstrw.32 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 4
   %0 = bitcast i8* %z to <4 x i16>*
-  %1 = load <4 x i16>, <4 x i16>* %0, align 8
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
   %2 = sext <4 x i16> %1 to <4 x i32>
   %3 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %2, <4 x i32>* %3, align 8
+  store <4 x i32> %2, <4 x i32>* %3, align 4
   ret i8* %z
 }
 
-define i8* @post_ldrhs32_3(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrhs32_3:
+define i8* @ldrhs32_3(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrhs32_3:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    adds r0, #3
 ; CHECK-NEXT:    vldrh.s32 q0, [r0]
@@ -227,49 +259,47 @@ define i8* @post_ldrhs32_3(i8* %x, i8* %y) {
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 3
   %0 = bitcast i8* %z to <4 x i16>*
-  %1 = load <4 x i16>, <4 x i16>* %0, align 8
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
   %2 = sext <4 x i16> %1 to <4 x i32>
   %3 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %2, <4 x i32>* %3, align 8
+  store <4 x i32> %2, <4 x i32>* %3, align 4
   ret i8* %z
 }
 
-define i8* @post_ldrhs32_2(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrhs32_2:
+define i8* @ldrhs32_2(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrhs32_2:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    adds r0, #2
-; CHECK-NEXT:    vldrh.s32 q0, [r0]
+; CHECK-NEXT:    vldrh.s32 q0, [r0, #2]!
 ; CHECK-NEXT:    vstrw.32 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 2
   %0 = bitcast i8* %z to <4 x i16>*
-  %1 = load <4 x i16>, <4 x i16>* %0, align 8
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
   %2 = sext <4 x i16> %1 to <4 x i32>
   %3 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %2, <4 x i32>* %3, align 8
+  store <4 x i32> %2, <4 x i32>* %3, align 4
   ret i8* %z
 }
 
-define i8* @post_ldrhs32_254(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrhs32_254:
+define i8* @ldrhs32_254(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrhs32_254:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    adds r0, #254
-; CHECK-NEXT:    vldrh.s32 q0, [r0]
+; CHECK-NEXT:    vldrh.s32 q0, [r0, #254]!
 ; CHECK-NEXT:    vstrw.32 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 254
   %0 = bitcast i8* %z to <4 x i16>*
-  %1 = load <4 x i16>, <4 x i16>* %0, align 8
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
   %2 = sext <4 x i16> %1 to <4 x i32>
   %3 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %2, <4 x i32>* %3, align 8
+  store <4 x i32> %2, <4 x i32>* %3, align 4
   ret i8* %z
 }
 
-define i8* @post_ldrhs32_256(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrhs32_256:
+define i8* @ldrhs32_256(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrhs32_256:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    add.w r0, r0, #256
 ; CHECK-NEXT:    vldrh.s32 q0, [r0]
@@ -278,148 +308,212 @@ define i8* @post_ldrhs32_256(i8* %x, i8* %y) {
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 256
   %0 = bitcast i8* %z to <4 x i16>*
-  %1 = load <4 x i16>, <4 x i16>* %0, align 8
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
   %2 = sext <4 x i16> %1 to <4 x i32>
   %3 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %2, <4 x i32>* %3, align 8
+  store <4 x i32> %2, <4 x i32>* %3, align 4
   ret i8* %z
 }
 
+define i8* @ldrhs32_m254(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrhs32_m254:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.s32 q0, [r0, #-254]!
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -254
+  %0 = bitcast i8* %z to <4 x i16>*
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
+  %2 = sext <4 x i16> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
 
-define i8* @post_ldrhu16_4(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrhu16_4:
+define i8* @ldrhs32_m256(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrhs32_m256:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r0, #4]
-; CHECK-NEXT:    adds r0, #4
+; CHECK-NEXT:    sub.w r0, r0, #256
+; CHECK-NEXT:    vldrh.s32 q0, [r0]
 ; CHECK-NEXT:    vstrw.32 q0, [r1]
 ; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -256
+  %0 = bitcast i8* %z to <4 x i16>*
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
+  %2 = sext <4 x i16> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
+
+
+define i8* @ldrhu16_4(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrhu16_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r0, #4]!
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 4
   %0 = bitcast i8* %z to <8 x i16>*
-  %1 = load <8 x i16>, <8 x i16>* %0, align 8
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
   %2 = bitcast i8* %y to <8 x i16>*
-  store <8 x i16> %1, <8 x i16>* %2, align 8
+  store <8 x i16> %1, <8 x i16>* %2, align 2
   ret i8* %z
 }
 
-define i8* @post_ldrhu16_3(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrhu16_3:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    adds r0, #3
-; CHECK-NEXT:    vldrw.u32 q0, [r0]
-; CHECK-NEXT:    vstrw.32 q0, [r1]
-; CHECK-NEXT:    bx lr
+define i8* @ldrhu16_3(i8* %x, i8* %y) {
+; CHECK-LE-LABEL: ldrhu16_3:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldrb.u8 q0, [r0, #3]!
+; CHECK-LE-NEXT:    vstrh.16 q0, [r1]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: ldrhu16_3:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    adds r0, #3
+; CHECK-BE-NEXT:    vldrh.u16 q0, [r0]
+; CHECK-BE-NEXT:    vstrh.16 q0, [r1]
+; CHECK-BE-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 3
   %0 = bitcast i8* %z to <8 x i16>*
-  %1 = load <8 x i16>, <8 x i16>* %0, align 8
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
   %2 = bitcast i8* %y to <8 x i16>*
-  store <8 x i16> %1, <8 x i16>* %2, align 8
+  store <8 x i16> %1, <8 x i16>* %2, align 2
   ret i8* %z
 }
 
-define i8* @post_ldrhu16_2(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrhu16_2:
+define i8* @ldrhu16_2(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrhu16_2:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    adds r0, #2
-; CHECK-NEXT:    vldrw.u32 q0, [r0]
-; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q0, [r0, #2]!
+; CHECK-NEXT:    vstrh.16 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 2
   %0 = bitcast i8* %z to <8 x i16>*
-  %1 = load <8 x i16>, <8 x i16>* %0, align 8
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
   %2 = bitcast i8* %y to <8 x i16>*
-  store <8 x i16> %1, <8 x i16>* %2, align 8
+  store <8 x i16> %1, <8 x i16>* %2, align 2
   ret i8* %z
 }
 
-define i8* @post_ldrhu16_254(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrhu16_254:
+define i8* @ldrhu16_254(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrhu16_254:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    adds r0, #254
-; CHECK-NEXT:    vldrw.u32 q0, [r0]
-; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q0, [r0, #254]!
+; CHECK-NEXT:    vstrh.16 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 254
   %0 = bitcast i8* %z to <8 x i16>*
-  %1 = load <8 x i16>, <8 x i16>* %0, align 8
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
   %2 = bitcast i8* %y to <8 x i16>*
-  store <8 x i16> %1, <8 x i16>* %2, align 8
+  store <8 x i16> %1, <8 x i16>* %2, align 2
   ret i8* %z
 }
 
-define i8* @post_ldrhu16_256(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrhu16_256:
+define i8* @ldrhu16_256(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrhu16_256:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    add.w r0, r0, #256
-; CHECK-NEXT:    vldrw.u32 q0, [r0]
-; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q0, [r0]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 256
   %0 = bitcast i8* %z to <8 x i16>*
-  %1 = load <8 x i16>, <8 x i16>* %0, align 8
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
   %2 = bitcast i8* %y to <8 x i16>*
-  store <8 x i16> %1, <8 x i16>* %2, align 8
+  store <8 x i16> %1, <8 x i16>* %2, align 2
+  ret i8* %z
+}
+
+define i8* @ldrhu16_m254(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrhu16_m254:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r0, #-254]!
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -254
+  %0 = bitcast i8* %z to <8 x i16>*
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
+  %2 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %1, <8 x i16>* %2, align 2
+  ret i8* %z
+}
+
+define i8* @ldrhu16_m256(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrhu16_m256:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    sub.w r0, r0, #256
+; CHECK-NEXT:    vldrh.u16 q0, [r0]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -256
+  %0 = bitcast i8* %z to <8 x i16>*
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
+  %2 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %1, <8 x i16>* %2, align 2
   ret i8* %z
 }
 
 
-define i8* @post_ldrbu32_4(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrbu32_4:
+define i8* @ldrbu32_4(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbu32_4:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrb.u32 q0, [r0, #4]
-; CHECK-NEXT:    adds r0, #4
+; CHECK-NEXT:    vldrb.u32 q0, [r0, #4]!
 ; CHECK-NEXT:    vstrw.32 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 4
   %0 = bitcast i8* %z to <4 x i8>*
-  %1 = load <4 x i8>, <4 x i8>* %0, align 8
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
   %2 = zext <4 x i8> %1 to <4 x i32>
   %3 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %2, <4 x i32>* %3, align 8
+  store <4 x i32> %2, <4 x i32>* %3, align 4
   ret i8* %z
 }
 
-define i8* @post_ldrbu32_3(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrbu32_3:
+define i8* @ldrbu32_3(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbu32_3:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    adds r0, #3
-; CHECK-NEXT:    vldrb.u32 q0, [r0]
+; CHECK-NEXT:    vldrb.u32 q0, [r0, #3]!
 ; CHECK-NEXT:    vstrw.32 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 3
   %0 = bitcast i8* %z to <4 x i8>*
-  %1 = load <4 x i8>, <4 x i8>* %0, align 8
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
   %2 = zext <4 x i8> %1 to <4 x i32>
   %3 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %2, <4 x i32>* %3, align 8
+  store <4 x i32> %2, <4 x i32>* %3, align 4
   ret i8* %z
 }
 
-define i8* @post_ldrbu32_127(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrbu32_127:
+define i8* @ldrbu32_127(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbu32_127:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    adds r0, #127
-; CHECK-NEXT:    vldrb.u32 q0, [r0]
+; CHECK-NEXT:    vldrb.u32 q0, [r0, #127]!
 ; CHECK-NEXT:    vstrw.32 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 127
   %0 = bitcast i8* %z to <4 x i8>*
-  %1 = load <4 x i8>, <4 x i8>* %0, align 8
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
   %2 = zext <4 x i8> %1 to <4 x i32>
   %3 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %2, <4 x i32>* %3, align 8
+  store <4 x i32> %2, <4 x i32>* %3, align 4
   ret i8* %z
 }
 
-define i8* @post_ldrbu32_128(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrbu32_128:
+define i8* @ldrbu32_128(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbu32_128:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    adds r0, #128
 ; CHECK-NEXT:    vldrb.u32 q0, [r0]
@@ -428,67 +522,97 @@ define i8* @post_ldrbu32_128(i8* %x, i8* %y) {
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 128
   %0 = bitcast i8* %z to <4 x i8>*
-  %1 = load <4 x i8>, <4 x i8>* %0, align 8
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
+  %2 = zext <4 x i8> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
+
+define i8* @ldrbu32_m127(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbu32_m127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u32 q0, [r0, #-127]!
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -127
+  %0 = bitcast i8* %z to <4 x i8>*
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
+  %2 = zext <4 x i8> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
+
+define i8* @ldrbu32_m128(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbu32_m128:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    subs r0, #128
+; CHECK-NEXT:    vldrb.u32 q0, [r0]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -128
+  %0 = bitcast i8* %z to <4 x i8>*
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
   %2 = zext <4 x i8> %1 to <4 x i32>
   %3 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %2, <4 x i32>* %3, align 8
+  store <4 x i32> %2, <4 x i32>* %3, align 4
   ret i8* %z
 }
 
 
-define i8* @post_ldrbs32_4(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrbs32_4:
+define i8* @ldrbs32_4(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbs32_4:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrb.s32 q0, [r0, #4]
-; CHECK-NEXT:    adds r0, #4
+; CHECK-NEXT:    vldrb.s32 q0, [r0, #4]!
 ; CHECK-NEXT:    vstrw.32 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 4
   %0 = bitcast i8* %z to <4 x i8>*
-  %1 = load <4 x i8>, <4 x i8>* %0, align 8
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
   %2 = sext <4 x i8> %1 to <4 x i32>
   %3 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %2, <4 x i32>* %3, align 8
+  store <4 x i32> %2, <4 x i32>* %3, align 4
   ret i8* %z
 }
 
-define i8* @post_ldrbs32_3(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrbs32_3:
+define i8* @ldrbs32_3(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbs32_3:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    adds r0, #3
-; CHECK-NEXT:    vldrb.s32 q0, [r0]
+; CHECK-NEXT:    vldrb.s32 q0, [r0, #3]!
 ; CHECK-NEXT:    vstrw.32 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 3
   %0 = bitcast i8* %z to <4 x i8>*
-  %1 = load <4 x i8>, <4 x i8>* %0, align 8
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
   %2 = sext <4 x i8> %1 to <4 x i32>
   %3 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %2, <4 x i32>* %3, align 8
+  store <4 x i32> %2, <4 x i32>* %3, align 4
   ret i8* %z
 }
 
-define i8* @post_ldrbs32_127(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrbs32_127:
+define i8* @ldrbs32_127(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbs32_127:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    adds r0, #127
-; CHECK-NEXT:    vldrb.s32 q0, [r0]
+; CHECK-NEXT:    vldrb.s32 q0, [r0, #127]!
 ; CHECK-NEXT:    vstrw.32 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 127
   %0 = bitcast i8* %z to <4 x i8>*
-  %1 = load <4 x i8>, <4 x i8>* %0, align 8
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
   %2 = sext <4 x i8> %1 to <4 x i32>
   %3 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %2, <4 x i32>* %3, align 8
+  store <4 x i32> %2, <4 x i32>* %3, align 4
   ret i8* %z
 }
 
-define i8* @post_ldrbs32_128(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrbs32_128:
+define i8* @ldrbs32_128(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbs32_128:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    adds r0, #128
 ; CHECK-NEXT:    vldrb.s32 q0, [r0]
@@ -497,318 +621,590 @@ define i8* @post_ldrbs32_128(i8* %x, i8* %y) {
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 128
   %0 = bitcast i8* %z to <4 x i8>*
-  %1 = load <4 x i8>, <4 x i8>* %0, align 8
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
   %2 = sext <4 x i8> %1 to <4 x i32>
   %3 = bitcast i8* %y to <4 x i32>*
-  store <4 x i32> %2, <4 x i32>* %3, align 8
+  store <4 x i32> %2, <4 x i32>* %3, align 4
   ret i8* %z
 }
 
+define i8* @ldrbs32_m127(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbs32_m127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.s32 q0, [r0, #-127]!
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -127
+  %0 = bitcast i8* %z to <4 x i8>*
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
+  %2 = sext <4 x i8> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
 
-define i8* @post_ldrbu16_4(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrbu16_4:
+define i8* @ldrbs32_m128(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbs32_m128:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrb.u16 q0, [r0, #4]
-; CHECK-NEXT:    adds r0, #4
+; CHECK-NEXT:    subs r0, #128
+; CHECK-NEXT:    vldrb.s32 q0, [r0]
 ; CHECK-NEXT:    vstrw.32 q0, [r1]
 ; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -128
+  %0 = bitcast i8* %z to <4 x i8>*
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
+  %2 = sext <4 x i8> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
+
+
+define i8* @ldrbu16_4(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbu16_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u16 q0, [r0, #4]!
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 4
   %0 = bitcast i8* %z to <8 x i8>*
-  %1 = load <8 x i8>, <8 x i8>* %0, align 8
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
   %2 = zext <8 x i8> %1 to <8 x i16>
   %3 = bitcast i8* %y to <8 x i16>*
-  store <8 x i16> %2, <8 x i16>* %3, align 8
+  store <8 x i16> %2, <8 x i16>* %3, align 2
   ret i8* %z
 }
 
-define i8* @post_ldrbu16_3(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrbu16_3:
+define i8* @ldrbu16_3(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbu16_3:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    adds r0, #3
-; CHECK-NEXT:    vldrb.u16 q0, [r0]
-; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vldrb.u16 q0, [r0, #3]!
+; CHECK-NEXT:    vstrh.16 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 3
   %0 = bitcast i8* %z to <8 x i8>*
-  %1 = load <8 x i8>, <8 x i8>* %0, align 8
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
   %2 = zext <8 x i8> %1 to <8 x i16>
   %3 = bitcast i8* %y to <8 x i16>*
-  store <8 x i16> %2, <8 x i16>* %3, align 8
+  store <8 x i16> %2, <8 x i16>* %3, align 2
   ret i8* %z
 }
 
-define i8* @post_ldrbu16_127(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrbu16_127:
+define i8* @ldrbu16_127(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbu16_127:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    adds r0, #127
-; CHECK-NEXT:    vldrb.u16 q0, [r0]
-; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vldrb.u16 q0, [r0, #127]!
+; CHECK-NEXT:    vstrh.16 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 127
   %0 = bitcast i8* %z to <8 x i8>*
-  %1 = load <8 x i8>, <8 x i8>* %0, align 8
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
   %2 = zext <8 x i8> %1 to <8 x i16>
   %3 = bitcast i8* %y to <8 x i16>*
-  store <8 x i16> %2, <8 x i16>* %3, align 8
+  store <8 x i16> %2, <8 x i16>* %3, align 2
   ret i8* %z
 }
 
-define i8* @post_ldrbu16_128(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrbu16_128:
+define i8* @ldrbu16_128(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbu16_128:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    adds r0, #128
 ; CHECK-NEXT:    vldrb.u16 q0, [r0]
-; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 128
   %0 = bitcast i8* %z to <8 x i8>*
-  %1 = load <8 x i8>, <8 x i8>* %0, align 8
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
   %2 = zext <8 x i8> %1 to <8 x i16>
   %3 = bitcast i8* %y to <8 x i16>*
-  store <8 x i16> %2, <8 x i16>* %3, align 8
+  store <8 x i16> %2, <8 x i16>* %3, align 2
   ret i8* %z
 }
 
+define i8* @ldrbu16_m127(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbu16_m127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u16 q0, [r0, #-127]!
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -127
+  %0 = bitcast i8* %z to <8 x i8>*
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
+  %2 = zext <8 x i8> %1 to <8 x i16>
+  %3 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %2, <8 x i16>* %3, align 2
+  ret i8* %z
+}
 
-define i8* @post_ldrbs16_4(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrbs16_4:
+define i8* @ldrbu16_m128(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbu16_m128:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrb.s16 q0, [r0, #4]
-; CHECK-NEXT:    adds r0, #4
-; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    subs r0, #128
+; CHECK-NEXT:    vldrb.u16 q0, [r0]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -128
+  %0 = bitcast i8* %z to <8 x i8>*
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
+  %2 = zext <8 x i8> %1 to <8 x i16>
+  %3 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %2, <8 x i16>* %3, align 2
+  ret i8* %z
+}
+
+
+define i8* @ldrbs16_4(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbs16_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.s16 q0, [r0, #4]!
+; CHECK-NEXT:    vstrh.16 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 4
   %0 = bitcast i8* %z to <8 x i8>*
-  %1 = load <8 x i8>, <8 x i8>* %0, align 8
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
   %2 = sext <8 x i8> %1 to <8 x i16>
   %3 = bitcast i8* %y to <8 x i16>*
-  store <8 x i16> %2, <8 x i16>* %3, align 8
+  store <8 x i16> %2, <8 x i16>* %3, align 2
   ret i8* %z
 }
 
-define i8* @post_ldrbs16_3(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrbs16_3:
+define i8* @ldrbs16_3(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbs16_3:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    adds r0, #3
-; CHECK-NEXT:    vldrb.s16 q0, [r0]
-; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vldrb.s16 q0, [r0, #3]!
+; CHECK-NEXT:    vstrh.16 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 3
   %0 = bitcast i8* %z to <8 x i8>*
-  %1 = load <8 x i8>, <8 x i8>* %0, align 8
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
   %2 = sext <8 x i8> %1 to <8 x i16>
   %3 = bitcast i8* %y to <8 x i16>*
-  store <8 x i16> %2, <8 x i16>* %3, align 8
+  store <8 x i16> %2, <8 x i16>* %3, align 2
   ret i8* %z
 }
 
-define i8* @post_ldrbs16_127(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrbs16_127:
+define i8* @ldrbs16_127(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbs16_127:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    adds r0, #127
-; CHECK-NEXT:    vldrb.s16 q0, [r0]
-; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vldrb.s16 q0, [r0, #127]!
+; CHECK-NEXT:    vstrh.16 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 127
   %0 = bitcast i8* %z to <8 x i8>*
-  %1 = load <8 x i8>, <8 x i8>* %0, align 8
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
   %2 = sext <8 x i8> %1 to <8 x i16>
   %3 = bitcast i8* %y to <8 x i16>*
-  store <8 x i16> %2, <8 x i16>* %3, align 8
+  store <8 x i16> %2, <8 x i16>* %3, align 2
   ret i8* %z
 }
 
-define i8* @post_ldrbs16_128(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrbs16_128:
+define i8* @ldrbs16_128(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbs16_128:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    adds r0, #128
 ; CHECK-NEXT:    vldrb.s16 q0, [r0]
-; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 128
   %0 = bitcast i8* %z to <8 x i8>*
-  %1 = load <8 x i8>, <8 x i8>* %0, align 8
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
   %2 = sext <8 x i8> %1 to <8 x i16>
   %3 = bitcast i8* %y to <8 x i16>*
-  store <8 x i16> %2, <8 x i16>* %3, align 8
+  store <8 x i16> %2, <8 x i16>* %3, align 2
   ret i8* %z
 }
 
+define i8* @ldrbs16_m127(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbs16_m127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.s16 q0, [r0, #-127]!
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -127
+  %0 = bitcast i8* %z to <8 x i8>*
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
+  %2 = sext <8 x i8> %1 to <8 x i16>
+  %3 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %2, <8 x i16>* %3, align 2
+  ret i8* %z
+}
 
-define i8* @post_ldrbu8_4(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrbu8_4:
+define i8* @ldrbs16_m128(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbs16_m128:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r0, #4]
-; CHECK-NEXT:    adds r0, #4
-; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    subs r0, #128
+; CHECK-NEXT:    vldrb.s16 q0, [r0]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -128
+  %0 = bitcast i8* %z to <8 x i8>*
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
+  %2 = sext <8 x i8> %1 to <8 x i16>
+  %3 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %2, <8 x i16>* %3, align 2
+  ret i8* %z
+}
+
+
+define i8* @ldrbu8_4(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbu8_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u8 q0, [r0, #4]!
+; CHECK-NEXT:    vstrb.8 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 4
   %0 = bitcast i8* %z to <16 x i8>*
-  %1 = load <16 x i8>, <16 x i8>* %0, align 8
+  %1 = load <16 x i8>, <16 x i8>* %0, align 1
   %2 = bitcast i8* %y to <16 x i8>*
-  store <16 x i8> %1, <16 x i8>* %2, align 8
+  store <16 x i8> %1, <16 x i8>* %2, align 1
   ret i8* %z
 }
 
-define i8* @post_ldrbu8_3(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrbu8_3:
+define i8* @ldrbu8_3(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbu8_3:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    adds r0, #3
-; CHECK-NEXT:    vldrw.u32 q0, [r0]
-; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vldrb.u8 q0, [r0, #3]!
+; CHECK-NEXT:    vstrb.8 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 3
   %0 = bitcast i8* %z to <16 x i8>*
-  %1 = load <16 x i8>, <16 x i8>* %0, align 8
+  %1 = load <16 x i8>, <16 x i8>* %0, align 1
   %2 = bitcast i8* %y to <16 x i8>*
-  store <16 x i8> %1, <16 x i8>* %2, align 8
+  store <16 x i8> %1, <16 x i8>* %2, align 1
   ret i8* %z
 }
 
-define i8* @post_ldrbu8_127(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrbu8_127:
+define i8* @ldrbu8_127(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbu8_127:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    adds r0, #127
-; CHECK-NEXT:    vldrw.u32 q0, [r0]
-; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vldrb.u8 q0, [r0, #127]!
+; CHECK-NEXT:    vstrb.8 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 127
   %0 = bitcast i8* %z to <16 x i8>*
-  %1 = load <16 x i8>, <16 x i8>* %0, align 8
+  %1 = load <16 x i8>, <16 x i8>* %0, align 1
   %2 = bitcast i8* %y to <16 x i8>*
-  store <16 x i8> %1, <16 x i8>* %2, align 8
+  store <16 x i8> %1, <16 x i8>* %2, align 1
   ret i8* %z
 }
 
-define i8* @post_ldrbu8_128(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrbu8_128:
+define i8* @ldrbu8_128(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbu8_128:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r0, #128]
 ; CHECK-NEXT:    adds r0, #128
-; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vldrb.u8 q0, [r0]
+; CHECK-NEXT:    vstrb.8 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 128
   %0 = bitcast i8* %z to <16 x i8>*
-  %1 = load <16 x i8>, <16 x i8>* %0, align 8
+  %1 = load <16 x i8>, <16 x i8>* %0, align 1
+  %2 = bitcast i8* %y to <16 x i8>*
+  store <16 x i8> %1, <16 x i8>* %2, align 1
+  ret i8* %z
+}
+
+define i8* @ldrbu8_m127(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbu8_m127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u8 q0, [r0, #-127]!
+; CHECK-NEXT:    vstrb.8 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -127
+  %0 = bitcast i8* %z to <16 x i8>*
+  %1 = load <16 x i8>, <16 x i8>* %0, align 1
   %2 = bitcast i8* %y to <16 x i8>*
-  store <16 x i8> %1, <16 x i8>* %2, align 8
+  store <16 x i8> %1, <16 x i8>* %2, align 1
   ret i8* %z
 }
 
-define i8* @post_ldrwf32_4(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrwf32_4:
+define i8* @ldrbu8_m128(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrbu8_m128:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r0, #4]
-; CHECK-NEXT:    adds r0, #4
+; CHECK-NEXT:    subs r0, #128
+; CHECK-NEXT:    vldrb.u8 q0, [r0]
+; CHECK-NEXT:    vstrb.8 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -128
+  %0 = bitcast i8* %z to <16 x i8>*
+  %1 = load <16 x i8>, <16 x i8>* %0, align 1
+  %2 = bitcast i8* %y to <16 x i8>*
+  store <16 x i8> %1, <16 x i8>* %2, align 1
+  ret i8* %z
+}
+
+
+define i8* @ldrwf32_4(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrwf32_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #4]!
 ; CHECK-NEXT:    vstrw.32 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 4
   %0 = bitcast i8* %z to <4 x float>*
-  %1 = load <4 x float>, <4 x float>* %0, align 8
+  %1 = load <4 x float>, <4 x float>* %0, align 4
   %2 = bitcast i8* %y to <4 x float>*
-  store <4 x float> %1, <4 x float>* %2, align 8
+  store <4 x float> %1, <4 x float>* %2, align 4
   ret i8* %z
 }
 
-define i8* @post_ldrwf16_4(i8* %x, i8* %y) {
-; CHECK-LABEL: post_ldrwf16_4:
+define i8* @ldrwf16_4(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrwf16_4:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r0, #4]
-; CHECK-NEXT:    adds r0, #4
-; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q0, [r0, #4]!
+; CHECK-NEXT:    vstrh.16 q0, [r1]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 4
   %0 = bitcast i8* %z to <8 x half>*
-  %1 = load <8 x half>, <8 x half>* %0, align 8
+  %1 = load <8 x half>, <8 x half>* %0, align 2
   %2 = bitcast i8* %y to <8 x half>*
-  store <8 x half> %1, <8 x half>* %2, align 8
+  store <8 x half> %1, <8 x half>* %2, align 2
   ret i8* %z
 }
 
+define i8* @ldrwi32_align1(i8* %x, i8* %y) {
+; CHECK-LE-LABEL: ldrwi32_align1:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldrb.u8 q0, [r0, #3]!
+; CHECK-LE-NEXT:    vstrw.32 q0, [r1]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: ldrwi32_align1:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vldrb.u8 q0, [r0, #3]
+; CHECK-BE-NEXT:    adds r0, #3
+; CHECK-BE-NEXT:    vrev32.8 q0, q0
+; CHECK-BE-NEXT:    vstrw.32 q0, [r1]
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 3
+  %0 = bitcast i8* %z to <4 x i32>*
+  %1 = load <4 x i32>, <4 x i32>* %0, align 1
+  %2 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %1, <4 x i32>* %2, align 4
+  ret i8* %z
+}
 
+define i8* @ldrhi16_align1(i8* %x, i8* %y) {
+; CHECK-LE-LABEL: ldrhi16_align1:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldrb.u8 q0, [r0, #3]!
+; CHECK-LE-NEXT:    vstrh.16 q0, [r1]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: ldrhi16_align1:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vldrb.u8 q0, [r0, #3]
+; CHECK-BE-NEXT:    adds r0, #3
+; CHECK-BE-NEXT:    vrev16.8 q0, q0
+; CHECK-BE-NEXT:    vstrh.16 q0, [r1]
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 3
+  %0 = bitcast i8* %z to <8 x i16>*
+  %1 = load <8 x i16>, <8 x i16>* %0, align 1
+  %2 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %1, <8 x i16>* %2, align 2
+  ret i8* %z
+}
 
-
-
-define i8* @post_strw32_4(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strw32_4:
+define i8* @ldrhi32_align1(i8* %x, i8* %y) {
+; CHECK-LABEL: ldrhi32_align1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-NEXT:    vstrw.32 q0, [r0, #4]
-; CHECK-NEXT:    adds r0, #4
+; CHECK-NEXT:    .pad #8
+; CHECK-NEXT:    sub sp, #8
+; CHECK-NEXT:    ldr r2, [r0, #3]!
+; CHECK-NEXT:    str r2, [sp]
+; CHECK-NEXT:    ldr r2, [r0, #4]
+; CHECK-NEXT:    str r2, [sp, #4]
+; CHECK-NEXT:    mov r2, sp
+; CHECK-NEXT:    vldrh.s32 q0, [r2]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    add sp, #8
 ; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 3
+  %0 = bitcast i8* %z to <4 x i16>*
+  %1 = load <4 x i16>, <4 x i16>* %0, align 1
+  %2 = bitcast i8* %y to <4 x i32>*
+  %3 = sext <4 x i16> %1 to <4 x i32>
+  store <4 x i32> %3, <4 x i32>* %2, align 4
+  ret i8* %z
+}
+
+define i8* @ldrf32_align1(i8* %x, i8* %y) {
+; CHECK-LE-LABEL: ldrf32_align1:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldrb.u8 q0, [r0, #3]!
+; CHECK-LE-NEXT:    vstrw.32 q0, [r1]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: ldrf32_align1:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vldrb.u8 q0, [r0, #3]
+; CHECK-BE-NEXT:    adds r0, #3
+; CHECK-BE-NEXT:    vrev32.8 q0, q0
+; CHECK-BE-NEXT:    vstrw.32 q0, [r1]
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 3
+  %0 = bitcast i8* %z to <4 x float>*
+  %1 = load <4 x float>, <4 x float>* %0, align 1
+  %2 = bitcast i8* %y to <4 x float>*
+  store <4 x float> %1, <4 x float>* %2, align 4
+  ret i8* %z
+}
+
+define i8* @ldrf16_align1(i8* %x, i8* %y) {
+; CHECK-LE-LABEL: ldrf16_align1:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldrb.u8 q0, [r0, #3]!
+; CHECK-LE-NEXT:    vstrh.16 q0, [r1]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: ldrf16_align1:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vldrb.u8 q0, [r0, #3]
+; CHECK-BE-NEXT:    adds r0, #3
+; CHECK-BE-NEXT:    vrev16.8 q0, q0
+; CHECK-BE-NEXT:    vstrh.16 q0, [r1]
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 3
+  %0 = bitcast i8* %z to <8 x half>*
+  %1 = load <8 x half>, <8 x half>* %0, align 1
+  %2 = bitcast i8* %y to <8 x half>*
+  store <8 x half> %1, <8 x half>* %2, align 2
+  ret i8* %z
+}
+
+define i8* @ldrh16_align8(i8* %x, i8* %y) {
+; CHECK-LE-LABEL: ldrh16_align8:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldrw.u32 q0, [r0, #4]!
+; CHECK-LE-NEXT:    vstrh.16 q0, [r1]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: ldrh16_align8:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vldrh.u16 q0, [r0, #4]!
+; CHECK-BE-NEXT:    vstrh.16 q0, [r1]
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 4
+  %0 = bitcast i8* %z to <8 x i16>*
+  %1 = load <8 x i16>, <8 x i16>* %0, align 8
+  %2 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %1, <8 x i16>* %2, align 2
+  ret i8* %z
+}
+
+
+
+
+
+define i8* @strw32_4(i8* %y, i8* %x) {
+; CHECK-LE-LABEL: strw32_4:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-LE-NEXT:    vstrb.8 q0, [r0, #4]!
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: strw32_4:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-BE-NEXT:    vstrw.32 q0, [r0, #4]!
+; CHECK-BE-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 4
   %0 = bitcast i8* %x to <4 x i32>*
-  %1 = load <4 x i32>, <4 x i32>* %0, align 8
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
   %2 = bitcast i8* %z to <4 x i32>*
-  store <4 x i32> %1, <4 x i32>* %2, align 8
+  store <4 x i32> %1, <4 x i32>* %2, align 4
   ret i8* %z
 }
 
-define i8* @post_strw32_3(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strw32_3:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    adds r0, #3
-; CHECK-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-NEXT:    vstrw.32 q0, [r0]
-; CHECK-NEXT:    bx lr
+define i8* @strw32_3(i8* %y, i8* %x) {
+; CHECK-LE-LABEL: strw32_3:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-LE-NEXT:    vstrb.8 q0, [r0, #3]!
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: strw32_3:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    adds r0, #3
+; CHECK-BE-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-BE-NEXT:    vstrw.32 q0, [r0]
+; CHECK-BE-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 3
   %0 = bitcast i8* %x to <4 x i32>*
-  %1 = load <4 x i32>, <4 x i32>* %0, align 8
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
   %2 = bitcast i8* %z to <4 x i32>*
-  store <4 x i32> %1, <4 x i32>* %2, align 8
+  store <4 x i32> %1, <4 x i32>* %2, align 4
   ret i8* %z
 }
 
-define i8* @post_strw32_m4(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strw32_m4:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-NEXT:    vstrw.32 q0, [r0, #-4]
-; CHECK-NEXT:    subs r0, #4
-; CHECK-NEXT:    bx lr
+define i8* @strw32_m4(i8* %y, i8* %x) {
+; CHECK-LE-LABEL: strw32_m4:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-LE-NEXT:    vstrb.8 q0, [r0, #-4]!
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: strw32_m4:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-BE-NEXT:    vstrw.32 q0, [r0, #-4]!
+; CHECK-BE-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 -4
   %0 = bitcast i8* %x to <4 x i32>*
-  %1 = load <4 x i32>, <4 x i32>* %0, align 8
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
   %2 = bitcast i8* %z to <4 x i32>*
-  store <4 x i32> %1, <4 x i32>* %2, align 8
+  store <4 x i32> %1, <4 x i32>* %2, align 4
   ret i8* %z
 }
 
-define i8* @post_strw32_508(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strw32_508:
+define i8* @strw32_508(i8* %y, i8* %x) {
+; CHECK-LABEL: strw32_508:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    add.w r0, r0, #508
 ; CHECK-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-NEXT:    vstrw.32 q0, [r0]
+; CHECK-NEXT:    vstrw.32 q0, [r0, #508]!
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 508
   %0 = bitcast i8* %x to <4 x i32>*
-  %1 = load <4 x i32>, <4 x i32>* %0, align 8
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
   %2 = bitcast i8* %z to <4 x i32>*
-  store <4 x i32> %1, <4 x i32>* %2, align 8
+  store <4 x i32> %1, <4 x i32>* %2, align 4
   ret i8* %z
 }
 
-define i8* @post_strw32_512(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strw32_512:
+define i8* @strw32_512(i8* %y, i8* %x) {
+; CHECK-LABEL: strw32_512:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    add.w r0, r0, #512
 ; CHECK-NEXT:    vldrw.u32 q0, [r1]
@@ -817,30 +1213,29 @@ define i8* @post_strw32_512(i8* %y, i8* %x) {
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 512
   %0 = bitcast i8* %x to <4 x i32>*
-  %1 = load <4 x i32>, <4 x i32>* %0, align 8
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
   %2 = bitcast i8* %z to <4 x i32>*
-  store <4 x i32> %1, <4 x i32>* %2, align 8
+  store <4 x i32> %1, <4 x i32>* %2, align 4
   ret i8* %z
 }
 
-define i8* @post_strw32_m508(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strw32_m508:
+define i8* @strw32_m508(i8* %y, i8* %x) {
+; CHECK-LABEL: strw32_m508:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    sub.w r0, r0, #508
 ; CHECK-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-NEXT:    vstrw.32 q0, [r0]
+; CHECK-NEXT:    vstrw.32 q0, [r0, #-508]!
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 -508
   %0 = bitcast i8* %x to <4 x i32>*
-  %1 = load <4 x i32>, <4 x i32>* %0, align 8
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
   %2 = bitcast i8* %z to <4 x i32>*
-  store <4 x i32> %1, <4 x i32>* %2, align 8
+  store <4 x i32> %1, <4 x i32>* %2, align 4
   ret i8* %z
 }
 
-define i8* @post_strw32_m512(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strw32_m512:
+define i8* @strw32_m512(i8* %y, i8* %x) {
+; CHECK-LABEL: strw32_m512:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    sub.w r0, r0, #512
 ; CHECK-NEXT:    vldrw.u32 q0, [r1]
@@ -849,31 +1244,30 @@ define i8* @post_strw32_m512(i8* %y, i8* %x) {
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 -512
   %0 = bitcast i8* %x to <4 x i32>*
-  %1 = load <4 x i32>, <4 x i32>* %0, align 8
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
   %2 = bitcast i8* %z to <4 x i32>*
-  store <4 x i32> %1, <4 x i32>* %2, align 8
+  store <4 x i32> %1, <4 x i32>* %2, align 4
   ret i8* %z
 }
 
 
-define i8* @post_strh32_4(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strh32_4:
+define i8* @strh32_4(i8* %y, i8* %x) {
+; CHECK-LABEL: strh32_4:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vldrh.u32 q0, [r1]
-; CHECK-NEXT:    vstrh.32 q0, [r0, #4]
-; CHECK-NEXT:    adds r0, #4
+; CHECK-NEXT:    vstrh.32 q0, [r0, #4]!
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 4
   %0 = bitcast i8* %x to <4 x i16>*
-  %1 = load <4 x i16>, <4 x i16>* %0, align 8
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
   %2 = bitcast i8* %z to <4 x i16>*
-  store <4 x i16> %1, <4 x i16>* %2, align 8
+  store <4 x i16> %1, <4 x i16>* %2, align 2
   ret i8* %z
 }
 
-define i8* @post_strh32_3(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strh32_3:
+define i8* @strh32_3(i8* %y, i8* %x) {
+; CHECK-LABEL: strh32_3:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    adds r0, #3
 ; CHECK-NEXT:    vldrh.u32 q0, [r1]
@@ -882,46 +1276,44 @@ define i8* @post_strh32_3(i8* %y, i8* %x) {
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 3
   %0 = bitcast i8* %x to <4 x i16>*
-  %1 = load <4 x i16>, <4 x i16>* %0, align 8
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
   %2 = bitcast i8* %z to <4 x i16>*
-  store <4 x i16> %1, <4 x i16>* %2, align 8
+  store <4 x i16> %1, <4 x i16>* %2, align 2
   ret i8* %z
 }
 
-define i8* @post_strh32_2(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strh32_2:
+define i8* @strh32_2(i8* %y, i8* %x) {
+; CHECK-LABEL: strh32_2:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    adds r0, #2
 ; CHECK-NEXT:    vldrh.u32 q0, [r1]
-; CHECK-NEXT:    vstrh.32 q0, [r0]
+; CHECK-NEXT:    vstrh.32 q0, [r0, #2]!
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 2
   %0 = bitcast i8* %x to <4 x i16>*
-  %1 = load <4 x i16>, <4 x i16>* %0, align 8
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
   %2 = bitcast i8* %z to <4 x i16>*
-  store <4 x i16> %1, <4 x i16>* %2, align 8
+  store <4 x i16> %1, <4 x i16>* %2, align 2
   ret i8* %z
 }
 
-define i8* @post_strh32_254(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strh32_254:
+define i8* @strh32_254(i8* %y, i8* %x) {
+; CHECK-LABEL: strh32_254:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    adds r0, #254
 ; CHECK-NEXT:    vldrh.u32 q0, [r1]
-; CHECK-NEXT:    vstrh.32 q0, [r0]
+; CHECK-NEXT:    vstrh.32 q0, [r0, #254]!
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 254
   %0 = bitcast i8* %x to <4 x i16>*
-  %1 = load <4 x i16>, <4 x i16>* %0, align 8
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
   %2 = bitcast i8* %z to <4 x i16>*
-  store <4 x i16> %1, <4 x i16>* %2, align 8
+  store <4 x i16> %1, <4 x i16>* %2, align 2
   ret i8* %z
 }
 
-define i8* @post_strh32_256(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strh32_256:
+define i8* @strh32_256(i8* %y, i8* %x) {
+; CHECK-LABEL: strh32_256:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    add.w r0, r0, #256
 ; CHECK-NEXT:    vldrh.u32 q0, [r1]
@@ -930,144 +1322,218 @@ define i8* @post_strh32_256(i8* %y, i8* %x) {
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 256
   %0 = bitcast i8* %x to <4 x i16>*
-  %1 = load <4 x i16>, <4 x i16>* %0, align 8
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
   %2 = bitcast i8* %z to <4 x i16>*
-  store <4 x i16> %1, <4 x i16>* %2, align 8
+  store <4 x i16> %1, <4 x i16>* %2, align 2
   ret i8* %z
 }
 
+define i8* @strh32_m254(i8* %y, i8* %x) {
+; CHECK-LABEL: strh32_m254:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u32 q0, [r1]
+; CHECK-NEXT:    vstrh.32 q0, [r0, #-254]!
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -254
+  %0 = bitcast i8* %x to <4 x i16>*
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
+  %2 = bitcast i8* %z to <4 x i16>*
+  store <4 x i16> %1, <4 x i16>* %2, align 2
+  ret i8* %z
+}
 
-define i8* @post_strh16_4(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strh16_4:
+define i8* @strh32_m256(i8* %y, i8* %x) {
+; CHECK-LABEL: strh32_m256:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-NEXT:    vstrw.32 q0, [r0, #4]
-; CHECK-NEXT:    adds r0, #4
+; CHECK-NEXT:    sub.w r0, r0, #256
+; CHECK-NEXT:    vldrh.u32 q0, [r1]
+; CHECK-NEXT:    vstrh.32 q0, [r0]
 ; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -256
+  %0 = bitcast i8* %x to <4 x i16>*
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
+  %2 = bitcast i8* %z to <4 x i16>*
+  store <4 x i16> %1, <4 x i16>* %2, align 2
+  ret i8* %z
+}
+
+
+define i8* @strh16_4(i8* %y, i8* %x) {
+; CHECK-LE-LABEL: strh16_4:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-LE-NEXT:    vstrb.8 q0, [r0, #4]!
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: strh16_4:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-BE-NEXT:    vstrh.16 q0, [r0, #4]!
+; CHECK-BE-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 4
   %0 = bitcast i8* %x to <8 x i16>*
-  %1 = load <8 x i16>, <8 x i16>* %0, align 8
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
   %2 = bitcast i8* %z to <8 x i16>*
-  store <8 x i16> %1, <8 x i16>* %2, align 8
+  store <8 x i16> %1, <8 x i16>* %2, align 2
   ret i8* %z
 }
 
-define i8* @post_strh16_3(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strh16_3:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    adds r0, #3
-; CHECK-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-NEXT:    vstrw.32 q0, [r0]
-; CHECK-NEXT:    bx lr
+define i8* @strh16_3(i8* %y, i8* %x) {
+; CHECK-LE-LABEL: strh16_3:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-LE-NEXT:    vstrb.8 q0, [r0, #3]!
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: strh16_3:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    adds r0, #3
+; CHECK-BE-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-BE-NEXT:    vstrh.16 q0, [r0]
+; CHECK-BE-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 3
   %0 = bitcast i8* %x to <8 x i16>*
-  %1 = load <8 x i16>, <8 x i16>* %0, align 8
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
   %2 = bitcast i8* %z to <8 x i16>*
-  store <8 x i16> %1, <8 x i16>* %2, align 8
+  store <8 x i16> %1, <8 x i16>* %2, align 2
   ret i8* %z
 }
 
-define i8* @post_strh16_2(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strh16_2:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    adds r0, #2
-; CHECK-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-NEXT:    vstrw.32 q0, [r0]
-; CHECK-NEXT:    bx lr
+define i8* @strh16_2(i8* %y, i8* %x) {
+; CHECK-LE-LABEL: strh16_2:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-LE-NEXT:    vstrb.8 q0, [r0, #2]!
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: strh16_2:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-BE-NEXT:    vstrh.16 q0, [r0, #2]!
+; CHECK-BE-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 2
   %0 = bitcast i8* %x to <8 x i16>*
-  %1 = load <8 x i16>, <8 x i16>* %0, align 8
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
   %2 = bitcast i8* %z to <8 x i16>*
-  store <8 x i16> %1, <8 x i16>* %2, align 8
+  store <8 x i16> %1, <8 x i16>* %2, align 2
   ret i8* %z
 }
 
-define i8* @post_strh16_254(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strh16_254:
+define i8* @strh16_254(i8* %y, i8* %x) {
+; CHECK-LABEL: strh16_254:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    adds r0, #254
-; CHECK-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-NEXT:    vstrw.32 q0, [r0]
+; CHECK-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-NEXT:    vstrh.16 q0, [r0, #254]!
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 254
   %0 = bitcast i8* %x to <8 x i16>*
-  %1 = load <8 x i16>, <8 x i16>* %0, align 8
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
   %2 = bitcast i8* %z to <8 x i16>*
-  store <8 x i16> %1, <8 x i16>* %2, align 8
+  store <8 x i16> %1, <8 x i16>* %2, align 2
   ret i8* %z
 }
 
-define i8* @post_strh16_256(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strh16_256:
+define i8* @strh16_256(i8* %y, i8* %x) {
+; CHECK-LABEL: strh16_256:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    add.w r0, r0, #256
-; CHECK-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-NEXT:    vstrw.32 q0, [r0]
+; CHECK-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-NEXT:    vstrh.16 q0, [r0]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 256
   %0 = bitcast i8* %x to <8 x i16>*
-  %1 = load <8 x i16>, <8 x i16>* %0, align 8
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
   %2 = bitcast i8* %z to <8 x i16>*
-  store <8 x i16> %1, <8 x i16>* %2, align 8
+  store <8 x i16> %1, <8 x i16>* %2, align 2
+  ret i8* %z
+}
+
+define i8* @strh16_m254(i8* %y, i8* %x) {
+; CHECK-LABEL: strh16_m254:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-NEXT:    vstrh.16 q0, [r0, #-254]!
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -254
+  %0 = bitcast i8* %x to <8 x i16>*
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
+  %2 = bitcast i8* %z to <8 x i16>*
+  store <8 x i16> %1, <8 x i16>* %2, align 2
+  ret i8* %z
+}
+
+define i8* @strh16_m256(i8* %y, i8* %x) {
+; CHECK-LABEL: strh16_m256:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    sub.w r0, r0, #256
+; CHECK-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-NEXT:    vstrh.16 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -256
+  %0 = bitcast i8* %x to <8 x i16>*
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
+  %2 = bitcast i8* %z to <8 x i16>*
+  store <8 x i16> %1, <8 x i16>* %2, align 2
   ret i8* %z
 }
 
 
-define i8* @post_strb32_4(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strb32_4:
+define i8* @strb32_4(i8* %y, i8* %x) {
+; CHECK-LABEL: strb32_4:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vldrb.u32 q0, [r1]
-; CHECK-NEXT:    vstrb.32 q0, [r0, #4]
-; CHECK-NEXT:    adds r0, #4
+; CHECK-NEXT:    vstrb.32 q0, [r0, #4]!
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 4
   %0 = bitcast i8* %x to <4 x i8>*
-  %1 = load <4 x i8>, <4 x i8>* %0, align 8
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
   %2 = bitcast i8* %z to <4 x i8>*
-  store <4 x i8> %1, <4 x i8>* %2, align 8
+  store <4 x i8> %1, <4 x i8>* %2, align 1
   ret i8* %z
 }
 
-define i8* @post_strb32_3(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strb32_3:
+define i8* @strb32_3(i8* %y, i8* %x) {
+; CHECK-LABEL: strb32_3:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    adds r0, #3
 ; CHECK-NEXT:    vldrb.u32 q0, [r1]
-; CHECK-NEXT:    vstrb.32 q0, [r0]
+; CHECK-NEXT:    vstrb.32 q0, [r0, #3]!
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 3
   %0 = bitcast i8* %x to <4 x i8>*
-  %1 = load <4 x i8>, <4 x i8>* %0, align 8
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
   %2 = bitcast i8* %z to <4 x i8>*
-  store <4 x i8> %1, <4 x i8>* %2, align 8
+  store <4 x i8> %1, <4 x i8>* %2, align 1
   ret i8* %z
 }
 
-define i8* @post_strb32_127(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strb32_127:
+define i8* @strb32_127(i8* %y, i8* %x) {
+; CHECK-LABEL: strb32_127:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    adds r0, #127
 ; CHECK-NEXT:    vldrb.u32 q0, [r1]
-; CHECK-NEXT:    vstrb.32 q0, [r0]
+; CHECK-NEXT:    vstrb.32 q0, [r0, #127]!
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 127
   %0 = bitcast i8* %x to <4 x i8>*
-  %1 = load <4 x i8>, <4 x i8>* %0, align 8
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
   %2 = bitcast i8* %z to <4 x i8>*
-  store <4 x i8> %1, <4 x i8>* %2, align 8
+  store <4 x i8> %1, <4 x i8>* %2, align 1
   ret i8* %z
 }
 
-define i8* @post_strb32_128(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strb32_128:
+define i8* @strb32_128(i8* %y, i8* %x) {
+; CHECK-LABEL: strb32_128:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    adds r0, #128
 ; CHECK-NEXT:    vldrb.u32 q0, [r1]
@@ -1076,63 +1542,91 @@ define i8* @post_strb32_128(i8* %y, i8* %x) {
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 128
   %0 = bitcast i8* %x to <4 x i8>*
-  %1 = load <4 x i8>, <4 x i8>* %0, align 8
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
   %2 = bitcast i8* %z to <4 x i8>*
-  store <4 x i8> %1, <4 x i8>* %2, align 8
+  store <4 x i8> %1, <4 x i8>* %2, align 1
   ret i8* %z
 }
 
+define i8* @strb32_m127(i8* %y, i8* %x) {
+; CHECK-LABEL: strb32_m127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u32 q0, [r1]
+; CHECK-NEXT:    vstrb.32 q0, [r0, #-127]!
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -127
+  %0 = bitcast i8* %x to <4 x i8>*
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
+  %2 = bitcast i8* %z to <4 x i8>*
+  store <4 x i8> %1, <4 x i8>* %2, align 1
+  ret i8* %z
+}
 
-define i8* @post_strb16_4(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strb16_4:
+define i8* @strb32_m128(i8* %y, i8* %x) {
+; CHECK-LABEL: strb32_m128:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    subs r0, #128
+; CHECK-NEXT:    vldrb.u32 q0, [r1]
+; CHECK-NEXT:    vstrb.32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -128
+  %0 = bitcast i8* %x to <4 x i8>*
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
+  %2 = bitcast i8* %z to <4 x i8>*
+  store <4 x i8> %1, <4 x i8>* %2, align 1
+  ret i8* %z
+}
+
+
+define i8* @strb16_4(i8* %y, i8* %x) {
+; CHECK-LABEL: strb16_4:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vldrb.u16 q0, [r1]
-; CHECK-NEXT:    vstrb.16 q0, [r0, #4]
-; CHECK-NEXT:    adds r0, #4
+; CHECK-NEXT:    vstrb.16 q0, [r0, #4]!
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 4
   %0 = bitcast i8* %x to <8 x i8>*
-  %1 = load <8 x i8>, <8 x i8>* %0, align 8
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
   %2 = bitcast i8* %z to <8 x i8>*
-  store <8 x i8> %1, <8 x i8>* %2, align 8
+  store <8 x i8> %1, <8 x i8>* %2, align 1
   ret i8* %z
 }
 
-define i8* @post_strb16_3(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strb16_3:
+define i8* @strb16_3(i8* %y, i8* %x) {
+; CHECK-LABEL: strb16_3:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    adds r0, #3
 ; CHECK-NEXT:    vldrb.u16 q0, [r1]
-; CHECK-NEXT:    vstrb.16 q0, [r0]
+; CHECK-NEXT:    vstrb.16 q0, [r0, #3]!
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 3
   %0 = bitcast i8* %x to <8 x i8>*
-  %1 = load <8 x i8>, <8 x i8>* %0, align 8
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
   %2 = bitcast i8* %z to <8 x i8>*
-  store <8 x i8> %1, <8 x i8>* %2, align 8
+  store <8 x i8> %1, <8 x i8>* %2, align 1
   ret i8* %z
 }
 
-define i8* @post_strb16_127(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strb16_127:
+define i8* @strb16_127(i8* %y, i8* %x) {
+; CHECK-LABEL: strb16_127:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    adds r0, #127
 ; CHECK-NEXT:    vldrb.u16 q0, [r1]
-; CHECK-NEXT:    vstrb.16 q0, [r0]
+; CHECK-NEXT:    vstrb.16 q0, [r0, #127]!
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 127
   %0 = bitcast i8* %x to <8 x i8>*
-  %1 = load <8 x i8>, <8 x i8>* %0, align 8
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
   %2 = bitcast i8* %z to <8 x i8>*
-  store <8 x i8> %1, <8 x i8>* %2, align 8
+  store <8 x i8> %1, <8 x i8>* %2, align 1
   ret i8* %z
 }
 
-define i8* @post_strb16_128(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strb16_128:
+define i8* @strb16_128(i8* %y, i8* %x) {
+; CHECK-LABEL: strb16_128:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    adds r0, #128
 ; CHECK-NEXT:    vldrb.u16 q0, [r1]
@@ -1141,105 +1635,311 @@ define i8* @post_strb16_128(i8* %y, i8* %x) {
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 128
   %0 = bitcast i8* %x to <8 x i8>*
-  %1 = load <8 x i8>, <8 x i8>* %0, align 8
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
+  %2 = bitcast i8* %z to <8 x i8>*
+  store <8 x i8> %1, <8 x i8>* %2, align 1
+  ret i8* %z
+}
+
+define i8* @strb16_m127(i8* %y, i8* %x) {
+; CHECK-LABEL: strb16_m127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u16 q0, [r1]
+; CHECK-NEXT:    vstrb.16 q0, [r0, #-127]!
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -127
+  %0 = bitcast i8* %x to <8 x i8>*
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
   %2 = bitcast i8* %z to <8 x i8>*
-  store <8 x i8> %1, <8 x i8>* %2, align 8
+  store <8 x i8> %1, <8 x i8>* %2, align 1
   ret i8* %z
 }
 
+define i8* @strb16_m128(i8* %y, i8* %x) {
+; CHECK-LABEL: strb16_m128:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    subs r0, #128
+; CHECK-NEXT:    vldrb.u16 q0, [r1]
+; CHECK-NEXT:    vstrb.16 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -128
+  %0 = bitcast i8* %x to <8 x i8>*
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
+  %2 = bitcast i8* %z to <8 x i8>*
+  store <8 x i8> %1, <8 x i8>* %2, align 1
+  ret i8* %z
+}
 
-define i8* @post_strb8_4(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strb8_4:
+
+define i8* @strb8_4(i8* %y, i8* %x) {
+; CHECK-LABEL: strb8_4:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-NEXT:    vstrw.32 q0, [r0, #4]
-; CHECK-NEXT:    adds r0, #4
+; CHECK-NEXT:    vldrb.u8 q0, [r1]
+; CHECK-NEXT:    vstrb.8 q0, [r0, #4]!
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 4
   %0 = bitcast i8* %x to <16 x i8>*
-  %1 = load <16 x i8>, <16 x i8>* %0, align 8
+  %1 = load <16 x i8>, <16 x i8>* %0, align 1
   %2 = bitcast i8* %z to <16 x i8>*
-  store <16 x i8> %1, <16 x i8>* %2, align 8
+  store <16 x i8> %1, <16 x i8>* %2, align 1
   ret i8* %z
 }
 
-define i8* @post_strb8_3(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strb8_3:
+define i8* @strb8_3(i8* %y, i8* %x) {
+; CHECK-LABEL: strb8_3:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    adds r0, #3
-; CHECK-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-NEXT:    vstrw.32 q0, [r0]
+; CHECK-NEXT:    vldrb.u8 q0, [r1]
+; CHECK-NEXT:    vstrb.8 q0, [r0, #3]!
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 3
   %0 = bitcast i8* %x to <16 x i8>*
-  %1 = load <16 x i8>, <16 x i8>* %0, align 8
+  %1 = load <16 x i8>, <16 x i8>* %0, align 1
   %2 = bitcast i8* %z to <16 x i8>*
-  store <16 x i8> %1, <16 x i8>* %2, align 8
+  store <16 x i8> %1, <16 x i8>* %2, align 1
   ret i8* %z
 }
 
-define i8* @post_strb8_127(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strb8_127:
+define i8* @strb8_127(i8* %y, i8* %x) {
+; CHECK-LABEL: strb8_127:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    adds r0, #127
-; CHECK-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-NEXT:    vstrw.32 q0, [r0]
+; CHECK-NEXT:    vldrb.u8 q0, [r1]
+; CHECK-NEXT:    vstrb.8 q0, [r0, #127]!
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 127
   %0 = bitcast i8* %x to <16 x i8>*
-  %1 = load <16 x i8>, <16 x i8>* %0, align 8
+  %1 = load <16 x i8>, <16 x i8>* %0, align 1
   %2 = bitcast i8* %z to <16 x i8>*
-  store <16 x i8> %1, <16 x i8>* %2, align 8
+  store <16 x i8> %1, <16 x i8>* %2, align 1
   ret i8* %z
 }
 
-define i8* @post_strb8_128(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strb8_128:
+define i8* @strb8_128(i8* %y, i8* %x) {
+; CHECK-LABEL: strb8_128:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-NEXT:    vstrw.32 q0, [r0, #128]
 ; CHECK-NEXT:    adds r0, #128
+; CHECK-NEXT:    vldrb.u8 q0, [r1]
+; CHECK-NEXT:    vstrb.8 q0, [r0]
 ; CHECK-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 128
   %0 = bitcast i8* %x to <16 x i8>*
-  %1 = load <16 x i8>, <16 x i8>* %0, align 8
+  %1 = load <16 x i8>, <16 x i8>* %0, align 1
   %2 = bitcast i8* %z to <16 x i8>*
-  store <16 x i8> %1, <16 x i8>* %2, align 8
+  store <16 x i8> %1, <16 x i8>* %2, align 1
   ret i8* %z
 }
 
-define i8* @post_strf32_4(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strf32_4:
+define i8* @strb8_m127(i8* %y, i8* %x) {
+; CHECK-LABEL: strb8_m127:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-NEXT:    vstrw.32 q0, [r0, #4]
-; CHECK-NEXT:    adds r0, #4
+; CHECK-NEXT:    vldrb.u8 q0, [r1]
+; CHECK-NEXT:    vstrb.8 q0, [r0, #-127]!
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -127
+  %0 = bitcast i8* %x to <16 x i8>*
+  %1 = load <16 x i8>, <16 x i8>* %0, align 1
+  %2 = bitcast i8* %z to <16 x i8>*
+  store <16 x i8> %1, <16 x i8>* %2, align 1
+  ret i8* %z
+}
+
+define i8* @strb8_m128(i8* %y, i8* %x) {
+; CHECK-LABEL: strb8_m128:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    subs r0, #128
+; CHECK-NEXT:    vldrb.u8 q0, [r1]
+; CHECK-NEXT:    vstrb.8 q0, [r0]
 ; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -128
+  %0 = bitcast i8* %x to <16 x i8>*
+  %1 = load <16 x i8>, <16 x i8>* %0, align 1
+  %2 = bitcast i8* %z to <16 x i8>*
+  store <16 x i8> %1, <16 x i8>* %2, align 1
+  ret i8* %z
+}
+
+
+define i8* @strf32_4(i8* %y, i8* %x) {
+; CHECK-LE-LABEL: strf32_4:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-LE-NEXT:    vstrb.8 q0, [r0, #4]!
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: strf32_4:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-BE-NEXT:    vstrw.32 q0, [r0, #4]!
+; CHECK-BE-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 4
   %0 = bitcast i8* %x to <4 x float>*
-  %1 = load <4 x float>, <4 x float>* %0, align 8
+  %1 = load <4 x float>, <4 x float>* %0, align 4
   %2 = bitcast i8* %z to <4 x float>*
-  store <4 x float> %1, <4 x float>* %2, align 8
+  store <4 x float> %1, <4 x float>* %2, align 4
+  ret i8* %z
+}
+
+define i8* @strf16_4(i8* %y, i8* %x) {
+; CHECK-LE-LABEL: strf16_4:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-LE-NEXT:    vstrb.8 q0, [r0, #4]!
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: strf16_4:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-BE-NEXT:    vstrh.16 q0, [r0, #4]!
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 4
+  %0 = bitcast i8* %x to <8 x half>*
+  %1 = load <8 x half>, <8 x half>* %0, align 2
+  %2 = bitcast i8* %z to <8 x half>*
+  store <8 x half> %1, <8 x half>* %2, align 2
+  ret i8* %z
+}
+
+define i8* @strwi32_align1(i8* %y, i8* %x) {
+; CHECK-LE-LABEL: strwi32_align1:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-LE-NEXT:    vstrb.8 q0, [r0, #3]!
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: strwi32_align1:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-BE-NEXT:    vrev32.8 q0, q0
+; CHECK-BE-NEXT:    vstrb.8 q0, [r0, #3]
+; CHECK-BE-NEXT:    adds r0, #3
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 3
+  %0 = bitcast i8* %x to <4 x i32>*
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
+  %2 = bitcast i8* %z to <4 x i32>*
+  store <4 x i32> %1, <4 x i32>* %2, align 1
   ret i8* %z
 }
 
-define i8* @post_strf16_4(i8* %y, i8* %x) {
-; CHECK-LABEL: post_strf16_4:
+define i8* @strhi16_align1(i8* %y, i8* %x) {
+; CHECK-LE-LABEL: strhi16_align1:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-LE-NEXT:    vstrb.8 q0, [r0, #3]!
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: strhi16_align1:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-BE-NEXT:    vrev16.8 q0, q0
+; CHECK-BE-NEXT:    vstrb.8 q0, [r0, #3]
+; CHECK-BE-NEXT:    adds r0, #3
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 3
+  %0 = bitcast i8* %x to <8 x i16>*
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
+  %2 = bitcast i8* %z to <8 x i16>*
+  store <8 x i16> %1, <8 x i16>* %2, align 1
+  ret i8* %z
+}
+
+define i8* @strhi32_align1(i8* %y, i8* %x) {
+; CHECK-LABEL: strhi32_align1:
 ; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .pad #8
+; CHECK-NEXT:    sub sp, #8
 ; CHECK-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-NEXT:    vstrw.32 q0, [r0, #4]
-; CHECK-NEXT:    adds r0, #4
+; CHECK-NEXT:    mov r1, sp
+; CHECK-NEXT:    vstrh.32 q0, [r1]
+; CHECK-NEXT:    ldrd r1, r2, [sp]
+; CHECK-NEXT:    str r1, [r0, #3]!
+; CHECK-NEXT:    str r2, [r0, #4]
+; CHECK-NEXT:    add sp, #8
 ; CHECK-NEXT:    bx lr
 entry:
-  %z = getelementptr inbounds i8, i8* %y, i32 4
+  %z = getelementptr inbounds i8, i8* %y, i32 3
+  %0 = bitcast i8* %x to <4 x i32>*
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
+  %2 = bitcast i8* %z to <4 x i16>*
+  %3 = trunc <4 x i32> %1 to <4 x i16>
+  store <4 x i16> %3, <4 x i16>* %2, align 1
+  ret i8* %z
+}
+
+define i8* @strf32_align1(i8* %y, i8* %x) {
+; CHECK-LE-LABEL: strf32_align1:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-LE-NEXT:    vstrb.8 q0, [r0, #3]!
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: strf32_align1:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-BE-NEXT:    vrev32.8 q0, q0
+; CHECK-BE-NEXT:    vstrb.8 q0, [r0, #3]
+; CHECK-BE-NEXT:    adds r0, #3
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 3
+  %0 = bitcast i8* %x to <4 x float>*
+  %1 = load <4 x float>, <4 x float>* %0, align 4
+  %2 = bitcast i8* %z to <4 x float>*
+  store <4 x float> %1, <4 x float>* %2, align 1
+  ret i8* %z
+}
+
+define i8* @strf16_align1(i8* %y, i8* %x) {
+; CHECK-LE-LABEL: strf16_align1:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-LE-NEXT:    vstrb.8 q0, [r0, #3]!
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: strf16_align1:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-BE-NEXT:    vrev16.8 q0, q0
+; CHECK-BE-NEXT:    vstrb.8 q0, [r0, #3]
+; CHECK-BE-NEXT:    adds r0, #3
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 3
   %0 = bitcast i8* %x to <8 x half>*
-  %1 = load <8 x half>, <8 x half>* %0, align 8
+  %1 = load <8 x half>, <8 x half>* %0, align 2
   %2 = bitcast i8* %z to <8 x half>*
-  store <8 x half> %1, <8 x half>* %2, align 8
+  store <8 x half> %1, <8 x half>* %2, align 1
+  ret i8* %z
+}
+
+define i8* @strf16_align8(i8* %y, i8* %x) {
+; CHECK-LE-LABEL: strf16_align8:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-LE-NEXT:    vstrb.8 q0, [r0, #16]!
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: strf16_align8:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-BE-NEXT:    vstrh.16 q0, [r0, #16]!
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 16
+  %0 = bitcast i8* %x to <8 x i16>*
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
+  %2 = bitcast i8* %z to <8 x i16>*
+  store <8 x i16> %1, <8 x i16>* %2, align 8
   ret i8* %z
 }
diff --git a/llvm/test/CodeGen/Thumb2/mve-ldst-regimm.ll b/llvm/test/CodeGen/Thumb2/mve-ldst-regimm.ll
index e71310f3d..e57708d1e 100644
--- a/llvm/test/CodeGen/Thumb2/mve-ldst-regimm.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-ldst-regimm.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve %s -o - | FileCheck %s
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s
 
 %struct.s_int8_t = type { [16 x i8], [16 x i8] }
 %struct.s_int16_t = type { [8 x i16], [8 x i16] }
@@ -55,8 +55,8 @@ entry:
 define hidden void @fwd_float16_t(%struct.s_float16_t* noalias nocapture %v) local_unnamed_addr #0 {
 ; CHECK-LABEL: fwd_float16_t:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrh.u16 q0, [r0]
-; CHECK-NEXT:    vstrh.16 q0, [r0, #16]
+; CHECK-NEXT:    vldrh.u16 q0, [r0], #16
+; CHECK-NEXT:    vstrh.16 q0, [r0]
 ; CHECK-NEXT:    bx lr
 entry:
   %arrayidx3 = getelementptr inbounds %struct.s_float16_t, %struct.s_float16_t* %v, i32 0, i32 1, i32 0
@@ -130,8 +130,8 @@ for.end:
 define hidden void @bwd_float16_t(%struct.s_float16_t* noalias nocapture %v) local_unnamed_addr #0 {
 ; CHECK-LABEL: bwd_float16_t:
 ; CHECK:       @ %bb.0: @ %for.end
-; CHECK-NEXT:    vldrh.u16 q0, [r0]
-; CHECK-NEXT:    vstrh.16 q0, [r0, #-16]
+; CHECK-NEXT:    vldrh.u16 q0, [r0], #-16
+; CHECK-NEXT:    vstrh.16 q0, [r0]
 ; CHECK-NEXT:    bx lr
 for.end:
   %0 = bitcast %struct.s_float16_t* %v to <8 x half>*
diff --git a/llvm/test/CodeGen/Thumb2/mve-loadstore.ll b/llvm/test/CodeGen/Thumb2/mve-loadstore.ll
index 9f148c4d8..20fce95d6 100644
--- a/llvm/test/CodeGen/Thumb2/mve-loadstore.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-loadstore.ll
@@ -1,72 +1,137 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve %s -o - | FileCheck %s
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-LE
+; RUN: llc -mtriple=thumbebv8.1m.main-arm-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-BE
 
 define arm_aapcs_vfpcc <4 x i32> @load_4xi32_a4(<4 x i32>* %vp) {
-; CHECK-LABEL: load_4xi32_a4:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r0]
-; CHECK-NEXT:    bx lr
+; CHECK-LE-LABEL: load_4xi32_a4:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-LE-NEXT:    vshr.u32 q0, q0, #1
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: load_4xi32_a4:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-BE-NEXT:    vshr.u32 q1, q0, #1
+; CHECK-BE-NEXT:    vrev64.32 q0, q1
+; CHECK-BE-NEXT:    bx lr
 entry:
   %0 = load <4 x i32>, <4 x i32>* %vp, align 4
-  ret <4 x i32> %0
+  %1 = lshr <4 x i32> %0, <i32 1, i32 1, i32 1, i32 1>
+  ret <4 x i32> %1
 }
 
 define arm_aapcs_vfpcc <4 x i32> @load_4xi32_a2(<4 x i32>* %vp) {
-; CHECK-LABEL: load_4xi32_a2:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrh.u16 q0, [r0]
-; CHECK-NEXT:    bx lr
+; CHECK-LE-LABEL: load_4xi32_a2:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldrh.u16 q0, [r0]
+; CHECK-LE-NEXT:    vshr.u32 q0, q0, #1
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: load_4xi32_a2:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vldrb.u8 q0, [r0]
+; CHECK-BE-NEXT:    vrev32.8 q0, q0
+; CHECK-BE-NEXT:    vshr.u32 q1, q0, #1
+; CHECK-BE-NEXT:    vrev64.32 q0, q1
+; CHECK-BE-NEXT:    bx lr
 entry:
   %0 = load <4 x i32>, <4 x i32>* %vp, align 2
-  ret <4 x i32> %0
+  %1 = lshr <4 x i32> %0, <i32 1, i32 1, i32 1, i32 1>
+  ret <4 x i32> %1
 }
 
 define arm_aapcs_vfpcc <4 x i32> @load_4xi32_a1(<4 x i32>* %vp) {
-; CHECK-LABEL: load_4xi32_a1:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrb.u8 q0, [r0]
-; CHECK-NEXT:    bx lr
+; CHECK-LE-LABEL: load_4xi32_a1:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldrb.u8 q0, [r0]
+; CHECK-LE-NEXT:    vshr.u32 q0, q0, #1
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: load_4xi32_a1:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vldrb.u8 q0, [r0]
+; CHECK-BE-NEXT:    vrev32.8 q0, q0
+; CHECK-BE-NEXT:    vshr.u32 q1, q0, #1
+; CHECK-BE-NEXT:    vrev64.32 q0, q1
+; CHECK-BE-NEXT:    bx lr
 entry:
   %0 = load <4 x i32>, <4 x i32>* %vp, align 1
-  ret <4 x i32> %0
+  %1 = lshr <4 x i32> %0, <i32 1, i32 1, i32 1, i32 1>
+  ret <4 x i32> %1
 }
 
 define arm_aapcs_vfpcc void @store_4xi32_a4(<4 x i32>* %vp, <4 x i32> %val) {
-; CHECK-LABEL: store_4xi32_a4:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vstrw.32 q0, [r0]
-; CHECK-NEXT:    bx lr
+; CHECK-LE-LABEL: store_4xi32_a4:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vshr.u32 q0, q0, #1
+; CHECK-LE-NEXT:    vstrw.32 q0, [r0]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: store_4xi32_a4:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    vshr.u32 q0, q1, #1
+; CHECK-BE-NEXT:    vstrw.32 q0, [r0]
+; CHECK-BE-NEXT:    bx lr
 entry:
-  store <4 x i32> %val, <4 x i32>* %vp, align 4
+  %0 = lshr <4 x i32> %val, <i32 1, i32 1, i32 1, i32 1>
+  store <4 x i32> %0, <4 x i32>* %vp, align 4
   ret void
 }
 
 define arm_aapcs_vfpcc void @store_4xi32_a2(<4 x i32>* %vp, <4 x i32> %val) {
-; CHECK-LABEL: store_4xi32_a2:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vstrh.16 q0, [r0]
-; CHECK-NEXT:    bx lr
+; CHECK-LE-LABEL: store_4xi32_a2:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vshr.u32 q0, q0, #1
+; CHECK-LE-NEXT:    vstrh.16 q0, [r0]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: store_4xi32_a2:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    vshr.u32 q0, q1, #1
+; CHECK-BE-NEXT:    vrev32.8 q0, q0
+; CHECK-BE-NEXT:    vstrb.8 q0, [r0]
+; CHECK-BE-NEXT:    bx lr
 entry:
-  store <4 x i32> %val, <4 x i32>* %vp, align 2
+  %0 = lshr <4 x i32> %val, <i32 1, i32 1, i32 1, i32 1>
+  store <4 x i32> %0, <4 x i32>* %vp, align 2
   ret void
 }
 
 define arm_aapcs_vfpcc void @store_4xi32_a1(<4 x i32>* %vp, <4 x i32> %val) {
-; CHECK-LABEL: store_4xi32_a1:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vstrb.8 q0, [r0]
-; CHECK-NEXT:    bx lr
+; CHECK-LE-LABEL: store_4xi32_a1:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vshr.u32 q0, q0, #1
+; CHECK-LE-NEXT:    vstrb.8 q0, [r0]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: store_4xi32_a1:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    vshr.u32 q0, q1, #1
+; CHECK-BE-NEXT:    vrev32.8 q0, q0
+; CHECK-BE-NEXT:    vstrb.8 q0, [r0]
+; CHECK-BE-NEXT:    bx lr
 entry:
-  store <4 x i32> %val, <4 x i32>* %vp, align 1
+  %0 = lshr <4 x i32> %val, <i32 1, i32 1, i32 1, i32 1>
+  store <4 x i32> %0, <4 x i32>* %vp, align 1
   ret void
 }
 
 define arm_aapcs_vfpcc <4 x i32> @load_4xi32_a4_offset_pos(i32* %ip) {
-; CHECK-LABEL: load_4xi32_a4_offset_pos:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    add.w r0, r0, #508
-; CHECK-NEXT:    vldrw.u32 q0, [r0]
-; CHECK-NEXT:    bx lr
+; CHECK-LE-LABEL: load_4xi32_a4_offset_pos:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldrw.u32 q0, [r0, #508]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: load_4xi32_a4_offset_pos:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    add.w r0, r0, #508
+; CHECK-BE-NEXT:    vldrb.u8 q1, [r0]
+; CHECK-BE-NEXT:    vrev64.8 q0, q1
+; CHECK-BE-NEXT:    bx lr
 entry:
   %ipoffset = getelementptr inbounds i32, i32* %ip, i32 127
   %vp = bitcast i32* %ipoffset to <4 x i32>*
@@ -75,11 +140,17 @@ entry:
 }
 
 define arm_aapcs_vfpcc <4 x i32> @load_4xi32_a4_offset_neg(i32* %ip) {
-; CHECK-LABEL: load_4xi32_a4_offset_neg:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    sub.w r0, r0, #508
-; CHECK-NEXT:    vldrw.u32 q0, [r0]
-; CHECK-NEXT:    bx lr
+; CHECK-LE-LABEL: load_4xi32_a4_offset_neg:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldrw.u32 q0, [r0, #-508]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: load_4xi32_a4_offset_neg:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    sub.w r0, r0, #508
+; CHECK-BE-NEXT:    vldrb.u8 q1, [r0]
+; CHECK-BE-NEXT:    vrev64.8 q0, q1
+; CHECK-BE-NEXT:    bx lr
 entry:
   %ipoffset = getelementptr inbounds i32, i32* %ip, i32 -127
   %vp = bitcast i32* %ipoffset to <4 x i32>*
@@ -88,19 +159,34 @@ entry:
 }
 
 define arm_aapcs_vfpcc <4 x i32> @loadstore_4xi32_stack_off16() {
-; CHECK-LABEL: loadstore_4xi32_stack_off16:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .pad #40
-; CHECK-NEXT:    sub sp, #40
-; CHECK-NEXT:    vmov.i32 q0, #0x1
-; CHECK-NEXT:    mov r0, sp
-; CHECK-NEXT:    vstrw.32 q0, [r0]
-; CHECK-NEXT:    movs r0, #3
-; CHECK-NEXT:    vstrw.32 q0, [sp, #16]
-; CHECK-NEXT:    str r0, [sp, #16]
-; CHECK-NEXT:    vldrw.u32 q0, [sp, #16]
-; CHECK-NEXT:    add sp, #40
-; CHECK-NEXT:    bx lr
+; CHECK-LE-LABEL: loadstore_4xi32_stack_off16:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    .pad #40
+; CHECK-LE-NEXT:    sub sp, #40
+; CHECK-LE-NEXT:    vmov.i32 q0, #0x1
+; CHECK-LE-NEXT:    mov r0, sp
+; CHECK-LE-NEXT:    vstrw.32 q0, [r0]
+; CHECK-LE-NEXT:    movs r0, #3
+; CHECK-LE-NEXT:    vstrw.32 q0, [sp, #16]
+; CHECK-LE-NEXT:    str r0, [sp, #16]
+; CHECK-LE-NEXT:    vldrw.u32 q0, [sp, #16]
+; CHECK-LE-NEXT:    add sp, #40
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: loadstore_4xi32_stack_off16:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    .pad #40
+; CHECK-BE-NEXT:    sub sp, #40
+; CHECK-BE-NEXT:    vmov.i32 q0, #0x1
+; CHECK-BE-NEXT:    mov r0, sp
+; CHECK-BE-NEXT:    vstrw.32 q0, [r0]
+; CHECK-BE-NEXT:    movs r0, #3
+; CHECK-BE-NEXT:    vstrw.32 q0, [sp, #16]
+; CHECK-BE-NEXT:    str r0, [sp, #16]
+; CHECK-BE-NEXT:    vldrb.u8 q1, [sp, #16]
+; CHECK-BE-NEXT:    vrev64.8 q0, q1
+; CHECK-BE-NEXT:    add sp, #40
+; CHECK-BE-NEXT:    bx lr
 entry:
   %c = alloca [1 x [5 x [2 x i32]]], align 4
   %0 = bitcast [1 x [5 x [2 x i32]]]* %c to i8*
@@ -116,19 +202,34 @@ entry:
 }
 
 define arm_aapcs_vfpcc <8 x i16> @loadstore_8xi16_stack_off16() {
-; CHECK-LABEL: loadstore_8xi16_stack_off16:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .pad #40
-; CHECK-NEXT:    sub sp, #40
-; CHECK-NEXT:    vmov.i16 q0, #0x1
-; CHECK-NEXT:    mov r0, sp
-; CHECK-NEXT:    vstrh.16 q0, [r0]
-; CHECK-NEXT:    movs r0, #3
-; CHECK-NEXT:    vstrh.16 q0, [sp, #16]
-; CHECK-NEXT:    strh.w r0, [sp, #16]
-; CHECK-NEXT:    vldrh.u16 q0, [sp, #16]
-; CHECK-NEXT:    add sp, #40
-; CHECK-NEXT:    bx lr
+; CHECK-LE-LABEL: loadstore_8xi16_stack_off16:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    .pad #40
+; CHECK-LE-NEXT:    sub sp, #40
+; CHECK-LE-NEXT:    vmov.i16 q0, #0x1
+; CHECK-LE-NEXT:    mov r0, sp
+; CHECK-LE-NEXT:    vstrh.16 q0, [r0]
+; CHECK-LE-NEXT:    movs r0, #3
+; CHECK-LE-NEXT:    vstrh.16 q0, [sp, #16]
+; CHECK-LE-NEXT:    strh.w r0, [sp, #16]
+; CHECK-LE-NEXT:    vldrh.u16 q0, [sp, #16]
+; CHECK-LE-NEXT:    add sp, #40
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: loadstore_8xi16_stack_off16:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    .pad #40
+; CHECK-BE-NEXT:    sub sp, #40
+; CHECK-BE-NEXT:    vmov.i16 q0, #0x1
+; CHECK-BE-NEXT:    mov r0, sp
+; CHECK-BE-NEXT:    vstrh.16 q0, [r0]
+; CHECK-BE-NEXT:    movs r0, #3
+; CHECK-BE-NEXT:    vstrh.16 q0, [sp, #16]
+; CHECK-BE-NEXT:    strh.w r0, [sp, #16]
+; CHECK-BE-NEXT:    vldrb.u8 q1, [sp, #16]
+; CHECK-BE-NEXT:    vrev64.8 q0, q1
+; CHECK-BE-NEXT:    add sp, #40
+; CHECK-BE-NEXT:    bx lr
 entry:
   %c = alloca [1 x [10 x [2 x i16]]], align 2
   %0 = bitcast [1 x [10 x [2 x i16]]]* %c to i8*
@@ -144,19 +245,34 @@ entry:
 }
 
 define arm_aapcs_vfpcc <16 x i8> @loadstore_16xi8_stack_off16() {
-; CHECK-LABEL: loadstore_16xi8_stack_off16:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .pad #40
-; CHECK-NEXT:    sub sp, #40
-; CHECK-NEXT:    vmov.i8 q0, #0x1
-; CHECK-NEXT:    mov r0, sp
-; CHECK-NEXT:    vstrb.8 q0, [r0]
-; CHECK-NEXT:    movs r0, #3
-; CHECK-NEXT:    vstrb.8 q0, [sp, #16]
-; CHECK-NEXT:    strb.w r0, [sp, #16]
-; CHECK-NEXT:    vldrb.u8 q0, [sp, #16]
-; CHECK-NEXT:    add sp, #40
-; CHECK-NEXT:    bx lr
+; CHECK-LE-LABEL: loadstore_16xi8_stack_off16:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    .pad #40
+; CHECK-LE-NEXT:    sub sp, #40
+; CHECK-LE-NEXT:    vmov.i8 q0, #0x1
+; CHECK-LE-NEXT:    mov r0, sp
+; CHECK-LE-NEXT:    vstrb.8 q0, [r0]
+; CHECK-LE-NEXT:    movs r0, #3
+; CHECK-LE-NEXT:    vstrb.8 q0, [sp, #16]
+; CHECK-LE-NEXT:    strb.w r0, [sp, #16]
+; CHECK-LE-NEXT:    vldrb.u8 q0, [sp, #16]
+; CHECK-LE-NEXT:    add sp, #40
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: loadstore_16xi8_stack_off16:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    .pad #40
+; CHECK-BE-NEXT:    sub sp, #40
+; CHECK-BE-NEXT:    vmov.i8 q0, #0x1
+; CHECK-BE-NEXT:    mov r0, sp
+; CHECK-BE-NEXT:    vstrb.8 q0, [r0]
+; CHECK-BE-NEXT:    movs r0, #3
+; CHECK-BE-NEXT:    vstrb.8 q0, [sp, #16]
+; CHECK-BE-NEXT:    strb.w r0, [sp, #16]
+; CHECK-BE-NEXT:    vldrb.u8 q1, [sp, #16]
+; CHECK-BE-NEXT:    vrev64.8 q0, q1
+; CHECK-BE-NEXT:    add sp, #40
+; CHECK-BE-NEXT:    bx lr
 entry:
   %c = alloca [1 x [20 x [2 x i8]]], align 1
   %0 = bitcast [1 x [20 x [2 x i8]]]* %c to i8*
diff --git a/llvm/test/CodeGen/Thumb2/mve-masked-ldst-offset.ll b/llvm/test/CodeGen/Thumb2/mve-masked-ldst-offset.ll
new file mode 100644
index 000000000..5e2e85017
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-masked-ldst-offset.ll
@@ -0,0 +1,2646 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-LE
+; RUN: llc -mtriple=thumbebv8.1m.main-arm-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-BE
+
+define i8* @ldrwu32_4(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrwu32_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrwt.u32 q0, [r0, #4]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 4
+  %0 = bitcast i8* %z to <4 x i32>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %0, i32 4, <4 x i1> %c, <4 x i32> undef)
+  %2 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %1, <4 x i32>* %2, align 4
+  ret i8* %x
+}
+
+define i8* @ldrwu32_3(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrwu32_3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    adds r3, r0, #3
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrwt.u32 q0, [r3]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 3
+  %0 = bitcast i8* %z to <4 x i32>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %0, i32 4, <4 x i1> %c, <4 x i32> undef)
+  %2 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %1, <4 x i32>* %2, align 4
+  ret i8* %x
+}
+
+define i8* @ldrwu32_2(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrwu32_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    adds r3, r0, #2
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrwt.u32 q0, [r3]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 2
+  %0 = bitcast i8* %z to <4 x i32>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %0, i32 4, <4 x i1> %c, <4 x i32> undef)
+  %2 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %1, <4 x i32>* %2, align 4
+  ret i8* %x
+}
+
+define i8* @ldrwu32_508(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrwu32_508:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrwt.u32 q0, [r0, #508]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 508
+  %0 = bitcast i8* %z to <4 x i32>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %0, i32 4, <4 x i1> %c, <4 x i32> undef)
+  %2 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %1, <4 x i32>* %2, align 4
+  ret i8* %x
+}
+
+define i8* @ldrwu32_512(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrwu32_512:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    add.w r3, r0, #512
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrwt.u32 q0, [r3]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 512
+  %0 = bitcast i8* %z to <4 x i32>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %0, i32 4, <4 x i1> %c, <4 x i32> undef)
+  %2 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %1, <4 x i32>* %2, align 4
+  ret i8* %x
+}
+
+define i8* @ldrwu32_m508(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrwu32_m508:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrwt.u32 q0, [r0, #-508]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -508
+  %0 = bitcast i8* %z to <4 x i32>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %0, i32 4, <4 x i1> %c, <4 x i32> undef)
+  %2 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %1, <4 x i32>* %2, align 4
+  ret i8* %x
+}
+
+define i8* @ldrwu32_m512(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrwu32_m512:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    sub.w r3, r0, #512
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrwt.u32 q0, [r3]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -512
+  %0 = bitcast i8* %z to <4 x i32>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %0, i32 4, <4 x i1> %c, <4 x i32> undef)
+  %2 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %1, <4 x i32>* %2, align 4
+  ret i8* %x
+}
+
+define i8* @ldrhu32_4(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrhu32_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrht.u32 q0, [r0, #4]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 4
+  %0 = bitcast i8* %z to <4 x i16>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %0, i32 2, <4 x i1> %c, <4 x i16> undef)
+  %2 = zext <4 x i16> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %x
+}
+
+define i8* @ldrhu32_3(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrhu32_3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    adds r3, r0, #3
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrht.u32 q0, [r3]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 3
+  %0 = bitcast i8* %z to <4 x i16>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %0, i32 2, <4 x i1> %c, <4 x i16> undef)
+  %2 = zext <4 x i16> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %x
+}
+
+define i8* @ldrhu32_2(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrhu32_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrht.u32 q0, [r0, #2]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 2
+  %0 = bitcast i8* %z to <4 x i16>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %0, i32 2, <4 x i1> %c, <4 x i16> undef)
+  %2 = zext <4 x i16> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %x
+}
+
+define i8* @ldrhu32_254(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrhu32_254:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrht.u32 q0, [r0, #254]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 254
+  %0 = bitcast i8* %z to <4 x i16>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %0, i32 2, <4 x i1> %c, <4 x i16> undef)
+  %2 = zext <4 x i16> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %x
+}
+
+define i8* @ldrhu32_256(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrhu32_256:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    add.w r3, r0, #256
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrht.u32 q0, [r3]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 256
+  %0 = bitcast i8* %z to <4 x i16>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %0, i32 2, <4 x i1> %c, <4 x i16> undef)
+  %2 = zext <4 x i16> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %x
+}
+
+define i8* @ldrhu32_m254(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrhu32_m254:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrht.u32 q0, [r0, #-254]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -254
+  %0 = bitcast i8* %z to <4 x i16>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %0, i32 2, <4 x i1> %c, <4 x i16> undef)
+  %2 = zext <4 x i16> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %x
+}
+
+define i8* @ldrhu32_m256(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrhu32_m256:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    sub.w r3, r0, #256
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrht.u32 q0, [r3]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -256
+  %0 = bitcast i8* %z to <4 x i16>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %0, i32 2, <4 x i1> %c, <4 x i16> undef)
+  %2 = zext <4 x i16> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %x
+}
+
+define i8* @ldrhs32_4(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrhs32_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrht.s32 q0, [r0, #4]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 4
+  %0 = bitcast i8* %z to <4 x i16>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %0, i32 2, <4 x i1> %c, <4 x i16> undef)
+  %2 = sext <4 x i16> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %x
+}
+
+define i8* @ldrhs32_3(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrhs32_3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    adds r3, r0, #3
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrht.s32 q0, [r3]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 3
+  %0 = bitcast i8* %z to <4 x i16>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %0, i32 2, <4 x i1> %c, <4 x i16> undef)
+  %2 = sext <4 x i16> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %x
+}
+
+define i8* @ldrhs32_2(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrhs32_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrht.s32 q0, [r0, #2]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 2
+  %0 = bitcast i8* %z to <4 x i16>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %0, i32 2, <4 x i1> %c, <4 x i16> undef)
+  %2 = sext <4 x i16> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %x
+}
+
+define i8* @ldrhs32_254(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrhs32_254:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrht.s32 q0, [r0, #254]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 254
+  %0 = bitcast i8* %z to <4 x i16>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %0, i32 2, <4 x i1> %c, <4 x i16> undef)
+  %2 = sext <4 x i16> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %x
+}
+
+define i8* @ldrhs32_256(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrhs32_256:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    add.w r3, r0, #256
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrht.s32 q0, [r3]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 256
+  %0 = bitcast i8* %z to <4 x i16>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %0, i32 2, <4 x i1> %c, <4 x i16> undef)
+  %2 = sext <4 x i16> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %x
+}
+
+define i8* @ldrhs32_m254(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrhs32_m254:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrht.s32 q0, [r0, #-254]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -254
+  %0 = bitcast i8* %z to <4 x i16>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %0, i32 2, <4 x i1> %c, <4 x i16> undef)
+  %2 = sext <4 x i16> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %x
+}
+
+define i8* @ldrhs32_m256(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrhs32_m256:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    sub.w r3, r0, #256
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrht.s32 q0, [r3]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -256
+  %0 = bitcast i8* %z to <4 x i16>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %0, i32 2, <4 x i1> %c, <4 x i16> undef)
+  %2 = sext <4 x i16> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %x
+}
+
+define i8* @ldrhu16_4(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrhu16_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrht.u16 q0, [r0, #4]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 4
+  %0 = bitcast i8* %z to <8 x i16>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x i16> @llvm.masked.load.v8i16.p0v4i16(<8 x i16>* %0, i32 2, <8 x i1> %c, <8 x i16> undef)
+  %2 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %1, <8 x i16>* %2, align 2
+  ret i8* %x
+}
+
+define i8* @ldrhu16_3(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrhu16_3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    adds r3, r0, #3
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrht.u16 q0, [r3]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 3
+  %0 = bitcast i8* %z to <8 x i16>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x i16> @llvm.masked.load.v8i16.p0v4i16(<8 x i16>* %0, i32 2, <8 x i1> %c, <8 x i16> undef)
+  %2 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %1, <8 x i16>* %2, align 2
+  ret i8* %x
+}
+
+define i8* @ldrhu16_2(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrhu16_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrht.u16 q0, [r0, #2]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 2
+  %0 = bitcast i8* %z to <8 x i16>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x i16> @llvm.masked.load.v8i16.p0v4i16(<8 x i16>* %0, i32 2, <8 x i1> %c, <8 x i16> undef)
+  %2 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %1, <8 x i16>* %2, align 2
+  ret i8* %x
+}
+
+define i8* @ldrhu16_254(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrhu16_254:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrht.u16 q0, [r0, #254]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 254
+  %0 = bitcast i8* %z to <8 x i16>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x i16> @llvm.masked.load.v8i16.p0v4i16(<8 x i16>* %0, i32 2, <8 x i1> %c, <8 x i16> undef)
+  %2 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %1, <8 x i16>* %2, align 2
+  ret i8* %x
+}
+
+define i8* @ldrhu16_256(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrhu16_256:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    add.w r3, r0, #256
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrht.u16 q0, [r3]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 256
+  %0 = bitcast i8* %z to <8 x i16>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x i16> @llvm.masked.load.v8i16.p0v4i16(<8 x i16>* %0, i32 2, <8 x i1> %c, <8 x i16> undef)
+  %2 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %1, <8 x i16>* %2, align 2
+  ret i8* %x
+}
+
+define i8* @ldrhu16_m254(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrhu16_m254:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrht.u16 q0, [r0, #-254]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -254
+  %0 = bitcast i8* %z to <8 x i16>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x i16> @llvm.masked.load.v8i16.p0v4i16(<8 x i16>* %0, i32 2, <8 x i1> %c, <8 x i16> undef)
+  %2 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %1, <8 x i16>* %2, align 2
+  ret i8* %x
+}
+
+define i8* @ldrhu16_m256(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrhu16_m256:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    sub.w r3, r0, #256
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrht.u16 q0, [r3]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -256
+  %0 = bitcast i8* %z to <8 x i16>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x i16> @llvm.masked.load.v8i16.p0v4i16(<8 x i16>* %0, i32 2, <8 x i1> %c, <8 x i16> undef)
+  %2 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %1, <8 x i16>* %2, align 2
+  ret i8* %x
+}
+
+define i8* @ldrbu32_4(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrbu32_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrbt.u32 q0, [r0, #4]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 4
+  %0 = bitcast i8* %z to <4 x i8>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %0, i32 1, <4 x i1> %c, <4 x i8> undef)
+  %2 = zext <4 x i8> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %x
+}
+
+define i8* @ldrbu32_3(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrbu32_3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrbt.u32 q0, [r0, #3]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 3
+  %0 = bitcast i8* %z to <4 x i8>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %0, i32 1, <4 x i1> %c, <4 x i8> undef)
+  %2 = zext <4 x i8> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %x
+}
+
+define i8* @ldrbu32_2(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrbu32_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrbt.u32 q0, [r0, #2]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 2
+  %0 = bitcast i8* %z to <4 x i8>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %0, i32 1, <4 x i1> %c, <4 x i8> undef)
+  %2 = zext <4 x i8> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %x
+}
+
+define i8* @ldrbu32_127(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrbu32_127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrbt.u32 q0, [r0, #127]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 127
+  %0 = bitcast i8* %z to <4 x i8>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %0, i32 1, <4 x i1> %c, <4 x i8> undef)
+  %2 = zext <4 x i8> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %x
+}
+
+define i8* @ldrbu32_128(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrbu32_128:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    add.w r3, r0, #128
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrbt.u32 q0, [r3]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 128
+  %0 = bitcast i8* %z to <4 x i8>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %0, i32 1, <4 x i1> %c, <4 x i8> undef)
+  %2 = zext <4 x i8> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %x
+}
+
+define i8* @ldrbu32_m127(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrbu32_m127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrbt.u32 q0, [r0, #-127]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -127
+  %0 = bitcast i8* %z to <4 x i8>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %0, i32 1, <4 x i1> %c, <4 x i8> undef)
+  %2 = zext <4 x i8> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %x
+}
+
+define i8* @ldrbu32_m128(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrbu32_m128:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    sub.w r3, r0, #128
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrbt.u32 q0, [r3]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -128
+  %0 = bitcast i8* %z to <4 x i8>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %0, i32 1, <4 x i1> %c, <4 x i8> undef)
+  %2 = zext <4 x i8> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %x
+}
+
+define i8* @ldrbs32_4(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrbs32_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrbt.s32 q0, [r0, #4]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 4
+  %0 = bitcast i8* %z to <4 x i8>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %0, i32 1, <4 x i1> %c, <4 x i8> undef)
+  %2 = sext <4 x i8> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %x
+}
+
+define i8* @ldrbs32_3(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrbs32_3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrbt.s32 q0, [r0, #3]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 3
+  %0 = bitcast i8* %z to <4 x i8>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %0, i32 1, <4 x i1> %c, <4 x i8> undef)
+  %2 = sext <4 x i8> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %x
+}
+
+define i8* @ldrbs32_2(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrbs32_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrbt.s32 q0, [r0, #2]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 2
+  %0 = bitcast i8* %z to <4 x i8>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %0, i32 1, <4 x i1> %c, <4 x i8> undef)
+  %2 = sext <4 x i8> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %x
+}
+
+define i8* @ldrbs32_127(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrbs32_127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrbt.s32 q0, [r0, #127]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 127
+  %0 = bitcast i8* %z to <4 x i8>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %0, i32 1, <4 x i1> %c, <4 x i8> undef)
+  %2 = sext <4 x i8> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %x
+}
+
+define i8* @ldrbs32_128(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrbs32_128:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    add.w r3, r0, #128
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrbt.s32 q0, [r3]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 128
+  %0 = bitcast i8* %z to <4 x i8>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %0, i32 1, <4 x i1> %c, <4 x i8> undef)
+  %2 = sext <4 x i8> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %x
+}
+
+define i8* @ldrbs32_m127(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrbs32_m127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrbt.s32 q0, [r0, #-127]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -127
+  %0 = bitcast i8* %z to <4 x i8>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %0, i32 1, <4 x i1> %c, <4 x i8> undef)
+  %2 = sext <4 x i8> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %x
+}
+
+define i8* @ldrbs32_m128(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrbs32_m128:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    sub.w r3, r0, #128
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrbt.s32 q0, [r3]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -128
+  %0 = bitcast i8* %z to <4 x i8>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %0, i32 1, <4 x i1> %c, <4 x i8> undef)
+  %2 = sext <4 x i8> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %x
+}
+
+define i8* @ldrbu16_4(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrbu16_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrbt.u16 q0, [r0, #4]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 4
+  %0 = bitcast i8* %z to <8 x i8>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %0, i32 1, <8 x i1> %c, <8 x i8> undef)
+  %2 = zext <8 x i8> %1 to <8 x i16>
+  %3 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %2, <8 x i16>* %3, align 2
+  ret i8* %x
+}
+
+define i8* @ldrbu16_3(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrbu16_3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrbt.u16 q0, [r0, #3]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 3
+  %0 = bitcast i8* %z to <8 x i8>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %0, i32 1, <8 x i1> %c, <8 x i8> undef)
+  %2 = zext <8 x i8> %1 to <8 x i16>
+  %3 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %2, <8 x i16>* %3, align 2
+  ret i8* %x
+}
+
+define i8* @ldrbu16_2(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrbu16_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrbt.u16 q0, [r0, #2]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 2
+  %0 = bitcast i8* %z to <8 x i8>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %0, i32 1, <8 x i1> %c, <8 x i8> undef)
+  %2 = zext <8 x i8> %1 to <8 x i16>
+  %3 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %2, <8 x i16>* %3, align 2
+  ret i8* %x
+}
+
+define i8* @ldrbu16_127(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrbu16_127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrbt.u16 q0, [r0, #127]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 127
+  %0 = bitcast i8* %z to <8 x i8>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %0, i32 1, <8 x i1> %c, <8 x i8> undef)
+  %2 = zext <8 x i8> %1 to <8 x i16>
+  %3 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %2, <8 x i16>* %3, align 2
+  ret i8* %x
+}
+
+define i8* @ldrbu16_128(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrbu16_128:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    add.w r3, r0, #128
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrbt.u16 q0, [r3]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 128
+  %0 = bitcast i8* %z to <8 x i8>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %0, i32 1, <8 x i1> %c, <8 x i8> undef)
+  %2 = zext <8 x i8> %1 to <8 x i16>
+  %3 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %2, <8 x i16>* %3, align 2
+  ret i8* %x
+}
+
+define i8* @ldrbu16_m127(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrbu16_m127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrbt.u16 q0, [r0, #-127]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -127
+  %0 = bitcast i8* %z to <8 x i8>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %0, i32 1, <8 x i1> %c, <8 x i8> undef)
+  %2 = zext <8 x i8> %1 to <8 x i16>
+  %3 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %2, <8 x i16>* %3, align 2
+  ret i8* %x
+}
+
+define i8* @ldrbu16_m128(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrbu16_m128:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    sub.w r3, r0, #128
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrbt.u16 q0, [r3]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -128
+  %0 = bitcast i8* %z to <8 x i8>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %0, i32 1, <8 x i1> %c, <8 x i8> undef)
+  %2 = zext <8 x i8> %1 to <8 x i16>
+  %3 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %2, <8 x i16>* %3, align 2
+  ret i8* %x
+}
+
+define i8* @ldrbs16_4(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrbs16_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrbt.s16 q0, [r0, #4]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 4
+  %0 = bitcast i8* %z to <8 x i8>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %0, i32 1, <8 x i1> %c, <8 x i8> undef)
+  %2 = sext <8 x i8> %1 to <8 x i16>
+  %3 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %2, <8 x i16>* %3, align 2
+  ret i8* %x
+}
+
+define i8* @ldrbs16_3(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrbs16_3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrbt.s16 q0, [r0, #3]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 3
+  %0 = bitcast i8* %z to <8 x i8>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %0, i32 1, <8 x i1> %c, <8 x i8> undef)
+  %2 = sext <8 x i8> %1 to <8 x i16>
+  %3 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %2, <8 x i16>* %3, align 2
+  ret i8* %x
+}
+
+define i8* @ldrbs16_2(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrbs16_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrbt.s16 q0, [r0, #2]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 2
+  %0 = bitcast i8* %z to <8 x i8>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %0, i32 1, <8 x i1> %c, <8 x i8> undef)
+  %2 = sext <8 x i8> %1 to <8 x i16>
+  %3 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %2, <8 x i16>* %3, align 2
+  ret i8* %x
+}
+
+define i8* @ldrbs16_127(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrbs16_127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrbt.s16 q0, [r0, #127]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 127
+  %0 = bitcast i8* %z to <8 x i8>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %0, i32 1, <8 x i1> %c, <8 x i8> undef)
+  %2 = sext <8 x i8> %1 to <8 x i16>
+  %3 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %2, <8 x i16>* %3, align 2
+  ret i8* %x
+}
+
+define i8* @ldrbs16_128(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrbs16_128:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    add.w r3, r0, #128
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrbt.s16 q0, [r3]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 128
+  %0 = bitcast i8* %z to <8 x i8>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %0, i32 1, <8 x i1> %c, <8 x i8> undef)
+  %2 = sext <8 x i8> %1 to <8 x i16>
+  %3 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %2, <8 x i16>* %3, align 2
+  ret i8* %x
+}
+
+define i8* @ldrbs16_m127(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrbs16_m127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrbt.s16 q0, [r0, #-127]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -127
+  %0 = bitcast i8* %z to <8 x i8>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %0, i32 1, <8 x i1> %c, <8 x i8> undef)
+  %2 = sext <8 x i8> %1 to <8 x i16>
+  %3 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %2, <8 x i16>* %3, align 2
+  ret i8* %x
+}
+
+define i8* @ldrbs16_m128(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrbs16_m128:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    sub.w r3, r0, #128
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrbt.s16 q0, [r3]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -128
+  %0 = bitcast i8* %z to <8 x i8>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %0, i32 1, <8 x i1> %c, <8 x i8> undef)
+  %2 = sext <8 x i8> %1 to <8 x i16>
+  %3 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %2, <8 x i16>* %3, align 2
+  ret i8* %x
+}
+
+define i8* @ldrbu8_4(i8* %x, i8* %y, <16 x i8> *%m) {
+; CHECK-LABEL: ldrbu8_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u8 q0, [r2]
+; CHECK-NEXT:    vpt.i8 ne, q0, zr
+; CHECK-NEXT:    vldrbt.u8 q0, [r0, #4]
+; CHECK-NEXT:    vstrb.8 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 4
+  %0 = bitcast i8* %z to <16 x i8>*
+  %mask = load <16 x i8>, <16 x i8>* %m, align 1
+  %c = icmp ne <16 x i8> %mask, zeroinitializer
+  %1 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %0, i32 1, <16 x i1> %c, <16 x i8> undef)
+  %2 = bitcast i8* %y to <16 x i8>*
+  store <16 x i8> %1, <16 x i8>* %2, align 1
+  ret i8* %x
+}
+
+define i8* @ldrbu8_3(i8* %x, i8* %y, <16 x i8> *%m) {
+; CHECK-LABEL: ldrbu8_3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u8 q0, [r2]
+; CHECK-NEXT:    vpt.i8 ne, q0, zr
+; CHECK-NEXT:    vldrbt.u8 q0, [r0, #3]
+; CHECK-NEXT:    vstrb.8 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 3
+  %0 = bitcast i8* %z to <16 x i8>*
+  %mask = load <16 x i8>, <16 x i8>* %m, align 1
+  %c = icmp ne <16 x i8> %mask, zeroinitializer
+  %1 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %0, i32 1, <16 x i1> %c, <16 x i8> undef)
+  %2 = bitcast i8* %y to <16 x i8>*
+  store <16 x i8> %1, <16 x i8>* %2, align 1
+  ret i8* %x
+}
+
+define i8* @ldrbu8_2(i8* %x, i8* %y, <16 x i8> *%m) {
+; CHECK-LABEL: ldrbu8_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u8 q0, [r2]
+; CHECK-NEXT:    vpt.i8 ne, q0, zr
+; CHECK-NEXT:    vldrbt.u8 q0, [r0, #2]
+; CHECK-NEXT:    vstrb.8 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 2
+  %0 = bitcast i8* %z to <16 x i8>*
+  %mask = load <16 x i8>, <16 x i8>* %m, align 1
+  %c = icmp ne <16 x i8> %mask, zeroinitializer
+  %1 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %0, i32 1, <16 x i1> %c, <16 x i8> undef)
+  %2 = bitcast i8* %y to <16 x i8>*
+  store <16 x i8> %1, <16 x i8>* %2, align 1
+  ret i8* %x
+}
+
+define i8* @ldrbu8_127(i8* %x, i8* %y, <16 x i8> *%m) {
+; CHECK-LABEL: ldrbu8_127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u8 q0, [r2]
+; CHECK-NEXT:    vpt.i8 ne, q0, zr
+; CHECK-NEXT:    vldrbt.u8 q0, [r0, #127]
+; CHECK-NEXT:    vstrb.8 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 127
+  %0 = bitcast i8* %z to <16 x i8>*
+  %mask = load <16 x i8>, <16 x i8>* %m, align 1
+  %c = icmp ne <16 x i8> %mask, zeroinitializer
+  %1 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %0, i32 1, <16 x i1> %c, <16 x i8> undef)
+  %2 = bitcast i8* %y to <16 x i8>*
+  store <16 x i8> %1, <16 x i8>* %2, align 1
+  ret i8* %x
+}
+
+define i8* @ldrbu8_128(i8* %x, i8* %y, <16 x i8> *%m) {
+; CHECK-LABEL: ldrbu8_128:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u8 q0, [r2]
+; CHECK-NEXT:    add.w r3, r0, #128
+; CHECK-NEXT:    vpt.i8 ne, q0, zr
+; CHECK-NEXT:    vldrbt.u8 q0, [r3]
+; CHECK-NEXT:    vstrb.8 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 128
+  %0 = bitcast i8* %z to <16 x i8>*
+  %mask = load <16 x i8>, <16 x i8>* %m, align 1
+  %c = icmp ne <16 x i8> %mask, zeroinitializer
+  %1 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %0, i32 1, <16 x i1> %c, <16 x i8> undef)
+  %2 = bitcast i8* %y to <16 x i8>*
+  store <16 x i8> %1, <16 x i8>* %2, align 1
+  ret i8* %x
+}
+
+define i8* @ldrbu8_m127(i8* %x, i8* %y, <16 x i8> *%m) {
+; CHECK-LABEL: ldrbu8_m127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u8 q0, [r2]
+; CHECK-NEXT:    vpt.i8 ne, q0, zr
+; CHECK-NEXT:    vldrbt.u8 q0, [r0, #-127]
+; CHECK-NEXT:    vstrb.8 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -127
+  %0 = bitcast i8* %z to <16 x i8>*
+  %mask = load <16 x i8>, <16 x i8>* %m, align 1
+  %c = icmp ne <16 x i8> %mask, zeroinitializer
+  %1 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %0, i32 1, <16 x i1> %c, <16 x i8> undef)
+  %2 = bitcast i8* %y to <16 x i8>*
+  store <16 x i8> %1, <16 x i8>* %2, align 1
+  ret i8* %x
+}
+
+define i8* @ldrbu8_m128(i8* %x, i8* %y, <16 x i8> *%m) {
+; CHECK-LABEL: ldrbu8_m128:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u8 q0, [r2]
+; CHECK-NEXT:    sub.w r3, r0, #128
+; CHECK-NEXT:    vpt.i8 ne, q0, zr
+; CHECK-NEXT:    vldrbt.u8 q0, [r3]
+; CHECK-NEXT:    vstrb.8 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -128
+  %0 = bitcast i8* %z to <16 x i8>*
+  %mask = load <16 x i8>, <16 x i8>* %m, align 1
+  %c = icmp ne <16 x i8> %mask, zeroinitializer
+  %1 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %0, i32 1, <16 x i1> %c, <16 x i8> undef)
+  %2 = bitcast i8* %y to <16 x i8>*
+  store <16 x i8> %1, <16 x i8>* %2, align 1
+  ret i8* %x
+}
+
+define i8* @ldrwf32_4(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrwf32_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrwt.u32 q0, [r0, #4]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 4
+  %0 = bitcast i8* %z to <4 x float>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* %0, i32 4, <4 x i1> %c, <4 x float> undef)
+  %2 = bitcast i8* %y to <4 x float>*
+  store <4 x float> %1, <4 x float>* %2, align 4
+  ret i8* %x
+}
+
+define i8* @ldrwf32_3(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrwf32_3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    adds r3, r0, #3
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrwt.u32 q0, [r3]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 3
+  %0 = bitcast i8* %z to <4 x float>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* %0, i32 4, <4 x i1> %c, <4 x float> undef)
+  %2 = bitcast i8* %y to <4 x float>*
+  store <4 x float> %1, <4 x float>* %2, align 4
+  ret i8* %x
+}
+
+define i8* @ldrwf32_2(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrwf32_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    adds r3, r0, #2
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrwt.u32 q0, [r3]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 2
+  %0 = bitcast i8* %z to <4 x float>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* %0, i32 4, <4 x i1> %c, <4 x float> undef)
+  %2 = bitcast i8* %y to <4 x float>*
+  store <4 x float> %1, <4 x float>* %2, align 4
+  ret i8* %x
+}
+
+define i8* @ldrwf32_508(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrwf32_508:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrwt.u32 q0, [r0, #508]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 508
+  %0 = bitcast i8* %z to <4 x float>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* %0, i32 4, <4 x i1> %c, <4 x float> undef)
+  %2 = bitcast i8* %y to <4 x float>*
+  store <4 x float> %1, <4 x float>* %2, align 4
+  ret i8* %x
+}
+
+define i8* @ldrwf32_512(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrwf32_512:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    add.w r3, r0, #512
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrwt.u32 q0, [r3]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 512
+  %0 = bitcast i8* %z to <4 x float>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* %0, i32 4, <4 x i1> %c, <4 x float> undef)
+  %2 = bitcast i8* %y to <4 x float>*
+  store <4 x float> %1, <4 x float>* %2, align 4
+  ret i8* %x
+}
+
+define i8* @ldrwf32_m508(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrwf32_m508:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrwt.u32 q0, [r0, #-508]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -508
+  %0 = bitcast i8* %z to <4 x float>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* %0, i32 4, <4 x i1> %c, <4 x float> undef)
+  %2 = bitcast i8* %y to <4 x float>*
+  store <4 x float> %1, <4 x float>* %2, align 4
+  ret i8* %x
+}
+
+define i8* @ldrwf32_m512(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrwf32_m512:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    sub.w r3, r0, #512
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrwt.u32 q0, [r3]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -512
+  %0 = bitcast i8* %z to <4 x float>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* %0, i32 4, <4 x i1> %c, <4 x float> undef)
+  %2 = bitcast i8* %y to <4 x float>*
+  store <4 x float> %1, <4 x float>* %2, align 4
+  ret i8* %x
+}
+
+define i8* @ldrhf16_4(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrhf16_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrht.u16 q0, [r0, #4]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 4
+  %0 = bitcast i8* %z to <8 x half>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>* %0, i32 2, <8 x i1> %c, <8 x half> undef)
+  %2 = bitcast i8* %y to <8 x half>*
+  store <8 x half> %1, <8 x half>* %2, align 2
+  ret i8* %x
+}
+
+define i8* @ldrhf16_3(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrhf16_3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    adds r3, r0, #3
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrht.u16 q0, [r3]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 3
+  %0 = bitcast i8* %z to <8 x half>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>* %0, i32 2, <8 x i1> %c, <8 x half> undef)
+  %2 = bitcast i8* %y to <8 x half>*
+  store <8 x half> %1, <8 x half>* %2, align 2
+  ret i8* %x
+}
+
+define i8* @ldrhf16_2(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrhf16_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrht.u16 q0, [r0, #2]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 2
+  %0 = bitcast i8* %z to <8 x half>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>* %0, i32 2, <8 x i1> %c, <8 x half> undef)
+  %2 = bitcast i8* %y to <8 x half>*
+  store <8 x half> %1, <8 x half>* %2, align 2
+  ret i8* %x
+}
+
+define i8* @ldrhf16_254(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrhf16_254:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrht.u16 q0, [r0, #254]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 254
+  %0 = bitcast i8* %z to <8 x half>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>* %0, i32 2, <8 x i1> %c, <8 x half> undef)
+  %2 = bitcast i8* %y to <8 x half>*
+  store <8 x half> %1, <8 x half>* %2, align 2
+  ret i8* %x
+}
+
+define i8* @ldrhf16_256(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrhf16_256:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    add.w r3, r0, #256
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrht.u16 q0, [r3]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 256
+  %0 = bitcast i8* %z to <8 x half>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>* %0, i32 2, <8 x i1> %c, <8 x half> undef)
+  %2 = bitcast i8* %y to <8 x half>*
+  store <8 x half> %1, <8 x half>* %2, align 2
+  ret i8* %x
+}
+
+define i8* @ldrhf16_m254(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrhf16_m254:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrht.u16 q0, [r0, #-254]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -254
+  %0 = bitcast i8* %z to <8 x half>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>* %0, i32 2, <8 x i1> %c, <8 x half> undef)
+  %2 = bitcast i8* %y to <8 x half>*
+  store <8 x half> %1, <8 x half>* %2, align 2
+  ret i8* %x
+}
+
+define i8* @ldrhf16_m256(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrhf16_m256:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    sub.w r3, r0, #256
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrht.u16 q0, [r3]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -256
+  %0 = bitcast i8* %z to <8 x half>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>* %0, i32 2, <8 x i1> %c, <8 x half> undef)
+  %2 = bitcast i8* %y to <8 x half>*
+  store <8 x half> %1, <8 x half>* %2, align 2
+  ret i8* %x
+}
+
+
+
+
+define i8* @strw32_4(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strw32_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrwt.32 q0, [r0, #4]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 4
+  %0 = bitcast i8* %x to <4 x i32>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
+  %2 = bitcast i8* %z to <4 x i32>*
+  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %1, <4 x i32>* %2, i32 4, <4 x i1> %c)
+  ret i8* %y
+}
+
+define i8* @strw32_3(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strw32_3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    adds r1, r0, #3
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrwt.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 3
+  %0 = bitcast i8* %x to <4 x i32>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
+  %2 = bitcast i8* %z to <4 x i32>*
+  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %1, <4 x i32>* %2, i32 4, <4 x i1> %c)
+  ret i8* %y
+}
+
+define i8* @strw32_2(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strw32_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    adds r1, r0, #2
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrwt.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 2
+  %0 = bitcast i8* %x to <4 x i32>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
+  %2 = bitcast i8* %z to <4 x i32>*
+  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %1, <4 x i32>* %2, i32 4, <4 x i1> %c)
+  ret i8* %y
+}
+
+define i8* @strw32_508(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strw32_508:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrwt.32 q0, [r0, #508]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 508
+  %0 = bitcast i8* %x to <4 x i32>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
+  %2 = bitcast i8* %z to <4 x i32>*
+  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %1, <4 x i32>* %2, i32 4, <4 x i1> %c)
+  ret i8* %y
+}
+
+define i8* @strw32_512(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strw32_512:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    add.w r1, r0, #512
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrwt.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 512
+  %0 = bitcast i8* %x to <4 x i32>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
+  %2 = bitcast i8* %z to <4 x i32>*
+  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %1, <4 x i32>* %2, i32 4, <4 x i1> %c)
+  ret i8* %y
+}
+
+define i8* @strw32_m508(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strw32_m508:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrwt.32 q0, [r0, #-508]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -508
+  %0 = bitcast i8* %x to <4 x i32>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
+  %2 = bitcast i8* %z to <4 x i32>*
+  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %1, <4 x i32>* %2, i32 4, <4 x i1> %c)
+  ret i8* %y
+}
+
+define i8* @strw32_m512(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strw32_m512:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    sub.w r1, r0, #512
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrwt.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -512
+  %0 = bitcast i8* %x to <4 x i32>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
+  %2 = bitcast i8* %z to <4 x i32>*
+  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %1, <4 x i32>* %2, i32 4, <4 x i1> %c)
+  ret i8* %y
+}
+
+define i8* @strh32_4(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strh32_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrht.32 q0, [r0, #4]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 4
+  %0 = bitcast i8* %x to <4 x i16>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
+  %2 = bitcast i8* %z to <4 x i16>*
+  call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %1, <4 x i16>* %2, i32 2, <4 x i1> %c)
+  ret i8* %y
+}
+
+define i8* @strh32_3(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strh32_3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u32 q0, [r1]
+; CHECK-NEXT:    adds r1, r0, #3
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrht.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 3
+  %0 = bitcast i8* %x to <4 x i16>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
+  %2 = bitcast i8* %z to <4 x i16>*
+  call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %1, <4 x i16>* %2, i32 2, <4 x i1> %c)
+  ret i8* %y
+}
+
+define i8* @strh32_2(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strh32_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrht.32 q0, [r0, #2]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 2
+  %0 = bitcast i8* %x to <4 x i16>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
+  %2 = bitcast i8* %z to <4 x i16>*
+  call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %1, <4 x i16>* %2, i32 2, <4 x i1> %c)
+  ret i8* %y
+}
+
+define i8* @strh32_254(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strh32_254:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrht.32 q0, [r0, #254]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 254
+  %0 = bitcast i8* %x to <4 x i16>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
+  %2 = bitcast i8* %z to <4 x i16>*
+  call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %1, <4 x i16>* %2, i32 2, <4 x i1> %c)
+  ret i8* %y
+}
+
+define i8* @strh32_256(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strh32_256:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u32 q0, [r1]
+; CHECK-NEXT:    add.w r1, r0, #256
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrht.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 256
+  %0 = bitcast i8* %x to <4 x i16>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
+  %2 = bitcast i8* %z to <4 x i16>*
+  call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %1, <4 x i16>* %2, i32 2, <4 x i1> %c)
+  ret i8* %y
+}
+
+define i8* @strh32_m254(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strh32_m254:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrht.32 q0, [r0, #-254]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -254
+  %0 = bitcast i8* %x to <4 x i16>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
+  %2 = bitcast i8* %z to <4 x i16>*
+  call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %1, <4 x i16>* %2, i32 2, <4 x i1> %c)
+  ret i8* %y
+}
+
+define i8* @strh32_m256(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strh32_m256:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u32 q0, [r1]
+; CHECK-NEXT:    sub.w r1, r0, #256
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrht.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -256
+  %0 = bitcast i8* %x to <4 x i16>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
+  %2 = bitcast i8* %z to <4 x i16>*
+  call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %1, <4 x i16>* %2, i32 2, <4 x i1> %c)
+  ret i8* %y
+}
+
+define i8* @strh16_4(i8* %y, i8* %x, <8 x i16> *%m) {
+; CHECK-LABEL: strh16_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q1, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q1, zr
+; CHECK-NEXT:    vstrht.16 q0, [r0, #4]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 4
+  %0 = bitcast i8* %x to <8 x i16>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
+  %2 = bitcast i8* %z to <8 x i16>*
+  call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %1, <8 x i16>* %2, i32 2, <8 x i1> %c)
+  ret i8* %y
+}
+
+define i8* @strh16_3(i8* %y, i8* %x, <8 x i16> *%m) {
+; CHECK-LABEL: strh16_3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-NEXT:    adds r1, r0, #3
+; CHECK-NEXT:    vldrh.u16 q1, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q1, zr
+; CHECK-NEXT:    vstrht.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 3
+  %0 = bitcast i8* %x to <8 x i16>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
+  %2 = bitcast i8* %z to <8 x i16>*
+  call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %1, <8 x i16>* %2, i32 2, <8 x i1> %c)
+  ret i8* %y
+}
+
+define i8* @strh16_2(i8* %y, i8* %x, <8 x i16> *%m) {
+; CHECK-LABEL: strh16_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q1, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q1, zr
+; CHECK-NEXT:    vstrht.16 q0, [r0, #2]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 2
+  %0 = bitcast i8* %x to <8 x i16>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
+  %2 = bitcast i8* %z to <8 x i16>*
+  call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %1, <8 x i16>* %2, i32 2, <8 x i1> %c)
+  ret i8* %y
+}
+
+define i8* @strh16_254(i8* %y, i8* %x, <8 x i16> *%m) {
+; CHECK-LABEL: strh16_254:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q1, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q1, zr
+; CHECK-NEXT:    vstrht.16 q0, [r0, #254]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 254
+  %0 = bitcast i8* %x to <8 x i16>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
+  %2 = bitcast i8* %z to <8 x i16>*
+  call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %1, <8 x i16>* %2, i32 2, <8 x i1> %c)
+  ret i8* %y
+}
+
+define i8* @strh16_256(i8* %y, i8* %x, <8 x i16> *%m) {
+; CHECK-LABEL: strh16_256:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-NEXT:    add.w r1, r0, #256
+; CHECK-NEXT:    vldrh.u16 q1, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q1, zr
+; CHECK-NEXT:    vstrht.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 256
+  %0 = bitcast i8* %x to <8 x i16>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
+  %2 = bitcast i8* %z to <8 x i16>*
+  call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %1, <8 x i16>* %2, i32 2, <8 x i1> %c)
+  ret i8* %y
+}
+
+define i8* @strh16_m254(i8* %y, i8* %x, <8 x i16> *%m) {
+; CHECK-LABEL: strh16_m254:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q1, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q1, zr
+; CHECK-NEXT:    vstrht.16 q0, [r0, #-254]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -254
+  %0 = bitcast i8* %x to <8 x i16>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
+  %2 = bitcast i8* %z to <8 x i16>*
+  call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %1, <8 x i16>* %2, i32 2, <8 x i1> %c)
+  ret i8* %y
+}
+
+define i8* @strh16_m256(i8* %y, i8* %x, <8 x i16> *%m) {
+; CHECK-LABEL: strh16_m256:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-NEXT:    sub.w r1, r0, #256
+; CHECK-NEXT:    vldrh.u16 q1, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q1, zr
+; CHECK-NEXT:    vstrht.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -256
+  %0 = bitcast i8* %x to <8 x i16>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
+  %2 = bitcast i8* %z to <8 x i16>*
+  call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %1, <8 x i16>* %2, i32 2, <8 x i1> %c)
+  ret i8* %y
+}
+
+define i8* @strb32_4(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strb32_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrbt.32 q0, [r0, #4]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 4
+  %0 = bitcast i8* %x to <4 x i8>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
+  %2 = bitcast i8* %z to <4 x i8>*
+  call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %1, <4 x i8>* %2, i32 1, <4 x i1> %c)
+  ret i8* %y
+}
+
+define i8* @strb32_3(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strb32_3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrbt.32 q0, [r0, #3]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 3
+  %0 = bitcast i8* %x to <4 x i8>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
+  %2 = bitcast i8* %z to <4 x i8>*
+  call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %1, <4 x i8>* %2, i32 1, <4 x i1> %c)
+  ret i8* %y
+}
+
+define i8* @strb32_2(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strb32_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrbt.32 q0, [r0, #2]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 2
+  %0 = bitcast i8* %x to <4 x i8>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
+  %2 = bitcast i8* %z to <4 x i8>*
+  call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %1, <4 x i8>* %2, i32 1, <4 x i1> %c)
+  ret i8* %y
+}
+
+define i8* @strb32_127(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strb32_127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrbt.32 q0, [r0, #127]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 127
+  %0 = bitcast i8* %x to <4 x i8>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
+  %2 = bitcast i8* %z to <4 x i8>*
+  call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %1, <4 x i8>* %2, i32 1, <4 x i1> %c)
+  ret i8* %y
+}
+
+define i8* @strb32_128(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strb32_128:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u32 q0, [r1]
+; CHECK-NEXT:    add.w r1, r0, #128
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrbt.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 128
+  %0 = bitcast i8* %x to <4 x i8>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
+  %2 = bitcast i8* %z to <4 x i8>*
+  call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %1, <4 x i8>* %2, i32 1, <4 x i1> %c)
+  ret i8* %y
+}
+
+define i8* @strb32_m127(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strb32_m127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrbt.32 q0, [r0, #-127]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -127
+  %0 = bitcast i8* %x to <4 x i8>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
+  %2 = bitcast i8* %z to <4 x i8>*
+  call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %1, <4 x i8>* %2, i32 1, <4 x i1> %c)
+  ret i8* %y
+}
+
+define i8* @strb32_m128(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strb32_m128:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u32 q0, [r1]
+; CHECK-NEXT:    sub.w r1, r0, #128
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrbt.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -128
+  %0 = bitcast i8* %x to <4 x i8>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
+  %2 = bitcast i8* %z to <4 x i8>*
+  call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %1, <4 x i8>* %2, i32 1, <4 x i1> %c)
+  ret i8* %y
+}
+
+define i8* @strb16_4(i8* %y, i8* %x, <8 x i16> *%m) {
+; CHECK-LABEL: strb16_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u16 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q1, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q1, zr
+; CHECK-NEXT:    vstrbt.16 q0, [r0, #4]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 4
+  %0 = bitcast i8* %x to <8 x i8>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
+  %2 = bitcast i8* %z to <8 x i8>*
+  call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %1, <8 x i8>* %2, i32 1, <8 x i1> %c)
+  ret i8* %y
+}
+
+define i8* @strb16_3(i8* %y, i8* %x, <8 x i16> *%m) {
+; CHECK-LABEL: strb16_3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u16 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q1, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q1, zr
+; CHECK-NEXT:    vstrbt.16 q0, [r0, #3]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 3
+  %0 = bitcast i8* %x to <8 x i8>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
+  %2 = bitcast i8* %z to <8 x i8>*
+  call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %1, <8 x i8>* %2, i32 1, <8 x i1> %c)
+  ret i8* %y
+}
+
+define i8* @strb16_2(i8* %y, i8* %x, <8 x i16> *%m) {
+; CHECK-LABEL: strb16_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u16 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q1, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q1, zr
+; CHECK-NEXT:    vstrbt.16 q0, [r0, #2]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 2
+  %0 = bitcast i8* %x to <8 x i8>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
+  %2 = bitcast i8* %z to <8 x i8>*
+  call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %1, <8 x i8>* %2, i32 1, <8 x i1> %c)
+  ret i8* %y
+}
+
+define i8* @strb16_127(i8* %y, i8* %x, <8 x i16> *%m) {
+; CHECK-LABEL: strb16_127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u16 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q1, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q1, zr
+; CHECK-NEXT:    vstrbt.16 q0, [r0, #127]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 127
+  %0 = bitcast i8* %x to <8 x i8>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
+  %2 = bitcast i8* %z to <8 x i8>*
+  call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %1, <8 x i8>* %2, i32 1, <8 x i1> %c)
+  ret i8* %y
+}
+
+define i8* @strb16_128(i8* %y, i8* %x, <8 x i16> *%m) {
+; CHECK-LABEL: strb16_128:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u16 q0, [r1]
+; CHECK-NEXT:    add.w r1, r0, #128
+; CHECK-NEXT:    vldrh.u16 q1, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q1, zr
+; CHECK-NEXT:    vstrbt.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 128
+  %0 = bitcast i8* %x to <8 x i8>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
+  %2 = bitcast i8* %z to <8 x i8>*
+  call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %1, <8 x i8>* %2, i32 1, <8 x i1> %c)
+  ret i8* %y
+}
+
+define i8* @strb16_m127(i8* %y, i8* %x, <8 x i16> *%m) {
+; CHECK-LABEL: strb16_m127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u16 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q1, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q1, zr
+; CHECK-NEXT:    vstrbt.16 q0, [r0, #-127]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -127
+  %0 = bitcast i8* %x to <8 x i8>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
+  %2 = bitcast i8* %z to <8 x i8>*
+  call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %1, <8 x i8>* %2, i32 1, <8 x i1> %c)
+  ret i8* %y
+}
+
+define i8* @strb16_m128(i8* %y, i8* %x, <8 x i16> *%m) {
+; CHECK-LABEL: strb16_m128:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u16 q0, [r1]
+; CHECK-NEXT:    sub.w r1, r0, #128
+; CHECK-NEXT:    vldrh.u16 q1, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q1, zr
+; CHECK-NEXT:    vstrbt.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -128
+  %0 = bitcast i8* %x to <8 x i8>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
+  %2 = bitcast i8* %z to <8 x i8>*
+  call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %1, <8 x i8>* %2, i32 1, <8 x i1> %c)
+  ret i8* %y
+}
+
+define i8* @strb8_4(i8* %y, i8* %x, <16 x i8> *%m) {
+; CHECK-LABEL: strb8_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u8 q0, [r1]
+; CHECK-NEXT:    vldrb.u8 q1, [r2]
+; CHECK-NEXT:    vpt.i8 ne, q1, zr
+; CHECK-NEXT:    vstrbt.8 q0, [r0, #4]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 4
+  %0 = bitcast i8* %x to <16 x i8>*
+  %mask = load <16 x i8>, <16 x i8>* %m, align 1
+  %c = icmp ne <16 x i8> %mask, zeroinitializer
+  %1 = load <16 x i8>, <16 x i8>* %0, align 1
+  %2 = bitcast i8* %z to <16 x i8>*
+  call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %1, <16 x i8>* %2, i32 1, <16 x i1> %c)
+  ret i8* %y
+}
+
+define i8* @strb8_3(i8* %y, i8* %x, <16 x i8> *%m) {
+; CHECK-LABEL: strb8_3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u8 q0, [r1]
+; CHECK-NEXT:    vldrb.u8 q1, [r2]
+; CHECK-NEXT:    vpt.i8 ne, q1, zr
+; CHECK-NEXT:    vstrbt.8 q0, [r0, #3]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 3
+  %0 = bitcast i8* %x to <16 x i8>*
+  %mask = load <16 x i8>, <16 x i8>* %m, align 1
+  %c = icmp ne <16 x i8> %mask, zeroinitializer
+  %1 = load <16 x i8>, <16 x i8>* %0, align 1
+  %2 = bitcast i8* %z to <16 x i8>*
+  call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %1, <16 x i8>* %2, i32 1, <16 x i1> %c)
+  ret i8* %y
+}
+
+define i8* @strb8_2(i8* %y, i8* %x, <16 x i8> *%m) {
+; CHECK-LABEL: strb8_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u8 q0, [r1]
+; CHECK-NEXT:    vldrb.u8 q1, [r2]
+; CHECK-NEXT:    vpt.i8 ne, q1, zr
+; CHECK-NEXT:    vstrbt.8 q0, [r0, #2]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 2
+  %0 = bitcast i8* %x to <16 x i8>*
+  %mask = load <16 x i8>, <16 x i8>* %m, align 1
+  %c = icmp ne <16 x i8> %mask, zeroinitializer
+  %1 = load <16 x i8>, <16 x i8>* %0, align 1
+  %2 = bitcast i8* %z to <16 x i8>*
+  call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %1, <16 x i8>* %2, i32 1, <16 x i1> %c)
+  ret i8* %y
+}
+
+define i8* @strb8_127(i8* %y, i8* %x, <16 x i8> *%m) {
+; CHECK-LABEL: strb8_127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u8 q0, [r1]
+; CHECK-NEXT:    vldrb.u8 q1, [r2]
+; CHECK-NEXT:    vpt.i8 ne, q1, zr
+; CHECK-NEXT:    vstrbt.8 q0, [r0, #127]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 127
+  %0 = bitcast i8* %x to <16 x i8>*
+  %mask = load <16 x i8>, <16 x i8>* %m, align 1
+  %c = icmp ne <16 x i8> %mask, zeroinitializer
+  %1 = load <16 x i8>, <16 x i8>* %0, align 1
+  %2 = bitcast i8* %z to <16 x i8>*
+  call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %1, <16 x i8>* %2, i32 1, <16 x i1> %c)
+  ret i8* %y
+}
+
+define i8* @strb8_128(i8* %y, i8* %x, <16 x i8> *%m) {
+; CHECK-LABEL: strb8_128:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u8 q0, [r1]
+; CHECK-NEXT:    add.w r1, r0, #128
+; CHECK-NEXT:    vldrb.u8 q1, [r2]
+; CHECK-NEXT:    vpt.i8 ne, q1, zr
+; CHECK-NEXT:    vstrbt.8 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 128
+  %0 = bitcast i8* %x to <16 x i8>*
+  %mask = load <16 x i8>, <16 x i8>* %m, align 1
+  %c = icmp ne <16 x i8> %mask, zeroinitializer
+  %1 = load <16 x i8>, <16 x i8>* %0, align 1
+  %2 = bitcast i8* %z to <16 x i8>*
+  call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %1, <16 x i8>* %2, i32 1, <16 x i1> %c)
+  ret i8* %y
+}
+
+define i8* @strb8_m127(i8* %y, i8* %x, <16 x i8> *%m) {
+; CHECK-LABEL: strb8_m127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u8 q0, [r1]
+; CHECK-NEXT:    vldrb.u8 q1, [r2]
+; CHECK-NEXT:    vpt.i8 ne, q1, zr
+; CHECK-NEXT:    vstrbt.8 q0, [r0, #-127]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -127
+  %0 = bitcast i8* %x to <16 x i8>*
+  %mask = load <16 x i8>, <16 x i8>* %m, align 1
+  %c = icmp ne <16 x i8> %mask, zeroinitializer
+  %1 = load <16 x i8>, <16 x i8>* %0, align 1
+  %2 = bitcast i8* %z to <16 x i8>*
+  call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %1, <16 x i8>* %2, i32 1, <16 x i1> %c)
+  ret i8* %y
+}
+
+define i8* @strb8_m128(i8* %y, i8* %x, <16 x i8> *%m) {
+; CHECK-LABEL: strb8_m128:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u8 q0, [r1]
+; CHECK-NEXT:    sub.w r1, r0, #128
+; CHECK-NEXT:    vldrb.u8 q1, [r2]
+; CHECK-NEXT:    vpt.i8 ne, q1, zr
+; CHECK-NEXT:    vstrbt.8 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -128
+  %0 = bitcast i8* %x to <16 x i8>*
+  %mask = load <16 x i8>, <16 x i8>* %m, align 1
+  %c = icmp ne <16 x i8> %mask, zeroinitializer
+  %1 = load <16 x i8>, <16 x i8>* %0, align 1
+  %2 = bitcast i8* %z to <16 x i8>*
+  call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %1, <16 x i8>* %2, i32 1, <16 x i1> %c)
+  ret i8* %y
+}
+
+define i8* @strwf32_4(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strwf32_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrwt.32 q0, [r0, #4]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 4
+  %0 = bitcast i8* %x to <4 x float>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x float>, <4 x float>* %0, align 4
+  %2 = bitcast i8* %z to <4 x float>*
+  call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %1, <4 x float>* %2, i32 4, <4 x i1> %c)
+  ret i8* %y
+}
+
+define i8* @strwf32_3(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strwf32_3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    adds r1, r0, #3
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrwt.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 3
+  %0 = bitcast i8* %x to <4 x float>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x float>, <4 x float>* %0, align 4
+  %2 = bitcast i8* %z to <4 x float>*
+  call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %1, <4 x float>* %2, i32 4, <4 x i1> %c)
+  ret i8* %y
+}
+
+define i8* @strwf32_2(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strwf32_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    adds r1, r0, #2
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrwt.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 2
+  %0 = bitcast i8* %x to <4 x float>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x float>, <4 x float>* %0, align 4
+  %2 = bitcast i8* %z to <4 x float>*
+  call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %1, <4 x float>* %2, i32 4, <4 x i1> %c)
+  ret i8* %y
+}
+
+define i8* @strwf32_508(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strwf32_508:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrwt.32 q0, [r0, #508]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 508
+  %0 = bitcast i8* %x to <4 x float>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x float>, <4 x float>* %0, align 4
+  %2 = bitcast i8* %z to <4 x float>*
+  call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %1, <4 x float>* %2, i32 4, <4 x i1> %c)
+  ret i8* %y
+}
+
+define i8* @strwf32_512(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strwf32_512:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    add.w r1, r0, #512
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrwt.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 512
+  %0 = bitcast i8* %x to <4 x float>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x float>, <4 x float>* %0, align 4
+  %2 = bitcast i8* %z to <4 x float>*
+  call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %1, <4 x float>* %2, i32 4, <4 x i1> %c)
+  ret i8* %y
+}
+
+define i8* @strwf32_m508(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strwf32_m508:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrwt.32 q0, [r0, #-508]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -508
+  %0 = bitcast i8* %x to <4 x float>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x float>, <4 x float>* %0, align 4
+  %2 = bitcast i8* %z to <4 x float>*
+  call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %1, <4 x float>* %2, i32 4, <4 x i1> %c)
+  ret i8* %y
+}
+
+define i8* @strwf32_m512(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strwf32_m512:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    sub.w r1, r0, #512
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrwt.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -512
+  %0 = bitcast i8* %x to <4 x float>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x float>, <4 x float>* %0, align 4
+  %2 = bitcast i8* %z to <4 x float>*
+  call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %1, <4 x float>* %2, i32 4, <4 x i1> %c)
+  ret i8* %y
+}
+
+define i8* @strhf16_4(i8* %y, i8* %x, <8 x i16> *%m) {
+; CHECK-LABEL: strhf16_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q1, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q1, zr
+; CHECK-NEXT:    vstrht.16 q0, [r0, #4]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 4
+  %0 = bitcast i8* %x to <8 x half>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = load <8 x half>, <8 x half>* %0, align 2
+  %2 = bitcast i8* %z to <8 x half>*
+  call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %1, <8 x half>* %2, i32 2, <8 x i1> %c)
+  ret i8* %y
+}
+
+define i8* @strhf16_3(i8* %y, i8* %x, <8 x i16> *%m) {
+; CHECK-LABEL: strhf16_3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-NEXT:    adds r1, r0, #3
+; CHECK-NEXT:    vldrh.u16 q1, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q1, zr
+; CHECK-NEXT:    vstrht.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 3
+  %0 = bitcast i8* %x to <8 x half>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = load <8 x half>, <8 x half>* %0, align 2
+  %2 = bitcast i8* %z to <8 x half>*
+  call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %1, <8 x half>* %2, i32 2, <8 x i1> %c)
+  ret i8* %y
+}
+
+define i8* @strhf16_2(i8* %y, i8* %x, <8 x i16> *%m) {
+; CHECK-LABEL: strhf16_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q1, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q1, zr
+; CHECK-NEXT:    vstrht.16 q0, [r0, #2]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 2
+  %0 = bitcast i8* %x to <8 x half>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = load <8 x half>, <8 x half>* %0, align 2
+  %2 = bitcast i8* %z to <8 x half>*
+  call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %1, <8 x half>* %2, i32 2, <8 x i1> %c)
+  ret i8* %y
+}
+
+define i8* @strhf16_254(i8* %y, i8* %x, <8 x i16> *%m) {
+; CHECK-LABEL: strhf16_254:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q1, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q1, zr
+; CHECK-NEXT:    vstrht.16 q0, [r0, #254]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 254
+  %0 = bitcast i8* %x to <8 x half>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = load <8 x half>, <8 x half>* %0, align 2
+  %2 = bitcast i8* %z to <8 x half>*
+  call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %1, <8 x half>* %2, i32 2, <8 x i1> %c)
+  ret i8* %y
+}
+
+define i8* @strhf16_256(i8* %y, i8* %x, <8 x i16> *%m) {
+; CHECK-LABEL: strhf16_256:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-NEXT:    add.w r1, r0, #256
+; CHECK-NEXT:    vldrh.u16 q1, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q1, zr
+; CHECK-NEXT:    vstrht.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 256
+  %0 = bitcast i8* %x to <8 x half>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = load <8 x half>, <8 x half>* %0, align 2
+  %2 = bitcast i8* %z to <8 x half>*
+  call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %1, <8 x half>* %2, i32 2, <8 x i1> %c)
+  ret i8* %y
+}
+
+define i8* @strhf16_m254(i8* %y, i8* %x, <8 x i16> *%m) {
+; CHECK-LABEL: strhf16_m254:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q1, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q1, zr
+; CHECK-NEXT:    vstrht.16 q0, [r0, #-254]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -254
+  %0 = bitcast i8* %x to <8 x half>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = load <8 x half>, <8 x half>* %0, align 2
+  %2 = bitcast i8* %z to <8 x half>*
+  call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %1, <8 x half>* %2, i32 2, <8 x i1> %c)
+  ret i8* %y
+}
+
+define i8* @strhf16_m256(i8* %y, i8* %x, <8 x i16> *%m) {
+; CHECK-LABEL: strhf16_m256:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-NEXT:    sub.w r1, r0, #256
+; CHECK-NEXT:    vldrh.u16 q1, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q1, zr
+; CHECK-NEXT:    vstrht.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -256
+  %0 = bitcast i8* %x to <8 x half>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = load <8 x half>, <8 x half>* %0, align 2
+  %2 = bitcast i8* %z to <8 x half>*
+  call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %1, <8 x half>* %2, i32 2, <8 x i1> %c)
+  ret i8* %y
+}
+
+declare <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>*, i32, <4 x i1>, <4 x i32>)
+declare <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>*, i32, <4 x i1>, <4 x i16>)
+declare <8 x i16> @llvm.masked.load.v8i16.p0v4i16(<8 x i16>*, i32, <8 x i1>, <8 x i16>)
+declare <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>*, i32, <4 x i1>, <4 x i8>)
+declare <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>*, i32, <8 x i1>, <8 x i8>)
+declare <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>*, i32, <16 x i1>, <16 x i8>)
+declare <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>*, i32, <4 x i1>, <4 x float>)
+declare <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>*, i32, <8 x i1>, <8 x half>)
+
+declare void @llvm.masked.store.v4i32.p0v4i32(<4 x i32>, <4 x i32>*, i32, <4 x i1>)
+declare void @llvm.masked.store.v8i16.p0v8i16(<8 x i16>, <8 x i16>*, i32, <8 x i1>)
+declare void @llvm.masked.store.v4i16.p0v4i16(<4 x i16>, <4 x i16>*, i32, <4 x i1>)
+declare void @llvm.masked.store.v16i8.p0v16i8(<16 x i8>, <16 x i8>*, i32, <16 x i1>)
+declare void @llvm.masked.store.v8i8.p0v8i8(<8 x i8>, <8 x i8>*, i32, <8 x i1>)
+declare void @llvm.masked.store.v4i8.p0v4i8(<4 x i8>, <4 x i8>*, i32, <4 x i1>)
+declare void @llvm.masked.store.v4f32.p0v4f32(<4 x float>, <4 x float>*, i32, <4 x i1>)
+declare void @llvm.masked.store.v8f16.p0v8f16(<8 x half>, <8 x half>*, i32, <8 x i1>)
diff --git a/llvm/test/CodeGen/Thumb2/mve-masked-ldst-postinc.ll b/llvm/test/CodeGen/Thumb2/mve-masked-ldst-postinc.ll
new file mode 100644
index 000000000..f80372209
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-masked-ldst-postinc.ll
@@ -0,0 +1,2646 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-LE
+; RUN: llc -mtriple=thumbebv8.1m.main-arm-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-BE
+
+define i8* @ldrwu32_4(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrwu32_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrwt.u32 q0, [r0], #4
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 4
+  %0 = bitcast i8* %x to <4 x i32>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %0, i32 4, <4 x i1> %c, <4 x i32> undef)
+  %2 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %1, <4 x i32>* %2, align 4
+  ret i8* %z
+}
+
+define i8* @ldrwu32_3(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrwu32_3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrwt.u32 q0, [r0]
+; CHECK-NEXT:    adds r0, #3
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 3
+  %0 = bitcast i8* %x to <4 x i32>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %0, i32 4, <4 x i1> %c, <4 x i32> undef)
+  %2 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %1, <4 x i32>* %2, align 4
+  ret i8* %z
+}
+
+define i8* @ldrwu32_2(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrwu32_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrwt.u32 q0, [r0]
+; CHECK-NEXT:    adds r0, #2
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 2
+  %0 = bitcast i8* %x to <4 x i32>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %0, i32 4, <4 x i1> %c, <4 x i32> undef)
+  %2 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %1, <4 x i32>* %2, align 4
+  ret i8* %z
+}
+
+define i8* @ldrwu32_508(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrwu32_508:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrwt.u32 q0, [r0], #508
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 508
+  %0 = bitcast i8* %x to <4 x i32>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %0, i32 4, <4 x i1> %c, <4 x i32> undef)
+  %2 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %1, <4 x i32>* %2, align 4
+  ret i8* %z
+}
+
+define i8* @ldrwu32_512(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrwu32_512:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrwt.u32 q0, [r0]
+; CHECK-NEXT:    add.w r0, r0, #512
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 512
+  %0 = bitcast i8* %x to <4 x i32>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %0, i32 4, <4 x i1> %c, <4 x i32> undef)
+  %2 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %1, <4 x i32>* %2, align 4
+  ret i8* %z
+}
+
+define i8* @ldrwu32_m508(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrwu32_m508:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrwt.u32 q0, [r0], #-508
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -508
+  %0 = bitcast i8* %x to <4 x i32>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %0, i32 4, <4 x i1> %c, <4 x i32> undef)
+  %2 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %1, <4 x i32>* %2, align 4
+  ret i8* %z
+}
+
+define i8* @ldrwu32_m512(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrwu32_m512:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrwt.u32 q0, [r0]
+; CHECK-NEXT:    sub.w r0, r0, #512
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -512
+  %0 = bitcast i8* %x to <4 x i32>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %0, i32 4, <4 x i1> %c, <4 x i32> undef)
+  %2 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %1, <4 x i32>* %2, align 4
+  ret i8* %z
+}
+
+define i8* @ldrhu32_4(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrhu32_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrht.u32 q0, [r0], #4
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 4
+  %0 = bitcast i8* %x to <4 x i16>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %0, i32 2, <4 x i1> %c, <4 x i16> undef)
+  %2 = zext <4 x i16> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
+
+define i8* @ldrhu32_3(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrhu32_3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrht.u32 q0, [r0]
+; CHECK-NEXT:    adds r0, #3
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 3
+  %0 = bitcast i8* %x to <4 x i16>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %0, i32 2, <4 x i1> %c, <4 x i16> undef)
+  %2 = zext <4 x i16> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
+
+define i8* @ldrhu32_2(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrhu32_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrht.u32 q0, [r0], #2
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 2
+  %0 = bitcast i8* %x to <4 x i16>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %0, i32 2, <4 x i1> %c, <4 x i16> undef)
+  %2 = zext <4 x i16> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
+
+define i8* @ldrhu32_254(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrhu32_254:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrht.u32 q0, [r0], #254
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 254
+  %0 = bitcast i8* %x to <4 x i16>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %0, i32 2, <4 x i1> %c, <4 x i16> undef)
+  %2 = zext <4 x i16> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
+
+define i8* @ldrhu32_256(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrhu32_256:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrht.u32 q0, [r0]
+; CHECK-NEXT:    add.w r0, r0, #256
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 256
+  %0 = bitcast i8* %x to <4 x i16>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %0, i32 2, <4 x i1> %c, <4 x i16> undef)
+  %2 = zext <4 x i16> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
+
+define i8* @ldrhu32_m254(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrhu32_m254:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrht.u32 q0, [r0], #-254
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -254
+  %0 = bitcast i8* %x to <4 x i16>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %0, i32 2, <4 x i1> %c, <4 x i16> undef)
+  %2 = zext <4 x i16> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
+
+define i8* @ldrhu32_m256(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrhu32_m256:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrht.u32 q0, [r0]
+; CHECK-NEXT:    sub.w r0, r0, #256
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -256
+  %0 = bitcast i8* %x to <4 x i16>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %0, i32 2, <4 x i1> %c, <4 x i16> undef)
+  %2 = zext <4 x i16> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
+
+define i8* @ldrhs32_4(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrhs32_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrht.s32 q0, [r0], #4
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 4
+  %0 = bitcast i8* %x to <4 x i16>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %0, i32 2, <4 x i1> %c, <4 x i16> undef)
+  %2 = sext <4 x i16> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
+
+define i8* @ldrhs32_3(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrhs32_3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrht.s32 q0, [r0]
+; CHECK-NEXT:    adds r0, #3
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 3
+  %0 = bitcast i8* %x to <4 x i16>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %0, i32 2, <4 x i1> %c, <4 x i16> undef)
+  %2 = sext <4 x i16> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
+
+define i8* @ldrhs32_2(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrhs32_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrht.s32 q0, [r0], #2
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 2
+  %0 = bitcast i8* %x to <4 x i16>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %0, i32 2, <4 x i1> %c, <4 x i16> undef)
+  %2 = sext <4 x i16> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
+
+define i8* @ldrhs32_254(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrhs32_254:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrht.s32 q0, [r0], #254
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 254
+  %0 = bitcast i8* %x to <4 x i16>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %0, i32 2, <4 x i1> %c, <4 x i16> undef)
+  %2 = sext <4 x i16> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
+
+define i8* @ldrhs32_256(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrhs32_256:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrht.s32 q0, [r0]
+; CHECK-NEXT:    add.w r0, r0, #256
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 256
+  %0 = bitcast i8* %x to <4 x i16>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %0, i32 2, <4 x i1> %c, <4 x i16> undef)
+  %2 = sext <4 x i16> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
+
+define i8* @ldrhs32_m254(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrhs32_m254:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrht.s32 q0, [r0], #-254
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -254
+  %0 = bitcast i8* %x to <4 x i16>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %0, i32 2, <4 x i1> %c, <4 x i16> undef)
+  %2 = sext <4 x i16> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
+
+define i8* @ldrhs32_m256(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrhs32_m256:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrht.s32 q0, [r0]
+; CHECK-NEXT:    sub.w r0, r0, #256
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -256
+  %0 = bitcast i8* %x to <4 x i16>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %0, i32 2, <4 x i1> %c, <4 x i16> undef)
+  %2 = sext <4 x i16> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
+
+define i8* @ldrhu16_4(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrhu16_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrht.u16 q0, [r0], #4
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 4
+  %0 = bitcast i8* %x to <8 x i16>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %0, i32 2, <8 x i1> %c, <8 x i16> undef)
+  %2 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %1, <8 x i16>* %2, align 2
+  ret i8* %z
+}
+
+define i8* @ldrhu16_3(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrhu16_3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrht.u16 q0, [r0]
+; CHECK-NEXT:    adds r0, #3
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 3
+  %0 = bitcast i8* %x to <8 x i16>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %0, i32 2, <8 x i1> %c, <8 x i16> undef)
+  %2 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %1, <8 x i16>* %2, align 2
+  ret i8* %z
+}
+
+define i8* @ldrhu16_2(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrhu16_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrht.u16 q0, [r0], #2
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 2
+  %0 = bitcast i8* %x to <8 x i16>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %0, i32 2, <8 x i1> %c, <8 x i16> undef)
+  %2 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %1, <8 x i16>* %2, align 2
+  ret i8* %z
+}
+
+define i8* @ldrhu16_254(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrhu16_254:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrht.u16 q0, [r0], #254
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 254
+  %0 = bitcast i8* %x to <8 x i16>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %0, i32 2, <8 x i1> %c, <8 x i16> undef)
+  %2 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %1, <8 x i16>* %2, align 2
+  ret i8* %z
+}
+
+define i8* @ldrhu16_256(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrhu16_256:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrht.u16 q0, [r0]
+; CHECK-NEXT:    add.w r0, r0, #256
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 256
+  %0 = bitcast i8* %x to <8 x i16>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %0, i32 2, <8 x i1> %c, <8 x i16> undef)
+  %2 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %1, <8 x i16>* %2, align 2
+  ret i8* %z
+}
+
+define i8* @ldrhu16_m254(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrhu16_m254:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrht.u16 q0, [r0], #-254
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -254
+  %0 = bitcast i8* %x to <8 x i16>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %0, i32 2, <8 x i1> %c, <8 x i16> undef)
+  %2 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %1, <8 x i16>* %2, align 2
+  ret i8* %z
+}
+
+define i8* @ldrhu16_m256(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrhu16_m256:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrht.u16 q0, [r0]
+; CHECK-NEXT:    sub.w r0, r0, #256
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -256
+  %0 = bitcast i8* %x to <8 x i16>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %0, i32 2, <8 x i1> %c, <8 x i16> undef)
+  %2 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %1, <8 x i16>* %2, align 2
+  ret i8* %z
+}
+
+define i8* @ldrbu32_4(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrbu32_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrbt.u32 q0, [r0], #4
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 4
+  %0 = bitcast i8* %x to <4 x i8>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %0, i32 1, <4 x i1> %c, <4 x i8> undef)
+  %2 = zext <4 x i8> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
+
+define i8* @ldrbu32_3(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrbu32_3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrbt.u32 q0, [r0], #3
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 3
+  %0 = bitcast i8* %x to <4 x i8>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %0, i32 1, <4 x i1> %c, <4 x i8> undef)
+  %2 = zext <4 x i8> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
+
+define i8* @ldrbu32_2(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrbu32_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrbt.u32 q0, [r0], #2
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 2
+  %0 = bitcast i8* %x to <4 x i8>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %0, i32 1, <4 x i1> %c, <4 x i8> undef)
+  %2 = zext <4 x i8> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
+
+define i8* @ldrbu32_127(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrbu32_127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrbt.u32 q0, [r0], #127
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 127
+  %0 = bitcast i8* %x to <4 x i8>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %0, i32 1, <4 x i1> %c, <4 x i8> undef)
+  %2 = zext <4 x i8> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
+
+define i8* @ldrbu32_128(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrbu32_128:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrbt.u32 q0, [r0]
+; CHECK-NEXT:    adds r0, #128
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 128
+  %0 = bitcast i8* %x to <4 x i8>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %0, i32 1, <4 x i1> %c, <4 x i8> undef)
+  %2 = zext <4 x i8> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
+
+define i8* @ldrbu32_m127(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrbu32_m127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrbt.u32 q0, [r0], #-127
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -127
+  %0 = bitcast i8* %x to <4 x i8>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %0, i32 1, <4 x i1> %c, <4 x i8> undef)
+  %2 = zext <4 x i8> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
+
+define i8* @ldrbu32_m128(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrbu32_m128:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrbt.u32 q0, [r0]
+; CHECK-NEXT:    subs r0, #128
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -128
+  %0 = bitcast i8* %x to <4 x i8>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %0, i32 1, <4 x i1> %c, <4 x i8> undef)
+  %2 = zext <4 x i8> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
+
+define i8* @ldrbs32_4(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrbs32_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrbt.s32 q0, [r0], #4
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 4
+  %0 = bitcast i8* %x to <4 x i8>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %0, i32 1, <4 x i1> %c, <4 x i8> undef)
+  %2 = sext <4 x i8> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
+
+define i8* @ldrbs32_3(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrbs32_3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrbt.s32 q0, [r0], #3
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 3
+  %0 = bitcast i8* %x to <4 x i8>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %0, i32 1, <4 x i1> %c, <4 x i8> undef)
+  %2 = sext <4 x i8> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
+
+define i8* @ldrbs32_2(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrbs32_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrbt.s32 q0, [r0], #2
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 2
+  %0 = bitcast i8* %x to <4 x i8>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %0, i32 1, <4 x i1> %c, <4 x i8> undef)
+  %2 = sext <4 x i8> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
+
+define i8* @ldrbs32_127(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrbs32_127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrbt.s32 q0, [r0], #127
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 127
+  %0 = bitcast i8* %x to <4 x i8>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %0, i32 1, <4 x i1> %c, <4 x i8> undef)
+  %2 = sext <4 x i8> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
+
+define i8* @ldrbs32_128(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrbs32_128:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrbt.s32 q0, [r0]
+; CHECK-NEXT:    adds r0, #128
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 128
+  %0 = bitcast i8* %x to <4 x i8>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %0, i32 1, <4 x i1> %c, <4 x i8> undef)
+  %2 = sext <4 x i8> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
+
+define i8* @ldrbs32_m127(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrbs32_m127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrbt.s32 q0, [r0], #-127
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -127
+  %0 = bitcast i8* %x to <4 x i8>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %0, i32 1, <4 x i1> %c, <4 x i8> undef)
+  %2 = sext <4 x i8> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
+
+define i8* @ldrbs32_m128(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrbs32_m128:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrbt.s32 q0, [r0]
+; CHECK-NEXT:    subs r0, #128
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -128
+  %0 = bitcast i8* %x to <4 x i8>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %0, i32 1, <4 x i1> %c, <4 x i8> undef)
+  %2 = sext <4 x i8> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
+
+define i8* @ldrbu16_4(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrbu16_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrbt.u16 q0, [r0], #4
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 4
+  %0 = bitcast i8* %x to <8 x i8>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %0, i32 1, <8 x i1> %c, <8 x i8> undef)
+  %2 = zext <8 x i8> %1 to <8 x i16>
+  %3 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %2, <8 x i16>* %3, align 2
+  ret i8* %z
+}
+
+define i8* @ldrbu16_3(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrbu16_3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrbt.u16 q0, [r0], #3
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 3
+  %0 = bitcast i8* %x to <8 x i8>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %0, i32 1, <8 x i1> %c, <8 x i8> undef)
+  %2 = zext <8 x i8> %1 to <8 x i16>
+  %3 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %2, <8 x i16>* %3, align 2
+  ret i8* %z
+}
+
+define i8* @ldrbu16_2(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrbu16_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrbt.u16 q0, [r0], #2
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 2
+  %0 = bitcast i8* %x to <8 x i8>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %0, i32 1, <8 x i1> %c, <8 x i8> undef)
+  %2 = zext <8 x i8> %1 to <8 x i16>
+  %3 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %2, <8 x i16>* %3, align 2
+  ret i8* %z
+}
+
+define i8* @ldrbu16_127(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrbu16_127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrbt.u16 q0, [r0], #127
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 127
+  %0 = bitcast i8* %x to <8 x i8>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %0, i32 1, <8 x i1> %c, <8 x i8> undef)
+  %2 = zext <8 x i8> %1 to <8 x i16>
+  %3 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %2, <8 x i16>* %3, align 2
+  ret i8* %z
+}
+
+define i8* @ldrbu16_128(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrbu16_128:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrbt.u16 q0, [r0]
+; CHECK-NEXT:    adds r0, #128
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 128
+  %0 = bitcast i8* %x to <8 x i8>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %0, i32 1, <8 x i1> %c, <8 x i8> undef)
+  %2 = zext <8 x i8> %1 to <8 x i16>
+  %3 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %2, <8 x i16>* %3, align 2
+  ret i8* %z
+}
+
+define i8* @ldrbu16_m127(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrbu16_m127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrbt.u16 q0, [r0], #-127
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -127
+  %0 = bitcast i8* %x to <8 x i8>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %0, i32 1, <8 x i1> %c, <8 x i8> undef)
+  %2 = zext <8 x i8> %1 to <8 x i16>
+  %3 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %2, <8 x i16>* %3, align 2
+  ret i8* %z
+}
+
+define i8* @ldrbu16_m128(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrbu16_m128:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrbt.u16 q0, [r0]
+; CHECK-NEXT:    subs r0, #128
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -128
+  %0 = bitcast i8* %x to <8 x i8>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %0, i32 1, <8 x i1> %c, <8 x i8> undef)
+  %2 = zext <8 x i8> %1 to <8 x i16>
+  %3 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %2, <8 x i16>* %3, align 2
+  ret i8* %z
+}
+
+define i8* @ldrbs16_4(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrbs16_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrbt.s16 q0, [r0], #4
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 4
+  %0 = bitcast i8* %x to <8 x i8>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %0, i32 1, <8 x i1> %c, <8 x i8> undef)
+  %2 = sext <8 x i8> %1 to <8 x i16>
+  %3 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %2, <8 x i16>* %3, align 2
+  ret i8* %z
+}
+
+define i8* @ldrbs16_3(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrbs16_3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrbt.s16 q0, [r0], #3
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 3
+  %0 = bitcast i8* %x to <8 x i8>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %0, i32 1, <8 x i1> %c, <8 x i8> undef)
+  %2 = sext <8 x i8> %1 to <8 x i16>
+  %3 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %2, <8 x i16>* %3, align 2
+  ret i8* %z
+}
+
+define i8* @ldrbs16_2(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrbs16_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrbt.s16 q0, [r0], #2
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 2
+  %0 = bitcast i8* %x to <8 x i8>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %0, i32 1, <8 x i1> %c, <8 x i8> undef)
+  %2 = sext <8 x i8> %1 to <8 x i16>
+  %3 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %2, <8 x i16>* %3, align 2
+  ret i8* %z
+}
+
+define i8* @ldrbs16_127(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrbs16_127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrbt.s16 q0, [r0], #127
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 127
+  %0 = bitcast i8* %x to <8 x i8>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %0, i32 1, <8 x i1> %c, <8 x i8> undef)
+  %2 = sext <8 x i8> %1 to <8 x i16>
+  %3 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %2, <8 x i16>* %3, align 2
+  ret i8* %z
+}
+
+define i8* @ldrbs16_128(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrbs16_128:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrbt.s16 q0, [r0]
+; CHECK-NEXT:    adds r0, #128
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 128
+  %0 = bitcast i8* %x to <8 x i8>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %0, i32 1, <8 x i1> %c, <8 x i8> undef)
+  %2 = sext <8 x i8> %1 to <8 x i16>
+  %3 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %2, <8 x i16>* %3, align 2
+  ret i8* %z
+}
+
+define i8* @ldrbs16_m127(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrbs16_m127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrbt.s16 q0, [r0], #-127
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -127
+  %0 = bitcast i8* %x to <8 x i8>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %0, i32 1, <8 x i1> %c, <8 x i8> undef)
+  %2 = sext <8 x i8> %1 to <8 x i16>
+  %3 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %2, <8 x i16>* %3, align 2
+  ret i8* %z
+}
+
+define i8* @ldrbs16_m128(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrbs16_m128:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrbt.s16 q0, [r0]
+; CHECK-NEXT:    subs r0, #128
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -128
+  %0 = bitcast i8* %x to <8 x i8>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %0, i32 1, <8 x i1> %c, <8 x i8> undef)
+  %2 = sext <8 x i8> %1 to <8 x i16>
+  %3 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %2, <8 x i16>* %3, align 2
+  ret i8* %z
+}
+
+define i8* @ldrbu8_4(i8* %x, i8* %y, <16 x i8> *%m) {
+; CHECK-LABEL: ldrbu8_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u8 q0, [r2]
+; CHECK-NEXT:    vpt.i8 ne, q0, zr
+; CHECK-NEXT:    vldrbt.u8 q0, [r0], #4
+; CHECK-NEXT:    vstrb.8 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 4
+  %0 = bitcast i8* %x to <16 x i8>*
+  %mask = load <16 x i8>, <16 x i8>* %m, align 1
+  %c = icmp ne <16 x i8> %mask, zeroinitializer
+  %1 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %0, i32 1, <16 x i1> %c, <16 x i8> undef)
+  %2 = bitcast i8* %y to <16 x i8>*
+  store <16 x i8> %1, <16 x i8>* %2, align 1
+  ret i8* %z
+}
+
+define i8* @ldrbu8_3(i8* %x, i8* %y, <16 x i8> *%m) {
+; CHECK-LABEL: ldrbu8_3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u8 q0, [r2]
+; CHECK-NEXT:    vpt.i8 ne, q0, zr
+; CHECK-NEXT:    vldrbt.u8 q0, [r0], #3
+; CHECK-NEXT:    vstrb.8 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 3
+  %0 = bitcast i8* %x to <16 x i8>*
+  %mask = load <16 x i8>, <16 x i8>* %m, align 1
+  %c = icmp ne <16 x i8> %mask, zeroinitializer
+  %1 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %0, i32 1, <16 x i1> %c, <16 x i8> undef)
+  %2 = bitcast i8* %y to <16 x i8>*
+  store <16 x i8> %1, <16 x i8>* %2, align 1
+  ret i8* %z
+}
+
+define i8* @ldrbu8_2(i8* %x, i8* %y, <16 x i8> *%m) {
+; CHECK-LABEL: ldrbu8_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u8 q0, [r2]
+; CHECK-NEXT:    vpt.i8 ne, q0, zr
+; CHECK-NEXT:    vldrbt.u8 q0, [r0], #2
+; CHECK-NEXT:    vstrb.8 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 2
+  %0 = bitcast i8* %x to <16 x i8>*
+  %mask = load <16 x i8>, <16 x i8>* %m, align 1
+  %c = icmp ne <16 x i8> %mask, zeroinitializer
+  %1 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %0, i32 1, <16 x i1> %c, <16 x i8> undef)
+  %2 = bitcast i8* %y to <16 x i8>*
+  store <16 x i8> %1, <16 x i8>* %2, align 1
+  ret i8* %z
+}
+
+define i8* @ldrbu8_127(i8* %x, i8* %y, <16 x i8> *%m) {
+; CHECK-LABEL: ldrbu8_127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u8 q0, [r2]
+; CHECK-NEXT:    vpt.i8 ne, q0, zr
+; CHECK-NEXT:    vldrbt.u8 q0, [r0], #127
+; CHECK-NEXT:    vstrb.8 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 127
+  %0 = bitcast i8* %x to <16 x i8>*
+  %mask = load <16 x i8>, <16 x i8>* %m, align 1
+  %c = icmp ne <16 x i8> %mask, zeroinitializer
+  %1 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %0, i32 1, <16 x i1> %c, <16 x i8> undef)
+  %2 = bitcast i8* %y to <16 x i8>*
+  store <16 x i8> %1, <16 x i8>* %2, align 1
+  ret i8* %z
+}
+
+define i8* @ldrbu8_128(i8* %x, i8* %y, <16 x i8> *%m) {
+; CHECK-LABEL: ldrbu8_128:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u8 q0, [r2]
+; CHECK-NEXT:    vpt.i8 ne, q0, zr
+; CHECK-NEXT:    vldrbt.u8 q0, [r0]
+; CHECK-NEXT:    adds r0, #128
+; CHECK-NEXT:    vstrb.8 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 128
+  %0 = bitcast i8* %x to <16 x i8>*
+  %mask = load <16 x i8>, <16 x i8>* %m, align 1
+  %c = icmp ne <16 x i8> %mask, zeroinitializer
+  %1 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %0, i32 1, <16 x i1> %c, <16 x i8> undef)
+  %2 = bitcast i8* %y to <16 x i8>*
+  store <16 x i8> %1, <16 x i8>* %2, align 1
+  ret i8* %z
+}
+
+define i8* @ldrbu8_m127(i8* %x, i8* %y, <16 x i8> *%m) {
+; CHECK-LABEL: ldrbu8_m127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u8 q0, [r2]
+; CHECK-NEXT:    vpt.i8 ne, q0, zr
+; CHECK-NEXT:    vldrbt.u8 q0, [r0], #-127
+; CHECK-NEXT:    vstrb.8 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -127
+  %0 = bitcast i8* %x to <16 x i8>*
+  %mask = load <16 x i8>, <16 x i8>* %m, align 1
+  %c = icmp ne <16 x i8> %mask, zeroinitializer
+  %1 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %0, i32 1, <16 x i1> %c, <16 x i8> undef)
+  %2 = bitcast i8* %y to <16 x i8>*
+  store <16 x i8> %1, <16 x i8>* %2, align 1
+  ret i8* %z
+}
+
+define i8* @ldrbu8_m128(i8* %x, i8* %y, <16 x i8> *%m) {
+; CHECK-LABEL: ldrbu8_m128:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u8 q0, [r2]
+; CHECK-NEXT:    vpt.i8 ne, q0, zr
+; CHECK-NEXT:    vldrbt.u8 q0, [r0]
+; CHECK-NEXT:    subs r0, #128
+; CHECK-NEXT:    vstrb.8 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -128
+  %0 = bitcast i8* %x to <16 x i8>*
+  %mask = load <16 x i8>, <16 x i8>* %m, align 1
+  %c = icmp ne <16 x i8> %mask, zeroinitializer
+  %1 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %0, i32 1, <16 x i1> %c, <16 x i8> undef)
+  %2 = bitcast i8* %y to <16 x i8>*
+  store <16 x i8> %1, <16 x i8>* %2, align 1
+  ret i8* %z
+}
+
+define i8* @ldrwf32_4(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrwf32_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrwt.u32 q0, [r0], #4
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 4
+  %0 = bitcast i8* %x to <4 x float>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* %0, i32 4, <4 x i1> %c, <4 x float> undef)
+  %2 = bitcast i8* %y to <4 x float>*
+  store <4 x float> %1, <4 x float>* %2, align 4
+  ret i8* %z
+}
+
+define i8* @ldrwf32_3(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrwf32_3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrwt.u32 q0, [r0]
+; CHECK-NEXT:    adds r0, #3
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 3
+  %0 = bitcast i8* %x to <4 x float>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* %0, i32 4, <4 x i1> %c, <4 x float> undef)
+  %2 = bitcast i8* %y to <4 x float>*
+  store <4 x float> %1, <4 x float>* %2, align 4
+  ret i8* %z
+}
+
+define i8* @ldrwf32_2(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrwf32_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrwt.u32 q0, [r0]
+; CHECK-NEXT:    adds r0, #2
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 2
+  %0 = bitcast i8* %x to <4 x float>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* %0, i32 4, <4 x i1> %c, <4 x float> undef)
+  %2 = bitcast i8* %y to <4 x float>*
+  store <4 x float> %1, <4 x float>* %2, align 4
+  ret i8* %z
+}
+
+define i8* @ldrwf32_508(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrwf32_508:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrwt.u32 q0, [r0], #508
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 508
+  %0 = bitcast i8* %x to <4 x float>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* %0, i32 4, <4 x i1> %c, <4 x float> undef)
+  %2 = bitcast i8* %y to <4 x float>*
+  store <4 x float> %1, <4 x float>* %2, align 4
+  ret i8* %z
+}
+
+define i8* @ldrwf32_512(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrwf32_512:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrwt.u32 q0, [r0]
+; CHECK-NEXT:    add.w r0, r0, #512
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 512
+  %0 = bitcast i8* %x to <4 x float>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* %0, i32 4, <4 x i1> %c, <4 x float> undef)
+  %2 = bitcast i8* %y to <4 x float>*
+  store <4 x float> %1, <4 x float>* %2, align 4
+  ret i8* %z
+}
+
+define i8* @ldrwf32_m508(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrwf32_m508:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrwt.u32 q0, [r0], #-508
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -508
+  %0 = bitcast i8* %x to <4 x float>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* %0, i32 4, <4 x i1> %c, <4 x float> undef)
+  %2 = bitcast i8* %y to <4 x float>*
+  store <4 x float> %1, <4 x float>* %2, align 4
+  ret i8* %z
+}
+
+define i8* @ldrwf32_m512(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrwf32_m512:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrwt.u32 q0, [r0]
+; CHECK-NEXT:    sub.w r0, r0, #512
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -512
+  %0 = bitcast i8* %x to <4 x float>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* %0, i32 4, <4 x i1> %c, <4 x float> undef)
+  %2 = bitcast i8* %y to <4 x float>*
+  store <4 x float> %1, <4 x float>* %2, align 4
+  ret i8* %z
+}
+
+define i8* @ldrhf16_4(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrhf16_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrht.u16 q0, [r0], #4
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 4
+  %0 = bitcast i8* %x to <8 x half>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>* %0, i32 2, <8 x i1> %c, <8 x half> undef)
+  %2 = bitcast i8* %y to <8 x half>*
+  store <8 x half> %1, <8 x half>* %2, align 2
+  ret i8* %z
+}
+
+define i8* @ldrhf16_3(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrhf16_3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrht.u16 q0, [r0]
+; CHECK-NEXT:    adds r0, #3
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 3
+  %0 = bitcast i8* %x to <8 x half>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>* %0, i32 2, <8 x i1> %c, <8 x half> undef)
+  %2 = bitcast i8* %y to <8 x half>*
+  store <8 x half> %1, <8 x half>* %2, align 2
+  ret i8* %z
+}
+
+define i8* @ldrhf16_2(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrhf16_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrht.u16 q0, [r0], #2
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 2
+  %0 = bitcast i8* %x to <8 x half>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>* %0, i32 2, <8 x i1> %c, <8 x half> undef)
+  %2 = bitcast i8* %y to <8 x half>*
+  store <8 x half> %1, <8 x half>* %2, align 2
+  ret i8* %z
+}
+
+define i8* @ldrhf16_254(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrhf16_254:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrht.u16 q0, [r0], #254
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 254
+  %0 = bitcast i8* %x to <8 x half>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>* %0, i32 2, <8 x i1> %c, <8 x half> undef)
+  %2 = bitcast i8* %y to <8 x half>*
+  store <8 x half> %1, <8 x half>* %2, align 2
+  ret i8* %z
+}
+
+define i8* @ldrhf16_256(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrhf16_256:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrht.u16 q0, [r0]
+; CHECK-NEXT:    add.w r0, r0, #256
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 256
+  %0 = bitcast i8* %x to <8 x half>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>* %0, i32 2, <8 x i1> %c, <8 x half> undef)
+  %2 = bitcast i8* %y to <8 x half>*
+  store <8 x half> %1, <8 x half>* %2, align 2
+  ret i8* %z
+}
+
+define i8* @ldrhf16_m254(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrhf16_m254:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrht.u16 q0, [r0], #-254
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -254
+  %0 = bitcast i8* %x to <8 x half>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>* %0, i32 2, <8 x i1> %c, <8 x half> undef)
+  %2 = bitcast i8* %y to <8 x half>*
+  store <8 x half> %1, <8 x half>* %2, align 2
+  ret i8* %z
+}
+
+define i8* @ldrhf16_m256(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrhf16_m256:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrht.u16 q0, [r0]
+; CHECK-NEXT:    sub.w r0, r0, #256
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -256
+  %0 = bitcast i8* %x to <8 x half>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>* %0, i32 2, <8 x i1> %c, <8 x half> undef)
+  %2 = bitcast i8* %y to <8 x half>*
+  store <8 x half> %1, <8 x half>* %2, align 2
+  ret i8* %z
+}
+
+
+
+
+define i8* @strw32_4(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strw32_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrwt.32 q0, [r0], #4
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 4
+  %0 = bitcast i8* %x to <4 x i32>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
+  %2 = bitcast i8* %y to <4 x i32>*
+  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %1, <4 x i32>* %2, i32 4, <4 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strw32_3(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strw32_3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrwt.32 q0, [r0]
+; CHECK-NEXT:    adds r0, #3
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 3
+  %0 = bitcast i8* %x to <4 x i32>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
+  %2 = bitcast i8* %y to <4 x i32>*
+  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %1, <4 x i32>* %2, i32 4, <4 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strw32_2(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strw32_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrwt.32 q0, [r0]
+; CHECK-NEXT:    adds r0, #2
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 2
+  %0 = bitcast i8* %x to <4 x i32>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
+  %2 = bitcast i8* %y to <4 x i32>*
+  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %1, <4 x i32>* %2, i32 4, <4 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strw32_508(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strw32_508:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrwt.32 q0, [r0], #508
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 508
+  %0 = bitcast i8* %x to <4 x i32>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
+  %2 = bitcast i8* %y to <4 x i32>*
+  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %1, <4 x i32>* %2, i32 4, <4 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strw32_512(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strw32_512:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrwt.32 q0, [r0]
+; CHECK-NEXT:    add.w r0, r0, #512
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 512
+  %0 = bitcast i8* %x to <4 x i32>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
+  %2 = bitcast i8* %y to <4 x i32>*
+  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %1, <4 x i32>* %2, i32 4, <4 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strw32_m508(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strw32_m508:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrwt.32 q0, [r0], #-508
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -508
+  %0 = bitcast i8* %x to <4 x i32>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
+  %2 = bitcast i8* %y to <4 x i32>*
+  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %1, <4 x i32>* %2, i32 4, <4 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strw32_m512(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strw32_m512:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrwt.32 q0, [r0]
+; CHECK-NEXT:    sub.w r0, r0, #512
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -512
+  %0 = bitcast i8* %x to <4 x i32>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
+  %2 = bitcast i8* %y to <4 x i32>*
+  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %1, <4 x i32>* %2, i32 4, <4 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strh32_4(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strh32_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrht.32 q0, [r0], #4
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 4
+  %0 = bitcast i8* %x to <4 x i16>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
+  %2 = bitcast i8* %y to <4 x i16>*
+  call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %1, <4 x i16>* %2, i32 2, <4 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strh32_3(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strh32_3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrht.32 q0, [r0]
+; CHECK-NEXT:    adds r0, #3
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 3
+  %0 = bitcast i8* %x to <4 x i16>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
+  %2 = bitcast i8* %y to <4 x i16>*
+  call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %1, <4 x i16>* %2, i32 2, <4 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strh32_2(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strh32_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrht.32 q0, [r0], #2
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 2
+  %0 = bitcast i8* %x to <4 x i16>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
+  %2 = bitcast i8* %y to <4 x i16>*
+  call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %1, <4 x i16>* %2, i32 2, <4 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strh32_254(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strh32_254:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrht.32 q0, [r0], #254
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 254
+  %0 = bitcast i8* %x to <4 x i16>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
+  %2 = bitcast i8* %y to <4 x i16>*
+  call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %1, <4 x i16>* %2, i32 2, <4 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strh32_256(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strh32_256:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrht.32 q0, [r0]
+; CHECK-NEXT:    add.w r0, r0, #256
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 256
+  %0 = bitcast i8* %x to <4 x i16>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
+  %2 = bitcast i8* %y to <4 x i16>*
+  call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %1, <4 x i16>* %2, i32 2, <4 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strh32_m254(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strh32_m254:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrht.32 q0, [r0], #-254
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -254
+  %0 = bitcast i8* %x to <4 x i16>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
+  %2 = bitcast i8* %y to <4 x i16>*
+  call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %1, <4 x i16>* %2, i32 2, <4 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strh32_m256(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strh32_m256:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrht.32 q0, [r0]
+; CHECK-NEXT:    sub.w r0, r0, #256
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -256
+  %0 = bitcast i8* %x to <4 x i16>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
+  %2 = bitcast i8* %y to <4 x i16>*
+  call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %1, <4 x i16>* %2, i32 2, <4 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strh16_4(i8* %y, i8* %x, <8 x i16> *%m) {
+; CHECK-LABEL: strh16_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q1, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q1, zr
+; CHECK-NEXT:    vstrht.16 q0, [r0], #4
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 4
+  %0 = bitcast i8* %x to <8 x i16>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
+  %2 = bitcast i8* %y to <8 x i16>*
+  call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %1, <8 x i16>* %2, i32 2, <8 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strh16_3(i8* %y, i8* %x, <8 x i16> *%m) {
+; CHECK-LABEL: strh16_3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q1, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q1, zr
+; CHECK-NEXT:    vstrht.16 q0, [r0]
+; CHECK-NEXT:    adds r0, #3
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 3
+  %0 = bitcast i8* %x to <8 x i16>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
+  %2 = bitcast i8* %y to <8 x i16>*
+  call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %1, <8 x i16>* %2, i32 2, <8 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strh16_2(i8* %y, i8* %x, <8 x i16> *%m) {
+; CHECK-LABEL: strh16_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q1, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q1, zr
+; CHECK-NEXT:    vstrht.16 q0, [r0], #2
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 2
+  %0 = bitcast i8* %x to <8 x i16>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
+  %2 = bitcast i8* %y to <8 x i16>*
+  call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %1, <8 x i16>* %2, i32 2, <8 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strh16_254(i8* %y, i8* %x, <8 x i16> *%m) {
+; CHECK-LABEL: strh16_254:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q1, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q1, zr
+; CHECK-NEXT:    vstrht.16 q0, [r0], #254
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 254
+  %0 = bitcast i8* %x to <8 x i16>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
+  %2 = bitcast i8* %y to <8 x i16>*
+  call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %1, <8 x i16>* %2, i32 2, <8 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strh16_256(i8* %y, i8* %x, <8 x i16> *%m) {
+; CHECK-LABEL: strh16_256:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q1, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q1, zr
+; CHECK-NEXT:    vstrht.16 q0, [r0]
+; CHECK-NEXT:    add.w r0, r0, #256
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 256
+  %0 = bitcast i8* %x to <8 x i16>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
+  %2 = bitcast i8* %y to <8 x i16>*
+  call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %1, <8 x i16>* %2, i32 2, <8 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strh16_m254(i8* %y, i8* %x, <8 x i16> *%m) {
+; CHECK-LABEL: strh16_m254:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q1, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q1, zr
+; CHECK-NEXT:    vstrht.16 q0, [r0], #-254
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -254
+  %0 = bitcast i8* %x to <8 x i16>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
+  %2 = bitcast i8* %y to <8 x i16>*
+  call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %1, <8 x i16>* %2, i32 2, <8 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strh16_m256(i8* %y, i8* %x, <8 x i16> *%m) {
+; CHECK-LABEL: strh16_m256:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q1, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q1, zr
+; CHECK-NEXT:    vstrht.16 q0, [r0]
+; CHECK-NEXT:    sub.w r0, r0, #256
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -256
+  %0 = bitcast i8* %x to <8 x i16>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
+  %2 = bitcast i8* %y to <8 x i16>*
+  call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %1, <8 x i16>* %2, i32 2, <8 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strb32_4(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strb32_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrbt.32 q0, [r0], #4
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 4
+  %0 = bitcast i8* %x to <4 x i8>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
+  %2 = bitcast i8* %y to <4 x i8>*
+  call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %1, <4 x i8>* %2, i32 1, <4 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strb32_3(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strb32_3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrbt.32 q0, [r0], #3
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 3
+  %0 = bitcast i8* %x to <4 x i8>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
+  %2 = bitcast i8* %y to <4 x i8>*
+  call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %1, <4 x i8>* %2, i32 1, <4 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strb32_2(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strb32_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrbt.32 q0, [r0], #2
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 2
+  %0 = bitcast i8* %x to <4 x i8>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
+  %2 = bitcast i8* %y to <4 x i8>*
+  call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %1, <4 x i8>* %2, i32 1, <4 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strb32_127(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strb32_127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrbt.32 q0, [r0], #127
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 127
+  %0 = bitcast i8* %x to <4 x i8>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
+  %2 = bitcast i8* %y to <4 x i8>*
+  call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %1, <4 x i8>* %2, i32 1, <4 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strb32_128(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strb32_128:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrbt.32 q0, [r0]
+; CHECK-NEXT:    adds r0, #128
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 128
+  %0 = bitcast i8* %x to <4 x i8>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
+  %2 = bitcast i8* %y to <4 x i8>*
+  call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %1, <4 x i8>* %2, i32 1, <4 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strb32_m127(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strb32_m127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrbt.32 q0, [r0], #-127
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -127
+  %0 = bitcast i8* %x to <4 x i8>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
+  %2 = bitcast i8* %y to <4 x i8>*
+  call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %1, <4 x i8>* %2, i32 1, <4 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strb32_m128(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strb32_m128:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrbt.32 q0, [r0]
+; CHECK-NEXT:    subs r0, #128
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -128
+  %0 = bitcast i8* %x to <4 x i8>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
+  %2 = bitcast i8* %y to <4 x i8>*
+  call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %1, <4 x i8>* %2, i32 1, <4 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strb16_4(i8* %y, i8* %x, <8 x i16> *%m) {
+; CHECK-LABEL: strb16_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u16 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q1, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q1, zr
+; CHECK-NEXT:    vstrbt.16 q0, [r0], #4
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 4
+  %0 = bitcast i8* %x to <8 x i8>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
+  %2 = bitcast i8* %y to <8 x i8>*
+  call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %1, <8 x i8>* %2, i32 1, <8 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strb16_3(i8* %y, i8* %x, <8 x i16> *%m) {
+; CHECK-LABEL: strb16_3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u16 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q1, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q1, zr
+; CHECK-NEXT:    vstrbt.16 q0, [r0], #3
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 3
+  %0 = bitcast i8* %x to <8 x i8>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
+  %2 = bitcast i8* %y to <8 x i8>*
+  call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %1, <8 x i8>* %2, i32 1, <8 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strb16_2(i8* %y, i8* %x, <8 x i16> *%m) {
+; CHECK-LABEL: strb16_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u16 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q1, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q1, zr
+; CHECK-NEXT:    vstrbt.16 q0, [r0], #2
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 2
+  %0 = bitcast i8* %x to <8 x i8>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
+  %2 = bitcast i8* %y to <8 x i8>*
+  call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %1, <8 x i8>* %2, i32 1, <8 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strb16_127(i8* %y, i8* %x, <8 x i16> *%m) {
+; CHECK-LABEL: strb16_127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u16 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q1, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q1, zr
+; CHECK-NEXT:    vstrbt.16 q0, [r0], #127
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 127
+  %0 = bitcast i8* %x to <8 x i8>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
+  %2 = bitcast i8* %y to <8 x i8>*
+  call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %1, <8 x i8>* %2, i32 1, <8 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strb16_128(i8* %y, i8* %x, <8 x i16> *%m) {
+; CHECK-LABEL: strb16_128:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u16 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q1, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q1, zr
+; CHECK-NEXT:    vstrbt.16 q0, [r0]
+; CHECK-NEXT:    adds r0, #128
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 128
+  %0 = bitcast i8* %x to <8 x i8>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
+  %2 = bitcast i8* %y to <8 x i8>*
+  call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %1, <8 x i8>* %2, i32 1, <8 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strb16_m127(i8* %y, i8* %x, <8 x i16> *%m) {
+; CHECK-LABEL: strb16_m127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u16 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q1, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q1, zr
+; CHECK-NEXT:    vstrbt.16 q0, [r0], #-127
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -127
+  %0 = bitcast i8* %x to <8 x i8>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
+  %2 = bitcast i8* %y to <8 x i8>*
+  call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %1, <8 x i8>* %2, i32 1, <8 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strb16_m128(i8* %y, i8* %x, <8 x i16> *%m) {
+; CHECK-LABEL: strb16_m128:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u16 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q1, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q1, zr
+; CHECK-NEXT:    vstrbt.16 q0, [r0]
+; CHECK-NEXT:    subs r0, #128
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -128
+  %0 = bitcast i8* %x to <8 x i8>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
+  %2 = bitcast i8* %y to <8 x i8>*
+  call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %1, <8 x i8>* %2, i32 1, <8 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strb8_4(i8* %y, i8* %x, <16 x i8> *%m) {
+; CHECK-LABEL: strb8_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u8 q0, [r1]
+; CHECK-NEXT:    vldrb.u8 q1, [r2]
+; CHECK-NEXT:    vpt.i8 ne, q1, zr
+; CHECK-NEXT:    vstrbt.8 q0, [r0], #4
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 4
+  %0 = bitcast i8* %x to <16 x i8>*
+  %mask = load <16 x i8>, <16 x i8>* %m, align 1
+  %c = icmp ne <16 x i8> %mask, zeroinitializer
+  %1 = load <16 x i8>, <16 x i8>* %0, align 1
+  %2 = bitcast i8* %y to <16 x i8>*
+  call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %1, <16 x i8>* %2, i32 1, <16 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strb8_3(i8* %y, i8* %x, <16 x i8> *%m) {
+; CHECK-LABEL: strb8_3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u8 q0, [r1]
+; CHECK-NEXT:    vldrb.u8 q1, [r2]
+; CHECK-NEXT:    vpt.i8 ne, q1, zr
+; CHECK-NEXT:    vstrbt.8 q0, [r0], #3
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 3
+  %0 = bitcast i8* %x to <16 x i8>*
+  %mask = load <16 x i8>, <16 x i8>* %m, align 1
+  %c = icmp ne <16 x i8> %mask, zeroinitializer
+  %1 = load <16 x i8>, <16 x i8>* %0, align 1
+  %2 = bitcast i8* %y to <16 x i8>*
+  call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %1, <16 x i8>* %2, i32 1, <16 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strb8_2(i8* %y, i8* %x, <16 x i8> *%m) {
+; CHECK-LABEL: strb8_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u8 q0, [r1]
+; CHECK-NEXT:    vldrb.u8 q1, [r2]
+; CHECK-NEXT:    vpt.i8 ne, q1, zr
+; CHECK-NEXT:    vstrbt.8 q0, [r0], #2
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 2
+  %0 = bitcast i8* %x to <16 x i8>*
+  %mask = load <16 x i8>, <16 x i8>* %m, align 1
+  %c = icmp ne <16 x i8> %mask, zeroinitializer
+  %1 = load <16 x i8>, <16 x i8>* %0, align 1
+  %2 = bitcast i8* %y to <16 x i8>*
+  call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %1, <16 x i8>* %2, i32 1, <16 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strb8_127(i8* %y, i8* %x, <16 x i8> *%m) {
+; CHECK-LABEL: strb8_127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u8 q0, [r1]
+; CHECK-NEXT:    vldrb.u8 q1, [r2]
+; CHECK-NEXT:    vpt.i8 ne, q1, zr
+; CHECK-NEXT:    vstrbt.8 q0, [r0], #127
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 127
+  %0 = bitcast i8* %x to <16 x i8>*
+  %mask = load <16 x i8>, <16 x i8>* %m, align 1
+  %c = icmp ne <16 x i8> %mask, zeroinitializer
+  %1 = load <16 x i8>, <16 x i8>* %0, align 1
+  %2 = bitcast i8* %y to <16 x i8>*
+  call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %1, <16 x i8>* %2, i32 1, <16 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strb8_128(i8* %y, i8* %x, <16 x i8> *%m) {
+; CHECK-LABEL: strb8_128:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u8 q0, [r1]
+; CHECK-NEXT:    vldrb.u8 q1, [r2]
+; CHECK-NEXT:    vpt.i8 ne, q1, zr
+; CHECK-NEXT:    vstrbt.8 q0, [r0]
+; CHECK-NEXT:    adds r0, #128
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 128
+  %0 = bitcast i8* %x to <16 x i8>*
+  %mask = load <16 x i8>, <16 x i8>* %m, align 1
+  %c = icmp ne <16 x i8> %mask, zeroinitializer
+  %1 = load <16 x i8>, <16 x i8>* %0, align 1
+  %2 = bitcast i8* %y to <16 x i8>*
+  call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %1, <16 x i8>* %2, i32 1, <16 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strb8_m127(i8* %y, i8* %x, <16 x i8> *%m) {
+; CHECK-LABEL: strb8_m127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u8 q0, [r1]
+; CHECK-NEXT:    vldrb.u8 q1, [r2]
+; CHECK-NEXT:    vpt.i8 ne, q1, zr
+; CHECK-NEXT:    vstrbt.8 q0, [r0], #-127
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -127
+  %0 = bitcast i8* %x to <16 x i8>*
+  %mask = load <16 x i8>, <16 x i8>* %m, align 1
+  %c = icmp ne <16 x i8> %mask, zeroinitializer
+  %1 = load <16 x i8>, <16 x i8>* %0, align 1
+  %2 = bitcast i8* %y to <16 x i8>*
+  call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %1, <16 x i8>* %2, i32 1, <16 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strb8_m128(i8* %y, i8* %x, <16 x i8> *%m) {
+; CHECK-LABEL: strb8_m128:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u8 q0, [r1]
+; CHECK-NEXT:    vldrb.u8 q1, [r2]
+; CHECK-NEXT:    vpt.i8 ne, q1, zr
+; CHECK-NEXT:    vstrbt.8 q0, [r0]
+; CHECK-NEXT:    subs r0, #128
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -128
+  %0 = bitcast i8* %x to <16 x i8>*
+  %mask = load <16 x i8>, <16 x i8>* %m, align 1
+  %c = icmp ne <16 x i8> %mask, zeroinitializer
+  %1 = load <16 x i8>, <16 x i8>* %0, align 1
+  %2 = bitcast i8* %y to <16 x i8>*
+  call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %1, <16 x i8>* %2, i32 1, <16 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strwf32_4(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strwf32_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrwt.32 q0, [r0], #4
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 4
+  %0 = bitcast i8* %x to <4 x float>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x float>, <4 x float>* %0, align 4
+  %2 = bitcast i8* %y to <4 x float>*
+  call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %1, <4 x float>* %2, i32 4, <4 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strwf32_3(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strwf32_3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrwt.32 q0, [r0]
+; CHECK-NEXT:    adds r0, #3
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 3
+  %0 = bitcast i8* %x to <4 x float>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x float>, <4 x float>* %0, align 4
+  %2 = bitcast i8* %y to <4 x float>*
+  call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %1, <4 x float>* %2, i32 4, <4 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strwf32_2(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strwf32_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrwt.32 q0, [r0]
+; CHECK-NEXT:    adds r0, #2
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 2
+  %0 = bitcast i8* %x to <4 x float>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x float>, <4 x float>* %0, align 4
+  %2 = bitcast i8* %y to <4 x float>*
+  call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %1, <4 x float>* %2, i32 4, <4 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strwf32_508(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strwf32_508:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrwt.32 q0, [r0], #508
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 508
+  %0 = bitcast i8* %x to <4 x float>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x float>, <4 x float>* %0, align 4
+  %2 = bitcast i8* %y to <4 x float>*
+  call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %1, <4 x float>* %2, i32 4, <4 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strwf32_512(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strwf32_512:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrwt.32 q0, [r0]
+; CHECK-NEXT:    add.w r0, r0, #512
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 512
+  %0 = bitcast i8* %x to <4 x float>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x float>, <4 x float>* %0, align 4
+  %2 = bitcast i8* %y to <4 x float>*
+  call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %1, <4 x float>* %2, i32 4, <4 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strwf32_m508(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strwf32_m508:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrwt.32 q0, [r0], #-508
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -508
+  %0 = bitcast i8* %x to <4 x float>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x float>, <4 x float>* %0, align 4
+  %2 = bitcast i8* %y to <4 x float>*
+  call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %1, <4 x float>* %2, i32 4, <4 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strwf32_m512(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strwf32_m512:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrwt.32 q0, [r0]
+; CHECK-NEXT:    sub.w r0, r0, #512
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -512
+  %0 = bitcast i8* %x to <4 x float>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x float>, <4 x float>* %0, align 4
+  %2 = bitcast i8* %y to <4 x float>*
+  call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %1, <4 x float>* %2, i32 4, <4 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strhf16_4(i8* %y, i8* %x, <8 x i16> *%m) {
+; CHECK-LABEL: strhf16_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q1, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q1, zr
+; CHECK-NEXT:    vstrht.16 q0, [r0], #4
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 4
+  %0 = bitcast i8* %x to <8 x half>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = load <8 x half>, <8 x half>* %0, align 2
+  %2 = bitcast i8* %y to <8 x half>*
+  call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %1, <8 x half>* %2, i32 2, <8 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strhf16_3(i8* %y, i8* %x, <8 x i16> *%m) {
+; CHECK-LABEL: strhf16_3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q1, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q1, zr
+; CHECK-NEXT:    vstrht.16 q0, [r0]
+; CHECK-NEXT:    adds r0, #3
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 3
+  %0 = bitcast i8* %x to <8 x half>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = load <8 x half>, <8 x half>* %0, align 2
+  %2 = bitcast i8* %y to <8 x half>*
+  call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %1, <8 x half>* %2, i32 2, <8 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strhf16_2(i8* %y, i8* %x, <8 x i16> *%m) {
+; CHECK-LABEL: strhf16_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q1, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q1, zr
+; CHECK-NEXT:    vstrht.16 q0, [r0], #2
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 2
+  %0 = bitcast i8* %x to <8 x half>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = load <8 x half>, <8 x half>* %0, align 2
+  %2 = bitcast i8* %y to <8 x half>*
+  call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %1, <8 x half>* %2, i32 2, <8 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strhf16_254(i8* %y, i8* %x, <8 x i16> *%m) {
+; CHECK-LABEL: strhf16_254:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q1, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q1, zr
+; CHECK-NEXT:    vstrht.16 q0, [r0], #254
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 254
+  %0 = bitcast i8* %x to <8 x half>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = load <8 x half>, <8 x half>* %0, align 2
+  %2 = bitcast i8* %y to <8 x half>*
+  call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %1, <8 x half>* %2, i32 2, <8 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strhf16_256(i8* %y, i8* %x, <8 x i16> *%m) {
+; CHECK-LABEL: strhf16_256:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q1, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q1, zr
+; CHECK-NEXT:    vstrht.16 q0, [r0]
+; CHECK-NEXT:    add.w r0, r0, #256
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 256
+  %0 = bitcast i8* %x to <8 x half>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = load <8 x half>, <8 x half>* %0, align 2
+  %2 = bitcast i8* %y to <8 x half>*
+  call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %1, <8 x half>* %2, i32 2, <8 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strhf16_m254(i8* %y, i8* %x, <8 x i16> *%m) {
+; CHECK-LABEL: strhf16_m254:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q1, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q1, zr
+; CHECK-NEXT:    vstrht.16 q0, [r0], #-254
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -254
+  %0 = bitcast i8* %x to <8 x half>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = load <8 x half>, <8 x half>* %0, align 2
+  %2 = bitcast i8* %y to <8 x half>*
+  call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %1, <8 x half>* %2, i32 2, <8 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strhf16_m256(i8* %y, i8* %x, <8 x i16> *%m) {
+; CHECK-LABEL: strhf16_m256:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q1, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q1, zr
+; CHECK-NEXT:    vstrht.16 q0, [r0]
+; CHECK-NEXT:    sub.w r0, r0, #256
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -256
+  %0 = bitcast i8* %x to <8 x half>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = load <8 x half>, <8 x half>* %0, align 2
+  %2 = bitcast i8* %y to <8 x half>*
+  call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %1, <8 x half>* %2, i32 2, <8 x i1> %c)
+  ret i8* %z
+}
+
+declare <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>*, i32, <4 x i1>, <4 x i32>)
+declare <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>*, i32, <4 x i1>, <4 x i16>)
+declare <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>*, i32, <8 x i1>, <8 x i16>)
+declare <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>*, i32, <4 x i1>, <4 x i8>)
+declare <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>*, i32, <8 x i1>, <8 x i8>)
+declare <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>*, i32, <16 x i1>, <16 x i8>)
+declare <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>*, i32, <4 x i1>, <4 x float>)
+declare <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>*, i32, <8 x i1>, <8 x half>)
+
+declare void @llvm.masked.store.v4i32.p0v4i32(<4 x i32>, <4 x i32>*, i32, <4 x i1>)
+declare void @llvm.masked.store.v8i16.p0v8i16(<8 x i16>, <8 x i16>*, i32, <8 x i1>)
+declare void @llvm.masked.store.v4i16.p0v4i16(<4 x i16>, <4 x i16>*, i32, <4 x i1>)
+declare void @llvm.masked.store.v16i8.p0v16i8(<16 x i8>, <16 x i8>*, i32, <16 x i1>)
+declare void @llvm.masked.store.v8i8.p0v8i8(<8 x i8>, <8 x i8>*, i32, <8 x i1>)
+declare void @llvm.masked.store.v4i8.p0v4i8(<4 x i8>, <4 x i8>*, i32, <4 x i1>)
+declare void @llvm.masked.store.v4f32.p0v4f32(<4 x float>, <4 x float>*, i32, <4 x i1>)
+declare void @llvm.masked.store.v8f16.p0v8f16(<8 x half>, <8 x half>*, i32, <8 x i1>)
diff --git a/llvm/test/CodeGen/Thumb2/mve-masked-ldst-preinc.ll b/llvm/test/CodeGen/Thumb2/mve-masked-ldst-preinc.ll
new file mode 100644
index 000000000..424e16d22
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-masked-ldst-preinc.ll
@@ -0,0 +1,2646 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-LE
+; RUN: llc -mtriple=thumbebv8.1m.main-arm-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-BE
+
+define i8* @ldrwu32_4(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrwu32_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrwt.u32 q0, [r0, #4]!
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 4
+  %0 = bitcast i8* %z to <4 x i32>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %0, i32 4, <4 x i1> %c, <4 x i32> undef)
+  %2 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %1, <4 x i32>* %2, align 4
+  ret i8* %z
+}
+
+define i8* @ldrwu32_3(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrwu32_3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    adds r0, #3
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrwt.u32 q0, [r0]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 3
+  %0 = bitcast i8* %z to <4 x i32>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %0, i32 4, <4 x i1> %c, <4 x i32> undef)
+  %2 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %1, <4 x i32>* %2, align 4
+  ret i8* %z
+}
+
+define i8* @ldrwu32_2(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrwu32_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    adds r0, #2
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrwt.u32 q0, [r0]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 2
+  %0 = bitcast i8* %z to <4 x i32>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %0, i32 4, <4 x i1> %c, <4 x i32> undef)
+  %2 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %1, <4 x i32>* %2, align 4
+  ret i8* %z
+}
+
+define i8* @ldrwu32_508(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrwu32_508:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrwt.u32 q0, [r0, #508]!
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 508
+  %0 = bitcast i8* %z to <4 x i32>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %0, i32 4, <4 x i1> %c, <4 x i32> undef)
+  %2 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %1, <4 x i32>* %2, align 4
+  ret i8* %z
+}
+
+define i8* @ldrwu32_512(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrwu32_512:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    add.w r0, r0, #512
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrwt.u32 q0, [r0]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 512
+  %0 = bitcast i8* %z to <4 x i32>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %0, i32 4, <4 x i1> %c, <4 x i32> undef)
+  %2 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %1, <4 x i32>* %2, align 4
+  ret i8* %z
+}
+
+define i8* @ldrwu32_m508(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrwu32_m508:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrwt.u32 q0, [r0, #-508]!
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -508
+  %0 = bitcast i8* %z to <4 x i32>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %0, i32 4, <4 x i1> %c, <4 x i32> undef)
+  %2 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %1, <4 x i32>* %2, align 4
+  ret i8* %z
+}
+
+define i8* @ldrwu32_m512(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrwu32_m512:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    sub.w r0, r0, #512
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrwt.u32 q0, [r0]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -512
+  %0 = bitcast i8* %z to <4 x i32>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %0, i32 4, <4 x i1> %c, <4 x i32> undef)
+  %2 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %1, <4 x i32>* %2, align 4
+  ret i8* %z
+}
+
+define i8* @ldrhu32_4(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrhu32_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrht.u32 q0, [r0, #4]!
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 4
+  %0 = bitcast i8* %z to <4 x i16>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %0, i32 2, <4 x i1> %c, <4 x i16> undef)
+  %2 = zext <4 x i16> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
+
+define i8* @ldrhu32_3(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrhu32_3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    adds r0, #3
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrht.u32 q0, [r0]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 3
+  %0 = bitcast i8* %z to <4 x i16>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %0, i32 2, <4 x i1> %c, <4 x i16> undef)
+  %2 = zext <4 x i16> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
+
+define i8* @ldrhu32_2(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrhu32_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrht.u32 q0, [r0, #2]!
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 2
+  %0 = bitcast i8* %z to <4 x i16>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %0, i32 2, <4 x i1> %c, <4 x i16> undef)
+  %2 = zext <4 x i16> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
+
+define i8* @ldrhu32_254(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrhu32_254:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrht.u32 q0, [r0, #254]!
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 254
+  %0 = bitcast i8* %z to <4 x i16>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %0, i32 2, <4 x i1> %c, <4 x i16> undef)
+  %2 = zext <4 x i16> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
+
+define i8* @ldrhu32_256(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrhu32_256:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    add.w r0, r0, #256
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrht.u32 q0, [r0]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 256
+  %0 = bitcast i8* %z to <4 x i16>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %0, i32 2, <4 x i1> %c, <4 x i16> undef)
+  %2 = zext <4 x i16> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
+
+define i8* @ldrhu32_m254(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrhu32_m254:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrht.u32 q0, [r0, #-254]!
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -254
+  %0 = bitcast i8* %z to <4 x i16>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %0, i32 2, <4 x i1> %c, <4 x i16> undef)
+  %2 = zext <4 x i16> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
+
+define i8* @ldrhu32_m256(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrhu32_m256:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    sub.w r0, r0, #256
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrht.u32 q0, [r0]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -256
+  %0 = bitcast i8* %z to <4 x i16>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %0, i32 2, <4 x i1> %c, <4 x i16> undef)
+  %2 = zext <4 x i16> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
+
+define i8* @ldrhs32_4(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrhs32_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrht.s32 q0, [r0, #4]!
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 4
+  %0 = bitcast i8* %z to <4 x i16>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %0, i32 2, <4 x i1> %c, <4 x i16> undef)
+  %2 = sext <4 x i16> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
+
+define i8* @ldrhs32_3(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrhs32_3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    adds r0, #3
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrht.s32 q0, [r0]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 3
+  %0 = bitcast i8* %z to <4 x i16>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %0, i32 2, <4 x i1> %c, <4 x i16> undef)
+  %2 = sext <4 x i16> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
+
+define i8* @ldrhs32_2(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrhs32_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrht.s32 q0, [r0, #2]!
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 2
+  %0 = bitcast i8* %z to <4 x i16>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %0, i32 2, <4 x i1> %c, <4 x i16> undef)
+  %2 = sext <4 x i16> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
+
+define i8* @ldrhs32_254(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrhs32_254:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrht.s32 q0, [r0, #254]!
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 254
+  %0 = bitcast i8* %z to <4 x i16>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %0, i32 2, <4 x i1> %c, <4 x i16> undef)
+  %2 = sext <4 x i16> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
+
+define i8* @ldrhs32_256(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrhs32_256:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    add.w r0, r0, #256
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrht.s32 q0, [r0]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 256
+  %0 = bitcast i8* %z to <4 x i16>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %0, i32 2, <4 x i1> %c, <4 x i16> undef)
+  %2 = sext <4 x i16> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
+
+define i8* @ldrhs32_m254(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrhs32_m254:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrht.s32 q0, [r0, #-254]!
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -254
+  %0 = bitcast i8* %z to <4 x i16>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %0, i32 2, <4 x i1> %c, <4 x i16> undef)
+  %2 = sext <4 x i16> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
+
+define i8* @ldrhs32_m256(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrhs32_m256:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    sub.w r0, r0, #256
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrht.s32 q0, [r0]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -256
+  %0 = bitcast i8* %z to <4 x i16>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %0, i32 2, <4 x i1> %c, <4 x i16> undef)
+  %2 = sext <4 x i16> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
+
+define i8* @ldrhu16_4(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrhu16_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrht.u16 q0, [r0, #4]!
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 4
+  %0 = bitcast i8* %z to <8 x i16>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %0, i32 2, <8 x i1> %c, <8 x i16> undef)
+  %2 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %1, <8 x i16>* %2, align 2
+  ret i8* %z
+}
+
+define i8* @ldrhu16_3(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrhu16_3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    adds r0, #3
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrht.u16 q0, [r0]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 3
+  %0 = bitcast i8* %z to <8 x i16>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %0, i32 2, <8 x i1> %c, <8 x i16> undef)
+  %2 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %1, <8 x i16>* %2, align 2
+  ret i8* %z
+}
+
+define i8* @ldrhu16_2(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrhu16_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrht.u16 q0, [r0, #2]!
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 2
+  %0 = bitcast i8* %z to <8 x i16>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %0, i32 2, <8 x i1> %c, <8 x i16> undef)
+  %2 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %1, <8 x i16>* %2, align 2
+  ret i8* %z
+}
+
+define i8* @ldrhu16_254(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrhu16_254:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrht.u16 q0, [r0, #254]!
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 254
+  %0 = bitcast i8* %z to <8 x i16>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %0, i32 2, <8 x i1> %c, <8 x i16> undef)
+  %2 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %1, <8 x i16>* %2, align 2
+  ret i8* %z
+}
+
+define i8* @ldrhu16_256(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrhu16_256:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    add.w r0, r0, #256
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrht.u16 q0, [r0]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 256
+  %0 = bitcast i8* %z to <8 x i16>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %0, i32 2, <8 x i1> %c, <8 x i16> undef)
+  %2 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %1, <8 x i16>* %2, align 2
+  ret i8* %z
+}
+
+define i8* @ldrhu16_m254(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrhu16_m254:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrht.u16 q0, [r0, #-254]!
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -254
+  %0 = bitcast i8* %z to <8 x i16>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %0, i32 2, <8 x i1> %c, <8 x i16> undef)
+  %2 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %1, <8 x i16>* %2, align 2
+  ret i8* %z
+}
+
+define i8* @ldrhu16_m256(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrhu16_m256:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    sub.w r0, r0, #256
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrht.u16 q0, [r0]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -256
+  %0 = bitcast i8* %z to <8 x i16>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %0, i32 2, <8 x i1> %c, <8 x i16> undef)
+  %2 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %1, <8 x i16>* %2, align 2
+  ret i8* %z
+}
+
+define i8* @ldrbu32_4(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrbu32_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrbt.u32 q0, [r0, #4]!
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 4
+  %0 = bitcast i8* %z to <4 x i8>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %0, i32 1, <4 x i1> %c, <4 x i8> undef)
+  %2 = zext <4 x i8> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
+
+define i8* @ldrbu32_3(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrbu32_3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrbt.u32 q0, [r0, #3]!
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 3
+  %0 = bitcast i8* %z to <4 x i8>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %0, i32 1, <4 x i1> %c, <4 x i8> undef)
+  %2 = zext <4 x i8> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
+
+define i8* @ldrbu32_2(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrbu32_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrbt.u32 q0, [r0, #2]!
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 2
+  %0 = bitcast i8* %z to <4 x i8>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %0, i32 1, <4 x i1> %c, <4 x i8> undef)
+  %2 = zext <4 x i8> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
+
+define i8* @ldrbu32_127(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrbu32_127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrbt.u32 q0, [r0, #127]!
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 127
+  %0 = bitcast i8* %z to <4 x i8>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %0, i32 1, <4 x i1> %c, <4 x i8> undef)
+  %2 = zext <4 x i8> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
+
+define i8* @ldrbu32_128(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrbu32_128:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    adds r0, #128
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrbt.u32 q0, [r0]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 128
+  %0 = bitcast i8* %z to <4 x i8>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %0, i32 1, <4 x i1> %c, <4 x i8> undef)
+  %2 = zext <4 x i8> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
+
+define i8* @ldrbu32_m127(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrbu32_m127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrbt.u32 q0, [r0, #-127]!
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -127
+  %0 = bitcast i8* %z to <4 x i8>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %0, i32 1, <4 x i1> %c, <4 x i8> undef)
+  %2 = zext <4 x i8> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
+
+define i8* @ldrbu32_m128(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrbu32_m128:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    subs r0, #128
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrbt.u32 q0, [r0]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -128
+  %0 = bitcast i8* %z to <4 x i8>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %0, i32 1, <4 x i1> %c, <4 x i8> undef)
+  %2 = zext <4 x i8> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
+
+define i8* @ldrbs32_4(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrbs32_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrbt.s32 q0, [r0, #4]!
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 4
+  %0 = bitcast i8* %z to <4 x i8>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %0, i32 1, <4 x i1> %c, <4 x i8> undef)
+  %2 = sext <4 x i8> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
+
+define i8* @ldrbs32_3(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrbs32_3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrbt.s32 q0, [r0, #3]!
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 3
+  %0 = bitcast i8* %z to <4 x i8>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %0, i32 1, <4 x i1> %c, <4 x i8> undef)
+  %2 = sext <4 x i8> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
+
+define i8* @ldrbs32_2(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrbs32_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrbt.s32 q0, [r0, #2]!
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 2
+  %0 = bitcast i8* %z to <4 x i8>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %0, i32 1, <4 x i1> %c, <4 x i8> undef)
+  %2 = sext <4 x i8> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
+
+define i8* @ldrbs32_127(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrbs32_127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrbt.s32 q0, [r0, #127]!
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 127
+  %0 = bitcast i8* %z to <4 x i8>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %0, i32 1, <4 x i1> %c, <4 x i8> undef)
+  %2 = sext <4 x i8> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
+
+define i8* @ldrbs32_128(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrbs32_128:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    adds r0, #128
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrbt.s32 q0, [r0]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 128
+  %0 = bitcast i8* %z to <4 x i8>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %0, i32 1, <4 x i1> %c, <4 x i8> undef)
+  %2 = sext <4 x i8> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
+
+define i8* @ldrbs32_m127(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrbs32_m127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrbt.s32 q0, [r0, #-127]!
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -127
+  %0 = bitcast i8* %z to <4 x i8>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %0, i32 1, <4 x i1> %c, <4 x i8> undef)
+  %2 = sext <4 x i8> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
+
+define i8* @ldrbs32_m128(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrbs32_m128:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    subs r0, #128
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrbt.s32 q0, [r0]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -128
+  %0 = bitcast i8* %z to <4 x i8>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %0, i32 1, <4 x i1> %c, <4 x i8> undef)
+  %2 = sext <4 x i8> %1 to <4 x i32>
+  %3 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %3, align 4
+  ret i8* %z
+}
+
+define i8* @ldrbu16_4(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrbu16_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrbt.u16 q0, [r0, #4]!
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 4
+  %0 = bitcast i8* %z to <8 x i8>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %0, i32 1, <8 x i1> %c, <8 x i8> undef)
+  %2 = zext <8 x i8> %1 to <8 x i16>
+  %3 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %2, <8 x i16>* %3, align 2
+  ret i8* %z
+}
+
+define i8* @ldrbu16_3(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrbu16_3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrbt.u16 q0, [r0, #3]!
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 3
+  %0 = bitcast i8* %z to <8 x i8>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %0, i32 1, <8 x i1> %c, <8 x i8> undef)
+  %2 = zext <8 x i8> %1 to <8 x i16>
+  %3 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %2, <8 x i16>* %3, align 2
+  ret i8* %z
+}
+
+define i8* @ldrbu16_2(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrbu16_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrbt.u16 q0, [r0, #2]!
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 2
+  %0 = bitcast i8* %z to <8 x i8>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %0, i32 1, <8 x i1> %c, <8 x i8> undef)
+  %2 = zext <8 x i8> %1 to <8 x i16>
+  %3 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %2, <8 x i16>* %3, align 2
+  ret i8* %z
+}
+
+define i8* @ldrbu16_127(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrbu16_127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrbt.u16 q0, [r0, #127]!
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 127
+  %0 = bitcast i8* %z to <8 x i8>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %0, i32 1, <8 x i1> %c, <8 x i8> undef)
+  %2 = zext <8 x i8> %1 to <8 x i16>
+  %3 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %2, <8 x i16>* %3, align 2
+  ret i8* %z
+}
+
+define i8* @ldrbu16_128(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrbu16_128:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    adds r0, #128
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrbt.u16 q0, [r0]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 128
+  %0 = bitcast i8* %z to <8 x i8>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %0, i32 1, <8 x i1> %c, <8 x i8> undef)
+  %2 = zext <8 x i8> %1 to <8 x i16>
+  %3 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %2, <8 x i16>* %3, align 2
+  ret i8* %z
+}
+
+define i8* @ldrbu16_m127(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrbu16_m127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrbt.u16 q0, [r0, #-127]!
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -127
+  %0 = bitcast i8* %z to <8 x i8>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %0, i32 1, <8 x i1> %c, <8 x i8> undef)
+  %2 = zext <8 x i8> %1 to <8 x i16>
+  %3 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %2, <8 x i16>* %3, align 2
+  ret i8* %z
+}
+
+define i8* @ldrbu16_m128(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrbu16_m128:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    subs r0, #128
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrbt.u16 q0, [r0]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -128
+  %0 = bitcast i8* %z to <8 x i8>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %0, i32 1, <8 x i1> %c, <8 x i8> undef)
+  %2 = zext <8 x i8> %1 to <8 x i16>
+  %3 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %2, <8 x i16>* %3, align 2
+  ret i8* %z
+}
+
+define i8* @ldrbs16_4(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrbs16_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrbt.s16 q0, [r0, #4]!
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 4
+  %0 = bitcast i8* %z to <8 x i8>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %0, i32 1, <8 x i1> %c, <8 x i8> undef)
+  %2 = sext <8 x i8> %1 to <8 x i16>
+  %3 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %2, <8 x i16>* %3, align 2
+  ret i8* %z
+}
+
+define i8* @ldrbs16_3(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrbs16_3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrbt.s16 q0, [r0, #3]!
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 3
+  %0 = bitcast i8* %z to <8 x i8>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %0, i32 1, <8 x i1> %c, <8 x i8> undef)
+  %2 = sext <8 x i8> %1 to <8 x i16>
+  %3 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %2, <8 x i16>* %3, align 2
+  ret i8* %z
+}
+
+define i8* @ldrbs16_2(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrbs16_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrbt.s16 q0, [r0, #2]!
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 2
+  %0 = bitcast i8* %z to <8 x i8>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %0, i32 1, <8 x i1> %c, <8 x i8> undef)
+  %2 = sext <8 x i8> %1 to <8 x i16>
+  %3 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %2, <8 x i16>* %3, align 2
+  ret i8* %z
+}
+
+define i8* @ldrbs16_127(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrbs16_127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrbt.s16 q0, [r0, #127]!
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 127
+  %0 = bitcast i8* %z to <8 x i8>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %0, i32 1, <8 x i1> %c, <8 x i8> undef)
+  %2 = sext <8 x i8> %1 to <8 x i16>
+  %3 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %2, <8 x i16>* %3, align 2
+  ret i8* %z
+}
+
+define i8* @ldrbs16_128(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrbs16_128:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    adds r0, #128
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrbt.s16 q0, [r0]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 128
+  %0 = bitcast i8* %z to <8 x i8>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %0, i32 1, <8 x i1> %c, <8 x i8> undef)
+  %2 = sext <8 x i8> %1 to <8 x i16>
+  %3 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %2, <8 x i16>* %3, align 2
+  ret i8* %z
+}
+
+define i8* @ldrbs16_m127(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrbs16_m127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrbt.s16 q0, [r0, #-127]!
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -127
+  %0 = bitcast i8* %z to <8 x i8>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %0, i32 1, <8 x i1> %c, <8 x i8> undef)
+  %2 = sext <8 x i8> %1 to <8 x i16>
+  %3 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %2, <8 x i16>* %3, align 2
+  ret i8* %z
+}
+
+define i8* @ldrbs16_m128(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrbs16_m128:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    subs r0, #128
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrbt.s16 q0, [r0]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -128
+  %0 = bitcast i8* %z to <8 x i8>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %0, i32 1, <8 x i1> %c, <8 x i8> undef)
+  %2 = sext <8 x i8> %1 to <8 x i16>
+  %3 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %2, <8 x i16>* %3, align 2
+  ret i8* %z
+}
+
+define i8* @ldrbu8_4(i8* %x, i8* %y, <16 x i8> *%m) {
+; CHECK-LABEL: ldrbu8_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u8 q0, [r2]
+; CHECK-NEXT:    vpt.i8 ne, q0, zr
+; CHECK-NEXT:    vldrbt.u8 q0, [r0, #4]!
+; CHECK-NEXT:    vstrb.8 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 4
+  %0 = bitcast i8* %z to <16 x i8>*
+  %mask = load <16 x i8>, <16 x i8>* %m, align 1
+  %c = icmp ne <16 x i8> %mask, zeroinitializer
+  %1 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %0, i32 1, <16 x i1> %c, <16 x i8> undef)
+  %2 = bitcast i8* %y to <16 x i8>*
+  store <16 x i8> %1, <16 x i8>* %2, align 1
+  ret i8* %z
+}
+
+define i8* @ldrbu8_3(i8* %x, i8* %y, <16 x i8> *%m) {
+; CHECK-LABEL: ldrbu8_3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u8 q0, [r2]
+; CHECK-NEXT:    vpt.i8 ne, q0, zr
+; CHECK-NEXT:    vldrbt.u8 q0, [r0, #3]!
+; CHECK-NEXT:    vstrb.8 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 3
+  %0 = bitcast i8* %z to <16 x i8>*
+  %mask = load <16 x i8>, <16 x i8>* %m, align 1
+  %c = icmp ne <16 x i8> %mask, zeroinitializer
+  %1 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %0, i32 1, <16 x i1> %c, <16 x i8> undef)
+  %2 = bitcast i8* %y to <16 x i8>*
+  store <16 x i8> %1, <16 x i8>* %2, align 1
+  ret i8* %z
+}
+
+define i8* @ldrbu8_2(i8* %x, i8* %y, <16 x i8> *%m) {
+; CHECK-LABEL: ldrbu8_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u8 q0, [r2]
+; CHECK-NEXT:    vpt.i8 ne, q0, zr
+; CHECK-NEXT:    vldrbt.u8 q0, [r0, #2]!
+; CHECK-NEXT:    vstrb.8 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 2
+  %0 = bitcast i8* %z to <16 x i8>*
+  %mask = load <16 x i8>, <16 x i8>* %m, align 1
+  %c = icmp ne <16 x i8> %mask, zeroinitializer
+  %1 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %0, i32 1, <16 x i1> %c, <16 x i8> undef)
+  %2 = bitcast i8* %y to <16 x i8>*
+  store <16 x i8> %1, <16 x i8>* %2, align 1
+  ret i8* %z
+}
+
+define i8* @ldrbu8_127(i8* %x, i8* %y, <16 x i8> *%m) {
+; CHECK-LABEL: ldrbu8_127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u8 q0, [r2]
+; CHECK-NEXT:    vpt.i8 ne, q0, zr
+; CHECK-NEXT:    vldrbt.u8 q0, [r0, #127]!
+; CHECK-NEXT:    vstrb.8 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 127
+  %0 = bitcast i8* %z to <16 x i8>*
+  %mask = load <16 x i8>, <16 x i8>* %m, align 1
+  %c = icmp ne <16 x i8> %mask, zeroinitializer
+  %1 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %0, i32 1, <16 x i1> %c, <16 x i8> undef)
+  %2 = bitcast i8* %y to <16 x i8>*
+  store <16 x i8> %1, <16 x i8>* %2, align 1
+  ret i8* %z
+}
+
+define i8* @ldrbu8_128(i8* %x, i8* %y, <16 x i8> *%m) {
+; CHECK-LABEL: ldrbu8_128:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    adds r0, #128
+; CHECK-NEXT:    vldrb.u8 q0, [r2]
+; CHECK-NEXT:    vpt.i8 ne, q0, zr
+; CHECK-NEXT:    vldrbt.u8 q0, [r0]
+; CHECK-NEXT:    vstrb.8 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 128
+  %0 = bitcast i8* %z to <16 x i8>*
+  %mask = load <16 x i8>, <16 x i8>* %m, align 1
+  %c = icmp ne <16 x i8> %mask, zeroinitializer
+  %1 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %0, i32 1, <16 x i1> %c, <16 x i8> undef)
+  %2 = bitcast i8* %y to <16 x i8>*
+  store <16 x i8> %1, <16 x i8>* %2, align 1
+  ret i8* %z
+}
+
+define i8* @ldrbu8_m127(i8* %x, i8* %y, <16 x i8> *%m) {
+; CHECK-LABEL: ldrbu8_m127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u8 q0, [r2]
+; CHECK-NEXT:    vpt.i8 ne, q0, zr
+; CHECK-NEXT:    vldrbt.u8 q0, [r0, #-127]!
+; CHECK-NEXT:    vstrb.8 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -127
+  %0 = bitcast i8* %z to <16 x i8>*
+  %mask = load <16 x i8>, <16 x i8>* %m, align 1
+  %c = icmp ne <16 x i8> %mask, zeroinitializer
+  %1 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %0, i32 1, <16 x i1> %c, <16 x i8> undef)
+  %2 = bitcast i8* %y to <16 x i8>*
+  store <16 x i8> %1, <16 x i8>* %2, align 1
+  ret i8* %z
+}
+
+define i8* @ldrbu8_m128(i8* %x, i8* %y, <16 x i8> *%m) {
+; CHECK-LABEL: ldrbu8_m128:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    subs r0, #128
+; CHECK-NEXT:    vldrb.u8 q0, [r2]
+; CHECK-NEXT:    vpt.i8 ne, q0, zr
+; CHECK-NEXT:    vldrbt.u8 q0, [r0]
+; CHECK-NEXT:    vstrb.8 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -128
+  %0 = bitcast i8* %z to <16 x i8>*
+  %mask = load <16 x i8>, <16 x i8>* %m, align 1
+  %c = icmp ne <16 x i8> %mask, zeroinitializer
+  %1 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %0, i32 1, <16 x i1> %c, <16 x i8> undef)
+  %2 = bitcast i8* %y to <16 x i8>*
+  store <16 x i8> %1, <16 x i8>* %2, align 1
+  ret i8* %z
+}
+
+define i8* @ldrwf32_4(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrwf32_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrwt.u32 q0, [r0, #4]!
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 4
+  %0 = bitcast i8* %z to <4 x float>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* %0, i32 4, <4 x i1> %c, <4 x float> undef)
+  %2 = bitcast i8* %y to <4 x float>*
+  store <4 x float> %1, <4 x float>* %2, align 4
+  ret i8* %z
+}
+
+define i8* @ldrwf32_3(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrwf32_3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    adds r0, #3
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrwt.u32 q0, [r0]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 3
+  %0 = bitcast i8* %z to <4 x float>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* %0, i32 4, <4 x i1> %c, <4 x float> undef)
+  %2 = bitcast i8* %y to <4 x float>*
+  store <4 x float> %1, <4 x float>* %2, align 4
+  ret i8* %z
+}
+
+define i8* @ldrwf32_2(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrwf32_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    adds r0, #2
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrwt.u32 q0, [r0]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 2
+  %0 = bitcast i8* %z to <4 x float>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* %0, i32 4, <4 x i1> %c, <4 x float> undef)
+  %2 = bitcast i8* %y to <4 x float>*
+  store <4 x float> %1, <4 x float>* %2, align 4
+  ret i8* %z
+}
+
+define i8* @ldrwf32_508(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrwf32_508:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrwt.u32 q0, [r0, #508]!
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 508
+  %0 = bitcast i8* %z to <4 x float>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* %0, i32 4, <4 x i1> %c, <4 x float> undef)
+  %2 = bitcast i8* %y to <4 x float>*
+  store <4 x float> %1, <4 x float>* %2, align 4
+  ret i8* %z
+}
+
+define i8* @ldrwf32_512(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrwf32_512:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    add.w r0, r0, #512
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrwt.u32 q0, [r0]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 512
+  %0 = bitcast i8* %z to <4 x float>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* %0, i32 4, <4 x i1> %c, <4 x float> undef)
+  %2 = bitcast i8* %y to <4 x float>*
+  store <4 x float> %1, <4 x float>* %2, align 4
+  ret i8* %z
+}
+
+define i8* @ldrwf32_m508(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrwf32_m508:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrwt.u32 q0, [r0, #-508]!
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -508
+  %0 = bitcast i8* %z to <4 x float>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* %0, i32 4, <4 x i1> %c, <4 x float> undef)
+  %2 = bitcast i8* %y to <4 x float>*
+  store <4 x float> %1, <4 x float>* %2, align 4
+  ret i8* %z
+}
+
+define i8* @ldrwf32_m512(i8* %x, i8* %y, <4 x i32> *%m) {
+; CHECK-LABEL: ldrwf32_m512:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    sub.w r0, r0, #512
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vldrwt.u32 q0, [r0]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -512
+  %0 = bitcast i8* %z to <4 x float>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* %0, i32 4, <4 x i1> %c, <4 x float> undef)
+  %2 = bitcast i8* %y to <4 x float>*
+  store <4 x float> %1, <4 x float>* %2, align 4
+  ret i8* %z
+}
+
+define i8* @ldrhf16_4(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrhf16_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrht.u16 q0, [r0, #4]!
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 4
+  %0 = bitcast i8* %z to <8 x half>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>* %0, i32 2, <8 x i1> %c, <8 x half> undef)
+  %2 = bitcast i8* %y to <8 x half>*
+  store <8 x half> %1, <8 x half>* %2, align 2
+  ret i8* %z
+}
+
+define i8* @ldrhf16_3(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrhf16_3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    adds r0, #3
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrht.u16 q0, [r0]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 3
+  %0 = bitcast i8* %z to <8 x half>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>* %0, i32 2, <8 x i1> %c, <8 x half> undef)
+  %2 = bitcast i8* %y to <8 x half>*
+  store <8 x half> %1, <8 x half>* %2, align 2
+  ret i8* %z
+}
+
+define i8* @ldrhf16_2(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrhf16_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrht.u16 q0, [r0, #2]!
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 2
+  %0 = bitcast i8* %z to <8 x half>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>* %0, i32 2, <8 x i1> %c, <8 x half> undef)
+  %2 = bitcast i8* %y to <8 x half>*
+  store <8 x half> %1, <8 x half>* %2, align 2
+  ret i8* %z
+}
+
+define i8* @ldrhf16_254(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrhf16_254:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrht.u16 q0, [r0, #254]!
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 254
+  %0 = bitcast i8* %z to <8 x half>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>* %0, i32 2, <8 x i1> %c, <8 x half> undef)
+  %2 = bitcast i8* %y to <8 x half>*
+  store <8 x half> %1, <8 x half>* %2, align 2
+  ret i8* %z
+}
+
+define i8* @ldrhf16_256(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrhf16_256:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    add.w r0, r0, #256
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrht.u16 q0, [r0]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 256
+  %0 = bitcast i8* %z to <8 x half>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>* %0, i32 2, <8 x i1> %c, <8 x half> undef)
+  %2 = bitcast i8* %y to <8 x half>*
+  store <8 x half> %1, <8 x half>* %2, align 2
+  ret i8* %z
+}
+
+define i8* @ldrhf16_m254(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrhf16_m254:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrht.u16 q0, [r0, #-254]!
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -254
+  %0 = bitcast i8* %z to <8 x half>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>* %0, i32 2, <8 x i1> %c, <8 x half> undef)
+  %2 = bitcast i8* %y to <8 x half>*
+  store <8 x half> %1, <8 x half>* %2, align 2
+  ret i8* %z
+}
+
+define i8* @ldrhf16_m256(i8* %x, i8* %y, <8 x i16> *%m) {
+; CHECK-LABEL: ldrhf16_m256:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    sub.w r0, r0, #256
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vldrht.u16 q0, [r0]
+; CHECK-NEXT:    vstrh.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 -256
+  %0 = bitcast i8* %z to <8 x half>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = call <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>* %0, i32 2, <8 x i1> %c, <8 x half> undef)
+  %2 = bitcast i8* %y to <8 x half>*
+  store <8 x half> %1, <8 x half>* %2, align 2
+  ret i8* %z
+}
+
+
+
+
+define i8* @strw32_4(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strw32_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrwt.32 q0, [r0, #4]!
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 4
+  %0 = bitcast i8* %x to <4 x i32>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
+  %2 = bitcast i8* %z to <4 x i32>*
+  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %1, <4 x i32>* %2, i32 4, <4 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strw32_3(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strw32_3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    adds r0, #3
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrwt.32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 3
+  %0 = bitcast i8* %x to <4 x i32>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
+  %2 = bitcast i8* %z to <4 x i32>*
+  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %1, <4 x i32>* %2, i32 4, <4 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strw32_2(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strw32_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    adds r0, #2
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrwt.32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 2
+  %0 = bitcast i8* %x to <4 x i32>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
+  %2 = bitcast i8* %z to <4 x i32>*
+  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %1, <4 x i32>* %2, i32 4, <4 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strw32_508(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strw32_508:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrwt.32 q0, [r0, #508]!
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 508
+  %0 = bitcast i8* %x to <4 x i32>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
+  %2 = bitcast i8* %z to <4 x i32>*
+  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %1, <4 x i32>* %2, i32 4, <4 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strw32_512(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strw32_512:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    add.w r0, r0, #512
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrwt.32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 512
+  %0 = bitcast i8* %x to <4 x i32>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
+  %2 = bitcast i8* %z to <4 x i32>*
+  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %1, <4 x i32>* %2, i32 4, <4 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strw32_m508(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strw32_m508:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrwt.32 q0, [r0, #-508]!
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -508
+  %0 = bitcast i8* %x to <4 x i32>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
+  %2 = bitcast i8* %z to <4 x i32>*
+  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %1, <4 x i32>* %2, i32 4, <4 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strw32_m512(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strw32_m512:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    sub.w r0, r0, #512
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrwt.32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -512
+  %0 = bitcast i8* %x to <4 x i32>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
+  %2 = bitcast i8* %z to <4 x i32>*
+  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %1, <4 x i32>* %2, i32 4, <4 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strh32_4(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strh32_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrht.32 q0, [r0, #4]!
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 4
+  %0 = bitcast i8* %x to <4 x i16>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
+  %2 = bitcast i8* %z to <4 x i16>*
+  call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %1, <4 x i16>* %2, i32 2, <4 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strh32_3(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strh32_3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    adds r0, #3
+; CHECK-NEXT:    vldrh.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrht.32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 3
+  %0 = bitcast i8* %x to <4 x i16>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
+  %2 = bitcast i8* %z to <4 x i16>*
+  call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %1, <4 x i16>* %2, i32 2, <4 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strh32_2(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strh32_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrht.32 q0, [r0, #2]!
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 2
+  %0 = bitcast i8* %x to <4 x i16>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
+  %2 = bitcast i8* %z to <4 x i16>*
+  call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %1, <4 x i16>* %2, i32 2, <4 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strh32_254(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strh32_254:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrht.32 q0, [r0, #254]!
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 254
+  %0 = bitcast i8* %x to <4 x i16>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
+  %2 = bitcast i8* %z to <4 x i16>*
+  call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %1, <4 x i16>* %2, i32 2, <4 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strh32_256(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strh32_256:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    add.w r0, r0, #256
+; CHECK-NEXT:    vldrh.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrht.32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 256
+  %0 = bitcast i8* %x to <4 x i16>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
+  %2 = bitcast i8* %z to <4 x i16>*
+  call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %1, <4 x i16>* %2, i32 2, <4 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strh32_m254(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strh32_m254:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrht.32 q0, [r0, #-254]!
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -254
+  %0 = bitcast i8* %x to <4 x i16>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
+  %2 = bitcast i8* %z to <4 x i16>*
+  call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %1, <4 x i16>* %2, i32 2, <4 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strh32_m256(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strh32_m256:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    sub.w r0, r0, #256
+; CHECK-NEXT:    vldrh.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrht.32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -256
+  %0 = bitcast i8* %x to <4 x i16>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x i16>, <4 x i16>* %0, align 2
+  %2 = bitcast i8* %z to <4 x i16>*
+  call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %1, <4 x i16>* %2, i32 2, <4 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strh16_4(i8* %y, i8* %x, <8 x i16> *%m) {
+; CHECK-LABEL: strh16_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q1, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q1, zr
+; CHECK-NEXT:    vstrht.16 q0, [r0, #4]!
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 4
+  %0 = bitcast i8* %x to <8 x i16>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
+  %2 = bitcast i8* %z to <8 x i16>*
+  call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %1, <8 x i16>* %2, i32 2, <8 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strh16_3(i8* %y, i8* %x, <8 x i16> *%m) {
+; CHECK-LABEL: strh16_3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    adds r0, #3
+; CHECK-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q1, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q1, zr
+; CHECK-NEXT:    vstrht.16 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 3
+  %0 = bitcast i8* %x to <8 x i16>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
+  %2 = bitcast i8* %z to <8 x i16>*
+  call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %1, <8 x i16>* %2, i32 2, <8 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strh16_2(i8* %y, i8* %x, <8 x i16> *%m) {
+; CHECK-LABEL: strh16_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q1, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q1, zr
+; CHECK-NEXT:    vstrht.16 q0, [r0, #2]!
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 2
+  %0 = bitcast i8* %x to <8 x i16>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
+  %2 = bitcast i8* %z to <8 x i16>*
+  call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %1, <8 x i16>* %2, i32 2, <8 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strh16_254(i8* %y, i8* %x, <8 x i16> *%m) {
+; CHECK-LABEL: strh16_254:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q1, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q1, zr
+; CHECK-NEXT:    vstrht.16 q0, [r0, #254]!
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 254
+  %0 = bitcast i8* %x to <8 x i16>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
+  %2 = bitcast i8* %z to <8 x i16>*
+  call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %1, <8 x i16>* %2, i32 2, <8 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strh16_256(i8* %y, i8* %x, <8 x i16> *%m) {
+; CHECK-LABEL: strh16_256:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    add.w r0, r0, #256
+; CHECK-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q1, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q1, zr
+; CHECK-NEXT:    vstrht.16 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 256
+  %0 = bitcast i8* %x to <8 x i16>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
+  %2 = bitcast i8* %z to <8 x i16>*
+  call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %1, <8 x i16>* %2, i32 2, <8 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strh16_m254(i8* %y, i8* %x, <8 x i16> *%m) {
+; CHECK-LABEL: strh16_m254:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q1, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q1, zr
+; CHECK-NEXT:    vstrht.16 q0, [r0, #-254]!
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -254
+  %0 = bitcast i8* %x to <8 x i16>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
+  %2 = bitcast i8* %z to <8 x i16>*
+  call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %1, <8 x i16>* %2, i32 2, <8 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strh16_m256(i8* %y, i8* %x, <8 x i16> *%m) {
+; CHECK-LABEL: strh16_m256:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    sub.w r0, r0, #256
+; CHECK-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q1, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q1, zr
+; CHECK-NEXT:    vstrht.16 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -256
+  %0 = bitcast i8* %x to <8 x i16>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = load <8 x i16>, <8 x i16>* %0, align 2
+  %2 = bitcast i8* %z to <8 x i16>*
+  call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %1, <8 x i16>* %2, i32 2, <8 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strb32_4(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strb32_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrbt.32 q0, [r0, #4]!
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 4
+  %0 = bitcast i8* %x to <4 x i8>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
+  %2 = bitcast i8* %z to <4 x i8>*
+  call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %1, <4 x i8>* %2, i32 1, <4 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strb32_3(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strb32_3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrbt.32 q0, [r0, #3]!
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 3
+  %0 = bitcast i8* %x to <4 x i8>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
+  %2 = bitcast i8* %z to <4 x i8>*
+  call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %1, <4 x i8>* %2, i32 1, <4 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strb32_2(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strb32_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrbt.32 q0, [r0, #2]!
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 2
+  %0 = bitcast i8* %x to <4 x i8>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
+  %2 = bitcast i8* %z to <4 x i8>*
+  call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %1, <4 x i8>* %2, i32 1, <4 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strb32_127(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strb32_127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrbt.32 q0, [r0, #127]!
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 127
+  %0 = bitcast i8* %x to <4 x i8>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
+  %2 = bitcast i8* %z to <4 x i8>*
+  call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %1, <4 x i8>* %2, i32 1, <4 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strb32_128(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strb32_128:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    adds r0, #128
+; CHECK-NEXT:    vldrb.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrbt.32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 128
+  %0 = bitcast i8* %x to <4 x i8>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
+  %2 = bitcast i8* %z to <4 x i8>*
+  call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %1, <4 x i8>* %2, i32 1, <4 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strb32_m127(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strb32_m127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrbt.32 q0, [r0, #-127]!
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -127
+  %0 = bitcast i8* %x to <4 x i8>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
+  %2 = bitcast i8* %z to <4 x i8>*
+  call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %1, <4 x i8>* %2, i32 1, <4 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strb32_m128(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strb32_m128:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    subs r0, #128
+; CHECK-NEXT:    vldrb.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrbt.32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -128
+  %0 = bitcast i8* %x to <4 x i8>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x i8>, <4 x i8>* %0, align 1
+  %2 = bitcast i8* %z to <4 x i8>*
+  call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %1, <4 x i8>* %2, i32 1, <4 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strb16_4(i8* %y, i8* %x, <8 x i16> *%m) {
+; CHECK-LABEL: strb16_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u16 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q1, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q1, zr
+; CHECK-NEXT:    vstrbt.16 q0, [r0, #4]!
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 4
+  %0 = bitcast i8* %x to <8 x i8>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
+  %2 = bitcast i8* %z to <8 x i8>*
+  call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %1, <8 x i8>* %2, i32 1, <8 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strb16_3(i8* %y, i8* %x, <8 x i16> *%m) {
+; CHECK-LABEL: strb16_3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u16 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q1, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q1, zr
+; CHECK-NEXT:    vstrbt.16 q0, [r0, #3]!
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 3
+  %0 = bitcast i8* %x to <8 x i8>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
+  %2 = bitcast i8* %z to <8 x i8>*
+  call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %1, <8 x i8>* %2, i32 1, <8 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strb16_2(i8* %y, i8* %x, <8 x i16> *%m) {
+; CHECK-LABEL: strb16_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u16 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q1, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q1, zr
+; CHECK-NEXT:    vstrbt.16 q0, [r0, #2]!
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 2
+  %0 = bitcast i8* %x to <8 x i8>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
+  %2 = bitcast i8* %z to <8 x i8>*
+  call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %1, <8 x i8>* %2, i32 1, <8 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strb16_127(i8* %y, i8* %x, <8 x i16> *%m) {
+; CHECK-LABEL: strb16_127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u16 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q1, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q1, zr
+; CHECK-NEXT:    vstrbt.16 q0, [r0, #127]!
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 127
+  %0 = bitcast i8* %x to <8 x i8>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
+  %2 = bitcast i8* %z to <8 x i8>*
+  call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %1, <8 x i8>* %2, i32 1, <8 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strb16_128(i8* %y, i8* %x, <8 x i16> *%m) {
+; CHECK-LABEL: strb16_128:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    adds r0, #128
+; CHECK-NEXT:    vldrb.u16 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q1, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q1, zr
+; CHECK-NEXT:    vstrbt.16 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 128
+  %0 = bitcast i8* %x to <8 x i8>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
+  %2 = bitcast i8* %z to <8 x i8>*
+  call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %1, <8 x i8>* %2, i32 1, <8 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strb16_m127(i8* %y, i8* %x, <8 x i16> *%m) {
+; CHECK-LABEL: strb16_m127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u16 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q1, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q1, zr
+; CHECK-NEXT:    vstrbt.16 q0, [r0, #-127]!
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -127
+  %0 = bitcast i8* %x to <8 x i8>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
+  %2 = bitcast i8* %z to <8 x i8>*
+  call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %1, <8 x i8>* %2, i32 1, <8 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strb16_m128(i8* %y, i8* %x, <8 x i16> *%m) {
+; CHECK-LABEL: strb16_m128:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    subs r0, #128
+; CHECK-NEXT:    vldrb.u16 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q1, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q1, zr
+; CHECK-NEXT:    vstrbt.16 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -128
+  %0 = bitcast i8* %x to <8 x i8>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
+  %2 = bitcast i8* %z to <8 x i8>*
+  call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %1, <8 x i8>* %2, i32 1, <8 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strb8_4(i8* %y, i8* %x, <16 x i8> *%m) {
+; CHECK-LABEL: strb8_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u8 q0, [r1]
+; CHECK-NEXT:    vldrb.u8 q1, [r2]
+; CHECK-NEXT:    vpt.i8 ne, q1, zr
+; CHECK-NEXT:    vstrbt.8 q0, [r0, #4]!
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 4
+  %0 = bitcast i8* %x to <16 x i8>*
+  %mask = load <16 x i8>, <16 x i8>* %m, align 1
+  %c = icmp ne <16 x i8> %mask, zeroinitializer
+  %1 = load <16 x i8>, <16 x i8>* %0, align 1
+  %2 = bitcast i8* %z to <16 x i8>*
+  call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %1, <16 x i8>* %2, i32 1, <16 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strb8_3(i8* %y, i8* %x, <16 x i8> *%m) {
+; CHECK-LABEL: strb8_3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u8 q0, [r1]
+; CHECK-NEXT:    vldrb.u8 q1, [r2]
+; CHECK-NEXT:    vpt.i8 ne, q1, zr
+; CHECK-NEXT:    vstrbt.8 q0, [r0, #3]!
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 3
+  %0 = bitcast i8* %x to <16 x i8>*
+  %mask = load <16 x i8>, <16 x i8>* %m, align 1
+  %c = icmp ne <16 x i8> %mask, zeroinitializer
+  %1 = load <16 x i8>, <16 x i8>* %0, align 1
+  %2 = bitcast i8* %z to <16 x i8>*
+  call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %1, <16 x i8>* %2, i32 1, <16 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strb8_2(i8* %y, i8* %x, <16 x i8> *%m) {
+; CHECK-LABEL: strb8_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u8 q0, [r1]
+; CHECK-NEXT:    vldrb.u8 q1, [r2]
+; CHECK-NEXT:    vpt.i8 ne, q1, zr
+; CHECK-NEXT:    vstrbt.8 q0, [r0, #2]!
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 2
+  %0 = bitcast i8* %x to <16 x i8>*
+  %mask = load <16 x i8>, <16 x i8>* %m, align 1
+  %c = icmp ne <16 x i8> %mask, zeroinitializer
+  %1 = load <16 x i8>, <16 x i8>* %0, align 1
+  %2 = bitcast i8* %z to <16 x i8>*
+  call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %1, <16 x i8>* %2, i32 1, <16 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strb8_127(i8* %y, i8* %x, <16 x i8> *%m) {
+; CHECK-LABEL: strb8_127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u8 q0, [r1]
+; CHECK-NEXT:    vldrb.u8 q1, [r2]
+; CHECK-NEXT:    vpt.i8 ne, q1, zr
+; CHECK-NEXT:    vstrbt.8 q0, [r0, #127]!
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 127
+  %0 = bitcast i8* %x to <16 x i8>*
+  %mask = load <16 x i8>, <16 x i8>* %m, align 1
+  %c = icmp ne <16 x i8> %mask, zeroinitializer
+  %1 = load <16 x i8>, <16 x i8>* %0, align 1
+  %2 = bitcast i8* %z to <16 x i8>*
+  call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %1, <16 x i8>* %2, i32 1, <16 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strb8_128(i8* %y, i8* %x, <16 x i8> *%m) {
+; CHECK-LABEL: strb8_128:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    adds r0, #128
+; CHECK-NEXT:    vldrb.u8 q0, [r1]
+; CHECK-NEXT:    vldrb.u8 q1, [r2]
+; CHECK-NEXT:    vpt.i8 ne, q1, zr
+; CHECK-NEXT:    vstrbt.8 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 128
+  %0 = bitcast i8* %x to <16 x i8>*
+  %mask = load <16 x i8>, <16 x i8>* %m, align 1
+  %c = icmp ne <16 x i8> %mask, zeroinitializer
+  %1 = load <16 x i8>, <16 x i8>* %0, align 1
+  %2 = bitcast i8* %z to <16 x i8>*
+  call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %1, <16 x i8>* %2, i32 1, <16 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strb8_m127(i8* %y, i8* %x, <16 x i8> *%m) {
+; CHECK-LABEL: strb8_m127:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u8 q0, [r1]
+; CHECK-NEXT:    vldrb.u8 q1, [r2]
+; CHECK-NEXT:    vpt.i8 ne, q1, zr
+; CHECK-NEXT:    vstrbt.8 q0, [r0, #-127]!
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -127
+  %0 = bitcast i8* %x to <16 x i8>*
+  %mask = load <16 x i8>, <16 x i8>* %m, align 1
+  %c = icmp ne <16 x i8> %mask, zeroinitializer
+  %1 = load <16 x i8>, <16 x i8>* %0, align 1
+  %2 = bitcast i8* %z to <16 x i8>*
+  call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %1, <16 x i8>* %2, i32 1, <16 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strb8_m128(i8* %y, i8* %x, <16 x i8> *%m) {
+; CHECK-LABEL: strb8_m128:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    subs r0, #128
+; CHECK-NEXT:    vldrb.u8 q0, [r1]
+; CHECK-NEXT:    vldrb.u8 q1, [r2]
+; CHECK-NEXT:    vpt.i8 ne, q1, zr
+; CHECK-NEXT:    vstrbt.8 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -128
+  %0 = bitcast i8* %x to <16 x i8>*
+  %mask = load <16 x i8>, <16 x i8>* %m, align 1
+  %c = icmp ne <16 x i8> %mask, zeroinitializer
+  %1 = load <16 x i8>, <16 x i8>* %0, align 1
+  %2 = bitcast i8* %z to <16 x i8>*
+  call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %1, <16 x i8>* %2, i32 1, <16 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strwf32_4(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strwf32_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrwt.32 q0, [r0, #4]!
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 4
+  %0 = bitcast i8* %x to <4 x float>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x float>, <4 x float>* %0, align 4
+  %2 = bitcast i8* %z to <4 x float>*
+  call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %1, <4 x float>* %2, i32 4, <4 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strwf32_3(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strwf32_3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    adds r0, #3
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrwt.32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 3
+  %0 = bitcast i8* %x to <4 x float>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x float>, <4 x float>* %0, align 4
+  %2 = bitcast i8* %z to <4 x float>*
+  call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %1, <4 x float>* %2, i32 4, <4 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strwf32_2(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strwf32_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    adds r0, #2
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrwt.32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 2
+  %0 = bitcast i8* %x to <4 x float>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x float>, <4 x float>* %0, align 4
+  %2 = bitcast i8* %z to <4 x float>*
+  call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %1, <4 x float>* %2, i32 4, <4 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strwf32_508(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strwf32_508:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrwt.32 q0, [r0, #508]!
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 508
+  %0 = bitcast i8* %x to <4 x float>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x float>, <4 x float>* %0, align 4
+  %2 = bitcast i8* %z to <4 x float>*
+  call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %1, <4 x float>* %2, i32 4, <4 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strwf32_512(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strwf32_512:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    add.w r0, r0, #512
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrwt.32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 512
+  %0 = bitcast i8* %x to <4 x float>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x float>, <4 x float>* %0, align 4
+  %2 = bitcast i8* %z to <4 x float>*
+  call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %1, <4 x float>* %2, i32 4, <4 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strwf32_m508(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strwf32_m508:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrwt.32 q0, [r0, #-508]!
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -508
+  %0 = bitcast i8* %x to <4 x float>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x float>, <4 x float>* %0, align 4
+  %2 = bitcast i8* %z to <4 x float>*
+  call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %1, <4 x float>* %2, i32 4, <4 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strwf32_m512(i8* %y, i8* %x, <4 x i32> *%m) {
+; CHECK-LABEL: strwf32_m512:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    sub.w r0, r0, #512
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-NEXT:    vstrwt.32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -512
+  %0 = bitcast i8* %x to <4 x float>*
+  %mask = load <4 x i32>, <4 x i32>* %m, align 4
+  %c = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = load <4 x float>, <4 x float>* %0, align 4
+  %2 = bitcast i8* %z to <4 x float>*
+  call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %1, <4 x float>* %2, i32 4, <4 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strhf16_4(i8* %y, i8* %x, <8 x i16> *%m) {
+; CHECK-LABEL: strhf16_4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q1, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q1, zr
+; CHECK-NEXT:    vstrht.16 q0, [r0, #4]!
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 4
+  %0 = bitcast i8* %x to <8 x half>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = load <8 x half>, <8 x half>* %0, align 2
+  %2 = bitcast i8* %z to <8 x half>*
+  call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %1, <8 x half>* %2, i32 2, <8 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strhf16_3(i8* %y, i8* %x, <8 x i16> *%m) {
+; CHECK-LABEL: strhf16_3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    adds r0, #3
+; CHECK-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q1, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q1, zr
+; CHECK-NEXT:    vstrht.16 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 3
+  %0 = bitcast i8* %x to <8 x half>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = load <8 x half>, <8 x half>* %0, align 2
+  %2 = bitcast i8* %z to <8 x half>*
+  call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %1, <8 x half>* %2, i32 2, <8 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strhf16_2(i8* %y, i8* %x, <8 x i16> *%m) {
+; CHECK-LABEL: strhf16_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q1, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q1, zr
+; CHECK-NEXT:    vstrht.16 q0, [r0, #2]!
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 2
+  %0 = bitcast i8* %x to <8 x half>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = load <8 x half>, <8 x half>* %0, align 2
+  %2 = bitcast i8* %z to <8 x half>*
+  call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %1, <8 x half>* %2, i32 2, <8 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strhf16_254(i8* %y, i8* %x, <8 x i16> *%m) {
+; CHECK-LABEL: strhf16_254:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q1, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q1, zr
+; CHECK-NEXT:    vstrht.16 q0, [r0, #254]!
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 254
+  %0 = bitcast i8* %x to <8 x half>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = load <8 x half>, <8 x half>* %0, align 2
+  %2 = bitcast i8* %z to <8 x half>*
+  call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %1, <8 x half>* %2, i32 2, <8 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strhf16_256(i8* %y, i8* %x, <8 x i16> *%m) {
+; CHECK-LABEL: strhf16_256:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    add.w r0, r0, #256
+; CHECK-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q1, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q1, zr
+; CHECK-NEXT:    vstrht.16 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 256
+  %0 = bitcast i8* %x to <8 x half>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = load <8 x half>, <8 x half>* %0, align 2
+  %2 = bitcast i8* %z to <8 x half>*
+  call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %1, <8 x half>* %2, i32 2, <8 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strhf16_m254(i8* %y, i8* %x, <8 x i16> *%m) {
+; CHECK-LABEL: strhf16_m254:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q1, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q1, zr
+; CHECK-NEXT:    vstrht.16 q0, [r0, #-254]!
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -254
+  %0 = bitcast i8* %x to <8 x half>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = load <8 x half>, <8 x half>* %0, align 2
+  %2 = bitcast i8* %z to <8 x half>*
+  call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %1, <8 x half>* %2, i32 2, <8 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @strhf16_m256(i8* %y, i8* %x, <8 x i16> *%m) {
+; CHECK-LABEL: strhf16_m256:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    sub.w r0, r0, #256
+; CHECK-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q1, [r2]
+; CHECK-NEXT:    vpt.i16 ne, q1, zr
+; CHECK-NEXT:    vstrht.16 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 -256
+  %0 = bitcast i8* %x to <8 x half>*
+  %mask = load <8 x i16>, <8 x i16>* %m, align 2
+  %c = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = load <8 x half>, <8 x half>* %0, align 2
+  %2 = bitcast i8* %z to <8 x half>*
+  call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %1, <8 x half>* %2, i32 2, <8 x i1> %c)
+  ret i8* %z
+}
+
+declare <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>*, i32, <4 x i1>, <4 x i32>)
+declare <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>*, i32, <4 x i1>, <4 x i16>)
+declare <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>*, i32, <8 x i1>, <8 x i16>)
+declare <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>*, i32, <4 x i1>, <4 x i8>)
+declare <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>*, i32, <8 x i1>, <8 x i8>)
+declare <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>*, i32, <16 x i1>, <16 x i8>)
+declare <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>*, i32, <4 x i1>, <4 x float>)
+declare <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>*, i32, <8 x i1>, <8 x half>)
+
+declare void @llvm.masked.store.v4i32.p0v4i32(<4 x i32>, <4 x i32>*, i32, <4 x i1>)
+declare void @llvm.masked.store.v8i16.p0v8i16(<8 x i16>, <8 x i16>*, i32, <8 x i1>)
+declare void @llvm.masked.store.v4i16.p0v4i16(<4 x i16>, <4 x i16>*, i32, <4 x i1>)
+declare void @llvm.masked.store.v16i8.p0v16i8(<16 x i8>, <16 x i8>*, i32, <16 x i1>)
+declare void @llvm.masked.store.v8i8.p0v8i8(<8 x i8>, <8 x i8>*, i32, <8 x i1>)
+declare void @llvm.masked.store.v4i8.p0v4i8(<4 x i8>, <4 x i8>*, i32, <4 x i1>)
+declare void @llvm.masked.store.v4f32.p0v4f32(<4 x float>, <4 x float>*, i32, <4 x i1>)
+declare void @llvm.masked.store.v8f16.p0v8f16(<8 x half>, <8 x half>*, i32, <8 x i1>)
diff --git a/llvm/test/CodeGen/Thumb2/mve-masked-ldst.ll b/llvm/test/CodeGen/Thumb2/mve-masked-ldst.ll
new file mode 100644
index 000000000..b92163bcb
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-masked-ldst.ll
@@ -0,0 +1,1151 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve,+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-LE
+; RUN: llc -mtriple=thumbebv8.1m.main-arm-none-eabi -mattr=+mve,+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-BE
+
+define void @foo_v4i32_v4i32(<4 x i32> *%dest, <4 x i32> *%mask, <4 x i32> *%src) {
+; CHECK-LABEL: foo_v4i32_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    vptt.s32 gt, q0, zr
+; CHECK-NEXT:    vldrwt.u32 q0, [r2]
+; CHECK-NEXT:    vstrwt.32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = load <4 x i32>, <4 x i32>* %mask, align 4
+  %1 = icmp sgt <4 x i32> %0, zeroinitializer
+  %2 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %src, i32 4, <4 x i1> %1, <4 x i32> undef)
+  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %2, <4 x i32>* %dest, i32 4, <4 x i1> %1)
+  ret void
+}
+
+define void @foo_sext_v4i32_v4i8(<4 x i32> *%dest, <4 x i32> *%mask, <4 x i8> *%src) {
+; CHECK-LABEL: foo_sext_v4i32_v4i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    vptt.s32 gt, q0, zr
+; CHECK-NEXT:    vldrbt.s32 q0, [r2]
+; CHECK-NEXT:    vstrwt.32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = load <4 x i32>, <4 x i32>* %mask, align 4
+  %1 = icmp sgt <4 x i32> %0, zeroinitializer
+  %2 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %src, i32 1, <4 x i1> %1, <4 x i8> undef)
+  %3 = sext <4 x i8> %2 to <4 x i32>
+  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %3, <4 x i32>* %dest, i32 4, <4 x i1> %1)
+  ret void
+}
+
+define void @foo_sext_v4i32_v4i16(<4 x i32> *%dest, <4 x i32> *%mask, <4 x i16> *%src) {
+; CHECK-LABEL: foo_sext_v4i32_v4i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    vptt.s32 gt, q0, zr
+; CHECK-NEXT:    vldrht.s32 q0, [r2]
+; CHECK-NEXT:    vstrwt.32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = load <4 x i32>, <4 x i32>* %mask, align 4
+  %1 = icmp sgt <4 x i32> %0, zeroinitializer
+  %2 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %src, i32 2, <4 x i1> %1, <4 x i16> undef)
+  %3 = sext <4 x i16> %2 to <4 x i32>
+  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %3, <4 x i32>* %dest, i32 4, <4 x i1> %1)
+  ret void
+}
+
+define void @foo_zext_v4i32_v4i8(<4 x i32> *%dest, <4 x i32> *%mask, <4 x i8> *%src) {
+; CHECK-LABEL: foo_zext_v4i32_v4i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    vptt.s32 gt, q0, zr
+; CHECK-NEXT:    vldrbt.u32 q0, [r2]
+; CHECK-NEXT:    vstrwt.32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = load <4 x i32>, <4 x i32>* %mask, align 4
+  %1 = icmp sgt <4 x i32> %0, zeroinitializer
+  %2 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %src, i32 1, <4 x i1> %1, <4 x i8> undef)
+  %3 = zext <4 x i8> %2 to <4 x i32>
+  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %3, <4 x i32>* %dest, i32 4, <4 x i1> %1)
+  ret void
+}
+
+define void @foo_zext_v4i32_v4i16(<4 x i32> *%dest, <4 x i32> *%mask, <4 x i16> *%src) {
+; CHECK-LABEL: foo_zext_v4i32_v4i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    vptt.s32 gt, q0, zr
+; CHECK-NEXT:    vldrht.u32 q0, [r2]
+; CHECK-NEXT:    vstrwt.32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = load <4 x i32>, <4 x i32>* %mask, align 4
+  %1 = icmp sgt <4 x i32> %0, zeroinitializer
+  %2 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %src, i32 2, <4 x i1> %1, <4 x i16> undef)
+  %3 = zext <4 x i16> %2 to <4 x i32>
+  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %3, <4 x i32>* %dest, i32 4, <4 x i1> %1)
+  ret void
+}
+
+define void @foo_sext_v2i64_v2i32(<2 x i64> *%dest, <2 x i32> *%mask, <2 x i32> *%src) {
+; CHECK-LE-LABEL: foo_sext_v2i64_v2i32:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    .save {r4, r5, r7, lr}
+; CHECK-LE-NEXT:    push {r4, r5, r7, lr}
+; CHECK-LE-NEXT:    .pad #4
+; CHECK-LE-NEXT:    sub sp, #4
+; CHECK-LE-NEXT:    ldrd lr, r12, [r1]
+; CHECK-LE-NEXT:    movs r1, #0
+; CHECK-LE-NEXT:    @ implicit-def: $q1
+; CHECK-LE-NEXT:    movs r4, #0
+; CHECK-LE-NEXT:    rsbs.w r3, lr, #0
+; CHECK-LE-NEXT:    vmov.32 q0[0], lr
+; CHECK-LE-NEXT:    sbcs.w r3, r1, lr, asr #31
+; CHECK-LE-NEXT:    mov.w lr, #0
+; CHECK-LE-NEXT:    it lt
+; CHECK-LE-NEXT:    movlt.w lr, #1
+; CHECK-LE-NEXT:    rsbs.w r3, r12, #0
+; CHECK-LE-NEXT:    sbcs.w r3, r1, r12, asr #31
+; CHECK-LE-NEXT:    it lt
+; CHECK-LE-NEXT:    movlt r1, #1
+; CHECK-LE-NEXT:    cmp r1, #0
+; CHECK-LE-NEXT:    it ne
+; CHECK-LE-NEXT:    mvnne r1, #1
+; CHECK-LE-NEXT:    bfi r1, lr, #0, #1
+; CHECK-LE-NEXT:    vmov.32 q0[2], r12
+; CHECK-LE-NEXT:    and r3, r1, #3
+; CHECK-LE-NEXT:    lsls r1, r1, #31
+; CHECK-LE-NEXT:    itt ne
+; CHECK-LE-NEXT:    ldrne r1, [r2]
+; CHECK-LE-NEXT:    vmovne.32 q1[0], r1
+; CHECK-LE-NEXT:    lsls r1, r3, #30
+; CHECK-LE-NEXT:    itt mi
+; CHECK-LE-NEXT:    ldrmi r1, [r2, #4]
+; CHECK-LE-NEXT:    vmovmi.32 q1[2], r1
+; CHECK-LE-NEXT:    vmov r2, s0
+; CHECK-LE-NEXT:    vmov r3, s4
+; CHECK-LE-NEXT:    vmov r1, s6
+; CHECK-LE-NEXT:    vmov.32 q1[0], r3
+; CHECK-LE-NEXT:    rsbs r5, r2, #0
+; CHECK-LE-NEXT:    sbcs.w r2, r4, r2, asr #31
+; CHECK-LE-NEXT:    vmov r2, s2
+; CHECK-LE-NEXT:    asr.w lr, r3, #31
+; CHECK-LE-NEXT:    vmov.32 q1[1], lr
+; CHECK-LE-NEXT:    asr.w r12, r1, #31
+; CHECK-LE-NEXT:    vmov.32 q1[2], r1
+; CHECK-LE-NEXT:    mov.w r1, #0
+; CHECK-LE-NEXT:    it lt
+; CHECK-LE-NEXT:    movlt r1, #1
+; CHECK-LE-NEXT:    vmov.32 q1[3], r12
+; CHECK-LE-NEXT:    rsbs r3, r2, #0
+; CHECK-LE-NEXT:    sbcs.w r2, r4, r2, asr #31
+; CHECK-LE-NEXT:    it lt
+; CHECK-LE-NEXT:    movlt r4, #1
+; CHECK-LE-NEXT:    cmp r4, #0
+; CHECK-LE-NEXT:    it ne
+; CHECK-LE-NEXT:    mvnne r4, #1
+; CHECK-LE-NEXT:    bfi r4, r1, #0, #1
+; CHECK-LE-NEXT:    and r1, r4, #3
+; CHECK-LE-NEXT:    lsls r2, r4, #31
+; CHECK-LE-NEXT:    it ne
+; CHECK-LE-NEXT:    vstrne d2, [r0]
+; CHECK-LE-NEXT:    lsls r1, r1, #30
+; CHECK-LE-NEXT:    it mi
+; CHECK-LE-NEXT:    vstrmi d3, [r0, #8]
+; CHECK-LE-NEXT:    add sp, #4
+; CHECK-LE-NEXT:    pop {r4, r5, r7, pc}
+;
+; CHECK-BE-LABEL: foo_sext_v2i64_v2i32:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    .save {r4, r5, r7, lr}
+; CHECK-BE-NEXT:    push {r4, r5, r7, lr}
+; CHECK-BE-NEXT:    .pad #4
+; CHECK-BE-NEXT:    sub sp, #4
+; CHECK-BE-NEXT:    ldrd r12, lr, [r1]
+; CHECK-BE-NEXT:    rsbs.w r1, lr, #0
+; CHECK-BE-NEXT:    mov.w r3, #0
+; CHECK-BE-NEXT:    sbcs.w r1, r3, lr, asr #31
+; CHECK-BE-NEXT:    vmov.32 q0[1], r12
+; CHECK-BE-NEXT:    @ implicit-def: $q2
+; CHECK-BE-NEXT:    vmov.32 q0[3], lr
+; CHECK-BE-NEXT:    mov.w lr, #0
+; CHECK-BE-NEXT:    it lt
+; CHECK-BE-NEXT:    movlt.w lr, #1
+; CHECK-BE-NEXT:    rsbs.w r1, r12, #0
+; CHECK-BE-NEXT:    sbcs.w r1, r3, r12, asr #31
+; CHECK-BE-NEXT:    it lt
+; CHECK-BE-NEXT:    movlt r3, #1
+; CHECK-BE-NEXT:    cmp r3, #0
+; CHECK-BE-NEXT:    it ne
+; CHECK-BE-NEXT:    mvnne r3, #1
+; CHECK-BE-NEXT:    bfi r3, lr, #0, #1
+; CHECK-BE-NEXT:    and r1, r3, #3
+; CHECK-BE-NEXT:    lsls r3, r3, #31
+; CHECK-BE-NEXT:    beq .LBB5_2
+; CHECK-BE-NEXT:  @ %bb.1: @ %cond.load
+; CHECK-BE-NEXT:    ldr r3, [r2]
+; CHECK-BE-NEXT:    vmov.32 q1[1], r3
+; CHECK-BE-NEXT:    vrev64.32 q2, q1
+; CHECK-BE-NEXT:  .LBB5_2: @ %else
+; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    lsls r1, r1, #30
+; CHECK-BE-NEXT:    bpl .LBB5_4
+; CHECK-BE-NEXT:  @ %bb.3: @ %cond.load1
+; CHECK-BE-NEXT:    ldr r1, [r2, #4]
+; CHECK-BE-NEXT:    vrev64.32 q0, q2
+; CHECK-BE-NEXT:    vmov.32 q0[3], r1
+; CHECK-BE-NEXT:    vrev64.32 q2, q0
+; CHECK-BE-NEXT:  .LBB5_4: @ %else2
+; CHECK-BE-NEXT:    vrev64.32 q0, q2
+; CHECK-BE-NEXT:    vrev64.32 q2, q1
+; CHECK-BE-NEXT:    vmov r2, s11
+; CHECK-BE-NEXT:    movs r4, #0
+; CHECK-BE-NEXT:    vmov r3, s1
+; CHECK-BE-NEXT:    vmov r1, s3
+; CHECK-BE-NEXT:    rsbs r5, r2, #0
+; CHECK-BE-NEXT:    sbcs.w r2, r4, r2, asr #31
+; CHECK-BE-NEXT:    vmov r2, s9
+; CHECK-BE-NEXT:    asr.w lr, r3, #31
+; CHECK-BE-NEXT:    vmov.32 q1[0], lr
+; CHECK-BE-NEXT:    asr.w r12, r1, #31
+; CHECK-BE-NEXT:    vmov.32 q1[1], r3
+; CHECK-BE-NEXT:    vmov.32 q1[2], r12
+; CHECK-BE-NEXT:    vmov.32 q1[3], r1
+; CHECK-BE-NEXT:    mov.w r1, #0
+; CHECK-BE-NEXT:    it lt
+; CHECK-BE-NEXT:    movlt r1, #1
+; CHECK-BE-NEXT:    vrev64.32 q0, q1
+; CHECK-BE-NEXT:    rsbs r3, r2, #0
+; CHECK-BE-NEXT:    sbcs.w r2, r4, r2, asr #31
+; CHECK-BE-NEXT:    it lt
+; CHECK-BE-NEXT:    movlt r4, #1
+; CHECK-BE-NEXT:    cmp r4, #0
+; CHECK-BE-NEXT:    it ne
+; CHECK-BE-NEXT:    mvnne r4, #1
+; CHECK-BE-NEXT:    bfi r4, r1, #0, #1
+; CHECK-BE-NEXT:    and r1, r4, #3
+; CHECK-BE-NEXT:    lsls r2, r4, #31
+; CHECK-BE-NEXT:    it ne
+; CHECK-BE-NEXT:    vstrne d0, [r0]
+; CHECK-BE-NEXT:    lsls r1, r1, #30
+; CHECK-BE-NEXT:    it mi
+; CHECK-BE-NEXT:    vstrmi d1, [r0, #8]
+; CHECK-BE-NEXT:    add sp, #4
+; CHECK-BE-NEXT:    pop {r4, r5, r7, pc}
+entry:
+  %0 = load <2 x i32>, <2 x i32>* %mask, align 4
+  %1 = icmp sgt <2 x i32> %0, zeroinitializer
+  %2 = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* %src, i32 4, <2 x i1> %1, <2 x i32> undef)
+  %3 = sext <2 x i32> %2 to <2 x i64>
+  call void @llvm.masked.store.v2i64.p0v2i64(<2 x i64> %3, <2 x i64>* %dest, i32 8, <2 x i1> %1)
+  ret void
+}
+
+define void @foo_sext_v2i64_v2i32_unaligned(<2 x i64> *%dest, <2 x i32> *%mask, <2 x i32> *%src) {
+; CHECK-LE-LABEL: foo_sext_v2i64_v2i32_unaligned:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    .save {r4, r5, r7, lr}
+; CHECK-LE-NEXT:    push {r4, r5, r7, lr}
+; CHECK-LE-NEXT:    .pad #4
+; CHECK-LE-NEXT:    sub sp, #4
+; CHECK-LE-NEXT:    ldrd lr, r12, [r1]
+; CHECK-LE-NEXT:    movs r1, #0
+; CHECK-LE-NEXT:    @ implicit-def: $q1
+; CHECK-LE-NEXT:    movs r4, #0
+; CHECK-LE-NEXT:    rsbs.w r3, lr, #0
+; CHECK-LE-NEXT:    vmov.32 q0[0], lr
+; CHECK-LE-NEXT:    sbcs.w r3, r1, lr, asr #31
+; CHECK-LE-NEXT:    mov.w lr, #0
+; CHECK-LE-NEXT:    it lt
+; CHECK-LE-NEXT:    movlt.w lr, #1
+; CHECK-LE-NEXT:    rsbs.w r3, r12, #0
+; CHECK-LE-NEXT:    sbcs.w r3, r1, r12, asr #31
+; CHECK-LE-NEXT:    it lt
+; CHECK-LE-NEXT:    movlt r1, #1
+; CHECK-LE-NEXT:    cmp r1, #0
+; CHECK-LE-NEXT:    it ne
+; CHECK-LE-NEXT:    mvnne r1, #1
+; CHECK-LE-NEXT:    bfi r1, lr, #0, #1
+; CHECK-LE-NEXT:    vmov.32 q0[2], r12
+; CHECK-LE-NEXT:    and r3, r1, #3
+; CHECK-LE-NEXT:    lsls r1, r1, #31
+; CHECK-LE-NEXT:    itt ne
+; CHECK-LE-NEXT:    ldrne r1, [r2]
+; CHECK-LE-NEXT:    vmovne.32 q1[0], r1
+; CHECK-LE-NEXT:    lsls r1, r3, #30
+; CHECK-LE-NEXT:    itt mi
+; CHECK-LE-NEXT:    ldrmi r1, [r2, #4]
+; CHECK-LE-NEXT:    vmovmi.32 q1[2], r1
+; CHECK-LE-NEXT:    vmov r2, s0
+; CHECK-LE-NEXT:    vmov r3, s4
+; CHECK-LE-NEXT:    vmov r1, s6
+; CHECK-LE-NEXT:    vmov.32 q1[0], r3
+; CHECK-LE-NEXT:    rsbs r5, r2, #0
+; CHECK-LE-NEXT:    sbcs.w r2, r4, r2, asr #31
+; CHECK-LE-NEXT:    vmov r2, s2
+; CHECK-LE-NEXT:    asr.w lr, r3, #31
+; CHECK-LE-NEXT:    vmov.32 q1[1], lr
+; CHECK-LE-NEXT:    asr.w r12, r1, #31
+; CHECK-LE-NEXT:    vmov.32 q1[2], r1
+; CHECK-LE-NEXT:    mov.w r1, #0
+; CHECK-LE-NEXT:    it lt
+; CHECK-LE-NEXT:    movlt r1, #1
+; CHECK-LE-NEXT:    vmov.32 q1[3], r12
+; CHECK-LE-NEXT:    rsbs r3, r2, #0
+; CHECK-LE-NEXT:    sbcs.w r2, r4, r2, asr #31
+; CHECK-LE-NEXT:    it lt
+; CHECK-LE-NEXT:    movlt r4, #1
+; CHECK-LE-NEXT:    cmp r4, #0
+; CHECK-LE-NEXT:    it ne
+; CHECK-LE-NEXT:    mvnne r4, #1
+; CHECK-LE-NEXT:    bfi r4, r1, #0, #1
+; CHECK-LE-NEXT:    and r1, r4, #3
+; CHECK-LE-NEXT:    lsls r2, r4, #31
+; CHECK-LE-NEXT:    itt ne
+; CHECK-LE-NEXT:    vmovne r2, r3, d2
+; CHECK-LE-NEXT:    strdne r2, r3, [r0]
+; CHECK-LE-NEXT:    lsls r1, r1, #30
+; CHECK-LE-NEXT:    itt mi
+; CHECK-LE-NEXT:    vmovmi r1, r2, d3
+; CHECK-LE-NEXT:    strdmi r1, r2, [r0, #8]
+; CHECK-LE-NEXT:    add sp, #4
+; CHECK-LE-NEXT:    pop {r4, r5, r7, pc}
+;
+; CHECK-BE-LABEL: foo_sext_v2i64_v2i32_unaligned:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    .save {r4, r5, r7, lr}
+; CHECK-BE-NEXT:    push {r4, r5, r7, lr}
+; CHECK-BE-NEXT:    .pad #4
+; CHECK-BE-NEXT:    sub sp, #4
+; CHECK-BE-NEXT:    ldrd r12, lr, [r1]
+; CHECK-BE-NEXT:    rsbs.w r1, lr, #0
+; CHECK-BE-NEXT:    mov.w r3, #0
+; CHECK-BE-NEXT:    sbcs.w r1, r3, lr, asr #31
+; CHECK-BE-NEXT:    vmov.32 q0[1], r12
+; CHECK-BE-NEXT:    @ implicit-def: $q2
+; CHECK-BE-NEXT:    vmov.32 q0[3], lr
+; CHECK-BE-NEXT:    mov.w lr, #0
+; CHECK-BE-NEXT:    it lt
+; CHECK-BE-NEXT:    movlt.w lr, #1
+; CHECK-BE-NEXT:    rsbs.w r1, r12, #0
+; CHECK-BE-NEXT:    sbcs.w r1, r3, r12, asr #31
+; CHECK-BE-NEXT:    it lt
+; CHECK-BE-NEXT:    movlt r3, #1
+; CHECK-BE-NEXT:    cmp r3, #0
+; CHECK-BE-NEXT:    it ne
+; CHECK-BE-NEXT:    mvnne r3, #1
+; CHECK-BE-NEXT:    bfi r3, lr, #0, #1
+; CHECK-BE-NEXT:    and r1, r3, #3
+; CHECK-BE-NEXT:    lsls r3, r3, #31
+; CHECK-BE-NEXT:    beq .LBB6_2
+; CHECK-BE-NEXT:  @ %bb.1: @ %cond.load
+; CHECK-BE-NEXT:    ldr r3, [r2]
+; CHECK-BE-NEXT:    vmov.32 q1[1], r3
+; CHECK-BE-NEXT:    vrev64.32 q2, q1
+; CHECK-BE-NEXT:  .LBB6_2: @ %else
+; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    lsls r1, r1, #30
+; CHECK-BE-NEXT:    bpl .LBB6_4
+; CHECK-BE-NEXT:  @ %bb.3: @ %cond.load1
+; CHECK-BE-NEXT:    ldr r1, [r2, #4]
+; CHECK-BE-NEXT:    vrev64.32 q0, q2
+; CHECK-BE-NEXT:    vmov.32 q0[3], r1
+; CHECK-BE-NEXT:    vrev64.32 q2, q0
+; CHECK-BE-NEXT:  .LBB6_4: @ %else2
+; CHECK-BE-NEXT:    vrev64.32 q0, q2
+; CHECK-BE-NEXT:    vrev64.32 q2, q1
+; CHECK-BE-NEXT:    vmov r2, s11
+; CHECK-BE-NEXT:    movs r4, #0
+; CHECK-BE-NEXT:    vmov r3, s1
+; CHECK-BE-NEXT:    vmov r1, s3
+; CHECK-BE-NEXT:    rsbs r5, r2, #0
+; CHECK-BE-NEXT:    sbcs.w r2, r4, r2, asr #31
+; CHECK-BE-NEXT:    vmov r2, s9
+; CHECK-BE-NEXT:    asr.w lr, r3, #31
+; CHECK-BE-NEXT:    vmov.32 q1[0], lr
+; CHECK-BE-NEXT:    asr.w r12, r1, #31
+; CHECK-BE-NEXT:    vmov.32 q1[1], r3
+; CHECK-BE-NEXT:    vmov.32 q1[2], r12
+; CHECK-BE-NEXT:    vmov.32 q1[3], r1
+; CHECK-BE-NEXT:    mov.w r1, #0
+; CHECK-BE-NEXT:    it lt
+; CHECK-BE-NEXT:    movlt r1, #1
+; CHECK-BE-NEXT:    vrev64.32 q0, q1
+; CHECK-BE-NEXT:    rsbs r3, r2, #0
+; CHECK-BE-NEXT:    sbcs.w r2, r4, r2, asr #31
+; CHECK-BE-NEXT:    it lt
+; CHECK-BE-NEXT:    movlt r4, #1
+; CHECK-BE-NEXT:    cmp r4, #0
+; CHECK-BE-NEXT:    it ne
+; CHECK-BE-NEXT:    mvnne r4, #1
+; CHECK-BE-NEXT:    bfi r4, r1, #0, #1
+; CHECK-BE-NEXT:    and r1, r4, #3
+; CHECK-BE-NEXT:    lsls r2, r4, #31
+; CHECK-BE-NEXT:    itt ne
+; CHECK-BE-NEXT:    vmovne r2, r3, d0
+; CHECK-BE-NEXT:    strdne r3, r2, [r0]
+; CHECK-BE-NEXT:    lsls r1, r1, #30
+; CHECK-BE-NEXT:    itt mi
+; CHECK-BE-NEXT:    vmovmi r1, r2, d1
+; CHECK-BE-NEXT:    strdmi r2, r1, [r0, #8]
+; CHECK-BE-NEXT:    add sp, #4
+; CHECK-BE-NEXT:    pop {r4, r5, r7, pc}
+entry:
+  %0 = load <2 x i32>, <2 x i32>* %mask, align 4
+  %1 = icmp sgt <2 x i32> %0, zeroinitializer
+  %2 = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* %src, i32 2, <2 x i1> %1, <2 x i32> undef)
+  %3 = sext <2 x i32> %2 to <2 x i64>
+  call void @llvm.masked.store.v2i64.p0v2i64(<2 x i64> %3, <2 x i64>* %dest, i32 4, <2 x i1> %1)
+  ret void
+}
+
+define void @foo_zext_v2i64_v2i32(<2 x i64> *%dest, <2 x i32> *%mask, <2 x i32> *%src) {
+; CHECK-LE-LABEL: foo_zext_v2i64_v2i32:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    .save {r7, lr}
+; CHECK-LE-NEXT:    push {r7, lr}
+; CHECK-LE-NEXT:    .pad #4
+; CHECK-LE-NEXT:    sub sp, #4
+; CHECK-LE-NEXT:    ldrd lr, r12, [r1]
+; CHECK-LE-NEXT:    movs r1, #0
+; CHECK-LE-NEXT:    @ implicit-def: $q1
+; CHECK-LE-NEXT:    rsbs.w r3, lr, #0
+; CHECK-LE-NEXT:    vmov.32 q0[0], lr
+; CHECK-LE-NEXT:    sbcs.w r3, r1, lr, asr #31
+; CHECK-LE-NEXT:    mov.w lr, #0
+; CHECK-LE-NEXT:    it lt
+; CHECK-LE-NEXT:    movlt.w lr, #1
+; CHECK-LE-NEXT:    rsbs.w r3, r12, #0
+; CHECK-LE-NEXT:    sbcs.w r3, r1, r12, asr #31
+; CHECK-LE-NEXT:    it lt
+; CHECK-LE-NEXT:    movlt r1, #1
+; CHECK-LE-NEXT:    cmp r1, #0
+; CHECK-LE-NEXT:    it ne
+; CHECK-LE-NEXT:    mvnne r1, #1
+; CHECK-LE-NEXT:    bfi r1, lr, #0, #1
+; CHECK-LE-NEXT:    vmov.32 q0[2], r12
+; CHECK-LE-NEXT:    and r3, r1, #3
+; CHECK-LE-NEXT:    adr.w r12, .LCPI7_0
+; CHECK-LE-NEXT:    lsls r1, r1, #31
+; CHECK-LE-NEXT:    itt ne
+; CHECK-LE-NEXT:    ldrne r1, [r2]
+; CHECK-LE-NEXT:    vmovne.32 q1[0], r1
+; CHECK-LE-NEXT:    lsls r1, r3, #30
+; CHECK-LE-NEXT:    vmov r3, s0
+; CHECK-LE-NEXT:    itt mi
+; CHECK-LE-NEXT:    ldrmi r1, [r2, #4]
+; CHECK-LE-NEXT:    vmovmi.32 q1[2], r1
+; CHECK-LE-NEXT:    movs r2, #0
+; CHECK-LE-NEXT:    vldrw.u32 q2, [r12]
+; CHECK-LE-NEXT:    mov.w r12, #0
+; CHECK-LE-NEXT:    vand q1, q1, q2
+; CHECK-LE-NEXT:    rsbs r1, r3, #0
+; CHECK-LE-NEXT:    sbcs.w r1, r2, r3, asr #31
+; CHECK-LE-NEXT:    vmov r3, s2
+; CHECK-LE-NEXT:    it lt
+; CHECK-LE-NEXT:    movlt.w r12, #1
+; CHECK-LE-NEXT:    rsbs r1, r3, #0
+; CHECK-LE-NEXT:    sbcs.w r1, r2, r3, asr #31
+; CHECK-LE-NEXT:    it lt
+; CHECK-LE-NEXT:    movlt r2, #1
+; CHECK-LE-NEXT:    cmp r2, #0
+; CHECK-LE-NEXT:    it ne
+; CHECK-LE-NEXT:    mvnne r2, #1
+; CHECK-LE-NEXT:    bfi r2, r12, #0, #1
+; CHECK-LE-NEXT:    and r1, r2, #3
+; CHECK-LE-NEXT:    lsls r2, r2, #31
+; CHECK-LE-NEXT:    it ne
+; CHECK-LE-NEXT:    vstrne d2, [r0]
+; CHECK-LE-NEXT:    lsls r1, r1, #30
+; CHECK-LE-NEXT:    it mi
+; CHECK-LE-NEXT:    vstrmi d3, [r0, #8]
+; CHECK-LE-NEXT:    add sp, #4
+; CHECK-LE-NEXT:    pop {r7, pc}
+; CHECK-LE-NEXT:    .p2align 4
+; CHECK-LE-NEXT:  @ %bb.1:
+; CHECK-LE-NEXT:  .LCPI7_0:
+; CHECK-LE-NEXT:    .long 4294967295 @ 0xffffffff
+; CHECK-LE-NEXT:    .long 0 @ 0x0
+; CHECK-LE-NEXT:    .long 4294967295 @ 0xffffffff
+; CHECK-LE-NEXT:    .long 0 @ 0x0
+;
+; CHECK-BE-LABEL: foo_zext_v2i64_v2i32:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    .save {r7, lr}
+; CHECK-BE-NEXT:    push {r7, lr}
+; CHECK-BE-NEXT:    .pad #4
+; CHECK-BE-NEXT:    sub sp, #4
+; CHECK-BE-NEXT:    ldrd r12, lr, [r1]
+; CHECK-BE-NEXT:    rsbs.w r1, lr, #0
+; CHECK-BE-NEXT:    mov.w r3, #0
+; CHECK-BE-NEXT:    sbcs.w r1, r3, lr, asr #31
+; CHECK-BE-NEXT:    vmov.32 q0[1], r12
+; CHECK-BE-NEXT:    @ implicit-def: $q1
+; CHECK-BE-NEXT:    vmov.32 q0[3], lr
+; CHECK-BE-NEXT:    mov.w lr, #0
+; CHECK-BE-NEXT:    it lt
+; CHECK-BE-NEXT:    movlt.w lr, #1
+; CHECK-BE-NEXT:    rsbs.w r1, r12, #0
+; CHECK-BE-NEXT:    sbcs.w r1, r3, r12, asr #31
+; CHECK-BE-NEXT:    it lt
+; CHECK-BE-NEXT:    movlt r3, #1
+; CHECK-BE-NEXT:    cmp r3, #0
+; CHECK-BE-NEXT:    it ne
+; CHECK-BE-NEXT:    mvnne r3, #1
+; CHECK-BE-NEXT:    bfi r3, lr, #0, #1
+; CHECK-BE-NEXT:    and r1, r3, #3
+; CHECK-BE-NEXT:    lsls r3, r3, #31
+; CHECK-BE-NEXT:    beq .LBB7_2
+; CHECK-BE-NEXT:  @ %bb.1: @ %cond.load
+; CHECK-BE-NEXT:    ldr r3, [r2]
+; CHECK-BE-NEXT:    vmov.32 q2[1], r3
+; CHECK-BE-NEXT:    vrev64.32 q1, q2
+; CHECK-BE-NEXT:  .LBB7_2: @ %else
+; CHECK-BE-NEXT:    vrev64.32 q2, q0
+; CHECK-BE-NEXT:    lsls r1, r1, #30
+; CHECK-BE-NEXT:    bpl .LBB7_4
+; CHECK-BE-NEXT:  @ %bb.3: @ %cond.load1
+; CHECK-BE-NEXT:    ldr r1, [r2, #4]
+; CHECK-BE-NEXT:    vrev64.32 q0, q1
+; CHECK-BE-NEXT:    vmov.32 q0[3], r1
+; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:  .LBB7_4: @ %else2
+; CHECK-BE-NEXT:    vrev64.32 q3, q2
+; CHECK-BE-NEXT:    movs r2, #0
+; CHECK-BE-NEXT:    vmov r3, s15
+; CHECK-BE-NEXT:    adr.w r12, .LCPI7_0
+; CHECK-BE-NEXT:    vldrb.u8 q0, [r12]
+; CHECK-BE-NEXT:    mov.w r12, #0
+; CHECK-BE-NEXT:    vrev64.8 q2, q0
+; CHECK-BE-NEXT:    vand q0, q1, q2
+; CHECK-BE-NEXT:    rsbs r1, r3, #0
+; CHECK-BE-NEXT:    sbcs.w r1, r2, r3, asr #31
+; CHECK-BE-NEXT:    vmov r3, s13
+; CHECK-BE-NEXT:    it lt
+; CHECK-BE-NEXT:    movlt.w r12, #1
+; CHECK-BE-NEXT:    rsbs r1, r3, #0
+; CHECK-BE-NEXT:    sbcs.w r1, r2, r3, asr #31
+; CHECK-BE-NEXT:    it lt
+; CHECK-BE-NEXT:    movlt r2, #1
+; CHECK-BE-NEXT:    cmp r2, #0
+; CHECK-BE-NEXT:    it ne
+; CHECK-BE-NEXT:    mvnne r2, #1
+; CHECK-BE-NEXT:    bfi r2, r12, #0, #1
+; CHECK-BE-NEXT:    and r1, r2, #3
+; CHECK-BE-NEXT:    lsls r2, r2, #31
+; CHECK-BE-NEXT:    it ne
+; CHECK-BE-NEXT:    vstrne d0, [r0]
+; CHECK-BE-NEXT:    lsls r1, r1, #30
+; CHECK-BE-NEXT:    it mi
+; CHECK-BE-NEXT:    vstrmi d1, [r0, #8]
+; CHECK-BE-NEXT:    add sp, #4
+; CHECK-BE-NEXT:    pop {r7, pc}
+; CHECK-BE-NEXT:    .p2align 4
+; CHECK-BE-NEXT:  @ %bb.5:
+; CHECK-BE-NEXT:  .LCPI7_0:
+; CHECK-BE-NEXT:    .long 0 @ 0x0
+; CHECK-BE-NEXT:    .long 4294967295 @ 0xffffffff
+; CHECK-BE-NEXT:    .long 0 @ 0x0
+; CHECK-BE-NEXT:    .long 4294967295 @ 0xffffffff
+entry:
+  %0 = load <2 x i32>, <2 x i32>* %mask, align 4
+  %1 = icmp sgt <2 x i32> %0, zeroinitializer
+  %2 = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* %src, i32 4, <2 x i1> %1, <2 x i32> undef)
+  %3 = zext <2 x i32> %2 to <2 x i64>
+  call void @llvm.masked.store.v2i64.p0v2i64(<2 x i64> %3, <2 x i64>* %dest, i32 8, <2 x i1> %1)
+  ret void
+}
+
+define void @foo_zext_v2i64_v2i32_unaligned(<2 x i64> *%dest, <2 x i32> *%mask, <2 x i32> *%src) {
+; CHECK-LE-LABEL: foo_zext_v2i64_v2i32_unaligned:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    .save {r7, lr}
+; CHECK-LE-NEXT:    push {r7, lr}
+; CHECK-LE-NEXT:    .pad #4
+; CHECK-LE-NEXT:    sub sp, #4
+; CHECK-LE-NEXT:    ldrd lr, r12, [r1]
+; CHECK-LE-NEXT:    movs r1, #0
+; CHECK-LE-NEXT:    @ implicit-def: $q1
+; CHECK-LE-NEXT:    rsbs.w r3, lr, #0
+; CHECK-LE-NEXT:    vmov.32 q0[0], lr
+; CHECK-LE-NEXT:    sbcs.w r3, r1, lr, asr #31
+; CHECK-LE-NEXT:    mov.w lr, #0
+; CHECK-LE-NEXT:    it lt
+; CHECK-LE-NEXT:    movlt.w lr, #1
+; CHECK-LE-NEXT:    rsbs.w r3, r12, #0
+; CHECK-LE-NEXT:    sbcs.w r3, r1, r12, asr #31
+; CHECK-LE-NEXT:    it lt
+; CHECK-LE-NEXT:    movlt r1, #1
+; CHECK-LE-NEXT:    cmp r1, #0
+; CHECK-LE-NEXT:    it ne
+; CHECK-LE-NEXT:    mvnne r1, #1
+; CHECK-LE-NEXT:    bfi r1, lr, #0, #1
+; CHECK-LE-NEXT:    vmov.32 q0[2], r12
+; CHECK-LE-NEXT:    and r3, r1, #3
+; CHECK-LE-NEXT:    adr.w r12, .LCPI8_0
+; CHECK-LE-NEXT:    lsls r1, r1, #31
+; CHECK-LE-NEXT:    itt ne
+; CHECK-LE-NEXT:    ldrne r1, [r2]
+; CHECK-LE-NEXT:    vmovne.32 q1[0], r1
+; CHECK-LE-NEXT:    lsls r1, r3, #30
+; CHECK-LE-NEXT:    vmov r3, s0
+; CHECK-LE-NEXT:    itt mi
+; CHECK-LE-NEXT:    ldrmi r1, [r2, #4]
+; CHECK-LE-NEXT:    vmovmi.32 q1[2], r1
+; CHECK-LE-NEXT:    movs r2, #0
+; CHECK-LE-NEXT:    vldrw.u32 q2, [r12]
+; CHECK-LE-NEXT:    mov.w r12, #0
+; CHECK-LE-NEXT:    vand q1, q1, q2
+; CHECK-LE-NEXT:    rsbs r1, r3, #0
+; CHECK-LE-NEXT:    sbcs.w r1, r2, r3, asr #31
+; CHECK-LE-NEXT:    vmov r3, s2
+; CHECK-LE-NEXT:    it lt
+; CHECK-LE-NEXT:    movlt.w r12, #1
+; CHECK-LE-NEXT:    rsbs r1, r3, #0
+; CHECK-LE-NEXT:    sbcs.w r1, r2, r3, asr #31
+; CHECK-LE-NEXT:    it lt
+; CHECK-LE-NEXT:    movlt r2, #1
+; CHECK-LE-NEXT:    cmp r2, #0
+; CHECK-LE-NEXT:    it ne
+; CHECK-LE-NEXT:    mvnne r2, #1
+; CHECK-LE-NEXT:    bfi r2, r12, #0, #1
+; CHECK-LE-NEXT:    and r1, r2, #3
+; CHECK-LE-NEXT:    lsls r2, r2, #31
+; CHECK-LE-NEXT:    itt ne
+; CHECK-LE-NEXT:    vmovne r2, r3, d2
+; CHECK-LE-NEXT:    strdne r2, r3, [r0]
+; CHECK-LE-NEXT:    lsls r1, r1, #30
+; CHECK-LE-NEXT:    itt mi
+; CHECK-LE-NEXT:    vmovmi r1, r2, d3
+; CHECK-LE-NEXT:    strdmi r1, r2, [r0, #8]
+; CHECK-LE-NEXT:    add sp, #4
+; CHECK-LE-NEXT:    pop {r7, pc}
+; CHECK-LE-NEXT:    .p2align 4
+; CHECK-LE-NEXT:  @ %bb.1:
+; CHECK-LE-NEXT:  .LCPI8_0:
+; CHECK-LE-NEXT:    .long 4294967295 @ 0xffffffff
+; CHECK-LE-NEXT:    .long 0 @ 0x0
+; CHECK-LE-NEXT:    .long 4294967295 @ 0xffffffff
+; CHECK-LE-NEXT:    .long 0 @ 0x0
+;
+; CHECK-BE-LABEL: foo_zext_v2i64_v2i32_unaligned:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    .save {r7, lr}
+; CHECK-BE-NEXT:    push {r7, lr}
+; CHECK-BE-NEXT:    .pad #4
+; CHECK-BE-NEXT:    sub sp, #4
+; CHECK-BE-NEXT:    ldrd r12, lr, [r1]
+; CHECK-BE-NEXT:    rsbs.w r1, lr, #0
+; CHECK-BE-NEXT:    mov.w r3, #0
+; CHECK-BE-NEXT:    sbcs.w r1, r3, lr, asr #31
+; CHECK-BE-NEXT:    vmov.32 q0[1], r12
+; CHECK-BE-NEXT:    @ implicit-def: $q1
+; CHECK-BE-NEXT:    vmov.32 q0[3], lr
+; CHECK-BE-NEXT:    mov.w lr, #0
+; CHECK-BE-NEXT:    it lt
+; CHECK-BE-NEXT:    movlt.w lr, #1
+; CHECK-BE-NEXT:    rsbs.w r1, r12, #0
+; CHECK-BE-NEXT:    sbcs.w r1, r3, r12, asr #31
+; CHECK-BE-NEXT:    it lt
+; CHECK-BE-NEXT:    movlt r3, #1
+; CHECK-BE-NEXT:    cmp r3, #0
+; CHECK-BE-NEXT:    it ne
+; CHECK-BE-NEXT:    mvnne r3, #1
+; CHECK-BE-NEXT:    bfi r3, lr, #0, #1
+; CHECK-BE-NEXT:    and r1, r3, #3
+; CHECK-BE-NEXT:    lsls r3, r3, #31
+; CHECK-BE-NEXT:    beq .LBB8_2
+; CHECK-BE-NEXT:  @ %bb.1: @ %cond.load
+; CHECK-BE-NEXT:    ldr r3, [r2]
+; CHECK-BE-NEXT:    vmov.32 q2[1], r3
+; CHECK-BE-NEXT:    vrev64.32 q1, q2
+; CHECK-BE-NEXT:  .LBB8_2: @ %else
+; CHECK-BE-NEXT:    vrev64.32 q2, q0
+; CHECK-BE-NEXT:    lsls r1, r1, #30
+; CHECK-BE-NEXT:    bpl .LBB8_4
+; CHECK-BE-NEXT:  @ %bb.3: @ %cond.load1
+; CHECK-BE-NEXT:    ldr r1, [r2, #4]
+; CHECK-BE-NEXT:    vrev64.32 q0, q1
+; CHECK-BE-NEXT:    vmov.32 q0[3], r1
+; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:  .LBB8_4: @ %else2
+; CHECK-BE-NEXT:    vrev64.32 q3, q2
+; CHECK-BE-NEXT:    movs r2, #0
+; CHECK-BE-NEXT:    vmov r3, s15
+; CHECK-BE-NEXT:    adr.w r12, .LCPI8_0
+; CHECK-BE-NEXT:    vldrb.u8 q0, [r12]
+; CHECK-BE-NEXT:    mov.w r12, #0
+; CHECK-BE-NEXT:    vrev64.8 q2, q0
+; CHECK-BE-NEXT:    vand q0, q1, q2
+; CHECK-BE-NEXT:    rsbs r1, r3, #0
+; CHECK-BE-NEXT:    sbcs.w r1, r2, r3, asr #31
+; CHECK-BE-NEXT:    vmov r3, s13
+; CHECK-BE-NEXT:    it lt
+; CHECK-BE-NEXT:    movlt.w r12, #1
+; CHECK-BE-NEXT:    rsbs r1, r3, #0
+; CHECK-BE-NEXT:    sbcs.w r1, r2, r3, asr #31
+; CHECK-BE-NEXT:    it lt
+; CHECK-BE-NEXT:    movlt r2, #1
+; CHECK-BE-NEXT:    cmp r2, #0
+; CHECK-BE-NEXT:    it ne
+; CHECK-BE-NEXT:    mvnne r2, #1
+; CHECK-BE-NEXT:    bfi r2, r12, #0, #1
+; CHECK-BE-NEXT:    and r1, r2, #3
+; CHECK-BE-NEXT:    lsls r2, r2, #31
+; CHECK-BE-NEXT:    itt ne
+; CHECK-BE-NEXT:    vmovne r2, r3, d0
+; CHECK-BE-NEXT:    strdne r3, r2, [r0]
+; CHECK-BE-NEXT:    lsls r1, r1, #30
+; CHECK-BE-NEXT:    itt mi
+; CHECK-BE-NEXT:    vmovmi r1, r2, d1
+; CHECK-BE-NEXT:    strdmi r2, r1, [r0, #8]
+; CHECK-BE-NEXT:    add sp, #4
+; CHECK-BE-NEXT:    pop {r7, pc}
+; CHECK-BE-NEXT:    .p2align 4
+; CHECK-BE-NEXT:  @ %bb.5:
+; CHECK-BE-NEXT:  .LCPI8_0:
+; CHECK-BE-NEXT:    .long 0 @ 0x0
+; CHECK-BE-NEXT:    .long 4294967295 @ 0xffffffff
+; CHECK-BE-NEXT:    .long 0 @ 0x0
+; CHECK-BE-NEXT:    .long 4294967295 @ 0xffffffff
+entry:
+  %0 = load <2 x i32>, <2 x i32>* %mask, align 4
+  %1 = icmp sgt <2 x i32> %0, zeroinitializer
+  %2 = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* %src, i32 2, <2 x i1> %1, <2 x i32> undef)
+  %3 = zext <2 x i32> %2 to <2 x i64>
+  call void @llvm.masked.store.v2i64.p0v2i64(<2 x i64> %3, <2 x i64>* %dest, i32 4, <2 x i1> %1)
+  ret void
+}
+
+define void @foo_v8i16_v8i16(<8 x i16> *%dest, <8 x i16> *%mask, <8 x i16> *%src) {
+; CHECK-LABEL: foo_v8i16_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-NEXT:    vptt.s16 gt, q0, zr
+; CHECK-NEXT:    vldrht.u16 q0, [r2]
+; CHECK-NEXT:    vstrht.16 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = load <8 x i16>, <8 x i16>* %mask, align 2
+  %1 = icmp sgt <8 x i16> %0, zeroinitializer
+  %2 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %src, i32 2, <8 x i1> %1, <8 x i16> undef)
+  call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %2, <8 x i16>* %dest, i32 2, <8 x i1> %1)
+  ret void
+}
+
+define void @foo_sext_v8i16_v8i8(<8 x i16> *%dest, <8 x i16> *%mask, <8 x i8> *%src) {
+; CHECK-LABEL: foo_sext_v8i16_v8i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-NEXT:    vptt.s16 gt, q0, zr
+; CHECK-NEXT:    vldrbt.s16 q0, [r2]
+; CHECK-NEXT:    vstrht.16 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = load <8 x i16>, <8 x i16>* %mask, align 2
+  %1 = icmp sgt <8 x i16> %0, zeroinitializer
+  %2 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %src, i32 1, <8 x i1> %1, <8 x i8> undef)
+  %3 = sext <8 x i8> %2 to <8 x i16>
+  call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %3, <8 x i16>* %dest, i32 2, <8 x i1> %1)
+  ret void
+}
+
+define void @foo_zext_v8i16_v8i8(<8 x i16> *%dest, <8 x i16> *%mask, <8 x i8> *%src) {
+; CHECK-LABEL: foo_zext_v8i16_v8i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-NEXT:    vptt.s16 gt, q0, zr
+; CHECK-NEXT:    vldrbt.u16 q0, [r2]
+; CHECK-NEXT:    vstrht.16 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = load <8 x i16>, <8 x i16>* %mask, align 2
+  %1 = icmp sgt <8 x i16> %0, zeroinitializer
+  %2 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %src, i32 1, <8 x i1> %1, <8 x i8> undef)
+  %3 = zext <8 x i8> %2 to <8 x i16>
+  call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %3, <8 x i16>* %dest, i32 2, <8 x i1> %1)
+  ret void
+}
+
+define void @foo_v16i8_v16i8(<16 x i8> *%dest, <16 x i8> *%mask, <16 x i8> *%src) {
+; CHECK-LABEL: foo_v16i8_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u8 q0, [r1]
+; CHECK-NEXT:    vptt.s8 gt, q0, zr
+; CHECK-NEXT:    vldrbt.u8 q0, [r2]
+; CHECK-NEXT:    vstrbt.8 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = load <16 x i8>, <16 x i8>* %mask, align 1
+  %1 = icmp sgt <16 x i8> %0, zeroinitializer
+  %2 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %src, i32 1, <16 x i1> %1, <16 x i8> undef)
+  call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %2, <16 x i8>* %dest, i32 1, <16 x i1> %1)
+  ret void
+}
+
+define void @foo_trunc_v8i8_v8i16(<8 x i8> *%dest, <8 x i16> *%mask, <8 x i16> *%src) {
+; CHECK-LABEL: foo_trunc_v8i8_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-NEXT:    vptt.s16 gt, q0, zr
+; CHECK-NEXT:    vldrht.u16 q0, [r2]
+; CHECK-NEXT:    vstrbt.16 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = load <8 x i16>, <8 x i16>* %mask, align 2
+  %1 = icmp sgt <8 x i16> %0, zeroinitializer
+  %2 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %src, i32 2, <8 x i1> %1, <8 x i16> undef)
+  %3 = trunc <8 x i16> %2 to <8 x i8>
+  call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %3, <8 x i8>* %dest, i32 1, <8 x i1> %1)
+  ret void
+}
+
+define void @foo_trunc_v4i8_v4i32(<4 x i8> *%dest, <4 x i32> *%mask, <4 x i32> *%src) {
+; CHECK-LABEL: foo_trunc_v4i8_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    vptt.s32 gt, q0, zr
+; CHECK-NEXT:    vldrwt.u32 q0, [r2]
+; CHECK-NEXT:    vstrbt.32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = load <4 x i32>, <4 x i32>* %mask, align 4
+  %1 = icmp sgt <4 x i32> %0, zeroinitializer
+  %2 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %src, i32 4, <4 x i1> %1, <4 x i32> undef)
+  %3 = trunc <4 x i32> %2 to <4 x i8>
+  call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %3, <4 x i8>* %dest, i32 1, <4 x i1> %1)
+  ret void
+}
+
+define void @foo_trunc_v4i16_v4i32(<4 x i16> *%dest, <4 x i32> *%mask, <4 x i32> *%src) {
+; CHECK-LABEL: foo_trunc_v4i16_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    vptt.s32 gt, q0, zr
+; CHECK-NEXT:    vldrwt.u32 q0, [r2]
+; CHECK-NEXT:    vstrht.32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = load <4 x i32>, <4 x i32>* %mask, align 4
+  %1 = icmp sgt <4 x i32> %0, zeroinitializer
+  %2 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %src, i32 4, <4 x i1> %1, <4 x i32> undef)
+  %3 = trunc <4 x i32> %2 to <4 x i16>
+  call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %3, <4 x i16>* %dest, i32 2, <4 x i1> %1)
+  ret void
+}
+
+define void @foo_v4f32_v4f32(<4 x float> *%dest, <4 x i32> *%mask, <4 x float> *%src) {
+; CHECK-LABEL: foo_v4f32_v4f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    vptt.s32 gt, q0, zr
+; CHECK-NEXT:    vldrwt.u32 q0, [r2]
+; CHECK-NEXT:    vstrwt.32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = load <4 x i32>, <4 x i32>* %mask, align 4
+  %1 = icmp sgt <4 x i32> %0, zeroinitializer
+  %2 = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* %src, i32 4, <4 x i1> %1, <4 x float> undef)
+  call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %2, <4 x float>* %dest, i32 4, <4 x i1> %1)
+  ret void
+}
+
+define void @foo_v8f16_v8f16(<8 x half> *%dest, <8 x i16> *%mask, <8 x half> *%src) {
+; CHECK-LABEL: foo_v8f16_v8f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-NEXT:    vptt.s16 gt, q0, zr
+; CHECK-NEXT:    vldrht.u16 q0, [r2]
+; CHECK-NEXT:    vstrht.16 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = load <8 x i16>, <8 x i16>* %mask, align 2
+  %1 = icmp sgt <8 x i16> %0, zeroinitializer
+  %2 = call <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>* %src, i32 2, <8 x i1> %1, <8 x half> undef)
+  call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %2, <8 x half>* %dest, i32 2, <8 x i1> %1)
+  ret void
+}
+
+define void @foo_v4f32_v4f16(<4 x float> *%dest, <4 x i16> *%mask, <4 x half> *%src) {
+; CHECK-LABEL: foo_v4f32_v4f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    .pad #24
+; CHECK-NEXT:    sub sp, #24
+; CHECK-NEXT:    vldrh.s32 q0, [r1]
+; CHECK-NEXT:    mov.w lr, #0
+; CHECK-NEXT:    @ implicit-def: $q1
+; CHECK-NEXT:    vcmp.s32 gt, q0, zr
+; CHECK-NEXT:    vmrs r3, p0
+; CHECK-NEXT:    and r1, r3, #1
+; CHECK-NEXT:    rsb.w r12, r1, #0
+; CHECK-NEXT:    ubfx r1, r3, #4, #1
+; CHECK-NEXT:    bfi lr, r12, #0, #1
+; CHECK-NEXT:    rsbs r1, r1, #0
+; CHECK-NEXT:    bfi lr, r1, #1, #1
+; CHECK-NEXT:    ubfx r1, r3, #8, #1
+; CHECK-NEXT:    rsbs r1, r1, #0
+; CHECK-NEXT:    bfi lr, r1, #2, #1
+; CHECK-NEXT:    ubfx r1, r3, #12, #1
+; CHECK-NEXT:    rsbs r1, r1, #0
+; CHECK-NEXT:    bfi lr, r1, #3, #1
+; CHECK-NEXT:    lsls.w r1, lr, #31
+; CHECK-NEXT:    beq .LBB18_2
+; CHECK-NEXT:  @ %bb.1: @ %cond.load
+; CHECK-NEXT:    vldr.16 s4, [r2]
+; CHECK-NEXT:  .LBB18_2: @ %else
+; CHECK-NEXT:    lsls.w r1, lr, #30
+; CHECK-NEXT:    bpl .LBB18_6
+; CHECK-NEXT:  @ %bb.3: @ %cond.load1
+; CHECK-NEXT:    vldr.16 s0, [r2, #2]
+; CHECK-NEXT:    vmov r3, s4
+; CHECK-NEXT:    vmovx.f16 s4, s5
+; CHECK-NEXT:    vmov r1, s0
+; CHECK-NEXT:    vmov.16 q0[0], r3
+; CHECK-NEXT:    vmov.16 q0[1], r1
+; CHECK-NEXT:    vmov r1, s5
+; CHECK-NEXT:    vmov.16 q0[2], r1
+; CHECK-NEXT:    vmov r1, s4
+; CHECK-NEXT:    vmov.16 q0[3], r1
+; CHECK-NEXT:    lsls.w r1, lr, #29
+; CHECK-NEXT:    bmi .LBB18_7
+; CHECK-NEXT:  .LBB18_4:
+; CHECK-NEXT:    vmov q2, q0
+; CHECK-NEXT:    lsls.w r1, lr, #28
+; CHECK-NEXT:    bmi .LBB18_8
+; CHECK-NEXT:  .LBB18_5:
+; CHECK-NEXT:    vmov q1, q2
+; CHECK-NEXT:    b .LBB18_9
+; CHECK-NEXT:  .LBB18_6:
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    lsls.w r1, lr, #29
+; CHECK-NEXT:    bpl .LBB18_4
+; CHECK-NEXT:  .LBB18_7: @ %cond.load4
+; CHECK-NEXT:    vmovx.f16 s4, s0
+; CHECK-NEXT:    vmov r1, s0
+; CHECK-NEXT:    vmov r3, s4
+; CHECK-NEXT:    vldr.16 s4, [r2, #4]
+; CHECK-NEXT:    vmov.16 q2[0], r1
+; CHECK-NEXT:    vmovx.f16 s0, s1
+; CHECK-NEXT:    vmov.16 q2[1], r3
+; CHECK-NEXT:    vmov r1, s4
+; CHECK-NEXT:    vmov.16 q2[2], r1
+; CHECK-NEXT:    vmov r1, s0
+; CHECK-NEXT:    vmov.16 q2[3], r1
+; CHECK-NEXT:    lsls.w r1, lr, #28
+; CHECK-NEXT:    bpl .LBB18_5
+; CHECK-NEXT:  .LBB18_8: @ %cond.load7
+; CHECK-NEXT:    vmovx.f16 s0, s8
+; CHECK-NEXT:    vmov r3, s8
+; CHECK-NEXT:    vmov r1, s0
+; CHECK-NEXT:    vmov.16 q1[0], r3
+; CHECK-NEXT:    vldr.16 s0, [r2, #6]
+; CHECK-NEXT:    vmov.16 q1[1], r1
+; CHECK-NEXT:    vmov r1, s9
+; CHECK-NEXT:    vmov.16 q1[2], r1
+; CHECK-NEXT:    vmov r1, s0
+; CHECK-NEXT:    vmov.16 q1[3], r1
+; CHECK-NEXT:  .LBB18_9: @ %else8
+; CHECK-NEXT:    vmrs r2, p0
+; CHECK-NEXT:    vmovx.f16 s0, s5
+; CHECK-NEXT:    vcvtb.f32.f16 s3, s0
+; CHECK-NEXT:    vmovx.f16 s8, s4
+; CHECK-NEXT:    vcvtb.f32.f16 s2, s5
+; CHECK-NEXT:    movs r1, #0
+; CHECK-NEXT:    vcvtb.f32.f16 s1, s8
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s4
+; CHECK-NEXT:    and r3, r2, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    bfi r1, r3, #0, #1
+; CHECK-NEXT:    ubfx r3, r2, #4, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    bfi r1, r3, #1, #1
+; CHECK-NEXT:    ubfx r3, r2, #8, #1
+; CHECK-NEXT:    ubfx r2, r2, #12, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    bfi r1, r3, #2, #1
+; CHECK-NEXT:    rsbs r2, r2, #0
+; CHECK-NEXT:    bfi r1, r2, #3, #1
+; CHECK-NEXT:    lsls r2, r1, #31
+; CHECK-NEXT:    ittt ne
+; CHECK-NEXT:    vstrne s0, [sp, #12]
+; CHECK-NEXT:    ldrne r2, [sp, #12]
+; CHECK-NEXT:    strne r2, [r0]
+; CHECK-NEXT:    lsls r2, r1, #30
+; CHECK-NEXT:    ittt mi
+; CHECK-NEXT:    vstrmi s1, [sp, #8]
+; CHECK-NEXT:    ldrmi r2, [sp, #8]
+; CHECK-NEXT:    strmi r2, [r0, #4]
+; CHECK-NEXT:    lsls r2, r1, #29
+; CHECK-NEXT:    ittt mi
+; CHECK-NEXT:    vstrmi s2, [sp, #4]
+; CHECK-NEXT:    ldrmi r2, [sp, #4]
+; CHECK-NEXT:    strmi r2, [r0, #8]
+; CHECK-NEXT:    lsls r1, r1, #28
+; CHECK-NEXT:    ittt mi
+; CHECK-NEXT:    vstrmi s3, [sp]
+; CHECK-NEXT:    ldrmi r1, [sp]
+; CHECK-NEXT:    strmi r1, [r0, #12]
+; CHECK-NEXT:    add sp, #24
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %0 = load <4 x i16>, <4 x i16>* %mask, align 2
+  %1 = icmp sgt <4 x i16> %0, zeroinitializer
+  %2 = call <4 x half> @llvm.masked.load.v4f16.p0v4f16(<4 x half>* %src, i32 2, <4 x i1> %1, <4 x half> undef)
+  %3 = fpext <4 x half> %2 to <4 x float>
+  call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %3, <4 x float>* %dest, i32 2, <4 x i1> %1)
+  ret void
+}
+
+define void @foo_v4f32_v4f16_unaligned(<4 x float> *%dest, <4 x i16> *%mask, <4 x half> *%src) {
+; CHECK-LABEL: foo_v4f32_v4f16_unaligned:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    .pad #24
+; CHECK-NEXT:    sub sp, #24
+; CHECK-NEXT:    vldrh.s32 q0, [r1]
+; CHECK-NEXT:    mov.w lr, #0
+; CHECK-NEXT:    @ implicit-def: $q1
+; CHECK-NEXT:    vcmp.s32 gt, q0, zr
+; CHECK-NEXT:    vmrs r3, p0
+; CHECK-NEXT:    and r1, r3, #1
+; CHECK-NEXT:    rsb.w r12, r1, #0
+; CHECK-NEXT:    ubfx r1, r3, #4, #1
+; CHECK-NEXT:    bfi lr, r12, #0, #1
+; CHECK-NEXT:    rsbs r1, r1, #0
+; CHECK-NEXT:    bfi lr, r1, #1, #1
+; CHECK-NEXT:    ubfx r1, r3, #8, #1
+; CHECK-NEXT:    rsbs r1, r1, #0
+; CHECK-NEXT:    bfi lr, r1, #2, #1
+; CHECK-NEXT:    ubfx r1, r3, #12, #1
+; CHECK-NEXT:    rsbs r1, r1, #0
+; CHECK-NEXT:    bfi lr, r1, #3, #1
+; CHECK-NEXT:    lsls.w r1, lr, #31
+; CHECK-NEXT:    beq .LBB19_2
+; CHECK-NEXT:  @ %bb.1: @ %cond.load
+; CHECK-NEXT:    vldr.16 s4, [r2]
+; CHECK-NEXT:  .LBB19_2: @ %else
+; CHECK-NEXT:    lsls.w r1, lr, #30
+; CHECK-NEXT:    bpl .LBB19_6
+; CHECK-NEXT:  @ %bb.3: @ %cond.load1
+; CHECK-NEXT:    vldr.16 s0, [r2, #2]
+; CHECK-NEXT:    vmov r3, s4
+; CHECK-NEXT:    vmovx.f16 s4, s5
+; CHECK-NEXT:    vmov r1, s0
+; CHECK-NEXT:    vmov.16 q0[0], r3
+; CHECK-NEXT:    vmov.16 q0[1], r1
+; CHECK-NEXT:    vmov r1, s5
+; CHECK-NEXT:    vmov.16 q0[2], r1
+; CHECK-NEXT:    vmov r1, s4
+; CHECK-NEXT:    vmov.16 q0[3], r1
+; CHECK-NEXT:    lsls.w r1, lr, #29
+; CHECK-NEXT:    bmi .LBB19_7
+; CHECK-NEXT:  .LBB19_4:
+; CHECK-NEXT:    vmov q2, q0
+; CHECK-NEXT:    lsls.w r1, lr, #28
+; CHECK-NEXT:    bmi .LBB19_8
+; CHECK-NEXT:  .LBB19_5:
+; CHECK-NEXT:    vmov q1, q2
+; CHECK-NEXT:    b .LBB19_9
+; CHECK-NEXT:  .LBB19_6:
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    lsls.w r1, lr, #29
+; CHECK-NEXT:    bpl .LBB19_4
+; CHECK-NEXT:  .LBB19_7: @ %cond.load4
+; CHECK-NEXT:    vmovx.f16 s4, s0
+; CHECK-NEXT:    vmov r1, s0
+; CHECK-NEXT:    vmov r3, s4
+; CHECK-NEXT:    vldr.16 s4, [r2, #4]
+; CHECK-NEXT:    vmov.16 q2[0], r1
+; CHECK-NEXT:    vmovx.f16 s0, s1
+; CHECK-NEXT:    vmov.16 q2[1], r3
+; CHECK-NEXT:    vmov r1, s4
+; CHECK-NEXT:    vmov.16 q2[2], r1
+; CHECK-NEXT:    vmov r1, s0
+; CHECK-NEXT:    vmov.16 q2[3], r1
+; CHECK-NEXT:    lsls.w r1, lr, #28
+; CHECK-NEXT:    bpl .LBB19_5
+; CHECK-NEXT:  .LBB19_8: @ %cond.load7
+; CHECK-NEXT:    vmovx.f16 s0, s8
+; CHECK-NEXT:    vmov r3, s8
+; CHECK-NEXT:    vmov r1, s0
+; CHECK-NEXT:    vmov.16 q1[0], r3
+; CHECK-NEXT:    vldr.16 s0, [r2, #6]
+; CHECK-NEXT:    vmov.16 q1[1], r1
+; CHECK-NEXT:    vmov r1, s9
+; CHECK-NEXT:    vmov.16 q1[2], r1
+; CHECK-NEXT:    vmov r1, s0
+; CHECK-NEXT:    vmov.16 q1[3], r1
+; CHECK-NEXT:  .LBB19_9: @ %else8
+; CHECK-NEXT:    vmrs r2, p0
+; CHECK-NEXT:    vmovx.f16 s0, s5
+; CHECK-NEXT:    vcvtb.f32.f16 s3, s0
+; CHECK-NEXT:    vmovx.f16 s8, s4
+; CHECK-NEXT:    vcvtb.f32.f16 s2, s5
+; CHECK-NEXT:    movs r1, #0
+; CHECK-NEXT:    vcvtb.f32.f16 s1, s8
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s4
+; CHECK-NEXT:    and r3, r2, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    bfi r1, r3, #0, #1
+; CHECK-NEXT:    ubfx r3, r2, #4, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    bfi r1, r3, #1, #1
+; CHECK-NEXT:    ubfx r3, r2, #8, #1
+; CHECK-NEXT:    ubfx r2, r2, #12, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    bfi r1, r3, #2, #1
+; CHECK-NEXT:    rsbs r2, r2, #0
+; CHECK-NEXT:    bfi r1, r2, #3, #1
+; CHECK-NEXT:    lsls r2, r1, #31
+; CHECK-NEXT:    ittt ne
+; CHECK-NEXT:    vstrne s0, [sp, #12]
+; CHECK-NEXT:    ldrne r2, [sp, #12]
+; CHECK-NEXT:    strne r2, [r0]
+; CHECK-NEXT:    lsls r2, r1, #30
+; CHECK-NEXT:    ittt mi
+; CHECK-NEXT:    vstrmi s1, [sp, #8]
+; CHECK-NEXT:    ldrmi r2, [sp, #8]
+; CHECK-NEXT:    strmi r2, [r0, #4]
+; CHECK-NEXT:    lsls r2, r1, #29
+; CHECK-NEXT:    ittt mi
+; CHECK-NEXT:    vstrmi s2, [sp, #4]
+; CHECK-NEXT:    ldrmi r2, [sp, #4]
+; CHECK-NEXT:    strmi r2, [r0, #8]
+; CHECK-NEXT:    lsls r1, r1, #28
+; CHECK-NEXT:    ittt mi
+; CHECK-NEXT:    vstrmi s3, [sp]
+; CHECK-NEXT:    ldrmi r1, [sp]
+; CHECK-NEXT:    strmi r1, [r0, #12]
+; CHECK-NEXT:    add sp, #24
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %0 = load <4 x i16>, <4 x i16>* %mask, align 2
+  %1 = icmp sgt <4 x i16> %0, zeroinitializer
+  %2 = call <4 x half> @llvm.masked.load.v4f16.p0v4f16(<4 x half>* %src, i32 2, <4 x i1> %1, <4 x half> undef)
+  %3 = fpext <4 x half> %2 to <4 x float>
+  call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %3, <4 x float>* %dest, i32 1, <4 x i1> %1)
+  ret void
+}
+
+declare void @llvm.masked.store.v4i32.p0v4i32(<4 x i32>, <4 x i32>*, i32, <4 x i1>)
+declare void @llvm.masked.store.v8i16.p0v8i16(<8 x i16>, <8 x i16>*, i32, <8 x i1>)
+declare void @llvm.masked.store.v16i8.p0v16i8(<16 x i8>, <16 x i8>*, i32, <16 x i1>)
+declare void @llvm.masked.store.v8f16.p0v8f16(<8 x half>, <8 x half>*, i32, <8 x i1>)
+declare void @llvm.masked.store.v4f32.p0v4f32(<4 x float>, <4 x float>*, i32, <4 x i1>)
+declare <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>*, i32, <16 x i1>, <16 x i8>)
+declare <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>*, i32, <8 x i1>, <8 x i16>)
+declare <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>*, i32, <2 x i1>, <2 x i32>)
+declare <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>*, i32, <4 x i1>, <4 x i32>)
+declare <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>*, i32, <4 x i1>, <4 x float>)
+declare <4 x half> @llvm.masked.load.v4f16.p0v4f16(<4 x half>*, i32, <4 x i1>, <4 x half>)
+declare <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>*, i32, <8 x i1>, <8 x half>)
+
+declare void @llvm.masked.store.v8i8.p0v8i8(<8 x i8>, <8 x i8>*, i32, <8 x i1>)
+declare void @llvm.masked.store.v4i8.p0v4i8(<4 x i8>, <4 x i8>*, i32, <4 x i1>)
+declare void @llvm.masked.store.v4i16.p0v4i16(<4 x i16>, <4 x i16>*, i32, <4 x i1>)
+declare void @llvm.masked.store.v2i64.p0v2i64(<2 x i64>, <2 x i64>*, i32, <2 x i1>)
+declare <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>*, i32, <4 x i1>, <4 x i16>)
+declare <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>*, i32, <4 x i1>, <4 x i8>)
+declare <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>*, i32, <8 x i1>, <8 x i8>)
diff --git a/llvm/test/CodeGen/Thumb2/mve-masked-load.ll b/llvm/test/CodeGen/Thumb2/mve-masked-load.ll
new file mode 100644
index 000000000..5bddc5ed8
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-masked-load.ll
@@ -0,0 +1,2196 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve,+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-LE
+; RUN: llc -mtriple=thumbebv8.1m.main-arm-none-eabi -mattr=+mve,+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-BE
+
+define arm_aapcs_vfpcc <4 x i32> @masked_v4i32_align4_zero(<4 x i32> *%dest, <4 x i32> %a) {
+; CHECK-LE-LABEL: masked_v4i32_align4_zero:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vpt.s32 gt, q0, zr
+; CHECK-LE-NEXT:    vldrwt.u32 q0, [r0]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: masked_v4i32_align4_zero:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    vpt.s32 gt, q1, zr
+; CHECK-BE-NEXT:    vldrwt.u32 q1, [r0]
+; CHECK-BE-NEXT:    vrev64.32 q0, q1
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = icmp sgt <4 x i32> %a, zeroinitializer
+  %l = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %dest, i32 4, <4 x i1> %c, <4 x i32> zeroinitializer)
+  ret <4 x i32> %l
+}
+
+define arm_aapcs_vfpcc <4 x i32> @masked_v4i32_align4_undef(<4 x i32> *%dest, <4 x i32> %a) {
+; CHECK-LE-LABEL: masked_v4i32_align4_undef:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vpt.s32 gt, q0, zr
+; CHECK-LE-NEXT:    vldrwt.u32 q0, [r0]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: masked_v4i32_align4_undef:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    vpt.s32 gt, q1, zr
+; CHECK-BE-NEXT:    vldrwt.u32 q1, [r0]
+; CHECK-BE-NEXT:    vrev64.32 q0, q1
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = icmp sgt <4 x i32> %a, zeroinitializer
+  %l = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %dest, i32 4, <4 x i1> %c, <4 x i32> undef)
+  ret <4 x i32> %l
+}
+
+define arm_aapcs_vfpcc <4 x i32> @masked_v4i32_align1_undef(<4 x i32> *%dest, <4 x i32> %a) {
+; CHECK-LE-LABEL: masked_v4i32_align1_undef:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    .pad #4
+; CHECK-LE-NEXT:    sub sp, #4
+; CHECK-LE-NEXT:    vcmp.s32 gt, q0, zr
+; CHECK-LE-NEXT:    @ implicit-def: $q0
+; CHECK-LE-NEXT:    vmrs r2, p0
+; CHECK-LE-NEXT:    and r1, r2, #1
+; CHECK-LE-NEXT:    rsbs r3, r1, #0
+; CHECK-LE-NEXT:    movs r1, #0
+; CHECK-LE-NEXT:    bfi r1, r3, #0, #1
+; CHECK-LE-NEXT:    ubfx r3, r2, #4, #1
+; CHECK-LE-NEXT:    rsbs r3, r3, #0
+; CHECK-LE-NEXT:    bfi r1, r3, #1, #1
+; CHECK-LE-NEXT:    ubfx r3, r2, #8, #1
+; CHECK-LE-NEXT:    ubfx r2, r2, #12, #1
+; CHECK-LE-NEXT:    rsbs r3, r3, #0
+; CHECK-LE-NEXT:    bfi r1, r3, #2, #1
+; CHECK-LE-NEXT:    rsbs r2, r2, #0
+; CHECK-LE-NEXT:    bfi r1, r2, #3, #1
+; CHECK-LE-NEXT:    lsls r2, r1, #31
+; CHECK-LE-NEXT:    itt ne
+; CHECK-LE-NEXT:    ldrne r2, [r0]
+; CHECK-LE-NEXT:    vmovne.32 q0[0], r2
+; CHECK-LE-NEXT:    lsls r2, r1, #30
+; CHECK-LE-NEXT:    itt mi
+; CHECK-LE-NEXT:    ldrmi r2, [r0, #4]
+; CHECK-LE-NEXT:    vmovmi.32 q0[1], r2
+; CHECK-LE-NEXT:    lsls r2, r1, #29
+; CHECK-LE-NEXT:    itt mi
+; CHECK-LE-NEXT:    ldrmi r2, [r0, #8]
+; CHECK-LE-NEXT:    vmovmi.32 q0[2], r2
+; CHECK-LE-NEXT:    lsls r1, r1, #28
+; CHECK-LE-NEXT:    itt mi
+; CHECK-LE-NEXT:    ldrmi r0, [r0, #12]
+; CHECK-LE-NEXT:    vmovmi.32 q0[3], r0
+; CHECK-LE-NEXT:    add sp, #4
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: masked_v4i32_align1_undef:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    .pad #4
+; CHECK-BE-NEXT:    sub sp, #4
+; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    vcmp.s32 gt, q1, zr
+; CHECK-BE-NEXT:    @ implicit-def: $q1
+; CHECK-BE-NEXT:    vmrs r2, p0
+; CHECK-BE-NEXT:    and r1, r2, #1
+; CHECK-BE-NEXT:    rsbs r3, r1, #0
+; CHECK-BE-NEXT:    movs r1, #0
+; CHECK-BE-NEXT:    bfi r1, r3, #0, #1
+; CHECK-BE-NEXT:    ubfx r3, r2, #4, #1
+; CHECK-BE-NEXT:    rsbs r3, r3, #0
+; CHECK-BE-NEXT:    bfi r1, r3, #1, #1
+; CHECK-BE-NEXT:    ubfx r3, r2, #8, #1
+; CHECK-BE-NEXT:    ubfx r2, r2, #12, #1
+; CHECK-BE-NEXT:    rsbs r3, r3, #0
+; CHECK-BE-NEXT:    bfi r1, r3, #2, #1
+; CHECK-BE-NEXT:    rsbs r2, r2, #0
+; CHECK-BE-NEXT:    bfi r1, r2, #3, #1
+; CHECK-BE-NEXT:    lsls r2, r1, #31
+; CHECK-BE-NEXT:    itt ne
+; CHECK-BE-NEXT:    ldrne r2, [r0]
+; CHECK-BE-NEXT:    vmovne.32 q1[0], r2
+; CHECK-BE-NEXT:    lsls r2, r1, #30
+; CHECK-BE-NEXT:    itt mi
+; CHECK-BE-NEXT:    ldrmi r2, [r0, #4]
+; CHECK-BE-NEXT:    vmovmi.32 q1[1], r2
+; CHECK-BE-NEXT:    lsls r2, r1, #29
+; CHECK-BE-NEXT:    itt mi
+; CHECK-BE-NEXT:    ldrmi r2, [r0, #8]
+; CHECK-BE-NEXT:    vmovmi.32 q1[2], r2
+; CHECK-BE-NEXT:    lsls r1, r1, #28
+; CHECK-BE-NEXT:    itt mi
+; CHECK-BE-NEXT:    ldrmi r0, [r0, #12]
+; CHECK-BE-NEXT:    vmovmi.32 q1[3], r0
+; CHECK-BE-NEXT:    vrev64.32 q0, q1
+; CHECK-BE-NEXT:    add sp, #4
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = icmp sgt <4 x i32> %a, zeroinitializer
+  %l = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %dest, i32 1, <4 x i1> %c, <4 x i32> undef)
+  ret <4 x i32> %l
+}
+
+define arm_aapcs_vfpcc <4 x i32> @masked_v4i32_align4_other(<4 x i32> *%dest, <4 x i32> %a) {
+; CHECK-LE-LABEL: masked_v4i32_align4_other:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vpt.s32 gt, q0, zr
+; CHECK-LE-NEXT:    vldrwt.u32 q1, [r0]
+; CHECK-LE-NEXT:    vpsel q0, q1, q0
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: masked_v4i32_align4_other:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    vpt.s32 gt, q1, zr
+; CHECK-BE-NEXT:    vldrwt.u32 q0, [r0]
+; CHECK-BE-NEXT:    vpsel q1, q0, q1
+; CHECK-BE-NEXT:    vrev64.32 q0, q1
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = icmp sgt <4 x i32> %a, zeroinitializer
+  %l = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %dest, i32 4, <4 x i1> %c, <4 x i32> %a)
+  ret <4 x i32> %l
+}
+
+define arm_aapcs_vfpcc <4 x i32> @zext16_masked_v4i32_align2_zero(<4 x i16> *%dest, <4 x i32> %a) {
+; CHECK-LE-LABEL: zext16_masked_v4i32_align2_zero:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vpt.s32 gt, q0, zr
+; CHECK-LE-NEXT:    vldrht.u32 q0, [r0]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: zext16_masked_v4i32_align2_zero:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    vpt.s32 gt, q1, zr
+; CHECK-BE-NEXT:    vldrht.u32 q1, [r0]
+; CHECK-BE-NEXT:    vrev64.32 q0, q1
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = icmp sgt <4 x i32> %a, zeroinitializer
+  %l = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %dest, i32 2, <4 x i1> %c, <4 x i16> zeroinitializer)
+  %ext = zext <4 x i16> %l to <4 x i32>
+  ret <4 x i32> %ext
+}
+
+define arm_aapcs_vfpcc <4 x i32> @zext16_masked_v4i32_align2_undef(<4 x i16> *%dest, <4 x i32> %a) {
+; CHECK-LE-LABEL: zext16_masked_v4i32_align2_undef:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vpt.s32 gt, q0, zr
+; CHECK-LE-NEXT:    vldrht.u32 q0, [r0]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: zext16_masked_v4i32_align2_undef:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    vpt.s32 gt, q1, zr
+; CHECK-BE-NEXT:    vldrht.u32 q1, [r0]
+; CHECK-BE-NEXT:    vrev64.32 q0, q1
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = icmp sgt <4 x i32> %a, zeroinitializer
+  %l = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %dest, i32 2, <4 x i1> %c, <4 x i16> undef)
+  %ext = zext <4 x i16> %l to <4 x i32>
+  ret <4 x i32> %ext
+}
+
+define arm_aapcs_vfpcc <4 x i32> @zext16_masked_v4i32_align1_undef(<4 x i16> *%dest, <4 x i32> %a) {
+; CHECK-LE-LABEL: zext16_masked_v4i32_align1_undef:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    .pad #4
+; CHECK-LE-NEXT:    sub sp, #4
+; CHECK-LE-NEXT:    vcmp.s32 gt, q0, zr
+; CHECK-LE-NEXT:    @ implicit-def: $q0
+; CHECK-LE-NEXT:    vmrs r2, p0
+; CHECK-LE-NEXT:    and r1, r2, #1
+; CHECK-LE-NEXT:    rsbs r3, r1, #0
+; CHECK-LE-NEXT:    movs r1, #0
+; CHECK-LE-NEXT:    bfi r1, r3, #0, #1
+; CHECK-LE-NEXT:    ubfx r3, r2, #4, #1
+; CHECK-LE-NEXT:    rsbs r3, r3, #0
+; CHECK-LE-NEXT:    bfi r1, r3, #1, #1
+; CHECK-LE-NEXT:    ubfx r3, r2, #8, #1
+; CHECK-LE-NEXT:    ubfx r2, r2, #12, #1
+; CHECK-LE-NEXT:    rsbs r3, r3, #0
+; CHECK-LE-NEXT:    bfi r1, r3, #2, #1
+; CHECK-LE-NEXT:    rsbs r2, r2, #0
+; CHECK-LE-NEXT:    bfi r1, r2, #3, #1
+; CHECK-LE-NEXT:    lsls r2, r1, #31
+; CHECK-LE-NEXT:    itt ne
+; CHECK-LE-NEXT:    ldrhne r2, [r0]
+; CHECK-LE-NEXT:    vmovne.32 q0[0], r2
+; CHECK-LE-NEXT:    lsls r2, r1, #30
+; CHECK-LE-NEXT:    itt mi
+; CHECK-LE-NEXT:    ldrhmi r2, [r0, #2]
+; CHECK-LE-NEXT:    vmovmi.32 q0[1], r2
+; CHECK-LE-NEXT:    lsls r2, r1, #29
+; CHECK-LE-NEXT:    itt mi
+; CHECK-LE-NEXT:    ldrhmi r2, [r0, #4]
+; CHECK-LE-NEXT:    vmovmi.32 q0[2], r2
+; CHECK-LE-NEXT:    lsls r1, r1, #28
+; CHECK-LE-NEXT:    itt mi
+; CHECK-LE-NEXT:    ldrhmi r0, [r0, #6]
+; CHECK-LE-NEXT:    vmovmi.32 q0[3], r0
+; CHECK-LE-NEXT:    vmovlb.s16 q0, q0
+; CHECK-LE-NEXT:    add sp, #4
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: zext16_masked_v4i32_align1_undef:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    .pad #4
+; CHECK-BE-NEXT:    sub sp, #4
+; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    @ implicit-def: $q0
+; CHECK-BE-NEXT:    vcmp.s32 gt, q1, zr
+; CHECK-BE-NEXT:    vmrs r2, p0
+; CHECK-BE-NEXT:    and r1, r2, #1
+; CHECK-BE-NEXT:    rsbs r3, r1, #0
+; CHECK-BE-NEXT:    movs r1, #0
+; CHECK-BE-NEXT:    bfi r1, r3, #0, #1
+; CHECK-BE-NEXT:    ubfx r3, r2, #4, #1
+; CHECK-BE-NEXT:    rsbs r3, r3, #0
+; CHECK-BE-NEXT:    bfi r1, r3, #1, #1
+; CHECK-BE-NEXT:    ubfx r3, r2, #8, #1
+; CHECK-BE-NEXT:    ubfx r2, r2, #12, #1
+; CHECK-BE-NEXT:    rsbs r3, r3, #0
+; CHECK-BE-NEXT:    bfi r1, r3, #2, #1
+; CHECK-BE-NEXT:    rsbs r2, r2, #0
+; CHECK-BE-NEXT:    bfi r1, r2, #3, #1
+; CHECK-BE-NEXT:    lsls r2, r1, #31
+; CHECK-BE-NEXT:    itt ne
+; CHECK-BE-NEXT:    ldrhne r2, [r0]
+; CHECK-BE-NEXT:    vmovne.32 q0[0], r2
+; CHECK-BE-NEXT:    lsls r2, r1, #30
+; CHECK-BE-NEXT:    itt mi
+; CHECK-BE-NEXT:    ldrhmi r2, [r0, #2]
+; CHECK-BE-NEXT:    vmovmi.32 q0[1], r2
+; CHECK-BE-NEXT:    lsls r2, r1, #29
+; CHECK-BE-NEXT:    itt mi
+; CHECK-BE-NEXT:    ldrhmi r2, [r0, #4]
+; CHECK-BE-NEXT:    vmovmi.32 q0[2], r2
+; CHECK-BE-NEXT:    lsls r1, r1, #28
+; CHECK-BE-NEXT:    itt mi
+; CHECK-BE-NEXT:    ldrhmi r0, [r0, #6]
+; CHECK-BE-NEXT:    vmovmi.32 q0[3], r0
+; CHECK-BE-NEXT:    vmovlb.s16 q1, q0
+; CHECK-BE-NEXT:    vrev64.32 q0, q1
+; CHECK-BE-NEXT:    add sp, #4
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = icmp sgt <4 x i32> %a, zeroinitializer
+  %l = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %dest, i32 1, <4 x i1> %c, <4 x i16> undef)
+  %ext = sext <4 x i16> %l to <4 x i32>
+  ret <4 x i32> %ext
+}
+
+define arm_aapcs_vfpcc <4 x i32> @zext16_masked_v4i32_align2_other(<4 x i16> *%dest, <4 x i16> %a) {
+; CHECK-LE-LABEL: zext16_masked_v4i32_align2_other:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vmovlb.u16 q1, q0
+; CHECK-LE-NEXT:    vmovlb.s16 q0, q0
+; CHECK-LE-NEXT:    vpt.s32 gt, q0, zr
+; CHECK-LE-NEXT:    vldrht.u32 q0, [r0]
+; CHECK-LE-NEXT:    vpsel q0, q0, q1
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: zext16_masked_v4i32_align2_other:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    vmovlb.u16 q0, q1
+; CHECK-BE-NEXT:    vmovlb.s16 q1, q1
+; CHECK-BE-NEXT:    vpt.s32 gt, q1, zr
+; CHECK-BE-NEXT:    vldrht.u32 q1, [r0]
+; CHECK-BE-NEXT:    vpsel q1, q1, q0
+; CHECK-BE-NEXT:    vrev64.32 q0, q1
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = icmp sgt <4 x i16> %a, zeroinitializer
+  %l = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %dest, i32 2, <4 x i1> %c, <4 x i16> %a)
+  %ext = zext <4 x i16> %l to <4 x i32>
+  ret <4 x i32> %ext
+}
+
+define arm_aapcs_vfpcc <4 x i32> @sext16_masked_v4i32_align2_zero(<4 x i16> *%dest, <4 x i32> %a) {
+; CHECK-LE-LABEL: sext16_masked_v4i32_align2_zero:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vpt.s32 gt, q0, zr
+; CHECK-LE-NEXT:    vldrht.s32 q0, [r0]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: sext16_masked_v4i32_align2_zero:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    vpt.s32 gt, q1, zr
+; CHECK-BE-NEXT:    vldrht.s32 q1, [r0]
+; CHECK-BE-NEXT:    vrev64.32 q0, q1
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = icmp sgt <4 x i32> %a, zeroinitializer
+  %l = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %dest, i32 2, <4 x i1> %c, <4 x i16> zeroinitializer)
+  %sext = sext <4 x i16> %l to <4 x i32>
+  ret <4 x i32> %sext
+}
+
+define arm_aapcs_vfpcc <4 x i32> @sext16_masked_v4i32_align2_undef(<4 x i16> *%dest, <4 x i32> %a) {
+; CHECK-LE-LABEL: sext16_masked_v4i32_align2_undef:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vpt.s32 gt, q0, zr
+; CHECK-LE-NEXT:    vldrht.s32 q0, [r0]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: sext16_masked_v4i32_align2_undef:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    vpt.s32 gt, q1, zr
+; CHECK-BE-NEXT:    vldrht.s32 q1, [r0]
+; CHECK-BE-NEXT:    vrev64.32 q0, q1
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = icmp sgt <4 x i32> %a, zeroinitializer
+  %l = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %dest, i32 2, <4 x i1> %c, <4 x i16> undef)
+  %sext = sext <4 x i16> %l to <4 x i32>
+  ret <4 x i32> %sext
+}
+
+define arm_aapcs_vfpcc <4 x i32> @sext16_masked_v4i32_align1_undef(<4 x i16> *%dest, <4 x i32> %a) {
+; CHECK-LE-LABEL: sext16_masked_v4i32_align1_undef:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    .pad #4
+; CHECK-LE-NEXT:    sub sp, #4
+; CHECK-LE-NEXT:    vcmp.s32 gt, q0, zr
+; CHECK-LE-NEXT:    @ implicit-def: $q0
+; CHECK-LE-NEXT:    vmrs r2, p0
+; CHECK-LE-NEXT:    and r1, r2, #1
+; CHECK-LE-NEXT:    rsbs r3, r1, #0
+; CHECK-LE-NEXT:    movs r1, #0
+; CHECK-LE-NEXT:    bfi r1, r3, #0, #1
+; CHECK-LE-NEXT:    ubfx r3, r2, #4, #1
+; CHECK-LE-NEXT:    rsbs r3, r3, #0
+; CHECK-LE-NEXT:    bfi r1, r3, #1, #1
+; CHECK-LE-NEXT:    ubfx r3, r2, #8, #1
+; CHECK-LE-NEXT:    ubfx r2, r2, #12, #1
+; CHECK-LE-NEXT:    rsbs r3, r3, #0
+; CHECK-LE-NEXT:    bfi r1, r3, #2, #1
+; CHECK-LE-NEXT:    rsbs r2, r2, #0
+; CHECK-LE-NEXT:    bfi r1, r2, #3, #1
+; CHECK-LE-NEXT:    lsls r2, r1, #31
+; CHECK-LE-NEXT:    itt ne
+; CHECK-LE-NEXT:    ldrhne r2, [r0]
+; CHECK-LE-NEXT:    vmovne.32 q0[0], r2
+; CHECK-LE-NEXT:    lsls r2, r1, #30
+; CHECK-LE-NEXT:    itt mi
+; CHECK-LE-NEXT:    ldrhmi r2, [r0, #2]
+; CHECK-LE-NEXT:    vmovmi.32 q0[1], r2
+; CHECK-LE-NEXT:    lsls r2, r1, #29
+; CHECK-LE-NEXT:    itt mi
+; CHECK-LE-NEXT:    ldrhmi r2, [r0, #4]
+; CHECK-LE-NEXT:    vmovmi.32 q0[2], r2
+; CHECK-LE-NEXT:    lsls r1, r1, #28
+; CHECK-LE-NEXT:    itt mi
+; CHECK-LE-NEXT:    ldrhmi r0, [r0, #6]
+; CHECK-LE-NEXT:    vmovmi.32 q0[3], r0
+; CHECK-LE-NEXT:    vmovlb.s16 q0, q0
+; CHECK-LE-NEXT:    add sp, #4
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: sext16_masked_v4i32_align1_undef:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    .pad #4
+; CHECK-BE-NEXT:    sub sp, #4
+; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    @ implicit-def: $q0
+; CHECK-BE-NEXT:    vcmp.s32 gt, q1, zr
+; CHECK-BE-NEXT:    vmrs r2, p0
+; CHECK-BE-NEXT:    and r1, r2, #1
+; CHECK-BE-NEXT:    rsbs r3, r1, #0
+; CHECK-BE-NEXT:    movs r1, #0
+; CHECK-BE-NEXT:    bfi r1, r3, #0, #1
+; CHECK-BE-NEXT:    ubfx r3, r2, #4, #1
+; CHECK-BE-NEXT:    rsbs r3, r3, #0
+; CHECK-BE-NEXT:    bfi r1, r3, #1, #1
+; CHECK-BE-NEXT:    ubfx r3, r2, #8, #1
+; CHECK-BE-NEXT:    ubfx r2, r2, #12, #1
+; CHECK-BE-NEXT:    rsbs r3, r3, #0
+; CHECK-BE-NEXT:    bfi r1, r3, #2, #1
+; CHECK-BE-NEXT:    rsbs r2, r2, #0
+; CHECK-BE-NEXT:    bfi r1, r2, #3, #1
+; CHECK-BE-NEXT:    lsls r2, r1, #31
+; CHECK-BE-NEXT:    itt ne
+; CHECK-BE-NEXT:    ldrhne r2, [r0]
+; CHECK-BE-NEXT:    vmovne.32 q0[0], r2
+; CHECK-BE-NEXT:    lsls r2, r1, #30
+; CHECK-BE-NEXT:    itt mi
+; CHECK-BE-NEXT:    ldrhmi r2, [r0, #2]
+; CHECK-BE-NEXT:    vmovmi.32 q0[1], r2
+; CHECK-BE-NEXT:    lsls r2, r1, #29
+; CHECK-BE-NEXT:    itt mi
+; CHECK-BE-NEXT:    ldrhmi r2, [r0, #4]
+; CHECK-BE-NEXT:    vmovmi.32 q0[2], r2
+; CHECK-BE-NEXT:    lsls r1, r1, #28
+; CHECK-BE-NEXT:    itt mi
+; CHECK-BE-NEXT:    ldrhmi r0, [r0, #6]
+; CHECK-BE-NEXT:    vmovmi.32 q0[3], r0
+; CHECK-BE-NEXT:    vmovlb.s16 q1, q0
+; CHECK-BE-NEXT:    vrev64.32 q0, q1
+; CHECK-BE-NEXT:    add sp, #4
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = icmp sgt <4 x i32> %a, zeroinitializer
+  %l = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %dest, i32 1, <4 x i1> %c, <4 x i16> undef)
+  %sext = sext <4 x i16> %l to <4 x i32>
+  ret <4 x i32> %sext
+}
+
+define arm_aapcs_vfpcc <4 x i32> @sext16_masked_v4i32_align2_other(<4 x i16> *%dest, <4 x i16> %a) {
+; CHECK-LE-LABEL: sext16_masked_v4i32_align2_other:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vmovlb.s16 q0, q0
+; CHECK-LE-NEXT:    vpt.s32 gt, q0, zr
+; CHECK-LE-NEXT:    vldrht.s32 q1, [r0]
+; CHECK-LE-NEXT:    vpsel q0, q1, q0
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: sext16_masked_v4i32_align2_other:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    vmovlb.s16 q0, q1
+; CHECK-BE-NEXT:    vpt.s32 gt, q0, zr
+; CHECK-BE-NEXT:    vldrht.s32 q1, [r0]
+; CHECK-BE-NEXT:    vpsel q1, q1, q0
+; CHECK-BE-NEXT:    vrev64.32 q0, q1
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = icmp sgt <4 x i16> %a, zeroinitializer
+  %l = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %dest, i32 2, <4 x i1> %c, <4 x i16> %a)
+  %sext = sext <4 x i16> %l to <4 x i32>
+  ret <4 x i32> %sext
+}
+
+define arm_aapcs_vfpcc i8* @masked_v4i32_preinc(i8* %x, i8* %y, <4 x i32> %a) {
+; CHECK-LE-LABEL: masked_v4i32_preinc:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vpt.s32 gt, q0, zr
+; CHECK-LE-NEXT:    vldrwt.u32 q0, [r0, #4]!
+; CHECK-LE-NEXT:    vstrw.32 q0, [r1]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: masked_v4i32_preinc:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    vpt.s32 gt, q1, zr
+; CHECK-BE-NEXT:    vldrwt.u32 q0, [r0, #4]!
+; CHECK-BE-NEXT:    vstrw.32 q0, [r1]
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 4
+  %0 = bitcast i8* %z to <4 x i32>*
+  %c = icmp sgt <4 x i32> %a, zeroinitializer
+  %1 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %0, i32 4, <4 x i1> %c, <4 x i32> undef)
+  %2 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %1, <4 x i32>* %2, align 4
+  ret i8* %z
+}
+
+define arm_aapcs_vfpcc i8* @masked_v4i32_postinc(i8* %x, i8* %y, <4 x i32> %a) {
+; CHECK-LE-LABEL: masked_v4i32_postinc:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vpt.s32 gt, q0, zr
+; CHECK-LE-NEXT:    vldrwt.u32 q0, [r0], #4
+; CHECK-LE-NEXT:    vstrw.32 q0, [r1]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: masked_v4i32_postinc:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    vpt.s32 gt, q1, zr
+; CHECK-BE-NEXT:    vldrwt.u32 q0, [r0], #4
+; CHECK-BE-NEXT:    vstrw.32 q0, [r1]
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 4
+  %0 = bitcast i8* %x to <4 x i32>*
+  %c = icmp sgt <4 x i32> %a, zeroinitializer
+  %1 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %0, i32 4, <4 x i1> %c, <4 x i32> undef)
+  %2 = bitcast i8* %y to <4 x i32>*
+  store <4 x i32> %1, <4 x i32>* %2, align 4
+  ret i8* %z
+}
+
+define arm_aapcs_vfpcc <8 x i16> @masked_v8i16_align4_zero(<8 x i16> *%dest, <8 x i16> %a) {
+; CHECK-LE-LABEL: masked_v8i16_align4_zero:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vpt.s16 gt, q0, zr
+; CHECK-LE-NEXT:    vldrht.u16 q0, [r0]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: masked_v8i16_align4_zero:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vrev64.16 q1, q0
+; CHECK-BE-NEXT:    vpt.s16 gt, q1, zr
+; CHECK-BE-NEXT:    vldrht.u16 q1, [r0]
+; CHECK-BE-NEXT:    vrev64.16 q0, q1
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = icmp sgt <8 x i16> %a, zeroinitializer
+  %l = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %dest, i32 2, <8 x i1> %c, <8 x i16> zeroinitializer)
+  ret <8 x i16> %l
+}
+
+define arm_aapcs_vfpcc <8 x i16> @masked_v8i16_align2_undef(<8 x i16> *%dest, <8 x i16> %a) {
+; CHECK-LE-LABEL: masked_v8i16_align2_undef:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vpt.s16 gt, q0, zr
+; CHECK-LE-NEXT:    vldrht.u16 q0, [r0]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: masked_v8i16_align2_undef:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vrev64.16 q1, q0
+; CHECK-BE-NEXT:    vpt.s16 gt, q1, zr
+; CHECK-BE-NEXT:    vldrht.u16 q1, [r0]
+; CHECK-BE-NEXT:    vrev64.16 q0, q1
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = icmp sgt <8 x i16> %a, zeroinitializer
+  %l = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %dest, i32 2, <8 x i1> %c, <8 x i16> undef)
+  ret <8 x i16> %l
+}
+
+define arm_aapcs_vfpcc <8 x i16> @masked_v8i16_align1_undef(<8 x i16> *%dest, <8 x i16> %a) {
+; CHECK-LE-LABEL: masked_v8i16_align1_undef:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    .pad #8
+; CHECK-LE-NEXT:    sub sp, #8
+; CHECK-LE-NEXT:    vcmp.s16 gt, q0, zr
+; CHECK-LE-NEXT:    @ implicit-def: $q0
+; CHECK-LE-NEXT:    vmrs r1, p0
+; CHECK-LE-NEXT:    and r2, r1, #1
+; CHECK-LE-NEXT:    rsbs r3, r2, #0
+; CHECK-LE-NEXT:    movs r2, #0
+; CHECK-LE-NEXT:    bfi r2, r3, #0, #1
+; CHECK-LE-NEXT:    ubfx r3, r1, #2, #1
+; CHECK-LE-NEXT:    rsbs r3, r3, #0
+; CHECK-LE-NEXT:    bfi r2, r3, #1, #1
+; CHECK-LE-NEXT:    ubfx r3, r1, #4, #1
+; CHECK-LE-NEXT:    rsbs r3, r3, #0
+; CHECK-LE-NEXT:    bfi r2, r3, #2, #1
+; CHECK-LE-NEXT:    ubfx r3, r1, #6, #1
+; CHECK-LE-NEXT:    rsbs r3, r3, #0
+; CHECK-LE-NEXT:    bfi r2, r3, #3, #1
+; CHECK-LE-NEXT:    ubfx r3, r1, #8, #1
+; CHECK-LE-NEXT:    rsbs r3, r3, #0
+; CHECK-LE-NEXT:    bfi r2, r3, #4, #1
+; CHECK-LE-NEXT:    ubfx r3, r1, #10, #1
+; CHECK-LE-NEXT:    rsbs r3, r3, #0
+; CHECK-LE-NEXT:    bfi r2, r3, #5, #1
+; CHECK-LE-NEXT:    ubfx r3, r1, #12, #1
+; CHECK-LE-NEXT:    ubfx r1, r1, #14, #1
+; CHECK-LE-NEXT:    rsbs r3, r3, #0
+; CHECK-LE-NEXT:    bfi r2, r3, #6, #1
+; CHECK-LE-NEXT:    rsbs r1, r1, #0
+; CHECK-LE-NEXT:    bfi r2, r1, #7, #1
+; CHECK-LE-NEXT:    uxtb r1, r2
+; CHECK-LE-NEXT:    lsls r2, r2, #31
+; CHECK-LE-NEXT:    itt ne
+; CHECK-LE-NEXT:    ldrhne r2, [r0]
+; CHECK-LE-NEXT:    vmovne.16 q0[0], r2
+; CHECK-LE-NEXT:    lsls r2, r1, #30
+; CHECK-LE-NEXT:    itt mi
+; CHECK-LE-NEXT:    ldrhmi r2, [r0, #2]
+; CHECK-LE-NEXT:    vmovmi.16 q0[1], r2
+; CHECK-LE-NEXT:    lsls r2, r1, #29
+; CHECK-LE-NEXT:    itt mi
+; CHECK-LE-NEXT:    ldrhmi r2, [r0, #4]
+; CHECK-LE-NEXT:    vmovmi.16 q0[2], r2
+; CHECK-LE-NEXT:    lsls r2, r1, #28
+; CHECK-LE-NEXT:    itt mi
+; CHECK-LE-NEXT:    ldrhmi r2, [r0, #6]
+; CHECK-LE-NEXT:    vmovmi.16 q0[3], r2
+; CHECK-LE-NEXT:    lsls r2, r1, #27
+; CHECK-LE-NEXT:    itt mi
+; CHECK-LE-NEXT:    ldrhmi r2, [r0, #8]
+; CHECK-LE-NEXT:    vmovmi.16 q0[4], r2
+; CHECK-LE-NEXT:    lsls r2, r1, #26
+; CHECK-LE-NEXT:    itt mi
+; CHECK-LE-NEXT:    ldrhmi r2, [r0, #10]
+; CHECK-LE-NEXT:    vmovmi.16 q0[5], r2
+; CHECK-LE-NEXT:    lsls r2, r1, #25
+; CHECK-LE-NEXT:    itt mi
+; CHECK-LE-NEXT:    ldrhmi r2, [r0, #12]
+; CHECK-LE-NEXT:    vmovmi.16 q0[6], r2
+; CHECK-LE-NEXT:    lsls r1, r1, #24
+; CHECK-LE-NEXT:    itt mi
+; CHECK-LE-NEXT:    ldrhmi r0, [r0, #14]
+; CHECK-LE-NEXT:    vmovmi.16 q0[7], r0
+; CHECK-LE-NEXT:    add sp, #8
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: masked_v8i16_align1_undef:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    .pad #8
+; CHECK-BE-NEXT:    sub sp, #8
+; CHECK-BE-NEXT:    vrev64.16 q1, q0
+; CHECK-BE-NEXT:    vcmp.s16 gt, q1, zr
+; CHECK-BE-NEXT:    @ implicit-def: $q1
+; CHECK-BE-NEXT:    vmrs r1, p0
+; CHECK-BE-NEXT:    and r2, r1, #1
+; CHECK-BE-NEXT:    rsbs r3, r2, #0
+; CHECK-BE-NEXT:    movs r2, #0
+; CHECK-BE-NEXT:    bfi r2, r3, #0, #1
+; CHECK-BE-NEXT:    ubfx r3, r1, #2, #1
+; CHECK-BE-NEXT:    rsbs r3, r3, #0
+; CHECK-BE-NEXT:    bfi r2, r3, #1, #1
+; CHECK-BE-NEXT:    ubfx r3, r1, #4, #1
+; CHECK-BE-NEXT:    rsbs r3, r3, #0
+; CHECK-BE-NEXT:    bfi r2, r3, #2, #1
+; CHECK-BE-NEXT:    ubfx r3, r1, #6, #1
+; CHECK-BE-NEXT:    rsbs r3, r3, #0
+; CHECK-BE-NEXT:    bfi r2, r3, #3, #1
+; CHECK-BE-NEXT:    ubfx r3, r1, #8, #1
+; CHECK-BE-NEXT:    rsbs r3, r3, #0
+; CHECK-BE-NEXT:    bfi r2, r3, #4, #1
+; CHECK-BE-NEXT:    ubfx r3, r1, #10, #1
+; CHECK-BE-NEXT:    rsbs r3, r3, #0
+; CHECK-BE-NEXT:    bfi r2, r3, #5, #1
+; CHECK-BE-NEXT:    ubfx r3, r1, #12, #1
+; CHECK-BE-NEXT:    ubfx r1, r1, #14, #1
+; CHECK-BE-NEXT:    rsbs r3, r3, #0
+; CHECK-BE-NEXT:    bfi r2, r3, #6, #1
+; CHECK-BE-NEXT:    rsbs r1, r1, #0
+; CHECK-BE-NEXT:    bfi r2, r1, #7, #1
+; CHECK-BE-NEXT:    uxtb r1, r2
+; CHECK-BE-NEXT:    lsls r2, r2, #31
+; CHECK-BE-NEXT:    itt ne
+; CHECK-BE-NEXT:    ldrhne r2, [r0]
+; CHECK-BE-NEXT:    vmovne.16 q1[0], r2
+; CHECK-BE-NEXT:    lsls r2, r1, #30
+; CHECK-BE-NEXT:    itt mi
+; CHECK-BE-NEXT:    ldrhmi r2, [r0, #2]
+; CHECK-BE-NEXT:    vmovmi.16 q1[1], r2
+; CHECK-BE-NEXT:    lsls r2, r1, #29
+; CHECK-BE-NEXT:    itt mi
+; CHECK-BE-NEXT:    ldrhmi r2, [r0, #4]
+; CHECK-BE-NEXT:    vmovmi.16 q1[2], r2
+; CHECK-BE-NEXT:    lsls r2, r1, #28
+; CHECK-BE-NEXT:    itt mi
+; CHECK-BE-NEXT:    ldrhmi r2, [r0, #6]
+; CHECK-BE-NEXT:    vmovmi.16 q1[3], r2
+; CHECK-BE-NEXT:    lsls r2, r1, #27
+; CHECK-BE-NEXT:    itt mi
+; CHECK-BE-NEXT:    ldrhmi r2, [r0, #8]
+; CHECK-BE-NEXT:    vmovmi.16 q1[4], r2
+; CHECK-BE-NEXT:    lsls r2, r1, #26
+; CHECK-BE-NEXT:    itt mi
+; CHECK-BE-NEXT:    ldrhmi r2, [r0, #10]
+; CHECK-BE-NEXT:    vmovmi.16 q1[5], r2
+; CHECK-BE-NEXT:    lsls r2, r1, #25
+; CHECK-BE-NEXT:    itt mi
+; CHECK-BE-NEXT:    ldrhmi r2, [r0, #12]
+; CHECK-BE-NEXT:    vmovmi.16 q1[6], r2
+; CHECK-BE-NEXT:    lsls r1, r1, #24
+; CHECK-BE-NEXT:    itt mi
+; CHECK-BE-NEXT:    ldrhmi r0, [r0, #14]
+; CHECK-BE-NEXT:    vmovmi.16 q1[7], r0
+; CHECK-BE-NEXT:    vrev64.16 q0, q1
+; CHECK-BE-NEXT:    add sp, #8
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = icmp sgt <8 x i16> %a, zeroinitializer
+  %l = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %dest, i32 1, <8 x i1> %c, <8 x i16> undef)
+  ret <8 x i16> %l
+}
+
+define arm_aapcs_vfpcc <8 x i16> @masked_v8i16_align4_other(<8 x i16> *%dest, <8 x i16> %a) {
+; CHECK-LE-LABEL: masked_v8i16_align4_other:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vpt.s16 gt, q0, zr
+; CHECK-LE-NEXT:    vldrht.u16 q1, [r0]
+; CHECK-LE-NEXT:    vpsel q0, q1, q0
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: masked_v8i16_align4_other:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vrev64.16 q1, q0
+; CHECK-BE-NEXT:    vpt.s16 gt, q1, zr
+; CHECK-BE-NEXT:    vldrht.u16 q0, [r0]
+; CHECK-BE-NEXT:    vpsel q1, q0, q1
+; CHECK-BE-NEXT:    vrev64.16 q0, q1
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = icmp sgt <8 x i16> %a, zeroinitializer
+  %l = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %dest, i32 2, <8 x i1> %c, <8 x i16> %a)
+  ret <8 x i16> %l
+}
+
+define arm_aapcs_vfpcc <8 x i16> @sext8_masked_v8i16_align1_zero(<8 x i8> *%dest, <8 x i8> %a) {
+; CHECK-LE-LABEL: sext8_masked_v8i16_align1_zero:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vmovlb.s8 q0, q0
+; CHECK-LE-NEXT:    vpt.s16 gt, q0, zr
+; CHECK-LE-NEXT:    vldrbt.s16 q0, [r0]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: sext8_masked_v8i16_align1_zero:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vrev64.16 q1, q0
+; CHECK-BE-NEXT:    vmovlb.s8 q0, q1
+; CHECK-BE-NEXT:    vpt.s16 gt, q0, zr
+; CHECK-BE-NEXT:    vldrbt.s16 q1, [r0]
+; CHECK-BE-NEXT:    vrev64.16 q0, q1
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = icmp sgt <8 x i8> %a, zeroinitializer
+  %l = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %dest, i32 1, <8 x i1> %c, <8 x i8> zeroinitializer)
+  %ext = sext <8 x i8> %l to <8 x i16>
+  ret <8 x i16> %ext
+}
+
+define arm_aapcs_vfpcc <8 x i16> @sext8_masked_v8i16_align1_undef(<8 x i8> *%dest, <8 x i8> %a) {
+; CHECK-LE-LABEL: sext8_masked_v8i16_align1_undef:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vmovlb.s8 q0, q0
+; CHECK-LE-NEXT:    vpt.s16 gt, q0, zr
+; CHECK-LE-NEXT:    vldrbt.s16 q0, [r0]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: sext8_masked_v8i16_align1_undef:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vrev64.16 q1, q0
+; CHECK-BE-NEXT:    vmovlb.s8 q0, q1
+; CHECK-BE-NEXT:    vpt.s16 gt, q0, zr
+; CHECK-BE-NEXT:    vldrbt.s16 q1, [r0]
+; CHECK-BE-NEXT:    vrev64.16 q0, q1
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = icmp sgt <8 x i8> %a, zeroinitializer
+  %l = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %dest, i32 1, <8 x i1> %c, <8 x i8> undef)
+  %ext = sext <8 x i8> %l to <8 x i16>
+  ret <8 x i16> %ext
+}
+
+define arm_aapcs_vfpcc <8 x i16> @sext8_masked_v8i16_align1_other(<8 x i8> *%dest, <8 x i8> %a) {
+; CHECK-LE-LABEL: sext8_masked_v8i16_align1_other:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vmovlb.s8 q0, q0
+; CHECK-LE-NEXT:    vpt.s16 gt, q0, zr
+; CHECK-LE-NEXT:    vldrbt.s16 q1, [r0]
+; CHECK-LE-NEXT:    vpsel q0, q1, q0
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: sext8_masked_v8i16_align1_other:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vrev64.16 q1, q0
+; CHECK-BE-NEXT:    vmovlb.s8 q0, q1
+; CHECK-BE-NEXT:    vpt.s16 gt, q0, zr
+; CHECK-BE-NEXT:    vldrbt.s16 q1, [r0]
+; CHECK-BE-NEXT:    vpsel q1, q1, q0
+; CHECK-BE-NEXT:    vrev64.16 q0, q1
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = icmp sgt <8 x i8> %a, zeroinitializer
+  %l = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %dest, i32 1, <8 x i1> %c, <8 x i8> %a)
+  %ext = sext <8 x i8> %l to <8 x i16>
+  ret <8 x i16> %ext
+}
+
+define arm_aapcs_vfpcc <4 x i32> @sext8_masked_v4i32_align1_zero(<4 x i8> *%dest, <4 x i8> %a) {
+; CHECK-LE-LABEL: sext8_masked_v4i32_align1_zero:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vmovlb.s8 q0, q0
+; CHECK-LE-NEXT:    vmovlb.s16 q0, q0
+; CHECK-LE-NEXT:    vpt.s32 gt, q0, zr
+; CHECK-LE-NEXT:    vldrbt.s32 q0, [r0]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: sext8_masked_v4i32_align1_zero:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    vmovlb.s8 q0, q1
+; CHECK-BE-NEXT:    vmovlb.s16 q0, q0
+; CHECK-BE-NEXT:    vpt.s32 gt, q0, zr
+; CHECK-BE-NEXT:    vldrbt.s32 q1, [r0]
+; CHECK-BE-NEXT:    vrev64.32 q0, q1
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = icmp sgt <4 x i8> %a, zeroinitializer
+  %l = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %dest, i32 1, <4 x i1> %c, <4 x i8> zeroinitializer)
+  %ext = sext <4 x i8> %l to <4 x i32>
+  ret <4 x i32> %ext
+}
+
+define arm_aapcs_vfpcc <4 x i32> @sext8_masked_v4i32_align1_undef(<4 x i8> *%dest, <4 x i8> %a) {
+; CHECK-LE-LABEL: sext8_masked_v4i32_align1_undef:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vmovlb.s8 q0, q0
+; CHECK-LE-NEXT:    vmovlb.s16 q0, q0
+; CHECK-LE-NEXT:    vpt.s32 gt, q0, zr
+; CHECK-LE-NEXT:    vldrbt.s32 q0, [r0]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: sext8_masked_v4i32_align1_undef:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    vmovlb.s8 q0, q1
+; CHECK-BE-NEXT:    vmovlb.s16 q0, q0
+; CHECK-BE-NEXT:    vpt.s32 gt, q0, zr
+; CHECK-BE-NEXT:    vldrbt.s32 q1, [r0]
+; CHECK-BE-NEXT:    vrev64.32 q0, q1
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = icmp sgt <4 x i8> %a, zeroinitializer
+  %l = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %dest, i32 1, <4 x i1> %c, <4 x i8> undef)
+  %ext = sext <4 x i8> %l to <4 x i32>
+  ret <4 x i32> %ext
+}
+
+define arm_aapcs_vfpcc <4 x i32> @sext8_masked_v4i32_align1_other(<4 x i8> *%dest, <4 x i8> %a) {
+; CHECK-LE-LABEL: sext8_masked_v4i32_align1_other:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vmovlb.s8 q0, q0
+; CHECK-LE-NEXT:    vmovlb.s16 q0, q0
+; CHECK-LE-NEXT:    vpt.s32 gt, q0, zr
+; CHECK-LE-NEXT:    vldrbt.s32 q1, [r0]
+; CHECK-LE-NEXT:    vpsel q0, q1, q0
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: sext8_masked_v4i32_align1_other:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    vmovlb.s8 q0, q1
+; CHECK-BE-NEXT:    vmovlb.s16 q0, q0
+; CHECK-BE-NEXT:    vpt.s32 gt, q0, zr
+; CHECK-BE-NEXT:    vldrbt.s32 q1, [r0]
+; CHECK-BE-NEXT:    vpsel q1, q1, q0
+; CHECK-BE-NEXT:    vrev64.32 q0, q1
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = icmp sgt <4 x i8> %a, zeroinitializer
+  %l = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %dest, i32 1, <4 x i1> %c, <4 x i8> %a)
+  %ext = sext <4 x i8> %l to <4 x i32>
+  ret <4 x i32> %ext
+}
+
+define arm_aapcs_vfpcc <4 x i32> @zext8_masked_v4i32_align1_zero(<4 x i8> *%dest, <4 x i8> %a) {
+; CHECK-LE-LABEL: zext8_masked_v4i32_align1_zero:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vmovlb.s8 q0, q0
+; CHECK-LE-NEXT:    vmovlb.s16 q0, q0
+; CHECK-LE-NEXT:    vpt.s32 gt, q0, zr
+; CHECK-LE-NEXT:    vldrbt.u32 q0, [r0]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: zext8_masked_v4i32_align1_zero:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    vmovlb.s8 q0, q1
+; CHECK-BE-NEXT:    vmovlb.s16 q0, q0
+; CHECK-BE-NEXT:    vpt.s32 gt, q0, zr
+; CHECK-BE-NEXT:    vldrbt.u32 q1, [r0]
+; CHECK-BE-NEXT:    vrev64.32 q0, q1
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = icmp sgt <4 x i8> %a, zeroinitializer
+  %l = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %dest, i32 1, <4 x i1> %c, <4 x i8> zeroinitializer)
+  %ext = zext <4 x i8> %l to <4 x i32>
+  ret <4 x i32> %ext
+}
+
+define arm_aapcs_vfpcc <4 x i32> @zext8_masked_v4i32_align1_undef(<4 x i8> *%dest, <4 x i8> %a) {
+; CHECK-LE-LABEL: zext8_masked_v4i32_align1_undef:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vmovlb.s8 q0, q0
+; CHECK-LE-NEXT:    vmovlb.s16 q0, q0
+; CHECK-LE-NEXT:    vpt.s32 gt, q0, zr
+; CHECK-LE-NEXT:    vldrbt.u32 q0, [r0]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: zext8_masked_v4i32_align1_undef:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    vmovlb.s8 q0, q1
+; CHECK-BE-NEXT:    vmovlb.s16 q0, q0
+; CHECK-BE-NEXT:    vpt.s32 gt, q0, zr
+; CHECK-BE-NEXT:    vldrbt.u32 q1, [r0]
+; CHECK-BE-NEXT:    vrev64.32 q0, q1
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = icmp sgt <4 x i8> %a, zeroinitializer
+  %l = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %dest, i32 1, <4 x i1> %c, <4 x i8> undef)
+  %ext = zext <4 x i8> %l to <4 x i32>
+  ret <4 x i32> %ext
+}
+
+define arm_aapcs_vfpcc <4 x i32> @zext8_masked_v4i32_align1_other(<4 x i8> *%dest, <4 x i8> %a) {
+; CHECK-LE-LABEL: zext8_masked_v4i32_align1_other:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vmov.i32 q1, #0xff
+; CHECK-LE-NEXT:    vand q1, q0, q1
+; CHECK-LE-NEXT:    vmovlb.s8 q0, q0
+; CHECK-LE-NEXT:    vmovlb.s16 q0, q0
+; CHECK-LE-NEXT:    vpt.s32 gt, q0, zr
+; CHECK-LE-NEXT:    vldrbt.u32 q0, [r0]
+; CHECK-LE-NEXT:    vpsel q0, q0, q1
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: zext8_masked_v4i32_align1_other:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vmov.i32 q1, #0xff
+; CHECK-BE-NEXT:    vrev64.32 q2, q0
+; CHECK-BE-NEXT:    vand q0, q2, q1
+; CHECK-BE-NEXT:    vmovlb.s8 q1, q2
+; CHECK-BE-NEXT:    vmovlb.s16 q1, q1
+; CHECK-BE-NEXT:    vpt.s32 gt, q1, zr
+; CHECK-BE-NEXT:    vldrbt.u32 q1, [r0]
+; CHECK-BE-NEXT:    vpsel q1, q1, q0
+; CHECK-BE-NEXT:    vrev64.32 q0, q1
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = icmp sgt <4 x i8> %a, zeroinitializer
+  %l = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %dest, i32 1, <4 x i1> %c, <4 x i8> %a)
+  %ext = zext <4 x i8> %l to <4 x i32>
+  ret <4 x i32> %ext
+}
+
+define arm_aapcs_vfpcc <8 x i16> @zext8_masked_v8i16_align1_zero(<8 x i8> *%dest, <8 x i8> %a) {
+; CHECK-LE-LABEL: zext8_masked_v8i16_align1_zero:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vmovlb.s8 q0, q0
+; CHECK-LE-NEXT:    vpt.s16 gt, q0, zr
+; CHECK-LE-NEXT:    vldrbt.u16 q0, [r0]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: zext8_masked_v8i16_align1_zero:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vrev64.16 q1, q0
+; CHECK-BE-NEXT:    vmovlb.s8 q0, q1
+; CHECK-BE-NEXT:    vpt.s16 gt, q0, zr
+; CHECK-BE-NEXT:    vldrbt.u16 q1, [r0]
+; CHECK-BE-NEXT:    vrev64.16 q0, q1
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = icmp sgt <8 x i8> %a, zeroinitializer
+  %l = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %dest, i32 1, <8 x i1> %c, <8 x i8> zeroinitializer)
+  %ext = zext <8 x i8> %l to <8 x i16>
+  ret <8 x i16> %ext
+}
+
+define arm_aapcs_vfpcc <8 x i16> @zext8_masked_v8i16_align1_undef(<8 x i8> *%dest, <8 x i8> %a) {
+; CHECK-LE-LABEL: zext8_masked_v8i16_align1_undef:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vmovlb.s8 q0, q0
+; CHECK-LE-NEXT:    vpt.s16 gt, q0, zr
+; CHECK-LE-NEXT:    vldrbt.u16 q0, [r0]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: zext8_masked_v8i16_align1_undef:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vrev64.16 q1, q0
+; CHECK-BE-NEXT:    vmovlb.s8 q0, q1
+; CHECK-BE-NEXT:    vpt.s16 gt, q0, zr
+; CHECK-BE-NEXT:    vldrbt.u16 q1, [r0]
+; CHECK-BE-NEXT:    vrev64.16 q0, q1
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = icmp sgt <8 x i8> %a, zeroinitializer
+  %l = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %dest, i32 1, <8 x i1> %c, <8 x i8> undef)
+  %ext = zext <8 x i8> %l to <8 x i16>
+  ret <8 x i16> %ext
+}
+
+define arm_aapcs_vfpcc <8 x i16> @zext8_masked_v8i16_align1_other(<8 x i8> *%dest, <8 x i8> %a) {
+; CHECK-LE-LABEL: zext8_masked_v8i16_align1_other:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vmovlb.u8 q1, q0
+; CHECK-LE-NEXT:    vmovlb.s8 q0, q0
+; CHECK-LE-NEXT:    vpt.s16 gt, q0, zr
+; CHECK-LE-NEXT:    vldrbt.u16 q0, [r0]
+; CHECK-LE-NEXT:    vpsel q0, q0, q1
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: zext8_masked_v8i16_align1_other:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vrev64.16 q1, q0
+; CHECK-BE-NEXT:    vmovlb.u8 q0, q1
+; CHECK-BE-NEXT:    vmovlb.s8 q1, q1
+; CHECK-BE-NEXT:    vpt.s16 gt, q1, zr
+; CHECK-BE-NEXT:    vldrbt.u16 q1, [r0]
+; CHECK-BE-NEXT:    vpsel q1, q1, q0
+; CHECK-BE-NEXT:    vrev64.16 q0, q1
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = icmp sgt <8 x i8> %a, zeroinitializer
+  %l = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %dest, i32 1, <8 x i1> %c, <8 x i8> %a)
+  %ext = zext <8 x i8> %l to <8 x i16>
+  ret <8 x i16> %ext
+}
+
+define i8* @masked_v8i16_preinc(i8* %x, i8* %y, <8 x i16> %a) {
+; CHECK-LE-LABEL: masked_v8i16_preinc:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldr d1, [sp]
+; CHECK-LE-NEXT:    vmov d0, r2, r3
+; CHECK-LE-NEXT:    vpt.s16 gt, q0, zr
+; CHECK-LE-NEXT:    vldrht.u16 q0, [r0, #4]!
+; CHECK-LE-NEXT:    vstrw.32 q0, [r1]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: masked_v8i16_preinc:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vldr d1, [sp]
+; CHECK-BE-NEXT:    vmov d0, r3, r2
+; CHECK-BE-NEXT:    vrev64.16 q1, q0
+; CHECK-BE-NEXT:    vpt.s16 gt, q1, zr
+; CHECK-BE-NEXT:    vldrht.u16 q0, [r0, #4]!
+; CHECK-BE-NEXT:    vstrh.16 q0, [r1]
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 4
+  %0 = bitcast i8* %z to <8 x i16>*
+  %c = icmp sgt <8 x i16> %a, zeroinitializer
+  %1 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %0, i32 4, <8 x i1> %c, <8 x i16> undef)
+  %2 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %1, <8 x i16>* %2, align 4
+  ret i8* %z
+}
+
+define arm_aapcs_vfpcc i8* @masked_v8i16_postinc(i8* %x, i8* %y, <8 x i16> %a) {
+; CHECK-LE-LABEL: masked_v8i16_postinc:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vpt.s16 gt, q0, zr
+; CHECK-LE-NEXT:    vldrht.u16 q0, [r0], #4
+; CHECK-LE-NEXT:    vstrw.32 q0, [r1]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: masked_v8i16_postinc:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vrev64.16 q1, q0
+; CHECK-BE-NEXT:    vpt.s16 gt, q1, zr
+; CHECK-BE-NEXT:    vldrht.u16 q0, [r0], #4
+; CHECK-BE-NEXT:    vstrh.16 q0, [r1]
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 4
+  %0 = bitcast i8* %x to <8 x i16>*
+  %c = icmp sgt <8 x i16> %a, zeroinitializer
+  %1 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %0, i32 4, <8 x i1> %c, <8 x i16> undef)
+  %2 = bitcast i8* %y to <8 x i16>*
+  store <8 x i16> %1, <8 x i16>* %2, align 4
+  ret i8* %z
+}
+
+
+define arm_aapcs_vfpcc <16 x i8> @masked_v16i8_align4_zero(<16 x i8> *%dest, <16 x i8> %a) {
+; CHECK-LE-LABEL: masked_v16i8_align4_zero:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vpt.s8 gt, q0, zr
+; CHECK-LE-NEXT:    vldrbt.u8 q0, [r0]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: masked_v16i8_align4_zero:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vrev64.8 q1, q0
+; CHECK-BE-NEXT:    vpt.s8 gt, q1, zr
+; CHECK-BE-NEXT:    vldrbt.u8 q1, [r0]
+; CHECK-BE-NEXT:    vrev64.8 q0, q1
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = icmp sgt <16 x i8> %a, zeroinitializer
+  %l = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %dest, i32 1, <16 x i1> %c, <16 x i8> zeroinitializer)
+  ret <16 x i8> %l
+}
+
+define arm_aapcs_vfpcc <16 x i8> @masked_v16i8_align4_undef(<16 x i8> *%dest, <16 x i8> %a) {
+; CHECK-LE-LABEL: masked_v16i8_align4_undef:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vpt.s8 gt, q0, zr
+; CHECK-LE-NEXT:    vldrbt.u8 q0, [r0]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: masked_v16i8_align4_undef:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vrev64.8 q1, q0
+; CHECK-BE-NEXT:    vpt.s8 gt, q1, zr
+; CHECK-BE-NEXT:    vldrbt.u8 q1, [r0]
+; CHECK-BE-NEXT:    vrev64.8 q0, q1
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = icmp sgt <16 x i8> %a, zeroinitializer
+  %l = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %dest, i32 1, <16 x i1> %c, <16 x i8> undef)
+  ret <16 x i8> %l
+}
+
+define arm_aapcs_vfpcc <16 x i8> @masked_v16i8_align4_other(<16 x i8> *%dest, <16 x i8> %a) {
+; CHECK-LE-LABEL: masked_v16i8_align4_other:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vpt.s8 gt, q0, zr
+; CHECK-LE-NEXT:    vldrbt.u8 q1, [r0]
+; CHECK-LE-NEXT:    vpsel q0, q1, q0
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: masked_v16i8_align4_other:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vrev64.8 q1, q0
+; CHECK-BE-NEXT:    vpt.s8 gt, q1, zr
+; CHECK-BE-NEXT:    vldrbt.u8 q0, [r0]
+; CHECK-BE-NEXT:    vpsel q1, q0, q1
+; CHECK-BE-NEXT:    vrev64.8 q0, q1
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = icmp sgt <16 x i8> %a, zeroinitializer
+  %l = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %dest, i32 1, <16 x i1> %c, <16 x i8> %a)
+  ret <16 x i8> %l
+}
+
+define arm_aapcs_vfpcc i8* @masked_v16i8_preinc(i8* %x, i8* %y, <16 x i8> %a) {
+; CHECK-LE-LABEL: masked_v16i8_preinc:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vpt.s8 gt, q0, zr
+; CHECK-LE-NEXT:    vldrbt.u8 q0, [r0, #4]!
+; CHECK-LE-NEXT:    vstrw.32 q0, [r1]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: masked_v16i8_preinc:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vrev64.8 q1, q0
+; CHECK-BE-NEXT:    vpt.s8 gt, q1, zr
+; CHECK-BE-NEXT:    vldrbt.u8 q0, [r0, #4]!
+; CHECK-BE-NEXT:    vstrb.8 q0, [r1]
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 4
+  %0 = bitcast i8* %z to <16 x i8>*
+  %c = icmp sgt <16 x i8> %a, zeroinitializer
+  %1 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %0, i32 4, <16 x i1> %c, <16 x i8> undef)
+  %2 = bitcast i8* %y to <16 x i8>*
+  store <16 x i8> %1, <16 x i8>* %2, align 4
+  ret i8* %z
+}
+
+define arm_aapcs_vfpcc i8* @masked_v16i8_postinc(i8* %x, i8* %y, <16 x i8> %a) {
+; CHECK-LE-LABEL: masked_v16i8_postinc:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vpt.s8 gt, q0, zr
+; CHECK-LE-NEXT:    vldrbt.u8 q0, [r0], #4
+; CHECK-LE-NEXT:    vstrw.32 q0, [r1]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: masked_v16i8_postinc:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vrev64.8 q1, q0
+; CHECK-BE-NEXT:    vpt.s8 gt, q1, zr
+; CHECK-BE-NEXT:    vldrbt.u8 q0, [r0], #4
+; CHECK-BE-NEXT:    vstrb.8 q0, [r1]
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 4
+  %0 = bitcast i8* %x to <16 x i8>*
+  %c = icmp sgt <16 x i8> %a, zeroinitializer
+  %1 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %0, i32 4, <16 x i1> %c, <16 x i8> undef)
+  %2 = bitcast i8* %y to <16 x i8>*
+  store <16 x i8> %1, <16 x i8>* %2, align 4
+  ret i8* %z
+}
+
+
+define arm_aapcs_vfpcc <4 x float> @masked_v4f32_align4_zero(<4 x float> *%dest, <4 x i32> %a) {
+; CHECK-LE-LABEL: masked_v4f32_align4_zero:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vpt.s32 gt, q0, zr
+; CHECK-LE-NEXT:    vldrwt.u32 q0, [r0]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: masked_v4f32_align4_zero:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    vpt.s32 gt, q1, zr
+; CHECK-BE-NEXT:    vldrwt.u32 q1, [r0]
+; CHECK-BE-NEXT:    vrev64.32 q0, q1
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = icmp sgt <4 x i32> %a, zeroinitializer
+  %l = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* %dest, i32 4, <4 x i1> %c, <4 x float> zeroinitializer)
+  ret <4 x float> %l
+}
+
+define arm_aapcs_vfpcc <4 x float> @masked_v4f32_align4_undef(<4 x float> *%dest, <4 x i32> %a) {
+; CHECK-LE-LABEL: masked_v4f32_align4_undef:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vpt.s32 gt, q0, zr
+; CHECK-LE-NEXT:    vldrwt.u32 q0, [r0]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: masked_v4f32_align4_undef:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    vpt.s32 gt, q1, zr
+; CHECK-BE-NEXT:    vldrwt.u32 q1, [r0]
+; CHECK-BE-NEXT:    vrev64.32 q0, q1
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = icmp sgt <4 x i32> %a, zeroinitializer
+  %l = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* %dest, i32 4, <4 x i1> %c, <4 x float> undef)
+  ret <4 x float> %l
+}
+
+define arm_aapcs_vfpcc <4 x float> @masked_v4f32_align1_undef(<4 x float> *%dest, <4 x i32> %a) {
+; CHECK-LE-LABEL: masked_v4f32_align1_undef:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    .pad #4
+; CHECK-LE-NEXT:    sub sp, #4
+; CHECK-LE-NEXT:    vcmp.s32 gt, q0, zr
+; CHECK-LE-NEXT:    @ implicit-def: $q0
+; CHECK-LE-NEXT:    vmrs r2, p0
+; CHECK-LE-NEXT:    and r1, r2, #1
+; CHECK-LE-NEXT:    rsbs r3, r1, #0
+; CHECK-LE-NEXT:    movs r1, #0
+; CHECK-LE-NEXT:    bfi r1, r3, #0, #1
+; CHECK-LE-NEXT:    ubfx r3, r2, #4, #1
+; CHECK-LE-NEXT:    rsbs r3, r3, #0
+; CHECK-LE-NEXT:    bfi r1, r3, #1, #1
+; CHECK-LE-NEXT:    ubfx r3, r2, #8, #1
+; CHECK-LE-NEXT:    ubfx r2, r2, #12, #1
+; CHECK-LE-NEXT:    rsbs r3, r3, #0
+; CHECK-LE-NEXT:    bfi r1, r3, #2, #1
+; CHECK-LE-NEXT:    rsbs r2, r2, #0
+; CHECK-LE-NEXT:    bfi r1, r2, #3, #1
+; CHECK-LE-NEXT:    lsls r2, r1, #31
+; CHECK-LE-NEXT:    itt ne
+; CHECK-LE-NEXT:    ldrne r2, [r0]
+; CHECK-LE-NEXT:    vmovne s0, r2
+; CHECK-LE-NEXT:    lsls r2, r1, #30
+; CHECK-LE-NEXT:    itt mi
+; CHECK-LE-NEXT:    ldrmi r2, [r0, #4]
+; CHECK-LE-NEXT:    vmovmi s1, r2
+; CHECK-LE-NEXT:    lsls r2, r1, #29
+; CHECK-LE-NEXT:    itt mi
+; CHECK-LE-NEXT:    ldrmi r2, [r0, #8]
+; CHECK-LE-NEXT:    vmovmi s2, r2
+; CHECK-LE-NEXT:    lsls r1, r1, #28
+; CHECK-LE-NEXT:    itt mi
+; CHECK-LE-NEXT:    ldrmi r0, [r0, #12]
+; CHECK-LE-NEXT:    vmovmi s3, r0
+; CHECK-LE-NEXT:    add sp, #4
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: masked_v4f32_align1_undef:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    .pad #4
+; CHECK-BE-NEXT:    sub sp, #4
+; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    vcmp.s32 gt, q1, zr
+; CHECK-BE-NEXT:    @ implicit-def: $q1
+; CHECK-BE-NEXT:    vmrs r2, p0
+; CHECK-BE-NEXT:    and r1, r2, #1
+; CHECK-BE-NEXT:    rsbs r3, r1, #0
+; CHECK-BE-NEXT:    movs r1, #0
+; CHECK-BE-NEXT:    bfi r1, r3, #0, #1
+; CHECK-BE-NEXT:    ubfx r3, r2, #4, #1
+; CHECK-BE-NEXT:    rsbs r3, r3, #0
+; CHECK-BE-NEXT:    bfi r1, r3, #1, #1
+; CHECK-BE-NEXT:    ubfx r3, r2, #8, #1
+; CHECK-BE-NEXT:    ubfx r2, r2, #12, #1
+; CHECK-BE-NEXT:    rsbs r3, r3, #0
+; CHECK-BE-NEXT:    bfi r1, r3, #2, #1
+; CHECK-BE-NEXT:    rsbs r2, r2, #0
+; CHECK-BE-NEXT:    bfi r1, r2, #3, #1
+; CHECK-BE-NEXT:    lsls r2, r1, #31
+; CHECK-BE-NEXT:    itt ne
+; CHECK-BE-NEXT:    ldrne r2, [r0]
+; CHECK-BE-NEXT:    vmovne s4, r2
+; CHECK-BE-NEXT:    lsls r2, r1, #30
+; CHECK-BE-NEXT:    itt mi
+; CHECK-BE-NEXT:    ldrmi r2, [r0, #4]
+; CHECK-BE-NEXT:    vmovmi s5, r2
+; CHECK-BE-NEXT:    lsls r2, r1, #29
+; CHECK-BE-NEXT:    itt mi
+; CHECK-BE-NEXT:    ldrmi r2, [r0, #8]
+; CHECK-BE-NEXT:    vmovmi s6, r2
+; CHECK-BE-NEXT:    lsls r1, r1, #28
+; CHECK-BE-NEXT:    itt mi
+; CHECK-BE-NEXT:    ldrmi r0, [r0, #12]
+; CHECK-BE-NEXT:    vmovmi s7, r0
+; CHECK-BE-NEXT:    vrev64.32 q0, q1
+; CHECK-BE-NEXT:    add sp, #4
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = icmp sgt <4 x i32> %a, zeroinitializer
+  %l = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* %dest, i32 1, <4 x i1> %c, <4 x float> undef)
+  ret <4 x float> %l
+}
+
+define arm_aapcs_vfpcc <4 x float> @masked_v4f32_align4_other(<4 x float> *%dest, <4 x i32> %a, <4 x float> %b) {
+; CHECK-LE-LABEL: masked_v4f32_align4_other:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vpt.s32 gt, q0, zr
+; CHECK-LE-NEXT:    vldrwt.u32 q0, [r0]
+; CHECK-LE-NEXT:    vpsel q0, q0, q1
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: masked_v4f32_align4_other:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vrev64.32 q2, q1
+; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    vpt.s32 gt, q1, zr
+; CHECK-BE-NEXT:    vldrwt.u32 q0, [r0]
+; CHECK-BE-NEXT:    vpsel q1, q0, q2
+; CHECK-BE-NEXT:    vrev64.32 q0, q1
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = icmp sgt <4 x i32> %a, zeroinitializer
+  %l = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* %dest, i32 4, <4 x i1> %c, <4 x float> %b)
+  ret <4 x float> %l
+}
+
+define arm_aapcs_vfpcc i8* @masked_v4f32_preinc(i8* %x, i8* %y, <4 x i32> %a) {
+; CHECK-LE-LABEL: masked_v4f32_preinc:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vpt.s32 gt, q0, zr
+; CHECK-LE-NEXT:    vldrwt.u32 q0, [r0, #4]!
+; CHECK-LE-NEXT:    vstrw.32 q0, [r1]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: masked_v4f32_preinc:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    vpt.s32 gt, q1, zr
+; CHECK-BE-NEXT:    vldrwt.u32 q0, [r0, #4]!
+; CHECK-BE-NEXT:    vstrw.32 q0, [r1]
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 4
+  %0 = bitcast i8* %z to <4 x float>*
+  %c = icmp sgt <4 x i32> %a, zeroinitializer
+  %1 = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* %0, i32 4, <4 x i1> %c, <4 x float> undef)
+  %2 = bitcast i8* %y to <4 x float>*
+  store <4 x float> %1, <4 x float>* %2, align 4
+  ret i8* %z
+}
+
+define arm_aapcs_vfpcc i8* @masked_v4f32_postinc(i8* %x, i8* %y, <4 x i32> %a) {
+; CHECK-LE-LABEL: masked_v4f32_postinc:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vpt.s32 gt, q0, zr
+; CHECK-LE-NEXT:    vldrwt.u32 q0, [r0], #4
+; CHECK-LE-NEXT:    vstrw.32 q0, [r1]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: masked_v4f32_postinc:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    vpt.s32 gt, q1, zr
+; CHECK-BE-NEXT:    vldrwt.u32 q0, [r0], #4
+; CHECK-BE-NEXT:    vstrw.32 q0, [r1]
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 4
+  %0 = bitcast i8* %x to <4 x float>*
+  %c = icmp sgt <4 x i32> %a, zeroinitializer
+  %1 = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* %0, i32 4, <4 x i1> %c, <4 x float> undef)
+  %2 = bitcast i8* %y to <4 x float>*
+  store <4 x float> %1, <4 x float>* %2, align 4
+  ret i8* %z
+}
+
+
+define arm_aapcs_vfpcc <8 x half> @masked_v8f16_align4_zero(<8 x half> *%dest, <8 x i16> %a) {
+; CHECK-LE-LABEL: masked_v8f16_align4_zero:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vpt.s16 gt, q0, zr
+; CHECK-LE-NEXT:    vldrht.u16 q0, [r0]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: masked_v8f16_align4_zero:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vrev64.16 q1, q0
+; CHECK-BE-NEXT:    vpt.s16 gt, q1, zr
+; CHECK-BE-NEXT:    vldrht.u16 q1, [r0]
+; CHECK-BE-NEXT:    vrev64.16 q0, q1
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = icmp sgt <8 x i16> %a, zeroinitializer
+  %l = call <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>* %dest, i32 2, <8 x i1> %c, <8 x half> zeroinitializer)
+  ret <8 x half> %l
+}
+
+define arm_aapcs_vfpcc <8 x half> @masked_v8f16_align4_undef(<8 x half> *%dest, <8 x i16> %a) {
+; CHECK-LE-LABEL: masked_v8f16_align4_undef:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vpt.s16 gt, q0, zr
+; CHECK-LE-NEXT:    vldrht.u16 q0, [r0]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: masked_v8f16_align4_undef:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vrev64.16 q1, q0
+; CHECK-BE-NEXT:    vpt.s16 gt, q1, zr
+; CHECK-BE-NEXT:    vldrht.u16 q1, [r0]
+; CHECK-BE-NEXT:    vrev64.16 q0, q1
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = icmp sgt <8 x i16> %a, zeroinitializer
+  %l = call <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>* %dest, i32 2, <8 x i1> %c, <8 x half> undef)
+  ret <8 x half> %l
+}
+
+define arm_aapcs_vfpcc <8 x half> @masked_v8f16_align1_undef(<8 x half> *%dest, <8 x i16> %a) {
+; CHECK-LE-LABEL: masked_v8f16_align1_undef:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    .pad #40
+; CHECK-LE-NEXT:    sub sp, #40
+; CHECK-LE-NEXT:    vcmp.s16 gt, q0, zr
+; CHECK-LE-NEXT:    @ implicit-def: $q0
+; CHECK-LE-NEXT:    vmrs r1, p0
+; CHECK-LE-NEXT:    and r2, r1, #1
+; CHECK-LE-NEXT:    rsbs r3, r2, #0
+; CHECK-LE-NEXT:    movs r2, #0
+; CHECK-LE-NEXT:    bfi r2, r3, #0, #1
+; CHECK-LE-NEXT:    ubfx r3, r1, #2, #1
+; CHECK-LE-NEXT:    rsbs r3, r3, #0
+; CHECK-LE-NEXT:    bfi r2, r3, #1, #1
+; CHECK-LE-NEXT:    ubfx r3, r1, #4, #1
+; CHECK-LE-NEXT:    rsbs r3, r3, #0
+; CHECK-LE-NEXT:    bfi r2, r3, #2, #1
+; CHECK-LE-NEXT:    ubfx r3, r1, #6, #1
+; CHECK-LE-NEXT:    rsbs r3, r3, #0
+; CHECK-LE-NEXT:    bfi r2, r3, #3, #1
+; CHECK-LE-NEXT:    ubfx r3, r1, #8, #1
+; CHECK-LE-NEXT:    rsbs r3, r3, #0
+; CHECK-LE-NEXT:    bfi r2, r3, #4, #1
+; CHECK-LE-NEXT:    ubfx r3, r1, #10, #1
+; CHECK-LE-NEXT:    rsbs r3, r3, #0
+; CHECK-LE-NEXT:    bfi r2, r3, #5, #1
+; CHECK-LE-NEXT:    ubfx r3, r1, #12, #1
+; CHECK-LE-NEXT:    ubfx r1, r1, #14, #1
+; CHECK-LE-NEXT:    rsbs r3, r3, #0
+; CHECK-LE-NEXT:    bfi r2, r3, #6, #1
+; CHECK-LE-NEXT:    rsbs r1, r1, #0
+; CHECK-LE-NEXT:    bfi r2, r1, #7, #1
+; CHECK-LE-NEXT:    uxtb r1, r2
+; CHECK-LE-NEXT:    lsls r2, r2, #31
+; CHECK-LE-NEXT:    bne .LBB45_9
+; CHECK-LE-NEXT:  @ %bb.1: @ %else
+; CHECK-LE-NEXT:    lsls r2, r1, #30
+; CHECK-LE-NEXT:    bmi .LBB45_10
+; CHECK-LE-NEXT:  .LBB45_2: @ %else2
+; CHECK-LE-NEXT:    lsls r2, r1, #29
+; CHECK-LE-NEXT:    bmi .LBB45_11
+; CHECK-LE-NEXT:  .LBB45_3: @ %else5
+; CHECK-LE-NEXT:    lsls r2, r1, #28
+; CHECK-LE-NEXT:    bmi .LBB45_12
+; CHECK-LE-NEXT:  .LBB45_4: @ %else8
+; CHECK-LE-NEXT:    lsls r2, r1, #27
+; CHECK-LE-NEXT:    bmi .LBB45_13
+; CHECK-LE-NEXT:  .LBB45_5: @ %else11
+; CHECK-LE-NEXT:    lsls r2, r1, #26
+; CHECK-LE-NEXT:    bmi .LBB45_14
+; CHECK-LE-NEXT:  .LBB45_6: @ %else14
+; CHECK-LE-NEXT:    lsls r2, r1, #25
+; CHECK-LE-NEXT:    bmi .LBB45_15
+; CHECK-LE-NEXT:  .LBB45_7: @ %else17
+; CHECK-LE-NEXT:    lsls r1, r1, #24
+; CHECK-LE-NEXT:    bmi .LBB45_16
+; CHECK-LE-NEXT:  .LBB45_8: @ %else20
+; CHECK-LE-NEXT:    add sp, #40
+; CHECK-LE-NEXT:    bx lr
+; CHECK-LE-NEXT:  .LBB45_9: @ %cond.load
+; CHECK-LE-NEXT:    ldrh r2, [r0]
+; CHECK-LE-NEXT:    strh.w r2, [sp, #28]
+; CHECK-LE-NEXT:    vldr.16 s0, [sp, #28]
+; CHECK-LE-NEXT:    lsls r2, r1, #30
+; CHECK-LE-NEXT:    bpl .LBB45_2
+; CHECK-LE-NEXT:  .LBB45_10: @ %cond.load1
+; CHECK-LE-NEXT:    ldrh r2, [r0, #2]
+; CHECK-LE-NEXT:    strh.w r2, [sp, #24]
+; CHECK-LE-NEXT:    vldr.16 s4, [sp, #24]
+; CHECK-LE-NEXT:    vmov r2, s4
+; CHECK-LE-NEXT:    vmov.16 q0[1], r2
+; CHECK-LE-NEXT:    lsls r2, r1, #29
+; CHECK-LE-NEXT:    bpl .LBB45_3
+; CHECK-LE-NEXT:  .LBB45_11: @ %cond.load4
+; CHECK-LE-NEXT:    ldrh r2, [r0, #4]
+; CHECK-LE-NEXT:    strh.w r2, [sp, #20]
+; CHECK-LE-NEXT:    vldr.16 s4, [sp, #20]
+; CHECK-LE-NEXT:    vmov r2, s4
+; CHECK-LE-NEXT:    vmov.16 q0[2], r2
+; CHECK-LE-NEXT:    lsls r2, r1, #28
+; CHECK-LE-NEXT:    bpl .LBB45_4
+; CHECK-LE-NEXT:  .LBB45_12: @ %cond.load7
+; CHECK-LE-NEXT:    ldrh r2, [r0, #6]
+; CHECK-LE-NEXT:    strh.w r2, [sp, #16]
+; CHECK-LE-NEXT:    vldr.16 s4, [sp, #16]
+; CHECK-LE-NEXT:    vmov r2, s4
+; CHECK-LE-NEXT:    vmov.16 q0[3], r2
+; CHECK-LE-NEXT:    lsls r2, r1, #27
+; CHECK-LE-NEXT:    bpl .LBB45_5
+; CHECK-LE-NEXT:  .LBB45_13: @ %cond.load10
+; CHECK-LE-NEXT:    ldrh r2, [r0, #8]
+; CHECK-LE-NEXT:    strh.w r2, [sp, #12]
+; CHECK-LE-NEXT:    vldr.16 s4, [sp, #12]
+; CHECK-LE-NEXT:    vmov r2, s4
+; CHECK-LE-NEXT:    vmov.16 q0[4], r2
+; CHECK-LE-NEXT:    lsls r2, r1, #26
+; CHECK-LE-NEXT:    bpl .LBB45_6
+; CHECK-LE-NEXT:  .LBB45_14: @ %cond.load13
+; CHECK-LE-NEXT:    ldrh r2, [r0, #10]
+; CHECK-LE-NEXT:    strh.w r2, [sp, #8]
+; CHECK-LE-NEXT:    vldr.16 s4, [sp, #8]
+; CHECK-LE-NEXT:    vmov r2, s4
+; CHECK-LE-NEXT:    vmov.16 q0[5], r2
+; CHECK-LE-NEXT:    lsls r2, r1, #25
+; CHECK-LE-NEXT:    bpl .LBB45_7
+; CHECK-LE-NEXT:  .LBB45_15: @ %cond.load16
+; CHECK-LE-NEXT:    ldrh r2, [r0, #12]
+; CHECK-LE-NEXT:    strh.w r2, [sp, #4]
+; CHECK-LE-NEXT:    vldr.16 s4, [sp, #4]
+; CHECK-LE-NEXT:    vmov r2, s4
+; CHECK-LE-NEXT:    vmov.16 q0[6], r2
+; CHECK-LE-NEXT:    lsls r1, r1, #24
+; CHECK-LE-NEXT:    bpl .LBB45_8
+; CHECK-LE-NEXT:  .LBB45_16: @ %cond.load19
+; CHECK-LE-NEXT:    ldrh r0, [r0, #14]
+; CHECK-LE-NEXT:    strh.w r0, [sp]
+; CHECK-LE-NEXT:    vldr.16 s4, [sp]
+; CHECK-LE-NEXT:    vmov r0, s4
+; CHECK-LE-NEXT:    vmov.16 q0[7], r0
+; CHECK-LE-NEXT:    add sp, #40
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: masked_v8f16_align1_undef:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    .pad #40
+; CHECK-BE-NEXT:    sub sp, #40
+; CHECK-BE-NEXT:    vrev64.16 q1, q0
+; CHECK-BE-NEXT:    vcmp.s16 gt, q1, zr
+; CHECK-BE-NEXT:    @ implicit-def: $q1
+; CHECK-BE-NEXT:    vmrs r1, p0
+; CHECK-BE-NEXT:    and r2, r1, #1
+; CHECK-BE-NEXT:    rsbs r3, r2, #0
+; CHECK-BE-NEXT:    movs r2, #0
+; CHECK-BE-NEXT:    bfi r2, r3, #0, #1
+; CHECK-BE-NEXT:    ubfx r3, r1, #2, #1
+; CHECK-BE-NEXT:    rsbs r3, r3, #0
+; CHECK-BE-NEXT:    bfi r2, r3, #1, #1
+; CHECK-BE-NEXT:    ubfx r3, r1, #4, #1
+; CHECK-BE-NEXT:    rsbs r3, r3, #0
+; CHECK-BE-NEXT:    bfi r2, r3, #2, #1
+; CHECK-BE-NEXT:    ubfx r3, r1, #6, #1
+; CHECK-BE-NEXT:    rsbs r3, r3, #0
+; CHECK-BE-NEXT:    bfi r2, r3, #3, #1
+; CHECK-BE-NEXT:    ubfx r3, r1, #8, #1
+; CHECK-BE-NEXT:    rsbs r3, r3, #0
+; CHECK-BE-NEXT:    bfi r2, r3, #4, #1
+; CHECK-BE-NEXT:    ubfx r3, r1, #10, #1
+; CHECK-BE-NEXT:    rsbs r3, r3, #0
+; CHECK-BE-NEXT:    bfi r2, r3, #5, #1
+; CHECK-BE-NEXT:    ubfx r3, r1, #12, #1
+; CHECK-BE-NEXT:    ubfx r1, r1, #14, #1
+; CHECK-BE-NEXT:    rsbs r3, r3, #0
+; CHECK-BE-NEXT:    bfi r2, r3, #6, #1
+; CHECK-BE-NEXT:    rsbs r1, r1, #0
+; CHECK-BE-NEXT:    bfi r2, r1, #7, #1
+; CHECK-BE-NEXT:    uxtb r1, r2
+; CHECK-BE-NEXT:    lsls r2, r2, #31
+; CHECK-BE-NEXT:    bne .LBB45_10
+; CHECK-BE-NEXT:  @ %bb.1: @ %else
+; CHECK-BE-NEXT:    lsls r2, r1, #30
+; CHECK-BE-NEXT:    bmi .LBB45_11
+; CHECK-BE-NEXT:  .LBB45_2: @ %else2
+; CHECK-BE-NEXT:    lsls r2, r1, #29
+; CHECK-BE-NEXT:    bmi .LBB45_12
+; CHECK-BE-NEXT:  .LBB45_3: @ %else5
+; CHECK-BE-NEXT:    lsls r2, r1, #28
+; CHECK-BE-NEXT:    bmi .LBB45_13
+; CHECK-BE-NEXT:  .LBB45_4: @ %else8
+; CHECK-BE-NEXT:    lsls r2, r1, #27
+; CHECK-BE-NEXT:    bmi .LBB45_14
+; CHECK-BE-NEXT:  .LBB45_5: @ %else11
+; CHECK-BE-NEXT:    lsls r2, r1, #26
+; CHECK-BE-NEXT:    bmi .LBB45_15
+; CHECK-BE-NEXT:  .LBB45_6: @ %else14
+; CHECK-BE-NEXT:    lsls r2, r1, #25
+; CHECK-BE-NEXT:    bmi .LBB45_16
+; CHECK-BE-NEXT:  .LBB45_7: @ %else17
+; CHECK-BE-NEXT:    lsls r1, r1, #24
+; CHECK-BE-NEXT:    bpl .LBB45_9
+; CHECK-BE-NEXT:  .LBB45_8: @ %cond.load19
+; CHECK-BE-NEXT:    ldrh r0, [r0, #14]
+; CHECK-BE-NEXT:    strh.w r0, [sp]
+; CHECK-BE-NEXT:    vldr.16 s0, [sp]
+; CHECK-BE-NEXT:    vmov r0, s0
+; CHECK-BE-NEXT:    vmov.16 q1[7], r0
+; CHECK-BE-NEXT:  .LBB45_9: @ %else20
+; CHECK-BE-NEXT:    vrev64.16 q0, q1
+; CHECK-BE-NEXT:    add sp, #40
+; CHECK-BE-NEXT:    bx lr
+; CHECK-BE-NEXT:  .LBB45_10: @ %cond.load
+; CHECK-BE-NEXT:    ldrh r2, [r0]
+; CHECK-BE-NEXT:    strh.w r2, [sp, #28]
+; CHECK-BE-NEXT:    vldr.16 s4, [sp, #28]
+; CHECK-BE-NEXT:    lsls r2, r1, #30
+; CHECK-BE-NEXT:    bpl .LBB45_2
+; CHECK-BE-NEXT:  .LBB45_11: @ %cond.load1
+; CHECK-BE-NEXT:    ldrh r2, [r0, #2]
+; CHECK-BE-NEXT:    strh.w r2, [sp, #24]
+; CHECK-BE-NEXT:    vldr.16 s0, [sp, #24]
+; CHECK-BE-NEXT:    vmov r2, s0
+; CHECK-BE-NEXT:    vmov.16 q1[1], r2
+; CHECK-BE-NEXT:    lsls r2, r1, #29
+; CHECK-BE-NEXT:    bpl .LBB45_3
+; CHECK-BE-NEXT:  .LBB45_12: @ %cond.load4
+; CHECK-BE-NEXT:    ldrh r2, [r0, #4]
+; CHECK-BE-NEXT:    strh.w r2, [sp, #20]
+; CHECK-BE-NEXT:    vldr.16 s0, [sp, #20]
+; CHECK-BE-NEXT:    vmov r2, s0
+; CHECK-BE-NEXT:    vmov.16 q1[2], r2
+; CHECK-BE-NEXT:    lsls r2, r1, #28
+; CHECK-BE-NEXT:    bpl .LBB45_4
+; CHECK-BE-NEXT:  .LBB45_13: @ %cond.load7
+; CHECK-BE-NEXT:    ldrh r2, [r0, #6]
+; CHECK-BE-NEXT:    strh.w r2, [sp, #16]
+; CHECK-BE-NEXT:    vldr.16 s0, [sp, #16]
+; CHECK-BE-NEXT:    vmov r2, s0
+; CHECK-BE-NEXT:    vmov.16 q1[3], r2
+; CHECK-BE-NEXT:    lsls r2, r1, #27
+; CHECK-BE-NEXT:    bpl .LBB45_5
+; CHECK-BE-NEXT:  .LBB45_14: @ %cond.load10
+; CHECK-BE-NEXT:    ldrh r2, [r0, #8]
+; CHECK-BE-NEXT:    strh.w r2, [sp, #12]
+; CHECK-BE-NEXT:    vldr.16 s0, [sp, #12]
+; CHECK-BE-NEXT:    vmov r2, s0
+; CHECK-BE-NEXT:    vmov.16 q1[4], r2
+; CHECK-BE-NEXT:    lsls r2, r1, #26
+; CHECK-BE-NEXT:    bpl .LBB45_6
+; CHECK-BE-NEXT:  .LBB45_15: @ %cond.load13
+; CHECK-BE-NEXT:    ldrh r2, [r0, #10]
+; CHECK-BE-NEXT:    strh.w r2, [sp, #8]
+; CHECK-BE-NEXT:    vldr.16 s0, [sp, #8]
+; CHECK-BE-NEXT:    vmov r2, s0
+; CHECK-BE-NEXT:    vmov.16 q1[5], r2
+; CHECK-BE-NEXT:    lsls r2, r1, #25
+; CHECK-BE-NEXT:    bpl .LBB45_7
+; CHECK-BE-NEXT:  .LBB45_16: @ %cond.load16
+; CHECK-BE-NEXT:    ldrh r2, [r0, #12]
+; CHECK-BE-NEXT:    strh.w r2, [sp, #4]
+; CHECK-BE-NEXT:    vldr.16 s0, [sp, #4]
+; CHECK-BE-NEXT:    vmov r2, s0
+; CHECK-BE-NEXT:    vmov.16 q1[6], r2
+; CHECK-BE-NEXT:    lsls r1, r1, #24
+; CHECK-BE-NEXT:    bmi .LBB45_8
+; CHECK-BE-NEXT:    b .LBB45_9
+entry:
+  %c = icmp sgt <8 x i16> %a, zeroinitializer
+  %l = call <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>* %dest, i32 1, <8 x i1> %c, <8 x half> undef)
+  ret <8 x half> %l
+}
+
+define arm_aapcs_vfpcc <8 x half> @masked_v8f16_align4_other(<8 x half> *%dest, <8 x i16> %a, <8 x half> %b) {
+; CHECK-LE-LABEL: masked_v8f16_align4_other:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vpt.s16 gt, q0, zr
+; CHECK-LE-NEXT:    vldrht.u16 q0, [r0]
+; CHECK-LE-NEXT:    vpsel q0, q0, q1
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: masked_v8f16_align4_other:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vrev64.16 q2, q1
+; CHECK-BE-NEXT:    vrev64.16 q1, q0
+; CHECK-BE-NEXT:    vpt.s16 gt, q1, zr
+; CHECK-BE-NEXT:    vldrht.u16 q0, [r0]
+; CHECK-BE-NEXT:    vpsel q1, q0, q2
+; CHECK-BE-NEXT:    vrev64.16 q0, q1
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = icmp sgt <8 x i16> %a, zeroinitializer
+  %l = call <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>* %dest, i32 2, <8 x i1> %c, <8 x half> %b)
+  ret <8 x half> %l
+}
+
+define arm_aapcs_vfpcc i8* @masked_v8f16_preinc(i8* %x, i8* %y, <8 x i16> %a) {
+; CHECK-LE-LABEL: masked_v8f16_preinc:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vpt.s16 gt, q0, zr
+; CHECK-LE-NEXT:    vldrht.u16 q0, [r0, #4]!
+; CHECK-LE-NEXT:    vstrw.32 q0, [r1]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: masked_v8f16_preinc:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vrev64.16 q1, q0
+; CHECK-BE-NEXT:    vpt.s16 gt, q1, zr
+; CHECK-BE-NEXT:    vldrht.u16 q0, [r0, #4]!
+; CHECK-BE-NEXT:    vstrh.16 q0, [r1]
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 4
+  %0 = bitcast i8* %z to <8 x half>*
+  %c = icmp sgt <8 x i16> %a, zeroinitializer
+  %1 = call <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>* %0, i32 4, <8 x i1> %c, <8 x half> undef)
+  %2 = bitcast i8* %y to <8 x half>*
+  store <8 x half> %1, <8 x half>* %2, align 4
+  ret i8* %z
+}
+
+define arm_aapcs_vfpcc i8* @masked_v8f16_postinc(i8* %x, i8* %y, <8 x i16> %a) {
+; CHECK-LE-LABEL: masked_v8f16_postinc:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vpt.s16 gt, q0, zr
+; CHECK-LE-NEXT:    vldrht.u16 q0, [r0], #4
+; CHECK-LE-NEXT:    vstrw.32 q0, [r1]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: masked_v8f16_postinc:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vrev64.16 q1, q0
+; CHECK-BE-NEXT:    vpt.s16 gt, q1, zr
+; CHECK-BE-NEXT:    vldrht.u16 q0, [r0], #4
+; CHECK-BE-NEXT:    vstrh.16 q0, [r1]
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %x, i32 4
+  %0 = bitcast i8* %x to <8 x half>*
+  %c = icmp sgt <8 x i16> %a, zeroinitializer
+  %1 = call <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>* %0, i32 4, <8 x i1> %c, <8 x half> undef)
+  %2 = bitcast i8* %y to <8 x half>*
+  store <8 x half> %1, <8 x half>* %2, align 4
+  ret i8* %z
+}
+
+
+define arm_aapcs_vfpcc <2 x i64> @masked_v2i64_align4_zero(<2 x i64> *%dest, <2 x i64> %a) {
+; CHECK-LE-LABEL: masked_v2i64_align4_zero:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    .pad #4
+; CHECK-LE-NEXT:    sub sp, #4
+; CHECK-LE-NEXT:    vmov r3, s0
+; CHECK-LE-NEXT:    movs r2, #0
+; CHECK-LE-NEXT:    vmov r1, s1
+; CHECK-LE-NEXT:    vmov r12, s3
+; CHECK-LE-NEXT:    rsbs r3, r3, #0
+; CHECK-LE-NEXT:    vmov r3, s2
+; CHECK-LE-NEXT:    sbcs.w r1, r2, r1
+; CHECK-LE-NEXT:    mov.w r1, #0
+; CHECK-LE-NEXT:    it lt
+; CHECK-LE-NEXT:    movlt r1, #1
+; CHECK-LE-NEXT:    rsbs r3, r3, #0
+; CHECK-LE-NEXT:    sbcs.w r3, r2, r12
+; CHECK-LE-NEXT:    it lt
+; CHECK-LE-NEXT:    movlt r2, #1
+; CHECK-LE-NEXT:    cmp r2, #0
+; CHECK-LE-NEXT:    it ne
+; CHECK-LE-NEXT:    mvnne r2, #1
+; CHECK-LE-NEXT:    bfi r2, r1, #0, #1
+; CHECK-LE-NEXT:    and r1, r2, #3
+; CHECK-LE-NEXT:    lsls r2, r2, #31
+; CHECK-LE-NEXT:    beq .LBB49_2
+; CHECK-LE-NEXT:  @ %bb.1: @ %cond.load
+; CHECK-LE-NEXT:    vldr d1, .LCPI49_0
+; CHECK-LE-NEXT:    vldr d0, [r0]
+; CHECK-LE-NEXT:    b .LBB49_3
+; CHECK-LE-NEXT:  .LBB49_2:
+; CHECK-LE-NEXT:    vmov.i32 q0, #0x0
+; CHECK-LE-NEXT:  .LBB49_3: @ %else
+; CHECK-LE-NEXT:    lsls r1, r1, #30
+; CHECK-LE-NEXT:    it mi
+; CHECK-LE-NEXT:    vldrmi d1, [r0, #8]
+; CHECK-LE-NEXT:    add sp, #4
+; CHECK-LE-NEXT:    bx lr
+; CHECK-LE-NEXT:    .p2align 3
+; CHECK-LE-NEXT:  @ %bb.4:
+; CHECK-LE-NEXT:  .LCPI49_0:
+; CHECK-LE-NEXT:    .long 0 @ double 0
+; CHECK-LE-NEXT:    .long 0
+;
+; CHECK-BE-LABEL: masked_v2i64_align4_zero:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    .pad #4
+; CHECK-BE-NEXT:    sub sp, #4
+; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    movs r2, #0
+; CHECK-BE-NEXT:    vmov r3, s7
+; CHECK-BE-NEXT:    vmov r1, s6
+; CHECK-BE-NEXT:    vmov r12, s4
+; CHECK-BE-NEXT:    rsbs r3, r3, #0
+; CHECK-BE-NEXT:    vmov r3, s5
+; CHECK-BE-NEXT:    sbcs.w r1, r2, r1
+; CHECK-BE-NEXT:    mov.w r1, #0
+; CHECK-BE-NEXT:    it lt
+; CHECK-BE-NEXT:    movlt r1, #1
+; CHECK-BE-NEXT:    rsbs r3, r3, #0
+; CHECK-BE-NEXT:    sbcs.w r3, r2, r12
+; CHECK-BE-NEXT:    it lt
+; CHECK-BE-NEXT:    movlt r2, #1
+; CHECK-BE-NEXT:    cmp r2, #0
+; CHECK-BE-NEXT:    it ne
+; CHECK-BE-NEXT:    mvnne r2, #1
+; CHECK-BE-NEXT:    bfi r2, r1, #0, #1
+; CHECK-BE-NEXT:    and r1, r2, #3
+; CHECK-BE-NEXT:    lsls r2, r2, #31
+; CHECK-BE-NEXT:    beq .LBB49_2
+; CHECK-BE-NEXT:  @ %bb.1: @ %cond.load
+; CHECK-BE-NEXT:    vldr d1, .LCPI49_0
+; CHECK-BE-NEXT:    vldr d0, [r0]
+; CHECK-BE-NEXT:    b .LBB49_3
+; CHECK-BE-NEXT:  .LBB49_2:
+; CHECK-BE-NEXT:    vmov.i32 q1, #0x0
+; CHECK-BE-NEXT:    vrev64.32 q0, q1
+; CHECK-BE-NEXT:  .LBB49_3: @ %else
+; CHECK-BE-NEXT:    lsls r1, r1, #30
+; CHECK-BE-NEXT:    it mi
+; CHECK-BE-NEXT:    vldrmi d1, [r0, #8]
+; CHECK-BE-NEXT:    add sp, #4
+; CHECK-BE-NEXT:    bx lr
+; CHECK-BE-NEXT:    .p2align 3
+; CHECK-BE-NEXT:  @ %bb.4:
+; CHECK-BE-NEXT:  .LCPI49_0:
+; CHECK-BE-NEXT:    .long 0 @ double 0
+; CHECK-BE-NEXT:    .long 0
+entry:
+  %c = icmp sgt <2 x i64> %a, zeroinitializer
+  %l = call <2 x i64> @llvm.masked.load.v2i64.p0v2i64(<2 x i64>* %dest, i32 8, <2 x i1> %c, <2 x i64> zeroinitializer)
+  ret <2 x i64> %l
+}
+
+define arm_aapcs_vfpcc <2 x double> @masked_v2f64_align4_zero(<2 x double> *%dest, <2 x double> %a, <2 x i64> %b) {
+; CHECK-LE-LABEL: masked_v2f64_align4_zero:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    .pad #4
+; CHECK-LE-NEXT:    sub sp, #4
+; CHECK-LE-NEXT:    vmov r3, s4
+; CHECK-LE-NEXT:    movs r2, #0
+; CHECK-LE-NEXT:    vmov r1, s5
+; CHECK-LE-NEXT:    vmov r12, s7
+; CHECK-LE-NEXT:    rsbs r3, r3, #0
+; CHECK-LE-NEXT:    vmov r3, s6
+; CHECK-LE-NEXT:    sbcs.w r1, r2, r1
+; CHECK-LE-NEXT:    mov.w r1, #0
+; CHECK-LE-NEXT:    it lt
+; CHECK-LE-NEXT:    movlt r1, #1
+; CHECK-LE-NEXT:    rsbs r3, r3, #0
+; CHECK-LE-NEXT:    sbcs.w r3, r2, r12
+; CHECK-LE-NEXT:    it lt
+; CHECK-LE-NEXT:    movlt r2, #1
+; CHECK-LE-NEXT:    cmp r2, #0
+; CHECK-LE-NEXT:    it ne
+; CHECK-LE-NEXT:    mvnne r2, #1
+; CHECK-LE-NEXT:    bfi r2, r1, #0, #1
+; CHECK-LE-NEXT:    and r1, r2, #3
+; CHECK-LE-NEXT:    lsls r2, r2, #31
+; CHECK-LE-NEXT:    beq .LBB50_2
+; CHECK-LE-NEXT:  @ %bb.1: @ %cond.load
+; CHECK-LE-NEXT:    vldr d1, .LCPI50_0
+; CHECK-LE-NEXT:    vldr d0, [r0]
+; CHECK-LE-NEXT:    b .LBB50_3
+; CHECK-LE-NEXT:  .LBB50_2:
+; CHECK-LE-NEXT:    vmov.i32 q0, #0x0
+; CHECK-LE-NEXT:  .LBB50_3: @ %else
+; CHECK-LE-NEXT:    lsls r1, r1, #30
+; CHECK-LE-NEXT:    it mi
+; CHECK-LE-NEXT:    vldrmi d1, [r0, #8]
+; CHECK-LE-NEXT:    add sp, #4
+; CHECK-LE-NEXT:    bx lr
+; CHECK-LE-NEXT:    .p2align 3
+; CHECK-LE-NEXT:  @ %bb.4:
+; CHECK-LE-NEXT:  .LCPI50_0:
+; CHECK-LE-NEXT:    .long 0 @ double 0
+; CHECK-LE-NEXT:    .long 0
+;
+; CHECK-BE-LABEL: masked_v2f64_align4_zero:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    .pad #4
+; CHECK-BE-NEXT:    sub sp, #4
+; CHECK-BE-NEXT:    vrev64.32 q0, q1
+; CHECK-BE-NEXT:    movs r2, #0
+; CHECK-BE-NEXT:    vmov r3, s3
+; CHECK-BE-NEXT:    vmov r1, s2
+; CHECK-BE-NEXT:    vmov r12, s0
+; CHECK-BE-NEXT:    rsbs r3, r3, #0
+; CHECK-BE-NEXT:    vmov r3, s1
+; CHECK-BE-NEXT:    sbcs.w r1, r2, r1
+; CHECK-BE-NEXT:    mov.w r1, #0
+; CHECK-BE-NEXT:    it lt
+; CHECK-BE-NEXT:    movlt r1, #1
+; CHECK-BE-NEXT:    rsbs r3, r3, #0
+; CHECK-BE-NEXT:    sbcs.w r3, r2, r12
+; CHECK-BE-NEXT:    it lt
+; CHECK-BE-NEXT:    movlt r2, #1
+; CHECK-BE-NEXT:    cmp r2, #0
+; CHECK-BE-NEXT:    it ne
+; CHECK-BE-NEXT:    mvnne r2, #1
+; CHECK-BE-NEXT:    bfi r2, r1, #0, #1
+; CHECK-BE-NEXT:    and r1, r2, #3
+; CHECK-BE-NEXT:    lsls r2, r2, #31
+; CHECK-BE-NEXT:    beq .LBB50_2
+; CHECK-BE-NEXT:  @ %bb.1: @ %cond.load
+; CHECK-BE-NEXT:    vldr d1, .LCPI50_0
+; CHECK-BE-NEXT:    vldr d0, [r0]
+; CHECK-BE-NEXT:    b .LBB50_3
+; CHECK-BE-NEXT:  .LBB50_2:
+; CHECK-BE-NEXT:    vmov.i32 q1, #0x0
+; CHECK-BE-NEXT:    vrev64.32 q0, q1
+; CHECK-BE-NEXT:  .LBB50_3: @ %else
+; CHECK-BE-NEXT:    lsls r1, r1, #30
+; CHECK-BE-NEXT:    it mi
+; CHECK-BE-NEXT:    vldrmi d1, [r0, #8]
+; CHECK-BE-NEXT:    add sp, #4
+; CHECK-BE-NEXT:    bx lr
+; CHECK-BE-NEXT:    .p2align 3
+; CHECK-BE-NEXT:  @ %bb.4:
+; CHECK-BE-NEXT:  .LCPI50_0:
+; CHECK-BE-NEXT:    .long 0 @ double 0
+; CHECK-BE-NEXT:    .long 0
+entry:
+  %c = icmp sgt <2 x i64> %b, zeroinitializer
+  %l = call <2 x double> @llvm.masked.load.v2f64.p0v2f64(<2 x double>* %dest, i32 8, <2 x i1> %c, <2 x double> zeroinitializer)
+  ret <2 x double> %l
+}
+
+define arm_aapcs_vfpcc <4 x i16> @anyext_v4i16(<4 x i16> *%dest, <4 x i32> %a) {
+; CHECK-LE-LABEL: anyext_v4i16:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vpt.s32 gt, q0, zr
+; CHECK-LE-NEXT:    vldrht.u32 q0, [r0]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: anyext_v4i16:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    vpt.s32 gt, q1, zr
+; CHECK-BE-NEXT:    vldrht.u32 q1, [r0]
+; CHECK-BE-NEXT:    vrev64.32 q0, q1
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = icmp sgt <4 x i32> %a, zeroinitializer
+  %l = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %dest, i32 2, <4 x i1> %c, <4 x i16> zeroinitializer)
+  ret <4 x i16> %l
+}
+
+define arm_aapcs_vfpcc <4 x i16> @anyext_v4i16_align1(<4 x i16> *%dest, <4 x i32> %a) {
+; CHECK-LE-LABEL: anyext_v4i16_align1:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    .pad #4
+; CHECK-LE-NEXT:    sub sp, #4
+; CHECK-LE-NEXT:    vcmp.s32 gt, q0, zr
+; CHECK-LE-NEXT:    mov.w r12, #0
+; CHECK-LE-NEXT:    vmrs r3, p0
+; CHECK-LE-NEXT:    and r1, r3, #1
+; CHECK-LE-NEXT:    rsbs r2, r1, #0
+; CHECK-LE-NEXT:    movs r1, #0
+; CHECK-LE-NEXT:    bfi r1, r2, #0, #1
+; CHECK-LE-NEXT:    ubfx r2, r3, #4, #1
+; CHECK-LE-NEXT:    rsbs r2, r2, #0
+; CHECK-LE-NEXT:    bfi r1, r2, #1, #1
+; CHECK-LE-NEXT:    ubfx r2, r3, #8, #1
+; CHECK-LE-NEXT:    rsbs r2, r2, #0
+; CHECK-LE-NEXT:    bfi r1, r2, #2, #1
+; CHECK-LE-NEXT:    ubfx r2, r3, #12, #1
+; CHECK-LE-NEXT:    rsbs r2, r2, #0
+; CHECK-LE-NEXT:    bfi r1, r2, #3, #1
+; CHECK-LE-NEXT:    lsls r2, r1, #31
+; CHECK-LE-NEXT:    beq .LBB52_2
+; CHECK-LE-NEXT:  @ %bb.1: @ %cond.load
+; CHECK-LE-NEXT:    ldrh r2, [r0]
+; CHECK-LE-NEXT:    vdup.32 q0, r12
+; CHECK-LE-NEXT:    vmov.32 q0[0], r2
+; CHECK-LE-NEXT:    b .LBB52_3
+; CHECK-LE-NEXT:  .LBB52_2:
+; CHECK-LE-NEXT:    vmov.i32 q0, #0x0
+; CHECK-LE-NEXT:  .LBB52_3: @ %else
+; CHECK-LE-NEXT:    lsls r2, r1, #30
+; CHECK-LE-NEXT:    itt mi
+; CHECK-LE-NEXT:    ldrhmi r2, [r0, #2]
+; CHECK-LE-NEXT:    vmovmi.32 q0[1], r2
+; CHECK-LE-NEXT:    lsls r2, r1, #29
+; CHECK-LE-NEXT:    itt mi
+; CHECK-LE-NEXT:    ldrhmi r2, [r0, #4]
+; CHECK-LE-NEXT:    vmovmi.32 q0[2], r2
+; CHECK-LE-NEXT:    lsls r1, r1, #28
+; CHECK-LE-NEXT:    itt mi
+; CHECK-LE-NEXT:    ldrhmi r0, [r0, #6]
+; CHECK-LE-NEXT:    vmovmi.32 q0[3], r0
+; CHECK-LE-NEXT:    add sp, #4
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: anyext_v4i16_align1:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    .pad #4
+; CHECK-BE-NEXT:    sub sp, #4
+; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    mov.w r12, #0
+; CHECK-BE-NEXT:    vcmp.s32 gt, q1, zr
+; CHECK-BE-NEXT:    vmrs r3, p0
+; CHECK-BE-NEXT:    and r1, r3, #1
+; CHECK-BE-NEXT:    rsbs r2, r1, #0
+; CHECK-BE-NEXT:    movs r1, #0
+; CHECK-BE-NEXT:    bfi r1, r2, #0, #1
+; CHECK-BE-NEXT:    ubfx r2, r3, #4, #1
+; CHECK-BE-NEXT:    rsbs r2, r2, #0
+; CHECK-BE-NEXT:    bfi r1, r2, #1, #1
+; CHECK-BE-NEXT:    ubfx r2, r3, #8, #1
+; CHECK-BE-NEXT:    rsbs r2, r2, #0
+; CHECK-BE-NEXT:    bfi r1, r2, #2, #1
+; CHECK-BE-NEXT:    ubfx r2, r3, #12, #1
+; CHECK-BE-NEXT:    rsbs r2, r2, #0
+; CHECK-BE-NEXT:    bfi r1, r2, #3, #1
+; CHECK-BE-NEXT:    lsls r2, r1, #31
+; CHECK-BE-NEXT:    beq .LBB52_2
+; CHECK-BE-NEXT:  @ %bb.1: @ %cond.load
+; CHECK-BE-NEXT:    ldrh r2, [r0]
+; CHECK-BE-NEXT:    vdup.32 q1, r12
+; CHECK-BE-NEXT:    vmov.32 q1[0], r2
+; CHECK-BE-NEXT:    b .LBB52_3
+; CHECK-BE-NEXT:  .LBB52_2:
+; CHECK-BE-NEXT:    vmov.i32 q1, #0x0
+; CHECK-BE-NEXT:  .LBB52_3: @ %else
+; CHECK-BE-NEXT:    lsls r2, r1, #30
+; CHECK-BE-NEXT:    itt mi
+; CHECK-BE-NEXT:    ldrhmi r2, [r0, #2]
+; CHECK-BE-NEXT:    vmovmi.32 q1[1], r2
+; CHECK-BE-NEXT:    lsls r2, r1, #29
+; CHECK-BE-NEXT:    itt mi
+; CHECK-BE-NEXT:    ldrhmi r2, [r0, #4]
+; CHECK-BE-NEXT:    vmovmi.32 q1[2], r2
+; CHECK-BE-NEXT:    lsls r1, r1, #28
+; CHECK-BE-NEXT:    itt mi
+; CHECK-BE-NEXT:    ldrhmi r0, [r0, #6]
+; CHECK-BE-NEXT:    vmovmi.32 q1[3], r0
+; CHECK-BE-NEXT:    vrev64.32 q0, q1
+; CHECK-BE-NEXT:    add sp, #4
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = icmp sgt <4 x i32> %a, zeroinitializer
+  %l = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %dest, i32 1, <4 x i1> %c, <4 x i16> zeroinitializer)
+  ret <4 x i16> %l
+}
+
+define arm_aapcs_vfpcc <4 x i8> @anyext_v4i8(<4 x i8> *%dest, <4 x i32> %a) {
+; CHECK-LE-LABEL: anyext_v4i8:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vpt.s32 gt, q0, zr
+; CHECK-LE-NEXT:    vldrbt.u32 q0, [r0]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: anyext_v4i8:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    vpt.s32 gt, q1, zr
+; CHECK-BE-NEXT:    vldrbt.u32 q1, [r0]
+; CHECK-BE-NEXT:    vrev64.32 q0, q1
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = icmp sgt <4 x i32> %a, zeroinitializer
+  %l = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %dest, i32 1, <4 x i1> %c, <4 x i8> zeroinitializer)
+  ret <4 x i8> %l
+}
+
+define arm_aapcs_vfpcc <8 x i8> @anyext_v8i8(<8 x i8> *%dest, <8 x i16> %a) {
+; CHECK-LE-LABEL: anyext_v8i8:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vpt.s16 gt, q0, zr
+; CHECK-LE-NEXT:    vldrbt.u16 q0, [r0]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: anyext_v8i8:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vrev64.16 q1, q0
+; CHECK-BE-NEXT:    vpt.s16 gt, q1, zr
+; CHECK-BE-NEXT:    vldrbt.u16 q1, [r0]
+; CHECK-BE-NEXT:    vrev64.16 q0, q1
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = icmp sgt <8 x i16> %a, zeroinitializer
+  %l = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %dest, i32 1, <8 x i1> %c, <8 x i8> zeroinitializer)
+  ret <8 x i8> %l
+}
+
+define arm_aapcs_vfpcc <4 x i32> @multi_user_zext(<4 x i16> *%dest, <4 x i32> %a) {
+; CHECK-LE-LABEL: multi_user_zext:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    .save {r7, lr}
+; CHECK-LE-NEXT:    push {r7, lr}
+; CHECK-LE-NEXT:    .vsave {d8, d9}
+; CHECK-LE-NEXT:    vpush {d8, d9}
+; CHECK-LE-NEXT:    vpt.s32 gt, q0, zr
+; CHECK-LE-NEXT:    vldrht.u32 q4, [r0]
+; CHECK-LE-NEXT:    vmov r0, r1, d8
+; CHECK-LE-NEXT:    vmov r2, r3, d9
+; CHECK-LE-NEXT:    bl foo
+; CHECK-LE-NEXT:    vmovlb.u16 q0, q4
+; CHECK-LE-NEXT:    vpop {d8, d9}
+; CHECK-LE-NEXT:    pop {r7, pc}
+;
+; CHECK-BE-LABEL: multi_user_zext:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    .save {r7, lr}
+; CHECK-BE-NEXT:    push {r7, lr}
+; CHECK-BE-NEXT:    .vsave {d8, d9}
+; CHECK-BE-NEXT:    vpush {d8, d9}
+; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    vpt.s32 gt, q1, zr
+; CHECK-BE-NEXT:    vldrht.u32 q4, [r0]
+; CHECK-BE-NEXT:    vrev64.32 q0, q4
+; CHECK-BE-NEXT:    vmov r1, r0, d0
+; CHECK-BE-NEXT:    vmov r3, r2, d1
+; CHECK-BE-NEXT:    bl foo
+; CHECK-BE-NEXT:    vmovlb.u16 q1, q4
+; CHECK-BE-NEXT:    vrev64.32 q0, q1
+; CHECK-BE-NEXT:    vpop {d8, d9}
+; CHECK-BE-NEXT:    pop {r7, pc}
+entry:
+  %c = icmp sgt <4 x i32> %a, zeroinitializer
+  %l = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %dest, i32 2, <4 x i1> %c, <4 x i16> zeroinitializer)
+  call void @foo(<4 x i16> %l)
+  %ext = zext <4 x i16> %l to <4 x i32>
+  ret <4 x i32> %ext
+}
+
+define arm_aapcs_vfpcc <4 x i32> @multi_user_sext(<4 x i16> *%dest, <4 x i32> %a) {
+; CHECK-LE-LABEL: multi_user_sext:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    .save {r7, lr}
+; CHECK-LE-NEXT:    push {r7, lr}
+; CHECK-LE-NEXT:    .vsave {d8, d9}
+; CHECK-LE-NEXT:    vpush {d8, d9}
+; CHECK-LE-NEXT:    vpt.s32 gt, q0, zr
+; CHECK-LE-NEXT:    vldrht.u32 q4, [r0]
+; CHECK-LE-NEXT:    vmov r0, r1, d8
+; CHECK-LE-NEXT:    vmov r2, r3, d9
+; CHECK-LE-NEXT:    bl foo
+; CHECK-LE-NEXT:    vmovlb.s16 q0, q4
+; CHECK-LE-NEXT:    vpop {d8, d9}
+; CHECK-LE-NEXT:    pop {r7, pc}
+;
+; CHECK-BE-LABEL: multi_user_sext:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    .save {r7, lr}
+; CHECK-BE-NEXT:    push {r7, lr}
+; CHECK-BE-NEXT:    .vsave {d8, d9}
+; CHECK-BE-NEXT:    vpush {d8, d9}
+; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    vpt.s32 gt, q1, zr
+; CHECK-BE-NEXT:    vldrht.u32 q4, [r0]
+; CHECK-BE-NEXT:    vrev64.32 q0, q4
+; CHECK-BE-NEXT:    vmov r1, r0, d0
+; CHECK-BE-NEXT:    vmov r3, r2, d1
+; CHECK-BE-NEXT:    bl foo
+; CHECK-BE-NEXT:    vmovlb.s16 q1, q4
+; CHECK-BE-NEXT:    vrev64.32 q0, q1
+; CHECK-BE-NEXT:    vpop {d8, d9}
+; CHECK-BE-NEXT:    pop {r7, pc}
+entry:
+  %c = icmp sgt <4 x i32> %a, zeroinitializer
+  %l = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %dest, i32 2, <4 x i1> %c, <4 x i16> zeroinitializer)
+  call void @foo(<4 x i16> %l)
+  %ext = sext <4 x i16> %l to <4 x i32>
+  ret <4 x i32> %ext
+}
+
+declare <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>*, i32, <4 x i1>, <4 x i16>)
+declare <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>*, i32, <4 x i1>, <4 x i32>)
+declare <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>*, i32, <8 x i1>, <8 x i16>)
+declare <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>*, i32, <4 x i1>, <4 x i8>)
+declare <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>*, i32, <8 x i1>, <8 x i8>)
+declare <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>*, i32, <16 x i1>, <16 x i8>)
+declare <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>*, i32, <4 x i1>, <4 x float>)
+declare <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>*, i32, <8 x i1>, <8 x half>)
+declare <2 x i64> @llvm.masked.load.v2i64.p0v2i64(<2 x i64>*, i32, <2 x i1>, <2 x i64>)
+declare <2 x double> @llvm.masked.load.v2f64.p0v2f64(<2 x double>*, i32, <2 x i1>, <2 x double>)
+declare void @foo(<4 x i16>)
diff --git a/llvm/test/CodeGen/Thumb2/mve-masked-store.ll b/llvm/test/CodeGen/Thumb2/mve-masked-store.ll
new file mode 100644
index 000000000..7e320626d
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-masked-store.ll
@@ -0,0 +1,1819 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve,+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-LE
+; RUN: llc -mtriple=thumbebv8.1m.main-arm-none-eabi -mattr=+mve,+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-BE
+
+define arm_aapcs_vfpcc void @masked_v4i32(<4 x i32> *%dest, <4 x i32> %a) {
+; CHECK-LE-LABEL: masked_v4i32:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vpt.s32 gt, q0, zr
+; CHECK-LE-NEXT:    vstrwt.32 q0, [r0]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: masked_v4i32:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    vpt.s32 gt, q1, zr
+; CHECK-BE-NEXT:    vstrwt.32 q1, [r0]
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = icmp sgt <4 x i32> %a, zeroinitializer
+  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %a, <4 x i32>* %dest, i32 4, <4 x i1> %c)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @masked_v4i32_align1(<4 x i32> *%dest, <4 x i32> %a) {
+; CHECK-LE-LABEL: masked_v4i32_align1:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    .pad #4
+; CHECK-LE-NEXT:    sub sp, #4
+; CHECK-LE-NEXT:    vcmp.s32 gt, q0, zr
+; CHECK-LE-NEXT:    vmrs r2, p0
+; CHECK-LE-NEXT:    and r1, r2, #1
+; CHECK-LE-NEXT:    rsbs r3, r1, #0
+; CHECK-LE-NEXT:    movs r1, #0
+; CHECK-LE-NEXT:    bfi r1, r3, #0, #1
+; CHECK-LE-NEXT:    ubfx r3, r2, #4, #1
+; CHECK-LE-NEXT:    rsbs r3, r3, #0
+; CHECK-LE-NEXT:    bfi r1, r3, #1, #1
+; CHECK-LE-NEXT:    ubfx r3, r2, #8, #1
+; CHECK-LE-NEXT:    ubfx r2, r2, #12, #1
+; CHECK-LE-NEXT:    rsbs r3, r3, #0
+; CHECK-LE-NEXT:    bfi r1, r3, #2, #1
+; CHECK-LE-NEXT:    rsbs r2, r2, #0
+; CHECK-LE-NEXT:    bfi r1, r2, #3, #1
+; CHECK-LE-NEXT:    lsls r2, r1, #31
+; CHECK-LE-NEXT:    itt ne
+; CHECK-LE-NEXT:    vmovne r2, s0
+; CHECK-LE-NEXT:    strne r2, [r0]
+; CHECK-LE-NEXT:    lsls r2, r1, #30
+; CHECK-LE-NEXT:    itt mi
+; CHECK-LE-NEXT:    vmovmi r2, s1
+; CHECK-LE-NEXT:    strmi r2, [r0, #4]
+; CHECK-LE-NEXT:    lsls r2, r1, #29
+; CHECK-LE-NEXT:    itt mi
+; CHECK-LE-NEXT:    vmovmi r2, s2
+; CHECK-LE-NEXT:    strmi r2, [r0, #8]
+; CHECK-LE-NEXT:    lsls r1, r1, #28
+; CHECK-LE-NEXT:    itt mi
+; CHECK-LE-NEXT:    vmovmi r1, s3
+; CHECK-LE-NEXT:    strmi r1, [r0, #12]
+; CHECK-LE-NEXT:    add sp, #4
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: masked_v4i32_align1:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    .pad #4
+; CHECK-BE-NEXT:    sub sp, #4
+; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    vcmp.s32 gt, q1, zr
+; CHECK-BE-NEXT:    vmrs r2, p0
+; CHECK-BE-NEXT:    and r1, r2, #1
+; CHECK-BE-NEXT:    rsbs r3, r1, #0
+; CHECK-BE-NEXT:    movs r1, #0
+; CHECK-BE-NEXT:    bfi r1, r3, #0, #1
+; CHECK-BE-NEXT:    ubfx r3, r2, #4, #1
+; CHECK-BE-NEXT:    rsbs r3, r3, #0
+; CHECK-BE-NEXT:    bfi r1, r3, #1, #1
+; CHECK-BE-NEXT:    ubfx r3, r2, #8, #1
+; CHECK-BE-NEXT:    ubfx r2, r2, #12, #1
+; CHECK-BE-NEXT:    rsbs r3, r3, #0
+; CHECK-BE-NEXT:    bfi r1, r3, #2, #1
+; CHECK-BE-NEXT:    rsbs r2, r2, #0
+; CHECK-BE-NEXT:    bfi r1, r2, #3, #1
+; CHECK-BE-NEXT:    lsls r2, r1, #31
+; CHECK-BE-NEXT:    itt ne
+; CHECK-BE-NEXT:    vmovne r2, s4
+; CHECK-BE-NEXT:    strne r2, [r0]
+; CHECK-BE-NEXT:    lsls r2, r1, #30
+; CHECK-BE-NEXT:    itt mi
+; CHECK-BE-NEXT:    vmovmi r2, s5
+; CHECK-BE-NEXT:    strmi r2, [r0, #4]
+; CHECK-BE-NEXT:    lsls r2, r1, #29
+; CHECK-BE-NEXT:    itt mi
+; CHECK-BE-NEXT:    vmovmi r2, s6
+; CHECK-BE-NEXT:    strmi r2, [r0, #8]
+; CHECK-BE-NEXT:    lsls r1, r1, #28
+; CHECK-BE-NEXT:    itt mi
+; CHECK-BE-NEXT:    vmovmi r1, s7
+; CHECK-BE-NEXT:    strmi r1, [r0, #12]
+; CHECK-BE-NEXT:    add sp, #4
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = icmp sgt <4 x i32> %a, zeroinitializer
+  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %a, <4 x i32>* %dest, i32 1, <4 x i1> %c)
+  ret void
+}
+
+define i8* @masked_v4i32_pre(i8* %y, i8* %x, <4 x i32> %a) {
+; CHECK-LE-LABEL: masked_v4i32_pre:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldr d1, [sp]
+; CHECK-LE-NEXT:    vldrw.u32 q1, [r1]
+; CHECK-LE-NEXT:    vmov d0, r2, r3
+; CHECK-LE-NEXT:    vpt.s32 gt, q0, zr
+; CHECK-LE-NEXT:    vstrwt.32 q1, [r0, #4]!
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: masked_v4i32_pre:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vldr d1, [sp]
+; CHECK-BE-NEXT:    vldrw.u32 q1, [r1]
+; CHECK-BE-NEXT:    vmov d0, r3, r2
+; CHECK-BE-NEXT:    vrev64.32 q2, q0
+; CHECK-BE-NEXT:    vpt.s32 gt, q2, zr
+; CHECK-BE-NEXT:    vstrwt.32 q1, [r0, #4]!
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 4
+  %0 = bitcast i8* %x to <4 x i32>*
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
+  %2 = bitcast i8* %z to <4 x i32>*
+  %c = icmp sgt <4 x i32> %a, zeroinitializer
+  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %1, <4 x i32>* %2, i32 4, <4 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @masked_v4i32_post(i8* %y, i8* %x, <4 x i32> %a) {
+; CHECK-LE-LABEL: masked_v4i32_post:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldr d1, [sp]
+; CHECK-LE-NEXT:    vldrw.u32 q1, [r1]
+; CHECK-LE-NEXT:    vmov d0, r2, r3
+; CHECK-LE-NEXT:    vpt.s32 gt, q0, zr
+; CHECK-LE-NEXT:    vstrwt.32 q1, [r0], #4
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: masked_v4i32_post:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vldr d1, [sp]
+; CHECK-BE-NEXT:    vldrw.u32 q1, [r1]
+; CHECK-BE-NEXT:    vmov d0, r3, r2
+; CHECK-BE-NEXT:    vrev64.32 q2, q0
+; CHECK-BE-NEXT:    vpt.s32 gt, q2, zr
+; CHECK-BE-NEXT:    vstrwt.32 q1, [r0], #4
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 4
+  %0 = bitcast i8* %x to <4 x i32>*
+  %1 = load <4 x i32>, <4 x i32>* %0, align 4
+  %2 = bitcast i8* %y to <4 x i32>*
+  %c = icmp sgt <4 x i32> %a, zeroinitializer
+  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %1, <4 x i32>* %2, i32 4, <4 x i1> %c)
+  ret i8* %z
+}
+
+
+define arm_aapcs_vfpcc void @masked_v8i16(<8 x i16> *%dest, <8 x i16> %a) {
+; CHECK-LE-LABEL: masked_v8i16:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vpt.s16 gt, q0, zr
+; CHECK-LE-NEXT:    vstrht.16 q0, [r0]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: masked_v8i16:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vrev64.16 q1, q0
+; CHECK-BE-NEXT:    vpt.s16 gt, q1, zr
+; CHECK-BE-NEXT:    vstrht.16 q1, [r0]
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = icmp sgt <8 x i16> %a, zeroinitializer
+  call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %a, <8 x i16>* %dest, i32 2, <8 x i1> %c)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @masked_v8i16_align1(<8 x i16> *%dest, <8 x i16> %a) {
+; CHECK-LE-LABEL: masked_v8i16_align1:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    .pad #8
+; CHECK-LE-NEXT:    sub sp, #8
+; CHECK-LE-NEXT:    vcmp.s16 gt, q0, zr
+; CHECK-LE-NEXT:    vmrs r1, p0
+; CHECK-LE-NEXT:    and r2, r1, #1
+; CHECK-LE-NEXT:    rsbs r3, r2, #0
+; CHECK-LE-NEXT:    movs r2, #0
+; CHECK-LE-NEXT:    bfi r2, r3, #0, #1
+; CHECK-LE-NEXT:    ubfx r3, r1, #2, #1
+; CHECK-LE-NEXT:    rsbs r3, r3, #0
+; CHECK-LE-NEXT:    bfi r2, r3, #1, #1
+; CHECK-LE-NEXT:    ubfx r3, r1, #4, #1
+; CHECK-LE-NEXT:    rsbs r3, r3, #0
+; CHECK-LE-NEXT:    bfi r2, r3, #2, #1
+; CHECK-LE-NEXT:    ubfx r3, r1, #6, #1
+; CHECK-LE-NEXT:    rsbs r3, r3, #0
+; CHECK-LE-NEXT:    bfi r2, r3, #3, #1
+; CHECK-LE-NEXT:    ubfx r3, r1, #8, #1
+; CHECK-LE-NEXT:    rsbs r3, r3, #0
+; CHECK-LE-NEXT:    bfi r2, r3, #4, #1
+; CHECK-LE-NEXT:    ubfx r3, r1, #10, #1
+; CHECK-LE-NEXT:    rsbs r3, r3, #0
+; CHECK-LE-NEXT:    bfi r2, r3, #5, #1
+; CHECK-LE-NEXT:    ubfx r3, r1, #12, #1
+; CHECK-LE-NEXT:    ubfx r1, r1, #14, #1
+; CHECK-LE-NEXT:    rsbs r3, r3, #0
+; CHECK-LE-NEXT:    bfi r2, r3, #6, #1
+; CHECK-LE-NEXT:    rsbs r1, r1, #0
+; CHECK-LE-NEXT:    bfi r2, r1, #7, #1
+; CHECK-LE-NEXT:    uxtb r1, r2
+; CHECK-LE-NEXT:    lsls r2, r2, #31
+; CHECK-LE-NEXT:    itt ne
+; CHECK-LE-NEXT:    vmovne.u16 r2, q0[0]
+; CHECK-LE-NEXT:    strhne r2, [r0]
+; CHECK-LE-NEXT:    lsls r2, r1, #30
+; CHECK-LE-NEXT:    itt mi
+; CHECK-LE-NEXT:    vmovmi.u16 r2, q0[1]
+; CHECK-LE-NEXT:    strhmi r2, [r0, #2]
+; CHECK-LE-NEXT:    lsls r2, r1, #29
+; CHECK-LE-NEXT:    itt mi
+; CHECK-LE-NEXT:    vmovmi.u16 r2, q0[2]
+; CHECK-LE-NEXT:    strhmi r2, [r0, #4]
+; CHECK-LE-NEXT:    lsls r2, r1, #28
+; CHECK-LE-NEXT:    itt mi
+; CHECK-LE-NEXT:    vmovmi.u16 r2, q0[3]
+; CHECK-LE-NEXT:    strhmi r2, [r0, #6]
+; CHECK-LE-NEXT:    lsls r2, r1, #27
+; CHECK-LE-NEXT:    itt mi
+; CHECK-LE-NEXT:    vmovmi.u16 r2, q0[4]
+; CHECK-LE-NEXT:    strhmi r2, [r0, #8]
+; CHECK-LE-NEXT:    lsls r2, r1, #26
+; CHECK-LE-NEXT:    itt mi
+; CHECK-LE-NEXT:    vmovmi.u16 r2, q0[5]
+; CHECK-LE-NEXT:    strhmi r2, [r0, #10]
+; CHECK-LE-NEXT:    lsls r2, r1, #25
+; CHECK-LE-NEXT:    itt mi
+; CHECK-LE-NEXT:    vmovmi.u16 r2, q0[6]
+; CHECK-LE-NEXT:    strhmi r2, [r0, #12]
+; CHECK-LE-NEXT:    lsls r1, r1, #24
+; CHECK-LE-NEXT:    itt mi
+; CHECK-LE-NEXT:    vmovmi.u16 r1, q0[7]
+; CHECK-LE-NEXT:    strhmi r1, [r0, #14]
+; CHECK-LE-NEXT:    add sp, #8
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: masked_v8i16_align1:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    .pad #8
+; CHECK-BE-NEXT:    sub sp, #8
+; CHECK-BE-NEXT:    vrev64.16 q1, q0
+; CHECK-BE-NEXT:    vcmp.s16 gt, q1, zr
+; CHECK-BE-NEXT:    vmrs r1, p0
+; CHECK-BE-NEXT:    and r2, r1, #1
+; CHECK-BE-NEXT:    rsbs r3, r2, #0
+; CHECK-BE-NEXT:    movs r2, #0
+; CHECK-BE-NEXT:    bfi r2, r3, #0, #1
+; CHECK-BE-NEXT:    ubfx r3, r1, #2, #1
+; CHECK-BE-NEXT:    rsbs r3, r3, #0
+; CHECK-BE-NEXT:    bfi r2, r3, #1, #1
+; CHECK-BE-NEXT:    ubfx r3, r1, #4, #1
+; CHECK-BE-NEXT:    rsbs r3, r3, #0
+; CHECK-BE-NEXT:    bfi r2, r3, #2, #1
+; CHECK-BE-NEXT:    ubfx r3, r1, #6, #1
+; CHECK-BE-NEXT:    rsbs r3, r3, #0
+; CHECK-BE-NEXT:    bfi r2, r3, #3, #1
+; CHECK-BE-NEXT:    ubfx r3, r1, #8, #1
+; CHECK-BE-NEXT:    rsbs r3, r3, #0
+; CHECK-BE-NEXT:    bfi r2, r3, #4, #1
+; CHECK-BE-NEXT:    ubfx r3, r1, #10, #1
+; CHECK-BE-NEXT:    rsbs r3, r3, #0
+; CHECK-BE-NEXT:    bfi r2, r3, #5, #1
+; CHECK-BE-NEXT:    ubfx r3, r1, #12, #1
+; CHECK-BE-NEXT:    ubfx r1, r1, #14, #1
+; CHECK-BE-NEXT:    rsbs r3, r3, #0
+; CHECK-BE-NEXT:    bfi r2, r3, #6, #1
+; CHECK-BE-NEXT:    rsbs r1, r1, #0
+; CHECK-BE-NEXT:    bfi r2, r1, #7, #1
+; CHECK-BE-NEXT:    uxtb r1, r2
+; CHECK-BE-NEXT:    lsls r2, r2, #31
+; CHECK-BE-NEXT:    itt ne
+; CHECK-BE-NEXT:    vmovne.u16 r2, q1[0]
+; CHECK-BE-NEXT:    strhne r2, [r0]
+; CHECK-BE-NEXT:    lsls r2, r1, #30
+; CHECK-BE-NEXT:    itt mi
+; CHECK-BE-NEXT:    vmovmi.u16 r2, q1[1]
+; CHECK-BE-NEXT:    strhmi r2, [r0, #2]
+; CHECK-BE-NEXT:    lsls r2, r1, #29
+; CHECK-BE-NEXT:    itt mi
+; CHECK-BE-NEXT:    vmovmi.u16 r2, q1[2]
+; CHECK-BE-NEXT:    strhmi r2, [r0, #4]
+; CHECK-BE-NEXT:    lsls r2, r1, #28
+; CHECK-BE-NEXT:    itt mi
+; CHECK-BE-NEXT:    vmovmi.u16 r2, q1[3]
+; CHECK-BE-NEXT:    strhmi r2, [r0, #6]
+; CHECK-BE-NEXT:    lsls r2, r1, #27
+; CHECK-BE-NEXT:    itt mi
+; CHECK-BE-NEXT:    vmovmi.u16 r2, q1[4]
+; CHECK-BE-NEXT:    strhmi r2, [r0, #8]
+; CHECK-BE-NEXT:    lsls r2, r1, #26
+; CHECK-BE-NEXT:    itt mi
+; CHECK-BE-NEXT:    vmovmi.u16 r2, q1[5]
+; CHECK-BE-NEXT:    strhmi r2, [r0, #10]
+; CHECK-BE-NEXT:    lsls r2, r1, #25
+; CHECK-BE-NEXT:    itt mi
+; CHECK-BE-NEXT:    vmovmi.u16 r2, q1[6]
+; CHECK-BE-NEXT:    strhmi r2, [r0, #12]
+; CHECK-BE-NEXT:    lsls r1, r1, #24
+; CHECK-BE-NEXT:    itt mi
+; CHECK-BE-NEXT:    vmovmi.u16 r1, q1[7]
+; CHECK-BE-NEXT:    strhmi r1, [r0, #14]
+; CHECK-BE-NEXT:    add sp, #8
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = icmp sgt <8 x i16> %a, zeroinitializer
+  call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %a, <8 x i16>* %dest, i32 1, <8 x i1> %c)
+  ret void
+}
+
+define i8* @masked_v8i16_pre(i8* %y, i8* %x, <8 x i16> %a) {
+; CHECK-LE-LABEL: masked_v8i16_pre:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldr d1, [sp]
+; CHECK-LE-NEXT:    vldrw.u32 q1, [r1]
+; CHECK-LE-NEXT:    vmov d0, r2, r3
+; CHECK-LE-NEXT:    vpt.s16 gt, q0, zr
+; CHECK-LE-NEXT:    vstrht.16 q1, [r0, #4]!
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: masked_v8i16_pre:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vldr d1, [sp]
+; CHECK-BE-NEXT:    vldrh.u16 q1, [r1]
+; CHECK-BE-NEXT:    vmov d0, r3, r2
+; CHECK-BE-NEXT:    vrev64.16 q2, q0
+; CHECK-BE-NEXT:    vpt.s16 gt, q2, zr
+; CHECK-BE-NEXT:    vstrht.16 q1, [r0, #4]!
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 4
+  %0 = bitcast i8* %x to <8 x i16>*
+  %1 = load <8 x i16>, <8 x i16>* %0, align 4
+  %2 = bitcast i8* %z to <8 x i16>*
+  %c = icmp sgt <8 x i16> %a, zeroinitializer
+  call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %1, <8 x i16>* %2, i32 2, <8 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @masked_v8i16_post(i8* %y, i8* %x, <8 x i16> %a) {
+; CHECK-LE-LABEL: masked_v8i16_post:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldr d1, [sp]
+; CHECK-LE-NEXT:    vldrw.u32 q1, [r1]
+; CHECK-LE-NEXT:    vmov d0, r2, r3
+; CHECK-LE-NEXT:    vpt.s16 gt, q0, zr
+; CHECK-LE-NEXT:    vstrht.16 q1, [r0], #4
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: masked_v8i16_post:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vldr d1, [sp]
+; CHECK-BE-NEXT:    vldrh.u16 q1, [r1]
+; CHECK-BE-NEXT:    vmov d0, r3, r2
+; CHECK-BE-NEXT:    vrev64.16 q2, q0
+; CHECK-BE-NEXT:    vpt.s16 gt, q2, zr
+; CHECK-BE-NEXT:    vstrht.16 q1, [r0], #4
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 4
+  %0 = bitcast i8* %x to <8 x i16>*
+  %1 = load <8 x i16>, <8 x i16>* %0, align 4
+  %2 = bitcast i8* %y to <8 x i16>*
+  %c = icmp sgt <8 x i16> %a, zeroinitializer
+  call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %1, <8 x i16>* %2, i32 2, <8 x i1> %c)
+  ret i8* %z
+}
+
+
+define arm_aapcs_vfpcc void @masked_v16i8(<16 x i8> *%dest, <16 x i8> %a) {
+; CHECK-LE-LABEL: masked_v16i8:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vpt.s8 gt, q0, zr
+; CHECK-LE-NEXT:    vstrbt.8 q0, [r0]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: masked_v16i8:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vrev64.8 q1, q0
+; CHECK-BE-NEXT:    vpt.s8 gt, q1, zr
+; CHECK-BE-NEXT:    vstrbt.8 q1, [r0]
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = icmp sgt <16 x i8> %a, zeroinitializer
+  call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %a, <16 x i8>* %dest, i32 1, <16 x i1> %c)
+  ret void
+}
+
+define i8* @masked_v16i8_pre(i8* %y, i8* %x, <16 x i8> %a) {
+; CHECK-LE-LABEL: masked_v16i8_pre:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldr d1, [sp]
+; CHECK-LE-NEXT:    vldrw.u32 q1, [r1]
+; CHECK-LE-NEXT:    vmov d0, r2, r3
+; CHECK-LE-NEXT:    vpt.s8 gt, q0, zr
+; CHECK-LE-NEXT:    vstrbt.8 q1, [r0, #4]!
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: masked_v16i8_pre:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vldr d1, [sp]
+; CHECK-BE-NEXT:    vldrb.u8 q1, [r1]
+; CHECK-BE-NEXT:    vmov d0, r3, r2
+; CHECK-BE-NEXT:    vrev64.8 q2, q0
+; CHECK-BE-NEXT:    vpt.s8 gt, q2, zr
+; CHECK-BE-NEXT:    vstrbt.8 q1, [r0, #4]!
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 4
+  %0 = bitcast i8* %x to <16 x i8>*
+  %1 = load <16 x i8>, <16 x i8>* %0, align 4
+  %2 = bitcast i8* %z to <16 x i8>*
+  %c = icmp sgt <16 x i8> %a, zeroinitializer
+  call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %1, <16 x i8>* %2, i32 1, <16 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @masked_v16i8_post(i8* %y, i8* %x, <16 x i8> %a) {
+; CHECK-LE-LABEL: masked_v16i8_post:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldr d1, [sp]
+; CHECK-LE-NEXT:    vldrw.u32 q1, [r1]
+; CHECK-LE-NEXT:    vmov d0, r2, r3
+; CHECK-LE-NEXT:    vpt.s8 gt, q0, zr
+; CHECK-LE-NEXT:    vstrbt.8 q1, [r0], #4
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: masked_v16i8_post:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vldr d1, [sp]
+; CHECK-BE-NEXT:    vldrb.u8 q1, [r1]
+; CHECK-BE-NEXT:    vmov d0, r3, r2
+; CHECK-BE-NEXT:    vrev64.8 q2, q0
+; CHECK-BE-NEXT:    vpt.s8 gt, q2, zr
+; CHECK-BE-NEXT:    vstrbt.8 q1, [r0], #4
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 4
+  %0 = bitcast i8* %x to <16 x i8>*
+  %1 = load <16 x i8>, <16 x i8>* %0, align 4
+  %2 = bitcast i8* %y to <16 x i8>*
+  %c = icmp sgt <16 x i8> %a, zeroinitializer
+  call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %1, <16 x i8>* %2, i32 1, <16 x i1> %c)
+  ret i8* %z
+}
+
+
+define arm_aapcs_vfpcc void @masked_v4f32(<4 x float> *%dest, <4 x float> %a, <4 x i32> %b) {
+; CHECK-LE-LABEL: masked_v4f32:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-LE-NEXT:    vstrwt.32 q0, [r0]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: masked_v4f32:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vrev64.32 q2, q1
+; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    vpt.i32 ne, q2, zr
+; CHECK-BE-NEXT:    vstrwt.32 q1, [r0]
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = icmp ugt <4 x i32> %b, zeroinitializer
+  call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %a, <4 x float>* %dest, i32 4, <4 x i1> %c)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @masked_v4f32_align1(<4 x float> *%dest, <4 x float> %a, <4 x i32> %b) {
+; CHECK-LE-LABEL: masked_v4f32_align1:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    .pad #20
+; CHECK-LE-NEXT:    sub sp, #20
+; CHECK-LE-NEXT:    vcmp.i32 ne, q1, zr
+; CHECK-LE-NEXT:    movs r1, #0
+; CHECK-LE-NEXT:    vmrs r2, p0
+; CHECK-LE-NEXT:    and r3, r2, #1
+; CHECK-LE-NEXT:    rsbs r3, r3, #0
+; CHECK-LE-NEXT:    bfi r1, r3, #0, #1
+; CHECK-LE-NEXT:    ubfx r3, r2, #4, #1
+; CHECK-LE-NEXT:    rsbs r3, r3, #0
+; CHECK-LE-NEXT:    bfi r1, r3, #1, #1
+; CHECK-LE-NEXT:    ubfx r3, r2, #8, #1
+; CHECK-LE-NEXT:    ubfx r2, r2, #12, #1
+; CHECK-LE-NEXT:    rsbs r3, r3, #0
+; CHECK-LE-NEXT:    bfi r1, r3, #2, #1
+; CHECK-LE-NEXT:    rsbs r2, r2, #0
+; CHECK-LE-NEXT:    bfi r1, r2, #3, #1
+; CHECK-LE-NEXT:    lsls r2, r1, #31
+; CHECK-LE-NEXT:    ittt ne
+; CHECK-LE-NEXT:    vstrne s0, [sp, #12]
+; CHECK-LE-NEXT:    ldrne r2, [sp, #12]
+; CHECK-LE-NEXT:    strne r2, [r0]
+; CHECK-LE-NEXT:    lsls r2, r1, #30
+; CHECK-LE-NEXT:    ittt mi
+; CHECK-LE-NEXT:    vstrmi s1, [sp, #8]
+; CHECK-LE-NEXT:    ldrmi r2, [sp, #8]
+; CHECK-LE-NEXT:    strmi r2, [r0, #4]
+; CHECK-LE-NEXT:    lsls r2, r1, #29
+; CHECK-LE-NEXT:    ittt mi
+; CHECK-LE-NEXT:    vstrmi s2, [sp, #4]
+; CHECK-LE-NEXT:    ldrmi r2, [sp, #4]
+; CHECK-LE-NEXT:    strmi r2, [r0, #8]
+; CHECK-LE-NEXT:    lsls r1, r1, #28
+; CHECK-LE-NEXT:    ittt mi
+; CHECK-LE-NEXT:    vstrmi s3, [sp]
+; CHECK-LE-NEXT:    ldrmi r1, [sp]
+; CHECK-LE-NEXT:    strmi r1, [r0, #12]
+; CHECK-LE-NEXT:    add sp, #20
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: masked_v4f32_align1:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    .pad #20
+; CHECK-BE-NEXT:    sub sp, #20
+; CHECK-BE-NEXT:    vrev64.32 q2, q1
+; CHECK-BE-NEXT:    movs r1, #0
+; CHECK-BE-NEXT:    vcmp.i32 ne, q2, zr
+; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    vmrs r2, p0
+; CHECK-BE-NEXT:    and r3, r2, #1
+; CHECK-BE-NEXT:    rsbs r3, r3, #0
+; CHECK-BE-NEXT:    bfi r1, r3, #0, #1
+; CHECK-BE-NEXT:    ubfx r3, r2, #4, #1
+; CHECK-BE-NEXT:    rsbs r3, r3, #0
+; CHECK-BE-NEXT:    bfi r1, r3, #1, #1
+; CHECK-BE-NEXT:    ubfx r3, r2, #8, #1
+; CHECK-BE-NEXT:    ubfx r2, r2, #12, #1
+; CHECK-BE-NEXT:    rsbs r3, r3, #0
+; CHECK-BE-NEXT:    bfi r1, r3, #2, #1
+; CHECK-BE-NEXT:    rsbs r2, r2, #0
+; CHECK-BE-NEXT:    bfi r1, r2, #3, #1
+; CHECK-BE-NEXT:    lsls r2, r1, #31
+; CHECK-BE-NEXT:    ittt ne
+; CHECK-BE-NEXT:    vstrne s4, [sp, #12]
+; CHECK-BE-NEXT:    ldrne r2, [sp, #12]
+; CHECK-BE-NEXT:    strne r2, [r0]
+; CHECK-BE-NEXT:    lsls r2, r1, #30
+; CHECK-BE-NEXT:    ittt mi
+; CHECK-BE-NEXT:    vstrmi s5, [sp, #8]
+; CHECK-BE-NEXT:    ldrmi r2, [sp, #8]
+; CHECK-BE-NEXT:    strmi r2, [r0, #4]
+; CHECK-BE-NEXT:    lsls r2, r1, #29
+; CHECK-BE-NEXT:    ittt mi
+; CHECK-BE-NEXT:    vstrmi s6, [sp, #4]
+; CHECK-BE-NEXT:    ldrmi r2, [sp, #4]
+; CHECK-BE-NEXT:    strmi r2, [r0, #8]
+; CHECK-BE-NEXT:    lsls r1, r1, #28
+; CHECK-BE-NEXT:    ittt mi
+; CHECK-BE-NEXT:    vstrmi s7, [sp]
+; CHECK-BE-NEXT:    ldrmi r1, [sp]
+; CHECK-BE-NEXT:    strmi r1, [r0, #12]
+; CHECK-BE-NEXT:    add sp, #20
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = icmp ugt <4 x i32> %b, zeroinitializer
+  call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %a, <4 x float>* %dest, i32 1, <4 x i1> %c)
+  ret void
+}
+
+define i8* @masked_v4f32_pre(i8* %y, i8* %x, <4 x i32> %a) {
+; CHECK-LE-LABEL: masked_v4f32_pre:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldr d1, [sp]
+; CHECK-LE-NEXT:    vldrw.u32 q1, [r1]
+; CHECK-LE-NEXT:    vmov d0, r2, r3
+; CHECK-LE-NEXT:    vpt.s32 gt, q0, zr
+; CHECK-LE-NEXT:    vstrwt.32 q1, [r0, #4]!
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: masked_v4f32_pre:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vldr d1, [sp]
+; CHECK-BE-NEXT:    vldrw.u32 q1, [r1]
+; CHECK-BE-NEXT:    vmov d0, r3, r2
+; CHECK-BE-NEXT:    vrev64.32 q2, q0
+; CHECK-BE-NEXT:    vpt.s32 gt, q2, zr
+; CHECK-BE-NEXT:    vstrwt.32 q1, [r0, #4]!
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 4
+  %0 = bitcast i8* %x to <4 x float>*
+  %1 = load <4 x float>, <4 x float>* %0, align 4
+  %2 = bitcast i8* %z to <4 x float>*
+  %c = icmp sgt <4 x i32> %a, zeroinitializer
+  call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %1, <4 x float>* %2, i32 4, <4 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @masked_v4f32_post(i8* %y, i8* %x, <4 x i32> %a) {
+; CHECK-LE-LABEL: masked_v4f32_post:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldr d1, [sp]
+; CHECK-LE-NEXT:    vldrw.u32 q1, [r1]
+; CHECK-LE-NEXT:    vmov d0, r2, r3
+; CHECK-LE-NEXT:    vpt.s32 gt, q0, zr
+; CHECK-LE-NEXT:    vstrwt.32 q1, [r0], #4
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: masked_v4f32_post:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vldr d1, [sp]
+; CHECK-BE-NEXT:    vldrw.u32 q1, [r1]
+; CHECK-BE-NEXT:    vmov d0, r3, r2
+; CHECK-BE-NEXT:    vrev64.32 q2, q0
+; CHECK-BE-NEXT:    vpt.s32 gt, q2, zr
+; CHECK-BE-NEXT:    vstrwt.32 q1, [r0], #4
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 4
+  %0 = bitcast i8* %x to <4 x float>*
+  %1 = load <4 x float>, <4 x float>* %0, align 4
+  %2 = bitcast i8* %y to <4 x float>*
+  %c = icmp sgt <4 x i32> %a, zeroinitializer
+  call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %1, <4 x float>* %2, i32 4, <4 x i1> %c)
+  ret i8* %z
+}
+
+
+define arm_aapcs_vfpcc void @masked_v8f16(<8 x half> *%dest, <8 x half> %a, <8 x i16> %b) {
+; CHECK-LE-LABEL: masked_v8f16:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vpt.i16 ne, q1, zr
+; CHECK-LE-NEXT:    vstrht.16 q0, [r0]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: masked_v8f16:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vrev64.16 q2, q1
+; CHECK-BE-NEXT:    vrev64.16 q1, q0
+; CHECK-BE-NEXT:    vpt.i16 ne, q2, zr
+; CHECK-BE-NEXT:    vstrht.16 q1, [r0]
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = icmp ugt <8 x i16> %b, zeroinitializer
+  call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %a, <8 x half>* %dest, i32 2, <8 x i1> %c)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @masked_v8f16_align1(<8 x half> *%dest, <8 x half> %a, <8 x i16> %b) {
+; CHECK-LE-LABEL: masked_v8f16_align1:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    .pad #40
+; CHECK-LE-NEXT:    sub sp, #40
+; CHECK-LE-NEXT:    vcmp.i16 ne, q1, zr
+; CHECK-LE-NEXT:    movs r2, #0
+; CHECK-LE-NEXT:    vmrs r1, p0
+; CHECK-LE-NEXT:    and r3, r1, #1
+; CHECK-LE-NEXT:    rsbs r3, r3, #0
+; CHECK-LE-NEXT:    bfi r2, r3, #0, #1
+; CHECK-LE-NEXT:    ubfx r3, r1, #2, #1
+; CHECK-LE-NEXT:    rsbs r3, r3, #0
+; CHECK-LE-NEXT:    bfi r2, r3, #1, #1
+; CHECK-LE-NEXT:    ubfx r3, r1, #4, #1
+; CHECK-LE-NEXT:    rsbs r3, r3, #0
+; CHECK-LE-NEXT:    bfi r2, r3, #2, #1
+; CHECK-LE-NEXT:    ubfx r3, r1, #6, #1
+; CHECK-LE-NEXT:    rsbs r3, r3, #0
+; CHECK-LE-NEXT:    bfi r2, r3, #3, #1
+; CHECK-LE-NEXT:    ubfx r3, r1, #8, #1
+; CHECK-LE-NEXT:    rsbs r3, r3, #0
+; CHECK-LE-NEXT:    bfi r2, r3, #4, #1
+; CHECK-LE-NEXT:    ubfx r3, r1, #10, #1
+; CHECK-LE-NEXT:    rsbs r3, r3, #0
+; CHECK-LE-NEXT:    bfi r2, r3, #5, #1
+; CHECK-LE-NEXT:    ubfx r3, r1, #12, #1
+; CHECK-LE-NEXT:    ubfx r1, r1, #14, #1
+; CHECK-LE-NEXT:    rsbs r3, r3, #0
+; CHECK-LE-NEXT:    bfi r2, r3, #6, #1
+; CHECK-LE-NEXT:    rsbs r1, r1, #0
+; CHECK-LE-NEXT:    bfi r2, r1, #7, #1
+; CHECK-LE-NEXT:    uxtb r1, r2
+; CHECK-LE-NEXT:    lsls r2, r2, #31
+; CHECK-LE-NEXT:    bne .LBB16_9
+; CHECK-LE-NEXT:  @ %bb.1: @ %else
+; CHECK-LE-NEXT:    lsls r2, r1, #30
+; CHECK-LE-NEXT:    bmi .LBB16_10
+; CHECK-LE-NEXT:  .LBB16_2: @ %else2
+; CHECK-LE-NEXT:    lsls r2, r1, #29
+; CHECK-LE-NEXT:    bmi .LBB16_11
+; CHECK-LE-NEXT:  .LBB16_3: @ %else4
+; CHECK-LE-NEXT:    lsls r2, r1, #28
+; CHECK-LE-NEXT:    bmi .LBB16_12
+; CHECK-LE-NEXT:  .LBB16_4: @ %else6
+; CHECK-LE-NEXT:    lsls r2, r1, #27
+; CHECK-LE-NEXT:    bmi .LBB16_13
+; CHECK-LE-NEXT:  .LBB16_5: @ %else8
+; CHECK-LE-NEXT:    lsls r2, r1, #26
+; CHECK-LE-NEXT:    bmi .LBB16_14
+; CHECK-LE-NEXT:  .LBB16_6: @ %else10
+; CHECK-LE-NEXT:    lsls r2, r1, #25
+; CHECK-LE-NEXT:    bmi .LBB16_15
+; CHECK-LE-NEXT:  .LBB16_7: @ %else12
+; CHECK-LE-NEXT:    lsls r1, r1, #24
+; CHECK-LE-NEXT:    bmi .LBB16_16
+; CHECK-LE-NEXT:  .LBB16_8: @ %else14
+; CHECK-LE-NEXT:    add sp, #40
+; CHECK-LE-NEXT:    bx lr
+; CHECK-LE-NEXT:  .LBB16_9: @ %cond.store
+; CHECK-LE-NEXT:    vstr.16 s0, [sp, #28]
+; CHECK-LE-NEXT:    ldrh.w r2, [sp, #28]
+; CHECK-LE-NEXT:    strh r2, [r0]
+; CHECK-LE-NEXT:    lsls r2, r1, #30
+; CHECK-LE-NEXT:    bpl .LBB16_2
+; CHECK-LE-NEXT:  .LBB16_10: @ %cond.store1
+; CHECK-LE-NEXT:    vmovx.f16 s4, s0
+; CHECK-LE-NEXT:    vstr.16 s4, [sp, #24]
+; CHECK-LE-NEXT:    ldrh.w r2, [sp, #24]
+; CHECK-LE-NEXT:    strh r2, [r0, #2]
+; CHECK-LE-NEXT:    lsls r2, r1, #29
+; CHECK-LE-NEXT:    bpl .LBB16_3
+; CHECK-LE-NEXT:  .LBB16_11: @ %cond.store3
+; CHECK-LE-NEXT:    vstr.16 s1, [sp, #20]
+; CHECK-LE-NEXT:    ldrh.w r2, [sp, #20]
+; CHECK-LE-NEXT:    strh r2, [r0, #4]
+; CHECK-LE-NEXT:    lsls r2, r1, #28
+; CHECK-LE-NEXT:    bpl .LBB16_4
+; CHECK-LE-NEXT:  .LBB16_12: @ %cond.store5
+; CHECK-LE-NEXT:    vmovx.f16 s4, s1
+; CHECK-LE-NEXT:    vstr.16 s4, [sp, #16]
+; CHECK-LE-NEXT:    ldrh.w r2, [sp, #16]
+; CHECK-LE-NEXT:    strh r2, [r0, #6]
+; CHECK-LE-NEXT:    lsls r2, r1, #27
+; CHECK-LE-NEXT:    bpl .LBB16_5
+; CHECK-LE-NEXT:  .LBB16_13: @ %cond.store7
+; CHECK-LE-NEXT:    vstr.16 s2, [sp, #12]
+; CHECK-LE-NEXT:    ldrh.w r2, [sp, #12]
+; CHECK-LE-NEXT:    strh r2, [r0, #8]
+; CHECK-LE-NEXT:    lsls r2, r1, #26
+; CHECK-LE-NEXT:    bpl .LBB16_6
+; CHECK-LE-NEXT:  .LBB16_14: @ %cond.store9
+; CHECK-LE-NEXT:    vmovx.f16 s4, s2
+; CHECK-LE-NEXT:    vstr.16 s4, [sp, #8]
+; CHECK-LE-NEXT:    ldrh.w r2, [sp, #8]
+; CHECK-LE-NEXT:    strh r2, [r0, #10]
+; CHECK-LE-NEXT:    lsls r2, r1, #25
+; CHECK-LE-NEXT:    bpl .LBB16_7
+; CHECK-LE-NEXT:  .LBB16_15: @ %cond.store11
+; CHECK-LE-NEXT:    vstr.16 s3, [sp, #4]
+; CHECK-LE-NEXT:    ldrh.w r2, [sp, #4]
+; CHECK-LE-NEXT:    strh r2, [r0, #12]
+; CHECK-LE-NEXT:    lsls r1, r1, #24
+; CHECK-LE-NEXT:    bpl .LBB16_8
+; CHECK-LE-NEXT:  .LBB16_16: @ %cond.store13
+; CHECK-LE-NEXT:    vmovx.f16 s0, s3
+; CHECK-LE-NEXT:    vstr.16 s0, [sp]
+; CHECK-LE-NEXT:    ldrh.w r1, [sp]
+; CHECK-LE-NEXT:    strh r1, [r0, #14]
+; CHECK-LE-NEXT:    add sp, #40
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: masked_v8f16_align1:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    .pad #40
+; CHECK-BE-NEXT:    sub sp, #40
+; CHECK-BE-NEXT:    vrev64.16 q2, q1
+; CHECK-BE-NEXT:    vrev64.16 q1, q0
+; CHECK-BE-NEXT:    vcmp.i16 ne, q2, zr
+; CHECK-BE-NEXT:    vmrs r1, p0
+; CHECK-BE-NEXT:    and r2, r1, #1
+; CHECK-BE-NEXT:    rsbs r3, r2, #0
+; CHECK-BE-NEXT:    movs r2, #0
+; CHECK-BE-NEXT:    bfi r2, r3, #0, #1
+; CHECK-BE-NEXT:    ubfx r3, r1, #2, #1
+; CHECK-BE-NEXT:    rsbs r3, r3, #0
+; CHECK-BE-NEXT:    bfi r2, r3, #1, #1
+; CHECK-BE-NEXT:    ubfx r3, r1, #4, #1
+; CHECK-BE-NEXT:    rsbs r3, r3, #0
+; CHECK-BE-NEXT:    bfi r2, r3, #2, #1
+; CHECK-BE-NEXT:    ubfx r3, r1, #6, #1
+; CHECK-BE-NEXT:    rsbs r3, r3, #0
+; CHECK-BE-NEXT:    bfi r2, r3, #3, #1
+; CHECK-BE-NEXT:    ubfx r3, r1, #8, #1
+; CHECK-BE-NEXT:    rsbs r3, r3, #0
+; CHECK-BE-NEXT:    bfi r2, r3, #4, #1
+; CHECK-BE-NEXT:    ubfx r3, r1, #10, #1
+; CHECK-BE-NEXT:    rsbs r3, r3, #0
+; CHECK-BE-NEXT:    bfi r2, r3, #5, #1
+; CHECK-BE-NEXT:    ubfx r3, r1, #12, #1
+; CHECK-BE-NEXT:    ubfx r1, r1, #14, #1
+; CHECK-BE-NEXT:    rsbs r3, r3, #0
+; CHECK-BE-NEXT:    bfi r2, r3, #6, #1
+; CHECK-BE-NEXT:    rsbs r1, r1, #0
+; CHECK-BE-NEXT:    bfi r2, r1, #7, #1
+; CHECK-BE-NEXT:    uxtb r1, r2
+; CHECK-BE-NEXT:    lsls r2, r2, #31
+; CHECK-BE-NEXT:    bne .LBB16_9
+; CHECK-BE-NEXT:  @ %bb.1: @ %else
+; CHECK-BE-NEXT:    lsls r2, r1, #30
+; CHECK-BE-NEXT:    bmi .LBB16_10
+; CHECK-BE-NEXT:  .LBB16_2: @ %else2
+; CHECK-BE-NEXT:    lsls r2, r1, #29
+; CHECK-BE-NEXT:    bmi .LBB16_11
+; CHECK-BE-NEXT:  .LBB16_3: @ %else4
+; CHECK-BE-NEXT:    lsls r2, r1, #28
+; CHECK-BE-NEXT:    bmi .LBB16_12
+; CHECK-BE-NEXT:  .LBB16_4: @ %else6
+; CHECK-BE-NEXT:    lsls r2, r1, #27
+; CHECK-BE-NEXT:    bmi .LBB16_13
+; CHECK-BE-NEXT:  .LBB16_5: @ %else8
+; CHECK-BE-NEXT:    lsls r2, r1, #26
+; CHECK-BE-NEXT:    bmi .LBB16_14
+; CHECK-BE-NEXT:  .LBB16_6: @ %else10
+; CHECK-BE-NEXT:    lsls r2, r1, #25
+; CHECK-BE-NEXT:    bmi .LBB16_15
+; CHECK-BE-NEXT:  .LBB16_7: @ %else12
+; CHECK-BE-NEXT:    lsls r1, r1, #24
+; CHECK-BE-NEXT:    bmi .LBB16_16
+; CHECK-BE-NEXT:  .LBB16_8: @ %else14
+; CHECK-BE-NEXT:    add sp, #40
+; CHECK-BE-NEXT:    bx lr
+; CHECK-BE-NEXT:  .LBB16_9: @ %cond.store
+; CHECK-BE-NEXT:    vstr.16 s4, [sp, #28]
+; CHECK-BE-NEXT:    ldrh.w r2, [sp, #28]
+; CHECK-BE-NEXT:    strh r2, [r0]
+; CHECK-BE-NEXT:    lsls r2, r1, #30
+; CHECK-BE-NEXT:    bpl .LBB16_2
+; CHECK-BE-NEXT:  .LBB16_10: @ %cond.store1
+; CHECK-BE-NEXT:    vmovx.f16 s0, s4
+; CHECK-BE-NEXT:    vstr.16 s0, [sp, #24]
+; CHECK-BE-NEXT:    ldrh.w r2, [sp, #24]
+; CHECK-BE-NEXT:    strh r2, [r0, #2]
+; CHECK-BE-NEXT:    lsls r2, r1, #29
+; CHECK-BE-NEXT:    bpl .LBB16_3
+; CHECK-BE-NEXT:  .LBB16_11: @ %cond.store3
+; CHECK-BE-NEXT:    vstr.16 s5, [sp, #20]
+; CHECK-BE-NEXT:    ldrh.w r2, [sp, #20]
+; CHECK-BE-NEXT:    strh r2, [r0, #4]
+; CHECK-BE-NEXT:    lsls r2, r1, #28
+; CHECK-BE-NEXT:    bpl .LBB16_4
+; CHECK-BE-NEXT:  .LBB16_12: @ %cond.store5
+; CHECK-BE-NEXT:    vmovx.f16 s0, s5
+; CHECK-BE-NEXT:    vstr.16 s0, [sp, #16]
+; CHECK-BE-NEXT:    ldrh.w r2, [sp, #16]
+; CHECK-BE-NEXT:    strh r2, [r0, #6]
+; CHECK-BE-NEXT:    lsls r2, r1, #27
+; CHECK-BE-NEXT:    bpl .LBB16_5
+; CHECK-BE-NEXT:  .LBB16_13: @ %cond.store7
+; CHECK-BE-NEXT:    vstr.16 s6, [sp, #12]
+; CHECK-BE-NEXT:    ldrh.w r2, [sp, #12]
+; CHECK-BE-NEXT:    strh r2, [r0, #8]
+; CHECK-BE-NEXT:    lsls r2, r1, #26
+; CHECK-BE-NEXT:    bpl .LBB16_6
+; CHECK-BE-NEXT:  .LBB16_14: @ %cond.store9
+; CHECK-BE-NEXT:    vmovx.f16 s0, s6
+; CHECK-BE-NEXT:    vstr.16 s0, [sp, #8]
+; CHECK-BE-NEXT:    ldrh.w r2, [sp, #8]
+; CHECK-BE-NEXT:    strh r2, [r0, #10]
+; CHECK-BE-NEXT:    lsls r2, r1, #25
+; CHECK-BE-NEXT:    bpl .LBB16_7
+; CHECK-BE-NEXT:  .LBB16_15: @ %cond.store11
+; CHECK-BE-NEXT:    vstr.16 s7, [sp, #4]
+; CHECK-BE-NEXT:    ldrh.w r2, [sp, #4]
+; CHECK-BE-NEXT:    strh r2, [r0, #12]
+; CHECK-BE-NEXT:    lsls r1, r1, #24
+; CHECK-BE-NEXT:    bpl .LBB16_8
+; CHECK-BE-NEXT:  .LBB16_16: @ %cond.store13
+; CHECK-BE-NEXT:    vmovx.f16 s0, s7
+; CHECK-BE-NEXT:    vstr.16 s0, [sp]
+; CHECK-BE-NEXT:    ldrh.w r1, [sp]
+; CHECK-BE-NEXT:    strh r1, [r0, #14]
+; CHECK-BE-NEXT:    add sp, #40
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = icmp ugt <8 x i16> %b, zeroinitializer
+  call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %a, <8 x half>* %dest, i32 1, <8 x i1> %c)
+  ret void
+}
+
+define i8* @masked_v8f16_pre(i8* %y, i8* %x, <8 x i16> %a) {
+; CHECK-LE-LABEL: masked_v8f16_pre:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldr d1, [sp]
+; CHECK-LE-NEXT:    vldrw.u32 q1, [r1]
+; CHECK-LE-NEXT:    vmov d0, r2, r3
+; CHECK-LE-NEXT:    vpt.s16 gt, q0, zr
+; CHECK-LE-NEXT:    vstrht.16 q1, [r0, #4]!
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: masked_v8f16_pre:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vldr d1, [sp]
+; CHECK-BE-NEXT:    vldrh.u16 q1, [r1]
+; CHECK-BE-NEXT:    vmov d0, r3, r2
+; CHECK-BE-NEXT:    vrev64.16 q2, q0
+; CHECK-BE-NEXT:    vpt.s16 gt, q2, zr
+; CHECK-BE-NEXT:    vstrht.16 q1, [r0, #4]!
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 4
+  %0 = bitcast i8* %x to <8 x half>*
+  %1 = load <8 x half>, <8 x half>* %0, align 4
+  %2 = bitcast i8* %z to <8 x half>*
+  %c = icmp sgt <8 x i16> %a, zeroinitializer
+  call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %1, <8 x half>* %2, i32 2, <8 x i1> %c)
+  ret i8* %z
+}
+
+define i8* @masked_v8f16_post(i8* %y, i8* %x, <8 x i16> %a) {
+; CHECK-LE-LABEL: masked_v8f16_post:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vldr d1, [sp]
+; CHECK-LE-NEXT:    vldrw.u32 q1, [r1]
+; CHECK-LE-NEXT:    vmov d0, r2, r3
+; CHECK-LE-NEXT:    vpt.s16 gt, q0, zr
+; CHECK-LE-NEXT:    vstrht.16 q1, [r0], #4
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: masked_v8f16_post:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vldr d1, [sp]
+; CHECK-BE-NEXT:    vldrh.u16 q1, [r1]
+; CHECK-BE-NEXT:    vmov d0, r3, r2
+; CHECK-BE-NEXT:    vrev64.16 q2, q0
+; CHECK-BE-NEXT:    vpt.s16 gt, q2, zr
+; CHECK-BE-NEXT:    vstrht.16 q1, [r0], #4
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds i8, i8* %y, i32 4
+  %0 = bitcast i8* %x to <8 x half>*
+  %1 = load <8 x half>, <8 x half>* %0, align 4
+  %2 = bitcast i8* %y to <8 x half>*
+  %c = icmp sgt <8 x i16> %a, zeroinitializer
+  call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %1, <8 x half>* %2, i32 2, <8 x i1> %c)
+  ret i8* %z
+}
+
+
+define arm_aapcs_vfpcc void @masked_v2i64(<2 x i64> *%dest, <2 x i64> %a) {
+; CHECK-LE-LABEL: masked_v2i64:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    .pad #4
+; CHECK-LE-NEXT:    sub sp, #4
+; CHECK-LE-NEXT:    vmov r2, s0
+; CHECK-LE-NEXT:    movs r3, #0
+; CHECK-LE-NEXT:    vmov r1, s1
+; CHECK-LE-NEXT:    vmov r12, s3
+; CHECK-LE-NEXT:    rsbs r2, r2, #0
+; CHECK-LE-NEXT:    vmov r2, s2
+; CHECK-LE-NEXT:    sbcs.w r1, r3, r1
+; CHECK-LE-NEXT:    mov.w r1, #0
+; CHECK-LE-NEXT:    it lt
+; CHECK-LE-NEXT:    movlt r1, #1
+; CHECK-LE-NEXT:    rsbs r2, r2, #0
+; CHECK-LE-NEXT:    sbcs.w r2, r3, r12
+; CHECK-LE-NEXT:    it lt
+; CHECK-LE-NEXT:    movlt r3, #1
+; CHECK-LE-NEXT:    cmp r3, #0
+; CHECK-LE-NEXT:    it ne
+; CHECK-LE-NEXT:    mvnne r3, #1
+; CHECK-LE-NEXT:    bfi r3, r1, #0, #1
+; CHECK-LE-NEXT:    and r1, r3, #3
+; CHECK-LE-NEXT:    lsls r2, r3, #31
+; CHECK-LE-NEXT:    it ne
+; CHECK-LE-NEXT:    vstrne d0, [r0]
+; CHECK-LE-NEXT:    lsls r1, r1, #30
+; CHECK-LE-NEXT:    it mi
+; CHECK-LE-NEXT:    vstrmi d1, [r0, #8]
+; CHECK-LE-NEXT:    add sp, #4
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: masked_v2i64:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    .pad #4
+; CHECK-BE-NEXT:    sub sp, #4
+; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    movs r3, #0
+; CHECK-BE-NEXT:    vmov r2, s7
+; CHECK-BE-NEXT:    vmov r1, s6
+; CHECK-BE-NEXT:    vmov r12, s4
+; CHECK-BE-NEXT:    rsbs r2, r2, #0
+; CHECK-BE-NEXT:    vmov r2, s5
+; CHECK-BE-NEXT:    sbcs.w r1, r3, r1
+; CHECK-BE-NEXT:    mov.w r1, #0
+; CHECK-BE-NEXT:    it lt
+; CHECK-BE-NEXT:    movlt r1, #1
+; CHECK-BE-NEXT:    rsbs r2, r2, #0
+; CHECK-BE-NEXT:    sbcs.w r2, r3, r12
+; CHECK-BE-NEXT:    it lt
+; CHECK-BE-NEXT:    movlt r3, #1
+; CHECK-BE-NEXT:    cmp r3, #0
+; CHECK-BE-NEXT:    it ne
+; CHECK-BE-NEXT:    mvnne r3, #1
+; CHECK-BE-NEXT:    bfi r3, r1, #0, #1
+; CHECK-BE-NEXT:    and r1, r3, #3
+; CHECK-BE-NEXT:    lsls r2, r3, #31
+; CHECK-BE-NEXT:    it ne
+; CHECK-BE-NEXT:    vstrne d0, [r0]
+; CHECK-BE-NEXT:    lsls r1, r1, #30
+; CHECK-BE-NEXT:    it mi
+; CHECK-BE-NEXT:    vstrmi d1, [r0, #8]
+; CHECK-BE-NEXT:    add sp, #4
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = icmp sgt <2 x i64> %a, zeroinitializer
+  call void @llvm.masked.store.v2i64.p0v2i64(<2 x i64> %a, <2 x i64>* %dest, i32 8, <2 x i1> %c)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @masked_v2f64(<2 x double> *%dest, <2 x double> %a, <2 x i64> %b) {
+; CHECK-LE-LABEL: masked_v2f64:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    .pad #4
+; CHECK-LE-NEXT:    sub sp, #4
+; CHECK-LE-NEXT:    vmov r2, s4
+; CHECK-LE-NEXT:    movs r3, #0
+; CHECK-LE-NEXT:    vmov r1, s5
+; CHECK-LE-NEXT:    vmov r12, s7
+; CHECK-LE-NEXT:    rsbs r2, r2, #0
+; CHECK-LE-NEXT:    vmov r2, s6
+; CHECK-LE-NEXT:    sbcs.w r1, r3, r1
+; CHECK-LE-NEXT:    mov.w r1, #0
+; CHECK-LE-NEXT:    it lt
+; CHECK-LE-NEXT:    movlt r1, #1
+; CHECK-LE-NEXT:    rsbs r2, r2, #0
+; CHECK-LE-NEXT:    sbcs.w r2, r3, r12
+; CHECK-LE-NEXT:    it lt
+; CHECK-LE-NEXT:    movlt r3, #1
+; CHECK-LE-NEXT:    cmp r3, #0
+; CHECK-LE-NEXT:    it ne
+; CHECK-LE-NEXT:    mvnne r3, #1
+; CHECK-LE-NEXT:    bfi r3, r1, #0, #1
+; CHECK-LE-NEXT:    and r1, r3, #3
+; CHECK-LE-NEXT:    lsls r2, r3, #31
+; CHECK-LE-NEXT:    it ne
+; CHECK-LE-NEXT:    vstrne d0, [r0]
+; CHECK-LE-NEXT:    lsls r1, r1, #30
+; CHECK-LE-NEXT:    it mi
+; CHECK-LE-NEXT:    vstrmi d1, [r0, #8]
+; CHECK-LE-NEXT:    add sp, #4
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: masked_v2f64:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    .pad #4
+; CHECK-BE-NEXT:    sub sp, #4
+; CHECK-BE-NEXT:    vrev64.32 q2, q1
+; CHECK-BE-NEXT:    movs r3, #0
+; CHECK-BE-NEXT:    vmov r2, s11
+; CHECK-BE-NEXT:    vmov r1, s10
+; CHECK-BE-NEXT:    vmov r12, s8
+; CHECK-BE-NEXT:    rsbs r2, r2, #0
+; CHECK-BE-NEXT:    vmov r2, s9
+; CHECK-BE-NEXT:    sbcs.w r1, r3, r1
+; CHECK-BE-NEXT:    mov.w r1, #0
+; CHECK-BE-NEXT:    it lt
+; CHECK-BE-NEXT:    movlt r1, #1
+; CHECK-BE-NEXT:    rsbs r2, r2, #0
+; CHECK-BE-NEXT:    sbcs.w r2, r3, r12
+; CHECK-BE-NEXT:    it lt
+; CHECK-BE-NEXT:    movlt r3, #1
+; CHECK-BE-NEXT:    cmp r3, #0
+; CHECK-BE-NEXT:    it ne
+; CHECK-BE-NEXT:    mvnne r3, #1
+; CHECK-BE-NEXT:    bfi r3, r1, #0, #1
+; CHECK-BE-NEXT:    and r1, r3, #3
+; CHECK-BE-NEXT:    lsls r2, r3, #31
+; CHECK-BE-NEXT:    it ne
+; CHECK-BE-NEXT:    vstrne d0, [r0]
+; CHECK-BE-NEXT:    lsls r1, r1, #30
+; CHECK-BE-NEXT:    it mi
+; CHECK-BE-NEXT:    vstrmi d1, [r0, #8]
+; CHECK-BE-NEXT:    add sp, #4
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = icmp sgt <2 x i64> %b, zeroinitializer
+  call void @llvm.masked.store.v2f64.p0v2f64(<2 x double> %a, <2 x double>* %dest, i32 8, <2 x i1> %c)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @masked_v4i16(<4 x i16> *%dest, <4 x i32> %a) {
+; CHECK-LE-LABEL: masked_v4i16:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vpt.s32 gt, q0, zr
+; CHECK-LE-NEXT:    vstrht.32 q0, [r0]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: masked_v4i16:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    vpt.s32 gt, q1, zr
+; CHECK-BE-NEXT:    vstrht.32 q1, [r0]
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = icmp sgt <4 x i32> %a, zeroinitializer
+  %trunc = trunc <4 x i32> %a to <4 x i16>
+  call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %trunc, <4 x i16>* %dest, i32 2, <4 x i1> %c)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @masked_v4i8(<4 x i8> *%dest, <4 x i32> %a) {
+; CHECK-LE-LABEL: masked_v4i8:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vpt.s32 gt, q0, zr
+; CHECK-LE-NEXT:    vstrbt.32 q0, [r0]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: masked_v4i8:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    vpt.s32 gt, q1, zr
+; CHECK-BE-NEXT:    vstrbt.32 q1, [r0]
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = icmp sgt <4 x i32> %a, zeroinitializer
+  %trunc = trunc <4 x i32> %a to <4 x i8>
+  call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %trunc, <4 x i8>* %dest, i32 1, <4 x i1> %c)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @masked_v8i8(<8 x i8> *%dest, <8 x i16> %a) {
+; CHECK-LE-LABEL: masked_v8i8:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vpt.s16 gt, q0, zr
+; CHECK-LE-NEXT:    vstrbt.16 q0, [r0]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: masked_v8i8:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vrev64.16 q1, q0
+; CHECK-BE-NEXT:    vpt.s16 gt, q1, zr
+; CHECK-BE-NEXT:    vstrbt.16 q1, [r0]
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = icmp sgt <8 x i16> %a, zeroinitializer
+  %trunc = trunc <8 x i16> %a to <8 x i8>
+  call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %trunc, <8 x i8>* %dest, i32 1, <8 x i1> %c)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @masked_v4i16_align1(<4 x i16> *%dest, <4 x i32> %a) {
+; CHECK-LE-LABEL: masked_v4i16_align1:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    .pad #4
+; CHECK-LE-NEXT:    sub sp, #4
+; CHECK-LE-NEXT:    vcmp.s32 gt, q0, zr
+; CHECK-LE-NEXT:    vmrs r2, p0
+; CHECK-LE-NEXT:    and r1, r2, #1
+; CHECK-LE-NEXT:    rsbs r3, r1, #0
+; CHECK-LE-NEXT:    movs r1, #0
+; CHECK-LE-NEXT:    bfi r1, r3, #0, #1
+; CHECK-LE-NEXT:    ubfx r3, r2, #4, #1
+; CHECK-LE-NEXT:    rsbs r3, r3, #0
+; CHECK-LE-NEXT:    bfi r1, r3, #1, #1
+; CHECK-LE-NEXT:    ubfx r3, r2, #8, #1
+; CHECK-LE-NEXT:    ubfx r2, r2, #12, #1
+; CHECK-LE-NEXT:    rsbs r3, r3, #0
+; CHECK-LE-NEXT:    bfi r1, r3, #2, #1
+; CHECK-LE-NEXT:    rsbs r2, r2, #0
+; CHECK-LE-NEXT:    bfi r1, r2, #3, #1
+; CHECK-LE-NEXT:    lsls r2, r1, #31
+; CHECK-LE-NEXT:    itt ne
+; CHECK-LE-NEXT:    vmovne r2, s0
+; CHECK-LE-NEXT:    strhne r2, [r0]
+; CHECK-LE-NEXT:    lsls r2, r1, #30
+; CHECK-LE-NEXT:    itt mi
+; CHECK-LE-NEXT:    vmovmi r2, s1
+; CHECK-LE-NEXT:    strhmi r2, [r0, #2]
+; CHECK-LE-NEXT:    lsls r2, r1, #29
+; CHECK-LE-NEXT:    itt mi
+; CHECK-LE-NEXT:    vmovmi r2, s2
+; CHECK-LE-NEXT:    strhmi r2, [r0, #4]
+; CHECK-LE-NEXT:    lsls r1, r1, #28
+; CHECK-LE-NEXT:    itt mi
+; CHECK-LE-NEXT:    vmovmi r1, s3
+; CHECK-LE-NEXT:    strhmi r1, [r0, #6]
+; CHECK-LE-NEXT:    add sp, #4
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: masked_v4i16_align1:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    .pad #4
+; CHECK-BE-NEXT:    sub sp, #4
+; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    vcmp.s32 gt, q1, zr
+; CHECK-BE-NEXT:    vmrs r2, p0
+; CHECK-BE-NEXT:    and r1, r2, #1
+; CHECK-BE-NEXT:    rsbs r3, r1, #0
+; CHECK-BE-NEXT:    movs r1, #0
+; CHECK-BE-NEXT:    bfi r1, r3, #0, #1
+; CHECK-BE-NEXT:    ubfx r3, r2, #4, #1
+; CHECK-BE-NEXT:    rsbs r3, r3, #0
+; CHECK-BE-NEXT:    bfi r1, r3, #1, #1
+; CHECK-BE-NEXT:    ubfx r3, r2, #8, #1
+; CHECK-BE-NEXT:    ubfx r2, r2, #12, #1
+; CHECK-BE-NEXT:    rsbs r3, r3, #0
+; CHECK-BE-NEXT:    bfi r1, r3, #2, #1
+; CHECK-BE-NEXT:    rsbs r2, r2, #0
+; CHECK-BE-NEXT:    bfi r1, r2, #3, #1
+; CHECK-BE-NEXT:    lsls r2, r1, #31
+; CHECK-BE-NEXT:    itt ne
+; CHECK-BE-NEXT:    vmovne r2, s4
+; CHECK-BE-NEXT:    strhne r2, [r0]
+; CHECK-BE-NEXT:    lsls r2, r1, #30
+; CHECK-BE-NEXT:    itt mi
+; CHECK-BE-NEXT:    vmovmi r2, s5
+; CHECK-BE-NEXT:    strhmi r2, [r0, #2]
+; CHECK-BE-NEXT:    lsls r2, r1, #29
+; CHECK-BE-NEXT:    itt mi
+; CHECK-BE-NEXT:    vmovmi r2, s6
+; CHECK-BE-NEXT:    strhmi r2, [r0, #4]
+; CHECK-BE-NEXT:    lsls r1, r1, #28
+; CHECK-BE-NEXT:    itt mi
+; CHECK-BE-NEXT:    vmovmi r1, s7
+; CHECK-BE-NEXT:    strhmi r1, [r0, #6]
+; CHECK-BE-NEXT:    add sp, #4
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = icmp sgt <4 x i32> %a, zeroinitializer
+  %trunc = trunc <4 x i32> %a to <4 x i16>
+  call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %trunc, <4 x i16>* %dest, i32 1, <4 x i1> %c)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @masked_v4f16_align4(<4 x half> *%dest, <4 x float> %a) {
+; CHECK-LE-LABEL: masked_v4f16_align4:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    .pad #4
+; CHECK-LE-NEXT:    sub sp, #4
+; CHECK-LE-NEXT:    vcvtb.f16.f32 s4, s0
+; CHECK-LE-NEXT:    vcmp.f32 s0, #0
+; CHECK-LE-NEXT:    vmov r1, s4
+; CHECK-LE-NEXT:    vcvtb.f16.f32 s4, s1
+; CHECK-LE-NEXT:    vmov r2, s4
+; CHECK-LE-NEXT:    vmov.16 q1[0], r1
+; CHECK-LE-NEXT:    vcvtb.f16.f32 s8, s2
+; CHECK-LE-NEXT:    vmov.16 q1[1], r2
+; CHECK-LE-NEXT:    vmov r1, s8
+; CHECK-LE-NEXT:    vcvtb.f16.f32 s8, s3
+; CHECK-LE-NEXT:    vmov.16 q1[2], r1
+; CHECK-LE-NEXT:    vmov r1, s8
+; CHECK-LE-NEXT:    vmov.16 q1[3], r1
+; CHECK-LE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-LE-NEXT:    mov.w r1, #0
+; CHECK-LE-NEXT:    vcmp.f32 s1, #0
+; CHECK-LE-NEXT:    it gt
+; CHECK-LE-NEXT:    movgt r1, #1
+; CHECK-LE-NEXT:    cmp r1, #0
+; CHECK-LE-NEXT:    cset r1, ne
+; CHECK-LE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-LE-NEXT:    and r1, r1, #1
+; CHECK-LE-NEXT:    vcmp.f32 s2, #0
+; CHECK-LE-NEXT:    rsb.w r3, r1, #0
+; CHECK-LE-NEXT:    mov.w r1, #0
+; CHECK-LE-NEXT:    bfi r1, r3, #0, #1
+; CHECK-LE-NEXT:    mov.w r3, #0
+; CHECK-LE-NEXT:    it gt
+; CHECK-LE-NEXT:    movgt r3, #1
+; CHECK-LE-NEXT:    cmp r3, #0
+; CHECK-LE-NEXT:    cset r3, ne
+; CHECK-LE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-LE-NEXT:    and r3, r3, #1
+; CHECK-LE-NEXT:    vcmp.f32 s3, #0
+; CHECK-LE-NEXT:    rsb.w r3, r3, #0
+; CHECK-LE-NEXT:    mov.w r2, #0
+; CHECK-LE-NEXT:    bfi r1, r3, #1, #1
+; CHECK-LE-NEXT:    mov.w r3, #0
+; CHECK-LE-NEXT:    it gt
+; CHECK-LE-NEXT:    movgt r3, #1
+; CHECK-LE-NEXT:    cmp r3, #0
+; CHECK-LE-NEXT:    cset r3, ne
+; CHECK-LE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-LE-NEXT:    it gt
+; CHECK-LE-NEXT:    movgt r2, #1
+; CHECK-LE-NEXT:    cmp r2, #0
+; CHECK-LE-NEXT:    and r3, r3, #1
+; CHECK-LE-NEXT:    cset r2, ne
+; CHECK-LE-NEXT:    and r2, r2, #1
+; CHECK-LE-NEXT:    rsbs r3, r3, #0
+; CHECK-LE-NEXT:    bfi r1, r3, #2, #1
+; CHECK-LE-NEXT:    rsbs r2, r2, #0
+; CHECK-LE-NEXT:    bfi r1, r2, #3, #1
+; CHECK-LE-NEXT:    lsls r2, r1, #31
+; CHECK-LE-NEXT:    bne .LBB25_5
+; CHECK-LE-NEXT:  @ %bb.1: @ %else
+; CHECK-LE-NEXT:    lsls r2, r1, #30
+; CHECK-LE-NEXT:    bmi .LBB25_6
+; CHECK-LE-NEXT:  .LBB25_2: @ %else2
+; CHECK-LE-NEXT:    lsls r2, r1, #29
+; CHECK-LE-NEXT:    bmi .LBB25_7
+; CHECK-LE-NEXT:  .LBB25_3: @ %else4
+; CHECK-LE-NEXT:    lsls r1, r1, #28
+; CHECK-LE-NEXT:    bmi .LBB25_8
+; CHECK-LE-NEXT:  .LBB25_4: @ %else6
+; CHECK-LE-NEXT:    add sp, #4
+; CHECK-LE-NEXT:    bx lr
+; CHECK-LE-NEXT:  .LBB25_5: @ %cond.store
+; CHECK-LE-NEXT:    vstr.16 s4, [r0]
+; CHECK-LE-NEXT:    lsls r2, r1, #30
+; CHECK-LE-NEXT:    bpl .LBB25_2
+; CHECK-LE-NEXT:  .LBB25_6: @ %cond.store1
+; CHECK-LE-NEXT:    vmovx.f16 s0, s4
+; CHECK-LE-NEXT:    vstr.16 s0, [r0, #2]
+; CHECK-LE-NEXT:    lsls r2, r1, #29
+; CHECK-LE-NEXT:    bpl .LBB25_3
+; CHECK-LE-NEXT:  .LBB25_7: @ %cond.store3
+; CHECK-LE-NEXT:    vstr.16 s5, [r0, #4]
+; CHECK-LE-NEXT:    lsls r1, r1, #28
+; CHECK-LE-NEXT:    bpl .LBB25_4
+; CHECK-LE-NEXT:  .LBB25_8: @ %cond.store5
+; CHECK-LE-NEXT:    vmovx.f16 s0, s5
+; CHECK-LE-NEXT:    vstr.16 s0, [r0, #6]
+; CHECK-LE-NEXT:    add sp, #4
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: masked_v4f16_align4:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    .pad #4
+; CHECK-BE-NEXT:    sub sp, #4
+; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    vcvtb.f16.f32 s0, s4
+; CHECK-BE-NEXT:    vcmp.f32 s4, #0
+; CHECK-BE-NEXT:    vmov r1, s0
+; CHECK-BE-NEXT:    vcvtb.f16.f32 s0, s5
+; CHECK-BE-NEXT:    vmov r2, s0
+; CHECK-BE-NEXT:    vmov.16 q0[0], r1
+; CHECK-BE-NEXT:    vcvtb.f16.f32 s8, s6
+; CHECK-BE-NEXT:    vmov.16 q0[1], r2
+; CHECK-BE-NEXT:    vmov r1, s8
+; CHECK-BE-NEXT:    vcvtb.f16.f32 s8, s7
+; CHECK-BE-NEXT:    vmov.16 q0[2], r1
+; CHECK-BE-NEXT:    vmov r1, s8
+; CHECK-BE-NEXT:    vmov.16 q0[3], r1
+; CHECK-BE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-BE-NEXT:    mov.w r1, #0
+; CHECK-BE-NEXT:    vcmp.f32 s5, #0
+; CHECK-BE-NEXT:    it gt
+; CHECK-BE-NEXT:    movgt r1, #1
+; CHECK-BE-NEXT:    cmp r1, #0
+; CHECK-BE-NEXT:    cset r1, ne
+; CHECK-BE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-BE-NEXT:    and r1, r1, #1
+; CHECK-BE-NEXT:    vcmp.f32 s6, #0
+; CHECK-BE-NEXT:    rsb.w r3, r1, #0
+; CHECK-BE-NEXT:    mov.w r1, #0
+; CHECK-BE-NEXT:    bfi r1, r3, #0, #1
+; CHECK-BE-NEXT:    mov.w r3, #0
+; CHECK-BE-NEXT:    it gt
+; CHECK-BE-NEXT:    movgt r3, #1
+; CHECK-BE-NEXT:    cmp r3, #0
+; CHECK-BE-NEXT:    cset r3, ne
+; CHECK-BE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-BE-NEXT:    and r3, r3, #1
+; CHECK-BE-NEXT:    vcmp.f32 s7, #0
+; CHECK-BE-NEXT:    rsb.w r3, r3, #0
+; CHECK-BE-NEXT:    mov.w r2, #0
+; CHECK-BE-NEXT:    bfi r1, r3, #1, #1
+; CHECK-BE-NEXT:    mov.w r3, #0
+; CHECK-BE-NEXT:    it gt
+; CHECK-BE-NEXT:    movgt r3, #1
+; CHECK-BE-NEXT:    cmp r3, #0
+; CHECK-BE-NEXT:    cset r3, ne
+; CHECK-BE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-BE-NEXT:    it gt
+; CHECK-BE-NEXT:    movgt r2, #1
+; CHECK-BE-NEXT:    cmp r2, #0
+; CHECK-BE-NEXT:    and r3, r3, #1
+; CHECK-BE-NEXT:    cset r2, ne
+; CHECK-BE-NEXT:    and r2, r2, #1
+; CHECK-BE-NEXT:    rsbs r3, r3, #0
+; CHECK-BE-NEXT:    bfi r1, r3, #2, #1
+; CHECK-BE-NEXT:    rsbs r2, r2, #0
+; CHECK-BE-NEXT:    bfi r1, r2, #3, #1
+; CHECK-BE-NEXT:    lsls r2, r1, #31
+; CHECK-BE-NEXT:    bne .LBB25_5
+; CHECK-BE-NEXT:  @ %bb.1: @ %else
+; CHECK-BE-NEXT:    lsls r2, r1, #30
+; CHECK-BE-NEXT:    bmi .LBB25_6
+; CHECK-BE-NEXT:  .LBB25_2: @ %else2
+; CHECK-BE-NEXT:    lsls r2, r1, #29
+; CHECK-BE-NEXT:    bmi .LBB25_7
+; CHECK-BE-NEXT:  .LBB25_3: @ %else4
+; CHECK-BE-NEXT:    lsls r1, r1, #28
+; CHECK-BE-NEXT:    bmi .LBB25_8
+; CHECK-BE-NEXT:  .LBB25_4: @ %else6
+; CHECK-BE-NEXT:    add sp, #4
+; CHECK-BE-NEXT:    bx lr
+; CHECK-BE-NEXT:  .LBB25_5: @ %cond.store
+; CHECK-BE-NEXT:    vstr.16 s0, [r0]
+; CHECK-BE-NEXT:    lsls r2, r1, #30
+; CHECK-BE-NEXT:    bpl .LBB25_2
+; CHECK-BE-NEXT:  .LBB25_6: @ %cond.store1
+; CHECK-BE-NEXT:    vmovx.f16 s4, s0
+; CHECK-BE-NEXT:    vstr.16 s4, [r0, #2]
+; CHECK-BE-NEXT:    lsls r2, r1, #29
+; CHECK-BE-NEXT:    bpl .LBB25_3
+; CHECK-BE-NEXT:  .LBB25_7: @ %cond.store3
+; CHECK-BE-NEXT:    vstr.16 s1, [r0, #4]
+; CHECK-BE-NEXT:    lsls r1, r1, #28
+; CHECK-BE-NEXT:    bpl .LBB25_4
+; CHECK-BE-NEXT:  .LBB25_8: @ %cond.store5
+; CHECK-BE-NEXT:    vmovx.f16 s0, s1
+; CHECK-BE-NEXT:    vstr.16 s0, [r0, #6]
+; CHECK-BE-NEXT:    add sp, #4
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = fcmp ogt <4 x float> %a, zeroinitializer
+  %trunc = fptrunc <4 x float> %a to <4 x half>
+  call void @llvm.masked.store.v4f16.p0v4f16(<4 x half> %trunc, <4 x half>* %dest, i32 4, <4 x i1> %c)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @masked_v4f16_align2(<4 x half> *%dest, <4 x float> %a) {
+; CHECK-LE-LABEL: masked_v4f16_align2:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    .pad #4
+; CHECK-LE-NEXT:    sub sp, #4
+; CHECK-LE-NEXT:    vcvtb.f16.f32 s4, s0
+; CHECK-LE-NEXT:    vcmp.f32 s0, #0
+; CHECK-LE-NEXT:    vmov r1, s4
+; CHECK-LE-NEXT:    vcvtb.f16.f32 s4, s1
+; CHECK-LE-NEXT:    vmov r2, s4
+; CHECK-LE-NEXT:    vmov.16 q1[0], r1
+; CHECK-LE-NEXT:    vcvtb.f16.f32 s8, s2
+; CHECK-LE-NEXT:    vmov.16 q1[1], r2
+; CHECK-LE-NEXT:    vmov r1, s8
+; CHECK-LE-NEXT:    vcvtb.f16.f32 s8, s3
+; CHECK-LE-NEXT:    vmov.16 q1[2], r1
+; CHECK-LE-NEXT:    vmov r1, s8
+; CHECK-LE-NEXT:    vmov.16 q1[3], r1
+; CHECK-LE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-LE-NEXT:    mov.w r1, #0
+; CHECK-LE-NEXT:    vcmp.f32 s1, #0
+; CHECK-LE-NEXT:    it gt
+; CHECK-LE-NEXT:    movgt r1, #1
+; CHECK-LE-NEXT:    cmp r1, #0
+; CHECK-LE-NEXT:    cset r1, ne
+; CHECK-LE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-LE-NEXT:    and r1, r1, #1
+; CHECK-LE-NEXT:    vcmp.f32 s2, #0
+; CHECK-LE-NEXT:    rsb.w r3, r1, #0
+; CHECK-LE-NEXT:    mov.w r1, #0
+; CHECK-LE-NEXT:    bfi r1, r3, #0, #1
+; CHECK-LE-NEXT:    mov.w r3, #0
+; CHECK-LE-NEXT:    it gt
+; CHECK-LE-NEXT:    movgt r3, #1
+; CHECK-LE-NEXT:    cmp r3, #0
+; CHECK-LE-NEXT:    cset r3, ne
+; CHECK-LE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-LE-NEXT:    and r3, r3, #1
+; CHECK-LE-NEXT:    vcmp.f32 s3, #0
+; CHECK-LE-NEXT:    rsb.w r3, r3, #0
+; CHECK-LE-NEXT:    mov.w r2, #0
+; CHECK-LE-NEXT:    bfi r1, r3, #1, #1
+; CHECK-LE-NEXT:    mov.w r3, #0
+; CHECK-LE-NEXT:    it gt
+; CHECK-LE-NEXT:    movgt r3, #1
+; CHECK-LE-NEXT:    cmp r3, #0
+; CHECK-LE-NEXT:    cset r3, ne
+; CHECK-LE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-LE-NEXT:    it gt
+; CHECK-LE-NEXT:    movgt r2, #1
+; CHECK-LE-NEXT:    cmp r2, #0
+; CHECK-LE-NEXT:    and r3, r3, #1
+; CHECK-LE-NEXT:    cset r2, ne
+; CHECK-LE-NEXT:    and r2, r2, #1
+; CHECK-LE-NEXT:    rsbs r3, r3, #0
+; CHECK-LE-NEXT:    bfi r1, r3, #2, #1
+; CHECK-LE-NEXT:    rsbs r2, r2, #0
+; CHECK-LE-NEXT:    bfi r1, r2, #3, #1
+; CHECK-LE-NEXT:    lsls r2, r1, #31
+; CHECK-LE-NEXT:    bne .LBB26_5
+; CHECK-LE-NEXT:  @ %bb.1: @ %else
+; CHECK-LE-NEXT:    lsls r2, r1, #30
+; CHECK-LE-NEXT:    bmi .LBB26_6
+; CHECK-LE-NEXT:  .LBB26_2: @ %else2
+; CHECK-LE-NEXT:    lsls r2, r1, #29
+; CHECK-LE-NEXT:    bmi .LBB26_7
+; CHECK-LE-NEXT:  .LBB26_3: @ %else4
+; CHECK-LE-NEXT:    lsls r1, r1, #28
+; CHECK-LE-NEXT:    bmi .LBB26_8
+; CHECK-LE-NEXT:  .LBB26_4: @ %else6
+; CHECK-LE-NEXT:    add sp, #4
+; CHECK-LE-NEXT:    bx lr
+; CHECK-LE-NEXT:  .LBB26_5: @ %cond.store
+; CHECK-LE-NEXT:    vstr.16 s4, [r0]
+; CHECK-LE-NEXT:    lsls r2, r1, #30
+; CHECK-LE-NEXT:    bpl .LBB26_2
+; CHECK-LE-NEXT:  .LBB26_6: @ %cond.store1
+; CHECK-LE-NEXT:    vmovx.f16 s0, s4
+; CHECK-LE-NEXT:    vstr.16 s0, [r0, #2]
+; CHECK-LE-NEXT:    lsls r2, r1, #29
+; CHECK-LE-NEXT:    bpl .LBB26_3
+; CHECK-LE-NEXT:  .LBB26_7: @ %cond.store3
+; CHECK-LE-NEXT:    vstr.16 s5, [r0, #4]
+; CHECK-LE-NEXT:    lsls r1, r1, #28
+; CHECK-LE-NEXT:    bpl .LBB26_4
+; CHECK-LE-NEXT:  .LBB26_8: @ %cond.store5
+; CHECK-LE-NEXT:    vmovx.f16 s0, s5
+; CHECK-LE-NEXT:    vstr.16 s0, [r0, #6]
+; CHECK-LE-NEXT:    add sp, #4
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: masked_v4f16_align2:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    .pad #4
+; CHECK-BE-NEXT:    sub sp, #4
+; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    vcvtb.f16.f32 s0, s4
+; CHECK-BE-NEXT:    vcmp.f32 s4, #0
+; CHECK-BE-NEXT:    vmov r1, s0
+; CHECK-BE-NEXT:    vcvtb.f16.f32 s0, s5
+; CHECK-BE-NEXT:    vmov r2, s0
+; CHECK-BE-NEXT:    vmov.16 q0[0], r1
+; CHECK-BE-NEXT:    vcvtb.f16.f32 s8, s6
+; CHECK-BE-NEXT:    vmov.16 q0[1], r2
+; CHECK-BE-NEXT:    vmov r1, s8
+; CHECK-BE-NEXT:    vcvtb.f16.f32 s8, s7
+; CHECK-BE-NEXT:    vmov.16 q0[2], r1
+; CHECK-BE-NEXT:    vmov r1, s8
+; CHECK-BE-NEXT:    vmov.16 q0[3], r1
+; CHECK-BE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-BE-NEXT:    mov.w r1, #0
+; CHECK-BE-NEXT:    vcmp.f32 s5, #0
+; CHECK-BE-NEXT:    it gt
+; CHECK-BE-NEXT:    movgt r1, #1
+; CHECK-BE-NEXT:    cmp r1, #0
+; CHECK-BE-NEXT:    cset r1, ne
+; CHECK-BE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-BE-NEXT:    and r1, r1, #1
+; CHECK-BE-NEXT:    vcmp.f32 s6, #0
+; CHECK-BE-NEXT:    rsb.w r3, r1, #0
+; CHECK-BE-NEXT:    mov.w r1, #0
+; CHECK-BE-NEXT:    bfi r1, r3, #0, #1
+; CHECK-BE-NEXT:    mov.w r3, #0
+; CHECK-BE-NEXT:    it gt
+; CHECK-BE-NEXT:    movgt r3, #1
+; CHECK-BE-NEXT:    cmp r3, #0
+; CHECK-BE-NEXT:    cset r3, ne
+; CHECK-BE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-BE-NEXT:    and r3, r3, #1
+; CHECK-BE-NEXT:    vcmp.f32 s7, #0
+; CHECK-BE-NEXT:    rsb.w r3, r3, #0
+; CHECK-BE-NEXT:    mov.w r2, #0
+; CHECK-BE-NEXT:    bfi r1, r3, #1, #1
+; CHECK-BE-NEXT:    mov.w r3, #0
+; CHECK-BE-NEXT:    it gt
+; CHECK-BE-NEXT:    movgt r3, #1
+; CHECK-BE-NEXT:    cmp r3, #0
+; CHECK-BE-NEXT:    cset r3, ne
+; CHECK-BE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-BE-NEXT:    it gt
+; CHECK-BE-NEXT:    movgt r2, #1
+; CHECK-BE-NEXT:    cmp r2, #0
+; CHECK-BE-NEXT:    and r3, r3, #1
+; CHECK-BE-NEXT:    cset r2, ne
+; CHECK-BE-NEXT:    and r2, r2, #1
+; CHECK-BE-NEXT:    rsbs r3, r3, #0
+; CHECK-BE-NEXT:    bfi r1, r3, #2, #1
+; CHECK-BE-NEXT:    rsbs r2, r2, #0
+; CHECK-BE-NEXT:    bfi r1, r2, #3, #1
+; CHECK-BE-NEXT:    lsls r2, r1, #31
+; CHECK-BE-NEXT:    bne .LBB26_5
+; CHECK-BE-NEXT:  @ %bb.1: @ %else
+; CHECK-BE-NEXT:    lsls r2, r1, #30
+; CHECK-BE-NEXT:    bmi .LBB26_6
+; CHECK-BE-NEXT:  .LBB26_2: @ %else2
+; CHECK-BE-NEXT:    lsls r2, r1, #29
+; CHECK-BE-NEXT:    bmi .LBB26_7
+; CHECK-BE-NEXT:  .LBB26_3: @ %else4
+; CHECK-BE-NEXT:    lsls r1, r1, #28
+; CHECK-BE-NEXT:    bmi .LBB26_8
+; CHECK-BE-NEXT:  .LBB26_4: @ %else6
+; CHECK-BE-NEXT:    add sp, #4
+; CHECK-BE-NEXT:    bx lr
+; CHECK-BE-NEXT:  .LBB26_5: @ %cond.store
+; CHECK-BE-NEXT:    vstr.16 s0, [r0]
+; CHECK-BE-NEXT:    lsls r2, r1, #30
+; CHECK-BE-NEXT:    bpl .LBB26_2
+; CHECK-BE-NEXT:  .LBB26_6: @ %cond.store1
+; CHECK-BE-NEXT:    vmovx.f16 s4, s0
+; CHECK-BE-NEXT:    vstr.16 s4, [r0, #2]
+; CHECK-BE-NEXT:    lsls r2, r1, #29
+; CHECK-BE-NEXT:    bpl .LBB26_3
+; CHECK-BE-NEXT:  .LBB26_7: @ %cond.store3
+; CHECK-BE-NEXT:    vstr.16 s1, [r0, #4]
+; CHECK-BE-NEXT:    lsls r1, r1, #28
+; CHECK-BE-NEXT:    bpl .LBB26_4
+; CHECK-BE-NEXT:  .LBB26_8: @ %cond.store5
+; CHECK-BE-NEXT:    vmovx.f16 s0, s1
+; CHECK-BE-NEXT:    vstr.16 s0, [r0, #6]
+; CHECK-BE-NEXT:    add sp, #4
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = fcmp ogt <4 x float> %a, zeroinitializer
+  %trunc = fptrunc <4 x float> %a to <4 x half>
+  call void @llvm.masked.store.v4f16.p0v4f16(<4 x half> %trunc, <4 x half>* %dest, i32 2, <4 x i1> %c)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @masked_v4f16_align1(<4 x half> *%dest, <4 x float> %a) {
+; CHECK-LE-LABEL: masked_v4f16_align1:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    .pad #20
+; CHECK-LE-NEXT:    sub sp, #20
+; CHECK-LE-NEXT:    vcvtb.f16.f32 s4, s0
+; CHECK-LE-NEXT:    vcmp.f32 s0, #0
+; CHECK-LE-NEXT:    vmov r1, s4
+; CHECK-LE-NEXT:    vcvtb.f16.f32 s4, s1
+; CHECK-LE-NEXT:    vmov r2, s4
+; CHECK-LE-NEXT:    vmov.16 q1[0], r1
+; CHECK-LE-NEXT:    vcvtb.f16.f32 s8, s2
+; CHECK-LE-NEXT:    vmov.16 q1[1], r2
+; CHECK-LE-NEXT:    vmov r1, s8
+; CHECK-LE-NEXT:    vcvtb.f16.f32 s8, s3
+; CHECK-LE-NEXT:    vmov.16 q1[2], r1
+; CHECK-LE-NEXT:    vmov r1, s8
+; CHECK-LE-NEXT:    vmov.16 q1[3], r1
+; CHECK-LE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-LE-NEXT:    mov.w r1, #0
+; CHECK-LE-NEXT:    vcmp.f32 s1, #0
+; CHECK-LE-NEXT:    it gt
+; CHECK-LE-NEXT:    movgt r1, #1
+; CHECK-LE-NEXT:    cmp r1, #0
+; CHECK-LE-NEXT:    cset r1, ne
+; CHECK-LE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-LE-NEXT:    and r1, r1, #1
+; CHECK-LE-NEXT:    vcmp.f32 s2, #0
+; CHECK-LE-NEXT:    rsb.w r3, r1, #0
+; CHECK-LE-NEXT:    mov.w r1, #0
+; CHECK-LE-NEXT:    bfi r1, r3, #0, #1
+; CHECK-LE-NEXT:    mov.w r3, #0
+; CHECK-LE-NEXT:    it gt
+; CHECK-LE-NEXT:    movgt r3, #1
+; CHECK-LE-NEXT:    cmp r3, #0
+; CHECK-LE-NEXT:    cset r3, ne
+; CHECK-LE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-LE-NEXT:    and r3, r3, #1
+; CHECK-LE-NEXT:    vcmp.f32 s3, #0
+; CHECK-LE-NEXT:    rsb.w r3, r3, #0
+; CHECK-LE-NEXT:    mov.w r2, #0
+; CHECK-LE-NEXT:    bfi r1, r3, #1, #1
+; CHECK-LE-NEXT:    mov.w r3, #0
+; CHECK-LE-NEXT:    it gt
+; CHECK-LE-NEXT:    movgt r3, #1
+; CHECK-LE-NEXT:    cmp r3, #0
+; CHECK-LE-NEXT:    cset r3, ne
+; CHECK-LE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-LE-NEXT:    it gt
+; CHECK-LE-NEXT:    movgt r2, #1
+; CHECK-LE-NEXT:    cmp r2, #0
+; CHECK-LE-NEXT:    and r3, r3, #1
+; CHECK-LE-NEXT:    cset r2, ne
+; CHECK-LE-NEXT:    and r2, r2, #1
+; CHECK-LE-NEXT:    rsbs r3, r3, #0
+; CHECK-LE-NEXT:    bfi r1, r3, #2, #1
+; CHECK-LE-NEXT:    rsbs r2, r2, #0
+; CHECK-LE-NEXT:    bfi r1, r2, #3, #1
+; CHECK-LE-NEXT:    lsls r2, r1, #31
+; CHECK-LE-NEXT:    bne .LBB27_5
+; CHECK-LE-NEXT:  @ %bb.1: @ %else
+; CHECK-LE-NEXT:    lsls r2, r1, #30
+; CHECK-LE-NEXT:    bmi .LBB27_6
+; CHECK-LE-NEXT:  .LBB27_2: @ %else2
+; CHECK-LE-NEXT:    lsls r2, r1, #29
+; CHECK-LE-NEXT:    bmi .LBB27_7
+; CHECK-LE-NEXT:  .LBB27_3: @ %else4
+; CHECK-LE-NEXT:    lsls r1, r1, #28
+; CHECK-LE-NEXT:    bmi .LBB27_8
+; CHECK-LE-NEXT:  .LBB27_4: @ %else6
+; CHECK-LE-NEXT:    add sp, #20
+; CHECK-LE-NEXT:    bx lr
+; CHECK-LE-NEXT:  .LBB27_5: @ %cond.store
+; CHECK-LE-NEXT:    vstr.16 s4, [sp, #12]
+; CHECK-LE-NEXT:    ldrh.w r2, [sp, #12]
+; CHECK-LE-NEXT:    strh r2, [r0]
+; CHECK-LE-NEXT:    lsls r2, r1, #30
+; CHECK-LE-NEXT:    bpl .LBB27_2
+; CHECK-LE-NEXT:  .LBB27_6: @ %cond.store1
+; CHECK-LE-NEXT:    vmovx.f16 s0, s4
+; CHECK-LE-NEXT:    vstr.16 s0, [sp, #8]
+; CHECK-LE-NEXT:    ldrh.w r2, [sp, #8]
+; CHECK-LE-NEXT:    strh r2, [r0, #2]
+; CHECK-LE-NEXT:    lsls r2, r1, #29
+; CHECK-LE-NEXT:    bpl .LBB27_3
+; CHECK-LE-NEXT:  .LBB27_7: @ %cond.store3
+; CHECK-LE-NEXT:    vstr.16 s5, [sp, #4]
+; CHECK-LE-NEXT:    ldrh.w r2, [sp, #4]
+; CHECK-LE-NEXT:    strh r2, [r0, #4]
+; CHECK-LE-NEXT:    lsls r1, r1, #28
+; CHECK-LE-NEXT:    bpl .LBB27_4
+; CHECK-LE-NEXT:  .LBB27_8: @ %cond.store5
+; CHECK-LE-NEXT:    vmovx.f16 s0, s5
+; CHECK-LE-NEXT:    vstr.16 s0, [sp]
+; CHECK-LE-NEXT:    ldrh.w r1, [sp]
+; CHECK-LE-NEXT:    strh r1, [r0, #6]
+; CHECK-LE-NEXT:    add sp, #20
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: masked_v4f16_align1:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    .pad #20
+; CHECK-BE-NEXT:    sub sp, #20
+; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    vcvtb.f16.f32 s0, s4
+; CHECK-BE-NEXT:    vcmp.f32 s4, #0
+; CHECK-BE-NEXT:    vmov r1, s0
+; CHECK-BE-NEXT:    vcvtb.f16.f32 s0, s5
+; CHECK-BE-NEXT:    vmov r2, s0
+; CHECK-BE-NEXT:    vmov.16 q0[0], r1
+; CHECK-BE-NEXT:    vcvtb.f16.f32 s8, s6
+; CHECK-BE-NEXT:    vmov.16 q0[1], r2
+; CHECK-BE-NEXT:    vmov r1, s8
+; CHECK-BE-NEXT:    vcvtb.f16.f32 s8, s7
+; CHECK-BE-NEXT:    vmov.16 q0[2], r1
+; CHECK-BE-NEXT:    vmov r1, s8
+; CHECK-BE-NEXT:    vmov.16 q0[3], r1
+; CHECK-BE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-BE-NEXT:    mov.w r1, #0
+; CHECK-BE-NEXT:    vcmp.f32 s5, #0
+; CHECK-BE-NEXT:    it gt
+; CHECK-BE-NEXT:    movgt r1, #1
+; CHECK-BE-NEXT:    cmp r1, #0
+; CHECK-BE-NEXT:    cset r1, ne
+; CHECK-BE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-BE-NEXT:    and r1, r1, #1
+; CHECK-BE-NEXT:    vcmp.f32 s6, #0
+; CHECK-BE-NEXT:    rsb.w r3, r1, #0
+; CHECK-BE-NEXT:    mov.w r1, #0
+; CHECK-BE-NEXT:    bfi r1, r3, #0, #1
+; CHECK-BE-NEXT:    mov.w r3, #0
+; CHECK-BE-NEXT:    it gt
+; CHECK-BE-NEXT:    movgt r3, #1
+; CHECK-BE-NEXT:    cmp r3, #0
+; CHECK-BE-NEXT:    cset r3, ne
+; CHECK-BE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-BE-NEXT:    and r3, r3, #1
+; CHECK-BE-NEXT:    vcmp.f32 s7, #0
+; CHECK-BE-NEXT:    rsb.w r3, r3, #0
+; CHECK-BE-NEXT:    mov.w r2, #0
+; CHECK-BE-NEXT:    bfi r1, r3, #1, #1
+; CHECK-BE-NEXT:    mov.w r3, #0
+; CHECK-BE-NEXT:    it gt
+; CHECK-BE-NEXT:    movgt r3, #1
+; CHECK-BE-NEXT:    cmp r3, #0
+; CHECK-BE-NEXT:    cset r3, ne
+; CHECK-BE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-BE-NEXT:    it gt
+; CHECK-BE-NEXT:    movgt r2, #1
+; CHECK-BE-NEXT:    cmp r2, #0
+; CHECK-BE-NEXT:    and r3, r3, #1
+; CHECK-BE-NEXT:    cset r2, ne
+; CHECK-BE-NEXT:    and r2, r2, #1
+; CHECK-BE-NEXT:    rsbs r3, r3, #0
+; CHECK-BE-NEXT:    bfi r1, r3, #2, #1
+; CHECK-BE-NEXT:    rsbs r2, r2, #0
+; CHECK-BE-NEXT:    bfi r1, r2, #3, #1
+; CHECK-BE-NEXT:    lsls r2, r1, #31
+; CHECK-BE-NEXT:    bne .LBB27_5
+; CHECK-BE-NEXT:  @ %bb.1: @ %else
+; CHECK-BE-NEXT:    lsls r2, r1, #30
+; CHECK-BE-NEXT:    bmi .LBB27_6
+; CHECK-BE-NEXT:  .LBB27_2: @ %else2
+; CHECK-BE-NEXT:    lsls r2, r1, #29
+; CHECK-BE-NEXT:    bmi .LBB27_7
+; CHECK-BE-NEXT:  .LBB27_3: @ %else4
+; CHECK-BE-NEXT:    lsls r1, r1, #28
+; CHECK-BE-NEXT:    bmi .LBB27_8
+; CHECK-BE-NEXT:  .LBB27_4: @ %else6
+; CHECK-BE-NEXT:    add sp, #20
+; CHECK-BE-NEXT:    bx lr
+; CHECK-BE-NEXT:  .LBB27_5: @ %cond.store
+; CHECK-BE-NEXT:    vstr.16 s0, [sp, #12]
+; CHECK-BE-NEXT:    ldrh.w r2, [sp, #12]
+; CHECK-BE-NEXT:    strh r2, [r0]
+; CHECK-BE-NEXT:    lsls r2, r1, #30
+; CHECK-BE-NEXT:    bpl .LBB27_2
+; CHECK-BE-NEXT:  .LBB27_6: @ %cond.store1
+; CHECK-BE-NEXT:    vmovx.f16 s4, s0
+; CHECK-BE-NEXT:    vstr.16 s4, [sp, #8]
+; CHECK-BE-NEXT:    ldrh.w r2, [sp, #8]
+; CHECK-BE-NEXT:    strh r2, [r0, #2]
+; CHECK-BE-NEXT:    lsls r2, r1, #29
+; CHECK-BE-NEXT:    bpl .LBB27_3
+; CHECK-BE-NEXT:  .LBB27_7: @ %cond.store3
+; CHECK-BE-NEXT:    vstr.16 s1, [sp, #4]
+; CHECK-BE-NEXT:    ldrh.w r2, [sp, #4]
+; CHECK-BE-NEXT:    strh r2, [r0, #4]
+; CHECK-BE-NEXT:    lsls r1, r1, #28
+; CHECK-BE-NEXT:    bpl .LBB27_4
+; CHECK-BE-NEXT:  .LBB27_8: @ %cond.store5
+; CHECK-BE-NEXT:    vmovx.f16 s0, s1
+; CHECK-BE-NEXT:    vstr.16 s0, [sp]
+; CHECK-BE-NEXT:    ldrh.w r1, [sp]
+; CHECK-BE-NEXT:    strh r1, [r0, #6]
+; CHECK-BE-NEXT:    add sp, #20
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = fcmp ogt <4 x float> %a, zeroinitializer
+  %trunc = fptrunc <4 x float> %a to <4 x half>
+  call void @llvm.masked.store.v4f16.p0v4f16(<4 x half> %trunc, <4 x half>* %dest, i32 1, <4 x i1> %c)
+  ret void
+}
+
+declare void @llvm.masked.store.v4i8.p0v4i8(<4 x i8>, <4 x i8>*, i32, <4 x i1>)
+declare void @llvm.masked.store.v8i8.p0v8i8(<8 x i8>, <8 x i8>*, i32, <8 x i1>)
+declare void @llvm.masked.store.v4i16.p0v4i16(<4 x i16>, <4 x i16>*, i32, <4 x i1>)
+declare void @llvm.masked.store.v4i32.p0v4i32(<4 x i32>, <4 x i32>*, i32, <4 x i1>)
+declare void @llvm.masked.store.v8i16.p0v8i16(<8 x i16>, <8 x i16>*, i32, <8 x i1>)
+declare void @llvm.masked.store.v16i8.p0v16i8(<16 x i8>, <16 x i8>*, i32, <16 x i1>)
+declare void @llvm.masked.store.v4f32.p0v4f32(<4 x float>, <4 x float>*, i32, <4 x i1>)
+declare void @llvm.masked.store.v4f16.p0v4f16(<4 x half>, <4 x half>*, i32, <4 x i1>)
+declare void @llvm.masked.store.v8f16.p0v8f16(<8 x half>, <8 x half>*, i32, <8 x i1>)
+declare void @llvm.masked.store.v2i64.p0v2i64(<2 x i64>, <2 x i64>*, i32, <2 x i1>)
+declare void @llvm.masked.store.v2f64.p0v2f64(<2 x double>, <2 x double>*, i32, <2 x i1>)
diff --git a/llvm/test/CodeGen/Thumb2/mve-minmax.ll b/llvm/test/CodeGen/Thumb2/mve-minmax.ll
index 0b6308f88..14b1f466b 100644
--- a/llvm/test/CodeGen/Thumb2/mve-minmax.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-minmax.ll
@@ -55,15 +55,13 @@ define arm_aapcs_vfpcc <2 x i64> @smin_v2i64(<2 x i64> %s1, <2 x i64> %s2) {
 ; CHECK-NEXT:    it lt
 ; CHECK-NEXT:    movlt r1, #1
 ; CHECK-NEXT:    cmp r1, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r1, #-1
+; CHECK-NEXT:    csetm r1, ne
 ; CHECK-NEXT:    subs r2, r3, r2
 ; CHECK-NEXT:    sbcs.w r2, lr, r12
 ; CHECK-NEXT:    it lt
 ; CHECK-NEXT:    movlt r0, #1
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csetm r0, ne
 ; CHECK-NEXT:    vmov.32 q2[0], r0
 ; CHECK-NEXT:    vmov.32 q2[1], r0
 ; CHECK-NEXT:    vmov.32 q2[2], r1
@@ -131,15 +129,13 @@ define arm_aapcs_vfpcc <2 x i64> @umin_v2i64(<2 x i64> %s1, <2 x i64> %s2) {
 ; CHECK-NEXT:    it lo
 ; CHECK-NEXT:    movlo r1, #1
 ; CHECK-NEXT:    cmp r1, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r1, #-1
+; CHECK-NEXT:    csetm r1, ne
 ; CHECK-NEXT:    subs r2, r3, r2
 ; CHECK-NEXT:    sbcs.w r2, lr, r12
 ; CHECK-NEXT:    it lo
 ; CHECK-NEXT:    movlo r0, #1
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csetm r0, ne
 ; CHECK-NEXT:    vmov.32 q2[0], r0
 ; CHECK-NEXT:    vmov.32 q2[1], r0
 ; CHECK-NEXT:    vmov.32 q2[2], r1
@@ -208,15 +204,13 @@ define arm_aapcs_vfpcc <2 x i64> @smax_v2i64(<2 x i64> %s1, <2 x i64> %s2) {
 ; CHECK-NEXT:    it lt
 ; CHECK-NEXT:    movlt r1, #1
 ; CHECK-NEXT:    cmp r1, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r1, #-1
+; CHECK-NEXT:    csetm r1, ne
 ; CHECK-NEXT:    subs r2, r3, r2
 ; CHECK-NEXT:    sbcs.w r2, lr, r12
 ; CHECK-NEXT:    it lt
 ; CHECK-NEXT:    movlt r0, #1
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csetm r0, ne
 ; CHECK-NEXT:    vmov.32 q2[0], r0
 ; CHECK-NEXT:    vmov.32 q2[1], r0
 ; CHECK-NEXT:    vmov.32 q2[2], r1
@@ -284,15 +278,13 @@ define arm_aapcs_vfpcc <2 x i64> @umax_v2i64(<2 x i64> %s1, <2 x i64> %s2) {
 ; CHECK-NEXT:    it lo
 ; CHECK-NEXT:    movlo r1, #1
 ; CHECK-NEXT:    cmp r1, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r1, #-1
+; CHECK-NEXT:    csetm r1, ne
 ; CHECK-NEXT:    subs r2, r3, r2
 ; CHECK-NEXT:    sbcs.w r2, lr, r12
 ; CHECK-NEXT:    it lo
 ; CHECK-NEXT:    movlo r0, #1
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csetm r0, ne
 ; CHECK-NEXT:    vmov.32 q2[0], r0
 ; CHECK-NEXT:    vmov.32 q2[1], r0
 ; CHECK-NEXT:    vmov.32 q2[2], r1
@@ -331,60 +323,36 @@ entry:
 define arm_aapcs_vfpcc <8 x half> @minnm_float16_t(<8 x half> %src1, <8 x half> %src2) {
 ; CHECK-MVE-LABEL: minnm_float16_t:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[0]
-; CHECK-MVE-NEXT:    vmov.u16 r1, q0[1]
-; CHECK-MVE-NEXT:    vmov s8, r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q1[0]
-; CHECK-MVE-NEXT:    vmov s10, r0
-; CHECK-MVE-NEXT:    vmov.u16 r2, q1[1]
-; CHECK-MVE-NEXT:    vminnm.f16 s8, s10, s8
-; CHECK-MVE-NEXT:    vmov s10, r2
+; CHECK-MVE-NEXT:    vminnm.f16 s8, s4, s0
+; CHECK-MVE-NEXT:    vmovx.f16 s10, s4
 ; CHECK-MVE-NEXT:    vmov r0, s8
-; CHECK-MVE-NEXT:    vmov s8, r1
+; CHECK-MVE-NEXT:    vmovx.f16 s8, s0
 ; CHECK-MVE-NEXT:    vminnm.f16 s8, s10, s8
+; CHECK-MVE-NEXT:    vminnm.f16 s12, s5, s1
 ; CHECK-MVE-NEXT:    vmov r1, s8
 ; CHECK-MVE-NEXT:    vmov.16 q2[0], r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[2]
+; CHECK-MVE-NEXT:    vmov r0, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s1
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s5
 ; CHECK-MVE-NEXT:    vmov.16 q2[1], r1
-; CHECK-MVE-NEXT:    vmov s12, r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q1[2]
-; CHECK-MVE-NEXT:    vmov s14, r0
 ; CHECK-MVE-NEXT:    vminnm.f16 s12, s14, s12
-; CHECK-MVE-NEXT:    vmov r0, s12
 ; CHECK-MVE-NEXT:    vmov.16 q2[2], r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[3]
-; CHECK-MVE-NEXT:    vmov s12, r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q1[3]
-; CHECK-MVE-NEXT:    vmov s14, r0
-; CHECK-MVE-NEXT:    vminnm.f16 s12, s14, s12
 ; CHECK-MVE-NEXT:    vmov r0, s12
+; CHECK-MVE-NEXT:    vminnm.f16 s12, s6, s2
 ; CHECK-MVE-NEXT:    vmov.16 q2[3], r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[4]
-; CHECK-MVE-NEXT:    vmov s12, r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q1[4]
-; CHECK-MVE-NEXT:    vmov s14, r0
-; CHECK-MVE-NEXT:    vminnm.f16 s12, s14, s12
 ; CHECK-MVE-NEXT:    vmov r0, s12
-; CHECK-MVE-NEXT:    vmov.16 q2[4], r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[5]
-; CHECK-MVE-NEXT:    vmov s12, r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q1[5]
-; CHECK-MVE-NEXT:    vmov s14, r0
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s2
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s6
 ; CHECK-MVE-NEXT:    vminnm.f16 s12, s14, s12
+; CHECK-MVE-NEXT:    vmov.16 q2[4], r0
 ; CHECK-MVE-NEXT:    vmov r0, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s7
+; CHECK-MVE-NEXT:    vminnm.f16 s12, s7, s3
 ; CHECK-MVE-NEXT:    vmov.16 q2[5], r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[6]
-; CHECK-MVE-NEXT:    vmov s12, r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q1[6]
-; CHECK-MVE-NEXT:    vmov s14, r0
-; CHECK-MVE-NEXT:    vminnm.f16 s12, s14, s12
 ; CHECK-MVE-NEXT:    vmov r0, s12
-; CHECK-MVE-NEXT:    vmov.16 q2[6], r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[7]
-; CHECK-MVE-NEXT:    vmov s0, r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q1[7]
-; CHECK-MVE-NEXT:    vmov s2, r0
 ; CHECK-MVE-NEXT:    vminnm.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov.16 q2[6], r0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q2[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q2
@@ -412,22 +380,20 @@ define arm_aapcs_vfpcc <2 x double> @maxnm_float64_t(<2 x double> %src1, <2 x do
 ; CHECK-NEXT:    vmov r0, r1, d9
 ; CHECK-NEXT:    vmov r2, r3, d11
 ; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    vmov r0, r1, d8
+; CHECK-NEXT:    vmov r12, r1, d8
+; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    vmov r2, r3, d10
-; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r4, #1
-; CHECK-NEXT:    cmp r4, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r4, #-1
+; CHECK-NEXT:    movne r0, #1
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    csetm r4, ne
+; CHECK-NEXT:    mov r0, r12
 ; CHECK-NEXT:    bl __aeabi_dcmpgt
 ; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    it ne
 ; CHECK-NEXT:    movne r0, #1
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csetm r0, ne
 ; CHECK-NEXT:    vmov.32 q0[0], r0
 ; CHECK-NEXT:    vmov.32 q0[1], r0
 ; CHECK-NEXT:    vmov.32 q0[2], r4
diff --git a/llvm/test/CodeGen/Thumb2/mve-multivec-spill.ll b/llvm/test/CodeGen/Thumb2/mve-multivec-spill.ll
new file mode 100644
index 000000000..6c71b0069
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-multivec-spill.ll
@@ -0,0 +1,103 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -O3 -mattr=+mve %s -o - | FileCheck %s
+
+declare void @external_function()
+
+define arm_aapcs_vfpcc void @spill_multivector(<4 x i32>* %p) {
+; CHECK-LABEL: spill_multivector:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, r5, r6, lr}
+; CHECK-NEXT:    push {r4, r5, r6, lr}
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    .pad #112
+; CHECK-NEXT:    sub sp, #112
+; CHECK-NEXT:    vld20.32 {q0, q1}, [r0]
+; CHECK-NEXT:    add.w lr, sp, #64
+; CHECK-NEXT:    mov r4, r0
+; CHECK-NEXT:    vld21.32 {q0, q1}, [r0]
+; CHECK-NEXT:    adds r0, #64
+; CHECK-NEXT:    vstmia lr, {d0, d1, d2, d3} @ 32-byte Spill
+; CHECK-NEXT:    add.w lr, sp, #32
+; CHECK-NEXT:    vld20.32 {q0, q1}, [r0]
+; CHECK-NEXT:    vld21.32 {q0, q1}, [r0]
+; CHECK-NEXT:    add.w r0, r4, #128
+; CHECK-NEXT:    vstmia lr, {d0, d1, d2, d3} @ 32-byte Spill
+; CHECK-NEXT:    vld20.32 {q0, q1}, [r0]
+; CHECK-NEXT:    vld21.32 {q0, q1}, [r0]
+; CHECK-NEXT:    add.w r0, r4, #192
+; CHECK-NEXT:    vstmia sp, {d0, d1, d2, d3} @ 32-byte Spill
+; CHECK-NEXT:    vld20.32 {q6, q7}, [r0]
+; CHECK-NEXT:    vld21.32 {q6, q7}, [r0]
+; CHECK-NEXT:    add.w r0, r4, #256
+; CHECK-NEXT:    vld20.32 {q4, q5}, [r0]
+; CHECK-NEXT:    vld21.32 {q4, q5}, [r0]
+; CHECK-NEXT:    bl external_function
+; CHECK-NEXT:    vldmia sp, {d2, d3, d4, d5} @ 32-byte Reload
+; CHECK-NEXT:    add r0, sp, #32
+; CHECK-NEXT:    vstrw.32 q2, [r4, #80]
+; CHECK-NEXT:    vstrw.32 q5, [r4, #144]
+; CHECK-NEXT:    vstrw.32 q4, [r4, #128]
+; CHECK-NEXT:    vstrw.32 q7, [r4, #112]
+; CHECK-NEXT:    vstrw.32 q1, [r4, #64]
+; CHECK-NEXT:    vldmia r0, {d2, d3, d4, d5} @ 32-byte Reload
+; CHECK-NEXT:    add r0, sp, #64
+; CHECK-NEXT:    vstrw.32 q2, [r4, #48]
+; CHECK-NEXT:    vstrw.32 q6, [r4, #96]
+; CHECK-NEXT:    vstrw.32 q1, [r4, #32]
+; CHECK-NEXT:    vldmia r0, {d2, d3, d4, d5} @ 32-byte Reload
+; CHECK-NEXT:    vstrw.32 q2, [r4, #16]
+; CHECK-NEXT:    vstrw.32 q1, [r4]
+; CHECK-NEXT:    add sp, #112
+; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    pop {r4, r5, r6, pc}
+entry:
+  %ip01 = bitcast <4 x i32>* %p to i32*
+  %v01 = call { <4 x i32>, <4 x i32> } @llvm.arm.mve.vld2q.v4i32.v4i32.p0i32(i32* %ip01)
+  %ip23 = getelementptr i32, i32* %ip01, i32 16
+  %v23 = call { <4 x i32>, <4 x i32> } @llvm.arm.mve.vld2q.v4i32.v4i32.p0i32(i32* %ip23)
+  %ip45 = getelementptr i32, i32* %ip23, i32 16
+  %v45 = call { <4 x i32>, <4 x i32> } @llvm.arm.mve.vld2q.v4i32.v4i32.p0i32(i32* %ip45)
+  %ip67 = getelementptr i32, i32* %ip45, i32 16
+  %v67 = call { <4 x i32>, <4 x i32> } @llvm.arm.mve.vld2q.v4i32.v4i32.p0i32(i32* %ip67)
+  %ip89 = getelementptr i32, i32* %ip67, i32 16
+  %v89 = call { <4 x i32>, <4 x i32> } @llvm.arm.mve.vld2q.v4i32.v4i32.p0i32(i32* %ip89)
+  call void @external_function()
+
+  %v0 = extractvalue { <4 x i32>, <4 x i32> } %v01, 0
+  %v1 = extractvalue { <4 x i32>, <4 x i32> } %v01, 1
+  store <4 x i32> %v0, <4 x i32>* %p, align 4
+  %p1 = getelementptr <4 x i32>, <4 x i32>* %p, i32 1
+  store <4 x i32> %v1, <4 x i32>* %p1, align 4
+
+  %v2 = extractvalue { <4 x i32>, <4 x i32> } %v23, 0
+  %v3 = extractvalue { <4 x i32>, <4 x i32> } %v23, 1
+  %p2 = getelementptr <4 x i32>, <4 x i32>* %p, i32 2
+  store <4 x i32> %v2, <4 x i32>* %p2, align 4
+  %p3 = getelementptr <4 x i32>, <4 x i32>* %p, i32 3
+  store <4 x i32> %v3, <4 x i32>* %p3, align 4
+
+  %v4 = extractvalue { <4 x i32>, <4 x i32> } %v45, 0
+  %v5 = extractvalue { <4 x i32>, <4 x i32> } %v45, 1
+  %p4 = getelementptr <4 x i32>, <4 x i32>* %p, i32 4
+  store <4 x i32> %v4, <4 x i32>* %p4, align 4
+  %p5 = getelementptr <4 x i32>, <4 x i32>* %p, i32 5
+  store <4 x i32> %v5, <4 x i32>* %p5, align 4
+
+  %v6 = extractvalue { <4 x i32>, <4 x i32> } %v67, 0
+  %v7 = extractvalue { <4 x i32>, <4 x i32> } %v67, 1
+  %p6 = getelementptr <4 x i32>, <4 x i32>* %p, i32 6
+  store <4 x i32> %v6, <4 x i32>* %p6, align 4
+  %p7 = getelementptr <4 x i32>, <4 x i32>* %p, i32 7
+  store <4 x i32> %v7, <4 x i32>* %p7, align 4
+
+  %v8 = extractvalue { <4 x i32>, <4 x i32> } %v89, 0
+  %v9 = extractvalue { <4 x i32>, <4 x i32> } %v89, 1
+  %p8 = getelementptr <4 x i32>, <4 x i32>* %p, i32 8
+  store <4 x i32> %v8, <4 x i32>* %p8, align 4
+  %p9 = getelementptr <4 x i32>, <4 x i32>* %p, i32 9
+  store <4 x i32> %v9, <4 x i32>* %p9, align 4
+  ret void
+}
+
+declare { <4 x i32>, <4 x i32> } @llvm.arm.mve.vld2q.v4i32.v4i32.p0i32(i32*)
diff --git a/llvm/test/CodeGen/Thumb2/mve-phireg.ll b/llvm/test/CodeGen/Thumb2/mve-phireg.ll
new file mode 100644
index 000000000..dde14da7e
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-phireg.ll
@@ -0,0 +1,285 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -O3 -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve.fp -verify-machineinstrs %s -o - | FileCheck %s
+
+; verify-machineinstrs previously caught the incorrect use of QPR in the stack reloads.
+
+define arm_aapcs_vfpcc void @k() {
+; CHECK-LABEL: k:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, r5, r6, lr}
+; CHECK-NEXT:    push {r4, r5, r6, lr}
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    .pad #16
+; CHECK-NEXT:    sub sp, #16
+; CHECK-NEXT:    adr r5, .LCPI0_0
+; CHECK-NEXT:    adr r4, .LCPI0_1
+; CHECK-NEXT:    vldrw.u32 q5, [r5]
+; CHECK-NEXT:    vldrw.u32 q6, [r4]
+; CHECK-NEXT:    vmov.i32 q0, #0x1
+; CHECK-NEXT:    vmov.i8 q1, #0x0
+; CHECK-NEXT:    vmov.i8 q2, #0xff
+; CHECK-NEXT:    vmov.i16 q3, #0x6
+; CHECK-NEXT:    vmov.i16 q4, #0x3
+; CHECK-NEXT:    movs r0, #0
+; CHECK-NEXT:  .LBB0_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vand q5, q5, q0
+; CHECK-NEXT:    vand q6, q6, q0
+; CHECK-NEXT:    vcmp.i32 eq, q5, zr
+; CHECK-NEXT:    vpsel q5, q2, q1
+; CHECK-NEXT:    vcmp.i32 eq, q6, zr
+; CHECK-NEXT:    vpsel q7, q2, q1
+; CHECK-NEXT:    vmov r1, s28
+; CHECK-NEXT:    vmov.16 q6[0], r1
+; CHECK-NEXT:    vmov r1, s29
+; CHECK-NEXT:    vmov.16 q6[1], r1
+; CHECK-NEXT:    vmov r1, s30
+; CHECK-NEXT:    vmov.16 q6[2], r1
+; CHECK-NEXT:    vmov r1, s31
+; CHECK-NEXT:    vmov.16 q6[3], r1
+; CHECK-NEXT:    vmov r1, s20
+; CHECK-NEXT:    vmov.16 q6[4], r1
+; CHECK-NEXT:    vmov r1, s21
+; CHECK-NEXT:    vmov.16 q6[5], r1
+; CHECK-NEXT:    vmov r1, s22
+; CHECK-NEXT:    vmov.16 q6[6], r1
+; CHECK-NEXT:    vmov r1, s23
+; CHECK-NEXT:    vmov.16 q6[7], r1
+; CHECK-NEXT:    vcmp.i16 ne, q6, zr
+; CHECK-NEXT:    vmov.i32 q6, #0x0
+; CHECK-NEXT:    vpsel q5, q4, q3
+; CHECK-NEXT:    vstrh.16 q5, [r0]
+; CHECK-NEXT:    vmov q5, q6
+; CHECK-NEXT:    cbz r0, .LBB0_2
+; CHECK-NEXT:    le .LBB0_1
+; CHECK-NEXT:  .LBB0_2: @ %for.cond4.preheader
+; CHECK-NEXT:    movs r6, #0
+; CHECK-NEXT:    cbnz r6, .LBB0_5
+; CHECK-NEXT:  .LBB0_3: @ %for.body10
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    cbnz r6, .LBB0_4
+; CHECK-NEXT:    le .LBB0_3
+; CHECK-NEXT:  .LBB0_4: @ %for.cond4.loopexit
+; CHECK-NEXT:    bl l
+; CHECK-NEXT:  .LBB0_5: @ %vector.body105.preheader
+; CHECK-NEXT:    vldrw.u32 q0, [r5]
+; CHECK-NEXT:    vldrw.u32 q1, [r4]
+; CHECK-NEXT:    vmov.i32 q2, #0x8
+; CHECK-NEXT:  .LBB0_6: @ %vector.body105
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vadd.i32 q1, q1, q2
+; CHECK-NEXT:    vadd.i32 q0, q0, q2
+; CHECK-NEXT:    cbz r6, .LBB0_7
+; CHECK-NEXT:    le .LBB0_6
+; CHECK-NEXT:  .LBB0_7: @ %vector.body115.ph
+; CHECK-NEXT:    vldrw.u32 q0, [r4]
+; CHECK-NEXT:    vstrw.32 q0, [sp] @ 16-byte Spill
+; CHECK-NEXT:    @APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    @NO_APP
+; CHECK-NEXT:    vldrw.u32 q1, [sp] @ 16-byte Reload
+; CHECK-NEXT:    vmov.i32 q0, #0x4
+; CHECK-NEXT:  .LBB0_8: @ %vector.body115
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vadd.i32 q1, q1, q0
+; CHECK-NEXT:    b .LBB0_8
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  @ %bb.9:
+; CHECK-NEXT:  .LCPI0_0:
+; CHECK-NEXT:    .long 4 @ 0x4
+; CHECK-NEXT:    .long 5 @ 0x5
+; CHECK-NEXT:    .long 6 @ 0x6
+; CHECK-NEXT:    .long 7 @ 0x7
+; CHECK-NEXT:  .LCPI0_1:
+; CHECK-NEXT:    .long 0 @ 0x0
+; CHECK-NEXT:    .long 1 @ 0x1
+; CHECK-NEXT:    .long 2 @ 0x2
+; CHECK-NEXT:    .long 3 @ 0x3
+entry:
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %entry
+  %vec.ind = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, %entry ], [ zeroinitializer, %vector.body ]
+  %0 = and <8 x i32> %vec.ind, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  %1 = icmp eq <8 x i32> %0, zeroinitializer
+  %2 = select <8 x i1> %1, <8 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>, <8 x i16> <i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6>
+  %3 = bitcast i16* undef to <8 x i16>*
+  store <8 x i16> %2, <8 x i16>* %3, align 2
+  %4 = icmp eq i32 undef, 128
+  br i1 %4, label %for.cond4.preheader, label %vector.body
+
+for.cond4.preheader:                              ; preds = %vector.body
+  br i1 undef, label %vector.body105, label %for.body10
+
+for.cond4.loopexit:                               ; preds = %for.body10
+  %call5 = call arm_aapcs_vfpcc i32 bitcast (i32 (...)* @l to i32 ()*)()
+  br label %vector.body105
+
+for.body10:                                       ; preds = %for.body10, %for.cond4.preheader
+  %exitcond88 = icmp eq i32 undef, 7
+  br i1 %exitcond88, label %for.cond4.loopexit, label %for.body10
+
+vector.body105:                                   ; preds = %vector.body105, %for.cond4.loopexit, %for.cond4.preheader
+  %vec.ind113 = phi <8 x i32> [ %vec.ind.next114, %vector.body105 ], [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, %for.cond4.loopexit ], [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, %for.cond4.preheader ]
+  %5 = and <8 x i32> %vec.ind113, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  %vec.ind.next114 = add <8 x i32> %vec.ind113, <i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8>
+  %6 = icmp eq i32 undef, 256
+  br i1 %6, label %vector.body115.ph, label %vector.body105
+
+vector.body115.ph:                                ; preds = %vector.body105
+  tail call void asm sideeffect "nop", "~{s0},~{s4},~{s8},~{s12},~{s16},~{s20},~{s24},~{s28},~{memory}"()
+  br label %vector.body115
+
+vector.body115:                                   ; preds = %vector.body115, %vector.body115.ph
+  %vec.ind123 = phi <4 x i32> [ %vec.ind.next124, %vector.body115 ], [ <i32 0, i32 1, i32 2, i32 3>, %vector.body115.ph ]
+  %7 = icmp eq <4 x i32> %vec.ind123, zeroinitializer
+  %vec.ind.next124 = add <4 x i32> %vec.ind123, <i32 4, i32 4, i32 4, i32 4>
+  br label %vector.body115
+}
+
+
+@a = external dso_local global i32, align 4
+@b = dso_local local_unnamed_addr global i32 ptrtoint (i32* @a to i32), align 4
+@c = dso_local global i32 2, align 4
+@d = dso_local global i32 2, align 4
+
+define dso_local i32 @e() #0 {
+; CHECK-LABEL: e:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; CHECK-NEXT:    .pad #4
+; CHECK-NEXT:    sub sp, #4
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    .pad #440
+; CHECK-NEXT:    sub sp, #440
+; CHECK-NEXT:    vldr s20, .LCPI1_0
+; CHECK-NEXT:    movw r9, :lower16:.L_MergedGlobals
+; CHECK-NEXT:    vldr s23, .LCPI1_1
+; CHECK-NEXT:    movt r9, :upper16:.L_MergedGlobals
+; CHECK-NEXT:    mov.w r8, #4
+; CHECK-NEXT:    mov r5, r9
+; CHECK-NEXT:    strh.w r8, [sp, #438]
+; CHECK-NEXT:    movs r6, #0
+; CHECK-NEXT:    vstr s23, [sp, #48]
+; CHECK-NEXT:    mov r7, r9
+; CHECK-NEXT:    vstr s23, [sp, #40]
+; CHECK-NEXT:    movw r4, :lower16:e
+; CHECK-NEXT:    ldr r1, [r5, #4]!
+; CHECK-NEXT:    movt r4, :upper16:e
+; CHECK-NEXT:    str r6, [sp, #76]
+; CHECK-NEXT:    vmov s5, r4
+; CHECK-NEXT:    vmov.32 q7[0], r5
+; CHECK-NEXT:    ldr r0, [r7, #8]!
+; CHECK-NEXT:    vmov q0, q7
+; CHECK-NEXT:    ldr r2, [sp, #48]
+; CHECK-NEXT:    vmov.32 q0[1], r5
+; CHECK-NEXT:    vmov s21, r5
+; CHECK-NEXT:    vmov.32 q0[2], r2
+; CHECK-NEXT:    ldr r2, [sp, #40]
+; CHECK-NEXT:    vdup.32 q2, r5
+; CHECK-NEXT:    vmov.32 q4[0], r7
+; CHECK-NEXT:    vmov q6, q4
+; CHECK-NEXT:    vstrw.32 q2, [sp] @ 16-byte Spill
+; CHECK-NEXT:    vmov.32 q6[1], r2
+; CHECK-NEXT:    vldrw.u32 q3, [sp] @ 16-byte Reload
+; CHECK-NEXT:    vmov.f32 s22, s21
+; CHECK-NEXT:    vmov.32 q6[2], r7
+; CHECK-NEXT:    vmov.f32 s4, s20
+; CHECK-NEXT:    vstrw.32 q2, [sp, #16] @ 16-byte Spill
+; CHECK-NEXT:    vmov.f32 s6, s21
+; CHECK-NEXT:    vmov.32 q3[0], r4
+; CHECK-NEXT:    vmov.32 q2[1], r4
+; CHECK-NEXT:    vmov.32 q6[3], r4
+; CHECK-NEXT:    vmov.32 q0[3], r4
+; CHECK-NEXT:    vmov.f32 s7, s23
+; CHECK-NEXT:    str r1, [sp, #72]
+; CHECK-NEXT:    vstrw.32 q6, [sp, #124]
+; CHECK-NEXT:    str r1, [r0]
+; CHECK-NEXT:    movs r1, #64
+; CHECK-NEXT:    str r0, [r0]
+; CHECK-NEXT:    vstrw.32 q5, [sp, #92]
+; CHECK-NEXT:    vstrw.32 q1, [r0]
+; CHECK-NEXT:    vstrw.32 q2, [r0]
+; CHECK-NEXT:    vstrw.32 q3, [r0]
+; CHECK-NEXT:    vstrw.32 q0, [r0]
+; CHECK-NEXT:    bl __aeabi_memclr4
+; CHECK-NEXT:    vstr s23, [sp, #44]
+; CHECK-NEXT:    vmov.32 q7[1], r7
+; CHECK-NEXT:    ldr r0, [sp, #44]
+; CHECK-NEXT:    vldrw.u32 q0, [sp, #16] @ 16-byte Reload
+; CHECK-NEXT:    vmov.32 q4[1], r4
+; CHECK-NEXT:    vmov.32 q7[2], r5
+; CHECK-NEXT:    vmov.32 q4[2], r5
+; CHECK-NEXT:    vmov.32 q0[0], r6
+; CHECK-NEXT:    vmov.32 q4[3], r7
+; CHECK-NEXT:    vmov.32 q7[3], r0
+; CHECK-NEXT:    vstrw.32 q4, [r0]
+; CHECK-NEXT:    str.w r6, [r9]
+; CHECK-NEXT:    vstrw.32 q0, [r0]
+; CHECK-NEXT:    vstrw.32 q7, [r0]
+; CHECK-NEXT:    str.w r8, [sp, #356]
+; CHECK-NEXT:  .LBB1_1: @ %for.cond
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    b .LBB1_1
+; CHECK-NEXT:    .p2align 2
+; CHECK-NEXT:  @ %bb.2:
+; CHECK-NEXT:  .LCPI1_0:
+; CHECK-NEXT:    .long 4 @ float 5.60519386E-45
+; CHECK-NEXT:  .LCPI1_1:
+; CHECK-NEXT:    .long 0 @ float 0
+entry:
+  %f = alloca i16, align 2
+  %g = alloca [3 x [8 x [4 x i16*]]], align 4
+  store i16 4, i16* %f, align 2
+  %0 = load i32, i32* @c, align 4
+  %1 = load i32, i32* @d, align 4
+  %arrayinit.element7 = getelementptr inbounds [3 x [8 x [4 x i16*]]], [3 x [8 x [4 x i16*]]]* %g, i32 0, i32 0, i32 1, i32 1
+  %2 = bitcast i16** %arrayinit.element7 to i32*
+  store i32 %0, i32* %2, align 4
+  %arrayinit.element8 = getelementptr inbounds [3 x [8 x [4 x i16*]]], [3 x [8 x [4 x i16*]]]* %g, i32 0, i32 0, i32 1, i32 2
+  store i16* null, i16** %arrayinit.element8, align 4
+  %3 = bitcast i16** undef to i32*
+  store i32 %1, i32* %3, align 4
+  %4 = bitcast i16** undef to i32*
+  store i32 %0, i32* %4, align 4
+  %arrayinit.element13 = getelementptr inbounds [3 x [8 x [4 x i16*]]], [3 x [8 x [4 x i16*]]]* %g, i32 0, i32 0, i32 2, i32 2
+  %5 = bitcast i16** %arrayinit.element13 to <4 x i16*>*
+  store <4 x i16*> <i16* inttoptr (i32 4 to i16*), i16* bitcast (i32* @c to i16*), i16* bitcast (i32* @c to i16*), i16* null>, <4 x i16*>* %5, align 4
+  %arrayinit.element24 = getelementptr inbounds [3 x [8 x [4 x i16*]]], [3 x [8 x [4 x i16*]]]* %g, i32 0, i32 0, i32 4, i32 2
+  %6 = bitcast i16** %arrayinit.element24 to <4 x i16*>*
+  store <4 x i16*> <i16* bitcast (i32* @d to i16*), i16* null, i16* bitcast (i32* @d to i16*), i16* bitcast (i32 ()* @e to i16*)>, <4 x i16*>* %6, align 4
+  %7 = bitcast i16** undef to <4 x i16*>*
+  store <4 x i16*> <i16* inttoptr (i32 4 to i16*), i16* bitcast (i32 ()* @e to i16*), i16* bitcast (i32* @c to i16*), i16* null>, <4 x i16*>* %7, align 4
+  %8 = bitcast i16** undef to <4 x i16*>*
+  store <4 x i16*> <i16* bitcast (i32* @c to i16*), i16* bitcast (i32 ()* @e to i16*), i16* bitcast (i32* @c to i16*), i16* bitcast (i32* @c to i16*)>, <4 x i16*>* %8, align 4
+  %9 = bitcast i16** undef to <4 x i16*>*
+  store <4 x i16*> <i16* bitcast (i32 ()* @e to i16*), i16* bitcast (i32* @c to i16*), i16* bitcast (i32* @c to i16*), i16* bitcast (i32* @c to i16*)>, <4 x i16*>* %9, align 4
+  %10 = bitcast i16** undef to <4 x i16*>*
+  store <4 x i16*> <i16* bitcast (i32* @c to i16*), i16* bitcast (i32* @c to i16*), i16* null, i16* bitcast (i32 ()* @e to i16*)>, <4 x i16*>* %10, align 4
+  call void @llvm.memset.p0i8.i32(i8* nonnull align 4 dereferenceable(64) undef, i8 0, i32 64, i1 false)
+  %11 = bitcast i16** undef to <4 x i16*>*
+  store <4 x i16*> <i16* bitcast (i32* @d to i16*), i16* bitcast (i32 ()* @e to i16*), i16* bitcast (i32* @c to i16*), i16* bitcast (i32* @d to i16*)>, <4 x i16*>* %11, align 4
+  %12 = bitcast i16** undef to <4 x i16*>*
+  store <4 x i16*> <i16* null, i16* bitcast (i32* @c to i16*), i16* bitcast (i32* @c to i16*), i16* bitcast (i32* @c to i16*)>, <4 x i16*>* %12, align 4
+  %13 = bitcast i16** undef to <4 x i16*>*
+  store <4 x i16*> <i16* bitcast (i32* @c to i16*), i16* bitcast (i32* @d to i16*), i16* bitcast (i32* @c to i16*), i16* null>, <4 x i16*>* %13, align 4
+  %arrayinit.begin78 = getelementptr inbounds [3 x [8 x [4 x i16*]]], [3 x [8 x [4 x i16*]]]* %g, i32 0, i32 2, i32 3, i32 0
+  store i16* inttoptr (i32 4 to i16*), i16** %arrayinit.begin78, align 4
+  store i32 0, i32* @b, align 4
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.cond, %entry
+  br label %for.cond
+}
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.memset.p0i8.i32(i8* nocapture writeonly, i8, i32, i1 immarg) #1
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.memset.p0i8.i64(i8* nocapture writeonly, i8, i64, i1 immarg) #1
+
+
+declare arm_aapcs_vfpcc i32 @l(...)
diff --git a/llvm/test/CodeGen/Thumb2/mve-pred-and.ll b/llvm/test/CodeGen/Thumb2/mve-pred-and.ll
new file mode 100644
index 000000000..4567f60b4
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-pred-and.ll
@@ -0,0 +1,714 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s
+
+define arm_aapcs_vfpcc <4 x i32> @cmpeqz_v4i1(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: cmpeqz_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vorr q2, q0, q1
+; CHECK-NEXT:    vcmp.i32 eq, q2, zr
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %c2 = icmp eq <4 x i32> %b, zeroinitializer
+  %o = and <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpnez_v4i1(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: cmpnez_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
+; CHECK-NEXT:    vcmpt.i32 ne, q1, zr
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %c2 = icmp ne <4 x i32> %b, zeroinitializer
+  %o = and <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpsltz_v4i1(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: cmpsltz_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
+; CHECK-NEXT:    vcmpt.s32 lt, q1, zr
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %c2 = icmp slt <4 x i32> %b, zeroinitializer
+  %o = and <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpsgtz_v4i1(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: cmpsgtz_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
+; CHECK-NEXT:    vcmpt.s32 gt, q1, zr
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %c2 = icmp sgt <4 x i32> %b, zeroinitializer
+  %o = and <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpslez_v4i1(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: cmpslez_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
+; CHECK-NEXT:    vcmpt.s32 le, q1, zr
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %c2 = icmp sle <4 x i32> %b, zeroinitializer
+  %o = and <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpsgez_v4i1(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: cmpsgez_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
+; CHECK-NEXT:    vcmpt.s32 ge, q1, zr
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %c2 = icmp sge <4 x i32> %b, zeroinitializer
+  %o = and <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpultz_v4i1(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: cmpultz_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %c2 = icmp ult <4 x i32> %b, zeroinitializer
+  %o = and <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpugtz_v4i1(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: cmpugtz_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
+; CHECK-NEXT:    vcmpt.i32 ne, q1, zr
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %c2 = icmp ugt <4 x i32> %b, zeroinitializer
+  %o = and <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpulez_v4i1(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: cmpulez_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
+; CHECK-NEXT:    vcmpt.u32 cs, q1, zr
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %c2 = icmp ule <4 x i32> %b, zeroinitializer
+  %o = and <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpugez_v4i1(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: cmpugez_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i32 eq, q0, zr
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %c2 = icmp uge <4 x i32> %b, zeroinitializer
+  %o = and <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+
+
+define arm_aapcs_vfpcc <4 x i32> @cmpeq_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LABEL: cmpeq_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
+; CHECK-NEXT:    vcmpt.i32 eq, q1, q2
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %c2 = icmp eq <4 x i32> %b, %c
+  %o = and <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpne_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LABEL: cmpne_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
+; CHECK-NEXT:    vcmpt.i32 ne, q1, q2
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %c2 = icmp ne <4 x i32> %b, %c
+  %o = and <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpslt_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LABEL: cmpslt_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
+; CHECK-NEXT:    vcmpt.s32 gt, q2, q1
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %c2 = icmp slt <4 x i32> %b, %c
+  %o = and <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpsgt_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LABEL: cmpsgt_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
+; CHECK-NEXT:    vcmpt.s32 gt, q1, q2
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %c2 = icmp sgt <4 x i32> %b, %c
+  %o = and <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpsle_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LABEL: cmpsle_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
+; CHECK-NEXT:    vcmpt.s32 ge, q2, q1
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %c2 = icmp sle <4 x i32> %b, %c
+  %o = and <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpsge_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LABEL: cmpsge_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
+; CHECK-NEXT:    vcmpt.s32 ge, q1, q2
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %c2 = icmp sge <4 x i32> %b, %c
+  %o = and <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpult_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LABEL: cmpult_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
+; CHECK-NEXT:    vcmpt.u32 hi, q2, q1
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %c2 = icmp ult <4 x i32> %b, %c
+  %o = and <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpugt_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LABEL: cmpugt_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
+; CHECK-NEXT:    vcmpt.u32 hi, q1, q2
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %c2 = icmp ugt <4 x i32> %b, %c
+  %o = and <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpule_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LABEL: cmpule_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
+; CHECK-NEXT:    vcmpt.u32 cs, q2, q1
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %c2 = icmp ule <4 x i32> %b, %c
+  %o = and <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpuge_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LABEL: cmpuge_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
+; CHECK-NEXT:    vcmpt.u32 cs, q1, q2
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %c2 = icmp uge <4 x i32> %b, %c
+  %o = and <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+
+define arm_aapcs_vfpcc <4 x i32> @cmpeqr_v4i1(<4 x i32> %a, <4 x i32> %b, i32 %c) {
+; CHECK-LABEL: cmpeqr_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
+; CHECK-NEXT:    vcmpt.i32 eq, q1, r0
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %i = insertelement <4 x i32> undef, i32 %c, i32 0
+  %sp = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %c2 = icmp eq <4 x i32> %b, %sp
+  %o = and <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpner_v4i1(<4 x i32> %a, <4 x i32> %b, i32 %c) {
+; CHECK-LABEL: cmpner_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
+; CHECK-NEXT:    vcmpt.i32 ne, q1, r0
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %i = insertelement <4 x i32> undef, i32 %c, i32 0
+  %sp = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %c2 = icmp ne <4 x i32> %b, %sp
+  %o = and <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpsltr_v4i1(<4 x i32> %a, <4 x i32> %b, i32 %c) {
+; CHECK-LABEL: cmpsltr_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
+; CHECK-NEXT:    vcmpt.s32 lt, q1, r0
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %i = insertelement <4 x i32> undef, i32 %c, i32 0
+  %sp = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %c2 = icmp slt <4 x i32> %b, %sp
+  %o = and <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpsgtr_v4i1(<4 x i32> %a, <4 x i32> %b, i32 %c) {
+; CHECK-LABEL: cmpsgtr_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
+; CHECK-NEXT:    vcmpt.s32 gt, q1, r0
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %i = insertelement <4 x i32> undef, i32 %c, i32 0
+  %sp = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %c2 = icmp sgt <4 x i32> %b, %sp
+  %o = and <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpsler_v4i1(<4 x i32> %a, <4 x i32> %b, i32 %c) {
+; CHECK-LABEL: cmpsler_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
+; CHECK-NEXT:    vcmpt.s32 le, q1, r0
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %i = insertelement <4 x i32> undef, i32 %c, i32 0
+  %sp = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %c2 = icmp sle <4 x i32> %b, %sp
+  %o = and <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpsger_v4i1(<4 x i32> %a, <4 x i32> %b, i32 %c) {
+; CHECK-LABEL: cmpsger_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
+; CHECK-NEXT:    vcmpt.s32 ge, q1, r0
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %i = insertelement <4 x i32> undef, i32 %c, i32 0
+  %sp = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %c2 = icmp sge <4 x i32> %b, %sp
+  %o = and <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpultr_v4i1(<4 x i32> %a, <4 x i32> %b, i32 %c) {
+; CHECK-LABEL: cmpultr_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdup.32 q2, r0
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
+; CHECK-NEXT:    vcmpt.u32 hi, q2, q1
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %i = insertelement <4 x i32> undef, i32 %c, i32 0
+  %sp = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %c2 = icmp ult <4 x i32> %b, %sp
+  %o = and <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpugtr_v4i1(<4 x i32> %a, <4 x i32> %b, i32 %c) {
+; CHECK-LABEL: cmpugtr_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
+; CHECK-NEXT:    vcmpt.u32 hi, q1, r0
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %i = insertelement <4 x i32> undef, i32 %c, i32 0
+  %sp = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %c2 = icmp ugt <4 x i32> %b, %sp
+  %o = and <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpuler_v4i1(<4 x i32> %a, <4 x i32> %b, i32 %c) {
+; CHECK-LABEL: cmpuler_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdup.32 q2, r0
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
+; CHECK-NEXT:    vcmpt.u32 cs, q2, q1
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %i = insertelement <4 x i32> undef, i32 %c, i32 0
+  %sp = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %c2 = icmp ule <4 x i32> %b, %sp
+  %o = and <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpuger_v4i1(<4 x i32> %a, <4 x i32> %b, i32 %c) {
+; CHECK-LABEL: cmpuger_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
+; CHECK-NEXT:    vcmpt.u32 cs, q1, r0
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %i = insertelement <4 x i32> undef, i32 %c, i32 0
+  %sp = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %c2 = icmp uge <4 x i32> %b, %sp
+  %o = and <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+
+
+define arm_aapcs_vfpcc <8 x i16> @cmpeqz_v8i1(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: cmpeqz_v8i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vorr q2, q0, q1
+; CHECK-NEXT:    vcmp.i16 eq, q2, zr
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <8 x i16> %a, zeroinitializer
+  %c2 = icmp eq <8 x i16> %b, zeroinitializer
+  %o = and <8 x i1> %c1, %c2
+  %s = select <8 x i1> %o, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+define arm_aapcs_vfpcc <8 x i16> @cmpeq_v8i1(<8 x i16> %a, <8 x i16> %b, <8 x i16> %c) {
+; CHECK-LABEL: cmpeq_v8i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i16 eq, q0, zr
+; CHECK-NEXT:    vcmpt.i16 eq, q1, q2
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <8 x i16> %a, zeroinitializer
+  %c2 = icmp eq <8 x i16> %b, %c
+  %o = and <8 x i1> %c1, %c2
+  %s = select <8 x i1> %o, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+define arm_aapcs_vfpcc <8 x i16> @cmpeqr_v8i1(<8 x i16> %a, <8 x i16> %b, i16 %c) {
+; CHECK-LABEL: cmpeqr_v8i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i16 eq, q0, zr
+; CHECK-NEXT:    vcmpt.i16 eq, q1, r0
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <8 x i16> %a, zeroinitializer
+  %i = insertelement <8 x i16> undef, i16 %c, i32 0
+  %sp = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
+  %c2 = icmp eq <8 x i16> %b, %sp
+  %o = and <8 x i1> %c1, %c2
+  %s = select <8 x i1> %o, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+
+define arm_aapcs_vfpcc <16 x i8> @cmpeqz_v16i1(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: cmpeqz_v16i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vorr q2, q0, q1
+; CHECK-NEXT:    vcmp.i8 eq, q2, zr
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <16 x i8> %a, zeroinitializer
+  %c2 = icmp eq <16 x i8> %b, zeroinitializer
+  %o = and <16 x i1> %c1, %c2
+  %s = select <16 x i1> %o, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %s
+}
+
+define arm_aapcs_vfpcc <16 x i8> @cmpeq_v16i1(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c) {
+; CHECK-LABEL: cmpeq_v16i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i8 eq, q0, zr
+; CHECK-NEXT:    vcmpt.i8 eq, q1, q2
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <16 x i8> %a, zeroinitializer
+  %c2 = icmp eq <16 x i8> %b, %c
+  %o = and <16 x i1> %c1, %c2
+  %s = select <16 x i1> %o, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %s
+}
+
+define arm_aapcs_vfpcc <16 x i8> @cmpeqr_v16i1(<16 x i8> %a, <16 x i8> %b, i8 %c) {
+; CHECK-LABEL: cmpeqr_v16i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i8 eq, q0, zr
+; CHECK-NEXT:    vcmpt.i8 eq, q1, r0
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <16 x i8> %a, zeroinitializer
+  %i = insertelement <16 x i8> undef, i8 %c, i32 0
+  %sp = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
+  %c2 = icmp eq <16 x i8> %b, %sp
+  %o = and <16 x i1> %c1, %c2
+  %s = select <16 x i1> %o, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %s
+}
+
+
+define arm_aapcs_vfpcc <2 x i64> @cmpeqz_v2i1(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: cmpeqz_v2i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vorr q2, q0, q1
+; CHECK-NEXT:    vmov r0, s9
+; CHECK-NEXT:    vmov r1, s8
+; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    vmov r1, s10
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q3[0], r0
+; CHECK-NEXT:    vmov.32 q3[1], r0
+; CHECK-NEXT:    vmov r0, s11
+; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q3[2], r0
+; CHECK-NEXT:    vmov.32 q3[3], r0
+; CHECK-NEXT:    vbic q1, q1, q3
+; CHECK-NEXT:    vand q0, q0, q3
+; CHECK-NEXT:    vorr q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <2 x i64> %a, zeroinitializer
+  %c2 = icmp eq <2 x i64> %b, zeroinitializer
+  %o = and <2 x i1> %c1, %c2
+  %s = select <2 x i1> %o, <2 x i64> %a, <2 x i64> %b
+  ret <2 x i64> %s
+}
+
+define arm_aapcs_vfpcc <2 x i64> @cmpeq_v2i1(<2 x i64> %a, <2 x i64> %b, <2 x i64> %c) {
+; CHECK-LABEL: cmpeq_v2i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov r0, s9
+; CHECK-NEXT:    vmov r1, s5
+; CHECK-NEXT:    vmov r2, s4
+; CHECK-NEXT:    eors r0, r1
+; CHECK-NEXT:    vmov r1, s8
+; CHECK-NEXT:    eors r1, r2
+; CHECK-NEXT:    vmov r2, s6
+; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    vmov r1, s7
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q3[0], r0
+; CHECK-NEXT:    vmov.32 q3[1], r0
+; CHECK-NEXT:    vmov r0, s11
+; CHECK-NEXT:    eors r0, r1
+; CHECK-NEXT:    vmov r1, s10
+; CHECK-NEXT:    eors r1, r2
+; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    vmov r1, s0
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q3[2], r0
+; CHECK-NEXT:    vmov.32 q3[3], r0
+; CHECK-NEXT:    vmov r0, s1
+; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    vmov r1, s2
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q2[0], r0
+; CHECK-NEXT:    vmov.32 q2[1], r0
+; CHECK-NEXT:    vmov r0, s3
+; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q2[2], r0
+; CHECK-NEXT:    vmov.32 q2[3], r0
+; CHECK-NEXT:    vand q2, q2, q3
+; CHECK-NEXT:    vbic q1, q1, q2
+; CHECK-NEXT:    vand q0, q0, q2
+; CHECK-NEXT:    vorr q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <2 x i64> %a, zeroinitializer
+  %c2 = icmp eq <2 x i64> %b, %c
+  %o = and <2 x i1> %c1, %c2
+  %s = select <2 x i1> %o, <2 x i64> %a, <2 x i64> %b
+  ret <2 x i64> %s
+}
+
+define arm_aapcs_vfpcc <2 x i64> @cmpeqr_v2i1(<2 x i64> %a, <2 x i64> %b, i64 %c) {
+; CHECK-LABEL: cmpeqr_v2i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov r2, s5
+; CHECK-NEXT:    vmov r3, s4
+; CHECK-NEXT:    eors r2, r1
+; CHECK-NEXT:    eors r3, r0
+; CHECK-NEXT:    orrs r2, r3
+; CHECK-NEXT:    cset r2, eq
+; CHECK-NEXT:    tst.w r2, #1
+; CHECK-NEXT:    csetm r2, ne
+; CHECK-NEXT:    vmov.32 q2[0], r2
+; CHECK-NEXT:    vmov.32 q2[1], r2
+; CHECK-NEXT:    vmov r2, s7
+; CHECK-NEXT:    eors r1, r2
+; CHECK-NEXT:    vmov r2, s6
+; CHECK-NEXT:    eors r0, r2
+; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    vmov r1, s0
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q2[2], r0
+; CHECK-NEXT:    vmov.32 q2[3], r0
+; CHECK-NEXT:    vmov r0, s1
+; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    vmov r1, s2
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q3[0], r0
+; CHECK-NEXT:    vmov.32 q3[1], r0
+; CHECK-NEXT:    vmov r0, s3
+; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q3[2], r0
+; CHECK-NEXT:    vmov.32 q3[3], r0
+; CHECK-NEXT:    vand q2, q3, q2
+; CHECK-NEXT:    vbic q1, q1, q2
+; CHECK-NEXT:    vand q0, q0, q2
+; CHECK-NEXT:    vorr q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <2 x i64> %a, zeroinitializer
+  %i = insertelement <2 x i64> undef, i64 %c, i32 0
+  %sp = shufflevector <2 x i64> %i, <2 x i64> undef, <2 x i32> zeroinitializer
+  %c2 = icmp eq <2 x i64> %b, %sp
+  %o = and <2 x i1> %c1, %c2
+  %s = select <2 x i1> %o, <2 x i64> %a, <2 x i64> %b
+  ret <2 x i64> %s
+}
diff --git a/llvm/test/CodeGen/Thumb2/mve-pred-bitcast.ll b/llvm/test/CodeGen/Thumb2/mve-pred-bitcast.ll
new file mode 100644
index 000000000..61a78eb72
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-pred-bitcast.ll
@@ -0,0 +1,431 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-LE
+; RUN: llc -mtriple=thumbebv8.1m.main-arm-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-BE
+
+define arm_aapcs_vfpcc <4 x i32> @bitcast_to_v4i1(i4 %b, <4 x i32> %a) {
+; CHECK-LE-LABEL: bitcast_to_v4i1:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    .pad #4
+; CHECK-LE-NEXT:    sub sp, #4
+; CHECK-LE-NEXT:    and r0, r0, #15
+; CHECK-LE-NEXT:    vmov.i8 q1, #0x0
+; CHECK-LE-NEXT:    vmov.i8 q2, #0xff
+; CHECK-LE-NEXT:    vmsr p0, r0
+; CHECK-LE-NEXT:    vpsel q1, q2, q1
+; CHECK-LE-NEXT:    vmov.u8 r0, q1[0]
+; CHECK-LE-NEXT:    vmov.32 q2[0], r0
+; CHECK-LE-NEXT:    vmov.u8 r0, q1[1]
+; CHECK-LE-NEXT:    vmov.32 q2[1], r0
+; CHECK-LE-NEXT:    vmov.u8 r0, q1[2]
+; CHECK-LE-NEXT:    vmov.32 q2[2], r0
+; CHECK-LE-NEXT:    vmov.u8 r0, q1[3]
+; CHECK-LE-NEXT:    vmov.32 q2[3], r0
+; CHECK-LE-NEXT:    vmov.i32 q1, #0x0
+; CHECK-LE-NEXT:    vcmp.i32 ne, q2, zr
+; CHECK-LE-NEXT:    vpsel q0, q0, q1
+; CHECK-LE-NEXT:    add sp, #4
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: bitcast_to_v4i1:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    .pad #4
+; CHECK-BE-NEXT:    sub sp, #4
+; CHECK-BE-NEXT:    and r0, r0, #15
+; CHECK-BE-NEXT:    vmov.i8 q1, #0x0
+; CHECK-BE-NEXT:    vmov.i8 q2, #0xff
+; CHECK-BE-NEXT:    vmsr p0, r0
+; CHECK-BE-NEXT:    vpsel q1, q2, q1
+; CHECK-BE-NEXT:    vmov.u8 r0, q1[0]
+; CHECK-BE-NEXT:    vmov.32 q2[0], r0
+; CHECK-BE-NEXT:    vmov.u8 r0, q1[1]
+; CHECK-BE-NEXT:    vmov.32 q2[1], r0
+; CHECK-BE-NEXT:    vmov.u8 r0, q1[2]
+; CHECK-BE-NEXT:    vmov.32 q2[2], r0
+; CHECK-BE-NEXT:    vmov.u8 r0, q1[3]
+; CHECK-BE-NEXT:    vmov.32 q2[3], r0
+; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    vcmp.i32 ne, q2, zr
+; CHECK-BE-NEXT:    vmov.i32 q0, #0x0
+; CHECK-BE-NEXT:    vpsel q1, q1, q0
+; CHECK-BE-NEXT:    vrev64.32 q0, q1
+; CHECK-BE-NEXT:    add sp, #4
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = bitcast i4 %b to <4 x i1>
+  %s = select <4 x i1> %c, <4 x i32> %a, <4 x i32> zeroinitializer
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <8 x i16> @bitcast_to_v8i1(i8 %b, <8 x i16> %a) {
+; CHECK-LE-LABEL: bitcast_to_v8i1:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    .pad #8
+; CHECK-LE-NEXT:    sub sp, #8
+; CHECK-LE-NEXT:    uxtb r0, r0
+; CHECK-LE-NEXT:    vmov.i8 q1, #0x0
+; CHECK-LE-NEXT:    vmov.i8 q2, #0xff
+; CHECK-LE-NEXT:    vmsr p0, r0
+; CHECK-LE-NEXT:    vpsel q2, q2, q1
+; CHECK-LE-NEXT:    vmov.u8 r0, q2[0]
+; CHECK-LE-NEXT:    vmov.16 q1[0], r0
+; CHECK-LE-NEXT:    vmov.u8 r0, q2[1]
+; CHECK-LE-NEXT:    vmov.16 q1[1], r0
+; CHECK-LE-NEXT:    vmov.u8 r0, q2[2]
+; CHECK-LE-NEXT:    vmov.16 q1[2], r0
+; CHECK-LE-NEXT:    vmov.u8 r0, q2[3]
+; CHECK-LE-NEXT:    vmov.16 q1[3], r0
+; CHECK-LE-NEXT:    vmov.u8 r0, q2[4]
+; CHECK-LE-NEXT:    vmov.16 q1[4], r0
+; CHECK-LE-NEXT:    vmov.u8 r0, q2[5]
+; CHECK-LE-NEXT:    vmov.16 q1[5], r0
+; CHECK-LE-NEXT:    vmov.u8 r0, q2[6]
+; CHECK-LE-NEXT:    vmov.16 q1[6], r0
+; CHECK-LE-NEXT:    vmov.u8 r0, q2[7]
+; CHECK-LE-NEXT:    vmov.16 q1[7], r0
+; CHECK-LE-NEXT:    vcmp.i16 ne, q1, zr
+; CHECK-LE-NEXT:    vmov.i32 q1, #0x0
+; CHECK-LE-NEXT:    vpsel q0, q0, q1
+; CHECK-LE-NEXT:    add sp, #8
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: bitcast_to_v8i1:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    .pad #8
+; CHECK-BE-NEXT:    sub sp, #8
+; CHECK-BE-NEXT:    uxtb r0, r0
+; CHECK-BE-NEXT:    vmov.i8 q1, #0x0
+; CHECK-BE-NEXT:    vmov.i8 q2, #0xff
+; CHECK-BE-NEXT:    vmsr p0, r0
+; CHECK-BE-NEXT:    vpsel q2, q2, q1
+; CHECK-BE-NEXT:    vmov.u8 r0, q2[0]
+; CHECK-BE-NEXT:    vmov.16 q1[0], r0
+; CHECK-BE-NEXT:    vmov.u8 r0, q2[1]
+; CHECK-BE-NEXT:    vmov.16 q1[1], r0
+; CHECK-BE-NEXT:    vmov.u8 r0, q2[2]
+; CHECK-BE-NEXT:    vmov.16 q1[2], r0
+; CHECK-BE-NEXT:    vmov.u8 r0, q2[3]
+; CHECK-BE-NEXT:    vmov.16 q1[3], r0
+; CHECK-BE-NEXT:    vmov.u8 r0, q2[4]
+; CHECK-BE-NEXT:    vmov.16 q1[4], r0
+; CHECK-BE-NEXT:    vmov.u8 r0, q2[5]
+; CHECK-BE-NEXT:    vmov.16 q1[5], r0
+; CHECK-BE-NEXT:    vmov.u8 r0, q2[6]
+; CHECK-BE-NEXT:    vmov.16 q1[6], r0
+; CHECK-BE-NEXT:    vmov.u8 r0, q2[7]
+; CHECK-BE-NEXT:    vmov.16 q1[7], r0
+; CHECK-BE-NEXT:    vcmp.i16 ne, q1, zr
+; CHECK-BE-NEXT:    vrev64.16 q1, q0
+; CHECK-BE-NEXT:    vmov.i32 q0, #0x0
+; CHECK-BE-NEXT:    vrev32.16 q0, q0
+; CHECK-BE-NEXT:    vpsel q1, q1, q0
+; CHECK-BE-NEXT:    vrev64.16 q0, q1
+; CHECK-BE-NEXT:    add sp, #8
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = bitcast i8 %b to <8 x i1>
+  %s = select <8 x i1> %c, <8 x i16> %a, <8 x i16> zeroinitializer
+  ret <8 x i16> %s
+}
+
+define arm_aapcs_vfpcc <16 x i8> @bitcast_to_v16i1(i16 %b, <16 x i8> %a) {
+; CHECK-LE-LABEL: bitcast_to_v16i1:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    .save {r4, r6, r7, lr}
+; CHECK-LE-NEXT:    push {r4, r6, r7, lr}
+; CHECK-LE-NEXT:    .setfp r7, sp, #8
+; CHECK-LE-NEXT:    add r7, sp, #8
+; CHECK-LE-NEXT:    .pad #16
+; CHECK-LE-NEXT:    sub sp, #16
+; CHECK-LE-NEXT:    mov r4, sp
+; CHECK-LE-NEXT:    bfc r4, #0, #4
+; CHECK-LE-NEXT:    mov sp, r4
+; CHECK-LE-NEXT:    uxth r0, r0
+; CHECK-LE-NEXT:    sub.w r4, r7, #8
+; CHECK-LE-NEXT:    vmov.i32 q1, #0x0
+; CHECK-LE-NEXT:    vmsr p0, r0
+; CHECK-LE-NEXT:    vpsel q0, q0, q1
+; CHECK-LE-NEXT:    mov sp, r4
+; CHECK-LE-NEXT:    pop {r4, r6, r7, pc}
+;
+; CHECK-BE-LABEL: bitcast_to_v16i1:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    .save {r4, r6, r7, lr}
+; CHECK-BE-NEXT:    push {r4, r6, r7, lr}
+; CHECK-BE-NEXT:    .setfp r7, sp, #8
+; CHECK-BE-NEXT:    add r7, sp, #8
+; CHECK-BE-NEXT:    .pad #16
+; CHECK-BE-NEXT:    sub sp, #16
+; CHECK-BE-NEXT:    mov r4, sp
+; CHECK-BE-NEXT:    bfc r4, #0, #4
+; CHECK-BE-NEXT:    mov sp, r4
+; CHECK-BE-NEXT:    vrev64.8 q1, q0
+; CHECK-BE-NEXT:    vmov.i32 q0, #0x0
+; CHECK-BE-NEXT:    uxth r0, r0
+; CHECK-BE-NEXT:    sub.w r4, r7, #8
+; CHECK-BE-NEXT:    vrev32.8 q0, q0
+; CHECK-BE-NEXT:    vmsr p0, r0
+; CHECK-BE-NEXT:    vpsel q1, q1, q0
+; CHECK-BE-NEXT:    vrev64.8 q0, q1
+; CHECK-BE-NEXT:    mov sp, r4
+; CHECK-BE-NEXT:    pop {r4, r6, r7, pc}
+entry:
+  %c = bitcast i16 %b to <16 x i1>
+  %s = select <16 x i1> %c, <16 x i8> %a, <16 x i8> zeroinitializer
+  ret <16 x i8> %s
+}
+
+define arm_aapcs_vfpcc <2 x i64> @bitcast_to_v2i1(i2 %b, <2 x i64> %a) {
+; CHECK-LE-LABEL: bitcast_to_v2i1:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    .pad #4
+; CHECK-LE-NEXT:    sub sp, #4
+; CHECK-LE-NEXT:    and r0, r0, #3
+; CHECK-LE-NEXT:    sbfx r1, r0, #0, #1
+; CHECK-LE-NEXT:    sbfx r0, r0, #1, #1
+; CHECK-LE-NEXT:    vmov.32 q1[0], r1
+; CHECK-LE-NEXT:    vmov.32 q1[1], r1
+; CHECK-LE-NEXT:    vmov.32 q1[2], r0
+; CHECK-LE-NEXT:    vmov.32 q1[3], r0
+; CHECK-LE-NEXT:    vand q0, q0, q1
+; CHECK-LE-NEXT:    add sp, #4
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: bitcast_to_v2i1:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    .pad #4
+; CHECK-BE-NEXT:    sub sp, #4
+; CHECK-BE-NEXT:    and r0, r0, #3
+; CHECK-BE-NEXT:    sbfx r1, r0, #0, #1
+; CHECK-BE-NEXT:    sbfx r0, r0, #1, #1
+; CHECK-BE-NEXT:    vmov.32 q1[0], r1
+; CHECK-BE-NEXT:    vmov.32 q1[1], r1
+; CHECK-BE-NEXT:    vmov.32 q1[2], r0
+; CHECK-BE-NEXT:    vmov.32 q1[3], r0
+; CHECK-BE-NEXT:    vrev64.32 q2, q1
+; CHECK-BE-NEXT:    vand q0, q0, q2
+; CHECK-BE-NEXT:    add sp, #4
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = bitcast i2 %b to <2 x i1>
+  %s = select <2 x i1> %c, <2 x i64> %a, <2 x i64> zeroinitializer
+  ret <2 x i64> %s
+}
+
+
+define arm_aapcs_vfpcc i4 @bitcast_from_v4i1(<4 x i32> %a) {
+; CHECK-LE-LABEL: bitcast_from_v4i1:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    .pad #4
+; CHECK-LE-NEXT:    sub sp, #4
+; CHECK-LE-NEXT:    vcmp.i32 eq, q0, zr
+; CHECK-LE-NEXT:    vmrs r1, p0
+; CHECK-LE-NEXT:    and r0, r1, #1
+; CHECK-LE-NEXT:    rsbs r2, r0, #0
+; CHECK-LE-NEXT:    movs r0, #0
+; CHECK-LE-NEXT:    bfi r0, r2, #0, #1
+; CHECK-LE-NEXT:    ubfx r2, r1, #4, #1
+; CHECK-LE-NEXT:    rsbs r2, r2, #0
+; CHECK-LE-NEXT:    bfi r0, r2, #1, #1
+; CHECK-LE-NEXT:    ubfx r2, r1, #8, #1
+; CHECK-LE-NEXT:    ubfx r1, r1, #12, #1
+; CHECK-LE-NEXT:    rsbs r2, r2, #0
+; CHECK-LE-NEXT:    bfi r0, r2, #2, #1
+; CHECK-LE-NEXT:    rsbs r1, r1, #0
+; CHECK-LE-NEXT:    bfi r0, r1, #3, #1
+; CHECK-LE-NEXT:    add sp, #4
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: bitcast_from_v4i1:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    .pad #4
+; CHECK-BE-NEXT:    sub sp, #4
+; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    vcmp.i32 eq, q1, zr
+; CHECK-BE-NEXT:    vmrs r1, p0
+; CHECK-BE-NEXT:    and r0, r1, #1
+; CHECK-BE-NEXT:    rsbs r2, r0, #0
+; CHECK-BE-NEXT:    movs r0, #0
+; CHECK-BE-NEXT:    bfi r0, r2, #0, #1
+; CHECK-BE-NEXT:    ubfx r2, r1, #4, #1
+; CHECK-BE-NEXT:    rsbs r2, r2, #0
+; CHECK-BE-NEXT:    bfi r0, r2, #1, #1
+; CHECK-BE-NEXT:    ubfx r2, r1, #8, #1
+; CHECK-BE-NEXT:    ubfx r1, r1, #12, #1
+; CHECK-BE-NEXT:    rsbs r2, r2, #0
+; CHECK-BE-NEXT:    bfi r0, r2, #2, #1
+; CHECK-BE-NEXT:    rsbs r1, r1, #0
+; CHECK-BE-NEXT:    bfi r0, r1, #3, #1
+; CHECK-BE-NEXT:    add sp, #4
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = icmp eq <4 x i32> %a, zeroinitializer
+  %b = bitcast <4 x i1> %c to i4
+  ret i4 %b
+}
+
+define arm_aapcs_vfpcc i8 @bitcast_from_v8i1(<8 x i16> %a) {
+; CHECK-LE-LABEL: bitcast_from_v8i1:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    .pad #8
+; CHECK-LE-NEXT:    sub sp, #8
+; CHECK-LE-NEXT:    vcmp.i16 eq, q0, zr
+; CHECK-LE-NEXT:    vmrs r1, p0
+; CHECK-LE-NEXT:    and r0, r1, #1
+; CHECK-LE-NEXT:    rsbs r2, r0, #0
+; CHECK-LE-NEXT:    movs r0, #0
+; CHECK-LE-NEXT:    bfi r0, r2, #0, #1
+; CHECK-LE-NEXT:    ubfx r2, r1, #2, #1
+; CHECK-LE-NEXT:    rsbs r2, r2, #0
+; CHECK-LE-NEXT:    bfi r0, r2, #1, #1
+; CHECK-LE-NEXT:    ubfx r2, r1, #4, #1
+; CHECK-LE-NEXT:    rsbs r2, r2, #0
+; CHECK-LE-NEXT:    bfi r0, r2, #2, #1
+; CHECK-LE-NEXT:    ubfx r2, r1, #6, #1
+; CHECK-LE-NEXT:    rsbs r2, r2, #0
+; CHECK-LE-NEXT:    bfi r0, r2, #3, #1
+; CHECK-LE-NEXT:    ubfx r2, r1, #8, #1
+; CHECK-LE-NEXT:    rsbs r2, r2, #0
+; CHECK-LE-NEXT:    bfi r0, r2, #4, #1
+; CHECK-LE-NEXT:    ubfx r2, r1, #10, #1
+; CHECK-LE-NEXT:    rsbs r2, r2, #0
+; CHECK-LE-NEXT:    bfi r0, r2, #5, #1
+; CHECK-LE-NEXT:    ubfx r2, r1, #12, #1
+; CHECK-LE-NEXT:    ubfx r1, r1, #14, #1
+; CHECK-LE-NEXT:    rsbs r2, r2, #0
+; CHECK-LE-NEXT:    bfi r0, r2, #6, #1
+; CHECK-LE-NEXT:    rsbs r1, r1, #0
+; CHECK-LE-NEXT:    bfi r0, r1, #7, #1
+; CHECK-LE-NEXT:    uxtb r0, r0
+; CHECK-LE-NEXT:    add sp, #8
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: bitcast_from_v8i1:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    .pad #8
+; CHECK-BE-NEXT:    sub sp, #8
+; CHECK-BE-NEXT:    vrev64.16 q1, q0
+; CHECK-BE-NEXT:    vcmp.i16 eq, q1, zr
+; CHECK-BE-NEXT:    vmrs r1, p0
+; CHECK-BE-NEXT:    and r0, r1, #1
+; CHECK-BE-NEXT:    rsbs r2, r0, #0
+; CHECK-BE-NEXT:    movs r0, #0
+; CHECK-BE-NEXT:    bfi r0, r2, #0, #1
+; CHECK-BE-NEXT:    ubfx r2, r1, #2, #1
+; CHECK-BE-NEXT:    rsbs r2, r2, #0
+; CHECK-BE-NEXT:    bfi r0, r2, #1, #1
+; CHECK-BE-NEXT:    ubfx r2, r1, #4, #1
+; CHECK-BE-NEXT:    rsbs r2, r2, #0
+; CHECK-BE-NEXT:    bfi r0, r2, #2, #1
+; CHECK-BE-NEXT:    ubfx r2, r1, #6, #1
+; CHECK-BE-NEXT:    rsbs r2, r2, #0
+; CHECK-BE-NEXT:    bfi r0, r2, #3, #1
+; CHECK-BE-NEXT:    ubfx r2, r1, #8, #1
+; CHECK-BE-NEXT:    rsbs r2, r2, #0
+; CHECK-BE-NEXT:    bfi r0, r2, #4, #1
+; CHECK-BE-NEXT:    ubfx r2, r1, #10, #1
+; CHECK-BE-NEXT:    rsbs r2, r2, #0
+; CHECK-BE-NEXT:    bfi r0, r2, #5, #1
+; CHECK-BE-NEXT:    ubfx r2, r1, #12, #1
+; CHECK-BE-NEXT:    ubfx r1, r1, #14, #1
+; CHECK-BE-NEXT:    rsbs r2, r2, #0
+; CHECK-BE-NEXT:    bfi r0, r2, #6, #1
+; CHECK-BE-NEXT:    rsbs r1, r1, #0
+; CHECK-BE-NEXT:    bfi r0, r1, #7, #1
+; CHECK-BE-NEXT:    uxtb r0, r0
+; CHECK-BE-NEXT:    add sp, #8
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = icmp eq <8 x i16> %a, zeroinitializer
+  %b = bitcast <8 x i1> %c to i8
+  ret i8 %b
+}
+
+define arm_aapcs_vfpcc i16 @bitcast_from_v16i1(<16 x i8> %a) {
+; CHECK-LE-LABEL: bitcast_from_v16i1:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    .save {r4, r6, r7, lr}
+; CHECK-LE-NEXT:    push {r4, r6, r7, lr}
+; CHECK-LE-NEXT:    .setfp r7, sp, #8
+; CHECK-LE-NEXT:    add r7, sp, #8
+; CHECK-LE-NEXT:    .pad #16
+; CHECK-LE-NEXT:    sub sp, #16
+; CHECK-LE-NEXT:    mov r4, sp
+; CHECK-LE-NEXT:    bfc r4, #0, #4
+; CHECK-LE-NEXT:    mov sp, r4
+; CHECK-LE-NEXT:    vcmp.i8 eq, q0, zr
+; CHECK-LE-NEXT:    sub.w r4, r7, #8
+; CHECK-LE-NEXT:    vmrs r0, p0
+; CHECK-LE-NEXT:    uxth r0, r0
+; CHECK-LE-NEXT:    mov sp, r4
+; CHECK-LE-NEXT:    pop {r4, r6, r7, pc}
+;
+; CHECK-BE-LABEL: bitcast_from_v16i1:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    .save {r4, r6, r7, lr}
+; CHECK-BE-NEXT:    push {r4, r6, r7, lr}
+; CHECK-BE-NEXT:    .setfp r7, sp, #8
+; CHECK-BE-NEXT:    add r7, sp, #8
+; CHECK-BE-NEXT:    .pad #16
+; CHECK-BE-NEXT:    sub sp, #16
+; CHECK-BE-NEXT:    mov r4, sp
+; CHECK-BE-NEXT:    bfc r4, #0, #4
+; CHECK-BE-NEXT:    mov sp, r4
+; CHECK-BE-NEXT:    vrev64.8 q1, q0
+; CHECK-BE-NEXT:    sub.w r4, r7, #8
+; CHECK-BE-NEXT:    vcmp.i8 eq, q1, zr
+; CHECK-BE-NEXT:    vmrs r0, p0
+; CHECK-BE-NEXT:    uxth r0, r0
+; CHECK-BE-NEXT:    mov sp, r4
+; CHECK-BE-NEXT:    pop {r4, r6, r7, pc}
+entry:
+  %c = icmp eq <16 x i8> %a, zeroinitializer
+  %b = bitcast <16 x i1> %c to i16
+  ret i16 %b
+}
+
+define arm_aapcs_vfpcc i2 @bitcast_from_v2i1(<2 x i64> %a) {
+; CHECK-LE-LABEL: bitcast_from_v2i1:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    .pad #4
+; CHECK-LE-NEXT:    sub sp, #4
+; CHECK-LE-NEXT:    vmov r0, s1
+; CHECK-LE-NEXT:    vmov r1, s0
+; CHECK-LE-NEXT:    vmov r2, s2
+; CHECK-LE-NEXT:    orrs r0, r1
+; CHECK-LE-NEXT:    vmov r1, s3
+; CHECK-LE-NEXT:    cset r0, eq
+; CHECK-LE-NEXT:    orrs r1, r2
+; CHECK-LE-NEXT:    cset r1, eq
+; CHECK-LE-NEXT:    ands r1, r1, #1
+; CHECK-LE-NEXT:    it ne
+; CHECK-LE-NEXT:    mvnne r1, #1
+; CHECK-LE-NEXT:    bfi r1, r0, #0, #1
+; CHECK-LE-NEXT:    and r0, r1, #3
+; CHECK-LE-NEXT:    add sp, #4
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: bitcast_from_v2i1:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    .pad #4
+; CHECK-BE-NEXT:    sub sp, #4
+; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    vmov r0, s6
+; CHECK-BE-NEXT:    vmov r1, s7
+; CHECK-BE-NEXT:    vmov r2, s5
+; CHECK-BE-NEXT:    orrs r0, r1
+; CHECK-BE-NEXT:    vmov r1, s4
+; CHECK-BE-NEXT:    cset r0, eq
+; CHECK-BE-NEXT:    orrs r1, r2
+; CHECK-BE-NEXT:    cset r1, eq
+; CHECK-BE-NEXT:    ands r1, r1, #1
+; CHECK-BE-NEXT:    it ne
+; CHECK-BE-NEXT:    mvnne r1, #1
+; CHECK-BE-NEXT:    bfi r1, r0, #0, #1
+; CHECK-BE-NEXT:    and r0, r1, #3
+; CHECK-BE-NEXT:    add sp, #4
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = icmp eq <2 x i64> %a, zeroinitializer
+  %b = bitcast <2 x i1> %c to i2
+  ret i2 %b
+}
diff --git a/llvm/test/CodeGen/Thumb2/mve-pred-build-const.ll b/llvm/test/CodeGen/Thumb2/mve-pred-build-const.ll
new file mode 100644
index 000000000..959b162ad
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-pred-build-const.ll
@@ -0,0 +1,196 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s
+
+
+define arm_aapcs_vfpcc <4 x i32> @build_true_v4i1(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: build_true_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    bx lr
+entry:
+  %s = select <4 x i1> <i1 1, i1 1, i1 1, i1 1>, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @build_false_v4i1(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: build_false_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %s = select <4 x i1> <i1 0, i1 0, i1 0, i1 0>, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @build_upper_v4i1(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: build_upper_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    mov.w r0, #65280
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %s = select <4 x i1> <i1 0, i1 0, i1 1, i1 1>, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @build_lower_v4i1(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: build_lower_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    movs r0, #255
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %s = select <4 x i1> <i1 1, i1 1, i1 0, i1 0>, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+
+define arm_aapcs_vfpcc <8 x i16> @build_true_v8i1(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: build_true_v8i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    bx lr
+entry:
+  %s = select <8 x i1> <i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1>, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+define arm_aapcs_vfpcc <8 x i16> @build_false_v8i1(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: build_false_v8i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %s = select <8 x i1> <i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0>, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+define arm_aapcs_vfpcc <8 x i16> @build_upper_v8i1(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: build_upper_v8i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    mov.w r0, #65280
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %s = select <8 x i1> <i1 0, i1 0, i1 0, i1 0, i1 1, i1 1, i1 1, i1 1>, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+define arm_aapcs_vfpcc <8 x i16> @build_lower_v8i1(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: build_lower_v8i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    movs r0, #255
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %s = select <8 x i1> <i1 1, i1 1, i1 1, i1 1, i1 0, i1 0, i1 0, i1 0>, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+
+define arm_aapcs_vfpcc <16 x i8> @build_true_v16i1(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: build_true_v16i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    bx lr
+entry:
+  %s = select <16 x i1> <i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1>, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %s
+}
+
+define arm_aapcs_vfpcc <16 x i8> @build_false_v16i1(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: build_false_v16i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %s = select <16 x i1> <i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0>, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %s
+}
+
+define arm_aapcs_vfpcc <16 x i8> @build_upper_v16i1(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: build_upper_v16i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    mov.w r0, #65280
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %s = select <16 x i1> <i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1>, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %s
+}
+
+define arm_aapcs_vfpcc <16 x i8> @build_lower_v16i1(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: build_lower_v16i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    movs r0, #255
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %s = select <16 x i1> <i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0>, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %s
+}
+
+
+define arm_aapcs_vfpcc <2 x i64> @build_true_v2i1(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: build_true_v2i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    bx lr
+entry:
+  %s = select <2 x i1> <i1 1, i1 1>, <2 x i64> %a, <2 x i64> %b
+  ret <2 x i64> %s
+}
+
+define arm_aapcs_vfpcc <2 x i64> @build_false_v2i1(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: build_false_v2i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %s = select <2 x i1> <i1 0, i1 0>, <2 x i64> %a, <2 x i64> %b
+  ret <2 x i64> %s
+}
+
+define arm_aapcs_vfpcc <2 x i64> @build_upper_v2i1(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: build_upper_v2i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    adr r0, .LCPI14_0
+; CHECK-NEXT:    vldrw.u32 q2, [r0]
+; CHECK-NEXT:    vbic q1, q1, q2
+; CHECK-NEXT:    vand q0, q0, q2
+; CHECK-NEXT:    vorr q0, q0, q1
+; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI14_0:
+; CHECK-NEXT:    .long 0 @ 0x0
+; CHECK-NEXT:    .long 0 @ 0x0
+; CHECK-NEXT:    .long 4294967295 @ 0xffffffff
+; CHECK-NEXT:    .long 4294967295 @ 0xffffffff
+entry:
+  %s = select <2 x i1> <i1 0, i1 1>, <2 x i64> %a, <2 x i64> %b
+  ret <2 x i64> %s
+}
+
+define arm_aapcs_vfpcc <2 x i64> @build_lower_v2i1(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: build_lower_v2i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    adr r0, .LCPI15_0
+; CHECK-NEXT:    vldrw.u32 q2, [r0]
+; CHECK-NEXT:    vbic q1, q1, q2
+; CHECK-NEXT:    vand q0, q0, q2
+; CHECK-NEXT:    vorr q0, q0, q1
+; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI15_0:
+; CHECK-NEXT:    .long 4294967295 @ 0xffffffff
+; CHECK-NEXT:    .long 4294967295 @ 0xffffffff
+; CHECK-NEXT:    .long 0 @ 0x0
+; CHECK-NEXT:    .long 0 @ 0x0
+entry:
+  %s = select <2 x i1> <i1 1, i1 0>, <2 x i64> %a, <2 x i64> %b
+  ret <2 x i64> %s
+}
diff --git a/llvm/test/CodeGen/Thumb2/mve-pred-build-var.ll b/llvm/test/CodeGen/Thumb2/mve-pred-build-var.ll
new file mode 100644
index 000000000..46b418daf
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-pred-build-var.ll
@@ -0,0 +1,246 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s
+
+
+define arm_aapcs_vfpcc <4 x i32> @build_var0_v4i1(i32 %s, i32 %t, <4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: build_var0_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    cmp r0, r1
+; CHECK-NEXT:    mov.w r1, #0
+; CHECK-NEXT:    cset r0, lo
+; CHECK-NEXT:    and r0, r0, #1
+; CHECK-NEXT:    rsbs r0, r0, #0
+; CHECK-NEXT:    bfi r1, r0, #0, #4
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp ult i32 %s, %t
+  %vc = insertelement <4 x i1> zeroinitializer, i1 %c, i64 0
+  %r = select <4 x i1> %vc, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %r
+}
+
+define arm_aapcs_vfpcc <4 x i32> @build_var3_v4i1(i32 %s, i32 %t, <4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: build_var3_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    cmp r0, r1
+; CHECK-NEXT:    mov.w r1, #0
+; CHECK-NEXT:    cset r0, lo
+; CHECK-NEXT:    and r0, r0, #1
+; CHECK-NEXT:    rsbs r0, r0, #0
+; CHECK-NEXT:    bfi r1, r0, #12, #4
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp ult i32 %s, %t
+  %vc = insertelement <4 x i1> zeroinitializer, i1 %c, i64 3
+  %r = select <4 x i1> %vc, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %r
+}
+
+define arm_aapcs_vfpcc <4 x i32> @build_varN_v4i1(i32 %s, i32 %t, <4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: build_varN_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    cmp r0, r1
+; CHECK-NEXT:    cset r0, lo
+; CHECK-NEXT:    and r0, r0, #1
+; CHECK-NEXT:    rsbs r0, r0, #0
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp ult i32 %s, %t
+  %vc1 = insertelement <4 x i1> undef, i1 %c, i64 0
+  %vc4 = shufflevector <4 x i1> %vc1, <4 x i1> undef, <4 x i32> zeroinitializer
+  %r = select <4 x i1> %vc4, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %r
+}
+
+
+define arm_aapcs_vfpcc <8 x i16> @build_var0_v8i1(i32 %s, i32 %t, <8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: build_var0_v8i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    cmp r0, r1
+; CHECK-NEXT:    mov.w r1, #0
+; CHECK-NEXT:    cset r0, lo
+; CHECK-NEXT:    and r0, r0, #1
+; CHECK-NEXT:    rsbs r0, r0, #0
+; CHECK-NEXT:    bfi r1, r0, #0, #2
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp ult i32 %s, %t
+  %vc = insertelement <8 x i1> zeroinitializer, i1 %c, i64 0
+  %r = select <8 x i1> %vc, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %r
+}
+
+define arm_aapcs_vfpcc <8 x i16> @build_var3_v8i1(i32 %s, i32 %t, <8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: build_var3_v8i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    cmp r0, r1
+; CHECK-NEXT:    mov.w r1, #0
+; CHECK-NEXT:    cset r0, lo
+; CHECK-NEXT:    and r0, r0, #1
+; CHECK-NEXT:    rsbs r0, r0, #0
+; CHECK-NEXT:    bfi r1, r0, #6, #2
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp ult i32 %s, %t
+  %vc = insertelement <8 x i1> zeroinitializer, i1 %c, i64 3
+  %r = select <8 x i1> %vc, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %r
+}
+
+define arm_aapcs_vfpcc <8 x i16> @build_varN_v8i1(i32 %s, i32 %t, <8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: build_varN_v8i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    cmp r0, r1
+; CHECK-NEXT:    cset r0, lo
+; CHECK-NEXT:    and r0, r0, #1
+; CHECK-NEXT:    rsbs r0, r0, #0
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp ult i32 %s, %t
+  %vc1 = insertelement <8 x i1> undef, i1 %c, i64 0
+  %vc4 = shufflevector <8 x i1> %vc1, <8 x i1> undef, <8 x i32> zeroinitializer
+  %r = select <8 x i1> %vc4, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %r
+}
+
+
+define arm_aapcs_vfpcc <16 x i8> @build_var0_v16i1(i32 %s, i32 %t, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: build_var0_v16i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    cmp r0, r1
+; CHECK-NEXT:    mov.w r1, #0
+; CHECK-NEXT:    cset r0, lo
+; CHECK-NEXT:    and r0, r0, #1
+; CHECK-NEXT:    rsbs r0, r0, #0
+; CHECK-NEXT:    bfi r1, r0, #0, #1
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp ult i32 %s, %t
+  %vc = insertelement <16 x i1> zeroinitializer, i1 %c, i64 0
+  %r = select <16 x i1> %vc, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %r
+}
+
+define arm_aapcs_vfpcc <16 x i8> @build_var3_v16i1(i32 %s, i32 %t, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: build_var3_v16i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    cmp r0, r1
+; CHECK-NEXT:    mov.w r1, #0
+; CHECK-NEXT:    cset r0, lo
+; CHECK-NEXT:    and r0, r0, #1
+; CHECK-NEXT:    rsbs r0, r0, #0
+; CHECK-NEXT:    bfi r1, r0, #3, #1
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp ult i32 %s, %t
+  %vc = insertelement <16 x i1> zeroinitializer, i1 %c, i64 3
+  %r = select <16 x i1> %vc, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %r
+}
+
+define arm_aapcs_vfpcc <16 x i8> @build_varN_v16i1(i32 %s, i32 %t, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: build_varN_v16i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    cmp r0, r1
+; CHECK-NEXT:    cset r0, lo
+; CHECK-NEXT:    and r0, r0, #1
+; CHECK-NEXT:    rsbs r0, r0, #0
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp ult i32 %s, %t
+  %vc1 = insertelement <16 x i1> undef, i1 %c, i64 0
+  %vc4 = shufflevector <16 x i1> %vc1, <16 x i1> undef, <16 x i32> zeroinitializer
+  %r = select <16 x i1> %vc4, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %r
+}
+
+
+define arm_aapcs_vfpcc <2 x i64> @build_var0_v2i1(i32 %s, i32 %t, <2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: build_var0_v2i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    cmp r0, r1
+; CHECK-NEXT:    cset r0, lo
+; CHECK-NEXT:    and r0, r0, #1
+; CHECK-NEXT:    rsbs r0, r0, #0
+; CHECK-NEXT:    vmov s8, r0
+; CHECK-NEXT:    vldr s10, .LCPI9_0
+; CHECK-NEXT:    vmov.f32 s9, s8
+; CHECK-NEXT:    vmov.f32 s11, s10
+; CHECK-NEXT:    vbic q1, q1, q2
+; CHECK-NEXT:    vand q0, q0, q2
+; CHECK-NEXT:    vorr q0, q0, q1
+; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    .p2align 2
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI9_0:
+; CHECK-NEXT:    .long 0 @ float 0
+entry:
+  %c = icmp ult i32 %s, %t
+  %vc = insertelement <2 x i1> zeroinitializer, i1 %c, i64 0
+  %r = select <2 x i1> %vc, <2 x i64> %a, <2 x i64> %b
+  ret <2 x i64> %r
+}
+
+define arm_aapcs_vfpcc <2 x i64> @build_var1_v2i1(i32 %s, i32 %t, <2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: build_var1_v2i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    cmp r0, r1
+; CHECK-NEXT:    cset r0, lo
+; CHECK-NEXT:    and r0, r0, #1
+; CHECK-NEXT:    rsbs r0, r0, #0
+; CHECK-NEXT:    vmov s10, r0
+; CHECK-NEXT:    vldr s8, .LCPI10_0
+; CHECK-NEXT:    vmov.f32 s9, s8
+; CHECK-NEXT:    vmov.f32 s11, s10
+; CHECK-NEXT:    vbic q1, q1, q2
+; CHECK-NEXT:    vand q0, q0, q2
+; CHECK-NEXT:    vorr q0, q0, q1
+; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    .p2align 2
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI10_0:
+; CHECK-NEXT:    .long 0 @ float 0
+entry:
+  %c = icmp ult i32 %s, %t
+  %vc = insertelement <2 x i1> zeroinitializer, i1 %c, i64 1
+  %r = select <2 x i1> %vc, <2 x i64> %a, <2 x i64> %b
+  ret <2 x i64> %r
+}
+
+define arm_aapcs_vfpcc <2 x i64> @build_varN_v2i1(i32 %s, i32 %t, <2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: build_varN_v2i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    cmp r0, r1
+; CHECK-NEXT:    cset r0, lo
+; CHECK-NEXT:    and r0, r0, #1
+; CHECK-NEXT:    rsbs r0, r0, #0
+; CHECK-NEXT:    vdup.32 q2, r0
+; CHECK-NEXT:    vbic q1, q1, q2
+; CHECK-NEXT:    vand q0, q0, q2
+; CHECK-NEXT:    vorr q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp ult i32 %s, %t
+  %vc1 = insertelement <2 x i1> undef, i1 %c, i64 0
+  %vc4 = shufflevector <2 x i1> %vc1, <2 x i1> undef, <2 x i32> zeroinitializer
+  %r = select <2 x i1> %vc4, <2 x i64> %a, <2 x i64> %b
+  ret <2 x i64> %r
+}
diff --git a/llvm/test/CodeGen/Thumb2/mve-pred-ext.ll b/llvm/test/CodeGen/Thumb2/mve-pred-ext.ll
new file mode 100644
index 000000000..99bd003c8
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-pred-ext.ll
@@ -0,0 +1,335 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve.fp -verify-machineinstrs %s -o - | FileCheck %s
+
+define arm_aapcs_vfpcc <4 x i32> @sext_v4i1_v4i32(<4 x i32> %src) {
+; CHECK-LABEL: sext_v4i1_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.i32 q1, #0x0
+; CHECK-NEXT:    vmov.i8 q2, #0xff
+; CHECK-NEXT:    vcmp.s32 gt, q0, zr
+; CHECK-NEXT:    vpsel q0, q2, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp sgt <4 x i32> %src, zeroinitializer
+  %0 = sext <4 x i1> %c to <4 x i32>
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @sext_v8i1_v8i16(<8 x i16> %src) {
+; CHECK-LABEL: sext_v8i1_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.i16 q1, #0x0
+; CHECK-NEXT:    vmov.i8 q2, #0xff
+; CHECK-NEXT:    vcmp.s16 gt, q0, zr
+; CHECK-NEXT:    vpsel q0, q2, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp sgt <8 x i16> %src, zeroinitializer
+  %0 = sext <8 x i1> %c to <8 x i16>
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @sext_v16i1_v16i8(<16 x i8> %src) {
+; CHECK-LABEL: sext_v16i1_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.i8 q1, #0x0
+; CHECK-NEXT:    vmov.i8 q2, #0xff
+; CHECK-NEXT:    vcmp.s8 gt, q0, zr
+; CHECK-NEXT:    vpsel q0, q2, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp sgt <16 x i8> %src, zeroinitializer
+  %0 = sext <16 x i1> %c to <16 x i8>
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <2 x i64> @sext_v2i1_v2i64(<2 x i64> %src) {
+; CHECK-LABEL: sext_v2i1_v2i64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov r1, s2
+; CHECK-NEXT:    movs r2, #0
+; CHECK-NEXT:    vmov r0, s3
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    rsbs r1, r1, #0
+; CHECK-NEXT:    vmov r1, s1
+; CHECK-NEXT:    sbcs.w r0, r2, r0
+; CHECK-NEXT:    mov.w r0, #0
+; CHECK-NEXT:    it lt
+; CHECK-NEXT:    movlt r0, #1
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    sbcs.w r1, r2, r1
+; CHECK-NEXT:    it lt
+; CHECK-NEXT:    movlt r2, #1
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    csetm r1, ne
+; CHECK-NEXT:    vmov.32 q0[0], r1
+; CHECK-NEXT:    vmov.32 q0[1], r1
+; CHECK-NEXT:    vmov.32 q0[2], r0
+; CHECK-NEXT:    vmov.32 q0[3], r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp sgt <2 x i64> %src, zeroinitializer
+  %0 = sext <2 x i1> %c to <2 x i64>
+  ret <2 x i64> %0
+}
+
+
+define arm_aapcs_vfpcc <4 x i32> @zext_v4i1_v4i32(<4 x i32> %src) {
+; CHECK-LABEL: zext_v4i1_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.i32 q1, #0x0
+; CHECK-NEXT:    vmov.i32 q2, #0x1
+; CHECK-NEXT:    vcmp.s32 gt, q0, zr
+; CHECK-NEXT:    vpsel q0, q2, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp sgt <4 x i32> %src, zeroinitializer
+  %0 = zext <4 x i1> %c to <4 x i32>
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @zext_v8i1_v8i16(<8 x i16> %src) {
+; CHECK-LABEL: zext_v8i1_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.i16 q1, #0x0
+; CHECK-NEXT:    vmov.i16 q2, #0x1
+; CHECK-NEXT:    vcmp.s16 gt, q0, zr
+; CHECK-NEXT:    vpsel q0, q2, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp sgt <8 x i16> %src, zeroinitializer
+  %0 = zext <8 x i1> %c to <8 x i16>
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @zext_v16i1_v16i8(<16 x i8> %src) {
+; CHECK-LABEL: zext_v16i1_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.i8 q1, #0x0
+; CHECK-NEXT:    vmov.i8 q2, #0x1
+; CHECK-NEXT:    vcmp.s8 gt, q0, zr
+; CHECK-NEXT:    vpsel q0, q2, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp sgt <16 x i8> %src, zeroinitializer
+  %0 = zext <16 x i1> %c to <16 x i8>
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <2 x i64> @zext_v2i1_v2i64(<2 x i64> %src) {
+; CHECK-LABEL: zext_v2i1_v2i64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov r2, s2
+; CHECK-NEXT:    adr r1, .LCPI7_0
+; CHECK-NEXT:    vldrw.u32 q1, [r1]
+; CHECK-NEXT:    vmov r1, s3
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    movs r0, #0
+; CHECK-NEXT:    rsbs r2, r2, #0
+; CHECK-NEXT:    vmov r2, s1
+; CHECK-NEXT:    sbcs.w r1, r0, r1
+; CHECK-NEXT:    mov.w r1, #0
+; CHECK-NEXT:    it lt
+; CHECK-NEXT:    movlt r1, #1
+; CHECK-NEXT:    cmp r1, #0
+; CHECK-NEXT:    csetm r1, ne
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    sbcs.w r2, r0, r2
+; CHECK-NEXT:    it lt
+; CHECK-NEXT:    movlt r0, #1
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q0[0], r0
+; CHECK-NEXT:    vmov.32 q0[2], r1
+; CHECK-NEXT:    vand q0, q0, q1
+; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI7_0:
+; CHECK-NEXT:    .long 1 @ 0x1
+; CHECK-NEXT:    .long 0 @ 0x0
+; CHECK-NEXT:    .long 1 @ 0x1
+; CHECK-NEXT:    .long 0 @ 0x0
+entry:
+  %c = icmp sgt <2 x i64> %src, zeroinitializer
+  %0 = zext <2 x i1> %c to <2 x i64>
+  ret <2 x i64> %0
+}
+
+
+define arm_aapcs_vfpcc <4 x i32> @trunc_v4i1_v4i32(<4 x i32> %src) {
+; CHECK-LABEL: trunc_v4i1_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.i32 q1, #0x0
+; CHECK-NEXT:    vcmp.i32 ne, q0, zr
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = trunc <4 x i32> %src to <4 x i1>
+  %1 = select <4 x i1> %0, <4 x i32> %src, <4 x i32> zeroinitializer
+  ret <4 x i32> %1
+}
+
+define arm_aapcs_vfpcc <8 x i16> @trunc_v8i1_v8i16(<8 x i16> %src) {
+; CHECK-LABEL: trunc_v8i1_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.i32 q1, #0x0
+; CHECK-NEXT:    vcmp.i32 ne, q0, zr
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = trunc <8 x i16> %src to <8 x i1>
+  %1 = select <8 x i1> %0, <8 x i16> %src, <8 x i16> zeroinitializer
+  ret <8 x i16> %1
+}
+
+define arm_aapcs_vfpcc <16 x i8> @trunc_v16i1_v16i8(<16 x i8> %src) {
+; CHECK-LABEL: trunc_v16i1_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.i32 q1, #0x0
+; CHECK-NEXT:    vcmp.i32 ne, q0, zr
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = trunc <16 x i8> %src to <16 x i1>
+  %1 = select <16 x i1> %0, <16 x i8> %src, <16 x i8> zeroinitializer
+  ret <16 x i8> %1
+}
+
+define arm_aapcs_vfpcc <2 x i64> @trunc_v2i1_v2i64(<2 x i64> %src) {
+; CHECK-LABEL: trunc_v2i1_v2i64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov r1, s0
+; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    and r1, r1, #1
+; CHECK-NEXT:    rsbs r1, r1, #0
+; CHECK-NEXT:    and r0, r0, #1
+; CHECK-NEXT:    vmov.32 q1[0], r1
+; CHECK-NEXT:    rsbs r0, r0, #0
+; CHECK-NEXT:    vmov.32 q1[1], r1
+; CHECK-NEXT:    vmov.32 q1[2], r0
+; CHECK-NEXT:    vmov.32 q1[3], r0
+; CHECK-NEXT:    vand q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = trunc <2 x i64> %src to <2 x i1>
+  %1 = select <2 x i1> %0, <2 x i64> %src, <2 x i64> zeroinitializer
+  ret <2 x i64> %1
+}
+
+
+define arm_aapcs_vfpcc <4 x float> @uitofp_v4i1_v4f32(<4 x i32> %src) {
+; CHECK-LABEL: uitofp_v4i1_v4f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.i32 q1, #0x0
+; CHECK-NEXT:    vmov.f32 q2, #1.000000e+00
+; CHECK-NEXT:    vcmp.s32 gt, q0, zr
+; CHECK-NEXT:    vpsel q0, q2, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp sgt <4 x i32> %src, zeroinitializer
+  %0 = uitofp <4 x i1> %c to <4 x float>
+  ret <4 x float> %0
+}
+
+define arm_aapcs_vfpcc <4 x float> @sitofp_v4i1_v4f32(<4 x i32> %src) {
+; CHECK-LABEL: sitofp_v4i1_v4f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.i32 q1, #0x0
+; CHECK-NEXT:    vmov.f32 q2, #-1.000000e+00
+; CHECK-NEXT:    vcmp.s32 gt, q0, zr
+; CHECK-NEXT:    vpsel q0, q2, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp sgt <4 x i32> %src, zeroinitializer
+  %0 = sitofp <4 x i1> %c to <4 x float>
+  ret <4 x float> %0
+}
+
+define arm_aapcs_vfpcc <4 x float> @fptoui_v4i1_v4f32(<4 x float> %src) {
+; CHECK-LABEL: fptoui_v4i1_v4f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.i32 q1, #0x0
+; CHECK-NEXT:    vmov.f32 q2, #1.000000e+00
+; CHECK-NEXT:    vcmp.f32 ne, q0, zr
+; CHECK-NEXT:    vpsel q0, q2, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = fptoui <4 x float> %src to <4 x i1>
+  %s = select <4 x i1> %0, <4 x float> <float 1.0, float 1.0, float 1.0, float 1.0>, <4 x float> zeroinitializer
+  ret <4 x float> %s
+}
+
+define arm_aapcs_vfpcc <4 x float> @fptosi_v4i1_v4f32(<4 x float> %src) {
+; CHECK-LABEL: fptosi_v4i1_v4f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.i32 q1, #0x0
+; CHECK-NEXT:    vmov.f32 q2, #1.000000e+00
+; CHECK-NEXT:    vcmp.f32 ne, q0, zr
+; CHECK-NEXT:    vpsel q0, q2, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = fptosi <4 x float> %src to <4 x i1>
+  %s = select <4 x i1> %0, <4 x float> <float 1.0, float 1.0, float 1.0, float 1.0>, <4 x float> zeroinitializer
+  ret <4 x float> %s
+}
+
+
+
+define arm_aapcs_vfpcc <8 x half> @uitofp_v8i1_v8f16(<8 x i16> %src) {
+; CHECK-LABEL: uitofp_v8i1_v8f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.i16 q1, #0x0
+; CHECK-NEXT:    vmov.i16 q2, #0x3c00
+; CHECK-NEXT:    vcmp.s16 gt, q0, zr
+; CHECK-NEXT:    vpsel q0, q2, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp sgt <8 x i16> %src, zeroinitializer
+  %0 = uitofp <8 x i1> %c to <8 x half>
+  ret <8 x half> %0
+}
+
+define arm_aapcs_vfpcc <8 x half> @sitofp_v8i1_v8f16(<8 x i16> %src) {
+; CHECK-LABEL: sitofp_v8i1_v8f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.i16 q1, #0x0
+; CHECK-NEXT:    vmov.i16 q2, #0xbc00
+; CHECK-NEXT:    vcmp.s16 gt, q0, zr
+; CHECK-NEXT:    vpsel q0, q2, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp sgt <8 x i16> %src, zeroinitializer
+  %0 = sitofp <8 x i1> %c to <8 x half>
+  ret <8 x half> %0
+}
+
+define arm_aapcs_vfpcc <8 x half> @fptoui_v8i1_v8f16(<8 x half> %src) {
+; CHECK-LABEL: fptoui_v8i1_v8f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.i32 q1, #0x0
+; CHECK-NEXT:    vmov.i16 q2, #0x3c00
+; CHECK-NEXT:    vcmp.f16 ne, q0, zr
+; CHECK-NEXT:    vpsel q0, q2, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = fptoui <8 x half> %src to <8 x i1>
+  %s = select <8 x i1> %0, <8 x half> <half 1.0, half 1.0, half 1.0, half 1.0, half 1.0, half 1.0, half 1.0, half 1.0>, <8 x half> zeroinitializer
+  ret <8 x half> %s
+}
+
+define arm_aapcs_vfpcc <8 x half> @fptosi_v8i1_v8f16(<8 x half> %src) {
+; CHECK-LABEL: fptosi_v8i1_v8f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.i32 q1, #0x0
+; CHECK-NEXT:    vmov.i16 q2, #0x3c00
+; CHECK-NEXT:    vcmp.f16 ne, q0, zr
+; CHECK-NEXT:    vpsel q0, q2, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = fptosi <8 x half> %src to <8 x i1>
+  %s = select <8 x i1> %0, <8 x half> <half 1.0, half 1.0, half 1.0, half 1.0, half 1.0, half 1.0, half 1.0, half 1.0>, <8 x half> zeroinitializer
+  ret <8 x half> %s
+}
diff --git a/llvm/test/CodeGen/Thumb2/mve-pred-loadstore.ll b/llvm/test/CodeGen/Thumb2/mve-pred-loadstore.ll
new file mode 100644
index 000000000..21af26117
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-pred-loadstore.ll
@@ -0,0 +1,355 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-LE
+; RUN: llc -mtriple=thumbebv8.1m.main-arm-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-BE
+
+define arm_aapcs_vfpcc <4 x i32> @load_v4i1(<4 x i1> *%src, <4 x i32> %a) {
+; CHECK-LE-LABEL: load_v4i1:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    ldrb r0, [r0]
+; CHECK-LE-NEXT:    vmov.i8 q1, #0x0
+; CHECK-LE-NEXT:    vmov.i8 q2, #0xff
+; CHECK-LE-NEXT:    vmsr p0, r0
+; CHECK-LE-NEXT:    vpsel q1, q2, q1
+; CHECK-LE-NEXT:    vmov.u8 r0, q1[0]
+; CHECK-LE-NEXT:    vmov.32 q2[0], r0
+; CHECK-LE-NEXT:    vmov.u8 r0, q1[1]
+; CHECK-LE-NEXT:    vmov.32 q2[1], r0
+; CHECK-LE-NEXT:    vmov.u8 r0, q1[2]
+; CHECK-LE-NEXT:    vmov.32 q2[2], r0
+; CHECK-LE-NEXT:    vmov.u8 r0, q1[3]
+; CHECK-LE-NEXT:    vmov.32 q2[3], r0
+; CHECK-LE-NEXT:    vmov.i32 q1, #0x0
+; CHECK-LE-NEXT:    vcmp.i32 ne, q2, zr
+; CHECK-LE-NEXT:    vpsel q0, q0, q1
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: load_v4i1:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    ldrb r0, [r0]
+; CHECK-BE-NEXT:    vmov.i8 q1, #0x0
+; CHECK-BE-NEXT:    vmov.i8 q2, #0xff
+; CHECK-BE-NEXT:    vmsr p0, r0
+; CHECK-BE-NEXT:    vpsel q1, q2, q1
+; CHECK-BE-NEXT:    vmov.u8 r0, q1[0]
+; CHECK-BE-NEXT:    vmov.32 q2[0], r0
+; CHECK-BE-NEXT:    vmov.u8 r0, q1[1]
+; CHECK-BE-NEXT:    vmov.32 q2[1], r0
+; CHECK-BE-NEXT:    vmov.u8 r0, q1[2]
+; CHECK-BE-NEXT:    vmov.32 q2[2], r0
+; CHECK-BE-NEXT:    vmov.u8 r0, q1[3]
+; CHECK-BE-NEXT:    vmov.32 q2[3], r0
+; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    vcmp.i32 ne, q2, zr
+; CHECK-BE-NEXT:    vmov.i32 q0, #0x0
+; CHECK-BE-NEXT:    vpsel q1, q1, q0
+; CHECK-BE-NEXT:    vrev64.32 q0, q1
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = load <4 x i1>, <4 x i1>* %src
+  %s = select <4 x i1> %c, <4 x i32> %a, <4 x i32> zeroinitializer
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <8 x i16> @load_v8i1(<8 x i1> *%src, <8 x i16> %a) {
+; CHECK-LE-LABEL: load_v8i1:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    ldrb r0, [r0]
+; CHECK-LE-NEXT:    vmov.i8 q1, #0x0
+; CHECK-LE-NEXT:    vmov.i8 q2, #0xff
+; CHECK-LE-NEXT:    vmsr p0, r0
+; CHECK-LE-NEXT:    vpsel q2, q2, q1
+; CHECK-LE-NEXT:    vmov.u8 r0, q2[0]
+; CHECK-LE-NEXT:    vmov.16 q1[0], r0
+; CHECK-LE-NEXT:    vmov.u8 r0, q2[1]
+; CHECK-LE-NEXT:    vmov.16 q1[1], r0
+; CHECK-LE-NEXT:    vmov.u8 r0, q2[2]
+; CHECK-LE-NEXT:    vmov.16 q1[2], r0
+; CHECK-LE-NEXT:    vmov.u8 r0, q2[3]
+; CHECK-LE-NEXT:    vmov.16 q1[3], r0
+; CHECK-LE-NEXT:    vmov.u8 r0, q2[4]
+; CHECK-LE-NEXT:    vmov.16 q1[4], r0
+; CHECK-LE-NEXT:    vmov.u8 r0, q2[5]
+; CHECK-LE-NEXT:    vmov.16 q1[5], r0
+; CHECK-LE-NEXT:    vmov.u8 r0, q2[6]
+; CHECK-LE-NEXT:    vmov.16 q1[6], r0
+; CHECK-LE-NEXT:    vmov.u8 r0, q2[7]
+; CHECK-LE-NEXT:    vmov.16 q1[7], r0
+; CHECK-LE-NEXT:    vcmp.i16 ne, q1, zr
+; CHECK-LE-NEXT:    vmov.i32 q1, #0x0
+; CHECK-LE-NEXT:    vpsel q0, q0, q1
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: load_v8i1:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    ldrb r0, [r0]
+; CHECK-BE-NEXT:    vmov.i8 q1, #0x0
+; CHECK-BE-NEXT:    vmov.i8 q2, #0xff
+; CHECK-BE-NEXT:    vmsr p0, r0
+; CHECK-BE-NEXT:    vpsel q2, q2, q1
+; CHECK-BE-NEXT:    vmov.u8 r0, q2[0]
+; CHECK-BE-NEXT:    vmov.16 q1[0], r0
+; CHECK-BE-NEXT:    vmov.u8 r0, q2[1]
+; CHECK-BE-NEXT:    vmov.16 q1[1], r0
+; CHECK-BE-NEXT:    vmov.u8 r0, q2[2]
+; CHECK-BE-NEXT:    vmov.16 q1[2], r0
+; CHECK-BE-NEXT:    vmov.u8 r0, q2[3]
+; CHECK-BE-NEXT:    vmov.16 q1[3], r0
+; CHECK-BE-NEXT:    vmov.u8 r0, q2[4]
+; CHECK-BE-NEXT:    vmov.16 q1[4], r0
+; CHECK-BE-NEXT:    vmov.u8 r0, q2[5]
+; CHECK-BE-NEXT:    vmov.16 q1[5], r0
+; CHECK-BE-NEXT:    vmov.u8 r0, q2[6]
+; CHECK-BE-NEXT:    vmov.16 q1[6], r0
+; CHECK-BE-NEXT:    vmov.u8 r0, q2[7]
+; CHECK-BE-NEXT:    vmov.16 q1[7], r0
+; CHECK-BE-NEXT:    vcmp.i16 ne, q1, zr
+; CHECK-BE-NEXT:    vrev64.16 q1, q0
+; CHECK-BE-NEXT:    vmov.i32 q0, #0x0
+; CHECK-BE-NEXT:    vrev32.16 q0, q0
+; CHECK-BE-NEXT:    vpsel q1, q1, q0
+; CHECK-BE-NEXT:    vrev64.16 q0, q1
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = load <8 x i1>, <8 x i1>* %src
+  %s = select <8 x i1> %c, <8 x i16> %a, <8 x i16> zeroinitializer
+  ret <8 x i16> %s
+}
+
+define arm_aapcs_vfpcc <16 x i8> @load_v16i1(<16 x i1> *%src, <16 x i8> %a) {
+; CHECK-LE-LABEL: load_v16i1:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    ldrh r0, [r0]
+; CHECK-LE-NEXT:    vmov.i32 q1, #0x0
+; CHECK-LE-NEXT:    vmsr p0, r0
+; CHECK-LE-NEXT:    vpsel q0, q0, q1
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: load_v16i1:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    ldrh r0, [r0]
+; CHECK-BE-NEXT:    vrev64.8 q1, q0
+; CHECK-BE-NEXT:    vmov.i32 q0, #0x0
+; CHECK-BE-NEXT:    vrev32.8 q0, q0
+; CHECK-BE-NEXT:    vmsr p0, r0
+; CHECK-BE-NEXT:    vpsel q1, q1, q0
+; CHECK-BE-NEXT:    vrev64.8 q0, q1
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = load <16 x i1>, <16 x i1>* %src
+  %s = select <16 x i1> %c, <16 x i8> %a, <16 x i8> zeroinitializer
+  ret <16 x i8> %s
+}
+
+define arm_aapcs_vfpcc <2 x i64> @load_v2i1(<2 x i1> *%src, <2 x i64> %a) {
+; CHECK-LE-LABEL: load_v2i1:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    ldrb r0, [r0]
+; CHECK-LE-NEXT:    sbfx r1, r0, #0, #1
+; CHECK-LE-NEXT:    sbfx r0, r0, #1, #1
+; CHECK-LE-NEXT:    vmov.32 q1[0], r1
+; CHECK-LE-NEXT:    vmov.32 q1[1], r1
+; CHECK-LE-NEXT:    vmov.32 q1[2], r0
+; CHECK-LE-NEXT:    vmov.32 q1[3], r0
+; CHECK-LE-NEXT:    vand q0, q0, q1
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: load_v2i1:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    ldrb r0, [r0]
+; CHECK-BE-NEXT:    sbfx r1, r0, #0, #1
+; CHECK-BE-NEXT:    sbfx r0, r0, #1, #1
+; CHECK-BE-NEXT:    vmov.32 q1[0], r1
+; CHECK-BE-NEXT:    vmov.32 q1[1], r1
+; CHECK-BE-NEXT:    vmov.32 q1[2], r0
+; CHECK-BE-NEXT:    vmov.32 q1[3], r0
+; CHECK-BE-NEXT:    vrev64.32 q2, q1
+; CHECK-BE-NEXT:    vand q0, q0, q2
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = load <2 x i1>, <2 x i1>* %src
+  %s = select <2 x i1> %c, <2 x i64> %a, <2 x i64> zeroinitializer
+  ret <2 x i64> %s
+}
+
+
+define arm_aapcs_vfpcc void @store_v4i1(<4 x i1> *%dst, <4 x i32> %a) {
+; CHECK-LE-LABEL: store_v4i1:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vcmp.i32 eq, q0, zr
+; CHECK-LE-NEXT:    movs r3, #0
+; CHECK-LE-NEXT:    vmrs r1, p0
+; CHECK-LE-NEXT:    and r2, r1, #1
+; CHECK-LE-NEXT:    rsbs r2, r2, #0
+; CHECK-LE-NEXT:    bfi r3, r2, #0, #1
+; CHECK-LE-NEXT:    ubfx r2, r1, #4, #1
+; CHECK-LE-NEXT:    rsbs r2, r2, #0
+; CHECK-LE-NEXT:    bfi r3, r2, #1, #1
+; CHECK-LE-NEXT:    ubfx r2, r1, #8, #1
+; CHECK-LE-NEXT:    ubfx r1, r1, #12, #1
+; CHECK-LE-NEXT:    rsbs r2, r2, #0
+; CHECK-LE-NEXT:    bfi r3, r2, #2, #1
+; CHECK-LE-NEXT:    rsbs r1, r1, #0
+; CHECK-LE-NEXT:    bfi r3, r1, #3, #1
+; CHECK-LE-NEXT:    strb r3, [r0]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: store_v4i1:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    movs r3, #0
+; CHECK-BE-NEXT:    vcmp.i32 eq, q1, zr
+; CHECK-BE-NEXT:    vmrs r1, p0
+; CHECK-BE-NEXT:    and r2, r1, #1
+; CHECK-BE-NEXT:    rsbs r2, r2, #0
+; CHECK-BE-NEXT:    bfi r3, r2, #0, #1
+; CHECK-BE-NEXT:    ubfx r2, r1, #4, #1
+; CHECK-BE-NEXT:    rsbs r2, r2, #0
+; CHECK-BE-NEXT:    bfi r3, r2, #1, #1
+; CHECK-BE-NEXT:    ubfx r2, r1, #8, #1
+; CHECK-BE-NEXT:    ubfx r1, r1, #12, #1
+; CHECK-BE-NEXT:    rsbs r2, r2, #0
+; CHECK-BE-NEXT:    bfi r3, r2, #2, #1
+; CHECK-BE-NEXT:    rsbs r1, r1, #0
+; CHECK-BE-NEXT:    bfi r3, r1, #3, #1
+; CHECK-BE-NEXT:    strb r3, [r0]
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = icmp eq <4 x i32> %a, zeroinitializer
+  store <4 x i1> %c, <4 x i1>* %dst
+  ret void
+}
+
+define arm_aapcs_vfpcc void @store_v8i1(<8 x i1> *%dst, <8 x i16> %a) {
+; CHECK-LE-LABEL: store_v8i1:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vcmp.i16 eq, q0, zr
+; CHECK-LE-NEXT:    vmrs r2, p0
+; CHECK-LE-NEXT:    and r1, r2, #1
+; CHECK-LE-NEXT:    rsbs r3, r1, #0
+; CHECK-LE-NEXT:    movs r1, #0
+; CHECK-LE-NEXT:    bfi r1, r3, #0, #1
+; CHECK-LE-NEXT:    ubfx r3, r2, #2, #1
+; CHECK-LE-NEXT:    rsbs r3, r3, #0
+; CHECK-LE-NEXT:    bfi r1, r3, #1, #1
+; CHECK-LE-NEXT:    ubfx r3, r2, #4, #1
+; CHECK-LE-NEXT:    rsbs r3, r3, #0
+; CHECK-LE-NEXT:    bfi r1, r3, #2, #1
+; CHECK-LE-NEXT:    ubfx r3, r2, #6, #1
+; CHECK-LE-NEXT:    rsbs r3, r3, #0
+; CHECK-LE-NEXT:    bfi r1, r3, #3, #1
+; CHECK-LE-NEXT:    ubfx r3, r2, #8, #1
+; CHECK-LE-NEXT:    rsbs r3, r3, #0
+; CHECK-LE-NEXT:    bfi r1, r3, #4, #1
+; CHECK-LE-NEXT:    ubfx r3, r2, #10, #1
+; CHECK-LE-NEXT:    rsbs r3, r3, #0
+; CHECK-LE-NEXT:    bfi r1, r3, #5, #1
+; CHECK-LE-NEXT:    ubfx r3, r2, #12, #1
+; CHECK-LE-NEXT:    ubfx r2, r2, #14, #1
+; CHECK-LE-NEXT:    rsbs r3, r3, #0
+; CHECK-LE-NEXT:    bfi r1, r3, #6, #1
+; CHECK-LE-NEXT:    rsbs r2, r2, #0
+; CHECK-LE-NEXT:    bfi r1, r2, #7, #1
+; CHECK-LE-NEXT:    strb r1, [r0]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: store_v8i1:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vrev64.16 q1, q0
+; CHECK-BE-NEXT:    vcmp.i16 eq, q1, zr
+; CHECK-BE-NEXT:    vmrs r2, p0
+; CHECK-BE-NEXT:    and r1, r2, #1
+; CHECK-BE-NEXT:    rsbs r3, r1, #0
+; CHECK-BE-NEXT:    movs r1, #0
+; CHECK-BE-NEXT:    bfi r1, r3, #0, #1
+; CHECK-BE-NEXT:    ubfx r3, r2, #2, #1
+; CHECK-BE-NEXT:    rsbs r3, r3, #0
+; CHECK-BE-NEXT:    bfi r1, r3, #1, #1
+; CHECK-BE-NEXT:    ubfx r3, r2, #4, #1
+; CHECK-BE-NEXT:    rsbs r3, r3, #0
+; CHECK-BE-NEXT:    bfi r1, r3, #2, #1
+; CHECK-BE-NEXT:    ubfx r3, r2, #6, #1
+; CHECK-BE-NEXT:    rsbs r3, r3, #0
+; CHECK-BE-NEXT:    bfi r1, r3, #3, #1
+; CHECK-BE-NEXT:    ubfx r3, r2, #8, #1
+; CHECK-BE-NEXT:    rsbs r3, r3, #0
+; CHECK-BE-NEXT:    bfi r1, r3, #4, #1
+; CHECK-BE-NEXT:    ubfx r3, r2, #10, #1
+; CHECK-BE-NEXT:    rsbs r3, r3, #0
+; CHECK-BE-NEXT:    bfi r1, r3, #5, #1
+; CHECK-BE-NEXT:    ubfx r3, r2, #12, #1
+; CHECK-BE-NEXT:    ubfx r2, r2, #14, #1
+; CHECK-BE-NEXT:    rsbs r3, r3, #0
+; CHECK-BE-NEXT:    bfi r1, r3, #6, #1
+; CHECK-BE-NEXT:    rsbs r2, r2, #0
+; CHECK-BE-NEXT:    bfi r1, r2, #7, #1
+; CHECK-BE-NEXT:    strb r1, [r0]
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = icmp eq <8 x i16> %a, zeroinitializer
+  store <8 x i1> %c, <8 x i1>* %dst
+  ret void
+}
+
+define arm_aapcs_vfpcc void @store_v16i1(<16 x i1> *%dst, <16 x i8> %a) {
+; CHECK-LE-LABEL: store_v16i1:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vcmp.i8 eq, q0, zr
+; CHECK-LE-NEXT:    vmrs r1, p0
+; CHECK-LE-NEXT:    strh r1, [r0]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: store_v16i1:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vrev64.8 q1, q0
+; CHECK-BE-NEXT:    vcmp.i8 eq, q1, zr
+; CHECK-BE-NEXT:    vmrs r1, p0
+; CHECK-BE-NEXT:    strh r1, [r0]
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = icmp eq <16 x i8> %a, zeroinitializer
+  store <16 x i1> %c, <16 x i1>* %dst
+  ret void
+}
+
+define arm_aapcs_vfpcc void @store_v2i1(<2 x i1> *%dst, <2 x i64> %a) {
+; CHECK-LE-LABEL: store_v2i1:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vmov r1, s1
+; CHECK-LE-NEXT:    vmov r2, s0
+; CHECK-LE-NEXT:    vmov r3, s2
+; CHECK-LE-NEXT:    orrs r1, r2
+; CHECK-LE-NEXT:    vmov r2, s3
+; CHECK-LE-NEXT:    cset r1, eq
+; CHECK-LE-NEXT:    orrs r2, r3
+; CHECK-LE-NEXT:    cset r2, eq
+; CHECK-LE-NEXT:    ands r2, r2, #1
+; CHECK-LE-NEXT:    it ne
+; CHECK-LE-NEXT:    mvnne r2, #1
+; CHECK-LE-NEXT:    bfi r2, r1, #0, #1
+; CHECK-LE-NEXT:    and r1, r2, #3
+; CHECK-LE-NEXT:    strb r1, [r0]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: store_v2i1:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    vmov r1, s6
+; CHECK-BE-NEXT:    vmov r2, s7
+; CHECK-BE-NEXT:    vmov r3, s5
+; CHECK-BE-NEXT:    orrs r1, r2
+; CHECK-BE-NEXT:    vmov r2, s4
+; CHECK-BE-NEXT:    cset r1, eq
+; CHECK-BE-NEXT:    orrs r2, r3
+; CHECK-BE-NEXT:    cset r2, eq
+; CHECK-BE-NEXT:    ands r2, r2, #1
+; CHECK-BE-NEXT:    it ne
+; CHECK-BE-NEXT:    mvnne r2, #1
+; CHECK-BE-NEXT:    bfi r2, r1, #0, #1
+; CHECK-BE-NEXT:    and r1, r2, #3
+; CHECK-BE-NEXT:    strb r1, [r0]
+; CHECK-BE-NEXT:    bx lr
+entry:
+  %c = icmp eq <2 x i64> %a, zeroinitializer
+  store <2 x i1> %c, <2 x i1>* %dst
+  ret void
+}
diff --git a/llvm/test/CodeGen/Thumb2/mve-pred-not.ll b/llvm/test/CodeGen/Thumb2/mve-pred-not.ll
new file mode 100644
index 000000000..bc94f8ba9
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-pred-not.ll
@@ -0,0 +1,402 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s
+
+define arm_aapcs_vfpcc <4 x i32> @cmpeqz_v4i1(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: cmpeqz_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i32 eq, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %o = xor <4 x i1> %c1, <i1 -1, i1 -1, i1 -1, i1 -1>
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpnez_v4i1(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: cmpnez_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i32 eq, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %o = xor <4 x i1> %c1, <i1 -1, i1 -1, i1 -1, i1 -1>
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpsltz_v4i1(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: cmpsltz_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i32 eq, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %o = xor <4 x i1> %c1, <i1 -1, i1 -1, i1 -1, i1 -1>
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpsgtz_v4i1(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: cmpsgtz_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i32 eq, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %o = xor <4 x i1> %c1, <i1 -1, i1 -1, i1 -1, i1 -1>
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpslez_v4i1(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: cmpslez_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i32 eq, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %o = xor <4 x i1> %c1, <i1 -1, i1 -1, i1 -1, i1 -1>
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpsgez_v4i1(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: cmpsgez_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i32 eq, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %o = xor <4 x i1> %c1, <i1 -1, i1 -1, i1 -1, i1 -1>
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpultz_v4i1(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: cmpultz_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i32 eq, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %o = xor <4 x i1> %c1, <i1 -1, i1 -1, i1 -1, i1 -1>
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpugtz_v4i1(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: cmpugtz_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i32 eq, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %o = xor <4 x i1> %c1, <i1 -1, i1 -1, i1 -1, i1 -1>
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpulez_v4i1(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: cmpulez_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i32 eq, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %o = xor <4 x i1> %c1, <i1 -1, i1 -1, i1 -1, i1 -1>
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpugez_v4i1(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: cmpugez_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i32 eq, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %o = xor <4 x i1> %c1, <i1 -1, i1 -1, i1 -1, i1 -1>
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+
+
+define arm_aapcs_vfpcc <4 x i32> @cmpeq_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LABEL: cmpeq_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i32 eq, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %o = xor <4 x i1> %c1, <i1 -1, i1 -1, i1 -1, i1 -1>
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpne_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LABEL: cmpne_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i32 eq, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %o = xor <4 x i1> %c1, <i1 -1, i1 -1, i1 -1, i1 -1>
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpslt_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LABEL: cmpslt_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i32 eq, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %o = xor <4 x i1> %c1, <i1 -1, i1 -1, i1 -1, i1 -1>
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpsgt_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LABEL: cmpsgt_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i32 eq, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %o = xor <4 x i1> %c1, <i1 -1, i1 -1, i1 -1, i1 -1>
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpsle_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LABEL: cmpsle_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i32 eq, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %o = xor <4 x i1> %c1, <i1 -1, i1 -1, i1 -1, i1 -1>
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpsge_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LABEL: cmpsge_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i32 eq, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %o = xor <4 x i1> %c1, <i1 -1, i1 -1, i1 -1, i1 -1>
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpult_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LABEL: cmpult_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i32 eq, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %o = xor <4 x i1> %c1, <i1 -1, i1 -1, i1 -1, i1 -1>
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpugt_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LABEL: cmpugt_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i32 eq, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %o = xor <4 x i1> %c1, <i1 -1, i1 -1, i1 -1, i1 -1>
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpule_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LABEL: cmpule_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i32 eq, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %o = xor <4 x i1> %c1, <i1 -1, i1 -1, i1 -1, i1 -1>
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpuge_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LABEL: cmpuge_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i32 eq, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %o = xor <4 x i1> %c1, <i1 -1, i1 -1, i1 -1, i1 -1>
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+
+
+
+define arm_aapcs_vfpcc <8 x i16> @cmpeqz_v8i1(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: cmpeqz_v8i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i16 eq, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <8 x i16> %a, zeroinitializer
+  %o = xor <8 x i1> %c1, <i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1>
+  %s = select <8 x i1> %o, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+define arm_aapcs_vfpcc <8 x i16> @cmpeq_v8i1(<8 x i16> %a, <8 x i16> %b, <8 x i16> %c) {
+; CHECK-LABEL: cmpeq_v8i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i16 eq, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <8 x i16> %a, zeroinitializer
+  %o = xor <8 x i1> %c1, <i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1>
+  %s = select <8 x i1> %o, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+
+define arm_aapcs_vfpcc <16 x i8> @cmpeqz_v16i1(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: cmpeqz_v16i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i8 eq, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <16 x i8> %a, zeroinitializer
+  %o = xor <16 x i1> %c1, <i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1>
+  %s = select <16 x i1> %o, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %s
+}
+
+define arm_aapcs_vfpcc <16 x i8> @cmpeq_v16i1(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c) {
+; CHECK-LABEL: cmpeq_v16i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i8 eq, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <16 x i8> %a, zeroinitializer
+  %o = xor <16 x i1> %c1, <i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1>
+  %s = select <16 x i1> %o, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %s
+}
+
+
+define arm_aapcs_vfpcc <2 x i64> @cmpeqz_v2i1(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: cmpeqz_v2i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov r0, s1
+; CHECK-NEXT:    vmov r1, s0
+; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    vmov r1, s2
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q2[0], r0
+; CHECK-NEXT:    vmov.32 q2[1], r0
+; CHECK-NEXT:    vmov r0, s3
+; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q2[2], r0
+; CHECK-NEXT:    vmov.32 q2[3], r0
+; CHECK-NEXT:    vbic q0, q0, q2
+; CHECK-NEXT:    vand q1, q1, q2
+; CHECK-NEXT:    vorr q0, q1, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <2 x i64> %a, zeroinitializer
+  %o = xor <2 x i1> %c1, <i1 -1, i1 -1>
+  %s = select <2 x i1> %o, <2 x i64> %a, <2 x i64> %b
+  ret <2 x i64> %s
+}
+
+define arm_aapcs_vfpcc <2 x i64> @cmpeq_v2i1(<2 x i64> %a, <2 x i64> %b, <2 x i64> %c) {
+; CHECK-LABEL: cmpeq_v2i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov r0, s1
+; CHECK-NEXT:    vmov r1, s0
+; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    vmov r1, s2
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q2[0], r0
+; CHECK-NEXT:    vmov.32 q2[1], r0
+; CHECK-NEXT:    vmov r0, s3
+; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q2[2], r0
+; CHECK-NEXT:    vmov.32 q2[3], r0
+; CHECK-NEXT:    vbic q0, q0, q2
+; CHECK-NEXT:    vand q1, q1, q2
+; CHECK-NEXT:    vorr q0, q1, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <2 x i64> %a, zeroinitializer
+  %o = xor <2 x i1> %c1, <i1 -1, i1 -1>
+  %s = select <2 x i1> %o, <2 x i64> %a, <2 x i64> %b
+  ret <2 x i64> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @vpnot_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LABEL: vpnot_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.s32 lt, q0, zr
+; CHECK-NEXT:    vcmpt.s32 gt, q1, zr
+; CHECK-NEXT:    vpnot
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmpt.i32 eq, q2, zr
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp slt <4 x i32> %a, zeroinitializer
+  %c2 = icmp sgt <4 x i32> %b, zeroinitializer
+  %c3 = icmp eq <4 x i32> %c, zeroinitializer
+  %o1 = and <4 x i1> %c1, %c2
+  %o2 = xor <4 x i1> %o1, <i1 -1, i1 -1, i1 -1, i1 -1>
+  %o = and <4 x i1> %c3, %o2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
diff --git a/llvm/test/CodeGen/Thumb2/mve-pred-or.ll b/llvm/test/CodeGen/Thumb2/mve-pred-or.ll
new file mode 100644
index 000000000..26ff81c6f
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-pred-or.ll
@@ -0,0 +1,508 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s
+
+define arm_aapcs_vfpcc <4 x i32> @cmpeqz_v4i1(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: cmpeqz_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vcmpt.i32 ne, q1, zr
+; CHECK-NEXT:    vpnot
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %c2 = icmp eq <4 x i32> %b, zeroinitializer
+  %o = or <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpnez_v4i1(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: cmpnez_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vcmpt.i32 eq, q1, zr
+; CHECK-NEXT:    vpnot
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %c2 = icmp ne <4 x i32> %b, zeroinitializer
+  %o = or <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpsltz_v4i1(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: cmpsltz_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vcmpt.s32 ge, q1, zr
+; CHECK-NEXT:    vpnot
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %c2 = icmp slt <4 x i32> %b, zeroinitializer
+  %o = or <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpsgtz_v4i1(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: cmpsgtz_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vcmpt.s32 le, q1, zr
+; CHECK-NEXT:    vpnot
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %c2 = icmp sgt <4 x i32> %b, zeroinitializer
+  %o = or <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpslez_v4i1(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: cmpslez_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vcmpt.s32 gt, q1, zr
+; CHECK-NEXT:    vpnot
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %c2 = icmp sle <4 x i32> %b, zeroinitializer
+  %o = or <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpsgez_v4i1(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: cmpsgez_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vcmpt.s32 lt, q1, zr
+; CHECK-NEXT:    vpnot
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %c2 = icmp sge <4 x i32> %b, zeroinitializer
+  %o = or <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpultz_v4i1(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: cmpultz_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i32 eq, q0, zr
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %c2 = icmp ult <4 x i32> %b, zeroinitializer
+  %o = or <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpugtz_v4i1(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: cmpugtz_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vcmpt.i32 eq, q1, zr
+; CHECK-NEXT:    vpnot
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %c2 = icmp ugt <4 x i32> %b, zeroinitializer
+  %o = or <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpulez_v4i1(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: cmpulez_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.u32 cs, q1, zr
+; CHECK-NEXT:    vmrs r0, p0
+; CHECK-NEXT:    vcmp.i32 eq, q0, zr
+; CHECK-NEXT:    vmrs r1, p0
+; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %c2 = icmp ule <4 x i32> %b, zeroinitializer
+  %o = or <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpugez_v4i1(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: cmpugez_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %c2 = icmp uge <4 x i32> %b, zeroinitializer
+  %o = or <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+
+
+define arm_aapcs_vfpcc <4 x i32> @cmpeq_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LABEL: cmpeq_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vcmpt.i32 ne, q1, q2
+; CHECK-NEXT:    vpnot
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %c2 = icmp eq <4 x i32> %b, %c
+  %o = or <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpne_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LABEL: cmpne_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vcmpt.i32 eq, q1, q2
+; CHECK-NEXT:    vpnot
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %c2 = icmp ne <4 x i32> %b, %c
+  %o = or <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpslt_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LABEL: cmpslt_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vcmpt.s32 le, q2, q1
+; CHECK-NEXT:    vpnot
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %c2 = icmp slt <4 x i32> %b, %c
+  %o = or <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpsgt_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LABEL: cmpsgt_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vcmpt.s32 le, q1, q2
+; CHECK-NEXT:    vpnot
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %c2 = icmp sgt <4 x i32> %b, %c
+  %o = or <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpsle_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LABEL: cmpsle_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vcmpt.s32 lt, q2, q1
+; CHECK-NEXT:    vpnot
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %c2 = icmp sle <4 x i32> %b, %c
+  %o = or <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpsge_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LABEL: cmpsge_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
+; CHECK-NEXT:    vcmpt.s32 lt, q1, q2
+; CHECK-NEXT:    vpnot
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %c2 = icmp sge <4 x i32> %b, %c
+  %o = or <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpult_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LABEL: cmpult_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.u32 hi, q2, q1
+; CHECK-NEXT:    vmrs r0, p0
+; CHECK-NEXT:    vcmp.i32 eq, q0, zr
+; CHECK-NEXT:    vmrs r1, p0
+; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %c2 = icmp ult <4 x i32> %b, %c
+  %o = or <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpugt_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LABEL: cmpugt_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.u32 hi, q1, q2
+; CHECK-NEXT:    vmrs r0, p0
+; CHECK-NEXT:    vcmp.i32 eq, q0, zr
+; CHECK-NEXT:    vmrs r1, p0
+; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %c2 = icmp ugt <4 x i32> %b, %c
+  %o = or <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpule_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LABEL: cmpule_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.u32 cs, q2, q1
+; CHECK-NEXT:    vmrs r0, p0
+; CHECK-NEXT:    vcmp.i32 eq, q0, zr
+; CHECK-NEXT:    vmrs r1, p0
+; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %c2 = icmp ule <4 x i32> %b, %c
+  %o = or <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpuge_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LABEL: cmpuge_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.u32 cs, q1, q2
+; CHECK-NEXT:    vmrs r0, p0
+; CHECK-NEXT:    vcmp.i32 eq, q0, zr
+; CHECK-NEXT:    vmrs r1, p0
+; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %c2 = icmp uge <4 x i32> %b, %c
+  %o = or <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+
+
+
+define arm_aapcs_vfpcc <8 x i16> @cmpeqz_v8i1(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: cmpeqz_v8i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vcmpt.i16 ne, q1, zr
+; CHECK-NEXT:    vpnot
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <8 x i16> %a, zeroinitializer
+  %c2 = icmp eq <8 x i16> %b, zeroinitializer
+  %o = or <8 x i1> %c1, %c2
+  %s = select <8 x i1> %o, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+define arm_aapcs_vfpcc <8 x i16> @cmpeq_v8i1(<8 x i16> %a, <8 x i16> %b, <8 x i16> %c) {
+; CHECK-LABEL: cmpeq_v8i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
+; CHECK-NEXT:    vcmpt.i16 ne, q1, q2
+; CHECK-NEXT:    vpnot
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <8 x i16> %a, zeroinitializer
+  %c2 = icmp eq <8 x i16> %b, %c
+  %o = or <8 x i1> %c1, %c2
+  %s = select <8 x i1> %o, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+
+define arm_aapcs_vfpcc <16 x i8> @cmpeqz_v16i1(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: cmpeqz_v16i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i8 ne, q0, zr
+; CHECK-NEXT:    vcmpt.i8 ne, q1, zr
+; CHECK-NEXT:    vpnot
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <16 x i8> %a, zeroinitializer
+  %c2 = icmp eq <16 x i8> %b, zeroinitializer
+  %o = or <16 x i1> %c1, %c2
+  %s = select <16 x i1> %o, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %s
+}
+
+define arm_aapcs_vfpcc <16 x i8> @cmpeq_v16i1(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c) {
+; CHECK-LABEL: cmpeq_v16i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i8 ne, q0, zr
+; CHECK-NEXT:    vcmpt.i8 ne, q1, q2
+; CHECK-NEXT:    vpnot
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <16 x i8> %a, zeroinitializer
+  %c2 = icmp eq <16 x i8> %b, %c
+  %o = or <16 x i1> %c1, %c2
+  %s = select <16 x i1> %o, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %s
+}
+
+
+define arm_aapcs_vfpcc <2 x i64> @cmpeqz_v2i1(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: cmpeqz_v2i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov r0, s5
+; CHECK-NEXT:    vmov r1, s4
+; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    vmov r1, s6
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q2[0], r0
+; CHECK-NEXT:    vmov.32 q2[1], r0
+; CHECK-NEXT:    vmov r0, s7
+; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    vmov r1, s0
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q2[2], r0
+; CHECK-NEXT:    vmov.32 q2[3], r0
+; CHECK-NEXT:    vmov r0, s1
+; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    vmov r1, s2
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q3[0], r0
+; CHECK-NEXT:    vmov.32 q3[1], r0
+; CHECK-NEXT:    vmov r0, s3
+; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q3[2], r0
+; CHECK-NEXT:    vmov.32 q3[3], r0
+; CHECK-NEXT:    vorr q2, q3, q2
+; CHECK-NEXT:    vbic q1, q1, q2
+; CHECK-NEXT:    vand q0, q0, q2
+; CHECK-NEXT:    vorr q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <2 x i64> %a, zeroinitializer
+  %c2 = icmp eq <2 x i64> %b, zeroinitializer
+  %o = or <2 x i1> %c1, %c2
+  %s = select <2 x i1> %o, <2 x i64> %a, <2 x i64> %b
+  ret <2 x i64> %s
+}
+
+define arm_aapcs_vfpcc <2 x i64> @cmpeq_v2i1(<2 x i64> %a, <2 x i64> %b, <2 x i64> %c) {
+; CHECK-LABEL: cmpeq_v2i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov r0, s9
+; CHECK-NEXT:    vmov r1, s5
+; CHECK-NEXT:    vmov r2, s4
+; CHECK-NEXT:    eors r0, r1
+; CHECK-NEXT:    vmov r1, s8
+; CHECK-NEXT:    eors r1, r2
+; CHECK-NEXT:    vmov r2, s6
+; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    vmov r1, s7
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q3[0], r0
+; CHECK-NEXT:    vmov.32 q3[1], r0
+; CHECK-NEXT:    vmov r0, s11
+; CHECK-NEXT:    eors r0, r1
+; CHECK-NEXT:    vmov r1, s10
+; CHECK-NEXT:    eors r1, r2
+; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    vmov r1, s0
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q3[2], r0
+; CHECK-NEXT:    vmov.32 q3[3], r0
+; CHECK-NEXT:    vmov r0, s1
+; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    vmov r1, s2
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q2[0], r0
+; CHECK-NEXT:    vmov.32 q2[1], r0
+; CHECK-NEXT:    vmov r0, s3
+; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q2[2], r0
+; CHECK-NEXT:    vmov.32 q2[3], r0
+; CHECK-NEXT:    vorr q2, q2, q3
+; CHECK-NEXT:    vbic q1, q1, q2
+; CHECK-NEXT:    vand q0, q0, q2
+; CHECK-NEXT:    vorr q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <2 x i64> %a, zeroinitializer
+  %c2 = icmp eq <2 x i64> %b, %c
+  %o = or <2 x i1> %c1, %c2
+  %s = select <2 x i1> %o, <2 x i64> %a, <2 x i64> %b
+  ret <2 x i64> %s
+}
+
+
diff --git a/llvm/test/CodeGen/Thumb2/mve-pred-shuffle.ll b/llvm/test/CodeGen/Thumb2/mve-pred-shuffle.ll
new file mode 100644
index 000000000..1b00b708d
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-pred-shuffle.ll
@@ -0,0 +1,563 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s
+
+define <4 x i32> @shuffle1_v4i32(<4 x i32> %src, <4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: shuffle1_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov d1, r2, r3
+; CHECK-NEXT:    vmov d0, r0, r1
+; CHECK-NEXT:    vcmp.i32 eq, q0, zr
+; CHECK-NEXT:    vmrs r0, p0
+; CHECK-NEXT:    rbit r0, r0
+; CHECK-NEXT:    lsrs r0, r0, #16
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    add r0, sp, #16
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    mov r0, sp
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    vmov r0, r1, d0
+; CHECK-NEXT:    vmov r2, r3, d1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <4 x i32> %src, zeroinitializer
+  %sh = shufflevector <4 x i1> %c, <4 x i1> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %s = select <4 x i1> %sh, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define <8 x i16> @shuffle1_v8i16(<8 x i16> %src, <8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: shuffle1_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov d1, r2, r3
+; CHECK-NEXT:    vmov d0, r0, r1
+; CHECK-NEXT:    vcmp.i16 eq, q0, zr
+; CHECK-NEXT:    vmrs r0, p0
+; CHECK-NEXT:    rbit r0, r0
+; CHECK-NEXT:    lsrs r0, r0, #16
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    add r0, sp, #16
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    mov r0, sp
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    vmov r0, r1, d0
+; CHECK-NEXT:    vmov r2, r3, d1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <8 x i16> %src, zeroinitializer
+  %sh = shufflevector <8 x i1> %c, <8 x i1> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  %s = select <8 x i1> %sh, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+define <16 x i8> @shuffle1_v16i8(<16 x i8> %src, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: shuffle1_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov d1, r2, r3
+; CHECK-NEXT:    vmov d0, r0, r1
+; CHECK-NEXT:    vcmp.i8 eq, q0, zr
+; CHECK-NEXT:    vmrs r0, p0
+; CHECK-NEXT:    rbit r0, r0
+; CHECK-NEXT:    lsrs r0, r0, #16
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    add r0, sp, #16
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    mov r0, sp
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    vmov r0, r1, d0
+; CHECK-NEXT:    vmov r2, r3, d1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <16 x i8> %src, zeroinitializer
+  %sh = shufflevector <16 x i1> %c, <16 x i1> undef, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  %s = select <16 x i1> %sh, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %s
+}
+
+define <4 x i32> @shuffle2_v4i32(<4 x i32> %src, <4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: shuffle2_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov d1, r2, r3
+; CHECK-NEXT:    vmov d0, r0, r1
+; CHECK-NEXT:    add r0, sp, #16
+; CHECK-NEXT:    vcmp.i32 eq, q0, zr
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    mov r0, sp
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    vmov r0, r1, d0
+; CHECK-NEXT:    vmov r2, r3, d1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <4 x i32> %src, zeroinitializer
+  %sh = shufflevector <4 x i1> %c, <4 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %s = select <4 x i1> %sh, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define <8 x i16> @shuffle2_v8i16(<8 x i16> %src, <8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: shuffle2_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov d1, r2, r3
+; CHECK-NEXT:    vmov d0, r0, r1
+; CHECK-NEXT:    add r0, sp, #16
+; CHECK-NEXT:    vcmp.i16 eq, q0, zr
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    mov r0, sp
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    vmov r0, r1, d0
+; CHECK-NEXT:    vmov r2, r3, d1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <8 x i16> %src, zeroinitializer
+  %sh = shufflevector <8 x i1> %c, <8 x i1> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %s = select <8 x i1> %sh, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+define <16 x i8> @shuffle2_v16i8(<16 x i8> %src, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: shuffle2_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov d1, r2, r3
+; CHECK-NEXT:    vmov d0, r0, r1
+; CHECK-NEXT:    add r0, sp, #16
+; CHECK-NEXT:    vcmp.i8 eq, q0, zr
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    mov r0, sp
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    vmov r0, r1, d0
+; CHECK-NEXT:    vmov r2, r3, d1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <16 x i8> %src, zeroinitializer
+  %sh = shufflevector <16 x i1> %c, <16 x i1> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %s = select <16 x i1> %sh, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %s
+}
+
+define <4 x i32> @shuffle3_v4i32(<4 x i32> %src, <4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: shuffle3_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov d1, r2, r3
+; CHECK-NEXT:    vmov.i8 q1, #0xff
+; CHECK-NEXT:    vmov d0, r0, r1
+; CHECK-NEXT:    vcmp.i32 eq, q0, zr
+; CHECK-NEXT:    vmov.i8 q0, #0x0
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    vmov.32 r0, q0[0]
+; CHECK-NEXT:    vdup.32 q0, r0
+; CHECK-NEXT:    add r0, sp, #16
+; CHECK-NEXT:    vcmp.i32 ne, q0, zr
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    mov r0, sp
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    vmov r0, r1, d0
+; CHECK-NEXT:    vmov r2, r3, d1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <4 x i32> %src, zeroinitializer
+  %sh = shufflevector <4 x i1> %c, <4 x i1> undef, <4 x i32> <i32 0, i32 0, i32 0, i32 0>
+  %s = select <4 x i1> %sh, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define <8 x i16> @shuffle3_v8i16(<8 x i16> %src, <8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: shuffle3_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov d1, r2, r3
+; CHECK-NEXT:    vmov.i8 q1, #0xff
+; CHECK-NEXT:    vmov d0, r0, r1
+; CHECK-NEXT:    vcmp.i16 eq, q0, zr
+; CHECK-NEXT:    vmov.i8 q0, #0x0
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    vmov.u16 r0, q0[0]
+; CHECK-NEXT:    vdup.16 q0, r0
+; CHECK-NEXT:    add r0, sp, #16
+; CHECK-NEXT:    vcmp.i16 ne, q0, zr
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    mov r0, sp
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    vmov r0, r1, d0
+; CHECK-NEXT:    vmov r2, r3, d1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <8 x i16> %src, zeroinitializer
+  %sh = shufflevector <8 x i1> %c, <8 x i1> undef, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
+  %s = select <8 x i1> %sh, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+define <16 x i8> @shuffle3_v16i8(<16 x i8> %src, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: shuffle3_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov d1, r2, r3
+; CHECK-NEXT:    vmov.i8 q1, #0xff
+; CHECK-NEXT:    vmov d0, r0, r1
+; CHECK-NEXT:    vcmp.i8 eq, q0, zr
+; CHECK-NEXT:    vmov.i8 q0, #0x0
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    vmov.u8 r0, q0[0]
+; CHECK-NEXT:    vdup.8 q0, r0
+; CHECK-NEXT:    add r0, sp, #16
+; CHECK-NEXT:    vcmp.i8 ne, q0, zr
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    mov r0, sp
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    vmov r0, r1, d0
+; CHECK-NEXT:    vmov r2, r3, d1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <16 x i8> %src, zeroinitializer
+  %sh = shufflevector <16 x i1> %c, <16 x i1> undef, <16 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
+  %s = select <16 x i1> %sh, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %s
+}
+
+define <4 x i32> @shuffle4_v4i32(<4 x i32> %src, <4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: shuffle4_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov d1, r2, r3
+; CHECK-NEXT:    vmov.i8 q1, #0xff
+; CHECK-NEXT:    vmov d0, r0, r1
+; CHECK-NEXT:    add r0, sp, #16
+; CHECK-NEXT:    vcmp.i32 eq, q0, zr
+; CHECK-NEXT:    vmov.i8 q0, #0x0
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    vmov.f32 s4, s0
+; CHECK-NEXT:    vmov.f32 s5, s0
+; CHECK-NEXT:    vmov.f32 s6, s0
+; CHECK-NEXT:    vmov.f32 s7, s1
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    mov r0, sp
+; CHECK-NEXT:    vcmp.i32 ne, q1, zr
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    vmov r0, r1, d0
+; CHECK-NEXT:    vmov r2, r3, d1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <4 x i32> %src, zeroinitializer
+  %sh = shufflevector <4 x i1> %c, <4 x i1> undef, <4 x i32> <i32 0, i32 0, i32 0, i32 1>
+  %s = select <4 x i1> %sh, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define <8 x i16> @shuffle4_v8i16(<8 x i16> %src, <8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: shuffle4_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov d1, r2, r3
+; CHECK-NEXT:    vmov.i8 q1, #0xff
+; CHECK-NEXT:    vmov d0, r0, r1
+; CHECK-NEXT:    vcmp.i16 eq, q0, zr
+; CHECK-NEXT:    vmov.i8 q0, #0x0
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    vmov.u16 r0, q0[0]
+; CHECK-NEXT:    vdup.16 q1, r0
+; CHECK-NEXT:    add r0, sp, #16
+; CHECK-NEXT:    vmov.f32 s7, s0
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    mov r0, sp
+; CHECK-NEXT:    vcmp.i16 ne, q1, zr
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    vmov r0, r1, d0
+; CHECK-NEXT:    vmov r2, r3, d1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <8 x i16> %src, zeroinitializer
+  %sh = shufflevector <8 x i1> %c, <8 x i1> undef, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 1>
+  %s = select <8 x i1> %sh, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+define <16 x i8> @shuffle4_v16i8(<16 x i8> %src, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: shuffle4_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov d1, r2, r3
+; CHECK-NEXT:    vmov.i8 q1, #0xff
+; CHECK-NEXT:    vmov d0, r0, r1
+; CHECK-NEXT:    vcmp.i8 eq, q0, zr
+; CHECK-NEXT:    vmov.i8 q0, #0x0
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    vmov.u8 r0, q0[0]
+; CHECK-NEXT:    vdup.8 q1, r0
+; CHECK-NEXT:    vmov.u8 r0, q0[1]
+; CHECK-NEXT:    vmov.8 q1[15], r0
+; CHECK-NEXT:    add r0, sp, #16
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    mov r0, sp
+; CHECK-NEXT:    vcmp.i8 ne, q1, zr
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    vmov r0, r1, d0
+; CHECK-NEXT:    vmov r2, r3, d1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <16 x i8> %src, zeroinitializer
+  %sh = shufflevector <16 x i1> %c, <16 x i1> undef, <16 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 1>
+  %s = select <16 x i1> %sh, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %s
+}
+
+define <4 x i32> @shuffle5_b_v4i32(<8 x i16> %src, <4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: shuffle5_b_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov d1, r2, r3
+; CHECK-NEXT:    vmov.i8 q1, #0xff
+; CHECK-NEXT:    vmov d0, r0, r1
+; CHECK-NEXT:    vcmp.i16 eq, q0, zr
+; CHECK-NEXT:    vmov.i8 q0, #0x0
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    vmov.u16 r0, q0[0]
+; CHECK-NEXT:    vmov.32 q1[0], r0
+; CHECK-NEXT:    vmov.u16 r0, q0[1]
+; CHECK-NEXT:    vmov.32 q1[1], r0
+; CHECK-NEXT:    vmov.u16 r0, q0[2]
+; CHECK-NEXT:    vmov.32 q1[2], r0
+; CHECK-NEXT:    vmov.u16 r0, q0[3]
+; CHECK-NEXT:    vmov.32 q1[3], r0
+; CHECK-NEXT:    add r0, sp, #16
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    mov r0, sp
+; CHECK-NEXT:    vcmp.i32 ne, q1, zr
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    vmov r0, r1, d0
+; CHECK-NEXT:    vmov r2, r3, d1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <8 x i16> %src, zeroinitializer
+  %sh = shufflevector <8 x i1> %c, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %s = select <4 x i1> %sh, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define <4 x i32> @shuffle5_t_v4i32(<8 x i16> %src, <4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: shuffle5_t_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov d1, r2, r3
+; CHECK-NEXT:    vmov.i8 q1, #0xff
+; CHECK-NEXT:    vmov d0, r0, r1
+; CHECK-NEXT:    vcmp.i16 eq, q0, zr
+; CHECK-NEXT:    vmov.i8 q0, #0x0
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    vmov.u16 r0, q0[4]
+; CHECK-NEXT:    vmov.32 q1[0], r0
+; CHECK-NEXT:    vmov.u16 r0, q0[5]
+; CHECK-NEXT:    vmov.32 q1[1], r0
+; CHECK-NEXT:    vmov.u16 r0, q0[6]
+; CHECK-NEXT:    vmov.32 q1[2], r0
+; CHECK-NEXT:    vmov.u16 r0, q0[7]
+; CHECK-NEXT:    vmov.32 q1[3], r0
+; CHECK-NEXT:    add r0, sp, #16
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    mov r0, sp
+; CHECK-NEXT:    vcmp.i32 ne, q1, zr
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    vmov r0, r1, d0
+; CHECK-NEXT:    vmov r2, r3, d1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <8 x i16> %src, zeroinitializer
+  %sh = shufflevector <8 x i1> %c, <8 x i1> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %s = select <4 x i1> %sh, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define <8 x i16> @shuffle5_b_v8i16(<16 x i8> %src, <8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: shuffle5_b_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov d1, r2, r3
+; CHECK-NEXT:    vmov.i8 q1, #0xff
+; CHECK-NEXT:    vmov d0, r0, r1
+; CHECK-NEXT:    vcmp.i8 eq, q0, zr
+; CHECK-NEXT:    vmov.i8 q0, #0x0
+; CHECK-NEXT:    vpsel q1, q1, q0
+; CHECK-NEXT:    vmov.u8 r0, q1[0]
+; CHECK-NEXT:    vmov.16 q0[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[1]
+; CHECK-NEXT:    vmov.16 q0[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[2]
+; CHECK-NEXT:    vmov.16 q0[2], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[3]
+; CHECK-NEXT:    vmov.16 q0[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[4]
+; CHECK-NEXT:    vmov.16 q0[4], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[5]
+; CHECK-NEXT:    vmov.16 q0[5], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[6]
+; CHECK-NEXT:    vmov.16 q0[6], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[7]
+; CHECK-NEXT:    vmov.16 q0[7], r0
+; CHECK-NEXT:    add r0, sp, #16
+; CHECK-NEXT:    vcmp.i16 ne, q0, zr
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    mov r0, sp
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    vmov r0, r1, d0
+; CHECK-NEXT:    vmov r2, r3, d1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <16 x i8> %src, zeroinitializer
+  %sh = shufflevector <16 x i1> %c, <16 x i1> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %s = select <8 x i1> %sh, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+define <8 x i16> @shuffle5_t_v8i16(<16 x i8> %src, <8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: shuffle5_t_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov d1, r2, r3
+; CHECK-NEXT:    vmov.i8 q1, #0xff
+; CHECK-NEXT:    vmov d0, r0, r1
+; CHECK-NEXT:    vcmp.i8 eq, q0, zr
+; CHECK-NEXT:    vmov.i8 q0, #0x0
+; CHECK-NEXT:    vpsel q1, q1, q0
+; CHECK-NEXT:    vmov.u8 r0, q1[8]
+; CHECK-NEXT:    vmov.16 q0[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[9]
+; CHECK-NEXT:    vmov.16 q0[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[10]
+; CHECK-NEXT:    vmov.16 q0[2], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[11]
+; CHECK-NEXT:    vmov.16 q0[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[12]
+; CHECK-NEXT:    vmov.16 q0[4], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[13]
+; CHECK-NEXT:    vmov.16 q0[5], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[14]
+; CHECK-NEXT:    vmov.16 q0[6], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[15]
+; CHECK-NEXT:    vmov.16 q0[7], r0
+; CHECK-NEXT:    add r0, sp, #16
+; CHECK-NEXT:    vcmp.i16 ne, q0, zr
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    mov r0, sp
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    vmov r0, r1, d0
+; CHECK-NEXT:    vmov r2, r3, d1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <16 x i8> %src, zeroinitializer
+  %sh = shufflevector <16 x i1> %c, <16 x i1> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %s = select <8 x i1> %sh, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+define <8 x i16> @shuffle6_v4i32(<4 x i32> %src1, <4 x i32> %src2, <8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: shuffle6_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov d1, r2, r3
+; CHECK-NEXT:    vmov.i8 q1, #0x0
+; CHECK-NEXT:    vmov d0, r0, r1
+; CHECK-NEXT:    vmov.i8 q2, #0xff
+; CHECK-NEXT:    vcmp.i32 eq, q0, zr
+; CHECK-NEXT:    vpsel q3, q2, q1
+; CHECK-NEXT:    vmov r0, s12
+; CHECK-NEXT:    vmov.16 q0[0], r0
+; CHECK-NEXT:    vmov r0, s13
+; CHECK-NEXT:    vmov.16 q0[1], r0
+; CHECK-NEXT:    vmov r0, s14
+; CHECK-NEXT:    vmov.16 q0[2], r0
+; CHECK-NEXT:    vmov r0, s15
+; CHECK-NEXT:    vmov.16 q0[3], r0
+; CHECK-NEXT:    mov r0, sp
+; CHECK-NEXT:    vldrw.u32 q3, [r0]
+; CHECK-NEXT:    vcmp.i32 eq, q3, zr
+; CHECK-NEXT:    vpsel q1, q2, q1
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmov.16 q0[4], r0
+; CHECK-NEXT:    vmov r0, s5
+; CHECK-NEXT:    vmov.16 q0[5], r0
+; CHECK-NEXT:    vmov r0, s6
+; CHECK-NEXT:    vmov.16 q0[6], r0
+; CHECK-NEXT:    vmov r0, s7
+; CHECK-NEXT:    vmov.16 q0[7], r0
+; CHECK-NEXT:    add r0, sp, #32
+; CHECK-NEXT:    vcmp.i16 ne, q0, zr
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    add r0, sp, #16
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    vmov r0, r1, d0
+; CHECK-NEXT:    vmov r2, r3, d1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %src1, zeroinitializer
+  %c2 = icmp eq <4 x i32> %src2, zeroinitializer
+  %sh = shufflevector <4 x i1> %c1, <4 x i1> %c2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %s = select <8 x i1> %sh, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+define <16 x i8> @shuffle6_v8i16(<8 x i16> %src1, <8 x i16> %src2, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: shuffle6_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov d1, r2, r3
+; CHECK-NEXT:    vmov.i8 q1, #0x0
+; CHECK-NEXT:    vmov d0, r0, r1
+; CHECK-NEXT:    vmov.i8 q2, #0xff
+; CHECK-NEXT:    vcmp.i16 eq, q0, zr
+; CHECK-NEXT:    vpsel q3, q2, q1
+; CHECK-NEXT:    vmov.u16 r0, q3[0]
+; CHECK-NEXT:    vmov.8 q0[0], r0
+; CHECK-NEXT:    vmov.u16 r0, q3[1]
+; CHECK-NEXT:    vmov.8 q0[1], r0
+; CHECK-NEXT:    vmov.u16 r0, q3[2]
+; CHECK-NEXT:    vmov.8 q0[2], r0
+; CHECK-NEXT:    vmov.u16 r0, q3[3]
+; CHECK-NEXT:    vmov.8 q0[3], r0
+; CHECK-NEXT:    vmov.u16 r0, q3[4]
+; CHECK-NEXT:    vmov.8 q0[4], r0
+; CHECK-NEXT:    vmov.u16 r0, q3[5]
+; CHECK-NEXT:    vmov.8 q0[5], r0
+; CHECK-NEXT:    vmov.u16 r0, q3[6]
+; CHECK-NEXT:    vmov.8 q0[6], r0
+; CHECK-NEXT:    vmov.u16 r0, q3[7]
+; CHECK-NEXT:    vmov.8 q0[7], r0
+; CHECK-NEXT:    mov r0, sp
+; CHECK-NEXT:    vldrw.u32 q3, [r0]
+; CHECK-NEXT:    vcmp.i16 eq, q3, zr
+; CHECK-NEXT:    vpsel q1, q2, q1
+; CHECK-NEXT:    vmov.u16 r0, q1[0]
+; CHECK-NEXT:    vmov.8 q0[8], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[1]
+; CHECK-NEXT:    vmov.8 q0[9], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[2]
+; CHECK-NEXT:    vmov.8 q0[10], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[3]
+; CHECK-NEXT:    vmov.8 q0[11], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[4]
+; CHECK-NEXT:    vmov.8 q0[12], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[5]
+; CHECK-NEXT:    vmov.8 q0[13], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[6]
+; CHECK-NEXT:    vmov.8 q0[14], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[7]
+; CHECK-NEXT:    vmov.8 q0[15], r0
+; CHECK-NEXT:    add r0, sp, #32
+; CHECK-NEXT:    vcmp.i8 ne, q0, zr
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    add r0, sp, #16
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    vmov r0, r1, d0
+; CHECK-NEXT:    vmov r2, r3, d1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <8 x i16> %src1, zeroinitializer
+  %c2 = icmp eq <8 x i16> %src2, zeroinitializer
+  %sh = shufflevector <8 x i1> %c1, <8 x i1> %c2, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %s = select <16 x i1> %sh, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %s
+}
diff --git a/llvm/test/CodeGen/Thumb2/mve-pred-spill.ll b/llvm/test/CodeGen/Thumb2/mve-pred-spill.ll
new file mode 100644
index 000000000..ccbe50335
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-pred-spill.ll
@@ -0,0 +1,165 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-LE
+; RUN: llc -mtriple=thumbebv8.1m.main-arm-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-BE
+
+declare arm_aapcs_vfpcc <4 x i32> @ext_i32(<4 x i32> %c)
+declare arm_aapcs_vfpcc <8 x i16> @ext_i16(<8 x i16> %c)
+declare arm_aapcs_vfpcc <16 x i8> @ext_i8(<16 x i8> %c)
+
+define arm_aapcs_vfpcc <4 x i32> @shuffle1_v4i32(<4 x i32> %src, <4 x i32> %a) {
+; CHECK-LE-LABEL: shuffle1_v4i32:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    .save {r7, lr}
+; CHECK-LE-NEXT:    push {r7, lr}
+; CHECK-LE-NEXT:    .vsave {d8, d9}
+; CHECK-LE-NEXT:    vpush {d8, d9}
+; CHECK-LE-NEXT:    .pad #8
+; CHECK-LE-NEXT:    sub sp, #8
+; CHECK-LE-NEXT:    vcmp.i32 eq, q0, zr
+; CHECK-LE-NEXT:    vmov.i32 q0, #0x0
+; CHECK-LE-NEXT:    vpsel q0, q1, q0
+; CHECK-LE-NEXT:    vmov q4, q1
+; CHECK-LE-NEXT:    vstr p0, [sp, #4] @ 4-byte Spill
+; CHECK-LE-NEXT:    bl ext_i32
+; CHECK-LE-NEXT:    vldr p0, [sp, #4] @ 4-byte Reload
+; CHECK-LE-NEXT:    vpsel q0, q4, q0
+; CHECK-LE-NEXT:    add sp, #8
+; CHECK-LE-NEXT:    vpop {d8, d9}
+; CHECK-LE-NEXT:    pop {r7, pc}
+;
+; CHECK-BE-LABEL: shuffle1_v4i32:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    .save {r7, lr}
+; CHECK-BE-NEXT:    push {r7, lr}
+; CHECK-BE-NEXT:    .vsave {d8, d9}
+; CHECK-BE-NEXT:    vpush {d8, d9}
+; CHECK-BE-NEXT:    .pad #8
+; CHECK-BE-NEXT:    sub sp, #8
+; CHECK-BE-NEXT:    vrev64.32 q4, q1
+; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    vcmp.i32 eq, q1, zr
+; CHECK-BE-NEXT:    vmov.i32 q0, #0x0
+; CHECK-BE-NEXT:    vpsel q1, q4, q0
+; CHECK-BE-NEXT:    vstr p0, [sp, #4] @ 4-byte Spill
+; CHECK-BE-NEXT:    vrev64.32 q0, q1
+; CHECK-BE-NEXT:    bl ext_i32
+; CHECK-BE-NEXT:    vldr p0, [sp, #4] @ 4-byte Reload
+; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    vpsel q1, q4, q1
+; CHECK-BE-NEXT:    vrev64.32 q0, q1
+; CHECK-BE-NEXT:    add sp, #8
+; CHECK-BE-NEXT:    vpop {d8, d9}
+; CHECK-BE-NEXT:    pop {r7, pc}
+entry:
+  %c = icmp eq <4 x i32> %src, zeroinitializer
+  %s1 = select <4 x i1> %c, <4 x i32> %a, <4 x i32> zeroinitializer
+  %ext = call arm_aapcs_vfpcc <4 x i32> @ext_i32(<4 x i32> %s1)
+  %s = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %ext
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <8 x i16> @shuffle1_v8i16(<8 x i16> %src, <8 x i16> %a) {
+; CHECK-LE-LABEL: shuffle1_v8i16:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    .save {r7, lr}
+; CHECK-LE-NEXT:    push {r7, lr}
+; CHECK-LE-NEXT:    .vsave {d8, d9}
+; CHECK-LE-NEXT:    vpush {d8, d9}
+; CHECK-LE-NEXT:    .pad #8
+; CHECK-LE-NEXT:    sub sp, #8
+; CHECK-LE-NEXT:    vcmp.i16 eq, q0, zr
+; CHECK-LE-NEXT:    vmov.i32 q0, #0x0
+; CHECK-LE-NEXT:    vpsel q0, q1, q0
+; CHECK-LE-NEXT:    vmov q4, q1
+; CHECK-LE-NEXT:    vstr p0, [sp, #4] @ 4-byte Spill
+; CHECK-LE-NEXT:    bl ext_i16
+; CHECK-LE-NEXT:    vldr p0, [sp, #4] @ 4-byte Reload
+; CHECK-LE-NEXT:    vpsel q0, q4, q0
+; CHECK-LE-NEXT:    add sp, #8
+; CHECK-LE-NEXT:    vpop {d8, d9}
+; CHECK-LE-NEXT:    pop {r7, pc}
+;
+; CHECK-BE-LABEL: shuffle1_v8i16:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    .save {r7, lr}
+; CHECK-BE-NEXT:    push {r7, lr}
+; CHECK-BE-NEXT:    .vsave {d8, d9}
+; CHECK-BE-NEXT:    vpush {d8, d9}
+; CHECK-BE-NEXT:    .pad #8
+; CHECK-BE-NEXT:    sub sp, #8
+; CHECK-BE-NEXT:    vrev64.16 q4, q1
+; CHECK-BE-NEXT:    vmov.i32 q1, #0x0
+; CHECK-BE-NEXT:    vrev64.16 q2, q0
+; CHECK-BE-NEXT:    vrev32.16 q1, q1
+; CHECK-BE-NEXT:    vcmp.i16 eq, q2, zr
+; CHECK-BE-NEXT:    vpsel q1, q4, q1
+; CHECK-BE-NEXT:    vstr p0, [sp, #4] @ 4-byte Spill
+; CHECK-BE-NEXT:    vrev64.16 q0, q1
+; CHECK-BE-NEXT:    bl ext_i16
+; CHECK-BE-NEXT:    vldr p0, [sp, #4] @ 4-byte Reload
+; CHECK-BE-NEXT:    vrev64.16 q1, q0
+; CHECK-BE-NEXT:    vpsel q1, q4, q1
+; CHECK-BE-NEXT:    vrev64.16 q0, q1
+; CHECK-BE-NEXT:    add sp, #8
+; CHECK-BE-NEXT:    vpop {d8, d9}
+; CHECK-BE-NEXT:    pop {r7, pc}
+entry:
+  %c = icmp eq <8 x i16> %src, zeroinitializer
+  %s1 = select <8 x i1> %c, <8 x i16> %a, <8 x i16> zeroinitializer
+  %ext = call arm_aapcs_vfpcc <8 x i16> @ext_i16(<8 x i16> %s1)
+  %s = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %ext
+  ret <8 x i16> %s
+}
+
+define arm_aapcs_vfpcc <16 x i8> @shuffle1_v16i8(<16 x i8> %src, <16 x i8> %a) {
+; CHECK-LE-LABEL: shuffle1_v16i8:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    .save {r7, lr}
+; CHECK-LE-NEXT:    push {r7, lr}
+; CHECK-LE-NEXT:    .vsave {d8, d9}
+; CHECK-LE-NEXT:    vpush {d8, d9}
+; CHECK-LE-NEXT:    .pad #8
+; CHECK-LE-NEXT:    sub sp, #8
+; CHECK-LE-NEXT:    vcmp.i8 eq, q0, zr
+; CHECK-LE-NEXT:    vmov.i32 q0, #0x0
+; CHECK-LE-NEXT:    vpsel q0, q1, q0
+; CHECK-LE-NEXT:    vmov q4, q1
+; CHECK-LE-NEXT:    vstr p0, [sp, #4] @ 4-byte Spill
+; CHECK-LE-NEXT:    bl ext_i8
+; CHECK-LE-NEXT:    vldr p0, [sp, #4] @ 4-byte Reload
+; CHECK-LE-NEXT:    vpsel q0, q4, q0
+; CHECK-LE-NEXT:    add sp, #8
+; CHECK-LE-NEXT:    vpop {d8, d9}
+; CHECK-LE-NEXT:    pop {r7, pc}
+;
+; CHECK-BE-LABEL: shuffle1_v16i8:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    .save {r7, lr}
+; CHECK-BE-NEXT:    push {r7, lr}
+; CHECK-BE-NEXT:    .vsave {d8, d9}
+; CHECK-BE-NEXT:    vpush {d8, d9}
+; CHECK-BE-NEXT:    .pad #8
+; CHECK-BE-NEXT:    sub sp, #8
+; CHECK-BE-NEXT:    vrev64.8 q4, q1
+; CHECK-BE-NEXT:    vmov.i32 q1, #0x0
+; CHECK-BE-NEXT:    vrev64.8 q2, q0
+; CHECK-BE-NEXT:    vrev32.8 q1, q1
+; CHECK-BE-NEXT:    vcmp.i8 eq, q2, zr
+; CHECK-BE-NEXT:    vpsel q1, q4, q1
+; CHECK-BE-NEXT:    vstr p0, [sp, #4] @ 4-byte Spill
+; CHECK-BE-NEXT:    vrev64.8 q0, q1
+; CHECK-BE-NEXT:    bl ext_i8
+; CHECK-BE-NEXT:    vldr p0, [sp, #4] @ 4-byte Reload
+; CHECK-BE-NEXT:    vrev64.8 q1, q0
+; CHECK-BE-NEXT:    vpsel q1, q4, q1
+; CHECK-BE-NEXT:    vrev64.8 q0, q1
+; CHECK-BE-NEXT:    add sp, #8
+; CHECK-BE-NEXT:    vpop {d8, d9}
+; CHECK-BE-NEXT:    pop {r7, pc}
+entry:
+  %c = icmp eq <16 x i8> %src, zeroinitializer
+  %s1 = select <16 x i1> %c, <16 x i8> %a, <16 x i8> zeroinitializer
+  %ext = call arm_aapcs_vfpcc <16 x i8> @ext_i8(<16 x i8> %s1)
+  %s = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %ext
+  ret <16 x i8> %s
+}
diff --git a/llvm/test/CodeGen/Thumb2/mve-pred-threshold.ll b/llvm/test/CodeGen/Thumb2/mve-pred-threshold.ll
new file mode 100644
index 000000000..3d4779ff4
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-pred-threshold.ll
@@ -0,0 +1,615 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve.fp -enable-arm-maskedldst -verify-machineinstrs %s -o - | FileCheck %s
+
+define arm_aapcs_vfpcc void @thres_i32(i32* %data, i16 zeroext %N, i32 %T) {
+; CHECK-LABEL: thres_i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    cmp r1, #0
+; CHECK-NEXT:    it eq
+; CHECK-NEXT:    popeq {r7, pc}
+; CHECK-NEXT:    mvn r3, #3
+; CHECK-NEXT:    add.w r1, r3, r1, lsl #2
+; CHECK-NEXT:    movs r3, #1
+; CHECK-NEXT:    vmov.i32 q0, #0x0
+; CHECK-NEXT:    add.w lr, r3, r1, lsr #2
+; CHECK-NEXT:    rsbs r1, r2, #0
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB0_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vpt.s32 ge, q1, r2
+; CHECK-NEXT:    vcmpt.s32 le, q1, r1
+; CHECK-NEXT:    vpnot
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrwt.32 q0, [r0], #16
+; CHECK-NEXT:    le lr, .LBB0_1
+; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %conv = zext i16 %N to i32
+  %mul = shl nuw nsw i32 %conv, 2
+  %cmp15 = icmp eq i16 %N, 0
+  br i1 %cmp15, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %sub = sub nsw i32 0, %T
+  %broadcast.splatinsert17 = insertelement <4 x i32> undef, i32 %T, i32 0
+  %broadcast.splat18 = shufflevector <4 x i32> %broadcast.splatinsert17, <4 x i32> undef, <4 x i32> zeroinitializer
+  %broadcast.splatinsert19 = insertelement <4 x i32> undef, i32 %sub, i32 0
+  %broadcast.splat20 = shufflevector <4 x i32> %broadcast.splatinsert19, <4 x i32> undef, <4 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %0 = getelementptr inbounds i32, i32* %data, i32 %index
+  %1 = bitcast i32* %0 to <4 x i32>*
+  %wide.load = load <4 x i32>, <4 x i32>* %1, align 4
+  %2 = icmp slt <4 x i32> %wide.load, %broadcast.splat18
+  %3 = icmp sgt <4 x i32> %wide.load, %broadcast.splat20
+  %4 = or <4 x i1> %2, %3
+  %5 = bitcast i32* %0 to <4 x i32>*
+  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> zeroinitializer, <4 x i32>* %5, i32 4, <4 x i1> %4)
+  %index.next = add i32 %index, 4
+  %6 = icmp eq i32 %index.next, %mul
+  br i1 %6, label %for.cond.cleanup, label %vector.body
+
+for.cond.cleanup:                                 ; preds = %vector.body, %entry
+  ret void
+}
+
+define arm_aapcs_vfpcc void @thresh_i16(i16* %data, i16 zeroext %N, i16 signext %T) {
+; CHECK-LABEL: thresh_i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    cmp r1, #0
+; CHECK-NEXT:    it eq
+; CHECK-NEXT:    popeq {r7, pc}
+; CHECK-NEXT:    mvn r3, #7
+; CHECK-NEXT:    add.w r1, r3, r1, lsl #3
+; CHECK-NEXT:    movs r3, #1
+; CHECK-NEXT:    vmov.i32 q0, #0x0
+; CHECK-NEXT:    add.w lr, r3, r1, lsr #3
+; CHECK-NEXT:    rsbs r1, r2, #0
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB1_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrh.u16 q1, [r0]
+; CHECK-NEXT:    vpt.s16 ge, q1, r2
+; CHECK-NEXT:    vcmpt.s16 le, q1, r1
+; CHECK-NEXT:    vpnot
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrht.16 q0, [r0], #16
+; CHECK-NEXT:    le lr, .LBB1_1
+; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %conv2 = zext i16 %N to i32
+  %mul = shl nuw nsw i32 %conv2, 3
+  %cmp22 = icmp eq i16 %N, 0
+  br i1 %cmp22, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %sub = sub i16 0, %T
+  %broadcast.splatinsert24 = insertelement <8 x i16> undef, i16 %T, i32 0
+  %broadcast.splat25 = shufflevector <8 x i16> %broadcast.splatinsert24, <8 x i16> undef, <8 x i32> zeroinitializer
+  %broadcast.splatinsert26 = insertelement <8 x i16> undef, i16 %sub, i32 0
+  %broadcast.splat27 = shufflevector <8 x i16> %broadcast.splatinsert26, <8 x i16> undef, <8 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %0 = getelementptr inbounds i16, i16* %data, i32 %index
+  %1 = bitcast i16* %0 to <8 x i16>*
+  %wide.load = load <8 x i16>, <8 x i16>* %1, align 2
+  %2 = icmp slt <8 x i16> %wide.load, %broadcast.splat25
+  %3 = icmp sgt <8 x i16> %wide.load, %broadcast.splat27
+  %4 = or <8 x i1> %2, %3
+  %5 = bitcast i16* %0 to <8 x i16>*
+  call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> zeroinitializer, <8 x i16>* %5, i32 2, <8 x i1> %4)
+  %index.next = add i32 %index, 8
+  %6 = icmp eq i32 %index.next, %mul
+  br i1 %6, label %for.cond.cleanup, label %vector.body
+
+for.cond.cleanup:                                 ; preds = %vector.body, %entry
+  ret void
+}
+
+define arm_aapcs_vfpcc void @thresh_i8(i8* %data, i16 zeroext %N, i8 signext %T) {
+; CHECK-LABEL: thresh_i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    cmp r1, #0
+; CHECK-NEXT:    it eq
+; CHECK-NEXT:    popeq {r7, pc}
+; CHECK-NEXT:    mvn r3, #15
+; CHECK-NEXT:    add.w r1, r3, r1, lsl #4
+; CHECK-NEXT:    movs r3, #1
+; CHECK-NEXT:    vmov.i32 q0, #0x0
+; CHECK-NEXT:    add.w lr, r3, r1, lsr #4
+; CHECK-NEXT:    rsbs r1, r2, #0
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB2_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrb.u8 q1, [r0]
+; CHECK-NEXT:    vpt.s8 ge, q1, r2
+; CHECK-NEXT:    vcmpt.s8 le, q1, r1
+; CHECK-NEXT:    vpnot
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrbt.8 q0, [r0], #16
+; CHECK-NEXT:    le lr, .LBB2_1
+; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %conv2 = zext i16 %N to i32
+  %mul = shl nuw nsw i32 %conv2, 4
+  %cmp20 = icmp eq i16 %N, 0
+  br i1 %cmp20, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %sub = sub i8 0, %T
+  %broadcast.splatinsert22 = insertelement <16 x i8> undef, i8 %T, i32 0
+  %broadcast.splat23 = shufflevector <16 x i8> %broadcast.splatinsert22, <16 x i8> undef, <16 x i32> zeroinitializer
+  %broadcast.splatinsert24 = insertelement <16 x i8> undef, i8 %sub, i32 0
+  %broadcast.splat25 = shufflevector <16 x i8> %broadcast.splatinsert24, <16 x i8> undef, <16 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %0 = getelementptr inbounds i8, i8* %data, i32 %index
+  %1 = bitcast i8* %0 to <16 x i8>*
+  %wide.load = load <16 x i8>, <16 x i8>* %1, align 1
+  %2 = icmp slt <16 x i8> %wide.load, %broadcast.splat23
+  %3 = icmp sgt <16 x i8> %wide.load, %broadcast.splat25
+  %4 = or <16 x i1> %2, %3
+  %5 = bitcast i8* %0 to <16 x i8>*
+  call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> zeroinitializer, <16 x i8>* %5, i32 1, <16 x i1> %4)
+  %index.next = add i32 %index, 16
+  %6 = icmp eq i32 %index.next, %mul
+  br i1 %6, label %for.cond.cleanup, label %vector.body
+
+for.cond.cleanup:                                 ; preds = %vector.body, %entry
+  ret void
+}
+
+define arm_aapcs_vfpcc void @thresh_f32(float* %data, i16 zeroext %N, float %T) {
+; CHECK-LABEL: thresh_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    cmp r1, #0
+; CHECK-NEXT:    it eq
+; CHECK-NEXT:    popeq {r7, pc}
+; CHECK-NEXT:    vneg.f32 s4, s0
+; CHECK-NEXT:    mvn r2, #3
+; CHECK-NEXT:    add.w r1, r2, r1, lsl #2
+; CHECK-NEXT:    movs r2, #1
+; CHECK-NEXT:    vmov.i32 q2, #0x0
+; CHECK-NEXT:    add.w lr, r2, r1, lsr #2
+; CHECK-NEXT:    vmov r1, s0
+; CHECK-NEXT:    vdup.32 q0, r1
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:    vmov r1, s4
+; CHECK-NEXT:    vdup.32 q1, r1
+; CHECK-NEXT:  .LBB3_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrw.u32 q3, [r0]
+; CHECK-NEXT:    vpt.f32 le, q0, q3
+; CHECK-NEXT:    vcmpt.f32 le, q3, q1
+; CHECK-NEXT:    vpnot
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrwt.32 q2, [r0], #16
+; CHECK-NEXT:    le lr, .LBB3_1
+; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %conv = zext i16 %N to i32
+  %mul = shl nuw nsw i32 %conv, 2
+  %cmp15 = icmp eq i16 %N, 0
+  br i1 %cmp15, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %fneg = fneg fast float %T
+  %broadcast.splatinsert17 = insertelement <4 x float> undef, float %T, i32 0
+  %broadcast.splat18 = shufflevector <4 x float> %broadcast.splatinsert17, <4 x float> undef, <4 x i32> zeroinitializer
+  %broadcast.splatinsert19 = insertelement <4 x float> undef, float %fneg, i32 0
+  %broadcast.splat20 = shufflevector <4 x float> %broadcast.splatinsert19, <4 x float> undef, <4 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %0 = getelementptr inbounds float, float* %data, i32 %index
+  %1 = bitcast float* %0 to <4 x float>*
+  %wide.load = load <4 x float>, <4 x float>* %1, align 4
+  %2 = fcmp fast olt <4 x float> %wide.load, %broadcast.splat18
+  %3 = fcmp fast ogt <4 x float> %wide.load, %broadcast.splat20
+  %4 = or <4 x i1> %2, %3
+  %5 = bitcast float* %0 to <4 x float>*
+  call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> zeroinitializer, <4 x float>* %5, i32 4, <4 x i1> %4)
+  %index.next = add i32 %index, 4
+  %6 = icmp eq i32 %index.next, %mul
+  br i1 %6, label %for.cond.cleanup, label %vector.body
+
+for.cond.cleanup:                                 ; preds = %vector.body, %entry
+  ret void
+}
+
+define arm_aapcs_vfpcc void @thresh_f16(half* %data, i16 zeroext %N, float %T.coerce) {
+; CHECK-LABEL: thresh_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    cmp r1, #0
+; CHECK-NEXT:    it eq
+; CHECK-NEXT:    popeq {r7, pc}
+; CHECK-NEXT:    mvn r2, #7
+; CHECK-NEXT:    add.w r1, r2, r1, lsl #3
+; CHECK-NEXT:    movs r2, #1
+; CHECK-NEXT:    vmov.i32 q2, #0x0
+; CHECK-NEXT:    add.w lr, r2, r1, lsr #3
+; CHECK-NEXT:    vmov r1, s0
+; CHECK-NEXT:    vneg.f16 s0, s0
+; CHECK-NEXT:    vdup.16 q1, r1
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:    vdup.16 q0, r2
+; CHECK-NEXT:  .LBB4_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrh.u16 q3, [r0]
+; CHECK-NEXT:    vpt.f16 le, q1, q3
+; CHECK-NEXT:    vcmpt.f16 le, q3, q0
+; CHECK-NEXT:    vpnot
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrht.16 q2, [r0], #16
+; CHECK-NEXT:    le lr, .LBB4_1
+; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %0 = bitcast float %T.coerce to i32
+  %tmp.0.extract.trunc = trunc i32 %0 to i16
+  %1 = bitcast i16 %tmp.0.extract.trunc to half
+  %conv = zext i16 %N to i32
+  %mul = shl nuw nsw i32 %conv, 3
+  %cmp17 = icmp eq i16 %N, 0
+  br i1 %cmp17, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %fneg = fneg fast half %1
+  %broadcast.splatinsert19 = insertelement <8 x half> undef, half %1, i32 0
+  %broadcast.splat20 = shufflevector <8 x half> %broadcast.splatinsert19, <8 x half> undef, <8 x i32> zeroinitializer
+  %broadcast.splatinsert21 = insertelement <8 x half> undef, half %fneg, i32 0
+  %broadcast.splat22 = shufflevector <8 x half> %broadcast.splatinsert21, <8 x half> undef, <8 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %2 = getelementptr inbounds half, half* %data, i32 %index
+  %3 = bitcast half* %2 to <8 x half>*
+  %wide.load = load <8 x half>, <8 x half>* %3, align 2
+  %4 = fcmp fast olt <8 x half> %wide.load, %broadcast.splat20
+  %5 = fcmp fast ogt <8 x half> %wide.load, %broadcast.splat22
+  %6 = or <8 x i1> %4, %5
+  %7 = bitcast half* %2 to <8 x half>*
+  call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> zeroinitializer, <8 x half>* %7, i32 2, <8 x i1> %6)
+  %index.next = add i32 %index, 8
+  %8 = icmp eq i32 %index.next, %mul
+  br i1 %8, label %for.cond.cleanup, label %vector.body
+
+for.cond.cleanup:                                 ; preds = %vector.body, %entry
+  ret void
+}
+
+
+
+define arm_aapcs_vfpcc void @thres_rev_i32(i32* %data, i16 zeroext %N, i32 %T) {
+; CHECK-LABEL: thres_rev_i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    cmp r1, #0
+; CHECK-NEXT:    it eq
+; CHECK-NEXT:    popeq {r7, pc}
+; CHECK-NEXT:    mvn r3, #3
+; CHECK-NEXT:    add.w r1, r3, r1, lsl #2
+; CHECK-NEXT:    movs r3, #1
+; CHECK-NEXT:    vmov.i32 q0, #0x0
+; CHECK-NEXT:    add.w lr, r3, r1, lsr #2
+; CHECK-NEXT:    rsbs r1, r2, #0
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB5_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vpt.s32 ge, q1, r2
+; CHECK-NEXT:    vcmpt.s32 le, q1, r1
+; CHECK-NEXT:    vpnot
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrwt.32 q0, [r0], #16
+; CHECK-NEXT:    le lr, .LBB5_1
+; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %conv = zext i16 %N to i32
+  %mul = shl nuw nsw i32 %conv, 2
+  %cmp15 = icmp eq i16 %N, 0
+  br i1 %cmp15, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %sub = sub nsw i32 0, %T
+  %broadcast.splatinsert17 = insertelement <4 x i32> undef, i32 %T, i32 0
+  %broadcast.splat18 = shufflevector <4 x i32> %broadcast.splatinsert17, <4 x i32> undef, <4 x i32> zeroinitializer
+  %broadcast.splatinsert19 = insertelement <4 x i32> undef, i32 %sub, i32 0
+  %broadcast.splat20 = shufflevector <4 x i32> %broadcast.splatinsert19, <4 x i32> undef, <4 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %0 = getelementptr inbounds i32, i32* %data, i32 %index
+  %1 = bitcast i32* %0 to <4 x i32>*
+  %wide.load = load <4 x i32>, <4 x i32>* %1, align 4
+  %2 = icmp sgt <4 x i32> %broadcast.splat18, %wide.load
+  %3 = icmp slt <4 x i32> %broadcast.splat20, %wide.load
+  %4 = or <4 x i1> %2, %3
+  %5 = bitcast i32* %0 to <4 x i32>*
+  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> zeroinitializer, <4 x i32>* %5, i32 4, <4 x i1> %4)
+  %index.next = add i32 %index, 4
+  %6 = icmp eq i32 %index.next, %mul
+  br i1 %6, label %for.cond.cleanup, label %vector.body
+
+for.cond.cleanup:                                 ; preds = %vector.body, %entry
+  ret void
+}
+
+define arm_aapcs_vfpcc void @thresh_rev_i16(i16* %data, i16 zeroext %N, i16 signext %T) {
+; CHECK-LABEL: thresh_rev_i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    cmp r1, #0
+; CHECK-NEXT:    it eq
+; CHECK-NEXT:    popeq {r7, pc}
+; CHECK-NEXT:    mvn r3, #7
+; CHECK-NEXT:    add.w r1, r3, r1, lsl #3
+; CHECK-NEXT:    movs r3, #1
+; CHECK-NEXT:    vmov.i32 q0, #0x0
+; CHECK-NEXT:    add.w lr, r3, r1, lsr #3
+; CHECK-NEXT:    rsbs r1, r2, #0
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB6_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrh.u16 q1, [r0]
+; CHECK-NEXT:    vpt.s16 ge, q1, r2
+; CHECK-NEXT:    vcmpt.s16 le, q1, r1
+; CHECK-NEXT:    vpnot
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrht.16 q0, [r0], #16
+; CHECK-NEXT:    le lr, .LBB6_1
+; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %conv2 = zext i16 %N to i32
+  %mul = shl nuw nsw i32 %conv2, 3
+  %cmp22 = icmp eq i16 %N, 0
+  br i1 %cmp22, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %sub = sub i16 0, %T
+  %broadcast.splatinsert24 = insertelement <8 x i16> undef, i16 %T, i32 0
+  %broadcast.splat25 = shufflevector <8 x i16> %broadcast.splatinsert24, <8 x i16> undef, <8 x i32> zeroinitializer
+  %broadcast.splatinsert26 = insertelement <8 x i16> undef, i16 %sub, i32 0
+  %broadcast.splat27 = shufflevector <8 x i16> %broadcast.splatinsert26, <8 x i16> undef, <8 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %0 = getelementptr inbounds i16, i16* %data, i32 %index
+  %1 = bitcast i16* %0 to <8 x i16>*
+  %wide.load = load <8 x i16>, <8 x i16>* %1, align 2
+  %2 = icmp sgt <8 x i16> %broadcast.splat25, %wide.load
+  %3 = icmp slt <8 x i16> %broadcast.splat27, %wide.load
+  %4 = or <8 x i1> %2, %3
+  %5 = bitcast i16* %0 to <8 x i16>*
+  call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> zeroinitializer, <8 x i16>* %5, i32 2, <8 x i1> %4)
+  %index.next = add i32 %index, 8
+  %6 = icmp eq i32 %index.next, %mul
+  br i1 %6, label %for.cond.cleanup, label %vector.body
+
+for.cond.cleanup:                                 ; preds = %vector.body, %entry
+  ret void
+}
+
+define arm_aapcs_vfpcc void @thresh_rev_i8(i8* %data, i16 zeroext %N, i8 signext %T) {
+; CHECK-LABEL: thresh_rev_i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    cmp r1, #0
+; CHECK-NEXT:    it eq
+; CHECK-NEXT:    popeq {r7, pc}
+; CHECK-NEXT:    mvn r3, #15
+; CHECK-NEXT:    add.w r1, r3, r1, lsl #4
+; CHECK-NEXT:    movs r3, #1
+; CHECK-NEXT:    vmov.i32 q0, #0x0
+; CHECK-NEXT:    add.w lr, r3, r1, lsr #4
+; CHECK-NEXT:    rsbs r1, r2, #0
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB7_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrb.u8 q1, [r0]
+; CHECK-NEXT:    vpt.s8 ge, q1, r2
+; CHECK-NEXT:    vcmpt.s8 le, q1, r1
+; CHECK-NEXT:    vpnot
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrbt.8 q0, [r0], #16
+; CHECK-NEXT:    le lr, .LBB7_1
+; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %conv2 = zext i16 %N to i32
+  %mul = shl nuw nsw i32 %conv2, 4
+  %cmp20 = icmp eq i16 %N, 0
+  br i1 %cmp20, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %sub = sub i8 0, %T
+  %broadcast.splatinsert22 = insertelement <16 x i8> undef, i8 %T, i32 0
+  %broadcast.splat23 = shufflevector <16 x i8> %broadcast.splatinsert22, <16 x i8> undef, <16 x i32> zeroinitializer
+  %broadcast.splatinsert24 = insertelement <16 x i8> undef, i8 %sub, i32 0
+  %broadcast.splat25 = shufflevector <16 x i8> %broadcast.splatinsert24, <16 x i8> undef, <16 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %0 = getelementptr inbounds i8, i8* %data, i32 %index
+  %1 = bitcast i8* %0 to <16 x i8>*
+  %wide.load = load <16 x i8>, <16 x i8>* %1, align 1
+  %2 = icmp sgt <16 x i8> %broadcast.splat23, %wide.load
+  %3 = icmp slt <16 x i8> %broadcast.splat25, %wide.load
+  %4 = or <16 x i1> %2, %3
+  %5 = bitcast i8* %0 to <16 x i8>*
+  call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> zeroinitializer, <16 x i8>* %5, i32 1, <16 x i1> %4)
+  %index.next = add i32 %index, 16
+  %6 = icmp eq i32 %index.next, %mul
+  br i1 %6, label %for.cond.cleanup, label %vector.body
+
+for.cond.cleanup:                                 ; preds = %vector.body, %entry
+  ret void
+}
+
+define arm_aapcs_vfpcc void @thresh_rev_f32(float* %data, i16 zeroext %N, float %T) {
+; CHECK-LABEL: thresh_rev_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    cmp r1, #0
+; CHECK-NEXT:    it eq
+; CHECK-NEXT:    popeq {r7, pc}
+; CHECK-NEXT:    vneg.f32 s4, s0
+; CHECK-NEXT:    mvn r2, #3
+; CHECK-NEXT:    add.w r1, r2, r1, lsl #2
+; CHECK-NEXT:    movs r2, #1
+; CHECK-NEXT:    vmov.i32 q2, #0x0
+; CHECK-NEXT:    add.w lr, r2, r1, lsr #2
+; CHECK-NEXT:    vmov r1, s0
+; CHECK-NEXT:    vdup.32 q0, r1
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:    vmov r1, s4
+; CHECK-NEXT:    vdup.32 q1, r1
+; CHECK-NEXT:  .LBB8_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrw.u32 q3, [r0]
+; CHECK-NEXT:    vpt.f32 le, q0, q3
+; CHECK-NEXT:    vcmpt.f32 le, q3, q1
+; CHECK-NEXT:    vpnot
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrwt.32 q2, [r0], #16
+; CHECK-NEXT:    le lr, .LBB8_1
+; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %conv = zext i16 %N to i32
+  %mul = shl nuw nsw i32 %conv, 2
+  %cmp15 = icmp eq i16 %N, 0
+  br i1 %cmp15, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %fneg = fneg fast float %T
+  %broadcast.splatinsert17 = insertelement <4 x float> undef, float %T, i32 0
+  %broadcast.splat18 = shufflevector <4 x float> %broadcast.splatinsert17, <4 x float> undef, <4 x i32> zeroinitializer
+  %broadcast.splatinsert19 = insertelement <4 x float> undef, float %fneg, i32 0
+  %broadcast.splat20 = shufflevector <4 x float> %broadcast.splatinsert19, <4 x float> undef, <4 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %0 = getelementptr inbounds float, float* %data, i32 %index
+  %1 = bitcast float* %0 to <4 x float>*
+  %wide.load = load <4 x float>, <4 x float>* %1, align 4
+  %2 = fcmp fast ogt <4 x float> %broadcast.splat18, %wide.load
+  %3 = fcmp fast olt <4 x float> %broadcast.splat20, %wide.load
+  %4 = or <4 x i1> %2, %3
+  %5 = bitcast float* %0 to <4 x float>*
+  call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> zeroinitializer, <4 x float>* %5, i32 4, <4 x i1> %4)
+  %index.next = add i32 %index, 4
+  %6 = icmp eq i32 %index.next, %mul
+  br i1 %6, label %for.cond.cleanup, label %vector.body
+
+for.cond.cleanup:                                 ; preds = %vector.body, %entry
+  ret void
+}
+
+define arm_aapcs_vfpcc void @thresh_rev_f16(half* %data, i16 zeroext %N, float %T.coerce) {
+; CHECK-LABEL: thresh_rev_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    cmp r1, #0
+; CHECK-NEXT:    it eq
+; CHECK-NEXT:    popeq {r7, pc}
+; CHECK-NEXT:    mvn r2, #7
+; CHECK-NEXT:    add.w r1, r2, r1, lsl #3
+; CHECK-NEXT:    movs r2, #1
+; CHECK-NEXT:    vmov.i32 q2, #0x0
+; CHECK-NEXT:    add.w lr, r2, r1, lsr #3
+; CHECK-NEXT:    vmov r1, s0
+; CHECK-NEXT:    vneg.f16 s0, s0
+; CHECK-NEXT:    vdup.16 q1, r1
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:    vdup.16 q0, r2
+; CHECK-NEXT:  .LBB9_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrh.u16 q3, [r0]
+; CHECK-NEXT:    vpt.f16 le, q1, q3
+; CHECK-NEXT:    vcmpt.f16 le, q3, q0
+; CHECK-NEXT:    vpnot
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrht.16 q2, [r0], #16
+; CHECK-NEXT:    le lr, .LBB9_1
+; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %0 = bitcast float %T.coerce to i32
+  %tmp.0.extract.trunc = trunc i32 %0 to i16
+  %1 = bitcast i16 %tmp.0.extract.trunc to half
+  %conv = zext i16 %N to i32
+  %mul = shl nuw nsw i32 %conv, 3
+  %cmp17 = icmp eq i16 %N, 0
+  br i1 %cmp17, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %fneg = fneg fast half %1
+  %broadcast.splatinsert19 = insertelement <8 x half> undef, half %1, i32 0
+  %broadcast.splat20 = shufflevector <8 x half> %broadcast.splatinsert19, <8 x half> undef, <8 x i32> zeroinitializer
+  %broadcast.splatinsert21 = insertelement <8 x half> undef, half %fneg, i32 0
+  %broadcast.splat22 = shufflevector <8 x half> %broadcast.splatinsert21, <8 x half> undef, <8 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %2 = getelementptr inbounds half, half* %data, i32 %index
+  %3 = bitcast half* %2 to <8 x half>*
+  %wide.load = load <8 x half>, <8 x half>* %3, align 2
+  %4 = fcmp fast ogt <8 x half> %broadcast.splat20, %wide.load
+  %5 = fcmp fast olt <8 x half> %broadcast.splat22, %wide.load
+  %6 = or <8 x i1> %4, %5
+  %7 = bitcast half* %2 to <8 x half>*
+  call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> zeroinitializer, <8 x half>* %7, i32 2, <8 x i1> %6)
+  %index.next = add i32 %index, 8
+  %8 = icmp eq i32 %index.next, %mul
+  br i1 %8, label %for.cond.cleanup, label %vector.body
+
+for.cond.cleanup:                                 ; preds = %vector.body, %entry
+  ret void
+}
+
+
+
+
+declare void @llvm.masked.store.v4i32.p0v4i32(<4 x i32>, <4 x i32>*, i32 immarg, <4 x i1>)
+declare void @llvm.masked.store.v8i16.p0v8i16(<8 x i16>, <8 x i16>*, i32 immarg, <8 x i1>)
+declare void @llvm.masked.store.v16i8.p0v16i8(<16 x i8>, <16 x i8>*, i32 immarg, <16 x i1>)
+declare void @llvm.masked.store.v4f32.p0v4f32(<4 x float>, <4 x float>*, i32 immarg, <4 x i1>)
+declare void @llvm.masked.store.v8f16.p0v8f16(<8 x half>, <8 x half>*, i32 immarg, <8 x i1>)
diff --git a/llvm/test/CodeGen/Thumb2/mve-pred-xor.ll b/llvm/test/CodeGen/Thumb2/mve-pred-xor.ll
new file mode 100644
index 000000000..07821a29e
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-pred-xor.ll
@@ -0,0 +1,561 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s
+
+define arm_aapcs_vfpcc <4 x i32> @cmpeqz_v4i1(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: cmpeqz_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i32 eq, q1, zr
+; CHECK-NEXT:    vmrs r0, p0
+; CHECK-NEXT:    vcmp.i32 eq, q0, zr
+; CHECK-NEXT:    vmrs r1, p0
+; CHECK-NEXT:    eors r0, r1
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %c2 = icmp eq <4 x i32> %b, zeroinitializer
+  %o = xor <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpnez_v4i1(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: cmpnez_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i32 ne, q1, zr
+; CHECK-NEXT:    vmrs r0, p0
+; CHECK-NEXT:    vcmp.i32 eq, q0, zr
+; CHECK-NEXT:    vmrs r1, p0
+; CHECK-NEXT:    eors r0, r1
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %c2 = icmp ne <4 x i32> %b, zeroinitializer
+  %o = xor <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpsltz_v4i1(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: cmpsltz_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.s32 lt, q1, zr
+; CHECK-NEXT:    vmrs r0, p0
+; CHECK-NEXT:    vcmp.i32 eq, q0, zr
+; CHECK-NEXT:    vmrs r1, p0
+; CHECK-NEXT:    eors r0, r1
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %c2 = icmp slt <4 x i32> %b, zeroinitializer
+  %o = xor <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpsgtz_v4i1(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: cmpsgtz_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.s32 gt, q1, zr
+; CHECK-NEXT:    vmrs r0, p0
+; CHECK-NEXT:    vcmp.i32 eq, q0, zr
+; CHECK-NEXT:    vmrs r1, p0
+; CHECK-NEXT:    eors r0, r1
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %c2 = icmp sgt <4 x i32> %b, zeroinitializer
+  %o = xor <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpslez_v4i1(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: cmpslez_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.s32 le, q1, zr
+; CHECK-NEXT:    vmrs r0, p0
+; CHECK-NEXT:    vcmp.i32 eq, q0, zr
+; CHECK-NEXT:    vmrs r1, p0
+; CHECK-NEXT:    eors r0, r1
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %c2 = icmp sle <4 x i32> %b, zeroinitializer
+  %o = xor <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpsgez_v4i1(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: cmpsgez_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.s32 ge, q1, zr
+; CHECK-NEXT:    vmrs r0, p0
+; CHECK-NEXT:    vcmp.i32 eq, q0, zr
+; CHECK-NEXT:    vmrs r1, p0
+; CHECK-NEXT:    eors r0, r1
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %c2 = icmp sge <4 x i32> %b, zeroinitializer
+  %o = xor <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpultz_v4i1(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: cmpultz_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i32 eq, q0, zr
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %c2 = icmp ult <4 x i32> %b, zeroinitializer
+  %o = xor <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpugtz_v4i1(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: cmpugtz_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i32 ne, q1, zr
+; CHECK-NEXT:    vmrs r0, p0
+; CHECK-NEXT:    vcmp.i32 eq, q0, zr
+; CHECK-NEXT:    vmrs r1, p0
+; CHECK-NEXT:    eors r0, r1
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %c2 = icmp ugt <4 x i32> %b, zeroinitializer
+  %o = xor <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpulez_v4i1(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: cmpulez_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.u32 cs, q1, zr
+; CHECK-NEXT:    vmrs r0, p0
+; CHECK-NEXT:    vcmp.i32 eq, q0, zr
+; CHECK-NEXT:    vmrs r1, p0
+; CHECK-NEXT:    eors r0, r1
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %c2 = icmp ule <4 x i32> %b, zeroinitializer
+  %o = xor <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpugez_v4i1(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: cmpugez_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i32 ne, q0, zr
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %c2 = icmp uge <4 x i32> %b, zeroinitializer
+  %o = xor <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+
+
+define arm_aapcs_vfpcc <4 x i32> @cmpeq_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LABEL: cmpeq_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i32 eq, q1, q2
+; CHECK-NEXT:    vmrs r0, p0
+; CHECK-NEXT:    vcmp.i32 eq, q0, zr
+; CHECK-NEXT:    vmrs r1, p0
+; CHECK-NEXT:    eors r0, r1
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %c2 = icmp eq <4 x i32> %b, %c
+  %o = xor <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpne_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LABEL: cmpne_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i32 ne, q1, q2
+; CHECK-NEXT:    vmrs r0, p0
+; CHECK-NEXT:    vcmp.i32 eq, q0, zr
+; CHECK-NEXT:    vmrs r1, p0
+; CHECK-NEXT:    eors r0, r1
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %c2 = icmp ne <4 x i32> %b, %c
+  %o = xor <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpslt_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LABEL: cmpslt_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.s32 gt, q2, q1
+; CHECK-NEXT:    vmrs r0, p0
+; CHECK-NEXT:    vcmp.i32 eq, q0, zr
+; CHECK-NEXT:    vmrs r1, p0
+; CHECK-NEXT:    eors r0, r1
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %c2 = icmp slt <4 x i32> %b, %c
+  %o = xor <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpsgt_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LABEL: cmpsgt_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.s32 gt, q1, q2
+; CHECK-NEXT:    vmrs r0, p0
+; CHECK-NEXT:    vcmp.i32 eq, q0, zr
+; CHECK-NEXT:    vmrs r1, p0
+; CHECK-NEXT:    eors r0, r1
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %c2 = icmp sgt <4 x i32> %b, %c
+  %o = xor <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpsle_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LABEL: cmpsle_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.s32 ge, q2, q1
+; CHECK-NEXT:    vmrs r0, p0
+; CHECK-NEXT:    vcmp.i32 eq, q0, zr
+; CHECK-NEXT:    vmrs r1, p0
+; CHECK-NEXT:    eors r0, r1
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %c2 = icmp sle <4 x i32> %b, %c
+  %o = xor <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpsge_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LABEL: cmpsge_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.s32 ge, q1, q2
+; CHECK-NEXT:    vmrs r0, p0
+; CHECK-NEXT:    vcmp.i32 eq, q0, zr
+; CHECK-NEXT:    vmrs r1, p0
+; CHECK-NEXT:    eors r0, r1
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %c2 = icmp sge <4 x i32> %b, %c
+  %o = xor <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpult_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LABEL: cmpult_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.u32 hi, q2, q1
+; CHECK-NEXT:    vmrs r0, p0
+; CHECK-NEXT:    vcmp.i32 eq, q0, zr
+; CHECK-NEXT:    vmrs r1, p0
+; CHECK-NEXT:    eors r0, r1
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %c2 = icmp ult <4 x i32> %b, %c
+  %o = xor <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpugt_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LABEL: cmpugt_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.u32 hi, q1, q2
+; CHECK-NEXT:    vmrs r0, p0
+; CHECK-NEXT:    vcmp.i32 eq, q0, zr
+; CHECK-NEXT:    vmrs r1, p0
+; CHECK-NEXT:    eors r0, r1
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %c2 = icmp ugt <4 x i32> %b, %c
+  %o = xor <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpule_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LABEL: cmpule_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.u32 cs, q2, q1
+; CHECK-NEXT:    vmrs r0, p0
+; CHECK-NEXT:    vcmp.i32 eq, q0, zr
+; CHECK-NEXT:    vmrs r1, p0
+; CHECK-NEXT:    eors r0, r1
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %c2 = icmp ule <4 x i32> %b, %c
+  %o = xor <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cmpuge_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LABEL: cmpuge_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.u32 cs, q1, q2
+; CHECK-NEXT:    vmrs r0, p0
+; CHECK-NEXT:    vcmp.i32 eq, q0, zr
+; CHECK-NEXT:    vmrs r1, p0
+; CHECK-NEXT:    eors r0, r1
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <4 x i32> %a, zeroinitializer
+  %c2 = icmp uge <4 x i32> %b, %c
+  %o = xor <4 x i1> %c1, %c2
+  %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+
+
+
+define arm_aapcs_vfpcc <8 x i16> @cmpeqz_v8i1(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: cmpeqz_v8i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i16 eq, q1, zr
+; CHECK-NEXT:    vmrs r0, p0
+; CHECK-NEXT:    vcmp.i16 eq, q0, zr
+; CHECK-NEXT:    vmrs r1, p0
+; CHECK-NEXT:    eors r0, r1
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <8 x i16> %a, zeroinitializer
+  %c2 = icmp eq <8 x i16> %b, zeroinitializer
+  %o = xor <8 x i1> %c1, %c2
+  %s = select <8 x i1> %o, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+define arm_aapcs_vfpcc <8 x i16> @cmpeq_v8i1(<8 x i16> %a, <8 x i16> %b, <8 x i16> %c) {
+; CHECK-LABEL: cmpeq_v8i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i16 eq, q1, q2
+; CHECK-NEXT:    vmrs r0, p0
+; CHECK-NEXT:    vcmp.i16 eq, q0, zr
+; CHECK-NEXT:    vmrs r1, p0
+; CHECK-NEXT:    eors r0, r1
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <8 x i16> %a, zeroinitializer
+  %c2 = icmp eq <8 x i16> %b, %c
+  %o = xor <8 x i1> %c1, %c2
+  %s = select <8 x i1> %o, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+
+define arm_aapcs_vfpcc <16 x i8> @cmpeqz_v16i1(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: cmpeqz_v16i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i8 eq, q1, zr
+; CHECK-NEXT:    vmrs r0, p0
+; CHECK-NEXT:    vcmp.i8 eq, q0, zr
+; CHECK-NEXT:    vmrs r1, p0
+; CHECK-NEXT:    eors r0, r1
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <16 x i8> %a, zeroinitializer
+  %c2 = icmp eq <16 x i8> %b, zeroinitializer
+  %o = xor <16 x i1> %c1, %c2
+  %s = select <16 x i1> %o, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %s
+}
+
+define arm_aapcs_vfpcc <16 x i8> @cmpeq_v16i1(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c) {
+; CHECK-LABEL: cmpeq_v16i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i8 eq, q1, q2
+; CHECK-NEXT:    vmrs r0, p0
+; CHECK-NEXT:    vcmp.i8 eq, q0, zr
+; CHECK-NEXT:    vmrs r1, p0
+; CHECK-NEXT:    eors r0, r1
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <16 x i8> %a, zeroinitializer
+  %c2 = icmp eq <16 x i8> %b, %c
+  %o = xor <16 x i1> %c1, %c2
+  %s = select <16 x i1> %o, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %s
+}
+
+
+define arm_aapcs_vfpcc <2 x i64> @cmpeqz_v2i1(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: cmpeqz_v2i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov r0, s5
+; CHECK-NEXT:    vmov r1, s4
+; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    vmov r1, s6
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q2[0], r0
+; CHECK-NEXT:    vmov.32 q2[1], r0
+; CHECK-NEXT:    vmov r0, s7
+; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    vmov r1, s0
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q2[2], r0
+; CHECK-NEXT:    vmov.32 q2[3], r0
+; CHECK-NEXT:    vmov r0, s1
+; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    vmov r1, s2
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q3[0], r0
+; CHECK-NEXT:    vmov.32 q3[1], r0
+; CHECK-NEXT:    vmov r0, s3
+; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q3[2], r0
+; CHECK-NEXT:    vmov.32 q3[3], r0
+; CHECK-NEXT:    veor q2, q3, q2
+; CHECK-NEXT:    vbic q1, q1, q2
+; CHECK-NEXT:    vand q0, q0, q2
+; CHECK-NEXT:    vorr q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <2 x i64> %a, zeroinitializer
+  %c2 = icmp eq <2 x i64> %b, zeroinitializer
+  %o = xor <2 x i1> %c1, %c2
+  %s = select <2 x i1> %o, <2 x i64> %a, <2 x i64> %b
+  ret <2 x i64> %s
+}
+
+define arm_aapcs_vfpcc <2 x i64> @cmpeq_v2i1(<2 x i64> %a, <2 x i64> %b, <2 x i64> %c) {
+; CHECK-LABEL: cmpeq_v2i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov r0, s9
+; CHECK-NEXT:    vmov r1, s5
+; CHECK-NEXT:    vmov r2, s4
+; CHECK-NEXT:    eors r0, r1
+; CHECK-NEXT:    vmov r1, s8
+; CHECK-NEXT:    eors r1, r2
+; CHECK-NEXT:    vmov r2, s6
+; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    vmov r1, s7
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q3[0], r0
+; CHECK-NEXT:    vmov.32 q3[1], r0
+; CHECK-NEXT:    vmov r0, s11
+; CHECK-NEXT:    eors r0, r1
+; CHECK-NEXT:    vmov r1, s10
+; CHECK-NEXT:    eors r1, r2
+; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    vmov r1, s0
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q3[2], r0
+; CHECK-NEXT:    vmov.32 q3[3], r0
+; CHECK-NEXT:    vmov r0, s1
+; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    vmov r1, s2
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q2[0], r0
+; CHECK-NEXT:    vmov.32 q2[1], r0
+; CHECK-NEXT:    vmov r0, s3
+; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q2[2], r0
+; CHECK-NEXT:    vmov.32 q2[3], r0
+; CHECK-NEXT:    veor q2, q2, q3
+; CHECK-NEXT:    vbic q1, q1, q2
+; CHECK-NEXT:    vand q0, q0, q2
+; CHECK-NEXT:    vorr q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c1 = icmp eq <2 x i64> %a, zeroinitializer
+  %c2 = icmp eq <2 x i64> %b, %c
+  %o = xor <2 x i1> %c1, %c2
+  %s = select <2 x i1> %o, <2 x i64> %a, <2 x i64> %b
+  ret <2 x i64> %s
+}
+
+
diff --git a/llvm/test/CodeGen/Thumb2/mve-saturating-arith.ll b/llvm/test/CodeGen/Thumb2/mve-saturating-arith.ll
new file mode 100644
index 000000000..3610c2a65
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-saturating-arith.ll
@@ -0,0 +1,408 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s
+
+define arm_aapcs_vfpcc <16 x i8> @sadd_int8_t(<16 x i8> %src1, <16 x i8> %src2) {
+; CHECK-LABEL: sadd_int8_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqadd.s8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> %src1, <16 x i8> %src2)
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @sadd_int16_t(<8 x i16> %src1, <8 x i16> %src2) {
+; CHECK-LABEL: sadd_int16_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqadd.s16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> %src1, <8 x i16> %src2)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @sadd_int32_t(<4 x i32> %src1, <4 x i32> %src2) {
+; CHECK-LABEL: sadd_int32_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqadd.s32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> %src1, <4 x i32> %src2)
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <2 x i64> @sadd_int64_t(<2 x i64> %src1, <2 x i64> %src2) {
+; CHECK-LABEL: sadd_int64_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, lr}
+; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, lr}
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmov r5, s0
+; CHECK-NEXT:    vmov r8, s5
+; CHECK-NEXT:    vmov r4, s1
+; CHECK-NEXT:    vmov r7, s2
+; CHECK-NEXT:    vmov r3, s7
+; CHECK-NEXT:    vmov r6, s3
+; CHECK-NEXT:    adds.w r12, r5, r0
+; CHECK-NEXT:    adc.w r0, r4, r8
+; CHECK-NEXT:    asrs r2, r0, #31
+; CHECK-NEXT:    vmov.32 q2[0], r2
+; CHECK-NEXT:    vmov.32 q2[1], r2
+; CHECK-NEXT:    vmov r2, s6
+; CHECK-NEXT:    adds.w lr, r7, r2
+; CHECK-NEXT:    adc.w r2, r6, r3
+; CHECK-NEXT:    subs.w r5, r12, r5
+; CHECK-NEXT:    sbcs.w r4, r0, r4
+; CHECK-NEXT:    asr.w r1, r2, #31
+; CHECK-NEXT:    mov.w r4, #0
+; CHECK-NEXT:    vmov.32 q2[2], r1
+; CHECK-NEXT:    it lt
+; CHECK-NEXT:    movlt r4, #1
+; CHECK-NEXT:    vmov.32 q2[3], r1
+; CHECK-NEXT:    adr r1, .LCPI3_0
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    adr r1, .LCPI3_1
+; CHECK-NEXT:    vldrw.u32 q1, [r1]
+; CHECK-NEXT:    cmp r4, #0
+; CHECK-NEXT:    vbic q0, q0, q2
+; CHECK-NEXT:    csetm r4, ne
+; CHECK-NEXT:    vand q1, q1, q2
+; CHECK-NEXT:    movs r1, #0
+; CHECK-NEXT:    vorr q0, q1, q0
+; CHECK-NEXT:    vmov.32 q1[0], r4
+; CHECK-NEXT:    vmov.32 q1[1], r4
+; CHECK-NEXT:    subs.w r4, lr, r7
+; CHECK-NEXT:    sbcs.w r4, r2, r6
+; CHECK-NEXT:    it lt
+; CHECK-NEXT:    movlt r1, #1
+; CHECK-NEXT:    cmp r1, #0
+; CHECK-NEXT:    csetm r1, ne
+; CHECK-NEXT:    vmov.32 q1[2], r1
+; CHECK-NEXT:    vmov.32 q1[3], r1
+; CHECK-NEXT:    asr.w r1, r8, #31
+; CHECK-NEXT:    vmov.32 q2[0], r1
+; CHECK-NEXT:    vmov.32 q2[1], r1
+; CHECK-NEXT:    asrs r1, r3, #31
+; CHECK-NEXT:    vmov.32 q2[2], r1
+; CHECK-NEXT:    vmov.32 q2[3], r1
+; CHECK-NEXT:    veor q1, q2, q1
+; CHECK-NEXT:    vmov.32 q2[0], r12
+; CHECK-NEXT:    vmov.32 q2[1], r0
+; CHECK-NEXT:    vand q0, q0, q1
+; CHECK-NEXT:    vmov.32 q2[2], lr
+; CHECK-NEXT:    vmov.32 q2[3], r2
+; CHECK-NEXT:    vbic q1, q2, q1
+; CHECK-NEXT:    vorr q0, q0, q1
+; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, pc}
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI3_0:
+; CHECK-NEXT:    .long 0 @ 0x0
+; CHECK-NEXT:    .long 2147483648 @ 0x80000000
+; CHECK-NEXT:    .long 0 @ 0x0
+; CHECK-NEXT:    .long 2147483648 @ 0x80000000
+; CHECK-NEXT:  .LCPI3_1:
+; CHECK-NEXT:    .long 4294967295 @ 0xffffffff
+; CHECK-NEXT:    .long 2147483647 @ 0x7fffffff
+; CHECK-NEXT:    .long 4294967295 @ 0xffffffff
+; CHECK-NEXT:    .long 2147483647 @ 0x7fffffff
+entry:
+  %0 = call <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64> %src1, <2 x i64> %src2)
+  ret <2 x i64> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @uadd_int8_t(<16 x i8> %src1, <16 x i8> %src2) {
+; CHECK-LABEL: uadd_int8_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqadd.u8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> %src1, <16 x i8> %src2)
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @uadd_int16_t(<8 x i16> %src1, <8 x i16> %src2) {
+; CHECK-LABEL: uadd_int16_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqadd.u16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> %src1, <8 x i16> %src2)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @uadd_int32_t(<4 x i32> %src1, <4 x i32> %src2) {
+; CHECK-LABEL: uadd_int32_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqadd.u32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> %src1, <4 x i32> %src2)
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <2 x i64> @uadd_int64_t(<2 x i64> %src1, <2 x i64> %src2) {
+; CHECK-LABEL: uadd_int64_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, lr}
+; CHECK-NEXT:    push {r4, lr}
+; CHECK-NEXT:    vmov r2, s4
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    vmov r0, s5
+; CHECK-NEXT:    vmov r1, s1
+; CHECK-NEXT:    vmov r4, s2
+; CHECK-NEXT:    adds.w lr, r3, r2
+; CHECK-NEXT:    vmov r2, s6
+; CHECK-NEXT:    adc.w r12, r1, r0
+; CHECK-NEXT:    subs.w r3, lr, r3
+; CHECK-NEXT:    sbcs.w r1, r12, r1
+; CHECK-NEXT:    vmov r3, s3
+; CHECK-NEXT:    mov.w r1, #0
+; CHECK-NEXT:    mov.w r0, #0
+; CHECK-NEXT:    it lo
+; CHECK-NEXT:    movlo r1, #1
+; CHECK-NEXT:    cmp r1, #0
+; CHECK-NEXT:    csetm r1, ne
+; CHECK-NEXT:    vmov.32 q0[0], lr
+; CHECK-NEXT:    vmov.32 q2[0], r1
+; CHECK-NEXT:    vmov.32 q0[1], r12
+; CHECK-NEXT:    vmov.32 q2[1], r1
+; CHECK-NEXT:    vmov r1, s7
+; CHECK-NEXT:    adds r2, r2, r4
+; CHECK-NEXT:    vmov.32 q0[2], r2
+; CHECK-NEXT:    adcs r1, r3
+; CHECK-NEXT:    subs r4, r2, r4
+; CHECK-NEXT:    sbcs.w r3, r1, r3
+; CHECK-NEXT:    it lo
+; CHECK-NEXT:    movlo r0, #1
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    vmov.32 q0[3], r1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q2[2], r0
+; CHECK-NEXT:    vmov.32 q2[3], r0
+; CHECK-NEXT:    vorr q0, q0, q2
+; CHECK-NEXT:    pop {r4, pc}
+entry:
+  %0 = call <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64> %src1, <2 x i64> %src2)
+  ret <2 x i64> %0
+}
+
+
+define arm_aapcs_vfpcc <16 x i8> @ssub_int8_t(<16 x i8> %src1, <16 x i8> %src2) {
+; CHECK-LABEL: ssub_int8_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqsub.s8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> %src1, <16 x i8> %src2)
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @ssub_int16_t(<8 x i16> %src1, <8 x i16> %src2) {
+; CHECK-LABEL: ssub_int16_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqsub.s16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> %src1, <8 x i16> %src2)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @ssub_int32_t(<4 x i32> %src1, <4 x i32> %src2) {
+; CHECK-LABEL: ssub_int32_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqsub.s32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> %src1, <4 x i32> %src2)
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <2 x i64> @ssub_int64_t(<2 x i64> %src1, <2 x i64> %src2) {
+; CHECK-LABEL: ssub_int64_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, r5, r6, lr}
+; CHECK-NEXT:    push {r4, r5, r6, lr}
+; CHECK-NEXT:    .vsave {d8, d9}
+; CHECK-NEXT:    vpush {d8, d9}
+; CHECK-NEXT:    vmov r2, s4
+; CHECK-NEXT:    movs r0, #0
+; CHECK-NEXT:    vmov lr, s5
+; CHECK-NEXT:    vmov r12, s7
+; CHECK-NEXT:    vmov r5, s0
+; CHECK-NEXT:    vmov r4, s1
+; CHECK-NEXT:    rsbs r3, r2, #0
+; CHECK-NEXT:    sbcs.w r3, r0, lr
+; CHECK-NEXT:    mov.w r3, #0
+; CHECK-NEXT:    it lt
+; CHECK-NEXT:    movlt r3, #1
+; CHECK-NEXT:    cmp r3, #0
+; CHECK-NEXT:    csetm r3, ne
+; CHECK-NEXT:    vmov.32 q2[0], r3
+; CHECK-NEXT:    vmov.32 q2[1], r3
+; CHECK-NEXT:    vmov r3, s6
+; CHECK-NEXT:    rsbs r1, r3, #0
+; CHECK-NEXT:    sbcs.w r1, r0, r12
+; CHECK-NEXT:    mov.w r1, #0
+; CHECK-NEXT:    it lt
+; CHECK-NEXT:    movlt r1, #1
+; CHECK-NEXT:    cmp r1, #0
+; CHECK-NEXT:    csetm r1, ne
+; CHECK-NEXT:    subs r6, r5, r2
+; CHECK-NEXT:    vmov.32 q2[2], r1
+; CHECK-NEXT:    vmov.32 q2[3], r1
+; CHECK-NEXT:    sbc.w r1, r4, lr
+; CHECK-NEXT:    subs r5, r6, r5
+; CHECK-NEXT:    sbcs.w r5, r1, r4
+; CHECK-NEXT:    vmov r4, s2
+; CHECK-NEXT:    mov.w r5, #0
+; CHECK-NEXT:    it lt
+; CHECK-NEXT:    movlt r5, #1
+; CHECK-NEXT:    cmp r5, #0
+; CHECK-NEXT:    csetm r5, ne
+; CHECK-NEXT:    vmov.32 q1[0], r5
+; CHECK-NEXT:    vmov.32 q1[1], r5
+; CHECK-NEXT:    vmov r5, s3
+; CHECK-NEXT:    subs r3, r4, r3
+; CHECK-NEXT:    sbc.w r2, r5, r12
+; CHECK-NEXT:    subs r4, r3, r4
+; CHECK-NEXT:    sbcs.w r5, r2, r5
+; CHECK-NEXT:    it lt
+; CHECK-NEXT:    movlt r0, #1
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q1[2], r0
+; CHECK-NEXT:    vmov.32 q1[3], r0
+; CHECK-NEXT:    asrs r0, r1, #31
+; CHECK-NEXT:    veor q0, q2, q1
+; CHECK-NEXT:    vmov.32 q2[0], r0
+; CHECK-NEXT:    vmov.32 q2[1], r0
+; CHECK-NEXT:    asrs r0, r2, #31
+; CHECK-NEXT:    vmov.32 q2[2], r0
+; CHECK-NEXT:    vmov.32 q1[0], r6
+; CHECK-NEXT:    vmov.32 q2[3], r0
+; CHECK-NEXT:    adr r0, .LCPI11_0
+; CHECK-NEXT:    vldrw.u32 q3, [r0]
+; CHECK-NEXT:    adr r0, .LCPI11_1
+; CHECK-NEXT:    vldrw.u32 q4, [r0]
+; CHECK-NEXT:    vmov.32 q1[1], r1
+; CHECK-NEXT:    vmov.32 q1[2], r3
+; CHECK-NEXT:    vbic q3, q3, q2
+; CHECK-NEXT:    vand q2, q4, q2
+; CHECK-NEXT:    vmov.32 q1[3], r2
+; CHECK-NEXT:    vorr q2, q2, q3
+; CHECK-NEXT:    vbic q1, q1, q0
+; CHECK-NEXT:    vand q0, q2, q0
+; CHECK-NEXT:    vorr q0, q0, q1
+; CHECK-NEXT:    vpop {d8, d9}
+; CHECK-NEXT:    pop {r4, r5, r6, pc}
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI11_0:
+; CHECK-NEXT:    .long 0 @ 0x0
+; CHECK-NEXT:    .long 2147483648 @ 0x80000000
+; CHECK-NEXT:    .long 0 @ 0x0
+; CHECK-NEXT:    .long 2147483648 @ 0x80000000
+; CHECK-NEXT:  .LCPI11_1:
+; CHECK-NEXT:    .long 4294967295 @ 0xffffffff
+; CHECK-NEXT:    .long 2147483647 @ 0x7fffffff
+; CHECK-NEXT:    .long 4294967295 @ 0xffffffff
+; CHECK-NEXT:    .long 2147483647 @ 0x7fffffff
+entry:
+  %0 = call <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64> %src1, <2 x i64> %src2)
+  ret <2 x i64> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @usub_int8_t(<16 x i8> %src1, <16 x i8> %src2) {
+; CHECK-LABEL: usub_int8_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqsub.u8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> %src1, <16 x i8> %src2)
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @usub_int16_t(<8 x i16> %src1, <8 x i16> %src2) {
+; CHECK-LABEL: usub_int16_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqsub.u16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> %src1, <8 x i16> %src2)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @usub_int32_t(<4 x i32> %src1, <4 x i32> %src2) {
+; CHECK-LABEL: usub_int32_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqsub.u32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> %src1, <4 x i32> %src2)
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <2 x i64> @usub_int64_t(<2 x i64> %src1, <2 x i64> %src2) {
+; CHECK-LABEL: usub_int64_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, lr}
+; CHECK-NEXT:    push {r4, lr}
+; CHECK-NEXT:    vmov r2, s4
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    vmov r0, s5
+; CHECK-NEXT:    vmov r1, s1
+; CHECK-NEXT:    vmov r4, s2
+; CHECK-NEXT:    subs.w lr, r3, r2
+; CHECK-NEXT:    vmov r2, s6
+; CHECK-NEXT:    sbc.w r12, r1, r0
+; CHECK-NEXT:    subs.w r3, r3, lr
+; CHECK-NEXT:    sbcs.w r1, r1, r12
+; CHECK-NEXT:    vmov r3, s3
+; CHECK-NEXT:    mov.w r1, #0
+; CHECK-NEXT:    mov.w r0, #0
+; CHECK-NEXT:    it lo
+; CHECK-NEXT:    movlo r1, #1
+; CHECK-NEXT:    cmp r1, #0
+; CHECK-NEXT:    csetm r1, ne
+; CHECK-NEXT:    vmov.32 q0[0], lr
+; CHECK-NEXT:    vmov.32 q2[0], r1
+; CHECK-NEXT:    vmov.32 q0[1], r12
+; CHECK-NEXT:    vmov.32 q2[1], r1
+; CHECK-NEXT:    vmov r1, s7
+; CHECK-NEXT:    subs r2, r4, r2
+; CHECK-NEXT:    vmov.32 q0[2], r2
+; CHECK-NEXT:    sbc.w r1, r3, r1
+; CHECK-NEXT:    subs r4, r4, r2
+; CHECK-NEXT:    sbcs r3, r1
+; CHECK-NEXT:    it lo
+; CHECK-NEXT:    movlo r0, #1
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    vmov.32 q0[3], r1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q2[2], r0
+; CHECK-NEXT:    vmov.32 q2[3], r0
+; CHECK-NEXT:    vbic q0, q0, q2
+; CHECK-NEXT:    pop {r4, pc}
+entry:
+  %0 = call <2 x i64> @llvm.usub.sat.v2i64(<2 x i64> %src1, <2 x i64> %src2)
+  ret <2 x i64> %0
+}
+
+
+declare <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> %src1, <16 x i8> %src2)
+declare <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> %src1, <8 x i16> %src2)
+declare <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> %src1, <4 x i32> %src2)
+declare <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64> %src1, <2 x i64> %src2)
+declare <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> %src1, <16 x i8> %src2)
+declare <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> %src1, <8 x i16> %src2)
+declare <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> %src1, <4 x i32> %src2)
+declare <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64> %src1, <2 x i64> %src2)
+declare <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> %src1, <16 x i8> %src2)
+declare <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> %src1, <8 x i16> %src2)
+declare <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> %src1, <4 x i32> %src2)
+declare <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64> %src1, <2 x i64> %src2)
+declare <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> %src1, <16 x i8> %src2)
+declare <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> %src1, <8 x i16> %src2)
+declare <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> %src1, <4 x i32> %src2)
+declare <2 x i64> @llvm.usub.sat.v2i64(<2 x i64> %src1, <2 x i64> %src2)
diff --git a/llvm/test/CodeGen/Thumb2/mve-sext.ll b/llvm/test/CodeGen/Thumb2/mve-sext.ll
index 452e68405..ffade88a9 100644
--- a/llvm/test/CodeGen/Thumb2/mve-sext.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-sext.ll
@@ -1,6 +1,104 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s
 
+define arm_aapcs_vfpcc <4 x i32> @sext_v4i32_v4i32_v4i1(<4 x i32> %m) {
+; CHECK-LABEL: sext_v4i32_v4i32_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vshl.i32 q0, q0, #31
+; CHECK-NEXT:    vshr.s32 q0, q0, #31
+; CHECK-NEXT:    bx lr
+entry:
+  %shl = shl <4 x i32> %m, <i32 31, i32 31, i32 31, i32 31>
+  %shr = ashr exact <4 x i32> %shl, <i32 31, i32 31, i32 31, i32 31>
+  ret <4 x i32> %shr
+}
+
+define arm_aapcs_vfpcc <4 x i32> @sext_v4i32_v4i32_v4i8(<4 x i32> %m) {
+; CHECK-LABEL: sext_v4i32_v4i32_v4i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmovlb.s8 q0, q0
+; CHECK-NEXT:    vmovlb.s16 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %shl = shl <4 x i32> %m, <i32 24, i32 24, i32 24, i32 24>
+  %shr = ashr exact <4 x i32> %shl, <i32 24, i32 24, i32 24, i32 24>
+  ret <4 x i32> %shr
+}
+
+define arm_aapcs_vfpcc <4 x i32> @sext_v4i32_v4i32_v4i16(<4 x i32> %m) {
+; CHECK-LABEL: sext_v4i32_v4i32_v4i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmovlb.s16 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %shl = shl <4 x i32> %m, <i32 16, i32 16, i32 16, i32 16>
+  %shr = ashr exact <4 x i32> %shl, <i32 16, i32 16, i32 16, i32 16>
+  ret <4 x i32> %shr
+}
+
+define arm_aapcs_vfpcc <8 x i16> @sext_v8i16_v8i16_v8i8(<8 x i16> %m) {
+; CHECK-LABEL: sext_v8i16_v8i16_v8i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmovlb.s8 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %shl = shl <8 x i16> %m, <i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8>
+  %shr = ashr exact <8 x i16> %shl, <i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8>
+  ret <8 x i16> %shr
+}
+
+define arm_aapcs_vfpcc <8 x i16> @sext_v8i16_v8i16_v8i1(<8 x i16> %m) {
+; CHECK-LABEL: sext_v8i16_v8i16_v8i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vshl.i16 q0, q0, #15
+; CHECK-NEXT:    vshr.s16 q0, q0, #15
+; CHECK-NEXT:    bx lr
+entry:
+  %shl = shl <8 x i16> %m, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+  %shr = ashr exact <8 x i16> %shl, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+  ret <8 x i16> %shr
+}
+
+define arm_aapcs_vfpcc <2 x i64> @sext_v2i64_v2i64_v2i32(<2 x i64> %m) {
+; CHECK-LABEL: sext_v2i64_v2i64_v2i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vmov.32 q1[0], r0
+; CHECK-NEXT:    asrs r0, r0, #31
+; CHECK-NEXT:    vmov.32 q1[1], r0
+; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    vmov.32 q1[2], r0
+; CHECK-NEXT:    asrs r0, r0, #31
+; CHECK-NEXT:    vmov.32 q1[3], r0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %shl = shl <2 x i64> %m, <i64 32, i64 32>
+  %shr = ashr exact <2 x i64> %shl, <i64 32, i64 32>
+  ret <2 x i64> %shr
+}
+
+define arm_aapcs_vfpcc <2 x i64> @sext_v2i64_v2i64_v2i35(<2 x i64> %m) {
+; CHECK-LABEL: sext_v2i64_v2i64_v2i35:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vmov.32 q1[0], r0
+; CHECK-NEXT:    vmov r0, s1
+; CHECK-NEXT:    sbfx r0, r0, #0, #3
+; CHECK-NEXT:    vmov.32 q1[1], r0
+; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    vmov.32 q1[2], r0
+; CHECK-NEXT:    vmov r0, s3
+; CHECK-NEXT:    sbfx r0, r0, #0, #3
+; CHECK-NEXT:    vmov.32 q1[3], r0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %shl = shl <2 x i64> %m, <i64 29, i64 29>
+  %shr = ashr exact <2 x i64> %shl, <i64 29, i64 29>
+  ret <2 x i64> %shr
+}
+
 define arm_aapcs_vfpcc <8 x i16> @sext_v8i8_v8i16(<8 x i8> %src) {
 ; CHECK-LABEL: sext_v8i8_v8i16:
 ; CHECK:       @ %bb.0: @ %entry
@@ -32,6 +130,131 @@ entry:
   ret <4 x i32> %0
 }
 
+define arm_aapcs_vfpcc <16 x i16> @sext_v16i8_v16i16(<16 x i8> %src) {
+; CHECK-LABEL: sext_v16i8_v16i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.u8 r0, q0[0]
+; CHECK-NEXT:    vmov.16 q1[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[1]
+; CHECK-NEXT:    vmov.16 q1[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[2]
+; CHECK-NEXT:    vmov.16 q1[2], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[3]
+; CHECK-NEXT:    vmov.16 q1[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[4]
+; CHECK-NEXT:    vmov.16 q1[4], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[5]
+; CHECK-NEXT:    vmov.16 q1[5], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[6]
+; CHECK-NEXT:    vmov.16 q1[6], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[7]
+; CHECK-NEXT:    vmov.16 q1[7], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[8]
+; CHECK-NEXT:    vmovlb.s8 q2, q1
+; CHECK-NEXT:    vmov.16 q1[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[9]
+; CHECK-NEXT:    vmov.16 q1[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[10]
+; CHECK-NEXT:    vmov.16 q1[2], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[11]
+; CHECK-NEXT:    vmov.16 q1[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[12]
+; CHECK-NEXT:    vmov.16 q1[4], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[13]
+; CHECK-NEXT:    vmov.16 q1[5], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[14]
+; CHECK-NEXT:    vmov.16 q1[6], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[15]
+; CHECK-NEXT:    vmov.16 q1[7], r0
+; CHECK-NEXT:    vmov q0, q2
+; CHECK-NEXT:    vmovlb.s8 q1, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = sext <16 x i8> %src to <16 x i16>
+  ret <16 x i16> %0
+}
+
+define arm_aapcs_vfpcc <8 x i32> @sext_v8i16_v8i32(<8 x i16> %src) {
+; CHECK-LABEL: sext_v8i16_v8i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.u16 r0, q0[0]
+; CHECK-NEXT:    vmov.32 q1[0], r0
+; CHECK-NEXT:    vmov.u16 r0, q0[1]
+; CHECK-NEXT:    vmov.32 q1[1], r0
+; CHECK-NEXT:    vmov.u16 r0, q0[2]
+; CHECK-NEXT:    vmov.32 q1[2], r0
+; CHECK-NEXT:    vmov.u16 r0, q0[3]
+; CHECK-NEXT:    vmov.32 q1[3], r0
+; CHECK-NEXT:    vmov.u16 r0, q0[4]
+; CHECK-NEXT:    vmovlb.s16 q2, q1
+; CHECK-NEXT:    vmov.32 q1[0], r0
+; CHECK-NEXT:    vmov.u16 r0, q0[5]
+; CHECK-NEXT:    vmov.32 q1[1], r0
+; CHECK-NEXT:    vmov.u16 r0, q0[6]
+; CHECK-NEXT:    vmov.32 q1[2], r0
+; CHECK-NEXT:    vmov.u16 r0, q0[7]
+; CHECK-NEXT:    vmov.32 q1[3], r0
+; CHECK-NEXT:    vmov q0, q2
+; CHECK-NEXT:    vmovlb.s16 q1, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = sext <8 x i16> %src to <8 x i32>
+  ret <8 x i32> %0
+}
+
+define arm_aapcs_vfpcc <16 x i32> @sext_v16i8_v16i32(<16 x i8> %src) {
+; CHECK-LABEL: sext_v16i8_v16i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9}
+; CHECK-NEXT:    vpush {d8, d9}
+; CHECK-NEXT:    vmov.u8 r0, q0[0]
+; CHECK-NEXT:    vmov.32 q1[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[1]
+; CHECK-NEXT:    vmov.32 q1[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[2]
+; CHECK-NEXT:    vmov.32 q1[2], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[3]
+; CHECK-NEXT:    vmov.32 q1[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[4]
+; CHECK-NEXT:    vmovlb.s8 q1, q1
+; CHECK-NEXT:    vmovlb.s16 q4, q1
+; CHECK-NEXT:    vmov.32 q1[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[5]
+; CHECK-NEXT:    vmov.32 q1[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[6]
+; CHECK-NEXT:    vmov.32 q1[2], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[7]
+; CHECK-NEXT:    vmov.32 q1[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[8]
+; CHECK-NEXT:    vmov.32 q2[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[9]
+; CHECK-NEXT:    vmov.32 q2[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[10]
+; CHECK-NEXT:    vmov.32 q2[2], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[11]
+; CHECK-NEXT:    vmov.32 q2[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[12]
+; CHECK-NEXT:    vmov.32 q3[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[13]
+; CHECK-NEXT:    vmov.32 q3[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[14]
+; CHECK-NEXT:    vmov.32 q3[2], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[15]
+; CHECK-NEXT:    vmov.32 q3[3], r0
+; CHECK-NEXT:    vmovlb.s8 q1, q1
+; CHECK-NEXT:    vmovlb.s8 q2, q2
+; CHECK-NEXT:    vmovlb.s8 q0, q3
+; CHECK-NEXT:    vmovlb.s16 q3, q0
+; CHECK-NEXT:    vmovlb.s16 q1, q1
+; CHECK-NEXT:    vmovlb.s16 q2, q2
+; CHECK-NEXT:    vmov q0, q4
+; CHECK-NEXT:    vpop {d8, d9}
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = sext <16 x i8> %src to <16 x i32>
+  ret <16 x i32> %0
+}
+
 define arm_aapcs_vfpcc <2 x i64> @sext_v2i32_v2i64(<2 x i32> %src) {
 ; CHECK-LABEL: sext_v2i32_v2i64:
 ; CHECK:       @ %bb.0: @ %entry
@@ -82,16 +305,138 @@ entry:
   ret <4 x i32> %0
 }
 
+define arm_aapcs_vfpcc <16 x i16> @zext_v16i8_v16i16(<16 x i8> %src) {
+; CHECK-LABEL: zext_v16i8_v16i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.u8 r0, q0[0]
+; CHECK-NEXT:    vmov.16 q1[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[1]
+; CHECK-NEXT:    vmov.16 q1[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[2]
+; CHECK-NEXT:    vmov.16 q1[2], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[3]
+; CHECK-NEXT:    vmov.16 q1[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[4]
+; CHECK-NEXT:    vmov.16 q1[4], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[5]
+; CHECK-NEXT:    vmov.16 q1[5], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[6]
+; CHECK-NEXT:    vmov.16 q1[6], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[7]
+; CHECK-NEXT:    vmov.16 q1[7], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[8]
+; CHECK-NEXT:    vmovlb.u8 q2, q1
+; CHECK-NEXT:    vmov.16 q1[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[9]
+; CHECK-NEXT:    vmov.16 q1[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[10]
+; CHECK-NEXT:    vmov.16 q1[2], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[11]
+; CHECK-NEXT:    vmov.16 q1[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[12]
+; CHECK-NEXT:    vmov.16 q1[4], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[13]
+; CHECK-NEXT:    vmov.16 q1[5], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[14]
+; CHECK-NEXT:    vmov.16 q1[6], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[15]
+; CHECK-NEXT:    vmov.16 q1[7], r0
+; CHECK-NEXT:    vmov q0, q2
+; CHECK-NEXT:    vmovlb.u8 q1, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext <16 x i8> %src to <16 x i16>
+  ret <16 x i16> %0
+}
+
+define arm_aapcs_vfpcc <8 x i32> @zext_v8i16_v8i32(<8 x i16> %src) {
+; CHECK-LABEL: zext_v8i16_v8i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.u16 r0, q0[0]
+; CHECK-NEXT:    vmov.32 q1[0], r0
+; CHECK-NEXT:    vmov.u16 r0, q0[1]
+; CHECK-NEXT:    vmov.32 q1[1], r0
+; CHECK-NEXT:    vmov.u16 r0, q0[2]
+; CHECK-NEXT:    vmov.32 q1[2], r0
+; CHECK-NEXT:    vmov.u16 r0, q0[3]
+; CHECK-NEXT:    vmov.32 q1[3], r0
+; CHECK-NEXT:    vmov.u16 r0, q0[4]
+; CHECK-NEXT:    vmovlb.u16 q2, q1
+; CHECK-NEXT:    vmov.32 q1[0], r0
+; CHECK-NEXT:    vmov.u16 r0, q0[5]
+; CHECK-NEXT:    vmov.32 q1[1], r0
+; CHECK-NEXT:    vmov.u16 r0, q0[6]
+; CHECK-NEXT:    vmov.32 q1[2], r0
+; CHECK-NEXT:    vmov.u16 r0, q0[7]
+; CHECK-NEXT:    vmov.32 q1[3], r0
+; CHECK-NEXT:    vmov q0, q2
+; CHECK-NEXT:    vmovlb.u16 q1, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext <8 x i16> %src to <8 x i32>
+  ret <8 x i32> %0
+}
+
+define arm_aapcs_vfpcc <16 x i32> @zext_v16i8_v16i32(<16 x i8> %src) {
+; CHECK-LABEL: zext_v16i8_v16i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11}
+; CHECK-NEXT:    vmov.u8 r0, q0[0]
+; CHECK-NEXT:    vmov.i32 q3, #0xff
+; CHECK-NEXT:    vmov.32 q1[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[1]
+; CHECK-NEXT:    vmov.32 q1[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[2]
+; CHECK-NEXT:    vmov.32 q1[2], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[3]
+; CHECK-NEXT:    vmov.32 q1[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[4]
+; CHECK-NEXT:    vand q4, q1, q3
+; CHECK-NEXT:    vmov.32 q1[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[5]
+; CHECK-NEXT:    vmov.32 q1[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[6]
+; CHECK-NEXT:    vmov.32 q1[2], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[7]
+; CHECK-NEXT:    vmov.32 q1[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[8]
+; CHECK-NEXT:    vmov.32 q2[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[9]
+; CHECK-NEXT:    vmov.32 q2[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[10]
+; CHECK-NEXT:    vmov.32 q2[2], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[11]
+; CHECK-NEXT:    vmov.32 q2[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[12]
+; CHECK-NEXT:    vmov.32 q5[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[13]
+; CHECK-NEXT:    vmov.32 q5[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[14]
+; CHECK-NEXT:    vmov.32 q5[2], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[15]
+; CHECK-NEXT:    vmov.32 q5[3], r0
+; CHECK-NEXT:    vand q1, q1, q3
+; CHECK-NEXT:    vand q2, q2, q3
+; CHECK-NEXT:    vand q3, q5, q3
+; CHECK-NEXT:    vmov q0, q4
+; CHECK-NEXT:    vpop {d8, d9, d10, d11}
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext <16 x i8> %src to <16 x i32>
+  ret <16 x i32> %0
+}
+
 define arm_aapcs_vfpcc <2 x i64> @zext_v2i32_v2i64(<2 x i32> %src) {
 ; CHECK-LABEL: zext_v2i32_v2i64:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    adr r0, .LCPI7_0
+; CHECK-NEXT:    adr r0, .LCPI20_0
 ; CHECK-NEXT:    vldrw.u32 q1, [r0]
 ; CHECK-NEXT:    vand q0, q0, q1
 ; CHECK-NEXT:    bx lr
 ; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  @ %bb.1:
-; CHECK-NEXT:  .LCPI7_0:
+; CHECK-NEXT:  .LCPI20_0:
 ; CHECK-NEXT:    .long 4294967295 @ 0xffffffff
 ; CHECK-NEXT:    .long 0 @ 0x0
 ; CHECK-NEXT:    .long 4294967295 @ 0xffffffff
@@ -129,6 +474,119 @@ entry:
   ret <4 x i8> %0
 }
 
+define arm_aapcs_vfpcc <16 x i8> @trunc_v16i16_v16i8(<16 x i16> %src) {
+; CHECK-LABEL: trunc_v16i16_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov q2, q0
+; CHECK-NEXT:    vmov.u16 r0, q0[0]
+; CHECK-NEXT:    vmov.8 q0[0], r0
+; CHECK-NEXT:    vmov.u16 r0, q2[1]
+; CHECK-NEXT:    vmov.8 q0[1], r0
+; CHECK-NEXT:    vmov.u16 r0, q2[2]
+; CHECK-NEXT:    vmov.8 q0[2], r0
+; CHECK-NEXT:    vmov.u16 r0, q2[3]
+; CHECK-NEXT:    vmov.8 q0[3], r0
+; CHECK-NEXT:    vmov.u16 r0, q2[4]
+; CHECK-NEXT:    vmov.8 q0[4], r0
+; CHECK-NEXT:    vmov.u16 r0, q2[5]
+; CHECK-NEXT:    vmov.8 q0[5], r0
+; CHECK-NEXT:    vmov.u16 r0, q2[6]
+; CHECK-NEXT:    vmov.8 q0[6], r0
+; CHECK-NEXT:    vmov.u16 r0, q2[7]
+; CHECK-NEXT:    vmov.8 q0[7], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[0]
+; CHECK-NEXT:    vmov.8 q0[8], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[1]
+; CHECK-NEXT:    vmov.8 q0[9], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[2]
+; CHECK-NEXT:    vmov.8 q0[10], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[3]
+; CHECK-NEXT:    vmov.8 q0[11], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[4]
+; CHECK-NEXT:    vmov.8 q0[12], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[5]
+; CHECK-NEXT:    vmov.8 q0[13], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[6]
+; CHECK-NEXT:    vmov.8 q0[14], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[7]
+; CHECK-NEXT:    vmov.8 q0[15], r0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = trunc <16 x i16> %src to <16 x i8>
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @trunc_v8i32_v8i16(<8 x i32> %src) {
+; CHECK-LABEL: trunc_v8i32_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov q2, q0
+; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    vmov.16 q0[0], r0
+; CHECK-NEXT:    vmov r0, s9
+; CHECK-NEXT:    vmov.16 q0[1], r0
+; CHECK-NEXT:    vmov r0, s10
+; CHECK-NEXT:    vmov.16 q0[2], r0
+; CHECK-NEXT:    vmov r0, s11
+; CHECK-NEXT:    vmov.16 q0[3], r0
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmov.16 q0[4], r0
+; CHECK-NEXT:    vmov r0, s5
+; CHECK-NEXT:    vmov.16 q0[5], r0
+; CHECK-NEXT:    vmov r0, s6
+; CHECK-NEXT:    vmov.16 q0[6], r0
+; CHECK-NEXT:    vmov r0, s7
+; CHECK-NEXT:    vmov.16 q0[7], r0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = trunc <8 x i32> %src to <8 x i16>
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @trunc_v16i32_v16i8(<16 x i32> %src) {
+; CHECK-LABEL: trunc_v16i32_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9}
+; CHECK-NEXT:    vpush {d8, d9}
+; CHECK-NEXT:    vmov q4, q0
+; CHECK-NEXT:    vmov r0, s16
+; CHECK-NEXT:    vmov.8 q0[0], r0
+; CHECK-NEXT:    vmov r0, s17
+; CHECK-NEXT:    vmov.8 q0[1], r0
+; CHECK-NEXT:    vmov r0, s18
+; CHECK-NEXT:    vmov.8 q0[2], r0
+; CHECK-NEXT:    vmov r0, s19
+; CHECK-NEXT:    vmov.8 q0[3], r0
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmov.8 q0[4], r0
+; CHECK-NEXT:    vmov r0, s5
+; CHECK-NEXT:    vmov.8 q0[5], r0
+; CHECK-NEXT:    vmov r0, s6
+; CHECK-NEXT:    vmov.8 q0[6], r0
+; CHECK-NEXT:    vmov r0, s7
+; CHECK-NEXT:    vmov.8 q0[7], r0
+; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    vmov.8 q0[8], r0
+; CHECK-NEXT:    vmov r0, s9
+; CHECK-NEXT:    vmov.8 q0[9], r0
+; CHECK-NEXT:    vmov r0, s10
+; CHECK-NEXT:    vmov.8 q0[10], r0
+; CHECK-NEXT:    vmov r0, s11
+; CHECK-NEXT:    vmov.8 q0[11], r0
+; CHECK-NEXT:    vmov r0, s12
+; CHECK-NEXT:    vmov.8 q0[12], r0
+; CHECK-NEXT:    vmov r0, s13
+; CHECK-NEXT:    vmov.8 q0[13], r0
+; CHECK-NEXT:    vmov r0, s14
+; CHECK-NEXT:    vmov.8 q0[14], r0
+; CHECK-NEXT:    vmov r0, s15
+; CHECK-NEXT:    vmov.8 q0[15], r0
+; CHECK-NEXT:    vpop {d8, d9}
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = trunc <16 x i32> %src to <16 x i8>
+  ret <16 x i8> %0
+}
+
 define arm_aapcs_vfpcc <2 x i32> @trunc_v2i64_v2i32(<2 x i64> %src) {
 ; CHECK-LABEL: trunc_v2i64_v2i32:
 ; CHECK:       @ %bb.0: @ %entry
diff --git a/llvm/test/CodeGen/Thumb2/mve-shifts-scalar.ll b/llvm/test/CodeGen/Thumb2/mve-shifts-scalar.ll
new file mode 100644
index 000000000..d2e8896d7
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-shifts-scalar.ll
@@ -0,0 +1,422 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -O3 -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve %s --verify-machineinstrs -o - | FileCheck %s
+
+define dso_local arm_aapcs_vfpcc void @sink_shl_i32(i32* nocapture readonly %in, i32* noalias nocapture %out, i32 %shift, i32 %N) {
+; CHECK-LABEL: sink_shl_i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    bic r3, r3, #3
+; CHECK-NEXT:    subs r0, #16
+; CHECK-NEXT:    sub.w r12, r3, #4
+; CHECK-NEXT:    movs r3, #1
+; CHECK-NEXT:    subs r1, #16
+; CHECK-NEXT:    add.w lr, r3, r12, lsr #2
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB0_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #16]!
+; CHECK-NEXT:    vshl.u32 q0, r2
+; CHECK-NEXT:    vstrb.8 q0, [r1, #16]!
+; CHECK-NEXT:    le lr, .LBB0_1
+; CHECK-NEXT:  @ %bb.2: @ %exit
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  br label %vector.ph
+
+vector.ph:
+  %n.vec = and i32 %N, -4
+  %broadcast.splatinsert10 = insertelement <4 x i32> undef, i32 %shift, i32 0
+  %broadcast.splat11 = shufflevector <4 x i32> %broadcast.splatinsert10, <4 x i32> undef, <4 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %gep.in = getelementptr inbounds i32, i32* %in, i32 %index
+  %cast.in = bitcast i32* %gep.in to <4 x i32>*
+  %wide.load = load <4 x i32>, <4 x i32>* %cast.in, align 4
+  %res = shl <4 x i32> %wide.load, %broadcast.splat11
+  %gep.out = getelementptr inbounds i32, i32* %out, i32 %index
+  %cast.out = bitcast i32* %gep.out to <4 x i32>*
+  store <4 x i32> %res, <4 x i32>* %cast.out, align 4
+  %index.next = add i32 %index, 4
+  %cmp = icmp eq i32 %index.next, %n.vec
+  br i1 %cmp, label %exit, label %vector.body
+
+exit:
+  ret void
+}
+
+define dso_local arm_aapcs_vfpcc void @sink_shl_i16(i16* nocapture readonly %in, i16* noalias nocapture %out, i16 %shift, i32 %N) {
+; CHECK-LABEL: sink_shl_i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    bic r3, r3, #3
+; CHECK-NEXT:    subs r0, #8
+; CHECK-NEXT:    sub.w r12, r3, #4
+; CHECK-NEXT:    movs r3, #1
+; CHECK-NEXT:    subs r1, #8
+; CHECK-NEXT:    add.w lr, r3, r12, lsr #2
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB1_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #8]!
+; CHECK-NEXT:    vshl.u16 q0, r2
+; CHECK-NEXT:    vstrb.8 q0, [r1, #8]!
+; CHECK-NEXT:    le lr, .LBB1_1
+; CHECK-NEXT:  @ %bb.2: @ %exit
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  br label %vector.ph
+
+vector.ph:
+  %n.vec = and i32 %N, -4
+  %broadcast.splatinsert10 = insertelement <8 x i16> undef, i16 %shift, i32 0
+  %broadcast.splat11 = shufflevector <8 x i16> %broadcast.splatinsert10, <8 x i16> undef, <8 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %gep.in = getelementptr inbounds i16, i16* %in, i32 %index
+  %cast.in = bitcast i16* %gep.in to <8 x i16>*
+  %wide.load = load <8 x i16>, <8 x i16>* %cast.in, align 4
+  %res = shl <8 x i16> %wide.load, %broadcast.splat11
+  %gep.out = getelementptr inbounds i16, i16* %out, i32 %index
+  %cast.out = bitcast i16* %gep.out to <8 x i16>*
+  store <8 x i16> %res, <8 x i16>* %cast.out, align 4
+  %index.next = add i32 %index, 4
+  %cmp = icmp eq i32 %index.next, %n.vec
+  br i1 %cmp, label %exit, label %vector.body
+
+exit:
+  ret void
+}
+
+define dso_local arm_aapcs_vfpcc void @sink_shl_i8(i8* nocapture readonly %in, i8* noalias nocapture %out, i8 %shift, i32 %N) {
+; CHECK-LABEL: sink_shl_i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    bic r3, r3, #3
+; CHECK-NEXT:    subs r0, #4
+; CHECK-NEXT:    sub.w r12, r3, #4
+; CHECK-NEXT:    movs r3, #1
+; CHECK-NEXT:    subs r1, #4
+; CHECK-NEXT:    add.w lr, r3, r12, lsr #2
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB2_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #4]!
+; CHECK-NEXT:    vshl.u8 q0, r2
+; CHECK-NEXT:    vstrb.8 q0, [r1, #4]!
+; CHECK-NEXT:    le lr, .LBB2_1
+; CHECK-NEXT:  @ %bb.2: @ %exit
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  br label %vector.ph
+
+vector.ph:
+  %n.vec = and i32 %N, -4
+  %broadcast.splatinsert10 = insertelement <16 x i8> undef, i8 %shift, i32 0
+  %broadcast.splat11 = shufflevector <16 x i8> %broadcast.splatinsert10, <16 x i8> undef, <16 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %gep.in = getelementptr inbounds i8, i8* %in, i32 %index
+  %cast.in = bitcast i8* %gep.in to <16 x i8>*
+  %wide.load = load <16 x i8>, <16 x i8>* %cast.in, align 4
+  %res = shl <16 x i8> %wide.load, %broadcast.splat11
+  %gep.out = getelementptr inbounds i8, i8* %out, i32 %index
+  %cast.out = bitcast i8* %gep.out to <16 x i8>*
+  store <16 x i8> %res, <16 x i8>* %cast.out, align 4
+  %index.next = add i32 %index, 4
+  %cmp = icmp eq i32 %index.next, %n.vec
+  br i1 %cmp, label %exit, label %vector.body
+
+exit:
+  ret void
+}
+
+define dso_local arm_aapcs_vfpcc void @sink_lshr_i32(i32* nocapture readonly %in, i32* noalias nocapture %out, i32 %shift, i32 %N) {
+; CHECK-LABEL: sink_lshr_i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    bic r3, r3, #3
+; CHECK-NEXT:    subs r0, #16
+; CHECK-NEXT:    sub.w r12, r3, #4
+; CHECK-NEXT:    movs r3, #1
+; CHECK-NEXT:    subs r1, #16
+; CHECK-NEXT:    rsbs r2, r2, #0
+; CHECK-NEXT:    add.w lr, r3, r12, lsr #2
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB3_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #16]!
+; CHECK-NEXT:    vshl.u32 q0, r2
+; CHECK-NEXT:    vstrb.8 q0, [r1, #16]!
+; CHECK-NEXT:    le lr, .LBB3_1
+; CHECK-NEXT:  @ %bb.2: @ %exit
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  br label %vector.ph
+
+vector.ph:
+  %n.vec = and i32 %N, -4
+  %broadcast.splatinsert10 = insertelement <4 x i32> undef, i32 %shift, i32 0
+  %broadcast.splat11 = shufflevector <4 x i32> %broadcast.splatinsert10, <4 x i32> undef, <4 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %gep.in = getelementptr inbounds i32, i32* %in, i32 %index
+  %cast.in = bitcast i32* %gep.in to <4 x i32>*
+  %wide.load = load <4 x i32>, <4 x i32>* %cast.in, align 4
+  %res = lshr <4 x i32> %wide.load, %broadcast.splat11
+  %gep.out = getelementptr inbounds i32, i32* %out, i32 %index
+  %cast.out = bitcast i32* %gep.out to <4 x i32>*
+  store <4 x i32> %res, <4 x i32>* %cast.out, align 4
+  %index.next = add i32 %index, 4
+  %cmp = icmp eq i32 %index.next, %n.vec
+  br i1 %cmp, label %exit, label %vector.body
+
+exit:
+  ret void
+}
+
+define dso_local arm_aapcs_vfpcc void @sink_lshr_i16(i16* nocapture readonly %in, i16* noalias nocapture %out, i16 %shift, i32 %N) {
+; CHECK-LABEL: sink_lshr_i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    bic r3, r3, #3
+; CHECK-NEXT:    subs r0, #8
+; CHECK-NEXT:    sub.w r12, r3, #4
+; CHECK-NEXT:    movs r3, #1
+; CHECK-NEXT:    subs r1, #8
+; CHECK-NEXT:    rsbs r2, r2, #0
+; CHECK-NEXT:    add.w lr, r3, r12, lsr #2
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB4_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #8]!
+; CHECK-NEXT:    vshl.u16 q0, r2
+; CHECK-NEXT:    vstrb.8 q0, [r1, #8]!
+; CHECK-NEXT:    le lr, .LBB4_1
+; CHECK-NEXT:  @ %bb.2: @ %exit
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  br label %vector.ph
+
+vector.ph:
+  %n.vec = and i32 %N, -4
+  %broadcast.splatinsert10 = insertelement <8 x i16> undef, i16 %shift, i32 0
+  %broadcast.splat11 = shufflevector <8 x i16> %broadcast.splatinsert10, <8 x i16> undef, <8 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %gep.in = getelementptr inbounds i16, i16* %in, i32 %index
+  %cast.in = bitcast i16* %gep.in to <8 x i16>*
+  %wide.load = load <8 x i16>, <8 x i16>* %cast.in, align 4
+  %res = lshr <8 x i16> %wide.load, %broadcast.splat11
+  %gep.out = getelementptr inbounds i16, i16* %out, i32 %index
+  %cast.out = bitcast i16* %gep.out to <8 x i16>*
+  store <8 x i16> %res, <8 x i16>* %cast.out, align 4
+  %index.next = add i32 %index, 4
+  %cmp = icmp eq i32 %index.next, %n.vec
+  br i1 %cmp, label %exit, label %vector.body
+
+exit:
+  ret void
+}
+
+define dso_local arm_aapcs_vfpcc void @sink_lshr_i8(i8* nocapture readonly %in, i8* noalias nocapture %out, i8 %shift, i32 %N) {
+; CHECK-LABEL: sink_lshr_i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    bic r3, r3, #3
+; CHECK-NEXT:    subs r0, #4
+; CHECK-NEXT:    sub.w r12, r3, #4
+; CHECK-NEXT:    movs r3, #1
+; CHECK-NEXT:    subs r1, #4
+; CHECK-NEXT:    rsbs r2, r2, #0
+; CHECK-NEXT:    add.w lr, r3, r12, lsr #2
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB5_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #4]!
+; CHECK-NEXT:    vshl.u8 q0, r2
+; CHECK-NEXT:    vstrb.8 q0, [r1, #4]!
+; CHECK-NEXT:    le lr, .LBB5_1
+; CHECK-NEXT:  @ %bb.2: @ %exit
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  br label %vector.ph
+
+vector.ph:
+  %n.vec = and i32 %N, -4
+  %broadcast.splatinsert10 = insertelement <16 x i8> undef, i8 %shift, i32 0
+  %broadcast.splat11 = shufflevector <16 x i8> %broadcast.splatinsert10, <16 x i8> undef, <16 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %gep.in = getelementptr inbounds i8, i8* %in, i32 %index
+  %cast.in = bitcast i8* %gep.in to <16 x i8>*
+  %wide.load = load <16 x i8>, <16 x i8>* %cast.in, align 4
+  %res = lshr <16 x i8> %wide.load, %broadcast.splat11
+  %gep.out = getelementptr inbounds i8, i8* %out, i32 %index
+  %cast.out = bitcast i8* %gep.out to <16 x i8>*
+  store <16 x i8> %res, <16 x i8>* %cast.out, align 4
+  %index.next = add i32 %index, 4
+  %cmp = icmp eq i32 %index.next, %n.vec
+  br i1 %cmp, label %exit, label %vector.body
+
+exit:
+  ret void
+}
+
+define dso_local arm_aapcs_vfpcc void @sink_ashr_i32(i32* nocapture readonly %in, i32* noalias nocapture %out, i32 %shift, i32 %N) {
+; CHECK-LABEL: sink_ashr_i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    bic r3, r3, #3
+; CHECK-NEXT:    subs r0, #16
+; CHECK-NEXT:    sub.w r12, r3, #4
+; CHECK-NEXT:    movs r3, #1
+; CHECK-NEXT:    subs r1, #16
+; CHECK-NEXT:    rsbs r2, r2, #0
+; CHECK-NEXT:    add.w lr, r3, r12, lsr #2
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB6_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #16]!
+; CHECK-NEXT:    vshl.s32 q0, r2
+; CHECK-NEXT:    vstrb.8 q0, [r1, #16]!
+; CHECK-NEXT:    le lr, .LBB6_1
+; CHECK-NEXT:  @ %bb.2: @ %exit
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  br label %vector.ph
+
+vector.ph:
+  %n.vec = and i32 %N, -4
+  %broadcast.splatinsert10 = insertelement <4 x i32> undef, i32 %shift, i32 0
+  %broadcast.splat11 = shufflevector <4 x i32> %broadcast.splatinsert10, <4 x i32> undef, <4 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %gep.in = getelementptr inbounds i32, i32* %in, i32 %index
+  %cast.in = bitcast i32* %gep.in to <4 x i32>*
+  %wide.load = load <4 x i32>, <4 x i32>* %cast.in, align 4
+  %res = ashr <4 x i32> %wide.load, %broadcast.splat11
+  %gep.out = getelementptr inbounds i32, i32* %out, i32 %index
+  %cast.out = bitcast i32* %gep.out to <4 x i32>*
+  store <4 x i32> %res, <4 x i32>* %cast.out, align 4
+  %index.next = add i32 %index, 4
+  %cmp = icmp eq i32 %index.next, %n.vec
+  br i1 %cmp, label %exit, label %vector.body
+
+exit:
+  ret void
+}
+
+define dso_local arm_aapcs_vfpcc void @sink_ashr_i16(i16* nocapture readonly %in, i16* noalias nocapture %out, i16 %shift, i32 %N) {
+; CHECK-LABEL: sink_ashr_i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    bic r3, r3, #3
+; CHECK-NEXT:    subs r0, #8
+; CHECK-NEXT:    sub.w r12, r3, #4
+; CHECK-NEXT:    movs r3, #1
+; CHECK-NEXT:    subs r1, #8
+; CHECK-NEXT:    rsbs r2, r2, #0
+; CHECK-NEXT:    add.w lr, r3, r12, lsr #2
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB7_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #8]!
+; CHECK-NEXT:    vshl.s16 q0, r2
+; CHECK-NEXT:    vstrb.8 q0, [r1, #8]!
+; CHECK-NEXT:    le lr, .LBB7_1
+; CHECK-NEXT:  @ %bb.2: @ %exit
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  br label %vector.ph
+
+vector.ph:
+  %n.vec = and i32 %N, -4
+  %broadcast.splatinsert10 = insertelement <8 x i16> undef, i16 %shift, i32 0
+  %broadcast.splat11 = shufflevector <8 x i16> %broadcast.splatinsert10, <8 x i16> undef, <8 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %gep.in = getelementptr inbounds i16, i16* %in, i32 %index
+  %cast.in = bitcast i16* %gep.in to <8 x i16>*
+  %wide.load = load <8 x i16>, <8 x i16>* %cast.in, align 4
+  %res = ashr <8 x i16> %wide.load, %broadcast.splat11
+  %gep.out = getelementptr inbounds i16, i16* %out, i32 %index
+  %cast.out = bitcast i16* %gep.out to <8 x i16>*
+  store <8 x i16> %res, <8 x i16>* %cast.out, align 4
+  %index.next = add i32 %index, 4
+  %cmp = icmp eq i32 %index.next, %n.vec
+  br i1 %cmp, label %exit, label %vector.body
+
+exit:
+  ret void
+}
+
+define dso_local arm_aapcs_vfpcc void @sink_ashr_i8(i8* nocapture readonly %in, i8* noalias nocapture %out, i8 %shift, i32 %N) {
+; CHECK-LABEL: sink_ashr_i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    bic r3, r3, #3
+; CHECK-NEXT:    subs r0, #4
+; CHECK-NEXT:    sub.w r12, r3, #4
+; CHECK-NEXT:    movs r3, #1
+; CHECK-NEXT:    subs r1, #4
+; CHECK-NEXT:    rsbs r2, r2, #0
+; CHECK-NEXT:    add.w lr, r3, r12, lsr #2
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB8_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #4]!
+; CHECK-NEXT:    vshl.s8 q0, r2
+; CHECK-NEXT:    vstrb.8 q0, [r1, #4]!
+; CHECK-NEXT:    le lr, .LBB8_1
+; CHECK-NEXT:  @ %bb.2: @ %exit
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  br label %vector.ph
+
+vector.ph:
+  %n.vec = and i32 %N, -4
+  %broadcast.splatinsert10 = insertelement <16 x i8> undef, i8 %shift, i32 0
+  %broadcast.splat11 = shufflevector <16 x i8> %broadcast.splatinsert10, <16 x i8> undef, <16 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %gep.in = getelementptr inbounds i8, i8* %in, i32 %index
+  %cast.in = bitcast i8* %gep.in to <16 x i8>*
+  %wide.load = load <16 x i8>, <16 x i8>* %cast.in, align 4
+  %res = ashr <16 x i8> %wide.load, %broadcast.splat11
+  %gep.out = getelementptr inbounds i8, i8* %out, i32 %index
+  %cast.out = bitcast i8* %gep.out to <16 x i8>*
+  store <16 x i8> %res, <16 x i8>* %cast.out, align 4
+  %index.next = add i32 %index, 4
+  %cmp = icmp eq i32 %index.next, %n.vec
+  br i1 %cmp, label %exit, label %vector.body
+
+exit:
+  ret void
+}
diff --git a/llvm/test/CodeGen/Thumb2/mve-shifts.ll b/llvm/test/CodeGen/Thumb2/mve-shifts.ll
index a321c2dd3..69e91a54c 100644
--- a/llvm/test/CodeGen/Thumb2/mve-shifts.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-shifts.ll
@@ -383,9 +383,8 @@ entry:
 define arm_aapcs_vfpcc <16 x i8> @shru_qr_int8_t(<16 x i8> %src1, i8 %src2) {
 ; CHECK-LABEL: shru_qr_int8_t:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vdup.8 q1, r0
-; CHECK-NEXT:    vneg.s8 q1, q1
-; CHECK-NEXT:    vshl.u8 q0, q0, q1
+; CHECK-NEXT:    rsbs r0, r0, #0
+; CHECK-NEXT:    vshl.u8 q0, r0
 ; CHECK-NEXT:    bx lr
 entry:
   %i = insertelement <16 x i8> undef, i8 %src2, i32 0
@@ -397,9 +396,8 @@ entry:
 define arm_aapcs_vfpcc <8 x i16> @shru_qr_int16_t(<8 x i16> %src1, i16 %src2) {
 ; CHECK-LABEL: shru_qr_int16_t:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vdup.16 q1, r0
-; CHECK-NEXT:    vneg.s16 q1, q1
-; CHECK-NEXT:    vshl.u16 q0, q0, q1
+; CHECK-NEXT:    rsbs r0, r0, #0
+; CHECK-NEXT:    vshl.u16 q0, r0
 ; CHECK-NEXT:    bx lr
 entry:
   %i = insertelement <8 x i16> undef, i16 %src2, i32 0
@@ -411,9 +409,8 @@ entry:
 define arm_aapcs_vfpcc <4 x i32> @shru_qr_int32_t(<4 x i32> %src1, i32 %src2) {
 ; CHECK-LABEL: shru_qr_int32_t:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vdup.32 q1, r0
-; CHECK-NEXT:    vneg.s32 q1, q1
-; CHECK-NEXT:    vshl.u32 q0, q0, q1
+; CHECK-NEXT:    rsbs r0, r0, #0
+; CHECK-NEXT:    vshl.u32 q0, r0
 ; CHECK-NEXT:    bx lr
 entry:
   %i = insertelement <4 x i32> undef, i32 %src2, i32 0
@@ -449,9 +446,8 @@ entry:
 define arm_aapcs_vfpcc <16 x i8> @shrs_qr_int8_t(<16 x i8> %src1, i8 %src2) {
 ; CHECK-LABEL: shrs_qr_int8_t:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vdup.8 q1, r0
-; CHECK-NEXT:    vneg.s8 q1, q1
-; CHECK-NEXT:    vshl.s8 q0, q0, q1
+; CHECK-NEXT:    rsbs r0, r0, #0
+; CHECK-NEXT:    vshl.s8 q0, r0
 ; CHECK-NEXT:    bx lr
 entry:
   %i = insertelement <16 x i8> undef, i8 %src2, i32 0
@@ -463,9 +459,8 @@ entry:
 define arm_aapcs_vfpcc <8 x i16> @shrs_qr_int16_t(<8 x i16> %src1, i16 %src2) {
 ; CHECK-LABEL: shrs_qr_int16_t:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vdup.16 q1, r0
-; CHECK-NEXT:    vneg.s16 q1, q1
-; CHECK-NEXT:    vshl.s16 q0, q0, q1
+; CHECK-NEXT:    rsbs r0, r0, #0
+; CHECK-NEXT:    vshl.s16 q0, r0
 ; CHECK-NEXT:    bx lr
 entry:
   %i = insertelement <8 x i16> undef, i16 %src2, i32 0
@@ -477,9 +472,8 @@ entry:
 define arm_aapcs_vfpcc <4 x i32> @shrs_qr_int32_t(<4 x i32> %src1, i32 %src2) {
 ; CHECK-LABEL: shrs_qr_int32_t:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vdup.32 q1, r0
-; CHECK-NEXT:    vneg.s32 q1, q1
-; CHECK-NEXT:    vshl.s32 q0, q0, q1
+; CHECK-NEXT:    rsbs r0, r0, #0
+; CHECK-NEXT:    vshl.s32 q0, r0
 ; CHECK-NEXT:    bx lr
 entry:
   %i = insertelement <4 x i32> undef, i32 %src2, i32 0
diff --git a/llvm/test/CodeGen/Thumb2/mve-shuffle.ll b/llvm/test/CodeGen/Thumb2/mve-shuffle.ll
index b5c96f9f8..2f6541b7b 100644
--- a/llvm/test/CodeGen/Thumb2/mve-shuffle.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-shuffle.ll
@@ -42,7 +42,8 @@ entry:
 define arm_aapcs_vfpcc <4 x i32> @shuffle5_i32(<4 x i32> %src) {
 ; CHECK-LABEL: shuffle5_i32:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vrev64.32 q0, q0
+; CHECK-NEXT:    vrev64.32 q1, q0
+; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %out = shufflevector <4 x i32> %src, <4 x i32> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
@@ -97,11 +98,7 @@ define arm_aapcs_vfpcc <8 x i16> @shuffle3_i16(<8 x i16> %src) {
 ; CHECK-LABEL: shuffle3_i16:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vmov q1, q0
-; CHECK-NEXT:    vmov.u16 r0, q0[4]
-; CHECK-NEXT:    vmov.16 q0[0], r0
-; CHECK-NEXT:    vmov.u16 r0, q1[5]
-; CHECK-NEXT:    vmov.16 q0[1], r0
-; CHECK-NEXT:    vmov.u16 r0, q1[7]
+; CHECK-NEXT:    vmov.u16 r0, q0[7]
 ; CHECK-NEXT:    vmov.16 q0[2], r0
 ; CHECK-NEXT:    vmov.u16 r0, q1[6]
 ; CHECK-NEXT:    vmov.16 q0[3], r0
@@ -113,6 +110,7 @@ define arm_aapcs_vfpcc <8 x i16> @shuffle3_i16(<8 x i16> %src) {
 ; CHECK-NEXT:    vmov.16 q0[6], r0
 ; CHECK-NEXT:    vmov.u16 r0, q1[0]
 ; CHECK-NEXT:    vmov.16 q0[7], r0
+; CHECK-NEXT:    vmov.f32 s0, s6
 ; CHECK-NEXT:    bx lr
 entry:
   %out = shufflevector <8 x i16> %src, <8 x i16> undef, <8 x i32> <i32 4, i32 5, i32 7, i32 6, i32 3, i32 1, i32 2, i32 0>
@@ -122,7 +120,8 @@ entry:
 define arm_aapcs_vfpcc <8 x i16> @shuffle5_i16(<8 x i16> %src) {
 ; CHECK-LABEL: shuffle5_i16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vrev64.16 q0, q0
+; CHECK-NEXT:    vrev64.16 q1, q0
+; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %out = shufflevector <8 x i16> %src, <8 x i16> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
@@ -235,7 +234,8 @@ entry:
 define arm_aapcs_vfpcc <16 x i8> @shuffle5_i8(<16 x i8> %src) {
 ; CHECK-LABEL: shuffle5_i8:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vrev64.8 q0, q0
+; CHECK-NEXT:    vrev64.8 q1, q0
+; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %out = shufflevector <16 x i8> %src, <16 x i8> undef, <16 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8>
@@ -334,7 +334,8 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @shuffle5_f32(<4 x float> %src) {
 ; CHECK-LABEL: shuffle5_f32:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vrev64.32 q0, q0
+; CHECK-NEXT:    vrev64.32 q1, q0
+; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %out = shufflevector <4 x float> %src, <4 x float> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
@@ -344,21 +345,25 @@ entry:
 define arm_aapcs_vfpcc <8 x half> @shuffle1_f16(<8 x half> %src) {
 ; CHECK-LABEL: shuffle1_f16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov.u16 r0, q0[7]
-; CHECK-NEXT:    vmov.u16 r1, q0[6]
-; CHECK-NEXT:    vmov.16 q1[0], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[5]
-; CHECK-NEXT:    vmov.16 q1[1], r1
+; CHECK-NEXT:    vmovx.f16 s4, s3
+; CHECK-NEXT:    vmov r0, s3
+; CHECK-NEXT:    vmov r1, s4
+; CHECK-NEXT:    vmovx.f16 s8, s2
+; CHECK-NEXT:    vmov.16 q1[0], r1
+; CHECK-NEXT:    vmov.16 q1[1], r0
+; CHECK-NEXT:    vmov r0, s8
 ; CHECK-NEXT:    vmov.16 q1[2], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[4]
+; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    vmovx.f16 s8, s1
 ; CHECK-NEXT:    vmov.16 q1[3], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[3]
+; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    vmovx.f16 s8, s0
 ; CHECK-NEXT:    vmov.16 q1[4], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[2]
+; CHECK-NEXT:    vmov r0, s1
 ; CHECK-NEXT:    vmov.16 q1[5], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[1]
+; CHECK-NEXT:    vmov r0, s8
 ; CHECK-NEXT:    vmov.16 q1[6], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[0]
+; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vmov.16 q1[7], r0
 ; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    bx lr
@@ -379,22 +384,22 @@ entry:
 define arm_aapcs_vfpcc <8 x half> @shuffle3_f16(<8 x half> %src) {
 ; CHECK-LABEL: shuffle3_f16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov.u16 r0, q0[4]
-; CHECK-NEXT:    vmov.u16 r1, q0[5]
-; CHECK-NEXT:    vmov.16 q1[0], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[7]
-; CHECK-NEXT:    vmov.16 q1[1], r1
-; CHECK-NEXT:    vmov.16 q1[2], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[6]
+; CHECK-NEXT:    vmovx.f16 s4, s3
+; CHECK-NEXT:    vmov r0, s3
+; CHECK-NEXT:    vmov r1, s4
+; CHECK-NEXT:    vmovx.f16 s8, s1
+; CHECK-NEXT:    vmov.16 q1[2], r1
 ; CHECK-NEXT:    vmov.16 q1[3], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[3]
+; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    vmovx.f16 s8, s0
 ; CHECK-NEXT:    vmov.16 q1[4], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[1]
+; CHECK-NEXT:    vmov r0, s8
 ; CHECK-NEXT:    vmov.16 q1[5], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[2]
+; CHECK-NEXT:    vmov r0, s1
 ; CHECK-NEXT:    vmov.16 q1[6], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[0]
+; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vmov.16 q1[7], r0
+; CHECK-NEXT:    vmov.f32 s4, s2
 ; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
@@ -405,7 +410,8 @@ entry:
 define arm_aapcs_vfpcc <8 x half> @shuffle5_f16(<8 x half> %src) {
 ; CHECK-LABEL: shuffle5_f16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vrev64.16 q0, q0
+; CHECK-NEXT:    vrev64.16 q1, q0
+; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %out = shufflevector <8 x half> %src, <8 x half> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
@@ -676,8 +682,6 @@ entry:
 define arm_aapcs_vfpcc half @extract_f16_0(<8 x half> %a) {
 ; CHECK-LABEL: extract_f16_0:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov.u16 r1, q0[0]
-; CHECK-NEXT:    vmov s0, r1
 ; CHECK-NEXT:    vstr.16 s0, [r0]
 ; CHECK-NEXT:    bx lr
 entry:
@@ -688,8 +692,7 @@ entry:
 define arm_aapcs_vfpcc half @extract_f16_3(<8 x half> %a) {
 ; CHECK-LABEL: extract_f16_3:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov.u16 r1, q0[3]
-; CHECK-NEXT:    vmov s0, r1
+; CHECK-NEXT:    vmovx.f16 s0, s1
 ; CHECK-NEXT:    vstr.16 s0, [r0]
 ; CHECK-NEXT:    bx lr
 entry:
diff --git a/llvm/test/CodeGen/Thumb2/mve-shuffleext.ll b/llvm/test/CodeGen/Thumb2/mve-shuffleext.ll
new file mode 100644
index 000000000..74596ff39
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-shuffleext.ll
@@ -0,0 +1,94 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK
+
+define arm_aapcs_vfpcc <4 x i32> @sext_0246(<8 x i16> %src) {
+; CHECK-LABEL: sext_0246:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmovlb.s16 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %strided.vec = shufflevector <8 x i16> %src, <8 x i16> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  %out = sext <4 x i16> %strided.vec to <4 x i32>
+  ret <4 x i32> %out
+}
+
+define arm_aapcs_vfpcc <4 x i32> @sext_1357(<8 x i16> %src) {
+; CHECK-LABEL: sext_1357:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vrev32.16 q0, q0
+; CHECK-NEXT:    vmovlb.s16 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %strided.vec = shufflevector <8 x i16> %src, <8 x i16> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+  %out = sext <4 x i16> %strided.vec to <4 x i32>
+  ret <4 x i32> %out
+}
+
+define arm_aapcs_vfpcc <4 x i32> @zext_0246(<8 x i16> %src) {
+; CHECK-LABEL: zext_0246:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmovlb.u16 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %strided.vec = shufflevector <8 x i16> %src, <8 x i16> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  %out = zext <4 x i16> %strided.vec to <4 x i32>
+  ret <4 x i32> %out
+}
+
+define arm_aapcs_vfpcc <4 x i32> @zext_1357(<8 x i16> %src) {
+; CHECK-LABEL: zext_1357:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vrev32.16 q0, q0
+; CHECK-NEXT:    vmovlb.u16 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %strided.vec = shufflevector <8 x i16> %src, <8 x i16> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+  %out = zext <4 x i16> %strided.vec to <4 x i32>
+  ret <4 x i32> %out
+}
+
+define arm_aapcs_vfpcc <8 x i16> @sext_02468101214(<16 x i8> %src) {
+; CHECK-LABEL: sext_02468101214:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmovlb.s8 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %strided.vec = shufflevector <16 x i8> %src, <16 x i8> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+  %out = sext <8 x i8> %strided.vec to <8 x i16>
+  ret <8 x i16> %out
+}
+
+define arm_aapcs_vfpcc <8 x i16> @sext_13579111315(<16 x i8> %src) {
+; CHECK-LABEL: sext_13579111315:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vrev16.8 q0, q0
+; CHECK-NEXT:    vmovlb.s8 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %strided.vec = shufflevector <16 x i8> %src, <16 x i8> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+  %out = sext <8 x i8> %strided.vec to <8 x i16>
+  ret <8 x i16> %out
+}
+
+define arm_aapcs_vfpcc <8 x i16> @zext_02468101214(<16 x i8> %src) {
+; CHECK-LABEL: zext_02468101214:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmovlb.u8 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %strided.vec = shufflevector <16 x i8> %src, <16 x i8> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+  %out = zext <8 x i8> %strided.vec to <8 x i16>
+  ret <8 x i16> %out
+}
+
+define arm_aapcs_vfpcc <8 x i16> @zext_13579111315(<16 x i8> %src) {
+; CHECK-LABEL: zext_13579111315:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vrev16.8 q0, q0
+; CHECK-NEXT:    vmovlb.u8 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %strided.vec = shufflevector <16 x i8> %src, <16 x i8> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+  %out = zext <8 x i8> %strided.vec to <8 x i16>
+  ret <8 x i16> %out
+}
diff --git a/llvm/test/CodeGen/Thumb2/mve-shufflemov.ll b/llvm/test/CodeGen/Thumb2/mve-shufflemov.ll
new file mode 100644
index 000000000..164d42f15
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-shufflemov.ll
@@ -0,0 +1,442 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve,+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve.fp -verify-machineinstrs %s -o - | FileCheck %s
+
+; i16
+
+define arm_aapcs_vfpcc <8 x i16> @shuffle_i16_45670123(<8 x i16> %s1, <8 x i16> %s2) {
+; CHECK-LABEL: shuffle_i16_45670123:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.f32 s4, s2
+; CHECK-NEXT:    vmov.f32 s5, s3
+; CHECK-NEXT:    vmov.f32 s6, s0
+; CHECK-NEXT:    vmov.f32 s7, s1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %out = shufflevector <8 x i16> %s1, <8 x i16> %s2, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>
+  ret <8 x i16> %out
+}
+
+define arm_aapcs_vfpcc <8 x i16> @shuffle_i16_67452301(<8 x i16> %s1, <8 x i16> %s2) {
+; CHECK-LABEL: shuffle_i16_67452301:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.f32 s4, s3
+; CHECK-NEXT:    vmov.f32 s5, s2
+; CHECK-NEXT:    vmov.f32 s6, s1
+; CHECK-NEXT:    vmov.f32 s7, s0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %out = shufflevector <8 x i16> %s1, <8 x i16> %s2, <8 x i32> <i32 6, i32 7, i32 4, i32 5, i32 2, i32 3, i32 0, i32 1>
+  ret <8 x i16> %out
+}
+
+define arm_aapcs_vfpcc <8 x i16> @shuffle_i16_76543210(<8 x i16> %s1, <8 x i16> %s2) {
+; CHECK-LABEL: shuffle_i16_76543210:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov q1, q0
+; CHECK-NEXT:    vmov.u16 r0, q0[7]
+; CHECK-NEXT:    vmov.16 q0[0], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[6]
+; CHECK-NEXT:    vmov.16 q0[1], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[5]
+; CHECK-NEXT:    vmov.16 q0[2], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[4]
+; CHECK-NEXT:    vmov.16 q0[3], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[3]
+; CHECK-NEXT:    vmov.16 q0[4], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[2]
+; CHECK-NEXT:    vmov.16 q0[5], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[1]
+; CHECK-NEXT:    vmov.16 q0[6], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[0]
+; CHECK-NEXT:    vmov.16 q0[7], r0
+; CHECK-NEXT:    bx lr
+entry:
+  %out = shufflevector <8 x i16> %s1, <8 x i16> %s2, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <8 x i16> %out
+}
+
+define arm_aapcs_vfpcc <8 x i16> @shuffle_i16_01234567(<8 x i16> %s1, <8 x i16> %s2) {
+; CHECK-LABEL: shuffle_i16_01234567:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    bx lr
+entry:
+  %out = shufflevector <8 x i16> %s1, <8 x i16> %s2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  ret <8 x i16> %out
+}
+
+define arm_aapcs_vfpcc <8 x i16> @shuffle_i16_0123cdef(<8 x i16> %s1, <8 x i16> %s2) {
+; CHECK-LABEL: shuffle_i16_0123cdef:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.f32 s2, s6
+; CHECK-NEXT:    vmov.f32 s3, s7
+; CHECK-NEXT:    bx lr
+entry:
+  %out = shufflevector <8 x i16> %s1, <8 x i16> %s2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
+  ret <8 x i16> %out
+}
+
+define arm_aapcs_vfpcc <8 x i16> @shuffle_i16_u7u5u3u1(<8 x i16> %s1, <8 x i16> %s2) {
+; CHECK-LABEL: shuffle_i16_u7u5u3u1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.f32 s4, s3
+; CHECK-NEXT:    vmov.f32 s5, s2
+; CHECK-NEXT:    vmov.f32 s6, s1
+; CHECK-NEXT:    vmov.f32 s7, s0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %out = shufflevector <8 x i16> %s1, <8 x i16> %s2, <8 x i32> <i32 undef, i32 7, i32 undef, i32 5, i32 undef, i32 3, i32 undef, i32 1>
+  ret <8 x i16> %out
+}
+
+define arm_aapcs_vfpcc <8 x i16> @shuffle_i16_6u4u2u0u(<8 x i16> %s1, <8 x i16> %s2) {
+; CHECK-LABEL: shuffle_i16_6u4u2u0u:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.f32 s4, s3
+; CHECK-NEXT:    vmov.f32 s5, s2
+; CHECK-NEXT:    vmov.f32 s6, s1
+; CHECK-NEXT:    vmov.f32 s7, s0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %out = shufflevector <8 x i16> %s1, <8 x i16> %s2, <8 x i32> <i32 6, i32 undef, i32 4, i32 undef, i32 2, i32 undef, i32 0, i32 undef>
+  ret <8 x i16> %out
+}
+
+define arm_aapcs_vfpcc <8 x i16> @shuffle_i16_0uuuuuuu(<8 x i16> %s1, <8 x i16> %s2) {
+; CHECK-LABEL: shuffle_i16_0uuuuuuu:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    bx lr
+entry:
+  %out = shufflevector <8 x i16> %s1, <8 x i16> %s2, <8 x i32> <i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  ret <8 x i16> %out
+}
+
+define arm_aapcs_vfpcc <8 x i16> @shuffle_i16_uuuu0uuu(<8 x i16> %s1, <8 x i16> %s2) {
+; CHECK-LABEL: shuffle_i16_uuuu0uuu:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.u16 r0, q0[0]
+; CHECK-NEXT:    vdup.16 q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %out = shufflevector <8 x i16> %s1, <8 x i16> %s2, <8 x i32> <i32 undef, i32 undef, i32 undef, i32 undef, i32 0, i32 undef, i32 undef, i32 undef>
+  ret <8 x i16> %out
+}
+
+
+; i8
+
+define arm_aapcs_vfpcc <16 x i8> @shuffle_i8_cdef89ab45670123(<16 x i8> %s1, <16 x i8> %s2) {
+; CHECK-LABEL: shuffle_i8_cdef89ab45670123:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.f32 s4, s3
+; CHECK-NEXT:    vmov.f32 s5, s2
+; CHECK-NEXT:    vmov.f32 s6, s1
+; CHECK-NEXT:    vmov.f32 s7, s0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %out = shufflevector <16 x i8> %s1, <16 x i8> %s2, <16 x i32> <i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>
+  ret <16 x i8> %out
+}
+
+define arm_aapcs_vfpcc <16 x i8> @shuffle_i8_efcdab8967452301(<16 x i8> %s1, <16 x i8> %s2) {
+; CHECK-LABEL: shuffle_i8_efcdab8967452301:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov q1, q0
+; CHECK-NEXT:    vmov.u8 r0, q0[14]
+; CHECK-NEXT:    vmov.8 q0[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[15]
+; CHECK-NEXT:    vmov.8 q0[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[12]
+; CHECK-NEXT:    vmov.8 q0[2], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[13]
+; CHECK-NEXT:    vmov.8 q0[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[10]
+; CHECK-NEXT:    vmov.8 q0[4], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[11]
+; CHECK-NEXT:    vmov.8 q0[5], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[8]
+; CHECK-NEXT:    vmov.8 q0[6], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[9]
+; CHECK-NEXT:    vmov.8 q0[7], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[6]
+; CHECK-NEXT:    vmov.8 q0[8], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[7]
+; CHECK-NEXT:    vmov.8 q0[9], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[4]
+; CHECK-NEXT:    vmov.8 q0[10], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[5]
+; CHECK-NEXT:    vmov.8 q0[11], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[2]
+; CHECK-NEXT:    vmov.8 q0[12], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[3]
+; CHECK-NEXT:    vmov.8 q0[13], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[0]
+; CHECK-NEXT:    vmov.8 q0[14], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[1]
+; CHECK-NEXT:    vmov.8 q0[15], r0
+; CHECK-NEXT:    bx lr
+entry:
+  %out = shufflevector <16 x i8> %s1, <16 x i8> %s2, <16 x i32> <i32 14, i32 15, i32 12, i32 13, i32 10, i32 11, i32 8, i32 9, i32 6, i32 7, i32 4, i32 5, i32 2, i32 3, i32 0, i32 1>
+  ret <16 x i8> %out
+}
+
+define arm_aapcs_vfpcc <16 x i8> @shuffle_i8_fedcba9876543210(<16 x i8> %s1, <16 x i8> %s2) {
+; CHECK-LABEL: shuffle_i8_fedcba9876543210:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov q1, q0
+; CHECK-NEXT:    vmov.u8 r0, q0[15]
+; CHECK-NEXT:    vmov.8 q0[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[14]
+; CHECK-NEXT:    vmov.8 q0[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[13]
+; CHECK-NEXT:    vmov.8 q0[2], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[12]
+; CHECK-NEXT:    vmov.8 q0[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[11]
+; CHECK-NEXT:    vmov.8 q0[4], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[10]
+; CHECK-NEXT:    vmov.8 q0[5], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[9]
+; CHECK-NEXT:    vmov.8 q0[6], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[8]
+; CHECK-NEXT:    vmov.8 q0[7], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[7]
+; CHECK-NEXT:    vmov.8 q0[8], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[6]
+; CHECK-NEXT:    vmov.8 q0[9], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[5]
+; CHECK-NEXT:    vmov.8 q0[10], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[4]
+; CHECK-NEXT:    vmov.8 q0[11], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[3]
+; CHECK-NEXT:    vmov.8 q0[12], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[2]
+; CHECK-NEXT:    vmov.8 q0[13], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[1]
+; CHECK-NEXT:    vmov.8 q0[14], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[0]
+; CHECK-NEXT:    vmov.8 q0[15], r0
+; CHECK-NEXT:    bx lr
+entry:
+  %out = shufflevector <16 x i8> %s1, <16 x i8> %s2, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <16 x i8> %out
+}
+
+define arm_aapcs_vfpcc <16 x i8> @shuffle_i8_0123456789abcdef(<16 x i8> %s1, <16 x i8> %s2) {
+; CHECK-LABEL: shuffle_i8_0123456789abcdef:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    bx lr
+entry:
+  %out = shufflevector <16 x i8> %s1, <16 x i8> %s2, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  ret <16 x i8> %out
+}
+
+define arm_aapcs_vfpcc <16 x i8> @shuffle_i8_0123ghij4567klmn(<16 x i8> %s1, <16 x i8> %s2) {
+; CHECK-LABEL: shuffle_i8_0123ghij4567klmn:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.f32 s8, s0
+; CHECK-NEXT:    vmov.f32 s9, s4
+; CHECK-NEXT:    vmov.f32 s10, s1
+; CHECK-NEXT:    vmov.f32 s11, s5
+; CHECK-NEXT:    vmov q0, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %out = shufflevector <16 x i8> %s1, <16 x i8> %s2, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 20, i32 21, i32 22, i32 23>
+  ret <16 x i8> %out
+}
+
+define arm_aapcs_vfpcc <16 x i8> @shuffle_i8_cdeu89ub4u67u123(<16 x i8> %s1, <16 x i8> %s2) {
+; CHECK-LABEL: shuffle_i8_cdeu89ub4u67u123:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.f32 s4, s3
+; CHECK-NEXT:    vmov.f32 s5, s2
+; CHECK-NEXT:    vmov.f32 s6, s1
+; CHECK-NEXT:    vmov.f32 s7, s0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %out = shufflevector <16 x i8> %s1, <16 x i8> %s2, <16 x i32> <i32 12, i32 13, i32 14, i32 undef, i32 8, i32 9, i32 undef, i32 11, i32 4, i32 undef, i32 6, i32 7, i32 undef, i32 1, i32 2, i32 3>
+  ret <16 x i8> %out
+}
+
+define arm_aapcs_vfpcc <16 x i8> @shuffle_i8_cduu8uubuu67u12u(<16 x i8> %s1, <16 x i8> %s2) {
+; CHECK-LABEL: shuffle_i8_cduu8uubuu67u12u:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.f32 s4, s3
+; CHECK-NEXT:    vmov.f32 s5, s2
+; CHECK-NEXT:    vmov.f32 s6, s1
+; CHECK-NEXT:    vmov.f32 s7, s0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %out = shufflevector <16 x i8> %s1, <16 x i8> %s2, <16 x i32> <i32 12, i32 13, i32 undef, i32 undef, i32 8, i32 undef, i32 undef, i32 11, i32 undef, i32 undef, i32 6, i32 7, i32 undef, i32 1, i32 2, i32 undef>
+  ret <16 x i8> %out
+}
+
+define arm_aapcs_vfpcc <16 x i8> @shuffle_i8_cuuuuuubuu6uuu2u(<16 x i8> %s1, <16 x i8> %s2) {
+; CHECK-LABEL: shuffle_i8_cuuuuuubuu6uuu2u:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.f32 s4, s3
+; CHECK-NEXT:    vmov.f32 s5, s2
+; CHECK-NEXT:    vmov.f32 s6, s1
+; CHECK-NEXT:    vmov.f32 s7, s0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %out = shufflevector <16 x i8> %s1, <16 x i8> %s2, <16 x i32> <i32 12, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 11, i32 undef, i32 undef, i32 6, i32 undef, i32 undef, i32 undef, i32 2, i32 undef>
+  ret <16 x i8> %out
+}
+
+define arm_aapcs_vfpcc <16 x i8> @shuffle_i8_cdef89ab45u700123(<16 x i8> %s1, <16 x i8> %s2) {
+; CHECK-LABEL: shuffle_i8_cdef89ab45u700123:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.u8 r0, q0[4]
+; CHECK-NEXT:    vmov.8 q1[8], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[5]
+; CHECK-NEXT:    vmov.8 q1[9], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[0]
+; CHECK-NEXT:    vmov.8 q1[11], r0
+; CHECK-NEXT:    vmov.f32 s4, s3
+; CHECK-NEXT:    vmov.f32 s5, s2
+; CHECK-NEXT:    vmov.f32 s7, s0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %out = shufflevector <16 x i8> %s1, <16 x i8> %s2, <16 x i32> <i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 undef, i32 0, i32 0, i32 1, i32 2, i32 3>
+  ret <16 x i8> %out
+}
+
+
+
+; f16
+
+define arm_aapcs_vfpcc <8 x half> @shuffle_f16_45670123(<8 x half> %s1, <8 x half> %s2) {
+; CHECK-LABEL: shuffle_f16_45670123:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.f32 s4, s2
+; CHECK-NEXT:    vmov.f32 s5, s3
+; CHECK-NEXT:    vmov.f32 s6, s0
+; CHECK-NEXT:    vmov.f32 s7, s1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %out = shufflevector <8 x half> %s1, <8 x half> %s2, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>
+  ret <8 x half> %out
+}
+
+define arm_aapcs_vfpcc <8 x half> @shuffle_f16_67452301(<8 x half> %s1, <8 x half> %s2) {
+; CHECK-LABEL: shuffle_f16_67452301:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.f32 s4, s3
+; CHECK-NEXT:    vmov.f32 s5, s2
+; CHECK-NEXT:    vmov.f32 s6, s1
+; CHECK-NEXT:    vmov.f32 s7, s0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %out = shufflevector <8 x half> %s1, <8 x half> %s2, <8 x i32> <i32 6, i32 7, i32 4, i32 5, i32 2, i32 3, i32 0, i32 1>
+  ret <8 x half> %out
+}
+
+define arm_aapcs_vfpcc <8 x half> @shuffle_f16_76543210(<8 x half> %s1, <8 x half> %s2) {
+; CHECK-LABEL: shuffle_f16_76543210:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmovx.f16 s4, s3
+; CHECK-NEXT:    vmov r0, s3
+; CHECK-NEXT:    vmov r1, s4
+; CHECK-NEXT:    vmovx.f16 s8, s2
+; CHECK-NEXT:    vmov.16 q1[0], r1
+; CHECK-NEXT:    vmov.16 q1[1], r0
+; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    vmov.16 q1[2], r0
+; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    vmovx.f16 s8, s1
+; CHECK-NEXT:    vmov.16 q1[3], r0
+; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    vmovx.f16 s8, s0
+; CHECK-NEXT:    vmov.16 q1[4], r0
+; CHECK-NEXT:    vmov r0, s1
+; CHECK-NEXT:    vmov.16 q1[5], r0
+; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    vmov.16 q1[6], r0
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vmov.16 q1[7], r0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %out = shufflevector <8 x half> %s1, <8 x half> %s2, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <8 x half> %out
+}
+
+define arm_aapcs_vfpcc <8 x half> @shuffle_f16_01234567(<8 x half> %s1, <8 x half> %s2) {
+; CHECK-LABEL: shuffle_f16_01234567:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    bx lr
+entry:
+  %out = shufflevector <8 x half> %s1, <8 x half> %s2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  ret <8 x half> %out
+}
+
+define arm_aapcs_vfpcc <8 x half> @shuffle_f16_0123cdef(<8 x half> %s1, <8 x half> %s2) {
+; CHECK-LABEL: shuffle_f16_0123cdef:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.f32 s2, s6
+; CHECK-NEXT:    vmov.f32 s3, s7
+; CHECK-NEXT:    bx lr
+entry:
+  %out = shufflevector <8 x half> %s1, <8 x half> %s2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
+  ret <8 x half> %out
+}
+
+define arm_aapcs_vfpcc <8 x half> @shuffle_f16_u7u5u3u1(<8 x half> %s1, <8 x half> %s2) {
+; CHECK-LABEL: shuffle_f16_u7u5u3u1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.f32 s4, s3
+; CHECK-NEXT:    vmov.f32 s5, s2
+; CHECK-NEXT:    vmov.f32 s6, s1
+; CHECK-NEXT:    vmov.f32 s7, s0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %out = shufflevector <8 x half> %s1, <8 x half> %s2, <8 x i32> <i32 undef, i32 7, i32 undef, i32 5, i32 undef, i32 3, i32 undef, i32 1>
+  ret <8 x half> %out
+}
+
+define arm_aapcs_vfpcc <8 x half> @shuffle_f16_6u4u2u0u(<8 x half> %s1, <8 x half> %s2) {
+; CHECK-LABEL: shuffle_f16_6u4u2u0u:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.f32 s4, s3
+; CHECK-NEXT:    vmov.f32 s5, s2
+; CHECK-NEXT:    vmov.f32 s6, s1
+; CHECK-NEXT:    vmov.f32 s7, s0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %out = shufflevector <8 x half> %s1, <8 x half> %s2, <8 x i32> <i32 6, i32 undef, i32 4, i32 undef, i32 2, i32 undef, i32 0, i32 undef>
+  ret <8 x half> %out
+}
+
+define arm_aapcs_vfpcc <8 x half> @shuffle_f16_0uuuuuuu(<8 x half> %s1, <8 x half> %s2) {
+; CHECK-LABEL: shuffle_f16_0uuuuuuu:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    bx lr
+entry:
+  %out = shufflevector <8 x half> %s1, <8 x half> %s2, <8 x i32> <i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  ret <8 x half> %out
+}
+
+define arm_aapcs_vfpcc <8 x half> @shuffle_f16_uuuu0uuu(<8 x half> %s1, <8 x half> %s2) {
+; CHECK-LABEL: shuffle_f16_uuuu0uuu:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.u16 r0, q0[0]
+; CHECK-NEXT:    vdup.16 q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %out = shufflevector <8 x half> %s1, <8 x half> %s2, <8 x i32> <i32 undef, i32 undef, i32 undef, i32 undef, i32 0, i32 undef, i32 undef, i32 undef>
+  ret <8 x half> %out
+}
diff --git a/llvm/test/CodeGen/Thumb2/mve-simple-arith.ll b/llvm/test/CodeGen/Thumb2/mve-simple-arith.ll
index ecad0c1c5..92b6ae533 100644
--- a/llvm/test/CodeGen/Thumb2/mve-simple-arith.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-simple-arith.ll
@@ -81,60 +81,36 @@ entry:
 define arm_aapcs_vfpcc <8 x half> @add_float16_t(<8 x half> %src1, <8 x half> %src2) {
 ; CHECK-MVE-LABEL: add_float16_t:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[0]
-; CHECK-MVE-NEXT:    vmov.u16 r1, q0[1]
-; CHECK-MVE-NEXT:    vmov s8, r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q1[0]
-; CHECK-MVE-NEXT:    vmov s10, r0
-; CHECK-MVE-NEXT:    vmov.u16 r2, q1[1]
-; CHECK-MVE-NEXT:    vadd.f16 s8, s10, s8
-; CHECK-MVE-NEXT:    vmov s10, r2
+; CHECK-MVE-NEXT:    vadd.f16 s8, s4, s0
+; CHECK-MVE-NEXT:    vmovx.f16 s10, s4
 ; CHECK-MVE-NEXT:    vmov r0, s8
-; CHECK-MVE-NEXT:    vmov s8, r1
+; CHECK-MVE-NEXT:    vmovx.f16 s8, s0
 ; CHECK-MVE-NEXT:    vadd.f16 s8, s10, s8
+; CHECK-MVE-NEXT:    vadd.f16 s12, s5, s1
 ; CHECK-MVE-NEXT:    vmov r1, s8
 ; CHECK-MVE-NEXT:    vmov.16 q2[0], r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[2]
+; CHECK-MVE-NEXT:    vmov r0, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s1
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s5
 ; CHECK-MVE-NEXT:    vmov.16 q2[1], r1
-; CHECK-MVE-NEXT:    vmov s12, r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q1[2]
-; CHECK-MVE-NEXT:    vmov s14, r0
 ; CHECK-MVE-NEXT:    vadd.f16 s12, s14, s12
-; CHECK-MVE-NEXT:    vmov r0, s12
 ; CHECK-MVE-NEXT:    vmov.16 q2[2], r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[3]
-; CHECK-MVE-NEXT:    vmov s12, r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q1[3]
-; CHECK-MVE-NEXT:    vmov s14, r0
-; CHECK-MVE-NEXT:    vadd.f16 s12, s14, s12
 ; CHECK-MVE-NEXT:    vmov r0, s12
+; CHECK-MVE-NEXT:    vadd.f16 s12, s6, s2
 ; CHECK-MVE-NEXT:    vmov.16 q2[3], r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[4]
-; CHECK-MVE-NEXT:    vmov s12, r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q1[4]
-; CHECK-MVE-NEXT:    vmov s14, r0
-; CHECK-MVE-NEXT:    vadd.f16 s12, s14, s12
 ; CHECK-MVE-NEXT:    vmov r0, s12
-; CHECK-MVE-NEXT:    vmov.16 q2[4], r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[5]
-; CHECK-MVE-NEXT:    vmov s12, r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q1[5]
-; CHECK-MVE-NEXT:    vmov s14, r0
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s2
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s6
 ; CHECK-MVE-NEXT:    vadd.f16 s12, s14, s12
+; CHECK-MVE-NEXT:    vmov.16 q2[4], r0
 ; CHECK-MVE-NEXT:    vmov r0, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s7
+; CHECK-MVE-NEXT:    vadd.f16 s12, s7, s3
 ; CHECK-MVE-NEXT:    vmov.16 q2[5], r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[6]
-; CHECK-MVE-NEXT:    vmov s12, r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q1[6]
-; CHECK-MVE-NEXT:    vmov s14, r0
-; CHECK-MVE-NEXT:    vadd.f16 s12, s14, s12
 ; CHECK-MVE-NEXT:    vmov r0, s12
-; CHECK-MVE-NEXT:    vmov.16 q2[6], r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[7]
-; CHECK-MVE-NEXT:    vmov s0, r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q1[7]
-; CHECK-MVE-NEXT:    vmov s2, r0
 ; CHECK-MVE-NEXT:    vadd.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov.16 q2[6], r0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q2[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q2
@@ -256,60 +232,36 @@ entry:
 define arm_aapcs_vfpcc <8 x half> @sub_float16_t(<8 x half> %src1, <8 x half> %src2) {
 ; CHECK-MVE-LABEL: sub_float16_t:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[0]
-; CHECK-MVE-NEXT:    vmov.u16 r1, q0[1]
-; CHECK-MVE-NEXT:    vmov s8, r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q1[0]
-; CHECK-MVE-NEXT:    vmov s10, r0
-; CHECK-MVE-NEXT:    vmov.u16 r2, q1[1]
-; CHECK-MVE-NEXT:    vsub.f16 s8, s10, s8
-; CHECK-MVE-NEXT:    vmov s10, r2
+; CHECK-MVE-NEXT:    vsub.f16 s8, s4, s0
+; CHECK-MVE-NEXT:    vmovx.f16 s10, s4
 ; CHECK-MVE-NEXT:    vmov r0, s8
-; CHECK-MVE-NEXT:    vmov s8, r1
+; CHECK-MVE-NEXT:    vmovx.f16 s8, s0
 ; CHECK-MVE-NEXT:    vsub.f16 s8, s10, s8
+; CHECK-MVE-NEXT:    vsub.f16 s12, s5, s1
 ; CHECK-MVE-NEXT:    vmov r1, s8
 ; CHECK-MVE-NEXT:    vmov.16 q2[0], r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[2]
+; CHECK-MVE-NEXT:    vmov r0, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s1
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s5
 ; CHECK-MVE-NEXT:    vmov.16 q2[1], r1
-; CHECK-MVE-NEXT:    vmov s12, r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q1[2]
-; CHECK-MVE-NEXT:    vmov s14, r0
 ; CHECK-MVE-NEXT:    vsub.f16 s12, s14, s12
-; CHECK-MVE-NEXT:    vmov r0, s12
 ; CHECK-MVE-NEXT:    vmov.16 q2[2], r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[3]
-; CHECK-MVE-NEXT:    vmov s12, r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q1[3]
-; CHECK-MVE-NEXT:    vmov s14, r0
-; CHECK-MVE-NEXT:    vsub.f16 s12, s14, s12
 ; CHECK-MVE-NEXT:    vmov r0, s12
+; CHECK-MVE-NEXT:    vsub.f16 s12, s6, s2
 ; CHECK-MVE-NEXT:    vmov.16 q2[3], r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[4]
-; CHECK-MVE-NEXT:    vmov s12, r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q1[4]
-; CHECK-MVE-NEXT:    vmov s14, r0
-; CHECK-MVE-NEXT:    vsub.f16 s12, s14, s12
 ; CHECK-MVE-NEXT:    vmov r0, s12
-; CHECK-MVE-NEXT:    vmov.16 q2[4], r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[5]
-; CHECK-MVE-NEXT:    vmov s12, r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q1[5]
-; CHECK-MVE-NEXT:    vmov s14, r0
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s2
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s6
 ; CHECK-MVE-NEXT:    vsub.f16 s12, s14, s12
+; CHECK-MVE-NEXT:    vmov.16 q2[4], r0
 ; CHECK-MVE-NEXT:    vmov r0, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s7
+; CHECK-MVE-NEXT:    vsub.f16 s12, s7, s3
 ; CHECK-MVE-NEXT:    vmov.16 q2[5], r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[6]
-; CHECK-MVE-NEXT:    vmov s12, r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q1[6]
-; CHECK-MVE-NEXT:    vmov s14, r0
-; CHECK-MVE-NEXT:    vsub.f16 s12, s14, s12
 ; CHECK-MVE-NEXT:    vmov r0, s12
-; CHECK-MVE-NEXT:    vmov.16 q2[6], r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[7]
-; CHECK-MVE-NEXT:    vmov s0, r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q1[7]
-; CHECK-MVE-NEXT:    vmov s2, r0
 ; CHECK-MVE-NEXT:    vsub.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov.16 q2[6], r0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q2[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q2
@@ -414,60 +366,36 @@ entry:
 define arm_aapcs_vfpcc <8 x half> @mul_float16_t(<8 x half> %src1, <8 x half> %src2) {
 ; CHECK-MVE-LABEL: mul_float16_t:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[0]
-; CHECK-MVE-NEXT:    vmov.u16 r1, q0[1]
-; CHECK-MVE-NEXT:    vmov s8, r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q1[0]
-; CHECK-MVE-NEXT:    vmov s10, r0
-; CHECK-MVE-NEXT:    vmov.u16 r2, q1[1]
-; CHECK-MVE-NEXT:    vmul.f16 s8, s10, s8
-; CHECK-MVE-NEXT:    vmov s10, r2
+; CHECK-MVE-NEXT:    vmul.f16 s8, s4, s0
+; CHECK-MVE-NEXT:    vmovx.f16 s10, s4
 ; CHECK-MVE-NEXT:    vmov r0, s8
-; CHECK-MVE-NEXT:    vmov s8, r1
+; CHECK-MVE-NEXT:    vmovx.f16 s8, s0
 ; CHECK-MVE-NEXT:    vmul.f16 s8, s10, s8
+; CHECK-MVE-NEXT:    vmul.f16 s12, s5, s1
 ; CHECK-MVE-NEXT:    vmov r1, s8
 ; CHECK-MVE-NEXT:    vmov.16 q2[0], r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[2]
+; CHECK-MVE-NEXT:    vmov r0, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s1
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s5
 ; CHECK-MVE-NEXT:    vmov.16 q2[1], r1
-; CHECK-MVE-NEXT:    vmov s12, r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q1[2]
-; CHECK-MVE-NEXT:    vmov s14, r0
 ; CHECK-MVE-NEXT:    vmul.f16 s12, s14, s12
-; CHECK-MVE-NEXT:    vmov r0, s12
 ; CHECK-MVE-NEXT:    vmov.16 q2[2], r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[3]
-; CHECK-MVE-NEXT:    vmov s12, r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q1[3]
-; CHECK-MVE-NEXT:    vmov s14, r0
-; CHECK-MVE-NEXT:    vmul.f16 s12, s14, s12
 ; CHECK-MVE-NEXT:    vmov r0, s12
+; CHECK-MVE-NEXT:    vmul.f16 s12, s6, s2
 ; CHECK-MVE-NEXT:    vmov.16 q2[3], r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[4]
-; CHECK-MVE-NEXT:    vmov s12, r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q1[4]
-; CHECK-MVE-NEXT:    vmov s14, r0
-; CHECK-MVE-NEXT:    vmul.f16 s12, s14, s12
 ; CHECK-MVE-NEXT:    vmov r0, s12
-; CHECK-MVE-NEXT:    vmov.16 q2[4], r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[5]
-; CHECK-MVE-NEXT:    vmov s12, r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q1[5]
-; CHECK-MVE-NEXT:    vmov s14, r0
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s2
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s6
 ; CHECK-MVE-NEXT:    vmul.f16 s12, s14, s12
+; CHECK-MVE-NEXT:    vmov.16 q2[4], r0
 ; CHECK-MVE-NEXT:    vmov r0, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s7
+; CHECK-MVE-NEXT:    vmul.f16 s12, s7, s3
 ; CHECK-MVE-NEXT:    vmov.16 q2[5], r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[6]
-; CHECK-MVE-NEXT:    vmov s12, r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q1[6]
-; CHECK-MVE-NEXT:    vmov s14, r0
-; CHECK-MVE-NEXT:    vmul.f16 s12, s14, s12
 ; CHECK-MVE-NEXT:    vmov r0, s12
-; CHECK-MVE-NEXT:    vmov.16 q2[6], r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[7]
-; CHECK-MVE-NEXT:    vmov s0, r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q1[7]
-; CHECK-MVE-NEXT:    vmov s2, r0
 ; CHECK-MVE-NEXT:    vmul.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov.16 q2[6], r0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q2[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q2
diff --git a/llvm/test/CodeGen/Thumb2/mve-stack.ll b/llvm/test/CodeGen/Thumb2/mve-stack.ll
new file mode 100644
index 000000000..41cc1162e
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-stack.ll
@@ -0,0 +1,278 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s
+
+define arm_aapcs_vfpcc void @vstrw32() {
+; CHECK-LABEL: vstrw32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    .pad #16
+; CHECK-NEXT:    sub sp, #16
+; CHECK-NEXT:    vmov.i32 q0, #0x0
+; CHECK-NEXT:    mov r0, sp
+; CHECK-NEXT:    vstrw.32 q0, [sp, #8]
+; CHECK-NEXT:    bl func
+; CHECK-NEXT:    add sp, #16
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %d = alloca [4 x i32], align 2
+  %g = getelementptr inbounds [4 x i32], [4 x i32]* %d, i32 0, i32 2
+  %b = bitcast i32* %g to <4 x i32>*
+  store <4 x i32> zeroinitializer, <4 x i32>* %b, align 2
+  %arraydecay = getelementptr inbounds [4 x i32], [4 x i32]* %d, i32 0, i32 0
+  call arm_aapcs_vfpcc void bitcast (void (...)* @func to void (i32*)*)(i32* %arraydecay)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @vstrh16() {
+; CHECK-LABEL: vstrh16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    .pad #16
+; CHECK-NEXT:    sub sp, #16
+; CHECK-NEXT:    vmov.i32 q0, #0x0
+; CHECK-NEXT:    mov r0, sp
+; CHECK-NEXT:    vstrh.16 q0, [sp, #4]
+; CHECK-NEXT:    bl func
+; CHECK-NEXT:    add sp, #16
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %d = alloca [8 x i16], align 2
+  %g = getelementptr inbounds [8 x i16], [8 x i16]* %d, i32 0, i32 2
+  %b = bitcast i16* %g to <8 x i16>*
+  store <8 x i16> zeroinitializer, <8 x i16>* %b, align 2
+  %arraydecay = getelementptr inbounds [8 x i16], [8 x i16]* %d, i32 0, i32 0
+  call arm_aapcs_vfpcc void bitcast (void (...)* @func to void (i16*)*)(i16* %arraydecay)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @vstrb8() {
+; CHECK-LABEL: vstrb8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    .pad #16
+; CHECK-NEXT:    sub sp, #16
+; CHECK-NEXT:    vmov.i32 q0, #0x0
+; CHECK-NEXT:    mov r0, sp
+; CHECK-NEXT:    vstrh.16 q0, [sp, #2]
+; CHECK-NEXT:    bl func
+; CHECK-NEXT:    add sp, #16
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %d = alloca [16 x i8], align 2
+  %g = getelementptr inbounds [16 x i8], [16 x i8]* %d, i32 0, i32 2
+  %b = bitcast i8* %g to <16 x i8>*
+  store <16 x i8> zeroinitializer, <16 x i8>* %b, align 2
+  %arraydecay = getelementptr inbounds [16 x i8], [16 x i8]* %d, i32 0, i32 0
+  call arm_aapcs_vfpcc void bitcast (void (...)* @func to void (i8*)*)(i8* %arraydecay)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @vstrh32() {
+; CHECK-LABEL: vstrh32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    .pad #8
+; CHECK-NEXT:    sub sp, #8
+; CHECK-NEXT:    mov r0, sp
+; CHECK-NEXT:    vmov.i32 q0, #0x6
+; CHECK-NEXT:    vstrh.32 q0, [r0, #4]
+; CHECK-NEXT:    bl func
+; CHECK-NEXT:    add sp, #8
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %d = alloca [4 x i16], align 2
+  %g = getelementptr inbounds [4 x i16], [4 x i16]* %d, i32 0, i32 2
+  %b = bitcast i16* %g to <4 x i16>*
+  store <4 x i16> <i16 6, i16 6, i16 6, i16 6>, <4 x i16>* %b, align 2
+  %arraydecay = getelementptr inbounds [4 x i16], [4 x i16]* %d, i32 0, i32 0
+  call arm_aapcs_vfpcc void bitcast (void (...)* @func to void (i16*)*)(i16* %arraydecay)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @vstrb32() {
+; CHECK-LABEL: vstrb32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    .pad #8
+; CHECK-NEXT:    sub sp, #8
+; CHECK-NEXT:    add r0, sp, #4
+; CHECK-NEXT:    vmov.i32 q0, #0x6
+; CHECK-NEXT:    vstrb.32 q0, [r0, #2]
+; CHECK-NEXT:    bl func
+; CHECK-NEXT:    add sp, #8
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %d = alloca [4 x i8], align 2
+  %g = getelementptr inbounds [4 x i8], [4 x i8]* %d, i32 0, i32 2
+  %b = bitcast i8* %g to <4 x i8>*
+  store <4 x i8> <i8 6, i8 6, i8 6, i8 6>, <4 x i8>* %b, align 2
+  %arraydecay = getelementptr inbounds [4 x i8], [4 x i8]* %d, i32 0, i32 0
+  call arm_aapcs_vfpcc void bitcast (void (...)* @func to void (i8*)*)(i8* %arraydecay)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @vstrb16() {
+; CHECK-LABEL: vstrb16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    .pad #8
+; CHECK-NEXT:    sub sp, #8
+; CHECK-NEXT:    mov r0, sp
+; CHECK-NEXT:    vmov.i32 q0, #0x0
+; CHECK-NEXT:    vstrb.16 q0, [r0, #2]
+; CHECK-NEXT:    bl func
+; CHECK-NEXT:    add sp, #8
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %d = alloca [8 x i8], align 2
+  %g = getelementptr inbounds [8 x i8], [8 x i8]* %d, i32 0, i32 2
+  %b = bitcast i8* %g to <8 x i8>*
+  store <8 x i8> zeroinitializer, <8 x i8>* %b, align 2
+  %arraydecay = getelementptr inbounds [8 x i8], [8 x i8]* %d, i32 0, i32 0
+  call arm_aapcs_vfpcc void bitcast (void (...)* @func to void (i8*)*)(i8* %arraydecay)
+  ret void
+}
+
+
+define arm_aapcs_vfpcc <4 x i32> @vldrw32() {
+; CHECK-LABEL: vldrw32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    .pad #16
+; CHECK-NEXT:    sub sp, #16
+; CHECK-NEXT:    mov r0, sp
+; CHECK-NEXT:    bl func
+; CHECK-NEXT:    vldrw.u32 q0, [sp, #8]
+; CHECK-NEXT:    add sp, #16
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %d = alloca [4 x i32], align 2
+  %arraydecay = getelementptr inbounds [4 x i32], [4 x i32]* %d, i32 0, i32 0
+  call arm_aapcs_vfpcc void bitcast (void (...)* @func to void (i32*)*)(i32* %arraydecay)
+  %g = getelementptr inbounds [4 x i32], [4 x i32]* %d, i32 0, i32 2
+  %b = bitcast i32* %g to <4 x i32>*
+  %l = load <4 x i32>, <4 x i32>* %b, align 2
+  ret <4 x i32> %l
+}
+
+define arm_aapcs_vfpcc <8 x i16> @vldrh16() {
+; CHECK-LABEL: vldrh16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    .pad #16
+; CHECK-NEXT:    sub sp, #16
+; CHECK-NEXT:    mov r0, sp
+; CHECK-NEXT:    bl func
+; CHECK-NEXT:    vldrh.u16 q0, [sp, #4]
+; CHECK-NEXT:    add sp, #16
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %d = alloca [8 x i16], align 2
+  %arraydecay = getelementptr inbounds [8 x i16], [8 x i16]* %d, i32 0, i32 0
+  call arm_aapcs_vfpcc void bitcast (void (...)* @func to void (i16*)*)(i16* %arraydecay)
+  %g = getelementptr inbounds [8 x i16], [8 x i16]* %d, i32 0, i32 2
+  %b = bitcast i16* %g to <8 x i16>*
+  %l = load <8 x i16>, <8 x i16>* %b, align 2
+  ret <8 x i16> %l
+}
+
+define arm_aapcs_vfpcc <16 x i8> @vldrb8() {
+; CHECK-LABEL: vldrb8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    .pad #16
+; CHECK-NEXT:    sub sp, #16
+; CHECK-NEXT:    mov r0, sp
+; CHECK-NEXT:    bl func
+; CHECK-NEXT:    vldrh.u16 q0, [sp, #2]
+; CHECK-NEXT:    add sp, #16
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %d = alloca [16 x i8], align 2
+  %arraydecay = getelementptr inbounds [16 x i8], [16 x i8]* %d, i32 0, i32 0
+  call arm_aapcs_vfpcc void bitcast (void (...)* @func to void (i8*)*)(i8* %arraydecay)
+  %g = getelementptr inbounds [16 x i8], [16 x i8]* %d, i32 0, i32 2
+  %b = bitcast i8* %g to <16 x i8>*
+  %l = load <16 x i8>, <16 x i8>* %b, align 2
+  ret <16 x i8> %l
+}
+
+define arm_aapcs_vfpcc <4 x i16> @vldrh32() {
+; CHECK-LABEL: vldrh32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, lr}
+; CHECK-NEXT:    push {r4, lr}
+; CHECK-NEXT:    .pad #8
+; CHECK-NEXT:    sub sp, #8
+; CHECK-NEXT:    mov r4, sp
+; CHECK-NEXT:    mov r0, r4
+; CHECK-NEXT:    bl func
+; CHECK-NEXT:    vldrh.u32 q0, [r4, #4]
+; CHECK-NEXT:    add sp, #8
+; CHECK-NEXT:    pop {r4, pc}
+entry:
+  %d = alloca [4 x i16], align 2
+  %arraydecay = getelementptr inbounds [4 x i16], [4 x i16]* %d, i32 0, i32 0
+  call arm_aapcs_vfpcc void bitcast (void (...)* @func to void (i16*)*)(i16* %arraydecay)
+  %g = getelementptr inbounds [4 x i16], [4 x i16]* %d, i32 0, i32 2
+  %b = bitcast i16* %g to <4 x i16>*
+  %l = load <4 x i16>, <4 x i16>* %b, align 2
+  ret <4 x i16> %l
+}
+
+define arm_aapcs_vfpcc <4 x i8> @vldrb32() {
+; CHECK-LABEL: vldrb32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, lr}
+; CHECK-NEXT:    push {r4, lr}
+; CHECK-NEXT:    .pad #8
+; CHECK-NEXT:    sub sp, #8
+; CHECK-NEXT:    add r4, sp, #4
+; CHECK-NEXT:    mov r0, r4
+; CHECK-NEXT:    bl func
+; CHECK-NEXT:    vldrb.u32 q0, [r4, #2]
+; CHECK-NEXT:    add sp, #8
+; CHECK-NEXT:    pop {r4, pc}
+entry:
+  %d = alloca [4 x i8], align 2
+  %arraydecay = getelementptr inbounds [4 x i8], [4 x i8]* %d, i32 0, i32 0
+  call arm_aapcs_vfpcc void bitcast (void (...)* @func to void (i8*)*)(i8* %arraydecay)
+  %g = getelementptr inbounds [4 x i8], [4 x i8]* %d, i32 0, i32 2
+  %b = bitcast i8* %g to <4 x i8>*
+  %l = load <4 x i8>, <4 x i8>* %b, align 2
+  ret <4 x i8> %l
+}
+
+define arm_aapcs_vfpcc <8 x i8> @vldrb16() {
+; CHECK-LABEL: vldrb16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, lr}
+; CHECK-NEXT:    push {r4, lr}
+; CHECK-NEXT:    .pad #8
+; CHECK-NEXT:    sub sp, #8
+; CHECK-NEXT:    mov r4, sp
+; CHECK-NEXT:    mov r0, r4
+; CHECK-NEXT:    bl func
+; CHECK-NEXT:    vldrb.u16 q0, [r4, #2]
+; CHECK-NEXT:    add sp, #8
+; CHECK-NEXT:    pop {r4, pc}
+entry:
+  %d = alloca [8 x i8], align 2
+  %arraydecay = getelementptr inbounds [8 x i8], [8 x i8]* %d, i32 0, i32 0
+  call arm_aapcs_vfpcc void bitcast (void (...)* @func to void (i8*)*)(i8* %arraydecay)
+  %g = getelementptr inbounds [8 x i8], [8 x i8]* %d, i32 0, i32 2
+  %b = bitcast i8* %g to <8 x i8>*
+  %l = load <8 x i8>, <8 x i8>* %b, align 2
+  ret <8 x i8> %l
+}
+
+declare dso_local arm_aapcs_vfpcc void @func(...)
diff --git a/llvm/test/CodeGen/Thumb2/mve-vaddqr.ll b/llvm/test/CodeGen/Thumb2/mve-vaddqr.ll
new file mode 100644
index 000000000..dd97bd52d
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-vaddqr.ll
@@ -0,0 +1,74 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -verify-machineinstrs -mattr=+mve %s -o - | FileCheck %s
+
+define arm_aapcs_vfpcc <4 x i32> @vaddqr_v4i32(<4 x i32> %src, i32 %src2, <4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: vaddqr_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vadd.i32 q0, q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <4 x i32> undef, i32 %src2, i32 0
+  %sp = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %c = add <4 x i32> %src, %sp
+  ret <4 x i32> %c
+}
+
+define arm_aapcs_vfpcc <8 x i16> @vaddqr_v8i16(<8 x i16> %src, i16 %src2, <8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: vaddqr_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vadd.i16 q0, q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <8 x i16> undef, i16 %src2, i32 0
+  %sp = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
+  %c = add <8 x i16> %src, %sp
+  ret <8 x i16> %c
+}
+
+define arm_aapcs_vfpcc <16 x i8> @vaddqr_v16i8(<16 x i8> %src, i8 %src2, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: vaddqr_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vadd.i8 q0, q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <16 x i8> undef, i8 %src2, i32 0
+  %sp = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
+  %c = add <16 x i8> %src, %sp
+  ret <16 x i8> %c
+}
+
+define arm_aapcs_vfpcc <4 x i32> @vaddqr_v4i32_2(<4 x i32> %src, i32 %src2, <4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: vaddqr_v4i32_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vadd.i32 q0, q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <4 x i32> undef, i32 %src2, i32 0
+  %sp = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %c = add <4 x i32> %sp, %src
+  ret <4 x i32> %c
+}
+
+define arm_aapcs_vfpcc <8 x i16> @vaddqr_v8i16_2(<8 x i16> %src, i16 %src2, <8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: vaddqr_v8i16_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vadd.i16 q0, q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <8 x i16> undef, i16 %src2, i32 0
+  %sp = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
+  %c = add <8 x i16> %sp, %src
+  ret <8 x i16> %c
+}
+
+define arm_aapcs_vfpcc <16 x i8> @vaddqr_v16i8_2(<16 x i8> %src, i8 %src2, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: vaddqr_v16i8_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vadd.i8 q0, q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <16 x i8> undef, i8 %src2, i32 0
+  %sp = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
+  %c = add <16 x i8> %sp, %src
+  ret <16 x i8> %c
+}
diff --git a/llvm/test/CodeGen/Thumb2/mve-vaddv.ll b/llvm/test/CodeGen/Thumb2/mve-vaddv.ll
new file mode 100644
index 000000000..94356e392
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-vaddv.ll
@@ -0,0 +1,105 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve.fp %s -o - | FileCheck %s
+
+declare i64 @llvm.experimental.vector.reduce.add.i64.v2i64(<2 x i64>)
+declare i32 @llvm.experimental.vector.reduce.add.i32.v4i32(<4 x i32>)
+declare i16 @llvm.experimental.vector.reduce.add.i16.v8i16(<8 x i16>)
+declare i8 @llvm.experimental.vector.reduce.add.i8.v16i8(<16 x i8>)
+
+define arm_aapcs_vfpcc i64 @vaddv_v2i64_i64(<2 x i64> %s1) {
+; CHECK-LABEL: vaddv_v2i64_i64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    vmov r1, s3
+; CHECK-NEXT:    vmov r2, s1
+; CHECK-NEXT:    adds r0, r0, r3
+; CHECK-NEXT:    adcs r1, r2
+; CHECK-NEXT:    bx lr
+entry:
+  %r = call i64 @llvm.experimental.vector.reduce.add.i64.v2i64(<2 x i64> %s1)
+  ret i64 %r
+}
+
+define arm_aapcs_vfpcc i32 @vaddv_v4i32_i32(<4 x i32> %s1) {
+; CHECK-LABEL: vaddv_v4i32_i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vaddv.u32 r0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %r = call i32 @llvm.experimental.vector.reduce.add.i32.v4i32(<4 x i32> %s1)
+  ret i32 %r
+}
+
+define arm_aapcs_vfpcc i16 @vaddv_v16i16_i16(<8 x i16> %s1) {
+; CHECK-LABEL: vaddv_v16i16_i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vaddv.u16 r0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %r = call i16 @llvm.experimental.vector.reduce.add.i16.v8i16(<8 x i16> %s1)
+  ret i16 %r
+}
+
+define arm_aapcs_vfpcc i8 @vaddv_v16i8_i8(<16 x i8> %s1) {
+; CHECK-LABEL: vaddv_v16i8_i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vaddv.u8 r0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %r = call i8 @llvm.experimental.vector.reduce.add.i8.v16i8(<16 x i8> %s1)
+  ret i8 %r
+}
+
+define arm_aapcs_vfpcc i64 @vaddva_v2i64_i64(<2 x i64> %s1, i64 %x) {
+; CHECK-LABEL: vaddva_v2i64_i64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    vmov r2, s2
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    vmov r12, s3
+; CHECK-NEXT:    vmov lr, s1
+; CHECK-NEXT:    adds r2, r2, r3
+; CHECK-NEXT:    adc.w r3, lr, r12
+; CHECK-NEXT:    adds r0, r0, r2
+; CHECK-NEXT:    adcs r1, r3
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %t = call i64 @llvm.experimental.vector.reduce.add.i64.v2i64(<2 x i64> %s1)
+  %r = add i64 %t, %x
+  ret i64 %r
+}
+
+define arm_aapcs_vfpcc i32 @vaddva_v4i32_i32(<4 x i32> %s1, i32 %x) {
+; CHECK-LABEL: vaddva_v4i32_i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vaddva.u32 r0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %t = call i32 @llvm.experimental.vector.reduce.add.i32.v4i32(<4 x i32> %s1)
+  %r = add i32 %t, %x
+  ret i32 %r
+}
+
+define arm_aapcs_vfpcc i16 @vaddva_v8i16_i16(<8 x i16> %s1, i16 %x) {
+; CHECK-LABEL: vaddva_v8i16_i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vaddva.u16 r0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %t = call i16 @llvm.experimental.vector.reduce.add.i16.v8i16(<8 x i16> %s1)
+  %r = add i16 %t, %x
+  ret i16 %r
+}
+
+define arm_aapcs_vfpcc i8 @vaddva_v16i8_i8(<16 x i8> %s1, i8 %x) {
+; CHECK-LABEL: vaddva_v16i8_i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vaddva.u8 r0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %t = call i8 @llvm.experimental.vector.reduce.add.i8.v16i8(<16 x i8> %s1)
+  %r = add i8 %t, %x
+  ret i8 %r
+}
diff --git a/llvm/test/CodeGen/Thumb2/mve-vcmp.ll b/llvm/test/CodeGen/Thumb2/mve-vcmp.ll
new file mode 100644
index 000000000..87f0e66fe
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-vcmp.ll
@@ -0,0 +1,537 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s
+
+define arm_aapcs_vfpcc <4 x i32> @vcmp_eq_v4i32(<4 x i32> %src, <4 x i32> %srcb, <4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: vcmp_eq_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i32 eq, q0, q1
+; CHECK-NEXT:    vpsel q0, q2, q3
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <4 x i32> %src, %srcb
+  %s = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @vcmp_ne_v4i32(<4 x i32> %src, <4 x i32> %srcb, <4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: vcmp_ne_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i32 ne, q0, q1
+; CHECK-NEXT:    vpsel q0, q2, q3
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp ne <4 x i32> %src, %srcb
+  %s = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @vcmp_sgt_v4i32(<4 x i32> %src, <4 x i32> %srcb, <4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: vcmp_sgt_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.s32 gt, q0, q1
+; CHECK-NEXT:    vpsel q0, q2, q3
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp sgt <4 x i32> %src, %srcb
+  %s = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @vcmp_sge_v4i32(<4 x i32> %src, <4 x i32> %srcb, <4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: vcmp_sge_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.s32 ge, q0, q1
+; CHECK-NEXT:    vpsel q0, q2, q3
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp sge <4 x i32> %src, %srcb
+  %s = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @vcmp_slt_v4i32(<4 x i32> %src, <4 x i32> %srcb, <4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: vcmp_slt_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.s32 gt, q1, q0
+; CHECK-NEXT:    vpsel q0, q2, q3
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp slt <4 x i32> %src, %srcb
+  %s = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @vcmp_sle_v4i32(<4 x i32> %src, <4 x i32> %srcb, <4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: vcmp_sle_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.s32 ge, q1, q0
+; CHECK-NEXT:    vpsel q0, q2, q3
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp sle <4 x i32> %src, %srcb
+  %s = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @vcmp_ugt_v4i32(<4 x i32> %src, <4 x i32> %srcb, <4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: vcmp_ugt_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.u32 hi, q0, q1
+; CHECK-NEXT:    vpsel q0, q2, q3
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp ugt <4 x i32> %src, %srcb
+  %s = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @vcmp_uge_v4i32(<4 x i32> %src, <4 x i32> %srcb, <4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: vcmp_uge_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.u32 cs, q0, q1
+; CHECK-NEXT:    vpsel q0, q2, q3
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp uge <4 x i32> %src, %srcb
+  %s = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @vcmp_ult_v4i32(<4 x i32> %src, <4 x i32> %srcb, <4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: vcmp_ult_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.u32 hi, q1, q0
+; CHECK-NEXT:    vpsel q0, q2, q3
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp ult <4 x i32> %src, %srcb
+  %s = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @vcmp_ule_v4i32(<4 x i32> %src, <4 x i32> %srcb, <4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: vcmp_ule_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.u32 cs, q1, q0
+; CHECK-NEXT:    vpsel q0, q2, q3
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp ule <4 x i32> %src, %srcb
+  %s = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+
+define arm_aapcs_vfpcc <8 x i16> @vcmp_eq_v8i16(<8 x i16> %src, <8 x i16> %srcb, <8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: vcmp_eq_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i16 eq, q0, q1
+; CHECK-NEXT:    vpsel q0, q2, q3
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <8 x i16> %src, %srcb
+  %s = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+define arm_aapcs_vfpcc <8 x i16> @vcmp_ne_v8i16(<8 x i16> %src, <8 x i16> %srcb, <8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: vcmp_ne_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i16 ne, q0, q1
+; CHECK-NEXT:    vpsel q0, q2, q3
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp ne <8 x i16> %src, %srcb
+  %s = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+define arm_aapcs_vfpcc <8 x i16> @vcmp_sgt_v8i16(<8 x i16> %src, <8 x i16> %srcb, <8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: vcmp_sgt_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.s16 gt, q0, q1
+; CHECK-NEXT:    vpsel q0, q2, q3
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp sgt <8 x i16> %src, %srcb
+  %s = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+define arm_aapcs_vfpcc <8 x i16> @vcmp_sge_v8i16(<8 x i16> %src, <8 x i16> %srcb, <8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: vcmp_sge_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.s16 ge, q0, q1
+; CHECK-NEXT:    vpsel q0, q2, q3
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp sge <8 x i16> %src, %srcb
+  %s = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+define arm_aapcs_vfpcc <8 x i16> @vcmp_slt_v8i16(<8 x i16> %src, <8 x i16> %srcb, <8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: vcmp_slt_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.s16 gt, q1, q0
+; CHECK-NEXT:    vpsel q0, q2, q3
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp slt <8 x i16> %src, %srcb
+  %s = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+define arm_aapcs_vfpcc <8 x i16> @vcmp_sle_v8i16(<8 x i16> %src, <8 x i16> %srcb, <8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: vcmp_sle_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.s16 ge, q1, q0
+; CHECK-NEXT:    vpsel q0, q2, q3
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp sle <8 x i16> %src, %srcb
+  %s = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+define arm_aapcs_vfpcc <8 x i16> @vcmp_ugt_v8i16(<8 x i16> %src, <8 x i16> %srcb, <8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: vcmp_ugt_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.u16 hi, q0, q1
+; CHECK-NEXT:    vpsel q0, q2, q3
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp ugt <8 x i16> %src, %srcb
+  %s = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+define arm_aapcs_vfpcc <8 x i16> @vcmp_uge_v8i16(<8 x i16> %src, <8 x i16> %srcb, <8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: vcmp_uge_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.u16 cs, q0, q1
+; CHECK-NEXT:    vpsel q0, q2, q3
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp uge <8 x i16> %src, %srcb
+  %s = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+define arm_aapcs_vfpcc <8 x i16> @vcmp_ult_v8i16(<8 x i16> %src, <8 x i16> %srcb, <8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: vcmp_ult_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.u16 hi, q1, q0
+; CHECK-NEXT:    vpsel q0, q2, q3
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp ult <8 x i16> %src, %srcb
+  %s = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+define arm_aapcs_vfpcc <8 x i16> @vcmp_ule_v8i16(<8 x i16> %src, <8 x i16> %srcb, <8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: vcmp_ule_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.u16 cs, q1, q0
+; CHECK-NEXT:    vpsel q0, q2, q3
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp ule <8 x i16> %src, %srcb
+  %s = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+
+define arm_aapcs_vfpcc <16 x i8> @vcmp_eq_v16i8(<16 x i8> %src, <16 x i8> %srcb, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: vcmp_eq_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i8 eq, q0, q1
+; CHECK-NEXT:    vpsel q0, q2, q3
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <16 x i8> %src, %srcb
+  %s = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %s
+}
+
+define arm_aapcs_vfpcc <16 x i8> @vcmp_ne_v16i8(<16 x i8> %src, <16 x i8> %srcb, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: vcmp_ne_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i8 ne, q0, q1
+; CHECK-NEXT:    vpsel q0, q2, q3
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp ne <16 x i8> %src, %srcb
+  %s = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %s
+}
+
+define arm_aapcs_vfpcc <16 x i8> @vcmp_sgt_v16i8(<16 x i8> %src, <16 x i8> %srcb, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: vcmp_sgt_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.s8 gt, q0, q1
+; CHECK-NEXT:    vpsel q0, q2, q3
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp sgt <16 x i8> %src, %srcb
+  %s = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %s
+}
+
+define arm_aapcs_vfpcc <16 x i8> @vcmp_sge_v16i8(<16 x i8> %src, <16 x i8> %srcb, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: vcmp_sge_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.s8 ge, q0, q1
+; CHECK-NEXT:    vpsel q0, q2, q3
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp sge <16 x i8> %src, %srcb
+  %s = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %s
+}
+
+define arm_aapcs_vfpcc <16 x i8> @vcmp_slt_v16i8(<16 x i8> %src, <16 x i8> %srcb, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: vcmp_slt_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.s8 gt, q1, q0
+; CHECK-NEXT:    vpsel q0, q2, q3
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp slt <16 x i8> %src, %srcb
+  %s = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %s
+}
+
+define arm_aapcs_vfpcc <16 x i8> @vcmp_sle_v16i8(<16 x i8> %src, <16 x i8> %srcb, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: vcmp_sle_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.s8 ge, q1, q0
+; CHECK-NEXT:    vpsel q0, q2, q3
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp sle <16 x i8> %src, %srcb
+  %s = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %s
+}
+
+define arm_aapcs_vfpcc <16 x i8> @vcmp_ugt_v16i8(<16 x i8> %src, <16 x i8> %srcb, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: vcmp_ugt_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.u8 hi, q0, q1
+; CHECK-NEXT:    vpsel q0, q2, q3
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp ugt <16 x i8> %src, %srcb
+  %s = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %s
+}
+
+define arm_aapcs_vfpcc <16 x i8> @vcmp_uge_v16i8(<16 x i8> %src, <16 x i8> %srcb, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: vcmp_uge_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.u8 cs, q0, q1
+; CHECK-NEXT:    vpsel q0, q2, q3
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp uge <16 x i8> %src, %srcb
+  %s = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %s
+}
+
+define arm_aapcs_vfpcc <16 x i8> @vcmp_ult_v16i8(<16 x i8> %src, <16 x i8> %srcb, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: vcmp_ult_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.u8 hi, q1, q0
+; CHECK-NEXT:    vpsel q0, q2, q3
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp ult <16 x i8> %src, %srcb
+  %s = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %s
+}
+
+define arm_aapcs_vfpcc <16 x i8> @vcmp_ule_v16i8(<16 x i8> %src, <16 x i8> %srcb, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: vcmp_ule_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.u8 cs, q1, q0
+; CHECK-NEXT:    vpsel q0, q2, q3
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp ule <16 x i8> %src, %srcb
+  %s = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %s
+}
+
+
+define arm_aapcs_vfpcc <2 x i64> @vcmp_eq_v2i64(<2 x i64> %src, <2 x i64> %srcb, <2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: vcmp_eq_v2i64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9}
+; CHECK-NEXT:    vpush {d8, d9}
+; CHECK-NEXT:    vmov r0, s5
+; CHECK-NEXT:    vmov r1, s1
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    eors r0, r1
+; CHECK-NEXT:    vmov r1, s4
+; CHECK-NEXT:    eors r1, r2
+; CHECK-NEXT:    vmov r2, s2
+; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    vmov r1, s3
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q4[0], r0
+; CHECK-NEXT:    vmov.32 q4[1], r0
+; CHECK-NEXT:    vmov r0, s7
+; CHECK-NEXT:    eors r0, r1
+; CHECK-NEXT:    vmov r1, s6
+; CHECK-NEXT:    eors r1, r2
+; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q4[2], r0
+; CHECK-NEXT:    vmov.32 q4[3], r0
+; CHECK-NEXT:    vbic q0, q3, q4
+; CHECK-NEXT:    vand q1, q2, q4
+; CHECK-NEXT:    vorr q0, q1, q0
+; CHECK-NEXT:    vpop {d8, d9}
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <2 x i64> %src, %srcb
+  %s = select <2 x i1> %c, <2 x i64> %a, <2 x i64> %b
+  ret <2 x i64> %s
+}
+
+define arm_aapcs_vfpcc <2 x i32> @vcmp_eq_v2i32(<2 x i64> %src, <2 x i64> %srcb, <2 x i32> %a, <2 x i32> %b) {
+; CHECK-LABEL: vcmp_eq_v2i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9}
+; CHECK-NEXT:    vpush {d8, d9}
+; CHECK-NEXT:    vmov r0, s5
+; CHECK-NEXT:    vmov r1, s1
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    eors r0, r1
+; CHECK-NEXT:    vmov r1, s4
+; CHECK-NEXT:    eors r1, r2
+; CHECK-NEXT:    vmov r2, s2
+; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    vmov r1, s3
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q4[0], r0
+; CHECK-NEXT:    vmov.32 q4[1], r0
+; CHECK-NEXT:    vmov r0, s7
+; CHECK-NEXT:    eors r0, r1
+; CHECK-NEXT:    vmov r1, s6
+; CHECK-NEXT:    eors r1, r2
+; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q4[2], r0
+; CHECK-NEXT:    vmov.32 q4[3], r0
+; CHECK-NEXT:    vbic q0, q3, q4
+; CHECK-NEXT:    vand q1, q2, q4
+; CHECK-NEXT:    vorr q0, q1, q0
+; CHECK-NEXT:    vpop {d8, d9}
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <2 x i64> %src, %srcb
+  %s = select <2 x i1> %c, <2 x i32> %a, <2 x i32> %b
+  ret <2 x i32> %s
+}
+
+define arm_aapcs_vfpcc <2 x i32> @vcmp_multi_v2i32(<2 x i64> %a, <2 x i32> %b, <2 x i32> %c) {
+; CHECK-LABEL: vcmp_multi_v2i32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11}
+; CHECK-NEXT:    vmov r0, s1
+; CHECK-NEXT:    movs r3, #0
+; CHECK-NEXT:    vmov r1, s0
+; CHECK-NEXT:    vmov r2, s8
+; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    vmov r1, s2
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q3[0], r0
+; CHECK-NEXT:    vmov.32 q3[1], r0
+; CHECK-NEXT:    vmov r0, s3
+; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q3[2], r0
+; CHECK-NEXT:    vmov.32 q3[3], r0
+; CHECK-NEXT:    vbic q0, q2, q3
+; CHECK-NEXT:    vmov lr, s0
+; CHECK-NEXT:    subs.w r1, lr, r2
+; CHECK-NEXT:    asr.w r12, lr, #31
+; CHECK-NEXT:    sbcs.w r1, r12, r2, asr #31
+; CHECK-NEXT:    mov.w r1, #0
+; CHECK-NEXT:    vmov r2, s10
+; CHECK-NEXT:    it lt
+; CHECK-NEXT:    movlt r1, #1
+; CHECK-NEXT:    cmp r1, #0
+; CHECK-NEXT:    csetm r1, ne
+; CHECK-NEXT:    vmov.32 q3[0], r1
+; CHECK-NEXT:    vmov.32 q3[1], r1
+; CHECK-NEXT:    vmov r1, s2
+; CHECK-NEXT:    subs r0, r1, r2
+; CHECK-NEXT:    asr.w r12, r1, #31
+; CHECK-NEXT:    sbcs.w r0, r12, r2, asr #31
+; CHECK-NEXT:    it lt
+; CHECK-NEXT:    movlt r3, #1
+; CHECK-NEXT:    cmp r3, #0
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    cmp.w lr, #0
+; CHECK-NEXT:    vmov.32 q3[2], r0
+; CHECK-NEXT:    vmov.32 q3[3], r0
+; CHECK-NEXT:    cset r0, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    cmp r1, #0
+; CHECK-NEXT:    vmov.32 q4[0], r0
+; CHECK-NEXT:    vmov.32 q4[1], r0
+; CHECK-NEXT:    cset r0, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q4[2], r0
+; CHECK-NEXT:    vmov.32 q4[3], r0
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    cset r0, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q5[0], r0
+; CHECK-NEXT:    vmov.32 q5[1], r0
+; CHECK-NEXT:    vmov r0, s6
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    cset r0, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q5[2], r0
+; CHECK-NEXT:    vmov.32 q5[3], r0
+; CHECK-NEXT:    vand q1, q5, q4
+; CHECK-NEXT:    vand q1, q3, q1
+; CHECK-NEXT:    vbic q0, q0, q1
+; CHECK-NEXT:    vand q1, q2, q1
+; CHECK-NEXT:    vorr q0, q1, q0
+; CHECK-NEXT:    vpop {d8, d9, d10, d11}
+; CHECK-NEXT:    pop {r7, pc}
+  %a4 = icmp eq <2 x i64> %a, zeroinitializer
+  %a5 = select <2 x i1> %a4, <2 x i32> zeroinitializer, <2 x i32> %c
+  %a6 = icmp ne <2 x i32> %b, zeroinitializer
+  %a7 = icmp slt <2 x i32> %a5, %c
+  %a8 = icmp ne <2 x i32> %a5, zeroinitializer
+  %a9 = and <2 x i1> %a6, %a8
+  %a10 = and <2 x i1> %a7, %a9
+  %a11 = select <2 x i1> %a10, <2 x i32> %c, <2 x i32> %a5
+  ret <2 x i32> %a11
+}
diff --git a/llvm/test/CodeGen/Thumb2/mve-vcmpf.ll b/llvm/test/CodeGen/Thumb2/mve-vcmpf.ll
new file mode 100644
index 000000000..9e793caac
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-vcmpf.ll
@@ -0,0 +1,2541 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve,+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-MVE
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve.fp -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-MVEFP
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_oeq_v4f32(<4 x float> %src, <4 x float> %src2, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_oeq_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s5
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s2, s6
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s3, s7
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_oeq_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vcmp.f32 eq, q0, q1
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp oeq <4 x float> %src, %src2
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_one_v4f32(<4 x float> %src, <4 x float> %src2, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_one_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r1, #1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s5
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r2, #1
+; CHECK-MVE-NEXT:    vcmp.f32 s2, s6
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r3, #1
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s3, s7
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r0, #1
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_one_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vpt.f32 le, q1, q0
+; CHECK-MVEFP-NEXT:    vcmpt.f32 le, q0, q1
+; CHECK-MVEFP-NEXT:    vpnot
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp one <4 x float> %src, %src2
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_ogt_v4f32(<4 x float> %src, <4 x float> %src2, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_ogt_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s5
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s2, s6
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s3, s7
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_ogt_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vcmp.f32 gt, q0, q1
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp ogt <4 x float> %src, %src2
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_oge_v4f32(<4 x float> %src, <4 x float> %src2, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_oge_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ge
+; CHECK-MVE-NEXT:    movge r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s5
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s2, s6
+; CHECK-MVE-NEXT:    it ge
+; CHECK-MVE-NEXT:    movge r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s3, s7
+; CHECK-MVE-NEXT:    it ge
+; CHECK-MVE-NEXT:    movge r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ge
+; CHECK-MVE-NEXT:    movge r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_oge_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vcmp.f32 ge, q0, q1
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp oge <4 x float> %src, %src2
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_olt_v4f32(<4 x float> %src, <4 x float> %src2, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_olt_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s5
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s2, s6
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s3, s7
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_olt_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vcmp.f32 gt, q1, q0
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp olt <4 x float> %src, %src2
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_ole_v4f32(<4 x float> %src, <4 x float> %src2, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_ole_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ls
+; CHECK-MVE-NEXT:    movls r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s5
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s2, s6
+; CHECK-MVE-NEXT:    it ls
+; CHECK-MVE-NEXT:    movls r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s3, s7
+; CHECK-MVE-NEXT:    it ls
+; CHECK-MVE-NEXT:    movls r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ls
+; CHECK-MVE-NEXT:    movls r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_ole_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vcmp.f32 ge, q1, q0
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp ole <4 x float> %src, %src2
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_ueq_v4f32(<4 x float> %src, <4 x float> %src2, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_ueq_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r1, #1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s5
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r2, #1
+; CHECK-MVE-NEXT:    vcmp.f32 s2, s6
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r3, #1
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s3, s7
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r0, #1
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_ueq_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vpt.f32 le, q1, q0
+; CHECK-MVEFP-NEXT:    vcmpt.f32 le, q0, q1
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp ueq <4 x float> %src, %src2
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_une_v4f32(<4 x float> %src, <4 x float> %src2, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_une_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ne
+; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s5
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s2, s6
+; CHECK-MVE-NEXT:    it ne
+; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s3, s7
+; CHECK-MVE-NEXT:    it ne
+; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ne
+; CHECK-MVE-NEXT:    movne r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_une_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vcmp.f32 ne, q0, q1
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp une <4 x float> %src, %src2
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_ugt_v4f32(<4 x float> %src, <4 x float> %src2, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_ugt_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it hi
+; CHECK-MVE-NEXT:    movhi r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s5
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s2, s6
+; CHECK-MVE-NEXT:    it hi
+; CHECK-MVE-NEXT:    movhi r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s3, s7
+; CHECK-MVE-NEXT:    it hi
+; CHECK-MVE-NEXT:    movhi r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it hi
+; CHECK-MVE-NEXT:    movhi r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_ugt_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vcmp.f32 ge, q1, q0
+; CHECK-MVEFP-NEXT:    vpnot
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp ugt <4 x float> %src, %src2
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_uge_v4f32(<4 x float> %src, <4 x float> %src2, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_uge_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it pl
+; CHECK-MVE-NEXT:    movpl r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s5
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s2, s6
+; CHECK-MVE-NEXT:    it pl
+; CHECK-MVE-NEXT:    movpl r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s3, s7
+; CHECK-MVE-NEXT:    it pl
+; CHECK-MVE-NEXT:    movpl r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it pl
+; CHECK-MVE-NEXT:    movpl r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_uge_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vcmp.f32 gt, q1, q0
+; CHECK-MVEFP-NEXT:    vpnot
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp uge <4 x float> %src, %src2
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_ult_v4f32(<4 x float> %src, <4 x float> %src2, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_ult_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it lt
+; CHECK-MVE-NEXT:    movlt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s5
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s2, s6
+; CHECK-MVE-NEXT:    it lt
+; CHECK-MVE-NEXT:    movlt r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s3, s7
+; CHECK-MVE-NEXT:    it lt
+; CHECK-MVE-NEXT:    movlt r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it lt
+; CHECK-MVE-NEXT:    movlt r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_ult_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vcmp.f32 ge, q0, q1
+; CHECK-MVEFP-NEXT:    vpnot
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp ult <4 x float> %src, %src2
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_ule_v4f32(<4 x float> %src, <4 x float> %src2, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_ule_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it le
+; CHECK-MVE-NEXT:    movle r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s5
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s2, s6
+; CHECK-MVE-NEXT:    it le
+; CHECK-MVE-NEXT:    movle r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s3, s7
+; CHECK-MVE-NEXT:    it le
+; CHECK-MVE-NEXT:    movle r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it le
+; CHECK-MVE-NEXT:    movle r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_ule_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vcmp.f32 gt, q0, q1
+; CHECK-MVEFP-NEXT:    vpnot
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp ule <4 x float> %src, %src2
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_ord_v4f32(<4 x float> %src, <4 x float> %src2, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_ord_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it vc
+; CHECK-MVE-NEXT:    movvc r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s5
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s2, s6
+; CHECK-MVE-NEXT:    it vc
+; CHECK-MVE-NEXT:    movvc r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s3, s7
+; CHECK-MVE-NEXT:    it vc
+; CHECK-MVE-NEXT:    movvc r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it vc
+; CHECK-MVE-NEXT:    movvc r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_ord_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vpt.f32 le, q1, q0
+; CHECK-MVEFP-NEXT:    vcmpt.f32 lt, q0, q1
+; CHECK-MVEFP-NEXT:    vpnot
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp ord <4 x float> %src, %src2
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_uno_v4f32(<4 x float> %src, <4 x float> %src2, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_uno_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s5
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s2, s6
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s3, s7
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_uno_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vpt.f32 le, q1, q0
+; CHECK-MVEFP-NEXT:    vcmpt.f32 lt, q0, q1
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp uno <4 x float> %src, %src2
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_oeq_v8f16(<8 x half> %src, <8 x half> %src2, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_oeq_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9, d10, d11}
+; CHECK-MVE-NEXT:    vpush {d8, d9, d10, d11}
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s4
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s0
+; CHECK-MVE-NEXT:    vcmp.f16 s18, s16
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s0, s4
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s8
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s12
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vcmp.f16 s1, s5
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s12, s8
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
+; CHECK-MVE-NEXT:    vmov.16 q4[0], r2
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q4[1], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s13, s9
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s5
+; CHECK-MVE-NEXT:    vcmp.f16 s22, s20
+; CHECK-MVE-NEXT:    vmov.16 q4[2], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s13
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
+; CHECK-MVE-NEXT:    vcmp.f16 s2, s6
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov.16 q4[3], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s2
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s14, s10
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s6
+; CHECK-MVE-NEXT:    vcmp.f16 s22, s20
+; CHECK-MVE-NEXT:    vmov.16 q4[4], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s14
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
+; CHECK-MVE-NEXT:    vcmp.f16 s3, s7
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov.16 q4[5], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vcmp.f16 s0, s4
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s15, s11
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmov.16 q4[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q4[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q4
+; CHECK-MVE-NEXT:    vpop {d8, d9, d10, d11}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_oeq_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vcmp.f16 eq, q0, q1
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp oeq <8 x half> %src, %src2
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_one_v8f16(<8 x half> %src, <8 x half> %src2, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_one_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9, d10, d11}
+; CHECK-MVE-NEXT:    vpush {d8, d9, d10, d11}
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s4
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s0
+; CHECK-MVE-NEXT:    vcmp.f16 s18, s16
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r1, #1
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vcmp.f16 s0, s4
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s8
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s12
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r2, #1
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vcmp.f16 s1, s5
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s12, s8
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
+; CHECK-MVE-NEXT:    vmov.16 q4[0], r2
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q4[1], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r1, #1
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s13, s9
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s5
+; CHECK-MVE-NEXT:    vcmp.f16 s22, s20
+; CHECK-MVE-NEXT:    vmov.16 q4[2], r1
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r1, #1
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s13
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s2, s6
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s2
+; CHECK-MVE-NEXT:    vmov.16 q4[3], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r1, #1
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s14, s10
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s6
+; CHECK-MVE-NEXT:    vcmp.f16 s22, s20
+; CHECK-MVE-NEXT:    vmov.16 q4[4], r1
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r1, #1
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s14
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s3, s7
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    vcmp.f16 s0, s4
+; CHECK-MVE-NEXT:    vmov.16 q4[5], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r1, #1
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s11
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s15, s11
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r0, #1
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmov.16 q4[6], r1
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q4[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q4
+; CHECK-MVE-NEXT:    vpop {d8, d9, d10, d11}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_one_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vpt.f16 le, q1, q0
+; CHECK-MVEFP-NEXT:    vcmpt.f16 le, q0, q1
+; CHECK-MVEFP-NEXT:    vpnot
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp one <8 x half> %src, %src2
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_ogt_v8f16(<8 x half> %src, <8 x half> %src2, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_ogt_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9, d10, d11}
+; CHECK-MVE-NEXT:    vpush {d8, d9, d10, d11}
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s4
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s0
+; CHECK-MVE-NEXT:    vcmp.f16 s18, s16
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s0, s4
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s8
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s12
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vcmp.f16 s1, s5
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s12, s8
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
+; CHECK-MVE-NEXT:    vmov.16 q4[0], r2
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q4[1], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s13, s9
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s5
+; CHECK-MVE-NEXT:    vcmp.f16 s22, s20
+; CHECK-MVE-NEXT:    vmov.16 q4[2], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s13
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
+; CHECK-MVE-NEXT:    vcmp.f16 s2, s6
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov.16 q4[3], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s2
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s14, s10
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s6
+; CHECK-MVE-NEXT:    vcmp.f16 s22, s20
+; CHECK-MVE-NEXT:    vmov.16 q4[4], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s14
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
+; CHECK-MVE-NEXT:    vcmp.f16 s3, s7
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov.16 q4[5], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vcmp.f16 s0, s4
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s15, s11
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmov.16 q4[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q4[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q4
+; CHECK-MVE-NEXT:    vpop {d8, d9, d10, d11}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_ogt_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vcmp.f16 gt, q0, q1
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp ogt <8 x half> %src, %src2
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_oge_v8f16(<8 x half> %src, <8 x half> %src2, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_oge_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9, d10, d11}
+; CHECK-MVE-NEXT:    vpush {d8, d9, d10, d11}
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s4
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s0
+; CHECK-MVE-NEXT:    vcmp.f16 s18, s16
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ge
+; CHECK-MVE-NEXT:    movge r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s0, s4
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s8
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s12
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    it ge
+; CHECK-MVE-NEXT:    movge r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vcmp.f16 s1, s5
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s12, s8
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
+; CHECK-MVE-NEXT:    vmov.16 q4[0], r2
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q4[1], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it ge
+; CHECK-MVE-NEXT:    movge r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s13, s9
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s5
+; CHECK-MVE-NEXT:    vcmp.f16 s22, s20
+; CHECK-MVE-NEXT:    vmov.16 q4[2], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it ge
+; CHECK-MVE-NEXT:    movge r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s13
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
+; CHECK-MVE-NEXT:    vcmp.f16 s2, s6
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov.16 q4[3], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it ge
+; CHECK-MVE-NEXT:    movge r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s2
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s14, s10
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s6
+; CHECK-MVE-NEXT:    vcmp.f16 s22, s20
+; CHECK-MVE-NEXT:    vmov.16 q4[4], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it ge
+; CHECK-MVE-NEXT:    movge r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s14
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
+; CHECK-MVE-NEXT:    vcmp.f16 s3, s7
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov.16 q4[5], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it ge
+; CHECK-MVE-NEXT:    movge r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vcmp.f16 s0, s4
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s15, s11
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ge
+; CHECK-MVE-NEXT:    movge r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmov.16 q4[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q4[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q4
+; CHECK-MVE-NEXT:    vpop {d8, d9, d10, d11}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_oge_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vcmp.f16 ge, q0, q1
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp oge <8 x half> %src, %src2
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_olt_v8f16(<8 x half> %src, <8 x half> %src2, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_olt_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9, d10, d11}
+; CHECK-MVE-NEXT:    vpush {d8, d9, d10, d11}
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s4
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s0
+; CHECK-MVE-NEXT:    vcmp.f16 s18, s16
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s0, s4
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s8
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s12
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vcmp.f16 s1, s5
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s12, s8
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
+; CHECK-MVE-NEXT:    vmov.16 q4[0], r2
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q4[1], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s13, s9
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s5
+; CHECK-MVE-NEXT:    vcmp.f16 s22, s20
+; CHECK-MVE-NEXT:    vmov.16 q4[2], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s13
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
+; CHECK-MVE-NEXT:    vcmp.f16 s2, s6
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov.16 q4[3], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s2
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s14, s10
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s6
+; CHECK-MVE-NEXT:    vcmp.f16 s22, s20
+; CHECK-MVE-NEXT:    vmov.16 q4[4], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s14
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
+; CHECK-MVE-NEXT:    vcmp.f16 s3, s7
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov.16 q4[5], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vcmp.f16 s0, s4
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s15, s11
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmov.16 q4[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q4[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q4
+; CHECK-MVE-NEXT:    vpop {d8, d9, d10, d11}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_olt_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vcmp.f16 gt, q1, q0
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp olt <8 x half> %src, %src2
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_ole_v8f16(<8 x half> %src, <8 x half> %src2, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_ole_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9, d10, d11}
+; CHECK-MVE-NEXT:    vpush {d8, d9, d10, d11}
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s4
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s0
+; CHECK-MVE-NEXT:    vcmp.f16 s18, s16
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ls
+; CHECK-MVE-NEXT:    movls r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s0, s4
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s8
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s12
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    it ls
+; CHECK-MVE-NEXT:    movls r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vcmp.f16 s1, s5
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s12, s8
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
+; CHECK-MVE-NEXT:    vmov.16 q4[0], r2
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q4[1], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it ls
+; CHECK-MVE-NEXT:    movls r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s13, s9
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s5
+; CHECK-MVE-NEXT:    vcmp.f16 s22, s20
+; CHECK-MVE-NEXT:    vmov.16 q4[2], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it ls
+; CHECK-MVE-NEXT:    movls r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s13
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
+; CHECK-MVE-NEXT:    vcmp.f16 s2, s6
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov.16 q4[3], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it ls
+; CHECK-MVE-NEXT:    movls r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s2
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s14, s10
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s6
+; CHECK-MVE-NEXT:    vcmp.f16 s22, s20
+; CHECK-MVE-NEXT:    vmov.16 q4[4], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it ls
+; CHECK-MVE-NEXT:    movls r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s14
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
+; CHECK-MVE-NEXT:    vcmp.f16 s3, s7
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov.16 q4[5], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it ls
+; CHECK-MVE-NEXT:    movls r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vcmp.f16 s0, s4
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s15, s11
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ls
+; CHECK-MVE-NEXT:    movls r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmov.16 q4[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q4[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q4
+; CHECK-MVE-NEXT:    vpop {d8, d9, d10, d11}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_ole_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vcmp.f16 ge, q1, q0
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp ole <8 x half> %src, %src2
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_ueq_v8f16(<8 x half> %src, <8 x half> %src2, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_ueq_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9, d10, d11}
+; CHECK-MVE-NEXT:    vpush {d8, d9, d10, d11}
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s4
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s0
+; CHECK-MVE-NEXT:    vcmp.f16 s18, s16
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r1, #1
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vcmp.f16 s0, s4
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s8
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s12
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r2, #1
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vcmp.f16 s1, s5
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s12, s8
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
+; CHECK-MVE-NEXT:    vmov.16 q4[0], r2
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q4[1], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r1, #1
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s13, s9
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s5
+; CHECK-MVE-NEXT:    vcmp.f16 s22, s20
+; CHECK-MVE-NEXT:    vmov.16 q4[2], r1
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r1, #1
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s13
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s2, s6
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s2
+; CHECK-MVE-NEXT:    vmov.16 q4[3], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r1, #1
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s14, s10
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s6
+; CHECK-MVE-NEXT:    vcmp.f16 s22, s20
+; CHECK-MVE-NEXT:    vmov.16 q4[4], r1
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r1, #1
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s14
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s3, s7
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    vcmp.f16 s0, s4
+; CHECK-MVE-NEXT:    vmov.16 q4[5], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r1, #1
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s11
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s15, s11
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r0, #1
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmov.16 q4[6], r1
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q4[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q4
+; CHECK-MVE-NEXT:    vpop {d8, d9, d10, d11}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_ueq_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vpt.f16 le, q1, q0
+; CHECK-MVEFP-NEXT:    vcmpt.f16 le, q0, q1
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp ueq <8 x half> %src, %src2
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_une_v8f16(<8 x half> %src, <8 x half> %src2, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_une_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9, d10, d11}
+; CHECK-MVE-NEXT:    vpush {d8, d9, d10, d11}
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s4
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s0
+; CHECK-MVE-NEXT:    vcmp.f16 s18, s16
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ne
+; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s0, s4
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s8
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s12
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    it ne
+; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vcmp.f16 s1, s5
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s12, s8
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
+; CHECK-MVE-NEXT:    vmov.16 q4[0], r2
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q4[1], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it ne
+; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s13, s9
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s5
+; CHECK-MVE-NEXT:    vcmp.f16 s22, s20
+; CHECK-MVE-NEXT:    vmov.16 q4[2], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it ne
+; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s13
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
+; CHECK-MVE-NEXT:    vcmp.f16 s2, s6
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov.16 q4[3], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it ne
+; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s2
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s14, s10
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s6
+; CHECK-MVE-NEXT:    vcmp.f16 s22, s20
+; CHECK-MVE-NEXT:    vmov.16 q4[4], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it ne
+; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s14
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
+; CHECK-MVE-NEXT:    vcmp.f16 s3, s7
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov.16 q4[5], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it ne
+; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vcmp.f16 s0, s4
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s15, s11
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ne
+; CHECK-MVE-NEXT:    movne r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmov.16 q4[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q4[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q4
+; CHECK-MVE-NEXT:    vpop {d8, d9, d10, d11}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_une_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vcmp.f16 ne, q0, q1
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp une <8 x half> %src, %src2
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_ugt_v8f16(<8 x half> %src, <8 x half> %src2, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_ugt_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9, d10, d11}
+; CHECK-MVE-NEXT:    vpush {d8, d9, d10, d11}
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s4
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s0
+; CHECK-MVE-NEXT:    vcmp.f16 s18, s16
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it hi
+; CHECK-MVE-NEXT:    movhi r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s0, s4
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s8
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s12
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    it hi
+; CHECK-MVE-NEXT:    movhi r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vcmp.f16 s1, s5
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s12, s8
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
+; CHECK-MVE-NEXT:    vmov.16 q4[0], r2
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q4[1], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it hi
+; CHECK-MVE-NEXT:    movhi r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s13, s9
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s5
+; CHECK-MVE-NEXT:    vcmp.f16 s22, s20
+; CHECK-MVE-NEXT:    vmov.16 q4[2], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it hi
+; CHECK-MVE-NEXT:    movhi r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s13
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
+; CHECK-MVE-NEXT:    vcmp.f16 s2, s6
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov.16 q4[3], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it hi
+; CHECK-MVE-NEXT:    movhi r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s2
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s14, s10
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s6
+; CHECK-MVE-NEXT:    vcmp.f16 s22, s20
+; CHECK-MVE-NEXT:    vmov.16 q4[4], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it hi
+; CHECK-MVE-NEXT:    movhi r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s14
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
+; CHECK-MVE-NEXT:    vcmp.f16 s3, s7
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov.16 q4[5], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it hi
+; CHECK-MVE-NEXT:    movhi r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vcmp.f16 s0, s4
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s15, s11
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it hi
+; CHECK-MVE-NEXT:    movhi r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmov.16 q4[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q4[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q4
+; CHECK-MVE-NEXT:    vpop {d8, d9, d10, d11}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_ugt_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vcmp.f16 ge, q1, q0
+; CHECK-MVEFP-NEXT:    vpnot
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp ugt <8 x half> %src, %src2
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_uge_v8f16(<8 x half> %src, <8 x half> %src2, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_uge_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9, d10, d11}
+; CHECK-MVE-NEXT:    vpush {d8, d9, d10, d11}
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s4
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s0
+; CHECK-MVE-NEXT:    vcmp.f16 s18, s16
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it pl
+; CHECK-MVE-NEXT:    movpl r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s0, s4
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s8
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s12
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    it pl
+; CHECK-MVE-NEXT:    movpl r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vcmp.f16 s1, s5
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s12, s8
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
+; CHECK-MVE-NEXT:    vmov.16 q4[0], r2
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q4[1], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it pl
+; CHECK-MVE-NEXT:    movpl r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s13, s9
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s5
+; CHECK-MVE-NEXT:    vcmp.f16 s22, s20
+; CHECK-MVE-NEXT:    vmov.16 q4[2], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it pl
+; CHECK-MVE-NEXT:    movpl r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s13
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
+; CHECK-MVE-NEXT:    vcmp.f16 s2, s6
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov.16 q4[3], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it pl
+; CHECK-MVE-NEXT:    movpl r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s2
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s14, s10
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s6
+; CHECK-MVE-NEXT:    vcmp.f16 s22, s20
+; CHECK-MVE-NEXT:    vmov.16 q4[4], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it pl
+; CHECK-MVE-NEXT:    movpl r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s14
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
+; CHECK-MVE-NEXT:    vcmp.f16 s3, s7
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov.16 q4[5], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it pl
+; CHECK-MVE-NEXT:    movpl r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vcmp.f16 s0, s4
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s15, s11
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it pl
+; CHECK-MVE-NEXT:    movpl r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmov.16 q4[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q4[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q4
+; CHECK-MVE-NEXT:    vpop {d8, d9, d10, d11}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_uge_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vcmp.f16 gt, q1, q0
+; CHECK-MVEFP-NEXT:    vpnot
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp uge <8 x half> %src, %src2
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_ult_v8f16(<8 x half> %src, <8 x half> %src2, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_ult_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9, d10, d11}
+; CHECK-MVE-NEXT:    vpush {d8, d9, d10, d11}
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s4
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s0
+; CHECK-MVE-NEXT:    vcmp.f16 s18, s16
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it lt
+; CHECK-MVE-NEXT:    movlt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s0, s4
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s8
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s12
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    it lt
+; CHECK-MVE-NEXT:    movlt r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vcmp.f16 s1, s5
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s12, s8
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
+; CHECK-MVE-NEXT:    vmov.16 q4[0], r2
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q4[1], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it lt
+; CHECK-MVE-NEXT:    movlt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s13, s9
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s5
+; CHECK-MVE-NEXT:    vcmp.f16 s22, s20
+; CHECK-MVE-NEXT:    vmov.16 q4[2], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it lt
+; CHECK-MVE-NEXT:    movlt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s13
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
+; CHECK-MVE-NEXT:    vcmp.f16 s2, s6
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov.16 q4[3], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it lt
+; CHECK-MVE-NEXT:    movlt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s2
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s14, s10
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s6
+; CHECK-MVE-NEXT:    vcmp.f16 s22, s20
+; CHECK-MVE-NEXT:    vmov.16 q4[4], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it lt
+; CHECK-MVE-NEXT:    movlt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s14
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
+; CHECK-MVE-NEXT:    vcmp.f16 s3, s7
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov.16 q4[5], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it lt
+; CHECK-MVE-NEXT:    movlt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vcmp.f16 s0, s4
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s15, s11
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it lt
+; CHECK-MVE-NEXT:    movlt r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmov.16 q4[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q4[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q4
+; CHECK-MVE-NEXT:    vpop {d8, d9, d10, d11}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_ult_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vcmp.f16 ge, q0, q1
+; CHECK-MVEFP-NEXT:    vpnot
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp ult <8 x half> %src, %src2
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_ule_v8f16(<8 x half> %src, <8 x half> %src2, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_ule_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9, d10, d11}
+; CHECK-MVE-NEXT:    vpush {d8, d9, d10, d11}
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s4
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s0
+; CHECK-MVE-NEXT:    vcmp.f16 s18, s16
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it le
+; CHECK-MVE-NEXT:    movle r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s0, s4
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s8
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s12
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    it le
+; CHECK-MVE-NEXT:    movle r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vcmp.f16 s1, s5
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s12, s8
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
+; CHECK-MVE-NEXT:    vmov.16 q4[0], r2
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q4[1], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it le
+; CHECK-MVE-NEXT:    movle r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s13, s9
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s5
+; CHECK-MVE-NEXT:    vcmp.f16 s22, s20
+; CHECK-MVE-NEXT:    vmov.16 q4[2], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it le
+; CHECK-MVE-NEXT:    movle r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s13
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
+; CHECK-MVE-NEXT:    vcmp.f16 s2, s6
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov.16 q4[3], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it le
+; CHECK-MVE-NEXT:    movle r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s2
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s14, s10
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s6
+; CHECK-MVE-NEXT:    vcmp.f16 s22, s20
+; CHECK-MVE-NEXT:    vmov.16 q4[4], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it le
+; CHECK-MVE-NEXT:    movle r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s14
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
+; CHECK-MVE-NEXT:    vcmp.f16 s3, s7
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov.16 q4[5], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it le
+; CHECK-MVE-NEXT:    movle r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vcmp.f16 s0, s4
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s15, s11
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it le
+; CHECK-MVE-NEXT:    movle r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmov.16 q4[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q4[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q4
+; CHECK-MVE-NEXT:    vpop {d8, d9, d10, d11}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_ule_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vcmp.f16 gt, q0, q1
+; CHECK-MVEFP-NEXT:    vpnot
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp ule <8 x half> %src, %src2
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_ord_v8f16(<8 x half> %src, <8 x half> %src2, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_ord_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9, d10, d11}
+; CHECK-MVE-NEXT:    vpush {d8, d9, d10, d11}
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s4
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s0
+; CHECK-MVE-NEXT:    vcmp.f16 s18, s16
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it vc
+; CHECK-MVE-NEXT:    movvc r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s0, s4
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s8
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s12
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    it vc
+; CHECK-MVE-NEXT:    movvc r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vcmp.f16 s1, s5
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s12, s8
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
+; CHECK-MVE-NEXT:    vmov.16 q4[0], r2
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q4[1], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it vc
+; CHECK-MVE-NEXT:    movvc r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s13, s9
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s5
+; CHECK-MVE-NEXT:    vcmp.f16 s22, s20
+; CHECK-MVE-NEXT:    vmov.16 q4[2], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it vc
+; CHECK-MVE-NEXT:    movvc r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s13
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
+; CHECK-MVE-NEXT:    vcmp.f16 s2, s6
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov.16 q4[3], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it vc
+; CHECK-MVE-NEXT:    movvc r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s2
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s14, s10
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s6
+; CHECK-MVE-NEXT:    vcmp.f16 s22, s20
+; CHECK-MVE-NEXT:    vmov.16 q4[4], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it vc
+; CHECK-MVE-NEXT:    movvc r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s14
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
+; CHECK-MVE-NEXT:    vcmp.f16 s3, s7
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov.16 q4[5], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it vc
+; CHECK-MVE-NEXT:    movvc r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vcmp.f16 s0, s4
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s15, s11
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it vc
+; CHECK-MVE-NEXT:    movvc r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmov.16 q4[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q4[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q4
+; CHECK-MVE-NEXT:    vpop {d8, d9, d10, d11}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_ord_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vpt.f16 le, q1, q0
+; CHECK-MVEFP-NEXT:    vcmpt.f16 lt, q0, q1
+; CHECK-MVEFP-NEXT:    vpnot
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp ord <8 x half> %src, %src2
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_uno_v8f16(<8 x half> %src, <8 x half> %src2, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_uno_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9, d10, d11}
+; CHECK-MVE-NEXT:    vpush {d8, d9, d10, d11}
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s4
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s0
+; CHECK-MVE-NEXT:    vcmp.f16 s18, s16
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s0, s4
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s8
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s12
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vcmp.f16 s1, s5
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s12, s8
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
+; CHECK-MVE-NEXT:    vmov.16 q4[0], r2
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q4[1], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s13, s9
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s5
+; CHECK-MVE-NEXT:    vcmp.f16 s22, s20
+; CHECK-MVE-NEXT:    vmov.16 q4[2], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s13
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
+; CHECK-MVE-NEXT:    vcmp.f16 s2, s6
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov.16 q4[3], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s2
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s14, s10
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s6
+; CHECK-MVE-NEXT:    vcmp.f16 s22, s20
+; CHECK-MVE-NEXT:    vmov.16 q4[4], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s14
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
+; CHECK-MVE-NEXT:    vcmp.f16 s3, s7
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov.16 q4[5], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vcmp.f16 s0, s4
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s15, s11
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmov r1, s20
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmov.16 q4[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q4[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q4
+; CHECK-MVE-NEXT:    vpop {d8, d9, d10, d11}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_uno_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vpt.f16 le, q1, q0
+; CHECK-MVEFP-NEXT:    vcmpt.f16 lt, q0, q1
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp uno <8 x half> %src, %src2
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
diff --git a/llvm/test/CodeGen/Thumb2/mve-vcmpfr.ll b/llvm/test/CodeGen/Thumb2/mve-vcmpfr.ll
new file mode 100644
index 000000000..79700e046
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-vcmpfr.ll
@@ -0,0 +1,5222 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve,+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-MVE
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve.fp -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-MVEFP
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_oeq_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_oeq_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s4
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s2, s4
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s3, s4
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_oeq_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vmov r0, s4
+; CHECK-MVEFP-NEXT:    vcmp.f32 eq, q0, r0
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %i = insertelement <4 x float> undef, float %src2, i32 0
+  %sp = shufflevector <4 x float> %i, <4 x float> undef, <4 x i32> zeroinitializer
+  %c = fcmp oeq <4 x float> %src, %sp
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_one_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_one_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r1, #1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s4
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r2, #1
+; CHECK-MVE-NEXT:    vcmp.f32 s2, s4
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r3, #1
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s3, s4
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r0, #1
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_one_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vmov r0, s4
+; CHECK-MVEFP-NEXT:    vpt.f32 ge, q0, r0
+; CHECK-MVEFP-NEXT:    vcmpt.f32 le, q0, r0
+; CHECK-MVEFP-NEXT:    vpnot
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %i = insertelement <4 x float> undef, float %src2, i32 0
+  %sp = shufflevector <4 x float> %i, <4 x float> undef, <4 x i32> zeroinitializer
+  %c = fcmp one <4 x float> %src, %sp
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_ogt_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_ogt_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s4
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s2, s4
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s3, s4
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_ogt_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vmov r0, s4
+; CHECK-MVEFP-NEXT:    vcmp.f32 gt, q0, r0
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %i = insertelement <4 x float> undef, float %src2, i32 0
+  %sp = shufflevector <4 x float> %i, <4 x float> undef, <4 x i32> zeroinitializer
+  %c = fcmp ogt <4 x float> %src, %sp
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_oge_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_oge_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ge
+; CHECK-MVE-NEXT:    movge r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s4
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s2, s4
+; CHECK-MVE-NEXT:    it ge
+; CHECK-MVE-NEXT:    movge r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s3, s4
+; CHECK-MVE-NEXT:    it ge
+; CHECK-MVE-NEXT:    movge r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ge
+; CHECK-MVE-NEXT:    movge r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_oge_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vmov r0, s4
+; CHECK-MVEFP-NEXT:    vcmp.f32 ge, q0, r0
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %i = insertelement <4 x float> undef, float %src2, i32 0
+  %sp = shufflevector <4 x float> %i, <4 x float> undef, <4 x i32> zeroinitializer
+  %c = fcmp oge <4 x float> %src, %sp
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_olt_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_olt_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s4
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s2, s4
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s3, s4
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_olt_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vmov r0, s4
+; CHECK-MVEFP-NEXT:    vcmp.f32 lt, q0, r0
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %i = insertelement <4 x float> undef, float %src2, i32 0
+  %sp = shufflevector <4 x float> %i, <4 x float> undef, <4 x i32> zeroinitializer
+  %c = fcmp olt <4 x float> %src, %sp
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_ole_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_ole_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ls
+; CHECK-MVE-NEXT:    movls r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s4
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s2, s4
+; CHECK-MVE-NEXT:    it ls
+; CHECK-MVE-NEXT:    movls r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s3, s4
+; CHECK-MVE-NEXT:    it ls
+; CHECK-MVE-NEXT:    movls r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ls
+; CHECK-MVE-NEXT:    movls r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_ole_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vmov r0, s4
+; CHECK-MVEFP-NEXT:    vcmp.f32 le, q0, r0
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %i = insertelement <4 x float> undef, float %src2, i32 0
+  %sp = shufflevector <4 x float> %i, <4 x float> undef, <4 x i32> zeroinitializer
+  %c = fcmp ole <4 x float> %src, %sp
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_ueq_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_ueq_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r1, #1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s4
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r2, #1
+; CHECK-MVE-NEXT:    vcmp.f32 s2, s4
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r3, #1
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s3, s4
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r0, #1
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_ueq_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vmov r0, s4
+; CHECK-MVEFP-NEXT:    vpt.f32 ge, q0, r0
+; CHECK-MVEFP-NEXT:    vcmpt.f32 le, q0, r0
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %i = insertelement <4 x float> undef, float %src2, i32 0
+  %sp = shufflevector <4 x float> %i, <4 x float> undef, <4 x i32> zeroinitializer
+  %c = fcmp ueq <4 x float> %src, %sp
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_une_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_une_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ne
+; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s4
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s2, s4
+; CHECK-MVE-NEXT:    it ne
+; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s3, s4
+; CHECK-MVE-NEXT:    it ne
+; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ne
+; CHECK-MVE-NEXT:    movne r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_une_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vmov r0, s4
+; CHECK-MVEFP-NEXT:    vcmp.f32 ne, q0, r0
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %i = insertelement <4 x float> undef, float %src2, i32 0
+  %sp = shufflevector <4 x float> %i, <4 x float> undef, <4 x i32> zeroinitializer
+  %c = fcmp une <4 x float> %src, %sp
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_ugt_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_ugt_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it hi
+; CHECK-MVE-NEXT:    movhi r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s4
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s2, s4
+; CHECK-MVE-NEXT:    it hi
+; CHECK-MVE-NEXT:    movhi r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s3, s4
+; CHECK-MVE-NEXT:    it hi
+; CHECK-MVE-NEXT:    movhi r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it hi
+; CHECK-MVE-NEXT:    movhi r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_ugt_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vmov r0, s4
+; CHECK-MVEFP-NEXT:    vcmp.f32 le, q0, r0
+; CHECK-MVEFP-NEXT:    vpnot
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %i = insertelement <4 x float> undef, float %src2, i32 0
+  %sp = shufflevector <4 x float> %i, <4 x float> undef, <4 x i32> zeroinitializer
+  %c = fcmp ugt <4 x float> %src, %sp
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_uge_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_uge_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it pl
+; CHECK-MVE-NEXT:    movpl r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s4
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s2, s4
+; CHECK-MVE-NEXT:    it pl
+; CHECK-MVE-NEXT:    movpl r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s3, s4
+; CHECK-MVE-NEXT:    it pl
+; CHECK-MVE-NEXT:    movpl r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it pl
+; CHECK-MVE-NEXT:    movpl r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_uge_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vmov r0, s4
+; CHECK-MVEFP-NEXT:    vcmp.f32 lt, q0, r0
+; CHECK-MVEFP-NEXT:    vpnot
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %i = insertelement <4 x float> undef, float %src2, i32 0
+  %sp = shufflevector <4 x float> %i, <4 x float> undef, <4 x i32> zeroinitializer
+  %c = fcmp uge <4 x float> %src, %sp
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_ult_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_ult_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it lt
+; CHECK-MVE-NEXT:    movlt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s4
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s2, s4
+; CHECK-MVE-NEXT:    it lt
+; CHECK-MVE-NEXT:    movlt r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s3, s4
+; CHECK-MVE-NEXT:    it lt
+; CHECK-MVE-NEXT:    movlt r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it lt
+; CHECK-MVE-NEXT:    movlt r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_ult_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vmov r0, s4
+; CHECK-MVEFP-NEXT:    vcmp.f32 ge, q0, r0
+; CHECK-MVEFP-NEXT:    vpnot
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %i = insertelement <4 x float> undef, float %src2, i32 0
+  %sp = shufflevector <4 x float> %i, <4 x float> undef, <4 x i32> zeroinitializer
+  %c = fcmp ult <4 x float> %src, %sp
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_ule_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_ule_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it le
+; CHECK-MVE-NEXT:    movle r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s4
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s2, s4
+; CHECK-MVE-NEXT:    it le
+; CHECK-MVE-NEXT:    movle r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s3, s4
+; CHECK-MVE-NEXT:    it le
+; CHECK-MVE-NEXT:    movle r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it le
+; CHECK-MVE-NEXT:    movle r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_ule_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vmov r0, s4
+; CHECK-MVEFP-NEXT:    vcmp.f32 gt, q0, r0
+; CHECK-MVEFP-NEXT:    vpnot
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %i = insertelement <4 x float> undef, float %src2, i32 0
+  %sp = shufflevector <4 x float> %i, <4 x float> undef, <4 x i32> zeroinitializer
+  %c = fcmp ule <4 x float> %src, %sp
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_ord_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_ord_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it vc
+; CHECK-MVE-NEXT:    movvc r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s4
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s2, s4
+; CHECK-MVE-NEXT:    it vc
+; CHECK-MVE-NEXT:    movvc r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s3, s4
+; CHECK-MVE-NEXT:    it vc
+; CHECK-MVE-NEXT:    movvc r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it vc
+; CHECK-MVE-NEXT:    movvc r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_ord_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vmov r0, s4
+; CHECK-MVEFP-NEXT:    vpt.f32 ge, q0, r0
+; CHECK-MVEFP-NEXT:    vcmpt.f32 lt, q0, r0
+; CHECK-MVEFP-NEXT:    vpnot
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %i = insertelement <4 x float> undef, float %src2, i32 0
+  %sp = shufflevector <4 x float> %i, <4 x float> undef, <4 x i32> zeroinitializer
+  %c = fcmp ord <4 x float> %src, %sp
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_uno_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_uno_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s4
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s2, s4
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s3, s4
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_uno_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vmov r0, s4
+; CHECK-MVEFP-NEXT:    vpt.f32 ge, q0, r0
+; CHECK-MVEFP-NEXT:    vcmpt.f32 lt, q0, r0
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %i = insertelement <4 x float> undef, float %src2, i32 0
+  %sp = shufflevector <4 x float> %i, <4 x float> undef, <4 x i32> zeroinitializer
+  %c = fcmp uno <4 x float> %src, %sp
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_oeq_v8f16(<8 x half> %src, half* %src2p, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_oeq_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9, d10}
+; CHECK-MVE-NEXT:    vpush {d8, d9, d10}
+; CHECK-MVE-NEXT:    vldr.16 s16, [r0]
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s0
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
+; CHECK-MVE-NEXT:    vcmp.f16 s12, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s0, s16
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    movs r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmov r0, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s1, s16
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[1], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s9, s5
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s1
+; CHECK-MVE-NEXT:    vcmp.f16 s18, s16
+; CHECK-MVE-NEXT:    vmov.16 q3[2], r0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s5
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s2, s16
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
+; CHECK-MVE-NEXT:    vmov.16 q3[3], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s10, s6
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s2
+; CHECK-MVE-NEXT:    vcmp.f16 s18, s16
+; CHECK-MVE-NEXT:    vmov.16 q3[4], r0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s6
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s3, s16
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vcmp.f16 s0, s16
+; CHECK-MVE-NEXT:    vmov.16 q3[5], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s11, s7
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r1, #1
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[6], r0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q3
+; CHECK-MVE-NEXT:    vpop {d8, d9, d10}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_oeq_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vldr.16 s12, [r0]
+; CHECK-MVEFP-NEXT:    vmov r0, s12
+; CHECK-MVEFP-NEXT:    vcmp.f16 eq, q0, r0
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %src2 = load half, half* %src2p
+  %i = insertelement <8 x half> undef, half %src2, i32 0
+  %sp = shufflevector <8 x half> %i, <8 x half> undef, <8 x i32> zeroinitializer
+  %c = fcmp oeq <8 x half> %src, %sp
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_one_v8f16(<8 x half> %src, half* %src2p, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_one_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9, d10}
+; CHECK-MVE-NEXT:    vpush {d8, d9, d10}
+; CHECK-MVE-NEXT:    vldr.16 s16, [r0]
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s0
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
+; CHECK-MVE-NEXT:    vcmp.f16 s12, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r0, #1
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vcmp.f16 s0, s16
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r2, #1
+; CHECK-MVE-NEXT:    vmov r0, s12
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vcmp.f16 s1, s16
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[1], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r0, #1
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s9, s5
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s1
+; CHECK-MVE-NEXT:    vcmp.f16 s18, s16
+; CHECK-MVE-NEXT:    vmov.16 q3[2], r0
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r0, #1
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s5
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vcmp.f16 s2, s16
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
+; CHECK-MVE-NEXT:    vmov.16 q3[3], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r0, #1
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s10, s6
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s2
+; CHECK-MVE-NEXT:    vcmp.f16 s18, s16
+; CHECK-MVE-NEXT:    vmov.16 q3[4], r0
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r0, #1
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s6
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vcmp.f16 s3, s16
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vcmp.f16 s0, s16
+; CHECK-MVE-NEXT:    vmov.16 q3[5], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r0, #1
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s11, s7
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r1, #1
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[6], r0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q3
+; CHECK-MVE-NEXT:    vpop {d8, d9, d10}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_one_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vldr.16 s12, [r0]
+; CHECK-MVEFP-NEXT:    vmov r0, s12
+; CHECK-MVEFP-NEXT:    vpt.f16 ge, q0, r0
+; CHECK-MVEFP-NEXT:    vcmpt.f16 le, q0, r0
+; CHECK-MVEFP-NEXT:    vpnot
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %src2 = load half, half* %src2p
+  %i = insertelement <8 x half> undef, half %src2, i32 0
+  %sp = shufflevector <8 x half> %i, <8 x half> undef, <8 x i32> zeroinitializer
+  %c = fcmp one <8 x half> %src, %sp
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_ogt_v8f16(<8 x half> %src, half* %src2p, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_ogt_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9, d10}
+; CHECK-MVE-NEXT:    vpush {d8, d9, d10}
+; CHECK-MVE-NEXT:    vldr.16 s16, [r0]
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s0
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
+; CHECK-MVE-NEXT:    vcmp.f16 s12, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s0, s16
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    movs r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmov r0, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s1, s16
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[1], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s9, s5
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s1
+; CHECK-MVE-NEXT:    vcmp.f16 s18, s16
+; CHECK-MVE-NEXT:    vmov.16 q3[2], r0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s5
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s2, s16
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
+; CHECK-MVE-NEXT:    vmov.16 q3[3], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s10, s6
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s2
+; CHECK-MVE-NEXT:    vcmp.f16 s18, s16
+; CHECK-MVE-NEXT:    vmov.16 q3[4], r0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s6
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s3, s16
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vcmp.f16 s0, s16
+; CHECK-MVE-NEXT:    vmov.16 q3[5], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s11, s7
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r1, #1
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[6], r0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q3
+; CHECK-MVE-NEXT:    vpop {d8, d9, d10}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_ogt_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vldr.16 s12, [r0]
+; CHECK-MVEFP-NEXT:    vmov r0, s12
+; CHECK-MVEFP-NEXT:    vcmp.f16 gt, q0, r0
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %src2 = load half, half* %src2p
+  %i = insertelement <8 x half> undef, half %src2, i32 0
+  %sp = shufflevector <8 x half> %i, <8 x half> undef, <8 x i32> zeroinitializer
+  %c = fcmp ogt <8 x half> %src, %sp
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_oge_v8f16(<8 x half> %src, half* %src2p, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_oge_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9, d10}
+; CHECK-MVE-NEXT:    vpush {d8, d9, d10}
+; CHECK-MVE-NEXT:    vldr.16 s16, [r0]
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s0
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
+; CHECK-MVE-NEXT:    vcmp.f16 s12, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ge
+; CHECK-MVE-NEXT:    movge r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s0, s16
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    movs r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ge
+; CHECK-MVE-NEXT:    movge r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmov r0, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s1, s16
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[1], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it ge
+; CHECK-MVE-NEXT:    movge r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s9, s5
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s1
+; CHECK-MVE-NEXT:    vcmp.f16 s18, s16
+; CHECK-MVE-NEXT:    vmov.16 q3[2], r0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it ge
+; CHECK-MVE-NEXT:    movge r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s5
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s2, s16
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
+; CHECK-MVE-NEXT:    vmov.16 q3[3], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it ge
+; CHECK-MVE-NEXT:    movge r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s10, s6
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s2
+; CHECK-MVE-NEXT:    vcmp.f16 s18, s16
+; CHECK-MVE-NEXT:    vmov.16 q3[4], r0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it ge
+; CHECK-MVE-NEXT:    movge r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s6
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s3, s16
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vcmp.f16 s0, s16
+; CHECK-MVE-NEXT:    vmov.16 q3[5], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it ge
+; CHECK-MVE-NEXT:    movge r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s11, s7
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ge
+; CHECK-MVE-NEXT:    movge r1, #1
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[6], r0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q3
+; CHECK-MVE-NEXT:    vpop {d8, d9, d10}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_oge_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vldr.16 s12, [r0]
+; CHECK-MVEFP-NEXT:    vmov r0, s12
+; CHECK-MVEFP-NEXT:    vcmp.f16 ge, q0, r0
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %src2 = load half, half* %src2p
+  %i = insertelement <8 x half> undef, half %src2, i32 0
+  %sp = shufflevector <8 x half> %i, <8 x half> undef, <8 x i32> zeroinitializer
+  %c = fcmp oge <8 x half> %src, %sp
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_olt_v8f16(<8 x half> %src, half* %src2p, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_olt_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9, d10}
+; CHECK-MVE-NEXT:    vpush {d8, d9, d10}
+; CHECK-MVE-NEXT:    vldr.16 s16, [r0]
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s0
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
+; CHECK-MVE-NEXT:    vcmp.f16 s12, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s0, s16
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    movs r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmov r0, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s1, s16
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[1], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s9, s5
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s1
+; CHECK-MVE-NEXT:    vcmp.f16 s18, s16
+; CHECK-MVE-NEXT:    vmov.16 q3[2], r0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s5
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s2, s16
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
+; CHECK-MVE-NEXT:    vmov.16 q3[3], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s10, s6
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s2
+; CHECK-MVE-NEXT:    vcmp.f16 s18, s16
+; CHECK-MVE-NEXT:    vmov.16 q3[4], r0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s6
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s3, s16
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vcmp.f16 s0, s16
+; CHECK-MVE-NEXT:    vmov.16 q3[5], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s11, s7
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r1, #1
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[6], r0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q3
+; CHECK-MVE-NEXT:    vpop {d8, d9, d10}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_olt_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vldr.16 s12, [r0]
+; CHECK-MVEFP-NEXT:    vmov r0, s12
+; CHECK-MVEFP-NEXT:    vcmp.f16 lt, q0, r0
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %src2 = load half, half* %src2p
+  %i = insertelement <8 x half> undef, half %src2, i32 0
+  %sp = shufflevector <8 x half> %i, <8 x half> undef, <8 x i32> zeroinitializer
+  %c = fcmp olt <8 x half> %src, %sp
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_ole_v8f16(<8 x half> %src, half* %src2p, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_ole_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9, d10}
+; CHECK-MVE-NEXT:    vpush {d8, d9, d10}
+; CHECK-MVE-NEXT:    vldr.16 s16, [r0]
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s0
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
+; CHECK-MVE-NEXT:    vcmp.f16 s12, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ls
+; CHECK-MVE-NEXT:    movls r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s0, s16
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    movs r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ls
+; CHECK-MVE-NEXT:    movls r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmov r0, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s1, s16
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[1], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it ls
+; CHECK-MVE-NEXT:    movls r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s9, s5
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s1
+; CHECK-MVE-NEXT:    vcmp.f16 s18, s16
+; CHECK-MVE-NEXT:    vmov.16 q3[2], r0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it ls
+; CHECK-MVE-NEXT:    movls r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s5
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s2, s16
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
+; CHECK-MVE-NEXT:    vmov.16 q3[3], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it ls
+; CHECK-MVE-NEXT:    movls r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s10, s6
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s2
+; CHECK-MVE-NEXT:    vcmp.f16 s18, s16
+; CHECK-MVE-NEXT:    vmov.16 q3[4], r0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it ls
+; CHECK-MVE-NEXT:    movls r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s6
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s3, s16
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vcmp.f16 s0, s16
+; CHECK-MVE-NEXT:    vmov.16 q3[5], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it ls
+; CHECK-MVE-NEXT:    movls r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s11, s7
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ls
+; CHECK-MVE-NEXT:    movls r1, #1
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[6], r0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q3
+; CHECK-MVE-NEXT:    vpop {d8, d9, d10}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_ole_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vldr.16 s12, [r0]
+; CHECK-MVEFP-NEXT:    vmov r0, s12
+; CHECK-MVEFP-NEXT:    vcmp.f16 le, q0, r0
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %src2 = load half, half* %src2p
+  %i = insertelement <8 x half> undef, half %src2, i32 0
+  %sp = shufflevector <8 x half> %i, <8 x half> undef, <8 x i32> zeroinitializer
+  %c = fcmp ole <8 x half> %src, %sp
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_ueq_v8f16(<8 x half> %src, half* %src2p, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_ueq_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9, d10}
+; CHECK-MVE-NEXT:    vpush {d8, d9, d10}
+; CHECK-MVE-NEXT:    vldr.16 s16, [r0]
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s0
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
+; CHECK-MVE-NEXT:    vcmp.f16 s12, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r0, #1
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vcmp.f16 s0, s16
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r2, #1
+; CHECK-MVE-NEXT:    vmov r0, s12
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vcmp.f16 s1, s16
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[1], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r0, #1
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s9, s5
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s1
+; CHECK-MVE-NEXT:    vcmp.f16 s18, s16
+; CHECK-MVE-NEXT:    vmov.16 q3[2], r0
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r0, #1
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s5
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vcmp.f16 s2, s16
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
+; CHECK-MVE-NEXT:    vmov.16 q3[3], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r0, #1
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s10, s6
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s2
+; CHECK-MVE-NEXT:    vcmp.f16 s18, s16
+; CHECK-MVE-NEXT:    vmov.16 q3[4], r0
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r0, #1
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s6
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vcmp.f16 s3, s16
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vcmp.f16 s0, s16
+; CHECK-MVE-NEXT:    vmov.16 q3[5], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r0, #1
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s11, s7
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r1, #1
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[6], r0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q3
+; CHECK-MVE-NEXT:    vpop {d8, d9, d10}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_ueq_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vldr.16 s12, [r0]
+; CHECK-MVEFP-NEXT:    vmov r0, s12
+; CHECK-MVEFP-NEXT:    vpt.f16 ge, q0, r0
+; CHECK-MVEFP-NEXT:    vcmpt.f16 le, q0, r0
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %src2 = load half, half* %src2p
+  %i = insertelement <8 x half> undef, half %src2, i32 0
+  %sp = shufflevector <8 x half> %i, <8 x half> undef, <8 x i32> zeroinitializer
+  %c = fcmp ueq <8 x half> %src, %sp
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_une_v8f16(<8 x half> %src, half* %src2p, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_une_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9, d10}
+; CHECK-MVE-NEXT:    vpush {d8, d9, d10}
+; CHECK-MVE-NEXT:    vldr.16 s16, [r0]
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s0
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
+; CHECK-MVE-NEXT:    vcmp.f16 s12, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ne
+; CHECK-MVE-NEXT:    movne r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s0, s16
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    movs r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ne
+; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmov r0, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s1, s16
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[1], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it ne
+; CHECK-MVE-NEXT:    movne r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s9, s5
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s1
+; CHECK-MVE-NEXT:    vcmp.f16 s18, s16
+; CHECK-MVE-NEXT:    vmov.16 q3[2], r0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it ne
+; CHECK-MVE-NEXT:    movne r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s5
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s2, s16
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
+; CHECK-MVE-NEXT:    vmov.16 q3[3], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it ne
+; CHECK-MVE-NEXT:    movne r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s10, s6
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s2
+; CHECK-MVE-NEXT:    vcmp.f16 s18, s16
+; CHECK-MVE-NEXT:    vmov.16 q3[4], r0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it ne
+; CHECK-MVE-NEXT:    movne r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s6
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s3, s16
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vcmp.f16 s0, s16
+; CHECK-MVE-NEXT:    vmov.16 q3[5], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it ne
+; CHECK-MVE-NEXT:    movne r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s11, s7
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ne
+; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[6], r0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q3
+; CHECK-MVE-NEXT:    vpop {d8, d9, d10}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_une_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vldr.16 s12, [r0]
+; CHECK-MVEFP-NEXT:    vmov r0, s12
+; CHECK-MVEFP-NEXT:    vcmp.f16 ne, q0, r0
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %src2 = load half, half* %src2p
+  %i = insertelement <8 x half> undef, half %src2, i32 0
+  %sp = shufflevector <8 x half> %i, <8 x half> undef, <8 x i32> zeroinitializer
+  %c = fcmp une <8 x half> %src, %sp
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_ugt_v8f16(<8 x half> %src, half* %src2p, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_ugt_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9, d10}
+; CHECK-MVE-NEXT:    vpush {d8, d9, d10}
+; CHECK-MVE-NEXT:    vldr.16 s16, [r0]
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s0
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
+; CHECK-MVE-NEXT:    vcmp.f16 s12, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it hi
+; CHECK-MVE-NEXT:    movhi r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s0, s16
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    movs r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it hi
+; CHECK-MVE-NEXT:    movhi r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmov r0, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s1, s16
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[1], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it hi
+; CHECK-MVE-NEXT:    movhi r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s9, s5
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s1
+; CHECK-MVE-NEXT:    vcmp.f16 s18, s16
+; CHECK-MVE-NEXT:    vmov.16 q3[2], r0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it hi
+; CHECK-MVE-NEXT:    movhi r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s5
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s2, s16
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
+; CHECK-MVE-NEXT:    vmov.16 q3[3], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it hi
+; CHECK-MVE-NEXT:    movhi r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s10, s6
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s2
+; CHECK-MVE-NEXT:    vcmp.f16 s18, s16
+; CHECK-MVE-NEXT:    vmov.16 q3[4], r0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it hi
+; CHECK-MVE-NEXT:    movhi r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s6
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s3, s16
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vcmp.f16 s0, s16
+; CHECK-MVE-NEXT:    vmov.16 q3[5], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it hi
+; CHECK-MVE-NEXT:    movhi r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s11, s7
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it hi
+; CHECK-MVE-NEXT:    movhi r1, #1
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[6], r0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q3
+; CHECK-MVE-NEXT:    vpop {d8, d9, d10}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_ugt_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vldr.16 s12, [r0]
+; CHECK-MVEFP-NEXT:    vmov r0, s12
+; CHECK-MVEFP-NEXT:    vcmp.f16 le, q0, r0
+; CHECK-MVEFP-NEXT:    vpnot
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %src2 = load half, half* %src2p
+  %i = insertelement <8 x half> undef, half %src2, i32 0
+  %sp = shufflevector <8 x half> %i, <8 x half> undef, <8 x i32> zeroinitializer
+  %c = fcmp ugt <8 x half> %src, %sp
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_uge_v8f16(<8 x half> %src, half* %src2p, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_uge_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9, d10}
+; CHECK-MVE-NEXT:    vpush {d8, d9, d10}
+; CHECK-MVE-NEXT:    vldr.16 s16, [r0]
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s0
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
+; CHECK-MVE-NEXT:    vcmp.f16 s12, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it pl
+; CHECK-MVE-NEXT:    movpl r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s0, s16
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    movs r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it pl
+; CHECK-MVE-NEXT:    movpl r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmov r0, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s1, s16
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[1], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it pl
+; CHECK-MVE-NEXT:    movpl r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s9, s5
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s1
+; CHECK-MVE-NEXT:    vcmp.f16 s18, s16
+; CHECK-MVE-NEXT:    vmov.16 q3[2], r0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it pl
+; CHECK-MVE-NEXT:    movpl r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s5
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s2, s16
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
+; CHECK-MVE-NEXT:    vmov.16 q3[3], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it pl
+; CHECK-MVE-NEXT:    movpl r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s10, s6
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s2
+; CHECK-MVE-NEXT:    vcmp.f16 s18, s16
+; CHECK-MVE-NEXT:    vmov.16 q3[4], r0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it pl
+; CHECK-MVE-NEXT:    movpl r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s6
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s3, s16
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vcmp.f16 s0, s16
+; CHECK-MVE-NEXT:    vmov.16 q3[5], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it pl
+; CHECK-MVE-NEXT:    movpl r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s11, s7
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it pl
+; CHECK-MVE-NEXT:    movpl r1, #1
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[6], r0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q3
+; CHECK-MVE-NEXT:    vpop {d8, d9, d10}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_uge_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vldr.16 s12, [r0]
+; CHECK-MVEFP-NEXT:    vmov r0, s12
+; CHECK-MVEFP-NEXT:    vcmp.f16 lt, q0, r0
+; CHECK-MVEFP-NEXT:    vpnot
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %src2 = load half, half* %src2p
+  %i = insertelement <8 x half> undef, half %src2, i32 0
+  %sp = shufflevector <8 x half> %i, <8 x half> undef, <8 x i32> zeroinitializer
+  %c = fcmp uge <8 x half> %src, %sp
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_ult_v8f16(<8 x half> %src, half* %src2p, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_ult_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9, d10}
+; CHECK-MVE-NEXT:    vpush {d8, d9, d10}
+; CHECK-MVE-NEXT:    vldr.16 s16, [r0]
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s0
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
+; CHECK-MVE-NEXT:    vcmp.f16 s12, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it lt
+; CHECK-MVE-NEXT:    movlt r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s0, s16
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    movs r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it lt
+; CHECK-MVE-NEXT:    movlt r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmov r0, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s1, s16
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[1], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it lt
+; CHECK-MVE-NEXT:    movlt r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s9, s5
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s1
+; CHECK-MVE-NEXT:    vcmp.f16 s18, s16
+; CHECK-MVE-NEXT:    vmov.16 q3[2], r0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it lt
+; CHECK-MVE-NEXT:    movlt r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s5
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s2, s16
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
+; CHECK-MVE-NEXT:    vmov.16 q3[3], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it lt
+; CHECK-MVE-NEXT:    movlt r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s10, s6
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s2
+; CHECK-MVE-NEXT:    vcmp.f16 s18, s16
+; CHECK-MVE-NEXT:    vmov.16 q3[4], r0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it lt
+; CHECK-MVE-NEXT:    movlt r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s6
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s3, s16
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vcmp.f16 s0, s16
+; CHECK-MVE-NEXT:    vmov.16 q3[5], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it lt
+; CHECK-MVE-NEXT:    movlt r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s11, s7
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it lt
+; CHECK-MVE-NEXT:    movlt r1, #1
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[6], r0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q3
+; CHECK-MVE-NEXT:    vpop {d8, d9, d10}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_ult_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vldr.16 s12, [r0]
+; CHECK-MVEFP-NEXT:    vmov r0, s12
+; CHECK-MVEFP-NEXT:    vcmp.f16 ge, q0, r0
+; CHECK-MVEFP-NEXT:    vpnot
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %src2 = load half, half* %src2p
+  %i = insertelement <8 x half> undef, half %src2, i32 0
+  %sp = shufflevector <8 x half> %i, <8 x half> undef, <8 x i32> zeroinitializer
+  %c = fcmp ult <8 x half> %src, %sp
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_ule_v8f16(<8 x half> %src, half* %src2p, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_ule_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9, d10}
+; CHECK-MVE-NEXT:    vpush {d8, d9, d10}
+; CHECK-MVE-NEXT:    vldr.16 s16, [r0]
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s0
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
+; CHECK-MVE-NEXT:    vcmp.f16 s12, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it le
+; CHECK-MVE-NEXT:    movle r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s0, s16
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    movs r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it le
+; CHECK-MVE-NEXT:    movle r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmov r0, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s1, s16
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[1], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it le
+; CHECK-MVE-NEXT:    movle r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s9, s5
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s1
+; CHECK-MVE-NEXT:    vcmp.f16 s18, s16
+; CHECK-MVE-NEXT:    vmov.16 q3[2], r0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it le
+; CHECK-MVE-NEXT:    movle r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s5
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s2, s16
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
+; CHECK-MVE-NEXT:    vmov.16 q3[3], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it le
+; CHECK-MVE-NEXT:    movle r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s10, s6
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s2
+; CHECK-MVE-NEXT:    vcmp.f16 s18, s16
+; CHECK-MVE-NEXT:    vmov.16 q3[4], r0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it le
+; CHECK-MVE-NEXT:    movle r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s6
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s3, s16
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vcmp.f16 s0, s16
+; CHECK-MVE-NEXT:    vmov.16 q3[5], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it le
+; CHECK-MVE-NEXT:    movle r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s11, s7
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it le
+; CHECK-MVE-NEXT:    movle r1, #1
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[6], r0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q3
+; CHECK-MVE-NEXT:    vpop {d8, d9, d10}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_ule_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vldr.16 s12, [r0]
+; CHECK-MVEFP-NEXT:    vmov r0, s12
+; CHECK-MVEFP-NEXT:    vcmp.f16 gt, q0, r0
+; CHECK-MVEFP-NEXT:    vpnot
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %src2 = load half, half* %src2p
+  %i = insertelement <8 x half> undef, half %src2, i32 0
+  %sp = shufflevector <8 x half> %i, <8 x half> undef, <8 x i32> zeroinitializer
+  %c = fcmp ule <8 x half> %src, %sp
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_ord_v8f16(<8 x half> %src, half* %src2p, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_ord_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9, d10}
+; CHECK-MVE-NEXT:    vpush {d8, d9, d10}
+; CHECK-MVE-NEXT:    vldr.16 s16, [r0]
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s0
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
+; CHECK-MVE-NEXT:    vcmp.f16 s12, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it vc
+; CHECK-MVE-NEXT:    movvc r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s0, s16
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    movs r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it vc
+; CHECK-MVE-NEXT:    movvc r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmov r0, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s1, s16
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[1], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it vc
+; CHECK-MVE-NEXT:    movvc r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s9, s5
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s1
+; CHECK-MVE-NEXT:    vcmp.f16 s18, s16
+; CHECK-MVE-NEXT:    vmov.16 q3[2], r0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it vc
+; CHECK-MVE-NEXT:    movvc r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s5
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s2, s16
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
+; CHECK-MVE-NEXT:    vmov.16 q3[3], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it vc
+; CHECK-MVE-NEXT:    movvc r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s10, s6
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s2
+; CHECK-MVE-NEXT:    vcmp.f16 s18, s16
+; CHECK-MVE-NEXT:    vmov.16 q3[4], r0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it vc
+; CHECK-MVE-NEXT:    movvc r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s6
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s3, s16
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vcmp.f16 s0, s16
+; CHECK-MVE-NEXT:    vmov.16 q3[5], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it vc
+; CHECK-MVE-NEXT:    movvc r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s11, s7
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it vc
+; CHECK-MVE-NEXT:    movvc r1, #1
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[6], r0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q3
+; CHECK-MVE-NEXT:    vpop {d8, d9, d10}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_ord_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vldr.16 s12, [r0]
+; CHECK-MVEFP-NEXT:    vmov r0, s12
+; CHECK-MVEFP-NEXT:    vpt.f16 ge, q0, r0
+; CHECK-MVEFP-NEXT:    vcmpt.f16 lt, q0, r0
+; CHECK-MVEFP-NEXT:    vpnot
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %src2 = load half, half* %src2p
+  %i = insertelement <8 x half> undef, half %src2, i32 0
+  %sp = shufflevector <8 x half> %i, <8 x half> undef, <8 x i32> zeroinitializer
+  %c = fcmp ord <8 x half> %src, %sp
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_uno_v8f16(<8 x half> %src, half* %src2p, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_uno_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9, d10}
+; CHECK-MVE-NEXT:    vpush {d8, d9, d10}
+; CHECK-MVE-NEXT:    vldr.16 s16, [r0]
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s0
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
+; CHECK-MVE-NEXT:    vcmp.f16 s12, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s0, s16
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    movs r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmov r0, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s1, s16
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[1], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s9, s5
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s1
+; CHECK-MVE-NEXT:    vcmp.f16 s18, s16
+; CHECK-MVE-NEXT:    vmov.16 q3[2], r0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s5
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s2, s16
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
+; CHECK-MVE-NEXT:    vmov.16 q3[3], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s10, s6
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s2
+; CHECK-MVE-NEXT:    vcmp.f16 s18, s16
+; CHECK-MVE-NEXT:    vmov.16 q3[4], r0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s6
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s3, s16
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vcmp.f16 s0, s16
+; CHECK-MVE-NEXT:    vmov.16 q3[5], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s11, s7
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r1, #1
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[6], r0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q3
+; CHECK-MVE-NEXT:    vpop {d8, d9, d10}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_uno_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vldr.16 s12, [r0]
+; CHECK-MVEFP-NEXT:    vmov r0, s12
+; CHECK-MVEFP-NEXT:    vpt.f16 ge, q0, r0
+; CHECK-MVEFP-NEXT:    vcmpt.f16 lt, q0, r0
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %src2 = load half, half* %src2p
+  %i = insertelement <8 x half> undef, half %src2, i32 0
+  %sp = shufflevector <8 x half> %i, <8 x half> undef, <8 x i32> zeroinitializer
+  %c = fcmp uno <8 x half> %src, %sp
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
+
+
+; Reversed
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_r_oeq_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_r_oeq_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s1
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s2
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s3
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_r_oeq_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vmov r0, s4
+; CHECK-MVEFP-NEXT:    vcmp.f32 eq, q0, r0
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %i = insertelement <4 x float> undef, float %src2, i32 0
+  %sp = shufflevector <4 x float> %i, <4 x float> undef, <4 x i32> zeroinitializer
+  %c = fcmp oeq <4 x float> %sp, %src
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_r_one_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_r_one_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s0
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r1, #1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s1
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r2, #1
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s2
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r3, #1
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s3
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r0, #1
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_r_one_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vmov r0, s4
+; CHECK-MVEFP-NEXT:    vpt.f32 le, q0, r0
+; CHECK-MVEFP-NEXT:    vcmpt.f32 ge, q0, r0
+; CHECK-MVEFP-NEXT:    vpnot
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %i = insertelement <4 x float> undef, float %src2, i32 0
+  %sp = shufflevector <4 x float> %i, <4 x float> undef, <4 x i32> zeroinitializer
+  %c = fcmp one <4 x float> %sp, %src
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_r_ogt_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_r_ogt_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s1
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s2
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s3
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_r_ogt_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vmov r0, s4
+; CHECK-MVEFP-NEXT:    vcmp.f32 lt, q0, r0
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %i = insertelement <4 x float> undef, float %src2, i32 0
+  %sp = shufflevector <4 x float> %i, <4 x float> undef, <4 x i32> zeroinitializer
+  %c = fcmp ogt <4 x float> %sp, %src
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_r_oge_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_r_oge_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ge
+; CHECK-MVE-NEXT:    movge r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s1
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s2
+; CHECK-MVE-NEXT:    it ge
+; CHECK-MVE-NEXT:    movge r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s3
+; CHECK-MVE-NEXT:    it ge
+; CHECK-MVE-NEXT:    movge r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ge
+; CHECK-MVE-NEXT:    movge r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_r_oge_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vmov r0, s4
+; CHECK-MVEFP-NEXT:    vcmp.f32 le, q0, r0
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %i = insertelement <4 x float> undef, float %src2, i32 0
+  %sp = shufflevector <4 x float> %i, <4 x float> undef, <4 x i32> zeroinitializer
+  %c = fcmp oge <4 x float> %sp, %src
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_r_olt_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_r_olt_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s1
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s2
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s3
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_r_olt_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vmov r0, s4
+; CHECK-MVEFP-NEXT:    vcmp.f32 gt, q0, r0
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %i = insertelement <4 x float> undef, float %src2, i32 0
+  %sp = shufflevector <4 x float> %i, <4 x float> undef, <4 x i32> zeroinitializer
+  %c = fcmp olt <4 x float> %sp, %src
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_r_ole_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_r_ole_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ls
+; CHECK-MVE-NEXT:    movls r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s1
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s2
+; CHECK-MVE-NEXT:    it ls
+; CHECK-MVE-NEXT:    movls r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s3
+; CHECK-MVE-NEXT:    it ls
+; CHECK-MVE-NEXT:    movls r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ls
+; CHECK-MVE-NEXT:    movls r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_r_ole_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vmov r0, s4
+; CHECK-MVEFP-NEXT:    vcmp.f32 ge, q0, r0
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %i = insertelement <4 x float> undef, float %src2, i32 0
+  %sp = shufflevector <4 x float> %i, <4 x float> undef, <4 x i32> zeroinitializer
+  %c = fcmp ole <4 x float> %sp, %src
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_r_ueq_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_r_ueq_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s0
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r1, #1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s1
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r2, #1
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s2
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r3, #1
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s3
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r0, #1
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_r_ueq_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vmov r0, s4
+; CHECK-MVEFP-NEXT:    vpt.f32 le, q0, r0
+; CHECK-MVEFP-NEXT:    vcmpt.f32 ge, q0, r0
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %i = insertelement <4 x float> undef, float %src2, i32 0
+  %sp = shufflevector <4 x float> %i, <4 x float> undef, <4 x i32> zeroinitializer
+  %c = fcmp ueq <4 x float> %sp, %src
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_r_une_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_r_une_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ne
+; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s1
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s2
+; CHECK-MVE-NEXT:    it ne
+; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s3
+; CHECK-MVE-NEXT:    it ne
+; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ne
+; CHECK-MVE-NEXT:    movne r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_r_une_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vmov r0, s4
+; CHECK-MVEFP-NEXT:    vcmp.f32 ne, q0, r0
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %i = insertelement <4 x float> undef, float %src2, i32 0
+  %sp = shufflevector <4 x float> %i, <4 x float> undef, <4 x i32> zeroinitializer
+  %c = fcmp une <4 x float> %sp, %src
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_r_ugt_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_r_ugt_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it hi
+; CHECK-MVE-NEXT:    movhi r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s1
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s2
+; CHECK-MVE-NEXT:    it hi
+; CHECK-MVE-NEXT:    movhi r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s3
+; CHECK-MVE-NEXT:    it hi
+; CHECK-MVE-NEXT:    movhi r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it hi
+; CHECK-MVE-NEXT:    movhi r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_r_ugt_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vmov r0, s4
+; CHECK-MVEFP-NEXT:    vcmp.f32 ge, q0, r0
+; CHECK-MVEFP-NEXT:    vpnot
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %i = insertelement <4 x float> undef, float %src2, i32 0
+  %sp = shufflevector <4 x float> %i, <4 x float> undef, <4 x i32> zeroinitializer
+  %c = fcmp ugt <4 x float> %sp, %src
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_r_uge_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_r_uge_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it pl
+; CHECK-MVE-NEXT:    movpl r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s1
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s2
+; CHECK-MVE-NEXT:    it pl
+; CHECK-MVE-NEXT:    movpl r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s3
+; CHECK-MVE-NEXT:    it pl
+; CHECK-MVE-NEXT:    movpl r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it pl
+; CHECK-MVE-NEXT:    movpl r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_r_uge_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vmov r0, s4
+; CHECK-MVEFP-NEXT:    vcmp.f32 gt, q0, r0
+; CHECK-MVEFP-NEXT:    vpnot
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %i = insertelement <4 x float> undef, float %src2, i32 0
+  %sp = shufflevector <4 x float> %i, <4 x float> undef, <4 x i32> zeroinitializer
+  %c = fcmp uge <4 x float> %sp, %src
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_r_ult_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_r_ult_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it lt
+; CHECK-MVE-NEXT:    movlt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s1
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s2
+; CHECK-MVE-NEXT:    it lt
+; CHECK-MVE-NEXT:    movlt r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s3
+; CHECK-MVE-NEXT:    it lt
+; CHECK-MVE-NEXT:    movlt r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it lt
+; CHECK-MVE-NEXT:    movlt r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_r_ult_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vmov r0, s4
+; CHECK-MVEFP-NEXT:    vcmp.f32 le, q0, r0
+; CHECK-MVEFP-NEXT:    vpnot
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %i = insertelement <4 x float> undef, float %src2, i32 0
+  %sp = shufflevector <4 x float> %i, <4 x float> undef, <4 x i32> zeroinitializer
+  %c = fcmp ult <4 x float> %sp, %src
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_r_ule_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_r_ule_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it le
+; CHECK-MVE-NEXT:    movle r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s1
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s2
+; CHECK-MVE-NEXT:    it le
+; CHECK-MVE-NEXT:    movle r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s3
+; CHECK-MVE-NEXT:    it le
+; CHECK-MVE-NEXT:    movle r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it le
+; CHECK-MVE-NEXT:    movle r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_r_ule_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vmov r0, s4
+; CHECK-MVEFP-NEXT:    vcmp.f32 lt, q0, r0
+; CHECK-MVEFP-NEXT:    vpnot
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %i = insertelement <4 x float> undef, float %src2, i32 0
+  %sp = shufflevector <4 x float> %i, <4 x float> undef, <4 x i32> zeroinitializer
+  %c = fcmp ule <4 x float> %sp, %src
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_r_ord_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_r_ord_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it vc
+; CHECK-MVE-NEXT:    movvc r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s1
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s2
+; CHECK-MVE-NEXT:    it vc
+; CHECK-MVE-NEXT:    movvc r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s3
+; CHECK-MVE-NEXT:    it vc
+; CHECK-MVE-NEXT:    movvc r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it vc
+; CHECK-MVE-NEXT:    movvc r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_r_ord_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vmov r0, s4
+; CHECK-MVEFP-NEXT:    vpt.f32 le, q0, r0
+; CHECK-MVEFP-NEXT:    vcmpt.f32 gt, q0, r0
+; CHECK-MVEFP-NEXT:    vpnot
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %i = insertelement <4 x float> undef, float %src2, i32 0
+  %sp = shufflevector <4 x float> %i, <4 x float> undef, <4 x i32> zeroinitializer
+  %c = fcmp ord <4 x float> %sp, %src
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_r_uno_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_r_uno_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s1
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s2
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s3
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_r_uno_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vmov r0, s4
+; CHECK-MVEFP-NEXT:    vpt.f32 le, q0, r0
+; CHECK-MVEFP-NEXT:    vcmpt.f32 gt, q0, r0
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %i = insertelement <4 x float> undef, float %src2, i32 0
+  %sp = shufflevector <4 x float> %i, <4 x float> undef, <4 x i32> zeroinitializer
+  %c = fcmp uno <4 x float> %sp, %src
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_r_oeq_v8f16(<8 x half> %src, half* %src2p, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_r_oeq_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9, d10}
+; CHECK-MVE-NEXT:    vpush {d8, d9, d10}
+; CHECK-MVE-NEXT:    vldr.16 s16, [r0]
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s0
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    movs r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmov r0, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s1
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[1], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s9, s5
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s1
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s18
+; CHECK-MVE-NEXT:    vmov.16 q3[2], r0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s5
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s2
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
+; CHECK-MVE-NEXT:    vmov.16 q3[3], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s10, s6
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s2
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s18
+; CHECK-MVE-NEXT:    vmov.16 q3[4], r0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s6
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s3
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[5], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s11, s7
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r1, #1
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[6], r0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q3
+; CHECK-MVE-NEXT:    vpop {d8, d9, d10}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_r_oeq_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vldr.16 s12, [r0]
+; CHECK-MVEFP-NEXT:    vmov r0, s12
+; CHECK-MVEFP-NEXT:    vcmp.f16 eq, q0, r0
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %src2 = load half, half* %src2p
+  %i = insertelement <8 x half> undef, half %src2, i32 0
+  %sp = shufflevector <8 x half> %i, <8 x half> undef, <8 x i32> zeroinitializer
+  %c = fcmp oeq <8 x half> %sp, %src
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_r_one_v8f16(<8 x half> %src, half* %src2p, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_r_one_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9, d10}
+; CHECK-MVE-NEXT:    vpush {d8, d9, d10}
+; CHECK-MVE-NEXT:    vldr.16 s16, [r0]
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s0
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r0, #1
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r2, #1
+; CHECK-MVE-NEXT:    vmov r0, s12
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s1
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[1], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r0, #1
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s9, s5
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s1
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s18
+; CHECK-MVE-NEXT:    vmov.16 q3[2], r0
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r0, #1
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s5
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s2
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
+; CHECK-MVE-NEXT:    vmov.16 q3[3], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r0, #1
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s10, s6
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s2
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s18
+; CHECK-MVE-NEXT:    vmov.16 q3[4], r0
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r0, #1
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s6
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s3
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[5], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r0, #1
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s11, s7
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r1, #1
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[6], r0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q3
+; CHECK-MVE-NEXT:    vpop {d8, d9, d10}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_r_one_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vldr.16 s12, [r0]
+; CHECK-MVEFP-NEXT:    vmov r0, s12
+; CHECK-MVEFP-NEXT:    vpt.f16 le, q0, r0
+; CHECK-MVEFP-NEXT:    vcmpt.f16 ge, q0, r0
+; CHECK-MVEFP-NEXT:    vpnot
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %src2 = load half, half* %src2p
+  %i = insertelement <8 x half> undef, half %src2, i32 0
+  %sp = shufflevector <8 x half> %i, <8 x half> undef, <8 x i32> zeroinitializer
+  %c = fcmp one <8 x half> %sp, %src
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_r_ogt_v8f16(<8 x half> %src, half* %src2p, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_r_ogt_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9, d10}
+; CHECK-MVE-NEXT:    vpush {d8, d9, d10}
+; CHECK-MVE-NEXT:    vldr.16 s16, [r0]
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s0
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    movs r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmov r0, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s1
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[1], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s9, s5
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s1
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s18
+; CHECK-MVE-NEXT:    vmov.16 q3[2], r0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s5
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s2
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
+; CHECK-MVE-NEXT:    vmov.16 q3[3], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s10, s6
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s2
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s18
+; CHECK-MVE-NEXT:    vmov.16 q3[4], r0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s6
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s3
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[5], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s11, s7
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r1, #1
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[6], r0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q3
+; CHECK-MVE-NEXT:    vpop {d8, d9, d10}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_r_ogt_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vldr.16 s12, [r0]
+; CHECK-MVEFP-NEXT:    vmov r0, s12
+; CHECK-MVEFP-NEXT:    vcmp.f16 lt, q0, r0
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %src2 = load half, half* %src2p
+  %i = insertelement <8 x half> undef, half %src2, i32 0
+  %sp = shufflevector <8 x half> %i, <8 x half> undef, <8 x i32> zeroinitializer
+  %c = fcmp ogt <8 x half> %sp, %src
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_r_oge_v8f16(<8 x half> %src, half* %src2p, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_r_oge_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9, d10}
+; CHECK-MVE-NEXT:    vpush {d8, d9, d10}
+; CHECK-MVE-NEXT:    vldr.16 s16, [r0]
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s0
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ge
+; CHECK-MVE-NEXT:    movge r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    movs r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ge
+; CHECK-MVE-NEXT:    movge r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmov r0, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s1
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[1], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it ge
+; CHECK-MVE-NEXT:    movge r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s9, s5
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s1
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s18
+; CHECK-MVE-NEXT:    vmov.16 q3[2], r0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it ge
+; CHECK-MVE-NEXT:    movge r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s5
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s2
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
+; CHECK-MVE-NEXT:    vmov.16 q3[3], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it ge
+; CHECK-MVE-NEXT:    movge r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s10, s6
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s2
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s18
+; CHECK-MVE-NEXT:    vmov.16 q3[4], r0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it ge
+; CHECK-MVE-NEXT:    movge r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s6
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s3
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[5], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it ge
+; CHECK-MVE-NEXT:    movge r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s11, s7
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ge
+; CHECK-MVE-NEXT:    movge r1, #1
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[6], r0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q3
+; CHECK-MVE-NEXT:    vpop {d8, d9, d10}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_r_oge_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vldr.16 s12, [r0]
+; CHECK-MVEFP-NEXT:    vmov r0, s12
+; CHECK-MVEFP-NEXT:    vcmp.f16 le, q0, r0
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %src2 = load half, half* %src2p
+  %i = insertelement <8 x half> undef, half %src2, i32 0
+  %sp = shufflevector <8 x half> %i, <8 x half> undef, <8 x i32> zeroinitializer
+  %c = fcmp oge <8 x half> %sp, %src
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_r_olt_v8f16(<8 x half> %src, half* %src2p, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_r_olt_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9, d10}
+; CHECK-MVE-NEXT:    vpush {d8, d9, d10}
+; CHECK-MVE-NEXT:    vldr.16 s16, [r0]
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s0
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    movs r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmov r0, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s1
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[1], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s9, s5
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s1
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s18
+; CHECK-MVE-NEXT:    vmov.16 q3[2], r0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s5
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s2
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
+; CHECK-MVE-NEXT:    vmov.16 q3[3], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s10, s6
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s2
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s18
+; CHECK-MVE-NEXT:    vmov.16 q3[4], r0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s6
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s3
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[5], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s11, s7
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r1, #1
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[6], r0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q3
+; CHECK-MVE-NEXT:    vpop {d8, d9, d10}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_r_olt_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vldr.16 s12, [r0]
+; CHECK-MVEFP-NEXT:    vmov r0, s12
+; CHECK-MVEFP-NEXT:    vcmp.f16 gt, q0, r0
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %src2 = load half, half* %src2p
+  %i = insertelement <8 x half> undef, half %src2, i32 0
+  %sp = shufflevector <8 x half> %i, <8 x half> undef, <8 x i32> zeroinitializer
+  %c = fcmp olt <8 x half> %sp, %src
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_r_ole_v8f16(<8 x half> %src, half* %src2p, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_r_ole_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9, d10}
+; CHECK-MVE-NEXT:    vpush {d8, d9, d10}
+; CHECK-MVE-NEXT:    vldr.16 s16, [r0]
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s0
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ls
+; CHECK-MVE-NEXT:    movls r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    movs r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ls
+; CHECK-MVE-NEXT:    movls r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmov r0, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s1
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[1], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it ls
+; CHECK-MVE-NEXT:    movls r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s9, s5
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s1
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s18
+; CHECK-MVE-NEXT:    vmov.16 q3[2], r0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it ls
+; CHECK-MVE-NEXT:    movls r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s5
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s2
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
+; CHECK-MVE-NEXT:    vmov.16 q3[3], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it ls
+; CHECK-MVE-NEXT:    movls r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s10, s6
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s2
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s18
+; CHECK-MVE-NEXT:    vmov.16 q3[4], r0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it ls
+; CHECK-MVE-NEXT:    movls r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s6
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s3
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[5], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it ls
+; CHECK-MVE-NEXT:    movls r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s11, s7
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ls
+; CHECK-MVE-NEXT:    movls r1, #1
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[6], r0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q3
+; CHECK-MVE-NEXT:    vpop {d8, d9, d10}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_r_ole_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vldr.16 s12, [r0]
+; CHECK-MVEFP-NEXT:    vmov r0, s12
+; CHECK-MVEFP-NEXT:    vcmp.f16 ge, q0, r0
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %src2 = load half, half* %src2p
+  %i = insertelement <8 x half> undef, half %src2, i32 0
+  %sp = shufflevector <8 x half> %i, <8 x half> undef, <8 x i32> zeroinitializer
+  %c = fcmp ole <8 x half> %sp, %src
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_r_ueq_v8f16(<8 x half> %src, half* %src2p, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_r_ueq_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9, d10}
+; CHECK-MVE-NEXT:    vpush {d8, d9, d10}
+; CHECK-MVE-NEXT:    vldr.16 s16, [r0]
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s0
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r0, #1
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r2, #1
+; CHECK-MVE-NEXT:    vmov r0, s12
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s1
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[1], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r0, #1
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s9, s5
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s1
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s18
+; CHECK-MVE-NEXT:    vmov.16 q3[2], r0
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r0, #1
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s5
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s2
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
+; CHECK-MVE-NEXT:    vmov.16 q3[3], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r0, #1
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s10, s6
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s2
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s18
+; CHECK-MVE-NEXT:    vmov.16 q3[4], r0
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r0, #1
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s6
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s3
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[5], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r0, #1
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s11, s7
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r1, #1
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[6], r0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q3
+; CHECK-MVE-NEXT:    vpop {d8, d9, d10}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_r_ueq_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vldr.16 s12, [r0]
+; CHECK-MVEFP-NEXT:    vmov r0, s12
+; CHECK-MVEFP-NEXT:    vpt.f16 le, q0, r0
+; CHECK-MVEFP-NEXT:    vcmpt.f16 ge, q0, r0
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %src2 = load half, half* %src2p
+  %i = insertelement <8 x half> undef, half %src2, i32 0
+  %sp = shufflevector <8 x half> %i, <8 x half> undef, <8 x i32> zeroinitializer
+  %c = fcmp ueq <8 x half> %sp, %src
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_r_une_v8f16(<8 x half> %src, half* %src2p, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_r_une_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9, d10}
+; CHECK-MVE-NEXT:    vpush {d8, d9, d10}
+; CHECK-MVE-NEXT:    vldr.16 s16, [r0]
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s0
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ne
+; CHECK-MVE-NEXT:    movne r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    movs r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ne
+; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmov r0, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s1
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[1], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it ne
+; CHECK-MVE-NEXT:    movne r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s9, s5
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s1
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s18
+; CHECK-MVE-NEXT:    vmov.16 q3[2], r0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it ne
+; CHECK-MVE-NEXT:    movne r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s5
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s2
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
+; CHECK-MVE-NEXT:    vmov.16 q3[3], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it ne
+; CHECK-MVE-NEXT:    movne r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s10, s6
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s2
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s18
+; CHECK-MVE-NEXT:    vmov.16 q3[4], r0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it ne
+; CHECK-MVE-NEXT:    movne r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s6
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s3
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[5], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it ne
+; CHECK-MVE-NEXT:    movne r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s11, s7
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ne
+; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[6], r0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q3
+; CHECK-MVE-NEXT:    vpop {d8, d9, d10}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_r_une_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vldr.16 s12, [r0]
+; CHECK-MVEFP-NEXT:    vmov r0, s12
+; CHECK-MVEFP-NEXT:    vcmp.f16 ne, q0, r0
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %src2 = load half, half* %src2p
+  %i = insertelement <8 x half> undef, half %src2, i32 0
+  %sp = shufflevector <8 x half> %i, <8 x half> undef, <8 x i32> zeroinitializer
+  %c = fcmp une <8 x half> %sp, %src
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_r_ugt_v8f16(<8 x half> %src, half* %src2p, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_r_ugt_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9, d10}
+; CHECK-MVE-NEXT:    vpush {d8, d9, d10}
+; CHECK-MVE-NEXT:    vldr.16 s16, [r0]
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s0
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it hi
+; CHECK-MVE-NEXT:    movhi r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    movs r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it hi
+; CHECK-MVE-NEXT:    movhi r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmov r0, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s1
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[1], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it hi
+; CHECK-MVE-NEXT:    movhi r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s9, s5
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s1
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s18
+; CHECK-MVE-NEXT:    vmov.16 q3[2], r0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it hi
+; CHECK-MVE-NEXT:    movhi r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s5
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s2
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
+; CHECK-MVE-NEXT:    vmov.16 q3[3], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it hi
+; CHECK-MVE-NEXT:    movhi r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s10, s6
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s2
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s18
+; CHECK-MVE-NEXT:    vmov.16 q3[4], r0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it hi
+; CHECK-MVE-NEXT:    movhi r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s6
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s3
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[5], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it hi
+; CHECK-MVE-NEXT:    movhi r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s11, s7
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it hi
+; CHECK-MVE-NEXT:    movhi r1, #1
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[6], r0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q3
+; CHECK-MVE-NEXT:    vpop {d8, d9, d10}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_r_ugt_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vldr.16 s12, [r0]
+; CHECK-MVEFP-NEXT:    vmov r0, s12
+; CHECK-MVEFP-NEXT:    vcmp.f16 ge, q0, r0
+; CHECK-MVEFP-NEXT:    vpnot
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %src2 = load half, half* %src2p
+  %i = insertelement <8 x half> undef, half %src2, i32 0
+  %sp = shufflevector <8 x half> %i, <8 x half> undef, <8 x i32> zeroinitializer
+  %c = fcmp ugt <8 x half> %sp, %src
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_r_uge_v8f16(<8 x half> %src, half* %src2p, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_r_uge_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9, d10}
+; CHECK-MVE-NEXT:    vpush {d8, d9, d10}
+; CHECK-MVE-NEXT:    vldr.16 s16, [r0]
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s0
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it pl
+; CHECK-MVE-NEXT:    movpl r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    movs r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it pl
+; CHECK-MVE-NEXT:    movpl r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmov r0, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s1
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[1], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it pl
+; CHECK-MVE-NEXT:    movpl r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s9, s5
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s1
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s18
+; CHECK-MVE-NEXT:    vmov.16 q3[2], r0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it pl
+; CHECK-MVE-NEXT:    movpl r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s5
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s2
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
+; CHECK-MVE-NEXT:    vmov.16 q3[3], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it pl
+; CHECK-MVE-NEXT:    movpl r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s10, s6
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s2
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s18
+; CHECK-MVE-NEXT:    vmov.16 q3[4], r0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it pl
+; CHECK-MVE-NEXT:    movpl r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s6
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s3
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[5], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it pl
+; CHECK-MVE-NEXT:    movpl r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s11, s7
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it pl
+; CHECK-MVE-NEXT:    movpl r1, #1
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[6], r0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q3
+; CHECK-MVE-NEXT:    vpop {d8, d9, d10}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_r_uge_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vldr.16 s12, [r0]
+; CHECK-MVEFP-NEXT:    vmov r0, s12
+; CHECK-MVEFP-NEXT:    vcmp.f16 gt, q0, r0
+; CHECK-MVEFP-NEXT:    vpnot
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %src2 = load half, half* %src2p
+  %i = insertelement <8 x half> undef, half %src2, i32 0
+  %sp = shufflevector <8 x half> %i, <8 x half> undef, <8 x i32> zeroinitializer
+  %c = fcmp uge <8 x half> %sp, %src
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_r_ult_v8f16(<8 x half> %src, half* %src2p, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_r_ult_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9, d10}
+; CHECK-MVE-NEXT:    vpush {d8, d9, d10}
+; CHECK-MVE-NEXT:    vldr.16 s16, [r0]
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s0
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it lt
+; CHECK-MVE-NEXT:    movlt r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    movs r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it lt
+; CHECK-MVE-NEXT:    movlt r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmov r0, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s1
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[1], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it lt
+; CHECK-MVE-NEXT:    movlt r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s9, s5
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s1
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s18
+; CHECK-MVE-NEXT:    vmov.16 q3[2], r0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it lt
+; CHECK-MVE-NEXT:    movlt r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s5
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s2
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
+; CHECK-MVE-NEXT:    vmov.16 q3[3], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it lt
+; CHECK-MVE-NEXT:    movlt r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s10, s6
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s2
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s18
+; CHECK-MVE-NEXT:    vmov.16 q3[4], r0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it lt
+; CHECK-MVE-NEXT:    movlt r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s6
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s3
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[5], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it lt
+; CHECK-MVE-NEXT:    movlt r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s11, s7
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it lt
+; CHECK-MVE-NEXT:    movlt r1, #1
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[6], r0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q3
+; CHECK-MVE-NEXT:    vpop {d8, d9, d10}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_r_ult_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vldr.16 s12, [r0]
+; CHECK-MVEFP-NEXT:    vmov r0, s12
+; CHECK-MVEFP-NEXT:    vcmp.f16 le, q0, r0
+; CHECK-MVEFP-NEXT:    vpnot
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %src2 = load half, half* %src2p
+  %i = insertelement <8 x half> undef, half %src2, i32 0
+  %sp = shufflevector <8 x half> %i, <8 x half> undef, <8 x i32> zeroinitializer
+  %c = fcmp ult <8 x half> %sp, %src
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_r_ule_v8f16(<8 x half> %src, half* %src2p, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_r_ule_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9, d10}
+; CHECK-MVE-NEXT:    vpush {d8, d9, d10}
+; CHECK-MVE-NEXT:    vldr.16 s16, [r0]
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s0
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it le
+; CHECK-MVE-NEXT:    movle r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    movs r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it le
+; CHECK-MVE-NEXT:    movle r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmov r0, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s1
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[1], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it le
+; CHECK-MVE-NEXT:    movle r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s9, s5
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s1
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s18
+; CHECK-MVE-NEXT:    vmov.16 q3[2], r0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it le
+; CHECK-MVE-NEXT:    movle r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s5
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s2
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
+; CHECK-MVE-NEXT:    vmov.16 q3[3], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it le
+; CHECK-MVE-NEXT:    movle r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s10, s6
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s2
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s18
+; CHECK-MVE-NEXT:    vmov.16 q3[4], r0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it le
+; CHECK-MVE-NEXT:    movle r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s6
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s3
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[5], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it le
+; CHECK-MVE-NEXT:    movle r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s11, s7
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it le
+; CHECK-MVE-NEXT:    movle r1, #1
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[6], r0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q3
+; CHECK-MVE-NEXT:    vpop {d8, d9, d10}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_r_ule_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vldr.16 s12, [r0]
+; CHECK-MVEFP-NEXT:    vmov r0, s12
+; CHECK-MVEFP-NEXT:    vcmp.f16 lt, q0, r0
+; CHECK-MVEFP-NEXT:    vpnot
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %src2 = load half, half* %src2p
+  %i = insertelement <8 x half> undef, half %src2, i32 0
+  %sp = shufflevector <8 x half> %i, <8 x half> undef, <8 x i32> zeroinitializer
+  %c = fcmp ule <8 x half> %sp, %src
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_r_ord_v8f16(<8 x half> %src, half* %src2p, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_r_ord_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9, d10}
+; CHECK-MVE-NEXT:    vpush {d8, d9, d10}
+; CHECK-MVE-NEXT:    vldr.16 s16, [r0]
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s0
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it vc
+; CHECK-MVE-NEXT:    movvc r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    movs r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it vc
+; CHECK-MVE-NEXT:    movvc r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmov r0, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s1
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[1], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it vc
+; CHECK-MVE-NEXT:    movvc r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s9, s5
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s1
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s18
+; CHECK-MVE-NEXT:    vmov.16 q3[2], r0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it vc
+; CHECK-MVE-NEXT:    movvc r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s5
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s2
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
+; CHECK-MVE-NEXT:    vmov.16 q3[3], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it vc
+; CHECK-MVE-NEXT:    movvc r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s10, s6
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s2
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s18
+; CHECK-MVE-NEXT:    vmov.16 q3[4], r0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it vc
+; CHECK-MVE-NEXT:    movvc r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s6
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s3
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[5], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it vc
+; CHECK-MVE-NEXT:    movvc r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s11, s7
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it vc
+; CHECK-MVE-NEXT:    movvc r1, #1
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[6], r0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q3
+; CHECK-MVE-NEXT:    vpop {d8, d9, d10}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_r_ord_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vldr.16 s12, [r0]
+; CHECK-MVEFP-NEXT:    vmov r0, s12
+; CHECK-MVEFP-NEXT:    vpt.f16 le, q0, r0
+; CHECK-MVEFP-NEXT:    vcmpt.f16 gt, q0, r0
+; CHECK-MVEFP-NEXT:    vpnot
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %src2 = load half, half* %src2p
+  %i = insertelement <8 x half> undef, half %src2, i32 0
+  %sp = shufflevector <8 x half> %i, <8 x half> undef, <8 x i32> zeroinitializer
+  %c = fcmp ord <8 x half> %sp, %src
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_r_uno_v8f16(<8 x half> %src, half* %src2p, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_r_uno_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9, d10}
+; CHECK-MVE-NEXT:    vpush {d8, d9, d10}
+; CHECK-MVE-NEXT:    vldr.16 s16, [r0]
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s0
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    movs r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmov r0, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s1
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[1], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s9, s5
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s1
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s18
+; CHECK-MVE-NEXT:    vmov.16 q3[2], r0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s5
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s2
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
+; CHECK-MVE-NEXT:    vmov.16 q3[3], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s10, s6
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s2
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s18
+; CHECK-MVE-NEXT:    vmov.16 q3[4], r0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s6
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s3
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[5], r0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s11, s7
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r1, #1
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[6], r0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q3
+; CHECK-MVE-NEXT:    vpop {d8, d9, d10}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_r_uno_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vldr.16 s12, [r0]
+; CHECK-MVEFP-NEXT:    vmov r0, s12
+; CHECK-MVEFP-NEXT:    vpt.f16 le, q0, r0
+; CHECK-MVEFP-NEXT:    vcmpt.f16 gt, q0, r0
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %src2 = load half, half* %src2p
+  %i = insertelement <8 x half> undef, half %src2, i32 0
+  %sp = shufflevector <8 x half> %i, <8 x half> undef, <8 x i32> zeroinitializer
+  %c = fcmp uno <8 x half> %sp, %src
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
diff --git a/llvm/test/CodeGen/Thumb2/mve-vcmpfz.ll b/llvm/test/CodeGen/Thumb2/mve-vcmpfz.ll
new file mode 100644
index 000000000..82ef5df34
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-vcmpfz.ll
@@ -0,0 +1,4970 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve,+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-MVE
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve.fp -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-MVEFP
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_oeq_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_oeq_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s2, #0
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s3, #0
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_oeq_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vcmp.f32 eq, q0, zr
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp oeq <4 x float> %src, zeroinitializer
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_one_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_one_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r1, #1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r2, #1
+; CHECK-MVE-NEXT:    vcmp.f32 s2, #0
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r3, #1
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s3, #0
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r0, #1
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_one_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vpt.f32 ge, q0, zr
+; CHECK-MVEFP-NEXT:    vcmpt.f32 le, q0, zr
+; CHECK-MVEFP-NEXT:    vpnot
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp one <4 x float> %src, zeroinitializer
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_ogt_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_ogt_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s2, #0
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s3, #0
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_ogt_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vcmp.f32 gt, q0, zr
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp ogt <4 x float> %src, zeroinitializer
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_oge_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_oge_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ge
+; CHECK-MVE-NEXT:    movge r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s2, #0
+; CHECK-MVE-NEXT:    it ge
+; CHECK-MVE-NEXT:    movge r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s3, #0
+; CHECK-MVE-NEXT:    it ge
+; CHECK-MVE-NEXT:    movge r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ge
+; CHECK-MVE-NEXT:    movge r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_oge_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vcmp.f32 ge, q0, zr
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp oge <4 x float> %src, zeroinitializer
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_olt_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_olt_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s2, #0
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s3, #0
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_olt_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vcmp.f32 lt, q0, zr
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp olt <4 x float> %src, zeroinitializer
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_ole_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_ole_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ls
+; CHECK-MVE-NEXT:    movls r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s2, #0
+; CHECK-MVE-NEXT:    it ls
+; CHECK-MVE-NEXT:    movls r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s3, #0
+; CHECK-MVE-NEXT:    it ls
+; CHECK-MVE-NEXT:    movls r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ls
+; CHECK-MVE-NEXT:    movls r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_ole_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vcmp.f32 le, q0, zr
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp ole <4 x float> %src, zeroinitializer
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_ueq_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_ueq_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r1, #1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r2, #1
+; CHECK-MVE-NEXT:    vcmp.f32 s2, #0
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r3, #1
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s3, #0
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r0, #1
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_ueq_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vpt.f32 ge, q0, zr
+; CHECK-MVEFP-NEXT:    vcmpt.f32 le, q0, zr
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp ueq <4 x float> %src, zeroinitializer
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_une_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_une_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ne
+; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s2, #0
+; CHECK-MVE-NEXT:    it ne
+; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s3, #0
+; CHECK-MVE-NEXT:    it ne
+; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ne
+; CHECK-MVE-NEXT:    movne r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_une_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vcmp.f32 ne, q0, zr
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp une <4 x float> %src, zeroinitializer
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_ugt_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_ugt_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it hi
+; CHECK-MVE-NEXT:    movhi r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s2, #0
+; CHECK-MVE-NEXT:    it hi
+; CHECK-MVE-NEXT:    movhi r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s3, #0
+; CHECK-MVE-NEXT:    it hi
+; CHECK-MVE-NEXT:    movhi r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it hi
+; CHECK-MVE-NEXT:    movhi r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_ugt_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vcmp.f32 le, q0, zr
+; CHECK-MVEFP-NEXT:    vpnot
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp ugt <4 x float> %src, zeroinitializer
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_uge_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_uge_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it pl
+; CHECK-MVE-NEXT:    movpl r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s2, #0
+; CHECK-MVE-NEXT:    it pl
+; CHECK-MVE-NEXT:    movpl r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s3, #0
+; CHECK-MVE-NEXT:    it pl
+; CHECK-MVE-NEXT:    movpl r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it pl
+; CHECK-MVE-NEXT:    movpl r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_uge_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vcmp.f32 lt, q0, zr
+; CHECK-MVEFP-NEXT:    vpnot
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp uge <4 x float> %src, zeroinitializer
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_ult_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_ult_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it lt
+; CHECK-MVE-NEXT:    movlt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s2, #0
+; CHECK-MVE-NEXT:    it lt
+; CHECK-MVE-NEXT:    movlt r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s3, #0
+; CHECK-MVE-NEXT:    it lt
+; CHECK-MVE-NEXT:    movlt r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it lt
+; CHECK-MVE-NEXT:    movlt r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_ult_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vcmp.f32 ge, q0, zr
+; CHECK-MVEFP-NEXT:    vpnot
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp ult <4 x float> %src, zeroinitializer
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_ule_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_ule_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it le
+; CHECK-MVE-NEXT:    movle r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s2, #0
+; CHECK-MVE-NEXT:    it le
+; CHECK-MVE-NEXT:    movle r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s3, #0
+; CHECK-MVE-NEXT:    it le
+; CHECK-MVE-NEXT:    movle r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it le
+; CHECK-MVE-NEXT:    movle r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_ule_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vcmp.f32 gt, q0, zr
+; CHECK-MVEFP-NEXT:    vpnot
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp ule <4 x float> %src, zeroinitializer
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_ord_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_ord_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s0, s0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it vc
+; CHECK-MVE-NEXT:    movvc r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s1
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s2, s2
+; CHECK-MVE-NEXT:    it vc
+; CHECK-MVE-NEXT:    movvc r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s3, s3
+; CHECK-MVE-NEXT:    it vc
+; CHECK-MVE-NEXT:    movvc r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it vc
+; CHECK-MVE-NEXT:    movvc r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_ord_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vpt.f32 ge, q0, zr
+; CHECK-MVEFP-NEXT:    vcmpt.f32 lt, q0, zr
+; CHECK-MVEFP-NEXT:    vpnot
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp ord <4 x float> %src, zeroinitializer
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_uno_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_uno_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s0, s0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s1
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s2, s2
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s3, s3
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_uno_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vpt.f32 ge, q0, zr
+; CHECK-MVEFP-NEXT:    vcmpt.f32 lt, q0, zr
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp uno <4 x float> %src, zeroinitializer
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_oeq_v8f16(<8 x half> %src, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_oeq_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9}
+; CHECK-MVE-NEXT:    vpush {d8, d9}
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s12, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s0, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmov r1, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s1, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
+; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q3[1], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s9, s5
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s1
+; CHECK-MVE-NEXT:    vcmp.f16 s16, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[2], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s5
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s2, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s10
+; CHECK-MVE-NEXT:    vmov.16 q3[3], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s10, s6
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s2
+; CHECK-MVE-NEXT:    vcmp.f16 s16, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[4], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s6
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s3, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vcmp.f16 s0, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[5], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s11, s7
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmov.16 q3[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q3
+; CHECK-MVE-NEXT:    vpop {d8, d9}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_oeq_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vcmp.f16 eq, q0, zr
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp oeq <8 x half> %src, zeroinitializer
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_one_v8f16(<8 x half> %src, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_one_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9}
+; CHECK-MVE-NEXT:    vpush {d8, d9}
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s12, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r1, #1
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vcmp.f16 s0, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r2, #1
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmov r1, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s1, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
+; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q3[1], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r1, #1
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s9, s5
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s1
+; CHECK-MVE-NEXT:    vcmp.f16 s16, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[2], r1
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r1, #1
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s5
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vcmp.f16 s2, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s10
+; CHECK-MVE-NEXT:    vmov.16 q3[3], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r1, #1
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s10, s6
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s2
+; CHECK-MVE-NEXT:    vcmp.f16 s16, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[4], r1
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r1, #1
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s6
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vcmp.f16 s3, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vcmp.f16 s0, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[5], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r1, #1
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s11, s7
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r0, #1
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmov.16 q3[6], r1
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q3
+; CHECK-MVE-NEXT:    vpop {d8, d9}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_one_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vpt.f16 ge, q0, zr
+; CHECK-MVEFP-NEXT:    vcmpt.f32 le, q0, zr
+; CHECK-MVEFP-NEXT:    vpnot
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp one <8 x half> %src, zeroinitializer
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_ogt_v8f16(<8 x half> %src, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_ogt_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9}
+; CHECK-MVE-NEXT:    vpush {d8, d9}
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s12, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s0, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmov r1, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s1, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
+; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q3[1], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s9, s5
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s1
+; CHECK-MVE-NEXT:    vcmp.f16 s16, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[2], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s5
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s2, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s10
+; CHECK-MVE-NEXT:    vmov.16 q3[3], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s10, s6
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s2
+; CHECK-MVE-NEXT:    vcmp.f16 s16, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[4], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s6
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s3, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vcmp.f16 s0, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[5], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s11, s7
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmov.16 q3[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q3
+; CHECK-MVE-NEXT:    vpop {d8, d9}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_ogt_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vcmp.f16 gt, q0, zr
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp ogt <8 x half> %src, zeroinitializer
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_oge_v8f16(<8 x half> %src, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_oge_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9}
+; CHECK-MVE-NEXT:    vpush {d8, d9}
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s12, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ge
+; CHECK-MVE-NEXT:    movge r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s0, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ge
+; CHECK-MVE-NEXT:    movge r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmov r1, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s1, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
+; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q3[1], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it ge
+; CHECK-MVE-NEXT:    movge r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s9, s5
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s1
+; CHECK-MVE-NEXT:    vcmp.f16 s16, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[2], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it ge
+; CHECK-MVE-NEXT:    movge r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s5
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s2, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s10
+; CHECK-MVE-NEXT:    vmov.16 q3[3], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it ge
+; CHECK-MVE-NEXT:    movge r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s10, s6
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s2
+; CHECK-MVE-NEXT:    vcmp.f16 s16, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[4], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it ge
+; CHECK-MVE-NEXT:    movge r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s6
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s3, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vcmp.f16 s0, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[5], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it ge
+; CHECK-MVE-NEXT:    movge r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s11, s7
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ge
+; CHECK-MVE-NEXT:    movge r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmov.16 q3[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q3
+; CHECK-MVE-NEXT:    vpop {d8, d9}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_oge_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vcmp.f16 ge, q0, zr
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp oge <8 x half> %src, zeroinitializer
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_olt_v8f16(<8 x half> %src, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_olt_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9}
+; CHECK-MVE-NEXT:    vpush {d8, d9}
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s12, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s0, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmov r1, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s1, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
+; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q3[1], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s9, s5
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s1
+; CHECK-MVE-NEXT:    vcmp.f16 s16, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[2], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s5
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s2, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s10
+; CHECK-MVE-NEXT:    vmov.16 q3[3], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s10, s6
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s2
+; CHECK-MVE-NEXT:    vcmp.f16 s16, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[4], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s6
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s3, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vcmp.f16 s0, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[5], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s11, s7
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmov.16 q3[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q3
+; CHECK-MVE-NEXT:    vpop {d8, d9}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_olt_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vcmp.f16 lt, q0, zr
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp olt <8 x half> %src, zeroinitializer
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_ole_v8f16(<8 x half> %src, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_ole_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9}
+; CHECK-MVE-NEXT:    vpush {d8, d9}
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s12, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ls
+; CHECK-MVE-NEXT:    movls r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s0, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ls
+; CHECK-MVE-NEXT:    movls r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmov r1, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s1, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
+; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q3[1], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it ls
+; CHECK-MVE-NEXT:    movls r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s9, s5
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s1
+; CHECK-MVE-NEXT:    vcmp.f16 s16, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[2], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it ls
+; CHECK-MVE-NEXT:    movls r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s5
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s2, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s10
+; CHECK-MVE-NEXT:    vmov.16 q3[3], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it ls
+; CHECK-MVE-NEXT:    movls r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s10, s6
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s2
+; CHECK-MVE-NEXT:    vcmp.f16 s16, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[4], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it ls
+; CHECK-MVE-NEXT:    movls r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s6
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s3, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vcmp.f16 s0, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[5], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it ls
+; CHECK-MVE-NEXT:    movls r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s11, s7
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ls
+; CHECK-MVE-NEXT:    movls r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmov.16 q3[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q3
+; CHECK-MVE-NEXT:    vpop {d8, d9}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_ole_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vcmp.f16 le, q0, zr
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp ole <8 x half> %src, zeroinitializer
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_ueq_v8f16(<8 x half> %src, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_ueq_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9}
+; CHECK-MVE-NEXT:    vpush {d8, d9}
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s12, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r1, #1
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vcmp.f16 s0, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r2, #1
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmov r1, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s1, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
+; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q3[1], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r1, #1
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s9, s5
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s1
+; CHECK-MVE-NEXT:    vcmp.f16 s16, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[2], r1
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r1, #1
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s5
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vcmp.f16 s2, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s10
+; CHECK-MVE-NEXT:    vmov.16 q3[3], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r1, #1
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s10, s6
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s2
+; CHECK-MVE-NEXT:    vcmp.f16 s16, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[4], r1
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r1, #1
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s6
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vcmp.f16 s3, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vcmp.f16 s0, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[5], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r1, #1
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s11, s7
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r0, #1
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmov.16 q3[6], r1
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q3
+; CHECK-MVE-NEXT:    vpop {d8, d9}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_ueq_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vpt.f16 ge, q0, zr
+; CHECK-MVEFP-NEXT:    vcmpt.f32 le, q0, zr
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp ueq <8 x half> %src, zeroinitializer
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_une_v8f16(<8 x half> %src, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_une_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9}
+; CHECK-MVE-NEXT:    vpush {d8, d9}
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s12, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ne
+; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s0, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ne
+; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmov r1, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s1, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
+; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q3[1], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it ne
+; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s9, s5
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s1
+; CHECK-MVE-NEXT:    vcmp.f16 s16, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[2], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it ne
+; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s5
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s2, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s10
+; CHECK-MVE-NEXT:    vmov.16 q3[3], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it ne
+; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s10, s6
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s2
+; CHECK-MVE-NEXT:    vcmp.f16 s16, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[4], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it ne
+; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s6
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s3, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vcmp.f16 s0, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[5], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it ne
+; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s11, s7
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ne
+; CHECK-MVE-NEXT:    movne r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmov.16 q3[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q3
+; CHECK-MVE-NEXT:    vpop {d8, d9}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_une_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vcmp.f16 ne, q0, zr
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp une <8 x half> %src, zeroinitializer
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_ugt_v8f16(<8 x half> %src, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_ugt_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9}
+; CHECK-MVE-NEXT:    vpush {d8, d9}
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s12, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it hi
+; CHECK-MVE-NEXT:    movhi r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s0, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it hi
+; CHECK-MVE-NEXT:    movhi r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmov r1, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s1, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
+; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q3[1], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it hi
+; CHECK-MVE-NEXT:    movhi r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s9, s5
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s1
+; CHECK-MVE-NEXT:    vcmp.f16 s16, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[2], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it hi
+; CHECK-MVE-NEXT:    movhi r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s5
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s2, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s10
+; CHECK-MVE-NEXT:    vmov.16 q3[3], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it hi
+; CHECK-MVE-NEXT:    movhi r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s10, s6
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s2
+; CHECK-MVE-NEXT:    vcmp.f16 s16, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[4], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it hi
+; CHECK-MVE-NEXT:    movhi r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s6
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s3, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vcmp.f16 s0, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[5], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it hi
+; CHECK-MVE-NEXT:    movhi r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s11, s7
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it hi
+; CHECK-MVE-NEXT:    movhi r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmov.16 q3[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q3
+; CHECK-MVE-NEXT:    vpop {d8, d9}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_ugt_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vcmp.f16 le, q0, zr
+; CHECK-MVEFP-NEXT:    vpnot
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp ugt <8 x half> %src, zeroinitializer
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_uge_v8f16(<8 x half> %src, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_uge_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9}
+; CHECK-MVE-NEXT:    vpush {d8, d9}
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s12, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it pl
+; CHECK-MVE-NEXT:    movpl r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s0, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it pl
+; CHECK-MVE-NEXT:    movpl r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmov r1, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s1, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
+; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q3[1], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it pl
+; CHECK-MVE-NEXT:    movpl r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s9, s5
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s1
+; CHECK-MVE-NEXT:    vcmp.f16 s16, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[2], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it pl
+; CHECK-MVE-NEXT:    movpl r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s5
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s2, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s10
+; CHECK-MVE-NEXT:    vmov.16 q3[3], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it pl
+; CHECK-MVE-NEXT:    movpl r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s10, s6
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s2
+; CHECK-MVE-NEXT:    vcmp.f16 s16, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[4], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it pl
+; CHECK-MVE-NEXT:    movpl r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s6
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s3, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vcmp.f16 s0, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[5], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it pl
+; CHECK-MVE-NEXT:    movpl r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s11, s7
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it pl
+; CHECK-MVE-NEXT:    movpl r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmov.16 q3[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q3
+; CHECK-MVE-NEXT:    vpop {d8, d9}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_uge_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vcmp.f16 lt, q0, zr
+; CHECK-MVEFP-NEXT:    vpnot
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp uge <8 x half> %src, zeroinitializer
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_ult_v8f16(<8 x half> %src, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_ult_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9}
+; CHECK-MVE-NEXT:    vpush {d8, d9}
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s12, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it lt
+; CHECK-MVE-NEXT:    movlt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s0, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it lt
+; CHECK-MVE-NEXT:    movlt r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmov r1, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s1, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
+; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q3[1], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it lt
+; CHECK-MVE-NEXT:    movlt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s9, s5
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s1
+; CHECK-MVE-NEXT:    vcmp.f16 s16, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[2], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it lt
+; CHECK-MVE-NEXT:    movlt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s5
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s2, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s10
+; CHECK-MVE-NEXT:    vmov.16 q3[3], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it lt
+; CHECK-MVE-NEXT:    movlt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s10, s6
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s2
+; CHECK-MVE-NEXT:    vcmp.f16 s16, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[4], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it lt
+; CHECK-MVE-NEXT:    movlt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s6
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s3, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vcmp.f16 s0, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[5], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it lt
+; CHECK-MVE-NEXT:    movlt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s11, s7
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it lt
+; CHECK-MVE-NEXT:    movlt r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmov.16 q3[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q3
+; CHECK-MVE-NEXT:    vpop {d8, d9}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_ult_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vcmp.f16 ge, q0, zr
+; CHECK-MVEFP-NEXT:    vpnot
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp ult <8 x half> %src, zeroinitializer
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_ule_v8f16(<8 x half> %src, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_ule_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9}
+; CHECK-MVE-NEXT:    vpush {d8, d9}
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s12, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it le
+; CHECK-MVE-NEXT:    movle r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s0, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it le
+; CHECK-MVE-NEXT:    movle r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmov r1, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s1, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
+; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q3[1], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it le
+; CHECK-MVE-NEXT:    movle r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s9, s5
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s1
+; CHECK-MVE-NEXT:    vcmp.f16 s16, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[2], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it le
+; CHECK-MVE-NEXT:    movle r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s5
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s2, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s10
+; CHECK-MVE-NEXT:    vmov.16 q3[3], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it le
+; CHECK-MVE-NEXT:    movle r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s10, s6
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s2
+; CHECK-MVE-NEXT:    vcmp.f16 s16, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[4], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it le
+; CHECK-MVE-NEXT:    movle r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s6
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s3, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vcmp.f16 s0, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[5], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it le
+; CHECK-MVE-NEXT:    movle r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s11, s7
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it le
+; CHECK-MVE-NEXT:    movle r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmov.16 q3[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q3
+; CHECK-MVE-NEXT:    vpop {d8, d9}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_ule_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vcmp.f16 gt, q0, zr
+; CHECK-MVEFP-NEXT:    vpnot
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp ule <8 x half> %src, zeroinitializer
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_ord_v8f16(<8 x half> %src, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_ord_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9}
+; CHECK-MVE-NEXT:    vpush {d8, d9}
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s12, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it vc
+; CHECK-MVE-NEXT:    movvc r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s0, s0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it vc
+; CHECK-MVE-NEXT:    movvc r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmov r1, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s1, s1
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
+; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q3[1], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it vc
+; CHECK-MVE-NEXT:    movvc r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s9, s5
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s1
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s16
+; CHECK-MVE-NEXT:    vmov.16 q3[2], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it vc
+; CHECK-MVE-NEXT:    movvc r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s5
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s2, s2
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s10
+; CHECK-MVE-NEXT:    vmov.16 q3[3], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it vc
+; CHECK-MVE-NEXT:    movvc r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s10, s6
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s2
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s16
+; CHECK-MVE-NEXT:    vmov.16 q3[4], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it vc
+; CHECK-MVE-NEXT:    movvc r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s6
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s3, s3
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vcmp.f16 s0, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[5], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it vc
+; CHECK-MVE-NEXT:    movvc r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s11, s7
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it vc
+; CHECK-MVE-NEXT:    movvc r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmov.16 q3[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q3
+; CHECK-MVE-NEXT:    vpop {d8, d9}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_ord_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vpt.f16 ge, q0, zr
+; CHECK-MVEFP-NEXT:    vcmpt.f32 lt, q0, zr
+; CHECK-MVEFP-NEXT:    vpnot
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp ord <8 x half> %src, zeroinitializer
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_uno_v8f16(<8 x half> %src, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_uno_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9}
+; CHECK-MVE-NEXT:    vpush {d8, d9}
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s12, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s0, s0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmov r1, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s1, s1
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
+; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q3[1], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s9, s5
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s1
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s16
+; CHECK-MVE-NEXT:    vmov.16 q3[2], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s5
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s2, s2
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s10
+; CHECK-MVE-NEXT:    vmov.16 q3[3], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s10, s6
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s2
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s16
+; CHECK-MVE-NEXT:    vmov.16 q3[4], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s6
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s3, s3
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vcmp.f16 s0, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[5], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s11, s7
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmov.16 q3[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q3
+; CHECK-MVE-NEXT:    vpop {d8, d9}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_uno_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vpt.f16 ge, q0, zr
+; CHECK-MVEFP-NEXT:    vcmpt.f32 lt, q0, zr
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp uno <8 x half> %src, zeroinitializer
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
+
+
+; Reversed
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_r_oeq_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_r_oeq_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s2, #0
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s3, #0
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_r_oeq_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vcmp.f32 eq, q0, zr
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp oeq <4 x float> zeroinitializer, %src
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_r_one_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_r_one_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r1, #1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r2, #1
+; CHECK-MVE-NEXT:    vcmp.f32 s2, #0
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r3, #1
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s3, #0
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r0, #1
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_r_one_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vpt.f32 le, q0, zr
+; CHECK-MVEFP-NEXT:    vcmpt.f32 ge, q0, zr
+; CHECK-MVEFP-NEXT:    vpnot
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp one <4 x float> zeroinitializer, %src
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_r_ogt_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_r_ogt_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s2, #0
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s3, #0
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_r_ogt_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vcmp.f32 lt, q0, zr
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp ogt <4 x float> zeroinitializer, %src
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_r_oge_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_r_oge_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ls
+; CHECK-MVE-NEXT:    movls r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s2, #0
+; CHECK-MVE-NEXT:    it ls
+; CHECK-MVE-NEXT:    movls r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s3, #0
+; CHECK-MVE-NEXT:    it ls
+; CHECK-MVE-NEXT:    movls r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ls
+; CHECK-MVE-NEXT:    movls r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_r_oge_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vcmp.f32 le, q0, zr
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp oge <4 x float> zeroinitializer, %src
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_r_olt_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_r_olt_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s2, #0
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s3, #0
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_r_olt_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vcmp.f32 gt, q0, zr
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp olt <4 x float> zeroinitializer, %src
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_r_ole_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_r_ole_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ge
+; CHECK-MVE-NEXT:    movge r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s2, #0
+; CHECK-MVE-NEXT:    it ge
+; CHECK-MVE-NEXT:    movge r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s3, #0
+; CHECK-MVE-NEXT:    it ge
+; CHECK-MVE-NEXT:    movge r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ge
+; CHECK-MVE-NEXT:    movge r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_r_ole_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vcmp.f32 ge, q0, zr
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp ole <4 x float> zeroinitializer, %src
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_r_ueq_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_r_ueq_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r1, #1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r2, #1
+; CHECK-MVE-NEXT:    vcmp.f32 s2, #0
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r3, #1
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s3, #0
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r0, #1
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_r_ueq_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vpt.f32 le, q0, zr
+; CHECK-MVEFP-NEXT:    vcmpt.f32 ge, q0, zr
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp ueq <4 x float> zeroinitializer, %src
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_r_une_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_r_une_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ne
+; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s2, #0
+; CHECK-MVE-NEXT:    it ne
+; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s3, #0
+; CHECK-MVE-NEXT:    it ne
+; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ne
+; CHECK-MVE-NEXT:    movne r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_r_une_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vcmp.f32 ne, q0, zr
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp une <4 x float> zeroinitializer, %src
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_r_ugt_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_r_ugt_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it lt
+; CHECK-MVE-NEXT:    movlt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s2, #0
+; CHECK-MVE-NEXT:    it lt
+; CHECK-MVE-NEXT:    movlt r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s3, #0
+; CHECK-MVE-NEXT:    it lt
+; CHECK-MVE-NEXT:    movlt r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it lt
+; CHECK-MVE-NEXT:    movlt r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_r_ugt_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vcmp.f32 ge, q0, zr
+; CHECK-MVEFP-NEXT:    vpnot
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp ugt <4 x float> zeroinitializer, %src
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_r_uge_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_r_uge_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it le
+; CHECK-MVE-NEXT:    movle r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s2, #0
+; CHECK-MVE-NEXT:    it le
+; CHECK-MVE-NEXT:    movle r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s3, #0
+; CHECK-MVE-NEXT:    it le
+; CHECK-MVE-NEXT:    movle r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it le
+; CHECK-MVE-NEXT:    movle r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_r_uge_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vcmp.f32 gt, q0, zr
+; CHECK-MVEFP-NEXT:    vpnot
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp uge <4 x float> zeroinitializer, %src
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_r_ult_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_r_ult_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it hi
+; CHECK-MVE-NEXT:    movhi r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s2, #0
+; CHECK-MVE-NEXT:    it hi
+; CHECK-MVE-NEXT:    movhi r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s3, #0
+; CHECK-MVE-NEXT:    it hi
+; CHECK-MVE-NEXT:    movhi r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it hi
+; CHECK-MVE-NEXT:    movhi r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_r_ult_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vcmp.f32 le, q0, zr
+; CHECK-MVEFP-NEXT:    vpnot
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp ult <4 x float> zeroinitializer, %src
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_r_ule_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_r_ule_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it pl
+; CHECK-MVE-NEXT:    movpl r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s2, #0
+; CHECK-MVE-NEXT:    it pl
+; CHECK-MVE-NEXT:    movpl r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s3, #0
+; CHECK-MVE-NEXT:    it pl
+; CHECK-MVE-NEXT:    movpl r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it pl
+; CHECK-MVE-NEXT:    movpl r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_r_ule_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vcmp.f32 lt, q0, zr
+; CHECK-MVEFP-NEXT:    vpnot
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp ule <4 x float> zeroinitializer, %src
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_r_ord_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_r_ord_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s0, s0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it vc
+; CHECK-MVE-NEXT:    movvc r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s1
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s2, s2
+; CHECK-MVE-NEXT:    it vc
+; CHECK-MVE-NEXT:    movvc r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s3, s3
+; CHECK-MVE-NEXT:    it vc
+; CHECK-MVE-NEXT:    movvc r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it vc
+; CHECK-MVE-NEXT:    movvc r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_r_ord_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vpt.f32 le, q0, zr
+; CHECK-MVEFP-NEXT:    vcmpt.f32 gt, q0, zr
+; CHECK-MVEFP-NEXT:    vpnot
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp ord <4 x float> zeroinitializer, %src
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+define arm_aapcs_vfpcc <4 x float> @vcmp_r_uno_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
+; CHECK-MVE-LABEL: vcmp_r_uno_v4f32:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vcmp.f32 s0, s0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s1
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s2, s2
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s3, s3
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r3, #1
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    cset r3, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_r_uno_v4f32:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vpt.f32 le, q0, zr
+; CHECK-MVEFP-NEXT:    vcmpt.f32 gt, q0, zr
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp uno <4 x float> zeroinitializer, %src
+  %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %s
+}
+
+
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_r_oeq_v8f16(<8 x half> %src, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_r_oeq_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9}
+; CHECK-MVE-NEXT:    vpush {d8, d9}
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s12, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s0, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmov r1, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s1, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
+; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q3[1], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s9, s5
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s1
+; CHECK-MVE-NEXT:    vcmp.f16 s16, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[2], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s5
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s2, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s10
+; CHECK-MVE-NEXT:    vmov.16 q3[3], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s10, s6
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s2
+; CHECK-MVE-NEXT:    vcmp.f16 s16, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[4], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s6
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s3, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vcmp.f16 s0, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[5], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s11, s7
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmov.16 q3[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q3
+; CHECK-MVE-NEXT:    vpop {d8, d9}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_r_oeq_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vcmp.f16 eq, q0, zr
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp oeq <8 x half> zeroinitializer, %src
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_r_one_v8f16(<8 x half> %src, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_r_one_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9}
+; CHECK-MVE-NEXT:    vpush {d8, d9}
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s12, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r1, #1
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vcmp.f16 s0, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r2, #1
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmov r1, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s1, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
+; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q3[1], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r1, #1
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s9, s5
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s1
+; CHECK-MVE-NEXT:    vcmp.f16 s16, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[2], r1
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r1, #1
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s5
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vcmp.f16 s2, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s10
+; CHECK-MVE-NEXT:    vmov.16 q3[3], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r1, #1
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s10, s6
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s2
+; CHECK-MVE-NEXT:    vcmp.f16 s16, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[4], r1
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r1, #1
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s6
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vcmp.f16 s3, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vcmp.f16 s0, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[5], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r1, #1
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s11, s7
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r0, #1
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmov.16 q3[6], r1
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q3
+; CHECK-MVE-NEXT:    vpop {d8, d9}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_r_one_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vpt.f16 le, q0, zr
+; CHECK-MVEFP-NEXT:    vcmpt.f32 ge, q0, zr
+; CHECK-MVEFP-NEXT:    vpnot
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp one <8 x half> zeroinitializer, %src
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_r_ogt_v8f16(<8 x half> %src, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_r_ogt_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9}
+; CHECK-MVE-NEXT:    vpush {d8, d9}
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s12, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s0, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmov r1, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s1, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
+; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q3[1], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s9, s5
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s1
+; CHECK-MVE-NEXT:    vcmp.f16 s16, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[2], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s5
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s2, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s10
+; CHECK-MVE-NEXT:    vmov.16 q3[3], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s10, s6
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s2
+; CHECK-MVE-NEXT:    vcmp.f16 s16, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[4], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s6
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s3, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vcmp.f16 s0, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[5], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s11, s7
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it mi
+; CHECK-MVE-NEXT:    movmi r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmov.16 q3[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q3
+; CHECK-MVE-NEXT:    vpop {d8, d9}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_r_ogt_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vcmp.f16 lt, q0, zr
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp ogt <8 x half> zeroinitializer, %src
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_r_oge_v8f16(<8 x half> %src, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_r_oge_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9}
+; CHECK-MVE-NEXT:    vpush {d8, d9}
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s12, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ls
+; CHECK-MVE-NEXT:    movls r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s0, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ls
+; CHECK-MVE-NEXT:    movls r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmov r1, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s1, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
+; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q3[1], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it ls
+; CHECK-MVE-NEXT:    movls r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s9, s5
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s1
+; CHECK-MVE-NEXT:    vcmp.f16 s16, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[2], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it ls
+; CHECK-MVE-NEXT:    movls r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s5
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s2, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s10
+; CHECK-MVE-NEXT:    vmov.16 q3[3], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it ls
+; CHECK-MVE-NEXT:    movls r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s10, s6
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s2
+; CHECK-MVE-NEXT:    vcmp.f16 s16, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[4], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it ls
+; CHECK-MVE-NEXT:    movls r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s6
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s3, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vcmp.f16 s0, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[5], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it ls
+; CHECK-MVE-NEXT:    movls r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s11, s7
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ls
+; CHECK-MVE-NEXT:    movls r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmov.16 q3[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q3
+; CHECK-MVE-NEXT:    vpop {d8, d9}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_r_oge_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vcmp.f16 le, q0, zr
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp oge <8 x half> zeroinitializer, %src
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_r_olt_v8f16(<8 x half> %src, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_r_olt_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9}
+; CHECK-MVE-NEXT:    vpush {d8, d9}
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s12, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s0, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmov r1, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s1, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
+; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q3[1], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s9, s5
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s1
+; CHECK-MVE-NEXT:    vcmp.f16 s16, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[2], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s5
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s2, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s10
+; CHECK-MVE-NEXT:    vmov.16 q3[3], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s10, s6
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s2
+; CHECK-MVE-NEXT:    vcmp.f16 s16, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[4], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s6
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s3, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vcmp.f16 s0, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[5], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s11, s7
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it gt
+; CHECK-MVE-NEXT:    movgt r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmov.16 q3[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q3
+; CHECK-MVE-NEXT:    vpop {d8, d9}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_r_olt_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vcmp.f16 gt, q0, zr
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp olt <8 x half> zeroinitializer, %src
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_r_ole_v8f16(<8 x half> %src, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_r_ole_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9}
+; CHECK-MVE-NEXT:    vpush {d8, d9}
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s12, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ge
+; CHECK-MVE-NEXT:    movge r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s0, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ge
+; CHECK-MVE-NEXT:    movge r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmov r1, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s1, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
+; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q3[1], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it ge
+; CHECK-MVE-NEXT:    movge r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s9, s5
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s1
+; CHECK-MVE-NEXT:    vcmp.f16 s16, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[2], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it ge
+; CHECK-MVE-NEXT:    movge r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s5
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s2, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s10
+; CHECK-MVE-NEXT:    vmov.16 q3[3], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it ge
+; CHECK-MVE-NEXT:    movge r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s10, s6
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s2
+; CHECK-MVE-NEXT:    vcmp.f16 s16, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[4], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it ge
+; CHECK-MVE-NEXT:    movge r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s6
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s3, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vcmp.f16 s0, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[5], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it ge
+; CHECK-MVE-NEXT:    movge r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s11, s7
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ge
+; CHECK-MVE-NEXT:    movge r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmov.16 q3[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q3
+; CHECK-MVE-NEXT:    vpop {d8, d9}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_r_ole_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vcmp.f16 ge, q0, zr
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp ole <8 x half> zeroinitializer, %src
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_r_ueq_v8f16(<8 x half> %src, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_r_ueq_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9}
+; CHECK-MVE-NEXT:    vpush {d8, d9}
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s12, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r1, #1
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vcmp.f16 s0, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r2, #1
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmov r1, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s1, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
+; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q3[1], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r1, #1
+; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s9, s5
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s1
+; CHECK-MVE-NEXT:    vcmp.f16 s16, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[2], r1
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r1, #1
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s5
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vcmp.f16 s2, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s10
+; CHECK-MVE-NEXT:    vmov.16 q3[3], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r1, #1
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s10, s6
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s2
+; CHECK-MVE-NEXT:    vcmp.f16 s16, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[4], r1
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r1, #1
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s6
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vcmp.f16 s3, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vcmp.f16 s0, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[5], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r1, #1
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s11, s7
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it eq
+; CHECK-MVE-NEXT:    moveq r0, #1
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmov.16 q3[6], r1
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q3
+; CHECK-MVE-NEXT:    vpop {d8, d9}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_r_ueq_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vpt.f16 le, q0, zr
+; CHECK-MVEFP-NEXT:    vcmpt.f32 ge, q0, zr
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp ueq <8 x half> zeroinitializer, %src
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_r_une_v8f16(<8 x half> %src, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_r_une_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9}
+; CHECK-MVE-NEXT:    vpush {d8, d9}
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s12, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ne
+; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s0, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ne
+; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmov r1, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s1, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
+; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q3[1], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it ne
+; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s9, s5
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s1
+; CHECK-MVE-NEXT:    vcmp.f16 s16, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[2], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it ne
+; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s5
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s2, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s10
+; CHECK-MVE-NEXT:    vmov.16 q3[3], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it ne
+; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s10, s6
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s2
+; CHECK-MVE-NEXT:    vcmp.f16 s16, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[4], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it ne
+; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s6
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s3, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vcmp.f16 s0, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[5], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it ne
+; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s11, s7
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it ne
+; CHECK-MVE-NEXT:    movne r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmov.16 q3[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q3
+; CHECK-MVE-NEXT:    vpop {d8, d9}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_r_une_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vcmp.f16 ne, q0, zr
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp une <8 x half> zeroinitializer, %src
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_r_ugt_v8f16(<8 x half> %src, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_r_ugt_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9}
+; CHECK-MVE-NEXT:    vpush {d8, d9}
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s12, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it lt
+; CHECK-MVE-NEXT:    movlt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s0, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it lt
+; CHECK-MVE-NEXT:    movlt r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmov r1, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s1, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
+; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q3[1], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it lt
+; CHECK-MVE-NEXT:    movlt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s9, s5
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s1
+; CHECK-MVE-NEXT:    vcmp.f16 s16, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[2], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it lt
+; CHECK-MVE-NEXT:    movlt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s5
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s2, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s10
+; CHECK-MVE-NEXT:    vmov.16 q3[3], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it lt
+; CHECK-MVE-NEXT:    movlt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s10, s6
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s2
+; CHECK-MVE-NEXT:    vcmp.f16 s16, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[4], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it lt
+; CHECK-MVE-NEXT:    movlt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s6
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s3, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vcmp.f16 s0, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[5], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it lt
+; CHECK-MVE-NEXT:    movlt r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s11, s7
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it lt
+; CHECK-MVE-NEXT:    movlt r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmov.16 q3[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q3
+; CHECK-MVE-NEXT:    vpop {d8, d9}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_r_ugt_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vcmp.f16 ge, q0, zr
+; CHECK-MVEFP-NEXT:    vpnot
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp ugt <8 x half> zeroinitializer, %src
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_r_uge_v8f16(<8 x half> %src, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_r_uge_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9}
+; CHECK-MVE-NEXT:    vpush {d8, d9}
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s12, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it le
+; CHECK-MVE-NEXT:    movle r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s0, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it le
+; CHECK-MVE-NEXT:    movle r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmov r1, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s1, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
+; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q3[1], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it le
+; CHECK-MVE-NEXT:    movle r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s9, s5
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s1
+; CHECK-MVE-NEXT:    vcmp.f16 s16, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[2], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it le
+; CHECK-MVE-NEXT:    movle r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s5
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s2, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s10
+; CHECK-MVE-NEXT:    vmov.16 q3[3], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it le
+; CHECK-MVE-NEXT:    movle r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s10, s6
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s2
+; CHECK-MVE-NEXT:    vcmp.f16 s16, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[4], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it le
+; CHECK-MVE-NEXT:    movle r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s6
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s3, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vcmp.f16 s0, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[5], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it le
+; CHECK-MVE-NEXT:    movle r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s11, s7
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it le
+; CHECK-MVE-NEXT:    movle r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmov.16 q3[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q3
+; CHECK-MVE-NEXT:    vpop {d8, d9}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_r_uge_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vcmp.f16 gt, q0, zr
+; CHECK-MVEFP-NEXT:    vpnot
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp uge <8 x half> zeroinitializer, %src
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_r_ult_v8f16(<8 x half> %src, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_r_ult_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9}
+; CHECK-MVE-NEXT:    vpush {d8, d9}
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s12, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it hi
+; CHECK-MVE-NEXT:    movhi r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s0, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it hi
+; CHECK-MVE-NEXT:    movhi r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmov r1, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s1, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
+; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q3[1], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it hi
+; CHECK-MVE-NEXT:    movhi r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s9, s5
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s1
+; CHECK-MVE-NEXT:    vcmp.f16 s16, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[2], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it hi
+; CHECK-MVE-NEXT:    movhi r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s5
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s2, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s10
+; CHECK-MVE-NEXT:    vmov.16 q3[3], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it hi
+; CHECK-MVE-NEXT:    movhi r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s10, s6
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s2
+; CHECK-MVE-NEXT:    vcmp.f16 s16, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[4], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it hi
+; CHECK-MVE-NEXT:    movhi r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s6
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s3, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vcmp.f16 s0, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[5], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it hi
+; CHECK-MVE-NEXT:    movhi r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s11, s7
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it hi
+; CHECK-MVE-NEXT:    movhi r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmov.16 q3[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q3
+; CHECK-MVE-NEXT:    vpop {d8, d9}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_r_ult_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vcmp.f16 le, q0, zr
+; CHECK-MVEFP-NEXT:    vpnot
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp ult <8 x half> zeroinitializer, %src
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_r_ule_v8f16(<8 x half> %src, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_r_ule_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9}
+; CHECK-MVE-NEXT:    vpush {d8, d9}
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s12, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it pl
+; CHECK-MVE-NEXT:    movpl r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s0, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it pl
+; CHECK-MVE-NEXT:    movpl r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmov r1, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s1, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
+; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q3[1], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it pl
+; CHECK-MVE-NEXT:    movpl r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s9, s5
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s1
+; CHECK-MVE-NEXT:    vcmp.f16 s16, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[2], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it pl
+; CHECK-MVE-NEXT:    movpl r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s5
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s2, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s10
+; CHECK-MVE-NEXT:    vmov.16 q3[3], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it pl
+; CHECK-MVE-NEXT:    movpl r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s10, s6
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s2
+; CHECK-MVE-NEXT:    vcmp.f16 s16, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[4], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it pl
+; CHECK-MVE-NEXT:    movpl r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s6
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s3, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vcmp.f16 s0, #0
+; CHECK-MVE-NEXT:    vmov.16 q3[5], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it pl
+; CHECK-MVE-NEXT:    movpl r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s11, s7
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it pl
+; CHECK-MVE-NEXT:    movpl r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmov.16 q3[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q3
+; CHECK-MVE-NEXT:    vpop {d8, d9}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_r_ule_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vcmp.f16 lt, q0, zr
+; CHECK-MVEFP-NEXT:    vpnot
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp ule <8 x half> zeroinitializer, %src
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_r_ord_v8f16(<8 x half> %src, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_r_ord_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9}
+; CHECK-MVE-NEXT:    vpush {d8, d9}
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s12, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it vc
+; CHECK-MVE-NEXT:    movvc r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s0, s0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it vc
+; CHECK-MVE-NEXT:    movvc r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmov r1, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s1, s1
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
+; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q3[1], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it vc
+; CHECK-MVE-NEXT:    movvc r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s9, s5
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s1
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s16
+; CHECK-MVE-NEXT:    vmov.16 q3[2], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it vc
+; CHECK-MVE-NEXT:    movvc r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s5
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s2, s2
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s10
+; CHECK-MVE-NEXT:    vmov.16 q3[3], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it vc
+; CHECK-MVE-NEXT:    movvc r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s10, s6
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s2
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s16
+; CHECK-MVE-NEXT:    vmov.16 q3[4], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it vc
+; CHECK-MVE-NEXT:    movvc r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s6
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s3, s3
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vcmp.f16 s0, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[5], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it vc
+; CHECK-MVE-NEXT:    movvc r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s11, s7
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it vc
+; CHECK-MVE-NEXT:    movvc r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmov.16 q3[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q3
+; CHECK-MVE-NEXT:    vpop {d8, d9}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_r_ord_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vpt.f16 le, q0, zr
+; CHECK-MVEFP-NEXT:    vcmpt.f32 gt, q0, zr
+; CHECK-MVEFP-NEXT:    vpnot
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp ord <8 x half> zeroinitializer, %src
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
+
+define arm_aapcs_vfpcc <8 x half> @vcmp_r_uno_v8f16(<8 x half> %src, <8 x half> %a, <8 x half> %b) {
+; CHECK-MVE-LABEL: vcmp_r_uno_v8f16:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    .vsave {d8, d9}
+; CHECK-MVE-NEXT:    vpush {d8, d9}
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s0
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s12, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s12, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vcmp.f16 s0, s0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    mov.w r2, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r2, #1
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    cset r2, ne
+; CHECK-MVE-NEXT:    vmov r1, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s1, s1
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
+; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
+; CHECK-MVE-NEXT:    vmov.16 q3[1], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s9, s5
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s1
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s16
+; CHECK-MVE-NEXT:    vmov.16 q3[2], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s5
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s2, s2
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s10
+; CHECK-MVE-NEXT:    vmov.16 q3[3], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s10, s6
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s2
+; CHECK-MVE-NEXT:    vcmp.f16 s16, s16
+; CHECK-MVE-NEXT:    vmov.16 q3[4], r1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s16, s6
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s3, s3
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vcmp.f16 s0, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[5], r1
+; CHECK-MVE-NEXT:    mov.w r1, #0
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r1, #1
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    cset r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s11, s7
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    it vs
+; CHECK-MVE-NEXT:    movvs r0, #1
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cset r0, ne
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmov.16 q3[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    vmov r0, s0
+; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
+; CHECK-MVE-NEXT:    vmov q0, q3
+; CHECK-MVE-NEXT:    vpop {d8, d9}
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: vcmp_r_uno_v8f16:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vpt.f16 le, q0, zr
+; CHECK-MVEFP-NEXT:    vcmpt.f32 gt, q0, zr
+; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %c = fcmp uno <8 x half> zeroinitializer, %src
+  %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
+  ret <8 x half> %s
+}
diff --git a/llvm/test/CodeGen/Thumb2/mve-vcmpr.ll b/llvm/test/CodeGen/Thumb2/mve-vcmpr.ll
new file mode 100644
index 000000000..c832c241e
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-vcmpr.ll
@@ -0,0 +1,1186 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s
+
+define arm_aapcs_vfpcc <4 x i32> @vcmp_eq_v4i32(<4 x i32> %src, i32 %src2, <4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: vcmp_eq_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i32 eq, q0, r0
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <4 x i32> undef, i32 %src2, i32 0
+  %sp = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %c = icmp eq <4 x i32> %src, %sp
+  %s = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @vcmp_ne_v4i32(<4 x i32> %src, i32 %src2, <4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: vcmp_ne_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i32 ne, q0, r0
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <4 x i32> undef, i32 %src2, i32 0
+  %sp = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %c = icmp ne <4 x i32> %src, %sp
+  %s = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @vcmp_sgt_v4i32(<4 x i32> %src, i32 %src2, <4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: vcmp_sgt_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.s32 gt, q0, r0
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <4 x i32> undef, i32 %src2, i32 0
+  %sp = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %c = icmp sgt <4 x i32> %src, %sp
+  %s = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @vcmp_sge_v4i32(<4 x i32> %src, i32 %src2, <4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: vcmp_sge_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.s32 ge, q0, r0
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <4 x i32> undef, i32 %src2, i32 0
+  %sp = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %c = icmp sge <4 x i32> %src, %sp
+  %s = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @vcmp_slt_v4i32(<4 x i32> %src, i32 %src2, <4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: vcmp_slt_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.s32 lt, q0, r0
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <4 x i32> undef, i32 %src2, i32 0
+  %sp = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %c = icmp slt <4 x i32> %src, %sp
+  %s = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @vcmp_sle_v4i32(<4 x i32> %src, i32 %src2, <4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: vcmp_sle_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.s32 le, q0, r0
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <4 x i32> undef, i32 %src2, i32 0
+  %sp = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %c = icmp sle <4 x i32> %src, %sp
+  %s = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @vcmp_ugt_v4i32(<4 x i32> %src, i32 %src2, <4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: vcmp_ugt_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.u32 hi, q0, r0
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <4 x i32> undef, i32 %src2, i32 0
+  %sp = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %c = icmp ugt <4 x i32> %src, %sp
+  %s = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @vcmp_uge_v4i32(<4 x i32> %src, i32 %src2, <4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: vcmp_uge_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.u32 cs, q0, r0
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <4 x i32> undef, i32 %src2, i32 0
+  %sp = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %c = icmp uge <4 x i32> %src, %sp
+  %s = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @vcmp_ult_v4i32(<4 x i32> %src, i32 %src2, <4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: vcmp_ult_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdup.32 q3, r0
+; CHECK-NEXT:    vcmp.u32 hi, q3, q0
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <4 x i32> undef, i32 %src2, i32 0
+  %sp = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %c = icmp ult <4 x i32> %src, %sp
+  %s = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @vcmp_ule_v4i32(<4 x i32> %src, i32 %src2, <4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: vcmp_ule_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdup.32 q3, r0
+; CHECK-NEXT:    vcmp.u32 cs, q3, q0
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <4 x i32> undef, i32 %src2, i32 0
+  %sp = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %c = icmp ule <4 x i32> %src, %sp
+  %s = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+
+define arm_aapcs_vfpcc <8 x i16> @vcmp_eq_v8i16(<8 x i16> %src, i16 %src2, <8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: vcmp_eq_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i16 eq, q0, r0
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <8 x i16> undef, i16 %src2, i32 0
+  %sp = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
+  %c = icmp eq <8 x i16> %src, %sp
+  %s = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+define arm_aapcs_vfpcc <8 x i16> @vcmp_ne_v8i16(<8 x i16> %src, i16 %src2, <8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: vcmp_ne_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i16 ne, q0, r0
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <8 x i16> undef, i16 %src2, i32 0
+  %sp = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
+  %c = icmp ne <8 x i16> %src, %sp
+  %s = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+define arm_aapcs_vfpcc <8 x i16> @vcmp_sgt_v8i16(<8 x i16> %src, i16 %src2, <8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: vcmp_sgt_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.s16 gt, q0, r0
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <8 x i16> undef, i16 %src2, i32 0
+  %sp = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
+  %c = icmp sgt <8 x i16> %src, %sp
+  %s = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+define arm_aapcs_vfpcc <8 x i16> @vcmp_sge_v8i16(<8 x i16> %src, i16 %src2, <8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: vcmp_sge_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.s16 ge, q0, r0
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <8 x i16> undef, i16 %src2, i32 0
+  %sp = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
+  %c = icmp sge <8 x i16> %src, %sp
+  %s = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+define arm_aapcs_vfpcc <8 x i16> @vcmp_slt_v8i16(<8 x i16> %src, i16 %src2, <8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: vcmp_slt_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.s16 lt, q0, r0
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <8 x i16> undef, i16 %src2, i32 0
+  %sp = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
+  %c = icmp slt <8 x i16> %src, %sp
+  %s = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+define arm_aapcs_vfpcc <8 x i16> @vcmp_sle_v8i16(<8 x i16> %src, i16 %src2, <8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: vcmp_sle_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.s16 le, q0, r0
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <8 x i16> undef, i16 %src2, i32 0
+  %sp = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
+  %c = icmp sle <8 x i16> %src, %sp
+  %s = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+define arm_aapcs_vfpcc <8 x i16> @vcmp_ugt_v8i16(<8 x i16> %src, i16 %src2, <8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: vcmp_ugt_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.u16 hi, q0, r0
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <8 x i16> undef, i16 %src2, i32 0
+  %sp = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
+  %c = icmp ugt <8 x i16> %src, %sp
+  %s = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+define arm_aapcs_vfpcc <8 x i16> @vcmp_uge_v8i16(<8 x i16> %src, i16 %src2, <8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: vcmp_uge_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.u16 cs, q0, r0
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <8 x i16> undef, i16 %src2, i32 0
+  %sp = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
+  %c = icmp uge <8 x i16> %src, %sp
+  %s = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+define arm_aapcs_vfpcc <8 x i16> @vcmp_ult_v8i16(<8 x i16> %src, i16 %src2, <8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: vcmp_ult_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdup.16 q3, r0
+; CHECK-NEXT:    vcmp.u16 hi, q3, q0
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <8 x i16> undef, i16 %src2, i32 0
+  %sp = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
+  %c = icmp ult <8 x i16> %src, %sp
+  %s = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+define arm_aapcs_vfpcc <8 x i16> @vcmp_ule_v8i16(<8 x i16> %src, i16 %src2, <8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: vcmp_ule_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdup.16 q3, r0
+; CHECK-NEXT:    vcmp.u16 cs, q3, q0
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <8 x i16> undef, i16 %src2, i32 0
+  %sp = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
+  %c = icmp ule <8 x i16> %src, %sp
+  %s = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+
+define arm_aapcs_vfpcc <16 x i8> @vcmp_eq_v16i8(<16 x i8> %src, i8 %src2, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: vcmp_eq_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i8 eq, q0, r0
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <16 x i8> undef, i8 %src2, i32 0
+  %sp = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
+  %c = icmp eq <16 x i8> %src, %sp
+  %s = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %s
+}
+
+define arm_aapcs_vfpcc <16 x i8> @vcmp_ne_v16i8(<16 x i8> %src, i8 %src2, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: vcmp_ne_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i8 ne, q0, r0
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <16 x i8> undef, i8 %src2, i32 0
+  %sp = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
+  %c = icmp ne <16 x i8> %src, %sp
+  %s = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %s
+}
+
+define arm_aapcs_vfpcc <16 x i8> @vcmp_sgt_v16i8(<16 x i8> %src, i8 %src2, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: vcmp_sgt_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.s8 gt, q0, r0
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <16 x i8> undef, i8 %src2, i32 0
+  %sp = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
+  %c = icmp sgt <16 x i8> %src, %sp
+  %s = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %s
+}
+
+define arm_aapcs_vfpcc <16 x i8> @vcmp_sge_v16i8(<16 x i8> %src, i8 %src2, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: vcmp_sge_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.s8 ge, q0, r0
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <16 x i8> undef, i8 %src2, i32 0
+  %sp = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
+  %c = icmp sge <16 x i8> %src, %sp
+  %s = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %s
+}
+
+define arm_aapcs_vfpcc <16 x i8> @vcmp_slt_v16i8(<16 x i8> %src, i8 %src2, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: vcmp_slt_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.s8 lt, q0, r0
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <16 x i8> undef, i8 %src2, i32 0
+  %sp = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
+  %c = icmp slt <16 x i8> %src, %sp
+  %s = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %s
+}
+
+define arm_aapcs_vfpcc <16 x i8> @vcmp_sle_v16i8(<16 x i8> %src, i8 %src2, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: vcmp_sle_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.s8 le, q0, r0
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <16 x i8> undef, i8 %src2, i32 0
+  %sp = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
+  %c = icmp sle <16 x i8> %src, %sp
+  %s = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %s
+}
+
+define arm_aapcs_vfpcc <16 x i8> @vcmp_ugt_v16i8(<16 x i8> %src, i8 %src2, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: vcmp_ugt_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.u8 hi, q0, r0
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <16 x i8> undef, i8 %src2, i32 0
+  %sp = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
+  %c = icmp ugt <16 x i8> %src, %sp
+  %s = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %s
+}
+
+define arm_aapcs_vfpcc <16 x i8> @vcmp_uge_v16i8(<16 x i8> %src, i8 %src2, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: vcmp_uge_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.u8 cs, q0, r0
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <16 x i8> undef, i8 %src2, i32 0
+  %sp = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
+  %c = icmp uge <16 x i8> %src, %sp
+  %s = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %s
+}
+
+define arm_aapcs_vfpcc <16 x i8> @vcmp_ult_v16i8(<16 x i8> %src, i8 %src2, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: vcmp_ult_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdup.8 q3, r0
+; CHECK-NEXT:    vcmp.u8 hi, q3, q0
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <16 x i8> undef, i8 %src2, i32 0
+  %sp = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
+  %c = icmp ult <16 x i8> %src, %sp
+  %s = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %s
+}
+
+define arm_aapcs_vfpcc <16 x i8> @vcmp_ule_v16i8(<16 x i8> %src, i8 %src2, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: vcmp_ule_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdup.8 q3, r0
+; CHECK-NEXT:    vcmp.u8 cs, q3, q0
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <16 x i8> undef, i8 %src2, i32 0
+  %sp = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
+  %c = icmp ule <16 x i8> %src, %sp
+  %s = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %s
+}
+
+
+define arm_aapcs_vfpcc <2 x i64> @vcmp_eq_v2i64(<2 x i64> %src, i64 %src2, <2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: vcmp_eq_v2i64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov r2, s1
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    eors r2, r1
+; CHECK-NEXT:    eors r3, r0
+; CHECK-NEXT:    orrs r2, r3
+; CHECK-NEXT:    cset r2, eq
+; CHECK-NEXT:    tst.w r2, #1
+; CHECK-NEXT:    csetm r2, ne
+; CHECK-NEXT:    vmov.32 q3[0], r2
+; CHECK-NEXT:    vmov.32 q3[1], r2
+; CHECK-NEXT:    vmov r2, s3
+; CHECK-NEXT:    eors r1, r2
+; CHECK-NEXT:    vmov r2, s2
+; CHECK-NEXT:    eors r0, r2
+; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q3[2], r0
+; CHECK-NEXT:    vmov.32 q3[3], r0
+; CHECK-NEXT:    vbic q0, q2, q3
+; CHECK-NEXT:    vand q1, q1, q3
+; CHECK-NEXT:    vorr q0, q1, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <2 x i64> undef, i64 %src2, i32 0
+  %sp = shufflevector <2 x i64> %i, <2 x i64> undef, <2 x i32> zeroinitializer
+  %c = icmp eq <2 x i64> %src, %sp
+  %s = select <2 x i1> %c, <2 x i64> %a, <2 x i64> %b
+  ret <2 x i64> %s
+}
+
+define arm_aapcs_vfpcc <2 x i32> @vcmp_eq_v2i32(<2 x i64> %src, i64 %src2, <2 x i32> %a, <2 x i32> %b) {
+; CHECK-LABEL: vcmp_eq_v2i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov r2, s1
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    eors r2, r1
+; CHECK-NEXT:    eors r3, r0
+; CHECK-NEXT:    orrs r2, r3
+; CHECK-NEXT:    cset r2, eq
+; CHECK-NEXT:    tst.w r2, #1
+; CHECK-NEXT:    csetm r2, ne
+; CHECK-NEXT:    vmov.32 q3[0], r2
+; CHECK-NEXT:    vmov.32 q3[1], r2
+; CHECK-NEXT:    vmov r2, s3
+; CHECK-NEXT:    eors r1, r2
+; CHECK-NEXT:    vmov r2, s2
+; CHECK-NEXT:    eors r0, r2
+; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q3[2], r0
+; CHECK-NEXT:    vmov.32 q3[3], r0
+; CHECK-NEXT:    vbic q0, q2, q3
+; CHECK-NEXT:    vand q1, q1, q3
+; CHECK-NEXT:    vorr q0, q1, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <2 x i64> undef, i64 %src2, i32 0
+  %sp = shufflevector <2 x i64> %i, <2 x i64> undef, <2 x i32> zeroinitializer
+  %c = icmp eq <2 x i64> %src, %sp
+  %s = select <2 x i1> %c, <2 x i32> %a, <2 x i32> %b
+  ret <2 x i32> %s
+}
+
+define arm_aapcs_vfpcc <2 x i32> @vcmp_multi_v2i32(<2 x i64> %a, <2 x i32> %b, <2 x i32> %c) {
+; CHECK-LABEL: vcmp_multi_v2i32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11}
+; CHECK-NEXT:    vmov r0, s1
+; CHECK-NEXT:    movs r3, #0
+; CHECK-NEXT:    vmov r1, s0
+; CHECK-NEXT:    vmov r2, s8
+; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    vmov r1, s2
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q3[0], r0
+; CHECK-NEXT:    vmov.32 q3[1], r0
+; CHECK-NEXT:    vmov r0, s3
+; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q3[2], r0
+; CHECK-NEXT:    vmov.32 q3[3], r0
+; CHECK-NEXT:    vbic q0, q2, q3
+; CHECK-NEXT:    vmov lr, s0
+; CHECK-NEXT:    subs.w r1, lr, r2
+; CHECK-NEXT:    asr.w r12, lr, #31
+; CHECK-NEXT:    sbcs.w r1, r12, r2, asr #31
+; CHECK-NEXT:    mov.w r1, #0
+; CHECK-NEXT:    vmov r2, s10
+; CHECK-NEXT:    it lt
+; CHECK-NEXT:    movlt r1, #1
+; CHECK-NEXT:    cmp r1, #0
+; CHECK-NEXT:    csetm r1, ne
+; CHECK-NEXT:    vmov.32 q3[0], r1
+; CHECK-NEXT:    vmov.32 q3[1], r1
+; CHECK-NEXT:    vmov r1, s2
+; CHECK-NEXT:    subs r0, r1, r2
+; CHECK-NEXT:    asr.w r12, r1, #31
+; CHECK-NEXT:    sbcs.w r0, r12, r2, asr #31
+; CHECK-NEXT:    it lt
+; CHECK-NEXT:    movlt r3, #1
+; CHECK-NEXT:    cmp r3, #0
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    cmp.w lr, #0
+; CHECK-NEXT:    vmov.32 q3[2], r0
+; CHECK-NEXT:    vmov.32 q3[3], r0
+; CHECK-NEXT:    cset r0, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    cmp r1, #0
+; CHECK-NEXT:    vmov.32 q4[0], r0
+; CHECK-NEXT:    vmov.32 q4[1], r0
+; CHECK-NEXT:    cset r0, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q4[2], r0
+; CHECK-NEXT:    vmov.32 q4[3], r0
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    cset r0, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q5[0], r0
+; CHECK-NEXT:    vmov.32 q5[1], r0
+; CHECK-NEXT:    vmov r0, s6
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    cset r0, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q5[2], r0
+; CHECK-NEXT:    vmov.32 q5[3], r0
+; CHECK-NEXT:    vand q1, q5, q4
+; CHECK-NEXT:    vand q1, q3, q1
+; CHECK-NEXT:    vbic q0, q0, q1
+; CHECK-NEXT:    vand q1, q2, q1
+; CHECK-NEXT:    vorr q0, q1, q0
+; CHECK-NEXT:    vpop {d8, d9, d10, d11}
+; CHECK-NEXT:    pop {r7, pc}
+  %a4 = icmp eq <2 x i64> %a, zeroinitializer
+  %a5 = select <2 x i1> %a4, <2 x i32> zeroinitializer, <2 x i32> %c
+  %a6 = icmp ne <2 x i32> %b, zeroinitializer
+  %a7 = icmp slt <2 x i32> %a5, %c
+  %a8 = icmp ne <2 x i32> %a5, zeroinitializer
+  %a9 = and <2 x i1> %a6, %a8
+  %a10 = and <2 x i1> %a7, %a9
+  %a11 = select <2 x i1> %a10, <2 x i32> %c, <2 x i32> %a5
+  ret <2 x i32> %a11
+}
+
+; Reversed
+
+define arm_aapcs_vfpcc <4 x i32> @vcmp_r_eq_v4i32(<4 x i32> %src, i32 %src2, <4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: vcmp_r_eq_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i32 eq, q0, r0
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <4 x i32> undef, i32 %src2, i32 0
+  %sp = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %c = icmp eq <4 x i32> %sp, %src
+  %s = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @vcmp_r_ne_v4i32(<4 x i32> %src, i32 %src2, <4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: vcmp_r_ne_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i32 ne, q0, r0
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <4 x i32> undef, i32 %src2, i32 0
+  %sp = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %c = icmp ne <4 x i32> %sp, %src
+  %s = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @vcmp_r_sgt_v4i32(<4 x i32> %src, i32 %src2, <4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: vcmp_r_sgt_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.s32 lt, q0, r0
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <4 x i32> undef, i32 %src2, i32 0
+  %sp = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %c = icmp sgt <4 x i32> %sp, %src
+  %s = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @vcmp_r_sge_v4i32(<4 x i32> %src, i32 %src2, <4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: vcmp_r_sge_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.s32 le, q0, r0
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <4 x i32> undef, i32 %src2, i32 0
+  %sp = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %c = icmp sge <4 x i32> %sp, %src
+  %s = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @vcmp_r_slt_v4i32(<4 x i32> %src, i32 %src2, <4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: vcmp_r_slt_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.s32 gt, q0, r0
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <4 x i32> undef, i32 %src2, i32 0
+  %sp = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %c = icmp slt <4 x i32> %sp, %src
+  %s = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @vcmp_r_sle_v4i32(<4 x i32> %src, i32 %src2, <4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: vcmp_r_sle_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.s32 ge, q0, r0
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <4 x i32> undef, i32 %src2, i32 0
+  %sp = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %c = icmp sle <4 x i32> %sp, %src
+  %s = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @vcmp_r_ugt_v4i32(<4 x i32> %src, i32 %src2, <4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: vcmp_r_ugt_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdup.32 q3, r0
+; CHECK-NEXT:    vcmp.u32 hi, q3, q0
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <4 x i32> undef, i32 %src2, i32 0
+  %sp = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %c = icmp ugt <4 x i32> %sp, %src
+  %s = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @vcmp_r_uge_v4i32(<4 x i32> %src, i32 %src2, <4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: vcmp_r_uge_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdup.32 q3, r0
+; CHECK-NEXT:    vcmp.u32 cs, q3, q0
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <4 x i32> undef, i32 %src2, i32 0
+  %sp = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %c = icmp uge <4 x i32> %sp, %src
+  %s = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @vcmp_r_ult_v4i32(<4 x i32> %src, i32 %src2, <4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: vcmp_r_ult_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.u32 hi, q0, r0
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <4 x i32> undef, i32 %src2, i32 0
+  %sp = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %c = icmp ult <4 x i32> %sp, %src
+  %s = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @vcmp_r_ule_v4i32(<4 x i32> %src, i32 %src2, <4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: vcmp_r_ule_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.u32 cs, q0, r0
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <4 x i32> undef, i32 %src2, i32 0
+  %sp = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %c = icmp ule <4 x i32> %sp, %src
+  %s = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+
+define arm_aapcs_vfpcc <8 x i16> @vcmp_r_eq_v8i16(<8 x i16> %src, i16 %src2, <8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: vcmp_r_eq_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i16 eq, q0, r0
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <8 x i16> undef, i16 %src2, i32 0
+  %sp = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
+  %c = icmp eq <8 x i16> %sp, %src
+  %s = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+define arm_aapcs_vfpcc <8 x i16> @vcmp_r_ne_v8i16(<8 x i16> %src, i16 %src2, <8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: vcmp_r_ne_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i16 ne, q0, r0
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <8 x i16> undef, i16 %src2, i32 0
+  %sp = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
+  %c = icmp ne <8 x i16> %sp, %src
+  %s = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+define arm_aapcs_vfpcc <8 x i16> @vcmp_r_sgt_v8i16(<8 x i16> %src, i16 %src2, <8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: vcmp_r_sgt_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.s16 lt, q0, r0
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <8 x i16> undef, i16 %src2, i32 0
+  %sp = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
+  %c = icmp sgt <8 x i16> %sp, %src
+  %s = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+define arm_aapcs_vfpcc <8 x i16> @vcmp_r_sge_v8i16(<8 x i16> %src, i16 %src2, <8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: vcmp_r_sge_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.s16 le, q0, r0
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <8 x i16> undef, i16 %src2, i32 0
+  %sp = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
+  %c = icmp sge <8 x i16> %sp, %src
+  %s = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+define arm_aapcs_vfpcc <8 x i16> @vcmp_r_slt_v8i16(<8 x i16> %src, i16 %src2, <8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: vcmp_r_slt_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.s16 gt, q0, r0
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <8 x i16> undef, i16 %src2, i32 0
+  %sp = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
+  %c = icmp slt <8 x i16> %sp, %src
+  %s = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+define arm_aapcs_vfpcc <8 x i16> @vcmp_r_sle_v8i16(<8 x i16> %src, i16 %src2, <8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: vcmp_r_sle_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.s16 ge, q0, r0
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <8 x i16> undef, i16 %src2, i32 0
+  %sp = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
+  %c = icmp sle <8 x i16> %sp, %src
+  %s = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+define arm_aapcs_vfpcc <8 x i16> @vcmp_r_ugt_v8i16(<8 x i16> %src, i16 %src2, <8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: vcmp_r_ugt_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdup.16 q3, r0
+; CHECK-NEXT:    vcmp.u16 hi, q3, q0
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <8 x i16> undef, i16 %src2, i32 0
+  %sp = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
+  %c = icmp ugt <8 x i16> %sp, %src
+  %s = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+define arm_aapcs_vfpcc <8 x i16> @vcmp_r_uge_v8i16(<8 x i16> %src, i16 %src2, <8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: vcmp_r_uge_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdup.16 q3, r0
+; CHECK-NEXT:    vcmp.u16 cs, q3, q0
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <8 x i16> undef, i16 %src2, i32 0
+  %sp = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
+  %c = icmp uge <8 x i16> %sp, %src
+  %s = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+define arm_aapcs_vfpcc <8 x i16> @vcmp_r_ult_v8i16(<8 x i16> %src, i16 %src2, <8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: vcmp_r_ult_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.u16 hi, q0, r0
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <8 x i16> undef, i16 %src2, i32 0
+  %sp = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
+  %c = icmp ult <8 x i16> %sp, %src
+  %s = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+define arm_aapcs_vfpcc <8 x i16> @vcmp_r_ule_v8i16(<8 x i16> %src, i16 %src2, <8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: vcmp_r_ule_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.u16 cs, q0, r0
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <8 x i16> undef, i16 %src2, i32 0
+  %sp = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
+  %c = icmp ule <8 x i16> %sp, %src
+  %s = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+
+define arm_aapcs_vfpcc <16 x i8> @vcmp_r_eq_v16i8(<16 x i8> %src, i8 %src2, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: vcmp_r_eq_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i8 eq, q0, r0
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <16 x i8> undef, i8 %src2, i32 0
+  %sp = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
+  %c = icmp eq <16 x i8> %sp, %src
+  %s = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %s
+}
+
+define arm_aapcs_vfpcc <16 x i8> @vcmp_r_ne_v16i8(<16 x i8> %src, i8 %src2, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: vcmp_r_ne_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i8 ne, q0, r0
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <16 x i8> undef, i8 %src2, i32 0
+  %sp = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
+  %c = icmp ne <16 x i8> %sp, %src
+  %s = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %s
+}
+
+define arm_aapcs_vfpcc <16 x i8> @vcmp_r_sgt_v16i8(<16 x i8> %src, i8 %src2, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: vcmp_r_sgt_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.s8 lt, q0, r0
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <16 x i8> undef, i8 %src2, i32 0
+  %sp = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
+  %c = icmp sgt <16 x i8> %sp, %src
+  %s = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %s
+}
+
+define arm_aapcs_vfpcc <16 x i8> @vcmp_r_sge_v16i8(<16 x i8> %src, i8 %src2, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: vcmp_r_sge_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.s8 le, q0, r0
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <16 x i8> undef, i8 %src2, i32 0
+  %sp = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
+  %c = icmp sge <16 x i8> %sp, %src
+  %s = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %s
+}
+
+define arm_aapcs_vfpcc <16 x i8> @vcmp_r_slt_v16i8(<16 x i8> %src, i8 %src2, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: vcmp_r_slt_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.s8 gt, q0, r0
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <16 x i8> undef, i8 %src2, i32 0
+  %sp = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
+  %c = icmp slt <16 x i8> %sp, %src
+  %s = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %s
+}
+
+define arm_aapcs_vfpcc <16 x i8> @vcmp_r_sle_v16i8(<16 x i8> %src, i8 %src2, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: vcmp_r_sle_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.s8 ge, q0, r0
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <16 x i8> undef, i8 %src2, i32 0
+  %sp = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
+  %c = icmp sle <16 x i8> %sp, %src
+  %s = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %s
+}
+
+define arm_aapcs_vfpcc <16 x i8> @vcmp_r_ugt_v16i8(<16 x i8> %src, i8 %src2, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: vcmp_r_ugt_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdup.8 q3, r0
+; CHECK-NEXT:    vcmp.u8 hi, q3, q0
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <16 x i8> undef, i8 %src2, i32 0
+  %sp = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
+  %c = icmp ugt <16 x i8> %sp, %src
+  %s = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %s
+}
+
+define arm_aapcs_vfpcc <16 x i8> @vcmp_r_uge_v16i8(<16 x i8> %src, i8 %src2, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: vcmp_r_uge_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdup.8 q3, r0
+; CHECK-NEXT:    vcmp.u8 cs, q3, q0
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <16 x i8> undef, i8 %src2, i32 0
+  %sp = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
+  %c = icmp uge <16 x i8> %sp, %src
+  %s = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %s
+}
+
+define arm_aapcs_vfpcc <16 x i8> @vcmp_r_ult_v16i8(<16 x i8> %src, i8 %src2, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: vcmp_r_ult_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.u8 hi, q0, r0
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <16 x i8> undef, i8 %src2, i32 0
+  %sp = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
+  %c = icmp ult <16 x i8> %sp, %src
+  %s = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %s
+}
+
+define arm_aapcs_vfpcc <16 x i8> @vcmp_r_ule_v16i8(<16 x i8> %src, i8 %src2, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: vcmp_r_ule_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.u8 cs, q0, r0
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <16 x i8> undef, i8 %src2, i32 0
+  %sp = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
+  %c = icmp ule <16 x i8> %sp, %src
+  %s = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %s
+}
+
+
+define arm_aapcs_vfpcc <2 x i64> @vcmp_r_eq_v2i64(<2 x i64> %src, i64 %src2, <2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: vcmp_r_eq_v2i64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov r2, s1
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    eors r2, r1
+; CHECK-NEXT:    eors r3, r0
+; CHECK-NEXT:    orrs r2, r3
+; CHECK-NEXT:    cset r2, eq
+; CHECK-NEXT:    tst.w r2, #1
+; CHECK-NEXT:    csetm r2, ne
+; CHECK-NEXT:    vmov.32 q3[0], r2
+; CHECK-NEXT:    vmov.32 q3[1], r2
+; CHECK-NEXT:    vmov r2, s3
+; CHECK-NEXT:    eors r1, r2
+; CHECK-NEXT:    vmov r2, s2
+; CHECK-NEXT:    eors r0, r2
+; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q3[2], r0
+; CHECK-NEXT:    vmov.32 q3[3], r0
+; CHECK-NEXT:    vbic q0, q2, q3
+; CHECK-NEXT:    vand q1, q1, q3
+; CHECK-NEXT:    vorr q0, q1, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <2 x i64> undef, i64 %src2, i32 0
+  %sp = shufflevector <2 x i64> %i, <2 x i64> undef, <2 x i32> zeroinitializer
+  %c = icmp eq <2 x i64> %sp, %src
+  %s = select <2 x i1> %c, <2 x i64> %a, <2 x i64> %b
+  ret <2 x i64> %s
+}
+
+define arm_aapcs_vfpcc <2 x i32> @vcmp_r_eq_v2i32(<2 x i64> %src, i64 %src2, <2 x i32> %a, <2 x i32> %b) {
+; CHECK-LABEL: vcmp_r_eq_v2i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov r2, s1
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    eors r2, r1
+; CHECK-NEXT:    eors r3, r0
+; CHECK-NEXT:    orrs r2, r3
+; CHECK-NEXT:    cset r2, eq
+; CHECK-NEXT:    tst.w r2, #1
+; CHECK-NEXT:    csetm r2, ne
+; CHECK-NEXT:    vmov.32 q3[0], r2
+; CHECK-NEXT:    vmov.32 q3[1], r2
+; CHECK-NEXT:    vmov r2, s3
+; CHECK-NEXT:    eors r1, r2
+; CHECK-NEXT:    vmov r2, s2
+; CHECK-NEXT:    eors r0, r2
+; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q3[2], r0
+; CHECK-NEXT:    vmov.32 q3[3], r0
+; CHECK-NEXT:    vbic q0, q2, q3
+; CHECK-NEXT:    vand q1, q1, q3
+; CHECK-NEXT:    vorr q0, q1, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <2 x i64> undef, i64 %src2, i32 0
+  %sp = shufflevector <2 x i64> %i, <2 x i64> undef, <2 x i32> zeroinitializer
+  %c = icmp eq <2 x i64> %sp, %src
+  %s = select <2 x i1> %c, <2 x i32> %a, <2 x i32> %b
+  ret <2 x i32> %s
+}
+
+define arm_aapcs_vfpcc <2 x i32> @vcmp_r_multi_v2i32(<2 x i64> %a, <2 x i32> %b, <2 x i32> %c) {
+; CHECK-LABEL: vcmp_r_multi_v2i32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11}
+; CHECK-NEXT:    vmov r0, s1
+; CHECK-NEXT:    movs r3, #0
+; CHECK-NEXT:    vmov r1, s0
+; CHECK-NEXT:    vmov r2, s8
+; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    vmov r1, s2
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q3[0], r0
+; CHECK-NEXT:    vmov.32 q3[1], r0
+; CHECK-NEXT:    vmov r0, s3
+; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q3[2], r0
+; CHECK-NEXT:    vmov.32 q3[3], r0
+; CHECK-NEXT:    vbic q0, q2, q3
+; CHECK-NEXT:    vmov lr, s0
+; CHECK-NEXT:    subs.w r1, lr, r2
+; CHECK-NEXT:    asr.w r12, lr, #31
+; CHECK-NEXT:    sbcs.w r1, r12, r2, asr #31
+; CHECK-NEXT:    mov.w r1, #0
+; CHECK-NEXT:    vmov r2, s10
+; CHECK-NEXT:    it lt
+; CHECK-NEXT:    movlt r1, #1
+; CHECK-NEXT:    cmp r1, #0
+; CHECK-NEXT:    csetm r1, ne
+; CHECK-NEXT:    vmov.32 q3[0], r1
+; CHECK-NEXT:    vmov.32 q3[1], r1
+; CHECK-NEXT:    vmov r1, s2
+; CHECK-NEXT:    subs r0, r1, r2
+; CHECK-NEXT:    asr.w r12, r1, #31
+; CHECK-NEXT:    sbcs.w r0, r12, r2, asr #31
+; CHECK-NEXT:    it lt
+; CHECK-NEXT:    movlt r3, #1
+; CHECK-NEXT:    cmp r3, #0
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    cmp.w lr, #0
+; CHECK-NEXT:    vmov.32 q3[2], r0
+; CHECK-NEXT:    vmov.32 q3[3], r0
+; CHECK-NEXT:    cset r0, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    cmp r1, #0
+; CHECK-NEXT:    vmov.32 q4[0], r0
+; CHECK-NEXT:    vmov.32 q4[1], r0
+; CHECK-NEXT:    cset r0, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q4[2], r0
+; CHECK-NEXT:    vmov.32 q4[3], r0
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    cset r0, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q5[0], r0
+; CHECK-NEXT:    vmov.32 q5[1], r0
+; CHECK-NEXT:    vmov r0, s6
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    cset r0, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q5[2], r0
+; CHECK-NEXT:    vmov.32 q5[3], r0
+; CHECK-NEXT:    vand q1, q5, q4
+; CHECK-NEXT:    vand q1, q3, q1
+; CHECK-NEXT:    vbic q0, q0, q1
+; CHECK-NEXT:    vand q1, q2, q1
+; CHECK-NEXT:    vorr q0, q1, q0
+; CHECK-NEXT:    vpop {d8, d9, d10, d11}
+; CHECK-NEXT:    pop {r7, pc}
+  %a4 = icmp eq <2 x i64> %a, zeroinitializer
+  %a5 = select <2 x i1> %a4, <2 x i32> zeroinitializer, <2 x i32> %c
+  %a6 = icmp ne <2 x i32> %b, zeroinitializer
+  %a7 = icmp slt <2 x i32> %a5, %c
+  %a8 = icmp ne <2 x i32> %a5, zeroinitializer
+  %a9 = and <2 x i1> %a6, %a8
+  %a10 = and <2 x i1> %a7, %a9
+  %a11 = select <2 x i1> %a10, <2 x i32> %c, <2 x i32> %a5
+  ret <2 x i32> %a11
+}
diff --git a/llvm/test/CodeGen/Thumb2/mve-vcmpz.ll b/llvm/test/CodeGen/Thumb2/mve-vcmpz.ll
new file mode 100644
index 000000000..6d08abc72
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-vcmpz.ll
@@ -0,0 +1,835 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s
+
+define arm_aapcs_vfpcc <4 x i32> @vcmp_eqz_v4i32(<4 x i32> %src, <4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: vcmp_eqz_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i32 eq, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <4 x i32> %src, zeroinitializer
+  %s = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @vcmp_nez_v4i32(<4 x i32> %src, <4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: vcmp_nez_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i32 ne, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp ne <4 x i32> %src, zeroinitializer
+  %s = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @vcmp_sgtz_v4i32(<4 x i32> %src, <4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: vcmp_sgtz_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.s32 gt, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp sgt <4 x i32> %src, zeroinitializer
+  %s = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @vcmp_sgez_v4i32(<4 x i32> %src, <4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: vcmp_sgez_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.s32 ge, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp sge <4 x i32> %src, zeroinitializer
+  %s = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @vcmp_sltz_v4i32(<4 x i32> %src, <4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: vcmp_sltz_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.s32 lt, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp slt <4 x i32> %src, zeroinitializer
+  %s = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @vcmp_slez_v4i32(<4 x i32> %src, <4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: vcmp_slez_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.s32 le, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp sle <4 x i32> %src, zeroinitializer
+  %s = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @vcmp_ugtz_v4i32(<4 x i32> %src, <4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: vcmp_ugtz_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i32 ne, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp ugt <4 x i32> %src, zeroinitializer
+  %s = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @vcmp_ugez_v4i32(<4 x i32> %src, <4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: vcmp_ugez_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp uge <4 x i32> %src, zeroinitializer
+  %s = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @vcmp_ultz_v4i32(<4 x i32> %src, <4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: vcmp_ultz_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov q0, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp ult <4 x i32> %src, zeroinitializer
+  %s = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @vcmp_ulez_v4i32(<4 x i32> %src, <4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: vcmp_ulez_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.u32 cs, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp ule <4 x i32> %src, zeroinitializer
+  %s = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+
+define arm_aapcs_vfpcc <8 x i16> @vcmp_eqz_v8i16(<8 x i16> %src, <8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: vcmp_eqz_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i16 eq, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <8 x i16> %src, zeroinitializer
+  %s = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+define arm_aapcs_vfpcc <8 x i16> @vcmp_nez_v8i16(<8 x i16> %src, <8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: vcmp_nez_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i16 ne, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp ne <8 x i16> %src, zeroinitializer
+  %s = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+define arm_aapcs_vfpcc <8 x i16> @vcmp_sgtz_v8i16(<8 x i16> %src, <8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: vcmp_sgtz_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.s16 gt, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp sgt <8 x i16> %src, zeroinitializer
+  %s = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+define arm_aapcs_vfpcc <8 x i16> @vcmp_sgez_v8i16(<8 x i16> %src, <8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: vcmp_sgez_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.s16 ge, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp sge <8 x i16> %src, zeroinitializer
+  %s = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+define arm_aapcs_vfpcc <8 x i16> @vcmp_sltz_v8i16(<8 x i16> %src, <8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: vcmp_sltz_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.s16 lt, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp slt <8 x i16> %src, zeroinitializer
+  %s = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+define arm_aapcs_vfpcc <8 x i16> @vcmp_slez_v8i16(<8 x i16> %src, <8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: vcmp_slez_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.s16 le, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp sle <8 x i16> %src, zeroinitializer
+  %s = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+define arm_aapcs_vfpcc <8 x i16> @vcmp_ugtz_v8i16(<8 x i16> %src, <8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: vcmp_ugtz_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i16 ne, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp ugt <8 x i16> %src, zeroinitializer
+  %s = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+define arm_aapcs_vfpcc <8 x i16> @vcmp_ugez_v8i16(<8 x i16> %src, <8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: vcmp_ugez_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp uge <8 x i16> %src, zeroinitializer
+  %s = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+define arm_aapcs_vfpcc <8 x i16> @vcmp_ultz_v8i16(<8 x i16> %src, <8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: vcmp_ultz_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov q0, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp ult <8 x i16> %src, zeroinitializer
+  %s = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+define arm_aapcs_vfpcc <8 x i16> @vcmp_ulez_v8i16(<8 x i16> %src, <8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: vcmp_ulez_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.u16 cs, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp ule <8 x i16> %src, zeroinitializer
+  %s = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+
+define arm_aapcs_vfpcc <16 x i8> @vcmp_eqz_v16i8(<16 x i8> %src, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: vcmp_eqz_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i8 eq, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <16 x i8> %src, zeroinitializer
+  %s = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %s
+}
+
+define arm_aapcs_vfpcc <16 x i8> @vcmp_nez_v16i8(<16 x i8> %src, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: vcmp_nez_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i8 ne, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp ne <16 x i8> %src, zeroinitializer
+  %s = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %s
+}
+
+define arm_aapcs_vfpcc <16 x i8> @vcmp_sgtz_v16i8(<16 x i8> %src, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: vcmp_sgtz_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.s8 gt, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp sgt <16 x i8> %src, zeroinitializer
+  %s = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %s
+}
+
+define arm_aapcs_vfpcc <16 x i8> @vcmp_sgez_v16i8(<16 x i8> %src, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: vcmp_sgez_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.s8 ge, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp sge <16 x i8> %src, zeroinitializer
+  %s = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %s
+}
+
+define arm_aapcs_vfpcc <16 x i8> @vcmp_sltz_v16i8(<16 x i8> %src, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: vcmp_sltz_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.s8 lt, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp slt <16 x i8> %src, zeroinitializer
+  %s = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %s
+}
+
+define arm_aapcs_vfpcc <16 x i8> @vcmp_slez_v16i8(<16 x i8> %src, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: vcmp_slez_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.s8 le, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp sle <16 x i8> %src, zeroinitializer
+  %s = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %s
+}
+
+define arm_aapcs_vfpcc <16 x i8> @vcmp_ugtz_v16i8(<16 x i8> %src, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: vcmp_ugtz_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i8 ne, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp ugt <16 x i8> %src, zeroinitializer
+  %s = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %s
+}
+
+define arm_aapcs_vfpcc <16 x i8> @vcmp_ugez_v16i8(<16 x i8> %src, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: vcmp_ugez_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp uge <16 x i8> %src, zeroinitializer
+  %s = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %s
+}
+
+define arm_aapcs_vfpcc <16 x i8> @vcmp_ultz_v16i8(<16 x i8> %src, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: vcmp_ultz_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov q0, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp ult <16 x i8> %src, zeroinitializer
+  %s = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %s
+}
+
+define arm_aapcs_vfpcc <16 x i8> @vcmp_ulez_v16i8(<16 x i8> %src, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: vcmp_ulez_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.u8 cs, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp ule <16 x i8> %src, zeroinitializer
+  %s = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %s
+}
+
+
+define arm_aapcs_vfpcc <2 x i64> @vcmp_eqz_v2i64(<2 x i64> %src, <2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: vcmp_eqz_v2i64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov r0, s1
+; CHECK-NEXT:    vmov r1, s0
+; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    vmov r1, s2
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q3[0], r0
+; CHECK-NEXT:    vmov.32 q3[1], r0
+; CHECK-NEXT:    vmov r0, s3
+; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q3[2], r0
+; CHECK-NEXT:    vmov.32 q3[3], r0
+; CHECK-NEXT:    vbic q0, q2, q3
+; CHECK-NEXT:    vand q1, q1, q3
+; CHECK-NEXT:    vorr q0, q1, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <2 x i64> %src, zeroinitializer
+  %s = select <2 x i1> %c, <2 x i64> %a, <2 x i64> %b
+  ret <2 x i64> %s
+}
+
+define arm_aapcs_vfpcc <2 x i32> @vcmp_eqz_v2i32(<2 x i64> %src, <2 x i32> %a, <2 x i32> %b) {
+; CHECK-LABEL: vcmp_eqz_v2i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov r0, s1
+; CHECK-NEXT:    vmov r1, s0
+; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    vmov r1, s2
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q3[0], r0
+; CHECK-NEXT:    vmov.32 q3[1], r0
+; CHECK-NEXT:    vmov r0, s3
+; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q3[2], r0
+; CHECK-NEXT:    vmov.32 q3[3], r0
+; CHECK-NEXT:    vbic q0, q2, q3
+; CHECK-NEXT:    vand q1, q1, q3
+; CHECK-NEXT:    vorr q0, q1, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <2 x i64> %src, zeroinitializer
+  %s = select <2 x i1> %c, <2 x i32> %a, <2 x i32> %b
+  ret <2 x i32> %s
+}
+
+
+; Reversed
+
+define arm_aapcs_vfpcc <4 x i32> @vcmp_r_eqz_v4i32(<4 x i32> %src, <4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: vcmp_r_eqz_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i32 eq, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <4 x i32> zeroinitializer, %src
+  %s = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @vcmp_r_nez_v4i32(<4 x i32> %src, <4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: vcmp_r_nez_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i32 ne, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp ne <4 x i32> zeroinitializer, %src
+  %s = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @vcmp_r_sgtz_v4i32(<4 x i32> %src, <4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: vcmp_r_sgtz_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.s32 lt, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp sgt <4 x i32> zeroinitializer, %src
+  %s = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @vcmp_r_sgez_v4i32(<4 x i32> %src, <4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: vcmp_r_sgez_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.s32 le, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp sge <4 x i32> zeroinitializer, %src
+  %s = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @vcmp_r_sltz_v4i32(<4 x i32> %src, <4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: vcmp_r_sltz_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.s32 gt, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp slt <4 x i32> zeroinitializer, %src
+  %s = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @vcmp_r_slez_v4i32(<4 x i32> %src, <4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: vcmp_r_slez_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.s32 ge, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp sle <4 x i32> zeroinitializer, %src
+  %s = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @vcmp_r_ugtz_v4i32(<4 x i32> %src, <4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: vcmp_r_ugtz_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov q0, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp ugt <4 x i32> zeroinitializer, %src
+  %s = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @vcmp_r_ugez_v4i32(<4 x i32> %src, <4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: vcmp_r_ugez_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.u32 cs, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp uge <4 x i32> zeroinitializer, %src
+  %s = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @vcmp_r_ultz_v4i32(<4 x i32> %src, <4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: vcmp_r_ultz_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i32 ne, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp ult <4 x i32> zeroinitializer, %src
+  %s = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+define arm_aapcs_vfpcc <4 x i32> @vcmp_r_ulez_v4i32(<4 x i32> %src, <4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: vcmp_r_ulez_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp ule <4 x i32> zeroinitializer, %src
+  %s = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %s
+}
+
+
+define arm_aapcs_vfpcc <8 x i16> @vcmp_r_eqz_v8i16(<8 x i16> %src, <8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: vcmp_r_eqz_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i16 eq, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <8 x i16> zeroinitializer, %src
+  %s = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+define arm_aapcs_vfpcc <8 x i16> @vcmp_r_nez_v8i16(<8 x i16> %src, <8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: vcmp_r_nez_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i16 ne, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp ne <8 x i16> zeroinitializer, %src
+  %s = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+define arm_aapcs_vfpcc <8 x i16> @vcmp_r_sgtz_v8i16(<8 x i16> %src, <8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: vcmp_r_sgtz_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.s16 lt, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp sgt <8 x i16> zeroinitializer, %src
+  %s = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+define arm_aapcs_vfpcc <8 x i16> @vcmp_r_sgez_v8i16(<8 x i16> %src, <8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: vcmp_r_sgez_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.s16 le, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp sge <8 x i16> zeroinitializer, %src
+  %s = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+define arm_aapcs_vfpcc <8 x i16> @vcmp_r_sltz_v8i16(<8 x i16> %src, <8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: vcmp_r_sltz_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.s16 gt, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp slt <8 x i16> zeroinitializer, %src
+  %s = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+define arm_aapcs_vfpcc <8 x i16> @vcmp_r_slez_v8i16(<8 x i16> %src, <8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: vcmp_r_slez_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.s16 ge, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp sle <8 x i16> zeroinitializer, %src
+  %s = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+define arm_aapcs_vfpcc <8 x i16> @vcmp_r_ugtz_v8i16(<8 x i16> %src, <8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: vcmp_r_ugtz_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov q0, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp ugt <8 x i16> zeroinitializer, %src
+  %s = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+define arm_aapcs_vfpcc <8 x i16> @vcmp_r_ugez_v8i16(<8 x i16> %src, <8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: vcmp_r_ugez_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.u16 cs, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp uge <8 x i16> zeroinitializer, %src
+  %s = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+define arm_aapcs_vfpcc <8 x i16> @vcmp_r_ultz_v8i16(<8 x i16> %src, <8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: vcmp_r_ultz_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i16 ne, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp ult <8 x i16> zeroinitializer, %src
+  %s = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+define arm_aapcs_vfpcc <8 x i16> @vcmp_r_ulez_v8i16(<8 x i16> %src, <8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: vcmp_r_ulez_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp ule <8 x i16> zeroinitializer, %src
+  %s = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %s
+}
+
+
+define arm_aapcs_vfpcc <16 x i8> @vcmp_r_eqz_v16i8(<16 x i8> %src, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: vcmp_r_eqz_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i8 eq, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <16 x i8> zeroinitializer, %src
+  %s = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %s
+}
+
+define arm_aapcs_vfpcc <16 x i8> @vcmp_r_nez_v16i8(<16 x i8> %src, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: vcmp_r_nez_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i8 ne, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp ne <16 x i8> zeroinitializer, %src
+  %s = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %s
+}
+
+define arm_aapcs_vfpcc <16 x i8> @vcmp_r_sgtz_v16i8(<16 x i8> %src, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: vcmp_r_sgtz_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.s8 lt, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp sgt <16 x i8> zeroinitializer, %src
+  %s = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %s
+}
+
+define arm_aapcs_vfpcc <16 x i8> @vcmp_r_sgez_v16i8(<16 x i8> %src, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: vcmp_r_sgez_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.s8 le, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp sge <16 x i8> zeroinitializer, %src
+  %s = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %s
+}
+
+define arm_aapcs_vfpcc <16 x i8> @vcmp_r_sltz_v16i8(<16 x i8> %src, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: vcmp_r_sltz_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.s8 gt, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp slt <16 x i8> zeroinitializer, %src
+  %s = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %s
+}
+
+define arm_aapcs_vfpcc <16 x i8> @vcmp_r_slez_v16i8(<16 x i8> %src, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: vcmp_r_slez_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.s8 ge, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp sle <16 x i8> zeroinitializer, %src
+  %s = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %s
+}
+
+define arm_aapcs_vfpcc <16 x i8> @vcmp_r_ugtz_v16i8(<16 x i8> %src, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: vcmp_r_ugtz_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov q0, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp ugt <16 x i8> zeroinitializer, %src
+  %s = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %s
+}
+
+define arm_aapcs_vfpcc <16 x i8> @vcmp_r_ugez_v16i8(<16 x i8> %src, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: vcmp_r_ugez_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.u8 cs, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp uge <16 x i8> zeroinitializer, %src
+  %s = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %s
+}
+
+define arm_aapcs_vfpcc <16 x i8> @vcmp_r_ultz_v16i8(<16 x i8> %src, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: vcmp_r_ultz_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i8 ne, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp ult <16 x i8> zeroinitializer, %src
+  %s = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %s
+}
+
+define arm_aapcs_vfpcc <16 x i8> @vcmp_r_ulez_v16i8(<16 x i8> %src, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: vcmp_r_ulez_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp ule <16 x i8> zeroinitializer, %src
+  %s = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %s
+}
+
+
+define arm_aapcs_vfpcc <2 x i64> @vcmp_r_eqz_v2i64(<2 x i64> %src, <2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: vcmp_r_eqz_v2i64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov r0, s1
+; CHECK-NEXT:    vmov r1, s0
+; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    vmov r1, s2
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q3[0], r0
+; CHECK-NEXT:    vmov.32 q3[1], r0
+; CHECK-NEXT:    vmov r0, s3
+; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q3[2], r0
+; CHECK-NEXT:    vmov.32 q3[3], r0
+; CHECK-NEXT:    vbic q0, q2, q3
+; CHECK-NEXT:    vand q1, q1, q3
+; CHECK-NEXT:    vorr q0, q1, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <2 x i64> zeroinitializer, %src
+  %s = select <2 x i1> %c, <2 x i64> %a, <2 x i64> %b
+  ret <2 x i64> %s
+}
+
+define arm_aapcs_vfpcc <2 x i32> @vcmp_r_eqz_v2i32(<2 x i64> %src, <2 x i32> %a, <2 x i32> %b) {
+; CHECK-LABEL: vcmp_r_eqz_v2i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov r0, s1
+; CHECK-NEXT:    vmov r1, s0
+; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    vmov r1, s2
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q3[0], r0
+; CHECK-NEXT:    vmov.32 q3[1], r0
+; CHECK-NEXT:    vmov r0, s3
+; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q3[2], r0
+; CHECK-NEXT:    vmov.32 q3[3], r0
+; CHECK-NEXT:    vbic q0, q2, q3
+; CHECK-NEXT:    vand q1, q1, q3
+; CHECK-NEXT:    vorr q0, q1, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <2 x i64> %src, zeroinitializer
+  %s = select <2 x i1> %c, <2 x i32> %a, <2 x i32> %b
+  ret <2 x i32> %s
+}
diff --git a/llvm/test/CodeGen/Thumb2/mve-vctp.ll b/llvm/test/CodeGen/Thumb2/mve-vctp.ll
new file mode 100644
index 000000000..d6e4d492f
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-vctp.ll
@@ -0,0 +1,54 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-none-eabi -mattr=+mve --verify-machineinstrs %s -o - | FileCheck %s
+
+define void @vctp8(i32 %arg, <16 x i8> *%in, <16 x i8>* %out) {
+; CHECK-LABEL: vctp8:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vldrw.u32 q1, [r1]
+; CHECK-NEXT:    vmov.i32 q0, #0x0
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    vstrw.32 q0, [r2]
+; CHECK-NEXT:    bx lr
+  %pred = call <16 x i1> @llvm.arm.mve.vctp8(i32 %arg)
+  %ld = load <16 x i8>, <16 x i8>* %in
+  %res = select <16 x i1> %pred, <16 x i8> %ld, <16 x i8> zeroinitializer
+  store <16 x i8> %res, <16 x i8>* %out
+  ret void
+}
+
+define void @vctp16(i32 %arg, <8 x i16> *%in, <8 x i16>* %out) {
+; CHECK-LABEL: vctp16:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vldrw.u32 q1, [r1]
+; CHECK-NEXT:    vmov.i32 q0, #0x0
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    vstrw.32 q0, [r2]
+; CHECK-NEXT:    bx lr
+  %pred = call <8 x i1> @llvm.arm.mve.vctp16(i32 %arg)
+  %ld = load <8 x i16>, <8 x i16>* %in
+  %res = select <8 x i1> %pred, <8 x i16> %ld, <8 x i16> zeroinitializer
+  store <8 x i16> %res, <8 x i16>* %out
+  ret void
+}
+
+define void @vctp32(i32 %arg, <4 x i32> *%in, <4 x i32>* %out) {
+; CHECK-LABEL: vctp32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vldrw.u32 q1, [r1]
+; CHECK-NEXT:    vmov.i32 q0, #0x0
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    vstrw.32 q0, [r2]
+; CHECK-NEXT:    bx lr
+  %pred = call <4 x i1> @llvm.arm.mve.vctp32(i32 %arg)
+  %ld = load <4 x i32>, <4 x i32>* %in
+  %res = select <4 x i1> %pred, <4 x i32> %ld, <4 x i32> zeroinitializer
+  store <4 x i32> %res, <4 x i32>* %out
+  ret void
+}
+
+declare <16 x i1> @llvm.arm.mve.vctp8(i32)
+declare <8 x i1> @llvm.arm.mve.vctp16(i32)
+declare <4 x i1> @llvm.arm.mve.vctp32(i32)
diff --git a/llvm/test/CodeGen/Thumb2/mve-vcvt.ll b/llvm/test/CodeGen/Thumb2/mve-vcvt.ll
index 524ec692c..a2d8b582a 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vcvt.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vcvt.ll
@@ -213,29 +213,17 @@ entry:
 define arm_aapcs_vfpcc <8 x i16> @foo_int16_half(<8 x half> %src) {
 ; CHECK-MVE-LABEL: foo_int16_half:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[7]
-; CHECK-MVE-NEXT:    vmov s4, r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[5]
-; CHECK-MVE-NEXT:    vmov s6, r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[6]
-; CHECK-MVE-NEXT:    vmov s8, r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[3]
-; CHECK-MVE-NEXT:    vmov s10, r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[4]
-; CHECK-MVE-NEXT:    vmov s12, r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[1]
-; CHECK-MVE-NEXT:    vmov s14, r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[2]
-; CHECK-MVE-NEXT:    vmov s5, r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[0]
-; CHECK-MVE-NEXT:    vmov s0, r0
+; CHECK-MVE-NEXT:    vmovx.f16 s4, s3
+; CHECK-MVE-NEXT:    vmovx.f16 s6, s2
+; CHECK-MVE-NEXT:    vmovx.f16 s10, s1
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s0
 ; CHECK-MVE-NEXT:    vcvt.s32.f16 s4, s4
 ; CHECK-MVE-NEXT:    vcvt.s32.f16 s6, s6
-; CHECK-MVE-NEXT:    vcvt.s32.f16 s8, s8
+; CHECK-MVE-NEXT:    vcvt.s32.f16 s8, s3
 ; CHECK-MVE-NEXT:    vcvt.s32.f16 s10, s10
-; CHECK-MVE-NEXT:    vcvt.s32.f16 s12, s12
+; CHECK-MVE-NEXT:    vcvt.s32.f16 s12, s2
 ; CHECK-MVE-NEXT:    vcvt.s32.f16 s14, s14
-; CHECK-MVE-NEXT:    vcvt.s32.f16 s5, s5
+; CHECK-MVE-NEXT:    vcvt.s32.f16 s5, s1
 ; CHECK-MVE-NEXT:    vcvt.s32.f16 s0, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q0[0], r0
@@ -267,29 +255,17 @@ entry:
 define arm_aapcs_vfpcc <8 x i16> @foo_uint16_half(<8 x half> %src) {
 ; CHECK-MVE-LABEL: foo_uint16_half:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[7]
-; CHECK-MVE-NEXT:    vmov s4, r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[5]
-; CHECK-MVE-NEXT:    vmov s6, r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[6]
-; CHECK-MVE-NEXT:    vmov s8, r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[3]
-; CHECK-MVE-NEXT:    vmov s10, r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[4]
-; CHECK-MVE-NEXT:    vmov s12, r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[1]
-; CHECK-MVE-NEXT:    vmov s14, r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[2]
-; CHECK-MVE-NEXT:    vmov s5, r0
-; CHECK-MVE-NEXT:    vmov.u16 r0, q0[0]
-; CHECK-MVE-NEXT:    vmov s0, r0
+; CHECK-MVE-NEXT:    vmovx.f16 s4, s3
+; CHECK-MVE-NEXT:    vmovx.f16 s6, s2
+; CHECK-MVE-NEXT:    vmovx.f16 s10, s1
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s0
 ; CHECK-MVE-NEXT:    vcvt.s32.f16 s4, s4
 ; CHECK-MVE-NEXT:    vcvt.s32.f16 s6, s6
-; CHECK-MVE-NEXT:    vcvt.s32.f16 s8, s8
+; CHECK-MVE-NEXT:    vcvt.s32.f16 s8, s3
 ; CHECK-MVE-NEXT:    vcvt.s32.f16 s10, s10
-; CHECK-MVE-NEXT:    vcvt.s32.f16 s12, s12
+; CHECK-MVE-NEXT:    vcvt.s32.f16 s12, s2
 ; CHECK-MVE-NEXT:    vcvt.s32.f16 s14, s14
-; CHECK-MVE-NEXT:    vcvt.s32.f16 s5, s5
+; CHECK-MVE-NEXT:    vcvt.s32.f16 s5, s1
 ; CHECK-MVE-NEXT:    vcvt.s32.f16 s0, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q0[0], r0
@@ -382,14 +358,12 @@ define arm_aapcs_vfpcc <2 x i64> @foo_int64_float(<2 x double> %src) {
 ; CHECK-NEXT:    bl __aeabi_d2lz
 ; CHECK-NEXT:    vmov r2, r3, d9
 ; CHECK-NEXT:    vmov.32 q4[0], r0
-; CHECK-NEXT:    lsrl r0, r1, #32
-; CHECK-NEXT:    vmov.32 q4[1], r0
+; CHECK-NEXT:    vmov.32 q4[1], r1
 ; CHECK-NEXT:    mov r0, r2
 ; CHECK-NEXT:    mov r1, r3
 ; CHECK-NEXT:    bl __aeabi_d2lz
 ; CHECK-NEXT:    vmov.32 q4[2], r0
-; CHECK-NEXT:    lsrl r0, r1, #32
-; CHECK-NEXT:    vmov.32 q4[3], r0
+; CHECK-NEXT:    vmov.32 q4[3], r1
 ; CHECK-NEXT:    vmov q0, q4
 ; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    pop {r7, pc}
@@ -410,14 +384,12 @@ define arm_aapcs_vfpcc <2 x i64> @foo_uint64_float(<2 x double> %src) {
 ; CHECK-NEXT:    bl __aeabi_d2ulz
 ; CHECK-NEXT:    vmov r2, r3, d9
 ; CHECK-NEXT:    vmov.32 q4[0], r0
-; CHECK-NEXT:    lsrl r0, r1, #32
-; CHECK-NEXT:    vmov.32 q4[1], r0
+; CHECK-NEXT:    vmov.32 q4[1], r1
 ; CHECK-NEXT:    mov r0, r2
 ; CHECK-NEXT:    mov r1, r3
 ; CHECK-NEXT:    bl __aeabi_d2ulz
 ; CHECK-NEXT:    vmov.32 q4[2], r0
-; CHECK-NEXT:    lsrl r0, r1, #32
-; CHECK-NEXT:    vmov.32 q4[3], r0
+; CHECK-NEXT:    vmov.32 q4[3], r1
 ; CHECK-NEXT:    vmov q0, q4
 ; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    pop {r7, pc}
diff --git a/llvm/test/CodeGen/Thumb2/mve-vecreduce-fminmax.ll b/llvm/test/CodeGen/Thumb2/mve-vecreduce-fminmax.ll
new file mode 100644
index 000000000..26541e6ec
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-vecreduce-fminmax.ll
@@ -0,0 +1,2264 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve.fp,+fp64 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-FP
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve,+fullfp16,+fp64 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-NOFP
+
+; FIXME minnum nonan X, +Inf -> X   ?
+define arm_aapcs_vfpcc float @fmin_v2f32(<2 x float> %x) {
+; CHECK-LABEL: fmin_v2f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldr s4, .LCPI0_0
+; CHECK-NEXT:    vminnm.f32 s0, s0, s1
+; CHECK-NEXT:    vminnm.f32 s0, s0, s4
+; CHECK-NEXT:    vminnm.f32 s0, s0, s4
+; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    .p2align 2
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI0_0:
+; CHECK-NEXT:    .long 2139095040 @ float +Inf
+entry:
+  %z = call fast float @llvm.experimental.vector.reduce.fmin.v2f32(<2 x float> %x)
+  ret float %z
+}
+
+define arm_aapcs_vfpcc float @fmin_v4f32(<4 x float> %x) {
+; CHECK-LABEL: fmin_v4f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vminnm.f32 s4, s0, s1
+; CHECK-NEXT:    vminnm.f32 s4, s4, s2
+; CHECK-NEXT:    vminnm.f32 s0, s4, s3
+; CHECK-NEXT:    bx lr
+entry:
+  %z = call fast float @llvm.experimental.vector.reduce.fmin.v4f32(<4 x float> %x)
+  ret float %z
+}
+
+; FIXME fminnum (vector) -> fminnum (scalar)  ?
+define arm_aapcs_vfpcc float @fmin_v8f32(<8 x float> %x) {
+; CHECK-FP-LABEL: fmin_v8f32:
+; CHECK-FP:       @ %bb.0: @ %entry
+; CHECK-FP-NEXT:    vminnm.f32 q0, q0, q1
+; CHECK-FP-NEXT:    vminnm.f32 s4, s0, s1
+; CHECK-FP-NEXT:    vminnm.f32 s4, s4, s2
+; CHECK-FP-NEXT:    vminnm.f32 s0, s4, s3
+; CHECK-FP-NEXT:    bx lr
+;
+; CHECK-NOFP-LABEL: fmin_v8f32:
+; CHECK-NOFP:       @ %bb.0: @ %entry
+; CHECK-NOFP-NEXT:    vcmp.f32 s5, s1
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f32 s4, s0
+; CHECK-NOFP-NEXT:    vselgt.f32 s8, s1, s5
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f32 s6, s2
+; CHECK-NOFP-NEXT:    vselgt.f32 s10, s0, s4
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f32 s7, s3
+; CHECK-NOFP-NEXT:    vselgt.f32 s12, s2, s6
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f32 s0, s3, s7
+; CHECK-NOFP-NEXT:    vminnm.f32 s2, s10, s8
+; CHECK-NOFP-NEXT:    vminnm.f32 s2, s2, s12
+; CHECK-NOFP-NEXT:    vminnm.f32 s0, s2, s0
+; CHECK-NOFP-NEXT:    bx lr
+entry:
+  %z = call fast float @llvm.experimental.vector.reduce.fmin.v8f32(<8 x float> %x)
+  ret float %z
+}
+
+define arm_aapcs_vfpcc half @fmin_v4f16(<4 x half> %x) {
+; CHECK-LABEL: fmin_v4f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmovx.f16 s4, s0
+; CHECK-NEXT:    vminnm.f16 s4, s0, s4
+; CHECK-NEXT:    vmovx.f16 s0, s1
+; CHECK-NEXT:    vminnm.f16 s4, s4, s1
+; CHECK-NEXT:    vldr.16 s2, .LCPI3_0
+; CHECK-NEXT:    vminnm.f16 s0, s4, s0
+; CHECK-NEXT:    vminnm.f16 s0, s0, s2
+; CHECK-NEXT:    vminnm.f16 s0, s0, s2
+; CHECK-NEXT:    vminnm.f16 s0, s0, s2
+; CHECK-NEXT:    vminnm.f16 s0, s0, s2
+; CHECK-NEXT:    vstr.16 s0, [r0]
+; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    .p2align 1
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI3_0:
+; CHECK-NEXT:    .short 31744 @ half +Inf
+entry:
+  %z = call fast half @llvm.experimental.vector.reduce.fmin.v4f16(<4 x half> %x)
+  ret half %z
+}
+
+define arm_aapcs_vfpcc half @fmin_v8f16(<8 x half> %x) {
+; CHECK-LABEL: fmin_v8f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmovx.f16 s4, s0
+; CHECK-NEXT:    vmovx.f16 s6, s1
+; CHECK-NEXT:    vminnm.f16 s4, s0, s4
+; CHECK-NEXT:    vmovx.f16 s0, s3
+; CHECK-NEXT:    vminnm.f16 s4, s4, s1
+; CHECK-NEXT:    vminnm.f16 s4, s4, s6
+; CHECK-NEXT:    vmovx.f16 s6, s2
+; CHECK-NEXT:    vminnm.f16 s4, s4, s2
+; CHECK-NEXT:    vminnm.f16 s4, s4, s6
+; CHECK-NEXT:    vminnm.f16 s4, s4, s3
+; CHECK-NEXT:    vminnm.f16 s0, s4, s0
+; CHECK-NEXT:    vstr.16 s0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = call fast half @llvm.experimental.vector.reduce.fmin.v8f16(<8 x half> %x)
+  ret half %z
+}
+
+define arm_aapcs_vfpcc half @fmin_v16f16(<16 x half> %x) {
+; CHECK-FP-LABEL: fmin_v16f16:
+; CHECK-FP:       @ %bb.0: @ %entry
+; CHECK-FP-NEXT:    vminnm.f16 q0, q0, q1
+; CHECK-FP-NEXT:    vmovx.f16 s4, s0
+; CHECK-FP-NEXT:    vmovx.f16 s6, s1
+; CHECK-FP-NEXT:    vminnm.f16 s4, s0, s4
+; CHECK-FP-NEXT:    vmovx.f16 s0, s3
+; CHECK-FP-NEXT:    vminnm.f16 s4, s4, s1
+; CHECK-FP-NEXT:    vminnm.f16 s4, s4, s6
+; CHECK-FP-NEXT:    vmovx.f16 s6, s2
+; CHECK-FP-NEXT:    vminnm.f16 s4, s4, s2
+; CHECK-FP-NEXT:    vminnm.f16 s4, s4, s6
+; CHECK-FP-NEXT:    vminnm.f16 s4, s4, s3
+; CHECK-FP-NEXT:    vminnm.f16 s0, s4, s0
+; CHECK-FP-NEXT:    vstr.16 s0, [r0]
+; CHECK-FP-NEXT:    bx lr
+;
+; CHECK-NOFP-LABEL: fmin_v16f16:
+; CHECK-NOFP:       @ %bb.0: @ %entry
+; CHECK-NOFP-NEXT:    vmovx.f16 s8, s4
+; CHECK-NOFP-NEXT:    vmovx.f16 s10, s0
+; CHECK-NOFP-NEXT:    vcmp.f16 s8, s10
+; CHECK-NOFP-NEXT:    vmovx.f16 s12, s1
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f16 s4, s0
+; CHECK-NOFP-NEXT:    vselgt.f16 s8, s10, s8
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f16 s5, s1
+; CHECK-NOFP-NEXT:    vselgt.f16 s10, s0, s4
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vminnm.f16 s8, s10, s8
+; CHECK-NOFP-NEXT:    vmovx.f16 s4, s7
+; CHECK-NOFP-NEXT:    vmovx.f16 s0, s3
+; CHECK-NOFP-NEXT:    vselgt.f16 s10, s1, s5
+; CHECK-NOFP-NEXT:    vminnm.f16 s8, s8, s10
+; CHECK-NOFP-NEXT:    vmovx.f16 s10, s5
+; CHECK-NOFP-NEXT:    vcmp.f16 s10, s12
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f16 s6, s2
+; CHECK-NOFP-NEXT:    vselgt.f16 s10, s12, s10
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vminnm.f16 s8, s8, s10
+; CHECK-NOFP-NEXT:    vmovx.f16 s12, s2
+; CHECK-NOFP-NEXT:    vselgt.f16 s10, s2, s6
+; CHECK-NOFP-NEXT:    vminnm.f16 s8, s8, s10
+; CHECK-NOFP-NEXT:    vmovx.f16 s10, s6
+; CHECK-NOFP-NEXT:    vcmp.f16 s10, s12
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f16 s7, s3
+; CHECK-NOFP-NEXT:    vselgt.f16 s10, s12, s10
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f16 s4, s0
+; CHECK-NOFP-NEXT:    vminnm.f16 s8, s8, s10
+; CHECK-NOFP-NEXT:    vselgt.f16 s10, s3, s7
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vminnm.f16 s8, s8, s10
+; CHECK-NOFP-NEXT:    vselgt.f16 s0, s0, s4
+; CHECK-NOFP-NEXT:    vminnm.f16 s0, s8, s0
+; CHECK-NOFP-NEXT:    vstr.16 s0, [r0]
+; CHECK-NOFP-NEXT:    bx lr
+entry:
+  %z = call fast half @llvm.experimental.vector.reduce.fmin.v16f16(<16 x half> %x)
+  ret half %z
+}
+
+define arm_aapcs_vfpcc double @fmin_v1f64(<1 x double> %x) {
+; CHECK-LABEL: fmin_v1f64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    bx lr
+entry:
+  %z = call fast double @llvm.experimental.vector.reduce.fmin.v1f64(<1 x double> %x)
+  ret double %z
+}
+
+define arm_aapcs_vfpcc double @fmin_v2f64(<2 x double> %x) {
+; CHECK-LABEL: fmin_v2f64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vminnm.f64 d0, d0, d1
+; CHECK-NEXT:    bx lr
+entry:
+  %z = call fast double @llvm.experimental.vector.reduce.fmin.v2f64(<2 x double> %x)
+  ret double %z
+}
+
+define arm_aapcs_vfpcc double @fmin_v4f64(<4 x double> %x) {
+; CHECK-LABEL: fmin_v4f64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.f64 d3, d1
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    vcmp.f64 d2, d0
+; CHECK-NEXT:    vselgt.f64 d4, d1, d3
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    vselgt.f64 d0, d0, d2
+; CHECK-NEXT:    vminnm.f64 d0, d0, d4
+; CHECK-NEXT:    bx lr
+entry:
+  %z = call fast double @llvm.experimental.vector.reduce.fmin.v4f64(<4 x double> %x)
+  ret double %z
+}
+
+; FIXME should not be vminnm
+; FIXME better reductions (no vmovs/vdups)
+define arm_aapcs_vfpcc float @fmin_v2f32_nofast(<2 x float> %x) {
+; CHECK-FP-LABEL: fmin_v2f32_nofast:
+; CHECK-FP:       @ %bb.0: @ %entry
+; CHECK-FP-NEXT:    vmov.32 r0, q0[1]
+; CHECK-FP-NEXT:    vdup.32 q1, r0
+; CHECK-FP-NEXT:    vminnm.f32 q0, q0, q1
+; CHECK-FP-NEXT:    @ kill: def $s0 killed $s0 killed $q0
+; CHECK-FP-NEXT:    bx lr
+;
+; CHECK-NOFP-LABEL: fmin_v2f32_nofast:
+; CHECK-NOFP:       @ %bb.0: @ %entry
+; CHECK-NOFP-NEXT:    vmov.32 r0, q0[1]
+; CHECK-NOFP-NEXT:    vdup.32 q1, r0
+; CHECK-NOFP-NEXT:    vcmp.f32 s4, s0
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f32 s0, s0, s4
+; CHECK-NOFP-NEXT:    bx lr
+entry:
+  %z = call float @llvm.experimental.vector.reduce.fmin.v2f32(<2 x float> %x)
+  ret float %z
+}
+
+define arm_aapcs_vfpcc float @fmin_v4f32_nofast(<4 x float> %x) {
+; CHECK-FP-LABEL: fmin_v4f32_nofast:
+; CHECK-FP:       @ %bb.0: @ %entry
+; CHECK-FP-NEXT:    vmov.f64 d2, d1
+; CHECK-FP-NEXT:    vmov.f32 s5, s3
+; CHECK-FP-NEXT:    vminnm.f32 q0, q0, q1
+; CHECK-FP-NEXT:    vmov.32 r0, q0[1]
+; CHECK-FP-NEXT:    vdup.32 q1, r0
+; CHECK-FP-NEXT:    vminnm.f32 q0, q0, q1
+; CHECK-FP-NEXT:    @ kill: def $s0 killed $s0 killed $q0
+; CHECK-FP-NEXT:    bx lr
+;
+; CHECK-NOFP-LABEL: fmin_v4f32_nofast:
+; CHECK-NOFP:       @ %bb.0: @ %entry
+; CHECK-NOFP-NEXT:    vcmp.f32 s3, s1
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vmov.f64 d2, d1
+; CHECK-NOFP-NEXT:    vmov.f32 s5, s3
+; CHECK-NOFP-NEXT:    vcmp.f32 s4, s0
+; CHECK-NOFP-NEXT:    vselgt.f32 s8, s1, s3
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f32 s0, s0, s4
+; CHECK-NOFP-NEXT:    vcmp.f32 s8, s0
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f32 s0, s0, s8
+; CHECK-NOFP-NEXT:    bx lr
+entry:
+  %z = call float @llvm.experimental.vector.reduce.fmin.v4f32(<4 x float> %x)
+  ret float %z
+}
+
+define arm_aapcs_vfpcc float @fmin_v8f32_nofast(<8 x float> %x) {
+; CHECK-FP-LABEL: fmin_v8f32_nofast:
+; CHECK-FP:       @ %bb.0: @ %entry
+; CHECK-FP-NEXT:    vminnm.f32 q0, q0, q1
+; CHECK-FP-NEXT:    vmov.f64 d2, d1
+; CHECK-FP-NEXT:    vmov.f32 s5, s3
+; CHECK-FP-NEXT:    vminnm.f32 q0, q0, q1
+; CHECK-FP-NEXT:    vmov.32 r0, q0[1]
+; CHECK-FP-NEXT:    vdup.32 q1, r0
+; CHECK-FP-NEXT:    vminnm.f32 q0, q0, q1
+; CHECK-FP-NEXT:    @ kill: def $s0 killed $s0 killed $q0
+; CHECK-FP-NEXT:    bx lr
+;
+; CHECK-NOFP-LABEL: fmin_v8f32_nofast:
+; CHECK-NOFP:       @ %bb.0: @ %entry
+; CHECK-NOFP-NEXT:    vcmp.f32 s7, s3
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f32 s5, s1
+; CHECK-NOFP-NEXT:    vselgt.f32 s8, s3, s7
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f32 s6, s2
+; CHECK-NOFP-NEXT:    vselgt.f32 s10, s1, s5
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f32 s4, s0
+; CHECK-NOFP-NEXT:    vselgt.f32 s12, s2, s6
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f32 s8, s10
+; CHECK-NOFP-NEXT:    vselgt.f32 s0, s0, s4
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f32 s12, s0
+; CHECK-NOFP-NEXT:    vselgt.f32 s2, s10, s8
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f32 s0, s0, s12
+; CHECK-NOFP-NEXT:    vcmp.f32 s2, s0
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f32 s0, s0, s2
+; CHECK-NOFP-NEXT:    bx lr
+entry:
+  %z = call float @llvm.experimental.vector.reduce.fmin.v8f32(<8 x float> %x)
+  ret float %z
+}
+
+define arm_aapcs_vfpcc half @fmin_v4f16_nofast(<4 x half> %x) {
+; CHECK-FP-LABEL: fmin_v4f16_nofast:
+; CHECK-FP:       @ %bb.0: @ %entry
+; CHECK-FP-NEXT:    vmov.32 r1, q0[1]
+; CHECK-FP-NEXT:    vdup.32 q1, r1
+; CHECK-FP-NEXT:    vminnm.f16 q0, q0, q1
+; CHECK-FP-NEXT:    vmov.u16 r1, q0[1]
+; CHECK-FP-NEXT:    vdup.16 q1, r1
+; CHECK-FP-NEXT:    vminnm.f16 q0, q0, q1
+; CHECK-FP-NEXT:    vstr.16 s0, [r0]
+; CHECK-FP-NEXT:    bx lr
+;
+; CHECK-NOFP-LABEL: fmin_v4f16_nofast:
+; CHECK-NOFP:       @ %bb.0: @ %entry
+; CHECK-NOFP-NEXT:    vmov.32 r1, q0[1]
+; CHECK-NOFP-NEXT:    vmovx.f16 s10, s0
+; CHECK-NOFP-NEXT:    vdup.32 q1, r1
+; CHECK-NOFP-NEXT:    vmovx.f16 s8, s4
+; CHECK-NOFP-NEXT:    vcmp.f16 s8, s10
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f16 s4, s0
+; CHECK-NOFP-NEXT:    vselgt.f16 s8, s10, s8
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s0, s0, s4
+; CHECK-NOFP-NEXT:    vcmp.f16 s8, s0
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s0, s0, s8
+; CHECK-NOFP-NEXT:    vstr.16 s0, [r0]
+; CHECK-NOFP-NEXT:    bx lr
+entry:
+  %z = call half @llvm.experimental.vector.reduce.fmin.v4f16(<4 x half> %x)
+  ret half %z
+}
+
+define arm_aapcs_vfpcc half @fmin_v8f16_nofast(<8 x half> %x) {
+; CHECK-FP-LABEL: fmin_v8f16_nofast:
+; CHECK-FP:       @ %bb.0: @ %entry
+; CHECK-FP-NEXT:    vmov.f64 d2, d1
+; CHECK-FP-NEXT:    vmov.f32 s5, s3
+; CHECK-FP-NEXT:    vminnm.f16 q0, q0, q1
+; CHECK-FP-NEXT:    vmov.32 r1, q0[1]
+; CHECK-FP-NEXT:    vdup.32 q1, r1
+; CHECK-FP-NEXT:    vminnm.f16 q0, q0, q1
+; CHECK-FP-NEXT:    vmov.u16 r1, q0[1]
+; CHECK-FP-NEXT:    vdup.16 q1, r1
+; CHECK-FP-NEXT:    vminnm.f16 q0, q0, q1
+; CHECK-FP-NEXT:    vstr.16 s0, [r0]
+; CHECK-FP-NEXT:    bx lr
+;
+; CHECK-NOFP-LABEL: fmin_v8f16_nofast:
+; CHECK-NOFP:       @ %bb.0: @ %entry
+; CHECK-NOFP-NEXT:    vmovx.f16 s8, s3
+; CHECK-NOFP-NEXT:    vmovx.f16 s10, s1
+; CHECK-NOFP-NEXT:    vcmp.f16 s8, s10
+; CHECK-NOFP-NEXT:    vmov.f64 d2, d1
+; CHECK-NOFP-NEXT:    vmovx.f16 s12, s0
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vmov.f32 s5, s3
+; CHECK-NOFP-NEXT:    vselgt.f16 s8, s10, s8
+; CHECK-NOFP-NEXT:    vmovx.f16 s10, s4
+; CHECK-NOFP-NEXT:    vcmp.f16 s10, s12
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s10, s12, s10
+; CHECK-NOFP-NEXT:    vcmp.f16 s8, s10
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f16 s3, s1
+; CHECK-NOFP-NEXT:    vselgt.f16 s8, s10, s8
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f16 s4, s0
+; CHECK-NOFP-NEXT:    vselgt.f16 s10, s1, s3
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s0, s0, s4
+; CHECK-NOFP-NEXT:    vcmp.f16 s10, s0
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s0, s0, s10
+; CHECK-NOFP-NEXT:    vcmp.f16 s8, s0
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s0, s0, s8
+; CHECK-NOFP-NEXT:    vstr.16 s0, [r0]
+; CHECK-NOFP-NEXT:    bx lr
+entry:
+  %z = call half @llvm.experimental.vector.reduce.fmin.v8f16(<8 x half> %x)
+  ret half %z
+}
+
+define arm_aapcs_vfpcc half @fmin_v16f16_nofast(<16 x half> %x) {
+; CHECK-FP-LABEL: fmin_v16f16_nofast:
+; CHECK-FP:       @ %bb.0: @ %entry
+; CHECK-FP-NEXT:    vminnm.f16 q0, q0, q1
+; CHECK-FP-NEXT:    vmov.f64 d2, d1
+; CHECK-FP-NEXT:    vmov.f32 s5, s3
+; CHECK-FP-NEXT:    vminnm.f16 q0, q0, q1
+; CHECK-FP-NEXT:    vmov.32 r1, q0[1]
+; CHECK-FP-NEXT:    vdup.32 q1, r1
+; CHECK-FP-NEXT:    vminnm.f16 q0, q0, q1
+; CHECK-FP-NEXT:    vmov.u16 r1, q0[1]
+; CHECK-FP-NEXT:    vdup.16 q1, r1
+; CHECK-FP-NEXT:    vminnm.f16 q0, q0, q1
+; CHECK-FP-NEXT:    vstr.16 s0, [r0]
+; CHECK-FP-NEXT:    bx lr
+;
+; CHECK-NOFP-LABEL: fmin_v16f16_nofast:
+; CHECK-NOFP:       @ %bb.0: @ %entry
+; CHECK-NOFP-NEXT:    vmovx.f16 s8, s7
+; CHECK-NOFP-NEXT:    vmovx.f16 s10, s3
+; CHECK-NOFP-NEXT:    vcmp.f16 s8, s10
+; CHECK-NOFP-NEXT:    vmovx.f16 s12, s1
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vmovx.f16 s14, s0
+; CHECK-NOFP-NEXT:    vselgt.f16 s8, s10, s8
+; CHECK-NOFP-NEXT:    vmovx.f16 s10, s5
+; CHECK-NOFP-NEXT:    vcmp.f16 s10, s12
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s10, s12, s10
+; CHECK-NOFP-NEXT:    vmovx.f16 s12, s2
+; CHECK-NOFP-NEXT:    vcmp.f16 s8, s10
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s8, s10, s8
+; CHECK-NOFP-NEXT:    vmovx.f16 s10, s6
+; CHECK-NOFP-NEXT:    vcmp.f16 s10, s12
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s10, s12, s10
+; CHECK-NOFP-NEXT:    vmovx.f16 s12, s4
+; CHECK-NOFP-NEXT:    vcmp.f16 s12, s14
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s12, s14, s12
+; CHECK-NOFP-NEXT:    vcmp.f16 s10, s12
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s10, s12, s10
+; CHECK-NOFP-NEXT:    vcmp.f16 s8, s10
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f16 s7, s3
+; CHECK-NOFP-NEXT:    vselgt.f16 s8, s10, s8
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f16 s5, s1
+; CHECK-NOFP-NEXT:    vselgt.f16 s10, s3, s7
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s12, s1, s5
+; CHECK-NOFP-NEXT:    vcmp.f16 s10, s12
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f16 s6, s2
+; CHECK-NOFP-NEXT:    vselgt.f16 s10, s12, s10
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f16 s4, s0
+; CHECK-NOFP-NEXT:    vselgt.f16 s12, s2, s6
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s0, s0, s4
+; CHECK-NOFP-NEXT:    vcmp.f16 s12, s0
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s0, s0, s12
+; CHECK-NOFP-NEXT:    vcmp.f16 s10, s0
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s0, s0, s10
+; CHECK-NOFP-NEXT:    vcmp.f16 s8, s0
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s0, s0, s8
+; CHECK-NOFP-NEXT:    vstr.16 s0, [r0]
+; CHECK-NOFP-NEXT:    bx lr
+entry:
+  %z = call half @llvm.experimental.vector.reduce.fmin.v16f16(<16 x half> %x)
+  ret half %z
+}
+
+define arm_aapcs_vfpcc double @fmin_v1f64_nofast(<1 x double> %x) {
+; CHECK-LABEL: fmin_v1f64_nofast:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    bx lr
+entry:
+  %z = call double @llvm.experimental.vector.reduce.fmin.v1f64(<1 x double> %x)
+  ret double %z
+}
+
+define arm_aapcs_vfpcc double @fmin_v2f64_nofast(<2 x double> %x) {
+; CHECK-LABEL: fmin_v2f64_nofast:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.f64 d1, d0
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    vselgt.f64 d0, d0, d1
+; CHECK-NEXT:    bx lr
+entry:
+  %z = call double @llvm.experimental.vector.reduce.fmin.v2f64(<2 x double> %x)
+  ret double %z
+}
+
+define arm_aapcs_vfpcc double @fmin_v4f64_nofast(<4 x double> %x) {
+; CHECK-LABEL: fmin_v4f64_nofast:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.f64 d3, d1
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    vcmp.f64 d2, d0
+; CHECK-NEXT:    vselgt.f64 d4, d1, d3
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    vselgt.f64 d0, d0, d2
+; CHECK-NEXT:    vcmp.f64 d4, d0
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    vselgt.f64 d0, d0, d4
+; CHECK-NEXT:    bx lr
+entry:
+  %z = call double @llvm.experimental.vector.reduce.fmin.v4f64(<4 x double> %x)
+  ret double %z
+}
+
+define arm_aapcs_vfpcc float @fmin_v2f32_acc(<2 x float> %x, float %y) {
+; CHECK-LABEL: fmin_v2f32_acc:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldr s6, .LCPI18_0
+; CHECK-NEXT:    vminnm.f32 s0, s0, s1
+; CHECK-NEXT:    vminnm.f32 s0, s0, s6
+; CHECK-NEXT:    vminnm.f32 s0, s0, s6
+; CHECK-NEXT:    vminnm.f32 s0, s4, s0
+; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    .p2align 2
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI18_0:
+; CHECK-NEXT:    .long 2139095040 @ float +Inf
+entry:
+  %z = call fast float @llvm.experimental.vector.reduce.fmin.v2f32(<2 x float> %x)
+  %c = fcmp fast olt float %y, %z
+  %r = select i1 %c, float %y, float %z
+  ret float %r
+}
+
+define arm_aapcs_vfpcc float @fmin_v4f32_acc(<4 x float> %x, float %y) {
+; CHECK-LABEL: fmin_v4f32_acc:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vminnm.f32 s6, s0, s1
+; CHECK-NEXT:    vminnm.f32 s6, s6, s2
+; CHECK-NEXT:    vminnm.f32 s0, s6, s3
+; CHECK-NEXT:    vminnm.f32 s0, s4, s0
+; CHECK-NEXT:    bx lr
+entry:
+  %z = call fast float @llvm.experimental.vector.reduce.fmin.v4f32(<4 x float> %x)
+  %c = fcmp fast olt float %y, %z
+  %r = select i1 %c, float %y, float %z
+  ret float %r
+}
+
+define arm_aapcs_vfpcc float @fmin_v8f32_acc(<8 x float> %x, float %y) {
+; CHECK-FP-LABEL: fmin_v8f32_acc:
+; CHECK-FP:       @ %bb.0: @ %entry
+; CHECK-FP-NEXT:    vminnm.f32 q0, q0, q1
+; CHECK-FP-NEXT:    vminnm.f32 s4, s0, s1
+; CHECK-FP-NEXT:    vminnm.f32 s4, s4, s2
+; CHECK-FP-NEXT:    vminnm.f32 s0, s4, s3
+; CHECK-FP-NEXT:    vminnm.f32 s0, s8, s0
+; CHECK-FP-NEXT:    bx lr
+;
+; CHECK-NOFP-LABEL: fmin_v8f32_acc:
+; CHECK-NOFP:       @ %bb.0: @ %entry
+; CHECK-NOFP-NEXT:    vcmp.f32 s5, s1
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f32 s4, s0
+; CHECK-NOFP-NEXT:    vselgt.f32 s10, s1, s5
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f32 s6, s2
+; CHECK-NOFP-NEXT:    vselgt.f32 s12, s0, s4
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f32 s7, s3
+; CHECK-NOFP-NEXT:    vselgt.f32 s14, s2, s6
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f32 s0, s3, s7
+; CHECK-NOFP-NEXT:    vminnm.f32 s2, s12, s10
+; CHECK-NOFP-NEXT:    vminnm.f32 s2, s2, s14
+; CHECK-NOFP-NEXT:    vminnm.f32 s0, s2, s0
+; CHECK-NOFP-NEXT:    vminnm.f32 s0, s8, s0
+; CHECK-NOFP-NEXT:    bx lr
+entry:
+  %z = call fast float @llvm.experimental.vector.reduce.fmin.v8f32(<8 x float> %x)
+  %c = fcmp fast olt float %y, %z
+  %r = select i1 %c, float %y, float %z
+  ret float %r
+}
+
+define arm_aapcs_vfpcc void @fmin_v4f16_acc(<4 x half> %x, half* %yy) {
+; CHECK-LABEL: fmin_v4f16_acc:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmovx.f16 s4, s0
+; CHECK-NEXT:    vminnm.f16 s4, s0, s4
+; CHECK-NEXT:    vmovx.f16 s0, s1
+; CHECK-NEXT:    vminnm.f16 s4, s4, s1
+; CHECK-NEXT:    vldr.16 s2, .LCPI21_0
+; CHECK-NEXT:    vminnm.f16 s0, s4, s0
+; CHECK-NEXT:    vminnm.f16 s0, s0, s2
+; CHECK-NEXT:    vminnm.f16 s0, s0, s2
+; CHECK-NEXT:    vminnm.f16 s0, s0, s2
+; CHECK-NEXT:    vminnm.f16 s0, s0, s2
+; CHECK-NEXT:    vldr.16 s2, [r0]
+; CHECK-NEXT:    vminnm.f16 s0, s2, s0
+; CHECK-NEXT:    vstr.16 s0, [r0]
+; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    .p2align 1
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI21_0:
+; CHECK-NEXT:    .short 31744 @ half +Inf
+entry:
+  %y = load half, half* %yy
+  %z = call fast half @llvm.experimental.vector.reduce.fmin.v4f16(<4 x half> %x)
+  %c = fcmp fast olt half %y, %z
+  %r = select i1 %c, half %y, half %z
+  store half %r, half* %yy
+  ret void
+}
+
+define arm_aapcs_vfpcc void @fmin_v8f16_acc(<8 x half> %x, half* %yy) {
+; CHECK-LABEL: fmin_v8f16_acc:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmovx.f16 s4, s0
+; CHECK-NEXT:    vmovx.f16 s6, s1
+; CHECK-NEXT:    vminnm.f16 s4, s0, s4
+; CHECK-NEXT:    vmovx.f16 s0, s3
+; CHECK-NEXT:    vminnm.f16 s4, s4, s1
+; CHECK-NEXT:    vminnm.f16 s4, s4, s6
+; CHECK-NEXT:    vmovx.f16 s6, s2
+; CHECK-NEXT:    vminnm.f16 s4, s4, s2
+; CHECK-NEXT:    vldr.16 s2, [r0]
+; CHECK-NEXT:    vminnm.f16 s4, s4, s6
+; CHECK-NEXT:    vminnm.f16 s4, s4, s3
+; CHECK-NEXT:    vminnm.f16 s0, s4, s0
+; CHECK-NEXT:    vminnm.f16 s0, s2, s0
+; CHECK-NEXT:    vstr.16 s0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %y = load half, half* %yy
+  %z = call fast half @llvm.experimental.vector.reduce.fmin.v8f16(<8 x half> %x)
+  %c = fcmp fast olt half %y, %z
+  %r = select i1 %c, half %y, half %z
+  store half %r, half* %yy
+  ret void
+}
+
+define arm_aapcs_vfpcc void @fmin_v16f16_acc(<16 x half> %x, half* %yy) {
+; CHECK-FP-LABEL: fmin_v16f16_acc:
+; CHECK-FP:       @ %bb.0: @ %entry
+; CHECK-FP-NEXT:    vminnm.f16 q0, q0, q1
+; CHECK-FP-NEXT:    vmovx.f16 s4, s0
+; CHECK-FP-NEXT:    vmovx.f16 s6, s1
+; CHECK-FP-NEXT:    vminnm.f16 s4, s0, s4
+; CHECK-FP-NEXT:    vmovx.f16 s0, s3
+; CHECK-FP-NEXT:    vminnm.f16 s4, s4, s1
+; CHECK-FP-NEXT:    vminnm.f16 s4, s4, s6
+; CHECK-FP-NEXT:    vmovx.f16 s6, s2
+; CHECK-FP-NEXT:    vminnm.f16 s4, s4, s2
+; CHECK-FP-NEXT:    vldr.16 s2, [r0]
+; CHECK-FP-NEXT:    vminnm.f16 s4, s4, s6
+; CHECK-FP-NEXT:    vminnm.f16 s4, s4, s3
+; CHECK-FP-NEXT:    vminnm.f16 s0, s4, s0
+; CHECK-FP-NEXT:    vminnm.f16 s0, s2, s0
+; CHECK-FP-NEXT:    vstr.16 s0, [r0]
+; CHECK-FP-NEXT:    bx lr
+;
+; CHECK-NOFP-LABEL: fmin_v16f16_acc:
+; CHECK-NOFP:       @ %bb.0: @ %entry
+; CHECK-NOFP-NEXT:    vmovx.f16 s8, s4
+; CHECK-NOFP-NEXT:    vmovx.f16 s10, s0
+; CHECK-NOFP-NEXT:    vcmp.f16 s8, s10
+; CHECK-NOFP-NEXT:    vmovx.f16 s12, s1
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f16 s4, s0
+; CHECK-NOFP-NEXT:    vselgt.f16 s8, s10, s8
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f16 s5, s1
+; CHECK-NOFP-NEXT:    vselgt.f16 s10, s0, s4
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vminnm.f16 s8, s10, s8
+; CHECK-NOFP-NEXT:    vmovx.f16 s4, s7
+; CHECK-NOFP-NEXT:    vmovx.f16 s0, s3
+; CHECK-NOFP-NEXT:    vselgt.f16 s10, s1, s5
+; CHECK-NOFP-NEXT:    vminnm.f16 s8, s8, s10
+; CHECK-NOFP-NEXT:    vmovx.f16 s10, s5
+; CHECK-NOFP-NEXT:    vcmp.f16 s10, s12
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f16 s6, s2
+; CHECK-NOFP-NEXT:    vselgt.f16 s10, s12, s10
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vminnm.f16 s8, s8, s10
+; CHECK-NOFP-NEXT:    vmovx.f16 s12, s2
+; CHECK-NOFP-NEXT:    vselgt.f16 s10, s2, s6
+; CHECK-NOFP-NEXT:    vldr.16 s2, [r0]
+; CHECK-NOFP-NEXT:    vminnm.f16 s8, s8, s10
+; CHECK-NOFP-NEXT:    vmovx.f16 s10, s6
+; CHECK-NOFP-NEXT:    vcmp.f16 s10, s12
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f16 s7, s3
+; CHECK-NOFP-NEXT:    vselgt.f16 s10, s12, s10
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f16 s4, s0
+; CHECK-NOFP-NEXT:    vminnm.f16 s8, s8, s10
+; CHECK-NOFP-NEXT:    vselgt.f16 s10, s3, s7
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vminnm.f16 s8, s8, s10
+; CHECK-NOFP-NEXT:    vselgt.f16 s0, s0, s4
+; CHECK-NOFP-NEXT:    vminnm.f16 s0, s8, s0
+; CHECK-NOFP-NEXT:    vminnm.f16 s0, s2, s0
+; CHECK-NOFP-NEXT:    vstr.16 s0, [r0]
+; CHECK-NOFP-NEXT:    bx lr
+entry:
+  %y = load half, half* %yy
+  %z = call fast half @llvm.experimental.vector.reduce.fmin.v16f16(<16 x half> %x)
+  %c = fcmp fast olt half %y, %z
+  %r = select i1 %c, half %y, half %z
+  store half %r, half* %yy
+  ret void
+}
+
+define arm_aapcs_vfpcc double @fmin_v1f64_acc(<1 x double> %x, double %y) {
+; CHECK-LABEL: fmin_v1f64_acc:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vminnm.f64 d0, d1, d0
+; CHECK-NEXT:    bx lr
+entry:
+  %z = call fast double @llvm.experimental.vector.reduce.fmin.v1f64(<1 x double> %x)
+  %c = fcmp fast olt double %y, %z
+  %r = select i1 %c, double %y, double %z
+  ret double %r
+}
+
+define arm_aapcs_vfpcc double @fmin_v2f64_acc(<2 x double> %x, double %y) {
+; CHECK-LABEL: fmin_v2f64_acc:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vminnm.f64 d0, d0, d1
+; CHECK-NEXT:    vminnm.f64 d0, d2, d0
+; CHECK-NEXT:    bx lr
+entry:
+  %z = call fast double @llvm.experimental.vector.reduce.fmin.v2f64(<2 x double> %x)
+  %c = fcmp fast olt double %y, %z
+  %r = select i1 %c, double %y, double %z
+  ret double %r
+}
+
+define arm_aapcs_vfpcc double @fmin_v4f64_acc(<4 x double> %x, double %y) {
+; CHECK-LABEL: fmin_v4f64_acc:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.f64 d3, d1
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    vcmp.f64 d2, d0
+; CHECK-NEXT:    vselgt.f64 d5, d1, d3
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    vselgt.f64 d0, d0, d2
+; CHECK-NEXT:    vminnm.f64 d0, d0, d5
+; CHECK-NEXT:    vminnm.f64 d0, d4, d0
+; CHECK-NEXT:    bx lr
+entry:
+  %z = call fast double @llvm.experimental.vector.reduce.fmin.v4f64(<4 x double> %x)
+  %c = fcmp fast olt double %y, %z
+  %r = select i1 %c, double %y, double %z
+  ret double %r
+}
+
+define arm_aapcs_vfpcc float @fmin_v2f32_acc_nofast(<2 x float> %x, float %y) {
+; CHECK-FP-LABEL: fmin_v2f32_acc_nofast:
+; CHECK-FP:       @ %bb.0: @ %entry
+; CHECK-FP-NEXT:    vmov.32 r0, q0[1]
+; CHECK-FP-NEXT:    vdup.32 q2, r0
+; CHECK-FP-NEXT:    vminnm.f32 q0, q0, q2
+; CHECK-FP-NEXT:    vcmp.f32 s0, s4
+; CHECK-FP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-FP-NEXT:    vselgt.f32 s0, s4, s0
+; CHECK-FP-NEXT:    bx lr
+;
+; CHECK-NOFP-LABEL: fmin_v2f32_acc_nofast:
+; CHECK-NOFP:       @ %bb.0: @ %entry
+; CHECK-NOFP-NEXT:    vmov.32 r0, q0[1]
+; CHECK-NOFP-NEXT:    vdup.32 q2, r0
+; CHECK-NOFP-NEXT:    vcmp.f32 s8, s0
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f32 s0, s0, s8
+; CHECK-NOFP-NEXT:    vcmp.f32 s0, s4
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f32 s0, s4, s0
+; CHECK-NOFP-NEXT:    bx lr
+entry:
+  %z = call float @llvm.experimental.vector.reduce.fmin.v2f32(<2 x float> %x)
+  %c = fcmp olt float %y, %z
+  %r = select i1 %c, float %y, float %z
+  ret float %r
+}
+
+define arm_aapcs_vfpcc float @fmin_v4f32_acc_nofast(<4 x float> %x, float %y) {
+; CHECK-FP-LABEL: fmin_v4f32_acc_nofast:
+; CHECK-FP:       @ %bb.0: @ %entry
+; CHECK-FP-NEXT:    vmov.f64 d4, d1
+; CHECK-FP-NEXT:    vmov.f32 s9, s3
+; CHECK-FP-NEXT:    vminnm.f32 q0, q0, q2
+; CHECK-FP-NEXT:    vmov.32 r0, q0[1]
+; CHECK-FP-NEXT:    vdup.32 q2, r0
+; CHECK-FP-NEXT:    vminnm.f32 q0, q0, q2
+; CHECK-FP-NEXT:    vcmp.f32 s0, s4
+; CHECK-FP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-FP-NEXT:    vselgt.f32 s0, s4, s0
+; CHECK-FP-NEXT:    bx lr
+;
+; CHECK-NOFP-LABEL: fmin_v4f32_acc_nofast:
+; CHECK-NOFP:       @ %bb.0: @ %entry
+; CHECK-NOFP-NEXT:    vcmp.f32 s3, s1
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vmov.f64 d4, d1
+; CHECK-NOFP-NEXT:    vmov.f32 s9, s3
+; CHECK-NOFP-NEXT:    vcmp.f32 s8, s0
+; CHECK-NOFP-NEXT:    vselgt.f32 s6, s1, s3
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f32 s0, s0, s8
+; CHECK-NOFP-NEXT:    vcmp.f32 s6, s0
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f32 s0, s0, s6
+; CHECK-NOFP-NEXT:    vcmp.f32 s0, s4
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f32 s0, s4, s0
+; CHECK-NOFP-NEXT:    bx lr
+entry:
+  %z = call float @llvm.experimental.vector.reduce.fmin.v4f32(<4 x float> %x)
+  %c = fcmp olt float %y, %z
+  %r = select i1 %c, float %y, float %z
+  ret float %r
+}
+
+define arm_aapcs_vfpcc float @fmin_v8f32_acc_nofast(<8 x float> %x, float %y) {
+; CHECK-FP-LABEL: fmin_v8f32_acc_nofast:
+; CHECK-FP:       @ %bb.0: @ %entry
+; CHECK-FP-NEXT:    vminnm.f32 q0, q0, q1
+; CHECK-FP-NEXT:    vmov.f64 d2, d1
+; CHECK-FP-NEXT:    vmov.f32 s5, s3
+; CHECK-FP-NEXT:    vminnm.f32 q0, q0, q1
+; CHECK-FP-NEXT:    vmov.32 r0, q0[1]
+; CHECK-FP-NEXT:    vdup.32 q1, r0
+; CHECK-FP-NEXT:    vminnm.f32 q0, q0, q1
+; CHECK-FP-NEXT:    vcmp.f32 s0, s8
+; CHECK-FP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-FP-NEXT:    vselgt.f32 s0, s8, s0
+; CHECK-FP-NEXT:    bx lr
+;
+; CHECK-NOFP-LABEL: fmin_v8f32_acc_nofast:
+; CHECK-NOFP:       @ %bb.0: @ %entry
+; CHECK-NOFP-NEXT:    vcmp.f32 s7, s3
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f32 s5, s1
+; CHECK-NOFP-NEXT:    vselgt.f32 s10, s3, s7
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f32 s6, s2
+; CHECK-NOFP-NEXT:    vselgt.f32 s12, s1, s5
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f32 s4, s0
+; CHECK-NOFP-NEXT:    vselgt.f32 s14, s2, s6
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f32 s10, s12
+; CHECK-NOFP-NEXT:    vselgt.f32 s0, s0, s4
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f32 s14, s0
+; CHECK-NOFP-NEXT:    vselgt.f32 s2, s12, s10
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f32 s0, s0, s14
+; CHECK-NOFP-NEXT:    vcmp.f32 s2, s0
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f32 s0, s0, s2
+; CHECK-NOFP-NEXT:    vcmp.f32 s0, s8
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f32 s0, s8, s0
+; CHECK-NOFP-NEXT:    bx lr
+entry:
+  %z = call float @llvm.experimental.vector.reduce.fmin.v8f32(<8 x float> %x)
+  %c = fcmp olt float %y, %z
+  %r = select i1 %c, float %y, float %z
+  ret float %r
+}
+
+define arm_aapcs_vfpcc void @fmin_v4f16_acc_nofast(<4 x half> %x, half* %yy) {
+; CHECK-FP-LABEL: fmin_v4f16_acc_nofast:
+; CHECK-FP:       @ %bb.0: @ %entry
+; CHECK-FP-NEXT:    vmov.32 r1, q0[1]
+; CHECK-FP-NEXT:    vdup.32 q1, r1
+; CHECK-FP-NEXT:    vminnm.f16 q0, q0, q1
+; CHECK-FP-NEXT:    vmov.u16 r1, q0[1]
+; CHECK-FP-NEXT:    vdup.16 q1, r1
+; CHECK-FP-NEXT:    vminnm.f16 q0, q0, q1
+; CHECK-FP-NEXT:    vldr.16 s4, [r0]
+; CHECK-FP-NEXT:    vcmp.f16 s0, s4
+; CHECK-FP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-FP-NEXT:    vselgt.f16 s0, s4, s0
+; CHECK-FP-NEXT:    vstr.16 s0, [r0]
+; CHECK-FP-NEXT:    bx lr
+;
+; CHECK-NOFP-LABEL: fmin_v4f16_acc_nofast:
+; CHECK-NOFP:       @ %bb.0: @ %entry
+; CHECK-NOFP-NEXT:    vmov.32 r1, q0[1]
+; CHECK-NOFP-NEXT:    vmovx.f16 s10, s0
+; CHECK-NOFP-NEXT:    vdup.32 q1, r1
+; CHECK-NOFP-NEXT:    vmovx.f16 s8, s4
+; CHECK-NOFP-NEXT:    vcmp.f16 s8, s10
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f16 s4, s0
+; CHECK-NOFP-NEXT:    vselgt.f16 s8, s10, s8
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s0, s0, s4
+; CHECK-NOFP-NEXT:    vldr.16 s2, [r0]
+; CHECK-NOFP-NEXT:    vcmp.f16 s8, s0
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s0, s0, s8
+; CHECK-NOFP-NEXT:    vcmp.f16 s0, s2
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s0, s2, s0
+; CHECK-NOFP-NEXT:    vstr.16 s0, [r0]
+; CHECK-NOFP-NEXT:    bx lr
+entry:
+  %y = load half, half* %yy
+  %z = call half @llvm.experimental.vector.reduce.fmin.v4f16(<4 x half> %x)
+  %c = fcmp olt half %y, %z
+  %r = select i1 %c, half %y, half %z
+  store half %r, half* %yy
+  ret void
+}
+
+define arm_aapcs_vfpcc void @fmin_v8f16_acc_nofast(<8 x half> %x, half* %yy) {
+; CHECK-FP-LABEL: fmin_v8f16_acc_nofast:
+; CHECK-FP:       @ %bb.0: @ %entry
+; CHECK-FP-NEXT:    vmov.f64 d2, d1
+; CHECK-FP-NEXT:    vmov.f32 s5, s3
+; CHECK-FP-NEXT:    vminnm.f16 q0, q0, q1
+; CHECK-FP-NEXT:    vmov.32 r1, q0[1]
+; CHECK-FP-NEXT:    vdup.32 q1, r1
+; CHECK-FP-NEXT:    vminnm.f16 q0, q0, q1
+; CHECK-FP-NEXT:    vmov.u16 r1, q0[1]
+; CHECK-FP-NEXT:    vdup.16 q1, r1
+; CHECK-FP-NEXT:    vminnm.f16 q0, q0, q1
+; CHECK-FP-NEXT:    vldr.16 s4, [r0]
+; CHECK-FP-NEXT:    vcmp.f16 s0, s4
+; CHECK-FP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-FP-NEXT:    vselgt.f16 s0, s4, s0
+; CHECK-FP-NEXT:    vstr.16 s0, [r0]
+; CHECK-FP-NEXT:    bx lr
+;
+; CHECK-NOFP-LABEL: fmin_v8f16_acc_nofast:
+; CHECK-NOFP:       @ %bb.0: @ %entry
+; CHECK-NOFP-NEXT:    vmovx.f16 s8, s3
+; CHECK-NOFP-NEXT:    vmovx.f16 s10, s1
+; CHECK-NOFP-NEXT:    vcmp.f16 s8, s10
+; CHECK-NOFP-NEXT:    vmov.f64 d2, d1
+; CHECK-NOFP-NEXT:    vmovx.f16 s12, s0
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vmov.f32 s5, s3
+; CHECK-NOFP-NEXT:    vselgt.f16 s8, s10, s8
+; CHECK-NOFP-NEXT:    vmovx.f16 s10, s4
+; CHECK-NOFP-NEXT:    vcmp.f16 s10, s12
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s10, s12, s10
+; CHECK-NOFP-NEXT:    vcmp.f16 s8, s10
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f16 s3, s1
+; CHECK-NOFP-NEXT:    vselgt.f16 s8, s10, s8
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f16 s4, s0
+; CHECK-NOFP-NEXT:    vselgt.f16 s10, s1, s3
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s0, s0, s4
+; CHECK-NOFP-NEXT:    vldr.16 s2, [r0]
+; CHECK-NOFP-NEXT:    vcmp.f16 s10, s0
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s0, s0, s10
+; CHECK-NOFP-NEXT:    vcmp.f16 s8, s0
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s0, s0, s8
+; CHECK-NOFP-NEXT:    vcmp.f16 s0, s2
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s0, s2, s0
+; CHECK-NOFP-NEXT:    vstr.16 s0, [r0]
+; CHECK-NOFP-NEXT:    bx lr
+entry:
+  %y = load half, half* %yy
+  %z = call half @llvm.experimental.vector.reduce.fmin.v8f16(<8 x half> %x)
+  %c = fcmp olt half %y, %z
+  %r = select i1 %c, half %y, half %z
+  store half %r, half* %yy
+  ret void
+}
+
+define arm_aapcs_vfpcc void @fmin_v16f16_acc_nofast(<16 x half> %x, half* %yy) {
+; CHECK-FP-LABEL: fmin_v16f16_acc_nofast:
+; CHECK-FP:       @ %bb.0: @ %entry
+; CHECK-FP-NEXT:    vminnm.f16 q0, q0, q1
+; CHECK-FP-NEXT:    vmov.f64 d2, d1
+; CHECK-FP-NEXT:    vmov.f32 s5, s3
+; CHECK-FP-NEXT:    vminnm.f16 q0, q0, q1
+; CHECK-FP-NEXT:    vmov.32 r1, q0[1]
+; CHECK-FP-NEXT:    vdup.32 q1, r1
+; CHECK-FP-NEXT:    vminnm.f16 q0, q0, q1
+; CHECK-FP-NEXT:    vmov.u16 r1, q0[1]
+; CHECK-FP-NEXT:    vdup.16 q1, r1
+; CHECK-FP-NEXT:    vminnm.f16 q0, q0, q1
+; CHECK-FP-NEXT:    vldr.16 s4, [r0]
+; CHECK-FP-NEXT:    vcmp.f16 s0, s4
+; CHECK-FP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-FP-NEXT:    vselgt.f16 s0, s4, s0
+; CHECK-FP-NEXT:    vstr.16 s0, [r0]
+; CHECK-FP-NEXT:    bx lr
+;
+; CHECK-NOFP-LABEL: fmin_v16f16_acc_nofast:
+; CHECK-NOFP:       @ %bb.0: @ %entry
+; CHECK-NOFP-NEXT:    vmovx.f16 s8, s7
+; CHECK-NOFP-NEXT:    vmovx.f16 s10, s3
+; CHECK-NOFP-NEXT:    vcmp.f16 s8, s10
+; CHECK-NOFP-NEXT:    vmovx.f16 s12, s1
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vmovx.f16 s14, s0
+; CHECK-NOFP-NEXT:    vselgt.f16 s8, s10, s8
+; CHECK-NOFP-NEXT:    vmovx.f16 s10, s5
+; CHECK-NOFP-NEXT:    vcmp.f16 s10, s12
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s10, s12, s10
+; CHECK-NOFP-NEXT:    vmovx.f16 s12, s2
+; CHECK-NOFP-NEXT:    vcmp.f16 s8, s10
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s8, s10, s8
+; CHECK-NOFP-NEXT:    vmovx.f16 s10, s6
+; CHECK-NOFP-NEXT:    vcmp.f16 s10, s12
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s10, s12, s10
+; CHECK-NOFP-NEXT:    vmovx.f16 s12, s4
+; CHECK-NOFP-NEXT:    vcmp.f16 s12, s14
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s12, s14, s12
+; CHECK-NOFP-NEXT:    vcmp.f16 s10, s12
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s10, s12, s10
+; CHECK-NOFP-NEXT:    vcmp.f16 s8, s10
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f16 s7, s3
+; CHECK-NOFP-NEXT:    vselgt.f16 s8, s10, s8
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f16 s5, s1
+; CHECK-NOFP-NEXT:    vselgt.f16 s10, s3, s7
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s12, s1, s5
+; CHECK-NOFP-NEXT:    vcmp.f16 s10, s12
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f16 s6, s2
+; CHECK-NOFP-NEXT:    vselgt.f16 s10, s12, s10
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f16 s4, s0
+; CHECK-NOFP-NEXT:    vselgt.f16 s12, s2, s6
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s0, s0, s4
+; CHECK-NOFP-NEXT:    vldr.16 s2, [r0]
+; CHECK-NOFP-NEXT:    vcmp.f16 s12, s0
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s0, s0, s12
+; CHECK-NOFP-NEXT:    vcmp.f16 s10, s0
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s0, s0, s10
+; CHECK-NOFP-NEXT:    vcmp.f16 s8, s0
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s0, s0, s8
+; CHECK-NOFP-NEXT:    vcmp.f16 s0, s2
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s0, s2, s0
+; CHECK-NOFP-NEXT:    vstr.16 s0, [r0]
+; CHECK-NOFP-NEXT:    bx lr
+entry:
+  %y = load half, half* %yy
+  %z = call half @llvm.experimental.vector.reduce.fmin.v16f16(<16 x half> %x)
+  %c = fcmp olt half %y, %z
+  %r = select i1 %c, half %y, half %z
+  store half %r, half* %yy
+  ret void
+}
+
+define arm_aapcs_vfpcc double @fmin_v1f64_acc_nofast(<1 x double> %x, double %y) {
+; CHECK-LABEL: fmin_v1f64_acc_nofast:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.f64 d0, d1
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    vselgt.f64 d0, d1, d0
+; CHECK-NEXT:    bx lr
+entry:
+  %z = call double @llvm.experimental.vector.reduce.fmin.v1f64(<1 x double> %x)
+  %c = fcmp olt double %y, %z
+  %r = select i1 %c, double %y, double %z
+  ret double %r
+}
+
+define arm_aapcs_vfpcc double @fmin_v2f64_acc_nofast(<2 x double> %x, double %y) {
+; CHECK-LABEL: fmin_v2f64_acc_nofast:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.f64 d1, d0
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    vselgt.f64 d0, d0, d1
+; CHECK-NEXT:    vcmp.f64 d0, d2
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    vselgt.f64 d0, d2, d0
+; CHECK-NEXT:    bx lr
+entry:
+  %z = call double @llvm.experimental.vector.reduce.fmin.v2f64(<2 x double> %x)
+  %c = fcmp olt double %y, %z
+  %r = select i1 %c, double %y, double %z
+  ret double %r
+}
+
+define arm_aapcs_vfpcc double @fmin_v4f64_acc_nofast(<4 x double> %x, double %y) {
+; CHECK-LABEL: fmin_v4f64_acc_nofast:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.f64 d3, d1
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    vcmp.f64 d2, d0
+; CHECK-NEXT:    vselgt.f64 d5, d1, d3
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    vselgt.f64 d0, d0, d2
+; CHECK-NEXT:    vcmp.f64 d5, d0
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    vselgt.f64 d0, d0, d5
+; CHECK-NEXT:    vcmp.f64 d0, d4
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    vselgt.f64 d0, d4, d0
+; CHECK-NEXT:    bx lr
+entry:
+  %z = call double @llvm.experimental.vector.reduce.fmin.v4f64(<4 x double> %x)
+  %c = fcmp olt double %y, %z
+  %r = select i1 %c, double %y, double %z
+  ret double %r
+}
+
+define arm_aapcs_vfpcc float @fmax_v2f32(<2 x float> %x) {
+; CHECK-LABEL: fmax_v2f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldr s4, .LCPI36_0
+; CHECK-NEXT:    vmaxnm.f32 s0, s0, s1
+; CHECK-NEXT:    vmaxnm.f32 s0, s0, s4
+; CHECK-NEXT:    vmaxnm.f32 s0, s0, s4
+; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    .p2align 2
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI36_0:
+; CHECK-NEXT:    .long 4286578688 @ float -Inf
+entry:
+  %z = call fast float @llvm.experimental.vector.reduce.fmax.v2f32(<2 x float> %x)
+  ret float %z
+}
+
+define arm_aapcs_vfpcc float @fmax_v4f32(<4 x float> %x) {
+; CHECK-LABEL: fmax_v4f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmaxnm.f32 s4, s0, s1
+; CHECK-NEXT:    vmaxnm.f32 s4, s4, s2
+; CHECK-NEXT:    vmaxnm.f32 s0, s4, s3
+; CHECK-NEXT:    bx lr
+entry:
+  %z = call fast float @llvm.experimental.vector.reduce.fmax.v4f32(<4 x float> %x)
+  ret float %z
+}
+
+define arm_aapcs_vfpcc float @fmax_v8f32(<8 x float> %x) {
+; CHECK-FP-LABEL: fmax_v8f32:
+; CHECK-FP:       @ %bb.0: @ %entry
+; CHECK-FP-NEXT:    vmaxnm.f32 q0, q0, q1
+; CHECK-FP-NEXT:    vmaxnm.f32 s4, s0, s1
+; CHECK-FP-NEXT:    vmaxnm.f32 s4, s4, s2
+; CHECK-FP-NEXT:    vmaxnm.f32 s0, s4, s3
+; CHECK-FP-NEXT:    bx lr
+;
+; CHECK-NOFP-LABEL: fmax_v8f32:
+; CHECK-NOFP:       @ %bb.0: @ %entry
+; CHECK-NOFP-NEXT:    vcmp.f32 s1, s5
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f32 s0, s4
+; CHECK-NOFP-NEXT:    vselgt.f32 s8, s1, s5
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f32 s2, s6
+; CHECK-NOFP-NEXT:    vselgt.f32 s10, s0, s4
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f32 s3, s7
+; CHECK-NOFP-NEXT:    vselgt.f32 s12, s2, s6
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f32 s0, s3, s7
+; CHECK-NOFP-NEXT:    vmaxnm.f32 s2, s10, s8
+; CHECK-NOFP-NEXT:    vmaxnm.f32 s2, s2, s12
+; CHECK-NOFP-NEXT:    vmaxnm.f32 s0, s2, s0
+; CHECK-NOFP-NEXT:    bx lr
+entry:
+  %z = call fast float @llvm.experimental.vector.reduce.fmax.v8f32(<8 x float> %x)
+  ret float %z
+}
+
+define arm_aapcs_vfpcc half @fmax_v4f16(<4 x half> %x) {
+; CHECK-LABEL: fmax_v4f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmovx.f16 s4, s0
+; CHECK-NEXT:    vmaxnm.f16 s4, s0, s4
+; CHECK-NEXT:    vmovx.f16 s0, s1
+; CHECK-NEXT:    vmaxnm.f16 s4, s4, s1
+; CHECK-NEXT:    vldr.16 s2, .LCPI39_0
+; CHECK-NEXT:    vmaxnm.f16 s0, s4, s0
+; CHECK-NEXT:    vmaxnm.f16 s0, s0, s2
+; CHECK-NEXT:    vmaxnm.f16 s0, s0, s2
+; CHECK-NEXT:    vmaxnm.f16 s0, s0, s2
+; CHECK-NEXT:    vmaxnm.f16 s0, s0, s2
+; CHECK-NEXT:    vstr.16 s0, [r0]
+; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    .p2align 1
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI39_0:
+; CHECK-NEXT:    .short 64512 @ half -Inf
+entry:
+  %z = call fast half @llvm.experimental.vector.reduce.fmax.v4f16(<4 x half> %x)
+  ret half %z
+}
+
+define arm_aapcs_vfpcc half @fmax_v8f16(<8 x half> %x) {
+; CHECK-LABEL: fmax_v8f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmovx.f16 s4, s0
+; CHECK-NEXT:    vmovx.f16 s6, s1
+; CHECK-NEXT:    vmaxnm.f16 s4, s0, s4
+; CHECK-NEXT:    vmovx.f16 s0, s3
+; CHECK-NEXT:    vmaxnm.f16 s4, s4, s1
+; CHECK-NEXT:    vmaxnm.f16 s4, s4, s6
+; CHECK-NEXT:    vmovx.f16 s6, s2
+; CHECK-NEXT:    vmaxnm.f16 s4, s4, s2
+; CHECK-NEXT:    vmaxnm.f16 s4, s4, s6
+; CHECK-NEXT:    vmaxnm.f16 s4, s4, s3
+; CHECK-NEXT:    vmaxnm.f16 s0, s4, s0
+; CHECK-NEXT:    vstr.16 s0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %z = call fast half @llvm.experimental.vector.reduce.fmax.v8f16(<8 x half> %x)
+  ret half %z
+}
+
+define arm_aapcs_vfpcc half @fmax_v16f16(<16 x half> %x) {
+; CHECK-FP-LABEL: fmax_v16f16:
+; CHECK-FP:       @ %bb.0: @ %entry
+; CHECK-FP-NEXT:    vmaxnm.f16 q0, q0, q1
+; CHECK-FP-NEXT:    vmovx.f16 s4, s0
+; CHECK-FP-NEXT:    vmovx.f16 s6, s1
+; CHECK-FP-NEXT:    vmaxnm.f16 s4, s0, s4
+; CHECK-FP-NEXT:    vmovx.f16 s0, s3
+; CHECK-FP-NEXT:    vmaxnm.f16 s4, s4, s1
+; CHECK-FP-NEXT:    vmaxnm.f16 s4, s4, s6
+; CHECK-FP-NEXT:    vmovx.f16 s6, s2
+; CHECK-FP-NEXT:    vmaxnm.f16 s4, s4, s2
+; CHECK-FP-NEXT:    vmaxnm.f16 s4, s4, s6
+; CHECK-FP-NEXT:    vmaxnm.f16 s4, s4, s3
+; CHECK-FP-NEXT:    vmaxnm.f16 s0, s4, s0
+; CHECK-FP-NEXT:    vstr.16 s0, [r0]
+; CHECK-FP-NEXT:    bx lr
+;
+; CHECK-NOFP-LABEL: fmax_v16f16:
+; CHECK-NOFP:       @ %bb.0: @ %entry
+; CHECK-NOFP-NEXT:    vmovx.f16 s8, s4
+; CHECK-NOFP-NEXT:    vmovx.f16 s10, s0
+; CHECK-NOFP-NEXT:    vcmp.f16 s10, s8
+; CHECK-NOFP-NEXT:    vmovx.f16 s12, s1
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f16 s0, s4
+; CHECK-NOFP-NEXT:    vselgt.f16 s8, s10, s8
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f16 s1, s5
+; CHECK-NOFP-NEXT:    vselgt.f16 s10, s0, s4
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vmaxnm.f16 s8, s10, s8
+; CHECK-NOFP-NEXT:    vmovx.f16 s4, s7
+; CHECK-NOFP-NEXT:    vmovx.f16 s0, s3
+; CHECK-NOFP-NEXT:    vselgt.f16 s10, s1, s5
+; CHECK-NOFP-NEXT:    vmaxnm.f16 s8, s8, s10
+; CHECK-NOFP-NEXT:    vmovx.f16 s10, s5
+; CHECK-NOFP-NEXT:    vcmp.f16 s12, s10
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f16 s2, s6
+; CHECK-NOFP-NEXT:    vselgt.f16 s10, s12, s10
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vmaxnm.f16 s8, s8, s10
+; CHECK-NOFP-NEXT:    vmovx.f16 s12, s2
+; CHECK-NOFP-NEXT:    vselgt.f16 s10, s2, s6
+; CHECK-NOFP-NEXT:    vmaxnm.f16 s8, s8, s10
+; CHECK-NOFP-NEXT:    vmovx.f16 s10, s6
+; CHECK-NOFP-NEXT:    vcmp.f16 s12, s10
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f16 s3, s7
+; CHECK-NOFP-NEXT:    vselgt.f16 s10, s12, s10
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f16 s0, s4
+; CHECK-NOFP-NEXT:    vmaxnm.f16 s8, s8, s10
+; CHECK-NOFP-NEXT:    vselgt.f16 s10, s3, s7
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vmaxnm.f16 s8, s8, s10
+; CHECK-NOFP-NEXT:    vselgt.f16 s0, s0, s4
+; CHECK-NOFP-NEXT:    vmaxnm.f16 s0, s8, s0
+; CHECK-NOFP-NEXT:    vstr.16 s0, [r0]
+; CHECK-NOFP-NEXT:    bx lr
+entry:
+  %z = call fast half @llvm.experimental.vector.reduce.fmax.v16f16(<16 x half> %x)
+  ret half %z
+}
+
+define arm_aapcs_vfpcc double @fmax_v1f64(<1 x double> %x) {
+; CHECK-LABEL: fmax_v1f64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    bx lr
+entry:
+  %z = call fast double @llvm.experimental.vector.reduce.fmax.v1f64(<1 x double> %x)
+  ret double %z
+}
+
+define arm_aapcs_vfpcc double @fmax_v2f64(<2 x double> %x) {
+; CHECK-LABEL: fmax_v2f64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmaxnm.f64 d0, d0, d1
+; CHECK-NEXT:    bx lr
+entry:
+  %z = call fast double @llvm.experimental.vector.reduce.fmax.v2f64(<2 x double> %x)
+  ret double %z
+}
+
+define arm_aapcs_vfpcc double @fmax_v4f64(<4 x double> %x) {
+; CHECK-LABEL: fmax_v4f64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.f64 d1, d3
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    vcmp.f64 d0, d2
+; CHECK-NEXT:    vselgt.f64 d4, d1, d3
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    vselgt.f64 d0, d0, d2
+; CHECK-NEXT:    vmaxnm.f64 d0, d0, d4
+; CHECK-NEXT:    bx lr
+entry:
+  %z = call fast double @llvm.experimental.vector.reduce.fmax.v4f64(<4 x double> %x)
+  ret double %z
+}
+
+define arm_aapcs_vfpcc float @fmax_v2f32_nofast(<2 x float> %x) {
+; CHECK-FP-LABEL: fmax_v2f32_nofast:
+; CHECK-FP:       @ %bb.0: @ %entry
+; CHECK-FP-NEXT:    vmov.32 r0, q0[1]
+; CHECK-FP-NEXT:    vdup.32 q1, r0
+; CHECK-FP-NEXT:    vmaxnm.f32 q0, q0, q1
+; CHECK-FP-NEXT:    @ kill: def $s0 killed $s0 killed $q0
+; CHECK-FP-NEXT:    bx lr
+;
+; CHECK-NOFP-LABEL: fmax_v2f32_nofast:
+; CHECK-NOFP:       @ %bb.0: @ %entry
+; CHECK-NOFP-NEXT:    vmov.32 r0, q0[1]
+; CHECK-NOFP-NEXT:    vdup.32 q1, r0
+; CHECK-NOFP-NEXT:    vcmp.f32 s0, s4
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f32 s0, s0, s4
+; CHECK-NOFP-NEXT:    bx lr
+entry:
+  %z = call float @llvm.experimental.vector.reduce.fmax.v2f32(<2 x float> %x)
+  ret float %z
+}
+
+define arm_aapcs_vfpcc float @fmax_v4f32_nofast(<4 x float> %x) {
+; CHECK-FP-LABEL: fmax_v4f32_nofast:
+; CHECK-FP:       @ %bb.0: @ %entry
+; CHECK-FP-NEXT:    vmov.f64 d2, d1
+; CHECK-FP-NEXT:    vmov.f32 s5, s3
+; CHECK-FP-NEXT:    vmaxnm.f32 q0, q0, q1
+; CHECK-FP-NEXT:    vmov.32 r0, q0[1]
+; CHECK-FP-NEXT:    vdup.32 q1, r0
+; CHECK-FP-NEXT:    vmaxnm.f32 q0, q0, q1
+; CHECK-FP-NEXT:    @ kill: def $s0 killed $s0 killed $q0
+; CHECK-FP-NEXT:    bx lr
+;
+; CHECK-NOFP-LABEL: fmax_v4f32_nofast:
+; CHECK-NOFP:       @ %bb.0: @ %entry
+; CHECK-NOFP-NEXT:    vcmp.f32 s1, s3
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vmov.f64 d2, d1
+; CHECK-NOFP-NEXT:    vmov.f32 s5, s3
+; CHECK-NOFP-NEXT:    vcmp.f32 s0, s4
+; CHECK-NOFP-NEXT:    vselgt.f32 s8, s1, s3
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f32 s0, s0, s4
+; CHECK-NOFP-NEXT:    vcmp.f32 s0, s8
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f32 s0, s0, s8
+; CHECK-NOFP-NEXT:    bx lr
+entry:
+  %z = call float @llvm.experimental.vector.reduce.fmax.v4f32(<4 x float> %x)
+  ret float %z
+}
+
+define arm_aapcs_vfpcc float @fmax_v8f32_nofast(<8 x float> %x) {
+; CHECK-FP-LABEL: fmax_v8f32_nofast:
+; CHECK-FP:       @ %bb.0: @ %entry
+; CHECK-FP-NEXT:    vmaxnm.f32 q0, q0, q1
+; CHECK-FP-NEXT:    vmov.f64 d2, d1
+; CHECK-FP-NEXT:    vmov.f32 s5, s3
+; CHECK-FP-NEXT:    vmaxnm.f32 q0, q0, q1
+; CHECK-FP-NEXT:    vmov.32 r0, q0[1]
+; CHECK-FP-NEXT:    vdup.32 q1, r0
+; CHECK-FP-NEXT:    vmaxnm.f32 q0, q0, q1
+; CHECK-FP-NEXT:    @ kill: def $s0 killed $s0 killed $q0
+; CHECK-FP-NEXT:    bx lr
+;
+; CHECK-NOFP-LABEL: fmax_v8f32_nofast:
+; CHECK-NOFP:       @ %bb.0: @ %entry
+; CHECK-NOFP-NEXT:    vcmp.f32 s3, s7
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f32 s1, s5
+; CHECK-NOFP-NEXT:    vselgt.f32 s8, s3, s7
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f32 s2, s6
+; CHECK-NOFP-NEXT:    vselgt.f32 s10, s1, s5
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f32 s0, s4
+; CHECK-NOFP-NEXT:    vselgt.f32 s12, s2, s6
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f32 s10, s8
+; CHECK-NOFP-NEXT:    vselgt.f32 s0, s0, s4
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f32 s0, s12
+; CHECK-NOFP-NEXT:    vselgt.f32 s2, s10, s8
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f32 s0, s0, s12
+; CHECK-NOFP-NEXT:    vcmp.f32 s0, s2
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f32 s0, s0, s2
+; CHECK-NOFP-NEXT:    bx lr
+entry:
+  %z = call float @llvm.experimental.vector.reduce.fmax.v8f32(<8 x float> %x)
+  ret float %z
+}
+
+define arm_aapcs_vfpcc half @fmax_v4f16_nofast(<4 x half> %x) {
+; CHECK-FP-LABEL: fmax_v4f16_nofast:
+; CHECK-FP:       @ %bb.0: @ %entry
+; CHECK-FP-NEXT:    vmov.32 r1, q0[1]
+; CHECK-FP-NEXT:    vdup.32 q1, r1
+; CHECK-FP-NEXT:    vmaxnm.f16 q0, q0, q1
+; CHECK-FP-NEXT:    vmov.u16 r1, q0[1]
+; CHECK-FP-NEXT:    vdup.16 q1, r1
+; CHECK-FP-NEXT:    vmaxnm.f16 q0, q0, q1
+; CHECK-FP-NEXT:    vstr.16 s0, [r0]
+; CHECK-FP-NEXT:    bx lr
+;
+; CHECK-NOFP-LABEL: fmax_v4f16_nofast:
+; CHECK-NOFP:       @ %bb.0: @ %entry
+; CHECK-NOFP-NEXT:    vmov.32 r1, q0[1]
+; CHECK-NOFP-NEXT:    vmovx.f16 s10, s0
+; CHECK-NOFP-NEXT:    vdup.32 q1, r1
+; CHECK-NOFP-NEXT:    vmovx.f16 s8, s4
+; CHECK-NOFP-NEXT:    vcmp.f16 s10, s8
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f16 s0, s4
+; CHECK-NOFP-NEXT:    vselgt.f16 s8, s10, s8
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s0, s0, s4
+; CHECK-NOFP-NEXT:    vcmp.f16 s0, s8
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s0, s0, s8
+; CHECK-NOFP-NEXT:    vstr.16 s0, [r0]
+; CHECK-NOFP-NEXT:    bx lr
+entry:
+  %z = call half @llvm.experimental.vector.reduce.fmax.v4f16(<4 x half> %x)
+  ret half %z
+}
+
+define arm_aapcs_vfpcc half @fmax_v8f16_nofast(<8 x half> %x) {
+; CHECK-FP-LABEL: fmax_v8f16_nofast:
+; CHECK-FP:       @ %bb.0: @ %entry
+; CHECK-FP-NEXT:    vmov.f64 d2, d1
+; CHECK-FP-NEXT:    vmov.f32 s5, s3
+; CHECK-FP-NEXT:    vmaxnm.f16 q0, q0, q1
+; CHECK-FP-NEXT:    vmov.32 r1, q0[1]
+; CHECK-FP-NEXT:    vdup.32 q1, r1
+; CHECK-FP-NEXT:    vmaxnm.f16 q0, q0, q1
+; CHECK-FP-NEXT:    vmov.u16 r1, q0[1]
+; CHECK-FP-NEXT:    vdup.16 q1, r1
+; CHECK-FP-NEXT:    vmaxnm.f16 q0, q0, q1
+; CHECK-FP-NEXT:    vstr.16 s0, [r0]
+; CHECK-FP-NEXT:    bx lr
+;
+; CHECK-NOFP-LABEL: fmax_v8f16_nofast:
+; CHECK-NOFP:       @ %bb.0: @ %entry
+; CHECK-NOFP-NEXT:    vmovx.f16 s8, s3
+; CHECK-NOFP-NEXT:    vmovx.f16 s10, s1
+; CHECK-NOFP-NEXT:    vcmp.f16 s10, s8
+; CHECK-NOFP-NEXT:    vmov.f64 d2, d1
+; CHECK-NOFP-NEXT:    vmovx.f16 s12, s0
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vmov.f32 s5, s3
+; CHECK-NOFP-NEXT:    vselgt.f16 s8, s10, s8
+; CHECK-NOFP-NEXT:    vmovx.f16 s10, s4
+; CHECK-NOFP-NEXT:    vcmp.f16 s12, s10
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s10, s12, s10
+; CHECK-NOFP-NEXT:    vcmp.f16 s10, s8
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f16 s1, s3
+; CHECK-NOFP-NEXT:    vselgt.f16 s8, s10, s8
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f16 s0, s4
+; CHECK-NOFP-NEXT:    vselgt.f16 s10, s1, s3
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s0, s0, s4
+; CHECK-NOFP-NEXT:    vcmp.f16 s0, s10
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s0, s0, s10
+; CHECK-NOFP-NEXT:    vcmp.f16 s0, s8
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s0, s0, s8
+; CHECK-NOFP-NEXT:    vstr.16 s0, [r0]
+; CHECK-NOFP-NEXT:    bx lr
+entry:
+  %z = call half @llvm.experimental.vector.reduce.fmax.v8f16(<8 x half> %x)
+  ret half %z
+}
+
+define arm_aapcs_vfpcc half @fmax_v16f16_nofast(<16 x half> %x) {
+; CHECK-FP-LABEL: fmax_v16f16_nofast:
+; CHECK-FP:       @ %bb.0: @ %entry
+; CHECK-FP-NEXT:    vmaxnm.f16 q0, q0, q1
+; CHECK-FP-NEXT:    vmov.f64 d2, d1
+; CHECK-FP-NEXT:    vmov.f32 s5, s3
+; CHECK-FP-NEXT:    vmaxnm.f16 q0, q0, q1
+; CHECK-FP-NEXT:    vmov.32 r1, q0[1]
+; CHECK-FP-NEXT:    vdup.32 q1, r1
+; CHECK-FP-NEXT:    vmaxnm.f16 q0, q0, q1
+; CHECK-FP-NEXT:    vmov.u16 r1, q0[1]
+; CHECK-FP-NEXT:    vdup.16 q1, r1
+; CHECK-FP-NEXT:    vmaxnm.f16 q0, q0, q1
+; CHECK-FP-NEXT:    vstr.16 s0, [r0]
+; CHECK-FP-NEXT:    bx lr
+;
+; CHECK-NOFP-LABEL: fmax_v16f16_nofast:
+; CHECK-NOFP:       @ %bb.0: @ %entry
+; CHECK-NOFP-NEXT:    vmovx.f16 s8, s7
+; CHECK-NOFP-NEXT:    vmovx.f16 s10, s3
+; CHECK-NOFP-NEXT:    vcmp.f16 s10, s8
+; CHECK-NOFP-NEXT:    vmovx.f16 s12, s1
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vmovx.f16 s14, s0
+; CHECK-NOFP-NEXT:    vselgt.f16 s8, s10, s8
+; CHECK-NOFP-NEXT:    vmovx.f16 s10, s5
+; CHECK-NOFP-NEXT:    vcmp.f16 s12, s10
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s10, s12, s10
+; CHECK-NOFP-NEXT:    vmovx.f16 s12, s2
+; CHECK-NOFP-NEXT:    vcmp.f16 s10, s8
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s8, s10, s8
+; CHECK-NOFP-NEXT:    vmovx.f16 s10, s6
+; CHECK-NOFP-NEXT:    vcmp.f16 s12, s10
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s10, s12, s10
+; CHECK-NOFP-NEXT:    vmovx.f16 s12, s4
+; CHECK-NOFP-NEXT:    vcmp.f16 s14, s12
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s12, s14, s12
+; CHECK-NOFP-NEXT:    vcmp.f16 s12, s10
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s10, s12, s10
+; CHECK-NOFP-NEXT:    vcmp.f16 s10, s8
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f16 s3, s7
+; CHECK-NOFP-NEXT:    vselgt.f16 s8, s10, s8
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f16 s1, s5
+; CHECK-NOFP-NEXT:    vselgt.f16 s10, s3, s7
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s12, s1, s5
+; CHECK-NOFP-NEXT:    vcmp.f16 s12, s10
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f16 s2, s6
+; CHECK-NOFP-NEXT:    vselgt.f16 s10, s12, s10
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f16 s0, s4
+; CHECK-NOFP-NEXT:    vselgt.f16 s12, s2, s6
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s0, s0, s4
+; CHECK-NOFP-NEXT:    vcmp.f16 s0, s12
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s0, s0, s12
+; CHECK-NOFP-NEXT:    vcmp.f16 s0, s10
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s0, s0, s10
+; CHECK-NOFP-NEXT:    vcmp.f16 s0, s8
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s0, s0, s8
+; CHECK-NOFP-NEXT:    vstr.16 s0, [r0]
+; CHECK-NOFP-NEXT:    bx lr
+entry:
+  %z = call half @llvm.experimental.vector.reduce.fmax.v16f16(<16 x half> %x)
+  ret half %z
+}
+
+define arm_aapcs_vfpcc double @fmax_v1f64_nofast(<1 x double> %x) {
+; CHECK-LABEL: fmax_v1f64_nofast:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    bx lr
+entry:
+  %z = call double @llvm.experimental.vector.reduce.fmax.v1f64(<1 x double> %x)
+  ret double %z
+}
+
+define arm_aapcs_vfpcc double @fmax_v2f64_nofast(<2 x double> %x) {
+; CHECK-LABEL: fmax_v2f64_nofast:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.f64 d0, d1
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    vselgt.f64 d0, d0, d1
+; CHECK-NEXT:    bx lr
+entry:
+  %z = call double @llvm.experimental.vector.reduce.fmax.v2f64(<2 x double> %x)
+  ret double %z
+}
+
+define arm_aapcs_vfpcc double @fmax_v4f64_nofast(<4 x double> %x) {
+; CHECK-LABEL: fmax_v4f64_nofast:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.f64 d1, d3
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    vcmp.f64 d0, d2
+; CHECK-NEXT:    vselgt.f64 d4, d1, d3
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    vselgt.f64 d0, d0, d2
+; CHECK-NEXT:    vcmp.f64 d0, d4
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    vselgt.f64 d0, d0, d4
+; CHECK-NEXT:    bx lr
+entry:
+  %z = call double @llvm.experimental.vector.reduce.fmax.v4f64(<4 x double> %x)
+  ret double %z
+}
+
+define arm_aapcs_vfpcc float @fmax_v2f32_acc(<2 x float> %x, float %y) {
+; CHECK-LABEL: fmax_v2f32_acc:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldr s6, .LCPI54_0
+; CHECK-NEXT:    vmaxnm.f32 s0, s0, s1
+; CHECK-NEXT:    vmaxnm.f32 s0, s0, s6
+; CHECK-NEXT:    vmaxnm.f32 s0, s0, s6
+; CHECK-NEXT:    vmaxnm.f32 s0, s4, s0
+; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    .p2align 2
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI54_0:
+; CHECK-NEXT:    .long 4286578688 @ float -Inf
+entry:
+  %z = call fast float @llvm.experimental.vector.reduce.fmax.v2f32(<2 x float> %x)
+  %c = fcmp fast ogt float %y, %z
+  %r = select i1 %c, float %y, float %z
+  ret float %r
+}
+
+define arm_aapcs_vfpcc float @fmax_v4f32_acc(<4 x float> %x, float %y) {
+; CHECK-LABEL: fmax_v4f32_acc:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmaxnm.f32 s6, s0, s1
+; CHECK-NEXT:    vmaxnm.f32 s6, s6, s2
+; CHECK-NEXT:    vmaxnm.f32 s0, s6, s3
+; CHECK-NEXT:    vmaxnm.f32 s0, s4, s0
+; CHECK-NEXT:    bx lr
+entry:
+  %z = call fast float @llvm.experimental.vector.reduce.fmax.v4f32(<4 x float> %x)
+  %c = fcmp fast ogt float %y, %z
+  %r = select i1 %c, float %y, float %z
+  ret float %r
+}
+
+define arm_aapcs_vfpcc float @fmax_v8f32_acc(<8 x float> %x, float %y) {
+; CHECK-FP-LABEL: fmax_v8f32_acc:
+; CHECK-FP:       @ %bb.0: @ %entry
+; CHECK-FP-NEXT:    vmaxnm.f32 q0, q0, q1
+; CHECK-FP-NEXT:    vmaxnm.f32 s4, s0, s1
+; CHECK-FP-NEXT:    vmaxnm.f32 s4, s4, s2
+; CHECK-FP-NEXT:    vmaxnm.f32 s0, s4, s3
+; CHECK-FP-NEXT:    vmaxnm.f32 s0, s8, s0
+; CHECK-FP-NEXT:    bx lr
+;
+; CHECK-NOFP-LABEL: fmax_v8f32_acc:
+; CHECK-NOFP:       @ %bb.0: @ %entry
+; CHECK-NOFP-NEXT:    vcmp.f32 s1, s5
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f32 s0, s4
+; CHECK-NOFP-NEXT:    vselgt.f32 s10, s1, s5
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f32 s2, s6
+; CHECK-NOFP-NEXT:    vselgt.f32 s12, s0, s4
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f32 s3, s7
+; CHECK-NOFP-NEXT:    vselgt.f32 s14, s2, s6
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f32 s0, s3, s7
+; CHECK-NOFP-NEXT:    vmaxnm.f32 s2, s12, s10
+; CHECK-NOFP-NEXT:    vmaxnm.f32 s2, s2, s14
+; CHECK-NOFP-NEXT:    vmaxnm.f32 s0, s2, s0
+; CHECK-NOFP-NEXT:    vmaxnm.f32 s0, s8, s0
+; CHECK-NOFP-NEXT:    bx lr
+entry:
+  %z = call fast float @llvm.experimental.vector.reduce.fmax.v8f32(<8 x float> %x)
+  %c = fcmp fast ogt float %y, %z
+  %r = select i1 %c, float %y, float %z
+  ret float %r
+}
+
+define arm_aapcs_vfpcc void @fmax_v4f16_acc(<4 x half> %x, half* %yy) {
+; CHECK-LABEL: fmax_v4f16_acc:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmovx.f16 s4, s0
+; CHECK-NEXT:    vmaxnm.f16 s4, s0, s4
+; CHECK-NEXT:    vmovx.f16 s0, s1
+; CHECK-NEXT:    vmaxnm.f16 s4, s4, s1
+; CHECK-NEXT:    vldr.16 s2, .LCPI57_0
+; CHECK-NEXT:    vmaxnm.f16 s0, s4, s0
+; CHECK-NEXT:    vmaxnm.f16 s0, s0, s2
+; CHECK-NEXT:    vmaxnm.f16 s0, s0, s2
+; CHECK-NEXT:    vmaxnm.f16 s0, s0, s2
+; CHECK-NEXT:    vmaxnm.f16 s0, s0, s2
+; CHECK-NEXT:    vldr.16 s2, [r0]
+; CHECK-NEXT:    vmaxnm.f16 s0, s2, s0
+; CHECK-NEXT:    vstr.16 s0, [r0]
+; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    .p2align 1
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI57_0:
+; CHECK-NEXT:    .short 64512 @ half -Inf
+entry:
+  %y = load half, half* %yy
+  %z = call fast half @llvm.experimental.vector.reduce.fmax.v4f16(<4 x half> %x)
+  %c = fcmp fast ogt half %y, %z
+  %r = select i1 %c, half %y, half %z
+  store half %r, half* %yy
+  ret void
+}
+
+define arm_aapcs_vfpcc void @fmax_v8f16_acc(<8 x half> %x, half* %yy) {
+; CHECK-LABEL: fmax_v8f16_acc:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmovx.f16 s4, s0
+; CHECK-NEXT:    vmovx.f16 s6, s1
+; CHECK-NEXT:    vmaxnm.f16 s4, s0, s4
+; CHECK-NEXT:    vmovx.f16 s0, s3
+; CHECK-NEXT:    vmaxnm.f16 s4, s4, s1
+; CHECK-NEXT:    vmaxnm.f16 s4, s4, s6
+; CHECK-NEXT:    vmovx.f16 s6, s2
+; CHECK-NEXT:    vmaxnm.f16 s4, s4, s2
+; CHECK-NEXT:    vldr.16 s2, [r0]
+; CHECK-NEXT:    vmaxnm.f16 s4, s4, s6
+; CHECK-NEXT:    vmaxnm.f16 s4, s4, s3
+; CHECK-NEXT:    vmaxnm.f16 s0, s4, s0
+; CHECK-NEXT:    vmaxnm.f16 s0, s2, s0
+; CHECK-NEXT:    vstr.16 s0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %y = load half, half* %yy
+  %z = call fast half @llvm.experimental.vector.reduce.fmax.v8f16(<8 x half> %x)
+  %c = fcmp fast ogt half %y, %z
+  %r = select i1 %c, half %y, half %z
+  store half %r, half* %yy
+  ret void
+}
+
+define arm_aapcs_vfpcc void @fmax_v16f16_acc(<16 x half> %x, half* %yy) {
+; CHECK-FP-LABEL: fmax_v16f16_acc:
+; CHECK-FP:       @ %bb.0: @ %entry
+; CHECK-FP-NEXT:    vmaxnm.f16 q0, q0, q1
+; CHECK-FP-NEXT:    vmovx.f16 s4, s0
+; CHECK-FP-NEXT:    vmovx.f16 s6, s1
+; CHECK-FP-NEXT:    vmaxnm.f16 s4, s0, s4
+; CHECK-FP-NEXT:    vmovx.f16 s0, s3
+; CHECK-FP-NEXT:    vmaxnm.f16 s4, s4, s1
+; CHECK-FP-NEXT:    vmaxnm.f16 s4, s4, s6
+; CHECK-FP-NEXT:    vmovx.f16 s6, s2
+; CHECK-FP-NEXT:    vmaxnm.f16 s4, s4, s2
+; CHECK-FP-NEXT:    vldr.16 s2, [r0]
+; CHECK-FP-NEXT:    vmaxnm.f16 s4, s4, s6
+; CHECK-FP-NEXT:    vmaxnm.f16 s4, s4, s3
+; CHECK-FP-NEXT:    vmaxnm.f16 s0, s4, s0
+; CHECK-FP-NEXT:    vmaxnm.f16 s0, s2, s0
+; CHECK-FP-NEXT:    vstr.16 s0, [r0]
+; CHECK-FP-NEXT:    bx lr
+;
+; CHECK-NOFP-LABEL: fmax_v16f16_acc:
+; CHECK-NOFP:       @ %bb.0: @ %entry
+; CHECK-NOFP-NEXT:    vmovx.f16 s8, s4
+; CHECK-NOFP-NEXT:    vmovx.f16 s10, s0
+; CHECK-NOFP-NEXT:    vcmp.f16 s10, s8
+; CHECK-NOFP-NEXT:    vmovx.f16 s12, s1
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f16 s0, s4
+; CHECK-NOFP-NEXT:    vselgt.f16 s8, s10, s8
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f16 s1, s5
+; CHECK-NOFP-NEXT:    vselgt.f16 s10, s0, s4
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vmaxnm.f16 s8, s10, s8
+; CHECK-NOFP-NEXT:    vmovx.f16 s4, s7
+; CHECK-NOFP-NEXT:    vmovx.f16 s0, s3
+; CHECK-NOFP-NEXT:    vselgt.f16 s10, s1, s5
+; CHECK-NOFP-NEXT:    vmaxnm.f16 s8, s8, s10
+; CHECK-NOFP-NEXT:    vmovx.f16 s10, s5
+; CHECK-NOFP-NEXT:    vcmp.f16 s12, s10
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f16 s2, s6
+; CHECK-NOFP-NEXT:    vselgt.f16 s10, s12, s10
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vmaxnm.f16 s8, s8, s10
+; CHECK-NOFP-NEXT:    vmovx.f16 s12, s2
+; CHECK-NOFP-NEXT:    vselgt.f16 s10, s2, s6
+; CHECK-NOFP-NEXT:    vldr.16 s2, [r0]
+; CHECK-NOFP-NEXT:    vmaxnm.f16 s8, s8, s10
+; CHECK-NOFP-NEXT:    vmovx.f16 s10, s6
+; CHECK-NOFP-NEXT:    vcmp.f16 s12, s10
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f16 s3, s7
+; CHECK-NOFP-NEXT:    vselgt.f16 s10, s12, s10
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f16 s0, s4
+; CHECK-NOFP-NEXT:    vmaxnm.f16 s8, s8, s10
+; CHECK-NOFP-NEXT:    vselgt.f16 s10, s3, s7
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vmaxnm.f16 s8, s8, s10
+; CHECK-NOFP-NEXT:    vselgt.f16 s0, s0, s4
+; CHECK-NOFP-NEXT:    vmaxnm.f16 s0, s8, s0
+; CHECK-NOFP-NEXT:    vmaxnm.f16 s0, s2, s0
+; CHECK-NOFP-NEXT:    vstr.16 s0, [r0]
+; CHECK-NOFP-NEXT:    bx lr
+entry:
+  %y = load half, half* %yy
+  %z = call fast half @llvm.experimental.vector.reduce.fmax.v16f16(<16 x half> %x)
+  %c = fcmp fast ogt half %y, %z
+  %r = select i1 %c, half %y, half %z
+  store half %r, half* %yy
+  ret void
+}
+
+define arm_aapcs_vfpcc double @fmax_v1f64_acc(<1 x double> %x, double %y) {
+; CHECK-LABEL: fmax_v1f64_acc:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmaxnm.f64 d0, d1, d0
+; CHECK-NEXT:    bx lr
+entry:
+  %z = call fast double @llvm.experimental.vector.reduce.fmax.v1f64(<1 x double> %x)
+  %c = fcmp fast ogt double %y, %z
+  %r = select i1 %c, double %y, double %z
+  ret double %r
+}
+
+define arm_aapcs_vfpcc double @fmax_v2f64_acc(<2 x double> %x, double %y) {
+; CHECK-LABEL: fmax_v2f64_acc:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmaxnm.f64 d0, d0, d1
+; CHECK-NEXT:    vmaxnm.f64 d0, d2, d0
+; CHECK-NEXT:    bx lr
+entry:
+  %z = call fast double @llvm.experimental.vector.reduce.fmax.v2f64(<2 x double> %x)
+  %c = fcmp fast ogt double %y, %z
+  %r = select i1 %c, double %y, double %z
+  ret double %r
+}
+
+define arm_aapcs_vfpcc double @fmax_v4f64_acc(<4 x double> %x, double %y) {
+; CHECK-LABEL: fmax_v4f64_acc:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.f64 d1, d3
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    vcmp.f64 d0, d2
+; CHECK-NEXT:    vselgt.f64 d5, d1, d3
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    vselgt.f64 d0, d0, d2
+; CHECK-NEXT:    vmaxnm.f64 d0, d0, d5
+; CHECK-NEXT:    vmaxnm.f64 d0, d4, d0
+; CHECK-NEXT:    bx lr
+entry:
+  %z = call fast double @llvm.experimental.vector.reduce.fmax.v4f64(<4 x double> %x)
+  %c = fcmp fast ogt double %y, %z
+  %r = select i1 %c, double %y, double %z
+  ret double %r
+}
+
+define arm_aapcs_vfpcc float @fmax_v2f32_acc_nofast(<2 x float> %x, float %y) {
+; CHECK-FP-LABEL: fmax_v2f32_acc_nofast:
+; CHECK-FP:       @ %bb.0: @ %entry
+; CHECK-FP-NEXT:    vmov.32 r0, q0[1]
+; CHECK-FP-NEXT:    vdup.32 q2, r0
+; CHECK-FP-NEXT:    vmaxnm.f32 q0, q0, q2
+; CHECK-FP-NEXT:    vcmp.f32 s4, s0
+; CHECK-FP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-FP-NEXT:    vselgt.f32 s0, s4, s0
+; CHECK-FP-NEXT:    bx lr
+;
+; CHECK-NOFP-LABEL: fmax_v2f32_acc_nofast:
+; CHECK-NOFP:       @ %bb.0: @ %entry
+; CHECK-NOFP-NEXT:    vmov.32 r0, q0[1]
+; CHECK-NOFP-NEXT:    vdup.32 q2, r0
+; CHECK-NOFP-NEXT:    vcmp.f32 s0, s8
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f32 s0, s0, s8
+; CHECK-NOFP-NEXT:    vcmp.f32 s4, s0
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f32 s0, s4, s0
+; CHECK-NOFP-NEXT:    bx lr
+entry:
+  %z = call float @llvm.experimental.vector.reduce.fmax.v2f32(<2 x float> %x)
+  %c = fcmp ogt float %y, %z
+  %r = select i1 %c, float %y, float %z
+  ret float %r
+}
+
+define arm_aapcs_vfpcc float @fmax_v4f32_acc_nofast(<4 x float> %x, float %y) {
+; CHECK-FP-LABEL: fmax_v4f32_acc_nofast:
+; CHECK-FP:       @ %bb.0: @ %entry
+; CHECK-FP-NEXT:    vmov.f64 d4, d1
+; CHECK-FP-NEXT:    vmov.f32 s9, s3
+; CHECK-FP-NEXT:    vmaxnm.f32 q0, q0, q2
+; CHECK-FP-NEXT:    vmov.32 r0, q0[1]
+; CHECK-FP-NEXT:    vdup.32 q2, r0
+; CHECK-FP-NEXT:    vmaxnm.f32 q0, q0, q2
+; CHECK-FP-NEXT:    vcmp.f32 s4, s0
+; CHECK-FP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-FP-NEXT:    vselgt.f32 s0, s4, s0
+; CHECK-FP-NEXT:    bx lr
+;
+; CHECK-NOFP-LABEL: fmax_v4f32_acc_nofast:
+; CHECK-NOFP:       @ %bb.0: @ %entry
+; CHECK-NOFP-NEXT:    vcmp.f32 s1, s3
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vmov.f64 d4, d1
+; CHECK-NOFP-NEXT:    vmov.f32 s9, s3
+; CHECK-NOFP-NEXT:    vcmp.f32 s0, s8
+; CHECK-NOFP-NEXT:    vselgt.f32 s6, s1, s3
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f32 s0, s0, s8
+; CHECK-NOFP-NEXT:    vcmp.f32 s0, s6
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f32 s0, s0, s6
+; CHECK-NOFP-NEXT:    vcmp.f32 s4, s0
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f32 s0, s4, s0
+; CHECK-NOFP-NEXT:    bx lr
+entry:
+  %z = call float @llvm.experimental.vector.reduce.fmax.v4f32(<4 x float> %x)
+  %c = fcmp ogt float %y, %z
+  %r = select i1 %c, float %y, float %z
+  ret float %r
+}
+
+define arm_aapcs_vfpcc float @fmax_v8f32_acc_nofast(<8 x float> %x, float %y) {
+; CHECK-FP-LABEL: fmax_v8f32_acc_nofast:
+; CHECK-FP:       @ %bb.0: @ %entry
+; CHECK-FP-NEXT:    vmaxnm.f32 q0, q0, q1
+; CHECK-FP-NEXT:    vmov.f64 d2, d1
+; CHECK-FP-NEXT:    vmov.f32 s5, s3
+; CHECK-FP-NEXT:    vmaxnm.f32 q0, q0, q1
+; CHECK-FP-NEXT:    vmov.32 r0, q0[1]
+; CHECK-FP-NEXT:    vdup.32 q1, r0
+; CHECK-FP-NEXT:    vmaxnm.f32 q0, q0, q1
+; CHECK-FP-NEXT:    vcmp.f32 s8, s0
+; CHECK-FP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-FP-NEXT:    vselgt.f32 s0, s8, s0
+; CHECK-FP-NEXT:    bx lr
+;
+; CHECK-NOFP-LABEL: fmax_v8f32_acc_nofast:
+; CHECK-NOFP:       @ %bb.0: @ %entry
+; CHECK-NOFP-NEXT:    vcmp.f32 s3, s7
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f32 s1, s5
+; CHECK-NOFP-NEXT:    vselgt.f32 s10, s3, s7
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f32 s2, s6
+; CHECK-NOFP-NEXT:    vselgt.f32 s12, s1, s5
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f32 s0, s4
+; CHECK-NOFP-NEXT:    vselgt.f32 s14, s2, s6
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f32 s12, s10
+; CHECK-NOFP-NEXT:    vselgt.f32 s0, s0, s4
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f32 s0, s14
+; CHECK-NOFP-NEXT:    vselgt.f32 s2, s12, s10
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f32 s0, s0, s14
+; CHECK-NOFP-NEXT:    vcmp.f32 s0, s2
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f32 s0, s0, s2
+; CHECK-NOFP-NEXT:    vcmp.f32 s8, s0
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f32 s0, s8, s0
+; CHECK-NOFP-NEXT:    bx lr
+entry:
+  %z = call float @llvm.experimental.vector.reduce.fmax.v8f32(<8 x float> %x)
+  %c = fcmp ogt float %y, %z
+  %r = select i1 %c, float %y, float %z
+  ret float %r
+}
+
+define arm_aapcs_vfpcc void @fmax_v4f16_acc_nofast(<4 x half> %x, half* %yy) {
+; CHECK-FP-LABEL: fmax_v4f16_acc_nofast:
+; CHECK-FP:       @ %bb.0: @ %entry
+; CHECK-FP-NEXT:    vmov.32 r1, q0[1]
+; CHECK-FP-NEXT:    vdup.32 q1, r1
+; CHECK-FP-NEXT:    vmaxnm.f16 q0, q0, q1
+; CHECK-FP-NEXT:    vmov.u16 r1, q0[1]
+; CHECK-FP-NEXT:    vdup.16 q1, r1
+; CHECK-FP-NEXT:    vmaxnm.f16 q0, q0, q1
+; CHECK-FP-NEXT:    vldr.16 s4, [r0]
+; CHECK-FP-NEXT:    vcmp.f16 s4, s0
+; CHECK-FP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-FP-NEXT:    vselgt.f16 s0, s4, s0
+; CHECK-FP-NEXT:    vstr.16 s0, [r0]
+; CHECK-FP-NEXT:    bx lr
+;
+; CHECK-NOFP-LABEL: fmax_v4f16_acc_nofast:
+; CHECK-NOFP:       @ %bb.0: @ %entry
+; CHECK-NOFP-NEXT:    vmov.32 r1, q0[1]
+; CHECK-NOFP-NEXT:    vmovx.f16 s10, s0
+; CHECK-NOFP-NEXT:    vdup.32 q1, r1
+; CHECK-NOFP-NEXT:    vmovx.f16 s8, s4
+; CHECK-NOFP-NEXT:    vcmp.f16 s10, s8
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f16 s0, s4
+; CHECK-NOFP-NEXT:    vselgt.f16 s8, s10, s8
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s0, s0, s4
+; CHECK-NOFP-NEXT:    vldr.16 s2, [r0]
+; CHECK-NOFP-NEXT:    vcmp.f16 s0, s8
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s0, s0, s8
+; CHECK-NOFP-NEXT:    vcmp.f16 s2, s0
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s0, s2, s0
+; CHECK-NOFP-NEXT:    vstr.16 s0, [r0]
+; CHECK-NOFP-NEXT:    bx lr
+entry:
+  %y = load half, half* %yy
+  %z = call half @llvm.experimental.vector.reduce.fmax.v4f16(<4 x half> %x)
+  %c = fcmp ogt half %y, %z
+  %r = select i1 %c, half %y, half %z
+  store half %r, half* %yy
+  ret void
+}
+
+define arm_aapcs_vfpcc void @fmax_v8f16_acc_nofast(<8 x half> %x, half* %yy) {
+; CHECK-FP-LABEL: fmax_v8f16_acc_nofast:
+; CHECK-FP:       @ %bb.0: @ %entry
+; CHECK-FP-NEXT:    vmov.f64 d2, d1
+; CHECK-FP-NEXT:    vmov.f32 s5, s3
+; CHECK-FP-NEXT:    vmaxnm.f16 q0, q0, q1
+; CHECK-FP-NEXT:    vmov.32 r1, q0[1]
+; CHECK-FP-NEXT:    vdup.32 q1, r1
+; CHECK-FP-NEXT:    vmaxnm.f16 q0, q0, q1
+; CHECK-FP-NEXT:    vmov.u16 r1, q0[1]
+; CHECK-FP-NEXT:    vdup.16 q1, r1
+; CHECK-FP-NEXT:    vmaxnm.f16 q0, q0, q1
+; CHECK-FP-NEXT:    vldr.16 s4, [r0]
+; CHECK-FP-NEXT:    vcmp.f16 s4, s0
+; CHECK-FP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-FP-NEXT:    vselgt.f16 s0, s4, s0
+; CHECK-FP-NEXT:    vstr.16 s0, [r0]
+; CHECK-FP-NEXT:    bx lr
+;
+; CHECK-NOFP-LABEL: fmax_v8f16_acc_nofast:
+; CHECK-NOFP:       @ %bb.0: @ %entry
+; CHECK-NOFP-NEXT:    vmovx.f16 s8, s3
+; CHECK-NOFP-NEXT:    vmovx.f16 s10, s1
+; CHECK-NOFP-NEXT:    vcmp.f16 s10, s8
+; CHECK-NOFP-NEXT:    vmov.f64 d2, d1
+; CHECK-NOFP-NEXT:    vmovx.f16 s12, s0
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vmov.f32 s5, s3
+; CHECK-NOFP-NEXT:    vselgt.f16 s8, s10, s8
+; CHECK-NOFP-NEXT:    vmovx.f16 s10, s4
+; CHECK-NOFP-NEXT:    vcmp.f16 s12, s10
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s10, s12, s10
+; CHECK-NOFP-NEXT:    vcmp.f16 s10, s8
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f16 s1, s3
+; CHECK-NOFP-NEXT:    vselgt.f16 s8, s10, s8
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f16 s0, s4
+; CHECK-NOFP-NEXT:    vselgt.f16 s10, s1, s3
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s0, s0, s4
+; CHECK-NOFP-NEXT:    vldr.16 s2, [r0]
+; CHECK-NOFP-NEXT:    vcmp.f16 s0, s10
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s0, s0, s10
+; CHECK-NOFP-NEXT:    vcmp.f16 s0, s8
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s0, s0, s8
+; CHECK-NOFP-NEXT:    vcmp.f16 s2, s0
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s0, s2, s0
+; CHECK-NOFP-NEXT:    vstr.16 s0, [r0]
+; CHECK-NOFP-NEXT:    bx lr
+entry:
+  %y = load half, half* %yy
+  %z = call half @llvm.experimental.vector.reduce.fmax.v8f16(<8 x half> %x)
+  %c = fcmp ogt half %y, %z
+  %r = select i1 %c, half %y, half %z
+  store half %r, half* %yy
+  ret void
+}
+
+define arm_aapcs_vfpcc void @fmax_v16f16_acc_nofast(<16 x half> %x, half* %yy) {
+; CHECK-FP-LABEL: fmax_v16f16_acc_nofast:
+; CHECK-FP:       @ %bb.0: @ %entry
+; CHECK-FP-NEXT:    vmaxnm.f16 q0, q0, q1
+; CHECK-FP-NEXT:    vmov.f64 d2, d1
+; CHECK-FP-NEXT:    vmov.f32 s5, s3
+; CHECK-FP-NEXT:    vmaxnm.f16 q0, q0, q1
+; CHECK-FP-NEXT:    vmov.32 r1, q0[1]
+; CHECK-FP-NEXT:    vdup.32 q1, r1
+; CHECK-FP-NEXT:    vmaxnm.f16 q0, q0, q1
+; CHECK-FP-NEXT:    vmov.u16 r1, q0[1]
+; CHECK-FP-NEXT:    vdup.16 q1, r1
+; CHECK-FP-NEXT:    vmaxnm.f16 q0, q0, q1
+; CHECK-FP-NEXT:    vldr.16 s4, [r0]
+; CHECK-FP-NEXT:    vcmp.f16 s4, s0
+; CHECK-FP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-FP-NEXT:    vselgt.f16 s0, s4, s0
+; CHECK-FP-NEXT:    vstr.16 s0, [r0]
+; CHECK-FP-NEXT:    bx lr
+;
+; CHECK-NOFP-LABEL: fmax_v16f16_acc_nofast:
+; CHECK-NOFP:       @ %bb.0: @ %entry
+; CHECK-NOFP-NEXT:    vmovx.f16 s8, s7
+; CHECK-NOFP-NEXT:    vmovx.f16 s10, s3
+; CHECK-NOFP-NEXT:    vcmp.f16 s10, s8
+; CHECK-NOFP-NEXT:    vmovx.f16 s12, s1
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vmovx.f16 s14, s0
+; CHECK-NOFP-NEXT:    vselgt.f16 s8, s10, s8
+; CHECK-NOFP-NEXT:    vmovx.f16 s10, s5
+; CHECK-NOFP-NEXT:    vcmp.f16 s12, s10
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s10, s12, s10
+; CHECK-NOFP-NEXT:    vmovx.f16 s12, s2
+; CHECK-NOFP-NEXT:    vcmp.f16 s10, s8
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s8, s10, s8
+; CHECK-NOFP-NEXT:    vmovx.f16 s10, s6
+; CHECK-NOFP-NEXT:    vcmp.f16 s12, s10
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s10, s12, s10
+; CHECK-NOFP-NEXT:    vmovx.f16 s12, s4
+; CHECK-NOFP-NEXT:    vcmp.f16 s14, s12
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s12, s14, s12
+; CHECK-NOFP-NEXT:    vcmp.f16 s12, s10
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s10, s12, s10
+; CHECK-NOFP-NEXT:    vcmp.f16 s10, s8
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f16 s3, s7
+; CHECK-NOFP-NEXT:    vselgt.f16 s8, s10, s8
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f16 s1, s5
+; CHECK-NOFP-NEXT:    vselgt.f16 s10, s3, s7
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s12, s1, s5
+; CHECK-NOFP-NEXT:    vcmp.f16 s12, s10
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f16 s2, s6
+; CHECK-NOFP-NEXT:    vselgt.f16 s10, s12, s10
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vcmp.f16 s0, s4
+; CHECK-NOFP-NEXT:    vselgt.f16 s12, s2, s6
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s0, s0, s4
+; CHECK-NOFP-NEXT:    vldr.16 s2, [r0]
+; CHECK-NOFP-NEXT:    vcmp.f16 s0, s12
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s0, s0, s12
+; CHECK-NOFP-NEXT:    vcmp.f16 s0, s10
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s0, s0, s10
+; CHECK-NOFP-NEXT:    vcmp.f16 s0, s8
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s0, s0, s8
+; CHECK-NOFP-NEXT:    vcmp.f16 s2, s0
+; CHECK-NOFP-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NOFP-NEXT:    vselgt.f16 s0, s2, s0
+; CHECK-NOFP-NEXT:    vstr.16 s0, [r0]
+; CHECK-NOFP-NEXT:    bx lr
+entry:
+  %y = load half, half* %yy
+  %z = call half @llvm.experimental.vector.reduce.fmax.v16f16(<16 x half> %x)
+  %c = fcmp ogt half %y, %z
+  %r = select i1 %c, half %y, half %z
+  store half %r, half* %yy
+  ret void
+}
+
+define arm_aapcs_vfpcc double @fmax_v1f64_acc_nofast(<1 x double> %x, double %y) {
+; CHECK-LABEL: fmax_v1f64_acc_nofast:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.f64 d1, d0
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    vselgt.f64 d0, d1, d0
+; CHECK-NEXT:    bx lr
+entry:
+  %z = call double @llvm.experimental.vector.reduce.fmax.v1f64(<1 x double> %x)
+  %c = fcmp ogt double %y, %z
+  %r = select i1 %c, double %y, double %z
+  ret double %r
+}
+
+define arm_aapcs_vfpcc double @fmax_v2f64_acc_nofast(<2 x double> %x, double %y) {
+; CHECK-LABEL: fmax_v2f64_acc_nofast:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.f64 d0, d1
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    vselgt.f64 d0, d0, d1
+; CHECK-NEXT:    vcmp.f64 d2, d0
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    vselgt.f64 d0, d2, d0
+; CHECK-NEXT:    bx lr
+entry:
+  %z = call double @llvm.experimental.vector.reduce.fmax.v2f64(<2 x double> %x)
+  %c = fcmp ogt double %y, %z
+  %r = select i1 %c, double %y, double %z
+  ret double %r
+}
+
+define arm_aapcs_vfpcc double @fmax_v4f64_acc_nofast(<4 x double> %x, double %y) {
+; CHECK-LABEL: fmax_v4f64_acc_nofast:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.f64 d1, d3
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    vcmp.f64 d0, d2
+; CHECK-NEXT:    vselgt.f64 d5, d1, d3
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    vselgt.f64 d0, d0, d2
+; CHECK-NEXT:    vcmp.f64 d0, d5
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    vselgt.f64 d0, d0, d5
+; CHECK-NEXT:    vcmp.f64 d4, d0
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    vselgt.f64 d0, d4, d0
+; CHECK-NEXT:    bx lr
+entry:
+  %z = call double @llvm.experimental.vector.reduce.fmax.v4f64(<4 x double> %x)
+  %c = fcmp ogt double %y, %z
+  %r = select i1 %c, double %y, double %z
+  ret double %r
+}
+
+declare double @llvm.experimental.vector.reduce.fmax.v1f64(<1 x double>)
+declare double @llvm.experimental.vector.reduce.fmax.v2f64(<2 x double>)
+declare double @llvm.experimental.vector.reduce.fmax.v4f64(<4 x double>)
+declare double @llvm.experimental.vector.reduce.fmin.v1f64(<1 x double>)
+declare double @llvm.experimental.vector.reduce.fmin.v2f64(<2 x double>)
+declare double @llvm.experimental.vector.reduce.fmin.v4f64(<4 x double>)
+declare float @llvm.experimental.vector.reduce.fmax.v2f32(<2 x float>)
+declare float @llvm.experimental.vector.reduce.fmax.v4f32(<4 x float>)
+declare float @llvm.experimental.vector.reduce.fmax.v8f32(<8 x float>)
+declare float @llvm.experimental.vector.reduce.fmin.v2f32(<2 x float>)
+declare float @llvm.experimental.vector.reduce.fmin.v4f32(<4 x float>)
+declare float @llvm.experimental.vector.reduce.fmin.v8f32(<8 x float>)
+declare half @llvm.experimental.vector.reduce.fmax.v16f16(<16 x half>)
+declare half @llvm.experimental.vector.reduce.fmax.v4f16(<4 x half>)
+declare half @llvm.experimental.vector.reduce.fmax.v8f16(<8 x half>)
+declare half @llvm.experimental.vector.reduce.fmin.v16f16(<16 x half>)
+declare half @llvm.experimental.vector.reduce.fmin.v4f16(<4 x half>)
+declare half @llvm.experimental.vector.reduce.fmin.v8f16(<8 x half>)
diff --git a/llvm/test/CodeGen/Thumb2/mve-vector-spill.ll b/llvm/test/CodeGen/Thumb2/mve-vector-spill.ll
new file mode 100644
index 000000000..f7a0a04af
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-vector-spill.ll
@@ -0,0 +1,163 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -O0 -mattr=+mve %s -o - | FileCheck %s
+
+declare void @external_function()
+
+define arm_aapcs_vfpcc void @spill_vector_i32(<4 x i32> %v, <4 x i32>* %p) {
+; CHECK-LABEL: spill_vector_i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    .pad #40
+; CHECK-NEXT:    sub sp, #40
+; CHECK-NEXT:    vmov q1, q0
+; CHECK-NEXT:    str r0, [sp, #36] @ 4-byte Spill
+; CHECK-NEXT:    vstrw.32 q0, [sp, #16] @ 16-byte Spill
+; CHECK-NEXT:    vstrw.32 q1, [sp] @ 16-byte Spill
+; CHECK-NEXT:    bl external_function
+; CHECK-NEXT:    vldrw.u32 q0, [sp, #16] @ 16-byte Reload
+; CHECK-NEXT:    ldr r0, [sp, #36] @ 4-byte Reload
+; CHECK-NEXT:    vstrw.32 q0, [r0]
+; CHECK-NEXT:    add sp, #40
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  call void @external_function()
+  store <4 x i32> %v, <4 x i32>* %p, align 4
+  ret void
+}
+
+define arm_aapcs_vfpcc void @spill_vector_i16(<8 x i16> %v, <8 x i16>* %p) {
+; CHECK-LABEL: spill_vector_i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    .pad #40
+; CHECK-NEXT:    sub sp, #40
+; CHECK-NEXT:    vmov q1, q0
+; CHECK-NEXT:    str r0, [sp, #36] @ 4-byte Spill
+; CHECK-NEXT:    vstrw.32 q0, [sp, #16] @ 16-byte Spill
+; CHECK-NEXT:    vstrw.32 q1, [sp] @ 16-byte Spill
+; CHECK-NEXT:    bl external_function
+; CHECK-NEXT:    vldrw.u32 q0, [sp, #16] @ 16-byte Reload
+; CHECK-NEXT:    ldr r0, [sp, #36] @ 4-byte Reload
+; CHECK-NEXT:    vstrh.16 q0, [r0]
+; CHECK-NEXT:    add sp, #40
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  call void @external_function()
+  store <8 x i16> %v, <8 x i16>* %p, align 2
+  ret void
+}
+
+define arm_aapcs_vfpcc void @spill_vector_i8(<16 x i8> %v, <16 x i8>* %p) {
+; CHECK-LABEL: spill_vector_i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    .pad #40
+; CHECK-NEXT:    sub sp, #40
+; CHECK-NEXT:    vmov q1, q0
+; CHECK-NEXT:    str r0, [sp, #36] @ 4-byte Spill
+; CHECK-NEXT:    vstrw.32 q0, [sp, #16] @ 16-byte Spill
+; CHECK-NEXT:    vstrw.32 q1, [sp] @ 16-byte Spill
+; CHECK-NEXT:    bl external_function
+; CHECK-NEXT:    vldrw.u32 q0, [sp, #16] @ 16-byte Reload
+; CHECK-NEXT:    ldr r0, [sp, #36] @ 4-byte Reload
+; CHECK-NEXT:    vstrb.8 q0, [r0]
+; CHECK-NEXT:    add sp, #40
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  call void @external_function()
+  store <16 x i8> %v, <16 x i8>* %p, align 1
+  ret void
+}
+
+define arm_aapcs_vfpcc void @spill_vector_i64(<2 x i64> %v, <2 x i64>* %p) {
+; CHECK-LABEL: spill_vector_i64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    .pad #40
+; CHECK-NEXT:    sub sp, #40
+; CHECK-NEXT:    vmov q1, q0
+; CHECK-NEXT:    str r0, [sp, #36] @ 4-byte Spill
+; CHECK-NEXT:    vstrw.32 q0, [sp, #16] @ 16-byte Spill
+; CHECK-NEXT:    vstrw.32 q1, [sp] @ 16-byte Spill
+; CHECK-NEXT:    bl external_function
+; CHECK-NEXT:    vldrw.u32 q0, [sp, #16] @ 16-byte Reload
+; CHECK-NEXT:    ldr r0, [sp, #36] @ 4-byte Reload
+; CHECK-NEXT:    vstrw.32 q0, [r0]
+; CHECK-NEXT:    add sp, #40
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  call void @external_function()
+  store <2 x i64> %v, <2 x i64>* %p, align 8
+  ret void
+}
+
+define arm_aapcs_vfpcc void @spill_vector_f32(<4 x float> %v, <4 x float>* %p) {
+; CHECK-LABEL: spill_vector_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    .pad #40
+; CHECK-NEXT:    sub sp, #40
+; CHECK-NEXT:    vmov q1, q0
+; CHECK-NEXT:    str r0, [sp, #36] @ 4-byte Spill
+; CHECK-NEXT:    vstrw.32 q0, [sp, #16] @ 16-byte Spill
+; CHECK-NEXT:    vstrw.32 q1, [sp] @ 16-byte Spill
+; CHECK-NEXT:    bl external_function
+; CHECK-NEXT:    vldrw.u32 q0, [sp, #16] @ 16-byte Reload
+; CHECK-NEXT:    ldr r0, [sp, #36] @ 4-byte Reload
+; CHECK-NEXT:    vstrw.32 q0, [r0]
+; CHECK-NEXT:    add sp, #40
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  call void @external_function()
+  store <4 x float> %v, <4 x float>* %p, align 8
+  ret void
+}
+
+define arm_aapcs_vfpcc void @spill_vector_f16(<8 x half> %v, <8 x half>* %p) {
+; CHECK-LABEL: spill_vector_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    .pad #40
+; CHECK-NEXT:    sub sp, #40
+; CHECK-NEXT:    vmov q1, q0
+; CHECK-NEXT:    str r0, [sp, #36] @ 4-byte Spill
+; CHECK-NEXT:    vstrw.32 q0, [sp, #16] @ 16-byte Spill
+; CHECK-NEXT:    vstrw.32 q1, [sp] @ 16-byte Spill
+; CHECK-NEXT:    bl external_function
+; CHECK-NEXT:    vldrw.u32 q0, [sp, #16] @ 16-byte Reload
+; CHECK-NEXT:    ldr r0, [sp, #36] @ 4-byte Reload
+; CHECK-NEXT:    vstrw.32 q0, [r0]
+; CHECK-NEXT:    add sp, #40
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  call void @external_function()
+  store <8 x half> %v, <8 x half>* %p, align 8
+  ret void
+}
+
+define arm_aapcs_vfpcc void @spill_vector_f64(<2 x double> %v, <2 x double>* %p) {
+; CHECK-LABEL: spill_vector_f64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    .pad #24
+; CHECK-NEXT:    sub sp, #24
+; CHECK-NEXT:    str r0, [sp, #20] @ 4-byte Spill
+; CHECK-NEXT:    vstrw.32 q0, [sp] @ 16-byte Spill
+; CHECK-NEXT:    bl external_function
+; CHECK-NEXT:    vldrw.u32 q0, [sp] @ 16-byte Reload
+; CHECK-NEXT:    ldr r0, [sp, #20] @ 4-byte Reload
+; CHECK-NEXT:    vstrw.32 q0, [r0]
+; CHECK-NEXT:    add sp, #24
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  call void @external_function()
+  store <2 x double> %v, <2 x double>* %p, align 8
+  ret void
+}
diff --git a/llvm/test/CodeGen/Thumb2/mve-vfma.ll b/llvm/test/CodeGen/Thumb2/mve-vfma.ll
new file mode 100644
index 000000000..8ed8b8120
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-vfma.ll
@@ -0,0 +1,25 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve.fp %s -o - | FileCheck %s
+
+define arm_aapcs_vfpcc <4 x float> @fma_v4f32(<4 x float> %dst, <4 x float> %s1, <4 x float> %s2) {
+; CHECK-LABEL: fma_v4f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vfma.f32 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call fast <4 x float> @llvm.fma.v4f32(<4 x float> %s1, <4 x float> %s2, <4 x float> %dst)
+  ret <4 x float> %0
+}
+
+define arm_aapcs_vfpcc <8 x half> @fma_v8f16(<8 x half> %dst, <8 x half> %s1, <8 x half> %s2) {
+; CHECK-LABEL: fma_v8f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vfma.f16 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call fast <8 x half> @llvm.fma.v8f16(<8 x half> %s1, <8 x half> %s2, <8 x half> %dst)
+  ret <8 x half> %0
+}
+
+declare <4 x float> @llvm.fma.v4f32(<4 x float>, <4 x float>, <4 x float>)
+declare <8 x half> @llvm.fma.v8f16(<8 x half>, <8 x half>, <8 x half>)
diff --git a/llvm/test/CodeGen/Thumb2/mve-vhaddsub.ll b/llvm/test/CodeGen/Thumb2/mve-vhaddsub.ll
new file mode 100644
index 000000000..83534e2c3
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-vhaddsub.ll
@@ -0,0 +1,305 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve %s -o - | FileCheck %s
+
+define arm_aapcs_vfpcc <16 x i8> @add_ashr_v16i8(<16 x i8> %src1, <16 x i8> %src2) {
+; CHECK-LABEL: add_ashr_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vadd.i8 q0, q0, q1
+; CHECK-NEXT:    vshr.s8 q0, q0, #1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = add <16 x i8> %src1, %src2
+  %1 = ashr <16 x i8> %0, <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
+  ret <16 x i8> %1
+}
+
+define arm_aapcs_vfpcc <8 x i16> @add_ashr_v8i16(<8 x i16> %src1, <8 x i16> %src2) {
+; CHECK-LABEL: add_ashr_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vadd.i16 q0, q0, q1
+; CHECK-NEXT:    vshr.s16 q0, q0, #1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = add <8 x i16> %src1, %src2
+  %1 = ashr <8 x i16> %0, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
+  ret <8 x i16> %1
+}
+
+define arm_aapcs_vfpcc <4 x i32> @add_ashr_v4i32(<4 x i32> %src1, <4 x i32> %src2) {
+; CHECK-LABEL: add_ashr_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vadd.i32 q0, q0, q1
+; CHECK-NEXT:    vshr.s32 q0, q0, #1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = add nsw <4 x i32> %src1, %src2
+  %1 = ashr <4 x i32> %0, <i32 1, i32 1, i32 1, i32 1>
+  ret <4 x i32> %1
+}
+
+define arm_aapcs_vfpcc <16 x i8> @add_lshr_v16i8(<16 x i8> %src1, <16 x i8> %src2) {
+; CHECK-LABEL: add_lshr_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vadd.i8 q0, q0, q1
+; CHECK-NEXT:    vshr.u8 q0, q0, #1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = add <16 x i8> %src1, %src2
+  %1 = lshr <16 x i8> %0, <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
+  ret <16 x i8> %1
+}
+
+define arm_aapcs_vfpcc <8 x i16> @add_lshr_v8i16(<8 x i16> %src1, <8 x i16> %src2) {
+; CHECK-LABEL: add_lshr_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vadd.i16 q0, q0, q1
+; CHECK-NEXT:    vshr.u16 q0, q0, #1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = add <8 x i16> %src1, %src2
+  %1 = lshr <8 x i16> %0, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
+  ret <8 x i16> %1
+}
+
+define arm_aapcs_vfpcc <4 x i32> @add_lshr_v4i32(<4 x i32> %src1, <4 x i32> %src2) {
+; CHECK-LABEL: add_lshr_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vadd.i32 q0, q0, q1
+; CHECK-NEXT:    vshr.u32 q0, q0, #1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = add nsw <4 x i32> %src1, %src2
+  %1 = lshr <4 x i32> %0, <i32 1, i32 1, i32 1, i32 1>
+  ret <4 x i32> %1
+}
+
+define arm_aapcs_vfpcc <16 x i8> @sub_ashr_v16i8(<16 x i8> %src1, <16 x i8> %src2) {
+; CHECK-LABEL: sub_ashr_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vsub.i8 q0, q0, q1
+; CHECK-NEXT:    vshr.s8 q0, q0, #1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = sub <16 x i8> %src1, %src2
+  %1 = ashr <16 x i8> %0, <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
+  ret <16 x i8> %1
+}
+
+define arm_aapcs_vfpcc <8 x i16> @sub_ashr_v8i16(<8 x i16> %src1, <8 x i16> %src2) {
+; CHECK-LABEL: sub_ashr_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vsub.i16 q0, q0, q1
+; CHECK-NEXT:    vshr.s16 q0, q0, #1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = sub <8 x i16> %src1, %src2
+  %1 = ashr <8 x i16> %0, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
+  ret <8 x i16> %1
+}
+
+define arm_aapcs_vfpcc <4 x i32> @sub_ashr_v4i32(<4 x i32> %src1, <4 x i32> %src2) {
+; CHECK-LABEL: sub_ashr_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vsub.i32 q0, q0, q1
+; CHECK-NEXT:    vshr.s32 q0, q0, #1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = sub nsw <4 x i32> %src1, %src2
+  %1 = ashr <4 x i32> %0, <i32 1, i32 1, i32 1, i32 1>
+  ret <4 x i32> %1
+}
+
+define arm_aapcs_vfpcc <16 x i8> @sub_lshr_v16i8(<16 x i8> %src1, <16 x i8> %src2) {
+; CHECK-LABEL: sub_lshr_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vsub.i8 q0, q0, q1
+; CHECK-NEXT:    vshr.u8 q0, q0, #1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = sub <16 x i8> %src1, %src2
+  %1 = lshr <16 x i8> %0, <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
+  ret <16 x i8> %1
+}
+
+define arm_aapcs_vfpcc <8 x i16> @sub_lshr_v8i16(<8 x i16> %src1, <8 x i16> %src2) {
+; CHECK-LABEL: sub_lshr_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vsub.i16 q0, q0, q1
+; CHECK-NEXT:    vshr.u16 q0, q0, #1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = sub <8 x i16> %src1, %src2
+  %1 = lshr <8 x i16> %0, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
+  ret <8 x i16> %1
+}
+
+define arm_aapcs_vfpcc <4 x i32> @sub_lshr_v4i32(<4 x i32> %src1, <4 x i32> %src2) {
+; CHECK-LABEL: sub_lshr_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vsub.i32 q0, q0, q1
+; CHECK-NEXT:    vshr.u32 q0, q0, #1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = sub nsw <4 x i32> %src1, %src2
+  %1 = lshr <4 x i32> %0, <i32 1, i32 1, i32 1, i32 1>
+  ret <4 x i32> %1
+}
+
+
+
+define arm_aapcs_vfpcc <16 x i8> @add_sdiv_v16i8(<16 x i8> %src1, <16 x i8> %src2) {
+; CHECK-LABEL: add_sdiv_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vadd.i8 q0, q0, q1
+; CHECK-NEXT:    vshr.u8 q1, q0, #7
+; CHECK-NEXT:    vadd.i8 q0, q0, q1
+; CHECK-NEXT:    vshr.s8 q0, q0, #1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = add <16 x i8> %src1, %src2
+  %1 = sdiv <16 x i8> %0, <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>
+  ret <16 x i8> %1
+}
+
+define arm_aapcs_vfpcc <8 x i16> @add_sdiv_v8i16(<8 x i16> %src1, <8 x i16> %src2) {
+; CHECK-LABEL: add_sdiv_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vadd.i16 q0, q0, q1
+; CHECK-NEXT:    vshr.u16 q1, q0, #15
+; CHECK-NEXT:    vadd.i16 q0, q0, q1
+; CHECK-NEXT:    vshr.s16 q0, q0, #1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = add <8 x i16> %src1, %src2
+  %1 = sdiv <8 x i16> %0, <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
+  ret <8 x i16> %1
+}
+
+define arm_aapcs_vfpcc <4 x i32> @add_sdiv_v4i32(<4 x i32> %src1, <4 x i32> %src2) {
+; CHECK-LABEL: add_sdiv_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vadd.i32 q0, q0, q1
+; CHECK-NEXT:    vshr.u32 q1, q0, #31
+; CHECK-NEXT:    vadd.i32 q0, q0, q1
+; CHECK-NEXT:    vshr.s32 q0, q0, #1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = add nsw <4 x i32> %src1, %src2
+  %1 = sdiv <4 x i32> %0, <i32 2, i32 2, i32 2, i32 2>
+  ret <4 x i32> %1
+}
+
+define arm_aapcs_vfpcc <16 x i8> @add_udiv_v16i8(<16 x i8> %src1, <16 x i8> %src2) {
+; CHECK-LABEL: add_udiv_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vadd.i8 q0, q0, q1
+; CHECK-NEXT:    vshr.u8 q0, q0, #1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = add <16 x i8> %src1, %src2
+  %1 = udiv <16 x i8> %0, <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>
+  ret <16 x i8> %1
+}
+
+define arm_aapcs_vfpcc <8 x i16> @add_udiv_v8i16(<8 x i16> %src1, <8 x i16> %src2) {
+; CHECK-LABEL: add_udiv_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vadd.i16 q0, q0, q1
+; CHECK-NEXT:    vshr.u16 q0, q0, #1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = add <8 x i16> %src1, %src2
+  %1 = udiv <8 x i16> %0, <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
+  ret <8 x i16> %1
+}
+
+define arm_aapcs_vfpcc <4 x i32> @add_udiv_v4i32(<4 x i32> %src1, <4 x i32> %src2) {
+; CHECK-LABEL: add_udiv_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vadd.i32 q0, q0, q1
+; CHECK-NEXT:    vshr.u32 q0, q0, #1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = add nsw <4 x i32> %src1, %src2
+  %1 = udiv <4 x i32> %0, <i32 2, i32 2, i32 2, i32 2>
+  ret <4 x i32> %1
+}
+
+define arm_aapcs_vfpcc <16 x i8> @sub_sdiv_v16i8(<16 x i8> %src1, <16 x i8> %src2) {
+; CHECK-LABEL: sub_sdiv_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vsub.i8 q0, q0, q1
+; CHECK-NEXT:    vshr.u8 q1, q0, #7
+; CHECK-NEXT:    vadd.i8 q0, q0, q1
+; CHECK-NEXT:    vshr.s8 q0, q0, #1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = sub <16 x i8> %src1, %src2
+  %1 = sdiv <16 x i8> %0, <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>
+  ret <16 x i8> %1
+}
+
+define arm_aapcs_vfpcc <8 x i16> @sub_sdiv_v8i16(<8 x i16> %src1, <8 x i16> %src2) {
+; CHECK-LABEL: sub_sdiv_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vsub.i16 q0, q0, q1
+; CHECK-NEXT:    vshr.u16 q1, q0, #15
+; CHECK-NEXT:    vadd.i16 q0, q0, q1
+; CHECK-NEXT:    vshr.s16 q0, q0, #1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = sub <8 x i16> %src1, %src2
+  %1 = sdiv <8 x i16> %0, <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
+  ret <8 x i16> %1
+}
+
+define arm_aapcs_vfpcc <4 x i32> @sub_sdiv_v4i32(<4 x i32> %src1, <4 x i32> %src2) {
+; CHECK-LABEL: sub_sdiv_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vsub.i32 q0, q0, q1
+; CHECK-NEXT:    vshr.u32 q1, q0, #31
+; CHECK-NEXT:    vadd.i32 q0, q0, q1
+; CHECK-NEXT:    vshr.s32 q0, q0, #1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = sub nsw <4 x i32> %src1, %src2
+  %1 = sdiv <4 x i32> %0, <i32 2, i32 2, i32 2, i32 2>
+  ret <4 x i32> %1
+}
+
+define arm_aapcs_vfpcc <16 x i8> @sub_udiv_v16i8(<16 x i8> %src1, <16 x i8> %src2) {
+; CHECK-LABEL: sub_udiv_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vsub.i8 q0, q0, q1
+; CHECK-NEXT:    vshr.u8 q0, q0, #1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = sub <16 x i8> %src1, %src2
+  %1 = udiv <16 x i8> %0, <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>
+  ret <16 x i8> %1
+}
+
+define arm_aapcs_vfpcc <8 x i16> @sub_udiv_v8i16(<8 x i16> %src1, <8 x i16> %src2) {
+; CHECK-LABEL: sub_udiv_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vsub.i16 q0, q0, q1
+; CHECK-NEXT:    vshr.u16 q0, q0, #1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = sub <8 x i16> %src1, %src2
+  %1 = udiv <8 x i16> %0, <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
+  ret <8 x i16> %1
+}
+
+define arm_aapcs_vfpcc <4 x i32> @sub_udiv_v4i32(<4 x i32> %src1, <4 x i32> %src2) {
+; CHECK-LABEL: sub_udiv_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vsub.i32 q0, q0, q1
+; CHECK-NEXT:    vshr.u32 q0, q0, #1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = sub nsw <4 x i32> %src1, %src2
+  %1 = udiv <4 x i32> %0, <i32 2, i32 2, i32 2, i32 2>
+  ret <4 x i32> %1
+}
+
diff --git a/llvm/test/CodeGen/Thumb2/mve-vld2.ll b/llvm/test/CodeGen/Thumb2/mve-vld2.ll
new file mode 100644
index 000000000..fbeb08558
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-vld2.ll
@@ -0,0 +1,615 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve.fp,+fp64 -verify-machineinstrs %s -o - | FileCheck %s
+
+; i32
+
+define void @vld2_v2i32(<4 x i32> *%src, <2 x i32> *%dst) {
+; CHECK-LABEL: vld2_v2i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vrev64.32 q1, q0
+; CHECK-NEXT:    vmov r2, s2
+; CHECK-NEXT:    vmov r0, s6
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    add r0, r2
+; CHECK-NEXT:    vmov r2, s4
+; CHECK-NEXT:    add r2, r3
+; CHECK-NEXT:    strd r2, r0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %l1 = load <4 x i32>, <4 x i32>* %src, align 4
+  %s1 = shufflevector <4 x i32> %l1, <4 x i32> undef, <2 x i32> <i32 0, i32 2>
+  %s2 = shufflevector <4 x i32> %l1, <4 x i32> undef, <2 x i32> <i32 1, i32 3>
+  %a = add <2 x i32> %s1, %s2
+  store <2 x i32> %a, <2 x i32> *%dst
+  ret void
+}
+
+define void @vld2_v4i32(<8 x i32> *%src, <4 x i32> *%dst) {
+; CHECK-LABEL: vld2_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vld20.32 {q0, q1}, [r0]
+; CHECK-NEXT:    vld21.32 {q0, q1}, [r0]
+; CHECK-NEXT:    vadd.i32 q0, q0, q1
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %l1 = load <8 x i32>, <8 x i32>* %src, align 4
+  %s1 = shufflevector <8 x i32> %l1, <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  %s2 = shufflevector <8 x i32> %l1, <8 x i32> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+  %a = add <4 x i32> %s1, %s2
+  store <4 x i32> %a, <4 x i32> *%dst
+  ret void
+}
+
+define void @vld2_v8i32(<16 x i32> *%src, <8 x i32> *%dst) {
+; CHECK-LABEL: vld2_v8i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    add.w r2, r0, #32
+; CHECK-NEXT:    vld20.32 {q0, q1}, [r0]
+; CHECK-NEXT:    vld20.32 {q2, q3}, [r2]
+; CHECK-NEXT:    vld21.32 {q0, q1}, [r0]
+; CHECK-NEXT:    vld21.32 {q2, q3}, [r2]
+; CHECK-NEXT:    vadd.i32 q0, q0, q1
+; CHECK-NEXT:    vadd.i32 q1, q2, q3
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vstrw.32 q1, [r1, #16]
+; CHECK-NEXT:    bx lr
+entry:
+  %l1 = load <16 x i32>, <16 x i32>* %src, align 4
+  %s1 = shufflevector <16 x i32> %l1, <16 x i32> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+  %s2 = shufflevector <16 x i32> %l1, <16 x i32> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+  %a = add <8 x i32> %s1, %s2
+  store <8 x i32> %a, <8 x i32> *%dst
+  ret void
+}
+
+define void @vld2_v16i32(<32 x i32> *%src, <16 x i32> *%dst) {
+; CHECK-LABEL: vld2_v16i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT:    vld20.32 {q0, q1}, [r0]
+; CHECK-NEXT:    add.w r12, r0, #96
+; CHECK-NEXT:    add.w r3, r0, #32
+; CHECK-NEXT:    add.w r2, r0, #64
+; CHECK-NEXT:    vld21.32 {q0, q1}, [r0]
+; CHECK-NEXT:    vadd.i32 q0, q0, q1
+; CHECK-NEXT:    vld20.32 {q1, q2}, [r2]
+; CHECK-NEXT:    vld20.32 {q3, q4}, [r12]
+; CHECK-NEXT:    vld20.32 {q5, q6}, [r3]
+; CHECK-NEXT:    vld21.32 {q5, q6}, [r3]
+; CHECK-NEXT:    vld21.32 {q1, q2}, [r2]
+; CHECK-NEXT:    vld21.32 {q3, q4}, [r12]
+; CHECK-NEXT:    @ kill: def $q1 killed $q1 killed $q1_q2
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vadd.i32 q5, q5, q6
+; CHECK-NEXT:    vadd.i32 q1, q1, q2
+; CHECK-NEXT:    vadd.i32 q3, q3, q4
+; CHECK-NEXT:    vstrw.32 q1, [r1, #32]
+; CHECK-NEXT:    vstrw.32 q3, [r1, #48]
+; CHECK-NEXT:    vstrw.32 q5, [r1, #16]
+; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT:    bx lr
+entry:
+  %l1 = load <32 x i32>, <32 x i32>* %src, align 4
+  %s1 = shufflevector <32 x i32> %l1, <32 x i32> undef, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
+  %s2 = shufflevector <32 x i32> %l1, <32 x i32> undef, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
+  %a = add <16 x i32> %s1, %s2
+  store <16 x i32> %a, <16 x i32> *%dst
+  ret void
+}
+
+; i16
+
+define void @vld2_v2i16(<4 x i16> *%src, <2 x i16> *%dst) {
+; CHECK-LABEL: vld2_v2i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u32 q0, [r0]
+; CHECK-NEXT:    vrev64.32 q1, q0
+; CHECK-NEXT:    vmov r2, s2
+; CHECK-NEXT:    vmov r0, s6
+; CHECK-NEXT:    add r0, r2
+; CHECK-NEXT:    strh r0, [r1, #2]
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    add r0, r2
+; CHECK-NEXT:    strh r0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %l1 = load <4 x i16>, <4 x i16>* %src, align 4
+  %s1 = shufflevector <4 x i16> %l1, <4 x i16> undef, <2 x i32> <i32 0, i32 2>
+  %s2 = shufflevector <4 x i16> %l1, <4 x i16> undef, <2 x i32> <i32 1, i32 3>
+  %a = add <2 x i16> %s1, %s2
+  store <2 x i16> %a, <2 x i16> *%dst
+  ret void
+}
+
+define void @vld2_v4i16(<8 x i16> *%src, <4 x i16> *%dst) {
+; CHECK-LABEL: vld2_v4i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vrev32.16 q1, q0
+; CHECK-NEXT:    vadd.i32 q0, q0, q1
+; CHECK-NEXT:    vstrh.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %l1 = load <8 x i16>, <8 x i16>* %src, align 4
+  %s1 = shufflevector <8 x i16> %l1, <8 x i16> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  %s2 = shufflevector <8 x i16> %l1, <8 x i16> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+  %a = add <4 x i16> %s1, %s2
+  store <4 x i16> %a, <4 x i16> *%dst
+  ret void
+}
+
+define void @vld2_v8i16(<16 x i16> *%src, <8 x i16> *%dst) {
+; CHECK-LABEL: vld2_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vld20.16 {q0, q1}, [r0]
+; CHECK-NEXT:    vld21.16 {q0, q1}, [r0]
+; CHECK-NEXT:    vadd.i16 q0, q0, q1
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %l1 = load <16 x i16>, <16 x i16>* %src, align 4
+  %s1 = shufflevector <16 x i16> %l1, <16 x i16> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+  %s2 = shufflevector <16 x i16> %l1, <16 x i16> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+  %a = add <8 x i16> %s1, %s2
+  store <8 x i16> %a, <8 x i16> *%dst
+  ret void
+}
+
+define void @vld2_v16i16(<32 x i16> *%src, <16 x i16> *%dst) {
+; CHECK-LABEL: vld2_v16i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    add.w r2, r0, #32
+; CHECK-NEXT:    vld20.16 {q0, q1}, [r0]
+; CHECK-NEXT:    vld20.16 {q2, q3}, [r2]
+; CHECK-NEXT:    vld21.16 {q0, q1}, [r0]
+; CHECK-NEXT:    vld21.16 {q2, q3}, [r2]
+; CHECK-NEXT:    vadd.i16 q0, q0, q1
+; CHECK-NEXT:    vadd.i16 q1, q2, q3
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vstrw.32 q1, [r1, #16]
+; CHECK-NEXT:    bx lr
+entry:
+  %l1 = load <32 x i16>, <32 x i16>* %src, align 4
+  %s1 = shufflevector <32 x i16> %l1, <32 x i16> undef, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
+  %s2 = shufflevector <32 x i16> %l1, <32 x i16> undef, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
+  %a = add <16 x i16> %s1, %s2
+  store <16 x i16> %a, <16 x i16> *%dst
+  ret void
+}
+
+; i8
+
+define void @vld2_v2i8(<4 x i8> *%src, <2 x i8> *%dst) {
+; CHECK-LABEL: vld2_v2i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u32 q0, [r0]
+; CHECK-NEXT:    vrev64.32 q1, q0
+; CHECK-NEXT:    vmov r2, s2
+; CHECK-NEXT:    vmov r0, s6
+; CHECK-NEXT:    add r0, r2
+; CHECK-NEXT:    strb r0, [r1, #1]
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    add r0, r2
+; CHECK-NEXT:    strb r0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %l1 = load <4 x i8>, <4 x i8>* %src, align 4
+  %s1 = shufflevector <4 x i8> %l1, <4 x i8> undef, <2 x i32> <i32 0, i32 2>
+  %s2 = shufflevector <4 x i8> %l1, <4 x i8> undef, <2 x i32> <i32 1, i32 3>
+  %a = add <2 x i8> %s1, %s2
+  store <2 x i8> %a, <2 x i8> *%dst
+  ret void
+}
+
+define void @vld2_v4i8(<8 x i8> *%src, <4 x i8> *%dst) {
+; CHECK-LABEL: vld2_v4i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u16 q0, [r0]
+; CHECK-NEXT:    vrev32.16 q1, q0
+; CHECK-NEXT:    vadd.i32 q0, q0, q1
+; CHECK-NEXT:    vstrb.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %l1 = load <8 x i8>, <8 x i8>* %src, align 4
+  %s1 = shufflevector <8 x i8> %l1, <8 x i8> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  %s2 = shufflevector <8 x i8> %l1, <8 x i8> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+  %a = add <4 x i8> %s1, %s2
+  store <4 x i8> %a, <4 x i8> *%dst
+  ret void
+}
+
+define void @vld2_v8i8(<16 x i8> *%src, <8 x i8> *%dst) {
+; CHECK-LABEL: vld2_v8i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vrev16.8 q1, q0
+; CHECK-NEXT:    vadd.i16 q0, q0, q1
+; CHECK-NEXT:    vstrb.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %l1 = load <16 x i8>, <16 x i8>* %src, align 4
+  %s1 = shufflevector <16 x i8> %l1, <16 x i8> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+  %s2 = shufflevector <16 x i8> %l1, <16 x i8> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+  %a = add <8 x i8> %s1, %s2
+  store <8 x i8> %a, <8 x i8> *%dst
+  ret void
+}
+
+define void @vld2_v16i8(<32 x i8> *%src, <16 x i8> *%dst) {
+; CHECK-LABEL: vld2_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vld20.8 {q0, q1}, [r0]
+; CHECK-NEXT:    vld21.8 {q0, q1}, [r0]
+; CHECK-NEXT:    vadd.i8 q0, q0, q1
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %l1 = load <32 x i8>, <32 x i8>* %src, align 4
+  %s1 = shufflevector <32 x i8> %l1, <32 x i8> undef, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
+  %s2 = shufflevector <32 x i8> %l1, <32 x i8> undef, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
+  %a = add <16 x i8> %s1, %s2
+  store <16 x i8> %a, <16 x i8> *%dst
+  ret void
+}
+
+; i64
+
+define void @vld2_v2i64(<4 x i64> *%src, <2 x i64> *%dst) {
+; CHECK-LABEL: vld2_v2i64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, lr}
+; CHECK-NEXT:    push {r4, lr}
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #16]
+; CHECK-NEXT:    vmov.f64 d2, d1
+; CHECK-NEXT:    vmov.f32 s5, s3
+; CHECK-NEXT:    vmov.f32 s6, s10
+; CHECK-NEXT:    vmov.f32 s2, s8
+; CHECK-NEXT:    vmov.f32 s3, s9
+; CHECK-NEXT:    vmov.f32 s7, s11
+; CHECK-NEXT:    vmov r3, s6
+; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    vmov r4, s0
+; CHECK-NEXT:    vmov r2, s3
+; CHECK-NEXT:    vmov r12, s7
+; CHECK-NEXT:    adds.w lr, r0, r3
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmov r3, s5
+; CHECK-NEXT:    adc.w r12, r12, r2
+; CHECK-NEXT:    vmov r2, s1
+; CHECK-NEXT:    adds r0, r0, r4
+; CHECK-NEXT:    vmov.32 q0[0], r0
+; CHECK-NEXT:    adcs r2, r3
+; CHECK-NEXT:    vmov.32 q0[1], r2
+; CHECK-NEXT:    vmov.32 q0[2], lr
+; CHECK-NEXT:    vmov.32 q0[3], r12
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    pop {r4, pc}
+entry:
+  %l1 = load <4 x i64>, <4 x i64>* %src, align 4
+  %s1 = shufflevector <4 x i64> %l1, <4 x i64> undef, <2 x i32> <i32 0, i32 2>
+  %s2 = shufflevector <4 x i64> %l1, <4 x i64> undef, <2 x i32> <i32 1, i32 3>
+  %a = add <2 x i64> %s1, %s2
+  store <2 x i64> %a, <2 x i64> *%dst
+  ret void
+}
+
+define void @vld2_v4i64(<8 x i64> *%src, <4 x i64> *%dst) {
+; CHECK-LABEL: vld2_v4i64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, lr}
+; CHECK-NEXT:    push {r4, lr}
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11}
+; CHECK-NEXT:    vldrw.u32 q3, [r0, #32]
+; CHECK-NEXT:    vldrw.u32 q5, [r0, #48]
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #16]
+; CHECK-NEXT:    vmov.f64 d8, d7
+; CHECK-NEXT:    vmov.f32 s17, s15
+; CHECK-NEXT:    vmov.f32 s18, s22
+; CHECK-NEXT:    vmov.f32 s14, s20
+; CHECK-NEXT:    vmov.f32 s15, s21
+; CHECK-NEXT:    vmov.f32 s19, s23
+; CHECK-NEXT:    vmov r3, s18
+; CHECK-NEXT:    vmov r0, s14
+; CHECK-NEXT:    vmov r4, s12
+; CHECK-NEXT:    vmov.f64 d2, d1
+; CHECK-NEXT:    vmov r12, s19
+; CHECK-NEXT:    vmov r2, s15
+; CHECK-NEXT:    vmov.f32 s5, s3
+; CHECK-NEXT:    vmov.f32 s6, s10
+; CHECK-NEXT:    vmov.f32 s2, s8
+; CHECK-NEXT:    vmov.f32 s7, s11
+; CHECK-NEXT:    vmov.f32 s3, s9
+; CHECK-NEXT:    adds.w lr, r0, r3
+; CHECK-NEXT:    vmov r0, s16
+; CHECK-NEXT:    vmov r3, s17
+; CHECK-NEXT:    adc.w r12, r12, r2
+; CHECK-NEXT:    vmov r2, s13
+; CHECK-NEXT:    adds r0, r0, r4
+; CHECK-NEXT:    vmov r4, s2
+; CHECK-NEXT:    vmov.32 q3[0], r0
+; CHECK-NEXT:    vmov r0, s7
+; CHECK-NEXT:    adcs r2, r3
+; CHECK-NEXT:    vmov r3, s6
+; CHECK-NEXT:    vmov.32 q3[1], r2
+; CHECK-NEXT:    vmov r2, s3
+; CHECK-NEXT:    vmov.32 q3[2], lr
+; CHECK-NEXT:    vmov.32 q3[3], r12
+; CHECK-NEXT:    vstrw.32 q3, [r1, #16]
+; CHECK-NEXT:    adds.w lr, r4, r3
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    adc.w r12, r2, r0
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmov r2, s5
+; CHECK-NEXT:    vmov r4, s1
+; CHECK-NEXT:    adds r0, r0, r3
+; CHECK-NEXT:    vmov.32 q0[0], r0
+; CHECK-NEXT:    adcs r2, r4
+; CHECK-NEXT:    vmov.32 q0[1], r2
+; CHECK-NEXT:    vmov.32 q0[2], lr
+; CHECK-NEXT:    vmov.32 q0[3], r12
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vpop {d8, d9, d10, d11}
+; CHECK-NEXT:    pop {r4, pc}
+entry:
+  %l1 = load <8 x i64>, <8 x i64>* %src, align 4
+  %s1 = shufflevector <8 x i64> %l1, <8 x i64> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  %s2 = shufflevector <8 x i64> %l1, <8 x i64> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+  %a = add <4 x i64> %s1, %s2
+  store <4 x i64> %a, <4 x i64> *%dst
+  ret void
+}
+
+; f32
+
+define void @vld2_v2f32(<4 x float> *%src, <2 x float> *%dst) {
+; CHECK-LABEL: vld2_v2f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vmov.f32 s4, s1
+; CHECK-NEXT:    vmov.f32 s5, s3
+; CHECK-NEXT:    vmov.f32 s1, s2
+; CHECK-NEXT:    vadd.f32 q0, q0, q1
+; CHECK-NEXT:    vstmia r1, {s0, s1}
+; CHECK-NEXT:    bx lr
+entry:
+  %l1 = load <4 x float>, <4 x float>* %src, align 4
+  %s1 = shufflevector <4 x float> %l1, <4 x float> undef, <2 x i32> <i32 0, i32 2>
+  %s2 = shufflevector <4 x float> %l1, <4 x float> undef, <2 x i32> <i32 1, i32 3>
+  %a = fadd <2 x float> %s1, %s2
+  store <2 x float> %a, <2 x float> *%dst
+  ret void
+}
+
+define void @vld2_v4f32(<8 x float> *%src, <4 x float> *%dst) {
+; CHECK-LABEL: vld2_v4f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vld20.32 {q0, q1}, [r0]
+; CHECK-NEXT:    vld21.32 {q0, q1}, [r0]
+; CHECK-NEXT:    vadd.f32 q0, q0, q1
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %l1 = load <8 x float>, <8 x float>* %src, align 4
+  %s1 = shufflevector <8 x float> %l1, <8 x float> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  %s2 = shufflevector <8 x float> %l1, <8 x float> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+  %a = fadd <4 x float> %s1, %s2
+  store <4 x float> %a, <4 x float> *%dst
+  ret void
+}
+
+define void @vld2_v8f32(<16 x float> *%src, <8 x float> *%dst) {
+; CHECK-LABEL: vld2_v8f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    add.w r2, r0, #32
+; CHECK-NEXT:    vld20.32 {q0, q1}, [r0]
+; CHECK-NEXT:    vld20.32 {q2, q3}, [r2]
+; CHECK-NEXT:    vld21.32 {q0, q1}, [r0]
+; CHECK-NEXT:    vld21.32 {q2, q3}, [r2]
+; CHECK-NEXT:    vadd.f32 q0, q0, q1
+; CHECK-NEXT:    vadd.f32 q1, q2, q3
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vstrw.32 q1, [r1, #16]
+; CHECK-NEXT:    bx lr
+entry:
+  %l1 = load <16 x float>, <16 x float>* %src, align 4
+  %s1 = shufflevector <16 x float> %l1, <16 x float> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+  %s2 = shufflevector <16 x float> %l1, <16 x float> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+  %a = fadd <8 x float> %s1, %s2
+  store <8 x float> %a, <8 x float> *%dst
+  ret void
+}
+
+define void @vld2_v16f32(<32 x float> *%src, <16 x float> *%dst) {
+; CHECK-LABEL: vld2_v16f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT:    vld20.32 {q0, q1}, [r0]
+; CHECK-NEXT:    add.w r12, r0, #96
+; CHECK-NEXT:    add.w r3, r0, #32
+; CHECK-NEXT:    add.w r2, r0, #64
+; CHECK-NEXT:    vld21.32 {q0, q1}, [r0]
+; CHECK-NEXT:    vadd.f32 q0, q0, q1
+; CHECK-NEXT:    vld20.32 {q1, q2}, [r2]
+; CHECK-NEXT:    vld20.32 {q3, q4}, [r12]
+; CHECK-NEXT:    vld20.32 {q5, q6}, [r3]
+; CHECK-NEXT:    vld21.32 {q5, q6}, [r3]
+; CHECK-NEXT:    vld21.32 {q1, q2}, [r2]
+; CHECK-NEXT:    vld21.32 {q3, q4}, [r12]
+; CHECK-NEXT:    @ kill: def $q1 killed $q1 killed $q1_q2
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vadd.f32 q5, q5, q6
+; CHECK-NEXT:    vadd.f32 q1, q1, q2
+; CHECK-NEXT:    vadd.f32 q3, q3, q4
+; CHECK-NEXT:    vstrw.32 q1, [r1, #32]
+; CHECK-NEXT:    vstrw.32 q3, [r1, #48]
+; CHECK-NEXT:    vstrw.32 q5, [r1, #16]
+; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT:    bx lr
+entry:
+  %l1 = load <32 x float>, <32 x float>* %src, align 4
+  %s1 = shufflevector <32 x float> %l1, <32 x float> undef, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
+  %s2 = shufflevector <32 x float> %l1, <32 x float> undef, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
+  %a = fadd <16 x float> %s1, %s2
+  store <16 x float> %a, <16 x float> *%dst
+  ret void
+}
+
+; f16
+
+define void @vld2_v2f16(<4 x half> *%src, <2 x half> *%dst) {
+; CHECK-LABEL: vld2_v2f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    ldrd r2, r0, [r0]
+; CHECK-NEXT:    vmov.32 q0[0], r2
+; CHECK-NEXT:    vmov.32 q0[1], r0
+; CHECK-NEXT:    vmovx.f16 s4, s1
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmovx.f16 s4, s0
+; CHECK-NEXT:    vmov r2, s4
+; CHECK-NEXT:    vmov.16 q1[0], r2
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vmov.16 q1[1], r0
+; CHECK-NEXT:    vmov r0, s1
+; CHECK-NEXT:    vmov.16 q0[0], r2
+; CHECK-NEXT:    vmov.16 q0[1], r0
+; CHECK-NEXT:    vadd.f16 q0, q0, q1
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    str r0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %l1 = load <4 x half>, <4 x half>* %src, align 4
+  %s1 = shufflevector <4 x half> %l1, <4 x half> undef, <2 x i32> <i32 0, i32 2>
+  %s2 = shufflevector <4 x half> %l1, <4 x half> undef, <2 x i32> <i32 1, i32 3>
+  %a = fadd <2 x half> %s1, %s2
+  store <2 x half> %a, <2 x half> *%dst
+  ret void
+}
+
+define void @vld2_v4f16(<8 x half> *%src, <4 x half> *%dst) {
+; CHECK-LABEL: vld2_v4f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vmovx.f16 s8, s0
+; CHECK-NEXT:    vmov r0, s1
+; CHECK-NEXT:    vmov.16 q1[0], r2
+; CHECK-NEXT:    vmov.16 q1[1], r0
+; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    vmov.16 q1[2], r0
+; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    vmovx.f16 s8, s1
+; CHECK-NEXT:    vmovx.f16 s12, s2
+; CHECK-NEXT:    vmov r2, s8
+; CHECK-NEXT:    vmov.16 q2[0], r0
+; CHECK-NEXT:    vmov r0, s12
+; CHECK-NEXT:    vmov.16 q2[1], r2
+; CHECK-NEXT:    vmovx.f16 s12, s3
+; CHECK-NEXT:    vmov.16 q2[2], r0
+; CHECK-NEXT:    vmov r0, s12
+; CHECK-NEXT:    vmov.16 q2[3], r0
+; CHECK-NEXT:    vmov r0, s3
+; CHECK-NEXT:    vmov.16 q1[3], r0
+; CHECK-NEXT:    vadd.f16 q0, q1, q2
+; CHECK-NEXT:    vmov r2, s1
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    strd r0, r2, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %l1 = load <8 x half>, <8 x half>* %src, align 4
+  %s1 = shufflevector <8 x half> %l1, <8 x half> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  %s2 = shufflevector <8 x half> %l1, <8 x half> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+  %a = fadd <4 x half> %s1, %s2
+  store <4 x half> %a, <4 x half> *%dst
+  ret void
+}
+
+define void @vld2_v8f16(<16 x half> *%src, <8 x half> *%dst) {
+; CHECK-LABEL: vld2_v8f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vld20.16 {q0, q1}, [r0]
+; CHECK-NEXT:    vld21.16 {q0, q1}, [r0]
+; CHECK-NEXT:    vadd.f16 q0, q0, q1
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %l1 = load <16 x half>, <16 x half>* %src, align 4
+  %s1 = shufflevector <16 x half> %l1, <16 x half> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+  %s2 = shufflevector <16 x half> %l1, <16 x half> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+  %a = fadd <8 x half> %s1, %s2
+  store <8 x half> %a, <8 x half> *%dst
+  ret void
+}
+
+define void @vld2_v16f16(<32 x half> *%src, <16 x half> *%dst) {
+; CHECK-LABEL: vld2_v16f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    add.w r2, r0, #32
+; CHECK-NEXT:    vld20.16 {q0, q1}, [r2]
+; CHECK-NEXT:    vld21.16 {q0, q1}, [r2]
+; CHECK-NEXT:    vadd.f16 q0, q0, q1
+; CHECK-NEXT:    vld20.16 {q1, q2}, [r0]
+; CHECK-NEXT:    vld21.16 {q1, q2}, [r0]
+; CHECK-NEXT:    vstrw.32 q0, [r1, #16]
+; CHECK-NEXT:    vadd.f16 q0, q1, q2
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %l1 = load <32 x half>, <32 x half>* %src, align 4
+  %s1 = shufflevector <32 x half> %l1, <32 x half> undef, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
+  %s2 = shufflevector <32 x half> %l1, <32 x half> undef, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
+  %a = fadd <16 x half> %s1, %s2
+  store <16 x half> %a, <16 x half> *%dst
+  ret void
+}
+
+; f64
+
+define void @vld2_v2f64(<4 x double> *%src, <2 x double> *%dst) {
+; CHECK-LABEL: vld2_v2f64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #16]
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vadd.f64 d1, d0, d1
+; CHECK-NEXT:    vadd.f64 d0, d2, d3
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %l1 = load <4 x double>, <4 x double>* %src, align 4
+  %s1 = shufflevector <4 x double> %l1, <4 x double> undef, <2 x i32> <i32 0, i32 2>
+  %s2 = shufflevector <4 x double> %l1, <4 x double> undef, <2 x i32> <i32 1, i32 3>
+  %a = fadd <2 x double> %s1, %s2
+  store <2 x double> %a, <2 x double> *%dst
+  ret void
+}
+
+define void @vld2_v4f64(<8 x double> *%src, <4 x double> *%dst) {
+; CHECK-LABEL: vld2_v4f64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #48]
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #32]
+; CHECK-NEXT:    vldrw.u32 q2, [r0]
+; CHECK-NEXT:    vadd.f64 d1, d0, d1
+; CHECK-NEXT:    vadd.f64 d0, d2, d3
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #16]
+; CHECK-NEXT:    vadd.f64 d3, d2, d3
+; CHECK-NEXT:    vstrw.32 q0, [r1, #16]
+; CHECK-NEXT:    vadd.f64 d2, d4, d5
+; CHECK-NEXT:    vstrw.32 q1, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %l1 = load <8 x double>, <8 x double>* %src, align 4
+  %s1 = shufflevector <8 x double> %l1, <8 x double> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  %s2 = shufflevector <8 x double> %l1, <8 x double> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+  %a = fadd <4 x double> %s1, %s2
+  store <4 x double> %a, <4 x double> *%dst
+  ret void
+}
diff --git a/llvm/test/CodeGen/Thumb2/mve-vld3.ll b/llvm/test/CodeGen/Thumb2/mve-vld3.ll
new file mode 100644
index 000000000..c7f3c1d2f
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-vld3.ll
@@ -0,0 +1,1745 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve.fp,+fp64 -verify-machineinstrs %s -o - | FileCheck %s
+
+; i32
+
+define void @vld3_v2i32(<6 x i32> *%src, <2 x i32> *%dst) {
+; CHECK-LABEL: vld3_v2i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    ldrd r2, r3, [r0, #16]
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vmov.32 q2[0], r2
+; CHECK-NEXT:    vmov.f64 d2, d0
+; CHECK-NEXT:    vmov.32 q2[2], r3
+; CHECK-NEXT:    vmov.32 r0, q0[2]
+; CHECK-NEXT:    vmov.f32 s12, s1
+; CHECK-NEXT:    vmov.f32 s6, s3
+; CHECK-NEXT:    vmov.f32 s14, s8
+; CHECK-NEXT:    vmov r2, s12
+; CHECK-NEXT:    vmov r12, s6
+; CHECK-NEXT:    vdup.32 q1, r0
+; CHECK-NEXT:    vmov r0, s14
+; CHECK-NEXT:    add r0, r12
+; CHECK-NEXT:    add r0, r3
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    add r2, r3
+; CHECK-NEXT:    vmov r3, s4
+; CHECK-NEXT:    add r2, r3
+; CHECK-NEXT:    strd r2, r0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %l1 = load <6 x i32>, <6 x i32>* %src, align 4
+  %s1 = shufflevector <6 x i32> %l1, <6 x i32> undef, <2 x i32> <i32 0, i32 3>
+  %s2 = shufflevector <6 x i32> %l1, <6 x i32> undef, <2 x i32> <i32 1, i32 4>
+  %s3 = shufflevector <6 x i32> %l1, <6 x i32> undef, <2 x i32> <i32 2, i32 5>
+  %a1 = add <2 x i32> %s1, %s2
+  %a = add <2 x i32> %a1, %s3
+  store <2 x i32> %a, <2 x i32> *%dst
+  ret void
+}
+
+define void @vld3_v4i32(<12 x i32> *%src, <4 x i32> *%dst) {
+; CHECK-LABEL: vld3_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11}
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #16]
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #32]
+; CHECK-NEXT:    vmov.f32 s12, s5
+; CHECK-NEXT:    vmov.f32 s13, s0
+; CHECK-NEXT:    vmov.32 r0, q2[2]
+; CHECK-NEXT:    vdup.32 q4, r0
+; CHECK-NEXT:    vmov.f32 s14, s3
+; CHECK-NEXT:    vmov.f32 s15, s19
+; CHECK-NEXT:    vmov.32 r0, q2[1]
+; CHECK-NEXT:    vmov.f64 d8, d2
+; CHECK-NEXT:    vdup.32 q5, r0
+; CHECK-NEXT:    vmov.f32 s17, s7
+; CHECK-NEXT:    vmov.f32 s18, s2
+; CHECK-NEXT:    vmov.f32 s0, s6
+; CHECK-NEXT:    vmov.f32 s19, s23
+; CHECK-NEXT:    vmov.f32 s10, s8
+; CHECK-NEXT:    vadd.i32 q3, q4, q3
+; CHECK-NEXT:    vmov.f32 s2, s8
+; CHECK-NEXT:    vmov.f32 s3, s11
+; CHECK-NEXT:    vadd.i32 q0, q3, q0
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vpop {d8, d9, d10, d11}
+; CHECK-NEXT:    bx lr
+entry:
+  %l1 = load <12 x i32>, <12 x i32>* %src, align 4
+  %s1 = shufflevector <12 x i32> %l1, <12 x i32> undef, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
+  %s2 = shufflevector <12 x i32> %l1, <12 x i32> undef, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
+  %s3 = shufflevector <12 x i32> %l1, <12 x i32> undef, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
+  %a1 = add <4 x i32> %s1, %s2
+  %a = add <4 x i32> %a1, %s3
+  store <4 x i32> %a, <4 x i32> *%dst
+  ret void
+}
+
+define void @vld3_v8i32(<24 x i32> *%src, <8 x i32> *%dst) {
+; CHECK-LABEL: vld3_v8i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #48]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #64]
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #80]
+; CHECK-NEXT:    vmov.f32 s12, s5
+; CHECK-NEXT:    vmov.f32 s13, s0
+; CHECK-NEXT:    vmov.32 r2, q2[2]
+; CHECK-NEXT:    vdup.32 q4, r2
+; CHECK-NEXT:    vmov.f32 s14, s3
+; CHECK-NEXT:    vmov.f32 s15, s19
+; CHECK-NEXT:    vmov.32 r2, q2[1]
+; CHECK-NEXT:    vmov.f64 d8, d2
+; CHECK-NEXT:    vdup.32 q5, r2
+; CHECK-NEXT:    vmov.f32 s17, s7
+; CHECK-NEXT:    vmov.f32 s18, s2
+; CHECK-NEXT:    vmov.f32 s0, s6
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #16]
+; CHECK-NEXT:    vmov.f32 s19, s23
+; CHECK-NEXT:    vmov.f32 s10, s8
+; CHECK-NEXT:    vadd.i32 q3, q4, q3
+; CHECK-NEXT:    vmov.f32 s2, s8
+; CHECK-NEXT:    vmov.f32 s3, s11
+; CHECK-NEXT:    vldrw.u32 q2, [r0]
+; CHECK-NEXT:    vadd.i32 q0, q3, q0
+; CHECK-NEXT:    vldrw.u32 q3, [r0, #32]
+; CHECK-NEXT:    vmov.f32 s16, s9
+; CHECK-NEXT:    vmov.f32 s17, s4
+; CHECK-NEXT:    vmov.32 r0, q3[2]
+; CHECK-NEXT:    vdup.32 q5, r0
+; CHECK-NEXT:    vmov.f32 s18, s7
+; CHECK-NEXT:    vmov.f32 s19, s23
+; CHECK-NEXT:    vmov.32 r0, q3[1]
+; CHECK-NEXT:    vmov.f64 d10, d4
+; CHECK-NEXT:    vdup.32 q6, r0
+; CHECK-NEXT:    vstrw.32 q0, [r1, #16]
+; CHECK-NEXT:    vmov.f32 s21, s11
+; CHECK-NEXT:    vmov.f32 s22, s6
+; CHECK-NEXT:    vmov.f32 s4, s10
+; CHECK-NEXT:    vmov.f32 s23, s27
+; CHECK-NEXT:    vmov.f32 s14, s12
+; CHECK-NEXT:    vadd.i32 q4, q5, q4
+; CHECK-NEXT:    vmov.f32 s6, s12
+; CHECK-NEXT:    vmov.f32 s7, s15
+; CHECK-NEXT:    vadd.i32 q1, q4, q1
+; CHECK-NEXT:    vstrw.32 q1, [r1]
+; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT:    bx lr
+entry:
+  %l1 = load <24 x i32>, <24 x i32>* %src, align 4
+  %s1 = shufflevector <24 x i32> %l1, <24 x i32> undef, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
+  %s2 = shufflevector <24 x i32> %l1, <24 x i32> undef, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22>
+  %s3 = shufflevector <24 x i32> %l1, <24 x i32> undef, <8 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23>
+  %a1 = add <8 x i32> %s1, %s2
+  %a = add <8 x i32> %a1, %s3
+  store <8 x i32> %a, <8 x i32> *%dst
+  ret void
+}
+
+define void @vld3_v16i32(<48 x i32> *%src, <16 x i32> *%dst) {
+; CHECK-LABEL: vld3_v16i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    .pad #16
+; CHECK-NEXT:    sub sp, #16
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #48]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #64]
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #80]
+; CHECK-NEXT:    vmov.f32 s12, s5
+; CHECK-NEXT:    vmov.f32 s13, s0
+; CHECK-NEXT:    vmov.32 r2, q2[2]
+; CHECK-NEXT:    vdup.32 q4, r2
+; CHECK-NEXT:    vmov.f32 s14, s3
+; CHECK-NEXT:    vmov.f32 s15, s19
+; CHECK-NEXT:    vmov.32 r2, q2[1]
+; CHECK-NEXT:    vmov.f64 d8, d2
+; CHECK-NEXT:    vdup.32 q5, r2
+; CHECK-NEXT:    vmov.f32 s17, s7
+; CHECK-NEXT:    vmov.f32 s18, s2
+; CHECK-NEXT:    vmov.f32 s0, s6
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #16]
+; CHECK-NEXT:    vmov.f32 s19, s23
+; CHECK-NEXT:    vmov.f32 s10, s8
+; CHECK-NEXT:    vadd.i32 q3, q4, q3
+; CHECK-NEXT:    vmov.f32 s2, s8
+; CHECK-NEXT:    vmov.f32 s3, s11
+; CHECK-NEXT:    vldrw.u32 q2, [r0]
+; CHECK-NEXT:    vadd.i32 q0, q3, q0
+; CHECK-NEXT:    vldrw.u32 q3, [r0, #32]
+; CHECK-NEXT:    vmov.f32 s16, s9
+; CHECK-NEXT:    vstrw.32 q0, [sp] @ 16-byte Spill
+; CHECK-NEXT:    vmov.f32 s17, s4
+; CHECK-NEXT:    vmov.32 r2, q3[2]
+; CHECK-NEXT:    vdup.32 q5, r2
+; CHECK-NEXT:    vmov.f32 s18, s7
+; CHECK-NEXT:    vmov.f32 s19, s23
+; CHECK-NEXT:    vmov.32 r2, q3[1]
+; CHECK-NEXT:    vmov.f64 d10, d4
+; CHECK-NEXT:    vdup.32 q6, r2
+; CHECK-NEXT:    vmov.f32 s21, s11
+; CHECK-NEXT:    vmov.f32 s22, s6
+; CHECK-NEXT:    vmov.f32 s4, s10
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #160]
+; CHECK-NEXT:    vmov.f32 s23, s27
+; CHECK-NEXT:    vmov.f32 s14, s12
+; CHECK-NEXT:    vadd.i32 q4, q5, q4
+; CHECK-NEXT:    vmov.f32 s6, s12
+; CHECK-NEXT:    vmov.f32 s7, s15
+; CHECK-NEXT:    vldrw.u32 q3, [r0, #144]
+; CHECK-NEXT:    vadd.i32 q1, q4, q1
+; CHECK-NEXT:    vldrw.u32 q4, [r0, #176]
+; CHECK-NEXT:    vmov.f32 s20, s13
+; CHECK-NEXT:    vmov.f32 s21, s8
+; CHECK-NEXT:    vmov.32 r2, q4[2]
+; CHECK-NEXT:    vdup.32 q6, r2
+; CHECK-NEXT:    vmov.f32 s22, s11
+; CHECK-NEXT:    vmov.f32 s23, s27
+; CHECK-NEXT:    vmov.32 r2, q4[1]
+; CHECK-NEXT:    vmov.f64 d12, d6
+; CHECK-NEXT:    vdup.32 q7, r2
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #128]
+; CHECK-NEXT:    vmov.f32 s25, s15
+; CHECK-NEXT:    vmov.f32 s26, s10
+; CHECK-NEXT:    vmov.f32 s8, s14
+; CHECK-NEXT:    vmov.f32 s27, s31
+; CHECK-NEXT:    vmov.f32 s18, s16
+; CHECK-NEXT:    vadd.i32 q5, q6, q5
+; CHECK-NEXT:    vmov.f32 s10, s16
+; CHECK-NEXT:    vmov.f32 s11, s19
+; CHECK-NEXT:    vldrw.u32 q4, [r0, #96]
+; CHECK-NEXT:    vadd.i32 q2, q5, q2
+; CHECK-NEXT:    vldrw.u32 q5, [r0, #112]
+; CHECK-NEXT:    vmov.f32 s24, s17
+; CHECK-NEXT:    vmov.32 r0, q0[2]
+; CHECK-NEXT:    vmov.f32 s25, s20
+; CHECK-NEXT:    vdup.32 q7, r0
+; CHECK-NEXT:    vmov.f64 d6, d8
+; CHECK-NEXT:    vmov.32 r0, q0[1]
+; CHECK-NEXT:    vstrw.32 q2, [r1, #48]
+; CHECK-NEXT:    vstrw.32 q1, [r1]
+; CHECK-NEXT:    vmov.f32 s26, s23
+; CHECK-NEXT:    vmov.f32 s13, s19
+; CHECK-NEXT:    vmov.f32 s27, s31
+; CHECK-NEXT:    vdup.32 q7, r0
+; CHECK-NEXT:    vmov.f32 s14, s22
+; CHECK-NEXT:    vmov.f32 s20, s18
+; CHECK-NEXT:    vmov.f32 s15, s31
+; CHECK-NEXT:    vmov.f32 s2, s0
+; CHECK-NEXT:    vadd.i32 q6, q3, q6
+; CHECK-NEXT:    vmov.f32 s22, s0
+; CHECK-NEXT:    vmov.f32 s23, s3
+; CHECK-NEXT:    vadd.i32 q0, q6, q5
+; CHECK-NEXT:    vstrw.32 q0, [r1, #32]
+; CHECK-NEXT:    vldrw.u32 q0, [sp] @ 16-byte Reload
+; CHECK-NEXT:    vstrw.32 q0, [r1, #16]
+; CHECK-NEXT:    add sp, #16
+; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    bx lr
+entry:
+  %l1 = load <48 x i32>, <48 x i32>* %src, align 4
+  %s1 = shufflevector <48 x i32> %l1, <48 x i32> undef, <16 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21, i32 24, i32 27, i32 30, i32 33, i32 36, i32 39, i32 42, i32 45>
+  %s2 = shufflevector <48 x i32> %l1, <48 x i32> undef, <16 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22, i32 25, i32 28, i32 31, i32 34, i32 37, i32 40, i32 43, i32 46>
+  %s3 = shufflevector <48 x i32> %l1, <48 x i32> undef, <16 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23, i32 26, i32 29, i32 32, i32 35, i32 38, i32 41, i32 44, i32 47>
+  %a1 = add <16 x i32> %s1, %s2
+  %a = add <16 x i32> %a1, %s3
+  store <16 x i32> %a, <16 x i32> *%dst
+  ret void
+}
+
+; i16
+
+define void @vld3_v2i16(<6 x i16> *%src, <2 x i16> *%dst) {
+; CHECK-LABEL: vld3_v2i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .pad #8
+; CHECK-NEXT:    sub sp, #8
+; CHECK-NEXT:    vldrh.u32 q0, [r0]
+; CHECK-NEXT:    ldr r2, [r0, #8]
+; CHECK-NEXT:    mov r3, sp
+; CHECK-NEXT:    str r2, [sp]
+; CHECK-NEXT:    vmov.f64 d2, d0
+; CHECK-NEXT:    vmov.f32 s6, s3
+; CHECK-NEXT:    vmov.f32 s8, s1
+; CHECK-NEXT:    vmov.f64 d6, d1
+; CHECK-NEXT:    vmov r0, s6
+; CHECK-NEXT:    vldrh.u32 q1, [r3]
+; CHECK-NEXT:    vmov.f32 s10, s4
+; CHECK-NEXT:    vmov.f32 s14, s5
+; CHECK-NEXT:    vmov r2, s10
+; CHECK-NEXT:    add r0, r2
+; CHECK-NEXT:    vmov r2, s14
+; CHECK-NEXT:    add r0, r2
+; CHECK-NEXT:    strh r0, [r1, #2]
+; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    add r0, r2
+; CHECK-NEXT:    vmov r2, s12
+; CHECK-NEXT:    add r0, r2
+; CHECK-NEXT:    strh r0, [r1]
+; CHECK-NEXT:    add sp, #8
+; CHECK-NEXT:    bx lr
+entry:
+  %l1 = load <6 x i16>, <6 x i16>* %src, align 4
+  %s1 = shufflevector <6 x i16> %l1, <6 x i16> undef, <2 x i32> <i32 0, i32 3>
+  %s2 = shufflevector <6 x i16> %l1, <6 x i16> undef, <2 x i32> <i32 1, i32 4>
+  %s3 = shufflevector <6 x i16> %l1, <6 x i16> undef, <2 x i32> <i32 2, i32 5>
+  %a1 = add <2 x i16> %s1, %s2
+  %a = add <2 x i16> %a1, %s3
+  store <2 x i16> %a, <2 x i16> *%dst
+  ret void
+}
+
+define void @vld3_v4i16(<12 x i16> *%src, <4 x i16> *%dst) {
+; CHECK-LABEL: vld3_v4i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vldrh.u32 q3, [r0, #16]
+; CHECK-NEXT:    vmov.u16 r2, q0[0]
+; CHECK-NEXT:    vmov r0, s14
+; CHECK-NEXT:    vmov.32 q1[0], r2
+; CHECK-NEXT:    vmov.u16 r2, q0[3]
+; CHECK-NEXT:    vmov.32 q1[1], r2
+; CHECK-NEXT:    vmov.u16 r2, q0[6]
+; CHECK-NEXT:    vmov.32 q1[2], r2
+; CHECK-NEXT:    vmov.u16 r2, q0[1]
+; CHECK-NEXT:    vmov.32 q2[0], r2
+; CHECK-NEXT:    vmov.u16 r2, q0[4]
+; CHECK-NEXT:    vmov.32 q2[1], r2
+; CHECK-NEXT:    vmov.u16 r2, q0[7]
+; CHECK-NEXT:    vmov.32 q2[2], r2
+; CHECK-NEXT:    vmov.32 q2[3], r0
+; CHECK-NEXT:    vmov r0, s13
+; CHECK-NEXT:    vmov.32 q1[3], r0
+; CHECK-NEXT:    vmov.u16 r0, q0[2]
+; CHECK-NEXT:    vadd.i32 q1, q1, q2
+; CHECK-NEXT:    vmov.32 q2[0], r0
+; CHECK-NEXT:    vmov.u16 r0, q0[5]
+; CHECK-NEXT:    vmov.32 q2[1], r0
+; CHECK-NEXT:    vmov r0, s12
+; CHECK-NEXT:    vmov.32 q2[2], r0
+; CHECK-NEXT:    vmov r0, s15
+; CHECK-NEXT:    vmov.32 q2[3], r0
+; CHECK-NEXT:    vadd.i32 q0, q1, q2
+; CHECK-NEXT:    vstrh.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %l1 = load <12 x i16>, <12 x i16>* %src, align 4
+  %s1 = shufflevector <12 x i16> %l1, <12 x i16> undef, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
+  %s2 = shufflevector <12 x i16> %l1, <12 x i16> undef, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
+  %s3 = shufflevector <12 x i16> %l1, <12 x i16> undef, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
+  %a1 = add <4 x i16> %s1, %s2
+  %a = add <4 x i16> %a1, %s3
+  store <4 x i16> %a, <4 x i16> *%dst
+  ret void
+}
+
+define void @vld3_v8i16(<24 x i16> *%src, <8 x i16> *%dst) {
+; CHECK-LABEL: vld3_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vldrw.u32 q3, [r0, #32]
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #16]
+; CHECK-NEXT:    vmov.u16 r2, q1[2]
+; CHECK-NEXT:    vmov.u16 r0, q3[4]
+; CHECK-NEXT:    vmov.16 q4[0], r2
+; CHECK-NEXT:    vmov.u16 r2, q1[5]
+; CHECK-NEXT:    vmov.16 q4[1], r2
+; CHECK-NEXT:    vmov.u16 r2, q2[0]
+; CHECK-NEXT:    vmov.16 q5[6], r0
+; CHECK-NEXT:    vmov.u16 r0, q3[7]
+; CHECK-NEXT:    vmov.16 q5[7], r0
+; CHECK-NEXT:    vmov.16 q4[2], r2
+; CHECK-NEXT:    vmov.u16 r2, q2[3]
+; CHECK-NEXT:    vmov.f32 s22, s12
+; CHECK-NEXT:    vmov.16 q4[3], r2
+; CHECK-NEXT:    vmov q6, q5
+; CHECK-NEXT:    vmov.f32 s18, s11
+; CHECK-NEXT:    vmov r2, s16
+; CHECK-NEXT:    vmovnb.i32 q6, q4
+; CHECK-NEXT:    vmov.32 q0[0], r2
+; CHECK-NEXT:    vmov r2, s17
+; CHECK-NEXT:    vmov.32 q0[1], r2
+; CHECK-NEXT:    vmov r0, s26
+; CHECK-NEXT:    vmov.32 q0[2], r0
+; CHECK-NEXT:    vmov r0, s23
+; CHECK-NEXT:    vmov.32 q0[3], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[0]
+; CHECK-NEXT:    vmov.16 q4[0], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[3]
+; CHECK-NEXT:    vmov.16 q4[1], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[6]
+; CHECK-NEXT:    vmov.16 q4[2], r0
+; CHECK-NEXT:    vmov.u16 r0, q2[1]
+; CHECK-NEXT:    vmov.16 q4[3], r0
+; CHECK-NEXT:    vmov.u16 r0, q2[4]
+; CHECK-NEXT:    vmov.16 q4[4], r0
+; CHECK-NEXT:    vmov.u16 r0, q3[2]
+; CHECK-NEXT:    vmov.16 q5[6], r0
+; CHECK-NEXT:    vmov.u16 r0, q3[5]
+; CHECK-NEXT:    vmov.16 q5[7], r0
+; CHECK-NEXT:    vmov.u16 r0, q2[7]
+; CHECK-NEXT:    vmov.16 q4[5], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[1]
+; CHECK-NEXT:    vmov.f32 s19, s23
+; CHECK-NEXT:    vmov.16 q5[0], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[4]
+; CHECK-NEXT:    vmov.16 q5[1], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[7]
+; CHECK-NEXT:    vmov.16 q5[2], r0
+; CHECK-NEXT:    vmov.u16 r0, q2[2]
+; CHECK-NEXT:    vmov.16 q5[3], r0
+; CHECK-NEXT:    vmov.u16 r0, q2[5]
+; CHECK-NEXT:    vmov.16 q5[4], r0
+; CHECK-NEXT:    vmov.u16 r0, q3[0]
+; CHECK-NEXT:    vmov.16 q1[5], r0
+; CHECK-NEXT:    vmov.u16 r0, q3[3]
+; CHECK-NEXT:    vmov.16 q1[6], r0
+; CHECK-NEXT:    vmov.u16 r0, q3[6]
+; CHECK-NEXT:    vmov.16 q1[7], r0
+; CHECK-NEXT:    vmov r0, s20
+; CHECK-NEXT:    vmov q2, q1
+; CHECK-NEXT:    vmov.32 q3[0], r0
+; CHECK-NEXT:    vmov r0, s21
+; CHECK-NEXT:    vmovnb.i32 q2, q5
+; CHECK-NEXT:    vmov.32 q3[1], r0
+; CHECK-NEXT:    vmov r0, s10
+; CHECK-NEXT:    vmov.32 q3[2], r0
+; CHECK-NEXT:    vmov r0, s7
+; CHECK-NEXT:    vmov.32 q3[3], r0
+; CHECK-NEXT:    vadd.i16 q1, q4, q3
+; CHECK-NEXT:    vadd.i16 q0, q1, q0
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT:    bx lr
+entry:
+  %l1 = load <24 x i16>, <24 x i16>* %src, align 4
+  %s1 = shufflevector <24 x i16> %l1, <24 x i16> undef, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
+  %s2 = shufflevector <24 x i16> %l1, <24 x i16> undef, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22>
+  %s3 = shufflevector <24 x i16> %l1, <24 x i16> undef, <8 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23>
+  %a1 = add <8 x i16> %s1, %s2
+  %a = add <8 x i16> %a1, %s3
+  store <8 x i16> %a, <8 x i16> *%dst
+  ret void
+}
+
+define void @vld3_v16i16(<48 x i16> *%src, <16 x i16> *%dst) {
+; CHECK-LABEL: vld3_v16i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #48]
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #64]
+; CHECK-NEXT:    vldrw.u32 q3, [r0, #80]
+; CHECK-NEXT:    vmov.u16 r2, q1[2]
+; CHECK-NEXT:    vmov.16 q4[0], r2
+; CHECK-NEXT:    vmov.u16 r2, q1[5]
+; CHECK-NEXT:    vmov.16 q4[1], r2
+; CHECK-NEXT:    vmov.u16 r2, q2[0]
+; CHECK-NEXT:    vmov.16 q4[2], r2
+; CHECK-NEXT:    vmov.u16 r2, q2[3]
+; CHECK-NEXT:    vmov.16 q4[3], r2
+; CHECK-NEXT:    vmov.f32 s18, s11
+; CHECK-NEXT:    vmov r2, s16
+; CHECK-NEXT:    vmov.32 q0[0], r2
+; CHECK-NEXT:    vmov r2, s17
+; CHECK-NEXT:    vmov.32 q0[1], r2
+; CHECK-NEXT:    vmov.u16 r2, q3[4]
+; CHECK-NEXT:    vmov.16 q5[6], r2
+; CHECK-NEXT:    vmov.u16 r2, q3[7]
+; CHECK-NEXT:    vmov.16 q5[7], r2
+; CHECK-NEXT:    vmov.f32 s22, s12
+; CHECK-NEXT:    vmov q6, q5
+; CHECK-NEXT:    vmovnb.i32 q6, q4
+; CHECK-NEXT:    vmov r2, s26
+; CHECK-NEXT:    vmov.32 q0[2], r2
+; CHECK-NEXT:    vmov r2, s23
+; CHECK-NEXT:    vmov.32 q0[3], r2
+; CHECK-NEXT:    vmov.u16 r2, q1[0]
+; CHECK-NEXT:    vmov.16 q4[0], r2
+; CHECK-NEXT:    vmov.u16 r2, q1[3]
+; CHECK-NEXT:    vmov.16 q4[1], r2
+; CHECK-NEXT:    vmov.u16 r2, q1[6]
+; CHECK-NEXT:    vmov.16 q4[2], r2
+; CHECK-NEXT:    vmov.u16 r2, q2[1]
+; CHECK-NEXT:    vmov.16 q4[3], r2
+; CHECK-NEXT:    vmov.u16 r2, q2[4]
+; CHECK-NEXT:    vmov.16 q4[4], r2
+; CHECK-NEXT:    vmov.u16 r2, q3[2]
+; CHECK-NEXT:    vmov.16 q5[6], r2
+; CHECK-NEXT:    vmov.u16 r2, q3[5]
+; CHECK-NEXT:    vmov.16 q5[7], r2
+; CHECK-NEXT:    vmov.u16 r2, q2[7]
+; CHECK-NEXT:    vmov.16 q4[5], r2
+; CHECK-NEXT:    vmov.u16 r2, q1[1]
+; CHECK-NEXT:    vmov.f32 s19, s23
+; CHECK-NEXT:    vmov.16 q5[0], r2
+; CHECK-NEXT:    vmov.u16 r2, q1[4]
+; CHECK-NEXT:    vmov.16 q5[1], r2
+; CHECK-NEXT:    vmov.u16 r2, q1[7]
+; CHECK-NEXT:    vmov.16 q5[2], r2
+; CHECK-NEXT:    vmov.u16 r2, q2[2]
+; CHECK-NEXT:    vmov.16 q5[3], r2
+; CHECK-NEXT:    vmov.u16 r2, q2[5]
+; CHECK-NEXT:    vmov.16 q5[4], r2
+; CHECK-NEXT:    vmov.u16 r2, q3[0]
+; CHECK-NEXT:    vmov.16 q1[5], r2
+; CHECK-NEXT:    vmov.u16 r2, q3[3]
+; CHECK-NEXT:    vmov.16 q1[6], r2
+; CHECK-NEXT:    vmov.u16 r2, q3[6]
+; CHECK-NEXT:    vmov.16 q1[7], r2
+; CHECK-NEXT:    vmov r2, s20
+; CHECK-NEXT:    vmov q2, q1
+; CHECK-NEXT:    vmov.32 q3[0], r2
+; CHECK-NEXT:    vmov r2, s21
+; CHECK-NEXT:    vmovnb.i32 q2, q5
+; CHECK-NEXT:    vmov.32 q3[1], r2
+; CHECK-NEXT:    vmov r2, s10
+; CHECK-NEXT:    vmov.32 q3[2], r2
+; CHECK-NEXT:    vmov r2, s7
+; CHECK-NEXT:    vmov.32 q3[3], r2
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #16]
+; CHECK-NEXT:    vadd.i16 q1, q4, q3
+; CHECK-NEXT:    vldrw.u32 q3, [r0, #32]
+; CHECK-NEXT:    vadd.i16 q0, q1, q0
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vmov.u16 r0, q3[4]
+; CHECK-NEXT:    vstrw.32 q0, [r1, #16]
+; CHECK-NEXT:    vmov.u16 r2, q1[2]
+; CHECK-NEXT:    vmov.16 q6[6], r0
+; CHECK-NEXT:    vmov.16 q5[0], r2
+; CHECK-NEXT:    vmov.u16 r2, q1[5]
+; CHECK-NEXT:    vmov.16 q5[1], r2
+; CHECK-NEXT:    vmov.u16 r2, q2[0]
+; CHECK-NEXT:    vmov.u16 r0, q3[7]
+; CHECK-NEXT:    vmov.16 q5[2], r2
+; CHECK-NEXT:    vmov.16 q6[7], r0
+; CHECK-NEXT:    vmov.u16 r2, q2[3]
+; CHECK-NEXT:    vmov.16 q5[3], r2
+; CHECK-NEXT:    vmov.f32 s26, s12
+; CHECK-NEXT:    vmov.f32 s22, s11
+; CHECK-NEXT:    vmov q7, q6
+; CHECK-NEXT:    vmov r0, s20
+; CHECK-NEXT:    vmovnb.i32 q7, q5
+; CHECK-NEXT:    vmov.32 q4[0], r0
+; CHECK-NEXT:    vmov r0, s21
+; CHECK-NEXT:    vmov.32 q4[1], r0
+; CHECK-NEXT:    vmov r0, s30
+; CHECK-NEXT:    vmov.32 q4[2], r0
+; CHECK-NEXT:    vmov r0, s27
+; CHECK-NEXT:    vmov.32 q4[3], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[0]
+; CHECK-NEXT:    vmov.16 q5[0], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[3]
+; CHECK-NEXT:    vmov.16 q5[1], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[6]
+; CHECK-NEXT:    vmov.16 q5[2], r0
+; CHECK-NEXT:    vmov.u16 r0, q2[1]
+; CHECK-NEXT:    vmov.16 q5[3], r0
+; CHECK-NEXT:    vmov.u16 r0, q2[4]
+; CHECK-NEXT:    vmov.16 q5[4], r0
+; CHECK-NEXT:    vmov.u16 r0, q3[2]
+; CHECK-NEXT:    vmov.16 q6[6], r0
+; CHECK-NEXT:    vmov.u16 r0, q3[5]
+; CHECK-NEXT:    vmov.16 q6[7], r0
+; CHECK-NEXT:    vmov.u16 r0, q2[7]
+; CHECK-NEXT:    vmov.16 q5[5], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[1]
+; CHECK-NEXT:    vmov.f32 s23, s27
+; CHECK-NEXT:    vmov.16 q6[0], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[4]
+; CHECK-NEXT:    vmov.16 q6[1], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[7]
+; CHECK-NEXT:    vmov.16 q6[2], r0
+; CHECK-NEXT:    vmov.u16 r0, q2[2]
+; CHECK-NEXT:    vmov.16 q6[3], r0
+; CHECK-NEXT:    vmov.u16 r0, q2[5]
+; CHECK-NEXT:    vmov.16 q6[4], r0
+; CHECK-NEXT:    vmov r0, s24
+; CHECK-NEXT:    vmov.32 q1[0], r0
+; CHECK-NEXT:    vmov r0, s25
+; CHECK-NEXT:    vmov.32 q1[1], r0
+; CHECK-NEXT:    vmov.u16 r0, q3[0]
+; CHECK-NEXT:    vmov.16 q2[5], r0
+; CHECK-NEXT:    vmov.u16 r0, q3[3]
+; CHECK-NEXT:    vmov.16 q2[6], r0
+; CHECK-NEXT:    vmov.u16 r0, q3[6]
+; CHECK-NEXT:    vmov.16 q2[7], r0
+; CHECK-NEXT:    vmov q3, q2
+; CHECK-NEXT:    vmovnb.i32 q3, q6
+; CHECK-NEXT:    vmov r0, s14
+; CHECK-NEXT:    vmov.32 q1[2], r0
+; CHECK-NEXT:    vmov r0, s11
+; CHECK-NEXT:    vmov.32 q1[3], r0
+; CHECK-NEXT:    vadd.i16 q1, q5, q1
+; CHECK-NEXT:    vadd.i16 q1, q1, q4
+; CHECK-NEXT:    vstrw.32 q1, [r1]
+; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    bx lr
+entry:
+  %l1 = load <48 x i16>, <48 x i16>* %src, align 4
+  %s1 = shufflevector <48 x i16> %l1, <48 x i16> undef, <16 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21, i32 24, i32 27, i32 30, i32 33, i32 36, i32 39, i32 42, i32 45>
+  %s2 = shufflevector <48 x i16> %l1, <48 x i16> undef, <16 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22, i32 25, i32 28, i32 31, i32 34, i32 37, i32 40, i32 43, i32 46>
+  %s3 = shufflevector <48 x i16> %l1, <48 x i16> undef, <16 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23, i32 26, i32 29, i32 32, i32 35, i32 38, i32 41, i32 44, i32 47>
+  %a1 = add <16 x i16> %s1, %s2
+  %a = add <16 x i16> %a1, %s3
+  store <16 x i16> %a, <16 x i16> *%dst
+  ret void
+}
+
+; i8
+
+define void @vld3_v2i8(<6 x i8> *%src, <2 x i8> *%dst) {
+; CHECK-LABEL: vld3_v2i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .pad #8
+; CHECK-NEXT:    sub sp, #8
+; CHECK-NEXT:    ldrd r2, r0, [r0]
+; CHECK-NEXT:    strd r2, r0, [sp]
+; CHECK-NEXT:    mov r0, sp
+; CHECK-NEXT:    vldrb.u16 q0, [r0]
+; CHECK-NEXT:    vmov.u16 r0, q0[4]
+; CHECK-NEXT:    vmov.u16 r2, q0[3]
+; CHECK-NEXT:    add r0, r2
+; CHECK-NEXT:    vmov.u16 r2, q0[5]
+; CHECK-NEXT:    add r0, r2
+; CHECK-NEXT:    strb r0, [r1, #1]
+; CHECK-NEXT:    vmov.u16 r0, q0[1]
+; CHECK-NEXT:    vmov.u16 r2, q0[0]
+; CHECK-NEXT:    add r0, r2
+; CHECK-NEXT:    vmov.u16 r2, q0[2]
+; CHECK-NEXT:    add r0, r2
+; CHECK-NEXT:    strb r0, [r1]
+; CHECK-NEXT:    add sp, #8
+; CHECK-NEXT:    bx lr
+entry:
+  %l1 = load <6 x i8>, <6 x i8>* %src, align 4
+  %s1 = shufflevector <6 x i8> %l1, <6 x i8> undef, <2 x i32> <i32 0, i32 3>
+  %s2 = shufflevector <6 x i8> %l1, <6 x i8> undef, <2 x i32> <i32 1, i32 4>
+  %s3 = shufflevector <6 x i8> %l1, <6 x i8> undef, <2 x i32> <i32 2, i32 5>
+  %a1 = add <2 x i8> %s1, %s2
+  %a = add <2 x i8> %a1, %s3
+  store <2 x i8> %a, <2 x i8> *%dst
+  ret void
+}
+
+define void @vld3_v4i8(<12 x i8> *%src, <4 x i8> *%dst) {
+; CHECK-LABEL: vld3_v4i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .pad #8
+; CHECK-NEXT:    sub sp, #8
+; CHECK-NEXT:    vldrb.u16 q2, [r0]
+; CHECK-NEXT:    ldr r3, [r0, #8]
+; CHECK-NEXT:    mov r2, sp
+; CHECK-NEXT:    str r3, [sp]
+; CHECK-NEXT:    vmov.u16 r0, q2[2]
+; CHECK-NEXT:    vmov.32 q0[0], r0
+; CHECK-NEXT:    vmov.u16 r0, q2[5]
+; CHECK-NEXT:    vmov.32 q0[1], r0
+; CHECK-NEXT:    vmov.u16 r0, q2[0]
+; CHECK-NEXT:    vmov.32 q1[0], r0
+; CHECK-NEXT:    vmov.u16 r0, q2[3]
+; CHECK-NEXT:    vmov.32 q1[1], r0
+; CHECK-NEXT:    vmov.u16 r0, q2[6]
+; CHECK-NEXT:    vmov.32 q1[2], r0
+; CHECK-NEXT:    vmov.u16 r0, q2[1]
+; CHECK-NEXT:    vmov.32 q3[0], r0
+; CHECK-NEXT:    vmov.u16 r0, q2[4]
+; CHECK-NEXT:    vmov.32 q3[1], r0
+; CHECK-NEXT:    vmov.u16 r0, q2[7]
+; CHECK-NEXT:    vldrb.u16 q2, [r2]
+; CHECK-NEXT:    vmov.32 q3[2], r0
+; CHECK-NEXT:    vmov.u16 r0, q2[2]
+; CHECK-NEXT:    vmov.32 q3[3], r0
+; CHECK-NEXT:    vmov.u16 r0, q2[1]
+; CHECK-NEXT:    vmov.32 q1[3], r0
+; CHECK-NEXT:    vmov.u16 r0, q2[0]
+; CHECK-NEXT:    vmov.32 q0[2], r0
+; CHECK-NEXT:    vmov.u16 r0, q2[3]
+; CHECK-NEXT:    vadd.i32 q1, q1, q3
+; CHECK-NEXT:    vmov.32 q0[3], r0
+; CHECK-NEXT:    vadd.i32 q0, q1, q0
+; CHECK-NEXT:    vstrb.32 q0, [r1]
+; CHECK-NEXT:    add sp, #8
+; CHECK-NEXT:    bx lr
+entry:
+  %l1 = load <12 x i8>, <12 x i8>* %src, align 4
+  %s1 = shufflevector <12 x i8> %l1, <12 x i8> undef, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
+  %s2 = shufflevector <12 x i8> %l1, <12 x i8> undef, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
+  %s3 = shufflevector <12 x i8> %l1, <12 x i8> undef, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
+  %a1 = add <4 x i8> %s1, %s2
+  %a = add <4 x i8> %a1, %s3
+  store <4 x i8> %a, <4 x i8> *%dst
+  ret void
+}
+
+define void @vld3_v8i8(<24 x i8> *%src, <8 x i8> *%dst) {
+; CHECK-LABEL: vld3_v8i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vldrb.u16 q1, [r0, #16]
+; CHECK-NEXT:    vmov.u8 r2, q0[0]
+; CHECK-NEXT:    vmov.u16 r0, q1[2]
+; CHECK-NEXT:    vmov.16 q2[0], r2
+; CHECK-NEXT:    vmov.u8 r2, q0[3]
+; CHECK-NEXT:    vmov.16 q2[1], r2
+; CHECK-NEXT:    vmov.u8 r2, q0[6]
+; CHECK-NEXT:    vmov.16 q2[2], r2
+; CHECK-NEXT:    vmov.u8 r2, q0[9]
+; CHECK-NEXT:    vmov.16 q2[3], r2
+; CHECK-NEXT:    vmov.u8 r2, q0[12]
+; CHECK-NEXT:    vmov.16 q2[4], r2
+; CHECK-NEXT:    vmov.u8 r2, q0[15]
+; CHECK-NEXT:    vmov.16 q2[5], r2
+; CHECK-NEXT:    vmov.16 q2[6], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[1]
+; CHECK-NEXT:    vmov.16 q3[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[4]
+; CHECK-NEXT:    vmov.16 q3[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[7]
+; CHECK-NEXT:    vmov.16 q3[2], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[10]
+; CHECK-NEXT:    vmov.16 q3[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[13]
+; CHECK-NEXT:    vmov.16 q3[4], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[0]
+; CHECK-NEXT:    vmov.16 q3[5], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[3]
+; CHECK-NEXT:    vmov.16 q3[6], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[6]
+; CHECK-NEXT:    vmov.16 q3[7], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[5]
+; CHECK-NEXT:    vmov.16 q2[7], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[2]
+; CHECK-NEXT:    vadd.i16 q2, q2, q3
+; CHECK-NEXT:    vmov.16 q3[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[5]
+; CHECK-NEXT:    vmov.16 q3[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[8]
+; CHECK-NEXT:    vmov.16 q3[2], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[11]
+; CHECK-NEXT:    vmov.16 q3[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[14]
+; CHECK-NEXT:    vmov.16 q3[4], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[1]
+; CHECK-NEXT:    vmov.16 q3[5], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[4]
+; CHECK-NEXT:    vmov.16 q3[6], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[7]
+; CHECK-NEXT:    vmov.16 q3[7], r0
+; CHECK-NEXT:    vadd.i16 q0, q2, q3
+; CHECK-NEXT:    vstrb.16 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %l1 = load <24 x i8>, <24 x i8>* %src, align 4
+  %s1 = shufflevector <24 x i8> %l1, <24 x i8> undef, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
+  %s2 = shufflevector <24 x i8> %l1, <24 x i8> undef, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22>
+  %s3 = shufflevector <24 x i8> %l1, <24 x i8> undef, <8 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23>
+  %a1 = add <8 x i8> %s1, %s2
+  %a = add <8 x i8> %a1, %s3
+  store <8 x i8> %a, <8 x i8> *%dst
+  ret void
+}
+
+define void @vld3_v16i8(<48 x i8> *%src, <16 x i8> *%dst) {
+; CHECK-LABEL: vld3_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11}
+; CHECK-NEXT:    vldrw.u32 q2, [r0]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #16]
+; CHECK-NEXT:    vmov.u8 r2, q2[0]
+; CHECK-NEXT:    vmov.8 q1[0], r2
+; CHECK-NEXT:    vmov.u8 r2, q2[3]
+; CHECK-NEXT:    vmov.8 q1[1], r2
+; CHECK-NEXT:    vmov.u8 r2, q2[6]
+; CHECK-NEXT:    vmov.8 q1[2], r2
+; CHECK-NEXT:    vmov.u8 r2, q2[9]
+; CHECK-NEXT:    vmov.8 q1[3], r2
+; CHECK-NEXT:    vmov.u8 r2, q2[12]
+; CHECK-NEXT:    vmov.8 q1[4], r2
+; CHECK-NEXT:    vmov.u8 r2, q2[15]
+; CHECK-NEXT:    vmov.8 q1[5], r2
+; CHECK-NEXT:    vmov.u8 r2, q0[2]
+; CHECK-NEXT:    vmov.8 q1[6], r2
+; CHECK-NEXT:    vmov.u8 r2, q0[5]
+; CHECK-NEXT:    vmov.8 q1[7], r2
+; CHECK-NEXT:    vmov r2, s4
+; CHECK-NEXT:    vmov.32 q3[0], r2
+; CHECK-NEXT:    vmov r2, s5
+; CHECK-NEXT:    vmov.32 q3[1], r2
+; CHECK-NEXT:    vmov.u8 r2, q0[8]
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #32]
+; CHECK-NEXT:    vmov.8 q4[8], r2
+; CHECK-NEXT:    vmov.u8 r2, q0[11]
+; CHECK-NEXT:    vmov.8 q4[9], r2
+; CHECK-NEXT:    vmov.u8 r2, q0[14]
+; CHECK-NEXT:    vmov.8 q4[10], r2
+; CHECK-NEXT:    vmov.u8 r0, q1[1]
+; CHECK-NEXT:    vmov.8 q4[11], r0
+; CHECK-NEXT:    vmov r0, s18
+; CHECK-NEXT:    vmov.32 q3[2], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[1]
+; CHECK-NEXT:    vmov.8 q5[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[4]
+; CHECK-NEXT:    vmov.8 q5[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[7]
+; CHECK-NEXT:    vmov.8 q5[2], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[10]
+; CHECK-NEXT:    vmov.8 q5[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[13]
+; CHECK-NEXT:    vmov.8 q5[4], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[0]
+; CHECK-NEXT:    vmov.8 q5[5], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[3]
+; CHECK-NEXT:    vmov.8 q5[6], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[6]
+; CHECK-NEXT:    vmov.8 q5[7], r0
+; CHECK-NEXT:    vmov r0, s20
+; CHECK-NEXT:    vmov.32 q4[0], r0
+; CHECK-NEXT:    vmov r0, s21
+; CHECK-NEXT:    vmov.32 q4[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[9]
+; CHECK-NEXT:    vmov.8 q5[8], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[12]
+; CHECK-NEXT:    vmov.8 q5[9], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[15]
+; CHECK-NEXT:    vmov.8 q5[10], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[2]
+; CHECK-NEXT:    vmov.8 q5[11], r0
+; CHECK-NEXT:    vmov r0, s22
+; CHECK-NEXT:    vmov.32 q4[2], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[5]
+; CHECK-NEXT:    vmov.8 q5[12], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[8]
+; CHECK-NEXT:    vmov.8 q5[13], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[11]
+; CHECK-NEXT:    vmov.8 q5[14], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[14]
+; CHECK-NEXT:    vmov.8 q5[15], r0
+; CHECK-NEXT:    vmov r0, s23
+; CHECK-NEXT:    vmov.32 q4[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[4]
+; CHECK-NEXT:    vmov.8 q5[12], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[7]
+; CHECK-NEXT:    vmov.8 q5[13], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[10]
+; CHECK-NEXT:    vmov.8 q5[14], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[13]
+; CHECK-NEXT:    vmov.8 q5[15], r0
+; CHECK-NEXT:    vmov r0, s23
+; CHECK-NEXT:    vmov.32 q3[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[2]
+; CHECK-NEXT:    vadd.i8 q3, q3, q4
+; CHECK-NEXT:    vmov.8 q4[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[5]
+; CHECK-NEXT:    vmov.8 q4[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[8]
+; CHECK-NEXT:    vmov.8 q4[2], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[11]
+; CHECK-NEXT:    vmov.8 q4[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[14]
+; CHECK-NEXT:    vmov.8 q4[4], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[1]
+; CHECK-NEXT:    vmov.8 q4[5], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[4]
+; CHECK-NEXT:    vmov.8 q4[6], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[7]
+; CHECK-NEXT:    vmov.8 q4[7], r0
+; CHECK-NEXT:    vmov r0, s16
+; CHECK-NEXT:    vmov.32 q2[0], r0
+; CHECK-NEXT:    vmov r0, s17
+; CHECK-NEXT:    vmov.32 q2[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[10]
+; CHECK-NEXT:    vmov.8 q4[8], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[13]
+; CHECK-NEXT:    vmov.8 q4[9], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[0]
+; CHECK-NEXT:    vmov.8 q4[10], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[3]
+; CHECK-NEXT:    vmov.8 q4[11], r0
+; CHECK-NEXT:    vmov r0, s18
+; CHECK-NEXT:    vmov.32 q2[2], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[6]
+; CHECK-NEXT:    vmov.8 q0[12], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[9]
+; CHECK-NEXT:    vmov.8 q0[13], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[12]
+; CHECK-NEXT:    vmov.8 q0[14], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[15]
+; CHECK-NEXT:    vmov.8 q0[15], r0
+; CHECK-NEXT:    vmov r0, s3
+; CHECK-NEXT:    vmov.32 q2[3], r0
+; CHECK-NEXT:    vadd.i8 q0, q3, q2
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vpop {d8, d9, d10, d11}
+; CHECK-NEXT:    bx lr
+entry:
+  %l1 = load <48 x i8>, <48 x i8>* %src, align 4
+  %s1 = shufflevector <48 x i8> %l1, <48 x i8> undef, <16 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21, i32 24, i32 27, i32 30, i32 33, i32 36, i32 39, i32 42, i32 45>
+  %s2 = shufflevector <48 x i8> %l1, <48 x i8> undef, <16 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22, i32 25, i32 28, i32 31, i32 34, i32 37, i32 40, i32 43, i32 46>
+  %s3 = shufflevector <48 x i8> %l1, <48 x i8> undef, <16 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23, i32 26, i32 29, i32 32, i32 35, i32 38, i32 41, i32 44, i32 47>
+  %a1 = add <16 x i8> %s1, %s2
+  %a = add <16 x i8> %a1, %s3
+  store <16 x i8> %a, <16 x i8> *%dst
+  ret void
+}
+
+; i64
+
+define void @vld3_v2i64(<6 x i64> *%src, <2 x i64> *%dst) {
+; CHECK-LABEL: vld3_v2i64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, lr}
+; CHECK-NEXT:    push {r4, lr}
+; CHECK-NEXT:    .vsave {d8, d9}
+; CHECK-NEXT:    vpush {d8, d9}
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #16]
+; CHECK-NEXT:    vldrw.u32 q4, [r0, #32]
+; CHECK-NEXT:    vmov.f64 d6, d3
+; CHECK-NEXT:    vmov.f32 s13, s7
+; CHECK-NEXT:    vmov.f32 s14, s16
+; CHECK-NEXT:    vmov.f32 s6, s10
+; CHECK-NEXT:    vmov.f32 s7, s11
+; CHECK-NEXT:    vmov.f32 s15, s17
+; CHECK-NEXT:    vmov r3, s14
+; CHECK-NEXT:    vmov r0, s6
+; CHECK-NEXT:    vmov.f64 d0, d4
+; CHECK-NEXT:    vmov.f32 s1, s9
+; CHECK-NEXT:    vmov.f32 s2, s18
+; CHECK-NEXT:    vmov.f32 s3, s19
+; CHECK-NEXT:    vmov r12, s15
+; CHECK-NEXT:    vmov r2, s7
+; CHECK-NEXT:    vmov r4, s4
+; CHECK-NEXT:    adds.w lr, r0, r3
+; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    vmov r3, s3
+; CHECK-NEXT:    adc.w r2, r2, r12
+; CHECK-NEXT:    adds.w lr, lr, r0
+; CHECK-NEXT:    vmov r0, s12
+; CHECK-NEXT:    adc.w r12, r2, r3
+; CHECK-NEXT:    vmov r3, s13
+; CHECK-NEXT:    vmov r2, s5
+; CHECK-NEXT:    adds r0, r0, r4
+; CHECK-NEXT:    vmov r4, s0
+; CHECK-NEXT:    adcs r2, r3
+; CHECK-NEXT:    vmov r3, s1
+; CHECK-NEXT:    adds r0, r0, r4
+; CHECK-NEXT:    vmov.32 q0[0], r0
+; CHECK-NEXT:    adcs r2, r3
+; CHECK-NEXT:    vmov.32 q0[1], r2
+; CHECK-NEXT:    vmov.32 q0[2], lr
+; CHECK-NEXT:    vmov.32 q0[3], r12
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vpop {d8, d9}
+; CHECK-NEXT:    pop {r4, pc}
+entry:
+  %l1 = load <6 x i64>, <6 x i64>* %src, align 4
+  %s1 = shufflevector <6 x i64> %l1, <6 x i64> undef, <2 x i32> <i32 0, i32 3>
+  %s2 = shufflevector <6 x i64> %l1, <6 x i64> undef, <2 x i32> <i32 1, i32 4>
+  %s3 = shufflevector <6 x i64> %l1, <6 x i64> undef, <2 x i32> <i32 2, i32 5>
+  %a1 = add <2 x i64> %s1, %s2
+  %a = add <2 x i64> %a1, %s3
+  store <2 x i64> %a, <2 x i64> *%dst
+  ret void
+}
+
+define void @vld3_v4i64(<12 x i64> *%src, <4 x i64> *%dst) {
+; CHECK-LABEL: vld3_v4i64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, lr}
+; CHECK-NEXT:    push {r4, lr}
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    .pad #24
+; CHECK-NEXT:    sub sp, #24
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #16]
+; CHECK-NEXT:    vldrw.u32 q5, [r0, #48]
+; CHECK-NEXT:    vldrw.u32 q6, [r0, #64]
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vmov.f64 d4, d0
+; CHECK-NEXT:    vstrw.32 q0, [sp] @ 16-byte Spill
+; CHECK-NEXT:    vldrw.u32 q4, [r0, #32]
+; CHECK-NEXT:    vmov.f32 s9, s1
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #80]
+; CHECK-NEXT:    vmov.f64 d14, d11
+; CHECK-NEXT:    vmov.f32 s29, s23
+; CHECK-NEXT:    vmov.f32 s30, s0
+; CHECK-NEXT:    vmov.f32 s22, s26
+; CHECK-NEXT:    vmov.f32 s23, s27
+; CHECK-NEXT:    vmov.f32 s31, s1
+; CHECK-NEXT:    vmov r3, s30
+; CHECK-NEXT:    vmov r0, s22
+; CHECK-NEXT:    vmov.f64 d6, d3
+; CHECK-NEXT:    vmov.f32 s13, s7
+; CHECK-NEXT:    vmov.f32 s10, s18
+; CHECK-NEXT:    vmov.f32 s14, s16
+; CHECK-NEXT:    vmov.f32 s11, s19
+; CHECK-NEXT:    vmov.f32 s15, s17
+; CHECK-NEXT:    vmov.f64 d8, d12
+; CHECK-NEXT:    vmov.f32 s17, s25
+; CHECK-NEXT:    vmov.f32 s18, s2
+; CHECK-NEXT:    vmov.f32 s19, s3
+; CHECK-NEXT:    vmov r12, s31
+; CHECK-NEXT:    vmov r2, s23
+; CHECK-NEXT:    adds.w lr, r0, r3
+; CHECK-NEXT:    vmov r0, s18
+; CHECK-NEXT:    vmov r4, s20
+; CHECK-NEXT:    vmov r3, s19
+; CHECK-NEXT:    adc.w r2, r2, r12
+; CHECK-NEXT:    adds.w lr, lr, r0
+; CHECK-NEXT:    vmov r0, s28
+; CHECK-NEXT:    adc.w r12, r2, r3
+; CHECK-NEXT:    vmov r3, s29
+; CHECK-NEXT:    vmov r2, s21
+; CHECK-NEXT:    adds r0, r0, r4
+; CHECK-NEXT:    vmov r4, s16
+; CHECK-NEXT:    adcs r2, r3
+; CHECK-NEXT:    vmov r3, s17
+; CHECK-NEXT:    adds r0, r0, r4
+; CHECK-NEXT:    vmov.32 q0[0], r0
+; CHECK-NEXT:    vmov r0, s15
+; CHECK-NEXT:    adcs r2, r3
+; CHECK-NEXT:    vmov r3, s14
+; CHECK-NEXT:    vmov.32 q0[1], r2
+; CHECK-NEXT:    vmov.32 q0[2], lr
+; CHECK-NEXT:    vmov.32 q0[3], r12
+; CHECK-NEXT:    vstrw.32 q0, [r1, #16]
+; CHECK-NEXT:    vldrw.u32 q0, [sp] @ 16-byte Reload
+; CHECK-NEXT:    vmov.f32 s6, s2
+; CHECK-NEXT:    vmov.f32 s7, s3
+; CHECK-NEXT:    vmov r4, s6
+; CHECK-NEXT:    vmov r2, s7
+; CHECK-NEXT:    adds r3, r3, r4
+; CHECK-NEXT:    vmov r4, s10
+; CHECK-NEXT:    adcs r0, r2
+; CHECK-NEXT:    vmov r2, s11
+; CHECK-NEXT:    adds.w lr, r3, r4
+; CHECK-NEXT:    vmov r3, s4
+; CHECK-NEXT:    vmov r4, s5
+; CHECK-NEXT:    adc.w r12, r0, r2
+; CHECK-NEXT:    vmov r0, s12
+; CHECK-NEXT:    vmov r2, s13
+; CHECK-NEXT:    adds r0, r0, r3
+; CHECK-NEXT:    vmov r3, s9
+; CHECK-NEXT:    adcs r2, r4
+; CHECK-NEXT:    vmov r4, s8
+; CHECK-NEXT:    adds r0, r0, r4
+; CHECK-NEXT:    adcs r2, r3
+; CHECK-NEXT:    vmov.32 q0[0], r0
+; CHECK-NEXT:    vmov.32 q0[1], r2
+; CHECK-NEXT:    vmov.32 q0[2], lr
+; CHECK-NEXT:    vmov.32 q0[3], r12
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    add sp, #24
+; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    pop {r4, pc}
+entry:
+  %l1 = load <12 x i64>, <12 x i64>* %src, align 4
+  %s1 = shufflevector <12 x i64> %l1, <12 x i64> undef, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
+  %s2 = shufflevector <12 x i64> %l1, <12 x i64> undef, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
+  %s3 = shufflevector <12 x i64> %l1, <12 x i64> undef, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
+  %a1 = add <4 x i64> %s1, %s2
+  %a = add <4 x i64> %a1, %s3
+  store <4 x i64> %a, <4 x i64> *%dst
+  ret void
+}
+
+; f32
+
+define void @vld3_v2f32(<6 x float> *%src, <2 x float> *%dst) {
+; CHECK-LABEL: vld3_v2f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q2, [r0]
+; CHECK-NEXT:    vldr s1, [r0, #16]
+; CHECK-NEXT:    vldr s5, [r0, #20]
+; CHECK-NEXT:    vmov.f64 d6, d4
+; CHECK-NEXT:    vmov.f32 s13, s11
+; CHECK-NEXT:    vmov.f32 s0, s9
+; CHECK-NEXT:    vadd.f32 q0, q3, q0
+; CHECK-NEXT:    vmov.f32 s4, s10
+; CHECK-NEXT:    vadd.f32 q0, q0, q1
+; CHECK-NEXT:    vstmia r1, {s0, s1}
+; CHECK-NEXT:    bx lr
+entry:
+  %l1 = load <6 x float>, <6 x float>* %src, align 4
+  %s1 = shufflevector <6 x float> %l1, <6 x float> undef, <2 x i32> <i32 0, i32 3>
+  %s2 = shufflevector <6 x float> %l1, <6 x float> undef, <2 x i32> <i32 1, i32 4>
+  %s3 = shufflevector <6 x float> %l1, <6 x float> undef, <2 x i32> <i32 2, i32 5>
+  %a1 = fadd <2 x float> %s1, %s2
+  %a = fadd <2 x float> %a1, %s3
+  store <2 x float> %a, <2 x float> *%dst
+  ret void
+}
+
+define void @vld3_v4f32(<12 x float> *%src, <4 x float> *%dst) {
+; CHECK-LABEL: vld3_v4f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11}
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #16]
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #32]
+; CHECK-NEXT:    vmov.f32 s12, s5
+; CHECK-NEXT:    vmov.f32 s13, s0
+; CHECK-NEXT:    vmov.32 r0, q2[2]
+; CHECK-NEXT:    vdup.32 q4, r0
+; CHECK-NEXT:    vmov.f32 s14, s3
+; CHECK-NEXT:    vmov.f32 s15, s19
+; CHECK-NEXT:    vmov.32 r0, q2[1]
+; CHECK-NEXT:    vmov.f64 d8, d2
+; CHECK-NEXT:    vdup.32 q5, r0
+; CHECK-NEXT:    vmov.f32 s17, s7
+; CHECK-NEXT:    vmov.f32 s18, s2
+; CHECK-NEXT:    vmov.f32 s0, s6
+; CHECK-NEXT:    vmov.f32 s19, s23
+; CHECK-NEXT:    vmov.f32 s10, s8
+; CHECK-NEXT:    vadd.f32 q3, q4, q3
+; CHECK-NEXT:    vmov.f32 s2, s8
+; CHECK-NEXT:    vmov.f32 s3, s11
+; CHECK-NEXT:    vadd.f32 q0, q3, q0
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vpop {d8, d9, d10, d11}
+; CHECK-NEXT:    bx lr
+entry:
+  %l1 = load <12 x float>, <12 x float>* %src, align 4
+  %s1 = shufflevector <12 x float> %l1, <12 x float> undef, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
+  %s2 = shufflevector <12 x float> %l1, <12 x float> undef, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
+  %s3 = shufflevector <12 x float> %l1, <12 x float> undef, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
+  %a1 = fadd <4 x float> %s1, %s2
+  %a = fadd <4 x float> %a1, %s3
+  store <4 x float> %a, <4 x float> *%dst
+  ret void
+}
+
+define void @vld3_v8f32(<24 x float> *%src, <8 x float> *%dst) {
+; CHECK-LABEL: vld3_v8f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #48]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #64]
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #80]
+; CHECK-NEXT:    vmov.f32 s12, s5
+; CHECK-NEXT:    vmov.f32 s13, s0
+; CHECK-NEXT:    vmov.32 r2, q2[2]
+; CHECK-NEXT:    vdup.32 q4, r2
+; CHECK-NEXT:    vmov.f32 s14, s3
+; CHECK-NEXT:    vmov.f32 s15, s19
+; CHECK-NEXT:    vmov.32 r2, q2[1]
+; CHECK-NEXT:    vmov.f64 d8, d2
+; CHECK-NEXT:    vdup.32 q5, r2
+; CHECK-NEXT:    vmov.f32 s17, s7
+; CHECK-NEXT:    vmov.f32 s18, s2
+; CHECK-NEXT:    vmov.f32 s0, s6
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #16]
+; CHECK-NEXT:    vmov.f32 s19, s23
+; CHECK-NEXT:    vmov.f32 s10, s8
+; CHECK-NEXT:    vadd.f32 q3, q4, q3
+; CHECK-NEXT:    vmov.f32 s2, s8
+; CHECK-NEXT:    vmov.f32 s3, s11
+; CHECK-NEXT:    vldrw.u32 q2, [r0]
+; CHECK-NEXT:    vadd.f32 q0, q3, q0
+; CHECK-NEXT:    vldrw.u32 q3, [r0, #32]
+; CHECK-NEXT:    vmov.f32 s16, s9
+; CHECK-NEXT:    vmov.f32 s17, s4
+; CHECK-NEXT:    vmov.32 r0, q3[2]
+; CHECK-NEXT:    vdup.32 q5, r0
+; CHECK-NEXT:    vmov.f32 s18, s7
+; CHECK-NEXT:    vmov.f32 s19, s23
+; CHECK-NEXT:    vmov.32 r0, q3[1]
+; CHECK-NEXT:    vmov.f64 d10, d4
+; CHECK-NEXT:    vdup.32 q6, r0
+; CHECK-NEXT:    vstrw.32 q0, [r1, #16]
+; CHECK-NEXT:    vmov.f32 s21, s11
+; CHECK-NEXT:    vmov.f32 s22, s6
+; CHECK-NEXT:    vmov.f32 s4, s10
+; CHECK-NEXT:    vmov.f32 s23, s27
+; CHECK-NEXT:    vmov.f32 s14, s12
+; CHECK-NEXT:    vadd.f32 q4, q5, q4
+; CHECK-NEXT:    vmov.f32 s6, s12
+; CHECK-NEXT:    vmov.f32 s7, s15
+; CHECK-NEXT:    vadd.f32 q1, q4, q1
+; CHECK-NEXT:    vstrw.32 q1, [r1]
+; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT:    bx lr
+entry:
+  %l1 = load <24 x float>, <24 x float>* %src, align 4
+  %s1 = shufflevector <24 x float> %l1, <24 x float> undef, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
+  %s2 = shufflevector <24 x float> %l1, <24 x float> undef, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22>
+  %s3 = shufflevector <24 x float> %l1, <24 x float> undef, <8 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23>
+  %a1 = fadd <8 x float> %s1, %s2
+  %a = fadd <8 x float> %a1, %s3
+  store <8 x float> %a, <8 x float> *%dst
+  ret void
+}
+
+define void @vld3_v16f32(<48 x float> *%src, <16 x float> *%dst) {
+; CHECK-LABEL: vld3_v16f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    .pad #16
+; CHECK-NEXT:    sub sp, #16
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #48]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #64]
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #80]
+; CHECK-NEXT:    vmov.f32 s12, s5
+; CHECK-NEXT:    vmov.f32 s13, s0
+; CHECK-NEXT:    vmov.32 r2, q2[2]
+; CHECK-NEXT:    vdup.32 q4, r2
+; CHECK-NEXT:    vmov.f32 s14, s3
+; CHECK-NEXT:    vmov.f32 s15, s19
+; CHECK-NEXT:    vmov.32 r2, q2[1]
+; CHECK-NEXT:    vmov.f64 d8, d2
+; CHECK-NEXT:    vdup.32 q5, r2
+; CHECK-NEXT:    vmov.f32 s17, s7
+; CHECK-NEXT:    vmov.f32 s18, s2
+; CHECK-NEXT:    vmov.f32 s0, s6
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #16]
+; CHECK-NEXT:    vmov.f32 s19, s23
+; CHECK-NEXT:    vmov.f32 s10, s8
+; CHECK-NEXT:    vadd.f32 q3, q4, q3
+; CHECK-NEXT:    vmov.f32 s2, s8
+; CHECK-NEXT:    vmov.f32 s3, s11
+; CHECK-NEXT:    vldrw.u32 q2, [r0]
+; CHECK-NEXT:    vadd.f32 q0, q3, q0
+; CHECK-NEXT:    vldrw.u32 q3, [r0, #32]
+; CHECK-NEXT:    vmov.f32 s16, s9
+; CHECK-NEXT:    vstrw.32 q0, [sp] @ 16-byte Spill
+; CHECK-NEXT:    vmov.f32 s17, s4
+; CHECK-NEXT:    vmov.32 r2, q3[2]
+; CHECK-NEXT:    vdup.32 q5, r2
+; CHECK-NEXT:    vmov.f32 s18, s7
+; CHECK-NEXT:    vmov.f32 s19, s23
+; CHECK-NEXT:    vmov.32 r2, q3[1]
+; CHECK-NEXT:    vmov.f64 d10, d4
+; CHECK-NEXT:    vdup.32 q6, r2
+; CHECK-NEXT:    vmov.f32 s21, s11
+; CHECK-NEXT:    vmov.f32 s22, s6
+; CHECK-NEXT:    vmov.f32 s4, s10
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #160]
+; CHECK-NEXT:    vmov.f32 s23, s27
+; CHECK-NEXT:    vmov.f32 s14, s12
+; CHECK-NEXT:    vadd.f32 q4, q5, q4
+; CHECK-NEXT:    vmov.f32 s6, s12
+; CHECK-NEXT:    vmov.f32 s7, s15
+; CHECK-NEXT:    vldrw.u32 q3, [r0, #144]
+; CHECK-NEXT:    vadd.f32 q1, q4, q1
+; CHECK-NEXT:    vldrw.u32 q4, [r0, #176]
+; CHECK-NEXT:    vmov.f32 s20, s13
+; CHECK-NEXT:    vmov.f32 s21, s8
+; CHECK-NEXT:    vmov.32 r2, q4[2]
+; CHECK-NEXT:    vdup.32 q6, r2
+; CHECK-NEXT:    vmov.f32 s22, s11
+; CHECK-NEXT:    vmov.f32 s23, s27
+; CHECK-NEXT:    vmov.32 r2, q4[1]
+; CHECK-NEXT:    vmov.f64 d12, d6
+; CHECK-NEXT:    vdup.32 q7, r2
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #128]
+; CHECK-NEXT:    vmov.f32 s25, s15
+; CHECK-NEXT:    vmov.f32 s26, s10
+; CHECK-NEXT:    vmov.f32 s8, s14
+; CHECK-NEXT:    vmov.f32 s27, s31
+; CHECK-NEXT:    vmov.f32 s18, s16
+; CHECK-NEXT:    vadd.f32 q5, q6, q5
+; CHECK-NEXT:    vmov.f32 s10, s16
+; CHECK-NEXT:    vmov.f32 s11, s19
+; CHECK-NEXT:    vldrw.u32 q4, [r0, #96]
+; CHECK-NEXT:    vadd.f32 q2, q5, q2
+; CHECK-NEXT:    vldrw.u32 q5, [r0, #112]
+; CHECK-NEXT:    vmov.f32 s24, s17
+; CHECK-NEXT:    vmov.32 r0, q0[2]
+; CHECK-NEXT:    vmov.f32 s25, s20
+; CHECK-NEXT:    vdup.32 q7, r0
+; CHECK-NEXT:    vmov.f64 d6, d8
+; CHECK-NEXT:    vmov.32 r0, q0[1]
+; CHECK-NEXT:    vstrw.32 q2, [r1, #48]
+; CHECK-NEXT:    vstrw.32 q1, [r1]
+; CHECK-NEXT:    vmov.f32 s26, s23
+; CHECK-NEXT:    vmov.f32 s13, s19
+; CHECK-NEXT:    vmov.f32 s27, s31
+; CHECK-NEXT:    vdup.32 q7, r0
+; CHECK-NEXT:    vmov.f32 s14, s22
+; CHECK-NEXT:    vmov.f32 s20, s18
+; CHECK-NEXT:    vmov.f32 s15, s31
+; CHECK-NEXT:    vmov.f32 s2, s0
+; CHECK-NEXT:    vadd.f32 q6, q3, q6
+; CHECK-NEXT:    vmov.f32 s22, s0
+; CHECK-NEXT:    vmov.f32 s23, s3
+; CHECK-NEXT:    vadd.f32 q0, q6, q5
+; CHECK-NEXT:    vstrw.32 q0, [r1, #32]
+; CHECK-NEXT:    vldrw.u32 q0, [sp] @ 16-byte Reload
+; CHECK-NEXT:    vstrw.32 q0, [r1, #16]
+; CHECK-NEXT:    add sp, #16
+; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    bx lr
+entry:
+  %l1 = load <48 x float>, <48 x float>* %src, align 4
+  %s1 = shufflevector <48 x float> %l1, <48 x float> undef, <16 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21, i32 24, i32 27, i32 30, i32 33, i32 36, i32 39, i32 42, i32 45>
+  %s2 = shufflevector <48 x float> %l1, <48 x float> undef, <16 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22, i32 25, i32 28, i32 31, i32 34, i32 37, i32 40, i32 43, i32 46>
+  %s3 = shufflevector <48 x float> %l1, <48 x float> undef, <16 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23, i32 26, i32 29, i32 32, i32 35, i32 38, i32 41, i32 44, i32 47>
+  %a1 = fadd <16 x float> %s1, %s2
+  %a = fadd <16 x float> %a1, %s3
+  store <16 x float> %a, <16 x float> *%dst
+  ret void
+}
+
+; f16
+
+define void @vld3_v2f16(<6 x half> *%src, <2 x half> *%dst) {
+; CHECK-LABEL: vld3_v2f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    ldrd r2, r3, [r0]
+; CHECK-NEXT:    ldr r0, [r0, #8]
+; CHECK-NEXT:    vmov.32 q0[0], r2
+; CHECK-NEXT:    vmov.32 q0[1], r3
+; CHECK-NEXT:    vmov.32 q0[2], r0
+; CHECK-NEXT:    vmovx.f16 s4, s0
+; CHECK-NEXT:    vmov r2, s2
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmovx.f16 s8, s1
+; CHECK-NEXT:    vmov.16 q1[0], r0
+; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    vmov.16 q1[1], r2
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vmov.16 q2[0], r2
+; CHECK-NEXT:    vmov r2, s1
+; CHECK-NEXT:    vmov.16 q2[1], r0
+; CHECK-NEXT:    vadd.f16 q1, q2, q1
+; CHECK-NEXT:    vmovx.f16 s8, s2
+; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    vmov.16 q0[0], r2
+; CHECK-NEXT:    vmov.16 q0[1], r0
+; CHECK-NEXT:    vadd.f16 q0, q1, q0
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    str r0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %l1 = load <6 x half>, <6 x half>* %src, align 4
+  %s1 = shufflevector <6 x half> %l1, <6 x half> undef, <2 x i32> <i32 0, i32 3>
+  %s2 = shufflevector <6 x half> %l1, <6 x half> undef, <2 x i32> <i32 1, i32 4>
+  %s3 = shufflevector <6 x half> %l1, <6 x half> undef, <2 x i32> <i32 2, i32 5>
+  %a1 = fadd <2 x half> %s1, %s2
+  %a = fadd <2 x half> %a1, %s3
+  store <2 x half> %a, <2 x half> *%dst
+  ret void
+}
+
+define void @vld3_v4f16(<12 x half> *%src, <4 x half> *%dst) {
+; CHECK-LABEL: vld3_v4f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8}
+; CHECK-NEXT:    vpush {d8}
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vmovx.f16 s4, s0
+; CHECK-NEXT:    vmov r2, s2
+; CHECK-NEXT:    vmov r3, s4
+; CHECK-NEXT:    vmovx.f16 s4, s3
+; CHECK-NEXT:    vmov.16 q2[0], r3
+; CHECK-NEXT:    vmovx.f16 s12, s1
+; CHECK-NEXT:    vmov.16 q2[1], r2
+; CHECK-NEXT:    vmov r2, s4
+; CHECK-NEXT:    vmov.16 q2[2], r2
+; CHECK-NEXT:    ldrd r2, r0, [r0, #16]
+; CHECK-NEXT:    vmov.32 q1[0], r2
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vmov.32 q1[1], r0
+; CHECK-NEXT:    vmovx.f16 s0, s2
+; CHECK-NEXT:    vmov r0, s5
+; CHECK-NEXT:    vmovx.f16 s16, s4
+; CHECK-NEXT:    vmov.16 q2[3], r0
+; CHECK-NEXT:    vmov r0, s12
+; CHECK-NEXT:    vmov.16 q3[0], r2
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vmov.16 q3[1], r0
+; CHECK-NEXT:    vmov r0, s3
+; CHECK-NEXT:    vmov.16 q3[2], r0
+; CHECK-NEXT:    vmov r0, s16
+; CHECK-NEXT:    vmov.16 q3[3], r0
+; CHECK-NEXT:    vmov r0, s1
+; CHECK-NEXT:    vmov.16 q0[0], r0
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmov.16 q0[1], r2
+; CHECK-NEXT:    vmovx.f16 s4, s5
+; CHECK-NEXT:    vmov.16 q0[2], r0
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vadd.f16 q2, q3, q2
+; CHECK-NEXT:    vmov.16 q0[3], r0
+; CHECK-NEXT:    vadd.f16 q0, q2, q0
+; CHECK-NEXT:    vmov r2, s1
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    strd r0, r2, [r1]
+; CHECK-NEXT:    vpop {d8}
+; CHECK-NEXT:    bx lr
+entry:
+  %l1 = load <12 x half>, <12 x half>* %src, align 4
+  %s1 = shufflevector <12 x half> %l1, <12 x half> undef, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
+  %s2 = shufflevector <12 x half> %l1, <12 x half> undef, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
+  %s3 = shufflevector <12 x half> %l1, <12 x half> undef, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
+  %a1 = fadd <4 x half> %s1, %s2
+  %a = fadd <4 x half> %a1, %s3
+  store <4 x half> %a, <4 x half> *%dst
+  ret void
+}
+
+define void @vld3_v8f16(<24 x half> *%src, <8 x half> *%dst) {
+; CHECK-LABEL: vld3_v8f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, lr}
+; CHECK-NEXT:    push {r4, lr}
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #16]
+; CHECK-NEXT:    vldrw.u32 q4, [r0, #32]
+; CHECK-NEXT:    vmovx.f16 s0, s6
+; CHECK-NEXT:    vmov r3, s5
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vmov.16 q0[0], r3
+; CHECK-NEXT:    vmov.16 q0[1], r2
+; CHECK-NEXT:    vmov r2, s8
+; CHECK-NEXT:    vmovx.f16 s12, s9
+; CHECK-NEXT:    vmov.16 q0[2], r2
+; CHECK-NEXT:    vmov r2, s12
+; CHECK-NEXT:    vmovx.f16 s12, s19
+; CHECK-NEXT:    vmov.16 q0[3], r2
+; CHECK-NEXT:    vmov r3, s18
+; CHECK-NEXT:    vmov.f32 s2, s11
+; CHECK-NEXT:    vmovx.f16 s20, s16
+; CHECK-NEXT:    vmov r0, s12
+; CHECK-NEXT:    vmov.16 q3[6], r3
+; CHECK-NEXT:    vmov.16 q3[7], r0
+; CHECK-NEXT:    vmov r0, s20
+; CHECK-NEXT:    vmov.f32 s14, s16
+; CHECK-NEXT:    vmovx.f16 s24, s8
+; CHECK-NEXT:    vmov r4, s0
+; CHECK-NEXT:    vmov r2, s2
+; CHECK-NEXT:    vmov.16 q5[4], r2
+; CHECK-NEXT:    vmov r2, s17
+; CHECK-NEXT:    vmov.16 q5[5], r0
+; CHECK-NEXT:    vmov r0, s19
+; CHECK-NEXT:    vmov lr, s22
+; CHECK-NEXT:    vmovx.f16 s20, s17
+; CHECK-NEXT:    vmov r3, s20
+; CHECK-NEXT:    vmov.16 q5[6], r3
+; CHECK-NEXT:    vmov.16 q5[7], r0
+; CHECK-NEXT:    vmov r0, s16
+; CHECK-NEXT:    vmov r12, s23
+; CHECK-NEXT:    vmovx.f16 s20, s10
+; CHECK-NEXT:    vmov r3, s20
+; CHECK-NEXT:    vmov.16 q5[4], r3
+; CHECK-NEXT:    vmov.16 q5[5], r0
+; CHECK-NEXT:    vmov r3, s22
+; CHECK-NEXT:    vmovx.f16 s20, s18
+; CHECK-NEXT:    vmov r0, s20
+; CHECK-NEXT:    vmov.16 q4[6], r2
+; CHECK-NEXT:    vmov.16 q4[7], r0
+; CHECK-NEXT:    vmovx.f16 s20, s5
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmov r2, s20
+; CHECK-NEXT:    vmov.16 q5[0], r0
+; CHECK-NEXT:    vmov.16 q5[1], r2
+; CHECK-NEXT:    vmov r0, s7
+; CHECK-NEXT:    vmov.16 q5[2], r0
+; CHECK-NEXT:    vmov r0, s24
+; CHECK-NEXT:    vmov.16 q5[3], r0
+; CHECK-NEXT:    vmov r0, s10
+; CHECK-NEXT:    vmovx.f16 s24, s11
+; CHECK-NEXT:    vmov.16 q5[4], r0
+; CHECK-NEXT:    vmov r0, s24
+; CHECK-NEXT:    vmovx.f16 s24, s4
+; CHECK-NEXT:    vmov r2, s24
+; CHECK-NEXT:    vmov.16 q5[5], r0
+; CHECK-NEXT:    vmov r0, s6
+; CHECK-NEXT:    vmov.16 q6[0], r2
+; CHECK-NEXT:    vmovx.f16 s4, s7
+; CHECK-NEXT:    vmov.16 q6[1], r0
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmov.32 q1[0], r4
+; CHECK-NEXT:    vmov.16 q6[2], r0
+; CHECK-NEXT:    vmov r0, s9
+; CHECK-NEXT:    vmov.16 q6[3], r0
+; CHECK-NEXT:    vmov r4, s1
+; CHECK-NEXT:    vmov r2, s24
+; CHECK-NEXT:    vmov.32 q1[1], r4
+; CHECK-NEXT:    vmov r0, s25
+; CHECK-NEXT:    vmov.32 q0[0], r2
+; CHECK-NEXT:    vmov.32 q0[1], r0
+; CHECK-NEXT:    vmov.32 q1[2], lr
+; CHECK-NEXT:    vmov.32 q0[2], r3
+; CHECK-NEXT:    vmov r4, s15
+; CHECK-NEXT:    vmov.f32 s23, s19
+; CHECK-NEXT:    vmov.32 q0[3], r12
+; CHECK-NEXT:    vmov.32 q1[3], r4
+; CHECK-NEXT:    vadd.f16 q0, q5, q0
+; CHECK-NEXT:    vadd.f16 q0, q0, q1
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT:    pop {r4, pc}
+entry:
+  %l1 = load <24 x half>, <24 x half>* %src, align 4
+  %s1 = shufflevector <24 x half> %l1, <24 x half> undef, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
+  %s2 = shufflevector <24 x half> %l1, <24 x half> undef, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22>
+  %s3 = shufflevector <24 x half> %l1, <24 x half> undef, <8 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23>
+  %a1 = fadd <8 x half> %s1, %s2
+  %a = fadd <8 x half> %a1, %s3
+  store <8 x half> %a, <8 x half> *%dst
+  ret void
+}
+
+define void @vld3_v16f16(<48 x half> *%src, <16 x half> *%dst) {
+; CHECK-LABEL: vld3_v16f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, r5, r7, lr}
+; CHECK-NEXT:    push {r4, r5, r7, lr}
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT:    vldrw.u32 q4, [r0, #80]
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #48]
+; CHECK-NEXT:    vldrw.u32 q3, [r0, #64]
+; CHECK-NEXT:    vmovx.f16 s0, s19
+; CHECK-NEXT:    vmovx.f16 s4, s16
+; CHECK-NEXT:    vmov r2, s18
+; CHECK-NEXT:    vmovx.f16 s20, s13
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    vmov.16 q0[6], r2
+; CHECK-NEXT:    vmov r12, s4
+; CHECK-NEXT:    vmovx.f16 s4, s10
+; CHECK-NEXT:    vmov r2, s9
+; CHECK-NEXT:    vmov.16 q0[7], r3
+; CHECK-NEXT:    vmov r3, s4
+; CHECK-NEXT:    vmov.16 q1[0], r2
+; CHECK-NEXT:    vmov.16 q1[1], r3
+; CHECK-NEXT:    vmov r2, s12
+; CHECK-NEXT:    vmov.16 q1[2], r2
+; CHECK-NEXT:    vmov r2, s20
+; CHECK-NEXT:    vmov.16 q1[3], r2
+; CHECK-NEXT:    vmov r3, s16
+; CHECK-NEXT:    vmov.f32 s6, s15
+; CHECK-NEXT:    vmovx.f16 s24, s12
+; CHECK-NEXT:    vmov.f32 s2, s16
+; CHECK-NEXT:    vmovx.f16 s16, s18
+; CHECK-NEXT:    vmov r4, s16
+; CHECK-NEXT:    vmov r5, s4
+; CHECK-NEXT:    vmov r2, s6
+; CHECK-NEXT:    vmov.16 q5[4], r2
+; CHECK-NEXT:    vmov.16 q5[5], r12
+; CHECK-NEXT:    vmov lr, s22
+; CHECK-NEXT:    vmovx.f16 s20, s14
+; CHECK-NEXT:    vmov r2, s20
+; CHECK-NEXT:    vmov.16 q5[4], r2
+; CHECK-NEXT:    vmov r2, s19
+; CHECK-NEXT:    vmov.16 q5[5], r3
+; CHECK-NEXT:    vmov r12, s22
+; CHECK-NEXT:    vmovx.f16 s20, s17
+; CHECK-NEXT:    vmov r3, s20
+; CHECK-NEXT:    vmov.16 q5[6], r3
+; CHECK-NEXT:    vmov r3, s17
+; CHECK-NEXT:    vmov.16 q5[7], r2
+; CHECK-NEXT:    vmov.16 q4[6], r3
+; CHECK-NEXT:    vmov r2, s23
+; CHECK-NEXT:    vmov.16 q4[7], r4
+; CHECK-NEXT:    vmovx.f16 s20, s9
+; CHECK-NEXT:    vmov r4, s8
+; CHECK-NEXT:    vmov r3, s20
+; CHECK-NEXT:    vmov.16 q5[0], r4
+; CHECK-NEXT:    vmov.16 q5[1], r3
+; CHECK-NEXT:    vmov r3, s11
+; CHECK-NEXT:    vmov.16 q5[2], r3
+; CHECK-NEXT:    vmov r3, s24
+; CHECK-NEXT:    vmov.16 q5[3], r3
+; CHECK-NEXT:    vmov r3, s14
+; CHECK-NEXT:    vmovx.f16 s24, s15
+; CHECK-NEXT:    vmov.16 q5[4], r3
+; CHECK-NEXT:    vmov r3, s24
+; CHECK-NEXT:    vmovx.f16 s24, s8
+; CHECK-NEXT:    vmov.16 q5[5], r3
+; CHECK-NEXT:    vmov r3, s24
+; CHECK-NEXT:    vmov r4, s10
+; CHECK-NEXT:    vmov.16 q6[0], r3
+; CHECK-NEXT:    vmovx.f16 s8, s11
+; CHECK-NEXT:    vmov.16 q6[1], r4
+; CHECK-NEXT:    vmov r3, s8
+; CHECK-NEXT:    vmov.32 q2[0], r5
+; CHECK-NEXT:    vmov.16 q6[2], r3
+; CHECK-NEXT:    vmov r3, s13
+; CHECK-NEXT:    vmov.16 q6[3], r3
+; CHECK-NEXT:    vmov r5, s5
+; CHECK-NEXT:    vmov r3, s24
+; CHECK-NEXT:    vmov.32 q2[1], r5
+; CHECK-NEXT:    vmov r5, s3
+; CHECK-NEXT:    vmov.32 q0[0], r3
+; CHECK-NEXT:    vmov r4, s25
+; CHECK-NEXT:    vmov.32 q2[2], lr
+; CHECK-NEXT:    vmov.32 q0[1], r4
+; CHECK-NEXT:    vmov.f32 s23, s19
+; CHECK-NEXT:    vmov.32 q0[2], r12
+; CHECK-NEXT:    vldrw.u32 q4, [r0, #32]
+; CHECK-NEXT:    vmov.32 q0[3], r2
+; CHECK-NEXT:    vmov.32 q2[3], r5
+; CHECK-NEXT:    vadd.f16 q0, q5, q0
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #16]
+; CHECK-NEXT:    vadd.f16 q0, q0, q2
+; CHECK-NEXT:    vldrw.u32 q2, [r0]
+; CHECK-NEXT:    vstrw.32 q0, [r1, #16]
+; CHECK-NEXT:    vmovx.f16 s0, s19
+; CHECK-NEXT:    vmov r0, s18
+; CHECK-NEXT:    vmovx.f16 s12, s16
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vmov.16 q0[6], r0
+; CHECK-NEXT:    vmov.16 q0[7], r2
+; CHECK-NEXT:    vmov r0, s12
+; CHECK-NEXT:    vmovx.f16 s12, s10
+; CHECK-NEXT:    vmov r2, s9
+; CHECK-NEXT:    vmov r3, s12
+; CHECK-NEXT:    vmov.16 q3[0], r2
+; CHECK-NEXT:    vmov.16 q3[1], r3
+; CHECK-NEXT:    vmov r2, s4
+; CHECK-NEXT:    vmovx.f16 s20, s5
+; CHECK-NEXT:    vmov.16 q3[2], r2
+; CHECK-NEXT:    vmov r2, s20
+; CHECK-NEXT:    vmovx.f16 s24, s4
+; CHECK-NEXT:    vmov.16 q3[3], r2
+; CHECK-NEXT:    vmov r3, s16
+; CHECK-NEXT:    vmov.f32 s14, s7
+; CHECK-NEXT:    vmov.f32 s2, s16
+; CHECK-NEXT:    vmovx.f16 s16, s18
+; CHECK-NEXT:    vmov r4, s16
+; CHECK-NEXT:    vmov r2, s14
+; CHECK-NEXT:    vmov.16 q5[4], r2
+; CHECK-NEXT:    vmov.16 q5[5], r0
+; CHECK-NEXT:    vmov r2, s22
+; CHECK-NEXT:    vmovx.f16 s20, s6
+; CHECK-NEXT:    vmov r0, s20
+; CHECK-NEXT:    vmov.16 q5[4], r0
+; CHECK-NEXT:    vmov r0, s12
+; CHECK-NEXT:    vmov.16 q5[5], r3
+; CHECK-NEXT:    vmov r3, s19
+; CHECK-NEXT:    vmov r12, s22
+; CHECK-NEXT:    vmovx.f16 s20, s17
+; CHECK-NEXT:    vmov r5, s20
+; CHECK-NEXT:    vmov.16 q5[6], r5
+; CHECK-NEXT:    vmov r5, s17
+; CHECK-NEXT:    vmov.16 q5[7], r3
+; CHECK-NEXT:    vmov.16 q4[6], r5
+; CHECK-NEXT:    vmov r3, s23
+; CHECK-NEXT:    vmov.16 q4[7], r4
+; CHECK-NEXT:    vmovx.f16 s20, s9
+; CHECK-NEXT:    vmov r4, s8
+; CHECK-NEXT:    vmov r5, s20
+; CHECK-NEXT:    vmov.16 q5[0], r4
+; CHECK-NEXT:    vmov.16 q5[1], r5
+; CHECK-NEXT:    vmov r5, s11
+; CHECK-NEXT:    vmov.16 q5[2], r5
+; CHECK-NEXT:    vmov r5, s24
+; CHECK-NEXT:    vmov.16 q5[3], r5
+; CHECK-NEXT:    vmov r5, s6
+; CHECK-NEXT:    vmovx.f16 s24, s7
+; CHECK-NEXT:    vmov.16 q5[4], r5
+; CHECK-NEXT:    vmov r5, s24
+; CHECK-NEXT:    vmovx.f16 s24, s8
+; CHECK-NEXT:    vmov.16 q5[5], r5
+; CHECK-NEXT:    vmov r5, s24
+; CHECK-NEXT:    vmov r4, s10
+; CHECK-NEXT:    vmov.16 q6[0], r5
+; CHECK-NEXT:    vmovx.f16 s8, s11
+; CHECK-NEXT:    vmov.16 q6[1], r4
+; CHECK-NEXT:    vmov r5, s8
+; CHECK-NEXT:    vmov.16 q6[2], r5
+; CHECK-NEXT:    vmov r5, s5
+; CHECK-NEXT:    vmov.16 q6[3], r5
+; CHECK-NEXT:    vmov.32 q1[0], r0
+; CHECK-NEXT:    vmov r0, s13
+; CHECK-NEXT:    vmov r5, s24
+; CHECK-NEXT:    vmov.32 q1[1], r0
+; CHECK-NEXT:    vmov r0, s3
+; CHECK-NEXT:    vmov.32 q0[0], r5
+; CHECK-NEXT:    vmov r4, s25
+; CHECK-NEXT:    vmov.32 q1[2], r2
+; CHECK-NEXT:    vmov.32 q0[1], r4
+; CHECK-NEXT:    vmov.f32 s23, s19
+; CHECK-NEXT:    vmov.32 q0[2], r12
+; CHECK-NEXT:    vmov.32 q1[3], r0
+; CHECK-NEXT:    vmov.32 q0[3], r3
+; CHECK-NEXT:    vadd.f16 q0, q5, q0
+; CHECK-NEXT:    vadd.f16 q0, q0, q1
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT:    pop {r4, r5, r7, pc}
+entry:
+  %l1 = load <48 x half>, <48 x half>* %src, align 4
+  %s1 = shufflevector <48 x half> %l1, <48 x half> undef, <16 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21, i32 24, i32 27, i32 30, i32 33, i32 36, i32 39, i32 42, i32 45>
+  %s2 = shufflevector <48 x half> %l1, <48 x half> undef, <16 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22, i32 25, i32 28, i32 31, i32 34, i32 37, i32 40, i32 43, i32 46>
+  %s3 = shufflevector <48 x half> %l1, <48 x half> undef, <16 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23, i32 26, i32 29, i32 32, i32 35, i32 38, i32 41, i32 44, i32 47>
+  %a1 = fadd <16 x half> %s1, %s2
+  %a = fadd <16 x half> %a1, %s3
+  store <16 x half> %a, <16 x half> *%dst
+  ret void
+}
+
+; f64
+
+define void @vld3_v2f64(<6 x double> *%src, <2 x double> *%dst) {
+; CHECK-LABEL: vld3_v2f64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #32]
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #16]
+; CHECK-NEXT:    vldrw.u32 q3, [r0]
+; CHECK-NEXT:    vadd.f64 d4, d3, d0
+; CHECK-NEXT:    vadd.f64 d5, d6, d7
+; CHECK-NEXT:    vadd.f64 d1, d4, d1
+; CHECK-NEXT:    vadd.f64 d0, d5, d2
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %l1 = load <6 x double>, <6 x double>* %src, align 4
+  %s1 = shufflevector <6 x double> %l1, <6 x double> undef, <2 x i32> <i32 0, i32 3>
+  %s2 = shufflevector <6 x double> %l1, <6 x double> undef, <2 x i32> <i32 1, i32 4>
+  %s3 = shufflevector <6 x double> %l1, <6 x double> undef, <2 x i32> <i32 2, i32 5>
+  %a1 = fadd <2 x double> %s1, %s2
+  %a = fadd <2 x double> %a1, %s3
+  store <2 x double> %a, <2 x double> *%dst
+  ret void
+}
+
+define void @vld3_v4f64(<12 x double> *%src, <4 x double> *%dst) {
+; CHECK-LABEL: vld3_v4f64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT:    vldrw.u32 q3, [r0, #48]
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #80]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #64]
+; CHECK-NEXT:    vldrw.u32 q4, [r0, #16]
+; CHECK-NEXT:    vadd.f64 d5, d6, d7
+; CHECK-NEXT:    vldrw.u32 q3, [r0, #32]
+; CHECK-NEXT:    vldrw.u32 q6, [r0]
+; CHECK-NEXT:    vadd.f64 d4, d1, d2
+; CHECK-NEXT:    vadd.f64 d10, d9, d6
+; CHECK-NEXT:    vadd.f64 d11, d12, d13
+; CHECK-NEXT:    vadd.f64 d3, d4, d3
+; CHECK-NEXT:    vadd.f64 d2, d5, d0
+; CHECK-NEXT:    vadd.f64 d1, d10, d7
+; CHECK-NEXT:    vstrw.32 q1, [r1, #16]
+; CHECK-NEXT:    vadd.f64 d0, d11, d8
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT:    bx lr
+entry:
+  %l1 = load <12 x double>, <12 x double>* %src, align 4
+  %s1 = shufflevector <12 x double> %l1, <12 x double> undef, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
+  %s2 = shufflevector <12 x double> %l1, <12 x double> undef, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
+  %s3 = shufflevector <12 x double> %l1, <12 x double> undef, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
+  %a1 = fadd <4 x double> %s1, %s2
+  %a = fadd <4 x double> %a1, %s3
+  store <4 x double> %a, <4 x double> *%dst
+  ret void
+}
diff --git a/llvm/test/CodeGen/Thumb2/mve-vld4.ll b/llvm/test/CodeGen/Thumb2/mve-vld4.ll
new file mode 100644
index 000000000..654ed2e7f
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-vld4.ll
@@ -0,0 +1,1235 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve.fp,+fp64 -mve-max-interleave-factor=4 -verify-machineinstrs %s -o - | FileCheck %s
+
+; i32
+
+define void @vld4_v2i32(<8 x i32> *%src, <2 x i32> *%dst) {
+; CHECK-LABEL: vld4_v2i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #16]
+; CHECK-NEXT:    vmov.f32 s8, s3
+; CHECK-NEXT:    vmov.32 r3, q1[0]
+; CHECK-NEXT:    vmov.f32 s10, s7
+; CHECK-NEXT:    vmov r2, s6
+; CHECK-NEXT:    vmov.f32 s12, s1
+; CHECK-NEXT:    vmov.f32 s14, s5
+; CHECK-NEXT:    vdup.32 q1, r3
+; CHECK-NEXT:    vmov r3, s6
+; CHECK-NEXT:    vmov r0, s10
+; CHECK-NEXT:    add r0, r2
+; CHECK-NEXT:    vmov r2, s14
+; CHECK-NEXT:    add r2, r3
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    add r0, r2
+; CHECK-NEXT:    str r0, [r1, #4]
+; CHECK-NEXT:    vmov.32 r2, q0[2]
+; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    vdup.32 q1, r2
+; CHECK-NEXT:    vmov r2, s4
+; CHECK-NEXT:    add r0, r2
+; CHECK-NEXT:    vmov r2, s12
+; CHECK-NEXT:    add r2, r3
+; CHECK-NEXT:    add r0, r2
+; CHECK-NEXT:    str r0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %l1 = load <8 x i32>, <8 x i32>* %src, align 4
+  %s1 = shufflevector <8 x i32> %l1, <8 x i32> undef, <2 x i32> <i32 0, i32 4>
+  %s2 = shufflevector <8 x i32> %l1, <8 x i32> undef, <2 x i32> <i32 1, i32 5>
+  %s3 = shufflevector <8 x i32> %l1, <8 x i32> undef, <2 x i32> <i32 2, i32 6>
+  %s4 = shufflevector <8 x i32> %l1, <8 x i32> undef, <2 x i32> <i32 3, i32 7>
+  %a1 = add <2 x i32> %s1, %s2
+  %a2 = add <2 x i32> %s3, %s4
+  %a3 = add <2 x i32> %a1, %a2
+  store <2 x i32> %a3, <2 x i32> *%dst
+  ret void
+}
+
+define void @vld4_v4i32(<16 x i32> *%src, <4 x i32> *%dst) {
+; CHECK-LABEL: vld4_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9}
+; CHECK-NEXT:    vpush {d8, d9}
+; CHECK-NEXT:    vld40.32 {q0, q1, q2, q3}, [r0]
+; CHECK-NEXT:    vld41.32 {q0, q1, q2, q3}, [r0]
+; CHECK-NEXT:    vld42.32 {q0, q1, q2, q3}, [r0]
+; CHECK-NEXT:    vld43.32 {q0, q1, q2, q3}, [r0]
+; CHECK-NEXT:    @ kill: def $q0 killed $q0 killed $q0_q1_q2_q3
+; CHECK-NEXT:    vadd.i32 q4, q2, q3
+; CHECK-NEXT:    vadd.i32 q0, q0, q1
+; CHECK-NEXT:    vadd.i32 q0, q0, q4
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vpop {d8, d9}
+; CHECK-NEXT:    bx lr
+entry:
+  %l1 = load <16 x i32>, <16 x i32>* %src, align 4
+  %s1 = shufflevector <16 x i32> %l1, <16 x i32> undef, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+  %s2 = shufflevector <16 x i32> %l1, <16 x i32> undef, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
+  %s3 = shufflevector <16 x i32> %l1, <16 x i32> undef, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
+  %s4 = shufflevector <16 x i32> %l1, <16 x i32> undef, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
+  %a1 = add <4 x i32> %s1, %s2
+  %a2 = add <4 x i32> %s3, %s4
+  %a3 = add <4 x i32> %a1, %a2
+  store <4 x i32> %a3, <4 x i32> *%dst
+  ret void
+}
+
+define void @vld4_v8i32(<32 x i32> *%src, <8 x i32> *%dst) {
+; CHECK-LABEL: vld4_v8i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, r5}
+; CHECK-NEXT:    push {r4, r5}
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    .pad #88
+; CHECK-NEXT:    sub sp, #88
+; CHECK-NEXT:    add.w r2, r0, #64
+; CHECK-NEXT:    vld40.32 {q0, q1, q2, q3}, [r2]
+; CHECK-NEXT:    vld41.32 {q0, q1, q2, q3}, [r2]
+; CHECK-NEXT:    vstmia sp, {d0, d1, d2, d3, d4, d5, d6, d7} @ 64-byte Spill
+; CHECK-NEXT:    vld40.32 {q4, q5, q6, q7}, [r0]
+; CHECK-NEXT:    vld41.32 {q4, q5, q6, q7}, [r0]
+; CHECK-NEXT:    vld42.32 {q4, q5, q6, q7}, [r0]
+; CHECK-NEXT:    vld43.32 {q4, q5, q6, q7}, [r0]
+; CHECK-NEXT:    vstrw.32 q5, [sp, #64] @ 16-byte Spill
+; CHECK-NEXT:    vmov q1, q4
+; CHECK-NEXT:    vldrw.u32 q0, [sp, #64] @ 16-byte Reload
+; CHECK-NEXT:    vadd.i32 q4, q6, q7
+; CHECK-NEXT:    vadd.i32 q5, q1, q0
+; CHECK-NEXT:    vldmia sp, {d0, d1, d2, d3, d4, d5, d6, d7} @ 64-byte Reload
+; CHECK-NEXT:    vadd.i32 q4, q5, q4
+; CHECK-NEXT:    vld42.32 {q0, q1, q2, q3}, [r2]
+; CHECK-NEXT:    vld43.32 {q0, q1, q2, q3}, [r2]
+; CHECK-NEXT:    vstrw.32 q4, [r1]
+; CHECK-NEXT:    @ kill: def $q0 killed $q0 killed $q0_q1_q2_q3
+; CHECK-NEXT:    vadd.i32 q5, q2, q3
+; CHECK-NEXT:    vadd.i32 q0, q0, q1
+; CHECK-NEXT:    vadd.i32 q0, q0, q5
+; CHECK-NEXT:    vstrw.32 q0, [r1, #16]
+; CHECK-NEXT:    add sp, #88
+; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    pop {r4, r5}
+; CHECK-NEXT:    bx lr
+entry:
+  %l1 = load <32 x i32>, <32 x i32>* %src, align 4
+  %s1 = shufflevector <32 x i32> %l1, <32 x i32> undef, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
+  %s2 = shufflevector <32 x i32> %l1, <32 x i32> undef, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>
+  %s3 = shufflevector <32 x i32> %l1, <32 x i32> undef, <8 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30>
+  %s4 = shufflevector <32 x i32> %l1, <32 x i32> undef, <8 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31>
+  %a1 = add <8 x i32> %s1, %s2
+  %a2 = add <8 x i32> %s3, %s4
+  %a3 = add <8 x i32> %a1, %a2
+  store <8 x i32> %a3, <8 x i32> *%dst
+  ret void
+}
+
+define void @vld4_v16i32(<64 x i32> *%src, <16 x i32> *%dst) {
+; CHECK-LABEL: vld4_v16i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, r5}
+; CHECK-NEXT:    push {r4, r5}
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    .pad #152
+; CHECK-NEXT:    sub sp, #152
+; CHECK-NEXT:    add.w r2, r0, #128
+; CHECK-NEXT:    add r3, sp, #64
+; CHECK-NEXT:    vld40.32 {q0, q1, q2, q3}, [r2]
+; CHECK-NEXT:    add r4, sp, #64
+; CHECK-NEXT:    vld41.32 {q0, q1, q2, q3}, [r2]
+; CHECK-NEXT:    vstmia r3, {d0, d1, d2, d3, d4, d5, d6, d7} @ 64-byte Spill
+; CHECK-NEXT:    add.w r3, r0, #64
+; CHECK-NEXT:    vld40.32 {q0, q1, q2, q3}, [r0]
+; CHECK-NEXT:    vld41.32 {q0, q1, q2, q3}, [r0]
+; CHECK-NEXT:    vld42.32 {q0, q1, q2, q3}, [r0]
+; CHECK-NEXT:    vld43.32 {q0, q1, q2, q3}, [r0]
+; CHECK-NEXT:    adds r0, #192
+; CHECK-NEXT:    vstrw.32 q1, [sp, #32] @ 16-byte Spill
+; CHECK-NEXT:    vadd.i32 q4, q2, q3
+; CHECK-NEXT:    vmov q5, q0
+; CHECK-NEXT:    vldrw.u32 q0, [sp, #32] @ 16-byte Reload
+; CHECK-NEXT:    vstrw.32 q4, [sp, #48] @ 16-byte Spill
+; CHECK-NEXT:    vadd.i32 q4, q5, q0
+; CHECK-NEXT:    vldmia r4, {d0, d1, d2, d3, d4, d5, d6, d7} @ 64-byte Reload
+; CHECK-NEXT:    vstrw.32 q4, [sp, #32] @ 16-byte Spill
+; CHECK-NEXT:    add r4, sp, #64
+; CHECK-NEXT:    vld42.32 {q0, q1, q2, q3}, [r2]
+; CHECK-NEXT:    vstmia r4, {d0, d1, d2, d3, d4, d5, d6, d7} @ 64-byte Spill
+; CHECK-NEXT:    vld40.32 {q4, q5, q6, q7}, [r3]
+; CHECK-NEXT:    vld41.32 {q4, q5, q6, q7}, [r3]
+; CHECK-NEXT:    vld42.32 {q4, q5, q6, q7}, [r3]
+; CHECK-NEXT:    vld43.32 {q4, q5, q6, q7}, [r3]
+; CHECK-NEXT:    vldrw.u32 q0, [sp, #48] @ 16-byte Reload
+; CHECK-NEXT:    vldrw.u32 q1, [sp, #32] @ 16-byte Reload
+; CHECK-NEXT:    add r3, sp, #64
+; CHECK-NEXT:    vstrw.32 q6, [sp, #16] @ 16-byte Spill
+; CHECK-NEXT:    vadd.i32 q4, q4, q5
+; CHECK-NEXT:    vadd.i32 q0, q1, q0
+; CHECK-NEXT:    vstrw.32 q0, [sp, #48] @ 16-byte Spill
+; CHECK-NEXT:    vldrw.u32 q0, [sp, #16] @ 16-byte Reload
+; CHECK-NEXT:    vadd.i32 q0, q0, q7
+; CHECK-NEXT:    vadd.i32 q0, q4, q0
+; CHECK-NEXT:    vstrw.32 q0, [sp, #32] @ 16-byte Spill
+; CHECK-NEXT:    vldmia r3, {d0, d1, d2, d3, d4, d5, d6, d7} @ 64-byte Reload
+; CHECK-NEXT:    vld43.32 {q0, q1, q2, q3}, [r2]
+; CHECK-NEXT:    add r2, sp, #64
+; CHECK-NEXT:    vstmia r2, {d0, d1, d2, d3, d4, d5, d6, d7} @ 64-byte Spill
+; CHECK-NEXT:    vld40.32 {q4, q5, q6, q7}, [r0]
+; CHECK-NEXT:    vld41.32 {q4, q5, q6, q7}, [r0]
+; CHECK-NEXT:    vld42.32 {q4, q5, q6, q7}, [r0]
+; CHECK-NEXT:    vld43.32 {q4, q5, q6, q7}, [r0]
+; CHECK-NEXT:    add r0, sp, #64
+; CHECK-NEXT:    @ kill: def $q4 killed $q4 killed $q4_q5_q6_q7
+; CHECK-NEXT:    vstrw.32 q7, [sp, #16] @ 16-byte Spill
+; CHECK-NEXT:    vmov q2, q5
+; CHECK-NEXT:    vldrw.u32 q0, [sp, #16] @ 16-byte Reload
+; CHECK-NEXT:    vadd.i32 q4, q4, q2
+; CHECK-NEXT:    vadd.i32 q5, q6, q0
+; CHECK-NEXT:    vldmia r0, {d0, d1, d2, d3, d4, d5, d6, d7} @ 64-byte Reload
+; CHECK-NEXT:    @ kill: def $q0 killed $q0 killed $q0_q1_q2_q3
+; CHECK-NEXT:    vadd.i32 q4, q4, q5
+; CHECK-NEXT:    vadd.i32 q5, q2, q3
+; CHECK-NEXT:    vadd.i32 q0, q0, q1
+; CHECK-NEXT:    vstrw.32 q4, [r1, #48]
+; CHECK-NEXT:    vadd.i32 q0, q0, q5
+; CHECK-NEXT:    vstrw.32 q0, [r1, #32]
+; CHECK-NEXT:    vldrw.u32 q0, [sp, #32] @ 16-byte Reload
+; CHECK-NEXT:    vstrw.32 q0, [r1, #16]
+; CHECK-NEXT:    vldrw.u32 q0, [sp, #48] @ 16-byte Reload
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    add sp, #152
+; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    pop {r4, r5}
+; CHECK-NEXT:    bx lr
+entry:
+  %l1 = load <64 x i32>, <64 x i32>* %src, align 4
+  %s1 = shufflevector <64 x i32> %l1, <64 x i32> undef, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28, i32 32, i32 36, i32 40, i32 44, i32 48, i32 52, i32 56, i32 60>
+  %s2 = shufflevector <64 x i32> %l1, <64 x i32> undef, <16 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29, i32 33, i32 37, i32 41, i32 45, i32 49, i32 53, i32 57, i32 61>
+  %s3 = shufflevector <64 x i32> %l1, <64 x i32> undef, <16 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30, i32 34, i32 38, i32 42, i32 46, i32 50, i32 54, i32 58, i32 62>
+  %s4 = shufflevector <64 x i32> %l1, <64 x i32> undef, <16 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31, i32 35, i32 39, i32 43, i32 47, i32 51, i32 55, i32 59, i32 63>
+  %a1 = add <16 x i32> %s1, %s2
+  %a2 = add <16 x i32> %s3, %s4
+  %a3 = add <16 x i32> %a1, %a2
+  store <16 x i32> %a3, <16 x i32> *%dst
+  ret void
+}
+
+; i16
+
+define void @vld4_v2i16(<8 x i16> *%src, <2 x i16> *%dst) {
+; CHECK-LABEL: vld4_v2i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vmov.u16 r0, q0[7]
+; CHECK-NEXT:    vmov.u16 r2, q0[6]
+; CHECK-NEXT:    add r0, r2
+; CHECK-NEXT:    vmov.u16 r2, q0[5]
+; CHECK-NEXT:    vmov.u16 r3, q0[4]
+; CHECK-NEXT:    add r2, r3
+; CHECK-NEXT:    vmov.u16 r3, q0[0]
+; CHECK-NEXT:    add r0, r2
+; CHECK-NEXT:    strh r0, [r1, #2]
+; CHECK-NEXT:    vmov.u16 r0, q0[3]
+; CHECK-NEXT:    vmov.u16 r2, q0[2]
+; CHECK-NEXT:    add r0, r2
+; CHECK-NEXT:    vmov.u16 r2, q0[1]
+; CHECK-NEXT:    add r2, r3
+; CHECK-NEXT:    add r0, r2
+; CHECK-NEXT:    strh r0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %l1 = load <8 x i16>, <8 x i16>* %src, align 4
+  %s1 = shufflevector <8 x i16> %l1, <8 x i16> undef, <2 x i32> <i32 0, i32 4>
+  %s2 = shufflevector <8 x i16> %l1, <8 x i16> undef, <2 x i32> <i32 1, i32 5>
+  %s3 = shufflevector <8 x i16> %l1, <8 x i16> undef, <2 x i32> <i32 2, i32 6>
+  %s4 = shufflevector <8 x i16> %l1, <8 x i16> undef, <2 x i32> <i32 3, i32 7>
+  %a1 = add <2 x i16> %s1, %s2
+  %a2 = add <2 x i16> %s3, %s4
+  %a3 = add <2 x i16> %a1, %a2
+  store <2 x i16> %a3, <2 x i16> *%dst
+  ret void
+}
+
+define void @vld4_v4i16(<16 x i16> *%src, <4 x i16> *%dst) {
+; CHECK-LABEL: vld4_v4i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9}
+; CHECK-NEXT:    vpush {d8, d9}
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #16]
+; CHECK-NEXT:    vmov.u16 r2, q1[3]
+; CHECK-NEXT:    vmov.u16 r0, q0[3]
+; CHECK-NEXT:    vmov.32 q2[0], r2
+; CHECK-NEXT:    vmov.u16 r2, q1[7]
+; CHECK-NEXT:    vmov.32 q2[1], r2
+; CHECK-NEXT:    vmov.32 q2[2], r0
+; CHECK-NEXT:    vmov.u16 r0, q0[7]
+; CHECK-NEXT:    vmov.32 q2[3], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[2]
+; CHECK-NEXT:    vmov.32 q3[0], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[6]
+; CHECK-NEXT:    vmov.32 q3[1], r0
+; CHECK-NEXT:    vmov.u16 r0, q0[2]
+; CHECK-NEXT:    vmov.32 q3[2], r0
+; CHECK-NEXT:    vmov.u16 r0, q0[6]
+; CHECK-NEXT:    vmov.32 q3[3], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[0]
+; CHECK-NEXT:    vadd.i32 q2, q3, q2
+; CHECK-NEXT:    vmov.32 q3[0], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[4]
+; CHECK-NEXT:    vmov.32 q3[1], r0
+; CHECK-NEXT:    vmov.u16 r0, q0[0]
+; CHECK-NEXT:    vmov.32 q3[2], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[1]
+; CHECK-NEXT:    vmov.32 q4[0], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[5]
+; CHECK-NEXT:    vmov.32 q4[1], r0
+; CHECK-NEXT:    vmov.u16 r0, q0[1]
+; CHECK-NEXT:    vmov.32 q4[2], r0
+; CHECK-NEXT:    vmov.u16 r0, q0[5]
+; CHECK-NEXT:    vmov.32 q4[3], r0
+; CHECK-NEXT:    vmov.u16 r0, q0[4]
+; CHECK-NEXT:    vmov.32 q3[3], r0
+; CHECK-NEXT:    vadd.i32 q0, q3, q4
+; CHECK-NEXT:    vadd.i32 q0, q0, q2
+; CHECK-NEXT:    vstrh.32 q0, [r1]
+; CHECK-NEXT:    vpop {d8, d9}
+; CHECK-NEXT:    bx lr
+entry:
+  %l1 = load <16 x i16>, <16 x i16>* %src, align 4
+  %s1 = shufflevector <16 x i16> %l1, <16 x i16> undef, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+  %s2 = shufflevector <16 x i16> %l1, <16 x i16> undef, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
+  %s3 = shufflevector <16 x i16> %l1, <16 x i16> undef, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
+  %s4 = shufflevector <16 x i16> %l1, <16 x i16> undef, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
+  %a1 = add <4 x i16> %s1, %s2
+  %a2 = add <4 x i16> %s3, %s4
+  %a3 = add <4 x i16> %a1, %a2
+  store <4 x i16> %a3, <4 x i16> *%dst
+  ret void
+}
+
+define void @vld4_v8i16(<32 x i16> *%src, <8 x i16> *%dst) {
+; CHECK-LABEL: vld4_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9}
+; CHECK-NEXT:    vpush {d8, d9}
+; CHECK-NEXT:    vld40.16 {q0, q1, q2, q3}, [r0]
+; CHECK-NEXT:    vld41.16 {q0, q1, q2, q3}, [r0]
+; CHECK-NEXT:    vld42.16 {q0, q1, q2, q3}, [r0]
+; CHECK-NEXT:    vld43.16 {q0, q1, q2, q3}, [r0]
+; CHECK-NEXT:    @ kill: def $q0 killed $q0 killed $q0_q1_q2_q3
+; CHECK-NEXT:    vadd.i16 q4, q2, q3
+; CHECK-NEXT:    vadd.i16 q0, q0, q1
+; CHECK-NEXT:    vadd.i16 q0, q0, q4
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vpop {d8, d9}
+; CHECK-NEXT:    bx lr
+entry:
+  %l1 = load <32 x i16>, <32 x i16>* %src, align 4
+  %s1 = shufflevector <32 x i16> %l1, <32 x i16> undef, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
+  %s2 = shufflevector <32 x i16> %l1, <32 x i16> undef, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>
+  %s3 = shufflevector <32 x i16> %l1, <32 x i16> undef, <8 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30>
+  %s4 = shufflevector <32 x i16> %l1, <32 x i16> undef, <8 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31>
+  %a1 = add <8 x i16> %s1, %s2
+  %a2 = add <8 x i16> %s3, %s4
+  %a3 = add <8 x i16> %a1, %a2
+  store <8 x i16> %a3, <8 x i16> *%dst
+  ret void
+}
+
+define void @vld4_v16i16(<64 x i16> *%src, <16 x i16> *%dst) {
+; CHECK-LABEL: vld4_v16i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, r5}
+; CHECK-NEXT:    push {r4, r5}
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    .pad #88
+; CHECK-NEXT:    sub sp, #88
+; CHECK-NEXT:    add.w r2, r0, #64
+; CHECK-NEXT:    vld40.16 {q0, q1, q2, q3}, [r2]
+; CHECK-NEXT:    vld41.16 {q0, q1, q2, q3}, [r2]
+; CHECK-NEXT:    vstmia sp, {d0, d1, d2, d3, d4, d5, d6, d7} @ 64-byte Spill
+; CHECK-NEXT:    vld40.16 {q4, q5, q6, q7}, [r0]
+; CHECK-NEXT:    vld41.16 {q4, q5, q6, q7}, [r0]
+; CHECK-NEXT:    vld42.16 {q4, q5, q6, q7}, [r0]
+; CHECK-NEXT:    vld43.16 {q4, q5, q6, q7}, [r0]
+; CHECK-NEXT:    vstrw.32 q5, [sp, #64] @ 16-byte Spill
+; CHECK-NEXT:    vmov q1, q4
+; CHECK-NEXT:    vldrw.u32 q0, [sp, #64] @ 16-byte Reload
+; CHECK-NEXT:    vadd.i16 q4, q6, q7
+; CHECK-NEXT:    vadd.i16 q5, q1, q0
+; CHECK-NEXT:    vldmia sp, {d0, d1, d2, d3, d4, d5, d6, d7} @ 64-byte Reload
+; CHECK-NEXT:    vadd.i16 q4, q5, q4
+; CHECK-NEXT:    vld42.16 {q0, q1, q2, q3}, [r2]
+; CHECK-NEXT:    vld43.16 {q0, q1, q2, q3}, [r2]
+; CHECK-NEXT:    vstrw.32 q4, [r1]
+; CHECK-NEXT:    @ kill: def $q0 killed $q0 killed $q0_q1_q2_q3
+; CHECK-NEXT:    vadd.i16 q5, q2, q3
+; CHECK-NEXT:    vadd.i16 q0, q0, q1
+; CHECK-NEXT:    vadd.i16 q0, q0, q5
+; CHECK-NEXT:    vstrw.32 q0, [r1, #16]
+; CHECK-NEXT:    add sp, #88
+; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    pop {r4, r5}
+; CHECK-NEXT:    bx lr
+entry:
+  %l1 = load <64 x i16>, <64 x i16>* %src, align 4
+  %s1 = shufflevector <64 x i16> %l1, <64 x i16> undef, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28, i32 32, i32 36, i32 40, i32 44, i32 48, i32 52, i32 56, i32 60>
+  %s2 = shufflevector <64 x i16> %l1, <64 x i16> undef, <16 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29, i32 33, i32 37, i32 41, i32 45, i32 49, i32 53, i32 57, i32 61>
+  %s3 = shufflevector <64 x i16> %l1, <64 x i16> undef, <16 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30, i32 34, i32 38, i32 42, i32 46, i32 50, i32 54, i32 58, i32 62>
+  %s4 = shufflevector <64 x i16> %l1, <64 x i16> undef, <16 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31, i32 35, i32 39, i32 43, i32 47, i32 51, i32 55, i32 59, i32 63>
+  %a1 = add <16 x i16> %s1, %s2
+  %a2 = add <16 x i16> %s3, %s4
+  %a3 = add <16 x i16> %a1, %a2
+  store <16 x i16> %a3, <16 x i16> *%dst
+  ret void
+}
+
+; i8
+
+define void @vld4_v2i8(<8 x i8> *%src, <2 x i8> *%dst) {
+; CHECK-LABEL: vld4_v2i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u16 q0, [r0]
+; CHECK-NEXT:    vmov.u16 r0, q0[7]
+; CHECK-NEXT:    vmov.u16 r2, q0[6]
+; CHECK-NEXT:    add r0, r2
+; CHECK-NEXT:    vmov.u16 r2, q0[5]
+; CHECK-NEXT:    vmov.u16 r3, q0[4]
+; CHECK-NEXT:    add r2, r3
+; CHECK-NEXT:    vmov.u16 r3, q0[0]
+; CHECK-NEXT:    add r0, r2
+; CHECK-NEXT:    strb r0, [r1, #1]
+; CHECK-NEXT:    vmov.u16 r0, q0[3]
+; CHECK-NEXT:    vmov.u16 r2, q0[2]
+; CHECK-NEXT:    add r0, r2
+; CHECK-NEXT:    vmov.u16 r2, q0[1]
+; CHECK-NEXT:    add r2, r3
+; CHECK-NEXT:    add r0, r2
+; CHECK-NEXT:    strb r0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %l1 = load <8 x i8>, <8 x i8>* %src, align 4
+  %s1 = shufflevector <8 x i8> %l1, <8 x i8> undef, <2 x i32> <i32 0, i32 4>
+  %s2 = shufflevector <8 x i8> %l1, <8 x i8> undef, <2 x i32> <i32 1, i32 5>
+  %s3 = shufflevector <8 x i8> %l1, <8 x i8> undef, <2 x i32> <i32 2, i32 6>
+  %s4 = shufflevector <8 x i8> %l1, <8 x i8> undef, <2 x i32> <i32 3, i32 7>
+  %a1 = add <2 x i8> %s1, %s2
+  %a2 = add <2 x i8> %s3, %s4
+  %a3 = add <2 x i8> %a1, %a2
+  store <2 x i8> %a3, <2 x i8> *%dst
+  ret void
+}
+
+define void @vld4_v4i8(<16 x i8> *%src, <4 x i8> *%dst) {
+; CHECK-LABEL: vld4_v4i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vmov.u8 r0, q0[2]
+; CHECK-NEXT:    vrev32.8 q2, q0
+; CHECK-NEXT:    vmov.32 q1[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[6]
+; CHECK-NEXT:    vmov.32 q1[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[10]
+; CHECK-NEXT:    vmov.32 q1[2], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[14]
+; CHECK-NEXT:    vmov.32 q1[3], r0
+; CHECK-NEXT:    vadd.i32 q1, q1, q2
+; CHECK-NEXT:    vrev16.8 q2, q0
+; CHECK-NEXT:    vadd.i32 q0, q0, q2
+; CHECK-NEXT:    vadd.i32 q0, q0, q1
+; CHECK-NEXT:    vstrb.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %l1 = load <16 x i8>, <16 x i8>* %src, align 4
+  %s1 = shufflevector <16 x i8> %l1, <16 x i8> undef, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+  %s2 = shufflevector <16 x i8> %l1, <16 x i8> undef, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
+  %s3 = shufflevector <16 x i8> %l1, <16 x i8> undef, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
+  %s4 = shufflevector <16 x i8> %l1, <16 x i8> undef, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
+  %a1 = add <4 x i8> %s1, %s2
+  %a2 = add <4 x i8> %s3, %s4
+  %a3 = add <4 x i8> %a1, %a2
+  store <4 x i8> %a3, <4 x i8> *%dst
+  ret void
+}
+
+define void @vld4_v8i8(<32 x i8> *%src, <8 x i8> *%dst) {
+; CHECK-LABEL: vld4_v8i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9}
+; CHECK-NEXT:    vpush {d8, d9}
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #16]
+; CHECK-NEXT:    vmov.u8 r2, q1[3]
+; CHECK-NEXT:    vmov.u8 r0, q0[3]
+; CHECK-NEXT:    vmov.16 q2[0], r2
+; CHECK-NEXT:    vmov.u8 r2, q1[7]
+; CHECK-NEXT:    vmov.16 q2[1], r2
+; CHECK-NEXT:    vmov.u8 r2, q1[11]
+; CHECK-NEXT:    vmov.16 q2[2], r2
+; CHECK-NEXT:    vmov.u8 r2, q1[15]
+; CHECK-NEXT:    vmov.16 q2[3], r2
+; CHECK-NEXT:    vmov.16 q2[4], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[7]
+; CHECK-NEXT:    vmov.16 q2[5], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[11]
+; CHECK-NEXT:    vmov.16 q2[6], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[15]
+; CHECK-NEXT:    vmov.16 q2[7], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[2]
+; CHECK-NEXT:    vmov.16 q3[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[6]
+; CHECK-NEXT:    vmov.16 q3[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[10]
+; CHECK-NEXT:    vmov.16 q3[2], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[14]
+; CHECK-NEXT:    vmov.16 q3[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[2]
+; CHECK-NEXT:    vmov.16 q3[4], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[6]
+; CHECK-NEXT:    vmov.16 q3[5], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[10]
+; CHECK-NEXT:    vmov.16 q3[6], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[14]
+; CHECK-NEXT:    vmov.16 q3[7], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[0]
+; CHECK-NEXT:    vadd.i16 q2, q3, q2
+; CHECK-NEXT:    vmov.16 q3[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[4]
+; CHECK-NEXT:    vmov.16 q3[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[8]
+; CHECK-NEXT:    vmov.16 q3[2], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[12]
+; CHECK-NEXT:    vmov.16 q3[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[0]
+; CHECK-NEXT:    vmov.16 q3[4], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[4]
+; CHECK-NEXT:    vmov.16 q3[5], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[8]
+; CHECK-NEXT:    vmov.16 q3[6], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[1]
+; CHECK-NEXT:    vmov.16 q4[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[5]
+; CHECK-NEXT:    vmov.16 q4[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[9]
+; CHECK-NEXT:    vmov.16 q4[2], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[13]
+; CHECK-NEXT:    vmov.16 q4[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[1]
+; CHECK-NEXT:    vmov.16 q4[4], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[5]
+; CHECK-NEXT:    vmov.16 q4[5], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[9]
+; CHECK-NEXT:    vmov.16 q4[6], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[13]
+; CHECK-NEXT:    vmov.16 q4[7], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[12]
+; CHECK-NEXT:    vmov.16 q3[7], r0
+; CHECK-NEXT:    vadd.i16 q0, q3, q4
+; CHECK-NEXT:    vadd.i16 q0, q0, q2
+; CHECK-NEXT:    vstrb.16 q0, [r1]
+; CHECK-NEXT:    vpop {d8, d9}
+; CHECK-NEXT:    bx lr
+entry:
+  %l1 = load <32 x i8>, <32 x i8>* %src, align 4
+  %s1 = shufflevector <32 x i8> %l1, <32 x i8> undef, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
+  %s2 = shufflevector <32 x i8> %l1, <32 x i8> undef, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>
+  %s3 = shufflevector <32 x i8> %l1, <32 x i8> undef, <8 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30>
+  %s4 = shufflevector <32 x i8> %l1, <32 x i8> undef, <8 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31>
+  %a1 = add <8 x i8> %s1, %s2
+  %a2 = add <8 x i8> %s3, %s4
+  %a3 = add <8 x i8> %a1, %a2
+  store <8 x i8> %a3, <8 x i8> *%dst
+  ret void
+}
+
+define void @vld4_v16i8(<64 x i8> *%src, <16 x i8> *%dst) {
+; CHECK-LABEL: vld4_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9}
+; CHECK-NEXT:    vpush {d8, d9}
+; CHECK-NEXT:    vld40.8 {q0, q1, q2, q3}, [r0]
+; CHECK-NEXT:    vld41.8 {q0, q1, q2, q3}, [r0]
+; CHECK-NEXT:    vld42.8 {q0, q1, q2, q3}, [r0]
+; CHECK-NEXT:    vld43.8 {q0, q1, q2, q3}, [r0]
+; CHECK-NEXT:    @ kill: def $q0 killed $q0 killed $q0_q1_q2_q3
+; CHECK-NEXT:    vadd.i8 q4, q2, q3
+; CHECK-NEXT:    vadd.i8 q0, q0, q1
+; CHECK-NEXT:    vadd.i8 q0, q0, q4
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vpop {d8, d9}
+; CHECK-NEXT:    bx lr
+entry:
+  %l1 = load <64 x i8>, <64 x i8>* %src, align 4
+  %s1 = shufflevector <64 x i8> %l1, <64 x i8> undef, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28, i32 32, i32 36, i32 40, i32 44, i32 48, i32 52, i32 56, i32 60>
+  %s2 = shufflevector <64 x i8> %l1, <64 x i8> undef, <16 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29, i32 33, i32 37, i32 41, i32 45, i32 49, i32 53, i32 57, i32 61>
+  %s3 = shufflevector <64 x i8> %l1, <64 x i8> undef, <16 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30, i32 34, i32 38, i32 42, i32 46, i32 50, i32 54, i32 58, i32 62>
+  %s4 = shufflevector <64 x i8> %l1, <64 x i8> undef, <16 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31, i32 35, i32 39, i32 43, i32 47, i32 51, i32 55, i32 59, i32 63>
+  %a1 = add <16 x i8> %s1, %s2
+  %a2 = add <16 x i8> %s3, %s4
+  %a3 = add <16 x i8> %a1, %a2
+  store <16 x i8> %a3, <16 x i8> *%dst
+  ret void
+}
+
+; i64
+
+define void @vld4_v2i64(<8 x i64> *%src, <2 x i64> *%dst) {
+; CHECK-LABEL: vld4_v2i64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, r5, r6, lr}
+; CHECK-NEXT:    push {r4, r5, r6, lr}
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11}
+; CHECK-NEXT:    vldrw.u32 q3, [r0, #16]
+; CHECK-NEXT:    vldrw.u32 q5, [r0, #48]
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #32]
+; CHECK-NEXT:    vmov.f64 d8, d7
+; CHECK-NEXT:    vmov.f32 s17, s15
+; CHECK-NEXT:    vmov.f32 s18, s22
+; CHECK-NEXT:    vmov.f32 s14, s20
+; CHECK-NEXT:    vmov.f32 s19, s23
+; CHECK-NEXT:    vmov.f32 s15, s21
+; CHECK-NEXT:    vmov r3, s18
+; CHECK-NEXT:    vmov r0, s14
+; CHECK-NEXT:    vmov.f64 d2, d1
+; CHECK-NEXT:    vmov r12, s19
+; CHECK-NEXT:    vmov r2, s15
+; CHECK-NEXT:    vmov.f32 s5, s3
+; CHECK-NEXT:    vmov.f32 s6, s10
+; CHECK-NEXT:    vmov.f32 s2, s8
+; CHECK-NEXT:    vmov.f32 s3, s9
+; CHECK-NEXT:    vmov.f32 s7, s11
+; CHECK-NEXT:    vmov r4, s2
+; CHECK-NEXT:    vmov r5, s4
+; CHECK-NEXT:    vmov r6, s0
+; CHECK-NEXT:    adds.w lr, r0, r3
+; CHECK-NEXT:    vmov r3, s7
+; CHECK-NEXT:    vmov r0, s3
+; CHECK-NEXT:    adc.w r12, r12, r2
+; CHECK-NEXT:    vmov r2, s6
+; CHECK-NEXT:    adds r2, r2, r4
+; CHECK-NEXT:    vmov r4, s13
+; CHECK-NEXT:    adcs r0, r3
+; CHECK-NEXT:    adds.w lr, lr, r2
+; CHECK-NEXT:    adc.w r12, r12, r0
+; CHECK-NEXT:    vmov r0, s16
+; CHECK-NEXT:    vmov r2, s12
+; CHECK-NEXT:    vmov r3, s17
+; CHECK-NEXT:    adds r0, r0, r2
+; CHECK-NEXT:    adc.w r2, r4, r3
+; CHECK-NEXT:    vmov r3, s5
+; CHECK-NEXT:    vmov r4, s1
+; CHECK-NEXT:    adds r5, r5, r6
+; CHECK-NEXT:    adcs r3, r4
+; CHECK-NEXT:    adds r0, r0, r5
+; CHECK-NEXT:    adcs r2, r3
+; CHECK-NEXT:    vmov.32 q0[0], r0
+; CHECK-NEXT:    vmov.32 q0[1], r2
+; CHECK-NEXT:    vmov.32 q0[2], lr
+; CHECK-NEXT:    vmov.32 q0[3], r12
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vpop {d8, d9, d10, d11}
+; CHECK-NEXT:    pop {r4, r5, r6, pc}
+entry:
+  %l1 = load <8 x i64>, <8 x i64>* %src, align 4
+  %s1 = shufflevector <8 x i64> %l1, <8 x i64> undef, <2 x i32> <i32 0, i32 4>
+  %s2 = shufflevector <8 x i64> %l1, <8 x i64> undef, <2 x i32> <i32 1, i32 5>
+  %s3 = shufflevector <8 x i64> %l1, <8 x i64> undef, <2 x i32> <i32 2, i32 6>
+  %s4 = shufflevector <8 x i64> %l1, <8 x i64> undef, <2 x i32> <i32 3, i32 7>
+  %a1 = add <2 x i64> %s1, %s2
+  %a2 = add <2 x i64> %s3, %s4
+  %a3 = add <2 x i64> %a1, %a2
+  store <2 x i64> %a3, <2 x i64> *%dst
+  ret void
+}
+
+define void @vld4_v4i64(<16 x i64> *%src, <4 x i64> *%dst) {
+; CHECK-LABEL: vld4_v4i64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, lr}
+; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, lr}
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    .pad #72
+; CHECK-NEXT:    sub sp, #72
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #64]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #96]
+; CHECK-NEXT:    vldrw.u32 q5, [r0, #80]
+; CHECK-NEXT:    vldrw.u32 q7, [r0, #16]
+; CHECK-NEXT:    vmov.f64 d8, d3
+; CHECK-NEXT:    vstrw.32 q0, [sp, #32] @ 16-byte Spill
+; CHECK-NEXT:    vstrw.32 q1, [sp, #48] @ 16-byte Spill
+; CHECK-NEXT:    vmov.f32 s17, s7
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vmov.f32 s18, s2
+; CHECK-NEXT:    vmov.f32 s19, s3
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #112]
+; CHECK-NEXT:    vmov.f64 d12, d11
+; CHECK-NEXT:    vstrw.32 q0, [sp, #16] @ 16-byte Spill
+; CHECK-NEXT:    vmov.f32 s25, s23
+; CHECK-NEXT:    vmov.f32 s26, s2
+; CHECK-NEXT:    vmov.f64 d6, d3
+; CHECK-NEXT:    vmov.f32 s27, s3
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #32]
+; CHECK-NEXT:    vmov.f32 s13, s7
+; CHECK-NEXT:    vmov.f32 s14, s2
+; CHECK-NEXT:    vstrw.32 q0, [sp] @ 16-byte Spill
+; CHECK-NEXT:    vmov.f64 d4, d15
+; CHECK-NEXT:    vmov.f32 s15, s3
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #48]
+; CHECK-NEXT:    vmov.f32 s9, s31
+; CHECK-NEXT:    vmov.f32 s10, s2
+; CHECK-NEXT:    vmov.f32 s30, s0
+; CHECK-NEXT:    vmov.f32 s11, s3
+; CHECK-NEXT:    vmov.f32 s31, s1
+; CHECK-NEXT:    vldrw.u32 q0, [sp] @ 16-byte Reload
+; CHECK-NEXT:    vmov r3, s10
+; CHECK-NEXT:    vmov r0, s30
+; CHECK-NEXT:    vmov.f32 s6, s0
+; CHECK-NEXT:    vmov.f32 s7, s1
+; CHECK-NEXT:    vldrw.u32 q0, [sp, #16] @ 16-byte Reload
+; CHECK-NEXT:    vmov r4, s6
+; CHECK-NEXT:    vstrw.32 q1, [sp] @ 16-byte Spill
+; CHECK-NEXT:    vmov r12, s11
+; CHECK-NEXT:    vmov r2, s31
+; CHECK-NEXT:    vmov.f32 s22, s0
+; CHECK-NEXT:    vmov.f32 s23, s1
+; CHECK-NEXT:    vldrw.u32 q0, [sp, #48] @ 16-byte Reload
+; CHECK-NEXT:    vmov r5, s18
+; CHECK-NEXT:    vmov r7, s16
+; CHECK-NEXT:    adds.w lr, r0, r3
+; CHECK-NEXT:    vmov r3, s14
+; CHECK-NEXT:    vmov r0, s7
+; CHECK-NEXT:    vldrw.u32 q1, [sp, #32] @ 16-byte Reload
+; CHECK-NEXT:    vmov.f32 s2, s4
+; CHECK-NEXT:    vmov.f32 s3, s5
+; CHECK-NEXT:    vldrw.u32 q1, [sp] @ 16-byte Reload
+; CHECK-NEXT:    adc.w r12, r12, r2
+; CHECK-NEXT:    vmov r2, s15
+; CHECK-NEXT:    vmov r6, s2
+; CHECK-NEXT:    adds r3, r3, r4
+; CHECK-NEXT:    vmov r4, s23
+; CHECK-NEXT:    adcs r0, r2
+; CHECK-NEXT:    adds.w lr, lr, r3
+; CHECK-NEXT:    adc.w r12, r12, r0
+; CHECK-NEXT:    vmov r0, s26
+; CHECK-NEXT:    vmov r2, s22
+; CHECK-NEXT:    vmov r3, s27
+; CHECK-NEXT:    adds r0, r0, r2
+; CHECK-NEXT:    adc.w r2, r4, r3
+; CHECK-NEXT:    vmov r3, s19
+; CHECK-NEXT:    vmov r4, s3
+; CHECK-NEXT:    adds r5, r5, r6
+; CHECK-NEXT:    vmov r6, s20
+; CHECK-NEXT:    adcs r3, r4
+; CHECK-NEXT:    adds r0, r0, r5
+; CHECK-NEXT:    vmov r5, s24
+; CHECK-NEXT:    adc.w r8, r3, r2
+; CHECK-NEXT:    vmov r2, s25
+; CHECK-NEXT:    vmov r4, s21
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    adds r5, r5, r6
+; CHECK-NEXT:    vmov r6, s1
+; CHECK-NEXT:    adcs r2, r4
+; CHECK-NEXT:    vmov r4, s17
+; CHECK-NEXT:    adds r3, r3, r7
+; CHECK-NEXT:    vmov r7, s28
+; CHECK-NEXT:    adcs r4, r6
+; CHECK-NEXT:    adds r3, r3, r5
+; CHECK-NEXT:    vmov r6, s8
+; CHECK-NEXT:    adcs r2, r4
+; CHECK-NEXT:    vmov r4, s9
+; CHECK-NEXT:    vmov.32 q0[0], r3
+; CHECK-NEXT:    vmov r5, s29
+; CHECK-NEXT:    vmov.32 q0[1], r2
+; CHECK-NEXT:    vmov.32 q0[2], r0
+; CHECK-NEXT:    vmov r0, s12
+; CHECK-NEXT:    vmov r3, s13
+; CHECK-NEXT:    vmov.32 q0[3], r8
+; CHECK-NEXT:    vmov r2, s5
+; CHECK-NEXT:    vstrw.32 q0, [r1, #16]
+; CHECK-NEXT:    adds r6, r6, r7
+; CHECK-NEXT:    adcs r4, r5
+; CHECK-NEXT:    vmov r5, s4
+; CHECK-NEXT:    adds r0, r0, r5
+; CHECK-NEXT:    adcs r2, r3
+; CHECK-NEXT:    adds r0, r0, r6
+; CHECK-NEXT:    adcs r2, r4
+; CHECK-NEXT:    vmov.32 q0[0], r0
+; CHECK-NEXT:    vmov.32 q0[1], r2
+; CHECK-NEXT:    vmov.32 q0[2], lr
+; CHECK-NEXT:    vmov.32 q0[3], r12
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    add sp, #72
+; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, pc}
+entry:
+  %l1 = load <16 x i64>, <16 x i64>* %src, align 4
+  %s1 = shufflevector <16 x i64> %l1, <16 x i64> undef, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+  %s2 = shufflevector <16 x i64> %l1, <16 x i64> undef, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
+  %s3 = shufflevector <16 x i64> %l1, <16 x i64> undef, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
+  %s4 = shufflevector <16 x i64> %l1, <16 x i64> undef, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
+  %a1 = add <4 x i64> %s1, %s2
+  %a2 = add <4 x i64> %s3, %s4
+  %a3 = add <4 x i64> %a1, %a2
+  store <4 x i64> %a3, <4 x i64> *%dst
+  ret void
+}
+
+; f32
+
+define void @vld4_v2f32(<8 x float> *%src, <2 x float> *%dst) {
+; CHECK-LABEL: vld4_v2f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #16]
+; CHECK-NEXT:    vmov.f32 s8, s7
+; CHECK-NEXT:    vmov.f64 d6, d3
+; CHECK-NEXT:    vmov.f32 s9, s3
+; CHECK-NEXT:    vmov.f32 s13, s2
+; CHECK-NEXT:    vadd.f32 q2, q3, q2
+; CHECK-NEXT:    vmov.f32 s12, s5
+; CHECK-NEXT:    vmov.f32 s13, s1
+; CHECK-NEXT:    vmov.f32 s5, s0
+; CHECK-NEXT:    vadd.f32 q0, q1, q3
+; CHECK-NEXT:    vadd.f32 q0, q0, q2
+; CHECK-NEXT:    vstmia r1, {s0, s1}
+; CHECK-NEXT:    bx lr
+entry:
+  %l1 = load <8 x float>, <8 x float>* %src, align 4
+  %s1 = shufflevector <8 x float> %l1, <8 x float> undef, <2 x i32> <i32 0, i32 4>
+  %s2 = shufflevector <8 x float> %l1, <8 x float> undef, <2 x i32> <i32 1, i32 5>
+  %s3 = shufflevector <8 x float> %l1, <8 x float> undef, <2 x i32> <i32 2, i32 6>
+  %s4 = shufflevector <8 x float> %l1, <8 x float> undef, <2 x i32> <i32 3, i32 7>
+  %a1 = fadd <2 x float> %s1, %s2
+  %a2 = fadd <2 x float> %s3, %s4
+  %a3 = fadd <2 x float> %a1, %a2
+  store <2 x float> %a3, <2 x float> *%dst
+  ret void
+}
+
+define void @vld4_v4f32(<16 x float> *%src, <4 x float> *%dst) {
+; CHECK-LABEL: vld4_v4f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9}
+; CHECK-NEXT:    vpush {d8, d9}
+; CHECK-NEXT:    vld40.32 {q0, q1, q2, q3}, [r0]
+; CHECK-NEXT:    vld41.32 {q0, q1, q2, q3}, [r0]
+; CHECK-NEXT:    vld42.32 {q0, q1, q2, q3}, [r0]
+; CHECK-NEXT:    vld43.32 {q0, q1, q2, q3}, [r0]
+; CHECK-NEXT:    @ kill: def $q0 killed $q0 killed $q0_q1_q2_q3
+; CHECK-NEXT:    vadd.f32 q4, q2, q3
+; CHECK-NEXT:    vadd.f32 q0, q0, q1
+; CHECK-NEXT:    vadd.f32 q0, q0, q4
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vpop {d8, d9}
+; CHECK-NEXT:    bx lr
+entry:
+  %l1 = load <16 x float>, <16 x float>* %src, align 4
+  %s1 = shufflevector <16 x float> %l1, <16 x float> undef, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+  %s2 = shufflevector <16 x float> %l1, <16 x float> undef, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
+  %s3 = shufflevector <16 x float> %l1, <16 x float> undef, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
+  %s4 = shufflevector <16 x float> %l1, <16 x float> undef, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
+  %a1 = fadd <4 x float> %s1, %s2
+  %a2 = fadd <4 x float> %s3, %s4
+  %a3 = fadd <4 x float> %a1, %a2
+  store <4 x float> %a3, <4 x float> *%dst
+  ret void
+}
+
+define void @vld4_v8f32(<32 x float> *%src, <8 x float> *%dst) {
+; CHECK-LABEL: vld4_v8f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, r5}
+; CHECK-NEXT:    push {r4, r5}
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    .pad #88
+; CHECK-NEXT:    sub sp, #88
+; CHECK-NEXT:    add.w r2, r0, #64
+; CHECK-NEXT:    vld40.32 {q0, q1, q2, q3}, [r2]
+; CHECK-NEXT:    vld41.32 {q0, q1, q2, q3}, [r2]
+; CHECK-NEXT:    vstmia sp, {d0, d1, d2, d3, d4, d5, d6, d7} @ 64-byte Spill
+; CHECK-NEXT:    vld40.32 {q4, q5, q6, q7}, [r0]
+; CHECK-NEXT:    vld41.32 {q4, q5, q6, q7}, [r0]
+; CHECK-NEXT:    vld42.32 {q4, q5, q6, q7}, [r0]
+; CHECK-NEXT:    vld43.32 {q4, q5, q6, q7}, [r0]
+; CHECK-NEXT:    vstrw.32 q5, [sp, #64] @ 16-byte Spill
+; CHECK-NEXT:    vmov q1, q4
+; CHECK-NEXT:    vldrw.u32 q0, [sp, #64] @ 16-byte Reload
+; CHECK-NEXT:    vadd.f32 q4, q6, q7
+; CHECK-NEXT:    vadd.f32 q5, q1, q0
+; CHECK-NEXT:    vldmia sp, {d0, d1, d2, d3, d4, d5, d6, d7} @ 64-byte Reload
+; CHECK-NEXT:    vadd.f32 q4, q5, q4
+; CHECK-NEXT:    vld42.32 {q0, q1, q2, q3}, [r2]
+; CHECK-NEXT:    vld43.32 {q0, q1, q2, q3}, [r2]
+; CHECK-NEXT:    vstrw.32 q4, [r1]
+; CHECK-NEXT:    @ kill: def $q0 killed $q0 killed $q0_q1_q2_q3
+; CHECK-NEXT:    vadd.f32 q5, q2, q3
+; CHECK-NEXT:    vadd.f32 q0, q0, q1
+; CHECK-NEXT:    vadd.f32 q0, q0, q5
+; CHECK-NEXT:    vstrw.32 q0, [r1, #16]
+; CHECK-NEXT:    add sp, #88
+; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    pop {r4, r5}
+; CHECK-NEXT:    bx lr
+entry:
+  %l1 = load <32 x float>, <32 x float>* %src, align 4
+  %s1 = shufflevector <32 x float> %l1, <32 x float> undef, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
+  %s2 = shufflevector <32 x float> %l1, <32 x float> undef, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>
+  %s3 = shufflevector <32 x float> %l1, <32 x float> undef, <8 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30>
+  %s4 = shufflevector <32 x float> %l1, <32 x float> undef, <8 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31>
+  %a1 = fadd <8 x float> %s1, %s2
+  %a2 = fadd <8 x float> %s3, %s4
+  %a3 = fadd <8 x float> %a1, %a2
+  store <8 x float> %a3, <8 x float> *%dst
+  ret void
+}
+
+define void @vld4_v16f32(<64 x float> *%src, <16 x float> *%dst) {
+; CHECK-LABEL: vld4_v16f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, r5}
+; CHECK-NEXT:    push {r4, r5}
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    .pad #152
+; CHECK-NEXT:    sub sp, #152
+; CHECK-NEXT:    add.w r2, r0, #128
+; CHECK-NEXT:    add r3, sp, #64
+; CHECK-NEXT:    vld40.32 {q0, q1, q2, q3}, [r2]
+; CHECK-NEXT:    add r4, sp, #64
+; CHECK-NEXT:    vld41.32 {q0, q1, q2, q3}, [r2]
+; CHECK-NEXT:    vstmia r3, {d0, d1, d2, d3, d4, d5, d6, d7} @ 64-byte Spill
+; CHECK-NEXT:    add.w r3, r0, #64
+; CHECK-NEXT:    vld40.32 {q0, q1, q2, q3}, [r0]
+; CHECK-NEXT:    vld41.32 {q0, q1, q2, q3}, [r0]
+; CHECK-NEXT:    vld42.32 {q0, q1, q2, q3}, [r0]
+; CHECK-NEXT:    vld43.32 {q0, q1, q2, q3}, [r0]
+; CHECK-NEXT:    adds r0, #192
+; CHECK-NEXT:    vstrw.32 q1, [sp, #32] @ 16-byte Spill
+; CHECK-NEXT:    vadd.f32 q4, q2, q3
+; CHECK-NEXT:    vmov q5, q0
+; CHECK-NEXT:    vldrw.u32 q0, [sp, #32] @ 16-byte Reload
+; CHECK-NEXT:    vstrw.32 q4, [sp, #48] @ 16-byte Spill
+; CHECK-NEXT:    vadd.f32 q4, q5, q0
+; CHECK-NEXT:    vldmia r4, {d0, d1, d2, d3, d4, d5, d6, d7} @ 64-byte Reload
+; CHECK-NEXT:    vstrw.32 q4, [sp, #32] @ 16-byte Spill
+; CHECK-NEXT:    add r4, sp, #64
+; CHECK-NEXT:    vld42.32 {q0, q1, q2, q3}, [r2]
+; CHECK-NEXT:    vstmia r4, {d0, d1, d2, d3, d4, d5, d6, d7} @ 64-byte Spill
+; CHECK-NEXT:    vld40.32 {q4, q5, q6, q7}, [r3]
+; CHECK-NEXT:    vld41.32 {q4, q5, q6, q7}, [r3]
+; CHECK-NEXT:    vld42.32 {q4, q5, q6, q7}, [r3]
+; CHECK-NEXT:    vld43.32 {q4, q5, q6, q7}, [r3]
+; CHECK-NEXT:    vldrw.u32 q0, [sp, #48] @ 16-byte Reload
+; CHECK-NEXT:    vldrw.u32 q1, [sp, #32] @ 16-byte Reload
+; CHECK-NEXT:    add r3, sp, #64
+; CHECK-NEXT:    vstrw.32 q6, [sp, #16] @ 16-byte Spill
+; CHECK-NEXT:    vadd.f32 q4, q4, q5
+; CHECK-NEXT:    vadd.f32 q0, q1, q0
+; CHECK-NEXT:    vstrw.32 q0, [sp, #48] @ 16-byte Spill
+; CHECK-NEXT:    vldrw.u32 q0, [sp, #16] @ 16-byte Reload
+; CHECK-NEXT:    vadd.f32 q0, q0, q7
+; CHECK-NEXT:    vadd.f32 q0, q4, q0
+; CHECK-NEXT:    vstrw.32 q0, [sp, #32] @ 16-byte Spill
+; CHECK-NEXT:    vldmia r3, {d0, d1, d2, d3, d4, d5, d6, d7} @ 64-byte Reload
+; CHECK-NEXT:    vld43.32 {q0, q1, q2, q3}, [r2]
+; CHECK-NEXT:    add r2, sp, #64
+; CHECK-NEXT:    vstmia r2, {d0, d1, d2, d3, d4, d5, d6, d7} @ 64-byte Spill
+; CHECK-NEXT:    vld40.32 {q4, q5, q6, q7}, [r0]
+; CHECK-NEXT:    vld41.32 {q4, q5, q6, q7}, [r0]
+; CHECK-NEXT:    vld42.32 {q4, q5, q6, q7}, [r0]
+; CHECK-NEXT:    vld43.32 {q4, q5, q6, q7}, [r0]
+; CHECK-NEXT:    add r0, sp, #64
+; CHECK-NEXT:    @ kill: def $q4 killed $q4 killed $q4_q5_q6_q7
+; CHECK-NEXT:    vstrw.32 q7, [sp, #16] @ 16-byte Spill
+; CHECK-NEXT:    vmov q2, q5
+; CHECK-NEXT:    vldrw.u32 q0, [sp, #16] @ 16-byte Reload
+; CHECK-NEXT:    vadd.f32 q4, q4, q2
+; CHECK-NEXT:    vadd.f32 q5, q6, q0
+; CHECK-NEXT:    vldmia r0, {d0, d1, d2, d3, d4, d5, d6, d7} @ 64-byte Reload
+; CHECK-NEXT:    @ kill: def $q0 killed $q0 killed $q0_q1_q2_q3
+; CHECK-NEXT:    vadd.f32 q4, q4, q5
+; CHECK-NEXT:    vadd.f32 q5, q2, q3
+; CHECK-NEXT:    vadd.f32 q0, q0, q1
+; CHECK-NEXT:    vstrw.32 q4, [r1, #48]
+; CHECK-NEXT:    vadd.f32 q0, q0, q5
+; CHECK-NEXT:    vstrw.32 q0, [r1, #32]
+; CHECK-NEXT:    vldrw.u32 q0, [sp, #32] @ 16-byte Reload
+; CHECK-NEXT:    vstrw.32 q0, [r1, #16]
+; CHECK-NEXT:    vldrw.u32 q0, [sp, #48] @ 16-byte Reload
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    add sp, #152
+; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    pop {r4, r5}
+; CHECK-NEXT:    bx lr
+entry:
+  %l1 = load <64 x float>, <64 x float>* %src, align 4
+  %s1 = shufflevector <64 x float> %l1, <64 x float> undef, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28, i32 32, i32 36, i32 40, i32 44, i32 48, i32 52, i32 56, i32 60>
+  %s2 = shufflevector <64 x float> %l1, <64 x float> undef, <16 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29, i32 33, i32 37, i32 41, i32 45, i32 49, i32 53, i32 57, i32 61>
+  %s3 = shufflevector <64 x float> %l1, <64 x float> undef, <16 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30, i32 34, i32 38, i32 42, i32 46, i32 50, i32 54, i32 58, i32 62>
+  %s4 = shufflevector <64 x float> %l1, <64 x float> undef, <16 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31, i32 35, i32 39, i32 43, i32 47, i32 51, i32 55, i32 59, i32 63>
+  %a1 = fadd <16 x float> %s1, %s2
+  %a2 = fadd <16 x float> %s3, %s4
+  %a3 = fadd <16 x float> %a1, %a2
+  store <16 x float> %a3, <16 x float> *%dst
+  ret void
+}
+
+; f16
+
+define void @vld4_v2f16(<8 x half> *%src, <2 x half> *%dst) {
+; CHECK-LABEL: vld4_v2f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vmovx.f16 s4, s3
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmovx.f16 s4, s1
+; CHECK-NEXT:    vmov r2, s4
+; CHECK-NEXT:    vmov.16 q1[0], r2
+; CHECK-NEXT:    vmov r2, s1
+; CHECK-NEXT:    vmov.16 q1[1], r0
+; CHECK-NEXT:    vmov r0, s3
+; CHECK-NEXT:    vmov.16 q2[0], r2
+; CHECK-NEXT:    vmov.16 q2[1], r0
+; CHECK-NEXT:    vadd.f16 q1, q2, q1
+; CHECK-NEXT:    vmovx.f16 s8, s0
+; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    vmovx.f16 s8, s2
+; CHECK-NEXT:    vmov r2, s8
+; CHECK-NEXT:    vmov.16 q2[0], r0
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vmov.16 q2[1], r2
+; CHECK-NEXT:    vmov r2, s2
+; CHECK-NEXT:    vmov.16 q0[0], r0
+; CHECK-NEXT:    vmov.16 q0[1], r2
+; CHECK-NEXT:    vadd.f16 q0, q0, q2
+; CHECK-NEXT:    vadd.f16 q0, q0, q1
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    str r0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %l1 = load <8 x half>, <8 x half>* %src, align 4
+  %s1 = shufflevector <8 x half> %l1, <8 x half> undef, <2 x i32> <i32 0, i32 4>
+  %s2 = shufflevector <8 x half> %l1, <8 x half> undef, <2 x i32> <i32 1, i32 5>
+  %s3 = shufflevector <8 x half> %l1, <8 x half> undef, <2 x i32> <i32 2, i32 6>
+  %s4 = shufflevector <8 x half> %l1, <8 x half> undef, <2 x i32> <i32 3, i32 7>
+  %a1 = fadd <2 x half> %s1, %s2
+  %a2 = fadd <2 x half> %s3, %s4
+  %a3 = fadd <2 x half> %a1, %a2
+  store <2 x half> %a3, <2 x half> *%dst
+  ret void
+}
+
+define void @vld4_v4f16(<16 x half> *%src, <4 x half> *%dst) {
+; CHECK-LABEL: vld4_v4f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8}
+; CHECK-NEXT:    vpush {d8}
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #16]
+; CHECK-NEXT:    vmov r2, s5
+; CHECK-NEXT:    vmovx.f16 s12, s5
+; CHECK-NEXT:    vmov r3, s7
+; CHECK-NEXT:    vmov.16 q2[0], r2
+; CHECK-NEXT:    vmov.16 q2[1], r3
+; CHECK-NEXT:    vmov r0, s1
+; CHECK-NEXT:    vmov.16 q2[2], r0
+; CHECK-NEXT:    vmov r0, s12
+; CHECK-NEXT:    vmovx.f16 s12, s7
+; CHECK-NEXT:    vmovx.f16 s16, s1
+; CHECK-NEXT:    vmov r2, s12
+; CHECK-NEXT:    vmov.16 q3[0], r0
+; CHECK-NEXT:    vmov r0, s16
+; CHECK-NEXT:    vmov.16 q3[1], r2
+; CHECK-NEXT:    vmovx.f16 s16, s3
+; CHECK-NEXT:    vmov.16 q3[2], r0
+; CHECK-NEXT:    vmov r0, s16
+; CHECK-NEXT:    vmovx.f16 s16, s0
+; CHECK-NEXT:    vmov.16 q3[3], r0
+; CHECK-NEXT:    vmov r0, s3
+; CHECK-NEXT:    vmov.16 q2[3], r0
+; CHECK-NEXT:    vadd.f16 q2, q2, q3
+; CHECK-NEXT:    vmovx.f16 s12, s4
+; CHECK-NEXT:    vmov r0, s12
+; CHECK-NEXT:    vmovx.f16 s12, s6
+; CHECK-NEXT:    vmov r2, s12
+; CHECK-NEXT:    vmov.16 q3[0], r0
+; CHECK-NEXT:    vmov.16 q3[1], r2
+; CHECK-NEXT:    vmov r0, s16
+; CHECK-NEXT:    vmovx.f16 s16, s2
+; CHECK-NEXT:    vmov.16 q3[2], r0
+; CHECK-NEXT:    vmov r0, s16
+; CHECK-NEXT:    vmov.16 q3[3], r0
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmov r2, s6
+; CHECK-NEXT:    vmov.16 q1[0], r0
+; CHECK-NEXT:    vmov.16 q1[1], r2
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vmov.16 q1[2], r0
+; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    vmov.16 q1[3], r0
+; CHECK-NEXT:    vadd.f16 q0, q1, q3
+; CHECK-NEXT:    vadd.f16 q0, q0, q2
+; CHECK-NEXT:    vmov r2, s1
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    strd r0, r2, [r1]
+; CHECK-NEXT:    vpop {d8}
+; CHECK-NEXT:    bx lr
+entry:
+  %l1 = load <16 x half>, <16 x half>* %src, align 4
+  %s1 = shufflevector <16 x half> %l1, <16 x half> undef, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+  %s2 = shufflevector <16 x half> %l1, <16 x half> undef, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
+  %s3 = shufflevector <16 x half> %l1, <16 x half> undef, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
+  %s4 = shufflevector <16 x half> %l1, <16 x half> undef, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
+  %a1 = fadd <4 x half> %s1, %s2
+  %a2 = fadd <4 x half> %s3, %s4
+  %a3 = fadd <4 x half> %a1, %a2
+  store <4 x half> %a3, <4 x half> *%dst
+  ret void
+}
+
+define void @vld4_v8f16(<32 x half> *%src, <8 x half> *%dst) {
+; CHECK-LABEL: vld4_v8f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9}
+; CHECK-NEXT:    vpush {d8, d9}
+; CHECK-NEXT:    vld40.16 {q0, q1, q2, q3}, [r0]
+; CHECK-NEXT:    vld41.16 {q0, q1, q2, q3}, [r0]
+; CHECK-NEXT:    vld42.16 {q0, q1, q2, q3}, [r0]
+; CHECK-NEXT:    vld43.16 {q0, q1, q2, q3}, [r0]
+; CHECK-NEXT:    @ kill: def $q0 killed $q0 killed $q0_q1_q2_q3
+; CHECK-NEXT:    vadd.f16 q4, q2, q3
+; CHECK-NEXT:    vadd.f16 q0, q0, q1
+; CHECK-NEXT:    vadd.f16 q0, q0, q4
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vpop {d8, d9}
+; CHECK-NEXT:    bx lr
+entry:
+  %l1 = load <32 x half>, <32 x half>* %src, align 4
+  %s1 = shufflevector <32 x half> %l1, <32 x half> undef, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
+  %s2 = shufflevector <32 x half> %l1, <32 x half> undef, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>
+  %s3 = shufflevector <32 x half> %l1, <32 x half> undef, <8 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30>
+  %s4 = shufflevector <32 x half> %l1, <32 x half> undef, <8 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31>
+  %a1 = fadd <8 x half> %s1, %s2
+  %a2 = fadd <8 x half> %s3, %s4
+  %a3 = fadd <8 x half> %a1, %a2
+  store <8 x half> %a3, <8 x half> *%dst
+  ret void
+}
+
+define void @vld4_v16f16(<64 x half> *%src, <16 x half> *%dst) {
+; CHECK-LABEL: vld4_v16f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9}
+; CHECK-NEXT:    vpush {d8, d9}
+; CHECK-NEXT:    add.w r2, r0, #64
+; CHECK-NEXT:    vld40.16 {q0, q1, q2, q3}, [r2]
+; CHECK-NEXT:    vld41.16 {q0, q1, q2, q3}, [r2]
+; CHECK-NEXT:    vld42.16 {q0, q1, q2, q3}, [r2]
+; CHECK-NEXT:    vld43.16 {q0, q1, q2, q3}, [r2]
+; CHECK-NEXT:    @ kill: def $q0 killed $q0 killed $q0_q1_q2_q3
+; CHECK-NEXT:    vadd.f16 q4, q2, q3
+; CHECK-NEXT:    vadd.f16 q0, q0, q1
+; CHECK-NEXT:    vadd.f16 q0, q0, q4
+; CHECK-NEXT:    vld40.16 {q1, q2, q3, q4}, [r0]
+; CHECK-NEXT:    vld41.16 {q1, q2, q3, q4}, [r0]
+; CHECK-NEXT:    vld42.16 {q1, q2, q3, q4}, [r0]
+; CHECK-NEXT:    vld43.16 {q1, q2, q3, q4}, [r0]
+; CHECK-NEXT:    vstrw.32 q0, [r1, #16]
+; CHECK-NEXT:    @ kill: def $q1 killed $q1 killed $q1_q2_q3_q4
+; CHECK-NEXT:    vadd.f16 q0, q3, q4
+; CHECK-NEXT:    vadd.f16 q1, q1, q2
+; CHECK-NEXT:    vadd.f16 q0, q1, q0
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vpop {d8, d9}
+; CHECK-NEXT:    bx lr
+entry:
+  %l1 = load <64 x half>, <64 x half>* %src, align 4
+  %s1 = shufflevector <64 x half> %l1, <64 x half> undef, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28, i32 32, i32 36, i32 40, i32 44, i32 48, i32 52, i32 56, i32 60>
+  %s2 = shufflevector <64 x half> %l1, <64 x half> undef, <16 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29, i32 33, i32 37, i32 41, i32 45, i32 49, i32 53, i32 57, i32 61>
+  %s3 = shufflevector <64 x half> %l1, <64 x half> undef, <16 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30, i32 34, i32 38, i32 42, i32 46, i32 50, i32 54, i32 58, i32 62>
+  %s4 = shufflevector <64 x half> %l1, <64 x half> undef, <16 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31, i32 35, i32 39, i32 43, i32 47, i32 51, i32 55, i32 59, i32 63>
+  %a1 = fadd <16 x half> %s1, %s2
+  %a2 = fadd <16 x half> %s3, %s4
+  %a3 = fadd <16 x half> %a1, %a2
+  store <16 x half> %a3, <16 x half> *%dst
+  ret void
+}
+
+; f64
+
+define void @vld4_v2f64(<8 x double> *%src, <2 x double> *%dst) {
+; CHECK-LABEL: vld4_v2f64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #48]
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #32]
+; CHECK-NEXT:    vldrw.u32 q2, [r0]
+; CHECK-NEXT:    vadd.f64 d0, d0, d1
+; CHECK-NEXT:    vadd.f64 d1, d2, d3
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #16]
+; CHECK-NEXT:    vadd.f64 d2, d2, d3
+; CHECK-NEXT:    vadd.f64 d3, d4, d5
+; CHECK-NEXT:    vadd.f64 d1, d1, d0
+; CHECK-NEXT:    vadd.f64 d0, d3, d2
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %l1 = load <8 x double>, <8 x double>* %src, align 4
+  %s1 = shufflevector <8 x double> %l1, <8 x double> undef, <2 x i32> <i32 0, i32 4>
+  %s2 = shufflevector <8 x double> %l1, <8 x double> undef, <2 x i32> <i32 1, i32 5>
+  %s3 = shufflevector <8 x double> %l1, <8 x double> undef, <2 x i32> <i32 2, i32 6>
+  %s4 = shufflevector <8 x double> %l1, <8 x double> undef, <2 x i32> <i32 3, i32 7>
+  %a1 = fadd <2 x double> %s1, %s2
+  %a2 = fadd <2 x double> %s3, %s4
+  %a3 = fadd <2 x double> %a1, %a2
+  store <2 x double> %a3, <2 x double> *%dst
+  ret void
+}
+
+define void @vld4_v4f64(<16 x double> *%src, <4 x double> *%dst) {
+; CHECK-LABEL: vld4_v4f64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9}
+; CHECK-NEXT:    vpush {d8, d9}
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #112]
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #96]
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #64]
+; CHECK-NEXT:    vldrw.u32 q3, [r0, #32]
+; CHECK-NEXT:    vadd.f64 d0, d0, d1
+; CHECK-NEXT:    vldrw.u32 q4, [r0]
+; CHECK-NEXT:    vadd.f64 d1, d2, d3
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #80]
+; CHECK-NEXT:    vadd.f64 d2, d2, d3
+; CHECK-NEXT:    vadd.f64 d3, d4, d5
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #48]
+; CHECK-NEXT:    vadd.f64 d4, d4, d5
+; CHECK-NEXT:    vadd.f64 d5, d6, d7
+; CHECK-NEXT:    vldrw.u32 q3, [r0, #16]
+; CHECK-NEXT:    vadd.f64 d6, d6, d7
+; CHECK-NEXT:    vadd.f64 d7, d8, d9
+; CHECK-NEXT:    vadd.f64 d1, d1, d0
+; CHECK-NEXT:    vadd.f64 d0, d3, d2
+; CHECK-NEXT:    vadd.f64 d3, d5, d4
+; CHECK-NEXT:    vstrw.32 q0, [r1, #16]
+; CHECK-NEXT:    vadd.f64 d2, d7, d6
+; CHECK-NEXT:    vstrw.32 q1, [r1]
+; CHECK-NEXT:    vpop {d8, d9}
+; CHECK-NEXT:    bx lr
+entry:
+  %l1 = load <16 x double>, <16 x double>* %src, align 4
+  %s1 = shufflevector <16 x double> %l1, <16 x double> undef, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+  %s2 = shufflevector <16 x double> %l1, <16 x double> undef, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
+  %s3 = shufflevector <16 x double> %l1, <16 x double> undef, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
+  %s4 = shufflevector <16 x double> %l1, <16 x double> undef, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
+  %a1 = fadd <4 x double> %s1, %s2
+  %a2 = fadd <4 x double> %s3, %s4
+  %a3 = fadd <4 x double> %a1, %a2
+  store <4 x double> %a3, <4 x double> *%dst
+  ret void
+}
diff --git a/llvm/test/CodeGen/Thumb2/mve-vldst4.ll b/llvm/test/CodeGen/Thumb2/mve-vldst4.ll
new file mode 100644
index 000000000..4b5f8b762
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-vldst4.ll
@@ -0,0 +1,284 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve.fp -verify-machineinstrs %s -o - | FileCheck %s
+
+define void @vldst4(half* nocapture readonly %pIn, half* nocapture %pOut, i32 %numRows, i32 %numCols, i32 %scale.coerce) #0 {
+; CHECK-LABEL: vldst4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    .pad #104
+; CHECK-NEXT:    sub sp, #104
+; CHECK-NEXT:    muls r2, r3, r2
+; CHECK-NEXT:    movs r3, #0
+; CHECK-NEXT:    cmp.w r3, r2, lsr #2
+; CHECK-NEXT:    beq.w .LBB0_3
+; CHECK-NEXT:  @ %bb.1: @ %vector.ph
+; CHECK-NEXT:    mvn r3, #7
+; CHECK-NEXT:    and.w r2, r3, r2, lsr #2
+; CHECK-NEXT:    vldr.16 s0, [sp, #176]
+; CHECK-NEXT:    subs r2, #8
+; CHECK-NEXT:    movs r3, #1
+; CHECK-NEXT:    sub.w r12, r0, #64
+; CHECK-NEXT:    add.w lr, r3, r2, lsr #3
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vdup.16 q0, r2
+; CHECK-NEXT:    subs r1, #64
+; CHECK-NEXT:    vstrw.32 q0, [sp, #80] @ 16-byte Spill
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB0_2: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrh.u16 q0, [r12, #64]!
+; CHECK-NEXT:    vldrh.u16 q6, [r12, #32]
+; CHECK-NEXT:    vldrh.u16 q4, [r12, #48]
+; CHECK-NEXT:    vldrh.u16 q7, [r12, #16]
+; CHECK-NEXT:    vmov r2, s24
+; CHECK-NEXT:    vmovx.f16 s12, s16
+; CHECK-NEXT:    vmov.16 q1[4], r2
+; CHECK-NEXT:    vmov r3, s26
+; CHECK-NEXT:    vmov.16 q1[5], r3
+; CHECK-NEXT:    vmov r2, s16
+; CHECK-NEXT:    vmov.16 q1[6], r2
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vmov.16 q2[0], r2
+; CHECK-NEXT:    vmov r3, s2
+; CHECK-NEXT:    vmov.16 q2[1], r3
+; CHECK-NEXT:    vmov r2, s28
+; CHECK-NEXT:    vmov.16 q2[2], r2
+; CHECK-NEXT:    vmov r2, s18
+; CHECK-NEXT:    vmov.16 q1[7], r2
+; CHECK-NEXT:    vmov r2, s30
+; CHECK-NEXT:    vmov.16 q2[3], r2
+; CHECK-NEXT:    vmov.f32 s10, s6
+; CHECK-NEXT:    vmov.f32 s11, s7
+; CHECK-NEXT:    vldrw.u32 q1, [sp, #80] @ 16-byte Reload
+; CHECK-NEXT:    vmul.f16 q5, q2, q1
+; CHECK-NEXT:    vmovx.f16 s4, s2
+; CHECK-NEXT:    vmov r3, s4
+; CHECK-NEXT:    vmovx.f16 s4, s0
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmovx.f16 s8, s28
+; CHECK-NEXT:    vmov.16 q1[0], r0
+; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    vmov.16 q1[1], r3
+; CHECK-NEXT:    vmovx.f16 s8, s26
+; CHECK-NEXT:    vmov.16 q1[2], r0
+; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    vmovx.f16 s8, s24
+; CHECK-NEXT:    vmov r2, s22
+; CHECK-NEXT:    vmov r3, s8
+; CHECK-NEXT:    vmovx.f16 s0, s1
+; CHECK-NEXT:    vmov.16 q2[4], r3
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    vmov.16 q2[5], r0
+; CHECK-NEXT:    vmov r0, s12
+; CHECK-NEXT:    vmovx.f16 s12, s18
+; CHECK-NEXT:    vmov.16 q2[6], r0
+; CHECK-NEXT:    vmov r0, s12
+; CHECK-NEXT:    vmovx.f16 s12, s30
+; CHECK-NEXT:    vmov.16 q2[7], r0
+; CHECK-NEXT:    vmov r0, s12
+; CHECK-NEXT:    vmov.16 q1[3], r0
+; CHECK-NEXT:    vldrw.u32 q3, [sp, #80] @ 16-byte Reload
+; CHECK-NEXT:    vmov.f32 s6, s10
+; CHECK-NEXT:    vstrw.32 q5, [sp, #16] @ 16-byte Spill
+; CHECK-NEXT:    vmov.f32 s7, s11
+; CHECK-NEXT:    vmov.16 q2[0], r2
+; CHECK-NEXT:    vmul.f16 q1, q1, q3
+; CHECK-NEXT:    vmov r2, s27
+; CHECK-NEXT:    vmov r0, s6
+; CHECK-NEXT:    vstrw.32 q1, [sp, #32] @ 16-byte Spill
+; CHECK-NEXT:    vmovx.f16 s4, s22
+; CHECK-NEXT:    vmov.16 q2[1], r0
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmov.16 q2[4], r0
+; CHECK-NEXT:    vmov r0, s25
+; CHECK-NEXT:    vmov.16 q1[4], r0
+; CHECK-NEXT:    vmov r0, s17
+; CHECK-NEXT:    vmov.16 q1[5], r2
+; CHECK-NEXT:    vmov r2, s1
+; CHECK-NEXT:    vstrw.32 q2, [sp, #64] @ 16-byte Spill
+; CHECK-NEXT:    vmov.16 q1[6], r0
+; CHECK-NEXT:    vmov r0, s3
+; CHECK-NEXT:    vmov.16 q2[0], r2
+; CHECK-NEXT:    vmov.16 q2[1], r0
+; CHECK-NEXT:    vmov r0, s29
+; CHECK-NEXT:    vmov.16 q2[2], r0
+; CHECK-NEXT:    vmov r0, s19
+; CHECK-NEXT:    vmov.16 q1[7], r0
+; CHECK-NEXT:    vmov r0, s31
+; CHECK-NEXT:    vmov.16 q2[3], r0
+; CHECK-NEXT:    vmov.f32 s10, s6
+; CHECK-NEXT:    vmov.f32 s11, s7
+; CHECK-NEXT:    vmovx.f16 s4, s3
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmov.16 q0[0], r3
+; CHECK-NEXT:    vmovx.f16 s4, s29
+; CHECK-NEXT:    vmov.16 q0[1], r0
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmovx.f16 s4, s27
+; CHECK-NEXT:    vmov.16 q0[2], r0
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmovx.f16 s4, s25
+; CHECK-NEXT:    vmul.f16 q5, q2, q3
+; CHECK-NEXT:    vmov r3, s4
+; CHECK-NEXT:    vmovx.f16 s8, s17
+; CHECK-NEXT:    vmov.16 q1[4], r3
+; CHECK-NEXT:    vmov r2, s23
+; CHECK-NEXT:    vmov.16 q1[5], r0
+; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    vmovx.f16 s8, s19
+; CHECK-NEXT:    vmov.16 q1[6], r0
+; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    vmovx.f16 s8, s31
+; CHECK-NEXT:    vmov.16 q1[7], r0
+; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    vmov.16 q0[3], r0
+; CHECK-NEXT:    vldrw.u32 q2, [sp, #32] @ 16-byte Reload
+; CHECK-NEXT:    vmov.f32 s2, s6
+; CHECK-NEXT:    vmov.f32 s3, s7
+; CHECK-NEXT:    vmov.16 q1[2], r2
+; CHECK-NEXT:    vmul.f16 q6, q0, q3
+; CHECK-NEXT:    vmovx.f16 s0, s23
+; CHECK-NEXT:    vmov r0, s27
+; CHECK-NEXT:    vmov.16 q1[3], r0
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vmovx.f16 s0, s27
+; CHECK-NEXT:    vmov.16 q1[6], r0
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vmov.16 q1[7], r0
+; CHECK-NEXT:    vmov r2, s11
+; CHECK-NEXT:    vstrw.32 q1, [sp, #48] @ 16-byte Spill
+; CHECK-NEXT:    vldrw.u32 q1, [sp, #16] @ 16-byte Reload
+; CHECK-NEXT:    vmov r0, s7
+; CHECK-NEXT:    vmovx.f16 s0, s7
+; CHECK-NEXT:    vmov.16 q4[0], r0
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vmov.16 q4[1], r2
+; CHECK-NEXT:    vmovx.f16 s0, s11
+; CHECK-NEXT:    vmov.16 q4[4], r0
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vmov.16 q4[5], r0
+; CHECK-NEXT:    vmov r0, s21
+; CHECK-NEXT:    vmov r2, s25
+; CHECK-NEXT:    vmov.16 q3[2], r0
+; CHECK-NEXT:    vmovx.f16 s0, s21
+; CHECK-NEXT:    vmov.16 q3[3], r2
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vmovx.f16 s0, s25
+; CHECK-NEXT:    vmov.16 q3[6], r0
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vmov.16 q3[7], r0
+; CHECK-NEXT:    vmov r0, s5
+; CHECK-NEXT:    vmov r2, s9
+; CHECK-NEXT:    vmov.16 q7[0], r0
+; CHECK-NEXT:    vmovx.f16 s0, s5
+; CHECK-NEXT:    vmov.16 q7[1], r2
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vmovx.f16 s0, s9
+; CHECK-NEXT:    vmov.16 q7[4], r0
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vmov.16 q7[5], r0
+; CHECK-NEXT:    vmov r0, s20
+; CHECK-NEXT:    vstrw.32 q3, [sp] @ 16-byte Spill
+; CHECK-NEXT:    vmovx.f16 s0, s20
+; CHECK-NEXT:    vmov.16 q3[2], r0
+; CHECK-NEXT:    vmov r2, s24
+; CHECK-NEXT:    vmov.16 q3[3], r2
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vmovx.f16 s0, s24
+; CHECK-NEXT:    vmov.16 q3[6], r0
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vmovx.f16 s20, s22
+; CHECK-NEXT:    vmov.16 q3[7], r0
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmov r2, s8
+; CHECK-NEXT:    vmov.16 q0[0], r0
+; CHECK-NEXT:    vmovx.f16 s4, s4
+; CHECK-NEXT:    vmov.16 q0[1], r2
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmovx.f16 s4, s8
+; CHECK-NEXT:    vmov.16 q0[4], r0
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmov.16 q0[5], r0
+; CHECK-NEXT:    vmov r0, s22
+; CHECK-NEXT:    vmov.16 q1[2], r0
+; CHECK-NEXT:    vmov r2, s26
+; CHECK-NEXT:    vmov r0, s20
+; CHECK-NEXT:    vmov.16 q1[3], r2
+; CHECK-NEXT:    vmovx.f16 s20, s26
+; CHECK-NEXT:    vmov.16 q1[6], r0
+; CHECK-NEXT:    vmov r0, s20
+; CHECK-NEXT:    vmovx.f16 s20, s10
+; CHECK-NEXT:    vmov.16 q1[7], r0
+; CHECK-NEXT:    vmov r0, s20
+; CHECK-NEXT:    vldrw.u32 q5, [sp, #64] @ 16-byte Reload
+; CHECK-NEXT:    vldrw.u32 q2, [sp] @ 16-byte Reload
+; CHECK-NEXT:    vldrw.u32 q6, [sp, #48] @ 16-byte Reload
+; CHECK-NEXT:    vmov.f32 s1, s13
+; CHECK-NEXT:    vmov.f32 s29, s9
+; CHECK-NEXT:    vmov.16 q5[5], r0
+; CHECK-NEXT:    vmov.f32 s31, s11
+; CHECK-NEXT:    vmov q2, q5
+; CHECK-NEXT:    vmov.f32 s17, s25
+; CHECK-NEXT:    vmov.f32 s9, s5
+; CHECK-NEXT:    vmov.f32 s3, s15
+; CHECK-NEXT:    vstrb.8 q0, [r1, #64]!
+; CHECK-NEXT:    vmov.f32 s19, s27
+; CHECK-NEXT:    vmov.f32 s11, s7
+; CHECK-NEXT:    vstrh.16 q4, [r1, #48]
+; CHECK-NEXT:    vstrh.16 q2, [r1, #32]
+; CHECK-NEXT:    vstrh.16 q7, [r1, #16]
+; CHECK-NEXT:    le lr, .LBB0_2
+; CHECK-NEXT:  .LBB0_3: @ %while.end
+; CHECK-NEXT:    add sp, #104
+; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %tmp.0.extract.trunc = trunc i32 %scale.coerce to i16
+  %l0 = bitcast i16 %tmp.0.extract.trunc to half
+  %mul = mul i32 %numCols, %numRows
+  %shr = lshr i32 %mul, 2
+  %cmp38 = icmp eq i32 %shr, 0
+  br i1 %cmp38, label %while.end, label %vector.ph
+
+vector.ph:                                        ; preds = %vector.memcheck
+  %n.vec = and i32 %shr, 1073741816
+  %l2 = shl nuw i32 %n.vec, 2
+  %ind.end = getelementptr half, half* %pIn, i32 %l2
+  %l3 = shl nuw i32 %n.vec, 2
+  %ind.end48 = getelementptr half, half* %pOut, i32 %l3
+  %ind.end50 = sub nsw i32 %shr, %n.vec
+  %broadcast.splatinsert55 = insertelement <8 x half> undef, half %l0, i32 0
+  %broadcast.splat56 = shufflevector <8 x half> %broadcast.splatinsert55, <8 x half> undef, <8 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %l4 = shl i32 %index, 2
+  %next.gep = getelementptr half, half* %pIn, i32 %l4
+  %l5 = shl i32 %index, 2
+  %l6 = bitcast half* %next.gep to <32 x half>*
+  %wide.vec = load <32 x half>, <32 x half>* %l6, align 2
+  %strided.vec = shufflevector <32 x half> %wide.vec, <32 x half> undef, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
+  %strided.vec52 = shufflevector <32 x half> %wide.vec, <32 x half> undef, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>
+  %strided.vec53 = shufflevector <32 x half> %wide.vec, <32 x half> undef, <8 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30>
+  %strided.vec54 = shufflevector <32 x half> %wide.vec, <32 x half> undef, <8 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31>
+  %l7 = fmul <8 x half> %strided.vec, %broadcast.splat56
+  %l8 = fmul <8 x half> %strided.vec52, %broadcast.splat56
+  %l9 = fmul <8 x half> %strided.vec53, %broadcast.splat56
+  %l10 = fmul <8 x half> %strided.vec54, %broadcast.splat56
+  %l11 = getelementptr inbounds half, half* %pOut, i32 %l5
+  %l12 = bitcast half* %l11 to <32 x half>*
+  %l13 = shufflevector <8 x half> %l7, <8 x half> %l8, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %l14 = shufflevector <8 x half> %l9, <8 x half> %l10, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %interleaved.vec = shufflevector <16 x half> %l13, <16 x half> %l14, <32 x i32> <i32 0, i32 8, i32 16, i32 24, i32 1, i32 9, i32 17, i32 25, i32 2, i32 10, i32 18, i32 26, i32 3, i32 11, i32 19, i32 27, i32 4, i32 12, i32 20, i32 28, i32 5, i32 13, i32 21, i32 29, i32 6, i32 14, i32 22, i32 30, i32 7, i32 15, i32 23, i32 31>
+  store <32 x half> %interleaved.vec, <32 x half>* %l12, align 2
+  %index.next = add i32 %index, 8
+  %l15 = icmp eq i32 %index.next, %n.vec
+  br i1 %l15, label %while.end, label %vector.body
+
+while.end:                                        ; preds = %while.body, %middle.block, %entry
+  ret void
+}
diff --git a/llvm/test/CodeGen/Thumb2/mve-vmaxv.ll b/llvm/test/CodeGen/Thumb2/mve-vmaxv.ll
new file mode 100644
index 000000000..4c705b75f
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-vmaxv.ll
@@ -0,0 +1,136 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve.fp %s -o - | FileCheck %s
+
+declare i8 @llvm.experimental.vector.reduce.smax.v16i8(<16 x i8>)
+declare i16 @llvm.experimental.vector.reduce.smax.v8i16(<8 x i16>)
+declare i32 @llvm.experimental.vector.reduce.smax.v4i32(<4 x i32>)
+declare i8 @llvm.experimental.vector.reduce.umax.v16i8(<16 x i8>)
+declare i16 @llvm.experimental.vector.reduce.umax.v8i16(<8 x i16>)
+declare i32 @llvm.experimental.vector.reduce.umax.v4i32(<4 x i32>)
+declare i8 @llvm.experimental.vector.reduce.smin.v16i8(<16 x i8>)
+declare i16 @llvm.experimental.vector.reduce.smin.v8i16(<8 x i16>)
+declare i32 @llvm.experimental.vector.reduce.smin.v4i32(<4 x i32>)
+declare i8 @llvm.experimental.vector.reduce.umin.v16i8(<16 x i8>)
+declare i16 @llvm.experimental.vector.reduce.umin.v8i16(<8 x i16>)
+declare i32 @llvm.experimental.vector.reduce.umin.v4i32(<4 x i32>)
+
+define arm_aapcs_vfpcc i8 @vmaxv_s_v16i8_i32(<16 x i8> %s1) {
+; CHECK-LABEL: vmaxv_s_v16i8_i32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    mvn r0, #127
+; CHECK-NEXT:    vmaxv.s8 r0, q0
+; CHECK-NEXT:    bx lr
+  %r = call i8 @llvm.experimental.vector.reduce.smax.v16i8(<16 x i8> %s1)
+  ret i8 %r
+}
+
+define arm_aapcs_vfpcc i16 @vmaxv_s_v8i16_i32(<8 x i16> %s1) {
+; CHECK-LABEL: vmaxv_s_v8i16_i32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    movw r0, #32768
+; CHECK-NEXT:    movt r0, #65535
+; CHECK-NEXT:    vmaxv.s16 r0, q0
+; CHECK-NEXT:    bx lr
+  %r = call i16 @llvm.experimental.vector.reduce.smax.v8i16(<8 x i16> %s1)
+  ret i16 %r
+}
+
+define arm_aapcs_vfpcc i32 @vmaxv_s_v4i32_i32(<4 x i32> %s1) {
+; CHECK-LABEL: vmaxv_s_v4i32_i32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    mov.w r0, #-2147483648
+; CHECK-NEXT:    vmaxv.s32 r0, q0
+; CHECK-NEXT:    bx lr
+  %r = call i32 @llvm.experimental.vector.reduce.smax.v4i32(<4 x i32> %s1)
+  ret i32 %r
+}
+
+define arm_aapcs_vfpcc i8 @vmaxv_u_v16i8_i32(<16 x i8> %s1) {
+; CHECK-LABEL: vmaxv_u_v16i8_i32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    movs r0, #0
+; CHECK-NEXT:    vmaxv.u8 r0, q0
+; CHECK-NEXT:    bx lr
+  %r = call i8 @llvm.experimental.vector.reduce.umax.v16i8(<16 x i8> %s1)
+  ret i8 %r
+}
+
+define arm_aapcs_vfpcc i16 @vmaxv_u_v8i16_i32(<8 x i16> %s1) {
+; CHECK-LABEL: vmaxv_u_v8i16_i32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    movs r0, #0
+; CHECK-NEXT:    vmaxv.u16 r0, q0
+; CHECK-NEXT:    bx lr
+  %r = call i16 @llvm.experimental.vector.reduce.umax.v8i16(<8 x i16> %s1)
+  ret i16 %r
+}
+
+define arm_aapcs_vfpcc i32 @vmaxv_u_v4i32_i32(<4 x i32> %s1) {
+; CHECK-LABEL: vmaxv_u_v4i32_i32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    movs r0, #0
+; CHECK-NEXT:    vmaxv.u32 r0, q0
+; CHECK-NEXT:    bx lr
+  %r = call i32 @llvm.experimental.vector.reduce.umax.v4i32(<4 x i32> %s1)
+  ret i32 %r
+}
+
+define arm_aapcs_vfpcc i8 @vminv_s_v16i8_i32(<16 x i8> %s1) {
+; CHECK-LABEL: vminv_s_v16i8_i32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    movs r0, #127
+; CHECK-NEXT:    vminv.s8 r0, q0
+; CHECK-NEXT:    bx lr
+  %r = call i8 @llvm.experimental.vector.reduce.smin.v16i8(<16 x i8> %s1)
+  ret i8 %r
+}
+
+define arm_aapcs_vfpcc i16 @vminv_s_v8i16_i32(<8 x i16> %s1) {
+; CHECK-LABEL: vminv_s_v8i16_i32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    movw r0, #32767
+; CHECK-NEXT:    vminv.s16 r0, q0
+; CHECK-NEXT:    bx lr
+  %r = call i16 @llvm.experimental.vector.reduce.smin.v8i16(<8 x i16> %s1)
+  ret i16 %r
+}
+
+define arm_aapcs_vfpcc i32 @vminv_s_v4i32_i32(<4 x i32> %s1) {
+; CHECK-LABEL: vminv_s_v4i32_i32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    mvn r0, #-2147483648
+; CHECK-NEXT:    vminv.s32 r0, q0
+; CHECK-NEXT:    bx lr
+  %r = call i32 @llvm.experimental.vector.reduce.smin.v4i32(<4 x i32> %s1)
+  ret i32 %r
+}
+
+define arm_aapcs_vfpcc i8 @vminv_u_v16i8_i32(<16 x i8> %s1) {
+; CHECK-LABEL: vminv_u_v16i8_i32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    movs r0, #255
+; CHECK-NEXT:    vminv.u8 r0, q0
+; CHECK-NEXT:    bx lr
+  %r = call i8 @llvm.experimental.vector.reduce.umin.v16i8(<16 x i8> %s1)
+  ret i8 %r
+}
+
+define arm_aapcs_vfpcc i16 @vminv_u_v8i16_i32(<8 x i16> %s1) {
+; CHECK-LABEL: vminv_u_v8i16_i32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    movw r0, #65535
+; CHECK-NEXT:    vminv.u16 r0, q0
+; CHECK-NEXT:    bx lr
+  %r = call i16 @llvm.experimental.vector.reduce.umin.v8i16(<8 x i16> %s1)
+  ret i16 %r
+}
+
+define arm_aapcs_vfpcc i32 @vminv_u_v4i32_i32(<4 x i32> %s1) {
+; CHECK-LABEL: vminv_u_v4i32_i32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    mov.w r0, #-1
+; CHECK-NEXT:    vminv.u32 r0, q0
+; CHECK-NEXT:    bx lr
+  %r = call i32 @llvm.experimental.vector.reduce.umin.v4i32(<4 x i32> %s1)
+  ret i32 %r
+}
diff --git a/llvm/test/CodeGen/Thumb2/mve-vmla.ll b/llvm/test/CodeGen/Thumb2/mve-vmla.ll
new file mode 100644
index 000000000..b75fc964d
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-vmla.ll
@@ -0,0 +1,200 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve.fp %s -o - | FileCheck %s
+
+define arm_aapcs_vfpcc <4 x i32> @vmlau32(<4 x i32> %A, <4 x i32> %B, i32 %X) nounwind {
+; CHECK-LABEL: vmlau32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmla.u32 q0, q1, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = insertelement <4 x i32> undef, i32 %X, i32 0
+  %1 = shufflevector <4 x i32> %0, <4 x i32> undef, <4 x i32> zeroinitializer
+  %2 = mul nsw <4 x i32> %B, %1
+  %3 = add nsw <4 x i32> %A, %2
+  ret <4 x i32> %3
+}
+
+define arm_aapcs_vfpcc <4 x i32> @vmlau32b(<4 x i32> %A, <4 x i32> %B, i32 %X) nounwind {
+; CHECK-LABEL: vmlau32b:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmla.u32 q0, q1, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = insertelement <4 x i32> undef, i32 %X, i32 0
+  %1 = shufflevector <4 x i32> %0, <4 x i32> undef, <4 x i32> zeroinitializer
+  %2 = mul nsw <4 x i32> %1, %B
+  %3 = add nsw <4 x i32> %2, %A
+  ret <4 x i32> %3
+}
+
+define arm_aapcs_vfpcc <8 x i16> @vmlau16(<8 x i16> %A, <8 x i16> %B, i16 %X) nounwind {
+; CHECK-LABEL: vmlau16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmla.u16 q0, q1, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = insertelement <8 x i16> undef, i16 %X, i32 0
+  %1 = shufflevector <8 x i16> %0, <8 x i16> undef, <8 x i32> zeroinitializer
+  %2 = mul nsw <8 x i16> %B, %1
+  %3 = add nsw <8 x i16> %A, %2
+  ret <8 x i16> %3
+}
+
+define arm_aapcs_vfpcc <8 x i16> @vmlau16b(<8 x i16> %A, <8 x i16> %B, i16 %X) nounwind {
+; CHECK-LABEL: vmlau16b:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmla.u16 q0, q1, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = insertelement <8 x i16> undef, i16 %X, i32 0
+  %1 = shufflevector <8 x i16> %0, <8 x i16> undef, <8 x i32> zeroinitializer
+  %2 = mul nsw <8 x i16> %1, %B
+  %3 = add nsw <8 x i16> %2, %A
+  ret <8 x i16> %3
+}
+
+define arm_aapcs_vfpcc <16 x i8> @vmlau8(<16 x i8> %A, <16 x i8> %B, i8 %X) nounwind {
+; CHECK-LABEL: vmlau8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmla.u8 q0, q1, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = insertelement <16 x i8> undef, i8 %X, i32 0
+  %1 = shufflevector <16 x i8> %0, <16 x i8> undef, <16 x i32> zeroinitializer
+  %2 = mul nsw <16 x i8> %B, %1
+  %3 = add nsw <16 x i8> %A, %2
+  ret <16 x i8> %3
+}
+
+define arm_aapcs_vfpcc <16 x i8> @vmlau8b(<16 x i8> %A, <16 x i8> %B, i8 %X) nounwind {
+; CHECK-LABEL: vmlau8b:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmla.u8 q0, q1, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = insertelement <16 x i8> undef, i8 %X, i32 0
+  %1 = shufflevector <16 x i8> %0, <16 x i8> undef, <16 x i32> zeroinitializer
+  %2 = mul nsw <16 x i8> %1, %B
+  %3 = add nsw <16 x i8> %2, %A
+  ret <16 x i8> %3
+}
+
+define void @vmla32_in_loop(i32* %s1, i32 %x, i32* %d, i32 %n) {
+; CHECK-LABEL: vmla32_in_loop:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    subs r0, #16
+; CHECK-NEXT:    subs r2, #16
+; CHECK-NEXT:  .LBB6_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #16]!
+; CHECK-NEXT:    vldrw.u32 q1, [r2, #16]!
+; CHECK-NEXT:    subs r3, #4
+; CHECK-NEXT:    vmla.u32 q1, q0, r1
+; CHECK-NEXT:    vstrw.32 q1, [r2]
+; CHECK-NEXT:    bne .LBB6_1
+; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
+; CHECK-NEXT:    bx lr
+entry:
+  %broadcast.splatinsert8 = insertelement <4 x i32> undef, i32 %x, i32 0
+  %broadcast.splat9 = shufflevector <4 x i32> %broadcast.splatinsert8, <4 x i32> undef, <4 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:
+  %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]
+  %0 = getelementptr inbounds i32, i32* %s1, i32 %index
+  %1 = bitcast i32* %0 to <4 x i32>*
+  %wide.load = load <4 x i32>, <4 x i32>* %1, align 4
+  %2 = mul nsw <4 x i32> %wide.load, %broadcast.splat9
+  %3 = getelementptr inbounds i32, i32* %d, i32 %index
+  %4 = bitcast i32* %3 to <4 x i32>*
+  %wide.load10 = load <4 x i32>, <4 x i32>* %4, align 4
+  %5 = add nsw <4 x i32> %wide.load10, %2
+  %6 = bitcast i32* %3 to <4 x i32>*
+  store <4 x i32> %5, <4 x i32>* %6, align 4
+  %index.next = add i32 %index, 4
+  %7 = icmp eq i32 %index.next, %n
+  br i1 %7, label %for.cond.cleanup, label %vector.body
+
+for.cond.cleanup:
+  ret void
+}
+
+define void @vmla16_in_loop(i16* %s1, i16 %x, i16* %d, i32 %n) {
+; CHECK-LABEL: vmla16_in_loop:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    subs r0, #16
+; CHECK-NEXT:    subs r2, #16
+; CHECK-NEXT:  .LBB7_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrh.u16 q0, [r0, #16]!
+; CHECK-NEXT:    vldrh.u16 q1, [r2, #16]!
+; CHECK-NEXT:    subs r3, #8
+; CHECK-NEXT:    vmla.u16 q1, q0, r1
+; CHECK-NEXT:    vstrh.16 q1, [r2]
+; CHECK-NEXT:    bne .LBB7_1
+; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
+; CHECK-NEXT:    bx lr
+entry:
+  %broadcast.splatinsert11 = insertelement <8 x i16> undef, i16 %x, i32 0
+  %broadcast.splat12 = shufflevector <8 x i16> %broadcast.splatinsert11, <8 x i16> undef, <8 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:
+  %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]
+  %0 = getelementptr inbounds i16, i16* %s1, i32 %index
+  %1 = bitcast i16* %0 to <8 x i16>*
+  %wide.load = load <8 x i16>, <8 x i16>* %1, align 2
+  %2 = mul <8 x i16> %wide.load, %broadcast.splat12
+  %3 = getelementptr inbounds i16, i16* %d, i32 %index
+  %4 = bitcast i16* %3 to <8 x i16>*
+  %wide.load13 = load <8 x i16>, <8 x i16>* %4, align 2
+  %5 = add <8 x i16> %2, %wide.load13
+  %6 = bitcast i16* %3 to <8 x i16>*
+  store <8 x i16> %5, <8 x i16>* %6, align 2
+  %index.next = add i32 %index, 8
+  %7 = icmp eq i32 %index.next, %n
+  br i1 %7, label %for.cond.cleanup, label %vector.body
+
+for.cond.cleanup:
+  ret void
+}
+
+define void @vmla8_in_loop(i8* %s1, i8 %x, i8* %d, i32 %n) {
+; CHECK-LABEL: vmla8_in_loop:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    subs r0, #16
+; CHECK-NEXT:    subs r2, #16
+; CHECK-NEXT:  .LBB8_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrh.u16 q0, [r0, #16]!
+; CHECK-NEXT:    vldrh.u16 q1, [r2, #16]!
+; CHECK-NEXT:    subs r3, #16
+; CHECK-NEXT:    vmla.u8 q1, q0, r1
+; CHECK-NEXT:    vstrh.16 q1, [r2]
+; CHECK-NEXT:    bne .LBB8_1
+; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
+; CHECK-NEXT:    bx lr
+entry:
+  %broadcast.splatinsert11 = insertelement <16 x i8> undef, i8 %x, i32 0
+  %broadcast.splat12 = shufflevector <16 x i8> %broadcast.splatinsert11, <16 x i8> undef, <16 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:
+  %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]
+  %0 = getelementptr inbounds i8, i8* %s1, i32 %index
+  %1 = bitcast i8* %0 to <16 x i8>*
+  %wide.load = load <16 x i8>, <16 x i8>* %1, align 2
+  %2 = mul <16 x i8> %wide.load, %broadcast.splat12
+  %3 = getelementptr inbounds i8, i8* %d, i32 %index
+  %4 = bitcast i8* %3 to <16 x i8>*
+  %wide.load13 = load <16 x i8>, <16 x i8>* %4, align 2
+  %5 = add <16 x i8> %2, %wide.load13
+  %6 = bitcast i8* %3 to <16 x i8>*
+  store <16 x i8> %5, <16 x i8>* %6, align 2
+  %index.next = add i32 %index, 16
+  %7 = icmp eq i32 %index.next, %n
+  br i1 %7, label %for.cond.cleanup, label %vector.body
+
+for.cond.cleanup:
+  ret void
+}
diff --git a/llvm/test/CodeGen/Thumb2/mve-vmovn.ll b/llvm/test/CodeGen/Thumb2/mve-vmovn.ll
new file mode 100644
index 000000000..20f790d0a
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-vmovn.ll
@@ -0,0 +1,422 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s
+
+define arm_aapcs_vfpcc <8 x i16> @vmovn32_trunc1(<4 x i32> %src1, <4 x i32> %src2) {
+; CHECK-LABEL: vmovn32_trunc1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmovnt.i32 q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %strided.vec = shufflevector <4 x i32> %src1, <4 x i32> %src2, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+  %out = trunc <8 x i32> %strided.vec to <8 x i16>
+  ret <8 x i16> %out
+}
+
+define arm_aapcs_vfpcc <8 x i16> @vmovn32_trunc2(<4 x i32> %src1, <4 x i32> %src2) {
+; CHECK-LABEL: vmovn32_trunc2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmovnt.i32 q1, q0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %strided.vec = shufflevector <4 x i32> %src1, <4 x i32> %src2, <8 x i32> <i32 4, i32 0, i32 5, i32 1, i32 6, i32 2, i32 7, i32 3>
+  %out = trunc <8 x i32> %strided.vec to <8 x i16>
+  ret <8 x i16> %out
+}
+
+define arm_aapcs_vfpcc <16 x i8> @vmovn16_trunc1(<8 x i16> %src1, <8 x i16> %src2) {
+; CHECK-LABEL: vmovn16_trunc1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmovnt.i16 q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %strided.vec = shufflevector <8 x i16> %src1, <8 x i16> %src2, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
+  %out = trunc <16 x i16> %strided.vec to <16 x i8>
+  ret <16 x i8> %out
+}
+
+define arm_aapcs_vfpcc <16 x i8> @vmovn16_trunc2(<8 x i16> %src1, <8 x i16> %src2) {
+; CHECK-LABEL: vmovn16_trunc2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmovnt.i16 q1, q0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %strided.vec = shufflevector <8 x i16> %src1, <8 x i16> %src2, <16 x i32> <i32 8, i32 0, i32 9, i32 1, i32 10, i32 2, i32 11, i32 3, i32 12, i32 4, i32 13, i32 5, i32 14, i32 6, i32 15, i32 7>
+  %out = trunc <16 x i16> %strided.vec to <16 x i8>
+  ret <16 x i8> %out
+}
+
+
+define arm_aapcs_vfpcc <2 x i64> @vmovn64_t1(<2 x i64> %src1, <2 x i64> %src2) {
+; CHECK-LABEL: vmovn64_t1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.f32 s2, s4
+; CHECK-NEXT:    vmov.f32 s3, s5
+; CHECK-NEXT:    bx lr
+entry:
+  %out = shufflevector <2 x i64> %src1, <2 x i64> %src2, <2 x i32> <i32 0, i32 2>
+  ret <2 x i64> %out
+}
+
+define arm_aapcs_vfpcc <2 x i64> @vmovn64_t2(<2 x i64> %src1, <2 x i64> %src2) {
+; CHECK-LABEL: vmovn64_t2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.f32 s6, s0
+; CHECK-NEXT:    vmov.f32 s7, s1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %out = shufflevector <2 x i64> %src1, <2 x i64> %src2, <2 x i32> <i32 2, i32 0>
+  ret <2 x i64> %out
+}
+
+define arm_aapcs_vfpcc <2 x i64> @vmovn64_b1(<2 x i64> %src1, <2 x i64> %src2) {
+; CHECK-LABEL: vmovn64_b1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.f32 s2, s6
+; CHECK-NEXT:    vmov.f32 s3, s7
+; CHECK-NEXT:    bx lr
+entry:
+  %out = shufflevector <2 x i64> %src1, <2 x i64> %src2, <2 x i32> <i32 0, i32 3>
+  ret <2 x i64> %out
+}
+
+define arm_aapcs_vfpcc <2 x i64> @vmovn64_b2(<2 x i64> %src1, <2 x i64> %src2) {
+; CHECK-LABEL: vmovn64_b2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.f32 s4, s6
+; CHECK-NEXT:    vmov.f32 s5, s7
+; CHECK-NEXT:    vmov.f32 s6, s0
+; CHECK-NEXT:    vmov.f32 s7, s1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %out = shufflevector <2 x i64> %src1, <2 x i64> %src2, <2 x i32> <i32 3, i32 0>
+  ret <2 x i64> %out
+}
+
+define arm_aapcs_vfpcc <2 x i64> @vmovn64_b3(<2 x i64> %src1, <2 x i64> %src2) {
+; CHECK-LABEL: vmovn64_b3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.f32 s0, s2
+; CHECK-NEXT:    vmov.f32 s1, s3
+; CHECK-NEXT:    vmov.f32 s2, s4
+; CHECK-NEXT:    vmov.f32 s3, s5
+; CHECK-NEXT:    bx lr
+entry:
+  %out = shufflevector <2 x i64> %src1, <2 x i64> %src2, <2 x i32> <i32 1, i32 2>
+  ret <2 x i64> %out
+}
+
+define arm_aapcs_vfpcc <2 x i64> @vmovn64_b4(<2 x i64> %src1, <2 x i64> %src2) {
+; CHECK-LABEL: vmovn64_b4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.f32 s6, s2
+; CHECK-NEXT:    vmov.f32 s7, s3
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %out = shufflevector <2 x i64> %src1, <2 x i64> %src2, <2 x i32> <i32 2, i32 1>
+  ret <2 x i64> %out
+}
+
+
+
+define arm_aapcs_vfpcc <4 x i32> @vmovn32_t1(<4 x i32> %src1, <4 x i32> %src2) {
+; CHECK-LABEL: vmovn32_t1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.f32 s1, s4
+; CHECK-NEXT:    vmov.f32 s3, s6
+; CHECK-NEXT:    bx lr
+entry:
+  %out = shufflevector <4 x i32> %src1, <4 x i32> %src2, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
+  ret <4 x i32> %out
+}
+
+define arm_aapcs_vfpcc <4 x i32> @vmovn32_t2(<4 x i32> %src1, <4 x i32> %src2) {
+; CHECK-LABEL: vmovn32_t2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.f32 s5, s0
+; CHECK-NEXT:    vmov.f32 s7, s2
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %out = shufflevector <4 x i32> %src1, <4 x i32> %src2, <4 x i32> <i32 4, i32 0, i32 6, i32 2>
+  ret <4 x i32> %out
+}
+
+define arm_aapcs_vfpcc <4 x i32> @vmovn32_b1(<4 x i32> %src1, <4 x i32> %src2) {
+; CHECK-LABEL: vmovn32_b1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.f32 s1, s5
+; CHECK-NEXT:    vmov.f32 s3, s7
+; CHECK-NEXT:    bx lr
+entry:
+  %out = shufflevector <4 x i32> %src1, <4 x i32> %src2, <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+  ret <4 x i32> %out
+}
+
+define arm_aapcs_vfpcc <4 x i32> @vmovn32_b2(<4 x i32> %src1, <4 x i32> %src2) {
+; CHECK-LABEL: vmovn32_b2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.f32 s8, s5
+; CHECK-NEXT:    vmov.f32 s9, s0
+; CHECK-NEXT:    vmov.f32 s10, s7
+; CHECK-NEXT:    vmov.f32 s11, s2
+; CHECK-NEXT:    vmov q0, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %out = shufflevector <4 x i32> %src1, <4 x i32> %src2, <4 x i32> <i32 5, i32 0, i32 7, i32 2>
+  ret <4 x i32> %out
+}
+
+define arm_aapcs_vfpcc <4 x i32> @vmovn32_b3(<4 x i32> %src1, <4 x i32> %src2) {
+; CHECK-LABEL: vmovn32_b3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.f32 s8, s1
+; CHECK-NEXT:    vmov.f32 s9, s4
+; CHECK-NEXT:    vmov.f32 s10, s3
+; CHECK-NEXT:    vmov.f32 s11, s6
+; CHECK-NEXT:    vmov q0, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %out = shufflevector <4 x i32> %src1, <4 x i32> %src2, <4 x i32> <i32 1, i32 4, i32 3, i32 6>
+  ret <4 x i32> %out
+}
+
+define arm_aapcs_vfpcc <4 x i32> @vmovn32_b4(<4 x i32> %src1, <4 x i32> %src2) {
+; CHECK-LABEL: vmovn32_b4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.f32 s5, s1
+; CHECK-NEXT:    vmov.f32 s7, s3
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %out = shufflevector <4 x i32> %src1, <4 x i32> %src2, <4 x i32> <i32 4, i32 1, i32 6, i32 3>
+  ret <4 x i32> %out
+}
+
+
+
+
+define arm_aapcs_vfpcc <8 x i16> @vmovn16_t1(<8 x i16> %src1, <8 x i16> %src2) {
+; CHECK-LABEL: vmovn16_t1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmovnt.i32 q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %out = shufflevector <8 x i16> %src1, <8 x i16> %src2, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
+  ret <8 x i16> %out
+}
+
+define arm_aapcs_vfpcc <8 x i16> @vmovn16_t2(<8 x i16> %src1, <8 x i16> %src2) {
+; CHECK-LABEL: vmovn16_t2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmovnt.i32 q1, q0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %out = shufflevector <8 x i16> %src1, <8 x i16> %src2, <8 x i32> <i32 8, i32 0, i32 10, i32 2, i32 12, i32 4, i32 14, i32 6>
+  ret <8 x i16> %out
+}
+
+define arm_aapcs_vfpcc <8 x i16> @vmovn16_b1(<8 x i16> %src1, <8 x i16> %src2) {
+; CHECK-LABEL: vmovn16_b1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmovnb.i32 q1, q0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %out = shufflevector <8 x i16> %src1, <8 x i16> %src2, <8 x i32> <i32 0, i32 9, i32 2, i32 11, i32 4, i32 13, i32 6, i32 15>
+  ret <8 x i16> %out
+}
+
+define arm_aapcs_vfpcc <8 x i16> @vmovn16_b2(<8 x i16> %src1, <8 x i16> %src2) {
+; CHECK-LABEL: vmovn16_b2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov q2, q0
+; CHECK-NEXT:    vmov.u16 r0, q1[1]
+; CHECK-NEXT:    vmov.16 q0[0], r0
+; CHECK-NEXT:    vmov.u16 r0, q2[0]
+; CHECK-NEXT:    vmov.16 q0[1], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[3]
+; CHECK-NEXT:    vmov.16 q0[2], r0
+; CHECK-NEXT:    vmov.u16 r0, q2[2]
+; CHECK-NEXT:    vmov.16 q0[3], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[5]
+; CHECK-NEXT:    vmov.16 q0[4], r0
+; CHECK-NEXT:    vmov.u16 r0, q2[4]
+; CHECK-NEXT:    vmov.16 q0[5], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[7]
+; CHECK-NEXT:    vmov.16 q0[6], r0
+; CHECK-NEXT:    vmov.u16 r0, q2[6]
+; CHECK-NEXT:    vmov.16 q0[7], r0
+; CHECK-NEXT:    bx lr
+entry:
+  %out = shufflevector <8 x i16> %src1, <8 x i16> %src2, <8 x i32> <i32 9, i32 0, i32 11, i32 2, i32 13, i32 4, i32 15, i32 6>
+  ret <8 x i16> %out
+}
+
+define arm_aapcs_vfpcc <8 x i16> @vmovn16_b3(<8 x i16> %src1, <8 x i16> %src2) {
+; CHECK-LABEL: vmovn16_b3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.u16 r0, q0[1]
+; CHECK-NEXT:    vmov q2, q0
+; CHECK-NEXT:    vmov.16 q0[0], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[0]
+; CHECK-NEXT:    vmov.16 q0[1], r0
+; CHECK-NEXT:    vmov.u16 r0, q2[3]
+; CHECK-NEXT:    vmov.16 q0[2], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[2]
+; CHECK-NEXT:    vmov.16 q0[3], r0
+; CHECK-NEXT:    vmov.u16 r0, q2[5]
+; CHECK-NEXT:    vmov.16 q0[4], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[4]
+; CHECK-NEXT:    vmov.16 q0[5], r0
+; CHECK-NEXT:    vmov.u16 r0, q2[7]
+; CHECK-NEXT:    vmov.16 q0[6], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[6]
+; CHECK-NEXT:    vmov.16 q0[7], r0
+; CHECK-NEXT:    bx lr
+entry:
+  %out = shufflevector <8 x i16> %src1, <8 x i16> %src2, <8 x i32> <i32 1, i32 8, i32 3, i32 10, i32 5, i32 12, i32 7, i32 14>
+  ret <8 x i16> %out
+}
+
+define arm_aapcs_vfpcc <8 x i16> @vmovn16_b4(<8 x i16> %src1, <8 x i16> %src2) {
+; CHECK-LABEL: vmovn16_b4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmovnb.i32 q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %out = shufflevector <8 x i16> %src1, <8 x i16> %src2, <8 x i32> <i32 8, i32 1, i32 10, i32 3, i32 12, i32 5, i32 14, i32 7>
+  ret <8 x i16> %out
+}
+
+
+define arm_aapcs_vfpcc <16 x i8> @vmovn8_b1(<16 x i8> %src1, <16 x i8> %src2) {
+; CHECK-LABEL: vmovn8_b1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmovnt.i16 q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %out = shufflevector <16 x i8> %src1, <16 x i8> %src2, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
+  ret <16 x i8> %out
+}
+
+define arm_aapcs_vfpcc <16 x i8> @vmovn8_b2(<16 x i8> %src1, <16 x i8> %src2) {
+; CHECK-LABEL: vmovn8_b2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmovnt.i16 q1, q0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %out = shufflevector <16 x i8> %src1, <16 x i8> %src2, <16 x i32> <i32 16, i32 0, i32 18, i32 2, i32 20, i32 4, i32 22, i32 6, i32 24, i32 8, i32 26, i32 10, i32 28, i32 12, i32 30, i32 14>
+  ret <16 x i8> %out
+}
+
+define arm_aapcs_vfpcc <16 x i8> @vmovn8_t1(<16 x i8> %src1, <16 x i8> %src2) {
+; CHECK-LABEL: vmovn8_t1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmovnb.i16 q1, q0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %out = shufflevector <16 x i8> %src1, <16 x i8> %src2, <16 x i32> <i32 0, i32 17, i32 2, i32 19, i32 4, i32 21, i32 6, i32 23, i32 8, i32 25, i32 10, i32 27, i32 12, i32 29, i32 14, i32 31>
+  ret <16 x i8> %out
+}
+
+define arm_aapcs_vfpcc <16 x i8> @vmovn8_t2(<16 x i8> %src1, <16 x i8> %src2) {
+; CHECK-LABEL: vmovn8_t2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov q2, q0
+; CHECK-NEXT:    vmov.u8 r0, q1[1]
+; CHECK-NEXT:    vmov.8 q0[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[0]
+; CHECK-NEXT:    vmov.8 q0[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[3]
+; CHECK-NEXT:    vmov.8 q0[2], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[2]
+; CHECK-NEXT:    vmov.8 q0[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[5]
+; CHECK-NEXT:    vmov.8 q0[4], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[4]
+; CHECK-NEXT:    vmov.8 q0[5], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[7]
+; CHECK-NEXT:    vmov.8 q0[6], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[6]
+; CHECK-NEXT:    vmov.8 q0[7], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[9]
+; CHECK-NEXT:    vmov.8 q0[8], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[8]
+; CHECK-NEXT:    vmov.8 q0[9], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[11]
+; CHECK-NEXT:    vmov.8 q0[10], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[10]
+; CHECK-NEXT:    vmov.8 q0[11], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[13]
+; CHECK-NEXT:    vmov.8 q0[12], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[12]
+; CHECK-NEXT:    vmov.8 q0[13], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[15]
+; CHECK-NEXT:    vmov.8 q0[14], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[14]
+; CHECK-NEXT:    vmov.8 q0[15], r0
+; CHECK-NEXT:    bx lr
+entry:
+  %out = shufflevector <16 x i8> %src1, <16 x i8> %src2, <16 x i32> <i32 17, i32 0, i32 19, i32 2, i32 21, i32 4, i32 23, i32 6, i32 25, i32 8, i32 27, i32 10, i32 29, i32 12, i32 31, i32 14>
+  ret <16 x i8> %out
+}
+
+define arm_aapcs_vfpcc <16 x i8> @vmovn8_t3(<16 x i8> %src1, <16 x i8> %src2) {
+; CHECK-LABEL: vmovn8_t3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.u8 r0, q0[1]
+; CHECK-NEXT:    vmov q2, q0
+; CHECK-NEXT:    vmov.8 q0[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[0]
+; CHECK-NEXT:    vmov.8 q0[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[3]
+; CHECK-NEXT:    vmov.8 q0[2], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[2]
+; CHECK-NEXT:    vmov.8 q0[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[5]
+; CHECK-NEXT:    vmov.8 q0[4], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[4]
+; CHECK-NEXT:    vmov.8 q0[5], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[7]
+; CHECK-NEXT:    vmov.8 q0[6], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[6]
+; CHECK-NEXT:    vmov.8 q0[7], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[9]
+; CHECK-NEXT:    vmov.8 q0[8], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[8]
+; CHECK-NEXT:    vmov.8 q0[9], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[11]
+; CHECK-NEXT:    vmov.8 q0[10], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[10]
+; CHECK-NEXT:    vmov.8 q0[11], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[13]
+; CHECK-NEXT:    vmov.8 q0[12], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[12]
+; CHECK-NEXT:    vmov.8 q0[13], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[15]
+; CHECK-NEXT:    vmov.8 q0[14], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[14]
+; CHECK-NEXT:    vmov.8 q0[15], r0
+; CHECK-NEXT:    bx lr
+entry:
+  %out = shufflevector <16 x i8> %src1, <16 x i8> %src2, <16 x i32> <i32 1, i32 16, i32 3, i32 18, i32 5, i32 20, i32 7, i32 22, i32 9, i32 24, i32 11, i32 26, i32 13, i32 28, i32 15, i32 30>
+  ret <16 x i8> %out
+}
+
+define arm_aapcs_vfpcc <16 x i8> @vmovn8_t4(<16 x i8> %src1, <16 x i8> %src2) {
+; CHECK-LABEL: vmovn8_t4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmovnb.i16 q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %out = shufflevector <16 x i8> %src1, <16 x i8> %src2, <16 x i32> <i32 16, i32 1, i32 18, i32 3, i32 20, i32 5, i32 22, i32 7, i32 24, i32 9, i32 26, i32 11, i32 28, i32 13, i32 30, i32 15>
+  ret <16 x i8> %out
+}
diff --git a/llvm/test/CodeGen/Thumb2/mve-vmulqr.ll b/llvm/test/CodeGen/Thumb2/mve-vmulqr.ll
new file mode 100644
index 000000000..5c5c39c15
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-vmulqr.ll
@@ -0,0 +1,74 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -verify-machineinstrs -mattr=+mve %s -o - | FileCheck %s
+
+define arm_aapcs_vfpcc <4 x i32> @vmulqr_v4i32(<4 x i32> %src, i32 %src2, <4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: vmulqr_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmul.i32 q0, q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <4 x i32> undef, i32 %src2, i32 0
+  %sp = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %c = mul <4 x i32> %src, %sp
+  ret <4 x i32> %c
+}
+
+define arm_aapcs_vfpcc <8 x i16> @vmulqr_v8i16(<8 x i16> %src, i16 %src2, <8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: vmulqr_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmul.i16 q0, q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <8 x i16> undef, i16 %src2, i32 0
+  %sp = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
+  %c = mul <8 x i16> %src, %sp
+  ret <8 x i16> %c
+}
+
+define arm_aapcs_vfpcc <16 x i8> @vmulqr_v16i8(<16 x i8> %src, i8 %src2, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: vmulqr_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmul.i8 q0, q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <16 x i8> undef, i8 %src2, i32 0
+  %sp = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
+  %c = mul <16 x i8> %src, %sp
+  ret <16 x i8> %c
+}
+
+define arm_aapcs_vfpcc <4 x i32> @vmulqr_v4i32_2(<4 x i32> %src, i32 %src2, <4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: vmulqr_v4i32_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmul.i32 q0, q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <4 x i32> undef, i32 %src2, i32 0
+  %sp = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %c = mul <4 x i32> %sp, %src
+  ret <4 x i32> %c
+}
+
+define arm_aapcs_vfpcc <8 x i16> @vmulqr_v8i16_2(<8 x i16> %src, i16 %src2, <8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: vmulqr_v8i16_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmul.i16 q0, q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <8 x i16> undef, i16 %src2, i32 0
+  %sp = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
+  %c = mul <8 x i16> %sp, %src
+  ret <8 x i16> %c
+}
+
+define arm_aapcs_vfpcc <16 x i8> @vmulqr_v16i8_2(<16 x i8> %src, i8 %src2, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: vmulqr_v16i8_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmul.i8 q0, q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <16 x i8> undef, i8 %src2, i32 0
+  %sp = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
+  %c = mul <16 x i8> %sp, %src
+  ret <16 x i8> %c
+}
diff --git a/llvm/test/CodeGen/Thumb2/mve-vpsel.ll b/llvm/test/CodeGen/Thumb2/mve-vpsel.ll
new file mode 100644
index 000000000..5da0aa518
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-vpsel.ll
@@ -0,0 +1,77 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -O3 -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve.fp %s -o - | FileCheck %s
+
+define arm_aapcs_vfpcc <16 x i8> @vpsel_i8(<16 x i8> %mask, <16 x i8> %src1, <16 x i8> %src2) {
+; CHECK-LABEL: vpsel_i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i8 ne, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = icmp ne <16 x i8> %mask, zeroinitializer
+  %1 = select <16 x i1> %0, <16 x i8> %src1, <16 x i8> %src2
+  ret <16 x i8> %1
+}
+
+define arm_aapcs_vfpcc <8 x i16> @vpsel_i16(<8 x i16> %mask, <8 x i16> %src1, <8 x i16> %src2) {
+; CHECK-LABEL: vpsel_i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i16 ne, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = select <8 x i1> %0, <8 x i16> %src1, <8 x i16> %src2
+  ret <8 x i16> %1
+}
+
+define arm_aapcs_vfpcc <4 x i32> @vpsel_i32(<4 x i32> %mask, <4 x i32> %src1, <4 x i32> %src2) {
+; CHECK-LABEL: vpsel_i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i32 ne, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = select <4 x i1> %0, <4 x i32> %src1, <4 x i32> %src2
+  ret <4 x i32> %1
+}
+
+define arm_aapcs_vfpcc <8 x half> @vpsel_f16(<8 x i16> %mask, <8 x half> %src1, <8 x half> %src2) {
+; CHECK-LABEL: vpsel_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i16 ne, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = icmp ne <8 x i16> %mask, zeroinitializer
+  %1 = select <8 x i1> %0, <8 x half> %src1, <8 x half> %src2
+  ret <8 x half> %1
+}
+
+define arm_aapcs_vfpcc <4 x float> @vpsel_f32(<4 x i32> %mask, <4 x float> %src1, <4 x float> %src2) {
+; CHECK-LABEL: vpsel_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i32 ne, q0, zr
+; CHECK-NEXT:    vpsel q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = icmp ne <4 x i32> %mask, zeroinitializer
+  %1 = select <4 x i1> %0, <4 x float> %src1, <4 x float> %src2
+  ret <4 x float> %1
+}
+
+define arm_aapcs_vfpcc <4 x i32> @foo(<4 x i32> %vec.ind) {
+; CHECK-LABEL: foo:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmov.i32 q2, #0x1
+; CHECK-NEXT:    vmov.i32 q1, #0x0
+; CHECK-NEXT:    vand q2, q0, q2
+; CHECK-NEXT:    vcmp.i32 eq, q2, zr
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+  %tmp = and <4 x i32> %vec.ind, <i32 1, i32 1, i32 1, i32 1>
+  %tmp1 = icmp eq <4 x i32> %tmp, zeroinitializer
+  %tmp2 = select <4 x i1> %tmp1, <4 x i32> %vec.ind, <4 x i32> zeroinitializer
+  ret <4 x i32> %tmp2
+}
diff --git a/llvm/test/CodeGen/Thumb2/mve-vpt-from-intrinsics.ll b/llvm/test/CodeGen/Thumb2/mve-vpt-from-intrinsics.ll
new file mode 100644
index 000000000..07e5185b7
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-vpt-from-intrinsics.ll
@@ -0,0 +1,45 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: opt -instcombine %s | llc -mtriple=thumbv8.1m.main-none-eabi -mattr=+mve --verify-machineinstrs -o - | FileCheck %s
+
+target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
+
+define arm_aapcs_vfpcc <8 x i16> @test_vpt_block(<8 x i16> %v_inactive, <8 x i16> %v1, <8 x i16> %v2, <8 x i16> %v3) {
+; CHECK-LABEL: test_vpt_block:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i16 eq, q1, q2
+; CHECK-NEXT:    vaddt.i16 q0, q3, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = icmp eq <8 x i16> %v1, %v2
+  %1 = call i32 @llvm.arm.mve.pred.v2i.v8i1(<8 x i1> %0)
+  %2 = trunc i32 %1 to i16
+  %3 = zext i16 %2 to i32
+  %4 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %3)
+  %5 = call <8 x i16> @llvm.arm.mve.add.predicated.v8i16.v8i1(<8 x i16> %v3, <8 x i16> %v2, <8 x i1> %4, <8 x i16> %v_inactive)
+  ret <8 x i16> %5
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vpnot(<8 x i16> %v, <8 x i16> %w, <8 x i16> %x, i32 %n) {
+; CHECK-LABEL: test_vpnot:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpnot
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vaddt.i16 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %1 = call i32 @llvm.arm.mve.pred.v2i.v8i1(<8 x i1> %0)
+  %2 = trunc i32 %1 to i16
+  %3 = xor i16 %2, -1
+  %4 = zext i16 %3 to i32
+  %5 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %4)
+  %6 = call <8 x i16> @llvm.arm.mve.add.predicated.v8i16.v8i1(<8 x i16> %w, <8 x i16> %x, <8 x i1> %5, <8 x i16> %v)
+  ret <8 x i16> %6
+}
+
+declare i32 @llvm.arm.mve.pred.v2i.v8i1(<8 x i1>)
+declare <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32)
+declare <8 x i16> @llvm.arm.mve.add.predicated.v8i16.v8i1(<8 x i16>, <8 x i16>, <8 x i1>, <8 x i16>)
+declare <8 x i1> @llvm.arm.mve.vctp16(i32)
+
diff --git a/llvm/test/CodeGen/Thumb2/mve-vst2.ll b/llvm/test/CodeGen/Thumb2/mve-vst2.ll
new file mode 100644
index 000000000..afe3935cc
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-vst2.ll
@@ -0,0 +1,606 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve.fp,+fp64 -verify-machineinstrs %s -o - | FileCheck %s
+
+; i32
+
+define void @vst2_v2i32(<2 x i32> *%src, <4 x i32> *%dst) {
+; CHECK-LABEL: vst2_v2i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    ldrd r2, r12, [r0]
+; CHECK-NEXT:    ldrd r3, r0, [r0, #8]
+; CHECK-NEXT:    vmov.32 q0[0], r2
+; CHECK-NEXT:    vmov.32 q1[0], r3
+; CHECK-NEXT:    vmov.32 q0[2], r12
+; CHECK-NEXT:    vmov.f64 d4, d1
+; CHECK-NEXT:    vmov.32 q1[2], r0
+; CHECK-NEXT:    vmov.f32 s9, s3
+; CHECK-NEXT:    vmov.f32 s2, s4
+; CHECK-NEXT:    vmov.f32 s3, s5
+; CHECK-NEXT:    vmov.f32 s10, s6
+; CHECK-NEXT:    vmov.f32 s1, s2
+; CHECK-NEXT:    vmov.f32 s11, s7
+; CHECK-NEXT:    vmov.f32 s2, s8
+; CHECK-NEXT:    vmov.f32 s3, s10
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %s1 = getelementptr <2 x i32>, <2 x i32>* %src, i32 0
+  %l1 = load <2 x i32>, <2 x i32>* %s1, align 4
+  %s2 = getelementptr <2 x i32>, <2 x i32>* %src, i32 1
+  %l2 = load <2 x i32>, <2 x i32>* %s2, align 4
+  %s = shufflevector <2 x i32> %l1, <2 x i32> %l2, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+  store <4 x i32> %s, <4 x i32> *%dst
+  ret void
+}
+
+define void @vst2_v4i32(<4 x i32> *%src, <8 x i32> *%dst) {
+; CHECK-LABEL: vst2_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #16]
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vst20.32 {q0, q1}, [r1]
+; CHECK-NEXT:    vst21.32 {q0, q1}, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %s1 = getelementptr <4 x i32>, <4 x i32>* %src, i32 0
+  %l1 = load <4 x i32>, <4 x i32>* %s1, align 4
+  %s2 = getelementptr <4 x i32>, <4 x i32>* %src, i32 1
+  %l2 = load <4 x i32>, <4 x i32>* %s2, align 4
+  %s = shufflevector <4 x i32> %l1, <4 x i32> %l2, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+  store <8 x i32> %s, <8 x i32> *%dst
+  ret void
+}
+
+define void @vst2_v8i32(<8 x i32> *%src, <16 x i32> *%dst) {
+; CHECK-LABEL: vst2_v8i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #32]
+; CHECK-NEXT:    vldrw.u32 q3, [r0, #48]
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #16]
+; CHECK-NEXT:    add.w r0, r1, #32
+; CHECK-NEXT:    vst20.32 {q0, q1}, [r1]
+; CHECK-NEXT:    vst21.32 {q0, q1}, [r1]
+; CHECK-NEXT:    vst20.32 {q2, q3}, [r0]
+; CHECK-NEXT:    vst21.32 {q2, q3}, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %s1 = getelementptr <8 x i32>, <8 x i32>* %src, i32 0
+  %l1 = load <8 x i32>, <8 x i32>* %s1, align 4
+  %s2 = getelementptr <8 x i32>, <8 x i32>* %src, i32 1
+  %l2 = load <8 x i32>, <8 x i32>* %s2, align 4
+  %s = shufflevector <8 x i32> %l1, <8 x i32> %l2, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
+  store <16 x i32> %s, <16 x i32> *%dst
+  ret void
+}
+
+define void @vst2_v16i32(<16 x i32> *%src, <32 x i32> *%dst) {
+; CHECK-LABEL: vst2_v16i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #112]
+; CHECK-NEXT:    vldrw.u32 q3, [r0, #96]
+; CHECK-NEXT:    vldrw.u32 q5, [r0, #80]
+; CHECK-NEXT:    vldrw.u32 q7, [r0, #64]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #48]
+; CHECK-NEXT:    vldrw.u32 q6, [r0]
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #32]
+; CHECK-NEXT:    vldrw.u32 q4, [r0, #16]
+; CHECK-NEXT:    add.w r0, r1, #96
+; CHECK-NEXT:    add.w r2, r1, #64
+; CHECK-NEXT:    add.w r3, r1, #32
+; CHECK-NEXT:    vst20.32 {q6, q7}, [r1]
+; CHECK-NEXT:    vst21.32 {q6, q7}, [r1]
+; CHECK-NEXT:    vst20.32 {q4, q5}, [r3]
+; CHECK-NEXT:    vst21.32 {q4, q5}, [r3]
+; CHECK-NEXT:    vst20.32 {q2, q3}, [r2]
+; CHECK-NEXT:    vst21.32 {q2, q3}, [r2]
+; CHECK-NEXT:    vst20.32 {q0, q1}, [r0]
+; CHECK-NEXT:    vst21.32 {q0, q1}, [r0]
+; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    bx lr
+entry:
+  %s1 = getelementptr <16 x i32>, <16 x i32>* %src, i32 0
+  %l1 = load <16 x i32>, <16 x i32>* %s1, align 4
+  %s2 = getelementptr <16 x i32>, <16 x i32>* %src, i32 1
+  %l2 = load <16 x i32>, <16 x i32>* %s2, align 4
+  %s = shufflevector <16 x i32> %l1, <16 x i32> %l2, <32 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23, i32 8, i32 24, i32 9, i32 25, i32 10, i32 26, i32 11, i32 27, i32 12, i32 28, i32 13, i32 29, i32 14, i32 30, i32 15, i32 31>
+  store <32 x i32> %s, <32 x i32> *%dst
+  ret void
+}
+
+; i16
+
+define void @vst2_v2i16(<2 x i16> *%src, <4 x i16> *%dst) {
+; CHECK-LABEL: vst2_v2i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    ldrh r3, [r0]
+; CHECK-NEXT:    ldrh r2, [r0, #4]
+; CHECK-NEXT:    vmov.32 q0[0], r3
+; CHECK-NEXT:    ldrh.w r12, [r0, #6]
+; CHECK-NEXT:    ldrh r0, [r0, #2]
+; CHECK-NEXT:    vmov.32 q0[1], r2
+; CHECK-NEXT:    vmov.32 q0[2], r0
+; CHECK-NEXT:    vmov.32 q0[3], r12
+; CHECK-NEXT:    vstrh.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %s1 = getelementptr <2 x i16>, <2 x i16>* %src, i32 0
+  %l1 = load <2 x i16>, <2 x i16>* %s1, align 4
+  %s2 = getelementptr <2 x i16>, <2 x i16>* %src, i32 1
+  %l2 = load <2 x i16>, <2 x i16>* %s2, align 4
+  %s = shufflevector <2 x i16> %l1, <2 x i16> %l2, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+  store <4 x i16> %s, <4 x i16> *%dst
+  ret void
+}
+
+define void @vst2_v4i16(<4 x i16> *%src, <8 x i16> *%dst) {
+; CHECK-LABEL: vst2_v4i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u32 q0, [r0, #8]
+; CHECK-NEXT:    vldrh.u32 q1, [r0]
+; CHECK-NEXT:    vmovnt.i32 q1, q0
+; CHECK-NEXT:    vstrw.32 q1, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %s1 = getelementptr <4 x i16>, <4 x i16>* %src, i32 0
+  %l1 = load <4 x i16>, <4 x i16>* %s1, align 4
+  %s2 = getelementptr <4 x i16>, <4 x i16>* %src, i32 1
+  %l2 = load <4 x i16>, <4 x i16>* %s2, align 4
+  %s = shufflevector <4 x i16> %l1, <4 x i16> %l2, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+  store <8 x i16> %s, <8 x i16> *%dst
+  ret void
+}
+
+define void @vst2_v8i16(<8 x i16> *%src, <16 x i16> *%dst) {
+; CHECK-LABEL: vst2_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #16]
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vst20.16 {q0, q1}, [r1]
+; CHECK-NEXT:    vst21.16 {q0, q1}, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %s1 = getelementptr <8 x i16>, <8 x i16>* %src, i32 0
+  %l1 = load <8 x i16>, <8 x i16>* %s1, align 4
+  %s2 = getelementptr <8 x i16>, <8 x i16>* %src, i32 1
+  %l2 = load <8 x i16>, <8 x i16>* %s2, align 4
+  %s = shufflevector <8 x i16> %l1, <8 x i16> %l2, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
+  store <16 x i16> %s, <16 x i16> *%dst
+  ret void
+}
+
+define void @vst2_v16i16(<16 x i16> *%src, <32 x i16> *%dst) {
+; CHECK-LABEL: vst2_v16i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #32]
+; CHECK-NEXT:    vldrw.u32 q3, [r0, #48]
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #16]
+; CHECK-NEXT:    add.w r0, r1, #32
+; CHECK-NEXT:    vst20.16 {q0, q1}, [r1]
+; CHECK-NEXT:    vst21.16 {q0, q1}, [r1]
+; CHECK-NEXT:    vst20.16 {q2, q3}, [r0]
+; CHECK-NEXT:    vst21.16 {q2, q3}, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %s1 = getelementptr <16 x i16>, <16 x i16>* %src, i32 0
+  %l1 = load <16 x i16>, <16 x i16>* %s1, align 4
+  %s2 = getelementptr <16 x i16>, <16 x i16>* %src, i32 1
+  %l2 = load <16 x i16>, <16 x i16>* %s2, align 4
+  %s = shufflevector <16 x i16> %l1, <16 x i16> %l2, <32 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23, i32 8, i32 24, i32 9, i32 25, i32 10, i32 26, i32 11, i32 27, i32 12, i32 28, i32 13, i32 29, i32 14, i32 30, i32 15, i32 31>
+  store <32 x i16> %s, <32 x i16> *%dst
+  ret void
+}
+
+; i8
+
+define void @vst2_v2i8(<2 x i8> *%src, <4 x i8> *%dst) {
+; CHECK-LABEL: vst2_v2i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    ldrb r2, [r0]
+; CHECK-NEXT:    ldrb r3, [r0, #2]
+; CHECK-NEXT:    vmov.32 q0[0], r2
+; CHECK-NEXT:    ldrb.w r12, [r0, #1]
+; CHECK-NEXT:    vmov.32 q0[1], r3
+; CHECK-NEXT:    ldrb r0, [r0, #3]
+; CHECK-NEXT:    vmov.32 q0[2], r12
+; CHECK-NEXT:    vmov.32 q0[3], r0
+; CHECK-NEXT:    vstrb.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %s1 = getelementptr <2 x i8>, <2 x i8>* %src, i32 0
+  %l1 = load <2 x i8>, <2 x i8>* %s1, align 4
+  %s2 = getelementptr <2 x i8>, <2 x i8>* %src, i32 1
+  %l2 = load <2 x i8>, <2 x i8>* %s2, align 4
+  %s = shufflevector <2 x i8> %l1, <2 x i8> %l2, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+  store <4 x i8> %s, <4 x i8> *%dst
+  ret void
+}
+
+define void @vst2_v4i8(<4 x i8> *%src, <8 x i8> *%dst) {
+; CHECK-LABEL: vst2_v4i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u32 q0, [r0, #4]
+; CHECK-NEXT:    vldrb.u32 q1, [r0]
+; CHECK-NEXT:    vmovnt.i32 q1, q0
+; CHECK-NEXT:    vstrb.16 q1, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %s1 = getelementptr <4 x i8>, <4 x i8>* %src, i32 0
+  %l1 = load <4 x i8>, <4 x i8>* %s1, align 4
+  %s2 = getelementptr <4 x i8>, <4 x i8>* %src, i32 1
+  %l2 = load <4 x i8>, <4 x i8>* %s2, align 4
+  %s = shufflevector <4 x i8> %l1, <4 x i8> %l2, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+  store <8 x i8> %s, <8 x i8> *%dst
+  ret void
+}
+
+define void @vst2_v8i8(<8 x i8> *%src, <16 x i8> *%dst) {
+; CHECK-LABEL: vst2_v8i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u16 q0, [r0, #8]
+; CHECK-NEXT:    vldrb.u16 q1, [r0]
+; CHECK-NEXT:    vmovnt.i16 q1, q0
+; CHECK-NEXT:    vstrw.32 q1, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %s1 = getelementptr <8 x i8>, <8 x i8>* %src, i32 0
+  %l1 = load <8 x i8>, <8 x i8>* %s1, align 4
+  %s2 = getelementptr <8 x i8>, <8 x i8>* %src, i32 1
+  %l2 = load <8 x i8>, <8 x i8>* %s2, align 4
+  %s = shufflevector <8 x i8> %l1, <8 x i8> %l2, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
+  store <16 x i8> %s, <16 x i8> *%dst
+  ret void
+}
+
+define void @vst2_v16i8(<16 x i8> *%src, <32 x i8> *%dst) {
+; CHECK-LABEL: vst2_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #16]
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vst20.8 {q0, q1}, [r1]
+; CHECK-NEXT:    vst21.8 {q0, q1}, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %s1 = getelementptr <16 x i8>, <16 x i8>* %src, i32 0
+  %l1 = load <16 x i8>, <16 x i8>* %s1, align 4
+  %s2 = getelementptr <16 x i8>, <16 x i8>* %src, i32 1
+  %l2 = load <16 x i8>, <16 x i8>* %s2, align 4
+  %s = shufflevector <16 x i8> %l1, <16 x i8> %l2, <32 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23, i32 8, i32 24, i32 9, i32 25, i32 10, i32 26, i32 11, i32 27, i32 12, i32 28, i32 13, i32 29, i32 14, i32 30, i32 15, i32 31>
+  store <32 x i8> %s, <32 x i8> *%dst
+  ret void
+}
+
+; i64
+
+define void @vst2_v2i64(<2 x i64> *%src, <4 x i64> *%dst) {
+; CHECK-LABEL: vst2_v2i64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #16]
+; CHECK-NEXT:    vmov.f64 d4, d2
+; CHECK-NEXT:    vmov.f32 s9, s5
+; CHECK-NEXT:    vmov.f32 s10, s0
+; CHECK-NEXT:    vmov.f32 s11, s1
+; CHECK-NEXT:    vmov.f32 s0, s6
+; CHECK-NEXT:    vstrb.8 q2, [r1], #16
+; CHECK-NEXT:    vmov.f32 s1, s7
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %s1 = getelementptr <2 x i64>, <2 x i64>* %src, i32 0
+  %l1 = load <2 x i64>, <2 x i64>* %s1, align 4
+  %s2 = getelementptr <2 x i64>, <2 x i64>* %src, i32 1
+  %l2 = load <2 x i64>, <2 x i64>* %s2, align 4
+  %s = shufflevector <2 x i64> %l1, <2 x i64> %l2, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+  store <4 x i64> %s, <4 x i64> *%dst
+  ret void
+}
+
+define void @vst2_v4i64(<4 x i64> *%src, <8 x i64> *%dst) {
+; CHECK-LABEL: vst2_v4i64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11}
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #16]
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #32]
+; CHECK-NEXT:    vldrw.u32 q4, [r0, #48]
+; CHECK-NEXT:    vmov.f64 d6, d1
+; CHECK-NEXT:    mov r0, r1
+; CHECK-NEXT:    vmov.f64 d10, d3
+; CHECK-NEXT:    vmov.f32 s13, s3
+; CHECK-NEXT:    vmov.f32 s21, s7
+; CHECK-NEXT:    vmov.f32 s6, s8
+; CHECK-NEXT:    vmov.f32 s14, s18
+; CHECK-NEXT:    vmov.f32 s22, s10
+; CHECK-NEXT:    vmov.f32 s2, s16
+; CHECK-NEXT:    vmov.f32 s7, s9
+; CHECK-NEXT:    vmov.f32 s23, s11
+; CHECK-NEXT:    vstrb.8 q1, [r0], #48
+; CHECK-NEXT:    vmov.f32 s15, s19
+; CHECK-NEXT:    vstrw.32 q5, [r1, #16]
+; CHECK-NEXT:    vmov.f32 s3, s17
+; CHECK-NEXT:    vstrw.32 q3, [r0]
+; CHECK-NEXT:    vstrw.32 q0, [r1, #32]
+; CHECK-NEXT:    vpop {d8, d9, d10, d11}
+; CHECK-NEXT:    bx lr
+entry:
+  %s1 = getelementptr <4 x i64>, <4 x i64>* %src, i32 0
+  %l1 = load <4 x i64>, <4 x i64>* %s1, align 4
+  %s2 = getelementptr <4 x i64>, <4 x i64>* %src, i32 1
+  %l2 = load <4 x i64>, <4 x i64>* %s2, align 4
+  %s = shufflevector <4 x i64> %l1, <4 x i64> %l2, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+  store <8 x i64> %s, <8 x i64> *%dst
+  ret void
+}
+
+; f32
+
+define void @vst2_v2f32(<2 x float> *%src, <4 x float> *%dst) {
+; CHECK-LABEL: vst2_v2f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldr s0, [r0]
+; CHECK-NEXT:    vldr s4, [r0, #4]
+; CHECK-NEXT:    vldr s1, [r0, #8]
+; CHECK-NEXT:    vldr s5, [r0, #12]
+; CHECK-NEXT:    vmov.f32 s2, s4
+; CHECK-NEXT:    vmov.f32 s3, s5
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %s1 = getelementptr <2 x float>, <2 x float>* %src, i32 0
+  %l1 = load <2 x float>, <2 x float>* %s1, align 4
+  %s2 = getelementptr <2 x float>, <2 x float>* %src, i32 1
+  %l2 = load <2 x float>, <2 x float>* %s2, align 4
+  %s = shufflevector <2 x float> %l1, <2 x float> %l2, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+  store <4 x float> %s, <4 x float> *%dst
+  ret void
+}
+
+define void @vst2_v4f32(<4 x float> *%src, <8 x float> *%dst) {
+; CHECK-LABEL: vst2_v4f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #16]
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vst20.32 {q0, q1}, [r1]
+; CHECK-NEXT:    vst21.32 {q0, q1}, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %s1 = getelementptr <4 x float>, <4 x float>* %src, i32 0
+  %l1 = load <4 x float>, <4 x float>* %s1, align 4
+  %s2 = getelementptr <4 x float>, <4 x float>* %src, i32 1
+  %l2 = load <4 x float>, <4 x float>* %s2, align 4
+  %s = shufflevector <4 x float> %l1, <4 x float> %l2, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+  store <8 x float> %s, <8 x float> *%dst
+  ret void
+}
+
+define void @vst2_v8f32(<8 x float> *%src, <16 x float> *%dst) {
+; CHECK-LABEL: vst2_v8f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #32]
+; CHECK-NEXT:    vldrw.u32 q3, [r0, #48]
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #16]
+; CHECK-NEXT:    add.w r0, r1, #32
+; CHECK-NEXT:    vst20.32 {q0, q1}, [r1]
+; CHECK-NEXT:    vst21.32 {q0, q1}, [r1]
+; CHECK-NEXT:    vst20.32 {q2, q3}, [r0]
+; CHECK-NEXT:    vst21.32 {q2, q3}, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %s1 = getelementptr <8 x float>, <8 x float>* %src, i32 0
+  %l1 = load <8 x float>, <8 x float>* %s1, align 4
+  %s2 = getelementptr <8 x float>, <8 x float>* %src, i32 1
+  %l2 = load <8 x float>, <8 x float>* %s2, align 4
+  %s = shufflevector <8 x float> %l1, <8 x float> %l2, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
+  store <16 x float> %s, <16 x float> *%dst
+  ret void
+}
+
+define void @vst2_v16f32(<16 x float> *%src, <32 x float> *%dst) {
+; CHECK-LABEL: vst2_v16f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #112]
+; CHECK-NEXT:    vldrw.u32 q3, [r0, #96]
+; CHECK-NEXT:    vldrw.u32 q5, [r0, #80]
+; CHECK-NEXT:    vldrw.u32 q7, [r0, #64]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #48]
+; CHECK-NEXT:    vldrw.u32 q6, [r0]
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #32]
+; CHECK-NEXT:    vldrw.u32 q4, [r0, #16]
+; CHECK-NEXT:    add.w r0, r1, #96
+; CHECK-NEXT:    add.w r2, r1, #64
+; CHECK-NEXT:    add.w r3, r1, #32
+; CHECK-NEXT:    vst20.32 {q6, q7}, [r1]
+; CHECK-NEXT:    vst21.32 {q6, q7}, [r1]
+; CHECK-NEXT:    vst20.32 {q4, q5}, [r3]
+; CHECK-NEXT:    vst21.32 {q4, q5}, [r3]
+; CHECK-NEXT:    vst20.32 {q2, q3}, [r2]
+; CHECK-NEXT:    vst21.32 {q2, q3}, [r2]
+; CHECK-NEXT:    vst20.32 {q0, q1}, [r0]
+; CHECK-NEXT:    vst21.32 {q0, q1}, [r0]
+; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    bx lr
+entry:
+  %s1 = getelementptr <16 x float>, <16 x float>* %src, i32 0
+  %l1 = load <16 x float>, <16 x float>* %s1, align 4
+  %s2 = getelementptr <16 x float>, <16 x float>* %src, i32 1
+  %l2 = load <16 x float>, <16 x float>* %s2, align 4
+  %s = shufflevector <16 x float> %l1, <16 x float> %l2, <32 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23, i32 8, i32 24, i32 9, i32 25, i32 10, i32 26, i32 11, i32 27, i32 12, i32 28, i32 13, i32 29, i32 14, i32 30, i32 15, i32 31>
+  store <32 x float> %s, <32 x float> *%dst
+  ret void
+}
+
+; f16
+
+define void @vst2_v2f16(<2 x half> *%src, <4 x half> *%dst) {
+; CHECK-LABEL: vst2_v2f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    ldrd r2, r0, [r0]
+; CHECK-NEXT:    vmov.32 q0[0], r2
+; CHECK-NEXT:    vmov.32 q1[0], r0
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vmovx.f16 s0, s0
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmov.16 q2[0], r2
+; CHECK-NEXT:    vmov.16 q2[1], r0
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vmovx.f16 s0, s4
+; CHECK-NEXT:    vmov.16 q2[2], r0
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vmov.16 q2[3], r0
+; CHECK-NEXT:    vmov r2, s9
+; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    strd r0, r2, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %s1 = getelementptr <2 x half>, <2 x half>* %src, i32 0
+  %l1 = load <2 x half>, <2 x half>* %s1, align 4
+  %s2 = getelementptr <2 x half>, <2 x half>* %src, i32 1
+  %l2 = load <2 x half>, <2 x half>* %s2, align 4
+  %s = shufflevector <2 x half> %l1, <2 x half> %l2, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+  store <4 x half> %s, <4 x half> *%dst
+  ret void
+}
+
+define void @vst2_v4f16(<4 x half> *%src, <8 x half> *%dst) {
+; CHECK-LABEL: vst2_v4f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    ldm.w r0, {r2, r3, r12}
+; CHECK-NEXT:    vmov.32 q0[0], r12
+; CHECK-NEXT:    ldr r0, [r0, #12]
+; CHECK-NEXT:    vmov.32 q2[0], r2
+; CHECK-NEXT:    vmov.32 q2[1], r3
+; CHECK-NEXT:    vmov.32 q0[1], r0
+; CHECK-NEXT:    vmov r2, s8
+; CHECK-NEXT:    vmovx.f16 s12, s8
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vmov.16 q1[0], r2
+; CHECK-NEXT:    vmov.16 q1[1], r0
+; CHECK-NEXT:    vmov r0, s12
+; CHECK-NEXT:    vmovx.f16 s12, s0
+; CHECK-NEXT:    vmov.16 q1[2], r0
+; CHECK-NEXT:    vmov r0, s12
+; CHECK-NEXT:    vmovx.f16 s8, s9
+; CHECK-NEXT:    vmov.16 q1[3], r0
+; CHECK-NEXT:    vmov r0, s9
+; CHECK-NEXT:    vmov.16 q1[4], r0
+; CHECK-NEXT:    vmov r0, s1
+; CHECK-NEXT:    vmov.16 q1[5], r0
+; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    vmovx.f16 s0, s1
+; CHECK-NEXT:    vmov.16 q1[6], r0
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vmov.16 q1[7], r0
+; CHECK-NEXT:    vstrw.32 q1, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %s1 = getelementptr <4 x half>, <4 x half>* %src, i32 0
+  %l1 = load <4 x half>, <4 x half>* %s1, align 4
+  %s2 = getelementptr <4 x half>, <4 x half>* %src, i32 1
+  %l2 = load <4 x half>, <4 x half>* %s2, align 4
+  %s = shufflevector <4 x half> %l1, <4 x half> %l2, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+  store <8 x half> %s, <8 x half> *%dst
+  ret void
+}
+
+define void @vst2_v8f16(<8 x half> *%src, <16 x half> *%dst) {
+; CHECK-LABEL: vst2_v8f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #16]
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vst20.16 {q0, q1}, [r1]
+; CHECK-NEXT:    vst21.16 {q0, q1}, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %s1 = getelementptr <8 x half>, <8 x half>* %src, i32 0
+  %l1 = load <8 x half>, <8 x half>* %s1, align 4
+  %s2 = getelementptr <8 x half>, <8 x half>* %src, i32 1
+  %l2 = load <8 x half>, <8 x half>* %s2, align 4
+  %s = shufflevector <8 x half> %l1, <8 x half> %l2, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
+  store <16 x half> %s, <16 x half> *%dst
+  ret void
+}
+
+define void @vst2_v16f16(<16 x half> *%src, <32 x half> *%dst) {
+; CHECK-LABEL: vst2_v16f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #48]
+; CHECK-NEXT:    vldrw.u32 q3, [r0, #32]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #16]
+; CHECK-NEXT:    vldrw.u32 q2, [r0]
+; CHECK-NEXT:    add.w r0, r1, #32
+; CHECK-NEXT:    vst20.16 {q2, q3}, [r1]
+; CHECK-NEXT:    vst21.16 {q2, q3}, [r1]
+; CHECK-NEXT:    vst20.16 {q0, q1}, [r0]
+; CHECK-NEXT:    vst21.16 {q0, q1}, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %s1 = getelementptr <16 x half>, <16 x half>* %src, i32 0
+  %l1 = load <16 x half>, <16 x half>* %s1, align 4
+  %s2 = getelementptr <16 x half>, <16 x half>* %src, i32 1
+  %l2 = load <16 x half>, <16 x half>* %s2, align 4
+  %s = shufflevector <16 x half> %l1, <16 x half> %l2, <32 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23, i32 8, i32 24, i32 9, i32 25, i32 10, i32 26, i32 11, i32 27, i32 12, i32 28, i32 13, i32 29, i32 14, i32 30, i32 15, i32 31>
+  store <32 x half> %s, <32 x half> *%dst
+  ret void
+}
+
+; f64
+
+define void @vst2_v2f64(<2 x double> *%src, <4 x double> *%dst) {
+; CHECK-LABEL: vst2_v2f64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #16]
+; CHECK-NEXT:    vmov.f64 d4, d3
+; CHECK-NEXT:    vmov.f64 d5, d1
+; CHECK-NEXT:    vmov.f64 d3, d0
+; CHECK-NEXT:    vstrw.32 q2, [r1, #16]
+; CHECK-NEXT:    vstrw.32 q1, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %s1 = getelementptr <2 x double>, <2 x double>* %src, i32 0
+  %l1 = load <2 x double>, <2 x double>* %s1, align 4
+  %s2 = getelementptr <2 x double>, <2 x double>* %src, i32 1
+  %l2 = load <2 x double>, <2 x double>* %s2, align 4
+  %s = shufflevector <2 x double> %l1, <2 x double> %l2, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+  store <4 x double> %s, <4 x double> *%dst
+  ret void
+}
+
+define void @vst2_v4f64(<4 x double> *%src, <8 x double> *%dst) {
+; CHECK-LABEL: vst2_v4f64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9}
+; CHECK-NEXT:    vpush {d8, d9}
+; CHECK-NEXT:    vldrw.u32 q2, [r0]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #32]
+; CHECK-NEXT:    vldrw.u32 q3, [r0, #16]
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #48]
+; CHECK-NEXT:    vmov.f64 d8, d4
+; CHECK-NEXT:    vmov.f64 d9, d0
+; CHECK-NEXT:    vmov.f64 d0, d5
+; CHECK-NEXT:    vstrw.32 q4, [r1]
+; CHECK-NEXT:    vmov.f64 d4, d6
+; CHECK-NEXT:    vstrw.32 q0, [r1, #16]
+; CHECK-NEXT:    vmov.f64 d5, d2
+; CHECK-NEXT:    vmov.f64 d2, d7
+; CHECK-NEXT:    vstrw.32 q2, [r1, #32]
+; CHECK-NEXT:    vstrw.32 q1, [r1, #48]
+; CHECK-NEXT:    vpop {d8, d9}
+; CHECK-NEXT:    bx lr
+entry:
+  %s1 = getelementptr <4 x double>, <4 x double>* %src, i32 0
+  %l1 = load <4 x double>, <4 x double>* %s1, align 4
+  %s2 = getelementptr <4 x double>, <4 x double>* %src, i32 1
+  %l2 = load <4 x double>, <4 x double>* %s2, align 4
+  %s = shufflevector <4 x double> %l1, <4 x double> %l2, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+  store <8 x double> %s, <8 x double> *%dst
+  ret void
+}
diff --git a/llvm/test/CodeGen/Thumb2/mve-vst3.ll b/llvm/test/CodeGen/Thumb2/mve-vst3.ll
new file mode 100644
index 000000000..97c595c01
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-vst3.ll
@@ -0,0 +1,1967 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve.fp,+fp64 -verify-machineinstrs %s -o - | FileCheck %s
+
+; i32
+
+define void @vst3_v2i32(<2 x i32> *%src, <6 x i32> *%dst) {
+; CHECK-LABEL: vst3_v2i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, lr}
+; CHECK-NEXT:    push {r4, lr}
+; CHECK-NEXT:    ldm.w r0, {r2, r3, r12, lr}
+; CHECK-NEXT:    ldrd r4, r0, [r0, #16]
+; CHECK-NEXT:    vmov.32 q1[0], r2
+; CHECK-NEXT:    vmov.32 q1[1], r3
+; CHECK-NEXT:    vmov.32 q0[0], r4
+; CHECK-NEXT:    vmov.32 q1[2], r12
+; CHECK-NEXT:    vmov.32 q0[1], r0
+; CHECK-NEXT:    vmov.32 q1[3], lr
+; CHECK-NEXT:    vmov.f32 s8, s7
+; CHECK-NEXT:    vmov.f32 s10, s1
+; CHECK-NEXT:    vmov r2, s8
+; CHECK-NEXT:    vmov r0, s10
+; CHECK-NEXT:    vmov.f64 d4, d2
+; CHECK-NEXT:    vmov.f32 s9, s6
+; CHECK-NEXT:    vmov.f32 s10, s0
+; CHECK-NEXT:    vmov.f32 s11, s5
+; CHECK-NEXT:    strd r2, r0, [r1, #16]
+; CHECK-NEXT:    vstrw.32 q2, [r1]
+; CHECK-NEXT:    pop {r4, pc}
+entry:
+  %s1 = getelementptr <2 x i32>, <2 x i32>* %src, i32 0
+  %l1 = load <2 x i32>, <2 x i32>* %s1, align 4
+  %s2 = getelementptr <2 x i32>, <2 x i32>* %src, i32 1
+  %l2 = load <2 x i32>, <2 x i32>* %s2, align 4
+  %s3 = getelementptr <2 x i32>, <2 x i32>* %src, i32 2
+  %l3 = load <2 x i32>, <2 x i32>* %s3, align 4
+  %t1 = shufflevector <2 x i32> %l1, <2 x i32> %l2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %t2 = shufflevector <2 x i32> %l3, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+  %s = shufflevector <4 x i32> %t1, <4 x i32> %t2, <6 x i32> <i32 0, i32 2, i32 4, i32 1, i32 3, i32 5>
+  store <6 x i32> %s, <6 x i32> *%dst
+  ret void
+}
+
+define void @vst3_v4i32(<4 x i32> *%src, <12 x i32> *%dst) {
+; CHECK-LABEL: vst3_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11}
+; CHECK-NEXT:    vldrw.u32 q4, [r0]
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #16]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #32]
+; CHECK-NEXT:    vmov.f32 s4, s9
+; CHECK-NEXT:    vmov.f64 d6, d8
+; CHECK-NEXT:    vmov.32 r0, q0[0]
+; CHECK-NEXT:    vdup.32 q5, r0
+; CHECK-NEXT:    vmov.32 r0, q2[3]
+; CHECK-NEXT:    vmov.f32 s5, s1
+; CHECK-NEXT:    vmov.f32 s13, s8
+; CHECK-NEXT:    vmov.f32 s0, s2
+; CHECK-NEXT:    vmov.f32 s7, s10
+; CHECK-NEXT:    vdup.32 q2, r0
+; CHECK-NEXT:    vmov.f32 s15, s17
+; CHECK-NEXT:    vmov.f32 s1, s19
+; CHECK-NEXT:    vmov.f32 s6, s18
+; CHECK-NEXT:    vmov.f32 s14, s22
+; CHECK-NEXT:    vstrw.32 q1, [r1, #16]
+; CHECK-NEXT:    vmov.f32 s2, s10
+; CHECK-NEXT:    vstrw.32 q3, [r1]
+; CHECK-NEXT:    vstrw.32 q0, [r1, #32]
+; CHECK-NEXT:    vpop {d8, d9, d10, d11}
+; CHECK-NEXT:    bx lr
+entry:
+  %s1 = getelementptr <4 x i32>, <4 x i32>* %src, i32 0
+  %l1 = load <4 x i32>, <4 x i32>* %s1, align 4
+  %s2 = getelementptr <4 x i32>, <4 x i32>* %src, i32 1
+  %l2 = load <4 x i32>, <4 x i32>* %s2, align 4
+  %s3 = getelementptr <4 x i32>, <4 x i32>* %src, i32 2
+  %l3 = load <4 x i32>, <4 x i32>* %s3, align 4
+  %t1 = shufflevector <4 x i32> %l1, <4 x i32> %l2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %t2 = shufflevector <4 x i32> %l3, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+  %s = shufflevector <8 x i32> %t1, <8 x i32> %t2, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+  store <12 x i32> %s, <12 x i32> *%dst
+  ret void
+}
+
+define void @vst3_v8i32(<8 x i32> *%src, <24 x i32> *%dst) {
+; CHECK-LABEL: vst3_v8i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    .pad #48
+; CHECK-NEXT:    sub sp, #48
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #80]
+; CHECK-NEXT:    vldrw.u32 q5, [r0, #16]
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #48]
+; CHECK-NEXT:    vmov.f64 d6, d1
+; CHECK-NEXT:    vmov.32 r2, q1[3]
+; CHECK-NEXT:    vldrw.u32 q4, [r0]
+; CHECK-NEXT:    vldrw.u32 q6, [r0, #32]
+; CHECK-NEXT:    vdup.32 q2, r2
+; CHECK-NEXT:    vldrw.u32 q7, [r0, #64]
+; CHECK-NEXT:    vstrw.32 q4, [sp, #32] @ 16-byte Spill
+; CHECK-NEXT:    vmov.f32 s13, s23
+; CHECK-NEXT:    vstrw.32 q6, [sp, #16] @ 16-byte Spill
+; CHECK-NEXT:    vmov.f32 s15, s3
+; CHECK-NEXT:    vstrw.32 q5, [sp] @ 16-byte Spill
+; CHECK-NEXT:    vmov.f32 s14, s10
+; CHECK-NEXT:    vmov.32 r0, q7[0]
+; CHECK-NEXT:    vmov.f64 d4, d8
+; CHECK-NEXT:    vmov.f32 s9, s24
+; CHECK-NEXT:    vmov.f32 s11, s17
+; CHECK-NEXT:    vmov q4, q5
+; CHECK-NEXT:    vmov.f32 s21, s4
+; CHECK-NEXT:    vmov.f32 s23, s17
+; CHECK-NEXT:    vdup.32 q4, r0
+; CHECK-NEXT:    vmov.32 r0, q0[0]
+; CHECK-NEXT:    vmov.f32 s0, s5
+; CHECK-NEXT:    vmov.f32 s10, s18
+; CHECK-NEXT:    vldrw.u32 q4, [sp, #16] @ 16-byte Reload
+; CHECK-NEXT:    vmov.f32 s3, s6
+; CHECK-NEXT:    vldrw.u32 q1, [sp] @ 16-byte Reload
+; CHECK-NEXT:    vdup.32 q6, r0
+; CHECK-NEXT:    vmov.f32 s22, s26
+; CHECK-NEXT:    vldrw.u32 q6, [sp, #32] @ 16-byte Reload
+; CHECK-NEXT:    vmov.f32 s2, s6
+; CHECK-NEXT:    vmov.32 r0, q4[3]
+; CHECK-NEXT:    vmov.f32 s4, s17
+; CHECK-NEXT:    vstrw.32 q5, [r1, #48]
+; CHECK-NEXT:    vmov.f32 s5, s29
+; CHECK-NEXT:    vstrw.32 q0, [r1, #64]
+; CHECK-NEXT:    vmov.f32 s28, s30
+; CHECK-NEXT:    vstrw.32 q3, [r1, #80]
+; CHECK-NEXT:    vmov.f32 s7, s18
+; CHECK-NEXT:    vdup.32 q4, r0
+; CHECK-NEXT:    vmov.f32 s29, s27
+; CHECK-NEXT:    vstrw.32 q2, [r1]
+; CHECK-NEXT:    vmov.f32 s6, s26
+; CHECK-NEXT:    vmov.f32 s30, s18
+; CHECK-NEXT:    vstrw.32 q1, [r1, #16]
+; CHECK-NEXT:    vstrw.32 q7, [r1, #32]
+; CHECK-NEXT:    add sp, #48
+; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    bx lr
+entry:
+  %s1 = getelementptr <8 x i32>, <8 x i32>* %src, i32 0
+  %l1 = load <8 x i32>, <8 x i32>* %s1, align 4
+  %s2 = getelementptr <8 x i32>, <8 x i32>* %src, i32 1
+  %l2 = load <8 x i32>, <8 x i32>* %s2, align 4
+  %s3 = getelementptr <8 x i32>, <8 x i32>* %src, i32 2
+  %l3 = load <8 x i32>, <8 x i32>* %s3, align 4
+  %t1 = shufflevector <8 x i32> %l1, <8 x i32> %l2, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %t2 = shufflevector <8 x i32> %l3, <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %s = shufflevector <16 x i32> %t1, <16 x i32> %t2, <24 x i32> <i32 0, i32 8, i32 16, i32 1, i32 9, i32 17, i32 2, i32 10, i32 18, i32 3, i32 11, i32 19, i32 4, i32 12, i32 20, i32 5, i32 13, i32 21, i32 6, i32 14, i32 22, i32 7, i32 15, i32 23>
+  store <24 x i32> %s, <24 x i32> *%dst
+  ret void
+}
+
+define void @vst3_v16i32(<16 x i32> *%src, <48 x i32> *%dst) {
+; CHECK-LABEL: vst3_v16i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    .pad #160
+; CHECK-NEXT:    sub sp, #160
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #80]
+; CHECK-NEXT:    vldrw.u32 q6, [r0, #64]
+; CHECK-NEXT:    vldrw.u32 q7, [r0, #128]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #144]
+; CHECK-NEXT:    vstrw.32 q2, [sp, #144] @ 16-byte Spill
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #16]
+; CHECK-NEXT:    vmov.f32 s4, s25
+; CHECK-NEXT:    vldrw.u32 q5, [r0]
+; CHECK-NEXT:    vstrw.32 q2, [sp, #128] @ 16-byte Spill
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #160]
+; CHECK-NEXT:    vmov.f32 s5, s29
+; CHECK-NEXT:    vstrw.32 q0, [sp] @ 16-byte Spill
+; CHECK-NEXT:    vstrw.32 q2, [sp, #112] @ 16-byte Spill
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #96]
+; CHECK-NEXT:    vmov.f32 s7, s26
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #48]
+; CHECK-NEXT:    vstrw.32 q2, [sp, #96] @ 16-byte Spill
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #32]
+; CHECK-NEXT:    vmov.f32 s6, s22
+; CHECK-NEXT:    vldrw.u32 q3, [r0, #112]
+; CHECK-NEXT:    vstrw.32 q2, [sp, #16] @ 16-byte Spill
+; CHECK-NEXT:    vmov.f64 d4, d15
+; CHECK-NEXT:    vldrw.u32 q4, [r0, #176]
+; CHECK-NEXT:    vstrw.32 q0, [sp, #32] @ 16-byte Spill
+; CHECK-NEXT:    vstrw.32 q1, [r1, #16]
+; CHECK-NEXT:    vmov.32 r0, q6[3]
+; CHECK-NEXT:    vdup.32 q1, r0
+; CHECK-NEXT:    vmov.f32 s9, s23
+; CHECK-NEXT:    vmov.f32 s11, s31
+; CHECK-NEXT:    vmov.f32 s10, s6
+; CHECK-NEXT:    vmov.f64 d2, d9
+; CHECK-NEXT:    vstrw.32 q2, [r1, #32]
+; CHECK-NEXT:    vmov.32 r0, q3[3]
+; CHECK-NEXT:    vmov.f32 s5, s3
+; CHECK-NEXT:    vmov.f32 s7, s19
+; CHECK-NEXT:    vmov q2, q1
+; CHECK-NEXT:    vdup.32 q1, r0
+; CHECK-NEXT:    vmov.f32 s10, s6
+; CHECK-NEXT:    vmov.f64 d2, d10
+; CHECK-NEXT:    vstrw.32 q2, [sp, #80] @ 16-byte Spill
+; CHECK-NEXT:    vmov.32 r0, q7[0]
+; CHECK-NEXT:    vldrw.u32 q2, [sp, #32] @ 16-byte Reload
+; CHECK-NEXT:    vdup.32 q0, r0
+; CHECK-NEXT:    vmov.f32 s5, s24
+; CHECK-NEXT:    vmov.f32 s7, s21
+; CHECK-NEXT:    vmov.f32 s6, s2
+; CHECK-NEXT:    vmov.f64 d0, d4
+; CHECK-NEXT:    vstrw.32 q1, [sp, #64] @ 16-byte Spill
+; CHECK-NEXT:    vmov.32 r0, q4[0]
+; CHECK-NEXT:    vmov.f32 s1, s12
+; CHECK-NEXT:    vmov.f32 s3, s9
+; CHECK-NEXT:    vmov q1, q0
+; CHECK-NEXT:    vdup.32 q0, r0
+; CHECK-NEXT:    vmov.f32 s6, s2
+; CHECK-NEXT:    vldrw.u32 q0, [sp, #112] @ 16-byte Reload
+; CHECK-NEXT:    vstrw.32 q1, [sp, #48] @ 16-byte Spill
+; CHECK-NEXT:    vldrw.u32 q1, [sp, #96] @ 16-byte Reload
+; CHECK-NEXT:    vmov.f32 s16, s13
+; CHECK-NEXT:    vmov.f32 s24, s5
+; CHECK-NEXT:    vmov.f32 s25, s1
+; CHECK-NEXT:    vmov.f32 s19, s14
+; CHECK-NEXT:    vmov q3, q1
+; CHECK-NEXT:    vmov.f32 s27, s6
+; CHECK-NEXT:    vldrw.u32 q1, [sp, #16] @ 16-byte Reload
+; CHECK-NEXT:    vmov.f32 s18, s10
+; CHECK-NEXT:    vmov.f64 d4, d1
+; CHECK-NEXT:    vstrw.32 q4, [sp, #32] @ 16-byte Spill
+; CHECK-NEXT:    vmov.32 r0, q3[3]
+; CHECK-NEXT:    vmov q5, q1
+; CHECK-NEXT:    vldrw.u32 q4, [sp, #144] @ 16-byte Reload
+; CHECK-NEXT:    vmov.f32 s9, s7
+; CHECK-NEXT:    vmov.f32 s11, s3
+; CHECK-NEXT:    vdup.32 q0, r0
+; CHECK-NEXT:    vmov.f32 s10, s2
+; CHECK-NEXT:    vldrw.u32 q0, [sp] @ 16-byte Reload
+; CHECK-NEXT:    vmov.f32 s26, s6
+; CHECK-NEXT:    vldrw.u32 q1, [sp, #128] @ 16-byte Reload
+; CHECK-NEXT:    vmov.f64 d6, d1
+; CHECK-NEXT:    vmov.32 r0, q4[3]
+; CHECK-NEXT:    vmov q7, q0
+; CHECK-NEXT:    vmov.f32 s13, s7
+; CHECK-NEXT:    vldrw.u32 q1, [sp, #96] @ 16-byte Reload
+; CHECK-NEXT:    vmov.f32 s15, s3
+; CHECK-NEXT:    vdup.32 q0, r0
+; CHECK-NEXT:    vmov.f32 s14, s2
+; CHECK-NEXT:    vmov q0, q5
+; CHECK-NEXT:    vmov.f32 s21, s4
+; CHECK-NEXT:    vmov.f32 s23, s1
+; CHECK-NEXT:    vldrw.u32 q0, [sp, #112] @ 16-byte Reload
+; CHECK-NEXT:    vmov.32 r0, q0[0]
+; CHECK-NEXT:    vdup.32 q0, r0
+; CHECK-NEXT:    vmov.f32 s22, s2
+; CHECK-NEXT:    vldrw.u32 q0, [sp, #128] @ 16-byte Reload
+; CHECK-NEXT:    vmov.32 r0, q7[0]
+; CHECK-NEXT:    vstrw.32 q3, [r1, #80]
+; CHECK-NEXT:    vmov.f64 d2, d0
+; CHECK-NEXT:    vstrw.32 q5, [r1, #96]
+; CHECK-NEXT:    vstrw.32 q6, [r1, #112]
+; CHECK-NEXT:    vstrw.32 q2, [r1, #128]
+; CHECK-NEXT:    vmov.f32 s5, s16
+; CHECK-NEXT:    vmov q4, q0
+; CHECK-NEXT:    vmov.f32 s7, s1
+; CHECK-NEXT:    vdup.32 q0, r0
+; CHECK-NEXT:    vmov.f32 s6, s2
+; CHECK-NEXT:    vldrw.u32 q0, [sp, #144] @ 16-byte Reload
+; CHECK-NEXT:    vstrw.32 q1, [r1, #48]
+; CHECK-NEXT:    vmov.f32 s28, s1
+; CHECK-NEXT:    vmov.f32 s31, s2
+; CHECK-NEXT:    vldrw.u32 q0, [sp, #48] @ 16-byte Reload
+; CHECK-NEXT:    vmov.f32 s30, s18
+; CHECK-NEXT:    vstrw.32 q0, [r1, #144]
+; CHECK-NEXT:    vldrw.u32 q0, [sp, #32] @ 16-byte Reload
+; CHECK-NEXT:    vstrw.32 q7, [r1, #64]
+; CHECK-NEXT:    vstrw.32 q0, [r1, #160]
+; CHECK-NEXT:    vldrw.u32 q0, [sp, #80] @ 16-byte Reload
+; CHECK-NEXT:    vstrw.32 q0, [r1, #176]
+; CHECK-NEXT:    vldrw.u32 q0, [sp, #64] @ 16-byte Reload
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    add sp, #160
+; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    bx lr
+entry:
+  %s1 = getelementptr <16 x i32>, <16 x i32>* %src, i32 0
+  %l1 = load <16 x i32>, <16 x i32>* %s1, align 4
+  %s2 = getelementptr <16 x i32>, <16 x i32>* %src, i32 1
+  %l2 = load <16 x i32>, <16 x i32>* %s2, align 4
+  %s3 = getelementptr <16 x i32>, <16 x i32>* %src, i32 2
+  %l3 = load <16 x i32>, <16 x i32>* %s3, align 4
+  %t1 = shufflevector <16 x i32> %l1, <16 x i32> %l2, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+  %t2 = shufflevector <16 x i32> %l3, <16 x i32> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %s = shufflevector <32 x i32> %t1, <32 x i32> %t2, <48 x i32> <i32 0, i32 16, i32 32, i32 1, i32 17, i32 33, i32 2, i32 18, i32 34, i32 3, i32 19, i32 35, i32 4, i32 20, i32 36, i32 5, i32 21, i32 37, i32 6, i32 22, i32 38, i32 7, i32 23, i32 39, i32 8, i32 24, i32 40, i32 9, i32 25, i32 41, i32 10, i32 26, i32 42, i32 11, i32 27, i32 43, i32 12, i32 28, i32 44, i32 13, i32 29, i32 45, i32 14, i32 30, i32 46, i32 15, i32 31, i32 47>
+  store <48 x i32> %s, <48 x i32> *%dst
+  ret void
+}
+
+; i16
+
+define void @vst3_v2i16(<2 x i16> *%src, <6 x i16> *%dst) {
+; CHECK-LABEL: vst3_v2i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, lr}
+; CHECK-NEXT:    push {r4, lr}
+; CHECK-NEXT:    ldrh r2, [r0, #6]
+; CHECK-NEXT:    ldrh r3, [r0, #4]
+; CHECK-NEXT:    ldrh.w r12, [r0, #8]
+; CHECK-NEXT:    vmov.16 q0[4], r2
+; CHECK-NEXT:    ldrh.w lr, [r0, #2]
+; CHECK-NEXT:    vmov.32 q1[0], r3
+; CHECK-NEXT:    ldrh r4, [r0]
+; CHECK-NEXT:    vmov.32 q1[2], r2
+; CHECK-NEXT:    ldrh r0, [r0, #10]
+; CHECK-NEXT:    vmov.16 q0[5], r0
+; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    vmov.32 q0[0], r4
+; CHECK-NEXT:    vmov.32 q0[2], lr
+; CHECK-NEXT:    vmov.f32 s1, s4
+; CHECK-NEXT:    vdup.32 q1, r12
+; CHECK-NEXT:    vmov.f32 s3, s2
+; CHECK-NEXT:    vmov.f32 s2, s6
+; CHECK-NEXT:    vstrh.32 q0, [r1]
+; CHECK-NEXT:    str r0, [r1, #8]
+; CHECK-NEXT:    pop {r4, pc}
+entry:
+  %s1 = getelementptr <2 x i16>, <2 x i16>* %src, i32 0
+  %l1 = load <2 x i16>, <2 x i16>* %s1, align 4
+  %s2 = getelementptr <2 x i16>, <2 x i16>* %src, i32 1
+  %l2 = load <2 x i16>, <2 x i16>* %s2, align 4
+  %s3 = getelementptr <2 x i16>, <2 x i16>* %src, i32 2
+  %l3 = load <2 x i16>, <2 x i16>* %s3, align 4
+  %t1 = shufflevector <2 x i16> %l1, <2 x i16> %l2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %t2 = shufflevector <2 x i16> %l3, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+  %s = shufflevector <4 x i16> %t1, <4 x i16> %t2, <6 x i32> <i32 0, i32 2, i32 4, i32 1, i32 3, i32 5>
+  store <6 x i16> %s, <6 x i16> *%dst
+  ret void
+}
+
+define void @vst3_v4i16(<4 x i16> *%src, <12 x i16> *%dst) {
+; CHECK-LABEL: vst3_v4i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9}
+; CHECK-NEXT:    vpush {d8, d9}
+; CHECK-NEXT:    vldrh.u32 q2, [r0, #16]
+; CHECK-NEXT:    vldrh.u32 q1, [r0]
+; CHECK-NEXT:    vldrh.u32 q3, [r0, #8]
+; CHECK-NEXT:    vmov.f64 d0, d5
+; CHECK-NEXT:    vmov.32 r0, q3[3]
+; CHECK-NEXT:    vdup.32 q4, r0
+; CHECK-NEXT:    vmov.f32 s1, s7
+; CHECK-NEXT:    vmov.f32 s3, s11
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmov.f32 s2, s18
+; CHECK-NEXT:    vmov.16 q4[0], r0
+; CHECK-NEXT:    vmov r0, s12
+; CHECK-NEXT:    vstrh.32 q0, [r1, #16]
+; CHECK-NEXT:    vmov.16 q4[1], r0
+; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    vmov.16 q4[2], r0
+; CHECK-NEXT:    vmov r0, s5
+; CHECK-NEXT:    vmov.16 q4[3], r0
+; CHECK-NEXT:    vmov r0, s13
+; CHECK-NEXT:    vmov.16 q4[4], r0
+; CHECK-NEXT:    vmov r0, s9
+; CHECK-NEXT:    vmov.16 q4[5], r0
+; CHECK-NEXT:    vmov r0, s6
+; CHECK-NEXT:    vmov.16 q4[6], r0
+; CHECK-NEXT:    vmov r0, s14
+; CHECK-NEXT:    vmov.16 q4[7], r0
+; CHECK-NEXT:    vstrw.32 q4, [r1]
+; CHECK-NEXT:    vpop {d8, d9}
+; CHECK-NEXT:    bx lr
+entry:
+  %s1 = getelementptr <4 x i16>, <4 x i16>* %src, i32 0
+  %l1 = load <4 x i16>, <4 x i16>* %s1, align 4
+  %s2 = getelementptr <4 x i16>, <4 x i16>* %src, i32 1
+  %l2 = load <4 x i16>, <4 x i16>* %s2, align 4
+  %s3 = getelementptr <4 x i16>, <4 x i16>* %src, i32 2
+  %l3 = load <4 x i16>, <4 x i16>* %s3, align 4
+  %t1 = shufflevector <4 x i16> %l1, <4 x i16> %l2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %t2 = shufflevector <4 x i16> %l3, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+  %s = shufflevector <8 x i16> %t1, <8 x i16> %t2, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+  store <12 x i16> %s, <12 x i16> *%dst
+  ret void
+}
+
+define void @vst3_v8i16(<8 x i16> *%src, <24 x i16> *%dst) {
+; CHECK-LABEL: vst3_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #32]
+; CHECK-NEXT:    vldrw.u32 q2, [r0]
+; CHECK-NEXT:    vldrw.u32 q4, [r0, #16]
+; CHECK-NEXT:    vmov.u16 r2, q1[2]
+; CHECK-NEXT:    vmov.16 q0[0], r2
+; CHECK-NEXT:    vmov.u16 r2, q2[3]
+; CHECK-NEXT:    vmov.16 q0[1], r2
+; CHECK-NEXT:    vmov.u16 r2, q1[4]
+; CHECK-NEXT:    vmov.16 q0[6], r2
+; CHECK-NEXT:    vmov.u16 r2, q2[5]
+; CHECK-NEXT:    vmov.16 q0[7], r2
+; CHECK-NEXT:    vmov.u16 r2, q2[0]
+; CHECK-NEXT:    vmov.16 q3[0], r2
+; CHECK-NEXT:    vmov.u16 r0, q4[0]
+; CHECK-NEXT:    vmov.16 q3[1], r0
+; CHECK-NEXT:    vmov.u16 r0, q4[1]
+; CHECK-NEXT:    vmov.16 q3[4], r0
+; CHECK-NEXT:    vmov.u16 r0, q2[2]
+; CHECK-NEXT:    vmov.16 q3[6], r0
+; CHECK-NEXT:    vmov.u16 r0, q4[2]
+; CHECK-NEXT:    vmov.32 r2, q1[0]
+; CHECK-NEXT:    vmov.16 q3[7], r0
+; CHECK-NEXT:    vdup.32 q5, r2
+; CHECK-NEXT:    vmov.f32 s13, s8
+; CHECK-NEXT:    vmov.u16 r2, q5[2]
+; CHECK-NEXT:    vmov.u16 r0, q3[3]
+; CHECK-NEXT:    vmov.16 q6[2], r2
+; CHECK-NEXT:    vmov.32 r2, q2[3]
+; CHECK-NEXT:    vmov.16 q6[3], r0
+; CHECK-NEXT:    vmov.u16 r0, q3[4]
+; CHECK-NEXT:    vmov.16 q6[4], r0
+; CHECK-NEXT:    vmov.u16 r0, q5[5]
+; CHECK-NEXT:    vmov.16 q6[5], r0
+; CHECK-NEXT:    vmov.u16 r0, q4[5]
+; CHECK-NEXT:    vmov.16 q5[0], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[5]
+; CHECK-NEXT:    vmov.16 q5[1], r0
+; CHECK-NEXT:    vmov.u16 r0, q4[6]
+; CHECK-NEXT:    vmov.16 q5[3], r0
+; CHECK-NEXT:    vmov.u16 r0, q4[7]
+; CHECK-NEXT:    vmov.f32 s13, s25
+; CHECK-NEXT:    vmov.16 q5[6], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[7]
+; CHECK-NEXT:    vmov.f32 s14, s26
+; CHECK-NEXT:    vmov.16 q5[7], r0
+; CHECK-NEXT:    vdup.32 q6, r2
+; CHECK-NEXT:    vmov.f32 s1, s5
+; CHECK-NEXT:    vmov.u16 r2, q6[2]
+; CHECK-NEXT:    vmov.f32 s22, s7
+; CHECK-NEXT:    vrev32.16 q4, q4
+; CHECK-NEXT:    vmov.16 q7[2], r2
+; CHECK-NEXT:    vmov.u16 r0, q5[3]
+; CHECK-NEXT:    vmov.u16 r2, q4[2]
+; CHECK-NEXT:    vmov.f32 s2, s10
+; CHECK-NEXT:    vmov.16 q7[3], r0
+; CHECK-NEXT:    vmov.u16 r0, q5[4]
+; CHECK-NEXT:    vstrw.32 q3, [r1]
+; CHECK-NEXT:    vmov.16 q3[2], r2
+; CHECK-NEXT:    vmov.u16 r2, q0[3]
+; CHECK-NEXT:    vmov.16 q7[4], r0
+; CHECK-NEXT:    vmov.u16 r0, q6[5]
+; CHECK-NEXT:    vmov.16 q3[3], r2
+; CHECK-NEXT:    vmov.u16 r2, q0[4]
+; CHECK-NEXT:    vmov.16 q7[5], r0
+; CHECK-NEXT:    vmov.u16 r0, q4[5]
+; CHECK-NEXT:    vmov.16 q3[4], r2
+; CHECK-NEXT:    vmov.16 q3[5], r0
+; CHECK-NEXT:    vmov.f32 s21, s29
+; CHECK-NEXT:    vmov.f32 s1, s13
+; CHECK-NEXT:    vmov.f32 s22, s30
+; CHECK-NEXT:    vmov.f32 s2, s14
+; CHECK-NEXT:    vstrw.32 q5, [r1, #32]
+; CHECK-NEXT:    vstrw.32 q0, [r1, #16]
+; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    bx lr
+entry:
+  %s1 = getelementptr <8 x i16>, <8 x i16>* %src, i32 0
+  %l1 = load <8 x i16>, <8 x i16>* %s1, align 4
+  %s2 = getelementptr <8 x i16>, <8 x i16>* %src, i32 1
+  %l2 = load <8 x i16>, <8 x i16>* %s2, align 4
+  %s3 = getelementptr <8 x i16>, <8 x i16>* %src, i32 2
+  %l3 = load <8 x i16>, <8 x i16>* %s3, align 4
+  %t1 = shufflevector <8 x i16> %l1, <8 x i16> %l2, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %t2 = shufflevector <8 x i16> %l3, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %s = shufflevector <16 x i16> %t1, <16 x i16> %t2, <24 x i32> <i32 0, i32 8, i32 16, i32 1, i32 9, i32 17, i32 2, i32 10, i32 18, i32 3, i32 11, i32 19, i32 4, i32 12, i32 20, i32 5, i32 13, i32 21, i32 6, i32 14, i32 22, i32 7, i32 15, i32 23>
+  store <24 x i16> %s, <24 x i16> *%dst
+  ret void
+}
+
+define void @vst3_v16i16(<16 x i16> *%src, <48 x i16> *%dst) {
+; CHECK-LABEL: vst3_v16i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    .pad #160
+; CHECK-NEXT:    sub sp, #160
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #16]
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #48]
+; CHECK-NEXT:    vmov.u16 r2, q0[0]
+; CHECK-NEXT:    vstrw.32 q0, [sp, #128] @ 16-byte Spill
+; CHECK-NEXT:    vmov.16 q2[0], r2
+; CHECK-NEXT:    vmov.u16 r2, q1[0]
+; CHECK-NEXT:    vmov.16 q2[1], r2
+; CHECK-NEXT:    vmov.u16 r2, q1[1]
+; CHECK-NEXT:    vmov.16 q2[4], r2
+; CHECK-NEXT:    vmov.u16 r2, q0[2]
+; CHECK-NEXT:    vmov.16 q2[6], r2
+; CHECK-NEXT:    vmov.u16 r2, q1[2]
+; CHECK-NEXT:    vmov.16 q2[7], r2
+; CHECK-NEXT:    vmov q7, q1
+; CHECK-NEXT:    vmov.f32 s9, s0
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #80]
+; CHECK-NEXT:    vstrw.32 q2, [sp, #96] @ 16-byte Spill
+; CHECK-NEXT:    vmov.u16 r2, q2[3]
+; CHECK-NEXT:    vmov.32 r3, q0[0]
+; CHECK-NEXT:    vmov q1, q0
+; CHECK-NEXT:    vdup.32 q0, r3
+; CHECK-NEXT:    vmov q3, q2
+; CHECK-NEXT:    vmov.u16 r3, q0[2]
+; CHECK-NEXT:    vmov.16 q2[2], r3
+; CHECK-NEXT:    vmov.16 q2[3], r2
+; CHECK-NEXT:    vmov.u16 r2, q3[4]
+; CHECK-NEXT:    vldrw.u32 q3, [r0, #32]
+; CHECK-NEXT:    vmov.16 q2[4], r2
+; CHECK-NEXT:    vmov.u16 r2, q0[5]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #64]
+; CHECK-NEXT:    vmov.16 q2[5], r2
+; CHECK-NEXT:    vmov.u16 r2, q3[5]
+; CHECK-NEXT:    vmov.16 q5[0], r2
+; CHECK-NEXT:    vmov.u16 r2, q0[5]
+; CHECK-NEXT:    vmov.16 q5[1], r2
+; CHECK-NEXT:    vmov.u16 r2, q3[6]
+; CHECK-NEXT:    vmov.16 q5[3], r2
+; CHECK-NEXT:    vmov.u16 r2, q3[7]
+; CHECK-NEXT:    vmov.16 q5[6], r2
+; CHECK-NEXT:    vmov.u16 r2, q0[7]
+; CHECK-NEXT:    vmov.16 q5[7], r2
+; CHECK-NEXT:    vstrw.32 q0, [sp, #144] @ 16-byte Spill
+; CHECK-NEXT:    vmov.f32 s22, s3
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vstrw.32 q2, [sp, #112] @ 16-byte Spill
+; CHECK-NEXT:    vmov.u16 r2, q5[3]
+; CHECK-NEXT:    vmov.32 r0, q0[3]
+; CHECK-NEXT:    vmov q2, q0
+; CHECK-NEXT:    vdup.32 q0, r0
+; CHECK-NEXT:    vstrw.32 q2, [sp, #48] @ 16-byte Spill
+; CHECK-NEXT:    vmov.u16 r0, q0[2]
+; CHECK-NEXT:    vmov.16 q4[2], r0
+; CHECK-NEXT:    vmov.u16 r0, q5[4]
+; CHECK-NEXT:    vmov.16 q4[3], r2
+; CHECK-NEXT:    vmov.16 q4[4], r0
+; CHECK-NEXT:    vmov.u16 r0, q0[5]
+; CHECK-NEXT:    vmov.16 q4[5], r0
+; CHECK-NEXT:    vmov.u16 r0, q2[0]
+; CHECK-NEXT:    vmov.16 q6[0], r0
+; CHECK-NEXT:    vmov.u16 r0, q3[0]
+; CHECK-NEXT:    vmov.16 q6[1], r0
+; CHECK-NEXT:    vmov.u16 r0, q3[1]
+; CHECK-NEXT:    vldrw.u32 q0, [sp, #144] @ 16-byte Reload
+; CHECK-NEXT:    vmov.16 q6[4], r0
+; CHECK-NEXT:    vmov.u16 r0, q2[2]
+; CHECK-NEXT:    vstrw.32 q4, [sp, #80] @ 16-byte Spill
+; CHECK-NEXT:    vmov.32 r2, q0[0]
+; CHECK-NEXT:    vmov.16 q6[6], r0
+; CHECK-NEXT:    vmov.u16 r0, q3[2]
+; CHECK-NEXT:    vdup.32 q0, r2
+; CHECK-NEXT:    vmov.16 q6[7], r0
+; CHECK-NEXT:    vmov.u16 r2, q0[2]
+; CHECK-NEXT:    vmov.f32 s25, s8
+; CHECK-NEXT:    vmov.16 q2[2], r2
+; CHECK-NEXT:    vmov.u16 r0, q6[3]
+; CHECK-NEXT:    vmov q4, q1
+; CHECK-NEXT:    vmov.16 q2[3], r0
+; CHECK-NEXT:    vmov.u16 r0, q6[4]
+; CHECK-NEXT:    vmov.16 q2[4], r0
+; CHECK-NEXT:    vmov.u16 r0, q0[5]
+; CHECK-NEXT:    vmov.16 q2[5], r0
+; CHECK-NEXT:    vmov.u16 r0, q7[5]
+; CHECK-NEXT:    vmov.16 q0[0], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[5]
+; CHECK-NEXT:    vmov.16 q0[1], r0
+; CHECK-NEXT:    vmov.u16 r0, q7[6]
+; CHECK-NEXT:    vmov.16 q0[3], r0
+; CHECK-NEXT:    vmov.u16 r0, q7[7]
+; CHECK-NEXT:    vmov.16 q0[6], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[7]
+; CHECK-NEXT:    vldrw.u32 q1, [sp, #128] @ 16-byte Reload
+; CHECK-NEXT:    vstrw.32 q2, [sp, #64] @ 16-byte Spill
+; CHECK-NEXT:    vstrw.32 q7, [sp, #32] @ 16-byte Spill
+; CHECK-NEXT:    vmov.16 q0[7], r0
+; CHECK-NEXT:    vmov.32 r2, q1[3]
+; CHECK-NEXT:    vmov.f32 s2, s19
+; CHECK-NEXT:    vdup.32 q7, r2
+; CHECK-NEXT:    vmov.u16 r0, q0[3]
+; CHECK-NEXT:    vmov.u16 r2, q7[2]
+; CHECK-NEXT:    vrev32.16 q3, q3
+; CHECK-NEXT:    vmov.16 q2[2], r2
+; CHECK-NEXT:    vstrw.32 q3, [sp] @ 16-byte Spill
+; CHECK-NEXT:    vmov.16 q2[3], r0
+; CHECK-NEXT:    vmov.u16 r0, q0[4]
+; CHECK-NEXT:    vmov.16 q2[4], r0
+; CHECK-NEXT:    vmov.u16 r0, q7[5]
+; CHECK-NEXT:    vmov.16 q2[5], r0
+; CHECK-NEXT:    vmov.u16 r0, q4[2]
+; CHECK-NEXT:    vmov.16 q3[0], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[3]
+; CHECK-NEXT:    vmov.16 q3[1], r0
+; CHECK-NEXT:    vmov.u16 r0, q4[4]
+; CHECK-NEXT:    vldrw.u32 q7, [sp, #32] @ 16-byte Reload
+; CHECK-NEXT:    vmov.16 q3[6], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[5]
+; CHECK-NEXT:    vldrw.u32 q1, [sp, #144] @ 16-byte Reload
+; CHECK-NEXT:    vstrw.32 q4, [sp, #16] @ 16-byte Spill
+; CHECK-NEXT:    vldrw.u32 q4, [sp, #48] @ 16-byte Reload
+; CHECK-NEXT:    vrev32.16 q7, q7
+; CHECK-NEXT:    vmov.16 q3[7], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[2]
+; CHECK-NEXT:    vstrw.32 q7, [sp, #32] @ 16-byte Spill
+; CHECK-NEXT:    vmov.16 q7[0], r0
+; CHECK-NEXT:    vmov.u16 r0, q4[3]
+; CHECK-NEXT:    vmov.16 q7[1], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[4]
+; CHECK-NEXT:    vldrw.u32 q1, [sp, #64] @ 16-byte Reload
+; CHECK-NEXT:    vmov.f32 s1, s9
+; CHECK-NEXT:    vmov.f32 s2, s10
+; CHECK-NEXT:    vldrw.u32 q2, [sp, #96] @ 16-byte Reload
+; CHECK-NEXT:    vmov.f32 s25, s5
+; CHECK-NEXT:    vmov.16 q7[6], r0
+; CHECK-NEXT:    vmov.f32 s26, s6
+; CHECK-NEXT:    vldrw.u32 q1, [sp, #80] @ 16-byte Reload
+; CHECK-NEXT:    vmov.u16 r0, q4[5]
+; CHECK-NEXT:    vstrw.32 q0, [r1, #80]
+; CHECK-NEXT:    vmov.f32 s21, s5
+; CHECK-NEXT:    vldrw.u32 q0, [sp] @ 16-byte Reload
+; CHECK-NEXT:    vmov.f32 s22, s6
+; CHECK-NEXT:    vldrw.u32 q1, [sp, #112] @ 16-byte Reload
+; CHECK-NEXT:    vmov.16 q7[7], r0
+; CHECK-NEXT:    vmov.u16 r2, q0[2]
+; CHECK-NEXT:    vmov.f32 s9, s5
+; CHECK-NEXT:    vmov.u16 r0, q0[5]
+; CHECK-NEXT:    vmov.f32 s10, s6
+; CHECK-NEXT:    vldrw.u32 q1, [sp, #144] @ 16-byte Reload
+; CHECK-NEXT:    vmov.16 q0[2], r2
+; CHECK-NEXT:    vstrw.32 q5, [r1, #32]
+; CHECK-NEXT:    vmov.f32 s29, s5
+; CHECK-NEXT:    vldrw.u32 q1, [sp, #16] @ 16-byte Reload
+; CHECK-NEXT:    vmov.f32 s30, s18
+; CHECK-NEXT:    vstrw.32 q2, [r1, #48]
+; CHECK-NEXT:    vmov.u16 r2, q7[3]
+; CHECK-NEXT:    vmov.f32 s13, s5
+; CHECK-NEXT:    vmov.16 q0[3], r2
+; CHECK-NEXT:    vmov.u16 r2, q7[4]
+; CHECK-NEXT:    vmov.16 q0[4], r2
+; CHECK-NEXT:    vldrw.u32 q1, [sp, #128] @ 16-byte Reload
+; CHECK-NEXT:    vmov.16 q0[5], r0
+; CHECK-NEXT:    vstrw.32 q6, [r1]
+; CHECK-NEXT:    vmov.f32 s29, s1
+; CHECK-NEXT:    vmov.f32 s30, s2
+; CHECK-NEXT:    vldrw.u32 q0, [sp, #32] @ 16-byte Reload
+; CHECK-NEXT:    vmov.f32 s14, s6
+; CHECK-NEXT:    vstrw.32 q7, [r1, #16]
+; CHECK-NEXT:    vmov.u16 r2, q0[2]
+; CHECK-NEXT:    vmov.u16 r0, q0[5]
+; CHECK-NEXT:    vmov.16 q0[2], r2
+; CHECK-NEXT:    vmov.u16 r2, q3[3]
+; CHECK-NEXT:    vmov.16 q0[3], r2
+; CHECK-NEXT:    vmov.u16 r2, q3[4]
+; CHECK-NEXT:    vmov.16 q0[4], r2
+; CHECK-NEXT:    vmov.16 q0[5], r0
+; CHECK-NEXT:    vmov.f32 s13, s1
+; CHECK-NEXT:    vmov.f32 s14, s2
+; CHECK-NEXT:    vstrw.32 q3, [r1, #64]
+; CHECK-NEXT:    add sp, #160
+; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    bx lr
+entry:
+  %s1 = getelementptr <16 x i16>, <16 x i16>* %src, i32 0
+  %l1 = load <16 x i16>, <16 x i16>* %s1, align 4
+  %s2 = getelementptr <16 x i16>, <16 x i16>* %src, i32 1
+  %l2 = load <16 x i16>, <16 x i16>* %s2, align 4
+  %s3 = getelementptr <16 x i16>, <16 x i16>* %src, i32 2
+  %l3 = load <16 x i16>, <16 x i16>* %s3, align 4
+  %t1 = shufflevector <16 x i16> %l1, <16 x i16> %l2, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+  %t2 = shufflevector <16 x i16> %l3, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %s = shufflevector <32 x i16> %t1, <32 x i16> %t2, <48 x i32> <i32 0, i32 16, i32 32, i32 1, i32 17, i32 33, i32 2, i32 18, i32 34, i32 3, i32 19, i32 35, i32 4, i32 20, i32 36, i32 5, i32 21, i32 37, i32 6, i32 22, i32 38, i32 7, i32 23, i32 39, i32 8, i32 24, i32 40, i32 9, i32 25, i32 41, i32 10, i32 26, i32 42, i32 11, i32 27, i32 43, i32 12, i32 28, i32 44, i32 13, i32 29, i32 45, i32 14, i32 30, i32 46, i32 15, i32 31, i32 47>
+  store <48 x i16> %s, <48 x i16> *%dst
+  ret void
+}
+
+; i8
+
+define void @vst3_v2i8(<2 x i8> *%src, <6 x i8> *%dst) {
+; CHECK-LABEL: vst3_v2i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, r5, r6, lr}
+; CHECK-NEXT:    push {r4, r5, r6, lr}
+; CHECK-NEXT:    .pad #16
+; CHECK-NEXT:    sub sp, #16
+; CHECK-NEXT:    ldrb r2, [r0]
+; CHECK-NEXT:    movs r6, #0
+; CHECK-NEXT:    ldrb r3, [r0, #1]
+; CHECK-NEXT:    vmov.32 q0[0], r2
+; CHECK-NEXT:    ldrb.w r12, [r0, #2]
+; CHECK-NEXT:    vmov.32 q0[2], r3
+; CHECK-NEXT:    ldrb.w lr, [r0, #3]
+; CHECK-NEXT:    vmov r4, s0
+; CHECK-NEXT:    ldrb r5, [r0, #5]
+; CHECK-NEXT:    vmov.16 q0[0], r4
+; CHECK-NEXT:    ldrb r0, [r0, #4]
+; CHECK-NEXT:    vmov.16 q0[1], r12
+; CHECK-NEXT:    mov r2, sp
+; CHECK-NEXT:    vmov.16 q0[2], r0
+; CHECK-NEXT:    add r0, sp, #8
+; CHECK-NEXT:    vmov.16 q0[3], r3
+; CHECK-NEXT:    vmov.16 q0[4], lr
+; CHECK-NEXT:    vmov.16 q0[5], r5
+; CHECK-NEXT:    vmov.16 q0[6], r6
+; CHECK-NEXT:    vmov.16 q0[7], r6
+; CHECK-NEXT:    vstrb.16 q0, [r2]
+; CHECK-NEXT:    vstrb.16 q0, [r0]
+; CHECK-NEXT:    vldrh.u32 q0, [r0]
+; CHECK-NEXT:    ldr r2, [sp]
+; CHECK-NEXT:    str r2, [r1]
+; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    strh r0, [r1, #4]
+; CHECK-NEXT:    add sp, #16
+; CHECK-NEXT:    pop {r4, r5, r6, pc}
+entry:
+  %s1 = getelementptr <2 x i8>, <2 x i8>* %src, i32 0
+  %l1 = load <2 x i8>, <2 x i8>* %s1, align 4
+  %s2 = getelementptr <2 x i8>, <2 x i8>* %src, i32 1
+  %l2 = load <2 x i8>, <2 x i8>* %s2, align 4
+  %s3 = getelementptr <2 x i8>, <2 x i8>* %src, i32 2
+  %l3 = load <2 x i8>, <2 x i8>* %s3, align 4
+  %t1 = shufflevector <2 x i8> %l1, <2 x i8> %l2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %t2 = shufflevector <2 x i8> %l3, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+  %s = shufflevector <4 x i8> %t1, <4 x i8> %t2, <6 x i32> <i32 0, i32 2, i32 4, i32 1, i32 3, i32 5>
+  store <6 x i8> %s, <6 x i8> *%dst
+  ret void
+}
+
+define void @vst3_v4i8(<4 x i8> *%src, <12 x i8> *%dst) {
+; CHECK-LABEL: vst3_v4i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9}
+; CHECK-NEXT:    vpush {d8, d9}
+; CHECK-NEXT:    vldrb.u32 q1, [r0]
+; CHECK-NEXT:    vldrb.u32 q2, [r0, #4]
+; CHECK-NEXT:    vldrb.u32 q3, [r0, #8]
+; CHECK-NEXT:    vmov r2, s4
+; CHECK-NEXT:    vmov.16 q0[0], r2
+; CHECK-NEXT:    vmov r2, s8
+; CHECK-NEXT:    vmov.16 q0[1], r2
+; CHECK-NEXT:    vmov r0, s12
+; CHECK-NEXT:    vmov.16 q0[2], r0
+; CHECK-NEXT:    vmov r0, s5
+; CHECK-NEXT:    vmov.16 q0[3], r0
+; CHECK-NEXT:    vmov r0, s9
+; CHECK-NEXT:    vmov.16 q0[4], r0
+; CHECK-NEXT:    vmov r0, s13
+; CHECK-NEXT:    vmov.16 q0[5], r0
+; CHECK-NEXT:    vmov r0, s6
+; CHECK-NEXT:    vmov.16 q0[6], r0
+; CHECK-NEXT:    vmov r0, s10
+; CHECK-NEXT:    vmov.16 q0[7], r0
+; CHECK-NEXT:    vmov r0, s14
+; CHECK-NEXT:    vmov.8 q4[8], r0
+; CHECK-NEXT:    vmov r0, s7
+; CHECK-NEXT:    vmov.8 q4[9], r0
+; CHECK-NEXT:    vmov r0, s11
+; CHECK-NEXT:    vmov.8 q4[10], r0
+; CHECK-NEXT:    vmov r0, s15
+; CHECK-NEXT:    vmov.8 q4[11], r0
+; CHECK-NEXT:    vstrb.16 q0, [r1]
+; CHECK-NEXT:    vmov r0, s18
+; CHECK-NEXT:    str r0, [r1, #8]
+; CHECK-NEXT:    vpop {d8, d9}
+; CHECK-NEXT:    bx lr
+entry:
+  %s1 = getelementptr <4 x i8>, <4 x i8>* %src, i32 0
+  %l1 = load <4 x i8>, <4 x i8>* %s1, align 4
+  %s2 = getelementptr <4 x i8>, <4 x i8>* %src, i32 1
+  %l2 = load <4 x i8>, <4 x i8>* %s2, align 4
+  %s3 = getelementptr <4 x i8>, <4 x i8>* %src, i32 2
+  %l3 = load <4 x i8>, <4 x i8>* %s3, align 4
+  %t1 = shufflevector <4 x i8> %l1, <4 x i8> %l2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %t2 = shufflevector <4 x i8> %l3, <4 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+  %s = shufflevector <8 x i8> %t1, <8 x i8> %t2, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+  store <12 x i8> %s, <12 x i8> *%dst
+  ret void
+}
+
+define void @vst3_v8i8(<8 x i8> *%src, <24 x i8> *%dst) {
+; CHECK-LABEL: vst3_v8i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11}
+; CHECK-NEXT:    vldrb.u16 q1, [r0, #8]
+; CHECK-NEXT:    vldrb.u16 q2, [r0, #16]
+; CHECK-NEXT:    vldrb.u16 q3, [r0]
+; CHECK-NEXT:    vmov.u16 r2, q1[5]
+; CHECK-NEXT:    vmov.16 q0[0], r2
+; CHECK-NEXT:    vmov.u16 r2, q2[5]
+; CHECK-NEXT:    vmov.16 q0[1], r2
+; CHECK-NEXT:    vmov.u16 r2, q1[6]
+; CHECK-NEXT:    vmov.16 q0[3], r2
+; CHECK-NEXT:    vmov.u16 r2, q1[7]
+; CHECK-NEXT:    vmov.16 q0[6], r2
+; CHECK-NEXT:    vmov.u16 r2, q2[7]
+; CHECK-NEXT:    vmov.32 r0, q3[3]
+; CHECK-NEXT:    vmov.16 q0[7], r2
+; CHECK-NEXT:    vdup.32 q4, r0
+; CHECK-NEXT:    vmov.f32 s2, s11
+; CHECK-NEXT:    vmov.u16 r0, q4[2]
+; CHECK-NEXT:    vmov.u16 r2, q0[3]
+; CHECK-NEXT:    vmov.16 q5[2], r0
+; CHECK-NEXT:    vmov.u16 r0, q0[4]
+; CHECK-NEXT:    vmov.16 q5[3], r2
+; CHECK-NEXT:    vmov.16 q5[4], r0
+; CHECK-NEXT:    vmov.u16 r0, q4[5]
+; CHECK-NEXT:    vmov.16 q5[5], r0
+; CHECK-NEXT:    vmov.u16 r0, q3[0]
+; CHECK-NEXT:    vmov.8 q4[0], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[0]
+; CHECK-NEXT:    vmov.8 q4[1], r0
+; CHECK-NEXT:    vmov.u16 r0, q2[0]
+; CHECK-NEXT:    vmov.8 q4[2], r0
+; CHECK-NEXT:    vmov.u16 r0, q3[1]
+; CHECK-NEXT:    vmov.8 q4[3], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[1]
+; CHECK-NEXT:    vmov.8 q4[4], r0
+; CHECK-NEXT:    vmov.u16 r0, q2[1]
+; CHECK-NEXT:    vmov.8 q4[5], r0
+; CHECK-NEXT:    vmov.u16 r0, q3[2]
+; CHECK-NEXT:    vmov.8 q4[6], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[2]
+; CHECK-NEXT:    vmov.8 q4[7], r0
+; CHECK-NEXT:    vmov.u16 r0, q2[2]
+; CHECK-NEXT:    vmov.8 q4[8], r0
+; CHECK-NEXT:    vmov.u16 r0, q3[3]
+; CHECK-NEXT:    vmov.8 q4[9], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[3]
+; CHECK-NEXT:    vmov.8 q4[10], r0
+; CHECK-NEXT:    vmov.u16 r0, q2[3]
+; CHECK-NEXT:    vmov.8 q4[11], r0
+; CHECK-NEXT:    vmov.u16 r0, q3[4]
+; CHECK-NEXT:    vmov.8 q4[12], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[4]
+; CHECK-NEXT:    vmov.8 q4[13], r0
+; CHECK-NEXT:    vmov.u16 r0, q2[4]
+; CHECK-NEXT:    vmov.f32 s1, s21
+; CHECK-NEXT:    vmov.8 q4[14], r0
+; CHECK-NEXT:    vmov.u16 r0, q3[5]
+; CHECK-NEXT:    vmov.f32 s2, s22
+; CHECK-NEXT:    vmov.8 q4[15], r0
+; CHECK-NEXT:    vstrb.16 q0, [r1, #16]
+; CHECK-NEXT:    vstrw.32 q4, [r1]
+; CHECK-NEXT:    vpop {d8, d9, d10, d11}
+; CHECK-NEXT:    bx lr
+entry:
+  %s1 = getelementptr <8 x i8>, <8 x i8>* %src, i32 0
+  %l1 = load <8 x i8>, <8 x i8>* %s1, align 4
+  %s2 = getelementptr <8 x i8>, <8 x i8>* %src, i32 1
+  %l2 = load <8 x i8>, <8 x i8>* %s2, align 4
+  %s3 = getelementptr <8 x i8>, <8 x i8>* %src, i32 2
+  %l3 = load <8 x i8>, <8 x i8>* %s3, align 4
+  %t1 = shufflevector <8 x i8> %l1, <8 x i8> %l2, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %t2 = shufflevector <8 x i8> %l3, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %s = shufflevector <16 x i8> %t1, <16 x i8> %t2, <24 x i32> <i32 0, i32 8, i32 16, i32 1, i32 9, i32 17, i32 2, i32 10, i32 18, i32 3, i32 11, i32 19, i32 4, i32 12, i32 20, i32 5, i32 13, i32 21, i32 6, i32 14, i32 22, i32 7, i32 15, i32 23>
+  store <24 x i8> %s, <24 x i8> *%dst
+  ret void
+}
+
+define void @vst3_v16i8(<16 x i8> *%src, <48 x i8> *%dst) {
+; CHECK-LABEL: vst3_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT:    vldrw.u32 q3, [r0]
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #16]
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #32]
+; CHECK-NEXT:    vmov.u8 r3, q3[0]
+; CHECK-NEXT:    vmov.u8 r0, q2[0]
+; CHECK-NEXT:    vmov.8 q5[0], r3
+; CHECK-NEXT:    vmov.u8 r2, q1[0]
+; CHECK-NEXT:    vmov.8 q5[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q3[1]
+; CHECK-NEXT:    vmov.8 q5[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[1]
+; CHECK-NEXT:    vmov.8 q5[4], r0
+; CHECK-NEXT:    vmov.u8 r0, q3[2]
+; CHECK-NEXT:    vmov.8 q5[6], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[2]
+; CHECK-NEXT:    vmov.8 q5[7], r0
+; CHECK-NEXT:    vmov.u8 r0, q3[3]
+; CHECK-NEXT:    vmov.8 q5[9], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[3]
+; CHECK-NEXT:    vmov.8 q5[10], r0
+; CHECK-NEXT:    vmov.u8 r0, q3[4]
+; CHECK-NEXT:    vmov.8 q4[2], r2
+; CHECK-NEXT:    vmov.u8 r2, q1[2]
+; CHECK-NEXT:    vmov.8 q5[12], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[4]
+; CHECK-NEXT:    vmov.8 q4[8], r2
+; CHECK-NEXT:    vmov.u8 r2, q1[3]
+; CHECK-NEXT:    vmov.8 q5[13], r0
+; CHECK-NEXT:    vmov.u8 r0, q3[5]
+; CHECK-NEXT:    vmov.8 q5[15], r0
+; CHECK-NEXT:    vmov.8 q4[11], r2
+; CHECK-NEXT:    vmov.u8 r2, q1[4]
+; CHECK-NEXT:    vmov.u8 r0, q5[0]
+; CHECK-NEXT:    vmov.8 q4[14], r2
+; CHECK-NEXT:    vmov.8 q0[0], r0
+; CHECK-NEXT:    vmov.f32 s17, s4
+; CHECK-NEXT:    vmov.u8 r0, q5[1]
+; CHECK-NEXT:    vmov.8 q0[1], r0
+; CHECK-NEXT:    vmov.u8 r2, q4[2]
+; CHECK-NEXT:    vmov.8 q0[2], r2
+; CHECK-NEXT:    vmov.u8 r0, q5[3]
+; CHECK-NEXT:    vmov.8 q0[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q5[4]
+; CHECK-NEXT:    vmov.8 q0[4], r0
+; CHECK-NEXT:    vmov.u8 r0, q4[5]
+; CHECK-NEXT:    vmov.8 q0[5], r0
+; CHECK-NEXT:    vmov.u8 r0, q5[6]
+; CHECK-NEXT:    vmov.8 q0[6], r0
+; CHECK-NEXT:    vmov.u8 r0, q5[7]
+; CHECK-NEXT:    vmov.8 q0[7], r0
+; CHECK-NEXT:    vmov.u8 r0, q4[8]
+; CHECK-NEXT:    vmov.8 q0[8], r0
+; CHECK-NEXT:    vmov.u8 r0, q5[9]
+; CHECK-NEXT:    vmov.8 q0[9], r0
+; CHECK-NEXT:    vmov.u8 r0, q5[10]
+; CHECK-NEXT:    vmov.8 q0[10], r0
+; CHECK-NEXT:    vmov.u8 r0, q4[11]
+; CHECK-NEXT:    vmov.8 q0[11], r0
+; CHECK-NEXT:    vmov.u8 r0, q5[12]
+; CHECK-NEXT:    vmov.8 q0[12], r0
+; CHECK-NEXT:    vmov.u8 r0, q5[13]
+; CHECK-NEXT:    vmov.8 q0[13], r0
+; CHECK-NEXT:    vmov.u8 r0, q4[14]
+; CHECK-NEXT:    vmov.8 q0[14], r0
+; CHECK-NEXT:    vmov.u8 r0, q5[15]
+; CHECK-NEXT:    vmov.8 q0[15], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[5]
+; CHECK-NEXT:    vmov.8 q5[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[5]
+; CHECK-NEXT:    vmov.8 q5[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[6]
+; CHECK-NEXT:    vmov.8 q5[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[6]
+; CHECK-NEXT:    vmov.8 q5[4], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[7]
+; CHECK-NEXT:    vmov.8 q5[6], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[7]
+; CHECK-NEXT:    vmov.8 q5[7], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[8]
+; CHECK-NEXT:    vmov.8 q5[9], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[8]
+; CHECK-NEXT:    vmov.8 q5[10], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[9]
+; CHECK-NEXT:    vmov.8 q5[12], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[9]
+; CHECK-NEXT:    vmov.8 q5[13], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[10]
+; CHECK-NEXT:    vmov.8 q5[15], r0
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vmov.u8 r0, q5[0]
+; CHECK-NEXT:    vmov.8 q4[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q5[1]
+; CHECK-NEXT:    vmov.8 q4[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q3[7]
+; CHECK-NEXT:    vmov.8 q6[5], r0
+; CHECK-NEXT:    vmov.u8 r0, q3[8]
+; CHECK-NEXT:    vmov.8 q6[8], r0
+; CHECK-NEXT:    vmov.u8 r0, q3[9]
+; CHECK-NEXT:    vmov.8 q6[11], r0
+; CHECK-NEXT:    vmov.f32 s24, s13
+; CHECK-NEXT:    vmov.f32 s27, s14
+; CHECK-NEXT:    vmov.u8 r0, q6[2]
+; CHECK-NEXT:    vmov.8 q4[2], r0
+; CHECK-NEXT:    vmov.u8 r0, q5[3]
+; CHECK-NEXT:    vmov.8 q4[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q5[4]
+; CHECK-NEXT:    vmov.8 q4[4], r0
+; CHECK-NEXT:    vmov.u8 r0, q6[5]
+; CHECK-NEXT:    vmov.8 q4[5], r0
+; CHECK-NEXT:    vmov.u8 r0, q5[6]
+; CHECK-NEXT:    vmov.8 q4[6], r0
+; CHECK-NEXT:    vmov.u8 r0, q5[7]
+; CHECK-NEXT:    vmov.8 q4[7], r0
+; CHECK-NEXT:    vmov.u8 r0, q6[8]
+; CHECK-NEXT:    vmov.8 q4[8], r0
+; CHECK-NEXT:    vmov.u8 r0, q5[9]
+; CHECK-NEXT:    vmov.8 q4[9], r0
+; CHECK-NEXT:    vmov.u8 r0, q5[10]
+; CHECK-NEXT:    vmov.8 q4[10], r0
+; CHECK-NEXT:    vmov.u8 r0, q6[11]
+; CHECK-NEXT:    vmov.8 q4[11], r0
+; CHECK-NEXT:    vmov.u8 r0, q5[12]
+; CHECK-NEXT:    vmov.8 q4[12], r0
+; CHECK-NEXT:    vmov.u8 r0, q5[13]
+; CHECK-NEXT:    vmov.8 q4[13], r0
+; CHECK-NEXT:    vmov.u8 r0, q6[14]
+; CHECK-NEXT:    vmov.8 q4[14], r0
+; CHECK-NEXT:    vmov.u8 r0, q5[15]
+; CHECK-NEXT:    vmov.8 q4[15], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[10]
+; CHECK-NEXT:    vmov.8 q5[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q3[11]
+; CHECK-NEXT:    vmov.8 q5[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[11]
+; CHECK-NEXT:    vmov.8 q5[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q3[12]
+; CHECK-NEXT:    vmov.8 q5[4], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[12]
+; CHECK-NEXT:    vmov.8 q5[6], r0
+; CHECK-NEXT:    vmov.u8 r0, q3[13]
+; CHECK-NEXT:    vmov.8 q5[7], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[13]
+; CHECK-NEXT:    vmov.8 q5[9], r0
+; CHECK-NEXT:    vmov.u8 r0, q3[14]
+; CHECK-NEXT:    vmov.8 q5[10], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[14]
+; CHECK-NEXT:    vmov.8 q5[12], r0
+; CHECK-NEXT:    vmov.u8 r0, q3[15]
+; CHECK-NEXT:    vmov.8 q5[13], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[15]
+; CHECK-NEXT:    vmov.8 q5[15], r0
+; CHECK-NEXT:    vstrw.32 q4, [r1, #16]
+; CHECK-NEXT:    vmov.u8 r0, q5[0]
+; CHECK-NEXT:    vmov.8 q1[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q5[1]
+; CHECK-NEXT:    vmov.8 q1[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[11]
+; CHECK-NEXT:    vmov.8 q3[2], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[12]
+; CHECK-NEXT:    vmov.8 q3[5], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[13]
+; CHECK-NEXT:    vmov.8 q3[8], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[14]
+; CHECK-NEXT:    vmov.8 q3[11], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[15]
+; CHECK-NEXT:    vmov.8 q3[14], r0
+; CHECK-NEXT:    vmov.u8 r0, q3[2]
+; CHECK-NEXT:    vmov.8 q1[2], r0
+; CHECK-NEXT:    vmov.u8 r0, q5[3]
+; CHECK-NEXT:    vmov.8 q1[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q5[4]
+; CHECK-NEXT:    vmov.8 q1[4], r0
+; CHECK-NEXT:    vmov.u8 r0, q3[5]
+; CHECK-NEXT:    vmov.8 q1[5], r0
+; CHECK-NEXT:    vmov.u8 r0, q5[6]
+; CHECK-NEXT:    vmov.8 q1[6], r0
+; CHECK-NEXT:    vmov.u8 r0, q5[7]
+; CHECK-NEXT:    vmov.8 q1[7], r0
+; CHECK-NEXT:    vmov.u8 r0, q3[8]
+; CHECK-NEXT:    vmov.8 q1[8], r0
+; CHECK-NEXT:    vmov.u8 r0, q5[9]
+; CHECK-NEXT:    vmov.8 q1[9], r0
+; CHECK-NEXT:    vmov.u8 r0, q5[10]
+; CHECK-NEXT:    vmov.8 q1[10], r0
+; CHECK-NEXT:    vmov.u8 r0, q3[11]
+; CHECK-NEXT:    vmov.8 q1[11], r0
+; CHECK-NEXT:    vmov.u8 r0, q5[12]
+; CHECK-NEXT:    vmov.8 q1[12], r0
+; CHECK-NEXT:    vmov.u8 r0, q5[13]
+; CHECK-NEXT:    vmov.8 q1[13], r0
+; CHECK-NEXT:    vmov.u8 r0, q3[14]
+; CHECK-NEXT:    vmov.8 q1[14], r0
+; CHECK-NEXT:    vmov.u8 r0, q5[15]
+; CHECK-NEXT:    vmov.8 q1[15], r0
+; CHECK-NEXT:    vstrw.32 q1, [r1, #32]
+; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT:    bx lr
+entry:
+  %s1 = getelementptr <16 x i8>, <16 x i8>* %src, i32 0
+  %l1 = load <16 x i8>, <16 x i8>* %s1, align 4
+  %s2 = getelementptr <16 x i8>, <16 x i8>* %src, i32 1
+  %l2 = load <16 x i8>, <16 x i8>* %s2, align 4
+  %s3 = getelementptr <16 x i8>, <16 x i8>* %src, i32 2
+  %l3 = load <16 x i8>, <16 x i8>* %s3, align 4
+  %t1 = shufflevector <16 x i8> %l1, <16 x i8> %l2, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+  %t2 = shufflevector <16 x i8> %l3, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %s = shufflevector <32 x i8> %t1, <32 x i8> %t2, <48 x i32> <i32 0, i32 16, i32 32, i32 1, i32 17, i32 33, i32 2, i32 18, i32 34, i32 3, i32 19, i32 35, i32 4, i32 20, i32 36, i32 5, i32 21, i32 37, i32 6, i32 22, i32 38, i32 7, i32 23, i32 39, i32 8, i32 24, i32 40, i32 9, i32 25, i32 41, i32 10, i32 26, i32 42, i32 11, i32 27, i32 43, i32 12, i32 28, i32 44, i32 13, i32 29, i32 45, i32 14, i32 30, i32 46, i32 15, i32 31, i32 47>
+  store <48 x i8> %s, <48 x i8> *%dst
+  ret void
+}
+
+; i64
+
+define void @vst3_v2i64(<2 x i64> *%src, <6 x i64> *%dst) {
+; CHECK-LABEL: vst3_v2i64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #16]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #32]
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    mov r0, r1
+; CHECK-NEXT:    vmov.f64 d6, d5
+; CHECK-NEXT:    vmov.f32 s13, s11
+; CHECK-NEXT:    vmov.f32 s14, s2
+; CHECK-NEXT:    vmov.f32 s15, s3
+; CHECK-NEXT:    vmov.f32 s2, s6
+; CHECK-NEXT:    vmov.f32 s3, s7
+; CHECK-NEXT:    vmov.f32 s6, s8
+; CHECK-NEXT:    vstrw.32 q0, [r1, #16]
+; CHECK-NEXT:    vmov.f32 s7, s9
+; CHECK-NEXT:    vstrb.8 q1, [r0], #32
+; CHECK-NEXT:    vstrw.32 q3, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %s1 = getelementptr <2 x i64>, <2 x i64>* %src, i32 0
+  %l1 = load <2 x i64>, <2 x i64>* %s1, align 4
+  %s2 = getelementptr <2 x i64>, <2 x i64>* %src, i32 1
+  %l2 = load <2 x i64>, <2 x i64>* %s2, align 4
+  %s3 = getelementptr <2 x i64>, <2 x i64>* %src, i32 2
+  %l3 = load <2 x i64>, <2 x i64>* %s3, align 4
+  %t1 = shufflevector <2 x i64> %l1, <2 x i64> %l2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %t2 = shufflevector <2 x i64> %l3, <2 x i64> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+  %s = shufflevector <4 x i64> %t1, <4 x i64> %t2, <6 x i32> <i32 0, i32 2, i32 4, i32 1, i32 3, i32 5>
+  store <6 x i64> %s, <6 x i64> *%dst
+  ret void
+}
+
+define void @vst3_v4i64(<4 x i64> *%src, <12 x i64> *%dst) {
+; CHECK-LABEL: vst3_v4i64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    .pad #16
+; CHECK-NEXT:    sub sp, #16
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vldrw.u32 q7, [r0, #32]
+; CHECK-NEXT:    vldrw.u32 q6, [r0, #16]
+; CHECK-NEXT:    vldrw.u32 q3, [r0, #48]
+; CHECK-NEXT:    vmov.f64 d10, d2
+; CHECK-NEXT:    vstrw.32 q7, [sp] @ 16-byte Spill
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #80]
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #64]
+; CHECK-NEXT:    vmov.f32 s21, s5
+; CHECK-NEXT:    vmov.f32 s22, s28
+; CHECK-NEXT:    vmov.f32 s23, s29
+; CHECK-NEXT:    vmov.f64 d14, d12
+; CHECK-NEXT:    vstrw.32 q5, [r1]
+; CHECK-NEXT:    vmov.f32 s29, s25
+; CHECK-NEXT:    vmov.f64 d8, d7
+; CHECK-NEXT:    vmov.f32 s30, s12
+; CHECK-NEXT:    vmov.f32 s17, s15
+; CHECK-NEXT:    vmov.f32 s31, s13
+; CHECK-NEXT:    vldrw.u32 q3, [sp] @ 16-byte Reload
+; CHECK-NEXT:    vmov.f32 s18, s2
+; CHECK-NEXT:    vstrw.32 q7, [r1, #48]
+; CHECK-NEXT:    vmov.f32 s4, s8
+; CHECK-NEXT:    vmov.f32 s19, s3
+; CHECK-NEXT:    vmov.f32 s2, s26
+; CHECK-NEXT:    vstrw.32 q4, [r1, #80]
+; CHECK-NEXT:    vmov.f32 s5, s9
+; CHECK-NEXT:    vmov.f32 s8, s14
+; CHECK-NEXT:    vstrw.32 q1, [r1, #16]
+; CHECK-NEXT:    vmov.f32 s3, s27
+; CHECK-NEXT:    vmov.f32 s9, s15
+; CHECK-NEXT:    vstrw.32 q0, [r1, #64]
+; CHECK-NEXT:    vstrw.32 q2, [r1, #32]
+; CHECK-NEXT:    add sp, #16
+; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    bx lr
+entry:
+  %s1 = getelementptr <4 x i64>, <4 x i64>* %src, i32 0
+  %l1 = load <4 x i64>, <4 x i64>* %s1, align 4
+  %s2 = getelementptr <4 x i64>, <4 x i64>* %src, i32 1
+  %l2 = load <4 x i64>, <4 x i64>* %s2, align 4
+  %s3 = getelementptr <4 x i64>, <4 x i64>* %src, i32 2
+  %l3 = load <4 x i64>, <4 x i64>* %s3, align 4
+  %t1 = shufflevector <4 x i64> %l1, <4 x i64> %l2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %t2 = shufflevector <4 x i64> %l3, <4 x i64> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+  %s = shufflevector <8 x i64> %t1, <8 x i64> %t2, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+  store <12 x i64> %s, <12 x i64> *%dst
+  ret void
+}
+
+; f32
+
+define void @vst3_v2f32(<2 x float> *%src, <6 x float> *%dst) {
+; CHECK-LABEL: vst3_v2f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldr s0, [r0]
+; CHECK-NEXT:    vldr s3, [r0, #4]
+; CHECK-NEXT:    vldr s1, [r0, #8]
+; CHECK-NEXT:    ldr r2, [r0, #20]
+; CHECK-NEXT:    vldr s2, [r0, #16]
+; CHECK-NEXT:    ldr r0, [r0, #12]
+; CHECK-NEXT:    strd r0, r2, [r1, #16]
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %s1 = getelementptr <2 x float>, <2 x float>* %src, i32 0
+  %l1 = load <2 x float>, <2 x float>* %s1, align 4
+  %s2 = getelementptr <2 x float>, <2 x float>* %src, i32 1
+  %l2 = load <2 x float>, <2 x float>* %s2, align 4
+  %s3 = getelementptr <2 x float>, <2 x float>* %src, i32 2
+  %l3 = load <2 x float>, <2 x float>* %s3, align 4
+  %t1 = shufflevector <2 x float> %l1, <2 x float> %l2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %t2 = shufflevector <2 x float> %l3, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+  %s = shufflevector <4 x float> %t1, <4 x float> %t2, <6 x i32> <i32 0, i32 2, i32 4, i32 1, i32 3, i32 5>
+  store <6 x float> %s, <6 x float> *%dst
+  ret void
+}
+
+define void @vst3_v4f32(<4 x float> *%src, <12 x float> *%dst) {
+; CHECK-LABEL: vst3_v4f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11}
+; CHECK-NEXT:    vldrw.u32 q4, [r0]
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #16]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #32]
+; CHECK-NEXT:    vmov.f32 s4, s9
+; CHECK-NEXT:    vmov.f64 d6, d8
+; CHECK-NEXT:    vmov.32 r0, q0[0]
+; CHECK-NEXT:    vdup.32 q5, r0
+; CHECK-NEXT:    vmov.32 r0, q2[3]
+; CHECK-NEXT:    vmov.f32 s5, s1
+; CHECK-NEXT:    vmov.f32 s13, s8
+; CHECK-NEXT:    vmov.f32 s0, s2
+; CHECK-NEXT:    vmov.f32 s7, s10
+; CHECK-NEXT:    vdup.32 q2, r0
+; CHECK-NEXT:    vmov.f32 s15, s17
+; CHECK-NEXT:    vmov.f32 s1, s19
+; CHECK-NEXT:    vmov.f32 s6, s18
+; CHECK-NEXT:    vmov.f32 s14, s22
+; CHECK-NEXT:    vstrw.32 q1, [r1, #16]
+; CHECK-NEXT:    vmov.f32 s2, s10
+; CHECK-NEXT:    vstrw.32 q3, [r1]
+; CHECK-NEXT:    vstrw.32 q0, [r1, #32]
+; CHECK-NEXT:    vpop {d8, d9, d10, d11}
+; CHECK-NEXT:    bx lr
+entry:
+  %s1 = getelementptr <4 x float>, <4 x float>* %src, i32 0
+  %l1 = load <4 x float>, <4 x float>* %s1, align 4
+  %s2 = getelementptr <4 x float>, <4 x float>* %src, i32 1
+  %l2 = load <4 x float>, <4 x float>* %s2, align 4
+  %s3 = getelementptr <4 x float>, <4 x float>* %src, i32 2
+  %l3 = load <4 x float>, <4 x float>* %s3, align 4
+  %t1 = shufflevector <4 x float> %l1, <4 x float> %l2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %t2 = shufflevector <4 x float> %l3, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+  %s = shufflevector <8 x float> %t1, <8 x float> %t2, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+  store <12 x float> %s, <12 x float> *%dst
+  ret void
+}
+
+define void @vst3_v8f32(<8 x float> *%src, <24 x float> *%dst) {
+; CHECK-LABEL: vst3_v8f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    .pad #48
+; CHECK-NEXT:    sub sp, #48
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #80]
+; CHECK-NEXT:    vldrw.u32 q5, [r0, #16]
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #48]
+; CHECK-NEXT:    vmov.f64 d6, d1
+; CHECK-NEXT:    vmov.32 r2, q1[3]
+; CHECK-NEXT:    vldrw.u32 q4, [r0]
+; CHECK-NEXT:    vldrw.u32 q6, [r0, #32]
+; CHECK-NEXT:    vdup.32 q2, r2
+; CHECK-NEXT:    vldrw.u32 q7, [r0, #64]
+; CHECK-NEXT:    vstrw.32 q4, [sp, #32] @ 16-byte Spill
+; CHECK-NEXT:    vmov.f32 s13, s23
+; CHECK-NEXT:    vstrw.32 q6, [sp, #16] @ 16-byte Spill
+; CHECK-NEXT:    vmov.f32 s15, s3
+; CHECK-NEXT:    vstrw.32 q5, [sp] @ 16-byte Spill
+; CHECK-NEXT:    vmov.f32 s14, s10
+; CHECK-NEXT:    vmov.32 r0, q7[0]
+; CHECK-NEXT:    vmov.f64 d4, d8
+; CHECK-NEXT:    vmov.f32 s9, s24
+; CHECK-NEXT:    vmov.f32 s11, s17
+; CHECK-NEXT:    vmov q4, q5
+; CHECK-NEXT:    vmov.f32 s21, s4
+; CHECK-NEXT:    vmov.f32 s23, s17
+; CHECK-NEXT:    vdup.32 q4, r0
+; CHECK-NEXT:    vmov.32 r0, q0[0]
+; CHECK-NEXT:    vmov.f32 s0, s5
+; CHECK-NEXT:    vmov.f32 s10, s18
+; CHECK-NEXT:    vldrw.u32 q4, [sp, #16] @ 16-byte Reload
+; CHECK-NEXT:    vmov.f32 s3, s6
+; CHECK-NEXT:    vldrw.u32 q1, [sp] @ 16-byte Reload
+; CHECK-NEXT:    vdup.32 q6, r0
+; CHECK-NEXT:    vmov.f32 s22, s26
+; CHECK-NEXT:    vldrw.u32 q6, [sp, #32] @ 16-byte Reload
+; CHECK-NEXT:    vmov.f32 s2, s6
+; CHECK-NEXT:    vmov.32 r0, q4[3]
+; CHECK-NEXT:    vmov.f32 s4, s17
+; CHECK-NEXT:    vstrw.32 q5, [r1, #48]
+; CHECK-NEXT:    vmov.f32 s5, s29
+; CHECK-NEXT:    vstrw.32 q0, [r1, #64]
+; CHECK-NEXT:    vmov.f32 s28, s30
+; CHECK-NEXT:    vstrw.32 q3, [r1, #80]
+; CHECK-NEXT:    vmov.f32 s7, s18
+; CHECK-NEXT:    vdup.32 q4, r0
+; CHECK-NEXT:    vmov.f32 s29, s27
+; CHECK-NEXT:    vstrw.32 q2, [r1]
+; CHECK-NEXT:    vmov.f32 s6, s26
+; CHECK-NEXT:    vmov.f32 s30, s18
+; CHECK-NEXT:    vstrw.32 q1, [r1, #16]
+; CHECK-NEXT:    vstrw.32 q7, [r1, #32]
+; CHECK-NEXT:    add sp, #48
+; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    bx lr
+entry:
+  %s1 = getelementptr <8 x float>, <8 x float>* %src, i32 0
+  %l1 = load <8 x float>, <8 x float>* %s1, align 4
+  %s2 = getelementptr <8 x float>, <8 x float>* %src, i32 1
+  %l2 = load <8 x float>, <8 x float>* %s2, align 4
+  %s3 = getelementptr <8 x float>, <8 x float>* %src, i32 2
+  %l3 = load <8 x float>, <8 x float>* %s3, align 4
+  %t1 = shufflevector <8 x float> %l1, <8 x float> %l2, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %t2 = shufflevector <8 x float> %l3, <8 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %s = shufflevector <16 x float> %t1, <16 x float> %t2, <24 x i32> <i32 0, i32 8, i32 16, i32 1, i32 9, i32 17, i32 2, i32 10, i32 18, i32 3, i32 11, i32 19, i32 4, i32 12, i32 20, i32 5, i32 13, i32 21, i32 6, i32 14, i32 22, i32 7, i32 15, i32 23>
+  store <24 x float> %s, <24 x float> *%dst
+  ret void
+}
+
+define void @vst3_v16f32(<16 x float> *%src, <48 x float> *%dst) {
+; CHECK-LABEL: vst3_v16f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    .pad #160
+; CHECK-NEXT:    sub sp, #160
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #80]
+; CHECK-NEXT:    vldrw.u32 q6, [r0, #64]
+; CHECK-NEXT:    vldrw.u32 q7, [r0, #128]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #144]
+; CHECK-NEXT:    vstrw.32 q2, [sp, #144] @ 16-byte Spill
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #16]
+; CHECK-NEXT:    vmov.f32 s4, s25
+; CHECK-NEXT:    vldrw.u32 q5, [r0]
+; CHECK-NEXT:    vstrw.32 q2, [sp, #128] @ 16-byte Spill
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #160]
+; CHECK-NEXT:    vmov.f32 s5, s29
+; CHECK-NEXT:    vstrw.32 q0, [sp] @ 16-byte Spill
+; CHECK-NEXT:    vstrw.32 q2, [sp, #112] @ 16-byte Spill
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #96]
+; CHECK-NEXT:    vmov.f32 s7, s26
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #48]
+; CHECK-NEXT:    vstrw.32 q2, [sp, #96] @ 16-byte Spill
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #32]
+; CHECK-NEXT:    vmov.f32 s6, s22
+; CHECK-NEXT:    vldrw.u32 q3, [r0, #112]
+; CHECK-NEXT:    vstrw.32 q2, [sp, #16] @ 16-byte Spill
+; CHECK-NEXT:    vmov.f64 d4, d15
+; CHECK-NEXT:    vldrw.u32 q4, [r0, #176]
+; CHECK-NEXT:    vstrw.32 q0, [sp, #32] @ 16-byte Spill
+; CHECK-NEXT:    vstrw.32 q1, [r1, #16]
+; CHECK-NEXT:    vmov.32 r0, q6[3]
+; CHECK-NEXT:    vdup.32 q1, r0
+; CHECK-NEXT:    vmov.f32 s9, s23
+; CHECK-NEXT:    vmov.f32 s11, s31
+; CHECK-NEXT:    vmov.f32 s10, s6
+; CHECK-NEXT:    vmov.f64 d2, d9
+; CHECK-NEXT:    vstrw.32 q2, [r1, #32]
+; CHECK-NEXT:    vmov.32 r0, q3[3]
+; CHECK-NEXT:    vmov.f32 s5, s3
+; CHECK-NEXT:    vmov.f32 s7, s19
+; CHECK-NEXT:    vmov q2, q1
+; CHECK-NEXT:    vdup.32 q1, r0
+; CHECK-NEXT:    vmov.f32 s10, s6
+; CHECK-NEXT:    vmov.f64 d2, d10
+; CHECK-NEXT:    vstrw.32 q2, [sp, #80] @ 16-byte Spill
+; CHECK-NEXT:    vmov.32 r0, q7[0]
+; CHECK-NEXT:    vldrw.u32 q2, [sp, #32] @ 16-byte Reload
+; CHECK-NEXT:    vdup.32 q0, r0
+; CHECK-NEXT:    vmov.f32 s5, s24
+; CHECK-NEXT:    vmov.f32 s7, s21
+; CHECK-NEXT:    vmov.f32 s6, s2
+; CHECK-NEXT:    vmov.f64 d0, d4
+; CHECK-NEXT:    vstrw.32 q1, [sp, #64] @ 16-byte Spill
+; CHECK-NEXT:    vmov.32 r0, q4[0]
+; CHECK-NEXT:    vmov.f32 s1, s12
+; CHECK-NEXT:    vmov.f32 s3, s9
+; CHECK-NEXT:    vmov q1, q0
+; CHECK-NEXT:    vdup.32 q0, r0
+; CHECK-NEXT:    vmov.f32 s6, s2
+; CHECK-NEXT:    vldrw.u32 q0, [sp, #112] @ 16-byte Reload
+; CHECK-NEXT:    vstrw.32 q1, [sp, #48] @ 16-byte Spill
+; CHECK-NEXT:    vldrw.u32 q1, [sp, #96] @ 16-byte Reload
+; CHECK-NEXT:    vmov.f32 s16, s13
+; CHECK-NEXT:    vmov.f32 s24, s5
+; CHECK-NEXT:    vmov.f32 s25, s1
+; CHECK-NEXT:    vmov.f32 s19, s14
+; CHECK-NEXT:    vmov q3, q1
+; CHECK-NEXT:    vmov.f32 s27, s6
+; CHECK-NEXT:    vldrw.u32 q1, [sp, #16] @ 16-byte Reload
+; CHECK-NEXT:    vmov.f32 s18, s10
+; CHECK-NEXT:    vmov.f64 d4, d1
+; CHECK-NEXT:    vstrw.32 q4, [sp, #32] @ 16-byte Spill
+; CHECK-NEXT:    vmov.32 r0, q3[3]
+; CHECK-NEXT:    vmov q5, q1
+; CHECK-NEXT:    vldrw.u32 q4, [sp, #144] @ 16-byte Reload
+; CHECK-NEXT:    vmov.f32 s9, s7
+; CHECK-NEXT:    vmov.f32 s11, s3
+; CHECK-NEXT:    vdup.32 q0, r0
+; CHECK-NEXT:    vmov.f32 s10, s2
+; CHECK-NEXT:    vldrw.u32 q0, [sp] @ 16-byte Reload
+; CHECK-NEXT:    vmov.f32 s26, s6
+; CHECK-NEXT:    vldrw.u32 q1, [sp, #128] @ 16-byte Reload
+; CHECK-NEXT:    vmov.f64 d6, d1
+; CHECK-NEXT:    vmov.32 r0, q4[3]
+; CHECK-NEXT:    vmov q7, q0
+; CHECK-NEXT:    vmov.f32 s13, s7
+; CHECK-NEXT:    vldrw.u32 q1, [sp, #96] @ 16-byte Reload
+; CHECK-NEXT:    vmov.f32 s15, s3
+; CHECK-NEXT:    vdup.32 q0, r0
+; CHECK-NEXT:    vmov.f32 s14, s2
+; CHECK-NEXT:    vmov q0, q5
+; CHECK-NEXT:    vmov.f32 s21, s4
+; CHECK-NEXT:    vmov.f32 s23, s1
+; CHECK-NEXT:    vldrw.u32 q0, [sp, #112] @ 16-byte Reload
+; CHECK-NEXT:    vmov.32 r0, q0[0]
+; CHECK-NEXT:    vdup.32 q0, r0
+; CHECK-NEXT:    vmov.f32 s22, s2
+; CHECK-NEXT:    vldrw.u32 q0, [sp, #128] @ 16-byte Reload
+; CHECK-NEXT:    vmov.32 r0, q7[0]
+; CHECK-NEXT:    vstrw.32 q3, [r1, #80]
+; CHECK-NEXT:    vmov.f64 d2, d0
+; CHECK-NEXT:    vstrw.32 q5, [r1, #96]
+; CHECK-NEXT:    vstrw.32 q6, [r1, #112]
+; CHECK-NEXT:    vstrw.32 q2, [r1, #128]
+; CHECK-NEXT:    vmov.f32 s5, s16
+; CHECK-NEXT:    vmov q4, q0
+; CHECK-NEXT:    vmov.f32 s7, s1
+; CHECK-NEXT:    vdup.32 q0, r0
+; CHECK-NEXT:    vmov.f32 s6, s2
+; CHECK-NEXT:    vldrw.u32 q0, [sp, #144] @ 16-byte Reload
+; CHECK-NEXT:    vstrw.32 q1, [r1, #48]
+; CHECK-NEXT:    vmov.f32 s28, s1
+; CHECK-NEXT:    vmov.f32 s31, s2
+; CHECK-NEXT:    vldrw.u32 q0, [sp, #48] @ 16-byte Reload
+; CHECK-NEXT:    vmov.f32 s30, s18
+; CHECK-NEXT:    vstrw.32 q0, [r1, #144]
+; CHECK-NEXT:    vldrw.u32 q0, [sp, #32] @ 16-byte Reload
+; CHECK-NEXT:    vstrw.32 q7, [r1, #64]
+; CHECK-NEXT:    vstrw.32 q0, [r1, #160]
+; CHECK-NEXT:    vldrw.u32 q0, [sp, #80] @ 16-byte Reload
+; CHECK-NEXT:    vstrw.32 q0, [r1, #176]
+; CHECK-NEXT:    vldrw.u32 q0, [sp, #64] @ 16-byte Reload
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    add sp, #160
+; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    bx lr
+entry:
+  %s1 = getelementptr <16 x float>, <16 x float>* %src, i32 0
+  %l1 = load <16 x float>, <16 x float>* %s1, align 4
+  %s2 = getelementptr <16 x float>, <16 x float>* %src, i32 1
+  %l2 = load <16 x float>, <16 x float>* %s2, align 4
+  %s3 = getelementptr <16 x float>, <16 x float>* %src, i32 2
+  %l3 = load <16 x float>, <16 x float>* %s3, align 4
+  %t1 = shufflevector <16 x float> %l1, <16 x float> %l2, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+  %t2 = shufflevector <16 x float> %l3, <16 x float> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %s = shufflevector <32 x float> %t1, <32 x float> %t2, <48 x i32> <i32 0, i32 16, i32 32, i32 1, i32 17, i32 33, i32 2, i32 18, i32 34, i32 3, i32 19, i32 35, i32 4, i32 20, i32 36, i32 5, i32 21, i32 37, i32 6, i32 22, i32 38, i32 7, i32 23, i32 39, i32 8, i32 24, i32 40, i32 9, i32 25, i32 41, i32 10, i32 26, i32 42, i32 11, i32 27, i32 43, i32 12, i32 28, i32 44, i32 13, i32 29, i32 45, i32 14, i32 30, i32 46, i32 15, i32 31, i32 47>
+  store <48 x float> %s, <48 x float> *%dst
+  ret void
+}
+
+; f16
+
+define void @vst3_v2f16(<2 x half> *%src, <6 x half> *%dst) {
+; CHECK-LABEL: vst3_v2f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldmia r0, {s4, s5}
+; CHECK-NEXT:    vmov r2, s5
+; CHECK-NEXT:    ldr r0, [r0, #8]
+; CHECK-NEXT:    vmov r3, s4
+; CHECK-NEXT:    vmovx.f16 s12, s4
+; CHECK-NEXT:    vmov.16 q0[0], r3
+; CHECK-NEXT:    vmov.32 q2[0], r0
+; CHECK-NEXT:    vmov.16 q0[1], r2
+; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    vmov.16 q0[2], r0
+; CHECK-NEXT:    vmov r0, s12
+; CHECK-NEXT:    vmovx.f16 s4, s5
+; CHECK-NEXT:    vmov.16 q0[3], r0
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmovx.f16 s4, s8
+; CHECK-NEXT:    vmov.16 q0[4], r0
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmov.16 q0[5], r0
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vmov r2, s1
+; CHECK-NEXT:    vmov r3, s2
+; CHECK-NEXT:    stm r1!, {r0, r2, r3}
+; CHECK-NEXT:    bx lr
+entry:
+  %s1 = getelementptr <2 x half>, <2 x half>* %src, i32 0
+  %l1 = load <2 x half>, <2 x half>* %s1, align 4
+  %s2 = getelementptr <2 x half>, <2 x half>* %src, i32 1
+  %l2 = load <2 x half>, <2 x half>* %s2, align 4
+  %s3 = getelementptr <2 x half>, <2 x half>* %src, i32 2
+  %l3 = load <2 x half>, <2 x half>* %s3, align 4
+  %t1 = shufflevector <2 x half> %l1, <2 x half> %l2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %t2 = shufflevector <2 x half> %l3, <2 x half> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+  %s = shufflevector <4 x half> %t1, <4 x half> %t2, <6 x i32> <i32 0, i32 2, i32 4, i32 1, i32 3, i32 5>
+  store <6 x half> %s, <6 x half> *%dst
+  ret void
+}
+
+define void @vst3_v4f16(<4 x half> *%src, <12 x half> *%dst) {
+; CHECK-LABEL: vst3_v4f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    ldm.w r0, {r2, r3, r12, lr}
+; CHECK-NEXT:    vmov.32 q0[0], r2
+; CHECK-NEXT:    vmov.32 q0[1], r3
+; CHECK-NEXT:    vmov.32 q0[2], r12
+; CHECK-NEXT:    vmov.32 q0[3], lr
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    vmovx.f16 s12, s0
+; CHECK-NEXT:    vmov r2, s2
+; CHECK-NEXT:    vmov.16 q2[0], r3
+; CHECK-NEXT:    vmov.16 q2[1], r2
+; CHECK-NEXT:    ldrd r2, r0, [r0, #16]
+; CHECK-NEXT:    vmovx.f16 s0, s3
+; CHECK-NEXT:    vmov.32 q1[0], r2
+; CHECK-NEXT:    vmov.32 q1[1], r0
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmov.16 q2[2], r0
+; CHECK-NEXT:    vmov r0, s12
+; CHECK-NEXT:    vmovx.f16 s12, s2
+; CHECK-NEXT:    vmov.16 q2[3], r0
+; CHECK-NEXT:    vmov r0, s12
+; CHECK-NEXT:    vmovx.f16 s12, s4
+; CHECK-NEXT:    vmov.16 q2[4], r0
+; CHECK-NEXT:    vmov r0, s12
+; CHECK-NEXT:    vmov.16 q2[5], r0
+; CHECK-NEXT:    vmov r0, s1
+; CHECK-NEXT:    vmov.16 q2[6], r0
+; CHECK-NEXT:    vmov r0, s3
+; CHECK-NEXT:    vmov.16 q2[7], r0
+; CHECK-NEXT:    vmov r2, s5
+; CHECK-NEXT:    vstrw.32 q2, [r1]
+; CHECK-NEXT:    vmovx.f16 s8, s1
+; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    vmov.16 q2[0], r2
+; CHECK-NEXT:    vmov.16 q2[1], r0
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vmovx.f16 s0, s5
+; CHECK-NEXT:    vmov.16 q2[2], r0
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vmov.16 q2[3], r0
+; CHECK-NEXT:    vmov r0, s9
+; CHECK-NEXT:    vmov r2, s8
+; CHECK-NEXT:    strd r2, r0, [r1, #16]
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %s1 = getelementptr <4 x half>, <4 x half>* %src, i32 0
+  %l1 = load <4 x half>, <4 x half>* %s1, align 4
+  %s2 = getelementptr <4 x half>, <4 x half>* %src, i32 1
+  %l2 = load <4 x half>, <4 x half>* %s2, align 4
+  %s3 = getelementptr <4 x half>, <4 x half>* %src, i32 2
+  %l3 = load <4 x half>, <4 x half>* %s3, align 4
+  %t1 = shufflevector <4 x half> %l1, <4 x half> %l2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %t2 = shufflevector <4 x half> %l3, <4 x half> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+  %s = shufflevector <8 x half> %t1, <8 x half> %t2, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+  store <12 x half> %s, <12 x half> *%dst
+  ret void
+}
+
+define void @vst3_v8f16(<8 x half> *%src, <24 x half> *%dst) {
+; CHECK-LABEL: vst3_v8f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    .pad #16
+; CHECK-NEXT:    sub sp, #16
+; CHECK-NEXT:    vldrw.u32 q2, [r0]
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #16]
+; CHECK-NEXT:    vldrw.u32 q5, [r0, #32]
+; CHECK-NEXT:    vmov r3, s8
+; CHECK-NEXT:    vmovx.f16 s12, s4
+; CHECK-NEXT:    vmov r2, s4
+; CHECK-NEXT:    vmov.16 q0[0], r3
+; CHECK-NEXT:    vmov.32 r0, q5[0]
+; CHECK-NEXT:    vmov.16 q0[1], r2
+; CHECK-NEXT:    vmov r2, s12
+; CHECK-NEXT:    vdup.32 q4, r0
+; CHECK-NEXT:    vmov.16 q0[4], r2
+; CHECK-NEXT:    vmov r2, s9
+; CHECK-NEXT:    vmov.16 q0[6], r2
+; CHECK-NEXT:    vmov r2, s5
+; CHECK-NEXT:    vmovx.f16 s12, s8
+; CHECK-NEXT:    vmov r0, s17
+; CHECK-NEXT:    vmov.16 q0[7], r2
+; CHECK-NEXT:    vmov r2, s12
+; CHECK-NEXT:    vmov.16 q3[2], r0
+; CHECK-NEXT:    vmov.f32 s1, s8
+; CHECK-NEXT:    vmov.16 q3[3], r2
+; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    vmovx.f16 s16, s18
+; CHECK-NEXT:    vmov.16 q3[4], r0
+; CHECK-NEXT:    vmov r0, s16
+; CHECK-NEXT:    vmovx.f16 s16, s6
+; CHECK-NEXT:    vmov.16 q3[5], r0
+; CHECK-NEXT:    vmov r0, s16
+; CHECK-NEXT:    vmovx.f16 s16, s22
+; CHECK-NEXT:    vmovx.f16 s24, s7
+; CHECK-NEXT:    vmov r2, s16
+; CHECK-NEXT:    vmov.16 q4[0], r0
+; CHECK-NEXT:    vmov.16 q4[1], r2
+; CHECK-NEXT:    vmov r0, s7
+; CHECK-NEXT:    vmov.16 q4[3], r0
+; CHECK-NEXT:    vmov r0, s24
+; CHECK-NEXT:    vmovx.f16 s24, s23
+; CHECK-NEXT:    vmov.16 q4[6], r0
+; CHECK-NEXT:    vmov r0, s24
+; CHECK-NEXT:    vstrw.32 q1, [sp] @ 16-byte Spill
+; CHECK-NEXT:    vmov.16 q4[7], r0
+; CHECK-NEXT:    vmov.32 r2, q2[3]
+; CHECK-NEXT:    vmov.f32 s18, s23
+; CHECK-NEXT:    vdup.32 q7, r2
+; CHECK-NEXT:    vmovx.f16 s24, s17
+; CHECK-NEXT:    vmov r2, s29
+; CHECK-NEXT:    vmov r0, s24
+; CHECK-NEXT:    vmov.16 q6[2], r2
+; CHECK-NEXT:    vmov.16 q6[3], r0
+; CHECK-NEXT:    vmovx.f16 s28, s30
+; CHECK-NEXT:    vmovx.f16 s4, s10
+; CHECK-NEXT:    vmov.f32 s1, s13
+; CHECK-NEXT:    vmov.f32 s2, s14
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vmov r0, s18
+; CHECK-NEXT:    vmov.16 q6[4], r0
+; CHECK-NEXT:    vmov r0, s28
+; CHECK-NEXT:    vmov.16 q6[5], r0
+; CHECK-NEXT:    vmovx.f16 s28, s9
+; CHECK-NEXT:    vmov r0, s21
+; CHECK-NEXT:    vmov r2, s28
+; CHECK-NEXT:    vmov.16 q7[0], r0
+; CHECK-NEXT:    vmov.16 q7[1], r2
+; CHECK-NEXT:    vmov r0, s22
+; CHECK-NEXT:    vmov.16 q7[6], r0
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmov.16 q7[7], r0
+; CHECK-NEXT:    vmov.f32 s17, s25
+; CHECK-NEXT:    vmov.f32 s29, s21
+; CHECK-NEXT:    vmov.f32 s30, s10
+; CHECK-NEXT:    vmovx.f16 s4, s29
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vldrw.u32 q1, [sp] @ 16-byte Reload
+; CHECK-NEXT:    vmov.f32 s18, s26
+; CHECK-NEXT:    vrev32.16 q2, q1
+; CHECK-NEXT:    vstrw.32 q4, [r1, #32]
+; CHECK-NEXT:    vmov r2, s9
+; CHECK-NEXT:    vmovx.f16 s8, s10
+; CHECK-NEXT:    vmov.16 q1[2], r2
+; CHECK-NEXT:    vmov.16 q1[3], r0
+; CHECK-NEXT:    vmov r0, s30
+; CHECK-NEXT:    vmov.16 q1[4], r0
+; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    vmov.16 q1[5], r0
+; CHECK-NEXT:    vmov.f32 s29, s5
+; CHECK-NEXT:    vmov.f32 s30, s6
+; CHECK-NEXT:    vstrw.32 q7, [r1, #16]
+; CHECK-NEXT:    add sp, #16
+; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    bx lr
+entry:
+  %s1 = getelementptr <8 x half>, <8 x half>* %src, i32 0
+  %l1 = load <8 x half>, <8 x half>* %s1, align 4
+  %s2 = getelementptr <8 x half>, <8 x half>* %src, i32 1
+  %l2 = load <8 x half>, <8 x half>* %s2, align 4
+  %s3 = getelementptr <8 x half>, <8 x half>* %src, i32 2
+  %l3 = load <8 x half>, <8 x half>* %s3, align 4
+  %t1 = shufflevector <8 x half> %l1, <8 x half> %l2, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %t2 = shufflevector <8 x half> %l3, <8 x half> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %s = shufflevector <16 x half> %t1, <16 x half> %t2, <24 x i32> <i32 0, i32 8, i32 16, i32 1, i32 9, i32 17, i32 2, i32 10, i32 18, i32 3, i32 11, i32 19, i32 4, i32 12, i32 20, i32 5, i32 13, i32 21, i32 6, i32 14, i32 22, i32 7, i32 15, i32 23>
+  store <24 x half> %s, <24 x half> *%dst
+  ret void
+}
+
+define void @vst3_v16f16(<16 x half> *%src, <48 x half> *%dst) {
+; CHECK-LABEL: vst3_v16f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    .pad #144
+; CHECK-NEXT:    sub sp, #144
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #32]
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    vmov q3, q0
+; CHECK-NEXT:    vmov r2, s8
+; CHECK-NEXT:    vmov.16 q1[0], r3
+; CHECK-NEXT:    vmovx.f16 s0, s8
+; CHECK-NEXT:    vmov.16 q1[1], r2
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vmovx.f16 s0, s12
+; CHECK-NEXT:    vmov.16 q1[4], r2
+; CHECK-NEXT:    vmov r2, s13
+; CHECK-NEXT:    vmov.16 q1[6], r2
+; CHECK-NEXT:    vmov r2, s9
+; CHECK-NEXT:    vmov.16 q1[7], r2
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #64]
+; CHECK-NEXT:    vmov.f32 s5, s12
+; CHECK-NEXT:    vstrw.32 q3, [sp, #32] @ 16-byte Spill
+; CHECK-NEXT:    vstrw.32 q1, [sp, #128] @ 16-byte Spill
+; CHECK-NEXT:    vmov.32 r3, q0[0]
+; CHECK-NEXT:    vmov q4, q0
+; CHECK-NEXT:    vdup.32 q0, r3
+; CHECK-NEXT:    vmov q5, q3
+; CHECK-NEXT:    vmov r3, s1
+; CHECK-NEXT:    vmovx.f16 s0, s2
+; CHECK-NEXT:    vmov.16 q3[2], r3
+; CHECK-NEXT:    vstrw.32 q2, [sp, #80] @ 16-byte Spill
+; CHECK-NEXT:    vmov.16 q3[3], r2
+; CHECK-NEXT:    vmov r2, s6
+; CHECK-NEXT:    vmov.16 q3[4], r2
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vmovx.f16 s0, s10
+; CHECK-NEXT:    vmov.16 q3[5], r2
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vmovx.f16 s0, s18
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    vmov.16 q1[0], r2
+; CHECK-NEXT:    vmov.16 q1[1], r3
+; CHECK-NEXT:    vmov r2, s11
+; CHECK-NEXT:    vmovx.f16 s0, s11
+; CHECK-NEXT:    vmov.16 q1[3], r2
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vmovx.f16 s0, s19
+; CHECK-NEXT:    vmov.16 q1[6], r2
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vmov.16 q1[7], r2
+; CHECK-NEXT:    vstrw.32 q3, [sp, #112] @ 16-byte Spill
+; CHECK-NEXT:    vmov.f32 s6, s19
+; CHECK-NEXT:    vmovx.f16 s0, s5
+; CHECK-NEXT:    vstrw.32 q4, [sp, #48] @ 16-byte Spill
+; CHECK-NEXT:    vstrw.32 q1, [sp, #96] @ 16-byte Spill
+; CHECK-NEXT:    vmov.32 r3, q5[3]
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vdup.32 q0, r3
+; CHECK-NEXT:    vldrw.u32 q5, [r0, #16]
+; CHECK-NEXT:    vmov r3, s1
+; CHECK-NEXT:    vmovx.f16 s0, s2
+; CHECK-NEXT:    vmov.16 q2[2], r3
+; CHECK-NEXT:    vmov r3, s20
+; CHECK-NEXT:    vmov.16 q2[3], r2
+; CHECK-NEXT:    vmov.16 q7[0], r3
+; CHECK-NEXT:    vmov r2, s6
+; CHECK-NEXT:    vldrw.u32 q3, [r0, #80]
+; CHECK-NEXT:    vmov.16 q2[4], r2
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vmov.16 q2[5], r2
+; CHECK-NEXT:    vstrw.32 q2, [sp, #64] @ 16-byte Spill
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #48]
+; CHECK-NEXT:    vmov.32 r0, q3[0]
+; CHECK-NEXT:    vmov r2, s8
+; CHECK-NEXT:    vmovx.f16 s0, s8
+; CHECK-NEXT:    vmov.16 q7[1], r2
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vmov.16 q7[4], r2
+; CHECK-NEXT:    vmov r2, s21
+; CHECK-NEXT:    vmov.16 q7[6], r2
+; CHECK-NEXT:    vmov r2, s9
+; CHECK-NEXT:    vmovx.f16 s0, s20
+; CHECK-NEXT:    vmov.16 q7[7], r2
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vdup.32 q0, r0
+; CHECK-NEXT:    vmov r0, s1
+; CHECK-NEXT:    vmovx.f16 s0, s2
+; CHECK-NEXT:    vmov.16 q1[2], r0
+; CHECK-NEXT:    vmov.f32 s29, s20
+; CHECK-NEXT:    vmov.16 q1[3], r2
+; CHECK-NEXT:    vmov r0, s30
+; CHECK-NEXT:    vmov.16 q1[4], r0
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vmov.16 q1[5], r0
+; CHECK-NEXT:    vstrw.32 q1, [sp, #16] @ 16-byte Spill
+; CHECK-NEXT:    vmov q1, q2
+; CHECK-NEXT:    vmovx.f16 s0, s6
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vmovx.f16 s0, s14
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vmov.16 q2[0], r0
+; CHECK-NEXT:    vmov.16 q2[1], r2
+; CHECK-NEXT:    vmov r0, s7
+; CHECK-NEXT:    vmovx.f16 s0, s7
+; CHECK-NEXT:    vmov.16 q2[3], r0
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vmovx.f16 s0, s15
+; CHECK-NEXT:    vmov.16 q2[6], r0
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vmov.16 q2[7], r0
+; CHECK-NEXT:    vmov.32 r2, q5[3]
+; CHECK-NEXT:    vmov.f32 s10, s15
+; CHECK-NEXT:    vmovx.f16 s0, s9
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vdup.32 q0, r2
+; CHECK-NEXT:    vmov r2, s1
+; CHECK-NEXT:    vmovx.f16 s0, s2
+; CHECK-NEXT:    vmov.16 q6[2], r2
+; CHECK-NEXT:    vmov.16 q6[3], r0
+; CHECK-NEXT:    vmov r0, s10
+; CHECK-NEXT:    vmov.16 q6[4], r0
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vmov.16 q6[5], r0
+; CHECK-NEXT:    vmov r0, s13
+; CHECK-NEXT:    vmovx.f16 s0, s21
+; CHECK-NEXT:    vmov.16 q4[0], r0
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vmovx.f16 s0, s22
+; CHECK-NEXT:    vmov.16 q4[1], r2
+; CHECK-NEXT:    vmov r0, s14
+; CHECK-NEXT:    vmov.16 q4[6], r0
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vmov.16 q4[7], r0
+; CHECK-NEXT:    vmov.f32 s9, s25
+; CHECK-NEXT:    vmov.f32 s17, s13
+; CHECK-NEXT:    vldrw.u32 q3, [sp, #32] @ 16-byte Reload
+; CHECK-NEXT:    vmov.f32 s18, s22
+; CHECK-NEXT:    vmovx.f16 s0, s17
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vrev32.16 q0, q1
+; CHECK-NEXT:    vmov r2, s1
+; CHECK-NEXT:    vmovx.f16 s0, s2
+; CHECK-NEXT:    vmov.16 q1[2], r2
+; CHECK-NEXT:    vmov.f32 s10, s26
+; CHECK-NEXT:    vmov.16 q1[3], r0
+; CHECK-NEXT:    vldrw.u32 q6, [sp, #112] @ 16-byte Reload
+; CHECK-NEXT:    vstrw.32 q2, [r1, #80]
+; CHECK-NEXT:    vmov r0, s18
+; CHECK-NEXT:    vmov.16 q1[4], r0
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vmov.16 q1[5], r0
+; CHECK-NEXT:    vmovx.f16 s0, s13
+; CHECK-NEXT:    vstrw.32 q1, [sp] @ 16-byte Spill
+; CHECK-NEXT:    vldrw.u32 q1, [sp, #48] @ 16-byte Reload
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vmovx.f16 s0, s14
+; CHECK-NEXT:    vmov r0, s5
+; CHECK-NEXT:    vmov.16 q5[0], r0
+; CHECK-NEXT:    vmov r0, s6
+; CHECK-NEXT:    vmov.16 q5[1], r2
+; CHECK-NEXT:    vmov.16 q5[6], r0
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vmov.16 q5[7], r0
+; CHECK-NEXT:    vmov.f32 s21, s5
+; CHECK-NEXT:    vldrw.u32 q1, [sp, #64] @ 16-byte Reload
+; CHECK-NEXT:    vmov.f32 s22, s14
+; CHECK-NEXT:    vmovx.f16 s0, s21
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vldrw.u32 q0, [sp, #80] @ 16-byte Reload
+; CHECK-NEXT:    vrev32.16 q3, q0
+; CHECK-NEXT:    vmov r2, s13
+; CHECK-NEXT:    vmovx.f16 s12, s14
+; CHECK-NEXT:    vmov.16 q0[2], r2
+; CHECK-NEXT:    vmov.16 q0[3], r0
+; CHECK-NEXT:    vmov r0, s22
+; CHECK-NEXT:    vmov.16 q0[4], r0
+; CHECK-NEXT:    vmov r0, s12
+; CHECK-NEXT:    vldrw.u32 q3, [sp, #128] @ 16-byte Reload
+; CHECK-NEXT:    vmov.16 q0[5], r0
+; CHECK-NEXT:    vmov.f32 s13, s25
+; CHECK-NEXT:    vmov.f32 s14, s26
+; CHECK-NEXT:    vldrw.u32 q6, [sp, #96] @ 16-byte Reload
+; CHECK-NEXT:    vmov.f32 s25, s5
+; CHECK-NEXT:    vstrw.32 q3, [r1]
+; CHECK-NEXT:    vmov.f32 s21, s1
+; CHECK-NEXT:    vmov.f32 s26, s6
+; CHECK-NEXT:    vldrw.u32 q1, [sp, #16] @ 16-byte Reload
+; CHECK-NEXT:    vmov.f32 s22, s2
+; CHECK-NEXT:    vldrw.u32 q0, [sp] @ 16-byte Reload
+; CHECK-NEXT:    vmov.f32 s29, s5
+; CHECK-NEXT:    vstrw.32 q6, [r1, #32]
+; CHECK-NEXT:    vmov.f32 s17, s1
+; CHECK-NEXT:    vstrw.32 q5, [r1, #16]
+; CHECK-NEXT:    vmov.f32 s30, s6
+; CHECK-NEXT:    vmov.f32 s18, s2
+; CHECK-NEXT:    vstrw.32 q7, [r1, #48]
+; CHECK-NEXT:    vstrw.32 q4, [r1, #64]
+; CHECK-NEXT:    add sp, #144
+; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    bx lr
+entry:
+  %s1 = getelementptr <16 x half>, <16 x half>* %src, i32 0
+  %l1 = load <16 x half>, <16 x half>* %s1, align 4
+  %s2 = getelementptr <16 x half>, <16 x half>* %src, i32 1
+  %l2 = load <16 x half>, <16 x half>* %s2, align 4
+  %s3 = getelementptr <16 x half>, <16 x half>* %src, i32 2
+  %l3 = load <16 x half>, <16 x half>* %s3, align 4
+  %t1 = shufflevector <16 x half> %l1, <16 x half> %l2, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+  %t2 = shufflevector <16 x half> %l3, <16 x half> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %s = shufflevector <32 x half> %t1, <32 x half> %t2, <48 x i32> <i32 0, i32 16, i32 32, i32 1, i32 17, i32 33, i32 2, i32 18, i32 34, i32 3, i32 19, i32 35, i32 4, i32 20, i32 36, i32 5, i32 21, i32 37, i32 6, i32 22, i32 38, i32 7, i32 23, i32 39, i32 8, i32 24, i32 40, i32 9, i32 25, i32 41, i32 10, i32 26, i32 42, i32 11, i32 27, i32 43, i32 12, i32 28, i32 44, i32 13, i32 29, i32 45, i32 14, i32 30, i32 46, i32 15, i32 31, i32 47>
+  store <48 x half> %s, <48 x half> *%dst
+  ret void
+}
+
+; f64
+
+define void @vst3_v2f64(<2 x double> *%src, <6 x double> *%dst) {
+; CHECK-LABEL: vst3_v2f64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #32]
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #16]
+; CHECK-NEXT:    vmov.f64 d6, d2
+; CHECK-NEXT:    vmov.f64 d7, d1
+; CHECK-NEXT:    vmov.f64 d1, d4
+; CHECK-NEXT:    vstrw.32 q3, [r1, #16]
+; CHECK-NEXT:    vmov.f64 d2, d5
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vstrw.32 q1, [r1, #32]
+; CHECK-NEXT:    bx lr
+entry:
+  %s1 = getelementptr <2 x double>, <2 x double>* %src, i32 0
+  %l1 = load <2 x double>, <2 x double>* %s1, align 4
+  %s2 = getelementptr <2 x double>, <2 x double>* %src, i32 1
+  %l2 = load <2 x double>, <2 x double>* %s2, align 4
+  %s3 = getelementptr <2 x double>, <2 x double>* %src, i32 2
+  %l3 = load <2 x double>, <2 x double>* %s3, align 4
+  %t1 = shufflevector <2 x double> %l1, <2 x double> %l2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %t2 = shufflevector <2 x double> %l3, <2 x double> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+  %s = shufflevector <4 x double> %t1, <4 x double> %t2, <6 x i32> <i32 0, i32 2, i32 4, i32 1, i32 3, i32 5>
+  store <6 x double> %s, <6 x double> *%dst
+  ret void
+}
+
+define void @vst3_v4f64(<4 x double> *%src, <12 x double> *%dst) {
+; CHECK-LABEL: vst3_v4f64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    .pad #16
+; CHECK-NEXT:    sub sp, #16
+; CHECK-NEXT:    vldrw.u32 q7, [r0, #48]
+; CHECK-NEXT:    vldrw.u32 q6, [r0, #32]
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #80]
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #16]
+; CHECK-NEXT:    vmov.f64 d6, d15
+; CHECK-NEXT:    vstrw.32 q6, [sp] @ 16-byte Spill
+; CHECK-NEXT:    vldrw.u32 q4, [r0, #64]
+; CHECK-NEXT:    vmov.f64 d10, d2
+; CHECK-NEXT:    vmov.f64 d7, d1
+; CHECK-NEXT:    vmov.f64 d11, d12
+; CHECK-NEXT:    vstrw.32 q3, [r1, #80]
+; CHECK-NEXT:    vmov.f64 d12, d4
+; CHECK-NEXT:    vstrw.32 q5, [r1]
+; CHECK-NEXT:    vmov.f64 d1, d5
+; CHECK-NEXT:    vldrw.u32 q2, [sp] @ 16-byte Reload
+; CHECK-NEXT:    vmov.f64 d2, d8
+; CHECK-NEXT:    vstrw.32 q0, [r1, #64]
+; CHECK-NEXT:    vmov.f64 d13, d14
+; CHECK-NEXT:    vstrw.32 q1, [r1, #16]
+; CHECK-NEXT:    vmov.f64 d8, d5
+; CHECK-NEXT:    vstrw.32 q6, [r1, #48]
+; CHECK-NEXT:    vstrw.32 q4, [r1, #32]
+; CHECK-NEXT:    add sp, #16
+; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    bx lr
+entry:
+  %s1 = getelementptr <4 x double>, <4 x double>* %src, i32 0
+  %l1 = load <4 x double>, <4 x double>* %s1, align 4
+  %s2 = getelementptr <4 x double>, <4 x double>* %src, i32 1
+  %l2 = load <4 x double>, <4 x double>* %s2, align 4
+  %s3 = getelementptr <4 x double>, <4 x double>* %src, i32 2
+  %l3 = load <4 x double>, <4 x double>* %s3, align 4
+  %t1 = shufflevector <4 x double> %l1, <4 x double> %l2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %t2 = shufflevector <4 x double> %l3, <4 x double> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+  %s = shufflevector <8 x double> %t1, <8 x double> %t2, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+  store <12 x double> %s, <12 x double> *%dst
+  ret void
+}
diff --git a/llvm/test/CodeGen/Thumb2/mve-vst4.ll b/llvm/test/CodeGen/Thumb2/mve-vst4.ll
new file mode 100644
index 000000000..177046a74
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-vst4.ll
@@ -0,0 +1,1126 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve.fp,+fp64 -mve-max-interleave-factor=4 -verify-machineinstrs %s -o - | FileCheck %s
+
+; i32
+
+define void @vst4_v2i32(<2 x i32> *%src, <8 x i32> *%dst) {
+; CHECK-LABEL: vst4_v2i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, lr}
+; CHECK-NEXT:    push {r4, lr}
+; CHECK-NEXT:    ldm.w r0, {r2, r3, r12, lr}
+; CHECK-NEXT:    ldrd r4, r0, [r0, #16]
+; CHECK-NEXT:    vmov.32 q1[0], r4
+; CHECK-NEXT:    vmov.32 q1[2], r0
+; CHECK-NEXT:    vmov.f64 d0, d2
+; CHECK-NEXT:    vmov.f32 s1, s6
+; CHECK-NEXT:    vmov.f32 s2, s4
+; CHECK-NEXT:    vmov.f32 s3, s6
+; CHECK-NEXT:    vmov.32 q1[0], r2
+; CHECK-NEXT:    vmov.32 q1[1], r3
+; CHECK-NEXT:    vmov.32 q1[2], r12
+; CHECK-NEXT:    vmov.32 q1[3], lr
+; CHECK-NEXT:    vmov.f64 d4, d2
+; CHECK-NEXT:    vmov.f32 s9, s6
+; CHECK-NEXT:    vmov.f32 s10, s0
+; CHECK-NEXT:    vmov.f32 s11, s2
+; CHECK-NEXT:    vstrw.32 q2, [r1]
+; CHECK-NEXT:    vmov.f32 s8, s5
+; CHECK-NEXT:    vmov.f32 s9, s7
+; CHECK-NEXT:    vmov.f32 s10, s1
+; CHECK-NEXT:    vmov.f32 s11, s3
+; CHECK-NEXT:    vstrw.32 q2, [r1, #16]
+; CHECK-NEXT:    pop {r4, pc}
+entry:
+  %s1 = getelementptr <2 x i32>, <2 x i32>* %src, i32 0
+  %l1 = load <2 x i32>, <2 x i32>* %s1, align 4
+  %s2 = getelementptr <2 x i32>, <2 x i32>* %src, i32 1
+  %l2 = load <2 x i32>, <2 x i32>* %s2, align 4
+  %s3 = getelementptr <2 x i32>, <2 x i32>* %src, i32 2
+  %l3 = load <2 x i32>, <2 x i32>* %s3, align 4
+  %s4 = getelementptr <2 x i32>, <2 x i32>* %src, i32 3
+  %l4 = load <2 x i32>, <2 x i32>* %s3, align 4
+  %t1 = shufflevector <2 x i32> %l1, <2 x i32> %l2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %t2 = shufflevector <2 x i32> %l3, <2 x i32> %l4, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %s = shufflevector <4 x i32> %t1, <4 x i32> %t2, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 1, i32 3, i32 5, i32 7>
+  store <8 x i32> %s, <8 x i32> *%dst
+  ret void
+}
+
+define void @vst4_v4i32(<4 x i32> *%src, <16 x i32> *%dst) {
+; CHECK-LABEL: vst4_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #32]
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #16]
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vmov q3, q2
+; CHECK-NEXT:    vst40.32 {q0, q1, q2, q3}, [r1]
+; CHECK-NEXT:    vst41.32 {q0, q1, q2, q3}, [r1]
+; CHECK-NEXT:    vst42.32 {q0, q1, q2, q3}, [r1]
+; CHECK-NEXT:    vst43.32 {q0, q1, q2, q3}, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %s1 = getelementptr <4 x i32>, <4 x i32>* %src, i32 0
+  %l1 = load <4 x i32>, <4 x i32>* %s1, align 4
+  %s2 = getelementptr <4 x i32>, <4 x i32>* %src, i32 1
+  %l2 = load <4 x i32>, <4 x i32>* %s2, align 4
+  %s3 = getelementptr <4 x i32>, <4 x i32>* %src, i32 2
+  %l3 = load <4 x i32>, <4 x i32>* %s3, align 4
+  %s4 = getelementptr <4 x i32>, <4 x i32>* %src, i32 3
+  %l4 = load <4 x i32>, <4 x i32>* %s3, align 4
+  %t1 = shufflevector <4 x i32> %l1, <4 x i32> %l2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %t2 = shufflevector <4 x i32> %l3, <4 x i32> %l4, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %s = shufflevector <8 x i32> %t1, <8 x i32> %t2, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 1, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
+  store <16 x i32> %s, <16 x i32> *%dst
+  ret void
+}
+
+define void @vst4_v8i32(<8 x i32> *%src, <32 x i32> *%dst) {
+; CHECK-LABEL: vst4_v8i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #80]
+; CHECK-NEXT:    vldrw.u32 q6, [r0, #64]
+; CHECK-NEXT:    vldrw.u32 q5, [r0, #32]
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #48]
+; CHECK-NEXT:    vldrw.u32 q4, [r0]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #16]
+; CHECK-NEXT:    vmov q7, q6
+; CHECK-NEXT:    add.w r0, r1, #64
+; CHECK-NEXT:    vmov q3, q2
+; CHECK-NEXT:    vst40.32 {q4, q5, q6, q7}, [r1]
+; CHECK-NEXT:    vst41.32 {q4, q5, q6, q7}, [r1]
+; CHECK-NEXT:    vst42.32 {q4, q5, q6, q7}, [r1]
+; CHECK-NEXT:    vst43.32 {q4, q5, q6, q7}, [r1]
+; CHECK-NEXT:    vst40.32 {q0, q1, q2, q3}, [r0]
+; CHECK-NEXT:    vst41.32 {q0, q1, q2, q3}, [r0]
+; CHECK-NEXT:    vst42.32 {q0, q1, q2, q3}, [r0]
+; CHECK-NEXT:    vst43.32 {q0, q1, q2, q3}, [r0]
+; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    bx lr
+entry:
+  %s1 = getelementptr <8 x i32>, <8 x i32>* %src, i32 0
+  %l1 = load <8 x i32>, <8 x i32>* %s1, align 4
+  %s2 = getelementptr <8 x i32>, <8 x i32>* %src, i32 1
+  %l2 = load <8 x i32>, <8 x i32>* %s2, align 4
+  %s3 = getelementptr <8 x i32>, <8 x i32>* %src, i32 2
+  %l3 = load <8 x i32>, <8 x i32>* %s3, align 4
+  %s4 = getelementptr <8 x i32>, <8 x i32>* %src, i32 3
+  %l4 = load <8 x i32>, <8 x i32>* %s3, align 4
+  %t1 = shufflevector <8 x i32> %l1, <8 x i32> %l2, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %t2 = shufflevector <8 x i32> %l3, <8 x i32> %l4, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %s = shufflevector <16 x i32> %t1, <16 x i32> %t2, <32 x i32> <i32 0, i32 8, i32 16, i32 24, i32 1, i32 9, i32 17, i32 25, i32 2, i32 10, i32 18, i32 26, i32 3, i32 11, i32 19, i32 27, i32 4, i32 12, i32 20, i32 28, i32 5, i32 13, i32 21, i32 29, i32 6, i32 14, i32 22, i32 30, i32 7, i32 15, i32 23, i32 31>
+  store <32 x i32> %s, <32 x i32> *%dst
+  ret void
+}
+
+define void @vst4_v16i32(<16 x i32> *%src, <64 x i32> *%dst) {
+; CHECK-LABEL: vst4_v16i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, r5}
+; CHECK-NEXT:    push {r4, r5}
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    .pad #152
+; CHECK-NEXT:    sub sp, #152
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #176]
+; CHECK-NEXT:    add r2, sp, #64
+; CHECK-NEXT:    vstmia sp, {d0, d1, d2, d3, d4, d5, d6, d7} @ 64-byte Spill
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #160]
+; CHECK-NEXT:    vldrw.u32 q6, [r0, #128]
+; CHECK-NEXT:    vstmia r2, {d0, d1, d2, d3, d4, d5, d6, d7} @ 64-byte Spill
+; CHECK-NEXT:    add r2, sp, #64
+; CHECK-NEXT:    vldrw.u32 q5, [r0, #64]
+; CHECK-NEXT:    vldmia sp, {d0, d1, d2, d3, d4, d5, d6, d7} @ 64-byte Reload
+; CHECK-NEXT:    vldrw.u32 q4, [r0]
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #112]
+; CHECK-NEXT:    vmov q7, q6
+; CHECK-NEXT:    vstmia sp, {d0, d1, d2, d3, d4, d5, d6, d7} @ 64-byte Spill
+; CHECK-NEXT:    vldmia r2, {d0, d1, d2, d3, d4, d5, d6, d7} @ 64-byte Reload
+; CHECK-NEXT:    add r2, sp, #64
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #96]
+; CHECK-NEXT:    vstmia r2, {d0, d1, d2, d3, d4, d5, d6, d7} @ 64-byte Spill
+; CHECK-NEXT:    add r2, sp, #64
+; CHECK-NEXT:    vldmia sp, {d0, d1, d2, d3, d4, d5, d6, d7} @ 64-byte Reload
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #48]
+; CHECK-NEXT:    vstmia sp, {d0, d1, d2, d3, d4, d5, d6, d7} @ 64-byte Spill
+; CHECK-NEXT:    vldmia r2, {d0, d1, d2, d3, d4, d5, d6, d7} @ 64-byte Reload
+; CHECK-NEXT:    add r2, sp, #64
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #32]
+; CHECK-NEXT:    vstmia r2, {d0, d1, d2, d3, d4, d5, d6, d7} @ 64-byte Spill
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #144]
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #80]
+; CHECK-NEXT:    add r2, sp, #64
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #16]
+; CHECK-NEXT:    add.w r0, r1, #64
+; CHECK-NEXT:    vmov q3, q2
+; CHECK-NEXT:    vst40.32 {q4, q5, q6, q7}, [r1]
+; CHECK-NEXT:    vst41.32 {q4, q5, q6, q7}, [r1]
+; CHECK-NEXT:    vst42.32 {q4, q5, q6, q7}, [r1]
+; CHECK-NEXT:    vst43.32 {q4, q5, q6, q7}, [r1]
+; CHECK-NEXT:    vst40.32 {q0, q1, q2, q3}, [r0]
+; CHECK-NEXT:    vst41.32 {q0, q1, q2, q3}, [r0]
+; CHECK-NEXT:    vst42.32 {q0, q1, q2, q3}, [r0]
+; CHECK-NEXT:    vst43.32 {q0, q1, q2, q3}, [r0]
+; CHECK-NEXT:    add.w r0, r1, #192
+; CHECK-NEXT:    vldmia sp, {d0, d1, d2, d3, d4, d5, d6, d7} @ 64-byte Reload
+; CHECK-NEXT:    adds r1, #128
+; CHECK-NEXT:    vmov q3, q2
+; CHECK-NEXT:    vldmia r2, {d8, d9, d10, d11, d12, d13, d14, d15} @ 64-byte Reload
+; CHECK-NEXT:    vmov q7, q6
+; CHECK-NEXT:    vst40.32 {q4, q5, q6, q7}, [r1]
+; CHECK-NEXT:    vst41.32 {q4, q5, q6, q7}, [r1]
+; CHECK-NEXT:    vst42.32 {q4, q5, q6, q7}, [r1]
+; CHECK-NEXT:    vst43.32 {q4, q5, q6, q7}, [r1]
+; CHECK-NEXT:    vst40.32 {q0, q1, q2, q3}, [r0]
+; CHECK-NEXT:    vst41.32 {q0, q1, q2, q3}, [r0]
+; CHECK-NEXT:    vst42.32 {q0, q1, q2, q3}, [r0]
+; CHECK-NEXT:    vst43.32 {q0, q1, q2, q3}, [r0]
+; CHECK-NEXT:    add sp, #152
+; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    pop {r4, r5}
+; CHECK-NEXT:    bx lr
+entry:
+  %s1 = getelementptr <16 x i32>, <16 x i32>* %src, i32 0
+  %l1 = load <16 x i32>, <16 x i32>* %s1, align 4
+  %s2 = getelementptr <16 x i32>, <16 x i32>* %src, i32 1
+  %l2 = load <16 x i32>, <16 x i32>* %s2, align 4
+  %s3 = getelementptr <16 x i32>, <16 x i32>* %src, i32 2
+  %l3 = load <16 x i32>, <16 x i32>* %s3, align 4
+  %s4 = getelementptr <16 x i32>, <16 x i32>* %src, i32 3
+  %l4 = load <16 x i32>, <16 x i32>* %s3, align 4
+  %t1 = shufflevector <16 x i32> %l1, <16 x i32> %l2, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+  %t2 = shufflevector <16 x i32> %l3, <16 x i32> %l4, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+  %s = shufflevector <32 x i32> %t1, <32 x i32> %t2, <64 x i32> <i32 0, i32 16, i32 32, i32 48, i32 1, i32 17, i32 33, i32 49, i32 2, i32 18, i32 34, i32 50, i32 3, i32 19, i32 35, i32 51, i32 4, i32 20, i32 36, i32 52, i32 5, i32 21, i32 37, i32 53, i32 6, i32 22, i32 38, i32 54, i32 7, i32 23, i32 39, i32 55, i32 8, i32 24, i32 40, i32 56, i32 9, i32 25, i32 41, i32 57, i32 10, i32 26, i32 42, i32 58, i32 11, i32 27, i32 43, i32 59, i32 12, i32 28, i32 44, i32 60, i32 13, i32 29, i32 45, i32 61, i32 14, i32 30, i32 46, i32 62, i32 15, i32 31, i32 47, i32 63>
+  store <64 x i32> %s, <64 x i32> *%dst
+  ret void
+}
+
+; i16
+
+define void @vst4_v2i16(<2 x i16> *%src, <8 x i16> *%dst) {
+; CHECK-LABEL: vst4_v2i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, lr}
+; CHECK-NEXT:    push {r4, lr}
+; CHECK-NEXT:    ldrh r4, [r0]
+; CHECK-NEXT:    ldrh.w lr, [r0, #4]
+; CHECK-NEXT:    ldrh r3, [r0, #8]
+; CHECK-NEXT:    vmov.32 q0[0], r4
+; CHECK-NEXT:    ldrh.w r12, [r0, #6]
+; CHECK-NEXT:    ldrh r2, [r0, #10]
+; CHECK-NEXT:    ldrh r0, [r0, #2]
+; CHECK-NEXT:    vmov.32 q0[2], r0
+; CHECK-NEXT:    vmov r4, s0
+; CHECK-NEXT:    vmov.16 q0[0], r4
+; CHECK-NEXT:    vmov.16 q0[1], lr
+; CHECK-NEXT:    vmov.16 q0[2], r3
+; CHECK-NEXT:    vmov.16 q0[3], r3
+; CHECK-NEXT:    vmov.16 q0[4], r0
+; CHECK-NEXT:    vmov.16 q0[5], r12
+; CHECK-NEXT:    vmov.16 q0[6], r2
+; CHECK-NEXT:    vmov.16 q0[7], r2
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    pop {r4, pc}
+entry:
+  %s1 = getelementptr <2 x i16>, <2 x i16>* %src, i32 0
+  %l1 = load <2 x i16>, <2 x i16>* %s1, align 4
+  %s2 = getelementptr <2 x i16>, <2 x i16>* %src, i32 1
+  %l2 = load <2 x i16>, <2 x i16>* %s2, align 4
+  %s3 = getelementptr <2 x i16>, <2 x i16>* %src, i32 2
+  %l3 = load <2 x i16>, <2 x i16>* %s3, align 4
+  %s4 = getelementptr <2 x i16>, <2 x i16>* %src, i32 3
+  %l4 = load <2 x i16>, <2 x i16>* %s3, align 4
+  %t1 = shufflevector <2 x i16> %l1, <2 x i16> %l2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %t2 = shufflevector <2 x i16> %l3, <2 x i16> %l4, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %s = shufflevector <4 x i16> %t1, <4 x i16> %t2, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 1, i32 3, i32 5, i32 7>
+  store <8 x i16> %s, <8 x i16> *%dst
+  ret void
+}
+
+define void @vst4_v4i16(<4 x i16> *%src, <16 x i16> *%dst) {
+; CHECK-LABEL: vst4_v4i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9}
+; CHECK-NEXT:    vpush {d8, d9}
+; CHECK-NEXT:    vldrh.u32 q1, [r0]
+; CHECK-NEXT:    vldrh.u32 q2, [r0, #8]
+; CHECK-NEXT:    vldrh.u32 q3, [r0, #16]
+; CHECK-NEXT:    vmov r2, s6
+; CHECK-NEXT:    vmov.16 q0[0], r2
+; CHECK-NEXT:    vmov r2, s10
+; CHECK-NEXT:    vmov.16 q0[1], r2
+; CHECK-NEXT:    vmov r0, s14
+; CHECK-NEXT:    vmov.16 q0[2], r0
+; CHECK-NEXT:    vmov.16 q0[3], r0
+; CHECK-NEXT:    vmov r0, s7
+; CHECK-NEXT:    vmov.16 q0[4], r0
+; CHECK-NEXT:    vmov r0, s11
+; CHECK-NEXT:    vmov.16 q0[5], r0
+; CHECK-NEXT:    vmov r0, s15
+; CHECK-NEXT:    vmov.16 q0[6], r0
+; CHECK-NEXT:    vmov.16 q0[7], r0
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmov.16 q4[0], r0
+; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    vmov.16 q4[1], r0
+; CHECK-NEXT:    vmov r0, s12
+; CHECK-NEXT:    vmov.16 q4[2], r0
+; CHECK-NEXT:    vstrw.32 q0, [r1, #16]
+; CHECK-NEXT:    vmov.16 q4[3], r0
+; CHECK-NEXT:    vmov r0, s5
+; CHECK-NEXT:    vmov.16 q4[4], r0
+; CHECK-NEXT:    vmov r0, s9
+; CHECK-NEXT:    vmov.16 q4[5], r0
+; CHECK-NEXT:    vmov r0, s13
+; CHECK-NEXT:    vmov.16 q4[6], r0
+; CHECK-NEXT:    vmov.16 q4[7], r0
+; CHECK-NEXT:    vstrw.32 q4, [r1]
+; CHECK-NEXT:    vpop {d8, d9}
+; CHECK-NEXT:    bx lr
+entry:
+  %s1 = getelementptr <4 x i16>, <4 x i16>* %src, i32 0
+  %l1 = load <4 x i16>, <4 x i16>* %s1, align 4
+  %s2 = getelementptr <4 x i16>, <4 x i16>* %src, i32 1
+  %l2 = load <4 x i16>, <4 x i16>* %s2, align 4
+  %s3 = getelementptr <4 x i16>, <4 x i16>* %src, i32 2
+  %l3 = load <4 x i16>, <4 x i16>* %s3, align 4
+  %s4 = getelementptr <4 x i16>, <4 x i16>* %src, i32 3
+  %l4 = load <4 x i16>, <4 x i16>* %s3, align 4
+  %t1 = shufflevector <4 x i16> %l1, <4 x i16> %l2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %t2 = shufflevector <4 x i16> %l3, <4 x i16> %l4, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %s = shufflevector <8 x i16> %t1, <8 x i16> %t2, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 1, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
+  store <16 x i16> %s, <16 x i16> *%dst
+  ret void
+}
+
+define void @vst4_v8i16(<8 x i16> *%src, <32 x i16> *%dst) {
+; CHECK-LABEL: vst4_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #32]
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #16]
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vmov q3, q2
+; CHECK-NEXT:    vst40.16 {q0, q1, q2, q3}, [r1]
+; CHECK-NEXT:    vst41.16 {q0, q1, q2, q3}, [r1]
+; CHECK-NEXT:    vst42.16 {q0, q1, q2, q3}, [r1]
+; CHECK-NEXT:    vst43.16 {q0, q1, q2, q3}, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %s1 = getelementptr <8 x i16>, <8 x i16>* %src, i32 0
+  %l1 = load <8 x i16>, <8 x i16>* %s1, align 4
+  %s2 = getelementptr <8 x i16>, <8 x i16>* %src, i32 1
+  %l2 = load <8 x i16>, <8 x i16>* %s2, align 4
+  %s3 = getelementptr <8 x i16>, <8 x i16>* %src, i32 2
+  %l3 = load <8 x i16>, <8 x i16>* %s3, align 4
+  %s4 = getelementptr <8 x i16>, <8 x i16>* %src, i32 3
+  %l4 = load <8 x i16>, <8 x i16>* %s3, align 4
+  %t1 = shufflevector <8 x i16> %l1, <8 x i16> %l2, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %t2 = shufflevector <8 x i16> %l3, <8 x i16> %l4, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %s = shufflevector <16 x i16> %t1, <16 x i16> %t2, <32 x i32> <i32 0, i32 8, i32 16, i32 24, i32 1, i32 9, i32 17, i32 25, i32 2, i32 10, i32 18, i32 26, i32 3, i32 11, i32 19, i32 27, i32 4, i32 12, i32 20, i32 28, i32 5, i32 13, i32 21, i32 29, i32 6, i32 14, i32 22, i32 30, i32 7, i32 15, i32 23, i32 31>
+  store <32 x i16> %s, <32 x i16> *%dst
+  ret void
+}
+
+define void @vst4_v16i16(<16 x i16> *%src, <64 x i16> *%dst) {
+; CHECK-LABEL: vst4_v16i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #80]
+; CHECK-NEXT:    vldrw.u32 q6, [r0, #64]
+; CHECK-NEXT:    vldrw.u32 q5, [r0, #32]
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #48]
+; CHECK-NEXT:    vldrw.u32 q4, [r0]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #16]
+; CHECK-NEXT:    vmov q7, q6
+; CHECK-NEXT:    add.w r0, r1, #64
+; CHECK-NEXT:    vmov q3, q2
+; CHECK-NEXT:    vst40.16 {q4, q5, q6, q7}, [r1]
+; CHECK-NEXT:    vst41.16 {q4, q5, q6, q7}, [r1]
+; CHECK-NEXT:    vst42.16 {q4, q5, q6, q7}, [r1]
+; CHECK-NEXT:    vst43.16 {q4, q5, q6, q7}, [r1]
+; CHECK-NEXT:    vst40.16 {q0, q1, q2, q3}, [r0]
+; CHECK-NEXT:    vst41.16 {q0, q1, q2, q3}, [r0]
+; CHECK-NEXT:    vst42.16 {q0, q1, q2, q3}, [r0]
+; CHECK-NEXT:    vst43.16 {q0, q1, q2, q3}, [r0]
+; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    bx lr
+entry:
+  %s1 = getelementptr <16 x i16>, <16 x i16>* %src, i32 0
+  %l1 = load <16 x i16>, <16 x i16>* %s1, align 4
+  %s2 = getelementptr <16 x i16>, <16 x i16>* %src, i32 1
+  %l2 = load <16 x i16>, <16 x i16>* %s2, align 4
+  %s3 = getelementptr <16 x i16>, <16 x i16>* %src, i32 2
+  %l3 = load <16 x i16>, <16 x i16>* %s3, align 4
+  %s4 = getelementptr <16 x i16>, <16 x i16>* %src, i32 3
+  %l4 = load <16 x i16>, <16 x i16>* %s3, align 4
+  %t1 = shufflevector <16 x i16> %l1, <16 x i16> %l2, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+  %t2 = shufflevector <16 x i16> %l3, <16 x i16> %l4, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+  %s = shufflevector <32 x i16> %t1, <32 x i16> %t2, <64 x i32> <i32 0, i32 16, i32 32, i32 48, i32 1, i32 17, i32 33, i32 49, i32 2, i32 18, i32 34, i32 50, i32 3, i32 19, i32 35, i32 51, i32 4, i32 20, i32 36, i32 52, i32 5, i32 21, i32 37, i32 53, i32 6, i32 22, i32 38, i32 54, i32 7, i32 23, i32 39, i32 55, i32 8, i32 24, i32 40, i32 56, i32 9, i32 25, i32 41, i32 57, i32 10, i32 26, i32 42, i32 58, i32 11, i32 27, i32 43, i32 59, i32 12, i32 28, i32 44, i32 60, i32 13, i32 29, i32 45, i32 61, i32 14, i32 30, i32 46, i32 62, i32 15, i32 31, i32 47, i32 63>
+  store <64 x i16> %s, <64 x i16> *%dst
+  ret void
+}
+
+; i8
+
+define void @vst4_v2i8(<2 x i8> *%src, <8 x i8> *%dst) {
+; CHECK-LABEL: vst4_v2i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, lr}
+; CHECK-NEXT:    push {r4, lr}
+; CHECK-NEXT:    ldrb r2, [r0]
+; CHECK-NEXT:    ldrb r3, [r0, #1]
+; CHECK-NEXT:    vmov.32 q0[0], r2
+; CHECK-NEXT:    ldrb.w r12, [r0, #2]
+; CHECK-NEXT:    vmov.32 q0[2], r3
+; CHECK-NEXT:    ldrb.w lr, [r0, #3]
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    ldrb r4, [r0, #5]
+; CHECK-NEXT:    vmov.16 q0[0], r2
+; CHECK-NEXT:    ldrb r0, [r0, #4]
+; CHECK-NEXT:    vmov.16 q0[1], r12
+; CHECK-NEXT:    vmov.16 q0[2], r0
+; CHECK-NEXT:    vmov.16 q0[3], r0
+; CHECK-NEXT:    vmov.16 q0[4], r3
+; CHECK-NEXT:    vmov.16 q0[5], lr
+; CHECK-NEXT:    vmov.16 q0[6], r4
+; CHECK-NEXT:    vmov.16 q0[7], r4
+; CHECK-NEXT:    vstrb.16 q0, [r1]
+; CHECK-NEXT:    pop {r4, pc}
+entry:
+  %s1 = getelementptr <2 x i8>, <2 x i8>* %src, i32 0
+  %l1 = load <2 x i8>, <2 x i8>* %s1, align 4
+  %s2 = getelementptr <2 x i8>, <2 x i8>* %src, i32 1
+  %l2 = load <2 x i8>, <2 x i8>* %s2, align 4
+  %s3 = getelementptr <2 x i8>, <2 x i8>* %src, i32 2
+  %l3 = load <2 x i8>, <2 x i8>* %s3, align 4
+  %s4 = getelementptr <2 x i8>, <2 x i8>* %src, i32 3
+  %l4 = load <2 x i8>, <2 x i8>* %s3, align 4
+  %t1 = shufflevector <2 x i8> %l1, <2 x i8> %l2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %t2 = shufflevector <2 x i8> %l3, <2 x i8> %l4, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %s = shufflevector <4 x i8> %t1, <4 x i8> %t2, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 1, i32 3, i32 5, i32 7>
+  store <8 x i8> %s, <8 x i8> *%dst
+  ret void
+}
+
+define void @vst4_v4i8(<4 x i8> *%src, <16 x i8> *%dst) {
+; CHECK-LABEL: vst4_v4i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u32 q1, [r0]
+; CHECK-NEXT:    vldrb.u32 q2, [r0, #4]
+; CHECK-NEXT:    vldrb.u32 q3, [r0, #8]
+; CHECK-NEXT:    vmov r2, s4
+; CHECK-NEXT:    vmov.8 q0[0], r2
+; CHECK-NEXT:    vmov r2, s8
+; CHECK-NEXT:    vmov.8 q0[1], r2
+; CHECK-NEXT:    vmov r0, s12
+; CHECK-NEXT:    vmov.8 q0[2], r0
+; CHECK-NEXT:    vmov.8 q0[3], r0
+; CHECK-NEXT:    vmov r0, s5
+; CHECK-NEXT:    vmov.8 q0[4], r0
+; CHECK-NEXT:    vmov r0, s9
+; CHECK-NEXT:    vmov.8 q0[5], r0
+; CHECK-NEXT:    vmov r0, s13
+; CHECK-NEXT:    vmov.8 q0[6], r0
+; CHECK-NEXT:    vmov.8 q0[7], r0
+; CHECK-NEXT:    vmov r0, s6
+; CHECK-NEXT:    vmov.8 q0[8], r0
+; CHECK-NEXT:    vmov r0, s10
+; CHECK-NEXT:    vmov.8 q0[9], r0
+; CHECK-NEXT:    vmov r0, s14
+; CHECK-NEXT:    vmov.8 q0[10], r0
+; CHECK-NEXT:    vmov.8 q0[11], r0
+; CHECK-NEXT:    vmov r0, s7
+; CHECK-NEXT:    vmov.8 q0[12], r0
+; CHECK-NEXT:    vmov r0, s11
+; CHECK-NEXT:    vmov.8 q0[13], r0
+; CHECK-NEXT:    vmov r0, s15
+; CHECK-NEXT:    vmov.8 q0[14], r0
+; CHECK-NEXT:    vmov.8 q0[15], r0
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %s1 = getelementptr <4 x i8>, <4 x i8>* %src, i32 0
+  %l1 = load <4 x i8>, <4 x i8>* %s1, align 4
+  %s2 = getelementptr <4 x i8>, <4 x i8>* %src, i32 1
+  %l2 = load <4 x i8>, <4 x i8>* %s2, align 4
+  %s3 = getelementptr <4 x i8>, <4 x i8>* %src, i32 2
+  %l3 = load <4 x i8>, <4 x i8>* %s3, align 4
+  %s4 = getelementptr <4 x i8>, <4 x i8>* %src, i32 3
+  %l4 = load <4 x i8>, <4 x i8>* %s3, align 4
+  %t1 = shufflevector <4 x i8> %l1, <4 x i8> %l2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %t2 = shufflevector <4 x i8> %l3, <4 x i8> %l4, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %s = shufflevector <8 x i8> %t1, <8 x i8> %t2, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 1, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
+  store <16 x i8> %s, <16 x i8> *%dst
+  ret void
+}
+
+define void @vst4_v8i8(<8 x i8> *%src, <32 x i8> *%dst) {
+; CHECK-LABEL: vst4_v8i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9}
+; CHECK-NEXT:    vpush {d8, d9}
+; CHECK-NEXT:    vldrb.u16 q1, [r0]
+; CHECK-NEXT:    vldrb.u16 q2, [r0, #8]
+; CHECK-NEXT:    vldrb.u16 q3, [r0, #16]
+; CHECK-NEXT:    vmov.u16 r2, q1[4]
+; CHECK-NEXT:    vmov.8 q0[0], r2
+; CHECK-NEXT:    vmov.u16 r2, q2[4]
+; CHECK-NEXT:    vmov.8 q0[1], r2
+; CHECK-NEXT:    vmov.u16 r0, q3[4]
+; CHECK-NEXT:    vmov.8 q0[2], r0
+; CHECK-NEXT:    vmov.8 q0[3], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[5]
+; CHECK-NEXT:    vmov.8 q0[4], r0
+; CHECK-NEXT:    vmov.u16 r0, q2[5]
+; CHECK-NEXT:    vmov.8 q0[5], r0
+; CHECK-NEXT:    vmov.u16 r0, q3[5]
+; CHECK-NEXT:    vmov.8 q0[6], r0
+; CHECK-NEXT:    vmov.8 q0[7], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[6]
+; CHECK-NEXT:    vmov.8 q0[8], r0
+; CHECK-NEXT:    vmov.u16 r0, q2[6]
+; CHECK-NEXT:    vmov.8 q0[9], r0
+; CHECK-NEXT:    vmov.u16 r0, q3[6]
+; CHECK-NEXT:    vmov.8 q0[10], r0
+; CHECK-NEXT:    vmov.8 q0[11], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[7]
+; CHECK-NEXT:    vmov.8 q0[12], r0
+; CHECK-NEXT:    vmov.u16 r0, q2[7]
+; CHECK-NEXT:    vmov.8 q0[13], r0
+; CHECK-NEXT:    vmov.u16 r0, q3[7]
+; CHECK-NEXT:    vmov.8 q0[14], r0
+; CHECK-NEXT:    vmov.8 q0[15], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[0]
+; CHECK-NEXT:    vmov.8 q4[0], r0
+; CHECK-NEXT:    vmov.u16 r0, q2[0]
+; CHECK-NEXT:    vmov.8 q4[1], r0
+; CHECK-NEXT:    vmov.u16 r0, q3[0]
+; CHECK-NEXT:    vmov.8 q4[2], r0
+; CHECK-NEXT:    vstrw.32 q0, [r1, #16]
+; CHECK-NEXT:    vmov.8 q4[3], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[1]
+; CHECK-NEXT:    vmov.8 q4[4], r0
+; CHECK-NEXT:    vmov.u16 r0, q2[1]
+; CHECK-NEXT:    vmov.8 q4[5], r0
+; CHECK-NEXT:    vmov.u16 r0, q3[1]
+; CHECK-NEXT:    vmov.8 q4[6], r0
+; CHECK-NEXT:    vmov.8 q4[7], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[2]
+; CHECK-NEXT:    vmov.8 q4[8], r0
+; CHECK-NEXT:    vmov.u16 r0, q2[2]
+; CHECK-NEXT:    vmov.8 q4[9], r0
+; CHECK-NEXT:    vmov.u16 r0, q3[2]
+; CHECK-NEXT:    vmov.8 q4[10], r0
+; CHECK-NEXT:    vmov.8 q4[11], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[3]
+; CHECK-NEXT:    vmov.8 q4[12], r0
+; CHECK-NEXT:    vmov.u16 r0, q2[3]
+; CHECK-NEXT:    vmov.8 q4[13], r0
+; CHECK-NEXT:    vmov.u16 r0, q3[3]
+; CHECK-NEXT:    vmov.8 q4[14], r0
+; CHECK-NEXT:    vmov.8 q4[15], r0
+; CHECK-NEXT:    vstrw.32 q4, [r1]
+; CHECK-NEXT:    vpop {d8, d9}
+; CHECK-NEXT:    bx lr
+entry:
+  %s1 = getelementptr <8 x i8>, <8 x i8>* %src, i32 0
+  %l1 = load <8 x i8>, <8 x i8>* %s1, align 4
+  %s2 = getelementptr <8 x i8>, <8 x i8>* %src, i32 1
+  %l2 = load <8 x i8>, <8 x i8>* %s2, align 4
+  %s3 = getelementptr <8 x i8>, <8 x i8>* %src, i32 2
+  %l3 = load <8 x i8>, <8 x i8>* %s3, align 4
+  %s4 = getelementptr <8 x i8>, <8 x i8>* %src, i32 3
+  %l4 = load <8 x i8>, <8 x i8>* %s3, align 4
+  %t1 = shufflevector <8 x i8> %l1, <8 x i8> %l2, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %t2 = shufflevector <8 x i8> %l3, <8 x i8> %l4, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %s = shufflevector <16 x i8> %t1, <16 x i8> %t2, <32 x i32> <i32 0, i32 8, i32 16, i32 24, i32 1, i32 9, i32 17, i32 25, i32 2, i32 10, i32 18, i32 26, i32 3, i32 11, i32 19, i32 27, i32 4, i32 12, i32 20, i32 28, i32 5, i32 13, i32 21, i32 29, i32 6, i32 14, i32 22, i32 30, i32 7, i32 15, i32 23, i32 31>
+  store <32 x i8> %s, <32 x i8> *%dst
+  ret void
+}
+
+define void @vst4_v16i8(<16 x i8> *%src, <64 x i8> *%dst) {
+; CHECK-LABEL: vst4_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #32]
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #16]
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vmov q3, q2
+; CHECK-NEXT:    vst40.8 {q0, q1, q2, q3}, [r1]
+; CHECK-NEXT:    vst41.8 {q0, q1, q2, q3}, [r1]
+; CHECK-NEXT:    vst42.8 {q0, q1, q2, q3}, [r1]
+; CHECK-NEXT:    vst43.8 {q0, q1, q2, q3}, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %s1 = getelementptr <16 x i8>, <16 x i8>* %src, i32 0
+  %l1 = load <16 x i8>, <16 x i8>* %s1, align 4
+  %s2 = getelementptr <16 x i8>, <16 x i8>* %src, i32 1
+  %l2 = load <16 x i8>, <16 x i8>* %s2, align 4
+  %s3 = getelementptr <16 x i8>, <16 x i8>* %src, i32 2
+  %l3 = load <16 x i8>, <16 x i8>* %s3, align 4
+  %s4 = getelementptr <16 x i8>, <16 x i8>* %src, i32 3
+  %l4 = load <16 x i8>, <16 x i8>* %s3, align 4
+  %t1 = shufflevector <16 x i8> %l1, <16 x i8> %l2, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+  %t2 = shufflevector <16 x i8> %l3, <16 x i8> %l4, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+  %s = shufflevector <32 x i8> %t1, <32 x i8> %t2, <64 x i32> <i32 0, i32 16, i32 32, i32 48, i32 1, i32 17, i32 33, i32 49, i32 2, i32 18, i32 34, i32 50, i32 3, i32 19, i32 35, i32 51, i32 4, i32 20, i32 36, i32 52, i32 5, i32 21, i32 37, i32 53, i32 6, i32 22, i32 38, i32 54, i32 7, i32 23, i32 39, i32 55, i32 8, i32 24, i32 40, i32 56, i32 9, i32 25, i32 41, i32 57, i32 10, i32 26, i32 42, i32 58, i32 11, i32 27, i32 43, i32 59, i32 12, i32 28, i32 44, i32 60, i32 13, i32 29, i32 45, i32 61, i32 14, i32 30, i32 46, i32 62, i32 15, i32 31, i32 47, i32 63>
+  store <64 x i8> %s, <64 x i8> *%dst
+  ret void
+}
+
+; i64
+
+define void @vst4_v2i64(<2 x i64> *%src, <8 x i64> *%dst) {
+; CHECK-LABEL: vst4_v2i64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q3, [r0]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #16]
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #32]
+; CHECK-NEXT:    vmov.f64 d4, d6
+; CHECK-NEXT:    vmov.f32 s9, s13
+; CHECK-NEXT:    vmov.f32 s10, s0
+; CHECK-NEXT:    vmov.f32 s11, s1
+; CHECK-NEXT:    vmov.f32 s0, s14
+; CHECK-NEXT:    vstrw.32 q2, [r1]
+; CHECK-NEXT:    vmov.f32 s1, s15
+; CHECK-NEXT:    vmov.f64 d6, d2
+; CHECK-NEXT:    vstrw.32 q0, [r1, #32]
+; CHECK-NEXT:    vmov.f32 s13, s5
+; CHECK-NEXT:    vmov.f32 s14, s4
+; CHECK-NEXT:    vmov.f32 s15, s5
+; CHECK-NEXT:    vmov.f32 s4, s6
+; CHECK-NEXT:    vstrw.32 q3, [r1, #16]
+; CHECK-NEXT:    vmov.f32 s5, s7
+; CHECK-NEXT:    vstrw.32 q1, [r1, #48]
+; CHECK-NEXT:    bx lr
+entry:
+  %s1 = getelementptr <2 x i64>, <2 x i64>* %src, i32 0
+  %l1 = load <2 x i64>, <2 x i64>* %s1, align 4
+  %s2 = getelementptr <2 x i64>, <2 x i64>* %src, i32 1
+  %l2 = load <2 x i64>, <2 x i64>* %s2, align 4
+  %s3 = getelementptr <2 x i64>, <2 x i64>* %src, i32 2
+  %l3 = load <2 x i64>, <2 x i64>* %s3, align 4
+  %s4 = getelementptr <2 x i64>, <2 x i64>* %src, i32 3
+  %l4 = load <2 x i64>, <2 x i64>* %s3, align 4
+  %t1 = shufflevector <2 x i64> %l1, <2 x i64> %l2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %t2 = shufflevector <2 x i64> %l3, <2 x i64> %l4, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %s = shufflevector <4 x i64> %t1, <4 x i64> %t2, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 1, i32 3, i32 5, i32 7>
+  store <8 x i64> %s, <8 x i64> *%dst
+  ret void
+}
+
+define void @vst4_v4i64(<4 x i64> *%src, <16 x i64> *%dst) {
+; CHECK-LABEL: vst4_v4i64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    vldrw.u32 q5, [r0]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #32]
+; CHECK-NEXT:    vldrw.u32 q6, [r0, #16]
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #48]
+; CHECK-NEXT:    vmov.f64 d6, d10
+; CHECK-NEXT:    vldrw.u32 q4, [r0, #80]
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #64]
+; CHECK-NEXT:    vmov.f32 s13, s21
+; CHECK-NEXT:    vmov.f32 s14, s0
+; CHECK-NEXT:    vmov.f32 s15, s1
+; CHECK-NEXT:    vmov.f32 s0, s22
+; CHECK-NEXT:    vstrw.32 q3, [r1]
+; CHECK-NEXT:    vmov.f32 s1, s23
+; CHECK-NEXT:    vmov.f64 d10, d12
+; CHECK-NEXT:    vstrw.32 q0, [r1, #32]
+; CHECK-NEXT:    vmov.f32 s21, s25
+; CHECK-NEXT:    vmov.f32 s22, s8
+; CHECK-NEXT:    vmov.f32 s23, s9
+; CHECK-NEXT:    vmov.f32 s8, s26
+; CHECK-NEXT:    vstrw.32 q5, [r1, #64]
+; CHECK-NEXT:    vmov.f32 s9, s27
+; CHECK-NEXT:    vmov.f64 d12, d2
+; CHECK-NEXT:    vstrw.32 q2, [r1, #96]
+; CHECK-NEXT:    vmov.f64 d14, d8
+; CHECK-NEXT:    vmov.f32 s25, s5
+; CHECK-NEXT:    vmov.f32 s29, s17
+; CHECK-NEXT:    vmov.f32 s26, s4
+; CHECK-NEXT:    vmov.f32 s30, s16
+; CHECK-NEXT:    vmov.f32 s27, s5
+; CHECK-NEXT:    vmov.f32 s4, s6
+; CHECK-NEXT:    vstrw.32 q6, [r1, #16]
+; CHECK-NEXT:    vmov.f32 s31, s17
+; CHECK-NEXT:    vmov.f32 s16, s18
+; CHECK-NEXT:    vstrw.32 q7, [r1, #80]
+; CHECK-NEXT:    vmov.f32 s5, s7
+; CHECK-NEXT:    vmov.f32 s17, s19
+; CHECK-NEXT:    vstrw.32 q1, [r1, #48]
+; CHECK-NEXT:    vstrw.32 q4, [r1, #112]
+; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    bx lr
+entry:
+  %s1 = getelementptr <4 x i64>, <4 x i64>* %src, i32 0
+  %l1 = load <4 x i64>, <4 x i64>* %s1, align 4
+  %s2 = getelementptr <4 x i64>, <4 x i64>* %src, i32 1
+  %l2 = load <4 x i64>, <4 x i64>* %s2, align 4
+  %s3 = getelementptr <4 x i64>, <4 x i64>* %src, i32 2
+  %l3 = load <4 x i64>, <4 x i64>* %s3, align 4
+  %s4 = getelementptr <4 x i64>, <4 x i64>* %src, i32 3
+  %l4 = load <4 x i64>, <4 x i64>* %s3, align 4
+  %t1 = shufflevector <4 x i64> %l1, <4 x i64> %l2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %t2 = shufflevector <4 x i64> %l3, <4 x i64> %l4, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %s = shufflevector <8 x i64> %t1, <8 x i64> %t2, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 1, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
+  store <16 x i64> %s, <16 x i64> *%dst
+  ret void
+}
+
+; f32
+
+define void @vst4_v2f32(<2 x float> *%src, <8 x float> *%dst) {
+; CHECK-LABEL: vst4_v2f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldr s0, [r0]
+; CHECK-NEXT:    vldr s4, [r0, #4]
+; CHECK-NEXT:    vldr s1, [r0, #8]
+; CHECK-NEXT:    vldr s5, [r0, #12]
+; CHECK-NEXT:    vldr s2, [r0, #16]
+; CHECK-NEXT:    vldr s6, [r0, #20]
+; CHECK-NEXT:    vmov.f32 s3, s2
+; CHECK-NEXT:    vmov.f32 s7, s6
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vstrw.32 q1, [r1, #16]
+; CHECK-NEXT:    bx lr
+entry:
+  %s1 = getelementptr <2 x float>, <2 x float>* %src, i32 0
+  %l1 = load <2 x float>, <2 x float>* %s1, align 4
+  %s2 = getelementptr <2 x float>, <2 x float>* %src, i32 1
+  %l2 = load <2 x float>, <2 x float>* %s2, align 4
+  %s3 = getelementptr <2 x float>, <2 x float>* %src, i32 2
+  %l3 = load <2 x float>, <2 x float>* %s3, align 4
+  %s4 = getelementptr <2 x float>, <2 x float>* %src, i32 3
+  %l4 = load <2 x float>, <2 x float>* %s3, align 4
+  %t1 = shufflevector <2 x float> %l1, <2 x float> %l2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %t2 = shufflevector <2 x float> %l3, <2 x float> %l4, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %s = shufflevector <4 x float> %t1, <4 x float> %t2, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 1, i32 3, i32 5, i32 7>
+  store <8 x float> %s, <8 x float> *%dst
+  ret void
+}
+
+define void @vst4_v4f32(<4 x float> *%src, <16 x float> *%dst) {
+; CHECK-LABEL: vst4_v4f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #32]
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #16]
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vmov q3, q2
+; CHECK-NEXT:    vst40.32 {q0, q1, q2, q3}, [r1]
+; CHECK-NEXT:    vst41.32 {q0, q1, q2, q3}, [r1]
+; CHECK-NEXT:    vst42.32 {q0, q1, q2, q3}, [r1]
+; CHECK-NEXT:    vst43.32 {q0, q1, q2, q3}, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %s1 = getelementptr <4 x float>, <4 x float>* %src, i32 0
+  %l1 = load <4 x float>, <4 x float>* %s1, align 4
+  %s2 = getelementptr <4 x float>, <4 x float>* %src, i32 1
+  %l2 = load <4 x float>, <4 x float>* %s2, align 4
+  %s3 = getelementptr <4 x float>, <4 x float>* %src, i32 2
+  %l3 = load <4 x float>, <4 x float>* %s3, align 4
+  %s4 = getelementptr <4 x float>, <4 x float>* %src, i32 3
+  %l4 = load <4 x float>, <4 x float>* %s3, align 4
+  %t1 = shufflevector <4 x float> %l1, <4 x float> %l2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %t2 = shufflevector <4 x float> %l3, <4 x float> %l4, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %s = shufflevector <8 x float> %t1, <8 x float> %t2, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 1, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
+  store <16 x float> %s, <16 x float> *%dst
+  ret void
+}
+
+define void @vst4_v8f32(<8 x float> *%src, <32 x float> *%dst) {
+; CHECK-LABEL: vst4_v8f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #80]
+; CHECK-NEXT:    vldrw.u32 q6, [r0, #64]
+; CHECK-NEXT:    vldrw.u32 q5, [r0, #32]
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #48]
+; CHECK-NEXT:    vldrw.u32 q4, [r0]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #16]
+; CHECK-NEXT:    vmov q7, q6
+; CHECK-NEXT:    add.w r0, r1, #64
+; CHECK-NEXT:    vmov q3, q2
+; CHECK-NEXT:    vst40.32 {q4, q5, q6, q7}, [r1]
+; CHECK-NEXT:    vst41.32 {q4, q5, q6, q7}, [r1]
+; CHECK-NEXT:    vst42.32 {q4, q5, q6, q7}, [r1]
+; CHECK-NEXT:    vst43.32 {q4, q5, q6, q7}, [r1]
+; CHECK-NEXT:    vst40.32 {q0, q1, q2, q3}, [r0]
+; CHECK-NEXT:    vst41.32 {q0, q1, q2, q3}, [r0]
+; CHECK-NEXT:    vst42.32 {q0, q1, q2, q3}, [r0]
+; CHECK-NEXT:    vst43.32 {q0, q1, q2, q3}, [r0]
+; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    bx lr
+entry:
+  %s1 = getelementptr <8 x float>, <8 x float>* %src, i32 0
+  %l1 = load <8 x float>, <8 x float>* %s1, align 4
+  %s2 = getelementptr <8 x float>, <8 x float>* %src, i32 1
+  %l2 = load <8 x float>, <8 x float>* %s2, align 4
+  %s3 = getelementptr <8 x float>, <8 x float>* %src, i32 2
+  %l3 = load <8 x float>, <8 x float>* %s3, align 4
+  %s4 = getelementptr <8 x float>, <8 x float>* %src, i32 3
+  %l4 = load <8 x float>, <8 x float>* %s3, align 4
+  %t1 = shufflevector <8 x float> %l1, <8 x float> %l2, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %t2 = shufflevector <8 x float> %l3, <8 x float> %l4, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %s = shufflevector <16 x float> %t1, <16 x float> %t2, <32 x i32> <i32 0, i32 8, i32 16, i32 24, i32 1, i32 9, i32 17, i32 25, i32 2, i32 10, i32 18, i32 26, i32 3, i32 11, i32 19, i32 27, i32 4, i32 12, i32 20, i32 28, i32 5, i32 13, i32 21, i32 29, i32 6, i32 14, i32 22, i32 30, i32 7, i32 15, i32 23, i32 31>
+  store <32 x float> %s, <32 x float> *%dst
+  ret void
+}
+
+define void @vst4_v16f32(<16 x float> *%src, <64 x float> *%dst) {
+; CHECK-LABEL: vst4_v16f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, r5}
+; CHECK-NEXT:    push {r4, r5}
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    .pad #152
+; CHECK-NEXT:    sub sp, #152
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #176]
+; CHECK-NEXT:    add r2, sp, #64
+; CHECK-NEXT:    vstmia sp, {d0, d1, d2, d3, d4, d5, d6, d7} @ 64-byte Spill
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #160]
+; CHECK-NEXT:    vldrw.u32 q6, [r0, #128]
+; CHECK-NEXT:    vstmia r2, {d0, d1, d2, d3, d4, d5, d6, d7} @ 64-byte Spill
+; CHECK-NEXT:    add r2, sp, #64
+; CHECK-NEXT:    vldrw.u32 q5, [r0, #64]
+; CHECK-NEXT:    vldmia sp, {d0, d1, d2, d3, d4, d5, d6, d7} @ 64-byte Reload
+; CHECK-NEXT:    vldrw.u32 q4, [r0]
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #112]
+; CHECK-NEXT:    vmov q7, q6
+; CHECK-NEXT:    vstmia sp, {d0, d1, d2, d3, d4, d5, d6, d7} @ 64-byte Spill
+; CHECK-NEXT:    vldmia r2, {d0, d1, d2, d3, d4, d5, d6, d7} @ 64-byte Reload
+; CHECK-NEXT:    add r2, sp, #64
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #96]
+; CHECK-NEXT:    vstmia r2, {d0, d1, d2, d3, d4, d5, d6, d7} @ 64-byte Spill
+; CHECK-NEXT:    add r2, sp, #64
+; CHECK-NEXT:    vldmia sp, {d0, d1, d2, d3, d4, d5, d6, d7} @ 64-byte Reload
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #48]
+; CHECK-NEXT:    vstmia sp, {d0, d1, d2, d3, d4, d5, d6, d7} @ 64-byte Spill
+; CHECK-NEXT:    vldmia r2, {d0, d1, d2, d3, d4, d5, d6, d7} @ 64-byte Reload
+; CHECK-NEXT:    add r2, sp, #64
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #32]
+; CHECK-NEXT:    vstmia r2, {d0, d1, d2, d3, d4, d5, d6, d7} @ 64-byte Spill
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #144]
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #80]
+; CHECK-NEXT:    add r2, sp, #64
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #16]
+; CHECK-NEXT:    add.w r0, r1, #64
+; CHECK-NEXT:    vmov q3, q2
+; CHECK-NEXT:    vst40.32 {q4, q5, q6, q7}, [r1]
+; CHECK-NEXT:    vst41.32 {q4, q5, q6, q7}, [r1]
+; CHECK-NEXT:    vst42.32 {q4, q5, q6, q7}, [r1]
+; CHECK-NEXT:    vst43.32 {q4, q5, q6, q7}, [r1]
+; CHECK-NEXT:    vst40.32 {q0, q1, q2, q3}, [r0]
+; CHECK-NEXT:    vst41.32 {q0, q1, q2, q3}, [r0]
+; CHECK-NEXT:    vst42.32 {q0, q1, q2, q3}, [r0]
+; CHECK-NEXT:    vst43.32 {q0, q1, q2, q3}, [r0]
+; CHECK-NEXT:    add.w r0, r1, #192
+; CHECK-NEXT:    vldmia sp, {d0, d1, d2, d3, d4, d5, d6, d7} @ 64-byte Reload
+; CHECK-NEXT:    adds r1, #128
+; CHECK-NEXT:    vmov q3, q2
+; CHECK-NEXT:    vldmia r2, {d8, d9, d10, d11, d12, d13, d14, d15} @ 64-byte Reload
+; CHECK-NEXT:    vmov q7, q6
+; CHECK-NEXT:    vst40.32 {q4, q5, q6, q7}, [r1]
+; CHECK-NEXT:    vst41.32 {q4, q5, q6, q7}, [r1]
+; CHECK-NEXT:    vst42.32 {q4, q5, q6, q7}, [r1]
+; CHECK-NEXT:    vst43.32 {q4, q5, q6, q7}, [r1]
+; CHECK-NEXT:    vst40.32 {q0, q1, q2, q3}, [r0]
+; CHECK-NEXT:    vst41.32 {q0, q1, q2, q3}, [r0]
+; CHECK-NEXT:    vst42.32 {q0, q1, q2, q3}, [r0]
+; CHECK-NEXT:    vst43.32 {q0, q1, q2, q3}, [r0]
+; CHECK-NEXT:    add sp, #152
+; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    pop {r4, r5}
+; CHECK-NEXT:    bx lr
+entry:
+  %s1 = getelementptr <16 x float>, <16 x float>* %src, i32 0
+  %l1 = load <16 x float>, <16 x float>* %s1, align 4
+  %s2 = getelementptr <16 x float>, <16 x float>* %src, i32 1
+  %l2 = load <16 x float>, <16 x float>* %s2, align 4
+  %s3 = getelementptr <16 x float>, <16 x float>* %src, i32 2
+  %l3 = load <16 x float>, <16 x float>* %s3, align 4
+  %s4 = getelementptr <16 x float>, <16 x float>* %src, i32 3
+  %l4 = load <16 x float>, <16 x float>* %s3, align 4
+  %t1 = shufflevector <16 x float> %l1, <16 x float> %l2, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+  %t2 = shufflevector <16 x float> %l3, <16 x float> %l4, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+  %s = shufflevector <32 x float> %t1, <32 x float> %t2, <64 x i32> <i32 0, i32 16, i32 32, i32 48, i32 1, i32 17, i32 33, i32 49, i32 2, i32 18, i32 34, i32 50, i32 3, i32 19, i32 35, i32 51, i32 4, i32 20, i32 36, i32 52, i32 5, i32 21, i32 37, i32 53, i32 6, i32 22, i32 38, i32 54, i32 7, i32 23, i32 39, i32 55, i32 8, i32 24, i32 40, i32 56, i32 9, i32 25, i32 41, i32 57, i32 10, i32 26, i32 42, i32 58, i32 11, i32 27, i32 43, i32 59, i32 12, i32 28, i32 44, i32 60, i32 13, i32 29, i32 45, i32 61, i32 14, i32 30, i32 46, i32 62, i32 15, i32 31, i32 47, i32 63>
+  store <64 x float> %s, <64 x float> *%dst
+  ret void
+}
+
+; f16
+
+define void @vst4_v2f16(<2 x half> *%src, <8 x half> *%dst) {
+; CHECK-LABEL: vst4_v2f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldmia r0, {s4, s5}
+; CHECK-NEXT:    vmov r2, s5
+; CHECK-NEXT:    ldr r0, [r0, #8]
+; CHECK-NEXT:    vmov r3, s4
+; CHECK-NEXT:    vmovx.f16 s12, s4
+; CHECK-NEXT:    vmov.16 q0[0], r3
+; CHECK-NEXT:    vdup.32 q2, r0
+; CHECK-NEXT:    vmov.16 q0[1], r2
+; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    vmov.16 q0[2], r0
+; CHECK-NEXT:    vmov r0, s9
+; CHECK-NEXT:    vmov.16 q0[3], r0
+; CHECK-NEXT:    vmov r0, s12
+; CHECK-NEXT:    vmovx.f16 s4, s5
+; CHECK-NEXT:    vmov.16 q0[4], r0
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmovx.f16 s4, s8
+; CHECK-NEXT:    vmov.16 q0[5], r0
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmovx.f16 s4, s9
+; CHECK-NEXT:    vmov.16 q0[6], r0
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmov.16 q0[7], r0
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %s1 = getelementptr <2 x half>, <2 x half>* %src, i32 0
+  %l1 = load <2 x half>, <2 x half>* %s1, align 4
+  %s2 = getelementptr <2 x half>, <2 x half>* %src, i32 1
+  %l2 = load <2 x half>, <2 x half>* %s2, align 4
+  %s3 = getelementptr <2 x half>, <2 x half>* %src, i32 2
+  %l3 = load <2 x half>, <2 x half>* %s3, align 4
+  %s4 = getelementptr <2 x half>, <2 x half>* %src, i32 3
+  %l4 = load <2 x half>, <2 x half>* %s3, align 4
+  %t1 = shufflevector <2 x half> %l1, <2 x half> %l2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %t2 = shufflevector <2 x half> %l3, <2 x half> %l4, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %s = shufflevector <4 x half> %t1, <4 x half> %t2, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 1, i32 3, i32 5, i32 7>
+  store <8 x half> %s, <8 x half> *%dst
+  ret void
+}
+
+define void @vst4_v4f16(<4 x half> *%src, <16 x half> *%dst) {
+; CHECK-LABEL: vst4_v4f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    ldm.w r0, {r2, r3, r12, lr}
+; CHECK-NEXT:    vmov.32 q0[0], r2
+; CHECK-NEXT:    vmov.32 q0[1], r3
+; CHECK-NEXT:    vmov.32 q0[2], r12
+; CHECK-NEXT:    vmov.32 q0[3], lr
+; CHECK-NEXT:    vmov r3, s1
+; CHECK-NEXT:    vmovx.f16 s12, s1
+; CHECK-NEXT:    vmov r2, s3
+; CHECK-NEXT:    vmov.16 q2[0], r3
+; CHECK-NEXT:    vmov.16 q2[1], r2
+; CHECK-NEXT:    ldrd r2, r0, [r0, #16]
+; CHECK-NEXT:    vmov.32 q1[0], r2
+; CHECK-NEXT:    vmov.32 q1[1], r0
+; CHECK-NEXT:    vmov.32 q1[2], r2
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vmov.32 q1[3], r0
+; CHECK-NEXT:    vmov r0, s5
+; CHECK-NEXT:    vmov.16 q2[2], r0
+; CHECK-NEXT:    vmov r0, s7
+; CHECK-NEXT:    vmov.16 q2[3], r0
+; CHECK-NEXT:    vmov r0, s12
+; CHECK-NEXT:    vmovx.f16 s12, s3
+; CHECK-NEXT:    vmov.16 q2[4], r0
+; CHECK-NEXT:    vmov r0, s12
+; CHECK-NEXT:    vmovx.f16 s12, s5
+; CHECK-NEXT:    vmov.16 q2[5], r0
+; CHECK-NEXT:    vmov r0, s12
+; CHECK-NEXT:    vmovx.f16 s12, s7
+; CHECK-NEXT:    vmov.16 q2[6], r0
+; CHECK-NEXT:    vmov r0, s12
+; CHECK-NEXT:    vmovx.f16 s12, s0
+; CHECK-NEXT:    vmov.16 q2[7], r0
+; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    vstrw.32 q2, [r1, #16]
+; CHECK-NEXT:    vmov.16 q2[0], r2
+; CHECK-NEXT:    vmov.16 q2[1], r0
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmov.16 q2[2], r0
+; CHECK-NEXT:    vmov r0, s6
+; CHECK-NEXT:    vmov.16 q2[3], r0
+; CHECK-NEXT:    vmov r0, s12
+; CHECK-NEXT:    vmovx.f16 s0, s2
+; CHECK-NEXT:    vmov.16 q2[4], r0
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vmovx.f16 s0, s4
+; CHECK-NEXT:    vmov.16 q2[5], r0
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vmovx.f16 s0, s6
+; CHECK-NEXT:    vmov.16 q2[6], r0
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vmov.16 q2[7], r0
+; CHECK-NEXT:    vstrw.32 q2, [r1]
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %s1 = getelementptr <4 x half>, <4 x half>* %src, i32 0
+  %l1 = load <4 x half>, <4 x half>* %s1, align 4
+  %s2 = getelementptr <4 x half>, <4 x half>* %src, i32 1
+  %l2 = load <4 x half>, <4 x half>* %s2, align 4
+  %s3 = getelementptr <4 x half>, <4 x half>* %src, i32 2
+  %l3 = load <4 x half>, <4 x half>* %s3, align 4
+  %s4 = getelementptr <4 x half>, <4 x half>* %src, i32 3
+  %l4 = load <4 x half>, <4 x half>* %s3, align 4
+  %t1 = shufflevector <4 x half> %l1, <4 x half> %l2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %t2 = shufflevector <4 x half> %l3, <4 x half> %l4, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %s = shufflevector <8 x half> %t1, <8 x half> %t2, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 1, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
+  store <16 x half> %s, <16 x half> *%dst
+  ret void
+}
+
+define void @vst4_v8f16(<8 x half> *%src, <32 x half> *%dst) {
+; CHECK-LABEL: vst4_v8f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #32]
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #16]
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vmov q3, q2
+; CHECK-NEXT:    vst40.16 {q0, q1, q2, q3}, [r1]
+; CHECK-NEXT:    vst41.16 {q0, q1, q2, q3}, [r1]
+; CHECK-NEXT:    vst42.16 {q0, q1, q2, q3}, [r1]
+; CHECK-NEXT:    vst43.16 {q0, q1, q2, q3}, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  %s1 = getelementptr <8 x half>, <8 x half>* %src, i32 0
+  %l1 = load <8 x half>, <8 x half>* %s1, align 4
+  %s2 = getelementptr <8 x half>, <8 x half>* %src, i32 1
+  %l2 = load <8 x half>, <8 x half>* %s2, align 4
+  %s3 = getelementptr <8 x half>, <8 x half>* %src, i32 2
+  %l3 = load <8 x half>, <8 x half>* %s3, align 4
+  %s4 = getelementptr <8 x half>, <8 x half>* %src, i32 3
+  %l4 = load <8 x half>, <8 x half>* %s3, align 4
+  %t1 = shufflevector <8 x half> %l1, <8 x half> %l2, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %t2 = shufflevector <8 x half> %l3, <8 x half> %l4, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %s = shufflevector <16 x half> %t1, <16 x half> %t2, <32 x i32> <i32 0, i32 8, i32 16, i32 24, i32 1, i32 9, i32 17, i32 25, i32 2, i32 10, i32 18, i32 26, i32 3, i32 11, i32 19, i32 27, i32 4, i32 12, i32 20, i32 28, i32 5, i32 13, i32 21, i32 29, i32 6, i32 14, i32 22, i32 30, i32 7, i32 15, i32 23, i32 31>
+  store <32 x half> %s, <32 x half> *%dst
+  ret void
+}
+
+define void @vst4_v16f16(<16 x half> *%src, <64 x half> *%dst) {
+; CHECK-LABEL: vst4_v16f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #80]
+; CHECK-NEXT:    vldrw.u32 q6, [r0, #64]
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #48]
+; CHECK-NEXT:    vldrw.u32 q5, [r0, #32]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #16]
+; CHECK-NEXT:    vldrw.u32 q4, [r0]
+; CHECK-NEXT:    vmov q7, q6
+; CHECK-NEXT:    vmov q3, q2
+; CHECK-NEXT:    add.w r0, r1, #64
+; CHECK-NEXT:    vst40.16 {q4, q5, q6, q7}, [r1]
+; CHECK-NEXT:    vst41.16 {q4, q5, q6, q7}, [r1]
+; CHECK-NEXT:    vst42.16 {q4, q5, q6, q7}, [r1]
+; CHECK-NEXT:    vst43.16 {q4, q5, q6, q7}, [r1]
+; CHECK-NEXT:    vst40.16 {q0, q1, q2, q3}, [r0]
+; CHECK-NEXT:    vst41.16 {q0, q1, q2, q3}, [r0]
+; CHECK-NEXT:    vst42.16 {q0, q1, q2, q3}, [r0]
+; CHECK-NEXT:    vst43.16 {q0, q1, q2, q3}, [r0]
+; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    bx lr
+entry:
+  %s1 = getelementptr <16 x half>, <16 x half>* %src, i32 0
+  %l1 = load <16 x half>, <16 x half>* %s1, align 4
+  %s2 = getelementptr <16 x half>, <16 x half>* %src, i32 1
+  %l2 = load <16 x half>, <16 x half>* %s2, align 4
+  %s3 = getelementptr <16 x half>, <16 x half>* %src, i32 2
+  %l3 = load <16 x half>, <16 x half>* %s3, align 4
+  %s4 = getelementptr <16 x half>, <16 x half>* %src, i32 3
+  %l4 = load <16 x half>, <16 x half>* %s3, align 4
+  %t1 = shufflevector <16 x half> %l1, <16 x half> %l2, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+  %t2 = shufflevector <16 x half> %l3, <16 x half> %l4, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+  %s = shufflevector <32 x half> %t1, <32 x half> %t2, <64 x i32> <i32 0, i32 16, i32 32, i32 48, i32 1, i32 17, i32 33, i32 49, i32 2, i32 18, i32 34, i32 50, i32 3, i32 19, i32 35, i32 51, i32 4, i32 20, i32 36, i32 52, i32 5, i32 21, i32 37, i32 53, i32 6, i32 22, i32 38, i32 54, i32 7, i32 23, i32 39, i32 55, i32 8, i32 24, i32 40, i32 56, i32 9, i32 25, i32 41, i32 57, i32 10, i32 26, i32 42, i32 58, i32 11, i32 27, i32 43, i32 59, i32 12, i32 28, i32 44, i32 60, i32 13, i32 29, i32 45, i32 61, i32 14, i32 30, i32 46, i32 62, i32 15, i32 31, i32 47, i32 63>
+  store <64 x half> %s, <64 x half> *%dst
+  ret void
+}
+
+; f64
+
+define void @vst4_v2f64(<2 x double> *%src, <8 x double> *%dst) {
+; CHECK-LABEL: vst4_v2f64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #16]
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #32]
+; CHECK-NEXT:    vmov.f64 d6, d2
+; CHECK-NEXT:    vmov.f64 d7, d0
+; CHECK-NEXT:    vmov.f64 d0, d3
+; CHECK-NEXT:    vstrw.32 q3, [r1]
+; CHECK-NEXT:    vmov.f64 d2, d4
+; CHECK-NEXT:    vstrw.32 q0, [r1, #32]
+; CHECK-NEXT:    vmov.f64 d3, d4
+; CHECK-NEXT:    vmov.f64 d4, d5
+; CHECK-NEXT:    vstrw.32 q1, [r1, #16]
+; CHECK-NEXT:    vstrw.32 q2, [r1, #48]
+; CHECK-NEXT:    bx lr
+entry:
+  %s1 = getelementptr <2 x double>, <2 x double>* %src, i32 0
+  %l1 = load <2 x double>, <2 x double>* %s1, align 4
+  %s2 = getelementptr <2 x double>, <2 x double>* %src, i32 1
+  %l2 = load <2 x double>, <2 x double>* %s2, align 4
+  %s3 = getelementptr <2 x double>, <2 x double>* %src, i32 2
+  %l3 = load <2 x double>, <2 x double>* %s3, align 4
+  %s4 = getelementptr <2 x double>, <2 x double>* %src, i32 3
+  %l4 = load <2 x double>, <2 x double>* %s3, align 4
+  %t1 = shufflevector <2 x double> %l1, <2 x double> %l2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %t2 = shufflevector <2 x double> %l3, <2 x double> %l4, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %s = shufflevector <4 x double> %t1, <4 x double> %t2, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 1, i32 3, i32 5, i32 7>
+  store <8 x double> %s, <8 x double> *%dst
+  ret void
+}
+
+define void @vst4_v4f64(<4 x double> *%src, <16 x double> *%dst) {
+; CHECK-LABEL: vst4_v4f64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    vldrw.u32 q5, [r0]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #32]
+; CHECK-NEXT:    vldrw.u32 q6, [r0, #16]
+; CHECK-NEXT:    vldrw.u32 q4, [r0, #80]
+; CHECK-NEXT:    vmov.f64 d4, d10
+; CHECK-NEXT:    vldrw.u32 q3, [r0, #48]
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #64]
+; CHECK-NEXT:    vmov.f64 d5, d0
+; CHECK-NEXT:    vmov.f64 d0, d11
+; CHECK-NEXT:    vstrw.32 q2, [r1]
+; CHECK-NEXT:    vmov.f64 d10, d12
+; CHECK-NEXT:    vstrw.32 q0, [r1, #32]
+; CHECK-NEXT:    vmov.f64 d11, d6
+; CHECK-NEXT:    vmov.f64 d6, d13
+; CHECK-NEXT:    vstrw.32 q5, [r1, #64]
+; CHECK-NEXT:    vmov.f64 d12, d2
+; CHECK-NEXT:    vstrw.32 q3, [r1, #96]
+; CHECK-NEXT:    vmov.f64 d14, d8
+; CHECK-NEXT:    vmov.f64 d13, d2
+; CHECK-NEXT:    vmov.f64 d15, d8
+; CHECK-NEXT:    vstrw.32 q6, [r1, #16]
+; CHECK-NEXT:    vmov.f64 d2, d3
+; CHECK-NEXT:    vstrw.32 q7, [r1, #80]
+; CHECK-NEXT:    vmov.f64 d8, d9
+; CHECK-NEXT:    vstrw.32 q1, [r1, #48]
+; CHECK-NEXT:    vstrw.32 q4, [r1, #112]
+; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    bx lr
+entry:
+  %s1 = getelementptr <4 x double>, <4 x double>* %src, i32 0
+  %l1 = load <4 x double>, <4 x double>* %s1, align 4
+  %s2 = getelementptr <4 x double>, <4 x double>* %src, i32 1
+  %l2 = load <4 x double>, <4 x double>* %s2, align 4
+  %s3 = getelementptr <4 x double>, <4 x double>* %src, i32 2
+  %l3 = load <4 x double>, <4 x double>* %s3, align 4
+  %s4 = getelementptr <4 x double>, <4 x double>* %src, i32 3
+  %l4 = load <4 x double>, <4 x double>* %s3, align 4
+  %t1 = shufflevector <4 x double> %l1, <4 x double> %l2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %t2 = shufflevector <4 x double> %l3, <4 x double> %l4, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %s = shufflevector <8 x double> %t1, <8 x double> %t2, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 1, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
+  store <16 x double> %s, <16 x double> *%dst
+  ret void
+}
diff --git a/llvm/test/CodeGen/Thumb2/mve-vsubqr.ll b/llvm/test/CodeGen/Thumb2/mve-vsubqr.ll
new file mode 100644
index 000000000..ec2f7c11b
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-vsubqr.ll
@@ -0,0 +1,77 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -verify-machineinstrs -mattr=+mve %s -o - | FileCheck %s
+
+define arm_aapcs_vfpcc <4 x i32> @vsubqr_v4i32(<4 x i32> %src, i32 %src2, <4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: vsubqr_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vsub.i32 q0, q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <4 x i32> undef, i32 %src2, i32 0
+  %sp = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %c = sub <4 x i32> %src, %sp
+  ret <4 x i32> %c
+}
+
+define arm_aapcs_vfpcc <8 x i16> @vsubqr_v8i16(<8 x i16> %src, i16 %src2, <8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: vsubqr_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vsub.i16 q0, q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <8 x i16> undef, i16 %src2, i32 0
+  %sp = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
+  %c = sub <8 x i16> %src, %sp
+  ret <8 x i16> %c
+}
+
+define arm_aapcs_vfpcc <16 x i8> @vsubqr_v16i8(<16 x i8> %src, i8 %src2, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: vsubqr_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vsub.i8 q0, q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <16 x i8> undef, i8 %src2, i32 0
+  %sp = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
+  %c = sub <16 x i8> %src, %sp
+  ret <16 x i8> %c
+}
+
+define arm_aapcs_vfpcc <4 x i32> @vsubqr_v4i32_2(<4 x i32> %src, i32 %src2, <4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: vsubqr_v4i32_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdup.32 q1, r0
+; CHECK-NEXT:    vsub.i32 q0, q1, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <4 x i32> undef, i32 %src2, i32 0
+  %sp = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %c = sub <4 x i32> %sp, %src
+  ret <4 x i32> %c
+}
+
+define arm_aapcs_vfpcc <8 x i16> @vsubqr_v8i16_2(<8 x i16> %src, i16 %src2, <8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: vsubqr_v8i16_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdup.16 q1, r0
+; CHECK-NEXT:    vsub.i16 q0, q1, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <8 x i16> undef, i16 %src2, i32 0
+  %sp = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
+  %c = sub <8 x i16> %sp, %src
+  ret <8 x i16> %c
+}
+
+define arm_aapcs_vfpcc <16 x i8> @vsubqr_v16i8_2(<16 x i8> %src, i8 %src2, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: vsubqr_v16i8_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdup.8 q1, r0
+; CHECK-NEXT:    vsub.i8 q0, q1, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %i = insertelement <16 x i8> undef, i8 %src2, i32 0
+  %sp = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
+  %c = sub <16 x i8> %sp, %src
+  ret <16 x i8> %c
+}
diff --git a/llvm/test/CodeGen/Thumb2/mve-widen-narrow.ll b/llvm/test/CodeGen/Thumb2/mve-widen-narrow.ll
index 3ae4f8ab4..4c3b4d3e1 100644
--- a/llvm/test/CodeGen/Thumb2/mve-widen-narrow.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-widen-narrow.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve %s -o - | FileCheck %s
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-LE
+; RUN: llc -mtriple=thumbebv8.1m.main-arm-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-BE
 
 define void @foo_int8_int32(<4 x i8>* %dest, <4 x i32>* readonly %src, i32 %n) {
 ; CHECK-LABEL: foo_int8_int32:
@@ -14,7 +15,6 @@ entry:
   ret void
 }
 
-
 define void @foo_int16_int32(<4 x i16>* %dest, <4 x i32>* readonly %src, i32 %n) {
 ; CHECK-LABEL: foo_int16_int32:
 ; CHECK:       @ %bb.0: @ %entry
@@ -28,7 +28,6 @@ entry:
   ret void
 }
 
-
 define void @foo_int8_int16(<8 x i8>* %dest, <8 x i16>* readonly %src, i32 %n) {
 ; CHECK-LABEL: foo_int8_int16:
 ; CHECK:       @ %bb.0: @ %entry
@@ -43,6 +42,56 @@ entry:
 }
 
 
+define void @foo_int8_int32_double(<16 x i8>* %dest, <16 x i32>* readonly %src, i32 %n) {
+; CHECK-LABEL: foo_int8_int32_double:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r1, #16]
+; CHECK-NEXT:    vldrw.u32 q2, [r1, #32]
+; CHECK-NEXT:    vldrw.u32 q3, [r1, #48]
+; CHECK-NEXT:    vstrb.32 q1, [r0, #4]
+; CHECK-NEXT:    vstrb.32 q0, [r0]
+; CHECK-NEXT:    vstrb.32 q3, [r0, #12]
+; CHECK-NEXT:    vstrb.32 q2, [r0, #8]
+; CHECK-NEXT:    bx lr
+entry:
+  %wide.load = load <16 x i32>, <16 x i32>* %src, align 4
+  %0 = trunc <16 x i32> %wide.load to <16 x i8>
+  store <16 x i8> %0, <16 x i8>* %dest, align 1
+  ret void
+}
+
+define void @foo_int16_int32_double(<8 x i16>* %dest, <8 x i32>* readonly %src, i32 %n) {
+; CHECK-LABEL: foo_int16_int32_double:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r1, #16]
+; CHECK-NEXT:    vstrh.32 q1, [r0, #8]
+; CHECK-NEXT:    vstrh.32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %wide.load = load <8 x i32>, <8 x i32>* %src, align 4
+  %0 = trunc <8 x i32> %wide.load to <8 x i16>
+  store <8 x i16> %0, <8 x i16>* %dest, align 2
+  ret void
+}
+
+define void @foo_int8_int16_double(<16 x i8>* %dest, <16 x i16>* readonly %src, i32 %n) {
+; CHECK-LABEL: foo_int8_int16_double:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q1, [r1, #16]
+; CHECK-NEXT:    vstrb.16 q1, [r0, #8]
+; CHECK-NEXT:    vstrb.16 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %wide.load = load <16 x i16>, <16 x i16>* %src, align 2
+  %0 = trunc <16 x i16> %wide.load to <16 x i8>
+  store <16 x i8> %0, <16 x i8>* %dest, align 1
+  ret void
+}
+
+
 define void @foo_int32_int8(<4 x i32>* %dest, <4 x i8>* readonly %src, i32 %n) {
 ; CHECK-LABEL: foo_int32_int8:
 ; CHECK:       @ %bb.0: @ %entry
@@ -56,7 +105,6 @@ entry:
   ret void
 }
 
-
 define void @foo_int16_int8(<8 x i16>* %dest, <8 x i8>* readonly %src, i32 %n) {
 ; CHECK-LABEL: foo_int16_int8:
 ; CHECK:       @ %bb.0: @ %entry
@@ -70,7 +118,6 @@ entry:
   ret void
 }
 
-
 define void @foo_int32_int16(<4 x i32>* %dest, <4 x i16>* readonly %src, i32 %n) {
 ; CHECK-LABEL: foo_int32_int16:
 ; CHECK:       @ %bb.0: @ %entry
@@ -84,6 +131,55 @@ entry:
   ret void
 }
 
+define void @foo_int32_int8_double(<16 x i32>* %dest, <16 x i8>* readonly %src, i32 %n) {
+; CHECK-LABEL: foo_int32_int8_double:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.s32 q0, [r1, #4]
+; CHECK-NEXT:    vldrb.s32 q1, [r1]
+; CHECK-NEXT:    vldrb.s32 q2, [r1, #12]
+; CHECK-NEXT:    vldrb.s32 q3, [r1, #8]
+; CHECK-NEXT:    vstrw.32 q1, [r0]
+; CHECK-NEXT:    vstrw.32 q0, [r0, #16]
+; CHECK-NEXT:    vstrw.32 q3, [r0, #32]
+; CHECK-NEXT:    vstrw.32 q2, [r0, #48]
+; CHECK-NEXT:    bx lr
+entry:
+  %wide.load = load <16 x i8>, <16 x i8>* %src, align 1
+  %0 = sext <16 x i8> %wide.load to <16 x i32>
+  store <16 x i32> %0, <16 x i32>* %dest, align 4
+  ret void
+}
+
+define void @foo_int16_int8_double(<16 x i16>* %dest, <16 x i8>* readonly %src, i32 %n) {
+; CHECK-LABEL: foo_int16_int8_double:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.s16 q0, [r1]
+; CHECK-NEXT:    vldrb.s16 q1, [r1, #8]
+; CHECK-NEXT:    vstrh.16 q1, [r0, #16]
+; CHECK-NEXT:    vstrh.16 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %wide.load = load <16 x i8>, <16 x i8>* %src, align 1
+  %0 = sext <16 x i8> %wide.load to <16 x i16>
+  store <16 x i16> %0, <16 x i16>* %dest, align 2
+  ret void
+}
+
+define void @foo_int32_int16_double(<8 x i32>* %dest, <8 x i16>* readonly %src, i32 %n) {
+; CHECK-LABEL: foo_int32_int16_double:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.s32 q0, [r1]
+; CHECK-NEXT:    vldrh.s32 q1, [r1, #8]
+; CHECK-NEXT:    vstrw.32 q1, [r0, #16]
+; CHECK-NEXT:    vstrw.32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %wide.load = load <8 x i16>, <8 x i16>* %src, align 2
+  %0 = sext <8 x i16> %wide.load to <8 x i32>
+  store <8 x i32> %0, <8 x i32>* %dest, align 4
+  ret void
+}
+
 
 define void @foo_uint32_uint8(<4 x i32>* %dest, <4 x i8>* readonly %src, i32 %n) {
 ; CHECK-LABEL: foo_uint32_uint8:
@@ -98,7 +194,6 @@ entry:
   ret void
 }
 
-
 define void @foo_uint16_uint8(<8 x i16>* %dest, <8 x i8>* readonly %src, i32 %n) {
 ; CHECK-LABEL: foo_uint16_uint8:
 ; CHECK:       @ %bb.0: @ %entry
@@ -112,7 +207,6 @@ entry:
   ret void
 }
 
-
 define void @foo_uint32_uint16(<4 x i32>* %dest, <4 x i16>* readonly %src, i32 %n) {
 ; CHECK-LABEL: foo_uint32_uint16:
 ; CHECK:       @ %bb.0: @ %entry
@@ -125,3 +219,207 @@ entry:
   store <4 x i32> %0, <4 x i32>* %dest, align 4
   ret void
 }
+
+
+define void @foo_uint32_uint8_double(<16 x i32>* %dest, <16 x i8>* readonly %src, i32 %n) {
+; CHECK-LABEL: foo_uint32_uint8_double:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u32 q0, [r1, #4]
+; CHECK-NEXT:    vldrb.u32 q1, [r1]
+; CHECK-NEXT:    vldrb.u32 q2, [r1, #12]
+; CHECK-NEXT:    vldrb.u32 q3, [r1, #8]
+; CHECK-NEXT:    vstrw.32 q1, [r0]
+; CHECK-NEXT:    vstrw.32 q0, [r0, #16]
+; CHECK-NEXT:    vstrw.32 q3, [r0, #32]
+; CHECK-NEXT:    vstrw.32 q2, [r0, #48]
+; CHECK-NEXT:    bx lr
+entry:
+  %wide.load = load <16 x i8>, <16 x i8>* %src, align 1
+  %0 = zext <16 x i8> %wide.load to <16 x i32>
+  store <16 x i32> %0, <16 x i32>* %dest, align 4
+  ret void
+}
+
+define void @foo_uint16_uint8_double(<16 x i16>* %dest, <16 x i8>* readonly %src, i32 %n) {
+; CHECK-LABEL: foo_uint16_uint8_double:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u16 q0, [r1]
+; CHECK-NEXT:    vldrb.u16 q1, [r1, #8]
+; CHECK-NEXT:    vstrh.16 q1, [r0, #16]
+; CHECK-NEXT:    vstrh.16 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %wide.load = load <16 x i8>, <16 x i8>* %src, align 1
+  %0 = zext <16 x i8> %wide.load to <16 x i16>
+  store <16 x i16> %0, <16 x i16>* %dest, align 2
+  ret void
+}
+
+define void @foo_uint32_uint16_double(<8 x i32>* %dest, <8 x i16>* readonly %src, i32 %n) {
+; CHECK-LABEL: foo_uint32_uint16_double:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u32 q0, [r1]
+; CHECK-NEXT:    vldrh.u32 q1, [r1, #8]
+; CHECK-NEXT:    vstrw.32 q1, [r0, #16]
+; CHECK-NEXT:    vstrw.32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %wide.load = load <8 x i16>, <8 x i16>* %src, align 2
+  %0 = zext <8 x i16> %wide.load to <8 x i32>
+  store <8 x i32> %0, <8 x i32>* %dest, align 4
+  ret void
+}
+
+
+define void @foo_int32_int8_both(<16 x i32>* %dest, <16 x i8>* readonly %src, i32 %n) {
+; CHECK-LABEL: foo_int32_int8_both:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.s16 q1, [r1, #8]
+; CHECK-NEXT:    vmov.u16 r2, q1[4]
+; CHECK-NEXT:    vmov.32 q0[0], r2
+; CHECK-NEXT:    vmov.u16 r2, q1[5]
+; CHECK-NEXT:    vmov.32 q0[1], r2
+; CHECK-NEXT:    vmov.u16 r2, q1[6]
+; CHECK-NEXT:    vmov.32 q0[2], r2
+; CHECK-NEXT:    vmov.u16 r2, q1[7]
+; CHECK-NEXT:    vmov.32 q0[3], r2
+; CHECK-NEXT:    vmovlb.u16 q2, q0
+; CHECK-NEXT:    vldrb.s16 q0, [r1]
+; CHECK-NEXT:    vmov.u16 r1, q1[0]
+; CHECK-NEXT:    vstrw.32 q2, [r0, #48]
+; CHECK-NEXT:    vmov.32 q2[0], r1
+; CHECK-NEXT:    vmov.u16 r1, q1[1]
+; CHECK-NEXT:    vmov.32 q2[1], r1
+; CHECK-NEXT:    vmov.u16 r1, q1[2]
+; CHECK-NEXT:    vmov.32 q2[2], r1
+; CHECK-NEXT:    vmov.u16 r1, q1[3]
+; CHECK-NEXT:    vmov.32 q2[3], r1
+; CHECK-NEXT:    vmov.u16 r1, q0[4]
+; CHECK-NEXT:    vmovlb.u16 q1, q2
+; CHECK-NEXT:    vstrw.32 q1, [r0, #32]
+; CHECK-NEXT:    vmov.32 q1[0], r1
+; CHECK-NEXT:    vmov.u16 r1, q0[5]
+; CHECK-NEXT:    vmov.32 q1[1], r1
+; CHECK-NEXT:    vmov.u16 r1, q0[6]
+; CHECK-NEXT:    vmov.32 q1[2], r1
+; CHECK-NEXT:    vmov.u16 r1, q0[7]
+; CHECK-NEXT:    vmov.32 q1[3], r1
+; CHECK-NEXT:    vmov.u16 r1, q0[0]
+; CHECK-NEXT:    vmovlb.u16 q1, q1
+; CHECK-NEXT:    vstrw.32 q1, [r0, #16]
+; CHECK-NEXT:    vmov.32 q1[0], r1
+; CHECK-NEXT:    vmov.u16 r1, q0[1]
+; CHECK-NEXT:    vmov.32 q1[1], r1
+; CHECK-NEXT:    vmov.u16 r1, q0[2]
+; CHECK-NEXT:    vmov.32 q1[2], r1
+; CHECK-NEXT:    vmov.u16 r1, q0[3]
+; CHECK-NEXT:    vmov.32 q1[3], r1
+; CHECK-NEXT:    vmovlb.u16 q0, q1
+; CHECK-NEXT:    vstrw.32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %wide.load = load <16 x i8>, <16 x i8>* %src, align 1
+  %0 = sext <16 x i8> %wide.load to <16 x i16>
+  %1 = zext <16 x i16> %0 to <16 x i32>
+  store <16 x i32> %1, <16 x i32>* %dest, align 4
+  ret void
+}
+
+define <8 x i16>* @foo_uint32_uint16_double_offset(<8 x i32>* %dest, <8 x i16>* readonly %src, i32 %n) {
+; CHECK-LABEL: foo_uint32_uint16_double_offset:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.s32 q0, [r1, #16]!
+; CHECK-NEXT:    vldrh.s32 q1, [r1, #8]
+; CHECK-NEXT:    vstrw.32 q0, [r0]
+; CHECK-NEXT:    vstrw.32 q1, [r0, #16]
+; CHECK-NEXT:    mov r0, r1
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds <8 x i16>, <8 x i16>* %src, i32 1
+  %wide.load = load <8 x i16>, <8 x i16>* %z, align 2
+  %0 = sext <8 x i16> %wide.load to <8 x i32>
+  store <8 x i32> %0, <8 x i32>* %dest, align 4
+  ret <8 x i16>* %z
+}
+
+define <16 x i16>* @foo_uint32_uint16_quad_offset(<16 x i32>* %dest, <16 x i16>* readonly %src, i32 %n) {
+; CHECK-LABEL: foo_uint32_uint16_quad_offset:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.s32 q0, [r1, #32]!
+; CHECK-NEXT:    vldrh.s32 q1, [r1, #8]
+; CHECK-NEXT:    vldrh.s32 q2, [r1, #24]
+; CHECK-NEXT:    vldrh.s32 q3, [r1, #16]
+; CHECK-NEXT:    vstrw.32 q0, [r0]
+; CHECK-NEXT:    vstrw.32 q2, [r0, #48]
+; CHECK-NEXT:    vstrw.32 q1, [r0, #16]
+; CHECK-NEXT:    vstrw.32 q3, [r0, #32]
+; CHECK-NEXT:    mov r0, r1
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds <16 x i16>, <16 x i16>* %src, i32 1
+  %wide.load = load <16 x i16>, <16 x i16>* %z, align 2
+  %0 = sext <16 x i16> %wide.load to <16 x i32>
+  store <16 x i32> %0, <16 x i32>* %dest, align 4
+  ret <16 x i16>* %z
+}
+
+
+define void @foo_int16_int32_align1(<4 x i16>* %dest, <4 x i32>* readonly %src, i32 %n) {
+; CHECK-LABEL: foo_int16_int32_align1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .pad #8
+; CHECK-NEXT:    sub sp, #8
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    mov r1, sp
+; CHECK-NEXT:    vstrh.32 q0, [r1]
+; CHECK-NEXT:    ldrd r1, r2, [sp]
+; CHECK-NEXT:    str r1, [r0]
+; CHECK-NEXT:    str r2, [r0, #4]
+; CHECK-NEXT:    add sp, #8
+; CHECK-NEXT:    bx lr
+entry:
+  %wide.load = load <4 x i32>, <4 x i32>* %src, align 4
+  %0 = trunc <4 x i32> %wide.load to <4 x i16>
+  store <4 x i16> %0, <4 x i16>* %dest, align 1
+  ret void
+}
+
+define void @foo_int32_int16_align1(<4 x i32>* %dest, <4 x i16>* readonly %src, i32 %n) {
+; CHECK-LABEL: foo_int32_int16_align1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .pad #8
+; CHECK-NEXT:    sub sp, #8
+; CHECK-NEXT:    ldr r2, [r1]
+; CHECK-NEXT:    ldr r1, [r1, #4]
+; CHECK-NEXT:    strd r2, r1, [sp]
+; CHECK-NEXT:    mov r1, sp
+; CHECK-NEXT:    vldrh.s32 q0, [r1]
+; CHECK-NEXT:    vstrw.32 q0, [r0]
+; CHECK-NEXT:    add sp, #8
+; CHECK-NEXT:    bx lr
+entry:
+  %wide.load = load <4 x i16>, <4 x i16>* %src, align 1
+  %0 = sext <4 x i16> %wide.load to <4 x i32>
+  store <4 x i32> %0, <4 x i32>* %dest, align 4
+  ret void
+}
+
+define void @foo_uint32_uint16_align1(<4 x i32>* %dest, <4 x i16>* readonly %src, i32 %n) {
+; CHECK-LABEL: foo_uint32_uint16_align1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .pad #8
+; CHECK-NEXT:    sub sp, #8
+; CHECK-NEXT:    ldr r2, [r1]
+; CHECK-NEXT:    ldr r1, [r1, #4]
+; CHECK-NEXT:    strd r2, r1, [sp]
+; CHECK-NEXT:    mov r1, sp
+; CHECK-NEXT:    vldrh.u32 q0, [r1]
+; CHECK-NEXT:    vstrw.32 q0, [r0]
+; CHECK-NEXT:    add sp, #8
+; CHECK-NEXT:    bx lr
+entry:
+  %wide.load = load <4 x i16>, <4 x i16>* %src, align 1
+  %0 = zext <4 x i16> %wide.load to <4 x i32>
+  store <4 x i32> %0, <4 x i32>* %dest, align 4
+  ret void
+}
diff --git a/llvm/test/CodeGen/Thumb2/segmented-stacks.ll b/llvm/test/CodeGen/Thumb2/segmented-stacks.ll
index 49627fb7c..ee4dd0186 100644
--- a/llvm/test/CodeGen/Thumb2/segmented-stacks.ll
+++ b/llvm/test/CodeGen/Thumb2/segmented-stacks.ll
@@ -1,34 +1,70 @@
-; RUN: llc < %s -mtriple=thumb-linux-androideabi -mcpu=arm1156t2-s -mattr=+thumb2 -verify-machineinstrs | FileCheck %s -check-prefix=Thumb-android
-; RUN: llc < %s -mtriple=thumb-linux-androideabi -mcpu=arm1156t2-s -mattr=+thumb2 -filetype=obj
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=thumb-linux-androideabi -mcpu=arm1156t2-s -mattr=+thumb2 -verify-machineinstrs | FileCheck %s -check-prefix=THUMB
+; RUN: llc < %s -mtriple=arm-linux-androideabi -mcpu=arm1156t2-s -verify-machineinstrs | FileCheck %s -check-prefix=ARM
 
 
 ; Just to prevent the alloca from being optimized away
 declare void @dummy_use(i32*, i32)
 
 define void @test_basic() #0 {
-        %mem = alloca i32, i32 10
-        call void @dummy_use (i32* %mem, i32 10)
-	ret void
-
-; Thumb-android:      test_basic:
-
-; Thumb-android:      push    {r4, r5}
-; Thumb-android-NEXT: mrc     p15, #0, r4, c13, c0, #3
-; Thumb-android-NEXT: mov     r5, sp
-; Thumb-android-NEXT: ldr     r4, [r4, #252]
-; Thumb-android-NEXT: cmp     r4, r5
-; Thumb-android-NEXT: blo     .LBB0_2
-
-; Thumb-android:      mov     r4, #48
-; Thumb-android-NEXT: mov     r5, #0
-; Thumb-android-NEXT: push    {lr}
-; Thumb-android-NEXT: bl      __morestack
-; Thumb-android-NEXT: ldr     lr, [sp], #4
-; Thumb-android-NEXT: pop     {r4, r5}
-; Thumb-android-NEXT: bx      lr
-
-; Thumb-android:      pop     {r4, r5}
-
+; THUMB-LABEL: test_basic:
+; THUMB:       @ %bb.0:
+; THUMB-NEXT:    push {r4, r5}
+; THUMB-NEXT:    mrc p15, #0, r4, c13, c0, #3
+; THUMB-NEXT:    mov r5, sp
+; THUMB-NEXT:    ldr.w r4, [r4, #252]
+; THUMB-NEXT:    cmp r4, r5
+; THUMB-NEXT:    blo .LBB0_2
+; THUMB-NEXT:  @ %bb.1:
+; THUMB-NEXT:    mov r4, #48
+; THUMB-NEXT:    mov r5, #0
+; THUMB-NEXT:    push {lr}
+; THUMB-NEXT:    bl __morestack
+; THUMB-NEXT:    ldr lr, [sp], #4
+; THUMB-NEXT:    pop {r4, r5}
+; THUMB-NEXT:    bx lr
+; THUMB-NEXT:  .LBB0_2:
+; THUMB-NEXT:    pop {r4, r5}
+; THUMB-NEXT:    .save {r7, lr}
+; THUMB-NEXT:    push {r7, lr}
+; THUMB-NEXT:    .pad #40
+; THUMB-NEXT:    sub sp, #40
+; THUMB-NEXT:    mov r0, sp
+; THUMB-NEXT:    movs r1, #10
+; THUMB-NEXT:    bl dummy_use
+; THUMB-NEXT:    add sp, #40
+; THUMB-NEXT:    pop {r7, pc}
+;
+; ARM-LABEL: test_basic:
+; ARM:       @ %bb.0:
+; ARM-NEXT:    push {r4, r5}
+; ARM-NEXT:    mrc p15, #0, r4, c13, c0, #3
+; ARM-NEXT:    mov r5, sp
+; ARM-NEXT:    ldr r4, [r4, #252]
+; ARM-NEXT:    cmp r4, r5
+; ARM-NEXT:    blo .LBB0_2
+; ARM-NEXT:  @ %bb.1:
+; ARM-NEXT:    mov r4, #48
+; ARM-NEXT:    mov r5, #0
+; ARM-NEXT:    stmdb sp!, {lr}
+; ARM-NEXT:    bl __morestack
+; ARM-NEXT:    ldm sp!, {lr}
+; ARM-NEXT:    pop {r4, r5}
+; ARM-NEXT:    bx lr
+; ARM-NEXT:  .LBB0_2:
+; ARM-NEXT:    pop {r4, r5}
+; ARM-NEXT:    .save {r11, lr}
+; ARM-NEXT:    push {r11, lr}
+; ARM-NEXT:    .pad #40
+; ARM-NEXT:    sub sp, sp, #40
+; ARM-NEXT:    mov r0, sp
+; ARM-NEXT:    mov r1, #10
+; ARM-NEXT:    bl dummy_use
+; ARM-NEXT:    add sp, sp, #40
+; ARM-NEXT:    pop {r11, pc}
+  %mem = alloca i32, i32 10
+  call void @dummy_use (i32* %mem, i32 10)
+  ret void
 }
 
 attributes #0 = { "split-stack" }
diff --git a/llvm/test/CodeGen/Thumb2/stack_guard_remat.ll b/llvm/test/CodeGen/Thumb2/stack_guard_remat.ll
index 839a506b3..b007419b1 100644
--- a/llvm/test/CodeGen/Thumb2/stack_guard_remat.ll
+++ b/llvm/test/CodeGen/Thumb2/stack_guard_remat.ll
@@ -40,4 +40,4 @@ declare void @foo3(i32*)
 ; Function Attrs: nounwind
 declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture)
 
-attributes #0 = { nounwind ssp "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind ssp "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/Thumb2/t2peephole-t2ADDrr-to-t2ADDri.ll b/llvm/test/CodeGen/Thumb2/t2peephole-t2ADDrr-to-t2ADDri.ll
new file mode 100644
index 000000000..817eac566
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/t2peephole-t2ADDrr-to-t2ADDri.ll
@@ -0,0 +1,10 @@
+; RUN: llc -mtriple=thumb-eabi --stop-after=peephole-opt -mcpu=arm1156t2-s -mattr=+thumb2 %s -o - | FileCheck %s
+define i32 @t2_const_var2_1_ok_2(i32 %lhs) {
+; CHECK: [[R0:%0|%[1-9][0-9]*]]:gprnopc = COPY $r0
+; CHECK-NEXT: [[R1:%0|%[1-9][0-9]*]]:rgpr = t2ADDri [[R0]], 11206656
+; CHECK-NEXT: [[R2:%0|%[1-9][0-9]*]]:rgpr = t2ADDri killed [[R1]], 187
+; CHECK-NEXT: $r0 = COPY [[R2]]
+  %ret = add i32 %lhs, 11206843 ; 0x00ab00bb
+  ret i32 %ret
+}
+
diff --git a/llvm/test/CodeGen/Thumb2/thumb2-ldm.ll b/llvm/test/CodeGen/Thumb2/thumb2-ldm.ll
index abda4bac5..a33efd204 100644
--- a/llvm/test/CodeGen/Thumb2/thumb2-ldm.ll
+++ b/llvm/test/CodeGen/Thumb2/thumb2-ldm.ll
@@ -3,7 +3,7 @@
 
 @X = external global [0 x i32]          ; <[0 x i32]*> [#uses=5]
 
-define i32 @t1() "no-frame-pointer-elim"="true" {
+define i32 @t1() "frame-pointer"="all" {
 ; ALL-LABEL: t1:
 ; ALL: push {r7, lr}
 ; CHECK: ldrd
@@ -16,7 +16,7 @@ define i32 @t1() "no-frame-pointer-elim"="true" {
         ret i32 %tmp4
 }
 
-define i32 @t2() "no-frame-pointer-elim"="true" {
+define i32 @t2() "frame-pointer"="all" {
 ; ALL-LABEL: t2:
 ; ALL: push {r7, lr}
 ; CHECK: ldm
@@ -30,7 +30,7 @@ define i32 @t2() "no-frame-pointer-elim"="true" {
         ret i32 %tmp6
 }
 
-define i32 @t3() "no-frame-pointer-elim"="true" {
+define i32 @t3() "frame-pointer"="all" {
 ; ALL-LABEL: t3:
 ; ALL: push {r7, lr}
 ; CHECK: ldm
@@ -46,7 +46,7 @@ define i32 @t3() "no-frame-pointer-elim"="true" {
 
 @g = common global i32* null
 
-define void @t4(i32 %a0, i32 %a1, i32 %a2) "no-frame-pointer-elim"="true" {
+define void @t4(i32 %a0, i32 %a1, i32 %a2) "frame-pointer"="all" {
 ; ALL-LABEL: t4:
 ; ALL: stm.w sp, {r0, r1, r2}
 ; ALL: bl _ext
diff --git a/llvm/test/CodeGen/Thumb2/thumb2-uxtb.ll b/llvm/test/CodeGen/Thumb2/thumb2-uxtb.ll
index 8f6d556f8..5d2456ba4 100644
--- a/llvm/test/CodeGen/Thumb2/thumb2-uxtb.ll
+++ b/llvm/test/CodeGen/Thumb2/thumb2-uxtb.ll
@@ -1,63 +1,100 @@
-; RUN: llc -mtriple=thumb-eabi -mcpu=cortex-a8 %s -o - | FileCheck %s -check-prefix=CHECK-DSP
-; RUN: llc -mtriple=thumb-eabi -mcpu=cortex-m3 %s -o - | FileCheck %s -check-prefix=CHECK-NO-DSP
-; RUN: llc -mtriple=thumbv7em-eabi %s -o - | FileCheck %s -check-prefix=CHECK-DSP
-; RUN: llc -mtriple=thumbv8m.main-none-eabi %s -o - | FileCheck %s -check-prefix=CHECK-NO-DSP
-; RUN: llc -mtriple=thumbv8m.main-none-eabi -mattr=+dsp %s -o - | FileCheck %s -check-prefix=CHECK-DSP
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumb-eabi -mcpu=cortex-a8 %s -o - | FileCheck %s -check-prefixes=CHECK,CHECK-DSP
+; RUN: llc -mtriple=thumb-eabi -mcpu=cortex-m3 %s -o - | FileCheck %s -check-prefixes=CHECK,CHECK-NO-DSP
+; RUN: llc -mtriple=thumbv7em-eabi %s -o - | FileCheck %s -check-prefixes=CHECK,CHECK-DSP
+; RUN: llc -mtriple=thumbv8m.main-none-eabi %s -o - | FileCheck %s -check-prefixes=CHECK,CHECK-NO-DSP
+; RUN: llc -mtriple=thumbv8m.main-none-eabi -mattr=+dsp %s -o - | FileCheck %s -check-prefixes=CHECK,CHECK-DSP
 
 define i32 @test1(i32 %x) {
-; CHECK-LABEL: test1
-; CHECK-DSP: uxtb16 r0, r0
-; CHECK-NO-DSP: bic r0, r0, #-16711936
+; CHECK-DSP-LABEL: test1:
+; CHECK-DSP:       @ %bb.0:
+; CHECK-DSP-NEXT:    uxtb16 r0, r0
+; CHECK-DSP-NEXT:    bx lr
+;
+; CHECK-NO-DSP-LABEL: test1:
+; CHECK-NO-DSP:       @ %bb.0:
+; CHECK-NO-DSP-NEXT:    bic r0, r0, #-16711936
+; CHECK-NO-DSP-NEXT:    bx lr
 	%tmp1 = and i32 %x, 16711935		; <i32> [#uses=1]
 	ret i32 %tmp1
 }
 
 ; PR7503
 define i32 @test2(i32 %x) {
-; CHECK-LABEL: test2
-; CHECK-DSP: uxtb16  r0, r0, ror #8
-; CHECK-NO-DSP: mov.w r1, #16711935
-; CHECK-NO-DSP: and.w r0, r1, r0, lsr #8
+; CHECK-DSP-LABEL: test2:
+; CHECK-DSP:       @ %bb.0:
+; CHECK-DSP-NEXT:    uxtb16 r0, r0, ror #8
+; CHECK-DSP-NEXT:    bx lr
+;
+; CHECK-NO-DSP-LABEL: test2:
+; CHECK-NO-DSP:       @ %bb.0:
+; CHECK-NO-DSP-NEXT:    mov.w r1, #16711935
+; CHECK-NO-DSP-NEXT:    and.w r0, r1, r0, lsr #8
+; CHECK-NO-DSP-NEXT:    bx lr
 	%tmp1 = lshr i32 %x, 8		; <i32> [#uses=1]
 	%tmp2 = and i32 %tmp1, 16711935		; <i32> [#uses=1]
 	ret i32 %tmp2
 }
 
 define i32 @test3(i32 %x) {
-; CHECK-LABEL: test3
-; CHECK-DSP: uxtb16  r0, r0, ror #8
-; CHECK-NO-DSP: mov.w r1, #16711935
-; CHECK-NO-DSP: and.w r0, r1, r0, lsr #8
+; CHECK-DSP-LABEL: test3:
+; CHECK-DSP:       @ %bb.0:
+; CHECK-DSP-NEXT:    uxtb16 r0, r0, ror #8
+; CHECK-DSP-NEXT:    bx lr
+;
+; CHECK-NO-DSP-LABEL: test3:
+; CHECK-NO-DSP:       @ %bb.0:
+; CHECK-NO-DSP-NEXT:    mov.w r1, #16711935
+; CHECK-NO-DSP-NEXT:    and.w r0, r1, r0, lsr #8
+; CHECK-NO-DSP-NEXT:    bx lr
 	%tmp1 = lshr i32 %x, 8		; <i32> [#uses=1]
 	%tmp2 = and i32 %tmp1, 16711935		; <i32> [#uses=1]
 	ret i32 %tmp2
 }
 
 define i32 @test4(i32 %x) {
-; CHECK-LABEL: test4
-; CHECK-DSP: uxtb16  r0, r0, ror #8
-; CHECK-NO-DSP: mov.w r1, #16711935
-; CHECK-NO-DSP: and.w r0, r1, r0, lsr #8
+; CHECK-DSP-LABEL: test4:
+; CHECK-DSP:       @ %bb.0:
+; CHECK-DSP-NEXT:    uxtb16 r0, r0, ror #8
+; CHECK-DSP-NEXT:    bx lr
+;
+; CHECK-NO-DSP-LABEL: test4:
+; CHECK-NO-DSP:       @ %bb.0:
+; CHECK-NO-DSP-NEXT:    mov.w r1, #16711935
+; CHECK-NO-DSP-NEXT:    and.w r0, r1, r0, lsr #8
+; CHECK-NO-DSP-NEXT:    bx lr
 	%tmp1 = lshr i32 %x, 8		; <i32> [#uses=1]
 	%tmp6 = and i32 %tmp1, 16711935		; <i32> [#uses=1]
 	ret i32 %tmp6
 }
 
 define i32 @test5(i32 %x) {
-; CHECK-LABEL: test5
-; CHECK-DSP: uxtb16  r0, r0, ror #8
-; CHECK-NO-DSP: mov.w r1, #16711935
-; CHECK-NO-DSP: and.w r0, r1, r0, lsr #8
+; CHECK-DSP-LABEL: test5:
+; CHECK-DSP:       @ %bb.0:
+; CHECK-DSP-NEXT:    uxtb16 r0, r0, ror #8
+; CHECK-DSP-NEXT:    bx lr
+;
+; CHECK-NO-DSP-LABEL: test5:
+; CHECK-NO-DSP:       @ %bb.0:
+; CHECK-NO-DSP-NEXT:    mov.w r1, #16711935
+; CHECK-NO-DSP-NEXT:    and.w r0, r1, r0, lsr #8
+; CHECK-NO-DSP-NEXT:    bx lr
 	%tmp1 = lshr i32 %x, 8		; <i32> [#uses=1]
 	%tmp2 = and i32 %tmp1, 16711935		; <i32> [#uses=1]
 	ret i32 %tmp2
 }
 
 define i32 @test6(i32 %x) {
-; CHECK-LABEL: test6
-; CHECK-DSP: uxtb16  r0, r0, ror #16
-; CHECK-NO-DSP: mov.w r1, #16711935
-; CHECK-NO-DSP: and.w r0, r1, r0, ror #16
+; CHECK-DSP-LABEL: test6:
+; CHECK-DSP:       @ %bb.0:
+; CHECK-DSP-NEXT:    uxtb16 r0, r0, ror #16
+; CHECK-DSP-NEXT:    bx lr
+;
+; CHECK-NO-DSP-LABEL: test6:
+; CHECK-NO-DSP:       @ %bb.0:
+; CHECK-NO-DSP-NEXT:    mov.w r1, #16711935
+; CHECK-NO-DSP-NEXT:    and.w r0, r1, r0, ror #16
+; CHECK-NO-DSP-NEXT:    bx lr
 	%tmp1 = lshr i32 %x, 16		; <i32> [#uses=1]
 	%tmp2 = and i32 %tmp1, 255		; <i32> [#uses=1]
 	%tmp4 = shl i32 %x, 16		; <i32> [#uses=1]
@@ -67,10 +104,16 @@ define i32 @test6(i32 %x) {
 }
 
 define i32 @test7(i32 %x) {
-; CHECK-LABEL: test7
-; CHECK-DSP: uxtb16  r0, r0, ror #16
-; CHECK-NO-DSP: mov.w r1, #16711935
-; CHECK-NO-DSP: and.w r0, r1, r0, ror #16
+; CHECK-DSP-LABEL: test7:
+; CHECK-DSP:       @ %bb.0:
+; CHECK-DSP-NEXT:    uxtb16 r0, r0, ror #16
+; CHECK-DSP-NEXT:    bx lr
+;
+; CHECK-NO-DSP-LABEL: test7:
+; CHECK-NO-DSP:       @ %bb.0:
+; CHECK-NO-DSP-NEXT:    mov.w r1, #16711935
+; CHECK-NO-DSP-NEXT:    and.w r0, r1, r0, ror #16
+; CHECK-NO-DSP-NEXT:    bx lr
 	%tmp1 = lshr i32 %x, 16		; <i32> [#uses=1]
 	%tmp2 = and i32 %tmp1, 255		; <i32> [#uses=1]
 	%tmp4 = shl i32 %x, 16		; <i32> [#uses=1]
@@ -80,10 +123,16 @@ define i32 @test7(i32 %x) {
 }
 
 define i32 @test8(i32 %x) {
-; CHECK-LABEL: test8
-; CHECK-DSP: uxtb16  r0, r0, ror #24
-; CHECK-NO-DSP: mov.w r1, #16711935
-; CHECK-NO-DSP: and.w r0, r1, r0, ror #24
+; CHECK-DSP-LABEL: test8:
+; CHECK-DSP:       @ %bb.0:
+; CHECK-DSP-NEXT:    uxtb16 r0, r0, ror #24
+; CHECK-DSP-NEXT:    bx lr
+;
+; CHECK-NO-DSP-LABEL: test8:
+; CHECK-NO-DSP:       @ %bb.0:
+; CHECK-NO-DSP-NEXT:    mov.w r1, #16711935
+; CHECK-NO-DSP-NEXT:    and.w r0, r1, r0, ror #24
+; CHECK-NO-DSP-NEXT:    bx lr
 	%tmp1 = shl i32 %x, 8		; <i32> [#uses=1]
 	%tmp2 = and i32 %tmp1, 16711680		; <i32> [#uses=1]
 	%tmp5 = lshr i32 %x, 24		; <i32> [#uses=1]
@@ -92,10 +141,16 @@ define i32 @test8(i32 %x) {
 }
 
 define i32 @test9(i32 %x) {
-; CHECK-LABEL: test9
-; CHECK-DSP: uxtb16  r0, r0, ror #24
-; CHECK-NO-DSP: mov.w r1, #16711935
-; CHECK-NO-DSP: and.w r0, r1, r0, ror #24
+; CHECK-DSP-LABEL: test9:
+; CHECK-DSP:       @ %bb.0:
+; CHECK-DSP-NEXT:    uxtb16 r0, r0, ror #24
+; CHECK-DSP-NEXT:    bx lr
+;
+; CHECK-NO-DSP-LABEL: test9:
+; CHECK-NO-DSP:       @ %bb.0:
+; CHECK-NO-DSP-NEXT:    mov.w r1, #16711935
+; CHECK-NO-DSP-NEXT:    and.w r0, r1, r0, ror #24
+; CHECK-NO-DSP-NEXT:    bx lr
 	%tmp1 = lshr i32 %x, 24		; <i32> [#uses=1]
 	%tmp4 = shl i32 %x, 8		; <i32> [#uses=1]
 	%tmp5 = and i32 %tmp4, 16711680		; <i32> [#uses=1]
@@ -104,18 +159,24 @@ define i32 @test9(i32 %x) {
 }
 
 define i32 @test10(i32 %p0) {
-; CHECK-LABEL: test10
-; CHECK-DSP: mov.w r1, #16253176
-; CHECK-DSP: and.w r0, r1, r0, lsr #7
-; CHECK-DSP: lsrs  r1, r0, #5
-; CHECK-DSP: uxtb16  r1, r1
-; CHECk-DSP: adds r0, r1
+; CHECK-DSP-LABEL: test10:
+; CHECK-DSP:       @ %bb.0:
+; CHECK-DSP-NEXT:    mov.w r1, #16253176
+; CHECK-DSP-NEXT:    and.w r0, r1, r0, lsr #7
+; CHECK-DSP-NEXT:    lsrs r1, r0, #5
+; CHECK-DSP-NEXT:    uxtb16 r1, r1
+; CHECK-DSP-NEXT:    add r0, r1
+; CHECK-DSP-NEXT:    bx lr
+;
+; CHECK-NO-DSP-LABEL: test10:
+; CHECK-NO-DSP:       @ %bb.0:
+; CHECK-NO-DSP-NEXT:    mov.w r1, #16253176
+; CHECK-NO-DSP-NEXT:    and.w r0, r1, r0, lsr #7
+; CHECK-NO-DSP-NEXT:    mov.w r1, #458759
+; CHECK-NO-DSP-NEXT:    and.w r1, r1, r0, lsr #5
+; CHECK-NO-DSP-NEXT:    add r0, r1
+; CHECK-NO-DSP-NEXT:    bx lr
 
-; CHECK-NO-DSP: mov.w r1, #16253176
-; CHECK-NO-DSP: and.w r0, r1, r0, lsr #7
-; CHECK-NO-DSP: mov.w r1, #458759
-; CHECK-NO-DSP: and.w r1, r1, r0, lsr #5
-; CHECK-NO-DSP: add r0, r1
 	%tmp1 = lshr i32 %p0, 7		; <i32> [#uses=1]
 	%tmp2 = and i32 %tmp1, 16253176		; <i32> [#uses=2]
 	%tmp4 = lshr i32 %tmp2, 5		; <i32> [#uses=1]
diff --git a/llvm/test/CodeGen/Thumb2/vqabs.ll b/llvm/test/CodeGen/Thumb2/vqabs.ll
new file mode 100644
index 000000000..7a366ea5e
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/vqabs.ll
@@ -0,0 +1,50 @@
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve %s -o - | FileCheck %s
+
+define arm_aapcs_vfpcc <16 x i8> @vqabs_test16(<16 x i8> %A) nounwind {
+; CHECK-LABEL: vqabs_test16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqabs.s8 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+
+  %0 = icmp sgt <16 x i8> %A, zeroinitializer
+  %1 = icmp eq <16 x i8> %A, <i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128>
+  %2 = sub nsw <16 x i8> zeroinitializer, %A
+  %3 = select <16 x i1> %1, <16 x i8> <i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127>, <16 x i8> %2
+  %4 = select <16 x i1> %0, <16 x i8> %A, <16 x i8> %3
+  
+  ret <16 x i8> %4
+}
+
+define arm_aapcs_vfpcc <8 x i16> @vqabs_test8(<8 x i16> %A) nounwind {
+; CHECK-LABEL: vqabs_test8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqabs.s16 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+
+  %0 = icmp sgt <8 x i16> %A, zeroinitializer
+  %1 = icmp eq <8 x i16> %A, <i16 -32768, i16 -32768, i16 -32768, i16 -32768, i16 -32768, i16 -32768, i16 -32768, i16 -32768>
+  %2 = sub nsw <8 x i16> zeroinitializer, %A
+  %3 = select <8 x i1> %1, <8 x i16> <i16 32767, i16 32767, i16 32767, i16 32767, i16 32767, i16 32767, i16 32767, i16 32767>, <8 x i16> %2
+  %4 = select <8 x i1> %0, <8 x i16> %A, <8 x i16> %3
+  
+  ret <8 x i16> %4
+}
+
+define arm_aapcs_vfpcc <4 x i32> @vqabs_test4(<4 x i32> %A) nounwind {
+; CHECK-LABEL: vqabs_test4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqabs.s32 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+
+  %0 = icmp sgt <4 x i32> %A, zeroinitializer
+  %1 = icmp eq <4 x i32> %A, <i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 -2147483648>
+  %2 = sub nsw <4 x i32> zeroinitializer, %A
+  %3 = select <4 x i1> %1, <4 x i32> <i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647>, <4 x i32> %2
+  %4 = select <4 x i1> %0, <4 x i32> %A, <4 x i32> %3
+  
+  ret <4 x i32> %4
+}
+
diff --git a/llvm/test/CodeGen/Thumb2/vqneg.ll b/llvm/test/CodeGen/Thumb2/vqneg.ll
new file mode 100644
index 000000000..b92a7ab27
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/vqneg.ll
@@ -0,0 +1,44 @@
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve %s -o - | FileCheck %s
+
+define arm_aapcs_vfpcc <16 x i8> @vqneg_test16(<16 x i8> %A) nounwind {
+; CHECK-LABEL: vqneg_test16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqneg.s8 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+
+  %0 = icmp eq <16 x i8> %A, <i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128>
+  %1 = sub nsw <16 x i8> zeroinitializer, %A
+  %2 = select <16 x i1> %0, <16 x i8> <i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127>, <16 x i8> %1
+
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @vqneg_test8(<8 x i16> %A) nounwind {
+; CHECK-LABEL: vqneg_test8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqneg.s16 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+
+  %0 = icmp eq <8 x i16> %A, <i16 -32768, i16 -32768, i16 -32768, i16 -32768, i16 -32768, i16 -32768, i16 -32768, i16 -32768>
+  %1 = sub nsw <8 x i16> zeroinitializer, %A
+  %2 = select <8 x i1> %0, <8 x i16> <i16 32767, i16 32767, i16 32767, i16 32767, i16 32767, i16 32767, i16 32767, i16 32767>, <8 x i16> %1
+
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @vqneg_test4(<4 x i32> %A) nounwind {
+; CHECK-LABEL: vqneg_test4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vqneg.s32 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+
+  %0 = icmp eq <4 x i32> %A, <i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 -2147483648>
+  %1 = sub nsw <4 x i32> zeroinitializer, %A
+  %2 = select <4 x i1> %0, <4 x i32> <i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647>, <4 x i32> %1
+
+  ret <4 x i32> %2
+}
+
diff --git a/llvm/test/CodeGen/VE/simple_prologue_epilogue.ll b/llvm/test/CodeGen/VE/simple_prologue_epilogue.ll
new file mode 100644
index 000000000..8470e763f
--- /dev/null
+++ b/llvm/test/CodeGen/VE/simple_prologue_epilogue.ll
@@ -0,0 +1,32 @@
+; RUN: llc < %s -mtriple=ve-unknown-unknown | FileCheck %s
+
+define void @func() {
+; CHECK-LABEL: func:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:  st %s9, (,%s11)
+; CHECK-NEXT:  st %s10, 8(,%s11)
+; CHECK-NEXT:  st %s15, 24(,%s11)
+; CHECK-NEXT:  st %s16, 32(,%s11)
+; CHECK-NEXT:  or %s9, 0, %s11
+; CHECK-NEXT:  lea %s13, -176
+; CHECK-NEXT:  and %s13, %s13, (32)0
+; CHECK-NEXT:  lea.sl %s11, -1(%s11, %s13)
+; CHECK-NEXT:  brge.l %s11, %s8, .LBB0_2
+; CHECK-NEXT:  # %bb.1:
+; CHECK-NEXT:  ld %s61, 24(,%s14)
+; CHECK-NEXT:  or %s62, 0, %s0
+; CHECK-NEXT:  lea %s63, 315
+; CHECK-NEXT:  shm.l %s63, (%s61)
+; CHECK-NEXT:  shm.l %s8, 8(%s61)
+; CHECK-NEXT:  shm.l %s11, 16(%s61)
+; CHECK-NEXT:  monc
+; CHECK-NEXT:  or %s0, 0, %s62
+; CHECK-NEXT: .LBB0_2:
+; CHECK-NEXT:  or %s11, 0, %s9
+; CHECK-NEXT:  ld %s16, 32(,%s11)
+; CHECK-NEXT:  ld %s15, 24(,%s11)
+; CHECK-NEXT:  ld %s10, 8(,%s11)
+; CHECK-NEXT:  ld %s9, (,%s11)
+; CHECK-NEXT:  b.l (,%lr)
+  ret void
+}
diff --git a/llvm/test/CodeGen/VE/target_support.ll b/llvm/test/CodeGen/VE/target_support.ll
new file mode 100644
index 000000000..336d9cd36
--- /dev/null
+++ b/llvm/test/CodeGen/VE/target_support.ll
@@ -0,0 +1,2 @@
+; RUN: llc --version | FileCheck %s
+; CHECK:    ve     - VE
diff --git a/llvm/test/CodeGen/WebAssembly/atomic-fence.ll b/llvm/test/CodeGen/WebAssembly/atomic-fence.ll
index 22084e48e..43a1a66f9 100644
--- a/llvm/test/CodeGen/WebAssembly/atomic-fence.ll
+++ b/llvm/test/CodeGen/WebAssembly/atomic-fence.ll
@@ -1,19 +1,12 @@
 ; RUN: llc < %s | FileCheck %s --check-prefix NOATOMIC
-; RUN: not llc < %s -mtriple=wasm32-unknown-unknown -mattr=+atomics,+sign-ext 2>&1 | FileCheck %s --check-prefixes NOEMSCRIPTEN
-; RUN: not llc < %s -mtriple=wasm32-unknown-wasi -mattr=+atomics,+sign-ext 2>&1 | FileCheck %s --check-prefixes NOEMSCRIPTEN
-; RUN: llc < %s -mtriple=wasm32-unknown-emscripten -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=+atomics,+sign-ext | FileCheck %s
+; RUN: llc < %s -asm-verbose=false -wasm-disable-explicit-locals -wasm-keep-registers -mattr=+atomics | FileCheck %s
 
 target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
 target triple = "wasm32-unknown-unknown"
 
-; NOEMSCRIPTEN: LLVM ERROR: ATOMIC_FENCE is not yet supported in non-emscripten OSes
-
-; A multithread fence turns into 'global.get $__stack_pointer' followed by an
-; idempotent atomicrmw instruction.
+; A multithread fence is lowered to an atomic.fence instruction.
 ; CHECK-LABEL: multithread_fence:
-; CHECK:      global.get  $push[[SP:[0-9]+]]=, __stack_pointer
-; CHECK-NEXT: i32.const $push[[ZERO:[0-9]+]]=, 0
-; CHECK-NEXT: i32.atomic.rmw.or  $drop=, 0($pop[[SP]]), $pop[[ZERO]]
+; CHECK:  atomic.fence
 ; NOATOMIC-NOT: i32.atomic.rmw.or
 define void @multithread_fence() {
   fence seq_cst
@@ -23,10 +16,9 @@ define void @multithread_fence() {
 ; Fences with weaker memory orderings than seq_cst should be treated the same
 ; because atomic memory access in wasm are sequentially consistent.
 ; CHECK-LABEL: multithread_weak_fence:
-; CHECK:  global.get  $push{{.+}}=, __stack_pointer
-; CHECK:  i32.atomic.rmw.or
-; CHECK:  i32.atomic.rmw.or
-; CHECK:  i32.atomic.rmw.or
+; CHECK:       atomic.fence
+; CHECK-NEXT:  atomic.fence
+; CHECK-NEXT:  atomic.fence
 define void @multithread_weak_fence() {
   fence acquire
   fence release
@@ -37,7 +29,8 @@ define void @multithread_weak_fence() {
 ; A singlethread fence becomes compiler_fence instruction, a pseudo instruction
 ; that acts as a compiler barrier. The barrier should not be emitted to .s file.
 ; CHECK-LABEL: singlethread_fence:
-; CHECK-NOT:  compiler_fence
+; CHECK-NOT: compiler_fence
+; CHECK-NOT: atomic_fence
 define void @singlethread_fence() {
   fence syncscope("singlethread") seq_cst
   fence syncscope("singlethread") acquire
diff --git a/llvm/test/CodeGen/WebAssembly/bulk-memory.ll b/llvm/test/CodeGen/WebAssembly/bulk-memory.ll
index 657451108..89ea7e8f8 100644
--- a/llvm/test/CodeGen/WebAssembly/bulk-memory.ll
+++ b/llvm/test/CodeGen/WebAssembly/bulk-memory.ll
@@ -142,8 +142,8 @@ define void @memset_1024(i8* %dest, i8 %val) {
 }
 
 ; The following tests check that frame index elimination works for
-; bulk memory instructions. The stack pointer is bumped by 16 instead
-; of 10 because the stack pointer in WebAssembly is currently always
+; bulk memory instructions. The stack pointer is bumped by 112 instead
+; of 100 because the stack pointer in WebAssembly is currently always
 ; 16-byte aligned, even in leaf functions, although it is not written
 ; back to the global in this case.
 
@@ -156,17 +156,17 @@ define void @memset_1024(i8* %dest, i8 %val) {
 ; NO-BULK-MEM-NOT: memory.copy
 ; BULK-MEM-NEXT: .functype memcpy_alloca_src (i32) -> ()
 ; BULK-MEM-NEXT: global.get $push[[L0:[0-9]+]]=, __stack_pointer
-; BULK-MEM-NEXT: i32.const $push[[L1:[0-9]+]]=, 16
+; BULK-MEM-NEXT: i32.const $push[[L1:[0-9]+]]=, 112
 ; BULK-MEM-NEXT: i32.sub $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]
-; BULK-MEM-NEXT: i32.const $push[[L3:[0-9]+]]=, 6
+; BULK-MEM-NEXT: i32.const $push[[L3:[0-9]+]]=, 12
 ; BULK-MEM-NEXT: i32.add $push[[L4:[0-9]+]]=, $pop[[L2]], $pop[[L3]]
-; BULK-MEM-NEXT: i32.const $push[[L5:[0-9]+]]=, 10
+; BULK-MEM-NEXT: i32.const $push[[L5:[0-9]+]]=, 100
 ; BULK-MEM-NEXT: memory.copy 0, 0, $0, $pop[[L4]], $pop[[L5]]
 ; BULK-MEM-NEXT: return
 define void @memcpy_alloca_src(i8* %dst) {
-  %a = alloca [10 x i8]
-  %p = bitcast [10 x i8]* %a to i8*
-  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %dst, i8* %p, i32 10, i1 false)
+  %a = alloca [100 x i8]
+  %p = bitcast [100 x i8]* %a to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %dst, i8* %p, i32 100, i1 false)
   ret void
 }
 
@@ -174,17 +174,17 @@ define void @memcpy_alloca_src(i8* %dst) {
 ; NO-BULK-MEM-NOT: memory.copy
 ; BULK-MEM-NEXT: .functype memcpy_alloca_dst (i32) -> ()
 ; BULK-MEM-NEXT: global.get $push[[L0:[0-9]+]]=, __stack_pointer
-; BULK-MEM-NEXT: i32.const $push[[L1:[0-9]+]]=, 16
+; BULK-MEM-NEXT: i32.const $push[[L1:[0-9]+]]=, 112
 ; BULK-MEM-NEXT: i32.sub $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]
-; BULK-MEM-NEXT: i32.const $push[[L3:[0-9]+]]=, 6
+; BULK-MEM-NEXT: i32.const $push[[L3:[0-9]+]]=, 12
 ; BULK-MEM-NEXT: i32.add $push[[L4:[0-9]+]]=, $pop[[L2]], $pop[[L3]]
-; BULK-MEM-NEXT: i32.const $push[[L5:[0-9]+]]=, 10
+; BULK-MEM-NEXT: i32.const $push[[L5:[0-9]+]]=, 100
 ; BULK-MEM-NEXT: memory.copy 0, 0, $pop[[L4]], $0, $pop[[L5]]
 ; BULK-MEM-NEXT: return
 define void @memcpy_alloca_dst(i8* %src) {
-  %a = alloca [10 x i8]
-  %p = bitcast [10 x i8]* %a to i8*
-  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %p, i8* %src, i32 10, i1 false)
+  %a = alloca [100 x i8]
+  %p = bitcast [100 x i8]* %a to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %p, i8* %src, i32 100, i1 false)
   ret void
 }
 
@@ -192,16 +192,16 @@ define void @memcpy_alloca_dst(i8* %src) {
 ; NO-BULK-MEM-NOT: memory.fill
 ; BULK-MEM-NEXT: .functype memset_alloca (i32) -> ()
 ; BULK-MEM-NEXT: global.get $push[[L0:[0-9]+]]=, __stack_pointer
-; BULK-MEM-NEXT: i32.const $push[[L1:[0-9]+]]=, 16
+; BULK-MEM-NEXT: i32.const $push[[L1:[0-9]+]]=, 112
 ; BULK-MEM-NEXT: i32.sub $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]
-; BULK-MEM-NEXT: i32.const $push[[L3:[0-9]+]]=, 6
+; BULK-MEM-NEXT: i32.const $push[[L3:[0-9]+]]=, 12
 ; BULK-MEM-NEXT: i32.add $push[[L4:[0-9]+]]=, $pop[[L2]], $pop[[L3]]
-; BULK-MEM-NEXT: i32.const $push[[L5:[0-9]+]]=, 10
+; BULK-MEM-NEXT: i32.const $push[[L5:[0-9]+]]=, 100
 ; BULK-MEM-NEXT: memory.fill 0, $pop[[L4]], $0, $pop[[L5]]
 ; BULK-MEM-NEXT: return
 define void @memset_alloca(i8 %val) {
-  %a = alloca [10 x i8]
-  %p = bitcast [10 x i8]* %a to i8*
-  call void @llvm.memset.p0i8.i32(i8* %p, i8 %val, i32 10, i1 false)
+  %a = alloca [100 x i8]
+  %p = bitcast [100 x i8]* %a to i8*
+  call void @llvm.memset.p0i8.i32(i8* %p, i8 %val, i32 100, i1 false)
   ret void
 }
diff --git a/llvm/test/CodeGen/WebAssembly/cfg-stackify-eh.ll b/llvm/test/CodeGen/WebAssembly/cfg-stackify-eh.ll
index 9cb5a057e..858a3b7a9 100644
--- a/llvm/test/CodeGen/WebAssembly/cfg-stackify-eh.ll
+++ b/llvm/test/CodeGen/WebAssembly/cfg-stackify-eh.ll
@@ -1,6 +1,8 @@
+; REQUIRES: asserts
 ; RUN: llc < %s -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -disable-block-placement -verify-machineinstrs -fast-isel=false -machine-sink-split-probability-threshold=0 -cgp-freq-ratio-to-skip-merge=1000 -exception-model=wasm -mattr=+exception-handling | FileCheck %s
 ; RUN: llc < %s -O0 -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -verify-machineinstrs -exception-model=wasm -mattr=+exception-handling | FileCheck %s --check-prefix=NOOPT
 ; RUN: llc < %s -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -disable-block-placement -verify-machineinstrs -fast-isel=false -machine-sink-split-probability-threshold=0 -cgp-freq-ratio-to-skip-merge=1000 -exception-model=wasm -mattr=+exception-handling -wasm-disable-ehpad-sort | FileCheck %s --check-prefix=NOSORT
+; RUN: llc < %s -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -disable-block-placement -verify-machineinstrs -fast-isel=false -machine-sink-split-probability-threshold=0 -cgp-freq-ratio-to-skip-merge=1000 -exception-model=wasm -mattr=+exception-handling -wasm-disable-ehpad-sort -stats 2>&1 | FileCheck %s --check-prefix=NOSORT-STAT
 
 target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
 target triple = "wasm32-unknown-unknown"
@@ -436,13 +438,17 @@ try.cont:                                         ; preds = %catch.start1, %catc
 ; the right destination (label4), from which we rethrow the exception to the
 ; caller.
 
+; And the return value of 'baz' should NOT be stackified because the BB is split
+; during fixing unwind mismatches.
+
 ; NOSORT-LABEL: test6
 ; NOSORT:   try
 ; NOSORT:     call      foo
 ; --- Nested try/catch/end_try starts
 ; NOSORT:     try
 ; NOSORT:       call      bar
-; NOSORT:       call      bar
+; NOSORT:       i32.call  ${{[0-9]+}}=, baz
+; NOSORT-NOT:   i32.call  $push{{.*}}=, baz
 ; NOSORT:     catch     $[[REG:[0-9]+]]=
 ; NOSORT:       br        1                            # 1: down to label35
 ; NOSORT:     end_try
@@ -460,7 +466,8 @@ bb0:
 
 bb1:                                              ; preds = %bb0
   call void @bar()
-  call void @bar()
+  %call = call i32 @baz()
+  call void @nothrow(i32 %call) #0
   ret void
 
 catch.dispatch0:                                  ; preds = %bb0
@@ -616,8 +623,124 @@ try.cont:                                         ; preds = %catch.start1, %catc
   ret void
 }
 
+; In CFGSort, EH pads should be sorted as soon as it is available and
+; 'Preferred' queue and should NOT be entered into 'Ready' queue unless we are
+; in the middle of sorting another region that does not contain the EH pad. In
+; this example, 'catch.start' should be sorted right after 'if.then' is sorted
+; (before 'cont' is sorted) and there should not be any unwind destination
+; mismatches in CFGStackify.
+
+; NOOPT: block
+; NOOPT:   try
+; NOOPT:     call      foo
+; NOOPT:   catch
+; NOOPT:   end_try
+; NOOPT:   call      foo
+; NOOPT: end_block
+; NOOPT: return
+define void @test9(i32 %arg) personality i8* bitcast (i32 (...)* @__gxx_wasm_personality_v0 to i8*) {
+entry:
+  %tobool = icmp ne i32 %arg, 0
+  br i1 %tobool, label %if.then, label %if.end
+
+catch.dispatch:                                   ; preds = %if.then
+  %0 = catchswitch within none [label %catch.start] unwind to caller
+
+catch.start:                                      ; preds = %catch.dispatch
+  %1 = catchpad within %0 [i8* null]
+  %2 = call i8* @llvm.wasm.get.exception(token %1)
+  %3 = call i32 @llvm.wasm.get.ehselector(token %1)
+  %4 = call i8* @__cxa_begin_catch(i8* %2) [ "funclet"(token %1) ]
+  call void @__cxa_end_catch() [ "funclet"(token %1) ]
+  catchret from %1 to label %if.end
+
+if.then:                                          ; preds = %entry
+  invoke void @foo()
+          to label %cont unwind label %catch.dispatch
+
+cont:                                             ; preds = %if.then
+  call void @foo()
+  br label %if.end
+
+if.end:                                           ; preds = %cont, %catch.start, %entry
+  ret void
+}
+
+%class.Object = type { i8 }
+
+; Intrinsics like memcpy, memmove, and memset don't throw and are lowered into
+; calls to external symbols (not global addresses) in instruction selection,
+; which will be eventually lowered to library function calls.
+; Because this test runs with -wasm-disable-ehpad-sort, these library calls in
+; invoke.cont BB fall within try~end_try, but they shouldn't cause crashes or
+; unwinding destination mismatches in CFGStackify.
+
+; NOSORT-LABEL: test10
+; NOSORT: try
+; NOSORT:   call  foo
+; NOSORT:   i32.call {{.*}} memcpy
+; NOSORT:   i32.call {{.*}} memmove
+; NOSORT:   i32.call {{.*}} memset
+; NOSORT:   return
+; NOSORT: catch
+; NOSORT:   rethrow
+; NOSORT: end_try
+define void @test10(i8* %a, i8* %b) personality i8* bitcast (i32 (...)* @__gxx_wasm_personality_v0 to i8*) {
+entry:
+  %o = alloca %class.Object, align 1
+  invoke void @foo()
+          to label %invoke.cont unwind label %ehcleanup
+
+invoke.cont:                                      ; preds = %entry
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %a, i8* %b, i32 100, i1 false)
+  call void @llvm.memmove.p0i8.p0i8.i32(i8* %a, i8* %b, i32 100, i1 false)
+  call void @llvm.memset.p0i8.i32(i8* %a, i8 0, i32 100, i1 false)
+  %call = call %class.Object* @_ZN6ObjectD2Ev(%class.Object* %o) #1
+  ret void
+
+ehcleanup:                                        ; preds = %entry
+  %0 = cleanuppad within none []
+  %call2 = call %class.Object* @_ZN6ObjectD2Ev(%class.Object* %o) #1 [ "funclet"(token %0) ]
+  cleanupret from %0 unwind to caller
+}
+
+; Tests if 'try' marker is placed correctly. In this test, 'try' should be
+; placed before the call to 'nothrow_i32' and not between the call to
+; 'nothrow_i32' and 'fun', because the return value of 'nothrow_i32' is
+; stackified and pushed onto the stack to be consumed by the call to 'fun'.
+
+; CHECK-LABEL: test11
+; CHECK: try
+; CHECK: i32.call  $push{{.*}}=, nothrow_i32
+; CHECK: call      fun, $pop{{.*}}
+define void @test11() personality i8* bitcast (i32 (...)* @__gxx_wasm_personality_v0 to i8*) {
+entry:
+  %call = call i32 @nothrow_i32()
+  invoke void @fun(i32 %call)
+          to label %invoke.cont unwind label %terminate
+
+invoke.cont:                                      ; preds = %entry
+  ret void
+
+terminate:                                        ; preds = %entry
+  %0 = cleanuppad within none []
+  %1 = tail call i8* @llvm.wasm.get.exception(token %0)
+  call void @__clang_call_terminate(i8* %1) [ "funclet"(token %0) ]
+  unreachable
+}
+
+; Check if the unwind destination mismatch stats are correct
+; NOSORT-STAT: 11 wasm-cfg-stackify    - Number of EH pad unwind mismatches found
+
 declare void @foo()
 declare void @bar()
+declare i32 @baz()
+declare void @fun(i32)
+; Function Attrs: nounwind
+declare void @nothrow(i32) #0
+declare i32 @nothrow_i32() #0
+; Function Attrs: nounwind
+declare %class.Object* @_ZN6ObjectD2Ev(%class.Object* returned) #0
 declare i32 @__gxx_wasm_personality_v0(...)
 declare i8* @llvm.wasm.get.exception(token)
 declare i32 @llvm.wasm.get.ehselector(token)
@@ -627,3 +750,11 @@ declare i8* @__cxa_begin_catch(i8*)
 declare void @__cxa_end_catch()
 declare void @__clang_call_terminate(i8*)
 declare void @_ZSt9terminatev()
+; Function Attrs: nounwind
+declare void @llvm.memcpy.p0i8.p0i8.i32(i8* noalias nocapture writeonly, i8* noalias nocapture readonly, i32, i1 immarg) #0
+; Function Attrs: nounwind
+declare void @llvm.memmove.p0i8.p0i8.i32(i8* nocapture, i8* nocapture readonly, i32, i1 immarg) #0
+; Function Attrs: nounwind
+declare void @llvm.memset.p0i8.i32(i8* nocapture writeonly, i8, i32, i1 immarg) #0
+
+attributes #0 = { nounwind }
diff --git a/llvm/test/CodeGen/WebAssembly/conv-trap.ll b/llvm/test/CodeGen/WebAssembly/conv-trap.ll
index eff10ff9c..67519ee68 100644
--- a/llvm/test/CodeGen/WebAssembly/conv-trap.ll
+++ b/llvm/test/CodeGen/WebAssembly/conv-trap.ll
@@ -165,3 +165,83 @@ define i64 @i64_trunc_u_f64(double %x) {
   %a = fptoui double %x to i64
   ret i64 %a
 }
+
+; CHECK-LABEL: llvm_wasm_trunc_signed_i32_f32:
+; CHECK-NEXT: .functype llvm_wasm_trunc_signed_i32_f32 (f32) -> (i32)
+; CHECK-NEXT: i32.trunc_f32_s $push[[L0:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: return $pop[[L0]]{{$}}
+declare i32 @llvm.wasm.trunc.signed.i32.f32(float)
+define i32 @llvm_wasm_trunc_signed_i32_f32(float %f) {
+  %a = call i32 @llvm.wasm.trunc.signed.i32.f32(float %f)
+  ret i32 %a
+}
+
+; CHECK-LABEL: llvm_wasm_trunc_unsigned_i32_f32:
+; CHECK-NEXT: .functype llvm_wasm_trunc_unsigned_i32_f32 (f32) -> (i32)
+; CHECK-NEXT: i32.trunc_f32_u $push[[L0:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: return $pop[[L0]]{{$}}
+declare i32 @llvm.wasm.trunc.unsigned.i32.f32(float)
+define i32 @llvm_wasm_trunc_unsigned_i32_f32(float %f) {
+  %a = call i32 @llvm.wasm.trunc.unsigned.i32.f32(float %f)
+  ret i32 %a
+}
+
+; CHECK-LABEL: llvm_wasm_trunc_signed_i32_f64:
+; CHECK-NEXT: .functype llvm_wasm_trunc_signed_i32_f64 (f64) -> (i32)
+; CHECK-NEXT: i32.trunc_f64_s $push[[L0:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: return $pop[[L0]]{{$}}
+declare i32 @llvm.wasm.trunc.signed.i32.f64(double)
+define i32 @llvm_wasm_trunc_signed_i32_f64(double %f) {
+  %a = call i32 @llvm.wasm.trunc.signed.i32.f64(double %f)
+  ret i32 %a
+}
+
+; CHECK-LABEL: llvm_wasm_trunc_unsigned_i32_f64:
+; CHECK-NEXT: .functype llvm_wasm_trunc_unsigned_i32_f64 (f64) -> (i32)
+; CHECK-NEXT: i32.trunc_f64_u $push[[L0:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: return $pop[[L0]]{{$}}
+declare i32 @llvm.wasm.trunc.unsigned.i32.f64(double)
+define i32 @llvm_wasm_trunc_unsigned_i32_f64(double %f) {
+  %a = call i32 @llvm.wasm.trunc.unsigned.i32.f64(double %f)
+  ret i32 %a
+}
+
+; CHECK-LABEL: llvm_wasm_trunc_signed_i64_f32:
+; CHECK-NEXT: .functype llvm_wasm_trunc_signed_i64_f32 (f32) -> (i64)
+; CHECK-NEXT: i64.trunc_f32_s $push[[L0:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: return $pop[[L0]]{{$}}
+declare i64 @llvm.wasm.trunc.signed.i64.f32(float)
+define i64 @llvm_wasm_trunc_signed_i64_f32(float %f) {
+  %a = call i64 @llvm.wasm.trunc.signed.i64.f32(float %f)
+  ret i64 %a
+}
+
+; CHECK-LABEL: llvm_wasm_trunc_unsigned_i64_f32:
+; CHECK-NEXT: .functype llvm_wasm_trunc_unsigned_i64_f32 (f32) -> (i64)
+; CHECK-NEXT: i64.trunc_f32_u $push[[L0:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: return $pop[[L0]]{{$}}
+declare i64 @llvm.wasm.trunc.unsigned.i64.f32(float)
+define i64 @llvm_wasm_trunc_unsigned_i64_f32(float %f) {
+  %a = call i64 @llvm.wasm.trunc.unsigned.i64.f32(float %f)
+  ret i64 %a
+}
+
+; CHECK-LABEL: llvm_wasm_trunc_signed_i64_f64:
+; CHECK-NEXT: .functype llvm_wasm_trunc_signed_i64_f64 (f64) -> (i64)
+; CHECK-NEXT: i64.trunc_f64_s $push[[L0:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: return $pop[[L0]]{{$}}
+declare i64 @llvm.wasm.trunc.signed.i64.f64(double)
+define i64 @llvm_wasm_trunc_signed_i64_f64(double %f) {
+  %a = call i64 @llvm.wasm.trunc.signed.i64.f64(double %f)
+  ret i64 %a
+}
+
+; CHECK-LABEL: llvm_wasm_trunc_unsigned_i64_f64:
+; CHECK-NEXT: .functype llvm_wasm_trunc_unsigned_i64_f64 (f64) -> (i64)
+; CHECK-NEXT: i64.trunc_f64_u $push[[L0:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: return $pop[[L0]]{{$}}
+declare i64 @llvm.wasm.trunc.unsigned.i64.f64(double)
+define i64 @llvm_wasm_trunc_unsigned_i64_f64(double %f) {
+  %a = call i64 @llvm.wasm.trunc.unsigned.i64.f64(double %f)
+  ret i64 %a
+}
diff --git a/llvm/test/CodeGen/WebAssembly/cpus.ll b/llvm/test/CodeGen/WebAssembly/cpus.ll
index 8ede6cbb5..01964e9c8 100644
--- a/llvm/test/CodeGen/WebAssembly/cpus.ll
+++ b/llvm/test/CodeGen/WebAssembly/cpus.ll
@@ -1,16 +1,17 @@
 ; This tests that llc accepts all valid WebAssembly CPUs.
 
 ; RUN: llc < %s -asm-verbose=false -mtriple=wasm32-unknown-unknown -mcpu=mvp 2>&1 | FileCheck %s
-; RUN: llc < %s -asm-verbose=false -mtriple=wasm64-unknown-unknown-wasm -mcpu=mvp 2>&1 | FileCheck %s
+; RUN: not llc < %s -asm-verbose=false -mtriple=wasm64-unknown-unknown-wasm -mcpu=mvp 2>&1 | FileCheck %s --check-prefix=WASM64
 ; RUN: llc < %s -asm-verbose=false -mtriple=wasm32-unknown-unknown -mcpu=generic 2>&1 | FileCheck %s
-; RUN: llc < %s -asm-verbose=false -mtriple=wasm64-unknown-unknown-wasm -mcpu=generic 2>&1 | FileCheck %s
+; RUN: not llc < %s -asm-verbose=false -mtriple=wasm64-unknown-unknown-wasm -mcpu=generic 2>&1 | FileCheck %s  --check-prefix=WASM64
 ; RUN: llc < %s -asm-verbose=false -mtriple=wasm32-unknown-unknown -mcpu=bleeding-edge 2>&1 | FileCheck %s
-; RUN: llc < %s -asm-verbose=false -mtriple=wasm64-unknown-unknown-wasm -mcpu=bleeding-edge 2>&1 | FileCheck %s
+; RUN: not llc < %s -asm-verbose=false -mtriple=wasm64-unknown-unknown-wasm -mcpu=bleeding-edge 2>&1 | FileCheck %s  --check-prefix=WASM64
 ; RUN: llc < %s -asm-verbose=false -mtriple=wasm32-unknown-unknown -mcpu=invalidcpu 2>&1 | FileCheck %s --check-prefix=INVALID
-; RUN: llc < %s -asm-verbose=false -mtriple=wasm64-unknown-unknown-wasm -mcpu=invalidcpu 2>&1 | FileCheck %s --check-prefix=INVALID
+; RUN: not llc < %s -asm-verbose=false -mtriple=wasm64-unknown-unknown-wasm -mcpu=invalidcpu 2>&1 | FileCheck %s --check-prefix=WASM64
 
 ; CHECK-NOT: is not a recognized processor for this target
 ; INVALID: {{.+}} is not a recognized processor for this target
+; WASM64: 64-bit WebAssembly (wasm64) is not currently supported
 
 define i32 @f(i32 %i_like_the_web) {
   ret i32 %i_like_the_web
diff --git a/llvm/test/CodeGen/WebAssembly/exception.ll b/llvm/test/CodeGen/WebAssembly/exception.ll
index fc61f4099..af470b27e 100644
--- a/llvm/test/CodeGen/WebAssembly/exception.ll
+++ b/llvm/test/CodeGen/WebAssembly/exception.ll
@@ -138,7 +138,8 @@ ehcleanup:                                        ; preds = %entry
 ; CHECK:     catch
 ; CHECK:       block     i32
 ; CHECK:         br_on_exn   0, __cpp_exception
-; CHECK:         call      __clang_call_terminate, 0
+; CHECK:         i32.const  ${{.*}}=, 0
+; CHECK:         call      __clang_call_terminate
 ; CHECK:         unreachable
 ; CHECK:       end_block
 ; CHECK:       call      __clang_call_terminate
diff --git a/llvm/test/CodeGen/WebAssembly/export-name.ll b/llvm/test/CodeGen/WebAssembly/export-name.ll
new file mode 100644
index 000000000..d1d4c2115
--- /dev/null
+++ b/llvm/test/CodeGen/WebAssembly/export-name.ll
@@ -0,0 +1,17 @@
+; RUN: llc < %s -asm-verbose=false -wasm-keep-registers | FileCheck %s
+
+target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
+target triple = "wasm32-unknown-unknown"
+
+define void @test() #0 {
+  ret void
+}
+
+declare void @test2() #1
+
+
+attributes #0 = { "wasm-export-name"="foo" }
+attributes #1 = { "wasm-export-name"="bar" }
+
+; CHECK: .export_name test, foo
+; CHECK: .export_name test2, bar
diff --git a/llvm/test/CodeGen/WebAssembly/function-bitcasts.ll b/llvm/test/CodeGen/WebAssembly/function-bitcasts.ll
index eff69ebfe..91bb1b6db 100644
--- a/llvm/test/CodeGen/WebAssembly/function-bitcasts.ll
+++ b/llvm/test/CodeGen/WebAssembly/function-bitcasts.ll
@@ -5,7 +5,11 @@
 target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
 target triple = "wasm32-unknown-unknown"
 
-declare void @has_i32_arg(i32)
+define void @has_i32_arg(i32) {
+entry:
+  ret void
+}
+
 declare void @has_struct_arg({i32})
 declare i32 @has_i32_ret()
 declare void @vararg(...)
@@ -54,6 +58,19 @@ entry:
   ret void
 }
 
+; Calling aliases should also generate a wrapper
+
+@alias_i32_arg = weak hidden alias void (i32), void (i32)* @has_i32_arg
+
+; CHECK-LABEL: test_alias:
+; CHECK: call    .Lhas_i32_arg_bitcast.2
+define void @test_alias() {
+entry:
+  call void bitcast (void (i32)* @alias_i32_arg to void ()*)()
+  ret void
+}
+
+
 ; CHECK-LABEL: test_structs:
 ; CHECK: call     .Lhas_i32_arg_bitcast.1, $pop{{[0-9]+}}, $pop{{[0-9]+$}}
 ; CHECK: call     .Lhas_i32_arg_bitcast, $0, $pop2
diff --git a/llvm/test/CodeGen/WebAssembly/indirect-import.ll b/llvm/test/CodeGen/WebAssembly/indirect-import.ll
index eea1ce822..2c7f17e48 100644
--- a/llvm/test/CodeGen/WebAssembly/indirect-import.ll
+++ b/llvm/test/CodeGen/WebAssembly/indirect-import.ll
@@ -66,8 +66,8 @@ declare void @extern_sret(%struct.big* sret) #1
 
 declare i128 @extern_i128ret(i64) #1
 
-attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="generic" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="generic" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="generic" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="generic" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 
 ; CHECK: .functype extern_fd (f64) -> (f32)
diff --git a/llvm/test/CodeGen/WebAssembly/llround-conv-i32.ll b/llvm/test/CodeGen/WebAssembly/llround-conv-i32.ll
index eb2768cb9..5c84b33de 100644
--- a/llvm/test/CodeGen/WebAssembly/llround-conv-i32.ll
+++ b/llvm/test/CodeGen/WebAssembly/llround-conv-i32.ll
@@ -7,7 +7,7 @@ define i64 @testmsxs_builtin(float %x) {
 ; CHECK-NEXT:  # %bb.0: # %entry
 ; CHECK-NEXT:    local.get 0
 ; CHECK-NEXT:    i64.call llroundf
-; CHECK-NEXT:    # fallthrough-return-value
+; CHECK-NEXT:    # fallthrough-return
 ; CHECK-NEXT:    end_function
 entry:
   %0 = tail call i64 @llvm.llround.f32(float %x)
@@ -20,7 +20,7 @@ define i64 @testmsxd_builtin(double %x) {
 ; CHECK-NEXT:  # %bb.0: # %entry
 ; CHECK-NEXT:    local.get 0
 ; CHECK-NEXT:    i64.call llround
-; CHECK-NEXT:    # fallthrough-return-value
+; CHECK-NEXT:    # fallthrough-return
 ; CHECK-NEXT:    end_function
 entry:
   %0 = tail call i64 @llvm.llround.f64(double %x)
diff --git a/llvm/test/CodeGen/WebAssembly/lower-em-exceptions-resume-only.ll b/llvm/test/CodeGen/WebAssembly/lower-em-exceptions-resume-only.ll
new file mode 100644
index 000000000..4360d0219
--- /dev/null
+++ b/llvm/test/CodeGen/WebAssembly/lower-em-exceptions-resume-only.ll
@@ -0,0 +1,17 @@
+; RUN: opt < %s -wasm-lower-em-ehsjlj -S | FileCheck %s
+
+target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
+target triple = "wasm32-unknown-unknown"
+
+; Checks if a module that only contains a resume but not an invoke works
+; correctly and does not crash.
+; CHECK-LABEL: @resume_only
+; CHECK: call void @__resumeException
+define void @resume_only() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+entry:
+  %val0 = insertvalue { i8*, i32 } undef, i8* null, 0
+  %val1 = insertvalue { i8*, i32} %val0, i32 0, 1
+  resume { i8*, i32 } %val1
+}
+
+declare i32 @__gxx_personality_v0(...)
diff --git a/llvm/test/CodeGen/WebAssembly/lower-em-exceptions-whitelist.ll b/llvm/test/CodeGen/WebAssembly/lower-em-exceptions-whitelist.ll
index 5fcc39909..ce33a9f4b 100644
--- a/llvm/test/CodeGen/WebAssembly/lower-em-exceptions-whitelist.ll
+++ b/llvm/test/CodeGen/WebAssembly/lower-em-exceptions-whitelist.ll
@@ -38,7 +38,7 @@ entry:
           to label %invoke.cont unwind label %lpad
 ; CHECK: entry:
 ; CHECK-NEXT: store i32 0, i32*
-; CHECK-NEXT: call void @__invoke_void(void ()* @foo)
+; CHECK-NEXT: call cc{{.*}} void @__invoke_void(void ()* @foo)
 
 invoke.cont:                                      ; preds = %entry
   br label %try.cont
diff --git a/llvm/test/CodeGen/WebAssembly/lower-em-exceptions.ll b/llvm/test/CodeGen/WebAssembly/lower-em-exceptions.ll
index e54cb2fd4..1f9831067 100644
--- a/llvm/test/CodeGen/WebAssembly/lower-em-exceptions.ll
+++ b/llvm/test/CodeGen/WebAssembly/lower-em-exceptions.ll
@@ -16,7 +16,7 @@ entry:
           to label %invoke.cont unwind label %lpad
 ; CHECK: entry:
 ; CHECK-NEXT: store i32 0, i32* @__THREW__
-; CHECK-NEXT: call void @__invoke_void_i32(void (i32)* @foo, i32 3)
+; CHECK-NEXT: call cc{{.*}} void @__invoke_void_i32(void (i32)* @foo, i32 3)
 ; CHECK-NEXT: %[[__THREW__VAL:.*]] = load i32, i32* @__THREW__
 ; CHECK-NEXT: store i32 0, i32* @__THREW__
 ; CHECK-NEXT: %cmp = icmp eq i32 %[[__THREW__VAL]], 1
@@ -72,7 +72,7 @@ entry:
           to label %invoke.cont unwind label %lpad
 ; CHECK: entry:
 ; CHECK-NEXT: store i32 0, i32* @__THREW__
-; CHECK-NEXT: call void @__invoke_void_i32(void (i32)* @foo, i32 3)
+; CHECK-NEXT: call cc{{.*}} void @__invoke_void_i32(void (i32)* @foo, i32 3)
 ; CHECK-NEXT: %[[__THREW__VAL:.*]] = load i32, i32* @__THREW__
 ; CHECK-NEXT: store i32 0, i32* @__THREW__
 ; CHECK-NEXT: %cmp = icmp eq i32 %[[__THREW__VAL]], 1
@@ -123,7 +123,7 @@ entry:
           to label %invoke.cont unwind label %lpad
 ; CHECK: entry:
 ; CHECK-NEXT: store i32 0, i32* @__THREW__
-; CHECK-NEXT: %0 = call noalias i8* @"__invoke_i8*_i8_i8"(i8* (i8, i8)* @bar, i8 signext 1, i8 zeroext 2)
+; CHECK-NEXT: %0 = call cc{{.*}} noalias i8* @"__invoke_i8*_i8_i8"(i8* (i8, i8)* @bar, i8 signext 1, i8 zeroext 2)
 
 invoke.cont:                                      ; preds = %entry
   br label %try.cont
diff --git a/llvm/test/CodeGen/WebAssembly/lower-em-sjlj-alias.ll b/llvm/test/CodeGen/WebAssembly/lower-em-sjlj-alias.ll
new file mode 100644
index 000000000..6cd8e123d
--- /dev/null
+++ b/llvm/test/CodeGen/WebAssembly/lower-em-sjlj-alias.ll
@@ -0,0 +1,43 @@
+; RUN: opt < %s -wasm-lower-em-ehsjlj -S | FileCheck %s
+
+target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
+target triple = "wasm32-unknown-emscripten"
+
+; Tests if an alias to a function (here malloc) is correctly handled as a
+; function that cannot longjmp.
+
+%struct.__jmp_buf_tag = type { [6 x i32], i32, [32 x i32] }
+@malloc = weak alias i8* (i32), i8* (i32)* @dlmalloc
+
+; CHECK-LABEL: @malloc_test
+define void @malloc_test() {
+entry:
+; CHECK-LABEL: entry
+  ; All setjmp table preparations have to happen within the entry block. These
+  ; check lines list only some of the instructions for that.
+  ; CHECK: call i8* @malloc
+  ; CHECK: call i32* @saveSetjmp
+  ; CHECK: call i32 @getTempRet0
+  %retval = alloca i32, align 4
+  %jmp = alloca [1 x %struct.__jmp_buf_tag], align 16
+  store i32 0, i32* %retval, align 4
+  %arraydecay = getelementptr inbounds [1 x %struct.__jmp_buf_tag], [1 x %struct.__jmp_buf_tag]* %jmp, i32 0, i32 0
+  %call = call i32 @setjmp(%struct.__jmp_buf_tag* %arraydecay) #0
+  ret void
+
+; CHECK-LABEL: entry.split
+  ; CHECK: call void @free
+  ; CHECK: ret void
+}
+
+; This is a dummy dlmalloc implemenation only to make compiler pass, because an
+; alias (malloc) has to point an actual definition.
+define i8* @dlmalloc(i32) {
+  %p = inttoptr i32 0 to i8*
+  ret i8* %p
+}
+
+; Function Attrs: returns_twice
+declare i32 @setjmp(%struct.__jmp_buf_tag*) #0
+
+attributes #0 = { returns_twice }
diff --git a/llvm/test/CodeGen/WebAssembly/lower-em-sjlj-sret.ll b/llvm/test/CodeGen/WebAssembly/lower-em-sjlj-sret.ll
new file mode 100644
index 000000000..eab3b53d5
--- /dev/null
+++ b/llvm/test/CodeGen/WebAssembly/lower-em-sjlj-sret.ll
@@ -0,0 +1,27 @@
+; RUN: llc < %s -asm-verbose=false -enable-emscripten-sjlj -wasm-keep-registers | FileCheck %s
+
+target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
+target triple = "wasm32-unknown-unknown"
+
+%struct.__jmp_buf_tag = type { [6 x i32], i32, [32 x i32] }
+
+declare i32 @setjmp(%struct.__jmp_buf_tag*) #0
+declare {i32, i32} @returns_struct()
+
+; Test the combination of backend legalization of large return types and the
+; Emscripten sjlj transformation
+define {i32, i32} @legalized_to_sret() {
+entry:
+  %env = alloca [1 x %struct.__jmp_buf_tag], align 16
+  %arraydecay = getelementptr inbounds [1 x %struct.__jmp_buf_tag], [1 x %struct.__jmp_buf_tag]* %env, i32 0, i32 0
+  %call = call i32 @setjmp(%struct.__jmp_buf_tag* %arraydecay) #0
+  ; This is the function pointer to pass to invoke.
+  ; It needs to be the first argument (that's what we're testing here)
+  ; CHECK: i32.const $push[[FPTR:[0-9]+]]=, returns_struct
+  ; This is the sret stack region (as an offset from the stack pointer local)
+  ; CHECK: call "__invoke_{i32.i32}", $pop[[FPTR]]
+  %ret = call {i32, i32} @returns_struct()
+  ret {i32, i32} %ret
+}
+
+attributes #0 = { returns_twice }
diff --git a/llvm/test/CodeGen/WebAssembly/lower-em-sjlj.ll b/llvm/test/CodeGen/WebAssembly/lower-em-sjlj.ll
index b65af1434..540b13bdb 100644
--- a/llvm/test/CodeGen/WebAssembly/lower-em-sjlj.ll
+++ b/llvm/test/CodeGen/WebAssembly/lower-em-sjlj.ll
@@ -34,7 +34,7 @@ entry:
 ; CHECK-NEXT: phi i32 [ 0, %entry ], [ %[[LONGJMP_RESULT:.*]], %if.end ]
 ; CHECK-NEXT: %[[ARRAYDECAY1:.*]] = getelementptr inbounds [1 x %struct.__jmp_buf_tag], [1 x %struct.__jmp_buf_tag]* %[[BUF]], i32 0, i32 0
 ; CHECK-NEXT: store i32 0, i32* @__THREW__
-; CHECK-NEXT: call void @"__invoke_void_%struct.__jmp_buf_tag*_i32"(void (%struct.__jmp_buf_tag*, i32)* @emscripten_longjmp_jmpbuf, %struct.__jmp_buf_tag* %[[ARRAYDECAY1]], i32 1)
+; CHECK-NEXT: call cc{{.*}} void @"__invoke_void_%struct.__jmp_buf_tag*_i32"(void (%struct.__jmp_buf_tag*, i32)* @emscripten_longjmp_jmpbuf, %struct.__jmp_buf_tag* %[[ARRAYDECAY1]], i32 1)
 ; CHECK-NEXT: %[[__THREW__VAL:.*]] = load i32, i32* @__THREW__
 ; CHECK-NEXT: store i32 0, i32* @__THREW__
 ; CHECK-NEXT: %[[CMP0:.*]] = icmp ne i32 %__THREW__.val, 0
@@ -85,7 +85,7 @@ entry:
 ; CHECK: %[[SETJMP_TABLE:.*]] = call i32* @saveSetjmp(
 
 ; CHECK: entry.split:
-; CHECK: call void @__invoke_void(void ()* @foo)
+; CHECK: @__invoke_void(void ()* @foo)
 
 ; CHECK: entry.split.split:
 ; CHECK-NEXT: %[[BUF:.*]] = bitcast i32* %[[SETJMP_TABLE]] to i8*
@@ -94,7 +94,7 @@ entry:
 }
 
 ; Test a case when a function call is within try-catch, after a setjmp
-define hidden void @exception_and_longjmp() #3 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+define hidden void @exception_and_longjmp() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 ; CHECK-LABEL: @exception_and_longjmp
 entry:
   %buf = alloca [1 x %struct.__jmp_buf_tag], align 16
@@ -105,7 +105,7 @@ entry:
 
 ; CHECK: entry.split:
 ; CHECK: store i32 0, i32* @__THREW__
-; CHECK-NEXT: call void @__invoke_void(void ()* @foo)
+; CHECK-NEXT: call cc{{.*}} void @__invoke_void(void ()* @foo)
 ; CHECK-NEXT: %[[__THREW__VAL:.*]] = load i32, i32* @__THREW__
 ; CHECK-NEXT: store i32 0, i32* @__THREW__
 ; CHECK-NEXT: %[[CMP0:.*]] = icmp ne i32 %[[__THREW__VAL]], 0
@@ -185,7 +185,7 @@ entry:
   call void @longjmp(%struct.__jmp_buf_tag* %arraydecay, i32 5) #1
   unreachable
 ; CHECK: %[[ARRAYDECAY:.*]] = getelementptr inbounds
-; CHECK-NEXT: call void @emscripten_longjmp_jmpbuf(%struct.__jmp_buf_tag* %[[ARRAYDECAY]], i32 5) #1
+; CHECK-NEXT: call void @emscripten_longjmp_jmpbuf(%struct.__jmp_buf_tag* %[[ARRAYDECAY]], i32 5)
 }
 
 ; Test inline asm handling
@@ -203,6 +203,49 @@ entry:
   ret void
 }
 
+; Test that the allocsize attribute is being transformed properly
+declare i8 *@allocator(i32, %struct.__jmp_buf_tag*) #3
+define hidden i8 *@allocsize() {
+; CHECK-LABEL: @allocsize
+entry:
+  %buf = alloca [1 x %struct.__jmp_buf_tag], align 16
+  %arraydecay = getelementptr inbounds [1 x %struct.__jmp_buf_tag], [1 x %struct.__jmp_buf_tag]* %buf, i32 0, i32 0
+  %call = call i32 @setjmp(%struct.__jmp_buf_tag* %arraydecay) #0
+; CHECK: call cc{{.*}} i8* @"__invoke_i8*_i32_%struct.__jmp_buf_tag*"([[ARGS:.*]]) #[[ALLOCSIZE_ATTR:[0-9]+]]
+  %alloc = call i8* @allocator(i32 20, %struct.__jmp_buf_tag* %arraydecay) #3
+  ret i8 *%alloc
+}
+
+; Tests if program does not crash when there's no setjmp function calls in the
+; module.
+@buffer = global [1 x %struct.__jmp_buf_tag] zeroinitializer, align 16
+define void @longjmp_only() {
+entry:
+  ; CHECK: call void @emscripten_longjmp_jmpbuf
+  call void @longjmp(%struct.__jmp_buf_tag* getelementptr inbounds ([1 x %struct.__jmp_buf_tag], [1 x %struct.__jmp_buf_tag]* @buffer, i32 0, i32 0), i32 1) #1
+  unreachable
+}
+
+; Tests if SSA rewrite works when a use and its def are within the same BB.
+define void @ssa_rewite_in_same_bb() {
+entry:
+  call void @foo()
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  ; CHECK: %{{.*}} = phi i32 [ %var[[VARNO:.*]], %for.inc.split ]
+  %0 = phi i32 [ %var, %for.inc ], [ undef, %entry ]
+  %var = add i32 0, 0
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.cond
+  %call5 = call i32 @setjmp(%struct.__jmp_buf_tag* undef) #0
+  br label %for.cond
+
+; CHECK: for.inc.split:
+  ; CHECK: %var[[VARNO]] = phi i32 [ %var, %for.inc ]
+}
+
 declare void @foo()
 ; Function Attrs: returns_twice
 declare i32 @setjmp(%struct.__jmp_buf_tag*) #0
@@ -227,3 +270,5 @@ declare void @free(i8*)
 attributes #0 = { returns_twice }
 attributes #1 = { noreturn }
 attributes #2 = { nounwind }
+attributes #3 = { allocsize(0) }
+; CHECK: attributes #[[ALLOCSIZE_ATTR]] = { allocsize(1) }
diff --git a/llvm/test/CodeGen/WebAssembly/multivalue.ll b/llvm/test/CodeGen/WebAssembly/multivalue.ll
index cbf8d4e0a..ee32852ef 100644
--- a/llvm/test/CodeGen/WebAssembly/multivalue.ll
+++ b/llvm/test/CodeGen/WebAssembly/multivalue.ll
@@ -1,7 +1,8 @@
 ; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=+multivalue | FileCheck %s
+; RUN: llc < %s --filetype=obj -mattr=+multivalue | obj2yaml | FileCheck %s --check-prefix OBJ
 
-; Test that the multivalue attribute is accepted
-; TODO(tlively): implement multivalue
+; Test that the multivalue returns, function types, and block types
+; work as expected.
 
 target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
 target triple = "wasm32-unknown-unknown"
@@ -9,20 +10,52 @@ target triple = "wasm32-unknown-unknown"
 %pair = type { i32, i32 }
 %packed_pair = type <{ i32, i32 }>
 
-; CHECK-LABEL: sret:
-; CHECK-NEXT: sret (i32, i32, i32) -> ()
-define %pair @sret(%pair %p) {
+; CHECK-LABEL: pair_ident:
+; CHECK-NEXT: .functype pair_ident (i32, i32) -> (i32, i32)
+; CHECK-NEXT: return $0, $1{{$}}
+define %pair @pair_ident(%pair %p) {
   ret %pair %p
 }
 
-; CHECK-LABEL: packed_sret:
-; CHECK-NEXT: packed_sret (i32, i32, i32) -> ()
-define %packed_pair @packed_sret(%packed_pair %p) {
+; CHECK-LABEL: packed_pair_ident:
+; CHECK-NEXT: .functype packed_pair_ident (i32, i32) -> (i32, i32)
+; CHECK-NEXT: return $0, $1{{$}}
+define %packed_pair @packed_pair_ident(%packed_pair %p) {
   ret %packed_pair %p
 }
 
+; CHECK-LABEL: minimal_loop:
+; CHECK-NEXT: .functype minimal_loop (i32) -> (i32, i64)
+; CHECK-NEXT: .LBB{{[0-9]+}}_1:
+; CHECK-NEXT: loop () -> (i32, i64)
+; CHECK-NEXT: br 0{{$}}
+; CHECK-NEXT: .LBB{{[0-9]+}}_2:
+; CHECK-NEXT: end_loop{{$}}
+define {i32, i64} @minimal_loop(i32* %p) {
+entry:
+  br label %loop
+loop:
+  br label %loop
+}
+
 ; CHECK-LABEL: .section .custom_section.target_features
 ; CHECK-NEXT: .int8 1
 ; CHECK-NEXT: .int8 43
 ; CHECK-NEXT: .int8 10
 ; CHECK-NEXT: .ascii "multivalue"
+
+; OBJ-LABEL:  - Type:            TYPE
+; OBJ-NEXT:     Signatures:
+; OBJ-NEXT:       - Index:           0
+; OBJ-NEXT:         ParamTypes:
+; OBJ-NEXT:           - I32
+; OBJ-NEXT:           - I32
+; OBJ-NEXT:         ReturnTypes:
+; OBJ-NEXT:           - I32
+; OBJ-NEXT:           - I32
+; OBJ-NEXT:       - Index:           1
+; OBJ-NEXT:         ParamTypes:
+; OBJ-NEXT:           - I32
+; OBJ-NEXT:         ReturnTypes:
+; OBJ-NEXT:           - I32
+; OBJ-NEXT:           - I64
diff --git a/llvm/test/CodeGen/WebAssembly/negative-base-reg.ll b/llvm/test/CodeGen/WebAssembly/negative-base-reg.ll
index 7408e14fc..c4f1639f0 100644
--- a/llvm/test/CodeGen/WebAssembly/negative-base-reg.ll
+++ b/llvm/test/CodeGen/WebAssembly/negative-base-reg.ll
@@ -31,7 +31,7 @@ for.end:                                          ; preds = %for.body
   ret i32 0
 }
 
-attributes #0 = { norecurse nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="generic" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { norecurse nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="generic" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.ident = !{!0}
 
diff --git a/llvm/test/CodeGen/WebAssembly/reg-stackify.ll b/llvm/test/CodeGen/WebAssembly/reg-stackify.ll
index c0668afce..9a96016c9 100644
--- a/llvm/test/CodeGen/WebAssembly/reg-stackify.ll
+++ b/llvm/test/CodeGen/WebAssembly/reg-stackify.ll
@@ -655,7 +655,7 @@ define i32 @stackpointer_dependency(i8* readnone) {
 ; NOREGS-NEXT: local.tee 0
 ; NOREGS:      i32.load  0
 ; NOREGS-NEXT: i32.load  0
-; NOREGS-NEXT: i32.call_indirect
+; NOREGS-NEXT: i32.call_indirect (i32, i32) -> (i32)
 %class.call_indirect = type { i32 (...)** }
 define i32 @call_indirect_stackify(%class.call_indirect** %objptr, i32 %arg) {
   %obj = load %class.call_indirect*, %class.call_indirect** %objptr
diff --git a/llvm/test/CodeGen/WebAssembly/select.ll b/llvm/test/CodeGen/WebAssembly/select.ll
index ef18d9183..5bf607cb2 100644
--- a/llvm/test/CodeGen/WebAssembly/select.ll
+++ b/llvm/test/CodeGen/WebAssembly/select.ll
@@ -17,10 +17,10 @@ define i32 @select_i32_bool(i1 zeroext %a, i32 %b, i32 %c) {
 
 ; CHECK-LABEL: select_i32_bool_nozext:
 ; CHECK-NEXT: .functype select_i32_bool_nozext (i32, i32, i32) -> (i32){{$}}
-; SLOW-NEXT: i32.const  $push0=, 1{{$}}
-; SLOW-NEXT: i32.and    $push1=, $0, $pop0{{$}}
-; SLOW-NEXT: i32.select $push2=, $1, $2, $pop1{{$}}
-; SLOW-NEXT: return     $pop2{{$}}
+; CHECK-NEXT: i32.const  $push0=, 1{{$}}
+; CHECK-NEXT: i32.and    $push1=, $0, $pop0{{$}}
+; CHECK-NEXT: i32.select $push2=, $1, $2, $pop1{{$}}
+; CHECK-NEXT: return     $pop2{{$}}
 define i32 @select_i32_bool_nozext(i1 %a, i32 %b, i32 %c) {
   %cond = select i1 %a, i32 %b, i32 %c
   ret i32 %cond
@@ -57,10 +57,10 @@ define i64 @select_i64_bool(i1 zeroext %a, i64 %b, i64 %c) {
 
 ; CHECK-LABEL: select_i64_bool_nozext:
 ; CHECK-NEXT: .functype select_i64_bool_nozext (i32, i64, i64) -> (i64){{$}}
-; SLOW-NEXT: i32.const  $push0=, 1{{$}}
-; SLOW-NEXT: i32.and    $push1=, $0, $pop0{{$}}
-; SLOW-NEXT: i64.select $push2=, $1, $2, $pop1{{$}}
-; SLOW-NEXT: return     $pop2{{$}}
+; CHECK-NEXT: i32.const  $push0=, 1{{$}}
+; CHECK-NEXT: i32.and    $push1=, $0, $pop0{{$}}
+; CHECK-NEXT: i64.select $push2=, $1, $2, $pop1{{$}}
+; CHECK-NEXT: return     $pop2{{$}}
 define i64 @select_i64_bool_nozext(i1 %a, i64 %b, i64 %c) {
   %cond = select i1 %a, i64 %b, i64 %c
   ret i64 %cond
@@ -97,8 +97,10 @@ define float @select_f32_bool(i1 zeroext %a, float %b, float %c) {
 
 ; CHECK-LABEL: select_f32_bool_nozext:
 ; CHECK-NEXT: .functype select_f32_bool_nozext (i32, f32, f32) -> (f32){{$}}
-; SLOW-NEXT: f32.select $push0=, $1, $2, $0{{$}}
-; SLOW-NEXT: return     $pop0{{$}}
+; CHECK-NEXT: i32.const  $push0=, 1{{$}}
+; CHECK-NEXT: i32.and    $push1=, $0, $pop0{{$}}
+; CHECK-NEXT: f32.select $push2=, $1, $2, $pop1{{$}}
+; CHECK-NEXT: return     $pop2{{$}}
 define float @select_f32_bool_nozext(i1 %a, float %b, float %c) {
   %cond = select i1 %a, float %b, float %c
   ret float %cond
@@ -135,8 +137,10 @@ define double @select_f64_bool(i1 zeroext %a, double %b, double %c) {
 
 ; CHECK-LABEL: select_f64_bool_nozext:
 ; CHECK-NEXT: .functype select_f64_bool_nozext (i32, f64, f64) -> (f64){{$}}
-; SLOW-NEXT: f64.select $push0=, $1, $2, $0{{$}}
-; SLOW-NEXT: return     $pop0{{$}}
+; CHECK-NEXT: i32.const  $push0=, 1{{$}}
+; CHECK-NEXT: i32.and    $push1=, $0, $pop0{{$}}
+; CHECK-NEXT: f64.select $push2=, $1, $2, $pop1{{$}}
+; CHECK-NEXT: return     $pop2{{$}}
 define double @select_f64_bool_nozext(i1 %a, double %b, double %c) {
   %cond = select i1 %a, double %b, double %c
   ret double %cond
@@ -162,15 +166,56 @@ define double @select_f64_ne(i32 %a, double %b, double %c) {
   ret double %cond
 }
 
-; CHECK-LABEL: pr40805:
-; CHECK-NEXT: .functype pr40805 (i32, i32, i32) -> (i32){{$}}
+; CHECK-LABEL: pr40805_i32:
+; CHECK-NEXT: .functype pr40805_i32 (i32, i32, i32) -> (i32){{$}}
 ; SLOW-NEXT: i32.const  $push0=, 1{{$}}
 ; SLOW-NEXT: i32.and    $push1=, $0, $pop0{{$}}
 ; SLOW-NEXT: i32.select $push2=, $1, $2, $pop1{{$}}
 ; SLOW-NEXT: return     $pop2{{$}}
-define i32 @pr40805(i32 %x, i32 %y, i32 %z) {
+define i32 @pr40805_i32(i32 %x, i32 %y, i32 %z) {
   %a = and i32 %x, 1
   %b = icmp ne i32 %a, 0
   %c = select i1 %b, i32 %y, i32 %z
   ret i32 %c
 }
+
+; CHECK-LABEL: pr40805_i64:
+; CHECK-NEXT: .functype pr40805_i64 (i64, i64, i64) -> (i64){{$}}
+; SLOW-NEXT: i32.wrap_i64 $push0=, $0{{$}}
+; SLOW-NEXT: i32.const  $push1=, 1{{$}}
+; SLOW-NEXT: i32.and    $push2=, $pop0, $pop1{{$}}
+; SLOW-NEXT: i64.select $push3=, $1, $2, $pop2{{$}}
+; SLOW-NEXT: return     $pop3{{$}}
+define i64 @pr40805_i64(i64 %x, i64 %y, i64 %z) {
+  %a = and i64 %x, 1
+  %b = icmp ne i64 %a, 0
+  %c = select i1 %b, i64 %y, i64 %z
+  ret i64 %c
+}
+
+; CHECK-LABEL: pr44012_i32:
+; CHECK-NEXT: .functype pr44012_i32 (i32, f32, f32) -> (f32){{$}}
+; SLOW-NEXT: i32.const  $push0=, 1{{$}}
+; SLOW-NEXT: i32.and    $push1=, $0, $pop0{{$}}
+; SLOW-NEXT: f32.select $push2=, $1, $2, $pop1{{$}}
+; SLOW-NEXT: return     $pop2{{$}}
+define float @pr44012_i32(i32 %x, float %y, float %z) {
+  %a = and i32 %x, 1
+  %b = icmp ne i32 %a, 0
+  %c = select i1 %b, float %y, float %z
+  ret float %c
+}
+
+; CHECK-LABEL: pr44012_i64:
+; CHECK-NEXT: .functype pr44012_i64 (i64, f32, f32) -> (f32){{$}}
+; SLOW-NEXT: i32.wrap_i64 $push0=, $0{{$}}
+; SLOW-NEXT: i32.const  $push1=, 1{{$}}
+; SLOW-NEXT: i32.and    $push2=, $pop0, $pop1{{$}}
+; SLOW-NEXT: f32.select $push3=, $1, $2, $pop2{{$}}
+; SLOW-NEXT: return     $pop3{{$}}
+define float @pr44012_i64(i64 %x, float %y, float %z) {
+  %a = and i64 %x, 1
+  %b = icmp ne i64 %a, 0
+  %c = select i1 %b, float %y, float %z
+  ret float %c
+}
diff --git a/llvm/test/CodeGen/WebAssembly/simd-arith.ll b/llvm/test/CodeGen/WebAssembly/simd-arith.ll
index 410b2e6f5..180f7d066 100644
--- a/llvm/test/CodeGen/WebAssembly/simd-arith.ll
+++ b/llvm/test/CodeGen/WebAssembly/simd-arith.ll
@@ -47,6 +47,77 @@ define <16 x i8> @mul_v16i8(<16 x i8> %x, <16 x i8> %y) {
   ret <16 x i8> %a
 }
 
+; CHECK-LABEL: min_s_v16i8:
+; NO-SIMD128-NOT: i8x16
+; SIMD128-NEXT: .functype min_s_v16i8 (v128, v128) -> (v128){{$}}
+; SIMD128-NEXT: i8x16.min_s $push[[R:[0-9]+]]=, $0, $1{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <16 x i8> @min_s_v16i8(<16 x i8> %x, <16 x i8> %y) {
+  %c = icmp slt <16 x i8> %x, %y
+  %a = select <16 x i1> %c, <16 x i8> %x, <16 x i8> %y
+  ret <16 x i8> %a
+}
+
+; CHECK-LABEL: min_u_v16i8:
+; NO-SIMD128-NOT: i8x16
+; SIMD128-NEXT: .functype min_u_v16i8 (v128, v128) -> (v128){{$}}
+; SIMD128-NEXT: i8x16.min_u $push[[R:[0-9]+]]=, $0, $1{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <16 x i8> @min_u_v16i8(<16 x i8> %x, <16 x i8> %y) {
+  %c = icmp ult <16 x i8> %x, %y
+  %a = select <16 x i1> %c, <16 x i8> %x, <16 x i8> %y
+  ret <16 x i8> %a
+}
+
+; CHECK-LABEL: max_s_v16i8:
+; NO-SIMD128-NOT: i8x16
+; SIMD128-NEXT: .functype max_s_v16i8 (v128, v128) -> (v128){{$}}
+; SIMD128-NEXT: i8x16.max_s $push[[R:[0-9]+]]=, $0, $1{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <16 x i8> @max_s_v16i8(<16 x i8> %x, <16 x i8> %y) {
+  %c = icmp sgt <16 x i8> %x, %y
+  %a = select <16 x i1> %c, <16 x i8> %x, <16 x i8> %y
+  ret <16 x i8> %a
+}
+
+; CHECK-LABEL: max_u_v16i8:
+; NO-SIMD128-NOT: i8x16
+; SIMD128-NEXT: .functype max_u_v16i8 (v128, v128) -> (v128){{$}}
+; SIMD128-NEXT: i8x16.max_u $push[[R:[0-9]+]]=, $0, $1{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <16 x i8> @max_u_v16i8(<16 x i8> %x, <16 x i8> %y) {
+  %c = icmp ugt <16 x i8> %x, %y
+  %a = select <16 x i1> %c, <16 x i8> %x, <16 x i8> %y
+  ret <16 x i8> %a
+}
+
+; CHECK-LABEL: avgr_u_v16i8:
+; NO-SIMD128-NOT: i8x16
+; SIMD128-NEXT: .functype avgr_u_v16i8 (v128, v128) -> (v128){{$}}
+; SIMD128-NEXT: i8x16.avgr_u $push[[R:[0-9]+]]=, $0, $1{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <16 x i8> @avgr_u_v16i8(<16 x i8> %x, <16 x i8> %y) {
+  %a = add nuw <16 x i8> %x, %y
+  %b = add nuw <16 x i8> %a, <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1,
+                              i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
+  %c = udiv <16 x i8> %b, <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2,
+                           i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>
+  ret <16 x i8> %c
+}
+
+; CHECK-LABEL: avgr_u_v16i8_wrap:
+; NO-SIMD128-NOT: i8x16
+; SIMD128-NEXT: .functype avgr_u_v16i8_wrap (v128, v128) -> (v128){{$}}
+; SIMD128-NOT: i8x16.avgr_u
+define <16 x i8> @avgr_u_v16i8_wrap(<16 x i8> %x, <16 x i8> %y) {
+  %a = add <16 x i8> %x, %y
+  %b = add <16 x i8> %a, <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1,
+                          i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
+  %c = udiv <16 x i8> %b, <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2,
+                           i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>
+  ret <16 x i8> %c
+}
+
 ; CHECK-LABEL: neg_v16i8:
 ; NO-SIMD128-NOT: i8x16
 ; SIMD128-NEXT: .functype neg_v16i8 (v128) -> (v128){{$}}
@@ -90,18 +161,18 @@ define <16 x i8> @shl_const_v16i8(<16 x i8> %v) {
 ; NO-SIMD128-NOT: i8x16
 ; SIMD128-NEXT: .functype shl_vec_v16i8 (v128, v128) -> (v128){{$}}
 ; SIMD128-NEXT: i8x16.extract_lane_s $push[[L0:[0-9]+]]=, $0, 0{{$}}
+; SIMD128-NEXT: i8x16.extract_lane_s $push[[L1:[0-9]+]]=, $1, 0{{$}}
 ; SIMD128-NEXT: i32.const $push[[M0:[0-9]+]]=, 7{{$}}
-; SIMD128-NEXT: i8x16.splat $push[[M1:[0-9]+]]=, $pop[[M0]]{{$}}
-; SIMD128-NEXT: v128.and $push[[M2:[0-9]+]]=, $1, $pop[[M1]]{{$}}
-; SIMD128-NEXT: local.tee $push[[M:[0-9]+]]=, $1=, $pop[[M2]]{{$}}
-; SIMD128-NEXT: i8x16.extract_lane_u $push[[L1:[0-9]+]]=, $pop[[M]], 0{{$}}
-; SIMD128-NEXT: i32.shl $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
-; SIMD128-NEXT: i8x16.splat $push[[L3:[0-9]+]]=, $pop[[L2]]{{$}}
+; SIMD128-NEXT: i32.and $push[[M1:[0-9]+]]=, $pop[[L1]], $pop[[M0]]{{$}}
+; SIMD128-NEXT: i32.shl $push[[M2:[0-9]+]]=, $pop[[L0]], $pop[[M1]]
+; SIMD128-NEXT: i8x16.splat $push[[M3:[0-9]+]]=, $pop[[M2]]
 ; Skip 14 lanes
 ; SIMD128:      i8x16.extract_lane_s $push[[L4:[0-9]+]]=, $0, 15{{$}}
-; SIMD128-NEXT: i8x16.extract_lane_u $push[[L5:[0-9]+]]=, $1, 15{{$}}
-; SIMD128-NEXT: i32.shl $push[[L6:[0-9]+]]=, $pop[[L4]], $pop[[L5]]{{$}}
-; SIMD128-NEXT: i8x16.replace_lane $push[[R:[0-9]+]]=, $pop[[L7:[0-9]+]], 15, $pop[[L6]]{{$}}
+; SIMD128-NEXT: i8x16.extract_lane_s $push[[L5:[0-9]+]]=, $1, 15{{$}}
+; SIMD128-NEXT: i32.const $push[[M4:[0-9]+]]=, 7{{$}}
+; SIMD128-NEXT: i32.and $push[[M5:[0-9]+]]=, $pop[[L5]], $pop[[M4]]{{$}}
+; SIMD128-NEXT: i32.shl $push[[M6:[0-9]+]]=, $pop[[L4]], $pop[[M5]]{{$}}
+; SIMD128-NEXT: i8x16.replace_lane $push[[R:[0-9]+]]=, $pop[[M7:[0-9]+]], 15, $pop[[M6]]{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <16 x i8> @shl_vec_v16i8(<16 x i8> %v, <16 x i8> %x) {
   %a = shl <16 x i8> %v, %x
@@ -126,18 +197,18 @@ define <16 x i8> @shr_s_v16i8(<16 x i8> %v, i8 %x) {
 ; NO-SIMD128-NOT: i8x16
 ; SIMD128-NEXT: .functype shr_s_vec_v16i8 (v128, v128) -> (v128){{$}}
 ; SIMD128-NEXT: i8x16.extract_lane_s $push[[L0:[0-9]+]]=, $0, 0{{$}}
+; SIMD128-NEXT: i8x16.extract_lane_s $push[[L1:[0-9]+]]=, $1, 0{{$}}
 ; SIMD128-NEXT: i32.const $push[[M0:[0-9]+]]=, 7{{$}}
-; SIMD128-NEXT: i8x16.splat $push[[M1:[0-9]+]]=, $pop[[M0]]{{$}}
-; SIMD128-NEXT: v128.and $push[[M2:[0-9]+]]=, $1, $pop[[M1]]{{$}}
-; SIMD128-NEXT: local.tee $push[[M:[0-9]+]]=, $1=, $pop[[M2]]{{$}}
-; SIMD128-NEXT: i8x16.extract_lane_u $push[[L1:[0-9]+]]=, $pop[[M]], 0{{$}}
-; SIMD128-NEXT: i32.shr_s $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
-; SIMD128-NEXT: i8x16.splat $push[[L3:[0-9]+]]=, $pop[[L2]]{{$}}
+; SIMD128-NEXT: i32.and $push[[M1:[0-9]+]]=, $pop[[L1]], $pop[[M0]]{{$}}
+; SIMD128-NEXT: i32.shr_s $push[[M2:[0-9]+]]=, $pop[[L0]], $pop[[M1]]
+; SIMD128-NEXT: i8x16.splat $push[[M3:[0-9]+]]=, $pop[[M2]]
 ; Skip 14 lanes
-; SIMD128:      i8x16.extract_lane_s $push[[L0:[0-9]+]]=, $0, 15{{$}}
-; SIMD128-NEXT: i8x16.extract_lane_u $push[[L1:[0-9]+]]=, $1, 15{{$}}
-; SIMD128-NEXT: i32.shr_s $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
-; SIMD128-NEXT: i8x16.replace_lane $push[[R:[0-9]+]]=, $pop{{[0-9]+}}, 15, $pop[[L2]]{{$}}
+; SIMD128:      i8x16.extract_lane_s $push[[L4:[0-9]+]]=, $0, 15{{$}}
+; SIMD128-NEXT: i8x16.extract_lane_s $push[[L5:[0-9]+]]=, $1, 15{{$}}
+; SIMD128-NEXT: i32.const $push[[M4:[0-9]+]]=, 7{{$}}
+; SIMD128-NEXT: i32.and $push[[M5:[0-9]+]]=, $pop[[L5]], $pop[[M4]]{{$}}
+; SIMD128-NEXT: i32.shr_s $push[[M6:[0-9]+]]=, $pop[[L4]], $pop[[M5]]{{$}}
+; SIMD128-NEXT: i8x16.replace_lane $push[[R:[0-9]+]]=, $pop[[M7:[0-9]+]], 15, $pop[[M6]]{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <16 x i8> @shr_s_vec_v16i8(<16 x i8> %v, <16 x i8> %x) {
   %a = ashr <16 x i8> %v, %x
@@ -162,18 +233,18 @@ define <16 x i8> @shr_u_v16i8(<16 x i8> %v, i8 %x) {
 ; NO-SIMD128-NOT: i8x16
 ; SIMD128-NEXT: .functype shr_u_vec_v16i8 (v128, v128) -> (v128){{$}}
 ; SIMD128-NEXT: i8x16.extract_lane_u $push[[L0:[0-9]+]]=, $0, 0{{$}}
+; SIMD128-NEXT: i8x16.extract_lane_s $push[[L1:[0-9]+]]=, $1, 0{{$}}
 ; SIMD128-NEXT: i32.const $push[[M0:[0-9]+]]=, 7{{$}}
-; SIMD128-NEXT: i8x16.splat $push[[M1:[0-9]+]]=, $pop[[M0]]{{$}}
-; SIMD128-NEXT: v128.and $push[[M2:[0-9]+]]=, $1, $pop[[M1]]{{$}}
-; SIMD128-NEXT: local.tee $push[[M:[0-9]+]]=, $1=, $pop[[M2]]{{$}}
-; SIMD128-NEXT: i8x16.extract_lane_u $push[[L1:[0-9]+]]=, $pop[[M]], 0{{$}}
-; SIMD128-NEXT: i32.shr_u $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
-; SIMD128-NEXT: i8x16.splat $push[[L3:[0-9]+]]=, $pop[[L2]]{{$}}
+; SIMD128-NEXT: i32.and $push[[M1:[0-9]+]]=, $pop[[L1]], $pop[[M0]]{{$}}
+; SIMD128-NEXT: i32.shr_u $push[[M2:[0-9]+]]=, $pop[[L0]], $pop[[M1]]
+; SIMD128-NEXT: i8x16.splat $push[[M3:[0-9]+]]=, $pop[[M2]]
 ; Skip 14 lanes
 ; SIMD128:      i8x16.extract_lane_u $push[[L4:[0-9]+]]=, $0, 15{{$}}
-; SIMD128-NEXT: i8x16.extract_lane_u $push[[L5:[0-9]+]]=, $1, 15{{$}}
-; SIMD128-NEXT: i32.shr_u $push[[L6:[0-9]+]]=, $pop[[L4]], $pop[[L5]]{{$}}
-; SIMD128-NEXT: i8x16.replace_lane $push[[R:[0-9]+]]=, $pop[[L7:[0-9]+]], 15, $pop[[L6]]{{$}}
+; SIMD128-NEXT: i8x16.extract_lane_s $push[[L5:[0-9]+]]=, $1, 15{{$}}
+; SIMD128-NEXT: i32.const $push[[M4:[0-9]+]]=, 7{{$}}
+; SIMD128-NEXT: i32.and $push[[M5:[0-9]+]]=, $pop[[L5]], $pop[[M4]]{{$}}
+; SIMD128-NEXT: i32.shr_u $push[[M6:[0-9]+]]=, $pop[[L4]], $pop[[M5]]{{$}}
+; SIMD128-NEXT: i8x16.replace_lane $push[[R:[0-9]+]]=, $pop[[M7:[0-9]+]], 15, $pop[[M6]]{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <16 x i8> @shr_u_vec_v16i8(<16 x i8> %v, <16 x i8> %x) {
   %a = lshr <16 x i8> %v, %x
@@ -223,6 +294,23 @@ define <16 x i8> @not_v16i8(<16 x i8> %x) {
   ret <16 x i8> %a
 }
 
+; CHECK-LABEL: andnot_v16i8:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: v128.andnot
+; SIMD128-NEXT: .functype andnot_v16i8 (v128, v128) -> (v128){{$}}
+; SIMD128-SLOW-NEXT: v128.andnot $push[[R:[0-9]+]]=, $0, $1{{$}}
+; SIMD128-SLOW-NEXT: return $pop[[R]]{{$}}
+; SIMD128-FAST-NEXT: v128.not
+; SIMD128-FAST-NEXT: v128.and
+; SIMD128-FAST-NEXT: return
+define <16 x i8> @andnot_v16i8(<16 x i8> %x, <16 x i8> %y) {
+ %inv_y = xor <16 x i8> %y,
+   <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1,
+    i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
+ %a = and <16 x i8> %x, %inv_y
+ ret <16 x i8> %a
+}
+
 ; CHECK-LABEL: bitselect_v16i8:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype bitselect_v16i8 (v128, v128, v128) -> (v128){{$}}
@@ -276,6 +364,73 @@ define <8 x i16> @mul_v8i16(<8 x i16> %x, <8 x i16> %y) {
   ret <8 x i16> %a
 }
 
+; CHECK-LABEL: min_s_v8i16:
+; NO-SIMD128-NOT: i16x8
+; SIMD128-NEXT: .functype min_s_v8i16 (v128, v128) -> (v128){{$}}
+; SIMD128-NEXT: i16x8.min_s $push[[R:[0-9]+]]=, $0, $1{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <8 x i16> @min_s_v8i16(<8 x i16> %x, <8 x i16> %y) {
+  %c = icmp slt <8 x i16> %x, %y
+  %a = select <8 x i1> %c, <8 x i16> %x, <8 x i16> %y
+  ret <8 x i16> %a
+}
+
+; CHECK-LABEL: min_u_v8i16:
+; NO-SIMD128-NOT: i16x8
+; SIMD128-NEXT: .functype min_u_v8i16 (v128, v128) -> (v128){{$}}
+; SIMD128-NEXT: i16x8.min_u $push[[R:[0-9]+]]=, $0, $1{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <8 x i16> @min_u_v8i16(<8 x i16> %x, <8 x i16> %y) {
+  %c = icmp ult <8 x i16> %x, %y
+  %a = select <8 x i1> %c, <8 x i16> %x, <8 x i16> %y
+  ret <8 x i16> %a
+}
+
+; CHECK-LABEL: max_s_v8i16:
+; NO-SIMD128-NOT: i16x8
+; SIMD128-NEXT: .functype max_s_v8i16 (v128, v128) -> (v128){{$}}
+; SIMD128-NEXT: i16x8.max_s $push[[R:[0-9]+]]=, $0, $1{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <8 x i16> @max_s_v8i16(<8 x i16> %x, <8 x i16> %y) {
+  %c = icmp sgt <8 x i16> %x, %y
+  %a = select <8 x i1> %c, <8 x i16> %x, <8 x i16> %y
+  ret <8 x i16> %a
+}
+
+; CHECK-LABEL: max_u_v8i16:
+; NO-SIMD128-NOT: i16x8
+; SIMD128-NEXT: .functype max_u_v8i16 (v128, v128) -> (v128){{$}}
+; SIMD128-NEXT: i16x8.max_u $push[[R:[0-9]+]]=, $0, $1{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <8 x i16> @max_u_v8i16(<8 x i16> %x, <8 x i16> %y) {
+  %c = icmp ugt <8 x i16> %x, %y
+  %a = select <8 x i1> %c, <8 x i16> %x, <8 x i16> %y
+  ret <8 x i16> %a
+}
+
+; CHECK-LABEL: avgr_u_v8i16:
+; NO-SIMD128-NOT: i16x8
+; SIMD128-NEXT: .functype avgr_u_v8i16 (v128, v128) -> (v128){{$}}
+; SIMD128-NEXT: i16x8.avgr_u $push[[R:[0-9]+]]=, $0, $1{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <8 x i16> @avgr_u_v8i16(<8 x i16> %x, <8 x i16> %y) {
+  %a = add nuw <8 x i16> %x, %y
+  %b = add nuw <8 x i16> %a, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
+  %c = udiv <8 x i16> %b, <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
+  ret <8 x i16> %c
+}
+
+; CHECK-LABEL: avgr_u_v8i16_wrap:
+; NO-SIMD128-NOT: i16x8
+; SIMD128-NEXT: .functype avgr_u_v8i16_wrap (v128, v128) -> (v128){{$}}
+; SIMD128-NOT: i16x8.avgr_u
+define <8 x i16> @avgr_u_v8i16_wrap(<8 x i16> %x, <8 x i16> %y) {
+  %a = add <8 x i16> %x, %y
+  %b = add <8 x i16> %a, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
+  %c = udiv <8 x i16> %b, <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
+  ret <8 x i16> %c
+}
+
 ; CHECK-LABEL: neg_v8i16:
 ; NO-SIMD128-NOT: i16x8
 ; SIMD128-NEXT: .functype neg_v8i16 (v128) -> (v128){{$}}
@@ -316,18 +471,18 @@ define <8 x i16> @shl_const_v8i16(<8 x i16> %v) {
 ; NO-SIMD128-NOT: i16x8
 ; SIMD128-NEXT: .functype shl_vec_v8i16 (v128, v128) -> (v128){{$}}
 ; SIMD128-NEXT: i16x8.extract_lane_s $push[[L0:[0-9]+]]=, $0, 0{{$}}
+; SIMD128-NEXT: i16x8.extract_lane_s $push[[L1:[0-9]+]]=, $1, 0{{$}}
 ; SIMD128-NEXT: i32.const $push[[M0:[0-9]+]]=, 15{{$}}
-; SIMD128-NEXT: i16x8.splat $push[[M1:[0-9]+]]=, $pop[[M0]]{{$}}
-; SIMD128-NEXT: v128.and $push[[M2:[0-9]+]]=, $1, $pop[[M1]]{{$}}
-; SIMD128-NEXT: local.tee $push[[M:[0-9]+]]=, $1=, $pop[[M2]]{{$}}
-; SIMD128-NEXT: i16x8.extract_lane_u $push[[L1:[0-9]+]]=, $pop[[M]], 0{{$}}
-; SIMD128-NEXT: i32.shl $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
-; SIMD128-NEXT: i16x8.splat $push[[L3:[0-9]+]]=, $pop[[L2]]{{$}}
+; SIMD128-NEXT: i32.and $push[[M1:[0-9]+]]=, $pop[[L1]], $pop[[M0]]{{$}}
+; SIMD128-NEXT: i32.shl $push[[M2:[0-9]+]]=, $pop[[L0]], $pop[[M1]]{{$}}
+; SIMD128-NEXT: i16x8.splat $push[[M3:[0-9]+]]=, $pop[[M2]]{{$}}
 ; Skip 6 lanes
 ; SIMD128:      i16x8.extract_lane_s $push[[L4:[0-9]+]]=, $0, 7{{$}}
-; SIMD128-NEXT: i16x8.extract_lane_u $push[[L5:[0-9]+]]=, $1, 7{{$}}
-; SIMD128-NEXT: i32.shl $push[[L6:[0-9]+]]=, $pop[[L4]], $pop[[L5]]{{$}}
-; SIMD128-NEXT: i16x8.replace_lane $push[[R:[0-9]+]]=, $pop[[L7:[0-9]+]], 7, $pop[[L6]]{{$}}
+; SIMD128-NEXT: i16x8.extract_lane_s $push[[L5:[0-9]+]]=, $1, 7{{$}}
+; SIMD128-NEXT: i32.const $push[[M4:[0-9]+]]=, 15{{$}}
+; SIMD128-NEXT: i32.and $push[[M5:[0-9]+]]=, $pop[[L5]], $pop[[M4]]{{$}}
+; SIMD128-NEXT: i32.shl $push[[M6:[0-9]+]]=, $pop[[L4]], $pop[[M5]]{{$}}
+; SIMD128-NEXT: i16x8.replace_lane $push[[R:[0-9]+]]=, $pop[[M7:[0-9]+]], 7, $pop[[M6]]{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <8 x i16> @shl_vec_v8i16(<8 x i16> %v, <8 x i16> %x) {
   %a = shl <8 x i16> %v, %x
@@ -351,18 +506,18 @@ define <8 x i16> @shr_s_v8i16(<8 x i16> %v, i16 %x) {
 ; NO-SIMD128-NOT: i16x8
 ; SIMD128-NEXT: .functype shr_s_vec_v8i16 (v128, v128) -> (v128){{$}}
 ; SIMD128-NEXT: i16x8.extract_lane_s $push[[L0:[0-9]+]]=, $0, 0{{$}}
+; SIMD128-NEXT: i16x8.extract_lane_s $push[[L1:[0-9]+]]=, $1, 0{{$}}
 ; SIMD128-NEXT: i32.const $push[[M0:[0-9]+]]=, 15{{$}}
-; SIMD128-NEXT: i16x8.splat $push[[M1:[0-9]+]]=, $pop[[M0]]{{$}}
-; SIMD128-NEXT: v128.and $push[[M2:[0-9]+]]=, $1, $pop[[M1]]{{$}}
-; SIMD128-NEXT: local.tee $push[[M:[0-9]+]]=, $1=, $pop[[M2]]{{$}}
-; SIMD128-NEXT: i16x8.extract_lane_u $push[[L1:[0-9]+]]=, $pop[[M]], 0{{$}}
-; SIMD128-NEXT: i32.shr_s $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
-; SIMD128-NEXT: i16x8.splat $push[[L3:[0-9]+]]=, $pop[[L2]]{{$}}
+; SIMD128-NEXT: i32.and $push[[M1:[0-9]+]]=, $pop[[L1]], $pop[[M0]]{{$}}
+; SIMD128-NEXT: i32.shr_s $push[[M2:[0-9]+]]=, $pop[[L0]], $pop[[M1]]{{$}}
+; SIMD128-NEXT: i16x8.splat $push[[M3:[0-9]+]]=, $pop[[M2]]{{$}}
 ; Skip 6 lanes
-; SIMD128:      i16x8.extract_lane_s $push[[L0:[0-9]+]]=, $0, 7{{$}}
-; SIMD128-NEXT: i16x8.extract_lane_u $push[[L1:[0-9]+]]=, $1, 7{{$}}
-; SIMD128-NEXT: i32.shr_s $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
-; SIMD128-NEXT: i16x8.replace_lane $push[[R:[0-9]+]]=, $pop{{[0-9]+}}, 7, $pop[[L2]]{{$}}
+; SIMD128:      i16x8.extract_lane_s $push[[L4:[0-9]+]]=, $0, 7{{$}}
+; SIMD128-NEXT: i16x8.extract_lane_s $push[[L5:[0-9]+]]=, $1, 7{{$}}
+; SIMD128-NEXT: i32.const $push[[M4:[0-9]+]]=, 15{{$}}
+; SIMD128-NEXT: i32.and $push[[M5:[0-9]+]]=, $pop[[L5]], $pop[[M4]]{{$}}
+; SIMD128-NEXT: i32.shr_s $push[[M6:[0-9]+]]=, $pop[[L4]], $pop[[M5]]{{$}}
+; SIMD128-NEXT: i16x8.replace_lane $push[[R:[0-9]+]]=, $pop[[M7:[0-9]+]], 7, $pop[[M6]]{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <8 x i16> @shr_s_vec_v8i16(<8 x i16> %v, <8 x i16> %x) {
   %a = ashr <8 x i16> %v, %x
@@ -386,18 +541,18 @@ define <8 x i16> @shr_u_v8i16(<8 x i16> %v, i16 %x) {
 ; NO-SIMD128-NOT: i16x8
 ; SIMD128-NEXT: .functype shr_u_vec_v8i16 (v128, v128) -> (v128){{$}}
 ; SIMD128-NEXT: i16x8.extract_lane_u $push[[L0:[0-9]+]]=, $0, 0{{$}}
+; SIMD128-NEXT: i16x8.extract_lane_s $push[[L1:[0-9]+]]=, $1, 0{{$}}
 ; SIMD128-NEXT: i32.const $push[[M0:[0-9]+]]=, 15{{$}}
-; SIMD128-NEXT: i16x8.splat $push[[M1:[0-9]+]]=, $pop[[M0]]{{$}}
-; SIMD128-NEXT: v128.and $push[[M2:[0-9]+]]=, $1, $pop[[M1]]{{$}}
-; SIMD128-NEXT: local.tee $push[[M:[0-9]+]]=, $1=, $pop[[M2]]{{$}}
-; SIMD128-NEXT: i16x8.extract_lane_u $push[[L1:[0-9]+]]=, $pop[[M]], 0{{$}}
-; SIMD128-NEXT: i32.shr_u $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
-; SIMD128-NEXT: i16x8.splat $push[[L3:[0-9]+]]=, $pop[[L2]]{{$}}
+; SIMD128-NEXT: i32.and $push[[M1:[0-9]+]]=, $pop[[L1]], $pop[[M0]]{{$}}
+; SIMD128-NEXT: i32.shr_u $push[[M2:[0-9]+]]=, $pop[[L0]], $pop[[M1]]{{$}}
+; SIMD128-NEXT: i16x8.splat $push[[M3:[0-9]+]]=, $pop[[M2]]{{$}}
 ; Skip 6 lanes
 ; SIMD128:      i16x8.extract_lane_u $push[[L4:[0-9]+]]=, $0, 7{{$}}
-; SIMD128-NEXT: i16x8.extract_lane_u $push[[L5:[0-9]+]]=, $1, 7{{$}}
-; SIMD128-NEXT: i32.shr_u $push[[L6:[0-9]+]]=, $pop[[L4]], $pop[[L5]]{{$}}
-; SIMD128-NEXT: i16x8.replace_lane $push[[R:[0-9]+]]=, $pop[[L7:[0-9]+]], 7, $pop[[L6]]{{$}}
+; SIMD128-NEXT: i16x8.extract_lane_s $push[[L5:[0-9]+]]=, $1, 7{{$}}
+; SIMD128-NEXT: i32.const $push[[M4:[0-9]+]]=, 15{{$}}
+; SIMD128-NEXT: i32.and $push[[M5:[0-9]+]]=, $pop[[L5]], $pop[[M4]]{{$}}
+; SIMD128-NEXT: i32.shr_u $push[[M6:[0-9]+]]=, $pop[[L4]], $pop[[M5]]{{$}}
+; SIMD128-NEXT: i16x8.replace_lane $push[[R:[0-9]+]]=, $pop[[M7:[0-9]+]], 7, $pop[[M6]]{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <8 x i16> @shr_u_vec_v8i16(<8 x i16> %v, <8 x i16> %x) {
   %a = lshr <8 x i16> %v, %x
@@ -445,6 +600,22 @@ define <8 x i16> @not_v8i16(<8 x i16> %x) {
   ret <8 x i16> %a
 }
 
+; CHECK-LABEL: andnot_v8i16:
+; SIMD128-VM-NOT: v128.andnot
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype andnot_v8i16 (v128, v128) -> (v128){{$}}
+; SIMD128-SLOW-NEXT: v128.andnot $push[[R:[0-9]+]]=, $0, $1{{$}}
+; SIMD128-SLOW-NEXT: return $pop[[R]]{{$}}
+; SIMD128-FAST-NEXT: v128.not
+; SIMD128-FAST-NEXT: v128.and
+; SIMD128-FAST-NEXT: return
+define <8 x i16> @andnot_v8i16(<8 x i16> %x, <8 x i16> %y) {
+ %inv_y = xor <8 x i16> %y,
+   <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>
+ %a = and <8 x i16> %x, %inv_y
+ ret <8 x i16> %a
+}
+
 ; CHECK-LABEL: bitselect_v8i16:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype bitselect_v8i16 (v128, v128, v128) -> (v128){{$}}
@@ -498,6 +669,50 @@ define <4 x i32> @mul_v4i32(<4 x i32> %x, <4 x i32> %y) {
   ret <4 x i32> %a
 }
 
+; CHECK-LABEL: min_s_v4i32:
+; NO-SIMD128-NOT: i32x4
+; SIMD128-NEXT: .functype min_s_v4i32 (v128, v128) -> (v128){{$}}
+; SIMD128-NEXT: i32x4.min_s $push[[R:[0-9]+]]=, $0, $1{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x i32> @min_s_v4i32(<4 x i32> %x, <4 x i32> %y) {
+  %c = icmp slt <4 x i32> %x, %y
+  %a = select <4 x i1> %c, <4 x i32> %x, <4 x i32> %y
+  ret <4 x i32> %a
+}
+
+; CHECK-LABEL: min_u_v4i32:
+; NO-SIMD128-NOT: i32x4
+; SIMD128-NEXT: .functype min_u_v4i32 (v128, v128) -> (v128){{$}}
+; SIMD128-NEXT: i32x4.min_u $push[[R:[0-9]+]]=, $0, $1{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x i32> @min_u_v4i32(<4 x i32> %x, <4 x i32> %y) {
+  %c = icmp ult <4 x i32> %x, %y
+  %a = select <4 x i1> %c, <4 x i32> %x, <4 x i32> %y
+  ret <4 x i32> %a
+}
+
+; CHECK-LABEL: max_s_v4i32:
+; NO-SIMD128-NOT: i32x4
+; SIMD128-NEXT: .functype max_s_v4i32 (v128, v128) -> (v128){{$}}
+; SIMD128-NEXT: i32x4.max_s $push[[R:[0-9]+]]=, $0, $1{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x i32> @max_s_v4i32(<4 x i32> %x, <4 x i32> %y) {
+  %c = icmp sgt <4 x i32> %x, %y
+  %a = select <4 x i1> %c, <4 x i32> %x, <4 x i32> %y
+  ret <4 x i32> %a
+}
+
+; CHECK-LABEL: max_u_v4i32:
+; NO-SIMD128-NOT: i32x4
+; SIMD128-NEXT: .functype max_u_v4i32 (v128, v128) -> (v128){{$}}
+; SIMD128-NEXT: i32x4.max_u $push[[R:[0-9]+]]=, $0, $1{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x i32> @max_u_v4i32(<4 x i32> %x, <4 x i32> %y) {
+  %c = icmp ugt <4 x i32> %x, %y
+  %a = select <4 x i1> %c, <4 x i32> %x, <4 x i32> %y
+  ret <4 x i32> %a
+}
+
 ; CHECK-LABEL: neg_v4i32:
 ; NO-SIMD128-NOT: i32x4
 ; SIMD128-NEXT: .functype neg_v4i32 (v128) -> (v128){{$}}
@@ -652,6 +867,21 @@ define <4 x i32> @not_v4i32(<4 x i32> %x) {
   ret <4 x i32> %a
 }
 
+; CHECK-LABEL: andnot_v4i32:
+; SIMD128-VM-NOT: v128.andnot
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype andnot_v4i32 (v128, v128) -> (v128){{$}}
+; SIMD128-SLOW-NEXT: v128.andnot $push[[R:[0-9]+]]=, $0, $1{{$}}
+; SIMD128-SLOW-NEXT: return $pop[[R]]{{$}}
+; SIMD128-FAST-NEXT: v128.not
+; SIMD128-FAST-NEXT: v128.and
+; SIMD128-FAST-NEXT: return
+define <4 x i32> @andnot_v4i32(<4 x i32> %x, <4 x i32> %y) {
+ %inv_y = xor <4 x i32> %y, <i32 -1, i32 -1, i32 -1, i32 -1>
+ %a = and <4 x i32> %x, %inv_y
+ ret <4 x i32> %a
+}
+
 ; CHECK-LABEL: bitselect_v4i32:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype bitselect_v4i32 (v128, v128, v128) -> (v128){{$}}
@@ -962,6 +1192,21 @@ define <2 x i64> @not_v2i64(<2 x i64> %x) {
   ret <2 x i64> %a
 }
 
+; CHECK-LABEL: andnot_v2i64:
+; SIMD128-VM-NOT: v128.andnot
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype andnot_v2i64 (v128, v128) -> (v128){{$}}
+; SIMD128-SLOW-NEXT: v128.andnot $push[[R:[0-9]+]]=, $0, $1{{$}}
+; SIMD128-SLOW-NEXT: return $pop[[R]]{{$}}
+; SIMD128-FAST-NEXT: v128.not
+; SIMD128-FAST-NEXT: v128.and
+; SIMD128-FAST-NEXT: return
+define <2 x i64> @andnot_v2i64(<2 x i64> %x, <2 x i64> %y) {
+ %inv_y = xor <2 x i64> %y, <i64 -1, i64 -1>
+ %a = and <2 x i64> %x, %inv_y
+ ret <2 x i64> %a
+}
+
 ; CHECK-LABEL: bitselect_v2i64:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-VM-NOT: v128
diff --git a/llvm/test/CodeGen/WebAssembly/simd-build-vector.ll b/llvm/test/CodeGen/WebAssembly/simd-build-vector.ll
index 41a320c92..469c50ae2 100644
--- a/llvm/test/CodeGen/WebAssembly/simd-build-vector.ll
+++ b/llvm/test/CodeGen/WebAssembly/simd-build-vector.ll
@@ -7,13 +7,12 @@
 target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
 target triple = "wasm32-unknown-unknown"
 
-; CHECK-LABEL: same_const_one_replaced_i8x16:
-; CHECK-NEXT:  .functype       same_const_one_replaced_i8x16 (i32) -> (v128)
-; CHECK-NEXT:  i32.const       $push[[L0:[0-9]+]]=, 42
-; CHECK-NEXT:  i16x8.splat     $push[[L1:[0-9]+]]=, $pop[[L0]]
-; CHECK-NEXT:  i16x8.replace_lane      $push[[L2:[0-9]+]]=, $pop[[L1]], 5, $0
-; CHECK-NEXT:  return          $pop[[L2]]
-define <8 x i16> @same_const_one_replaced_i8x16(i16 %x) {
+; CHECK-LABEL: same_const_one_replaced_i16x8:
+; CHECK-NEXT:  .functype       same_const_one_replaced_i16x8 (i32) -> (v128)
+; CHECK-NEXT:  v128.const      $push[[L0:[0-9]+]]=, 42, 42, 42, 42, 42, 0, 42, 42
+; CHECK-NEXT:  i16x8.replace_lane      $push[[L1:[0-9]+]]=, $pop[[L0]], 5, $0
+; CHECK-NEXT:  return          $pop[[L1]]
+define <8 x i16> @same_const_one_replaced_i16x8(i16 %x) {
   %v = insertelement
     <8 x i16> <i16 42, i16 42, i16 42, i16 42, i16 42, i16 42, i16 42, i16 42>,
     i16 %x,
@@ -21,12 +20,12 @@ define <8 x i16> @same_const_one_replaced_i8x16(i16 %x) {
   ret <8 x i16> %v
 }
 
-; CHECK-LABEL: different_const_one_replaced_i8x16:
-; CHECK-NEXT:  .functype       different_const_one_replaced_i8x16 (i32) -> (v128)
+; CHECK-LABEL: different_const_one_replaced_i16x8:
+; CHECK-NEXT:  .functype       different_const_one_replaced_i16x8 (i32) -> (v128)
 ; CHECK-NEXT:  v128.const      $push[[L0:[0-9]+]]=, 1, -2, 3, -4, 5, 0, 7, -8
 ; CHECK-NEXT:  i16x8.replace_lane      $push[[L1:[0-9]+]]=, $pop[[L0]], 5, $0
 ; CHECK-NEXT:  return          $pop[[L1]]
-define <8 x i16> @different_const_one_replaced_i8x16(i16 %x) {
+define <8 x i16> @different_const_one_replaced_i16x8(i16 %x) {
   %v = insertelement
     <8 x i16> <i16 1, i16 -2, i16 3, i16 -4, i16 5, i16 -6, i16 7, i16 -8>,
     i16 %x,
@@ -36,10 +35,9 @@ define <8 x i16> @different_const_one_replaced_i8x16(i16 %x) {
 
 ; CHECK-LABEL: same_const_one_replaced_f32x4:
 ; CHECK-NEXT:  .functype       same_const_one_replaced_f32x4 (f32) -> (v128)
-; CHECK-NEXT:  f32.const       $push[[L0:[0-9]+]]=, 0x1.5p5
-; CHECK-NEXT:  f32x4.splat     $push[[L1:[0-9]+]]=, $pop[[L0]]
-; CHECK-NEXT:  f32x4.replace_lane      $push[[L2:[0-9]+]]=, $pop[[L1]], 2, $0
-; CHECK-NEXT:  return          $pop[[L2]]
+; CHECK-NEXT:  v128.const      $push[[L0:[0-9]+]]=, 0x1.5p5, 0x1.5p5, 0x0p0, 0x1.5p5
+; CHECK-NEXT:  f32x4.replace_lane      $push[[L1:[0-9]+]]=, $pop[[L0]], 2, $0
+; CHECK-NEXT:  return          $pop[[L1]]
 define <4 x float> @same_const_one_replaced_f32x4(float %x) {
   %v = insertelement
     <4 x float> <float 42., float 42., float 42., float 42.>,
@@ -63,11 +61,8 @@ define <4 x float> @different_const_one_replaced_f32x4(float %x) {
 
 ; CHECK-LABEL: splat_common_const_i32x4:
 ; CHECK-NEXT:  .functype       splat_common_const_i32x4 () -> (v128)
-; CHECK-NEXT:  i32.const       $push[[L0:[0-9]+]]=, 3
-; CHECK-NEXT:  i32x4.splat     $push[[L1:[0-9]+]]=, $pop[[L0]]
-; CHECK-NEXT:  i32.const       $push[[L2:[0-9]+]]=, 1
-; CHECK-NEXT:  i32x4.replace_lane      $push[[L3:[0-9]+]]=, $pop[[L1]], 3, $pop[[L2]]
-; CHECK-NEXT:  return          $pop[[L3]]
+; CHECK-NEXT:  v128.const      $push[[L0:[0-9]+]]=, 0, 3, 3, 1
+; CHECK-NEXT:  return          $pop[[L0]]
 define <4 x i32> @splat_common_const_i32x4() {
   ret <4 x i32> <i32 undef, i32 3, i32 3, i32 1>
 }
@@ -92,11 +87,159 @@ define <8 x i16> @splat_common_arg_i16x8(i16 %a, i16 %b, i16 %c) {
   ret <8 x i16> %v7
 }
 
+; CHECK-LABEL: swizzle_one_i8x16:
+; CHECK-NEXT:  .functype       swizzle_one_i8x16 (v128, v128) -> (v128)
+; CHECK-NEXT:  v8x16.swizzle   $push[[L0:[0-9]+]]=, $0, $1
+; CHECK-NEXT:  return          $pop[[L0]]
+define <16 x i8> @swizzle_one_i8x16(<16 x i8> %src, <16 x i8> %mask) {
+  %m0 = extractelement <16 x i8> %mask, i32 0
+  %s0 = extractelement <16 x i8> %src, i8 %m0
+  %v0 = insertelement <16 x i8> undef, i8 %s0, i32 0
+  ret <16 x i8> %v0
+}
+
+; CHECK-LABEL: swizzle_all_i8x16:
+; CHECK-NEXT:  .functype       swizzle_all_i8x16 (v128, v128) -> (v128)
+; CHECK-NEXT:  v8x16.swizzle   $push[[L0:[0-9]+]]=, $0, $1
+; CHECK-NEXT:  return          $pop[[L0]]
+define <16 x i8> @swizzle_all_i8x16(<16 x i8> %src, <16 x i8> %mask) {
+  %m0 = extractelement <16 x i8> %mask, i32 0
+  %s0 = extractelement <16 x i8> %src, i8 %m0
+  %v0 = insertelement <16 x i8> undef, i8 %s0, i32 0
+  %m1 = extractelement <16 x i8> %mask, i32 1
+  %s1 = extractelement <16 x i8> %src, i8 %m1
+  %v1 = insertelement <16 x i8> %v0, i8 %s1, i32 1
+  %m2 = extractelement <16 x i8> %mask, i32 2
+  %s2 = extractelement <16 x i8> %src, i8 %m2
+  %v2 = insertelement <16 x i8> %v1, i8 %s2, i32 2
+  %m3 = extractelement <16 x i8> %mask, i32 3
+  %s3 = extractelement <16 x i8> %src, i8 %m3
+  %v3 = insertelement <16 x i8> %v2, i8 %s3, i32 3
+  %m4 = extractelement <16 x i8> %mask, i32 4
+  %s4 = extractelement <16 x i8> %src, i8 %m4
+  %v4 = insertelement <16 x i8> %v3, i8 %s4, i32 4
+  %m5 = extractelement <16 x i8> %mask, i32 5
+  %s5 = extractelement <16 x i8> %src, i8 %m5
+  %v5 = insertelement <16 x i8> %v4, i8 %s5, i32 5
+  %m6 = extractelement <16 x i8> %mask, i32 6
+  %s6 = extractelement <16 x i8> %src, i8 %m6
+  %v6 = insertelement <16 x i8> %v5, i8 %s6, i32 6
+  %m7 = extractelement <16 x i8> %mask, i32 7
+  %s7 = extractelement <16 x i8> %src, i8 %m7
+  %v7 = insertelement <16 x i8> %v6, i8 %s7, i32 7
+  %m8 = extractelement <16 x i8> %mask, i32 8
+  %s8 = extractelement <16 x i8> %src, i8 %m8
+  %v8 = insertelement <16 x i8> %v7, i8 %s8, i32 8
+  %m9 = extractelement <16 x i8> %mask, i32 9
+  %s9 = extractelement <16 x i8> %src, i8 %m9
+  %v9 = insertelement <16 x i8> %v8, i8 %s9, i32 9
+  %m10 = extractelement <16 x i8> %mask, i32 10
+  %s10 = extractelement <16 x i8> %src, i8 %m10
+  %v10 = insertelement <16 x i8> %v9, i8 %s10, i32 10
+  %m11 = extractelement <16 x i8> %mask, i32 11
+  %s11 = extractelement <16 x i8> %src, i8 %m11
+  %v11 = insertelement <16 x i8> %v10, i8 %s11, i32 11
+  %m12 = extractelement <16 x i8> %mask, i32 12
+  %s12 = extractelement <16 x i8> %src, i8 %m12
+  %v12 = insertelement <16 x i8> %v11, i8 %s12, i32 12
+  %m13 = extractelement <16 x i8> %mask, i32 13
+  %s13 = extractelement <16 x i8> %src, i8 %m13
+  %v13 = insertelement <16 x i8> %v12, i8 %s13, i32 13
+  %m14 = extractelement <16 x i8> %mask, i32 14
+  %s14 = extractelement <16 x i8> %src, i8 %m14
+  %v14 = insertelement <16 x i8> %v13, i8 %s14, i32 14
+  %m15 = extractelement <16 x i8> %mask, i32 15
+  %s15 = extractelement <16 x i8> %src, i8 %m15
+  %v15 = insertelement <16 x i8> %v14, i8 %s15, i32 15
+  ret <16 x i8> %v15
+}
+
+; CHECK-LABEL: swizzle_one_i16x8:
+; CHECK-NEXT:  .functype       swizzle_one_i16x8 (v128, v128) -> (v128)
+; CHECK-NOT:    swizzle
+; CHECK:        return
+define <8 x i16> @swizzle_one_i16x8(<8 x i16> %src, <8 x i16> %mask) {
+  %m0 = extractelement <8 x i16> %mask, i32 0
+  %s0 = extractelement <8 x i16> %src, i16 %m0
+  %v0 = insertelement <8 x i16> undef, i16 %s0, i32 0
+  ret <8 x i16> %v0
+}
+
+; CHECK-LABEL: mashup_swizzle_i8x16:
+; CHECK-NEXT:  .functype       mashup_swizzle_i8x16 (v128, v128, i32) -> (v128)
+; CHECK-NEXT:  v8x16.swizzle   $push[[L0:[0-9]+]]=, $0, $1
+; CHECK:       i8x16.replace_lane
+; CHECK:       i8x16.replace_lane
+; CHECK:       i8x16.replace_lane
+; CHECK:       i8x16.replace_lane
+; CHECK:       return
+define <16 x i8> @mashup_swizzle_i8x16(<16 x i8> %src, <16 x i8> %mask, i8 %splatted) {
+  ; swizzle 0
+  %m0 = extractelement <16 x i8> %mask, i32 0
+  %s0 = extractelement <16 x i8> %src, i8 %m0
+  %v0 = insertelement <16 x i8> undef, i8 %s0, i32 0
+  ; swizzle 7
+  %m1 = extractelement <16 x i8> %mask, i32 7
+  %s1 = extractelement <16 x i8> %src, i8 %m1
+  %v1 = insertelement <16 x i8> %v0, i8 %s1, i32 7
+  ; splat 3
+  %v2 = insertelement <16 x i8> %v1, i8 %splatted, i32 3
+  ; splat 12
+  %v3 = insertelement <16 x i8> %v2, i8 %splatted, i32 12
+  ; const 4
+  %v4 = insertelement <16 x i8> %v3, i8 42, i32 4
+  ; const 14
+  %v5 = insertelement <16 x i8> %v4, i8 42, i32 14
+  ret <16 x i8> %v5
+}
+
+; CHECK-LABEL: mashup_const_i8x16:
+; CHECK-NEXT:  .functype       mashup_const_i8x16 (v128, v128, i32) -> (v128)
+; CHECK:       v128.const      $push[[L0:[0-9]+]]=, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0
+; CHECK:       i8x16.replace_lane
+; CHECK:       i8x16.replace_lane
+; CHECK:       i8x16.replace_lane
+; CHECK:       return
+define <16 x i8> @mashup_const_i8x16(<16 x i8> %src, <16 x i8> %mask, i8 %splatted) {
+  ; swizzle 0
+  %m0 = extractelement <16 x i8> %mask, i32 0
+  %s0 = extractelement <16 x i8> %src, i8 %m0
+  %v0 = insertelement <16 x i8> undef, i8 %s0, i32 0
+  ; splat 3
+  %v1 = insertelement <16 x i8> %v0, i8 %splatted, i32 3
+  ; splat 12
+  %v2 = insertelement <16 x i8> %v1, i8 %splatted, i32 12
+  ; const 4
+  %v3 = insertelement <16 x i8> %v2, i8 42, i32 4
+  ; const 14
+  %v4 = insertelement <16 x i8> %v3, i8 42, i32 14
+  ret <16 x i8> %v4
+}
+
+; CHECK-LABEL: mashup_splat_i8x16:
+; CHECK-NEXT:  .functype       mashup_splat_i8x16 (v128, v128, i32) -> (v128)
+; CHECK:       i8x16.splat     $push[[L0:[0-9]+]]=, $2
+; CHECK:       i8x16.replace_lane
+; CHECK:       i8x16.replace_lane
+; CHECK:       return
+define <16 x i8> @mashup_splat_i8x16(<16 x i8> %src, <16 x i8> %mask, i8 %splatted) {
+  ; swizzle 0
+  %m0 = extractelement <16 x i8> %mask, i32 0
+  %s0 = extractelement <16 x i8> %src, i8 %m0
+  %v0 = insertelement <16 x i8> undef, i8 %s0, i32 0
+  ; splat 3
+  %v1 = insertelement <16 x i8> %v0, i8 %splatted, i32 3
+  ; splat 12
+  %v2 = insertelement <16 x i8> %v1, i8 %splatted, i32 12
+  ; const 4
+  %v3 = insertelement <16 x i8> %v2, i8 42, i32 4
+  ret <16 x i8> %v3
+}
+
 ; CHECK-LABEL: undef_const_insert_f32x4:
 ; CHECK-NEXT:  .functype       undef_const_insert_f32x4 () -> (v128)
-; CHECK-NEXT:  f32.const       $push[[L0:[0-9]+]]=, 0x1.5p5
-; CHECK-NEXT:  f32x4.splat     $push[[L1:[0-9]+]]=, $pop[[L0]]
-; CHECK-NEXT:  return          $pop[[L1]]
+; CHECK-NEXT:  v128.const      $push[[L0:[0-9]+]]=, 0x0p0, 0x1.5p5, 0x0p0, 0x0p0
+; CHECK-NEXT:  return          $pop[[L0]]
 define <4 x float> @undef_const_insert_f32x4() {
   %v = insertelement <4 x float> undef, float 42., i32 1
   ret <4 x float> %v
diff --git a/llvm/test/CodeGen/WebAssembly/simd-comparisons.ll b/llvm/test/CodeGen/WebAssembly/simd-comparisons.ll
index 2decd38fa..3b6af73eb 100644
--- a/llvm/test/CodeGen/WebAssembly/simd-comparisons.ll
+++ b/llvm/test/CodeGen/WebAssembly/simd-comparisons.ll
@@ -637,6 +637,156 @@ define <4 x i32> @compare_sext_uge_v4i32 (<4 x i32> %x, <4 x i32> %y) {
   ret <4 x i32> %res
 }
 
+; CHECK-LABEL: compare_eq_v2i64:
+; SIMD128-NEXT: .functype compare_eq_v2i64 (v128, v128) -> (v128){{$}}
+define <2 x i1> @compare_eq_v2i64 (<2 x i64> %x, <2 x i64> %y) {
+  %res = icmp eq <2 x i64> %x, %y
+  ret <2 x i1> %res
+}
+
+; CHECK-LABEL: compare_sext_eq_v2i64:
+; SIMD128-NEXT: .functype compare_sext_eq_v2i64 (v128, v128) -> (v128){{$}}
+define <2 x i64> @compare_sext_eq_v2i64 (<2 x i64> %x, <2 x i64> %y) {
+  %cmp = icmp eq <2 x i64> %x, %y
+  %res = sext <2 x i1> %cmp to <2 x i64>
+  ret <2 x i64> %res
+}
+
+; CHECK-LABEL: compare_ne_v2i64:
+; SIMD128-NEXT: .functype compare_ne_v2i64 (v128, v128) -> (v128){{$}}
+define <2 x i1> @compare_ne_v2i64 (<2 x i64> %x, <2 x i64> %y) {
+  %res = icmp ne <2 x i64> %x, %y
+  ret <2 x i1> %res
+}
+
+; CHECK-LABEL: compare_sext_ne_v2i64:
+; SIMD128-NEXT: .functype compare_sext_ne_v2i64 (v128, v128) -> (v128){{$}}
+define <2 x i64> @compare_sext_ne_v2i64 (<2 x i64> %x, <2 x i64> %y) {
+  %cmp = icmp ne <2 x i64> %x, %y
+  %res = sext <2 x i1> %cmp to <2 x i64>
+  ret <2 x i64> %res
+}
+
+; CHECK-LABEL: compare_slt_v2i64:
+; SIMD128-NEXT: .functype compare_slt_v2i64 (v128, v128) -> (v128){{$}}
+define <2 x i1> @compare_slt_v2i64 (<2 x i64> %x, <2 x i64> %y) {
+  %res = icmp slt <2 x i64> %x, %y
+  ret <2 x i1> %res
+}
+
+; CHECK-LABEL: compare_sext_slt_v2i64:
+; SIMD128-NEXT: .functype compare_sext_slt_v2i64 (v128, v128) -> (v128){{$}}
+define <2 x i64> @compare_sext_slt_v2i64 (<2 x i64> %x, <2 x i64> %y) {
+  %cmp = icmp slt <2 x i64> %x, %y
+  %res = sext <2 x i1> %cmp to <2 x i64>
+  ret <2 x i64> %res
+}
+
+; CHECK-LABEL: compare_ult_v2i64:
+; SIMD128-NEXT: .functype compare_ult_v2i64 (v128, v128) -> (v128){{$}}
+define <2 x i1> @compare_ult_v2i64 (<2 x i64> %x, <2 x i64> %y) {
+  %res = icmp ult <2 x i64> %x, %y
+  ret <2 x i1> %res
+}
+
+; CHECK-LABEL: compare_sext_ult_v2i64:
+; SIMD128-NEXT: .functype compare_sext_ult_v2i64 (v128, v128) -> (v128){{$}}
+define <2 x i64> @compare_sext_ult_v2i64 (<2 x i64> %x, <2 x i64> %y) {
+  %cmp = icmp ult <2 x i64> %x, %y
+  %res = sext <2 x i1> %cmp to <2 x i64>
+  ret <2 x i64> %res
+}
+
+; CHECK-LABEL: compare_sle_v2i64:
+; SIMD128-NEXT: .functype compare_sle_v2i64 (v128, v128) -> (v128){{$}}
+define <2 x i1> @compare_sle_v2i64 (<2 x i64> %x, <2 x i64> %y) {
+  %res = icmp sle <2 x i64> %x, %y
+  ret <2 x i1> %res
+}
+
+; CHECK-LABEL: compare_sext_sle_v2i64:
+; SIMD128-NEXT: .functype compare_sext_sle_v2i64 (v128, v128) -> (v128){{$}}
+define <2 x i64> @compare_sext_sle_v2i64 (<2 x i64> %x, <2 x i64> %y) {
+  %cmp = icmp sle <2 x i64> %x, %y
+  %res = sext <2 x i1> %cmp to <2 x i64>
+  ret <2 x i64> %res
+}
+
+; CHECK-LABEL: compare_ule_v2i64:
+; SIMD128-NEXT: .functype compare_ule_v2i64 (v128, v128) -> (v128){{$}}
+define <2 x i1> @compare_ule_v2i64 (<2 x i64> %x, <2 x i64> %y) {
+  %res = icmp ule <2 x i64> %x, %y
+  ret <2 x i1> %res
+}
+
+; CHECK-LABEL: compare_sext_ule_v2i64:
+; SIMD128-NEXT: .functype compare_sext_ule_v2i64 (v128, v128) -> (v128){{$}}
+define <2 x i64> @compare_sext_ule_v2i64 (<2 x i64> %x, <2 x i64> %y) {
+  %cmp = icmp ule <2 x i64> %x, %y
+  %res = sext <2 x i1> %cmp to <2 x i64>
+  ret <2 x i64> %res
+}
+
+; CHECK-LABEL: compare_sgt_v2i64:
+; SIMD128-NEXT: .functype compare_sgt_v2i64 (v128, v128) -> (v128){{$}}
+define <2 x i1> @compare_sgt_v2i64 (<2 x i64> %x, <2 x i64> %y) {
+  %res = icmp sgt <2 x i64> %x, %y
+  ret <2 x i1> %res
+}
+
+; CHECK-LABEL: compare_sext_sgt_v2i64:
+; SIMD128-NEXT: .functype compare_sext_sgt_v2i64 (v128, v128) -> (v128){{$}}
+define <2 x i64> @compare_sext_sgt_v2i64 (<2 x i64> %x, <2 x i64> %y) {
+  %cmp = icmp sgt <2 x i64> %x, %y
+  %res = sext <2 x i1> %cmp to <2 x i64>
+  ret <2 x i64> %res
+}
+
+; CHECK-LABEL: compare_ugt_v2i64:
+; SIMD128-NEXT: .functype compare_ugt_v2i64 (v128, v128) -> (v128){{$}}
+define <2 x i1> @compare_ugt_v2i64 (<2 x i64> %x, <2 x i64> %y) {
+  %res = icmp ugt <2 x i64> %x, %y
+  ret <2 x i1> %res
+}
+
+; CHECK-LABEL: compare_sext_ugt_v2i64:
+; SIMD128-NEXT: .functype compare_sext_ugt_v2i64 (v128, v128) -> (v128){{$}}
+define <2 x i64> @compare_sext_ugt_v2i64 (<2 x i64> %x, <2 x i64> %y) {
+  %cmp = icmp ugt <2 x i64> %x, %y
+  %res = sext <2 x i1> %cmp to <2 x i64>
+  ret <2 x i64> %res
+}
+
+; CHECK-LABEL: compare_sge_v2i64:
+; SIMD128-NEXT: .functype compare_sge_v2i64 (v128, v128) -> (v128){{$}}
+define <2 x i1> @compare_sge_v2i64 (<2 x i64> %x, <2 x i64> %y) {
+  %res = icmp sge <2 x i64> %x, %y
+  ret <2 x i1> %res
+}
+
+; CHECK-LABEL: compare_sext_sge_v2i64:
+; SIMD128-NEXT: .functype compare_sext_sge_v2i64 (v128, v128) -> (v128){{$}}
+define <2 x i64> @compare_sext_sge_v2i64 (<2 x i64> %x, <2 x i64> %y) {
+  %cmp = icmp sge <2 x i64> %x, %y
+  %res = sext <2 x i1> %cmp to <2 x i64>
+  ret <2 x i64> %res
+}
+
+; CHECK-LABEL: compare_uge_v2i64:
+; SIMD128-NEXT: .functype compare_uge_v2i64 (v128, v128) -> (v128){{$}}
+define <2 x i1> @compare_uge_v2i64 (<2 x i64> %x, <2 x i64> %y) {
+  %res = icmp uge <2 x i64> %x, %y
+  ret <2 x i1> %res
+}
+
+; CHECK-LABEL: compare_sext_uge_v2i64:
+; SIMD128-NEXT: .functype compare_sext_uge_v2i64 (v128, v128) -> (v128){{$}}
+define <2 x i64> @compare_sext_uge_v2i64 (<2 x i64> %x, <2 x i64> %y) {
+  %cmp = icmp uge <2 x i64> %x, %y
+  %res = sext <2 x i1> %cmp to <2 x i64>
+  ret <2 x i64> %res
+}
+
 ; CHECK-LABEL: compare_oeq_v4f32:
 ; NO-SIMD128-NOT: f32x4
 ; SIMD128-NEXT: .functype compare_oeq_v4f32 (v128, v128) -> (v128){{$}}
diff --git a/llvm/test/CodeGen/WebAssembly/simd-extended-extract.ll b/llvm/test/CodeGen/WebAssembly/simd-extended-extract.ll
index b6e35f5f9..149b1842b 100644
--- a/llvm/test/CodeGen/WebAssembly/simd-extended-extract.ll
+++ b/llvm/test/CodeGen/WebAssembly/simd-extended-extract.ll
@@ -16,39 +16,58 @@
 target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
 target triple = "wasm32-unknown-unknown"
 
-; CHECK-LABEL: foo:
-
-; CHECK: i32.load8_u
-; CHECK: i32x4.splat
-; CHECK: i32.load8_u
-; CHECK: i32x4.replace_lane   1
-; CHECK: i32.load8_u
-; CHECK: i32x4.replace_lane   2
-; CHECK: i32.load8_u
-; CHECK: i32x4.replace_lane   3
-
-; CHECK: i8x16.extract_lane_s 0
-; CHECK: f64.convert_i32_s
-; CHECK: f32.demote_f64
-; CHECK: f32x4.splat
-
-; CHECK: i8x16.extract_lane_s 4
-; CHECK: f64.convert_i32_s
-; CHECK: f32.demote_f64
-; CHECK: f32x4.replace_lane   1
-
-; CHECK: i8x16.extract_lane_s 8
-; CHECK: f64.convert_i32_s
-; CHECK: f32.demote_f64
-; CHECK: f32x4.replace_lane   2
-
-; CHECK: i8x16.extract_lane_s 12
-; CHECK: f64.convert_i32_s
-; CHECK: f32.demote_f64
-; CHECK: f32x4.replace_lane   3
-
-; CHECK: v128.store
 define void @foo(<4 x i8>* %p) {
+; CHECK-LABEL: foo:
+; CHECK:         .functype foo (i32) -> ()
+; CHECK-NEXT:    i32.load8_u 0
+; CHECK-NEXT:    i32x4.splat
+; CHECK-NEXT:    local.tee
+; CHECK-NEXT:    i8x16.extract_lane_s 0
+; CHECK-NEXT:    f64.convert_i32_s
+; CHECK-NEXT:    f64.const 0x0p0
+; CHECK-NEXT:    f64.mul
+; CHECK-NEXT:    f64.const 0x0p0
+; CHECK-NEXT:    f64.add
+; CHECK-NEXT:    f32.demote_f64
+; CHECK-NEXT:    f32x4.splat
+; CHECK-NEXT:    i32.load8_u 1
+; CHECK-NEXT:    i32x4.replace_lane 1
+; CHECK-NEXT:    local.tee
+; CHECK-NEXT:    i8x16.extract_lane_s 4
+; CHECK-NEXT:    f64.convert_i32_s
+; CHECK-NEXT:    f64.const 0x0p0
+; CHECK-NEXT:    f64.mul
+; CHECK-NEXT:    f64.const 0x0p0
+; CHECK-NEXT:    f64.add
+; CHECK-NEXT:    f32.demote_f64
+; CHECK-NEXT:    f32x4.replace_lane 1
+; CHECK-NEXT:    i32.const 2
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    i32.load8_u 0
+; CHECK-NEXT:    i32x4.replace_lane 2
+; CHECK-NEXT:    local.tee
+; CHECK-NEXT:    i8x16.extract_lane_s 8
+; CHECK-NEXT:    f64.convert_i32_s
+; CHECK-NEXT:    f64.const 0x0p0
+; CHECK-NEXT:    f64.mul
+; CHECK-NEXT:    f64.const 0x0p0
+; CHECK-NEXT:    f64.add
+; CHECK-NEXT:    f32.demote_f64
+; CHECK-NEXT:    f32x4.replace_lane 2
+; CHECK-NEXT:    i32.const 3
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    i32.load8_u 0
+; CHECK-NEXT:    i32x4.replace_lane 3
+; CHECK-NEXT:    i8x16.extract_lane_s 12
+; CHECK-NEXT:    f64.convert_i32_s
+; CHECK-NEXT:    f64.const 0x0p0
+; CHECK-NEXT:    f64.mul
+; CHECK-NEXT:    f64.const 0x0p0
+; CHECK-NEXT:    f64.add
+; CHECK-NEXT:    f32.demote_f64
+; CHECK-NEXT:    f32x4.replace_lane 3
+; CHECK-NEXT:    v128.store 0
+; CHECK-NEXT:    return
   %1 = load <4 x i8>, <4 x i8>* %p
   %2 = sitofp <4 x i8> %1 to <4 x double>
   %3 = fmul <4 x double> zeroinitializer, %2
diff --git a/llvm/test/CodeGen/WebAssembly/simd-intrinsics.ll b/llvm/test/CodeGen/WebAssembly/simd-intrinsics.ll
index 53c98d272..b6680dd36 100644
--- a/llvm/test/CodeGen/WebAssembly/simd-intrinsics.ll
+++ b/llvm/test/CodeGen/WebAssembly/simd-intrinsics.ll
@@ -11,6 +11,16 @@ target triple = "wasm32-unknown-unknown"
 ; ==============================================================================
 ; 16 x i8
 ; ==============================================================================
+; CHECK-LABEL: swizzle_v16i8:
+; SIMD128-NEXT: .functype swizzle_v16i8 (v128, v128) -> (v128){{$}}
+; SIMD128-NEXT: v8x16.swizzle $push[[R:[0-9]+]]=, $0, $1{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+declare <16 x i8> @llvm.wasm.swizzle(<16 x i8>, <16 x i8>)
+define <16 x i8> @swizzle_v16i8(<16 x i8> %x, <16 x i8> %y) {
+  %a = call <16 x i8> @llvm.wasm.swizzle(<16 x i8> %x, <16 x i8> %y)
+  ret <16 x i8> %a
+}
+
 ; CHECK-LABEL: add_sat_s_v16i8:
 ; SIMD128-NEXT: .functype add_sat_s_v16i8 (v128, v128) -> (v128){{$}}
 ; SIMD128-NEXT: i8x16.add_saturate_s $push[[R:[0-9]+]]=, $0, $1{{$}}
@@ -55,6 +65,16 @@ define <16 x i8> @sub_sat_u_v16i8(<16 x i8> %x, <16 x i8> %y) {
   ret <16 x i8> %a
 }
 
+; CHECK-LABEL: avgr_u_v16i8:
+; SIMD128-NEXT: .functype avgr_u_v16i8 (v128, v128) -> (v128){{$}}
+; SIMD128-NEXT: i8x16.avgr_u $push[[R:[0-9]+]]=, $0, $1{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+declare <16 x i8> @llvm.wasm.avgr.unsigned.v16i8(<16 x i8>, <16 x i8>)
+define <16 x i8> @avgr_u_v16i8(<16 x i8> %x, <16 x i8> %y) {
+  %a = call <16 x i8> @llvm.wasm.avgr.unsigned.v16i8(<16 x i8> %x, <16 x i8> %y)
+  ret <16 x i8> %a
+}
+
 ; CHECK-LABEL: any_v16i8:
 ; SIMD128-NEXT: .functype any_v16i8 (v128) -> (i32){{$}}
 ; SIMD128-NEXT: i8x16.any_true $push[[R:[0-9]+]]=, $0{{$}}
@@ -87,6 +107,30 @@ define <16 x i8> @bitselect_v16i8(<16 x i8> %v1, <16 x i8> %v2, <16 x i8> %c) {
   ret <16 x i8> %a
 }
 
+; CHECK-LABEL: narrow_signed_v16i8:
+; SIMD128-NEXT: .functype narrow_signed_v16i8 (v128, v128) -> (v128){{$}}
+; SIMD128-NEXT: i8x16.narrow_i16x8_s $push[[R:[0-9]+]]=, $0, $1{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+declare <16 x i8> @llvm.wasm.narrow.signed.v16i8.v8i16(<8 x i16>, <8 x i16>)
+define <16 x i8> @narrow_signed_v16i8(<8 x i16> %low, <8 x i16> %high) {
+  %a = call <16 x i8> @llvm.wasm.narrow.signed.v16i8.v8i16(
+    <8 x i16> %low, <8 x i16> %high
+  )
+  ret <16 x i8> %a
+}
+
+; CHECK-LABEL: narrow_unsigned_v16i8:
+; SIMD128-NEXT: .functype narrow_unsigned_v16i8 (v128, v128) -> (v128){{$}}
+; SIMD128-NEXT: i8x16.narrow_i16x8_u $push[[R:[0-9]+]]=, $0, $1{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+declare <16 x i8> @llvm.wasm.narrow.unsigned.v16i8.v8i16(<8 x i16>, <8 x i16>)
+define <16 x i8> @narrow_unsigned_v16i8(<8 x i16> %low, <8 x i16> %high) {
+  %a = call <16 x i8> @llvm.wasm.narrow.unsigned.v16i8.v8i16(
+    <8 x i16> %low, <8 x i16> %high
+  )
+  ret <16 x i8> %a
+}
+
 ; ==============================================================================
 ; 8 x i16
 ; ==============================================================================
@@ -134,6 +178,16 @@ define <8 x i16> @sub_sat_u_v8i16(<8 x i16> %x, <8 x i16> %y) {
   ret <8 x i16> %a
 }
 
+; CHECK-LABEL: avgr_u_v8i16:
+; SIMD128-NEXT: .functype avgr_u_v8i16 (v128, v128) -> (v128){{$}}
+; SIMD128-NEXT: i16x8.avgr_u $push[[R:[0-9]+]]=, $0, $1{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+declare <8 x i16> @llvm.wasm.avgr.unsigned.v8i16(<8 x i16>, <8 x i16>)
+define <8 x i16> @avgr_u_v8i16(<8 x i16> %x, <8 x i16> %y) {
+  %a = call <8 x i16> @llvm.wasm.avgr.unsigned.v8i16(<8 x i16> %x, <8 x i16> %y)
+  ret <8 x i16> %a
+}
+
 ; CHECK-LABEL: any_v8i16:
 ; SIMD128-NEXT: .functype any_v8i16 (v128) -> (i32){{$}}
 ; SIMD128-NEXT: i16x8.any_true $push[[R:[0-9]+]]=, $0{{$}}
@@ -166,9 +220,83 @@ define <8 x i16> @bitselect_v8i16(<8 x i16> %v1, <8 x i16> %v2, <8 x i16> %c) {
   ret <8 x i16> %a
 }
 
+; CHECK-LABEL: narrow_signed_v8i16:
+; SIMD128-NEXT: .functype narrow_signed_v8i16 (v128, v128) -> (v128){{$}}
+; SIMD128-NEXT: i16x8.narrow_i32x4_s $push[[R:[0-9]+]]=, $0, $1{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+declare <8 x i16> @llvm.wasm.narrow.signed.v8i16.v4i32(<4 x i32>, <4 x i32>)
+define <8 x i16> @narrow_signed_v8i16(<4 x i32> %low, <4 x i32> %high) {
+  %a = call <8 x i16> @llvm.wasm.narrow.signed.v8i16.v4i32(
+    <4 x i32> %low, <4 x i32> %high
+  )
+  ret <8 x i16> %a
+}
+
+; CHECK-LABEL: narrow_unsigned_v8i16:
+; SIMD128-NEXT: .functype narrow_unsigned_v8i16 (v128, v128) -> (v128){{$}}
+; SIMD128-NEXT: i16x8.narrow_i32x4_u $push[[R:[0-9]+]]=, $0, $1{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+declare <8 x i16> @llvm.wasm.narrow.unsigned.v8i16.v4i32(<4 x i32>, <4 x i32>)
+define <8 x i16> @narrow_unsigned_v8i16(<4 x i32> %low, <4 x i32> %high) {
+  %a = call <8 x i16> @llvm.wasm.narrow.unsigned.v8i16.v4i32(
+    <4 x i32> %low, <4 x i32> %high
+  )
+  ret <8 x i16> %a
+}
+
+; CHECK-LABEL: widen_low_signed_v8i16:
+; SIMD128-NEXT: .functype widen_low_signed_v8i16 (v128) -> (v128){{$}}
+; SIMD128-NEXT: i16x8.widen_low_i8x16_s $push[[R:[0-9]+]]=, $0{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+declare <8 x i16> @llvm.wasm.widen.low.signed.v8i16.v16i8(<16 x i8>)
+define <8 x i16> @widen_low_signed_v8i16(<16 x i8> %v) {
+  %a = call <8 x i16> @llvm.wasm.widen.low.signed.v8i16.v16i8(<16 x i8> %v)
+  ret <8 x i16> %a
+}
+
+; CHECK-LABEL: widen_high_signed_v8i16:
+; SIMD128-NEXT: .functype widen_high_signed_v8i16 (v128) -> (v128){{$}}
+; SIMD128-NEXT: i16x8.widen_high_i8x16_s $push[[R:[0-9]+]]=, $0{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+declare <8 x i16> @llvm.wasm.widen.high.signed.v8i16.v16i8(<16 x i8>)
+define <8 x i16> @widen_high_signed_v8i16(<16 x i8> %v) {
+  %a = call <8 x i16> @llvm.wasm.widen.high.signed.v8i16.v16i8(<16 x i8> %v)
+  ret <8 x i16> %a
+}
+
+; CHECK-LABEL: widen_low_unsigned_v8i16:
+; SIMD128-NEXT: .functype widen_low_unsigned_v8i16 (v128) -> (v128){{$}}
+; SIMD128-NEXT: i16x8.widen_low_i8x16_u $push[[R:[0-9]+]]=, $0{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+declare <8 x i16> @llvm.wasm.widen.low.unsigned.v8i16.v16i8(<16 x i8>)
+define <8 x i16> @widen_low_unsigned_v8i16(<16 x i8> %v) {
+  %a = call <8 x i16> @llvm.wasm.widen.low.unsigned.v8i16.v16i8(<16 x i8> %v)
+  ret <8 x i16> %a
+}
+
+; CHECK-LABEL: widen_high_unsigned_v8i16:
+; SIMD128-NEXT: .functype widen_high_unsigned_v8i16 (v128) -> (v128){{$}}
+; SIMD128-NEXT: i16x8.widen_high_i8x16_u $push[[R:[0-9]+]]=, $0{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+declare <8 x i16> @llvm.wasm.widen.high.unsigned.v8i16.v16i8(<16 x i8>)
+define <8 x i16> @widen_high_unsigned_v8i16(<16 x i8> %v) {
+  %a = call <8 x i16> @llvm.wasm.widen.high.unsigned.v8i16.v16i8(<16 x i8> %v)
+  ret <8 x i16> %a
+}
+
 ; ==============================================================================
 ; 4 x i32
 ; ==============================================================================
+; CHECK-LABEL: dot:
+; SIMD128-NEXT: .functype dot (v128, v128) -> (v128){{$}}
+; SIMD128-NEXT: i32x4.dot_i16x8_s $push[[R:[0-9]+]]=, $0, $1{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+declare <4 x i32> @llvm.wasm.dot(<8 x i16>, <8 x i16>)
+define <4 x i32> @dot(<8 x i16> %x, <8 x i16> %y) {
+  %a = call <4 x i32> @llvm.wasm.dot(<8 x i16> %x, <8 x i16> %y)
+  ret <4 x i32> %a
+}
+
 ; CHECK-LABEL: any_v4i32:
 ; SIMD128-NEXT: .functype any_v4i32 (v128) -> (i32){{$}}
 ; SIMD128-NEXT: i32x4.any_true $push[[R:[0-9]+]]=, $0{{$}}
@@ -223,6 +351,46 @@ define <4 x i32> @trunc_sat_u_v4i32(<4 x float> %x) {
   ret <4 x i32> %a
 }
 
+; CHECK-LABEL: widen_low_signed_v4i32:
+; SIMD128-NEXT: .functype widen_low_signed_v4i32 (v128) -> (v128){{$}}
+; SIMD128-NEXT: i32x4.widen_low_i16x8_s $push[[R:[0-9]+]]=, $0{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+declare <4 x i32> @llvm.wasm.widen.low.signed.v4i32.v8i16(<8 x i16>)
+define <4 x i32> @widen_low_signed_v4i32(<8 x i16> %v) {
+  %a = call <4 x i32> @llvm.wasm.widen.low.signed.v4i32.v8i16(<8 x i16> %v)
+  ret <4 x i32> %a
+}
+
+; CHECK-LABEL: widen_high_signed_v4i32:
+; SIMD128-NEXT: .functype widen_high_signed_v4i32 (v128) -> (v128){{$}}
+; SIMD128-NEXT: i32x4.widen_high_i16x8_s $push[[R:[0-9]+]]=, $0{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+declare <4 x i32> @llvm.wasm.widen.high.signed.v4i32.v8i16(<8 x i16>)
+define <4 x i32> @widen_high_signed_v4i32(<8 x i16> %v) {
+  %a = call <4 x i32> @llvm.wasm.widen.high.signed.v4i32.v8i16(<8 x i16> %v)
+  ret <4 x i32> %a
+}
+
+; CHECK-LABEL: widen_low_unsigned_v4i32:
+; SIMD128-NEXT: .functype widen_low_unsigned_v4i32 (v128) -> (v128){{$}}
+; SIMD128-NEXT: i32x4.widen_low_i16x8_u $push[[R:[0-9]+]]=, $0{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+declare <4 x i32> @llvm.wasm.widen.low.unsigned.v4i32.v8i16(<8 x i16>)
+define <4 x i32> @widen_low_unsigned_v4i32(<8 x i16> %v) {
+  %a = call <4 x i32> @llvm.wasm.widen.low.unsigned.v4i32.v8i16(<8 x i16> %v)
+  ret <4 x i32> %a
+}
+
+; CHECK-LABEL: widen_high_unsigned_v4i32:
+; SIMD128-NEXT: .functype widen_high_unsigned_v4i32 (v128) -> (v128){{$}}
+; SIMD128-NEXT: i32x4.widen_high_i16x8_u $push[[R:[0-9]+]]=, $0{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+declare <4 x i32> @llvm.wasm.widen.high.unsigned.v4i32.v8i16(<8 x i16>)
+define <4 x i32> @widen_high_unsigned_v4i32(<8 x i16> %v) {
+  %a = call <4 x i32> @llvm.wasm.widen.high.unsigned.v4i32.v8i16(<8 x i16> %v)
+  ret <4 x i32> %a
+}
+
 ; ==============================================================================
 ; 2 x i64
 ; ==============================================================================
@@ -290,11 +458,35 @@ define <2 x i64> @trunc_sat_u_v2i64(<2 x double> %x) {
 declare <4 x float> @llvm.wasm.bitselect.v4f32(<4 x float>, <4 x float>, <4 x float>)
 define <4 x float> @bitselect_v4f32(<4 x float> %v1, <4 x float> %v2, <4 x float> %c) {
   %a = call <4 x float> @llvm.wasm.bitselect.v4f32(
-     <4 x float> %v1, <4 x float> %v2, <4 x float> %c
+    <4 x float> %v1, <4 x float> %v2, <4 x float> %c
   )
   ret <4 x float> %a
 }
 
+; CHECK-LABEL: qfma_v4f32:
+; SIMD128-NEXT: .functype qfma_v4f32 (v128, v128, v128) -> (v128){{$}}
+; SIMD128-NEXT: f32x4.qfma $push[[R:[0-9]+]]=, $0, $1, $2{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+declare <4 x float> @llvm.wasm.qfma.v4f32(<4 x float>, <4 x float>, <4 x float>)
+define <4 x float> @qfma_v4f32(<4 x float> %a, <4 x float> %b, <4 x float> %c) {
+  %v = call <4 x float> @llvm.wasm.qfma.v4f32(
+    <4 x float> %a, <4 x float> %b, <4 x float> %c
+  )
+  ret <4 x float> %v
+}
+
+; CHECK-LABEL: qfms_v4f32:
+; SIMD128-NEXT: .functype qfms_v4f32 (v128, v128, v128) -> (v128){{$}}
+; SIMD128-NEXT: f32x4.qfms $push[[R:[0-9]+]]=, $0, $1, $2{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+declare <4 x float> @llvm.wasm.qfms.v4f32(<4 x float>, <4 x float>, <4 x float>)
+define <4 x float> @qfms_v4f32(<4 x float> %a, <4 x float> %b, <4 x float> %c) {
+  %v = call <4 x float> @llvm.wasm.qfms.v4f32(
+    <4 x float> %a, <4 x float> %b, <4 x float> %c
+  )
+  ret <4 x float> %v
+}
+
 ; ==============================================================================
 ; 2 x f64
 ; ==============================================================================
@@ -309,3 +501,27 @@ define <2 x double> @bitselect_v2f64(<2 x double> %v1, <2 x double> %v2, <2 x do
   )
   ret <2 x double> %a
 }
+
+; CHECK-LABEL: qfma_v2f64:
+; SIMD128-NEXT: .functype qfma_v2f64 (v128, v128, v128) -> (v128){{$}}
+; SIMD128-NEXT: f64x2.qfma $push[[R:[0-9]+]]=, $0, $1, $2{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+declare <2 x double> @llvm.wasm.qfma.v2f64(<2 x double>, <2 x double>, <2 x double>)
+define <2 x double> @qfma_v2f64(<2 x double> %a, <2 x double> %b, <2 x double> %c) {
+  %v = call <2 x double> @llvm.wasm.qfma.v2f64(
+    <2 x double> %a, <2 x double> %b, <2 x double> %c
+  )
+  ret <2 x double> %v
+}
+
+; CHECK-LABEL: qfms_v2f64:
+; SIMD128-NEXT: .functype qfms_v2f64 (v128, v128, v128) -> (v128){{$}}
+; SIMD128-NEXT: f64x2.qfms $push[[R:[0-9]+]]=, $0, $1, $2{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+declare <2 x double> @llvm.wasm.qfms.v2f64(<2 x double>, <2 x double>, <2 x double>)
+define <2 x double> @qfms_v2f64(<2 x double> %a, <2 x double> %b, <2 x double> %c) {
+  %v = call <2 x double> @llvm.wasm.qfms.v2f64(
+    <2 x double> %a, <2 x double> %b, <2 x double> %c
+  )
+  ret <2 x double> %v
+}
diff --git a/llvm/test/CodeGen/WebAssembly/simd-load-splat.ll b/llvm/test/CodeGen/WebAssembly/simd-load-splat.ll
new file mode 100644
index 000000000..4e693c285
--- /dev/null
+++ b/llvm/test/CodeGen/WebAssembly/simd-load-splat.ll
@@ -0,0 +1,21 @@
+; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-keep-registers -wasm-disable-explicit-locals -mattr=+unimplemented-simd128 | FileCheck %s
+
+; Regression test for an ISel failure when a splatted load had more
+; than one use. The main tests for load_splat are in simd-offset.ll.
+
+target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
+target triple = "wasm32-unknown-unknown"
+
+; CHECK-LABEL: load_splat:
+; CHECK-NEXT: .functype load_splat (i32, i32) -> (i32)
+; CHECK-NEXT: i32.load8_u $[[E:[0-9]+]]=, 0($0){{$}}
+; CHECK-NEXT: v8x16.load_splat $push[[V:[0-9]+]]=, 0($0){{$}}
+; CHECK-NEXT: v128.store 0($1), $pop[[V]]{{$}}
+; CHECK-NEXT: return $[[E]]{{$}}
+define i8 @load_splat(i8* %p, <16 x i8>* %out) {
+  %e = load i8, i8* %p
+  %v1 = insertelement <16 x i8> undef, i8 %e, i32 0
+  %v2 = shufflevector <16 x i8> %v1, <16 x i8> undef, <16 x i32> zeroinitializer
+  store <16 x i8> %v2, <16 x i8>* %out
+  ret i8 %e
+}
diff --git a/llvm/test/CodeGen/WebAssembly/simd-offset.ll b/llvm/test/CodeGen/WebAssembly/simd-offset.ll
index 61177ae92..03b6ca7c2 100644
--- a/llvm/test/CodeGen/WebAssembly/simd-offset.ll
+++ b/llvm/test/CodeGen/WebAssembly/simd-offset.ll
@@ -20,6 +20,19 @@ define <16 x i8> @load_v16i8(<16 x i8>* %p) {
   ret <16 x i8> %v
 }
 
+; CHECK-LABEL: load_splat_v16i8:
+; SIMD128-VM-NOT: v8x16.load_splat
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v16i8 (i32) -> (v128){{$}}
+; SIMD128-NEXT: v8x16.load_splat $push[[R:[0-9]+]]=, 0($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <16 x i8> @load_splat_v16i8(i8* %p) {
+  %e = load i8, i8* %p
+  %v1 = insertelement <16 x i8> undef, i8 %e, i32 0
+  %v2 = shufflevector <16 x i8> %v1, <16 x i8> undef, <16 x i32> zeroinitializer
+  ret <16 x i8> %v2
+}
+
 ; CHECK-LABEL: load_v16i8_with_folded_offset:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype load_v16i8_with_folded_offset (i32) -> (v128){{$}}
@@ -33,6 +46,21 @@ define <16 x i8> @load_v16i8_with_folded_offset(<16 x i8>* %p) {
   ret <16 x i8> %v
 }
 
+; CHECK-LABEL: load_splat_v16i8_with_folded_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v16i8_with_folded_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: v8x16.load_splat $push[[R:[0-9]+]]=, 16($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <16 x i8> @load_splat_v16i8_with_folded_offset(i8* %p) {
+  %q = ptrtoint i8* %p to i32
+  %r = add nuw i32 %q, 16
+  %s = inttoptr i32 %r to i8*
+  %e = load i8, i8* %s
+  %v1 = insertelement <16 x i8> undef, i8 %e, i32 0
+  %v2 = shufflevector <16 x i8> %v1, <16 x i8> undef, <16 x i32> zeroinitializer
+  ret <16 x i8> %v2
+}
+
 ; CHECK-LABEL: load_v16i8_with_folded_gep_offset:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype load_v16i8_with_folded_gep_offset (i32) -> (v128){{$}}
@@ -44,6 +72,19 @@ define <16 x i8> @load_v16i8_with_folded_gep_offset(<16 x i8>* %p) {
   ret <16 x i8> %v
 }
 
+; CHECK-LABEL: load_splat_v16i8_with_folded_gep_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v16i8_with_folded_gep_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: v8x16.load_splat $push[[R:[0-9]+]]=, 1($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <16 x i8> @load_splat_v16i8_with_folded_gep_offset(i8* %p) {
+  %s = getelementptr inbounds i8, i8* %p, i32 1
+  %e = load i8, i8* %s
+  %v1 = insertelement <16 x i8> undef, i8 %e, i32 0
+  %v2 = shufflevector <16 x i8> %v1, <16 x i8> undef, <16 x i32> zeroinitializer
+  ret <16 x i8> %v2
+}
+
 ; CHECK-LABEL: load_v16i8_with_unfolded_gep_negative_offset:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype load_v16i8_with_unfolded_gep_negative_offset (i32) -> (v128){{$}}
@@ -57,6 +98,21 @@ define <16 x i8> @load_v16i8_with_unfolded_gep_negative_offset(<16 x i8>* %p) {
   ret <16 x i8> %v
 }
 
+; CHECK-LABEL: load_splat_v16i8_with_unfolded_gep_negative_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v16i8_with_unfolded_gep_negative_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, -1{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: v8x16.load_splat $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <16 x i8> @load_splat_v16i8_with_unfolded_gep_negative_offset(i8* %p) {
+  %s = getelementptr inbounds i8, i8* %p, i32 -1
+  %e = load i8, i8* %s
+  %v1 = insertelement <16 x i8> undef, i8 %e, i32 0
+  %v2 = shufflevector <16 x i8> %v1, <16 x i8> undef, <16 x i32> zeroinitializer
+  ret <16 x i8> %v2
+}
+
 ; CHECK-LABEL: load_v16i8_with_unfolded_offset:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype load_v16i8_with_unfolded_offset (i32) -> (v128){{$}}
@@ -72,6 +128,23 @@ define <16 x i8> @load_v16i8_with_unfolded_offset(<16 x i8>* %p) {
   ret <16 x i8> %v
 }
 
+; CHECK-LABEL: load_splat_v16i8_with_unfolded_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v16i8_with_unfolded_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 16{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: v8x16.load_splat $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <16 x i8> @load_splat_v16i8_with_unfolded_offset(i8* %p) {
+  %q = ptrtoint i8* %p to i32
+  %r = add nsw i32 %q, 16
+  %s = inttoptr i32 %r to i8*
+  %e = load i8, i8* %s
+  %v1 = insertelement <16 x i8> undef, i8 %e, i32 0
+  %v2 = shufflevector <16 x i8> %v1, <16 x i8> undef, <16 x i32> zeroinitializer
+  ret <16 x i8> %v2
+}
+
 ; CHECK-LABEL: load_v16i8_with_unfolded_gep_offset:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype load_v16i8_with_unfolded_gep_offset (i32) -> (v128){{$}}
@@ -85,6 +158,21 @@ define <16 x i8> @load_v16i8_with_unfolded_gep_offset(<16 x i8>* %p) {
   ret <16 x i8> %v
 }
 
+; CHECK-LABEL: load_splat_v16i8_with_unfolded_gep_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v16i8_with_unfolded_gep_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 1{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: v8x16.load_splat $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <16 x i8> @load_splat_v16i8_with_unfolded_gep_offset(i8* %p) {
+  %s = getelementptr i8, i8* %p, i32 1
+  %e = load i8, i8* %s
+  %v1 = insertelement <16 x i8> undef, i8 %e, i32 0
+  %v2 = shufflevector <16 x i8> %v1, <16 x i8> undef, <16 x i32> zeroinitializer
+  ret <16 x i8> %v2
+}
+
 ; CHECK-LABEL: load_v16i8_from_numeric_address:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype load_v16i8_from_numeric_address () -> (v128){{$}}
@@ -97,6 +185,20 @@ define <16 x i8> @load_v16i8_from_numeric_address() {
   ret <16 x i8> %v
 }
 
+; CHECK-LABEL: load_splat_v16i8_from_numeric_address:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v16i8_from_numeric_address () -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
+; SIMD128-NEXT: v8x16.load_splat $push[[R:[0-9]+]]=, 32($pop[[L0]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <16 x i8> @load_splat_v16i8_from_numeric_address() {
+  %s = inttoptr i32 32 to i8*
+  %e = load i8, i8* %s
+  %v1 = insertelement <16 x i8> undef, i8 %e, i32 0
+  %v2 = shufflevector <16 x i8> %v1, <16 x i8> undef, <16 x i32> zeroinitializer
+  ret <16 x i8> %v2
+}
+
 ; CHECK-LABEL: load_v16i8_from_global_address:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype load_v16i8_from_global_address () -> (v128){{$}}
@@ -109,6 +211,20 @@ define <16 x i8> @load_v16i8_from_global_address() {
   ret <16 x i8> %v
 }
 
+; CHECK-LABEL: load_splat_v16i8_from_global_address:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v16i8_from_global_address () -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
+; SIMD128-NEXT: v8x16.load_splat $push[[R:[0-9]+]]=, gv_i8($pop[[L0]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+@gv_i8 = global i8 42
+define <16 x i8> @load_splat_v16i8_from_global_address() {
+  %e = load i8, i8* @gv_i8
+  %v1 = insertelement <16 x i8> undef, i8 %e, i32 0
+  %v2 = shufflevector <16 x i8> %v1, <16 x i8> undef, <16 x i32> zeroinitializer
+  ret <16 x i8> %v2
+}
+
 ; CHECK-LABEL: store_v16i8:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype store_v16i8 (v128, i32) -> (){{$}}
@@ -210,6 +326,50 @@ define <8 x i16> @load_v8i16(<8 x i16>* %p) {
   ret <8 x i16> %v
 }
 
+; CHECK-LABEL: load_splat_v8i16:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v8i16 (i32) -> (v128){{$}}
+; SIMD128-NEXT: v16x8.load_splat $push[[R:[0-9]+]]=, 0($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <8 x i16> @load_splat_v8i16(i16* %p) {
+  %e = load i16, i16* %p
+  %v1 = insertelement <8 x i16> undef, i16 %e, i32 0
+  %v2 = shufflevector <8 x i16> %v1, <8 x i16> undef, <8 x i32> zeroinitializer
+  ret <8 x i16> %v2
+}
+
+; CHECK-LABEL: load_sext_v8i16:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_sext_v8i16 (i32) -> (v128){{$}}
+; SIMD128-NEXT: i16x8.load8x8_s $push[[R:[0-9]+]]=, 0($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <8 x i16> @load_sext_v8i16(<8 x i8>* %p) {
+  %v = load <8 x i8>, <8 x i8>* %p
+  %v2 = sext <8 x i8> %v to <8 x i16>
+  ret <8 x i16> %v2
+}
+
+; CHECK-LABEL: load_zext_v8i16:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_zext_v8i16 (i32) -> (v128){{$}}
+; SIMD128-NEXT: i16x8.load8x8_u $push[[R:[0-9]+]]=, 0($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <8 x i16> @load_zext_v8i16(<8 x i8>* %p) {
+  %v = load <8 x i8>, <8 x i8>* %p
+  %v2 = zext <8 x i8> %v to <8 x i16>
+  ret <8 x i16> %v2
+}
+
+; CHECK-LABEL: load_ext_v8i16:
+; NO-SIMD128-NOT: load8x8
+; SIMD128-NEXT: .functype load_ext_v8i16 (i32) -> (v128){{$}}
+; SIMD128-NEXT: i16x8.load8x8_u $push[[R:[0-9]+]]=, 0($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <8 x i8> @load_ext_v8i16(<8 x i8>* %p) {
+  %v = load <8 x i8>, <8 x i8>* %p
+  ret <8 x i8> %v
+}
+
 ; CHECK-LABEL: load_v8i16_with_folded_offset:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype load_v8i16_with_folded_offset (i32) -> (v128){{$}}
@@ -223,6 +383,62 @@ define <8 x i16> @load_v8i16_with_folded_offset(<8 x i16>* %p) {
   ret <8 x i16> %v
 }
 
+; CHECK-LABEL: load_splat_v8i16_with_folded_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v8i16_with_folded_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: v16x8.load_splat $push[[R:[0-9]+]]=, 16($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <8 x i16> @load_splat_v8i16_with_folded_offset(i16* %p) {
+  %q = ptrtoint i16* %p to i32
+  %r = add nuw i32 %q, 16
+  %s = inttoptr i32 %r to i16*
+  %e = load i16, i16* %s
+  %v1 = insertelement <8 x i16> undef, i16 %e, i32 0
+  %v2 = shufflevector <8 x i16> %v1, <8 x i16> undef, <8 x i32> zeroinitializer
+  ret <8 x i16> %v2
+}
+
+; CHECK-LABEL: load_sext_v8i16_with_folded_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_sext_v8i16_with_folded_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i16x8.load8x8_s $push[[R:[0-9]+]]=, 16($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <8 x i16> @load_sext_v8i16_with_folded_offset(<8 x i8>* %p) {
+  %q = ptrtoint <8 x i8>* %p to i32
+  %r = add nuw i32 %q, 16
+  %s = inttoptr i32 %r to <8 x i8>*
+  %v = load <8 x i8>, <8 x i8>* %s
+  %v2 = sext <8 x i8> %v to <8 x i16>
+  ret <8 x i16> %v2
+}
+
+; CHECK-LABEL: load_zext_v8i16_with_folded_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_zext_v8i16_with_folded_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i16x8.load8x8_u $push[[R:[0-9]+]]=, 16($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <8 x i16> @load_zext_v8i16_with_folded_offset(<8 x i8>* %p) {
+  %q = ptrtoint <8 x i8>* %p to i32
+  %r = add nuw i32 %q, 16
+  %s = inttoptr i32 %r to <8 x i8>*
+  %v = load <8 x i8>, <8 x i8>* %s
+  %v2 = zext <8 x i8> %v to <8 x i16>
+  ret <8 x i16> %v2
+}
+
+; CHECK-LABEL: load_ext_v8i16_with_folded_offset:
+; NO-SIMD128-NOT: load8x8
+; SIMD128-NEXT: .functype load_ext_v8i16_with_folded_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i16x8.load8x8_u $push[[R:[0-9]+]]=, 16($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <8 x i8> @load_ext_v8i16_with_folded_offset(<8 x i8>* %p) {
+  %q = ptrtoint <8 x i8>* %p to i32
+  %r = add nuw i32 %q, 16
+  %s = inttoptr i32 %r to <8 x i8>*
+  %v = load <8 x i8>, <8 x i8>* %s
+  ret <8 x i8> %v
+}
+
 ; CHECK-LABEL: load_v8i16_with_folded_gep_offset:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype load_v8i16_with_folded_gep_offset (i32) -> (v128){{$}}
@@ -234,6 +450,54 @@ define <8 x i16> @load_v8i16_with_folded_gep_offset(<8 x i16>* %p) {
   ret <8 x i16> %v
 }
 
+; CHECK-LABEL: load_splat_v8i16_with_folded_gep_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v8i16_with_folded_gep_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: v16x8.load_splat $push[[R:[0-9]+]]=, 2($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <8 x i16> @load_splat_v8i16_with_folded_gep_offset(i16* %p) {
+  %s = getelementptr inbounds i16, i16* %p, i32 1
+  %e = load i16, i16* %s
+  %v1 = insertelement <8 x i16> undef, i16 %e, i32 0
+  %v2 = shufflevector <8 x i16> %v1, <8 x i16> undef, <8 x i32> zeroinitializer
+  ret <8 x i16> %v2
+}
+
+; CHECK-LABEL: load_sext_v8i16_with_folded_gep_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_sext_v8i16_with_folded_gep_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i16x8.load8x8_s $push[[R:[0-9]+]]=, 8($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <8 x i16> @load_sext_v8i16_with_folded_gep_offset(<8 x i8>* %p) {
+  %s = getelementptr inbounds <8 x i8>, <8 x i8>* %p, i32 1
+  %v = load <8 x i8>, <8 x i8>* %s
+  %v2 = sext <8 x i8> %v to <8 x i16>
+  ret <8 x i16> %v2
+}
+
+; CHECK-LABEL: load_zext_v8i16_with_folded_gep_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_zext_v8i16_with_folded_gep_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i16x8.load8x8_u $push[[R:[0-9]+]]=, 8($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <8 x i16> @load_zext_v8i16_with_folded_gep_offset(<8 x i8>* %p) {
+  %s = getelementptr inbounds <8 x i8>, <8 x i8>* %p, i32 1
+  %v = load <8 x i8>, <8 x i8>* %s
+  %v2 = zext <8 x i8> %v to <8 x i16>
+  ret <8 x i16> %v2
+}
+
+; CHECK-LABEL: load_ext_v8i16_with_folded_gep_offset:
+; NO-SIMD128-NOT: load8x8
+; SIMD128-NEXT: .functype load_ext_v8i16_with_folded_gep_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i16x8.load8x8_u $push[[R:[0-9]+]]=, 8($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <8 x i8> @load_ext_v8i16_with_folded_gep_offset(<8 x i8>* %p) {
+  %s = getelementptr inbounds <8 x i8>, <8 x i8>* %p, i32 1
+  %v = load <8 x i8>, <8 x i8>* %s
+  ret <8 x i8> %v
+}
+
 ; CHECK-LABEL: load_v8i16_with_unfolded_gep_negative_offset:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype load_v8i16_with_unfolded_gep_negative_offset (i32) -> (v128){{$}}
@@ -247,6 +511,62 @@ define <8 x i16> @load_v8i16_with_unfolded_gep_negative_offset(<8 x i16>* %p) {
   ret <8 x i16> %v
 }
 
+; CHECK-LABEL: load_splat_v8i16_with_unfolded_gep_negative_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v8i16_with_unfolded_gep_negative_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, -2{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: v16x8.load_splat $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <8 x i16> @load_splat_v8i16_with_unfolded_gep_negative_offset(i16* %p) {
+  %s = getelementptr inbounds i16, i16* %p, i32 -1
+  %e = load i16, i16* %s
+  %v1 = insertelement <8 x i16> undef, i16 %e, i32 0
+  %v2 = shufflevector <8 x i16> %v1, <8 x i16> undef, <8 x i32> zeroinitializer
+  ret <8 x i16> %v2
+}
+
+; CHECK-LABEL: load_sext_v8i16_with_unfolded_gep_negative_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_sext_v8i16_with_unfolded_gep_negative_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, -8{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: i16x8.load8x8_s $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <8 x i16> @load_sext_v8i16_with_unfolded_gep_negative_offset(<8 x i8>* %p) {
+  %s = getelementptr inbounds <8 x i8>, <8 x i8>* %p, i32 -1
+  %v = load <8 x i8>, <8 x i8>* %s
+  %v2 = sext <8 x i8> %v to <8 x i16>
+  ret <8 x i16> %v2
+}
+
+; CHECK-LABEL: load_zext_v8i16_with_unfolded_gep_negative_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_zext_v8i16_with_unfolded_gep_negative_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, -8{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: i16x8.load8x8_u $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <8 x i16> @load_zext_v8i16_with_unfolded_gep_negative_offset(<8 x i8>* %p) {
+  %s = getelementptr inbounds <8 x i8>, <8 x i8>* %p, i32 -1
+  %v = load <8 x i8>, <8 x i8>* %s
+  %v2 = zext <8 x i8> %v to <8 x i16>
+  ret <8 x i16> %v2
+}
+
+; CHECK-LABEL: load_ext_v8i16_with_unfolded_gep_negative_offset:
+; NO-SIMD128-NOT: load8x8
+; SIMD128-NEXT: .functype load_ext_v8i16_with_unfolded_gep_negative_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, -8{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: i16x8.load8x8_u $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <8 x i8> @load_ext_v8i16_with_unfolded_gep_negative_offset(<8 x i8>* %p) {
+  %s = getelementptr inbounds <8 x i8>, <8 x i8>* %p, i32 -1
+  %v = load <8 x i8>, <8 x i8>* %s
+  ret <8 x i8> %v
+}
+
 ; CHECK-LABEL: load_v8i16_with_unfolded_offset:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype load_v8i16_with_unfolded_offset (i32) -> (v128){{$}}
@@ -262,6 +582,70 @@ define <8 x i16> @load_v8i16_with_unfolded_offset(<8 x i16>* %p) {
   ret <8 x i16> %v
 }
 
+; CHECK-LABEL: load_splat_v8i16_with_unfolded_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v8i16_with_unfolded_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 16{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: v16x8.load_splat $push[[L0:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[L0]]{{$}}
+define <8 x i16> @load_splat_v8i16_with_unfolded_offset(i16* %p) {
+  %q = ptrtoint i16* %p to i32
+  %r = add nsw i32 %q, 16
+  %s = inttoptr i32 %r to i16*
+  %e = load i16, i16* %s
+  %v1 = insertelement <8 x i16> undef, i16 %e, i32 0
+  %v2 = shufflevector <8 x i16> %v1, <8 x i16> undef, <8 x i32> zeroinitializer
+  ret <8 x i16> %v2
+}
+
+; CHECK-LABEL: load_sext_v8i16_with_unfolded_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_sext_v8i16_with_unfolded_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 16{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: i16x8.load8x8_s $push[[L0:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[L0]]{{$}}
+define <8 x i16> @load_sext_v8i16_with_unfolded_offset(<8 x i8>* %p) {
+  %q = ptrtoint <8 x i8>* %p to i32
+  %r = add nsw i32 %q, 16
+  %s = inttoptr i32 %r to <8 x i8>*
+  %v = load <8 x i8>, <8 x i8>* %s
+  %v2 = sext <8 x i8> %v to <8 x i16>
+  ret <8 x i16> %v2
+}
+
+; CHECK-LABEL: load_zext_v8i16_with_unfolded_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_zext_v8i16_with_unfolded_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 16{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: i16x8.load8x8_u $push[[L0:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[L0]]{{$}}
+define <8 x i16> @load_zext_v8i16_with_unfolded_offset(<8 x i8>* %p) {
+  %q = ptrtoint <8 x i8>* %p to i32
+  %r = add nsw i32 %q, 16
+  %s = inttoptr i32 %r to <8 x i8>*
+  %v = load <8 x i8>, <8 x i8>* %s
+  %v2 = zext <8 x i8> %v to <8 x i16>
+  ret <8 x i16> %v2
+}
+
+; CHECK-LABEL: load_ext_v8i16_with_unfolded_offset:
+; NO-SIMD128-NOT: load8x8
+; SIMD128-NEXT: .functype load_ext_v8i16_with_unfolded_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 16{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: i16x8.load8x8_u $push[[L0:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[L0]]{{$}}
+define <8 x i8> @load_ext_v8i16_with_unfolded_offset(<8 x i8>* %p) {
+  %q = ptrtoint <8 x i8>* %p to i32
+  %r = add nsw i32 %q, 16
+  %s = inttoptr i32 %r to <8 x i8>*
+  %v = load <8 x i8>, <8 x i8>* %s
+  ret <8 x i8> %v
+}
+
 ; CHECK-LABEL: load_v8i16_with_unfolded_gep_offset:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype load_v8i16_with_unfolded_gep_offset (i32) -> (v128){{$}}
@@ -275,6 +659,62 @@ define <8 x i16> @load_v8i16_with_unfolded_gep_offset(<8 x i16>* %p) {
   ret <8 x i16> %v
 }
 
+; CHECK-LABEL: load_splat_v8i16_with_unfolded_gep_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v8i16_with_unfolded_gep_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 2{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: v16x8.load_splat $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <8 x i16> @load_splat_v8i16_with_unfolded_gep_offset(i16* %p) {
+  %s = getelementptr i16, i16* %p, i32 1
+  %e = load i16, i16* %s
+  %v1 = insertelement <8 x i16> undef, i16 %e, i32 0
+  %v2 = shufflevector <8 x i16> %v1, <8 x i16> undef, <8 x i32> zeroinitializer
+  ret <8 x i16> %v2
+}
+
+; CHECK-LABEL: load_sext_v8i16_with_unfolded_gep_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_sext_v8i16_with_unfolded_gep_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 8{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: i16x8.load8x8_s $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <8 x i16> @load_sext_v8i16_with_unfolded_gep_offset(<8 x i8>* %p) {
+  %s = getelementptr <8 x i8>, <8 x i8>* %p, i32 1
+  %v = load <8 x i8>, <8 x i8>* %s
+  %v2 = sext <8 x i8> %v to <8 x i16>
+  ret <8 x i16> %v2
+}
+
+; CHECK-LABEL: load_zext_v8i16_with_unfolded_gep_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_zext_v8i16_with_unfolded_gep_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 8{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: i16x8.load8x8_u $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <8 x i16> @load_zext_v8i16_with_unfolded_gep_offset(<8 x i8>* %p) {
+  %s = getelementptr <8 x i8>, <8 x i8>* %p, i32 1
+  %v = load <8 x i8>, <8 x i8>* %s
+  %v2 = zext <8 x i8> %v to <8 x i16>
+  ret <8 x i16> %v2
+}
+
+; CHECK-LABEL: load_ext_v8i16_with_unfolded_gep_offset:
+; NO-SIMD128-NOT: load8x8
+; SIMD128-NEXT: .functype load_ext_v8i16_with_unfolded_gep_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 8{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: i16x8.load8x8_u $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <8 x i8> @load_ext_v8i16_with_unfolded_gep_offset(<8 x i8>* %p) {
+  %s = getelementptr <8 x i8>, <8 x i8>* %p, i32 1
+  %v = load <8 x i8>, <8 x i8>* %s
+  ret <8 x i8> %v
+}
+
 ; CHECK-LABEL: load_v8i16_from_numeric_address:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype load_v8i16_from_numeric_address () -> (v128){{$}}
@@ -287,6 +727,58 @@ define <8 x i16> @load_v8i16_from_numeric_address() {
   ret <8 x i16> %v
 }
 
+; CHECK-LABEL: load_splat_v8i16_from_numeric_address:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v8i16_from_numeric_address () -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
+; SIMD128-NEXT: v16x8.load_splat $push[[R:[0-9]+]]=, 32($pop[[L0]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <8 x i16> @load_splat_v8i16_from_numeric_address() {
+  %s = inttoptr i32 32 to i16*
+  %e = load i16, i16* %s
+  %v1 = insertelement <8 x i16> undef, i16 %e, i32 0
+  %v2 = shufflevector <8 x i16> %v1, <8 x i16> undef, <8 x i32> zeroinitializer
+  ret <8 x i16> %v2
+}
+
+; CHECK-LABEL: load_sext_v8i16_from_numeric_address:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_sext_v8i16_from_numeric_address () -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
+; SIMD128-NEXT: i16x8.load8x8_s $push[[R:[0-9]+]]=, 32($pop[[L0]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <8 x i16> @load_sext_v8i16_from_numeric_address() {
+  %s = inttoptr i32 32 to <8 x i8>*
+  %v = load <8 x i8>, <8 x i8>* %s
+  %v2 = sext <8 x i8> %v to <8 x i16>
+  ret <8 x i16> %v2
+}
+
+; CHECK-LABEL: load_zext_v8i16_from_numeric_address:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_zext_v8i16_from_numeric_address () -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
+; SIMD128-NEXT: i16x8.load8x8_u $push[[R:[0-9]+]]=, 32($pop[[L0]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <8 x i16> @load_zext_v8i16_from_numeric_address() {
+  %s = inttoptr i32 32 to <8 x i8>*
+  %v = load <8 x i8>, <8 x i8>* %s
+  %v2 = zext <8 x i8> %v to <8 x i16>
+  ret <8 x i16> %v2
+}
+
+; CHECK-LABEL: load_ext_v8i16_from_numeric_address:
+; NO-SIMD128-NOT: load8x8
+; SIMD128-NEXT: .functype load_ext_v8i16_from_numeric_address () -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
+; SIMD128-NEXT: i16x8.load8x8_u $push[[R:[0-9]+]]=, 32($pop[[L0]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <8 x i8> @load_ext_v8i16_from_numeric_address() {
+  %s = inttoptr i32 32 to <8 x i8>*
+  %v = load <8 x i8>, <8 x i8>* %s
+  ret <8 x i8> %v
+}
+
 ; CHECK-LABEL: load_v8i16_from_global_address:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype load_v8i16_from_global_address () -> (v128){{$}}
@@ -299,6 +791,57 @@ define <8 x i16> @load_v8i16_from_global_address() {
   ret <8 x i16> %v
 }
 
+; CHECK-LABEL: load_splat_v8i16_from_global_address:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v8i16_from_global_address () -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
+; SIMD128-NEXT: v16x8.load_splat $push[[R:[0-9]+]]=, gv_i16($pop[[L0]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+@gv_i16 = global i16 42
+define <8 x i16> @load_splat_v8i16_from_global_address() {
+  %e = load i16, i16* @gv_i16
+  %v1 = insertelement <8 x i16> undef, i16 %e, i32 0
+  %v2 = shufflevector <8 x i16> %v1, <8 x i16> undef, <8 x i32> zeroinitializer
+  ret <8 x i16> %v2
+}
+
+; CHECK-LABEL: load_sext_v8i16_from_global_address:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_sext_v8i16_from_global_address () -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
+; SIMD128-NEXT: i16x8.load8x8_s $push[[R:[0-9]+]]=, gv_v8i8($pop[[L0]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+@gv_v8i8 = global <8 x i8> <i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42>
+define <8 x i16> @load_sext_v8i16_from_global_address() {
+  %v = load <8 x i8>, <8 x i8>* @gv_v8i8
+  %v2 = sext <8 x i8> %v to <8 x i16>
+  ret <8 x i16> %v2
+}
+
+; CHECK-LABEL: load_zext_v8i16_from_global_address:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_zext_v8i16_from_global_address () -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
+; SIMD128-NEXT: i16x8.load8x8_u $push[[R:[0-9]+]]=, gv_v8i8($pop[[L0]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <8 x i16> @load_zext_v8i16_from_global_address() {
+  %v = load <8 x i8>, <8 x i8>* @gv_v8i8
+  %v2 = zext <8 x i8> %v to <8 x i16>
+  ret <8 x i16> %v2
+}
+
+; CHECK-LABEL: load_ext_v8i16_from_global_address:
+; NO-SIMD128-NOT: load8x8
+; SIMD128-NEXT: .functype load_ext_v8i16_from_global_address () -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
+; SIMD128-NEXT: i16x8.load8x8_u $push[[R:[0-9]+]]=, gv_v8i8($pop[[L0]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <8 x i8> @load_ext_v8i16_from_global_address() {
+  %v = load <8 x i8>, <8 x i8>* @gv_v8i8
+  ret <8 x i8> %v
+}
+
+
 ; CHECK-LABEL: store_v8i16:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype store_v8i16 (v128, i32) -> (){{$}}
@@ -400,6 +943,49 @@ define <4 x i32> @load_v4i32(<4 x i32>* %p) {
   ret <4 x i32> %v
 }
 
+; CHECK-LABEL: load_splat_v4i32:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v4i32 (i32) -> (v128){{$}}
+; SIMD128-NEXT: v32x4.load_splat
+define <4 x i32> @load_splat_v4i32(i32* %addr) {
+  %e = load i32, i32* %addr, align 4
+  %v1 = insertelement <4 x i32> undef, i32 %e, i32 0
+  %v2 = shufflevector <4 x i32> %v1, <4 x i32> undef, <4 x i32> zeroinitializer
+  ret <4 x i32> %v2
+}
+
+; CHECK-LABEL: load_sext_v4i32:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_sext_v4i32 (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32x4.load16x4_s $push[[R:[0-9]+]]=, 0($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x i32> @load_sext_v4i32(<4 x i16>* %p) {
+  %v = load <4 x i16>, <4 x i16>* %p
+  %v2 = sext <4 x i16> %v to <4 x i32>
+  ret <4 x i32> %v2
+}
+
+; CHECK-LABEL: load_zext_v4i32:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_zext_v4i32 (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32x4.load16x4_u $push[[R:[0-9]+]]=, 0($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x i32> @load_zext_v4i32(<4 x i16>* %p) {
+  %v = load <4 x i16>, <4 x i16>* %p
+  %v2 = zext <4 x i16> %v to <4 x i32>
+  ret <4 x i32> %v2
+}
+
+; CHECK-LABEL: load_ext_v4i32:
+; NO-SIMD128-NOT: load16x4
+; SIMD128-NEXT: .functype load_ext_v4i32 (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32x4.load16x4_u $push[[R:[0-9]+]]=, 0($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x i16> @load_ext_v4i32(<4 x i16>* %p) {
+  %v = load <4 x i16>, <4 x i16>* %p
+  ret <4 x i16> %v
+}
+
 ; CHECK-LABEL: load_v4i32_with_folded_offset:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype load_v4i32_with_folded_offset (i32) -> (v128){{$}}
@@ -413,6 +999,62 @@ define <4 x i32> @load_v4i32_with_folded_offset(<4 x i32>* %p) {
   ret <4 x i32> %v
 }
 
+; CHECK-LABEL: load_splat_v4i32_with_folded_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v4i32_with_folded_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: v32x4.load_splat $push[[R:[0-9]+]]=, 16($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x i32> @load_splat_v4i32_with_folded_offset(i32* %p) {
+  %q = ptrtoint i32* %p to i32
+  %r = add nuw i32 %q, 16
+  %s = inttoptr i32 %r to i32*
+  %e = load i32, i32* %s
+  %v1 = insertelement <4 x i32> undef, i32 %e, i32 0
+  %v2 = shufflevector <4 x i32> %v1, <4 x i32> undef, <4 x i32> zeroinitializer
+  ret <4 x i32> %v2
+}
+
+; CHECK-LABEL: load_sext_v4i32_with_folded_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_sext_v4i32_with_folded_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32x4.load16x4_s $push[[R:[0-9]+]]=, 16($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x i32> @load_sext_v4i32_with_folded_offset(<4 x i16>* %p) {
+  %q = ptrtoint <4 x i16>* %p to i32
+  %r = add nuw i32 %q, 16
+  %s = inttoptr i32 %r to <4 x i16>*
+  %v = load <4 x i16>, <4 x i16>* %s
+  %v2 = sext <4 x i16> %v to <4 x i32>
+  ret <4 x i32> %v2
+}
+
+; CHECK-LABEL: load_zext_v4i32_with_folded_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_zext_v4i32_with_folded_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32x4.load16x4_u $push[[R:[0-9]+]]=, 16($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x i32> @load_zext_v4i32_with_folded_offset(<4 x i16>* %p) {
+  %q = ptrtoint <4 x i16>* %p to i32
+  %r = add nuw i32 %q, 16
+  %s = inttoptr i32 %r to <4 x i16>*
+  %v = load <4 x i16>, <4 x i16>* %s
+  %v2 = zext <4 x i16> %v to <4 x i32>
+  ret <4 x i32> %v2
+}
+
+; CHECK-LABEL: load_ext_v4i32_with_folded_offset:
+; NO-SIMD128-NOT: load16x4
+; SIMD128-NEXT: .functype load_ext_v4i32_with_folded_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32x4.load16x4_u $push[[R:[0-9]+]]=, 16($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x i16> @load_ext_v4i32_with_folded_offset(<4 x i16>* %p) {
+  %q = ptrtoint <4 x i16>* %p to i32
+  %r = add nuw i32 %q, 16
+  %s = inttoptr i32 %r to <4 x i16>*
+  %v = load <4 x i16>, <4 x i16>* %s
+  ret <4 x i16> %v
+}
+
 ; CHECK-LABEL: load_v4i32_with_folded_gep_offset:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype load_v4i32_with_folded_gep_offset (i32) -> (v128){{$}}
@@ -424,6 +1066,54 @@ define <4 x i32> @load_v4i32_with_folded_gep_offset(<4 x i32>* %p) {
   ret <4 x i32> %v
 }
 
+; CHECK-LABEL: load_splat_v4i32_with_folded_gep_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v4i32_with_folded_gep_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: v32x4.load_splat $push[[R:[0-9]+]]=, 4($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x i32> @load_splat_v4i32_with_folded_gep_offset(i32* %p) {
+  %s = getelementptr inbounds i32, i32* %p, i32 1
+  %e = load i32, i32* %s
+  %v1 = insertelement <4 x i32> undef, i32 %e, i32 0
+  %v2 = shufflevector <4 x i32> %v1, <4 x i32> undef, <4 x i32> zeroinitializer
+  ret <4 x i32> %v2
+}
+
+; CHECK-LABEL: load_sext_v4i32_with_folded_gep_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_sext_v4i32_with_folded_gep_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32x4.load16x4_s $push[[R:[0-9]+]]=, 8($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x i32> @load_sext_v4i32_with_folded_gep_offset(<4 x i16>* %p) {
+  %s = getelementptr inbounds <4 x i16>, <4 x i16>* %p, i32 1
+  %v = load <4 x i16>, <4 x i16>* %s
+  %v2 = sext <4 x i16> %v to <4 x i32>
+  ret <4 x i32> %v2
+}
+
+; CHECK-LABEL: load_zext_v4i32_with_folded_gep_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_zext_v4i32_with_folded_gep_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32x4.load16x4_u $push[[R:[0-9]+]]=, 8($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x i32> @load_zext_v4i32_with_folded_gep_offset(<4 x i16>* %p) {
+  %s = getelementptr inbounds <4 x i16>, <4 x i16>* %p, i32 1
+  %v = load <4 x i16>, <4 x i16>* %s
+  %v2 = zext <4 x i16> %v to <4 x i32>
+  ret <4 x i32> %v2
+}
+
+; CHECK-LABEL: load_ext_v4i32_with_folded_gep_offset:
+; NO-SIMD128-NOT: load16x4
+; SIMD128-NEXT: .functype load_ext_v4i32_with_folded_gep_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32x4.load16x4_u $push[[R:[0-9]+]]=, 8($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x i16> @load_ext_v4i32_with_folded_gep_offset(<4 x i16>* %p) {
+  %s = getelementptr inbounds <4 x i16>, <4 x i16>* %p, i32 1
+  %v = load <4 x i16>, <4 x i16>* %s
+  ret <4 x i16> %v
+}
+
 ; CHECK-LABEL: load_v4i32_with_unfolded_gep_negative_offset:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype load_v4i32_with_unfolded_gep_negative_offset (i32) -> (v128){{$}}
@@ -437,6 +1127,62 @@ define <4 x i32> @load_v4i32_with_unfolded_gep_negative_offset(<4 x i32>* %p) {
   ret <4 x i32> %v
 }
 
+; CHECK-LABEL: load_splat_v4i32_with_unfolded_gep_negative_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v4i32_with_unfolded_gep_negative_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, -4{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: v32x4.load_splat $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x i32> @load_splat_v4i32_with_unfolded_gep_negative_offset(i32* %p) {
+  %s = getelementptr inbounds i32, i32* %p, i32 -1
+  %e = load i32, i32* %s
+  %v1 = insertelement <4 x i32> undef, i32 %e, i32 0
+  %v2 = shufflevector <4 x i32> %v1, <4 x i32> undef, <4 x i32> zeroinitializer
+  ret <4 x i32> %v2
+}
+
+; CHECK-LABEL: load_sext_v4i32_with_unfolded_gep_negative_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_sext_v4i32_with_unfolded_gep_negative_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, -8{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: i32x4.load16x4_s $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x i32> @load_sext_v4i32_with_unfolded_gep_negative_offset(<4 x i16>* %p) {
+  %s = getelementptr inbounds <4 x i16>, <4 x i16>* %p, i32 -1
+  %v = load <4 x i16>, <4 x i16>* %s
+  %v2 = sext <4 x i16> %v to <4 x i32>
+  ret <4 x i32> %v2
+}
+
+; CHECK-LABEL: load_zext_v4i32_with_unfolded_gep_negative_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_zext_v4i32_with_unfolded_gep_negative_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, -8{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: i32x4.load16x4_u $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x i32> @load_zext_v4i32_with_unfolded_gep_negative_offset(<4 x i16>* %p) {
+  %s = getelementptr inbounds <4 x i16>, <4 x i16>* %p, i32 -1
+  %v = load <4 x i16>, <4 x i16>* %s
+  %v2 = zext <4 x i16> %v to <4 x i32>
+  ret <4 x i32> %v2
+}
+
+; CHECK-LABEL: load_ext_v4i32_with_unfolded_gep_negative_offset:
+; NO-SIMD128-NOT: load16x4
+; SIMD128-NEXT: .functype load_ext_v4i32_with_unfolded_gep_negative_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, -8{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: i32x4.load16x4_u $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x i16> @load_ext_v4i32_with_unfolded_gep_negative_offset(<4 x i16>* %p) {
+  %s = getelementptr inbounds <4 x i16>, <4 x i16>* %p, i32 -1
+  %v = load <4 x i16>, <4 x i16>* %s
+  ret <4 x i16> %v
+}
+
 ; CHECK-LABEL: load_v4i32_with_unfolded_offset:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype load_v4i32_with_unfolded_offset (i32) -> (v128){{$}}
@@ -452,6 +1198,70 @@ define <4 x i32> @load_v4i32_with_unfolded_offset(<4 x i32>* %p) {
   ret <4 x i32> %v
 }
 
+; CHECK-LABEL: load_splat_v4i32_with_unfolded_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v4i32_with_unfolded_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 16{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: v32x4.load_splat $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x i32> @load_splat_v4i32_with_unfolded_offset(i32* %p) {
+  %q = ptrtoint i32* %p to i32
+  %r = add nsw i32 %q, 16
+  %s = inttoptr i32 %r to i32*
+  %e = load i32, i32* %s
+  %v1 = insertelement <4 x i32> undef, i32 %e, i32 0
+  %v2 = shufflevector <4 x i32> %v1, <4 x i32> undef, <4 x i32> zeroinitializer
+  ret <4 x i32> %v2
+}
+
+; CHECK-LABEL: load_sext_v4i32_with_unfolded_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_sext_v4i32_with_unfolded_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 16{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: i32x4.load16x4_s $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x i32> @load_sext_v4i32_with_unfolded_offset(<4 x i16>* %p) {
+  %q = ptrtoint <4 x i16>* %p to i32
+  %r = add nsw i32 %q, 16
+  %s = inttoptr i32 %r to <4 x i16>*
+  %v = load <4 x i16>, <4 x i16>* %s
+  %v2 = sext <4 x i16> %v to <4 x i32>
+  ret <4 x i32> %v2
+}
+
+; CHECK-LABEL: load_zext_v4i32_with_unfolded_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_zext_v4i32_with_unfolded_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 16{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: i32x4.load16x4_u $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x i32> @load_zext_v4i32_with_unfolded_offset(<4 x i16>* %p) {
+  %q = ptrtoint <4 x i16>* %p to i32
+  %r = add nsw i32 %q, 16
+  %s = inttoptr i32 %r to <4 x i16>*
+  %v = load <4 x i16>, <4 x i16>* %s
+  %v2 = zext <4 x i16> %v to <4 x i32>
+  ret <4 x i32> %v2
+}
+
+; CHECK-LABEL: load_ext_v4i32_with_unfolded_offset:
+; NO-SIMD128-NOT: load16x4
+; SIMD128-NEXT: .functype load_ext_v4i32_with_unfolded_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 16{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: i32x4.load16x4_u $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x i16> @load_ext_v4i32_with_unfolded_offset(<4 x i16>* %p) {
+  %q = ptrtoint <4 x i16>* %p to i32
+  %r = add nsw i32 %q, 16
+  %s = inttoptr i32 %r to <4 x i16>*
+  %v = load <4 x i16>, <4 x i16>* %s
+  ret <4 x i16> %v
+}
+
 ; CHECK-LABEL: load_v4i32_with_unfolded_gep_offset:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype load_v4i32_with_unfolded_gep_offset (i32) -> (v128){{$}}
@@ -465,6 +1275,62 @@ define <4 x i32> @load_v4i32_with_unfolded_gep_offset(<4 x i32>* %p) {
   ret <4 x i32> %v
 }
 
+; CHECK-LABEL: load_splat_v4i32_with_unfolded_gep_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v4i32_with_unfolded_gep_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 4{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: v32x4.load_splat $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x i32> @load_splat_v4i32_with_unfolded_gep_offset(i32* %p) {
+  %s = getelementptr i32, i32* %p, i32 1
+  %e = load i32, i32* %s
+  %v1 = insertelement <4 x i32> undef, i32 %e, i32 0
+  %v2 = shufflevector <4 x i32> %v1, <4 x i32> undef, <4 x i32> zeroinitializer
+  ret <4 x i32> %v2
+}
+
+; CHECK-LABEL: load_sext_v4i32_with_unfolded_gep_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_sext_v4i32_with_unfolded_gep_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 8{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: i32x4.load16x4_s $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x i32> @load_sext_v4i32_with_unfolded_gep_offset(<4 x i16>* %p) {
+  %s = getelementptr <4 x i16>, <4 x i16>* %p, i32 1
+  %v = load <4 x i16>, <4 x i16>* %s
+  %v2 = sext <4 x i16> %v to <4 x i32>
+  ret <4 x i32> %v2
+}
+
+; CHECK-LABEL: load_zext_v4i32_with_unfolded_gep_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_zext_v4i32_with_unfolded_gep_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 8{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: i32x4.load16x4_u $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x i32> @load_zext_v4i32_with_unfolded_gep_offset(<4 x i16>* %p) {
+  %s = getelementptr <4 x i16>, <4 x i16>* %p, i32 1
+  %v = load <4 x i16>, <4 x i16>* %s
+  %v2 = zext <4 x i16> %v to <4 x i32>
+  ret <4 x i32> %v2
+}
+
+; CHECK-LABEL: load_ext_v4i32_with_unfolded_gep_offset:
+; NO-SIMD128-NOT: load16x4
+; SIMD128-NEXT: .functype load_ext_v4i32_with_unfolded_gep_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 8{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: i32x4.load16x4_u $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x i16> @load_ext_v4i32_with_unfolded_gep_offset(<4 x i16>* %p) {
+  %s = getelementptr <4 x i16>, <4 x i16>* %p, i32 1
+  %v = load <4 x i16>, <4 x i16>* %s
+  ret <4 x i16> %v
+}
+
 ; CHECK-LABEL: load_v4i32_from_numeric_address:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype load_v4i32_from_numeric_address () -> (v128){{$}}
@@ -477,6 +1343,58 @@ define <4 x i32> @load_v4i32_from_numeric_address() {
   ret <4 x i32> %v
 }
 
+; CHECK-LABEL: load_splat_v4i32_from_numeric_address:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v4i32_from_numeric_address () -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
+; SIMD128-NEXT: v32x4.load_splat $push[[R:[0-9]+]]=, 32($pop[[L0]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x i32> @load_splat_v4i32_from_numeric_address() {
+  %s = inttoptr i32 32 to i32*
+  %e = load i32, i32* %s
+  %v1 = insertelement <4 x i32> undef, i32 %e, i32 0
+  %v2 = shufflevector <4 x i32> %v1, <4 x i32> undef, <4 x i32> zeroinitializer
+  ret <4 x i32> %v2
+}
+
+; CHECK-LABEL: load_sext_v4i32_from_numeric_address:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_sext_v4i32_from_numeric_address () -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
+; SIMD128-NEXT: i32x4.load16x4_s $push[[R:[0-9]+]]=, 32($pop[[L0]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x i32> @load_sext_v4i32_from_numeric_address() {
+  %s = inttoptr i32 32 to <4 x i16>*
+  %v = load <4 x i16>, <4 x i16>* %s
+  %v2 = sext <4 x i16> %v to <4 x i32>
+  ret <4 x i32> %v2
+}
+
+; CHECK-LABEL: load_zext_v4i32_from_numeric_address:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_zext_v4i32_from_numeric_address () -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
+; SIMD128-NEXT: i32x4.load16x4_u $push[[R:[0-9]+]]=, 32($pop[[L0]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x i32> @load_zext_v4i32_from_numeric_address() {
+  %s = inttoptr i32 32 to <4 x i16>*
+  %v = load <4 x i16>, <4 x i16>* %s
+  %v2 = zext <4 x i16> %v to <4 x i32>
+  ret <4 x i32> %v2
+}
+
+; CHECK-LABEL: load_ext_v4i32_from_numeric_address:
+; NO-SIMD128-NOT: load16x4
+; SIMD128-NEXT: .functype load_ext_v4i32_from_numeric_address () -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
+; SIMD128-NEXT: i32x4.load16x4_u $push[[R:[0-9]+]]=, 32($pop[[L0]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x i16> @load_ext_v4i32_from_numeric_address() {
+  %s = inttoptr i32 32 to <4 x i16>*
+  %v = load <4 x i16>, <4 x i16>* %s
+  ret <4 x i16> %v
+}
+
 ; CHECK-LABEL: load_v4i32_from_global_address:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype load_v4i32_from_global_address () -> (v128){{$}}
@@ -489,6 +1407,56 @@ define <4 x i32> @load_v4i32_from_global_address() {
   ret <4 x i32> %v
 }
 
+; CHECK-LABEL: load_splat_v4i32_from_global_address:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v4i32_from_global_address () -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
+; SIMD128-NEXT: v32x4.load_splat $push[[R:[0-9]+]]=, gv_i32($pop[[L0]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+@gv_i32 = global i32 42
+define <4 x i32> @load_splat_v4i32_from_global_address() {
+  %e = load i32, i32* @gv_i32
+  %v1 = insertelement <4 x i32> undef, i32 %e, i32 0
+  %v2 = shufflevector <4 x i32> %v1, <4 x i32> undef, <4 x i32> zeroinitializer
+  ret <4 x i32> %v2
+}
+
+; CHECK-LABEL: load_sext_v4i32_from_global_address:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_sext_v4i32_from_global_address () -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
+; SIMD128-NEXT: i32x4.load16x4_s $push[[R:[0-9]+]]=, gv_v4i16($pop[[L0]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+@gv_v4i16 = global <4 x i16> <i16 42, i16 42, i16 42, i16 42>
+define <4 x i32> @load_sext_v4i32_from_global_address() {
+  %v = load <4 x i16>, <4 x i16>* @gv_v4i16
+  %v2 = sext <4 x i16> %v to <4 x i32>
+  ret <4 x i32> %v2
+}
+
+; CHECK-LABEL: load_zext_v4i32_from_global_address:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_zext_v4i32_from_global_address () -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
+; SIMD128-NEXT: i32x4.load16x4_u $push[[R:[0-9]+]]=, gv_v4i16($pop[[L0]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x i32> @load_zext_v4i32_from_global_address() {
+  %v = load <4 x i16>, <4 x i16>* @gv_v4i16
+  %v2 = zext <4 x i16> %v to <4 x i32>
+  ret <4 x i32> %v2
+}
+
+; CHECK-LABEL: load_ext_v4i32_from_global_address:
+; NO-SIMD128-NOT: load16x4
+; SIMD128-NEXT: .functype load_ext_v4i32_from_global_address () -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
+; SIMD128-NEXT: i32x4.load16x4_u $push[[R:[0-9]+]]=, gv_v4i16($pop[[L0]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x i16> @load_ext_v4i32_from_global_address() {
+  %v = load <4 x i16>, <4 x i16>* @gv_v4i16
+  ret <4 x i16> %v
+}
+
 ; CHECK-LABEL: store_v4i32:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype store_v4i32 (v128, i32) -> (){{$}}
@@ -586,9 +1554,57 @@ define void @store_v4i32_to_global_address(<4 x i32> %v) {
 ; SIMD128-NEXT: .functype load_v2i64 (i32) -> (v128){{$}}
 ; SIMD128-NEXT: v128.load $push[[R:[0-9]+]]=, 0($0){{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
-define <2 x i64> @load_v2i64(<2 x i64>* %p) {
-  %v = load <2 x i64>, <2 x i64>* %p
-  ret <2 x i64> %v
+define <2 x i64> @load_v2i64(<2 x i64>* %p) {
+  %v = load <2 x i64>, <2 x i64>* %p
+  ret <2 x i64> %v
+}
+
+; CHECK-LABEL: load_splat_v2i64:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v2i64 (i32) -> (v128){{$}}
+; SIMD128-NEXT: v64x2.load_splat $push[[R:[0-9]+]]=, 0($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x i64> @load_splat_v2i64(i64* %p) {
+  %e = load i64, i64* %p
+  %v1 = insertelement <2 x i64> undef, i64 %e, i32 0
+  %v2 = shufflevector <2 x i64> %v1, <2 x i64> undef, <2 x i32> zeroinitializer
+  ret <2 x i64> %v2
+}
+
+; CHECK-LABEL: load_sext_v2i64:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: v128
+; SIMD128-NEXT: .functype load_sext_v2i64 (i32) -> (v128){{$}}
+; SIMD128-NEXT: i64x2.load32x2_s $push[[R:[0-9]+]]=, 0($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x i64> @load_sext_v2i64(<2 x i32>* %p) {
+  %v = load <2 x i32>, <2 x i32>* %p
+  %v2 = sext <2 x i32> %v to <2 x i64>
+  ret <2 x i64> %v2
+}
+
+; CHECK-LABEL: load_zext_v2i64:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: v128
+; SIMD128-NEXT: .functype load_zext_v2i64 (i32) -> (v128){{$}}
+; SIMD128-NEXT: i64x2.load32x2_u $push[[R:[0-9]+]]=, 0($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x i64> @load_zext_v2i64(<2 x i32>* %p) {
+  %v = load <2 x i32>, <2 x i32>* %p
+  %v2 = zext <2 x i32> %v to <2 x i64>
+  ret <2 x i64> %v2
+}
+
+; CHECK-LABEL: load_ext_v2i64:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: load32x2
+; SIMD128-NEXT: .functype load_ext_v2i64 (i32) -> (v128){{$}}
+; SIMD128-NEXT: i64x2.load32x2_u $push[[R:[0-9]+]]=, 0($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x i32> @load_ext_v2i64(<2 x i32>* %p) {
+  %v = load <2 x i32>, <2 x i32>* %p
+  ret <2 x i32> %v
 }
 
 ; CHECK-LABEL: load_v2i64_with_folded_offset:
@@ -605,6 +1621,66 @@ define <2 x i64> @load_v2i64_with_folded_offset(<2 x i64>* %p) {
   ret <2 x i64> %v
 }
 
+; CHECK-LABEL: load_splat_v2i64_with_folded_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v2i64_with_folded_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: v64x2.load_splat $push[[R:[0-9]+]]=, 16($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x i64> @load_splat_v2i64_with_folded_offset(i64* %p) {
+  %q = ptrtoint i64* %p to i32
+  %r = add nuw i32 %q, 16
+  %s = inttoptr i32 %r to i64*
+  %e = load i64, i64* %s
+  %v1 = insertelement <2 x i64> undef, i64 %e, i32 0
+  %v2 = shufflevector <2 x i64> %v1, <2 x i64> undef, <2 x i32> zeroinitializer
+  ret <2 x i64> %v2
+}
+
+; CHECK-LABEL: load_sext_v2i64_with_folded_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: v128
+; SIMD128-NEXT: .functype load_sext_v2i64_with_folded_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i64x2.load32x2_s $push[[R:[0-9]+]]=, 16($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x i64> @load_sext_v2i64_with_folded_offset(<2 x i32>* %p) {
+  %q = ptrtoint <2 x i32>* %p to i32
+  %r = add nuw i32 %q, 16
+  %s = inttoptr i32 %r to <2 x i32>*
+  %v = load <2 x i32>, <2 x i32>* %s
+  %v2 = sext <2 x i32> %v to <2 x i64>
+  ret <2 x i64> %v2
+}
+
+; CHECK-LABEL: load_zext_v2i64_with_folded_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: v128
+; SIMD128-NEXT: .functype load_zext_v2i64_with_folded_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i64x2.load32x2_u $push[[R:[0-9]+]]=, 16($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x i64> @load_zext_v2i64_with_folded_offset(<2 x i32>* %p) {
+  %q = ptrtoint <2 x i32>* %p to i32
+  %r = add nuw i32 %q, 16
+  %s = inttoptr i32 %r to <2 x i32>*
+  %v = load <2 x i32>, <2 x i32>* %s
+  %v2 = zext <2 x i32> %v to <2 x i64>
+  ret <2 x i64> %v2
+}
+
+; CHECK-LABEL: load_ext_v2i64_with_folded_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: load32x2
+; SIMD128-NEXT: .functype load_ext_v2i64_with_folded_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i64x2.load32x2_u $push[[R:[0-9]+]]=, 16($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x i32> @load_ext_v2i64_with_folded_offset(<2 x i32>* %p) {
+  %q = ptrtoint <2 x i32>* %p to i32
+  %r = add nuw i32 %q, 16
+  %s = inttoptr i32 %r to <2 x i32>*
+  %v = load <2 x i32>, <2 x i32>* %s
+  ret <2 x i32> %v
+}
+
 ; CHECK-LABEL: load_v2i64_with_folded_gep_offset:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-VM-NOT: v128
@@ -617,6 +1693,58 @@ define <2 x i64> @load_v2i64_with_folded_gep_offset(<2 x i64>* %p) {
   ret <2 x i64> %v
 }
 
+; CHECK-LABEL: load_splat_v2i64_with_folded_gep_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v2i64_with_folded_gep_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: v64x2.load_splat $push[[R:[0-9]+]]=, 8($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x i64> @load_splat_v2i64_with_folded_gep_offset(i64* %p) {
+  %s = getelementptr inbounds i64, i64* %p, i32 1
+  %e = load i64, i64* %s
+  %v1 = insertelement <2 x i64> undef, i64 %e, i32 0
+  %v2 = shufflevector <2 x i64> %v1, <2 x i64> undef, <2 x i32> zeroinitializer
+  ret <2 x i64> %v2
+}
+
+; CHECK-LABEL: load_sext_v2i64_with_folded_gep_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: v128
+; SIMD128-NEXT: .functype load_sext_v2i64_with_folded_gep_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i64x2.load32x2_s $push[[R:[0-9]+]]=, 8($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x i64> @load_sext_v2i64_with_folded_gep_offset(<2 x i32>* %p) {
+  %s = getelementptr inbounds <2 x i32>, <2 x i32>* %p, i32 1
+  %v = load <2 x i32>, <2 x i32>* %s
+  %v2 = sext <2 x i32> %v to <2 x i64>
+  ret <2 x i64> %v2
+}
+
+; CHECK-LABEL: load_zext_v2i64_with_folded_gep_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: v128
+; SIMD128-NEXT: .functype load_zext_v2i64_with_folded_gep_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i64x2.load32x2_u $push[[R:[0-9]+]]=, 8($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x i64> @load_zext_v2i64_with_folded_gep_offset(<2 x i32>* %p) {
+  %s = getelementptr inbounds <2 x i32>, <2 x i32>* %p, i32 1
+  %v = load <2 x i32>, <2 x i32>* %s
+  %v2 = zext <2 x i32> %v to <2 x i64>
+  ret <2 x i64> %v2
+}
+
+; CHECK-LABEL: load_ext_v2i64_with_folded_gep_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: load32x2
+; SIMD128-NEXT: .functype load_ext_v2i64_with_folded_gep_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i64x2.load32x2_u $push[[R:[0-9]+]]=, 8($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x i32> @load_ext_v2i64_with_folded_gep_offset(<2 x i32>* %p) {
+  %s = getelementptr inbounds <2 x i32>, <2 x i32>* %p, i32 1
+  %v = load <2 x i32>, <2 x i32>* %s
+  ret <2 x i32> %v
+}
+
 ; CHECK-LABEL: load_v2i64_with_unfolded_gep_negative_offset:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-VM-NOT: v128
@@ -631,6 +1759,66 @@ define <2 x i64> @load_v2i64_with_unfolded_gep_negative_offset(<2 x i64>* %p) {
   ret <2 x i64> %v
 }
 
+; CHECK-LABEL: load_splat_v2i64_with_unfolded_gep_negative_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v2i64_with_unfolded_gep_negative_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, -8{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: v64x2.load_splat $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x i64> @load_splat_v2i64_with_unfolded_gep_negative_offset(i64* %p) {
+  %s = getelementptr inbounds i64, i64* %p, i32 -1
+  %e = load i64, i64* %s
+  %v1 = insertelement <2 x i64> undef, i64 %e, i32 0
+  %v2 = shufflevector <2 x i64> %v1, <2 x i64> undef, <2 x i32> zeroinitializer
+  ret <2 x i64> %v2
+}
+
+; CHECK-LABEL: load_sext_v2i64_with_unfolded_gep_negative_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: v128
+; SIMD128-NEXT: .functype load_sext_v2i64_with_unfolded_gep_negative_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, -8{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: i64x2.load32x2_s $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x i64> @load_sext_v2i64_with_unfolded_gep_negative_offset(<2 x i32>* %p) {
+  %s = getelementptr inbounds <2 x i32>, <2 x i32>* %p, i32 -1
+  %v = load <2 x i32>, <2 x i32>* %s
+  %v2 = sext <2 x i32> %v to <2 x i64>
+  ret <2 x i64> %v2
+}
+
+; CHECK-LABEL: load_zext_v2i64_with_unfolded_gep_negative_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: v128
+; SIMD128-NEXT: .functype load_zext_v2i64_with_unfolded_gep_negative_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, -8{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: i64x2.load32x2_u $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x i64> @load_zext_v2i64_with_unfolded_gep_negative_offset(<2 x i32>* %p) {
+  %s = getelementptr inbounds <2 x i32>, <2 x i32>* %p, i32 -1
+  %v = load <2 x i32>, <2 x i32>* %s
+  %v2 = zext <2 x i32> %v to <2 x i64>
+  ret <2 x i64> %v2
+}
+
+; CHECK-LABEL: load_ext_v2i64_with_unfolded_gep_negative_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: load32x2
+; SIMD128-NEXT: .functype load_ext_v2i64_with_unfolded_gep_negative_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, -8{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: i64x2.load32x2_u $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x i32> @load_ext_v2i64_with_unfolded_gep_negative_offset(<2 x i32>* %p) {
+  %s = getelementptr inbounds <2 x i32>, <2 x i32>* %p, i32 -1
+  %v = load <2 x i32>, <2 x i32>* %s
+  ret <2 x i32> %v
+}
+
 ; CHECK-LABEL: load_v2i64_with_unfolded_offset:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-VM-NOT: v128
@@ -647,6 +1835,74 @@ define <2 x i64> @load_v2i64_with_unfolded_offset(<2 x i64>* %p) {
   ret <2 x i64> %v
 }
 
+; CHECK-LABEL: load_splat_v2i64_with_unfolded_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v2i64_with_unfolded_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 16{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: v64x2.load_splat $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x i64> @load_splat_v2i64_with_unfolded_offset(i64* %p) {
+  %q = ptrtoint i64* %p to i32
+  %r = add nsw i32 %q, 16
+  %s = inttoptr i32 %r to i64*
+  %e = load i64, i64* %s
+  %v1 = insertelement <2 x i64> undef, i64 %e, i32 0
+  %v2 = shufflevector <2 x i64> %v1, <2 x i64> undef, <2 x i32> zeroinitializer
+  ret <2 x i64> %v2
+}
+
+; CHECK-LABEL: load_sext_v2i64_with_unfolded_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: v128
+; SIMD128-NEXT: .functype load_sext_v2i64_with_unfolded_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 16{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: i64x2.load32x2_s $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x i64> @load_sext_v2i64_with_unfolded_offset(<2 x i32>* %p) {
+  %q = ptrtoint <2 x i32>* %p to i32
+  %r = add nsw i32 %q, 16
+  %s = inttoptr i32 %r to <2 x i32>*
+  %v = load <2 x i32>, <2 x i32>* %s
+  %v2 = sext <2 x i32> %v to <2 x i64>
+  ret <2 x i64> %v2
+}
+
+; CHECK-LABEL: load_zext_v2i64_with_unfolded_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: v128
+; SIMD128-NEXT: .functype load_zext_v2i64_with_unfolded_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 16{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: i64x2.load32x2_u $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x i64> @load_zext_v2i64_with_unfolded_offset(<2 x i32>* %p) {
+  %q = ptrtoint <2 x i32>* %p to i32
+  %r = add nsw i32 %q, 16
+  %s = inttoptr i32 %r to <2 x i32>*
+  %v = load <2 x i32>, <2 x i32>* %s
+  %v2 = zext <2 x i32> %v to <2 x i64>
+  ret <2 x i64> %v2
+}
+
+; CHECK-LABEL: load_ext_v2i64_with_unfolded_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: load32x2
+; SIMD128-NEXT: .functype load_ext_v2i64_with_unfolded_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 16{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: i64x2.load32x2_u $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x i32> @load_ext_v2i64_with_unfolded_offset(<2 x i32>* %p) {
+  %q = ptrtoint <2 x i32>* %p to i32
+  %r = add nsw i32 %q, 16
+  %s = inttoptr i32 %r to <2 x i32>*
+  %v = load <2 x i32>, <2 x i32>* %s
+  ret <2 x i32> %v
+}
+
 ; CHECK-LABEL: load_v2i64_with_unfolded_gep_offset:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-VM-NOT: v128
@@ -661,6 +1917,66 @@ define <2 x i64> @load_v2i64_with_unfolded_gep_offset(<2 x i64>* %p) {
   ret <2 x i64> %v
 }
 
+; CHECK-LABEL: load_splat_v2i64_with_unfolded_gep_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v2i64_with_unfolded_gep_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 8{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: v64x2.load_splat $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x i64> @load_splat_v2i64_with_unfolded_gep_offset(i64* %p) {
+  %s = getelementptr i64, i64* %p, i32 1
+  %e = load i64, i64* %s
+  %v1 = insertelement <2 x i64> undef, i64 %e, i32 0
+  %v2 = shufflevector <2 x i64> %v1, <2 x i64> undef, <2 x i32> zeroinitializer
+  ret <2 x i64> %v2
+}
+
+; CHECK-LABEL: load_sext_v2i64_with_unfolded_gep_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: v128
+; SIMD128-NEXT: .functype load_sext_v2i64_with_unfolded_gep_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 8{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: i64x2.load32x2_s $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x i64> @load_sext_v2i64_with_unfolded_gep_offset(<2 x i32>* %p) {
+  %s = getelementptr <2 x i32>, <2 x i32>* %p, i32 1
+  %v = load <2 x i32>, <2 x i32>* %s
+  %v2 = sext <2 x i32> %v to <2 x i64>
+  ret <2 x i64> %v2
+}
+
+; CHECK-LABEL: load_zext_v2i64_with_unfolded_gep_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: v128
+; SIMD128-NEXT: .functype load_zext_v2i64_with_unfolded_gep_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 8{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: i64x2.load32x2_u $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x i64> @load_zext_v2i64_with_unfolded_gep_offset(<2 x i32>* %p) {
+  %s = getelementptr <2 x i32>, <2 x i32>* %p, i32 1
+  %v = load <2 x i32>, <2 x i32>* %s
+  %v2 = zext <2 x i32> %v to <2 x i64>
+  ret <2 x i64> %v2
+}
+
+; CHECK-LABEL: load_ext_v2i64_with_unfolded_gep_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: load32x2
+; SIMD128-NEXT: .functype load_ext_v2i64_with_unfolded_gep_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 8{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: i64x2.load32x2_u $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x i32> @load_ext_v2i64_with_unfolded_gep_offset(<2 x i32>* %p) {
+  %s = getelementptr <2 x i32>, <2 x i32>* %p, i32 1
+  %v = load <2 x i32>, <2 x i32>* %s
+  ret <2 x i32> %v
+}
+
 ; CHECK-LABEL: load_v2i64_from_numeric_address:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-VM-NOT: v128
@@ -674,6 +1990,62 @@ define <2 x i64> @load_v2i64_from_numeric_address() {
   ret <2 x i64> %v
 }
 
+; CHECK-LABEL: load_splat_v2i64_from_numeric_address:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v2i64_from_numeric_address () -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
+; SIMD128-NEXT: v64x2.load_splat $push[[R:[0-9]+]]=, 32($pop[[L0]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x i64> @load_splat_v2i64_from_numeric_address() {
+  %s = inttoptr i32 32 to i64*
+  %e = load i64, i64* %s
+  %v1 = insertelement <2 x i64> undef, i64 %e, i32 0
+  %v2 = shufflevector <2 x i64> %v1, <2 x i64> undef, <2 x i32> zeroinitializer
+  ret <2 x i64> %v2
+}
+
+; CHECK-LABEL: load_sext_v2i64_from_numeric_address:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: v128
+; SIMD128-NEXT: .functype load_sext_v2i64_from_numeric_address () -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
+; SIMD128-NEXT: i64x2.load32x2_s $push[[R:[0-9]+]]=, 32($pop[[L0]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x i64> @load_sext_v2i64_from_numeric_address() {
+  %s = inttoptr i32 32 to <2 x i32>*
+  %v = load <2 x i32>, <2 x i32>* %s
+  %v2 = sext <2 x i32> %v to <2 x i64>
+  ret <2 x i64> %v2
+}
+
+; CHECK-LABEL: load_zext_v2i64_from_numeric_address:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: v128
+; SIMD128-NEXT: .functype load_zext_v2i64_from_numeric_address () -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
+; SIMD128-NEXT: i64x2.load32x2_u $push[[R:[0-9]+]]=, 32($pop[[L0]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x i64> @load_zext_v2i64_from_numeric_address() {
+  %s = inttoptr i32 32 to <2 x i32>*
+  %v = load <2 x i32>, <2 x i32>* %s
+  %v2 = zext <2 x i32> %v to <2 x i64>
+  ret <2 x i64> %v2
+}
+
+; CHECK-LABEL: load_ext_v2i64_from_numeric_address:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: load32x2
+; SIMD128-NEXT: .functype load_ext_v2i64_from_numeric_address () -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
+; SIMD128-NEXT: i64x2.load32x2_u $push[[R:[0-9]+]]=, 32($pop[[L0]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x i32> @load_ext_v2i64_from_numeric_address() {
+  %s = inttoptr i32 32 to <2 x i32>*
+  %v = load <2 x i32>, <2 x i32>* %s
+  ret <2 x i32> %v
+}
+
 ; CHECK-LABEL: load_v2i64_from_global_address:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-VM-NOT: v128
@@ -687,6 +2059,60 @@ define <2 x i64> @load_v2i64_from_global_address() {
   ret <2 x i64> %v
 }
 
+; CHECK-LABEL: load_splat_v2i64_from_global_address:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v2i64_from_global_address () -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
+; SIMD128-NEXT: v64x2.load_splat $push[[R:[0-9]+]]=, gv_i64($pop[[L0]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+@gv_i64 = global i64 42
+define <2 x i64> @load_splat_v2i64_from_global_address() {
+  %e = load i64, i64* @gv_i64
+  %v1 = insertelement <2 x i64> undef, i64 %e, i32 0
+  %v2 = shufflevector <2 x i64> %v1, <2 x i64> undef, <2 x i32> zeroinitializer
+  ret <2 x i64> %v2
+}
+
+; CHECK-LABEL: load_sext_v2i64_from_global_address:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: v128
+; SIMD128-NEXT: .functype load_sext_v2i64_from_global_address () -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
+; SIMD128-NEXT: i64x2.load32x2_s $push[[R:[0-9]+]]=, gv_v2i32($pop[[L0]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+@gv_v2i32 = global <2 x i32> <i32 42, i32 42>
+define <2 x i64> @load_sext_v2i64_from_global_address() {
+  %v = load <2 x i32>, <2 x i32>* @gv_v2i32
+  %v2 = sext <2 x i32> %v to <2 x i64>
+  ret <2 x i64> %v2
+}
+
+; CHECK-LABEL: load_zext_v2i64_from_global_address:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: v128
+; SIMD128-NEXT: .functype load_zext_v2i64_from_global_address () -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
+; SIMD128-NEXT: i64x2.load32x2_u $push[[R:[0-9]+]]=, gv_v2i32($pop[[L0]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x i64> @load_zext_v2i64_from_global_address() {
+  %v = load <2 x i32>, <2 x i32>* @gv_v2i32
+  %v2 = zext <2 x i32> %v to <2 x i64>
+  ret <2 x i64> %v2
+}
+
+; CHECK-LABEL: load_ext_v2i64_from_global_address:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: load32x2
+; SIMD128-NEXT: .functype load_ext_v2i64_from_global_address () -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
+; SIMD128-NEXT: i64x2.load32x2_u $push[[R:[0-9]+]]=, gv_v2i32($pop[[L0]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x i32> @load_ext_v2i64_from_global_address() {
+  %v = load <2 x i32>, <2 x i32>* @gv_v2i32
+  ret <2 x i32> %v
+}
+
 ; CHECK-LABEL: store_v2i64:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-VM-NOT: v128
@@ -796,6 +2222,18 @@ define <4 x float> @load_v4f32(<4 x float>* %p) {
   ret <4 x float> %v
 }
 
+; CHECK-LABEL: load_splat_v4f32:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v4f32 (i32) -> (v128){{$}}
+; SIMD128-NEXT: v32x4.load_splat $push[[R:[0-9]+]]=, 0($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x float> @load_splat_v4f32(float* %p) {
+  %e = load float, float* %p
+  %v1 = insertelement <4 x float> undef, float %e, i32 0
+  %v2 = shufflevector <4 x float> %v1, <4 x float> undef, <4 x i32> zeroinitializer
+  ret <4 x float> %v2
+}
+
 ; CHECK-LABEL: load_v4f32_with_folded_offset:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype load_v4f32_with_folded_offset (i32) -> (v128){{$}}
@@ -809,6 +2247,21 @@ define <4 x float> @load_v4f32_with_folded_offset(<4 x float>* %p) {
   ret <4 x float> %v
 }
 
+; CHECK-LABEL: load_splat_v4f32_with_folded_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v4f32_with_folded_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: v32x4.load_splat $push[[R:[0-9]+]]=, 16($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x float> @load_splat_v4f32_with_folded_offset(float* %p) {
+  %q = ptrtoint float* %p to i32
+  %r = add nuw i32 %q, 16
+  %s = inttoptr i32 %r to float*
+  %e = load float, float* %s
+  %v1 = insertelement <4 x float> undef, float %e, i32 0
+  %v2 = shufflevector <4 x float> %v1, <4 x float> undef, <4 x i32> zeroinitializer
+  ret <4 x float> %v2
+}
+
 ; CHECK-LABEL: load_v4f32_with_folded_gep_offset:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype load_v4f32_with_folded_gep_offset (i32) -> (v128){{$}}
@@ -820,6 +2273,19 @@ define <4 x float> @load_v4f32_with_folded_gep_offset(<4 x float>* %p) {
   ret <4 x float> %v
 }
 
+; CHECK-LABEL: load_splat_v4f32_with_folded_gep_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v4f32_with_folded_gep_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: v32x4.load_splat $push[[R:[0-9]+]]=, 4($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x float> @load_splat_v4f32_with_folded_gep_offset(float* %p) {
+  %s = getelementptr inbounds float, float* %p, i32 1
+  %e = load float, float* %s
+  %v1 = insertelement <4 x float> undef, float %e, i32 0
+  %v2 = shufflevector <4 x float> %v1, <4 x float> undef, <4 x i32> zeroinitializer
+  ret <4 x float> %v2
+}
+
 ; CHECK-LABEL: load_v4f32_with_unfolded_gep_negative_offset:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype load_v4f32_with_unfolded_gep_negative_offset (i32) -> (v128){{$}}
@@ -833,6 +2299,21 @@ define <4 x float> @load_v4f32_with_unfolded_gep_negative_offset(<4 x float>* %p
   ret <4 x float> %v
 }
 
+; CHECK-LABEL: load_splat_v4f32_with_unfolded_gep_negative_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v4f32_with_unfolded_gep_negative_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, -4{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: v32x4.load_splat $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x float> @load_splat_v4f32_with_unfolded_gep_negative_offset(float* %p) {
+  %s = getelementptr inbounds float, float* %p, i32 -1
+  %e = load float, float* %s
+  %v1 = insertelement <4 x float> undef, float %e, i32 0
+  %v2 = shufflevector <4 x float> %v1, <4 x float> undef, <4 x i32> zeroinitializer
+  ret <4 x float> %v2
+}
+
 ; CHECK-LABEL: load_v4f32_with_unfolded_offset:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype load_v4f32_with_unfolded_offset (i32) -> (v128){{$}}
@@ -848,6 +2329,23 @@ define <4 x float> @load_v4f32_with_unfolded_offset(<4 x float>* %p) {
   ret <4 x float> %v
 }
 
+; CHECK-LABEL: load_splat_v4f32_with_unfolded_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v4f32_with_unfolded_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 16{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: v32x4.load_splat $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x float> @load_splat_v4f32_with_unfolded_offset(float* %p) {
+  %q = ptrtoint float* %p to i32
+  %r = add nsw i32 %q, 16
+  %s = inttoptr i32 %r to float*
+  %e = load float, float* %s
+  %v1 = insertelement <4 x float> undef, float %e, i32 0
+  %v2 = shufflevector <4 x float> %v1, <4 x float> undef, <4 x i32> zeroinitializer
+  ret <4 x float> %v2
+}
+
 ; CHECK-LABEL: load_v4f32_with_unfolded_gep_offset:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype load_v4f32_with_unfolded_gep_offset (i32) -> (v128){{$}}
@@ -861,6 +2359,21 @@ define <4 x float> @load_v4f32_with_unfolded_gep_offset(<4 x float>* %p) {
   ret <4 x float> %v
 }
 
+; CHECK-LABEL: load_splat_v4f32_with_unfolded_gep_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v4f32_with_unfolded_gep_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 4{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: v32x4.load_splat $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x float> @load_splat_v4f32_with_unfolded_gep_offset(float* %p) {
+  %s = getelementptr float, float* %p, i32 1
+  %e = load float, float* %s
+  %v1 = insertelement <4 x float> undef, float %e, i32 0
+  %v2 = shufflevector <4 x float> %v1, <4 x float> undef, <4 x i32> zeroinitializer
+  ret <4 x float> %v2
+}
+
 ; CHECK-LABEL: load_v4f32_from_numeric_address:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype load_v4f32_from_numeric_address () -> (v128){{$}}
@@ -873,6 +2386,20 @@ define <4 x float> @load_v4f32_from_numeric_address() {
   ret <4 x float> %v
 }
 
+; CHECK-LABEL: load_splat_v4f32_from_numeric_address:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v4f32_from_numeric_address () -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
+; SIMD128-NEXT: v32x4.load_splat $push[[R:[0-9]+]]=, 32($pop[[L0]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x float> @load_splat_v4f32_from_numeric_address() {
+  %s = inttoptr i32 32 to float*
+  %e = load float, float* %s
+  %v1 = insertelement <4 x float> undef, float %e, i32 0
+  %v2 = shufflevector <4 x float> %v1, <4 x float> undef, <4 x i32> zeroinitializer
+  ret <4 x float> %v2
+}
+
 ; CHECK-LABEL: load_v4f32_from_global_address:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype load_v4f32_from_global_address () -> (v128){{$}}
@@ -885,6 +2412,20 @@ define <4 x float> @load_v4f32_from_global_address() {
   ret <4 x float> %v
 }
 
+; CHECK-LABEL: load_splat_v4f32_from_global_address:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v4f32_from_global_address () -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
+; SIMD128-NEXT: v32x4.load_splat $push[[R:[0-9]+]]=, gv_f32($pop[[L0]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+@gv_f32 = global float 42.
+define <4 x float> @load_splat_v4f32_from_global_address() {
+  %e = load float, float* @gv_f32
+  %v1 = insertelement <4 x float> undef, float %e, i32 0
+  %v2 = shufflevector <4 x float> %v1, <4 x float> undef, <4 x i32> zeroinitializer
+  ret <4 x float> %v2
+}
+
 ; CHECK-LABEL: store_v4f32:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype store_v4f32 (v128, i32) -> (){{$}}
@@ -987,6 +2528,19 @@ define <2 x double> @load_v2f64(<2 x double>* %p) {
   ret <2 x double> %v
 }
 
+; CHECK-LABEL: load_splat_v2f64:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v2f64 (i32) -> (v128){{$}}
+; SIMD128-NEXT: v64x2.load_splat $push[[R:[0-9]+]]=, 0($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x double> @load_splat_v2f64(double* %p) {
+  %e = load double, double* %p
+  %v1 = insertelement <2 x double> undef, double %e, i32 0
+  %v2 = shufflevector <2 x double> %v1, <2 x double> undef, <2 x i32> zeroinitializer
+  ret <2 x double> %v2
+}
+
 ; CHECK-LABEL: load_v2f64_with_folded_offset:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-VM-NOT: v128
@@ -1001,6 +2555,22 @@ define <2 x double> @load_v2f64_with_folded_offset(<2 x double>* %p) {
   ret <2 x double> %v
 }
 
+; CHECK-LABEL: load_splat_v2f64_with_folded_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v2f64_with_folded_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: v64x2.load_splat $push[[R:[0-9]+]]=, 16($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x double> @load_splat_v2f64_with_folded_offset(double* %p) {
+  %q = ptrtoint double* %p to i32
+  %r = add nuw i32 %q, 16
+  %s = inttoptr i32 %r to double*
+  %e = load double, double* %s
+  %v1 = insertelement <2 x double> undef, double %e, i32 0
+  %v2 = shufflevector <2 x double> %v1, <2 x double> undef, <2 x i32> zeroinitializer
+  ret <2 x double> %v2
+}
+
 ; CHECK-LABEL: load_v2f64_with_folded_gep_offset:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-VM-NOT: v128
@@ -1013,6 +2583,20 @@ define <2 x double> @load_v2f64_with_folded_gep_offset(<2 x double>* %p) {
   ret <2 x double> %v
 }
 
+; CHECK-LABEL: load_splat_v2f64_with_folded_gep_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v2f64_with_folded_gep_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: v64x2.load_splat $push[[R:[0-9]+]]=, 8($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x double> @load_splat_v2f64_with_folded_gep_offset(double* %p) {
+  %s = getelementptr inbounds double, double* %p, i32 1
+  %e = load double, double* %s
+  %v1 = insertelement <2 x double> undef, double %e, i32 0
+  %v2 = shufflevector <2 x double> %v1, <2 x double> undef, <2 x i32> zeroinitializer
+  ret <2 x double> %v2
+}
+
 ; CHECK-LABEL: load_v2f64_with_unfolded_gep_negative_offset:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-VM-NOT: v128
@@ -1027,6 +2611,22 @@ define <2 x double> @load_v2f64_with_unfolded_gep_negative_offset(<2 x double>*
   ret <2 x double> %v
 }
 
+; CHECK-LABEL: load_splat_v2f64_with_unfolded_gep_negative_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v2f64_with_unfolded_gep_negative_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, -8{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: v64x2.load_splat $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x double> @load_splat_v2f64_with_unfolded_gep_negative_offset(double* %p) {
+  %s = getelementptr inbounds double, double* %p, i32 -1
+  %e = load double, double* %s
+  %v1 = insertelement <2 x double> undef, double %e, i32 0
+  %v2 = shufflevector <2 x double> %v1, <2 x double> undef, <2 x i32> zeroinitializer
+  ret <2 x double> %v2
+}
+
 ; CHECK-LABEL: load_v2f64_with_unfolded_offset:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-VM-NOT: v128
@@ -1043,6 +2643,24 @@ define <2 x double> @load_v2f64_with_unfolded_offset(<2 x double>* %p) {
   ret <2 x double> %v
 }
 
+; CHECK-LABEL: load_splat_v2f64_with_unfolded_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v2f64_with_unfolded_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 16{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: v64x2.load_splat $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x double> @load_splat_v2f64_with_unfolded_offset(double* %p) {
+  %q = ptrtoint double* %p to i32
+  %r = add nsw i32 %q, 16
+  %s = inttoptr i32 %r to double*
+  %e = load double, double* %s
+  %v1 = insertelement <2 x double> undef, double %e, i32 0
+  %v2 = shufflevector <2 x double> %v1, <2 x double> undef, <2 x i32> zeroinitializer
+  ret <2 x double> %v2
+}
+
 ; CHECK-LABEL: load_v2f64_with_unfolded_gep_offset:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-VM-NOT: v128
@@ -1057,6 +2675,22 @@ define <2 x double> @load_v2f64_with_unfolded_gep_offset(<2 x double>* %p) {
   ret <2 x double> %v
 }
 
+; CHECK-LABEL: load_splat_v2f64_with_unfolded_gep_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v2f64_with_unfolded_gep_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 8{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: v64x2.load_splat $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x double> @load_splat_v2f64_with_unfolded_gep_offset(double* %p) {
+  %s = getelementptr double, double* %p, i32 1
+  %e = load double, double* %s
+  %v1 = insertelement <2 x double> undef, double %e, i32 0
+  %v2 = shufflevector <2 x double> %v1, <2 x double> undef, <2 x i32> zeroinitializer
+  ret <2 x double> %v2
+}
+
 ; CHECK-LABEL: load_v2f64_from_numeric_address:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-VM-NOT: v128
@@ -1070,6 +2704,21 @@ define <2 x double> @load_v2f64_from_numeric_address() {
   ret <2 x double> %v
 }
 
+; CHECK-LABEL: load_splat_v2f64_from_numeric_address:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v2f64_from_numeric_address () -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
+; SIMD128-NEXT: v64x2.load_splat $push[[R:[0-9]+]]=, 32($pop[[L0]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x double> @load_splat_v2f64_from_numeric_address() {
+  %s = inttoptr i32 32 to double*
+  %e = load double, double* %s
+  %v1 = insertelement <2 x double> undef, double %e, i32 0
+  %v2 = shufflevector <2 x double> %v1, <2 x double> undef, <2 x i32> zeroinitializer
+  ret <2 x double> %v2
+}
+
 ; CHECK-LABEL: load_v2f64_from_global_address:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-VM-NOT: v128
@@ -1083,6 +2732,21 @@ define <2 x double> @load_v2f64_from_global_address() {
   ret <2 x double> %v
 }
 
+; CHECK-LABEL: load_splat_v2f64_from_global_address:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v2f64_from_global_address () -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
+; SIMD128-NEXT: v64x2.load_splat $push[[R:[0-9]+]]=, gv_f64($pop[[L0]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+@gv_f64 = global double 42.
+define <2 x double> @load_splat_v2f64_from_global_address() {
+  %e = load double, double* @gv_f64
+  %v1 = insertelement <2 x double> undef, double %e, i32 0
+  %v2 = shufflevector <2 x double> %v1, <2 x double> undef, <2 x i32> zeroinitializer
+  ret <2 x double> %v2
+}
+
 ; CHECK-LABEL: store_v2f64:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-VM-NOT: v128
diff --git a/llvm/test/CodeGen/WebAssembly/simd.ll b/llvm/test/CodeGen/WebAssembly/simd.ll
index 35f77c0da..b2d063806 100644
--- a/llvm/test/CodeGen/WebAssembly/simd.ll
+++ b/llvm/test/CodeGen/WebAssembly/simd.ll
@@ -193,13 +193,13 @@ define <16 x i8> @replace_var_v16i8(<16 x i8> %v, i32 %i, i8 %x) {
   ret <16 x i8> %res
 }
 
-; CHECK-LABEL: replace_undef_v16i8:
+; CHECK-LABEL: replace_zero_v16i8:
 ; NO-SIMD128-NOT: i8x16
-; SIMD128-NEXT: .functype replace_undef_v16i8 (v128, i32) -> (v128){{$}}
+; SIMD128-NEXT: .functype replace_zero_v16i8 (v128, i32) -> (v128){{$}}
 ; SIMD128-NEXT: i8x16.replace_lane $push[[R:[0-9]+]]=, $0, 0, $1{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
-define <16 x i8> @replace_undef_v16i8(<16 x i8> %v, i8 %x) {
-  %res = insertelement <16 x i8> %v, i8 %x, i32 undef
+define <16 x i8> @replace_zero_v16i8(<16 x i8> %v, i8 %x) {
+  %res = insertelement <16 x i8> %v, i8 %x, i32 0
   ret <16 x i8> %res
 }
 
@@ -464,13 +464,13 @@ define <8 x i16> @replace_var_v8i16(<8 x i16> %v, i32 %i, i16 %x) {
   ret <8 x i16> %res
 }
 
-; CHECK-LABEL: replace_undef_v8i16:
+; CHECK-LABEL: replace_zero_v8i16:
 ; NO-SIMD128-NOT: i16x8
-; SIMD128-NEXT: .functype replace_undef_v8i16 (v128, i32) -> (v128){{$}}
+; SIMD128-NEXT: .functype replace_zero_v8i16 (v128, i32) -> (v128){{$}}
 ; SIMD128-NEXT: i16x8.replace_lane $push[[R:[0-9]+]]=, $0, 0, $1{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
-define <8 x i16> @replace_undef_v8i16(<8 x i16> %v, i16 %x) {
-  %res = insertelement <8 x i16> %v, i16 %x, i32 undef
+define <8 x i16> @replace_zero_v8i16(<8 x i16> %v, i16 %x) {
+  %res = insertelement <8 x i16> %v, i16 %x, i32 0
   ret <8 x i16> %res
 }
 
@@ -584,13 +584,13 @@ define i32 @extract_var_v4i32(<4 x i32> %v, i32 %i) {
   ret i32 %elem
 }
 
-; CHECK-LABEL: extract_undef_v4i32:
+; CHECK-LABEL: extract_zero_v4i32:
 ; NO-SIMD128-NOT: i32x4
-; SIMD128-NEXT: .functype extract_undef_v4i32 (v128) -> (i32){{$}}
+; SIMD128-NEXT: .functype extract_zero_v4i32 (v128) -> (i32){{$}}
 ; SIMD128-NEXT: i32x4.extract_lane $push[[R:[0-9]+]]=, $0, 0{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
-define i32 @extract_undef_v4i32(<4 x i32> %v) {
-  %elem = extractelement <4 x i32> %v, i32 undef
+define i32 @extract_zero_v4i32(<4 x i32> %v) {
+  %elem = extractelement <4 x i32> %v, i32 0
   ret i32 %elem
 }
 
@@ -625,13 +625,13 @@ define <4 x i32> @replace_var_v4i32(<4 x i32> %v, i32 %i, i32 %x) {
   ret <4 x i32> %res
 }
 
-; CHECK-LABEL: replace_undef_v4i32:
+; CHECK-LABEL: replace_zero_v4i32:
 ; NO-SIMD128-NOT: i32x4
-; SIMD128-NEXT: .functype replace_undef_v4i32 (v128, i32) -> (v128){{$}}
+; SIMD128-NEXT: .functype replace_zero_v4i32 (v128, i32) -> (v128){{$}}
 ; SIMD128-NEXT: i32x4.replace_lane $push[[R:[0-9]+]]=, $0, 0, $1{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
-define <4 x i32> @replace_undef_v4i32(<4 x i32> %v, i32 %x) {
-  %res = insertelement <4 x i32> %v, i32 %x, i32 undef
+define <4 x i32> @replace_zero_v4i32(<4 x i32> %v, i32 %x) {
+  %res = insertelement <4 x i32> %v, i32 %x, i32 0
   ret <4 x i32> %res
 }
 
@@ -737,14 +737,14 @@ define i64 @extract_var_v2i64(<2 x i64> %v, i32 %i) {
   ret i64 %elem
 }
 
-; CHECK-LABEL: extract_undef_v2i64:
+; CHECK-LABEL: extract_zero_v2i64:
 ; NO-SIMD128-NOT: i64x2
 ; SIMD128-VM-NOT: i64x2
-; SIMD128-NEXT: .functype extract_undef_v2i64 (v128) -> (i64){{$}}
+; SIMD128-NEXT: .functype extract_zero_v2i64 (v128) -> (i64){{$}}
 ; SIMD128-NEXT: i64x2.extract_lane $push[[R:[0-9]+]]=, $0, 0{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
-define i64 @extract_undef_v2i64(<2 x i64> %v) {
-  %elem = extractelement <2 x i64> %v, i64 undef
+define i64 @extract_zero_v2i64(<2 x i64> %v) {
+  %elem = extractelement <2 x i64> %v, i64 0
   ret i64 %elem
 }
 
@@ -781,14 +781,14 @@ define <2 x i64> @replace_var_v2i64(<2 x i64> %v, i32 %i, i64 %x) {
   ret <2 x i64> %res
 }
 
-; CHECK-LABEL: replace_undef_v2i64:
+; CHECK-LABEL: replace_zero_v2i64:
 ; NO-SIMD128-NOT: i64x2
 ; SIMD128-VM-NOT: i64x2
-; SIMD128-NEXT: .functype replace_undef_v2i64 (v128, i64) -> (v128){{$}}
+; SIMD128-NEXT: .functype replace_zero_v2i64 (v128, i64) -> (v128){{$}}
 ; SIMD128-NEXT: i64x2.replace_lane $push[[R:[0-9]+]]=, $0, 0, $1{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
-define <2 x i64> @replace_undef_v2i64(<2 x i64> %v, i64 %x) {
-  %res = insertelement <2 x i64> %v, i64 %x, i32 undef
+define <2 x i64> @replace_zero_v2i64(<2 x i64> %v, i64 %x) {
+  %res = insertelement <2 x i64> %v, i64 %x, i32 0
   ret <2 x i64> %res
 }
 
@@ -890,13 +890,13 @@ define float @extract_var_v4f32(<4 x float> %v, i32 %i) {
   ret float %elem
 }
 
-; CHECK-LABEL: extract_undef_v4f32:
+; CHECK-LABEL: extract_zero_v4f32:
 ; NO-SIMD128-NOT: f32x4
-; SIMD128-NEXT: .functype extract_undef_v4f32 (v128) -> (f32){{$}}
+; SIMD128-NEXT: .functype extract_zero_v4f32 (v128) -> (f32){{$}}
 ; SIMD128-NEXT: f32x4.extract_lane $push[[R:[0-9]+]]=, $0, 0{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
-define float @extract_undef_v4f32(<4 x float> %v) {
-  %elem = extractelement <4 x float> %v, i32 undef
+define float @extract_zero_v4f32(<4 x float> %v) {
+  %elem = extractelement <4 x float> %v, i32 0
   ret float %elem
 }
 
@@ -931,13 +931,13 @@ define <4 x float> @replace_var_v4f32(<4 x float> %v, i32 %i, float %x) {
   ret <4 x float> %res
 }
 
-; CHECK-LABEL: replace_undef_v4f32:
+; CHECK-LABEL: replace_zero_v4f32:
 ; NO-SIMD128-NOT: f32x4
-; SIMD128-NEXT: .functype replace_undef_v4f32 (v128, f32) -> (v128){{$}}
+; SIMD128-NEXT: .functype replace_zero_v4f32 (v128, f32) -> (v128){{$}}
 ; SIMD128-NEXT: f32x4.replace_lane $push[[R:[0-9]+]]=, $0, 0, $1{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
-define <4 x float> @replace_undef_v4f32(<4 x float> %v, float %x) {
-  %res = insertelement <4 x float> %v, float %x, i32 undef
+define <4 x float> @replace_zero_v4f32(<4 x float> %v, float %x) {
+  %res = insertelement <4 x float> %v, float %x, i32 0
   ret <4 x float> %res
 }
 
@@ -1042,14 +1042,14 @@ define double @extract_var_v2f64(<2 x double> %v, i32 %i) {
   ret double %elem
 }
 
-; CHECK-LABEL: extract_undef_v2f64:
+; CHECK-LABEL: extract_zero_v2f64:
 ; NO-SIMD128-NOT: f64x2
 ; SIMD128-VM-NOT: f64x2
-; SIMD128-NEXT: .functype extract_undef_v2f64 (v128) -> (f64){{$}}
+; SIMD128-NEXT: .functype extract_zero_v2f64 (v128) -> (f64){{$}}
 ; SIMD128-NEXT: f64x2.extract_lane $push[[R:[0-9]+]]=, $0, 0{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
-define double @extract_undef_v2f64(<2 x double> %v) {
-  %elem = extractelement <2 x double> %v, i32 undef
+define double @extract_zero_v2f64(<2 x double> %v) {
+  %elem = extractelement <2 x double> %v, i32 0
   ret double %elem
 }
 
@@ -1086,14 +1086,14 @@ define <2 x double> @replace_var_v2f64(<2 x double> %v, i32 %i, double %x) {
   ret <2 x double> %res
 }
 
-; CHECK-LABEL: replace_undef_v2f64:
+; CHECK-LABEL: replace_zero_v2f64:
 ; NO-SIMD128-NOT: f64x2
 ; SIMD128-VM-NOT: f64x2
-; SIMD128-NEXT: .functype replace_undef_v2f64 (v128, f64) -> (v128){{$}}
+; SIMD128-NEXT: .functype replace_zero_v2f64 (v128, f64) -> (v128){{$}}
 ; SIMD128-NEXT: f64x2.replace_lane $push[[R:[0-9]+]]=, $0, 0, $1{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
-define <2 x double> @replace_undef_v2f64(<2 x double> %v, double %x) {
-  %res = insertelement <2 x double> %v, double %x, i32 undef
+define <2 x double> @replace_zero_v2f64(<2 x double> %v, double %x) {
+  %res = insertelement <2 x double> %v, double %x, i32 0
   ret <2 x double> %res
 }
 
diff --git a/llvm/test/CodeGen/WebAssembly/tailcall.ll b/llvm/test/CodeGen/WebAssembly/tailcall.ll
index 9061ed161..f4d4499bc 100644
--- a/llvm/test/CodeGen/WebAssembly/tailcall.ll
+++ b/llvm/test/CodeGen/WebAssembly/tailcall.ll
@@ -1,7 +1,8 @@
 ; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=+tail-call | FileCheck --check-prefixes=CHECK,SLOW %s
 ; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -fast-isel -mattr=+tail-call | FileCheck --check-prefixes=CHECK,FAST %s
+; RUN: llc < %s --filetype=obj -mattr=+tail-call | obj2yaml | FileCheck --check-prefix=YAML %s
 
-; Test that the tail-call attribute is accepted
+; Test that the tail calls lower correctly
 
 target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
 target triple = "wasm32-unknown-unknown"
@@ -124,6 +125,44 @@ define i32 @mismatched_prototypes() {
   ret i32 %v
 }
 
+; CHECK-LABEL: mismatched_return_void:
+; CHECK: i32.call $drop=, baz, $pop{{[0-9]+}}, $pop{{[0-9]+}}, $pop{{[0-9]+}}{{$}}
+; CHECK: return{{$}}
+define void @mismatched_return_void() {
+  %v = tail call i32 @baz(i32 0, i32 42, i32 6)
+  ret void
+}
+
+; CHECK-LABEL: mismatched_return_f32:
+; CHECK: i32.call $push[[L:[0-9]+]]=, baz, $pop{{[0-9]+}}, $pop{{[0-9]+}}, $pop{{[0-9]+}}{{$}}
+; CHECK: f32.reinterpret_i32 $push[[L1:[0-9]+]]=, $pop[[L]]{{$}}
+; CHECK: return $pop[[L1]]{{$}}
+define float @mismatched_return_f32() {
+  %v = tail call i32 @baz(i32 0, i32 42, i32 6)
+  %u = bitcast i32 %v to float
+  ret float %u
+}
+
+; CHECK-LABEL: mismatched_indirect_void:
+; CHECK: i32.call_indirect $drop=, $0, $1, $2, $0{{$}}
+; CHECK: return{{$}}
+define void @mismatched_indirect_void(%fn %f, i32 %x, i32 %y) {
+  %p = extractvalue %fn %f, 0
+  %v = tail call i32 %p(%fn %f, i32 %x, i32 %y)
+  ret void
+}
+
+; CHECK-LABEL: mismatched_indirect_f32:
+; CHECK: i32.call_indirect $push[[L:[0-9]+]]=, $0, $1, $2, $0{{$}}
+; CHECK: f32.reinterpret_i32 $push[[L1:[0-9]+]]=, $pop[[L]]{{$}}
+; CHECK: return $pop[[L1]]{{$}}
+define float @mismatched_indirect_f32(%fn %f, i32 %x, i32 %y) {
+  %p = extractvalue %fn %f, 0
+  %v = tail call i32 %p(%fn %f, i32 %x, i32 %y)
+  %u = bitcast i32 %v to float
+  ret float %u
+}
+
 ; CHECK-LABEL: mismatched_byval:
 ; CHECK: i32.store
 ; CHECK: return_call quux, $pop{{[0-9]+}}{{$}}
@@ -135,13 +174,60 @@ define i32 @mismatched_byval(i32* %x) {
 
 ; CHECK-LABEL: varargs:
 ; CHECK: i32.store
-; CHECK: return_call var, $1{{$}}
+; CHECK: i32.call $0=, var, $1{{$}}
+; CHECK: return $0{{$}}
 declare i32 @var(...)
 define i32 @varargs(i32 %x) {
   %v = tail call i32 (...) @var(i32 %x)
   ret i32 %v
 }
 
+; Type transformations inhibit tail calls, even when they are nops
+
+; CHECK-LABEL: mismatched_return_zext:
+; CHECK: i32.call
+define i32 @mismatched_return_zext() {
+  %v = tail call i1 @foo(i1 1)
+  %u = zext i1 %v to i32
+  ret i32 %u
+}
+
+; CHECK-LABEL: mismatched_return_sext:
+; CHECK: i32.call
+define i32 @mismatched_return_sext() {
+  %v = tail call i1 @foo(i1 1)
+  %u = sext i1 %v to i32
+  ret i32 %u
+}
+
+; CHECK-LABEL: mismatched_return_trunc:
+; CHECK: i32.call
+declare i32 @int()
+define i1 @mismatched_return_trunc() {
+  %v = tail call i32 @int()
+  %u = trunc i32 %v to i1
+  ret i1 %u
+}
+
+
+
+; Check that the signatures generated for external indirectly
+; return-called functions include the proper return types
+
+; YAML-LABEL: - Index:           8
+; YAML-NEXT:    ParamTypes:
+; YAML-NEXT:      - I32
+; YAML-NEXT:      - F32
+; YAML-NEXT:      - I64
+; YAML-NEXT:      - F64
+; YAML-NEXT:    ReturnTypes:
+; YAML-NEXT:      - I32
+define i32 @unique_caller(i32 (i32, float, i64, double)** %p) {
+  %f = load i32 (i32, float, i64, double)*, i32 (i32, float, i64, double)** %p
+  %v = tail call i32 %f(i32 0, float 0., i64 0, double 0.)
+  ret i32 %v
+}
+
 ; CHECK-LABEL: .section .custom_section.target_features
 ; CHECK-NEXT: .int8 1
 ; CHECK-NEXT: .int8 43
diff --git a/llvm/test/CodeGen/WebAssembly/tls-general-dynamic.ll b/llvm/test/CodeGen/WebAssembly/tls-general-dynamic.ll
index 3f6d9d325..41dbd476b 100644
--- a/llvm/test/CodeGen/WebAssembly/tls-general-dynamic.ll
+++ b/llvm/test/CodeGen/WebAssembly/tls-general-dynamic.ll
@@ -75,6 +75,35 @@ define i32 @tls_size() {
   ret i32 %1
 }
 
+; CHECK-LABEL: tls_align:
+; CHECK-NEXT: .functype tls_align () -> (i32)
+define i32 @tls_align() {
+; CHECK-NEXT: global.get __tls_align
+; CHECK-NEXT: return
+  %1 = call i32 @llvm.wasm.tls.align.i32()
+  ret i32 %1
+}
+
+; CHECK-LABEL: tls_base:
+; CHECK-NEXT: .functype tls_base () -> (i32)
+define i8* @tls_base() {
+; CHECK-NEXT: global.get __tls_base
+; CHECK-NEXT: return
+  %1 = call i8* @llvm.wasm.tls.base()
+  ret i8* %1
+}
+
+; CHECK-LABEL: tls_base_write:
+; CHECK-NEXT: .functype tls_base_write (i32) -> ()
+define void @tls_base_write(i8** %output) {
+; CHECK-NEXT: global.get __tls_base
+; CHECK-NEXT: i32.store 0
+; CHECK-NEXT: return
+  %1 = call i8* @llvm.wasm.tls.base()
+  store i8* %1, i8** %output
+  ret void
+}
+
 ; CHECK: .type tls,@object
 ; TLS-NEXT: .section .tbss.tls,"",@
 ; NO-TLS-NEXT: .section .bss.tls,"",@
@@ -84,3 +113,5 @@ define i32 @tls_size() {
 @tls = internal thread_local global i32 0
 
 declare i32 @llvm.wasm.tls.size.i32()
+declare i32 @llvm.wasm.tls.align.i32()
+declare i8* @llvm.wasm.tls.base()
diff --git a/llvm/test/CodeGen/WinCFGuard/cfguard-cast.ll b/llvm/test/CodeGen/WinCFGuard/cfguard-cast.ll
new file mode 100644
index 000000000..e66345b0d
--- /dev/null
+++ b/llvm/test/CodeGen/WinCFGuard/cfguard-cast.ll
@@ -0,0 +1,35 @@
+; RUN: llc < %s -mtriple=x86_64-pc-windows-msvc | FileCheck %s
+
+; Check how constant function pointer casts are handled.
+
+declare void @unprototyped(...)
+
+define i32 @call_unprototyped() {
+  call void bitcast (void (...)* @unprototyped to void ()*)()
+  ret i32 0
+}
+
+; CHECK-LABEL: call_unprototyped:
+; CHECK: callq unprototyped
+; CHECK: xorl %eax, %eax
+; CHECK: retq
+
+declare void @escaped_cast()
+
+define i32 @escape_it_with_cast(i8** %p) {
+  store i8* bitcast (void ()* @escaped_cast to i8*), i8** %p
+  ret i32 0
+}
+
+declare void @dead_constant()
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 2, !"cfguard", i32 1}
+
+!dead_constant_root = !{!1}
+!1 = !DITemplateValueParameter(name: "dead_constant", value: i8* bitcast (void ()* @dead_constant to i8*))
+
+; CHECK-LABEL: .section .gfids$y,"dr"
+; CHECK-NEXT:  .symidx escaped_cast
+; CHECK-NOT:   .symidx
+
diff --git a/llvm/test/CodeGen/WinCFGuard/cfguard.ll b/llvm/test/CodeGen/WinCFGuard/cfguard.ll
index 2ddd34632..3eb0e3f78 100644
--- a/llvm/test/CodeGen/WinCFGuard/cfguard.ll
+++ b/llvm/test/CodeGen/WinCFGuard/cfguard.ll
@@ -1,4 +1,5 @@
-; RUN: llc < %s | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-pc-windows-msvc | FileCheck %s
+; Control Flow Guard is currently only available on Windows
 
 ; CHECK: .set @feat.00, 2048
 
@@ -152,13 +153,13 @@ entry:
   ret void
 }
 
-attributes #0 = { noinline nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-features"="+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { noinline nounwind optnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-features"="+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noinline nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-features"="+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { noinline nounwind optnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-features"="+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind }
 
 !llvm.module.flags = !{!0, !1}
 !llvm.ident = !{!2}
 
-!0 = !{i32 2, !"cfguardtable", i32 1}
+!0 = !{i32 2, !"cfguard", i32 1}
 !1 = !{i32 1, !"wchar_size", i32 2}
 !2 = !{!"clang version 6.0.0 "}
diff --git a/llvm/test/CodeGen/WinEH/wineh-comdat.ll b/llvm/test/CodeGen/WinEH/wineh-comdat.ll
index 5484370a7..2ef1650d2 100644
--- a/llvm/test/CodeGen/WinEH/wineh-comdat.ll
+++ b/llvm/test/CodeGen/WinEH/wineh-comdat.ll
@@ -14,4 +14,4 @@ unwind:
 declare void @g()
 declare i32 @__CxxFrameHandler3(...)
 
-; CHECK: define internal i32 @"__ehhandler$f"(i8*, i8*, i8*, i8*){{ .+}} comdat($f) {
+; CHECK: define internal i32 @"__ehhandler$f"(i8* %0, i8* %1, i8* %2, i8* %3){{ .+}} comdat($f) {
diff --git a/llvm/test/CodeGen/WinEH/wineh-noret-cleanup.ll.x b/llvm/test/CodeGen/WinEH/wineh-noret-cleanup.ll.x
index 9f7c995e5..7d4d833aa 100644
--- a/llvm/test/CodeGen/WinEH/wineh-noret-cleanup.ll.x
+++ b/llvm/test/CodeGen/WinEH/wineh-noret-cleanup.ll.x
@@ -1,4 +1,3 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: sed -e s/.Cxx:// %s | llc -mtriple=x86_64-pc-windows-msvc | FileCheck %s --check-prefix=CXX
 ; RUN: sed -e s/.Seh:// %s | llc -mtriple=x86_64-pc-windows-msvc | FileCheck %s --check-prefix=SEH
 
@@ -69,13 +68,13 @@ catch.body.2:
 ; SEH-NEXT:    .long   .Ltmp0@IMGREL+1
 ; SEH-NEXT:    .long   .Ltmp1@IMGREL+1
 ; SEH-NEXT:    .long   dummy_filter@IMGREL
-; SEH-NEXT:    .long   .LBB0_2@IMGREL
+; SEH-NEXT:    .long   .LBB0_5@IMGREL
 ; SEH-NEXT:    .long   .Ltmp2@IMGREL+1
 ; SEH-NEXT:    .long   .Ltmp3@IMGREL+1
-; SEH-NEXT:    .long   "?dtor$5@?0?test@4HA"@IMGREL 
+; SEH-NEXT:    .long   "?dtor$2@?0?test@4HA"@IMGREL
 ; SEH-NEXT:    .long   0
 ; SEH-NEXT:    .long   .Ltmp2@IMGREL+1
 ; SEH-NEXT:    .long   .Ltmp3@IMGREL+1
 ; SEH-NEXT:    .long   dummy_filter@IMGREL
-; SEH-NEXT:    .long   .LBB0_2@IMGREL
+; SEH-NEXT:    .long   .LBB0_5@IMGREL
 ; SEH-NEXT:  .Llsda_end0:
diff --git a/llvm/test/CodeGen/WinEH/wineh-statenumbering.ll b/llvm/test/CodeGen/WinEH/wineh-statenumbering.ll
index 4f0b553b1..733724cb4 100644
--- a/llvm/test/CodeGen/WinEH/wineh-statenumbering.ll
+++ b/llvm/test/CodeGen/WinEH/wineh-statenumbering.ll
@@ -211,7 +211,7 @@ declare void @llvm.localescape(...)
 
 declare i32 @_except_handler3(...)
 
-attributes #0 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { noreturn }
 
 !llvm.ident = !{!0}
diff --git a/llvm/test/CodeGen/X86/2006-04-27-ISelFoldingBug.ll b/llvm/test/CodeGen/X86/2006-04-27-ISelFoldingBug.ll
index c70cbd9c3..982cc9e6e 100644
--- a/llvm/test/CodeGen/X86/2006-04-27-ISelFoldingBug.ll
+++ b/llvm/test/CodeGen/X86/2006-04-27-ISelFoldingBug.ll
@@ -1,11 +1,35 @@
-; RUN: llc < %s -mtriple=i686-apple-darwin8 -relocation-model=static > %t
-; RUN: grep "movl	_last" %t | count 1
-; RUN: grep "cmpl.*_last" %t | count 1
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-apple-darwin8 -relocation-model=static | FileCheck %s
 
 @block = external global i8*            ; <i8**> [#uses=1]
 @last = external global i32             ; <i32*> [#uses=3]
 
 define i1 @loadAndRLEsource_no_exit_2E_1_label_2E_0(i32 %tmp.21.reload, i32 %tmp.8) {
+; CHECK-LABEL: loadAndRLEsource_no_exit_2E_1_label_2E_0:
+; CHECK:       ## %bb.0: ## %newFuncRoot
+; CHECK-NEXT:    pushl %esi
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    .cfi_offset %esi, -8
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    movl _last, %edx
+; CHECK-NEXT:    leal 1(%edx), %esi
+; CHECK-NEXT:    movl %esi, _last
+; CHECK-NEXT:    movl _block, %esi
+; CHECK-NEXT:    movb %al, 1(%esi,%edx)
+; CHECK-NEXT:    cmpl %ecx, _last
+; CHECK-NEXT:    jge LBB0_3
+; CHECK-NEXT:  ## %bb.1: ## %label.0
+; CHECK-NEXT:    cmpl $257, %eax ## imm = 0x101
+; CHECK-NEXT:    je LBB0_3
+; CHECK-NEXT:  ## %bb.2: ## %label.0.no_exit.1_crit_edge.exitStub
+; CHECK-NEXT:    movb $1, %al
+; CHECK-NEXT:    popl %esi
+; CHECK-NEXT:    retl
+; CHECK-NEXT:  LBB0_3: ## %codeRepl5.exitStub
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    popl %esi
+; CHECK-NEXT:    retl
 newFuncRoot:
         br label %label.0
 label.0.no_exit.1_crit_edge.exitStub:           ; preds = %label.0
diff --git a/llvm/test/CodeGen/X86/2006-05-08-CoalesceSubRegClass.ll b/llvm/test/CodeGen/X86/2006-05-08-CoalesceSubRegClass.ll
index 6c100aa92..25bed30cc 100644
--- a/llvm/test/CodeGen/X86/2006-05-08-CoalesceSubRegClass.ll
+++ b/llvm/test/CodeGen/X86/2006-05-08-CoalesceSubRegClass.ll
@@ -1,13 +1,22 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; Coalescing from R32 to a subset R32_. Once another register coalescer bug is
 ; fixed, the movb should go away as well.
 
-; RUN: llc < %s -mtriple=i686-- -relocation-model=static | \
-; RUN:   grep movl
+; RUN: llc < %s -mtriple=i686-- -relocation-model=static | FileCheck %s
 
 @B = external global i32		; <i32*> [#uses=2]
 @C = external global i16*		; <i16**> [#uses=2]
 
 define void @test(i32 %A) {
+; CHECK-LABEL: test:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    movl %eax, %ecx
+; CHECK-NEXT:    andb $16, %cl
+; CHECK-NEXT:    shll %cl, B
+; CHECK-NEXT:    shrl $3, %eax
+; CHECK-NEXT:    addl %eax, C
+; CHECK-NEXT:    retl
 	%A.upgrd.1 = trunc i32 %A to i8		; <i8> [#uses=1]
 	%tmp2 = load i32, i32* @B		; <i32> [#uses=1]
 	%tmp3 = and i8 %A.upgrd.1, 16		; <i8> [#uses=1]
diff --git a/llvm/test/CodeGen/X86/2006-05-08-InstrSched.ll b/llvm/test/CodeGen/X86/2006-05-08-InstrSched.ll
index ffe1a9ce3..dfa003a8f 100644
--- a/llvm/test/CodeGen/X86/2006-05-08-InstrSched.ll
+++ b/llvm/test/CodeGen/X86/2006-05-08-InstrSched.ll
@@ -1,10 +1,24 @@
-; RUN: llc < %s -mtriple=i686-- -relocation-model=static | not grep "subl.*%esp"
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- -relocation-model=static | FileCheck %s
 
 @A = external global i16*		; <i16**> [#uses=1]
 @B = external global i32		; <i32*> [#uses=1]
 @C = external global i32		; <i32*> [#uses=2]
 
 define void @test() {
+; CHECK-LABEL: test:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl A, %eax
+; CHECK-NEXT:    movzwl 2(%eax), %eax
+; CHECK-NEXT:    movb B, %cl
+; CHECK-NEXT:    movl C, %edx
+; CHECK-NEXT:    andb $16, %cl
+; CHECK-NEXT:    shll %cl, %edx
+; CHECK-NEXT:    xorb $16, %cl
+; CHECK-NEXT:    shrl %cl, %eax
+; CHECK-NEXT:    orl %edx, %eax
+; CHECK-NEXT:    movl %eax, C
+; CHECK-NEXT:    retl
 	%tmp = load i16*, i16** @A		; <i16*> [#uses=1]
 	%tmp1 = getelementptr i16, i16* %tmp, i32 1		; <i16*> [#uses=1]
 	%tmp.upgrd.1 = load i16, i16* %tmp1		; <i16> [#uses=1]
diff --git a/llvm/test/CodeGen/X86/2006-07-28-AsmPrint-Long-As-Pointer.ll b/llvm/test/CodeGen/X86/2006-07-28-AsmPrint-Long-As-Pointer.ll
index 1c4c6478b..3819dcf07 100644
--- a/llvm/test/CodeGen/X86/2006-07-28-AsmPrint-Long-As-Pointer.ll
+++ b/llvm/test/CodeGen/X86/2006-07-28-AsmPrint-Long-As-Pointer.ll
@@ -1,5 +1,6 @@
-; RUN: llc < %s -mtriple=i686-- | grep -- 4294967240
+; RUN: llc < %s -mtriple=i686-- | FileCheck %s
 ; PR853
 
+; CHECK: 4294967240
 @X = global i32* inttoptr (i64 -56 to i32*)		; <i32**> [#uses=0]
 
diff --git a/llvm/test/CodeGen/X86/2006-08-21-ExtraMovInst.ll b/llvm/test/CodeGen/X86/2006-08-21-ExtraMovInst.ll
index 1601d684e..ac749bccb 100644
--- a/llvm/test/CodeGen/X86/2006-08-21-ExtraMovInst.ll
+++ b/llvm/test/CodeGen/X86/2006-08-21-ExtraMovInst.ll
@@ -1,7 +1,20 @@
-; RUN: llc < %s -mtriple=i686-- -mcpu=i386 | \
-; RUN:    not grep "movl %eax, %edx"
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- -mcpu=i386 | FileCheck %s
 
 define i32 @foo(i32 %t, i32 %C) {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    decl %eax
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB0_1: # %cond_true
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    incl %eax
+; CHECK-NEXT:    cmpl $40, %ecx
+; CHECK-NEXT:    jl .LBB0_1
+; CHECK-NEXT:  # %bb.2: # %bb12
+; CHECK-NEXT:    retl
 entry:
         br label %cond_true
 
diff --git a/llvm/test/CodeGen/X86/2006-10-10-FindModifiedNodeSlotBug.ll b/llvm/test/CodeGen/X86/2006-10-10-FindModifiedNodeSlotBug.ll
index 0b0fc0526..09ad58abb 100644
--- a/llvm/test/CodeGen/X86/2006-10-10-FindModifiedNodeSlotBug.ll
+++ b/llvm/test/CodeGen/X86/2006-10-10-FindModifiedNodeSlotBug.ll
@@ -1,9 +1,23 @@
-; RUN: llc < %s -mtriple=i686-- | grep shrl
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- | FileCheck %s
 ; Bug in FindModifiedNodeSlot cause tmp14 load to become a zextload and shr 31
 ; is then optimized away.
 @tree_code_type = external global [0 x i32]		; <[0 x i32]*> [#uses=1]
 
 define void @copy_if_shared_r() {
+; CHECK-LABEL: copy_if_shared_r:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl 0, %eax
+; CHECK-NEXT:    movzbl %al, %ecx
+; CHECK-NEXT:    movl tree_code_type(,%ecx,4), %ecx
+; CHECK-NEXT:    decl %ecx
+; CHECK-NEXT:    cmpl $2, %ecx
+; CHECK-NEXT:    ja .LBB0_2
+; CHECK-NEXT:  # %bb.1: # %cond_true
+; CHECK-NEXT:    shrl $31, %eax
+; CHECK-NEXT:    testb %al, %al
+; CHECK-NEXT:  .LBB0_2: # %cond_true17
+; CHECK-NEXT:    retl
 	%tmp = load i32, i32* null		; <i32> [#uses=1]
 	%tmp56 = and i32 %tmp, 255		; <i32> [#uses=1]
 	%gep.upgrd.1 = zext i32 %tmp56 to i64		; <i64> [#uses=1]
diff --git a/llvm/test/CodeGen/X86/2007-01-13-StackPtrIndex.ll b/llvm/test/CodeGen/X86/2007-01-13-StackPtrIndex.ll
index f8975e6c3..08ddd4222 100644
--- a/llvm/test/CodeGen/X86/2007-01-13-StackPtrIndex.ll
+++ b/llvm/test/CodeGen/X86/2007-01-13-StackPtrIndex.ll
@@ -1,5 +1,5 @@
-; RUN: llc < %s -mtriple=x86_64-- > %t
-; RUN: not grep ",%rsp)" %t
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-- | FileCheck %s
 ; PR1103
 
 target datalayout = "e-p:64:64"
@@ -7,6 +7,251 @@ target datalayout = "e-p:64:64"
 
 
 define void @foo(i32* %a0, i32* %a1, i32* %a2, i32* %a3, i32* %a4, i32* %a5) {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0: # %b
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movq %rsp, %rbp
+; CHECK-NEXT:    .cfi_def_cfa_register %rbp
+; CHECK-NEXT:    movslq (%rdi), %rax
+; CHECK-NEXT:    movslq (%rsi), %r8
+; CHECK-NEXT:    movslq (%rdx), %r10
+; CHECK-NEXT:    movl (%rcx), %edi
+; CHECK-NEXT:    movslq (%r9), %rcx
+; CHECK-NEXT:    movq %rsp, %rdx
+; CHECK-NEXT:    subl %eax, %r8d
+; CHECK-NEXT:    movslq %r8d, %rsi
+; CHECK-NEXT:    js .LBB0_1
+; CHECK-NEXT:  # %bb.11: # %b63
+; CHECK-NEXT:    testq %rsi, %rsi
+; CHECK-NEXT:    js .LBB0_14
+; CHECK-NEXT:  # %bb.12:
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB0_13: # %a25b
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    testb %al, %al
+; CHECK-NEXT:    je .LBB0_13
+; CHECK-NEXT:  .LBB0_14: # %b85
+; CHECK-NEXT:    movb $1, %al
+; CHECK-NEXT:    testb %al, %al
+; CHECK-NEXT:    jne .LBB0_1
+; CHECK-NEXT:  # %bb.15:
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB0_16: # %a25b140
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    testb %al, %al
+; CHECK-NEXT:    je .LBB0_16
+; CHECK-NEXT:  .LBB0_1: # %a29b
+; CHECK-NEXT:    cmpl %r10d, %edi
+; CHECK-NEXT:    js .LBB0_10
+; CHECK-NEXT:  # %bb.2: # %b158
+; CHECK-NEXT:    xorl %edi, %edi
+; CHECK-NEXT:    xorps %xmm0, %xmm0
+; CHECK-NEXT:    movb $1, %r10b
+; CHECK-NEXT:    jmp .LBB0_3
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB0_9: # %b1606
+; CHECK-NEXT:    # in Loop: Header=BB0_3 Depth=1
+; CHECK-NEXT:    testb %dil, %dil
+; CHECK-NEXT:    je .LBB0_10
+; CHECK-NEXT:  .LBB0_3: # %a29b173
+; CHECK-NEXT:    # =>This Loop Header: Depth=1
+; CHECK-NEXT:    # Child Loop BB0_37 Depth 2
+; CHECK-NEXT:    # Child Loop BB0_19 Depth 2
+; CHECK-NEXT:    # Child Loop BB0_20 Depth 2
+; CHECK-NEXT:    # Child Loop BB0_21 Depth 3
+; CHECK-NEXT:    # Child Loop BB0_23 Depth 2
+; CHECK-NEXT:    # Child Loop BB0_24 Depth 3
+; CHECK-NEXT:    # Child Loop BB0_26 Depth 2
+; CHECK-NEXT:    # Child Loop BB0_38 Depth 3
+; CHECK-NEXT:    # Child Loop BB0_29 Depth 3
+; CHECK-NEXT:    # Child Loop BB0_30 Depth 2
+; CHECK-NEXT:    # Child Loop BB0_39 Depth 3
+; CHECK-NEXT:    # Child Loop BB0_33 Depth 3
+; CHECK-NEXT:    # Child Loop BB0_34 Depth 2
+; CHECK-NEXT:    # Child Loop BB0_36 Depth 2
+; CHECK-NEXT:    testl %r8d, %r8d
+; CHECK-NEXT:    js .LBB0_4
+; CHECK-NEXT:  # %bb.17: # %b179
+; CHECK-NEXT:    # in Loop: Header=BB0_3 Depth=1
+; CHECK-NEXT:    testq %rsi, %rsi
+; CHECK-NEXT:    js .LBB0_18
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB0_37: # %a30b
+; CHECK-NEXT:    # Parent Loop BB0_3 Depth=1
+; CHECK-NEXT:    # => This Inner Loop Header: Depth=2
+; CHECK-NEXT:    testb %dil, %dil
+; CHECK-NEXT:    je .LBB0_37
+; CHECK-NEXT:  .LBB0_18: # %b188
+; CHECK-NEXT:    # in Loop: Header=BB0_3 Depth=1
+; CHECK-NEXT:    testb %r10b, %r10b
+; CHECK-NEXT:    jne .LBB0_4
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB0_19: # %a30b294
+; CHECK-NEXT:    # Parent Loop BB0_3 Depth=1
+; CHECK-NEXT:    # => This Inner Loop Header: Depth=2
+; CHECK-NEXT:    testb %dil, %dil
+; CHECK-NEXT:    je .LBB0_19
+; CHECK-NEXT:  .LBB0_4: # %a33b
+; CHECK-NEXT:    # in Loop: Header=BB0_3 Depth=1
+; CHECK-NEXT:    movl %ecx, %eax
+; CHECK-NEXT:    orl %r8d, %eax
+; CHECK-NEXT:    movl %eax, %r9d
+; CHECK-NEXT:    shrl $31, %r9d
+; CHECK-NEXT:    testl %eax, %eax
+; CHECK-NEXT:    jns .LBB0_20
+; CHECK-NEXT:  .LBB0_5: # %a50b
+; CHECK-NEXT:    # in Loop: Header=BB0_3 Depth=1
+; CHECK-NEXT:    movl %r8d, %eax
+; CHECK-NEXT:    orl %ecx, %eax
+; CHECK-NEXT:    movl %eax, %r11d
+; CHECK-NEXT:    shrl $31, %r11d
+; CHECK-NEXT:    testl %eax, %eax
+; CHECK-NEXT:    jns .LBB0_26
+; CHECK-NEXT:  .LBB0_6: # %a57b
+; CHECK-NEXT:    # in Loop: Header=BB0_3 Depth=1
+; CHECK-NEXT:    testb %r9b, %r9b
+; CHECK-NEXT:    je .LBB0_30
+; CHECK-NEXT:  .LBB0_7: # %a66b
+; CHECK-NEXT:    # in Loop: Header=BB0_3 Depth=1
+; CHECK-NEXT:    testb %r11b, %r11b
+; CHECK-NEXT:    jne .LBB0_8
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB0_34: # %a74b
+; CHECK-NEXT:    # Parent Loop BB0_3 Depth=1
+; CHECK-NEXT:    # => This Inner Loop Header: Depth=2
+; CHECK-NEXT:    testb %dil, %dil
+; CHECK-NEXT:    jne .LBB0_34
+; CHECK-NEXT:  # %bb.35: # %b1582
+; CHECK-NEXT:    # in Loop: Header=BB0_34 Depth=2
+; CHECK-NEXT:    testb %dil, %dil
+; CHECK-NEXT:    jne .LBB0_34
+; CHECK-NEXT:  .LBB0_8: # %a93b
+; CHECK-NEXT:    # in Loop: Header=BB0_3 Depth=1
+; CHECK-NEXT:    testl %r8d, %r8d
+; CHECK-NEXT:    js .LBB0_9
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB0_36: # %a97b
+; CHECK-NEXT:    # Parent Loop BB0_3 Depth=1
+; CHECK-NEXT:    # => This Inner Loop Header: Depth=2
+; CHECK-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; CHECK-NEXT:    addss %xmm0, %xmm1
+; CHECK-NEXT:    addss %xmm0, %xmm1
+; CHECK-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; CHECK-NEXT:    addss %xmm0, %xmm2
+; CHECK-NEXT:    addss %xmm1, %xmm2
+; CHECK-NEXT:    movss %xmm2, {{.*}}(%rip)
+; CHECK-NEXT:    testb %dil, %dil
+; CHECK-NEXT:    jne .LBB0_36
+; CHECK-NEXT:    jmp .LBB0_9
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB0_22: # %b463
+; CHECK-NEXT:    # in Loop: Header=BB0_20 Depth=2
+; CHECK-NEXT:    testb %dil, %dil
+; CHECK-NEXT:    je .LBB0_23
+; CHECK-NEXT:  .LBB0_20: # %b341
+; CHECK-NEXT:    # Parent Loop BB0_3 Depth=1
+; CHECK-NEXT:    # => This Loop Header: Depth=2
+; CHECK-NEXT:    # Child Loop BB0_21 Depth 3
+; CHECK-NEXT:    testq %rcx, %rcx
+; CHECK-NEXT:    js .LBB0_22
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB0_21: # %a35b
+; CHECK-NEXT:    # Parent Loop BB0_3 Depth=1
+; CHECK-NEXT:    # Parent Loop BB0_20 Depth=2
+; CHECK-NEXT:    # => This Inner Loop Header: Depth=3
+; CHECK-NEXT:    testb %dil, %dil
+; CHECK-NEXT:    je .LBB0_21
+; CHECK-NEXT:    jmp .LBB0_22
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB0_28: # %b1016
+; CHECK-NEXT:    # in Loop: Header=BB0_26 Depth=2
+; CHECK-NEXT:    testq %rcx, %rcx
+; CHECK-NEXT:    jle .LBB0_6
+; CHECK-NEXT:  .LBB0_26: # %b858
+; CHECK-NEXT:    # Parent Loop BB0_3 Depth=1
+; CHECK-NEXT:    # => This Loop Header: Depth=2
+; CHECK-NEXT:    # Child Loop BB0_38 Depth 3
+; CHECK-NEXT:    # Child Loop BB0_29 Depth 3
+; CHECK-NEXT:    testq %rsi, %rsi
+; CHECK-NEXT:    js .LBB0_27
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB0_38: # %a53b
+; CHECK-NEXT:    # Parent Loop BB0_3 Depth=1
+; CHECK-NEXT:    # Parent Loop BB0_26 Depth=2
+; CHECK-NEXT:    # => This Inner Loop Header: Depth=3
+; CHECK-NEXT:    testb %dil, %dil
+; CHECK-NEXT:    je .LBB0_38
+; CHECK-NEXT:  .LBB0_27: # %b879
+; CHECK-NEXT:    # in Loop: Header=BB0_26 Depth=2
+; CHECK-NEXT:    testb %r10b, %r10b
+; CHECK-NEXT:    jne .LBB0_28
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB0_29: # %a53b1019
+; CHECK-NEXT:    # Parent Loop BB0_3 Depth=1
+; CHECK-NEXT:    # Parent Loop BB0_26 Depth=2
+; CHECK-NEXT:    # => This Inner Loop Header: Depth=3
+; CHECK-NEXT:    testq %rsi, %rsi
+; CHECK-NEXT:    jle .LBB0_29
+; CHECK-NEXT:    jmp .LBB0_28
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB0_32: # %b1263
+; CHECK-NEXT:    # in Loop: Header=BB0_30 Depth=2
+; CHECK-NEXT:    testq %rsi, %rsi
+; CHECK-NEXT:    jle .LBB0_7
+; CHECK-NEXT:  .LBB0_30: # %b1117
+; CHECK-NEXT:    # Parent Loop BB0_3 Depth=1
+; CHECK-NEXT:    # => This Loop Header: Depth=2
+; CHECK-NEXT:    # Child Loop BB0_39 Depth 3
+; CHECK-NEXT:    # Child Loop BB0_33 Depth 3
+; CHECK-NEXT:    testq %rcx, %rcx
+; CHECK-NEXT:    js .LBB0_31
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB0_39: # %a63b
+; CHECK-NEXT:    # Parent Loop BB0_3 Depth=1
+; CHECK-NEXT:    # Parent Loop BB0_30 Depth=2
+; CHECK-NEXT:    # => This Inner Loop Header: Depth=3
+; CHECK-NEXT:    testq %rcx, %rcx
+; CHECK-NEXT:    jle .LBB0_39
+; CHECK-NEXT:  .LBB0_31: # %b1139
+; CHECK-NEXT:    # in Loop: Header=BB0_30 Depth=2
+; CHECK-NEXT:    testq %rcx, %rcx
+; CHECK-NEXT:    jle .LBB0_32
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB0_33: # %a63b1266
+; CHECK-NEXT:    # Parent Loop BB0_3 Depth=1
+; CHECK-NEXT:    # Parent Loop BB0_30 Depth=2
+; CHECK-NEXT:    # => This Inner Loop Header: Depth=3
+; CHECK-NEXT:    testq %rcx, %rcx
+; CHECK-NEXT:    jle .LBB0_33
+; CHECK-NEXT:    jmp .LBB0_32
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB0_25: # %b712
+; CHECK-NEXT:    # in Loop: Header=BB0_23 Depth=2
+; CHECK-NEXT:    testb %dil, %dil
+; CHECK-NEXT:    je .LBB0_5
+; CHECK-NEXT:  .LBB0_23: # %b535
+; CHECK-NEXT:    # Parent Loop BB0_3 Depth=1
+; CHECK-NEXT:    # => This Loop Header: Depth=2
+; CHECK-NEXT:    # Child Loop BB0_24 Depth 3
+; CHECK-NEXT:    testq %rsi, %rsi
+; CHECK-NEXT:    js .LBB0_25
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB0_24: # %a45b
+; CHECK-NEXT:    # Parent Loop BB0_3 Depth=1
+; CHECK-NEXT:    # Parent Loop BB0_23 Depth=2
+; CHECK-NEXT:    # => This Inner Loop Header: Depth=3
+; CHECK-NEXT:    testb %dil, %dil
+; CHECK-NEXT:    je .LBB0_24
+; CHECK-NEXT:    jmp .LBB0_25
+; CHECK-NEXT:  .LBB0_10: # %a109b
+; CHECK-NEXT:    movq %rbp, %rsp
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa %rsp, 8
+; CHECK-NEXT:    retq
 b:
 	%r = load i32, i32* %a0
 	%r2 = load i32, i32* %a1
diff --git a/llvm/test/CodeGen/X86/2007-02-16-BranchFold.ll b/llvm/test/CodeGen/X86/2007-02-16-BranchFold.ll
index 22e0a4e80..010b9f4be 100644
--- a/llvm/test/CodeGen/X86/2007-02-16-BranchFold.ll
+++ b/llvm/test/CodeGen/X86/2007-02-16-BranchFold.ll
@@ -1,6 +1,6 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -enable-tail-merge=0 | FileCheck %s
 ; PR 1200
-; RUN: llc < %s -enable-tail-merge=0 | not grep jmp 
-
 ; ModuleID = '<stdin>'
 target datalayout = "e-p:32:32"
 target triple = "i686-apple-darwin8"
@@ -27,6 +27,81 @@ target triple = "i686-apple-darwin8"
 declare i32 @fprintf(%struct.FILE*, i8*, ...)
 
 define i16 @main_bb_2E_i9_2E_i_2E_i932_2E_ce(%struct.list* %l_addr.01.0.i2.i.i929, %struct.operator** %tmp66.i62.i.out) {
+; CHECK-LABEL: main_bb_2E_i9_2E_i_2E_i932_2E_ce:
+; CHECK:       ## %bb.0: ## %newFuncRoot
+; CHECK-NEXT:    pushl %edi
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    pushl %esi
+; CHECK-NEXT:    .cfi_def_cfa_offset 12
+; CHECK-NEXT:    subl $20, %esp
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    .cfi_offset %esi, -12
+; CHECK-NEXT:    .cfi_offset %edi, -8
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    movl (%eax), %edi
+; CHECK-NEXT:    movl 8(%edi), %eax
+; CHECK-NEXT:    movl L_outfile$non_lazy_ptr, %ecx
+; CHECK-NEXT:    movl (%ecx), %ecx
+; CHECK-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movl L_str1$non_lazy_ptr, %eax
+; CHECK-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movl %ecx, (%esp)
+; CHECK-NEXT:    calll _fprintf
+; CHECK-NEXT:    movl 20(%edi), %eax
+; CHECK-NEXT:    testl %eax, %eax
+; CHECK-NEXT:    jle LBB0_6
+; CHECK-NEXT:  ## %bb.1: ## %NodeBlock4
+; CHECK-NEXT:    cmpl $2, %eax
+; CHECK-NEXT:    jge LBB0_2
+; CHECK-NEXT:  ## %bb.4: ## %LeafBlock2
+; CHECK-NEXT:    cmpl $1, %eax
+; CHECK-NEXT:    jne LBB0_3
+; CHECK-NEXT:  ## %bb.5: ## %bb20.i.i937.exitStub
+; CHECK-NEXT:    movl %edi, (%esi)
+; CHECK-NEXT:    movw $3, %ax
+; CHECK-NEXT:    addl $20, %esp
+; CHECK-NEXT:    popl %esi
+; CHECK-NEXT:    popl %edi
+; CHECK-NEXT:    retl
+; CHECK-NEXT:  LBB0_6: ## %NodeBlock
+; CHECK-NEXT:    js LBB0_9
+; CHECK-NEXT:  ## %bb.7: ## %LeafBlock1
+; CHECK-NEXT:    testl %eax, %eax
+; CHECK-NEXT:    jne LBB0_3
+; CHECK-NEXT:  ## %bb.8: ## %bb12.i.i935.exitStub
+; CHECK-NEXT:    movl %edi, (%esi)
+; CHECK-NEXT:    movw $2, %ax
+; CHECK-NEXT:    addl $20, %esp
+; CHECK-NEXT:    popl %esi
+; CHECK-NEXT:    popl %edi
+; CHECK-NEXT:    retl
+; CHECK-NEXT:  LBB0_2: ## %LeafBlock3
+; CHECK-NEXT:    jne LBB0_3
+; CHECK-NEXT:  ## %bb.11: ## %bb28.i.i938.exitStub
+; CHECK-NEXT:    movl %edi, (%esi)
+; CHECK-NEXT:    movw $4, %ax
+; CHECK-NEXT:    addl $20, %esp
+; CHECK-NEXT:    popl %esi
+; CHECK-NEXT:    popl %edi
+; CHECK-NEXT:    retl
+; CHECK-NEXT:  LBB0_9: ## %LeafBlock
+; CHECK-NEXT:    cmpl $-1, %eax
+; CHECK-NEXT:    je LBB0_10
+; CHECK-NEXT:  LBB0_3: ## %NewDefault
+; CHECK-NEXT:    movl %edi, (%esi)
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    addl $20, %esp
+; CHECK-NEXT:    popl %esi
+; CHECK-NEXT:    popl %edi
+; CHECK-NEXT:    retl
+; CHECK-NEXT:  LBB0_10: ## %bb.i14.i.exitStub
+; CHECK-NEXT:    movl %edi, (%esi)
+; CHECK-NEXT:    movw $1, %ax
+; CHECK-NEXT:    addl $20, %esp
+; CHECK-NEXT:    popl %esi
+; CHECK-NEXT:    popl %edi
+; CHECK-NEXT:    retl
 newFuncRoot:
 	br label %bb.i9.i.i932.ce
 
diff --git a/llvm/test/CodeGen/X86/2007-03-01-SpillerCrash.ll b/llvm/test/CodeGen/X86/2007-03-01-SpillerCrash.ll
index dbbb611dc..9f48313a5 100644
--- a/llvm/test/CodeGen/X86/2007-03-01-SpillerCrash.ll
+++ b/llvm/test/CodeGen/X86/2007-03-01-SpillerCrash.ll
@@ -1,7 +1,12 @@
-; RUN: llc < %s -mtriple=x86_64-apple-darwin8 -mattr=+sse2
-; RUN: llc < %s -mtriple=x86_64-apple-darwin8 -mattr=+sse2 | not grep movhlps
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-apple-darwin8 -mattr=+sse2 | FileCheck %s
 
 define void @test() nounwind {
+; CHECK-LABEL: test:
+; CHECK:       ## %bb.0: ## %test.exit
+; CHECK-NEXT:    movb $1, %al
+; CHECK-NEXT:    testb %al, %al
+; CHECK-NEXT:    ud2
 test.exit:
 	fmul <4 x float> zeroinitializer, zeroinitializer		; <<4 x float>>:0 [#uses=4]
 	load <4 x float>, <4 x float>* null		; <<4 x float>>:1 [#uses=1]
diff --git a/llvm/test/CodeGen/X86/2007-03-24-InlineAsmPModifier.ll b/llvm/test/CodeGen/X86/2007-03-24-InlineAsmPModifier.ll
index da9a81c07..806b3924b 100644
--- a/llvm/test/CodeGen/X86/2007-03-24-InlineAsmPModifier.ll
+++ b/llvm/test/CodeGen/X86/2007-03-24-InlineAsmPModifier.ll
@@ -1,8 +1,15 @@
-; RUN: llc < %s -no-integrated-as | grep "mov %gs:72, %eax"
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -no-integrated-as | FileCheck %s
 target datalayout = "e-p:32:32"
 target triple = "i686-apple-darwin9"
 
 define void @test() {
+; CHECK-LABEL: test:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    ## InlineAsm Start
+; CHECK-NEXT:    mov %gs:72, %eax
+; CHECK-NEXT:    ## InlineAsm End
+; CHECK-NEXT:    retl
 	%tmp1 = tail call i32* asm sideeffect "mov %gs:${1:P}, $0", "=r,i,~{dirflag},~{fpsr},~{flags}"( i32 72 )		; <%struct._pthread*> [#uses=1]
 	ret void
 }
diff --git a/llvm/test/CodeGen/X86/2007-04-27-InlineAsm-IntMemInput.ll b/llvm/test/CodeGen/X86/2007-04-27-InlineAsm-IntMemInput.ll
index cae68c9f3..edd2f67fd 100644
--- a/llvm/test/CodeGen/X86/2007-04-27-InlineAsm-IntMemInput.ll
+++ b/llvm/test/CodeGen/X86/2007-04-27-InlineAsm-IntMemInput.ll
@@ -1,10 +1,17 @@
-; RUN: llc < %s | not grep "bsrl.*10"
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s | FileCheck %s
 ; PR1356
 
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64"
 target triple = "i686-apple-darwin8"
 
 define i32 @main() {
+; CHECK-LABEL: main:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    ## InlineAsm Start
+; CHECK-NEXT:    bsrl LCPI0_0, %eax
+; CHECK-NEXT:    ## InlineAsm End
+; CHECK-NEXT:    retl
 entry:
         %tmp4 = tail call i32 asm "bsrl  $1, $0", "=r,ro,~{dirflag},~{fpsr},~{flags},~{cc}"( i32 10 )           ; <i32> [#uses=1]
         ret i32 %tmp4
diff --git a/llvm/test/CodeGen/X86/2007-08-09-IllegalX86-64Asm.ll b/llvm/test/CodeGen/X86/2007-08-09-IllegalX86-64Asm.ll
index 3bf8225ab..645c6478e 100644
--- a/llvm/test/CodeGen/X86/2007-08-09-IllegalX86-64Asm.ll
+++ b/llvm/test/CodeGen/X86/2007-08-09-IllegalX86-64Asm.ll
@@ -1,4 +1,5 @@
-; RUN: llc < %s -mtriple=x86_64-apple-darwin | not grep "movb   %ah, %r"
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-apple-darwin | FileCheck %s
 
 	%struct.FILE = type { i8*, i32, i32, i16, i16, %struct.__sbuf, i32, i8*, i32 (i8*)*, i32 (i8*, i8*, i32)*, i64 (i8*, i64, i32)*, i32 (i8*, i8*, i32)*, %struct.__sbuf, %struct.__sFILEX*, i32, [3 x i8], [1 x i8], %struct.__sbuf, i32, [4 x i8], i64 }
 	%struct.PyBoolScalarObject = type { i64, %struct._typeobject*, i8 }
@@ -19,6 +20,175 @@
 @.str5 = external constant [14 x i8]		; <[14 x i8]*> [#uses=1]
 
 define %struct.PyObject* @ubyte_divmod(%struct.PyObject* %a, %struct.PyObject* %b) {
+; CHECK-LABEL: ubyte_divmod:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    subq $40, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 80
+; CHECK-NEXT:    .cfi_offset %rbx, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movq %rsi, %r14
+; CHECK-NEXT:    movq %rdi, %rbx
+; CHECK-NEXT:    leaq {{[0-9]+}}(%rsp), %rsi
+; CHECK-NEXT:    callq __ubyte_convert_to_ctype
+; CHECK-NEXT:    testl %eax, %eax
+; CHECK-NEXT:    js LBB0_4
+; CHECK-NEXT:  ## %bb.1: ## %cond_next.i
+; CHECK-NEXT:    leaq {{[0-9]+}}(%rsp), %rsi
+; CHECK-NEXT:    movq %r14, %rdi
+; CHECK-NEXT:    callq __ubyte_convert_to_ctype
+; CHECK-NEXT:    movl %eax, %ecx
+; CHECK-NEXT:    sarl $31, %ecx
+; CHECK-NEXT:    andl %eax, %ecx
+; CHECK-NEXT:    cmpl $-2, %ecx
+; CHECK-NEXT:    je LBB0_8
+; CHECK-NEXT:  ## %bb.2: ## %cond_next.i
+; CHECK-NEXT:    cmpl $-1, %ecx
+; CHECK-NEXT:    jne LBB0_6
+; CHECK-NEXT:  LBB0_3: ## %bb4
+; CHECK-NEXT:    movq _PyArray_API@{{.*}}(%rip), %rax
+; CHECK-NEXT:    movq (%rax), %rax
+; CHECK-NEXT:    movq 16(%rax), %rax
+; CHECK-NEXT:    jmp LBB0_10
+; CHECK-NEXT:  LBB0_4: ## %_ubyte_convert2_to_ctypes.exit
+; CHECK-NEXT:    cmpl $-2, %eax
+; CHECK-NEXT:    je LBB0_8
+; CHECK-NEXT:  ## %bb.5: ## %_ubyte_convert2_to_ctypes.exit
+; CHECK-NEXT:    cmpl $-1, %eax
+; CHECK-NEXT:    je LBB0_3
+; CHECK-NEXT:  LBB0_6: ## %bb35
+; CHECK-NEXT:    movq _PyUFunc_API@{{.*}}(%rip), %rbp
+; CHECK-NEXT:    movq (%rbp), %rax
+; CHECK-NEXT:    callq *216(%rax)
+; CHECK-NEXT:    movb {{[0-9]+}}(%rsp), %dl
+; CHECK-NEXT:    testb %dl, %dl
+; CHECK-NEXT:    je LBB0_11
+; CHECK-NEXT:  ## %bb.7: ## %cond_false.i
+; CHECK-NEXT:    movb {{[0-9]+}}(%rsp), %bl
+; CHECK-NEXT:    movzbl %bl, %ecx
+; CHECK-NEXT:    movl %ecx, %eax
+; CHECK-NEXT:    divb %dl
+; CHECK-NEXT:    movl %eax, %r14d
+; CHECK-NEXT:    testb %cl, %cl
+; CHECK-NEXT:    jne LBB0_12
+; CHECK-NEXT:    jmp LBB0_14
+; CHECK-NEXT:  LBB0_8: ## %bb17
+; CHECK-NEXT:    callq _PyErr_Occurred
+; CHECK-NEXT:    testq %rax, %rax
+; CHECK-NEXT:    jne LBB0_27
+; CHECK-NEXT:  ## %bb.9: ## %cond_next
+; CHECK-NEXT:    movq _PyArray_API@{{.*}}(%rip), %rax
+; CHECK-NEXT:    movq (%rax), %rax
+; CHECK-NEXT:    movq 80(%rax), %rax
+; CHECK-NEXT:  LBB0_10: ## %bb4
+; CHECK-NEXT:    movq 96(%rax), %rax
+; CHECK-NEXT:    movq %rbx, %rdi
+; CHECK-NEXT:    movq %r14, %rsi
+; CHECK-NEXT:    callq *40(%rax)
+; CHECK-NEXT:    jmp LBB0_28
+; CHECK-NEXT:  LBB0_11: ## %cond_true.i
+; CHECK-NEXT:    movl $4, %edi
+; CHECK-NEXT:    callq _feraiseexcept
+; CHECK-NEXT:    movb {{[0-9]+}}(%rsp), %dl
+; CHECK-NEXT:    movb {{[0-9]+}}(%rsp), %bl
+; CHECK-NEXT:    xorl %r14d, %r14d
+; CHECK-NEXT:    testb %bl, %bl
+; CHECK-NEXT:    je LBB0_14
+; CHECK-NEXT:  LBB0_12: ## %cond_false.i
+; CHECK-NEXT:    testb %dl, %dl
+; CHECK-NEXT:    je LBB0_14
+; CHECK-NEXT:  ## %bb.13: ## %cond_next17.i
+; CHECK-NEXT:    movzbl %bl, %eax
+; CHECK-NEXT:    divb %dl
+; CHECK-NEXT:    movzbl %ah, %eax
+; CHECK-NEXT:    movl %eax, %r15d
+; CHECK-NEXT:    jmp LBB0_18
+; CHECK-NEXT:  LBB0_14: ## %cond_true.i200
+; CHECK-NEXT:    testb %dl, %dl
+; CHECK-NEXT:    jne LBB0_17
+; CHECK-NEXT:  ## %bb.16: ## %cond_true14.i
+; CHECK-NEXT:    movl $4, %edi
+; CHECK-NEXT:    callq _feraiseexcept
+; CHECK-NEXT:  LBB0_17: ## %ubyte_ctype_remainder.exit
+; CHECK-NEXT:    xorl %r15d, %r15d
+; CHECK-NEXT:  LBB0_18: ## %ubyte_ctype_remainder.exit
+; CHECK-NEXT:    movq (%rbp), %rax
+; CHECK-NEXT:    callq *224(%rax)
+; CHECK-NEXT:    testl %eax, %eax
+; CHECK-NEXT:    je LBB0_21
+; CHECK-NEXT:  ## %bb.19: ## %cond_true61
+; CHECK-NEXT:    movl %eax, %ebx
+; CHECK-NEXT:    movq (%rbp), %rax
+; CHECK-NEXT:    movq _.str5@{{.*}}(%rip), %rdi
+; CHECK-NEXT:    leaq {{[0-9]+}}(%rsp), %rsi
+; CHECK-NEXT:    leaq {{[0-9]+}}(%rsp), %rdx
+; CHECK-NEXT:    leaq {{[0-9]+}}(%rsp), %rcx
+; CHECK-NEXT:    callq *200(%rax)
+; CHECK-NEXT:    testl %eax, %eax
+; CHECK-NEXT:    js LBB0_27
+; CHECK-NEXT:  ## %bb.20: ## %cond_next73
+; CHECK-NEXT:    movl $1, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movq (%rbp), %rax
+; CHECK-NEXT:    movq {{[0-9]+}}(%rsp), %rsi
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %edi
+; CHECK-NEXT:    leaq {{[0-9]+}}(%rsp), %rcx
+; CHECK-NEXT:    movl %ebx, %edx
+; CHECK-NEXT:    callq *232(%rax)
+; CHECK-NEXT:    testl %eax, %eax
+; CHECK-NEXT:    jne LBB0_27
+; CHECK-NEXT:  LBB0_21: ## %cond_next89
+; CHECK-NEXT:    movl $2, %edi
+; CHECK-NEXT:    callq _PyTuple_New
+; CHECK-NEXT:    testq %rax, %rax
+; CHECK-NEXT:    je LBB0_27
+; CHECK-NEXT:  ## %bb.22: ## %cond_next97
+; CHECK-NEXT:    movq %rax, %rbx
+; CHECK-NEXT:    movq _PyArray_API@{{.*}}(%rip), %rbp
+; CHECK-NEXT:    movq (%rbp), %rax
+; CHECK-NEXT:    movq 200(%rax), %rdi
+; CHECK-NEXT:    xorl %esi, %esi
+; CHECK-NEXT:    callq *304(%rdi)
+; CHECK-NEXT:    testq %rax, %rax
+; CHECK-NEXT:    je LBB0_25
+; CHECK-NEXT:  ## %bb.23: ## %cond_next135
+; CHECK-NEXT:    movb %r14b, 16(%rax)
+; CHECK-NEXT:    movq %rax, 24(%rbx)
+; CHECK-NEXT:    movq (%rbp), %rax
+; CHECK-NEXT:    movq 200(%rax), %rdi
+; CHECK-NEXT:    xorl %esi, %esi
+; CHECK-NEXT:    callq *304(%rdi)
+; CHECK-NEXT:    testq %rax, %rax
+; CHECK-NEXT:    je LBB0_25
+; CHECK-NEXT:  ## %bb.24: ## %cond_next182
+; CHECK-NEXT:    movb %r15b, 16(%rax)
+; CHECK-NEXT:    movq %rax, 32(%rbx)
+; CHECK-NEXT:    movq %rbx, %rax
+; CHECK-NEXT:    jmp LBB0_28
+; CHECK-NEXT:  LBB0_25: ## %cond_true113
+; CHECK-NEXT:    decq (%rbx)
+; CHECK-NEXT:    jne LBB0_27
+; CHECK-NEXT:  ## %bb.26: ## %cond_true126
+; CHECK-NEXT:    movq 8(%rbx), %rax
+; CHECK-NEXT:    movq %rbx, %rdi
+; CHECK-NEXT:    callq *48(%rax)
+; CHECK-NEXT:  LBB0_27: ## %UnifiedReturnBlock
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:  LBB0_28: ## %UnifiedReturnBlock
+; CHECK-NEXT:    addq $40, %rsp
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    retq
 entry:
 	%arg1 = alloca i8, align 1		; <i8*> [#uses=3]
 	%arg2 = alloca i8, align 1		; <i8*> [#uses=3]
diff --git a/llvm/test/CodeGen/X86/2007-08-10-SignExtSubreg.ll b/llvm/test/CodeGen/X86/2007-08-10-SignExtSubreg.ll
index a798839fd..749172083 100644
--- a/llvm/test/CodeGen/X86/2007-08-10-SignExtSubreg.ll
+++ b/llvm/test/CodeGen/X86/2007-08-10-SignExtSubreg.ll
@@ -1,8 +1,15 @@
-; RUN: llc < %s -mtriple=i686-- | grep "movsbl"
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- | FileCheck %s
 
 @X = global i32 0               ; <i32*> [#uses=1]
 
 define i32 @_Z3fooi(i32 %x)   {
+; CHECK-LABEL: _Z3fooi:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    movl %eax, X
+; CHECK-NEXT:    movsbl %al, %eax
+; CHECK-NEXT:    retl
 entry:
         store i32 %x, i32* @X, align 4
         %retval67 = trunc i32 %x to i8          ; <i8> [#uses=1]
diff --git a/llvm/test/CodeGen/X86/2007-10-04-AvoidEFLAGSCopy.ll b/llvm/test/CodeGen/X86/2007-10-04-AvoidEFLAGSCopy.ll
index bcffac2cd..b2d801691 100644
--- a/llvm/test/CodeGen/X86/2007-10-04-AvoidEFLAGSCopy.ll
+++ b/llvm/test/CodeGen/X86/2007-10-04-AvoidEFLAGSCopy.ll
@@ -1,9 +1,26 @@
-; RUN: llc < %s -mtriple=i686-- | not grep pushf
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- | FileCheck %s
 
 	%struct.gl_texture_image = type { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i8*, i8* }
 	%struct.gl_texture_object = type { i32, i32, i32, float, [4 x i32], i32, i32, i32, i32, i32, float, [11 x %struct.gl_texture_image*], [1024 x i8], i32, i32, i32, i8, i8*, i8, void (%struct.gl_texture_object*, i32, float*, float*, float*, float*, i8*, i8*, i8*, i8*)*, %struct.gl_texture_object* }
 
 define fastcc void @sample_3d_linear(%struct.gl_texture_object* %tObj, %struct.gl_texture_image* %img, float %s, float %t, float %r, i8* %red, i8* %green, i8* %blue, i8* %alpha) {
+; CHECK-LABEL: sample_3d_linear:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushl %esi
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    .cfi_offset %esi, -8
+; CHECK-NEXT:    movl 0, %esi
+; CHECK-NEXT:    pushl $0
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 4
+; CHECK-NEXT:    calll floorf
+; CHECK-NEXT:    fstp %st(0)
+; CHECK-NEXT:    addl $4, %esp
+; CHECK-NEXT:    .cfi_adjust_cfa_offset -4
+; CHECK-NEXT:    cmpl $10497, %esi # imm = 0x2901
+; CHECK-NEXT:    popl %esi
+; CHECK-NEXT:    .cfi_def_cfa_offset 4
+; CHECK-NEXT:    retl
 entry:
 	%tmp15 = load i32, i32* null, align 4		; <i32> [#uses=1]
 	%tmp16 = icmp eq i32 %tmp15, 10497		; <i1> [#uses=1]
diff --git a/llvm/test/CodeGen/X86/2007-10-12-CoalesceExtSubReg.ll b/llvm/test/CodeGen/X86/2007-10-12-CoalesceExtSubReg.ll
index 6658536bd..51dbeea2b 100644
--- a/llvm/test/CodeGen/X86/2007-10-12-CoalesceExtSubReg.ll
+++ b/llvm/test/CodeGen/X86/2007-10-12-CoalesceExtSubReg.ll
@@ -1,6 +1,29 @@
-; RUN: llc < %s -mtriple=i686-- | not grep movb
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- | FileCheck %s
 
 define signext i16 @f(i32* %bp, i32* %ss)   {
+; CHECK-LABEL: f:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushl %esi
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    .cfi_offset %esi, -8
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB0_1: # %cond_next127
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movl (%eax), %edx
+; CHECK-NEXT:    movl (%ecx), %esi
+; CHECK-NEXT:    andl $15, %edx
+; CHECK-NEXT:    andl $15, %esi
+; CHECK-NEXT:    addl %esi, (%ecx)
+; CHECK-NEXT:    cmpl $63, %edx
+; CHECK-NEXT:    jb .LBB0_1
+; CHECK-NEXT:  # %bb.2: # %UnifiedReturnBlock
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    popl %esi
+; CHECK-NEXT:    .cfi_def_cfa_offset 4
+; CHECK-NEXT:    retl
 entry:
 	br label %cond_next127
 
diff --git a/llvm/test/CodeGen/X86/2007-10-12-SpillerUnfold1.ll b/llvm/test/CodeGen/X86/2007-10-12-SpillerUnfold1.ll
index 14cff04e1..2cd09606e 100644
--- a/llvm/test/CodeGen/X86/2007-10-12-SpillerUnfold1.ll
+++ b/llvm/test/CodeGen/X86/2007-10-12-SpillerUnfold1.ll
@@ -1,6 +1,36 @@
-; RUN: llc < %s -mtriple=i686-- -mattr=+sse2 | grep addss | not grep esp
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- -mattr=+sse2 | FileCheck %s
 
 define fastcc void @fht(float* %fz, i16 signext  %n) {
+; CHECK-LABEL: fht:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; CHECK-NEXT:    xorps %xmm0, %xmm0
+; CHECK-NEXT:    xorps %xmm2, %xmm2
+; CHECK-NEXT:    subss %xmm1, %xmm2
+; CHECK-NEXT:    movaps %xmm1, %xmm3
+; CHECK-NEXT:    mulss %xmm0, %xmm3
+; CHECK-NEXT:    addss %xmm1, %xmm3
+; CHECK-NEXT:    movaps %xmm1, %xmm4
+; CHECK-NEXT:    subss %xmm3, %xmm4
+; CHECK-NEXT:    addss %xmm1, %xmm3
+; CHECK-NEXT:    xorps %xmm5, %xmm5
+; CHECK-NEXT:    subss %xmm2, %xmm5
+; CHECK-NEXT:    addss %xmm0, %xmm2
+; CHECK-NEXT:    mulss %xmm0, %xmm3
+; CHECK-NEXT:    mulss %xmm0, %xmm5
+; CHECK-NEXT:    addss %xmm3, %xmm5
+; CHECK-NEXT:    addss %xmm0, %xmm5
+; CHECK-NEXT:    movss %xmm5, 0
+; CHECK-NEXT:    movss %xmm1, (%ecx)
+; CHECK-NEXT:    addss %xmm0, %xmm1
+; CHECK-NEXT:    movss %xmm1, 0
+; CHECK-NEXT:    mulss %xmm0, %xmm2
+; CHECK-NEXT:    mulss %xmm0, %xmm4
+; CHECK-NEXT:    addss %xmm2, %xmm4
+; CHECK-NEXT:    addss %xmm0, %xmm4
+; CHECK-NEXT:    movss %xmm4, (%ecx)
+; CHECK-NEXT:    retl
 entry:
 	br i1 true, label %bb171.preheader, label %bb431
 
diff --git a/llvm/test/CodeGen/X86/2007-10-12-SpillerUnfold2.ll b/llvm/test/CodeGen/X86/2007-10-12-SpillerUnfold2.ll
index 2601b4d4c..5805a136d 100644
--- a/llvm/test/CodeGen/X86/2007-10-12-SpillerUnfold2.ll
+++ b/llvm/test/CodeGen/X86/2007-10-12-SpillerUnfold2.ll
@@ -1,6 +1,22 @@
-; RUN: llc < %s -mtriple=i686-- -mcpu=corei7 | grep sarl | not grep esp
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- -mcpu=corei7 | FileCheck %s
 
 define signext   i16 @t(i16* %qmatrix, i16* %dct, i16* %acBaseTable, i16* %acExtTable, i16 signext  %acBaseRes, i16 signext  %acMaskRes, i16 signext  %acExtRes, i32* %bitptr, i32* %source, i32 %markerPrefix, i8** %byteptr, i32 %scale, i32 %round, i32 %bits) {
+; CHECK-LABEL: t:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB0_1: # %cond_next127
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movl %eax, %edx
+; CHECK-NEXT:    sarl %cl, %edx
+; CHECK-NEXT:    incl %edx
+; CHECK-NEXT:    cmpl $63, %edx
+; CHECK-NEXT:    jb .LBB0_1
+; CHECK-NEXT:  # %bb.2: # %UnifiedReturnBlock
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    retl
 entry:
 	br label %cond_next127
 
diff --git a/llvm/test/CodeGen/X86/2007-10-29-ExtendSetCC.ll b/llvm/test/CodeGen/X86/2007-10-29-ExtendSetCC.ll
index f338ff065..1feb09802 100644
--- a/llvm/test/CodeGen/X86/2007-10-29-ExtendSetCC.ll
+++ b/llvm/test/CodeGen/X86/2007-10-29-ExtendSetCC.ll
@@ -1,6 +1,13 @@
-; RUN: llc < %s -mtriple=i686-- | grep mov | count 1
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- | FileCheck %s
 
 define signext i16 @t()   {
+; CHECK-LABEL: t:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movswl 0, %eax
+; CHECK-NEXT:    testl %eax, %eax
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    retl
 entry:
 	%tmp180 = load i16, i16* null, align 2		; <i16> [#uses=3]
 	%tmp180181 = sext i16 %tmp180 to i32		; <i32> [#uses=1]
diff --git a/llvm/test/CodeGen/X86/2007-11-06-InstrSched.ll b/llvm/test/CodeGen/X86/2007-11-06-InstrSched.ll
index 3894029e5..91df29681 100644
--- a/llvm/test/CodeGen/X86/2007-11-06-InstrSched.ll
+++ b/llvm/test/CodeGen/X86/2007-11-06-InstrSched.ll
@@ -1,6 +1,38 @@
-; RUN: llc < %s -mtriple=i686-- -mcpu=generic -mattr=+sse2 | not grep lea
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- -mcpu=generic -mattr=+sse2 | FileCheck %s
 
 define float @foo(i32* %x, float* %y, i32 %c) nounwind {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushl %esi
+; CHECK-NEXT:    pushl %eax
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    testl %eax, %eax
+; CHECK-NEXT:    je .LBB0_1
+; CHECK-NEXT:  # %bb.2: # %bb18.preheader
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; CHECK-NEXT:    xorps %xmm0, %xmm0
+; CHECK-NEXT:    xorl %esi, %esi
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB0_3: # %bb18
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    xorps %xmm1, %xmm1
+; CHECK-NEXT:    cvtsi2ssl (%edx,%esi,4), %xmm1
+; CHECK-NEXT:    mulss (%ecx,%esi,4), %xmm1
+; CHECK-NEXT:    addss %xmm1, %xmm0
+; CHECK-NEXT:    incl %esi
+; CHECK-NEXT:    cmpl %eax, %esi
+; CHECK-NEXT:    jb .LBB0_3
+; CHECK-NEXT:    jmp .LBB0_4
+; CHECK-NEXT:  .LBB0_1:
+; CHECK-NEXT:    xorps %xmm0, %xmm0
+; CHECK-NEXT:  .LBB0_4: # %bb23
+; CHECK-NEXT:    movss %xmm0, (%esp)
+; CHECK-NEXT:    flds (%esp)
+; CHECK-NEXT:    addl $4, %esp
+; CHECK-NEXT:    popl %esi
+; CHECK-NEXT:    retl
 entry:
 	%tmp2132 = icmp eq i32 %c, 0		; <i1> [#uses=1]
 	br i1 %tmp2132, label %bb23, label %bb18
diff --git a/llvm/test/CodeGen/X86/2007-11-07-MulBy4.ll b/llvm/test/CodeGen/X86/2007-11-07-MulBy4.ll
index 9ecf0680f..aa6408d31 100644
--- a/llvm/test/CodeGen/X86/2007-11-07-MulBy4.ll
+++ b/llvm/test/CodeGen/X86/2007-11-07-MulBy4.ll
@@ -1,4 +1,5 @@
-; RUN: llc < %s -mtriple=i686-- | not grep imul
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- | FileCheck %s
 
 	%struct.eebb = type { %struct.eebb*, i16* }
 	%struct.hf = type { %struct.hf*, i16*, i8*, i32, i32, %struct.eebb*, i32, i32, i8*, i8*, i8*, i8*, i16*, i8*, i16*, %struct.ri, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, [30 x i32], %struct.eebb, i32, i8* }
@@ -6,6 +7,19 @@
 	%struct.ri = type { %struct.ri*, i32, i8*, i16*, i32*, i32 }
 
 define fastcc i32 @foo(i16* %eptr, i8* %ecode, %struct.foo_data* %md, i32 %ims) {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cmpl $0, 0
+; CHECK-NEXT:    je .LBB0_3
+; CHECK-NEXT:  # %bb.1: # %cond_next79
+; CHECK-NEXT:    cmpl $0, 0
+; CHECK-NEXT:    je .LBB0_3
+; CHECK-NEXT:  # %bb.2: # %cond_next130
+; CHECK-NEXT:    movb $1, %al
+; CHECK-NEXT:    testb %al, %al
+; CHECK-NEXT:  .LBB0_3: # %cond_true89
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    retl
 entry:
 	%tmp36 = load i32, i32* null, align 4		; <i32> [#uses=1]
 	%tmp37 = icmp ult i32 0, %tmp36		; <i1> [#uses=1]
diff --git a/llvm/test/CodeGen/X86/2007-12-18-LoadCSEBug.ll b/llvm/test/CodeGen/X86/2007-12-18-LoadCSEBug.ll
index 35aa016ec..d839ed6df 100644
--- a/llvm/test/CodeGen/X86/2007-12-18-LoadCSEBug.ll
+++ b/llvm/test/CodeGen/X86/2007-12-18-LoadCSEBug.ll
@@ -1,10 +1,34 @@
-; RUN: llc < %s -mtriple=i686-- -mcpu=generic | grep "(%esp)" | count 2
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- -mcpu=generic | FileCheck %s
 ; PR1872
 
 	%struct.c34007g__designated___XUB = type { i32, i32, i32, i32 }
 	%struct.c34007g__pkg__parent = type { i32*, %struct.c34007g__designated___XUB* }
 
 define void @_ada_c34007g() {
+; CHECK-LABEL: _ada_c34007g:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushl %ebp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    .cfi_offset %ebp, -8
+; CHECK-NEXT:    movl %esp, %ebp
+; CHECK-NEXT:    .cfi_def_cfa_register %ebp
+; CHECK-NEXT:    andl $-8, %esp
+; CHECK-NEXT:    subl $8, %esp
+; CHECK-NEXT:    movl (%esp), %eax
+; CHECK-NEXT:    testl %eax, %eax
+; CHECK-NEXT:    je .LBB0_3
+; CHECK-NEXT:  # %bb.1: # %entry
+; CHECK-NEXT:    orl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    jne .LBB0_3
+; CHECK-NEXT:  # %bb.2: # %entry
+; CHECK-NEXT:    movb $1, %al
+; CHECK-NEXT:    testb %al, %al
+; CHECK-NEXT:  .LBB0_3: # %bb5507
+; CHECK-NEXT:    movl %ebp, %esp
+; CHECK-NEXT:    popl %ebp
+; CHECK-NEXT:    .cfi_def_cfa %esp, 4
+; CHECK-NEXT:    retl
 entry:
 	%x8 = alloca %struct.c34007g__pkg__parent, align 8		; <%struct.c34007g__pkg__parent*> [#uses=2]
 	%tmp1272 = getelementptr %struct.c34007g__pkg__parent, %struct.c34007g__pkg__parent* %x8, i32 0, i32 0		; <i32**> [#uses=1]
diff --git a/llvm/test/CodeGen/X86/2008-02-22-LocalRegAllocBug.ll b/llvm/test/CodeGen/X86/2008-02-22-LocalRegAllocBug.ll
index 77ed1857e..c4572d3f6 100644
--- a/llvm/test/CodeGen/X86/2008-02-22-LocalRegAllocBug.ll
+++ b/llvm/test/CodeGen/X86/2008-02-22-LocalRegAllocBug.ll
@@ -1,8 +1,51 @@
-; RUN: llc < %s -regalloc=fast -optimize-regalloc=0 -mtriple=i686-- -mattr=+mmx | grep esi
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -regalloc=fast -optimize-regalloc=0 -mtriple=i686-- -mattr=+mmx | FileCheck %s
 ; PR2082
 ; Local register allocator was refusing to use ESI, EDI, and EBP so it ran out of
 ; registers.
 define void @transpose4x4(i8* %dst, i8* %src, i32 %dst_stride, i32 %src_stride) {
+; CHECK-LABEL: transpose4x4:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushl %ebx
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    pushl %edi
+; CHECK-NEXT:    .cfi_def_cfa_offset 12
+; CHECK-NEXT:    pushl %esi
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset %esi, -16
+; CHECK-NEXT:    .cfi_offset %edi, -12
+; CHECK-NEXT:    .cfi_offset %ebx, -8
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT:    leal (%ecx,%ecx,2), %edx
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; CHECK-NEXT:    leal (%edi,%edi,2), %ebx
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    movd (%esi), %mm0
+; CHECK-NEXT:    movd (%esi,%edi), %mm1
+; CHECK-NEXT:    movd (%esi,%edi,2), %mm2
+; CHECK-NEXT:    movd (%esi,%ebx), %mm3
+; CHECK-NEXT:    punpcklbw %mm1, %mm0 # mm0 = mm0[0],mm1[0],mm0[1],mm1[1],mm0[2],mm1[2],mm0[3],mm1[3]
+; CHECK-NEXT:    punpcklbw %mm3, %mm2 # mm2 = mm2[0],mm3[0],mm2[1],mm3[1],mm2[2],mm3[2],mm2[3],mm3[3]
+; CHECK-NEXT:    movq %mm0, %mm1
+; CHECK-NEXT:    punpcklwd %mm2, %mm0 # mm0 = mm0[0],mm2[0],mm0[1],mm2[1]
+; CHECK-NEXT:    punpckhwd %mm2, %mm1 # mm1 = mm1[2],mm2[2],mm1[3],mm2[3]
+; CHECK-NEXT:    movd %mm0, (%eax)
+; CHECK-NEXT:    punpckhdq %mm0, %mm0 # mm0 = mm0[1,1]
+; CHECK-NEXT:    movd %mm0, (%eax,%ecx)
+; CHECK-NEXT:    movd %mm1, (%eax,%ecx,2)
+; CHECK-NEXT:    punpckhdq %mm1, %mm1 # mm1 = mm1[1,1]
+; CHECK-NEXT:    movd %mm1, (%eax,%edx)
+; CHECK-EMPTY:
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    popl %esi
+; CHECK-NEXT:    .cfi_def_cfa_offset 12
+; CHECK-NEXT:    popl %edi
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    popl %ebx
+; CHECK-NEXT:    .cfi_def_cfa_offset 4
+; CHECK-NEXT:    retl
 entry:
 	%dst_addr = alloca i8*		; <i8**> [#uses=5]
 	%src_addr = alloca i8*		; <i8**> [#uses=5]
@@ -47,7 +90,7 @@ entry:
 	%tmp32 = load i8*, i8** %src_addr, align 4		; <i8*> [#uses=1]
 	%tmp33 = getelementptr i8, i8* %tmp32, i32 %tmp31		; <i8*> [#uses=1]
 	%tmp3334 = bitcast i8* %tmp33 to i32*		; <i32*> [#uses=1]
-	call void asm sideeffect "movd  $4, %mm0                \0A\09movd  $5, %mm1                \0A\09movd  $6, %mm2                \0A\09movd  $7, %mm3                \0A\09punpcklbw %mm1, %mm0         \0A\09punpcklbw %mm3, %mm2         \0A\09movq %mm0, %mm1              \0A\09punpcklwd %mm2, %mm0         \0A\09punpckhwd %mm2, %mm1         \0A\09movd  %mm0, $0                \0A\09punpckhdq %mm0, %mm0         \0A\09movd  %mm0, $1                \0A\09movd  %mm1, $2                \0A\09punpckhdq %mm1, %mm1         \0A\09movd  %mm1, $3                \0A\09", "=*m,=*m,=*m,=*m,*m,*m,*m,*m,~{dirflag},~{fpsr},~{flags}"( i32* %tmp12, i32* %tmp56, i32* %tmp1011, i32* %tmp1617, i32* %tmp1920, i32* %tmp2324, i32* %tmp2829, i32* %tmp3334 ) nounwind 
+	call void asm sideeffect "movd  $4, %mm0                \0A\09movd  $5, %mm1                \0A\09movd  $6, %mm2                \0A\09movd  $7, %mm3                \0A\09punpcklbw %mm1, %mm0         \0A\09punpcklbw %mm3, %mm2         \0A\09movq %mm0, %mm1              \0A\09punpcklwd %mm2, %mm0         \0A\09punpckhwd %mm2, %mm1         \0A\09movd  %mm0, $0                \0A\09punpckhdq %mm0, %mm0         \0A\09movd  %mm0, $1                \0A\09movd  %mm1, $2                \0A\09punpckhdq %mm1, %mm1         \0A\09movd  %mm1, $3                \0A\09", "=*m,=*m,=*m,=*m,*m,*m,*m,*m,~{dirflag},~{fpsr},~{flags}"( i32* %tmp12, i32* %tmp56, i32* %tmp1011, i32* %tmp1617, i32* %tmp1920, i32* %tmp2324, i32* %tmp2829, i32* %tmp3334 ) nounwind
 	br label %return
 
 return:		; preds = %entry
diff --git a/llvm/test/CodeGen/X86/2008-03-07-APIntBug.ll b/llvm/test/CodeGen/X86/2008-03-07-APIntBug.ll
index e17e963ec..0c25cb77c 100644
--- a/llvm/test/CodeGen/X86/2008-03-07-APIntBug.ll
+++ b/llvm/test/CodeGen/X86/2008-03-07-APIntBug.ll
@@ -1,4 +1,5 @@
-; RUN: llc < %s -mtriple=i686-- -mcpu=i386 | not grep 255
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- -mcpu=i386 | FileCheck %s
 
 	%struct.CONSTRAINT = type { i32, i32, i32, i32 }
 	%struct.FIRST_UNION = type { %struct.anon }
@@ -13,6 +14,16 @@
 	%struct.word_type = type { [2 x %struct.LIST], %struct.FIRST_UNION, %struct.SECOND_UNION, %struct.THIRD_UNION, [4 x i8] }
 
 define void @InsertSym_bb1163(%struct.rec** %s) {
+; CHECK-LABEL: InsertSym_bb1163:
+; CHECK:       # %bb.0: # %newFuncRoot
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    movl (%eax), %ecx
+; CHECK-NEXT:    movl 28(%ecx), %ecx
+; CHECK-NEXT:    orb $-128, 26(%ecx)
+; CHECK-NEXT:    orb $2, 25(%ecx)
+; CHECK-NEXT:    movl (%eax), %eax
+; CHECK-NEXT:    orb $1, 25(%eax)
+; CHECK-NEXT:    retl
 newFuncRoot:
 	br label %bb1163
 bb1233.exitStub:		; preds = %bb1163
diff --git a/llvm/test/CodeGen/X86/2008-03-31-SpillerFoldingBug.ll b/llvm/test/CodeGen/X86/2008-03-31-SpillerFoldingBug.ll
index fd0b3e751..9d3c4484f 100644
--- a/llvm/test/CodeGen/X86/2008-03-31-SpillerFoldingBug.ll
+++ b/llvm/test/CodeGen/X86/2008-03-31-SpillerFoldingBug.ll
@@ -1,4 +1,5 @@
-; RUN: llc < %s -mtriple=i386-apple-darwin -relocation-model=pic -frame-pointer=all | grep add | grep 12 | not grep non_lazy_ptr
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i386-apple-darwin -relocation-model=pic -frame-pointer=all | FileCheck %s
 ; Don't fold re-materialized load into a two address instruction
 
 	%"struct.Smarts::Runnable" = type { i32 (...)**, i32 }
@@ -14,10 +15,41 @@
 @_ZTVSt15basic_stringbufIcSt11char_traitsIcESaIcEE = external constant [16 x i32 (...)*]		; <[16 x i32 (...)*]*> [#uses=1]
 
 define void @_GLOBAL__I__ZN5Pooma5pinfoE() nounwind  {
+; CHECK-LABEL: _GLOBAL__I__ZN5Pooma5pinfoE:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    pushl %ebp
+; CHECK-NEXT:    movl %esp, %ebp
+; CHECK-NEXT:    pushl %ebx
+; CHECK-NEXT:    pushl %edi
+; CHECK-NEXT:    pushl %esi
+; CHECK-NEXT:    subl $12, %esp
+; CHECK-NEXT:    calll L0$pb
+; CHECK-NEXT:  L0$pb:
+; CHECK-NEXT:    popl %edi
+; CHECK-NEXT:    movl L__ZTVSt19basic_ostringstreamIcSt11char_traitsIcESaIcEE$non_lazy_ptr-L0$pb(%edi), %ebx
+; CHECK-NEXT:    leal 32(%ebx), %eax
+; CHECK-NEXT:    movl %eax, 0
+; CHECK-NEXT:    movl $180, (%esp)
+; CHECK-NEXT:    calll __Znwm
+; CHECK-NEXT:    movl %eax, %esi
+; CHECK-NEXT:    movl $0, (%esp)
+; CHECK-NEXT:    calll __ZNSt8ios_baseC2Ev
+; CHECK-NEXT:    movl $0, 0
+; CHECK-NEXT:    addl $12, %ebx
+; CHECK-NEXT:    movl %ebx, (%esi)
+; CHECK-NEXT:    movl L__ZTVSt15basic_streambufIcSt11char_traitsIcEE$non_lazy_ptr-L0$pb(%edi), %eax
+; CHECK-NEXT:    addl $8, %eax
+; CHECK-NEXT:    movl %eax, 0
+; CHECK-NEXT:    movl $0, (%esp)
+; CHECK-NEXT:    calll __ZNSt6localeC1Ev
+; CHECK-NEXT:    movl L__ZTVSt15basic_stringbufIcSt11char_traitsIcESaIcEE$non_lazy_ptr-L0$pb(%edi), %eax
+; CHECK-NEXT:    addl $8, %eax
+; CHECK-NEXT:    movl %eax, 0
+; CHECK-NEXT:    ud2
 entry:
 	store i32 (...)** getelementptr ([10 x i32 (...)*], [10 x i32 (...)*]* @_ZTVSt19basic_ostringstreamIcSt11char_traitsIcESaIcEE, i32 0, i32 8), i32 (...)*** null, align 4
 	%tmp96.i.i142.i = call i8* @_Znwm( i32 180 ) nounwind 		; <i8*> [#uses=2]
-	call void @_ZNSt8ios_baseC2Ev( %"struct.std::ios_base"* null ) nounwind 
+	call void @_ZNSt8ios_baseC2Ev( %"struct.std::ios_base"* null ) nounwind
 	store i32 (...)** getelementptr ([4 x i32 (...)*], [4 x i32 (...)*]* @_ZTVSt9basic_iosIcSt11char_traitsIcEE, i32 0, i32 2), i32 (...)*** null, align 4
 	store i32 (...)** null, i32 (...)*** null, align 4
 	%ctg2242.i.i163.i = getelementptr i8, i8* %tmp96.i.i142.i, i32 0		; <i8*> [#uses=1]
@@ -28,7 +60,7 @@ entry:
 	%tmp159.i.i167.i = bitcast i8* %tmp96.i.i142.i to i32 (...)***		; <i32 (...)***> [#uses=1]
 	store i32 (...)** getelementptr ([10 x i32 (...)*], [10 x i32 (...)*]* @_ZTVSt19basic_ostringstreamIcSt11char_traitsIcESaIcEE, i32 0, i32 3), i32 (...)*** %tmp159.i.i167.i, align 4
 	store i32 (...)** getelementptr ([16 x i32 (...)*], [16 x i32 (...)*]* @_ZTVSt15basic_streambufIcSt11char_traitsIcEE, i32 0, i32 2), i32 (...)*** null, align 4
-	call void @_ZNSt6localeC1Ev( %"struct.std::locale"* null ) nounwind 
+	call void @_ZNSt6localeC1Ev( %"struct.std::locale"* null ) nounwind
 	store i32 (...)** getelementptr ([16 x i32 (...)*], [16 x i32 (...)*]* @_ZTVSt15basic_stringbufIcSt11char_traitsIcESaIcEE, i32 0, i32 2), i32 (...)*** null, align 4
 	unreachable
 }
@@ -37,4 +69,4 @@ declare i8* @_Znwm(i32)
 
 declare void @_ZNSt8ios_baseC2Ev(%"struct.std::ios_base"*)
 
-declare void @_ZNSt6localeC1Ev(%"struct.std::locale"*) nounwind 
+declare void @_ZNSt6localeC1Ev(%"struct.std::locale"*) nounwind
diff --git a/llvm/test/CodeGen/X86/2008-04-09-BranchFolding.ll b/llvm/test/CodeGen/X86/2008-04-09-BranchFolding.ll
index 550507061..94fab3680 100644
--- a/llvm/test/CodeGen/X86/2008-04-09-BranchFolding.ll
+++ b/llvm/test/CodeGen/X86/2008-04-09-BranchFolding.ll
@@ -1,4 +1,5 @@
-; RUN: llc < %s -mtriple=i686-- | not grep jmp
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- | FileCheck %s
 
 	%struct..0anon = type { i32 }
 	%struct.binding_level = type { %struct.tree_node*, %struct.tree_node*, %struct.tree_node*, %struct.tree_node*, %struct.tree_node*, %struct.binding_level*, i8, i8, i8, i8, i8, i32, %struct.tree_node* }
@@ -8,6 +9,19 @@
 	%struct.tree_node = type { %struct.tree_decl }
 
 define fastcc %struct.tree_node* @pushdecl(%struct.tree_node* %x) nounwind  {
+; CHECK-LABEL: pushdecl:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movb $1, %al
+; CHECK-NEXT:    testb %al, %al
+; CHECK-NEXT:    jne .LBB0_1
+; CHECK-NEXT:  # %bb.2: # %bb17.i
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    retl
+; CHECK-NEXT:  .LBB0_1: # %bb160
+; CHECK-NEXT:    movb $1, %al
+; CHECK-NEXT:    testb %al, %al
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    retl
 entry:
 	%tmp3.i40 = icmp eq %struct.binding_level* null, null		; <i1> [#uses=2]
 	br label %bb140
@@ -39,10 +53,10 @@ bb226.i:		; preds = %bb73.i
 bb273.i:		; preds = %bb226.i
 	ret %struct.tree_node* null
 bb260:		; preds = %bb226.i
-	tail call void (i8*, i32, ...) @pedwarn_with_file_and_line( i8* %file.0, i32 %line.0, i8* null ) nounwind 
+	tail call void (i8*, i32, ...) @pedwarn_with_file_and_line( i8* %file.0, i32 %line.0, i8* null ) nounwind
 	ret %struct.tree_node* null
 bb344:		; preds = %bb174
 	ret %struct.tree_node* null
 }
 
-declare void @pedwarn_with_file_and_line(i8*, i32, ...) nounwind 
+declare void @pedwarn_with_file_and_line(i8*, i32, ...) nounwind
diff --git a/llvm/test/CodeGen/X86/2008-04-16-ReMatBug.ll b/llvm/test/CodeGen/X86/2008-04-16-ReMatBug.ll
index 9cae76f65..7be73f638 100644
--- a/llvm/test/CodeGen/X86/2008-04-16-ReMatBug.ll
+++ b/llvm/test/CodeGen/X86/2008-04-16-ReMatBug.ll
@@ -1,4 +1,5 @@
-; RUN: llc < %s -mtriple=i386-apple-darwin -disable-cgp-branch-opts | grep movw | not grep ", %e"
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i386-apple-darwin -disable-cgp-branch-opts | FileCheck %s
 
 	%struct.DBC_t = type { i32, i8*, i16, %struct.DBC_t*, i8*, i8*, i8*, i8*, i8*, %struct.DBC_t*, i32, i32, i32, i32, i8*, i8*, i8*, i8*, i8*, i32, i32, i32, i32, i32, i32, i32, i32, i16, i16, i32*, i8, i16, %struct.DRVOPT*, i16 }
 	%struct.DRVOPT = type { i16, i32, i8, %struct.DRVOPT* }
@@ -7,6 +8,89 @@
 @iodbcdm_global_lock = external global %struct.pthread_mutex_t		; <%struct.pthread_mutex_t*> [#uses=1]
 
 define i16 @SQLDriversW(i8* %henv, i16 zeroext  %fDir, i32* %szDrvDesc, i16 signext  %cbDrvDescMax, i16* %pcbDrvDesc, i32* %szDrvAttr, i16 signext  %cbDrvAttrMax, i16* %pcbDrvAttr) nounwind  {
+; CHECK-LABEL: SQLDriversW:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    pushl %ebp
+; CHECK-NEXT:    pushl %ebx
+; CHECK-NEXT:    pushl %edi
+; CHECK-NEXT:    pushl %esi
+; CHECK-NEXT:    subl $12, %esp
+; CHECK-NEXT:    movb $1, %al
+; CHECK-NEXT:    testb %al, %al
+; CHECK-NEXT:    je LBB0_1
+; CHECK-NEXT:  ## %bb.3: ## %bb28
+; CHECK-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT:    movzwl {{[0-9]+}}(%esp), %ebx
+; CHECK-NEXT:    movzwl {{[0-9]+}}(%esp), %ebp
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; CHECK-NEXT:    movw $-2, %si
+; CHECK-NEXT:    jne LBB0_6
+; CHECK-NEXT:  ## %bb.4: ## %bb37
+; CHECK-NEXT:    movw $0, 40(%edi)
+; CHECK-NEXT:    movb $1, %al
+; CHECK-NEXT:    testb %al, %al
+; CHECK-NEXT:    leal (,%ecx,4), %eax
+; CHECK-NEXT:    leal (,%ebx,4), %ecx
+; CHECK-NEXT:    subl $12, %esp
+; CHECK-NEXT:    movzwl %bp, %edx
+; CHECK-NEXT:    cwtl
+; CHECK-NEXT:    movswl %cx, %ecx
+; CHECK-NEXT:    pushl $87
+; CHECK-NEXT:    pushl {{[0-9]+}}(%esp)
+; CHECK-NEXT:    pushl %eax
+; CHECK-NEXT:    pushl $0
+; CHECK-NEXT:    pushl {{[0-9]+}}(%esp)
+; CHECK-NEXT:    pushl %ecx
+; CHECK-NEXT:    pushl $0
+; CHECK-NEXT:    pushl %edx
+; CHECK-NEXT:    pushl %edi
+; CHECK-NEXT:    calll _SQLDrivers_Internal
+; CHECK-NEXT:    addl $48, %esp
+; CHECK-NEXT:    movl %eax, %esi
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    testb %al, %al
+; CHECK-NEXT:    je LBB0_1
+; CHECK-NEXT:  ## %bb.5:
+; CHECK-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT:  LBB0_6: ## %done
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    testb %al, %al
+; CHECK-NEXT:    je LBB0_7
+; CHECK-NEXT:  ## %bb.8: ## %bb167
+; CHECK-NEXT:    subl $12, %esp
+; CHECK-NEXT:    movl L_iodbcdm_global_lock$non_lazy_ptr, %eax
+; CHECK-NEXT:    pushl %eax
+; CHECK-NEXT:    calll _pthread_mutex_unlock
+; CHECK-NEXT:    addl $16, %esp
+; CHECK-NEXT:    movl %esi, %eax
+; CHECK-NEXT:    jmp LBB0_2
+; CHECK-NEXT:  LBB0_7: ## %bb150
+; CHECK-NEXT:    movswl %si, %eax
+; CHECK-NEXT:    subl $8, %esp
+; CHECK-NEXT:    movswl %cx, %ecx
+; CHECK-NEXT:    movswl %bx, %edx
+; CHECK-NEXT:    movzwl %bp, %esi
+; CHECK-NEXT:    pushl {{[0-9]+}}(%esp)
+; CHECK-NEXT:    pushl %ecx
+; CHECK-NEXT:    pushl {{[0-9]+}}(%esp)
+; CHECK-NEXT:    pushl {{[0-9]+}}(%esp)
+; CHECK-NEXT:    pushl %edx
+; CHECK-NEXT:    pushl {{[0-9]+}}(%esp)
+; CHECK-NEXT:    pushl %esi
+; CHECK-NEXT:    pushl %edi
+; CHECK-NEXT:    pushl %eax
+; CHECK-NEXT:    pushl $1
+; CHECK-NEXT:    calll _trace_SQLDriversW
+; CHECK-NEXT:    addl $48, %esp
+; CHECK-NEXT:  LBB0_1: ## %bb
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:  LBB0_2: ## %bb
+; CHECK-NEXT:    addl $12, %esp
+; CHECK-NEXT:    popl %esi
+; CHECK-NEXT:    popl %edi
+; CHECK-NEXT:    popl %ebx
+; CHECK-NEXT:    popl %ebp
+; CHECK-NEXT:    retl
 entry:
 	%tmp12 = bitcast i8* %henv to %struct.GENV_t*		; <%struct.GENV_t*> [#uses=1]
 	br i1 true, label %bb28, label %bb
@@ -32,7 +116,7 @@ done:		; preds = %bb92, %bb28
 	br i1 false, label %bb167, label %bb150
 bb150:		; preds = %done
 	%tmp157158 = sext i16 %retcode.0 to i32		; <i32> [#uses=1]
-	tail call void @trace_SQLDriversW( i32 1, i32 %tmp157158, i8* %henv, i16 zeroext  %fDir, i32* %szDrvDesc, i16 signext  %cbDrvDescMax, i16* %pcbDrvDesc, i32* %szDrvAttr, i16 signext  %cbDrvAttrMax, i16* %pcbDrvAttr ) nounwind 
+	tail call void @trace_SQLDriversW( i32 1, i32 %tmp157158, i8* %henv, i16 zeroext  %fDir, i32* %szDrvDesc, i16 signext  %cbDrvDescMax, i16* %pcbDrvDesc, i32* %szDrvAttr, i16 signext  %cbDrvAttrMax, i16* %pcbDrvAttr ) nounwind
 	ret i16 0
 bb167:		; preds = %done
 	%tmp168 = tail call i32 @pthread_mutex_unlock( %struct.pthread_mutex_t* @iodbcdm_global_lock ) nounwind 		; <i32> [#uses=0]
@@ -41,6 +125,6 @@ bb167:		; preds = %done
 
 declare i32 @pthread_mutex_unlock(%struct.pthread_mutex_t*)
 
-declare i16 @SQLDrivers_Internal(i8*, i16 zeroext , i8*, i16 signext , i16*, i8*, i16 signext , i16*, i8 zeroext )  nounwind 
+declare i16 @SQLDrivers_Internal(i8*, i16 zeroext , i8*, i16 signext , i16*, i8*, i16 signext , i16*, i8 zeroext )  nounwind
 
 declare void @trace_SQLDriversW(i32, i32, i8*, i16 zeroext , i32*, i16 signext , i16*, i32*, i16 signext , i16*)
diff --git a/llvm/test/CodeGen/X86/2008-04-17-CoalescerBug.ll b/llvm/test/CodeGen/X86/2008-04-17-CoalescerBug.ll
index 3d3851cbd..65240d5ae 100644
--- a/llvm/test/CodeGen/X86/2008-04-17-CoalescerBug.ll
+++ b/llvm/test/CodeGen/X86/2008-04-17-CoalescerBug.ll
@@ -1,4 +1,5 @@
-; RUN: llc < %s -mtriple=i386-apple-darwin | grep xorl | grep "%e"
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i386-apple-darwin | FileCheck %s
 ; Make sure xorl operands are 32-bit registers.
 
 	%struct.tm = type { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i8* }
@@ -14,6 +15,179 @@
 @.str89 = external constant [5 x i32]		; <[5 x i32]*> [#uses=1]
 
 define void @_ZNK10wxDateTime6FormatEPKwRKNS_8TimeZoneE(%struct.wxString* noalias sret  %agg.result, %struct.wxDateTime* %this, i32* %format, %"struct.wxDateTime::TimeZone"* %tz, i1 %foo) personality i32 (...)* @__gxx_personality_v0 {
+; CHECK-LABEL: _ZNK10wxDateTime6FormatEPKwRKNS_8TimeZoneE:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    pushl %ebp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    pushl %ebx
+; CHECK-NEXT:    .cfi_def_cfa_offset 12
+; CHECK-NEXT:    pushl %edi
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushl %esi
+; CHECK-NEXT:    .cfi_def_cfa_offset 20
+; CHECK-NEXT:    subl $28, %esp
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    .cfi_offset %esi, -20
+; CHECK-NEXT:    .cfi_offset %edi, -16
+; CHECK-NEXT:    .cfi_offset %ebx, -12
+; CHECK-NEXT:    .cfi_offset %ebp, -8
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; CHECK-NEXT:    movb {{[0-9]+}}(%esp), %bl
+; CHECK-NEXT:    testb $1, %bl
+; CHECK-NEXT:    je LBB0_27
+; CHECK-NEXT:  ## %bb.1: ## %bb116.i
+; CHECK-NEXT:    je LBB0_27
+; CHECK-NEXT:  ## %bb.2: ## %bb52.i.i
+; CHECK-NEXT:    testb $1, %bl
+; CHECK-NEXT:    je LBB0_27
+; CHECK-NEXT:  ## %bb.3: ## %bb142.i
+; CHECK-NEXT:    je LBB0_27
+; CHECK-NEXT:  ## %bb.4:
+; CHECK-NEXT:    movl L_.str89$non_lazy_ptr, %edi
+; CHECK-NEXT:    movb $1, %bh
+; CHECK-NEXT:    movl $274877907, %ebp ## imm = 0x10624DD3
+; CHECK-NEXT:    jmp LBB0_5
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  LBB0_23: ## %bb7806
+; CHECK-NEXT:    ## in Loop: Header=BB0_5 Depth=1
+; CHECK-NEXT:  Ltmp16:
+; CHECK-NEXT:    movl $0, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movl $1, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movl $0, (%esp)
+; CHECK-NEXT:    calll __ZN12wxStringBase6appendEmw
+; CHECK-NEXT:  Ltmp17:
+; CHECK-NEXT:  LBB0_5: ## %bb3261
+; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    cmpl $37, 0
+; CHECK-NEXT:    jne LBB0_27
+; CHECK-NEXT:  ## %bb.6: ## %bb3306
+; CHECK-NEXT:    ## in Loop: Header=BB0_5 Depth=1
+; CHECK-NEXT:  Ltmp0:
+; CHECK-NEXT:    movl %edi, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movl $0, (%esp)
+; CHECK-NEXT:    calll __ZN12wxStringBaseaSEPKw
+; CHECK-NEXT:  Ltmp1:
+; CHECK-NEXT:  ## %bb.7: ## %bb3314
+; CHECK-NEXT:    ## in Loop: Header=BB0_5 Depth=1
+; CHECK-NEXT:    movl 0, %eax
+; CHECK-NEXT:    cmpl $121, %eax
+; CHECK-NEXT:    ja LBB0_27
+; CHECK-NEXT:  ## %bb.8: ## %bb3314
+; CHECK-NEXT:    ## in Loop: Header=BB0_5 Depth=1
+; CHECK-NEXT:    jmpl *LJTI0_0(,%eax,4)
+; CHECK-NEXT:  LBB0_10: ## %bb5809
+; CHECK-NEXT:    ## in Loop: Header=BB0_5 Depth=1
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    testb %al, %al
+; CHECK-NEXT:    jne LBB0_27
+; CHECK-NEXT:  ## %bb.11: ## %bb5809
+; CHECK-NEXT:    ## in Loop: Header=BB0_5 Depth=1
+; CHECK-NEXT:    testb %bh, %bh
+; CHECK-NEXT:    je LBB0_27
+; CHECK-NEXT:  ## %bb.12: ## %bb91.i8504
+; CHECK-NEXT:    ## in Loop: Header=BB0_5 Depth=1
+; CHECK-NEXT:    testb $1, %bl
+; CHECK-NEXT:    je LBB0_14
+; CHECK-NEXT:  ## %bb.13: ## %bb155.i8541
+; CHECK-NEXT:    ## in Loop: Header=BB0_5 Depth=1
+; CHECK-NEXT:  Ltmp4:
+; CHECK-NEXT:    movl $0, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movl $0, (%esp)
+; CHECK-NEXT:    calll _gmtime_r
+; CHECK-NEXT:  Ltmp5:
+; CHECK-NEXT:  LBB0_14: ## %bb182.i8560
+; CHECK-NEXT:    ## in Loop: Header=BB0_5 Depth=1
+; CHECK-NEXT:    testb $1, %bl
+; CHECK-NEXT:    je LBB0_15
+; CHECK-NEXT:  ## %bb.17: ## %bb278.i8617
+; CHECK-NEXT:    ## in Loop: Header=BB0_5 Depth=1
+; CHECK-NEXT:    je LBB0_19
+; CHECK-NEXT:  ## %bb.18: ## %bb440.i8663
+; CHECK-NEXT:    ## in Loop: Header=BB0_5 Depth=1
+; CHECK-NEXT:  Ltmp6:
+; CHECK-NEXT:    movl L_.str4$non_lazy_ptr, %eax
+; CHECK-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movl L_.str33$non_lazy_ptr, %eax
+; CHECK-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movl L__ZZNK10wxDateTime5GetTmERKNS_8TimeZoneEE12__FUNCTION__$non_lazy_ptr, %eax
+; CHECK-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movl L_.str$non_lazy_ptr, %eax
+; CHECK-NEXT:    movl %eax, (%esp)
+; CHECK-NEXT:    movl $1717, {{[0-9]+}}(%esp) ## imm = 0x6B5
+; CHECK-NEXT:    calll __Z10wxOnAssertPKwiPKcS0_S0_
+; CHECK-NEXT:  Ltmp7:
+; CHECK-NEXT:  LBB0_19: ## %bb448.i8694
+; CHECK-NEXT:    ## in Loop: Header=BB0_5 Depth=1
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    jmp LBB0_20
+; CHECK-NEXT:  LBB0_15: ## %bb187.i8591
+; CHECK-NEXT:    ## in Loop: Header=BB0_5 Depth=1
+; CHECK-NEXT:    jne LBB0_27
+; CHECK-NEXT:  ## %bb.16: ## %bb265.i8606
+; CHECK-NEXT:    ## in Loop: Header=BB0_5 Depth=1
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    imull %ebp
+; CHECK-NEXT:    movl %edx, %eax
+; CHECK-NEXT:    shrl $31, %eax
+; CHECK-NEXT:    shrl $6, %edx
+; CHECK-NEXT:    addl %eax, %edx
+; CHECK-NEXT:    imull $1000, %edx, %eax ## imm = 0x3E8
+; CHECK-NEXT:    negl %eax
+; CHECK-NEXT:  LBB0_20: ## %invcont5814
+; CHECK-NEXT:    ## in Loop: Header=BB0_5 Depth=1
+; CHECK-NEXT:    movzwl %ax, %eax
+; CHECK-NEXT:  Ltmp8:
+; CHECK-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movl $0, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movl $0, (%esp)
+; CHECK-NEXT:    calll __ZN8wxString6FormatEPKwz
+; CHECK-NEXT:    subl $4, %esp
+; CHECK-NEXT:  Ltmp9:
+; CHECK-NEXT:  ## %bb.21: ## %invcont5831
+; CHECK-NEXT:    ## in Loop: Header=BB0_5 Depth=1
+; CHECK-NEXT:  Ltmp10:
+; CHECK-NEXT:    movl $0, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movl $0, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movl $0, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movl $0, (%esp)
+; CHECK-NEXT:    calll __ZN12wxStringBase10ConcatSelfEmPKwm
+; CHECK-NEXT:  Ltmp11:
+; CHECK-NEXT:    jmp LBB0_5
+; CHECK-NEXT:  LBB0_22: ## %bb5968
+; CHECK-NEXT:  Ltmp2:
+; CHECK-NEXT:    movl $0, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movl $0, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movl $0, (%esp)
+; CHECK-NEXT:    calll __ZN8wxString6FormatEPKwz
+; CHECK-NEXT:    subl $4, %esp
+; CHECK-NEXT:  Ltmp3:
+; CHECK-NEXT:    jmp LBB0_27
+; CHECK-NEXT:  LBB0_9: ## %bb5657
+; CHECK-NEXT:  Ltmp13:
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    movl %eax, (%esp)
+; CHECK-NEXT:    calll __ZNK10wxDateTime12GetDayOfYearERKNS_8TimeZoneE
+; CHECK-NEXT:  Ltmp14:
+; CHECK-NEXT:  LBB0_27: ## %bb115.critedge.i
+; CHECK-NEXT:    movl %esi, %eax
+; CHECK-NEXT:    addl $28, %esp
+; CHECK-NEXT:    popl %esi
+; CHECK-NEXT:    popl %edi
+; CHECK-NEXT:    popl %ebx
+; CHECK-NEXT:    popl %ebp
+; CHECK-NEXT:    retl $4
+; CHECK-NEXT:  LBB0_25: ## %lpad.loopexit.split-lp
+; CHECK-NEXT:  Ltmp15:
+; CHECK-NEXT:    jmp LBB0_27
+; CHECK-NEXT:  LBB0_26: ## %lpad8185
+; CHECK-NEXT:  Ltmp12:
+; CHECK-NEXT:    jmp LBB0_27
+; CHECK-NEXT:  LBB0_24: ## %lpad.loopexit
+; CHECK-NEXT:  Ltmp18:
+; CHECK-NEXT:    jmp LBB0_27
+; CHECK-NEXT:  Lfunc_end0:
 entry:
 	br i1 %foo, label %bb116.i, label %bb115.critedge.i
 bb115.critedge.i:		; preds = %entry
@@ -75,7 +249,7 @@ bb5334:		; preds = %bb3314
 bb5484:		; preds = %bb3314
 	ret void
 bb5657:		; preds = %bb3314
-	%tmp5661 = invoke zeroext i16 @_ZNK10wxDateTime12GetDayOfYearERKNS_8TimeZoneE( %struct.wxDateTime* %this, %"struct.wxDateTime::TimeZone"* %tz )  
+	%tmp5661 = invoke zeroext i16 @_ZNK10wxDateTime12GetDayOfYearERKNS_8TimeZoneE( %struct.wxDateTime* %this, %"struct.wxDateTime::TimeZone"* %tz )
 			to label %invcont5660 unwind label %lpad		; <i16> [#uses=0]
 invcont5660:		; preds = %bb5657
 	ret void
@@ -120,7 +294,7 @@ invcont5814:		; preds = %bb448.i8694, %bb265.i8606
 	invoke void (%struct.wxString*, i32*, ...) @_ZN8wxString6FormatEPKwz( %struct.wxString* noalias sret  null, i32* null, i32 %tmp58165817 )
 			to label %invcont5831 unwind label %lpad
 invcont5831:		; preds = %invcont5814
-	%tmp5862 = invoke zeroext  i8 @_ZN12wxStringBase10ConcatSelfEmPKwm( %struct.wxStringBase* null, i32 0, i32* null, i32 0 ) 
+	%tmp5862 = invoke zeroext  i8 @_ZN12wxStringBase10ConcatSelfEmPKwm( %struct.wxStringBase* null, i32 0, i32* null, i32 0 )
 			to label %bb7834 unwind label %lpad8185		; <i8> [#uses=0]
 bb5968:		; preds = %bb3314
 	invoke void (%struct.wxString*, i32*, ...) @_ZN8wxString6FormatEPKwz( %struct.wxString* noalias sret  null, i32* null, i32 0 )
@@ -162,11 +336,11 @@ lpad8185:		; preds = %invcont5831
 
 declare void @_Z10wxOnAssertPKwiPKcS0_S0_(i32*, i32, i8*, i32*, i32*)
 
-declare zeroext  i8 @_ZN12wxStringBase10ConcatSelfEmPKwm(%struct.wxStringBase*, i32, i32*, i32) 
+declare zeroext  i8 @_ZN12wxStringBase10ConcatSelfEmPKwm(%struct.wxStringBase*, i32, i32*, i32)
 
 declare %struct.tm* @gmtime_r(i32*, %struct.tm*)
 
-declare zeroext  i16 @_ZNK10wxDateTime12GetDayOfYearERKNS_8TimeZoneE(%struct.wxDateTime*, %"struct.wxDateTime::TimeZone"*) 
+declare zeroext  i16 @_ZNK10wxDateTime12GetDayOfYearERKNS_8TimeZoneE(%struct.wxDateTime*, %"struct.wxDateTime::TimeZone"*)
 
 declare %struct.wxStringBase* @_ZN12wxStringBase6appendEmw(%struct.wxStringBase*, i32, i32)
 
diff --git a/llvm/test/CodeGen/X86/2008-04-24-MemCpyBug.ll b/llvm/test/CodeGen/X86/2008-04-24-MemCpyBug.ll
index cd0f3a385..fb6baa200 100644
--- a/llvm/test/CodeGen/X86/2008-04-24-MemCpyBug.ll
+++ b/llvm/test/CodeGen/X86/2008-04-24-MemCpyBug.ll
@@ -1,12 +1,31 @@
-; RUN: llc < %s -mtriple=i686-- | not grep 120
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- | FileCheck %s
 ; Don't accidentally add the offset twice for trailing bytes.
 
 	%struct.S63 = type { [63 x i8] }
 @g1s63 = external global %struct.S63		; <%struct.S63*> [#uses=1]
 
-declare void @test63(%struct.S63* byval align 4 ) nounwind 
+declare void @test63(%struct.S63* byval align 4 ) nounwind
 
 define void @testit63_entry_2E_ce() nounwind  {
-	tail call void @test63( %struct.S63* byval align 4  @g1s63 ) nounwind 
+; CHECK-LABEL: testit63_entry_2E_ce:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushl %edi
+; CHECK-NEXT:    pushl %esi
+; CHECK-NEXT:    subl $64, %esp
+; CHECK-NEXT:    movl $15, %ecx
+; CHECK-NEXT:    movl %esp, %edi
+; CHECK-NEXT:    movl $g1s63, %esi
+; CHECK-NEXT:    rep;movsl (%esi), %es:(%edi)
+; CHECK-NEXT:    movb g1s63+62, %al
+; CHECK-NEXT:    movb %al, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movzwl g1s63+60, %eax
+; CHECK-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    calll test63
+; CHECK-NEXT:    addl $64, %esp
+; CHECK-NEXT:    popl %esi
+; CHECK-NEXT:    popl %edi
+; CHECK-NEXT:    retl
+	tail call void @test63( %struct.S63* byval align 4  @g1s63 ) nounwind
 	ret void
 }
diff --git a/llvm/test/CodeGen/X86/2008-04-28-CoalescerBug.ll b/llvm/test/CodeGen/X86/2008-04-28-CoalescerBug.ll
index 06bbd74e8..f64789f9f 100644
--- a/llvm/test/CodeGen/X86/2008-04-28-CoalescerBug.ll
+++ b/llvm/test/CodeGen/X86/2008-04-28-CoalescerBug.ll
@@ -1,7 +1,5 @@
-; RUN: llc < %s -mtriple=x86_64-apple-darwin | grep movl > %t
-; RUN: not grep "r[abcd]x" %t
-; RUN: not grep "r[ds]i" %t
-; RUN: not grep "r[bs]p" %t
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-apple-darwin | FileCheck %s
 
 	%struct.BITMAP = type { i16, i16, i32, i32, i32, i32, i32, i32, i8*, i8* }
 	%struct.BltData = type { float, float, float, float }
@@ -11,6 +9,58 @@
 	%struct.BltSh = type { i8, i8, i8, i8, float, float*, float*, float*, float*, i32, i32, float*, float*, float* }
 
 define void @t(%struct.BltDepth* %depth, %struct.BltOp* %bop, i32 %mode) nounwind  {
+; CHECK-LABEL: t:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    cmpl $18, %edx
+; CHECK-NEXT:    je LBB0_3
+; CHECK-NEXT:  ## %bb.1: ## %entry
+; CHECK-NEXT:    cmpl $1, %edx
+; CHECK-NEXT:    jne LBB0_10
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  LBB0_2: ## %bb2898.us
+; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    jmp LBB0_2
+; CHECK-NEXT:  LBB0_3: ## %bb13086.preheader
+; CHECK-NEXT:    movb $1, %al
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  LBB0_4: ## %bb13088
+; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    testb %al, %al
+; CHECK-NEXT:    jne LBB0_5
+; CHECK-NEXT:  ## %bb.6: ## %bb13101
+; CHECK-NEXT:    ## in Loop: Header=BB0_4 Depth=1
+; CHECK-NEXT:    xorl %ecx, %ecx
+; CHECK-NEXT:    jmp LBB0_7
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  LBB0_5: ## in Loop: Header=BB0_4 Depth=1
+; CHECK-NEXT:    movl $65535, %ecx ## imm = 0xFFFF
+; CHECK-NEXT:  LBB0_7: ## %bb13107
+; CHECK-NEXT:    ## in Loop: Header=BB0_4 Depth=1
+; CHECK-NEXT:    movl %ecx, %edx
+; CHECK-NEXT:    shll $16, %edx
+; CHECK-NEXT:    subl %ecx, %edx
+; CHECK-NEXT:    incl %edx
+; CHECK-NEXT:    shrl $16, %edx
+; CHECK-NEXT:    subl %edx, %ecx
+; CHECK-NEXT:    testw %cx, %cx
+; CHECK-NEXT:    je LBB0_4
+; CHECK-NEXT:  ## %bb.8: ## %bb13236
+; CHECK-NEXT:    ## in Loop: Header=BB0_4 Depth=1
+; CHECK-NEXT:    cmpw $-1, %cx
+; CHECK-NEXT:    testb %al, %al
+; CHECK-NEXT:    jne LBB0_4
+; CHECK-NEXT:  ## %bb.9: ## %bb13572
+; CHECK-NEXT:    ## in Loop: Header=BB0_4 Depth=1
+; CHECK-NEXT:    movzwl %cx, %ecx
+; CHECK-NEXT:    movl %ecx, %edx
+; CHECK-NEXT:    shll $16, %edx
+; CHECK-NEXT:    subl %ecx, %edx
+; CHECK-NEXT:    incl %edx
+; CHECK-NEXT:    shrl $16, %edx
+; CHECK-NEXT:    movw %dx, 0
+; CHECK-NEXT:    jmp LBB0_4
+; CHECK-NEXT:  LBB0_10: ## %return
+; CHECK-NEXT:    retq
 entry:
 	switch i32 %mode, label %return [
 		 i32 1, label %bb2898.us
diff --git a/llvm/test/CodeGen/X86/2008-05-01-InvalidOrdCompare.ll b/llvm/test/CodeGen/X86/2008-05-01-InvalidOrdCompare.ll
index 42ccb21fa..43dc21972 100644
--- a/llvm/test/CodeGen/X86/2008-05-01-InvalidOrdCompare.ll
+++ b/llvm/test/CodeGen/X86/2008-05-01-InvalidOrdCompare.ll
@@ -1,9 +1,25 @@
-; RUN: llc < %s -enable-unsafe-fp-math -mtriple=i686-- | grep jp
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -enable-unsafe-fp-math -mtriple=i686-- | FileCheck %s
 ; rdar://5902801
 
 declare void @test2()
 
 define i32 @test(double %p) nounwind {
+; CHECK-LABEL: test:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fldl {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fucomp %st(0)
+; CHECK-NEXT:    fnstsw %ax
+; CHECK-NEXT:    # kill: def $ah killed $ah killed $ax
+; CHECK-NEXT:    sahf
+; CHECK-NEXT:    jp .LBB0_1
+; CHECK-NEXT:  # %bb.2: # %UnifiedReturnBlock
+; CHECK-NEXT:    movl $42, %eax
+; CHECK-NEXT:    retl
+; CHECK-NEXT:  .LBB0_1: # %bb
+; CHECK-NEXT:    calll test2
+; CHECK-NEXT:    movl $17, %eax
+; CHECK-NEXT:    retl
 	%tmp5 = fcmp uno double %p, 0.000000e+00
 	br i1 %tmp5, label %bb, label %UnifiedReturnBlock
 bb:
diff --git a/llvm/test/CodeGen/X86/2008-05-12-tailmerge-5.ll b/llvm/test/CodeGen/X86/2008-05-12-tailmerge-5.ll
index 0669a3267..4e17e996c 100644
--- a/llvm/test/CodeGen/X86/2008-05-12-tailmerge-5.ll
+++ b/llvm/test/CodeGen/X86/2008-05-12-tailmerge-5.ll
@@ -1,4 +1,5 @@
-; RUN: llc < %s | grep abort | count 1
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s | FileCheck %s
 ; Calls to abort should all be merged
 
 ; ModuleID = '5898899.c'
@@ -7,6 +8,53 @@ target triple = "x86_64-apple-darwin8"
 	%struct.BoundaryAlignment = type { [3 x i8], i8, i16, i16, i8, [2 x i8] }
 
 define void @passing2(i64 %str.0, i64 %str.1, i16 signext  %s, i32 %j, i8 signext  %c, i16 signext  %t, i16 signext  %u, i8 signext  %d) nounwind optsize {
+; CHECK-LABEL: passing2:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    subq $40, %rsp
+; CHECK-NEXT:    movq %rsi, %rax
+; CHECK-NEXT:    movq %rdi, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movb %al, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movb %ah, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    shrq $16, %rsi
+; CHECK-NEXT:    movb %sil, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    shrq $24, %rax
+; CHECK-NEXT:    movb %al, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movw %dx, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movl %ecx, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movb %r8b, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movw %r9w, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    shll $14, %edi
+; CHECK-NEXT:    sarl $23, %edi
+; CHECK-NEXT:    cmpl %ecx, %edi
+; CHECK-NEXT:    jne LBB0_6
+; CHECK-NEXT:  ## %bb.1: ## %bb27
+; CHECK-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; CHECK-NEXT:    cmpb {{[0-9]+}}(%rsp), %al
+; CHECK-NEXT:    jne LBB0_6
+; CHECK-NEXT:  ## %bb.2: ## %bb35
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    shll $7, %eax
+; CHECK-NEXT:    cwtl
+; CHECK-NEXT:    shrl $7, %eax
+; CHECK-NEXT:    cmpw {{[0-9]+}}(%rsp), %ax
+; CHECK-NEXT:    jne LBB0_6
+; CHECK-NEXT:  ## %bb.3: ## %bb51
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    shll $7, %eax
+; CHECK-NEXT:    cwtl
+; CHECK-NEXT:    shrl $7, %eax
+; CHECK-NEXT:    cmpw {{[0-9]+}}(%rsp), %ax
+; CHECK-NEXT:    jne LBB0_6
+; CHECK-NEXT:  ## %bb.4: ## %bb67
+; CHECK-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; CHECK-NEXT:    cmpb {{[0-9]+}}(%rsp), %al
+; CHECK-NEXT:    jne LBB0_6
+; CHECK-NEXT:  ## %bb.5: ## %bb75
+; CHECK-NEXT:    addq $40, %rsp
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  LBB0_6: ## %bb
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    callq _abort
 entry:
 	%str_addr = alloca %struct.BoundaryAlignment		; <%struct.BoundaryAlignment*> [#uses=7]
 	%s_addr = alloca i16		; <i16*> [#uses=1]
@@ -64,7 +112,7 @@ entry:
 	br i1 %toBool, label %bb, label %bb27
 
 bb:		; preds = %entry
-	call void (...) @abort( ) noreturn nounwind 
+	call void (...) @abort( ) noreturn nounwind
 	unreachable
 
 bb27:		; preds = %entry
@@ -77,7 +125,7 @@ bb27:		; preds = %entry
 	br i1 %toBool33, label %bb34, label %bb35
 
 bb34:		; preds = %bb27
-	call void (...) @abort( ) noreturn nounwind 
+	call void (...) @abort( ) noreturn nounwind
 	unreachable
 
 bb35:		; preds = %bb27
@@ -98,7 +146,7 @@ bb35:		; preds = %bb27
 	br i1 %toBool49, label %bb50, label %bb51
 
 bb50:		; preds = %bb35
-	call void (...) @abort( ) noreturn nounwind 
+	call void (...) @abort( ) noreturn nounwind
 	unreachable
 
 bb51:		; preds = %bb35
@@ -119,7 +167,7 @@ bb51:		; preds = %bb35
 	br i1 %toBool65, label %bb66, label %bb67
 
 bb66:		; preds = %bb51
-	call void (...) @abort( ) noreturn nounwind 
+	call void (...) @abort( ) noreturn nounwind
 	unreachable
 
 bb67:		; preds = %bb51
@@ -132,7 +180,7 @@ bb67:		; preds = %bb51
 	br i1 %toBool73, label %bb74, label %bb75
 
 bb74:		; preds = %bb67
-	call void (...) @abort( ) noreturn nounwind 
+	call void (...) @abort( ) noreturn nounwind
 	unreachable
 
 bb75:		; preds = %bb67
@@ -142,4 +190,4 @@ return:		; preds = %bb75
 	ret void
 }
 
-declare void @abort(...) noreturn nounwind 
+declare void @abort(...) noreturn nounwind
diff --git a/llvm/test/CodeGen/X86/2008-05-21-CoalescerBug.ll b/llvm/test/CodeGen/X86/2008-05-21-CoalescerBug.ll
index 4dc6c4e49..976e17e9b 100644
--- a/llvm/test/CodeGen/X86/2008-05-21-CoalescerBug.ll
+++ b/llvm/test/CodeGen/X86/2008-05-21-CoalescerBug.ll
@@ -1,4 +1,5 @@
-; RUN: llc < %s -mtriple=i686-- -O0 -fast-isel=false -optimize-regalloc -regalloc=basic | grep mov | count 5
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- -O0 -fast-isel=false -optimize-regalloc -regalloc=basic | FileCheck %s
 ; PR2343
 
 	%llvm.dbg.anchor.type = type { i32, i32 }
@@ -70,6 +71,33 @@
 @llvm.used = appending global [1 x i8*] [ i8* bitcast (%struct.edge_def* (%struct.edge_def*, %struct.basic_block_def*)* @tree_redirect_edge_and_branch to i8*) ], section "llvm.metadata"		; <[1 x i8*]*> [#uses=0]
 
 define %struct.edge_def* @tree_redirect_edge_and_branch(%struct.edge_def* %e1, %struct.basic_block_def* %dest2) nounwind  {
+; CHECK-LABEL: tree_redirect_edge_and_branch:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushl %esi
+; CHECK-NEXT:    movb $1, %cl
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    # implicit-def: $esi
+; CHECK-NEXT:    jmp .LBB0_2
+; CHECK-NEXT:  .LBB0_1: # %bb483
+; CHECK-NEXT:    # in Loop: Header=BB0_2 Depth=1
+; CHECK-NEXT:    movl 0, %edx
+; CHECK-NEXT:    xorl %ecx, %ecx
+; CHECK-NEXT:    movl %eax, %esi
+; CHECK-NEXT:    jmp .LBB0_2
+; CHECK-NEXT:  .LBB0_2: # %bb497
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movl %edx, %eax
+; CHECK-NEXT:    testb $1, %cl
+; CHECK-NEXT:    jne .LBB0_1
+; CHECK-NEXT:    jmp .LBB0_3
+; CHECK-NEXT:  .LBB0_3: # %bb502
+; CHECK-NEXT:    testb $1, %cl
+; CHECK-NEXT:    je .LBB0_5
+; CHECK-NEXT:    jmp .LBB0_4
+; CHECK-NEXT:  .LBB0_4: # %bb507
+; CHECK-NEXT:    movl $0, (%esi)
+; CHECK-NEXT:    jmp .LBB0_5
+; CHECK-NEXT:  .LBB0_5: # %bb841
 entry:
 	br label %bb497
 
diff --git a/llvm/test/CodeGen/X86/2008-06-13-VolatileLoadStore.ll b/llvm/test/CodeGen/X86/2008-06-13-VolatileLoadStore.ll
index 5a05ec13f..e9059c0c6 100644
--- a/llvm/test/CodeGen/X86/2008-06-13-VolatileLoadStore.ll
+++ b/llvm/test/CodeGen/X86/2008-06-13-VolatileLoadStore.ll
@@ -1,5 +1,5 @@
-; RUN: llc < %s -mtriple=i686-linux -mattr=+sse2 | grep movsd | count 5
-; RUN: llc < %s -mtriple=i686-linux -mattr=+sse2 | grep movl | count 2
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-linux -mattr=+sse2 | FileCheck %s
 
 @atomic = global double 0.000000e+00		; <double*> [#uses=1]
 @atomic2 = global double 0.000000e+00		; <double*> [#uses=1]
@@ -7,6 +7,20 @@
 @ioport = global i32 0		; <i32*> [#uses=2]
 
 define i16 @f(i64 %x, double %y) {
+; CHECK-LABEL: f:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
+; CHECK-NEXT:    movsd %xmm1, atomic
+; CHECK-NEXT:    xorps %xmm1, %xmm1
+; CHECK-NEXT:    movsd %xmm1, atomic2
+; CHECK-NEXT:    movsd %xmm0, anything
+; CHECK-NEXT:    movl ioport, %ecx
+; CHECK-NEXT:    movl ioport, %eax
+; CHECK-NEXT:    shrl $16, %eax
+; CHECK-NEXT:    addl %ecx, %eax
+; CHECK-NEXT:    # kill: def $ax killed $ax killed $eax
+; CHECK-NEXT:    retl
 	%b = bitcast i64 %x to double		; <double> [#uses=1]
 	store volatile double %b, double* @atomic ; one processor operation only
 	store volatile double 0.000000e+00, double* @atomic2 ; one processor operation only
diff --git a/llvm/test/CodeGen/X86/2008-06-16-SubregsBug.ll b/llvm/test/CodeGen/X86/2008-06-16-SubregsBug.ll
index cdd1b0bfe..c56be8314 100644
--- a/llvm/test/CodeGen/X86/2008-06-16-SubregsBug.ll
+++ b/llvm/test/CodeGen/X86/2008-06-16-SubregsBug.ll
@@ -1,6 +1,21 @@
-; RUN: llc < %s -mtriple=i386-apple-darwin | grep mov | count 4
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i386-apple-darwin | FileCheck %s
 
 define i16 @test(i16* %tmp179) nounwind  {
+; CHECK-LABEL: test:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    movzwl (%eax), %eax
+; CHECK-NEXT:    movl %eax, %ecx
+; CHECK-NEXT:    andl $64512, %ecx ## imm = 0xFC00
+; CHECK-NEXT:    cmpl $32768, %ecx ## imm = 0x8000
+; CHECK-NEXT:    jne LBB0_2
+; CHECK-NEXT:  ## %bb.1: ## %bb189
+; CHECK-NEXT:    ## kill: def $ax killed $ax killed $eax
+; CHECK-NEXT:    retl
+; CHECK-NEXT:  LBB0_2: ## %bb288
+; CHECK-NEXT:    movw $32, %ax
+; CHECK-NEXT:    retl
 	%tmp180 = load i16, i16* %tmp179, align 2		; <i16> [#uses=2]
 	%tmp184 = and i16 %tmp180, -1024		; <i16> [#uses=1]
 	%tmp186 = icmp eq i16 %tmp184, -32768		; <i1> [#uses=1]
diff --git a/llvm/test/CodeGen/X86/2008-06-25-VecISelBug.ll b/llvm/test/CodeGen/X86/2008-06-25-VecISelBug.ll
index 7836316b0..35140a211 100644
--- a/llvm/test/CodeGen/X86/2008-06-25-VecISelBug.ll
+++ b/llvm/test/CodeGen/X86/2008-06-25-VecISelBug.ll
@@ -1,6 +1,11 @@
-; RUN: llc < %s -mtriple=i686-- -mattr=+sse2 | not grep pslldq
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- -mattr=+sse2 | FileCheck %s
 
 define void @t() nounwind  {
+; CHECK-LABEL: t:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movaps {{.*#+}} xmm0 = [0.0E+0,0.0E+0,0.0E+0,1.0E+0]
+; CHECK-NEXT:    movaps %xmm0, 0
 entry:
 	%tmp1 = shufflevector <4 x float> zeroinitializer, <4 x float> < float 0.000000e+00, float 1.000000e+00, float 0.000000e+00, float 1.000000e+00 >, <4 x i32> < i32 0, i32 1, i32 4, i32 5 >
 	%tmp2 = insertelement <4 x float> %tmp1, float 1.000000e+00, i32 3
diff --git a/llvm/test/CodeGen/X86/2008-07-09-ELFSectionAttributes.ll b/llvm/test/CodeGen/X86/2008-07-09-ELFSectionAttributes.ll
index 1a786ef7a..c818d14d5 100644
--- a/llvm/test/CodeGen/X86/2008-07-09-ELFSectionAttributes.ll
+++ b/llvm/test/CodeGen/X86/2008-07-09-ELFSectionAttributes.ll
@@ -1,6 +1,8 @@
-; RUN: llc < %s | grep ax
+; RUN: llc < %s | FileCheck %s
 ; PR2024
 
+; CHECK: .init.text,"ax"
+
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32"
 target triple = "i386-pc-linux-gnu"
 
diff --git a/llvm/test/CodeGen/X86/2008-07-11-SHLBy1.ll b/llvm/test/CodeGen/X86/2008-07-11-SHLBy1.ll
index ef1594603..7779d2eb3 100644
--- a/llvm/test/CodeGen/X86/2008-07-11-SHLBy1.ll
+++ b/llvm/test/CodeGen/X86/2008-07-11-SHLBy1.ll
@@ -1,5 +1,13 @@
-; RUN: llc < %s -mtriple=x86_64-- -o - | not grep shr
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-- | FileCheck %s
+
 define i128 @sl(i128 %x) {
+; CHECK-LABEL: sl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %rsi, %rdx
+; CHECK-NEXT:    shldq $1, %rdi, %rdx
+; CHECK-NEXT:    leaq (%rdi,%rdi), %rax
+; CHECK-NEXT:    retq
         %t = shl i128 %x, 1
         ret i128 %t
 }
diff --git a/llvm/test/CodeGen/X86/2008-08-06-CmpStride.ll b/llvm/test/CodeGen/X86/2008-08-06-CmpStride.ll
index 1b6bd7ec0..799ba0582 100644
--- a/llvm/test/CodeGen/X86/2008-08-06-CmpStride.ll
+++ b/llvm/test/CodeGen/X86/2008-08-06-CmpStride.ll
@@ -1,10 +1,29 @@
-; RUN: llc -mtriple=x86_64-- < %s -o - | grep "cmpl	\$[1], %"
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-- < %s | FileCheck %s
 
 @.str = internal constant [4 x i8] c"%d\0A\00"
 
 declare i32 @printf(i8* noalias , ...) nounwind
 
 define i32 @main() nounwind {
+; CHECK-LABEL: main:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    movl $10271, %ebx # imm = 0x281F
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB0_1: # %forbody
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movl $.str, %edi
+; CHECK-NEXT:    movl %ebx, %esi
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    callq printf
+; CHECK-NEXT:    addl $-10, %ebx
+; CHECK-NEXT:    cmpl $1, %ebx
+; CHECK-NEXT:    jne .LBB0_1
+; CHECK-NEXT:  # %bb.2: # %afterfor
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    retq
 entry:
         br label %forbody
 
diff --git a/llvm/test/CodeGen/X86/2008-08-17-UComiCodeGenBug.ll b/llvm/test/CodeGen/X86/2008-08-17-UComiCodeGenBug.ll
index 907f4cc4c..eace85b25 100644
--- a/llvm/test/CodeGen/X86/2008-08-17-UComiCodeGenBug.ll
+++ b/llvm/test/CodeGen/X86/2008-08-17-UComiCodeGenBug.ll
@@ -1,6 +1,13 @@
-; RUN: llc < %s -mtriple=x86_64-apple-darwin | grep xorl
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-apple-darwin | FileCheck %s
 
 define i32 @foo(<4 x float> %a, <4 x float> %b) nounwind {
+; CHECK-LABEL: foo:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    ucomiss %xmm1, %xmm0
+; CHECK-NEXT:    setae %al
+; CHECK-NEXT:    retq
 entry:
 	tail call i32 @llvm.x86.sse.ucomige.ss( <4 x float> %a, <4 x float> %b ) nounwind readnone
 	ret i32 %0
diff --git a/llvm/test/CodeGen/X86/2008-09-05-sinttofp-2xi32.ll b/llvm/test/CodeGen/X86/2008-09-05-sinttofp-2xi32.ll
index 4b62d7c58..ac86279ca 100644
--- a/llvm/test/CodeGen/X86/2008-09-05-sinttofp-2xi32.ll
+++ b/llvm/test/CodeGen/X86/2008-09-05-sinttofp-2xi32.ll
@@ -7,7 +7,6 @@
 define <2 x double> @a(<2 x i32> %x) nounwind {
 ; CHECK-LABEL: a:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; CHECK-NEXT:    cvtdq2pd %xmm0, %xmm0
 ; CHECK-NEXT:    retl
 entry:
@@ -19,7 +18,6 @@ define <2 x i32> @b(<2 x double> %x) nounwind {
 ; CHECK-LABEL: b:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    cvttpd2dq %xmm0, %xmm0
-; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,1,3]
 ; CHECK-NEXT:    retl
 entry:
   %y = fptosi <2 x double> %x to <2 x i32>
diff --git a/llvm/test/CodeGen/X86/2008-09-11-CoalescerBug2.ll b/llvm/test/CodeGen/X86/2008-09-11-CoalescerBug2.ll
index cbccc90df..245885268 100644
--- a/llvm/test/CodeGen/X86/2008-09-11-CoalescerBug2.ll
+++ b/llvm/test/CodeGen/X86/2008-09-11-CoalescerBug2.ll
@@ -22,7 +22,6 @@ define i32 @func_44(i16 signext %p_46) nounwind {
 ; SOURCE-SCHED-NEXT:    jne .LBB0_2
 ; SOURCE-SCHED-NEXT:  # %bb.1: # %bb11
 ; SOURCE-SCHED-NEXT:    movzbl %al, %eax
-; SOURCE-SCHED-NEXT:    # kill: def $eax killed $eax def $ax
 ; SOURCE-SCHED-NEXT:    divb %dl
 ; SOURCE-SCHED-NEXT:    movzbl %ah, %eax
 ; SOURCE-SCHED-NEXT:  .LBB0_2: # %bb12
diff --git a/llvm/test/CodeGen/X86/2008-09-25-sseregparm-1.ll b/llvm/test/CodeGen/X86/2008-09-25-sseregparm-1.ll
index 9373c7986..a2dd55767 100644
--- a/llvm/test/CodeGen/X86/2008-09-25-sseregparm-1.ll
+++ b/llvm/test/CodeGen/X86/2008-09-25-sseregparm-1.ll
@@ -1,19 +1,35 @@
-; RUN: llc < %s -mtriple=i686-- -mattr=+sse2 | grep movs | count 2
-; RUN: llc < %s -mtriple=i686-- -mattr=+sse2 | grep fld | count 2
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- -mattr=+sse2 | FileCheck %s
 ; check 'inreg' attribute for sse_regparm
 
 define inreg double @foo1()  nounwind {
+; CHECK-LABEL: foo1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    retl
   ret double 1.0
 }
 
 define inreg float @foo2()  nounwind {
+; CHECK-LABEL: foo2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; CHECK-NEXT:    retl
   ret float 1.0
 }
 
 define double @bar() nounwind {
+; CHECK-LABEL: bar:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fld1
+; CHECK-NEXT:    retl
   ret double 1.0
 }
 
 define float @bar2() nounwind {
+; CHECK-LABEL: bar2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fld1
+; CHECK-NEXT:    retl
   ret float 1.0
 }
diff --git a/llvm/test/CodeGen/X86/2008-10-06-x87ld-nan-1.ll b/llvm/test/CodeGen/X86/2008-10-06-x87ld-nan-1.ll
index 6a78a37c9..7ec3b2301 100644
--- a/llvm/test/CodeGen/X86/2008-10-06-x87ld-nan-1.ll
+++ b/llvm/test/CodeGen/X86/2008-10-06-x87ld-nan-1.ll
@@ -1,12 +1,25 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mattr=-sse2,-sse3,-sse | FileCheck %s
+
 ; ModuleID = 'nan.bc'
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-f80:32:32-v64:64:64-v128:128:128-a0:0:64"
 target triple = "i686-apple-darwin8"
-; RUN: llc < %s -mattr=-sse2,-sse3,-sse | grep fldl
 ; This NaN should be shortened to a double (not a float).
 
 declare x86_stdcallcc void @_D3nan5printFeZv(x86_fp80 %f)
 
 define i32 @main() {
+; CHECK-LABEL: main:
+; CHECK:       ## %bb.0: ## %entry_nan.main
+; CHECK-NEXT:    subl $28, %esp
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    fldl LCPI0_0
+; CHECK-NEXT:    fstpt (%esp)
+; CHECK-NEXT:    calll __D3nan5printFeZv
+; CHECK-NEXT:    subl $16, %esp
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    addl $28, %esp
+; CHECK-NEXT:    retl
 entry_nan.main:
   call x86_stdcallcc void @_D3nan5printFeZv(x86_fp80 0xK7FFFC001234000000800)
   ret i32 0
diff --git a/llvm/test/CodeGen/X86/2008-10-06-x87ld-nan-2.ll b/llvm/test/CodeGen/X86/2008-10-06-x87ld-nan-2.ll
index e659eea4c..108d8800a 100644
--- a/llvm/test/CodeGen/X86/2008-10-06-x87ld-nan-2.ll
+++ b/llvm/test/CodeGen/X86/2008-10-06-x87ld-nan-2.ll
@@ -1,7 +1,8 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; ModuleID = 'nan.bc'
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-f80:32:32-v64:64:64-v128:128:128-a0:0:64"
 target triple = "i686-apple-darwin8"
-; RUN: llc < %s -mattr=-sse2,-sse3,-sse | grep fldt | count 3
+; RUN: llc < %s -mattr=-sse2,-sse3,-sse | FileCheck %s
 ; it is not safe to shorten any of these NaNs.
 
 declare x86_stdcallcc void @_D3nan5printFeZv(x86_fp80 %f)
@@ -9,6 +10,25 @@ declare x86_stdcallcc void @_D3nan5printFeZv(x86_fp80 %f)
 @_D3nan4rvale = global x86_fp80 0xK7FFF8001234000000000   ; <x86_fp80*> [#uses=1]
 
 define i32 @main() {
+; CHECK-LABEL: main:
+; CHECK:       ## %bb.0: ## %entry_nan.main
+; CHECK-NEXT:    subl $28, %esp
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    fldt __D3nan4rvale
+; CHECK-NEXT:    fstpt (%esp)
+; CHECK-NEXT:    calll __D3nan5printFeZv
+; CHECK-NEXT:    subl $16, %esp
+; CHECK-NEXT:    fldt LCPI0_0
+; CHECK-NEXT:    fstpt (%esp)
+; CHECK-NEXT:    calll __D3nan5printFeZv
+; CHECK-NEXT:    subl $16, %esp
+; CHECK-NEXT:    fldt LCPI0_1
+; CHECK-NEXT:    fstpt (%esp)
+; CHECK-NEXT:    calll __D3nan5printFeZv
+; CHECK-NEXT:    subl $16, %esp
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    addl $28, %esp
+; CHECK-NEXT:    retl
 entry_nan.main:
   %tmp = load x86_fp80, x86_fp80* @_D3nan4rvale   ; <x86_fp80> [#uses=1]
   call x86_stdcallcc void @_D3nan5printFeZv(x86_fp80 %tmp)
diff --git a/llvm/test/CodeGen/X86/2008-10-24-FlippedCompare.ll b/llvm/test/CodeGen/X86/2008-10-24-FlippedCompare.ll
index 6c29bfaf3..44ea429bb 100644
--- a/llvm/test/CodeGen/X86/2008-10-24-FlippedCompare.ll
+++ b/llvm/test/CodeGen/X86/2008-10-24-FlippedCompare.ll
@@ -1,6 +1,18 @@
-; RUN: llc < %s -mtriple=i686-- -mattr=+sse2 -o - | not grep "ucomiss[^,]*esp"
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- -mattr=+sse2 | FileCheck %s
 
 define void @f(float %wt) {
+; CHECK-LABEL: f:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushl $44
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 4
+; CHECK-NEXT:    calll g
+; CHECK-NEXT:    addl $4, %esp
+; CHECK-NEXT:    .cfi_adjust_cfa_offset -4
+; CHECK-NEXT:    xorps %xmm0, %xmm0
+; CHECK-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; CHECK-NEXT:    ucomiss %xmm0, %xmm1
+; CHECK-NEXT:    retl
 entry:
 	%0 = fcmp ogt float %wt, 0.000000e+00		; <i1> [#uses=1]
 	%1 = tail call i32 @g(i32 44)		; <i32> [#uses=3]
diff --git a/llvm/test/CodeGen/X86/2008-11-29-ULT-Sign.ll b/llvm/test/CodeGen/X86/2008-11-29-ULT-Sign.ll
index 03442d631..6944f9b09 100644
--- a/llvm/test/CodeGen/X86/2008-11-29-ULT-Sign.ll
+++ b/llvm/test/CodeGen/X86/2008-11-29-ULT-Sign.ll
@@ -1,8 +1,19 @@
-; RUN:  llc < %s -mtriple=i686-pc-linux-gnu | grep "jns" | count 1
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN:  llc < %s -mtriple=i686-pc-linux-gnu | FileCheck %s
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32"
 target triple = "i686-pc-linux-gnu"
 
 define i32 @a(i32 %x) nounwind {
+; CHECK-LABEL: a:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    subl $12, %esp
+; CHECK-NEXT:    cmpl $0, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    jns .LBB0_2
+; CHECK-NEXT:  # %bb.1: # %if.then
+; CHECK-NEXT:    calll b
+; CHECK-NEXT:  .LBB0_2: # %if.end
+; CHECK-NEXT:    addl $12, %esp
+; CHECK-NEXT:    retl
 entry:
 	%cmp = icmp ult i32 %x, -2147483648		; <i1> [#uses=1]
 	br i1 %cmp, label %if.end, label %if.then
diff --git a/llvm/test/CodeGen/X86/2008-12-01-loop-iv-used-outside-loop.ll b/llvm/test/CodeGen/X86/2008-12-01-loop-iv-used-outside-loop.ll
index 840b8ba0f..42c7a19b1 100644
--- a/llvm/test/CodeGen/X86/2008-12-01-loop-iv-used-outside-loop.ll
+++ b/llvm/test/CodeGen/X86/2008-12-01-loop-iv-used-outside-loop.ll
@@ -1,4 +1,5 @@
-; RUN: llc < %s -mtriple=i386-apple-darwin | not grep lea
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i386-apple-darwin |  FileCheck %s
 ; The inner loop should use [reg] addressing, not [reg+reg] addressing.
 ; rdar://6403965
 
@@ -6,6 +7,26 @@ target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f3
 target triple = "i386-apple-darwin9.5"
 
 define i8* @test(i8* %Q, i32* %L) nounwind {
+; CHECK-LABEL: test:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    jmp LBB0_2
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  LBB0_1: ## %bb
+; CHECK-NEXT:    ## in Loop: Header=BB0_2 Depth=1
+; CHECK-NEXT:    incl %eax
+; CHECK-NEXT:  LBB0_2: ## %bb1
+; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movzbl (%eax), %ecx
+; CHECK-NEXT:    cmpb $12, %cl
+; CHECK-NEXT:    je LBB0_1
+; CHECK-NEXT:  ## %bb.3: ## %bb1
+; CHECK-NEXT:    ## in Loop: Header=BB0_2 Depth=1
+; CHECK-NEXT:    cmpb $42, %cl
+; CHECK-NEXT:    je LBB0_1
+; CHECK-NEXT:  ## %bb.4: ## %bb3
+; CHECK-NEXT:    movb $4, 2(%eax)
+; CHECK-NEXT:    retl
 entry:
 	br label %bb1
 
diff --git a/llvm/test/CodeGen/X86/2008-12-02-dagcombine-1.ll b/llvm/test/CodeGen/X86/2008-12-02-dagcombine-1.ll
index 3c7227c87..a7644e6a0 100644
--- a/llvm/test/CodeGen/X86/2008-12-02-dagcombine-1.ll
+++ b/llvm/test/CodeGen/X86/2008-12-02-dagcombine-1.ll
@@ -1,8 +1,16 @@
-; RUN: llc < %s | grep "(%esp)" | count 2
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s | FileCheck %s
+
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
 target triple = "i386-apple-darwin9.5"
 ; a - a should be found and removed, leaving refs to only L and P
 define i8* @test(i8* %a, i8* %L, i8* %P) nounwind {
+; CHECK-LABEL: test:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    subl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    addl $-2, %eax
+; CHECK-NEXT:    retl
 entry:
         %0 = ptrtoint i8* %a to i32
         %1 = sub i32 -2, %0
diff --git a/llvm/test/CodeGen/X86/2008-12-02-dagcombine-2.ll b/llvm/test/CodeGen/X86/2008-12-02-dagcombine-2.ll
index 42846f05e..cbab2f6c9 100644
--- a/llvm/test/CodeGen/X86/2008-12-02-dagcombine-2.ll
+++ b/llvm/test/CodeGen/X86/2008-12-02-dagcombine-2.ll
@@ -1,8 +1,14 @@
-; RUN: llc < %s | grep "(%esp)" | count 2
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s | FileCheck %s
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
 target triple = "i386-apple-darwin9.5"
 ; a - a should be found and removed, leaving refs to only L and P
 define i8* @test(i8* %a, i8* %L, i8* %P) nounwind {
+; CHECK-LABEL: test:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    subl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    retl
 entry:
         %0 = ptrtoint i8* %a to i32
         %1 = ptrtoint i8* %P to i32
diff --git a/llvm/test/CodeGen/X86/2008-12-02-dagcombine-3.ll b/llvm/test/CodeGen/X86/2008-12-02-dagcombine-3.ll
index 013732315..9bdfd5739 100644
--- a/llvm/test/CodeGen/X86/2008-12-02-dagcombine-3.ll
+++ b/llvm/test/CodeGen/X86/2008-12-02-dagcombine-3.ll
@@ -1,9 +1,18 @@
-; RUN: llc < %s | grep add | count 2
-; RUN: llc < %s | grep sub | grep -v subsections | count 1
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s | FileCheck %s
+
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
 target triple = "i386-apple-darwin9.5"
 ; this should be rearranged to have two +s and one -
 define i32 @test(i8* %a, i8* %L, i8* %P) nounwind {
+; CHECK-LABEL: test:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT:    addl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT:    subl %ecx, %eax
+; CHECK-NEXT:    addl $-2, %eax
+; CHECK-NEXT:    retl
 entry:
         %0 = ptrtoint i8* %P to i32
         %1 = sub i32 -2, %0
diff --git a/llvm/test/CodeGen/X86/2008-12-16-dagcombine-4.ll b/llvm/test/CodeGen/X86/2008-12-16-dagcombine-4.ll
index 76141825d..28b064d33 100644
--- a/llvm/test/CodeGen/X86/2008-12-16-dagcombine-4.ll
+++ b/llvm/test/CodeGen/X86/2008-12-16-dagcombine-4.ll
@@ -1,8 +1,14 @@
-; RUN: llc < %s | grep "(%esp)" | count 2
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s | FileCheck %s
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
 target triple = "i386-apple-darwin9.5"
 ; a - a should be found and removed, leaving refs to only L and P
 define i32 @test(i32 %a, i32 %L, i32 %P) nounwind {
+; CHECK-LABEL: test:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    subl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    retl
 entry:
         %0 = sub i32 %a, %L
         %1 = add i32 %P, %0
diff --git a/llvm/test/CodeGen/X86/2008-12-22-dagcombine-5.ll b/llvm/test/CodeGen/X86/2008-12-22-dagcombine-5.ll
index 884afae92..ef0cd25cc 100644
--- a/llvm/test/CodeGen/X86/2008-12-22-dagcombine-5.ll
+++ b/llvm/test/CodeGen/X86/2008-12-22-dagcombine-5.ll
@@ -1,8 +1,14 @@
-; RUN: llc < %s | grep "(%esp)" | count 2
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s | FileCheck %s
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
 target triple = "i386-apple-darwin9.5"
 ; -(-a) - a should be found and removed, leaving refs to only L and P
 define i32 @test(i32 %a, i32 %L, i32 %P) nounwind {
+; CHECK-LABEL: test:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    subl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    retl
 entry:
         %0 = sub i32 %L, %a
         %1 = sub i32 %P, %0
diff --git a/llvm/test/CodeGen/X86/2008-12-23-crazy-address.ll b/llvm/test/CodeGen/X86/2008-12-23-crazy-address.ll
index 902b3afcf..de13fb5ea 100644
--- a/llvm/test/CodeGen/X86/2008-12-23-crazy-address.ll
+++ b/llvm/test/CodeGen/X86/2008-12-23-crazy-address.ll
@@ -1,8 +1,22 @@
-; RUN: llc < %s -mtriple=i686-- -relocation-model=static | grep "lea.*X.*esp" | count 2
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- -relocation-model=static | FileCheck %s
 
 @X = external global [0 x i32]
 
 define void @foo() nounwind {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushl %eax
+; CHECK-NEXT:    movl %esp, %eax
+; CHECK-NEXT:    pushl %eax
+; CHECK-NEXT:    calll frob
+; CHECK-NEXT:    addl $4, %esp
+; CHECK-NEXT:    leal X(%esp), %eax
+; CHECK-NEXT:    pushl %eax
+; CHECK-NEXT:    calll borf
+; CHECK-NEXT:    addl $4, %esp
+; CHECK-NEXT:    popl %eax
+; CHECK-NEXT:    retl
 entry:
 	%Y = alloca i32
 	call void @frob(i32* %Y) nounwind
@@ -14,6 +28,21 @@ entry:
 }
 
 define void @bar(i32 %i) nounwind {
+; CHECK-LABEL: bar:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushl %esi
+; CHECK-NEXT:    subl $40, %esp
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; CHECK-NEXT:    movl %esp, %eax
+; CHECK-NEXT:    pushl %eax
+; CHECK-NEXT:    calll frob
+; CHECK-NEXT:    addl $4, %esp
+; CHECK-NEXT:    leal X(%esp,%esi,4), %eax
+; CHECK-NEXT:    pushl %eax
+; CHECK-NEXT:    calll borf
+; CHECK-NEXT:    addl $44, %esp
+; CHECK-NEXT:    popl %esi
+; CHECK-NEXT:    retl
 entry:
 	%Y = alloca [10 x i32]
 	%0 = getelementptr [10 x i32], [10 x i32]* %Y, i32 0, i32 0
diff --git a/llvm/test/CodeGen/X86/2008-12-23-dagcombine-6.ll b/llvm/test/CodeGen/X86/2008-12-23-dagcombine-6.ll
index 1f588934b..250415522 100644
--- a/llvm/test/CodeGen/X86/2008-12-23-dagcombine-6.ll
+++ b/llvm/test/CodeGen/X86/2008-12-23-dagcombine-6.ll
@@ -1,8 +1,14 @@
-; RUN: llc < %s | grep "(%esp)" | count 4
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s | FileCheck %s
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
 target triple = "i386-apple-darwin9.5"
 ; a - a should be found and removed, leaving refs to only L and P
 define i32 @test(i32 %a, i32 %L, i32 %P) nounwind {
+; CHECK-LABEL: test:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    addl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    retl
 entry:
         %0 = add i32 %a, %L
         %1 = add i32 %P, %0
@@ -13,6 +19,11 @@ return:		; preds = %bb3
 	ret i32 %2
 }
 define i32 @test2(i32 %a, i32 %L, i32 %P) nounwind {
+; CHECK-LABEL: test2:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    addl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    retl
 entry:
         %0 = add i32 %L, %a
         %1 = add i32 %P, %0
diff --git a/llvm/test/CodeGen/X86/2009-01-31-BigShift.ll b/llvm/test/CodeGen/X86/2009-01-31-BigShift.ll
index 158839743..a45751f1e 100644
--- a/llvm/test/CodeGen/X86/2009-01-31-BigShift.ll
+++ b/llvm/test/CodeGen/X86/2009-01-31-BigShift.ll
@@ -1,7 +1,22 @@
-; RUN: llc < %s -mtriple=i686-- | not grep and
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- | FileCheck %s
 ; PR3401
 
 define void @x(i288 %i) nounwind {
+; CHECK-LABEL: x:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushl {{[0-9]+}}(%esp)
+; CHECK-NEXT:    pushl {{[0-9]+}}(%esp)
+; CHECK-NEXT:    pushl {{[0-9]+}}(%esp)
+; CHECK-NEXT:    pushl {{[0-9]+}}(%esp)
+; CHECK-NEXT:    pushl {{[0-9]+}}(%esp)
+; CHECK-NEXT:    pushl {{[0-9]+}}(%esp)
+; CHECK-NEXT:    pushl {{[0-9]+}}(%esp)
+; CHECK-NEXT:    pushl {{[0-9]+}}(%esp)
+; CHECK-NEXT:    pushl {{[0-9]+}}(%esp)
+; CHECK-NEXT:    calll add
+; CHECK-NEXT:    addl $36, %esp
+; CHECK-NEXT:    retl
 	call void @add(i288 %i)
 	ret void
 }
diff --git a/llvm/test/CodeGen/X86/2009-01-31-BigShift2.ll b/llvm/test/CodeGen/X86/2009-01-31-BigShift2.ll
index a6209280e..f81b71114 100644
--- a/llvm/test/CodeGen/X86/2009-01-31-BigShift2.ll
+++ b/llvm/test/CodeGen/X86/2009-01-31-BigShift2.ll
@@ -1,7 +1,17 @@
-; RUN: llc < %s -mtriple=i686-- | grep "mov.*56"
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- | FileCheck %s
 ; PR3449
 
 define void @test(<8 x double>* %P, i64* %Q) nounwind {
+; CHECK-LABEL: test:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT:    movl 56(%ecx), %edx
+; CHECK-NEXT:    movl 60(%ecx), %ecx
+; CHECK-NEXT:    movl %ecx, 4(%eax)
+; CHECK-NEXT:    movl %edx, (%eax)
+; CHECK-NEXT:    retl
 	%A = load <8 x double>, <8 x double>* %P		; <<8 x double>> [#uses=1]
 	%B = bitcast <8 x double> %A to i512		; <i512> [#uses=1]
 	%C = lshr i512 %B, 448		; <i512> [#uses=1]
diff --git a/llvm/test/CodeGen/X86/2009-02-21-ExtWeakInitializer.ll b/llvm/test/CodeGen/X86/2009-02-21-ExtWeakInitializer.ll
index b3dd13c50..4dd6f1956 100644
--- a/llvm/test/CodeGen/X86/2009-02-21-ExtWeakInitializer.ll
+++ b/llvm/test/CodeGen/X86/2009-02-21-ExtWeakInitializer.ll
@@ -1,6 +1,9 @@
-; RUN: llc < %s | grep weak | count 3
+; RUN: llc < %s | FileCheck %s
 ; PR3629
 
+; CHECK: weak
+; CHECK-NEXT: weak
+; CHECK-NEXT: weak
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
 target triple = "x86_64-unknown-freebsd7.1"
 module asm ".ident\09\22$FreeBSD$\22"
diff --git a/llvm/test/CodeGen/X86/2009-02-26-MachineLICMBug.ll b/llvm/test/CodeGen/X86/2009-02-26-MachineLICMBug.ll
index bfa40738f..f4b1ca511 100644
--- a/llvm/test/CodeGen/X86/2009-02-26-MachineLICMBug.ll
+++ b/llvm/test/CodeGen/X86/2009-02-26-MachineLICMBug.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; REQUIRES: asserts
-; RUN: llc < %s -mattr=+sse3,+sse4.1 -mcpu=penryn -stats 2>&1 | grep "6 machinelicm"
+; RUN: llc < %s -mattr=+sse3,+sse4.1 -mcpu=penryn -stats 2>&1 | grep "5 machinelicm"
 ; RUN: llc < %s -mattr=+sse3,+sse4.1 -mcpu=penryn | FileCheck %s
 ; rdar://6627786
 ; rdar://7792037
@@ -33,7 +33,7 @@ define %struct.__vv* @t(%struct.Key* %desc, i64 %p) nounwind ssp {
 ; CHECK-NEXT:    jne LBB0_1
 ; CHECK-NEXT:  ## %bb.2: ## %bb26
 ; CHECK-NEXT:    ## in Loop: Header=BB0_1 Depth=1
-; CHECK-NEXT:    cmpq $1048576, %r14 ## imm = 0x100000
+; CHECK-NEXT:    cmpl $1048576, %r14d ## imm = 0x100000
 ; CHECK-NEXT:    jne LBB0_1
 ; CHECK-NEXT:  ## %bb.3: ## %bb.i
 ; CHECK-NEXT:    ## in Loop: Header=BB0_1 Depth=1
diff --git a/llvm/test/CodeGen/X86/2009-03-23-MultiUseSched.ll b/llvm/test/CodeGen/X86/2009-03-23-MultiUseSched.ll
index 276d52366..28564ce2d 100644
--- a/llvm/test/CodeGen/X86/2009-03-23-MultiUseSched.ll
+++ b/llvm/test/CodeGen/X86/2009-03-23-MultiUseSched.ll
@@ -1,8 +1,6 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; REQUIRES: asserts
-; RUN: llc < %s -mtriple=x86_64-linux -mcpu=corei7 -relocation-model=static -o /dev/null -stats -info-output-file - > %t
-; RUN: not grep spill %t
-; RUN: not grep "%rsp" %t
-; RUN: not grep "%rbp" %t
+; RUN: llc < %s -mtriple=x86_64-linux -mcpu=corei7 -relocation-model=static | FileCheck %s
 
 ; The register-pressure scheduler should be able to schedule this in a
 ; way that does not require spills.
@@ -10,6 +8,229 @@
 @X = external global i64		; <i64*> [#uses=25]
 
 define fastcc i64 @foo() nounwind {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    movq {{.*}}(%rip), %r8
+; CHECK-NEXT:    movq {{.*}}(%rip), %r10
+; CHECK-NEXT:    movq {{.*}}(%rip), %r9
+; CHECK-NEXT:    movq {{.*}}(%rip), %r12
+; CHECK-NEXT:    movq {{.*}}(%rip), %r15
+; CHECK-NEXT:    movq {{.*}}(%rip), %r14
+; CHECK-NEXT:    movq {{.*}}(%rip), %r11
+; CHECK-NEXT:    movq {{.*}}(%rip), %rdx
+; CHECK-NEXT:    addq %r15, %rdx
+; CHECK-NEXT:    movq {{.*}}(%rip), %rsi
+; CHECK-NEXT:    bswapq %rsi
+; CHECK-NEXT:    leaq (%r11,%r14), %rbx
+; CHECK-NEXT:    addq %r15, %rbx
+; CHECK-NEXT:    addq %rdx, %rbx
+; CHECK-NEXT:    addq %rsi, %rbx
+; CHECK-NEXT:    leaq (%r9,%r10), %rsi
+; CHECK-NEXT:    leaq (%rsi,%r8), %rdx
+; CHECK-NEXT:    addq %rsi, %rdx
+; CHECK-NEXT:    movq {{.*}}(%rip), %rdi
+; CHECK-NEXT:    addq %rbx, %r12
+; CHECK-NEXT:    addq %r8, %rdx
+; CHECK-NEXT:    bswapq %rdi
+; CHECK-NEXT:    addq %rbx, %rdx
+; CHECK-NEXT:    leaq (%r15,%r14), %rsi
+; CHECK-NEXT:    addq %r12, %rsi
+; CHECK-NEXT:    addq %r11, %rdi
+; CHECK-NEXT:    addq %rsi, %rdi
+; CHECK-NEXT:    leaq (%r10,%r8), %rbx
+; CHECK-NEXT:    leaq (%rdx,%rbx), %rsi
+; CHECK-NEXT:    addq %rbx, %rsi
+; CHECK-NEXT:    movq {{.*}}(%rip), %rbx
+; CHECK-NEXT:    addq %r12, %rdi
+; CHECK-NEXT:    addq %rdi, %r9
+; CHECK-NEXT:    addq %rdx, %rsi
+; CHECK-NEXT:    addq %rdi, %rsi
+; CHECK-NEXT:    bswapq %rbx
+; CHECK-NEXT:    leaq (%r12,%r15), %rdi
+; CHECK-NEXT:    addq %r9, %rdi
+; CHECK-NEXT:    addq %r14, %rbx
+; CHECK-NEXT:    addq %rdi, %rbx
+; CHECK-NEXT:    leaq (%rdx,%r8), %rax
+; CHECK-NEXT:    leaq (%rsi,%rax), %rdi
+; CHECK-NEXT:    addq %rax, %rdi
+; CHECK-NEXT:    movq {{.*}}(%rip), %rcx
+; CHECK-NEXT:    addq %r9, %rbx
+; CHECK-NEXT:    addq %rbx, %r10
+; CHECK-NEXT:    addq %rsi, %rdi
+; CHECK-NEXT:    bswapq %rcx
+; CHECK-NEXT:    addq %rbx, %rdi
+; CHECK-NEXT:    leaq (%r9,%r12), %rax
+; CHECK-NEXT:    addq %r10, %rax
+; CHECK-NEXT:    addq %r15, %rcx
+; CHECK-NEXT:    addq %rax, %rcx
+; CHECK-NEXT:    leaq (%rsi,%rdx), %rbx
+; CHECK-NEXT:    leaq (%rdi,%rbx), %r11
+; CHECK-NEXT:    addq %rbx, %r11
+; CHECK-NEXT:    movq {{.*}}(%rip), %rbx
+; CHECK-NEXT:    addq %r10, %rcx
+; CHECK-NEXT:    addq %rcx, %r8
+; CHECK-NEXT:    addq %rdi, %r11
+; CHECK-NEXT:    addq %rcx, %r11
+; CHECK-NEXT:    bswapq %rbx
+; CHECK-NEXT:    leaq (%r10,%r9), %rcx
+; CHECK-NEXT:    addq %r8, %rcx
+; CHECK-NEXT:    addq %r12, %rbx
+; CHECK-NEXT:    addq %rcx, %rbx
+; CHECK-NEXT:    leaq (%rdi,%rsi), %rax
+; CHECK-NEXT:    leaq (%r11,%rax), %r14
+; CHECK-NEXT:    addq %rax, %r14
+; CHECK-NEXT:    movq {{.*}}(%rip), %rax
+; CHECK-NEXT:    addq %r8, %rbx
+; CHECK-NEXT:    addq %rbx, %rdx
+; CHECK-NEXT:    addq %r11, %r14
+; CHECK-NEXT:    bswapq %rax
+; CHECK-NEXT:    addq %rbx, %r14
+; CHECK-NEXT:    leaq (%r8,%r10), %rbx
+; CHECK-NEXT:    addq %rdx, %rbx
+; CHECK-NEXT:    addq %r9, %rax
+; CHECK-NEXT:    addq %rbx, %rax
+; CHECK-NEXT:    leaq (%r11,%rdi), %rbx
+; CHECK-NEXT:    leaq (%r14,%rbx), %r9
+; CHECK-NEXT:    addq %rbx, %r9
+; CHECK-NEXT:    movq {{.*}}(%rip), %rbx
+; CHECK-NEXT:    addq %rdx, %rax
+; CHECK-NEXT:    addq %rax, %rsi
+; CHECK-NEXT:    addq %r14, %r9
+; CHECK-NEXT:    addq %rax, %r9
+; CHECK-NEXT:    bswapq %rbx
+; CHECK-NEXT:    leaq (%rdx,%r8), %rax
+; CHECK-NEXT:    addq %rsi, %rax
+; CHECK-NEXT:    addq %r10, %rbx
+; CHECK-NEXT:    addq %rax, %rbx
+; CHECK-NEXT:    leaq (%r14,%r11), %rax
+; CHECK-NEXT:    leaq (%r9,%rax), %r10
+; CHECK-NEXT:    addq %rax, %r10
+; CHECK-NEXT:    movq {{.*}}(%rip), %rax
+; CHECK-NEXT:    addq %rsi, %rbx
+; CHECK-NEXT:    addq %rbx, %rdi
+; CHECK-NEXT:    addq %r9, %r10
+; CHECK-NEXT:    bswapq %rax
+; CHECK-NEXT:    addq %rbx, %r10
+; CHECK-NEXT:    leaq (%rsi,%rdx), %rbx
+; CHECK-NEXT:    addq %rdi, %rbx
+; CHECK-NEXT:    addq %r8, %rax
+; CHECK-NEXT:    addq %rbx, %rax
+; CHECK-NEXT:    leaq (%r9,%r14), %rbx
+; CHECK-NEXT:    leaq (%r10,%rbx), %r8
+; CHECK-NEXT:    addq %rbx, %r8
+; CHECK-NEXT:    movq {{.*}}(%rip), %rbx
+; CHECK-NEXT:    addq %rdi, %rax
+; CHECK-NEXT:    addq %rax, %r11
+; CHECK-NEXT:    addq %r10, %r8
+; CHECK-NEXT:    addq %rax, %r8
+; CHECK-NEXT:    bswapq %rbx
+; CHECK-NEXT:    leaq (%rdi,%rsi), %rax
+; CHECK-NEXT:    addq %r11, %rax
+; CHECK-NEXT:    addq %rdx, %rbx
+; CHECK-NEXT:    addq %rax, %rbx
+; CHECK-NEXT:    leaq (%r10,%r9), %rax
+; CHECK-NEXT:    leaq (%r8,%rax), %r15
+; CHECK-NEXT:    addq %rax, %r15
+; CHECK-NEXT:    movq {{.*}}(%rip), %rax
+; CHECK-NEXT:    addq %r11, %rbx
+; CHECK-NEXT:    addq %rbx, %r14
+; CHECK-NEXT:    addq %r8, %r15
+; CHECK-NEXT:    bswapq %rax
+; CHECK-NEXT:    addq %rbx, %r15
+; CHECK-NEXT:    leaq (%r11,%rdi), %rbx
+; CHECK-NEXT:    addq %r14, %rbx
+; CHECK-NEXT:    addq %rsi, %rax
+; CHECK-NEXT:    addq %rbx, %rax
+; CHECK-NEXT:    leaq (%r8,%r10), %rbx
+; CHECK-NEXT:    leaq (%r15,%rbx), %rsi
+; CHECK-NEXT:    addq %rbx, %rsi
+; CHECK-NEXT:    movq {{.*}}(%rip), %rbx
+; CHECK-NEXT:    addq %r14, %rax
+; CHECK-NEXT:    addq %rax, %r9
+; CHECK-NEXT:    addq %r15, %rsi
+; CHECK-NEXT:    addq %rax, %rsi
+; CHECK-NEXT:    bswapq %rbx
+; CHECK-NEXT:    leaq (%r14,%r11), %rax
+; CHECK-NEXT:    addq %r9, %rax
+; CHECK-NEXT:    addq %rdi, %rbx
+; CHECK-NEXT:    addq %rax, %rbx
+; CHECK-NEXT:    leaq (%r15,%r8), %rax
+; CHECK-NEXT:    leaq (%rsi,%rax), %r12
+; CHECK-NEXT:    addq %rax, %r12
+; CHECK-NEXT:    movq {{.*}}(%rip), %rcx
+; CHECK-NEXT:    addq %r9, %rbx
+; CHECK-NEXT:    addq %rbx, %r10
+; CHECK-NEXT:    addq %rsi, %r12
+; CHECK-NEXT:    bswapq %rcx
+; CHECK-NEXT:    addq %rbx, %r12
+; CHECK-NEXT:    leaq (%r9,%r14), %rax
+; CHECK-NEXT:    addq %r10, %rax
+; CHECK-NEXT:    addq %r11, %rcx
+; CHECK-NEXT:    addq %rax, %rcx
+; CHECK-NEXT:    leaq (%rsi,%r15), %rbx
+; CHECK-NEXT:    leaq (%r12,%rbx), %rax
+; CHECK-NEXT:    addq %rbx, %rax
+; CHECK-NEXT:    movq {{.*}}(%rip), %rbx
+; CHECK-NEXT:    addq %r10, %rcx
+; CHECK-NEXT:    addq %rcx, %r8
+; CHECK-NEXT:    addq %r12, %rax
+; CHECK-NEXT:    addq %rcx, %rax
+; CHECK-NEXT:    bswapq %rbx
+; CHECK-NEXT:    leaq (%r10,%r9), %rcx
+; CHECK-NEXT:    addq %r8, %rcx
+; CHECK-NEXT:    addq %r14, %rbx
+; CHECK-NEXT:    addq %rcx, %rbx
+; CHECK-NEXT:    leaq (%r12,%rsi), %rdx
+; CHECK-NEXT:    leaq (%rax,%rdx), %rcx
+; CHECK-NEXT:    addq %rdx, %rcx
+; CHECK-NEXT:    movq {{.*}}(%rip), %rdx
+; CHECK-NEXT:    addq %r8, %rbx
+; CHECK-NEXT:    addq %rbx, %r15
+; CHECK-NEXT:    addq %rax, %rcx
+; CHECK-NEXT:    bswapq %rdx
+; CHECK-NEXT:    addq %rbx, %rcx
+; CHECK-NEXT:    leaq (%r8,%r10), %rbx
+; CHECK-NEXT:    addq %r15, %rbx
+; CHECK-NEXT:    addq %r9, %rdx
+; CHECK-NEXT:    addq %rbx, %rdx
+; CHECK-NEXT:    leaq (%rax,%r12), %r9
+; CHECK-NEXT:    leaq (%rcx,%r9), %rbx
+; CHECK-NEXT:    addq %r9, %rbx
+; CHECK-NEXT:    addq %r15, %rdx
+; CHECK-NEXT:    addq %rdx, %rsi
+; CHECK-NEXT:    addq %rcx, %rbx
+; CHECK-NEXT:    addq %rdx, %rbx
+; CHECK-NEXT:    movq {{.*}}(%rip), %rdx
+; CHECK-NEXT:    bswapq %rdx
+; CHECK-NEXT:    addq %r10, %rdx
+; CHECK-NEXT:    leaq (%r15,%r8), %rdi
+; CHECK-NEXT:    addq %rsi, %rdi
+; CHECK-NEXT:    addq %rdi, %rdx
+; CHECK-NEXT:    addq %rax, %rcx
+; CHECK-NEXT:    leaq (%rbx,%rcx), %rdi
+; CHECK-NEXT:    addq %rcx, %rdi
+; CHECK-NEXT:    addq %rbx, %rdi
+; CHECK-NEXT:    addq %rsi, %rdx
+; CHECK-NEXT:    addq %rdx, %r12
+; CHECK-NEXT:    addq %rdx, %rdi
+; CHECK-NEXT:    addq %r15, %rsi
+; CHECK-NEXT:    movq {{.*}}(%rip), %rax
+; CHECK-NEXT:    bswapq %rax
+; CHECK-NEXT:    movq %rax, {{.*}}(%rip)
+; CHECK-NEXT:    addq %r8, %rax
+; CHECK-NEXT:    addq %r12, %rsi
+; CHECK-NEXT:    addq %rsi, %rax
+; CHECK-NEXT:    addq %r12, %rax
+; CHECK-NEXT:    addq %rdi, %rax
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    retq
 	%tmp = load volatile i64, i64* @X		; <i64> [#uses=7]
 	%tmp1 = load volatile i64, i64* @X		; <i64> [#uses=5]
 	%tmp2 = load volatile i64, i64* @X		; <i64> [#uses=3]
diff --git a/llvm/test/CodeGen/X86/2009-04-12-picrel.ll b/llvm/test/CodeGen/X86/2009-04-12-picrel.ll
index 697f14e22..319fc9303 100644
--- a/llvm/test/CodeGen/X86/2009-04-12-picrel.ll
+++ b/llvm/test/CodeGen/X86/2009-04-12-picrel.ll
@@ -1,10 +1,15 @@
-; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -relocation-model=static -code-model=small > %t
-; RUN: grep leaq %t | count 1
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -relocation-model=static -code-model=small | FileCheck %s
 
 @dst = external global [131072 x i32]
 @ptr = external global i32*
 
 define void @off01(i64 %i) nounwind {
+; CHECK-LABEL: off01:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    leaq dst+64(,%rdi,4), %rax
+; CHECK-NEXT:    movq %rax, {{.*}}(%rip)
+; CHECK-NEXT:    retq
 entry:
 	%.sum = add i64 %i, 16
 	%0 = getelementptr [131072 x i32], [131072 x i32]* @dst, i64 0, i64 %.sum
diff --git a/llvm/test/CodeGen/X86/2009-04-14-IllegalRegs.ll b/llvm/test/CodeGen/X86/2009-04-14-IllegalRegs.ll
index 8055ea880..48ad2a2c0 100644
--- a/llvm/test/CodeGen/X86/2009-04-14-IllegalRegs.ll
+++ b/llvm/test/CodeGen/X86/2009-04-14-IllegalRegs.ll
@@ -1,10 +1,63 @@
-; RUN: llc < %s -mtriple=i386-apple-darwin -O0 -regalloc=fast | not grep sil
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i386-apple-darwin -O0 -regalloc=fast | FileCheck %s
 ; rdar://6787136
 
 	%struct.X = type { i8, [32 x i8] }
 @llvm.used = appending global [1 x i8*] [i8* bitcast (i32 ()* @z to i8*)], section "llvm.metadata"		; <[1 x i8*]*> [#uses=0]
 
 define i32 @z() nounwind ssp {
+; CHECK-LABEL: z:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    pushl %ebx
+; CHECK-NEXT:    pushl %edi
+; CHECK-NEXT:    pushl %esi
+; CHECK-NEXT:    subl $144, %esp
+; CHECK-NEXT:    movl L___stack_chk_guard$non_lazy_ptr, %eax
+; CHECK-NEXT:    movl (%eax), %eax
+; CHECK-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movb $48, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT:    movb %cl, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movb $15, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movl %esp, %eax
+; CHECK-NEXT:    movl $8, %edx
+; CHECK-NEXT:    leal {{[0-9]+}}(%esp), %esi
+; CHECK-NEXT:    movl %edx, %ecx
+; CHECK-NEXT:    movl %eax, %edi
+; CHECK-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) ## 4-byte Spill
+; CHECK-NEXT:    rep;movsl (%esi), %es:(%edi)
+; CHECK-NEXT:    movl %eax, %ecx
+; CHECK-NEXT:    addl $36, %ecx
+; CHECK-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) ## 4-byte Spill
+; CHECK-NEXT:    movl %edx, %ecx
+; CHECK-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi ## 4-byte Reload
+; CHECK-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi ## 4-byte Reload
+; CHECK-NEXT:    rep;movsl (%esi), %es:(%edi)
+; CHECK-NEXT:    movb {{[0-9]+}}(%esp), %bl
+; CHECK-NEXT:    movb %bl, 32(%eax)
+; CHECK-NEXT:    movb %bl, 68(%eax)
+; CHECK-NEXT:    calll _f
+; CHECK-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; CHECK-NEXT:  ## %bb.1: ## %return
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    movl L___stack_chk_guard$non_lazy_ptr, %ecx
+; CHECK-NEXT:    movl (%ecx), %ecx
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; CHECK-NEXT:    cmpl %edx, %ecx
+; CHECK-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) ## 4-byte Spill
+; CHECK-NEXT:    jne LBB0_3
+; CHECK-NEXT:  ## %bb.2: ## %SP_return
+; CHECK-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax ## 4-byte Reload
+; CHECK-NEXT:    addl $144, %esp
+; CHECK-NEXT:    popl %esi
+; CHECK-NEXT:    popl %edi
+; CHECK-NEXT:    popl %ebx
+; CHECK-NEXT:    retl
+; CHECK-NEXT:  LBB0_3: ## %CallStackCheckFailBlk
+; CHECK-NEXT:    calll ___stack_chk_fail
+; CHECK-NEXT:    ud2
 entry:
 	%retval = alloca i32		; <i32*> [#uses=2]
 	%xxx = alloca %struct.X		; <%struct.X*> [#uses=6]
diff --git a/llvm/test/CodeGen/X86/2009-04-25-CoalescerBug.ll b/llvm/test/CodeGen/X86/2009-04-25-CoalescerBug.ll
index f6b5ecef6..38bc6c387 100644
--- a/llvm/test/CodeGen/X86/2009-04-25-CoalescerBug.ll
+++ b/llvm/test/CodeGen/X86/2009-04-25-CoalescerBug.ll
@@ -1,7 +1,19 @@
-; RUN: llc < %s -mtriple=x86_64-- | grep mov | count 1
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-- | FileCheck %s
 ; rdar://6806252
 
 define i64 @test(i32* %tmp13) nounwind {
+; CHECK-LABEL: test:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB0_1: # %while.cond
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movl (%rdi), %eax
+; CHECK-NEXT:    testb $1, %al
+; CHECK-NEXT:    jne .LBB0_1
+; CHECK-NEXT:  # %bb.2: # %while.end
+; CHECK-NEXT:    shrl %eax
+; CHECK-NEXT:    retq
 entry:
 	br label %while.cond
 
diff --git a/llvm/test/CodeGen/X86/2009-04-29-IndirectDestOperands.ll b/llvm/test/CodeGen/X86/2009-04-29-IndirectDestOperands.ll
index 5ddb5cae2..2615164c6 100644
--- a/llvm/test/CodeGen/X86/2009-04-29-IndirectDestOperands.ll
+++ b/llvm/test/CodeGen/X86/2009-04-29-IndirectDestOperands.ll
@@ -1,8 +1,28 @@
-; RUN: llc < %s | grep "movl.*%ebx, 8(%esi)"
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s | FileCheck %s
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
 target triple = "i386-apple-darwin9.0"
 
 define void @cpuid(i32* %data) nounwind {
+; CHECK-LABEL: cpuid:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    pushl %ebx
+; CHECK-NEXT:    pushl %esi
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; CHECK-NEXT:    movl 20(%esi), %eax
+; CHECK-NEXT:    movl 24(%esi), %ebx
+; CHECK-NEXT:    movl 28(%esi), %ecx
+; CHECK-NEXT:    movl 32(%esi), %edx
+; CHECK-NEXT:    ## InlineAsm Start
+; CHECK-NEXT:    cpuid
+; CHECK-NEXT:    ## InlineAsm End
+; CHECK-NEXT:    movl %ebx, 8(%esi)
+; CHECK-NEXT:    movl %ecx, 12(%esi)
+; CHECK-NEXT:    movl %edx, 16(%esi)
+; CHECK-NEXT:    movl %eax, 4(%esi)
+; CHECK-NEXT:    popl %esi
+; CHECK-NEXT:    popl %ebx
+; CHECK-NEXT:    retl
 entry:
 	%arrayidx = getelementptr i32, i32* %data, i32 1		; <i32*> [#uses=1]
 	%arrayidx2 = getelementptr i32, i32* %data, i32 2		; <i32*> [#uses=1]
diff --git a/llvm/test/CodeGen/X86/2009-05-30-ISelBug.ll b/llvm/test/CodeGen/X86/2009-05-30-ISelBug.ll
index 5aa8fe804..5c36163f0 100644
--- a/llvm/test/CodeGen/X86/2009-05-30-ISelBug.ll
+++ b/llvm/test/CodeGen/X86/2009-05-30-ISelBug.ll
@@ -1,6 +1,25 @@
-; RUN: llc < %s -mtriple=x86_64-- | not grep "movzbl	%[abcd]h,"
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-- | FileCheck %s
 
 define void @BZ2_bzDecompress_bb5_2E_outer_bb35_2E_i_bb54_2E_i(i32*, i32 %c_nblock_used.2.i, i32 %.reload51, i32* %.out, i32* %.out1, i32* %.out2, i32* %.out3) nounwind {
+; CHECK-LABEL: BZ2_bzDecompress_bb5_2E_outer_bb35_2E_i_bb54_2E_i:
+; CHECK:       # %bb.0: # %newFuncRoot
+; CHECK-NEXT:    movq {{[0-9]+}}(%rsp), %r10
+; CHECK-NEXT:    movl %edx, %edx
+; CHECK-NEXT:    movl (%rdi,%rdx,4), %edx
+; CHECK-NEXT:    movzbl %dl, %eax
+; CHECK-NEXT:    shrq $6, %rdx
+; CHECK-NEXT:    andl $67108860, %edx # imm = 0x3FFFFFC
+; CHECK-NEXT:    addl $4, %eax
+; CHECK-NEXT:    movl (%rdi,%rdx), %edx
+; CHECK-NEXT:    movzbl %dl, %edi
+; CHECK-NEXT:    shrl $8, %edx
+; CHECK-NEXT:    addl $5, %esi
+; CHECK-NEXT:    movl %eax, (%rcx)
+; CHECK-NEXT:    movl %edi, (%r8)
+; CHECK-NEXT:    movl %edx, (%r9)
+; CHECK-NEXT:    movl %esi, (%r10)
+; CHECK-NEXT:    retq
 newFuncRoot:
 	br label %bb54.i
 
diff --git a/llvm/test/CodeGen/X86/2009-06-05-VZextByteShort.ll b/llvm/test/CodeGen/X86/2009-06-05-VZextByteShort.ll
index be8563a9f..31efd4660 100644
--- a/llvm/test/CodeGen/X86/2009-06-05-VZextByteShort.ll
+++ b/llvm/test/CodeGen/X86/2009-06-05-VZextByteShort.ll
@@ -7,6 +7,7 @@ define <4 x i16> @a(i32* %x1) nounwind {
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; CHECK-NEXT:    movl (%eax), %eax
 ; CHECK-NEXT:    shrl %eax
+; CHECK-NEXT:    movzwl %ax, %eax
 ; CHECK-NEXT:    movd %eax, %xmm0
 ; CHECK-NEXT:    retl
 
@@ -40,7 +41,7 @@ define <8 x i8> @c(i32* %x1) nounwind {
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; CHECK-NEXT:    movl (%eax), %eax
 ; CHECK-NEXT:    shrl %eax
-; CHECK-NEXT:    movzwl %ax, %eax
+; CHECK-NEXT:    movzbl %al, %eax
 ; CHECK-NEXT:    movd %eax, %xmm0
 ; CHECK-NEXT:    retl
 
diff --git a/llvm/test/CodeGen/X86/2009-06-12-x86_64-tail-call-conv-out-of-sync-bug.ll b/llvm/test/CodeGen/X86/2009-06-12-x86_64-tail-call-conv-out-of-sync-bug.ll
index 5989e69bd..dd8efe971 100644
--- a/llvm/test/CodeGen/X86/2009-06-12-x86_64-tail-call-conv-out-of-sync-bug.ll
+++ b/llvm/test/CodeGen/X86/2009-06-12-x86_64-tail-call-conv-out-of-sync-bug.ll
@@ -1,13 +1,24 @@
-; RUN: llc < %s -tailcallopt -mattr=+sse2 -mtriple=x86_64-apple-darwin | grep fstpt
-; RUN: llc < %s -tailcallopt -mattr=+sse2 -mtriple=x86_64-apple-darwin | grep xmm
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -tailcallopt -mattr=+sse2 -mtriple=x86_64-apple-darwin | FileCheck %s
 
 ; Check that x86-64 tail calls support x86_fp80 and v2f32 types. (Tail call
 ; calling convention out of sync with standard c calling convention on x86_64)
 ; Bug 4278.
 
-declare fastcc double @tailcallee(x86_fp80, <2 x float>) 
-	
+declare fastcc double @tailcallee(x86_fp80, <2 x float>)
+
 define fastcc double @tailcall() {
+; CHECK-LABEL: tailcall:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    subq $40, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    movq {{[0-9]+}}(%rsp), %rax
+; CHECK-NEXT:    movq %rax, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    fld1
+; CHECK-NEXT:    fstpt {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movaps {{.*#+}} xmm0 = <1.0E+0,1.0E+0,u,u>
+; CHECK-NEXT:    addq $24, %rsp
+; CHECK-NEXT:    jmp _tailcallee ## TAILCALL
 entry:
   %tmp = fpext float 1.000000e+00 to x86_fp80
 	%tmp2 = tail call fastcc double @tailcallee( x86_fp80 %tmp,  <2 x float> <float 1.000000e+00, float 1.000000e+00>)
diff --git a/llvm/test/CodeGen/X86/2009-06-15-not-a-tail-call.ll b/llvm/test/CodeGen/X86/2009-06-15-not-a-tail-call.ll
index a927b87cb..f4b651365 100644
--- a/llvm/test/CodeGen/X86/2009-06-15-not-a-tail-call.ll
+++ b/llvm/test/CodeGen/X86/2009-06-15-not-a-tail-call.ll
@@ -1,10 +1,25 @@
-; RUN: llc < %s -mtriple=i686-- -tailcallopt | not grep TAILCALL
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- -tailcallopt | FileCheck %s
 
 ; Bug 4396. This tail call can NOT be optimized.
 
 declare fastcc i8* @_D3gcx2GC12mallocNoSyncMFmkZPv() nounwind
 
 define fastcc i8* @_D3gcx2GC12callocNoSyncMFmkZPv() nounwind {
+; CHECK-LABEL: _D3gcx2GC12callocNoSyncMFmkZPv:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushl %esi
+; CHECK-NEXT:    calll _D3gcx2GC12mallocNoSyncMFmkZPv
+; CHECK-NEXT:    movl %eax, %esi
+; CHECK-NEXT:    pushl $0
+; CHECK-NEXT:    pushl $2
+; CHECK-NEXT:    pushl $0
+; CHECK-NEXT:    pushl %eax
+; CHECK-NEXT:    calll memset
+; CHECK-NEXT:    addl $16, %esp
+; CHECK-NEXT:    movl %esi, %eax
+; CHECK-NEXT:    popl %esi
+; CHECK-NEXT:    retl
 entry:
 	%tmp6 = tail call fastcc i8* @_D3gcx2GC12mallocNoSyncMFmkZPv()		; <i8*> [#uses=2]
 	%tmp9 = tail call i8* @memset(i8* %tmp6, i32 0, i64 2)		; <i8*> [#uses=0]
diff --git a/llvm/test/CodeGen/X86/2009-08-12-badswitch.ll b/llvm/test/CodeGen/X86/2009-08-12-badswitch.ll
index a94fce04e..a74b81430 100644
--- a/llvm/test/CodeGen/X86/2009-08-12-badswitch.ll
+++ b/llvm/test/CodeGen/X86/2009-08-12-badswitch.ll
@@ -1,4 +1,5 @@
-; RUN: llc < %s | grep LJT
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
 target triple = "x86_64-apple-darwin10"
 
@@ -30,6 +31,153 @@ declare void @f25() nounwind readnone
 declare void @f26() nounwind readnone
 
 define internal fastcc i32 @foo(i64 %bar) nounwind ssp {
+; CHECK-LABEL: foo:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    addq $-2, %rdi
+; CHECK-NEXT:    cmpq $25, %rdi
+; CHECK-NEXT:    ja LBB0_2
+; CHECK-NEXT:  ## %bb.1: ## %bb49
+; CHECK-NEXT:    leaq {{.*}}(%rip), %rax
+; CHECK-NEXT:    movslq (%rax,%rdi,4), %rcx
+; CHECK-NEXT:    addq %rax, %rcx
+; CHECK-NEXT:    jmpq *%rcx
+; CHECK-NEXT:  LBB0_3: ## %RRETURN_6
+; CHECK-NEXT:    callq _f2
+; CHECK-NEXT:    jmp LBB0_28
+; CHECK-NEXT:  LBB0_2: ## %RETURN
+; CHECK-NEXT:    callq _f1
+; CHECK-NEXT:    jmp LBB0_28
+; CHECK-NEXT:  LBB0_4: ## %RRETURN_7
+; CHECK-NEXT:    callq _f3
+; CHECK-NEXT:    jmp LBB0_28
+; CHECK-NEXT:  LBB0_5: ## %RRETURN_14
+; CHECK-NEXT:    callq _f4
+; CHECK-NEXT:    jmp LBB0_28
+; CHECK-NEXT:  LBB0_6: ## %RRETURN_15
+; CHECK-NEXT:    callq _f5
+; CHECK-NEXT:    jmp LBB0_28
+; CHECK-NEXT:  LBB0_7: ## %RRETURN_16
+; CHECK-NEXT:    callq _f6
+; CHECK-NEXT:    jmp LBB0_28
+; CHECK-NEXT:  LBB0_8: ## %RRETURN_17
+; CHECK-NEXT:    callq _f7
+; CHECK-NEXT:    jmp LBB0_28
+; CHECK-NEXT:  LBB0_9: ## %RRETURN_18
+; CHECK-NEXT:    callq _f8
+; CHECK-NEXT:    jmp LBB0_28
+; CHECK-NEXT:  LBB0_10: ## %RRETURN_19
+; CHECK-NEXT:    callq _f9
+; CHECK-NEXT:    jmp LBB0_28
+; CHECK-NEXT:  LBB0_11: ## %RRETURN_20
+; CHECK-NEXT:    callq _f10
+; CHECK-NEXT:    jmp LBB0_28
+; CHECK-NEXT:  LBB0_12: ## %RRETURN_21
+; CHECK-NEXT:    callq _f11
+; CHECK-NEXT:    jmp LBB0_28
+; CHECK-NEXT:  LBB0_13: ## %RRETURN_22
+; CHECK-NEXT:    callq _f12
+; CHECK-NEXT:    jmp LBB0_28
+; CHECK-NEXT:  LBB0_14: ## %RRETURN_24
+; CHECK-NEXT:    callq _f13
+; CHECK-NEXT:    jmp LBB0_28
+; CHECK-NEXT:  LBB0_15: ## %RRETURN_26
+; CHECK-NEXT:    callq _f14
+; CHECK-NEXT:    jmp LBB0_28
+; CHECK-NEXT:  LBB0_16: ## %RRETURN_27
+; CHECK-NEXT:    callq _f15
+; CHECK-NEXT:    jmp LBB0_28
+; CHECK-NEXT:  LBB0_17: ## %RRETURN_28
+; CHECK-NEXT:    callq _f16
+; CHECK-NEXT:    jmp LBB0_28
+; CHECK-NEXT:  LBB0_18: ## %RRETURN_29
+; CHECK-NEXT:    callq _f17
+; CHECK-NEXT:    jmp LBB0_28
+; CHECK-NEXT:  LBB0_19: ## %RRETURN_30
+; CHECK-NEXT:    callq _f18
+; CHECK-NEXT:    jmp LBB0_28
+; CHECK-NEXT:  LBB0_20: ## %RRETURN_31
+; CHECK-NEXT:    callq _f19
+; CHECK-NEXT:    jmp LBB0_28
+; CHECK-NEXT:  LBB0_21: ## %RRETURN_38
+; CHECK-NEXT:    callq _f20
+; CHECK-NEXT:    jmp LBB0_28
+; CHECK-NEXT:  LBB0_22: ## %RRETURN_40
+; CHECK-NEXT:    callq _f21
+; CHECK-NEXT:    jmp LBB0_28
+; CHECK-NEXT:  LBB0_23: ## %RRETURN_42
+; CHECK-NEXT:    callq _f22
+; CHECK-NEXT:    jmp LBB0_28
+; CHECK-NEXT:  LBB0_24: ## %RRETURN_44
+; CHECK-NEXT:    callq _f23
+; CHECK-NEXT:    jmp LBB0_28
+; CHECK-NEXT:  LBB0_25: ## %RRETURN_48
+; CHECK-NEXT:    callq _f24
+; CHECK-NEXT:    jmp LBB0_28
+; CHECK-NEXT:  LBB0_26: ## %RRETURN_52
+; CHECK-NEXT:    callq _f25
+; CHECK-NEXT:    jmp LBB0_28
+; CHECK-NEXT:  LBB0_27: ## %RRETURN_1
+; CHECK-NEXT:    callq _f26
+; CHECK-NEXT:  LBB0_28: ## %EXIT
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    popq %rcx
+; CHECK-NEXT:    retq
+; CHECK-NEXT:    .p2align 2, 0x90
+; CHECK-NEXT:    .data_region jt32
+; CHECK-NEXT:  .set L0_0_set_3, LBB0_3-LJTI0_0
+; CHECK-NEXT:  .set L0_0_set_4, LBB0_4-LJTI0_0
+; CHECK-NEXT:  .set L0_0_set_5, LBB0_5-LJTI0_0
+; CHECK-NEXT:  .set L0_0_set_6, LBB0_6-LJTI0_0
+; CHECK-NEXT:  .set L0_0_set_7, LBB0_7-LJTI0_0
+; CHECK-NEXT:  .set L0_0_set_8, LBB0_8-LJTI0_0
+; CHECK-NEXT:  .set L0_0_set_9, LBB0_9-LJTI0_0
+; CHECK-NEXT:  .set L0_0_set_10, LBB0_10-LJTI0_0
+; CHECK-NEXT:  .set L0_0_set_11, LBB0_11-LJTI0_0
+; CHECK-NEXT:  .set L0_0_set_12, LBB0_12-LJTI0_0
+; CHECK-NEXT:  .set L0_0_set_13, LBB0_13-LJTI0_0
+; CHECK-NEXT:  .set L0_0_set_14, LBB0_14-LJTI0_0
+; CHECK-NEXT:  .set L0_0_set_15, LBB0_15-LJTI0_0
+; CHECK-NEXT:  .set L0_0_set_16, LBB0_16-LJTI0_0
+; CHECK-NEXT:  .set L0_0_set_17, LBB0_17-LJTI0_0
+; CHECK-NEXT:  .set L0_0_set_18, LBB0_18-LJTI0_0
+; CHECK-NEXT:  .set L0_0_set_19, LBB0_19-LJTI0_0
+; CHECK-NEXT:  .set L0_0_set_20, LBB0_20-LJTI0_0
+; CHECK-NEXT:  .set L0_0_set_21, LBB0_21-LJTI0_0
+; CHECK-NEXT:  .set L0_0_set_22, LBB0_22-LJTI0_0
+; CHECK-NEXT:  .set L0_0_set_23, LBB0_23-LJTI0_0
+; CHECK-NEXT:  .set L0_0_set_24, LBB0_24-LJTI0_0
+; CHECK-NEXT:  .set L0_0_set_25, LBB0_25-LJTI0_0
+; CHECK-NEXT:  .set L0_0_set_26, LBB0_26-LJTI0_0
+; CHECK-NEXT:  .set L0_0_set_27, LBB0_27-LJTI0_0
+; CHECK-NEXT:  LJTI0_0:
+; CHECK-NEXT:    .long L0_0_set_3
+; CHECK-NEXT:    .long L0_0_set_3
+; CHECK-NEXT:    .long L0_0_set_4
+; CHECK-NEXT:    .long L0_0_set_5
+; CHECK-NEXT:    .long L0_0_set_6
+; CHECK-NEXT:    .long L0_0_set_7
+; CHECK-NEXT:    .long L0_0_set_8
+; CHECK-NEXT:    .long L0_0_set_9
+; CHECK-NEXT:    .long L0_0_set_10
+; CHECK-NEXT:    .long L0_0_set_11
+; CHECK-NEXT:    .long L0_0_set_12
+; CHECK-NEXT:    .long L0_0_set_13
+; CHECK-NEXT:    .long L0_0_set_14
+; CHECK-NEXT:    .long L0_0_set_15
+; CHECK-NEXT:    .long L0_0_set_16
+; CHECK-NEXT:    .long L0_0_set_17
+; CHECK-NEXT:    .long L0_0_set_18
+; CHECK-NEXT:    .long L0_0_set_19
+; CHECK-NEXT:    .long L0_0_set_20
+; CHECK-NEXT:    .long L0_0_set_21
+; CHECK-NEXT:    .long L0_0_set_22
+; CHECK-NEXT:    .long L0_0_set_23
+; CHECK-NEXT:    .long L0_0_set_24
+; CHECK-NEXT:    .long L0_0_set_25
+; CHECK-NEXT:    .long L0_0_set_26
+; CHECK-NEXT:    .long L0_0_set_27
+; CHECK-NEXT:    .end_data_region
 entry:
         br label %bb49
 
diff --git a/llvm/test/CodeGen/X86/20090313-signext.ll b/llvm/test/CodeGen/X86/20090313-signext.ll
index 4162ee668..688cb07ac 100644
--- a/llvm/test/CodeGen/X86/20090313-signext.ll
+++ b/llvm/test/CodeGen/X86/20090313-signext.ll
@@ -1,11 +1,19 @@
-; RUN: llc < %s -mtriple=x86_64-- -relocation-model=pic > %t
-; RUN: grep "movswl	%ax, %edi" %t
-; RUN: grep "movw	(%rax), %ax" %t
-; XFAIL: *
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-- -relocation-model=pic | FileCheck %s
 
 @x = common global i16 0
 
 define signext i16 @f() nounwind {
+; CHECK-LABEL: f:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    callq h@PLT
+; CHECK-NEXT:    movswl %ax, %edi
+; CHECK-NEXT:    callq g@PLT
+; CHECK-NEXT:    movq x@{{.*}}(%rip), %rax
+; CHECK-NEXT:    movzwl (%rax), %eax
+; CHECK-NEXT:    popq %rcx
+; CHECK-NEXT:    retq
 entry:
 	%0 = tail call signext i16 @h() nounwind
 	%1 = sext i16 %0 to i32
diff --git a/llvm/test/CodeGen/X86/2010-02-19-TailCallRetAddrBug.ll b/llvm/test/CodeGen/X86/2010-02-19-TailCallRetAddrBug.ll
index 9d3e18efe..53d29e2ac 100644
--- a/llvm/test/CodeGen/X86/2010-02-19-TailCallRetAddrBug.ll
+++ b/llvm/test/CodeGen/X86/2010-02-19-TailCallRetAddrBug.ll
@@ -1,5 +1,5 @@
 ; RUN: llc -mcpu=generic -mtriple=i386-apple-darwin -tailcallopt -enable-misched=false < %s | FileCheck %s
-; Check that lowered argumens do not overwrite the return address before it is moved.
+; Check that lowered arguments do not overwrite the return address before it is moved.
 ; Bug 6225
 ;
 ; If a call is a fastcc tail call and tail call optimization is enabled, the
diff --git a/llvm/test/CodeGen/X86/2011-05-09-loaduse.ll b/llvm/test/CodeGen/X86/2011-05-09-loaduse.ll
index 3dc77cb00..53b710324 100644
--- a/llvm/test/CodeGen/X86/2011-05-09-loaduse.ll
+++ b/llvm/test/CodeGen/X86/2011-05-09-loaduse.ll
@@ -1,14 +1,28 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=i686-- -mcpu=corei7 | FileCheck %s
+; RUN: llc < %s -mtriple=i686-- -mcpu=corei7 | FileCheck %s --check-prefixes=CHECK,X86
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=corei7 | FileCheck %s --check-prefixes=CHECK,X64
 
 define float @test(<4 x float>* %A) nounwind {
-; CHECK-LABEL: test:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; CHECK-NEXT:    xorps %xmm0, %xmm0
-; CHECK-NEXT:    flds 12(%eax)
-; CHECK-NEXT:    movaps %xmm0, (%eax)
-; CHECK-NEXT:    retl
+; X86-LABEL: test:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movaps (%eax), %xmm0
+; X86-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; X86-NEXT:    xorps %xmm1, %xmm1
+; X86-NEXT:    movaps %xmm1, (%eax)
+; X86-NEXT:    movss %xmm0, (%esp)
+; X86-NEXT:    flds (%esp)
+; X86-NEXT:    popl %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: test:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    movaps (%rdi), %xmm0
+; X64-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; X64-NEXT:    xorps %xmm1, %xmm1
+; X64-NEXT:    movaps %xmm1, (%rdi)
+; X64-NEXT:    retq
 entry:
   %T = load <4 x float>, <4 x float>* %A
   %R = extractelement <4 x float> %T, i32 3
diff --git a/llvm/test/CodeGen/X86/2011-10-19-LegelizeLoad.ll b/llvm/test/CodeGen/X86/2011-10-19-LegelizeLoad.ll
index d8a6823f7..d181af7c7 100644
--- a/llvm/test/CodeGen/X86/2011-10-19-LegelizeLoad.ll
+++ b/llvm/test/CodeGen/X86/2011-10-19-LegelizeLoad.ll
@@ -17,19 +17,21 @@ target triple = "x86_64-unknown-linux-gnu"
 define i32 @main() nounwind uwtable {
 ; CHECK-LABEL: main:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    pmovsxbq {{.*}}(%rip), %xmm0
-; CHECK-NEXT:    pmovsxbq {{.*}}(%rip), %xmm1
-; CHECK-NEXT:    pextrq $1, %xmm1, %rax
-; CHECK-NEXT:    pextrq $1, %xmm0, %rcx
-; CHECK-NEXT:    cqto
-; CHECK-NEXT:    idivq %rcx
-; CHECK-NEXT:    movq %rax, %xmm2
-; CHECK-NEXT:    movq %xmm1, %rax
-; CHECK-NEXT:    movq %xmm0, %rcx
-; CHECK-NEXT:    cqto
-; CHECK-NEXT:    idivq %rcx
-; CHECK-NEXT:    movq %rax, %xmm0
-; CHECK-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; CHECK-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    pextrb $1, %xmm0, %ecx
+; CHECK-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
+; CHECK-NEXT:    pextrb $1, %xmm1, %eax
+; CHECK-NEXT:    cbtw
+; CHECK-NEXT:    pextrb $0, %xmm0, %edx
+; CHECK-NEXT:    pextrb $0, %xmm1, %esi
+; CHECK-NEXT:    idivb %cl
+; CHECK-NEXT:    movl %eax, %ecx
+; CHECK-NEXT:    movsbl %sil, %eax
+; CHECK-NEXT:    idivb %dl
+; CHECK-NEXT:    movzbl %cl, %ecx
+; CHECK-NEXT:    movzbl %al, %eax
+; CHECK-NEXT:    movd %eax, %xmm0
+; CHECK-NEXT:    pinsrb $1, %ecx, %xmm0
 ; CHECK-NEXT:    pextrw $0, %xmm0, {{.*}}(%rip)
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/2011-12-28-vselecti8.ll b/llvm/test/CodeGen/X86/2011-12-28-vselecti8.ll
index bb3b4ed47..cb2cf269e 100644
--- a/llvm/test/CodeGen/X86/2011-12-28-vselecti8.ll
+++ b/llvm/test/CodeGen/X86/2011-12-28-vselecti8.ll
@@ -18,10 +18,11 @@ target triple = "x86_64-apple-darwin11.2.0"
 define void @foo8(float* nocapture %RET) nounwind {
 ; CHECK-LABEL: foo8:
 ; CHECK:       ## %bb.0: ## %allocas
-; CHECK-NEXT:    movaps {{.*#+}} xmm0 = [1.0E+2,2.0E+0,1.0E+2,4.0E+0]
-; CHECK-NEXT:    movaps {{.*#+}} xmm1 = [1.0E+2,6.0E+0,1.0E+2,8.0E+0]
-; CHECK-NEXT:    movups %xmm1, 16(%rdi)
-; CHECK-NEXT:    movups %xmm0, (%rdi)
+; CHECK-NEXT:    pmovzxbd {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
+; CHECK-NEXT:    cvtdq2ps %xmm0, %xmm0
+; CHECK-NEXT:    movaps {{.*#+}} xmm1 = [1.0E+2,2.0E+0,1.0E+2,4.0E+0]
+; CHECK-NEXT:    movups %xmm1, (%rdi)
+; CHECK-NEXT:    movups %xmm0, 16(%rdi)
 ; CHECK-NEXT:    retq
 allocas:
   %resultvec.i = select <8 x i1> <i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true>, <8 x i8> <i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8>, <8 x i8> <i8 100, i8 100, i8 100, i8 100, i8 100, i8 100, i8 100, i8 100>
diff --git a/llvm/test/CodeGen/X86/2011-12-8-bitcastintprom.ll b/llvm/test/CodeGen/X86/2011-12-8-bitcastintprom.ll
index 7b5b205ac..44f2468f6 100644
--- a/llvm/test/CodeGen/X86/2011-12-8-bitcastintprom.ll
+++ b/llvm/test/CodeGen/X86/2011-12-8-bitcastintprom.ll
@@ -6,16 +6,12 @@
 define void @prom_bug(<4 x i8> %t, i16* %p) {
 ; SSE2-LABEL: prom_bug:
 ; SSE2:       ## %bb.0:
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
-; SSE2-NEXT:    packuswb %xmm0, %xmm0
-; SSE2-NEXT:    packuswb %xmm0, %xmm0
-; SSE2-NEXT:    pextrw $0, %xmm0, %eax
+; SSE2-NEXT:    movd %xmm0, %eax
 ; SSE2-NEXT:    movw %ax, (%rdi)
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: prom_bug:
 ; SSE41:       ## %bb.0:
-; SSE41-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
 ; SSE41-NEXT:    pextrw $0, %xmm0, (%rdi)
 ; SSE41-NEXT:    retq
   %r = bitcast <4 x i8> %t to <2 x i16>
diff --git a/llvm/test/CodeGen/X86/2012-01-18-vbitcast.ll b/llvm/test/CodeGen/X86/2012-01-18-vbitcast.ll
index ab57b6177..e4313060b 100644
--- a/llvm/test/CodeGen/X86/2012-01-18-vbitcast.ll
+++ b/llvm/test/CodeGen/X86/2012-01-18-vbitcast.ll
@@ -4,9 +4,8 @@
 define <2 x i32> @vcast(<2 x float> %a, <2 x float> %b) {
 ; CHECK-LABEL: vcast:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pmovzxdq {{.*#+}} xmm0 = mem[0],zero,mem[1],zero
-; CHECK-NEXT:    pmovzxdq {{.*#+}} xmm1 = mem[0],zero,mem[1],zero
-; CHECK-NEXT:    psubq %xmm1, %xmm0
+; CHECK-NEXT:    movdqa (%rcx), %xmm0
+; CHECK-NEXT:    psubd (%rdx), %xmm0
 ; CHECK-NEXT:    retq
   %af = bitcast <2 x float> %a to <2 x i32>
   %bf = bitcast <2 x float> %b to <2 x i32>
diff --git a/llvm/test/CodeGen/X86/2012-03-15-build_vector_wl.ll b/llvm/test/CodeGen/X86/2012-03-15-build_vector_wl.ll
index 95d78e474..9c1266504 100644
--- a/llvm/test/CodeGen/X86/2012-03-15-build_vector_wl.ll
+++ b/llvm/test/CodeGen/X86/2012-03-15-build_vector_wl.ll
@@ -4,7 +4,6 @@
 define <4 x i8> @build_vector_again(<16 x i8> %in) nounwind readnone {
 ; CHECK-LABEL: build_vector_again:
 ; CHECK:       ## %bb.0: ## %entry
-; CHECK-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; CHECK-NEXT:    retq
 entry:
   %out = shufflevector <16 x i8> %in, <16 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
diff --git a/llvm/test/CodeGen/X86/2012-04-26-sdglue.ll b/llvm/test/CodeGen/X86/2012-04-26-sdglue.ll
index afa8bf44c..2590a27a9 100644
--- a/llvm/test/CodeGen/X86/2012-04-26-sdglue.ll
+++ b/llvm/test/CodeGen/X86/2012-04-26-sdglue.ll
@@ -9,17 +9,17 @@ define void @func(<4 x float> %a, <16 x i8> %b, <16 x i8> %c, <8 x float> %d, <8
 ; CHECK:       ## %bb.0:
 ; CHECK-NEXT:    vmovdqu 0, %xmm0
 ; CHECK-NEXT:    vpalignr {{.*#+}} xmm1 = xmm0[4,5,6,7,8,9,10,11,12,13,14,15],xmm1[0,1,2,3]
-; CHECK-NEXT:    vxorps %xmm2, %xmm2, %xmm2
 ; CHECK-NEXT:    vmulps %xmm1, %xmm1, %xmm1
 ; CHECK-NEXT:    vmulps %xmm0, %xmm0, %xmm0
 ; CHECK-NEXT:    vaddps %xmm1, %xmm0, %xmm0
 ; CHECK-NEXT:    vaddps %xmm0, %xmm0, %xmm0
 ; CHECK-NEXT:    vmulps %xmm0, %xmm0, %xmm0
 ; CHECK-NEXT:    vperm2f128 {{.*#+}} ymm0 = zero,zero,ymm0[0,1]
+; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1
 ; CHECK-NEXT:    vaddps %ymm0, %ymm0, %ymm0
 ; CHECK-NEXT:    vhaddps %ymm4, %ymm0, %ymm0
 ; CHECK-NEXT:    vsubps %ymm0, %ymm0, %ymm0
-; CHECK-NEXT:    vhaddps %ymm0, %ymm2, %ymm0
+; CHECK-NEXT:    vhaddps %ymm0, %ymm1, %ymm0
 ; CHECK-NEXT:    vmovaps %ymm0, (%rdi)
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/2012-07-10-extload64.ll b/llvm/test/CodeGen/X86/2012-07-10-extload64.ll
index 60e3e3247..67cfa3a71 100644
--- a/llvm/test/CodeGen/X86/2012-07-10-extload64.ll
+++ b/llvm/test/CodeGen/X86/2012-07-10-extload64.ll
@@ -33,7 +33,7 @@ define <2 x i32> @load_64(<2 x i32>* %ptr) {
 ; CHECK-LABEL: load_64:
 ; CHECK:       # %bb.0: # %BB
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; CHECK-NEXT:    pmovzxdq {{.*#+}} xmm0 = mem[0],zero,mem[1],zero
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; CHECK-NEXT:    retl
 BB:
   %t = load <2 x i32>, <2 x i32>* %ptr
diff --git a/llvm/test/CodeGen/X86/2012-08-07-CmpISelBug.ll b/llvm/test/CodeGen/X86/2012-08-07-CmpISelBug.ll
index e844ce04d..e2c56e443 100644
--- a/llvm/test/CodeGen/X86/2012-08-07-CmpISelBug.ll
+++ b/llvm/test/CodeGen/X86/2012-08-07-CmpISelBug.ll
@@ -8,14 +8,13 @@
 define void @foo(i8 %arg4, i32 %arg5, i32* %arg14) nounwind {
 ; CHECK-LABEL: foo:
 ; CHECK:       ## %bb.0: ## %bb
+; CHECK-NEXT:    ## kill: def $edi killed $edi def $rdi
 ; CHECK-NEXT:    andl $32, %edi
-; CHECK-NEXT:    orl $1601159181, %edi ## imm = 0x5F6FC00D
-; CHECK-NEXT:    andl %edi, %esi
-; CHECK-NEXT:    xorb $-14, %dil
-; CHECK-NEXT:    addb $82, %dil
-; CHECK-NEXT:    shrl $5, %esi
-; CHECK-NEXT:    movzbl %dil, %eax
-; CHECK-NEXT:    testb %sil, %sil
+; CHECK-NEXT:    leal 13(%rdi), %eax
+; CHECK-NEXT:    xorb $-14, %al
+; CHECK-NEXT:    addb $82, %al
+; CHECK-NEXT:    movzbl %al, %eax
+; CHECK-NEXT:    testl %esi, %edi
 ; CHECK-NEXT:    movl $1, %ecx
 ; CHECK-NEXT:    cmovnel %eax, %ecx
 ; CHECK-NEXT:    xorb $81, %cl
diff --git a/llvm/test/CodeGen/X86/2013-03-13-VEX-DestReg.ll b/llvm/test/CodeGen/X86/2013-03-13-VEX-DestReg.ll
index f6827c1c7..4df2bf6fc 100644
--- a/llvm/test/CodeGen/X86/2013-03-13-VEX-DestReg.ll
+++ b/llvm/test/CodeGen/X86/2013-03-13-VEX-DestReg.ll
@@ -23,6 +23,6 @@ entry:
 
 declare <4 x float> @llvm.x86.sse.rcp.ss(<4 x float>) #1
 
-attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 attributes #2 = { nounwind }
diff --git a/llvm/test/CodeGen/X86/2014-08-29-CompactUnwind.ll b/llvm/test/CodeGen/X86/2014-08-29-CompactUnwind.ll
index f6d6bd3ed..a0b72581a 100644
--- a/llvm/test/CodeGen/X86/2014-08-29-CompactUnwind.ll
+++ b/llvm/test/CodeGen/X86/2014-08-29-CompactUnwind.ll
@@ -43,4 +43,4 @@ print_shadow_bytes.exit.i: ; preds = %print_shadow_bytes.exit.i, %0
 
 declare void @append(i64*, i8*, ...)
 
-attributes #0 = { "no-frame-pointer-elim"="false" }
+attributes #0 = { "frame-pointer"="none" }
diff --git a/llvm/test/CodeGen/X86/3dnow-intrinsics.ll b/llvm/test/CodeGen/X86/3dnow-intrinsics.ll
index 109b9fc02..611ba11c5 100644
--- a/llvm/test/CodeGen/X86/3dnow-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/3dnow-intrinsics.ll
@@ -13,9 +13,7 @@ define <8 x i8> @test_pavgusb(x86_mmx %a.coerce, x86_mmx %b.coerce) nounwind rea
 ; X64-LABEL: test_pavgusb:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    pavgusb %mm1, %mm0
-; X64-NEXT:    movq %mm0, -{{[0-9]+}}(%rsp)
-; X64-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X64-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X64-NEXT:    movq2dq %mm0, %xmm0
 ; X64-NEXT:    retq
 entry:
   %0 = bitcast x86_mmx %a.coerce to <8 x i8>
@@ -51,9 +49,7 @@ define <2 x i32> @test_pf2id(<2 x float> %a) nounwind readnone {
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    movdq2q %xmm0, %mm0
 ; X64-NEXT:    pf2id %mm0, %mm0
-; X64-NEXT:    movq %mm0, -{{[0-9]+}}(%rsp)
-; X64-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,1,3]
+; X64-NEXT:    movq2dq %mm0, %xmm0
 ; X64-NEXT:    retq
 entry:
   %0 = bitcast <2 x float> %a to x86_mmx
@@ -168,9 +164,7 @@ define <2 x i32> @test_pfcmpeq(<2 x float> %a, <2 x float> %b) nounwind readnone
 ; X64-NEXT:    movdq2q %xmm1, %mm0
 ; X64-NEXT:    movdq2q %xmm0, %mm1
 ; X64-NEXT:    pfcmpeq %mm0, %mm1
-; X64-NEXT:    movq %mm1, -{{[0-9]+}}(%rsp)
-; X64-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,1,3]
+; X64-NEXT:    movq2dq %mm1, %xmm0
 ; X64-NEXT:    retq
 entry:
   %0 = bitcast <2 x float> %a to x86_mmx
@@ -208,9 +202,7 @@ define <2 x i32> @test_pfcmpge(<2 x float> %a, <2 x float> %b) nounwind readnone
 ; X64-NEXT:    movdq2q %xmm1, %mm0
 ; X64-NEXT:    movdq2q %xmm0, %mm1
 ; X64-NEXT:    pfcmpge %mm0, %mm1
-; X64-NEXT:    movq %mm1, -{{[0-9]+}}(%rsp)
-; X64-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,1,3]
+; X64-NEXT:    movq2dq %mm1, %xmm0
 ; X64-NEXT:    retq
 entry:
   %0 = bitcast <2 x float> %a to x86_mmx
@@ -248,9 +240,7 @@ define <2 x i32> @test_pfcmpgt(<2 x float> %a, <2 x float> %b) nounwind readnone
 ; X64-NEXT:    movdq2q %xmm1, %mm0
 ; X64-NEXT:    movdq2q %xmm0, %mm1
 ; X64-NEXT:    pfcmpgt %mm0, %mm1
-; X64-NEXT:    movq %mm1, -{{[0-9]+}}(%rsp)
-; X64-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,1,3]
+; X64-NEXT:    movq2dq %mm1, %xmm0
 ; X64-NEXT:    retq
 entry:
   %0 = bitcast <2 x float> %a to x86_mmx
@@ -684,9 +674,7 @@ define <4 x i16> @test_pmulhrw(x86_mmx %a.coerce, x86_mmx %b.coerce) nounwind re
 ; X64-LABEL: test_pmulhrw:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    pmulhrw %mm1, %mm0
-; X64-NEXT:    movq %mm0, -{{[0-9]+}}(%rsp)
-; X64-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X64-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; X64-NEXT:    movq2dq %mm0, %xmm0
 ; X64-NEXT:    retq
 entry:
   %0 = bitcast x86_mmx %a.coerce to <4 x i16>
@@ -722,9 +710,7 @@ define <2 x i32> @test_pf2iw(<2 x float> %a) nounwind readnone {
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    movdq2q %xmm0, %mm0
 ; X64-NEXT:    pf2iw %mm0, %mm0
-; X64-NEXT:    movq %mm0, -{{[0-9]+}}(%rsp)
-; X64-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,1,3]
+; X64-NEXT:    movq2dq %mm0, %xmm0
 ; X64-NEXT:    retq
 entry:
   %0 = bitcast <2 x float> %a to x86_mmx
@@ -896,12 +882,9 @@ define <2 x i32> @test_pswapdsi(<2 x i32> %a) nounwind readnone {
 ;
 ; X64-LABEL: test_pswapdsi:
 ; X64:       # %bb.0: # %entry
-; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; X64-NEXT:    movdq2q %xmm0, %mm0
 ; X64-NEXT:    pswapd %mm0, %mm0 # mm0 = mm0[1,0]
-; X64-NEXT:    movq %mm0, -{{[0-9]+}}(%rsp)
-; X64-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,1,3]
+; X64-NEXT:    movq2dq %mm0, %xmm0
 ; X64-NEXT:    retq
 entry:
   %0 = bitcast <2 x i32> %a to x86_mmx
diff --git a/llvm/test/CodeGen/X86/4char-promote.ll b/llvm/test/CodeGen/X86/4char-promote.ll
index 27778be3b..fcecf3104 100644
--- a/llvm/test/CodeGen/X86/4char-promote.ll
+++ b/llvm/test/CodeGen/X86/4char-promote.ll
@@ -7,8 +7,11 @@ target triple = "x86_64-apple-darwin"
 define <4 x i8> @foo(<4 x i8> %x, <4 x i8> %y) {
 ; CHECK-LABEL: foo:
 ; CHECK:       ## %bb.0: ## %entry
-; CHECK-NEXT:    pmulld %xmm0, %xmm1
-; CHECK-NEXT:    paddd %xmm1, %xmm0
+; CHECK-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; CHECK-NEXT:    pmovzxbw {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; CHECK-NEXT:    pmullw %xmm1, %xmm2
+; CHECK-NEXT:    pshufb {{.*#+}} xmm2 = xmm2[0,2,4,6,u,u,u,u,u,u,u,u,u,u,u,u]
+; CHECK-NEXT:    paddb %xmm2, %xmm0
 ; CHECK-NEXT:    retq
 entry:
  %binop = mul <4 x i8> %x, %y
diff --git a/llvm/test/CodeGen/X86/DynamicCalleeSavedRegisters.ll b/llvm/test/CodeGen/X86/DynamicCalleeSavedRegisters.ll
index 72807922a..75c448105 100644
--- a/llvm/test/CodeGen/X86/DynamicCalleeSavedRegisters.ll
+++ b/llvm/test/CodeGen/X86/DynamicCalleeSavedRegisters.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=i386-linux-gnu  | FileCheck %s
 
 declare x86_regcallcc i32 @callee(i32 %a0, i32 %b0, i32 %c0, i32 %d0, i32 %e0);
@@ -5,47 +6,60 @@ declare x86_regcallcc i32 @callee(i32 %a0, i32 %b0, i32 %c0, i32 %d0, i32 %e0);
 ; In RegCall calling convention, ESI and EDI are callee saved registers.
 ; One might think that the caller could assume that ESI value is the same before
 ; and after calling the callee.
-; However, RegCall also says that a register that was used for 
-; passing/returning argumnets, can be assumed to be modified by the callee.
+; However, RegCall also says that a register that was used for
+; passing/returning arguments, can be assumed to be modified by the callee.
 ; In other words, it is no longer a callee saved register.
 ; In this case we want to see that EDX/ECX values are saved and EDI/ESI are assumed
 ; to be modified by the callee.
 ; This is a hipe CC function that doesn't save any register for the caller.
 ; So we can be sure that there is no other reason to save EDX/ECX.
-; The caller arguments are expected to be passed (in the following order) 
+; The caller arguments are expected to be passed (in the following order)
 ; in registers: ESI, EBP, EAX, EDX and ECX.
 define cc 11 i32 @caller(i32 %a0, i32 %b0, i32 %c0, i32 %d0, i32 %e0) nounwind {
+; CHECK-LABEL: caller:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subl $12, %esp
+; CHECK-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    movl %edx, %ebx
+; CHECK-NEXT:    movl %eax, %edx
+; CHECK-NEXT:    movl %esi, %eax
+; CHECK-NEXT:    movl %ebp, %ecx
+; CHECK-NEXT:    movl %ebx, %edi
+; CHECK-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; CHECK-NEXT:    movl %ebp, %esi
+; CHECK-NEXT:    calll callee
+; CHECK-NEXT:    leal (%eax,%ebx), %esi
+; CHECK-NEXT:    addl %ebp, %esi
+; CHECK-NEXT:    addl $12, %esp
+; CHECK-NEXT:    retl
   %b1 = call x86_regcallcc i32 @callee(i32 %a0, i32 %b0, i32 %c0, i32 %d0, i32 %e0)
   %b2 = add i32 %b1, %d0
   %b3 = add i32 %b2, %e0
   ret i32 %b3
 }
-; CHECK-LABEL:  caller
-; CHECK:        subl    $12, %esp
-; CHECK-NEXT:   movl    %ecx, 8(%esp)
-; CHECK-NEXT:   movl    %edx, %ebx
-; CHECK-NEXT:   movl    %eax, %edx
-; CHECK-NEXT:   movl    %esi, %eax
-; CHECK-NEXT:   movl    %ebp, %ecx
-; CHECK-NEXT:   movl    %ebx, %edi
-; CHECK-NEXT:   movl    8(%esp), %ebp
-; CHECK-NEXT:   movl    %ebp, %esi
-; CHECK-NEXT:   calll   callee
-; CHECK-NEXT:   leal    (%eax,%ebx), %esi
-; CHECK-NEXT:   addl    %ebp, %esi
-; CHECK-NEXT:   addl    $12, %esp
-; CHECK-NEXT:   retl
-
 !hipe.literals = !{ !0, !1, !2 }
 !0 = !{ !"P_NSP_LIMIT", i32 120 }
 !1 = !{ !"X86_LEAF_WORDS", i32 24 }
 !2 = !{ !"AMD64_LEAF_WORDS", i32 18 }
 
 ; Make sure that the callee doesn't save parameters that were passed as arguments.
-; The caller arguments are expected to be passed (in the following order) 
+; The caller arguments are expected to be passed (in the following order)
 ; in registers: EAX, ECX, EDX, EDI and ESI.
 ; The result will return in EAX, ECX and EDX.
 define x86_regcallcc {i32, i32, i32} @test_callee(i32 %a0, i32 %b0, i32 %c0, i32 %d0, i32 %e0) nounwind {
+; CHECK-LABEL: test_callee:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    leal (,%esi,8), %ecx
+; CHECK-NEXT:    subl %esi, %ecx
+; CHECK-NEXT:    movl $5, %eax
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    divl %esi
+; CHECK-NEXT:    movl %eax, %esi
+; CHECK-NEXT:    leal (,%edi,8), %edx
+; CHECK-NEXT:    subl %edi, %edx
+; CHECK-NEXT:    movl %ecx, %eax
+; CHECK-NEXT:    movl %esi, %ecx
+; CHECK-NEXT:    retl
   %b1 = mul i32 7, %e0
   %b2 = udiv i32 5, %e0
   %b3 = mul i32 7, %d0
@@ -54,7 +68,3 @@ define x86_regcallcc {i32, i32, i32} @test_callee(i32 %a0, i32 %b0, i32 %c0, i32
   %b6 = insertvalue {i32, i32, i32} %b5, i32 %b3, 2
   ret {i32, i32, i32} %b6
 }
-; CHECK-LABEL: test_callee
-; CHECK-NOT:   pushl %esi
-; CHECK-NOT:   pushl %edi
-; CHECK:       retl
diff --git a/llvm/test/CodeGen/X86/GlobalISel/add-ext.ll b/llvm/test/CodeGen/X86/GlobalISel/add-ext.ll
index bf081b355..d0db8fdc0 100644
--- a/llvm/test/CodeGen/X86/GlobalISel/add-ext.ll
+++ b/llvm/test/CodeGen/X86/GlobalISel/add-ext.ll
@@ -45,8 +45,7 @@ define i64 @add_nsw_sext_lsh_add(i32 %i, i64 %x) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addl $-5, %edi
 ; CHECK-NEXT:    movslq %edi, %rax
-; CHECK-NEXT:    movq $3, %rcx
-; CHECK-NEXT:    shlq %cl, %rax
+; CHECK-NEXT:    shlq $3, %rax
 ; CHECK-NEXT:    addq %rsi, %rax
 ; CHECK-NEXT:    retq
 
@@ -202,16 +201,19 @@ define void @PR20134_zext(i32* %a, i32 %i) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    # kill: def $esi killed $esi def $rsi
 ; CHECK-NEXT:    leal 1(%rsi), %eax
+; CHECK-NEXT:    movl %eax, %eax
 ; CHECK-NEXT:    movq $4, %rcx
 ; CHECK-NEXT:    imulq %rcx, %rax
 ; CHECK-NEXT:    addq %rdi, %rax
 ; CHECK-NEXT:    leal 2(%rsi), %edx
+; CHECK-NEXT:    movl %edx, %edx
 ; CHECK-NEXT:    imulq %rcx, %rdx
 ; CHECK-NEXT:    addq %rdi, %rdx
 ; CHECK-NEXT:    movl (%rdx), %edx
 ; CHECK-NEXT:    addl (%rax), %edx
-; CHECK-NEXT:    imulq %rcx, %rsi
-; CHECK-NEXT:    leaq (%rdi,%rsi), %rax
+; CHECK-NEXT:    movl %esi, %eax
+; CHECK-NEXT:    imulq %rcx, %rax
+; CHECK-NEXT:    addq %rdi, %rax
 ; CHECK-NEXT:    movl %edx, (%rax)
 ; CHECK-NEXT:    retq
 
diff --git a/llvm/test/CodeGen/X86/GlobalISel/add-scalar.ll b/llvm/test/CodeGen/X86/GlobalISel/add-scalar.ll
index e7b04ce22..fd526963a 100644
--- a/llvm/test/CodeGen/X86/GlobalISel/add-scalar.ll
+++ b/llvm/test/CodeGen/X86/GlobalISel/add-scalar.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=x86_64-linux-gnu -global-isel -verify-machineinstrs < %s -o - | FileCheck %s --check-prefix=ALL --check-prefix=X64
-; RUN: llc -mtriple=i386-linux-gnu   -global-isel -verify-machineinstrs < %s -o - | FileCheck %s --check-prefix=ALL --check-prefix=X32
+; RUN: llc -mtriple=x86_64-linux-gnu -global-isel -global-isel-abort=2 -verify-machineinstrs < %s -o - | FileCheck %s --check-prefix=ALL --check-prefix=X64
+; RUN: llc -mtriple=i386-linux-gnu   -global-isel -global-isel-abort=2 -verify-machineinstrs < %s -o - | FileCheck %s --check-prefix=ALL --check-prefix=X32
 
 define i64 @test_add_i64(i64 %arg1, i64 %arg2) {
 ; X64-LABEL: test_add_i64:
diff --git a/llvm/test/CodeGen/X86/GlobalISel/ashr-scalar.ll b/llvm/test/CodeGen/X86/GlobalISel/ashr-scalar.ll
index 9db8b8e34..849cf0d4c 100644
--- a/llvm/test/CodeGen/X86/GlobalISel/ashr-scalar.ll
+++ b/llvm/test/CodeGen/X86/GlobalISel/ashr-scalar.ll
@@ -17,8 +17,7 @@ define i64 @test_ashr_i64_imm(i64 %arg1) {
 ; X64-LABEL: test_ashr_i64_imm:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movq %rdi, %rax
-; X64-NEXT:    movq $5, %rcx
-; X64-NEXT:    sarq %cl, %rax
+; X64-NEXT:    sarq $5, %rax
 ; X64-NEXT:    retq
   %res = ashr i64 %arg1, 5
   ret i64 %res
@@ -50,8 +49,7 @@ define i32 @test_ashr_i32_imm(i32 %arg1) {
 ; X64-LABEL: test_ashr_i32_imm:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl %edi, %eax
-; X64-NEXT:    movl $5, %ecx
-; X64-NEXT:    sarl %cl, %eax
+; X64-NEXT:    sarl $5, %eax
 ; X64-NEXT:    retq
   %res = ashr i32 %arg1, 5
   ret i32 %res
@@ -86,8 +84,7 @@ define i16 @test_ashr_i16_imm(i32 %arg1) {
 ; X64-LABEL: test_ashr_i16_imm:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl %edi, %eax
-; X64-NEXT:    movw $5, %cx
-; X64-NEXT:    sarw %cl, %ax
+; X64-NEXT:    sarw $5, %ax
 ; X64-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X64-NEXT:    retq
   %a = trunc i32 %arg1 to i16
@@ -168,7 +165,7 @@ define i1 @test_ashr_i1_imm1(i32 %arg1) {
 ; X64-LABEL: test_ashr_i1_imm1:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl %edi, %eax
-; X64-NEXT:    movb $-1, %cl
+; X64-NEXT:    movb $1, %cl
 ; X64-NEXT:    shlb $7, %al
 ; X64-NEXT:    sarb $7, %al
 ; X64-NEXT:    andb $1, %cl
diff --git a/llvm/test/CodeGen/X86/GlobalISel/ext-x86-64.ll b/llvm/test/CodeGen/X86/GlobalISel/ext-x86-64.ll
index 498126fbc..ce9637b4b 100644
--- a/llvm/test/CodeGen/X86/GlobalISel/ext-x86-64.ll
+++ b/llvm/test/CodeGen/X86/GlobalISel/ext-x86-64.ll
@@ -18,10 +18,8 @@ define i64 @test_sext_i8(i8 %val) {
 ; X64-LABEL: test_sext_i8:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl %edi, %eax
-; X64-NEXT:    movq $56, %rcx
-; X64-NEXT:    shlq %cl, %rax
-; X64-NEXT:    movq $56, %rcx
-; X64-NEXT:    sarq %cl, %rax
+; X64-NEXT:    shlq $56, %rax
+; X64-NEXT:    sarq $56, %rax
 ; X64-NEXT:    retq
   %r = sext i8 %val to i64
   ret i64 %r
@@ -31,10 +29,8 @@ define i64 @test_sext_i16(i16 %val) {
 ; X64-LABEL: test_sext_i16:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl %edi, %eax
-; X64-NEXT:    movq $48, %rcx
-; X64-NEXT:    shlq %cl, %rax
-; X64-NEXT:    movq $48, %rcx
-; X64-NEXT:    sarq %cl, %rax
+; X64-NEXT:    shlq $48, %rax
+; X64-NEXT:    sarq $48, %rax
 ; X64-NEXT:    retq
   %r = sext i16 %val to i64
   ret i64 %r
diff --git a/llvm/test/CodeGen/X86/GlobalISel/ext.ll b/llvm/test/CodeGen/X86/GlobalISel/ext.ll
index a54656b55..953f3ff95 100644
--- a/llvm/test/CodeGen/X86/GlobalISel/ext.ll
+++ b/llvm/test/CodeGen/X86/GlobalISel/ext.ll
@@ -89,10 +89,8 @@ define i32 @test_sext_i8(i8 %val) {
 ; X64-LABEL: test_sext_i8:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl %edi, %eax
-; X64-NEXT:    movl $24, %ecx
-; X64-NEXT:    shll %cl, %eax
-; X64-NEXT:    movl $24, %ecx
-; X64-NEXT:    sarl %cl, %eax
+; X64-NEXT:    shll $24, %eax
+; X64-NEXT:    sarl $24, %eax
 ; X64-NEXT:    retq
 ;
 ; X32-LABEL: test_sext_i8:
@@ -107,10 +105,8 @@ define i32 @test_sext_i16(i16 %val) {
 ; X64-LABEL: test_sext_i16:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl %edi, %eax
-; X64-NEXT:    movl $16, %ecx
-; X64-NEXT:    shll %cl, %eax
-; X64-NEXT:    movl $16, %ecx
-; X64-NEXT:    sarl %cl, %eax
+; X64-NEXT:    shll $16, %eax
+; X64-NEXT:    sarl $16, %eax
 ; X64-NEXT:    retq
 ;
 ; X32-LABEL: test_sext_i16:
diff --git a/llvm/test/CodeGen/X86/GlobalISel/irtranslator-callingconv.ll b/llvm/test/CodeGen/X86/GlobalISel/irtranslator-callingconv.ll
index f68805df4..a281ed366 100644
--- a/llvm/test/CodeGen/X86/GlobalISel/irtranslator-callingconv.ll
+++ b/llvm/test/CodeGen/X86/GlobalISel/irtranslator-callingconv.ll
@@ -360,11 +360,11 @@ define void @test_simple_arg(i32 %in0, i32 %in1) {
   ; X32:   ADJCALLSTACKDOWN32 8, 0, 0, implicit-def $esp, implicit-def $eflags, implicit-def $ssp, implicit $esp, implicit $ssp
   ; X32:   [[COPY:%[0-9]+]]:_(p0) = COPY $esp
   ; X32:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-  ; X32:   [[GEP:%[0-9]+]]:_(p0) = G_GEP [[COPY]], [[C]](s32)
+  ; X32:   [[GEP:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s32)
   ; X32:   G_STORE [[LOAD1]](s32), [[GEP]](p0) :: (store 4 into stack, align 1)
   ; X32:   [[COPY1:%[0-9]+]]:_(p0) = COPY $esp
   ; X32:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-  ; X32:   [[GEP1:%[0-9]+]]:_(p0) = G_GEP [[COPY1]], [[C1]](s32)
+  ; X32:   [[GEP1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY1]], [[C1]](s32)
   ; X32:   G_STORE [[LOAD]](s32), [[GEP1]](p0) :: (store 4 into stack + 4, align 1)
   ; X32:   CALLpcrel32 @simple_arg_callee, csr_32, implicit $esp, implicit $ssp
   ; X32:   ADJCALLSTACKUP32 8, 0, implicit-def $esp, implicit-def $eflags, implicit-def $ssp, implicit $esp, implicit $ssp
@@ -393,35 +393,35 @@ define void @test_simple_arg8_call(i32 %in0) {
   ; X32:   ADJCALLSTACKDOWN32 32, 0, 0, implicit-def $esp, implicit-def $eflags, implicit-def $ssp, implicit $esp, implicit $ssp
   ; X32:   [[COPY:%[0-9]+]]:_(p0) = COPY $esp
   ; X32:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-  ; X32:   [[GEP:%[0-9]+]]:_(p0) = G_GEP [[COPY]], [[C]](s32)
+  ; X32:   [[GEP:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s32)
   ; X32:   G_STORE [[LOAD]](s32), [[GEP]](p0) :: (store 4 into stack, align 1)
   ; X32:   [[COPY1:%[0-9]+]]:_(p0) = COPY $esp
   ; X32:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-  ; X32:   [[GEP1:%[0-9]+]]:_(p0) = G_GEP [[COPY1]], [[C1]](s32)
+  ; X32:   [[GEP1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY1]], [[C1]](s32)
   ; X32:   G_STORE [[LOAD]](s32), [[GEP1]](p0) :: (store 4 into stack + 4, align 1)
   ; X32:   [[COPY2:%[0-9]+]]:_(p0) = COPY $esp
   ; X32:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-  ; X32:   [[GEP2:%[0-9]+]]:_(p0) = G_GEP [[COPY2]], [[C2]](s32)
+  ; X32:   [[GEP2:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY2]], [[C2]](s32)
   ; X32:   G_STORE [[LOAD]](s32), [[GEP2]](p0) :: (store 4 into stack + 8, align 1)
   ; X32:   [[COPY3:%[0-9]+]]:_(p0) = COPY $esp
   ; X32:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
-  ; X32:   [[GEP3:%[0-9]+]]:_(p0) = G_GEP [[COPY3]], [[C3]](s32)
+  ; X32:   [[GEP3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY3]], [[C3]](s32)
   ; X32:   G_STORE [[LOAD]](s32), [[GEP3]](p0) :: (store 4 into stack + 12, align 1)
   ; X32:   [[COPY4:%[0-9]+]]:_(p0) = COPY $esp
   ; X32:   [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-  ; X32:   [[GEP4:%[0-9]+]]:_(p0) = G_GEP [[COPY4]], [[C4]](s32)
+  ; X32:   [[GEP4:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY4]], [[C4]](s32)
   ; X32:   G_STORE [[LOAD]](s32), [[GEP4]](p0) :: (store 4 into stack + 16, align 1)
   ; X32:   [[COPY5:%[0-9]+]]:_(p0) = COPY $esp
   ; X32:   [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
-  ; X32:   [[GEP5:%[0-9]+]]:_(p0) = G_GEP [[COPY5]], [[C5]](s32)
+  ; X32:   [[GEP5:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY5]], [[C5]](s32)
   ; X32:   G_STORE [[LOAD]](s32), [[GEP5]](p0) :: (store 4 into stack + 20, align 1)
   ; X32:   [[COPY6:%[0-9]+]]:_(p0) = COPY $esp
   ; X32:   [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
-  ; X32:   [[GEP6:%[0-9]+]]:_(p0) = G_GEP [[COPY6]], [[C6]](s32)
+  ; X32:   [[GEP6:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY6]], [[C6]](s32)
   ; X32:   G_STORE [[LOAD]](s32), [[GEP6]](p0) :: (store 4 into stack + 24, align 1)
   ; X32:   [[COPY7:%[0-9]+]]:_(p0) = COPY $esp
   ; X32:   [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 28
-  ; X32:   [[GEP7:%[0-9]+]]:_(p0) = G_GEP [[COPY7]], [[C7]](s32)
+  ; X32:   [[GEP7:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY7]], [[C7]](s32)
   ; X32:   G_STORE [[LOAD]](s32), [[GEP7]](p0) :: (store 4 into stack + 28, align 1)
   ; X32:   CALLpcrel32 @simple_arg8_callee, csr_32, implicit $esp, implicit $ssp
   ; X32:   ADJCALLSTACKUP32 32, 0, implicit-def $esp, implicit-def $eflags, implicit-def $ssp, implicit $esp, implicit $ssp
@@ -439,11 +439,11 @@ define void @test_simple_arg8_call(i32 %in0) {
   ; X64:   $r9d = COPY [[COPY]](s32)
   ; X64:   [[COPY1:%[0-9]+]]:_(p0) = COPY $rsp
   ; X64:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-  ; X64:   [[GEP:%[0-9]+]]:_(p0) = G_GEP [[COPY1]], [[C]](s64)
+  ; X64:   [[GEP:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY1]], [[C]](s64)
   ; X64:   G_STORE [[COPY]](s32), [[GEP]](p0) :: (store 4 into stack, align 1)
   ; X64:   [[COPY2:%[0-9]+]]:_(p0) = COPY $rsp
   ; X64:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-  ; X64:   [[GEP1:%[0-9]+]]:_(p0) = G_GEP [[COPY2]], [[C1]](s64)
+  ; X64:   [[GEP1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY2]], [[C1]](s64)
   ; X64:   G_STORE [[COPY]](s32), [[GEP1]](p0) :: (store 4 into stack + 8, align 1)
   ; X64:   CALL64pcrel32 @simple_arg8_callee, csr_64, implicit $rsp, implicit $ssp, implicit $edi, implicit $esi, implicit $edx, implicit $ecx, implicit $r8d, implicit $r9d
   ; X64:   ADJCALLSTACKUP64 16, 0, implicit-def $rsp, implicit-def $eflags, implicit-def $ssp, implicit $rsp, implicit $ssp
@@ -460,7 +460,7 @@ define i32 @test_simple_return_callee() {
   ; X32:   ADJCALLSTACKDOWN32 4, 0, 0, implicit-def $esp, implicit-def $eflags, implicit-def $ssp, implicit $esp, implicit $ssp
   ; X32:   [[COPY:%[0-9]+]]:_(p0) = COPY $esp
   ; X32:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-  ; X32:   [[GEP:%[0-9]+]]:_(p0) = G_GEP [[COPY]], [[C1]](s32)
+  ; X32:   [[GEP:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C1]](s32)
   ; X32:   G_STORE [[C]](s32), [[GEP]](p0) :: (store 4 into stack, align 1)
   ; X32:   CALLpcrel32 @simple_return_callee, csr_32, implicit $esp, implicit $ssp, implicit-def $eax
   ; X32:   [[COPY1:%[0-9]+]]:_(s32) = COPY $eax
@@ -570,7 +570,7 @@ define void @test_abi_exts_call(i8* %addr) {
   ; X32:   ADJCALLSTACKDOWN32 4, 0, 0, implicit-def $esp, implicit-def $eflags, implicit-def $ssp, implicit $esp, implicit $ssp
   ; X32:   [[COPY:%[0-9]+]]:_(p0) = COPY $esp
   ; X32:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-  ; X32:   [[GEP:%[0-9]+]]:_(p0) = G_GEP [[COPY]], [[C]](s32)
+  ; X32:   [[GEP:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s32)
   ; X32:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[LOAD1]](s8)
   ; X32:   G_STORE [[ANYEXT]](s32), [[GEP]](p0) :: (store 4 into stack, align 1)
   ; X32:   CALLpcrel32 @take_char, csr_32, implicit $esp, implicit $ssp
@@ -578,7 +578,7 @@ define void @test_abi_exts_call(i8* %addr) {
   ; X32:   ADJCALLSTACKDOWN32 4, 0, 0, implicit-def $esp, implicit-def $eflags, implicit-def $ssp, implicit $esp, implicit $ssp
   ; X32:   [[COPY1:%[0-9]+]]:_(p0) = COPY $esp
   ; X32:   [[COPY2:%[0-9]+]]:_(s32) = COPY [[C]](s32)
-  ; X32:   [[GEP1:%[0-9]+]]:_(p0) = G_GEP [[COPY1]], [[COPY2]](s32)
+  ; X32:   [[GEP1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY1]], [[COPY2]](s32)
   ; X32:   [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[LOAD1]](s8)
   ; X32:   G_STORE [[SEXT]](s32), [[GEP1]](p0) :: (store 4 into stack, align 1)
   ; X32:   CALLpcrel32 @take_char, csr_32, implicit $esp, implicit $ssp
@@ -586,7 +586,7 @@ define void @test_abi_exts_call(i8* %addr) {
   ; X32:   ADJCALLSTACKDOWN32 4, 0, 0, implicit-def $esp, implicit-def $eflags, implicit-def $ssp, implicit $esp, implicit $ssp
   ; X32:   [[COPY3:%[0-9]+]]:_(p0) = COPY $esp
   ; X32:   [[COPY4:%[0-9]+]]:_(s32) = COPY [[C]](s32)
-  ; X32:   [[GEP2:%[0-9]+]]:_(p0) = G_GEP [[COPY3]], [[COPY4]](s32)
+  ; X32:   [[GEP2:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY3]], [[COPY4]](s32)
   ; X32:   [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[LOAD1]](s8)
   ; X32:   G_STORE [[ZEXT]](s32), [[GEP2]](p0) :: (store 4 into stack, align 1)
   ; X32:   CALLpcrel32 @take_char, csr_32, implicit $esp, implicit $ssp
@@ -633,11 +633,11 @@ define void @test_variadic_call_1(i8** %addr_ptr, i32* %val_ptr) {
   ; X32:   ADJCALLSTACKDOWN32 8, 0, 0, implicit-def $esp, implicit-def $eflags, implicit-def $ssp, implicit $esp, implicit $ssp
   ; X32:   [[COPY:%[0-9]+]]:_(p0) = COPY $esp
   ; X32:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-  ; X32:   [[GEP:%[0-9]+]]:_(p0) = G_GEP [[COPY]], [[C]](s32)
+  ; X32:   [[GEP:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s32)
   ; X32:   G_STORE [[LOAD2]](p0), [[GEP]](p0) :: (store 4 into stack, align 1)
   ; X32:   [[COPY1:%[0-9]+]]:_(p0) = COPY $esp
   ; X32:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-  ; X32:   [[GEP1:%[0-9]+]]:_(p0) = G_GEP [[COPY1]], [[C1]](s32)
+  ; X32:   [[GEP1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY1]], [[C1]](s32)
   ; X32:   G_STORE [[LOAD3]](s32), [[GEP1]](p0) :: (store 4 into stack + 4, align 1)
   ; X32:   CALLpcrel32 @variadic_callee, csr_32, implicit $esp, implicit $ssp
   ; X32:   ADJCALLSTACKUP32 8, 0, implicit-def $esp, implicit-def $eflags, implicit-def $ssp, implicit $esp, implicit $ssp
@@ -674,11 +674,11 @@ define void @test_variadic_call_2(i8** %addr_ptr, double* %val_ptr) {
   ; X32:   ADJCALLSTACKDOWN32 12, 0, 0, implicit-def $esp, implicit-def $eflags, implicit-def $ssp, implicit $esp, implicit $ssp
   ; X32:   [[COPY:%[0-9]+]]:_(p0) = COPY $esp
   ; X32:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-  ; X32:   [[GEP:%[0-9]+]]:_(p0) = G_GEP [[COPY]], [[C]](s32)
+  ; X32:   [[GEP:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s32)
   ; X32:   G_STORE [[LOAD2]](p0), [[GEP]](p0) :: (store 4 into stack, align 1)
   ; X32:   [[COPY1:%[0-9]+]]:_(p0) = COPY $esp
   ; X32:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-  ; X32:   [[GEP1:%[0-9]+]]:_(p0) = G_GEP [[COPY1]], [[C1]](s32)
+  ; X32:   [[GEP1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY1]], [[C1]](s32)
   ; X32:   G_STORE [[LOAD3]](s64), [[GEP1]](p0) :: (store 8 into stack + 4, align 1)
   ; X32:   CALLpcrel32 @variadic_callee, csr_32, implicit $esp, implicit $ssp
   ; X32:   ADJCALLSTACKUP32 12, 0, implicit-def $esp, implicit-def $eflags, implicit-def $ssp, implicit $esp, implicit $ssp
diff --git a/llvm/test/CodeGen/X86/GlobalISel/lshr-scalar.ll b/llvm/test/CodeGen/X86/GlobalISel/lshr-scalar.ll
index ef51cb8cb..5dd537512 100644
--- a/llvm/test/CodeGen/X86/GlobalISel/lshr-scalar.ll
+++ b/llvm/test/CodeGen/X86/GlobalISel/lshr-scalar.ll
@@ -17,8 +17,7 @@ define i64 @test_lshr_i64_imm(i64 %arg1) {
 ; X64-LABEL: test_lshr_i64_imm:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movq %rdi, %rax
-; X64-NEXT:    movq $5, %rcx
-; X64-NEXT:    shrq %cl, %rax
+; X64-NEXT:    shrq $5, %rax
 ; X64-NEXT:    retq
   %res = lshr i64 %arg1, 5
   ret i64 %res
@@ -50,8 +49,7 @@ define i32 @test_lshr_i32_imm(i32 %arg1) {
 ; X64-LABEL: test_lshr_i32_imm:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl %edi, %eax
-; X64-NEXT:    movl $5, %ecx
-; X64-NEXT:    shrl %cl, %eax
+; X64-NEXT:    shrl $5, %eax
 ; X64-NEXT:    retq
   %res = lshr i32 %arg1, 5
   ret i32 %res
@@ -86,8 +84,7 @@ define i16 @test_lshr_i16_imm(i32 %arg1) {
 ; X64-LABEL: test_lshr_i16_imm:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl %edi, %eax
-; X64-NEXT:    movw $5, %cx
-; X64-NEXT:    shrw %cl, %ax
+; X64-NEXT:    shrw $5, %ax
 ; X64-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X64-NEXT:    retq
   %a = trunc i32 %arg1 to i16
@@ -167,7 +164,7 @@ define i1 @test_lshr_i1_imm1(i32 %arg1) {
 ; X64-LABEL: test_lshr_i1_imm1:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl %edi, %eax
-; X64-NEXT:    movb $-1, %cl
+; X64-NEXT:    movb $1, %cl
 ; X64-NEXT:    andb $1, %al
 ; X64-NEXT:    andb $1, %cl
 ; X64-NEXT:    shrb %cl, %al
diff --git a/llvm/test/CodeGen/X86/GlobalISel/ptr-add.ll b/llvm/test/CodeGen/X86/GlobalISel/ptr-add.ll
new file mode 100644
index 000000000..94e8f5877
--- /dev/null
+++ b/llvm/test/CodeGen/X86/GlobalISel/ptr-add.ll
@@ -0,0 +1,140 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=x86_64-linux-gnu -global-isel -verify-machineinstrs < %s -o - | FileCheck %s --check-prefix=ALL --check-prefix=X64_GISEL
+; RUN: llc -mtriple=x86_64-linux-gnu              -verify-machineinstrs < %s -o - | FileCheck %s --check-prefix=ALL --check-prefix=X64
+
+define i32* @test_gep_i8(i32 *%arr, i8 %ind) {
+; X64_GISEL-LABEL: test_gep_i8:
+; X64_GISEL:       # %bb.0:
+; X64_GISEL-NEXT:    # kill: def $esi killed $esi def $rsi
+; X64_GISEL-NEXT:    shlq $56, %rsi
+; X64_GISEL-NEXT:    sarq $56, %rsi
+; X64_GISEL-NEXT:    movq $4, %rax
+; X64_GISEL-NEXT:    imulq %rsi, %rax
+; X64_GISEL-NEXT:    addq %rdi, %rax
+; X64_GISEL-NEXT:    retq
+;
+; X64-LABEL: test_gep_i8:
+; X64:       # %bb.0:
+; X64-NEXT:    # kill: def $esi killed $esi def $rsi
+; X64-NEXT:    movsbq %sil, %rax
+; X64-NEXT:    leaq (%rdi,%rax,4), %rax
+; X64-NEXT:    retq
+  %arrayidx = getelementptr i32, i32* %arr, i8 %ind
+  ret i32* %arrayidx
+}
+
+define i32* @test_gep_i8_const(i32 *%arr) {
+; X64_GISEL-LABEL: test_gep_i8_const:
+; X64_GISEL:       # %bb.0:
+; X64_GISEL-NEXT:    movq $80, %rax
+; X64_GISEL-NEXT:    addq %rdi, %rax
+; X64_GISEL-NEXT:    retq
+;
+; X64-LABEL: test_gep_i8_const:
+; X64:       # %bb.0:
+; X64-NEXT:    leaq 80(%rdi), %rax
+; X64-NEXT:    retq
+  %arrayidx = getelementptr i32, i32* %arr, i8 20
+  ret i32* %arrayidx
+}
+
+define i32* @test_gep_i16(i32 *%arr, i16 %ind) {
+; X64_GISEL-LABEL: test_gep_i16:
+; X64_GISEL:       # %bb.0:
+; X64_GISEL-NEXT:    # kill: def $esi killed $esi def $rsi
+; X64_GISEL-NEXT:    shlq $48, %rsi
+; X64_GISEL-NEXT:    sarq $48, %rsi
+; X64_GISEL-NEXT:    movq $4, %rax
+; X64_GISEL-NEXT:    imulq %rsi, %rax
+; X64_GISEL-NEXT:    addq %rdi, %rax
+; X64_GISEL-NEXT:    retq
+;
+; X64-LABEL: test_gep_i16:
+; X64:       # %bb.0:
+; X64-NEXT:    # kill: def $esi killed $esi def $rsi
+; X64-NEXT:    movswq %si, %rax
+; X64-NEXT:    leaq (%rdi,%rax,4), %rax
+; X64-NEXT:    retq
+  %arrayidx = getelementptr i32, i32* %arr, i16 %ind
+  ret i32* %arrayidx
+}
+
+define i32* @test_gep_i16_const(i32 *%arr) {
+; X64_GISEL-LABEL: test_gep_i16_const:
+; X64_GISEL:       # %bb.0:
+; X64_GISEL-NEXT:    movq $80, %rax
+; X64_GISEL-NEXT:    addq %rdi, %rax
+; X64_GISEL-NEXT:    retq
+;
+; X64-LABEL: test_gep_i16_const:
+; X64:       # %bb.0:
+; X64-NEXT:    leaq 80(%rdi), %rax
+; X64-NEXT:    retq
+  %arrayidx = getelementptr i32, i32* %arr, i16 20
+  ret i32* %arrayidx
+}
+
+define i32* @test_gep_i32(i32 *%arr, i32 %ind) {
+; X64_GISEL-LABEL: test_gep_i32:
+; X64_GISEL:       # %bb.0:
+; X64_GISEL-NEXT:    movslq %esi, %rax
+; X64_GISEL-NEXT:    movq $4, %rcx
+; X64_GISEL-NEXT:    imulq %rax, %rcx
+; X64_GISEL-NEXT:    leaq (%rdi,%rcx), %rax
+; X64_GISEL-NEXT:    retq
+;
+; X64-LABEL: test_gep_i32:
+; X64:       # %bb.0:
+; X64-NEXT:    movslq %esi, %rax
+; X64-NEXT:    leaq (%rdi,%rax,4), %rax
+; X64-NEXT:    retq
+  %arrayidx = getelementptr i32, i32* %arr, i32 %ind
+  ret i32* %arrayidx
+}
+
+define i32* @test_gep_i32_const(i32 *%arr) {
+; X64_GISEL-LABEL: test_gep_i32_const:
+; X64_GISEL:       # %bb.0:
+; X64_GISEL-NEXT:    movq $20, %rax
+; X64_GISEL-NEXT:    addq %rdi, %rax
+; X64_GISEL-NEXT:    retq
+;
+; X64-LABEL: test_gep_i32_const:
+; X64:       # %bb.0:
+; X64-NEXT:    leaq 20(%rdi), %rax
+; X64-NEXT:    retq
+  %arrayidx = getelementptr i32, i32* %arr, i32 5
+  ret i32* %arrayidx
+}
+
+define i32* @test_gep_i64(i32 *%arr, i64 %ind) {
+; X64_GISEL-LABEL: test_gep_i64:
+; X64_GISEL:       # %bb.0:
+; X64_GISEL-NEXT:    movq $4, %rax
+; X64_GISEL-NEXT:    imulq %rsi, %rax
+; X64_GISEL-NEXT:    addq %rdi, %rax
+; X64_GISEL-NEXT:    retq
+;
+; X64-LABEL: test_gep_i64:
+; X64:       # %bb.0:
+; X64-NEXT:    leaq (%rdi,%rsi,4), %rax
+; X64-NEXT:    retq
+  %arrayidx = getelementptr i32, i32* %arr, i64 %ind
+  ret i32* %arrayidx
+}
+
+define i32* @test_gep_i64_const(i32 *%arr) {
+; X64_GISEL-LABEL: test_gep_i64_const:
+; X64_GISEL:       # %bb.0:
+; X64_GISEL-NEXT:    movq $20, %rax
+; X64_GISEL-NEXT:    addq %rdi, %rax
+; X64_GISEL-NEXT:    retq
+;
+; X64-LABEL: test_gep_i64_const:
+; X64:       # %bb.0:
+; X64-NEXT:    leaq 20(%rdi), %rax
+; X64-NEXT:    retq
+  %arrayidx = getelementptr i32, i32* %arr, i64 5
+  ret i32* %arrayidx
+}
+
diff --git a/llvm/test/CodeGen/X86/GlobalISel/shl-scalar.ll b/llvm/test/CodeGen/X86/GlobalISel/shl-scalar.ll
index e7e134ba2..5ccc0eee5 100644
--- a/llvm/test/CodeGen/X86/GlobalISel/shl-scalar.ll
+++ b/llvm/test/CodeGen/X86/GlobalISel/shl-scalar.ll
@@ -17,8 +17,7 @@ define i64 @test_shl_i64_imm(i64 %arg1) {
 ; X64-LABEL: test_shl_i64_imm:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movq %rdi, %rax
-; X64-NEXT:    movq $5, %rcx
-; X64-NEXT:    shlq %cl, %rax
+; X64-NEXT:    shlq $5, %rax
 ; X64-NEXT:    retq
   %res = shl i64 %arg1, 5
   ret i64 %res
@@ -49,8 +48,7 @@ define i32 @test_shl_i32_imm(i32 %arg1) {
 ; X64-LABEL: test_shl_i32_imm:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl %edi, %eax
-; X64-NEXT:    movl $5, %ecx
-; X64-NEXT:    shll %cl, %eax
+; X64-NEXT:    shll $5, %eax
 ; X64-NEXT:    retq
   %res = shl i32 %arg1, 5
   ret i32 %res
@@ -85,8 +83,7 @@ define i16 @test_shl_i16_imm(i32 %arg1) {
 ; X64-LABEL: test_shl_i16_imm:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl %edi, %eax
-; X64-NEXT:    movw $5, %cx
-; X64-NEXT:    shlw %cl, %ax
+; X64-NEXT:    shlw $5, %ax
 ; X64-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X64-NEXT:    retq
   %a = trunc i32 %arg1 to i16
@@ -165,7 +162,7 @@ define i1 @test_shl_i1_imm1(i32 %arg1) {
 ; X64-LABEL: test_shl_i1_imm1:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl %edi, %eax
-; X64-NEXT:    movb $-1, %cl
+; X64-NEXT:    movb $1, %cl
 ; X64-NEXT:    andb $1, %cl
 ; X64-NEXT:    shlb %cl, %al
 ; X64-NEXT:    # kill: def $al killed $al killed $eax
diff --git a/llvm/test/CodeGen/X86/GlobalISel/x86_64-irtranslator-struct-return.ll b/llvm/test/CodeGen/X86/GlobalISel/x86_64-irtranslator-struct-return.ll
index 8b9679882..446024fa0 100644
--- a/llvm/test/CodeGen/X86/GlobalISel/x86_64-irtranslator-struct-return.ll
+++ b/llvm/test/CodeGen/X86/GlobalISel/x86_64-irtranslator-struct-return.ll
@@ -19,12 +19,7 @@ define float @test_return_f1(float %f.coerce) {
   ; ALL:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.retval
   ; ALL:   [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.1.f
   ; ALL:   G_STORE [[TRUNC]](s32), [[FRAME_INDEX1]](p0) :: (store 4 into %ir.coerce.dive2)
-  ; ALL:   ADJCALLSTACKDOWN64 0, 0, 0, implicit-def $rsp, implicit-def $eflags, implicit-def $ssp, implicit $rsp, implicit $ssp
-  ; ALL:   $rdi = COPY [[FRAME_INDEX]](p0)
-  ; ALL:   $rsi = COPY [[FRAME_INDEX1]](p0)
-  ; ALL:   $rdx = COPY [[C]](s64)
-  ; ALL:   CALL64pcrel32 &memcpy, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit $rsi, implicit $rdx
-  ; ALL:   ADJCALLSTACKUP64 0, 0, implicit-def $rsp, implicit-def $eflags, implicit-def $ssp, implicit $rsp, implicit $ssp
+  ; ALL:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[FRAME_INDEX]](p0), [[FRAME_INDEX1]](p0), [[C]](s64), 0 :: (store 1 into %ir.0, align 4), (load 1 from %ir.1, align 4)
   ; ALL:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX]](p0) :: (load 4 from %ir.coerce.dive13)
   ; ALL:   [[ANYEXT:%[0-9]+]]:_(s128) = G_ANYEXT [[LOAD]](s32)
   ; ALL:   $xmm0 = COPY [[ANYEXT]](s128)
@@ -54,12 +49,7 @@ define double @test_return_d1(double %d.coerce) {
   ; ALL:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.retval
   ; ALL:   [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.1.d
   ; ALL:   G_STORE [[TRUNC]](s64), [[FRAME_INDEX1]](p0) :: (store 8 into %ir.coerce.dive2)
-  ; ALL:   ADJCALLSTACKDOWN64 0, 0, 0, implicit-def $rsp, implicit-def $eflags, implicit-def $ssp, implicit $rsp, implicit $ssp
-  ; ALL:   $rdi = COPY [[FRAME_INDEX]](p0)
-  ; ALL:   $rsi = COPY [[FRAME_INDEX1]](p0)
-  ; ALL:   $rdx = COPY [[C]](s64)
-  ; ALL:   CALL64pcrel32 &memcpy, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit $rsi, implicit $rdx
-  ; ALL:   ADJCALLSTACKUP64 0, 0, implicit-def $rsp, implicit-def $eflags, implicit-def $ssp, implicit $rsp, implicit $ssp
+  ; ALL:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[FRAME_INDEX]](p0), [[FRAME_INDEX1]](p0), [[C]](s64), 0 :: (store 1 into %ir.0, align 8), (load 1 from %ir.1, align 8)
   ; ALL:   [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[FRAME_INDEX]](p0) :: (load 8 from %ir.coerce.dive13)
   ; ALL:   [[ANYEXT:%[0-9]+]]:_(s128) = G_ANYEXT [[LOAD]](s64)
   ; ALL:   $xmm0 = COPY [[ANYEXT]](s128)
@@ -90,16 +80,11 @@ define { double, double } @test_return_d2(double %d.coerce0, double %d.coerce1)
   ; ALL:   [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.1.d
   ; ALL:   G_STORE [[TRUNC]](s64), [[FRAME_INDEX1]](p0) :: (store 8 into %ir.1)
   ; ALL:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-  ; ALL:   [[GEP:%[0-9]+]]:_(p0) = G_GEP [[FRAME_INDEX1]], [[C1]](s64)
+  ; ALL:   [[GEP:%[0-9]+]]:_(p0) = G_PTR_ADD [[FRAME_INDEX1]], [[C1]](s64)
   ; ALL:   G_STORE [[TRUNC1]](s64), [[GEP]](p0) :: (store 8 into %ir.2)
-  ; ALL:   ADJCALLSTACKDOWN64 0, 0, 0, implicit-def $rsp, implicit-def $eflags, implicit-def $ssp, implicit $rsp, implicit $ssp
-  ; ALL:   $rdi = COPY [[FRAME_INDEX]](p0)
-  ; ALL:   $rsi = COPY [[FRAME_INDEX1]](p0)
-  ; ALL:   $rdx = COPY [[C]](s64)
-  ; ALL:   CALL64pcrel32 &memcpy, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit $rsi, implicit $rdx
-  ; ALL:   ADJCALLSTACKUP64 0, 0, implicit-def $rsp, implicit-def $eflags, implicit-def $ssp, implicit $rsp, implicit $ssp
+  ; ALL:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[FRAME_INDEX]](p0), [[FRAME_INDEX1]](p0), [[C]](s64), 0 :: (store 1 into %ir.3, align 8), (load 1 from %ir.4, align 8)
   ; ALL:   [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[FRAME_INDEX]](p0) :: (load 8 from %ir.5)
-  ; ALL:   [[GEP1:%[0-9]+]]:_(p0) = G_GEP [[FRAME_INDEX]], [[C1]](s64)
+  ; ALL:   [[GEP1:%[0-9]+]]:_(p0) = G_PTR_ADD [[FRAME_INDEX]], [[C1]](s64)
   ; ALL:   [[LOAD1:%[0-9]+]]:_(s64) = G_LOAD [[GEP1]](p0) :: (load 8 from %ir.5 + 8)
   ; ALL:   [[ANYEXT:%[0-9]+]]:_(s128) = G_ANYEXT [[LOAD]](s64)
   ; ALL:   $xmm0 = COPY [[ANYEXT]](s128)
@@ -131,12 +116,7 @@ define i32 @test_return_i1(i32 %i.coerce) {
   ; ALL:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.retval
   ; ALL:   [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.1.i
   ; ALL:   G_STORE [[COPY]](s32), [[FRAME_INDEX1]](p0) :: (store 4 into %ir.coerce.dive2)
-  ; ALL:   ADJCALLSTACKDOWN64 0, 0, 0, implicit-def $rsp, implicit-def $eflags, implicit-def $ssp, implicit $rsp, implicit $ssp
-  ; ALL:   $rdi = COPY [[FRAME_INDEX]](p0)
-  ; ALL:   $rsi = COPY [[FRAME_INDEX1]](p0)
-  ; ALL:   $rdx = COPY [[C]](s64)
-  ; ALL:   CALL64pcrel32 &memcpy, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit $rsi, implicit $rdx
-  ; ALL:   ADJCALLSTACKUP64 0, 0, implicit-def $rsp, implicit-def $eflags, implicit-def $ssp, implicit $rsp, implicit $ssp
+  ; ALL:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[FRAME_INDEX]](p0), [[FRAME_INDEX1]](p0), [[C]](s64), 0 :: (store 1 into %ir.0, align 4), (load 1 from %ir.1, align 4)
   ; ALL:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX]](p0) :: (load 4 from %ir.coerce.dive13)
   ; ALL:   $eax = COPY [[LOAD]](s32)
   ; ALL:   RET 0, implicit $eax
@@ -162,12 +142,7 @@ define i64 @test_return_i2(i64 %i.coerce) {
   ; ALL:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.retval
   ; ALL:   [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.1.i
   ; ALL:   G_STORE [[COPY]](s64), [[FRAME_INDEX1]](p0) :: (store 8 into %ir.0, align 4)
-  ; ALL:   ADJCALLSTACKDOWN64 0, 0, 0, implicit-def $rsp, implicit-def $eflags, implicit-def $ssp, implicit $rsp, implicit $ssp
-  ; ALL:   $rdi = COPY [[FRAME_INDEX]](p0)
-  ; ALL:   $rsi = COPY [[FRAME_INDEX1]](p0)
-  ; ALL:   $rdx = COPY [[C]](s64)
-  ; ALL:   CALL64pcrel32 &memcpy, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit $rsi, implicit $rdx
-  ; ALL:   ADJCALLSTACKUP64 0, 0, implicit-def $rsp, implicit-def $eflags, implicit-def $ssp, implicit $rsp, implicit $ssp
+  ; ALL:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[FRAME_INDEX]](p0), [[FRAME_INDEX1]](p0), [[C]](s64), 0 :: (store 1 into %ir.1, align 4), (load 1 from %ir.2, align 4)
   ; ALL:   [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[FRAME_INDEX]](p0) :: (load 8 from %ir.3, align 4)
   ; ALL:   $rax = COPY [[LOAD]](s64)
   ; ALL:   RET 0, implicit $rax
@@ -197,28 +172,13 @@ define { i64, i32 } @test_return_i3(i64 %i.coerce0, i32 %i.coerce1) {
   ; ALL:   [[FRAME_INDEX3:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.3.tmp
   ; ALL:   G_STORE [[COPY]](s64), [[FRAME_INDEX2]](p0) :: (store 8 into %ir.0, align 4)
   ; ALL:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-  ; ALL:   [[GEP:%[0-9]+]]:_(p0) = G_GEP [[FRAME_INDEX2]], [[C1]](s64)
+  ; ALL:   [[GEP:%[0-9]+]]:_(p0) = G_PTR_ADD [[FRAME_INDEX2]], [[C1]](s64)
   ; ALL:   G_STORE [[COPY1]](s32), [[GEP]](p0) :: (store 4 into %ir.1)
-  ; ALL:   ADJCALLSTACKDOWN64 0, 0, 0, implicit-def $rsp, implicit-def $eflags, implicit-def $ssp, implicit $rsp, implicit $ssp
-  ; ALL:   $rdi = COPY [[FRAME_INDEX1]](p0)
-  ; ALL:   $rsi = COPY [[FRAME_INDEX2]](p0)
-  ; ALL:   $rdx = COPY [[C]](s64)
-  ; ALL:   CALL64pcrel32 &memcpy, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit $rsi, implicit $rdx
-  ; ALL:   ADJCALLSTACKUP64 0, 0, implicit-def $rsp, implicit-def $eflags, implicit-def $ssp, implicit $rsp, implicit $ssp
-  ; ALL:   ADJCALLSTACKDOWN64 0, 0, 0, implicit-def $rsp, implicit-def $eflags, implicit-def $ssp, implicit $rsp, implicit $ssp
-  ; ALL:   $rdi = COPY [[FRAME_INDEX]](p0)
-  ; ALL:   $rsi = COPY [[FRAME_INDEX1]](p0)
-  ; ALL:   $rdx = COPY [[C]](s64)
-  ; ALL:   CALL64pcrel32 &memcpy, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit $rsi, implicit $rdx
-  ; ALL:   ADJCALLSTACKUP64 0, 0, implicit-def $rsp, implicit-def $eflags, implicit-def $ssp, implicit $rsp, implicit $ssp
-  ; ALL:   ADJCALLSTACKDOWN64 0, 0, 0, implicit-def $rsp, implicit-def $eflags, implicit-def $ssp, implicit $rsp, implicit $ssp
-  ; ALL:   $rdi = COPY [[FRAME_INDEX3]](p0)
-  ; ALL:   $rsi = COPY [[FRAME_INDEX]](p0)
-  ; ALL:   $rdx = COPY [[C]](s64)
-  ; ALL:   CALL64pcrel32 &memcpy, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit $rsi, implicit $rdx
-  ; ALL:   ADJCALLSTACKUP64 0, 0, implicit-def $rsp, implicit-def $eflags, implicit-def $ssp, implicit $rsp, implicit $ssp
+  ; ALL:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[FRAME_INDEX1]](p0), [[FRAME_INDEX2]](p0), [[C]](s64), 0 :: (store 1 into %ir.2, align 4), (load 1 from %ir.3, align 4)
+  ; ALL:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[FRAME_INDEX]](p0), [[FRAME_INDEX1]](p0), [[C]](s64), 0 :: (store 1 into %ir.4, align 4), (load 1 from %ir.5, align 4)
+  ; ALL:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[FRAME_INDEX3]](p0), [[FRAME_INDEX]](p0), [[C]](s64), 0 :: (store 1 into %ir.6, align 8), (load 1 from %ir.7, align 4)
   ; ALL:   [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[FRAME_INDEX3]](p0) :: (load 8 from %ir.tmp)
-  ; ALL:   [[GEP1:%[0-9]+]]:_(p0) = G_GEP [[FRAME_INDEX3]], [[C1]](s64)
+  ; ALL:   [[GEP1:%[0-9]+]]:_(p0) = G_PTR_ADD [[FRAME_INDEX3]], [[C1]](s64)
   ; ALL:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[GEP1]](p0) :: (load 4 from %ir.tmp + 8, align 8)
   ; ALL:   $rax = COPY [[LOAD]](s64)
   ; ALL:   $edx = COPY [[LOAD1]](s32)
@@ -256,16 +216,11 @@ define { i64, i64 } @test_return_i4(i64 %i.coerce0, i64 %i.coerce1) {
   ; ALL:   [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.1.i
   ; ALL:   G_STORE [[COPY]](s64), [[FRAME_INDEX1]](p0) :: (store 8 into %ir.1, align 4)
   ; ALL:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-  ; ALL:   [[GEP:%[0-9]+]]:_(p0) = G_GEP [[FRAME_INDEX1]], [[C1]](s64)
+  ; ALL:   [[GEP:%[0-9]+]]:_(p0) = G_PTR_ADD [[FRAME_INDEX1]], [[C1]](s64)
   ; ALL:   G_STORE [[COPY1]](s64), [[GEP]](p0) :: (store 8 into %ir.2, align 4)
-  ; ALL:   ADJCALLSTACKDOWN64 0, 0, 0, implicit-def $rsp, implicit-def $eflags, implicit-def $ssp, implicit $rsp, implicit $ssp
-  ; ALL:   $rdi = COPY [[FRAME_INDEX]](p0)
-  ; ALL:   $rsi = COPY [[FRAME_INDEX1]](p0)
-  ; ALL:   $rdx = COPY [[C]](s64)
-  ; ALL:   CALL64pcrel32 &memcpy, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit $rsi, implicit $rdx
-  ; ALL:   ADJCALLSTACKUP64 0, 0, implicit-def $rsp, implicit-def $eflags, implicit-def $ssp, implicit $rsp, implicit $ssp
+  ; ALL:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[FRAME_INDEX]](p0), [[FRAME_INDEX1]](p0), [[C]](s64), 0 :: (store 1 into %ir.3, align 4), (load 1 from %ir.4, align 4)
   ; ALL:   [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[FRAME_INDEX]](p0) :: (load 8 from %ir.5, align 4)
-  ; ALL:   [[GEP1:%[0-9]+]]:_(p0) = G_GEP [[FRAME_INDEX]], [[C1]](s64)
+  ; ALL:   [[GEP1:%[0-9]+]]:_(p0) = G_PTR_ADD [[FRAME_INDEX]], [[C1]](s64)
   ; ALL:   [[LOAD1:%[0-9]+]]:_(s64) = G_LOAD [[GEP1]](p0) :: (load 8 from %ir.5 + 8, align 4)
   ; ALL:   $rax = COPY [[LOAD]](s64)
   ; ALL:   $rdx = COPY [[LOAD1]](s64)
diff --git a/llvm/test/CodeGen/X86/O0-pipeline.ll b/llvm/test/CodeGen/X86/O0-pipeline.ll
index 271aec5fa..d6f2fef13 100644
--- a/llvm/test/CodeGen/X86/O0-pipeline.ll
+++ b/llvm/test/CodeGen/X86/O0-pipeline.ll
@@ -14,6 +14,7 @@
 ; CHECK-NEXT: Scoped NoAlias Alias Analysis
 ; CHECK-NEXT: Assumption Cache Tracker
 ; CHECK-NEXT: Create Garbage Collector Module Metadata
+; CHECK-NEXT: Profile summary info
 ; CHECK-NEXT: Machine Branch Probability Analysis
 ; CHECK-NEXT:   ModulePass Manager
 ; CHECK-NEXT:     Pre-ISel Intrinsic Lowering
@@ -24,6 +25,7 @@
 ; CHECK-NEXT:       Module Verifier
 ; CHECK-NEXT:       Lower Garbage Collection Instructions
 ; CHECK-NEXT:       Shadow Stack GC Lowering
+; CHECK-NEXT:       Lower constant intrinsics
 ; CHECK-NEXT:       Remove unreachable blocks from the CFG
 ; CHECK-NEXT:       Instrument function entry/exit with calls to e.g. mcount() (post inlining)
 ; CHECK-NEXT:       Scalarize Masked Memory Intrinsics
@@ -36,6 +38,10 @@
 ; CHECK-NEXT:       Safe Stack instrumentation pass
 ; CHECK-NEXT:       Insert stack protectors
 ; CHECK-NEXT:       Module Verifier
+; CHECK-NEXT:       Dominator Tree Construction
+; CHECK-NEXT:       Natural Loop Information
+; CHECK-NEXT:       Lazy Branch Probability Analysis
+; CHECK-NEXT:       Lazy Block Frequency Analysis
 ; CHECK-NEXT:       X86 DAG->DAG Instruction Selection
 ; CHECK-NEXT:       X86 PIC Global Base Reg Initialization
 ; CHECK-NEXT:       Finalize ISel and expand pseudo-instructions
@@ -55,6 +61,9 @@
 ; CHECK-NEXT:       Post-RA pseudo instruction expansion pass
 ; CHECK-NEXT:       X86 pseudo instruction expansion pass
 ; CHECK-NEXT:       Analyze Machine Code For Garbage Collection
+; CHECK-NEXT:       Insert fentry calls
+; CHECK-NEXT:       Insert XRay ops
+; CHECK-NEXT:       Implement the 'patchable-function' attribute
 ; CHECK-NEXT:       X86 Indirect Branch Tracking
 ; CHECK-NEXT:       X86 vzeroupper inserter
 ; CHECK-NEXT:       X86 Discriminate Memory Operands
@@ -62,9 +71,6 @@
 ; CHECK-NEXT:       Contiguously Lay Out Funclets
 ; CHECK-NEXT:       StackMap Liveness Analysis
 ; CHECK-NEXT:       Live DEBUG_VALUE analysis
-; CHECK-NEXT:       Insert fentry calls
-; CHECK-NEXT:       Insert XRay ops
-; CHECK-NEXT:       Implement the 'patchable-function' attribute
 ; CHECK-NEXT:       X86 Retpoline Thunks
 ; CHECK-NEXT:       Check CFA info and insert CFI instructions if needed
 ; CHECK-NEXT:       Lazy Machine Block Frequency Analysis
diff --git a/llvm/test/CodeGen/X86/O3-pipeline.ll b/llvm/test/CodeGen/X86/O3-pipeline.ll
index 48bc2992a..c645bb636 100644
--- a/llvm/test/CodeGen/X86/O3-pipeline.ll
+++ b/llvm/test/CodeGen/X86/O3-pipeline.ll
@@ -13,8 +13,8 @@
 ; CHECK-NEXT: Type-Based Alias Analysis
 ; CHECK-NEXT: Scoped NoAlias Alias Analysis
 ; CHECK-NEXT: Assumption Cache Tracker
-; CHECK-NEXT: Create Garbage Collector Module Metadata
 ; CHECK-NEXT: Profile summary info
+; CHECK-NEXT: Create Garbage Collector Module Metadata
 ; CHECK-NEXT: Machine Branch Probability Analysis
 ; CHECK-NEXT:   ModulePass Manager
 ; CHECK-NEXT:     Pre-ISel Intrinsic Lowering
@@ -32,9 +32,13 @@
 ; CHECK-NEXT:       Basic Alias Analysis (stateless AA impl)
 ; CHECK-NEXT:         Function Alias Analysis Results
 ; CHECK-NEXT:       Merge contiguous icmps into a memcmp
+; CHECK-NEXT:       Natural Loop Information
+; CHECK-NEXT:       Lazy Branch Probability Analysis
+; CHECK-NEXT:       Lazy Block Frequency Analysis
 ; CHECK-NEXT:       Expand memcmp() to load/stores
 ; CHECK-NEXT:       Lower Garbage Collection Instructions
 ; CHECK-NEXT:       Shadow Stack GC Lowering
+; CHECK-NEXT:       Lower constant intrinsics
 ; CHECK-NEXT:       Remove unreachable blocks from the CFG
 ; CHECK-NEXT:       Dominator Tree Construction
 ; CHECK-NEXT:       Natural Loop Information
@@ -63,12 +67,15 @@
 ; CHECK-NEXT:       Function Alias Analysis Results
 ; CHECK-NEXT:       Natural Loop Information
 ; CHECK-NEXT:       Branch Probability Analysis
+; CHECK-NEXT:       Lazy Branch Probability Analysis
+; CHECK-NEXT:       Lazy Block Frequency Analysis
 ; CHECK-NEXT:       X86 DAG->DAG Instruction Selection
 ; CHECK-NEXT:       MachineDominator Tree Construction
 ; CHECK-NEXT:       Local Dynamic TLS Access Clean-up
 ; CHECK-NEXT:       X86 PIC Global Base Reg Initialization
 ; CHECK-NEXT:        Finalize ISel and expand pseudo-instructions
 ; CHECK-NEXT:       X86 Domain Reassignment Pass
+; CHECK-NEXT:       Lazy Machine Block Frequency Analysis
 ; CHECK-NEXT:       Early Tail Duplication
 ; CHECK-NEXT:       Optimize machine instruction PHIs
 ; CHECK-NEXT:       Slot index numbering
@@ -79,11 +86,13 @@
 ; CHECK-NEXT:       Machine Natural Loop Construction
 ; CHECK-NEXT:       Machine Trace Metrics
 ; CHECK-NEXT:       Early If-Conversion
+; CHECK-NEXT:       Lazy Machine Block Frequency Analysis
 ; CHECK-NEXT:       Machine InstCombiner
 ; CHECK-NEXT:       X86 cmov Conversion
 ; CHECK-NEXT:       MachineDominator Tree Construction
 ; CHECK-NEXT:       Machine Natural Loop Construction
 ; CHECK-NEXT:       Early Machine Loop Invariant Code Motion
+; CHECK-NEXT:       MachineDominator Tree Construction
 ; CHECK-NEXT:       Machine Block Frequency Analysis
 ; CHECK-NEXT:       Machine Common Subexpression Elimination
 ; CHECK-NEXT:       MachinePostDominator Tree Construction
@@ -92,6 +101,7 @@
 ; CHECK-NEXT:       Remove dead machine instructions
 ; CHECK-NEXT:       Live Range Shrink
 ; CHECK-NEXT:       X86 Fixup SetCC
+; CHECK-NEXT:       Lazy Machine Block Frequency Analysis
 ; CHECK-NEXT:       X86 LEA Optimize
 ; CHECK-NEXT:       X86 Optimize Call Frame
 ; CHECK-NEXT:       X86 Avoid Store Forwarding Block
@@ -130,12 +140,14 @@
 ; CHECK-NEXT:       X86 FP Stackifier
 ; CHECK-NEXT:       PostRA Machine Sink
 ; CHECK-NEXT:       Machine Block Frequency Analysis
+; CHECK-NEXT:       MachineDominator Tree Construction
 ; CHECK-NEXT:       MachinePostDominator Tree Construction
 ; CHECK-NEXT:       Lazy Machine Block Frequency Analysis
 ; CHECK-NEXT:       Machine Optimization Remark Emitter
 ; CHECK-NEXT:       Shrink Wrapping analysis
 ; CHECK-NEXT:       Prologue/Epilogue Insertion & Frame Finalization
 ; CHECK-NEXT:       Control Flow Optimizer
+; CHECK-NEXT:       Lazy Machine Block Frequency Analysis
 ; CHECK-NEXT:       Tail Duplication
 ; CHECK-NEXT:       Machine Copy Propagation Pass
 ; CHECK-NEXT:       Post-RA pseudo instruction expansion pass
@@ -147,6 +159,9 @@
 ; CHECK-NEXT:       Machine Block Frequency Analysis
 ; CHECK-NEXT:       MachinePostDominator Tree Construction
 ; CHECK-NEXT:       Branch Probability Basic Block Placement
+; CHECK-NEXT:       Insert fentry calls
+; CHECK-NEXT:       Insert XRay ops
+; CHECK-NEXT:       Implement the 'patchable-function' attribute
 ; CHECK-NEXT:       ReachingDefAnalysis
 ; CHECK-NEXT:       X86 Execution Dependency Fix
 ; CHECK-NEXT:       BreakFalseDeps
@@ -154,7 +169,9 @@
 ; CHECK-NEXT:       X86 vzeroupper inserter
 ; CHECK-NEXT:       MachineDominator Tree Construction
 ; CHECK-NEXT:       Machine Natural Loop Construction
+; CHECK-NEXT:       Lazy Machine Block Frequency Analysis
 ; CHECK-NEXT:       X86 Byte/Word Instruction Fixup
+; CHECK-NEXT:       Lazy Machine Block Frequency Analysis
 ; CHECK-NEXT:       X86 Atom pad short functions
 ; CHECK-NEXT:       X86 LEA Fixup
 ; CHECK-NEXT:       Compressing EVEX instrs to VEX encoding when possible
@@ -163,9 +180,6 @@
 ; CHECK-NEXT:       Contiguously Lay Out Funclets
 ; CHECK-NEXT:       StackMap Liveness Analysis
 ; CHECK-NEXT:       Live DEBUG_VALUE analysis
-; CHECK-NEXT:       Insert fentry calls
-; CHECK-NEXT:       Insert XRay ops
-; CHECK-NEXT:       Implement the 'patchable-function' attribute
 ; CHECK-NEXT:       X86 Retpoline Thunks
 ; CHECK-NEXT:       Check CFA info and insert CFI instructions if needed
 ; CHECK-NEXT:       Lazy Machine Block Frequency Analysis
diff --git a/llvm/test/CodeGen/X86/SwitchLowering.ll b/llvm/test/CodeGen/X86/SwitchLowering.ll
index 6ee5152c0..8439e3c72 100644
--- a/llvm/test/CodeGen/X86/SwitchLowering.ll
+++ b/llvm/test/CodeGen/X86/SwitchLowering.ll
@@ -1,7 +1,43 @@
-; RUN: llc < %s -mtriple=i686-- | grep cmp | count 1
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- | FileCheck %s
 ; PR964
 
 define i8* @FindChar(i8* %CurPtr) {
+; CHECK-LABEL: FindChar:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushl %edi
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    pushl %esi
+; CHECK-NEXT:    .cfi_def_cfa_offset 12
+; CHECK-NEXT:    .cfi_offset %esi, -12
+; CHECK-NEXT:    .cfi_offset %edi, -8
+; CHECK-NEXT:    xorl %edi, %edi
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB0_1: # %bb
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movzbl (%esi,%edi), %eax
+; CHECK-NEXT:    incl %edi
+; CHECK-NEXT:    cmpb $120, %al
+; CHECK-NEXT:    je .LBB0_3
+; CHECK-NEXT:  # %bb.2: # %bb
+; CHECK-NEXT:    # in Loop: Header=BB0_1 Depth=1
+; CHECK-NEXT:    testb %al, %al
+; CHECK-NEXT:    jne .LBB0_1
+; CHECK-NEXT:  .LBB0_3: # %bb7
+; CHECK-NEXT:    movzbl %al, %eax
+; CHECK-NEXT:    pushl %eax
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 4
+; CHECK-NEXT:    calll foo
+; CHECK-NEXT:    addl $4, %esp
+; CHECK-NEXT:    .cfi_adjust_cfa_offset -4
+; CHECK-NEXT:    addl %edi, %esi
+; CHECK-NEXT:    movl %esi, %eax
+; CHECK-NEXT:    popl %esi
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    popl %edi
+; CHECK-NEXT:    .cfi_def_cfa_offset 4
+; CHECK-NEXT:    retl
 entry:
         br label %bb
 
diff --git a/llvm/test/CodeGen/X86/SwizzleShuff.ll b/llvm/test/CodeGen/X86/SwizzleShuff.ll
index e6519a60a..4019ee426 100644
--- a/llvm/test/CodeGen/X86/SwizzleShuff.ll
+++ b/llvm/test/CodeGen/X86/SwizzleShuff.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx -x86-experimental-vector-widening-legalization | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s
 
 ; Check that we perform a scalar XOR on i32.
 
diff --git a/llvm/test/CodeGen/X86/absolute-constant.ll b/llvm/test/CodeGen/X86/absolute-constant.ll
index d93fb276c..dba0ca629 100644
--- a/llvm/test/CodeGen/X86/absolute-constant.ll
+++ b/llvm/test/CodeGen/X86/absolute-constant.ll
@@ -1,5 +1,6 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s | FileCheck %s
-; RUN: llc -relocation-model=pic < %s | FileCheck %s
+; RUN: llc -relocation-model=pic < %s | FileCheck %s --check-prefix=PIC
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
@@ -7,10 +8,28 @@ target triple = "x86_64-unknown-linux-gnu"
 @foo = external global i8, align 1, !absolute_symbol !0
 
 define void @bar(i8* %x) {
+; CHECK-LABEL: bar:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    testb $foo, (%rdi)
+; CHECK-NEXT:    je .LBB0_1
+; CHECK-NEXT:  # %bb.2: # %if.then
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    jmp xf # TAILCALL
+; CHECK-NEXT:  .LBB0_1: # %if.end
+; CHECK-NEXT:    retq
+;
+; PIC-LABEL: bar:
+; PIC:       # %bb.0: # %entry
+; PIC-NEXT:    testb $foo, (%rdi)
+; PIC-NEXT:    je .LBB0_1
+; PIC-NEXT:  # %bb.2: # %if.then
+; PIC-NEXT:    xorl %eax, %eax
+; PIC-NEXT:    jmp xf@PLT # TAILCALL
+; PIC-NEXT:  .LBB0_1: # %if.end
+; PIC-NEXT:    retq
 entry:
   %0 = load i8, i8* %x, align 1
   %conv = sext i8 %0 to i32
-  ; CHECK: testb $foo, (%rdi)
   %and = and i32 %conv, sext (i8 ptrtoint (i8* @foo to i8) to i32)
   %tobool = icmp eq i32 %and, 0
   br i1 %tobool, label %if.end, label %if.then
diff --git a/llvm/test/CodeGen/X86/addcarry.ll b/llvm/test/CodeGen/X86/addcarry.ll
index 6c70fee99..9edcb9492 100644
--- a/llvm/test/CodeGen/X86/addcarry.ll
+++ b/llvm/test/CodeGen/X86/addcarry.ll
@@ -1,6 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-unknown | FileCheck %s
 
+declare { i8, i64 } @llvm.x86.addcarry.64(i8, i64, i64)
+declare { i64, i1 } @llvm.uadd.with.overflow.i64(i64, i64) #1
+
 define i128 @add128(i128 %a, i128 %b) nounwind {
 ; CHECK-LABEL: add128:
 ; CHECK:       # %bb.0: # %entry
@@ -411,3 +414,622 @@ define i128 @addcarry_to_subcarry(i64 %a, i64 %b) {
   %sub2 = add i128 %sum2, %notb128
   ret i128 %sub2
 }
+
+%struct.U320 = type { [5 x i64] }
+
+define i32 @add_U320_without_i128_add(%struct.U320* nocapture dereferenceable(40) %0, i64 %1, i64 %2, i64 %3, i64 %4, i64 %5) {
+; CHECK-LABEL: add_U320_without_i128_add:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    .cfi_offset %rbx, -24
+; CHECK-NEXT:    .cfi_offset %r14, -16
+; CHECK-NEXT:    movq 16(%rdi), %rax
+; CHECK-NEXT:    leaq (%rax,%rcx), %r10
+; CHECK-NEXT:    addq %rsi, (%rdi)
+; CHECK-NEXT:    adcq %rdx, 8(%rdi)
+; CHECK-NEXT:    movq %rax, %rdx
+; CHECK-NEXT:    adcq %rcx, %rdx
+; CHECK-NEXT:    movq 24(%rdi), %r11
+; CHECK-NEXT:    leaq (%r8,%r11), %r14
+; CHECK-NEXT:    xorl %ebx, %ebx
+; CHECK-NEXT:    cmpq %r10, %rdx
+; CHECK-NEXT:    setb %bl
+; CHECK-NEXT:    addq %rcx, %rax
+; CHECK-NEXT:    adcq %r14, %rbx
+; CHECK-NEXT:    movq 32(%rdi), %r10
+; CHECK-NEXT:    leaq (%r9,%r10), %rcx
+; CHECK-NEXT:    xorl %esi, %esi
+; CHECK-NEXT:    cmpq %r14, %rbx
+; CHECK-NEXT:    setb %sil
+; CHECK-NEXT:    addq %r11, %r8
+; CHECK-NEXT:    adcq %rcx, %rsi
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    cmpq %rcx, %rsi
+; CHECK-NEXT:    setb %al
+; CHECK-NEXT:    addq %r10, %r9
+; CHECK-NEXT:    movq %rdx, 16(%rdi)
+; CHECK-NEXT:    movq %rbx, 24(%rdi)
+; CHECK-NEXT:    movq %rsi, 32(%rdi)
+; CHECK-NEXT:    adcl $0, %eax
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
+  %7 = getelementptr inbounds %struct.U320, %struct.U320* %0, i64 0, i32 0, i64 0
+  %8 = load i64, i64* %7, align 8
+  %9 = getelementptr inbounds %struct.U320, %struct.U320* %0, i64 0, i32 0, i64 1
+  %10 = load i64, i64* %9, align 8
+  %11 = getelementptr inbounds %struct.U320, %struct.U320* %0, i64 0, i32 0, i64 2
+  %12 = load i64, i64* %11, align 8
+  %13 = getelementptr inbounds %struct.U320, %struct.U320* %0, i64 0, i32 0, i64 3
+  %14 = load i64, i64* %13, align 8
+  %15 = getelementptr inbounds %struct.U320, %struct.U320* %0, i64 0, i32 0, i64 4
+  %16 = load i64, i64* %15, align 8
+  %17 = add i64 %8, %1
+  %18 = add i64 %10, %2
+  %19 = icmp ult i64 %17, %1
+  %20 = zext i1 %19 to i64
+  %21 = add i64 %18, %20
+  %22 = add i64 %12, %3
+  %23 = icmp ult i64 %18, %10
+  %24 = zext i1 %23 to i64
+  %25 = icmp ult i64 %21, %18
+  %26 = zext i1 %25 to i64
+  %27 = add i64 %22, %24
+  %28 = add i64 %27, %26
+  %29 = add i64 %14, %4
+  %30 = icmp ult i64 %22, %12
+  %31 = zext i1 %30 to i64
+  %32 = icmp ult i64 %28, %22
+  %33 = zext i1 %32 to i64
+  %34 = add i64 %29, %31
+  %35 = add i64 %34, %33
+  %36 = add i64 %16, %5
+  %37 = icmp ult i64 %29, %14
+  %38 = zext i1 %37 to i64
+  %39 = icmp ult i64 %35, %29
+  %40 = zext i1 %39 to i64
+  %41 = add i64 %36, %38
+  %42 = add i64 %41, %40
+  store i64 %17, i64* %7, align 8
+  store i64 %21, i64* %9, align 8
+  store i64 %28, i64* %11, align 8
+  store i64 %35, i64* %13, align 8
+  store i64 %42, i64* %15, align 8
+  %43 = icmp ult i64 %36, %16
+  %44 = zext i1 %43 to i32
+  %45 = icmp ult i64 %42, %36
+  %46 = zext i1 %45 to i32
+  %47 = add nuw nsw i32 %46, %44
+  ret i32 %47
+}
+
+define i32 @add_U320_without_i128_or(%struct.U320* nocapture dereferenceable(40) %0, i64 %1, i64 %2, i64 %3, i64 %4, i64 %5) {
+; CHECK-LABEL: add_U320_without_i128_or:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addq %rsi, (%rdi)
+; CHECK-NEXT:    adcq %rdx, 8(%rdi)
+; CHECK-NEXT:    adcq %rcx, 16(%rdi)
+; CHECK-NEXT:    adcq %r8, 24(%rdi)
+; CHECK-NEXT:    adcq %r9, 32(%rdi)
+; CHECK-NEXT:    setb %al
+; CHECK-NEXT:    movzbl %al, %eax
+; CHECK-NEXT:    retq
+  %7 = getelementptr inbounds %struct.U320, %struct.U320* %0, i64 0, i32 0, i64 0
+  %8 = load i64, i64* %7, align 8
+  %9 = getelementptr inbounds %struct.U320, %struct.U320* %0, i64 0, i32 0, i64 1
+  %10 = load i64, i64* %9, align 8
+  %11 = getelementptr inbounds %struct.U320, %struct.U320* %0, i64 0, i32 0, i64 2
+  %12 = load i64, i64* %11, align 8
+  %13 = getelementptr inbounds %struct.U320, %struct.U320* %0, i64 0, i32 0, i64 3
+  %14 = load i64, i64* %13, align 8
+  %15 = getelementptr inbounds %struct.U320, %struct.U320* %0, i64 0, i32 0, i64 4
+  %16 = load i64, i64* %15, align 8
+  %17 = add i64 %8, %1
+  %18 = add i64 %10, %2
+  %19 = icmp ult i64 %17, %1
+  %20 = zext i1 %19 to i64
+  %21 = add i64 %18, %20
+  %22 = add i64 %12, %3
+  %23 = icmp ult i64 %18, %10
+  %24 = icmp ult i64 %21, %18
+  %25 = or i1 %23, %24
+  %26 = zext i1 %25 to i64
+  %27 = add i64 %22, %26
+  %28 = add i64 %14, %4
+  %29 = icmp ult i64 %22, %12
+  %30 = icmp ult i64 %27, %22
+  %31 = or i1 %29, %30
+  %32 = zext i1 %31 to i64
+  %33 = add i64 %28, %32
+  %34 = add i64 %16, %5
+  %35 = icmp ult i64 %28, %14
+  %36 = icmp ult i64 %33, %28
+  %37 = or i1 %35, %36
+  %38 = zext i1 %37 to i64
+  %39 = add i64 %34, %38
+  store i64 %17, i64* %7, align 8
+  store i64 %21, i64* %9, align 8
+  store i64 %27, i64* %11, align 8
+  store i64 %33, i64* %13, align 8
+  store i64 %39, i64* %15, align 8
+  %40 = icmp ult i64 %34, %16
+  %41 = icmp ult i64 %39, %34
+  %42 = or i1 %40, %41
+  %43 = zext i1 %42 to i32
+  ret i32 %43
+}
+
+define i32 @add_U320_without_i128_xor(%struct.U320* nocapture dereferenceable(40) %0, i64 %1, i64 %2, i64 %3, i64 %4, i64 %5) {
+; CHECK-LABEL: add_U320_without_i128_xor:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addq %rsi, (%rdi)
+; CHECK-NEXT:    adcq %rdx, 8(%rdi)
+; CHECK-NEXT:    adcq %rcx, 16(%rdi)
+; CHECK-NEXT:    adcq %r8, 24(%rdi)
+; CHECK-NEXT:    adcq %r9, 32(%rdi)
+; CHECK-NEXT:    setb %al
+; CHECK-NEXT:    movzbl %al, %eax
+; CHECK-NEXT:    retq
+  %7 = getelementptr inbounds %struct.U320, %struct.U320* %0, i64 0, i32 0, i64 0
+  %8 = load i64, i64* %7, align 8
+  %9 = getelementptr inbounds %struct.U320, %struct.U320* %0, i64 0, i32 0, i64 1
+  %10 = load i64, i64* %9, align 8
+  %11 = getelementptr inbounds %struct.U320, %struct.U320* %0, i64 0, i32 0, i64 2
+  %12 = load i64, i64* %11, align 8
+  %13 = getelementptr inbounds %struct.U320, %struct.U320* %0, i64 0, i32 0, i64 3
+  %14 = load i64, i64* %13, align 8
+  %15 = getelementptr inbounds %struct.U320, %struct.U320* %0, i64 0, i32 0, i64 4
+  %16 = load i64, i64* %15, align 8
+  %17 = add i64 %8, %1
+  %18 = add i64 %10, %2
+  %19 = icmp ult i64 %17, %1
+  %20 = zext i1 %19 to i64
+  %21 = add i64 %18, %20
+  %22 = add i64 %12, %3
+  %23 = icmp ult i64 %18, %10
+  %24 = icmp ult i64 %21, %18
+  %25 = xor i1 %23, %24
+  %26 = zext i1 %25 to i64
+  %27 = add i64 %22, %26
+  %28 = add i64 %14, %4
+  %29 = icmp ult i64 %22, %12
+  %30 = icmp ult i64 %27, %22
+  %31 = xor i1 %29, %30
+  %32 = zext i1 %31 to i64
+  %33 = add i64 %28, %32
+  %34 = add i64 %16, %5
+  %35 = icmp ult i64 %28, %14
+  %36 = icmp ult i64 %33, %28
+  %37 = xor i1 %35, %36
+  %38 = zext i1 %37 to i64
+  %39 = add i64 %34, %38
+  store i64 %17, i64* %7, align 8
+  store i64 %21, i64* %9, align 8
+  store i64 %27, i64* %11, align 8
+  store i64 %33, i64* %13, align 8
+  store i64 %39, i64* %15, align 8
+  %40 = icmp ult i64 %34, %16
+  %41 = icmp ult i64 %39, %34
+  %42 = xor i1 %40, %41
+  %43 = zext i1 %42 to i32
+  ret i32 %43
+}
+
+; Either the primary addition can overflow or the addition of the carry, but
+; they cannot both overflow.
+define i32 @bogus_add_U320_without_i128_and(%struct.U320* nocapture dereferenceable(40) %0, i64 %1, i64 %2, i64 %3, i64 %4, i64 %5) {
+; CHECK-LABEL: bogus_add_U320_without_i128_and:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addq %rsi, (%rdi)
+; CHECK-NEXT:    adcq %rdx, 8(%rdi)
+; CHECK-NEXT:    addq %rcx, 16(%rdi)
+; CHECK-NEXT:    addq %r8, 24(%rdi)
+; CHECK-NEXT:    addq %r9, 32(%rdi)
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    retq
+  %7 = getelementptr inbounds %struct.U320, %struct.U320* %0, i64 0, i32 0, i64 0
+  %8 = load i64, i64* %7, align 8
+  %9 = getelementptr inbounds %struct.U320, %struct.U320* %0, i64 0, i32 0, i64 1
+  %10 = load i64, i64* %9, align 8
+  %11 = getelementptr inbounds %struct.U320, %struct.U320* %0, i64 0, i32 0, i64 2
+  %12 = load i64, i64* %11, align 8
+  %13 = getelementptr inbounds %struct.U320, %struct.U320* %0, i64 0, i32 0, i64 3
+  %14 = load i64, i64* %13, align 8
+  %15 = getelementptr inbounds %struct.U320, %struct.U320* %0, i64 0, i32 0, i64 4
+  %16 = load i64, i64* %15, align 8
+  %17 = add i64 %8, %1
+  %18 = add i64 %10, %2
+  %19 = icmp ult i64 %17, %1
+  %20 = zext i1 %19 to i64
+  %21 = add i64 %18, %20
+  %22 = add i64 %12, %3
+  %23 = icmp ult i64 %18, %10
+  %24 = icmp ult i64 %21, %18
+  %25 = and i1 %23, %24
+  %26 = zext i1 %25 to i64
+  %27 = add i64 %22, %26
+  %28 = add i64 %14, %4
+  %29 = icmp ult i64 %22, %12
+  %30 = icmp ult i64 %27, %22
+  %31 = and i1 %29, %30
+  %32 = zext i1 %31 to i64
+  %33 = add i64 %28, %32
+  %34 = add i64 %16, %5
+  %35 = icmp ult i64 %28, %14
+  %36 = icmp ult i64 %33, %28
+  %37 = and i1 %35, %36
+  %38 = zext i1 %37 to i64
+  %39 = add i64 %34, %38
+  store i64 %17, i64* %7, align 8
+  store i64 %21, i64* %9, align 8
+  store i64 %27, i64* %11, align 8
+  store i64 %33, i64* %13, align 8
+  store i64 %39, i64* %15, align 8
+  %40 = icmp ult i64 %34, %16
+  %41 = icmp ult i64 %39, %34
+  %42 = and i1 %40, %41
+  %43 = zext i1 %42 to i32
+  ret i32 %43
+}
+
+define void @add_U320_without_i128_or_no_ret(%struct.U320* nocapture dereferenceable(40) %0, i64 %1, i64 %2, i64 %3, i64 %4, i64 %5) {
+; CHECK-LABEL: add_U320_without_i128_or_no_ret:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addq %rsi, (%rdi)
+; CHECK-NEXT:    adcq %rdx, 8(%rdi)
+; CHECK-NEXT:    adcq %rcx, 16(%rdi)
+; CHECK-NEXT:    adcq %r8, 24(%rdi)
+; CHECK-NEXT:    adcq %r9, 32(%rdi)
+; CHECK-NEXT:    retq
+  %7 = getelementptr inbounds %struct.U320, %struct.U320* %0, i64 0, i32 0, i64 0
+  %8 = load i64, i64* %7, align 8
+  %9 = getelementptr inbounds %struct.U320, %struct.U320* %0, i64 0, i32 0, i64 1
+  %10 = load i64, i64* %9, align 8
+  %11 = getelementptr inbounds %struct.U320, %struct.U320* %0, i64 0, i32 0, i64 2
+  %12 = load i64, i64* %11, align 8
+  %13 = getelementptr inbounds %struct.U320, %struct.U320* %0, i64 0, i32 0, i64 3
+  %14 = load i64, i64* %13, align 8
+  %15 = getelementptr inbounds %struct.U320, %struct.U320* %0, i64 0, i32 0, i64 4
+  %16 = load i64, i64* %15, align 8
+  %17 = add i64 %8, %1
+  %18 = add i64 %10, %2
+  %19 = icmp ult i64 %17, %1
+  %20 = zext i1 %19 to i64
+  %21 = add i64 %18, %20
+  %22 = add i64 %12, %3
+  %23 = icmp ult i64 %18, %10
+  %24 = icmp ult i64 %21, %18
+  %25 = or i1 %23, %24
+  %26 = zext i1 %25 to i64
+  %27 = add i64 %22, %26
+  %28 = add i64 %14, %4
+  %29 = icmp ult i64 %22, %12
+  %30 = icmp ult i64 %27, %22
+  %31 = or i1 %29, %30
+  %32 = zext i1 %31 to i64
+  %33 = add i64 %28, %32
+  %34 = add i64 %16, %5
+  %35 = icmp ult i64 %28, %14
+  %36 = icmp ult i64 %33, %28
+  %37 = or i1 %35, %36
+  %38 = zext i1 %37 to i64
+  %39 = add i64 %34, %38
+  store i64 %17, i64* %7, align 8
+  store i64 %21, i64* %9, align 8
+  store i64 %27, i64* %11, align 8
+  store i64 %33, i64* %13, align 8
+  store i64 %39, i64* %15, align 8
+  ret void
+}
+
+define i32 @add_U320_uaddo(%struct.U320* nocapture dereferenceable(40) %0, i64 %1, i64 %2, i64 %3, i64 %4, i64 %5) {
+; CHECK-LABEL: add_U320_uaddo:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addq %rsi, (%rdi)
+; CHECK-NEXT:    adcq %rdx, 8(%rdi)
+; CHECK-NEXT:    adcq %rcx, 16(%rdi)
+; CHECK-NEXT:    adcq %r8, 24(%rdi)
+; CHECK-NEXT:    adcq %r9, 32(%rdi)
+; CHECK-NEXT:    setb %al
+; CHECK-NEXT:    movzbl %al, %eax
+; CHECK-NEXT:    retq
+  %7 = getelementptr inbounds %struct.U320, %struct.U320* %0, i64 0, i32 0, i64 0
+  %8 = load i64, i64* %7, align 8
+  %9 = getelementptr inbounds %struct.U320, %struct.U320* %0, i64 0, i32 0, i64 1
+  %10 = load i64, i64* %9, align 8
+  %11 = getelementptr inbounds %struct.U320, %struct.U320* %0, i64 0, i32 0, i64 2
+  %12 = load i64, i64* %11, align 8
+  %13 = getelementptr inbounds %struct.U320, %struct.U320* %0, i64 0, i32 0, i64 3
+  %14 = load i64, i64* %13, align 8
+  %15 = getelementptr inbounds %struct.U320, %struct.U320* %0, i64 0, i32 0, i64 4
+  %16 = load i64, i64* %15, align 8
+  %17 = tail call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 %8, i64 %1)
+  %18 = extractvalue { i64, i1 } %17, 1
+  %19 = extractvalue { i64, i1 } %17, 0
+  %20 = zext i1 %18 to i64
+  %21 = tail call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 %10, i64 %2)
+  %22 = extractvalue { i64, i1 } %21, 1
+  %23 = extractvalue { i64, i1 } %21, 0
+  %24 = tail call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 %23, i64 %20)
+  %25 = extractvalue { i64, i1 } %24, 1
+  %26 = extractvalue { i64, i1 } %24, 0
+  %27 = or i1 %22, %25
+  %28 = zext i1 %27 to i64
+  %29 = tail call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 %12, i64 %3)
+  %30 = extractvalue { i64, i1 } %29, 1
+  %31 = extractvalue { i64, i1 } %29, 0
+  %32 = tail call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 %31, i64 %28)
+  %33 = extractvalue { i64, i1 } %32, 1
+  %34 = extractvalue { i64, i1 } %32, 0
+  %35 = or i1 %30, %33
+  %36 = zext i1 %35 to i64
+  %37 = tail call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 %14, i64 %4)
+  %38 = extractvalue { i64, i1 } %37, 1
+  %39 = extractvalue { i64, i1 } %37, 0
+  %40 = tail call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 %39, i64 %36)
+  %41 = extractvalue { i64, i1 } %40, 1
+  %42 = extractvalue { i64, i1 } %40, 0
+  %43 = or i1 %38, %41
+  %44 = zext i1 %43 to i64
+  %45 = tail call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 %16, i64 %5)
+  %46 = extractvalue { i64, i1 } %45, 1
+  %47 = extractvalue { i64, i1 } %45, 0
+  %48 = tail call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 %47, i64 %44)
+  %49 = extractvalue { i64, i1 } %48, 1
+  %50 = extractvalue { i64, i1 } %48, 0
+  %51 = or i1 %46, %49
+  store i64 %19, i64* %7, align 8
+  store i64 %26, i64* %9, align 8
+  store i64 %34, i64* %11, align 8
+  store i64 %42, i64* %13, align 8
+  store i64 %50, i64* %15, align 8
+  %52 = zext i1 %51 to i32
+  ret i32 %52
+}
+
+%struct.U192 = type { [3 x i64] }
+
+define void @PR39464(%struct.U192* noalias nocapture sret %0, %struct.U192* nocapture readonly dereferenceable(24) %1, %struct.U192* nocapture readonly dereferenceable(24) %2) {
+; CHECK-LABEL: PR39464:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    movq (%rsi), %rcx
+; CHECK-NEXT:    addq (%rdx), %rcx
+; CHECK-NEXT:    movq %rcx, (%rdi)
+; CHECK-NEXT:    movq 8(%rsi), %rcx
+; CHECK-NEXT:    adcq 8(%rdx), %rcx
+; CHECK-NEXT:    movq %rcx, 8(%rdi)
+; CHECK-NEXT:    movq 16(%rsi), %rcx
+; CHECK-NEXT:    adcq 16(%rdx), %rcx
+; CHECK-NEXT:    movq %rcx, 16(%rdi)
+; CHECK-NEXT:    retq
+  %4 = getelementptr inbounds %struct.U192, %struct.U192* %1, i64 0, i32 0, i64 0
+  %5 = load i64, i64* %4, align 8
+  %6 = getelementptr inbounds %struct.U192, %struct.U192* %2, i64 0, i32 0, i64 0
+  %7 = load i64, i64* %6, align 8
+  %8 = tail call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 %5, i64 %7)
+  %9 = extractvalue { i64, i1 } %8, 1
+  %10 = extractvalue { i64, i1 } %8, 0
+  %11 = zext i1 %9 to i64
+  %12 = getelementptr inbounds %struct.U192, %struct.U192* %0, i64 0, i32 0, i64 0
+  store i64 %10, i64* %12, align 8
+  %13 = getelementptr inbounds %struct.U192, %struct.U192* %1, i64 0, i32 0, i64 1
+  %14 = load i64, i64* %13, align 8
+  %15 = getelementptr inbounds %struct.U192, %struct.U192* %2, i64 0, i32 0, i64 1
+  %16 = load i64, i64* %15, align 8
+  %17 = tail call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 %14, i64 %16)
+  %18 = extractvalue { i64, i1 } %17, 1
+  %19 = extractvalue { i64, i1 } %17, 0
+  %20 = tail call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 %19, i64 %11)
+  %21 = extractvalue { i64, i1 } %20, 1
+  %22 = extractvalue { i64, i1 } %20, 0
+  %23 = or i1 %18, %21
+  %24 = zext i1 %23 to i64
+  %25 = getelementptr inbounds %struct.U192, %struct.U192* %0, i64 0, i32 0, i64 1
+  store i64 %22, i64* %25, align 8
+  %26 = getelementptr inbounds %struct.U192, %struct.U192* %1, i64 0, i32 0, i64 2
+  %27 = load i64, i64* %26, align 8
+  %28 = getelementptr inbounds %struct.U192, %struct.U192* %2, i64 0, i32 0, i64 2
+  %29 = load i64, i64* %28, align 8
+  %30 = add i64 %27, %29
+  %31 = add i64 %30, %24
+  %32 = getelementptr inbounds %struct.U192, %struct.U192* %0, i64 0, i32 0, i64 2
+  store i64 %31, i64* %32, align 8
+  ret void
+}
+
+
+%uint128 = type { i64, i64 }
+
+define zeroext i1 @uaddo_U128_without_i128_or(i64 %0, i64 %1, i64 %2, i64 %3, %uint128* nocapture %4) nounwind {
+; CHECK-LABEL: uaddo_U128_without_i128_or:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addq %rdx, %rdi
+; CHECK-NEXT:    adcq %rcx, %rsi
+; CHECK-NEXT:    setb %al
+; CHECK-NEXT:    movq %rsi, (%r8)
+; CHECK-NEXT:    movq %rdi, 8(%r8)
+; CHECK-NEXT:    retq
+  %6 = add i64 %2, %0
+  %7 = icmp ult i64 %6, %0
+  %8 = add i64 %3, %1
+  %9 = icmp ult i64 %8, %1
+  %10 = zext i1 %7 to i64
+  %11 = add i64 %8, %10
+  %12 = icmp ult i64 %11, %8
+  %13 = or i1 %9, %12
+  %14 = getelementptr inbounds %uint128, %uint128* %4, i64 0, i32 0
+  store i64 %11, i64* %14, align 8
+  %15 = getelementptr inbounds %uint128, %uint128* %4, i64 0, i32 1
+  store i64 %6, i64* %15, align 8
+  ret i1 %13
+}
+
+
+%uint192 = type { i64, i64, i64 }
+
+define void @add_U192_without_i128_or(%uint192* sret %0, i64 %1, i64 %2, i64 %3, i64 %4, i64 %5, i64 %6) nounwind {
+; CHECK-LABEL: add_U192_without_i128_or:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    addq %r8, %rsi
+; CHECK-NEXT:    adcq %r9, %rdx
+; CHECK-NEXT:    adcq {{[0-9]+}}(%rsp), %rcx
+; CHECK-NEXT:    movq %rcx, (%rdi)
+; CHECK-NEXT:    movq %rdx, 8(%rdi)
+; CHECK-NEXT:    movq %rsi, 16(%rdi)
+; CHECK-NEXT:    retq
+  %8 = add i64 %4, %1
+  %9 = icmp ult i64 %8, %1
+  %10 = add i64 %5, %2
+  %11 = icmp ult i64 %10, %2
+  %12 = zext i1 %9 to i64
+  %13 = add i64 %10, %12
+  %14 = icmp ult i64 %13, %10
+  %15 = or i1 %11, %14
+  %16 = add i64 %6, %3
+  %17 = zext i1 %15 to i64
+  %18 = add i64 %16, %17
+  %19 = getelementptr inbounds %uint192, %uint192* %0, i64 0, i32 0
+  store i64 %18, i64* %19, align 8
+  %20 = getelementptr inbounds %uint192, %uint192* %0, i64 0, i32 1
+  store i64 %13, i64* %20, align 8
+  %21 = getelementptr inbounds %uint192, %uint192* %0, i64 0, i32 2
+  store i64 %8, i64* %21, align 8
+  ret void
+}
+
+
+%uint256 = type { %uint128, %uint128 }
+
+; Classic unrolled 256-bit addition implementation using i64 as the word type.
+; It starts by adding least significant words and propagates carry to additions of the higher words.
+define void @add_U256_without_i128_or_by_i64_words(%uint256* sret %0, %uint256* %1, %uint256* %2) nounwind {
+; CHECK-LABEL: add_U256_without_i128_or_by_i64_words:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    movq (%rdx), %r8
+; CHECK-NEXT:    movq 8(%rdx), %rdi
+; CHECK-NEXT:    addq (%rsi), %r8
+; CHECK-NEXT:    adcq 8(%rsi), %rdi
+; CHECK-NEXT:    movq 16(%rdx), %rcx
+; CHECK-NEXT:    adcq 16(%rsi), %rcx
+; CHECK-NEXT:    movq 24(%rdx), %rdx
+; CHECK-NEXT:    adcq 24(%rsi), %rdx
+; CHECK-NEXT:    movq %rdx, (%rax)
+; CHECK-NEXT:    movq %rcx, 8(%rax)
+; CHECK-NEXT:    movq %rdi, 16(%rax)
+; CHECK-NEXT:    movq %r8, 24(%rax)
+; CHECK-NEXT:    retq
+  %4 = getelementptr inbounds %uint256, %uint256* %1, i64 0, i32 0, i32 0
+  %5 = load i64, i64* %4, align 8
+  %6 = getelementptr inbounds %uint256, %uint256* %2, i64 0, i32 0, i32 0
+  %7 = load i64, i64* %6, align 8
+  %8 = add i64 %7, %5
+  %9 = icmp ult i64 %8, %5
+  %10 = getelementptr inbounds %uint256, %uint256* %1, i64 0, i32 0, i32 1
+  %11 = load i64, i64* %10, align 8
+  %12 = getelementptr inbounds %uint256, %uint256* %2, i64 0, i32 0, i32 1
+  %13 = load i64, i64* %12, align 8
+  %14 = add i64 %13, %11
+  %15 = icmp ult i64 %14, %11
+  %16 = zext i1 %9 to i64
+  %17 = add i64 %14, %16
+  %18 = icmp ult i64 %17, %16
+  %19 = or i1 %15, %18
+  %20 = getelementptr inbounds %uint256, %uint256* %1, i64 0, i32 1, i32 0
+  %21 = load i64, i64* %20, align 8
+  %22 = getelementptr inbounds %uint256, %uint256* %2, i64 0, i32 1, i32 0
+  %23 = load i64, i64* %22, align 8
+  %24 = add i64 %23, %21
+  %25 = icmp ult i64 %24, %21
+  %26 = zext i1 %19 to i64
+  %27 = add i64 %24, %26
+  %28 = icmp ult i64 %27, %26
+  %29 = or i1 %25, %28
+  %30 = getelementptr inbounds %uint256, %uint256* %1, i64 0, i32 1, i32 1
+  %31 = load i64, i64* %30, align 8
+  %32 = getelementptr inbounds %uint256, %uint256* %2, i64 0, i32 1, i32 1
+  %33 = load i64, i64* %32, align 8
+  %34 = add i64 %33, %31
+  %35 = zext i1 %29 to i64
+  %36 = add i64 %34, %35
+  %37 = getelementptr inbounds %uint256, %uint256* %0, i64 0, i32 0, i32 0
+  store i64 %36, i64* %37, align 8
+  %38 = getelementptr inbounds %uint256, %uint256* %0, i64 0, i32 0, i32 1
+  store i64 %27, i64* %38, align 8
+  %39 = getelementptr inbounds %uint256, %uint256* %0, i64 0, i32 1, i32 0
+  store i64 %17, i64* %39, align 8
+  %40 = getelementptr inbounds %uint256, %uint256* %0, i64 0, i32 1, i32 1
+  store i64 %8, i64* %40, align 8
+  ret void
+}
+
+; The 256-bit addition implementation using two inlined uaddo procedures for U128 type { i64, i64 }.
+; This is similar to how LLVM legalize types in CodeGen.
+define void @add_U256_without_i128_or_recursive(%uint256* sret %0, %uint256* %1, %uint256* %2) nounwind {
+; CHECK-LABEL: add_U256_without_i128_or_recursive:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    movq (%rdx), %r8
+; CHECK-NEXT:    movq 8(%rdx), %rdi
+; CHECK-NEXT:    addq (%rsi), %r8
+; CHECK-NEXT:    adcq 8(%rsi), %rdi
+; CHECK-NEXT:    movq 16(%rdx), %rcx
+; CHECK-NEXT:    movq 24(%rdx), %rdx
+; CHECK-NEXT:    adcq 16(%rsi), %rcx
+; CHECK-NEXT:    adcq 24(%rsi), %rdx
+; CHECK-NEXT:    movq %r8, (%rax)
+; CHECK-NEXT:    movq %rdi, 8(%rax)
+; CHECK-NEXT:    movq %rcx, 16(%rax)
+; CHECK-NEXT:    movq %rdx, 24(%rax)
+; CHECK-NEXT:    retq
+  %4 = getelementptr inbounds %uint256, %uint256* %1, i64 0, i32 0, i32 0
+  %5 = load i64, i64* %4, align 8
+  %6 = getelementptr inbounds %uint256, %uint256* %1, i64 0, i32 0, i32 1
+  %7 = load i64, i64* %6, align 8
+  %8 = getelementptr inbounds %uint256, %uint256* %2, i64 0, i32 0, i32 0
+  %9 = load i64, i64* %8, align 8
+  %10 = getelementptr inbounds %uint256, %uint256* %2, i64 0, i32 0, i32 1
+  %11 = load i64, i64* %10, align 8
+  %12 = add i64 %9, %5
+  %13 = icmp ult i64 %12, %5
+  %14 = add i64 %11, %7
+  %15 = icmp ult i64 %14, %7
+  %16 = zext i1 %13 to i64
+  %17 = add i64 %14, %16
+  %18 = icmp ult i64 %17, %14
+  %19 = or i1 %15, %18
+  %20 = getelementptr inbounds %uint256, %uint256* %1, i64 0, i32 1, i32 0
+  %21 = load i64, i64* %20, align 8
+  %22 = getelementptr inbounds %uint256, %uint256* %1, i64 0, i32 1, i32 1
+  %23 = load i64, i64* %22, align 8
+  %24 = getelementptr inbounds %uint256, %uint256* %2, i64 0, i32 1, i32 0
+  %25 = load i64, i64* %24, align 8
+  %26 = getelementptr inbounds %uint256, %uint256* %2, i64 0, i32 1, i32 1
+  %27 = load i64, i64* %26, align 8
+  %28 = add i64 %25, %21
+  %29 = icmp ult i64 %28, %21
+  %30 = add i64 %27, %23
+  %31 = zext i1 %29 to i64
+  %32 = add i64 %30, %31
+  %33 = zext i1 %19 to i64
+  %34 = add i64 %28, %33
+  %35 = icmp ult i64 %34, %28
+  %36 = zext i1 %35 to i64
+  %37 = add i64 %32, %36
+  %38 = getelementptr inbounds %uint256, %uint256* %0, i64 0, i32 0, i32 0
+  store i64 %12, i64* %38, align 8
+  %39 = getelementptr inbounds %uint256, %uint256* %0, i64 0, i32 0, i32 1
+  store i64 %17, i64* %39, align 8
+  %40 = getelementptr inbounds %uint256, %uint256* %0, i64 0, i32 1, i32 0
+  store i64 %34, i64* %40, align 8
+  %41 = getelementptr inbounds %uint256, %uint256* %0, i64 0, i32 1, i32 1
+  store i64 %37, i64* %41, align 8
+  ret void
+}
diff --git a/llvm/test/CodeGen/X86/addr-label-difference.ll b/llvm/test/CodeGen/X86/addr-label-difference.ll
index 15fbec52e..acdc479d5 100644
--- a/llvm/test/CodeGen/X86/addr-label-difference.ll
+++ b/llvm/test/CodeGen/X86/addr-label-difference.ll
@@ -1,5 +1,8 @@
-; RUN: llc %s -o - | grep "__TEXT,__const"
+; RUN: llc < %s | FileCheck %s
 ; PR5929
+
+; CHECK: .section        __TEXT,__const
+
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128-n8:16:32"
 target triple = "i386-apple-darwin10.0"
 
diff --git a/llvm/test/CodeGen/X86/alias-static-alloca.ll b/llvm/test/CodeGen/X86/alias-static-alloca.ll
index f4ca7e39f..01d5e506f 100644
--- a/llvm/test/CodeGen/X86/alias-static-alloca.ll
+++ b/llvm/test/CodeGen/X86/alias-static-alloca.ll
@@ -1,22 +1,22 @@
-; RUN: llc -o - -mtriple=x86_64-linux-gnu %s | FileCheck %s
-
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
-target triple = "x86_64-unknown-linux-gnu"
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-linux-gnu | FileCheck %s
 
 ; We should be able to bypass the load values to their corresponding
 ; stores here.
 
-; CHECK-LABEL: foo
-; CHECK-DAG: movl	%esi, -8(%rsp)
-; CHECK-DAG: movl	%ecx, -16(%rsp)
-; CHECK-DAG: movl	%edi, -4(%rsp)
-; CHECK-DAG: movl	%edx, -12(%rsp)
-; CHECK: leal
-; CHECK: addl
-; CHECK: addl
-; CHECK: retq
-
 define i32 @foo(i32 %a, i32 %b, i32 %c, i32 %d) {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    # kill: def $esi killed $esi def $rsi
+; CHECK-NEXT:    # kill: def $edi killed $edi def $rdi
+; CHECK-NEXT:    movl %esi, -8(%rsp)
+; CHECK-NEXT:    movl %ecx, -16(%rsp)
+; CHECK-NEXT:    movl %edi, -4(%rsp)
+; CHECK-NEXT:    movl %edx, -12(%rsp)
+; CHECK-NEXT:    leal (%rdi,%rsi), %eax
+; CHECK-NEXT:    addl %edx, %eax
+; CHECK-NEXT:    addl %ecx, %eax
+; CHECK-NEXT:    retq
 entry:
   %a0 = alloca i32
   %a1 = alloca i32
diff --git a/llvm/test/CodeGen/X86/align-branch-boundary-default.ll b/llvm/test/CodeGen/X86/align-branch-boundary-default.ll
new file mode 100644
index 000000000..d2b443342
--- /dev/null
+++ b/llvm/test/CodeGen/X86/align-branch-boundary-default.ll
@@ -0,0 +1,37 @@
+; RUN: llc -verify-machineinstrs -O3 -mtriple=x86_64-unknown-unknown -mcpu=skylake -filetype=obj < %s | llvm-objdump -d --no-show-raw-insn - | FileCheck %s
+
+; TODO: At the moment, autopadding for SKX102 is not the default, but
+; eventually we'd like ti to be for the integrated assembler (only).
+
+target datalayout = "e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-linux-gnu"
+
+define void @test(i1 %c) {
+; CHECK: 0: pushq
+; CHECK-NEXT: 1: movl
+; CHECK-NEXT: 3: callq
+; CHECK-NEXT: 8: callq
+; CHECK-NEXT: d: callq
+; CHECK-NEXT: 12: callq
+; CHECK-NEXT: 17: callq
+; TODO: want a nop here
+; CHECK-NEXT: 1c: testb
+; CHECK-NEXT: 1f: je
+entry:
+  call void @foo()
+  call void @foo()
+  call void @foo()
+  call void @foo()
+  call void @foo()
+  br i1 %c, label %taken, label %untaken
+
+taken:
+  call void @foo()
+  ret void
+untaken:
+  call void @bar()
+  ret void
+}
+
+declare void @foo()
+declare void @bar()
diff --git a/llvm/test/CodeGen/X86/align-branch-boundary-noautopadding.ll b/llvm/test/CodeGen/X86/align-branch-boundary-noautopadding.ll
new file mode 100644
index 000000000..bebbea199
--- /dev/null
+++ b/llvm/test/CodeGen/X86/align-branch-boundary-noautopadding.ll
@@ -0,0 +1,36 @@
+; RUN: llc -verify-machineinstrs -O3 -mcpu=skylake -x86-align-branch-boundary=32 -x86-align-branch=call -filetype=obj < %s | llvm-objdump -d --no-show-raw-insn - | FileCheck %s
+
+;; This file is a companion to align-branch-boundary-suppressions.ll.
+;; It exists to demonstrate that suppressions are actually wired into the
+;; integrated assembler.
+
+target datalayout = "e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-linux-gnu"
+
+define void @test_statepoint(i32 addrspace(1)* %ptr) gc "statepoint-example" {
+; CHECK: 1: callq
+; CHECK-NEXT: 6: callq
+; CHECK-NEXT: b: callq
+; CHECK-NEXT: 10: callq
+; CHECK-NEXT: 15: callq
+; CHECK-NEXT: 1a: callq
+; CHECK-NEXT: 1f: callq
+entry:
+  ; Each of these will be 5 bytes, pushing the statepoint to offset=30.
+  ; For a normal call, this would force padding between the last normal
+  ; call and the safepoint, but since we've suppressed alignment that won't
+  ; happen for the safepoint.  That's non-ideal, we'd really prefer to do
+  ; the alignment and just keep the label with the statepoint call. (TODO)
+  call void @foo()
+  call void @foo()
+  call void @foo()
+  call void @foo()
+  call void @foo()
+  call void @foo()
+  call token (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 0)
+  ret void
+}
+
+declare void @foo()
+declare zeroext i1 @return_i1()
+declare token @llvm.experimental.gc.statepoint.p0f_i1f(i64, i32, i1 ()*, i32, i32, ...)
diff --git a/llvm/test/CodeGen/X86/align-branch-boundary-suppressions.ll b/llvm/test/CodeGen/X86/align-branch-boundary-suppressions.ll
new file mode 100644
index 000000000..292fa6b18
--- /dev/null
+++ b/llvm/test/CodeGen/X86/align-branch-boundary-suppressions.ll
@@ -0,0 +1,85 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -verify-machineinstrs -O3 -enable-implicit-null-checks -mcpu=skylake -x86-align-branch-boundary=32 -x86-align-branch=call+jmp+indirect+ret+jcc < %s | FileCheck %s
+
+;; The tests in this file check that various constructs which need to disable
+;; prefix and/or nop padding do so in the right places.  However, since we
+;; don't yet have assembler syntax for this, they're only able to check
+;; comments and must hope the assembler does the right thing.
+
+target datalayout = "e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-linux-gnu"
+
+; If we have autopadding enabled, make sure the label isn't separated from
+; the mov.
+define i32 @implicit_null_check(i32* %x) {
+; CHECK-LABEL: implicit_null_check:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    #noautopadding
+; CHECK-NEXT:  .Ltmp0:
+; CHECK-NEXT:    movl (%rdi), %eax # on-fault: .LBB0_1
+; CHECK-NEXT:    #autopadding
+; CHECK-NEXT:  # %bb.2: # %not_null
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  .LBB0_1: # %is_null
+; CHECK-NEXT:    movl $42, %eax
+; CHECK-NEXT:    retq
+
+ entry:
+  %c = icmp eq i32* %x, null
+  br i1 %c, label %is_null, label %not_null, !make.implicit !{}
+
+ is_null:
+  ret i32 42
+
+ not_null:
+  %t = load atomic i32, i32* %x unordered, align 4
+  ret i32 %t
+}
+
+; Label must bind to call before
+define void @test_statepoint(i32 addrspace(1)* %ptr) gc "statepoint-example" {
+; CHECK-LABEL: test_statepoint:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    #noautopadding
+; CHECK-NEXT:    callq return_i1
+; CHECK-NEXT:  .Ltmp1:
+; CHECK-NEXT:    #autopadding
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
+entry:
+  call token (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 0)
+  ret void
+}
+
+declare zeroext i1 @return_i1()
+declare token @llvm.experimental.gc.statepoint.p0f_i1f(i64, i32, i1 ()*, i32, i32, ...)
+
+
+; Label must bind to following nop sequence
+define void @patchpoint(i64 %a, i64 %b) {
+; CHECK-LABEL: patchpoint:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movq %rsp, %rbp
+; CHECK-NEXT:    .cfi_def_cfa_register %rbp
+; CHECK-NEXT:    #noautopadding
+; CHECK-NEXT:  .Ltmp2:
+; CHECK-NEXT:    nopw %cs:512(%rax,%rax)
+; CHECK-NEXT:    nopl 8(%rax,%rax)
+; CHECK-NEXT:    #autopadding
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa %rsp, 8
+; CHECK-NEXT:    retq
+entry:
+  call void (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.void(i64 4, i32 15, i8* null, i32 0, i64 %a, i64 %b)
+  ret void
+}
+
+
+declare void @llvm.experimental.stackmap(i64, i32, ...)
+declare void @llvm.experimental.patchpoint.void(i64, i32, i8*, i32, ...)
diff --git a/llvm/test/CodeGen/X86/align-down-const.ll b/llvm/test/CodeGen/X86/align-down-const.ll
new file mode 100644
index 000000000..86f2dacba
--- /dev/null
+++ b/llvm/test/CodeGen/X86/align-down-const.ll
@@ -0,0 +1,152 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=i686-unknown-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,X86
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,X64
+
+; Fold
+;   ptr - (ptr & C)
+; To
+;   ptr & (~C)
+;
+; This needs to be a backend-level fold because only by now pointers
+; are just registers; in middle-end IR this can only be done via @llvm.ptrmask()
+; intrinsic which is not sufficiently widely-spread yet.
+;
+; https://bugs.llvm.org/show_bug.cgi?id=44448
+
+; The basic positive tests
+
+define i32 @t0_32(i32 %ptr) nounwind {
+; X86-LABEL: t0_32:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    andl $-16, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: t0_32:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    andl $-16, %eax
+; X64-NEXT:    retq
+  %bias = and i32 %ptr, 15
+  %r = sub i32 %ptr, %bias
+  ret i32 %r
+}
+define i64 @t1_64(i64 %ptr) nounwind {
+; X86-LABEL: t1_64:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    andl $-16, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: t1_64:
+; X64:       # %bb.0:
+; X64-NEXT:    movq %rdi, %rax
+; X64-NEXT:    andq $-16, %rax
+; X64-NEXT:    retq
+  %bias = and i64 %ptr, 15
+  %r = sub i64 %ptr, %bias
+  ret i64 %r
+}
+
+define i32 @t2_powerof2(i32 %ptr) nounwind {
+; X86-LABEL: t2_powerof2:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    andl $-17, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: t2_powerof2:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    andl $-17, %eax
+; X64-NEXT:    retq
+  %bias = and i32 %ptr, 16
+  %r = sub i32 %ptr, %bias
+  ret i32 %r
+}
+define i32 @t3_random_constant(i32 %ptr) nounwind {
+; X86-LABEL: t3_random_constant:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    andl $-43, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: t3_random_constant:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    andl $-43, %eax
+; X64-NEXT:    retq
+  %bias = and i32 %ptr, 42
+  %r = sub i32 %ptr, %bias
+  ret i32 %r
+}
+
+; Extra use tests
+
+define i32 @t4_extrause(i32 %ptr, i32* %bias_storage) nounwind {
+; X86-LABEL: t4_extrause:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl %eax, %edx
+; X86-NEXT:    andl $15, %edx
+; X86-NEXT:    movl %edx, (%ecx)
+; X86-NEXT:    andl $-16, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: t4_extrause:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    movl %edi, %ecx
+; X64-NEXT:    andl $15, %ecx
+; X64-NEXT:    movl %ecx, (%rsi)
+; X64-NEXT:    andl $-16, %eax
+; X64-NEXT:    retq
+  %bias = and i32 %ptr, 15
+  store i32 %bias, i32* %bias_storage
+  %r = sub i32 %ptr, %bias
+  ret i32 %r
+}
+
+; Negative tests
+
+define i32 @n5_different_ptrs(i32 %ptr0, i32 %ptr1) nounwind {
+; X86-LABEL: n5_different_ptrs:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    andl $15, %ecx
+; X86-NEXT:    subl %ecx, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: n5_different_ptrs:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    andl $15, %esi
+; X64-NEXT:    subl %esi, %eax
+; X64-NEXT:    retq
+  %bias = and i32 %ptr1, 15 ; not %ptr0
+  %r = sub i32 %ptr0, %bias ; not %ptr1
+  ret i32 %r
+}
+
+define i32 @n6_sub_is_not_commutative(i32 %ptr) nounwind {
+; X86-LABEL: n6_sub_is_not_commutative:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    andl $15, %eax
+; X86-NEXT:    subl %ecx, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: n6_sub_is_not_commutative:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    andl $15, %eax
+; X64-NEXT:    subl %edi, %eax
+; X64-NEXT:    retq
+  %bias = and i32 %ptr, 15
+  %r = sub i32 %bias, %ptr ; wrong order
+  ret i32 %r
+}
diff --git a/llvm/test/CodeGen/X86/align-down.ll b/llvm/test/CodeGen/X86/align-down.ll
new file mode 100644
index 000000000..e14d31221
--- /dev/null
+++ b/llvm/test/CodeGen/X86/align-down.ll
@@ -0,0 +1,272 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=i686-unknown-linux-gnu             < %s | FileCheck %s   --check-prefixes=CHECK,NOBMI,X86,NOBMI-X86
+; RUN: llc -mtriple=i686-unknown-linux-gnu -mattr=+bmi < %s | FileCheck %s   --check-prefixes=CHECK,BMI,X86,BMI-X86
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu             < %s | FileCheck %s --check-prefixes=CHECK,NOBMI,X64,NOBMI-X64
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu -mattr=+bmi < %s | FileCheck %s --check-prefixes=CHECK,BMI,X64,BMI-X64
+
+; Fold
+;   ptr - (ptr & (alignment-1))
+; To
+;   ptr & (0 - alignment)
+;
+; This needs to be a backend-level fold because only by now pointers
+; are just registers; in middle-end IR this can only be done via @llvm.ptrmask()
+; intrinsic which is not sufficiently widely-spread yet.
+;
+; https://bugs.llvm.org/show_bug.cgi?id=44448
+
+; The basic positive tests
+
+define i32 @t0_32(i32 %ptr, i32 %alignment) nounwind {
+; X86-LABEL: t0_32:
+; X86:       # %bb.0:
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    subl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    andl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: t0_32:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %esi, %eax
+; X64-NEXT:    negl %eax
+; X64-NEXT:    andl %edi, %eax
+; X64-NEXT:    retq
+  %mask = add i32 %alignment, -1
+  %bias = and i32 %ptr, %mask
+  %r = sub i32 %ptr, %bias
+  ret i32 %r
+}
+define i64 @t1_64(i64 %ptr, i64 %alignment) nounwind {
+; X86-LABEL: t1_64:
+; X86:       # %bb.0:
+; X86-NEXT:    xorl %edx, %edx
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    subl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    sbbl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    andl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    andl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: t1_64:
+; X64:       # %bb.0:
+; X64-NEXT:    movq %rsi, %rax
+; X64-NEXT:    negq %rax
+; X64-NEXT:    andq %rdi, %rax
+; X64-NEXT:    retq
+  %mask = add i64 %alignment, -1
+  %bias = and i64 %ptr, %mask
+  %r = sub i64 %ptr, %bias
+  ret i64 %r
+}
+
+define i32 @t2_commutative(i32 %ptr, i32 %alignment) nounwind {
+; X86-LABEL: t2_commutative:
+; X86:       # %bb.0:
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    subl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    andl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: t2_commutative:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %esi, %eax
+; X64-NEXT:    negl %eax
+; X64-NEXT:    andl %edi, %eax
+; X64-NEXT:    retq
+  %mask = add i32 %alignment, -1
+  %bias = and i32 %mask, %ptr ; swapped
+  %r = sub i32 %ptr, %bias
+  ret i32 %r
+}
+
+; Extra use tests
+
+define i32 @t3_extrause0(i32 %ptr, i32 %alignment, i32* %mask_storage) nounwind {
+; X86-LABEL: t3_extrause0:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    leal -1(%eax), %edx
+; X86-NEXT:    movl %edx, (%ecx)
+; X86-NEXT:    negl %eax
+; X86-NEXT:    andl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: t3_extrause0:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %esi, %eax
+; X64-NEXT:    leal -1(%rax), %ecx
+; X64-NEXT:    movl %ecx, (%rdx)
+; X64-NEXT:    negl %eax
+; X64-NEXT:    andl %edi, %eax
+; X64-NEXT:    # kill: def $eax killed $eax killed $rax
+; X64-NEXT:    retq
+  %mask = add i32 %alignment, -1
+  store i32 %mask, i32* %mask_storage
+  %bias = and i32 %ptr, %mask
+  %r = sub i32 %ptr, %bias
+  ret i32 %r
+}
+define i32 @n4_extrause1(i32 %ptr, i32 %alignment, i32* %bias_storage) nounwind {
+; X86-LABEL: n4_extrause1:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    decl %edx
+; X86-NEXT:    andl %eax, %edx
+; X86-NEXT:    movl %edx, (%ecx)
+; X86-NEXT:    subl %edx, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: n4_extrause1:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    decl %esi
+; X64-NEXT:    andl %edi, %esi
+; X64-NEXT:    movl %esi, (%rdx)
+; X64-NEXT:    subl %esi, %eax
+; X64-NEXT:    retq
+  %mask = add i32 %alignment, -1
+  %bias = and i32 %ptr, %mask ; has extra uses, can't fold
+  store i32 %bias, i32* %bias_storage
+  %r = sub i32 %ptr, %bias
+  ret i32 %r
+}
+define i32 @n5_extrause2(i32 %ptr, i32 %alignment, i32* %mask_storage, i32* %bias_storage) nounwind {
+; X86-LABEL: n5_extrause2:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    decl %esi
+; X86-NEXT:    movl %esi, (%edx)
+; X86-NEXT:    andl %eax, %esi
+; X86-NEXT:    movl %esi, (%ecx)
+; X86-NEXT:    subl %esi, %eax
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
+;
+; X64-LABEL: n5_extrause2:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    decl %esi
+; X64-NEXT:    movl %esi, (%rdx)
+; X64-NEXT:    andl %edi, %esi
+; X64-NEXT:    movl %esi, (%rcx)
+; X64-NEXT:    subl %esi, %eax
+; X64-NEXT:    retq
+  %mask = add i32 %alignment, -1
+  store i32 %mask, i32* %mask_storage
+  %bias = and i32 %ptr, %mask ; has extra uses, can't fold
+  store i32 %bias, i32* %bias_storage
+  %r = sub i32 %ptr, %bias
+  ret i32 %r
+}
+
+; Negative tests
+
+define i32 @n6_different_ptrs(i32 %ptr0, i32 %ptr1, i32 %alignment) nounwind {
+; X86-LABEL: n6_different_ptrs:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    decl %ecx
+; X86-NEXT:    andl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    subl %ecx, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: n6_different_ptrs:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    decl %edx
+; X64-NEXT:    andl %esi, %edx
+; X64-NEXT:    subl %edx, %eax
+; X64-NEXT:    retq
+  %mask = add i32 %alignment, -1
+  %bias = and i32 %ptr1, %mask ; not %ptr0
+  %r = sub i32 %ptr0, %bias ; not %ptr1
+  ret i32 %r
+}
+define i32 @n7_different_ptrs_commutative(i32 %ptr0, i32 %ptr1, i32 %alignment) nounwind {
+; X86-LABEL: n7_different_ptrs_commutative:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    decl %ecx
+; X86-NEXT:    andl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    subl %ecx, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: n7_different_ptrs_commutative:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    decl %edx
+; X64-NEXT:    andl %esi, %edx
+; X64-NEXT:    subl %edx, %eax
+; X64-NEXT:    retq
+  %mask = add i32 %alignment, -1
+  %bias = and i32 %mask, %ptr1 ; swapped, not %ptr0
+  %r = sub i32 %ptr0, %bias ; not %ptr1
+  ret i32 %r
+}
+
+define i32 @n8_not_lowbit_mask(i32 %ptr, i32 %alignment) nounwind {
+; NOBMI-X86-LABEL: n8_not_lowbit_mask:
+; NOBMI-X86:       # %bb.0:
+; NOBMI-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; NOBMI-X86-NEXT:    incl %eax
+; NOBMI-X86-NEXT:    notl %eax
+; NOBMI-X86-NEXT:    andl {{[0-9]+}}(%esp), %eax
+; NOBMI-X86-NEXT:    retl
+;
+; BMI-X86-LABEL: n8_not_lowbit_mask:
+; BMI-X86:       # %bb.0:
+; BMI-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; BMI-X86-NEXT:    incl %eax
+; BMI-X86-NEXT:    andnl {{[0-9]+}}(%esp), %eax, %eax
+; BMI-X86-NEXT:    retl
+;
+; NOBMI-X64-LABEL: n8_not_lowbit_mask:
+; NOBMI-X64:       # %bb.0:
+; NOBMI-X64-NEXT:    movl %esi, %eax
+; NOBMI-X64-NEXT:    incl %eax
+; NOBMI-X64-NEXT:    notl %eax
+; NOBMI-X64-NEXT:    andl %edi, %eax
+; NOBMI-X64-NEXT:    retq
+;
+; BMI-X64-LABEL: n8_not_lowbit_mask:
+; BMI-X64:       # %bb.0:
+; BMI-X64-NEXT:    incl %esi
+; BMI-X64-NEXT:    andnl %edi, %esi, %eax
+; BMI-X64-NEXT:    retq
+  %mask = add i32 %alignment, 1 ; not -1
+  %bias = and i32 %ptr, %mask
+  %r = sub i32 %ptr, %bias
+  ret i32 %r
+}
+
+define i32 @n9_sub_is_not_commutative(i32 %ptr, i32 %alignment) nounwind {
+; X86-LABEL: n9_sub_is_not_commutative:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    decl %eax
+; X86-NEXT:    andl %ecx, %eax
+; X86-NEXT:    subl %ecx, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: n9_sub_is_not_commutative:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %esi, %eax
+; X64-NEXT:    decl %eax
+; X64-NEXT:    andl %edi, %eax
+; X64-NEXT:    subl %edi, %eax
+; X64-NEXT:    retq
+  %mask = add i32 %alignment, -1
+  %bias = and i32 %ptr, %mask
+  %r = sub i32 %bias, %ptr ; wrong order
+  ret i32 %r
+}
diff --git a/llvm/test/CodeGen/X86/aligned-comm.ll b/llvm/test/CodeGen/X86/aligned-comm.ll
index 1192b72c4..ba6b670ef 100644
--- a/llvm/test/CodeGen/X86/aligned-comm.ll
+++ b/llvm/test/CodeGen/X86/aligned-comm.ll
@@ -1,6 +1,11 @@
-; RUN: llc < %s -mtriple=i686--
-; RUN: llc < %s -mtriple=i386-apple-darwin10 | grep "array,16512,7"
-; RUN: llc < %s -mtriple=i386-apple-darwin9 | grep "array,16512,7"
+; RUN: llc < %s -mtriple=i686-- | FileCheck %s --check-prefix=i686
+; RUN: llc < %s -mtriple=i386-apple-darwin10 | FileCheck %s --check-prefix=DARWIN
+; RUN: llc < %s -mtriple=i386-apple-darwin9 | FileCheck %s --check-prefix=DARWIN
+; RUN: llc < %s -mtriple=i386-apple-darwin8 | FileCheck %s --check-prefix=DARWIN8
+
+; i686: array,16512
+; DARWIN8: array,16512
+; DARWIN: array,16512,7
 
 ; Darwin 9+ should get alignment on common symbols.
 @array = common global [4128 x i32] zeroinitializer, align 128
diff --git a/llvm/test/CodeGen/X86/all-ones-vector.ll b/llvm/test/CodeGen/X86/all-ones-vector.ll
index d64b3d7e2..cead6acb1 100644
--- a/llvm/test/CodeGen/X86/all-ones-vector.ll
+++ b/llvm/test/CodeGen/X86/all-ones-vector.ll
@@ -466,16 +466,10 @@ define <64 x i8> @allones_v64i8() nounwind {
 ; X32-AVX2-NEXT:    vpcmpeqd %ymm1, %ymm1, %ymm1
 ; X32-AVX2-NEXT:    retl
 ;
-; X32-KNL-LABEL: allones_v64i8:
-; X32-KNL:       # %bb.0:
-; X32-KNL-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
-; X32-KNL-NEXT:    vpcmpeqd %ymm1, %ymm1, %ymm1
-; X32-KNL-NEXT:    retl
-;
-; X32-SKX-LABEL: allones_v64i8:
-; X32-SKX:       # %bb.0:
-; X32-SKX-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0
-; X32-SKX-NEXT:    retl
+; X32-AVX512-LABEL: allones_v64i8:
+; X32-AVX512:       # %bb.0:
+; X32-AVX512-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0
+; X32-AVX512-NEXT:    retl
 ;
 ; X64-SSE-LABEL: allones_v64i8:
 ; X64-SSE:       # %bb.0:
@@ -498,16 +492,10 @@ define <64 x i8> @allones_v64i8() nounwind {
 ; X64-AVX2-NEXT:    vpcmpeqd %ymm1, %ymm1, %ymm1
 ; X64-AVX2-NEXT:    retq
 ;
-; X64-KNL-LABEL: allones_v64i8:
-; X64-KNL:       # %bb.0:
-; X64-KNL-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
-; X64-KNL-NEXT:    vpcmpeqd %ymm1, %ymm1, %ymm1
-; X64-KNL-NEXT:    retq
-;
-; X64-SKX-LABEL: allones_v64i8:
-; X64-SKX:       # %bb.0:
-; X64-SKX-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0
-; X64-SKX-NEXT:    retq
+; X64-AVX512-LABEL: allones_v64i8:
+; X64-AVX512:       # %bb.0:
+; X64-AVX512-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0
+; X64-AVX512-NEXT:    retq
   ret <64 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
 }
 
@@ -533,16 +521,10 @@ define <32 x i16> @allones_v32i16() nounwind {
 ; X32-AVX2-NEXT:    vpcmpeqd %ymm1, %ymm1, %ymm1
 ; X32-AVX2-NEXT:    retl
 ;
-; X32-KNL-LABEL: allones_v32i16:
-; X32-KNL:       # %bb.0:
-; X32-KNL-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
-; X32-KNL-NEXT:    vpcmpeqd %ymm1, %ymm1, %ymm1
-; X32-KNL-NEXT:    retl
-;
-; X32-SKX-LABEL: allones_v32i16:
-; X32-SKX:       # %bb.0:
-; X32-SKX-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0
-; X32-SKX-NEXT:    retl
+; X32-AVX512-LABEL: allones_v32i16:
+; X32-AVX512:       # %bb.0:
+; X32-AVX512-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0
+; X32-AVX512-NEXT:    retl
 ;
 ; X64-SSE-LABEL: allones_v32i16:
 ; X64-SSE:       # %bb.0:
@@ -565,16 +547,10 @@ define <32 x i16> @allones_v32i16() nounwind {
 ; X64-AVX2-NEXT:    vpcmpeqd %ymm1, %ymm1, %ymm1
 ; X64-AVX2-NEXT:    retq
 ;
-; X64-KNL-LABEL: allones_v32i16:
-; X64-KNL:       # %bb.0:
-; X64-KNL-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
-; X64-KNL-NEXT:    vpcmpeqd %ymm1, %ymm1, %ymm1
-; X64-KNL-NEXT:    retq
-;
-; X64-SKX-LABEL: allones_v32i16:
-; X64-SKX:       # %bb.0:
-; X64-SKX-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0
-; X64-SKX-NEXT:    retq
+; X64-AVX512-LABEL: allones_v32i16:
+; X64-AVX512:       # %bb.0:
+; X64-AVX512-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0
+; X64-AVX512-NEXT:    retq
   ret <32 x i16> <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>
 }
 
diff --git a/llvm/test/CodeGen/X86/and-load-fold.ll b/llvm/test/CodeGen/X86/and-load-fold.ll
index 367ef2a34..a2454e031 100644
--- a/llvm/test/CodeGen/X86/and-load-fold.ll
+++ b/llvm/test/CodeGen/X86/and-load-fold.ll
@@ -6,10 +6,8 @@
 define i8 @foo(<4 x i8>* %V) {
 ; CHECK-LABEL: foo:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; CHECK-NEXT:    pextrw $1, %xmm0, %eax
+; CHECK-NEXT:    movb 2(%rdi), %al
 ; CHECK-NEXT:    andb $95, %al
-; CHECK-NEXT:    # kill: def $al killed $al killed $eax
 ; CHECK-NEXT:    retq
   %Vp = bitcast <4 x i8>* %V to <3 x i8>*
   %V3i8 = load <3 x i8>, <3 x i8>* %Vp, align 4
diff --git a/llvm/test/CodeGen/X86/anyext.ll b/llvm/test/CodeGen/X86/anyext.ll
index 93206737e..7833658ff 100644
--- a/llvm/test/CodeGen/X86/anyext.ll
+++ b/llvm/test/CodeGen/X86/anyext.ll
@@ -8,7 +8,6 @@ define i32 @foo(i32 %p, i8 zeroext %x) nounwind {
 ; X32-LABEL: foo:
 ; X32:       # %bb.0:
 ; X32-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    # kill: def $eax killed $eax def $ax
 ; X32-NEXT:    divb {{[0-9]+}}(%esp)
 ; X32-NEXT:    movzbl %al, %eax
 ; X32-NEXT:    andl $1, %eax
@@ -17,7 +16,6 @@ define i32 @foo(i32 %p, i8 zeroext %x) nounwind {
 ; X64-LABEL: foo:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movzbl %dil, %eax
-; X64-NEXT:    # kill: def $eax killed $eax def $ax
 ; X64-NEXT:    divb %sil
 ; X64-NEXT:    movzbl %al, %eax
 ; X64-NEXT:    andl $1, %eax
diff --git a/llvm/test/CodeGen/X86/arg-cast.ll b/llvm/test/CodeGen/X86/arg-cast.ll
index c11151446..9559ff64e 100644
--- a/llvm/test/CodeGen/X86/arg-cast.ll
+++ b/llvm/test/CodeGen/X86/arg-cast.ll
@@ -1,13 +1,17 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; This should compile to movl $2147483647, %eax + andl only.
-; RUN: llc < %s | grep andl
-; RUN: llc < %s | not grep movsd
-; RUN: llc < %s | grep esp | not grep add
+; RUN: llc < %s | FileCheck %s
 ; rdar://5736574
 
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
 target triple = "i686-apple-darwin8"
 
 define i32 @foo(double %x) nounwind  {
+; CHECK-LABEL: foo:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    movl $2147483647, %eax ## imm = 0x7FFFFFFF
+; CHECK-NEXT:    andl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    retl
 entry:
 	%x15 = bitcast double %x to i64		; <i64> [#uses=1]
 	%tmp713 = lshr i64 %x15, 32		; <i64> [#uses=1]
diff --git a/llvm/test/CodeGen/X86/atom-fixup-lea4.ll b/llvm/test/CodeGen/X86/atom-fixup-lea4.ll
index f0da1d201..a0bfcd889 100644
--- a/llvm/test/CodeGen/X86/atom-fixup-lea4.ll
+++ b/llvm/test/CodeGen/X86/atom-fixup-lea4.ll
@@ -19,5 +19,5 @@ entry:
 ; Function Attrs: uwtable
 declare void @_ZN12ValueWrapperIS_IS_IdEEEC2Ev(%struct.ValueWrapper.7*) unnamed_addr #0 align 2
 
-attributes #0 = { uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
diff --git a/llvm/test/CodeGen/X86/atom-pad-short-functions.ll b/llvm/test/CodeGen/X86/atom-pad-short-functions.ll
index b9a39e08c..debf14e14 100644
--- a/llvm/test/CodeGen/X86/atom-pad-short-functions.ll
+++ b/llvm/test/CodeGen/X86/atom-pad-short-functions.ll
@@ -29,6 +29,13 @@ define i32 @test_minsize(i32 %a) nounwind minsize {
   ret i32 %a
 }
 
+define i32 @test_pgso(i32 %a) nounwind !prof !14 {
+; CHECK: test_pgso
+; CHECK: movl
+; CHECK-NEXT: ret
+  ret i32 %a
+}
+
 define i32 @test_add(i32 %a, i32 %b) nounwind {
 ; CHECK: test_add
 ; CHECK: addl
@@ -101,3 +108,19 @@ while.end:
   ret void
 }
 
+!llvm.module.flags = !{!0}
+!0 = !{i32 1, !"ProfileSummary", !1}
+!1 = !{!2, !3, !4, !5, !6, !7, !8, !9}
+!2 = !{!"ProfileFormat", !"InstrProf"}
+!3 = !{!"TotalCount", i64 10000}
+!4 = !{!"MaxCount", i64 10}
+!5 = !{!"MaxInternalCount", i64 1}
+!6 = !{!"MaxFunctionCount", i64 1000}
+!7 = !{!"NumCounts", i64 3}
+!8 = !{!"NumFunctions", i64 3}
+!9 = !{!"DetailedSummary", !10}
+!10 = !{!11, !12, !13}
+!11 = !{i32 10000, i64 100, i32 1}
+!12 = !{i32 999000, i64 100, i32 1}
+!13 = !{i32 999999, i64 1, i32 2}
+!14 = !{!"function_entry_count", i64 0}
diff --git a/llvm/test/CodeGen/X86/atomic-monotonic.ll b/llvm/test/CodeGen/X86/atomic-monotonic.ll
index 73eaeba14..9cab2d7d2 100644
--- a/llvm/test/CodeGen/X86/atomic-monotonic.ll
+++ b/llvm/test/CodeGen/X86/atomic-monotonic.ll
@@ -1,17 +1,12 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -O0 < %s -mtriple=x86_64-linux-generic -verify-machineinstrs -mattr=sse2 | FileCheck --check-prefix=CHECK-O0 %s
-; RUN: llc -O3 < %s -mtriple=x86_64-linux-generic -verify-machineinstrs -mattr=sse2 | FileCheck --check-prefix=CHECK-O3 %s
+; RUN: llc -O0 < %s -mtriple=x86_64-linux-generic -verify-machineinstrs -mattr=sse2 | FileCheck --check-prefixes=CHECK,CHECK-O0 %s
+; RUN: llc -O3 < %s -mtriple=x86_64-linux-generic -verify-machineinstrs -mattr=sse2 | FileCheck --check-prefixes=CHECK,CHECK-O3 %s
 
 define i8 @load_i8(i8* %ptr) {
-; CHECK-O0-LABEL: load_i8:
-; CHECK-O0:       # %bb.0:
-; CHECK-O0-NEXT:    movb (%rdi), %al
-; CHECK-O0-NEXT:    retq
-;
-; CHECK-O3-LABEL: load_i8:
-; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    movb (%rdi), %al
-; CHECK-O3-NEXT:    retq
+; CHECK-LABEL: load_i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movb (%rdi), %al
+; CHECK-NEXT:    retq
   %v = load atomic i8, i8* %ptr monotonic, align 1
   ret i8 %v
 }
@@ -62,57 +57,37 @@ define void @store_i16(i16* %ptr, i16 %v) {
 }
 
 define i32 @load_i32(i32* %ptr) {
-; CHECK-O0-LABEL: load_i32:
-; CHECK-O0:       # %bb.0:
-; CHECK-O0-NEXT:    movl (%rdi), %eax
-; CHECK-O0-NEXT:    retq
-;
-; CHECK-O3-LABEL: load_i32:
-; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    movl (%rdi), %eax
-; CHECK-O3-NEXT:    retq
+; CHECK-LABEL: load_i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl (%rdi), %eax
+; CHECK-NEXT:    retq
   %v = load atomic i32, i32* %ptr monotonic, align 4
   ret i32 %v
 }
 
 define void @store_i32(i32* %ptr, i32 %v) {
-; CHECK-O0-LABEL: store_i32:
-; CHECK-O0:       # %bb.0:
-; CHECK-O0-NEXT:    movl %esi, (%rdi)
-; CHECK-O0-NEXT:    retq
-;
-; CHECK-O3-LABEL: store_i32:
-; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    movl %esi, (%rdi)
-; CHECK-O3-NEXT:    retq
+; CHECK-LABEL: store_i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %esi, (%rdi)
+; CHECK-NEXT:    retq
   store atomic i32 %v, i32* %ptr monotonic, align 4
   ret void
 }
 
 define i64 @load_i64(i64* %ptr) {
-; CHECK-O0-LABEL: load_i64:
-; CHECK-O0:       # %bb.0:
-; CHECK-O0-NEXT:    movq (%rdi), %rax
-; CHECK-O0-NEXT:    retq
-;
-; CHECK-O3-LABEL: load_i64:
-; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    movq (%rdi), %rax
-; CHECK-O3-NEXT:    retq
+; CHECK-LABEL: load_i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq (%rdi), %rax
+; CHECK-NEXT:    retq
   %v = load atomic i64, i64* %ptr monotonic, align 8
   ret i64 %v
 }
 
 define void @store_i64(i64* %ptr, i64 %v) {
-; CHECK-O0-LABEL: store_i64:
-; CHECK-O0:       # %bb.0:
-; CHECK-O0-NEXT:    movq %rsi, (%rdi)
-; CHECK-O0-NEXT:    retq
-;
-; CHECK-O3-LABEL: store_i64:
-; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    movq %rsi, (%rdi)
-; CHECK-O3-NEXT:    retq
+; CHECK-LABEL: store_i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %rsi, (%rdi)
+; CHECK-NEXT:    retq
   store atomic i64 %v, i64* %ptr monotonic, align 8
   ret void
 }
diff --git a/llvm/test/CodeGen/X86/atomic-non-integer-fp128.ll b/llvm/test/CodeGen/X86/atomic-non-integer-fp128.ll
new file mode 100644
index 000000000..a3028e9ac
--- /dev/null
+++ b/llvm/test/CodeGen/X86/atomic-non-integer-fp128.ll
@@ -0,0 +1,35 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-linux-generic -verify-machineinstrs -mattr=-sse | FileCheck %s --check-prefix=X64-NOSSE
+; RUN: llc < %s -mtriple=x86_64-linux-generic -verify-machineinstrs | FileCheck %s --check-prefix=X64-SSE
+
+; Note: This test is testing that the lowering for atomics matches what we
+; currently emit for non-atomics + the atomic restriction.  The presence of
+; particular lowering detail in these tests should not be read as requiring
+; that detail for correctness unless it's related to the atomicity itself.
+; (Specifically, there were reviewer questions about the lowering for halfs
+;  and their calling convention which remain unresolved.)
+
+define void @store_fp128(fp128* %fptr, fp128 %v) {
+; X64-NOSSE-LABEL: store_fp128:
+; X64-NOSSE:       # %bb.0:
+; X64-NOSSE-NEXT:    pushq %rax
+; X64-NOSSE-NEXT:    .cfi_def_cfa_offset 16
+; X64-NOSSE-NEXT:    callq __sync_lock_test_and_set_16
+; X64-NOSSE-NEXT:    popq %rax
+; X64-NOSSE-NEXT:    .cfi_def_cfa_offset 8
+; X64-NOSSE-NEXT:    retq
+;
+; X64-SSE-LABEL: store_fp128:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    subq $24, %rsp
+; X64-SSE-NEXT:    .cfi_def_cfa_offset 32
+; X64-SSE-NEXT:    movaps %xmm0, (%rsp)
+; X64-SSE-NEXT:    movq (%rsp), %rsi
+; X64-SSE-NEXT:    movq {{[0-9]+}}(%rsp), %rdx
+; X64-SSE-NEXT:    callq __sync_lock_test_and_set_16
+; X64-SSE-NEXT:    addq $24, %rsp
+; X64-SSE-NEXT:    .cfi_def_cfa_offset 8
+; X64-SSE-NEXT:    retq
+  store atomic fp128 %v, fp128* %fptr unordered, align 16
+  ret void
+}
diff --git a/llvm/test/CodeGen/X86/atomic-non-integer.ll b/llvm/test/CodeGen/X86/atomic-non-integer.ll
index d40eb7631..8fd96b749 100644
--- a/llvm/test/CodeGen/X86/atomic-non-integer.ll
+++ b/llvm/test/CodeGen/X86/atomic-non-integer.ll
@@ -274,14 +274,29 @@ define void @store_fp128(fp128* %fptr, fp128 %v) {
 ; X86-NOSSE-NEXT:    .cfi_adjust_cfa_offset -56
 ; X86-NOSSE-NEXT:    retl
 ;
-; X64-LABEL: store_fp128:
-; X64:       # %bb.0:
-; X64-NEXT:    pushq %rax
-; X64-NEXT:    .cfi_def_cfa_offset 16
-; X64-NEXT:    callq __sync_lock_test_and_set_16
-; X64-NEXT:    popq %rax
-; X64-NEXT:    .cfi_def_cfa_offset 8
-; X64-NEXT:    retq
+; X64-SSE-LABEL: store_fp128:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    subq $24, %rsp
+; X64-SSE-NEXT:    .cfi_def_cfa_offset 32
+; X64-SSE-NEXT:    movaps %xmm0, (%rsp)
+; X64-SSE-NEXT:    movq (%rsp), %rsi
+; X64-SSE-NEXT:    movq {{[0-9]+}}(%rsp), %rdx
+; X64-SSE-NEXT:    callq __sync_lock_test_and_set_16
+; X64-SSE-NEXT:    addq $24, %rsp
+; X64-SSE-NEXT:    .cfi_def_cfa_offset 8
+; X64-SSE-NEXT:    retq
+;
+; X64-AVX-LABEL: store_fp128:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    subq $24, %rsp
+; X64-AVX-NEXT:    .cfi_def_cfa_offset 32
+; X64-AVX-NEXT:    vmovaps %xmm0, (%rsp)
+; X64-AVX-NEXT:    movq (%rsp), %rsi
+; X64-AVX-NEXT:    movq {{[0-9]+}}(%rsp), %rdx
+; X64-AVX-NEXT:    callq __sync_lock_test_and_set_16
+; X64-AVX-NEXT:    addq $24, %rsp
+; X64-AVX-NEXT:    .cfi_def_cfa_offset 8
+; X64-AVX-NEXT:    retq
   store atomic fp128 %v, fp128* %fptr unordered, align 16
   ret void
 }
@@ -636,18 +651,37 @@ define fp128 @load_fp128(fp128* %fptr) {
 ; X86-NOSSE-NEXT:    .cfi_def_cfa_offset 4
 ; X86-NOSSE-NEXT:    retl $4
 ;
-; X64-LABEL: load_fp128:
-; X64:       # %bb.0:
-; X64-NEXT:    pushq %rax
-; X64-NEXT:    .cfi_def_cfa_offset 16
-; X64-NEXT:    xorl %esi, %esi
-; X64-NEXT:    xorl %edx, %edx
-; X64-NEXT:    xorl %ecx, %ecx
-; X64-NEXT:    xorl %r8d, %r8d
-; X64-NEXT:    callq __sync_val_compare_and_swap_16
-; X64-NEXT:    popq %rcx
-; X64-NEXT:    .cfi_def_cfa_offset 8
-; X64-NEXT:    retq
+; X64-SSE-LABEL: load_fp128:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    subq $24, %rsp
+; X64-SSE-NEXT:    .cfi_def_cfa_offset 32
+; X64-SSE-NEXT:    xorl %esi, %esi
+; X64-SSE-NEXT:    xorl %edx, %edx
+; X64-SSE-NEXT:    xorl %ecx, %ecx
+; X64-SSE-NEXT:    xorl %r8d, %r8d
+; X64-SSE-NEXT:    callq __sync_val_compare_and_swap_16
+; X64-SSE-NEXT:    movq %rdx, {{[0-9]+}}(%rsp)
+; X64-SSE-NEXT:    movq %rax, (%rsp)
+; X64-SSE-NEXT:    movaps (%rsp), %xmm0
+; X64-SSE-NEXT:    addq $24, %rsp
+; X64-SSE-NEXT:    .cfi_def_cfa_offset 8
+; X64-SSE-NEXT:    retq
+;
+; X64-AVX-LABEL: load_fp128:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    subq $24, %rsp
+; X64-AVX-NEXT:    .cfi_def_cfa_offset 32
+; X64-AVX-NEXT:    xorl %esi, %esi
+; X64-AVX-NEXT:    xorl %edx, %edx
+; X64-AVX-NEXT:    xorl %ecx, %ecx
+; X64-AVX-NEXT:    xorl %r8d, %r8d
+; X64-AVX-NEXT:    callq __sync_val_compare_and_swap_16
+; X64-AVX-NEXT:    movq %rdx, {{[0-9]+}}(%rsp)
+; X64-AVX-NEXT:    movq %rax, (%rsp)
+; X64-AVX-NEXT:    vmovaps (%rsp), %xmm0
+; X64-AVX-NEXT:    addq $24, %rsp
+; X64-AVX-NEXT:    .cfi_def_cfa_offset 8
+; X64-AVX-NEXT:    retq
   %v = load atomic fp128, fp128* %fptr unordered, align 16
   ret fp128 %v
 }
diff --git a/llvm/test/CodeGen/X86/atomic-unordered.ll b/llvm/test/CodeGen/X86/atomic-unordered.ll
index a9df57bba..9843bf81e 100644
--- a/llvm/test/CodeGen/X86/atomic-unordered.ll
+++ b/llvm/test/CodeGen/X86/atomic-unordered.ll
@@ -1,17 +1,14 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -O0 < %s -mtriple=x86_64-linux-generic -verify-machineinstrs -mcpu=skylake | FileCheck --check-prefix=CHECK-O0 %s
-; RUN: llc -O3 < %s -mtriple=x86_64-linux-generic -verify-machineinstrs -mcpu=skylake | FileCheck --check-prefix=CHECK-O3 %s
+; RUN: llc -O0 < %s -mtriple=x86_64-linux-generic -verify-machineinstrs -mcpu=skylake -x86-experimental-unordered-atomic-isel=0 | FileCheck --check-prefixes=CHECK,CHECK-O0,CHECK-O0-CUR %s
+; RUN: llc -O3 < %s -mtriple=x86_64-linux-generic -verify-machineinstrs -mcpu=skylake  -x86-experimental-unordered-atomic-isel=0 | FileCheck --check-prefixes=CHECK,CHECK-O3,CHECK-O3-CUR %s
+; RUN: llc -O0 < %s -mtriple=x86_64-linux-generic -verify-machineinstrs -mcpu=skylake -x86-experimental-unordered-atomic-isel=1 | FileCheck --check-prefixes=CHECK,CHECK-O0,CHECK-O0-EX %s
+; RUN: llc -O3 < %s -mtriple=x86_64-linux-generic -verify-machineinstrs -mcpu=skylake -x86-experimental-unordered-atomic-isel=1 | FileCheck --check-prefixes=CHECK,CHECK-O3,CHECK-O3-EX %s
 
 define i8 @load_i8(i8* %ptr) {
-; CHECK-O0-LABEL: load_i8:
-; CHECK-O0:       # %bb.0:
-; CHECK-O0-NEXT:    movb (%rdi), %al
-; CHECK-O0-NEXT:    retq
-;
-; CHECK-O3-LABEL: load_i8:
-; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    movb (%rdi), %al
-; CHECK-O3-NEXT:    retq
+; CHECK-LABEL: load_i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movb (%rdi), %al
+; CHECK-NEXT:    retq
   %v = load atomic i8, i8* %ptr unordered, align 1
   ret i8 %v
 }
@@ -62,57 +59,37 @@ define void @store_i16(i16* %ptr, i16 %v) {
 }
 
 define i32 @load_i32(i32* %ptr) {
-; CHECK-O0-LABEL: load_i32:
-; CHECK-O0:       # %bb.0:
-; CHECK-O0-NEXT:    movl (%rdi), %eax
-; CHECK-O0-NEXT:    retq
-;
-; CHECK-O3-LABEL: load_i32:
-; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    movl (%rdi), %eax
-; CHECK-O3-NEXT:    retq
+; CHECK-LABEL: load_i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl (%rdi), %eax
+; CHECK-NEXT:    retq
   %v = load atomic i32, i32* %ptr unordered, align 4
   ret i32 %v
 }
 
 define void @store_i32(i32* %ptr, i32 %v) {
-; CHECK-O0-LABEL: store_i32:
-; CHECK-O0:       # %bb.0:
-; CHECK-O0-NEXT:    movl %esi, (%rdi)
-; CHECK-O0-NEXT:    retq
-;
-; CHECK-O3-LABEL: store_i32:
-; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    movl %esi, (%rdi)
-; CHECK-O3-NEXT:    retq
+; CHECK-LABEL: store_i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %esi, (%rdi)
+; CHECK-NEXT:    retq
   store atomic i32 %v, i32* %ptr unordered, align 4
   ret void
 }
 
 define i64 @load_i64(i64* %ptr) {
-; CHECK-O0-LABEL: load_i64:
-; CHECK-O0:       # %bb.0:
-; CHECK-O0-NEXT:    movq (%rdi), %rax
-; CHECK-O0-NEXT:    retq
-;
-; CHECK-O3-LABEL: load_i64:
-; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    movq (%rdi), %rax
-; CHECK-O3-NEXT:    retq
+; CHECK-LABEL: load_i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq (%rdi), %rax
+; CHECK-NEXT:    retq
   %v = load atomic i64, i64* %ptr unordered, align 8
   ret i64 %v
 }
 
 define void @store_i64(i64* %ptr, i64 %v) {
-; CHECK-O0-LABEL: store_i64:
-; CHECK-O0:       # %bb.0:
-; CHECK-O0-NEXT:    movq %rsi, (%rdi)
-; CHECK-O0-NEXT:    retq
-;
-; CHECK-O3-LABEL: store_i64:
-; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    movq %rsi, (%rdi)
-; CHECK-O3-NEXT:    retq
+; CHECK-LABEL: store_i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %rsi, (%rdi)
+; CHECK-NEXT:    retq
   store atomic i64 %v, i64* %ptr unordered, align 8
   ret void
 }
@@ -187,7 +164,6 @@ define void @narrow_writeback_xor(i64* %ptr) {
 ;; improve codegeneration.  Note that widening is only legal if the
 ;; resulting type would be atomic.  Each tests has a well aligned, and
 ;; unaligned variant to ensure we get correct codegen here.
-;;
 ;; Note: It's not a legality issue, but there's a gotcha here to be aware
 ;; of.  Once we widen a pair of atomic stores, we loose the information
 ;; that the original atomicity requirement was half the width.  Given that,
@@ -196,17 +172,11 @@ define void @narrow_writeback_xor(i64* %ptr) {
 
 ; Legal if wider type is also atomic (TODO)
 define void @widen_store(i32* %p0, i32 %v1, i32 %v2) {
-; CHECK-O0-LABEL: widen_store:
-; CHECK-O0:       # %bb.0:
-; CHECK-O0-NEXT:    movl %esi, (%rdi)
-; CHECK-O0-NEXT:    movl %edx, 4(%rdi)
-; CHECK-O0-NEXT:    retq
-;
-; CHECK-O3-LABEL: widen_store:
-; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    movl %esi, (%rdi)
-; CHECK-O3-NEXT:    movl %edx, 4(%rdi)
-; CHECK-O3-NEXT:    retq
+; CHECK-LABEL: widen_store:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %esi, (%rdi)
+; CHECK-NEXT:    movl %edx, 4(%rdi)
+; CHECK-NEXT:    retq
   %p1 = getelementptr i32, i32* %p0, i64 1
   store atomic i32 %v1, i32* %p0 unordered, align 8
   store atomic i32 %v2, i32* %p1 unordered, align 4
@@ -217,17 +187,11 @@ define void @widen_store(i32* %p0, i32 %v1, i32 %v2) {
 ; the wider type might cross a cache line and violate the
 ; atomicity requirement.
 define void @widen_store_unaligned(i32* %p0, i32 %v1, i32 %v2) {
-; CHECK-O0-LABEL: widen_store_unaligned:
-; CHECK-O0:       # %bb.0:
-; CHECK-O0-NEXT:    movl %esi, (%rdi)
-; CHECK-O0-NEXT:    movl %edx, 4(%rdi)
-; CHECK-O0-NEXT:    retq
-;
-; CHECK-O3-LABEL: widen_store_unaligned:
-; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    movl %esi, (%rdi)
-; CHECK-O3-NEXT:    movl %edx, 4(%rdi)
-; CHECK-O3-NEXT:    retq
+; CHECK-LABEL: widen_store_unaligned:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %esi, (%rdi)
+; CHECK-NEXT:    movl %edx, 4(%rdi)
+; CHECK-NEXT:    retq
   %p1 = getelementptr i32, i32* %p0, i64 1
   store atomic i32 %v1, i32* %p0 unordered, align 4
   store atomic i32 %v2, i32* %p1 unordered, align 4
@@ -236,17 +200,11 @@ define void @widen_store_unaligned(i32* %p0, i32 %v1, i32 %v2) {
 
 ; Legal if wider type is also atomic (TODO)
 define void @widen_broadcast(i32* %p0, i32 %v) {
-; CHECK-O0-LABEL: widen_broadcast:
-; CHECK-O0:       # %bb.0:
-; CHECK-O0-NEXT:    movl %esi, (%rdi)
-; CHECK-O0-NEXT:    movl %esi, 4(%rdi)
-; CHECK-O0-NEXT:    retq
-;
-; CHECK-O3-LABEL: widen_broadcast:
-; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    movl %esi, (%rdi)
-; CHECK-O3-NEXT:    movl %esi, 4(%rdi)
-; CHECK-O3-NEXT:    retq
+; CHECK-LABEL: widen_broadcast:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %esi, (%rdi)
+; CHECK-NEXT:    movl %esi, 4(%rdi)
+; CHECK-NEXT:    retq
   %p1 = getelementptr i32, i32* %p0, i64 1
   store atomic i32 %v, i32* %p0 unordered, align 8
   store atomic i32 %v, i32* %p1 unordered, align 4
@@ -255,17 +213,11 @@ define void @widen_broadcast(i32* %p0, i32 %v) {
 
 ; Not legal to widen due to alignment restriction
 define void @widen_broadcast_unaligned(i32* %p0, i32 %v) {
-; CHECK-O0-LABEL: widen_broadcast_unaligned:
-; CHECK-O0:       # %bb.0:
-; CHECK-O0-NEXT:    movl %esi, (%rdi)
-; CHECK-O0-NEXT:    movl %esi, 4(%rdi)
-; CHECK-O0-NEXT:    retq
-;
-; CHECK-O3-LABEL: widen_broadcast_unaligned:
-; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    movl %esi, (%rdi)
-; CHECK-O3-NEXT:    movl %esi, 4(%rdi)
-; CHECK-O3-NEXT:    retq
+; CHECK-LABEL: widen_broadcast_unaligned:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %esi, (%rdi)
+; CHECK-NEXT:    movl %esi, 4(%rdi)
+; CHECK-NEXT:    retq
   %p1 = getelementptr i32, i32* %p0, i64 1
   store atomic i32 %v, i32* %p0 unordered, align 4
   store atomic i32 %v, i32* %p1 unordered, align 4
@@ -457,21 +409,33 @@ define void @store_i256(i256* %ptr, i256 %v) {
 
 ; Legal if wider type is also atomic (TODO)
 define void @vec_store(i32* %p0, <2 x i32> %vec) {
-; CHECK-O0-LABEL: vec_store:
-; CHECK-O0:       # %bb.0:
-; CHECK-O0-NEXT:    vmovd %xmm0, %eax
-; CHECK-O0-NEXT:    vpextrd $2, %xmm0, %ecx
-; CHECK-O0-NEXT:    movl %eax, (%rdi)
-; CHECK-O0-NEXT:    movl %ecx, 4(%rdi)
-; CHECK-O0-NEXT:    retq
-;
-; CHECK-O3-LABEL: vec_store:
-; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    vmovd %xmm0, %eax
-; CHECK-O3-NEXT:    vpextrd $2, %xmm0, %ecx
-; CHECK-O3-NEXT:    movl %eax, (%rdi)
-; CHECK-O3-NEXT:    movl %ecx, 4(%rdi)
-; CHECK-O3-NEXT:    retq
+; CHECK-O0-CUR-LABEL: vec_store:
+; CHECK-O0-CUR:       # %bb.0:
+; CHECK-O0-CUR-NEXT:    vmovd %xmm0, %eax
+; CHECK-O0-CUR-NEXT:    vpextrd $1, %xmm0, %ecx
+; CHECK-O0-CUR-NEXT:    movl %eax, (%rdi)
+; CHECK-O0-CUR-NEXT:    movl %ecx, 4(%rdi)
+; CHECK-O0-CUR-NEXT:    retq
+;
+; CHECK-O3-CUR-LABEL: vec_store:
+; CHECK-O3-CUR:       # %bb.0:
+; CHECK-O3-CUR-NEXT:    vmovd %xmm0, %eax
+; CHECK-O3-CUR-NEXT:    vpextrd $1, %xmm0, %ecx
+; CHECK-O3-CUR-NEXT:    movl %eax, (%rdi)
+; CHECK-O3-CUR-NEXT:    movl %ecx, 4(%rdi)
+; CHECK-O3-CUR-NEXT:    retq
+;
+; CHECK-O0-EX-LABEL: vec_store:
+; CHECK-O0-EX:       # %bb.0:
+; CHECK-O0-EX-NEXT:    vmovd %xmm0, (%rdi)
+; CHECK-O0-EX-NEXT:    vpextrd $1, %xmm0, 4(%rdi)
+; CHECK-O0-EX-NEXT:    retq
+;
+; CHECK-O3-EX-LABEL: vec_store:
+; CHECK-O3-EX:       # %bb.0:
+; CHECK-O3-EX-NEXT:    vmovss %xmm0, (%rdi)
+; CHECK-O3-EX-NEXT:    vextractps $1, %xmm0, 4(%rdi)
+; CHECK-O3-EX-NEXT:    retq
   %v1 = extractelement <2 x i32> %vec, i32 0
   %v2 = extractelement <2 x i32> %vec, i32 1
   %p1 = getelementptr i32, i32* %p0, i64 1
@@ -482,21 +446,33 @@ define void @vec_store(i32* %p0, <2 x i32> %vec) {
 
 ; Not legal to widen due to alignment restriction
 define void @vec_store_unaligned(i32* %p0, <2 x i32> %vec) {
-; CHECK-O0-LABEL: vec_store_unaligned:
-; CHECK-O0:       # %bb.0:
-; CHECK-O0-NEXT:    vmovd %xmm0, %eax
-; CHECK-O0-NEXT:    vpextrd $2, %xmm0, %ecx
-; CHECK-O0-NEXT:    movl %eax, (%rdi)
-; CHECK-O0-NEXT:    movl %ecx, 4(%rdi)
-; CHECK-O0-NEXT:    retq
-;
-; CHECK-O3-LABEL: vec_store_unaligned:
-; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    vmovd %xmm0, %eax
-; CHECK-O3-NEXT:    vpextrd $2, %xmm0, %ecx
-; CHECK-O3-NEXT:    movl %eax, (%rdi)
-; CHECK-O3-NEXT:    movl %ecx, 4(%rdi)
-; CHECK-O3-NEXT:    retq
+; CHECK-O0-CUR-LABEL: vec_store_unaligned:
+; CHECK-O0-CUR:       # %bb.0:
+; CHECK-O0-CUR-NEXT:    vmovd %xmm0, %eax
+; CHECK-O0-CUR-NEXT:    vpextrd $1, %xmm0, %ecx
+; CHECK-O0-CUR-NEXT:    movl %eax, (%rdi)
+; CHECK-O0-CUR-NEXT:    movl %ecx, 4(%rdi)
+; CHECK-O0-CUR-NEXT:    retq
+;
+; CHECK-O3-CUR-LABEL: vec_store_unaligned:
+; CHECK-O3-CUR:       # %bb.0:
+; CHECK-O3-CUR-NEXT:    vmovd %xmm0, %eax
+; CHECK-O3-CUR-NEXT:    vpextrd $1, %xmm0, %ecx
+; CHECK-O3-CUR-NEXT:    movl %eax, (%rdi)
+; CHECK-O3-CUR-NEXT:    movl %ecx, 4(%rdi)
+; CHECK-O3-CUR-NEXT:    retq
+;
+; CHECK-O0-EX-LABEL: vec_store_unaligned:
+; CHECK-O0-EX:       # %bb.0:
+; CHECK-O0-EX-NEXT:    vmovd %xmm0, (%rdi)
+; CHECK-O0-EX-NEXT:    vpextrd $1, %xmm0, 4(%rdi)
+; CHECK-O0-EX-NEXT:    retq
+;
+; CHECK-O3-EX-LABEL: vec_store_unaligned:
+; CHECK-O3-EX:       # %bb.0:
+; CHECK-O3-EX-NEXT:    vmovss %xmm0, (%rdi)
+; CHECK-O3-EX-NEXT:    vextractps $1, %xmm0, 4(%rdi)
+; CHECK-O3-EX-NEXT:    retq
   %v1 = extractelement <2 x i32> %vec, i32 0
   %v2 = extractelement <2 x i32> %vec, i32 1
   %p1 = getelementptr i32, i32* %p0, i64 1
@@ -510,19 +486,31 @@ define void @vec_store_unaligned(i32* %p0, <2 x i32> %vec) {
 ; Legal if wider type is also atomic (TODO)
 ; Also, can avoid register move from xmm to eax (TODO)
 define void @widen_broadcast2(i32* %p0, <2 x i32> %vec) {
-; CHECK-O0-LABEL: widen_broadcast2:
-; CHECK-O0:       # %bb.0:
-; CHECK-O0-NEXT:    vmovd %xmm0, %eax
-; CHECK-O0-NEXT:    movl %eax, (%rdi)
-; CHECK-O0-NEXT:    movl %eax, 4(%rdi)
-; CHECK-O0-NEXT:    retq
-;
-; CHECK-O3-LABEL: widen_broadcast2:
-; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    vmovd %xmm0, %eax
-; CHECK-O3-NEXT:    movl %eax, (%rdi)
-; CHECK-O3-NEXT:    movl %eax, 4(%rdi)
-; CHECK-O3-NEXT:    retq
+; CHECK-O0-CUR-LABEL: widen_broadcast2:
+; CHECK-O0-CUR:       # %bb.0:
+; CHECK-O0-CUR-NEXT:    vmovd %xmm0, %eax
+; CHECK-O0-CUR-NEXT:    movl %eax, (%rdi)
+; CHECK-O0-CUR-NEXT:    movl %eax, 4(%rdi)
+; CHECK-O0-CUR-NEXT:    retq
+;
+; CHECK-O3-CUR-LABEL: widen_broadcast2:
+; CHECK-O3-CUR:       # %bb.0:
+; CHECK-O3-CUR-NEXT:    vmovd %xmm0, %eax
+; CHECK-O3-CUR-NEXT:    movl %eax, (%rdi)
+; CHECK-O3-CUR-NEXT:    movl %eax, 4(%rdi)
+; CHECK-O3-CUR-NEXT:    retq
+;
+; CHECK-O0-EX-LABEL: widen_broadcast2:
+; CHECK-O0-EX:       # %bb.0:
+; CHECK-O0-EX-NEXT:    vmovd %xmm0, (%rdi)
+; CHECK-O0-EX-NEXT:    vmovd %xmm0, 4(%rdi)
+; CHECK-O0-EX-NEXT:    retq
+;
+; CHECK-O3-EX-LABEL: widen_broadcast2:
+; CHECK-O3-EX:       # %bb.0:
+; CHECK-O3-EX-NEXT:    vmovss %xmm0, (%rdi)
+; CHECK-O3-EX-NEXT:    vmovss %xmm0, 4(%rdi)
+; CHECK-O3-EX-NEXT:    retq
   %v1 = extractelement <2 x i32> %vec, i32 0
   %p1 = getelementptr i32, i32* %p0, i64 1
   store atomic i32 %v1, i32* %p0 unordered, align 8
@@ -532,19 +520,31 @@ define void @widen_broadcast2(i32* %p0, <2 x i32> %vec) {
 
 ; Not legal to widen due to alignment restriction
 define void @widen_broadcast2_unaligned(i32* %p0, <2 x i32> %vec) {
-; CHECK-O0-LABEL: widen_broadcast2_unaligned:
-; CHECK-O0:       # %bb.0:
-; CHECK-O0-NEXT:    vmovd %xmm0, %eax
-; CHECK-O0-NEXT:    movl %eax, (%rdi)
-; CHECK-O0-NEXT:    movl %eax, 4(%rdi)
-; CHECK-O0-NEXT:    retq
-;
-; CHECK-O3-LABEL: widen_broadcast2_unaligned:
-; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    vmovd %xmm0, %eax
-; CHECK-O3-NEXT:    movl %eax, (%rdi)
-; CHECK-O3-NEXT:    movl %eax, 4(%rdi)
-; CHECK-O3-NEXT:    retq
+; CHECK-O0-CUR-LABEL: widen_broadcast2_unaligned:
+; CHECK-O0-CUR:       # %bb.0:
+; CHECK-O0-CUR-NEXT:    vmovd %xmm0, %eax
+; CHECK-O0-CUR-NEXT:    movl %eax, (%rdi)
+; CHECK-O0-CUR-NEXT:    movl %eax, 4(%rdi)
+; CHECK-O0-CUR-NEXT:    retq
+;
+; CHECK-O3-CUR-LABEL: widen_broadcast2_unaligned:
+; CHECK-O3-CUR:       # %bb.0:
+; CHECK-O3-CUR-NEXT:    vmovd %xmm0, %eax
+; CHECK-O3-CUR-NEXT:    movl %eax, (%rdi)
+; CHECK-O3-CUR-NEXT:    movl %eax, 4(%rdi)
+; CHECK-O3-CUR-NEXT:    retq
+;
+; CHECK-O0-EX-LABEL: widen_broadcast2_unaligned:
+; CHECK-O0-EX:       # %bb.0:
+; CHECK-O0-EX-NEXT:    vmovd %xmm0, (%rdi)
+; CHECK-O0-EX-NEXT:    vmovd %xmm0, 4(%rdi)
+; CHECK-O0-EX-NEXT:    retq
+;
+; CHECK-O3-EX-LABEL: widen_broadcast2_unaligned:
+; CHECK-O3-EX:       # %bb.0:
+; CHECK-O3-EX-NEXT:    vmovss %xmm0, (%rdi)
+; CHECK-O3-EX-NEXT:    vmovss %xmm0, 4(%rdi)
+; CHECK-O3-EX-NEXT:    retq
   %v1 = extractelement <2 x i32> %vec, i32 0
   %p1 = getelementptr i32, i32* %p0, i64 1
   store atomic i32 %v1, i32* %p0 unordered, align 4
@@ -554,17 +554,11 @@ define void @widen_broadcast2_unaligned(i32* %p0, <2 x i32> %vec) {
 
 ; Legal if wider type is also atomic (TODO)
 define void @widen_zero_init(i32* %p0, i32 %v1, i32 %v2) {
-; CHECK-O0-LABEL: widen_zero_init:
-; CHECK-O0:       # %bb.0:
-; CHECK-O0-NEXT:    movl $0, (%rdi)
-; CHECK-O0-NEXT:    movl $0, 4(%rdi)
-; CHECK-O0-NEXT:    retq
-;
-; CHECK-O3-LABEL: widen_zero_init:
-; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    movl $0, (%rdi)
-; CHECK-O3-NEXT:    movl $0, 4(%rdi)
-; CHECK-O3-NEXT:    retq
+; CHECK-LABEL: widen_zero_init:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl $0, (%rdi)
+; CHECK-NEXT:    movl $0, 4(%rdi)
+; CHECK-NEXT:    retq
   %p1 = getelementptr i32, i32* %p0, i64 1
   store atomic i32 0, i32* %p0 unordered, align 8
   store atomic i32 0, i32* %p1 unordered, align 4
@@ -573,17 +567,11 @@ define void @widen_zero_init(i32* %p0, i32 %v1, i32 %v2) {
 
 ; Not legal to widen due to alignment restriction
 define void @widen_zero_init_unaligned(i32* %p0, i32 %v1, i32 %v2) {
-; CHECK-O0-LABEL: widen_zero_init_unaligned:
-; CHECK-O0:       # %bb.0:
-; CHECK-O0-NEXT:    movl $0, (%rdi)
-; CHECK-O0-NEXT:    movl $0, 4(%rdi)
-; CHECK-O0-NEXT:    retq
-;
-; CHECK-O3-LABEL: widen_zero_init_unaligned:
-; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    movl $0, (%rdi)
-; CHECK-O3-NEXT:    movl $0, 4(%rdi)
-; CHECK-O3-NEXT:    retq
+; CHECK-LABEL: widen_zero_init_unaligned:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl $0, (%rdi)
+; CHECK-NEXT:    movl $0, 4(%rdi)
+; CHECK-NEXT:    retq
   %p1 = getelementptr i32, i32* %p0, i64 1
   store atomic i32 0, i32* %p0 unordered, align 4
   store atomic i32 0, i32* %p1 unordered, align 4
@@ -595,17 +583,11 @@ define void @widen_zero_init_unaligned(i32* %p0, i32 %v1, i32 %v2) {
 
 ; Legal, as expected
 define i64 @load_fold_add1(i64* %p) {
-; CHECK-O0-LABEL: load_fold_add1:
-; CHECK-O0:       # %bb.0:
-; CHECK-O0-NEXT:    movq (%rdi), %rax
-; CHECK-O0-NEXT:    addq $15, %rax
-; CHECK-O0-NEXT:    retq
-;
-; CHECK-O3-LABEL: load_fold_add1:
-; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    movq (%rdi), %rax
-; CHECK-O3-NEXT:    addq $15, %rax
-; CHECK-O3-NEXT:    retq
+; CHECK-LABEL: load_fold_add1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq (%rdi), %rax
+; CHECK-NEXT:    addq $15, %rax
+; CHECK-NEXT:    retq
   %v = load atomic i64, i64* %p unordered, align 8
   %ret = add i64 %v, 15
   ret i64 %ret
@@ -635,11 +617,17 @@ define i64 @load_fold_add3(i64* %p1, i64* %p2) {
 ; CHECK-O0-NEXT:    addq (%rsi), %rax
 ; CHECK-O0-NEXT:    retq
 ;
-; CHECK-O3-LABEL: load_fold_add3:
-; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    movq (%rsi), %rax
-; CHECK-O3-NEXT:    addq (%rdi), %rax
-; CHECK-O3-NEXT:    retq
+; CHECK-O3-CUR-LABEL: load_fold_add3:
+; CHECK-O3-CUR:       # %bb.0:
+; CHECK-O3-CUR-NEXT:    movq (%rsi), %rax
+; CHECK-O3-CUR-NEXT:    addq (%rdi), %rax
+; CHECK-O3-CUR-NEXT:    retq
+;
+; CHECK-O3-EX-LABEL: load_fold_add3:
+; CHECK-O3-EX:       # %bb.0:
+; CHECK-O3-EX-NEXT:    movq (%rdi), %rax
+; CHECK-O3-EX-NEXT:    addq (%rsi), %rax
+; CHECK-O3-EX-NEXT:    retq
   %v = load atomic i64, i64* %p1 unordered, align 8
   %v2 = load atomic i64, i64* %p2 unordered, align 8
   %ret = add i64 %v, %v2
@@ -665,34 +653,22 @@ define i64 @load_fold_sub1(i64* %p) {
 }
 
 define i64 @load_fold_sub2(i64* %p, i64 %v2) {
-; CHECK-O0-LABEL: load_fold_sub2:
-; CHECK-O0:       # %bb.0:
-; CHECK-O0-NEXT:    movq (%rdi), %rax
-; CHECK-O0-NEXT:    subq %rsi, %rax
-; CHECK-O0-NEXT:    retq
-;
-; CHECK-O3-LABEL: load_fold_sub2:
-; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    movq (%rdi), %rax
-; CHECK-O3-NEXT:    subq %rsi, %rax
-; CHECK-O3-NEXT:    retq
+; CHECK-LABEL: load_fold_sub2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq (%rdi), %rax
+; CHECK-NEXT:    subq %rsi, %rax
+; CHECK-NEXT:    retq
   %v = load atomic i64, i64* %p unordered, align 8
   %ret = sub i64 %v, %v2
   ret i64 %ret
 }
 
 define i64 @load_fold_sub3(i64* %p1, i64* %p2) {
-; CHECK-O0-LABEL: load_fold_sub3:
-; CHECK-O0:       # %bb.0:
-; CHECK-O0-NEXT:    movq (%rdi), %rax
-; CHECK-O0-NEXT:    subq (%rsi), %rax
-; CHECK-O0-NEXT:    retq
-;
-; CHECK-O3-LABEL: load_fold_sub3:
-; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    movq (%rdi), %rax
-; CHECK-O3-NEXT:    subq (%rsi), %rax
-; CHECK-O3-NEXT:    retq
+; CHECK-LABEL: load_fold_sub3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq (%rdi), %rax
+; CHECK-NEXT:    subq (%rsi), %rax
+; CHECK-NEXT:    retq
   %v = load atomic i64, i64* %p1 unordered, align 8
   %v2 = load atomic i64, i64* %p2 unordered, align 8
   %ret = sub i64 %v, %v2
@@ -741,11 +717,17 @@ define i64 @load_fold_mul3(i64* %p1, i64* %p2) {
 ; CHECK-O0-NEXT:    imulq (%rsi), %rax
 ; CHECK-O0-NEXT:    retq
 ;
-; CHECK-O3-LABEL: load_fold_mul3:
-; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    movq (%rsi), %rax
-; CHECK-O3-NEXT:    imulq (%rdi), %rax
-; CHECK-O3-NEXT:    retq
+; CHECK-O3-CUR-LABEL: load_fold_mul3:
+; CHECK-O3-CUR:       # %bb.0:
+; CHECK-O3-CUR-NEXT:    movq (%rsi), %rax
+; CHECK-O3-CUR-NEXT:    imulq (%rdi), %rax
+; CHECK-O3-CUR-NEXT:    retq
+;
+; CHECK-O3-EX-LABEL: load_fold_mul3:
+; CHECK-O3-EX:       # %bb.0:
+; CHECK-O3-EX-NEXT:    movq (%rdi), %rax
+; CHECK-O3-EX-NEXT:    imulq (%rsi), %rax
+; CHECK-O3-EX-NEXT:    retq
   %v = load atomic i64, i64* %p1 unordered, align 8
   %v2 = load atomic i64, i64* %p2 unordered, align 8
   %ret = mul i64 %v, %v2
@@ -853,14 +835,22 @@ define i64 @load_fold_udiv1(i64* %p) {
 ; CHECK-O0-NEXT:    divq %rsi
 ; CHECK-O0-NEXT:    retq
 ;
-; CHECK-O3-LABEL: load_fold_udiv1:
-; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    movq (%rdi), %rax
-; CHECK-O3-NEXT:    movabsq $-8608480567731124087, %rcx # imm = 0x8888888888888889
-; CHECK-O3-NEXT:    mulq %rcx
-; CHECK-O3-NEXT:    movq %rdx, %rax
-; CHECK-O3-NEXT:    shrq $3, %rax
-; CHECK-O3-NEXT:    retq
+; CHECK-O3-CUR-LABEL: load_fold_udiv1:
+; CHECK-O3-CUR:       # %bb.0:
+; CHECK-O3-CUR-NEXT:    movq (%rdi), %rax
+; CHECK-O3-CUR-NEXT:    movabsq $-8608480567731124087, %rcx # imm = 0x8888888888888889
+; CHECK-O3-CUR-NEXT:    mulq %rcx
+; CHECK-O3-CUR-NEXT:    movq %rdx, %rax
+; CHECK-O3-CUR-NEXT:    shrq $3, %rax
+; CHECK-O3-CUR-NEXT:    retq
+;
+; CHECK-O3-EX-LABEL: load_fold_udiv1:
+; CHECK-O3-EX:       # %bb.0:
+; CHECK-O3-EX-NEXT:    movabsq $-8608480567731124087, %rax # imm = 0x8888888888888889
+; CHECK-O3-EX-NEXT:    mulq (%rdi)
+; CHECK-O3-EX-NEXT:    movq %rdx, %rax
+; CHECK-O3-EX-NEXT:    shrq $3, %rax
+; CHECK-O3-EX-NEXT:    retq
   %v = load atomic i64, i64* %p unordered, align 8
   %ret = udiv i64 %v, 15
   ret i64 %ret
@@ -1129,17 +1119,11 @@ define i64 @load_fold_urem3(i64* %p1, i64* %p2) {
 
 ; Legal, as expected
 define i64 @load_fold_shl1(i64* %p) {
-; CHECK-O0-LABEL: load_fold_shl1:
-; CHECK-O0:       # %bb.0:
-; CHECK-O0-NEXT:    movq (%rdi), %rax
-; CHECK-O0-NEXT:    shlq $15, %rax
-; CHECK-O0-NEXT:    retq
-;
-; CHECK-O3-LABEL: load_fold_shl1:
-; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    movq (%rdi), %rax
-; CHECK-O3-NEXT:    shlq $15, %rax
-; CHECK-O3-NEXT:    retq
+; CHECK-LABEL: load_fold_shl1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq (%rdi), %rax
+; CHECK-NEXT:    shlq $15, %rax
+; CHECK-NEXT:    retq
   %v = load atomic i64, i64* %p unordered, align 8
   %ret = shl i64 %v, 15
   ret i64 %ret
@@ -1185,17 +1169,11 @@ define i64 @load_fold_shl3(i64* %p1, i64* %p2) {
 
 ; Legal, as expected
 define i64 @load_fold_lshr1(i64* %p) {
-; CHECK-O0-LABEL: load_fold_lshr1:
-; CHECK-O0:       # %bb.0:
-; CHECK-O0-NEXT:    movq (%rdi), %rax
-; CHECK-O0-NEXT:    shrq $15, %rax
-; CHECK-O0-NEXT:    retq
-;
-; CHECK-O3-LABEL: load_fold_lshr1:
-; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    movq (%rdi), %rax
-; CHECK-O3-NEXT:    shrq $15, %rax
-; CHECK-O3-NEXT:    retq
+; CHECK-LABEL: load_fold_lshr1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq (%rdi), %rax
+; CHECK-NEXT:    shrq $15, %rax
+; CHECK-NEXT:    retq
   %v = load atomic i64, i64* %p unordered, align 8
   %ret = lshr i64 %v, 15
   ret i64 %ret
@@ -1241,17 +1219,11 @@ define i64 @load_fold_lshr3(i64* %p1, i64* %p2) {
 
 ; Legal, as expected
 define i64 @load_fold_ashr1(i64* %p) {
-; CHECK-O0-LABEL: load_fold_ashr1:
-; CHECK-O0:       # %bb.0:
-; CHECK-O0-NEXT:    movq (%rdi), %rax
-; CHECK-O0-NEXT:    sarq $15, %rax
-; CHECK-O0-NEXT:    retq
-;
-; CHECK-O3-LABEL: load_fold_ashr1:
-; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    movq (%rdi), %rax
-; CHECK-O3-NEXT:    sarq $15, %rax
-; CHECK-O3-NEXT:    retq
+; CHECK-LABEL: load_fold_ashr1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq (%rdi), %rax
+; CHECK-NEXT:    sarq $15, %rax
+; CHECK-NEXT:    retq
   %v = load atomic i64, i64* %p unordered, align 8
   %ret = ashr i64 %v, 15
   ret i64 %ret
@@ -1337,11 +1309,17 @@ define i64 @load_fold_and3(i64* %p1, i64* %p2) {
 ; CHECK-O0-NEXT:    andq (%rsi), %rax
 ; CHECK-O0-NEXT:    retq
 ;
-; CHECK-O3-LABEL: load_fold_and3:
-; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    movq (%rsi), %rax
-; CHECK-O3-NEXT:    andq (%rdi), %rax
-; CHECK-O3-NEXT:    retq
+; CHECK-O3-CUR-LABEL: load_fold_and3:
+; CHECK-O3-CUR:       # %bb.0:
+; CHECK-O3-CUR-NEXT:    movq (%rsi), %rax
+; CHECK-O3-CUR-NEXT:    andq (%rdi), %rax
+; CHECK-O3-CUR-NEXT:    retq
+;
+; CHECK-O3-EX-LABEL: load_fold_and3:
+; CHECK-O3-EX:       # %bb.0:
+; CHECK-O3-EX-NEXT:    movq (%rdi), %rax
+; CHECK-O3-EX-NEXT:    andq (%rsi), %rax
+; CHECK-O3-EX-NEXT:    retq
   %v = load atomic i64, i64* %p1 unordered, align 8
   %v2 = load atomic i64, i64* %p2 unordered, align 8
   %ret = and i64 %v, %v2
@@ -1350,17 +1328,11 @@ define i64 @load_fold_and3(i64* %p1, i64* %p2) {
 
 ; Legal, as expected
 define i64 @load_fold_or1(i64* %p) {
-; CHECK-O0-LABEL: load_fold_or1:
-; CHECK-O0:       # %bb.0:
-; CHECK-O0-NEXT:    movq (%rdi), %rax
-; CHECK-O0-NEXT:    orq $15, %rax
-; CHECK-O0-NEXT:    retq
-;
-; CHECK-O3-LABEL: load_fold_or1:
-; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    movq (%rdi), %rax
-; CHECK-O3-NEXT:    orq $15, %rax
-; CHECK-O3-NEXT:    retq
+; CHECK-LABEL: load_fold_or1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq (%rdi), %rax
+; CHECK-NEXT:    orq $15, %rax
+; CHECK-NEXT:    retq
   %v = load atomic i64, i64* %p unordered, align 8
   %ret = or i64 %v, 15
   ret i64 %ret
@@ -1390,11 +1362,17 @@ define i64 @load_fold_or3(i64* %p1, i64* %p2) {
 ; CHECK-O0-NEXT:    orq (%rsi), %rax
 ; CHECK-O0-NEXT:    retq
 ;
-; CHECK-O3-LABEL: load_fold_or3:
-; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    movq (%rsi), %rax
-; CHECK-O3-NEXT:    orq (%rdi), %rax
-; CHECK-O3-NEXT:    retq
+; CHECK-O3-CUR-LABEL: load_fold_or3:
+; CHECK-O3-CUR:       # %bb.0:
+; CHECK-O3-CUR-NEXT:    movq (%rsi), %rax
+; CHECK-O3-CUR-NEXT:    orq (%rdi), %rax
+; CHECK-O3-CUR-NEXT:    retq
+;
+; CHECK-O3-EX-LABEL: load_fold_or3:
+; CHECK-O3-EX:       # %bb.0:
+; CHECK-O3-EX-NEXT:    movq (%rdi), %rax
+; CHECK-O3-EX-NEXT:    orq (%rsi), %rax
+; CHECK-O3-EX-NEXT:    retq
   %v = load atomic i64, i64* %p1 unordered, align 8
   %v2 = load atomic i64, i64* %p2 unordered, align 8
   %ret = or i64 %v, %v2
@@ -1403,17 +1381,11 @@ define i64 @load_fold_or3(i64* %p1, i64* %p2) {
 
 ; Legal, as expected
 define i64 @load_fold_xor1(i64* %p) {
-; CHECK-O0-LABEL: load_fold_xor1:
-; CHECK-O0:       # %bb.0:
-; CHECK-O0-NEXT:    movq (%rdi), %rax
-; CHECK-O0-NEXT:    xorq $15, %rax
-; CHECK-O0-NEXT:    retq
-;
-; CHECK-O3-LABEL: load_fold_xor1:
-; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    movq (%rdi), %rax
-; CHECK-O3-NEXT:    xorq $15, %rax
-; CHECK-O3-NEXT:    retq
+; CHECK-LABEL: load_fold_xor1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq (%rdi), %rax
+; CHECK-NEXT:    xorq $15, %rax
+; CHECK-NEXT:    retq
   %v = load atomic i64, i64* %p unordered, align 8
   %ret = xor i64 %v, 15
   ret i64 %ret
@@ -1443,11 +1415,17 @@ define i64 @load_fold_xor3(i64* %p1, i64* %p2) {
 ; CHECK-O0-NEXT:    xorq (%rsi), %rax
 ; CHECK-O0-NEXT:    retq
 ;
-; CHECK-O3-LABEL: load_fold_xor3:
-; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    movq (%rsi), %rax
-; CHECK-O3-NEXT:    xorq (%rdi), %rax
-; CHECK-O3-NEXT:    retq
+; CHECK-O3-CUR-LABEL: load_fold_xor3:
+; CHECK-O3-CUR:       # %bb.0:
+; CHECK-O3-CUR-NEXT:    movq (%rsi), %rax
+; CHECK-O3-CUR-NEXT:    xorq (%rdi), %rax
+; CHECK-O3-CUR-NEXT:    retq
+;
+; CHECK-O3-EX-LABEL: load_fold_xor3:
+; CHECK-O3-EX:       # %bb.0:
+; CHECK-O3-EX-NEXT:    movq (%rdi), %rax
+; CHECK-O3-EX-NEXT:    xorq (%rsi), %rax
+; CHECK-O3-EX-NEXT:    retq
   %v = load atomic i64, i64* %p1 unordered, align 8
   %v2 = load atomic i64, i64* %p2 unordered, align 8
   %ret = xor i64 %v, %v2
@@ -1505,12 +1483,19 @@ define i1 @load_fold_icmp3(i64* %p1, i64* %p2) {
 ; CHECK-O0-NEXT:    movb %dl, %al
 ; CHECK-O0-NEXT:    retq
 ;
-; CHECK-O3-LABEL: load_fold_icmp3:
-; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    movq (%rsi), %rax
-; CHECK-O3-NEXT:    cmpq %rax, (%rdi)
-; CHECK-O3-NEXT:    sete %al
-; CHECK-O3-NEXT:    retq
+; CHECK-O3-CUR-LABEL: load_fold_icmp3:
+; CHECK-O3-CUR:       # %bb.0:
+; CHECK-O3-CUR-NEXT:    movq (%rsi), %rax
+; CHECK-O3-CUR-NEXT:    cmpq %rax, (%rdi)
+; CHECK-O3-CUR-NEXT:    sete %al
+; CHECK-O3-CUR-NEXT:    retq
+;
+; CHECK-O3-EX-LABEL: load_fold_icmp3:
+; CHECK-O3-EX:       # %bb.0:
+; CHECK-O3-EX-NEXT:    movq (%rdi), %rax
+; CHECK-O3-EX-NEXT:    cmpq (%rsi), %rax
+; CHECK-O3-EX-NEXT:    sete %al
+; CHECK-O3-EX-NEXT:    retq
   %v = load atomic i64, i64* %p1 unordered, align 8
   %v2 = load atomic i64, i64* %p2 unordered, align 8
   %ret = icmp eq i64 %v, %v2
@@ -1601,21 +1586,13 @@ define void @rmw_fold_sub2(i64* %p, i64 %v) {
 
 ; Legal, as expected
 define void @rmw_fold_mul1(i64* %p, i64 %v) {
-; CHECK-O0-LABEL: rmw_fold_mul1:
-; CHECK-O0:       # %bb.0:
-; CHECK-O0-NEXT:    movq (%rdi), %rax
-; CHECK-O0-NEXT:    leaq (%rax,%rax,4), %rax
-; CHECK-O0-NEXT:    leaq (%rax,%rax,2), %rax
-; CHECK-O0-NEXT:    movq %rax, (%rdi)
-; CHECK-O0-NEXT:    retq
-;
-; CHECK-O3-LABEL: rmw_fold_mul1:
-; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    movq (%rdi), %rax
-; CHECK-O3-NEXT:    leaq (%rax,%rax,4), %rax
-; CHECK-O3-NEXT:    leaq (%rax,%rax,2), %rax
-; CHECK-O3-NEXT:    movq %rax, (%rdi)
-; CHECK-O3-NEXT:    retq
+; CHECK-LABEL: rmw_fold_mul1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq (%rdi), %rax
+; CHECK-NEXT:    leaq (%rax,%rax,4), %rax
+; CHECK-NEXT:    leaq (%rax,%rax,2), %rax
+; CHECK-NEXT:    movq %rax, (%rdi)
+; CHECK-NEXT:    retq
   %prev = load atomic i64, i64* %p unordered, align 8
   %val = mul i64 %prev, 15
   store atomic i64 %val, i64* %p unordered, align 8
@@ -1724,14 +1701,22 @@ define void @rmw_fold_udiv1(i64* %p, i64 %v) {
 ; CHECK-O0-NEXT:    movq %rdx, (%rdi)
 ; CHECK-O0-NEXT:    retq
 ;
-; CHECK-O3-LABEL: rmw_fold_udiv1:
-; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    movq (%rdi), %rax
-; CHECK-O3-NEXT:    movabsq $-8608480567731124087, %rcx # imm = 0x8888888888888889
-; CHECK-O3-NEXT:    mulq %rcx
-; CHECK-O3-NEXT:    shrq $3, %rdx
-; CHECK-O3-NEXT:    movq %rdx, (%rdi)
-; CHECK-O3-NEXT:    retq
+; CHECK-O3-CUR-LABEL: rmw_fold_udiv1:
+; CHECK-O3-CUR:       # %bb.0:
+; CHECK-O3-CUR-NEXT:    movq (%rdi), %rax
+; CHECK-O3-CUR-NEXT:    movabsq $-8608480567731124087, %rcx # imm = 0x8888888888888889
+; CHECK-O3-CUR-NEXT:    mulq %rcx
+; CHECK-O3-CUR-NEXT:    shrq $3, %rdx
+; CHECK-O3-CUR-NEXT:    movq %rdx, (%rdi)
+; CHECK-O3-CUR-NEXT:    retq
+;
+; CHECK-O3-EX-LABEL: rmw_fold_udiv1:
+; CHECK-O3-EX:       # %bb.0:
+; CHECK-O3-EX-NEXT:    movabsq $-8608480567731124087, %rax # imm = 0x8888888888888889
+; CHECK-O3-EX-NEXT:    mulq (%rdi)
+; CHECK-O3-EX-NEXT:    shrq $3, %rdx
+; CHECK-O3-EX-NEXT:    movq %rdx, (%rdi)
+; CHECK-O3-EX-NEXT:    retq
   %prev = load atomic i64, i64* %p unordered, align 8
   %val = udiv i64 %prev, 15
   store atomic i64 %val, i64* %p unordered, align 8
@@ -1930,12 +1915,17 @@ define void @rmw_fold_shl1(i64* %p, i64 %v) {
 ; CHECK-O0-NEXT:    movq %rax, (%rdi)
 ; CHECK-O0-NEXT:    retq
 ;
-; CHECK-O3-LABEL: rmw_fold_shl1:
-; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    movq (%rdi), %rax
-; CHECK-O3-NEXT:    shlq $15, %rax
-; CHECK-O3-NEXT:    movq %rax, (%rdi)
-; CHECK-O3-NEXT:    retq
+; CHECK-O3-CUR-LABEL: rmw_fold_shl1:
+; CHECK-O3-CUR:       # %bb.0:
+; CHECK-O3-CUR-NEXT:    movq (%rdi), %rax
+; CHECK-O3-CUR-NEXT:    shlq $15, %rax
+; CHECK-O3-CUR-NEXT:    movq %rax, (%rdi)
+; CHECK-O3-CUR-NEXT:    retq
+;
+; CHECK-O3-EX-LABEL: rmw_fold_shl1:
+; CHECK-O3-EX:       # %bb.0:
+; CHECK-O3-EX-NEXT:    shlq $15, (%rdi)
+; CHECK-O3-EX-NEXT:    retq
   %prev = load atomic i64, i64* %p unordered, align 8
   %val = shl i64 %prev, 15
   store atomic i64 %val, i64* %p unordered, align 8
@@ -1954,11 +1944,18 @@ define void @rmw_fold_shl2(i64* %p, i64 %v) {
 ; CHECK-O0-NEXT:    movq %rax, (%rdi)
 ; CHECK-O0-NEXT:    retq
 ;
-; CHECK-O3-LABEL: rmw_fold_shl2:
-; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    shlxq %rsi, (%rdi), %rax
-; CHECK-O3-NEXT:    movq %rax, (%rdi)
-; CHECK-O3-NEXT:    retq
+; CHECK-O3-CUR-LABEL: rmw_fold_shl2:
+; CHECK-O3-CUR:       # %bb.0:
+; CHECK-O3-CUR-NEXT:    shlxq %rsi, (%rdi), %rax
+; CHECK-O3-CUR-NEXT:    movq %rax, (%rdi)
+; CHECK-O3-CUR-NEXT:    retq
+;
+; CHECK-O3-EX-LABEL: rmw_fold_shl2:
+; CHECK-O3-EX:       # %bb.0:
+; CHECK-O3-EX-NEXT:    movq %rsi, %rcx
+; CHECK-O3-EX-NEXT:    # kill: def $cl killed $cl killed $rcx
+; CHECK-O3-EX-NEXT:    shlq %cl, (%rdi)
+; CHECK-O3-EX-NEXT:    retq
   %prev = load atomic i64, i64* %p unordered, align 8
   %val = shl i64 %prev, %v
   store atomic i64 %val, i64* %p unordered, align 8
@@ -1974,12 +1971,17 @@ define void @rmw_fold_lshr1(i64* %p, i64 %v) {
 ; CHECK-O0-NEXT:    movq %rax, (%rdi)
 ; CHECK-O0-NEXT:    retq
 ;
-; CHECK-O3-LABEL: rmw_fold_lshr1:
-; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    movq (%rdi), %rax
-; CHECK-O3-NEXT:    shrq $15, %rax
-; CHECK-O3-NEXT:    movq %rax, (%rdi)
-; CHECK-O3-NEXT:    retq
+; CHECK-O3-CUR-LABEL: rmw_fold_lshr1:
+; CHECK-O3-CUR:       # %bb.0:
+; CHECK-O3-CUR-NEXT:    movq (%rdi), %rax
+; CHECK-O3-CUR-NEXT:    shrq $15, %rax
+; CHECK-O3-CUR-NEXT:    movq %rax, (%rdi)
+; CHECK-O3-CUR-NEXT:    retq
+;
+; CHECK-O3-EX-LABEL: rmw_fold_lshr1:
+; CHECK-O3-EX:       # %bb.0:
+; CHECK-O3-EX-NEXT:    shrq $15, (%rdi)
+; CHECK-O3-EX-NEXT:    retq
   %prev = load atomic i64, i64* %p unordered, align 8
   %val = lshr i64 %prev, 15
   store atomic i64 %val, i64* %p unordered, align 8
@@ -1998,11 +2000,18 @@ define void @rmw_fold_lshr2(i64* %p, i64 %v) {
 ; CHECK-O0-NEXT:    movq %rax, (%rdi)
 ; CHECK-O0-NEXT:    retq
 ;
-; CHECK-O3-LABEL: rmw_fold_lshr2:
-; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    shrxq %rsi, (%rdi), %rax
-; CHECK-O3-NEXT:    movq %rax, (%rdi)
-; CHECK-O3-NEXT:    retq
+; CHECK-O3-CUR-LABEL: rmw_fold_lshr2:
+; CHECK-O3-CUR:       # %bb.0:
+; CHECK-O3-CUR-NEXT:    shrxq %rsi, (%rdi), %rax
+; CHECK-O3-CUR-NEXT:    movq %rax, (%rdi)
+; CHECK-O3-CUR-NEXT:    retq
+;
+; CHECK-O3-EX-LABEL: rmw_fold_lshr2:
+; CHECK-O3-EX:       # %bb.0:
+; CHECK-O3-EX-NEXT:    movq %rsi, %rcx
+; CHECK-O3-EX-NEXT:    # kill: def $cl killed $cl killed $rcx
+; CHECK-O3-EX-NEXT:    shrq %cl, (%rdi)
+; CHECK-O3-EX-NEXT:    retq
   %prev = load atomic i64, i64* %p unordered, align 8
   %val = lshr i64 %prev, %v
   store atomic i64 %val, i64* %p unordered, align 8
@@ -2018,12 +2027,17 @@ define void @rmw_fold_ashr1(i64* %p, i64 %v) {
 ; CHECK-O0-NEXT:    movq %rax, (%rdi)
 ; CHECK-O0-NEXT:    retq
 ;
-; CHECK-O3-LABEL: rmw_fold_ashr1:
-; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    movq (%rdi), %rax
-; CHECK-O3-NEXT:    sarq $15, %rax
-; CHECK-O3-NEXT:    movq %rax, (%rdi)
-; CHECK-O3-NEXT:    retq
+; CHECK-O3-CUR-LABEL: rmw_fold_ashr1:
+; CHECK-O3-CUR:       # %bb.0:
+; CHECK-O3-CUR-NEXT:    movq (%rdi), %rax
+; CHECK-O3-CUR-NEXT:    sarq $15, %rax
+; CHECK-O3-CUR-NEXT:    movq %rax, (%rdi)
+; CHECK-O3-CUR-NEXT:    retq
+;
+; CHECK-O3-EX-LABEL: rmw_fold_ashr1:
+; CHECK-O3-EX:       # %bb.0:
+; CHECK-O3-EX-NEXT:    sarq $15, (%rdi)
+; CHECK-O3-EX-NEXT:    retq
   %prev = load atomic i64, i64* %p unordered, align 8
   %val = ashr i64 %prev, 15
   store atomic i64 %val, i64* %p unordered, align 8
@@ -2042,11 +2056,18 @@ define void @rmw_fold_ashr2(i64* %p, i64 %v) {
 ; CHECK-O0-NEXT:    movq %rax, (%rdi)
 ; CHECK-O0-NEXT:    retq
 ;
-; CHECK-O3-LABEL: rmw_fold_ashr2:
-; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    sarxq %rsi, (%rdi), %rax
-; CHECK-O3-NEXT:    movq %rax, (%rdi)
-; CHECK-O3-NEXT:    retq
+; CHECK-O3-CUR-LABEL: rmw_fold_ashr2:
+; CHECK-O3-CUR:       # %bb.0:
+; CHECK-O3-CUR-NEXT:    sarxq %rsi, (%rdi), %rax
+; CHECK-O3-CUR-NEXT:    movq %rax, (%rdi)
+; CHECK-O3-CUR-NEXT:    retq
+;
+; CHECK-O3-EX-LABEL: rmw_fold_ashr2:
+; CHECK-O3-EX:       # %bb.0:
+; CHECK-O3-EX-NEXT:    movq %rsi, %rcx
+; CHECK-O3-EX-NEXT:    # kill: def $cl killed $cl killed $rcx
+; CHECK-O3-EX-NEXT:    sarq %cl, (%rdi)
+; CHECK-O3-EX-NEXT:    retq
   %prev = load atomic i64, i64* %p unordered, align 8
   %val = ashr i64 %prev, %v
   store atomic i64 %val, i64* %p unordered, align 8
@@ -2174,17 +2195,11 @@ define void @rmw_fold_xor2(i64* %p, i64 %v) {
 
 ; Legal to reduce the load width (TODO)
 define i32 @fold_trunc(i64* %p) {
-; CHECK-O0-LABEL: fold_trunc:
-; CHECK-O0:       # %bb.0:
-; CHECK-O0-NEXT:    movq (%rdi), %rax
-; CHECK-O0-NEXT:    # kill: def $eax killed $eax killed $rax
-; CHECK-O0-NEXT:    retq
-;
-; CHECK-O3-LABEL: fold_trunc:
-; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    movq (%rdi), %rax
-; CHECK-O3-NEXT:    # kill: def $eax killed $eax killed $rax
-; CHECK-O3-NEXT:    retq
+; CHECK-LABEL: fold_trunc:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq (%rdi), %rax
+; CHECK-NEXT:    # kill: def $eax killed $eax killed $rax
+; CHECK-NEXT:    retq
   %v = load atomic i64, i64* %p unordered, align 8
   %ret = trunc i64 %v to i32
   ret i32 %ret
@@ -2290,32 +2305,21 @@ define i32 @split_load(i64* %p) {
 
 ; TODO: should return constant
 define i64 @constant_folding(i64* %p) {
-; CHECK-O0-LABEL: constant_folding:
-; CHECK-O0:       # %bb.0:
-; CHECK-O0-NEXT:    movq (%rdi), %rax
-; CHECK-O0-NEXT:    retq
-;
-; CHECK-O3-LABEL: constant_folding:
-; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    movq (%rdi), %rax
-; CHECK-O3-NEXT:    retq
+; CHECK-LABEL: constant_folding:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq (%rdi), %rax
+; CHECK-NEXT:    retq
   %v = load atomic i64, i64* %p unordered, align 8
   ret i64 %v
 }
 
 ; Legal to forward and fold (TODO)
 define i64 @load_forwarding(i64* %p) {
-; CHECK-O0-LABEL: load_forwarding:
-; CHECK-O0:       # %bb.0:
-; CHECK-O0-NEXT:    movq (%rdi), %rax
-; CHECK-O0-NEXT:    orq (%rdi), %rax
-; CHECK-O0-NEXT:    retq
-;
-; CHECK-O3-LABEL: load_forwarding:
-; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    movq (%rdi), %rax
-; CHECK-O3-NEXT:    orq (%rdi), %rax
-; CHECK-O3-NEXT:    retq
+; CHECK-LABEL: load_forwarding:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq (%rdi), %rax
+; CHECK-NEXT:    orq (%rdi), %rax
+; CHECK-NEXT:    retq
   %v = load atomic i64, i64* %p unordered, align 8
   %v2 = load atomic i64, i64* %p unordered, align 8
   %ret = or i64 %v, %v2
@@ -2324,17 +2328,11 @@ define i64 @load_forwarding(i64* %p) {
 
 ; Legal to forward (TODO)
 define i64 @store_forward(i64* %p, i64 %v) {
-; CHECK-O0-LABEL: store_forward:
-; CHECK-O0:       # %bb.0:
-; CHECK-O0-NEXT:    movq %rsi, (%rdi)
-; CHECK-O0-NEXT:    movq (%rdi), %rax
-; CHECK-O0-NEXT:    retq
-;
-; CHECK-O3-LABEL: store_forward:
-; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    movq %rsi, (%rdi)
-; CHECK-O3-NEXT:    movq (%rdi), %rax
-; CHECK-O3-NEXT:    retq
+; CHECK-LABEL: store_forward:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %rsi, (%rdi)
+; CHECK-NEXT:    movq (%rdi), %rax
+; CHECK-NEXT:    retq
   store atomic i64 %v, i64* %p unordered, align 8
   %ret = load atomic i64, i64* %p unordered, align 8
   ret i64 %ret
@@ -2342,17 +2340,11 @@ define i64 @store_forward(i64* %p, i64 %v) {
 
 ; Legal to kill (TODO)
 define void @dead_writeback(i64* %p) {
-; CHECK-O0-LABEL: dead_writeback:
-; CHECK-O0:       # %bb.0:
-; CHECK-O0-NEXT:    movq (%rdi), %rax
-; CHECK-O0-NEXT:    movq %rax, (%rdi)
-; CHECK-O0-NEXT:    retq
-;
-; CHECK-O3-LABEL: dead_writeback:
-; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    movq (%rdi), %rax
-; CHECK-O3-NEXT:    movq %rax, (%rdi)
-; CHECK-O3-NEXT:    retq
+; CHECK-LABEL: dead_writeback:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq (%rdi), %rax
+; CHECK-NEXT:    movq %rax, (%rdi)
+; CHECK-NEXT:    retq
   %v = load atomic i64, i64* %p unordered, align 8
   store atomic i64 %v, i64* %p unordered, align 8
   ret void
@@ -2360,17 +2352,11 @@ define void @dead_writeback(i64* %p) {
 
 ; Legal to kill (TODO)
 define void @dead_store(i64* %p, i64 %v) {
-; CHECK-O0-LABEL: dead_store:
-; CHECK-O0:       # %bb.0:
-; CHECK-O0-NEXT:    movq $0, (%rdi)
-; CHECK-O0-NEXT:    movq %rsi, (%rdi)
-; CHECK-O0-NEXT:    retq
-;
-; CHECK-O3-LABEL: dead_store:
-; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    movq $0, (%rdi)
-; CHECK-O3-NEXT:    movq %rsi, (%rdi)
-; CHECK-O3-NEXT:    retq
+; CHECK-LABEL: dead_store:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq $0, (%rdi)
+; CHECK-NEXT:    movq %rsi, (%rdi)
+; CHECK-NEXT:    retq
   store atomic i64 0, i64* %p unordered, align 8
   store atomic i64 %v, i64* %p unordered, align 8
   ret void
@@ -2384,19 +2370,12 @@ define void @dead_store(i64* %p, i64 %v) {
 ;; isn't violated.
 
 define i64 @nofold_fence(i64* %p) {
-; CHECK-O0-LABEL: nofold_fence:
-; CHECK-O0:       # %bb.0:
-; CHECK-O0-NEXT:    movq (%rdi), %rax
-; CHECK-O0-NEXT:    mfence
-; CHECK-O0-NEXT:    addq $15, %rax
-; CHECK-O0-NEXT:    retq
-;
-; CHECK-O3-LABEL: nofold_fence:
-; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    movq (%rdi), %rax
-; CHECK-O3-NEXT:    mfence
-; CHECK-O3-NEXT:    addq $15, %rax
-; CHECK-O3-NEXT:    retq
+; CHECK-LABEL: nofold_fence:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq (%rdi), %rax
+; CHECK-NEXT:    mfence
+; CHECK-NEXT:    addq $15, %rax
+; CHECK-NEXT:    retq
   %v = load atomic i64, i64* %p unordered, align 8
   fence seq_cst
   %ret = add i64 %v, 15
@@ -2404,19 +2383,12 @@ define i64 @nofold_fence(i64* %p) {
 }
 
 define i64 @nofold_fence_acquire(i64* %p) {
-; CHECK-O0-LABEL: nofold_fence_acquire:
-; CHECK-O0:       # %bb.0:
-; CHECK-O0-NEXT:    movq (%rdi), %rax
-; CHECK-O0-NEXT:    #MEMBARRIER
-; CHECK-O0-NEXT:    addq $15, %rax
-; CHECK-O0-NEXT:    retq
-;
-; CHECK-O3-LABEL: nofold_fence_acquire:
-; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    movq (%rdi), %rax
-; CHECK-O3-NEXT:    #MEMBARRIER
-; CHECK-O3-NEXT:    addq $15, %rax
-; CHECK-O3-NEXT:    retq
+; CHECK-LABEL: nofold_fence_acquire:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq (%rdi), %rax
+; CHECK-NEXT:    #MEMBARRIER
+; CHECK-NEXT:    addq $15, %rax
+; CHECK-NEXT:    retq
   %v = load atomic i64, i64* %p unordered, align 8
   fence acquire
   %ret = add i64 %v, 15
@@ -2425,19 +2397,12 @@ define i64 @nofold_fence_acquire(i64* %p) {
 
 
 define i64 @nofold_stfence(i64* %p) {
-; CHECK-O0-LABEL: nofold_stfence:
-; CHECK-O0:       # %bb.0:
-; CHECK-O0-NEXT:    movq (%rdi), %rax
-; CHECK-O0-NEXT:    #MEMBARRIER
-; CHECK-O0-NEXT:    addq $15, %rax
-; CHECK-O0-NEXT:    retq
-;
-; CHECK-O3-LABEL: nofold_stfence:
-; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    movq (%rdi), %rax
-; CHECK-O3-NEXT:    #MEMBARRIER
-; CHECK-O3-NEXT:    addq $15, %rax
-; CHECK-O3-NEXT:    retq
+; CHECK-LABEL: nofold_stfence:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq (%rdi), %rax
+; CHECK-NEXT:    #MEMBARRIER
+; CHECK-NEXT:    addq $15, %rax
+; CHECK-NEXT:    retq
   %v = load atomic i64, i64* %p unordered, align 8
   fence syncscope("singlethread") seq_cst
   %ret = add i64 %v, 15
@@ -2473,12 +2438,19 @@ define i64 @fold_constant_clobber(i64* %p, i64 %arg) {
 ; CHECK-O0-NEXT:    addq %rsi, %rax
 ; CHECK-O0-NEXT:    retq
 ;
-; CHECK-O3-LABEL: fold_constant_clobber:
-; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    movq {{.*}}(%rip), %rax
-; CHECK-O3-NEXT:    movq $5, (%rdi)
-; CHECK-O3-NEXT:    addq %rsi, %rax
-; CHECK-O3-NEXT:    retq
+; CHECK-O3-CUR-LABEL: fold_constant_clobber:
+; CHECK-O3-CUR:       # %bb.0:
+; CHECK-O3-CUR-NEXT:    movq {{.*}}(%rip), %rax
+; CHECK-O3-CUR-NEXT:    movq $5, (%rdi)
+; CHECK-O3-CUR-NEXT:    addq %rsi, %rax
+; CHECK-O3-CUR-NEXT:    retq
+;
+; CHECK-O3-EX-LABEL: fold_constant_clobber:
+; CHECK-O3-EX:       # %bb.0:
+; CHECK-O3-EX-NEXT:    movq %rsi, %rax
+; CHECK-O3-EX-NEXT:    addq {{.*}}(%rip), %rax
+; CHECK-O3-EX-NEXT:    movq $5, (%rdi)
+; CHECK-O3-EX-NEXT:    retq
   %v = load atomic i64, i64* @Constant unordered, align 8
   store i64 5, i64* %p
   %ret = add i64 %v, %arg
@@ -2493,12 +2465,19 @@ define i64 @fold_constant_fence(i64 %arg) {
 ; CHECK-O0-NEXT:    addq %rdi, %rax
 ; CHECK-O0-NEXT:    retq
 ;
-; CHECK-O3-LABEL: fold_constant_fence:
-; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    movq {{.*}}(%rip), %rax
-; CHECK-O3-NEXT:    mfence
-; CHECK-O3-NEXT:    addq %rdi, %rax
-; CHECK-O3-NEXT:    retq
+; CHECK-O3-CUR-LABEL: fold_constant_fence:
+; CHECK-O3-CUR:       # %bb.0:
+; CHECK-O3-CUR-NEXT:    movq {{.*}}(%rip), %rax
+; CHECK-O3-CUR-NEXT:    mfence
+; CHECK-O3-CUR-NEXT:    addq %rdi, %rax
+; CHECK-O3-CUR-NEXT:    retq
+;
+; CHECK-O3-EX-LABEL: fold_constant_fence:
+; CHECK-O3-EX:       # %bb.0:
+; CHECK-O3-EX-NEXT:    movq %rdi, %rax
+; CHECK-O3-EX-NEXT:    addq {{.*}}(%rip), %rax
+; CHECK-O3-EX-NEXT:    mfence
+; CHECK-O3-EX-NEXT:    retq
   %v = load atomic i64, i64* @Constant unordered, align 8
   fence seq_cst
   %ret = add i64 %v, %arg
@@ -2513,12 +2492,19 @@ define i64 @fold_invariant_clobber(i64* dereferenceable(8) %p, i64 %arg) {
 ; CHECK-O0-NEXT:    addq %rsi, %rax
 ; CHECK-O0-NEXT:    retq
 ;
-; CHECK-O3-LABEL: fold_invariant_clobber:
-; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    movq (%rdi), %rax
-; CHECK-O3-NEXT:    movq $5, (%rdi)
-; CHECK-O3-NEXT:    addq %rsi, %rax
-; CHECK-O3-NEXT:    retq
+; CHECK-O3-CUR-LABEL: fold_invariant_clobber:
+; CHECK-O3-CUR:       # %bb.0:
+; CHECK-O3-CUR-NEXT:    movq (%rdi), %rax
+; CHECK-O3-CUR-NEXT:    movq $5, (%rdi)
+; CHECK-O3-CUR-NEXT:    addq %rsi, %rax
+; CHECK-O3-CUR-NEXT:    retq
+;
+; CHECK-O3-EX-LABEL: fold_invariant_clobber:
+; CHECK-O3-EX:       # %bb.0:
+; CHECK-O3-EX-NEXT:    movq %rsi, %rax
+; CHECK-O3-EX-NEXT:    addq (%rdi), %rax
+; CHECK-O3-EX-NEXT:    movq $5, (%rdi)
+; CHECK-O3-EX-NEXT:    retq
   %v = load atomic i64, i64* %p unordered, align 8, !invariant.load !{}
   store i64 5, i64* %p
   %ret = add i64 %v, %arg
@@ -2534,15 +2520,234 @@ define i64 @fold_invariant_fence(i64* dereferenceable(8) %p, i64 %arg) {
 ; CHECK-O0-NEXT:    addq %rsi, %rax
 ; CHECK-O0-NEXT:    retq
 ;
-; CHECK-O3-LABEL: fold_invariant_fence:
-; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    movq (%rdi), %rax
-; CHECK-O3-NEXT:    mfence
-; CHECK-O3-NEXT:    addq %rsi, %rax
-; CHECK-O3-NEXT:    retq
+; CHECK-O3-CUR-LABEL: fold_invariant_fence:
+; CHECK-O3-CUR:       # %bb.0:
+; CHECK-O3-CUR-NEXT:    movq (%rdi), %rax
+; CHECK-O3-CUR-NEXT:    mfence
+; CHECK-O3-CUR-NEXT:    addq %rsi, %rax
+; CHECK-O3-CUR-NEXT:    retq
+;
+; CHECK-O3-EX-LABEL: fold_invariant_fence:
+; CHECK-O3-EX:       # %bb.0:
+; CHECK-O3-EX-NEXT:    movq %rsi, %rax
+; CHECK-O3-EX-NEXT:    addq (%rdi), %rax
+; CHECK-O3-EX-NEXT:    mfence
+; CHECK-O3-EX-NEXT:    retq
   %v = load atomic i64, i64* %p unordered, align 8, !invariant.load !{}
   fence seq_cst
   %ret = add i64 %v, %arg
   ret i64 %ret
 }
 
+
+; Exercise a few cases involving any extend idioms
+
+define i16 @load_i8_anyext_i16(i8* %ptr) {
+; CHECK-O0-CUR-LABEL: load_i8_anyext_i16:
+; CHECK-O0-CUR:       # %bb.0:
+; CHECK-O0-CUR-NEXT:    movb (%rdi), %al
+; CHECK-O0-CUR-NEXT:    movzbl %al, %ecx
+; CHECK-O0-CUR-NEXT:    # kill: def $cx killed $cx killed $ecx
+; CHECK-O0-CUR-NEXT:    movw %cx, %ax
+; CHECK-O0-CUR-NEXT:    retq
+;
+; CHECK-O3-CUR-LABEL: load_i8_anyext_i16:
+; CHECK-O3-CUR:       # %bb.0:
+; CHECK-O3-CUR-NEXT:    movzbl (%rdi), %eax
+; CHECK-O3-CUR-NEXT:    # kill: def $ax killed $ax killed $eax
+; CHECK-O3-CUR-NEXT:    retq
+;
+; CHECK-O0-EX-LABEL: load_i8_anyext_i16:
+; CHECK-O0-EX:       # %bb.0:
+; CHECK-O0-EX-NEXT:    vpbroadcastb (%rdi), %xmm0
+; CHECK-O0-EX-NEXT:    vmovd %xmm0, %eax
+; CHECK-O0-EX-NEXT:    # kill: def $ax killed $ax killed $eax
+; CHECK-O0-EX-NEXT:    retq
+;
+; CHECK-O3-EX-LABEL: load_i8_anyext_i16:
+; CHECK-O3-EX:       # %bb.0:
+; CHECK-O3-EX-NEXT:    vpbroadcastb (%rdi), %xmm0
+; CHECK-O3-EX-NEXT:    vmovd %xmm0, %eax
+; CHECK-O3-EX-NEXT:    # kill: def $ax killed $ax killed $eax
+; CHECK-O3-EX-NEXT:    retq
+  %v = load atomic i8, i8* %ptr unordered, align 2
+  %vec = insertelement <2 x i8> undef, i8 %v, i32 0
+  %res = bitcast <2 x i8> %vec to i16
+  ret i16 %res
+}
+
+define i32 @load_i8_anyext_i32(i8* %ptr) {
+; CHECK-O0-CUR-LABEL: load_i8_anyext_i32:
+; CHECK-O0-CUR:       # %bb.0:
+; CHECK-O0-CUR-NEXT:    movb (%rdi), %al
+; CHECK-O0-CUR-NEXT:    movzbl %al, %eax
+; CHECK-O0-CUR-NEXT:    retq
+;
+; CHECK-O3-CUR-LABEL: load_i8_anyext_i32:
+; CHECK-O3-CUR:       # %bb.0:
+; CHECK-O3-CUR-NEXT:    movzbl (%rdi), %eax
+; CHECK-O3-CUR-NEXT:    retq
+;
+; CHECK-O0-EX-LABEL: load_i8_anyext_i32:
+; CHECK-O0-EX:       # %bb.0:
+; CHECK-O0-EX-NEXT:    vpbroadcastb (%rdi), %xmm0
+; CHECK-O0-EX-NEXT:    vmovd %xmm0, %eax
+; CHECK-O0-EX-NEXT:    retq
+;
+; CHECK-O3-EX-LABEL: load_i8_anyext_i32:
+; CHECK-O3-EX:       # %bb.0:
+; CHECK-O3-EX-NEXT:    vpbroadcastb (%rdi), %xmm0
+; CHECK-O3-EX-NEXT:    vmovd %xmm0, %eax
+; CHECK-O3-EX-NEXT:    retq
+  %v = load atomic i8, i8* %ptr unordered, align 4
+  %vec = insertelement <4 x i8> undef, i8 %v, i32 0
+  %res = bitcast <4 x i8> %vec to i32
+  ret i32 %res
+}
+
+define i32 @load_i16_anyext_i32(i16* %ptr) {
+; CHECK-O0-CUR-LABEL: load_i16_anyext_i32:
+; CHECK-O0-CUR:       # %bb.0:
+; CHECK-O0-CUR-NEXT:    movw (%rdi), %ax
+; CHECK-O0-CUR-NEXT:    # implicit-def: $ecx
+; CHECK-O0-CUR-NEXT:    movw %ax, %cx
+; CHECK-O0-CUR-NEXT:    movl %ecx, %eax
+; CHECK-O0-CUR-NEXT:    retq
+;
+; CHECK-O3-CUR-LABEL: load_i16_anyext_i32:
+; CHECK-O3-CUR:       # %bb.0:
+; CHECK-O3-CUR-NEXT:    movzwl (%rdi), %eax
+; CHECK-O3-CUR-NEXT:    retq
+;
+; CHECK-O0-EX-LABEL: load_i16_anyext_i32:
+; CHECK-O0-EX:       # %bb.0:
+; CHECK-O0-EX-NEXT:    vpbroadcastw (%rdi), %xmm0
+; CHECK-O0-EX-NEXT:    vmovd %xmm0, %eax
+; CHECK-O0-EX-NEXT:    retq
+;
+; CHECK-O3-EX-LABEL: load_i16_anyext_i32:
+; CHECK-O3-EX:       # %bb.0:
+; CHECK-O3-EX-NEXT:    vpbroadcastw (%rdi), %xmm0
+; CHECK-O3-EX-NEXT:    vmovd %xmm0, %eax
+; CHECK-O3-EX-NEXT:    retq
+  %v = load atomic i16, i16* %ptr unordered, align 4
+  %vec = insertelement <2 x i16> undef, i16 %v, i64 0
+  %res = bitcast <2 x i16> %vec to i32
+  ret i32 %res
+}
+
+define i64 @load_i16_anyext_i64(i16* %ptr) {
+; CHECK-O0-CUR-LABEL: load_i16_anyext_i64:
+; CHECK-O0-CUR:       # %bb.0:
+; CHECK-O0-CUR-NEXT:    movw (%rdi), %ax
+; CHECK-O0-CUR-NEXT:    # implicit-def: $ecx
+; CHECK-O0-CUR-NEXT:    movw %ax, %cx
+; CHECK-O0-CUR-NEXT:    vmovd %ecx, %xmm0
+; CHECK-O0-CUR-NEXT:    vmovq %xmm0, %rax
+; CHECK-O0-CUR-NEXT:    retq
+;
+; CHECK-O3-CUR-LABEL: load_i16_anyext_i64:
+; CHECK-O3-CUR:       # %bb.0:
+; CHECK-O3-CUR-NEXT:    movzwl (%rdi), %eax
+; CHECK-O3-CUR-NEXT:    vmovd %eax, %xmm0
+; CHECK-O3-CUR-NEXT:    vmovq %xmm0, %rax
+; CHECK-O3-CUR-NEXT:    retq
+;
+; CHECK-O0-EX-LABEL: load_i16_anyext_i64:
+; CHECK-O0-EX:       # %bb.0:
+; CHECK-O0-EX-NEXT:    vpbroadcastw (%rdi), %xmm0
+; CHECK-O0-EX-NEXT:    vmovq %xmm0, %rax
+; CHECK-O0-EX-NEXT:    retq
+;
+; CHECK-O3-EX-LABEL: load_i16_anyext_i64:
+; CHECK-O3-EX:       # %bb.0:
+; CHECK-O3-EX-NEXT:    vpbroadcastw (%rdi), %xmm0
+; CHECK-O3-EX-NEXT:    vmovq %xmm0, %rax
+; CHECK-O3-EX-NEXT:    retq
+  %v = load atomic i16, i16* %ptr unordered, align 8
+  %vec = insertelement <4 x i16> undef, i16 %v, i64 0
+  %res = bitcast <4 x i16> %vec to i64
+  ret i64 %res
+}
+
+; TODO: Would be legal to combine for legal atomic wider types
+define i16 @load_combine(i8* %p) {
+; CHECK-O0-LABEL: load_combine:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movb (%rdi), %al
+; CHECK-O0-NEXT:    movb 1(%rdi), %cl
+; CHECK-O0-NEXT:    movzbl %al, %edx
+; CHECK-O0-NEXT:    # kill: def $dx killed $dx killed $edx
+; CHECK-O0-NEXT:    movzbl %cl, %esi
+; CHECK-O0-NEXT:    # kill: def $si killed $si killed $esi
+; CHECK-O0-NEXT:    shlw $8, %si
+; CHECK-O0-NEXT:    orw %si, %dx
+; CHECK-O0-NEXT:    movw %dx, %ax
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: load_combine:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movzbl (%rdi), %ecx
+; CHECK-O3-NEXT:    movzbl 1(%rdi), %eax
+; CHECK-O3-NEXT:    shll $8, %eax
+; CHECK-O3-NEXT:    orl %ecx, %eax
+; CHECK-O3-NEXT:    # kill: def $ax killed $ax killed $eax
+; CHECK-O3-NEXT:    retq
+  %v1 = load atomic i8, i8* %p unordered, align 2
+  %p2 = getelementptr i8, i8* %p, i64 1
+  %v2 = load atomic i8, i8* %p2 unordered, align 1
+  %v1.ext = zext i8 %v1 to i16
+  %v2.ext = zext i8 %v2 to i16
+  %v2.sht = shl i16 %v2.ext, 8
+  %res = or i16 %v1.ext, %v2.sht
+  ret i16 %res
+}
+
+define i1 @fold_cmp_over_fence(i32* %p, i32 %v1) {
+; CHECK-O0-LABEL: fold_cmp_over_fence:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movl (%rdi), %eax
+; CHECK-O0-NEXT:    mfence
+; CHECK-O0-NEXT:    cmpl %eax, %esi
+; CHECK-O0-NEXT:    jne .LBB116_2
+; CHECK-O0-NEXT:  # %bb.1: # %taken
+; CHECK-O0-NEXT:    movb $1, %al
+; CHECK-O0-NEXT:    retq
+; CHECK-O0-NEXT:  .LBB116_2: # %untaken
+; CHECK-O0-NEXT:    xorl %eax, %eax
+; CHECK-O0-NEXT:    # kill: def $al killed $al killed $eax
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-CUR-LABEL: fold_cmp_over_fence:
+; CHECK-O3-CUR:       # %bb.0:
+; CHECK-O3-CUR-NEXT:    movl (%rdi), %eax
+; CHECK-O3-CUR-NEXT:    mfence
+; CHECK-O3-CUR-NEXT:    cmpl %eax, %esi
+; CHECK-O3-CUR-NEXT:    jne .LBB116_2
+; CHECK-O3-CUR-NEXT:  # %bb.1: # %taken
+; CHECK-O3-CUR-NEXT:    movb $1, %al
+; CHECK-O3-CUR-NEXT:    retq
+; CHECK-O3-CUR-NEXT:  .LBB116_2: # %untaken
+; CHECK-O3-CUR-NEXT:    xorl %eax, %eax
+; CHECK-O3-CUR-NEXT:    retq
+;
+; CHECK-O3-EX-LABEL: fold_cmp_over_fence:
+; CHECK-O3-EX:       # %bb.0:
+; CHECK-O3-EX-NEXT:    cmpl (%rdi), %esi
+; CHECK-O3-EX-NEXT:    mfence
+; CHECK-O3-EX-NEXT:    jne .LBB116_2
+; CHECK-O3-EX-NEXT:  # %bb.1: # %taken
+; CHECK-O3-EX-NEXT:    movb $1, %al
+; CHECK-O3-EX-NEXT:    retq
+; CHECK-O3-EX-NEXT:  .LBB116_2: # %untaken
+; CHECK-O3-EX-NEXT:    xorl %eax, %eax
+; CHECK-O3-EX-NEXT:    retq
+  %v2 = load atomic i32, i32* %p unordered, align 4
+  fence seq_cst
+  %cmp = icmp eq i32 %v1, %v2
+  br i1 %cmp, label %taken, label %untaken
+taken:
+  ret i1 true
+untaken:
+  ret i1 false
+}
diff --git a/llvm/test/CodeGen/X86/atomic32.ll b/llvm/test/CodeGen/X86/atomic32.ll
index 461e0b5be..4fb03356f 100644
--- a/llvm/test/CodeGen/X86/atomic32.ll
+++ b/llvm/test/CodeGen/X86/atomic32.ll
@@ -2,6 +2,7 @@
 ; RUN: llc < %s -O0 -mtriple=x86_64-unknown-unknown -mcpu=corei7 -verify-machineinstrs | FileCheck %s -check-prefixes=X64,X64-CMOV
 ; RUN: llc < %s -O0 -mtriple=i686-unknown-unknown -mcpu=corei7 -verify-machineinstrs | FileCheck %s -check-prefixes=X86,X86-CMOV
 ; RUN: llc < %s -O0 -mtriple=i686-unknown-unknown -mcpu=corei7 -mattr=-cmov,-sse -verify-machineinstrs | FileCheck %s --check-prefixes=X86,X86-NOCMOV
+; RUN: llc < %s -O0 -mtriple=i686-unknown-unknown -mcpu=corei7 -mattr=-cmov,-sse,-x87 -verify-machineinstrs | FileCheck %s --check-prefixes=X86,X86-NOX87
 
 @sc32 = external global i32
 @fsc32 = external global float
@@ -361,6 +362,48 @@ define void @atomic_fetch_max32(i32 %x) nounwind {
 ; X86-NOCMOV-NEXT:    popl %esi
 ; X86-NOCMOV-NEXT:    popl %ebx
 ; X86-NOCMOV-NEXT:    retl
+;
+; X86-NOX87-LABEL: atomic_fetch_max32:
+; X86-NOX87:       # %bb.0:
+; X86-NOX87-NEXT:    pushl %ebx
+; X86-NOX87-NEXT:    pushl %esi
+; X86-NOX87-NEXT:    subl $20, %esp
+; X86-NOX87-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOX87-NEXT:    movl sc32, %ecx
+; X86-NOX87-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NOX87-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NOX87-NEXT:  .LBB6_1: # %atomicrmw.start
+; X86-NOX87-NEXT:    # =>This Inner Loop Header: Depth=1
+; X86-NOX87-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NOX87-NEXT:    movl %eax, %ecx
+; X86-NOX87-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NOX87-NEXT:    subl %edx, %ecx
+; X86-NOX87-NEXT:    movl %eax, %esi
+; X86-NOX87-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NOX87-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NOX87-NEXT:    jge .LBB6_4
+; X86-NOX87-NEXT:  # %bb.3: # %atomicrmw.start
+; X86-NOX87-NEXT:    # in Loop: Header=BB6_1 Depth=1
+; X86-NOX87-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NOX87-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NOX87-NEXT:  .LBB6_4: # %atomicrmw.start
+; X86-NOX87-NEXT:    # in Loop: Header=BB6_1 Depth=1
+; X86-NOX87-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NOX87-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NOX87-NEXT:    movl %eax, (%esp) # 4-byte Spill
+; X86-NOX87-NEXT:    movl %ecx, %eax
+; X86-NOX87-NEXT:    movl (%esp), %edx # 4-byte Reload
+; X86-NOX87-NEXT:    lock cmpxchgl %edx, sc32
+; X86-NOX87-NEXT:    sete %bl
+; X86-NOX87-NEXT:    testb $1, %bl
+; X86-NOX87-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NOX87-NEXT:    jne .LBB6_2
+; X86-NOX87-NEXT:    jmp .LBB6_1
+; X86-NOX87-NEXT:  .LBB6_2: # %atomicrmw.end
+; X86-NOX87-NEXT:    addl $20, %esp
+; X86-NOX87-NEXT:    popl %esi
+; X86-NOX87-NEXT:    popl %ebx
+; X86-NOX87-NEXT:    retl
   %t1 = atomicrmw max  i32* @sc32, i32 %x acquire
   ret void
 }
@@ -456,6 +499,48 @@ define void @atomic_fetch_min32(i32 %x) nounwind {
 ; X86-NOCMOV-NEXT:    popl %esi
 ; X86-NOCMOV-NEXT:    popl %ebx
 ; X86-NOCMOV-NEXT:    retl
+;
+; X86-NOX87-LABEL: atomic_fetch_min32:
+; X86-NOX87:       # %bb.0:
+; X86-NOX87-NEXT:    pushl %ebx
+; X86-NOX87-NEXT:    pushl %esi
+; X86-NOX87-NEXT:    subl $20, %esp
+; X86-NOX87-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOX87-NEXT:    movl sc32, %ecx
+; X86-NOX87-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NOX87-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NOX87-NEXT:  .LBB7_1: # %atomicrmw.start
+; X86-NOX87-NEXT:    # =>This Inner Loop Header: Depth=1
+; X86-NOX87-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NOX87-NEXT:    movl %eax, %ecx
+; X86-NOX87-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NOX87-NEXT:    subl %edx, %ecx
+; X86-NOX87-NEXT:    movl %eax, %esi
+; X86-NOX87-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NOX87-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NOX87-NEXT:    jle .LBB7_4
+; X86-NOX87-NEXT:  # %bb.3: # %atomicrmw.start
+; X86-NOX87-NEXT:    # in Loop: Header=BB7_1 Depth=1
+; X86-NOX87-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NOX87-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NOX87-NEXT:  .LBB7_4: # %atomicrmw.start
+; X86-NOX87-NEXT:    # in Loop: Header=BB7_1 Depth=1
+; X86-NOX87-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NOX87-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NOX87-NEXT:    movl %eax, (%esp) # 4-byte Spill
+; X86-NOX87-NEXT:    movl %ecx, %eax
+; X86-NOX87-NEXT:    movl (%esp), %edx # 4-byte Reload
+; X86-NOX87-NEXT:    lock cmpxchgl %edx, sc32
+; X86-NOX87-NEXT:    sete %bl
+; X86-NOX87-NEXT:    testb $1, %bl
+; X86-NOX87-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NOX87-NEXT:    jne .LBB7_2
+; X86-NOX87-NEXT:    jmp .LBB7_1
+; X86-NOX87-NEXT:  .LBB7_2: # %atomicrmw.end
+; X86-NOX87-NEXT:    addl $20, %esp
+; X86-NOX87-NEXT:    popl %esi
+; X86-NOX87-NEXT:    popl %ebx
+; X86-NOX87-NEXT:    retl
   %t1 = atomicrmw min  i32* @sc32, i32 %x acquire
   ret void
 }
@@ -551,6 +636,48 @@ define void @atomic_fetch_umax32(i32 %x) nounwind {
 ; X86-NOCMOV-NEXT:    popl %esi
 ; X86-NOCMOV-NEXT:    popl %ebx
 ; X86-NOCMOV-NEXT:    retl
+;
+; X86-NOX87-LABEL: atomic_fetch_umax32:
+; X86-NOX87:       # %bb.0:
+; X86-NOX87-NEXT:    pushl %ebx
+; X86-NOX87-NEXT:    pushl %esi
+; X86-NOX87-NEXT:    subl $20, %esp
+; X86-NOX87-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOX87-NEXT:    movl sc32, %ecx
+; X86-NOX87-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NOX87-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NOX87-NEXT:  .LBB8_1: # %atomicrmw.start
+; X86-NOX87-NEXT:    # =>This Inner Loop Header: Depth=1
+; X86-NOX87-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NOX87-NEXT:    movl %eax, %ecx
+; X86-NOX87-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NOX87-NEXT:    subl %edx, %ecx
+; X86-NOX87-NEXT:    movl %eax, %esi
+; X86-NOX87-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NOX87-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NOX87-NEXT:    ja .LBB8_4
+; X86-NOX87-NEXT:  # %bb.3: # %atomicrmw.start
+; X86-NOX87-NEXT:    # in Loop: Header=BB8_1 Depth=1
+; X86-NOX87-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NOX87-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NOX87-NEXT:  .LBB8_4: # %atomicrmw.start
+; X86-NOX87-NEXT:    # in Loop: Header=BB8_1 Depth=1
+; X86-NOX87-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NOX87-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NOX87-NEXT:    movl %eax, (%esp) # 4-byte Spill
+; X86-NOX87-NEXT:    movl %ecx, %eax
+; X86-NOX87-NEXT:    movl (%esp), %edx # 4-byte Reload
+; X86-NOX87-NEXT:    lock cmpxchgl %edx, sc32
+; X86-NOX87-NEXT:    sete %bl
+; X86-NOX87-NEXT:    testb $1, %bl
+; X86-NOX87-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NOX87-NEXT:    jne .LBB8_2
+; X86-NOX87-NEXT:    jmp .LBB8_1
+; X86-NOX87-NEXT:  .LBB8_2: # %atomicrmw.end
+; X86-NOX87-NEXT:    addl $20, %esp
+; X86-NOX87-NEXT:    popl %esi
+; X86-NOX87-NEXT:    popl %ebx
+; X86-NOX87-NEXT:    retl
   %t1 = atomicrmw umax i32* @sc32, i32 %x acquire
   ret void
 }
@@ -646,6 +773,48 @@ define void @atomic_fetch_umin32(i32 %x) nounwind {
 ; X86-NOCMOV-NEXT:    popl %esi
 ; X86-NOCMOV-NEXT:    popl %ebx
 ; X86-NOCMOV-NEXT:    retl
+;
+; X86-NOX87-LABEL: atomic_fetch_umin32:
+; X86-NOX87:       # %bb.0:
+; X86-NOX87-NEXT:    pushl %ebx
+; X86-NOX87-NEXT:    pushl %esi
+; X86-NOX87-NEXT:    subl $20, %esp
+; X86-NOX87-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOX87-NEXT:    movl sc32, %ecx
+; X86-NOX87-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NOX87-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NOX87-NEXT:  .LBB9_1: # %atomicrmw.start
+; X86-NOX87-NEXT:    # =>This Inner Loop Header: Depth=1
+; X86-NOX87-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NOX87-NEXT:    movl %eax, %ecx
+; X86-NOX87-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NOX87-NEXT:    subl %edx, %ecx
+; X86-NOX87-NEXT:    movl %eax, %esi
+; X86-NOX87-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NOX87-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NOX87-NEXT:    jbe .LBB9_4
+; X86-NOX87-NEXT:  # %bb.3: # %atomicrmw.start
+; X86-NOX87-NEXT:    # in Loop: Header=BB9_1 Depth=1
+; X86-NOX87-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NOX87-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NOX87-NEXT:  .LBB9_4: # %atomicrmw.start
+; X86-NOX87-NEXT:    # in Loop: Header=BB9_1 Depth=1
+; X86-NOX87-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NOX87-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NOX87-NEXT:    movl %eax, (%esp) # 4-byte Spill
+; X86-NOX87-NEXT:    movl %ecx, %eax
+; X86-NOX87-NEXT:    movl (%esp), %edx # 4-byte Reload
+; X86-NOX87-NEXT:    lock cmpxchgl %edx, sc32
+; X86-NOX87-NEXT:    sete %bl
+; X86-NOX87-NEXT:    testb $1, %bl
+; X86-NOX87-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NOX87-NEXT:    jne .LBB9_2
+; X86-NOX87-NEXT:    jmp .LBB9_1
+; X86-NOX87-NEXT:  .LBB9_2: # %atomicrmw.end
+; X86-NOX87-NEXT:    addl $20, %esp
+; X86-NOX87-NEXT:    popl %esi
+; X86-NOX87-NEXT:    popl %ebx
+; X86-NOX87-NEXT:    retl
   %t1 = atomicrmw umin i32* @sc32, i32 %x acquire
   ret void
 }
@@ -721,6 +890,12 @@ define void @atomic_fetch_swapf32(float %x) nounwind {
 ; X86-NOCMOV-NEXT:    xchgl %eax, fsc32
 ; X86-NOCMOV-NEXT:    popl %eax
 ; X86-NOCMOV-NEXT:    retl
+;
+; X86-NOX87-LABEL: atomic_fetch_swapf32:
+; X86-NOX87:       # %bb.0:
+; X86-NOX87-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOX87-NEXT:    xchgl %eax, fsc32
+; X86-NOX87-NEXT:    retl
   %t1 = atomicrmw xchg float* @fsc32, float %x acquire
   ret void
 }
diff --git a/llvm/test/CodeGen/X86/atomicf128.ll b/llvm/test/CodeGen/X86/atomicf128.ll
index 6455fc5eb..cbec96c1c 100644
--- a/llvm/test/CodeGen/X86/atomicf128.ll
+++ b/llvm/test/CodeGen/X86/atomicf128.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-apple-macosx10.9 -verify-machineinstrs -mattr=cx16 | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-apple-macosx10.9 -verify-machineinstrs -mattr=cx16 -mattr=-sse | FileCheck %s --check-prefix=NOSSE
 
 ; FIXME: This test has a fatal error in 32-bit mode
 
@@ -9,19 +10,42 @@ define void @atomic_fetch_swapf128(fp128 %x) nounwind {
 ; CHECK-LABEL: atomic_fetch_swapf128:
 ; CHECK:       ## %bb.0:
 ; CHECK-NEXT:    pushq %rbx
-; CHECK-NEXT:    movq %rsi, %rcx
-; CHECK-NEXT:    movq %rdi, %rbx
 ; CHECK-NEXT:    movq _fsc128@{{.*}}(%rip), %rsi
-; CHECK-NEXT:    movq (%rsi), %rax
-; CHECK-NEXT:    movq 8(%rsi), %rdx
+; CHECK-NEXT:    movaps (%rsi), %xmm1
 ; CHECK-NEXT:    .p2align 4, 0x90
 ; CHECK-NEXT:  LBB0_1: ## %atomicrmw.start
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movq -{{[0-9]+}}(%rsp), %rbx
+; CHECK-NEXT:    movq -{{[0-9]+}}(%rsp), %rcx
+; CHECK-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movq -{{[0-9]+}}(%rsp), %rax
+; CHECK-NEXT:    movq -{{[0-9]+}}(%rsp), %rdx
 ; CHECK-NEXT:    lock cmpxchg16b (%rsi)
+; CHECK-NEXT:    movq %rdx, -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movaps -{{[0-9]+}}(%rsp), %xmm1
 ; CHECK-NEXT:    jne LBB0_1
 ; CHECK-NEXT:  ## %bb.2: ## %atomicrmw.end
 ; CHECK-NEXT:    popq %rbx
 ; CHECK-NEXT:    retq
+;
+; NOSSE-LABEL: atomic_fetch_swapf128:
+; NOSSE:       ## %bb.0:
+; NOSSE-NEXT:    pushq %rbx
+; NOSSE-NEXT:    movq %rsi, %rcx
+; NOSSE-NEXT:    movq %rdi, %rbx
+; NOSSE-NEXT:    movq _fsc128@{{.*}}(%rip), %rsi
+; NOSSE-NEXT:    movq (%rsi), %rax
+; NOSSE-NEXT:    movq 8(%rsi), %rdx
+; NOSSE-NEXT:    .p2align 4, 0x90
+; NOSSE-NEXT:  LBB0_1: ## %atomicrmw.start
+; NOSSE-NEXT:    ## =>This Inner Loop Header: Depth=1
+; NOSSE-NEXT:    lock cmpxchg16b (%rsi)
+; NOSSE-NEXT:    jne LBB0_1
+; NOSSE-NEXT:  ## %bb.2: ## %atomicrmw.end
+; NOSSE-NEXT:    popq %rbx
+; NOSSE-NEXT:    retq
   %t1 = atomicrmw xchg fp128* @fsc128, fp128 %x acquire
   ret void
 }
diff --git a/llvm/test/CodeGen/X86/avg-mask.ll b/llvm/test/CodeGen/X86/avg-mask.ll
index 3519faa85..ebe9e75ab 100644
--- a/llvm/test/CodeGen/X86/avg-mask.ll
+++ b/llvm/test/CodeGen/X86/avg-mask.ll
@@ -123,30 +123,33 @@ define <32 x i8> @avg_v32i8_maskz(<32 x i8> %a, <32 x i8> %b, i32 %mask) nounwin
 define <64 x i8> @avg_v64i8_mask(<64 x i8> %a, <64 x i8> %b, <64 x i8> %src, i64 %mask) nounwind {
 ; AVX512F-LABEL: avg_v64i8_mask:
 ; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm2, %ymm3
 ; AVX512F-NEXT:    movq %rdi, %rax
-; AVX512F-NEXT:    movq %rdi, %rcx
+; AVX512F-NEXT:    movl %edi, %ecx
 ; AVX512F-NEXT:    kmovw %edi, %k1
-; AVX512F-NEXT:    movl %edi, %edx
-; AVX512F-NEXT:    shrl $16, %edx
-; AVX512F-NEXT:    shrq $32, %rax
-; AVX512F-NEXT:    shrq $48, %rcx
-; AVX512F-NEXT:    vpavgb %ymm3, %ymm1, %ymm1
-; AVX512F-NEXT:    vpavgb %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    shrq $32, %rdi
+; AVX512F-NEXT:    shrq $48, %rax
+; AVX512F-NEXT:    shrl $16, %ecx
+; AVX512F-NEXT:    vpavgb %ymm1, %ymm0, %ymm4
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm1
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
+; AVX512F-NEXT:    vpavgb %ymm1, %ymm0, %ymm0
 ; AVX512F-NEXT:    kmovw %ecx, %k2
 ; AVX512F-NEXT:    kmovw %eax, %k3
-; AVX512F-NEXT:    kmovw %edx, %k4
-; AVX512F-NEXT:    vpternlogd $255, %zmm2, %zmm2, %zmm2 {%k1} {z}
-; AVX512F-NEXT:    vpmovdb %zmm2, %xmm2
-; AVX512F-NEXT:    vpternlogd $255, %zmm3, %zmm3, %zmm3 {%k4} {z}
-; AVX512F-NEXT:    vpmovdb %zmm3, %xmm3
-; AVX512F-NEXT:    vinserti128 $1, %xmm3, %ymm2, %ymm2
-; AVX512F-NEXT:    vpblendvb %ymm2, %ymm0, %ymm4, %ymm0
-; AVX512F-NEXT:    vpternlogd $255, %zmm2, %zmm2, %zmm2 {%k3} {z}
-; AVX512F-NEXT:    vpmovdb %zmm2, %xmm2
+; AVX512F-NEXT:    kmovw %edi, %k4
+; AVX512F-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k4} {z}
+; AVX512F-NEXT:    vpmovdb %zmm1, %xmm1
+; AVX512F-NEXT:    vpternlogd $255, %zmm5, %zmm5, %zmm5 {%k3} {z}
+; AVX512F-NEXT:    vpmovdb %zmm5, %xmm5
+; AVX512F-NEXT:    vinserti128 $1, %xmm5, %ymm1, %ymm1
+; AVX512F-NEXT:    vpblendvb %ymm1, %ymm0, %ymm3, %ymm0
+; AVX512F-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k1} {z}
+; AVX512F-NEXT:    vpmovdb %zmm1, %xmm1
 ; AVX512F-NEXT:    vpternlogd $255, %zmm3, %zmm3, %zmm3 {%k2} {z}
 ; AVX512F-NEXT:    vpmovdb %zmm3, %xmm3
-; AVX512F-NEXT:    vinserti128 $1, %xmm3, %ymm2, %ymm2
-; AVX512F-NEXT:    vpblendvb %ymm2, %ymm1, %ymm5, %ymm1
+; AVX512F-NEXT:    vinserti128 $1, %xmm3, %ymm1, %ymm1
+; AVX512F-NEXT:    vpblendvb %ymm1, %ymm4, %ymm2, %ymm1
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BWVL-LABEL: avg_v64i8_mask:
@@ -170,29 +173,31 @@ define <64 x i8> @avg_v64i8_maskz(<64 x i8> %a, <64 x i8> %b, i64 %mask) nounwin
 ; AVX512F-LABEL: avg_v64i8_maskz:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    movq %rdi, %rax
-; AVX512F-NEXT:    movq %rdi, %rcx
+; AVX512F-NEXT:    movl %edi, %ecx
 ; AVX512F-NEXT:    kmovw %edi, %k1
-; AVX512F-NEXT:    movl %edi, %edx
-; AVX512F-NEXT:    shrl $16, %edx
-; AVX512F-NEXT:    shrq $32, %rax
-; AVX512F-NEXT:    shrq $48, %rcx
-; AVX512F-NEXT:    vpavgb %ymm3, %ymm1, %ymm1
-; AVX512F-NEXT:    vpavgb %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    shrq $32, %rdi
+; AVX512F-NEXT:    shrq $48, %rax
+; AVX512F-NEXT:    shrl $16, %ecx
+; AVX512F-NEXT:    vpavgb %ymm1, %ymm0, %ymm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm1
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
+; AVX512F-NEXT:    vpavgb %ymm1, %ymm0, %ymm0
 ; AVX512F-NEXT:    kmovw %ecx, %k2
 ; AVX512F-NEXT:    kmovw %eax, %k3
-; AVX512F-NEXT:    kmovw %edx, %k4
-; AVX512F-NEXT:    vpternlogd $255, %zmm2, %zmm2, %zmm2 {%k1} {z}
-; AVX512F-NEXT:    vpmovdb %zmm2, %xmm2
-; AVX512F-NEXT:    vpternlogd $255, %zmm3, %zmm3, %zmm3 {%k4} {z}
+; AVX512F-NEXT:    kmovw %edi, %k4
+; AVX512F-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k4} {z}
+; AVX512F-NEXT:    vpmovdb %zmm1, %xmm1
+; AVX512F-NEXT:    vpternlogd $255, %zmm3, %zmm3, %zmm3 {%k3} {z}
 ; AVX512F-NEXT:    vpmovdb %zmm3, %xmm3
-; AVX512F-NEXT:    vinserti128 $1, %xmm3, %ymm2, %ymm2
-; AVX512F-NEXT:    vpand %ymm0, %ymm2, %ymm0
-; AVX512F-NEXT:    vpternlogd $255, %zmm2, %zmm2, %zmm2 {%k3} {z}
-; AVX512F-NEXT:    vpmovdb %zmm2, %xmm2
+; AVX512F-NEXT:    vinserti128 $1, %xmm3, %ymm1, %ymm1
+; AVX512F-NEXT:    vpand %ymm0, %ymm1, %ymm0
+; AVX512F-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k1} {z}
+; AVX512F-NEXT:    vpmovdb %zmm1, %xmm1
 ; AVX512F-NEXT:    vpternlogd $255, %zmm3, %zmm3, %zmm3 {%k2} {z}
 ; AVX512F-NEXT:    vpmovdb %zmm3, %xmm3
-; AVX512F-NEXT:    vinserti128 $1, %xmm3, %ymm2, %ymm2
-; AVX512F-NEXT:    vpand %ymm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vinserti128 $1, %xmm3, %ymm1, %ymm1
+; AVX512F-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BWVL-LABEL: avg_v64i8_maskz:
@@ -322,17 +327,21 @@ define <16 x i16> @avg_v16i16_maskz(<16 x i16> %a, <16 x i16> %b, i16 %mask) nou
 define <32 x i16> @avg_v32i16_mask(<32 x i16> %a, <32 x i16> %b, <32 x i16> %src, i32 %mask) nounwind {
 ; AVX512F-LABEL: avg_v32i16_mask:
 ; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm2, %ymm3
 ; AVX512F-NEXT:    kmovw %edi, %k1
 ; AVX512F-NEXT:    shrl $16, %edi
-; AVX512F-NEXT:    vpavgw %ymm3, %ymm1, %ymm1
-; AVX512F-NEXT:    vpavgw %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vpavgw %ymm1, %ymm0, %ymm4
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm1
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
+; AVX512F-NEXT:    vpavgw %ymm1, %ymm0, %ymm0
 ; AVX512F-NEXT:    kmovw %edi, %k2
-; AVX512F-NEXT:    vpternlogd $255, %zmm2, %zmm2, %zmm2 {%k1} {z}
-; AVX512F-NEXT:    vpmovdw %zmm2, %ymm2
-; AVX512F-NEXT:    vpblendvb %ymm2, %ymm0, %ymm4, %ymm0
-; AVX512F-NEXT:    vpternlogd $255, %zmm2, %zmm2, %zmm2 {%k2} {z}
-; AVX512F-NEXT:    vpmovdw %zmm2, %ymm2
-; AVX512F-NEXT:    vpblendvb %ymm2, %ymm1, %ymm5, %ymm1
+; AVX512F-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k2} {z}
+; AVX512F-NEXT:    vpmovdw %zmm1, %ymm1
+; AVX512F-NEXT:    vpblendvb %ymm1, %ymm0, %ymm3, %ymm0
+; AVX512F-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k1} {z}
+; AVX512F-NEXT:    vpmovdw %zmm1, %ymm1
+; AVX512F-NEXT:    vpblendvb %ymm1, %ymm4, %ymm2, %ymm1
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BWVL-LABEL: avg_v32i16_mask:
@@ -357,15 +366,18 @@ define <32 x i16> @avg_v32i16_maskz(<32 x i16> %a, <32 x i16> %b, i32 %mask) nou
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    kmovw %edi, %k1
 ; AVX512F-NEXT:    shrl $16, %edi
-; AVX512F-NEXT:    vpavgw %ymm3, %ymm1, %ymm1
-; AVX512F-NEXT:    vpavgw %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vpavgw %ymm1, %ymm0, %ymm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm1
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
+; AVX512F-NEXT:    vpavgw %ymm1, %ymm0, %ymm0
 ; AVX512F-NEXT:    kmovw %edi, %k2
-; AVX512F-NEXT:    vpternlogd $255, %zmm2, %zmm2, %zmm2 {%k1} {z}
-; AVX512F-NEXT:    vpmovdw %zmm2, %ymm2
-; AVX512F-NEXT:    vpand %ymm0, %ymm2, %ymm0
-; AVX512F-NEXT:    vpternlogd $255, %zmm2, %zmm2, %zmm2 {%k2} {z}
-; AVX512F-NEXT:    vpmovdw %zmm2, %ymm2
-; AVX512F-NEXT:    vpand %ymm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k2} {z}
+; AVX512F-NEXT:    vpmovdw %zmm1, %ymm1
+; AVX512F-NEXT:    vpand %ymm0, %ymm1, %ymm0
+; AVX512F-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k1} {z}
+; AVX512F-NEXT:    vpmovdw %zmm1, %ymm1
+; AVX512F-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BWVL-LABEL: avg_v32i16_maskz:
diff --git a/llvm/test/CodeGen/X86/avg.ll b/llvm/test/CodeGen/X86/avg.ll
index 86d8628c9..e4a5d1392 100644
--- a/llvm/test/CodeGen/X86/avg.ll
+++ b/llvm/test/CodeGen/X86/avg.ll
@@ -226,7 +226,7 @@ define void @avg_v48i8(<48 x i8>* %a, <48 x i8>* %b) nounwind {
 ; SSE2-NEXT:    psubd %xmm5, %xmm0
 ; SSE2-NEXT:    psrld $1, %xmm3
 ; SSE2-NEXT:    psrld $1, %xmm8
-; SSE2-NEXT:    movdqa {{.*#+}} xmm7 = [255,255,255,255]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm7 = [255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0]
 ; SSE2-NEXT:    pand %xmm7, %xmm8
 ; SSE2-NEXT:    pand %xmm7, %xmm3
 ; SSE2-NEXT:    packuswb %xmm8, %xmm3
@@ -378,63 +378,65 @@ define void @avg_v48i8(<48 x i8>* %a, <48 x i8>* %b) nounwind {
 ; AVX2-LABEL: avg_v48i8:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX2-NEXT:    vmovdqa 32(%rdi), %xmm1
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero
-; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
-; AVX2-NEXT:    vpbroadcastq 24(%rdi), %xmm3
+; AVX2-NEXT:    vmovdqa 16(%rdi), %xmm1
+; AVX2-NEXT:    vmovdqa 32(%rdi), %xmm2
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[2,3,0,1]
 ; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero,xmm3[4],zero,zero,zero,xmm3[5],zero,zero,zero,xmm3[6],zero,zero,zero,xmm3[7],zero,zero,zero
-; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm4 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm5 = xmm1[2,3,0,1]
-; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm5 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero,xmm5[2],zero,zero,zero,xmm5[3],zero,zero,zero,xmm5[4],zero,zero,zero,xmm5[5],zero,zero,zero,xmm5[6],zero,zero,zero,xmm5[7],zero,zero,zero
-; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm8 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero
-; AVX2-NEXT:    vmovdqa (%rsi), %xmm6
-; AVX2-NEXT:    vmovdqa 32(%rsi), %xmm7
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm6[2,3,0,1]
-; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero
-; AVX2-NEXT:    vpaddd %ymm1, %ymm2, %ymm1
-; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm2 = xmm6[0],zero,zero,zero,xmm6[1],zero,zero,zero,xmm6[2],zero,zero,zero,xmm6[3],zero,zero,zero,xmm6[4],zero,zero,zero,xmm6[5],zero,zero,zero,xmm6[6],zero,zero,zero,xmm6[7],zero,zero,zero
-; AVX2-NEXT:    vpaddd %ymm2, %ymm0, %ymm0
-; AVX2-NEXT:    vpbroadcastq 24(%rsi), %xmm2
-; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero
-; AVX2-NEXT:    vpaddd %ymm2, %ymm3, %ymm2
-; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm3 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
-; AVX2-NEXT:    vpaddd %ymm3, %ymm4, %ymm3
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm4 = xmm7[2,3,0,1]
+; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm4 = xmm1[2,3,0,1]
 ; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero,xmm4[4],zero,zero,zero,xmm4[5],zero,zero,zero,xmm4[6],zero,zero,zero,xmm4[7],zero,zero,zero
-; AVX2-NEXT:    vpaddd %ymm4, %ymm5, %ymm4
+; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm5 = xmm2[2,3,0,1]
+; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm9 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero,xmm5[2],zero,zero,zero,xmm5[3],zero,zero,zero,xmm5[4],zero,zero,zero,xmm5[5],zero,zero,zero,xmm5[6],zero,zero,zero,xmm5[7],zero,zero,zero
+; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm8 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero
+; AVX2-NEXT:    vmovdqa (%rsi), %xmm6
+; AVX2-NEXT:    vmovdqa 16(%rsi), %xmm7
+; AVX2-NEXT:    vmovdqa 32(%rsi), %xmm2
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm5 = xmm6[2,3,0,1]
+; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm5 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero,xmm5[2],zero,zero,zero,xmm5[3],zero,zero,zero,xmm5[4],zero,zero,zero,xmm5[5],zero,zero,zero,xmm5[6],zero,zero,zero,xmm5[7],zero,zero,zero
+; AVX2-NEXT:    vpaddd %ymm5, %ymm3, %ymm3
+; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm5 = xmm6[0],zero,zero,zero,xmm6[1],zero,zero,zero,xmm6[2],zero,zero,zero,xmm6[3],zero,zero,zero,xmm6[4],zero,zero,zero,xmm6[5],zero,zero,zero,xmm6[6],zero,zero,zero,xmm6[7],zero,zero,zero
+; AVX2-NEXT:    vpaddd %ymm5, %ymm0, %ymm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm5 = xmm7[2,3,0,1]
+; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm5 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero,xmm5[2],zero,zero,zero,xmm5[3],zero,zero,zero,xmm5[4],zero,zero,zero,xmm5[5],zero,zero,zero,xmm5[6],zero,zero,zero,xmm5[7],zero,zero,zero
+; AVX2-NEXT:    vpaddd %ymm5, %ymm4, %ymm4
 ; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm5 = xmm7[0],zero,zero,zero,xmm7[1],zero,zero,zero,xmm7[2],zero,zero,zero,xmm7[3],zero,zero,zero,xmm7[4],zero,zero,zero,xmm7[5],zero,zero,zero,xmm7[6],zero,zero,zero,xmm7[7],zero,zero,zero
-; AVX2-NEXT:    vpaddd %ymm5, %ymm8, %ymm5
+; AVX2-NEXT:    vpaddd %ymm5, %ymm1, %ymm1
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm5 = xmm2[2,3,0,1]
+; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm5 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero,xmm5[2],zero,zero,zero,xmm5[3],zero,zero,zero,xmm5[4],zero,zero,zero,xmm5[5],zero,zero,zero,xmm5[6],zero,zero,zero,xmm5[7],zero,zero,zero
+; AVX2-NEXT:    vpaddd %ymm5, %ymm9, %ymm5
+; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero
+; AVX2-NEXT:    vpaddd %ymm2, %ymm8, %ymm2
 ; AVX2-NEXT:    vpcmpeqd %ymm6, %ymm6, %ymm6
-; AVX2-NEXT:    vpsubd %ymm6, %ymm1, %ymm1
-; AVX2-NEXT:    vpsubd %ymm6, %ymm0, %ymm0
-; AVX2-NEXT:    vpsubd %ymm6, %ymm2, %ymm2
 ; AVX2-NEXT:    vpsubd %ymm6, %ymm3, %ymm3
+; AVX2-NEXT:    vpsubd %ymm6, %ymm0, %ymm0
 ; AVX2-NEXT:    vpsubd %ymm6, %ymm4, %ymm4
+; AVX2-NEXT:    vpsubd %ymm6, %ymm1, %ymm1
 ; AVX2-NEXT:    vpsubd %ymm6, %ymm5, %ymm5
+; AVX2-NEXT:    vpsubd %ymm6, %ymm2, %ymm2
+; AVX2-NEXT:    vpsrld $1, %ymm2, %ymm2
 ; AVX2-NEXT:    vpsrld $1, %ymm5, %ymm5
+; AVX2-NEXT:    vpsrld $1, %ymm1, %ymm1
 ; AVX2-NEXT:    vpsrld $1, %ymm4, %ymm4
-; AVX2-NEXT:    vpsrld $1, %ymm3, %ymm3
-; AVX2-NEXT:    vpsrld $1, %ymm2, %ymm2
 ; AVX2-NEXT:    vpsrld $1, %ymm0, %ymm0
-; AVX2-NEXT:    vpsrld $1, %ymm1, %ymm1
-; AVX2-NEXT:    vperm2i128 {{.*#+}} ymm6 = ymm0[2,3],ymm1[2,3]
-; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpsrld $1, %ymm3, %ymm3
+; AVX2-NEXT:    vperm2i128 {{.*#+}} ymm6 = ymm0[2,3],ymm3[2,3]
+; AVX2-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm0
 ; AVX2-NEXT:    vpackusdw %ymm6, %ymm0, %ymm0
-; AVX2-NEXT:    vmovdqa {{.*#+}} ymm1 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vperm2i128 {{.*#+}} ymm6 = ymm3[2,3],ymm2[2,3]
-; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm3, %ymm2
-; AVX2-NEXT:    vpackusdw %ymm6, %ymm2, %ymm2
-; AVX2-NEXT:    vpand %ymm1, %ymm2, %ymm2
-; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm3
+; AVX2-NEXT:    vmovdqa {{.*#+}} ymm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; AVX2-NEXT:    vpand %ymm3, %ymm0, %ymm0
+; AVX2-NEXT:    vperm2i128 {{.*#+}} ymm6 = ymm1[2,3],ymm4[2,3]
+; AVX2-NEXT:    vinserti128 $1, %xmm4, %ymm1, %ymm1
+; AVX2-NEXT:    vpackusdw %ymm6, %ymm1, %ymm1
+; AVX2-NEXT:    vpand %ymm3, %ymm1, %ymm1
+; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm4
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
-; AVX2-NEXT:    vpackuswb %ymm0, %ymm3, %ymm0
-; AVX2-NEXT:    vperm2i128 {{.*#+}} ymm2 = ymm5[2,3],ymm4[2,3]
-; AVX2-NEXT:    vinserti128 $1, %xmm4, %ymm5, %ymm3
-; AVX2-NEXT:    vpackusdw %ymm2, %ymm3, %ymm2
-; AVX2-NEXT:    vpand %ymm1, %ymm2, %ymm1
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-NEXT:    vpackuswb %ymm0, %ymm4, %ymm0
+; AVX2-NEXT:    vperm2i128 {{.*#+}} ymm1 = ymm2[2,3],ymm5[2,3]
+; AVX2-NEXT:    vinserti128 $1, %xmm5, %ymm2, %ymm2
+; AVX2-NEXT:    vpackusdw %ymm1, %ymm2, %ymm1
+; AVX2-NEXT:    vpand %ymm3, %ymm1, %ymm1
 ; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
 ; AVX2-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
 ; AVX2-NEXT:    vmovdqu %xmm1, (%rax)
@@ -460,14 +462,12 @@ define void @avg_v48i8(<48 x i8>* %a, <48 x i8>* %b) nounwind {
 ; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm0
 ; AVX512BW-NEXT:    vmovdqa 16(%rdi), %xmm1
 ; AVX512BW-NEXT:    vmovdqa 32(%rdi), %xmm2
-; AVX512BW-NEXT:    vpavgb 16(%rsi), %xmm1, %xmm1
+; AVX512BW-NEXT:    vpavgb 32(%rsi), %xmm2, %xmm2
 ; AVX512BW-NEXT:    vpavgb (%rsi), %xmm0, %xmm0
-; AVX512BW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX512BW-NEXT:    vpavgb 32(%rsi), %xmm2, %xmm1
-; AVX512BW-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm1
-; AVX512BW-NEXT:    vmovdqu %ymm0, (%rax)
-; AVX512BW-NEXT:    vextracti32x4 $2, %zmm1, (%rax)
-; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    vpavgb 16(%rsi), %xmm1, %xmm1
+; AVX512BW-NEXT:    vmovdqu %xmm1, (%rax)
+; AVX512BW-NEXT:    vmovdqu %xmm0, (%rax)
+; AVX512BW-NEXT:    vmovdqu %xmm2, (%rax)
 ; AVX512BW-NEXT:    retq
   %1 = load <48 x i8>, <48 x i8>* %a
   %2 = load <48 x i8>, <48 x i8>* %b
@@ -1535,8 +1535,11 @@ define <64 x i8> @avg_v64i8_3(<64 x i8> %a, <64 x i8> %b) nounwind {
 ;
 ; AVX512F-LABEL: avg_v64i8_3:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpavgb %ymm2, %ymm0, %ymm0
-; AVX512F-NEXT:    vpavgb %ymm3, %ymm1, %ymm1
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
+; AVX512F-NEXT:    vpavgb %ymm2, %ymm3, %ymm2
+; AVX512F-NEXT:    vpavgb %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: avg_v64i8_3:
@@ -1897,118 +1900,178 @@ define void @not_avg_v16i8_wide_constants(<16 x i8>* %a, <16 x i8>* %b) nounwind
 ; SSE2-NEXT:    pushq %r13
 ; SSE2-NEXT:    pushq %r12
 ; SSE2-NEXT:    pushq %rbx
-; SSE2-NEXT:    movaps (%rdi), %xmm0
-; SSE2-NEXT:    movaps (%rsi), %xmm1
-; SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    movaps (%rdi), %xmm1
+; SSE2-NEXT:    movaps (%rsi), %xmm0
+; SSE2-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
 ; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
 ; SSE2-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %r13d
 ; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
 ; SSE2-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
 ; SSE2-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %r14d
-; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %r15d
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %r13d
 ; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %r12d
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %r15d
 ; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %r11d
 ; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %r10d
 ; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %r9d
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %r8d
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %edx
 ; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %ecx
 ; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %edi
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %esi
+; SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
 ; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %ebp
+; SSE2-NEXT:    addq %r11, %rbp
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %r14d
+; SSE2-NEXT:    addq %r10, %r14
 ; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %ebx
+; SSE2-NEXT:    addq %r9, %rbx
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %r11d
+; SSE2-NEXT:    addq %r8, %r11
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %r10d
+; SSE2-NEXT:    addq %rdx, %r10
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %r8d
+; SSE2-NEXT:    addq %rcx, %r8
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %edi
+; SSE2-NEXT:    addq %rax, %rdi
 ; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %edx
-; SSE2-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    addq %rsi, %rdx
 ; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %esi
-; SSE2-NEXT:    leal -1(%rdx,%rsi), %edx
-; SSE2-NEXT:    movl %edx, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %edx
-; SSE2-NEXT:    leal -1(%rbx,%rdx), %edx
-; SSE2-NEXT:    movl %edx, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %edx
-; SSE2-NEXT:    leal -1(%rbp,%rdx), %edx
-; SSE2-NEXT:    movl %edx, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %edx
-; SSE2-NEXT:    leal -1(%rdi,%rdx), %r8d
-; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %edx
-; SSE2-NEXT:    leal -1(%rax,%rdx), %edi
-; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    leal -1(%rcx,%rax), %edx
-; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    leal -1(%r9,%rax), %ecx
+; SSE2-NEXT:    leaq -1(%r15,%rsi), %rax
+; SSE2-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %esi
-; SSE2-NEXT:    leal -1(%r10,%rsi), %eax
+; SSE2-NEXT:    leaq -1(%r12,%rsi), %rax
+; SSE2-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %esi
-; SSE2-NEXT:    leaq -1(%r11,%rsi), %rsi
-; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %ebx
-; SSE2-NEXT:    leaq -1(%r12,%rbx), %r12
-; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %ebx
-; SSE2-NEXT:    leaq -1(%r15,%rbx), %r15
-; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %ebx
-; SSE2-NEXT:    leaq -1(%r14,%rbx), %r14
-; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %ebx
-; SSE2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rbp # 8-byte Reload
-; SSE2-NEXT:    leaq -1(%rbp,%rbx), %r11
-; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %ebx
-; SSE2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rbp # 8-byte Reload
-; SSE2-NEXT:    leaq -1(%rbp,%rbx), %r10
-; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %ebx
-; SSE2-NEXT:    leaq -1(%r13,%rbx), %r9
-; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %ebx
-; SSE2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r13 # 8-byte Reload
-; SSE2-NEXT:    leaq -1(%r13,%rbx), %rbx
-; SSE2-NEXT:    shrl %eax
-; SSE2-NEXT:    movd %eax, %xmm8
-; SSE2-NEXT:    shrl %ecx
-; SSE2-NEXT:    movd %ecx, %xmm15
-; SSE2-NEXT:    shrl %edx
-; SSE2-NEXT:    movd %edx, %xmm9
-; SSE2-NEXT:    shrl %edi
-; SSE2-NEXT:    movd %edi, %xmm2
-; SSE2-NEXT:    shrl %r8d
-; SSE2-NEXT:    movd %r8d, %xmm10
-; SSE2-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
-; SSE2-NEXT:    shrl %eax
-; SSE2-NEXT:    movd %eax, %xmm6
-; SSE2-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
-; SSE2-NEXT:    shrl %eax
-; SSE2-NEXT:    movd %eax, %xmm11
-; SSE2-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
-; SSE2-NEXT:    shrl %eax
-; SSE2-NEXT:    movd %eax, %xmm4
-; SSE2-NEXT:    shrq %rsi
-; SSE2-NEXT:    movd %esi, %xmm12
-; SSE2-NEXT:    shrq %r12
-; SSE2-NEXT:    movd %r12d, %xmm3
-; SSE2-NEXT:    shrq %r15
-; SSE2-NEXT:    movd %r15d, %xmm13
-; SSE2-NEXT:    shrq %r14
-; SSE2-NEXT:    movd %r14d, %xmm7
-; SSE2-NEXT:    shrq %r11
-; SSE2-NEXT:    movd %r11d, %xmm14
-; SSE2-NEXT:    shrq %r10
-; SSE2-NEXT:    movd %r10d, %xmm5
-; SSE2-NEXT:    shrq %r9
-; SSE2-NEXT:    movd %r9d, %xmm0
-; SSE2-NEXT:    shrq %rbx
-; SSE2-NEXT:    movd %ebx, %xmm1
+; SSE2-NEXT:    leaq -1(%r13,%rsi), %rax
+; SSE2-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %esi
+; SSE2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; SSE2-NEXT:    leaq -1(%rax,%rsi), %rax
+; SSE2-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %esi
+; SSE2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; SSE2-NEXT:    leaq -1(%rax,%rsi), %rax
+; SSE2-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %esi
+; SSE2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; SSE2-NEXT:    leaq -1(%rax,%rsi), %rax
+; SSE2-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %esi
+; SSE2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; SSE2-NEXT:    leaq -1(%rax,%rsi), %rsi
+; SSE2-NEXT:    movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %esi
+; SSE2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; SSE2-NEXT:    leaq -1(%rax,%rsi), %rsi
+; SSE2-NEXT:    movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE2-NEXT:    addq $-1, %rbp
+; SSE2-NEXT:    movl $0, %r9d
+; SSE2-NEXT:    adcq $-1, %r9
+; SSE2-NEXT:    addq $-1, %r14
+; SSE2-NEXT:    movl $0, %esi
+; SSE2-NEXT:    adcq $-1, %rsi
+; SSE2-NEXT:    addq $-1, %rbx
+; SSE2-NEXT:    movl $0, %eax
+; SSE2-NEXT:    adcq $-1, %rax
+; SSE2-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE2-NEXT:    addq $-1, %r11
+; SSE2-NEXT:    movl $0, %r12d
+; SSE2-NEXT:    adcq $-1, %r12
+; SSE2-NEXT:    addq $-1, %r10
+; SSE2-NEXT:    movl $0, %r13d
+; SSE2-NEXT:    adcq $-1, %r13
+; SSE2-NEXT:    addq $-1, %r8
+; SSE2-NEXT:    movl $0, %r15d
+; SSE2-NEXT:    adcq $-1, %r15
+; SSE2-NEXT:    addq $-1, %rdi
+; SSE2-NEXT:    movl $0, %ecx
+; SSE2-NEXT:    adcq $-1, %rcx
+; SSE2-NEXT:    addq $-1, %rdx
+; SSE2-NEXT:    movl $0, %eax
+; SSE2-NEXT:    adcq $-1, %rax
+; SSE2-NEXT:    shldq $63, %rdx, %rax
+; SSE2-NEXT:    shldq $63, %rdi, %rcx
+; SSE2-NEXT:    movq %rcx, %rdx
+; SSE2-NEXT:    shldq $63, %r8, %r15
+; SSE2-NEXT:    shldq $63, %r10, %r13
+; SSE2-NEXT:    shldq $63, %r11, %r12
+; SSE2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
+; SSE2-NEXT:    shldq $63, %rbx, %rdi
+; SSE2-NEXT:    shldq $63, %r14, %rsi
+; SSE2-NEXT:    shldq $63, %rbp, %r9
+; SSE2-NEXT:    movq %r9, %xmm8
+; SSE2-NEXT:    movq %rsi, %xmm15
+; SSE2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
+; SSE2-NEXT:    shrq %rcx
+; SSE2-NEXT:    movq %rcx, %xmm9
+; SSE2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
+; SSE2-NEXT:    shrq %rcx
+; SSE2-NEXT:    movq %rcx, %xmm2
+; SSE2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
+; SSE2-NEXT:    shrq %rcx
+; SSE2-NEXT:    movq %rcx, %xmm10
+; SSE2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
+; SSE2-NEXT:    shrq %rcx
+; SSE2-NEXT:    movq %rcx, %xmm4
+; SSE2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
+; SSE2-NEXT:    shrq %rcx
+; SSE2-NEXT:    movq %rcx, %xmm11
+; SSE2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
+; SSE2-NEXT:    shrq %rcx
+; SSE2-NEXT:    movq %rcx, %xmm7
+; SSE2-NEXT:    movq %rdi, %xmm12
+; SSE2-NEXT:    movq %r12, %xmm0
+; SSE2-NEXT:    movq %r13, %xmm13
+; SSE2-NEXT:    movq %r15, %xmm6
+; SSE2-NEXT:    movq %rdx, %xmm14
+; SSE2-NEXT:    movq %rax, %xmm5
+; SSE2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; SSE2-NEXT:    shrq %rax
+; SSE2-NEXT:    movq %rax, %xmm3
+; SSE2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; SSE2-NEXT:    shrq %rax
+; SSE2-NEXT:    movq %rax, %xmm1
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm15 = xmm15[0],xmm8[0],xmm15[1],xmm8[1],xmm15[2],xmm8[2],xmm15[3],xmm8[3],xmm15[4],xmm8[4],xmm15[5],xmm8[5],xmm15[6],xmm8[6],xmm15[7],xmm8[7]
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm9[0],xmm2[1],xmm9[1],xmm2[2],xmm9[2],xmm2[3],xmm9[3],xmm2[4],xmm9[4],xmm2[5],xmm9[5],xmm2[6],xmm9[6],xmm2[7],xmm9[7]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm15[0],xmm2[1],xmm15[1],xmm2[2],xmm15[2],xmm2[3],xmm15[3]
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm10[0],xmm6[1],xmm10[1],xmm6[2],xmm10[2],xmm6[3],xmm10[3],xmm6[4],xmm10[4],xmm6[5],xmm10[5],xmm6[6],xmm10[6],xmm6[7],xmm10[7]
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm11[0],xmm4[1],xmm11[1],xmm4[2],xmm11[2],xmm4[3],xmm11[3],xmm4[4],xmm11[4],xmm4[5],xmm11[5],xmm4[6],xmm11[6],xmm4[7],xmm11[7]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm6[0],xmm4[1],xmm6[1],xmm4[2],xmm6[2],xmm4[3],xmm6[3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm12[0],xmm3[1],xmm12[1],xmm3[2],xmm12[2],xmm3[3],xmm12[3],xmm3[4],xmm12[4],xmm3[5],xmm12[5],xmm3[6],xmm12[6],xmm3[7],xmm12[7]
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm13[0],xmm7[1],xmm13[1],xmm7[2],xmm13[2],xmm7[3],xmm13[3],xmm7[4],xmm13[4],xmm7[5],xmm13[5],xmm7[6],xmm13[6],xmm7[7],xmm13[7]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm7 = xmm7[0],xmm3[0],xmm7[1],xmm3[1],xmm7[2],xmm3[2],xmm7[3],xmm3[3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm15[0,1,2,0]
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm8
+; SSE2-NEXT:    pslldq {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm2[0,1]
+; SSE2-NEXT:    por %xmm8, %xmm2
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm10[0],xmm4[1],xmm10[1],xmm4[2],xmm10[2],xmm4[3],xmm10[3],xmm4[4],xmm10[4],xmm4[5],xmm10[5],xmm4[6],xmm10[6],xmm4[7],xmm10[7]
+; SSE2-NEXT:    pslldq {{.*#+}} xmm4 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm4[0,1,2,3,4,5]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm11[0],xmm7[1],xmm11[1],xmm7[2],xmm11[2],xmm7[3],xmm11[3],xmm7[4],xmm11[4],xmm7[5],xmm11[5],xmm7[6],xmm11[6],xmm7[7],xmm11[7]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [65535,65535,65535,65535,65535,0,65535,65535]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[0,1,0,1]
+; SSE2-NEXT:    pand %xmm8, %xmm7
+; SSE2-NEXT:    pandn %xmm4, %xmm8
+; SSE2-NEXT:    por %xmm7, %xmm8
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm8[0,1,2,2]
+; SSE2-NEXT:    punpckhdq {{.*#+}} xmm4 = xmm4[2],xmm2[2],xmm4[3],xmm2[3]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm12[0],xmm0[1],xmm12[1],xmm0[2],xmm12[2],xmm0[3],xmm12[3],xmm0[4],xmm12[4],xmm0[5],xmm12[5],xmm0[6],xmm12[6],xmm0[7],xmm12[7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm13[0],xmm6[1],xmm13[1],xmm6[2],xmm13[2],xmm6[3],xmm13[3],xmm6[4],xmm13[4],xmm6[5],xmm13[5],xmm6[6],xmm13[6],xmm6[7],xmm13[7]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [65535,0,65535,65535,65535,65535,65535,65535]
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    pslld $16, %xmm6
+; SSE2-NEXT:    pandn %xmm6, %xmm2
+; SSE2-NEXT:    por %xmm0, %xmm2
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm14[0],xmm5[1],xmm14[1],xmm5[2],xmm14[2],xmm5[3],xmm14[3],xmm5[4],xmm14[4],xmm5[5],xmm14[5],xmm5[6],xmm14[6],xmm5[7],xmm14[7]
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1],xmm1[2],xmm5[2],xmm1[3],xmm5[3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm7[0],xmm1[1],xmm7[1]
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm1[0]
-; SSE2-NEXT:    movdqu %xmm4, (%rax)
+; SSE2-NEXT:    psllq $48, %xmm5
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [65535,65535,65535,0,65535,65535,65535,65535]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3],xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,1,1]
+; SSE2-NEXT:    pand %xmm0, %xmm1
+; SSE2-NEXT:    pandn %xmm5, %xmm0
+; SSE2-NEXT:    por %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
+; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,1],xmm4[2,3]
+; SSE2-NEXT:    movups %xmm2, (%rax)
 ; SSE2-NEXT:    popq %rbx
 ; SSE2-NEXT:    popq %r12
 ; SSE2-NEXT:    popq %r13
@@ -2025,118 +2088,181 @@ define void @not_avg_v16i8_wide_constants(<16 x i8>* %a, <16 x i8>* %b) nounwind
 ; AVX1-NEXT:    pushq %r13
 ; AVX1-NEXT:    pushq %r12
 ; AVX1-NEXT:    pushq %rbx
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm3 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
+; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm4 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
 ; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm5 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm6 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
-; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
-; AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm7 = xmm4[2],xmm2[2],xmm4[3],xmm2[3]
-; AVX1-NEXT:    vpextrq $1, %xmm7, %r15
-; AVX1-NEXT:    vmovq %xmm7, %r14
-; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero
-; AVX1-NEXT:    vpextrq $1, %xmm4, %r11
-; AVX1-NEXT:    vmovq %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
+; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
+; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
+; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm5 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm5 = xmm5[2],xmm3[2],xmm5[3],xmm3[3]
+; AVX1-NEXT:    vmovq %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
+; AVX1-NEXT:    vpextrq $1, %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7]
+; AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm6 = xmm5[2],xmm3[2],xmm5[3],xmm3[3]
+; AVX1-NEXT:    vmovq %xmm6, %r10
+; AVX1-NEXT:    vpextrq $1, %xmm6, %r9
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm6 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
+; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm7 = xmm6[0],zero,xmm6[1],zero
+; AVX1-NEXT:    vmovq %xmm7, %r8
+; AVX1-NEXT:    vpextrq $1, %xmm7, %rdi
+; AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm6 = xmm6[2],xmm3[2],xmm6[3],xmm3[3]
+; AVX1-NEXT:    vpextrq $1, %xmm6, %rcx
+; AVX1-NEXT:    vmovq %xmm6, %r14
+; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm6 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero
+; AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm6 = xmm6[2],xmm3[2],xmm6[3],xmm3[3]
+; AVX1-NEXT:    vpextrq $1, %xmm6, %rax
+; AVX1-NEXT:    vmovq %xmm6, %rbp
+; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm5 = xmm5[0],zero,xmm5[1],zero
+; AVX1-NEXT:    vpextrq $1, %xmm5, %r11
+; AVX1-NEXT:    vmovq %xmm5, %r15
+; AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm8 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
+; AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
+; AVX1-NEXT:    vpextrq $1, %xmm4, %rbx
+; AVX1-NEXT:    vmovq %xmm4, %rdx
 ; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm4 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
-; AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm4 = xmm4[2],xmm2[2],xmm4[3],xmm2[3]
-; AVX1-NEXT:    vpextrq $1, %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
-; AVX1-NEXT:    vmovq %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
-; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm4 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero
-; AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm7 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero
-; AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm8 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
-; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm5[4],xmm2[4],xmm5[5],xmm2[5],xmm5[6],xmm2[6],xmm5[7],xmm2[7]
-; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
-; AVX1-NEXT:    vmovd %xmm6, %ecx
-; AVX1-NEXT:    vpextrd $1, %xmm6, %edx
-; AVX1-NEXT:    vpextrd $2, %xmm6, %r13d
-; AVX1-NEXT:    vpextrd $3, %xmm6, %r12d
-; AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm6 = xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; AVX1-NEXT:    vmovd %xmm1, %ebx
-; AVX1-NEXT:    vpextrd $1, %xmm1, %ebp
-; AVX1-NEXT:    vpextrd $2, %xmm1, %esi
-; AVX1-NEXT:    vpextrd $3, %xmm1, %edi
-; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm1 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero
-; AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm5 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero
-; AVX1-NEXT:    vmovd %xmm7, %r8d
-; AVX1-NEXT:    leal -1(%r12,%rdi), %eax
-; AVX1-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; AVX1-NEXT:    vpextrd $2, %xmm7, %eax
-; AVX1-NEXT:    leal -1(%r13,%rsi), %esi
-; AVX1-NEXT:    movl %esi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; AVX1-NEXT:    vpextrd $2, %xmm4, %edi
-; AVX1-NEXT:    leal -1(%rdx,%rbp), %edx
-; AVX1-NEXT:    movl %edx, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; AVX1-NEXT:    vpextrd $3, %xmm4, %edx
-; AVX1-NEXT:    leal -1(%rcx,%rbx), %r10d
-; AVX1-NEXT:    vpextrd $3, %xmm1, %ecx
-; AVX1-NEXT:    leal -1(%rdx,%rcx), %r9d
-; AVX1-NEXT:    vpextrd $2, %xmm1, %ecx
-; AVX1-NEXT:    leal -1(%rdi,%rcx), %edi
-; AVX1-NEXT:    vpextrd $2, %xmm5, %ecx
-; AVX1-NEXT:    leal -1(%rax,%rcx), %eax
-; AVX1-NEXT:    vmovd %xmm5, %ecx
-; AVX1-NEXT:    leal -1(%r8,%rcx), %r8d
-; AVX1-NEXT:    vpextrq $1, %xmm6, %rdx
-; AVX1-NEXT:    leal -1(%r15,%rdx), %r15d
-; AVX1-NEXT:    vmovq %xmm6, %rdx
-; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm3[0],zero,xmm3[1],zero
-; AVX1-NEXT:    leal -1(%r14,%rdx), %r14d
-; AVX1-NEXT:    vpextrq $1, %xmm1, %rdx
-; AVX1-NEXT:    leal -1(%r11,%rdx), %edx
-; AVX1-NEXT:    vmovq %xmm1, %rcx
-; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
-; AVX1-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
-; AVX1-NEXT:    leal -1(%rsi,%rcx), %ecx
-; AVX1-NEXT:    vpextrq $1, %xmm1, %rsi
-; AVX1-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rbp # 8-byte Reload
-; AVX1-NEXT:    leal -1(%rbp,%rsi), %esi
-; AVX1-NEXT:    vmovq %xmm1, %rbx
-; AVX1-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rbp # 8-byte Reload
-; AVX1-NEXT:    leal -1(%rbp,%rbx), %ebx
-; AVX1-NEXT:    vpextrq $1, %xmm8, %r11
-; AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; AVX1-NEXT:    vpextrq $1, %xmm0, %r12
-; AVX1-NEXT:    leal -1(%r11,%r12), %r11d
-; AVX1-NEXT:    vmovq %xmm8, %r12
+; AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm4 = xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm7 = xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7]
+; AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm5 = xmm7[2],xmm3[2],xmm7[3],xmm3[3]
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]
+; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm6 = xmm0[0],zero,xmm0[1],zero
+; AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm0 = xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; AVX1-NEXT:    vpextrq $1, %xmm0, %rsi
+; AVX1-NEXT:    addq %rcx, %rsi
 ; AVX1-NEXT:    vmovq %xmm0, %r13
-; AVX1-NEXT:    leal -1(%r12,%r13), %ebp
-; AVX1-NEXT:    shrl %ebp
-; AVX1-NEXT:    vmovd %ebp, %xmm0
-; AVX1-NEXT:    shrl %r11d
-; AVX1-NEXT:    vpinsrb $1, %r11d, %xmm0, %xmm0
-; AVX1-NEXT:    shrl %ebx
-; AVX1-NEXT:    vpinsrb $2, %ebx, %xmm0, %xmm0
-; AVX1-NEXT:    shrl %esi
-; AVX1-NEXT:    vpinsrb $3, %esi, %xmm0, %xmm0
-; AVX1-NEXT:    shrl %ecx
-; AVX1-NEXT:    vpinsrb $4, %ecx, %xmm0, %xmm0
-; AVX1-NEXT:    shrl %edx
-; AVX1-NEXT:    vpinsrb $5, %edx, %xmm0, %xmm0
-; AVX1-NEXT:    shrl %r14d
-; AVX1-NEXT:    vpinsrb $6, %r14d, %xmm0, %xmm0
-; AVX1-NEXT:    shrl %r15d
-; AVX1-NEXT:    vpinsrb $7, %r15d, %xmm0, %xmm0
-; AVX1-NEXT:    shrl %r8d
-; AVX1-NEXT:    vpinsrb $8, %r8d, %xmm0, %xmm0
-; AVX1-NEXT:    shrl %eax
-; AVX1-NEXT:    vpinsrb $9, %eax, %xmm0, %xmm0
-; AVX1-NEXT:    shrl %edi
-; AVX1-NEXT:    vpinsrb $10, %edi, %xmm0, %xmm0
-; AVX1-NEXT:    shrl %r9d
-; AVX1-NEXT:    vpinsrb $11, %r9d, %xmm0, %xmm0
-; AVX1-NEXT:    shrl %r10d
-; AVX1-NEXT:    vpinsrb $12, %r10d, %xmm0, %xmm0
-; AVX1-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
-; AVX1-NEXT:    shrl %eax
-; AVX1-NEXT:    vpinsrb $13, %eax, %xmm0, %xmm0
-; AVX1-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
-; AVX1-NEXT:    shrl %eax
-; AVX1-NEXT:    vpinsrb $14, %eax, %xmm0, %xmm0
-; AVX1-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
-; AVX1-NEXT:    shrl %eax
-; AVX1-NEXT:    vpinsrb $15, %eax, %xmm0, %xmm0
+; AVX1-NEXT:    addq %r14, %r13
+; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
+; AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm0 = xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; AVX1-NEXT:    vpextrq $1, %xmm0, %r12
+; AVX1-NEXT:    addq %rax, %r12
+; AVX1-NEXT:    vmovq %xmm0, %r14
+; AVX1-NEXT:    addq %rbp, %r14
+; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm7[0],zero,xmm7[1],zero
+; AVX1-NEXT:    vpextrq $1, %xmm0, %rbp
+; AVX1-NEXT:    addq %r11, %rbp
+; AVX1-NEXT:    vmovq %xmm0, %r11
+; AVX1-NEXT:    addq %r15, %r11
+; AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
+; AVX1-NEXT:    vpextrq $1, %xmm0, %r15
+; AVX1-NEXT:    addq %rbx, %r15
+; AVX1-NEXT:    vmovq %xmm0, %rbx
+; AVX1-NEXT:    addq %rdx, %rbx
+; AVX1-NEXT:    vpextrq $1, %xmm6, %rax
+; AVX1-NEXT:    leaq -1(%rdi,%rax), %rax
+; AVX1-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX1-NEXT:    vmovq %xmm6, %rax
+; AVX1-NEXT:    leaq -1(%r8,%rax), %rax
+; AVX1-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX1-NEXT:    vpextrq $1, %xmm5, %rax
+; AVX1-NEXT:    leaq -1(%r9,%rax), %rax
+; AVX1-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX1-NEXT:    vmovq %xmm5, %rax
+; AVX1-NEXT:    leaq -1(%r10,%rax), %rax
+; AVX1-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX1-NEXT:    vpextrq $1, %xmm4, %rax
+; AVX1-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
+; AVX1-NEXT:    leaq -1(%rcx,%rax), %rax
+; AVX1-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX1-NEXT:    vmovq %xmm4, %rax
+; AVX1-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
+; AVX1-NEXT:    leaq -1(%rcx,%rax), %rax
+; AVX1-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX1-NEXT:    vpextrq $1, %xmm8, %rax
+; AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX1-NEXT:    vpextrq $1, %xmm0, %rcx
+; AVX1-NEXT:    leaq -1(%rax,%rcx), %rax
+; AVX1-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX1-NEXT:    vmovq %xmm8, %rax
+; AVX1-NEXT:    vmovq %xmm0, %rcx
+; AVX1-NEXT:    leaq -1(%rax,%rcx), %rax
+; AVX1-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX1-NEXT:    xorl %r10d, %r10d
+; AVX1-NEXT:    addq $-1, %rsi
+; AVX1-NEXT:    movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX1-NEXT:    movl $0, %ecx
+; AVX1-NEXT:    adcq $-1, %rcx
+; AVX1-NEXT:    addq $-1, %r13
+; AVX1-NEXT:    movl $0, %eax
+; AVX1-NEXT:    adcq $-1, %rax
+; AVX1-NEXT:    addq $-1, %r12
+; AVX1-NEXT:    movl $0, %edi
+; AVX1-NEXT:    adcq $-1, %rdi
+; AVX1-NEXT:    addq $-1, %r14
+; AVX1-NEXT:    movl $0, %esi
+; AVX1-NEXT:    adcq $-1, %rsi
+; AVX1-NEXT:    addq $-1, %rbp
+; AVX1-NEXT:    movl $0, %r9d
+; AVX1-NEXT:    adcq $-1, %r9
+; AVX1-NEXT:    addq $-1, %r11
+; AVX1-NEXT:    movl $0, %r8d
+; AVX1-NEXT:    adcq $-1, %r8
+; AVX1-NEXT:    addq $-1, %r15
+; AVX1-NEXT:    movl $0, %edx
+; AVX1-NEXT:    adcq $-1, %rdx
+; AVX1-NEXT:    addq $-1, %rbx
+; AVX1-NEXT:    adcq $-1, %r10
+; AVX1-NEXT:    shldq $63, %r11, %r8
+; AVX1-NEXT:    shldq $63, %rbp, %r9
+; AVX1-NEXT:    shldq $63, %r14, %rsi
+; AVX1-NEXT:    shldq $63, %r12, %rdi
+; AVX1-NEXT:    shldq $63, %r13, %rax
+; AVX1-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rbp # 8-byte Reload
+; AVX1-NEXT:    shldq $63, %rbp, %rcx
+; AVX1-NEXT:    shldq $63, %rbx, %r10
+; AVX1-NEXT:    shldq $63, %r15, %rdx
+; AVX1-NEXT:    vmovq %rcx, %xmm8
+; AVX1-NEXT:    vmovq %rax, %xmm9
+; AVX1-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; AVX1-NEXT:    shrq %rax
+; AVX1-NEXT:    vmovq %rax, %xmm0
+; AVX1-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; AVX1-NEXT:    shrq %rax
+; AVX1-NEXT:    vmovq %rax, %xmm11
+; AVX1-NEXT:    vmovq %rdi, %xmm12
+; AVX1-NEXT:    vmovq %rsi, %xmm13
+; AVX1-NEXT:    vmovq %rdx, %xmm14
+; AVX1-NEXT:    vmovq %r10, %xmm15
+; AVX1-NEXT:    vmovq %r9, %xmm10
+; AVX1-NEXT:    vmovq %r8, %xmm1
+; AVX1-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; AVX1-NEXT:    shrq %rax
+; AVX1-NEXT:    vmovq %rax, %xmm2
+; AVX1-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; AVX1-NEXT:    shrq %rax
+; AVX1-NEXT:    vmovq %rax, %xmm3
+; AVX1-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; AVX1-NEXT:    shrq %rax
+; AVX1-NEXT:    vmovq %rax, %xmm4
+; AVX1-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; AVX1-NEXT:    shrq %rax
+; AVX1-NEXT:    vmovq %rax, %xmm5
+; AVX1-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; AVX1-NEXT:    shrq %rax
+; AVX1-NEXT:    vmovq %rax, %xmm6
+; AVX1-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; AVX1-NEXT:    shrq %rax
+; AVX1-NEXT:    vmovq %rax, %xmm7
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3],xmm9[4],xmm8[4],xmm9[5],xmm8[5],xmm9[6],xmm8[6],xmm9[7],xmm8[7]
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm9 = xmm11[0],xmm0[0],xmm11[1],xmm0[1],xmm11[2],xmm0[2],xmm11[3],xmm0[3],xmm11[4],xmm0[4],xmm11[5],xmm0[5],xmm11[6],xmm0[6],xmm11[7],xmm0[7]
+; AVX1-NEXT:    vpsllq $48, %xmm8, %xmm8
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm9[0,0,1,1]
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm8 = xmm0[0,1,2],xmm8[3],xmm0[4,5,6,7]
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm13[0],xmm12[0],xmm13[1],xmm12[1],xmm13[2],xmm12[2],xmm13[3],xmm12[3],xmm13[4],xmm12[4],xmm13[5],xmm12[5],xmm13[6],xmm12[6],xmm13[7],xmm12[7]
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm9 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3],xmm15[4],xmm14[4],xmm15[5],xmm14[5],xmm15[6],xmm14[6],xmm15[7],xmm14[7]
+; AVX1-NEXT:    vpslld $16, %xmm0, %xmm0
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm9[0],xmm0[1],xmm9[2,3,4,5,6,7]
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm8[2,3],xmm0[4,5,6,7]
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm1[0],xmm10[0],xmm1[1],xmm10[1],xmm1[2],xmm10[2],xmm1[3],xmm10[3],xmm1[4],xmm10[4],xmm1[5],xmm10[5],xmm1[6],xmm10[6],xmm1[7],xmm10[7]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,1,2,0]
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; AVX1-NEXT:    vpslldq {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm2[0,1]
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,6],xmm2[7]
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3],xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]
+; AVX1-NEXT:    vpslldq {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm2[0,1,2,3,4,5]
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm3 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3],xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[0,1,0,1]
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3,4],xmm2[5],xmm3[6,7]
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1,2,3,4,5],xmm1[6,7]
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4,5,6,7]
 ; AVX1-NEXT:    vmovdqu %xmm0, (%rax)
 ; AVX1-NEXT:    popq %rbx
 ; AVX1-NEXT:    popq %r12
@@ -2154,123 +2280,230 @@ define void @not_avg_v16i8_wide_constants(<16 x i8>* %a, <16 x i8>* %b) nounwind
 ; AVX2-NEXT:    pushq %r13
 ; AVX2-NEXT:    pushq %r12
 ; AVX2-NEXT:    pushq %rbx
+; AVX2-NEXT:    subq $16, %rsp
 ; AVX2-NEXT:    vpmovzxbw {{.*#+}} ymm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero
 ; AVX2-NEXT:    vpmovzxbw {{.*#+}} ymm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero
-; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
-; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm10 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
-; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm1
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
+; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
+; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm3
+; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero
+; AVX2-NEXT:    vextracti128 $1, %ymm3, %xmm4
+; AVX2-NEXT:    vpextrq $1, %xmm4, %rbx
+; AVX2-NEXT:    vmovq %xmm4, %rbp
+; AVX2-NEXT:    vpextrq $1, %xmm3, %rdi
+; AVX2-NEXT:    vmovq %xmm3, %rcx
+; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
+; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm3
+; AVX2-NEXT:    vpextrq $1, %xmm3, %rdx
+; AVX2-NEXT:    vmovq %xmm3, %r9
+; AVX2-NEXT:    vpextrq $1, %xmm2, %r13
+; AVX2-NEXT:    vmovq %xmm2, %r12
 ; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
-; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm3
-; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm5 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero
-; AVX2-NEXT:    vextracti128 $1, %ymm5, %xmm4
-; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm9 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
-; AVX2-NEXT:    vextracti128 $1, %ymm9, %xmm7
-; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm1
-; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
 ; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX2-NEXT:    vpextrq $1, %xmm2, %r15
-; AVX2-NEXT:    vmovq %xmm2, %r14
+; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
+; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm3
+; AVX2-NEXT:    vpextrq $1, %xmm3, %r14
+; AVX2-NEXT:    vmovq %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
+; AVX2-NEXT:    vpextrq $1, %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
+; AVX2-NEXT:    vmovq %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
+; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
 ; AVX2-NEXT:    vpextrq $1, %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
-; AVX2-NEXT:    vmovq %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
-; AVX2-NEXT:    vextracti128 $1, %ymm10, %xmm1
-; AVX2-NEXT:    vpextrq $1, %xmm1, %r13
-; AVX2-NEXT:    vmovq %xmm1, %r11
-; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm11 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
-; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
+; AVX2-NEXT:    vmovq %xmm1, %r10
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm1
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm2
+; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
+; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm3
+; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero
+; AVX2-NEXT:    vextracti128 $1, %ymm3, %xmm4
+; AVX2-NEXT:    vpextrq $1, %xmm4, %rax
+; AVX2-NEXT:    addq %rbx, %rax
+; AVX2-NEXT:    movq %rax, %rbx
+; AVX2-NEXT:    vmovq %xmm4, %rsi
+; AVX2-NEXT:    addq %rbp, %rsi
+; AVX2-NEXT:    vpextrq $1, %xmm3, %rax
+; AVX2-NEXT:    addq %rdi, %rax
+; AVX2-NEXT:    movq %rax, %rdi
+; AVX2-NEXT:    vmovq %xmm3, %r11
+; AVX2-NEXT:    addq %rcx, %r11
+; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
+; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm3
+; AVX2-NEXT:    vpextrq $1, %xmm3, %rcx
+; AVX2-NEXT:    addq %rdx, %rcx
+; AVX2-NEXT:    vmovq %xmm3, %r8
+; AVX2-NEXT:    addq %r9, %r8
+; AVX2-NEXT:    vpextrq $1, %xmm2, %r9
+; AVX2-NEXT:    addq %r13, %r9
+; AVX2-NEXT:    vmovq %xmm2, %r15
+; AVX2-NEXT:    addq %r12, %r15
 ; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm8 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
-; AVX2-NEXT:    vextracti128 $1, %ymm8, %xmm1
-; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; AVX2-NEXT:    vextracti128 $1, %ymm3, %xmm6
-; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm0
-; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; AVX2-NEXT:    vmovd %xmm9, %r12d
-; AVX2-NEXT:    vpextrd $2, %xmm9, %r9d
-; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm0
-; AVX2-NEXT:    vmovd %xmm7, %ecx
-; AVX2-NEXT:    vpextrd $2, %xmm7, %edi
-; AVX2-NEXT:    vmovd %xmm5, %ebx
-; AVX2-NEXT:    vpextrd $2, %xmm5, %esi
-; AVX2-NEXT:    vmovd %xmm4, %edx
-; AVX2-NEXT:    vpextrd $2, %xmm4, %ebp
-; AVX2-NEXT:    vpextrd $2, %xmm1, %eax
-; AVX2-NEXT:    leal -1(%rbp,%rax), %eax
-; AVX2-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; AVX2-NEXT:    vmovd %xmm1, %eax
-; AVX2-NEXT:    leal -1(%rdx,%rax), %eax
-; AVX2-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; AVX2-NEXT:    vpextrd $2, %xmm8, %eax
-; AVX2-NEXT:    leal -1(%rsi,%rax), %eax
-; AVX2-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; AVX2-NEXT:    vmovd %xmm8, %eax
-; AVX2-NEXT:    leal -1(%rbx,%rax), %r10d
-; AVX2-NEXT:    vpextrd $2, %xmm6, %eax
-; AVX2-NEXT:    leal -1(%rdi,%rax), %r8d
-; AVX2-NEXT:    vmovd %xmm6, %eax
-; AVX2-NEXT:    leal -1(%rcx,%rax), %edi
-; AVX2-NEXT:    vpextrd $2, %xmm3, %eax
-; AVX2-NEXT:    leal -1(%r9,%rax), %r9d
-; AVX2-NEXT:    vmovd %xmm3, %ecx
-; AVX2-NEXT:    leal -1(%r12,%rcx), %r12d
-; AVX2-NEXT:    vpextrq $1, %xmm0, %rcx
-; AVX2-NEXT:    leal -1(%r15,%rcx), %r15d
-; AVX2-NEXT:    vmovq %xmm0, %rcx
-; AVX2-NEXT:    leal -1(%r14,%rcx), %r14d
-; AVX2-NEXT:    vpextrq $1, %xmm2, %rdx
-; AVX2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
-; AVX2-NEXT:    leal -1(%rax,%rdx), %edx
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm2
+; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
+; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm3
+; AVX2-NEXT:    vpextrq $1, %xmm3, %rax
+; AVX2-NEXT:    addq %r14, %rax
+; AVX2-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT:    vmovq %xmm3, %rax
+; AVX2-NEXT:    addq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Folded Reload
+; AVX2-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT:    vpextrq $1, %xmm2, %rax
+; AVX2-NEXT:    addq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Folded Reload
+; AVX2-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; AVX2-NEXT:    vmovq %xmm2, %rax
-; AVX2-NEXT:    vextracti128 $1, %ymm11, %xmm0
+; AVX2-NEXT:    addq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Folded Reload
+; AVX2-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX2-NEXT:    vpextrq $1, %xmm0, %rbp
+; AVX2-NEXT:    addq {{[-0-9]+}}(%r{{[sb]}}p), %rbp # 8-byte Folded Reload
+; AVX2-NEXT:    vmovq %xmm0, %r12
+; AVX2-NEXT:    addq %r10, %r12
+; AVX2-NEXT:    vpextrq $1, %xmm1, %rax
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
+; AVX2-NEXT:    vpextrq $1, %xmm0, %r10
+; AVX2-NEXT:    addq %rax, %r10
+; AVX2-NEXT:    vmovq %xmm1, %rax
+; AVX2-NEXT:    vmovq %xmm0, %rdx
+; AVX2-NEXT:    addq %rax, %rdx
+; AVX2-NEXT:    addq $-1, %rbx
+; AVX2-NEXT:    movq %rbx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT:    movl $0, %eax
+; AVX2-NEXT:    adcq $-1, %rax
+; AVX2-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT:    addq $-1, %rsi
+; AVX2-NEXT:    movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT:    movl $0, %eax
+; AVX2-NEXT:    adcq $-1, %rax
+; AVX2-NEXT:    movq %rax, (%rsp) # 8-byte Spill
+; AVX2-NEXT:    addq $-1, %rdi
+; AVX2-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT:    movl $0, %eax
+; AVX2-NEXT:    adcq $-1, %rax
+; AVX2-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT:    addq $-1, %r11
+; AVX2-NEXT:    movq %r11, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT:    movl $0, %eax
+; AVX2-NEXT:    adcq $-1, %rax
+; AVX2-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT:    addq $-1, %rcx
+; AVX2-NEXT:    movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT:    movl $0, %eax
+; AVX2-NEXT:    adcq $-1, %rax
+; AVX2-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT:    addq $-1, %r8
+; AVX2-NEXT:    movq %r8, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT:    movl $0, %eax
+; AVX2-NEXT:    adcq $-1, %rax
+; AVX2-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT:    addq $-1, %r9
+; AVX2-NEXT:    movq %r9, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT:    movl $0, %eax
+; AVX2-NEXT:    adcq $-1, %rax
+; AVX2-NEXT:    movq %rax, %rsi
+; AVX2-NEXT:    addq $-1, %r15
+; AVX2-NEXT:    movq %r15, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT:    movl $0, %r15d
+; AVX2-NEXT:    adcq $-1, %r15
+; AVX2-NEXT:    addq $-1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
+; AVX2-NEXT:    movl $0, %r13d
+; AVX2-NEXT:    adcq $-1, %r13
+; AVX2-NEXT:    addq $-1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
+; AVX2-NEXT:    movl $0, %r14d
+; AVX2-NEXT:    adcq $-1, %r14
+; AVX2-NEXT:    addq $-1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
+; AVX2-NEXT:    movl $0, %ebx
+; AVX2-NEXT:    adcq $-1, %rbx
+; AVX2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; AVX2-NEXT:    addq $-1, %rax
+; AVX2-NEXT:    movl $0, %r11d
+; AVX2-NEXT:    adcq $-1, %r11
+; AVX2-NEXT:    addq $-1, %rbp
+; AVX2-NEXT:    movl $0, %r9d
+; AVX2-NEXT:    adcq $-1, %r9
+; AVX2-NEXT:    addq $-1, %r12
+; AVX2-NEXT:    movl $0, %r8d
+; AVX2-NEXT:    adcq $-1, %r8
+; AVX2-NEXT:    addq $-1, %r10
+; AVX2-NEXT:    movl $0, %edi
+; AVX2-NEXT:    adcq $-1, %rdi
+; AVX2-NEXT:    addq $-1, %rdx
+; AVX2-NEXT:    movl $0, %ecx
+; AVX2-NEXT:    adcq $-1, %rcx
+; AVX2-NEXT:    shldq $63, %rdx, %rcx
+; AVX2-NEXT:    movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT:    shldq $63, %r10, %rdi
+; AVX2-NEXT:    shldq $63, %r12, %r8
+; AVX2-NEXT:    shldq $63, %rbp, %r9
+; AVX2-NEXT:    shldq $63, %rax, %r11
+; AVX2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
+; AVX2-NEXT:    shldq $63, %rdx, %rbx
+; AVX2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
+; AVX2-NEXT:    shldq $63, %rdx, %r14
+; AVX2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
+; AVX2-NEXT:    shldq $63, %rdx, %r13
+; AVX2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; AVX2-NEXT:    shldq $63, %rax, %r15
+; AVX2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; AVX2-NEXT:    shldq $63, %rax, %rsi
+; AVX2-NEXT:    movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
+; AVX2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; AVX2-NEXT:    shldq $63, %rax, %rsi
+; AVX2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r12 # 8-byte Reload
+; AVX2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; AVX2-NEXT:    shldq $63, %rax, %r12
 ; AVX2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
-; AVX2-NEXT:    leal -1(%rcx,%rax), %eax
-; AVX2-NEXT:    vpextrq $1, %xmm0, %rsi
-; AVX2-NEXT:    leal -1(%r13,%rsi), %esi
-; AVX2-NEXT:    vmovq %xmm0, %rbx
-; AVX2-NEXT:    leal -1(%r11,%rbx), %ebx
-; AVX2-NEXT:    vpextrq $1, %xmm10, %rcx
-; AVX2-NEXT:    vpextrq $1, %xmm11, %r13
-; AVX2-NEXT:    leal -1(%rcx,%r13), %ecx
-; AVX2-NEXT:    vmovq %xmm10, %r13
-; AVX2-NEXT:    vmovq %xmm11, %r11
-; AVX2-NEXT:    leaq -1(%r13,%r11), %rbp
-; AVX2-NEXT:    shrq %rbp
-; AVX2-NEXT:    vmovd %ebp, %xmm0
-; AVX2-NEXT:    shrl %ecx
-; AVX2-NEXT:    vpinsrb $1, %ecx, %xmm0, %xmm0
-; AVX2-NEXT:    shrl %ebx
-; AVX2-NEXT:    vpinsrb $2, %ebx, %xmm0, %xmm0
-; AVX2-NEXT:    shrl %esi
-; AVX2-NEXT:    vpinsrb $3, %esi, %xmm0, %xmm0
-; AVX2-NEXT:    shrl %eax
-; AVX2-NEXT:    vpinsrb $4, %eax, %xmm0, %xmm0
-; AVX2-NEXT:    shrl %edx
-; AVX2-NEXT:    vpinsrb $5, %edx, %xmm0, %xmm0
-; AVX2-NEXT:    shrl %r14d
-; AVX2-NEXT:    vpinsrb $6, %r14d, %xmm0, %xmm0
-; AVX2-NEXT:    shrl %r15d
-; AVX2-NEXT:    vpinsrb $7, %r15d, %xmm0, %xmm0
-; AVX2-NEXT:    shrl %r12d
-; AVX2-NEXT:    vpinsrb $8, %r12d, %xmm0, %xmm0
-; AVX2-NEXT:    shrl %r9d
-; AVX2-NEXT:    vpinsrb $9, %r9d, %xmm0, %xmm0
-; AVX2-NEXT:    shrl %edi
-; AVX2-NEXT:    vpinsrb $10, %edi, %xmm0, %xmm0
-; AVX2-NEXT:    shrl %r8d
-; AVX2-NEXT:    vpinsrb $11, %r8d, %xmm0, %xmm0
-; AVX2-NEXT:    shrl %r10d
-; AVX2-NEXT:    vpinsrb $12, %r10d, %xmm0, %xmm0
-; AVX2-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
-; AVX2-NEXT:    shrl %eax
-; AVX2-NEXT:    vpinsrb $13, %eax, %xmm0, %xmm0
-; AVX2-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
-; AVX2-NEXT:    shrl %eax
-; AVX2-NEXT:    vpinsrb $14, %eax, %xmm0, %xmm0
-; AVX2-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
-; AVX2-NEXT:    shrl %eax
-; AVX2-NEXT:    vpinsrb $15, %eax, %xmm0, %xmm0
+; AVX2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; AVX2-NEXT:    shldq $63, %rax, %rcx
+; AVX2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r10 # 8-byte Reload
+; AVX2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; AVX2-NEXT:    shldq $63, %rax, %r10
+; AVX2-NEXT:    movq (%rsp), %rax # 8-byte Reload
+; AVX2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
+; AVX2-NEXT:    shldq $63, %rdx, %rax
+; AVX2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
+; AVX2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rbp # 8-byte Reload
+; AVX2-NEXT:    shldq $63, %rdx, %rbp
+; AVX2-NEXT:    vmovq %rbp, %xmm8
+; AVX2-NEXT:    vmovq %rax, %xmm9
+; AVX2-NEXT:    vmovq %r10, %xmm0
+; AVX2-NEXT:    vmovq %rcx, %xmm1
+; AVX2-NEXT:    vmovq %r12, %xmm12
+; AVX2-NEXT:    vmovq %rsi, %xmm13
+; AVX2-NEXT:    vmovq {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 8-byte Folded Reload
+; AVX2-NEXT:    # xmm14 = mem[0],zero
+; AVX2-NEXT:    vmovq %r15, %xmm15
+; AVX2-NEXT:    vmovq %r13, %xmm10
+; AVX2-NEXT:    vmovq %r14, %xmm11
+; AVX2-NEXT:    vmovq %rbx, %xmm2
+; AVX2-NEXT:    vmovq %r11, %xmm3
+; AVX2-NEXT:    vmovq %r9, %xmm4
+; AVX2-NEXT:    vmovq %r8, %xmm5
+; AVX2-NEXT:    vmovq %rdi, %xmm6
+; AVX2-NEXT:    vmovq {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 8-byte Folded Reload
+; AVX2-NEXT:    # xmm7 = mem[0],zero
+; AVX2-NEXT:    vpunpcklbw {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3],xmm9[4],xmm8[4],xmm9[5],xmm8[5],xmm9[6],xmm8[6],xmm9[7],xmm8[7]
+; AVX2-NEXT:    vpunpcklbw {{.*#+}} xmm9 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX2-NEXT:    vpslldq {{.*#+}} xmm8 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm8[0,1]
+; AVX2-NEXT:    vpbroadcastw %xmm9, %xmm0
+; AVX2-NEXT:    vpblendw {{.*#+}} xmm8 = xmm0[0,1,2,3,4,5,6],xmm8[7]
+; AVX2-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm13[0],xmm12[0],xmm13[1],xmm12[1],xmm13[2],xmm12[2],xmm13[3],xmm12[3],xmm13[4],xmm12[4],xmm13[5],xmm12[5],xmm13[6],xmm12[6],xmm13[7],xmm12[7]
+; AVX2-NEXT:    vpunpcklbw {{.*#+}} xmm9 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3],xmm15[4],xmm14[4],xmm15[5],xmm14[5],xmm15[6],xmm14[6],xmm15[7],xmm14[7]
+; AVX2-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5]
+; AVX2-NEXT:    vpbroadcastw %xmm9, %xmm1
+; AVX2-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3,4],xmm0[5],xmm1[6,7]
+; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0,1,2],xmm8[3]
+; AVX2-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm11[0],xmm10[0],xmm11[1],xmm10[1],xmm11[2],xmm10[2],xmm11[3],xmm10[3],xmm11[4],xmm10[4],xmm11[5],xmm10[5],xmm11[6],xmm10[6],xmm11[7],xmm10[7]
+; AVX2-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; AVX2-NEXT:    vpsllq $48, %xmm1, %xmm1
+; AVX2-NEXT:    vpbroadcastw %xmm2, %xmm2
+; AVX2-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[3],xmm2[4,5,6,7]
+; AVX2-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3],xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]
+; AVX2-NEXT:    vpunpcklbw {{.*#+}} xmm3 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3],xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7]
+; AVX2-NEXT:    vpslld $16, %xmm3, %xmm3
+; AVX2-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0],xmm3[1],xmm2[2,3,4,5,6,7]
+; AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2,3]
+; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
 ; AVX2-NEXT:    vmovdqu %xmm0, (%rax)
+; AVX2-NEXT:    addq $16, %rsp
 ; AVX2-NEXT:    popq %rbx
 ; AVX2-NEXT:    popq %r12
 ; AVX2-NEXT:    popq %r13
@@ -2280,139 +2513,414 @@ define void @not_avg_v16i8_wide_constants(<16 x i8>* %a, <16 x i8>* %b) nounwind
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
-; AVX512-LABEL: not_avg_v16i8_wide_constants:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    pushq %rbp
-; AVX512-NEXT:    pushq %r15
-; AVX512-NEXT:    pushq %r14
-; AVX512-NEXT:    pushq %r13
-; AVX512-NEXT:    pushq %r12
-; AVX512-NEXT:    pushq %rbx
-; AVX512-NEXT:    vpmovzxbw {{.*#+}} ymm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero
-; AVX512-NEXT:    vpmovzxbw {{.*#+}} ymm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero
-; AVX512-NEXT:    vpmovzxwd {{.*#+}} ymm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
-; AVX512-NEXT:    vpmovzxdq {{.*#+}} ymm10 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
-; AVX512-NEXT:    vextracti128 $1, %ymm1, %xmm1
-; AVX512-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
-; AVX512-NEXT:    vextracti128 $1, %ymm1, %xmm3
-; AVX512-NEXT:    vpmovzxdq {{.*#+}} ymm5 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero
-; AVX512-NEXT:    vextracti128 $1, %ymm5, %xmm4
-; AVX512-NEXT:    vpmovzxdq {{.*#+}} ymm9 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
-; AVX512-NEXT:    vextracti128 $1, %ymm9, %xmm7
-; AVX512-NEXT:    vextracti128 $1, %ymm2, %xmm1
-; AVX512-NEXT:    vpmovzxdq {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
-; AVX512-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX512-NEXT:    vpextrq $1, %xmm2, %r15
-; AVX512-NEXT:    vmovq %xmm2, %r14
-; AVX512-NEXT:    vpextrq $1, %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
-; AVX512-NEXT:    vmovq %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
-; AVX512-NEXT:    vextracti128 $1, %ymm10, %xmm1
-; AVX512-NEXT:    vpextrq $1, %xmm1, %r13
-; AVX512-NEXT:    vmovq %xmm1, %r11
-; AVX512-NEXT:    vpmovzxwd {{.*#+}} ymm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX512-NEXT:    vpmovzxdq {{.*#+}} ymm11 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
-; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; AVX512-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpmovzxdq {{.*#+}} ymm8 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
-; AVX512-NEXT:    vextracti128 $1, %ymm8, %xmm1
-; AVX512-NEXT:    vpmovzxdq {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; AVX512-NEXT:    vextracti128 $1, %ymm3, %xmm6
-; AVX512-NEXT:    vextracti128 $1, %ymm2, %xmm0
-; AVX512-NEXT:    vpmovzxdq {{.*#+}} ymm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; AVX512-NEXT:    vmovd %xmm9, %r12d
-; AVX512-NEXT:    vpextrd $2, %xmm9, %r9d
-; AVX512-NEXT:    vextracti128 $1, %ymm2, %xmm0
-; AVX512-NEXT:    vmovd %xmm7, %ecx
-; AVX512-NEXT:    vpextrd $2, %xmm7, %edi
-; AVX512-NEXT:    vmovd %xmm5, %ebx
-; AVX512-NEXT:    vpextrd $2, %xmm5, %esi
-; AVX512-NEXT:    vmovd %xmm4, %edx
-; AVX512-NEXT:    vpextrd $2, %xmm4, %ebp
-; AVX512-NEXT:    vpextrd $2, %xmm1, %eax
-; AVX512-NEXT:    leal -1(%rbp,%rax), %eax
-; AVX512-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; AVX512-NEXT:    vmovd %xmm1, %eax
-; AVX512-NEXT:    leal -1(%rdx,%rax), %eax
-; AVX512-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; AVX512-NEXT:    vpextrd $2, %xmm8, %eax
-; AVX512-NEXT:    leal -1(%rsi,%rax), %eax
-; AVX512-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; AVX512-NEXT:    vmovd %xmm8, %eax
-; AVX512-NEXT:    leal -1(%rbx,%rax), %r10d
-; AVX512-NEXT:    vpextrd $2, %xmm6, %eax
-; AVX512-NEXT:    leal -1(%rdi,%rax), %r8d
-; AVX512-NEXT:    vmovd %xmm6, %eax
-; AVX512-NEXT:    leal -1(%rcx,%rax), %edi
-; AVX512-NEXT:    vpextrd $2, %xmm3, %eax
-; AVX512-NEXT:    leal -1(%r9,%rax), %r9d
-; AVX512-NEXT:    vmovd %xmm3, %ecx
-; AVX512-NEXT:    leal -1(%r12,%rcx), %r12d
-; AVX512-NEXT:    vpextrq $1, %xmm0, %rcx
-; AVX512-NEXT:    leal -1(%r15,%rcx), %r15d
-; AVX512-NEXT:    vmovq %xmm0, %rcx
-; AVX512-NEXT:    leal -1(%r14,%rcx), %r14d
-; AVX512-NEXT:    vpextrq $1, %xmm2, %rdx
-; AVX512-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
-; AVX512-NEXT:    leal -1(%rax,%rdx), %edx
-; AVX512-NEXT:    vmovq %xmm2, %rax
-; AVX512-NEXT:    vextracti128 $1, %ymm11, %xmm0
-; AVX512-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
-; AVX512-NEXT:    leal -1(%rcx,%rax), %eax
-; AVX512-NEXT:    vpextrq $1, %xmm0, %rsi
-; AVX512-NEXT:    leal -1(%r13,%rsi), %esi
-; AVX512-NEXT:    vmovq %xmm0, %rbx
-; AVX512-NEXT:    leal -1(%r11,%rbx), %ebx
-; AVX512-NEXT:    vpextrq $1, %xmm10, %rcx
-; AVX512-NEXT:    vpextrq $1, %xmm11, %r13
-; AVX512-NEXT:    leal -1(%rcx,%r13), %ecx
-; AVX512-NEXT:    vmovq %xmm10, %r13
-; AVX512-NEXT:    vmovq %xmm11, %r11
-; AVX512-NEXT:    leaq -1(%r13,%r11), %rbp
-; AVX512-NEXT:    shrq %rbp
-; AVX512-NEXT:    vmovd %ebp, %xmm0
-; AVX512-NEXT:    shrl %ecx
-; AVX512-NEXT:    vpinsrb $1, %ecx, %xmm0, %xmm0
-; AVX512-NEXT:    shrl %ebx
-; AVX512-NEXT:    vpinsrb $2, %ebx, %xmm0, %xmm0
-; AVX512-NEXT:    shrl %esi
-; AVX512-NEXT:    vpinsrb $3, %esi, %xmm0, %xmm0
-; AVX512-NEXT:    shrl %eax
-; AVX512-NEXT:    vpinsrb $4, %eax, %xmm0, %xmm0
-; AVX512-NEXT:    shrl %edx
-; AVX512-NEXT:    vpinsrb $5, %edx, %xmm0, %xmm0
-; AVX512-NEXT:    shrl %r14d
-; AVX512-NEXT:    vpinsrb $6, %r14d, %xmm0, %xmm0
-; AVX512-NEXT:    shrl %r15d
-; AVX512-NEXT:    vpinsrb $7, %r15d, %xmm0, %xmm0
-; AVX512-NEXT:    shrl %r12d
-; AVX512-NEXT:    vpinsrb $8, %r12d, %xmm0, %xmm0
-; AVX512-NEXT:    shrl %r9d
-; AVX512-NEXT:    vpinsrb $9, %r9d, %xmm0, %xmm0
-; AVX512-NEXT:    shrl %edi
-; AVX512-NEXT:    vpinsrb $10, %edi, %xmm0, %xmm0
-; AVX512-NEXT:    shrl %r8d
-; AVX512-NEXT:    vpinsrb $11, %r8d, %xmm0, %xmm0
-; AVX512-NEXT:    shrl %r10d
-; AVX512-NEXT:    vpinsrb $12, %r10d, %xmm0, %xmm0
-; AVX512-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
-; AVX512-NEXT:    shrl %eax
-; AVX512-NEXT:    vpinsrb $13, %eax, %xmm0, %xmm0
-; AVX512-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
-; AVX512-NEXT:    shrl %eax
-; AVX512-NEXT:    vpinsrb $14, %eax, %xmm0, %xmm0
-; AVX512-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
-; AVX512-NEXT:    shrl %eax
-; AVX512-NEXT:    vpinsrb $15, %eax, %xmm0, %xmm0
-; AVX512-NEXT:    vmovdqu %xmm0, (%rax)
-; AVX512-NEXT:    popq %rbx
-; AVX512-NEXT:    popq %r12
-; AVX512-NEXT:    popq %r13
-; AVX512-NEXT:    popq %r14
-; AVX512-NEXT:    popq %r15
-; AVX512-NEXT:    popq %rbp
-; AVX512-NEXT:    vzeroupper
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: not_avg_v16i8_wide_constants:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    pushq %rbp
+; AVX512F-NEXT:    pushq %r15
+; AVX512F-NEXT:    pushq %r14
+; AVX512F-NEXT:    pushq %r13
+; AVX512F-NEXT:    pushq %r12
+; AVX512F-NEXT:    pushq %rbx
+; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero
+; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm3 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} ymm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX512F-NEXT:    vpmovzxdq {{.*#+}} ymm0 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
+; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm1
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm4
+; AVX512F-NEXT:    vpmovzxdq {{.*#+}} ymm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero
+; AVX512F-NEXT:    vextracti128 $1, %ymm4, %xmm5
+; AVX512F-NEXT:    vpextrq $1, %xmm5, %rdx
+; AVX512F-NEXT:    vmovq %xmm5, %rcx
+; AVX512F-NEXT:    vpextrq $1, %xmm4, %rax
+; AVX512F-NEXT:    vmovq %xmm4, %rbx
+; AVX512F-NEXT:    vpmovzxdq {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm4
+; AVX512F-NEXT:    vpextrq $1, %xmm4, %rdi
+; AVX512F-NEXT:    vmovq %xmm4, %rsi
+; AVX512F-NEXT:    vpextrq $1, %xmm1, %r13
+; AVX512F-NEXT:    vmovq %xmm1, %r15
+; AVX512F-NEXT:    vextracti128 $1, %ymm2, %xmm1
+; AVX512F-NEXT:    vpmovzxdq {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm2
+; AVX512F-NEXT:    vpextrq $1, %xmm2, %r12
+; AVX512F-NEXT:    vmovq %xmm2, %r14
+; AVX512F-NEXT:    vpextrq $1, %xmm1, %r11
+; AVX512F-NEXT:    vmovq %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-NEXT:    vpextrq $1, %xmm1, %r10
+; AVX512F-NEXT:    vmovq %xmm1, %r9
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} ymm2 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
+; AVX512F-NEXT:    vpmovzxdq {{.*#+}} ymm1 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
+; AVX512F-NEXT:    vextracti128 $1, %ymm3, %xmm3
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} ymm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
+; AVX512F-NEXT:    vextracti128 $1, %ymm3, %xmm4
+; AVX512F-NEXT:    vpmovzxdq {{.*#+}} ymm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero
+; AVX512F-NEXT:    vextracti128 $1, %ymm4, %xmm5
+; AVX512F-NEXT:    vpextrq $1, %xmm5, %rbp
+; AVX512F-NEXT:    leal -1(%rdx,%rbp), %edx
+; AVX512F-NEXT:    movl %edx, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; AVX512F-NEXT:    vmovq %xmm5, %rbp
+; AVX512F-NEXT:    leal -1(%rcx,%rbp), %ecx
+; AVX512F-NEXT:    movl %ecx, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; AVX512F-NEXT:    vpextrq $1, %xmm4, %rbp
+; AVX512F-NEXT:    leal -1(%rax,%rbp), %eax
+; AVX512F-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; AVX512F-NEXT:    vmovq %xmm4, %rbp
+; AVX512F-NEXT:    vpmovzxdq {{.*#+}} ymm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero
+; AVX512F-NEXT:    vextracti128 $1, %ymm3, %xmm4
+; AVX512F-NEXT:    leal -1(%rbx,%rbp), %r8d
+; AVX512F-NEXT:    vpextrq $1, %xmm4, %rbp
+; AVX512F-NEXT:    leal -1(%rdi,%rbp), %edi
+; AVX512F-NEXT:    vmovq %xmm4, %rbp
+; AVX512F-NEXT:    leal -1(%rsi,%rbp), %esi
+; AVX512F-NEXT:    vpextrq $1, %xmm3, %rbp
+; AVX512F-NEXT:    leal -1(%r13,%rbp), %r13d
+; AVX512F-NEXT:    vmovq %xmm3, %rbp
+; AVX512F-NEXT:    vextracti128 $1, %ymm2, %xmm2
+; AVX512F-NEXT:    vpmovzxdq {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
+; AVX512F-NEXT:    vextracti128 $1, %ymm2, %xmm3
+; AVX512F-NEXT:    leal -1(%r15,%rbp), %r15d
+; AVX512F-NEXT:    vpextrq $1, %xmm3, %rbp
+; AVX512F-NEXT:    leal -1(%r12,%rbp), %r12d
+; AVX512F-NEXT:    vmovq %xmm3, %rbp
+; AVX512F-NEXT:    leal -1(%r14,%rbp), %r14d
+; AVX512F-NEXT:    vpextrq $1, %xmm2, %rdx
+; AVX512F-NEXT:    leal -1(%r11,%rdx), %r11d
+; AVX512F-NEXT:    vmovq %xmm2, %rbp
+; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm2
+; AVX512F-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; AVX512F-NEXT:    leal -1(%rax,%rbp), %ebp
+; AVX512F-NEXT:    vpextrq $1, %xmm2, %rcx
+; AVX512F-NEXT:    leal -1(%r10,%rcx), %ecx
+; AVX512F-NEXT:    vmovq %xmm2, %rax
+; AVX512F-NEXT:    leal -1(%r9,%rax), %eax
+; AVX512F-NEXT:    vpextrq $1, %xmm0, %rdx
+; AVX512F-NEXT:    vpextrq $1, %xmm1, %r10
+; AVX512F-NEXT:    leal -1(%rdx,%r10), %edx
+; AVX512F-NEXT:    vmovq %xmm0, %r10
+; AVX512F-NEXT:    vmovq %xmm1, %r9
+; AVX512F-NEXT:    leaq -1(%r10,%r9), %rbx
+; AVX512F-NEXT:    shrq %rbx
+; AVX512F-NEXT:    vmovd %ebx, %xmm0
+; AVX512F-NEXT:    shrl %edx
+; AVX512F-NEXT:    vpinsrb $1, %edx, %xmm0, %xmm0
+; AVX512F-NEXT:    shrl %eax
+; AVX512F-NEXT:    vpinsrb $2, %eax, %xmm0, %xmm0
+; AVX512F-NEXT:    shrl %ecx
+; AVX512F-NEXT:    vpinsrb $3, %ecx, %xmm0, %xmm0
+; AVX512F-NEXT:    shrl %ebp
+; AVX512F-NEXT:    vpinsrb $4, %ebp, %xmm0, %xmm0
+; AVX512F-NEXT:    shrl %r11d
+; AVX512F-NEXT:    vpinsrb $5, %r11d, %xmm0, %xmm0
+; AVX512F-NEXT:    shrl %r14d
+; AVX512F-NEXT:    vpinsrb $6, %r14d, %xmm0, %xmm0
+; AVX512F-NEXT:    shrl %r12d
+; AVX512F-NEXT:    vpinsrb $7, %r12d, %xmm0, %xmm0
+; AVX512F-NEXT:    shrl %r15d
+; AVX512F-NEXT:    vpinsrb $8, %r15d, %xmm0, %xmm0
+; AVX512F-NEXT:    shrl %r13d
+; AVX512F-NEXT:    vpinsrb $9, %r13d, %xmm0, %xmm0
+; AVX512F-NEXT:    shrl %esi
+; AVX512F-NEXT:    vpinsrb $10, %esi, %xmm0, %xmm0
+; AVX512F-NEXT:    shrl %edi
+; AVX512F-NEXT:    vpinsrb $11, %edi, %xmm0, %xmm0
+; AVX512F-NEXT:    shrl %r8d
+; AVX512F-NEXT:    vpinsrb $12, %r8d, %xmm0, %xmm0
+; AVX512F-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
+; AVX512F-NEXT:    shrl %eax
+; AVX512F-NEXT:    vpinsrb $13, %eax, %xmm0, %xmm0
+; AVX512F-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
+; AVX512F-NEXT:    shrl %eax
+; AVX512F-NEXT:    vpinsrb $14, %eax, %xmm0, %xmm0
+; AVX512F-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
+; AVX512F-NEXT:    shrl %eax
+; AVX512F-NEXT:    vpinsrb $15, %eax, %xmm0, %xmm0
+; AVX512F-NEXT:    vmovdqu %xmm0, (%rax)
+; AVX512F-NEXT:    popq %rbx
+; AVX512F-NEXT:    popq %r12
+; AVX512F-NEXT:    popq %r13
+; AVX512F-NEXT:    popq %r14
+; AVX512F-NEXT:    popq %r15
+; AVX512F-NEXT:    popq %rbp
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: not_avg_v16i8_wide_constants:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    pushq %rbp
+; AVX512BW-NEXT:    pushq %r15
+; AVX512BW-NEXT:    pushq %r14
+; AVX512BW-NEXT:    pushq %r13
+; AVX512BW-NEXT:    pushq %r12
+; AVX512BW-NEXT:    pushq %rbx
+; AVX512BW-NEXT:    subq $24, %rsp
+; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero
+; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero
+; AVX512BW-NEXT:    vpmovzxwd {{.*#+}} ymm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512BW-NEXT:    vpmovzxdq {{.*#+}} ymm3 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
+; AVX512BW-NEXT:    vextracti128 $1, %ymm3, %xmm4
+; AVX512BW-NEXT:    vmovq %xmm4, %rbx
+; AVX512BW-NEXT:    vpextrq $1, %xmm4, %rbp
+; AVX512BW-NEXT:    vmovq %xmm3, %rdi
+; AVX512BW-NEXT:    vpextrq $1, %xmm3, %rsi
+; AVX512BW-NEXT:    vextracti128 $1, %ymm2, %xmm2
+; AVX512BW-NEXT:    vpmovzxdq {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
+; AVX512BW-NEXT:    vextracti128 $1, %ymm2, %xmm3
+; AVX512BW-NEXT:    vmovq %xmm3, %rdx
+; AVX512BW-NEXT:    vpextrq $1, %xmm3, %r15
+; AVX512BW-NEXT:    vmovq %xmm2, %r8
+; AVX512BW-NEXT:    vpextrq $1, %xmm2, %r14
+; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm0
+; AVX512BW-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512BW-NEXT:    vpmovzxdq {{.*#+}} ymm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX512BW-NEXT:    vextracti128 $1, %ymm2, %xmm3
+; AVX512BW-NEXT:    vmovq %xmm3, %r9
+; AVX512BW-NEXT:    vpextrq $1, %xmm3, %r10
+; AVX512BW-NEXT:    vmovq %xmm2, %r11
+; AVX512BW-NEXT:    vpextrq $1, %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
+; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm0
+; AVX512BW-NEXT:    vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm2
+; AVX512BW-NEXT:    vmovq %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
+; AVX512BW-NEXT:    vpextrq $1, %xmm2, %r13
+; AVX512BW-NEXT:    vpmovzxwd {{.*#+}} ymm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX512BW-NEXT:    vpmovzxdq {{.*#+}} ymm3 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
+; AVX512BW-NEXT:    vextracti128 $1, %ymm3, %xmm4
+; AVX512BW-NEXT:    vmovq %xmm4, %rax
+; AVX512BW-NEXT:    addq %rbx, %rax
+; AVX512BW-NEXT:    movq %rax, %rbx
+; AVX512BW-NEXT:    vpextrq $1, %xmm4, %rax
+; AVX512BW-NEXT:    addq %rbp, %rax
+; AVX512BW-NEXT:    movq %rax, %rbp
+; AVX512BW-NEXT:    vmovq %xmm3, %rcx
+; AVX512BW-NEXT:    addq %rdi, %rcx
+; AVX512BW-NEXT:    vpextrq $1, %xmm3, %r12
+; AVX512BW-NEXT:    addq %rsi, %r12
+; AVX512BW-NEXT:    vextracti128 $1, %ymm2, %xmm2
+; AVX512BW-NEXT:    vpmovzxdq {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
+; AVX512BW-NEXT:    vextracti128 $1, %ymm2, %xmm3
+; AVX512BW-NEXT:    vmovq %xmm3, %rax
+; AVX512BW-NEXT:    addq %rdx, %rax
+; AVX512BW-NEXT:    movq %rax, %rdx
+; AVX512BW-NEXT:    vpextrq $1, %xmm3, %rax
+; AVX512BW-NEXT:    addq %r15, %rax
+; AVX512BW-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512BW-NEXT:    vmovq %xmm2, %rax
+; AVX512BW-NEXT:    addq %r8, %rax
+; AVX512BW-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512BW-NEXT:    vpextrq $1, %xmm2, %rax
+; AVX512BW-NEXT:    addq %r14, %rax
+; AVX512BW-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512BW-NEXT:    vextracti128 $1, %ymm1, %xmm1
+; AVX512BW-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX512BW-NEXT:    vpmovzxdq {{.*#+}} ymm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; AVX512BW-NEXT:    vextracti128 $1, %ymm2, %xmm3
+; AVX512BW-NEXT:    vmovq %xmm3, %rax
+; AVX512BW-NEXT:    addq %r9, %rax
+; AVX512BW-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512BW-NEXT:    vpextrq $1, %xmm3, %rax
+; AVX512BW-NEXT:    addq %r10, %rax
+; AVX512BW-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512BW-NEXT:    vmovq %xmm2, %rax
+; AVX512BW-NEXT:    addq %r11, %rax
+; AVX512BW-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512BW-NEXT:    vpextrq $1, %xmm2, %r14
+; AVX512BW-NEXT:    addq {{[-0-9]+}}(%r{{[sb]}}p), %r14 # 8-byte Folded Reload
+; AVX512BW-NEXT:    vextracti128 $1, %ymm1, %xmm1
+; AVX512BW-NEXT:    vpmovzxdq {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; AVX512BW-NEXT:    vextracti128 $1, %ymm1, %xmm2
+; AVX512BW-NEXT:    vmovq %xmm2, %r10
+; AVX512BW-NEXT:    addq {{[-0-9]+}}(%r{{[sb]}}p), %r10 # 8-byte Folded Reload
+; AVX512BW-NEXT:    vpextrq $1, %xmm2, %r9
+; AVX512BW-NEXT:    addq %r13, %r9
+; AVX512BW-NEXT:    vmovq %xmm0, %rax
+; AVX512BW-NEXT:    vmovq %xmm1, %r8
+; AVX512BW-NEXT:    addq %rax, %r8
+; AVX512BW-NEXT:    vpextrq $1, %xmm0, %rdi
+; AVX512BW-NEXT:    vpextrq $1, %xmm1, %rsi
+; AVX512BW-NEXT:    addq %rdi, %rsi
+; AVX512BW-NEXT:    addq $-1, %rbx
+; AVX512BW-NEXT:    movq %rbx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512BW-NEXT:    movl $0, %r15d
+; AVX512BW-NEXT:    adcq $-1, %r15
+; AVX512BW-NEXT:    addq $-1, %rbp
+; AVX512BW-NEXT:    movq %rbp, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512BW-NEXT:    movl $0, %ebx
+; AVX512BW-NEXT:    adcq $-1, %rbx
+; AVX512BW-NEXT:    addq $-1, %rcx
+; AVX512BW-NEXT:    movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512BW-NEXT:    movl $0, %r11d
+; AVX512BW-NEXT:    adcq $-1, %r11
+; AVX512BW-NEXT:    addq $-1, %r12
+; AVX512BW-NEXT:    movq %r12, (%rsp) # 8-byte Spill
+; AVX512BW-NEXT:    movl $0, %edi
+; AVX512BW-NEXT:    adcq $-1, %rdi
+; AVX512BW-NEXT:    addq $-1, %rdx
+; AVX512BW-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512BW-NEXT:    movl $0, %eax
+; AVX512BW-NEXT:    adcq $-1, %rax
+; AVX512BW-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512BW-NEXT:    addq $-1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
+; AVX512BW-NEXT:    movl $0, %eax
+; AVX512BW-NEXT:    adcq $-1, %rax
+; AVX512BW-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512BW-NEXT:    addq $-1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
+; AVX512BW-NEXT:    movl $0, %r13d
+; AVX512BW-NEXT:    adcq $-1, %r13
+; AVX512BW-NEXT:    addq $-1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
+; AVX512BW-NEXT:    movl $0, %r12d
+; AVX512BW-NEXT:    adcq $-1, %r12
+; AVX512BW-NEXT:    addq $-1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
+; AVX512BW-NEXT:    movl $0, %eax
+; AVX512BW-NEXT:    adcq $-1, %rax
+; AVX512BW-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512BW-NEXT:    addq $-1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
+; AVX512BW-NEXT:    movl $0, %eax
+; AVX512BW-NEXT:    adcq $-1, %rax
+; AVX512BW-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512BW-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
+; AVX512BW-NEXT:    addq $-1, %rcx
+; AVX512BW-NEXT:    movl $0, %eax
+; AVX512BW-NEXT:    adcq $-1, %rax
+; AVX512BW-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512BW-NEXT:    addq $-1, %r14
+; AVX512BW-NEXT:    movl $0, %eax
+; AVX512BW-NEXT:    adcq $-1, %rax
+; AVX512BW-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512BW-NEXT:    addq $-1, %r10
+; AVX512BW-NEXT:    movl $0, %eax
+; AVX512BW-NEXT:    adcq $-1, %rax
+; AVX512BW-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512BW-NEXT:    addq $-1, %r9
+; AVX512BW-NEXT:    movl $0, %edx
+; AVX512BW-NEXT:    adcq $-1, %rdx
+; AVX512BW-NEXT:    addq $-1, %r8
+; AVX512BW-NEXT:    movl $0, %eax
+; AVX512BW-NEXT:    adcq $-1, %rax
+; AVX512BW-NEXT:    addq $-1, %rsi
+; AVX512BW-NEXT:    movl $0, %ebp
+; AVX512BW-NEXT:    adcq $-1, %rbp
+; AVX512BW-NEXT:    shldq $63, %rsi, %rbp
+; AVX512BW-NEXT:    movq %rbp, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512BW-NEXT:    shldq $63, %r8, %rax
+; AVX512BW-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512BW-NEXT:    shldq $63, %r9, %rdx
+; AVX512BW-NEXT:    movq %rdx, %rbp
+; AVX512BW-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Reload
+; AVX512BW-NEXT:    shldq $63, %r10, %r8
+; AVX512BW-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r10 # 8-byte Reload
+; AVX512BW-NEXT:    shldq $63, %r14, %r10
+; AVX512BW-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r9 # 8-byte Reload
+; AVX512BW-NEXT:    shldq $63, %rcx, %r9
+; AVX512BW-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r14 # 8-byte Reload
+; AVX512BW-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; AVX512BW-NEXT:    shldq $63, %rax, %r14
+; AVX512BW-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; AVX512BW-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
+; AVX512BW-NEXT:    shldq $63, %rax, %rsi
+; AVX512BW-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; AVX512BW-NEXT:    shldq $63, %rax, %r12
+; AVX512BW-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; AVX512BW-NEXT:    shldq $63, %rax, %r13
+; AVX512BW-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; AVX512BW-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
+; AVX512BW-NEXT:    shldq $63, %rax, %rdx
+; AVX512BW-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
+; AVX512BW-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; AVX512BW-NEXT:    shldq $63, %rax, %rcx
+; AVX512BW-NEXT:    movq (%rsp), %rax # 8-byte Reload
+; AVX512BW-NEXT:    shldq $63, %rax, %rdi
+; AVX512BW-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; AVX512BW-NEXT:    shldq $63, %rax, %r11
+; AVX512BW-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; AVX512BW-NEXT:    shldq $63, %rax, %rbx
+; AVX512BW-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; AVX512BW-NEXT:    shldq $63, %rax, %r15
+; AVX512BW-NEXT:    vmovq %r15, %xmm0
+; AVX512BW-NEXT:    vmovq %rbx, %xmm1
+; AVX512BW-NEXT:    vmovq %r11, %xmm2
+; AVX512BW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512BW-NEXT:    vmovq %rdi, %xmm1
+; AVX512BW-NEXT:    vinserti128 $1, %xmm1, %ymm2, %ymm1
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512BW-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX512BW-NEXT:    vmovd %eax, %xmm2
+; AVX512BW-NEXT:    vpextrb $0, %xmm1, %eax
+; AVX512BW-NEXT:    vpinsrb $1, %eax, %xmm2, %xmm1
+; AVX512BW-NEXT:    vextracti32x4 $2, %zmm0, %xmm2
+; AVX512BW-NEXT:    vpextrb $0, %xmm2, %eax
+; AVX512BW-NEXT:    vpinsrb $2, %eax, %xmm1, %xmm1
+; AVX512BW-NEXT:    vextracti32x4 $3, %zmm0, %xmm0
+; AVX512BW-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX512BW-NEXT:    vpinsrb $3, %eax, %xmm1, %xmm0
+; AVX512BW-NEXT:    vmovq %rcx, %xmm1
+; AVX512BW-NEXT:    vmovq %rdx, %xmm2
+; AVX512BW-NEXT:    vmovq %r13, %xmm3
+; AVX512BW-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512BW-NEXT:    vmovq %r12, %xmm2
+; AVX512BW-NEXT:    vinserti128 $1, %xmm2, %ymm3, %ymm2
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm1, %zmm2, %zmm1
+; AVX512BW-NEXT:    vpextrb $0, %xmm1, %eax
+; AVX512BW-NEXT:    vpinsrb $4, %eax, %xmm0, %xmm0
+; AVX512BW-NEXT:    vextracti128 $1, %ymm1, %xmm2
+; AVX512BW-NEXT:    vpextrb $0, %xmm2, %eax
+; AVX512BW-NEXT:    vpinsrb $5, %eax, %xmm0, %xmm0
+; AVX512BW-NEXT:    vextracti32x4 $2, %zmm1, %xmm2
+; AVX512BW-NEXT:    vpextrb $0, %xmm2, %eax
+; AVX512BW-NEXT:    vpinsrb $6, %eax, %xmm0, %xmm0
+; AVX512BW-NEXT:    vextracti32x4 $3, %zmm1, %xmm1
+; AVX512BW-NEXT:    vpextrb $0, %xmm1, %eax
+; AVX512BW-NEXT:    vpinsrb $7, %eax, %xmm0, %xmm0
+; AVX512BW-NEXT:    vmovq %rsi, %xmm1
+; AVX512BW-NEXT:    vmovq %r14, %xmm2
+; AVX512BW-NEXT:    vmovq %r9, %xmm3
+; AVX512BW-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512BW-NEXT:    vmovq %r10, %xmm2
+; AVX512BW-NEXT:    vinserti128 $1, %xmm2, %ymm3, %ymm2
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm1, %zmm2, %zmm1
+; AVX512BW-NEXT:    vpextrb $0, %xmm1, %eax
+; AVX512BW-NEXT:    vpinsrb $8, %eax, %xmm0, %xmm0
+; AVX512BW-NEXT:    vextracti128 $1, %ymm1, %xmm2
+; AVX512BW-NEXT:    vpextrb $0, %xmm2, %eax
+; AVX512BW-NEXT:    vpinsrb $9, %eax, %xmm0, %xmm0
+; AVX512BW-NEXT:    vextracti32x4 $2, %zmm1, %xmm2
+; AVX512BW-NEXT:    vpextrb $0, %xmm2, %eax
+; AVX512BW-NEXT:    vpinsrb $10, %eax, %xmm0, %xmm0
+; AVX512BW-NEXT:    vextracti32x4 $3, %zmm1, %xmm1
+; AVX512BW-NEXT:    vpextrb $0, %xmm1, %eax
+; AVX512BW-NEXT:    vpinsrb $11, %eax, %xmm0, %xmm0
+; AVX512BW-NEXT:    vmovq %r8, %xmm1
+; AVX512BW-NEXT:    vmovq %rbp, %xmm2
+; AVX512BW-NEXT:    vmovq {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 8-byte Folded Reload
+; AVX512BW-NEXT:    # xmm3 = mem[0],zero
+; AVX512BW-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512BW-NEXT:    vmovq {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 8-byte Folded Reload
+; AVX512BW-NEXT:    # xmm2 = mem[0],zero
+; AVX512BW-NEXT:    vinserti128 $1, %xmm2, %ymm3, %ymm2
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm1, %zmm2, %zmm1
+; AVX512BW-NEXT:    vpextrb $0, %xmm1, %eax
+; AVX512BW-NEXT:    vpinsrb $12, %eax, %xmm0, %xmm0
+; AVX512BW-NEXT:    vextracti128 $1, %ymm1, %xmm2
+; AVX512BW-NEXT:    vpextrb $0, %xmm2, %eax
+; AVX512BW-NEXT:    vpinsrb $13, %eax, %xmm0, %xmm0
+; AVX512BW-NEXT:    vextracti32x4 $2, %zmm1, %xmm2
+; AVX512BW-NEXT:    vpextrb $0, %xmm2, %eax
+; AVX512BW-NEXT:    vpinsrb $14, %eax, %xmm0, %xmm0
+; AVX512BW-NEXT:    vextracti32x4 $3, %zmm1, %xmm1
+; AVX512BW-NEXT:    vpextrb $0, %xmm1, %eax
+; AVX512BW-NEXT:    vpinsrb $15, %eax, %xmm0, %xmm0
+; AVX512BW-NEXT:    vmovdqu %xmm0, (%rax)
+; AVX512BW-NEXT:    addq $24, %rsp
+; AVX512BW-NEXT:    popq %rbx
+; AVX512BW-NEXT:    popq %r12
+; AVX512BW-NEXT:    popq %r13
+; AVX512BW-NEXT:    popq %r14
+; AVX512BW-NEXT:    popq %r15
+; AVX512BW-NEXT:    popq %rbp
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
   %1 = load <16 x i8>, <16 x i8>* %a
   %2 = load <16 x i8>, <16 x i8>* %b
   %3 = zext <16 x i8> %1 to <16 x i128>
diff --git a/llvm/test/CodeGen/X86/avoid-sfb-overlaps.ll b/llvm/test/CodeGen/X86/avoid-sfb-overlaps.ll
index 4fc6ca5ee..38b4e2ee0 100644
--- a/llvm/test/CodeGen/X86/avoid-sfb-overlaps.ll
+++ b/llvm/test/CodeGen/X86/avoid-sfb-overlaps.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-linux -verify-machineinstrs | FileCheck %s -check-prefix=CHECK
-; RUN: llc < %s -mtriple=x86_64-linux --x86-disable-avoid-SFB -verify-machineinstrs | FileCheck %s --check-prefix=DISABLED
+; RUN: llc < %s -mtriple=x86_64-linux -mcpu=x86-64 -verify-machineinstrs | FileCheck %s -check-prefix=CHECK
+; RUN: llc < %s -mtriple=x86_64-linux -mcpu=x86-64 --x86-disable-avoid-SFB -verify-machineinstrs | FileCheck %s --check-prefix=DISABLED
 ; RUN: llc < %s -mtriple=x86_64-linux -mcpu=core-avx2 -verify-machineinstrs | FileCheck %s -check-prefix=CHECK-AVX2
 ; RUN: llc < %s -mtriple=x86_64-linux -mcpu=skx -verify-machineinstrs | FileCheck %s -check-prefix=CHECK-AVX512
 
@@ -515,7 +515,6 @@ entry:
   ret void
 }
 
-attributes #0 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { argmemonly nounwind }
-
+attributes #0 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
+attributes #1 = { argmemonly nounwind }
diff --git a/llvm/test/CodeGen/X86/avoid-sfb.ll b/llvm/test/CodeGen/X86/avoid-sfb.ll
index 9d6c6c9c0..37f235288 100644
--- a/llvm/test/CodeGen/X86/avoid-sfb.ll
+++ b/llvm/test/CodeGen/X86/avoid-sfb.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-linux -verify-machineinstrs | FileCheck %s -check-prefix=CHECK
-; RUN: llc < %s -mtriple=x86_64-linux --x86-disable-avoid-SFB -verify-machineinstrs | FileCheck %s --check-prefix=DISABLED
+; RUN: llc < %s -mtriple=x86_64-linux -mcpu=x86-64 -verify-machineinstrs | FileCheck %s -check-prefix=CHECK
+; RUN: llc < %s -mtriple=x86_64-linux -mcpu=x86-64 --x86-disable-avoid-SFB -verify-machineinstrs | FileCheck %s --check-prefix=DISABLED
 ; RUN: llc < %s -mtriple=x86_64-linux -mcpu=core-avx2 -verify-machineinstrs | FileCheck %s -check-prefix=CHECK-AVX2
 ; RUN: llc < %s -mtriple=x86_64-linux -mcpu=skx -verify-machineinstrs | FileCheck %s -check-prefix=CHECK-AVX512
 
@@ -1243,11 +1243,6 @@ if.end:                                           ; preds = %if.then, %entry
 declare void @bar(%struct.S*) local_unnamed_addr #1
 
 
-; Function Attrs: argmemonly nounwind
-declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture writeonly, i8* nocapture readonly, i64, i32, i1) #1
-
-attributes #0 = { nounwind uwtable "target-cpu"="x86-64" }
-
 %struct.S7 = type { float, float, float , float, float, float, float, float }
 
 ; Function Attrs: nounwind uwtable
@@ -1527,5 +1522,7 @@ entry:
   ret void
 }
 
+; Function Attrs: argmemonly nounwind
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture writeonly, i8* nocapture readonly, i64, i32, i1) #1
 
-
+attributes #0 = { nounwind uwtable }
diff --git a/llvm/test/CodeGen/X86/avx-basic.ll b/llvm/test/CodeGen/X86/avx-basic.ll
index b85fd4e64..67b029104 100644
--- a/llvm/test/CodeGen/X86/avx-basic.ll
+++ b/llvm/test/CodeGen/X86/avx-basic.ll
@@ -19,8 +19,8 @@ define void @zero128() nounwind ssp {
 define void @zero256() nounwind ssp {
 ; CHECK-LABEL: zero256:
 ; CHECK:       ## %bb.0:
-; CHECK-NEXT:    movq _x@{{.*}}(%rip), %rax
 ; CHECK-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; CHECK-NEXT:    movq _x@{{.*}}(%rip), %rax
 ; CHECK-NEXT:    vmovaps %ymm0, (%rax)
 ; CHECK-NEXT:    movq _y@{{.*}}(%rip), %rax
 ; CHECK-NEXT:    vmovaps %ymm0, (%rax)
diff --git a/llvm/test/CodeGen/X86/avx-bitcast.ll b/llvm/test/CodeGen/X86/avx-bitcast.ll
index 150c7ccfa..d4df92f01 100644
--- a/llvm/test/CodeGen/X86/avx-bitcast.ll
+++ b/llvm/test/CodeGen/X86/avx-bitcast.ll
@@ -1,9 +1,11 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -O0 -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s
 
 define i64 @bitcasti64tof64() {
 ; CHECK-LABEL: bitcasti64tof64:
 ; CHECK:       # %bb.0:
-; CHECK:         vmovsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    # implicit-def: $rax
+; CHECK-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; CHECK-NEXT:    vmovq %xmm0, %rax
 ; CHECK-NEXT:    retq
   %a = load double, double* undef
diff --git a/llvm/test/CodeGen/X86/avx-cvt-2.ll b/llvm/test/CodeGen/X86/avx-cvt-2.ll
index 7c2df3e99..63e823684 100644
--- a/llvm/test/CodeGen/X86/avx-cvt-2.ll
+++ b/llvm/test/CodeGen/X86/avx-cvt-2.ll
@@ -40,7 +40,7 @@ define void @fptoui8(%f32vec_t %a, %i8vec_t *%p) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vcvttps2dq %ymm0, %ymm0
 ; CHECK-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; CHECK-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
 ; CHECK-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
 ; CHECK-NEXT:    vmovq %xmm0, (%rdi)
 ; CHECK-NEXT:    vzeroupper
diff --git a/llvm/test/CodeGen/X86/avx-cvt.ll b/llvm/test/CodeGen/X86/avx-cvt.ll
index 2057abf41..653a88edd 100644
--- a/llvm/test/CodeGen/X86/avx-cvt.ll
+++ b/llvm/test/CodeGen/X86/avx-cvt.ll
@@ -190,6 +190,16 @@ define float @floor_f32_load(float* %aptr) optsize {
   ret float %res
 }
 
+define float @floor_f32_load_pgso(float* %aptr) !prof !14 {
+; CHECK-LABEL: floor_f32_load_pgso:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vroundss $9, (%rdi), %xmm0, %xmm0
+; CHECK-NEXT:    retq
+  %a = load float, float* %aptr
+  %res = call float @llvm.floor.f32(float %a)
+  ret float %res
+}
+
 define double @nearbyint_f64_load(double* %aptr) optsize {
 ; CHECK-LABEL: nearbyint_f64_load:
 ; CHECK:       # %bb.0:
@@ -200,3 +210,29 @@ define double @nearbyint_f64_load(double* %aptr) optsize {
   ret double %res
 }
 
+define double @nearbyint_f64_load_pgso(double* %aptr) !prof !14 {
+; CHECK-LABEL: nearbyint_f64_load_pgso:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vroundsd $12, (%rdi), %xmm0, %xmm0
+; CHECK-NEXT:    retq
+  %a = load double, double* %aptr
+  %res = call double @llvm.nearbyint.f64(double %a)
+  ret double %res
+}
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 1, !"ProfileSummary", !1}
+!1 = !{!2, !3, !4, !5, !6, !7, !8, !9}
+!2 = !{!"ProfileFormat", !"InstrProf"}
+!3 = !{!"TotalCount", i64 10000}
+!4 = !{!"MaxCount", i64 10}
+!5 = !{!"MaxInternalCount", i64 1}
+!6 = !{!"MaxFunctionCount", i64 1000}
+!7 = !{!"NumCounts", i64 3}
+!8 = !{!"NumFunctions", i64 3}
+!9 = !{!"DetailedSummary", !10}
+!10 = !{!11, !12, !13}
+!11 = !{i32 10000, i64 100, i32 1}
+!12 = !{i32 999000, i64 100, i32 1}
+!13 = !{i32 999999, i64 1, i32 2}
+!14 = !{!"function_entry_count", i64 0}
diff --git a/llvm/test/CodeGen/X86/avx-fp2int.ll b/llvm/test/CodeGen/X86/avx-fp2int.ll
index d1aa1f281..ac5fcfe4d 100644
--- a/llvm/test/CodeGen/X86/avx-fp2int.ll
+++ b/llvm/test/CodeGen/X86/avx-fp2int.ll
@@ -7,6 +7,7 @@ define <4 x i8> @test1(<4 x double> %d) {
 ; CHECK-LABEL: test1:
 ; CHECK:       ## %bb.0:
 ; CHECK-NEXT:    vcvttpd2dq %ymm0, %xmm0
+; CHECK-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retl
   %c = fptoui <4 x double> %d to <4 x i8>
@@ -16,6 +17,7 @@ define <4 x i8> @test2(<4 x double> %d) {
 ; CHECK-LABEL: test2:
 ; CHECK:       ## %bb.0:
 ; CHECK-NEXT:    vcvttpd2dq %ymm0, %xmm0
+; CHECK-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retl
   %c = fptosi <4 x double> %d to <4 x i8>
diff --git a/llvm/test/CodeGen/X86/avx-intel-ocl.ll b/llvm/test/CodeGen/X86/avx-intel-ocl.ll
index 456006178..89630eec7 100644
--- a/llvm/test/CodeGen/X86/avx-intel-ocl.ll
+++ b/llvm/test/CodeGen/X86/avx-intel-ocl.ll
@@ -51,8 +51,8 @@ define <16 x float> @testf16_inp(<16 x float> %a, <16 x float> %b) nounwind {
 ; preserved ymm8-ymm15
 ; X64-LABEL: testf16_regs
 ; X64: call
-; X64: vaddps  {{%ymm[8-9]}}, {{%ymm[0-1]}}, {{%ymm[0-1]}}
-; X64: vaddps  {{%ymm[8-9]}}, {{%ymm[0-1]}}, {{%ymm[0-1]}}
+; X64: vaddps  {{%ymm[0-1]}}, {{%ymm[8-9]}}, {{%ymm[0-1]}}
+; X64: vaddps  {{%ymm[0-1]}}, {{%ymm[8-9]}}, {{%ymm[0-1]}}
 ; X64: ret
 
 define <16 x float> @testf16_regs(<16 x float> %a, <16 x float> %b) nounwind {
diff --git a/llvm/test/CodeGen/X86/avx-isa-check.ll b/llvm/test/CodeGen/X86/avx-isa-check.ll
index 5d66dfde0..decf604b4 100644
--- a/llvm/test/CodeGen/X86/avx-isa-check.ll
+++ b/llvm/test/CodeGen/X86/avx-isa-check.ll
@@ -352,7 +352,7 @@ define <8 x float> @shuffle_v8f32_11335577(<8 x float> %a, <8 x float> %b) {
 }
 
 define <4 x float> @shuffle_v4f32_1133(<4 x float> %a, <4 x float> %b) {
-; vmovshdup 128 test 
+; vmovshdup 128 test
   %shuffle = shufflevector <4 x float> %a, <4 x float> %b, <4 x i32> <i32 1, i32 1, i32 3, i32 3>
   ret <4 x float> %shuffle
 }
diff --git a/llvm/test/CodeGen/X86/avx-load-store.ll b/llvm/test/CodeGen/X86/avx-load-store.ll
index 402e27051..1b3c35855 100644
--- a/llvm/test/CodeGen/X86/avx-load-store.ll
+++ b/llvm/test/CodeGen/X86/avx-load-store.ll
@@ -333,3 +333,26 @@ define void @add4i64a16(<4 x i64>* %ret, <4 x i64>* %bp) nounwind {
   ret void
 }
 
+; This used to crash.
+; v2i128 may not be a "simple" (MVT) type, but we can split that.
+; This example gets split further in legalization.
+
+define void @PR43916(<2 x i128> %y, <2 x i128>* %z) {
+; CHECK-LABEL: PR43916:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %rcx, 24(%r8)
+; CHECK-NEXT:    movq %rdx, 16(%r8)
+; CHECK-NEXT:    movq %rsi, 8(%r8)
+; CHECK-NEXT:    movq %rdi, (%r8)
+; CHECK-NEXT:    retq
+;
+; CHECK_O0-LABEL: PR43916:
+; CHECK_O0:       # %bb.0:
+; CHECK_O0-NEXT:    movq %rdi, (%r8)
+; CHECK_O0-NEXT:    movq %rsi, 8(%r8)
+; CHECK_O0-NEXT:    movq %rdx, 16(%r8)
+; CHECK_O0-NEXT:    movq %rcx, 24(%r8)
+; CHECK_O0-NEXT:    retq
+  store <2 x i128> %y, <2 x i128>* %z, align 16
+  ret void
+}
diff --git a/llvm/test/CodeGen/X86/avx-minmax.ll b/llvm/test/CodeGen/X86/avx-minmax.ll
index 002d99fd8..6da04c50c 100644
--- a/llvm/test/CodeGen/X86/avx-minmax.ll
+++ b/llvm/test/CodeGen/X86/avx-minmax.ll
@@ -1,64 +1,81 @@
-; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx -asm-verbose=false -enable-unsafe-fp-math -enable-no-nans-fp-math | FileCheck -check-prefix=UNSAFE %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx -enable-unsafe-fp-math -enable-no-nans-fp-math | FileCheck %s
 
-; UNSAFE-LABEL: maxpd:
-; UNSAFE: vmaxpd {{.+}}, %xmm
 define <2 x double> @maxpd(<2 x double> %x, <2 x double> %y) {
+; CHECK-LABEL: maxpd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmaxpd %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    retq
   %max_is_x = fcmp oge <2 x double> %x, %y
   %max = select <2 x i1> %max_is_x, <2 x double> %x, <2 x double> %y
   ret <2 x double> %max
 }
 
-; UNSAFE-LABEL: minpd:
-; UNSAFE: vminpd {{.+}}, %xmm
 define <2 x double> @minpd(<2 x double> %x, <2 x double> %y) {
+; CHECK-LABEL: minpd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vminpd %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    retq
   %min_is_x = fcmp ole <2 x double> %x, %y
   %min = select <2 x i1> %min_is_x, <2 x double> %x, <2 x double> %y
   ret <2 x double> %min
 }
 
-; UNSAFE-LABEL: maxps:
-; UNSAFE: vmaxps {{.+}}, %xmm
 define <4 x float> @maxps(<4 x float> %x, <4 x float> %y) {
+; CHECK-LABEL: maxps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmaxps %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    retq
   %max_is_x = fcmp oge <4 x float> %x, %y
   %max = select <4 x i1> %max_is_x, <4 x float> %x, <4 x float> %y
   ret <4 x float> %max
 }
 
-; UNSAFE-LABEL: minps:
-; UNSAFE: vminps {{.+}}, %xmm
 define <4 x float> @minps(<4 x float> %x, <4 x float> %y) {
+; CHECK-LABEL: minps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vminps %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    retq
   %min_is_x = fcmp ole <4 x float> %x, %y
   %min = select <4 x i1> %min_is_x, <4 x float> %x, <4 x float> %y
   ret <4 x float> %min
 }
 
-; UNSAFE-LABEL: vmaxpd:
-; UNSAFE: vmaxpd {{.+}}, %ymm
 define <4 x double> @vmaxpd(<4 x double> %x, <4 x double> %y) {
+; CHECK-LABEL: vmaxpd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmaxpd %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    retq
   %max_is_x = fcmp oge <4 x double> %x, %y
   %max = select <4 x i1> %max_is_x, <4 x double> %x, <4 x double> %y
   ret <4 x double> %max
 }
 
-; UNSAFE-LABEL: vminpd:
-; UNSAFE: vminpd {{.+}}, %ymm
 define <4 x double> @vminpd(<4 x double> %x, <4 x double> %y) {
+; CHECK-LABEL: vminpd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vminpd %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    retq
   %min_is_x = fcmp ole <4 x double> %x, %y
   %min = select <4 x i1> %min_is_x, <4 x double> %x, <4 x double> %y
   ret <4 x double> %min
 }
 
-; UNSAFE-LABEL: vmaxps:
-; UNSAFE: vmaxps {{.+}}, %ymm
 define <8 x float> @vmaxps(<8 x float> %x, <8 x float> %y) {
+; CHECK-LABEL: vmaxps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmaxps %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    retq
   %max_is_x = fcmp oge <8 x float> %x, %y
   %max = select <8 x i1> %max_is_x, <8 x float> %x, <8 x float> %y
   ret <8 x float> %max
 }
 
-; UNSAFE-LABEL: vminps:
-; UNSAFE: vminps {{.+}}, %ymm
 define <8 x float> @vminps(<8 x float> %x, <8 x float> %y) {
+; CHECK-LABEL: vminps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vminps %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    retq
   %min_is_x = fcmp ole <8 x float> %x, %y
   %min = select <8 x i1> %min_is_x, <8 x float> %x, <8 x float> %y
   ret <8 x float> %min
diff --git a/llvm/test/CodeGen/X86/avx-splat.ll b/llvm/test/CodeGen/X86/avx-splat.ll
index e1779ac8f..6a9f8a723 100644
--- a/llvm/test/CodeGen/X86/avx-splat.ll
+++ b/llvm/test/CodeGen/X86/avx-splat.ll
@@ -1,12 +1,13 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=CHECK,X86
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=CHECK,X64
 
 define <32 x i8> @funcA(<32 x i8> %a) nounwind uwtable readnone ssp {
 ; CHECK-LABEL: funcA:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5]
 ; CHECK-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
 entry:
   %shuffle = shufflevector <32 x i8> %a, <32 x i8> undef, <32 x i32> <i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5>
   ret <32 x i8> %shuffle
@@ -18,19 +19,24 @@ define <16 x i16> @funcB(<16 x i16> %a) nounwind uwtable readnone ssp {
 ; CHECK-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,5,6,7]
 ; CHECK-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,2,2,2]
 ; CHECK-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
 entry:
   %shuffle = shufflevector <16 x i16> %a, <16 x i16> undef, <16 x i32> <i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5>
   ret <16 x i16> %shuffle
 }
 
 define <4 x i64> @funcC(i64 %q) nounwind uwtable readnone ssp {
-; CHECK-LABEL: funcC:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vmovq %rdi, %xmm0
-; CHECK-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
-; CHECK-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
-; CHECK-NEXT:    retq
+; X86-LABEL: funcC:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    vbroadcastsd {{[0-9]+}}(%esp), %ymm0
+; X86-NEXT:    retl
+;
+; X64-LABEL: funcC:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    vmovq %rdi, %xmm0
+; X64-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
+; X64-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
+; X64-NEXT:    retq
 entry:
   %vecinit.i = insertelement <4 x i64> undef, i64 %q, i32 0
   %vecinit2.i = insertelement <4 x i64> %vecinit.i, i64 %q, i32 1
@@ -40,11 +46,16 @@ entry:
 }
 
 define <4 x double> @funcD(double %q) nounwind uwtable readnone ssp {
-; CHECK-LABEL: funcD:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vmovddup {{.*#+}} xmm0 = xmm0[0,0]
-; CHECK-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
-; CHECK-NEXT:    retq
+; X86-LABEL: funcD:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    vbroadcastsd {{[0-9]+}}(%esp), %ymm0
+; X86-NEXT:    retl
+;
+; X64-LABEL: funcD:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    vmovddup {{.*#+}} xmm0 = xmm0[0,0]
+; X64-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
+; X64-NEXT:    retq
 entry:
   %vecinit.i = insertelement <4 x double> undef, double %q, i32 0
   %vecinit2.i = insertelement <4 x double> %vecinit.i, double %q, i32 1
@@ -57,22 +68,39 @@ entry:
 ;   shuffle (scalar_to_vector (load (ptr + 4))), undef, <0, 0, 0, 0>
 ;
 define <8 x float> @funcE() nounwind {
-; CHECK-LABEL: funcE:
-; CHECK:       # %bb.0: # %allocas
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    # implicit-def: $ymm0
-; CHECK-NEXT:    jne .LBB4_2
-; CHECK-NEXT:  # %bb.1: # %load.i1247
-; CHECK-NEXT:    pushq %rbp
-; CHECK-NEXT:    movq %rsp, %rbp
-; CHECK-NEXT:    andq $-32, %rsp
-; CHECK-NEXT:    subq $1312, %rsp # imm = 0x520
-; CHECK-NEXT:    vbroadcastss {{[0-9]+}}(%rsp), %ymm0
-; CHECK-NEXT:    movq %rbp, %rsp
-; CHECK-NEXT:    popq %rbp
-; CHECK-NEXT:  .LBB4_2: # %__load_and_broadcast_32.exit1249
-; CHECK-NEXT:    retq
+; X86-LABEL: funcE:
+; X86:       # %bb.0: # %allocas
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    testb %al, %al
+; X86-NEXT:    # implicit-def: $ymm0
+; X86-NEXT:    jne .LBB4_2
+; X86-NEXT:  # %bb.1: # %load.i1247
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    movl %esp, %ebp
+; X86-NEXT:    andl $-32, %esp
+; X86-NEXT:    subl $1312, %esp # imm = 0x520
+; X86-NEXT:    vbroadcastss {{[0-9]+}}(%esp), %ymm0
+; X86-NEXT:    movl %ebp, %esp
+; X86-NEXT:    popl %ebp
+; X86-NEXT:  .LBB4_2: # %__load_and_broadcast_32.exit1249
+; X86-NEXT:    retl
+;
+; X64-LABEL: funcE:
+; X64:       # %bb.0: # %allocas
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    testb %al, %al
+; X64-NEXT:    # implicit-def: $ymm0
+; X64-NEXT:    jne .LBB4_2
+; X64-NEXT:  # %bb.1: # %load.i1247
+; X64-NEXT:    pushq %rbp
+; X64-NEXT:    movq %rsp, %rbp
+; X64-NEXT:    andq $-32, %rsp
+; X64-NEXT:    subq $1312, %rsp # imm = 0x520
+; X64-NEXT:    vbroadcastss {{[0-9]+}}(%rsp), %ymm0
+; X64-NEXT:    movq %rbp, %rsp
+; X64-NEXT:    popq %rbp
+; X64-NEXT:  .LBB4_2: # %__load_and_broadcast_32.exit1249
+; X64-NEXT:    retq
 allocas:
   %udx495 = alloca [18 x [18 x float]], align 32
   br label %for_test505.preheader
@@ -98,12 +126,17 @@ __load_and_broadcast_32.exit1249:                 ; preds = %load.i1247, %for_ex
 }
 
 define <8 x float> @funcF(i32 %val) nounwind {
-; CHECK-LABEL: funcF:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmovd %edi, %xmm0
-; CHECK-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,0]
-; CHECK-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
-; CHECK-NEXT:    retq
+; X86-LABEL: funcF:
+; X86:       # %bb.0:
+; X86-NEXT:    vbroadcastss {{[0-9]+}}(%esp), %ymm0
+; X86-NEXT:    retl
+;
+; X64-LABEL: funcF:
+; X64:       # %bb.0:
+; X64-NEXT:    vmovd %edi, %xmm0
+; X64-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,0]
+; X64-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
+; X64-NEXT:    retq
   %ret6 = insertelement <8 x i32> undef, i32 %val, i32 6
   %ret7 = insertelement <8 x i32> %ret6, i32 %val, i32 7
   %tmp = bitcast <8 x i32> %ret7 to <8 x float>
@@ -115,7 +148,7 @@ define <8 x float> @funcG(<8 x float> %a) nounwind uwtable readnone ssp {
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; CHECK-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
 entry:
   %shuffle = shufflevector <8 x float> %a, <8 x float> undef, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
   ret <8 x float> %shuffle
@@ -126,47 +159,71 @@ define <8 x float> @funcH(<8 x float> %a) nounwind uwtable readnone ssp {
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vpermilps {{.*#+}} ymm0 = ymm0[1,1,1,1,5,5,5,5]
 ; CHECK-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,2,3]
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
 entry:
   %shuffle = shufflevector <8 x float> %a, <8 x float> undef, <8 x i32> <i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5>
   ret <8 x float> %shuffle
 }
 
 define <2 x double> @splat_load_2f64_11(<2 x double>* %ptr) {
-; CHECK-LABEL: splat_load_2f64_11:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmovddup {{.*#+}} xmm0 = mem[0,0]
-; CHECK-NEXT:    retq
+; X86-LABEL: splat_load_2f64_11:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    vmovddup {{.*#+}} xmm0 = mem[0,0]
+; X86-NEXT:    retl
+;
+; X64-LABEL: splat_load_2f64_11:
+; X64:       # %bb.0:
+; X64-NEXT:    vmovddup {{.*#+}} xmm0 = mem[0,0]
+; X64-NEXT:    retq
   %x = load <2 x double>, <2 x double>* %ptr
   %x1 = shufflevector <2 x double> %x, <2 x double> undef, <2 x i32> <i32 1, i32 1>
   ret <2 x double> %x1
 }
 
 define <4 x double> @splat_load_4f64_2222(<4 x double>* %ptr) {
-; CHECK-LABEL: splat_load_4f64_2222:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vbroadcastsd 16(%rdi), %ymm0
-; CHECK-NEXT:    retq
+; X86-LABEL: splat_load_4f64_2222:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    vbroadcastsd 16(%eax), %ymm0
+; X86-NEXT:    retl
+;
+; X64-LABEL: splat_load_4f64_2222:
+; X64:       # %bb.0:
+; X64-NEXT:    vbroadcastsd 16(%rdi), %ymm0
+; X64-NEXT:    retq
   %x = load <4 x double>, <4 x double>* %ptr
   %x1 = shufflevector <4 x double> %x, <4 x double> undef, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
   ret <4 x double> %x1
 }
 
 define <4 x float> @splat_load_4f32_0000(<4 x float>* %ptr) {
-; CHECK-LABEL: splat_load_4f32_0000:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vbroadcastss (%rdi), %xmm0
-; CHECK-NEXT:    retq
+; X86-LABEL: splat_load_4f32_0000:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    vbroadcastss (%eax), %xmm0
+; X86-NEXT:    retl
+;
+; X64-LABEL: splat_load_4f32_0000:
+; X64:       # %bb.0:
+; X64-NEXT:    vbroadcastss (%rdi), %xmm0
+; X64-NEXT:    retq
   %x = load <4 x float>, <4 x float>* %ptr
   %x1 = shufflevector <4 x float> %x, <4 x float> undef, <4 x i32> <i32 0, i32 0, i32 0, i32 0>
   ret <4 x float> %x1
 }
 
 define <8 x float> @splat_load_8f32_77777777(<8 x float>* %ptr) {
-; CHECK-LABEL: splat_load_8f32_77777777:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vbroadcastss 28(%rdi), %ymm0
-; CHECK-NEXT:    retq
+; X86-LABEL: splat_load_8f32_77777777:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    vbroadcastss 28(%eax), %ymm0
+; X86-NEXT:    retl
+;
+; X64-LABEL: splat_load_8f32_77777777:
+; X64:       # %bb.0:
+; X64-NEXT:    vbroadcastss 28(%rdi), %ymm0
+; X64-NEXT:    retq
   %x = load <8 x float>, <8 x float>* %ptr
   %x1 = shufflevector <8 x float> %x, <8 x float> undef, <8 x i32> <i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7>
   ret <8 x float> %x1
diff --git a/llvm/test/CodeGen/X86/avx-unpack.ll b/llvm/test/CodeGen/X86/avx-unpack.ll
index 8c7098097..30b42a2d8 100644
--- a/llvm/test/CodeGen/X86/avx-unpack.ll
+++ b/llvm/test/CodeGen/X86/avx-unpack.ll
@@ -73,11 +73,9 @@ define <8 x float> @unpackhips_not(<8 x float> %src1, <8 x float> %src2) nounwin
 define <4 x double> @unpackhipd_not(<4 x double> %src1, <4 x double> %src2) nounwind uwtable readnone ssp {
 ; CHECK-LABEL: unpackhipd_not:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vextractf128 $1, %ymm1, %xmm1
-; CHECK-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; CHECK-NEXT:    vunpckhpd {{.*#+}} xmm2 = xmm0[1],xmm1[1]
-; CHECK-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; CHECK-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; CHECK-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm1[2,3,2,3]
+; CHECK-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,2,3]
+; CHECK-NEXT:    vshufpd {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[3],ymm1[3]
 ; CHECK-NEXT:    retq
   %shuffle.i = shufflevector <4 x double> %src1, <4 x double> %src2, <4 x i32> <i32 2, i32 6, i32 3, i32 7>
   ret <4 x double> %shuffle.i
diff --git a/llvm/test/CodeGen/X86/avx-vbroadcast.ll b/llvm/test/CodeGen/X86/avx-vbroadcast.ll
index d3a261e9e..609c02eee 100644
--- a/llvm/test/CodeGen/X86/avx-vbroadcast.ll
+++ b/llvm/test/CodeGen/X86/avx-vbroadcast.ll
@@ -159,18 +159,14 @@ define <4 x double> @C2(double* %ptr, double* %ptr2) nounwind uwtable readnone s
 ; X32:       ## %bb.0: ## %entry
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; X32-NEXT:    vmovsd %xmm0, (%eax)
-; X32-NEXT:    vmovddup {{.*#+}} xmm0 = xmm0[0,0]
-; X32-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
+; X32-NEXT:    vbroadcastsd (%ecx), %ymm0
+; X32-NEXT:    vmovlps %xmm0, (%eax)
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: C2:
 ; X64:       ## %bb.0: ## %entry
-; X64-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; X64-NEXT:    vmovsd %xmm0, (%rsi)
-; X64-NEXT:    vmovddup {{.*#+}} xmm0 = xmm0[0,0]
-; X64-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
+; X64-NEXT:    vbroadcastsd (%rdi), %ymm0
+; X64-NEXT:    vmovlps %xmm0, (%rsi)
 ; X64-NEXT:    retq
 entry:
   %q = load double, double* %ptr, align 8
@@ -231,18 +227,14 @@ define <8 x float> @D3(float* %ptr, float* %ptr2) nounwind uwtable readnone ssp
 ; X32:       ## %bb.0: ## %entry
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X32-NEXT:    vbroadcastss (%ecx), %ymm0
 ; X32-NEXT:    vmovss %xmm0, (%eax)
-; X32-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,0,0,0]
-; X32-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: D3:
 ; X64:       ## %bb.0: ## %entry
-; X64-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X64-NEXT:    vbroadcastss (%rdi), %ymm0
 ; X64-NEXT:    vmovss %xmm0, (%rsi)
-; X64-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,0,0,0]
-; X64-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; X64-NEXT:    retq
 entry:
   %q = load float, float* %ptr, align 4
@@ -285,16 +277,14 @@ define <4 x float> @e2(float* %ptr, float* %ptr2) nounwind uwtable readnone ssp
 ; X32:       ## %bb.0: ## %entry
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X32-NEXT:    vbroadcastss (%ecx), %xmm0
 ; X32-NEXT:    vmovss %xmm0, (%eax)
-; X32-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: e2:
 ; X64:       ## %bb.0: ## %entry
-; X64-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X64-NEXT:    vbroadcastss (%rdi), %xmm0
 ; X64-NEXT:    vmovss %xmm0, (%rsi)
-; X64-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; X64-NEXT:    retq
 entry:
   %q = load float, float* %ptr, align 4
@@ -669,16 +659,14 @@ define <2 x double> @I2(double* %ptr, double* %ptr2) nounwind uwtable readnone s
 ; X32:       ## %bb.0: ## %entry
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; X32-NEXT:    vmovsd %xmm0, (%eax)
-; X32-NEXT:    vmovddup {{.*#+}} xmm0 = xmm0[0,0]
+; X32-NEXT:    vmovddup {{.*#+}} xmm0 = mem[0,0]
+; X32-NEXT:    vmovlps %xmm0, (%eax)
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: I2:
 ; X64:       ## %bb.0: ## %entry
-; X64-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; X64-NEXT:    vmovsd %xmm0, (%rsi)
-; X64-NEXT:    vmovddup {{.*#+}} xmm0 = xmm0[0,0]
+; X64-NEXT:    vmovddup {{.*#+}} xmm0 = mem[0,0]
+; X64-NEXT:    vmovlps %xmm0, (%rsi)
 ; X64-NEXT:    retq
 entry:
   %q = load double, double* %ptr, align 4
@@ -884,7 +872,6 @@ define void @broadcast_v16i32(i32* %a, <16 x i32>* %b) {
 
 ;
 ; Broadcast scale factor for xyz vector - slp will have vectorized xy.
-; FIXME: Load as a broadcast and then use the scalar 0'th element.
 ;
 define double @broadcast_scale_xyz(double* nocapture readonly, double* nocapture readonly) nounwind {
 ; X32-LABEL: broadcast_scale_xyz:
@@ -892,9 +879,8 @@ define double @broadcast_scale_xyz(double* nocapture readonly, double* nocapture
 ; X32-NEXT:    subl $12, %esp
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; X32-NEXT:    vmovddup {{.*#+}} xmm1 = xmm0[0,0]
-; X32-NEXT:    vmulpd (%eax), %xmm1, %xmm1
+; X32-NEXT:    vmovddup {{.*#+}} xmm0 = mem[0,0]
+; X32-NEXT:    vmulpd (%eax), %xmm0, %xmm1
 ; X32-NEXT:    vmulsd 16(%eax), %xmm0, %xmm0
 ; X32-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm1[1,0]
 ; X32-NEXT:    vaddsd %xmm2, %xmm1, %xmm1
@@ -906,9 +892,8 @@ define double @broadcast_scale_xyz(double* nocapture readonly, double* nocapture
 ;
 ; X64-LABEL: broadcast_scale_xyz:
 ; X64:       ## %bb.0:
-; X64-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; X64-NEXT:    vmovddup {{.*#+}} xmm1 = xmm0[0,0]
-; X64-NEXT:    vmulpd (%rsi), %xmm1, %xmm1
+; X64-NEXT:    vmovddup {{.*#+}} xmm0 = mem[0,0]
+; X64-NEXT:    vmulpd (%rsi), %xmm0, %xmm1
 ; X64-NEXT:    vmulsd 16(%rsi), %xmm0, %xmm0
 ; X64-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm1[1,0]
 ; X64-NEXT:    vaddsd %xmm2, %xmm1, %xmm1
diff --git a/llvm/test/CodeGen/X86/avx-vpclmulqdq.ll b/llvm/test/CodeGen/X86/avx-vpclmulqdq.ll
index 2447ff090..4c791e080 100644
--- a/llvm/test/CodeGen/X86/avx-vpclmulqdq.ll
+++ b/llvm/test/CodeGen/X86/avx-vpclmulqdq.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=avx,vpclmulqdq -show-mc-encoding | FileCheck %s --check-prefix=AVX_VPCLMULQDQ
 
 ; Check for vpclmulqdq
diff --git a/llvm/test/CodeGen/X86/avx-vzeroupper.ll b/llvm/test/CodeGen/X86/avx-vzeroupper.ll
index 1abeb76de..4ae9b3144 100644
--- a/llvm/test/CodeGen/X86/avx-vzeroupper.ll
+++ b/llvm/test/CodeGen/X86/avx-vzeroupper.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -x86-use-vzeroupper -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=VZ --check-prefix=AVX
 ; RUN: llc < %s -x86-use-vzeroupper -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefix=ALL --check-prefix=VZ --check-prefix=AVX512
-; RUN: llc < %s -x86-use-vzeroupper -mtriple=x86_64-unknown-unknown -mattr=+avx,+fast-partial-ymm-or-zmm-write | FileCheck %s --check-prefix=ALL --check-prefix=NO-VZ --check-prefix=FAST-ymm-zmm
+; RUN: llc < %s -x86-use-vzeroupper -mtriple=x86_64-unknown-unknown -mattr=+avx,-vzeroupper | FileCheck %s --check-prefix=ALL --check-prefix=NO-VZ --check-prefix=DISABLE-VZ
 ; RUN: llc < %s -x86-use-vzeroupper -mtriple=x86_64-unknown-unknown -mcpu=bdver2 | FileCheck %s --check-prefix=ALL --check-prefix=NO-VZ --check-prefix=BDVER2
 ; RUN: llc < %s -x86-use-vzeroupper -mtriple=x86_64-unknown-unknown -mcpu=btver2 | FileCheck %s --check-prefix=ALL --check-prefix=NO-VZ --check-prefix=BTVER2
 
@@ -44,18 +44,18 @@ define <8 x float> @test01(<4 x float> %a, <4 x float> %b, <8 x float> %c) nounw
 ; VZ-NEXT:    addq $56, %rsp
 ; VZ-NEXT:    retq
 ;
-; FAST-ymm-zmm-LABEL: test01:
-; FAST-ymm-zmm:       # %bb.0:
-; FAST-ymm-zmm-NEXT:    subq $56, %rsp
-; FAST-ymm-zmm-NEXT:    vmovups %ymm2, (%rsp) # 32-byte Spill
-; FAST-ymm-zmm-NEXT:    vmovaps {{.*}}(%rip), %xmm0
-; FAST-ymm-zmm-NEXT:    callq do_sse
-; FAST-ymm-zmm-NEXT:    vmovaps %xmm0, {{.*}}(%rip)
-; FAST-ymm-zmm-NEXT:    callq do_sse
-; FAST-ymm-zmm-NEXT:    vmovaps %xmm0, {{.*}}(%rip)
-; FAST-ymm-zmm-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
-; FAST-ymm-zmm-NEXT:    addq $56, %rsp
-; FAST-ymm-zmm-NEXT:    retq
+; DISABLE-VZ-LABEL: test01:
+; DISABLE-VZ:       # %bb.0:
+; DISABLE-VZ-NEXT:    subq $56, %rsp
+; DISABLE-VZ-NEXT:    vmovups %ymm2, (%rsp) # 32-byte Spill
+; DISABLE-VZ-NEXT:    vmovaps {{.*}}(%rip), %xmm0
+; DISABLE-VZ-NEXT:    callq do_sse
+; DISABLE-VZ-NEXT:    vmovaps %xmm0, {{.*}}(%rip)
+; DISABLE-VZ-NEXT:    callq do_sse
+; DISABLE-VZ-NEXT:    vmovaps %xmm0, {{.*}}(%rip)
+; DISABLE-VZ-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
+; DISABLE-VZ-NEXT:    addq $56, %rsp
+; DISABLE-VZ-NEXT:    retq
 ;
 ; BDVER2-LABEL: test01:
 ; BDVER2:       # %bb.0:
@@ -83,6 +83,7 @@ define <8 x float> @test01(<4 x float> %a, <4 x float> %b, <8 x float> %c) nounw
 ; BTVER2-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
 ; BTVER2-NEXT:    addq $56, %rsp
 ; BTVER2-NEXT:    retq
+; DISABLE-VZ       # %bb.0:
   %tmp = load <4 x float>, <4 x float>* @x, align 16
   %call = tail call <4 x float> @do_sse(<4 x float> %tmp) nounwind
   store <4 x float> %call, <4 x float>* @x, align 16
@@ -100,10 +101,10 @@ define <4 x float> @test02(<8 x float> %a, <8 x float> %b) nounwind {
 ; VZ-NEXT:    vzeroupper
 ; VZ-NEXT:    jmp do_sse # TAILCALL
 ;
-; FAST-ymm-zmm-LABEL: test02:
-; FAST-ymm-zmm:       # %bb.0:
-; FAST-ymm-zmm-NEXT:    vaddps %xmm1, %xmm0, %xmm0
-; FAST-ymm-zmm-NEXT:    jmp do_sse # TAILCALL
+; DISABLE-VZ-LABEL: test02:
+; DISABLE-VZ:       # %bb.0:
+; DISABLE-VZ-NEXT:    vaddps %xmm1, %xmm0, %xmm0
+; DISABLE-VZ-NEXT:    jmp do_sse # TAILCALL
 ;
 ; BDVER2-LABEL: test02:
 ; BDVER2:       # %bb.0:
@@ -154,34 +155,34 @@ define <4 x float> @test03(<4 x float> %a, <4 x float> %b) nounwind {
 ; VZ-NEXT:    popq %rbx
 ; VZ-NEXT:    retq
 ;
-; FAST-ymm-zmm-LABEL: test03:
-; FAST-ymm-zmm:       # %bb.0: # %entry
-; FAST-ymm-zmm-NEXT:    pushq %rbx
-; FAST-ymm-zmm-NEXT:    subq $16, %rsp
-; FAST-ymm-zmm-NEXT:    vaddps %xmm1, %xmm0, %xmm0
-; FAST-ymm-zmm-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
-; FAST-ymm-zmm-NEXT:    .p2align 4, 0x90
-; FAST-ymm-zmm-NEXT:  .LBB3_1: # %while.cond
-; FAST-ymm-zmm-NEXT:    # =>This Inner Loop Header: Depth=1
-; FAST-ymm-zmm-NEXT:    callq foo
-; FAST-ymm-zmm-NEXT:    testl %eax, %eax
-; FAST-ymm-zmm-NEXT:    jne .LBB3_1
-; FAST-ymm-zmm-NEXT:  # %bb.2: # %for.body.preheader
-; FAST-ymm-zmm-NEXT:    movl $4, %ebx
-; FAST-ymm-zmm-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
-; FAST-ymm-zmm-NEXT:    .p2align 4, 0x90
-; FAST-ymm-zmm-NEXT:  .LBB3_3: # %for.body
-; FAST-ymm-zmm-NEXT:    # =>This Inner Loop Header: Depth=1
-; FAST-ymm-zmm-NEXT:    callq do_sse
-; FAST-ymm-zmm-NEXT:    callq do_sse
-; FAST-ymm-zmm-NEXT:    vmovaps g+{{.*}}(%rip), %xmm0
-; FAST-ymm-zmm-NEXT:    callq do_sse
-; FAST-ymm-zmm-NEXT:    decl %ebx
-; FAST-ymm-zmm-NEXT:    jne .LBB3_3
-; FAST-ymm-zmm-NEXT:  # %bb.4: # %for.end
-; FAST-ymm-zmm-NEXT:    addq $16, %rsp
-; FAST-ymm-zmm-NEXT:    popq %rbx
-; FAST-ymm-zmm-NEXT:    retq
+; DISABLE-VZ-LABEL: test03:
+; DISABLE-VZ:       # %bb.0: # %entry
+; DISABLE-VZ-NEXT:    pushq %rbx
+; DISABLE-VZ-NEXT:    subq $16, %rsp
+; DISABLE-VZ-NEXT:    vaddps %xmm1, %xmm0, %xmm0
+; DISABLE-VZ-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
+; DISABLE-VZ-NEXT:    .p2align 4, 0x90
+; DISABLE-VZ-NEXT:  .LBB3_1: # %while.cond
+; DISABLE-VZ-NEXT:    # =>This Inner Loop Header: Depth=1
+; DISABLE-VZ-NEXT:    callq foo
+; DISABLE-VZ-NEXT:    testl %eax, %eax
+; DISABLE-VZ-NEXT:    jne .LBB3_1
+; DISABLE-VZ-NEXT:  # %bb.2: # %for.body.preheader
+; DISABLE-VZ-NEXT:    movl $4, %ebx
+; DISABLE-VZ-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
+; DISABLE-VZ-NEXT:    .p2align 4, 0x90
+; DISABLE-VZ-NEXT:  .LBB3_3: # %for.body
+; DISABLE-VZ-NEXT:    # =>This Inner Loop Header: Depth=1
+; DISABLE-VZ-NEXT:    callq do_sse
+; DISABLE-VZ-NEXT:    callq do_sse
+; DISABLE-VZ-NEXT:    vmovaps g+{{.*}}(%rip), %xmm0
+; DISABLE-VZ-NEXT:    callq do_sse
+; DISABLE-VZ-NEXT:    decl %ebx
+; DISABLE-VZ-NEXT:    jne .LBB3_3
+; DISABLE-VZ-NEXT:  # %bb.4: # %for.end
+; DISABLE-VZ-NEXT:    addq $16, %rsp
+; DISABLE-VZ-NEXT:    popq %rbx
+; DISABLE-VZ-NEXT:    retq
 ;
 ; BDVER2-LABEL: test03:
 ; BDVER2:       # %bb.0: # %entry
@@ -279,15 +280,15 @@ define <4 x float> @test04(<4 x float> %a, <4 x float> %b) nounwind {
 ; VZ-NEXT:    vzeroupper
 ; VZ-NEXT:    retq
 ;
-; FAST-ymm-zmm-LABEL: test04:
-; FAST-ymm-zmm:       # %bb.0:
-; FAST-ymm-zmm-NEXT:    pushq %rax
-; FAST-ymm-zmm-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
-; FAST-ymm-zmm-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; FAST-ymm-zmm-NEXT:    callq do_avx
-; FAST-ymm-zmm-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
-; FAST-ymm-zmm-NEXT:    popq %rax
-; FAST-ymm-zmm-NEXT:    retq
+; DISABLE-VZ-LABEL: test04:
+; DISABLE-VZ:       # %bb.0:
+; DISABLE-VZ-NEXT:    pushq %rax
+; DISABLE-VZ-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; DISABLE-VZ-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; DISABLE-VZ-NEXT:    callq do_avx
+; DISABLE-VZ-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; DISABLE-VZ-NEXT:    popq %rax
+; DISABLE-VZ-NEXT:    retq
 ;
 ; BDVER2-LABEL: test04:
 ; BDVER2:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/avx2-conversions.ll b/llvm/test/CodeGen/X86/avx2-conversions.ll
index 070959eb7..b424873c9 100644
--- a/llvm/test/CodeGen/X86/avx2-conversions.ll
+++ b/llvm/test/CodeGen/X86/avx2-conversions.ll
@@ -117,14 +117,12 @@ define <8 x i32> @zext8(<8 x i16> %A) nounwind {
 define <8 x i32> @zext_8i8_8i32(<8 x i8> %A) nounwind {
 ; X32-LABEL: zext_8i8_8i32:
 ; X32:       # %bb.0:
-; X32-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
-; X32-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; X32-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: zext_8i8_8i32:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; X64-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; X64-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
 ; X64-NEXT:    retq
   %B = zext <8 x i8> %A to <8 x i32>
   ret <8 x i32>%B
diff --git a/llvm/test/CodeGen/X86/avx2-gather.ll b/llvm/test/CodeGen/X86/avx2-gather.ll
index 23a8c9ef8..cf4a624eb 100644
--- a/llvm/test/CodeGen/X86/avx2-gather.ll
+++ b/llvm/test/CodeGen/X86/avx2-gather.ll
@@ -124,16 +124,16 @@ define <2 x double> @test_mm_i32gather_pd(double *%a0, <2 x i64> %a1) {
 ; X32-LABEL: test_mm_i32gather_pd:
 ; X32:       # %bb.0:
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
 ; X32-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; X32-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
 ; X32-NEXT:    vgatherdpd %xmm2, (%eax,%xmm0,2), %xmm1
 ; X32-NEXT:    vmovapd %xmm1, %xmm0
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: test_mm_i32gather_pd:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
 ; X64-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; X64-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
 ; X64-NEXT:    vgatherdpd %xmm2, (%rdi,%xmm0,2), %xmm1
 ; X64-NEXT:    vmovapd %xmm1, %xmm0
 ; X64-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/avx2-intrinsics-x86.ll b/llvm/test/CodeGen/X86/avx2-intrinsics-x86.ll
index aa68cfd70..5721c6998 100644
--- a/llvm/test/CodeGen/X86/avx2-intrinsics-x86.ll
+++ b/llvm/test/CodeGen/X86/avx2-intrinsics-x86.ll
@@ -2192,3 +2192,35 @@ define <8 x float>  @test_gather_mask(<8 x float> %a0, float* %a, <8 x i32> %idx
 
   ret <8 x float> %res
 }
+
+define <2 x i64> @test_mask_demanded_bits(<2 x i64> %a0, i8* %a1, <2 x i64> %idx, <2 x i1> %mask) {
+; X86-AVX-LABEL: test_mask_demanded_bits:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-AVX-NEXT:    vpsllq $63, %xmm2, %xmm2 # encoding: [0xc5,0xe9,0x73,0xf2,0x3f]
+; X86-AVX-NEXT:    vpgatherqq %xmm2, (%eax,%xmm1,2), %xmm0 # encoding: [0xc4,0xe2,0xe9,0x91,0x04,0x48]
+; X86-AVX-NEXT:    retl # encoding: [0xc3]
+;
+; X86-AVX512VL-LABEL: test_mask_demanded_bits:
+; X86-AVX512VL:       # %bb.0:
+; X86-AVX512VL-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-AVX512VL-NEXT:    vpsllq $63, %xmm2, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0x73,0xf2,0x3f]
+; X86-AVX512VL-NEXT:    vpgatherqq %xmm2, (%eax,%xmm1,2), %xmm0 # encoding: [0xc4,0xe2,0xe9,0x91,0x04,0x48]
+; X86-AVX512VL-NEXT:    retl # encoding: [0xc3]
+;
+; X64-AVX-LABEL: test_mask_demanded_bits:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    vpsllq $63, %xmm2, %xmm2 # encoding: [0xc5,0xe9,0x73,0xf2,0x3f]
+; X64-AVX-NEXT:    vpgatherqq %xmm2, (%rdi,%xmm1,2), %xmm0 # encoding: [0xc4,0xe2,0xe9,0x91,0x04,0x4f]
+; X64-AVX-NEXT:    retq # encoding: [0xc3]
+;
+; X64-AVX512VL-LABEL: test_mask_demanded_bits:
+; X64-AVX512VL:       # %bb.0:
+; X64-AVX512VL-NEXT:    vpsllq $63, %xmm2, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0x73,0xf2,0x3f]
+; X64-AVX512VL-NEXT:    vpgatherqq %xmm2, (%rdi,%xmm1,2), %xmm0 # encoding: [0xc4,0xe2,0xe9,0x91,0x04,0x4f]
+; X64-AVX512VL-NEXT:    retq # encoding: [0xc3]
+  %mask1 = sext <2 x i1> %mask to <2 x i64>
+  %res = call <2 x i64> @llvm.x86.avx2.gather.q.q(<2 x i64> %a0,
+                            i8* %a1, <2 x i64> %idx, <2 x i64> %mask1, i8 2) ;
+  ret <2 x i64> %res
+}
diff --git a/llvm/test/CodeGen/X86/avx2-masked-gather.ll b/llvm/test/CodeGen/X86/avx2-masked-gather.ll
index f220c7959..1d4e6a552 100644
--- a/llvm/test/CodeGen/X86/avx2-masked-gather.ll
+++ b/llvm/test/CodeGen/X86/avx2-masked-gather.ll
@@ -9,44 +9,44 @@ declare <2 x i32> @llvm.masked.gather.v2i32(<2 x i32*> %ptrs, i32 %align, <2 x i
 define <2 x i32> @masked_gather_v2i32(<2 x i32*>* %ptr, <2 x i1> %masks, <2 x i32> %passthro) {
 ; X86-LABEL: masked_gather_v2i32:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero
-; X86-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
 ; X86-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,2],zero,zero
 ; X86-NEXT:    vpslld $31, %xmm0, %xmm0
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero
 ; X86-NEXT:    vpgatherdd %xmm0, (,%xmm2), %xmm1
-; X86-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero
+; X86-NEXT:    vmovdqa %xmm1, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: masked_gather_v2i32:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    vmovdqa (%rdi), %xmm2
-; X64-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
 ; X64-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; X64-NEXT:    vpslld $31, %xmm0, %xmm0
 ; X64-NEXT:    vpgatherqd %xmm0, (,%xmm2), %xmm1
-; X64-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero
+; X64-NEXT:    vmovdqa %xmm1, %xmm0
 ; X64-NEXT:    retq
 ;
 ; NOGATHER-LABEL: masked_gather_v2i32:
 ; NOGATHER:       # %bb.0: # %entry
 ; NOGATHER-NEXT:    vmovdqa (%rdi), %xmm2
-; NOGATHER-NEXT:    vpextrb $0, %xmm0, %eax
-; NOGATHER-NEXT:    testb $1, %al
-; NOGATHER-NEXT:    je .LBB0_2
-; NOGATHER-NEXT:  # %bb.1: # %cond.load
-; NOGATHER-NEXT:    vmovq %xmm2, %rax
-; NOGATHER-NEXT:    movl (%rax), %eax
-; NOGATHER-NEXT:    vpinsrq $0, %rax, %xmm1, %xmm1
-; NOGATHER-NEXT:  .LBB0_2: # %else
-; NOGATHER-NEXT:    vpextrb $8, %xmm0, %eax
+; NOGATHER-NEXT:    vpsllq $63, %xmm0, %xmm0
+; NOGATHER-NEXT:    vmovmskpd %xmm0, %eax
 ; NOGATHER-NEXT:    testb $1, %al
+; NOGATHER-NEXT:    jne .LBB0_1
+; NOGATHER-NEXT:  # %bb.2: # %else
+; NOGATHER-NEXT:    testb $2, %al
+; NOGATHER-NEXT:    jne .LBB0_3
+; NOGATHER-NEXT:  .LBB0_4: # %else2
+; NOGATHER-NEXT:    vmovdqa %xmm1, %xmm0
+; NOGATHER-NEXT:    retq
+; NOGATHER-NEXT:  .LBB0_1: # %cond.load
+; NOGATHER-NEXT:    vmovq %xmm2, %rcx
+; NOGATHER-NEXT:    vpinsrd $0, (%rcx), %xmm1, %xmm1
+; NOGATHER-NEXT:    testb $2, %al
 ; NOGATHER-NEXT:    je .LBB0_4
-; NOGATHER-NEXT:  # %bb.3: # %cond.load1
+; NOGATHER-NEXT:  .LBB0_3: # %cond.load1
 ; NOGATHER-NEXT:    vpextrq $1, %xmm2, %rax
-; NOGATHER-NEXT:    movl (%rax), %eax
-; NOGATHER-NEXT:    vpinsrq $1, %rax, %xmm1, %xmm1
-; NOGATHER-NEXT:  .LBB0_4: # %else2
+; NOGATHER-NEXT:    vpinsrd $1, (%rax), %xmm1, %xmm1
 ; NOGATHER-NEXT:    vmovdqa %xmm1, %xmm0
 ; NOGATHER-NEXT:    retq
 entry:
@@ -58,11 +58,10 @@ entry:
 define <4 x i32> @masked_gather_v2i32_concat(<2 x i32*>* %ptr, <2 x i1> %masks, <2 x i32> %passthro) {
 ; X86-LABEL: masked_gather_v2i32_concat:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero
-; X86-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
 ; X86-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,2],zero,zero
 ; X86-NEXT:    vpslld $31, %xmm0, %xmm0
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero
 ; X86-NEXT:    vpgatherdd %xmm0, (,%xmm2), %xmm1
 ; X86-NEXT:    vmovdqa %xmm1, %xmm0
 ; X86-NEXT:    retl
@@ -70,7 +69,6 @@ define <4 x i32> @masked_gather_v2i32_concat(<2 x i32*>* %ptr, <2 x i1> %masks,
 ; X64-LABEL: masked_gather_v2i32_concat:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    vmovdqa (%rdi), %xmm2
-; X64-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
 ; X64-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; X64-NEXT:    vpslld $31, %xmm0, %xmm0
 ; X64-NEXT:    vpgatherqd %xmm0, (,%xmm2), %xmm1
@@ -80,23 +78,25 @@ define <4 x i32> @masked_gather_v2i32_concat(<2 x i32*>* %ptr, <2 x i1> %masks,
 ; NOGATHER-LABEL: masked_gather_v2i32_concat:
 ; NOGATHER:       # %bb.0: # %entry
 ; NOGATHER-NEXT:    vmovdqa (%rdi), %xmm2
-; NOGATHER-NEXT:    vpextrb $0, %xmm0, %eax
-; NOGATHER-NEXT:    testb $1, %al
-; NOGATHER-NEXT:    je .LBB1_2
-; NOGATHER-NEXT:  # %bb.1: # %cond.load
-; NOGATHER-NEXT:    vmovq %xmm2, %rax
-; NOGATHER-NEXT:    movl (%rax), %eax
-; NOGATHER-NEXT:    vpinsrq $0, %rax, %xmm1, %xmm1
-; NOGATHER-NEXT:  .LBB1_2: # %else
-; NOGATHER-NEXT:    vpextrb $8, %xmm0, %eax
+; NOGATHER-NEXT:    vpsllq $63, %xmm0, %xmm0
+; NOGATHER-NEXT:    vmovmskpd %xmm0, %eax
 ; NOGATHER-NEXT:    testb $1, %al
+; NOGATHER-NEXT:    jne .LBB1_1
+; NOGATHER-NEXT:  # %bb.2: # %else
+; NOGATHER-NEXT:    testb $2, %al
+; NOGATHER-NEXT:    jne .LBB1_3
+; NOGATHER-NEXT:  .LBB1_4: # %else2
+; NOGATHER-NEXT:    vmovdqa %xmm1, %xmm0
+; NOGATHER-NEXT:    retq
+; NOGATHER-NEXT:  .LBB1_1: # %cond.load
+; NOGATHER-NEXT:    vmovq %xmm2, %rcx
+; NOGATHER-NEXT:    vpinsrd $0, (%rcx), %xmm1, %xmm1
+; NOGATHER-NEXT:    testb $2, %al
 ; NOGATHER-NEXT:    je .LBB1_4
-; NOGATHER-NEXT:  # %bb.3: # %cond.load1
+; NOGATHER-NEXT:  .LBB1_3: # %cond.load1
 ; NOGATHER-NEXT:    vpextrq $1, %xmm2, %rax
-; NOGATHER-NEXT:    movl (%rax), %eax
-; NOGATHER-NEXT:    vpinsrq $1, %rax, %xmm1, %xmm1
-; NOGATHER-NEXT:  .LBB1_4: # %else2
-; NOGATHER-NEXT:    vpshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
+; NOGATHER-NEXT:    vpinsrd $1, (%rax), %xmm1, %xmm1
+; NOGATHER-NEXT:    vmovdqa %xmm1, %xmm0
 ; NOGATHER-NEXT:    retq
 entry:
   %ld  = load <2 x i32*>, <2 x i32*>* %ptr
@@ -130,21 +130,25 @@ define <2 x float> @masked_gather_v2float(<2 x float*>* %ptr, <2 x i1> %masks, <
 ; NOGATHER-LABEL: masked_gather_v2float:
 ; NOGATHER:       # %bb.0: # %entry
 ; NOGATHER-NEXT:    vmovdqa (%rdi), %xmm2
-; NOGATHER-NEXT:    vpextrb $0, %xmm0, %eax
-; NOGATHER-NEXT:    testb $1, %al
-; NOGATHER-NEXT:    je .LBB2_2
-; NOGATHER-NEXT:  # %bb.1: # %cond.load
-; NOGATHER-NEXT:    vmovq %xmm2, %rax
-; NOGATHER-NEXT:    vmovss {{.*#+}} xmm3 = mem[0],zero,zero,zero
-; NOGATHER-NEXT:    vblendps {{.*#+}} xmm1 = xmm3[0],xmm1[1,2,3]
-; NOGATHER-NEXT:  .LBB2_2: # %else
-; NOGATHER-NEXT:    vpextrb $8, %xmm0, %eax
+; NOGATHER-NEXT:    vpsllq $63, %xmm0, %xmm0
+; NOGATHER-NEXT:    vmovmskpd %xmm0, %eax
 ; NOGATHER-NEXT:    testb $1, %al
+; NOGATHER-NEXT:    jne .LBB2_1
+; NOGATHER-NEXT:  # %bb.2: # %else
+; NOGATHER-NEXT:    testb $2, %al
+; NOGATHER-NEXT:    jne .LBB2_3
+; NOGATHER-NEXT:  .LBB2_4: # %else2
+; NOGATHER-NEXT:    vmovaps %xmm1, %xmm0
+; NOGATHER-NEXT:    retq
+; NOGATHER-NEXT:  .LBB2_1: # %cond.load
+; NOGATHER-NEXT:    vmovq %xmm2, %rcx
+; NOGATHER-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; NOGATHER-NEXT:    vblendps {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3]
+; NOGATHER-NEXT:    testb $2, %al
 ; NOGATHER-NEXT:    je .LBB2_4
-; NOGATHER-NEXT:  # %bb.3: # %cond.load1
+; NOGATHER-NEXT:  .LBB2_3: # %cond.load1
 ; NOGATHER-NEXT:    vpextrq $1, %xmm2, %rax
 ; NOGATHER-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[2,3]
-; NOGATHER-NEXT:  .LBB2_4: # %else2
 ; NOGATHER-NEXT:    vmovaps %xmm1, %xmm0
 ; NOGATHER-NEXT:    retq
 entry:
@@ -176,21 +180,25 @@ define <4 x float> @masked_gather_v2float_concat(<2 x float*>* %ptr, <2 x i1> %m
 ; NOGATHER-LABEL: masked_gather_v2float_concat:
 ; NOGATHER:       # %bb.0: # %entry
 ; NOGATHER-NEXT:    vmovdqa (%rdi), %xmm2
-; NOGATHER-NEXT:    vpextrb $0, %xmm0, %eax
-; NOGATHER-NEXT:    testb $1, %al
-; NOGATHER-NEXT:    je .LBB3_2
-; NOGATHER-NEXT:  # %bb.1: # %cond.load
-; NOGATHER-NEXT:    vmovq %xmm2, %rax
-; NOGATHER-NEXT:    vmovss {{.*#+}} xmm3 = mem[0],zero,zero,zero
-; NOGATHER-NEXT:    vblendps {{.*#+}} xmm1 = xmm3[0],xmm1[1,2,3]
-; NOGATHER-NEXT:  .LBB3_2: # %else
-; NOGATHER-NEXT:    vpextrb $8, %xmm0, %eax
+; NOGATHER-NEXT:    vpsllq $63, %xmm0, %xmm0
+; NOGATHER-NEXT:    vmovmskpd %xmm0, %eax
 ; NOGATHER-NEXT:    testb $1, %al
+; NOGATHER-NEXT:    jne .LBB3_1
+; NOGATHER-NEXT:  # %bb.2: # %else
+; NOGATHER-NEXT:    testb $2, %al
+; NOGATHER-NEXT:    jne .LBB3_3
+; NOGATHER-NEXT:  .LBB3_4: # %else2
+; NOGATHER-NEXT:    vmovaps %xmm1, %xmm0
+; NOGATHER-NEXT:    retq
+; NOGATHER-NEXT:  .LBB3_1: # %cond.load
+; NOGATHER-NEXT:    vmovq %xmm2, %rcx
+; NOGATHER-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; NOGATHER-NEXT:    vblendps {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3]
+; NOGATHER-NEXT:    testb $2, %al
 ; NOGATHER-NEXT:    je .LBB3_4
-; NOGATHER-NEXT:  # %bb.3: # %cond.load1
+; NOGATHER-NEXT:  .LBB3_3: # %cond.load1
 ; NOGATHER-NEXT:    vpextrq $1, %xmm2, %rax
 ; NOGATHER-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[2,3]
-; NOGATHER-NEXT:  .LBB3_4: # %else2
 ; NOGATHER-NEXT:    vmovaps %xmm1, %xmm0
 ; NOGATHER-NEXT:    retq
 entry:
@@ -221,35 +229,38 @@ define <4 x i32> @masked_gather_v4i32(<4 x i32*> %ptrs, <4 x i1> %masks, <4 x i3
 ;
 ; NOGATHER-LABEL: masked_gather_v4i32:
 ; NOGATHER:       # %bb.0: # %entry
-; NOGATHER-NEXT:    vpextrb $0, %xmm1, %eax
+; NOGATHER-NEXT:    vpslld $31, %xmm1, %xmm1
+; NOGATHER-NEXT:    vmovmskps %xmm1, %eax
 ; NOGATHER-NEXT:    testb $1, %al
 ; NOGATHER-NEXT:    je .LBB4_2
 ; NOGATHER-NEXT:  # %bb.1: # %cond.load
-; NOGATHER-NEXT:    vmovq %xmm0, %rax
-; NOGATHER-NEXT:    vpinsrd $0, (%rax), %xmm2, %xmm2
+; NOGATHER-NEXT:    vmovq %xmm0, %rcx
+; NOGATHER-NEXT:    vpinsrd $0, (%rcx), %xmm2, %xmm2
 ; NOGATHER-NEXT:  .LBB4_2: # %else
-; NOGATHER-NEXT:    vpextrb $4, %xmm1, %eax
-; NOGATHER-NEXT:    testb $1, %al
+; NOGATHER-NEXT:    testb $2, %al
 ; NOGATHER-NEXT:    je .LBB4_4
 ; NOGATHER-NEXT:  # %bb.3: # %cond.load1
-; NOGATHER-NEXT:    vpextrq $1, %xmm0, %rax
-; NOGATHER-NEXT:    vpinsrd $1, (%rax), %xmm2, %xmm2
+; NOGATHER-NEXT:    vpextrq $1, %xmm0, %rcx
+; NOGATHER-NEXT:    vpinsrd $1, (%rcx), %xmm2, %xmm2
 ; NOGATHER-NEXT:  .LBB4_4: # %else2
-; NOGATHER-NEXT:    vpextrb $8, %xmm1, %eax
 ; NOGATHER-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; NOGATHER-NEXT:    testb $1, %al
-; NOGATHER-NEXT:    je .LBB4_6
-; NOGATHER-NEXT:  # %bb.5: # %cond.load4
-; NOGATHER-NEXT:    vmovq %xmm0, %rax
-; NOGATHER-NEXT:    vpinsrd $2, (%rax), %xmm2, %xmm2
-; NOGATHER-NEXT:  .LBB4_6: # %else5
-; NOGATHER-NEXT:    vpextrb $12, %xmm1, %eax
-; NOGATHER-NEXT:    testb $1, %al
+; NOGATHER-NEXT:    testb $4, %al
+; NOGATHER-NEXT:    jne .LBB4_5
+; NOGATHER-NEXT:  # %bb.6: # %else5
+; NOGATHER-NEXT:    testb $8, %al
+; NOGATHER-NEXT:    jne .LBB4_7
+; NOGATHER-NEXT:  .LBB4_8: # %else8
+; NOGATHER-NEXT:    vmovdqa %xmm2, %xmm0
+; NOGATHER-NEXT:    vzeroupper
+; NOGATHER-NEXT:    retq
+; NOGATHER-NEXT:  .LBB4_5: # %cond.load4
+; NOGATHER-NEXT:    vmovq %xmm0, %rcx
+; NOGATHER-NEXT:    vpinsrd $2, (%rcx), %xmm2, %xmm2
+; NOGATHER-NEXT:    testb $8, %al
 ; NOGATHER-NEXT:    je .LBB4_8
-; NOGATHER-NEXT:  # %bb.7: # %cond.load7
+; NOGATHER-NEXT:  .LBB4_7: # %cond.load7
 ; NOGATHER-NEXT:    vpextrq $1, %xmm0, %rax
 ; NOGATHER-NEXT:    vpinsrd $3, (%rax), %xmm2, %xmm2
-; NOGATHER-NEXT:  .LBB4_8: # %else8
 ; NOGATHER-NEXT:    vmovdqa %xmm2, %xmm0
 ; NOGATHER-NEXT:    vzeroupper
 ; NOGATHER-NEXT:    retq
@@ -278,36 +289,39 @@ define <4 x float> @masked_gather_v4float(<4 x float*> %ptrs, <4 x i1> %masks, <
 ;
 ; NOGATHER-LABEL: masked_gather_v4float:
 ; NOGATHER:       # %bb.0: # %entry
-; NOGATHER-NEXT:    vpextrb $0, %xmm1, %eax
+; NOGATHER-NEXT:    vpslld $31, %xmm1, %xmm1
+; NOGATHER-NEXT:    vmovmskps %xmm1, %eax
 ; NOGATHER-NEXT:    testb $1, %al
 ; NOGATHER-NEXT:    je .LBB5_2
 ; NOGATHER-NEXT:  # %bb.1: # %cond.load
-; NOGATHER-NEXT:    vmovq %xmm0, %rax
-; NOGATHER-NEXT:    vmovss {{.*#+}} xmm3 = mem[0],zero,zero,zero
-; NOGATHER-NEXT:    vblendps {{.*#+}} xmm2 = xmm3[0],xmm2[1,2,3]
+; NOGATHER-NEXT:    vmovq %xmm0, %rcx
+; NOGATHER-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; NOGATHER-NEXT:    vblendps {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
 ; NOGATHER-NEXT:  .LBB5_2: # %else
-; NOGATHER-NEXT:    vpextrb $4, %xmm1, %eax
-; NOGATHER-NEXT:    testb $1, %al
+; NOGATHER-NEXT:    testb $2, %al
 ; NOGATHER-NEXT:    je .LBB5_4
 ; NOGATHER-NEXT:  # %bb.3: # %cond.load1
-; NOGATHER-NEXT:    vpextrq $1, %xmm0, %rax
+; NOGATHER-NEXT:    vpextrq $1, %xmm0, %rcx
 ; NOGATHER-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0],mem[0],xmm2[2,3]
 ; NOGATHER-NEXT:  .LBB5_4: # %else2
-; NOGATHER-NEXT:    vpextrb $8, %xmm1, %eax
 ; NOGATHER-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; NOGATHER-NEXT:    testb $1, %al
-; NOGATHER-NEXT:    je .LBB5_6
-; NOGATHER-NEXT:  # %bb.5: # %cond.load4
-; NOGATHER-NEXT:    vmovq %xmm0, %rax
+; NOGATHER-NEXT:    testb $4, %al
+; NOGATHER-NEXT:    jne .LBB5_5
+; NOGATHER-NEXT:  # %bb.6: # %else5
+; NOGATHER-NEXT:    testb $8, %al
+; NOGATHER-NEXT:    jne .LBB5_7
+; NOGATHER-NEXT:  .LBB5_8: # %else8
+; NOGATHER-NEXT:    vmovaps %xmm2, %xmm0
+; NOGATHER-NEXT:    vzeroupper
+; NOGATHER-NEXT:    retq
+; NOGATHER-NEXT:  .LBB5_5: # %cond.load4
+; NOGATHER-NEXT:    vmovq %xmm0, %rcx
 ; NOGATHER-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1],mem[0],xmm2[3]
-; NOGATHER-NEXT:  .LBB5_6: # %else5
-; NOGATHER-NEXT:    vpextrb $12, %xmm1, %eax
-; NOGATHER-NEXT:    testb $1, %al
+; NOGATHER-NEXT:    testb $8, %al
 ; NOGATHER-NEXT:    je .LBB5_8
-; NOGATHER-NEXT:  # %bb.7: # %cond.load7
+; NOGATHER-NEXT:  .LBB5_7: # %cond.load7
 ; NOGATHER-NEXT:    vpextrq $1, %xmm0, %rax
 ; NOGATHER-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1,2],mem[0]
-; NOGATHER-NEXT:  .LBB5_8: # %else8
 ; NOGATHER-NEXT:    vmovaps %xmm2, %xmm0
 ; NOGATHER-NEXT:    vzeroupper
 ; NOGATHER-NEXT:    retq
@@ -347,76 +361,82 @@ define <8 x i32> @masked_gather_v8i32(<8 x i32*>* %ptr, <8 x i1> %masks, <8 x i3
 ; NOGATHER:       # %bb.0: # %entry
 ; NOGATHER-NEXT:    vmovdqa (%rdi), %ymm3
 ; NOGATHER-NEXT:    vmovdqa 32(%rdi), %ymm2
-; NOGATHER-NEXT:    vpextrb $0, %xmm0, %eax
+; NOGATHER-NEXT:    vpsllw $15, %xmm0, %xmm0
+; NOGATHER-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
+; NOGATHER-NEXT:    vpmovmskb %xmm0, %eax
 ; NOGATHER-NEXT:    testb $1, %al
 ; NOGATHER-NEXT:    je .LBB6_2
 ; NOGATHER-NEXT:  # %bb.1: # %cond.load
-; NOGATHER-NEXT:    vmovq %xmm3, %rax
-; NOGATHER-NEXT:    vpinsrd $0, (%rax), %xmm1, %xmm4
-; NOGATHER-NEXT:    vblendps {{.*#+}} ymm1 = ymm4[0,1,2,3],ymm1[4,5,6,7]
+; NOGATHER-NEXT:    vmovq %xmm3, %rcx
+; NOGATHER-NEXT:    vpinsrd $0, (%rcx), %xmm1, %xmm0
+; NOGATHER-NEXT:    vblendps {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
 ; NOGATHER-NEXT:  .LBB6_2: # %else
-; NOGATHER-NEXT:    vpextrb $2, %xmm0, %eax
-; NOGATHER-NEXT:    testb $1, %al
+; NOGATHER-NEXT:    testb $2, %al
 ; NOGATHER-NEXT:    je .LBB6_4
 ; NOGATHER-NEXT:  # %bb.3: # %cond.load1
-; NOGATHER-NEXT:    vpextrq $1, %xmm3, %rax
-; NOGATHER-NEXT:    vpinsrd $1, (%rax), %xmm1, %xmm4
-; NOGATHER-NEXT:    vblendps {{.*#+}} ymm1 = ymm4[0,1,2,3],ymm1[4,5,6,7]
+; NOGATHER-NEXT:    vpextrq $1, %xmm3, %rcx
+; NOGATHER-NEXT:    vpinsrd $1, (%rcx), %xmm1, %xmm0
+; NOGATHER-NEXT:    vblendps {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
 ; NOGATHER-NEXT:  .LBB6_4: # %else2
-; NOGATHER-NEXT:    vpextrb $4, %xmm0, %eax
-; NOGATHER-NEXT:    vextractf128 $1, %ymm3, %xmm3
-; NOGATHER-NEXT:    testb $1, %al
-; NOGATHER-NEXT:    je .LBB6_6
-; NOGATHER-NEXT:  # %bb.5: # %cond.load4
-; NOGATHER-NEXT:    vmovq %xmm3, %rax
-; NOGATHER-NEXT:    vpinsrd $2, (%rax), %xmm1, %xmm4
-; NOGATHER-NEXT:    vblendps {{.*#+}} ymm1 = ymm4[0,1,2,3],ymm1[4,5,6,7]
-; NOGATHER-NEXT:  .LBB6_6: # %else5
-; NOGATHER-NEXT:    vpextrb $6, %xmm0, %eax
-; NOGATHER-NEXT:    testb $1, %al
-; NOGATHER-NEXT:    je .LBB6_8
-; NOGATHER-NEXT:  # %bb.7: # %cond.load7
-; NOGATHER-NEXT:    vpextrq $1, %xmm3, %rax
-; NOGATHER-NEXT:    vpinsrd $3, (%rax), %xmm1, %xmm3
-; NOGATHER-NEXT:    vblendps {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
+; NOGATHER-NEXT:    vextractf128 $1, %ymm3, %xmm0
+; NOGATHER-NEXT:    testb $4, %al
+; NOGATHER-NEXT:    jne .LBB6_5
+; NOGATHER-NEXT:  # %bb.6: # %else5
+; NOGATHER-NEXT:    testb $8, %al
+; NOGATHER-NEXT:    jne .LBB6_7
 ; NOGATHER-NEXT:  .LBB6_8: # %else8
-; NOGATHER-NEXT:    vpextrb $8, %xmm0, %eax
-; NOGATHER-NEXT:    testb $1, %al
-; NOGATHER-NEXT:    je .LBB6_10
-; NOGATHER-NEXT:  # %bb.9: # %cond.load10
-; NOGATHER-NEXT:    vmovq %xmm2, %rax
-; NOGATHER-NEXT:    vextractf128 $1, %ymm1, %xmm3
-; NOGATHER-NEXT:    vpinsrd $0, (%rax), %xmm3, %xmm3
-; NOGATHER-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
+; NOGATHER-NEXT:    testb $16, %al
+; NOGATHER-NEXT:    jne .LBB6_9
 ; NOGATHER-NEXT:  .LBB6_10: # %else11
-; NOGATHER-NEXT:    vpextrb $10, %xmm0, %eax
-; NOGATHER-NEXT:    testb $1, %al
+; NOGATHER-NEXT:    testb $32, %al
 ; NOGATHER-NEXT:    je .LBB6_12
-; NOGATHER-NEXT:  # %bb.11: # %cond.load13
-; NOGATHER-NEXT:    vpextrq $1, %xmm2, %rax
-; NOGATHER-NEXT:    vextractf128 $1, %ymm1, %xmm3
-; NOGATHER-NEXT:    vpinsrd $1, (%rax), %xmm3, %xmm3
-; NOGATHER-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
+; NOGATHER-NEXT:  .LBB6_11: # %cond.load13
+; NOGATHER-NEXT:    vpextrq $1, %xmm2, %rcx
+; NOGATHER-NEXT:    vextractf128 $1, %ymm1, %xmm0
+; NOGATHER-NEXT:    vpinsrd $1, (%rcx), %xmm0, %xmm0
+; NOGATHER-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm1
 ; NOGATHER-NEXT:  .LBB6_12: # %else14
-; NOGATHER-NEXT:    vpextrb $12, %xmm0, %eax
-; NOGATHER-NEXT:    vextractf128 $1, %ymm2, %xmm2
-; NOGATHER-NEXT:    testb $1, %al
-; NOGATHER-NEXT:    je .LBB6_14
-; NOGATHER-NEXT:  # %bb.13: # %cond.load16
-; NOGATHER-NEXT:    vmovq %xmm2, %rax
-; NOGATHER-NEXT:    vextractf128 $1, %ymm1, %xmm3
-; NOGATHER-NEXT:    vpinsrd $2, (%rax), %xmm3, %xmm3
-; NOGATHER-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
-; NOGATHER-NEXT:  .LBB6_14: # %else17
-; NOGATHER-NEXT:    vpextrb $14, %xmm0, %eax
-; NOGATHER-NEXT:    testb $1, %al
+; NOGATHER-NEXT:    vextractf128 $1, %ymm2, %xmm0
+; NOGATHER-NEXT:    testb $64, %al
+; NOGATHER-NEXT:    jne .LBB6_13
+; NOGATHER-NEXT:  # %bb.14: # %else17
+; NOGATHER-NEXT:    testb $-128, %al
+; NOGATHER-NEXT:    jne .LBB6_15
+; NOGATHER-NEXT:  .LBB6_16: # %else20
+; NOGATHER-NEXT:    vmovaps %ymm1, %ymm0
+; NOGATHER-NEXT:    retq
+; NOGATHER-NEXT:  .LBB6_5: # %cond.load4
+; NOGATHER-NEXT:    vmovq %xmm0, %rcx
+; NOGATHER-NEXT:    vpinsrd $2, (%rcx), %xmm1, %xmm3
+; NOGATHER-NEXT:    vblendps {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
+; NOGATHER-NEXT:    testb $8, %al
+; NOGATHER-NEXT:    je .LBB6_8
+; NOGATHER-NEXT:  .LBB6_7: # %cond.load7
+; NOGATHER-NEXT:    vpextrq $1, %xmm0, %rcx
+; NOGATHER-NEXT:    vpinsrd $3, (%rcx), %xmm1, %xmm0
+; NOGATHER-NEXT:    vblendps {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; NOGATHER-NEXT:    testb $16, %al
+; NOGATHER-NEXT:    je .LBB6_10
+; NOGATHER-NEXT:  .LBB6_9: # %cond.load10
+; NOGATHER-NEXT:    vmovq %xmm2, %rcx
+; NOGATHER-NEXT:    vextractf128 $1, %ymm1, %xmm0
+; NOGATHER-NEXT:    vpinsrd $0, (%rcx), %xmm0, %xmm0
+; NOGATHER-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm1
+; NOGATHER-NEXT:    testb $32, %al
+; NOGATHER-NEXT:    jne .LBB6_11
+; NOGATHER-NEXT:    jmp .LBB6_12
+; NOGATHER-NEXT:  .LBB6_13: # %cond.load16
+; NOGATHER-NEXT:    vmovq %xmm0, %rcx
+; NOGATHER-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; NOGATHER-NEXT:    vpinsrd $2, (%rcx), %xmm2, %xmm2
+; NOGATHER-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; NOGATHER-NEXT:    testb $-128, %al
 ; NOGATHER-NEXT:    je .LBB6_16
-; NOGATHER-NEXT:  # %bb.15: # %cond.load19
-; NOGATHER-NEXT:    vpextrq $1, %xmm2, %rax
+; NOGATHER-NEXT:  .LBB6_15: # %cond.load19
+; NOGATHER-NEXT:    vpextrq $1, %xmm0, %rax
 ; NOGATHER-NEXT:    vextractf128 $1, %ymm1, %xmm0
 ; NOGATHER-NEXT:    vpinsrd $3, (%rax), %xmm0, %xmm0
 ; NOGATHER-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm1
-; NOGATHER-NEXT:  .LBB6_16: # %else20
 ; NOGATHER-NEXT:    vmovaps %ymm1, %ymm0
 ; NOGATHER-NEXT:    retq
 entry:
@@ -456,77 +476,83 @@ define <8 x float> @masked_gather_v8float(<8 x float*>* %ptr, <8 x i1> %masks, <
 ; NOGATHER:       # %bb.0: # %entry
 ; NOGATHER-NEXT:    vmovdqa (%rdi), %ymm3
 ; NOGATHER-NEXT:    vmovdqa 32(%rdi), %ymm2
-; NOGATHER-NEXT:    vpextrb $0, %xmm0, %eax
+; NOGATHER-NEXT:    vpsllw $15, %xmm0, %xmm0
+; NOGATHER-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
+; NOGATHER-NEXT:    vpmovmskb %xmm0, %eax
 ; NOGATHER-NEXT:    testb $1, %al
 ; NOGATHER-NEXT:    je .LBB7_2
 ; NOGATHER-NEXT:  # %bb.1: # %cond.load
-; NOGATHER-NEXT:    vmovq %xmm3, %rax
-; NOGATHER-NEXT:    vmovss {{.*#+}} xmm4 = mem[0],zero,zero,zero
-; NOGATHER-NEXT:    vblendps {{.*#+}} ymm1 = ymm4[0],ymm1[1,2,3,4,5,6,7]
+; NOGATHER-NEXT:    vmovq %xmm3, %rcx
+; NOGATHER-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; NOGATHER-NEXT:    vblendps {{.*#+}} ymm1 = ymm0[0],ymm1[1,2,3,4,5,6,7]
 ; NOGATHER-NEXT:  .LBB7_2: # %else
-; NOGATHER-NEXT:    vpextrb $2, %xmm0, %eax
-; NOGATHER-NEXT:    testb $1, %al
+; NOGATHER-NEXT:    testb $2, %al
 ; NOGATHER-NEXT:    je .LBB7_4
 ; NOGATHER-NEXT:  # %bb.3: # %cond.load1
-; NOGATHER-NEXT:    vpextrq $1, %xmm3, %rax
-; NOGATHER-NEXT:    vinsertps {{.*#+}} xmm4 = xmm1[0],mem[0],xmm1[2,3]
-; NOGATHER-NEXT:    vblendps {{.*#+}} ymm1 = ymm4[0,1,2,3],ymm1[4,5,6,7]
+; NOGATHER-NEXT:    vpextrq $1, %xmm3, %rcx
+; NOGATHER-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],mem[0],xmm1[2,3]
+; NOGATHER-NEXT:    vblendps {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
 ; NOGATHER-NEXT:  .LBB7_4: # %else2
-; NOGATHER-NEXT:    vpextrb $4, %xmm0, %eax
-; NOGATHER-NEXT:    vextractf128 $1, %ymm3, %xmm3
-; NOGATHER-NEXT:    testb $1, %al
-; NOGATHER-NEXT:    je .LBB7_6
-; NOGATHER-NEXT:  # %bb.5: # %cond.load4
-; NOGATHER-NEXT:    vmovq %xmm3, %rax
-; NOGATHER-NEXT:    vinsertps {{.*#+}} xmm4 = xmm1[0,1],mem[0],xmm1[3]
-; NOGATHER-NEXT:    vblendps {{.*#+}} ymm1 = ymm4[0,1,2,3],ymm1[4,5,6,7]
-; NOGATHER-NEXT:  .LBB7_6: # %else5
-; NOGATHER-NEXT:    vpextrb $6, %xmm0, %eax
-; NOGATHER-NEXT:    testb $1, %al
-; NOGATHER-NEXT:    je .LBB7_8
-; NOGATHER-NEXT:  # %bb.7: # %cond.load7
-; NOGATHER-NEXT:    vpextrq $1, %xmm3, %rax
-; NOGATHER-NEXT:    vinsertps {{.*#+}} xmm3 = xmm1[0,1,2],mem[0]
-; NOGATHER-NEXT:    vblendps {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
+; NOGATHER-NEXT:    vextractf128 $1, %ymm3, %xmm0
+; NOGATHER-NEXT:    testb $4, %al
+; NOGATHER-NEXT:    jne .LBB7_5
+; NOGATHER-NEXT:  # %bb.6: # %else5
+; NOGATHER-NEXT:    testb $8, %al
+; NOGATHER-NEXT:    jne .LBB7_7
 ; NOGATHER-NEXT:  .LBB7_8: # %else8
-; NOGATHER-NEXT:    vpextrb $8, %xmm0, %eax
-; NOGATHER-NEXT:    testb $1, %al
-; NOGATHER-NEXT:    je .LBB7_10
-; NOGATHER-NEXT:  # %bb.9: # %cond.load10
-; NOGATHER-NEXT:    vmovq %xmm2, %rax
-; NOGATHER-NEXT:    vextractf128 $1, %ymm1, %xmm3
-; NOGATHER-NEXT:    vmovss {{.*#+}} xmm4 = mem[0],zero,zero,zero
-; NOGATHER-NEXT:    vblendps {{.*#+}} xmm3 = xmm4[0],xmm3[1,2,3]
-; NOGATHER-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
+; NOGATHER-NEXT:    testb $16, %al
+; NOGATHER-NEXT:    jne .LBB7_9
 ; NOGATHER-NEXT:  .LBB7_10: # %else11
-; NOGATHER-NEXT:    vpextrb $10, %xmm0, %eax
-; NOGATHER-NEXT:    testb $1, %al
+; NOGATHER-NEXT:    testb $32, %al
 ; NOGATHER-NEXT:    je .LBB7_12
-; NOGATHER-NEXT:  # %bb.11: # %cond.load13
-; NOGATHER-NEXT:    vpextrq $1, %xmm2, %rax
-; NOGATHER-NEXT:    vextractf128 $1, %ymm1, %xmm3
-; NOGATHER-NEXT:    vinsertps {{.*#+}} xmm3 = xmm3[0],mem[0],xmm3[2,3]
-; NOGATHER-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
+; NOGATHER-NEXT:  .LBB7_11: # %cond.load13
+; NOGATHER-NEXT:    vpextrq $1, %xmm2, %rcx
+; NOGATHER-NEXT:    vextractf128 $1, %ymm1, %xmm0
+; NOGATHER-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[2,3]
+; NOGATHER-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm1
 ; NOGATHER-NEXT:  .LBB7_12: # %else14
-; NOGATHER-NEXT:    vpextrb $12, %xmm0, %eax
-; NOGATHER-NEXT:    vextractf128 $1, %ymm2, %xmm2
-; NOGATHER-NEXT:    testb $1, %al
-; NOGATHER-NEXT:    je .LBB7_14
-; NOGATHER-NEXT:  # %bb.13: # %cond.load16
-; NOGATHER-NEXT:    vmovq %xmm2, %rax
-; NOGATHER-NEXT:    vextractf128 $1, %ymm1, %xmm3
-; NOGATHER-NEXT:    vinsertps {{.*#+}} xmm3 = xmm3[0,1],mem[0],xmm3[3]
-; NOGATHER-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
-; NOGATHER-NEXT:  .LBB7_14: # %else17
-; NOGATHER-NEXT:    vpextrb $14, %xmm0, %eax
-; NOGATHER-NEXT:    testb $1, %al
+; NOGATHER-NEXT:    vextractf128 $1, %ymm2, %xmm0
+; NOGATHER-NEXT:    testb $64, %al
+; NOGATHER-NEXT:    jne .LBB7_13
+; NOGATHER-NEXT:  # %bb.14: # %else17
+; NOGATHER-NEXT:    testb $-128, %al
+; NOGATHER-NEXT:    jne .LBB7_15
+; NOGATHER-NEXT:  .LBB7_16: # %else20
+; NOGATHER-NEXT:    vmovaps %ymm1, %ymm0
+; NOGATHER-NEXT:    retq
+; NOGATHER-NEXT:  .LBB7_5: # %cond.load4
+; NOGATHER-NEXT:    vmovq %xmm0, %rcx
+; NOGATHER-NEXT:    vinsertps {{.*#+}} xmm3 = xmm1[0,1],mem[0],xmm1[3]
+; NOGATHER-NEXT:    vblendps {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
+; NOGATHER-NEXT:    testb $8, %al
+; NOGATHER-NEXT:    je .LBB7_8
+; NOGATHER-NEXT:  .LBB7_7: # %cond.load7
+; NOGATHER-NEXT:    vpextrq $1, %xmm0, %rcx
+; NOGATHER-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],mem[0]
+; NOGATHER-NEXT:    vblendps {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; NOGATHER-NEXT:    testb $16, %al
+; NOGATHER-NEXT:    je .LBB7_10
+; NOGATHER-NEXT:  .LBB7_9: # %cond.load10
+; NOGATHER-NEXT:    vmovq %xmm2, %rcx
+; NOGATHER-NEXT:    vextractf128 $1, %ymm1, %xmm0
+; NOGATHER-NEXT:    vmovd {{.*#+}} xmm3 = mem[0],zero,zero,zero
+; NOGATHER-NEXT:    vpblendw {{.*#+}} xmm0 = xmm3[0,1],xmm0[2,3,4,5,6,7]
+; NOGATHER-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm1
+; NOGATHER-NEXT:    testb $32, %al
+; NOGATHER-NEXT:    jne .LBB7_11
+; NOGATHER-NEXT:    jmp .LBB7_12
+; NOGATHER-NEXT:  .LBB7_13: # %cond.load16
+; NOGATHER-NEXT:    vmovq %xmm0, %rcx
+; NOGATHER-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; NOGATHER-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1],mem[0],xmm2[3]
+; NOGATHER-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; NOGATHER-NEXT:    testb $-128, %al
 ; NOGATHER-NEXT:    je .LBB7_16
-; NOGATHER-NEXT:  # %bb.15: # %cond.load19
-; NOGATHER-NEXT:    vpextrq $1, %xmm2, %rax
+; NOGATHER-NEXT:  .LBB7_15: # %cond.load19
+; NOGATHER-NEXT:    vpextrq $1, %xmm0, %rax
 ; NOGATHER-NEXT:    vextractf128 $1, %ymm1, %xmm0
 ; NOGATHER-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],mem[0]
 ; NOGATHER-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm1
-; NOGATHER-NEXT:  .LBB7_16: # %else20
 ; NOGATHER-NEXT:    vmovaps %ymm1, %ymm0
 ; NOGATHER-NEXT:    retq
 entry:
@@ -560,41 +586,43 @@ define <4 x i64> @masked_gather_v4i64(<4 x i64*>* %ptr, <4 x i1> %masks, <4 x i6
 ; NOGATHER-LABEL: masked_gather_v4i64:
 ; NOGATHER:       # %bb.0: # %entry
 ; NOGATHER-NEXT:    vmovdqa (%rdi), %ymm2
-; NOGATHER-NEXT:    vpextrb $0, %xmm0, %eax
+; NOGATHER-NEXT:    vpslld $31, %xmm0, %xmm0
+; NOGATHER-NEXT:    vmovmskps %xmm0, %eax
 ; NOGATHER-NEXT:    testb $1, %al
 ; NOGATHER-NEXT:    je .LBB8_2
 ; NOGATHER-NEXT:  # %bb.1: # %cond.load
-; NOGATHER-NEXT:    vmovq %xmm2, %rax
-; NOGATHER-NEXT:    vpinsrq $0, (%rax), %xmm1, %xmm3
-; NOGATHER-NEXT:    vblendps {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
+; NOGATHER-NEXT:    vmovq %xmm2, %rcx
+; NOGATHER-NEXT:    vpinsrq $0, (%rcx), %xmm1, %xmm0
+; NOGATHER-NEXT:    vblendps {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
 ; NOGATHER-NEXT:  .LBB8_2: # %else
-; NOGATHER-NEXT:    vpextrb $4, %xmm0, %eax
-; NOGATHER-NEXT:    testb $1, %al
+; NOGATHER-NEXT:    testb $2, %al
 ; NOGATHER-NEXT:    je .LBB8_4
 ; NOGATHER-NEXT:  # %bb.3: # %cond.load1
-; NOGATHER-NEXT:    vpextrq $1, %xmm2, %rax
-; NOGATHER-NEXT:    vpinsrq $1, (%rax), %xmm1, %xmm3
-; NOGATHER-NEXT:    vblendps {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
+; NOGATHER-NEXT:    vpextrq $1, %xmm2, %rcx
+; NOGATHER-NEXT:    vpinsrq $1, (%rcx), %xmm1, %xmm0
+; NOGATHER-NEXT:    vblendps {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
 ; NOGATHER-NEXT:  .LBB8_4: # %else2
-; NOGATHER-NEXT:    vpextrb $8, %xmm0, %eax
-; NOGATHER-NEXT:    vextractf128 $1, %ymm2, %xmm2
-; NOGATHER-NEXT:    testb $1, %al
-; NOGATHER-NEXT:    je .LBB8_6
-; NOGATHER-NEXT:  # %bb.5: # %cond.load4
-; NOGATHER-NEXT:    vmovq %xmm2, %rax
-; NOGATHER-NEXT:    vextractf128 $1, %ymm1, %xmm3
-; NOGATHER-NEXT:    vpinsrq $0, (%rax), %xmm3, %xmm3
-; NOGATHER-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
-; NOGATHER-NEXT:  .LBB8_6: # %else5
-; NOGATHER-NEXT:    vpextrb $12, %xmm0, %eax
-; NOGATHER-NEXT:    testb $1, %al
+; NOGATHER-NEXT:    vextractf128 $1, %ymm2, %xmm0
+; NOGATHER-NEXT:    testb $4, %al
+; NOGATHER-NEXT:    jne .LBB8_5
+; NOGATHER-NEXT:  # %bb.6: # %else5
+; NOGATHER-NEXT:    testb $8, %al
+; NOGATHER-NEXT:    jne .LBB8_7
+; NOGATHER-NEXT:  .LBB8_8: # %else8
+; NOGATHER-NEXT:    vmovaps %ymm1, %ymm0
+; NOGATHER-NEXT:    retq
+; NOGATHER-NEXT:  .LBB8_5: # %cond.load4
+; NOGATHER-NEXT:    vmovq %xmm0, %rcx
+; NOGATHER-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; NOGATHER-NEXT:    vpinsrq $0, (%rcx), %xmm2, %xmm2
+; NOGATHER-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; NOGATHER-NEXT:    testb $8, %al
 ; NOGATHER-NEXT:    je .LBB8_8
-; NOGATHER-NEXT:  # %bb.7: # %cond.load7
-; NOGATHER-NEXT:    vpextrq $1, %xmm2, %rax
+; NOGATHER-NEXT:  .LBB8_7: # %cond.load7
+; NOGATHER-NEXT:    vpextrq $1, %xmm0, %rax
 ; NOGATHER-NEXT:    vextractf128 $1, %ymm1, %xmm0
 ; NOGATHER-NEXT:    vpinsrq $1, (%rax), %xmm0, %xmm0
 ; NOGATHER-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm1
-; NOGATHER-NEXT:  .LBB8_8: # %else8
 ; NOGATHER-NEXT:    vmovaps %ymm1, %ymm0
 ; NOGATHER-NEXT:    retq
 entry:
@@ -628,41 +656,43 @@ define <4 x double> @masked_gather_v4double(<4 x double*>* %ptr, <4 x i1> %masks
 ; NOGATHER-LABEL: masked_gather_v4double:
 ; NOGATHER:       # %bb.0: # %entry
 ; NOGATHER-NEXT:    vmovdqa (%rdi), %ymm2
-; NOGATHER-NEXT:    vpextrb $0, %xmm0, %eax
+; NOGATHER-NEXT:    vpslld $31, %xmm0, %xmm0
+; NOGATHER-NEXT:    vmovmskps %xmm0, %eax
 ; NOGATHER-NEXT:    testb $1, %al
 ; NOGATHER-NEXT:    je .LBB9_2
 ; NOGATHER-NEXT:  # %bb.1: # %cond.load
-; NOGATHER-NEXT:    vmovq %xmm2, %rax
-; NOGATHER-NEXT:    vmovsd {{.*#+}} xmm3 = mem[0],zero
-; NOGATHER-NEXT:    vblendps {{.*#+}} ymm1 = ymm3[0,1],ymm1[2,3,4,5,6,7]
+; NOGATHER-NEXT:    vmovq %xmm2, %rcx
+; NOGATHER-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; NOGATHER-NEXT:    vblendps {{.*#+}} ymm1 = ymm0[0,1],ymm1[2,3,4,5,6,7]
 ; NOGATHER-NEXT:  .LBB9_2: # %else
-; NOGATHER-NEXT:    vpextrb $4, %xmm0, %eax
-; NOGATHER-NEXT:    testb $1, %al
+; NOGATHER-NEXT:    testb $2, %al
 ; NOGATHER-NEXT:    je .LBB9_4
 ; NOGATHER-NEXT:  # %bb.3: # %cond.load1
-; NOGATHER-NEXT:    vpextrq $1, %xmm2, %rax
-; NOGATHER-NEXT:    vmovhps {{.*#+}} xmm3 = xmm1[0,1],mem[0,1]
-; NOGATHER-NEXT:    vblendps {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
+; NOGATHER-NEXT:    vpextrq $1, %xmm2, %rcx
+; NOGATHER-NEXT:    vmovhps {{.*#+}} xmm0 = xmm1[0,1],mem[0,1]
+; NOGATHER-NEXT:    vblendps {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
 ; NOGATHER-NEXT:  .LBB9_4: # %else2
-; NOGATHER-NEXT:    vpextrb $8, %xmm0, %eax
-; NOGATHER-NEXT:    vextractf128 $1, %ymm2, %xmm2
-; NOGATHER-NEXT:    testb $1, %al
-; NOGATHER-NEXT:    je .LBB9_6
-; NOGATHER-NEXT:  # %bb.5: # %cond.load4
-; NOGATHER-NEXT:    vmovq %xmm2, %rax
-; NOGATHER-NEXT:    vextractf128 $1, %ymm1, %xmm3
-; NOGATHER-NEXT:    vmovlps {{.*#+}} xmm3 = mem[0,1],xmm3[2,3]
-; NOGATHER-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
-; NOGATHER-NEXT:  .LBB9_6: # %else5
-; NOGATHER-NEXT:    vpextrb $12, %xmm0, %eax
-; NOGATHER-NEXT:    testb $1, %al
+; NOGATHER-NEXT:    vextractf128 $1, %ymm2, %xmm0
+; NOGATHER-NEXT:    testb $4, %al
+; NOGATHER-NEXT:    jne .LBB9_5
+; NOGATHER-NEXT:  # %bb.6: # %else5
+; NOGATHER-NEXT:    testb $8, %al
+; NOGATHER-NEXT:    jne .LBB9_7
+; NOGATHER-NEXT:  .LBB9_8: # %else8
+; NOGATHER-NEXT:    vmovaps %ymm1, %ymm0
+; NOGATHER-NEXT:    retq
+; NOGATHER-NEXT:  .LBB9_5: # %cond.load4
+; NOGATHER-NEXT:    vmovq %xmm0, %rcx
+; NOGATHER-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; NOGATHER-NEXT:    vmovlps {{.*#+}} xmm2 = mem[0,1],xmm2[2,3]
+; NOGATHER-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; NOGATHER-NEXT:    testb $8, %al
 ; NOGATHER-NEXT:    je .LBB9_8
-; NOGATHER-NEXT:  # %bb.7: # %cond.load7
-; NOGATHER-NEXT:    vpextrq $1, %xmm2, %rax
+; NOGATHER-NEXT:  .LBB9_7: # %cond.load7
+; NOGATHER-NEXT:    vpextrq $1, %xmm0, %rax
 ; NOGATHER-NEXT:    vextractf128 $1, %ymm1, %xmm0
 ; NOGATHER-NEXT:    vmovhps {{.*#+}} xmm0 = xmm0[0,1],mem[0,1]
 ; NOGATHER-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm1
-; NOGATHER-NEXT:  .LBB9_8: # %else8
 ; NOGATHER-NEXT:    vmovaps %ymm1, %ymm0
 ; NOGATHER-NEXT:    retq
 entry:
@@ -676,10 +706,10 @@ declare <2 x i64> @llvm.masked.gather.v2i64(<2 x i64*> %ptrs, i32 %align, <2 x i
 define <2 x i64> @masked_gather_v2i64(<2 x i64*>* %ptr, <2 x i1> %masks, <2 x i64> %passthro) {
 ; X86-LABEL: masked_gather_v2i64:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    vpmovsxdq (%eax), %xmm2
 ; X86-NEXT:    vpsllq $63, %xmm0, %xmm0
-; X86-NEXT:    vpgatherqq %xmm0, (,%xmm2), %xmm1
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero
+; X86-NEXT:    vpgatherdq %xmm0, (,%xmm2), %xmm1
 ; X86-NEXT:    vmovdqa %xmm1, %xmm0
 ; X86-NEXT:    retl
 ;
@@ -694,20 +724,24 @@ define <2 x i64> @masked_gather_v2i64(<2 x i64*>* %ptr, <2 x i1> %masks, <2 x i6
 ; NOGATHER-LABEL: masked_gather_v2i64:
 ; NOGATHER:       # %bb.0: # %entry
 ; NOGATHER-NEXT:    vmovdqa (%rdi), %xmm2
-; NOGATHER-NEXT:    vpextrb $0, %xmm0, %eax
-; NOGATHER-NEXT:    testb $1, %al
-; NOGATHER-NEXT:    je .LBB10_2
-; NOGATHER-NEXT:  # %bb.1: # %cond.load
-; NOGATHER-NEXT:    vmovq %xmm2, %rax
-; NOGATHER-NEXT:    vpinsrq $0, (%rax), %xmm1, %xmm1
-; NOGATHER-NEXT:  .LBB10_2: # %else
-; NOGATHER-NEXT:    vpextrb $8, %xmm0, %eax
+; NOGATHER-NEXT:    vpsllq $63, %xmm0, %xmm0
+; NOGATHER-NEXT:    vmovmskpd %xmm0, %eax
 ; NOGATHER-NEXT:    testb $1, %al
+; NOGATHER-NEXT:    jne .LBB10_1
+; NOGATHER-NEXT:  # %bb.2: # %else
+; NOGATHER-NEXT:    testb $2, %al
+; NOGATHER-NEXT:    jne .LBB10_3
+; NOGATHER-NEXT:  .LBB10_4: # %else2
+; NOGATHER-NEXT:    vmovdqa %xmm1, %xmm0
+; NOGATHER-NEXT:    retq
+; NOGATHER-NEXT:  .LBB10_1: # %cond.load
+; NOGATHER-NEXT:    vmovq %xmm2, %rcx
+; NOGATHER-NEXT:    vpinsrq $0, (%rcx), %xmm1, %xmm1
+; NOGATHER-NEXT:    testb $2, %al
 ; NOGATHER-NEXT:    je .LBB10_4
-; NOGATHER-NEXT:  # %bb.3: # %cond.load1
+; NOGATHER-NEXT:  .LBB10_3: # %cond.load1
 ; NOGATHER-NEXT:    vpextrq $1, %xmm2, %rax
 ; NOGATHER-NEXT:    vpinsrq $1, (%rax), %xmm1, %xmm1
-; NOGATHER-NEXT:  .LBB10_4: # %else2
 ; NOGATHER-NEXT:    vmovdqa %xmm1, %xmm0
 ; NOGATHER-NEXT:    retq
 entry:
@@ -721,10 +755,10 @@ declare <2 x double> @llvm.masked.gather.v2double(<2 x double*> %ptrs, i32 %alig
 define <2 x double> @masked_gather_v2double(<2 x double*>* %ptr, <2 x i1> %masks, <2 x double> %passthro) {
 ; X86-LABEL: masked_gather_v2double:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    vpmovsxdq (%eax), %xmm2
 ; X86-NEXT:    vpsllq $63, %xmm0, %xmm0
-; X86-NEXT:    vgatherqpd %xmm0, (,%xmm2), %xmm1
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    vmovsd {{.*#+}} xmm2 = mem[0],zero
+; X86-NEXT:    vgatherdpd %xmm0, (,%xmm2), %xmm1
 ; X86-NEXT:    vmovapd %xmm1, %xmm0
 ; X86-NEXT:    retl
 ;
@@ -739,20 +773,24 @@ define <2 x double> @masked_gather_v2double(<2 x double*>* %ptr, <2 x i1> %masks
 ; NOGATHER-LABEL: masked_gather_v2double:
 ; NOGATHER:       # %bb.0: # %entry
 ; NOGATHER-NEXT:    vmovdqa (%rdi), %xmm2
-; NOGATHER-NEXT:    vpextrb $0, %xmm0, %eax
+; NOGATHER-NEXT:    vpsllq $63, %xmm0, %xmm0
+; NOGATHER-NEXT:    vmovmskpd %xmm0, %eax
 ; NOGATHER-NEXT:    testb $1, %al
-; NOGATHER-NEXT:    je .LBB11_2
-; NOGATHER-NEXT:  # %bb.1: # %cond.load
-; NOGATHER-NEXT:    vmovq %xmm2, %rax
+; NOGATHER-NEXT:    jne .LBB11_1
+; NOGATHER-NEXT:  # %bb.2: # %else
+; NOGATHER-NEXT:    testb $2, %al
+; NOGATHER-NEXT:    jne .LBB11_3
+; NOGATHER-NEXT:  .LBB11_4: # %else2
+; NOGATHER-NEXT:    vmovaps %xmm1, %xmm0
+; NOGATHER-NEXT:    retq
+; NOGATHER-NEXT:  .LBB11_1: # %cond.load
+; NOGATHER-NEXT:    vmovq %xmm2, %rcx
 ; NOGATHER-NEXT:    vmovlps {{.*#+}} xmm1 = mem[0,1],xmm1[2,3]
-; NOGATHER-NEXT:  .LBB11_2: # %else
-; NOGATHER-NEXT:    vpextrb $8, %xmm0, %eax
-; NOGATHER-NEXT:    testb $1, %al
+; NOGATHER-NEXT:    testb $2, %al
 ; NOGATHER-NEXT:    je .LBB11_4
-; NOGATHER-NEXT:  # %bb.3: # %cond.load1
+; NOGATHER-NEXT:  .LBB11_3: # %cond.load1
 ; NOGATHER-NEXT:    vpextrq $1, %xmm2, %rax
 ; NOGATHER-NEXT:    vmovhps {{.*#+}} xmm1 = xmm1[0,1],mem[0,1]
-; NOGATHER-NEXT:  .LBB11_4: # %else2
 ; NOGATHER-NEXT:    vmovaps %xmm1, %xmm0
 ; NOGATHER-NEXT:    retq
 entry:
diff --git a/llvm/test/CodeGen/X86/avx2-shift.ll b/llvm/test/CodeGen/X86/avx2-shift.ll
index f4924b3e2..1a0993bd2 100644
--- a/llvm/test/CodeGen/X86/avx2-shift.ll
+++ b/llvm/test/CodeGen/X86/avx2-shift.ll
@@ -580,9 +580,8 @@ define <8 x i16> @variable_lshr16(<8 x i16> %lhs, <8  x i16> %rhs) {
 ; X32-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
 ; X32-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; X32-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm0
-; X32-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; X32-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; X32-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; X32-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; X32-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
 ; X32-NEXT:    vzeroupper
 ; X32-NEXT:    retl
 ;
@@ -591,9 +590,8 @@ define <8 x i16> @variable_lshr16(<8 x i16> %lhs, <8  x i16> %rhs) {
 ; X64-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
 ; X64-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; X64-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm0
-; X64-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; X64-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; X64-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; X64-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; X64-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
 ; X64-NEXT:    vzeroupper
 ; X64-NEXT:    retq
   %res = lshr <8 x i16> %lhs, %rhs
diff --git a/llvm/test/CodeGen/X86/avx2-vbroadcast.ll b/llvm/test/CodeGen/X86/avx2-vbroadcast.ll
index e7dd88196..da77ee5be 100644
--- a/llvm/test/CodeGen/X86/avx2-vbroadcast.ll
+++ b/llvm/test/CodeGen/X86/avx2-vbroadcast.ll
@@ -657,12 +657,12 @@ define <4 x float> @_e2(float* %ptr) nounwind uwtable readnone ssp {
 define <8 x i8> @_e4(i8* %ptr) nounwind uwtable readnone ssp {
 ; X32-LABEL: _e4:
 ; X32:       ## %bb.0:
-; X32-NEXT:    vmovaps {{.*#+}} xmm0 = [52,52,52,52,52,52,52,52]
+; X32-NEXT:    vmovaps {{.*#+}} xmm0 = <52,52,52,52,52,52,52,52,u,u,u,u,u,u,u,u>
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: _e4:
 ; X64:       ## %bb.0:
-; X64-NEXT:    vmovaps {{.*#+}} xmm0 = [52,52,52,52,52,52,52,52]
+; X64-NEXT:    vmovaps {{.*#+}} xmm0 = <52,52,52,52,52,52,52,52,u,u,u,u,u,u,u,u>
 ; X64-NEXT:    retq
   %vecinit0.i = insertelement <8 x i8> undef, i8       52, i32 0
   %vecinit1.i = insertelement <8 x i8> %vecinit0.i, i8 52, i32 1
diff --git a/llvm/test/CodeGen/X86/avx2-vector-shifts.ll b/llvm/test/CodeGen/X86/avx2-vector-shifts.ll
index d95f8a014..905101cd1 100644
--- a/llvm/test/CodeGen/X86/avx2-vector-shifts.ll
+++ b/llvm/test/CodeGen/X86/avx2-vector-shifts.ll
@@ -637,9 +637,8 @@ define <8 x i16> @lshr_8i16(<8 x i16> %r, <8 x i16> %a) nounwind {
 ; X32-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
 ; X32-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; X32-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm0
-; X32-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; X32-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; X32-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; X32-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; X32-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
 ; X32-NEXT:    vzeroupper
 ; X32-NEXT:    retl
 ;
@@ -648,9 +647,8 @@ define <8 x i16> @lshr_8i16(<8 x i16> %r, <8 x i16> %a) nounwind {
 ; X64-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
 ; X64-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; X64-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm0
-; X64-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; X64-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; X64-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; X64-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; X64-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
 ; X64-NEXT:    vzeroupper
 ; X64-NEXT:    retq
   %lshr = lshr <8 x i16> %r, %a
diff --git a/llvm/test/CodeGen/X86/avx512-any_extend_load.ll b/llvm/test/CodeGen/X86/avx512-any_extend_load.ll
index dd80a30fe..4c257a0ed 100644
--- a/llvm/test/CodeGen/X86/avx512-any_extend_load.ll
+++ b/llvm/test/CodeGen/X86/avx512-any_extend_load.ll
@@ -4,13 +4,25 @@
 
 
 define void @any_extend_load_v8i64(<8 x i8> * %ptr) {
-; ALL-LABEL: any_extend_load_v8i64:
-; ALL:       # %bb.0:
-; ALL-NEXT:    vpmovzxbq {{.*#+}} zmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero,mem[2],zero,zero,zero,zero,zero,zero,zero,mem[3],zero,zero,zero,zero,zero,zero,zero,mem[4],zero,zero,zero,zero,zero,zero,zero,mem[5],zero,zero,zero,zero,zero,zero,zero,mem[6],zero,zero,zero,zero,zero,zero,zero,mem[7],zero,zero,zero,zero,zero,zero,zero
-; ALL-NEXT:    vpaddq {{.*}}(%rip){1to8}, %zmm0, %zmm0
-; ALL-NEXT:    vpmovqb %zmm0, (%rdi)
-; ALL-NEXT:    vzeroupper
-; ALL-NEXT:    retq
+; KNL-LABEL: any_extend_load_v8i64:
+; KNL:       # %bb.0:
+; KNL-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
+; KNL-NEXT:    vpmovzxbq {{.*#+}} ymm1 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero,xmm0[2],zero,zero,zero,zero,zero,zero,zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero
+; KNL-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; KNL-NEXT:    vpmovzxbq {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero,xmm0[2],zero,zero,zero,zero,zero,zero,zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero
+; KNL-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; KNL-NEXT:    vpaddq {{.*}}(%rip){1to8}, %zmm0, %zmm0
+; KNL-NEXT:    vpmovqb %zmm0, (%rdi)
+; KNL-NEXT:    vzeroupper
+; KNL-NEXT:    retq
+;
+; SKX-LABEL: any_extend_load_v8i64:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpmovzxbq {{.*#+}} zmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero,mem[2],zero,zero,zero,zero,zero,zero,zero,mem[3],zero,zero,zero,zero,zero,zero,zero,mem[4],zero,zero,zero,zero,zero,zero,zero,mem[5],zero,zero,zero,zero,zero,zero,zero,mem[6],zero,zero,zero,zero,zero,zero,zero,mem[7],zero,zero,zero,zero,zero,zero,zero
+; SKX-NEXT:    vpaddq {{.*}}(%rip){1to8}, %zmm0, %zmm0
+; SKX-NEXT:    vpmovqb %zmm0, (%rdi)
+; SKX-NEXT:    vzeroupper
+; SKX-NEXT:    retq
   %wide.load = load <8 x i8>, <8 x i8>* %ptr, align 1
   %1 = zext <8 x i8> %wide.load to <8 x i64>
   %2 = add nuw nsw <8 x i64> %1, <i64 4, i64 4, i64 4, i64 4, i64 4, i64 4, i64 4, i64 4>
@@ -23,10 +35,12 @@ define void @any_extend_load_v8i64(<8 x i8> * %ptr) {
 define void @any_extend_load_v8i32(<8 x i8> * %ptr) {
 ; KNL-LABEL: any_extend_load_v8i32:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    vpmovzxbw {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
-; KNL-NEXT:    vpaddw {{.*}}(%rip), %xmm0, %xmm0
-; KNL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
+; KNL-NEXT:    vpmovzxbd {{.*#+}} ymm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
+; KNL-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [4,4,4,4,4,4,4,4]
+; KNL-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; KNL-NEXT:    vpmovdb %zmm0, %xmm0
 ; KNL-NEXT:    vmovq %xmm0, (%rdi)
+; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: any_extend_load_v8i32:
@@ -49,8 +63,8 @@ define void @any_extend_load_v8i16(<8 x i8> * %ptr) {
 ; KNL-LABEL: any_extend_load_v8i16:
 ; KNL:       # %bb.0:
 ; KNL-NEXT:    vpmovzxbw {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
-; KNL-NEXT:    vpaddb {{.*}}(%rip), %xmm0, %xmm0
-; KNL-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
+; KNL-NEXT:    vpaddw {{.*}}(%rip), %xmm0, %xmm0
+; KNL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
 ; KNL-NEXT:    vmovq %xmm0, (%rdi)
 ; KNL-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/avx512-arith.ll b/llvm/test/CodeGen/X86/avx512-arith.ll
index 29793a7e0..be88e3530 100644
--- a/llvm/test/CodeGen/X86/avx512-arith.ll
+++ b/llvm/test/CodeGen/X86/avx512-arith.ll
@@ -190,6 +190,52 @@ define <4 x i64> @imulq256(<4 x i64> %y, <4 x i64> %x) {
   ret <4 x i64>%z
 }
 
+define <4 x i64> @imulq256_bcast(<4 x i64> %x) {
+; AVX512F-LABEL: imulq256_bcast:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [1337,1337,1337,1337]
+; AVX512F-NEXT:    vpmuludq %ymm1, %ymm0, %ymm2
+; AVX512F-NEXT:    vpsrlq $32, %ymm0, %ymm0
+; AVX512F-NEXT:    vpmuludq %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vpsllq $32, %ymm0, %ymm0
+; AVX512F-NEXT:    vpaddq %ymm0, %ymm2, %ymm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: imulq256_bcast:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [1337,1337,1337,1337]
+; AVX512VL-NEXT:    vpmuludq %ymm1, %ymm0, %ymm2
+; AVX512VL-NEXT:    vpsrlq $32, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpmuludq %ymm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpsllq $32, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpaddq %ymm0, %ymm2, %ymm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: imulq256_bcast:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [1337,1337,1337,1337]
+; AVX512BW-NEXT:    vpmuludq %ymm1, %ymm0, %ymm2
+; AVX512BW-NEXT:    vpsrlq $32, %ymm0, %ymm0
+; AVX512BW-NEXT:    vpmuludq %ymm1, %ymm0, %ymm0
+; AVX512BW-NEXT:    vpsllq $32, %ymm0, %ymm0
+; AVX512BW-NEXT:    vpaddq %ymm0, %ymm2, %ymm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512DQ-LABEL: imulq256_bcast:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512DQ-NEXT:    vpmullq {{.*}}(%rip){1to8}, %zmm0, %zmm0
+; AVX512DQ-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512DQ-NEXT:    retq
+;
+; SKX-LABEL: imulq256_bcast:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpmullq {{.*}}(%rip){1to4}, %ymm0, %ymm0
+; SKX-NEXT:    retq
+  %z = mul <4 x i64> %x, <i64 1337, i64 1337, i64 1337, i64 1337>
+  ret <4 x i64>%z
+}
+
 define <2 x i64> @imulq128(<2 x i64> %y, <2 x i64> %x) {
 ; AVX512F-LABEL: imulq128:
 ; AVX512F:       # %bb.0:
@@ -244,6 +290,54 @@ define <2 x i64> @imulq128(<2 x i64> %y, <2 x i64> %x) {
   ret <2 x i64>%z
 }
 
+define <2 x i64> @imulq128_bcast(<2 x i64> %x) {
+; AVX512F-LABEL: imulq128_bcast:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm1 = [8086,8086]
+; AVX512F-NEXT:    vpmuludq %xmm1, %xmm0, %xmm2
+; AVX512F-NEXT:    vpsrlq $32, %xmm0, %xmm0
+; AVX512F-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    vpsllq $32, %xmm0, %xmm0
+; AVX512F-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: imulq128_bcast:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm1 = [8086,8086]
+; AVX512VL-NEXT:    vpmuludq %xmm1, %xmm0, %xmm2
+; AVX512VL-NEXT:    vpsrlq $32, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpsllq $32, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: imulq128_bcast:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm1 = [8086,8086]
+; AVX512BW-NEXT:    vpmuludq %xmm1, %xmm0, %xmm2
+; AVX512BW-NEXT:    vpsrlq $32, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpsllq $32, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512DQ-LABEL: imulq128_bcast:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm1 = [8086,8086]
+; AVX512DQ-NEXT:    vpmullq %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    retq
+;
+; SKX-LABEL: imulq128_bcast:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpmullq {{.*}}(%rip), %xmm0, %xmm0
+; SKX-NEXT:    retq
+  %z = mul <2 x i64> %x, <i64 8086, i64 8086>
+  ret <2 x i64>%z
+}
+
 define <8 x double> @mulpd512(<8 x double> %y, <8 x double> %x) {
 ; CHECK-LABEL: mulpd512:
 ; CHECK:       # %bb.0: # %entry
diff --git a/llvm/test/CodeGen/X86/avx512-broadcast-unfold.ll b/llvm/test/CodeGen/X86/avx512-broadcast-unfold.ll
new file mode 100644
index 000000000..aa07a4db8
--- /dev/null
+++ b/llvm/test/CodeGen/X86/avx512-broadcast-unfold.ll
@@ -0,0 +1,4635 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-pc-linux -mattr=avx512vl | FileCheck %s
+
+; Test that we can unfold constant pool loads when we're using avx512's
+; ability to fold a broadcast load into an operation.
+
+define void @bcast_unfold_add_v16i32(i32* %arg) {
+; CHECK-LABEL: bcast_unfold_add_v16i32:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
+; CHECK-NEXT:    vpbroadcastd {{.*#+}} zmm0 = [2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB0_1: # %bb2
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vpaddd 4096(%rdi,%rax), %zmm0, %zmm1
+; CHECK-NEXT:    vmovdqu64 %zmm1, 4096(%rdi,%rax)
+; CHECK-NEXT:    addq $64, %rax
+; CHECK-NEXT:    jne .LBB0_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb2
+
+bb2:                                              ; preds = %bb2, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb2 ]
+  %tmp3 = getelementptr inbounds i32, i32* %arg, i64 %tmp
+  %tmp4 = bitcast i32* %tmp3 to <16 x i32>*
+  %tmp5 = load <16 x i32>, <16 x i32>* %tmp4, align 4
+  %tmp6 = add nsw <16 x i32> %tmp5, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  %tmp7 = bitcast i32* %tmp3 to <16 x i32>*
+  store <16 x i32> %tmp6, <16 x i32>* %tmp7, align 4
+  %tmp8 = add i64 %tmp, 16
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb2
+
+bb10:                                             ; preds = %bb2
+  ret void
+}
+
+define void @bcast_unfold_add_v8i32(i32* %arg) {
+; CHECK-LABEL: bcast_unfold_add_v8i32:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
+; CHECK-NEXT:    vpbroadcastd {{.*#+}} ymm0 = [2,2,2,2,2,2,2,2]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB1_1: # %bb2
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vpaddd 4096(%rdi,%rax), %ymm0, %ymm1
+; CHECK-NEXT:    vmovdqu %ymm1, 4096(%rdi,%rax)
+; CHECK-NEXT:    addq $32, %rax
+; CHECK-NEXT:    jne .LBB1_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb2
+
+bb2:                                              ; preds = %bb2, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb2 ]
+  %tmp3 = getelementptr inbounds i32, i32* %arg, i64 %tmp
+  %tmp4 = bitcast i32* %tmp3 to <8 x i32>*
+  %tmp5 = load <8 x i32>, <8 x i32>* %tmp4, align 4
+  %tmp6 = add nsw <8 x i32> %tmp5, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  %tmp7 = bitcast i32* %tmp3 to <8 x i32>*
+  store <8 x i32> %tmp6, <8 x i32>* %tmp7, align 4
+  %tmp8 = add i64 %tmp, 8
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb2
+
+bb10:                                             ; preds = %bb2
+  ret void
+}
+
+define void @bcast_unfold_add_v4i32(i32* %arg) {
+; CHECK-LABEL: bcast_unfold_add_v4i32:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
+; CHECK-NEXT:    vpbroadcastd {{.*#+}} xmm0 = [2,2,2,2]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB2_1: # %bb2
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vpaddd 4096(%rdi,%rax), %xmm0, %xmm1
+; CHECK-NEXT:    vmovdqu %xmm1, 4096(%rdi,%rax)
+; CHECK-NEXT:    addq $16, %rax
+; CHECK-NEXT:    jne .LBB2_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    retq
+bb:
+  br label %bb2
+
+bb2:                                              ; preds = %bb2, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb2 ]
+  %tmp3 = getelementptr inbounds i32, i32* %arg, i64 %tmp
+  %tmp4 = bitcast i32* %tmp3 to <4 x i32>*
+  %tmp5 = load <4 x i32>, <4 x i32>* %tmp4, align 4
+  %tmp6 = add nsw <4 x i32> %tmp5, <i32 2, i32 2, i32 2, i32 2>
+  %tmp7 = bitcast i32* %tmp3 to <4 x i32>*
+  store <4 x i32> %tmp6, <4 x i32>* %tmp7, align 4
+  %tmp8 = add i64 %tmp, 4
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb2
+
+bb10:                                             ; preds = %bb2
+  ret void
+}
+
+define void @bcast_unfold_add_v8i64(i64* %arg) {
+; CHECK-LABEL: bcast_unfold_add_v8i64:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
+; CHECK-NEXT:    vpbroadcastq {{.*#+}} zmm0 = [2,2,2,2,2,2,2,2]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB3_1: # %bb2
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vpaddq 8192(%rdi,%rax), %zmm0, %zmm1
+; CHECK-NEXT:    vmovdqu64 %zmm1, 8192(%rdi,%rax)
+; CHECK-NEXT:    addq $64, %rax
+; CHECK-NEXT:    jne .LBB3_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb2
+
+bb2:                                              ; preds = %bb2, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb2 ]
+  %tmp3 = getelementptr inbounds i64, i64* %arg, i64 %tmp
+  %tmp4 = bitcast i64* %tmp3 to <8 x i64>*
+  %tmp5 = load <8 x i64>, <8 x i64>* %tmp4, align 8
+  %tmp6 = add nsw <8 x i64> %tmp5, <i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2>
+  %tmp7 = bitcast i64* %tmp3 to <8 x i64>*
+  store <8 x i64> %tmp6, <8 x i64>* %tmp7, align 8
+  %tmp8 = add i64 %tmp, 8
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb2
+
+bb10:                                             ; preds = %bb2
+  ret void
+}
+
+define void @bcast_unfold_add_v4i64(i64* %arg) {
+; CHECK-LABEL: bcast_unfold_add_v4i64:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
+; CHECK-NEXT:    vpbroadcastq {{.*#+}} ymm0 = [2,2,2,2]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB4_1: # %bb2
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vpaddq 8192(%rdi,%rax), %ymm0, %ymm1
+; CHECK-NEXT:    vmovdqu %ymm1, 8192(%rdi,%rax)
+; CHECK-NEXT:    addq $32, %rax
+; CHECK-NEXT:    jne .LBB4_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb2
+
+bb2:                                              ; preds = %bb2, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb2 ]
+  %tmp3 = getelementptr inbounds i64, i64* %arg, i64 %tmp
+  %tmp4 = bitcast i64* %tmp3 to <4 x i64>*
+  %tmp5 = load <4 x i64>, <4 x i64>* %tmp4, align 8
+  %tmp6 = add nsw <4 x i64> %tmp5, <i64 2, i64 2, i64 2, i64 2>
+  %tmp7 = bitcast i64* %tmp3 to <4 x i64>*
+  store <4 x i64> %tmp6, <4 x i64>* %tmp7, align 8
+  %tmp8 = add i64 %tmp, 4
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb2
+
+bb10:                                             ; preds = %bb2
+  ret void
+}
+
+define void @bcast_unfold_add_v2i64(i64* %arg) {
+; CHECK-LABEL: bcast_unfold_add_v2i64:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
+; CHECK-NEXT:    vmovdqa {{.*#+}} xmm0 = [2,2]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB5_1: # %bb2
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vpaddq 8192(%rdi,%rax), %xmm0, %xmm1
+; CHECK-NEXT:    vmovdqu %xmm1, 8192(%rdi,%rax)
+; CHECK-NEXT:    addq $16, %rax
+; CHECK-NEXT:    jne .LBB5_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    retq
+bb:
+  br label %bb2
+
+bb2:                                              ; preds = %bb2, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb2 ]
+  %tmp3 = getelementptr inbounds i64, i64* %arg, i64 %tmp
+  %tmp4 = bitcast i64* %tmp3 to <2 x i64>*
+  %tmp5 = load <2 x i64>, <2 x i64>* %tmp4, align 8
+  %tmp6 = add nsw <2 x i64> %tmp5, <i64 2, i64 2>
+  %tmp7 = bitcast i64* %tmp3 to <2 x i64>*
+  store <2 x i64> %tmp6, <2 x i64>* %tmp7, align 8
+  %tmp8 = add i64 %tmp, 2
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb2
+
+bb10:                                             ; preds = %bb2
+  ret void
+}
+
+define void @bcast_unfold_mul_v16i32(i32* %arg) {
+; CHECK-LABEL: bcast_unfold_mul_v16i32:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
+; CHECK-NEXT:    vpbroadcastd {{.*#+}} zmm0 = [3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB6_1: # %bb2
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vpmulld 4096(%rdi,%rax), %zmm0, %zmm1
+; CHECK-NEXT:    vmovdqu64 %zmm1, 4096(%rdi,%rax)
+; CHECK-NEXT:    addq $64, %rax
+; CHECK-NEXT:    jne .LBB6_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb2
+
+bb2:                                              ; preds = %bb2, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb2 ]
+  %tmp3 = getelementptr inbounds i32, i32* %arg, i64 %tmp
+  %tmp4 = bitcast i32* %tmp3 to <16 x i32>*
+  %tmp5 = load <16 x i32>, <16 x i32>* %tmp4, align 4
+  %tmp6 = mul nsw <16 x i32> %tmp5, <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %tmp7 = bitcast i32* %tmp3 to <16 x i32>*
+  store <16 x i32> %tmp6, <16 x i32>* %tmp7, align 4
+  %tmp8 = add i64 %tmp, 16
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb2
+
+bb10:                                             ; preds = %bb2
+  ret void
+}
+
+define void @bcast_unfold_mul_v8i32(i32* %arg) {
+; CHECK-LABEL: bcast_unfold_mul_v8i32:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
+; CHECK-NEXT:    vpbroadcastd {{.*#+}} ymm0 = [3,3,3,3,3,3,3,3]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB7_1: # %bb2
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vpmulld 4096(%rdi,%rax), %ymm0, %ymm1
+; CHECK-NEXT:    vmovdqu %ymm1, 4096(%rdi,%rax)
+; CHECK-NEXT:    addq $32, %rax
+; CHECK-NEXT:    jne .LBB7_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb2
+
+bb2:                                              ; preds = %bb2, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb2 ]
+  %tmp3 = getelementptr inbounds i32, i32* %arg, i64 %tmp
+  %tmp4 = bitcast i32* %tmp3 to <8 x i32>*
+  %tmp5 = load <8 x i32>, <8 x i32>* %tmp4, align 4
+  %tmp6 = mul nsw <8 x i32> %tmp5, <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %tmp7 = bitcast i32* %tmp3 to <8 x i32>*
+  store <8 x i32> %tmp6, <8 x i32>* %tmp7, align 4
+  %tmp8 = add i64 %tmp, 8
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb2
+
+bb10:                                             ; preds = %bb2
+  ret void
+}
+
+define void @bcast_unfold_mul_v4i32(i32* %arg) {
+; CHECK-LABEL: bcast_unfold_mul_v4i32:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
+; CHECK-NEXT:    vpbroadcastd {{.*#+}} xmm0 = [3,3,3,3]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB8_1: # %bb2
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vpmulld 4096(%rdi,%rax), %xmm0, %xmm1
+; CHECK-NEXT:    vmovdqu %xmm1, 4096(%rdi,%rax)
+; CHECK-NEXT:    addq $16, %rax
+; CHECK-NEXT:    jne .LBB8_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    retq
+bb:
+  br label %bb2
+
+bb2:                                              ; preds = %bb2, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb2 ]
+  %tmp3 = getelementptr inbounds i32, i32* %arg, i64 %tmp
+  %tmp4 = bitcast i32* %tmp3 to <4 x i32>*
+  %tmp5 = load <4 x i32>, <4 x i32>* %tmp4, align 4
+  %tmp6 = mul nsw <4 x i32> %tmp5, <i32 3, i32 3, i32 3, i32 3>
+  %tmp7 = bitcast i32* %tmp3 to <4 x i32>*
+  store <4 x i32> %tmp6, <4 x i32>* %tmp7, align 4
+  %tmp8 = add i64 %tmp, 4
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb2
+
+bb10:                                             ; preds = %bb2
+  ret void
+}
+
+define void @bcast_unfold_mul_v8i64(i64* %arg) {
+; CHECK-LABEL: bcast_unfold_mul_v8i64:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB9_1: # %bb2
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovdqu64 8192(%rdi,%rax), %zmm0
+; CHECK-NEXT:    vpaddq %zmm0, %zmm0, %zmm1
+; CHECK-NEXT:    vpaddq %zmm1, %zmm0, %zmm0
+; CHECK-NEXT:    vmovdqu64 %zmm0, 8192(%rdi,%rax)
+; CHECK-NEXT:    addq $64, %rax
+; CHECK-NEXT:    jne .LBB9_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb2
+
+bb2:                                              ; preds = %bb2, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb2 ]
+  %tmp3 = getelementptr inbounds i64, i64* %arg, i64 %tmp
+  %tmp4 = bitcast i64* %tmp3 to <8 x i64>*
+  %tmp5 = load <8 x i64>, <8 x i64>* %tmp4, align 8
+  %tmp6 = mul nsw <8 x i64> %tmp5, <i64 3, i64 3, i64 3, i64 3, i64 3, i64 3, i64 3, i64 3>
+  %tmp7 = bitcast i64* %tmp3 to <8 x i64>*
+  store <8 x i64> %tmp6, <8 x i64>* %tmp7, align 8
+  %tmp8 = add i64 %tmp, 8
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb2
+
+bb10:                                             ; preds = %bb2
+  ret void
+}
+
+define void @bcast_unfold_mul_v4i64(i64* %arg) {
+; CHECK-LABEL: bcast_unfold_mul_v4i64:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB10_1: # %bb2
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovdqu 8192(%rdi,%rax), %ymm0
+; CHECK-NEXT:    vpaddq %ymm0, %ymm0, %ymm1
+; CHECK-NEXT:    vpaddq %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    vmovdqu %ymm0, 8192(%rdi,%rax)
+; CHECK-NEXT:    addq $32, %rax
+; CHECK-NEXT:    jne .LBB10_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb2
+
+bb2:                                              ; preds = %bb2, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb2 ]
+  %tmp3 = getelementptr inbounds i64, i64* %arg, i64 %tmp
+  %tmp4 = bitcast i64* %tmp3 to <4 x i64>*
+  %tmp5 = load <4 x i64>, <4 x i64>* %tmp4, align 8
+  %tmp6 = mul nsw <4 x i64> %tmp5, <i64 3, i64 3, i64 3, i64 3>
+  %tmp7 = bitcast i64* %tmp3 to <4 x i64>*
+  store <4 x i64> %tmp6, <4 x i64>* %tmp7, align 8
+  %tmp8 = add i64 %tmp, 4
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb2
+
+bb10:                                             ; preds = %bb2
+  ret void
+}
+
+define void @bcast_unfold_mul_v2i64(i64* %arg) {
+; CHECK-LABEL: bcast_unfold_mul_v2i64:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB11_1: # %bb2
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovdqu 8192(%rdi,%rax), %xmm0
+; CHECK-NEXT:    vpaddq %xmm0, %xmm0, %xmm1
+; CHECK-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    vmovdqu %xmm0, 8192(%rdi,%rax)
+; CHECK-NEXT:    addq $16, %rax
+; CHECK-NEXT:    jne .LBB11_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    retq
+bb:
+  br label %bb2
+
+bb2:                                              ; preds = %bb2, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb2 ]
+  %tmp3 = getelementptr inbounds i64, i64* %arg, i64 %tmp
+  %tmp4 = bitcast i64* %tmp3 to <2 x i64>*
+  %tmp5 = load <2 x i64>, <2 x i64>* %tmp4, align 8
+  %tmp6 = mul nsw <2 x i64> %tmp5, <i64 3, i64 3>
+  %tmp7 = bitcast i64* %tmp3 to <2 x i64>*
+  store <2 x i64> %tmp6, <2 x i64>* %tmp7, align 8
+  %tmp8 = add i64 %tmp, 2
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb2
+
+bb10:                                             ; preds = %bb2
+  ret void
+}
+
+define void @bcast_unfold_or_v16i32(i32* %arg) {
+; CHECK-LABEL: bcast_unfold_or_v16i32:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
+; CHECK-NEXT:    vpbroadcastd {{.*#+}} zmm0 = [3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB12_1: # %bb2
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vpord 4096(%rdi,%rax), %zmm0, %zmm1
+; CHECK-NEXT:    vmovdqu64 %zmm1, 4096(%rdi,%rax)
+; CHECK-NEXT:    addq $64, %rax
+; CHECK-NEXT:    jne .LBB12_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb2
+
+bb2:                                              ; preds = %bb2, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb2 ]
+  %tmp3 = getelementptr inbounds i32, i32* %arg, i64 %tmp
+  %tmp4 = bitcast i32* %tmp3 to <16 x i32>*
+  %tmp5 = load <16 x i32>, <16 x i32>* %tmp4, align 4
+  %tmp6 = or <16 x i32> %tmp5, <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %tmp7 = bitcast i32* %tmp3 to <16 x i32>*
+  store <16 x i32> %tmp6, <16 x i32>* %tmp7, align 4
+  %tmp8 = add i64 %tmp, 16
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb2
+
+bb10:                                             ; preds = %bb2
+  ret void
+}
+
+define void @bcast_unfold_or_v8i32(i32* %arg) {
+; CHECK-LABEL: bcast_unfold_or_v8i32:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
+; CHECK-NEXT:    vbroadcastss {{.*#+}} ymm0 = [3,3,3,3,3,3,3,3]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB13_1: # %bb2
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vorps 4096(%rdi,%rax), %ymm0, %ymm1
+; CHECK-NEXT:    vmovups %ymm1, 4096(%rdi,%rax)
+; CHECK-NEXT:    addq $32, %rax
+; CHECK-NEXT:    jne .LBB13_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb2
+
+bb2:                                              ; preds = %bb2, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb2 ]
+  %tmp3 = getelementptr inbounds i32, i32* %arg, i64 %tmp
+  %tmp4 = bitcast i32* %tmp3 to <8 x i32>*
+  %tmp5 = load <8 x i32>, <8 x i32>* %tmp4, align 4
+  %tmp6 = or <8 x i32> %tmp5, <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %tmp7 = bitcast i32* %tmp3 to <8 x i32>*
+  store <8 x i32> %tmp6, <8 x i32>* %tmp7, align 4
+  %tmp8 = add i64 %tmp, 8
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb2
+
+bb10:                                             ; preds = %bb2
+  ret void
+}
+
+define void @bcast_unfold_or_v4i32(i32* %arg) {
+; CHECK-LABEL: bcast_unfold_or_v4i32:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
+; CHECK-NEXT:    vbroadcastss {{.*#+}} xmm0 = [3,3,3,3]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB14_1: # %bb2
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vorps 4096(%rdi,%rax), %xmm0, %xmm1
+; CHECK-NEXT:    vmovups %xmm1, 4096(%rdi,%rax)
+; CHECK-NEXT:    addq $16, %rax
+; CHECK-NEXT:    jne .LBB14_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    retq
+bb:
+  br label %bb2
+
+bb2:                                              ; preds = %bb2, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb2 ]
+  %tmp3 = getelementptr inbounds i32, i32* %arg, i64 %tmp
+  %tmp4 = bitcast i32* %tmp3 to <4 x i32>*
+  %tmp5 = load <4 x i32>, <4 x i32>* %tmp4, align 4
+  %tmp6 = or <4 x i32> %tmp5, <i32 3, i32 3, i32 3, i32 3>
+  %tmp7 = bitcast i32* %tmp3 to <4 x i32>*
+  store <4 x i32> %tmp6, <4 x i32>* %tmp7, align 4
+  %tmp8 = add i64 %tmp, 4
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb2
+
+bb10:                                             ; preds = %bb2
+  ret void
+}
+
+define void @bcast_unfold_or_v8i64(i64* %arg) {
+; CHECK-LABEL: bcast_unfold_or_v8i64:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
+; CHECK-NEXT:    vpbroadcastq {{.*#+}} zmm0 = [3,3,3,3,3,3,3,3]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB15_1: # %bb2
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vporq 8192(%rdi,%rax), %zmm0, %zmm1
+; CHECK-NEXT:    vmovdqu64 %zmm1, 8192(%rdi,%rax)
+; CHECK-NEXT:    addq $64, %rax
+; CHECK-NEXT:    jne .LBB15_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb2
+
+bb2:                                              ; preds = %bb2, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb2 ]
+  %tmp3 = getelementptr inbounds i64, i64* %arg, i64 %tmp
+  %tmp4 = bitcast i64* %tmp3 to <8 x i64>*
+  %tmp5 = load <8 x i64>, <8 x i64>* %tmp4, align 8
+  %tmp6 = or <8 x i64> %tmp5, <i64 3, i64 3, i64 3, i64 3, i64 3, i64 3, i64 3, i64 3>
+  %tmp7 = bitcast i64* %tmp3 to <8 x i64>*
+  store <8 x i64> %tmp6, <8 x i64>* %tmp7, align 8
+  %tmp8 = add i64 %tmp, 8
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb2
+
+bb10:                                             ; preds = %bb2
+  ret void
+}
+
+define void @bcast_unfold_or_v4i64(i64* %arg) {
+; CHECK-LABEL: bcast_unfold_or_v4i64:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
+; CHECK-NEXT:    vbroadcastsd {{.*#+}} ymm0 = [3,3,3,3]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB16_1: # %bb2
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vorps 8192(%rdi,%rax), %ymm0, %ymm1
+; CHECK-NEXT:    vmovups %ymm1, 8192(%rdi,%rax)
+; CHECK-NEXT:    addq $32, %rax
+; CHECK-NEXT:    jne .LBB16_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb2
+
+bb2:                                              ; preds = %bb2, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb2 ]
+  %tmp3 = getelementptr inbounds i64, i64* %arg, i64 %tmp
+  %tmp4 = bitcast i64* %tmp3 to <4 x i64>*
+  %tmp5 = load <4 x i64>, <4 x i64>* %tmp4, align 8
+  %tmp6 = or <4 x i64> %tmp5, <i64 3, i64 3, i64 3, i64 3>
+  %tmp7 = bitcast i64* %tmp3 to <4 x i64>*
+  store <4 x i64> %tmp6, <4 x i64>* %tmp7, align 8
+  %tmp8 = add i64 %tmp, 4
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb2
+
+bb10:                                             ; preds = %bb2
+  ret void
+}
+
+define void @bcast_unfold_or_v2i64(i64* %arg) {
+; CHECK-LABEL: bcast_unfold_or_v2i64:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
+; CHECK-NEXT:    vmovaps {{.*#+}} xmm0 = [3,3]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB17_1: # %bb2
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vorps 8192(%rdi,%rax), %xmm0, %xmm1
+; CHECK-NEXT:    vmovups %xmm1, 8192(%rdi,%rax)
+; CHECK-NEXT:    addq $16, %rax
+; CHECK-NEXT:    jne .LBB17_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    retq
+bb:
+  br label %bb2
+
+bb2:                                              ; preds = %bb2, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb2 ]
+  %tmp3 = getelementptr inbounds i64, i64* %arg, i64 %tmp
+  %tmp4 = bitcast i64* %tmp3 to <2 x i64>*
+  %tmp5 = load <2 x i64>, <2 x i64>* %tmp4, align 8
+  %tmp6 = or <2 x i64> %tmp5, <i64 3, i64 3>
+  %tmp7 = bitcast i64* %tmp3 to <2 x i64>*
+  store <2 x i64> %tmp6, <2 x i64>* %tmp7, align 8
+  %tmp8 = add i64 %tmp, 2
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb2
+
+bb10:                                             ; preds = %bb2
+  ret void
+}
+
+define void @bcast_unfold_fneg_v16f32(float* %arg) {
+; CHECK-LABEL: bcast_unfold_fneg_v16f32:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
+; CHECK-NEXT:    vpbroadcastd {{.*#+}} zmm0 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB18_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vpxord 4096(%rdi,%rax), %zmm0, %zmm1
+; CHECK-NEXT:    vmovdqu64 %zmm1, 4096(%rdi,%rax)
+; CHECK-NEXT:    addq $64, %rax
+; CHECK-NEXT:    jne .LBB18_1
+; CHECK-NEXT:  # %bb.2: # %bb9
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp7, %bb1 ]
+  %tmp2 = getelementptr inbounds float, float* %arg, i64 %tmp
+  %tmp3 = bitcast float* %tmp2 to <16 x float>*
+  %tmp4 = load <16 x float>, <16 x float>* %tmp3, align 4
+  %tmp5 = fneg <16 x float> %tmp4
+  %tmp6 = bitcast float* %tmp2 to <16 x float>*
+  store <16 x float> %tmp5, <16 x float>* %tmp6, align 4
+  %tmp7 = add i64 %tmp, 16
+  %tmp8 = icmp eq i64 %tmp7, 1024
+  br i1 %tmp8, label %bb9, label %bb1
+
+bb9:                                              ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_fneg_v8f32(float* %arg) {
+; CHECK-LABEL: bcast_unfold_fneg_v8f32:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
+; CHECK-NEXT:    vbroadcastss {{.*#+}} ymm0 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB19_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vxorps 4096(%rdi,%rax), %ymm0, %ymm1
+; CHECK-NEXT:    vmovups %ymm1, 4096(%rdi,%rax)
+; CHECK-NEXT:    addq $32, %rax
+; CHECK-NEXT:    jne .LBB19_1
+; CHECK-NEXT:  # %bb.2: # %bb9
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp7, %bb1 ]
+  %tmp2 = getelementptr inbounds float, float* %arg, i64 %tmp
+  %tmp3 = bitcast float* %tmp2 to <8 x float>*
+  %tmp4 = load <8 x float>, <8 x float>* %tmp3, align 4
+  %tmp5 = fneg <8 x float> %tmp4
+  %tmp6 = bitcast float* %tmp2 to <8 x float>*
+  store <8 x float> %tmp5, <8 x float>* %tmp6, align 4
+  %tmp7 = add i64 %tmp, 8
+  %tmp8 = icmp eq i64 %tmp7, 1024
+  br i1 %tmp8, label %bb9, label %bb1
+
+bb9:                                              ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_fneg_v4f32(float* %arg) {
+; CHECK-LABEL: bcast_unfold_fneg_v4f32:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
+; CHECK-NEXT:    vbroadcastss {{.*#+}} xmm0 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB20_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vxorps 4096(%rdi,%rax), %xmm0, %xmm1
+; CHECK-NEXT:    vmovups %xmm1, 4096(%rdi,%rax)
+; CHECK-NEXT:    addq $16, %rax
+; CHECK-NEXT:    jne .LBB20_1
+; CHECK-NEXT:  # %bb.2: # %bb9
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp7, %bb1 ]
+  %tmp2 = getelementptr inbounds float, float* %arg, i64 %tmp
+  %tmp3 = bitcast float* %tmp2 to <4 x float>*
+  %tmp4 = load <4 x float>, <4 x float>* %tmp3, align 4
+  %tmp5 = fneg <4 x float> %tmp4
+  %tmp6 = bitcast float* %tmp2 to <4 x float>*
+  store <4 x float> %tmp5, <4 x float>* %tmp6, align 4
+  %tmp7 = add i64 %tmp, 4
+  %tmp8 = icmp eq i64 %tmp7, 1024
+  br i1 %tmp8, label %bb9, label %bb1
+
+bb9:                                              ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_fneg_v8f64(double* %arg) {
+; CHECK-LABEL: bcast_unfold_fneg_v8f64:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
+; CHECK-NEXT:    vpbroadcastq {{.*#+}} zmm0 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB21_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vpxorq 8192(%rdi,%rax), %zmm0, %zmm1
+; CHECK-NEXT:    vmovdqu64 %zmm1, 8192(%rdi,%rax)
+; CHECK-NEXT:    addq $64, %rax
+; CHECK-NEXT:    jne .LBB21_1
+; CHECK-NEXT:  # %bb.2: # %bb9
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp7, %bb1 ]
+  %tmp2 = getelementptr inbounds double, double* %arg, i64 %tmp
+  %tmp3 = bitcast double* %tmp2 to <8 x double>*
+  %tmp4 = load <8 x double>, <8 x double>* %tmp3, align 8
+  %tmp5 = fneg <8 x double> %tmp4
+  %tmp6 = bitcast double* %tmp2 to <8 x double>*
+  store <8 x double> %tmp5, <8 x double>* %tmp6, align 8
+  %tmp7 = add i64 %tmp, 8
+  %tmp8 = icmp eq i64 %tmp7, 1024
+  br i1 %tmp8, label %bb9, label %bb1
+
+bb9:                                              ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_fneg_v4f64(double* %arg) {
+; CHECK-LABEL: bcast_unfold_fneg_v4f64:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
+; CHECK-NEXT:    vbroadcastsd {{.*#+}} ymm0 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB22_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vxorps 8192(%rdi,%rax), %ymm0, %ymm1
+; CHECK-NEXT:    vmovups %ymm1, 8192(%rdi,%rax)
+; CHECK-NEXT:    addq $32, %rax
+; CHECK-NEXT:    jne .LBB22_1
+; CHECK-NEXT:  # %bb.2: # %bb9
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp7, %bb1 ]
+  %tmp2 = getelementptr inbounds double, double* %arg, i64 %tmp
+  %tmp3 = bitcast double* %tmp2 to <4 x double>*
+  %tmp4 = load <4 x double>, <4 x double>* %tmp3, align 8
+  %tmp5 = fneg <4 x double> %tmp4
+  %tmp6 = bitcast double* %tmp2 to <4 x double>*
+  store <4 x double> %tmp5, <4 x double>* %tmp6, align 8
+  %tmp7 = add i64 %tmp, 4
+  %tmp8 = icmp eq i64 %tmp7, 1024
+  br i1 %tmp8, label %bb9, label %bb1
+
+bb9:                                              ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_fneg_v2f64(double* %arg) {
+; CHECK-LABEL: bcast_unfold_fneg_v2f64:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
+; CHECK-NEXT:    vmovaps {{.*#+}} xmm0 = [-0.0E+0,-0.0E+0]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB23_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vxorps 8192(%rdi,%rax), %xmm0, %xmm1
+; CHECK-NEXT:    vmovups %xmm1, 8192(%rdi,%rax)
+; CHECK-NEXT:    addq $16, %rax
+; CHECK-NEXT:    jne .LBB23_1
+; CHECK-NEXT:  # %bb.2: # %bb9
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp7, %bb1 ]
+  %tmp2 = getelementptr inbounds double, double* %arg, i64 %tmp
+  %tmp3 = bitcast double* %tmp2 to <2 x double>*
+  %tmp4 = load <2 x double>, <2 x double>* %tmp3, align 8
+  %tmp5 = fneg <2 x double> %tmp4
+  %tmp6 = bitcast double* %tmp2 to <2 x double>*
+  store <2 x double> %tmp5, <2 x double>* %tmp6, align 8
+  %tmp7 = add i64 %tmp, 2
+  %tmp8 = icmp eq i64 %tmp7, 1024
+  br i1 %tmp8, label %bb9, label %bb1
+
+bb9:                                              ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_fabs_v16f32(float* %arg) {
+; CHECK-LABEL: bcast_unfold_fabs_v16f32:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
+; CHECK-NEXT:    vpbroadcastd {{.*#+}} zmm0 = [NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB24_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vpandd 4096(%rdi,%rax), %zmm0, %zmm1
+; CHECK-NEXT:    vmovdqu64 %zmm1, 4096(%rdi,%rax)
+; CHECK-NEXT:    addq $64, %rax
+; CHECK-NEXT:    jne .LBB24_1
+; CHECK-NEXT:  # %bb.2: # %bb9
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp7, %bb1 ]
+  %tmp2 = getelementptr inbounds float, float* %arg, i64 %tmp
+  %tmp3 = bitcast float* %tmp2 to <16 x float>*
+  %tmp4 = load <16 x float>, <16 x float>* %tmp3, align 4
+  %tmp5 = call <16 x float> @llvm.fabs.v16f32(<16 x float> %tmp4)
+  %tmp6 = bitcast float* %tmp2 to <16 x float>*
+  store <16 x float> %tmp5, <16 x float>* %tmp6, align 4
+  %tmp7 = add i64 %tmp, 16
+  %tmp8 = icmp eq i64 %tmp7, 1024
+  br i1 %tmp8, label %bb9, label %bb1
+
+bb9:                                              ; preds = %bb1
+  ret void
+}
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare <16 x float> @llvm.fabs.v16f32(<16 x float>) #0
+
+define void @bcast_unfold_fabs_v8f32(float* %arg) {
+; CHECK-LABEL: bcast_unfold_fabs_v8f32:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
+; CHECK-NEXT:    vbroadcastss {{.*#+}} ymm0 = [NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB25_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vandps 4096(%rdi,%rax), %ymm0, %ymm1
+; CHECK-NEXT:    vmovups %ymm1, 4096(%rdi,%rax)
+; CHECK-NEXT:    addq $32, %rax
+; CHECK-NEXT:    jne .LBB25_1
+; CHECK-NEXT:  # %bb.2: # %bb9
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp7, %bb1 ]
+  %tmp2 = getelementptr inbounds float, float* %arg, i64 %tmp
+  %tmp3 = bitcast float* %tmp2 to <8 x float>*
+  %tmp4 = load <8 x float>, <8 x float>* %tmp3, align 4
+  %tmp5 = call <8 x float> @llvm.fabs.v8f32(<8 x float> %tmp4)
+  %tmp6 = bitcast float* %tmp2 to <8 x float>*
+  store <8 x float> %tmp5, <8 x float>* %tmp6, align 4
+  %tmp7 = add i64 %tmp, 8
+  %tmp8 = icmp eq i64 %tmp7, 1024
+  br i1 %tmp8, label %bb9, label %bb1
+
+bb9:                                              ; preds = %bb1
+  ret void
+}
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare <8 x float> @llvm.fabs.v8f32(<8 x float>) #0
+
+define void @bcast_unfold_fabs_v4f32(float* %arg) {
+; CHECK-LABEL: bcast_unfold_fabs_v4f32:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
+; CHECK-NEXT:    vbroadcastss {{.*#+}} xmm0 = [NaN,NaN,NaN,NaN]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB26_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vandps 4096(%rdi,%rax), %xmm0, %xmm1
+; CHECK-NEXT:    vmovups %xmm1, 4096(%rdi,%rax)
+; CHECK-NEXT:    addq $16, %rax
+; CHECK-NEXT:    jne .LBB26_1
+; CHECK-NEXT:  # %bb.2: # %bb9
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp7, %bb1 ]
+  %tmp2 = getelementptr inbounds float, float* %arg, i64 %tmp
+  %tmp3 = bitcast float* %tmp2 to <4 x float>*
+  %tmp4 = load <4 x float>, <4 x float>* %tmp3, align 4
+  %tmp5 = call <4 x float> @llvm.fabs.v4f32(<4 x float> %tmp4)
+  %tmp6 = bitcast float* %tmp2 to <4 x float>*
+  store <4 x float> %tmp5, <4 x float>* %tmp6, align 4
+  %tmp7 = add i64 %tmp, 4
+  %tmp8 = icmp eq i64 %tmp7, 1024
+  br i1 %tmp8, label %bb9, label %bb1
+
+bb9:                                              ; preds = %bb1
+  ret void
+}
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare <4 x float> @llvm.fabs.v4f32(<4 x float>) #0
+
+define void @bcast_unfold_fabs_v8f64(double* %arg) {
+; CHECK-LABEL: bcast_unfold_fabs_v8f64:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
+; CHECK-NEXT:    vpbroadcastq {{.*#+}} zmm0 = [NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB27_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vpandq 8192(%rdi,%rax), %zmm0, %zmm1
+; CHECK-NEXT:    vmovdqu64 %zmm1, 8192(%rdi,%rax)
+; CHECK-NEXT:    addq $64, %rax
+; CHECK-NEXT:    jne .LBB27_1
+; CHECK-NEXT:  # %bb.2: # %bb9
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp7, %bb1 ]
+  %tmp2 = getelementptr inbounds double, double* %arg, i64 %tmp
+  %tmp3 = bitcast double* %tmp2 to <8 x double>*
+  %tmp4 = load <8 x double>, <8 x double>* %tmp3, align 8
+  %tmp5 = call <8 x double> @llvm.fabs.v8f64(<8 x double> %tmp4)
+  %tmp6 = bitcast double* %tmp2 to <8 x double>*
+  store <8 x double> %tmp5, <8 x double>* %tmp6, align 8
+  %tmp7 = add i64 %tmp, 8
+  %tmp8 = icmp eq i64 %tmp7, 1024
+  br i1 %tmp8, label %bb9, label %bb1
+
+bb9:                                              ; preds = %bb1
+  ret void
+}
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare <8 x double> @llvm.fabs.v8f64(<8 x double>) #0
+
+define void @bcast_unfold_fabs_v4f64(double* %arg) {
+; CHECK-LABEL: bcast_unfold_fabs_v4f64:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
+; CHECK-NEXT:    vbroadcastsd {{.*#+}} ymm0 = [NaN,NaN,NaN,NaN]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB28_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vandps 8192(%rdi,%rax), %ymm0, %ymm1
+; CHECK-NEXT:    vmovups %ymm1, 8192(%rdi,%rax)
+; CHECK-NEXT:    addq $32, %rax
+; CHECK-NEXT:    jne .LBB28_1
+; CHECK-NEXT:  # %bb.2: # %bb9
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp7, %bb1 ]
+  %tmp2 = getelementptr inbounds double, double* %arg, i64 %tmp
+  %tmp3 = bitcast double* %tmp2 to <4 x double>*
+  %tmp4 = load <4 x double>, <4 x double>* %tmp3, align 8
+  %tmp5 = call <4 x double> @llvm.fabs.v4f64(<4 x double> %tmp4)
+  %tmp6 = bitcast double* %tmp2 to <4 x double>*
+  store <4 x double> %tmp5, <4 x double>* %tmp6, align 8
+  %tmp7 = add i64 %tmp, 4
+  %tmp8 = icmp eq i64 %tmp7, 1024
+  br i1 %tmp8, label %bb9, label %bb1
+
+bb9:                                              ; preds = %bb1
+  ret void
+}
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare <4 x double> @llvm.fabs.v4f64(<4 x double>) #0
+
+define void @bcast_unfold_fabs_v2f64(double* %arg) {
+; CHECK-LABEL: bcast_unfold_fabs_v2f64:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
+; CHECK-NEXT:    vmovaps {{.*#+}} xmm0 = [NaN,NaN]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB29_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vandps 8192(%rdi,%rax), %xmm0, %xmm1
+; CHECK-NEXT:    vmovups %xmm1, 8192(%rdi,%rax)
+; CHECK-NEXT:    addq $16, %rax
+; CHECK-NEXT:    jne .LBB29_1
+; CHECK-NEXT:  # %bb.2: # %bb9
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp7, %bb1 ]
+  %tmp2 = getelementptr inbounds double, double* %arg, i64 %tmp
+  %tmp3 = bitcast double* %tmp2 to <2 x double>*
+  %tmp4 = load <2 x double>, <2 x double>* %tmp3, align 8
+  %tmp5 = call <2 x double> @llvm.fabs.v2f64(<2 x double> %tmp4)
+  %tmp6 = bitcast double* %tmp2 to <2 x double>*
+  store <2 x double> %tmp5, <2 x double>* %tmp6, align 8
+  %tmp7 = add i64 %tmp, 2
+  %tmp8 = icmp eq i64 %tmp7, 1024
+  br i1 %tmp8, label %bb9, label %bb1
+
+bb9:                                              ; preds = %bb1
+  ret void
+}
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare <2 x double> @llvm.fabs.v2f64(<2 x double>) #0
+
+define void @bcast_unfold_fadd_v16f32(float* nocapture %arg) {
+; CHECK-LABEL: bcast_unfold_fadd_v16f32:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
+; CHECK-NEXT:    vbroadcastss {{.*#+}} zmm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB30_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vaddps 4096(%rdi,%rax), %zmm0, %zmm1
+; CHECK-NEXT:    vmovups %zmm1, 4096(%rdi,%rax)
+; CHECK-NEXT:    addq $64, %rax
+; CHECK-NEXT:    jne .LBB30_1
+; CHECK-NEXT:  # %bb.2: # %bb9
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp7, %bb1 ]
+  %tmp2 = getelementptr inbounds float, float* %arg, i64 %tmp
+  %tmp3 = bitcast float* %tmp2 to <16 x float>*
+  %tmp4 = load <16 x float>, <16 x float>* %tmp3, align 4
+  %tmp5 = fadd <16 x float> %tmp4, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
+  %tmp6 = bitcast float* %tmp2 to <16 x float>*
+  store <16 x float> %tmp5, <16 x float>* %tmp6, align 4
+  %tmp7 = add i64 %tmp, 16
+  %tmp8 = icmp eq i64 %tmp7, 1024
+  br i1 %tmp8, label %bb9, label %bb1
+
+bb9:                                              ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_fadd_v8f32(float* nocapture %arg) {
+; CHECK-LABEL: bcast_unfold_fadd_v8f32:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
+; CHECK-NEXT:    vbroadcastss {{.*#+}} ymm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB31_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vaddps 4096(%rdi,%rax), %ymm0, %ymm1
+; CHECK-NEXT:    vmovups %ymm1, 4096(%rdi,%rax)
+; CHECK-NEXT:    addq $32, %rax
+; CHECK-NEXT:    jne .LBB31_1
+; CHECK-NEXT:  # %bb.2: # %bb9
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp7, %bb1 ]
+  %tmp2 = getelementptr inbounds float, float* %arg, i64 %tmp
+  %tmp3 = bitcast float* %tmp2 to <8 x float>*
+  %tmp4 = load <8 x float>, <8 x float>* %tmp3, align 4
+  %tmp5 = fadd <8 x float> %tmp4, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
+  %tmp6 = bitcast float* %tmp2 to <8 x float>*
+  store <8 x float> %tmp5, <8 x float>* %tmp6, align 4
+  %tmp7 = add i64 %tmp, 8
+  %tmp8 = icmp eq i64 %tmp7, 1024
+  br i1 %tmp8, label %bb9, label %bb1
+
+bb9:                                              ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_fadd_v4f32(float* nocapture %arg) {
+; CHECK-LABEL: bcast_unfold_fadd_v4f32:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
+; CHECK-NEXT:    vbroadcastss {{.*#+}} xmm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB32_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vaddps 4096(%rdi,%rax), %xmm0, %xmm1
+; CHECK-NEXT:    vmovups %xmm1, 4096(%rdi,%rax)
+; CHECK-NEXT:    addq $16, %rax
+; CHECK-NEXT:    jne .LBB32_1
+; CHECK-NEXT:  # %bb.2: # %bb9
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp7, %bb1 ]
+  %tmp2 = getelementptr inbounds float, float* %arg, i64 %tmp
+  %tmp3 = bitcast float* %tmp2 to <4 x float>*
+  %tmp4 = load <4 x float>, <4 x float>* %tmp3, align 4
+  %tmp5 = fadd <4 x float> %tmp4, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
+  %tmp6 = bitcast float* %tmp2 to <4 x float>*
+  store <4 x float> %tmp5, <4 x float>* %tmp6, align 4
+  %tmp7 = add i64 %tmp, 4
+  %tmp8 = icmp eq i64 %tmp7, 1024
+  br i1 %tmp8, label %bb9, label %bb1
+
+bb9:                                              ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_fadd_v8f64(double* nocapture %arg) {
+; CHECK-LABEL: bcast_unfold_fadd_v8f64:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
+; CHECK-NEXT:    vbroadcastsd {{.*#+}} zmm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB33_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vaddpd 8192(%rdi,%rax), %zmm0, %zmm1
+; CHECK-NEXT:    vmovupd %zmm1, 8192(%rdi,%rax)
+; CHECK-NEXT:    addq $64, %rax
+; CHECK-NEXT:    jne .LBB33_1
+; CHECK-NEXT:  # %bb.2: # %bb9
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp7, %bb1 ]
+  %tmp2 = getelementptr inbounds double, double* %arg, i64 %tmp
+  %tmp3 = bitcast double* %tmp2 to <8 x double>*
+  %tmp4 = load <8 x double>, <8 x double>* %tmp3, align 8
+  %tmp5 = fadd <8 x double> %tmp4, <double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00>
+  %tmp6 = bitcast double* %tmp2 to <8 x double>*
+  store <8 x double> %tmp5, <8 x double>* %tmp6, align 8
+  %tmp7 = add i64 %tmp, 8
+  %tmp8 = icmp eq i64 %tmp7, 1024
+  br i1 %tmp8, label %bb9, label %bb1
+
+bb9:                                              ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_fadd_v4f64(double* nocapture %arg) {
+; CHECK-LABEL: bcast_unfold_fadd_v4f64:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
+; CHECK-NEXT:    vbroadcastsd {{.*#+}} ymm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB34_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vaddpd 8192(%rdi,%rax), %ymm0, %ymm1
+; CHECK-NEXT:    vmovupd %ymm1, 8192(%rdi,%rax)
+; CHECK-NEXT:    addq $32, %rax
+; CHECK-NEXT:    jne .LBB34_1
+; CHECK-NEXT:  # %bb.2: # %bb9
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp7, %bb1 ]
+  %tmp2 = getelementptr inbounds double, double* %arg, i64 %tmp
+  %tmp3 = bitcast double* %tmp2 to <4 x double>*
+  %tmp4 = load <4 x double>, <4 x double>* %tmp3, align 8
+  %tmp5 = fadd <4 x double> %tmp4, <double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00>
+  %tmp6 = bitcast double* %tmp2 to <4 x double>*
+  store <4 x double> %tmp5, <4 x double>* %tmp6, align 8
+  %tmp7 = add i64 %tmp, 4
+  %tmp8 = icmp eq i64 %tmp7, 1024
+  br i1 %tmp8, label %bb9, label %bb1
+
+bb9:                                              ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_fadd_v2f64(double* nocapture %arg) {
+; CHECK-LABEL: bcast_unfold_fadd_v2f64:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
+; CHECK-NEXT:    vmovapd {{.*#+}} xmm0 = [2.0E+0,2.0E+0]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB35_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vaddpd 8192(%rdi,%rax), %xmm0, %xmm1
+; CHECK-NEXT:    vmovupd %xmm1, 8192(%rdi,%rax)
+; CHECK-NEXT:    addq $16, %rax
+; CHECK-NEXT:    jne .LBB35_1
+; CHECK-NEXT:  # %bb.2: # %bb9
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp7, %bb1 ]
+  %tmp2 = getelementptr inbounds double, double* %arg, i64 %tmp
+  %tmp3 = bitcast double* %tmp2 to <2 x double>*
+  %tmp4 = load <2 x double>, <2 x double>* %tmp3, align 8
+  %tmp5 = fadd <2 x double> %tmp4, <double 2.000000e+00, double 2.000000e+00>
+  %tmp6 = bitcast double* %tmp2 to <2 x double>*
+  store <2 x double> %tmp5, <2 x double>* %tmp6, align 8
+  %tmp7 = add i64 %tmp, 2
+  %tmp8 = icmp eq i64 %tmp7, 1024
+  br i1 %tmp8, label %bb9, label %bb1
+
+bb9:                                              ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_fmul_v16f32(float* nocapture %arg) {
+; CHECK-LABEL: bcast_unfold_fmul_v16f32:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
+; CHECK-NEXT:    vbroadcastss {{.*#+}} zmm0 = [3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB36_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmulps 4096(%rdi,%rax), %zmm0, %zmm1
+; CHECK-NEXT:    vmovups %zmm1, 4096(%rdi,%rax)
+; CHECK-NEXT:    addq $64, %rax
+; CHECK-NEXT:    jne .LBB36_1
+; CHECK-NEXT:  # %bb.2: # %bb9
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp7, %bb1 ]
+  %tmp2 = getelementptr inbounds float, float* %arg, i64 %tmp
+  %tmp3 = bitcast float* %tmp2 to <16 x float>*
+  %tmp4 = load <16 x float>, <16 x float>* %tmp3, align 4
+  %tmp5 = fmul <16 x float> %tmp4, <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00>
+  %tmp6 = bitcast float* %tmp2 to <16 x float>*
+  store <16 x float> %tmp5, <16 x float>* %tmp6, align 4
+  %tmp7 = add i64 %tmp, 16
+  %tmp8 = icmp eq i64 %tmp7, 1024
+  br i1 %tmp8, label %bb9, label %bb1
+
+bb9:                                              ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_fmul_v8f32(float* nocapture %arg) {
+; CHECK-LABEL: bcast_unfold_fmul_v8f32:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
+; CHECK-NEXT:    vbroadcastss {{.*#+}} ymm0 = [3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB37_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmulps 4096(%rdi,%rax), %ymm0, %ymm1
+; CHECK-NEXT:    vmovups %ymm1, 4096(%rdi,%rax)
+; CHECK-NEXT:    addq $32, %rax
+; CHECK-NEXT:    jne .LBB37_1
+; CHECK-NEXT:  # %bb.2: # %bb9
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp7, %bb1 ]
+  %tmp2 = getelementptr inbounds float, float* %arg, i64 %tmp
+  %tmp3 = bitcast float* %tmp2 to <8 x float>*
+  %tmp4 = load <8 x float>, <8 x float>* %tmp3, align 4
+  %tmp5 = fmul <8 x float> %tmp4, <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00>
+  %tmp6 = bitcast float* %tmp2 to <8 x float>*
+  store <8 x float> %tmp5, <8 x float>* %tmp6, align 4
+  %tmp7 = add i64 %tmp, 8
+  %tmp8 = icmp eq i64 %tmp7, 1024
+  br i1 %tmp8, label %bb9, label %bb1
+
+bb9:                                              ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_fmul_v4f32(float* nocapture %arg) {
+; CHECK-LABEL: bcast_unfold_fmul_v4f32:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
+; CHECK-NEXT:    vbroadcastss {{.*#+}} xmm0 = [3.0E+0,3.0E+0,3.0E+0,3.0E+0]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB38_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmulps 4096(%rdi,%rax), %xmm0, %xmm1
+; CHECK-NEXT:    vmovups %xmm1, 4096(%rdi,%rax)
+; CHECK-NEXT:    addq $16, %rax
+; CHECK-NEXT:    jne .LBB38_1
+; CHECK-NEXT:  # %bb.2: # %bb9
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp7, %bb1 ]
+  %tmp2 = getelementptr inbounds float, float* %arg, i64 %tmp
+  %tmp3 = bitcast float* %tmp2 to <4 x float>*
+  %tmp4 = load <4 x float>, <4 x float>* %tmp3, align 4
+  %tmp5 = fmul <4 x float> %tmp4, <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00>
+  %tmp6 = bitcast float* %tmp2 to <4 x float>*
+  store <4 x float> %tmp5, <4 x float>* %tmp6, align 4
+  %tmp7 = add i64 %tmp, 4
+  %tmp8 = icmp eq i64 %tmp7, 1024
+  br i1 %tmp8, label %bb9, label %bb1
+
+bb9:                                              ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_fmul_v8f64(double* nocapture %arg) {
+; CHECK-LABEL: bcast_unfold_fmul_v8f64:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
+; CHECK-NEXT:    vbroadcastsd {{.*#+}} zmm0 = [3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB39_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmulpd 8192(%rdi,%rax), %zmm0, %zmm1
+; CHECK-NEXT:    vmovupd %zmm1, 8192(%rdi,%rax)
+; CHECK-NEXT:    addq $64, %rax
+; CHECK-NEXT:    jne .LBB39_1
+; CHECK-NEXT:  # %bb.2: # %bb9
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp7, %bb1 ]
+  %tmp2 = getelementptr inbounds double, double* %arg, i64 %tmp
+  %tmp3 = bitcast double* %tmp2 to <8 x double>*
+  %tmp4 = load <8 x double>, <8 x double>* %tmp3, align 8
+  %tmp5 = fmul <8 x double> %tmp4, <double 3.000000e+00, double 3.000000e+00, double 3.000000e+00, double 3.000000e+00, double 3.000000e+00, double 3.000000e+00, double 3.000000e+00, double 3.000000e+00>
+  %tmp6 = bitcast double* %tmp2 to <8 x double>*
+  store <8 x double> %tmp5, <8 x double>* %tmp6, align 8
+  %tmp7 = add i64 %tmp, 8
+  %tmp8 = icmp eq i64 %tmp7, 1024
+  br i1 %tmp8, label %bb9, label %bb1
+
+bb9:                                              ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_fmul_v4f64(double* nocapture %arg) {
+; CHECK-LABEL: bcast_unfold_fmul_v4f64:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
+; CHECK-NEXT:    vbroadcastsd {{.*#+}} ymm0 = [3.0E+0,3.0E+0,3.0E+0,3.0E+0]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB40_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmulpd 8192(%rdi,%rax), %ymm0, %ymm1
+; CHECK-NEXT:    vmovupd %ymm1, 8192(%rdi,%rax)
+; CHECK-NEXT:    addq $32, %rax
+; CHECK-NEXT:    jne .LBB40_1
+; CHECK-NEXT:  # %bb.2: # %bb9
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp7, %bb1 ]
+  %tmp2 = getelementptr inbounds double, double* %arg, i64 %tmp
+  %tmp3 = bitcast double* %tmp2 to <4 x double>*
+  %tmp4 = load <4 x double>, <4 x double>* %tmp3, align 8
+  %tmp5 = fmul <4 x double> %tmp4, <double 3.000000e+00, double 3.000000e+00, double 3.000000e+00, double 3.000000e+00>
+  %tmp6 = bitcast double* %tmp2 to <4 x double>*
+  store <4 x double> %tmp5, <4 x double>* %tmp6, align 8
+  %tmp7 = add i64 %tmp, 4
+  %tmp8 = icmp eq i64 %tmp7, 1024
+  br i1 %tmp8, label %bb9, label %bb1
+
+bb9:                                              ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_fmul_v2f64(double* nocapture %arg) {
+; CHECK-LABEL: bcast_unfold_fmul_v2f64:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
+; CHECK-NEXT:    vmovapd {{.*#+}} xmm0 = [3.0E+0,3.0E+0]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB41_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmulpd 8192(%rdi,%rax), %xmm0, %xmm1
+; CHECK-NEXT:    vmovupd %xmm1, 8192(%rdi,%rax)
+; CHECK-NEXT:    addq $16, %rax
+; CHECK-NEXT:    jne .LBB41_1
+; CHECK-NEXT:  # %bb.2: # %bb9
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp7, %bb1 ]
+  %tmp2 = getelementptr inbounds double, double* %arg, i64 %tmp
+  %tmp3 = bitcast double* %tmp2 to <2 x double>*
+  %tmp4 = load <2 x double>, <2 x double>* %tmp3, align 8
+  %tmp5 = fmul <2 x double> %tmp4, <double 3.000000e+00, double 3.000000e+00>
+  %tmp6 = bitcast double* %tmp2 to <2 x double>*
+  store <2 x double> %tmp5, <2 x double>* %tmp6, align 8
+  %tmp7 = add i64 %tmp, 2
+  %tmp8 = icmp eq i64 %tmp7, 1024
+  br i1 %tmp8, label %bb9, label %bb1
+
+bb9:                                              ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_fdiv_v16f32(float* nocapture %arg) {
+; CHECK-LABEL: bcast_unfold_fdiv_v16f32:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
+; CHECK-NEXT:    vbroadcastss {{.*#+}} zmm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB42_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovups 4096(%rdi,%rax), %zmm1
+; CHECK-NEXT:    vdivps %zmm0, %zmm1, %zmm1
+; CHECK-NEXT:    vmovups %zmm1, 4096(%rdi,%rax)
+; CHECK-NEXT:    addq $64, %rax
+; CHECK-NEXT:    jne .LBB42_1
+; CHECK-NEXT:  # %bb.2: # %bb9
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp7, %bb1 ]
+  %tmp2 = getelementptr inbounds float, float* %arg, i64 %tmp
+  %tmp3 = bitcast float* %tmp2 to <16 x float>*
+  %tmp4 = load <16 x float>, <16 x float>* %tmp3, align 4
+  %tmp5 = fdiv <16 x float> %tmp4, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
+  %tmp6 = bitcast float* %tmp2 to <16 x float>*
+  store <16 x float> %tmp5, <16 x float>* %tmp6, align 4
+  %tmp7 = add i64 %tmp, 16
+  %tmp8 = icmp eq i64 %tmp7, 1024
+  br i1 %tmp8, label %bb9, label %bb1
+
+bb9:                                              ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_fdiv_v8f32(float* nocapture %arg) {
+; CHECK-LABEL: bcast_unfold_fdiv_v8f32:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
+; CHECK-NEXT:    vbroadcastss {{.*#+}} ymm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB43_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovups 4096(%rdi,%rax), %ymm1
+; CHECK-NEXT:    vdivps %ymm0, %ymm1, %ymm1
+; CHECK-NEXT:    vmovups %ymm1, 4096(%rdi,%rax)
+; CHECK-NEXT:    addq $32, %rax
+; CHECK-NEXT:    jne .LBB43_1
+; CHECK-NEXT:  # %bb.2: # %bb9
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp7, %bb1 ]
+  %tmp2 = getelementptr inbounds float, float* %arg, i64 %tmp
+  %tmp3 = bitcast float* %tmp2 to <8 x float>*
+  %tmp4 = load <8 x float>, <8 x float>* %tmp3, align 4
+  %tmp5 = fdiv <8 x float> %tmp4, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
+  %tmp6 = bitcast float* %tmp2 to <8 x float>*
+  store <8 x float> %tmp5, <8 x float>* %tmp6, align 4
+  %tmp7 = add i64 %tmp, 8
+  %tmp8 = icmp eq i64 %tmp7, 1024
+  br i1 %tmp8, label %bb9, label %bb1
+
+bb9:                                              ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_fdiv_v4f32(float* nocapture %arg) {
+; CHECK-LABEL: bcast_unfold_fdiv_v4f32:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
+; CHECK-NEXT:    vbroadcastss {{.*#+}} xmm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB44_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovups 4096(%rdi,%rax), %xmm1
+; CHECK-NEXT:    vdivps %xmm0, %xmm1, %xmm1
+; CHECK-NEXT:    vmovups %xmm1, 4096(%rdi,%rax)
+; CHECK-NEXT:    addq $16, %rax
+; CHECK-NEXT:    jne .LBB44_1
+; CHECK-NEXT:  # %bb.2: # %bb9
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp7, %bb1 ]
+  %tmp2 = getelementptr inbounds float, float* %arg, i64 %tmp
+  %tmp3 = bitcast float* %tmp2 to <4 x float>*
+  %tmp4 = load <4 x float>, <4 x float>* %tmp3, align 4
+  %tmp5 = fdiv <4 x float> %tmp4, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
+  %tmp6 = bitcast float* %tmp2 to <4 x float>*
+  store <4 x float> %tmp5, <4 x float>* %tmp6, align 4
+  %tmp7 = add i64 %tmp, 4
+  %tmp8 = icmp eq i64 %tmp7, 1024
+  br i1 %tmp8, label %bb9, label %bb1
+
+bb9:                                              ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_fdiv_v8f64(double* nocapture %arg) {
+; CHECK-LABEL: bcast_unfold_fdiv_v8f64:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
+; CHECK-NEXT:    vbroadcastsd {{.*#+}} zmm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB45_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovupd 8192(%rdi,%rax), %zmm1
+; CHECK-NEXT:    vdivpd %zmm0, %zmm1, %zmm1
+; CHECK-NEXT:    vmovupd %zmm1, 8192(%rdi,%rax)
+; CHECK-NEXT:    addq $64, %rax
+; CHECK-NEXT:    jne .LBB45_1
+; CHECK-NEXT:  # %bb.2: # %bb9
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp7, %bb1 ]
+  %tmp2 = getelementptr inbounds double, double* %arg, i64 %tmp
+  %tmp3 = bitcast double* %tmp2 to <8 x double>*
+  %tmp4 = load <8 x double>, <8 x double>* %tmp3, align 8
+  %tmp5 = fdiv <8 x double> %tmp4, <double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00>
+  %tmp6 = bitcast double* %tmp2 to <8 x double>*
+  store <8 x double> %tmp5, <8 x double>* %tmp6, align 8
+  %tmp7 = add i64 %tmp, 8
+  %tmp8 = icmp eq i64 %tmp7, 1024
+  br i1 %tmp8, label %bb9, label %bb1
+
+bb9:                                              ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_fdiv_v4f64(double* nocapture %arg) {
+; CHECK-LABEL: bcast_unfold_fdiv_v4f64:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
+; CHECK-NEXT:    vbroadcastsd {{.*#+}} ymm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB46_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovupd 8192(%rdi,%rax), %ymm1
+; CHECK-NEXT:    vdivpd %ymm0, %ymm1, %ymm1
+; CHECK-NEXT:    vmovupd %ymm1, 8192(%rdi,%rax)
+; CHECK-NEXT:    addq $32, %rax
+; CHECK-NEXT:    jne .LBB46_1
+; CHECK-NEXT:  # %bb.2: # %bb9
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp7, %bb1 ]
+  %tmp2 = getelementptr inbounds double, double* %arg, i64 %tmp
+  %tmp3 = bitcast double* %tmp2 to <4 x double>*
+  %tmp4 = load <4 x double>, <4 x double>* %tmp3, align 8
+  %tmp5 = fdiv <4 x double> %tmp4, <double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00>
+  %tmp6 = bitcast double* %tmp2 to <4 x double>*
+  store <4 x double> %tmp5, <4 x double>* %tmp6, align 8
+  %tmp7 = add i64 %tmp, 4
+  %tmp8 = icmp eq i64 %tmp7, 1024
+  br i1 %tmp8, label %bb9, label %bb1
+
+bb9:                                              ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_fdiv_v2f64(double* nocapture %arg) {
+; CHECK-LABEL: bcast_unfold_fdiv_v2f64:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
+; CHECK-NEXT:    vmovapd {{.*#+}} xmm0 = [2.0E+0,2.0E+0]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB47_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovupd 8192(%rdi,%rax), %xmm1
+; CHECK-NEXT:    vdivpd %xmm0, %xmm1, %xmm1
+; CHECK-NEXT:    vmovupd %xmm1, 8192(%rdi,%rax)
+; CHECK-NEXT:    addq $16, %rax
+; CHECK-NEXT:    jne .LBB47_1
+; CHECK-NEXT:  # %bb.2: # %bb9
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp7, %bb1 ]
+  %tmp2 = getelementptr inbounds double, double* %arg, i64 %tmp
+  %tmp3 = bitcast double* %tmp2 to <2 x double>*
+  %tmp4 = load <2 x double>, <2 x double>* %tmp3, align 8
+  %tmp5 = fdiv <2 x double> %tmp4, <double 2.000000e+00, double 2.000000e+00>
+  %tmp6 = bitcast double* %tmp2 to <2 x double>*
+  store <2 x double> %tmp5, <2 x double>* %tmp6, align 8
+  %tmp7 = add i64 %tmp, 2
+  %tmp8 = icmp eq i64 %tmp7, 1024
+  br i1 %tmp8, label %bb9, label %bb1
+
+bb9:                                              ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_fma213_v4f32(float* %arg) {
+; CHECK-LABEL: bcast_unfold_fma213_v4f32:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
+; CHECK-NEXT:    vbroadcastss {{.*#+}} xmm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB48_1: # %bb2
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovups 4096(%rdi,%rax), %xmm1
+; CHECK-NEXT:    vfmadd213ps {{.*#+}} xmm1 = (xmm1 * xmm1) + xmm0
+; CHECK-NEXT:    vmovups %xmm1, 4096(%rdi,%rax)
+; CHECK-NEXT:    addq $16, %rax
+; CHECK-NEXT:    jne .LBB48_1
+; CHECK-NEXT:  # %bb.2: # %bb11
+; CHECK-NEXT:    retq
+bb:
+  br label %bb2
+
+bb2:                                              ; preds = %bb2, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp9, %bb2 ]
+  %tmp3 = getelementptr inbounds float, float* %arg, i64 %tmp
+  %tmp4 = bitcast float* %tmp3 to <4 x float>*
+  %tmp5 = load <4 x float>, <4 x float>* %tmp4, align 4
+  %tmp6 = fmul contract <4 x float> %tmp5, %tmp5
+  %tmp7 = fadd contract <4 x float> %tmp6, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
+  %tmp8 = bitcast float* %tmp3 to <4 x float>*
+  store <4 x float> %tmp7, <4 x float>* %tmp8, align 4
+  %tmp9 = add i64 %tmp, 4
+  %tmp10 = icmp eq i64 %tmp9, 1024
+  br i1 %tmp10, label %bb11, label %bb2
+
+bb11:                                             ; preds = %bb2
+  ret void
+}
+
+define void @bcast_unfold_fma231_v4f32(float* %arg) {
+; CHECK-LABEL: bcast_unfold_fma231_v4f32:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
+; CHECK-NEXT:    vbroadcastss {{.*#+}} xmm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB49_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovups 4096(%rdi,%rax), %xmm1
+; CHECK-NEXT:    vfmadd231ps {{.*#+}} xmm1 = (xmm1 * xmm0) + xmm1
+; CHECK-NEXT:    vmovups %xmm1, 4096(%rdi,%rax)
+; CHECK-NEXT:    addq $16, %rax
+; CHECK-NEXT:    jne .LBB49_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds float, float* %arg, i64 %tmp
+  %tmp3 = bitcast float* %tmp2 to <4 x float>*
+  %tmp4 = load <4 x float>, <4 x float>* %tmp3, align 4
+  %tmp5 = fmul contract <4 x float> %tmp4, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
+  %tmp6 = fadd contract <4 x float> %tmp4, %tmp5
+  %tmp7 = bitcast float* %tmp2 to <4 x float>*
+  store <4 x float> %tmp6, <4 x float>* %tmp7, align 4
+  %tmp8 = add i64 %tmp, 4
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_fma213_v8f32(float* %arg) {
+; CHECK-LABEL: bcast_unfold_fma213_v8f32:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
+; CHECK-NEXT:    vbroadcastss {{.*#+}} ymm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB50_1: # %bb2
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovups 4096(%rdi,%rax), %ymm1
+; CHECK-NEXT:    vfmadd213ps {{.*#+}} ymm1 = (ymm1 * ymm1) + ymm0
+; CHECK-NEXT:    vmovups %ymm1, 4096(%rdi,%rax)
+; CHECK-NEXT:    addq $32, %rax
+; CHECK-NEXT:    jne .LBB50_1
+; CHECK-NEXT:  # %bb.2: # %bb11
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb2
+
+bb2:                                              ; preds = %bb2, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp9, %bb2 ]
+  %tmp3 = getelementptr inbounds float, float* %arg, i64 %tmp
+  %tmp4 = bitcast float* %tmp3 to <8 x float>*
+  %tmp5 = load <8 x float>, <8 x float>* %tmp4, align 4
+  %tmp6 = fmul contract <8 x float> %tmp5, %tmp5
+  %tmp7 = fadd contract <8 x float> %tmp6, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
+  %tmp8 = bitcast float* %tmp3 to <8 x float>*
+  store <8 x float> %tmp7, <8 x float>* %tmp8, align 4
+  %tmp9 = add i64 %tmp, 8
+  %tmp10 = icmp eq i64 %tmp9, 1024
+  br i1 %tmp10, label %bb11, label %bb2
+
+bb11:                                             ; preds = %bb2
+  ret void
+}
+
+define void @bcast_unfold_fma231_v8f32(float* %arg) {
+; CHECK-LABEL: bcast_unfold_fma231_v8f32:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
+; CHECK-NEXT:    vbroadcastss {{.*#+}} ymm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB51_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovups 4096(%rdi,%rax), %ymm1
+; CHECK-NEXT:    vfmadd231ps {{.*#+}} ymm1 = (ymm1 * ymm0) + ymm1
+; CHECK-NEXT:    vmovups %ymm1, 4096(%rdi,%rax)
+; CHECK-NEXT:    addq $32, %rax
+; CHECK-NEXT:    jne .LBB51_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds float, float* %arg, i64 %tmp
+  %tmp3 = bitcast float* %tmp2 to <8 x float>*
+  %tmp4 = load <8 x float>, <8 x float>* %tmp3, align 4
+  %tmp5 = fmul contract <8 x float> %tmp4, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
+  %tmp6 = fadd contract <8 x float> %tmp4, %tmp5
+  %tmp7 = bitcast float* %tmp2 to <8 x float>*
+  store <8 x float> %tmp6, <8 x float>* %tmp7, align 4
+  %tmp8 = add i64 %tmp, 8
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_fma213_v16f32(float* %arg) {
+; CHECK-LABEL: bcast_unfold_fma213_v16f32:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
+; CHECK-NEXT:    vbroadcastss {{.*#+}} zmm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB52_1: # %bb2
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovups 4096(%rdi,%rax), %zmm1
+; CHECK-NEXT:    vfmadd213ps {{.*#+}} zmm1 = (zmm1 * zmm1) + zmm0
+; CHECK-NEXT:    vmovups %zmm1, 4096(%rdi,%rax)
+; CHECK-NEXT:    addq $64, %rax
+; CHECK-NEXT:    jne .LBB52_1
+; CHECK-NEXT:  # %bb.2: # %bb11
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb2
+
+bb2:                                              ; preds = %bb2, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp9, %bb2 ]
+  %tmp3 = getelementptr inbounds float, float* %arg, i64 %tmp
+  %tmp4 = bitcast float* %tmp3 to <16 x float>*
+  %tmp5 = load <16 x float>, <16 x float>* %tmp4, align 4
+  %tmp6 = fmul contract <16 x float> %tmp5, %tmp5
+  %tmp7 = fadd contract <16 x float> %tmp6, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
+  %tmp8 = bitcast float* %tmp3 to <16 x float>*
+  store <16 x float> %tmp7, <16 x float>* %tmp8, align 4
+  %tmp9 = add i64 %tmp, 16
+  %tmp10 = icmp eq i64 %tmp9, 1024
+  br i1 %tmp10, label %bb11, label %bb2
+
+bb11:                                             ; preds = %bb2
+  ret void
+}
+
+define void @bcast_unfold_fma231_v16f32(float* %arg) {
+; CHECK-LABEL: bcast_unfold_fma231_v16f32:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
+; CHECK-NEXT:    vbroadcastss {{.*#+}} zmm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB53_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovups 4096(%rdi,%rax), %zmm1
+; CHECK-NEXT:    vfmadd231ps {{.*#+}} zmm1 = (zmm1 * zmm0) + zmm1
+; CHECK-NEXT:    vmovups %zmm1, 4096(%rdi,%rax)
+; CHECK-NEXT:    addq $64, %rax
+; CHECK-NEXT:    jne .LBB53_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds float, float* %arg, i64 %tmp
+  %tmp3 = bitcast float* %tmp2 to <16 x float>*
+  %tmp4 = load <16 x float>, <16 x float>* %tmp3, align 4
+  %tmp5 = fmul contract <16 x float> %tmp4, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
+  %tmp6 = fadd contract <16 x float> %tmp4, %tmp5
+  %tmp7 = bitcast float* %tmp2 to <16 x float>*
+  store <16 x float> %tmp6, <16 x float>* %tmp7, align 4
+  %tmp8 = add i64 %tmp, 16
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_fma213_v2f64(double* %arg) {
+; CHECK-LABEL: bcast_unfold_fma213_v2f64:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
+; CHECK-NEXT:    vmovapd {{.*#+}} xmm0 = [2.0E+0,2.0E+0]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB54_1: # %bb2
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovupd 8192(%rdi,%rax), %xmm1
+; CHECK-NEXT:    vfmadd213pd {{.*#+}} xmm1 = (xmm1 * xmm1) + xmm0
+; CHECK-NEXT:    vmovupd %xmm1, 8192(%rdi,%rax)
+; CHECK-NEXT:    addq $16, %rax
+; CHECK-NEXT:    jne .LBB54_1
+; CHECK-NEXT:  # %bb.2: # %bb11
+; CHECK-NEXT:    retq
+bb:
+  br label %bb2
+
+bb2:                                              ; preds = %bb2, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp9, %bb2 ]
+  %tmp3 = getelementptr inbounds double, double* %arg, i64 %tmp
+  %tmp4 = bitcast double* %tmp3 to <2 x double>*
+  %tmp5 = load <2 x double>, <2 x double>* %tmp4, align 4
+  %tmp6 = fmul contract <2 x double> %tmp5, %tmp5
+  %tmp7 = fadd contract <2 x double> %tmp6, <double 2.000000e+00, double 2.000000e+00>
+  %tmp8 = bitcast double* %tmp3 to <2 x double>*
+  store <2 x double> %tmp7, <2 x double>* %tmp8, align 8
+  %tmp9 = add i64 %tmp, 2
+  %tmp10 = icmp eq i64 %tmp9, 1024
+  br i1 %tmp10, label %bb11, label %bb2
+
+bb11:                                             ; preds = %bb2
+  ret void
+}
+
+define void @bcast_unfold_fma231_v2f64(double* %arg) {
+; CHECK-LABEL: bcast_unfold_fma231_v2f64:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
+; CHECK-NEXT:    vmovapd {{.*#+}} xmm0 = [2.0E+0,2.0E+0]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB55_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovupd 8192(%rdi,%rax), %xmm1
+; CHECK-NEXT:    vfmadd231pd {{.*#+}} xmm1 = (xmm1 * xmm0) + xmm1
+; CHECK-NEXT:    vmovupd %xmm1, 8192(%rdi,%rax)
+; CHECK-NEXT:    addq $16, %rax
+; CHECK-NEXT:    jne .LBB55_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds double, double* %arg, i64 %tmp
+  %tmp3 = bitcast double* %tmp2 to <2 x double>*
+  %tmp4 = load <2 x double>, <2 x double>* %tmp3, align 8
+  %tmp5 = fmul contract <2 x double> %tmp4, <double 2.000000e+00, double 2.000000e+00>
+  %tmp6 = fadd contract <2 x double> %tmp4, %tmp5
+  %tmp7 = bitcast double* %tmp2 to <2 x double>*
+  store <2 x double> %tmp6, <2 x double>* %tmp7, align 8
+  %tmp8 = add i64 %tmp, 2
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_fma213_v4f64(double* %arg) {
+; CHECK-LABEL: bcast_unfold_fma213_v4f64:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
+; CHECK-NEXT:    vbroadcastsd {{.*#+}} ymm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB56_1: # %bb2
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovupd 8192(%rdi,%rax), %ymm1
+; CHECK-NEXT:    vfmadd213pd {{.*#+}} ymm1 = (ymm1 * ymm1) + ymm0
+; CHECK-NEXT:    vmovupd %ymm1, 8192(%rdi,%rax)
+; CHECK-NEXT:    addq $32, %rax
+; CHECK-NEXT:    jne .LBB56_1
+; CHECK-NEXT:  # %bb.2: # %bb11
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb2
+
+bb2:                                              ; preds = %bb2, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp9, %bb2 ]
+  %tmp3 = getelementptr inbounds double, double* %arg, i64 %tmp
+  %tmp4 = bitcast double* %tmp3 to <4 x double>*
+  %tmp5 = load <4 x double>, <4 x double>* %tmp4, align 8
+  %tmp6 = fmul contract <4 x double> %tmp5, %tmp5
+  %tmp7 = fadd contract <4 x double> %tmp6, <double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00>
+  %tmp8 = bitcast double* %tmp3 to <4 x double>*
+  store <4 x double> %tmp7, <4 x double>* %tmp8, align 8
+  %tmp9 = add i64 %tmp, 4
+  %tmp10 = icmp eq i64 %tmp9, 1024
+  br i1 %tmp10, label %bb11, label %bb2
+
+bb11:                                             ; preds = %bb2
+  ret void
+}
+
+define void @bcast_unfold_fma231_v4f64(double* %arg) {
+; CHECK-LABEL: bcast_unfold_fma231_v4f64:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
+; CHECK-NEXT:    vbroadcastsd {{.*#+}} ymm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB57_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovupd 8192(%rdi,%rax), %ymm1
+; CHECK-NEXT:    vfmadd231pd {{.*#+}} ymm1 = (ymm1 * ymm0) + ymm1
+; CHECK-NEXT:    vmovupd %ymm1, 8192(%rdi,%rax)
+; CHECK-NEXT:    addq $32, %rax
+; CHECK-NEXT:    jne .LBB57_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds double, double* %arg, i64 %tmp
+  %tmp3 = bitcast double* %tmp2 to <4 x double>*
+  %tmp4 = load <4 x double>, <4 x double>* %tmp3, align 8
+  %tmp5 = fmul contract <4 x double> %tmp4, <double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00>
+  %tmp6 = fadd contract <4 x double> %tmp4, %tmp5
+  %tmp7 = bitcast double* %tmp2 to <4 x double>*
+  store <4 x double> %tmp6, <4 x double>* %tmp7, align 8
+  %tmp8 = add i64 %tmp, 4
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_fma213_v8f64(double* %arg) {
+; CHECK-LABEL: bcast_unfold_fma213_v8f64:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
+; CHECK-NEXT:    vbroadcastsd {{.*#+}} zmm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB58_1: # %bb2
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovupd 8192(%rdi,%rax), %zmm1
+; CHECK-NEXT:    vfmadd213pd {{.*#+}} zmm1 = (zmm1 * zmm1) + zmm0
+; CHECK-NEXT:    vmovupd %zmm1, 8192(%rdi,%rax)
+; CHECK-NEXT:    addq $64, %rax
+; CHECK-NEXT:    jne .LBB58_1
+; CHECK-NEXT:  # %bb.2: # %bb11
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb2
+
+bb2:                                              ; preds = %bb2, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp9, %bb2 ]
+  %tmp3 = getelementptr inbounds double, double* %arg, i64 %tmp
+  %tmp4 = bitcast double* %tmp3 to <8 x double>*
+  %tmp5 = load <8 x double>, <8 x double>* %tmp4, align 8
+  %tmp6 = fmul contract <8 x double> %tmp5, %tmp5
+  %tmp7 = fadd contract <8 x double> %tmp6, <double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00>
+  %tmp8 = bitcast double* %tmp3 to <8 x double>*
+  store <8 x double> %tmp7, <8 x double>* %tmp8, align 8
+  %tmp9 = add i64 %tmp, 8
+  %tmp10 = icmp eq i64 %tmp9, 1024
+  br i1 %tmp10, label %bb11, label %bb2
+
+bb11:                                             ; preds = %bb2
+  ret void
+}
+
+define void @bcast_unfold_fma231_v8f64(double* %arg) {
+; CHECK-LABEL: bcast_unfold_fma231_v8f64:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
+; CHECK-NEXT:    vbroadcastsd {{.*#+}} zmm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB59_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovupd 8192(%rdi,%rax), %zmm1
+; CHECK-NEXT:    vfmadd231pd {{.*#+}} zmm1 = (zmm1 * zmm0) + zmm1
+; CHECK-NEXT:    vmovupd %zmm1, 8192(%rdi,%rax)
+; CHECK-NEXT:    addq $64, %rax
+; CHECK-NEXT:    jne .LBB59_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds double, double* %arg, i64 %tmp
+  %tmp3 = bitcast double* %tmp2 to <8 x double>*
+  %tmp4 = load <8 x double>, <8 x double>* %tmp3, align 8
+  %tmp5 = fmul contract <8 x double> %tmp4, <double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00>
+  %tmp6 = fadd contract <8 x double> %tmp4, %tmp5
+  %tmp7 = bitcast double* %tmp2 to <8 x double>*
+  store <8 x double> %tmp6, <8 x double>* %tmp7, align 8
+  %tmp8 = add i64 %tmp, 8
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_fmax_v4f32(float* %arg) {
+; CHECK-LABEL: bcast_unfold_fmax_v4f32:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
+; CHECK-NEXT:    vbroadcastss {{.*#+}} xmm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB60_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovups 4096(%rdi,%rax), %xmm1
+; CHECK-NEXT:    vmaxps %xmm0, %xmm1, %xmm1
+; CHECK-NEXT:    vmovups %xmm1, 4096(%rdi,%rax)
+; CHECK-NEXT:    addq $16, %rax
+; CHECK-NEXT:    jne .LBB60_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds float, float* %arg, i64 %tmp
+  %tmp3 = bitcast float* %tmp2 to <4 x float>*
+  %tmp4 = load <4 x float>, <4 x float>* %tmp3, align 4
+  %tmp5 = fcmp ogt <4 x float> %tmp4, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
+  %tmp6 = select <4 x i1> %tmp5, <4 x float> %tmp4, <4 x float> <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
+  %tmp7 = bitcast float* %tmp2 to <4 x float>*
+  store <4 x float> %tmp6, <4 x float>* %tmp7, align 4
+  %tmp8 = add i64 %tmp, 4
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_fmax_v8f32(float* %arg) {
+; CHECK-LABEL: bcast_unfold_fmax_v8f32:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
+; CHECK-NEXT:    vbroadcastss {{.*#+}} ymm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB61_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovups 4096(%rdi,%rax), %ymm1
+; CHECK-NEXT:    vmaxps %ymm0, %ymm1, %ymm1
+; CHECK-NEXT:    vmovups %ymm1, 4096(%rdi,%rax)
+; CHECK-NEXT:    addq $32, %rax
+; CHECK-NEXT:    jne .LBB61_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds float, float* %arg, i64 %tmp
+  %tmp3 = bitcast float* %tmp2 to <8 x float>*
+  %tmp4 = load <8 x float>, <8 x float>* %tmp3, align 4
+  %tmp5 = fcmp ogt <8 x float> %tmp4, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
+  %tmp6 = select <8 x i1> %tmp5, <8 x float> %tmp4, <8 x float> <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
+  %tmp7 = bitcast float* %tmp2 to <8 x float>*
+  store <8 x float> %tmp6, <8 x float>* %tmp7, align 4
+  %tmp8 = add i64 %tmp, 8
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_fmax_v16f32(float* %arg) {
+; CHECK-LABEL: bcast_unfold_fmax_v16f32:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
+; CHECK-NEXT:    vbroadcastss {{.*#+}} zmm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB62_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovups 4096(%rdi,%rax), %zmm1
+; CHECK-NEXT:    vmaxps %zmm0, %zmm1, %zmm1
+; CHECK-NEXT:    vmovups %zmm1, 4096(%rdi,%rax)
+; CHECK-NEXT:    addq $64, %rax
+; CHECK-NEXT:    jne .LBB62_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds float, float* %arg, i64 %tmp
+  %tmp3 = bitcast float* %tmp2 to <16 x float>*
+  %tmp4 = load <16 x float>, <16 x float>* %tmp3, align 4
+  %tmp5 = fcmp ogt <16 x float> %tmp4, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
+  %tmp6 = select <16 x i1> %tmp5, <16 x float> %tmp4, <16 x float> <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
+  %tmp7 = bitcast float* %tmp2 to <16 x float>*
+  store <16 x float> %tmp6, <16 x float>* %tmp7, align 4
+  %tmp8 = add i64 %tmp, 16
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_fmax_v2f64(double* %arg) {
+; CHECK-LABEL: bcast_unfold_fmax_v2f64:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
+; CHECK-NEXT:    vmovapd {{.*#+}} xmm0 = [2.0E+0,2.0E+0]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB63_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovupd 8192(%rdi,%rax), %xmm1
+; CHECK-NEXT:    vmaxpd %xmm0, %xmm1, %xmm1
+; CHECK-NEXT:    vmovupd %xmm1, 8192(%rdi,%rax)
+; CHECK-NEXT:    addq $16, %rax
+; CHECK-NEXT:    jne .LBB63_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds double, double* %arg, i64 %tmp
+  %tmp3 = bitcast double* %tmp2 to <2 x double>*
+  %tmp4 = load <2 x double>, <2 x double>* %tmp3, align 8
+  %tmp5 = fcmp ogt <2 x double> %tmp4, <double 2.000000e+00, double 2.000000e+00>
+  %tmp6 = select <2 x i1> %tmp5, <2 x double> %tmp4, <2 x double> <double 2.000000e+00, double 2.000000e+00>
+  %tmp7 = bitcast double* %tmp2 to <2 x double>*
+  store <2 x double> %tmp6, <2 x double>* %tmp7, align 8
+  %tmp8 = add i64 %tmp, 2
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_fmax_v4f64(double* %arg) {
+; CHECK-LABEL: bcast_unfold_fmax_v4f64:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
+; CHECK-NEXT:    vbroadcastsd {{.*#+}} ymm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB64_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovupd 8192(%rdi,%rax), %ymm1
+; CHECK-NEXT:    vmaxpd %ymm0, %ymm1, %ymm1
+; CHECK-NEXT:    vmovupd %ymm1, 8192(%rdi,%rax)
+; CHECK-NEXT:    addq $32, %rax
+; CHECK-NEXT:    jne .LBB64_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds double, double* %arg, i64 %tmp
+  %tmp3 = bitcast double* %tmp2 to <4 x double>*
+  %tmp4 = load <4 x double>, <4 x double>* %tmp3, align 8
+  %tmp5 = fcmp ogt <4 x double> %tmp4, <double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00>
+  %tmp6 = select <4 x i1> %tmp5, <4 x double> %tmp4, <4 x double> <double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00>
+  %tmp7 = bitcast double* %tmp2 to <4 x double>*
+  store <4 x double> %tmp6, <4 x double>* %tmp7, align 8
+  %tmp8 = add i64 %tmp, 4
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_fmax_v8f64(double* %arg) {
+; CHECK-LABEL: bcast_unfold_fmax_v8f64:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
+; CHECK-NEXT:    vbroadcastsd {{.*#+}} zmm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB65_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovupd 8192(%rdi,%rax), %zmm1
+; CHECK-NEXT:    vmaxpd %zmm0, %zmm1, %zmm1
+; CHECK-NEXT:    vmovupd %zmm1, 8192(%rdi,%rax)
+; CHECK-NEXT:    addq $64, %rax
+; CHECK-NEXT:    jne .LBB65_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds double, double* %arg, i64 %tmp
+  %tmp3 = bitcast double* %tmp2 to <8 x double>*
+  %tmp4 = load <8 x double>, <8 x double>* %tmp3, align 8
+  %tmp5 = fcmp ogt <8 x double> %tmp4, <double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00>
+  %tmp6 = select <8 x i1> %tmp5, <8 x double> %tmp4, <8 x double> <double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00>
+  %tmp7 = bitcast double* %tmp2 to <8 x double>*
+  store <8 x double> %tmp6, <8 x double>* %tmp7, align 8
+  %tmp8 = add i64 %tmp, 8
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_fmin_v4f32(float* %arg) {
+; CHECK-LABEL: bcast_unfold_fmin_v4f32:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
+; CHECK-NEXT:    vbroadcastss {{.*#+}} xmm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB66_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovups 4096(%rdi,%rax), %xmm1
+; CHECK-NEXT:    vminps %xmm0, %xmm1, %xmm1
+; CHECK-NEXT:    vmovups %xmm1, 4096(%rdi,%rax)
+; CHECK-NEXT:    addq $16, %rax
+; CHECK-NEXT:    jne .LBB66_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds float, float* %arg, i64 %tmp
+  %tmp3 = bitcast float* %tmp2 to <4 x float>*
+  %tmp4 = load <4 x float>, <4 x float>* %tmp3, align 4
+  %tmp5 = fcmp olt <4 x float> %tmp4, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
+  %tmp6 = select <4 x i1> %tmp5, <4 x float> %tmp4, <4 x float> <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
+  %tmp7 = bitcast float* %tmp2 to <4 x float>*
+  store <4 x float> %tmp6, <4 x float>* %tmp7, align 4
+  %tmp8 = add i64 %tmp, 4
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_fmin_v8f32(float* %arg) {
+; CHECK-LABEL: bcast_unfold_fmin_v8f32:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
+; CHECK-NEXT:    vbroadcastss {{.*#+}} ymm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB67_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovups 4096(%rdi,%rax), %ymm1
+; CHECK-NEXT:    vminps %ymm0, %ymm1, %ymm1
+; CHECK-NEXT:    vmovups %ymm1, 4096(%rdi,%rax)
+; CHECK-NEXT:    addq $32, %rax
+; CHECK-NEXT:    jne .LBB67_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds float, float* %arg, i64 %tmp
+  %tmp3 = bitcast float* %tmp2 to <8 x float>*
+  %tmp4 = load <8 x float>, <8 x float>* %tmp3, align 4
+  %tmp5 = fcmp olt <8 x float> %tmp4, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
+  %tmp6 = select <8 x i1> %tmp5, <8 x float> %tmp4, <8 x float> <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
+  %tmp7 = bitcast float* %tmp2 to <8 x float>*
+  store <8 x float> %tmp6, <8 x float>* %tmp7, align 4
+  %tmp8 = add i64 %tmp, 8
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_fmin_v16f32(float* %arg) {
+; CHECK-LABEL: bcast_unfold_fmin_v16f32:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
+; CHECK-NEXT:    vbroadcastss {{.*#+}} zmm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB68_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovups 4096(%rdi,%rax), %zmm1
+; CHECK-NEXT:    vminps %zmm0, %zmm1, %zmm1
+; CHECK-NEXT:    vmovups %zmm1, 4096(%rdi,%rax)
+; CHECK-NEXT:    addq $64, %rax
+; CHECK-NEXT:    jne .LBB68_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds float, float* %arg, i64 %tmp
+  %tmp3 = bitcast float* %tmp2 to <16 x float>*
+  %tmp4 = load <16 x float>, <16 x float>* %tmp3, align 4
+  %tmp5 = fcmp olt <16 x float> %tmp4, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
+  %tmp6 = select <16 x i1> %tmp5, <16 x float> %tmp4, <16 x float> <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
+  %tmp7 = bitcast float* %tmp2 to <16 x float>*
+  store <16 x float> %tmp6, <16 x float>* %tmp7, align 4
+  %tmp8 = add i64 %tmp, 16
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_fmin_v2f64(double* %arg) {
+; CHECK-LABEL: bcast_unfold_fmin_v2f64:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
+; CHECK-NEXT:    vmovapd {{.*#+}} xmm0 = [2.0E+0,2.0E+0]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB69_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovupd 8192(%rdi,%rax), %xmm1
+; CHECK-NEXT:    vminpd %xmm0, %xmm1, %xmm1
+; CHECK-NEXT:    vmovupd %xmm1, 8192(%rdi,%rax)
+; CHECK-NEXT:    addq $16, %rax
+; CHECK-NEXT:    jne .LBB69_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds double, double* %arg, i64 %tmp
+  %tmp3 = bitcast double* %tmp2 to <2 x double>*
+  %tmp4 = load <2 x double>, <2 x double>* %tmp3, align 8
+  %tmp5 = fcmp olt <2 x double> %tmp4, <double 2.000000e+00, double 2.000000e+00>
+  %tmp6 = select <2 x i1> %tmp5, <2 x double> %tmp4, <2 x double> <double 2.000000e+00, double 2.000000e+00>
+  %tmp7 = bitcast double* %tmp2 to <2 x double>*
+  store <2 x double> %tmp6, <2 x double>* %tmp7, align 8
+  %tmp8 = add i64 %tmp, 2
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_fmin_v4f64(double* %arg) {
+; CHECK-LABEL: bcast_unfold_fmin_v4f64:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
+; CHECK-NEXT:    vbroadcastsd {{.*#+}} ymm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB70_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovupd 8192(%rdi,%rax), %ymm1
+; CHECK-NEXT:    vminpd %ymm0, %ymm1, %ymm1
+; CHECK-NEXT:    vmovupd %ymm1, 8192(%rdi,%rax)
+; CHECK-NEXT:    addq $32, %rax
+; CHECK-NEXT:    jne .LBB70_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds double, double* %arg, i64 %tmp
+  %tmp3 = bitcast double* %tmp2 to <4 x double>*
+  %tmp4 = load <4 x double>, <4 x double>* %tmp3, align 8
+  %tmp5 = fcmp olt <4 x double> %tmp4, <double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00>
+  %tmp6 = select <4 x i1> %tmp5, <4 x double> %tmp4, <4 x double> <double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00>
+  %tmp7 = bitcast double* %tmp2 to <4 x double>*
+  store <4 x double> %tmp6, <4 x double>* %tmp7, align 8
+  %tmp8 = add i64 %tmp, 4
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_fmin_v8f64(double* %arg) {
+; CHECK-LABEL: bcast_unfold_fmin_v8f64:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
+; CHECK-NEXT:    vbroadcastsd {{.*#+}} zmm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB71_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovupd 8192(%rdi,%rax), %zmm1
+; CHECK-NEXT:    vminpd %zmm0, %zmm1, %zmm1
+; CHECK-NEXT:    vmovupd %zmm1, 8192(%rdi,%rax)
+; CHECK-NEXT:    addq $64, %rax
+; CHECK-NEXT:    jne .LBB71_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds double, double* %arg, i64 %tmp
+  %tmp3 = bitcast double* %tmp2 to <8 x double>*
+  %tmp4 = load <8 x double>, <8 x double>* %tmp3, align 8
+  %tmp5 = fcmp olt <8 x double> %tmp4, <double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00>
+  %tmp6 = select <8 x i1> %tmp5, <8 x double> %tmp4, <8 x double> <double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00>
+  %tmp7 = bitcast double* %tmp2 to <8 x double>*
+  store <8 x double> %tmp6, <8 x double>* %tmp7, align 8
+  %tmp8 = add i64 %tmp, 8
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_smin_v4i32(i32* %arg) {
+; CHECK-LABEL: bcast_unfold_smin_v4i32:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
+; CHECK-NEXT:    vpbroadcastd {{.*#+}} xmm0 = [2,2,2,2]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB72_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vpminsd 4096(%rdi,%rax), %xmm0, %xmm1
+; CHECK-NEXT:    vmovdqu %xmm1, 4096(%rdi,%rax)
+; CHECK-NEXT:    addq $16, %rax
+; CHECK-NEXT:    jne .LBB72_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds i32, i32* %arg, i64 %tmp
+  %tmp3 = bitcast i32* %tmp2 to <4 x i32>*
+  %tmp4 = load <4 x i32>, <4 x i32>* %tmp3, align 4
+  %tmp5 = icmp slt <4 x i32> %tmp4, <i32 2, i32 2, i32 2, i32 2>
+  %tmp6 = select <4 x i1> %tmp5, <4 x i32> %tmp4, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+  %tmp7 = bitcast i32* %tmp2 to <4 x i32>*
+  store <4 x i32> %tmp6, <4 x i32>* %tmp7, align 4
+  %tmp8 = add i64 %tmp, 4
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_smin_v8i32(i32* %arg) {
+; CHECK-LABEL: bcast_unfold_smin_v8i32:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
+; CHECK-NEXT:    vpbroadcastd {{.*#+}} ymm0 = [2,2,2,2,2,2,2,2]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB73_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vpminsd 4096(%rdi,%rax), %ymm0, %ymm1
+; CHECK-NEXT:    vmovdqu %ymm1, 4096(%rdi,%rax)
+; CHECK-NEXT:    addq $32, %rax
+; CHECK-NEXT:    jne .LBB73_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds i32, i32* %arg, i64 %tmp
+  %tmp3 = bitcast i32* %tmp2 to <8 x i32>*
+  %tmp4 = load <8 x i32>, <8 x i32>* %tmp3, align 4
+  %tmp5 = icmp slt <8 x i32> %tmp4, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  %tmp6 = select <8 x i1> %tmp5, <8 x i32> %tmp4, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  %tmp7 = bitcast i32* %tmp2 to <8 x i32>*
+  store <8 x i32> %tmp6, <8 x i32>* %tmp7, align 4
+  %tmp8 = add i64 %tmp, 8
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_smin_v16i32(i32* %arg) {
+; CHECK-LABEL: bcast_unfold_smin_v16i32:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
+; CHECK-NEXT:    vpbroadcastd {{.*#+}} zmm0 = [2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB74_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vpminsd 4096(%rdi,%rax), %zmm0, %zmm1
+; CHECK-NEXT:    vmovdqu64 %zmm1, 4096(%rdi,%rax)
+; CHECK-NEXT:    addq $64, %rax
+; CHECK-NEXT:    jne .LBB74_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds i32, i32* %arg, i64 %tmp
+  %tmp3 = bitcast i32* %tmp2 to <16 x i32>*
+  %tmp4 = load <16 x i32>, <16 x i32>* %tmp3, align 4
+  %tmp5 = icmp slt <16 x i32> %tmp4, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  %tmp6 = select <16 x i1> %tmp5, <16 x i32> %tmp4, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  %tmp7 = bitcast i32* %tmp2 to <16 x i32>*
+  store <16 x i32> %tmp6, <16 x i32>* %tmp7, align 4
+  %tmp8 = add i64 %tmp, 16
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_smin_v2i64(i64* %arg) {
+; CHECK-LABEL: bcast_unfold_smin_v2i64:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
+; CHECK-NEXT:    vmovdqa {{.*#+}} xmm0 = [2,2]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB75_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vpminsq 8192(%rdi,%rax), %xmm0, %xmm1
+; CHECK-NEXT:    vmovdqu %xmm1, 8192(%rdi,%rax)
+; CHECK-NEXT:    addq $16, %rax
+; CHECK-NEXT:    jne .LBB75_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds i64, i64* %arg, i64 %tmp
+  %tmp3 = bitcast i64* %tmp2 to <2 x i64>*
+  %tmp4 = load <2 x i64>, <2 x i64>* %tmp3, align 8
+  %tmp5 = icmp slt <2 x i64> %tmp4, <i64 2, i64 2>
+  %tmp6 = select <2 x i1> %tmp5, <2 x i64> %tmp4, <2 x i64> <i64 2, i64 2>
+  %tmp7 = bitcast i64* %tmp2 to <2 x i64>*
+  store <2 x i64> %tmp6, <2 x i64>* %tmp7, align 8
+  %tmp8 = add i64 %tmp, 2
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_smin_v4i64(i64* %arg) {
+; CHECK-LABEL: bcast_unfold_smin_v4i64:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
+; CHECK-NEXT:    vpbroadcastq {{.*#+}} ymm0 = [2,2,2,2]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB76_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vpminsq 8192(%rdi,%rax), %ymm0, %ymm1
+; CHECK-NEXT:    vmovdqu %ymm1, 8192(%rdi,%rax)
+; CHECK-NEXT:    addq $32, %rax
+; CHECK-NEXT:    jne .LBB76_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds i64, i64* %arg, i64 %tmp
+  %tmp3 = bitcast i64* %tmp2 to <4 x i64>*
+  %tmp4 = load <4 x i64>, <4 x i64>* %tmp3, align 8
+  %tmp5 = icmp slt <4 x i64> %tmp4, <i64 2, i64 2, i64 2, i64 2>
+  %tmp6 = select <4 x i1> %tmp5, <4 x i64> %tmp4, <4 x i64> <i64 2, i64 2, i64 2, i64 2>
+  %tmp7 = bitcast i64* %tmp2 to <4 x i64>*
+  store <4 x i64> %tmp6, <4 x i64>* %tmp7, align 8
+  %tmp8 = add i64 %tmp, 4
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_smin_v8i64(i64* %arg) {
+; CHECK-LABEL: bcast_unfold_smin_v8i64:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
+; CHECK-NEXT:    vpbroadcastq {{.*#+}} zmm0 = [2,2,2,2,2,2,2,2]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB77_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vpminsq 8192(%rdi,%rax), %zmm0, %zmm1
+; CHECK-NEXT:    vmovdqu64 %zmm1, 8192(%rdi,%rax)
+; CHECK-NEXT:    addq $64, %rax
+; CHECK-NEXT:    jne .LBB77_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds i64, i64* %arg, i64 %tmp
+  %tmp3 = bitcast i64* %tmp2 to <8 x i64>*
+  %tmp4 = load <8 x i64>, <8 x i64>* %tmp3, align 8
+  %tmp5 = icmp slt <8 x i64> %tmp4, <i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2>
+  %tmp6 = select <8 x i1> %tmp5, <8 x i64> %tmp4, <8 x i64> <i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2>
+  %tmp7 = bitcast i64* %tmp2 to <8 x i64>*
+  store <8 x i64> %tmp6, <8 x i64>* %tmp7, align 8
+  %tmp8 = add i64 %tmp, 8
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_smax_v4i32(i32* %arg) {
+; CHECK-LABEL: bcast_unfold_smax_v4i32:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
+; CHECK-NEXT:    vpbroadcastd {{.*#+}} xmm0 = [2,2,2,2]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB78_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vpmaxsd 4096(%rdi,%rax), %xmm0, %xmm1
+; CHECK-NEXT:    vmovdqu %xmm1, 4096(%rdi,%rax)
+; CHECK-NEXT:    addq $16, %rax
+; CHECK-NEXT:    jne .LBB78_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds i32, i32* %arg, i64 %tmp
+  %tmp3 = bitcast i32* %tmp2 to <4 x i32>*
+  %tmp4 = load <4 x i32>, <4 x i32>* %tmp3, align 4
+  %tmp5 = icmp sgt <4 x i32> %tmp4, <i32 2, i32 2, i32 2, i32 2>
+  %tmp6 = select <4 x i1> %tmp5, <4 x i32> %tmp4, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+  %tmp7 = bitcast i32* %tmp2 to <4 x i32>*
+  store <4 x i32> %tmp6, <4 x i32>* %tmp7, align 4
+  %tmp8 = add i64 %tmp, 4
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_smax_v8i32(i32* %arg) {
+; CHECK-LABEL: bcast_unfold_smax_v8i32:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
+; CHECK-NEXT:    vpbroadcastd {{.*#+}} ymm0 = [2,2,2,2,2,2,2,2]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB79_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vpmaxsd 4096(%rdi,%rax), %ymm0, %ymm1
+; CHECK-NEXT:    vmovdqu %ymm1, 4096(%rdi,%rax)
+; CHECK-NEXT:    addq $32, %rax
+; CHECK-NEXT:    jne .LBB79_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds i32, i32* %arg, i64 %tmp
+  %tmp3 = bitcast i32* %tmp2 to <8 x i32>*
+  %tmp4 = load <8 x i32>, <8 x i32>* %tmp3, align 4
+  %tmp5 = icmp sgt <8 x i32> %tmp4, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  %tmp6 = select <8 x i1> %tmp5, <8 x i32> %tmp4, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  %tmp7 = bitcast i32* %tmp2 to <8 x i32>*
+  store <8 x i32> %tmp6, <8 x i32>* %tmp7, align 4
+  %tmp8 = add i64 %tmp, 8
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_smax_v16i32(i32* %arg) {
+; CHECK-LABEL: bcast_unfold_smax_v16i32:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
+; CHECK-NEXT:    vpbroadcastd {{.*#+}} zmm0 = [2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB80_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vpmaxsd 4096(%rdi,%rax), %zmm0, %zmm1
+; CHECK-NEXT:    vmovdqu64 %zmm1, 4096(%rdi,%rax)
+; CHECK-NEXT:    addq $64, %rax
+; CHECK-NEXT:    jne .LBB80_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds i32, i32* %arg, i64 %tmp
+  %tmp3 = bitcast i32* %tmp2 to <16 x i32>*
+  %tmp4 = load <16 x i32>, <16 x i32>* %tmp3, align 4
+  %tmp5 = icmp sgt <16 x i32> %tmp4, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  %tmp6 = select <16 x i1> %tmp5, <16 x i32> %tmp4, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  %tmp7 = bitcast i32* %tmp2 to <16 x i32>*
+  store <16 x i32> %tmp6, <16 x i32>* %tmp7, align 4
+  %tmp8 = add i64 %tmp, 16
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_smax_v2i64(i64* %arg) {
+; CHECK-LABEL: bcast_unfold_smax_v2i64:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
+; CHECK-NEXT:    vmovdqa {{.*#+}} xmm0 = [2,2]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB81_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vpmaxsq 8192(%rdi,%rax), %xmm0, %xmm1
+; CHECK-NEXT:    vmovdqu %xmm1, 8192(%rdi,%rax)
+; CHECK-NEXT:    addq $16, %rax
+; CHECK-NEXT:    jne .LBB81_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds i64, i64* %arg, i64 %tmp
+  %tmp3 = bitcast i64* %tmp2 to <2 x i64>*
+  %tmp4 = load <2 x i64>, <2 x i64>* %tmp3, align 8
+  %tmp5 = icmp sgt <2 x i64> %tmp4, <i64 2, i64 2>
+  %tmp6 = select <2 x i1> %tmp5, <2 x i64> %tmp4, <2 x i64> <i64 2, i64 2>
+  %tmp7 = bitcast i64* %tmp2 to <2 x i64>*
+  store <2 x i64> %tmp6, <2 x i64>* %tmp7, align 8
+  %tmp8 = add i64 %tmp, 2
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_smax_v4i64(i64* %arg) {
+; CHECK-LABEL: bcast_unfold_smax_v4i64:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
+; CHECK-NEXT:    vpbroadcastq {{.*#+}} ymm0 = [2,2,2,2]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB82_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vpmaxsq 8192(%rdi,%rax), %ymm0, %ymm1
+; CHECK-NEXT:    vmovdqu %ymm1, 8192(%rdi,%rax)
+; CHECK-NEXT:    addq $32, %rax
+; CHECK-NEXT:    jne .LBB82_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds i64, i64* %arg, i64 %tmp
+  %tmp3 = bitcast i64* %tmp2 to <4 x i64>*
+  %tmp4 = load <4 x i64>, <4 x i64>* %tmp3, align 8
+  %tmp5 = icmp sgt <4 x i64> %tmp4, <i64 2, i64 2, i64 2, i64 2>
+  %tmp6 = select <4 x i1> %tmp5, <4 x i64> %tmp4, <4 x i64> <i64 2, i64 2, i64 2, i64 2>
+  %tmp7 = bitcast i64* %tmp2 to <4 x i64>*
+  store <4 x i64> %tmp6, <4 x i64>* %tmp7, align 8
+  %tmp8 = add i64 %tmp, 4
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_smax_v8i64(i64* %arg) {
+; CHECK-LABEL: bcast_unfold_smax_v8i64:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
+; CHECK-NEXT:    vpbroadcastq {{.*#+}} zmm0 = [2,2,2,2,2,2,2,2]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB83_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vpmaxsq 8192(%rdi,%rax), %zmm0, %zmm1
+; CHECK-NEXT:    vmovdqu64 %zmm1, 8192(%rdi,%rax)
+; CHECK-NEXT:    addq $64, %rax
+; CHECK-NEXT:    jne .LBB83_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds i64, i64* %arg, i64 %tmp
+  %tmp3 = bitcast i64* %tmp2 to <8 x i64>*
+  %tmp4 = load <8 x i64>, <8 x i64>* %tmp3, align 8
+  %tmp5 = icmp sgt <8 x i64> %tmp4, <i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2>
+  %tmp6 = select <8 x i1> %tmp5, <8 x i64> %tmp4, <8 x i64> <i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2>
+  %tmp7 = bitcast i64* %tmp2 to <8 x i64>*
+  store <8 x i64> %tmp6, <8 x i64>* %tmp7, align 8
+  %tmp8 = add i64 %tmp, 8
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_umin_v4i32(i32* %arg) {
+; CHECK-LABEL: bcast_unfold_umin_v4i32:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
+; CHECK-NEXT:    vpbroadcastd {{.*#+}} xmm0 = [2,2,2,2]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB84_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vpminud 4096(%rdi,%rax), %xmm0, %xmm1
+; CHECK-NEXT:    vmovdqu %xmm1, 4096(%rdi,%rax)
+; CHECK-NEXT:    addq $16, %rax
+; CHECK-NEXT:    jne .LBB84_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds i32, i32* %arg, i64 %tmp
+  %tmp3 = bitcast i32* %tmp2 to <4 x i32>*
+  %tmp4 = load <4 x i32>, <4 x i32>* %tmp3, align 4
+  %tmp5 = icmp ult <4 x i32> %tmp4, <i32 2, i32 2, i32 2, i32 2>
+  %tmp6 = select <4 x i1> %tmp5, <4 x i32> %tmp4, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+  %tmp7 = bitcast i32* %tmp2 to <4 x i32>*
+  store <4 x i32> %tmp6, <4 x i32>* %tmp7, align 4
+  %tmp8 = add i64 %tmp, 4
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_umin_v8i32(i32* %arg) {
+; CHECK-LABEL: bcast_unfold_umin_v8i32:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
+; CHECK-NEXT:    vpbroadcastd {{.*#+}} ymm0 = [2,2,2,2,2,2,2,2]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB85_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vpminud 4096(%rdi,%rax), %ymm0, %ymm1
+; CHECK-NEXT:    vmovdqu %ymm1, 4096(%rdi,%rax)
+; CHECK-NEXT:    addq $32, %rax
+; CHECK-NEXT:    jne .LBB85_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds i32, i32* %arg, i64 %tmp
+  %tmp3 = bitcast i32* %tmp2 to <8 x i32>*
+  %tmp4 = load <8 x i32>, <8 x i32>* %tmp3, align 4
+  %tmp5 = icmp ult <8 x i32> %tmp4, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  %tmp6 = select <8 x i1> %tmp5, <8 x i32> %tmp4, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  %tmp7 = bitcast i32* %tmp2 to <8 x i32>*
+  store <8 x i32> %tmp6, <8 x i32>* %tmp7, align 4
+  %tmp8 = add i64 %tmp, 8
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_umin_v16i32(i32* %arg) {
+; CHECK-LABEL: bcast_unfold_umin_v16i32:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
+; CHECK-NEXT:    vpbroadcastd {{.*#+}} zmm0 = [2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB86_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vpminud 4096(%rdi,%rax), %zmm0, %zmm1
+; CHECK-NEXT:    vmovdqu64 %zmm1, 4096(%rdi,%rax)
+; CHECK-NEXT:    addq $64, %rax
+; CHECK-NEXT:    jne .LBB86_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds i32, i32* %arg, i64 %tmp
+  %tmp3 = bitcast i32* %tmp2 to <16 x i32>*
+  %tmp4 = load <16 x i32>, <16 x i32>* %tmp3, align 4
+  %tmp5 = icmp ult <16 x i32> %tmp4, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  %tmp6 = select <16 x i1> %tmp5, <16 x i32> %tmp4, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  %tmp7 = bitcast i32* %tmp2 to <16 x i32>*
+  store <16 x i32> %tmp6, <16 x i32>* %tmp7, align 4
+  %tmp8 = add i64 %tmp, 16
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_umin_v2i64(i64* %arg) {
+; CHECK-LABEL: bcast_unfold_umin_v2i64:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
+; CHECK-NEXT:    vmovdqa {{.*#+}} xmm0 = [2,2]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB87_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vpminuq 8192(%rdi,%rax), %xmm0, %xmm1
+; CHECK-NEXT:    vmovdqu %xmm1, 8192(%rdi,%rax)
+; CHECK-NEXT:    addq $16, %rax
+; CHECK-NEXT:    jne .LBB87_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds i64, i64* %arg, i64 %tmp
+  %tmp3 = bitcast i64* %tmp2 to <2 x i64>*
+  %tmp4 = load <2 x i64>, <2 x i64>* %tmp3, align 8
+  %tmp5 = icmp ult <2 x i64> %tmp4, <i64 2, i64 2>
+  %tmp6 = select <2 x i1> %tmp5, <2 x i64> %tmp4, <2 x i64> <i64 2, i64 2>
+  %tmp7 = bitcast i64* %tmp2 to <2 x i64>*
+  store <2 x i64> %tmp6, <2 x i64>* %tmp7, align 8
+  %tmp8 = add i64 %tmp, 2
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_umin_v4i64(i64* %arg) {
+; CHECK-LABEL: bcast_unfold_umin_v4i64:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
+; CHECK-NEXT:    vpbroadcastq {{.*#+}} ymm0 = [2,2,2,2]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB88_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vpminuq 8192(%rdi,%rax), %ymm0, %ymm1
+; CHECK-NEXT:    vmovdqu %ymm1, 8192(%rdi,%rax)
+; CHECK-NEXT:    addq $32, %rax
+; CHECK-NEXT:    jne .LBB88_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds i64, i64* %arg, i64 %tmp
+  %tmp3 = bitcast i64* %tmp2 to <4 x i64>*
+  %tmp4 = load <4 x i64>, <4 x i64>* %tmp3, align 8
+  %tmp5 = icmp ult <4 x i64> %tmp4, <i64 2, i64 2, i64 2, i64 2>
+  %tmp6 = select <4 x i1> %tmp5, <4 x i64> %tmp4, <4 x i64> <i64 2, i64 2, i64 2, i64 2>
+  %tmp7 = bitcast i64* %tmp2 to <4 x i64>*
+  store <4 x i64> %tmp6, <4 x i64>* %tmp7, align 8
+  %tmp8 = add i64 %tmp, 4
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_umin_v8i64(i64* %arg) {
+; CHECK-LABEL: bcast_unfold_umin_v8i64:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
+; CHECK-NEXT:    vpbroadcastq {{.*#+}} zmm0 = [2,2,2,2,2,2,2,2]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB89_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vpminuq 8192(%rdi,%rax), %zmm0, %zmm1
+; CHECK-NEXT:    vmovdqu64 %zmm1, 8192(%rdi,%rax)
+; CHECK-NEXT:    addq $64, %rax
+; CHECK-NEXT:    jne .LBB89_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds i64, i64* %arg, i64 %tmp
+  %tmp3 = bitcast i64* %tmp2 to <8 x i64>*
+  %tmp4 = load <8 x i64>, <8 x i64>* %tmp3, align 8
+  %tmp5 = icmp ult <8 x i64> %tmp4, <i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2>
+  %tmp6 = select <8 x i1> %tmp5, <8 x i64> %tmp4, <8 x i64> <i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2>
+  %tmp7 = bitcast i64* %tmp2 to <8 x i64>*
+  store <8 x i64> %tmp6, <8 x i64>* %tmp7, align 8
+  %tmp8 = add i64 %tmp, 8
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_umax_v4i32(i32* %arg) {
+; CHECK-LABEL: bcast_unfold_umax_v4i32:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
+; CHECK-NEXT:    vpbroadcastd {{.*#+}} xmm0 = [2,2,2,2]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB90_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vpmaxud 4096(%rdi,%rax), %xmm0, %xmm1
+; CHECK-NEXT:    vmovdqu %xmm1, 4096(%rdi,%rax)
+; CHECK-NEXT:    addq $16, %rax
+; CHECK-NEXT:    jne .LBB90_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds i32, i32* %arg, i64 %tmp
+  %tmp3 = bitcast i32* %tmp2 to <4 x i32>*
+  %tmp4 = load <4 x i32>, <4 x i32>* %tmp3, align 4
+  %tmp5 = icmp ugt <4 x i32> %tmp4, <i32 2, i32 2, i32 2, i32 2>
+  %tmp6 = select <4 x i1> %tmp5, <4 x i32> %tmp4, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+  %tmp7 = bitcast i32* %tmp2 to <4 x i32>*
+  store <4 x i32> %tmp6, <4 x i32>* %tmp7, align 4
+  %tmp8 = add i64 %tmp, 4
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_umax_v8i32(i32* %arg) {
+; CHECK-LABEL: bcast_unfold_umax_v8i32:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
+; CHECK-NEXT:    vpbroadcastd {{.*#+}} ymm0 = [2,2,2,2,2,2,2,2]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB91_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vpmaxud 4096(%rdi,%rax), %ymm0, %ymm1
+; CHECK-NEXT:    vmovdqu %ymm1, 4096(%rdi,%rax)
+; CHECK-NEXT:    addq $32, %rax
+; CHECK-NEXT:    jne .LBB91_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds i32, i32* %arg, i64 %tmp
+  %tmp3 = bitcast i32* %tmp2 to <8 x i32>*
+  %tmp4 = load <8 x i32>, <8 x i32>* %tmp3, align 4
+  %tmp5 = icmp ugt <8 x i32> %tmp4, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  %tmp6 = select <8 x i1> %tmp5, <8 x i32> %tmp4, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  %tmp7 = bitcast i32* %tmp2 to <8 x i32>*
+  store <8 x i32> %tmp6, <8 x i32>* %tmp7, align 4
+  %tmp8 = add i64 %tmp, 8
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_umax_v16i32(i32* %arg) {
+; CHECK-LABEL: bcast_unfold_umax_v16i32:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
+; CHECK-NEXT:    vpbroadcastd {{.*#+}} zmm0 = [2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB92_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vpmaxud 4096(%rdi,%rax), %zmm0, %zmm1
+; CHECK-NEXT:    vmovdqu64 %zmm1, 4096(%rdi,%rax)
+; CHECK-NEXT:    addq $64, %rax
+; CHECK-NEXT:    jne .LBB92_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds i32, i32* %arg, i64 %tmp
+  %tmp3 = bitcast i32* %tmp2 to <16 x i32>*
+  %tmp4 = load <16 x i32>, <16 x i32>* %tmp3, align 4
+  %tmp5 = icmp ugt <16 x i32> %tmp4, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  %tmp6 = select <16 x i1> %tmp5, <16 x i32> %tmp4, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  %tmp7 = bitcast i32* %tmp2 to <16 x i32>*
+  store <16 x i32> %tmp6, <16 x i32>* %tmp7, align 4
+  %tmp8 = add i64 %tmp, 16
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_umax_v2i64(i64* %arg) {
+; CHECK-LABEL: bcast_unfold_umax_v2i64:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
+; CHECK-NEXT:    vmovdqa {{.*#+}} xmm0 = [2,2]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB93_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vpmaxuq 8192(%rdi,%rax), %xmm0, %xmm1
+; CHECK-NEXT:    vmovdqu %xmm1, 8192(%rdi,%rax)
+; CHECK-NEXT:    addq $16, %rax
+; CHECK-NEXT:    jne .LBB93_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds i64, i64* %arg, i64 %tmp
+  %tmp3 = bitcast i64* %tmp2 to <2 x i64>*
+  %tmp4 = load <2 x i64>, <2 x i64>* %tmp3, align 8
+  %tmp5 = icmp ugt <2 x i64> %tmp4, <i64 2, i64 2>
+  %tmp6 = select <2 x i1> %tmp5, <2 x i64> %tmp4, <2 x i64> <i64 2, i64 2>
+  %tmp7 = bitcast i64* %tmp2 to <2 x i64>*
+  store <2 x i64> %tmp6, <2 x i64>* %tmp7, align 8
+  %tmp8 = add i64 %tmp, 2
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_umax_v4i64(i64* %arg) {
+; CHECK-LABEL: bcast_unfold_umax_v4i64:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
+; CHECK-NEXT:    vpbroadcastq {{.*#+}} ymm0 = [2,2,2,2]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB94_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vpmaxuq 8192(%rdi,%rax), %ymm0, %ymm1
+; CHECK-NEXT:    vmovdqu %ymm1, 8192(%rdi,%rax)
+; CHECK-NEXT:    addq $32, %rax
+; CHECK-NEXT:    jne .LBB94_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds i64, i64* %arg, i64 %tmp
+  %tmp3 = bitcast i64* %tmp2 to <4 x i64>*
+  %tmp4 = load <4 x i64>, <4 x i64>* %tmp3, align 8
+  %tmp5 = icmp ugt <4 x i64> %tmp4, <i64 2, i64 2, i64 2, i64 2>
+  %tmp6 = select <4 x i1> %tmp5, <4 x i64> %tmp4, <4 x i64> <i64 2, i64 2, i64 2, i64 2>
+  %tmp7 = bitcast i64* %tmp2 to <4 x i64>*
+  store <4 x i64> %tmp6, <4 x i64>* %tmp7, align 8
+  %tmp8 = add i64 %tmp, 4
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_umax_v8i64(i64* %arg) {
+; CHECK-LABEL: bcast_unfold_umax_v8i64:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
+; CHECK-NEXT:    vpbroadcastq {{.*#+}} zmm0 = [2,2,2,2,2,2,2,2]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB95_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vpmaxuq 8192(%rdi,%rax), %zmm0, %zmm1
+; CHECK-NEXT:    vmovdqu64 %zmm1, 8192(%rdi,%rax)
+; CHECK-NEXT:    addq $64, %rax
+; CHECK-NEXT:    jne .LBB95_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds i64, i64* %arg, i64 %tmp
+  %tmp3 = bitcast i64* %tmp2 to <8 x i64>*
+  %tmp4 = load <8 x i64>, <8 x i64>* %tmp3, align 8
+  %tmp5 = icmp ugt <8 x i64> %tmp4, <i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2>
+  %tmp6 = select <8 x i1> %tmp5, <8 x i64> %tmp4, <8 x i64> <i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2>
+  %tmp7 = bitcast i64* %tmp2 to <8 x i64>*
+  store <8 x i64> %tmp6, <8 x i64>* %tmp7, align 8
+  %tmp8 = add i64 %tmp, 8
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_pcmpgt_v4i32(i32* %arg) {
+; CHECK-LABEL: bcast_unfold_pcmpgt_v4i32:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
+; CHECK-NEXT:    vpbroadcastd {{.*#+}} xmm0 = [1,1,1,1]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB96_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovdqu 4096(%rdi,%rax), %xmm1
+; CHECK-NEXT:    vpcmpgtd %xmm0, %xmm1, %k1
+; CHECK-NEXT:    vpbroadcastd {{.*}}(%rip), %xmm1 {%k1}
+; CHECK-NEXT:    vmovdqu %xmm1, 4096(%rdi,%rax)
+; CHECK-NEXT:    addq $16, %rax
+; CHECK-NEXT:    jne .LBB96_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds i32, i32* %arg, i64 %tmp
+  %tmp3 = bitcast i32* %tmp2 to <4 x i32>*
+  %tmp4 = load <4 x i32>, <4 x i32>* %tmp3, align 4
+  %tmp5 = icmp sgt <4 x i32> %tmp4, <i32 1, i32 1, i32 1, i32 1>
+  %tmp6 = select <4 x i1> %tmp5, <4 x i32> <i32 3, i32 3, i32 3, i32 3>, <4 x i32> %tmp4
+  %tmp7 = bitcast i32* %tmp2 to <4 x i32>*
+  store <4 x i32> %tmp6, <4 x i32>* %tmp7, align 4
+  %tmp8 = add i64 %tmp, 4
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_pcmpgt_v8i32(i32* %arg) {
+; CHECK-LABEL: bcast_unfold_pcmpgt_v8i32:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
+; CHECK-NEXT:    vpbroadcastd {{.*#+}} ymm0 = [1,1,1,1,1,1,1,1]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB97_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovdqu 4096(%rdi,%rax), %ymm1
+; CHECK-NEXT:    vpcmpgtd %ymm0, %ymm1, %k1
+; CHECK-NEXT:    vpbroadcastd {{.*}}(%rip), %ymm1 {%k1}
+; CHECK-NEXT:    vmovdqu %ymm1, 4096(%rdi,%rax)
+; CHECK-NEXT:    addq $32, %rax
+; CHECK-NEXT:    jne .LBB97_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds i32, i32* %arg, i64 %tmp
+  %tmp3 = bitcast i32* %tmp2 to <8 x i32>*
+  %tmp4 = load <8 x i32>, <8 x i32>* %tmp3, align 4
+  %tmp5 = icmp sgt <8 x i32> %tmp4, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  %tmp6 = select <8 x i1> %tmp5, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>, <8 x i32> %tmp4
+  %tmp7 = bitcast i32* %tmp2 to <8 x i32>*
+  store <8 x i32> %tmp6, <8 x i32>* %tmp7, align 4
+  %tmp8 = add i64 %tmp, 8
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_pcmpgt_v16i32(i32* %arg) {
+; CHECK-LABEL: bcast_unfold_pcmpgt_v16i32:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
+; CHECK-NEXT:    vpbroadcastd {{.*#+}} zmm0 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB98_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovdqu64 4096(%rdi,%rax), %zmm1
+; CHECK-NEXT:    vpcmpgtd %zmm0, %zmm1, %k1
+; CHECK-NEXT:    vpbroadcastd {{.*}}(%rip), %zmm1 {%k1}
+; CHECK-NEXT:    vmovdqu64 %zmm1, 4096(%rdi,%rax)
+; CHECK-NEXT:    addq $64, %rax
+; CHECK-NEXT:    jne .LBB98_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds i32, i32* %arg, i64 %tmp
+  %tmp3 = bitcast i32* %tmp2 to <16 x i32>*
+  %tmp4 = load <16 x i32>, <16 x i32>* %tmp3, align 4
+  %tmp5 = icmp sgt <16 x i32> %tmp4, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  %tmp6 = select <16 x i1> %tmp5, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>, <16 x i32> %tmp4
+  %tmp7 = bitcast i32* %tmp2 to <16 x i32>*
+  store <16 x i32> %tmp6, <16 x i32>* %tmp7, align 4
+  %tmp8 = add i64 %tmp, 16
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_pcmpgt_v2i64(i64* %arg) {
+; CHECK-LABEL: bcast_unfold_pcmpgt_v2i64:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
+; CHECK-NEXT:    vmovdqa {{.*#+}} xmm0 = [1,1]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB99_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovdqu 8192(%rdi,%rax), %xmm1
+; CHECK-NEXT:    vpcmpgtq %xmm0, %xmm1, %k1
+; CHECK-NEXT:    vmovdqa64 {{.*}}(%rip), %xmm1 {%k1}
+; CHECK-NEXT:    vmovdqu %xmm1, 8192(%rdi,%rax)
+; CHECK-NEXT:    addq $16, %rax
+; CHECK-NEXT:    jne .LBB99_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds i64, i64* %arg, i64 %tmp
+  %tmp3 = bitcast i64* %tmp2 to <2 x i64>*
+  %tmp4 = load <2 x i64>, <2 x i64>* %tmp3, align 4
+  %tmp5 = icmp sgt <2 x i64> %tmp4, <i64 1, i64 1>
+  %tmp6 = select <2 x i1> %tmp5, <2 x i64> <i64 3, i64 3>, <2 x i64> %tmp4
+  %tmp7 = bitcast i64* %tmp2 to <2 x i64>*
+  store <2 x i64> %tmp6, <2 x i64>* %tmp7, align 4
+  %tmp8 = add i64 %tmp, 2
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+define void @bcast_unfold_pcmpgt_v4i64(i64* %arg) {
+; CHECK-LABEL: bcast_unfold_pcmpgt_v4i64:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
+; CHECK-NEXT:    vpbroadcastq {{.*#+}} ymm0 = [1,1,1,1]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB100_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovdqu 8192(%rdi,%rax), %ymm1
+; CHECK-NEXT:    vpcmpgtq %ymm0, %ymm1, %k1
+; CHECK-NEXT:    vpbroadcastq {{.*}}(%rip), %ymm1 {%k1}
+; CHECK-NEXT:    vmovdqu %ymm1, 8192(%rdi,%rax)
+; CHECK-NEXT:    addq $32, %rax
+; CHECK-NEXT:    jne .LBB100_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds i64, i64* %arg, i64 %tmp
+  %tmp3 = bitcast i64* %tmp2 to <4 x i64>*
+  %tmp4 = load <4 x i64>, <4 x i64>* %tmp3, align 4
+  %tmp5 = icmp sgt <4 x i64> %tmp4, <i64 1, i64 1, i64 1, i64 1>
+  %tmp6 = select <4 x i1> %tmp5, <4 x i64> <i64 3, i64 3, i64 3, i64 3>, <4 x i64> %tmp4
+  %tmp7 = bitcast i64* %tmp2 to <4 x i64>*
+  store <4 x i64> %tmp6, <4 x i64>* %tmp7, align 4
+  %tmp8 = add i64 %tmp, 4
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_pcmpgt_v8i64(i64* %arg) {
+; CHECK-LABEL: bcast_unfold_pcmpgt_v8i64:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
+; CHECK-NEXT:    vpbroadcastq {{.*#+}} zmm0 = [1,1,1,1,1,1,1,1]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB101_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovdqu64 8192(%rdi,%rax), %zmm1
+; CHECK-NEXT:    vpcmpgtq %zmm0, %zmm1, %k1
+; CHECK-NEXT:    vpbroadcastq {{.*}}(%rip), %zmm1 {%k1}
+; CHECK-NEXT:    vmovdqu64 %zmm1, 8192(%rdi,%rax)
+; CHECK-NEXT:    addq $64, %rax
+; CHECK-NEXT:    jne .LBB101_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds i64, i64* %arg, i64 %tmp
+  %tmp3 = bitcast i64* %tmp2 to <8 x i64>*
+  %tmp4 = load <8 x i64>, <8 x i64>* %tmp3, align 4
+  %tmp5 = icmp sgt <8 x i64> %tmp4, <i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1>
+  %tmp6 = select <8 x i1> %tmp5, <8 x i64> <i64 3, i64 3, i64 3, i64 3, i64 3, i64 3, i64 3, i64 3>, <8 x i64> %tmp4
+  %tmp7 = bitcast i64* %tmp2 to <8 x i64>*
+  store <8 x i64> %tmp6, <8 x i64>* %tmp7, align 4
+  %tmp8 = add i64 %tmp, 8
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_pcmpeq_v4i32(i32* %arg) {
+; CHECK-LABEL: bcast_unfold_pcmpeq_v4i32:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
+; CHECK-NEXT:    vpbroadcastd {{.*#+}} xmm0 = [1,1,1,1]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB102_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovdqu 4096(%rdi,%rax), %xmm1
+; CHECK-NEXT:    vpcmpeqd %xmm0, %xmm1, %k1
+; CHECK-NEXT:    vpbroadcastd {{.*}}(%rip), %xmm1 {%k1}
+; CHECK-NEXT:    vmovdqu %xmm1, 4096(%rdi,%rax)
+; CHECK-NEXT:    addq $16, %rax
+; CHECK-NEXT:    jne .LBB102_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds i32, i32* %arg, i64 %tmp
+  %tmp3 = bitcast i32* %tmp2 to <4 x i32>*
+  %tmp4 = load <4 x i32>, <4 x i32>* %tmp3, align 4
+  %tmp5 = icmp eq <4 x i32> %tmp4, <i32 1, i32 1, i32 1, i32 1>
+  %tmp6 = select <4 x i1> %tmp5, <4 x i32> <i32 3, i32 3, i32 3, i32 3>, <4 x i32> %tmp4
+  %tmp7 = bitcast i32* %tmp2 to <4 x i32>*
+  store <4 x i32> %tmp6, <4 x i32>* %tmp7, align 4
+  %tmp8 = add i64 %tmp, 4
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_pcmpeq_v8i32(i32* %arg) {
+; CHECK-LABEL: bcast_unfold_pcmpeq_v8i32:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
+; CHECK-NEXT:    vpbroadcastd {{.*#+}} ymm0 = [1,1,1,1,1,1,1,1]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB103_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovdqu 4096(%rdi,%rax), %ymm1
+; CHECK-NEXT:    vpcmpeqd %ymm0, %ymm1, %k1
+; CHECK-NEXT:    vpbroadcastd {{.*}}(%rip), %ymm1 {%k1}
+; CHECK-NEXT:    vmovdqu %ymm1, 4096(%rdi,%rax)
+; CHECK-NEXT:    addq $32, %rax
+; CHECK-NEXT:    jne .LBB103_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds i32, i32* %arg, i64 %tmp
+  %tmp3 = bitcast i32* %tmp2 to <8 x i32>*
+  %tmp4 = load <8 x i32>, <8 x i32>* %tmp3, align 4
+  %tmp5 = icmp eq <8 x i32> %tmp4, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  %tmp6 = select <8 x i1> %tmp5, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>, <8 x i32> %tmp4
+  %tmp7 = bitcast i32* %tmp2 to <8 x i32>*
+  store <8 x i32> %tmp6, <8 x i32>* %tmp7, align 4
+  %tmp8 = add i64 %tmp, 8
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_pcmpeq_v16i32(i32* %arg) {
+; CHECK-LABEL: bcast_unfold_pcmpeq_v16i32:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
+; CHECK-NEXT:    vpbroadcastd {{.*#+}} zmm0 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB104_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovdqu64 4096(%rdi,%rax), %zmm1
+; CHECK-NEXT:    vpcmpeqd %zmm0, %zmm1, %k1
+; CHECK-NEXT:    vpbroadcastd {{.*}}(%rip), %zmm1 {%k1}
+; CHECK-NEXT:    vmovdqu64 %zmm1, 4096(%rdi,%rax)
+; CHECK-NEXT:    addq $64, %rax
+; CHECK-NEXT:    jne .LBB104_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds i32, i32* %arg, i64 %tmp
+  %tmp3 = bitcast i32* %tmp2 to <16 x i32>*
+  %tmp4 = load <16 x i32>, <16 x i32>* %tmp3, align 4
+  %tmp5 = icmp eq <16 x i32> %tmp4, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  %tmp6 = select <16 x i1> %tmp5, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>, <16 x i32> %tmp4
+  %tmp7 = bitcast i32* %tmp2 to <16 x i32>*
+  store <16 x i32> %tmp6, <16 x i32>* %tmp7, align 4
+  %tmp8 = add i64 %tmp, 16
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_pcmpeq_v2i64(i64* %arg) {
+; CHECK-LABEL: bcast_unfold_pcmpeq_v2i64:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
+; CHECK-NEXT:    vmovdqa {{.*#+}} xmm0 = [1,1]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB105_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovdqu 8192(%rdi,%rax), %xmm1
+; CHECK-NEXT:    vpcmpeqq %xmm0, %xmm1, %k1
+; CHECK-NEXT:    vmovdqa64 {{.*}}(%rip), %xmm1 {%k1}
+; CHECK-NEXT:    vmovdqu %xmm1, 8192(%rdi,%rax)
+; CHECK-NEXT:    addq $16, %rax
+; CHECK-NEXT:    jne .LBB105_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds i64, i64* %arg, i64 %tmp
+  %tmp3 = bitcast i64* %tmp2 to <2 x i64>*
+  %tmp4 = load <2 x i64>, <2 x i64>* %tmp3, align 4
+  %tmp5 = icmp eq <2 x i64> %tmp4, <i64 1, i64 1>
+  %tmp6 = select <2 x i1> %tmp5, <2 x i64> <i64 3, i64 3>, <2 x i64> %tmp4
+  %tmp7 = bitcast i64* %tmp2 to <2 x i64>*
+  store <2 x i64> %tmp6, <2 x i64>* %tmp7, align 4
+  %tmp8 = add i64 %tmp, 2
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+define void @bcast_unfold_pcmpeq_v4i64(i64* %arg) {
+; CHECK-LABEL: bcast_unfold_pcmpeq_v4i64:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
+; CHECK-NEXT:    vpbroadcastq {{.*#+}} ymm0 = [1,1,1,1]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB106_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovdqu 8192(%rdi,%rax), %ymm1
+; CHECK-NEXT:    vpcmpeqq %ymm0, %ymm1, %k1
+; CHECK-NEXT:    vpbroadcastq {{.*}}(%rip), %ymm1 {%k1}
+; CHECK-NEXT:    vmovdqu %ymm1, 8192(%rdi,%rax)
+; CHECK-NEXT:    addq $32, %rax
+; CHECK-NEXT:    jne .LBB106_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds i64, i64* %arg, i64 %tmp
+  %tmp3 = bitcast i64* %tmp2 to <4 x i64>*
+  %tmp4 = load <4 x i64>, <4 x i64>* %tmp3, align 4
+  %tmp5 = icmp eq <4 x i64> %tmp4, <i64 1, i64 1, i64 1, i64 1>
+  %tmp6 = select <4 x i1> %tmp5, <4 x i64> <i64 3, i64 3, i64 3, i64 3>, <4 x i64> %tmp4
+  %tmp7 = bitcast i64* %tmp2 to <4 x i64>*
+  store <4 x i64> %tmp6, <4 x i64>* %tmp7, align 4
+  %tmp8 = add i64 %tmp, 4
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_pcmpeq_v8i64(i64* %arg) {
+; CHECK-LABEL: bcast_unfold_pcmpeq_v8i64:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
+; CHECK-NEXT:    vpbroadcastq {{.*#+}} zmm0 = [1,1,1,1,1,1,1,1]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB107_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovdqu64 8192(%rdi,%rax), %zmm1
+; CHECK-NEXT:    vpcmpeqq %zmm0, %zmm1, %k1
+; CHECK-NEXT:    vpbroadcastq {{.*}}(%rip), %zmm1 {%k1}
+; CHECK-NEXT:    vmovdqu64 %zmm1, 8192(%rdi,%rax)
+; CHECK-NEXT:    addq $64, %rax
+; CHECK-NEXT:    jne .LBB107_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds i64, i64* %arg, i64 %tmp
+  %tmp3 = bitcast i64* %tmp2 to <8 x i64>*
+  %tmp4 = load <8 x i64>, <8 x i64>* %tmp3, align 4
+  %tmp5 = icmp eq <8 x i64> %tmp4, <i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1>
+  %tmp6 = select <8 x i1> %tmp5, <8 x i64> <i64 3, i64 3, i64 3, i64 3, i64 3, i64 3, i64 3, i64 3>, <8 x i64> %tmp4
+  %tmp7 = bitcast i64* %tmp2 to <8 x i64>*
+  store <8 x i64> %tmp6, <8 x i64>* %tmp7, align 4
+  %tmp8 = add i64 %tmp, 8
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_pcmp_v4i32(i32* %arg) {
+; CHECK-LABEL: bcast_unfold_pcmp_v4i32:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    vpbroadcastd {{.*#+}} xmm0 = [1,1,1,1]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB108_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovdqu (%rdi,%rax,4), %xmm1
+; CHECK-NEXT:    vpcmpltd %xmm0, %xmm1, %k1
+; CHECK-NEXT:    vpbroadcastd {{.*}}(%rip), %xmm1 {%k1}
+; CHECK-NEXT:    vmovdqu %xmm1, (%rdi,%rax,4)
+; CHECK-NEXT:    addq $4, %rax
+; CHECK-NEXT:    cmpq $1023, %rax # imm = 0x3FF
+; CHECK-NEXT:    jg .LBB108_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds i32, i32* %arg, i64 %tmp
+  %tmp3 = bitcast i32* %tmp2 to <4 x i32>*
+  %tmp4 = load <4 x i32>, <4 x i32>* %tmp3, align 4
+  %tmp5 = icmp slt <4 x i32> %tmp4, <i32 1, i32 1, i32 1, i32 1>
+  %tmp6 = select <4 x i1> %tmp5, <4 x i32> <i32 3, i32 3, i32 3, i32 3>, <4 x i32> %tmp4
+  %tmp7 = bitcast i32* %tmp2 to <4 x i32>*
+  store <4 x i32> %tmp6, <4 x i32>* %tmp7, align 4
+  %tmp8 = add i64 %tmp, 4
+  %tmp9 = icmp slt i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_pcmp_v8i32(i32* %arg) {
+; CHECK-LABEL: bcast_unfold_pcmp_v8i32:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    vpbroadcastd {{.*#+}} ymm0 = [1,1,1,1,1,1,1,1]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB109_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovdqu (%rdi,%rax,4), %ymm1
+; CHECK-NEXT:    vpcmpltd %ymm0, %ymm1, %k1
+; CHECK-NEXT:    vpbroadcastd {{.*}}(%rip), %ymm1 {%k1}
+; CHECK-NEXT:    vmovdqu %ymm1, (%rdi,%rax,4)
+; CHECK-NEXT:    addq $8, %rax
+; CHECK-NEXT:    cmpq $1023, %rax # imm = 0x3FF
+; CHECK-NEXT:    jg .LBB109_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds i32, i32* %arg, i64 %tmp
+  %tmp3 = bitcast i32* %tmp2 to <8 x i32>*
+  %tmp4 = load <8 x i32>, <8 x i32>* %tmp3, align 4
+  %tmp5 = icmp slt <8 x i32> %tmp4, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  %tmp6 = select <8 x i1> %tmp5, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>, <8 x i32> %tmp4
+  %tmp7 = bitcast i32* %tmp2 to <8 x i32>*
+  store <8 x i32> %tmp6, <8 x i32>* %tmp7, align 4
+  %tmp8 = add i64 %tmp, 8
+  %tmp9 = icmp slt i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_pcmp_v16i32(i32* %arg) {
+; CHECK-LABEL: bcast_unfold_pcmp_v16i32:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    vpbroadcastd {{.*#+}} zmm0 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB110_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovdqu64 (%rdi,%rax,4), %zmm1
+; CHECK-NEXT:    vpcmpltd %zmm0, %zmm1, %k1
+; CHECK-NEXT:    vpbroadcastd {{.*}}(%rip), %zmm1 {%k1}
+; CHECK-NEXT:    vmovdqu64 %zmm1, (%rdi,%rax,4)
+; CHECK-NEXT:    addq $16, %rax
+; CHECK-NEXT:    cmpq $1023, %rax # imm = 0x3FF
+; CHECK-NEXT:    jg .LBB110_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds i32, i32* %arg, i64 %tmp
+  %tmp3 = bitcast i32* %tmp2 to <16 x i32>*
+  %tmp4 = load <16 x i32>, <16 x i32>* %tmp3, align 4
+  %tmp5 = icmp slt <16 x i32> %tmp4, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  %tmp6 = select <16 x i1> %tmp5, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>, <16 x i32> %tmp4
+  %tmp7 = bitcast i32* %tmp2 to <16 x i32>*
+  store <16 x i32> %tmp6, <16 x i32>* %tmp7, align 4
+  %tmp8 = add i64 %tmp, 16
+  %tmp9 = icmp slt i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_pcmp_v2i64(i64* %arg) {
+; CHECK-LABEL: bcast_unfold_pcmp_v2i64:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    vmovdqa {{.*#+}} xmm0 = [1,1]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB111_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovdqu (%rdi,%rax,8), %xmm1
+; CHECK-NEXT:    vpcmpltq %xmm0, %xmm1, %k1
+; CHECK-NEXT:    vmovdqa64 {{.*}}(%rip), %xmm1 {%k1}
+; CHECK-NEXT:    vmovdqu %xmm1, (%rdi,%rax,8)
+; CHECK-NEXT:    addq $2, %rax
+; CHECK-NEXT:    cmpq $1023, %rax # imm = 0x3FF
+; CHECK-NEXT:    jg .LBB111_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds i64, i64* %arg, i64 %tmp
+  %tmp3 = bitcast i64* %tmp2 to <2 x i64>*
+  %tmp4 = load <2 x i64>, <2 x i64>* %tmp3, align 4
+  %tmp5 = icmp slt <2 x i64> %tmp4, <i64 1, i64 1>
+  %tmp6 = select <2 x i1> %tmp5, <2 x i64> <i64 3, i64 3>, <2 x i64> %tmp4
+  %tmp7 = bitcast i64* %tmp2 to <2 x i64>*
+  store <2 x i64> %tmp6, <2 x i64>* %tmp7, align 4
+  %tmp8 = add i64 %tmp, 2
+  %tmp9 = icmp slt i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+define void @bcast_unfold_pcmp_v4i64(i64* %arg) {
+; CHECK-LABEL: bcast_unfold_pcmp_v4i64:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    vpbroadcastq {{.*#+}} ymm0 = [1,1,1,1]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB112_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovdqu (%rdi,%rax,8), %ymm1
+; CHECK-NEXT:    vpcmpltq %ymm0, %ymm1, %k1
+; CHECK-NEXT:    vpbroadcastq {{.*}}(%rip), %ymm1 {%k1}
+; CHECK-NEXT:    vmovdqu %ymm1, (%rdi,%rax,8)
+; CHECK-NEXT:    addq $4, %rax
+; CHECK-NEXT:    cmpq $1023, %rax # imm = 0x3FF
+; CHECK-NEXT:    jg .LBB112_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds i64, i64* %arg, i64 %tmp
+  %tmp3 = bitcast i64* %tmp2 to <4 x i64>*
+  %tmp4 = load <4 x i64>, <4 x i64>* %tmp3, align 4
+  %tmp5 = icmp slt <4 x i64> %tmp4, <i64 1, i64 1, i64 1, i64 1>
+  %tmp6 = select <4 x i1> %tmp5, <4 x i64> <i64 3, i64 3, i64 3, i64 3>, <4 x i64> %tmp4
+  %tmp7 = bitcast i64* %tmp2 to <4 x i64>*
+  store <4 x i64> %tmp6, <4 x i64>* %tmp7, align 4
+  %tmp8 = add i64 %tmp, 4
+  %tmp9 = icmp slt i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_pcmp_v8i64(i64* %arg) {
+; CHECK-LABEL: bcast_unfold_pcmp_v8i64:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    vpbroadcastq {{.*#+}} zmm0 = [1,1,1,1,1,1,1,1]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB113_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovdqu64 (%rdi,%rax,8), %zmm1
+; CHECK-NEXT:    vpcmpltq %zmm0, %zmm1, %k1
+; CHECK-NEXT:    vpbroadcastq {{.*}}(%rip), %zmm1 {%k1}
+; CHECK-NEXT:    vmovdqu64 %zmm1, (%rdi,%rax,8)
+; CHECK-NEXT:    addq $8, %rax
+; CHECK-NEXT:    cmpq $1023, %rax # imm = 0x3FF
+; CHECK-NEXT:    jg .LBB113_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds i64, i64* %arg, i64 %tmp
+  %tmp3 = bitcast i64* %tmp2 to <8 x i64>*
+  %tmp4 = load <8 x i64>, <8 x i64>* %tmp3, align 4
+  %tmp5 = icmp slt <8 x i64> %tmp4, <i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1>
+  %tmp6 = select <8 x i1> %tmp5, <8 x i64> <i64 3, i64 3, i64 3, i64 3, i64 3, i64 3, i64 3, i64 3>, <8 x i64> %tmp4
+  %tmp7 = bitcast i64* %tmp2 to <8 x i64>*
+  store <8 x i64> %tmp6, <8 x i64>* %tmp7, align 4
+  %tmp8 = add i64 %tmp, 8
+  %tmp9 = icmp slt i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_pcmpu_v4i32(i32* %arg) {
+; CHECK-LABEL: bcast_unfold_pcmpu_v4i32:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    vpbroadcastd {{.*#+}} xmm0 = [2,2,2,2]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB114_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovdqu (%rdi,%rax,4), %xmm1
+; CHECK-NEXT:    vpcmpltud %xmm0, %xmm1, %k1
+; CHECK-NEXT:    vpbroadcastd {{.*}}(%rip), %xmm1 {%k1}
+; CHECK-NEXT:    vmovdqu %xmm1, (%rdi,%rax,4)
+; CHECK-NEXT:    addq $4, %rax
+; CHECK-NEXT:    cmpq $1023, %rax # imm = 0x3FF
+; CHECK-NEXT:    ja .LBB114_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds i32, i32* %arg, i64 %tmp
+  %tmp3 = bitcast i32* %tmp2 to <4 x i32>*
+  %tmp4 = load <4 x i32>, <4 x i32>* %tmp3, align 4
+  %tmp5 = icmp ult <4 x i32> %tmp4, <i32 2, i32 2, i32 2, i32 2>
+  %tmp6 = select <4 x i1> %tmp5, <4 x i32> <i32 3, i32 3, i32 3, i32 3>, <4 x i32> %tmp4
+  %tmp7 = bitcast i32* %tmp2 to <4 x i32>*
+  store <4 x i32> %tmp6, <4 x i32>* %tmp7, align 4
+  %tmp8 = add i64 %tmp, 4
+  %tmp9 = icmp ult i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_pcmpu_v8i32(i32* %arg) {
+; CHECK-LABEL: bcast_unfold_pcmpu_v8i32:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    vpbroadcastd {{.*#+}} ymm0 = [2,2,2,2,2,2,2,2]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB115_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovdqu (%rdi,%rax,4), %ymm1
+; CHECK-NEXT:    vpcmpltud %ymm0, %ymm1, %k1
+; CHECK-NEXT:    vpbroadcastd {{.*}}(%rip), %ymm1 {%k1}
+; CHECK-NEXT:    vmovdqu %ymm1, (%rdi,%rax,4)
+; CHECK-NEXT:    addq $8, %rax
+; CHECK-NEXT:    cmpq $1023, %rax # imm = 0x3FF
+; CHECK-NEXT:    ja .LBB115_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds i32, i32* %arg, i64 %tmp
+  %tmp3 = bitcast i32* %tmp2 to <8 x i32>*
+  %tmp4 = load <8 x i32>, <8 x i32>* %tmp3, align 4
+  %tmp5 = icmp ult <8 x i32> %tmp4, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  %tmp6 = select <8 x i1> %tmp5, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>, <8 x i32> %tmp4
+  %tmp7 = bitcast i32* %tmp2 to <8 x i32>*
+  store <8 x i32> %tmp6, <8 x i32>* %tmp7, align 4
+  %tmp8 = add i64 %tmp, 8
+  %tmp9 = icmp ult i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_pcmpu_v16i32(i32* %arg) {
+; CHECK-LABEL: bcast_unfold_pcmpu_v16i32:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    vpbroadcastd {{.*#+}} zmm0 = [2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB116_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovdqu64 (%rdi,%rax,4), %zmm1
+; CHECK-NEXT:    vpcmpltud %zmm0, %zmm1, %k1
+; CHECK-NEXT:    vpbroadcastd {{.*}}(%rip), %zmm1 {%k1}
+; CHECK-NEXT:    vmovdqu64 %zmm1, (%rdi,%rax,4)
+; CHECK-NEXT:    addq $16, %rax
+; CHECK-NEXT:    cmpq $1023, %rax # imm = 0x3FF
+; CHECK-NEXT:    ja .LBB116_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds i32, i32* %arg, i64 %tmp
+  %tmp3 = bitcast i32* %tmp2 to <16 x i32>*
+  %tmp4 = load <16 x i32>, <16 x i32>* %tmp3, align 4
+  %tmp5 = icmp ult <16 x i32> %tmp4, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  %tmp6 = select <16 x i1> %tmp5, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>, <16 x i32> %tmp4
+  %tmp7 = bitcast i32* %tmp2 to <16 x i32>*
+  store <16 x i32> %tmp6, <16 x i32>* %tmp7, align 4
+  %tmp8 = add i64 %tmp, 16
+  %tmp9 = icmp ult i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_pcmpu_v2i64(i64* %arg) {
+; CHECK-LABEL: bcast_unfold_pcmpu_v2i64:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    vmovdqa {{.*#+}} xmm0 = [2,2]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB117_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovdqu (%rdi,%rax,8), %xmm1
+; CHECK-NEXT:    vpcmpltuq %xmm0, %xmm1, %k1
+; CHECK-NEXT:    vmovdqa64 {{.*}}(%rip), %xmm1 {%k1}
+; CHECK-NEXT:    vmovdqu %xmm1, (%rdi,%rax,8)
+; CHECK-NEXT:    addq $2, %rax
+; CHECK-NEXT:    cmpq $1023, %rax # imm = 0x3FF
+; CHECK-NEXT:    ja .LBB117_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds i64, i64* %arg, i64 %tmp
+  %tmp3 = bitcast i64* %tmp2 to <2 x i64>*
+  %tmp4 = load <2 x i64>, <2 x i64>* %tmp3, align 4
+  %tmp5 = icmp ult <2 x i64> %tmp4, <i64 2, i64 2>
+  %tmp6 = select <2 x i1> %tmp5, <2 x i64> <i64 3, i64 3>, <2 x i64> %tmp4
+  %tmp7 = bitcast i64* %tmp2 to <2 x i64>*
+  store <2 x i64> %tmp6, <2 x i64>* %tmp7, align 4
+  %tmp8 = add i64 %tmp, 2
+  %tmp9 = icmp ult i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+define void @bcast_unfold_pcmpu_v4i64(i64* %arg) {
+; CHECK-LABEL: bcast_unfold_pcmpu_v4i64:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    vpbroadcastq {{.*#+}} ymm0 = [2,2,2,2]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB118_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovdqu (%rdi,%rax,8), %ymm1
+; CHECK-NEXT:    vpcmpltuq %ymm0, %ymm1, %k1
+; CHECK-NEXT:    vpbroadcastq {{.*}}(%rip), %ymm1 {%k1}
+; CHECK-NEXT:    vmovdqu %ymm1, (%rdi,%rax,8)
+; CHECK-NEXT:    addq $4, %rax
+; CHECK-NEXT:    cmpq $1023, %rax # imm = 0x3FF
+; CHECK-NEXT:    ja .LBB118_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds i64, i64* %arg, i64 %tmp
+  %tmp3 = bitcast i64* %tmp2 to <4 x i64>*
+  %tmp4 = load <4 x i64>, <4 x i64>* %tmp3, align 4
+  %tmp5 = icmp ult <4 x i64> %tmp4, <i64 2, i64 2, i64 2, i64 2>
+  %tmp6 = select <4 x i1> %tmp5, <4 x i64> <i64 3, i64 3, i64 3, i64 3>, <4 x i64> %tmp4
+  %tmp7 = bitcast i64* %tmp2 to <4 x i64>*
+  store <4 x i64> %tmp6, <4 x i64>* %tmp7, align 4
+  %tmp8 = add i64 %tmp, 4
+  %tmp9 = icmp ult i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_pcmpu_v8i64(i64* %arg) {
+; CHECK-LABEL: bcast_unfold_pcmpu_v8i64:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    vpbroadcastq {{.*#+}} zmm0 = [2,2,2,2,2,2,2,2]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB119_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovdqu64 (%rdi,%rax,8), %zmm1
+; CHECK-NEXT:    vpcmpltuq %zmm0, %zmm1, %k1
+; CHECK-NEXT:    vpbroadcastq {{.*}}(%rip), %zmm1 {%k1}
+; CHECK-NEXT:    vmovdqu64 %zmm1, (%rdi,%rax,8)
+; CHECK-NEXT:    addq $8, %rax
+; CHECK-NEXT:    cmpq $1023, %rax # imm = 0x3FF
+; CHECK-NEXT:    ja .LBB119_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds i64, i64* %arg, i64 %tmp
+  %tmp3 = bitcast i64* %tmp2 to <8 x i64>*
+  %tmp4 = load <8 x i64>, <8 x i64>* %tmp3, align 4
+  %tmp5 = icmp ult <8 x i64> %tmp4, <i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2>
+  %tmp6 = select <8 x i1> %tmp5, <8 x i64> <i64 3, i64 3, i64 3, i64 3, i64 3, i64 3, i64 3, i64 3>, <8 x i64> %tmp4
+  %tmp7 = bitcast i64* %tmp2 to <8 x i64>*
+  store <8 x i64> %tmp6, <8 x i64>* %tmp7, align 4
+  %tmp8 = add i64 %tmp, 8
+  %tmp9 = icmp ult i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_cmp_v4f32(float* %arg) {
+; CHECK-LABEL: bcast_unfold_cmp_v4f32:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
+; CHECK-NEXT:    vbroadcastss {{.*#+}} xmm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0]
+; CHECK-NEXT:    vbroadcastss {{.*#+}} xmm1 = [3.0E+0,3.0E+0,3.0E+0,3.0E+0]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB120_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovups 4096(%rdi,%rax), %xmm2
+; CHECK-NEXT:    vcmpltps %xmm0, %xmm2, %k1
+; CHECK-NEXT:    vblendmps %xmm2, %xmm1, %xmm2 {%k1}
+; CHECK-NEXT:    vmovups %xmm2, 4096(%rdi,%rax)
+; CHECK-NEXT:    addq $16, %rax
+; CHECK-NEXT:    jne .LBB120_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds float, float* %arg, i64 %tmp
+  %tmp3 = bitcast float* %tmp2 to <4 x float>*
+  %tmp4 = load <4 x float>, <4 x float>* %tmp3, align 4
+  %tmp5 = fcmp olt <4 x float> %tmp4, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
+  %tmp6 = select <4 x i1> %tmp5, <4 x float> %tmp4, <4 x float> <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00>
+  %tmp7 = bitcast float* %tmp2 to <4 x float>*
+  store <4 x float> %tmp6, <4 x float>* %tmp7, align 4
+  %tmp8 = add i64 %tmp, 4
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_cmp_v8f32(float* %arg) {
+; CHECK-LABEL: bcast_unfold_cmp_v8f32:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
+; CHECK-NEXT:    vbroadcastss {{.*#+}} ymm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]
+; CHECK-NEXT:    vbroadcastss {{.*#+}} ymm1 = [3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB121_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovups 4096(%rdi,%rax), %ymm2
+; CHECK-NEXT:    vcmpltps %ymm0, %ymm2, %k1
+; CHECK-NEXT:    vblendmps %ymm2, %ymm1, %ymm2 {%k1}
+; CHECK-NEXT:    vmovups %ymm2, 4096(%rdi,%rax)
+; CHECK-NEXT:    addq $32, %rax
+; CHECK-NEXT:    jne .LBB121_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds float, float* %arg, i64 %tmp
+  %tmp3 = bitcast float* %tmp2 to <8 x float>*
+  %tmp4 = load <8 x float>, <8 x float>* %tmp3, align 4
+  %tmp5 = fcmp olt <8 x float> %tmp4, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
+  %tmp6 = select <8 x i1> %tmp5, <8 x float> %tmp4, <8 x float> <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00>
+  %tmp7 = bitcast float* %tmp2 to <8 x float>*
+  store <8 x float> %tmp6, <8 x float>* %tmp7, align 4
+  %tmp8 = add i64 %tmp, 8
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_cmp_v16f32(float* %arg) {
+; CHECK-LABEL: bcast_unfold_cmp_v16f32:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
+; CHECK-NEXT:    vbroadcastss {{.*#+}} zmm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]
+; CHECK-NEXT:    vbroadcastss {{.*#+}} zmm1 = [3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB122_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovups 4096(%rdi,%rax), %zmm2
+; CHECK-NEXT:    vcmpltps %zmm0, %zmm2, %k1
+; CHECK-NEXT:    vblendmps %zmm2, %zmm1, %zmm2 {%k1}
+; CHECK-NEXT:    vmovups %zmm2, 4096(%rdi,%rax)
+; CHECK-NEXT:    addq $64, %rax
+; CHECK-NEXT:    jne .LBB122_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds float, float* %arg, i64 %tmp
+  %tmp3 = bitcast float* %tmp2 to <16 x float>*
+  %tmp4 = load <16 x float>, <16 x float>* %tmp3, align 4
+  %tmp5 = fcmp olt <16 x float> %tmp4, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
+  %tmp6 = select <16 x i1> %tmp5, <16 x float> %tmp4, <16 x float> <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00>
+  %tmp7 = bitcast float* %tmp2 to <16 x float>*
+  store <16 x float> %tmp6, <16 x float>* %tmp7, align 4
+  %tmp8 = add i64 %tmp, 16
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_cmp_v2f64(double* %arg) {
+; CHECK-LABEL: bcast_unfold_cmp_v2f64:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
+; CHECK-NEXT:    vmovapd {{.*#+}} xmm0 = [2.0E+0,2.0E+0]
+; CHECK-NEXT:    vmovapd {{.*#+}} xmm1 = [3.0E+0,3.0E+0]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB123_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovupd 8192(%rdi,%rax), %xmm2
+; CHECK-NEXT:    vcmpltpd %xmm0, %xmm2, %k1
+; CHECK-NEXT:    vblendmpd %xmm2, %xmm1, %xmm2 {%k1}
+; CHECK-NEXT:    vmovupd %xmm2, 8192(%rdi,%rax)
+; CHECK-NEXT:    addq $16, %rax
+; CHECK-NEXT:    jne .LBB123_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds double, double* %arg, i64 %tmp
+  %tmp3 = bitcast double* %tmp2 to <2 x double>*
+  %tmp4 = load <2 x double>, <2 x double>* %tmp3, align 8
+  %tmp5 = fcmp olt <2 x double> %tmp4, <double 2.000000e+00, double 2.000000e+00>
+  %tmp6 = select <2 x i1> %tmp5, <2 x double> %tmp4, <2 x double> <double 3.000000e+00, double 3.000000e+00>
+  %tmp7 = bitcast double* %tmp2 to <2 x double>*
+  store <2 x double> %tmp6, <2 x double>* %tmp7, align 8
+  %tmp8 = add i64 %tmp, 2
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_cmp_v4f64(double* %arg) {
+; CHECK-LABEL: bcast_unfold_cmp_v4f64:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
+; CHECK-NEXT:    vbroadcastsd {{.*#+}} ymm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0]
+; CHECK-NEXT:    vbroadcastsd {{.*#+}} ymm1 = [3.0E+0,3.0E+0,3.0E+0,3.0E+0]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB124_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovupd 8192(%rdi,%rax), %ymm2
+; CHECK-NEXT:    vcmpltpd %ymm0, %ymm2, %k1
+; CHECK-NEXT:    vblendmpd %ymm2, %ymm1, %ymm2 {%k1}
+; CHECK-NEXT:    vmovupd %ymm2, 8192(%rdi,%rax)
+; CHECK-NEXT:    addq $32, %rax
+; CHECK-NEXT:    jne .LBB124_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds double, double* %arg, i64 %tmp
+  %tmp3 = bitcast double* %tmp2 to <4 x double>*
+  %tmp4 = load <4 x double>, <4 x double>* %tmp3, align 8
+  %tmp5 = fcmp olt <4 x double> %tmp4, <double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00>
+  %tmp6 = select <4 x i1> %tmp5, <4 x double> %tmp4, <4 x double> <double 3.000000e+00, double 3.000000e+00, double 3.000000e+00, double 3.000000e+00>
+  %tmp7 = bitcast double* %tmp2 to <4 x double>*
+  store <4 x double> %tmp6, <4 x double>* %tmp7, align 8
+  %tmp8 = add i64 %tmp, 4
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_cmp_v8f64(double* %arg) {
+; CHECK-LABEL: bcast_unfold_cmp_v8f64:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
+; CHECK-NEXT:    vbroadcastsd {{.*#+}} zmm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]
+; CHECK-NEXT:    vbroadcastsd {{.*#+}} zmm1 = [3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB125_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovupd 8192(%rdi,%rax), %zmm2
+; CHECK-NEXT:    vcmpltpd %zmm0, %zmm2, %k1
+; CHECK-NEXT:    vblendmpd %zmm2, %zmm1, %zmm2 {%k1}
+; CHECK-NEXT:    vmovupd %zmm2, 8192(%rdi,%rax)
+; CHECK-NEXT:    addq $64, %rax
+; CHECK-NEXT:    jne .LBB125_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds double, double* %arg, i64 %tmp
+  %tmp3 = bitcast double* %tmp2 to <8 x double>*
+  %tmp4 = load <8 x double>, <8 x double>* %tmp3, align 8
+  %tmp5 = fcmp olt <8 x double> %tmp4, <double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00>
+  %tmp6 = select <8 x i1> %tmp5, <8 x double> %tmp4, <8 x double> <double 3.000000e+00, double 3.000000e+00, double 3.000000e+00, double 3.000000e+00, double 3.000000e+00, double 3.000000e+00, double 3.000000e+00, double 3.000000e+00>
+  %tmp7 = bitcast double* %tmp2 to <8 x double>*
+  store <8 x double> %tmp6, <8 x double>* %tmp7, align 8
+  %tmp8 = add i64 %tmp, 8
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_cmp_v8f32_refold(float* nocapture %0) {
+; CHECK-LABEL: bcast_unfold_cmp_v8f32_refold:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
+; CHECK-NEXT:    vbroadcastss {{.*#+}} ymm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]
+; CHECK-NEXT:    vbroadcastss {{.*#+}} ymm1 = [3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB126_1: # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vcmpgtps 4096(%rdi,%rax), %ymm0, %k1
+; CHECK-NEXT:    vblendmps {{.*}}(%rip){1to8}, %ymm1, %ymm2 {%k1}
+; CHECK-NEXT:    vmovups %ymm2, 4096(%rdi,%rax)
+; CHECK-NEXT:    addq $32, %rax
+; CHECK-NEXT:    jne .LBB126_1
+; CHECK-NEXT:  # %bb.2:
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  br label %2
+
+2:                                                ; preds = %2, %1
+  %3 = phi i64 [ 0, %1 ], [ %10, %2 ]
+  %4 = getelementptr inbounds float, float* %0, i64 %3
+  %5 = bitcast float* %4 to <8 x float>*
+  %6 = load <8 x float>, <8 x float>* %5, align 4
+  %7 = fcmp olt <8 x float> %6, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
+  %8 = select <8 x i1> %7, <8 x float> <float 4.000000e+00, float 4.000000e+00, float 4.000000e+00, float 4.000000e+00, float 4.000000e+00, float 4.000000e+00, float 4.000000e+00, float 4.000000e+00>, <8 x float> <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00>
+  %9 = bitcast float* %4 to <8 x float>*
+  store <8 x float> %8, <8 x float>* %9, align 4
+  %10 = add i64 %3, 8
+  %11 = icmp eq i64 %10, 1024
+  br i1 %11, label %12, label %2
+
+12:                                               ; preds = %2
+  ret void
+}
+
+define void @bcast_unfold_ptestm_v4i32(i32* %arg) {
+; CHECK-LABEL: bcast_unfold_ptestm_v4i32:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
+; CHECK-NEXT:    vpbroadcastd {{.*#+}} xmm0 = [2,2,2,2]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB127_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovdqu 4096(%rdi,%rax), %xmm1
+; CHECK-NEXT:    vptestmd %xmm0, %xmm1, %k1
+; CHECK-NEXT:    vpbroadcastd {{.*}}(%rip), %xmm1 {%k1}
+; CHECK-NEXT:    vmovdqu %xmm1, 4096(%rdi,%rax)
+; CHECK-NEXT:    addq $16, %rax
+; CHECK-NEXT:    jne .LBB127_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds i32, i32* %arg, i64 %tmp
+  %tmp3 = bitcast i32* %tmp2 to <4 x i32>*
+  %tmp4 = load <4 x i32>, <4 x i32>* %tmp3, align 4
+  %tmp4b = and <4 x i32> %tmp4, <i32 2, i32 2, i32 2, i32 2>
+  %tmp5 = icmp ne <4 x i32> %tmp4b, zeroinitializer
+  %tmp6 = select <4 x i1> %tmp5, <4 x i32> <i32 3, i32 3, i32 3, i32 3>, <4 x i32> %tmp4
+  %tmp7 = bitcast i32* %tmp2 to <4 x i32>*
+  store <4 x i32> %tmp6, <4 x i32>* %tmp7, align 4
+  %tmp8 = add i64 %tmp, 4
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_ptestnm_v4i32(i32* %arg) {
+; CHECK-LABEL: bcast_unfold_ptestnm_v4i32:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
+; CHECK-NEXT:    vpbroadcastd {{.*#+}} xmm0 = [2,2,2,2]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB128_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovdqu 4096(%rdi,%rax), %xmm1
+; CHECK-NEXT:    vptestnmd %xmm0, %xmm1, %k1
+; CHECK-NEXT:    vpbroadcastd {{.*}}(%rip), %xmm1 {%k1}
+; CHECK-NEXT:    vmovdqu %xmm1, 4096(%rdi,%rax)
+; CHECK-NEXT:    addq $16, %rax
+; CHECK-NEXT:    jne .LBB128_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds i32, i32* %arg, i64 %tmp
+  %tmp3 = bitcast i32* %tmp2 to <4 x i32>*
+  %tmp4 = load <4 x i32>, <4 x i32>* %tmp3, align 4
+  %tmp4b = and <4 x i32> %tmp4, <i32 2, i32 2, i32 2, i32 2>
+  %tmp5 = icmp eq <4 x i32> %tmp4b, zeroinitializer
+  %tmp6 = select <4 x i1> %tmp5, <4 x i32> <i32 3, i32 3, i32 3, i32 3>, <4 x i32> %tmp4
+  %tmp7 = bitcast i32* %tmp2 to <4 x i32>*
+  store <4 x i32> %tmp6, <4 x i32>* %tmp7, align 4
+  %tmp8 = add i64 %tmp, 4
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_ptestm_v4i64(i64* %arg) {
+; CHECK-LABEL: bcast_unfold_ptestm_v4i64:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
+; CHECK-NEXT:    vpbroadcastq {{.*#+}} ymm0 = [2,2,2,2]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB129_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovdqu 8192(%rdi,%rax), %ymm1
+; CHECK-NEXT:    vptestmq %ymm0, %ymm1, %k1
+; CHECK-NEXT:    vpbroadcastq {{.*}}(%rip), %ymm1 {%k1}
+; CHECK-NEXT:    vmovdqu %ymm1, 8192(%rdi,%rax)
+; CHECK-NEXT:    addq $32, %rax
+; CHECK-NEXT:    jne .LBB129_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds i64, i64* %arg, i64 %tmp
+  %tmp3 = bitcast i64* %tmp2 to <4 x i64>*
+  %tmp4 = load <4 x i64>, <4 x i64>* %tmp3, align 8
+  %tmp4b = and <4 x i64> %tmp4, <i64 2, i64 2, i64 2, i64 2>
+  %tmp5 = icmp ne <4 x i64> %tmp4b, zeroinitializer
+  %tmp6 = select <4 x i1> %tmp5, <4 x i64> <i64 3, i64 3, i64 3, i64 3>, <4 x i64> %tmp4
+  %tmp7 = bitcast i64* %tmp2 to <4 x i64>*
+  store <4 x i64> %tmp6, <4 x i64>* %tmp7, align 8
+  %tmp8 = add i64 %tmp, 4
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_ptestnm_v4i64(i64* %arg) {
+; CHECK-LABEL: bcast_unfold_ptestnm_v4i64:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
+; CHECK-NEXT:    vpbroadcastq {{.*#+}} ymm0 = [2,2,2,2]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB130_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovdqu 8192(%rdi,%rax), %ymm1
+; CHECK-NEXT:    vptestnmq %ymm0, %ymm1, %k1
+; CHECK-NEXT:    vpbroadcastq {{.*}}(%rip), %ymm1 {%k1}
+; CHECK-NEXT:    vmovdqu %ymm1, 8192(%rdi,%rax)
+; CHECK-NEXT:    addq $32, %rax
+; CHECK-NEXT:    jne .LBB130_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds i64, i64* %arg, i64 %tmp
+  %tmp3 = bitcast i64* %tmp2 to <4 x i64>*
+  %tmp4 = load <4 x i64>, <4 x i64>* %tmp3, align 8
+  %tmp4b = and <4 x i64> %tmp4, <i64 2, i64 2, i64 2, i64 2>
+  %tmp5 = icmp eq <4 x i64> %tmp4b, zeroinitializer
+  %tmp6 = select <4 x i1> %tmp5, <4 x i64> <i64 3, i64 3, i64 3, i64 3>, <4 x i64> %tmp4
+  %tmp7 = bitcast i64* %tmp2 to <4 x i64>*
+  store <4 x i64> %tmp6, <4 x i64>* %tmp7, align 8
+  %tmp8 = add i64 %tmp, 4
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
diff --git a/llvm/test/CodeGen/X86/avx512-bugfix-25270.ll b/llvm/test/CodeGen/X86/avx512-bugfix-25270.ll
index b48bf4fe2..c3466df4d 100644
--- a/llvm/test/CodeGen/X86/avx512-bugfix-25270.ll
+++ b/llvm/test/CodeGen/X86/avx512-bugfix-25270.ll
@@ -11,12 +11,12 @@ define void @bar__512(<16 x i32>* %var) #0 {
 ; CHECK-NEXT:    movq %rdi, %rbx
 ; CHECK-NEXT:    vmovups (%rdi), %zmm0
 ; CHECK-NEXT:    vmovups %zmm0, (%rsp) ## 64-byte Spill
-; CHECK-NEXT:    vbroadcastss {{.*}}(%rip), %zmm1
+; CHECK-NEXT:    vbroadcastss {{.*#+}} zmm1 = [4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4]
 ; CHECK-NEXT:    vmovaps %zmm1, (%rdi)
 ; CHECK-NEXT:    callq _Print__512
 ; CHECK-NEXT:    vmovups (%rsp), %zmm0 ## 64-byte Reload
 ; CHECK-NEXT:    callq _Print__512
-; CHECK-NEXT:    vbroadcastss {{.*}}(%rip), %zmm0
+; CHECK-NEXT:    vbroadcastss {{.*#+}} zmm0 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
 ; CHECK-NEXT:    vmovaps %zmm0, (%rbx)
 ; CHECK-NEXT:    addq $112, %rsp
 ; CHECK-NEXT:    popq %rbx
diff --git a/llvm/test/CodeGen/X86/avx512-calling-conv.ll b/llvm/test/CodeGen/X86/avx512-calling-conv.ll
index 4145fe98c..f07067f97 100644
--- a/llvm/test/CodeGen/X86/avx512-calling-conv.ll
+++ b/llvm/test/CodeGen/X86/avx512-calling-conv.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-apple-darwin9 -mcpu=knl | FileCheck %s --check-prefix=ALL_X64 --check-prefix=KNL
+; RUN: llc < %s -mtriple=x86_64-apple-darwin9 -mcpu=knl | FileCheck %s --check-prefix=ALL_X64 --check-prefix=KNL --check-prefix=KNL-NEW
+; RUN: llc < %s -mtriple=x86_64-apple-darwin9 -mcpu=knl -x86-enable-old-knl-abi | FileCheck %s --check-prefix=ALL_X64 --check-prefix=KNL --check-prefix=KNL-OLD
 ; RUN: llc < %s -mtriple=x86_64-apple-darwin9 -mcpu=skx | FileCheck %s --check-prefix=ALL_X64 --check-prefix=SKX
 ; RUN: llc < %s -mtriple=i686-apple-darwin9 -mcpu=knl | FileCheck %s --check-prefix=KNL_X32
 
@@ -397,3 +398,2315 @@ define <1 x i1> @test13(<1 x i1>* %foo) {
   %bar = load <1 x i1>, <1 x i1>* %foo
   ret <1 x i1> %bar
 }
+
+define void @test14(<32 x i16>* %x) {
+; KNL-NEW-LABEL: test14:
+; KNL-NEW:       ## %bb.0:
+; KNL-NEW-NEXT:    pushq %rbx
+; KNL-NEW-NEXT:    .cfi_def_cfa_offset 16
+; KNL-NEW-NEXT:    .cfi_offset %rbx, -16
+; KNL-NEW-NEXT:    movq %rdi, %rbx
+; KNL-NEW-NEXT:    vmovaps (%rdi), %zmm0
+; KNL-NEW-NEXT:    callq _test14_callee
+; KNL-NEW-NEXT:    vmovaps %zmm0, (%rbx)
+; KNL-NEW-NEXT:    popq %rbx
+; KNL-NEW-NEXT:    retq
+;
+; KNL-OLD-LABEL: test14:
+; KNL-OLD:       ## %bb.0:
+; KNL-OLD-NEXT:    pushq %rbx
+; KNL-OLD-NEXT:    .cfi_def_cfa_offset 16
+; KNL-OLD-NEXT:    .cfi_offset %rbx, -16
+; KNL-OLD-NEXT:    movq %rdi, %rbx
+; KNL-OLD-NEXT:    vmovaps (%rdi), %ymm0
+; KNL-OLD-NEXT:    vmovaps 32(%rdi), %ymm1
+; KNL-OLD-NEXT:    callq _test14_callee
+; KNL-OLD-NEXT:    vmovaps %ymm1, 32(%rbx)
+; KNL-OLD-NEXT:    vmovaps %ymm0, (%rbx)
+; KNL-OLD-NEXT:    popq %rbx
+; KNL-OLD-NEXT:    retq
+;
+; SKX-LABEL: test14:
+; SKX:       ## %bb.0:
+; SKX-NEXT:    pushq %rbx
+; SKX-NEXT:    .cfi_def_cfa_offset 16
+; SKX-NEXT:    .cfi_offset %rbx, -16
+; SKX-NEXT:    movq %rdi, %rbx
+; SKX-NEXT:    vmovaps (%rdi), %zmm0
+; SKX-NEXT:    callq _test14_callee
+; SKX-NEXT:    vmovaps %zmm0, (%rbx)
+; SKX-NEXT:    popq %rbx
+; SKX-NEXT:    vzeroupper
+; SKX-NEXT:    retq
+;
+; KNL_X32-LABEL: test14:
+; KNL_X32:       ## %bb.0:
+; KNL_X32-NEXT:    pushl %esi
+; KNL_X32-NEXT:    .cfi_def_cfa_offset 8
+; KNL_X32-NEXT:    subl $8, %esp
+; KNL_X32-NEXT:    .cfi_def_cfa_offset 16
+; KNL_X32-NEXT:    .cfi_offset %esi, -8
+; KNL_X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; KNL_X32-NEXT:    vmovaps (%esi), %zmm0
+; KNL_X32-NEXT:    calll _test14_callee
+; KNL_X32-NEXT:    vmovaps %zmm0, (%esi)
+; KNL_X32-NEXT:    addl $8, %esp
+; KNL_X32-NEXT:    popl %esi
+; KNL_X32-NEXT:    retl
+  %a = load <32 x i16>, <32 x i16>* %x
+  %b = call <32 x i16> @test14_callee(<32 x i16> %a)
+  store <32 x i16> %b, <32 x i16>* %x
+  ret void
+}
+declare <32 x i16> @test14_callee(<32 x i16>)
+
+define void @test15(<64 x i8>* %x) {
+; KNL-NEW-LABEL: test15:
+; KNL-NEW:       ## %bb.0:
+; KNL-NEW-NEXT:    pushq %rbx
+; KNL-NEW-NEXT:    .cfi_def_cfa_offset 16
+; KNL-NEW-NEXT:    .cfi_offset %rbx, -16
+; KNL-NEW-NEXT:    movq %rdi, %rbx
+; KNL-NEW-NEXT:    vmovaps (%rdi), %zmm0
+; KNL-NEW-NEXT:    callq _test15_callee
+; KNL-NEW-NEXT:    vmovaps %zmm0, (%rbx)
+; KNL-NEW-NEXT:    popq %rbx
+; KNL-NEW-NEXT:    retq
+;
+; KNL-OLD-LABEL: test15:
+; KNL-OLD:       ## %bb.0:
+; KNL-OLD-NEXT:    pushq %rbx
+; KNL-OLD-NEXT:    .cfi_def_cfa_offset 16
+; KNL-OLD-NEXT:    .cfi_offset %rbx, -16
+; KNL-OLD-NEXT:    movq %rdi, %rbx
+; KNL-OLD-NEXT:    vmovaps (%rdi), %ymm0
+; KNL-OLD-NEXT:    vmovaps 32(%rdi), %ymm1
+; KNL-OLD-NEXT:    callq _test15_callee
+; KNL-OLD-NEXT:    vmovaps %ymm1, 32(%rbx)
+; KNL-OLD-NEXT:    vmovaps %ymm0, (%rbx)
+; KNL-OLD-NEXT:    popq %rbx
+; KNL-OLD-NEXT:    retq
+;
+; SKX-LABEL: test15:
+; SKX:       ## %bb.0:
+; SKX-NEXT:    pushq %rbx
+; SKX-NEXT:    .cfi_def_cfa_offset 16
+; SKX-NEXT:    .cfi_offset %rbx, -16
+; SKX-NEXT:    movq %rdi, %rbx
+; SKX-NEXT:    vmovaps (%rdi), %zmm0
+; SKX-NEXT:    callq _test15_callee
+; SKX-NEXT:    vmovaps %zmm0, (%rbx)
+; SKX-NEXT:    popq %rbx
+; SKX-NEXT:    vzeroupper
+; SKX-NEXT:    retq
+;
+; KNL_X32-LABEL: test15:
+; KNL_X32:       ## %bb.0:
+; KNL_X32-NEXT:    pushl %esi
+; KNL_X32-NEXT:    .cfi_def_cfa_offset 8
+; KNL_X32-NEXT:    subl $8, %esp
+; KNL_X32-NEXT:    .cfi_def_cfa_offset 16
+; KNL_X32-NEXT:    .cfi_offset %esi, -8
+; KNL_X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; KNL_X32-NEXT:    vmovaps (%esi), %zmm0
+; KNL_X32-NEXT:    calll _test15_callee
+; KNL_X32-NEXT:    vmovaps %zmm0, (%esi)
+; KNL_X32-NEXT:    addl $8, %esp
+; KNL_X32-NEXT:    popl %esi
+; KNL_X32-NEXT:    retl
+  %a = load <64 x i8>, <64 x i8>* %x
+  %b = call <64 x i8> @test15_callee(<64 x i8> %a)
+  store <64 x i8> %b, <64 x i8>* %x
+  ret void
+}
+declare <64 x i8> @test15_callee(<64 x i8>)
+
+define <17 x i1> @test16(<17 x i1> %a, <17 x i1> %b) nounwind {
+; KNL-LABEL: test16:
+; KNL:       ## %bb.0:
+; KNL-NEXT:    pushq %rbp
+; KNL-NEXT:    pushq %r15
+; KNL-NEXT:    pushq %r14
+; KNL-NEXT:    pushq %r13
+; KNL-NEXT:    pushq %r12
+; KNL-NEXT:    pushq %rbx
+; KNL-NEXT:    movq %rdi, %rax
+; KNL-NEXT:    movw $-3, %di
+; KNL-NEXT:    kmovw %edi, %k1
+; KNL-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k0
+; KNL-NEXT:    kandw %k1, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k1
+; KNL-NEXT:    kshiftlw $15, %k1, %k1
+; KNL-NEXT:    kshiftrw $14, %k1, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    movw $-5, %di
+; KNL-NEXT:    kmovw %edi, %k1
+; KNL-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; KNL-NEXT:    kandw %k1, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k1
+; KNL-NEXT:    kshiftlw $15, %k1, %k1
+; KNL-NEXT:    kshiftrw $13, %k1, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    movw $-9, %di
+; KNL-NEXT:    kmovw %edi, %k1
+; KNL-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; KNL-NEXT:    kandw %k1, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k1
+; KNL-NEXT:    kshiftlw $15, %k1, %k1
+; KNL-NEXT:    kshiftrw $12, %k1, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    movw $-17, %di
+; KNL-NEXT:    kmovw %edi, %k1
+; KNL-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; KNL-NEXT:    kandw %k1, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k1
+; KNL-NEXT:    kshiftlw $15, %k1, %k1
+; KNL-NEXT:    kshiftrw $11, %k1, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    movw $-33, %di
+; KNL-NEXT:    kmovw %edi, %k1
+; KNL-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; KNL-NEXT:    kandw %k1, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k1
+; KNL-NEXT:    kshiftlw $15, %k1, %k1
+; KNL-NEXT:    kshiftrw $10, %k1, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    movw $-65, %di
+; KNL-NEXT:    kmovw %edi, %k1
+; KNL-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; KNL-NEXT:    kandw %k1, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k1
+; KNL-NEXT:    kshiftlw $15, %k1, %k1
+; KNL-NEXT:    kshiftrw $9, %k1, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    movw $-129, %di
+; KNL-NEXT:    kmovw %edi, %k1
+; KNL-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; KNL-NEXT:    kandw %k1, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k1
+; KNL-NEXT:    kshiftlw $15, %k1, %k1
+; KNL-NEXT:    kshiftrw $8, %k1, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    movw $-257, %di ## imm = 0xFEFF
+; KNL-NEXT:    kmovw %edi, %k1
+; KNL-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; KNL-NEXT:    kandw %k1, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k1
+; KNL-NEXT:    kshiftlw $15, %k1, %k1
+; KNL-NEXT:    kshiftrw $7, %k1, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    movw $-513, %di ## imm = 0xFDFF
+; KNL-NEXT:    kmovw %edi, %k5
+; KNL-NEXT:    kandw %k5, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k1
+; KNL-NEXT:    kshiftlw $15, %k1, %k1
+; KNL-NEXT:    kshiftrw $6, %k1, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    movw $-1025, %di ## imm = 0xFBFF
+; KNL-NEXT:    kmovw %edi, %k4
+; KNL-NEXT:    kandw %k4, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k1
+; KNL-NEXT:    kshiftlw $15, %k1, %k1
+; KNL-NEXT:    kshiftrw $5, %k1, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    movw $-2049, %di ## imm = 0xF7FF
+; KNL-NEXT:    kmovw %edi, %k3
+; KNL-NEXT:    kandw %k3, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k1
+; KNL-NEXT:    kshiftlw $15, %k1, %k1
+; KNL-NEXT:    kshiftrw $4, %k1, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    movw $-4097, %di ## imm = 0xEFFF
+; KNL-NEXT:    kmovw %edi, %k2
+; KNL-NEXT:    kandw %k2, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k1
+; KNL-NEXT:    kshiftlw $15, %k1, %k1
+; KNL-NEXT:    kshiftrw $3, %k1, %k1
+; KNL-NEXT:    korw %k1, %k0, %k1
+; KNL-NEXT:    movw $-8193, %di ## imm = 0xDFFF
+; KNL-NEXT:    kmovw %edi, %k0
+; KNL-NEXT:    kandw %k0, %k1, %k1
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k6
+; KNL-NEXT:    kshiftlw $15, %k6, %k6
+; KNL-NEXT:    kshiftrw $2, %k6, %k6
+; KNL-NEXT:    korw %k6, %k1, %k6
+; KNL-NEXT:    movw $-16385, %di ## imm = 0xBFFF
+; KNL-NEXT:    kmovw %edi, %k1
+; KNL-NEXT:    kandw %k1, %k6, %k6
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k7
+; KNL-NEXT:    kshiftlw $14, %k7, %k7
+; KNL-NEXT:    korw %k7, %k6, %k6
+; KNL-NEXT:    kshiftlw $1, %k6, %k6
+; KNL-NEXT:    kshiftrw $1, %k6, %k6
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    korw %k7, %k6, %k6
+; KNL-NEXT:    kmovw %k6, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; KNL-NEXT:    kmovw %esi, %k6
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 ## 2-byte Reload
+; KNL-NEXT:    kandw %k7, %k6, %k6
+; KNL-NEXT:    kmovw %edx, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $14, %k7, %k7
+; KNL-NEXT:    korw %k7, %k6, %k6
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 ## 2-byte Reload
+; KNL-NEXT:    kandw %k7, %k6, %k6
+; KNL-NEXT:    kmovw %ecx, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $13, %k7, %k7
+; KNL-NEXT:    korw %k7, %k6, %k6
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 ## 2-byte Reload
+; KNL-NEXT:    kandw %k7, %k6, %k6
+; KNL-NEXT:    kmovw %r8d, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $12, %k7, %k7
+; KNL-NEXT:    korw %k7, %k6, %k6
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 ## 2-byte Reload
+; KNL-NEXT:    kandw %k7, %k6, %k6
+; KNL-NEXT:    kmovw %r9d, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $11, %k7, %k7
+; KNL-NEXT:    korw %k7, %k6, %k6
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 ## 2-byte Reload
+; KNL-NEXT:    kandw %k7, %k6, %k6
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %cl
+; KNL-NEXT:    kmovw %ecx, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $10, %k7, %k7
+; KNL-NEXT:    korw %k7, %k6, %k6
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 ## 2-byte Reload
+; KNL-NEXT:    kandw %k7, %k6, %k6
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %cl
+; KNL-NEXT:    kmovw %ecx, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $9, %k7, %k7
+; KNL-NEXT:    korw %k7, %k6, %k6
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 ## 2-byte Reload
+; KNL-NEXT:    kandw %k7, %k6, %k6
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %cl
+; KNL-NEXT:    kmovw %ecx, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $8, %k7, %k7
+; KNL-NEXT:    korw %k7, %k6, %k6
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 ## 2-byte Reload
+; KNL-NEXT:    kandw %k7, %k6, %k6
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %cl
+; KNL-NEXT:    kmovw %ecx, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $7, %k7, %k7
+; KNL-NEXT:    korw %k7, %k6, %k6
+; KNL-NEXT:    kandw %k5, %k6, %k5
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %cl
+; KNL-NEXT:    kmovw %ecx, %k6
+; KNL-NEXT:    kshiftlw $15, %k6, %k6
+; KNL-NEXT:    kshiftrw $6, %k6, %k6
+; KNL-NEXT:    korw %k6, %k5, %k5
+; KNL-NEXT:    kandw %k4, %k5, %k4
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %cl
+; KNL-NEXT:    kmovw %ecx, %k5
+; KNL-NEXT:    kshiftlw $15, %k5, %k5
+; KNL-NEXT:    kshiftrw $5, %k5, %k5
+; KNL-NEXT:    korw %k5, %k4, %k4
+; KNL-NEXT:    kandw %k3, %k4, %k3
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %cl
+; KNL-NEXT:    kmovw %ecx, %k4
+; KNL-NEXT:    kshiftlw $15, %k4, %k4
+; KNL-NEXT:    kshiftrw $4, %k4, %k4
+; KNL-NEXT:    korw %k4, %k3, %k3
+; KNL-NEXT:    kandw %k2, %k3, %k2
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %cl
+; KNL-NEXT:    kmovw %ecx, %k3
+; KNL-NEXT:    kshiftlw $15, %k3, %k3
+; KNL-NEXT:    kshiftrw $3, %k3, %k3
+; KNL-NEXT:    korw %k3, %k2, %k2
+; KNL-NEXT:    kandw %k0, %k2, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %cl
+; KNL-NEXT:    kmovw %ecx, %k2
+; KNL-NEXT:    kshiftlw $15, %k2, %k2
+; KNL-NEXT:    kshiftrw $2, %k2, %k2
+; KNL-NEXT:    korw %k2, %k0, %k0
+; KNL-NEXT:    xorl %ecx, %ecx
+; KNL-NEXT:    testb $1, {{[0-9]+}}(%rsp)
+; KNL-NEXT:    movl $65535, %edx ## imm = 0xFFFF
+; KNL-NEXT:    movl $0, %esi
+; KNL-NEXT:    cmovnel %edx, %esi
+; KNL-NEXT:    testb $1, {{[0-9]+}}(%rsp)
+; KNL-NEXT:    cmovnel %edx, %ecx
+; KNL-NEXT:    kandw %k1, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dl
+; KNL-NEXT:    kmovw %edx, %k1
+; KNL-NEXT:    kshiftlw $14, %k1, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $1, %k0, %k0
+; KNL-NEXT:    kshiftrw $1, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dl
+; KNL-NEXT:    kmovw %edx, %k1
+; KNL-NEXT:    kshiftlw $15, %k1, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kmovw %esi, %k1
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 ## 2-byte Reload
+; KNL-NEXT:    kandw %k2, %k0, %k0
+; KNL-NEXT:    kmovw %ecx, %k2
+; KNL-NEXT:    kandw %k1, %k2, %k1
+; KNL-NEXT:    kmovw %k1, %r8d
+; KNL-NEXT:    kshiftrw $1, %k0, %k1
+; KNL-NEXT:    kmovw %k1, %r9d
+; KNL-NEXT:    kshiftrw $2, %k0, %k1
+; KNL-NEXT:    kmovw %k1, %r10d
+; KNL-NEXT:    kshiftrw $3, %k0, %k1
+; KNL-NEXT:    kmovw %k1, %r11d
+; KNL-NEXT:    kshiftrw $4, %k0, %k1
+; KNL-NEXT:    kmovw %k1, %r12d
+; KNL-NEXT:    kshiftrw $5, %k0, %k1
+; KNL-NEXT:    kmovw %k1, %r15d
+; KNL-NEXT:    kshiftrw $6, %k0, %k1
+; KNL-NEXT:    kmovw %k1, %r14d
+; KNL-NEXT:    kshiftrw $7, %k0, %k1
+; KNL-NEXT:    kmovw %k1, %r13d
+; KNL-NEXT:    kshiftrw $8, %k0, %k1
+; KNL-NEXT:    kmovw %k1, %ebx
+; KNL-NEXT:    kshiftrw $9, %k0, %k1
+; KNL-NEXT:    kmovw %k1, %esi
+; KNL-NEXT:    kshiftrw $10, %k0, %k1
+; KNL-NEXT:    kmovw %k1, %ebp
+; KNL-NEXT:    kshiftrw $11, %k0, %k1
+; KNL-NEXT:    kmovw %k1, %ecx
+; KNL-NEXT:    kshiftrw $12, %k0, %k1
+; KNL-NEXT:    kmovw %k1, %edx
+; KNL-NEXT:    kshiftrw $13, %k0, %k1
+; KNL-NEXT:    kmovw %k1, %edi
+; KNL-NEXT:    kshiftrw $14, %k0, %k1
+; KNL-NEXT:    andl $1, %r8d
+; KNL-NEXT:    movb %r8b, 2(%rax)
+; KNL-NEXT:    kmovw %k0, %r8d
+; KNL-NEXT:    andl $1, %r8d
+; KNL-NEXT:    andl $1, %r9d
+; KNL-NEXT:    leal (%r8,%r9,2), %r8d
+; KNL-NEXT:    kmovw %k1, %r9d
+; KNL-NEXT:    kshiftrw $15, %k0, %k0
+; KNL-NEXT:    andl $1, %r10d
+; KNL-NEXT:    leal (%r8,%r10,4), %r8d
+; KNL-NEXT:    kmovw %k0, %r10d
+; KNL-NEXT:    andl $1, %r11d
+; KNL-NEXT:    leal (%r8,%r11,8), %r8d
+; KNL-NEXT:    andl $1, %r12d
+; KNL-NEXT:    shll $4, %r12d
+; KNL-NEXT:    orl %r8d, %r12d
+; KNL-NEXT:    andl $1, %r15d
+; KNL-NEXT:    shll $5, %r15d
+; KNL-NEXT:    orl %r12d, %r15d
+; KNL-NEXT:    andl $1, %r14d
+; KNL-NEXT:    shll $6, %r14d
+; KNL-NEXT:    andl $1, %r13d
+; KNL-NEXT:    shll $7, %r13d
+; KNL-NEXT:    orl %r14d, %r13d
+; KNL-NEXT:    andl $1, %ebx
+; KNL-NEXT:    shll $8, %ebx
+; KNL-NEXT:    orl %r13d, %ebx
+; KNL-NEXT:    andl $1, %esi
+; KNL-NEXT:    shll $9, %esi
+; KNL-NEXT:    orl %ebx, %esi
+; KNL-NEXT:    andl $1, %ebp
+; KNL-NEXT:    shll $10, %ebp
+; KNL-NEXT:    orl %esi, %ebp
+; KNL-NEXT:    orl %r15d, %ebp
+; KNL-NEXT:    andl $1, %ecx
+; KNL-NEXT:    shll $11, %ecx
+; KNL-NEXT:    andl $1, %edx
+; KNL-NEXT:    shll $12, %edx
+; KNL-NEXT:    orl %ecx, %edx
+; KNL-NEXT:    andl $1, %edi
+; KNL-NEXT:    shll $13, %edi
+; KNL-NEXT:    orl %edx, %edi
+; KNL-NEXT:    andl $1, %r9d
+; KNL-NEXT:    shll $14, %r9d
+; KNL-NEXT:    orl %edi, %r9d
+; KNL-NEXT:    andl $1, %r10d
+; KNL-NEXT:    shll $15, %r10d
+; KNL-NEXT:    orl %r9d, %r10d
+; KNL-NEXT:    orl %ebp, %r10d
+; KNL-NEXT:    movw %r10w, (%rax)
+; KNL-NEXT:    popq %rbx
+; KNL-NEXT:    popq %r12
+; KNL-NEXT:    popq %r13
+; KNL-NEXT:    popq %r14
+; KNL-NEXT:    popq %r15
+; KNL-NEXT:    popq %rbp
+; KNL-NEXT:    retq
+;
+; SKX-LABEL: test16:
+; SKX:       ## %bb.0:
+; SKX-NEXT:    pushq %rbp
+; SKX-NEXT:    pushq %r15
+; SKX-NEXT:    pushq %r14
+; SKX-NEXT:    pushq %r13
+; SKX-NEXT:    pushq %r12
+; SKX-NEXT:    pushq %rbx
+; SKX-NEXT:    movq %rdi, %rax
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k0
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k1
+; SKX-NEXT:    movl $-3, %edi
+; SKX-NEXT:    kmovd %edi, %k2
+; SKX-NEXT:    kmovd %k2, {{[-0-9]+}}(%r{{[sb]}}p) ## 4-byte Spill
+; SKX-NEXT:    kandd %k2, %k0, %k0
+; SKX-NEXT:    kshiftld $31, %k1, %k1
+; SKX-NEXT:    kshiftrd $30, %k1, %k1
+; SKX-NEXT:    kord %k1, %k0, %k0
+; SKX-NEXT:    movl $-5, %edi
+; SKX-NEXT:    kmovd %edi, %k1
+; SKX-NEXT:    kmovd %k1, {{[-0-9]+}}(%r{{[sb]}}p) ## 4-byte Spill
+; SKX-NEXT:    kandd %k1, %k0, %k0
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k1
+; SKX-NEXT:    kshiftld $31, %k1, %k1
+; SKX-NEXT:    kshiftrd $29, %k1, %k1
+; SKX-NEXT:    kord %k1, %k0, %k0
+; SKX-NEXT:    movl $-9, %edi
+; SKX-NEXT:    kmovd %edi, %k1
+; SKX-NEXT:    kmovd %k1, {{[-0-9]+}}(%r{{[sb]}}p) ## 4-byte Spill
+; SKX-NEXT:    kandd %k1, %k0, %k0
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k1
+; SKX-NEXT:    kshiftld $31, %k1, %k1
+; SKX-NEXT:    kshiftrd $28, %k1, %k1
+; SKX-NEXT:    kord %k1, %k0, %k0
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k1
+; SKX-NEXT:    movl $-17, %edi
+; SKX-NEXT:    kmovd %edi, %k2
+; SKX-NEXT:    kmovd %k2, {{[-0-9]+}}(%r{{[sb]}}p) ## 4-byte Spill
+; SKX-NEXT:    kandd %k2, %k0, %k0
+; SKX-NEXT:    kshiftld $31, %k1, %k1
+; SKX-NEXT:    kshiftrd $27, %k1, %k1
+; SKX-NEXT:    kord %k1, %k0, %k0
+; SKX-NEXT:    movl $-33, %edi
+; SKX-NEXT:    kmovd %edi, %k1
+; SKX-NEXT:    kmovd %k1, {{[-0-9]+}}(%r{{[sb]}}p) ## 4-byte Spill
+; SKX-NEXT:    kandd %k1, %k0, %k0
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k1
+; SKX-NEXT:    kshiftld $31, %k1, %k1
+; SKX-NEXT:    kshiftrd $26, %k1, %k1
+; SKX-NEXT:    kord %k1, %k0, %k0
+; SKX-NEXT:    movl $-65, %edi
+; SKX-NEXT:    kmovd %edi, %k1
+; SKX-NEXT:    kmovd %k1, {{[-0-9]+}}(%r{{[sb]}}p) ## 4-byte Spill
+; SKX-NEXT:    kandd %k1, %k0, %k0
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k1
+; SKX-NEXT:    kshiftld $31, %k1, %k1
+; SKX-NEXT:    kshiftrd $25, %k1, %k1
+; SKX-NEXT:    kord %k1, %k0, %k0
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k1
+; SKX-NEXT:    movl $-129, %edi
+; SKX-NEXT:    kmovd %edi, %k2
+; SKX-NEXT:    kmovd %k2, {{[-0-9]+}}(%r{{[sb]}}p) ## 4-byte Spill
+; SKX-NEXT:    kandd %k2, %k0, %k0
+; SKX-NEXT:    kshiftld $31, %k1, %k1
+; SKX-NEXT:    kshiftrd $24, %k1, %k1
+; SKX-NEXT:    kord %k1, %k0, %k0
+; SKX-NEXT:    movl $-257, %edi ## imm = 0xFEFF
+; SKX-NEXT:    kmovd %edi, %k1
+; SKX-NEXT:    kmovd %k1, {{[-0-9]+}}(%r{{[sb]}}p) ## 4-byte Spill
+; SKX-NEXT:    kandd %k1, %k0, %k0
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k1
+; SKX-NEXT:    kshiftld $31, %k1, %k1
+; SKX-NEXT:    kshiftrd $23, %k1, %k1
+; SKX-NEXT:    kord %k1, %k0, %k0
+; SKX-NEXT:    movl $-513, %edi ## imm = 0xFDFF
+; SKX-NEXT:    kmovd %edi, %k1
+; SKX-NEXT:    kmovd %k1, {{[-0-9]+}}(%r{{[sb]}}p) ## 4-byte Spill
+; SKX-NEXT:    kandd %k1, %k0, %k0
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k1
+; SKX-NEXT:    kshiftld $31, %k1, %k1
+; SKX-NEXT:    kshiftrd $22, %k1, %k1
+; SKX-NEXT:    kord %k1, %k0, %k0
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k1
+; SKX-NEXT:    movl $-1025, %edi ## imm = 0xFBFF
+; SKX-NEXT:    kmovd %edi, %k6
+; SKX-NEXT:    kandd %k6, %k0, %k0
+; SKX-NEXT:    kshiftld $31, %k1, %k1
+; SKX-NEXT:    kshiftrd $21, %k1, %k1
+; SKX-NEXT:    kord %k1, %k0, %k0
+; SKX-NEXT:    movl $-2049, %edi ## imm = 0xF7FF
+; SKX-NEXT:    kmovd %edi, %k1
+; SKX-NEXT:    kmovd %k1, {{[-0-9]+}}(%r{{[sb]}}p) ## 4-byte Spill
+; SKX-NEXT:    kandd %k1, %k0, %k0
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k1
+; SKX-NEXT:    kshiftld $31, %k1, %k1
+; SKX-NEXT:    kshiftrd $20, %k1, %k1
+; SKX-NEXT:    kord %k1, %k0, %k0
+; SKX-NEXT:    movl $-4097, %edi ## imm = 0xEFFF
+; SKX-NEXT:    kmovd %edi, %k1
+; SKX-NEXT:    kmovd %k1, {{[-0-9]+}}(%r{{[sb]}}p) ## 4-byte Spill
+; SKX-NEXT:    kandd %k1, %k0, %k0
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k1
+; SKX-NEXT:    kshiftld $31, %k1, %k1
+; SKX-NEXT:    kshiftrd $19, %k1, %k1
+; SKX-NEXT:    kord %k1, %k0, %k0
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k1
+; SKX-NEXT:    movl $-8193, %edi ## imm = 0xDFFF
+; SKX-NEXT:    kmovd %edi, %k4
+; SKX-NEXT:    kandd %k4, %k0, %k0
+; SKX-NEXT:    kshiftld $31, %k1, %k1
+; SKX-NEXT:    kshiftrd $18, %k1, %k1
+; SKX-NEXT:    kord %k1, %k0, %k0
+; SKX-NEXT:    movl $-16385, %edi ## imm = 0xBFFF
+; SKX-NEXT:    kmovd %edi, %k5
+; SKX-NEXT:    kandd %k5, %k0, %k0
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k1
+; SKX-NEXT:    kshiftld $31, %k1, %k1
+; SKX-NEXT:    kshiftrd $17, %k1, %k1
+; SKX-NEXT:    kord %k1, %k0, %k0
+; SKX-NEXT:    movl $-32769, %edi ## imm = 0xFFFF7FFF
+; SKX-NEXT:    kmovd %edi, %k3
+; SKX-NEXT:    kandd %k3, %k0, %k0
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k7
+; SKX-NEXT:    kshiftld $31, %k7, %k7
+; SKX-NEXT:    kshiftrd $16, %k7, %k7
+; SKX-NEXT:    kord %k7, %k0, %k7
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k0
+; SKX-NEXT:    movl $-65537, %edi ## imm = 0xFFFEFFFF
+; SKX-NEXT:    kmovd %edi, %k2
+; SKX-NEXT:    kandd %k2, %k7, %k7
+; SKX-NEXT:    kshiftld $31, %k0, %k0
+; SKX-NEXT:    kshiftrd $15, %k0, %k0
+; SKX-NEXT:    kord %k0, %k7, %k0
+; SKX-NEXT:    kmovd %k0, {{[-0-9]+}}(%r{{[sb]}}p) ## 4-byte Spill
+; SKX-NEXT:    kmovd %esi, %k0
+; SKX-NEXT:    kmovd {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 4-byte Reload
+; SKX-NEXT:    kandd %k1, %k0, %k0
+; SKX-NEXT:    kmovd %edx, %k7
+; SKX-NEXT:    kshiftld $31, %k7, %k7
+; SKX-NEXT:    kshiftrd $30, %k7, %k7
+; SKX-NEXT:    kord %k7, %k0, %k0
+; SKX-NEXT:    kmovd {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 4-byte Reload
+; SKX-NEXT:    kandd %k1, %k0, %k0
+; SKX-NEXT:    kmovd %ecx, %k7
+; SKX-NEXT:    kshiftld $31, %k7, %k7
+; SKX-NEXT:    kshiftrd $29, %k7, %k7
+; SKX-NEXT:    kord %k7, %k0, %k0
+; SKX-NEXT:    kmovd {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 4-byte Reload
+; SKX-NEXT:    kandd %k1, %k0, %k0
+; SKX-NEXT:    kmovd %r8d, %k7
+; SKX-NEXT:    kshiftld $31, %k7, %k7
+; SKX-NEXT:    kshiftrd $28, %k7, %k7
+; SKX-NEXT:    kord %k7, %k0, %k0
+; SKX-NEXT:    kmovd {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 4-byte Reload
+; SKX-NEXT:    kandd %k1, %k0, %k0
+; SKX-NEXT:    kmovd %r9d, %k7
+; SKX-NEXT:    kshiftld $31, %k7, %k7
+; SKX-NEXT:    kshiftrd $27, %k7, %k7
+; SKX-NEXT:    kord %k7, %k0, %k0
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k7
+; SKX-NEXT:    kmovd {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 4-byte Reload
+; SKX-NEXT:    kandd %k1, %k0, %k1
+; SKX-NEXT:    kshiftld $31, %k7, %k7
+; SKX-NEXT:    kshiftrd $26, %k7, %k7
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k0
+; SKX-NEXT:    kord %k7, %k1, %k1
+; SKX-NEXT:    kmovd {{[-0-9]+}}(%r{{[sb]}}p), %k7 ## 4-byte Reload
+; SKX-NEXT:    kandd %k7, %k1, %k1
+; SKX-NEXT:    kshiftld $31, %k0, %k0
+; SKX-NEXT:    kshiftrd $25, %k0, %k0
+; SKX-NEXT:    kord %k0, %k1, %k0
+; SKX-NEXT:    kmovd {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 4-byte Reload
+; SKX-NEXT:    kandd %k1, %k0, %k0
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k1
+; SKX-NEXT:    kshiftld $31, %k1, %k1
+; SKX-NEXT:    kshiftrd $24, %k1, %k1
+; SKX-NEXT:    kord %k1, %k0, %k0
+; SKX-NEXT:    kmovd {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 4-byte Reload
+; SKX-NEXT:    kandd %k1, %k0, %k0
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k1
+; SKX-NEXT:    kshiftld $31, %k1, %k1
+; SKX-NEXT:    kshiftrd $23, %k1, %k1
+; SKX-NEXT:    kord %k1, %k0, %k0
+; SKX-NEXT:    kmovd {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 4-byte Reload
+; SKX-NEXT:    kandd %k1, %k0, %k0
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k1
+; SKX-NEXT:    kshiftld $31, %k1, %k1
+; SKX-NEXT:    kshiftrd $22, %k1, %k1
+; SKX-NEXT:    kord %k1, %k0, %k0
+; SKX-NEXT:    kandd %k6, %k0, %k0
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k1
+; SKX-NEXT:    kshiftld $31, %k1, %k1
+; SKX-NEXT:    kshiftrd $21, %k1, %k1
+; SKX-NEXT:    kord %k1, %k0, %k0
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k1
+; SKX-NEXT:    kmovd {{[-0-9]+}}(%r{{[sb]}}p), %k6 ## 4-byte Reload
+; SKX-NEXT:    kandd %k6, %k0, %k0
+; SKX-NEXT:    kshiftld $31, %k1, %k1
+; SKX-NEXT:    kshiftrd $20, %k1, %k1
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k6
+; SKX-NEXT:    kord %k1, %k0, %k0
+; SKX-NEXT:    kmovd {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 4-byte Reload
+; SKX-NEXT:    kandd %k1, %k0, %k0
+; SKX-NEXT:    kshiftld $31, %k6, %k1
+; SKX-NEXT:    kshiftrd $19, %k1, %k1
+; SKX-NEXT:    kord %k1, %k0, %k0
+; SKX-NEXT:    kandd %k4, %k0, %k0
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k1
+; SKX-NEXT:    kshiftld $31, %k1, %k1
+; SKX-NEXT:    kshiftrd $18, %k1, %k1
+; SKX-NEXT:    kord %k1, %k0, %k0
+; SKX-NEXT:    kandd %k5, %k0, %k0
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k1
+; SKX-NEXT:    kshiftld $31, %k1, %k1
+; SKX-NEXT:    kshiftrd $17, %k1, %k1
+; SKX-NEXT:    kord %k1, %k0, %k0
+; SKX-NEXT:    kandd %k3, %k0, %k0
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k1
+; SKX-NEXT:    kshiftld $31, %k1, %k1
+; SKX-NEXT:    kshiftrd $16, %k1, %k1
+; SKX-NEXT:    kord %k1, %k0, %k0
+; SKX-NEXT:    kandd %k2, %k0, %k0
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k1
+; SKX-NEXT:    kshiftld $31, %k1, %k1
+; SKX-NEXT:    kshiftrd $15, %k1, %k1
+; SKX-NEXT:    kord %k1, %k0, %k0
+; SKX-NEXT:    kmovd {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 4-byte Reload
+; SKX-NEXT:    kandd %k1, %k0, %k0
+; SKX-NEXT:    kshiftrd $16, %k0, %k1
+; SKX-NEXT:    kmovd %k1, %r8d
+; SKX-NEXT:    kshiftrd $1, %k0, %k1
+; SKX-NEXT:    kmovd %k1, %r9d
+; SKX-NEXT:    kshiftrd $2, %k0, %k1
+; SKX-NEXT:    kmovd %k1, %r10d
+; SKX-NEXT:    kshiftrd $3, %k0, %k1
+; SKX-NEXT:    kmovd %k1, %r11d
+; SKX-NEXT:    kshiftrd $4, %k0, %k1
+; SKX-NEXT:    kmovd %k1, %r12d
+; SKX-NEXT:    kshiftrd $5, %k0, %k1
+; SKX-NEXT:    kmovd %k1, %r15d
+; SKX-NEXT:    kshiftrd $6, %k0, %k1
+; SKX-NEXT:    kmovd %k1, %r14d
+; SKX-NEXT:    kshiftrd $7, %k0, %k1
+; SKX-NEXT:    kmovd %k1, %r13d
+; SKX-NEXT:    kshiftrd $8, %k0, %k1
+; SKX-NEXT:    kmovd %k1, %ebx
+; SKX-NEXT:    kshiftrd $9, %k0, %k1
+; SKX-NEXT:    kmovd %k1, %esi
+; SKX-NEXT:    kshiftrd $10, %k0, %k1
+; SKX-NEXT:    kmovd %k1, %ebp
+; SKX-NEXT:    kshiftrd $11, %k0, %k1
+; SKX-NEXT:    kmovd %k1, %ecx
+; SKX-NEXT:    kshiftrd $12, %k0, %k1
+; SKX-NEXT:    kmovd %k1, %edx
+; SKX-NEXT:    kshiftrd $13, %k0, %k1
+; SKX-NEXT:    kmovd %k1, %edi
+; SKX-NEXT:    kshiftrd $14, %k0, %k1
+; SKX-NEXT:    andl $1, %r8d
+; SKX-NEXT:    movb %r8b, 2(%rax)
+; SKX-NEXT:    kmovd %k0, %r8d
+; SKX-NEXT:    andl $1, %r8d
+; SKX-NEXT:    andl $1, %r9d
+; SKX-NEXT:    leal (%r8,%r9,2), %r8d
+; SKX-NEXT:    kmovd %k1, %r9d
+; SKX-NEXT:    kshiftrd $15, %k0, %k0
+; SKX-NEXT:    andl $1, %r10d
+; SKX-NEXT:    leal (%r8,%r10,4), %r8d
+; SKX-NEXT:    kmovd %k0, %r10d
+; SKX-NEXT:    andl $1, %r11d
+; SKX-NEXT:    leal (%r8,%r11,8), %r8d
+; SKX-NEXT:    andl $1, %r12d
+; SKX-NEXT:    shll $4, %r12d
+; SKX-NEXT:    orl %r8d, %r12d
+; SKX-NEXT:    andl $1, %r15d
+; SKX-NEXT:    shll $5, %r15d
+; SKX-NEXT:    orl %r12d, %r15d
+; SKX-NEXT:    andl $1, %r14d
+; SKX-NEXT:    shll $6, %r14d
+; SKX-NEXT:    andl $1, %r13d
+; SKX-NEXT:    shll $7, %r13d
+; SKX-NEXT:    orl %r14d, %r13d
+; SKX-NEXT:    andl $1, %ebx
+; SKX-NEXT:    shll $8, %ebx
+; SKX-NEXT:    orl %r13d, %ebx
+; SKX-NEXT:    andl $1, %esi
+; SKX-NEXT:    shll $9, %esi
+; SKX-NEXT:    orl %ebx, %esi
+; SKX-NEXT:    andl $1, %ebp
+; SKX-NEXT:    shll $10, %ebp
+; SKX-NEXT:    orl %esi, %ebp
+; SKX-NEXT:    orl %r15d, %ebp
+; SKX-NEXT:    andl $1, %ecx
+; SKX-NEXT:    shll $11, %ecx
+; SKX-NEXT:    andl $1, %edx
+; SKX-NEXT:    shll $12, %edx
+; SKX-NEXT:    orl %ecx, %edx
+; SKX-NEXT:    andl $1, %edi
+; SKX-NEXT:    shll $13, %edi
+; SKX-NEXT:    orl %edx, %edi
+; SKX-NEXT:    andl $1, %r9d
+; SKX-NEXT:    shll $14, %r9d
+; SKX-NEXT:    orl %edi, %r9d
+; SKX-NEXT:    andl $1, %r10d
+; SKX-NEXT:    shll $15, %r10d
+; SKX-NEXT:    orl %r9d, %r10d
+; SKX-NEXT:    orl %ebp, %r10d
+; SKX-NEXT:    movw %r10w, (%rax)
+; SKX-NEXT:    popq %rbx
+; SKX-NEXT:    popq %r12
+; SKX-NEXT:    popq %r13
+; SKX-NEXT:    popq %r14
+; SKX-NEXT:    popq %r15
+; SKX-NEXT:    popq %rbp
+; SKX-NEXT:    retq
+;
+; KNL_X32-LABEL: test16:
+; KNL_X32:       ## %bb.0:
+; KNL_X32-NEXT:    pushl %ebp
+; KNL_X32-NEXT:    pushl %ebx
+; KNL_X32-NEXT:    pushl %edi
+; KNL_X32-NEXT:    pushl %esi
+; KNL_X32-NEXT:    subl $20, %esp
+; KNL_X32-NEXT:    movw $-3, %ax
+; KNL_X32-NEXT:    kmovw %eax, %k1
+; KNL_X32-NEXT:    kmovw %k1, {{[-0-9]+}}(%e{{[sb]}}p) ## 2-byte Spill
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k0
+; KNL_X32-NEXT:    kandw %k1, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k1
+; KNL_X32-NEXT:    kshiftlw $15, %k1, %k1
+; KNL_X32-NEXT:    kshiftrw $14, %k1, %k1
+; KNL_X32-NEXT:    korw %k1, %k0, %k0
+; KNL_X32-NEXT:    movw $-5, %ax
+; KNL_X32-NEXT:    kmovw %eax, %k1
+; KNL_X32-NEXT:    kmovw %k1, {{[-0-9]+}}(%e{{[sb]}}p) ## 2-byte Spill
+; KNL_X32-NEXT:    kandw %k1, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k1
+; KNL_X32-NEXT:    kshiftlw $15, %k1, %k1
+; KNL_X32-NEXT:    kshiftrw $13, %k1, %k1
+; KNL_X32-NEXT:    korw %k1, %k0, %k0
+; KNL_X32-NEXT:    movw $-9, %ax
+; KNL_X32-NEXT:    kmovw %eax, %k1
+; KNL_X32-NEXT:    kmovw %k1, {{[-0-9]+}}(%e{{[sb]}}p) ## 2-byte Spill
+; KNL_X32-NEXT:    kandw %k1, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k1
+; KNL_X32-NEXT:    kshiftlw $15, %k1, %k1
+; KNL_X32-NEXT:    kshiftrw $12, %k1, %k1
+; KNL_X32-NEXT:    korw %k1, %k0, %k0
+; KNL_X32-NEXT:    movw $-17, %ax
+; KNL_X32-NEXT:    kmovw %eax, %k1
+; KNL_X32-NEXT:    kmovw %k1, {{[-0-9]+}}(%e{{[sb]}}p) ## 2-byte Spill
+; KNL_X32-NEXT:    kandw %k1, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k1
+; KNL_X32-NEXT:    kshiftlw $15, %k1, %k1
+; KNL_X32-NEXT:    kshiftrw $11, %k1, %k1
+; KNL_X32-NEXT:    korw %k1, %k0, %k0
+; KNL_X32-NEXT:    movw $-33, %ax
+; KNL_X32-NEXT:    kmovw %eax, %k1
+; KNL_X32-NEXT:    kmovw %k1, {{[-0-9]+}}(%e{{[sb]}}p) ## 2-byte Spill
+; KNL_X32-NEXT:    kandw %k1, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k1
+; KNL_X32-NEXT:    kshiftlw $15, %k1, %k1
+; KNL_X32-NEXT:    kshiftrw $10, %k1, %k1
+; KNL_X32-NEXT:    korw %k1, %k0, %k0
+; KNL_X32-NEXT:    movw $-65, %ax
+; KNL_X32-NEXT:    kmovw %eax, %k1
+; KNL_X32-NEXT:    kmovw %k1, {{[-0-9]+}}(%e{{[sb]}}p) ## 2-byte Spill
+; KNL_X32-NEXT:    kandw %k1, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k1
+; KNL_X32-NEXT:    kshiftlw $15, %k1, %k1
+; KNL_X32-NEXT:    kshiftrw $9, %k1, %k1
+; KNL_X32-NEXT:    korw %k1, %k0, %k0
+; KNL_X32-NEXT:    movw $-129, %ax
+; KNL_X32-NEXT:    kmovw %eax, %k1
+; KNL_X32-NEXT:    kmovw %k1, {{[-0-9]+}}(%e{{[sb]}}p) ## 2-byte Spill
+; KNL_X32-NEXT:    kandw %k1, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k1
+; KNL_X32-NEXT:    kshiftlw $15, %k1, %k1
+; KNL_X32-NEXT:    kshiftrw $8, %k1, %k1
+; KNL_X32-NEXT:    korw %k1, %k0, %k0
+; KNL_X32-NEXT:    movw $-257, %ax ## imm = 0xFEFF
+; KNL_X32-NEXT:    kmovw %eax, %k1
+; KNL_X32-NEXT:    kmovw %k1, {{[-0-9]+}}(%e{{[sb]}}p) ## 2-byte Spill
+; KNL_X32-NEXT:    kandw %k1, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k1
+; KNL_X32-NEXT:    kshiftlw $15, %k1, %k1
+; KNL_X32-NEXT:    kshiftrw $7, %k1, %k1
+; KNL_X32-NEXT:    korw %k1, %k0, %k0
+; KNL_X32-NEXT:    movw $-513, %ax ## imm = 0xFDFF
+; KNL_X32-NEXT:    kmovw %eax, %k5
+; KNL_X32-NEXT:    kandw %k5, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k1
+; KNL_X32-NEXT:    kshiftlw $15, %k1, %k1
+; KNL_X32-NEXT:    kshiftrw $6, %k1, %k1
+; KNL_X32-NEXT:    korw %k1, %k0, %k0
+; KNL_X32-NEXT:    movw $-1025, %ax ## imm = 0xFBFF
+; KNL_X32-NEXT:    kmovw %eax, %k4
+; KNL_X32-NEXT:    kandw %k4, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k1
+; KNL_X32-NEXT:    kshiftlw $15, %k1, %k1
+; KNL_X32-NEXT:    kshiftrw $5, %k1, %k1
+; KNL_X32-NEXT:    korw %k1, %k0, %k0
+; KNL_X32-NEXT:    movw $-2049, %ax ## imm = 0xF7FF
+; KNL_X32-NEXT:    kmovw %eax, %k3
+; KNL_X32-NEXT:    kandw %k3, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k1
+; KNL_X32-NEXT:    kshiftlw $15, %k1, %k1
+; KNL_X32-NEXT:    kshiftrw $4, %k1, %k1
+; KNL_X32-NEXT:    korw %k1, %k0, %k0
+; KNL_X32-NEXT:    movw $-4097, %ax ## imm = 0xEFFF
+; KNL_X32-NEXT:    kmovw %eax, %k2
+; KNL_X32-NEXT:    kandw %k2, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k1
+; KNL_X32-NEXT:    kshiftlw $15, %k1, %k1
+; KNL_X32-NEXT:    kshiftrw $3, %k1, %k1
+; KNL_X32-NEXT:    korw %k1, %k0, %k0
+; KNL_X32-NEXT:    movw $-8193, %ax ## imm = 0xDFFF
+; KNL_X32-NEXT:    kmovw %eax, %k1
+; KNL_X32-NEXT:    kandw %k1, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k6
+; KNL_X32-NEXT:    kshiftlw $15, %k6, %k6
+; KNL_X32-NEXT:    kshiftrw $2, %k6, %k6
+; KNL_X32-NEXT:    korw %k6, %k0, %k6
+; KNL_X32-NEXT:    movw $-16385, %ax ## imm = 0xBFFF
+; KNL_X32-NEXT:    kmovw %eax, %k0
+; KNL_X32-NEXT:    kandw %k0, %k6, %k6
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k7
+; KNL_X32-NEXT:    kshiftlw $14, %k7, %k7
+; KNL_X32-NEXT:    korw %k7, %k6, %k6
+; KNL_X32-NEXT:    kshiftlw $1, %k6, %k6
+; KNL_X32-NEXT:    kshiftrw $1, %k6, %k6
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k7
+; KNL_X32-NEXT:    kshiftlw $15, %k7, %k7
+; KNL_X32-NEXT:    korw %k7, %k6, %k6
+; KNL_X32-NEXT:    kmovw %k6, {{[-0-9]+}}(%e{{[sb]}}p) ## 2-byte Spill
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k6
+; KNL_X32-NEXT:    kmovw {{[-0-9]+}}(%e{{[sb]}}p), %k7 ## 2-byte Reload
+; KNL_X32-NEXT:    kandw %k7, %k6, %k6
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k7
+; KNL_X32-NEXT:    kshiftlw $15, %k7, %k7
+; KNL_X32-NEXT:    kshiftrw $14, %k7, %k7
+; KNL_X32-NEXT:    korw %k7, %k6, %k6
+; KNL_X32-NEXT:    kmovw {{[-0-9]+}}(%e{{[sb]}}p), %k7 ## 2-byte Reload
+; KNL_X32-NEXT:    kandw %k7, %k6, %k6
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k7
+; KNL_X32-NEXT:    kshiftlw $15, %k7, %k7
+; KNL_X32-NEXT:    kshiftrw $13, %k7, %k7
+; KNL_X32-NEXT:    korw %k7, %k6, %k6
+; KNL_X32-NEXT:    kmovw {{[-0-9]+}}(%e{{[sb]}}p), %k7 ## 2-byte Reload
+; KNL_X32-NEXT:    kandw %k7, %k6, %k6
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k7
+; KNL_X32-NEXT:    kshiftlw $15, %k7, %k7
+; KNL_X32-NEXT:    kshiftrw $12, %k7, %k7
+; KNL_X32-NEXT:    korw %k7, %k6, %k6
+; KNL_X32-NEXT:    kmovw {{[-0-9]+}}(%e{{[sb]}}p), %k7 ## 2-byte Reload
+; KNL_X32-NEXT:    kandw %k7, %k6, %k6
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k7
+; KNL_X32-NEXT:    kshiftlw $15, %k7, %k7
+; KNL_X32-NEXT:    kshiftrw $11, %k7, %k7
+; KNL_X32-NEXT:    korw %k7, %k6, %k6
+; KNL_X32-NEXT:    kmovw {{[-0-9]+}}(%e{{[sb]}}p), %k7 ## 2-byte Reload
+; KNL_X32-NEXT:    kandw %k7, %k6, %k6
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k7
+; KNL_X32-NEXT:    kshiftlw $15, %k7, %k7
+; KNL_X32-NEXT:    kshiftrw $10, %k7, %k7
+; KNL_X32-NEXT:    korw %k7, %k6, %k6
+; KNL_X32-NEXT:    kmovw {{[-0-9]+}}(%e{{[sb]}}p), %k7 ## 2-byte Reload
+; KNL_X32-NEXT:    kandw %k7, %k6, %k6
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k7
+; KNL_X32-NEXT:    kshiftlw $15, %k7, %k7
+; KNL_X32-NEXT:    kshiftrw $9, %k7, %k7
+; KNL_X32-NEXT:    korw %k7, %k6, %k6
+; KNL_X32-NEXT:    kmovw {{[-0-9]+}}(%e{{[sb]}}p), %k7 ## 2-byte Reload
+; KNL_X32-NEXT:    kandw %k7, %k6, %k6
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k7
+; KNL_X32-NEXT:    kshiftlw $15, %k7, %k7
+; KNL_X32-NEXT:    kshiftrw $8, %k7, %k7
+; KNL_X32-NEXT:    korw %k7, %k6, %k6
+; KNL_X32-NEXT:    kmovw {{[-0-9]+}}(%e{{[sb]}}p), %k7 ## 2-byte Reload
+; KNL_X32-NEXT:    kandw %k7, %k6, %k6
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k7
+; KNL_X32-NEXT:    kshiftlw $15, %k7, %k7
+; KNL_X32-NEXT:    kshiftrw $7, %k7, %k7
+; KNL_X32-NEXT:    korw %k7, %k6, %k6
+; KNL_X32-NEXT:    kandw %k5, %k6, %k5
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k6
+; KNL_X32-NEXT:    kshiftlw $15, %k6, %k6
+; KNL_X32-NEXT:    kshiftrw $6, %k6, %k6
+; KNL_X32-NEXT:    korw %k6, %k5, %k5
+; KNL_X32-NEXT:    kandw %k4, %k5, %k4
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k5
+; KNL_X32-NEXT:    kshiftlw $15, %k5, %k5
+; KNL_X32-NEXT:    kshiftrw $5, %k5, %k5
+; KNL_X32-NEXT:    korw %k5, %k4, %k4
+; KNL_X32-NEXT:    kandw %k3, %k4, %k3
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k4
+; KNL_X32-NEXT:    kshiftlw $15, %k4, %k4
+; KNL_X32-NEXT:    kshiftrw $4, %k4, %k4
+; KNL_X32-NEXT:    korw %k4, %k3, %k3
+; KNL_X32-NEXT:    kandw %k2, %k3, %k2
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k3
+; KNL_X32-NEXT:    kshiftlw $15, %k3, %k3
+; KNL_X32-NEXT:    kshiftrw $3, %k3, %k3
+; KNL_X32-NEXT:    korw %k3, %k2, %k2
+; KNL_X32-NEXT:    kandw %k1, %k2, %k1
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k2
+; KNL_X32-NEXT:    kshiftlw $15, %k2, %k2
+; KNL_X32-NEXT:    kshiftrw $2, %k2, %k2
+; KNL_X32-NEXT:    korw %k2, %k1, %k1
+; KNL_X32-NEXT:    xorl %eax, %eax
+; KNL_X32-NEXT:    testb $1, {{[0-9]+}}(%esp)
+; KNL_X32-NEXT:    movl $65535, %ecx ## imm = 0xFFFF
+; KNL_X32-NEXT:    movl $0, %edx
+; KNL_X32-NEXT:    cmovnel %ecx, %edx
+; KNL_X32-NEXT:    kandw %k0, %k1, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %bl
+; KNL_X32-NEXT:    kmovw %ebx, %k1
+; KNL_X32-NEXT:    kshiftlw $14, %k1, %k1
+; KNL_X32-NEXT:    korw %k1, %k0, %k0
+; KNL_X32-NEXT:    kshiftlw $1, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $1, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %bl
+; KNL_X32-NEXT:    kmovw %ebx, %k1
+; KNL_X32-NEXT:    kshiftlw $15, %k1, %k1
+; KNL_X32-NEXT:    korw %k1, %k0, %k0
+; KNL_X32-NEXT:    kmovw %edx, %k1
+; KNL_X32-NEXT:    testb $1, {{[0-9]+}}(%esp)
+; KNL_X32-NEXT:    cmovnel %ecx, %eax
+; KNL_X32-NEXT:    kmovw {{[-0-9]+}}(%e{{[sb]}}p), %k2 ## 2-byte Reload
+; KNL_X32-NEXT:    kandw %k2, %k0, %k0
+; KNL_X32-NEXT:    kmovw %eax, %k2
+; KNL_X32-NEXT:    kandw %k1, %k2, %k1
+; KNL_X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; KNL_X32-NEXT:    kmovw %k1, %ebx
+; KNL_X32-NEXT:    kshiftrw $1, %k0, %k1
+; KNL_X32-NEXT:    kmovw %k1, %esi
+; KNL_X32-NEXT:    kshiftrw $2, %k0, %k1
+; KNL_X32-NEXT:    kmovw %k1, %edi
+; KNL_X32-NEXT:    kshiftrw $3, %k0, %k1
+; KNL_X32-NEXT:    kmovw %k1, %ebp
+; KNL_X32-NEXT:    kshiftrw $4, %k0, %k1
+; KNL_X32-NEXT:    kmovw %k1, %edx
+; KNL_X32-NEXT:    kshiftrw $5, %k0, %k1
+; KNL_X32-NEXT:    kmovw %k1, %ecx
+; KNL_X32-NEXT:    kshiftrw $6, %k0, %k1
+; KNL_X32-NEXT:    andl $1, %ebx
+; KNL_X32-NEXT:    movb %bl, 2(%eax)
+; KNL_X32-NEXT:    kmovw %k0, %ebx
+; KNL_X32-NEXT:    andl $1, %ebx
+; KNL_X32-NEXT:    andl $1, %esi
+; KNL_X32-NEXT:    leal (%ebx,%esi,2), %esi
+; KNL_X32-NEXT:    kmovw %k1, %ebx
+; KNL_X32-NEXT:    kshiftrw $7, %k0, %k1
+; KNL_X32-NEXT:    andl $1, %edi
+; KNL_X32-NEXT:    leal (%esi,%edi,4), %esi
+; KNL_X32-NEXT:    kmovw %k1, %edi
+; KNL_X32-NEXT:    kshiftrw $8, %k0, %k1
+; KNL_X32-NEXT:    andl $1, %ebp
+; KNL_X32-NEXT:    leal (%esi,%ebp,8), %esi
+; KNL_X32-NEXT:    kmovw %k1, %ebp
+; KNL_X32-NEXT:    kshiftrw $9, %k0, %k1
+; KNL_X32-NEXT:    andl $1, %edx
+; KNL_X32-NEXT:    shll $4, %edx
+; KNL_X32-NEXT:    orl %esi, %edx
+; KNL_X32-NEXT:    kmovw %k1, %esi
+; KNL_X32-NEXT:    kshiftrw $10, %k0, %k1
+; KNL_X32-NEXT:    andl $1, %ecx
+; KNL_X32-NEXT:    shll $5, %ecx
+; KNL_X32-NEXT:    orl %edx, %ecx
+; KNL_X32-NEXT:    kmovw %k1, %edx
+; KNL_X32-NEXT:    kshiftrw $11, %k0, %k1
+; KNL_X32-NEXT:    andl $1, %ebx
+; KNL_X32-NEXT:    shll $6, %ebx
+; KNL_X32-NEXT:    andl $1, %edi
+; KNL_X32-NEXT:    shll $7, %edi
+; KNL_X32-NEXT:    orl %ebx, %edi
+; KNL_X32-NEXT:    kmovw %k1, %ebx
+; KNL_X32-NEXT:    kshiftrw $12, %k0, %k1
+; KNL_X32-NEXT:    andl $1, %ebp
+; KNL_X32-NEXT:    shll $8, %ebp
+; KNL_X32-NEXT:    orl %edi, %ebp
+; KNL_X32-NEXT:    kmovw %k1, %edi
+; KNL_X32-NEXT:    kshiftrw $13, %k0, %k1
+; KNL_X32-NEXT:    andl $1, %esi
+; KNL_X32-NEXT:    shll $9, %esi
+; KNL_X32-NEXT:    orl %ebp, %esi
+; KNL_X32-NEXT:    kmovw %k1, %ebp
+; KNL_X32-NEXT:    kshiftrw $14, %k0, %k1
+; KNL_X32-NEXT:    andl $1, %edx
+; KNL_X32-NEXT:    shll $10, %edx
+; KNL_X32-NEXT:    orl %esi, %edx
+; KNL_X32-NEXT:    kmovw %k1, %esi
+; KNL_X32-NEXT:    kshiftrw $15, %k0, %k0
+; KNL_X32-NEXT:    orl %ecx, %edx
+; KNL_X32-NEXT:    kmovw %k0, %ecx
+; KNL_X32-NEXT:    andl $1, %ebx
+; KNL_X32-NEXT:    shll $11, %ebx
+; KNL_X32-NEXT:    andl $1, %edi
+; KNL_X32-NEXT:    shll $12, %edi
+; KNL_X32-NEXT:    orl %ebx, %edi
+; KNL_X32-NEXT:    andl $1, %ebp
+; KNL_X32-NEXT:    shll $13, %ebp
+; KNL_X32-NEXT:    orl %edi, %ebp
+; KNL_X32-NEXT:    andl $1, %esi
+; KNL_X32-NEXT:    shll $14, %esi
+; KNL_X32-NEXT:    orl %ebp, %esi
+; KNL_X32-NEXT:    andl $1, %ecx
+; KNL_X32-NEXT:    shll $15, %ecx
+; KNL_X32-NEXT:    orl %esi, %ecx
+; KNL_X32-NEXT:    orl %edx, %ecx
+; KNL_X32-NEXT:    movw %cx, (%eax)
+; KNL_X32-NEXT:    addl $20, %esp
+; KNL_X32-NEXT:    popl %esi
+; KNL_X32-NEXT:    popl %edi
+; KNL_X32-NEXT:    popl %ebx
+; KNL_X32-NEXT:    popl %ebp
+; KNL_X32-NEXT:    retl $4
+  %c = and <17 x i1> %a, %b
+  ret <17 x i1> %c
+}
+
+define <7 x i1> @test17(<7 x i1> %a, <7 x i1> %b, <7 x i1> %c, <7 x i1> %d, <7 x i1>%e, <7 x i1>%f, <7 x i1> %g, <7 x i1> %h, <7 x i1> %i) nounwind {
+; KNL-LABEL: test17:
+; KNL:       ## %bb.0:
+; KNL-NEXT:    movq %rdi, %rax
+; KNL-NEXT:    movw $-3, %di
+; KNL-NEXT:    kmovw %edi, %k2
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k0
+; KNL-NEXT:    kandw %k2, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k1
+; KNL-NEXT:    kshiftlw $15, %k1, %k1
+; KNL-NEXT:    kshiftrw $14, %k1, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    movw $-5, %di
+; KNL-NEXT:    kmovw %edi, %k1
+; KNL-NEXT:    kandw %k1, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k3
+; KNL-NEXT:    kshiftlw $15, %k3, %k3
+; KNL-NEXT:    kshiftrw $13, %k3, %k3
+; KNL-NEXT:    korw %k3, %k0, %k0
+; KNL-NEXT:    movw $-9, %di
+; KNL-NEXT:    kmovw %edi, %k3
+; KNL-NEXT:    kandw %k3, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k4
+; KNL-NEXT:    kshiftlw $15, %k4, %k4
+; KNL-NEXT:    kshiftrw $12, %k4, %k4
+; KNL-NEXT:    korw %k4, %k0, %k0
+; KNL-NEXT:    movw $-17, %di
+; KNL-NEXT:    kmovw %edi, %k4
+; KNL-NEXT:    kandw %k4, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k5
+; KNL-NEXT:    kshiftlw $15, %k5, %k5
+; KNL-NEXT:    kshiftrw $11, %k5, %k5
+; KNL-NEXT:    korw %k5, %k0, %k0
+; KNL-NEXT:    movw $-33, %di
+; KNL-NEXT:    kmovw %edi, %k5
+; KNL-NEXT:    kandw %k5, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k6
+; KNL-NEXT:    kshiftlw $15, %k6, %k6
+; KNL-NEXT:    kshiftrw $10, %k6, %k6
+; KNL-NEXT:    korw %k6, %k0, %k0
+; KNL-NEXT:    movw $-65, %di
+; KNL-NEXT:    kmovw %edi, %k6
+; KNL-NEXT:    kandw %k6, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $9, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kmovw %k0, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k0
+; KNL-NEXT:    kandw %k2, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $14, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kandw %k1, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $13, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kandw %k3, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $12, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kandw %k4, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $11, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kandw %k5, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $10, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kandw %k6, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $9, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kmovw %k0, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k0
+; KNL-NEXT:    kandw %k2, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $14, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kandw %k1, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $13, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kandw %k3, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $12, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kandw %k4, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $11, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kandw %k5, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $10, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kandw %k6, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $9, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kmovw %k0, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k0
+; KNL-NEXT:    kandw %k2, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $14, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kandw %k1, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $13, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kandw %k3, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $12, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kandw %k4, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $11, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kandw %k5, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $10, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kandw %k6, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $9, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kmovw %k0, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k0
+; KNL-NEXT:    kandw %k2, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $14, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kandw %k1, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $13, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kandw %k3, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $12, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kandw %k4, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $11, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kandw %k5, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $10, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kandw %k6, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $9, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kmovw %k0, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k0
+; KNL-NEXT:    kandw %k2, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $14, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kandw %k1, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $13, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kandw %k3, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $12, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kandw %k4, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $11, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kandw %k5, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $10, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kandw %k6, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $9, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kmovw %k0, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k0
+; KNL-NEXT:    kandw %k2, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $14, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kandw %k1, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $13, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kandw %k3, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $12, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kandw %k4, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $11, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kandw %k5, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $10, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kandw %k6, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $9, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kmovw %k0, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; KNL-NEXT:    kmovw %esi, %k0
+; KNL-NEXT:    kandw %k2, %k0, %k0
+; KNL-NEXT:    kmovw %edx, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $14, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kandw %k1, %k0, %k0
+; KNL-NEXT:    kmovw %ecx, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $13, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kandw %k3, %k0, %k0
+; KNL-NEXT:    kmovw %r8d, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $12, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kandw %k4, %k0, %k0
+; KNL-NEXT:    kmovw %r9d, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $11, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kandw %k5, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %cl
+; KNL-NEXT:    kmovw %ecx, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $10, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kandw %k6, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %cl
+; KNL-NEXT:    kmovw %ecx, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $9, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %cl
+; KNL-NEXT:    kmovw %ecx, %k7
+; KNL-NEXT:    kandw %k2, %k7, %k2
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %cl
+; KNL-NEXT:    kmovw %ecx, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $14, %k7, %k7
+; KNL-NEXT:    korw %k7, %k2, %k2
+; KNL-NEXT:    kandw %k1, %k2, %k1
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %cl
+; KNL-NEXT:    kmovw %ecx, %k2
+; KNL-NEXT:    kshiftlw $15, %k2, %k2
+; KNL-NEXT:    kshiftrw $13, %k2, %k2
+; KNL-NEXT:    korw %k2, %k1, %k1
+; KNL-NEXT:    kandw %k3, %k1, %k1
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %cl
+; KNL-NEXT:    kmovw %ecx, %k2
+; KNL-NEXT:    kshiftlw $15, %k2, %k2
+; KNL-NEXT:    kshiftrw $12, %k2, %k2
+; KNL-NEXT:    korw %k2, %k1, %k1
+; KNL-NEXT:    kandw %k4, %k1, %k1
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %cl
+; KNL-NEXT:    kmovw %ecx, %k2
+; KNL-NEXT:    kshiftlw $15, %k2, %k2
+; KNL-NEXT:    kshiftrw $11, %k2, %k2
+; KNL-NEXT:    korw %k2, %k1, %k1
+; KNL-NEXT:    kandw %k5, %k1, %k1
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %cl
+; KNL-NEXT:    kmovw %ecx, %k2
+; KNL-NEXT:    kshiftlw $15, %k2, %k2
+; KNL-NEXT:    kshiftrw $10, %k2, %k2
+; KNL-NEXT:    korw %k2, %k1, %k1
+; KNL-NEXT:    kandw %k6, %k1, %k1
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %cl
+; KNL-NEXT:    kmovw %ecx, %k2
+; KNL-NEXT:    kshiftlw $15, %k2, %k2
+; KNL-NEXT:    kshiftrw $9, %k2, %k2
+; KNL-NEXT:    korw %k2, %k1, %k1
+; KNL-NEXT:    kandw %k1, %k0, %k0
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 2-byte Reload
+; KNL-NEXT:    kandw %k1, %k0, %k0
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 2-byte Reload
+; KNL-NEXT:    kandw %k1, %k0, %k0
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 2-byte Reload
+; KNL-NEXT:    kandw %k1, %k0, %k0
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 2-byte Reload
+; KNL-NEXT:    kandw %k1, %k0, %k0
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 2-byte Reload
+; KNL-NEXT:    kandw %k1, %k0, %k0
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 2-byte Reload
+; KNL-NEXT:    kandw %k1, %k0, %k0
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 2-byte Reload
+; KNL-NEXT:    kandw %k1, %k0, %k0
+; KNL-NEXT:    kshiftrw $6, %k0, %k1
+; KNL-NEXT:    kmovw %k1, %r8d
+; KNL-NEXT:    kshiftrw $5, %k0, %k1
+; KNL-NEXT:    kmovw %k1, %r9d
+; KNL-NEXT:    kshiftrw $4, %k0, %k1
+; KNL-NEXT:    kmovw %k1, %r10d
+; KNL-NEXT:    kshiftrw $3, %k0, %k1
+; KNL-NEXT:    kmovw %k1, %edi
+; KNL-NEXT:    kshiftrw $2, %k0, %k1
+; KNL-NEXT:    kmovw %k1, %ecx
+; KNL-NEXT:    kshiftrw $1, %k0, %k1
+; KNL-NEXT:    kmovw %k1, %edx
+; KNL-NEXT:    kmovw %k0, %esi
+; KNL-NEXT:    andb $1, %sil
+; KNL-NEXT:    andb $1, %dl
+; KNL-NEXT:    addb %dl, %dl
+; KNL-NEXT:    orb %sil, %dl
+; KNL-NEXT:    andb $1, %cl
+; KNL-NEXT:    shlb $2, %cl
+; KNL-NEXT:    orb %dl, %cl
+; KNL-NEXT:    andb $1, %dil
+; KNL-NEXT:    shlb $3, %dil
+; KNL-NEXT:    orb %cl, %dil
+; KNL-NEXT:    andb $1, %r10b
+; KNL-NEXT:    shlb $4, %r10b
+; KNL-NEXT:    orb %dil, %r10b
+; KNL-NEXT:    andb $1, %r9b
+; KNL-NEXT:    shlb $5, %r9b
+; KNL-NEXT:    orb %r10b, %r9b
+; KNL-NEXT:    shlb $6, %r8b
+; KNL-NEXT:    orb %r9b, %r8b
+; KNL-NEXT:    andb $127, %r8b
+; KNL-NEXT:    movb %r8b, (%rax)
+; KNL-NEXT:    retq
+;
+; SKX-LABEL: test17:
+; SKX:       ## %bb.0:
+; SKX-NEXT:    movq %rdi, %rax
+; SKX-NEXT:    movb $-3, %dil
+; SKX-NEXT:    kmovd %edi, %k1
+; SKX-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k0
+; SKX-NEXT:    kandb %k1, %k0, %k0
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k1
+; SKX-NEXT:    kshiftlb $7, %k1, %k1
+; SKX-NEXT:    kshiftrb $6, %k1, %k1
+; SKX-NEXT:    korb %k1, %k0, %k0
+; SKX-NEXT:    movb $-5, %dil
+; SKX-NEXT:    kmovd %edi, %k1
+; SKX-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; SKX-NEXT:    kandb %k1, %k0, %k0
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k2
+; SKX-NEXT:    kshiftlb $7, %k2, %k2
+; SKX-NEXT:    kshiftrb $5, %k2, %k2
+; SKX-NEXT:    korb %k2, %k0, %k0
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k3
+; SKX-NEXT:    movb $-9, %dil
+; SKX-NEXT:    kmovd %edi, %k7
+; SKX-NEXT:    kandb %k7, %k0, %k0
+; SKX-NEXT:    kshiftlb $7, %k3, %k3
+; SKX-NEXT:    kshiftrb $4, %k3, %k3
+; SKX-NEXT:    korb %k3, %k0, %k0
+; SKX-NEXT:    movb $-17, %dil
+; SKX-NEXT:    kmovd %edi, %k1
+; SKX-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; SKX-NEXT:    kandb %k1, %k0, %k0
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k4
+; SKX-NEXT:    kshiftlb $7, %k4, %k4
+; SKX-NEXT:    kshiftrb $3, %k4, %k4
+; SKX-NEXT:    korb %k4, %k0, %k0
+; SKX-NEXT:    movb $-33, %dil
+; SKX-NEXT:    kmovd %edi, %k4
+; SKX-NEXT:    kandb %k4, %k0, %k0
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k5
+; SKX-NEXT:    kshiftlb $7, %k5, %k5
+; SKX-NEXT:    kshiftrb $2, %k5, %k5
+; SKX-NEXT:    korb %k5, %k0, %k0
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k6
+; SKX-NEXT:    movb $-65, %dil
+; SKX-NEXT:    kmovd %edi, %k5
+; SKX-NEXT:    kandb %k5, %k0, %k1
+; SKX-NEXT:    kshiftlb $7, %k6, %k6
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k0
+; SKX-NEXT:    kshiftrb $1, %k6, %k6
+; SKX-NEXT:    korb %k6, %k1, %k1
+; SKX-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; SKX-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k6 ## 2-byte Reload
+; SKX-NEXT:    kandb %k6, %k0, %k0
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k1
+; SKX-NEXT:    kshiftlb $7, %k1, %k1
+; SKX-NEXT:    kshiftrb $6, %k1, %k1
+; SKX-NEXT:    korb %k1, %k0, %k0
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k1
+; SKX-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k3 ## 2-byte Reload
+; SKX-NEXT:    kandb %k3, %k0, %k2
+; SKX-NEXT:    kshiftlb $7, %k1, %k1
+; SKX-NEXT:    kshiftrb $5, %k1, %k1
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k0
+; SKX-NEXT:    korb %k1, %k2, %k1
+; SKX-NEXT:    kandb %k7, %k1, %k1
+; SKX-NEXT:    kshiftlb $7, %k0, %k0
+; SKX-NEXT:    kshiftrb $4, %k0, %k0
+; SKX-NEXT:    korb %k0, %k1, %k0
+; SKX-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 ## 2-byte Reload
+; SKX-NEXT:    kandb %k2, %k0, %k0
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k1
+; SKX-NEXT:    kshiftlb $7, %k1, %k1
+; SKX-NEXT:    kshiftrb $3, %k1, %k1
+; SKX-NEXT:    korb %k1, %k0, %k0
+; SKX-NEXT:    kandb %k4, %k0, %k0
+; SKX-NEXT:    kmovw %k4, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k1
+; SKX-NEXT:    kshiftlb $7, %k1, %k1
+; SKX-NEXT:    kshiftrb $2, %k1, %k1
+; SKX-NEXT:    korb %k1, %k0, %k0
+; SKX-NEXT:    kandb %k5, %k0, %k0
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k1
+; SKX-NEXT:    kshiftlb $7, %k1, %k1
+; SKX-NEXT:    kshiftrb $1, %k1, %k1
+; SKX-NEXT:    korb %k1, %k0, %k0
+; SKX-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 2-byte Reload
+; SKX-NEXT:    kandb %k1, %k0, %k0
+; SKX-NEXT:    kmovw %k0, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k0
+; SKX-NEXT:    kandb %k6, %k0, %k0
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k1
+; SKX-NEXT:    kshiftlb $7, %k1, %k1
+; SKX-NEXT:    kshiftrb $6, %k1, %k1
+; SKX-NEXT:    korb %k1, %k0, %k0
+; SKX-NEXT:    kandb %k3, %k0, %k0
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k1
+; SKX-NEXT:    kshiftlb $7, %k1, %k1
+; SKX-NEXT:    kshiftrb $5, %k1, %k1
+; SKX-NEXT:    korb %k1, %k0, %k0
+; SKX-NEXT:    kandb %k7, %k0, %k0
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k1
+; SKX-NEXT:    kshiftlb $7, %k1, %k1
+; SKX-NEXT:    kshiftrb $4, %k1, %k1
+; SKX-NEXT:    korb %k1, %k0, %k0
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k1
+; SKX-NEXT:    kmovq %k2, %k3
+; SKX-NEXT:    kandb %k2, %k0, %k0
+; SKX-NEXT:    kshiftlb $7, %k1, %k1
+; SKX-NEXT:    kshiftrb $3, %k1, %k1
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k2
+; SKX-NEXT:    korb %k1, %k0, %k0
+; SKX-NEXT:    kandb %k4, %k0, %k0
+; SKX-NEXT:    kshiftlb $7, %k2, %k1
+; SKX-NEXT:    kshiftrb $2, %k1, %k1
+; SKX-NEXT:    korb %k1, %k0, %k0
+; SKX-NEXT:    kandb %k5, %k0, %k0
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k1
+; SKX-NEXT:    kshiftlb $7, %k1, %k1
+; SKX-NEXT:    kshiftrb $1, %k1, %k1
+; SKX-NEXT:    korb %k1, %k0, %k0
+; SKX-NEXT:    kmovw %k0, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k1
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k2
+; SKX-NEXT:    kmovq %k6, %k0
+; SKX-NEXT:    kandb %k6, %k1, %k1
+; SKX-NEXT:    kshiftlb $7, %k2, %k2
+; SKX-NEXT:    kshiftrb $6, %k2, %k2
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k6
+; SKX-NEXT:    korb %k2, %k1, %k1
+; SKX-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k4 ## 2-byte Reload
+; SKX-NEXT:    kandb %k4, %k1, %k1
+; SKX-NEXT:    kshiftlb $7, %k6, %k2
+; SKX-NEXT:    kshiftrb $5, %k2, %k2
+; SKX-NEXT:    korb %k2, %k1, %k1
+; SKX-NEXT:    kandb %k7, %k1, %k1
+; SKX-NEXT:    kmovw %k7, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k2
+; SKX-NEXT:    kshiftlb $7, %k2, %k2
+; SKX-NEXT:    kshiftrb $4, %k2, %k2
+; SKX-NEXT:    korb %k2, %k1, %k1
+; SKX-NEXT:    kandb %k3, %k1, %k1
+; SKX-NEXT:    kmovq %k3, %k6
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k2
+; SKX-NEXT:    kshiftlb $7, %k2, %k2
+; SKX-NEXT:    kshiftrb $3, %k2, %k2
+; SKX-NEXT:    korb %k2, %k1, %k1
+; SKX-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 ## 2-byte Reload
+; SKX-NEXT:    kandb %k2, %k1, %k1
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k2
+; SKX-NEXT:    kshiftlb $7, %k2, %k2
+; SKX-NEXT:    kshiftrb $2, %k2, %k2
+; SKX-NEXT:    korb %k2, %k1, %k1
+; SKX-NEXT:    kandb %k5, %k1, %k1
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k2
+; SKX-NEXT:    kshiftlb $7, %k2, %k2
+; SKX-NEXT:    kshiftrb $1, %k2, %k2
+; SKX-NEXT:    korb %k2, %k1, %k1
+; SKX-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k1
+; SKX-NEXT:    kandb %k0, %k1, %k1
+; SKX-NEXT:    kmovq %k0, %k3
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k2
+; SKX-NEXT:    kshiftlb $7, %k2, %k2
+; SKX-NEXT:    kshiftrb $6, %k2, %k2
+; SKX-NEXT:    korb %k2, %k1, %k1
+; SKX-NEXT:    kandb %k4, %k1, %k1
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k2
+; SKX-NEXT:    kshiftlb $7, %k2, %k2
+; SKX-NEXT:    kshiftrb $5, %k2, %k2
+; SKX-NEXT:    korb %k2, %k1, %k1
+; SKX-NEXT:    kandb %k7, %k1, %k1
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k2
+; SKX-NEXT:    kshiftlb $7, %k2, %k2
+; SKX-NEXT:    kshiftrb $4, %k2, %k2
+; SKX-NEXT:    korb %k2, %k1, %k1
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k2
+; SKX-NEXT:    kandb %k6, %k1, %k1
+; SKX-NEXT:    kshiftlb $7, %k2, %k2
+; SKX-NEXT:    kshiftrb $3, %k2, %k2
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k0
+; SKX-NEXT:    korb %k2, %k1, %k1
+; SKX-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 ## 2-byte Reload
+; SKX-NEXT:    kandb %k7, %k1, %k1
+; SKX-NEXT:    kshiftlb $7, %k0, %k0
+; SKX-NEXT:    kshiftrb $2, %k0, %k0
+; SKX-NEXT:    korb %k0, %k1, %k0
+; SKX-NEXT:    kandb %k5, %k0, %k0
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k1
+; SKX-NEXT:    kshiftlb $7, %k1, %k1
+; SKX-NEXT:    kshiftrb $1, %k1, %k1
+; SKX-NEXT:    korb %k1, %k0, %k0
+; SKX-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 2-byte Reload
+; SKX-NEXT:    kandb %k1, %k0, %k0
+; SKX-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 2-byte Reload
+; SKX-NEXT:    kandb %k1, %k0, %k0
+; SKX-NEXT:    kmovw %k0, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k0
+; SKX-NEXT:    kandb %k3, %k0, %k0
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k1
+; SKX-NEXT:    kshiftlb $7, %k1, %k1
+; SKX-NEXT:    kshiftrb $6, %k1, %k1
+; SKX-NEXT:    korb %k1, %k0, %k0
+; SKX-NEXT:    kandb %k4, %k0, %k0
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k1
+; SKX-NEXT:    kshiftlb $7, %k1, %k1
+; SKX-NEXT:    kshiftrb $5, %k1, %k1
+; SKX-NEXT:    korb %k1, %k0, %k0
+; SKX-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k4 ## 2-byte Reload
+; SKX-NEXT:    kandb %k4, %k0, %k0
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k1
+; SKX-NEXT:    kshiftlb $7, %k1, %k1
+; SKX-NEXT:    kshiftrb $4, %k1, %k1
+; SKX-NEXT:    korb %k1, %k0, %k0
+; SKX-NEXT:    kandb %k6, %k0, %k0
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k1
+; SKX-NEXT:    kshiftlb $7, %k1, %k1
+; SKX-NEXT:    kshiftrb $3, %k1, %k1
+; SKX-NEXT:    korb %k1, %k0, %k0
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k1
+; SKX-NEXT:    kandb %k7, %k0, %k0
+; SKX-NEXT:    kshiftlb $7, %k1, %k1
+; SKX-NEXT:    kshiftrb $2, %k1, %k1
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k2
+; SKX-NEXT:    korb %k1, %k0, %k0
+; SKX-NEXT:    kmovw %k5, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; SKX-NEXT:    kandb %k5, %k0, %k0
+; SKX-NEXT:    kshiftlb $7, %k2, %k1
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k2
+; SKX-NEXT:    kshiftrb $1, %k1, %k1
+; SKX-NEXT:    korb %k1, %k0, %k0
+; SKX-NEXT:    kandb %k3, %k2, %k1
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k2
+; SKX-NEXT:    kshiftlb $7, %k2, %k2
+; SKX-NEXT:    kshiftrb $6, %k2, %k2
+; SKX-NEXT:    korb %k2, %k1, %k1
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k2
+; SKX-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k6 ## 2-byte Reload
+; SKX-NEXT:    kandb %k6, %k1, %k1
+; SKX-NEXT:    kshiftlb $7, %k2, %k2
+; SKX-NEXT:    kshiftrb $5, %k2, %k2
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k6
+; SKX-NEXT:    korb %k2, %k1, %k1
+; SKX-NEXT:    kandb %k4, %k1, %k1
+; SKX-NEXT:    kshiftlb $7, %k6, %k2
+; SKX-NEXT:    kshiftrb $4, %k2, %k2
+; SKX-NEXT:    korb %k2, %k1, %k1
+; SKX-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k6 ## 2-byte Reload
+; SKX-NEXT:    kandb %k6, %k1, %k1
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k2
+; SKX-NEXT:    kshiftlb $7, %k2, %k2
+; SKX-NEXT:    kshiftrb $3, %k2, %k2
+; SKX-NEXT:    korb %k2, %k1, %k1
+; SKX-NEXT:    kandb %k7, %k1, %k1
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k2
+; SKX-NEXT:    kshiftlb $7, %k2, %k2
+; SKX-NEXT:    kshiftrb $2, %k2, %k2
+; SKX-NEXT:    korb %k2, %k1, %k1
+; SKX-NEXT:    kandb %k5, %k1, %k1
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k2
+; SKX-NEXT:    kshiftlb $7, %k2, %k2
+; SKX-NEXT:    kshiftrb $1, %k2, %k2
+; SKX-NEXT:    korb %k2, %k1, %k1
+; SKX-NEXT:    kandb %k0, %k1, %k0
+; SKX-NEXT:    kmovw %k0, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k0
+; SKX-NEXT:    kandb %k3, %k0, %k0
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k1
+; SKX-NEXT:    kshiftlb $7, %k1, %k1
+; SKX-NEXT:    kshiftrb $6, %k1, %k1
+; SKX-NEXT:    korb %k1, %k0, %k0
+; SKX-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k3 ## 2-byte Reload
+; SKX-NEXT:    kandb %k3, %k0, %k0
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k1
+; SKX-NEXT:    kshiftlb $7, %k1, %k1
+; SKX-NEXT:    kshiftrb $5, %k1, %k1
+; SKX-NEXT:    korb %k1, %k0, %k0
+; SKX-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k5 ## 2-byte Reload
+; SKX-NEXT:    kandb %k5, %k0, %k0
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k1
+; SKX-NEXT:    kshiftlb $7, %k1, %k1
+; SKX-NEXT:    kshiftrb $4, %k1, %k1
+; SKX-NEXT:    korb %k1, %k0, %k0
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k1
+; SKX-NEXT:    kandb %k6, %k0, %k0
+; SKX-NEXT:    kshiftlb $7, %k1, %k1
+; SKX-NEXT:    kshiftrb $3, %k1, %k1
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k2
+; SKX-NEXT:    korb %k1, %k0, %k0
+; SKX-NEXT:    kandb %k7, %k0, %k0
+; SKX-NEXT:    kshiftlb $7, %k2, %k1
+; SKX-NEXT:    kshiftrb $2, %k1, %k1
+; SKX-NEXT:    korb %k1, %k0, %k0
+; SKX-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k4 ## 2-byte Reload
+; SKX-NEXT:    kandb %k4, %k0, %k0
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k1
+; SKX-NEXT:    kshiftlb $7, %k1, %k1
+; SKX-NEXT:    kshiftrb $1, %k1, %k1
+; SKX-NEXT:    korb %k1, %k0, %k0
+; SKX-NEXT:    kmovd %esi, %k1
+; SKX-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 ## 2-byte Reload
+; SKX-NEXT:    kandb %k2, %k1, %k1
+; SKX-NEXT:    kmovd %edx, %k2
+; SKX-NEXT:    kshiftlb $7, %k2, %k2
+; SKX-NEXT:    kshiftrb $6, %k2, %k2
+; SKX-NEXT:    korb %k2, %k1, %k1
+; SKX-NEXT:    kandb %k3, %k1, %k1
+; SKX-NEXT:    kmovd %ecx, %k2
+; SKX-NEXT:    kshiftlb $7, %k2, %k2
+; SKX-NEXT:    kshiftrb $5, %k2, %k2
+; SKX-NEXT:    korb %k2, %k1, %k1
+; SKX-NEXT:    kandb %k5, %k1, %k1
+; SKX-NEXT:    kmovd %r8d, %k2
+; SKX-NEXT:    kshiftlb $7, %k2, %k2
+; SKX-NEXT:    kshiftrb $4, %k2, %k2
+; SKX-NEXT:    korb %k2, %k1, %k1
+; SKX-NEXT:    kandb %k6, %k1, %k1
+; SKX-NEXT:    kmovd %r9d, %k2
+; SKX-NEXT:    kshiftlb $7, %k2, %k2
+; SKX-NEXT:    kshiftrb $3, %k2, %k2
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k3
+; SKX-NEXT:    korb %k2, %k1, %k1
+; SKX-NEXT:    kandb %k7, %k1, %k1
+; SKX-NEXT:    kshiftlb $7, %k3, %k2
+; SKX-NEXT:    kshiftrb $2, %k2, %k2
+; SKX-NEXT:    korb %k2, %k1, %k1
+; SKX-NEXT:    kandb %k4, %k1, %k1
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k2
+; SKX-NEXT:    kshiftlb $7, %k2, %k2
+; SKX-NEXT:    kshiftrb $1, %k2, %k2
+; SKX-NEXT:    korb %k2, %k1, %k1
+; SKX-NEXT:    kandb %k0, %k1, %k0
+; SKX-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 2-byte Reload
+; SKX-NEXT:    kandb %k1, %k0, %k0
+; SKX-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 2-byte Reload
+; SKX-NEXT:    kandb %k1, %k0, %k0
+; SKX-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 2-byte Reload
+; SKX-NEXT:    kandb %k1, %k0, %k0
+; SKX-NEXT:    kshiftrb $6, %k0, %k1
+; SKX-NEXT:    kmovd %k1, %r8d
+; SKX-NEXT:    kshiftrb $5, %k0, %k1
+; SKX-NEXT:    kmovd %k1, %r9d
+; SKX-NEXT:    kshiftrb $4, %k0, %k1
+; SKX-NEXT:    kmovd %k1, %r10d
+; SKX-NEXT:    kshiftrb $3, %k0, %k1
+; SKX-NEXT:    kmovd %k1, %edi
+; SKX-NEXT:    kshiftrb $2, %k0, %k1
+; SKX-NEXT:    kmovd %k1, %ecx
+; SKX-NEXT:    kshiftrb $1, %k0, %k1
+; SKX-NEXT:    kmovd %k1, %edx
+; SKX-NEXT:    kmovd %k0, %esi
+; SKX-NEXT:    andb $1, %sil
+; SKX-NEXT:    andb $1, %dl
+; SKX-NEXT:    addb %dl, %dl
+; SKX-NEXT:    orb %sil, %dl
+; SKX-NEXT:    andb $1, %cl
+; SKX-NEXT:    shlb $2, %cl
+; SKX-NEXT:    orb %dl, %cl
+; SKX-NEXT:    andb $1, %dil
+; SKX-NEXT:    shlb $3, %dil
+; SKX-NEXT:    orb %cl, %dil
+; SKX-NEXT:    andb $1, %r10b
+; SKX-NEXT:    shlb $4, %r10b
+; SKX-NEXT:    orb %dil, %r10b
+; SKX-NEXT:    andb $1, %r9b
+; SKX-NEXT:    shlb $5, %r9b
+; SKX-NEXT:    orb %r10b, %r9b
+; SKX-NEXT:    shlb $6, %r8b
+; SKX-NEXT:    orb %r9b, %r8b
+; SKX-NEXT:    andb $127, %r8b
+; SKX-NEXT:    movb %r8b, (%rax)
+; SKX-NEXT:    retq
+;
+; KNL_X32-LABEL: test17:
+; KNL_X32:       ## %bb.0:
+; KNL_X32-NEXT:    pushl %ebx
+; KNL_X32-NEXT:    subl $16, %esp
+; KNL_X32-NEXT:    movw $-3, %ax
+; KNL_X32-NEXT:    kmovw %eax, %k2
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k0
+; KNL_X32-NEXT:    kandw %k2, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k1
+; KNL_X32-NEXT:    kshiftlw $15, %k1, %k1
+; KNL_X32-NEXT:    kshiftrw $14, %k1, %k1
+; KNL_X32-NEXT:    korw %k1, %k0, %k0
+; KNL_X32-NEXT:    movw $-5, %ax
+; KNL_X32-NEXT:    kmovw %eax, %k1
+; KNL_X32-NEXT:    kandw %k1, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k3
+; KNL_X32-NEXT:    kshiftlw $15, %k3, %k3
+; KNL_X32-NEXT:    kshiftrw $13, %k3, %k3
+; KNL_X32-NEXT:    korw %k3, %k0, %k0
+; KNL_X32-NEXT:    movw $-9, %ax
+; KNL_X32-NEXT:    kmovw %eax, %k3
+; KNL_X32-NEXT:    kandw %k3, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k4
+; KNL_X32-NEXT:    kshiftlw $15, %k4, %k4
+; KNL_X32-NEXT:    kshiftrw $12, %k4, %k4
+; KNL_X32-NEXT:    korw %k4, %k0, %k0
+; KNL_X32-NEXT:    movw $-17, %ax
+; KNL_X32-NEXT:    kmovw %eax, %k4
+; KNL_X32-NEXT:    kandw %k4, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k5
+; KNL_X32-NEXT:    kshiftlw $15, %k5, %k5
+; KNL_X32-NEXT:    kshiftrw $11, %k5, %k5
+; KNL_X32-NEXT:    korw %k5, %k0, %k0
+; KNL_X32-NEXT:    movw $-33, %ax
+; KNL_X32-NEXT:    kmovw %eax, %k5
+; KNL_X32-NEXT:    kandw %k5, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k6
+; KNL_X32-NEXT:    kshiftlw $15, %k6, %k6
+; KNL_X32-NEXT:    kshiftrw $10, %k6, %k6
+; KNL_X32-NEXT:    korw %k6, %k0, %k0
+; KNL_X32-NEXT:    movw $-65, %ax
+; KNL_X32-NEXT:    kmovw %eax, %k6
+; KNL_X32-NEXT:    kandw %k6, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k7
+; KNL_X32-NEXT:    kshiftlw $15, %k7, %k7
+; KNL_X32-NEXT:    kshiftrw $9, %k7, %k7
+; KNL_X32-NEXT:    korw %k7, %k0, %k0
+; KNL_X32-NEXT:    kmovw %k0, {{[-0-9]+}}(%e{{[sb]}}p) ## 2-byte Spill
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k0
+; KNL_X32-NEXT:    kandw %k2, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k7
+; KNL_X32-NEXT:    kshiftlw $15, %k7, %k7
+; KNL_X32-NEXT:    kshiftrw $14, %k7, %k7
+; KNL_X32-NEXT:    korw %k7, %k0, %k0
+; KNL_X32-NEXT:    kandw %k1, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k7
+; KNL_X32-NEXT:    kshiftlw $15, %k7, %k7
+; KNL_X32-NEXT:    kshiftrw $13, %k7, %k7
+; KNL_X32-NEXT:    korw %k7, %k0, %k0
+; KNL_X32-NEXT:    kandw %k3, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k7
+; KNL_X32-NEXT:    kshiftlw $15, %k7, %k7
+; KNL_X32-NEXT:    kshiftrw $12, %k7, %k7
+; KNL_X32-NEXT:    korw %k7, %k0, %k0
+; KNL_X32-NEXT:    kandw %k4, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k7
+; KNL_X32-NEXT:    kshiftlw $15, %k7, %k7
+; KNL_X32-NEXT:    kshiftrw $11, %k7, %k7
+; KNL_X32-NEXT:    korw %k7, %k0, %k0
+; KNL_X32-NEXT:    kandw %k5, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k7
+; KNL_X32-NEXT:    kshiftlw $15, %k7, %k7
+; KNL_X32-NEXT:    kshiftrw $10, %k7, %k7
+; KNL_X32-NEXT:    korw %k7, %k0, %k0
+; KNL_X32-NEXT:    kandw %k6, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k7
+; KNL_X32-NEXT:    kshiftlw $15, %k7, %k7
+; KNL_X32-NEXT:    kshiftrw $9, %k7, %k7
+; KNL_X32-NEXT:    korw %k7, %k0, %k0
+; KNL_X32-NEXT:    kmovw %k0, {{[-0-9]+}}(%e{{[sb]}}p) ## 2-byte Spill
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k0
+; KNL_X32-NEXT:    kandw %k2, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k7
+; KNL_X32-NEXT:    kshiftlw $15, %k7, %k7
+; KNL_X32-NEXT:    kshiftrw $14, %k7, %k7
+; KNL_X32-NEXT:    korw %k7, %k0, %k0
+; KNL_X32-NEXT:    kandw %k1, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k7
+; KNL_X32-NEXT:    kshiftlw $15, %k7, %k7
+; KNL_X32-NEXT:    kshiftrw $13, %k7, %k7
+; KNL_X32-NEXT:    korw %k7, %k0, %k0
+; KNL_X32-NEXT:    kandw %k3, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k7
+; KNL_X32-NEXT:    kshiftlw $15, %k7, %k7
+; KNL_X32-NEXT:    kshiftrw $12, %k7, %k7
+; KNL_X32-NEXT:    korw %k7, %k0, %k0
+; KNL_X32-NEXT:    kandw %k4, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k7
+; KNL_X32-NEXT:    kshiftlw $15, %k7, %k7
+; KNL_X32-NEXT:    kshiftrw $11, %k7, %k7
+; KNL_X32-NEXT:    korw %k7, %k0, %k0
+; KNL_X32-NEXT:    kandw %k5, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k7
+; KNL_X32-NEXT:    kshiftlw $15, %k7, %k7
+; KNL_X32-NEXT:    kshiftrw $10, %k7, %k7
+; KNL_X32-NEXT:    korw %k7, %k0, %k0
+; KNL_X32-NEXT:    kandw %k6, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k7
+; KNL_X32-NEXT:    kshiftlw $15, %k7, %k7
+; KNL_X32-NEXT:    kshiftrw $9, %k7, %k7
+; KNL_X32-NEXT:    korw %k7, %k0, %k0
+; KNL_X32-NEXT:    kmovw %k0, {{[-0-9]+}}(%e{{[sb]}}p) ## 2-byte Spill
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k0
+; KNL_X32-NEXT:    kandw %k2, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k7
+; KNL_X32-NEXT:    kshiftlw $15, %k7, %k7
+; KNL_X32-NEXT:    kshiftrw $14, %k7, %k7
+; KNL_X32-NEXT:    korw %k7, %k0, %k0
+; KNL_X32-NEXT:    kandw %k1, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k7
+; KNL_X32-NEXT:    kshiftlw $15, %k7, %k7
+; KNL_X32-NEXT:    kshiftrw $13, %k7, %k7
+; KNL_X32-NEXT:    korw %k7, %k0, %k0
+; KNL_X32-NEXT:    kandw %k3, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k7
+; KNL_X32-NEXT:    kshiftlw $15, %k7, %k7
+; KNL_X32-NEXT:    kshiftrw $12, %k7, %k7
+; KNL_X32-NEXT:    korw %k7, %k0, %k0
+; KNL_X32-NEXT:    kandw %k4, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k7
+; KNL_X32-NEXT:    kshiftlw $15, %k7, %k7
+; KNL_X32-NEXT:    kshiftrw $11, %k7, %k7
+; KNL_X32-NEXT:    korw %k7, %k0, %k0
+; KNL_X32-NEXT:    kandw %k5, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k7
+; KNL_X32-NEXT:    kshiftlw $15, %k7, %k7
+; KNL_X32-NEXT:    kshiftrw $10, %k7, %k7
+; KNL_X32-NEXT:    korw %k7, %k0, %k0
+; KNL_X32-NEXT:    kandw %k6, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k7
+; KNL_X32-NEXT:    kshiftlw $15, %k7, %k7
+; KNL_X32-NEXT:    kshiftrw $9, %k7, %k7
+; KNL_X32-NEXT:    korw %k7, %k0, %k0
+; KNL_X32-NEXT:    kmovw %k0, {{[-0-9]+}}(%e{{[sb]}}p) ## 2-byte Spill
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k0
+; KNL_X32-NEXT:    kandw %k2, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k7
+; KNL_X32-NEXT:    kshiftlw $15, %k7, %k7
+; KNL_X32-NEXT:    kshiftrw $14, %k7, %k7
+; KNL_X32-NEXT:    korw %k7, %k0, %k0
+; KNL_X32-NEXT:    kandw %k1, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k7
+; KNL_X32-NEXT:    kshiftlw $15, %k7, %k7
+; KNL_X32-NEXT:    kshiftrw $13, %k7, %k7
+; KNL_X32-NEXT:    korw %k7, %k0, %k0
+; KNL_X32-NEXT:    kandw %k3, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k7
+; KNL_X32-NEXT:    kshiftlw $15, %k7, %k7
+; KNL_X32-NEXT:    kshiftrw $12, %k7, %k7
+; KNL_X32-NEXT:    korw %k7, %k0, %k0
+; KNL_X32-NEXT:    kandw %k4, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k7
+; KNL_X32-NEXT:    kshiftlw $15, %k7, %k7
+; KNL_X32-NEXT:    kshiftrw $11, %k7, %k7
+; KNL_X32-NEXT:    korw %k7, %k0, %k0
+; KNL_X32-NEXT:    kandw %k5, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k7
+; KNL_X32-NEXT:    kshiftlw $15, %k7, %k7
+; KNL_X32-NEXT:    kshiftrw $10, %k7, %k7
+; KNL_X32-NEXT:    korw %k7, %k0, %k0
+; KNL_X32-NEXT:    kandw %k6, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k7
+; KNL_X32-NEXT:    kshiftlw $15, %k7, %k7
+; KNL_X32-NEXT:    kshiftrw $9, %k7, %k7
+; KNL_X32-NEXT:    korw %k7, %k0, %k0
+; KNL_X32-NEXT:    kmovw %k0, {{[-0-9]+}}(%e{{[sb]}}p) ## 2-byte Spill
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k0
+; KNL_X32-NEXT:    kandw %k2, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k7
+; KNL_X32-NEXT:    kshiftlw $15, %k7, %k7
+; KNL_X32-NEXT:    kshiftrw $14, %k7, %k7
+; KNL_X32-NEXT:    korw %k7, %k0, %k0
+; KNL_X32-NEXT:    kandw %k1, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k7
+; KNL_X32-NEXT:    kshiftlw $15, %k7, %k7
+; KNL_X32-NEXT:    kshiftrw $13, %k7, %k7
+; KNL_X32-NEXT:    korw %k7, %k0, %k0
+; KNL_X32-NEXT:    kandw %k3, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k7
+; KNL_X32-NEXT:    kshiftlw $15, %k7, %k7
+; KNL_X32-NEXT:    kshiftrw $12, %k7, %k7
+; KNL_X32-NEXT:    korw %k7, %k0, %k0
+; KNL_X32-NEXT:    kandw %k4, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k7
+; KNL_X32-NEXT:    kshiftlw $15, %k7, %k7
+; KNL_X32-NEXT:    kshiftrw $11, %k7, %k7
+; KNL_X32-NEXT:    korw %k7, %k0, %k0
+; KNL_X32-NEXT:    kandw %k5, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k7
+; KNL_X32-NEXT:    kshiftlw $15, %k7, %k7
+; KNL_X32-NEXT:    kshiftrw $10, %k7, %k7
+; KNL_X32-NEXT:    korw %k7, %k0, %k0
+; KNL_X32-NEXT:    kandw %k6, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k7
+; KNL_X32-NEXT:    kshiftlw $15, %k7, %k7
+; KNL_X32-NEXT:    kshiftrw $9, %k7, %k7
+; KNL_X32-NEXT:    korw %k7, %k0, %k0
+; KNL_X32-NEXT:    kmovw %k0, {{[-0-9]+}}(%e{{[sb]}}p) ## 2-byte Spill
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k0
+; KNL_X32-NEXT:    kandw %k2, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k7
+; KNL_X32-NEXT:    kshiftlw $15, %k7, %k7
+; KNL_X32-NEXT:    kshiftrw $14, %k7, %k7
+; KNL_X32-NEXT:    korw %k7, %k0, %k0
+; KNL_X32-NEXT:    kandw %k1, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k7
+; KNL_X32-NEXT:    kshiftlw $15, %k7, %k7
+; KNL_X32-NEXT:    kshiftrw $13, %k7, %k7
+; KNL_X32-NEXT:    korw %k7, %k0, %k0
+; KNL_X32-NEXT:    kandw %k3, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k7
+; KNL_X32-NEXT:    kshiftlw $15, %k7, %k7
+; KNL_X32-NEXT:    kshiftrw $12, %k7, %k7
+; KNL_X32-NEXT:    korw %k7, %k0, %k0
+; KNL_X32-NEXT:    kandw %k4, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k7
+; KNL_X32-NEXT:    kshiftlw $15, %k7, %k7
+; KNL_X32-NEXT:    kshiftrw $11, %k7, %k7
+; KNL_X32-NEXT:    korw %k7, %k0, %k0
+; KNL_X32-NEXT:    kandw %k5, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k7
+; KNL_X32-NEXT:    kshiftlw $15, %k7, %k7
+; KNL_X32-NEXT:    kshiftrw $10, %k7, %k7
+; KNL_X32-NEXT:    korw %k7, %k0, %k0
+; KNL_X32-NEXT:    kandw %k6, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k7
+; KNL_X32-NEXT:    kshiftlw $15, %k7, %k7
+; KNL_X32-NEXT:    kshiftrw $9, %k7, %k7
+; KNL_X32-NEXT:    korw %k7, %k0, %k0
+; KNL_X32-NEXT:    kmovw %k0, {{[-0-9]+}}(%e{{[sb]}}p) ## 2-byte Spill
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k0
+; KNL_X32-NEXT:    kandw %k2, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k7
+; KNL_X32-NEXT:    kshiftlw $15, %k7, %k7
+; KNL_X32-NEXT:    kshiftrw $14, %k7, %k7
+; KNL_X32-NEXT:    korw %k7, %k0, %k0
+; KNL_X32-NEXT:    kandw %k1, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k7
+; KNL_X32-NEXT:    kshiftlw $15, %k7, %k7
+; KNL_X32-NEXT:    kshiftrw $13, %k7, %k7
+; KNL_X32-NEXT:    korw %k7, %k0, %k0
+; KNL_X32-NEXT:    kandw %k3, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k7
+; KNL_X32-NEXT:    kshiftlw $15, %k7, %k7
+; KNL_X32-NEXT:    kshiftrw $12, %k7, %k7
+; KNL_X32-NEXT:    korw %k7, %k0, %k0
+; KNL_X32-NEXT:    kandw %k4, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k7
+; KNL_X32-NEXT:    kshiftlw $15, %k7, %k7
+; KNL_X32-NEXT:    kshiftrw $11, %k7, %k7
+; KNL_X32-NEXT:    korw %k7, %k0, %k0
+; KNL_X32-NEXT:    kandw %k5, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k7
+; KNL_X32-NEXT:    kshiftlw $15, %k7, %k7
+; KNL_X32-NEXT:    kshiftrw $10, %k7, %k7
+; KNL_X32-NEXT:    korw %k7, %k0, %k0
+; KNL_X32-NEXT:    kandw %k6, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k7
+; KNL_X32-NEXT:    kshiftlw $15, %k7, %k7
+; KNL_X32-NEXT:    kshiftrw $9, %k7, %k7
+; KNL_X32-NEXT:    korw %k7, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k7
+; KNL_X32-NEXT:    kandw %k2, %k7, %k2
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k7
+; KNL_X32-NEXT:    kshiftlw $15, %k7, %k7
+; KNL_X32-NEXT:    kshiftrw $14, %k7, %k7
+; KNL_X32-NEXT:    korw %k7, %k2, %k2
+; KNL_X32-NEXT:    kandw %k1, %k2, %k1
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k2
+; KNL_X32-NEXT:    kshiftlw $15, %k2, %k2
+; KNL_X32-NEXT:    kshiftrw $13, %k2, %k2
+; KNL_X32-NEXT:    korw %k2, %k1, %k1
+; KNL_X32-NEXT:    kandw %k3, %k1, %k1
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k2
+; KNL_X32-NEXT:    kshiftlw $15, %k2, %k2
+; KNL_X32-NEXT:    kshiftrw $12, %k2, %k2
+; KNL_X32-NEXT:    korw %k2, %k1, %k1
+; KNL_X32-NEXT:    kandw %k4, %k1, %k1
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k2
+; KNL_X32-NEXT:    kshiftlw $15, %k2, %k2
+; KNL_X32-NEXT:    kshiftrw $11, %k2, %k2
+; KNL_X32-NEXT:    korw %k2, %k1, %k1
+; KNL_X32-NEXT:    kandw %k5, %k1, %k1
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k2
+; KNL_X32-NEXT:    kshiftlw $15, %k2, %k2
+; KNL_X32-NEXT:    kshiftrw $10, %k2, %k2
+; KNL_X32-NEXT:    korw %k2, %k1, %k1
+; KNL_X32-NEXT:    kandw %k6, %k1, %k1
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k2
+; KNL_X32-NEXT:    kshiftlw $15, %k2, %k2
+; KNL_X32-NEXT:    kshiftrw $9, %k2, %k2
+; KNL_X32-NEXT:    korw %k2, %k1, %k1
+; KNL_X32-NEXT:    kandw %k1, %k0, %k0
+; KNL_X32-NEXT:    kmovw {{[-0-9]+}}(%e{{[sb]}}p), %k1 ## 2-byte Reload
+; KNL_X32-NEXT:    kandw %k1, %k0, %k0
+; KNL_X32-NEXT:    kmovw {{[-0-9]+}}(%e{{[sb]}}p), %k1 ## 2-byte Reload
+; KNL_X32-NEXT:    kandw %k1, %k0, %k0
+; KNL_X32-NEXT:    kmovw {{[-0-9]+}}(%e{{[sb]}}p), %k1 ## 2-byte Reload
+; KNL_X32-NEXT:    kandw %k1, %k0, %k0
+; KNL_X32-NEXT:    kmovw {{[-0-9]+}}(%e{{[sb]}}p), %k1 ## 2-byte Reload
+; KNL_X32-NEXT:    kandw %k1, %k0, %k0
+; KNL_X32-NEXT:    kmovw {{[-0-9]+}}(%e{{[sb]}}p), %k1 ## 2-byte Reload
+; KNL_X32-NEXT:    kandw %k1, %k0, %k0
+; KNL_X32-NEXT:    kmovw {{[-0-9]+}}(%e{{[sb]}}p), %k1 ## 2-byte Reload
+; KNL_X32-NEXT:    kandw %k1, %k0, %k0
+; KNL_X32-NEXT:    kmovw {{[-0-9]+}}(%e{{[sb]}}p), %k1 ## 2-byte Reload
+; KNL_X32-NEXT:    kandw %k1, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $6, %k0, %k1
+; KNL_X32-NEXT:    kmovw %k1, %ecx
+; KNL_X32-NEXT:    kshiftrw $5, %k0, %k1
+; KNL_X32-NEXT:    kmovw %k1, %eax
+; KNL_X32-NEXT:    kshiftrw $1, %k0, %k1
+; KNL_X32-NEXT:    kmovw %k1, %edx
+; KNL_X32-NEXT:    kshiftrw $2, %k0, %k1
+; KNL_X32-NEXT:    kmovw %k0, %ebx
+; KNL_X32-NEXT:    andb $1, %bl
+; KNL_X32-NEXT:    andb $1, %dl
+; KNL_X32-NEXT:    addb %dl, %dl
+; KNL_X32-NEXT:    orb %bl, %dl
+; KNL_X32-NEXT:    kmovw %k1, %ebx
+; KNL_X32-NEXT:    kshiftrw $3, %k0, %k1
+; KNL_X32-NEXT:    andb $1, %bl
+; KNL_X32-NEXT:    shlb $2, %bl
+; KNL_X32-NEXT:    orb %dl, %bl
+; KNL_X32-NEXT:    kmovw %k1, %edx
+; KNL_X32-NEXT:    kshiftrw $4, %k0, %k0
+; KNL_X32-NEXT:    andb $1, %dl
+; KNL_X32-NEXT:    shlb $3, %dl
+; KNL_X32-NEXT:    orb %bl, %dl
+; KNL_X32-NEXT:    kmovw %k0, %ebx
+; KNL_X32-NEXT:    andb $1, %bl
+; KNL_X32-NEXT:    shlb $4, %bl
+; KNL_X32-NEXT:    orb %dl, %bl
+; KNL_X32-NEXT:    andb $1, %al
+; KNL_X32-NEXT:    shlb $5, %al
+; KNL_X32-NEXT:    orb %bl, %al
+; KNL_X32-NEXT:    shlb $6, %cl
+; KNL_X32-NEXT:    orb %al, %cl
+; KNL_X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; KNL_X32-NEXT:    andb $127, %cl
+; KNL_X32-NEXT:    movb %cl, (%eax)
+; KNL_X32-NEXT:    addl $16, %esp
+; KNL_X32-NEXT:    popl %ebx
+; KNL_X32-NEXT:    retl $4
+  %j = and <7 x i1> %a, %b
+  %k = and <7 x i1> %j, %c
+  %l = and <7 x i1> %k, %d
+  %m = and <7 x i1> %l, %e
+  %n = and <7 x i1> %m, %f
+  %o = and <7 x i1> %n, %g
+  %p = and <7 x i1> %o, %h
+  %q = and <7 x i1> %p, %i
+  ret <7 x i1> %q
+}
diff --git a/llvm/test/CodeGen/X86/avx512-cmp-kor-sequence.ll b/llvm/test/CodeGen/X86/avx512-cmp-kor-sequence.ll
index 72e48ae02..77f982a8c 100644
--- a/llvm/test/CodeGen/X86/avx512-cmp-kor-sequence.ll
+++ b/llvm/test/CodeGen/X86/avx512-cmp-kor-sequence.ll
@@ -48,5 +48,5 @@ entry:
 ; Function Attrs: nounwind readnone
 declare <16 x i1> @llvm.x86.avx512.cmp.ps.512(<16 x float>, <16 x float>, i32, i32) #1
 
-attributes #0 = { nounwind readnone uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="knl" "target-features"="+adx,+aes,+avx,+avx2,+avx512cd,+avx512er,+avx512f,+avx512pf,+bmi,+bmi2,+cx16,+f16c,+fma,+fsgsbase,+fxsr,+lzcnt,+mmx,+movbe,+pclmul,+popcnt,+prefetchwt1,+rdrnd,+rdseed,+rtm,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87,+xsave,+xsaveopt" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind readnone uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="knl" "target-features"="+adx,+aes,+avx,+avx2,+avx512cd,+avx512er,+avx512f,+avx512pf,+bmi,+bmi2,+cx16,+f16c,+fma,+fsgsbase,+fxsr,+lzcnt,+mmx,+movbe,+pclmul,+popcnt,+prefetchwt1,+rdrnd,+rdseed,+rtm,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87,+xsave,+xsaveopt" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
diff --git a/llvm/test/CodeGen/X86/avx512-cmp-mask.ll b/llvm/test/CodeGen/X86/avx512-cmp-mask.ll
new file mode 100644
index 000000000..f1e9717b3
--- /dev/null
+++ b/llvm/test/CodeGen/X86/avx512-cmp-mask.ll
@@ -0,0 +1,51 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=+avx512f | FileCheck %s --check-prefixes=CHECK,AVX512F
+; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=+avx512f,+avx512dq | FileCheck %s --check-prefixes=CHECK,AVX512DQ
+; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=+avx512f,+avx512vl,+avx512dq | FileCheck %s --check-prefixes=CHECK,AVX512DQVL
+
+define <4 x i64> @PR32546(<8 x float> %a, <8 x float> %b, <8 x float> %c, <8 x float> %d) {
+; AVX512F-LABEL: PR32546:
+; AVX512F:       ## %bb.0: ## %entry
+; AVX512F-NEXT:    ## kill: def $ymm3 killed $ymm3 def $zmm3
+; AVX512F-NEXT:    ## kill: def $ymm2 killed $ymm2 def $zmm2
+; AVX512F-NEXT:    ## kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512F-NEXT:    ## kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512F-NEXT:    vcmpltps %zmm3, %zmm2, %k1
+; AVX512F-NEXT:    vcmpltps %zmm1, %zmm0, %k0 {%k1}
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    movzbl %al, %eax
+; AVX512F-NEXT:    vmovd %eax, %xmm0
+; AVX512F-NEXT:    vpbroadcastd %xmm0, %ymm0
+; AVX512F-NEXT:    retq
+;
+; AVX512DQ-LABEL: PR32546:
+; AVX512DQ:       ## %bb.0: ## %entry
+; AVX512DQ-NEXT:    ## kill: def $ymm3 killed $ymm3 def $zmm3
+; AVX512DQ-NEXT:    ## kill: def $ymm2 killed $ymm2 def $zmm2
+; AVX512DQ-NEXT:    ## kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512DQ-NEXT:    ## kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512DQ-NEXT:    vcmpltps %zmm3, %zmm2, %k1
+; AVX512DQ-NEXT:    vcmpltps %zmm1, %zmm0, %k0 {%k1}
+; AVX512DQ-NEXT:    kmovb %k0, %eax
+; AVX512DQ-NEXT:    vmovd %eax, %xmm0
+; AVX512DQ-NEXT:    vpbroadcastd %xmm0, %ymm0
+; AVX512DQ-NEXT:    retq
+;
+; AVX512DQVL-LABEL: PR32546:
+; AVX512DQVL:       ## %bb.0: ## %entry
+; AVX512DQVL-NEXT:    vcmpltps %ymm3, %ymm2, %k1
+; AVX512DQVL-NEXT:    vcmpltps %ymm1, %ymm0, %k0 {%k1}
+; AVX512DQVL-NEXT:    kmovb %k0, %eax
+; AVX512DQVL-NEXT:    vpbroadcastd %eax, %ymm0
+; AVX512DQVL-NEXT:    retq
+entry:
+  %0 = tail call i8 @llvm.x86.avx512.mask.cmp.ps.256(<8 x float> %a, <8 x float> %b, i32 1, i8 -1)
+  %1 = tail call i8 @llvm.x86.avx512.mask.cmp.ps.256(<8 x float> %c, <8 x float> %d, i32 1, i8 -1)
+  %and17 = and i8 %1, %0
+  %and = zext i8 %and17 to i32
+  %2 = insertelement <8 x i32> undef, i32 %and, i32 0
+  %vecinit7.i = shufflevector <8 x i32> %2, <8 x i32> undef, <8 x i32> zeroinitializer
+  %3 = bitcast <8 x i32> %vecinit7.i to <4 x i64>
+  ret <4 x i64> %3
+}
+ declare i8 @llvm.x86.avx512.mask.cmp.ps.256(<8 x float>, <8 x float>, i32, i8)
diff --git a/llvm/test/CodeGen/X86/avx512-cmp.ll b/llvm/test/CodeGen/X86/avx512-cmp.ll
index ecdca99be..bd902dde2 100644
--- a/llvm/test/CodeGen/X86/avx512-cmp.ll
+++ b/llvm/test/CodeGen/X86/avx512-cmp.ll
@@ -70,9 +70,8 @@ define float @test5(float %p) #0 {
 ; ALL-NEXT:    retq
 ; ALL-NEXT:  LBB3_1: ## %if.end
 ; ALL-NEXT:    vcmpltss %xmm0, %xmm1, %k1
-; ALL-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; ALL-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; ALL-NEXT:    vmovss %xmm1, %xmm0, %xmm0 {%k1}
+; ALL-NEXT:    vmovss {{.*}}(%rip), %xmm0 {%k1}
 ; ALL-NEXT:    retq
 entry:
   %cmp = fcmp oeq float %p, 0.000000e+00
@@ -182,3 +181,39 @@ if.then.i:
 if.end.i:
   ret i32 6
 }
+
+; This test previously caused an infinite loop in legalize vector ops. Due to
+; CSE triggering on the call to UpdateNodeOperands and the resulting node not
+; being passed to LowerOperation. The add is needed to force the zext into a
+; sext on that path. The shuffle keeps the zext alive. The xor somehow
+; influences the zext to be visited before the sext exposing the CSE opportunity
+; for the sext since zext of setcc is custom legalized to a sext and shift.
+define <8 x i32> @legalize_loop(<8 x double> %arg) {
+; KNL-LABEL: legalize_loop:
+; KNL:       ## %bb.0:
+; KNL-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; KNL-NEXT:    vcmpnltpd %zmm0, %zmm1, %k1
+; KNL-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; KNL-NEXT:    vpsrld $31, %ymm0, %ymm1
+; KNL-NEXT:    vpshufd {{.*#+}} ymm1 = ymm1[3,2,1,0,7,6,5,4]
+; KNL-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,3,0,1]
+; KNL-NEXT:    vpsubd %ymm0, %ymm1, %ymm0
+; KNL-NEXT:    retq
+;
+; SKX-LABEL: legalize_loop:
+; SKX:       ## %bb.0:
+; SKX-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; SKX-NEXT:    vcmpnltpd %zmm0, %zmm1, %k0
+; SKX-NEXT:    vpmovm2d %k0, %ymm0
+; SKX-NEXT:    vpsrld $31, %ymm0, %ymm1
+; SKX-NEXT:    vpshufd {{.*#+}} ymm1 = ymm1[3,2,1,0,7,6,5,4]
+; SKX-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,3,0,1]
+; SKX-NEXT:    vpsubd %ymm0, %ymm1, %ymm0
+; SKX-NEXT:    retq
+  %tmp = fcmp ogt <8 x double> %arg, zeroinitializer
+  %tmp1 = xor <8 x i1> %tmp, <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>
+  %tmp2 = zext <8 x i1> %tmp1 to <8 x i32>
+  %tmp3 = shufflevector <8 x i32> %tmp2, <8 x i32> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  %tmp4 = add <8 x i32> %tmp2, %tmp3
+  ret <8 x i32> %tmp4
+}
diff --git a/llvm/test/CodeGen/X86/avx512-cvt.ll b/llvm/test/CodeGen/X86/avx512-cvt.ll
index 2c8978d4a..e6b43c07f 100644
--- a/llvm/test/CodeGen/X86/avx512-cvt.ll
+++ b/llvm/test/CodeGen/X86/avx512-cvt.ll
@@ -25,25 +25,25 @@ define <8 x double> @sltof864(<8 x i64> %a) {
 ; NODQ-NEXT:    vcvtsi2sd %rax, %xmm2, %xmm2
 ; NODQ-NEXT:    vmovq %xmm1, %rax
 ; NODQ-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm1
-; NODQ-NEXT:    vmovlhps {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; NODQ-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm1[0],xmm2[0]
 ; NODQ-NEXT:    vextracti32x4 $2, %zmm0, %xmm2
 ; NODQ-NEXT:    vpextrq $1, %xmm2, %rax
 ; NODQ-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm3
 ; NODQ-NEXT:    vmovq %xmm2, %rax
 ; NODQ-NEXT:    vcvtsi2sd %rax, %xmm4, %xmm2
-; NODQ-NEXT:    vmovlhps {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; NODQ-NEXT:    vunpcklpd {{.*#+}} xmm2 = xmm2[0],xmm3[0]
 ; NODQ-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
 ; NODQ-NEXT:    vextracti128 $1, %ymm0, %xmm2
 ; NODQ-NEXT:    vpextrq $1, %xmm2, %rax
 ; NODQ-NEXT:    vcvtsi2sd %rax, %xmm4, %xmm3
 ; NODQ-NEXT:    vmovq %xmm2, %rax
 ; NODQ-NEXT:    vcvtsi2sd %rax, %xmm4, %xmm2
-; NODQ-NEXT:    vmovlhps {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; NODQ-NEXT:    vunpcklpd {{.*#+}} xmm2 = xmm2[0],xmm3[0]
 ; NODQ-NEXT:    vpextrq $1, %xmm0, %rax
 ; NODQ-NEXT:    vcvtsi2sd %rax, %xmm4, %xmm3
 ; NODQ-NEXT:    vmovq %xmm0, %rax
 ; NODQ-NEXT:    vcvtsi2sd %rax, %xmm4, %xmm0
-; NODQ-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm3[0]
+; NODQ-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm3[0]
 ; NODQ-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; NODQ-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
 ; NODQ-NEXT:    retq
@@ -69,12 +69,12 @@ define <4 x double> @slto4f64(<4 x i64> %a) {
 ; NODQ-NEXT:    vcvtsi2sd %rax, %xmm2, %xmm2
 ; NODQ-NEXT:    vmovq %xmm1, %rax
 ; NODQ-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm1
-; NODQ-NEXT:    vmovlhps {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; NODQ-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm1[0],xmm2[0]
 ; NODQ-NEXT:    vpextrq $1, %xmm0, %rax
 ; NODQ-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm2
 ; NODQ-NEXT:    vmovq %xmm0, %rax
 ; NODQ-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm0
-; NODQ-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; NODQ-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm2[0]
 ; NODQ-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; NODQ-NEXT:    retq
 ;
@@ -100,7 +100,7 @@ define <2 x double> @slto2f64(<2 x i64> %a) {
 ; NODQ-NEXT:    vcvtsi2sd %rax, %xmm1, %xmm1
 ; NODQ-NEXT:    vmovq %xmm0, %rax
 ; NODQ-NEXT:    vcvtsi2sd %rax, %xmm2, %xmm0
-; NODQ-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; NODQ-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm1[0]
 ; NODQ-NEXT:    retq
 ;
 ; VLDQ-LABEL: slto2f64:
@@ -140,7 +140,7 @@ define <2 x float> @sltof2f32(<2 x i64> %a) {
 ; VLNODQ-NEXT:    vcvtsi2ss %rax, %xmm1, %xmm1
 ; VLNODQ-NEXT:    vmovq %xmm0, %rax
 ; VLNODQ-NEXT:    vcvtsi2ss %rax, %xmm2, %xmm0
-; VLNODQ-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; VLNODQ-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; VLNODQ-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; VLNODQ-NEXT:    retq
 ;
@@ -513,15 +513,14 @@ define <8 x i8> @f64to8uc(<8 x double> %f) {
 ; NOVL-LABEL: f64to8uc:
 ; NOVL:       # %bb.0:
 ; NOVL-NEXT:    vcvttpd2dq %zmm0, %ymm0
-; NOVL-NEXT:    vpmovdw %zmm0, %ymm0
-; NOVL-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; NOVL-NEXT:    vpmovdb %zmm0, %xmm0
 ; NOVL-NEXT:    vzeroupper
 ; NOVL-NEXT:    retq
 ;
 ; VL-LABEL: f64to8uc:
 ; VL:       # %bb.0:
 ; VL-NEXT:    vcvttpd2dq %zmm0, %ymm0
-; VL-NEXT:    vpmovdw %ymm0, %xmm0
+; VL-NEXT:    vpmovdb %ymm0, %xmm0
 ; VL-NEXT:    vzeroupper
 ; VL-NEXT:    retq
   %res = fptoui <8 x double> %f to <8 x i8>
@@ -657,15 +656,14 @@ define <8 x i8> @f64to8sc(<8 x double> %f) {
 ; NOVL-LABEL: f64to8sc:
 ; NOVL:       # %bb.0:
 ; NOVL-NEXT:    vcvttpd2dq %zmm0, %ymm0
-; NOVL-NEXT:    vpmovdw %zmm0, %ymm0
-; NOVL-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; NOVL-NEXT:    vpmovdb %zmm0, %xmm0
 ; NOVL-NEXT:    vzeroupper
 ; NOVL-NEXT:    retq
 ;
 ; VL-LABEL: f64to8sc:
 ; VL:       # %bb.0:
 ; VL-NEXT:    vcvttpd2dq %zmm0, %ymm0
-; VL-NEXT:    vpmovdw %ymm0, %xmm0
+; VL-NEXT:    vpmovdb %ymm0, %xmm0
 ; VL-NEXT:    vzeroupper
 ; VL-NEXT:    retq
   %res = fptosi <8 x double> %f to <8 x i8>
@@ -1042,13 +1040,13 @@ define <16 x float> @slto16f32(<16 x i64> %a) {
 ; NODQ-NEXT:    vcvtsi2ss %rax, %xmm5, %xmm1
 ; NODQ-NEXT:    vinsertps {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[0]
 ; NODQ-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
-; NODQ-NEXT:    vextracti32x4 $2, %zmm0, %xmm2
+; NODQ-NEXT:    vextractf32x4 $2, %zmm0, %xmm2
 ; NODQ-NEXT:    vpextrq $1, %xmm2, %rax
 ; NODQ-NEXT:    vcvtsi2ss %rax, %xmm5, %xmm3
 ; NODQ-NEXT:    vmovq %xmm2, %rax
 ; NODQ-NEXT:    vcvtsi2ss %rax, %xmm5, %xmm2
 ; NODQ-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[2,3]
-; NODQ-NEXT:    vextracti32x4 $3, %zmm0, %xmm3
+; NODQ-NEXT:    vextractf32x4 $3, %zmm0, %xmm3
 ; NODQ-NEXT:    vmovq %xmm3, %rax
 ; NODQ-NEXT:    vcvtsi2ss %rax, %xmm5, %xmm4
 ; NODQ-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1],xmm4[0],xmm2[3]
@@ -1096,25 +1094,25 @@ define <8 x double> @slto8f64(<8 x i64> %a) {
 ; NODQ-NEXT:    vcvtsi2sd %rax, %xmm2, %xmm2
 ; NODQ-NEXT:    vmovq %xmm1, %rax
 ; NODQ-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm1
-; NODQ-NEXT:    vmovlhps {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; NODQ-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm1[0],xmm2[0]
 ; NODQ-NEXT:    vextracti32x4 $2, %zmm0, %xmm2
 ; NODQ-NEXT:    vpextrq $1, %xmm2, %rax
 ; NODQ-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm3
 ; NODQ-NEXT:    vmovq %xmm2, %rax
 ; NODQ-NEXT:    vcvtsi2sd %rax, %xmm4, %xmm2
-; NODQ-NEXT:    vmovlhps {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; NODQ-NEXT:    vunpcklpd {{.*#+}} xmm2 = xmm2[0],xmm3[0]
 ; NODQ-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
 ; NODQ-NEXT:    vextracti128 $1, %ymm0, %xmm2
 ; NODQ-NEXT:    vpextrq $1, %xmm2, %rax
 ; NODQ-NEXT:    vcvtsi2sd %rax, %xmm4, %xmm3
 ; NODQ-NEXT:    vmovq %xmm2, %rax
 ; NODQ-NEXT:    vcvtsi2sd %rax, %xmm4, %xmm2
-; NODQ-NEXT:    vmovlhps {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; NODQ-NEXT:    vunpcklpd {{.*#+}} xmm2 = xmm2[0],xmm3[0]
 ; NODQ-NEXT:    vpextrq $1, %xmm0, %rax
 ; NODQ-NEXT:    vcvtsi2sd %rax, %xmm4, %xmm3
 ; NODQ-NEXT:    vmovq %xmm0, %rax
 ; NODQ-NEXT:    vcvtsi2sd %rax, %xmm4, %xmm0
-; NODQ-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm3[0]
+; NODQ-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm3[0]
 ; NODQ-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; NODQ-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
 ; NODQ-NEXT:    retq
@@ -1140,25 +1138,25 @@ define <16 x double> @slto16f64(<16 x i64> %a) {
 ; NODQ-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm3
 ; NODQ-NEXT:    vmovq %xmm2, %rax
 ; NODQ-NEXT:    vcvtsi2sd %rax, %xmm4, %xmm2
-; NODQ-NEXT:    vmovlhps {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; NODQ-NEXT:    vunpcklpd {{.*#+}} xmm2 = xmm2[0],xmm3[0]
 ; NODQ-NEXT:    vextracti32x4 $2, %zmm0, %xmm3
 ; NODQ-NEXT:    vpextrq $1, %xmm3, %rax
 ; NODQ-NEXT:    vcvtsi2sd %rax, %xmm4, %xmm4
 ; NODQ-NEXT:    vmovq %xmm3, %rax
 ; NODQ-NEXT:    vcvtsi2sd %rax, %xmm5, %xmm3
-; NODQ-NEXT:    vmovlhps {{.*#+}} xmm3 = xmm3[0],xmm4[0]
+; NODQ-NEXT:    vunpcklpd {{.*#+}} xmm3 = xmm3[0],xmm4[0]
 ; NODQ-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
 ; NODQ-NEXT:    vextracti128 $1, %ymm0, %xmm3
 ; NODQ-NEXT:    vpextrq $1, %xmm3, %rax
 ; NODQ-NEXT:    vcvtsi2sd %rax, %xmm5, %xmm4
 ; NODQ-NEXT:    vmovq %xmm3, %rax
 ; NODQ-NEXT:    vcvtsi2sd %rax, %xmm5, %xmm3
-; NODQ-NEXT:    vmovlhps {{.*#+}} xmm3 = xmm3[0],xmm4[0]
+; NODQ-NEXT:    vunpcklpd {{.*#+}} xmm3 = xmm3[0],xmm4[0]
 ; NODQ-NEXT:    vpextrq $1, %xmm0, %rax
 ; NODQ-NEXT:    vcvtsi2sd %rax, %xmm5, %xmm4
 ; NODQ-NEXT:    vmovq %xmm0, %rax
 ; NODQ-NEXT:    vcvtsi2sd %rax, %xmm5, %xmm0
-; NODQ-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm4[0]
+; NODQ-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm4[0]
 ; NODQ-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
 ; NODQ-NEXT:    vinsertf64x4 $1, %ymm2, %zmm0, %zmm0
 ; NODQ-NEXT:    vextracti32x4 $3, %zmm1, %xmm2
@@ -1166,25 +1164,25 @@ define <16 x double> @slto16f64(<16 x i64> %a) {
 ; NODQ-NEXT:    vcvtsi2sd %rax, %xmm5, %xmm3
 ; NODQ-NEXT:    vmovq %xmm2, %rax
 ; NODQ-NEXT:    vcvtsi2sd %rax, %xmm5, %xmm2
-; NODQ-NEXT:    vmovlhps {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; NODQ-NEXT:    vunpcklpd {{.*#+}} xmm2 = xmm2[0],xmm3[0]
 ; NODQ-NEXT:    vextracti32x4 $2, %zmm1, %xmm3
 ; NODQ-NEXT:    vpextrq $1, %xmm3, %rax
 ; NODQ-NEXT:    vcvtsi2sd %rax, %xmm5, %xmm4
 ; NODQ-NEXT:    vmovq %xmm3, %rax
 ; NODQ-NEXT:    vcvtsi2sd %rax, %xmm5, %xmm3
-; NODQ-NEXT:    vmovlhps {{.*#+}} xmm3 = xmm3[0],xmm4[0]
+; NODQ-NEXT:    vunpcklpd {{.*#+}} xmm3 = xmm3[0],xmm4[0]
 ; NODQ-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
 ; NODQ-NEXT:    vextracti128 $1, %ymm1, %xmm3
 ; NODQ-NEXT:    vpextrq $1, %xmm3, %rax
 ; NODQ-NEXT:    vcvtsi2sd %rax, %xmm5, %xmm4
 ; NODQ-NEXT:    vmovq %xmm3, %rax
 ; NODQ-NEXT:    vcvtsi2sd %rax, %xmm5, %xmm3
-; NODQ-NEXT:    vmovlhps {{.*#+}} xmm3 = xmm3[0],xmm4[0]
+; NODQ-NEXT:    vunpcklpd {{.*#+}} xmm3 = xmm3[0],xmm4[0]
 ; NODQ-NEXT:    vpextrq $1, %xmm1, %rax
 ; NODQ-NEXT:    vcvtsi2sd %rax, %xmm5, %xmm4
 ; NODQ-NEXT:    vmovq %xmm1, %rax
 ; NODQ-NEXT:    vcvtsi2sd %rax, %xmm5, %xmm1
-; NODQ-NEXT:    vmovlhps {{.*#+}} xmm1 = xmm1[0],xmm4[0]
+; NODQ-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm1[0],xmm4[0]
 ; NODQ-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
 ; NODQ-NEXT:    vinsertf64x4 $1, %ymm2, %zmm1, %zmm1
 ; NODQ-NEXT:    retq
@@ -1277,13 +1275,13 @@ define <16 x float> @ulto16f32(<16 x i64> %a) {
 ; NODQ-NEXT:    vcvtusi2ss %rax, %xmm5, %xmm1
 ; NODQ-NEXT:    vinsertps {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[0]
 ; NODQ-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
-; NODQ-NEXT:    vextracti32x4 $2, %zmm0, %xmm2
+; NODQ-NEXT:    vextractf32x4 $2, %zmm0, %xmm2
 ; NODQ-NEXT:    vpextrq $1, %xmm2, %rax
 ; NODQ-NEXT:    vcvtusi2ss %rax, %xmm5, %xmm3
 ; NODQ-NEXT:    vmovq %xmm2, %rax
 ; NODQ-NEXT:    vcvtusi2ss %rax, %xmm5, %xmm2
 ; NODQ-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[2,3]
-; NODQ-NEXT:    vextracti32x4 $3, %zmm0, %xmm3
+; NODQ-NEXT:    vextractf32x4 $3, %zmm0, %xmm3
 ; NODQ-NEXT:    vmovq %xmm3, %rax
 ; NODQ-NEXT:    vcvtusi2ss %rax, %xmm5, %xmm4
 ; NODQ-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1],xmm4[0],xmm2[3]
@@ -1557,9 +1555,7 @@ define <8 x double> @ssto16f64(<8 x i16> %a) {
 define <8 x double> @scto8f64(<8 x i8> %a) {
 ; ALL-LABEL: scto8f64:
 ; ALL:       # %bb.0:
-; ALL-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; ALL-NEXT:    vpslld $24, %ymm0, %ymm0
-; ALL-NEXT:    vpsrad $24, %ymm0, %ymm0
+; ALL-NEXT:    vpmovsxbd %xmm0, %ymm0
 ; ALL-NEXT:    vcvtdq2pd %ymm0, %zmm0
 ; ALL-NEXT:    retq
   %1 = sitofp <8 x i8> %a to <8 x double>
@@ -1724,13 +1720,30 @@ define <2 x float> @sbto2f32(<2 x float> %a) {
 }
 
 define <2 x double> @sbto2f64(<2 x double> %a) {
-; ALL-LABEL: sbto2f64:
-; ALL:       # %bb.0:
-; ALL-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
-; ALL-NEXT:    vcmpltpd %xmm0, %xmm1, %xmm0
-; ALL-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; ALL-NEXT:    vcvtdq2pd %xmm0, %xmm0
-; ALL-NEXT:    retq
+; NOVL-LABEL: sbto2f64:
+; NOVL:       # %bb.0:
+; NOVL-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; NOVL-NEXT:    vcmpltpd %xmm0, %xmm1, %xmm0
+; NOVL-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; NOVL-NEXT:    vcvtdq2pd %xmm0, %xmm0
+; NOVL-NEXT:    retq
+;
+; VLDQ-LABEL: sbto2f64:
+; VLDQ:       # %bb.0:
+; VLDQ-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; VLDQ-NEXT:    vcmpltpd %xmm0, %xmm1, %k0
+; VLDQ-NEXT:    vpmovm2d %k0, %xmm0
+; VLDQ-NEXT:    vcvtdq2pd %xmm0, %xmm0
+; VLDQ-NEXT:    retq
+;
+; VLNODQ-LABEL: sbto2f64:
+; VLNODQ:       # %bb.0:
+; VLNODQ-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; VLNODQ-NEXT:    vcmpltpd %xmm0, %xmm1, %k1
+; VLNODQ-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
+; VLNODQ-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
+; VLNODQ-NEXT:    vcvtdq2pd %xmm0, %xmm0
+; VLNODQ-NEXT:    retq
   %cmpres = fcmp ogt <2 x double> %a, zeroinitializer
   %1 = sitofp <2 x i1> %cmpres to <2 x double>
   ret <2 x double> %1
@@ -1749,8 +1762,7 @@ define <16 x float> @ucto16f32(<16 x i8> %a) {
 define <8 x double> @ucto8f64(<8 x i8> %a) {
 ; ALL-LABEL: ucto8f64:
 ; ALL:       # %bb.0:
-; ALL-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; ALL-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; ALL-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
 ; ALL-NEXT:    vcvtdq2pd %ymm0, %zmm0
 ; ALL-NEXT:    retq
   %b = uitofp <8 x i8> %a to <8 x double>
@@ -1993,29 +2005,42 @@ define <4 x double> @ubto4f64(<4 x i32> %a) {
 }
 
 define <2 x float> @ubto2f32(<2 x i32> %a) {
-; ALL-LABEL: ubto2f32:
-; ALL:       # %bb.0:
-; ALL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; ALL-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
-; ALL-NEXT:    vpcmpeqq %xmm1, %xmm0, %xmm0
-; ALL-NEXT:    vpandn {{.*}}(%rip), %xmm0, %xmm0
-; ALL-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; ALL-NEXT:    retq
+; NOVL-LABEL: ubto2f32:
+; NOVL:       # %bb.0:
+; NOVL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; NOVL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; NOVL-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [1065353216,1065353216,1065353216,1065353216]
+; NOVL-NEXT:    vpandn %xmm1, %xmm0, %xmm0
+; NOVL-NEXT:    retq
+;
+; VL-LABEL: ubto2f32:
+; VL:       # %bb.0:
+; VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; VL-NEXT:    vpandnd {{.*}}(%rip){1to4}, %xmm0, %xmm0
+; VL-NEXT:    retq
   %mask = icmp ne <2 x i32> %a, zeroinitializer
   %1 = uitofp <2 x i1> %mask to <2 x float>
   ret <2 x float> %1
 }
 
 define <2 x double> @ubto2f64(<2 x i32> %a) {
-; ALL-LABEL: ubto2f64:
-; ALL:       # %bb.0:
-; ALL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; ALL-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
-; ALL-NEXT:    vpcmpeqq %xmm1, %xmm0, %xmm0
-; ALL-NEXT:    vpandn {{.*}}(%rip), %xmm0, %xmm0
-; ALL-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; ALL-NEXT:    vcvtdq2pd %xmm0, %xmm0
-; ALL-NEXT:    retq
+; NOVL-LABEL: ubto2f64:
+; NOVL:       # %bb.0:
+; NOVL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; NOVL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; NOVL-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [1,1,1,1]
+; NOVL-NEXT:    vpandn %xmm1, %xmm0, %xmm0
+; NOVL-NEXT:    vcvtdq2pd %xmm0, %xmm0
+; NOVL-NEXT:    retq
+;
+; VL-LABEL: ubto2f64:
+; VL:       # %bb.0:
+; VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; VL-NEXT:    vpandnd {{.*}}(%rip){1to4}, %xmm0, %xmm0
+; VL-NEXT:    vcvtdq2pd %xmm0, %xmm0
+; VL-NEXT:    retq
   %mask = icmp ne <2 x i32> %a, zeroinitializer
   %1 = uitofp <2 x i1> %mask to <2 x double>
   ret <2 x double> %1
diff --git a/llvm/test/CodeGen/X86/avx512-ext.ll b/llvm/test/CodeGen/X86/avx512-ext.ll
index d2c0c7d66..349a7d63a 100644
--- a/llvm/test/CodeGen/X86/avx512-ext.ll
+++ b/llvm/test/CodeGen/X86/avx512-ext.ll
@@ -205,8 +205,8 @@ define <16 x i16> @sext_16x8_to_16x16_mask(<16 x i8> %a ,<16 x i1> %mask) nounwi
 define <32 x i16> @zext_32x8mem_to_32x16(<32 x i8> *%i , <32 x i1> %mask) nounwind readnone {
 ; KNL-LABEL: zext_32x8mem_to_32x16:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; KNL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; KNL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; KNL-NEXT:    vextracti128 $1, %ymm0, %xmm0
 ; KNL-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
 ; KNL-NEXT:    vpmovzxbw {{.*#+}} ymm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero
 ; KNL-NEXT:    vpmovzxbw {{.*#+}} ymm3 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero
@@ -216,6 +216,7 @@ define <32 x i16> @zext_32x8mem_to_32x16(<32 x i8> *%i , <32 x i1> %mask) nounwi
 ; KNL-NEXT:    vpsllw $15, %ymm1, %ymm1
 ; KNL-NEXT:    vpsraw $15, %ymm1, %ymm1
 ; KNL-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; KNL-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: zext_32x8mem_to_32x16:
@@ -227,8 +228,8 @@ define <32 x i16> @zext_32x8mem_to_32x16(<32 x i8> *%i , <32 x i1> %mask) nounwi
 ;
 ; AVX512DQNOBW-LABEL: zext_32x8mem_to_32x16:
 ; AVX512DQNOBW:       # %bb.0:
-; AVX512DQNOBW-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512DQNOBW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512DQNOBW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512DQNOBW-NEXT:    vextracti128 $1, %ymm0, %xmm0
 ; AVX512DQNOBW-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
 ; AVX512DQNOBW-NEXT:    vpmovzxbw {{.*#+}} ymm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero
 ; AVX512DQNOBW-NEXT:    vpmovzxbw {{.*#+}} ymm3 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero
@@ -238,6 +239,7 @@ define <32 x i16> @zext_32x8mem_to_32x16(<32 x i8> *%i , <32 x i1> %mask) nounwi
 ; AVX512DQNOBW-NEXT:    vpsllw $15, %ymm1, %ymm1
 ; AVX512DQNOBW-NEXT:    vpsraw $15, %ymm1, %ymm1
 ; AVX512DQNOBW-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX512DQNOBW-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; AVX512DQNOBW-NEXT:    retq
   %a   = load <32 x i8>,<32 x i8> *%i,align 1
   %x   = zext <32 x i8> %a to <32 x i16>
@@ -248,17 +250,18 @@ define <32 x i16> @zext_32x8mem_to_32x16(<32 x i8> *%i , <32 x i1> %mask) nounwi
 define <32 x i16> @sext_32x8mem_to_32x16(<32 x i8> *%i , <32 x i1> %mask) nounwind readnone {
 ; KNL-LABEL: sext_32x8mem_to_32x16:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; KNL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; KNL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; KNL-NEXT:    vextracti128 $1, %ymm0, %xmm0
 ; KNL-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
-; KNL-NEXT:    vpmovsxbw 16(%rdi), %ymm2
-; KNL-NEXT:    vpmovsxbw (%rdi), %ymm3
+; KNL-NEXT:    vpmovsxbw (%rdi), %ymm2
+; KNL-NEXT:    vpmovsxbw 16(%rdi), %ymm3
 ; KNL-NEXT:    vpsllw $15, %ymm0, %ymm0
 ; KNL-NEXT:    vpsraw $15, %ymm0, %ymm0
 ; KNL-NEXT:    vpand %ymm3, %ymm0, %ymm0
 ; KNL-NEXT:    vpsllw $15, %ymm1, %ymm1
 ; KNL-NEXT:    vpsraw $15, %ymm1, %ymm1
 ; KNL-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; KNL-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: sext_32x8mem_to_32x16:
@@ -270,17 +273,18 @@ define <32 x i16> @sext_32x8mem_to_32x16(<32 x i8> *%i , <32 x i1> %mask) nounwi
 ;
 ; AVX512DQNOBW-LABEL: sext_32x8mem_to_32x16:
 ; AVX512DQNOBW:       # %bb.0:
-; AVX512DQNOBW-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512DQNOBW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512DQNOBW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512DQNOBW-NEXT:    vextracti128 $1, %ymm0, %xmm0
 ; AVX512DQNOBW-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
-; AVX512DQNOBW-NEXT:    vpmovsxbw 16(%rdi), %ymm2
-; AVX512DQNOBW-NEXT:    vpmovsxbw (%rdi), %ymm3
+; AVX512DQNOBW-NEXT:    vpmovsxbw (%rdi), %ymm2
+; AVX512DQNOBW-NEXT:    vpmovsxbw 16(%rdi), %ymm3
 ; AVX512DQNOBW-NEXT:    vpsllw $15, %ymm0, %ymm0
 ; AVX512DQNOBW-NEXT:    vpsraw $15, %ymm0, %ymm0
 ; AVX512DQNOBW-NEXT:    vpand %ymm3, %ymm0, %ymm0
 ; AVX512DQNOBW-NEXT:    vpsllw $15, %ymm1, %ymm1
 ; AVX512DQNOBW-NEXT:    vpsraw $15, %ymm1, %ymm1
 ; AVX512DQNOBW-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX512DQNOBW-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; AVX512DQNOBW-NEXT:    retq
   %a   = load <32 x i8>,<32 x i8> *%i,align 1
   %x   = sext <32 x i8> %a to <32 x i16>
@@ -291,10 +295,10 @@ define <32 x i16> @sext_32x8mem_to_32x16(<32 x i8> *%i , <32 x i1> %mask) nounwi
 define <32 x i16> @zext_32x8_to_32x16(<32 x i8> %a ) nounwind readnone {
 ; KNL-LABEL: zext_32x8_to_32x16:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    vpmovzxbw {{.*#+}} ymm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
-; KNL-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; KNL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
-; KNL-NEXT:    vmovdqa %ymm2, %ymm0
+; KNL-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; KNL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; KNL-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; KNL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: zext_32x8_to_32x16:
@@ -304,10 +308,10 @@ define <32 x i16> @zext_32x8_to_32x16(<32 x i8> %a ) nounwind readnone {
 ;
 ; AVX512DQNOBW-LABEL: zext_32x8_to_32x16:
 ; AVX512DQNOBW:       # %bb.0:
-; AVX512DQNOBW-NEXT:    vpmovzxbw {{.*#+}} ymm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
-; AVX512DQNOBW-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; AVX512DQNOBW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
-; AVX512DQNOBW-NEXT:    vmovdqa %ymm2, %ymm0
+; AVX512DQNOBW-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512DQNOBW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512DQNOBW-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512DQNOBW-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512DQNOBW-NEXT:    retq
   %x   = zext <32 x i8> %a to <32 x i16>
   ret <32 x i16> %x
@@ -316,11 +320,11 @@ define <32 x i16> @zext_32x8_to_32x16(<32 x i8> %a ) nounwind readnone {
 define <32 x i16> @zext_32x8_to_32x16_mask(<32 x i8> %a ,<32 x i1> %mask) nounwind readnone {
 ; KNL-LABEL: zext_32x8_to_32x16_mask:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; KNL-NEXT:    vpmovzxbw {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero,xmm2[8],zero,xmm2[9],zero,xmm2[10],zero,xmm2[11],zero,xmm2[12],zero,xmm2[13],zero,xmm2[14],zero,xmm2[15],zero
+; KNL-NEXT:    vpmovzxbw {{.*#+}} ymm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; KNL-NEXT:    vextracti128 $1, %ymm1, %xmm1
 ; KNL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
-; KNL-NEXT:    vextracti128 $1, %ymm0, %xmm3
-; KNL-NEXT:    vpmovzxbw {{.*#+}} ymm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero,xmm3[8],zero,xmm3[9],zero,xmm3[10],zero,xmm3[11],zero,xmm3[12],zero,xmm3[13],zero,xmm3[14],zero,xmm3[15],zero
+; KNL-NEXT:    vpmovzxbw {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; KNL-NEXT:    vextracti128 $1, %ymm0, %xmm0
 ; KNL-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
 ; KNL-NEXT:    vpsllw $15, %ymm1, %ymm1
 ; KNL-NEXT:    vpsraw $15, %ymm1, %ymm1
@@ -328,6 +332,7 @@ define <32 x i16> @zext_32x8_to_32x16_mask(<32 x i8> %a ,<32 x i1> %mask) nounwi
 ; KNL-NEXT:    vpsllw $15, %ymm2, %ymm1
 ; KNL-NEXT:    vpsraw $15, %ymm1, %ymm1
 ; KNL-NEXT:    vpand %ymm3, %ymm1, %ymm1
+; KNL-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: zext_32x8_to_32x16_mask:
@@ -339,11 +344,11 @@ define <32 x i16> @zext_32x8_to_32x16_mask(<32 x i8> %a ,<32 x i1> %mask) nounwi
 ;
 ; AVX512DQNOBW-LABEL: zext_32x8_to_32x16_mask:
 ; AVX512DQNOBW:       # %bb.0:
-; AVX512DQNOBW-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX512DQNOBW-NEXT:    vpmovzxbw {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero,xmm2[8],zero,xmm2[9],zero,xmm2[10],zero,xmm2[11],zero,xmm2[12],zero,xmm2[13],zero,xmm2[14],zero,xmm2[15],zero
+; AVX512DQNOBW-NEXT:    vpmovzxbw {{.*#+}} ymm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512DQNOBW-NEXT:    vextracti128 $1, %ymm1, %xmm1
 ; AVX512DQNOBW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
-; AVX512DQNOBW-NEXT:    vextracti128 $1, %ymm0, %xmm3
-; AVX512DQNOBW-NEXT:    vpmovzxbw {{.*#+}} ymm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero,xmm3[8],zero,xmm3[9],zero,xmm3[10],zero,xmm3[11],zero,xmm3[12],zero,xmm3[13],zero,xmm3[14],zero,xmm3[15],zero
+; AVX512DQNOBW-NEXT:    vpmovzxbw {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512DQNOBW-NEXT:    vextracti128 $1, %ymm0, %xmm0
 ; AVX512DQNOBW-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
 ; AVX512DQNOBW-NEXT:    vpsllw $15, %ymm1, %ymm1
 ; AVX512DQNOBW-NEXT:    vpsraw $15, %ymm1, %ymm1
@@ -351,6 +356,7 @@ define <32 x i16> @zext_32x8_to_32x16_mask(<32 x i8> %a ,<32 x i1> %mask) nounwi
 ; AVX512DQNOBW-NEXT:    vpsllw $15, %ymm2, %ymm1
 ; AVX512DQNOBW-NEXT:    vpsraw $15, %ymm1, %ymm1
 ; AVX512DQNOBW-NEXT:    vpand %ymm3, %ymm1, %ymm1
+; AVX512DQNOBW-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; AVX512DQNOBW-NEXT:    retq
   %x   = zext <32 x i8> %a to <32 x i16>
   %ret = select <32 x i1> %mask, <32 x i16> %x, <32 x i16> zeroinitializer
@@ -360,10 +366,10 @@ define <32 x i16> @zext_32x8_to_32x16_mask(<32 x i8> %a ,<32 x i1> %mask) nounwi
 define <32 x i16> @sext_32x8_to_32x16(<32 x i8> %a ) nounwind readnone {
 ; KNL-LABEL: sext_32x8_to_32x16:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    vpmovsxbw %xmm0, %ymm2
-; KNL-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; KNL-NEXT:    vpmovsxbw %xmm0, %ymm1
-; KNL-NEXT:    vmovdqa %ymm2, %ymm0
+; KNL-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; KNL-NEXT:    vpmovsxbw %xmm1, %ymm1
+; KNL-NEXT:    vpmovsxbw %xmm0, %ymm0
+; KNL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: sext_32x8_to_32x16:
@@ -373,10 +379,10 @@ define <32 x i16> @sext_32x8_to_32x16(<32 x i8> %a ) nounwind readnone {
 ;
 ; AVX512DQNOBW-LABEL: sext_32x8_to_32x16:
 ; AVX512DQNOBW:       # %bb.0:
-; AVX512DQNOBW-NEXT:    vpmovsxbw %xmm0, %ymm2
-; AVX512DQNOBW-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; AVX512DQNOBW-NEXT:    vpmovsxbw %xmm0, %ymm1
-; AVX512DQNOBW-NEXT:    vmovdqa %ymm2, %ymm0
+; AVX512DQNOBW-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512DQNOBW-NEXT:    vpmovsxbw %xmm1, %ymm1
+; AVX512DQNOBW-NEXT:    vpmovsxbw %xmm0, %ymm0
+; AVX512DQNOBW-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512DQNOBW-NEXT:    retq
   %x   = sext <32 x i8> %a to <32 x i16>
   ret <32 x i16> %x
@@ -385,11 +391,11 @@ define <32 x i16> @sext_32x8_to_32x16(<32 x i8> %a ) nounwind readnone {
 define <32 x i16> @sext_32x8_to_32x16_mask(<32 x i8> %a ,<32 x i1> %mask) nounwind readnone {
 ; KNL-LABEL: sext_32x8_to_32x16_mask:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; KNL-NEXT:    vpmovzxbw {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero,xmm2[8],zero,xmm2[9],zero,xmm2[10],zero,xmm2[11],zero,xmm2[12],zero,xmm2[13],zero,xmm2[14],zero,xmm2[15],zero
+; KNL-NEXT:    vpmovzxbw {{.*#+}} ymm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; KNL-NEXT:    vextracti128 $1, %ymm1, %xmm1
 ; KNL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
-; KNL-NEXT:    vextracti128 $1, %ymm0, %xmm3
-; KNL-NEXT:    vpmovsxbw %xmm3, %ymm3
+; KNL-NEXT:    vpmovsxbw %xmm0, %ymm3
+; KNL-NEXT:    vextracti128 $1, %ymm0, %xmm0
 ; KNL-NEXT:    vpmovsxbw %xmm0, %ymm0
 ; KNL-NEXT:    vpsllw $15, %ymm1, %ymm1
 ; KNL-NEXT:    vpsraw $15, %ymm1, %ymm1
@@ -397,6 +403,7 @@ define <32 x i16> @sext_32x8_to_32x16_mask(<32 x i8> %a ,<32 x i1> %mask) nounwi
 ; KNL-NEXT:    vpsllw $15, %ymm2, %ymm1
 ; KNL-NEXT:    vpsraw $15, %ymm1, %ymm1
 ; KNL-NEXT:    vpand %ymm3, %ymm1, %ymm1
+; KNL-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: sext_32x8_to_32x16_mask:
@@ -408,11 +415,11 @@ define <32 x i16> @sext_32x8_to_32x16_mask(<32 x i8> %a ,<32 x i1> %mask) nounwi
 ;
 ; AVX512DQNOBW-LABEL: sext_32x8_to_32x16_mask:
 ; AVX512DQNOBW:       # %bb.0:
-; AVX512DQNOBW-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX512DQNOBW-NEXT:    vpmovzxbw {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero,xmm2[8],zero,xmm2[9],zero,xmm2[10],zero,xmm2[11],zero,xmm2[12],zero,xmm2[13],zero,xmm2[14],zero,xmm2[15],zero
+; AVX512DQNOBW-NEXT:    vpmovzxbw {{.*#+}} ymm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512DQNOBW-NEXT:    vextracti128 $1, %ymm1, %xmm1
 ; AVX512DQNOBW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
-; AVX512DQNOBW-NEXT:    vextracti128 $1, %ymm0, %xmm3
-; AVX512DQNOBW-NEXT:    vpmovsxbw %xmm3, %ymm3
+; AVX512DQNOBW-NEXT:    vpmovsxbw %xmm0, %ymm3
+; AVX512DQNOBW-NEXT:    vextracti128 $1, %ymm0, %xmm0
 ; AVX512DQNOBW-NEXT:    vpmovsxbw %xmm0, %ymm0
 ; AVX512DQNOBW-NEXT:    vpsllw $15, %ymm1, %ymm1
 ; AVX512DQNOBW-NEXT:    vpsraw $15, %ymm1, %ymm1
@@ -420,6 +427,7 @@ define <32 x i16> @sext_32x8_to_32x16_mask(<32 x i8> %a ,<32 x i1> %mask) nounwi
 ; AVX512DQNOBW-NEXT:    vpsllw $15, %ymm2, %ymm1
 ; AVX512DQNOBW-NEXT:    vpsraw $15, %ymm1, %ymm1
 ; AVX512DQNOBW-NEXT:    vpand %ymm3, %ymm1, %ymm1
+; AVX512DQNOBW-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; AVX512DQNOBW-NEXT:    retq
   %x   = sext <32 x i8> %a to <32 x i16>
   %ret = select <32 x i1> %mask, <32 x i16> %x, <32 x i16> zeroinitializer
@@ -1878,22 +1886,453 @@ define void @extload_v8i64(<8 x i8>* %a, <8 x i64>* %res) {
 define <64 x i16> @test21(<64 x i16> %x , <64 x i1> %mask) nounwind readnone {
 ; KNL-LABEL: test21:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    vpmovzxbw {{.*#+}} ymm7 = xmm7[0],zero,xmm7[1],zero,xmm7[2],zero,xmm7[3],zero,xmm7[4],zero,xmm7[5],zero,xmm7[6],zero,xmm7[7],zero,xmm7[8],zero,xmm7[9],zero,xmm7[10],zero,xmm7[11],zero,xmm7[12],zero,xmm7[13],zero,xmm7[14],zero,xmm7[15],zero
-; KNL-NEXT:    vpmovzxbw {{.*#+}} ymm6 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero,xmm6[4],zero,xmm6[5],zero,xmm6[6],zero,xmm6[7],zero,xmm6[8],zero,xmm6[9],zero,xmm6[10],zero,xmm6[11],zero,xmm6[12],zero,xmm6[13],zero,xmm6[14],zero,xmm6[15],zero
-; KNL-NEXT:    vpmovzxbw {{.*#+}} ymm5 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero,xmm5[8],zero,xmm5[9],zero,xmm5[10],zero,xmm5[11],zero,xmm5[12],zero,xmm5[13],zero,xmm5[14],zero,xmm5[15],zero
-; KNL-NEXT:    vpmovzxbw {{.*#+}} ymm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero,xmm4[8],zero,xmm4[9],zero,xmm4[10],zero,xmm4[11],zero,xmm4[12],zero,xmm4[13],zero,xmm4[14],zero,xmm4[15],zero
-; KNL-NEXT:    vpsllw $15, %ymm4, %ymm4
-; KNL-NEXT:    vpsraw $15, %ymm4, %ymm4
-; KNL-NEXT:    vpand %ymm0, %ymm4, %ymm0
-; KNL-NEXT:    vpsllw $15, %ymm5, %ymm4
-; KNL-NEXT:    vpsraw $15, %ymm4, %ymm4
+; KNL-NEXT:    movw $-3, %ax
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kmovw %edi, %k0
+; KNL-NEXT:    kandw %k1, %k0, %k0
+; KNL-NEXT:    kmovw %k1, %k2
+; KNL-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; KNL-NEXT:    kmovw %esi, %k1
+; KNL-NEXT:    kshiftlw $15, %k1, %k1
+; KNL-NEXT:    kshiftrw $14, %k1, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    movw $-5, %ax
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; KNL-NEXT:    kandw %k1, %k0, %k0
+; KNL-NEXT:    kmovw %edx, %k1
+; KNL-NEXT:    kshiftlw $15, %k1, %k1
+; KNL-NEXT:    kshiftrw $13, %k1, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    movw $-9, %ax
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; KNL-NEXT:    kandw %k1, %k0, %k0
+; KNL-NEXT:    kmovw %ecx, %k1
+; KNL-NEXT:    kshiftlw $15, %k1, %k1
+; KNL-NEXT:    kshiftrw $12, %k1, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    movw $-17, %ax
+; KNL-NEXT:    kmovw %eax, %k6
+; KNL-NEXT:    kandw %k6, %k0, %k0
+; KNL-NEXT:    kmovw %k6, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; KNL-NEXT:    kmovw %r8d, %k1
+; KNL-NEXT:    kshiftlw $15, %k1, %k1
+; KNL-NEXT:    kshiftrw $11, %k1, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    movw $-33, %ax
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kandw %k1, %k0, %k0
+; KNL-NEXT:    kmovw %k1, %k3
+; KNL-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; KNL-NEXT:    kmovw %r9d, %k1
+; KNL-NEXT:    kshiftlw $15, %k1, %k1
+; KNL-NEXT:    kshiftrw $10, %k1, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    movw $-65, %ax
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; KNL-NEXT:    kandw %k1, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kshiftlw $15, %k1, %k1
+; KNL-NEXT:    kshiftrw $9, %k1, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    movw $-129, %ax
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kandw %k1, %k0, %k0
+; KNL-NEXT:    kmovw %k1, %k4
+; KNL-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kshiftlw $15, %k1, %k1
+; KNL-NEXT:    kshiftrw $8, %k1, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    movw $-257, %ax # imm = 0xFEFF
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; KNL-NEXT:    kandw %k1, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kshiftlw $15, %k1, %k1
+; KNL-NEXT:    kshiftrw $7, %k1, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    movw $-513, %ax # imm = 0xFDFF
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kandw %k1, %k0, %k0
+; KNL-NEXT:    kmovw %k1, %k5
+; KNL-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kshiftlw $15, %k1, %k1
+; KNL-NEXT:    kshiftrw $6, %k1, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    movw $-1025, %ax # imm = 0xFBFF
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; KNL-NEXT:    kandw %k1, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kshiftlw $15, %k1, %k1
+; KNL-NEXT:    kshiftrw $5, %k1, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    movw $-2049, %ax # imm = 0xF7FF
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; KNL-NEXT:    kandw %k1, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kshiftlw $15, %k1, %k1
+; KNL-NEXT:    kshiftrw $4, %k1, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    movw $-4097, %ax # imm = 0xEFFF
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; KNL-NEXT:    kandw %k1, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kshiftlw $15, %k1, %k1
+; KNL-NEXT:    kshiftrw $3, %k1, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    movw $-8193, %ax # imm = 0xDFFF
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; KNL-NEXT:    kandw %k1, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kshiftlw $15, %k1, %k1
+; KNL-NEXT:    kshiftrw $2, %k1, %k1
+; KNL-NEXT:    korw %k1, %k0, %k1
+; KNL-NEXT:    movw $-16385, %ax # imm = 0xBFFF
+; KNL-NEXT:    kmovw %eax, %k0
+; KNL-NEXT:    kmovw %k0, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; KNL-NEXT:    kandw %k0, %k1, %k1
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $14, %k7, %k7
+; KNL-NEXT:    korw %k7, %k1, %k1
+; KNL-NEXT:    kshiftlw $1, %k1, %k1
+; KNL-NEXT:    kshiftrw $1, %k1, %k1
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    korw %k7, %k1, %k1
+; KNL-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kandw %k2, %k1, %k1
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $14, %k7, %k7
+; KNL-NEXT:    korw %k7, %k1, %k1
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 # 2-byte Reload
+; KNL-NEXT:    kandw %k0, %k1, %k1
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $13, %k7, %k7
+; KNL-NEXT:    korw %k7, %k1, %k1
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload
+; KNL-NEXT:    kandw %k2, %k1, %k1
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $12, %k7, %k7
+; KNL-NEXT:    korw %k7, %k1, %k1
+; KNL-NEXT:    kandw %k6, %k1, %k1
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $11, %k7, %k7
+; KNL-NEXT:    korw %k7, %k1, %k1
+; KNL-NEXT:    kandw %k3, %k1, %k1
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $10, %k7, %k7
+; KNL-NEXT:    korw %k7, %k1, %k1
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k6 # 2-byte Reload
+; KNL-NEXT:    kandw %k6, %k1, %k1
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $9, %k7, %k7
+; KNL-NEXT:    korw %k7, %k1, %k1
+; KNL-NEXT:    kandw %k4, %k1, %k1
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $8, %k7, %k7
+; KNL-NEXT:    korw %k7, %k1, %k1
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k3 # 2-byte Reload
+; KNL-NEXT:    kandw %k3, %k1, %k1
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $7, %k7, %k7
+; KNL-NEXT:    korw %k7, %k1, %k1
+; KNL-NEXT:    kandw %k5, %k1, %k1
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $6, %k7, %k7
+; KNL-NEXT:    korw %k7, %k1, %k1
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k4 # 2-byte Reload
+; KNL-NEXT:    kandw %k4, %k1, %k1
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $5, %k7, %k7
+; KNL-NEXT:    korw %k7, %k1, %k1
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k5 # 2-byte Reload
+; KNL-NEXT:    kandw %k5, %k1, %k1
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $4, %k7, %k7
+; KNL-NEXT:    korw %k7, %k1, %k1
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 # 2-byte Reload
+; KNL-NEXT:    kandw %k7, %k1, %k1
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $3, %k7, %k7
+; KNL-NEXT:    korw %k7, %k1, %k1
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 # 2-byte Reload
+; KNL-NEXT:    kandw %k7, %k1, %k1
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $2, %k7, %k7
+; KNL-NEXT:    korw %k7, %k1, %k1
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 # 2-byte Reload
+; KNL-NEXT:    kandw %k7, %k1, %k1
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $14, %k7, %k7
+; KNL-NEXT:    korw %k7, %k1, %k1
+; KNL-NEXT:    kshiftlw $1, %k1, %k1
+; KNL-NEXT:    kshiftrw $1, %k1, %k1
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    korw %k7, %k1, %k1
+; KNL-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 # 2-byte Reload
+; KNL-NEXT:    kandw %k7, %k1, %k1
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $14, %k7, %k7
+; KNL-NEXT:    korw %k7, %k1, %k1
+; KNL-NEXT:    kandw %k0, %k1, %k1
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $13, %k7, %k7
+; KNL-NEXT:    korw %k7, %k1, %k1
+; KNL-NEXT:    kandw %k2, %k1, %k1
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $12, %k7, %k7
+; KNL-NEXT:    korw %k7, %k1, %k1
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 # 2-byte Reload
+; KNL-NEXT:    kandw %k0, %k1, %k1
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $11, %k7, %k7
+; KNL-NEXT:    korw %k7, %k1, %k1
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload
+; KNL-NEXT:    kandw %k2, %k1, %k1
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $10, %k7, %k7
+; KNL-NEXT:    korw %k7, %k1, %k1
+; KNL-NEXT:    kandw %k6, %k1, %k1
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $9, %k7, %k7
+; KNL-NEXT:    korw %k7, %k1, %k1
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload
+; KNL-NEXT:    kandw %k2, %k1, %k1
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $8, %k7, %k7
+; KNL-NEXT:    korw %k7, %k1, %k1
+; KNL-NEXT:    kandw %k3, %k1, %k1
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $7, %k7, %k7
+; KNL-NEXT:    korw %k7, %k1, %k1
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k3 # 2-byte Reload
+; KNL-NEXT:    kandw %k3, %k1, %k1
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $6, %k7, %k7
+; KNL-NEXT:    korw %k7, %k1, %k1
+; KNL-NEXT:    kandw %k4, %k1, %k1
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $5, %k7, %k7
+; KNL-NEXT:    korw %k7, %k1, %k1
+; KNL-NEXT:    kandw %k5, %k1, %k1
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $4, %k7, %k7
+; KNL-NEXT:    korw %k7, %k1, %k1
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload
+; KNL-NEXT:    kandw %k2, %k1, %k1
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $3, %k7, %k7
+; KNL-NEXT:    korw %k7, %k1, %k1
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k5 # 2-byte Reload
+; KNL-NEXT:    kandw %k5, %k1, %k1
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $2, %k7, %k7
+; KNL-NEXT:    korw %k7, %k1, %k1
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k5 # 2-byte Reload
+; KNL-NEXT:    kandw %k5, %k1, %k1
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $14, %k7, %k7
+; KNL-NEXT:    korw %k7, %k1, %k1
+; KNL-NEXT:    kshiftlw $1, %k1, %k1
+; KNL-NEXT:    kshiftrw $1, %k1, %k1
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    korw %k7, %k1, %k1
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k5 # 2-byte Reload
+; KNL-NEXT:    kandw %k5, %k7, %k7
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k6
+; KNL-NEXT:    kshiftlw $15, %k6, %k6
+; KNL-NEXT:    kshiftrw $14, %k6, %k6
+; KNL-NEXT:    korw %k6, %k7, %k6
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k5 # 2-byte Reload
+; KNL-NEXT:    kandw %k5, %k6, %k6
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $13, %k7, %k7
+; KNL-NEXT:    korw %k7, %k6, %k6
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k5 # 2-byte Reload
+; KNL-NEXT:    kandw %k5, %k6, %k6
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $12, %k7, %k7
+; KNL-NEXT:    korw %k7, %k6, %k6
+; KNL-NEXT:    kandw %k0, %k6, %k6
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $11, %k7, %k7
+; KNL-NEXT:    korw %k7, %k6, %k6
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 # 2-byte Reload
+; KNL-NEXT:    kandw %k0, %k6, %k6
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $10, %k7, %k7
+; KNL-NEXT:    korw %k7, %k6, %k6
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 # 2-byte Reload
+; KNL-NEXT:    kandw %k0, %k6, %k6
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $9, %k7, %k7
+; KNL-NEXT:    korw %k7, %k6, %k6
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 # 2-byte Reload
+; KNL-NEXT:    kandw %k0, %k6, %k6
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $8, %k7, %k7
+; KNL-NEXT:    korw %k7, %k6, %k6
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 # 2-byte Reload
+; KNL-NEXT:    kandw %k0, %k6, %k6
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $7, %k7, %k7
+; KNL-NEXT:    korw %k7, %k6, %k6
+; KNL-NEXT:    kandw %k3, %k6, %k6
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $6, %k7, %k7
+; KNL-NEXT:    korw %k7, %k6, %k6
+; KNL-NEXT:    kandw %k4, %k6, %k5
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k6
+; KNL-NEXT:    kshiftlw $15, %k6, %k6
+; KNL-NEXT:    kshiftrw $5, %k6, %k6
+; KNL-NEXT:    korw %k6, %k5, %k5
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 # 2-byte Reload
+; KNL-NEXT:    kandw %k0, %k5, %k4
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k5
+; KNL-NEXT:    kshiftlw $15, %k5, %k5
+; KNL-NEXT:    kshiftrw $4, %k5, %k5
+; KNL-NEXT:    korw %k5, %k4, %k4
+; KNL-NEXT:    kandw %k2, %k4, %k3
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k4
+; KNL-NEXT:    kshiftlw $15, %k4, %k4
+; KNL-NEXT:    kshiftrw $3, %k4, %k4
+; KNL-NEXT:    korw %k4, %k3, %k3
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 # 2-byte Reload
+; KNL-NEXT:    kandw %k0, %k3, %k2
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k3
+; KNL-NEXT:    kshiftlw $15, %k3, %k3
+; KNL-NEXT:    kshiftrw $2, %k3, %k3
+; KNL-NEXT:    korw %k3, %k2, %k2
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 # 2-byte Reload
+; KNL-NEXT:    kandw %k0, %k2, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k2
+; KNL-NEXT:    kshiftlw $14, %k2, %k2
+; KNL-NEXT:    korw %k2, %k0, %k0
+; KNL-NEXT:    kshiftlw $1, %k0, %k0
+; KNL-NEXT:    kshiftrw $1, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k2
+; KNL-NEXT:    kshiftlw $15, %k2, %k2
+; KNL-NEXT:    korw %k2, %k0, %k2
+; KNL-NEXT:    vpternlogd $255, %zmm4, %zmm4, %zmm4 {%k2} {z}
+; KNL-NEXT:    vpternlogd $255, %zmm5, %zmm5, %zmm5 {%k1} {z}
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload
+; KNL-NEXT:    vpternlogd $255, %zmm6, %zmm6, %zmm6 {%k1} {z}
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload
+; KNL-NEXT:    vpternlogd $255, %zmm7, %zmm7, %zmm7 {%k1} {z}
+; KNL-NEXT:    vpmovdw %zmm4, %ymm4
 ; KNL-NEXT:    vpand %ymm1, %ymm4, %ymm1
-; KNL-NEXT:    vpsllw $15, %ymm6, %ymm4
-; KNL-NEXT:    vpsraw $15, %ymm4, %ymm4
+; KNL-NEXT:    vpmovdw %zmm5, %ymm4
 ; KNL-NEXT:    vpand %ymm2, %ymm4, %ymm2
-; KNL-NEXT:    vpsllw $15, %ymm7, %ymm4
-; KNL-NEXT:    vpsraw $15, %ymm4, %ymm4
+; KNL-NEXT:    vpmovdw %zmm6, %ymm4
 ; KNL-NEXT:    vpand %ymm3, %ymm4, %ymm3
+; KNL-NEXT:    vpmovdw %zmm7, %ymm4
+; KNL-NEXT:    vpand %ymm0, %ymm4, %ymm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: test21:
@@ -1907,22 +2346,453 @@ define <64 x i16> @test21(<64 x i16> %x , <64 x i1> %mask) nounwind readnone {
 ;
 ; AVX512DQNOBW-LABEL: test21:
 ; AVX512DQNOBW:       # %bb.0:
-; AVX512DQNOBW-NEXT:    vpmovzxbw {{.*#+}} ymm7 = xmm7[0],zero,xmm7[1],zero,xmm7[2],zero,xmm7[3],zero,xmm7[4],zero,xmm7[5],zero,xmm7[6],zero,xmm7[7],zero,xmm7[8],zero,xmm7[9],zero,xmm7[10],zero,xmm7[11],zero,xmm7[12],zero,xmm7[13],zero,xmm7[14],zero,xmm7[15],zero
-; AVX512DQNOBW-NEXT:    vpmovzxbw {{.*#+}} ymm6 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero,xmm6[4],zero,xmm6[5],zero,xmm6[6],zero,xmm6[7],zero,xmm6[8],zero,xmm6[9],zero,xmm6[10],zero,xmm6[11],zero,xmm6[12],zero,xmm6[13],zero,xmm6[14],zero,xmm6[15],zero
-; AVX512DQNOBW-NEXT:    vpmovzxbw {{.*#+}} ymm5 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero,xmm5[8],zero,xmm5[9],zero,xmm5[10],zero,xmm5[11],zero,xmm5[12],zero,xmm5[13],zero,xmm5[14],zero,xmm5[15],zero
-; AVX512DQNOBW-NEXT:    vpmovzxbw {{.*#+}} ymm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero,xmm4[8],zero,xmm4[9],zero,xmm4[10],zero,xmm4[11],zero,xmm4[12],zero,xmm4[13],zero,xmm4[14],zero,xmm4[15],zero
-; AVX512DQNOBW-NEXT:    vpsllw $15, %ymm4, %ymm4
-; AVX512DQNOBW-NEXT:    vpsraw $15, %ymm4, %ymm4
-; AVX512DQNOBW-NEXT:    vpand %ymm0, %ymm4, %ymm0
-; AVX512DQNOBW-NEXT:    vpsllw $15, %ymm5, %ymm4
-; AVX512DQNOBW-NEXT:    vpsraw $15, %ymm4, %ymm4
+; AVX512DQNOBW-NEXT:    movw $-3, %ax
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k1
+; AVX512DQNOBW-NEXT:    kmovw %edi, %k0
+; AVX512DQNOBW-NEXT:    kandw %k1, %k0, %k0
+; AVX512DQNOBW-NEXT:    kmovw %k1, %k2
+; AVX512DQNOBW-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; AVX512DQNOBW-NEXT:    kmovw %esi, %k1
+; AVX512DQNOBW-NEXT:    kshiftlw $15, %k1, %k1
+; AVX512DQNOBW-NEXT:    kshiftrw $14, %k1, %k1
+; AVX512DQNOBW-NEXT:    korw %k1, %k0, %k0
+; AVX512DQNOBW-NEXT:    movw $-5, %ax
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k1
+; AVX512DQNOBW-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; AVX512DQNOBW-NEXT:    kandw %k1, %k0, %k0
+; AVX512DQNOBW-NEXT:    kmovw %edx, %k1
+; AVX512DQNOBW-NEXT:    kshiftlw $15, %k1, %k1
+; AVX512DQNOBW-NEXT:    kshiftrw $13, %k1, %k1
+; AVX512DQNOBW-NEXT:    korw %k1, %k0, %k0
+; AVX512DQNOBW-NEXT:    movw $-9, %ax
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k1
+; AVX512DQNOBW-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; AVX512DQNOBW-NEXT:    kandw %k1, %k0, %k0
+; AVX512DQNOBW-NEXT:    kmovw %ecx, %k1
+; AVX512DQNOBW-NEXT:    kshiftlw $15, %k1, %k1
+; AVX512DQNOBW-NEXT:    kshiftrw $12, %k1, %k1
+; AVX512DQNOBW-NEXT:    korw %k1, %k0, %k0
+; AVX512DQNOBW-NEXT:    movw $-17, %ax
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k6
+; AVX512DQNOBW-NEXT:    kandw %k6, %k0, %k0
+; AVX512DQNOBW-NEXT:    kmovw %k6, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; AVX512DQNOBW-NEXT:    kmovw %r8d, %k1
+; AVX512DQNOBW-NEXT:    kshiftlw $15, %k1, %k1
+; AVX512DQNOBW-NEXT:    kshiftrw $11, %k1, %k1
+; AVX512DQNOBW-NEXT:    korw %k1, %k0, %k0
+; AVX512DQNOBW-NEXT:    movw $-33, %ax
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k1
+; AVX512DQNOBW-NEXT:    kandw %k1, %k0, %k0
+; AVX512DQNOBW-NEXT:    kmovw %k1, %k3
+; AVX512DQNOBW-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; AVX512DQNOBW-NEXT:    kmovw %r9d, %k1
+; AVX512DQNOBW-NEXT:    kshiftlw $15, %k1, %k1
+; AVX512DQNOBW-NEXT:    kshiftrw $10, %k1, %k1
+; AVX512DQNOBW-NEXT:    korw %k1, %k0, %k0
+; AVX512DQNOBW-NEXT:    movw $-65, %ax
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k1
+; AVX512DQNOBW-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; AVX512DQNOBW-NEXT:    kandw %k1, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k1
+; AVX512DQNOBW-NEXT:    kshiftlw $15, %k1, %k1
+; AVX512DQNOBW-NEXT:    kshiftrw $9, %k1, %k1
+; AVX512DQNOBW-NEXT:    korw %k1, %k0, %k0
+; AVX512DQNOBW-NEXT:    movw $-129, %ax
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k1
+; AVX512DQNOBW-NEXT:    kandw %k1, %k0, %k0
+; AVX512DQNOBW-NEXT:    kmovw %k1, %k4
+; AVX512DQNOBW-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k1
+; AVX512DQNOBW-NEXT:    kshiftlw $15, %k1, %k1
+; AVX512DQNOBW-NEXT:    kshiftrw $8, %k1, %k1
+; AVX512DQNOBW-NEXT:    korw %k1, %k0, %k0
+; AVX512DQNOBW-NEXT:    movw $-257, %ax # imm = 0xFEFF
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k1
+; AVX512DQNOBW-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; AVX512DQNOBW-NEXT:    kandw %k1, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k1
+; AVX512DQNOBW-NEXT:    kshiftlw $15, %k1, %k1
+; AVX512DQNOBW-NEXT:    kshiftrw $7, %k1, %k1
+; AVX512DQNOBW-NEXT:    korw %k1, %k0, %k0
+; AVX512DQNOBW-NEXT:    movw $-513, %ax # imm = 0xFDFF
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k1
+; AVX512DQNOBW-NEXT:    kandw %k1, %k0, %k0
+; AVX512DQNOBW-NEXT:    kmovw %k1, %k5
+; AVX512DQNOBW-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k1
+; AVX512DQNOBW-NEXT:    kshiftlw $15, %k1, %k1
+; AVX512DQNOBW-NEXT:    kshiftrw $6, %k1, %k1
+; AVX512DQNOBW-NEXT:    korw %k1, %k0, %k0
+; AVX512DQNOBW-NEXT:    movw $-1025, %ax # imm = 0xFBFF
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k1
+; AVX512DQNOBW-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; AVX512DQNOBW-NEXT:    kandw %k1, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k1
+; AVX512DQNOBW-NEXT:    kshiftlw $15, %k1, %k1
+; AVX512DQNOBW-NEXT:    kshiftrw $5, %k1, %k1
+; AVX512DQNOBW-NEXT:    korw %k1, %k0, %k0
+; AVX512DQNOBW-NEXT:    movw $-2049, %ax # imm = 0xF7FF
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k1
+; AVX512DQNOBW-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; AVX512DQNOBW-NEXT:    kandw %k1, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k1
+; AVX512DQNOBW-NEXT:    kshiftlw $15, %k1, %k1
+; AVX512DQNOBW-NEXT:    kshiftrw $4, %k1, %k1
+; AVX512DQNOBW-NEXT:    korw %k1, %k0, %k0
+; AVX512DQNOBW-NEXT:    movw $-4097, %ax # imm = 0xEFFF
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k1
+; AVX512DQNOBW-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; AVX512DQNOBW-NEXT:    kandw %k1, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k1
+; AVX512DQNOBW-NEXT:    kshiftlw $15, %k1, %k1
+; AVX512DQNOBW-NEXT:    kshiftrw $3, %k1, %k1
+; AVX512DQNOBW-NEXT:    korw %k1, %k0, %k0
+; AVX512DQNOBW-NEXT:    movw $-8193, %ax # imm = 0xDFFF
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k1
+; AVX512DQNOBW-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; AVX512DQNOBW-NEXT:    kandw %k1, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k1
+; AVX512DQNOBW-NEXT:    kshiftlw $15, %k1, %k1
+; AVX512DQNOBW-NEXT:    kshiftrw $2, %k1, %k1
+; AVX512DQNOBW-NEXT:    korw %k1, %k0, %k0
+; AVX512DQNOBW-NEXT:    movw $-16385, %ax # imm = 0xBFFF
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k1
+; AVX512DQNOBW-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; AVX512DQNOBW-NEXT:    kandw %k1, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k7
+; AVX512DQNOBW-NEXT:    kshiftlw $14, %k7, %k7
+; AVX512DQNOBW-NEXT:    korw %k7, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftlw $1, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftrw $1, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k7
+; AVX512DQNOBW-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQNOBW-NEXT:    korw %k7, %k0, %k0
+; AVX512DQNOBW-NEXT:    kmovw %k0, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k0
+; AVX512DQNOBW-NEXT:    kandw %k2, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k7
+; AVX512DQNOBW-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQNOBW-NEXT:    kshiftrw $14, %k7, %k7
+; AVX512DQNOBW-NEXT:    korw %k7, %k0, %k0
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    kandw %k1, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k7
+; AVX512DQNOBW-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQNOBW-NEXT:    kshiftrw $13, %k7, %k7
+; AVX512DQNOBW-NEXT:    korw %k7, %k0, %k0
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    kandw %k2, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k7
+; AVX512DQNOBW-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQNOBW-NEXT:    kshiftrw $12, %k7, %k7
+; AVX512DQNOBW-NEXT:    korw %k7, %k0, %k0
+; AVX512DQNOBW-NEXT:    kandw %k6, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k7
+; AVX512DQNOBW-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQNOBW-NEXT:    kshiftrw $11, %k7, %k7
+; AVX512DQNOBW-NEXT:    korw %k7, %k0, %k0
+; AVX512DQNOBW-NEXT:    kandw %k3, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k7
+; AVX512DQNOBW-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQNOBW-NEXT:    kshiftrw $10, %k7, %k7
+; AVX512DQNOBW-NEXT:    korw %k7, %k0, %k0
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k6 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    kandw %k6, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k7
+; AVX512DQNOBW-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQNOBW-NEXT:    kshiftrw $9, %k7, %k7
+; AVX512DQNOBW-NEXT:    korw %k7, %k0, %k0
+; AVX512DQNOBW-NEXT:    kandw %k4, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k7
+; AVX512DQNOBW-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQNOBW-NEXT:    kshiftrw $8, %k7, %k7
+; AVX512DQNOBW-NEXT:    korw %k7, %k0, %k0
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k3 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    kandw %k3, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k7
+; AVX512DQNOBW-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQNOBW-NEXT:    kshiftrw $7, %k7, %k7
+; AVX512DQNOBW-NEXT:    korw %k7, %k0, %k0
+; AVX512DQNOBW-NEXT:    kandw %k5, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k7
+; AVX512DQNOBW-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQNOBW-NEXT:    kshiftrw $6, %k7, %k7
+; AVX512DQNOBW-NEXT:    korw %k7, %k0, %k0
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k4 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    kandw %k4, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k7
+; AVX512DQNOBW-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQNOBW-NEXT:    kshiftrw $5, %k7, %k7
+; AVX512DQNOBW-NEXT:    korw %k7, %k0, %k0
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k5 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    kandw %k5, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k7
+; AVX512DQNOBW-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQNOBW-NEXT:    kshiftrw $4, %k7, %k7
+; AVX512DQNOBW-NEXT:    korw %k7, %k0, %k0
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    kandw %k7, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k7
+; AVX512DQNOBW-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQNOBW-NEXT:    kshiftrw $3, %k7, %k7
+; AVX512DQNOBW-NEXT:    korw %k7, %k0, %k0
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    kandw %k7, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k7
+; AVX512DQNOBW-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQNOBW-NEXT:    kshiftrw $2, %k7, %k7
+; AVX512DQNOBW-NEXT:    korw %k7, %k0, %k0
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    kandw %k7, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k7
+; AVX512DQNOBW-NEXT:    kshiftlw $14, %k7, %k7
+; AVX512DQNOBW-NEXT:    korw %k7, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftlw $1, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftrw $1, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k7
+; AVX512DQNOBW-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQNOBW-NEXT:    korw %k7, %k0, %k0
+; AVX512DQNOBW-NEXT:    kmovw %k0, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k0
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    kandw %k7, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k7
+; AVX512DQNOBW-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQNOBW-NEXT:    kshiftrw $14, %k7, %k7
+; AVX512DQNOBW-NEXT:    korw %k7, %k0, %k0
+; AVX512DQNOBW-NEXT:    kandw %k1, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k7
+; AVX512DQNOBW-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQNOBW-NEXT:    kshiftrw $13, %k7, %k7
+; AVX512DQNOBW-NEXT:    korw %k7, %k0, %k0
+; AVX512DQNOBW-NEXT:    kandw %k2, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k7
+; AVX512DQNOBW-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQNOBW-NEXT:    kshiftrw $12, %k7, %k7
+; AVX512DQNOBW-NEXT:    korw %k7, %k0, %k0
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    kandw %k1, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k7
+; AVX512DQNOBW-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQNOBW-NEXT:    kshiftrw $11, %k7, %k7
+; AVX512DQNOBW-NEXT:    korw %k7, %k0, %k0
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    kandw %k2, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k7
+; AVX512DQNOBW-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQNOBW-NEXT:    kshiftrw $10, %k7, %k7
+; AVX512DQNOBW-NEXT:    korw %k7, %k0, %k0
+; AVX512DQNOBW-NEXT:    kandw %k6, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k7
+; AVX512DQNOBW-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQNOBW-NEXT:    kshiftrw $9, %k7, %k7
+; AVX512DQNOBW-NEXT:    korw %k7, %k0, %k0
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    kandw %k2, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k7
+; AVX512DQNOBW-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQNOBW-NEXT:    kshiftrw $8, %k7, %k7
+; AVX512DQNOBW-NEXT:    korw %k7, %k0, %k0
+; AVX512DQNOBW-NEXT:    kandw %k3, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k7
+; AVX512DQNOBW-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQNOBW-NEXT:    kshiftrw $7, %k7, %k7
+; AVX512DQNOBW-NEXT:    korw %k7, %k0, %k0
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k3 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    kandw %k3, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k7
+; AVX512DQNOBW-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQNOBW-NEXT:    kshiftrw $6, %k7, %k7
+; AVX512DQNOBW-NEXT:    korw %k7, %k0, %k0
+; AVX512DQNOBW-NEXT:    kandw %k4, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k7
+; AVX512DQNOBW-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQNOBW-NEXT:    kshiftrw $5, %k7, %k7
+; AVX512DQNOBW-NEXT:    korw %k7, %k0, %k0
+; AVX512DQNOBW-NEXT:    kandw %k5, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k7
+; AVX512DQNOBW-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQNOBW-NEXT:    kshiftrw $4, %k7, %k7
+; AVX512DQNOBW-NEXT:    korw %k7, %k0, %k0
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    kandw %k2, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k7
+; AVX512DQNOBW-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQNOBW-NEXT:    kshiftrw $3, %k7, %k7
+; AVX512DQNOBW-NEXT:    korw %k7, %k0, %k0
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k5 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    kandw %k5, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k7
+; AVX512DQNOBW-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQNOBW-NEXT:    kshiftrw $2, %k7, %k7
+; AVX512DQNOBW-NEXT:    korw %k7, %k0, %k0
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k5 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    kandw %k5, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k7
+; AVX512DQNOBW-NEXT:    kshiftlw $14, %k7, %k7
+; AVX512DQNOBW-NEXT:    korw %k7, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftlw $1, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftrw $1, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k7
+; AVX512DQNOBW-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQNOBW-NEXT:    korw %k7, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k7
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k5 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    kandw %k5, %k7, %k7
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k6
+; AVX512DQNOBW-NEXT:    kshiftlw $15, %k6, %k6
+; AVX512DQNOBW-NEXT:    kshiftrw $14, %k6, %k6
+; AVX512DQNOBW-NEXT:    korw %k6, %k7, %k6
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k5 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    kandw %k5, %k6, %k6
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k7
+; AVX512DQNOBW-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQNOBW-NEXT:    kshiftrw $13, %k7, %k7
+; AVX512DQNOBW-NEXT:    korw %k7, %k6, %k6
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k5 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    kandw %k5, %k6, %k6
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k7
+; AVX512DQNOBW-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQNOBW-NEXT:    kshiftrw $12, %k7, %k7
+; AVX512DQNOBW-NEXT:    korw %k7, %k6, %k6
+; AVX512DQNOBW-NEXT:    kandw %k1, %k6, %k6
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k7
+; AVX512DQNOBW-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQNOBW-NEXT:    kshiftrw $11, %k7, %k7
+; AVX512DQNOBW-NEXT:    korw %k7, %k6, %k6
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    kandw %k1, %k6, %k6
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k7
+; AVX512DQNOBW-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQNOBW-NEXT:    kshiftrw $10, %k7, %k7
+; AVX512DQNOBW-NEXT:    korw %k7, %k6, %k6
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    kandw %k1, %k6, %k6
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k7
+; AVX512DQNOBW-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQNOBW-NEXT:    kshiftrw $9, %k7, %k7
+; AVX512DQNOBW-NEXT:    korw %k7, %k6, %k6
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    kandw %k1, %k6, %k6
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k7
+; AVX512DQNOBW-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQNOBW-NEXT:    kshiftrw $8, %k7, %k7
+; AVX512DQNOBW-NEXT:    korw %k7, %k6, %k6
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    kandw %k1, %k6, %k6
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k7
+; AVX512DQNOBW-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQNOBW-NEXT:    kshiftrw $7, %k7, %k7
+; AVX512DQNOBW-NEXT:    korw %k7, %k6, %k6
+; AVX512DQNOBW-NEXT:    kandw %k3, %k6, %k6
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k7
+; AVX512DQNOBW-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQNOBW-NEXT:    kshiftrw $6, %k7, %k7
+; AVX512DQNOBW-NEXT:    korw %k7, %k6, %k6
+; AVX512DQNOBW-NEXT:    kandw %k4, %k6, %k5
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k6
+; AVX512DQNOBW-NEXT:    kshiftlw $15, %k6, %k6
+; AVX512DQNOBW-NEXT:    kshiftrw $5, %k6, %k6
+; AVX512DQNOBW-NEXT:    korw %k6, %k5, %k5
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    kandw %k1, %k5, %k4
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k5
+; AVX512DQNOBW-NEXT:    kshiftlw $15, %k5, %k5
+; AVX512DQNOBW-NEXT:    kshiftrw $4, %k5, %k5
+; AVX512DQNOBW-NEXT:    korw %k5, %k4, %k4
+; AVX512DQNOBW-NEXT:    kandw %k2, %k4, %k3
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k4
+; AVX512DQNOBW-NEXT:    kshiftlw $15, %k4, %k4
+; AVX512DQNOBW-NEXT:    kshiftrw $3, %k4, %k4
+; AVX512DQNOBW-NEXT:    korw %k4, %k3, %k3
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    kandw %k1, %k3, %k2
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k3
+; AVX512DQNOBW-NEXT:    kshiftlw $15, %k3, %k3
+; AVX512DQNOBW-NEXT:    kshiftrw $2, %k3, %k3
+; AVX512DQNOBW-NEXT:    korw %k3, %k2, %k2
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    kandw %k1, %k2, %k1
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k2
+; AVX512DQNOBW-NEXT:    kshiftlw $14, %k2, %k2
+; AVX512DQNOBW-NEXT:    korw %k2, %k1, %k1
+; AVX512DQNOBW-NEXT:    kshiftlw $1, %k1, %k1
+; AVX512DQNOBW-NEXT:    kshiftrw $1, %k1, %k1
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k2
+; AVX512DQNOBW-NEXT:    kshiftlw $15, %k2, %k2
+; AVX512DQNOBW-NEXT:    korw %k2, %k1, %k1
+; AVX512DQNOBW-NEXT:    vpmovm2d %k1, %zmm4
+; AVX512DQNOBW-NEXT:    vpmovm2d %k0, %zmm5
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    vpmovm2d %k0, %zmm6
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    vpmovm2d %k0, %zmm7
+; AVX512DQNOBW-NEXT:    vpmovdw %zmm4, %ymm4
 ; AVX512DQNOBW-NEXT:    vpand %ymm1, %ymm4, %ymm1
-; AVX512DQNOBW-NEXT:    vpsllw $15, %ymm6, %ymm4
-; AVX512DQNOBW-NEXT:    vpsraw $15, %ymm4, %ymm4
+; AVX512DQNOBW-NEXT:    vpmovdw %zmm5, %ymm4
 ; AVX512DQNOBW-NEXT:    vpand %ymm2, %ymm4, %ymm2
-; AVX512DQNOBW-NEXT:    vpsllw $15, %ymm7, %ymm4
-; AVX512DQNOBW-NEXT:    vpsraw $15, %ymm4, %ymm4
+; AVX512DQNOBW-NEXT:    vpmovdw %zmm6, %ymm4
 ; AVX512DQNOBW-NEXT:    vpand %ymm3, %ymm4, %ymm3
+; AVX512DQNOBW-NEXT:    vpmovdw %zmm7, %ymm4
+; AVX512DQNOBW-NEXT:    vpand %ymm0, %ymm4, %ymm0
 ; AVX512DQNOBW-NEXT:    retq
   %ret = select <64 x i1> %mask, <64 x i16> %x, <64 x i16> zeroinitializer
   ret <64 x i16> %ret
@@ -2032,11 +2902,14 @@ define <4 x i64> @zext_8x32_to_4x64(<8 x i32> %a) {
 define <64 x i8> @zext_64xi1_to_64xi8(<64 x i8> %x, <64 x i8> %y) #0 {
 ; KNL-LABEL: zext_64xi1_to_64xi8:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    vpcmpeqb %ymm2, %ymm0, %ymm0
-; KNL-NEXT:    vmovdqa {{.*#+}} ymm2 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; KNL-NEXT:    vpand %ymm2, %ymm0, %ymm0
-; KNL-NEXT:    vpcmpeqb %ymm3, %ymm1, %ymm1
-; KNL-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; KNL-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
+; KNL-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
+; KNL-NEXT:    vpcmpeqb %ymm2, %ymm3, %ymm2
+; KNL-NEXT:    vmovdqa {{.*#+}} ymm3 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; KNL-NEXT:    vpand %ymm3, %ymm2, %ymm2
+; KNL-NEXT:    vpcmpeqb %ymm1, %ymm0, %ymm0
+; KNL-NEXT:    vpand %ymm3, %ymm0, %ymm0
+; KNL-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: zext_64xi1_to_64xi8:
@@ -2047,11 +2920,14 @@ define <64 x i8> @zext_64xi1_to_64xi8(<64 x i8> %x, <64 x i8> %y) #0 {
 ;
 ; AVX512DQNOBW-LABEL: zext_64xi1_to_64xi8:
 ; AVX512DQNOBW:       # %bb.0:
-; AVX512DQNOBW-NEXT:    vpcmpeqb %ymm2, %ymm0, %ymm0
-; AVX512DQNOBW-NEXT:    vmovdqa {{.*#+}} ymm2 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; AVX512DQNOBW-NEXT:    vpand %ymm2, %ymm0, %ymm0
-; AVX512DQNOBW-NEXT:    vpcmpeqb %ymm3, %ymm1, %ymm1
-; AVX512DQNOBW-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX512DQNOBW-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
+; AVX512DQNOBW-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
+; AVX512DQNOBW-NEXT:    vpcmpeqb %ymm2, %ymm3, %ymm2
+; AVX512DQNOBW-NEXT:    vmovdqa {{.*#+}} ymm3 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; AVX512DQNOBW-NEXT:    vpand %ymm3, %ymm2, %ymm2
+; AVX512DQNOBW-NEXT:    vpcmpeqb %ymm1, %ymm0, %ymm0
+; AVX512DQNOBW-NEXT:    vpand %ymm3, %ymm0, %ymm0
+; AVX512DQNOBW-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512DQNOBW-NEXT:    retq
   %mask = icmp eq <64 x i8> %x, %y
   %1 = zext <64 x i1> %mask to <64 x i8>
@@ -2061,10 +2937,13 @@ define <64 x i8> @zext_64xi1_to_64xi8(<64 x i8> %x, <64 x i8> %y) #0 {
 define <32 x i16> @zext_32xi1_to_32xi16(<32 x i16> %x, <32 x i16> %y) #0 {
 ; KNL-LABEL: zext_32xi1_to_32xi16:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    vpcmpeqw %ymm2, %ymm0, %ymm0
+; KNL-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
+; KNL-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
+; KNL-NEXT:    vpcmpeqw %ymm2, %ymm3, %ymm2
+; KNL-NEXT:    vpsrlw $15, %ymm2, %ymm2
+; KNL-NEXT:    vpcmpeqw %ymm1, %ymm0, %ymm0
 ; KNL-NEXT:    vpsrlw $15, %ymm0, %ymm0
-; KNL-NEXT:    vpcmpeqw %ymm3, %ymm1, %ymm1
-; KNL-NEXT:    vpsrlw $15, %ymm1, %ymm1
+; KNL-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: zext_32xi1_to_32xi16:
@@ -2076,10 +2955,13 @@ define <32 x i16> @zext_32xi1_to_32xi16(<32 x i16> %x, <32 x i16> %y) #0 {
 ;
 ; AVX512DQNOBW-LABEL: zext_32xi1_to_32xi16:
 ; AVX512DQNOBW:       # %bb.0:
-; AVX512DQNOBW-NEXT:    vpcmpeqw %ymm2, %ymm0, %ymm0
+; AVX512DQNOBW-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
+; AVX512DQNOBW-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
+; AVX512DQNOBW-NEXT:    vpcmpeqw %ymm2, %ymm3, %ymm2
+; AVX512DQNOBW-NEXT:    vpsrlw $15, %ymm2, %ymm2
+; AVX512DQNOBW-NEXT:    vpcmpeqw %ymm1, %ymm0, %ymm0
 ; AVX512DQNOBW-NEXT:    vpsrlw $15, %ymm0, %ymm0
-; AVX512DQNOBW-NEXT:    vpcmpeqw %ymm3, %ymm1, %ymm1
-; AVX512DQNOBW-NEXT:    vpsrlw $15, %ymm1, %ymm1
+; AVX512DQNOBW-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512DQNOBW-NEXT:    retq
   %mask = icmp eq <32 x i16> %x, %y
   %1 = zext <32 x i1> %mask to <32 x i16>
@@ -2101,11 +2983,13 @@ define <16 x i16> @zext_16xi1_to_16xi16(<16 x i16> %x, <16 x i16> %y) #0 {
 define <32 x i8> @zext_32xi1_to_32xi8(<32 x i16> %x, <32 x i16> %y) #0 {
 ; KNL-LABEL: zext_32xi1_to_32xi8:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    vpcmpeqw %ymm2, %ymm0, %ymm0
+; KNL-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
+; KNL-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
+; KNL-NEXT:    vpcmpeqw %ymm2, %ymm3, %ymm2
+; KNL-NEXT:    vpcmpeqw %ymm1, %ymm0, %ymm0
 ; KNL-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
 ; KNL-NEXT:    vpmovdb %zmm0, %xmm0
-; KNL-NEXT:    vpcmpeqw %ymm3, %ymm1, %ymm1
-; KNL-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
+; KNL-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
 ; KNL-NEXT:    vpmovdb %zmm1, %xmm1
 ; KNL-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
 ; KNL-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
@@ -2119,11 +3003,13 @@ define <32 x i8> @zext_32xi1_to_32xi8(<32 x i16> %x, <32 x i16> %y) #0 {
 ;
 ; AVX512DQNOBW-LABEL: zext_32xi1_to_32xi8:
 ; AVX512DQNOBW:       # %bb.0:
-; AVX512DQNOBW-NEXT:    vpcmpeqw %ymm2, %ymm0, %ymm0
+; AVX512DQNOBW-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
+; AVX512DQNOBW-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
+; AVX512DQNOBW-NEXT:    vpcmpeqw %ymm2, %ymm3, %ymm2
+; AVX512DQNOBW-NEXT:    vpcmpeqw %ymm1, %ymm0, %ymm0
 ; AVX512DQNOBW-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
 ; AVX512DQNOBW-NEXT:    vpmovdb %zmm0, %xmm0
-; AVX512DQNOBW-NEXT:    vpcmpeqw %ymm3, %ymm1, %ymm1
-; AVX512DQNOBW-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
+; AVX512DQNOBW-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
 ; AVX512DQNOBW-NEXT:    vpmovdb %zmm1, %xmm1
 ; AVX512DQNOBW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
 ; AVX512DQNOBW-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
@@ -2134,28 +3020,53 @@ define <32 x i8> @zext_32xi1_to_32xi8(<32 x i16> %x, <32 x i16> %y) #0 {
 }
 
 define <4 x i32> @zext_4xi1_to_4x32(<4 x i8> %x, <4 x i8> %y) #0 {
-; ALL-LABEL: zext_4xi1_to_4x32:
-; ALL:       # %bb.0:
-; ALL-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [255,255,255,255]
-; ALL-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; ALL-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; ALL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
-; ALL-NEXT:    vpsrld $31, %xmm0, %xmm0
-; ALL-NEXT:    retq
+; KNL-LABEL: zext_4xi1_to_4x32:
+; KNL:       # %bb.0:
+; KNL-NEXT:    vpcmpeqb %xmm1, %xmm0, %xmm0
+; KNL-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; KNL-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [1,1,1,1]
+; KNL-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; KNL-NEXT:    retq
+;
+; SKX-LABEL: zext_4xi1_to_4x32:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpcmpeqb %xmm1, %xmm0, %k0
+; SKX-NEXT:    vpmovm2d %k0, %xmm0
+; SKX-NEXT:    vpsrld $31, %xmm0, %xmm0
+; SKX-NEXT:    retq
+;
+; AVX512DQNOBW-LABEL: zext_4xi1_to_4x32:
+; AVX512DQNOBW:       # %bb.0:
+; AVX512DQNOBW-NEXT:    vpcmpeqb %xmm1, %xmm0, %xmm0
+; AVX512DQNOBW-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; AVX512DQNOBW-NEXT:    vpandd {{.*}}(%rip){1to4}, %xmm0, %xmm0
+; AVX512DQNOBW-NEXT:    retq
   %mask = icmp eq <4 x i8> %x, %y
   %1 = zext <4 x i1> %mask to <4 x i32>
   ret <4 x i32> %1
 }
 
 define <2 x i64> @zext_2xi1_to_2xi64(<2 x i8> %x, <2 x i8> %y) #0 {
-; ALL-LABEL: zext_2xi1_to_2xi64:
-; ALL:       # %bb.0:
-; ALL-NEXT:    vpbroadcastq {{.*#+}} xmm2 = [255,255]
-; ALL-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; ALL-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; ALL-NEXT:    vpcmpeqq %xmm1, %xmm0, %xmm0
-; ALL-NEXT:    vpsrlq $63, %xmm0, %xmm0
-; ALL-NEXT:    retq
+; KNL-LABEL: zext_2xi1_to_2xi64:
+; KNL:       # %bb.0:
+; KNL-NEXT:    vpcmpeqb %xmm1, %xmm0, %xmm0
+; KNL-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
+; KNL-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; KNL-NEXT:    retq
+;
+; SKX-LABEL: zext_2xi1_to_2xi64:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpcmpeqb %xmm1, %xmm0, %k0
+; SKX-NEXT:    vpmovm2q %k0, %xmm0
+; SKX-NEXT:    vpsrlq $63, %xmm0, %xmm0
+; SKX-NEXT:    retq
+;
+; AVX512DQNOBW-LABEL: zext_2xi1_to_2xi64:
+; AVX512DQNOBW:       # %bb.0:
+; AVX512DQNOBW-NEXT:    vpcmpeqb %xmm1, %xmm0, %xmm0
+; AVX512DQNOBW-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512DQNOBW-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX512DQNOBW-NEXT:    retq
   %mask = icmp eq <2 x i8> %x, %y
   %1 = zext <2 x i1> %mask to <2 x i64>
   ret <2 x i64> %1
diff --git a/llvm/test/CodeGen/X86/avx512-fma-commute.ll b/llvm/test/CodeGen/X86/avx512-fma-commute.ll
index 194255179..bf156b41a 100644
--- a/llvm/test/CodeGen/X86/avx512-fma-commute.ll
+++ b/llvm/test/CodeGen/X86/avx512-fma-commute.ll
@@ -9,7 +9,7 @@ declare <2 x double> @llvm.x86.avx512.mask3.vfmsub.sd(<2 x double>, <2 x double>
 define <4 x float> @test_int_x86_avx512_mask3_vfmadd_ss_load0(<4 x float>* %x0ptr, <4 x float> %x1, <4 x float> %x2) {
 ; CHECK-LABEL: test_int_x86_avx512_mask3_vfmadd_ss_load0:
 ; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vfmadd231ss (%rdi), %xmm0, %xmm1
+; CHECK-NEXT:    vfmadd231ss {{.*#+}} xmm1 = (xmm0 * mem) + xmm1
 ; CHECK-NEXT:    vmovaps %xmm1, %xmm0
 ; CHECK-NEXT:    retq
   %x0 = load <4 x float>, <4 x float>* %x0ptr
@@ -20,7 +20,7 @@ define <4 x float> @test_int_x86_avx512_mask3_vfmadd_ss_load0(<4 x float>* %x0pt
 define <4 x float> @test_int_x86_avx512_mask3_vfmadd_ss_load1(<4 x float> %x0, <4 x float>* %x1ptr, <4 x float> %x2){
 ; CHECK-LABEL: test_int_x86_avx512_mask3_vfmadd_ss_load1:
 ; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vfmadd231ss (%rdi), %xmm0, %xmm1
+; CHECK-NEXT:    vfmadd231ss {{.*#+}} xmm1 = (xmm0 * mem) + xmm1
 ; CHECK-NEXT:    vmovaps %xmm1, %xmm0
 ; CHECK-NEXT:    retq
   %x1 = load <4 x float>, <4 x float>* %x1ptr
@@ -31,7 +31,7 @@ define <4 x float> @test_int_x86_avx512_mask3_vfmadd_ss_load1(<4 x float> %x0, <
 define <2 x double> @test_int_x86_avx512_mask3_vfmadd_sd_load0(<2 x double>* %x0ptr, <2 x double> %x1, <2 x double> %x2) {
 ; CHECK-LABEL: test_int_x86_avx512_mask3_vfmadd_sd_load0:
 ; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vfmadd231sd (%rdi), %xmm0, %xmm1
+; CHECK-NEXT:    vfmadd231sd {{.*#+}} xmm1 = (xmm0 * mem) + xmm1
 ; CHECK-NEXT:    vmovapd %xmm1, %xmm0
 ; CHECK-NEXT:    retq
   %x0 = load <2 x double>, <2 x double>* %x0ptr
@@ -42,7 +42,7 @@ define <2 x double> @test_int_x86_avx512_mask3_vfmadd_sd_load0(<2 x double>* %x0
 define <2 x double> @test_int_x86_avx512_mask3_vfmadd_sd_load1(<2 x double> %x0, <2 x double>* %x1ptr, <2 x double> %x2){
 ; CHECK-LABEL: test_int_x86_avx512_mask3_vfmadd_sd_load1:
 ; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vfmadd231sd (%rdi), %xmm0, %xmm1
+; CHECK-NEXT:    vfmadd231sd {{.*#+}} xmm1 = (xmm0 * mem) + xmm1
 ; CHECK-NEXT:    vmovapd %xmm1, %xmm0
 ; CHECK-NEXT:    retq
   %x1 = load <2 x double>, <2 x double>* %x1ptr
@@ -53,7 +53,7 @@ define <2 x double> @test_int_x86_avx512_mask3_vfmadd_sd_load1(<2 x double> %x0,
 define <4 x float> @test_int_x86_avx512_mask3_vfmsub_ss_load0(<4 x float>* %x0ptr, <4 x float> %x1, <4 x float> %x2) {
 ; CHECK-LABEL: test_int_x86_avx512_mask3_vfmsub_ss_load0:
 ; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vfmsub231ss (%rdi), %xmm0, %xmm1
+; CHECK-NEXT:    vfmsub231ss {{.*#+}} xmm1 = (xmm0 * mem) - xmm1
 ; CHECK-NEXT:    vmovaps %xmm1, %xmm0
 ; CHECK-NEXT:    retq
   %x0 = load <4 x float>, <4 x float>* %x0ptr
@@ -64,7 +64,7 @@ define <4 x float> @test_int_x86_avx512_mask3_vfmsub_ss_load0(<4 x float>* %x0pt
 define <4 x float> @test_int_x86_avx512_mask3_vfmsub_ss_load1(<4 x float> %x0, <4 x float>* %x1ptr, <4 x float> %x2){
 ; CHECK-LABEL: test_int_x86_avx512_mask3_vfmsub_ss_load1:
 ; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vfmsub231ss (%rdi), %xmm0, %xmm1
+; CHECK-NEXT:    vfmsub231ss {{.*#+}} xmm1 = (xmm0 * mem) - xmm1
 ; CHECK-NEXT:    vmovaps %xmm1, %xmm0
 ; CHECK-NEXT:    retq
   %x1 = load <4 x float>, <4 x float>* %x1ptr
@@ -75,7 +75,7 @@ define <4 x float> @test_int_x86_avx512_mask3_vfmsub_ss_load1(<4 x float> %x0, <
 define <2 x double> @test_int_x86_avx512_mask3_vfmsub_sd_load0(<2 x double>* %x0ptr, <2 x double> %x1, <2 x double> %x2) {
 ; CHECK-LABEL: test_int_x86_avx512_mask3_vfmsub_sd_load0:
 ; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vfmsub231sd (%rdi), %xmm0, %xmm1
+; CHECK-NEXT:    vfmsub231sd {{.*#+}} xmm1 = (xmm0 * mem) - xmm1
 ; CHECK-NEXT:    vmovapd %xmm1, %xmm0
 ; CHECK-NEXT:    retq
   %x0 = load <2 x double>, <2 x double>* %x0ptr
@@ -86,7 +86,7 @@ define <2 x double> @test_int_x86_avx512_mask3_vfmsub_sd_load0(<2 x double>* %x0
 define <2 x double> @test_int_x86_avx512_mask3_vfmsub_sd_load1(<2 x double> %x0, <2 x double>* %x1ptr, <2 x double> %x2){
 ; CHECK-LABEL: test_int_x86_avx512_mask3_vfmsub_sd_load1:
 ; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vfmsub231sd (%rdi), %xmm0, %xmm1
+; CHECK-NEXT:    vfmsub231sd {{.*#+}} xmm1 = (xmm0 * mem) - xmm1
 ; CHECK-NEXT:    vmovapd %xmm1, %xmm0
 ; CHECK-NEXT:    retq
   %x1 = load <2 x double>, <2 x double>* %x1ptr
diff --git a/llvm/test/CodeGen/X86/avx512-inc-dec.ll b/llvm/test/CodeGen/X86/avx512-inc-dec.ll
index 4fa4f27be..73a2d67f8 100644
--- a/llvm/test/CodeGen/X86/avx512-inc-dec.ll
+++ b/llvm/test/CodeGen/X86/avx512-inc-dec.ll
@@ -1,10 +1,14 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl | FileCheck %s
 
-;CHECK-LABEL: test
-;CHECK-NOT: dec
-;CHECK-NOT: enc
-;CHECK: ret
 define i32 @test(i32 %a, i32 %b) {
+; CHECK-LABEL: test:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    ## kill: def $edi killed $edi def $rdi
+; CHECK-NEXT:    leal -1(%rdi), %eax
+; CHECK-NEXT:    addl $1, %esi
+; CHECK-NEXT:    imull %esi, %eax
+; CHECK-NEXT:    retq
  %a1 = add i32 %a, -1
  %b1 = add i32 %b, 1
  %res = mul i32 %a1, %b1
diff --git a/llvm/test/CodeGen/X86/avx512-insert-extract.ll b/llvm/test/CodeGen/X86/avx512-insert-extract.ll
index e197d278b..6139928cb 100644
--- a/llvm/test/CodeGen/X86/avx512-insert-extract.ll
+++ b/llvm/test/CodeGen/X86/avx512-insert-extract.ll
@@ -302,12 +302,13 @@ define i16 @test16(i1 *%addr, i16 %a) {
 ; KNL:       ## %bb.0:
 ; KNL-NEXT:    movb (%rdi), %al
 ; KNL-NEXT:    kmovw %esi, %k0
+; KNL-NEXT:    movw $-1025, %cx ## imm = 0xFBFF
+; KNL-NEXT:    kmovw %ecx, %k1
+; KNL-NEXT:    kandw %k1, %k0, %k0
 ; KNL-NEXT:    kmovw %eax, %k1
-; KNL-NEXT:    kshiftrw $10, %k0, %k2
-; KNL-NEXT:    kxorw %k1, %k2, %k1
 ; KNL-NEXT:    kshiftlw $15, %k1, %k1
 ; KNL-NEXT:    kshiftrw $5, %k1, %k1
-; KNL-NEXT:    kxorw %k1, %k0, %k0
+; KNL-NEXT:    korw %k1, %k0, %k0
 ; KNL-NEXT:    kmovw %k0, %eax
 ; KNL-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; KNL-NEXT:    retq
@@ -316,11 +317,12 @@ define i16 @test16(i1 *%addr, i16 %a) {
 ; SKX:       ## %bb.0:
 ; SKX-NEXT:    kmovb (%rdi), %k0
 ; SKX-NEXT:    kmovd %esi, %k1
-; SKX-NEXT:    kshiftrw $10, %k1, %k2
-; SKX-NEXT:    kxorw %k0, %k2, %k0
+; SKX-NEXT:    movw $-1025, %ax ## imm = 0xFBFF
+; SKX-NEXT:    kmovd %eax, %k2
+; SKX-NEXT:    kandw %k2, %k1, %k1
 ; SKX-NEXT:    kshiftlw $15, %k0, %k0
 ; SKX-NEXT:    kshiftrw $5, %k0, %k0
-; SKX-NEXT:    kxorw %k0, %k1, %k0
+; SKX-NEXT:    korw %k0, %k1, %k0
 ; SKX-NEXT:    kmovd %k0, %eax
 ; SKX-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; SKX-NEXT:    retq
@@ -336,12 +338,13 @@ define i8 @test17(i1 *%addr, i8 %a) {
 ; KNL:       ## %bb.0:
 ; KNL-NEXT:    movb (%rdi), %al
 ; KNL-NEXT:    kmovw %esi, %k0
+; KNL-NEXT:    movw $-17, %cx
+; KNL-NEXT:    kmovw %ecx, %k1
+; KNL-NEXT:    kandw %k1, %k0, %k0
 ; KNL-NEXT:    kmovw %eax, %k1
-; KNL-NEXT:    kshiftrw $4, %k0, %k2
-; KNL-NEXT:    kxorw %k1, %k2, %k1
 ; KNL-NEXT:    kshiftlw $15, %k1, %k1
 ; KNL-NEXT:    kshiftrw $11, %k1, %k1
-; KNL-NEXT:    kxorw %k1, %k0, %k0
+; KNL-NEXT:    korw %k1, %k0, %k0
 ; KNL-NEXT:    kmovw %k0, %eax
 ; KNL-NEXT:    ## kill: def $al killed $al killed $eax
 ; KNL-NEXT:    retq
@@ -350,11 +353,12 @@ define i8 @test17(i1 *%addr, i8 %a) {
 ; SKX:       ## %bb.0:
 ; SKX-NEXT:    kmovb (%rdi), %k0
 ; SKX-NEXT:    kmovd %esi, %k1
-; SKX-NEXT:    kshiftrb $4, %k1, %k2
-; SKX-NEXT:    kxorb %k0, %k2, %k0
+; SKX-NEXT:    movb $-17, %al
+; SKX-NEXT:    kmovd %eax, %k2
+; SKX-NEXT:    kandb %k2, %k1, %k1
 ; SKX-NEXT:    kshiftlb $7, %k0, %k0
 ; SKX-NEXT:    kshiftrb $3, %k0, %k0
-; SKX-NEXT:    kxorb %k0, %k1, %k0
+; SKX-NEXT:    korb %k0, %k1, %k0
 ; SKX-NEXT:    kmovd %k0, %eax
 ; SKX-NEXT:    ## kill: def $al killed $al killed $eax
 ; SKX-NEXT:    retq
@@ -618,11 +622,11 @@ define <4 x i32> @insert_v4i32(<4 x i32> %x, i32 %y, i32* %ptr) {
 define <32 x i16> @insert_v32i16(<32 x i16> %x, i16 %y, i16* %ptr) {
 ; KNL-LABEL: insert_v32i16:
 ; KNL:       ## %bb.0:
-; KNL-NEXT:    vpinsrw $1, (%rsi), %xmm0, %xmm2
-; KNL-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm0[4,5,6,7]
-; KNL-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; KNL-NEXT:    vpinsrw $1, %edi, %xmm0, %xmm0
-; KNL-NEXT:    vinserti128 $1, %xmm0, %ymm2, %ymm0
+; KNL-NEXT:    vpinsrw $1, (%rsi), %xmm0, %xmm1
+; KNL-NEXT:    vextracti128 $1, %ymm0, %xmm2
+; KNL-NEXT:    vpinsrw $1, %edi, %xmm2, %xmm2
+; KNL-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; KNL-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm0[4,5,6,7]
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: insert_v32i16:
@@ -669,11 +673,13 @@ define <8 x i16> @insert_v8i16(<8 x i16> %x, i16 %y, i16* %ptr) {
 define <64 x i8> @insert_v64i8(<64 x i8> %x, i8 %y, i8* %ptr) {
 ; KNL-LABEL: insert_v64i8:
 ; KNL:       ## %bb.0:
+; KNL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; KNL-NEXT:    vpinsrb $1, (%rsi), %xmm0, %xmm2
 ; KNL-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
 ; KNL-NEXT:    vextracti128 $1, %ymm1, %xmm2
 ; KNL-NEXT:    vpinsrb $2, %edi, %xmm2, %xmm2
 ; KNL-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; KNL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: insert_v64i8:
@@ -787,13 +793,13 @@ define i32 @test_insertelement_v32i1(i32 %a, i32 %b, <32 x i32> %x , <32 x i32>
 ; KNL-NEXT:    vpcmpltud %zmm3, %zmm1, %k0
 ; KNL-NEXT:    kmovw %k0, %ecx
 ; KNL-NEXT:    shll $16, %ecx
-; KNL-NEXT:    vpcmpltud %zmm2, %zmm0, %k0
-; KNL-NEXT:    kshiftrw $4, %k0, %k1
-; KNL-NEXT:    kmovw %eax, %k2
-; KNL-NEXT:    kxorw %k2, %k1, %k1
+; KNL-NEXT:    movw $-17, %dx
+; KNL-NEXT:    kmovw %edx, %k1
+; KNL-NEXT:    vpcmpltud %zmm2, %zmm0, %k0 {%k1}
+; KNL-NEXT:    kmovw %eax, %k1
 ; KNL-NEXT:    kshiftlw $15, %k1, %k1
 ; KNL-NEXT:    kshiftrw $11, %k1, %k1
-; KNL-NEXT:    kxorw %k1, %k0, %k0
+; KNL-NEXT:    korw %k1, %k0, %k0
 ; KNL-NEXT:    kmovw %k0, %eax
 ; KNL-NEXT:    orl %ecx, %eax
 ; KNL-NEXT:    vzeroupper
@@ -806,12 +812,13 @@ define i32 @test_insertelement_v32i1(i32 %a, i32 %b, <32 x i32> %x , <32 x i32>
 ; SKX-NEXT:    vpcmpltud %zmm2, %zmm0, %k0
 ; SKX-NEXT:    vpcmpltud %zmm3, %zmm1, %k1
 ; SKX-NEXT:    kunpckwd %k0, %k1, %k0
-; SKX-NEXT:    kshiftrd $4, %k0, %k1
-; SKX-NEXT:    kmovd %eax, %k2
-; SKX-NEXT:    kxord %k2, %k1, %k1
+; SKX-NEXT:    movl $-17, %ecx
+; SKX-NEXT:    kmovd %ecx, %k1
+; SKX-NEXT:    kandd %k1, %k0, %k0
+; SKX-NEXT:    kmovd %eax, %k1
 ; SKX-NEXT:    kshiftld $31, %k1, %k1
 ; SKX-NEXT:    kshiftrd $27, %k1, %k1
-; SKX-NEXT:    kxord %k1, %k0, %k0
+; SKX-NEXT:    kord %k1, %k0, %k0
 ; SKX-NEXT:    kmovd %k0, %eax
 ; SKX-NEXT:    vzeroupper
 ; SKX-NEXT:    retq
@@ -829,13 +836,13 @@ define i8 @test_iinsertelement_v4i1(i32 %a, i32 %b, <4 x i32> %x , <4 x i32> %y)
 ; KNL-NEXT:    ## kill: def $xmm0 killed $xmm0 def $zmm0
 ; KNL-NEXT:    cmpl %esi, %edi
 ; KNL-NEXT:    setb %al
-; KNL-NEXT:    vpcmpltud %zmm1, %zmm0, %k0
-; KNL-NEXT:    kshiftrw $2, %k0, %k1
-; KNL-NEXT:    kmovw %eax, %k2
-; KNL-NEXT:    kxorw %k2, %k1, %k1
+; KNL-NEXT:    movw $-5, %cx
+; KNL-NEXT:    kmovw %ecx, %k1
+; KNL-NEXT:    vpcmpltud %zmm1, %zmm0, %k0 {%k1}
+; KNL-NEXT:    kmovw %eax, %k1
 ; KNL-NEXT:    kshiftlw $15, %k1, %k1
 ; KNL-NEXT:    kshiftrw $13, %k1, %k1
-; KNL-NEXT:    kxorw %k1, %k0, %k0
+; KNL-NEXT:    korw %k1, %k0, %k0
 ; KNL-NEXT:    kmovw %k0, %eax
 ; KNL-NEXT:    ## kill: def $al killed $al killed $eax
 ; KNL-NEXT:    vzeroupper
@@ -845,13 +852,13 @@ define i8 @test_iinsertelement_v4i1(i32 %a, i32 %b, <4 x i32> %x , <4 x i32> %y)
 ; SKX:       ## %bb.0:
 ; SKX-NEXT:    cmpl %esi, %edi
 ; SKX-NEXT:    setb %al
-; SKX-NEXT:    vpcmpltud %xmm1, %xmm0, %k0
-; SKX-NEXT:    kshiftrb $2, %k0, %k1
-; SKX-NEXT:    kmovd %eax, %k2
-; SKX-NEXT:    kxorb %k2, %k1, %k1
+; SKX-NEXT:    movb $-5, %cl
+; SKX-NEXT:    kmovd %ecx, %k1
+; SKX-NEXT:    vpcmpltud %xmm1, %xmm0, %k0 {%k1}
+; SKX-NEXT:    kmovd %eax, %k1
 ; SKX-NEXT:    kshiftlb $7, %k1, %k1
 ; SKX-NEXT:    kshiftrb $5, %k1, %k1
-; SKX-NEXT:    kxorw %k1, %k0, %k0
+; SKX-NEXT:    korw %k1, %k0, %k0
 ; SKX-NEXT:    kmovd %k0, %eax
 ; SKX-NEXT:    ## kill: def $al killed $al killed $eax
 ; SKX-NEXT:    retq
@@ -1015,10 +1022,12 @@ define zeroext i8 @test_extractelement_v32i1(<32 x i8> %a, <32 x i8> %b) {
 define zeroext i8 @test_extractelement_v64i1(<64 x i8> %a, <64 x i8> %b) {
 ; KNL-LABEL: test_extractelement_v64i1:
 ; KNL:       ## %bb.0:
-; KNL-NEXT:    vextracti128 $1, %ymm3, %xmm0
+; KNL-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
+; KNL-NEXT:    vextracti64x4 $1, %zmm1, %ymm1
 ; KNL-NEXT:    vextracti128 $1, %ymm1, %xmm1
-; KNL-NEXT:    vpminub %xmm0, %xmm1, %xmm0
-; KNL-NEXT:    vpcmpeqb %xmm0, %xmm1, %xmm0
+; KNL-NEXT:    vextracti128 $1, %ymm0, %xmm0
+; KNL-NEXT:    vpminub %xmm1, %xmm0, %xmm1
+; KNL-NEXT:    vpcmpeqb %xmm1, %xmm0, %xmm0
 ; KNL-NEXT:    vpternlogq $15, %zmm0, %zmm0, %zmm0
 ; KNL-NEXT:    vpmovsxbd %xmm0, %zmm0
 ; KNL-NEXT:    vptestmd %zmm0, %zmm0, %k0
@@ -1049,10 +1058,12 @@ define zeroext i8 @test_extractelement_v64i1(<64 x i8> %a, <64 x i8> %b) {
 define zeroext i8 @extractelement_v64i1_alt(<64 x i8> %a, <64 x i8> %b) {
 ; KNL-LABEL: extractelement_v64i1_alt:
 ; KNL:       ## %bb.0:
-; KNL-NEXT:    vextracti128 $1, %ymm3, %xmm0
+; KNL-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
+; KNL-NEXT:    vextracti64x4 $1, %zmm1, %ymm1
 ; KNL-NEXT:    vextracti128 $1, %ymm1, %xmm1
-; KNL-NEXT:    vpminub %xmm0, %xmm1, %xmm0
-; KNL-NEXT:    vpcmpeqb %xmm0, %xmm1, %xmm0
+; KNL-NEXT:    vextracti128 $1, %ymm0, %xmm0
+; KNL-NEXT:    vpminub %xmm1, %xmm0, %xmm1
+; KNL-NEXT:    vpcmpeqb %xmm1, %xmm0, %xmm0
 ; KNL-NEXT:    vpternlogq $15, %zmm0, %zmm0, %zmm0
 ; KNL-NEXT:    vpmovsxbd %xmm0, %zmm0
 ; KNL-NEXT:    vptestmd %zmm0, %zmm0, %k0
@@ -1352,7 +1363,7 @@ define i16 @test_extractelement_variable_v32i16(<32 x i16> %t1, i32 %index) {
 ; KNL-NEXT:    andq $-64, %rsp
 ; KNL-NEXT:    subq $128, %rsp
 ; KNL-NEXT:    ## kill: def $edi killed $edi def $rdi
-; KNL-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; KNL-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; KNL-NEXT:    vmovaps %ymm0, (%rsp)
 ; KNL-NEXT:    andl $31, %edi
 ; KNL-NEXT:    movzwl (%rsp,%rdi,2), %eax
@@ -1428,7 +1439,7 @@ define i8 @test_extractelement_variable_v64i8(<64 x i8> %t1, i32 %index) {
 ; KNL-NEXT:    andq $-64, %rsp
 ; KNL-NEXT:    subq $128, %rsp
 ; KNL-NEXT:    ## kill: def $edi killed $edi def $rdi
-; KNL-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; KNL-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; KNL-NEXT:    vmovaps %ymm0, (%rsp)
 ; KNL-NEXT:    andl $63, %edi
 ; KNL-NEXT:    movb (%rsp,%rdi), %al
@@ -1470,7 +1481,7 @@ define i8 @test_extractelement_variable_v64i8_indexi8(<64 x i8> %t1, i8 %index)
 ; KNL-NEXT:    andq $-64, %rsp
 ; KNL-NEXT:    subq $128, %rsp
 ; KNL-NEXT:    addb %dil, %dil
-; KNL-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; KNL-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; KNL-NEXT:    vmovaps %ymm0, (%rsp)
 ; KNL-NEXT:    movzbl %dil, %eax
 ; KNL-NEXT:    andl $63, %eax
@@ -1764,6 +1775,7 @@ define i64 @test_insertelement_variable_v64i1(<64 x i8> %a, i8 %b, i32 %index) {
 ; KNL-NEXT:    andq $-64, %rsp
 ; KNL-NEXT:    subq $128, %rsp
 ; KNL-NEXT:    ## kill: def $esi killed $esi def $rsi
+; KNL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; KNL-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; KNL-NEXT:    vpcmpeqb %ymm2, %ymm0, %ymm0
 ; KNL-NEXT:    vpternlogq $15, %zmm0, %zmm0, %zmm0
@@ -2288,7 +2300,7 @@ define void @test_concat_v2i1(<2 x half>* %arg, <2 x half>* %arg1, <2 x half>* %
 ; KNL-NEXT:    movswl 2(%rsi), %eax
 ; KNL-NEXT:    vmovd %eax, %xmm1
 ; KNL-NEXT:    vcvtph2ps %xmm1, %xmm1
-; KNL-NEXT:    vmovss %xmm1, %xmm0, %xmm1 {%k2} {z}
+; KNL-NEXT:    vmovss %xmm1, %xmm1, %xmm1 {%k2} {z}
 ; KNL-NEXT:    vmovss %xmm0, %xmm0, %xmm0 {%k1} {z}
 ; KNL-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
 ; KNL-NEXT:    vmovd %xmm0, %eax
@@ -2336,7 +2348,7 @@ define void @test_concat_v2i1(<2 x half>* %arg, <2 x half>* %arg1, <2 x half>* %
 ; SKX-NEXT:    movswl 2(%rsi), %eax
 ; SKX-NEXT:    vmovd %eax, %xmm1
 ; SKX-NEXT:    vcvtph2ps %xmm1, %xmm1
-; SKX-NEXT:    vmovss %xmm1, %xmm0, %xmm1 {%k2} {z}
+; SKX-NEXT:    vmovss %xmm1, %xmm1, %xmm1 {%k2} {z}
 ; SKX-NEXT:    vmovss %xmm0, %xmm0, %xmm0 {%k1} {z}
 ; SKX-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
 ; SKX-NEXT:    vmovd %xmm0, %eax
diff --git a/llvm/test/CodeGen/X86/avx512-intel-ocl.ll b/llvm/test/CodeGen/X86/avx512-intel-ocl.ll
index defedd2a7..751d610c2 100644
--- a/llvm/test/CodeGen/X86/avx512-intel-ocl.ll
+++ b/llvm/test/CodeGen/X86/avx512-intel-ocl.ll
@@ -94,12 +94,12 @@ define <16 x float> @testf16_regs(<16 x float> %a, <16 x float> %b) nounwind {
 ; X32-NEXT:    movl %esp, %ebp
 ; X32-NEXT:    andl $-64, %esp
 ; X32-NEXT:    subl $256, %esp ## imm = 0x100
-; X32-NEXT:    vmovaps %zmm1, {{[0-9]+}}(%esp) ## 64-byte Spill
+; X32-NEXT:    vmovaps %zmm1, {{[-0-9]+}}(%e{{[sb]}}p) ## 64-byte Spill
 ; X32-NEXT:    vaddps %zmm1, %zmm0, %zmm0
 ; X32-NEXT:    leal {{[0-9]+}}(%esp), %eax
 ; X32-NEXT:    movl %eax, (%esp)
 ; X32-NEXT:    calll _func_float16_ptr
-; X32-NEXT:    vaddps {{[0-9]+}}(%esp), %zmm0, %zmm0 ## 64-byte Folded Reload
+; X32-NEXT:    vaddps {{[-0-9]+}}(%e{{[sb]}}p), %zmm0, %zmm0 ## 64-byte Folded Reload
 ; X32-NEXT:    vaddps {{[0-9]+}}(%esp), %zmm0, %zmm0
 ; X32-NEXT:    movl %ebp, %esp
 ; X32-NEXT:    popl %ebp
@@ -184,110 +184,110 @@ define intel_ocl_bicc <16 x float> @test_prolog_epilog(<16 x float> %a, <16 x fl
 ; WIN64-KNL-LABEL: test_prolog_epilog:
 ; WIN64-KNL:       # %bb.0:
 ; WIN64-KNL-NEXT:    pushq %rbp
-; WIN64-KNL-NEXT:    subq $1328, %rsp # imm = 0x530
+; WIN64-KNL-NEXT:    subq $1264, %rsp # imm = 0x4F0
 ; WIN64-KNL-NEXT:    leaq {{[0-9]+}}(%rsp), %rbp
-; WIN64-KNL-NEXT:    kmovw %k7, 1198(%rbp) # 2-byte Spill
-; WIN64-KNL-NEXT:    kmovw %k6, 1196(%rbp) # 2-byte Spill
-; WIN64-KNL-NEXT:    kmovw %k5, 1194(%rbp) # 2-byte Spill
-; WIN64-KNL-NEXT:    kmovw %k4, 1192(%rbp) # 2-byte Spill
-; WIN64-KNL-NEXT:    vmovaps %zmm21, 1104(%rbp) # 64-byte Spill
-; WIN64-KNL-NEXT:    vmovaps %zmm20, 992(%rbp) # 64-byte Spill
-; WIN64-KNL-NEXT:    vmovaps %zmm19, 896(%rbp) # 64-byte Spill
-; WIN64-KNL-NEXT:    vmovaps %zmm18, 832(%rbp) # 64-byte Spill
-; WIN64-KNL-NEXT:    vmovaps %zmm17, 768(%rbp) # 64-byte Spill
-; WIN64-KNL-NEXT:    vmovaps %zmm16, 704(%rbp) # 64-byte Spill
-; WIN64-KNL-NEXT:    vmovaps %zmm15, 640(%rbp) # 64-byte Spill
-; WIN64-KNL-NEXT:    vmovaps %zmm14, 576(%rbp) # 64-byte Spill
-; WIN64-KNL-NEXT:    vmovaps %zmm13, 512(%rbp) # 64-byte Spill
-; WIN64-KNL-NEXT:    vmovaps %zmm12, 448(%rbp) # 64-byte Spill
-; WIN64-KNL-NEXT:    vmovaps %zmm11, 384(%rbp) # 64-byte Spill
-; WIN64-KNL-NEXT:    vmovaps %zmm10, 320(%rbp) # 64-byte Spill
-; WIN64-KNL-NEXT:    vmovaps %zmm9, 256(%rbp) # 64-byte Spill
-; WIN64-KNL-NEXT:    vmovaps %zmm8, 192(%rbp) # 64-byte Spill
-; WIN64-KNL-NEXT:    vmovaps %zmm7, 128(%rbp) # 64-byte Spill
-; WIN64-KNL-NEXT:    vmovaps %zmm6, 64(%rbp) # 64-byte Spill
+; WIN64-KNL-NEXT:    kmovw %k7, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; WIN64-KNL-NEXT:    kmovw %k6, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; WIN64-KNL-NEXT:    kmovw %k5, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; WIN64-KNL-NEXT:    kmovw %k4, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; WIN64-KNL-NEXT:    vmovaps %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; WIN64-KNL-NEXT:    vmovaps %zmm20, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; WIN64-KNL-NEXT:    vmovaps %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; WIN64-KNL-NEXT:    vmovaps %zmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; WIN64-KNL-NEXT:    vmovaps %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; WIN64-KNL-NEXT:    vmovaps %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; WIN64-KNL-NEXT:    vmovaps %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; WIN64-KNL-NEXT:    vmovaps %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; WIN64-KNL-NEXT:    vmovaps %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; WIN64-KNL-NEXT:    vmovaps %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; WIN64-KNL-NEXT:    vmovaps %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; WIN64-KNL-NEXT:    vmovaps %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; WIN64-KNL-NEXT:    vmovaps %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; WIN64-KNL-NEXT:    vmovaps %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; WIN64-KNL-NEXT:    vmovaps %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; WIN64-KNL-NEXT:    vmovaps %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; WIN64-KNL-NEXT:    andq $-64, %rsp
 ; WIN64-KNL-NEXT:    vmovaps %zmm1, {{[0-9]+}}(%rsp)
 ; WIN64-KNL-NEXT:    vmovaps %zmm0, {{[0-9]+}}(%rsp)
 ; WIN64-KNL-NEXT:    leaq {{[0-9]+}}(%rsp), %rcx
 ; WIN64-KNL-NEXT:    leaq {{[0-9]+}}(%rsp), %rdx
 ; WIN64-KNL-NEXT:    callq func_float16
-; WIN64-KNL-NEXT:    vmovaps 64(%rbp), %zmm6 # 64-byte Reload
-; WIN64-KNL-NEXT:    vmovaps 128(%rbp), %zmm7 # 64-byte Reload
-; WIN64-KNL-NEXT:    vmovaps 192(%rbp), %zmm8 # 64-byte Reload
-; WIN64-KNL-NEXT:    vmovaps 256(%rbp), %zmm9 # 64-byte Reload
-; WIN64-KNL-NEXT:    vmovaps 320(%rbp), %zmm10 # 64-byte Reload
-; WIN64-KNL-NEXT:    vmovaps 384(%rbp), %zmm11 # 64-byte Reload
-; WIN64-KNL-NEXT:    vmovaps 448(%rbp), %zmm12 # 64-byte Reload
-; WIN64-KNL-NEXT:    vmovaps 512(%rbp), %zmm13 # 64-byte Reload
-; WIN64-KNL-NEXT:    vmovaps 576(%rbp), %zmm14 # 64-byte Reload
-; WIN64-KNL-NEXT:    vmovaps 640(%rbp), %zmm15 # 64-byte Reload
-; WIN64-KNL-NEXT:    vmovaps 704(%rbp), %zmm16 # 64-byte Reload
-; WIN64-KNL-NEXT:    vmovaps 768(%rbp), %zmm17 # 64-byte Reload
-; WIN64-KNL-NEXT:    vmovaps 832(%rbp), %zmm18 # 64-byte Reload
-; WIN64-KNL-NEXT:    vmovaps 896(%rbp), %zmm19 # 64-byte Reload
-; WIN64-KNL-NEXT:    vmovaps 992(%rbp), %zmm20 # 64-byte Reload
-; WIN64-KNL-NEXT:    vmovaps 1104(%rbp), %zmm21 # 64-byte Reload
-; WIN64-KNL-NEXT:    kmovw 1192(%rbp), %k4 # 2-byte Reload
-; WIN64-KNL-NEXT:    kmovw 1194(%rbp), %k5 # 2-byte Reload
-; WIN64-KNL-NEXT:    kmovw 1196(%rbp), %k6 # 2-byte Reload
-; WIN64-KNL-NEXT:    kmovw 1198(%rbp), %k7 # 2-byte Reload
-; WIN64-KNL-NEXT:    leaq 1200(%rbp), %rsp
+; WIN64-KNL-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; WIN64-KNL-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; WIN64-KNL-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; WIN64-KNL-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
+; WIN64-KNL-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
+; WIN64-KNL-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
+; WIN64-KNL-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; WIN64-KNL-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
+; WIN64-KNL-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
+; WIN64-KNL-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
+; WIN64-KNL-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
+; WIN64-KNL-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
+; WIN64-KNL-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 # 64-byte Reload
+; WIN64-KNL-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
+; WIN64-KNL-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %zmm20 # 64-byte Reload
+; WIN64-KNL-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %zmm21 # 64-byte Reload
+; WIN64-KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k4 # 2-byte Reload
+; WIN64-KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k5 # 2-byte Reload
+; WIN64-KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k6 # 2-byte Reload
+; WIN64-KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 # 2-byte Reload
+; WIN64-KNL-NEXT:    leaq 1136(%rbp), %rsp
 ; WIN64-KNL-NEXT:    popq %rbp
 ; WIN64-KNL-NEXT:    retq
 ;
 ; WIN64-SKX-LABEL: test_prolog_epilog:
 ; WIN64-SKX:       # %bb.0:
 ; WIN64-SKX-NEXT:    pushq %rbp
-; WIN64-SKX-NEXT:    subq $1328, %rsp # imm = 0x530
+; WIN64-SKX-NEXT:    subq $1264, %rsp # imm = 0x4F0
 ; WIN64-SKX-NEXT:    leaq {{[0-9]+}}(%rsp), %rbp
-; WIN64-SKX-NEXT:    kmovq %k7, 1192(%rbp) # 8-byte Spill
-; WIN64-SKX-NEXT:    kmovq %k6, 1184(%rbp) # 8-byte Spill
-; WIN64-SKX-NEXT:    kmovq %k5, 1176(%rbp) # 8-byte Spill
-; WIN64-SKX-NEXT:    kmovq %k4, 1168(%rbp) # 8-byte Spill
-; WIN64-SKX-NEXT:    vmovaps %zmm21, 1056(%rbp) # 64-byte Spill
-; WIN64-SKX-NEXT:    vmovaps %zmm20, 960(%rbp) # 64-byte Spill
-; WIN64-SKX-NEXT:    vmovaps %zmm19, 896(%rbp) # 64-byte Spill
-; WIN64-SKX-NEXT:    vmovaps %zmm18, 832(%rbp) # 64-byte Spill
-; WIN64-SKX-NEXT:    vmovaps %zmm17, 768(%rbp) # 64-byte Spill
-; WIN64-SKX-NEXT:    vmovaps %zmm16, 704(%rbp) # 64-byte Spill
-; WIN64-SKX-NEXT:    vmovaps %zmm15, 640(%rbp) # 64-byte Spill
-; WIN64-SKX-NEXT:    vmovaps %zmm14, 576(%rbp) # 64-byte Spill
-; WIN64-SKX-NEXT:    vmovaps %zmm13, 512(%rbp) # 64-byte Spill
-; WIN64-SKX-NEXT:    vmovaps %zmm12, 448(%rbp) # 64-byte Spill
-; WIN64-SKX-NEXT:    vmovaps %zmm11, 384(%rbp) # 64-byte Spill
-; WIN64-SKX-NEXT:    vmovaps %zmm10, 320(%rbp) # 64-byte Spill
-; WIN64-SKX-NEXT:    vmovaps %zmm9, 256(%rbp) # 64-byte Spill
-; WIN64-SKX-NEXT:    vmovaps %zmm8, 192(%rbp) # 64-byte Spill
-; WIN64-SKX-NEXT:    vmovaps %zmm7, 128(%rbp) # 64-byte Spill
-; WIN64-SKX-NEXT:    vmovaps %zmm6, 64(%rbp) # 64-byte Spill
+; WIN64-SKX-NEXT:    kmovq %k7, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; WIN64-SKX-NEXT:    kmovq %k6, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; WIN64-SKX-NEXT:    kmovq %k5, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; WIN64-SKX-NEXT:    kmovq %k4, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; WIN64-SKX-NEXT:    vmovaps %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; WIN64-SKX-NEXT:    vmovaps %zmm20, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; WIN64-SKX-NEXT:    vmovaps %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; WIN64-SKX-NEXT:    vmovaps %zmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; WIN64-SKX-NEXT:    vmovaps %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; WIN64-SKX-NEXT:    vmovaps %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; WIN64-SKX-NEXT:    vmovaps %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; WIN64-SKX-NEXT:    vmovaps %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; WIN64-SKX-NEXT:    vmovaps %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; WIN64-SKX-NEXT:    vmovaps %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; WIN64-SKX-NEXT:    vmovaps %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; WIN64-SKX-NEXT:    vmovaps %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; WIN64-SKX-NEXT:    vmovaps %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; WIN64-SKX-NEXT:    vmovaps %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; WIN64-SKX-NEXT:    vmovaps %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; WIN64-SKX-NEXT:    vmovaps %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; WIN64-SKX-NEXT:    andq $-64, %rsp
 ; WIN64-SKX-NEXT:    vmovaps %zmm1, {{[0-9]+}}(%rsp)
 ; WIN64-SKX-NEXT:    vmovaps %zmm0, {{[0-9]+}}(%rsp)
 ; WIN64-SKX-NEXT:    leaq {{[0-9]+}}(%rsp), %rcx
 ; WIN64-SKX-NEXT:    leaq {{[0-9]+}}(%rsp), %rdx
 ; WIN64-SKX-NEXT:    callq func_float16
-; WIN64-SKX-NEXT:    vmovaps 64(%rbp), %zmm6 # 64-byte Reload
-; WIN64-SKX-NEXT:    vmovaps 128(%rbp), %zmm7 # 64-byte Reload
-; WIN64-SKX-NEXT:    vmovaps 192(%rbp), %zmm8 # 64-byte Reload
-; WIN64-SKX-NEXT:    vmovaps 256(%rbp), %zmm9 # 64-byte Reload
-; WIN64-SKX-NEXT:    vmovaps 320(%rbp), %zmm10 # 64-byte Reload
-; WIN64-SKX-NEXT:    vmovaps 384(%rbp), %zmm11 # 64-byte Reload
-; WIN64-SKX-NEXT:    vmovaps 448(%rbp), %zmm12 # 64-byte Reload
-; WIN64-SKX-NEXT:    vmovaps 512(%rbp), %zmm13 # 64-byte Reload
-; WIN64-SKX-NEXT:    vmovaps 576(%rbp), %zmm14 # 64-byte Reload
-; WIN64-SKX-NEXT:    vmovaps 640(%rbp), %zmm15 # 64-byte Reload
-; WIN64-SKX-NEXT:    vmovaps 704(%rbp), %zmm16 # 64-byte Reload
-; WIN64-SKX-NEXT:    vmovaps 768(%rbp), %zmm17 # 64-byte Reload
-; WIN64-SKX-NEXT:    vmovaps 832(%rbp), %zmm18 # 64-byte Reload
-; WIN64-SKX-NEXT:    vmovaps 896(%rbp), %zmm19 # 64-byte Reload
-; WIN64-SKX-NEXT:    vmovaps 960(%rbp), %zmm20 # 64-byte Reload
-; WIN64-SKX-NEXT:    vmovaps 1056(%rbp), %zmm21 # 64-byte Reload
-; WIN64-SKX-NEXT:    kmovq 1168(%rbp), %k4 # 8-byte Reload
-; WIN64-SKX-NEXT:    kmovq 1176(%rbp), %k5 # 8-byte Reload
-; WIN64-SKX-NEXT:    kmovq 1184(%rbp), %k6 # 8-byte Reload
-; WIN64-SKX-NEXT:    kmovq 1192(%rbp), %k7 # 8-byte Reload
-; WIN64-SKX-NEXT:    leaq 1200(%rbp), %rsp
+; WIN64-SKX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; WIN64-SKX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; WIN64-SKX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; WIN64-SKX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
+; WIN64-SKX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
+; WIN64-SKX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
+; WIN64-SKX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; WIN64-SKX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
+; WIN64-SKX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
+; WIN64-SKX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
+; WIN64-SKX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
+; WIN64-SKX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
+; WIN64-SKX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 # 64-byte Reload
+; WIN64-SKX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
+; WIN64-SKX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %zmm20 # 64-byte Reload
+; WIN64-SKX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %zmm21 # 64-byte Reload
+; WIN64-SKX-NEXT:    kmovq {{[-0-9]+}}(%r{{[sb]}}p), %k4 # 8-byte Reload
+; WIN64-SKX-NEXT:    kmovq {{[-0-9]+}}(%r{{[sb]}}p), %k5 # 8-byte Reload
+; WIN64-SKX-NEXT:    kmovq {{[-0-9]+}}(%r{{[sb]}}p), %k6 # 8-byte Reload
+; WIN64-SKX-NEXT:    kmovq {{[-0-9]+}}(%r{{[sb]}}p), %k7 # 8-byte Reload
+; WIN64-SKX-NEXT:    leaq 1136(%rbp), %rsp
 ; WIN64-SKX-NEXT:    popq %rbp
 ; WIN64-SKX-NEXT:    retq
 ;
@@ -296,47 +296,47 @@ define intel_ocl_bicc <16 x float> @test_prolog_epilog(<16 x float> %a, <16 x fl
 ; X64-KNL-NEXT:    pushq %rsi
 ; X64-KNL-NEXT:    pushq %rdi
 ; X64-KNL-NEXT:    subq $1064, %rsp ## imm = 0x428
-; X64-KNL-NEXT:    kmovw %k7, {{[0-9]+}}(%rsp) ## 2-byte Spill
-; X64-KNL-NEXT:    kmovw %k6, {{[0-9]+}}(%rsp) ## 2-byte Spill
-; X64-KNL-NEXT:    kmovw %k5, {{[0-9]+}}(%rsp) ## 2-byte Spill
-; X64-KNL-NEXT:    kmovw %k4, {{[0-9]+}}(%rsp) ## 2-byte Spill
-; X64-KNL-NEXT:    vmovups %zmm31, {{[0-9]+}}(%rsp) ## 64-byte Spill
-; X64-KNL-NEXT:    vmovups %zmm30, {{[0-9]+}}(%rsp) ## 64-byte Spill
-; X64-KNL-NEXT:    vmovups %zmm29, {{[0-9]+}}(%rsp) ## 64-byte Spill
-; X64-KNL-NEXT:    vmovups %zmm28, {{[0-9]+}}(%rsp) ## 64-byte Spill
-; X64-KNL-NEXT:    vmovups %zmm27, {{[0-9]+}}(%rsp) ## 64-byte Spill
-; X64-KNL-NEXT:    vmovups %zmm26, {{[0-9]+}}(%rsp) ## 64-byte Spill
-; X64-KNL-NEXT:    vmovups %zmm25, {{[0-9]+}}(%rsp) ## 64-byte Spill
-; X64-KNL-NEXT:    vmovups %zmm24, {{[0-9]+}}(%rsp) ## 64-byte Spill
-; X64-KNL-NEXT:    vmovups %zmm23, {{[0-9]+}}(%rsp) ## 64-byte Spill
-; X64-KNL-NEXT:    vmovups %zmm22, {{[0-9]+}}(%rsp) ## 64-byte Spill
-; X64-KNL-NEXT:    vmovups %zmm21, {{[0-9]+}}(%rsp) ## 64-byte Spill
-; X64-KNL-NEXT:    vmovups %zmm20, {{[0-9]+}}(%rsp) ## 64-byte Spill
-; X64-KNL-NEXT:    vmovups %zmm19, {{[0-9]+}}(%rsp) ## 64-byte Spill
-; X64-KNL-NEXT:    vmovups %zmm18, {{[0-9]+}}(%rsp) ## 64-byte Spill
-; X64-KNL-NEXT:    vmovups %zmm17, {{[0-9]+}}(%rsp) ## 64-byte Spill
+; X64-KNL-NEXT:    kmovw %k7, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; X64-KNL-NEXT:    kmovw %k6, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; X64-KNL-NEXT:    kmovw %k5, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; X64-KNL-NEXT:    kmovw %k4, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; X64-KNL-NEXT:    vmovups %zmm31, {{[-0-9]+}}(%r{{[sb]}}p) ## 64-byte Spill
+; X64-KNL-NEXT:    vmovups %zmm30, {{[-0-9]+}}(%r{{[sb]}}p) ## 64-byte Spill
+; X64-KNL-NEXT:    vmovups %zmm29, {{[-0-9]+}}(%r{{[sb]}}p) ## 64-byte Spill
+; X64-KNL-NEXT:    vmovups %zmm28, {{[-0-9]+}}(%r{{[sb]}}p) ## 64-byte Spill
+; X64-KNL-NEXT:    vmovups %zmm27, {{[-0-9]+}}(%r{{[sb]}}p) ## 64-byte Spill
+; X64-KNL-NEXT:    vmovups %zmm26, {{[-0-9]+}}(%r{{[sb]}}p) ## 64-byte Spill
+; X64-KNL-NEXT:    vmovups %zmm25, {{[-0-9]+}}(%r{{[sb]}}p) ## 64-byte Spill
+; X64-KNL-NEXT:    vmovups %zmm24, {{[-0-9]+}}(%r{{[sb]}}p) ## 64-byte Spill
+; X64-KNL-NEXT:    vmovups %zmm23, {{[-0-9]+}}(%r{{[sb]}}p) ## 64-byte Spill
+; X64-KNL-NEXT:    vmovups %zmm22, {{[-0-9]+}}(%r{{[sb]}}p) ## 64-byte Spill
+; X64-KNL-NEXT:    vmovups %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) ## 64-byte Spill
+; X64-KNL-NEXT:    vmovups %zmm20, {{[-0-9]+}}(%r{{[sb]}}p) ## 64-byte Spill
+; X64-KNL-NEXT:    vmovups %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) ## 64-byte Spill
+; X64-KNL-NEXT:    vmovups %zmm18, {{[-0-9]+}}(%r{{[sb]}}p) ## 64-byte Spill
+; X64-KNL-NEXT:    vmovups %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) ## 64-byte Spill
 ; X64-KNL-NEXT:    vmovups %zmm16, (%rsp) ## 64-byte Spill
 ; X64-KNL-NEXT:    callq _func_float16
 ; X64-KNL-NEXT:    vmovups (%rsp), %zmm16 ## 64-byte Reload
-; X64-KNL-NEXT:    vmovups {{[0-9]+}}(%rsp), %zmm17 ## 64-byte Reload
-; X64-KNL-NEXT:    vmovups {{[0-9]+}}(%rsp), %zmm18 ## 64-byte Reload
-; X64-KNL-NEXT:    vmovups {{[0-9]+}}(%rsp), %zmm19 ## 64-byte Reload
-; X64-KNL-NEXT:    vmovups {{[0-9]+}}(%rsp), %zmm20 ## 64-byte Reload
-; X64-KNL-NEXT:    vmovups {{[0-9]+}}(%rsp), %zmm21 ## 64-byte Reload
-; X64-KNL-NEXT:    vmovups {{[0-9]+}}(%rsp), %zmm22 ## 64-byte Reload
-; X64-KNL-NEXT:    vmovups {{[0-9]+}}(%rsp), %zmm23 ## 64-byte Reload
-; X64-KNL-NEXT:    vmovups {{[0-9]+}}(%rsp), %zmm24 ## 64-byte Reload
-; X64-KNL-NEXT:    vmovups {{[0-9]+}}(%rsp), %zmm25 ## 64-byte Reload
-; X64-KNL-NEXT:    vmovups {{[0-9]+}}(%rsp), %zmm26 ## 64-byte Reload
-; X64-KNL-NEXT:    vmovups {{[0-9]+}}(%rsp), %zmm27 ## 64-byte Reload
-; X64-KNL-NEXT:    vmovups {{[0-9]+}}(%rsp), %zmm28 ## 64-byte Reload
-; X64-KNL-NEXT:    vmovups {{[0-9]+}}(%rsp), %zmm29 ## 64-byte Reload
-; X64-KNL-NEXT:    vmovups {{[0-9]+}}(%rsp), %zmm30 ## 64-byte Reload
-; X64-KNL-NEXT:    vmovups {{[0-9]+}}(%rsp), %zmm31 ## 64-byte Reload
-; X64-KNL-NEXT:    kmovw {{[0-9]+}}(%rsp), %k4 ## 2-byte Reload
-; X64-KNL-NEXT:    kmovw {{[0-9]+}}(%rsp), %k5 ## 2-byte Reload
-; X64-KNL-NEXT:    kmovw {{[0-9]+}}(%rsp), %k6 ## 2-byte Reload
-; X64-KNL-NEXT:    kmovw {{[0-9]+}}(%rsp), %k7 ## 2-byte Reload
+; X64-KNL-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 ## 64-byte Reload
+; X64-KNL-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 ## 64-byte Reload
+; X64-KNL-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 ## 64-byte Reload
+; X64-KNL-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm20 ## 64-byte Reload
+; X64-KNL-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm21 ## 64-byte Reload
+; X64-KNL-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm22 ## 64-byte Reload
+; X64-KNL-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm23 ## 64-byte Reload
+; X64-KNL-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm24 ## 64-byte Reload
+; X64-KNL-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm25 ## 64-byte Reload
+; X64-KNL-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm26 ## 64-byte Reload
+; X64-KNL-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm27 ## 64-byte Reload
+; X64-KNL-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm28 ## 64-byte Reload
+; X64-KNL-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm29 ## 64-byte Reload
+; X64-KNL-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm30 ## 64-byte Reload
+; X64-KNL-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm31 ## 64-byte Reload
+; X64-KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k4 ## 2-byte Reload
+; X64-KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k5 ## 2-byte Reload
+; X64-KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k6 ## 2-byte Reload
+; X64-KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 ## 2-byte Reload
 ; X64-KNL-NEXT:    addq $1064, %rsp ## imm = 0x428
 ; X64-KNL-NEXT:    popq %rdi
 ; X64-KNL-NEXT:    popq %rsi
@@ -346,49 +346,49 @@ define intel_ocl_bicc <16 x float> @test_prolog_epilog(<16 x float> %a, <16 x fl
 ; X64-SKX:       ## %bb.0:
 ; X64-SKX-NEXT:    pushq %rsi
 ; X64-SKX-NEXT:    pushq %rdi
-; X64-SKX-NEXT:    subq $1192, %rsp ## imm = 0x4A8
-; X64-SKX-NEXT:    kmovq %k7, {{[0-9]+}}(%rsp) ## 8-byte Spill
-; X64-SKX-NEXT:    kmovq %k6, {{[0-9]+}}(%rsp) ## 8-byte Spill
-; X64-SKX-NEXT:    kmovq %k5, {{[0-9]+}}(%rsp) ## 8-byte Spill
-; X64-SKX-NEXT:    kmovq %k4, {{[0-9]+}}(%rsp) ## 8-byte Spill
-; X64-SKX-NEXT:    vmovups %zmm31, {{[0-9]+}}(%rsp) ## 64-byte Spill
-; X64-SKX-NEXT:    vmovups %zmm30, {{[0-9]+}}(%rsp) ## 64-byte Spill
-; X64-SKX-NEXT:    vmovups %zmm29, {{[0-9]+}}(%rsp) ## 64-byte Spill
-; X64-SKX-NEXT:    vmovups %zmm28, {{[0-9]+}}(%rsp) ## 64-byte Spill
-; X64-SKX-NEXT:    vmovups %zmm27, {{[0-9]+}}(%rsp) ## 64-byte Spill
-; X64-SKX-NEXT:    vmovups %zmm26, {{[0-9]+}}(%rsp) ## 64-byte Spill
-; X64-SKX-NEXT:    vmovups %zmm25, {{[0-9]+}}(%rsp) ## 64-byte Spill
-; X64-SKX-NEXT:    vmovups %zmm24, {{[0-9]+}}(%rsp) ## 64-byte Spill
-; X64-SKX-NEXT:    vmovups %zmm23, {{[0-9]+}}(%rsp) ## 64-byte Spill
-; X64-SKX-NEXT:    vmovups %zmm22, {{[0-9]+}}(%rsp) ## 64-byte Spill
-; X64-SKX-NEXT:    vmovups %zmm21, {{[0-9]+}}(%rsp) ## 64-byte Spill
-; X64-SKX-NEXT:    vmovups %zmm20, {{[0-9]+}}(%rsp) ## 64-byte Spill
-; X64-SKX-NEXT:    vmovups %zmm19, {{[0-9]+}}(%rsp) ## 64-byte Spill
-; X64-SKX-NEXT:    vmovups %zmm18, {{[0-9]+}}(%rsp) ## 64-byte Spill
-; X64-SKX-NEXT:    vmovups %zmm17, {{[0-9]+}}(%rsp) ## 64-byte Spill
+; X64-SKX-NEXT:    subq $1064, %rsp ## imm = 0x428
+; X64-SKX-NEXT:    kmovq %k7, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
+; X64-SKX-NEXT:    kmovq %k6, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
+; X64-SKX-NEXT:    kmovq %k5, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
+; X64-SKX-NEXT:    kmovq %k4, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
+; X64-SKX-NEXT:    vmovups %zmm31, {{[-0-9]+}}(%r{{[sb]}}p) ## 64-byte Spill
+; X64-SKX-NEXT:    vmovups %zmm30, {{[-0-9]+}}(%r{{[sb]}}p) ## 64-byte Spill
+; X64-SKX-NEXT:    vmovups %zmm29, {{[-0-9]+}}(%r{{[sb]}}p) ## 64-byte Spill
+; X64-SKX-NEXT:    vmovups %zmm28, {{[-0-9]+}}(%r{{[sb]}}p) ## 64-byte Spill
+; X64-SKX-NEXT:    vmovups %zmm27, {{[-0-9]+}}(%r{{[sb]}}p) ## 64-byte Spill
+; X64-SKX-NEXT:    vmovups %zmm26, {{[-0-9]+}}(%r{{[sb]}}p) ## 64-byte Spill
+; X64-SKX-NEXT:    vmovups %zmm25, {{[-0-9]+}}(%r{{[sb]}}p) ## 64-byte Spill
+; X64-SKX-NEXT:    vmovups %zmm24, {{[-0-9]+}}(%r{{[sb]}}p) ## 64-byte Spill
+; X64-SKX-NEXT:    vmovups %zmm23, {{[-0-9]+}}(%r{{[sb]}}p) ## 64-byte Spill
+; X64-SKX-NEXT:    vmovups %zmm22, {{[-0-9]+}}(%r{{[sb]}}p) ## 64-byte Spill
+; X64-SKX-NEXT:    vmovups %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) ## 64-byte Spill
+; X64-SKX-NEXT:    vmovups %zmm20, {{[-0-9]+}}(%r{{[sb]}}p) ## 64-byte Spill
+; X64-SKX-NEXT:    vmovups %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) ## 64-byte Spill
+; X64-SKX-NEXT:    vmovups %zmm18, {{[-0-9]+}}(%r{{[sb]}}p) ## 64-byte Spill
+; X64-SKX-NEXT:    vmovups %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) ## 64-byte Spill
 ; X64-SKX-NEXT:    vmovups %zmm16, (%rsp) ## 64-byte Spill
 ; X64-SKX-NEXT:    callq _func_float16
 ; X64-SKX-NEXT:    vmovups (%rsp), %zmm16 ## 64-byte Reload
-; X64-SKX-NEXT:    vmovups {{[0-9]+}}(%rsp), %zmm17 ## 64-byte Reload
-; X64-SKX-NEXT:    vmovups {{[0-9]+}}(%rsp), %zmm18 ## 64-byte Reload
-; X64-SKX-NEXT:    vmovups {{[0-9]+}}(%rsp), %zmm19 ## 64-byte Reload
-; X64-SKX-NEXT:    vmovups {{[0-9]+}}(%rsp), %zmm20 ## 64-byte Reload
-; X64-SKX-NEXT:    vmovups {{[0-9]+}}(%rsp), %zmm21 ## 64-byte Reload
-; X64-SKX-NEXT:    vmovups {{[0-9]+}}(%rsp), %zmm22 ## 64-byte Reload
-; X64-SKX-NEXT:    vmovups {{[0-9]+}}(%rsp), %zmm23 ## 64-byte Reload
-; X64-SKX-NEXT:    vmovups {{[0-9]+}}(%rsp), %zmm24 ## 64-byte Reload
-; X64-SKX-NEXT:    vmovups {{[0-9]+}}(%rsp), %zmm25 ## 64-byte Reload
-; X64-SKX-NEXT:    vmovups {{[0-9]+}}(%rsp), %zmm26 ## 64-byte Reload
-; X64-SKX-NEXT:    vmovups {{[0-9]+}}(%rsp), %zmm27 ## 64-byte Reload
-; X64-SKX-NEXT:    vmovups {{[0-9]+}}(%rsp), %zmm28 ## 64-byte Reload
-; X64-SKX-NEXT:    vmovups {{[0-9]+}}(%rsp), %zmm29 ## 64-byte Reload
-; X64-SKX-NEXT:    vmovups {{[0-9]+}}(%rsp), %zmm30 ## 64-byte Reload
-; X64-SKX-NEXT:    vmovups {{[0-9]+}}(%rsp), %zmm31 ## 64-byte Reload
-; X64-SKX-NEXT:    kmovq {{[0-9]+}}(%rsp), %k4 ## 8-byte Reload
-; X64-SKX-NEXT:    kmovq {{[0-9]+}}(%rsp), %k5 ## 8-byte Reload
-; X64-SKX-NEXT:    kmovq {{[0-9]+}}(%rsp), %k6 ## 8-byte Reload
-; X64-SKX-NEXT:    kmovq {{[0-9]+}}(%rsp), %k7 ## 8-byte Reload
-; X64-SKX-NEXT:    addq $1192, %rsp ## imm = 0x4A8
+; X64-SKX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 ## 64-byte Reload
+; X64-SKX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 ## 64-byte Reload
+; X64-SKX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 ## 64-byte Reload
+; X64-SKX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm20 ## 64-byte Reload
+; X64-SKX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm21 ## 64-byte Reload
+; X64-SKX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm22 ## 64-byte Reload
+; X64-SKX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm23 ## 64-byte Reload
+; X64-SKX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm24 ## 64-byte Reload
+; X64-SKX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm25 ## 64-byte Reload
+; X64-SKX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm26 ## 64-byte Reload
+; X64-SKX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm27 ## 64-byte Reload
+; X64-SKX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm28 ## 64-byte Reload
+; X64-SKX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm29 ## 64-byte Reload
+; X64-SKX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm30 ## 64-byte Reload
+; X64-SKX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm31 ## 64-byte Reload
+; X64-SKX-NEXT:    kmovq {{[-0-9]+}}(%r{{[sb]}}p), %k4 ## 8-byte Reload
+; X64-SKX-NEXT:    kmovq {{[-0-9]+}}(%r{{[sb]}}p), %k5 ## 8-byte Reload
+; X64-SKX-NEXT:    kmovq {{[-0-9]+}}(%r{{[sb]}}p), %k6 ## 8-byte Reload
+; X64-SKX-NEXT:    kmovq {{[-0-9]+}}(%r{{[sb]}}p), %k7 ## 8-byte Reload
+; X64-SKX-NEXT:    addq $1064, %rsp ## imm = 0x428
 ; X64-SKX-NEXT:    popq %rdi
 ; X64-SKX-NEXT:    popq %rsi
 ; X64-SKX-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/avx512-intrinsics-fast-isel.ll b/llvm/test/CodeGen/X86/avx512-intrinsics-fast-isel.ll
index d7611d1de..a40273dce 100644
--- a/llvm/test/CodeGen/X86/avx512-intrinsics-fast-isel.ll
+++ b/llvm/test/CodeGen/X86/avx512-intrinsics-fast-isel.ll
@@ -1841,7 +1841,8 @@ define <2 x double> @test_mm_cvtu64_sd(<2 x double> %__A, i64 %__B) {
 ; X86-NEXT:    vpinsrd $1, {{[0-9]+}}(%esp), %xmm1, %xmm1
 ; X86-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]
 ; X86-NEXT:    vsubpd {{\.LCPI.*}}, %xmm1, %xmm1
-; X86-NEXT:    vhaddpd %xmm1, %xmm1, %xmm1
+; X86-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm1[1,0]
+; X86-NEXT:    vaddsd %xmm1, %xmm2, %xmm1
 ; X86-NEXT:    vblendpd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
 ; X86-NEXT:    retl
 ;
@@ -1885,11 +1886,9 @@ define <4 x float> @test_mm_cvtu64_ss(<4 x float> %__A, i64 %__B) {
 ; X86-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; X86-NEXT:    vpinsrd $1, %eax, %xmm1, %xmm1
 ; X86-NEXT:    vmovq %xmm1, {{[0-9]+}}(%esp)
-; X86-NEXT:    xorl %ecx, %ecx
-; X86-NEXT:    testl %eax, %eax
-; X86-NEXT:    setns %cl
+; X86-NEXT:    shrl $31, %eax
 ; X86-NEXT:    fildll {{[0-9]+}}(%esp)
-; X86-NEXT:    fadds {{\.LCPI.*}}(,%ecx,4)
+; X86-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
 ; X86-NEXT:    fstps {{[0-9]+}}(%esp)
 ; X86-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; X86-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
@@ -8319,7 +8318,7 @@ define float @test_mm512_reduce_max_ps(<16 x float> %__W) {
 ; X86-NEXT:    vmaxps %xmm1, %xmm0, %xmm0
 ; X86-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
 ; X86-NEXT:    vmaxps %xmm1, %xmm0, %xmm0
-; X86-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,0,3,2]
+; X86-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
 ; X86-NEXT:    vmaxss %xmm1, %xmm0, %xmm0
 ; X86-NEXT:    vmovss %xmm0, (%esp)
 ; X86-NEXT:    flds (%esp)
@@ -8336,7 +8335,7 @@ define float @test_mm512_reduce_max_ps(<16 x float> %__W) {
 ; X64-NEXT:    vmaxps %xmm1, %xmm0, %xmm0
 ; X64-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
 ; X64-NEXT:    vmaxps %xmm1, %xmm0, %xmm0
-; X64-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,0,3,2]
+; X64-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
 ; X64-NEXT:    vmaxss %xmm1, %xmm0, %xmm0
 ; X64-NEXT:    vzeroupper
 ; X64-NEXT:    retq
@@ -8445,7 +8444,7 @@ define float @test_mm512_reduce_min_ps(<16 x float> %__W) {
 ; X86-NEXT:    vminps %xmm1, %xmm0, %xmm0
 ; X86-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
 ; X86-NEXT:    vminps %xmm1, %xmm0, %xmm0
-; X86-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,0,3,2]
+; X86-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
 ; X86-NEXT:    vminss %xmm1, %xmm0, %xmm0
 ; X86-NEXT:    vmovss %xmm0, (%esp)
 ; X86-NEXT:    flds (%esp)
@@ -8462,7 +8461,7 @@ define float @test_mm512_reduce_min_ps(<16 x float> %__W) {
 ; X64-NEXT:    vminps %xmm1, %xmm0, %xmm0
 ; X64-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
 ; X64-NEXT:    vminps %xmm1, %xmm0, %xmm0
-; X64-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,0,3,2]
+; X64-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
 ; X64-NEXT:    vminss %xmm1, %xmm0, %xmm0
 ; X64-NEXT:    vzeroupper
 ; X64-NEXT:    retq
@@ -8623,7 +8622,7 @@ define float @test_mm512_mask_reduce_max_ps(i16 zeroext %__M, <16 x float> %__W)
 ; X86-NEXT:    vmaxps %xmm1, %xmm0, %xmm0
 ; X86-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
 ; X86-NEXT:    vmaxps %xmm1, %xmm0, %xmm0
-; X86-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,0,3,2]
+; X86-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
 ; X86-NEXT:    vmaxss %xmm1, %xmm0, %xmm0
 ; X86-NEXT:    vmovss %xmm0, (%esp)
 ; X86-NEXT:    flds (%esp)
@@ -8643,7 +8642,7 @@ define float @test_mm512_mask_reduce_max_ps(i16 zeroext %__M, <16 x float> %__W)
 ; X64-NEXT:    vmaxps %xmm1, %xmm0, %xmm0
 ; X64-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
 ; X64-NEXT:    vmaxps %xmm1, %xmm0, %xmm0
-; X64-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,0,3,2]
+; X64-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
 ; X64-NEXT:    vmaxss %xmm1, %xmm0, %xmm0
 ; X64-NEXT:    vzeroupper
 ; X64-NEXT:    retq
@@ -8808,7 +8807,7 @@ define float @test_mm512_mask_reduce_min_ps(i16 zeroext %__M, <16 x float> %__W)
 ; X86-NEXT:    vminps %xmm1, %xmm0, %xmm0
 ; X86-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
 ; X86-NEXT:    vminps %xmm1, %xmm0, %xmm0
-; X86-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,0,3,2]
+; X86-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
 ; X86-NEXT:    vminss %xmm1, %xmm0, %xmm0
 ; X86-NEXT:    vmovss %xmm0, (%esp)
 ; X86-NEXT:    flds (%esp)
@@ -8828,7 +8827,7 @@ define float @test_mm512_mask_reduce_min_ps(i16 zeroext %__M, <16 x float> %__W)
 ; X64-NEXT:    vminps %xmm1, %xmm0, %xmm0
 ; X64-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
 ; X64-NEXT:    vminps %xmm1, %xmm0, %xmm0
-; X64-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,0,3,2]
+; X64-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
 ; X64-NEXT:    vminss %xmm1, %xmm0, %xmm0
 ; X64-NEXT:    vzeroupper
 ; X64-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/avx512-intrinsics-upgrade.ll b/llvm/test/CodeGen/X86/avx512-intrinsics-upgrade.ll
index 0a1850e50..e7f132bcd 100644
--- a/llvm/test/CodeGen/X86/avx512-intrinsics-upgrade.ll
+++ b/llvm/test/CodeGen/X86/avx512-intrinsics-upgrade.ll
@@ -29,11 +29,10 @@ define i16 @unpckbw_test(i16 %a0, i16 %a1) {
 define <16 x i32>@test_int_x86_avx512_mask_pbroadcastd_gpr_512(i32 %x0, <16 x i32> %x1, i16 %mask) {
 ; X86-LABEL: test_int_x86_avx512_mask_pbroadcastd_gpr_512:
 ; X86:       ## %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax ## encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vpbroadcastd %eax, %zmm1 ## encoding: [0x62,0xf2,0x7d,0x48,0x7c,0xc8]
+; X86-NEXT:    vpbroadcastd {{[0-9]+}}(%esp), %zmm1 ## encoding: [0x62,0xf2,0x7d,0x48,0x58,0x4c,0x24,0x01]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 ## encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
-; X86-NEXT:    vpbroadcastd %eax, %zmm0 {%k1} ## encoding: [0x62,0xf2,0x7d,0x49,0x7c,0xc0]
-; X86-NEXT:    vpbroadcastd %eax, %zmm2 {%k1} {z} ## encoding: [0x62,0xf2,0x7d,0xc9,0x7c,0xd0]
+; X86-NEXT:    vmovdqa32 %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf1,0x7d,0x49,0x6f,0xc1]
+; X86-NEXT:    vmovdqa32 %zmm1, %zmm2 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xc9,0x6f,0xd1]
 ; X86-NEXT:    vpaddd %zmm2, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0x7d,0x48,0xfe,0xc2]
 ; X86-NEXT:    vpaddd %zmm0, %zmm1, %zmm0 ## encoding: [0x62,0xf1,0x75,0x48,0xfe,0xc0]
 ; X86-NEXT:    retl ## encoding: [0xc3]
@@ -2251,8 +2250,7 @@ define <8 x i64> @test_mask_add_epi64_rmb(<8 x i64> %a, i64* %ptr_b) {
 ; X86-LABEL: test_mask_add_epi64_rmb:
 ; X86:       ## %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax ## encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vpbroadcastq (%eax), %zmm1 ## encoding: [0x62,0xf2,0xfd,0x48,0x59,0x08]
-; X86-NEXT:    vpaddq %zmm1, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0xd4,0xc1]
+; X86-NEXT:    vpaddq (%eax){1to8}, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x58,0xd4,0x00]
 ; X86-NEXT:    retl ## encoding: [0xc3]
 ;
 ; X64-LABEL: test_mask_add_epi64_rmb:
@@ -2270,10 +2268,9 @@ define <8 x i64> @test_mask_add_epi64_rmbk(<8 x i64> %a, i64* %ptr_b, <8 x i64>
 ; X86-LABEL: test_mask_add_epi64_rmbk:
 ; X86:       ## %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax ## encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vpbroadcastq (%eax), %zmm2 ## encoding: [0x62,0xf2,0xfd,0x48,0x59,0x10]
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax ## encoding: [0x0f,0xb6,0x44,0x24,0x08]
-; X86-NEXT:    kmovw %eax, %k1 ## encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vpaddq %zmm2, %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0xfd,0x49,0xd4,0xca]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx ## encoding: [0x0f,0xb6,0x4c,0x24,0x08]
+; X86-NEXT:    kmovw %ecx, %k1 ## encoding: [0xc5,0xf8,0x92,0xc9]
+; X86-NEXT:    vpaddq (%eax){1to8}, %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0xfd,0x59,0xd4,0x08]
 ; X86-NEXT:    vmovdqa64 %zmm1, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc1]
 ; X86-NEXT:    retl ## encoding: [0xc3]
 ;
@@ -2294,10 +2291,9 @@ define <8 x i64> @test_mask_add_epi64_rmbkz(<8 x i64> %a, i64* %ptr_b, i8 %mask)
 ; X86-LABEL: test_mask_add_epi64_rmbkz:
 ; X86:       ## %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax ## encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vpbroadcastq (%eax), %zmm1 ## encoding: [0x62,0xf2,0xfd,0x48,0x59,0x08]
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax ## encoding: [0x0f,0xb6,0x44,0x24,0x08]
-; X86-NEXT:    kmovw %eax, %k1 ## encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vpaddq %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0xc9,0xd4,0xc1]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx ## encoding: [0x0f,0xb6,0x4c,0x24,0x08]
+; X86-NEXT:    kmovw %ecx, %k1 ## encoding: [0xc5,0xf8,0x92,0xc9]
+; X86-NEXT:    vpaddq (%eax){1to8}, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0xd9,0xd4,0x00]
 ; X86-NEXT:    retl ## encoding: [0xc3]
 ;
 ; X64-LABEL: test_mask_add_epi64_rmbkz:
@@ -2419,8 +2415,7 @@ define <8 x i64> @test_mask_sub_epi64_rmb(<8 x i64> %a, i64* %ptr_b) {
 ; X86-LABEL: test_mask_sub_epi64_rmb:
 ; X86:       ## %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax ## encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vpbroadcastq (%eax), %zmm1 ## encoding: [0x62,0xf2,0xfd,0x48,0x59,0x08]
-; X86-NEXT:    vpsubq %zmm1, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0xfb,0xc1]
+; X86-NEXT:    vpsubq (%eax){1to8}, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x58,0xfb,0x00]
 ; X86-NEXT:    retl ## encoding: [0xc3]
 ;
 ; X64-LABEL: test_mask_sub_epi64_rmb:
@@ -2438,10 +2433,9 @@ define <8 x i64> @test_mask_sub_epi64_rmbk(<8 x i64> %a, i64* %ptr_b, <8 x i64>
 ; X86-LABEL: test_mask_sub_epi64_rmbk:
 ; X86:       ## %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax ## encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vpbroadcastq (%eax), %zmm2 ## encoding: [0x62,0xf2,0xfd,0x48,0x59,0x10]
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax ## encoding: [0x0f,0xb6,0x44,0x24,0x08]
-; X86-NEXT:    kmovw %eax, %k1 ## encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vpsubq %zmm2, %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0xfd,0x49,0xfb,0xca]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx ## encoding: [0x0f,0xb6,0x4c,0x24,0x08]
+; X86-NEXT:    kmovw %ecx, %k1 ## encoding: [0xc5,0xf8,0x92,0xc9]
+; X86-NEXT:    vpsubq (%eax){1to8}, %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0xfd,0x59,0xfb,0x08]
 ; X86-NEXT:    vmovdqa64 %zmm1, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc1]
 ; X86-NEXT:    retl ## encoding: [0xc3]
 ;
@@ -2462,10 +2456,9 @@ define <8 x i64> @test_mask_sub_epi64_rmbkz(<8 x i64> %a, i64* %ptr_b, i8 %mask)
 ; X86-LABEL: test_mask_sub_epi64_rmbkz:
 ; X86:       ## %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax ## encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vpbroadcastq (%eax), %zmm1 ## encoding: [0x62,0xf2,0xfd,0x48,0x59,0x08]
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax ## encoding: [0x0f,0xb6,0x44,0x24,0x08]
-; X86-NEXT:    kmovw %eax, %k1 ## encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vpsubq %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0xc9,0xfb,0xc1]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx ## encoding: [0x0f,0xb6,0x4c,0x24,0x08]
+; X86-NEXT:    kmovw %ecx, %k1 ## encoding: [0xc5,0xf8,0x92,0xc9]
+; X86-NEXT:    vpsubq (%eax){1to8}, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0xd9,0xfb,0x00]
 ; X86-NEXT:    retl ## encoding: [0xc3]
 ;
 ; X64-LABEL: test_mask_sub_epi64_rmbkz:
@@ -5478,19 +5471,19 @@ define <8 x i8> @test_cmp_q_512(<8 x i64> %a0, <8 x i64> %a1) {
 ; CHECK-NEXT:    vpcmpgtq %zmm1, %zmm0, %k5 ## encoding: [0x62,0xf2,0xfd,0x48,0x37,0xe9]
 ; CHECK-NEXT:    kmovw %k0, %eax ## encoding: [0xc5,0xf8,0x93,0xc0]
 ; CHECK-NEXT:    vpxor %xmm0, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xef,0xc0]
-; CHECK-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xc4,0xc0,0x00]
+; CHECK-NEXT:    vpinsrb $0, %eax, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x00]
 ; CHECK-NEXT:    kmovw %k1, %eax ## encoding: [0xc5,0xf8,0x93,0xc1]
-; CHECK-NEXT:    vpinsrw $1, %eax, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xc4,0xc0,0x01]
+; CHECK-NEXT:    vpinsrb $1, %eax, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x01]
 ; CHECK-NEXT:    kmovw %k2, %eax ## encoding: [0xc5,0xf8,0x93,0xc2]
-; CHECK-NEXT:    vpinsrw $2, %eax, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xc4,0xc0,0x02]
+; CHECK-NEXT:    vpinsrb $2, %eax, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x02]
 ; CHECK-NEXT:    kmovw %k3, %eax ## encoding: [0xc5,0xf8,0x93,0xc3]
-; CHECK-NEXT:    vpinsrw $4, %eax, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xc4,0xc0,0x04]
+; CHECK-NEXT:    vpinsrb $4, %eax, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x04]
 ; CHECK-NEXT:    kmovw %k4, %eax ## encoding: [0xc5,0xf8,0x93,0xc4]
-; CHECK-NEXT:    vpinsrw $5, %eax, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xc4,0xc0,0x05]
+; CHECK-NEXT:    vpinsrb $5, %eax, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x05]
 ; CHECK-NEXT:    kmovw %k5, %eax ## encoding: [0xc5,0xf8,0x93,0xc5]
-; CHECK-NEXT:    vpinsrw $6, %eax, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xc4,0xc0,0x06]
+; CHECK-NEXT:    vpinsrb $6, %eax, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x06]
 ; CHECK-NEXT:    movl $255, %eax ## encoding: [0xb8,0xff,0x00,0x00,0x00]
-; CHECK-NEXT:    vpinsrw $7, %eax, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xc4,0xc0,0x07]
+; CHECK-NEXT:    vpinsrb $7, %eax, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x07]
 ; CHECK-NEXT:    vzeroupper ## encoding: [0xc5,0xf8,0x77]
 ; CHECK-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
   %res0 = call i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 0, i8 -1)
@@ -5515,7 +5508,7 @@ define <8 x i8> @test_cmp_q_512(<8 x i64> %a0, <8 x i64> %a1) {
 define <8 x i8> @test_mask_cmp_q_512(<8 x i64> %a0, <8 x i64> %a1, i8 %mask) {
 ; X86-LABEL: test_mask_cmp_q_512:
 ; X86:       ## %bb.0:
-; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax ## encoding: [0x0f,0xb7,0x44,0x24,0x04]
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax ## encoding: [0x8b,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 ## encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpcmpeqq %zmm1, %zmm0, %k0 {%k1} ## encoding: [0x62,0xf2,0xfd,0x49,0x29,0xc1]
 ; X86-NEXT:    vpcmpgtq %zmm0, %zmm1, %k2 {%k1} ## encoding: [0x62,0xf2,0xf5,0x49,0x37,0xd0]
@@ -5525,18 +5518,18 @@ define <8 x i8> @test_mask_cmp_q_512(<8 x i64> %a0, <8 x i64> %a1, i8 %mask) {
 ; X86-NEXT:    vpcmpgtq %zmm1, %zmm0, %k1 {%k1} ## encoding: [0x62,0xf2,0xfd,0x49,0x37,0xc9]
 ; X86-NEXT:    kmovw %k0, %ecx ## encoding: [0xc5,0xf8,0x93,0xc8]
 ; X86-NEXT:    vpxor %xmm0, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xef,0xc0]
-; X86-NEXT:    vpinsrw $0, %ecx, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xc4,0xc1,0x00]
+; X86-NEXT:    vpinsrb $0, %ecx, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x20,0xc1,0x00]
 ; X86-NEXT:    kmovw %k2, %ecx ## encoding: [0xc5,0xf8,0x93,0xca]
-; X86-NEXT:    vpinsrw $1, %ecx, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xc4,0xc1,0x01]
+; X86-NEXT:    vpinsrb $1, %ecx, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x20,0xc1,0x01]
 ; X86-NEXT:    kmovw %k3, %ecx ## encoding: [0xc5,0xf8,0x93,0xcb]
-; X86-NEXT:    vpinsrw $2, %ecx, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xc4,0xc1,0x02]
+; X86-NEXT:    vpinsrb $2, %ecx, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x20,0xc1,0x02]
 ; X86-NEXT:    kmovw %k4, %ecx ## encoding: [0xc5,0xf8,0x93,0xcc]
-; X86-NEXT:    vpinsrw $4, %ecx, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xc4,0xc1,0x04]
+; X86-NEXT:    vpinsrb $4, %ecx, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x20,0xc1,0x04]
 ; X86-NEXT:    kmovw %k5, %ecx ## encoding: [0xc5,0xf8,0x93,0xcd]
-; X86-NEXT:    vpinsrw $5, %ecx, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xc4,0xc1,0x05]
+; X86-NEXT:    vpinsrb $5, %ecx, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x20,0xc1,0x05]
 ; X86-NEXT:    kmovw %k1, %ecx ## encoding: [0xc5,0xf8,0x93,0xc9]
-; X86-NEXT:    vpinsrw $6, %ecx, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xc4,0xc1,0x06]
-; X86-NEXT:    vpinsrw $7, %eax, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xc4,0xc0,0x07]
+; X86-NEXT:    vpinsrb $6, %ecx, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x20,0xc1,0x06]
+; X86-NEXT:    vpinsrb $7, %eax, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x07]
 ; X86-NEXT:    vzeroupper ## encoding: [0xc5,0xf8,0x77]
 ; X86-NEXT:    retl ## encoding: [0xc3]
 ;
@@ -5551,18 +5544,18 @@ define <8 x i8> @test_mask_cmp_q_512(<8 x i64> %a0, <8 x i64> %a1, i8 %mask) {
 ; X64-NEXT:    vpcmpgtq %zmm1, %zmm0, %k1 {%k1} ## encoding: [0x62,0xf2,0xfd,0x49,0x37,0xc9]
 ; X64-NEXT:    kmovw %k0, %eax ## encoding: [0xc5,0xf8,0x93,0xc0]
 ; X64-NEXT:    vpxor %xmm0, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xef,0xc0]
-; X64-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xc4,0xc0,0x00]
+; X64-NEXT:    vpinsrb $0, %eax, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x00]
 ; X64-NEXT:    kmovw %k2, %eax ## encoding: [0xc5,0xf8,0x93,0xc2]
-; X64-NEXT:    vpinsrw $1, %eax, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xc4,0xc0,0x01]
+; X64-NEXT:    vpinsrb $1, %eax, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x01]
 ; X64-NEXT:    kmovw %k3, %eax ## encoding: [0xc5,0xf8,0x93,0xc3]
-; X64-NEXT:    vpinsrw $2, %eax, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xc4,0xc0,0x02]
+; X64-NEXT:    vpinsrb $2, %eax, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x02]
 ; X64-NEXT:    kmovw %k4, %eax ## encoding: [0xc5,0xf8,0x93,0xc4]
-; X64-NEXT:    vpinsrw $4, %eax, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xc4,0xc0,0x04]
+; X64-NEXT:    vpinsrb $4, %eax, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x04]
 ; X64-NEXT:    kmovw %k5, %eax ## encoding: [0xc5,0xf8,0x93,0xc5]
-; X64-NEXT:    vpinsrw $5, %eax, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xc4,0xc0,0x05]
+; X64-NEXT:    vpinsrb $5, %eax, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x05]
 ; X64-NEXT:    kmovw %k1, %eax ## encoding: [0xc5,0xf8,0x93,0xc1]
-; X64-NEXT:    vpinsrw $6, %eax, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xc4,0xc0,0x06]
-; X64-NEXT:    vpinsrw $7, %edi, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xc4,0xc7,0x07]
+; X64-NEXT:    vpinsrb $6, %eax, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x06]
+; X64-NEXT:    vpinsrb $7, %edi, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x20,0xc7,0x07]
 ; X64-NEXT:    vzeroupper ## encoding: [0xc5,0xf8,0x77]
 ; X64-NEXT:    retq ## encoding: [0xc3]
   %res0 = call i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 0, i8 %mask)
@@ -5597,19 +5590,19 @@ define <8 x i8> @test_ucmp_q_512(<8 x i64> %a0, <8 x i64> %a1) {
 ; CHECK-NEXT:    vpcmpnleuq %zmm1, %zmm0, %k5 ## encoding: [0x62,0xf3,0xfd,0x48,0x1e,0xe9,0x06]
 ; CHECK-NEXT:    kmovw %k0, %eax ## encoding: [0xc5,0xf8,0x93,0xc0]
 ; CHECK-NEXT:    vpxor %xmm0, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xef,0xc0]
-; CHECK-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xc4,0xc0,0x00]
+; CHECK-NEXT:    vpinsrb $0, %eax, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x00]
 ; CHECK-NEXT:    kmovw %k1, %eax ## encoding: [0xc5,0xf8,0x93,0xc1]
-; CHECK-NEXT:    vpinsrw $1, %eax, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xc4,0xc0,0x01]
+; CHECK-NEXT:    vpinsrb $1, %eax, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x01]
 ; CHECK-NEXT:    kmovw %k2, %eax ## encoding: [0xc5,0xf8,0x93,0xc2]
-; CHECK-NEXT:    vpinsrw $2, %eax, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xc4,0xc0,0x02]
+; CHECK-NEXT:    vpinsrb $2, %eax, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x02]
 ; CHECK-NEXT:    kmovw %k3, %eax ## encoding: [0xc5,0xf8,0x93,0xc3]
-; CHECK-NEXT:    vpinsrw $4, %eax, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xc4,0xc0,0x04]
+; CHECK-NEXT:    vpinsrb $4, %eax, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x04]
 ; CHECK-NEXT:    kmovw %k4, %eax ## encoding: [0xc5,0xf8,0x93,0xc4]
-; CHECK-NEXT:    vpinsrw $5, %eax, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xc4,0xc0,0x05]
+; CHECK-NEXT:    vpinsrb $5, %eax, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x05]
 ; CHECK-NEXT:    kmovw %k5, %eax ## encoding: [0xc5,0xf8,0x93,0xc5]
-; CHECK-NEXT:    vpinsrw $6, %eax, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xc4,0xc0,0x06]
+; CHECK-NEXT:    vpinsrb $6, %eax, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x06]
 ; CHECK-NEXT:    movl $255, %eax ## encoding: [0xb8,0xff,0x00,0x00,0x00]
-; CHECK-NEXT:    vpinsrw $7, %eax, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xc4,0xc0,0x07]
+; CHECK-NEXT:    vpinsrb $7, %eax, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x07]
 ; CHECK-NEXT:    vzeroupper ## encoding: [0xc5,0xf8,0x77]
 ; CHECK-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
   %res0 = call i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 0, i8 -1)
@@ -5634,7 +5627,7 @@ define <8 x i8> @test_ucmp_q_512(<8 x i64> %a0, <8 x i64> %a1) {
 define <8 x i8> @test_mask_ucmp_q_512(<8 x i64> %a0, <8 x i64> %a1, i8 %mask) {
 ; X86-LABEL: test_mask_ucmp_q_512:
 ; X86:       ## %bb.0:
-; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax ## encoding: [0x0f,0xb7,0x44,0x24,0x04]
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax ## encoding: [0x8b,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 ## encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpcmpeqq %zmm1, %zmm0, %k0 {%k1} ## encoding: [0x62,0xf2,0xfd,0x49,0x29,0xc1]
 ; X86-NEXT:    vpcmpltuq %zmm1, %zmm0, %k2 {%k1} ## encoding: [0x62,0xf3,0xfd,0x49,0x1e,0xd1,0x01]
@@ -5644,18 +5637,18 @@ define <8 x i8> @test_mask_ucmp_q_512(<8 x i64> %a0, <8 x i64> %a1, i8 %mask) {
 ; X86-NEXT:    vpcmpnleuq %zmm1, %zmm0, %k1 {%k1} ## encoding: [0x62,0xf3,0xfd,0x49,0x1e,0xc9,0x06]
 ; X86-NEXT:    kmovw %k0, %ecx ## encoding: [0xc5,0xf8,0x93,0xc8]
 ; X86-NEXT:    vpxor %xmm0, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xef,0xc0]
-; X86-NEXT:    vpinsrw $0, %ecx, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xc4,0xc1,0x00]
+; X86-NEXT:    vpinsrb $0, %ecx, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x20,0xc1,0x00]
 ; X86-NEXT:    kmovw %k2, %ecx ## encoding: [0xc5,0xf8,0x93,0xca]
-; X86-NEXT:    vpinsrw $1, %ecx, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xc4,0xc1,0x01]
+; X86-NEXT:    vpinsrb $1, %ecx, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x20,0xc1,0x01]
 ; X86-NEXT:    kmovw %k3, %ecx ## encoding: [0xc5,0xf8,0x93,0xcb]
-; X86-NEXT:    vpinsrw $2, %ecx, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xc4,0xc1,0x02]
+; X86-NEXT:    vpinsrb $2, %ecx, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x20,0xc1,0x02]
 ; X86-NEXT:    kmovw %k4, %ecx ## encoding: [0xc5,0xf8,0x93,0xcc]
-; X86-NEXT:    vpinsrw $4, %ecx, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xc4,0xc1,0x04]
+; X86-NEXT:    vpinsrb $4, %ecx, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x20,0xc1,0x04]
 ; X86-NEXT:    kmovw %k5, %ecx ## encoding: [0xc5,0xf8,0x93,0xcd]
-; X86-NEXT:    vpinsrw $5, %ecx, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xc4,0xc1,0x05]
+; X86-NEXT:    vpinsrb $5, %ecx, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x20,0xc1,0x05]
 ; X86-NEXT:    kmovw %k1, %ecx ## encoding: [0xc5,0xf8,0x93,0xc9]
-; X86-NEXT:    vpinsrw $6, %ecx, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xc4,0xc1,0x06]
-; X86-NEXT:    vpinsrw $7, %eax, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xc4,0xc0,0x07]
+; X86-NEXT:    vpinsrb $6, %ecx, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x20,0xc1,0x06]
+; X86-NEXT:    vpinsrb $7, %eax, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x07]
 ; X86-NEXT:    vzeroupper ## encoding: [0xc5,0xf8,0x77]
 ; X86-NEXT:    retl ## encoding: [0xc3]
 ;
@@ -5670,18 +5663,18 @@ define <8 x i8> @test_mask_ucmp_q_512(<8 x i64> %a0, <8 x i64> %a1, i8 %mask) {
 ; X64-NEXT:    vpcmpnleuq %zmm1, %zmm0, %k1 {%k1} ## encoding: [0x62,0xf3,0xfd,0x49,0x1e,0xc9,0x06]
 ; X64-NEXT:    kmovw %k0, %eax ## encoding: [0xc5,0xf8,0x93,0xc0]
 ; X64-NEXT:    vpxor %xmm0, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xef,0xc0]
-; X64-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xc4,0xc0,0x00]
+; X64-NEXT:    vpinsrb $0, %eax, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x00]
 ; X64-NEXT:    kmovw %k2, %eax ## encoding: [0xc5,0xf8,0x93,0xc2]
-; X64-NEXT:    vpinsrw $1, %eax, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xc4,0xc0,0x01]
+; X64-NEXT:    vpinsrb $1, %eax, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x01]
 ; X64-NEXT:    kmovw %k3, %eax ## encoding: [0xc5,0xf8,0x93,0xc3]
-; X64-NEXT:    vpinsrw $2, %eax, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xc4,0xc0,0x02]
+; X64-NEXT:    vpinsrb $2, %eax, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x02]
 ; X64-NEXT:    kmovw %k4, %eax ## encoding: [0xc5,0xf8,0x93,0xc4]
-; X64-NEXT:    vpinsrw $4, %eax, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xc4,0xc0,0x04]
+; X64-NEXT:    vpinsrb $4, %eax, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x04]
 ; X64-NEXT:    kmovw %k5, %eax ## encoding: [0xc5,0xf8,0x93,0xc5]
-; X64-NEXT:    vpinsrw $5, %eax, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xc4,0xc0,0x05]
+; X64-NEXT:    vpinsrb $5, %eax, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x05]
 ; X64-NEXT:    kmovw %k1, %eax ## encoding: [0xc5,0xf8,0x93,0xc1]
-; X64-NEXT:    vpinsrw $6, %eax, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xc4,0xc0,0x06]
-; X64-NEXT:    vpinsrw $7, %edi, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xc4,0xc7,0x07]
+; X64-NEXT:    vpinsrb $6, %eax, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x06]
+; X64-NEXT:    vpinsrb $7, %edi, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x20,0xc7,0x07]
 ; X64-NEXT:    vzeroupper ## encoding: [0xc5,0xf8,0x77]
 ; X64-NEXT:    retq ## encoding: [0xc3]
   %res0 = call i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 0, i8 %mask)
diff --git a/llvm/test/CodeGen/X86/avx512-intrinsics.ll b/llvm/test/CodeGen/X86/avx512-intrinsics.ll
index 036b60d51..e94f16a56 100644
--- a/llvm/test/CodeGen/X86/avx512-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/avx512-intrinsics.ll
@@ -755,7 +755,7 @@ define <8 x double> @test_getexp_round_pd_512(<8 x double> %a0) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vgetexppd {sae}, %zmm0, %zmm0
 ; CHECK-NEXT:    ret{{[l|q]}}
-  %res = call <8 x double> @llvm.x86.avx512.mask.getexp.pd.512(<8 x double> %a0,  <8 x double> zeroinitializer, i8 -1, i32 8)
+  %res = call <8 x double> @llvm.x86.avx512.mask.getexp.pd.512(<8 x double> %a0,  <8 x double> zeroinitializer, i8 -1, i32 12)
   ret <8 x double> %res
 }
 declare <8 x double> @llvm.x86.avx512.mask.getexp.pd.512(<8 x double>, <8 x double>, i8, i32) nounwind readnone
@@ -4557,17 +4557,17 @@ define i8@test_int_x86_avx512_mask_cmp_sd_all(<2 x double> %x0, <2 x double> %x1
 ; X64-NEXT:    kmovw %k0, %esi
 ; X64-NEXT:    vcmpnltsd {sae}, %xmm1, %xmm0, %k0 {%k1}
 ; X64-NEXT:    kmovw %k0, %eax
-; X64-NEXT:    orb %sil, %al
-; X64-NEXT:    orb %dl, %al
-; X64-NEXT:    orb %cl, %al
+; X64-NEXT:    orl %esi, %eax
+; X64-NEXT:    orl %edx, %eax
+; X64-NEXT:    orl %ecx, %eax
 ; X64-NEXT:    # kill: def $al killed $al killed $eax
 ; X64-NEXT:    retq
 ;
 ; X86-LABEL: test_int_x86_avx512_mask_cmp_sd_all:
 ; X86:       # %bb.0:
-; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %esi
 ; X86-NEXT:    .cfi_def_cfa_offset 8
-; X86-NEXT:    .cfi_offset %ebx, -8
+; X86-NEXT:    .cfi_offset %esi, -8
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vcmplesd %xmm1, %xmm0, %k0
@@ -4575,14 +4575,14 @@ define i8@test_int_x86_avx512_mask_cmp_sd_all(<2 x double> %x0, <2 x double> %x1
 ; X86-NEXT:    vcmpunordsd {sae}, %xmm1, %xmm0, %k0
 ; X86-NEXT:    kmovw %k0, %edx
 ; X86-NEXT:    vcmpneqsd %xmm1, %xmm0, %k0 {%k1}
-; X86-NEXT:    kmovw %k0, %ebx
+; X86-NEXT:    kmovw %k0, %esi
 ; X86-NEXT:    vcmpnltsd {sae}, %xmm1, %xmm0, %k0 {%k1}
 ; X86-NEXT:    kmovw %k0, %eax
-; X86-NEXT:    orb %bl, %al
-; X86-NEXT:    orb %dl, %al
-; X86-NEXT:    orb %cl, %al
+; X86-NEXT:    orl %esi, %eax
+; X86-NEXT:    orl %edx, %eax
+; X86-NEXT:    orl %ecx, %eax
 ; X86-NEXT:    # kill: def $al killed $al killed $eax
-; X86-NEXT:    popl %ebx
+; X86-NEXT:    popl %esi
 ; X86-NEXT:    .cfi_def_cfa_offset 4
 ; X86-NEXT:    retl
 
@@ -4634,17 +4634,17 @@ define i8@test_int_x86_avx512_mask_cmp_ss_all(<4 x float> %x0, <4 x float> %x1,
 ; X64-NEXT:    kmovw %k0, %esi
 ; X64-NEXT:    vcmpnltss {sae}, %xmm1, %xmm0, %k0 {%k1}
 ; X64-NEXT:    kmovw %k0, %eax
-; X64-NEXT:    andb %sil, %al
-; X64-NEXT:    andb %dl, %al
-; X64-NEXT:    andb %cl, %al
+; X64-NEXT:    andl %esi, %eax
+; X64-NEXT:    andl %edx, %eax
+; X64-NEXT:    andl %ecx, %eax
 ; X64-NEXT:    # kill: def $al killed $al killed $eax
 ; X64-NEXT:    retq
 ;
 ; X86-LABEL: test_int_x86_avx512_mask_cmp_ss_all:
 ; X86:       # %bb.0:
-; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %esi
 ; X86-NEXT:    .cfi_def_cfa_offset 8
-; X86-NEXT:    .cfi_offset %ebx, -8
+; X86-NEXT:    .cfi_offset %esi, -8
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vcmpless %xmm1, %xmm0, %k0
@@ -4652,14 +4652,14 @@ define i8@test_int_x86_avx512_mask_cmp_ss_all(<4 x float> %x0, <4 x float> %x1,
 ; X86-NEXT:    vcmpunordss {sae}, %xmm1, %xmm0, %k0
 ; X86-NEXT:    kmovw %k0, %edx
 ; X86-NEXT:    vcmpneqss %xmm1, %xmm0, %k0 {%k1}
-; X86-NEXT:    kmovw %k0, %ebx
+; X86-NEXT:    kmovw %k0, %esi
 ; X86-NEXT:    vcmpnltss {sae}, %xmm1, %xmm0, %k0 {%k1}
 ; X86-NEXT:    kmovw %k0, %eax
-; X86-NEXT:    andb %bl, %al
-; X86-NEXT:    andb %dl, %al
-; X86-NEXT:    andb %cl, %al
+; X86-NEXT:    andl %esi, %eax
+; X86-NEXT:    andl %edx, %eax
+; X86-NEXT:    andl %ecx, %eax
 ; X86-NEXT:    # kill: def $al killed $al killed $eax
-; X86-NEXT:    popl %ebx
+; X86-NEXT:    popl %esi
 ; X86-NEXT:    .cfi_def_cfa_offset 4
 ; X86-NEXT:    retl
   %res1 = call i8 @llvm.x86.avx512.mask.cmp.ss(<4 x float> %x0, <4 x float> %x1, i32 2, i8 -1, i32 4)
@@ -4798,6 +4798,22 @@ define <4 x float>@test_int_x86_avx512_mask_getmant_ss(<4 x float> %x0, <4 x flo
   ret <4 x float> %res13
 }
 
+define <4 x float> @test_int_x86_avx512_mask_getmant_ss_load(<4 x float> %x0, <4 x float>* %x1p) {
+; X64-LABEL: test_int_x86_avx512_mask_getmant_ss_load:
+; X64:       # %bb.0:
+; X64-NEXT:    vgetmantss $11, (%rdi), %xmm0, %xmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_getmant_ss_load:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    vgetmantss $11, (%eax), %xmm0, %xmm0
+; X86-NEXT:    retl
+  %x1 = load <4 x float>, <4 x float>* %x1p
+  %res = call <4 x float> @llvm.x86.avx512.mask.getmant.ss(<4 x float> %x0, <4 x float> %x1, i32 11, <4 x float> undef, i8 -1, i32 4)
+  ret <4 x float> %res
+}
+
 declare <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double>, <8 x i64>)
 
 define <8 x double>@test_int_x86_avx512_vpermilvar_pd_512(<8 x double> %x0, <8 x i64> %x1) {
diff --git a/llvm/test/CodeGen/X86/avx512-logic.ll b/llvm/test/CodeGen/X86/avx512-logic.ll
index 65d9d67b2..9d614ce6c 100644
--- a/llvm/test/CodeGen/X86/avx512-logic.ll
+++ b/llvm/test/CodeGen/X86/avx512-logic.ll
@@ -166,8 +166,7 @@ entry:
 define <64 x i8> @and_v64i8(<64 x i8> %a, <64 x i8> %b) {
 ; KNL-LABEL: and_v64i8:
 ; KNL:       ## %bb.0:
-; KNL-NEXT:    vandps %ymm2, %ymm0, %ymm0
-; KNL-NEXT:    vandps %ymm3, %ymm1, %ymm1
+; KNL-NEXT:    vpandd %zmm1, %zmm0, %zmm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: and_v64i8:
@@ -181,8 +180,11 @@ define <64 x i8> @and_v64i8(<64 x i8> %a, <64 x i8> %b) {
 define <64 x i8> @andn_v64i8(<64 x i8> %a, <64 x i8> %b) {
 ; KNL-LABEL: andn_v64i8:
 ; KNL:       ## %bb.0:
-; KNL-NEXT:    vandnps %ymm0, %ymm2, %ymm0
-; KNL-NEXT:    vandnps %ymm1, %ymm3, %ymm1
+; KNL-NEXT:    vextractf64x4 $1, %zmm0, %ymm2
+; KNL-NEXT:    vextractf64x4 $1, %zmm1, %ymm3
+; KNL-NEXT:    vandnps %ymm2, %ymm3, %ymm2
+; KNL-NEXT:    vandnps %ymm0, %ymm1, %ymm0
+; KNL-NEXT:    vinsertf64x4 $1, %ymm2, %zmm0, %zmm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: andn_v64i8:
@@ -200,8 +202,7 @@ define <64 x i8> @andn_v64i8(<64 x i8> %a, <64 x i8> %b) {
 define <64 x i8> @or_v64i8(<64 x i8> %a, <64 x i8> %b) {
 ; KNL-LABEL: or_v64i8:
 ; KNL:       ## %bb.0:
-; KNL-NEXT:    vorps %ymm2, %ymm0, %ymm0
-; KNL-NEXT:    vorps %ymm3, %ymm1, %ymm1
+; KNL-NEXT:    vpord %zmm1, %zmm0, %zmm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: or_v64i8:
@@ -215,8 +216,7 @@ define <64 x i8> @or_v64i8(<64 x i8> %a, <64 x i8> %b) {
 define <64 x i8> @xor_v64i8(<64 x i8> %a, <64 x i8> %b) {
 ; KNL-LABEL: xor_v64i8:
 ; KNL:       ## %bb.0:
-; KNL-NEXT:    vxorps %ymm2, %ymm0, %ymm0
-; KNL-NEXT:    vxorps %ymm3, %ymm1, %ymm1
+; KNL-NEXT:    vpxord %zmm1, %zmm0, %zmm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: xor_v64i8:
@@ -230,8 +230,7 @@ define <64 x i8> @xor_v64i8(<64 x i8> %a, <64 x i8> %b) {
 define <32 x i16> @and_v32i16(<32 x i16> %a, <32 x i16> %b) {
 ; KNL-LABEL: and_v32i16:
 ; KNL:       ## %bb.0:
-; KNL-NEXT:    vandps %ymm2, %ymm0, %ymm0
-; KNL-NEXT:    vandps %ymm3, %ymm1, %ymm1
+; KNL-NEXT:    vpandd %zmm1, %zmm0, %zmm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: and_v32i16:
@@ -245,8 +244,11 @@ define <32 x i16> @and_v32i16(<32 x i16> %a, <32 x i16> %b) {
 define <32 x i16> @andn_v32i16(<32 x i16> %a, <32 x i16> %b) {
 ; KNL-LABEL: andn_v32i16:
 ; KNL:       ## %bb.0:
-; KNL-NEXT:    vandnps %ymm0, %ymm2, %ymm0
-; KNL-NEXT:    vandnps %ymm1, %ymm3, %ymm1
+; KNL-NEXT:    vextractf64x4 $1, %zmm0, %ymm2
+; KNL-NEXT:    vextractf64x4 $1, %zmm1, %ymm3
+; KNL-NEXT:    vandnps %ymm2, %ymm3, %ymm2
+; KNL-NEXT:    vandnps %ymm0, %ymm1, %ymm0
+; KNL-NEXT:    vinsertf64x4 $1, %ymm2, %zmm0, %zmm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: andn_v32i16:
@@ -262,8 +264,7 @@ define <32 x i16> @andn_v32i16(<32 x i16> %a, <32 x i16> %b) {
 define <32 x i16> @or_v32i16(<32 x i16> %a, <32 x i16> %b) {
 ; KNL-LABEL: or_v32i16:
 ; KNL:       ## %bb.0:
-; KNL-NEXT:    vorps %ymm2, %ymm0, %ymm0
-; KNL-NEXT:    vorps %ymm3, %ymm1, %ymm1
+; KNL-NEXT:    vpord %zmm1, %zmm0, %zmm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: or_v32i16:
@@ -277,8 +278,7 @@ define <32 x i16> @or_v32i16(<32 x i16> %a, <32 x i16> %b) {
 define <32 x i16> @xor_v32i16(<32 x i16> %a, <32 x i16> %b) {
 ; KNL-LABEL: xor_v32i16:
 ; KNL:       ## %bb.0:
-; KNL-NEXT:    vxorps %ymm2, %ymm0, %ymm0
-; KNL-NEXT:    vxorps %ymm3, %ymm1, %ymm1
+; KNL-NEXT:    vpxord %zmm1, %zmm0, %zmm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: xor_v32i16:
diff --git a/llvm/test/CodeGen/X86/avx512-mask-op.ll b/llvm/test/CodeGen/X86/avx512-mask-op.ll
index 02cf7544e..1fb0b3891 100644
--- a/llvm/test/CodeGen/X86/avx512-mask-op.ll
+++ b/llvm/test/CodeGen/X86/avx512-mask-op.ll
@@ -484,8 +484,8 @@ define <4 x i32> @test4(<4 x i64> %x, <4 x i64> %y, <4 x i64> %x1, <4 x i64> %y1
 ; KNL-NEXT:    ## kill: def $ymm2 killed $ymm2 def $zmm2
 ; KNL-NEXT:    ## kill: def $ymm1 killed $ymm1 def $zmm1
 ; KNL-NEXT:    ## kill: def $ymm0 killed $ymm0 def $zmm0
-; KNL-NEXT:    vpcmpleq %zmm1, %zmm0, %k1
-; KNL-NEXT:    vpcmpgtq %zmm3, %zmm2, %k1 {%k1}
+; KNL-NEXT:    vpcmpgtq %zmm3, %zmm2, %k1
+; KNL-NEXT:    vpcmpleq %zmm1, %zmm0, %k1 {%k1}
 ; KNL-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
 ; KNL-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $zmm0
 ; KNL-NEXT:    vzeroupper
@@ -493,8 +493,8 @@ define <4 x i32> @test4(<4 x i64> %x, <4 x i64> %y, <4 x i64> %x1, <4 x i64> %y1
 ;
 ; SKX-LABEL: test4:
 ; SKX:       ## %bb.0:
-; SKX-NEXT:    vpcmpleq %ymm1, %ymm0, %k1
-; SKX-NEXT:    vpcmpgtq %ymm3, %ymm2, %k0 {%k1}
+; SKX-NEXT:    vpcmpgtq %ymm3, %ymm2, %k1
+; SKX-NEXT:    vpcmpleq %ymm1, %ymm0, %k0 {%k1}
 ; SKX-NEXT:    vpmovm2d %k0, %xmm0
 ; SKX-NEXT:    vzeroupper
 ; SKX-NEXT:    retq
@@ -505,8 +505,8 @@ define <4 x i32> @test4(<4 x i64> %x, <4 x i64> %y, <4 x i64> %x1, <4 x i64> %y1
 ; AVX512BW-NEXT:    ## kill: def $ymm2 killed $ymm2 def $zmm2
 ; AVX512BW-NEXT:    ## kill: def $ymm1 killed $ymm1 def $zmm1
 ; AVX512BW-NEXT:    ## kill: def $ymm0 killed $ymm0 def $zmm0
-; AVX512BW-NEXT:    vpcmpleq %zmm1, %zmm0, %k1
-; AVX512BW-NEXT:    vpcmpgtq %zmm3, %zmm2, %k1 {%k1}
+; AVX512BW-NEXT:    vpcmpgtq %zmm3, %zmm2, %k1
+; AVX512BW-NEXT:    vpcmpleq %zmm1, %zmm0, %k1 {%k1}
 ; AVX512BW-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
 ; AVX512BW-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $zmm0
 ; AVX512BW-NEXT:    vzeroupper
@@ -518,8 +518,8 @@ define <4 x i32> @test4(<4 x i64> %x, <4 x i64> %y, <4 x i64> %x1, <4 x i64> %y1
 ; AVX512DQ-NEXT:    ## kill: def $ymm2 killed $ymm2 def $zmm2
 ; AVX512DQ-NEXT:    ## kill: def $ymm1 killed $ymm1 def $zmm1
 ; AVX512DQ-NEXT:    ## kill: def $ymm0 killed $ymm0 def $zmm0
-; AVX512DQ-NEXT:    vpcmpleq %zmm1, %zmm0, %k1
-; AVX512DQ-NEXT:    vpcmpgtq %zmm3, %zmm2, %k0 {%k1}
+; AVX512DQ-NEXT:    vpcmpgtq %zmm3, %zmm2, %k1
+; AVX512DQ-NEXT:    vpcmpleq %zmm1, %zmm0, %k0 {%k1}
 ; AVX512DQ-NEXT:    vpmovm2d %k0, %zmm0
 ; AVX512DQ-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $zmm0
 ; AVX512DQ-NEXT:    vzeroupper
@@ -527,8 +527,8 @@ define <4 x i32> @test4(<4 x i64> %x, <4 x i64> %y, <4 x i64> %x1, <4 x i64> %y1
 ;
 ; X86-LABEL: test4:
 ; X86:       ## %bb.0:
-; X86-NEXT:    vpcmpleq %ymm1, %ymm0, %k1
-; X86-NEXT:    vpcmpgtq %ymm3, %ymm2, %k0 {%k1}
+; X86-NEXT:    vpcmpgtq %ymm3, %ymm2, %k1
+; X86-NEXT:    vpcmpleq %ymm1, %ymm0, %k0 {%k1}
 ; X86-NEXT:    vpmovm2d %k0, %xmm0
 ; X86-NEXT:    vzeroupper
 ; X86-NEXT:    retl
@@ -546,8 +546,8 @@ define <2 x i64> @test5(<2 x i64> %x, <2 x i64> %y, <2 x i64> %x1, <2 x i64> %y1
 ; KNL-NEXT:    ## kill: def $xmm2 killed $xmm2 def $zmm2
 ; KNL-NEXT:    ## kill: def $xmm1 killed $xmm1 def $zmm1
 ; KNL-NEXT:    ## kill: def $xmm0 killed $xmm0 def $zmm0
-; KNL-NEXT:    vpcmpleq %zmm3, %zmm2, %k1
-; KNL-NEXT:    vpcmpgtq %zmm0, %zmm1, %k1 {%k1}
+; KNL-NEXT:    vpcmpgtq %zmm0, %zmm1, %k1
+; KNL-NEXT:    vpcmpleq %zmm3, %zmm2, %k1 {%k1}
 ; KNL-NEXT:    vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
 ; KNL-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $zmm0
 ; KNL-NEXT:    vzeroupper
@@ -555,8 +555,8 @@ define <2 x i64> @test5(<2 x i64> %x, <2 x i64> %y, <2 x i64> %x1, <2 x i64> %y1
 ;
 ; SKX-LABEL: test5:
 ; SKX:       ## %bb.0:
-; SKX-NEXT:    vpcmpleq %xmm3, %xmm2, %k1
-; SKX-NEXT:    vpcmpgtq %xmm0, %xmm1, %k0 {%k1}
+; SKX-NEXT:    vpcmpgtq %xmm0, %xmm1, %k1
+; SKX-NEXT:    vpcmpleq %xmm3, %xmm2, %k0 {%k1}
 ; SKX-NEXT:    vpmovm2q %k0, %xmm0
 ; SKX-NEXT:    retq
 ;
@@ -566,8 +566,8 @@ define <2 x i64> @test5(<2 x i64> %x, <2 x i64> %y, <2 x i64> %x1, <2 x i64> %y1
 ; AVX512BW-NEXT:    ## kill: def $xmm2 killed $xmm2 def $zmm2
 ; AVX512BW-NEXT:    ## kill: def $xmm1 killed $xmm1 def $zmm1
 ; AVX512BW-NEXT:    ## kill: def $xmm0 killed $xmm0 def $zmm0
-; AVX512BW-NEXT:    vpcmpleq %zmm3, %zmm2, %k1
-; AVX512BW-NEXT:    vpcmpgtq %zmm0, %zmm1, %k1 {%k1}
+; AVX512BW-NEXT:    vpcmpgtq %zmm0, %zmm1, %k1
+; AVX512BW-NEXT:    vpcmpleq %zmm3, %zmm2, %k1 {%k1}
 ; AVX512BW-NEXT:    vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
 ; AVX512BW-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $zmm0
 ; AVX512BW-NEXT:    vzeroupper
@@ -579,8 +579,8 @@ define <2 x i64> @test5(<2 x i64> %x, <2 x i64> %y, <2 x i64> %x1, <2 x i64> %y1
 ; AVX512DQ-NEXT:    ## kill: def $xmm2 killed $xmm2 def $zmm2
 ; AVX512DQ-NEXT:    ## kill: def $xmm1 killed $xmm1 def $zmm1
 ; AVX512DQ-NEXT:    ## kill: def $xmm0 killed $xmm0 def $zmm0
-; AVX512DQ-NEXT:    vpcmpleq %zmm3, %zmm2, %k1
-; AVX512DQ-NEXT:    vpcmpgtq %zmm0, %zmm1, %k0 {%k1}
+; AVX512DQ-NEXT:    vpcmpgtq %zmm0, %zmm1, %k1
+; AVX512DQ-NEXT:    vpcmpleq %zmm3, %zmm2, %k0 {%k1}
 ; AVX512DQ-NEXT:    vpmovm2q %k0, %zmm0
 ; AVX512DQ-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $zmm0
 ; AVX512DQ-NEXT:    vzeroupper
@@ -588,8 +588,8 @@ define <2 x i64> @test5(<2 x i64> %x, <2 x i64> %y, <2 x i64> %x1, <2 x i64> %y1
 ;
 ; X86-LABEL: test5:
 ; X86:       ## %bb.0:
-; X86-NEXT:    vpcmpleq %xmm3, %xmm2, %k1
-; X86-NEXT:    vpcmpgtq %xmm0, %xmm1, %k0 {%k1}
+; X86-NEXT:    vpcmpgtq %xmm0, %xmm1, %k1
+; X86-NEXT:    vpcmpleq %xmm3, %xmm2, %k0 {%k1}
 ; X86-NEXT:    vpmovm2q %k0, %xmm0
 ; X86-NEXT:    retl
   %x_gt_y = icmp slt <2 x i64> %x, %y
@@ -1069,45 +1069,52 @@ define <64 x i8> @test16(i64 %x) {
 ; KNL-NEXT:    kmovw %ecx, %k1
 ; KNL-NEXT:    kmovw %eax, %k2
 ; KNL-NEXT:    kmovw %edi, %k3
-; KNL-NEXT:    kshiftrw $5, %k0, %k4
-; KNL-NEXT:    kxnorw %k0, %k0, %k5
-; KNL-NEXT:    kxorw %k5, %k4, %k4
+; KNL-NEXT:    movw $-33, %ax
+; KNL-NEXT:    kmovw %eax, %k4
+; KNL-NEXT:    kandw %k4, %k0, %k0
+; KNL-NEXT:    movb $1, %al
+; KNL-NEXT:    kmovw %eax, %k4
 ; KNL-NEXT:    kshiftlw $15, %k4, %k4
 ; KNL-NEXT:    kshiftrw $10, %k4, %k4
-; KNL-NEXT:    kxorw %k4, %k0, %k4
+; KNL-NEXT:    korw %k4, %k0, %k4
 ; KNL-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k3} {z}
 ; KNL-NEXT:    vpmovdb %zmm0, %xmm0
 ; KNL-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k2} {z}
 ; KNL-NEXT:    vpmovdb %zmm1, %xmm1
-; KNL-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; KNL-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k4} {z}
-; KNL-NEXT:    vpmovdb %zmm0, %xmm0
+; KNL-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; KNL-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k4} {z}
+; KNL-NEXT:    vpmovdb %zmm1, %xmm1
 ; KNL-NEXT:    vpternlogd $255, %zmm2, %zmm2, %zmm2 {%k1} {z}
 ; KNL-NEXT:    vpmovdb %zmm2, %xmm2
-; KNL-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
+; KNL-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; KNL-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: test16:
 ; SKX:       ## %bb.0:
 ; SKX-NEXT:    kmovq %rdi, %k0
-; SKX-NEXT:    kxnorw %k0, %k0, %k1
-; SKX-NEXT:    kshiftrq $5, %k0, %k2
-; SKX-NEXT:    kxorq %k1, %k2, %k1
+; SKX-NEXT:    movq $-33, %rax
+; SKX-NEXT:    kmovq %rax, %k1
+; SKX-NEXT:    kandq %k1, %k0, %k0
+; SKX-NEXT:    movb $1, %al
+; SKX-NEXT:    kmovd %eax, %k1
 ; SKX-NEXT:    kshiftlq $63, %k1, %k1
 ; SKX-NEXT:    kshiftrq $58, %k1, %k1
-; SKX-NEXT:    kxorq %k1, %k0, %k0
+; SKX-NEXT:    korq %k1, %k0, %k0
 ; SKX-NEXT:    vpmovm2b %k0, %zmm0
 ; SKX-NEXT:    retq
 ;
 ; AVX512BW-LABEL: test16:
 ; AVX512BW:       ## %bb.0:
 ; AVX512BW-NEXT:    kmovq %rdi, %k0
-; AVX512BW-NEXT:    kxnorw %k0, %k0, %k1
-; AVX512BW-NEXT:    kshiftrq $5, %k0, %k2
-; AVX512BW-NEXT:    kxorq %k1, %k2, %k1
+; AVX512BW-NEXT:    movq $-33, %rax
+; AVX512BW-NEXT:    kmovq %rax, %k1
+; AVX512BW-NEXT:    kandq %k1, %k0, %k0
+; AVX512BW-NEXT:    movb $1, %al
+; AVX512BW-NEXT:    kmovd %eax, %k1
 ; AVX512BW-NEXT:    kshiftlq $63, %k1, %k1
 ; AVX512BW-NEXT:    kshiftrq $58, %k1, %k1
-; AVX512BW-NEXT:    kxorq %k1, %k0, %k0
+; AVX512BW-NEXT:    korq %k1, %k0, %k0
 ; AVX512BW-NEXT:    vpmovm2b %k0, %zmm0
 ; AVX512BW-NEXT:    retq
 ;
@@ -1115,40 +1122,47 @@ define <64 x i8> @test16(i64 %x) {
 ; AVX512DQ:       ## %bb.0:
 ; AVX512DQ-NEXT:    movq %rdi, %rax
 ; AVX512DQ-NEXT:    movl %edi, %ecx
-; AVX512DQ-NEXT:    kmovw %edi, %k0
+; AVX512DQ-NEXT:    kmovw %edi, %k1
 ; AVX512DQ-NEXT:    shrq $32, %rdi
 ; AVX512DQ-NEXT:    shrq $48, %rax
 ; AVX512DQ-NEXT:    shrl $16, %ecx
-; AVX512DQ-NEXT:    kmovw %ecx, %k1
+; AVX512DQ-NEXT:    kmovw %ecx, %k0
 ; AVX512DQ-NEXT:    kmovw %eax, %k2
 ; AVX512DQ-NEXT:    kmovw %edi, %k3
-; AVX512DQ-NEXT:    kshiftrw $5, %k0, %k4
-; AVX512DQ-NEXT:    kxnorw %k0, %k0, %k5
-; AVX512DQ-NEXT:    kxorw %k5, %k4, %k4
+; AVX512DQ-NEXT:    movw $-33, %ax
+; AVX512DQ-NEXT:    kmovw %eax, %k4
+; AVX512DQ-NEXT:    kandw %k4, %k1, %k1
+; AVX512DQ-NEXT:    movb $1, %al
+; AVX512DQ-NEXT:    kmovw %eax, %k4
 ; AVX512DQ-NEXT:    kshiftlw $15, %k4, %k4
 ; AVX512DQ-NEXT:    kshiftrw $10, %k4, %k4
-; AVX512DQ-NEXT:    kxorw %k4, %k0, %k0
+; AVX512DQ-NEXT:    korw %k4, %k1, %k1
 ; AVX512DQ-NEXT:    vpmovm2d %k3, %zmm0
 ; AVX512DQ-NEXT:    vpmovdb %zmm0, %xmm0
 ; AVX512DQ-NEXT:    vpmovm2d %k2, %zmm1
 ; AVX512DQ-NEXT:    vpmovdb %zmm1, %xmm1
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512DQ-NEXT:    vpmovm2d %k0, %zmm0
-; AVX512DQ-NEXT:    vpmovdb %zmm0, %xmm0
-; AVX512DQ-NEXT:    vpmovm2d %k1, %zmm2
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpmovm2d %k1, %zmm1
+; AVX512DQ-NEXT:    vpmovdb %zmm1, %xmm1
+; AVX512DQ-NEXT:    vpmovm2d %k0, %zmm2
 ; AVX512DQ-NEXT:    vpmovdb %zmm2, %xmm2
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; AVX512DQ-NEXT:    retq
 ;
 ; X86-LABEL: test16:
 ; X86:       ## %bb.0:
 ; X86-NEXT:    kmovq {{[0-9]+}}(%esp), %k0
-; X86-NEXT:    kshiftrq $5, %k0, %k1
-; X86-NEXT:    kxnorw %k0, %k0, %k2
-; X86-NEXT:    kxorq %k2, %k1, %k1
-; X86-NEXT:    kshiftlq $63, %k1, %k1
-; X86-NEXT:    kshiftrq $58, %k1, %k1
-; X86-NEXT:    kxorq %k1, %k0, %k0
+; X86-NEXT:    kshiftrq $6, %k0, %k1
+; X86-NEXT:    kshiftlq $6, %k1, %k1
+; X86-NEXT:    kshiftlq $59, %k0, %k0
+; X86-NEXT:    kshiftrq $59, %k0, %k0
+; X86-NEXT:    movb $1, %al
+; X86-NEXT:    kmovd %eax, %k2
+; X86-NEXT:    kshiftlq $63, %k2, %k2
+; X86-NEXT:    kshiftrq $58, %k2, %k2
+; X86-NEXT:    korq %k2, %k1, %k1
+; X86-NEXT:    korq %k1, %k0, %k0
 ; X86-NEXT:    vpmovm2b %k0, %zmm0
 ; X86-NEXT:    retl
   %a = bitcast i64 %x to <64 x i1>
@@ -1172,22 +1186,24 @@ define <64 x i8> @test17(i64 %x, i32 %y, i32 %z) {
 ; KNL-NEXT:    kmovw %edi, %k3
 ; KNL-NEXT:    cmpl %edx, %esi
 ; KNL-NEXT:    setg %al
-; KNL-NEXT:    kshiftrw $5, %k0, %k4
-; KNL-NEXT:    kmovw %eax, %k5
-; KNL-NEXT:    kxorw %k5, %k4, %k4
+; KNL-NEXT:    movw $-33, %cx
+; KNL-NEXT:    kmovw %ecx, %k4
+; KNL-NEXT:    kandw %k4, %k0, %k0
+; KNL-NEXT:    kmovw %eax, %k4
 ; KNL-NEXT:    kshiftlw $15, %k4, %k4
 ; KNL-NEXT:    kshiftrw $10, %k4, %k4
-; KNL-NEXT:    kxorw %k4, %k0, %k4
+; KNL-NEXT:    korw %k4, %k0, %k4
 ; KNL-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k3} {z}
 ; KNL-NEXT:    vpmovdb %zmm0, %xmm0
 ; KNL-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k2} {z}
 ; KNL-NEXT:    vpmovdb %zmm1, %xmm1
-; KNL-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; KNL-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k4} {z}
-; KNL-NEXT:    vpmovdb %zmm0, %xmm0
+; KNL-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; KNL-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k4} {z}
+; KNL-NEXT:    vpmovdb %zmm1, %xmm1
 ; KNL-NEXT:    vpternlogd $255, %zmm2, %zmm2, %zmm2 {%k1} {z}
 ; KNL-NEXT:    vpmovdb %zmm2, %xmm2
-; KNL-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
+; KNL-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; KNL-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: test17:
@@ -1195,12 +1211,13 @@ define <64 x i8> @test17(i64 %x, i32 %y, i32 %z) {
 ; SKX-NEXT:    kmovq %rdi, %k0
 ; SKX-NEXT:    cmpl %edx, %esi
 ; SKX-NEXT:    setg %al
+; SKX-NEXT:    movq $-33, %rcx
+; SKX-NEXT:    kmovq %rcx, %k1
+; SKX-NEXT:    kandq %k1, %k0, %k0
 ; SKX-NEXT:    kmovd %eax, %k1
-; SKX-NEXT:    kshiftrq $5, %k0, %k2
-; SKX-NEXT:    kxorq %k1, %k2, %k1
 ; SKX-NEXT:    kshiftlq $63, %k1, %k1
 ; SKX-NEXT:    kshiftrq $58, %k1, %k1
-; SKX-NEXT:    kxorq %k1, %k0, %k0
+; SKX-NEXT:    korq %k1, %k0, %k0
 ; SKX-NEXT:    vpmovm2b %k0, %zmm0
 ; SKX-NEXT:    retq
 ;
@@ -1209,12 +1226,13 @@ define <64 x i8> @test17(i64 %x, i32 %y, i32 %z) {
 ; AVX512BW-NEXT:    kmovq %rdi, %k0
 ; AVX512BW-NEXT:    cmpl %edx, %esi
 ; AVX512BW-NEXT:    setg %al
+; AVX512BW-NEXT:    movq $-33, %rcx
+; AVX512BW-NEXT:    kmovq %rcx, %k1
+; AVX512BW-NEXT:    kandq %k1, %k0, %k0
 ; AVX512BW-NEXT:    kmovd %eax, %k1
-; AVX512BW-NEXT:    kshiftrq $5, %k0, %k2
-; AVX512BW-NEXT:    kxorq %k1, %k2, %k1
 ; AVX512BW-NEXT:    kshiftlq $63, %k1, %k1
 ; AVX512BW-NEXT:    kshiftrq $58, %k1, %k1
-; AVX512BW-NEXT:    kxorq %k1, %k0, %k0
+; AVX512BW-NEXT:    korq %k1, %k0, %k0
 ; AVX512BW-NEXT:    vpmovm2b %k0, %zmm0
 ; AVX512BW-NEXT:    retq
 ;
@@ -1222,31 +1240,33 @@ define <64 x i8> @test17(i64 %x, i32 %y, i32 %z) {
 ; AVX512DQ:       ## %bb.0:
 ; AVX512DQ-NEXT:    movq %rdi, %rax
 ; AVX512DQ-NEXT:    movl %edi, %ecx
-; AVX512DQ-NEXT:    kmovw %edi, %k0
+; AVX512DQ-NEXT:    kmovw %edi, %k1
 ; AVX512DQ-NEXT:    shrq $32, %rdi
 ; AVX512DQ-NEXT:    shrq $48, %rax
 ; AVX512DQ-NEXT:    shrl $16, %ecx
-; AVX512DQ-NEXT:    kmovw %ecx, %k1
+; AVX512DQ-NEXT:    kmovw %ecx, %k0
 ; AVX512DQ-NEXT:    kmovw %eax, %k2
 ; AVX512DQ-NEXT:    kmovw %edi, %k3
 ; AVX512DQ-NEXT:    cmpl %edx, %esi
 ; AVX512DQ-NEXT:    setg %al
-; AVX512DQ-NEXT:    kshiftrw $5, %k0, %k4
-; AVX512DQ-NEXT:    kmovw %eax, %k5
-; AVX512DQ-NEXT:    kxorw %k5, %k4, %k4
+; AVX512DQ-NEXT:    movw $-33, %cx
+; AVX512DQ-NEXT:    kmovw %ecx, %k4
+; AVX512DQ-NEXT:    kandw %k4, %k1, %k1
+; AVX512DQ-NEXT:    kmovw %eax, %k4
 ; AVX512DQ-NEXT:    kshiftlw $15, %k4, %k4
 ; AVX512DQ-NEXT:    kshiftrw $10, %k4, %k4
-; AVX512DQ-NEXT:    kxorw %k4, %k0, %k0
+; AVX512DQ-NEXT:    korw %k4, %k1, %k1
 ; AVX512DQ-NEXT:    vpmovm2d %k3, %zmm0
 ; AVX512DQ-NEXT:    vpmovdb %zmm0, %xmm0
 ; AVX512DQ-NEXT:    vpmovm2d %k2, %zmm1
 ; AVX512DQ-NEXT:    vpmovdb %zmm1, %xmm1
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512DQ-NEXT:    vpmovm2d %k0, %zmm0
-; AVX512DQ-NEXT:    vpmovdb %zmm0, %xmm0
-; AVX512DQ-NEXT:    vpmovm2d %k1, %zmm2
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpmovm2d %k1, %zmm1
+; AVX512DQ-NEXT:    vpmovdb %zmm1, %xmm1
+; AVX512DQ-NEXT:    vpmovm2d %k0, %zmm2
 ; AVX512DQ-NEXT:    vpmovdb %zmm2, %xmm2
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; AVX512DQ-NEXT:    retq
 ;
 ; X86-LABEL: test17:
@@ -1255,12 +1275,15 @@ define <64 x i8> @test17(i64 %x, i32 %y, i32 %z) {
 ; X86-NEXT:    kmovq {{[0-9]+}}(%esp), %k0
 ; X86-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    setg %al
-; X86-NEXT:    kmovd %eax, %k1
-; X86-NEXT:    kshiftrq $5, %k0, %k2
-; X86-NEXT:    kxorq %k1, %k2, %k1
-; X86-NEXT:    kshiftlq $63, %k1, %k1
-; X86-NEXT:    kshiftrq $58, %k1, %k1
-; X86-NEXT:    kxorq %k1, %k0, %k0
+; X86-NEXT:    kshiftrq $6, %k0, %k1
+; X86-NEXT:    kshiftlq $6, %k1, %k1
+; X86-NEXT:    kshiftlq $59, %k0, %k0
+; X86-NEXT:    kshiftrq $59, %k0, %k0
+; X86-NEXT:    kmovd %eax, %k2
+; X86-NEXT:    kshiftlq $63, %k2, %k2
+; X86-NEXT:    kshiftrq $58, %k2, %k2
+; X86-NEXT:    korq %k2, %k1, %k1
+; X86-NEXT:    korq %k1, %k0, %k0
 ; X86-NEXT:    vpmovm2b %k0, %zmm0
 ; X86-NEXT:    retl
   %a = bitcast i64 %x to <64 x i1>
@@ -1277,11 +1300,11 @@ define <8 x i1> @test18(i8 %a, i16 %y) {
 ; KNL-NEXT:    kmovw %esi, %k1
 ; KNL-NEXT:    kshiftrw $8, %k1, %k2
 ; KNL-NEXT:    kshiftrw $9, %k1, %k1
-; KNL-NEXT:    kshiftrw $6, %k0, %k3
-; KNL-NEXT:    kxorw %k1, %k3, %k1
-; KNL-NEXT:    kshiftlw $15, %k1, %k1
-; KNL-NEXT:    kshiftrw $9, %k1, %k1
-; KNL-NEXT:    kxorw %k1, %k0, %k0
+; KNL-NEXT:    movw $-65, %ax
+; KNL-NEXT:    kmovw %eax, %k3
+; KNL-NEXT:    kandw %k3, %k0, %k0
+; KNL-NEXT:    kshiftlw $6, %k1, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
 ; KNL-NEXT:    kshiftlw $9, %k0, %k0
 ; KNL-NEXT:    kshiftrw $9, %k0, %k0
 ; KNL-NEXT:    kshiftlw $7, %k2, %k1
@@ -1298,11 +1321,11 @@ define <8 x i1> @test18(i8 %a, i16 %y) {
 ; SKX-NEXT:    kmovd %esi, %k1
 ; SKX-NEXT:    kshiftrw $8, %k1, %k2
 ; SKX-NEXT:    kshiftrw $9, %k1, %k1
-; SKX-NEXT:    kshiftrb $6, %k0, %k3
-; SKX-NEXT:    kxorb %k1, %k3, %k1
-; SKX-NEXT:    kshiftlb $7, %k1, %k1
-; SKX-NEXT:    kshiftrb $1, %k1, %k1
-; SKX-NEXT:    kxorb %k1, %k0, %k0
+; SKX-NEXT:    movb $-65, %al
+; SKX-NEXT:    kmovd %eax, %k3
+; SKX-NEXT:    kandb %k3, %k0, %k0
+; SKX-NEXT:    kshiftlb $6, %k1, %k1
+; SKX-NEXT:    korb %k1, %k0, %k0
 ; SKX-NEXT:    kshiftlb $1, %k0, %k0
 ; SKX-NEXT:    kshiftrb $1, %k0, %k0
 ; SKX-NEXT:    kshiftlb $7, %k2, %k1
@@ -1316,11 +1339,11 @@ define <8 x i1> @test18(i8 %a, i16 %y) {
 ; AVX512BW-NEXT:    kmovd %esi, %k1
 ; AVX512BW-NEXT:    kshiftrw $8, %k1, %k2
 ; AVX512BW-NEXT:    kshiftrw $9, %k1, %k1
-; AVX512BW-NEXT:    kshiftrw $6, %k0, %k3
-; AVX512BW-NEXT:    kxorw %k1, %k3, %k1
-; AVX512BW-NEXT:    kshiftlw $15, %k1, %k1
-; AVX512BW-NEXT:    kshiftrw $9, %k1, %k1
-; AVX512BW-NEXT:    kxorw %k1, %k0, %k0
+; AVX512BW-NEXT:    movw $-65, %ax
+; AVX512BW-NEXT:    kmovd %eax, %k3
+; AVX512BW-NEXT:    kandw %k3, %k0, %k0
+; AVX512BW-NEXT:    kshiftlw $6, %k1, %k1
+; AVX512BW-NEXT:    korw %k1, %k0, %k0
 ; AVX512BW-NEXT:    kshiftlw $9, %k0, %k0
 ; AVX512BW-NEXT:    kshiftrw $9, %k0, %k0
 ; AVX512BW-NEXT:    kshiftlw $7, %k2, %k1
@@ -1336,11 +1359,11 @@ define <8 x i1> @test18(i8 %a, i16 %y) {
 ; AVX512DQ-NEXT:    kmovw %esi, %k1
 ; AVX512DQ-NEXT:    kshiftrw $8, %k1, %k2
 ; AVX512DQ-NEXT:    kshiftrw $9, %k1, %k1
-; AVX512DQ-NEXT:    kshiftrb $6, %k0, %k3
-; AVX512DQ-NEXT:    kxorb %k1, %k3, %k1
-; AVX512DQ-NEXT:    kshiftlb $7, %k1, %k1
-; AVX512DQ-NEXT:    kshiftrb $1, %k1, %k1
-; AVX512DQ-NEXT:    kxorb %k1, %k0, %k0
+; AVX512DQ-NEXT:    movb $-65, %al
+; AVX512DQ-NEXT:    kmovw %eax, %k3
+; AVX512DQ-NEXT:    kandb %k3, %k0, %k0
+; AVX512DQ-NEXT:    kshiftlb $6, %k1, %k1
+; AVX512DQ-NEXT:    korb %k1, %k0, %k0
 ; AVX512DQ-NEXT:    kshiftlb $1, %k0, %k0
 ; AVX512DQ-NEXT:    kshiftrb $1, %k0, %k0
 ; AVX512DQ-NEXT:    kshiftlb $7, %k2, %k1
@@ -1357,11 +1380,11 @@ define <8 x i1> @test18(i8 %a, i16 %y) {
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
 ; X86-NEXT:    kshiftrw $8, %k1, %k2
 ; X86-NEXT:    kshiftrw $9, %k1, %k1
-; X86-NEXT:    kshiftrb $6, %k0, %k3
-; X86-NEXT:    kxorb %k1, %k3, %k1
-; X86-NEXT:    kshiftlb $7, %k1, %k1
-; X86-NEXT:    kshiftrb $1, %k1, %k1
-; X86-NEXT:    kxorb %k1, %k0, %k0
+; X86-NEXT:    movb $-65, %al
+; X86-NEXT:    kmovd %eax, %k3
+; X86-NEXT:    kandb %k3, %k0, %k0
+; X86-NEXT:    kshiftlb $6, %k1, %k1
+; X86-NEXT:    korb %k1, %k0, %k0
 ; X86-NEXT:    kshiftlb $1, %k0, %k0
 ; X86-NEXT:    kshiftrb $1, %k0, %k0
 ; X86-NEXT:    kshiftlb $7, %k2, %k1
@@ -1379,15 +1402,17 @@ define <8 x i1> @test18(i8 %a, i16 %y) {
 define <32 x i16> @test21(<32 x i16> %x , <32 x i1> %mask) nounwind readnone {
 ; KNL-LABEL: test21:
 ; KNL:       ## %bb.0:
-; KNL-NEXT:    vextracti128 $1, %ymm2, %xmm3
-; KNL-NEXT:    vpmovzxbw {{.*#+}} ymm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero,xmm3[8],zero,xmm3[9],zero,xmm3[10],zero,xmm3[11],zero,xmm3[12],zero,xmm3[13],zero,xmm3[14],zero,xmm3[15],zero
-; KNL-NEXT:    vpmovzxbw {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero,xmm2[8],zero,xmm2[9],zero,xmm2[10],zero,xmm2[11],zero,xmm2[12],zero,xmm2[13],zero,xmm2[14],zero,xmm2[15],zero
+; KNL-NEXT:    vpmovzxbw {{.*#+}} ymm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; KNL-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
+; KNL-NEXT:    vextracti128 $1, %ymm1, %xmm1
+; KNL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; KNL-NEXT:    vpsllw $15, %ymm1, %ymm1
+; KNL-NEXT:    vpsraw $15, %ymm1, %ymm1
+; KNL-NEXT:    vpand %ymm3, %ymm1, %ymm1
 ; KNL-NEXT:    vpsllw $15, %ymm2, %ymm2
 ; KNL-NEXT:    vpsraw $15, %ymm2, %ymm2
 ; KNL-NEXT:    vpand %ymm0, %ymm2, %ymm0
-; KNL-NEXT:    vpsllw $15, %ymm3, %ymm2
-; KNL-NEXT:    vpsraw $15, %ymm2, %ymm2
-; KNL-NEXT:    vpand %ymm1, %ymm2, %ymm1
+; KNL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: test21:
@@ -1406,15 +1431,17 @@ define <32 x i16> @test21(<32 x i16> %x , <32 x i1> %mask) nounwind readnone {
 ;
 ; AVX512DQ-LABEL: test21:
 ; AVX512DQ:       ## %bb.0:
-; AVX512DQ-NEXT:    vextracti128 $1, %ymm2, %xmm3
-; AVX512DQ-NEXT:    vpmovzxbw {{.*#+}} ymm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero,xmm3[8],zero,xmm3[9],zero,xmm3[10],zero,xmm3[11],zero,xmm3[12],zero,xmm3[13],zero,xmm3[14],zero,xmm3[15],zero
-; AVX512DQ-NEXT:    vpmovzxbw {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero,xmm2[8],zero,xmm2[9],zero,xmm2[10],zero,xmm2[11],zero,xmm2[12],zero,xmm2[13],zero,xmm2[14],zero,xmm2[15],zero
+; AVX512DQ-NEXT:    vpmovzxbw {{.*#+}} ymm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
+; AVX512DQ-NEXT:    vextracti128 $1, %ymm1, %xmm1
+; AVX512DQ-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512DQ-NEXT:    vpsllw $15, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpsraw $15, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpand %ymm3, %ymm1, %ymm1
 ; AVX512DQ-NEXT:    vpsllw $15, %ymm2, %ymm2
 ; AVX512DQ-NEXT:    vpsraw $15, %ymm2, %ymm2
 ; AVX512DQ-NEXT:    vpand %ymm0, %ymm2, %ymm0
-; AVX512DQ-NEXT:    vpsllw $15, %ymm3, %ymm2
-; AVX512DQ-NEXT:    vpsraw $15, %ymm2, %ymm2
-; AVX512DQ-NEXT:    vpand %ymm1, %ymm2, %ymm1
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    retq
 ;
 ; X86-LABEL: test21:
@@ -1849,8 +1876,10 @@ define void @store_i8_i1(i8 %x, i1 *%y) {
 define <32 x i16> @test_build_vec_v32i1(<32 x i16> %x) {
 ; KNL-LABEL: test_build_vec_v32i1:
 ; KNL:       ## %bb.0:
-; KNL-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
+; KNL-NEXT:    vextractf64x4 $1, %zmm0, %ymm1
 ; KNL-NEXT:    vandps {{.*}}(%rip), %ymm1, %ymm1
+; KNL-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
+; KNL-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: test_build_vec_v32i1:
@@ -1865,8 +1894,10 @@ define <32 x i16> @test_build_vec_v32i1(<32 x i16> %x) {
 ;
 ; AVX512DQ-LABEL: test_build_vec_v32i1:
 ; AVX512DQ:       ## %bb.0:
-; AVX512DQ-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
+; AVX512DQ-NEXT:    vextractf64x4 $1, %zmm0, %ymm1
 ; AVX512DQ-NEXT:    vandps {{.*}}(%rip), %ymm1, %ymm1
+; AVX512DQ-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
+; AVX512DQ-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    retq
 ;
 ; X86-LABEL: test_build_vec_v32i1:
@@ -1880,8 +1911,10 @@ define <32 x i16> @test_build_vec_v32i1(<32 x i16> %x) {
 define <32 x i16> @test_build_vec_v32i1_optsize(<32 x i16> %x) optsize {
 ; KNL-LABEL: test_build_vec_v32i1_optsize:
 ; KNL:       ## %bb.0:
-; KNL-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
+; KNL-NEXT:    vextractf64x4 $1, %zmm0, %ymm1
 ; KNL-NEXT:    vandps {{.*}}(%rip), %ymm1, %ymm1
+; KNL-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
+; KNL-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: test_build_vec_v32i1_optsize:
@@ -1900,8 +1933,10 @@ define <32 x i16> @test_build_vec_v32i1_optsize(<32 x i16> %x) optsize {
 ;
 ; AVX512DQ-LABEL: test_build_vec_v32i1_optsize:
 ; AVX512DQ:       ## %bb.0:
-; AVX512DQ-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
+; AVX512DQ-NEXT:    vextractf64x4 $1, %zmm0, %ymm1
 ; AVX512DQ-NEXT:    vandps {{.*}}(%rip), %ymm1, %ymm1
+; AVX512DQ-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
+; AVX512DQ-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    retq
 ;
 ; X86-LABEL: test_build_vec_v32i1_optsize:
@@ -1914,11 +1949,54 @@ define <32 x i16> @test_build_vec_v32i1_optsize(<32 x i16> %x) optsize {
   ret <32 x i16> %ret
 }
 
+define <32 x i16> @test_build_vec_v32i1_pgso(<32 x i16> %x) !prof !14 {
+; KNL-LABEL: test_build_vec_v32i1_pgso:
+; KNL:       ## %bb.0:
+; KNL-NEXT:    vextractf64x4 $1, %zmm0, %ymm1
+; KNL-NEXT:    vandps {{.*}}(%rip), %ymm1, %ymm1
+; KNL-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
+; KNL-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
+; KNL-NEXT:    retq
+;
+; SKX-LABEL: test_build_vec_v32i1_pgso:
+; SKX:       ## %bb.0:
+; SKX-NEXT:    movl $1497715861, %eax ## imm = 0x59455495
+; SKX-NEXT:    kmovd %eax, %k1
+; SKX-NEXT:    vmovdqu16 %zmm0, %zmm0 {%k1} {z}
+; SKX-NEXT:    retq
+;
+; AVX512BW-LABEL: test_build_vec_v32i1_pgso:
+; AVX512BW:       ## %bb.0:
+; AVX512BW-NEXT:    movl $1497715861, %eax ## imm = 0x59455495
+; AVX512BW-NEXT:    kmovd %eax, %k1
+; AVX512BW-NEXT:    vmovdqu16 %zmm0, %zmm0 {%k1} {z}
+; AVX512BW-NEXT:    retq
+;
+; AVX512DQ-LABEL: test_build_vec_v32i1_pgso:
+; AVX512DQ:       ## %bb.0:
+; AVX512DQ-NEXT:    vextractf64x4 $1, %zmm0, %ymm1
+; AVX512DQ-NEXT:    vandps {{.*}}(%rip), %ymm1, %ymm1
+; AVX512DQ-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
+; AVX512DQ-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    retq
+;
+; X86-LABEL: test_build_vec_v32i1_pgso:
+; X86:       ## %bb.0:
+; X86-NEXT:    movl $1497715861, %eax ## imm = 0x59455495
+; X86-NEXT:    kmovd %eax, %k1
+; X86-NEXT:    vmovdqu16 %zmm0, %zmm0 {%k1} {z}
+; X86-NEXT:    retl
+  %ret = select <32 x i1> <i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 true, i1 false>, <32 x i16> %x, <32 x i16> zeroinitializer
+  ret <32 x i16> %ret
+}
+
 define <64 x i8> @test_build_vec_v64i1(<64 x i8> %x) {
 ; KNL-LABEL: test_build_vec_v64i1:
 ; KNL:       ## %bb.0:
-; KNL-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
+; KNL-NEXT:    vextractf64x4 $1, %zmm0, %ymm1
 ; KNL-NEXT:    vandps {{.*}}(%rip), %ymm1, %ymm1
+; KNL-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
+; KNL-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: test_build_vec_v64i1:
@@ -1933,8 +2011,10 @@ define <64 x i8> @test_build_vec_v64i1(<64 x i8> %x) {
 ;
 ; AVX512DQ-LABEL: test_build_vec_v64i1:
 ; AVX512DQ:       ## %bb.0:
-; AVX512DQ-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
+; AVX512DQ-NEXT:    vextractf64x4 $1, %zmm0, %ymm1
 ; AVX512DQ-NEXT:    vandps {{.*}}(%rip), %ymm1, %ymm1
+; AVX512DQ-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
+; AVX512DQ-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    retq
 ;
 ; X86-LABEL: test_build_vec_v64i1:
@@ -1948,70 +2028,66 @@ define <64 x i8> @test_build_vec_v64i1(<64 x i8> %x) {
 define void @ktest_1(<8 x double> %in, double * %base) {
 ; KNL-LABEL: ktest_1:
 ; KNL:       ## %bb.0:
-; KNL-NEXT:    vmovupd (%rdi), %zmm1
-; KNL-NEXT:    vcmpltpd %zmm0, %zmm1, %k1
+; KNL-NEXT:    vcmpgtpd (%rdi), %zmm0, %k1
 ; KNL-NEXT:    vmovupd 8(%rdi), %zmm1 {%k1} {z}
 ; KNL-NEXT:    vcmpltpd %zmm1, %zmm0, %k0 {%k1}
 ; KNL-NEXT:    kmovw %k0, %eax
 ; KNL-NEXT:    testb %al, %al
-; KNL-NEXT:    je LBB43_2
+; KNL-NEXT:    je LBB44_2
 ; KNL-NEXT:  ## %bb.1: ## %L1
 ; KNL-NEXT:    vmovapd %zmm0, (%rdi)
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    retq
-; KNL-NEXT:  LBB43_2: ## %L2
+; KNL-NEXT:  LBB44_2: ## %L2
 ; KNL-NEXT:    vmovapd %zmm0, 8(%rdi)
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: ktest_1:
 ; SKX:       ## %bb.0:
-; SKX-NEXT:    vmovupd (%rdi), %zmm1
-; SKX-NEXT:    vcmpltpd %zmm0, %zmm1, %k1
+; SKX-NEXT:    vcmpgtpd (%rdi), %zmm0, %k1
 ; SKX-NEXT:    vmovupd 8(%rdi), %zmm1 {%k1} {z}
-; SKX-NEXT:    vcmpltpd %zmm1, %zmm0, %k0 {%k1}
-; SKX-NEXT:    kortestb %k0, %k0
-; SKX-NEXT:    je LBB43_2
+; SKX-NEXT:    vcmpltpd %zmm1, %zmm0, %k0
+; SKX-NEXT:    ktestb %k0, %k1
+; SKX-NEXT:    je LBB44_2
 ; SKX-NEXT:  ## %bb.1: ## %L1
 ; SKX-NEXT:    vmovapd %zmm0, (%rdi)
 ; SKX-NEXT:    vzeroupper
 ; SKX-NEXT:    retq
-; SKX-NEXT:  LBB43_2: ## %L2
+; SKX-NEXT:  LBB44_2: ## %L2
 ; SKX-NEXT:    vmovapd %zmm0, 8(%rdi)
 ; SKX-NEXT:    vzeroupper
 ; SKX-NEXT:    retq
 ;
 ; AVX512BW-LABEL: ktest_1:
 ; AVX512BW:       ## %bb.0:
-; AVX512BW-NEXT:    vmovupd (%rdi), %zmm1
-; AVX512BW-NEXT:    vcmpltpd %zmm0, %zmm1, %k1
+; AVX512BW-NEXT:    vcmpgtpd (%rdi), %zmm0, %k1
 ; AVX512BW-NEXT:    vmovupd 8(%rdi), %zmm1 {%k1} {z}
 ; AVX512BW-NEXT:    vcmpltpd %zmm1, %zmm0, %k0 {%k1}
 ; AVX512BW-NEXT:    kmovd %k0, %eax
 ; AVX512BW-NEXT:    testb %al, %al
-; AVX512BW-NEXT:    je LBB43_2
+; AVX512BW-NEXT:    je LBB44_2
 ; AVX512BW-NEXT:  ## %bb.1: ## %L1
 ; AVX512BW-NEXT:    vmovapd %zmm0, (%rdi)
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
-; AVX512BW-NEXT:  LBB43_2: ## %L2
+; AVX512BW-NEXT:  LBB44_2: ## %L2
 ; AVX512BW-NEXT:    vmovapd %zmm0, 8(%rdi)
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512DQ-LABEL: ktest_1:
 ; AVX512DQ:       ## %bb.0:
-; AVX512DQ-NEXT:    vmovupd (%rdi), %zmm1
-; AVX512DQ-NEXT:    vcmpltpd %zmm0, %zmm1, %k1
+; AVX512DQ-NEXT:    vcmpgtpd (%rdi), %zmm0, %k1
 ; AVX512DQ-NEXT:    vmovupd 8(%rdi), %zmm1 {%k1} {z}
-; AVX512DQ-NEXT:    vcmpltpd %zmm1, %zmm0, %k0 {%k1}
-; AVX512DQ-NEXT:    kortestb %k0, %k0
-; AVX512DQ-NEXT:    je LBB43_2
+; AVX512DQ-NEXT:    vcmpltpd %zmm1, %zmm0, %k0
+; AVX512DQ-NEXT:    ktestb %k0, %k1
+; AVX512DQ-NEXT:    je LBB44_2
 ; AVX512DQ-NEXT:  ## %bb.1: ## %L1
 ; AVX512DQ-NEXT:    vmovapd %zmm0, (%rdi)
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
-; AVX512DQ-NEXT:  LBB43_2: ## %L2
+; AVX512DQ-NEXT:  LBB44_2: ## %L2
 ; AVX512DQ-NEXT:    vmovapd %zmm0, 8(%rdi)
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
@@ -2019,17 +2095,16 @@ define void @ktest_1(<8 x double> %in, double * %base) {
 ; X86-LABEL: ktest_1:
 ; X86:       ## %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    vmovupd (%eax), %zmm1
-; X86-NEXT:    vcmpltpd %zmm0, %zmm1, %k1
+; X86-NEXT:    vcmpgtpd (%eax), %zmm0, %k1
 ; X86-NEXT:    vmovupd 8(%eax), %zmm1 {%k1} {z}
-; X86-NEXT:    vcmpltpd %zmm1, %zmm0, %k0 {%k1}
-; X86-NEXT:    kortestb %k0, %k0
-; X86-NEXT:    je LBB43_2
+; X86-NEXT:    vcmpltpd %zmm1, %zmm0, %k0
+; X86-NEXT:    ktestb %k0, %k1
+; X86-NEXT:    je LBB44_2
 ; X86-NEXT:  ## %bb.1: ## %L1
 ; X86-NEXT:    vmovapd %zmm0, (%eax)
 ; X86-NEXT:    vzeroupper
 ; X86-NEXT:    retl
-; X86-NEXT:  LBB43_2: ## %L2
+; X86-NEXT:  LBB44_2: ## %L2
 ; X86-NEXT:    vmovapd %zmm0, 8(%eax)
 ; X86-NEXT:    vzeroupper
 ; X86-NEXT:    retl
@@ -2064,10 +2139,8 @@ define void @ktest_2(<32 x float> %in, float * %base) {
 ;
 ; KNL-LABEL: ktest_2:
 ; KNL:       ## %bb.0:
-; KNL-NEXT:    vmovups (%rdi), %zmm2
-; KNL-NEXT:    vmovups 64(%rdi), %zmm3
-; KNL-NEXT:    vcmpltps %zmm1, %zmm3, %k1
-; KNL-NEXT:    vcmpltps %zmm0, %zmm2, %k2
+; KNL-NEXT:    vcmpgtps 64(%rdi), %zmm1, %k1
+; KNL-NEXT:    vcmpgtps (%rdi), %zmm0, %k2
 ; KNL-NEXT:    vmovups 4(%rdi), %zmm2 {%k2} {z}
 ; KNL-NEXT:    vmovups 68(%rdi), %zmm3 {%k1} {z}
 ; KNL-NEXT:    vcmpltps %zmm3, %zmm1, %k0
@@ -2078,13 +2151,13 @@ define void @ktest_2(<32 x float> %in, float * %base) {
 ; KNL-NEXT:    kmovw %k0, %ecx
 ; KNL-NEXT:    shll $16, %ecx
 ; KNL-NEXT:    orl %eax, %ecx
-; KNL-NEXT:    je LBB44_2
+; KNL-NEXT:    je LBB45_2
 ; KNL-NEXT:  ## %bb.1: ## %L1
 ; KNL-NEXT:    vmovaps %zmm0, (%rdi)
 ; KNL-NEXT:    vmovaps %zmm1, 64(%rdi)
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    retq
-; KNL-NEXT:  LBB44_2: ## %L2
+; KNL-NEXT:  LBB45_2: ## %L2
 ; KNL-NEXT:    vmovaps %zmm0, 4(%rdi)
 ; KNL-NEXT:    vmovaps %zmm1, 68(%rdi)
 ; KNL-NEXT:    vzeroupper
@@ -2092,10 +2165,8 @@ define void @ktest_2(<32 x float> %in, float * %base) {
 ;
 ; SKX-LABEL: ktest_2:
 ; SKX:       ## %bb.0:
-; SKX-NEXT:    vmovups (%rdi), %zmm2
-; SKX-NEXT:    vmovups 64(%rdi), %zmm3
-; SKX-NEXT:    vcmpltps %zmm0, %zmm2, %k1
-; SKX-NEXT:    vcmpltps %zmm1, %zmm3, %k2
+; SKX-NEXT:    vcmpgtps (%rdi), %zmm0, %k1
+; SKX-NEXT:    vcmpgtps 64(%rdi), %zmm1, %k2
 ; SKX-NEXT:    kunpckwd %k1, %k2, %k0
 ; SKX-NEXT:    vmovups 68(%rdi), %zmm2 {%k2} {z}
 ; SKX-NEXT:    vmovups 4(%rdi), %zmm3 {%k1} {z}
@@ -2103,13 +2174,13 @@ define void @ktest_2(<32 x float> %in, float * %base) {
 ; SKX-NEXT:    vcmpltps %zmm2, %zmm1, %k2
 ; SKX-NEXT:    kunpckwd %k1, %k2, %k1
 ; SKX-NEXT:    kortestd %k1, %k0
-; SKX-NEXT:    je LBB44_2
+; SKX-NEXT:    je LBB45_2
 ; SKX-NEXT:  ## %bb.1: ## %L1
 ; SKX-NEXT:    vmovaps %zmm0, (%rdi)
 ; SKX-NEXT:    vmovaps %zmm1, 64(%rdi)
 ; SKX-NEXT:    vzeroupper
 ; SKX-NEXT:    retq
-; SKX-NEXT:  LBB44_2: ## %L2
+; SKX-NEXT:  LBB45_2: ## %L2
 ; SKX-NEXT:    vmovaps %zmm0, 4(%rdi)
 ; SKX-NEXT:    vmovaps %zmm1, 68(%rdi)
 ; SKX-NEXT:    vzeroupper
@@ -2117,10 +2188,8 @@ define void @ktest_2(<32 x float> %in, float * %base) {
 ;
 ; AVX512BW-LABEL: ktest_2:
 ; AVX512BW:       ## %bb.0:
-; AVX512BW-NEXT:    vmovups (%rdi), %zmm2
-; AVX512BW-NEXT:    vmovups 64(%rdi), %zmm3
-; AVX512BW-NEXT:    vcmpltps %zmm0, %zmm2, %k1
-; AVX512BW-NEXT:    vcmpltps %zmm1, %zmm3, %k2
+; AVX512BW-NEXT:    vcmpgtps (%rdi), %zmm0, %k1
+; AVX512BW-NEXT:    vcmpgtps 64(%rdi), %zmm1, %k2
 ; AVX512BW-NEXT:    kunpckwd %k1, %k2, %k0
 ; AVX512BW-NEXT:    vmovups 68(%rdi), %zmm2 {%k2} {z}
 ; AVX512BW-NEXT:    vmovups 4(%rdi), %zmm3 {%k1} {z}
@@ -2128,13 +2197,13 @@ define void @ktest_2(<32 x float> %in, float * %base) {
 ; AVX512BW-NEXT:    vcmpltps %zmm2, %zmm1, %k2
 ; AVX512BW-NEXT:    kunpckwd %k1, %k2, %k1
 ; AVX512BW-NEXT:    kortestd %k1, %k0
-; AVX512BW-NEXT:    je LBB44_2
+; AVX512BW-NEXT:    je LBB45_2
 ; AVX512BW-NEXT:  ## %bb.1: ## %L1
 ; AVX512BW-NEXT:    vmovaps %zmm0, (%rdi)
 ; AVX512BW-NEXT:    vmovaps %zmm1, 64(%rdi)
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
-; AVX512BW-NEXT:  LBB44_2: ## %L2
+; AVX512BW-NEXT:  LBB45_2: ## %L2
 ; AVX512BW-NEXT:    vmovaps %zmm0, 4(%rdi)
 ; AVX512BW-NEXT:    vmovaps %zmm1, 68(%rdi)
 ; AVX512BW-NEXT:    vzeroupper
@@ -2142,10 +2211,8 @@ define void @ktest_2(<32 x float> %in, float * %base) {
 ;
 ; AVX512DQ-LABEL: ktest_2:
 ; AVX512DQ:       ## %bb.0:
-; AVX512DQ-NEXT:    vmovups (%rdi), %zmm2
-; AVX512DQ-NEXT:    vmovups 64(%rdi), %zmm3
-; AVX512DQ-NEXT:    vcmpltps %zmm1, %zmm3, %k1
-; AVX512DQ-NEXT:    vcmpltps %zmm0, %zmm2, %k2
+; AVX512DQ-NEXT:    vcmpgtps 64(%rdi), %zmm1, %k1
+; AVX512DQ-NEXT:    vcmpgtps (%rdi), %zmm0, %k2
 ; AVX512DQ-NEXT:    vmovups 4(%rdi), %zmm2 {%k2} {z}
 ; AVX512DQ-NEXT:    vmovups 68(%rdi), %zmm3 {%k1} {z}
 ; AVX512DQ-NEXT:    vcmpltps %zmm3, %zmm1, %k0
@@ -2156,13 +2223,13 @@ define void @ktest_2(<32 x float> %in, float * %base) {
 ; AVX512DQ-NEXT:    kmovw %k0, %ecx
 ; AVX512DQ-NEXT:    shll $16, %ecx
 ; AVX512DQ-NEXT:    orl %eax, %ecx
-; AVX512DQ-NEXT:    je LBB44_2
+; AVX512DQ-NEXT:    je LBB45_2
 ; AVX512DQ-NEXT:  ## %bb.1: ## %L1
 ; AVX512DQ-NEXT:    vmovaps %zmm0, (%rdi)
 ; AVX512DQ-NEXT:    vmovaps %zmm1, 64(%rdi)
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
-; AVX512DQ-NEXT:  LBB44_2: ## %L2
+; AVX512DQ-NEXT:  LBB45_2: ## %L2
 ; AVX512DQ-NEXT:    vmovaps %zmm0, 4(%rdi)
 ; AVX512DQ-NEXT:    vmovaps %zmm1, 68(%rdi)
 ; AVX512DQ-NEXT:    vzeroupper
@@ -2171,10 +2238,8 @@ define void @ktest_2(<32 x float> %in, float * %base) {
 ; X86-LABEL: ktest_2:
 ; X86:       ## %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    vmovups (%eax), %zmm2
-; X86-NEXT:    vmovups 64(%eax), %zmm3
-; X86-NEXT:    vcmpltps %zmm0, %zmm2, %k1
-; X86-NEXT:    vcmpltps %zmm1, %zmm3, %k2
+; X86-NEXT:    vcmpgtps (%eax), %zmm0, %k1
+; X86-NEXT:    vcmpgtps 64(%eax), %zmm1, %k2
 ; X86-NEXT:    kunpckwd %k1, %k2, %k0
 ; X86-NEXT:    vmovups 68(%eax), %zmm2 {%k2} {z}
 ; X86-NEXT:    vmovups 4(%eax), %zmm3 {%k1} {z}
@@ -2182,13 +2247,13 @@ define void @ktest_2(<32 x float> %in, float * %base) {
 ; X86-NEXT:    vcmpltps %zmm2, %zmm1, %k2
 ; X86-NEXT:    kunpckwd %k1, %k2, %k1
 ; X86-NEXT:    kortestd %k1, %k0
-; X86-NEXT:    je LBB44_2
+; X86-NEXT:    je LBB45_2
 ; X86-NEXT:  ## %bb.1: ## %L1
 ; X86-NEXT:    vmovaps %zmm0, (%eax)
 ; X86-NEXT:    vmovaps %zmm1, 64(%eax)
 ; X86-NEXT:    vzeroupper
 ; X86-NEXT:    retl
-; X86-NEXT:  LBB44_2: ## %L2
+; X86-NEXT:  LBB45_2: ## %L2
 ; X86-NEXT:    vmovaps %zmm0, 4(%eax)
 ; X86-NEXT:    vmovaps %zmm1, 68(%eax)
 ; X86-NEXT:    vzeroupper
@@ -2296,21 +2361,22 @@ define <2 x i16> @load_2i1(<2 x i1>* %a) {
 ; KNL-LABEL: load_2i1:
 ; KNL:       ## %bb.0:
 ; KNL-NEXT:    kmovw (%rdi), %k1
-; KNL-NEXT:    vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
-; KNL-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $zmm0
+; KNL-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; KNL-NEXT:    vpmovdw %zmm0, %ymm0
+; KNL-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $ymm0
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: load_2i1:
 ; SKX:       ## %bb.0:
 ; SKX-NEXT:    kmovb (%rdi), %k0
-; SKX-NEXT:    vpmovm2q %k0, %xmm0
+; SKX-NEXT:    vpmovm2w %k0, %xmm0
 ; SKX-NEXT:    retq
 ;
 ; AVX512BW-LABEL: load_2i1:
 ; AVX512BW:       ## %bb.0:
-; AVX512BW-NEXT:    kmovw (%rdi), %k1
-; AVX512BW-NEXT:    vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512BW-NEXT:    kmovw (%rdi), %k0
+; AVX512BW-NEXT:    vpmovm2w %k0, %zmm0
 ; AVX512BW-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $zmm0
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -2318,8 +2384,9 @@ define <2 x i16> @load_2i1(<2 x i1>* %a) {
 ; AVX512DQ-LABEL: load_2i1:
 ; AVX512DQ:       ## %bb.0:
 ; AVX512DQ-NEXT:    kmovb (%rdi), %k0
-; AVX512DQ-NEXT:    vpmovm2q %k0, %zmm0
-; AVX512DQ-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512DQ-NEXT:    vpmovm2d %k0, %zmm0
+; AVX512DQ-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512DQ-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $ymm0
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
 ;
@@ -2327,7 +2394,7 @@ define <2 x i16> @load_2i1(<2 x i1>* %a) {
 ; X86:       ## %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    kmovb (%eax), %k0
-; X86-NEXT:    vpmovm2q %k0, %xmm0
+; X86-NEXT:    vpmovm2w %k0, %xmm0
 ; X86-NEXT:    retl
   %b = load <2 x i1>, <2 x i1>* %a
   %c = sext <2 x i1> %b to <2 x i16>
@@ -2339,20 +2406,21 @@ define <4 x i16> @load_4i1(<4 x i1>* %a) {
 ; KNL:       ## %bb.0:
 ; KNL-NEXT:    kmovw (%rdi), %k1
 ; KNL-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
-; KNL-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $zmm0
+; KNL-NEXT:    vpmovdw %zmm0, %ymm0
+; KNL-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $ymm0
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: load_4i1:
 ; SKX:       ## %bb.0:
 ; SKX-NEXT:    kmovb (%rdi), %k0
-; SKX-NEXT:    vpmovm2d %k0, %xmm0
+; SKX-NEXT:    vpmovm2w %k0, %xmm0
 ; SKX-NEXT:    retq
 ;
 ; AVX512BW-LABEL: load_4i1:
 ; AVX512BW:       ## %bb.0:
-; AVX512BW-NEXT:    kmovw (%rdi), %k1
-; AVX512BW-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512BW-NEXT:    kmovw (%rdi), %k0
+; AVX512BW-NEXT:    vpmovm2w %k0, %zmm0
 ; AVX512BW-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $zmm0
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -2361,7 +2429,8 @@ define <4 x i16> @load_4i1(<4 x i1>* %a) {
 ; AVX512DQ:       ## %bb.0:
 ; AVX512DQ-NEXT:    kmovb (%rdi), %k0
 ; AVX512DQ-NEXT:    vpmovm2d %k0, %zmm0
-; AVX512DQ-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512DQ-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512DQ-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $ymm0
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
 ;
@@ -2369,7 +2438,7 @@ define <4 x i16> @load_4i1(<4 x i1>* %a) {
 ; X86:       ## %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    kmovb (%eax), %k0
-; X86-NEXT:    vpmovm2d %k0, %xmm0
+; X86-NEXT:    vpmovm2w %k0, %xmm0
 ; X86-NEXT:    retl
   %b = load <4 x i1>, <4 x i1>* %a
   %c = sext <4 x i1> %b to <4 x i16>
@@ -2381,10 +2450,11 @@ define <32 x i16> @load_32i1(<32 x i1>* %a) {
 ; KNL:       ## %bb.0:
 ; KNL-NEXT:    kmovw (%rdi), %k1
 ; KNL-NEXT:    kmovw 2(%rdi), %k2
-; KNL-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; KNL-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k2} {z}
 ; KNL-NEXT:    vpmovdw %zmm0, %ymm0
-; KNL-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k2} {z}
+; KNL-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k1} {z}
 ; KNL-NEXT:    vpmovdw %zmm1, %ymm1
+; KNL-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: load_32i1:
@@ -2403,10 +2473,11 @@ define <32 x i16> @load_32i1(<32 x i1>* %a) {
 ; AVX512DQ:       ## %bb.0:
 ; AVX512DQ-NEXT:    kmovw (%rdi), %k0
 ; AVX512DQ-NEXT:    kmovw 2(%rdi), %k1
-; AVX512DQ-NEXT:    vpmovm2d %k0, %zmm0
+; AVX512DQ-NEXT:    vpmovm2d %k1, %zmm0
 ; AVX512DQ-NEXT:    vpmovdw %zmm0, %ymm0
-; AVX512DQ-NEXT:    vpmovm2d %k1, %zmm1
+; AVX512DQ-NEXT:    vpmovm2d %k0, %zmm1
 ; AVX512DQ-NEXT:    vpmovdw %zmm1, %ymm1
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; AVX512DQ-NEXT:    retq
 ;
 ; X86-LABEL: load_32i1:
@@ -2427,16 +2498,17 @@ define <64 x i8> @load_64i1(<64 x i1>* %a) {
 ; KNL-NEXT:    kmovw 2(%rdi), %k2
 ; KNL-NEXT:    kmovw 4(%rdi), %k3
 ; KNL-NEXT:    kmovw 6(%rdi), %k4
-; KNL-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; KNL-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k3} {z}
 ; KNL-NEXT:    vpmovdb %zmm0, %xmm0
-; KNL-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k2} {z}
+; KNL-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k4} {z}
 ; KNL-NEXT:    vpmovdb %zmm1, %xmm1
 ; KNL-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
-; KNL-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k3} {z}
+; KNL-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k1} {z}
 ; KNL-NEXT:    vpmovdb %zmm1, %xmm1
-; KNL-NEXT:    vpternlogd $255, %zmm2, %zmm2, %zmm2 {%k4} {z}
+; KNL-NEXT:    vpternlogd $255, %zmm2, %zmm2, %zmm2 {%k2} {z}
 ; KNL-NEXT:    vpmovdb %zmm2, %xmm2
 ; KNL-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; KNL-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: load_64i1:
@@ -2457,16 +2529,17 @@ define <64 x i8> @load_64i1(<64 x i1>* %a) {
 ; AVX512DQ-NEXT:    kmovw 2(%rdi), %k1
 ; AVX512DQ-NEXT:    kmovw 4(%rdi), %k2
 ; AVX512DQ-NEXT:    kmovw 6(%rdi), %k3
-; AVX512DQ-NEXT:    vpmovm2d %k0, %zmm0
+; AVX512DQ-NEXT:    vpmovm2d %k2, %zmm0
 ; AVX512DQ-NEXT:    vpmovdb %zmm0, %xmm0
-; AVX512DQ-NEXT:    vpmovm2d %k1, %zmm1
+; AVX512DQ-NEXT:    vpmovm2d %k3, %zmm1
 ; AVX512DQ-NEXT:    vpmovdb %zmm1, %xmm1
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpmovm2d %k2, %zmm1
+; AVX512DQ-NEXT:    vpmovm2d %k0, %zmm1
 ; AVX512DQ-NEXT:    vpmovdb %zmm1, %xmm1
-; AVX512DQ-NEXT:    vpmovm2d %k3, %zmm2
+; AVX512DQ-NEXT:    vpmovm2d %k1, %zmm2
 ; AVX512DQ-NEXT:    vpmovdb %zmm2, %xmm2
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; AVX512DQ-NEXT:    retq
 ;
 ; X86-LABEL: load_64i1:
@@ -2680,6 +2753,7 @@ define void @store_32i1(<32 x i1>* %a, <32 x i1> %v) {
 define void @store_32i1_1(<32 x i1>* %a, <32 x i16> %v) {
 ; KNL-LABEL: store_32i1_1:
 ; KNL:       ## %bb.0:
+; KNL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; KNL-NEXT:    vpmovsxwd %ymm0, %zmm0
 ; KNL-NEXT:    vpslld $31, %zmm0, %zmm0
 ; KNL-NEXT:    vptestmd %zmm0, %zmm0, %k0
@@ -2709,6 +2783,7 @@ define void @store_32i1_1(<32 x i1>* %a, <32 x i16> %v) {
 ;
 ; AVX512DQ-LABEL: store_32i1_1:
 ; AVX512DQ:       ## %bb.0:
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512DQ-NEXT:    vpmovsxwd %ymm0, %zmm0
 ; AVX512DQ-NEXT:    vpslld $31, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    vpmovd2m %zmm0, %k0
@@ -2738,23 +2813,446 @@ define void @store_64i1(<64 x i1>* %a, <64 x i1> %v) {
 ;
 ; KNL-LABEL: store_64i1:
 ; KNL:       ## %bb.0:
-; KNL-NEXT:    vpmovsxbd %xmm0, %zmm0
-; KNL-NEXT:    vpslld $31, %zmm0, %zmm0
-; KNL-NEXT:    vptestmd %zmm0, %zmm0, %k0
-; KNL-NEXT:    vpmovsxbd %xmm1, %zmm0
-; KNL-NEXT:    vpslld $31, %zmm0, %zmm0
-; KNL-NEXT:    vptestmd %zmm0, %zmm0, %k1
-; KNL-NEXT:    vpmovsxbd %xmm2, %zmm0
-; KNL-NEXT:    vpslld $31, %zmm0, %zmm0
-; KNL-NEXT:    vptestmd %zmm0, %zmm0, %k2
-; KNL-NEXT:    vpmovsxbd %xmm3, %zmm0
-; KNL-NEXT:    vpslld $31, %zmm0, %zmm0
-; KNL-NEXT:    vptestmd %zmm0, %zmm0, %k3
-; KNL-NEXT:    kmovw %k3, 6(%rdi)
-; KNL-NEXT:    kmovw %k2, 4(%rdi)
-; KNL-NEXT:    kmovw %k1, 2(%rdi)
+; KNL-NEXT:    movw $-3, %ax
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kmovw %esi, %k0
+; KNL-NEXT:    kandw %k1, %k0, %k0
+; KNL-NEXT:    kmovw %k1, %k2
+; KNL-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; KNL-NEXT:    kmovw %edx, %k1
+; KNL-NEXT:    kshiftlw $15, %k1, %k1
+; KNL-NEXT:    kshiftrw $14, %k1, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    movw $-5, %ax
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; KNL-NEXT:    kandw %k1, %k0, %k0
+; KNL-NEXT:    kmovw %ecx, %k1
+; KNL-NEXT:    kshiftlw $15, %k1, %k1
+; KNL-NEXT:    kshiftrw $13, %k1, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    movw $-9, %ax
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; KNL-NEXT:    kandw %k1, %k0, %k0
+; KNL-NEXT:    kmovw %r8d, %k1
+; KNL-NEXT:    kshiftlw $15, %k1, %k1
+; KNL-NEXT:    kshiftrw $12, %k1, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    movw $-17, %ax
+; KNL-NEXT:    kmovw %eax, %k6
+; KNL-NEXT:    kandw %k6, %k0, %k0
+; KNL-NEXT:    kmovw %k6, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; KNL-NEXT:    kmovw %r9d, %k1
+; KNL-NEXT:    kshiftlw $15, %k1, %k1
+; KNL-NEXT:    kshiftrw $11, %k1, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    movw $-33, %ax
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kandw %k1, %k0, %k0
+; KNL-NEXT:    kmovw %k1, %k3
+; KNL-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kshiftlw $15, %k1, %k1
+; KNL-NEXT:    kshiftrw $10, %k1, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    movw $-65, %ax
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; KNL-NEXT:    kandw %k1, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kshiftlw $15, %k1, %k1
+; KNL-NEXT:    kshiftrw $9, %k1, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    movw $-129, %ax
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kandw %k1, %k0, %k0
+; KNL-NEXT:    kmovw %k1, %k4
+; KNL-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kshiftlw $15, %k1, %k1
+; KNL-NEXT:    kshiftrw $8, %k1, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    movw $-257, %ax ## imm = 0xFEFF
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; KNL-NEXT:    kandw %k1, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kshiftlw $15, %k1, %k1
+; KNL-NEXT:    kshiftrw $7, %k1, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    movw $-513, %ax ## imm = 0xFDFF
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kandw %k1, %k0, %k0
+; KNL-NEXT:    kmovw %k1, %k5
+; KNL-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kshiftlw $15, %k1, %k1
+; KNL-NEXT:    kshiftrw $6, %k1, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    movw $-1025, %ax ## imm = 0xFBFF
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; KNL-NEXT:    kandw %k1, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kshiftlw $15, %k1, %k1
+; KNL-NEXT:    kshiftrw $5, %k1, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    movw $-2049, %ax ## imm = 0xF7FF
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; KNL-NEXT:    kandw %k1, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kshiftlw $15, %k1, %k1
+; KNL-NEXT:    kshiftrw $4, %k1, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    movw $-4097, %ax ## imm = 0xEFFF
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; KNL-NEXT:    kandw %k1, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kshiftlw $15, %k1, %k1
+; KNL-NEXT:    kshiftrw $3, %k1, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    movw $-8193, %ax ## imm = 0xDFFF
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; KNL-NEXT:    kandw %k1, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kshiftlw $15, %k1, %k1
+; KNL-NEXT:    kshiftrw $2, %k1, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    movw $-16385, %ax ## imm = 0xBFFF
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; KNL-NEXT:    kandw %k1, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $14, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kshiftlw $1, %k0, %k0
+; KNL-NEXT:    kshiftrw $1, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kmovw %k0, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k0
+; KNL-NEXT:    kandw %k2, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $14, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 2-byte Reload
+; KNL-NEXT:    kandw %k1, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $13, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 ## 2-byte Reload
+; KNL-NEXT:    kandw %k2, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $12, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kandw %k6, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $11, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kandw %k3, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $10, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k6 ## 2-byte Reload
+; KNL-NEXT:    kandw %k6, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $9, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kandw %k4, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $8, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k3 ## 2-byte Reload
+; KNL-NEXT:    kandw %k3, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $7, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kandw %k5, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $6, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k4 ## 2-byte Reload
+; KNL-NEXT:    kandw %k4, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $5, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k5 ## 2-byte Reload
+; KNL-NEXT:    kandw %k5, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $4, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 ## 2-byte Reload
+; KNL-NEXT:    kandw %k7, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $3, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 ## 2-byte Reload
+; KNL-NEXT:    kandw %k7, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $2, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 ## 2-byte Reload
+; KNL-NEXT:    kandw %k7, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $14, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kshiftlw $1, %k0, %k0
+; KNL-NEXT:    kshiftrw $1, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kmovw %k0, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k0
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 ## 2-byte Reload
+; KNL-NEXT:    kandw %k7, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $14, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kandw %k1, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $13, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kandw %k2, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $12, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 2-byte Reload
+; KNL-NEXT:    kandw %k1, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $11, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 ## 2-byte Reload
+; KNL-NEXT:    kandw %k2, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $10, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kandw %k6, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $9, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 ## 2-byte Reload
+; KNL-NEXT:    kandw %k2, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $8, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kandw %k3, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $7, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k3 ## 2-byte Reload
+; KNL-NEXT:    kandw %k3, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $6, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kandw %k4, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $5, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kandw %k5, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $4, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 ## 2-byte Reload
+; KNL-NEXT:    kandw %k2, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $3, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k5 ## 2-byte Reload
+; KNL-NEXT:    kandw %k5, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $2, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k5 ## 2-byte Reload
+; KNL-NEXT:    kandw %k5, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $14, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    kshiftlw $1, %k0, %k0
+; KNL-NEXT:    kshiftrw $1, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k5 ## 2-byte Reload
+; KNL-NEXT:    kandw %k5, %k7, %k7
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k6
+; KNL-NEXT:    kshiftlw $15, %k6, %k6
+; KNL-NEXT:    kshiftrw $14, %k6, %k6
+; KNL-NEXT:    korw %k6, %k7, %k6
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k5 ## 2-byte Reload
+; KNL-NEXT:    kandw %k5, %k6, %k6
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $13, %k7, %k7
+; KNL-NEXT:    korw %k7, %k6, %k6
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k5 ## 2-byte Reload
+; KNL-NEXT:    kandw %k5, %k6, %k6
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $12, %k7, %k7
+; KNL-NEXT:    korw %k7, %k6, %k6
+; KNL-NEXT:    kandw %k1, %k6, %k6
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $11, %k7, %k7
+; KNL-NEXT:    korw %k7, %k6, %k6
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 2-byte Reload
+; KNL-NEXT:    kandw %k1, %k6, %k6
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $10, %k7, %k7
+; KNL-NEXT:    korw %k7, %k6, %k6
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 2-byte Reload
+; KNL-NEXT:    kandw %k1, %k6, %k6
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $9, %k7, %k7
+; KNL-NEXT:    korw %k7, %k6, %k6
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 2-byte Reload
+; KNL-NEXT:    kandw %k1, %k6, %k6
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $8, %k7, %k7
+; KNL-NEXT:    korw %k7, %k6, %k6
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 2-byte Reload
+; KNL-NEXT:    kandw %k1, %k6, %k6
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $7, %k7, %k7
+; KNL-NEXT:    korw %k7, %k6, %k6
+; KNL-NEXT:    kandw %k3, %k6, %k6
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $6, %k7, %k7
+; KNL-NEXT:    korw %k7, %k6, %k6
+; KNL-NEXT:    kandw %k4, %k6, %k5
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k6
+; KNL-NEXT:    kshiftlw $15, %k6, %k6
+; KNL-NEXT:    kshiftrw $5, %k6, %k6
+; KNL-NEXT:    korw %k6, %k5, %k5
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 2-byte Reload
+; KNL-NEXT:    kandw %k1, %k5, %k4
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k5
+; KNL-NEXT:    kshiftlw $15, %k5, %k5
+; KNL-NEXT:    kshiftrw $4, %k5, %k5
+; KNL-NEXT:    korw %k5, %k4, %k4
+; KNL-NEXT:    kandw %k2, %k4, %k3
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k4
+; KNL-NEXT:    kshiftlw $15, %k4, %k4
+; KNL-NEXT:    kshiftrw $3, %k4, %k4
+; KNL-NEXT:    korw %k4, %k3, %k3
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 2-byte Reload
+; KNL-NEXT:    kandw %k1, %k3, %k2
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k3
+; KNL-NEXT:    kshiftlw $15, %k3, %k3
+; KNL-NEXT:    kshiftrw $2, %k3, %k3
+; KNL-NEXT:    korw %k3, %k2, %k2
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 2-byte Reload
+; KNL-NEXT:    kandw %k1, %k2, %k1
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k2
+; KNL-NEXT:    kshiftlw $14, %k2, %k2
+; KNL-NEXT:    korw %k2, %k1, %k1
+; KNL-NEXT:    kshiftlw $1, %k1, %k1
+; KNL-NEXT:    kshiftrw $1, %k1, %k1
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k2
+; KNL-NEXT:    kshiftlw $15, %k2, %k2
+; KNL-NEXT:    korw %k2, %k1, %k1
+; KNL-NEXT:    kmovw %k1, 6(%rdi)
+; KNL-NEXT:    kmovw %k0, 4(%rdi)
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 ## 2-byte Reload
+; KNL-NEXT:    kmovw %k0, 2(%rdi)
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 ## 2-byte Reload
 ; KNL-NEXT:    kmovw %k0, (%rdi)
-; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: store_64i1:
@@ -2775,23 +3273,446 @@ define void @store_64i1(<64 x i1>* %a, <64 x i1> %v) {
 ;
 ; AVX512DQ-LABEL: store_64i1:
 ; AVX512DQ:       ## %bb.0:
-; AVX512DQ-NEXT:    vpmovsxbd %xmm0, %zmm0
-; AVX512DQ-NEXT:    vpslld $31, %zmm0, %zmm0
-; AVX512DQ-NEXT:    vpmovd2m %zmm0, %k0
-; AVX512DQ-NEXT:    vpmovsxbd %xmm1, %zmm0
-; AVX512DQ-NEXT:    vpslld $31, %zmm0, %zmm0
-; AVX512DQ-NEXT:    vpmovd2m %zmm0, %k1
-; AVX512DQ-NEXT:    vpmovsxbd %xmm2, %zmm0
-; AVX512DQ-NEXT:    vpslld $31, %zmm0, %zmm0
-; AVX512DQ-NEXT:    vpmovd2m %zmm0, %k2
-; AVX512DQ-NEXT:    vpmovsxbd %xmm3, %zmm0
-; AVX512DQ-NEXT:    vpslld $31, %zmm0, %zmm0
-; AVX512DQ-NEXT:    vpmovd2m %zmm0, %k3
-; AVX512DQ-NEXT:    kmovw %k3, 6(%rdi)
-; AVX512DQ-NEXT:    kmovw %k2, 4(%rdi)
-; AVX512DQ-NEXT:    kmovw %k1, 2(%rdi)
+; AVX512DQ-NEXT:    movw $-3, %ax
+; AVX512DQ-NEXT:    kmovw %eax, %k1
+; AVX512DQ-NEXT:    kmovw %esi, %k0
+; AVX512DQ-NEXT:    kandw %k1, %k0, %k0
+; AVX512DQ-NEXT:    kmovw %k1, %k2
+; AVX512DQ-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; AVX512DQ-NEXT:    kmovw %edx, %k1
+; AVX512DQ-NEXT:    kshiftlw $15, %k1, %k1
+; AVX512DQ-NEXT:    kshiftrw $14, %k1, %k1
+; AVX512DQ-NEXT:    korw %k1, %k0, %k0
+; AVX512DQ-NEXT:    movw $-5, %ax
+; AVX512DQ-NEXT:    kmovw %eax, %k1
+; AVX512DQ-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; AVX512DQ-NEXT:    kandw %k1, %k0, %k0
+; AVX512DQ-NEXT:    kmovw %ecx, %k1
+; AVX512DQ-NEXT:    kshiftlw $15, %k1, %k1
+; AVX512DQ-NEXT:    kshiftrw $13, %k1, %k1
+; AVX512DQ-NEXT:    korw %k1, %k0, %k0
+; AVX512DQ-NEXT:    movw $-9, %ax
+; AVX512DQ-NEXT:    kmovw %eax, %k1
+; AVX512DQ-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; AVX512DQ-NEXT:    kandw %k1, %k0, %k0
+; AVX512DQ-NEXT:    kmovw %r8d, %k1
+; AVX512DQ-NEXT:    kshiftlw $15, %k1, %k1
+; AVX512DQ-NEXT:    kshiftrw $12, %k1, %k1
+; AVX512DQ-NEXT:    korw %k1, %k0, %k0
+; AVX512DQ-NEXT:    movw $-17, %ax
+; AVX512DQ-NEXT:    kmovw %eax, %k6
+; AVX512DQ-NEXT:    kandw %k6, %k0, %k0
+; AVX512DQ-NEXT:    kmovw %k6, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; AVX512DQ-NEXT:    kmovw %r9d, %k1
+; AVX512DQ-NEXT:    kshiftlw $15, %k1, %k1
+; AVX512DQ-NEXT:    kshiftrw $11, %k1, %k1
+; AVX512DQ-NEXT:    korw %k1, %k0, %k0
+; AVX512DQ-NEXT:    movw $-33, %ax
+; AVX512DQ-NEXT:    kmovw %eax, %k1
+; AVX512DQ-NEXT:    kandw %k1, %k0, %k0
+; AVX512DQ-NEXT:    kmovw %k1, %k3
+; AVX512DQ-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k1
+; AVX512DQ-NEXT:    kshiftlw $15, %k1, %k1
+; AVX512DQ-NEXT:    kshiftrw $10, %k1, %k1
+; AVX512DQ-NEXT:    korw %k1, %k0, %k0
+; AVX512DQ-NEXT:    movw $-65, %ax
+; AVX512DQ-NEXT:    kmovw %eax, %k1
+; AVX512DQ-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; AVX512DQ-NEXT:    kandw %k1, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k1
+; AVX512DQ-NEXT:    kshiftlw $15, %k1, %k1
+; AVX512DQ-NEXT:    kshiftrw $9, %k1, %k1
+; AVX512DQ-NEXT:    korw %k1, %k0, %k0
+; AVX512DQ-NEXT:    movw $-129, %ax
+; AVX512DQ-NEXT:    kmovw %eax, %k1
+; AVX512DQ-NEXT:    kandw %k1, %k0, %k0
+; AVX512DQ-NEXT:    kmovw %k1, %k4
+; AVX512DQ-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k1
+; AVX512DQ-NEXT:    kshiftlw $15, %k1, %k1
+; AVX512DQ-NEXT:    kshiftrw $8, %k1, %k1
+; AVX512DQ-NEXT:    korw %k1, %k0, %k0
+; AVX512DQ-NEXT:    movw $-257, %ax ## imm = 0xFEFF
+; AVX512DQ-NEXT:    kmovw %eax, %k1
+; AVX512DQ-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; AVX512DQ-NEXT:    kandw %k1, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k1
+; AVX512DQ-NEXT:    kshiftlw $15, %k1, %k1
+; AVX512DQ-NEXT:    kshiftrw $7, %k1, %k1
+; AVX512DQ-NEXT:    korw %k1, %k0, %k0
+; AVX512DQ-NEXT:    movw $-513, %ax ## imm = 0xFDFF
+; AVX512DQ-NEXT:    kmovw %eax, %k1
+; AVX512DQ-NEXT:    kandw %k1, %k0, %k0
+; AVX512DQ-NEXT:    kmovw %k1, %k5
+; AVX512DQ-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k1
+; AVX512DQ-NEXT:    kshiftlw $15, %k1, %k1
+; AVX512DQ-NEXT:    kshiftrw $6, %k1, %k1
+; AVX512DQ-NEXT:    korw %k1, %k0, %k0
+; AVX512DQ-NEXT:    movw $-1025, %ax ## imm = 0xFBFF
+; AVX512DQ-NEXT:    kmovw %eax, %k1
+; AVX512DQ-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; AVX512DQ-NEXT:    kandw %k1, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k1
+; AVX512DQ-NEXT:    kshiftlw $15, %k1, %k1
+; AVX512DQ-NEXT:    kshiftrw $5, %k1, %k1
+; AVX512DQ-NEXT:    korw %k1, %k0, %k0
+; AVX512DQ-NEXT:    movw $-2049, %ax ## imm = 0xF7FF
+; AVX512DQ-NEXT:    kmovw %eax, %k1
+; AVX512DQ-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; AVX512DQ-NEXT:    kandw %k1, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k1
+; AVX512DQ-NEXT:    kshiftlw $15, %k1, %k1
+; AVX512DQ-NEXT:    kshiftrw $4, %k1, %k1
+; AVX512DQ-NEXT:    korw %k1, %k0, %k0
+; AVX512DQ-NEXT:    movw $-4097, %ax ## imm = 0xEFFF
+; AVX512DQ-NEXT:    kmovw %eax, %k1
+; AVX512DQ-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; AVX512DQ-NEXT:    kandw %k1, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k1
+; AVX512DQ-NEXT:    kshiftlw $15, %k1, %k1
+; AVX512DQ-NEXT:    kshiftrw $3, %k1, %k1
+; AVX512DQ-NEXT:    korw %k1, %k0, %k0
+; AVX512DQ-NEXT:    movw $-8193, %ax ## imm = 0xDFFF
+; AVX512DQ-NEXT:    kmovw %eax, %k1
+; AVX512DQ-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; AVX512DQ-NEXT:    kandw %k1, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k1
+; AVX512DQ-NEXT:    kshiftlw $15, %k1, %k1
+; AVX512DQ-NEXT:    kshiftrw $2, %k1, %k1
+; AVX512DQ-NEXT:    korw %k1, %k0, %k0
+; AVX512DQ-NEXT:    movw $-16385, %ax ## imm = 0xBFFF
+; AVX512DQ-NEXT:    kmovw %eax, %k1
+; AVX512DQ-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; AVX512DQ-NEXT:    kandw %k1, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k7
+; AVX512DQ-NEXT:    kshiftlw $14, %k7, %k7
+; AVX512DQ-NEXT:    korw %k7, %k0, %k0
+; AVX512DQ-NEXT:    kshiftlw $1, %k0, %k0
+; AVX512DQ-NEXT:    kshiftrw $1, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k7
+; AVX512DQ-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQ-NEXT:    korw %k7, %k0, %k0
+; AVX512DQ-NEXT:    kmovw %k0, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k0
+; AVX512DQ-NEXT:    kandw %k2, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k7
+; AVX512DQ-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQ-NEXT:    kshiftrw $14, %k7, %k7
+; AVX512DQ-NEXT:    korw %k7, %k0, %k0
+; AVX512DQ-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 2-byte Reload
+; AVX512DQ-NEXT:    kandw %k1, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k7
+; AVX512DQ-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQ-NEXT:    kshiftrw $13, %k7, %k7
+; AVX512DQ-NEXT:    korw %k7, %k0, %k0
+; AVX512DQ-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 ## 2-byte Reload
+; AVX512DQ-NEXT:    kandw %k2, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k7
+; AVX512DQ-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQ-NEXT:    kshiftrw $12, %k7, %k7
+; AVX512DQ-NEXT:    korw %k7, %k0, %k0
+; AVX512DQ-NEXT:    kandw %k6, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k7
+; AVX512DQ-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQ-NEXT:    kshiftrw $11, %k7, %k7
+; AVX512DQ-NEXT:    korw %k7, %k0, %k0
+; AVX512DQ-NEXT:    kandw %k3, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k7
+; AVX512DQ-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQ-NEXT:    kshiftrw $10, %k7, %k7
+; AVX512DQ-NEXT:    korw %k7, %k0, %k0
+; AVX512DQ-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k6 ## 2-byte Reload
+; AVX512DQ-NEXT:    kandw %k6, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k7
+; AVX512DQ-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQ-NEXT:    kshiftrw $9, %k7, %k7
+; AVX512DQ-NEXT:    korw %k7, %k0, %k0
+; AVX512DQ-NEXT:    kandw %k4, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k7
+; AVX512DQ-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQ-NEXT:    kshiftrw $8, %k7, %k7
+; AVX512DQ-NEXT:    korw %k7, %k0, %k0
+; AVX512DQ-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k3 ## 2-byte Reload
+; AVX512DQ-NEXT:    kandw %k3, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k7
+; AVX512DQ-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQ-NEXT:    kshiftrw $7, %k7, %k7
+; AVX512DQ-NEXT:    korw %k7, %k0, %k0
+; AVX512DQ-NEXT:    kandw %k5, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k7
+; AVX512DQ-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQ-NEXT:    kshiftrw $6, %k7, %k7
+; AVX512DQ-NEXT:    korw %k7, %k0, %k0
+; AVX512DQ-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k4 ## 2-byte Reload
+; AVX512DQ-NEXT:    kandw %k4, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k7
+; AVX512DQ-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQ-NEXT:    kshiftrw $5, %k7, %k7
+; AVX512DQ-NEXT:    korw %k7, %k0, %k0
+; AVX512DQ-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k5 ## 2-byte Reload
+; AVX512DQ-NEXT:    kandw %k5, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k7
+; AVX512DQ-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQ-NEXT:    kshiftrw $4, %k7, %k7
+; AVX512DQ-NEXT:    korw %k7, %k0, %k0
+; AVX512DQ-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 ## 2-byte Reload
+; AVX512DQ-NEXT:    kandw %k7, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k7
+; AVX512DQ-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQ-NEXT:    kshiftrw $3, %k7, %k7
+; AVX512DQ-NEXT:    korw %k7, %k0, %k0
+; AVX512DQ-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 ## 2-byte Reload
+; AVX512DQ-NEXT:    kandw %k7, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k7
+; AVX512DQ-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQ-NEXT:    kshiftrw $2, %k7, %k7
+; AVX512DQ-NEXT:    korw %k7, %k0, %k0
+; AVX512DQ-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 ## 2-byte Reload
+; AVX512DQ-NEXT:    kandw %k7, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k7
+; AVX512DQ-NEXT:    kshiftlw $14, %k7, %k7
+; AVX512DQ-NEXT:    korw %k7, %k0, %k0
+; AVX512DQ-NEXT:    kshiftlw $1, %k0, %k0
+; AVX512DQ-NEXT:    kshiftrw $1, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k7
+; AVX512DQ-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQ-NEXT:    korw %k7, %k0, %k0
+; AVX512DQ-NEXT:    kmovw %k0, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k0
+; AVX512DQ-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 ## 2-byte Reload
+; AVX512DQ-NEXT:    kandw %k7, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k7
+; AVX512DQ-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQ-NEXT:    kshiftrw $14, %k7, %k7
+; AVX512DQ-NEXT:    korw %k7, %k0, %k0
+; AVX512DQ-NEXT:    kandw %k1, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k7
+; AVX512DQ-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQ-NEXT:    kshiftrw $13, %k7, %k7
+; AVX512DQ-NEXT:    korw %k7, %k0, %k0
+; AVX512DQ-NEXT:    kandw %k2, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k7
+; AVX512DQ-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQ-NEXT:    kshiftrw $12, %k7, %k7
+; AVX512DQ-NEXT:    korw %k7, %k0, %k0
+; AVX512DQ-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 2-byte Reload
+; AVX512DQ-NEXT:    kandw %k1, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k7
+; AVX512DQ-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQ-NEXT:    kshiftrw $11, %k7, %k7
+; AVX512DQ-NEXT:    korw %k7, %k0, %k0
+; AVX512DQ-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 ## 2-byte Reload
+; AVX512DQ-NEXT:    kandw %k2, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k7
+; AVX512DQ-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQ-NEXT:    kshiftrw $10, %k7, %k7
+; AVX512DQ-NEXT:    korw %k7, %k0, %k0
+; AVX512DQ-NEXT:    kandw %k6, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k7
+; AVX512DQ-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQ-NEXT:    kshiftrw $9, %k7, %k7
+; AVX512DQ-NEXT:    korw %k7, %k0, %k0
+; AVX512DQ-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 ## 2-byte Reload
+; AVX512DQ-NEXT:    kandw %k2, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k7
+; AVX512DQ-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQ-NEXT:    kshiftrw $8, %k7, %k7
+; AVX512DQ-NEXT:    korw %k7, %k0, %k0
+; AVX512DQ-NEXT:    kandw %k3, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k7
+; AVX512DQ-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQ-NEXT:    kshiftrw $7, %k7, %k7
+; AVX512DQ-NEXT:    korw %k7, %k0, %k0
+; AVX512DQ-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k3 ## 2-byte Reload
+; AVX512DQ-NEXT:    kandw %k3, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k7
+; AVX512DQ-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQ-NEXT:    kshiftrw $6, %k7, %k7
+; AVX512DQ-NEXT:    korw %k7, %k0, %k0
+; AVX512DQ-NEXT:    kandw %k4, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k7
+; AVX512DQ-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQ-NEXT:    kshiftrw $5, %k7, %k7
+; AVX512DQ-NEXT:    korw %k7, %k0, %k0
+; AVX512DQ-NEXT:    kandw %k5, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k7
+; AVX512DQ-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQ-NEXT:    kshiftrw $4, %k7, %k7
+; AVX512DQ-NEXT:    korw %k7, %k0, %k0
+; AVX512DQ-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 ## 2-byte Reload
+; AVX512DQ-NEXT:    kandw %k2, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k7
+; AVX512DQ-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQ-NEXT:    kshiftrw $3, %k7, %k7
+; AVX512DQ-NEXT:    korw %k7, %k0, %k0
+; AVX512DQ-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k5 ## 2-byte Reload
+; AVX512DQ-NEXT:    kandw %k5, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k7
+; AVX512DQ-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQ-NEXT:    kshiftrw $2, %k7, %k7
+; AVX512DQ-NEXT:    korw %k7, %k0, %k0
+; AVX512DQ-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k5 ## 2-byte Reload
+; AVX512DQ-NEXT:    kandw %k5, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k7
+; AVX512DQ-NEXT:    kshiftlw $14, %k7, %k7
+; AVX512DQ-NEXT:    korw %k7, %k0, %k0
+; AVX512DQ-NEXT:    kshiftlw $1, %k0, %k0
+; AVX512DQ-NEXT:    kshiftrw $1, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k7
+; AVX512DQ-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQ-NEXT:    korw %k7, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k7
+; AVX512DQ-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k5 ## 2-byte Reload
+; AVX512DQ-NEXT:    kandw %k5, %k7, %k7
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k6
+; AVX512DQ-NEXT:    kshiftlw $15, %k6, %k6
+; AVX512DQ-NEXT:    kshiftrw $14, %k6, %k6
+; AVX512DQ-NEXT:    korw %k6, %k7, %k6
+; AVX512DQ-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k5 ## 2-byte Reload
+; AVX512DQ-NEXT:    kandw %k5, %k6, %k6
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k7
+; AVX512DQ-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQ-NEXT:    kshiftrw $13, %k7, %k7
+; AVX512DQ-NEXT:    korw %k7, %k6, %k6
+; AVX512DQ-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k5 ## 2-byte Reload
+; AVX512DQ-NEXT:    kandw %k5, %k6, %k6
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k7
+; AVX512DQ-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQ-NEXT:    kshiftrw $12, %k7, %k7
+; AVX512DQ-NEXT:    korw %k7, %k6, %k6
+; AVX512DQ-NEXT:    kandw %k1, %k6, %k6
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k7
+; AVX512DQ-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQ-NEXT:    kshiftrw $11, %k7, %k7
+; AVX512DQ-NEXT:    korw %k7, %k6, %k6
+; AVX512DQ-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 2-byte Reload
+; AVX512DQ-NEXT:    kandw %k1, %k6, %k6
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k7
+; AVX512DQ-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQ-NEXT:    kshiftrw $10, %k7, %k7
+; AVX512DQ-NEXT:    korw %k7, %k6, %k6
+; AVX512DQ-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 2-byte Reload
+; AVX512DQ-NEXT:    kandw %k1, %k6, %k6
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k7
+; AVX512DQ-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQ-NEXT:    kshiftrw $9, %k7, %k7
+; AVX512DQ-NEXT:    korw %k7, %k6, %k6
+; AVX512DQ-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 2-byte Reload
+; AVX512DQ-NEXT:    kandw %k1, %k6, %k6
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k7
+; AVX512DQ-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQ-NEXT:    kshiftrw $8, %k7, %k7
+; AVX512DQ-NEXT:    korw %k7, %k6, %k6
+; AVX512DQ-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 2-byte Reload
+; AVX512DQ-NEXT:    kandw %k1, %k6, %k6
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k7
+; AVX512DQ-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQ-NEXT:    kshiftrw $7, %k7, %k7
+; AVX512DQ-NEXT:    korw %k7, %k6, %k6
+; AVX512DQ-NEXT:    kandw %k3, %k6, %k6
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k7
+; AVX512DQ-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQ-NEXT:    kshiftrw $6, %k7, %k7
+; AVX512DQ-NEXT:    korw %k7, %k6, %k6
+; AVX512DQ-NEXT:    kandw %k4, %k6, %k5
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k6
+; AVX512DQ-NEXT:    kshiftlw $15, %k6, %k6
+; AVX512DQ-NEXT:    kshiftrw $5, %k6, %k6
+; AVX512DQ-NEXT:    korw %k6, %k5, %k5
+; AVX512DQ-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 2-byte Reload
+; AVX512DQ-NEXT:    kandw %k1, %k5, %k4
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k5
+; AVX512DQ-NEXT:    kshiftlw $15, %k5, %k5
+; AVX512DQ-NEXT:    kshiftrw $4, %k5, %k5
+; AVX512DQ-NEXT:    korw %k5, %k4, %k4
+; AVX512DQ-NEXT:    kandw %k2, %k4, %k3
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k4
+; AVX512DQ-NEXT:    kshiftlw $15, %k4, %k4
+; AVX512DQ-NEXT:    kshiftrw $3, %k4, %k4
+; AVX512DQ-NEXT:    korw %k4, %k3, %k3
+; AVX512DQ-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 2-byte Reload
+; AVX512DQ-NEXT:    kandw %k1, %k3, %k2
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k3
+; AVX512DQ-NEXT:    kshiftlw $15, %k3, %k3
+; AVX512DQ-NEXT:    kshiftrw $2, %k3, %k3
+; AVX512DQ-NEXT:    korw %k3, %k2, %k2
+; AVX512DQ-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 2-byte Reload
+; AVX512DQ-NEXT:    kandw %k1, %k2, %k1
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k2
+; AVX512DQ-NEXT:    kshiftlw $14, %k2, %k2
+; AVX512DQ-NEXT:    korw %k2, %k1, %k1
+; AVX512DQ-NEXT:    kshiftlw $1, %k1, %k1
+; AVX512DQ-NEXT:    kshiftrw $1, %k1, %k1
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k2
+; AVX512DQ-NEXT:    kshiftlw $15, %k2, %k2
+; AVX512DQ-NEXT:    korw %k2, %k1, %k1
+; AVX512DQ-NEXT:    kmovw %k1, 6(%rdi)
+; AVX512DQ-NEXT:    kmovw %k0, 4(%rdi)
+; AVX512DQ-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 ## 2-byte Reload
+; AVX512DQ-NEXT:    kmovw %k0, 2(%rdi)
+; AVX512DQ-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 ## 2-byte Reload
 ; AVX512DQ-NEXT:    kmovw %k0, (%rdi)
-; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
 ;
 ; X86-LABEL: store_64i1:
@@ -3201,12 +4122,12 @@ define void @ktest_signed(<16 x i32> %x, <16 x i32> %y) {
 ; KNL-NEXT:    vptestnmd %zmm0, %zmm0, %k0
 ; KNL-NEXT:    kmovw %k0, %eax
 ; KNL-NEXT:    testw %ax, %ax
-; KNL-NEXT:    jle LBB65_1
+; KNL-NEXT:    jle LBB66_1
 ; KNL-NEXT:  ## %bb.2: ## %bb.2
 ; KNL-NEXT:    popq %rax
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    retq
-; KNL-NEXT:  LBB65_1: ## %bb.1
+; KNL-NEXT:  LBB66_1: ## %bb.1
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    callq _foo
 ; KNL-NEXT:    popq %rax
@@ -3220,12 +4141,12 @@ define void @ktest_signed(<16 x i32> %x, <16 x i32> %y) {
 ; SKX-NEXT:    vptestnmd %zmm0, %zmm0, %k0
 ; SKX-NEXT:    kmovd %k0, %eax
 ; SKX-NEXT:    testw %ax, %ax
-; SKX-NEXT:    jle LBB65_1
+; SKX-NEXT:    jle LBB66_1
 ; SKX-NEXT:  ## %bb.2: ## %bb.2
 ; SKX-NEXT:    popq %rax
 ; SKX-NEXT:    vzeroupper
 ; SKX-NEXT:    retq
-; SKX-NEXT:  LBB65_1: ## %bb.1
+; SKX-NEXT:  LBB66_1: ## %bb.1
 ; SKX-NEXT:    vzeroupper
 ; SKX-NEXT:    callq _foo
 ; SKX-NEXT:    popq %rax
@@ -3239,12 +4160,12 @@ define void @ktest_signed(<16 x i32> %x, <16 x i32> %y) {
 ; AVX512BW-NEXT:    vptestnmd %zmm0, %zmm0, %k0
 ; AVX512BW-NEXT:    kmovd %k0, %eax
 ; AVX512BW-NEXT:    testw %ax, %ax
-; AVX512BW-NEXT:    jle LBB65_1
+; AVX512BW-NEXT:    jle LBB66_1
 ; AVX512BW-NEXT:  ## %bb.2: ## %bb.2
 ; AVX512BW-NEXT:    popq %rax
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
-; AVX512BW-NEXT:  LBB65_1: ## %bb.1
+; AVX512BW-NEXT:  LBB66_1: ## %bb.1
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    callq _foo
 ; AVX512BW-NEXT:    popq %rax
@@ -3258,12 +4179,12 @@ define void @ktest_signed(<16 x i32> %x, <16 x i32> %y) {
 ; AVX512DQ-NEXT:    vptestnmd %zmm0, %zmm0, %k0
 ; AVX512DQ-NEXT:    kmovw %k0, %eax
 ; AVX512DQ-NEXT:    testw %ax, %ax
-; AVX512DQ-NEXT:    jle LBB65_1
+; AVX512DQ-NEXT:    jle LBB66_1
 ; AVX512DQ-NEXT:  ## %bb.2: ## %bb.2
 ; AVX512DQ-NEXT:    popq %rax
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
-; AVX512DQ-NEXT:  LBB65_1: ## %bb.1
+; AVX512DQ-NEXT:  LBB66_1: ## %bb.1
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    callq _foo
 ; AVX512DQ-NEXT:    popq %rax
@@ -3277,12 +4198,12 @@ define void @ktest_signed(<16 x i32> %x, <16 x i32> %y) {
 ; X86-NEXT:    vptestnmd %zmm0, %zmm0, %k0
 ; X86-NEXT:    kmovd %k0, %eax
 ; X86-NEXT:    testw %ax, %ax
-; X86-NEXT:    jle LBB65_1
+; X86-NEXT:    jle LBB66_1
 ; X86-NEXT:  ## %bb.2: ## %bb.2
 ; X86-NEXT:    addl $12, %esp
 ; X86-NEXT:    vzeroupper
 ; X86-NEXT:    retl
-; X86-NEXT:  LBB65_1: ## %bb.1
+; X86-NEXT:  LBB66_1: ## %bb.1
 ; X86-NEXT:    vzeroupper
 ; X86-NEXT:    calll _foo
 ; X86-NEXT:    addl $12, %esp
@@ -3310,11 +4231,11 @@ define void @ktest_allones(<16 x i32> %x, <16 x i32> %y) {
 ; CHECK-NEXT:    vpord %zmm1, %zmm0, %zmm0
 ; CHECK-NEXT:    vptestnmd %zmm0, %zmm0, %k0
 ; CHECK-NEXT:    kortestw %k0, %k0
-; CHECK-NEXT:    jb LBB66_2
+; CHECK-NEXT:    jb LBB67_2
 ; CHECK-NEXT:  ## %bb.1: ## %bb.1
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    callq _foo
-; CHECK-NEXT:  LBB66_2: ## %bb.2
+; CHECK-NEXT:  LBB67_2: ## %bb.2
 ; CHECK-NEXT:    popq %rax
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
@@ -3326,11 +4247,11 @@ define void @ktest_allones(<16 x i32> %x, <16 x i32> %y) {
 ; X86-NEXT:    vpord %zmm1, %zmm0, %zmm0
 ; X86-NEXT:    vptestnmd %zmm0, %zmm0, %k0
 ; X86-NEXT:    kortestw %k0, %k0
-; X86-NEXT:    jb LBB66_2
+; X86-NEXT:    jb LBB67_2
 ; X86-NEXT:  ## %bb.1: ## %bb.1
 ; X86-NEXT:    vzeroupper
 ; X86-NEXT:    calll _foo
-; X86-NEXT:  LBB66_2: ## %bb.2
+; X86-NEXT:  LBB67_2: ## %bb.2
 ; X86-NEXT:    addl $12, %esp
 ; X86-NEXT:    vzeroupper
 ; X86-NEXT:    retl
@@ -3518,12 +4439,12 @@ define void @ktest_3(<8 x i32> %w, <8 x i32> %x, <8 x i32> %y, <8 x i32> %z) {
 ; KNL-NEXT:    kandw %k1, %k0, %k0
 ; KNL-NEXT:    kmovw %k0, %eax
 ; KNL-NEXT:    testb %al, %al
-; KNL-NEXT:    je LBB72_1
+; KNL-NEXT:    je LBB73_1
 ; KNL-NEXT:  ## %bb.2: ## %exit
 ; KNL-NEXT:    popq %rax
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    retq
-; KNL-NEXT:  LBB72_1: ## %bar
+; KNL-NEXT:  LBB73_1: ## %bar
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    callq _foo
 ; KNL-NEXT:    popq %rax
@@ -3540,12 +4461,12 @@ define void @ktest_3(<8 x i32> %w, <8 x i32> %x, <8 x i32> %y, <8 x i32> %z) {
 ; SKX-NEXT:    vptestnmd %ymm3, %ymm3, %k2
 ; SKX-NEXT:    korb %k2, %k1, %k1
 ; SKX-NEXT:    ktestb %k1, %k0
-; SKX-NEXT:    je LBB72_1
+; SKX-NEXT:    je LBB73_1
 ; SKX-NEXT:  ## %bb.2: ## %exit
 ; SKX-NEXT:    popq %rax
 ; SKX-NEXT:    vzeroupper
 ; SKX-NEXT:    retq
-; SKX-NEXT:  LBB72_1: ## %bar
+; SKX-NEXT:  LBB73_1: ## %bar
 ; SKX-NEXT:    vzeroupper
 ; SKX-NEXT:    callq _foo
 ; SKX-NEXT:    popq %rax
@@ -3568,12 +4489,12 @@ define void @ktest_3(<8 x i32> %w, <8 x i32> %x, <8 x i32> %y, <8 x i32> %z) {
 ; AVX512BW-NEXT:    kandw %k1, %k0, %k0
 ; AVX512BW-NEXT:    kmovd %k0, %eax
 ; AVX512BW-NEXT:    testb %al, %al
-; AVX512BW-NEXT:    je LBB72_1
+; AVX512BW-NEXT:    je LBB73_1
 ; AVX512BW-NEXT:  ## %bb.2: ## %exit
 ; AVX512BW-NEXT:    popq %rax
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
-; AVX512BW-NEXT:  LBB72_1: ## %bar
+; AVX512BW-NEXT:  LBB73_1: ## %bar
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    callq _foo
 ; AVX512BW-NEXT:    popq %rax
@@ -3594,12 +4515,12 @@ define void @ktest_3(<8 x i32> %w, <8 x i32> %x, <8 x i32> %y, <8 x i32> %z) {
 ; AVX512DQ-NEXT:    korb %k1, %k0, %k0
 ; AVX512DQ-NEXT:    korb %k3, %k2, %k1
 ; AVX512DQ-NEXT:    ktestb %k1, %k0
-; AVX512DQ-NEXT:    je LBB72_1
+; AVX512DQ-NEXT:    je LBB73_1
 ; AVX512DQ-NEXT:  ## %bb.2: ## %exit
 ; AVX512DQ-NEXT:    popq %rax
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
-; AVX512DQ-NEXT:  LBB72_1: ## %bar
+; AVX512DQ-NEXT:  LBB73_1: ## %bar
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    callq _foo
 ; AVX512DQ-NEXT:    popq %rax
@@ -3616,12 +4537,12 @@ define void @ktest_3(<8 x i32> %w, <8 x i32> %x, <8 x i32> %y, <8 x i32> %z) {
 ; X86-NEXT:    vptestnmd %ymm3, %ymm3, %k2
 ; X86-NEXT:    korb %k2, %k1, %k1
 ; X86-NEXT:    ktestb %k1, %k0
-; X86-NEXT:    je LBB72_1
+; X86-NEXT:    je LBB73_1
 ; X86-NEXT:  ## %bb.2: ## %exit
 ; X86-NEXT:    addl $12, %esp
 ; X86-NEXT:    vzeroupper
 ; X86-NEXT:    retl
-; X86-NEXT:  LBB72_1: ## %bar
+; X86-NEXT:  LBB73_1: ## %bar
 ; X86-NEXT:    vzeroupper
 ; X86-NEXT:    calll _foo
 ; X86-NEXT:    addl $12, %esp
@@ -3659,12 +4580,12 @@ define void @ktest_4(<8 x i64> %w, <8 x i64> %x, <8 x i64> %y, <8 x i64> %z) {
 ; KNL-NEXT:    kandw %k1, %k0, %k0
 ; KNL-NEXT:    kmovw %k0, %eax
 ; KNL-NEXT:    testb %al, %al
-; KNL-NEXT:    je LBB73_1
+; KNL-NEXT:    je LBB74_1
 ; KNL-NEXT:  ## %bb.2: ## %exit
 ; KNL-NEXT:    popq %rax
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    retq
-; KNL-NEXT:  LBB73_1: ## %bar
+; KNL-NEXT:  LBB74_1: ## %bar
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    callq _foo
 ; KNL-NEXT:    popq %rax
@@ -3681,12 +4602,12 @@ define void @ktest_4(<8 x i64> %w, <8 x i64> %x, <8 x i64> %y, <8 x i64> %z) {
 ; SKX-NEXT:    vptestnmq %zmm3, %zmm3, %k2
 ; SKX-NEXT:    korb %k2, %k1, %k1
 ; SKX-NEXT:    ktestb %k1, %k0
-; SKX-NEXT:    je LBB73_1
+; SKX-NEXT:    je LBB74_1
 ; SKX-NEXT:  ## %bb.2: ## %exit
 ; SKX-NEXT:    popq %rax
 ; SKX-NEXT:    vzeroupper
 ; SKX-NEXT:    retq
-; SKX-NEXT:  LBB73_1: ## %bar
+; SKX-NEXT:  LBB74_1: ## %bar
 ; SKX-NEXT:    vzeroupper
 ; SKX-NEXT:    callq _foo
 ; SKX-NEXT:    popq %rax
@@ -3705,12 +4626,12 @@ define void @ktest_4(<8 x i64> %w, <8 x i64> %x, <8 x i64> %y, <8 x i64> %z) {
 ; AVX512BW-NEXT:    kandw %k1, %k0, %k0
 ; AVX512BW-NEXT:    kmovd %k0, %eax
 ; AVX512BW-NEXT:    testb %al, %al
-; AVX512BW-NEXT:    je LBB73_1
+; AVX512BW-NEXT:    je LBB74_1
 ; AVX512BW-NEXT:  ## %bb.2: ## %exit
 ; AVX512BW-NEXT:    popq %rax
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
-; AVX512BW-NEXT:  LBB73_1: ## %bar
+; AVX512BW-NEXT:  LBB74_1: ## %bar
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    callq _foo
 ; AVX512BW-NEXT:    popq %rax
@@ -3727,12 +4648,12 @@ define void @ktest_4(<8 x i64> %w, <8 x i64> %x, <8 x i64> %y, <8 x i64> %z) {
 ; AVX512DQ-NEXT:    vptestnmq %zmm3, %zmm3, %k2
 ; AVX512DQ-NEXT:    korb %k2, %k1, %k1
 ; AVX512DQ-NEXT:    ktestb %k1, %k0
-; AVX512DQ-NEXT:    je LBB73_1
+; AVX512DQ-NEXT:    je LBB74_1
 ; AVX512DQ-NEXT:  ## %bb.2: ## %exit
 ; AVX512DQ-NEXT:    popq %rax
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
-; AVX512DQ-NEXT:  LBB73_1: ## %bar
+; AVX512DQ-NEXT:  LBB74_1: ## %bar
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    callq _foo
 ; AVX512DQ-NEXT:    popq %rax
@@ -3749,12 +4670,12 @@ define void @ktest_4(<8 x i64> %w, <8 x i64> %x, <8 x i64> %y, <8 x i64> %z) {
 ; X86-NEXT:    vptestnmq %zmm3, %zmm3, %k2
 ; X86-NEXT:    korb %k2, %k1, %k1
 ; X86-NEXT:    ktestb %k1, %k0
-; X86-NEXT:    je LBB73_1
+; X86-NEXT:    je LBB74_1
 ; X86-NEXT:  ## %bb.2: ## %exit
 ; X86-NEXT:    addl $12, %esp
 ; X86-NEXT:    vzeroupper
 ; X86-NEXT:    retl
-; X86-NEXT:  LBB73_1: ## %bar
+; X86-NEXT:  LBB74_1: ## %bar
 ; X86-NEXT:    vzeroupper
 ; X86-NEXT:    calll _foo
 ; X86-NEXT:    addl $12, %esp
@@ -3791,12 +4712,12 @@ define void @ktest_5(<16 x i32> %w, <16 x i32> %x, <16 x i32> %y, <16 x i32> %z)
 ; KNL-NEXT:    korw %k2, %k1, %k1
 ; KNL-NEXT:    kandw %k1, %k0, %k0
 ; KNL-NEXT:    kortestw %k0, %k0
-; KNL-NEXT:    je LBB74_1
+; KNL-NEXT:    je LBB75_1
 ; KNL-NEXT:  ## %bb.2: ## %exit
 ; KNL-NEXT:    popq %rax
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    retq
-; KNL-NEXT:  LBB74_1: ## %bar
+; KNL-NEXT:  LBB75_1: ## %bar
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    callq _foo
 ; KNL-NEXT:    popq %rax
@@ -3813,12 +4734,12 @@ define void @ktest_5(<16 x i32> %w, <16 x i32> %x, <16 x i32> %y, <16 x i32> %z)
 ; SKX-NEXT:    vptestnmd %zmm3, %zmm3, %k2
 ; SKX-NEXT:    korw %k2, %k1, %k1
 ; SKX-NEXT:    ktestw %k1, %k0
-; SKX-NEXT:    je LBB74_1
+; SKX-NEXT:    je LBB75_1
 ; SKX-NEXT:  ## %bb.2: ## %exit
 ; SKX-NEXT:    popq %rax
 ; SKX-NEXT:    vzeroupper
 ; SKX-NEXT:    retq
-; SKX-NEXT:  LBB74_1: ## %bar
+; SKX-NEXT:  LBB75_1: ## %bar
 ; SKX-NEXT:    vzeroupper
 ; SKX-NEXT:    callq _foo
 ; SKX-NEXT:    popq %rax
@@ -3836,12 +4757,12 @@ define void @ktest_5(<16 x i32> %w, <16 x i32> %x, <16 x i32> %y, <16 x i32> %z)
 ; AVX512BW-NEXT:    korw %k2, %k1, %k1
 ; AVX512BW-NEXT:    kandw %k1, %k0, %k0
 ; AVX512BW-NEXT:    kortestw %k0, %k0
-; AVX512BW-NEXT:    je LBB74_1
+; AVX512BW-NEXT:    je LBB75_1
 ; AVX512BW-NEXT:  ## %bb.2: ## %exit
 ; AVX512BW-NEXT:    popq %rax
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
-; AVX512BW-NEXT:  LBB74_1: ## %bar
+; AVX512BW-NEXT:  LBB75_1: ## %bar
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    callq _foo
 ; AVX512BW-NEXT:    popq %rax
@@ -3858,12 +4779,12 @@ define void @ktest_5(<16 x i32> %w, <16 x i32> %x, <16 x i32> %y, <16 x i32> %z)
 ; AVX512DQ-NEXT:    vptestnmd %zmm3, %zmm3, %k2
 ; AVX512DQ-NEXT:    korw %k2, %k1, %k1
 ; AVX512DQ-NEXT:    ktestw %k1, %k0
-; AVX512DQ-NEXT:    je LBB74_1
+; AVX512DQ-NEXT:    je LBB75_1
 ; AVX512DQ-NEXT:  ## %bb.2: ## %exit
 ; AVX512DQ-NEXT:    popq %rax
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
-; AVX512DQ-NEXT:  LBB74_1: ## %bar
+; AVX512DQ-NEXT:  LBB75_1: ## %bar
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    callq _foo
 ; AVX512DQ-NEXT:    popq %rax
@@ -3880,12 +4801,12 @@ define void @ktest_5(<16 x i32> %w, <16 x i32> %x, <16 x i32> %y, <16 x i32> %z)
 ; X86-NEXT:    vptestnmd %zmm3, %zmm3, %k2
 ; X86-NEXT:    korw %k2, %k1, %k1
 ; X86-NEXT:    ktestw %k1, %k0
-; X86-NEXT:    je LBB74_1
+; X86-NEXT:    je LBB75_1
 ; X86-NEXT:  ## %bb.2: ## %exit
 ; X86-NEXT:    addl $12, %esp
 ; X86-NEXT:    vzeroupper
 ; X86-NEXT:    retl
-; X86-NEXT:  LBB74_1: ## %bar
+; X86-NEXT:  LBB75_1: ## %bar
 ; X86-NEXT:    vzeroupper
 ; X86-NEXT:    calll _foo
 ; X86-NEXT:    addl $12, %esp
@@ -3914,20 +4835,24 @@ define void @ktest_6(<32 x i16> %w, <32 x i16> %x, <32 x i16> %y, <32 x i16> %z)
 ; KNL:       ## %bb.0:
 ; KNL-NEXT:    pushq %rax
 ; KNL-NEXT:    .cfi_def_cfa_offset 16
+; KNL-NEXT:    vextracti64x4 $1, %zmm3, %ymm4
+; KNL-NEXT:    vextracti64x4 $1, %zmm2, %ymm5
+; KNL-NEXT:    vextracti64x4 $1, %zmm1, %ymm6
+; KNL-NEXT:    vextracti64x4 $1, %zmm0, %ymm7
 ; KNL-NEXT:    vpxor %xmm8, %xmm8, %xmm8
-; KNL-NEXT:    vpcmpeqw %ymm8, %ymm0, %ymm0
-; KNL-NEXT:    vpcmpeqw %ymm8, %ymm1, %ymm1
-; KNL-NEXT:    vpcmpeqw %ymm8, %ymm2, %ymm2
-; KNL-NEXT:    vpor %ymm2, %ymm0, %ymm0
-; KNL-NEXT:    vpcmpeqw %ymm8, %ymm3, %ymm2
-; KNL-NEXT:    vpor %ymm2, %ymm1, %ymm1
-; KNL-NEXT:    vpcmpeqw %ymm8, %ymm4, %ymm2
-; KNL-NEXT:    vpcmpeqw %ymm8, %ymm5, %ymm3
-; KNL-NEXT:    vpcmpeqw %ymm8, %ymm6, %ymm4
-; KNL-NEXT:    vpor %ymm4, %ymm2, %ymm2
+; KNL-NEXT:    vpcmpeqw %ymm0, %ymm8, %ymm0
+; KNL-NEXT:    vpcmpeqw %ymm7, %ymm8, %ymm7
+; KNL-NEXT:    vpcmpeqw %ymm1, %ymm8, %ymm1
+; KNL-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; KNL-NEXT:    vpcmpeqw %ymm6, %ymm8, %ymm1
+; KNL-NEXT:    vpor %ymm1, %ymm7, %ymm1
+; KNL-NEXT:    vpcmpeqw %ymm2, %ymm8, %ymm2
+; KNL-NEXT:    vpcmpeqw %ymm5, %ymm8, %ymm5
+; KNL-NEXT:    vpcmpeqw %ymm3, %ymm8, %ymm3
+; KNL-NEXT:    vpor %ymm3, %ymm2, %ymm2
 ; KNL-NEXT:    vpand %ymm2, %ymm0, %ymm0
-; KNL-NEXT:    vpcmpeqw %ymm8, %ymm7, %ymm2
-; KNL-NEXT:    vpor %ymm2, %ymm3, %ymm2
+; KNL-NEXT:    vpcmpeqw %ymm4, %ymm8, %ymm2
+; KNL-NEXT:    vpor %ymm2, %ymm5, %ymm2
 ; KNL-NEXT:    vpand %ymm2, %ymm1, %ymm1
 ; KNL-NEXT:    vpmovsxwd %ymm0, %zmm0
 ; KNL-NEXT:    vptestmd %zmm0, %zmm0, %k0
@@ -3937,12 +4862,12 @@ define void @ktest_6(<32 x i16> %w, <32 x i16> %x, <32 x i16> %y, <32 x i16> %z)
 ; KNL-NEXT:    kmovw %k0, %ecx
 ; KNL-NEXT:    shll $16, %ecx
 ; KNL-NEXT:    orl %eax, %ecx
-; KNL-NEXT:    je LBB75_1
+; KNL-NEXT:    je LBB76_1
 ; KNL-NEXT:  ## %bb.2: ## %exit
 ; KNL-NEXT:    popq %rax
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    retq
-; KNL-NEXT:  LBB75_1: ## %bar
+; KNL-NEXT:  LBB76_1: ## %bar
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    callq _foo
 ; KNL-NEXT:    popq %rax
@@ -3959,12 +4884,12 @@ define void @ktest_6(<32 x i16> %w, <32 x i16> %x, <32 x i16> %y, <32 x i16> %z)
 ; SKX-NEXT:    vptestnmw %zmm3, %zmm3, %k2
 ; SKX-NEXT:    kord %k2, %k1, %k1
 ; SKX-NEXT:    ktestd %k1, %k0
-; SKX-NEXT:    je LBB75_1
+; SKX-NEXT:    je LBB76_1
 ; SKX-NEXT:  ## %bb.2: ## %exit
 ; SKX-NEXT:    popq %rax
 ; SKX-NEXT:    vzeroupper
 ; SKX-NEXT:    retq
-; SKX-NEXT:  LBB75_1: ## %bar
+; SKX-NEXT:  LBB76_1: ## %bar
 ; SKX-NEXT:    vzeroupper
 ; SKX-NEXT:    callq _foo
 ; SKX-NEXT:    popq %rax
@@ -3981,12 +4906,12 @@ define void @ktest_6(<32 x i16> %w, <32 x i16> %x, <32 x i16> %y, <32 x i16> %z)
 ; AVX512BW-NEXT:    vptestnmw %zmm3, %zmm3, %k2
 ; AVX512BW-NEXT:    kord %k2, %k1, %k1
 ; AVX512BW-NEXT:    ktestd %k1, %k0
-; AVX512BW-NEXT:    je LBB75_1
+; AVX512BW-NEXT:    je LBB76_1
 ; AVX512BW-NEXT:  ## %bb.2: ## %exit
 ; AVX512BW-NEXT:    popq %rax
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
-; AVX512BW-NEXT:  LBB75_1: ## %bar
+; AVX512BW-NEXT:  LBB76_1: ## %bar
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    callq _foo
 ; AVX512BW-NEXT:    popq %rax
@@ -3996,20 +4921,24 @@ define void @ktest_6(<32 x i16> %w, <32 x i16> %x, <32 x i16> %y, <32 x i16> %z)
 ; AVX512DQ:       ## %bb.0:
 ; AVX512DQ-NEXT:    pushq %rax
 ; AVX512DQ-NEXT:    .cfi_def_cfa_offset 16
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm3, %ymm4
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm2, %ymm5
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm1, %ymm6
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm7
 ; AVX512DQ-NEXT:    vpxor %xmm8, %xmm8, %xmm8
-; AVX512DQ-NEXT:    vpcmpeqw %ymm8, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpcmpeqw %ymm8, %ymm1, %ymm1
-; AVX512DQ-NEXT:    vpcmpeqw %ymm8, %ymm2, %ymm2
-; AVX512DQ-NEXT:    vpor %ymm2, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpcmpeqw %ymm8, %ymm3, %ymm2
-; AVX512DQ-NEXT:    vpor %ymm2, %ymm1, %ymm1
-; AVX512DQ-NEXT:    vpcmpeqw %ymm8, %ymm4, %ymm2
-; AVX512DQ-NEXT:    vpcmpeqw %ymm8, %ymm5, %ymm3
-; AVX512DQ-NEXT:    vpcmpeqw %ymm8, %ymm6, %ymm4
-; AVX512DQ-NEXT:    vpor %ymm4, %ymm2, %ymm2
+; AVX512DQ-NEXT:    vpcmpeqw %ymm0, %ymm8, %ymm0
+; AVX512DQ-NEXT:    vpcmpeqw %ymm7, %ymm8, %ymm7
+; AVX512DQ-NEXT:    vpcmpeqw %ymm1, %ymm8, %ymm1
+; AVX512DQ-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpcmpeqw %ymm6, %ymm8, %ymm1
+; AVX512DQ-NEXT:    vpor %ymm1, %ymm7, %ymm1
+; AVX512DQ-NEXT:    vpcmpeqw %ymm2, %ymm8, %ymm2
+; AVX512DQ-NEXT:    vpcmpeqw %ymm5, %ymm8, %ymm5
+; AVX512DQ-NEXT:    vpcmpeqw %ymm3, %ymm8, %ymm3
+; AVX512DQ-NEXT:    vpor %ymm3, %ymm2, %ymm2
 ; AVX512DQ-NEXT:    vpand %ymm2, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpcmpeqw %ymm8, %ymm7, %ymm2
-; AVX512DQ-NEXT:    vpor %ymm2, %ymm3, %ymm2
+; AVX512DQ-NEXT:    vpcmpeqw %ymm4, %ymm8, %ymm2
+; AVX512DQ-NEXT:    vpor %ymm2, %ymm5, %ymm2
 ; AVX512DQ-NEXT:    vpand %ymm2, %ymm1, %ymm1
 ; AVX512DQ-NEXT:    vpmovsxwd %ymm0, %zmm0
 ; AVX512DQ-NEXT:    vpmovd2m %zmm0, %k0
@@ -4019,12 +4948,12 @@ define void @ktest_6(<32 x i16> %w, <32 x i16> %x, <32 x i16> %y, <32 x i16> %z)
 ; AVX512DQ-NEXT:    kmovw %k0, %ecx
 ; AVX512DQ-NEXT:    shll $16, %ecx
 ; AVX512DQ-NEXT:    orl %eax, %ecx
-; AVX512DQ-NEXT:    je LBB75_1
+; AVX512DQ-NEXT:    je LBB76_1
 ; AVX512DQ-NEXT:  ## %bb.2: ## %exit
 ; AVX512DQ-NEXT:    popq %rax
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
-; AVX512DQ-NEXT:  LBB75_1: ## %bar
+; AVX512DQ-NEXT:  LBB76_1: ## %bar
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    callq _foo
 ; AVX512DQ-NEXT:    popq %rax
@@ -4041,12 +4970,12 @@ define void @ktest_6(<32 x i16> %w, <32 x i16> %x, <32 x i16> %y, <32 x i16> %z)
 ; X86-NEXT:    vptestnmw %zmm3, %zmm3, %k2
 ; X86-NEXT:    kord %k2, %k1, %k1
 ; X86-NEXT:    ktestd %k1, %k0
-; X86-NEXT:    je LBB75_1
+; X86-NEXT:    je LBB76_1
 ; X86-NEXT:  ## %bb.2: ## %exit
 ; X86-NEXT:    addl $12, %esp
 ; X86-NEXT:    vzeroupper
 ; X86-NEXT:    retl
-; X86-NEXT:  LBB75_1: ## %bar
+; X86-NEXT:  LBB76_1: ## %bar
 ; X86-NEXT:    vzeroupper
 ; X86-NEXT:    calll _foo
 ; X86-NEXT:    addl $12, %esp
@@ -4075,59 +5004,63 @@ define void @ktest_7(<64 x i8> %w, <64 x i8> %x, <64 x i8> %y, <64 x i8> %z) {
 ; KNL:       ## %bb.0:
 ; KNL-NEXT:    pushq %rax
 ; KNL-NEXT:    .cfi_def_cfa_offset 16
+; KNL-NEXT:    vextracti64x4 $1, %zmm3, %ymm9
+; KNL-NEXT:    vextracti64x4 $1, %zmm2, %ymm10
+; KNL-NEXT:    vextracti64x4 $1, %zmm1, %ymm11
+; KNL-NEXT:    vextracti64x4 $1, %zmm0, %ymm7
 ; KNL-NEXT:    vpxor %xmm8, %xmm8, %xmm8
-; KNL-NEXT:    vpcmpeqb %ymm8, %ymm0, %ymm9
-; KNL-NEXT:    vextracti128 $1, %ymm9, %xmm0
-; KNL-NEXT:    vpcmpeqb %ymm8, %ymm1, %ymm10
-; KNL-NEXT:    vextracti128 $1, %ymm10, %xmm1
-; KNL-NEXT:    vpcmpeqb %ymm8, %ymm2, %ymm11
-; KNL-NEXT:    vextracti128 $1, %ymm11, %xmm2
-; KNL-NEXT:    vpor %xmm2, %xmm0, %xmm13
-; KNL-NEXT:    vpcmpeqb %ymm8, %ymm3, %ymm2
-; KNL-NEXT:    vextracti128 $1, %ymm2, %xmm3
-; KNL-NEXT:    vpor %xmm3, %xmm1, %xmm12
-; KNL-NEXT:    vpcmpeqb %ymm8, %ymm4, %ymm3
-; KNL-NEXT:    vextracti128 $1, %ymm3, %xmm4
-; KNL-NEXT:    vpcmpeqb %ymm8, %ymm5, %ymm5
-; KNL-NEXT:    vextracti128 $1, %ymm5, %xmm1
-; KNL-NEXT:    vpcmpeqb %ymm8, %ymm6, %ymm6
-; KNL-NEXT:    vextracti128 $1, %ymm6, %xmm0
+; KNL-NEXT:    vpcmpeqb %ymm0, %ymm8, %ymm13
+; KNL-NEXT:    vextracti128 $1, %ymm13, %xmm4
+; KNL-NEXT:    vpcmpeqb %ymm7, %ymm8, %ymm7
+; KNL-NEXT:    vextracti128 $1, %ymm7, %xmm5
+; KNL-NEXT:    vpcmpeqb %ymm1, %ymm8, %ymm1
+; KNL-NEXT:    vextracti128 $1, %ymm1, %xmm6
+; KNL-NEXT:    vpor %xmm6, %xmm4, %xmm12
+; KNL-NEXT:    vpcmpeqb %ymm8, %ymm11, %ymm6
+; KNL-NEXT:    vextracti128 $1, %ymm6, %xmm4
+; KNL-NEXT:    vpor %xmm4, %xmm5, %xmm11
+; KNL-NEXT:    vpcmpeqb %ymm2, %ymm8, %ymm2
+; KNL-NEXT:    vextracti128 $1, %ymm2, %xmm5
+; KNL-NEXT:    vpcmpeqb %ymm8, %ymm10, %ymm10
+; KNL-NEXT:    vextracti128 $1, %ymm10, %xmm4
+; KNL-NEXT:    vpcmpeqb %ymm3, %ymm8, %ymm3
+; KNL-NEXT:    vextracti128 $1, %ymm3, %xmm0
+; KNL-NEXT:    vpor %xmm0, %xmm5, %xmm0
+; KNL-NEXT:    vpand %xmm0, %xmm12, %xmm12
+; KNL-NEXT:    vpcmpeqb %ymm8, %ymm9, %ymm5
+; KNL-NEXT:    vextracti128 $1, %ymm5, %xmm0
 ; KNL-NEXT:    vpor %xmm0, %xmm4, %xmm0
-; KNL-NEXT:    vpand %xmm0, %xmm13, %xmm0
-; KNL-NEXT:    vpcmpeqb %ymm8, %ymm7, %ymm4
-; KNL-NEXT:    vextracti128 $1, %ymm4, %xmm7
-; KNL-NEXT:    vpor %xmm7, %xmm1, %xmm1
-; KNL-NEXT:    vpand %xmm1, %xmm12, %xmm1
-; KNL-NEXT:    vpor %xmm2, %xmm10, %xmm2
-; KNL-NEXT:    vpor %xmm11, %xmm9, %xmm7
-; KNL-NEXT:    vpor %xmm4, %xmm5, %xmm4
-; KNL-NEXT:    vpand %xmm4, %xmm2, %xmm2
-; KNL-NEXT:    vpor %xmm6, %xmm3, %xmm3
-; KNL-NEXT:    vpand %xmm3, %xmm7, %xmm3
-; KNL-NEXT:    vpmovsxbd %xmm3, %zmm3
-; KNL-NEXT:    vptestmd %zmm3, %zmm3, %k0
+; KNL-NEXT:    vpand %xmm0, %xmm11, %xmm0
+; KNL-NEXT:    vpor %xmm6, %xmm7, %xmm4
+; KNL-NEXT:    vpor %xmm1, %xmm13, %xmm1
+; KNL-NEXT:    vpor %xmm5, %xmm10, %xmm5
+; KNL-NEXT:    vpand %xmm5, %xmm4, %xmm4
+; KNL-NEXT:    vpor %xmm3, %xmm2, %xmm2
+; KNL-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; KNL-NEXT:    vpmovsxbd %xmm1, %zmm1
+; KNL-NEXT:    vptestmd %zmm1, %zmm1, %k0
 ; KNL-NEXT:    kmovw %k0, %eax
-; KNL-NEXT:    vpmovsxbd %xmm0, %zmm0
-; KNL-NEXT:    vptestmd %zmm0, %zmm0, %k0
+; KNL-NEXT:    vpmovsxbd %xmm12, %zmm1
+; KNL-NEXT:    vptestmd %zmm1, %zmm1, %k0
 ; KNL-NEXT:    kmovw %k0, %ecx
 ; KNL-NEXT:    shll $16, %ecx
 ; KNL-NEXT:    orl %eax, %ecx
-; KNL-NEXT:    vpmovsxbd %xmm2, %zmm0
-; KNL-NEXT:    vptestmd %zmm0, %zmm0, %k0
+; KNL-NEXT:    vpmovsxbd %xmm4, %zmm1
+; KNL-NEXT:    vptestmd %zmm1, %zmm1, %k0
 ; KNL-NEXT:    kmovw %k0, %eax
-; KNL-NEXT:    vpmovsxbd %xmm1, %zmm0
+; KNL-NEXT:    vpmovsxbd %xmm0, %zmm0
 ; KNL-NEXT:    vptestmd %zmm0, %zmm0, %k0
 ; KNL-NEXT:    kmovw %k0, %edx
 ; KNL-NEXT:    shll $16, %edx
 ; KNL-NEXT:    orl %eax, %edx
 ; KNL-NEXT:    shlq $32, %rdx
 ; KNL-NEXT:    orq %rcx, %rdx
-; KNL-NEXT:    je LBB76_1
+; KNL-NEXT:    je LBB77_1
 ; KNL-NEXT:  ## %bb.2: ## %exit
 ; KNL-NEXT:    popq %rax
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    retq
-; KNL-NEXT:  LBB76_1: ## %bar
+; KNL-NEXT:  LBB77_1: ## %bar
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    callq _foo
 ; KNL-NEXT:    popq %rax
@@ -4144,12 +5077,12 @@ define void @ktest_7(<64 x i8> %w, <64 x i8> %x, <64 x i8> %y, <64 x i8> %z) {
 ; SKX-NEXT:    vptestnmb %zmm3, %zmm3, %k2
 ; SKX-NEXT:    korq %k2, %k1, %k1
 ; SKX-NEXT:    ktestq %k1, %k0
-; SKX-NEXT:    je LBB76_1
+; SKX-NEXT:    je LBB77_1
 ; SKX-NEXT:  ## %bb.2: ## %exit
 ; SKX-NEXT:    popq %rax
 ; SKX-NEXT:    vzeroupper
 ; SKX-NEXT:    retq
-; SKX-NEXT:  LBB76_1: ## %bar
+; SKX-NEXT:  LBB77_1: ## %bar
 ; SKX-NEXT:    vzeroupper
 ; SKX-NEXT:    callq _foo
 ; SKX-NEXT:    popq %rax
@@ -4166,12 +5099,12 @@ define void @ktest_7(<64 x i8> %w, <64 x i8> %x, <64 x i8> %y, <64 x i8> %z) {
 ; AVX512BW-NEXT:    vptestnmb %zmm3, %zmm3, %k2
 ; AVX512BW-NEXT:    korq %k2, %k1, %k1
 ; AVX512BW-NEXT:    ktestq %k1, %k0
-; AVX512BW-NEXT:    je LBB76_1
+; AVX512BW-NEXT:    je LBB77_1
 ; AVX512BW-NEXT:  ## %bb.2: ## %exit
 ; AVX512BW-NEXT:    popq %rax
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
-; AVX512BW-NEXT:  LBB76_1: ## %bar
+; AVX512BW-NEXT:  LBB77_1: ## %bar
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    callq _foo
 ; AVX512BW-NEXT:    popq %rax
@@ -4181,59 +5114,63 @@ define void @ktest_7(<64 x i8> %w, <64 x i8> %x, <64 x i8> %y, <64 x i8> %z) {
 ; AVX512DQ:       ## %bb.0:
 ; AVX512DQ-NEXT:    pushq %rax
 ; AVX512DQ-NEXT:    .cfi_def_cfa_offset 16
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm3, %ymm9
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm2, %ymm10
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm1, %ymm11
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm7
 ; AVX512DQ-NEXT:    vpxor %xmm8, %xmm8, %xmm8
-; AVX512DQ-NEXT:    vpcmpeqb %ymm8, %ymm0, %ymm9
-; AVX512DQ-NEXT:    vextracti128 $1, %ymm9, %xmm0
-; AVX512DQ-NEXT:    vpcmpeqb %ymm8, %ymm1, %ymm10
-; AVX512DQ-NEXT:    vextracti128 $1, %ymm10, %xmm1
-; AVX512DQ-NEXT:    vpcmpeqb %ymm8, %ymm2, %ymm11
-; AVX512DQ-NEXT:    vextracti128 $1, %ymm11, %xmm2
-; AVX512DQ-NEXT:    vpor %xmm2, %xmm0, %xmm13
-; AVX512DQ-NEXT:    vpcmpeqb %ymm8, %ymm3, %ymm2
-; AVX512DQ-NEXT:    vextracti128 $1, %ymm2, %xmm3
-; AVX512DQ-NEXT:    vpor %xmm3, %xmm1, %xmm12
-; AVX512DQ-NEXT:    vpcmpeqb %ymm8, %ymm4, %ymm3
-; AVX512DQ-NEXT:    vextracti128 $1, %ymm3, %xmm4
-; AVX512DQ-NEXT:    vpcmpeqb %ymm8, %ymm5, %ymm5
-; AVX512DQ-NEXT:    vextracti128 $1, %ymm5, %xmm1
-; AVX512DQ-NEXT:    vpcmpeqb %ymm8, %ymm6, %ymm6
-; AVX512DQ-NEXT:    vextracti128 $1, %ymm6, %xmm0
+; AVX512DQ-NEXT:    vpcmpeqb %ymm0, %ymm8, %ymm13
+; AVX512DQ-NEXT:    vextracti128 $1, %ymm13, %xmm4
+; AVX512DQ-NEXT:    vpcmpeqb %ymm7, %ymm8, %ymm7
+; AVX512DQ-NEXT:    vextracti128 $1, %ymm7, %xmm5
+; AVX512DQ-NEXT:    vpcmpeqb %ymm1, %ymm8, %ymm1
+; AVX512DQ-NEXT:    vextracti128 $1, %ymm1, %xmm6
+; AVX512DQ-NEXT:    vpor %xmm6, %xmm4, %xmm12
+; AVX512DQ-NEXT:    vpcmpeqb %ymm8, %ymm11, %ymm6
+; AVX512DQ-NEXT:    vextracti128 $1, %ymm6, %xmm4
+; AVX512DQ-NEXT:    vpor %xmm4, %xmm5, %xmm11
+; AVX512DQ-NEXT:    vpcmpeqb %ymm2, %ymm8, %ymm2
+; AVX512DQ-NEXT:    vextracti128 $1, %ymm2, %xmm5
+; AVX512DQ-NEXT:    vpcmpeqb %ymm8, %ymm10, %ymm10
+; AVX512DQ-NEXT:    vextracti128 $1, %ymm10, %xmm4
+; AVX512DQ-NEXT:    vpcmpeqb %ymm3, %ymm8, %ymm3
+; AVX512DQ-NEXT:    vextracti128 $1, %ymm3, %xmm0
+; AVX512DQ-NEXT:    vpor %xmm0, %xmm5, %xmm0
+; AVX512DQ-NEXT:    vpand %xmm0, %xmm12, %xmm12
+; AVX512DQ-NEXT:    vpcmpeqb %ymm8, %ymm9, %ymm5
+; AVX512DQ-NEXT:    vextracti128 $1, %ymm5, %xmm0
 ; AVX512DQ-NEXT:    vpor %xmm0, %xmm4, %xmm0
-; AVX512DQ-NEXT:    vpand %xmm0, %xmm13, %xmm0
-; AVX512DQ-NEXT:    vpcmpeqb %ymm8, %ymm7, %ymm4
-; AVX512DQ-NEXT:    vextracti128 $1, %ymm4, %xmm7
-; AVX512DQ-NEXT:    vpor %xmm7, %xmm1, %xmm1
-; AVX512DQ-NEXT:    vpand %xmm1, %xmm12, %xmm1
-; AVX512DQ-NEXT:    vpor %xmm2, %xmm10, %xmm2
-; AVX512DQ-NEXT:    vpor %xmm11, %xmm9, %xmm7
-; AVX512DQ-NEXT:    vpor %xmm4, %xmm5, %xmm4
-; AVX512DQ-NEXT:    vpand %xmm4, %xmm2, %xmm2
-; AVX512DQ-NEXT:    vpor %xmm6, %xmm3, %xmm3
-; AVX512DQ-NEXT:    vpand %xmm3, %xmm7, %xmm3
-; AVX512DQ-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512DQ-NEXT:    vpmovd2m %zmm3, %k0
+; AVX512DQ-NEXT:    vpand %xmm0, %xmm11, %xmm0
+; AVX512DQ-NEXT:    vpor %xmm6, %xmm7, %xmm4
+; AVX512DQ-NEXT:    vpor %xmm1, %xmm13, %xmm1
+; AVX512DQ-NEXT:    vpor %xmm5, %xmm10, %xmm5
+; AVX512DQ-NEXT:    vpand %xmm5, %xmm4, %xmm4
+; AVX512DQ-NEXT:    vpor %xmm3, %xmm2, %xmm2
+; AVX512DQ-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX512DQ-NEXT:    vpmovsxbd %xmm1, %zmm1
+; AVX512DQ-NEXT:    vpmovd2m %zmm1, %k0
 ; AVX512DQ-NEXT:    kmovw %k0, %eax
-; AVX512DQ-NEXT:    vpmovsxbd %xmm0, %zmm0
-; AVX512DQ-NEXT:    vpmovd2m %zmm0, %k0
+; AVX512DQ-NEXT:    vpmovsxbd %xmm12, %zmm1
+; AVX512DQ-NEXT:    vpmovd2m %zmm1, %k0
 ; AVX512DQ-NEXT:    kmovw %k0, %ecx
 ; AVX512DQ-NEXT:    shll $16, %ecx
 ; AVX512DQ-NEXT:    orl %eax, %ecx
-; AVX512DQ-NEXT:    vpmovsxbd %xmm2, %zmm0
-; AVX512DQ-NEXT:    vpmovd2m %zmm0, %k0
+; AVX512DQ-NEXT:    vpmovsxbd %xmm4, %zmm1
+; AVX512DQ-NEXT:    vpmovd2m %zmm1, %k0
 ; AVX512DQ-NEXT:    kmovw %k0, %eax
-; AVX512DQ-NEXT:    vpmovsxbd %xmm1, %zmm0
+; AVX512DQ-NEXT:    vpmovsxbd %xmm0, %zmm0
 ; AVX512DQ-NEXT:    vpmovd2m %zmm0, %k0
 ; AVX512DQ-NEXT:    kmovw %k0, %edx
 ; AVX512DQ-NEXT:    shll $16, %edx
 ; AVX512DQ-NEXT:    orl %eax, %edx
 ; AVX512DQ-NEXT:    shlq $32, %rdx
 ; AVX512DQ-NEXT:    orq %rcx, %rdx
-; AVX512DQ-NEXT:    je LBB76_1
+; AVX512DQ-NEXT:    je LBB77_1
 ; AVX512DQ-NEXT:  ## %bb.2: ## %exit
 ; AVX512DQ-NEXT:    popq %rax
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
-; AVX512DQ-NEXT:  LBB76_1: ## %bar
+; AVX512DQ-NEXT:  LBB77_1: ## %bar
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    callq _foo
 ; AVX512DQ-NEXT:    popq %rax
@@ -4252,12 +5189,12 @@ define void @ktest_7(<64 x i8> %w, <64 x i8> %x, <64 x i8> %y, <64 x i8> %z) {
 ; X86-NEXT:    kandq %k1, %k0, %k0
 ; X86-NEXT:    kshiftrq $32, %k0, %k1
 ; X86-NEXT:    kortestd %k1, %k0
-; X86-NEXT:    je LBB76_1
+; X86-NEXT:    je LBB77_1
 ; X86-NEXT:  ## %bb.2: ## %exit
 ; X86-NEXT:    addl $12, %esp
 ; X86-NEXT:    vzeroupper
 ; X86-NEXT:    retl
-; X86-NEXT:  LBB76_1: ## %bar
+; X86-NEXT:  LBB77_1: ## %bar
 ; X86-NEXT:    vzeroupper
 ; X86-NEXT:    calll _foo
 ; X86-NEXT:    addl $12, %esp
@@ -4280,3 +5217,97 @@ bar:
 exit:
   ret void
 }
+
+define <64 x i1> @mask64_insert(i32 %a) {
+; KNL-LABEL: mask64_insert:
+; KNL:       ## %bb.0:
+; KNL-NEXT:    movq %rdi, %rax
+; KNL-NEXT:    movw $-4, %cx
+; KNL-NEXT:    kmovw %ecx, %k0
+; KNL-NEXT:    kshiftrw $1, %k0, %k0
+; KNL-NEXT:    kshiftlw $1, %k0, %k0
+; KNL-NEXT:    andl $1, %esi
+; KNL-NEXT:    kmovw %esi, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kmovw %k0, (%rdi)
+; KNL-NEXT:    movw $-3, 6(%rdi)
+; KNL-NEXT:    movl $-131075, 2(%rdi) ## imm = 0xFFFDFFFD
+; KNL-NEXT:    retq
+;
+; SKX-LABEL: mask64_insert:
+; SKX:       ## %bb.0:
+; SKX-NEXT:    kmovd %edi, %k0
+; SKX-NEXT:    kshiftlq $63, %k0, %k0
+; SKX-NEXT:    kshiftrq $63, %k0, %k0
+; SKX-NEXT:    movabsq $-562958543486980, %rax ## imm = 0xFFFDFFFDFFFDFFFC
+; SKX-NEXT:    kmovq %rax, %k1
+; SKX-NEXT:    kshiftrq $1, %k1, %k1
+; SKX-NEXT:    kshiftlq $1, %k1, %k1
+; SKX-NEXT:    korq %k0, %k1, %k0
+; SKX-NEXT:    vpmovm2b %k0, %zmm0
+; SKX-NEXT:    retq
+;
+; AVX512BW-LABEL: mask64_insert:
+; AVX512BW:       ## %bb.0:
+; AVX512BW-NEXT:    kmovd %edi, %k0
+; AVX512BW-NEXT:    kshiftlq $63, %k0, %k0
+; AVX512BW-NEXT:    kshiftrq $63, %k0, %k0
+; AVX512BW-NEXT:    movabsq $-562958543486980, %rax ## imm = 0xFFFDFFFDFFFDFFFC
+; AVX512BW-NEXT:    kmovq %rax, %k1
+; AVX512BW-NEXT:    kshiftrq $1, %k1, %k1
+; AVX512BW-NEXT:    kshiftlq $1, %k1, %k1
+; AVX512BW-NEXT:    korq %k0, %k1, %k0
+; AVX512BW-NEXT:    vpmovm2b %k0, %zmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512DQ-LABEL: mask64_insert:
+; AVX512DQ:       ## %bb.0:
+; AVX512DQ-NEXT:    movq %rdi, %rax
+; AVX512DQ-NEXT:    movw $-4, %cx
+; AVX512DQ-NEXT:    kmovw %ecx, %k0
+; AVX512DQ-NEXT:    kshiftrw $1, %k0, %k0
+; AVX512DQ-NEXT:    kshiftlw $1, %k0, %k0
+; AVX512DQ-NEXT:    andl $1, %esi
+; AVX512DQ-NEXT:    kmovw %esi, %k1
+; AVX512DQ-NEXT:    korw %k1, %k0, %k0
+; AVX512DQ-NEXT:    kmovw %k0, (%rdi)
+; AVX512DQ-NEXT:    movw $-3, 6(%rdi)
+; AVX512DQ-NEXT:    movl $-131075, 2(%rdi) ## imm = 0xFFFDFFFD
+; AVX512DQ-NEXT:    retq
+;
+; X86-LABEL: mask64_insert:
+; X86:       ## %bb.0:
+; X86-NEXT:    kmovb {{[0-9]+}}(%esp), %k0
+; X86-NEXT:    movl $-131076, %eax ## imm = 0xFFFDFFFC
+; X86-NEXT:    kmovd %eax, %k1
+; X86-NEXT:    movl $-131075, %eax ## imm = 0xFFFDFFFD
+; X86-NEXT:    kmovd %eax, %k2
+; X86-NEXT:    kunpckdq %k1, %k2, %k1
+; X86-NEXT:    kshiftrq $1, %k1, %k1
+; X86-NEXT:    kshiftlq $1, %k1, %k1
+; X86-NEXT:    kshiftlq $63, %k0, %k0
+; X86-NEXT:    kshiftrq $63, %k0, %k0
+; X86-NEXT:    korq %k0, %k1, %k0
+; X86-NEXT:    vpmovm2b %k0, %zmm0
+; X86-NEXT:    retl
+  %a_i = trunc i32 %a to i1
+  %maskv = insertelement <64 x i1> <i1 true, i1 false, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, i1 %a_i, i32 0
+  ret <64 x i1> %maskv
+}
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 1, !"ProfileSummary", !1}
+!1 = !{!2, !3, !4, !5, !6, !7, !8, !9}
+!2 = !{!"ProfileFormat", !"InstrProf"}
+!3 = !{!"TotalCount", i64 10000}
+!4 = !{!"MaxCount", i64 10}
+!5 = !{!"MaxInternalCount", i64 1}
+!6 = !{!"MaxFunctionCount", i64 1000}
+!7 = !{!"NumCounts", i64 3}
+!8 = !{!"NumFunctions", i64 3}
+!9 = !{!"DetailedSummary", !10}
+!10 = !{!11, !12, !13}
+!11 = !{i32 10000, i64 100, i32 1}
+!12 = !{i32 999000, i64 100, i32 1}
+!13 = !{i32 999999, i64 1, i32 2}
+!14 = !{!"function_entry_count", i64 0}
diff --git a/llvm/test/CodeGen/X86/avx512-mask-spills.ll b/llvm/test/CodeGen/X86/avx512-mask-spills.ll
index b9f483e99..44643f42e 100644
--- a/llvm/test/CodeGen/X86/avx512-mask-spills.ll
+++ b/llvm/test/CodeGen/X86/avx512-mask-spills.ll
@@ -10,9 +10,9 @@ define <4 x i1> @test_4i1(<4 x i32> %a, <4 x i32> %b) {
 ; CHECK-NEXT:    vpcmpnleud %xmm1, %xmm0, %k0
 ; CHECK-NEXT:    vpcmpgtd %xmm1, %xmm0, %k1
 ; CHECK-NEXT:    korw %k1, %k0, %k0
-; CHECK-NEXT:    kmovw %k0, {{[0-9]+}}(%rsp) ## 2-byte Spill
+; CHECK-NEXT:    kmovw %k0, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
 ; CHECK-NEXT:    callq _f
-; CHECK-NEXT:    kmovw {{[0-9]+}}(%rsp), %k0 ## 2-byte Reload
+; CHECK-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 ## 2-byte Reload
 ; CHECK-NEXT:    vpmovm2d %k0, %xmm0
 ; CHECK-NEXT:    popq %rax
 ; CHECK-NEXT:    retq
@@ -32,10 +32,10 @@ define <8 x i1> @test_8i1(<8 x i32> %a, <8 x i32> %b) {
 ; CHECK-NEXT:    vpcmpnleud %ymm1, %ymm0, %k0
 ; CHECK-NEXT:    vpcmpgtd %ymm1, %ymm0, %k1
 ; CHECK-NEXT:    korb %k1, %k0, %k0
-; CHECK-NEXT:    kmovw %k0, {{[0-9]+}}(%rsp) ## 2-byte Spill
+; CHECK-NEXT:    kmovw %k0, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    callq _f
-; CHECK-NEXT:    kmovw {{[0-9]+}}(%rsp), %k0 ## 2-byte Reload
+; CHECK-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 ## 2-byte Reload
 ; CHECK-NEXT:    vpmovm2w %k0, %xmm0
 ; CHECK-NEXT:    popq %rax
 ; CHECK-NEXT:    retq
@@ -55,10 +55,10 @@ define <16 x i1> @test_16i1(<16 x i32> %a, <16 x i32> %b) {
 ; CHECK-NEXT:    vpcmpnleud %zmm1, %zmm0, %k0
 ; CHECK-NEXT:    vpcmpgtd %zmm1, %zmm0, %k1
 ; CHECK-NEXT:    korw %k1, %k0, %k0
-; CHECK-NEXT:    kmovw %k0, {{[0-9]+}}(%rsp) ## 2-byte Spill
+; CHECK-NEXT:    kmovw %k0, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    callq _f
-; CHECK-NEXT:    kmovw {{[0-9]+}}(%rsp), %k0 ## 2-byte Reload
+; CHECK-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 ## 2-byte Reload
 ; CHECK-NEXT:    vpmovm2b %k0, %xmm0
 ; CHECK-NEXT:    popq %rax
 ; CHECK-NEXT:    retq
@@ -77,10 +77,10 @@ define <32 x i1> @test_32i1(<32 x i16> %a, <32 x i16> %b) {
 ; CHECK-NEXT:    vpcmpnleuw %zmm1, %zmm0, %k0
 ; CHECK-NEXT:    vpcmpgtw %zmm1, %zmm0, %k1
 ; CHECK-NEXT:    kord %k1, %k0, %k0
-; CHECK-NEXT:    kmovd %k0, {{[0-9]+}}(%rsp) ## 4-byte Spill
+; CHECK-NEXT:    kmovd %k0, {{[-0-9]+}}(%r{{[sb]}}p) ## 4-byte Spill
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    callq _f
-; CHECK-NEXT:    kmovd {{[0-9]+}}(%rsp), %k0 ## 4-byte Reload
+; CHECK-NEXT:    kmovd {{[-0-9]+}}(%r{{[sb]}}p), %k0 ## 4-byte Reload
 ; CHECK-NEXT:    vpmovm2b %k0, %ymm0
 ; CHECK-NEXT:    popq %rax
 ; CHECK-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/avx512-masked_memop-16-8.ll b/llvm/test/CodeGen/X86/avx512-masked_memop-16-8.ll
index bf8df5e56..e4f1e7f6a 100644
--- a/llvm/test/CodeGen/X86/avx512-masked_memop-16-8.ll
+++ b/llvm/test/CodeGen/X86/avx512-masked_memop-16-8.ll
@@ -158,13 +158,12 @@ define <16 x half> @test_mask_load_16xf16(<16 x i1> %mask, <16 x half>* %addr, <
 ; CHECK:       ## %bb.0:
 ; CHECK-NEXT:    movq %rdi, %rax
 ; CHECK-NEXT:    vpsllw $7, %xmm0, %xmm0
-; CHECK-NEXT:    vpmovb2m %xmm0, %k0
-; CHECK-NEXT:    kmovd %k0, %ecx
+; CHECK-NEXT:    vpmovmskb %xmm0, %ecx
 ; CHECK-NEXT:    testb $1, %cl
 ; CHECK-NEXT:    je LBB12_1
 ; CHECK-NEXT:  ## %bb.2: ## %cond.load
-; CHECK-NEXT:    movswl (%rsi), %ecx
-; CHECK-NEXT:    vmovd %ecx, %xmm0
+; CHECK-NEXT:    movswl (%rsi), %edx
+; CHECK-NEXT:    vmovd %edx, %xmm0
 ; CHECK-NEXT:    vcvtph2ps %xmm0, %xmm8
 ; CHECK-NEXT:    jmp LBB12_3
 ; CHECK-NEXT:  LBB12_1:
@@ -172,13 +171,11 @@ define <16 x half> @test_mask_load_16xf16(<16 x i1> %mask, <16 x half>* %addr, <
 ; CHECK-NEXT:  LBB12_3: ## %else
 ; CHECK-NEXT:    vxorps %xmm2, %xmm2, %xmm2
 ; CHECK-NEXT:    vxorps %xmm9, %xmm9, %xmm9
-; CHECK-NEXT:    kshiftrw $1, %k0, %k1
-; CHECK-NEXT:    kmovd %k1, %ecx
-; CHECK-NEXT:    testb $1, %cl
+; CHECK-NEXT:    testb $2, %cl
 ; CHECK-NEXT:    je LBB12_4
 ; CHECK-NEXT:  ## %bb.5: ## %cond.load1
-; CHECK-NEXT:    movswl 2(%rsi), %ecx
-; CHECK-NEXT:    vmovd %ecx, %xmm0
+; CHECK-NEXT:    movswl 2(%rsi), %edx
+; CHECK-NEXT:    vmovd %edx, %xmm0
 ; CHECK-NEXT:    vmovaps %xmm2, %xmm1
 ; CHECK-NEXT:    vmovaps %xmm2, %xmm7
 ; CHECK-NEXT:    vmovaps %xmm2, %xmm6
@@ -193,7 +190,9 @@ define <16 x half> @test_mask_load_16xf16(<16 x i1> %mask, <16 x half>* %addr, <
 ; CHECK-NEXT:    vmovaps %xmm2, %xmm11
 ; CHECK-NEXT:    vmovaps %xmm2, %xmm10
 ; CHECK-NEXT:    vcvtph2ps %xmm0, %xmm2
-; CHECK-NEXT:    jmp LBB12_6
+; CHECK-NEXT:    testb $4, %cl
+; CHECK-NEXT:    jne LBB12_7
+; CHECK-NEXT:    jmp LBB12_8
 ; CHECK-NEXT:  LBB12_4:
 ; CHECK-NEXT:    vmovaps %xmm2, %xmm1
 ; CHECK-NEXT:    vmovaps %xmm2, %xmm7
@@ -208,129 +207,52 @@ define <16 x half> @test_mask_load_16xf16(<16 x i1> %mask, <16 x half>* %addr, <
 ; CHECK-NEXT:    vmovaps %xmm2, %xmm12
 ; CHECK-NEXT:    vmovaps %xmm2, %xmm11
 ; CHECK-NEXT:    vmovaps %xmm2, %xmm10
-; CHECK-NEXT:  LBB12_6: ## %else2
-; CHECK-NEXT:    kshiftrw $2, %k0, %k1
-; CHECK-NEXT:    kmovd %k1, %ecx
-; CHECK-NEXT:    testb $1, %cl
+; CHECK-NEXT:    testb $4, %cl
 ; CHECK-NEXT:    je LBB12_8
-; CHECK-NEXT:  ## %bb.7: ## %cond.load4
-; CHECK-NEXT:    movswl 4(%rsi), %ecx
-; CHECK-NEXT:    vmovd %ecx, %xmm0
+; CHECK-NEXT:  LBB12_7: ## %cond.load4
+; CHECK-NEXT:    movswl 4(%rsi), %edx
+; CHECK-NEXT:    vmovd %edx, %xmm0
 ; CHECK-NEXT:    vcvtph2ps %xmm0, %xmm1
 ; CHECK-NEXT:  LBB12_8: ## %else5
-; CHECK-NEXT:    kshiftrw $3, %k0, %k1
-; CHECK-NEXT:    kmovd %k1, %ecx
-; CHECK-NEXT:    testb $1, %cl
-; CHECK-NEXT:    je LBB12_10
-; CHECK-NEXT:  ## %bb.9: ## %cond.load7
-; CHECK-NEXT:    movswl 6(%rsi), %ecx
-; CHECK-NEXT:    vmovd %ecx, %xmm0
-; CHECK-NEXT:    vcvtph2ps %xmm0, %xmm7
-; CHECK-NEXT:  LBB12_10: ## %else8
-; CHECK-NEXT:    kshiftrw $4, %k0, %k1
-; CHECK-NEXT:    kmovd %k1, %ecx
-; CHECK-NEXT:    testb $1, %cl
-; CHECK-NEXT:    je LBB12_12
-; CHECK-NEXT:  ## %bb.11: ## %cond.load10
-; CHECK-NEXT:    movswl 8(%rsi), %ecx
-; CHECK-NEXT:    vmovd %ecx, %xmm0
-; CHECK-NEXT:    vcvtph2ps %xmm0, %xmm6
+; CHECK-NEXT:    testb $8, %cl
+; CHECK-NEXT:    jne LBB12_9
+; CHECK-NEXT:  ## %bb.10: ## %else8
+; CHECK-NEXT:    testb $16, %cl
+; CHECK-NEXT:    jne LBB12_11
 ; CHECK-NEXT:  LBB12_12: ## %else11
-; CHECK-NEXT:    kshiftrw $5, %k0, %k1
-; CHECK-NEXT:    kmovd %k1, %ecx
-; CHECK-NEXT:    testb $1, %cl
-; CHECK-NEXT:    je LBB12_14
-; CHECK-NEXT:  ## %bb.13: ## %cond.load13
-; CHECK-NEXT:    movswl 10(%rsi), %ecx
-; CHECK-NEXT:    vmovd %ecx, %xmm0
-; CHECK-NEXT:    vcvtph2ps %xmm0, %xmm5
+; CHECK-NEXT:    testb $32, %cl
+; CHECK-NEXT:    jne LBB12_13
 ; CHECK-NEXT:  LBB12_14: ## %else14
-; CHECK-NEXT:    kshiftrw $6, %k0, %k1
-; CHECK-NEXT:    kmovd %k1, %ecx
-; CHECK-NEXT:    testb $1, %cl
-; CHECK-NEXT:    je LBB12_16
-; CHECK-NEXT:  ## %bb.15: ## %cond.load16
-; CHECK-NEXT:    movswl 12(%rsi), %ecx
-; CHECK-NEXT:    vmovd %ecx, %xmm0
-; CHECK-NEXT:    vcvtph2ps %xmm0, %xmm4
+; CHECK-NEXT:    testb $64, %cl
+; CHECK-NEXT:    jne LBB12_15
 ; CHECK-NEXT:  LBB12_16: ## %else17
-; CHECK-NEXT:    kshiftrw $7, %k0, %k1
-; CHECK-NEXT:    kmovd %k1, %ecx
-; CHECK-NEXT:    testb $1, %cl
-; CHECK-NEXT:    je LBB12_18
-; CHECK-NEXT:  ## %bb.17: ## %cond.load19
-; CHECK-NEXT:    movswl 14(%rsi), %ecx
-; CHECK-NEXT:    vmovd %ecx, %xmm0
-; CHECK-NEXT:    vcvtph2ps %xmm0, %xmm3
+; CHECK-NEXT:    testb $-128, %cl
+; CHECK-NEXT:    jne LBB12_17
 ; CHECK-NEXT:  LBB12_18: ## %else20
-; CHECK-NEXT:    kshiftrw $8, %k0, %k1
-; CHECK-NEXT:    kmovd %k1, %ecx
-; CHECK-NEXT:    testb $1, %cl
-; CHECK-NEXT:    je LBB12_20
-; CHECK-NEXT:  ## %bb.19: ## %cond.load22
-; CHECK-NEXT:    movswl 16(%rsi), %ecx
-; CHECK-NEXT:    vmovd %ecx, %xmm0
-; CHECK-NEXT:    vcvtph2ps %xmm0, %xmm16
+; CHECK-NEXT:    testl $256, %ecx ## imm = 0x100
+; CHECK-NEXT:    jne LBB12_19
 ; CHECK-NEXT:  LBB12_20: ## %else23
-; CHECK-NEXT:    kshiftrw $9, %k0, %k1
-; CHECK-NEXT:    kmovd %k1, %ecx
-; CHECK-NEXT:    testb $1, %cl
-; CHECK-NEXT:    je LBB12_22
-; CHECK-NEXT:  ## %bb.21: ## %cond.load25
-; CHECK-NEXT:    movswl 18(%rsi), %ecx
-; CHECK-NEXT:    vmovd %ecx, %xmm0
-; CHECK-NEXT:    vcvtph2ps %xmm0, %xmm15
+; CHECK-NEXT:    testl $512, %ecx ## imm = 0x200
+; CHECK-NEXT:    jne LBB12_21
 ; CHECK-NEXT:  LBB12_22: ## %else26
-; CHECK-NEXT:    kshiftrw $10, %k0, %k1
-; CHECK-NEXT:    kmovd %k1, %ecx
-; CHECK-NEXT:    testb $1, %cl
-; CHECK-NEXT:    je LBB12_24
-; CHECK-NEXT:  ## %bb.23: ## %cond.load28
-; CHECK-NEXT:    movswl 20(%rsi), %ecx
-; CHECK-NEXT:    vmovd %ecx, %xmm0
-; CHECK-NEXT:    vcvtph2ps %xmm0, %xmm14
+; CHECK-NEXT:    testl $1024, %ecx ## imm = 0x400
+; CHECK-NEXT:    jne LBB12_23
 ; CHECK-NEXT:  LBB12_24: ## %else29
-; CHECK-NEXT:    kshiftrw $11, %k0, %k1
-; CHECK-NEXT:    kmovd %k1, %ecx
-; CHECK-NEXT:    testb $1, %cl
-; CHECK-NEXT:    je LBB12_26
-; CHECK-NEXT:  ## %bb.25: ## %cond.load31
-; CHECK-NEXT:    movswl 22(%rsi), %ecx
-; CHECK-NEXT:    vmovd %ecx, %xmm0
-; CHECK-NEXT:    vcvtph2ps %xmm0, %xmm13
+; CHECK-NEXT:    testl $2048, %ecx ## imm = 0x800
+; CHECK-NEXT:    jne LBB12_25
 ; CHECK-NEXT:  LBB12_26: ## %else32
-; CHECK-NEXT:    kshiftrw $12, %k0, %k1
-; CHECK-NEXT:    kmovd %k1, %ecx
-; CHECK-NEXT:    testb $1, %cl
-; CHECK-NEXT:    je LBB12_28
-; CHECK-NEXT:  ## %bb.27: ## %cond.load34
-; CHECK-NEXT:    movswl 24(%rsi), %ecx
-; CHECK-NEXT:    vmovd %ecx, %xmm0
-; CHECK-NEXT:    vcvtph2ps %xmm0, %xmm12
+; CHECK-NEXT:    testl $4096, %ecx ## imm = 0x1000
+; CHECK-NEXT:    jne LBB12_27
 ; CHECK-NEXT:  LBB12_28: ## %else35
-; CHECK-NEXT:    kshiftrw $13, %k0, %k1
-; CHECK-NEXT:    kmovd %k1, %ecx
-; CHECK-NEXT:    testb $1, %cl
-; CHECK-NEXT:    je LBB12_30
-; CHECK-NEXT:  ## %bb.29: ## %cond.load37
-; CHECK-NEXT:    movswl 26(%rsi), %ecx
-; CHECK-NEXT:    vmovd %ecx, %xmm0
-; CHECK-NEXT:    vcvtph2ps %xmm0, %xmm11
+; CHECK-NEXT:    testl $8192, %ecx ## imm = 0x2000
+; CHECK-NEXT:    jne LBB12_29
 ; CHECK-NEXT:  LBB12_30: ## %else38
-; CHECK-NEXT:    kshiftrw $14, %k0, %k1
-; CHECK-NEXT:    kmovd %k1, %ecx
-; CHECK-NEXT:    testb $1, %cl
-; CHECK-NEXT:    je LBB12_32
-; CHECK-NEXT:  ## %bb.31: ## %cond.load40
-; CHECK-NEXT:    movswl 28(%rsi), %ecx
-; CHECK-NEXT:    vmovd %ecx, %xmm0
-; CHECK-NEXT:    vcvtph2ps %xmm0, %xmm10
+; CHECK-NEXT:    testl $16384, %ecx ## imm = 0x4000
+; CHECK-NEXT:    jne LBB12_31
 ; CHECK-NEXT:  LBB12_32: ## %else41
-; CHECK-NEXT:    kshiftrw $15, %k0, %k0
-; CHECK-NEXT:    kmovd %k0, %ecx
-; CHECK-NEXT:    testb $1, %cl
+; CHECK-NEXT:    testl $32768, %ecx ## imm = 0x8000
 ; CHECK-NEXT:    je LBB12_34
-; CHECK-NEXT:  ## %bb.33: ## %cond.load43
+; CHECK-NEXT:  LBB12_33: ## %cond.load43
 ; CHECK-NEXT:    movswl 30(%rsi), %ecx
 ; CHECK-NEXT:    vmovd %ecx, %xmm0
 ; CHECK-NEXT:    vcvtph2ps %xmm0, %xmm9
@@ -384,6 +306,79 @@ define <16 x half> @test_mask_load_16xf16(<16 x i1> %mask, <16 x half>* %addr, <
 ; CHECK-NEXT:    vmovd %xmm0, %ecx
 ; CHECK-NEXT:    movw %cx, 30(%rax)
 ; CHECK-NEXT:    retq
+; CHECK-NEXT:  LBB12_9: ## %cond.load7
+; CHECK-NEXT:    movswl 6(%rsi), %edx
+; CHECK-NEXT:    vmovd %edx, %xmm0
+; CHECK-NEXT:    vcvtph2ps %xmm0, %xmm7
+; CHECK-NEXT:    testb $16, %cl
+; CHECK-NEXT:    je LBB12_12
+; CHECK-NEXT:  LBB12_11: ## %cond.load10
+; CHECK-NEXT:    movswl 8(%rsi), %edx
+; CHECK-NEXT:    vmovd %edx, %xmm0
+; CHECK-NEXT:    vcvtph2ps %xmm0, %xmm6
+; CHECK-NEXT:    testb $32, %cl
+; CHECK-NEXT:    je LBB12_14
+; CHECK-NEXT:  LBB12_13: ## %cond.load13
+; CHECK-NEXT:    movswl 10(%rsi), %edx
+; CHECK-NEXT:    vmovd %edx, %xmm0
+; CHECK-NEXT:    vcvtph2ps %xmm0, %xmm5
+; CHECK-NEXT:    testb $64, %cl
+; CHECK-NEXT:    je LBB12_16
+; CHECK-NEXT:  LBB12_15: ## %cond.load16
+; CHECK-NEXT:    movswl 12(%rsi), %edx
+; CHECK-NEXT:    vmovd %edx, %xmm0
+; CHECK-NEXT:    vcvtph2ps %xmm0, %xmm4
+; CHECK-NEXT:    testb $-128, %cl
+; CHECK-NEXT:    je LBB12_18
+; CHECK-NEXT:  LBB12_17: ## %cond.load19
+; CHECK-NEXT:    movswl 14(%rsi), %edx
+; CHECK-NEXT:    vmovd %edx, %xmm0
+; CHECK-NEXT:    vcvtph2ps %xmm0, %xmm3
+; CHECK-NEXT:    testl $256, %ecx ## imm = 0x100
+; CHECK-NEXT:    je LBB12_20
+; CHECK-NEXT:  LBB12_19: ## %cond.load22
+; CHECK-NEXT:    movswl 16(%rsi), %edx
+; CHECK-NEXT:    vmovd %edx, %xmm0
+; CHECK-NEXT:    vcvtph2ps %xmm0, %xmm16
+; CHECK-NEXT:    testl $512, %ecx ## imm = 0x200
+; CHECK-NEXT:    je LBB12_22
+; CHECK-NEXT:  LBB12_21: ## %cond.load25
+; CHECK-NEXT:    movswl 18(%rsi), %edx
+; CHECK-NEXT:    vmovd %edx, %xmm0
+; CHECK-NEXT:    vcvtph2ps %xmm0, %xmm15
+; CHECK-NEXT:    testl $1024, %ecx ## imm = 0x400
+; CHECK-NEXT:    je LBB12_24
+; CHECK-NEXT:  LBB12_23: ## %cond.load28
+; CHECK-NEXT:    movswl 20(%rsi), %edx
+; CHECK-NEXT:    vmovd %edx, %xmm0
+; CHECK-NEXT:    vcvtph2ps %xmm0, %xmm14
+; CHECK-NEXT:    testl $2048, %ecx ## imm = 0x800
+; CHECK-NEXT:    je LBB12_26
+; CHECK-NEXT:  LBB12_25: ## %cond.load31
+; CHECK-NEXT:    movswl 22(%rsi), %edx
+; CHECK-NEXT:    vmovd %edx, %xmm0
+; CHECK-NEXT:    vcvtph2ps %xmm0, %xmm13
+; CHECK-NEXT:    testl $4096, %ecx ## imm = 0x1000
+; CHECK-NEXT:    je LBB12_28
+; CHECK-NEXT:  LBB12_27: ## %cond.load34
+; CHECK-NEXT:    movswl 24(%rsi), %edx
+; CHECK-NEXT:    vmovd %edx, %xmm0
+; CHECK-NEXT:    vcvtph2ps %xmm0, %xmm12
+; CHECK-NEXT:    testl $8192, %ecx ## imm = 0x2000
+; CHECK-NEXT:    je LBB12_30
+; CHECK-NEXT:  LBB12_29: ## %cond.load37
+; CHECK-NEXT:    movswl 26(%rsi), %edx
+; CHECK-NEXT:    vmovd %edx, %xmm0
+; CHECK-NEXT:    vcvtph2ps %xmm0, %xmm11
+; CHECK-NEXT:    testl $16384, %ecx ## imm = 0x4000
+; CHECK-NEXT:    je LBB12_32
+; CHECK-NEXT:  LBB12_31: ## %cond.load40
+; CHECK-NEXT:    movswl 28(%rsi), %edx
+; CHECK-NEXT:    vmovd %edx, %xmm0
+; CHECK-NEXT:    vcvtph2ps %xmm0, %xmm10
+; CHECK-NEXT:    testl $32768, %ecx ## imm = 0x8000
+; CHECK-NEXT:    jne LBB12_33
+; CHECK-NEXT:    jmp LBB12_34
   %res = call <16 x half> @llvm.masked.load.v16f16(<16 x half>* %addr, i32 4, <16 x i1>%mask, <16 x half> zeroinitializer)
   ret <16 x half> %res
 }
@@ -394,159 +389,159 @@ define void @test_mask_store_16xf16(<16 x i1> %mask, <16 x half>* %addr, <16 x h
 ; CHECK-LABEL: test_mask_store_16xf16:
 ; CHECK:       ## %bb.0:
 ; CHECK-NEXT:    vpsllw $7, %xmm0, %xmm0
-; CHECK-NEXT:    vpmovb2m %xmm0, %k0
-; CHECK-NEXT:    kmovd %k0, %eax
+; CHECK-NEXT:    vpmovmskb %xmm0, %eax
 ; CHECK-NEXT:    testb $1, %al
-; CHECK-NEXT:    je LBB13_2
-; CHECK-NEXT:  ## %bb.1: ## %cond.store
+; CHECK-NEXT:    jne LBB13_1
+; CHECK-NEXT:  ## %bb.2: ## %else
+; CHECK-NEXT:    testb $2, %al
+; CHECK-NEXT:    jne LBB13_3
+; CHECK-NEXT:  LBB13_4: ## %else2
+; CHECK-NEXT:    testb $4, %al
+; CHECK-NEXT:    jne LBB13_5
+; CHECK-NEXT:  LBB13_6: ## %else4
+; CHECK-NEXT:    testb $8, %al
+; CHECK-NEXT:    jne LBB13_7
+; CHECK-NEXT:  LBB13_8: ## %else6
+; CHECK-NEXT:    testb $16, %al
+; CHECK-NEXT:    jne LBB13_9
+; CHECK-NEXT:  LBB13_10: ## %else8
+; CHECK-NEXT:    testb $32, %al
+; CHECK-NEXT:    jne LBB13_11
+; CHECK-NEXT:  LBB13_12: ## %else10
+; CHECK-NEXT:    testb $64, %al
+; CHECK-NEXT:    jne LBB13_13
+; CHECK-NEXT:  LBB13_14: ## %else12
+; CHECK-NEXT:    testb $-128, %al
+; CHECK-NEXT:    jne LBB13_15
+; CHECK-NEXT:  LBB13_16: ## %else14
+; CHECK-NEXT:    testl $256, %eax ## imm = 0x100
+; CHECK-NEXT:    jne LBB13_17
+; CHECK-NEXT:  LBB13_18: ## %else16
+; CHECK-NEXT:    testl $512, %eax ## imm = 0x200
+; CHECK-NEXT:    jne LBB13_19
+; CHECK-NEXT:  LBB13_20: ## %else18
+; CHECK-NEXT:    testl $1024, %eax ## imm = 0x400
+; CHECK-NEXT:    jne LBB13_21
+; CHECK-NEXT:  LBB13_22: ## %else20
+; CHECK-NEXT:    testl $2048, %eax ## imm = 0x800
+; CHECK-NEXT:    jne LBB13_23
+; CHECK-NEXT:  LBB13_24: ## %else22
+; CHECK-NEXT:    testl $4096, %eax ## imm = 0x1000
+; CHECK-NEXT:    jne LBB13_25
+; CHECK-NEXT:  LBB13_26: ## %else24
+; CHECK-NEXT:    testl $8192, %eax ## imm = 0x2000
+; CHECK-NEXT:    jne LBB13_27
+; CHECK-NEXT:  LBB13_28: ## %else26
+; CHECK-NEXT:    testl $16384, %eax ## imm = 0x4000
+; CHECK-NEXT:    jne LBB13_29
+; CHECK-NEXT:  LBB13_30: ## %else28
+; CHECK-NEXT:    testl $32768, %eax ## imm = 0x8000
+; CHECK-NEXT:    jne LBB13_31
+; CHECK-NEXT:  LBB13_32: ## %else30
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  LBB13_1: ## %cond.store
 ; CHECK-NEXT:    vcvtps2ph $4, %xmm1, %xmm0
-; CHECK-NEXT:    vmovd %xmm0, %eax
-; CHECK-NEXT:    movw %ax, (%rdi)
-; CHECK-NEXT:  LBB13_2: ## %else
-; CHECK-NEXT:    kshiftrw $1, %k0, %k1
-; CHECK-NEXT:    kmovd %k1, %eax
-; CHECK-NEXT:    testb $1, %al
+; CHECK-NEXT:    vmovd %xmm0, %ecx
+; CHECK-NEXT:    movw %cx, (%rdi)
+; CHECK-NEXT:    testb $2, %al
 ; CHECK-NEXT:    je LBB13_4
-; CHECK-NEXT:  ## %bb.3: ## %cond.store1
+; CHECK-NEXT:  LBB13_3: ## %cond.store1
 ; CHECK-NEXT:    vcvtps2ph $4, %xmm2, %xmm0
-; CHECK-NEXT:    vmovd %xmm0, %eax
-; CHECK-NEXT:    movw %ax, 2(%rdi)
-; CHECK-NEXT:  LBB13_4: ## %else2
-; CHECK-NEXT:    kshiftrw $2, %k0, %k1
-; CHECK-NEXT:    kmovd %k1, %eax
-; CHECK-NEXT:    testb $1, %al
+; CHECK-NEXT:    vmovd %xmm0, %ecx
+; CHECK-NEXT:    movw %cx, 2(%rdi)
+; CHECK-NEXT:    testb $4, %al
 ; CHECK-NEXT:    je LBB13_6
-; CHECK-NEXT:  ## %bb.5: ## %cond.store3
+; CHECK-NEXT:  LBB13_5: ## %cond.store3
 ; CHECK-NEXT:    vcvtps2ph $4, %xmm3, %xmm0
-; CHECK-NEXT:    vmovd %xmm0, %eax
-; CHECK-NEXT:    movw %ax, 4(%rdi)
-; CHECK-NEXT:  LBB13_6: ## %else4
-; CHECK-NEXT:    kshiftrw $3, %k0, %k1
-; CHECK-NEXT:    kmovd %k1, %eax
-; CHECK-NEXT:    testb $1, %al
+; CHECK-NEXT:    vmovd %xmm0, %ecx
+; CHECK-NEXT:    movw %cx, 4(%rdi)
+; CHECK-NEXT:    testb $8, %al
 ; CHECK-NEXT:    je LBB13_8
-; CHECK-NEXT:  ## %bb.7: ## %cond.store5
+; CHECK-NEXT:  LBB13_7: ## %cond.store5
 ; CHECK-NEXT:    vcvtps2ph $4, %xmm4, %xmm0
-; CHECK-NEXT:    vmovd %xmm0, %eax
-; CHECK-NEXT:    movw %ax, 6(%rdi)
-; CHECK-NEXT:  LBB13_8: ## %else6
-; CHECK-NEXT:    kshiftrw $4, %k0, %k1
-; CHECK-NEXT:    kmovd %k1, %eax
-; CHECK-NEXT:    testb $1, %al
+; CHECK-NEXT:    vmovd %xmm0, %ecx
+; CHECK-NEXT:    movw %cx, 6(%rdi)
+; CHECK-NEXT:    testb $16, %al
 ; CHECK-NEXT:    je LBB13_10
-; CHECK-NEXT:  ## %bb.9: ## %cond.store7
+; CHECK-NEXT:  LBB13_9: ## %cond.store7
 ; CHECK-NEXT:    vcvtps2ph $4, %xmm5, %xmm0
-; CHECK-NEXT:    vmovd %xmm0, %eax
-; CHECK-NEXT:    movw %ax, 8(%rdi)
-; CHECK-NEXT:  LBB13_10: ## %else8
-; CHECK-NEXT:    kshiftrw $5, %k0, %k1
-; CHECK-NEXT:    kmovd %k1, %eax
-; CHECK-NEXT:    testb $1, %al
+; CHECK-NEXT:    vmovd %xmm0, %ecx
+; CHECK-NEXT:    movw %cx, 8(%rdi)
+; CHECK-NEXT:    testb $32, %al
 ; CHECK-NEXT:    je LBB13_12
-; CHECK-NEXT:  ## %bb.11: ## %cond.store9
+; CHECK-NEXT:  LBB13_11: ## %cond.store9
 ; CHECK-NEXT:    vcvtps2ph $4, %xmm6, %xmm0
-; CHECK-NEXT:    vmovd %xmm0, %eax
-; CHECK-NEXT:    movw %ax, 10(%rdi)
-; CHECK-NEXT:  LBB13_12: ## %else10
-; CHECK-NEXT:    kshiftrw $6, %k0, %k1
-; CHECK-NEXT:    kmovd %k1, %eax
-; CHECK-NEXT:    testb $1, %al
+; CHECK-NEXT:    vmovd %xmm0, %ecx
+; CHECK-NEXT:    movw %cx, 10(%rdi)
+; CHECK-NEXT:    testb $64, %al
 ; CHECK-NEXT:    je LBB13_14
-; CHECK-NEXT:  ## %bb.13: ## %cond.store11
+; CHECK-NEXT:  LBB13_13: ## %cond.store11
 ; CHECK-NEXT:    vcvtps2ph $4, %xmm7, %xmm0
-; CHECK-NEXT:    vmovd %xmm0, %eax
-; CHECK-NEXT:    movw %ax, 12(%rdi)
-; CHECK-NEXT:  LBB13_14: ## %else12
-; CHECK-NEXT:    kshiftrw $7, %k0, %k1
-; CHECK-NEXT:    kmovd %k1, %eax
-; CHECK-NEXT:    testb $1, %al
+; CHECK-NEXT:    vmovd %xmm0, %ecx
+; CHECK-NEXT:    movw %cx, 12(%rdi)
+; CHECK-NEXT:    testb $-128, %al
 ; CHECK-NEXT:    je LBB13_16
-; CHECK-NEXT:  ## %bb.15: ## %cond.store13
+; CHECK-NEXT:  LBB13_15: ## %cond.store13
 ; CHECK-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; CHECK-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; CHECK-NEXT:    vmovd %xmm0, %eax
-; CHECK-NEXT:    movw %ax, 14(%rdi)
-; CHECK-NEXT:  LBB13_16: ## %else14
-; CHECK-NEXT:    kshiftrw $8, %k0, %k1
-; CHECK-NEXT:    kmovd %k1, %eax
-; CHECK-NEXT:    testb $1, %al
+; CHECK-NEXT:    vmovd %xmm0, %ecx
+; CHECK-NEXT:    movw %cx, 14(%rdi)
+; CHECK-NEXT:    testl $256, %eax ## imm = 0x100
 ; CHECK-NEXT:    je LBB13_18
-; CHECK-NEXT:  ## %bb.17: ## %cond.store15
+; CHECK-NEXT:  LBB13_17: ## %cond.store15
 ; CHECK-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; CHECK-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; CHECK-NEXT:    vmovd %xmm0, %eax
-; CHECK-NEXT:    movw %ax, 16(%rdi)
-; CHECK-NEXT:  LBB13_18: ## %else16
-; CHECK-NEXT:    kshiftrw $9, %k0, %k1
-; CHECK-NEXT:    kmovd %k1, %eax
-; CHECK-NEXT:    testb $1, %al
+; CHECK-NEXT:    vmovd %xmm0, %ecx
+; CHECK-NEXT:    movw %cx, 16(%rdi)
+; CHECK-NEXT:    testl $512, %eax ## imm = 0x200
 ; CHECK-NEXT:    je LBB13_20
-; CHECK-NEXT:  ## %bb.19: ## %cond.store17
+; CHECK-NEXT:  LBB13_19: ## %cond.store17
 ; CHECK-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; CHECK-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; CHECK-NEXT:    vmovd %xmm0, %eax
-; CHECK-NEXT:    movw %ax, 18(%rdi)
-; CHECK-NEXT:  LBB13_20: ## %else18
-; CHECK-NEXT:    kshiftrw $10, %k0, %k1
-; CHECK-NEXT:    kmovd %k1, %eax
-; CHECK-NEXT:    testb $1, %al
+; CHECK-NEXT:    vmovd %xmm0, %ecx
+; CHECK-NEXT:    movw %cx, 18(%rdi)
+; CHECK-NEXT:    testl $1024, %eax ## imm = 0x400
 ; CHECK-NEXT:    je LBB13_22
-; CHECK-NEXT:  ## %bb.21: ## %cond.store19
+; CHECK-NEXT:  LBB13_21: ## %cond.store19
 ; CHECK-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; CHECK-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; CHECK-NEXT:    vmovd %xmm0, %eax
-; CHECK-NEXT:    movw %ax, 20(%rdi)
-; CHECK-NEXT:  LBB13_22: ## %else20
-; CHECK-NEXT:    kshiftrw $11, %k0, %k1
-; CHECK-NEXT:    kmovd %k1, %eax
-; CHECK-NEXT:    testb $1, %al
+; CHECK-NEXT:    vmovd %xmm0, %ecx
+; CHECK-NEXT:    movw %cx, 20(%rdi)
+; CHECK-NEXT:    testl $2048, %eax ## imm = 0x800
 ; CHECK-NEXT:    je LBB13_24
-; CHECK-NEXT:  ## %bb.23: ## %cond.store21
+; CHECK-NEXT:  LBB13_23: ## %cond.store21
 ; CHECK-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; CHECK-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; CHECK-NEXT:    vmovd %xmm0, %eax
-; CHECK-NEXT:    movw %ax, 22(%rdi)
-; CHECK-NEXT:  LBB13_24: ## %else22
-; CHECK-NEXT:    kshiftrw $12, %k0, %k1
-; CHECK-NEXT:    kmovd %k1, %eax
-; CHECK-NEXT:    testb $1, %al
+; CHECK-NEXT:    vmovd %xmm0, %ecx
+; CHECK-NEXT:    movw %cx, 22(%rdi)
+; CHECK-NEXT:    testl $4096, %eax ## imm = 0x1000
 ; CHECK-NEXT:    je LBB13_26
-; CHECK-NEXT:  ## %bb.25: ## %cond.store23
+; CHECK-NEXT:  LBB13_25: ## %cond.store23
 ; CHECK-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; CHECK-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; CHECK-NEXT:    vmovd %xmm0, %eax
-; CHECK-NEXT:    movw %ax, 24(%rdi)
-; CHECK-NEXT:  LBB13_26: ## %else24
-; CHECK-NEXT:    kshiftrw $13, %k0, %k1
-; CHECK-NEXT:    kmovd %k1, %eax
-; CHECK-NEXT:    testb $1, %al
+; CHECK-NEXT:    vmovd %xmm0, %ecx
+; CHECK-NEXT:    movw %cx, 24(%rdi)
+; CHECK-NEXT:    testl $8192, %eax ## imm = 0x2000
 ; CHECK-NEXT:    je LBB13_28
-; CHECK-NEXT:  ## %bb.27: ## %cond.store25
+; CHECK-NEXT:  LBB13_27: ## %cond.store25
 ; CHECK-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; CHECK-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; CHECK-NEXT:    vmovd %xmm0, %eax
-; CHECK-NEXT:    movw %ax, 26(%rdi)
-; CHECK-NEXT:  LBB13_28: ## %else26
-; CHECK-NEXT:    kshiftrw $14, %k0, %k1
-; CHECK-NEXT:    kmovd %k1, %eax
-; CHECK-NEXT:    testb $1, %al
+; CHECK-NEXT:    vmovd %xmm0, %ecx
+; CHECK-NEXT:    movw %cx, 26(%rdi)
+; CHECK-NEXT:    testl $16384, %eax ## imm = 0x4000
 ; CHECK-NEXT:    je LBB13_30
-; CHECK-NEXT:  ## %bb.29: ## %cond.store27
+; CHECK-NEXT:  LBB13_29: ## %cond.store27
 ; CHECK-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; CHECK-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; CHECK-NEXT:    vmovd %xmm0, %eax
-; CHECK-NEXT:    movw %ax, 28(%rdi)
-; CHECK-NEXT:  LBB13_30: ## %else28
-; CHECK-NEXT:    kshiftrw $15, %k0, %k0
-; CHECK-NEXT:    kmovd %k0, %eax
-; CHECK-NEXT:    testb $1, %al
+; CHECK-NEXT:    vmovd %xmm0, %ecx
+; CHECK-NEXT:    movw %cx, 28(%rdi)
+; CHECK-NEXT:    testl $32768, %eax ## imm = 0x8000
 ; CHECK-NEXT:    je LBB13_32
-; CHECK-NEXT:  ## %bb.31: ## %cond.store29
+; CHECK-NEXT:  LBB13_31: ## %cond.store29
 ; CHECK-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; CHECK-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
 ; CHECK-NEXT:    vmovd %xmm0, %eax
 ; CHECK-NEXT:    movw %ax, 30(%rdi)
-; CHECK-NEXT:  LBB13_32: ## %else30
 ; CHECK-NEXT:    retq
   call void @llvm.masked.store.v16f16.p0v16f16(<16 x half> %val, <16 x half>* %addr, i32 4, <16 x i1>%mask)
   ret void
diff --git a/llvm/test/CodeGen/X86/avx512-memfold.ll b/llvm/test/CodeGen/X86/avx512-memfold.ll
index d80891868..6feb622a0 100644
--- a/llvm/test/CodeGen/X86/avx512-memfold.ll
+++ b/llvm/test/CodeGen/X86/avx512-memfold.ll
@@ -58,7 +58,7 @@ define <2 x double> @test_int_x86_avx512_mask_vfmadd_sd(<2 x double> %a, <2 x do
 ; CHECK-LABEL: test_int_x86_avx512_mask_vfmadd_sd:
 ; CHECK:       ## %bb.0:
 ; CHECK-NEXT:    kmovw %esi, %k1
-; CHECK-NEXT:    vfmadd213sd (%rdi), %xmm1, %xmm0 {%k1}
+; CHECK-NEXT:    vfmadd213sd {{.*#+}} xmm0 = (xmm1 * xmm0) + mem
 ; CHECK-NEXT:    retq
   %c.val = load double, double* %c
   %cv0 = insertelement <2 x double> undef, double %c.val, i32 0
diff --git a/llvm/test/CodeGen/X86/avx512-nontemporal.ll b/llvm/test/CodeGen/X86/avx512-nontemporal.ll
index 9bc8a8f97..617b44f0b 100644
--- a/llvm/test/CodeGen/X86/avx512-nontemporal.ll
+++ b/llvm/test/CodeGen/X86/avx512-nontemporal.ll
@@ -1,33 +1,63 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512f,+avx512bw | FileCheck %s
 
 define i32 @f(<16 x float> %A, <16 x float> %AA, i8* %B, <8 x double> %C, <8 x double> %CC, <8 x i64> %E, <8 x i64> %EE, <16 x i32> %F, <16 x i32> %FF, <32 x i16> %G, <32 x i16> %GG, <64 x i8> %H, <64 x i8> %HH, i32 * %loadptr) {
-; CHECK: vmovntps %z
+; CHECK-LABEL: f:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movq %rsp, %rbp
+; CHECK-NEXT:    .cfi_def_cfa_register %rbp
+; CHECK-NEXT:    andq $-64, %rsp
+; CHECK-NEXT:    subq $64, %rsp
+; CHECK-NEXT:    vmovdqa64 144(%rbp), %zmm8
+; CHECK-NEXT:    vmovdqa64 16(%rbp), %zmm9
+; CHECK-NEXT:    movl (%rsi), %eax
+; CHECK-NEXT:    vaddps %zmm1, %zmm0, %zmm0
+; CHECK-NEXT:    vmovntps %zmm0, (%rdi)
+; CHECK-NEXT:    vpaddq %zmm5, %zmm4, %zmm0
+; CHECK-NEXT:    addl (%rsi), %eax
+; CHECK-NEXT:    vmovntdq %zmm0, (%rdi)
+; CHECK-NEXT:    vaddpd %zmm3, %zmm2, %zmm0
+; CHECK-NEXT:    addl (%rsi), %eax
+; CHECK-NEXT:    vmovntpd %zmm0, (%rdi)
+; CHECK-NEXT:    vpaddd %zmm7, %zmm6, %zmm0
+; CHECK-NEXT:    addl (%rsi), %eax
+; CHECK-NEXT:    vmovntdq %zmm0, (%rdi)
+; CHECK-NEXT:    vpaddw 80(%rbp), %zmm9, %zmm0
+; CHECK-NEXT:    addl (%rsi), %eax
+; CHECK-NEXT:    vmovntdq %zmm0, (%rdi)
+; CHECK-NEXT:    vpaddb 208(%rbp), %zmm8, %zmm0
+; CHECK-NEXT:    addl (%rsi), %eax
+; CHECK-NEXT:    vmovntdq %zmm0, (%rdi)
+; CHECK-NEXT:    addl (%rsi), %eax
+; CHECK-NEXT:    movq %rbp, %rsp
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa %rsp, 8
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
   %v0 = load i32, i32* %loadptr, align 1
   %cast = bitcast i8* %B to <16 x float>*
   %A2 = fadd <16 x float> %A, %AA
   store <16 x float> %A2, <16 x float>* %cast, align 64, !nontemporal !0
   %v1 = load i32, i32* %loadptr, align 1
-; CHECK: vmovntdq %z
   %cast1 = bitcast i8* %B to <8 x i64>*
   %E2 = add <8 x i64> %E, %EE
   store <8 x i64> %E2, <8 x i64>* %cast1, align 64, !nontemporal !0
   %v2 = load i32, i32* %loadptr, align 1
-; CHECK: vmovntpd %z
   %cast2 = bitcast i8* %B to <8 x double>*
   %C2 = fadd <8 x double> %C, %CC
   store <8 x double> %C2, <8 x double>* %cast2, align 64, !nontemporal !0
   %v3 = load i32, i32* %loadptr, align 1
-; CHECK: vmovntdq %z
   %cast3 = bitcast i8* %B to <16 x i32>*
   %F2 = add <16 x i32> %F, %FF
   store <16 x i32> %F2, <16 x i32>* %cast3, align 64, !nontemporal !0
   %v4 = load i32, i32* %loadptr, align 1
-; CHECK: vmovntdq %z
   %cast4 = bitcast i8* %B to <32 x i16>*
   %G2 = add <32 x i16> %G, %GG
   store <32 x i16> %G2, <32 x i16>* %cast4, align 64, !nontemporal !0
   %v5 = load i32, i32* %loadptr, align 1
-; CHECK: vmovntdq %z
   %cast5 = bitcast i8* %B to <64 x i8>*
   %H2 = add <64 x i8> %H, %HH
   store <64 x i8> %H2, <64 x i8>* %cast5, align 64, !nontemporal !0
diff --git a/llvm/test/CodeGen/X86/avx512-regcall-Mask.ll b/llvm/test/CodeGen/X86/avx512-regcall-Mask.ll
index 41dfe1640..01f89aa58 100644
--- a/llvm/test/CodeGen/X86/avx512-regcall-Mask.ll
+++ b/llvm/test/CodeGen/X86/avx512-regcall-Mask.ll
@@ -118,21 +118,21 @@ define i64 @caller_argv64i1() #0 {
 ; WIN64-LABEL: caller_argv64i1:
 ; WIN64:       # %bb.0: # %entry
 ; WIN64-NEXT:    pushq %r15
-; WIN64-NEXT:    .seh_pushreg 15
+; WIN64-NEXT:    .seh_pushreg %r15
 ; WIN64-NEXT:    pushq %r14
-; WIN64-NEXT:    .seh_pushreg 14
+; WIN64-NEXT:    .seh_pushreg %r14
 ; WIN64-NEXT:    pushq %r12
-; WIN64-NEXT:    .seh_pushreg 12
+; WIN64-NEXT:    .seh_pushreg %r12
 ; WIN64-NEXT:    pushq %rsi
-; WIN64-NEXT:    .seh_pushreg 6
+; WIN64-NEXT:    .seh_pushreg %rsi
 ; WIN64-NEXT:    pushq %rdi
-; WIN64-NEXT:    .seh_pushreg 7
+; WIN64-NEXT:    .seh_pushreg %rdi
 ; WIN64-NEXT:    subq $48, %rsp
 ; WIN64-NEXT:    .seh_stackalloc 48
 ; WIN64-NEXT:    vmovaps %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; WIN64-NEXT:    .seh_savexmm 7, 32
+; WIN64-NEXT:    .seh_savexmm %xmm7, 32
 ; WIN64-NEXT:    vmovaps %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; WIN64-NEXT:    .seh_savexmm 6, 16
+; WIN64-NEXT:    .seh_savexmm %xmm6, 16
 ; WIN64-NEXT:    .seh_endprologue
 ; WIN64-NEXT:    movabsq $4294967298, %rax # imm = 0x100000002
 ; WIN64-NEXT:    movq %rax, (%rsp)
@@ -244,15 +244,15 @@ define <64 x i1> @caller_retv64i1() #0 {
 ; WIN64-LABEL: caller_retv64i1:
 ; WIN64:       # %bb.0: # %entry
 ; WIN64-NEXT:    pushq %rsi
-; WIN64-NEXT:    .seh_pushreg 6
+; WIN64-NEXT:    .seh_pushreg %rsi
 ; WIN64-NEXT:    pushq %rdi
-; WIN64-NEXT:    .seh_pushreg 7
+; WIN64-NEXT:    .seh_pushreg %rdi
 ; WIN64-NEXT:    subq $40, %rsp
 ; WIN64-NEXT:    .seh_stackalloc 40
 ; WIN64-NEXT:    vmovaps %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; WIN64-NEXT:    .seh_savexmm 7, 16
+; WIN64-NEXT:    .seh_savexmm %xmm7, 16
 ; WIN64-NEXT:    vmovaps %xmm6, (%rsp) # 16-byte Spill
-; WIN64-NEXT:    .seh_savexmm 6, 0
+; WIN64-NEXT:    .seh_savexmm %xmm6, 0
 ; WIN64-NEXT:    .seh_endprologue
 ; WIN64-NEXT:    callq test_retv64i1
 ; WIN64-NEXT:    kmovq %rax, %k0
@@ -315,11 +315,11 @@ define x86_regcallcc i32 @test_argv32i1(<32 x i1> %x0, <32 x i1> %x1, <32 x i1>
 ; WIN64-LABEL: test_argv32i1:
 ; WIN64:       # %bb.0: # %entry
 ; WIN64-NEXT:    pushq %r11
-; WIN64-NEXT:    .seh_pushreg 11
+; WIN64-NEXT:    .seh_pushreg %r11
 ; WIN64-NEXT:    pushq %r10
-; WIN64-NEXT:    .seh_pushreg 10
+; WIN64-NEXT:    .seh_pushreg %r10
 ; WIN64-NEXT:    pushq %rsp
-; WIN64-NEXT:    .seh_pushreg 4
+; WIN64-NEXT:    .seh_pushreg %rsp
 ; WIN64-NEXT:    subq $32, %rsp
 ; WIN64-NEXT:    .seh_stackalloc 32
 ; WIN64-NEXT:    .seh_endprologue
@@ -409,15 +409,15 @@ define i32 @caller_argv32i1() #0 {
 ; WIN64-LABEL: caller_argv32i1:
 ; WIN64:       # %bb.0: # %entry
 ; WIN64-NEXT:    pushq %rsi
-; WIN64-NEXT:    .seh_pushreg 6
+; WIN64-NEXT:    .seh_pushreg %rsi
 ; WIN64-NEXT:    pushq %rdi
-; WIN64-NEXT:    .seh_pushreg 7
+; WIN64-NEXT:    .seh_pushreg %rdi
 ; WIN64-NEXT:    subq $40, %rsp
 ; WIN64-NEXT:    .seh_stackalloc 40
 ; WIN64-NEXT:    vmovaps %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; WIN64-NEXT:    .seh_savexmm 7, 16
+; WIN64-NEXT:    .seh_savexmm %xmm7, 16
 ; WIN64-NEXT:    vmovaps %xmm6, (%rsp) # 16-byte Spill
-; WIN64-NEXT:    .seh_savexmm 6, 0
+; WIN64-NEXT:    .seh_savexmm %xmm6, 0
 ; WIN64-NEXT:    .seh_endprologue
 ; WIN64-NEXT:    movl $1, %eax
 ; WIN64-NEXT:    movl $1, %ecx
@@ -476,15 +476,15 @@ define i32 @caller_retv32i1() #0 {
 ; WIN64-LABEL: caller_retv32i1:
 ; WIN64:       # %bb.0: # %entry
 ; WIN64-NEXT:    pushq %rsi
-; WIN64-NEXT:    .seh_pushreg 6
+; WIN64-NEXT:    .seh_pushreg %rsi
 ; WIN64-NEXT:    pushq %rdi
-; WIN64-NEXT:    .seh_pushreg 7
+; WIN64-NEXT:    .seh_pushreg %rdi
 ; WIN64-NEXT:    subq $40, %rsp
 ; WIN64-NEXT:    .seh_stackalloc 40
 ; WIN64-NEXT:    vmovaps %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; WIN64-NEXT:    .seh_savexmm 7, 16
+; WIN64-NEXT:    .seh_savexmm %xmm7, 16
 ; WIN64-NEXT:    vmovaps %xmm6, (%rsp) # 16-byte Spill
-; WIN64-NEXT:    .seh_savexmm 6, 0
+; WIN64-NEXT:    .seh_savexmm %xmm6, 0
 ; WIN64-NEXT:    .seh_endprologue
 ; WIN64-NEXT:    callq test_retv32i1
 ; WIN64-NEXT:    incl %eax
@@ -547,11 +547,11 @@ define x86_regcallcc i16 @test_argv16i1(<16 x i1> %x0, <16 x i1> %x1, <16 x i1>
 ; WIN64-LABEL: test_argv16i1:
 ; WIN64:       # %bb.0:
 ; WIN64-NEXT:    pushq %r11
-; WIN64-NEXT:    .seh_pushreg 11
+; WIN64-NEXT:    .seh_pushreg %r11
 ; WIN64-NEXT:    pushq %r10
-; WIN64-NEXT:    .seh_pushreg 10
+; WIN64-NEXT:    .seh_pushreg %r10
 ; WIN64-NEXT:    pushq %rsp
-; WIN64-NEXT:    .seh_pushreg 4
+; WIN64-NEXT:    .seh_pushreg %rsp
 ; WIN64-NEXT:    subq $32, %rsp
 ; WIN64-NEXT:    .seh_stackalloc 32
 ; WIN64-NEXT:    .seh_endprologue
@@ -640,15 +640,15 @@ define i16 @caller_argv16i1() #0 {
 ; WIN64-LABEL: caller_argv16i1:
 ; WIN64:       # %bb.0: # %entry
 ; WIN64-NEXT:    pushq %rsi
-; WIN64-NEXT:    .seh_pushreg 6
+; WIN64-NEXT:    .seh_pushreg %rsi
 ; WIN64-NEXT:    pushq %rdi
-; WIN64-NEXT:    .seh_pushreg 7
+; WIN64-NEXT:    .seh_pushreg %rdi
 ; WIN64-NEXT:    subq $40, %rsp
 ; WIN64-NEXT:    .seh_stackalloc 40
 ; WIN64-NEXT:    vmovaps %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; WIN64-NEXT:    .seh_savexmm 7, 16
+; WIN64-NEXT:    .seh_savexmm %xmm7, 16
 ; WIN64-NEXT:    vmovaps %xmm6, (%rsp) # 16-byte Spill
-; WIN64-NEXT:    .seh_savexmm 6, 0
+; WIN64-NEXT:    .seh_savexmm %xmm6, 0
 ; WIN64-NEXT:    .seh_endprologue
 ; WIN64-NEXT:    movl $1, %eax
 ; WIN64-NEXT:    movl $1, %ecx
@@ -709,15 +709,15 @@ define i16 @caller_retv16i1() #0 {
 ; WIN64-LABEL: caller_retv16i1:
 ; WIN64:       # %bb.0: # %entry
 ; WIN64-NEXT:    pushq %rsi
-; WIN64-NEXT:    .seh_pushreg 6
+; WIN64-NEXT:    .seh_pushreg %rsi
 ; WIN64-NEXT:    pushq %rdi
-; WIN64-NEXT:    .seh_pushreg 7
+; WIN64-NEXT:    .seh_pushreg %rdi
 ; WIN64-NEXT:    subq $40, %rsp
 ; WIN64-NEXT:    .seh_stackalloc 40
 ; WIN64-NEXT:    vmovaps %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; WIN64-NEXT:    .seh_savexmm 7, 16
+; WIN64-NEXT:    .seh_savexmm %xmm7, 16
 ; WIN64-NEXT:    vmovaps %xmm6, (%rsp) # 16-byte Spill
-; WIN64-NEXT:    .seh_savexmm 6, 0
+; WIN64-NEXT:    .seh_savexmm %xmm6, 0
 ; WIN64-NEXT:    .seh_endprologue
 ; WIN64-NEXT:    callq test_retv16i1
 ; WIN64-NEXT:    # kill: def $ax killed $ax def $eax
@@ -784,11 +784,11 @@ define x86_regcallcc i8 @test_argv8i1(<8 x i1> %x0, <8 x i1> %x1, <8 x i1> %x2)
 ; WIN64-LABEL: test_argv8i1:
 ; WIN64:       # %bb.0:
 ; WIN64-NEXT:    pushq %r11
-; WIN64-NEXT:    .seh_pushreg 11
+; WIN64-NEXT:    .seh_pushreg %r11
 ; WIN64-NEXT:    pushq %r10
-; WIN64-NEXT:    .seh_pushreg 10
+; WIN64-NEXT:    .seh_pushreg %r10
 ; WIN64-NEXT:    pushq %rsp
-; WIN64-NEXT:    .seh_pushreg 4
+; WIN64-NEXT:    .seh_pushreg %rsp
 ; WIN64-NEXT:    subq $32, %rsp
 ; WIN64-NEXT:    .seh_stackalloc 32
 ; WIN64-NEXT:    .seh_endprologue
@@ -877,15 +877,15 @@ define i8 @caller_argv8i1() #0 {
 ; WIN64-LABEL: caller_argv8i1:
 ; WIN64:       # %bb.0: # %entry
 ; WIN64-NEXT:    pushq %rsi
-; WIN64-NEXT:    .seh_pushreg 6
+; WIN64-NEXT:    .seh_pushreg %rsi
 ; WIN64-NEXT:    pushq %rdi
-; WIN64-NEXT:    .seh_pushreg 7
+; WIN64-NEXT:    .seh_pushreg %rdi
 ; WIN64-NEXT:    subq $40, %rsp
 ; WIN64-NEXT:    .seh_stackalloc 40
 ; WIN64-NEXT:    vmovaps %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; WIN64-NEXT:    .seh_savexmm 7, 16
+; WIN64-NEXT:    .seh_savexmm %xmm7, 16
 ; WIN64-NEXT:    vmovaps %xmm6, (%rsp) # 16-byte Spill
-; WIN64-NEXT:    .seh_savexmm 6, 0
+; WIN64-NEXT:    .seh_savexmm %xmm6, 0
 ; WIN64-NEXT:    .seh_endprologue
 ; WIN64-NEXT:    movl $1, %eax
 ; WIN64-NEXT:    movl $1, %ecx
@@ -948,15 +948,15 @@ define <8 x i1> @caller_retv8i1() #0 {
 ; WIN64-LABEL: caller_retv8i1:
 ; WIN64:       # %bb.0: # %entry
 ; WIN64-NEXT:    pushq %rsi
-; WIN64-NEXT:    .seh_pushreg 6
+; WIN64-NEXT:    .seh_pushreg %rsi
 ; WIN64-NEXT:    pushq %rdi
-; WIN64-NEXT:    .seh_pushreg 7
+; WIN64-NEXT:    .seh_pushreg %rdi
 ; WIN64-NEXT:    subq $40, %rsp
 ; WIN64-NEXT:    .seh_stackalloc 40
 ; WIN64-NEXT:    vmovaps %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; WIN64-NEXT:    .seh_savexmm 7, 16
+; WIN64-NEXT:    .seh_savexmm %xmm7, 16
 ; WIN64-NEXT:    vmovaps %xmm6, (%rsp) # 16-byte Spill
-; WIN64-NEXT:    .seh_savexmm 6, 0
+; WIN64-NEXT:    .seh_savexmm %xmm6, 0
 ; WIN64-NEXT:    .seh_endprologue
 ; WIN64-NEXT:    callq test_retv8i1
 ; WIN64-NEXT:    # kill: def $al killed $al def $eax
diff --git a/llvm/test/CodeGen/X86/avx512-regcall-NoMask.ll b/llvm/test/CodeGen/X86/avx512-regcall-NoMask.ll
index c03a8f6ab..e832e42fe 100644
--- a/llvm/test/CodeGen/X86/avx512-regcall-NoMask.ll
+++ b/llvm/test/CodeGen/X86/avx512-regcall-NoMask.ll
@@ -1,8 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; FIXME: Fix machine verifier issues and remove -verify-machineinstrs=0. PR39437.
-; RUN: llc < %s -mtriple=i386-pc-win32       -mattr=+avx512f -mattr=+avx512vl -mattr=+avx512bw -mattr=+avx512dq -verify-machineinstrs=0  | FileCheck %s --check-prefix=X32
-; RUN: llc < %s -mtriple=x86_64-win32        -mattr=+avx512f -mattr=+avx512vl -mattr=+avx512bw -mattr=+avx512dq -verify-machineinstrs=0  | FileCheck %s --check-prefix=WIN64
-; RUN: llc < %s -mtriple=x86_64-linux-gnu    -mattr=+avx512f -mattr=+avx512vl -mattr=+avx512bw -mattr=+avx512dq -verify-machineinstrs=0  | FileCheck %s --check-prefix=LINUXOSX64
+; RUN: llc < %s -mtriple=i386-pc-win32       -mattr=+avx512f -mattr=+avx512vl -mattr=+avx512bw -mattr=+avx512dq -verify-machineinstrs  | FileCheck %s --check-prefix=X32
+; RUN: llc < %s -mtriple=x86_64-win32        -mattr=+avx512f -mattr=+avx512vl -mattr=+avx512bw -mattr=+avx512dq -verify-machineinstrs  | FileCheck %s --check-prefix=WIN64
+; RUN: llc < %s -mtriple=x86_64-linux-gnu    -mattr=+avx512f -mattr=+avx512vl -mattr=+avx512bw -mattr=+avx512dq -verify-machineinstrs  | FileCheck %s --check-prefix=LINUXOSX64
 
 ; Test regcall when receiving/returning i1
 define x86_regcallcc i1 @test_argReti1(i1 %a)  {
@@ -42,7 +41,7 @@ define x86_regcallcc i1 @test_CallargReti1(i1 %a)  {
 ; WIN64-LABEL: test_CallargReti1:
 ; WIN64:       # %bb.0:
 ; WIN64-NEXT:    pushq %rsp
-; WIN64-NEXT:    .seh_pushreg 4
+; WIN64-NEXT:    .seh_pushreg %rsp
 ; WIN64-NEXT:    .seh_endprologue
 ; WIN64-NEXT:    incb %al
 ; WIN64-NEXT:    movzbl %al, %eax
@@ -110,7 +109,7 @@ define x86_regcallcc i8 @test_CallargReti8(i8 %a)  {
 ; WIN64-LABEL: test_CallargReti8:
 ; WIN64:       # %bb.0:
 ; WIN64-NEXT:    pushq %rsp
-; WIN64-NEXT:    .seh_pushreg 4
+; WIN64-NEXT:    .seh_pushreg %rsp
 ; WIN64-NEXT:    .seh_endprologue
 ; WIN64-NEXT:    incb %al
 ; WIN64-NEXT:    movzbl %al, %eax
@@ -179,7 +178,7 @@ define x86_regcallcc i16 @test_CallargReti16(i16 %a)  {
 ; WIN64-LABEL: test_CallargReti16:
 ; WIN64:       # %bb.0:
 ; WIN64-NEXT:    pushq %rsp
-; WIN64-NEXT:    .seh_pushreg 4
+; WIN64-NEXT:    .seh_pushreg %rsp
 ; WIN64-NEXT:    .seh_endprologue
 ; WIN64-NEXT:    incl %eax
 ; WIN64-NEXT:    callq test_argReti16
@@ -245,7 +244,7 @@ define x86_regcallcc i32 @test_CallargReti32(i32 %a)  {
 ; WIN64-LABEL: test_CallargReti32:
 ; WIN64:       # %bb.0:
 ; WIN64-NEXT:    pushq %rsp
-; WIN64-NEXT:    .seh_pushreg 4
+; WIN64-NEXT:    .seh_pushreg %rsp
 ; WIN64-NEXT:    .seh_endprologue
 ; WIN64-NEXT:    incl %eax
 ; WIN64-NEXT:    callq test_argReti32
@@ -312,7 +311,7 @@ define x86_regcallcc i64 @test_CallargReti64(i64 %a)  {
 ; WIN64-LABEL: test_CallargReti64:
 ; WIN64:       # %bb.0:
 ; WIN64-NEXT:    pushq %rsp
-; WIN64-NEXT:    .seh_pushreg 4
+; WIN64-NEXT:    .seh_pushreg %rsp
 ; WIN64-NEXT:    .seh_endprologue
 ; WIN64-NEXT:    incq %rax
 ; WIN64-NEXT:    callq test_argReti64
@@ -379,16 +378,16 @@ define x86_regcallcc float @test_CallargRetFloat(float %a)  {
 ; WIN64-LABEL: test_CallargRetFloat:
 ; WIN64:       # %bb.0:
 ; WIN64-NEXT:    pushq %rsp
-; WIN64-NEXT:    .seh_pushreg 4
+; WIN64-NEXT:    .seh_pushreg %rsp
 ; WIN64-NEXT:    subq $16, %rsp
 ; WIN64-NEXT:    .seh_stackalloc 16
 ; WIN64-NEXT:    vmovaps %xmm8, (%rsp) # 16-byte Spill
-; WIN64-NEXT:    .seh_savexmm 8, 0
+; WIN64-NEXT:    .seh_savexmm %xmm8, 0
 ; WIN64-NEXT:    .seh_endprologue
 ; WIN64-NEXT:    vmovss {{.*#+}} xmm8 = mem[0],zero,zero,zero
-; WIN64-NEXT:    vaddss %xmm8, %xmm0, %xmm0
+; WIN64-NEXT:    vaddss %xmm0, %xmm8, %xmm0
 ; WIN64-NEXT:    callq test_argRetFloat
-; WIN64-NEXT:    vaddss %xmm8, %xmm0, %xmm0
+; WIN64-NEXT:    vaddss %xmm0, %xmm8, %xmm0
 ; WIN64-NEXT:    vmovaps (%rsp), %xmm8 # 16-byte Reload
 ; WIN64-NEXT:    addq $16, %rsp
 ; WIN64-NEXT:    popq %rsp
@@ -407,9 +406,9 @@ define x86_regcallcc float @test_CallargRetFloat(float %a)  {
 ; LINUXOSX64-NEXT:    .cfi_offset %rsp, -16
 ; LINUXOSX64-NEXT:    .cfi_offset %xmm8, -32
 ; LINUXOSX64-NEXT:    vmovss {{.*#+}} xmm8 = mem[0],zero,zero,zero
-; LINUXOSX64-NEXT:    vaddss %xmm8, %xmm0, %xmm0
+; LINUXOSX64-NEXT:    vaddss %xmm0, %xmm8, %xmm0
 ; LINUXOSX64-NEXT:    callq test_argRetFloat
-; LINUXOSX64-NEXT:    vaddss %xmm8, %xmm0, %xmm0
+; LINUXOSX64-NEXT:    vaddss %xmm0, %xmm8, %xmm0
 ; LINUXOSX64-NEXT:    vmovaps (%rsp), %xmm8 # 16-byte Reload
 ; LINUXOSX64-NEXT:    addq $16, %rsp
 ; LINUXOSX64-NEXT:    .cfi_def_cfa_offset 16
@@ -461,16 +460,16 @@ define x86_regcallcc double @test_CallargRetDouble(double %a)  {
 ; WIN64-LABEL: test_CallargRetDouble:
 ; WIN64:       # %bb.0:
 ; WIN64-NEXT:    pushq %rsp
-; WIN64-NEXT:    .seh_pushreg 4
+; WIN64-NEXT:    .seh_pushreg %rsp
 ; WIN64-NEXT:    subq $16, %rsp
 ; WIN64-NEXT:    .seh_stackalloc 16
 ; WIN64-NEXT:    vmovaps %xmm8, (%rsp) # 16-byte Spill
-; WIN64-NEXT:    .seh_savexmm 8, 0
+; WIN64-NEXT:    .seh_savexmm %xmm8, 0
 ; WIN64-NEXT:    .seh_endprologue
 ; WIN64-NEXT:    vmovsd {{.*#+}} xmm8 = mem[0],zero
-; WIN64-NEXT:    vaddsd %xmm8, %xmm0, %xmm0
+; WIN64-NEXT:    vaddsd %xmm0, %xmm8, %xmm0
 ; WIN64-NEXT:    callq test_argRetDouble
-; WIN64-NEXT:    vaddsd %xmm8, %xmm0, %xmm0
+; WIN64-NEXT:    vaddsd %xmm0, %xmm8, %xmm0
 ; WIN64-NEXT:    vmovaps (%rsp), %xmm8 # 16-byte Reload
 ; WIN64-NEXT:    addq $16, %rsp
 ; WIN64-NEXT:    popq %rsp
@@ -489,9 +488,9 @@ define x86_regcallcc double @test_CallargRetDouble(double %a)  {
 ; LINUXOSX64-NEXT:    .cfi_offset %rsp, -16
 ; LINUXOSX64-NEXT:    .cfi_offset %xmm8, -32
 ; LINUXOSX64-NEXT:    vmovsd {{.*#+}} xmm8 = mem[0],zero
-; LINUXOSX64-NEXT:    vaddsd %xmm8, %xmm0, %xmm0
+; LINUXOSX64-NEXT:    vaddsd %xmm0, %xmm8, %xmm0
 ; LINUXOSX64-NEXT:    callq test_argRetDouble
-; LINUXOSX64-NEXT:    vaddsd %xmm8, %xmm0, %xmm0
+; LINUXOSX64-NEXT:    vaddsd %xmm0, %xmm8, %xmm0
 ; LINUXOSX64-NEXT:    vmovaps (%rsp), %xmm8 # 16-byte Reload
 ; LINUXOSX64-NEXT:    addq $16, %rsp
 ; LINUXOSX64-NEXT:    .cfi_def_cfa_offset 16
@@ -524,6 +523,37 @@ define x86_regcallcc x86_fp80 @test_argRetf80(x86_fp80 %a0) nounwind {
   ret x86_fp80 %r0
 }
 
+; Test regcall when receiving/returning long double
+define x86_regcallcc double @test_argParamf80(x86_fp80 %a0) nounwind {
+; X32-LABEL: test_argParamf80:
+; X32:       # %bb.0:
+; X32-NEXT:    pushl %ebp
+; X32-NEXT:    movl %esp, %ebp
+; X32-NEXT:    andl $-8, %esp
+; X32-NEXT:    subl $8, %esp
+; X32-NEXT:    fstpl (%esp)
+; X32-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X32-NEXT:    movl %ebp, %esp
+; X32-NEXT:    popl %ebp
+; X32-NEXT:    retl
+;
+; WIN64-LABEL: test_argParamf80:
+; WIN64:       # %bb.0:
+; WIN64-NEXT:    pushq %rax
+; WIN64-NEXT:    fstpl (%rsp)
+; WIN64-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; WIN64-NEXT:    popq %rax
+; WIN64-NEXT:    retq
+;
+; LINUXOSX64-LABEL: test_argParamf80:
+; LINUXOSX64:       # %bb.0:
+; LINUXOSX64-NEXT:    fstpl -{{[0-9]+}}(%rsp)
+; LINUXOSX64-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; LINUXOSX64-NEXT:    retq
+  %r0 = fptrunc x86_fp80 %a0 to double
+  ret double %r0
+}
+
 ; Test regcall when passing/retrieving long double
 define x86_regcallcc x86_fp80 @test_CallargRetf80(x86_fp80 %a)  {
 ; X32-LABEL: test_CallargRetf80:
@@ -538,7 +568,7 @@ define x86_regcallcc x86_fp80 @test_CallargRetf80(x86_fp80 %a)  {
 ; WIN64-LABEL: test_CallargRetf80:
 ; WIN64:       # %bb.0:
 ; WIN64-NEXT:    pushq %rsp
-; WIN64-NEXT:    .seh_pushreg 4
+; WIN64-NEXT:    .seh_pushreg %rsp
 ; WIN64-NEXT:    .seh_endprologue
 ; WIN64-NEXT:    fadd %st, %st(0)
 ; WIN64-NEXT:    callq test_argRetf80
@@ -566,6 +596,47 @@ define x86_regcallcc x86_fp80 @test_CallargRetf80(x86_fp80 %a)  {
   ret x86_fp80 %d
 }
 
+define x86_regcallcc double @test_CallargParamf80(x86_fp80 %a)  {
+; X32-LABEL: test_CallargParamf80:
+; X32:       # %bb.0:
+; X32-NEXT:    pushl %esp
+; X32-NEXT:    fadd %st, %st(0)
+; X32-NEXT:    calll _test_argParamf80
+; X32-NEXT:    vaddsd %xmm0, %xmm0, %xmm0
+; X32-NEXT:    popl %esp
+; X32-NEXT:    retl
+;
+; WIN64-LABEL: test_CallargParamf80:
+; WIN64:       # %bb.0:
+; WIN64-NEXT:    pushq %rsp
+; WIN64-NEXT:    .seh_pushreg %rsp
+; WIN64-NEXT:    .seh_endprologue
+; WIN64-NEXT:    fadd %st, %st(0)
+; WIN64-NEXT:    callq test_argParamf80
+; WIN64-NEXT:    vaddsd %xmm0, %xmm0, %xmm0
+; WIN64-NEXT:    popq %rsp
+; WIN64-NEXT:    retq
+; WIN64-NEXT:    .seh_handlerdata
+; WIN64-NEXT:    .text
+; WIN64-NEXT:    .seh_endproc
+;
+; LINUXOSX64-LABEL: test_CallargParamf80:
+; LINUXOSX64:       # %bb.0:
+; LINUXOSX64-NEXT:    pushq %rsp
+; LINUXOSX64-NEXT:    .cfi_def_cfa_offset 16
+; LINUXOSX64-NEXT:    .cfi_offset %rsp, -16
+; LINUXOSX64-NEXT:    fadd %st, %st(0)
+; LINUXOSX64-NEXT:    callq test_argParamf80
+; LINUXOSX64-NEXT:    vaddsd %xmm0, %xmm0, %xmm0
+; LINUXOSX64-NEXT:    popq %rsp
+; LINUXOSX64-NEXT:    .cfi_def_cfa_offset 8
+; LINUXOSX64-NEXT:    retq
+  %b = fadd x86_fp80 %a, %a
+  %c = call x86_regcallcc double @test_argParamf80(x86_fp80 %b)
+  %d = fadd double %c, %c
+  ret double %d
+}
+
 ; Test regcall when receiving/returning pointer
 define x86_regcallcc [4 x i32]* @test_argRetPointer([4 x i32]* %a)  {
 ; X32-LABEL: test_argRetPointer:
@@ -602,7 +673,7 @@ define x86_regcallcc [4 x i32]* @test_CallargRetPointer([4 x i32]* %a)  {
 ; WIN64-LABEL: test_CallargRetPointer:
 ; WIN64:       # %bb.0:
 ; WIN64-NEXT:    pushq %rsp
-; WIN64-NEXT:    .seh_pushreg 4
+; WIN64-NEXT:    .seh_pushreg %rsp
 ; WIN64-NEXT:    .seh_endprologue
 ; WIN64-NEXT:    incl %eax
 ; WIN64-NEXT:    callq test_argRetPointer
@@ -684,11 +755,11 @@ define x86_regcallcc <4 x i32> @test_CallargRet128Vector(<4 x i1> %x, <4 x i32>
 ; WIN64-LABEL: test_CallargRet128Vector:
 ; WIN64:       # %bb.0:
 ; WIN64-NEXT:    pushq %rsp
-; WIN64-NEXT:    .seh_pushreg 4
+; WIN64-NEXT:    .seh_pushreg %rsp
 ; WIN64-NEXT:    subq $32, %rsp
 ; WIN64-NEXT:    .seh_stackalloc 32
 ; WIN64-NEXT:    vmovaps %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; WIN64-NEXT:    .seh_savexmm 8, 16
+; WIN64-NEXT:    .seh_savexmm %xmm8, 16
 ; WIN64-NEXT:    .seh_endprologue
 ; WIN64-NEXT:    vmovdqa %xmm1, %xmm8
 ; WIN64-NEXT:    vpslld $31, %xmm0, %xmm1
@@ -780,7 +851,7 @@ define x86_regcallcc <8 x i32> @test_CallargRet256Vector(<8 x i1> %x, <8 x i32>
 ; WIN64-LABEL: test_CallargRet256Vector:
 ; WIN64:       # %bb.0:
 ; WIN64-NEXT:    pushq %rsp
-; WIN64-NEXT:    .seh_pushreg 4
+; WIN64-NEXT:    .seh_pushreg %rsp
 ; WIN64-NEXT:    subq $80, %rsp
 ; WIN64-NEXT:    .seh_stackalloc 80
 ; WIN64-NEXT:    .seh_endprologue
@@ -868,7 +939,7 @@ define x86_regcallcc <16 x i32> @test_CallargRet512Vector(<16 x i1> %x, <16 x i3
 ; WIN64-LABEL: test_CallargRet512Vector:
 ; WIN64:       # %bb.0:
 ; WIN64-NEXT:    pushq %rsp
-; WIN64-NEXT:    .seh_pushreg 4
+; WIN64-NEXT:    .seh_pushreg %rsp
 ; WIN64-NEXT:    subq $176, %rsp
 ; WIN64-NEXT:    .seh_stackalloc 176
 ; WIN64-NEXT:    .seh_endprologue
diff --git a/llvm/test/CodeGen/X86/avx512-scalar_mask.ll b/llvm/test/CodeGen/X86/avx512-scalar_mask.ll
index 1a98bd958..27468fecf 100644
--- a/llvm/test/CodeGen/X86/avx512-scalar_mask.ll
+++ b/llvm/test/CodeGen/X86/avx512-scalar_mask.ll
@@ -8,7 +8,7 @@ define <4 x float>@test_var_mask(<4 x float> %v0, <4 x float> %v1, <4 x float> %
 ; CHECK-LABEL: test_var_mask:
 ; CHECK:       ## %bb.0:
 ; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vfmadd213ss %xmm2, %xmm1, %xmm0 {%k1}
+; CHECK-NEXT:    vfmadd213ss {{.*#+}} xmm0 = (xmm1 * xmm0) + xmm2
 ; CHECK-NEXT:    retq
   %res = call <4 x float> @llvm.x86.avx512.mask.vfmadd.ss(<4 x float> %v0,<4 x float> %v1, <4 x float> %v2,  i8 %mask, i32 4)
   ret < 4 x float> %res
@@ -18,7 +18,7 @@ define <4 x float>@test_var_maskz(<4 x float> %v0, <4 x float> %v1, <4 x float>
 ; CHECK-LABEL: test_var_maskz:
 ; CHECK:       ## %bb.0:
 ; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vfmadd213ss %xmm2, %xmm1, %xmm0 {%k1} {z}
+; CHECK-NEXT:    vfmadd213ss {{.*#+}} xmm0 = (xmm1 * xmm0) + xmm2
 ; CHECK-NEXT:    retq
   %res = call <4 x float> @llvm.x86.avx512.maskz.vfmadd.ss(<4 x float> %v0,<4 x float> %v1, <4 x float> %v2,  i8 %mask, i32 4)
   ret < 4 x float> %res
@@ -67,7 +67,7 @@ define <4 x float>@test_const2_maskz(<4 x float> %v0, <4 x float> %v1, <4 x floa
 define <4 x float>@test_const_allone_mask(<4 x float> %v0, <4 x float> %v1, <4 x float> %v2) {
 ; CHECK-LABEL: test_const_allone_mask:
 ; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vfmadd213ss %xmm2, %xmm1, %xmm0
+; CHECK-NEXT:    vfmadd213ss {{.*#+}} xmm0 = (xmm1 * xmm0) + xmm2
 ; CHECK-NEXT:    retq
   %res = call <4 x float> @llvm.x86.avx512.mask.vfmadd.ss(<4 x float> %v0,<4 x float> %v1, <4 x float> %v2,  i8 -1, i32 4)
   ret < 4 x float> %res
@@ -76,7 +76,7 @@ define <4 x float>@test_const_allone_mask(<4 x float> %v0, <4 x float> %v1, <4 x
 define <4 x float>@test_const_allone_maskz(<4 x float> %v0, <4 x float> %v1, <4 x float> %v2) {
 ; CHECK-LABEL: test_const_allone_maskz:
 ; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vfmadd213ss %xmm2, %xmm1, %xmm0
+; CHECK-NEXT:    vfmadd213ss {{.*#+}} xmm0 = (xmm1 * xmm0) + xmm2
 ; CHECK-NEXT:    retq
   %res = call <4 x float> @llvm.x86.avx512.maskz.vfmadd.ss(<4 x float> %v0,<4 x float> %v1, <4 x float> %v2,  i8 -1, i32 4)
   ret < 4 x float> %res
@@ -85,7 +85,7 @@ define <4 x float>@test_const_allone_maskz(<4 x float> %v0, <4 x float> %v1, <4
 define <4 x float>@test_const_3_mask(<4 x float> %v0, <4 x float> %v1, <4 x float> %v2) {
 ; CHECK-LABEL: test_const_3_mask:
 ; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vfmadd213ss %xmm2, %xmm1, %xmm0
+; CHECK-NEXT:    vfmadd213ss {{.*#+}} xmm0 = (xmm1 * xmm0) + xmm2
 ; CHECK-NEXT:    retq
   %res = call <4 x float> @llvm.x86.avx512.mask.vfmadd.ss(<4 x float> %v0,<4 x float> %v1, <4 x float> %v2,  i8 3, i32 4)
   ret < 4 x float> %res
@@ -94,7 +94,7 @@ define <4 x float>@test_const_3_mask(<4 x float> %v0, <4 x float> %v1, <4 x floa
 define <4 x float>@test_const_3_maskz(<4 x float> %v0, <4 x float> %v1, <4 x float> %v2) {
 ; CHECK-LABEL: test_const_3_maskz:
 ; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vfmadd213ss %xmm2, %xmm1, %xmm0
+; CHECK-NEXT:    vfmadd213ss {{.*#+}} xmm0 = (xmm1 * xmm0) + xmm2
 ; CHECK-NEXT:    retq
   %res = call <4 x float> @llvm.x86.avx512.maskz.vfmadd.ss(<4 x float> %v0,<4 x float> %v1, <4 x float> %v2,  i8 3, i32 4)
   ret < 4 x float> %res
diff --git a/llvm/test/CodeGen/X86/avx512-select.ll b/llvm/test/CodeGen/X86/avx512-select.ll
index 1ed7b408b..cd4a70c4f 100644
--- a/llvm/test/CodeGen/X86/avx512-select.ll
+++ b/llvm/test/CodeGen/X86/avx512-select.ll
@@ -74,7 +74,7 @@ define float @select02(float %a, float %b, float %c, float %eps) {
 ; X64-LABEL: select02:
 ; X64:       # %bb.0:
 ; X64-NEXT:    vcmpless %xmm0, %xmm3, %k1
-; X64-NEXT:    vmovss %xmm2, %xmm0, %xmm1 {%k1}
+; X64-NEXT:    vmovss %xmm2, %xmm1, %xmm1 {%k1}
 ; X64-NEXT:    vmovaps %xmm1, %xmm0
 ; X64-NEXT:    retq
   %cmp = fcmp oge float %a, %eps
@@ -96,7 +96,7 @@ define double @select03(double %a, double %b, double %c, double %eps) {
 ; X64-LABEL: select03:
 ; X64:       # %bb.0:
 ; X64-NEXT:    vcmplesd %xmm0, %xmm3, %k1
-; X64-NEXT:    vmovsd %xmm2, %xmm0, %xmm1 {%k1}
+; X64-NEXT:    vmovsd %xmm2, %xmm1, %xmm1 {%k1}
 ; X64-NEXT:    vmovapd %xmm1, %xmm0
 ; X64-NEXT:    retq
   %cmp = fcmp oge double %a, %eps
@@ -436,32 +436,30 @@ define <16 x i16> @pr31515(<16 x i1> %a, <16 x i1> %b, <16 x i16> %c) nounwind {
 define <32 x i16> @pr42355_v32i16(i1 %c, <32 x i16> %x, <32 x i16> %y) {
 ; X86-AVX512F-LABEL: pr42355_v32i16:
 ; X86-AVX512F:       # %bb.0:
-; X86-AVX512F-NEXT:    pushl %ebp
-; X86-AVX512F-NEXT:    .cfi_def_cfa_offset 8
-; X86-AVX512F-NEXT:    .cfi_offset %ebp, -8
-; X86-AVX512F-NEXT:    movl %esp, %ebp
-; X86-AVX512F-NEXT:    .cfi_def_cfa_register %ebp
-; X86-AVX512F-NEXT:    andl $-32, %esp
-; X86-AVX512F-NEXT:    subl $32, %esp
-; X86-AVX512F-NEXT:    testb $1, 8(%ebp)
-; X86-AVX512F-NEXT:    jne .LBB14_2
-; X86-AVX512F-NEXT:  # %bb.1:
-; X86-AVX512F-NEXT:    vmovaps 40(%ebp), %ymm1
-; X86-AVX512F-NEXT:    vmovaps %ymm2, %ymm0
-; X86-AVX512F-NEXT:  .LBB14_2:
-; X86-AVX512F-NEXT:    movl %ebp, %esp
-; X86-AVX512F-NEXT:    popl %ebp
-; X86-AVX512F-NEXT:    .cfi_def_cfa %esp, 4
+; X86-AVX512F-NEXT:    testb $1, {{[0-9]+}}(%esp)
+; X86-AVX512F-NEXT:    jne .LBB14_1
+; X86-AVX512F-NEXT:  # %bb.2:
+; X86-AVX512F-NEXT:    vextractf64x4 $1, %zmm1, %ymm2
+; X86-AVX512F-NEXT:    vmovaps %ymm1, %ymm0
+; X86-AVX512F-NEXT:    vinsertf64x4 $1, %ymm2, %zmm0, %zmm0
+; X86-AVX512F-NEXT:    retl
+; X86-AVX512F-NEXT:  .LBB14_1:
+; X86-AVX512F-NEXT:    vextractf64x4 $1, %zmm0, %ymm2
+; X86-AVX512F-NEXT:    vinsertf64x4 $1, %ymm2, %zmm0, %zmm0
 ; X86-AVX512F-NEXT:    retl
 ;
 ; X64-AVX512F-LABEL: pr42355_v32i16:
 ; X64-AVX512F:       # %bb.0:
 ; X64-AVX512F-NEXT:    testb $1, %dil
-; X64-AVX512F-NEXT:    jne .LBB14_2
-; X64-AVX512F-NEXT:  # %bb.1:
-; X64-AVX512F-NEXT:    vmovaps %ymm2, %ymm0
-; X64-AVX512F-NEXT:    vmovaps %ymm3, %ymm1
-; X64-AVX512F-NEXT:  .LBB14_2:
+; X64-AVX512F-NEXT:    jne .LBB14_1
+; X64-AVX512F-NEXT:  # %bb.2:
+; X64-AVX512F-NEXT:    vextractf64x4 $1, %zmm1, %ymm2
+; X64-AVX512F-NEXT:    vmovaps %ymm1, %ymm0
+; X64-AVX512F-NEXT:    vinsertf64x4 $1, %ymm2, %zmm0, %zmm0
+; X64-AVX512F-NEXT:    retq
+; X64-AVX512F-NEXT:  .LBB14_1:
+; X64-AVX512F-NEXT:    vextractf64x4 $1, %zmm0, %ymm2
+; X64-AVX512F-NEXT:    vinsertf64x4 $1, %ymm2, %zmm0, %zmm0
 ; X64-AVX512F-NEXT:    retq
 ;
 ; X86-AVX512BW-LABEL: pr42355_v32i16:
@@ -488,32 +486,30 @@ define <32 x i16> @pr42355_v32i16(i1 %c, <32 x i16> %x, <32 x i16> %y) {
 define <64 x i8> @pr42355_v64i8(i1 %c, <64 x i8> %x, <64 x i8> %y) {
 ; X86-AVX512F-LABEL: pr42355_v64i8:
 ; X86-AVX512F:       # %bb.0:
-; X86-AVX512F-NEXT:    pushl %ebp
-; X86-AVX512F-NEXT:    .cfi_def_cfa_offset 8
-; X86-AVX512F-NEXT:    .cfi_offset %ebp, -8
-; X86-AVX512F-NEXT:    movl %esp, %ebp
-; X86-AVX512F-NEXT:    .cfi_def_cfa_register %ebp
-; X86-AVX512F-NEXT:    andl $-32, %esp
-; X86-AVX512F-NEXT:    subl $32, %esp
-; X86-AVX512F-NEXT:    testb $1, 8(%ebp)
-; X86-AVX512F-NEXT:    jne .LBB15_2
-; X86-AVX512F-NEXT:  # %bb.1:
-; X86-AVX512F-NEXT:    vmovaps 40(%ebp), %ymm1
-; X86-AVX512F-NEXT:    vmovaps %ymm2, %ymm0
-; X86-AVX512F-NEXT:  .LBB15_2:
-; X86-AVX512F-NEXT:    movl %ebp, %esp
-; X86-AVX512F-NEXT:    popl %ebp
-; X86-AVX512F-NEXT:    .cfi_def_cfa %esp, 4
+; X86-AVX512F-NEXT:    testb $1, {{[0-9]+}}(%esp)
+; X86-AVX512F-NEXT:    jne .LBB15_1
+; X86-AVX512F-NEXT:  # %bb.2:
+; X86-AVX512F-NEXT:    vextractf64x4 $1, %zmm1, %ymm2
+; X86-AVX512F-NEXT:    vmovaps %ymm1, %ymm0
+; X86-AVX512F-NEXT:    vinsertf64x4 $1, %ymm2, %zmm0, %zmm0
+; X86-AVX512F-NEXT:    retl
+; X86-AVX512F-NEXT:  .LBB15_1:
+; X86-AVX512F-NEXT:    vextractf64x4 $1, %zmm0, %ymm2
+; X86-AVX512F-NEXT:    vinsertf64x4 $1, %ymm2, %zmm0, %zmm0
 ; X86-AVX512F-NEXT:    retl
 ;
 ; X64-AVX512F-LABEL: pr42355_v64i8:
 ; X64-AVX512F:       # %bb.0:
 ; X64-AVX512F-NEXT:    testb $1, %dil
-; X64-AVX512F-NEXT:    jne .LBB15_2
-; X64-AVX512F-NEXT:  # %bb.1:
-; X64-AVX512F-NEXT:    vmovaps %ymm2, %ymm0
-; X64-AVX512F-NEXT:    vmovaps %ymm3, %ymm1
-; X64-AVX512F-NEXT:  .LBB15_2:
+; X64-AVX512F-NEXT:    jne .LBB15_1
+; X64-AVX512F-NEXT:  # %bb.2:
+; X64-AVX512F-NEXT:    vextractf64x4 $1, %zmm1, %ymm2
+; X64-AVX512F-NEXT:    vmovaps %ymm1, %ymm0
+; X64-AVX512F-NEXT:    vinsertf64x4 $1, %ymm2, %zmm0, %zmm0
+; X64-AVX512F-NEXT:    retq
+; X64-AVX512F-NEXT:  .LBB15_1:
+; X64-AVX512F-NEXT:    vextractf64x4 $1, %zmm0, %ymm2
+; X64-AVX512F-NEXT:    vinsertf64x4 $1, %ymm2, %zmm0, %zmm0
 ; X64-AVX512F-NEXT:    retq
 ;
 ; X86-AVX512BW-LABEL: pr42355_v64i8:
diff --git a/llvm/test/CodeGen/X86/avx512-shuffles/partial_permute.ll b/llvm/test/CodeGen/X86/avx512-shuffles/partial_permute.ll
index b3e154b31..ba06a7a13 100644
--- a/llvm/test/CodeGen/X86/avx512-shuffles/partial_permute.ll
+++ b/llvm/test/CodeGen/X86/avx512-shuffles/partial_permute.ll
@@ -225,12 +225,11 @@ define <8 x i16> @test_masked_z_16xi16_to_8xi16_perm_mem_mask1(<16 x i16>* %vp,
 define <8 x i16> @test_masked_16xi16_to_8xi16_perm_mem_mask2(<16 x i16>* %vp, <8 x i16> %vec2, <8 x i16> %mask) {
 ; CHECK-LABEL: test_masked_16xi16_to_8xi16_perm_mem_mask2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpbroadcastw 2(%rdi), %xmm2
-; CHECK-NEXT:    vmovdqa 16(%rdi), %xmm3
-; CHECK-NEXT:    vmovdqa {{.*#+}} xmm4 = [8,0,3,0,5,0,7,1]
-; CHECK-NEXT:    vpermi2w %xmm2, %xmm3, %xmm4
+; CHECK-NEXT:    vmovdqa 16(%rdi), %xmm2
+; CHECK-NEXT:    vmovdqa {{.*#+}} xmm3 = [9,0,3,0,5,0,7,1]
+; CHECK-NEXT:    vpermi2w (%rdi), %xmm2, %xmm3
 ; CHECK-NEXT:    vptestnmw %xmm1, %xmm1, %k1
-; CHECK-NEXT:    vmovdqu16 %xmm4, %xmm0 {%k1}
+; CHECK-NEXT:    vmovdqu16 %xmm3, %xmm0 {%k1}
 ; CHECK-NEXT:    retq
   %vec = load <16 x i16>, <16 x i16>* %vp
   %shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <8 x i32> <i32 1, i32 8, i32 11, i32 8, i32 13, i32 8, i32 15, i32 9>
@@ -242,11 +241,10 @@ define <8 x i16> @test_masked_16xi16_to_8xi16_perm_mem_mask2(<16 x i16>* %vp, <8
 define <8 x i16> @test_masked_z_16xi16_to_8xi16_perm_mem_mask2(<16 x i16>* %vp, <8 x i16> %mask) {
 ; CHECK-LABEL: test_masked_z_16xi16_to_8xi16_perm_mem_mask2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpbroadcastw 2(%rdi), %xmm2
-; CHECK-NEXT:    vmovdqa 16(%rdi), %xmm3
-; CHECK-NEXT:    vmovdqa {{.*#+}} xmm1 = [8,0,3,0,5,0,7,1]
+; CHECK-NEXT:    vmovdqa 16(%rdi), %xmm2
+; CHECK-NEXT:    vmovdqa {{.*#+}} xmm1 = [9,0,3,0,5,0,7,1]
 ; CHECK-NEXT:    vptestnmw %xmm0, %xmm0, %k1
-; CHECK-NEXT:    vpermi2w %xmm2, %xmm3, %xmm1 {%k1} {z}
+; CHECK-NEXT:    vpermi2w (%rdi), %xmm2, %xmm1 {%k1} {z}
 ; CHECK-NEXT:    vmovdqa %xmm1, %xmm0
 ; CHECK-NEXT:    retq
   %vec = load <16 x i16>, <16 x i16>* %vp
@@ -3768,10 +3766,10 @@ define <4 x double> @test_masked_z_8xdouble_to_4xdouble_perm_mask2(<8 x double>
 define <4 x double> @test_8xdouble_to_4xdouble_perm_mask3(<8 x double> %vec) {
 ; CHECK-LABEL: test_8xdouble_to_4xdouble_perm_mask3:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vextractf32x4 $2, %zmm0, %xmm2
-; CHECK-NEXT:    vmovapd {{.*#+}} ymm1 = [0,2,1,4]
-; CHECK-NEXT:    vpermi2pd %ymm2, %ymm0, %ymm1
-; CHECK-NEXT:    vmovapd %ymm1, %ymm0
+; CHECK-NEXT:    vbroadcasti64x4 {{.*#+}} zmm1 = [0,2,1,4,0,2,1,4]
+; CHECK-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3]
+; CHECK-NEXT:    vpermq %zmm0, %zmm1, %zmm0
+; CHECK-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; CHECK-NEXT:    retq
   %res = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 0, i32 2, i32 1, i32 4>
   ret <4 x double> %res
@@ -3779,12 +3777,12 @@ define <4 x double> @test_8xdouble_to_4xdouble_perm_mask3(<8 x double> %vec) {
 define <4 x double> @test_masked_8xdouble_to_4xdouble_perm_mask3(<8 x double> %vec, <4 x double> %vec2, <4 x double> %mask) {
 ; CHECK-LABEL: test_masked_8xdouble_to_4xdouble_perm_mask3:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vextractf32x4 $2, %zmm0, %xmm3
-; CHECK-NEXT:    vmovapd {{.*#+}} ymm4 = [0,2,1,4]
-; CHECK-NEXT:    vpermi2pd %ymm3, %ymm0, %ymm4
-; CHECK-NEXT:    vxorpd %xmm0, %xmm0, %xmm0
-; CHECK-NEXT:    vcmpeqpd %ymm0, %ymm2, %k1
-; CHECK-NEXT:    vblendmpd %ymm4, %ymm1, %ymm0 {%k1}
+; CHECK-NEXT:    vbroadcasti64x4 {{.*#+}} zmm3 = [0,2,1,4,0,2,1,4]
+; CHECK-NEXT:    # zmm3 = mem[0,1,2,3,0,1,2,3]
+; CHECK-NEXT:    vpermq %zmm0, %zmm3, %zmm0
+; CHECK-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; CHECK-NEXT:    vcmpeqpd %ymm3, %ymm2, %k1
+; CHECK-NEXT:    vblendmpd %ymm0, %ymm1, %ymm0 {%k1}
 ; CHECK-NEXT:    retq
   %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 0, i32 2, i32 1, i32 4>
   %cmp = fcmp oeq <4 x double> %mask, zeroinitializer
@@ -3795,12 +3793,11 @@ define <4 x double> @test_masked_8xdouble_to_4xdouble_perm_mask3(<8 x double> %v
 define <4 x double> @test_masked_z_8xdouble_to_4xdouble_perm_mask3(<8 x double> %vec, <4 x double> %mask) {
 ; CHECK-LABEL: test_masked_z_8xdouble_to_4xdouble_perm_mask3:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vextractf32x4 $2, %zmm0, %xmm3
 ; CHECK-NEXT:    vmovapd {{.*#+}} ymm2 = [0,2,1,4]
-; CHECK-NEXT:    vxorpd %xmm4, %xmm4, %xmm4
-; CHECK-NEXT:    vcmpeqpd %ymm4, %ymm1, %k1
-; CHECK-NEXT:    vpermi2pd %ymm3, %ymm0, %ymm2 {%k1} {z}
-; CHECK-NEXT:    vmovapd %ymm2, %ymm0
+; CHECK-NEXT:    vxorpd %xmm3, %xmm3, %xmm3
+; CHECK-NEXT:    vcmpeqpd %ymm3, %ymm1, %k1
+; CHECK-NEXT:    vpermpd %zmm0, %zmm2, %zmm0 {%k1} {z}
+; CHECK-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; CHECK-NEXT:    retq
   %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 0, i32 2, i32 1, i32 4>
   %cmp = fcmp oeq <4 x double> %mask, zeroinitializer
@@ -3871,9 +3868,9 @@ define <4 x double> @test_masked_z_8xdouble_to_4xdouble_perm_mask5(<8 x double>
 define <4 x double> @test_8xdouble_to_4xdouble_perm_mask6(<8 x double> %vec) {
 ; CHECK-LABEL: test_8xdouble_to_4xdouble_perm_mask6:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vbroadcasti64x4 {{.*#+}} zmm1 = [5,0,7,0,5,0,7,0]
+; CHECK-NEXT:    vbroadcasti64x4 {{.*#+}} zmm1 = [5,8,7,8,5,8,7,8]
 ; CHECK-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3]
-; CHECK-NEXT:    vpermq %zmm0, %zmm1, %zmm0
+; CHECK-NEXT:    vpermt2pd %zmm0, %zmm1, %zmm0
 ; CHECK-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; CHECK-NEXT:    retq
   %res = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 5, i32 0, i32 7, i32 0>
@@ -3882,12 +3879,12 @@ define <4 x double> @test_8xdouble_to_4xdouble_perm_mask6(<8 x double> %vec) {
 define <4 x double> @test_masked_8xdouble_to_4xdouble_perm_mask6(<8 x double> %vec, <4 x double> %vec2, <4 x double> %mask) {
 ; CHECK-LABEL: test_masked_8xdouble_to_4xdouble_perm_mask6:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vbroadcasti64x4 {{.*#+}} zmm3 = [5,0,7,0,5,0,7,0]
+; CHECK-NEXT:    vbroadcasti64x4 {{.*#+}} zmm3 = [5,8,7,8,5,8,7,8]
 ; CHECK-NEXT:    # zmm3 = mem[0,1,2,3,0,1,2,3]
-; CHECK-NEXT:    vpermq %zmm0, %zmm3, %zmm0
-; CHECK-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; CHECK-NEXT:    vcmpeqpd %ymm3, %ymm2, %k1
-; CHECK-NEXT:    vblendmpd %ymm0, %ymm1, %ymm0 {%k1}
+; CHECK-NEXT:    vpermi2pd %zmm0, %zmm0, %zmm3
+; CHECK-NEXT:    vxorpd %xmm0, %xmm0, %xmm0
+; CHECK-NEXT:    vcmpeqpd %ymm0, %ymm2, %k1
+; CHECK-NEXT:    vblendmpd %ymm3, %ymm1, %ymm0 {%k1}
 ; CHECK-NEXT:    retq
   %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 5, i32 0, i32 7, i32 0>
   %cmp = fcmp oeq <4 x double> %mask, zeroinitializer
@@ -3898,10 +3895,10 @@ define <4 x double> @test_masked_8xdouble_to_4xdouble_perm_mask6(<8 x double> %v
 define <4 x double> @test_masked_z_8xdouble_to_4xdouble_perm_mask6(<8 x double> %vec, <4 x double> %mask) {
 ; CHECK-LABEL: test_masked_z_8xdouble_to_4xdouble_perm_mask6:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmovapd {{.*#+}} ymm2 = [5,0,7,0]
+; CHECK-NEXT:    vmovapd {{.*#+}} ymm2 = [5,8,7,8]
 ; CHECK-NEXT:    vxorpd %xmm3, %xmm3, %xmm3
 ; CHECK-NEXT:    vcmpeqpd %ymm3, %ymm1, %k1
-; CHECK-NEXT:    vpermpd %zmm0, %zmm2, %zmm0 {%k1} {z}
+; CHECK-NEXT:    vpermt2pd %zmm0, %zmm2, %zmm0 {%k1} {z}
 ; CHECK-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; CHECK-NEXT:    retq
   %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 5, i32 0, i32 7, i32 0>
@@ -3985,9 +3982,8 @@ define <2 x double> @test_masked_z_8xdouble_to_2xdouble_perm_mask0(<8 x double>
 define <2 x double> @test_masked_8xdouble_to_2xdouble_perm_mask1(<8 x double> %vec, <2 x double> %vec2, <2 x double> %mask) {
 ; CHECK-LABEL: test_masked_8xdouble_to_2xdouble_perm_mask1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vextractf64x4 $1, %zmm0, %ymm3
-; CHECK-NEXT:    vunpckhpd {{.*#+}} ymm0 = ymm0[1],ymm3[1],ymm0[3],ymm3[3]
-; CHECK-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; CHECK-NEXT:    vmovapd {{.*#+}} xmm3 = [3,7]
+; CHECK-NEXT:    vpermpd %zmm0, %zmm3, %zmm0
 ; CHECK-NEXT:    vxorpd %xmm3, %xmm3, %xmm3
 ; CHECK-NEXT:    vcmpeqpd %xmm3, %xmm2, %k1
 ; CHECK-NEXT:    vblendmpd %xmm0, %xmm1, %xmm0 {%k1}
@@ -4002,12 +3998,11 @@ define <2 x double> @test_masked_8xdouble_to_2xdouble_perm_mask1(<8 x double> %v
 define <2 x double> @test_masked_z_8xdouble_to_2xdouble_perm_mask1(<8 x double> %vec, <2 x double> %mask) {
 ; CHECK-LABEL: test_masked_z_8xdouble_to_2xdouble_perm_mask1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vextractf64x4 $1, %zmm0, %ymm2
-; CHECK-NEXT:    vunpckhpd {{.*#+}} ymm0 = ymm0[1],ymm2[1],ymm0[3],ymm2[3]
-; CHECK-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; CHECK-NEXT:    vxorpd %xmm2, %xmm2, %xmm2
-; CHECK-NEXT:    vcmpeqpd %xmm2, %xmm1, %k1
-; CHECK-NEXT:    vmovapd %xmm0, %xmm0 {%k1} {z}
+; CHECK-NEXT:    vmovapd {{.*#+}} xmm2 = [3,7]
+; CHECK-NEXT:    vxorpd %xmm3, %xmm3, %xmm3
+; CHECK-NEXT:    vcmpeqpd %xmm3, %xmm1, %k1
+; CHECK-NEXT:    vpermpd %zmm0, %zmm2, %zmm0 {%k1} {z}
+; CHECK-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
   %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <2 x i32> <i32 3, i32 7>
@@ -4064,8 +4059,8 @@ define <4 x double> @test_masked_8xdouble_to_4xdouble_perm_mem_mask1(<8 x double
 ; CHECK-LABEL: test_masked_8xdouble_to_4xdouble_perm_mem_mask1:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vmovapd (%rdi), %ymm2
-; CHECK-NEXT:    vmovapd {{.*#+}} ymm3 = [3,4,2,4]
-; CHECK-NEXT:    vpermi2pd 32(%rdi), %ymm2, %ymm3
+; CHECK-NEXT:    vmovapd {{.*#+}} ymm3 = [3,4,2,6]
+; CHECK-NEXT:    vpermi2pd 32(%rdi){1to4}, %ymm2, %ymm3
 ; CHECK-NEXT:    vxorpd %xmm2, %xmm2, %xmm2
 ; CHECK-NEXT:    vcmpeqpd %ymm2, %ymm1, %k1
 ; CHECK-NEXT:    vmovapd %ymm3, %ymm0 {%k1}
@@ -4081,10 +4076,10 @@ define <4 x double> @test_masked_z_8xdouble_to_4xdouble_perm_mem_mask1(<8 x doub
 ; CHECK-LABEL: test_masked_z_8xdouble_to_4xdouble_perm_mem_mask1:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vmovapd (%rdi), %ymm2
-; CHECK-NEXT:    vmovapd {{.*#+}} ymm1 = [3,4,2,4]
+; CHECK-NEXT:    vmovapd {{.*#+}} ymm1 = [3,4,2,6]
 ; CHECK-NEXT:    vxorpd %xmm3, %xmm3, %xmm3
 ; CHECK-NEXT:    vcmpeqpd %ymm3, %ymm0, %k1
-; CHECK-NEXT:    vpermi2pd 32(%rdi), %ymm2, %ymm1 {%k1} {z}
+; CHECK-NEXT:    vpermi2pd 32(%rdi){1to4}, %ymm2, %ymm1 {%k1} {z}
 ; CHECK-NEXT:    vmovapd %ymm1, %ymm0
 ; CHECK-NEXT:    retq
   %vec = load <8 x double>, <8 x double>* %vp
@@ -4244,10 +4239,9 @@ define <4 x double> @test_masked_z_8xdouble_to_4xdouble_perm_mem_mask5(<8 x doub
 define <4 x double> @test_8xdouble_to_4xdouble_perm_mem_mask6(<8 x double>* %vp) {
 ; CHECK-LABEL: test_8xdouble_to_4xdouble_perm_mem_mask6:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vinsertf128 $1, (%rdi), %ymm0, %ymm1
-; CHECK-NEXT:    vmovapd 32(%rdi), %ymm2
-; CHECK-NEXT:    vmovapd {{.*#+}} ymm0 = [0,2,6,1]
-; CHECK-NEXT:    vpermi2pd %ymm1, %ymm2, %ymm0
+; CHECK-NEXT:    vmovapd 32(%rdi), %ymm1
+; CHECK-NEXT:    vmovapd {{.*#+}} ymm0 = [0,2,4,1]
+; CHECK-NEXT:    vpermi2pd (%rdi), %ymm1, %ymm0
 ; CHECK-NEXT:    retq
   %vec = load <8 x double>, <8 x double>* %vp
   %res = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 4, i32 6, i32 0, i32 5>
@@ -4256,13 +4250,12 @@ define <4 x double> @test_8xdouble_to_4xdouble_perm_mem_mask6(<8 x double>* %vp)
 define <4 x double> @test_masked_8xdouble_to_4xdouble_perm_mem_mask6(<8 x double>* %vp, <4 x double> %vec2, <4 x double> %mask) {
 ; CHECK-LABEL: test_masked_8xdouble_to_4xdouble_perm_mem_mask6:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vinsertf128 $1, (%rdi), %ymm0, %ymm2
-; CHECK-NEXT:    vmovapd 32(%rdi), %ymm3
-; CHECK-NEXT:    vmovapd {{.*#+}} ymm4 = [0,2,6,1]
-; CHECK-NEXT:    vpermi2pd %ymm2, %ymm3, %ymm4
+; CHECK-NEXT:    vmovapd 32(%rdi), %ymm2
+; CHECK-NEXT:    vmovapd {{.*#+}} ymm3 = [0,2,4,1]
+; CHECK-NEXT:    vpermi2pd (%rdi), %ymm2, %ymm3
 ; CHECK-NEXT:    vxorpd %xmm2, %xmm2, %xmm2
 ; CHECK-NEXT:    vcmpeqpd %ymm2, %ymm1, %k1
-; CHECK-NEXT:    vmovapd %ymm4, %ymm0 {%k1}
+; CHECK-NEXT:    vmovapd %ymm3, %ymm0 {%k1}
 ; CHECK-NEXT:    retq
   %vec = load <8 x double>, <8 x double>* %vp
   %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 4, i32 6, i32 0, i32 5>
@@ -4274,12 +4267,11 @@ define <4 x double> @test_masked_8xdouble_to_4xdouble_perm_mem_mask6(<8 x double
 define <4 x double> @test_masked_z_8xdouble_to_4xdouble_perm_mem_mask6(<8 x double>* %vp, <4 x double> %mask) {
 ; CHECK-LABEL: test_masked_z_8xdouble_to_4xdouble_perm_mem_mask6:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vinsertf128 $1, (%rdi), %ymm0, %ymm2
-; CHECK-NEXT:    vmovapd 32(%rdi), %ymm3
-; CHECK-NEXT:    vmovapd {{.*#+}} ymm1 = [0,2,6,1]
-; CHECK-NEXT:    vxorpd %xmm4, %xmm4, %xmm4
-; CHECK-NEXT:    vcmpeqpd %ymm4, %ymm0, %k1
-; CHECK-NEXT:    vpermi2pd %ymm2, %ymm3, %ymm1 {%k1} {z}
+; CHECK-NEXT:    vmovapd 32(%rdi), %ymm2
+; CHECK-NEXT:    vmovapd {{.*#+}} ymm1 = [0,2,4,1]
+; CHECK-NEXT:    vxorpd %xmm3, %xmm3, %xmm3
+; CHECK-NEXT:    vcmpeqpd %ymm3, %ymm0, %k1
+; CHECK-NEXT:    vpermi2pd (%rdi), %ymm2, %ymm1 {%k1} {z}
 ; CHECK-NEXT:    vmovapd %ymm1, %ymm0
 ; CHECK-NEXT:    retq
   %vec = load <8 x double>, <8 x double>* %vp
diff --git a/llvm/test/CodeGen/X86/avx512-skx-insert-subvec.ll b/llvm/test/CodeGen/X86/avx512-skx-insert-subvec.ll
index 5338eb3c3..a24c1d8c2 100644
--- a/llvm/test/CodeGen/X86/avx512-skx-insert-subvec.ll
+++ b/llvm/test/CodeGen/X86/avx512-skx-insert-subvec.ll
@@ -163,3 +163,53 @@ define <8 x i1> @test11(<4 x i1> %a, <4 x i1>%b) {
   %res = shufflevector <4 x i1> %a, <4 x i1> undef, <8 x i32> <i32 undef, i32 undef, i32 undef, i32 undef, i32 0, i32 1, i32 2, i32 3>
   ret <8 x i1> %res
 }
+
+define <16 x i1> @test12(<2 x i1> %a) {
+; CHECK-LABEL: test12:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsllq $63, %xmm0, %xmm0
+; CHECK-NEXT:    vpmovq2m %xmm0, %k0
+; CHECK-NEXT:    kshiftlw $10, %k0, %k0
+; CHECK-NEXT:    vpmovm2b %k0, %xmm0
+; CHECK-NEXT:    retq
+  %res = shufflevector <2 x i1> %a, <2 x i1> zeroinitializer, <16 x i32> <i32 2, i32 3, i32 2, i32 3, i32 2, i32 3, i32 2, i32 3, i32 2, i32 3, i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef>
+  ret <16 x i1> %res
+}
+
+define <32 x i1> @test13(<2 x i1> %a) {
+; CHECK-LABEL: test13:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsllq $63, %xmm0, %xmm0
+; CHECK-NEXT:    vpmovq2m %xmm0, %k0
+; CHECK-NEXT:    kshiftld $10, %k0, %k0
+; CHECK-NEXT:    vpmovm2b %k0, %ymm0
+; CHECK-NEXT:    retq
+  %res = shufflevector <2 x i1> %a, <2 x i1> zeroinitializer, <32 x i32> <i32 2, i32 3, i32 2, i32 3, i32 2, i32 3, i32 2, i32 3, i32 2, i32 3, i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  ret <32 x i1> %res
+}
+
+define <64 x i1> @test14(<2 x i1> %a) {
+; CHECK-LABEL: test14:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsllq $63, %xmm0, %xmm0
+; CHECK-NEXT:    vpmovq2m %xmm0, %k0
+; CHECK-NEXT:    kshiftlq $10, %k0, %k0
+; CHECK-NEXT:    vpmovm2b %k0, %zmm0
+; CHECK-NEXT:    retq
+  %res = shufflevector <2 x i1> %a, <2 x i1> zeroinitializer, <64 x i32> <i32 2, i32 3, i32 2, i32 3, i32 2, i32 3, i32 2, i32 3, i32 2, i32 3, i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  ret <64 x i1> %res
+}
+
+; Make sure we can recognize this shuffle as an insertion in to a zero vector.
+define i8 @test15(<2 x i64> %x) {
+; CHECK-LABEL: test15:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vptestnmq %xmm0, %xmm0, %k0
+; CHECK-NEXT:    kmovd %k0, %eax
+; CHECK-NEXT:    # kill: def $al killed $al killed $eax
+; CHECK-NEXT:    retq
+  %a = icmp eq <2 x i64> %x, zeroinitializer
+  %b = shufflevector <2 x i1> %a, <2 x i1> <i1 false, i1 undef>, <8 x i32> <i32 0, i32 1, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  %c = bitcast <8 x i1> %b to i8
+  ret i8 %c
+}
diff --git a/llvm/test/CodeGen/X86/avx512-trunc.ll b/llvm/test/CodeGen/X86/avx512-trunc.ll
index 263f7c904..bf49987ab 100644
--- a/llvm/test/CodeGen/X86/avx512-trunc.ll
+++ b/llvm/test/CodeGen/X86/avx512-trunc.ll
@@ -36,7 +36,7 @@ define <16 x i16> @trunc_v16i32_to_v16i16(<16 x i32> %x) #0 {
 define <8 x i8> @trunc_qb_512(<8 x i64> %i) #0 {
 ; ALL-LABEL: trunc_qb_512:
 ; ALL:       ## %bb.0:
-; ALL-NEXT:    vpmovqw %zmm0, %xmm0
+; ALL-NEXT:    vpmovqb %zmm0, %xmm0
 ; ALL-NEXT:    vzeroupper
 ; ALL-NEXT:    retq
   %x = trunc <8 x i64> %i to <8 x i8>
@@ -58,14 +58,13 @@ define <4 x i8> @trunc_qb_256(<4 x i64> %i) #0 {
 ; KNL-LABEL: trunc_qb_256:
 ; KNL:       ## %bb.0:
 ; KNL-NEXT:    ## kill: def $ymm0 killed $ymm0 def $zmm0
-; KNL-NEXT:    vpmovqd %zmm0, %ymm0
-; KNL-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $ymm0
+; KNL-NEXT:    vpmovqb %zmm0, %xmm0
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: trunc_qb_256:
 ; SKX:       ## %bb.0:
-; SKX-NEXT:    vpmovqd %ymm0, %xmm0
+; SKX-NEXT:    vpmovqb %ymm0, %xmm0
 ; SKX-NEXT:    vzeroupper
 ; SKX-NEXT:    retq
   %x = trunc <4 x i64> %i to <4 x i8>
@@ -76,8 +75,7 @@ define void @trunc_qb_256_mem(<4 x i64> %i, <4 x i8>* %res) #0 {
 ; KNL-LABEL: trunc_qb_256_mem:
 ; KNL:       ## %bb.0:
 ; KNL-NEXT:    ## kill: def $ymm0 killed $ymm0 def $zmm0
-; KNL-NEXT:    vpmovqd %zmm0, %ymm0
-; KNL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; KNL-NEXT:    vpmovqb %zmm0, %xmm0
 ; KNL-NEXT:    vmovd %xmm0, (%rdi)
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    retq
@@ -95,6 +93,7 @@ define void @trunc_qb_256_mem(<4 x i64> %i, <4 x i8>* %res) #0 {
 define <2 x i8> @trunc_qb_128(<2 x i64> %i) #0 {
 ; ALL-LABEL: trunc_qb_128:
 ; ALL:       ## %bb.0:
+; ALL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
 ; ALL-NEXT:    retq
   %x = trunc <2 x i64> %i to <2 x i8>
   ret <2 x i8> %x
@@ -141,14 +140,13 @@ define <4 x i16> @trunc_qw_256(<4 x i64> %i) #0 {
 ; KNL-LABEL: trunc_qw_256:
 ; KNL:       ## %bb.0:
 ; KNL-NEXT:    ## kill: def $ymm0 killed $ymm0 def $zmm0
-; KNL-NEXT:    vpmovqd %zmm0, %ymm0
-; KNL-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $ymm0
+; KNL-NEXT:    vpmovqw %zmm0, %xmm0
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: trunc_qw_256:
 ; SKX:       ## %bb.0:
-; SKX-NEXT:    vpmovqd %ymm0, %xmm0
+; SKX-NEXT:    vpmovqw %ymm0, %xmm0
 ; SKX-NEXT:    vzeroupper
 ; SKX-NEXT:    retq
   %x = trunc <4 x i64> %i to <4 x i16>
@@ -159,8 +157,7 @@ define void @trunc_qw_256_mem(<4 x i64> %i, <4 x i16>* %res) #0 {
 ; KNL-LABEL: trunc_qw_256_mem:
 ; KNL:       ## %bb.0:
 ; KNL-NEXT:    ## kill: def $ymm0 killed $ymm0 def $zmm0
-; KNL-NEXT:    vpmovqd %zmm0, %ymm0
-; KNL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
+; KNL-NEXT:    vpmovqw %zmm0, %xmm0
 ; KNL-NEXT:    vmovq %xmm0, (%rdi)
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    retq
@@ -176,9 +173,16 @@ define void @trunc_qw_256_mem(<4 x i64> %i, <4 x i16>* %res) #0 {
 }
 
 define <2 x i16> @trunc_qw_128(<2 x i64> %i) #0 {
-; ALL-LABEL: trunc_qw_128:
-; ALL:       ## %bb.0:
-; ALL-NEXT:    retq
+; KNL-LABEL: trunc_qw_128:
+; KNL:       ## %bb.0:
+; KNL-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; KNL-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; KNL-NEXT:    retq
+;
+; SKX-LABEL: trunc_qw_128:
+; SKX:       ## %bb.0:
+; SKX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,8,9,8,9,10,11,8,9,10,11,12,13,14,15]
+; SKX-NEXT:    retq
   %x = trunc <2 x i64> %i to <2 x i16>
   ret <2 x i16> %x
 }
@@ -260,6 +264,7 @@ define void @trunc_qd_256_mem(<4 x i64> %i, <4 x i32>* %res) #0 {
 define <2 x i32> @trunc_qd_128(<2 x i64> %i) #0 {
 ; ALL-LABEL: trunc_qd_128:
 ; ALL:       ## %bb.0:
+; ALL-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; ALL-NEXT:    retq
   %x = trunc <2 x i64> %i to <2 x i32>
   ret <2 x i32> %x
@@ -306,14 +311,13 @@ define <8 x i8> @trunc_db_256(<8 x i32> %i) #0 {
 ; KNL-LABEL: trunc_db_256:
 ; KNL:       ## %bb.0:
 ; KNL-NEXT:    ## kill: def $ymm0 killed $ymm0 def $zmm0
-; KNL-NEXT:    vpmovdw %zmm0, %ymm0
-; KNL-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $ymm0
+; KNL-NEXT:    vpmovdb %zmm0, %xmm0
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: trunc_db_256:
 ; SKX:       ## %bb.0:
-; SKX-NEXT:    vpmovdw %ymm0, %xmm0
+; SKX-NEXT:    vpmovdb %ymm0, %xmm0
 ; SKX-NEXT:    vzeroupper
 ; SKX-NEXT:    retq
   %x = trunc <8 x i32> %i to <8 x i8>
@@ -324,8 +328,7 @@ define void @trunc_db_256_mem(<8 x i32> %i, <8 x i8>* %res) #0 {
 ; KNL-LABEL: trunc_db_256_mem:
 ; KNL:       ## %bb.0:
 ; KNL-NEXT:    ## kill: def $ymm0 killed $ymm0 def $zmm0
-; KNL-NEXT:    vpmovdw %zmm0, %ymm0
-; KNL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
+; KNL-NEXT:    vpmovdb %zmm0, %xmm0
 ; KNL-NEXT:    vmovq %xmm0, (%rdi)
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    retq
@@ -343,6 +346,7 @@ define void @trunc_db_256_mem(<8 x i32> %i, <8 x i8>* %res) #0 {
 define <4 x i8> @trunc_db_128(<4 x i32> %i) #0 {
 ; ALL-LABEL: trunc_db_128:
 ; ALL:       ## %bb.0:
+; ALL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
 ; ALL-NEXT:    retq
   %x = trunc <4 x i32> %i to <4 x i8>
   ret <4 x i8> %x
@@ -440,6 +444,7 @@ define void @trunc_dw_128_mem(<4 x i32> %i, <4 x i16>* %res) #0 {
 define <32 x i8> @trunc_wb_512(<32 x i16> %i) #0 {
 ; KNL-LABEL: trunc_wb_512:
 ; KNL:       ## %bb.0:
+; KNL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; KNL-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
 ; KNL-NEXT:    vpmovdb %zmm0, %xmm0
 ; KNL-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
@@ -458,6 +463,7 @@ define <32 x i8> @trunc_wb_512(<32 x i16> %i) #0 {
 define void @trunc_wb_512_mem(<32 x i16> %i, <32 x i8>* %res) #0 {
 ; KNL-LABEL: trunc_wb_512_mem:
 ; KNL:       ## %bb.0:
+; KNL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; KNL-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
 ; KNL-NEXT:    vpmovdb %zmm1, 16(%rdi)
 ; KNL-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
@@ -510,9 +516,30 @@ define void @trunc_wb_256_mem(<16 x i16> %i, <16 x i8>* %res) #0 {
     ret void
 }
 
+define <16 x i8> @trunc_wb_256_mem_and_ret(<16 x i16> %i, <16 x i8>* %res) #0 {
+; KNL-LABEL: trunc_wb_256_mem_and_ret:
+; KNL:       ## %bb.0:
+; KNL-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
+; KNL-NEXT:    vpmovdb %zmm0, %xmm0
+; KNL-NEXT:    vmovdqa %xmm0, (%rdi)
+; KNL-NEXT:    vzeroupper
+; KNL-NEXT:    retq
+;
+; SKX-LABEL: trunc_wb_256_mem_and_ret:
+; SKX:       ## %bb.0:
+; SKX-NEXT:    vpmovwb %ymm0, %xmm0
+; SKX-NEXT:    vmovdqa %xmm0, (%rdi)
+; SKX-NEXT:    vzeroupper
+; SKX-NEXT:    retq
+    %x = trunc <16 x i16> %i to <16 x i8>
+    store <16 x i8> %x, <16 x i8>* %res
+    ret <16 x i8> %x
+}
+
 define <8 x i8> @trunc_wb_128(<8 x i16> %i) #0 {
 ; ALL-LABEL: trunc_wb_128:
 ; ALL:       ## %bb.0:
+; ALL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
 ; ALL-NEXT:    retq
   %x = trunc <8 x i16> %i to <8 x i8>
   ret <8 x i8> %x
@@ -663,10 +690,8 @@ define <32 x i8> @usat_trunc_db_1024(<32 x i32> %i) {
 define void @usat_trunc_db_1024_mem(<32 x i32> %i, <32 x i8>* %p) {
 ; ALL-LABEL: usat_trunc_db_1024_mem:
 ; ALL:       ## %bb.0:
-; ALL-NEXT:    vpmovusdb %zmm0, %xmm0
-; ALL-NEXT:    vpmovusdb %zmm1, %xmm1
-; ALL-NEXT:    vmovdqu %xmm1, 16(%rdi)
-; ALL-NEXT:    vmovdqu %xmm0, (%rdi)
+; ALL-NEXT:    vpmovusdb %zmm1, 16(%rdi)
+; ALL-NEXT:    vpmovusdb %zmm0, (%rdi)
 ; ALL-NEXT:    vzeroupper
 ; ALL-NEXT:    retq
   %x3 = icmp ult <32 x i32> %i, <i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255>
@@ -688,10 +713,16 @@ define <16 x i16> @usat_trunc_dw_512(<16 x i32> %i) {
 }
 
 define <8 x i8> @usat_trunc_wb_128(<8 x i16> %i) {
-; ALL-LABEL: usat_trunc_wb_128:
-; ALL:       ## %bb.0:
-; ALL-NEXT:    vpminuw {{.*}}(%rip), %xmm0, %xmm0
-; ALL-NEXT:    retq
+; KNL-LABEL: usat_trunc_wb_128:
+; KNL:       ## %bb.0:
+; KNL-NEXT:    vpminuw {{.*}}(%rip), %xmm0, %xmm0
+; KNL-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
+; KNL-NEXT:    retq
+;
+; SKX-LABEL: usat_trunc_wb_128:
+; SKX:       ## %bb.0:
+; SKX-NEXT:    vpmovuswb %xmm0, %xmm0
+; SKX-NEXT:    retq
   %x3 = icmp ult <8 x i16> %i, <i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255>
   %x5 = select <8 x i1> %x3, <8 x i16> %i, <8 x i16> <i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255>
   %x6 = trunc <8 x i16> %x5 to <8 x i8>
@@ -714,18 +745,14 @@ define <16 x i16> @usat_trunc_qw_1024(<16 x i64> %i) {
 define <16 x i8> @usat_trunc_db_256(<8 x i32> %x) {
 ; KNL-LABEL: usat_trunc_db_256:
 ; KNL:       ## %bb.0:
-; KNL-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [255,255,255,255,255,255,255,255]
-; KNL-NEXT:    vpminud %ymm1, %ymm0, %ymm0
-; KNL-NEXT:    vpmovdw %zmm0, %ymm0
-; KNL-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
+; KNL-NEXT:    ## kill: def $ymm0 killed $ymm0 def $zmm0
+; KNL-NEXT:    vpmovusdb %zmm0, %xmm0
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: usat_trunc_db_256:
 ; SKX:       ## %bb.0:
-; SKX-NEXT:    vpminud {{.*}}(%rip){1to8}, %ymm0, %ymm0
-; SKX-NEXT:    vpmovdw %ymm0, %xmm0
-; SKX-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
+; SKX-NEXT:    vpmovusdb %ymm0, %xmm0
 ; SKX-NEXT:    vzeroupper
 ; SKX-NEXT:    retq
   %tmp1 = icmp ult <8 x i32> %x, <i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255>
@@ -749,11 +776,9 @@ define <16 x i8> @usat_trunc_db_256(<8 x i32> %x) {
 define void @smax_usat_trunc_wb_256_mem1(<16 x i16> %i, <16 x i8>* %res) {
 ; KNL-LABEL: smax_usat_trunc_wb_256_mem1:
 ; KNL:       ## %bb.0:
-; KNL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; KNL-NEXT:    vpmaxsw %ymm1, %ymm0, %ymm0
-; KNL-NEXT:    vpminsw {{.*}}(%rip), %ymm0, %ymm0
-; KNL-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
-; KNL-NEXT:    vpmovdb %zmm0, (%rdi)
+; KNL-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; KNL-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; KNL-NEXT:    vmovdqu %xmm0, (%rdi)
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    retq
 ;
@@ -777,11 +802,9 @@ define void @smax_usat_trunc_wb_256_mem1(<16 x i16> %i, <16 x i8>* %res) {
 define void @smax_usat_trunc_wb_256_mem2(<16 x i16> %i, <16 x i8>* %res) {
 ; KNL-LABEL: smax_usat_trunc_wb_256_mem2:
 ; KNL:       ## %bb.0:
-; KNL-NEXT:    vpminsw {{.*}}(%rip), %ymm0, %ymm0
-; KNL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; KNL-NEXT:    vpmaxsw %ymm1, %ymm0, %ymm0
-; KNL-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
-; KNL-NEXT:    vpmovdb %zmm0, (%rdi)
+; KNL-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; KNL-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; KNL-NEXT:    vmovdqu %xmm0, (%rdi)
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    retq
 ;
@@ -804,11 +827,8 @@ define void @smax_usat_trunc_wb_256_mem2(<16 x i16> %i, <16 x i8>* %res) {
 define <16 x i8> @smax_usat_trunc_wb_256(<16 x i16> %i) {
 ; KNL-LABEL: smax_usat_trunc_wb_256:
 ; KNL:       ## %bb.0:
-; KNL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; KNL-NEXT:    vpmaxsw %ymm1, %ymm0, %ymm0
-; KNL-NEXT:    vpminsw {{.*}}(%rip), %ymm0, %ymm0
-; KNL-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
-; KNL-NEXT:    vpmovdb %zmm0, %xmm0
+; KNL-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; KNL-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    retq
 ;
@@ -830,9 +850,6 @@ define <16 x i8> @smax_usat_trunc_wb_256(<16 x i16> %i) {
 define void @smax_usat_trunc_wb_128_mem(<8 x i16> %i, <8 x i8>* %res) {
 ; KNL-LABEL: smax_usat_trunc_wb_128_mem:
 ; KNL:       ## %bb.0:
-; KNL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; KNL-NEXT:    vpmaxsw %xmm1, %xmm0, %xmm0
-; KNL-NEXT:    vpminsw {{.*}}(%rip), %xmm0, %xmm0
 ; KNL-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
 ; KNL-NEXT:    vmovq %xmm0, (%rdi)
 ; KNL-NEXT:    retq
@@ -942,12 +959,10 @@ define void @smax_usat_trunc_db_1024_mem(<32 x i32> %i, <32 x i8>* %p) {
 ; ALL-LABEL: smax_usat_trunc_db_1024_mem:
 ; ALL:       ## %bb.0:
 ; ALL-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; ALL-NEXT:    vpmaxsd %zmm2, %zmm1, %zmm1
 ; ALL-NEXT:    vpmaxsd %zmm2, %zmm0, %zmm0
-; ALL-NEXT:    vpmovusdb %zmm0, %xmm0
-; ALL-NEXT:    vpmovusdb %zmm1, %xmm1
-; ALL-NEXT:    vmovdqu %xmm1, 16(%rdi)
-; ALL-NEXT:    vmovdqu %xmm0, (%rdi)
+; ALL-NEXT:    vpmaxsd %zmm2, %zmm1, %zmm1
+; ALL-NEXT:    vpmovusdb %zmm1, 16(%rdi)
+; ALL-NEXT:    vpmovusdb %zmm0, (%rdi)
 ; ALL-NEXT:    vzeroupper
 ; ALL-NEXT:    retq
   %x1 = icmp sgt <32 x i32> %i, <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
@@ -1029,3 +1044,21 @@ define void @negative_test2_smax_usat_trunc_wb_256_mem(<16 x i16> %i, <16 x i8>*
   store <16 x i8> %x6, <16 x i8>* %res, align 1
   ret void
 }
+
+define void @ssat_trunc_db_1024_mem(<32 x i32> %i, <32 x i8>* %p) {
+; ALL-LABEL: ssat_trunc_db_1024_mem:
+; ALL:       ## %bb.0:
+; ALL-NEXT:    vpmovsdb %zmm1, 16(%rdi)
+; ALL-NEXT:    vpmovsdb %zmm0, (%rdi)
+; ALL-NEXT:    vzeroupper
+; ALL-NEXT:    retq
+  %x1 = icmp sgt <32 x i32> %i, <i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32
+-128, i32 -128>
+  %x2 = select <32 x i1> %x1, <32 x i32> %i, <32 x i32> <i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128>
+  %x3 = icmp slt <32 x i32> %x2, <i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127>
+  %x5 = select <32 x i1> %x3, <32 x i32> %x2, <32 x i32> <i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127>
+  %x6 = trunc <32 x i32> %x5 to <32 x i8>
+  store <32 x i8>%x6, <32 x i8>* %p, align 1
+  ret void
+}
+
diff --git a/llvm/test/CodeGen/X86/avx512-unsafe-fp-math.ll b/llvm/test/CodeGen/X86/avx512-unsafe-fp-math.ll
index 00c9e4c95..9282ed1ec 100644
--- a/llvm/test/CodeGen/X86/avx512-unsafe-fp-math.ll
+++ b/llvm/test/CodeGen/X86/avx512-unsafe-fp-math.ll
@@ -1,5 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64 -enable-unsafe-fp-math -mattr=+avx512f | FileCheck %s --check-prefix=CHECK_UNSAFE --check-prefix=AVX512F_UNSAFE
+; RUN: llc < %s -mtriple=x86_64 -enable-no-nans-fp-math -enable-no-signed-zeros-fp-math -mattr=+avx512f | FileCheck %s --check-prefix=CHECK_UNSAFE --check-prefix=AVX512F_UNSAFE
+; RUN: llc < %s -mtriple=x86_64 -enable-no-nans-fp-math -mattr=+avx512f | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512F
+; RUN: llc < %s -mtriple=x86_64 -enable-no-signed-zeros-fp-math -mattr=+avx512f | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512
+; RUN: llc < %s -mtriple=x86_64 -enable-unsafe-fp-math -mattr=+avx512f | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512
 ; RUN: llc < %s -mtriple=x86_64 -mattr=+avx512f | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512
 
 define <16 x float> @test_max_v16f32(<16 x float> * %a_ptr, <16 x float> %b)  {
diff --git a/llvm/test/CodeGen/X86/avx512-vbroadcast.ll b/llvm/test/CodeGen/X86/avx512-vbroadcast.ll
index 3277f99dc..5e46f26f0 100644
--- a/llvm/test/CodeGen/X86/avx512-vbroadcast.ll
+++ b/llvm/test/CodeGen/X86/avx512-vbroadcast.ll
@@ -350,7 +350,7 @@ define <64 x i8> @_invec32xi8(<32 x i8>%a)  {
 ; AVX512F-LABEL: _invec32xi8:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpbroadcastb %xmm0, %ymm0
-; AVX512F-NEXT:    vmovdqa %ymm0, %ymm1
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: _invec32xi8:
@@ -365,7 +365,7 @@ define <32 x i16> @_invec16xi16(<16 x i16>%a)  {
 ; AVX512F-LABEL: _invec16xi16:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpbroadcastw %xmm0, %ymm0
-; AVX512F-NEXT:    vmovdqa %ymm0, %ymm1
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: _invec16xi16:
diff --git a/llvm/test/CodeGen/X86/avx512-vbroadcasti128.ll b/llvm/test/CodeGen/X86/avx512-vbroadcasti128.ll
index 3ca4f9a44..4067690d7 100644
--- a/llvm/test/CodeGen/X86/avx512-vbroadcasti128.ll
+++ b/llvm/test/CodeGen/X86/avx512-vbroadcasti128.ll
@@ -134,9 +134,10 @@ define <16 x i32> @test_broadcast_4i32_16i32(<4 x i32> *%p) nounwind {
 define <32 x i16> @test_broadcast_8i16_32i16(<8 x i16> *%p) nounwind {
 ; X64-AVX512VL-LABEL: test_broadcast_8i16_32i16:
 ; X64-AVX512VL:       ## %bb.0:
-; X64-AVX512VL-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = mem[0,1,0,1]
-; X64-AVX512VL-NEXT:    vpaddw {{.*}}(%rip), %ymm1, %ymm0
-; X64-AVX512VL-NEXT:    vpaddw {{.*}}(%rip), %ymm1, %ymm1
+; X64-AVX512VL-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = mem[0,1,0,1]
+; X64-AVX512VL-NEXT:    vpaddw {{.*}}(%rip), %ymm0, %ymm1
+; X64-AVX512VL-NEXT:    vpaddw {{.*}}(%rip), %ymm0, %ymm0
+; X64-AVX512VL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; X64-AVX512VL-NEXT:    retq
 ;
 ; X64-AVX512BWVL-LABEL: test_broadcast_8i16_32i16:
@@ -147,9 +148,10 @@ define <32 x i16> @test_broadcast_8i16_32i16(<8 x i16> *%p) nounwind {
 ;
 ; X64-AVX512DQVL-LABEL: test_broadcast_8i16_32i16:
 ; X64-AVX512DQVL:       ## %bb.0:
-; X64-AVX512DQVL-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = mem[0,1,0,1]
-; X64-AVX512DQVL-NEXT:    vpaddw {{.*}}(%rip), %ymm1, %ymm0
-; X64-AVX512DQVL-NEXT:    vpaddw {{.*}}(%rip), %ymm1, %ymm1
+; X64-AVX512DQVL-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = mem[0,1,0,1]
+; X64-AVX512DQVL-NEXT:    vpaddw {{.*}}(%rip), %ymm0, %ymm1
+; X64-AVX512DQVL-NEXT:    vpaddw {{.*}}(%rip), %ymm0, %ymm0
+; X64-AVX512DQVL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; X64-AVX512DQVL-NEXT:    retq
  %1 = load <8 x i16>, <8 x i16> *%p
  %2 = shufflevector <8 x i16> %1, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -160,9 +162,10 @@ define <32 x i16> @test_broadcast_8i16_32i16(<8 x i16> *%p) nounwind {
 define <64 x i8> @test_broadcast_16i8_64i8(<16 x i8> *%p) nounwind {
 ; X64-AVX512VL-LABEL: test_broadcast_16i8_64i8:
 ; X64-AVX512VL:       ## %bb.0:
-; X64-AVX512VL-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = mem[0,1,0,1]
-; X64-AVX512VL-NEXT:    vpaddb {{.*}}(%rip), %ymm1, %ymm0
-; X64-AVX512VL-NEXT:    vpaddb {{.*}}(%rip), %ymm1, %ymm1
+; X64-AVX512VL-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = mem[0,1,0,1]
+; X64-AVX512VL-NEXT:    vpaddb {{.*}}(%rip), %ymm0, %ymm1
+; X64-AVX512VL-NEXT:    vpaddb {{.*}}(%rip), %ymm0, %ymm0
+; X64-AVX512VL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; X64-AVX512VL-NEXT:    retq
 ;
 ; X64-AVX512BWVL-LABEL: test_broadcast_16i8_64i8:
@@ -173,9 +176,10 @@ define <64 x i8> @test_broadcast_16i8_64i8(<16 x i8> *%p) nounwind {
 ;
 ; X64-AVX512DQVL-LABEL: test_broadcast_16i8_64i8:
 ; X64-AVX512DQVL:       ## %bb.0:
-; X64-AVX512DQVL-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = mem[0,1,0,1]
-; X64-AVX512DQVL-NEXT:    vpaddb {{.*}}(%rip), %ymm1, %ymm0
-; X64-AVX512DQVL-NEXT:    vpaddb {{.*}}(%rip), %ymm1, %ymm1
+; X64-AVX512DQVL-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = mem[0,1,0,1]
+; X64-AVX512DQVL-NEXT:    vpaddb {{.*}}(%rip), %ymm0, %ymm1
+; X64-AVX512DQVL-NEXT:    vpaddb {{.*}}(%rip), %ymm0, %ymm0
+; X64-AVX512DQVL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; X64-AVX512DQVL-NEXT:    retq
  %1 = load <16 x i8>, <16 x i8> *%p
  %2 = shufflevector <16 x i8> %1, <16 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
diff --git a/llvm/test/CodeGen/X86/avx512-vbroadcasti256.ll b/llvm/test/CodeGen/X86/avx512-vbroadcasti256.ll
index b7710f323..8b6082faf 100644
--- a/llvm/test/CodeGen/X86/avx512-vbroadcasti256.ll
+++ b/llvm/test/CodeGen/X86/avx512-vbroadcasti256.ll
@@ -54,9 +54,10 @@ define <16 x i32> @test_broadcast_8i32_16i32(<8 x i32> *%p) nounwind {
 define <32 x i16> @test_broadcast_16i16_32i16(<16 x i16> *%p) nounwind {
 ; X64-AVX512VL-LABEL: test_broadcast_16i16_32i16:
 ; X64-AVX512VL:       ## %bb.0:
-; X64-AVX512VL-NEXT:    vmovdqa (%rdi), %ymm1
-; X64-AVX512VL-NEXT:    vpaddw {{.*}}(%rip), %ymm1, %ymm0
-; X64-AVX512VL-NEXT:    vpaddw {{.*}}(%rip), %ymm1, %ymm1
+; X64-AVX512VL-NEXT:    vmovdqa (%rdi), %ymm0
+; X64-AVX512VL-NEXT:    vpaddw {{.*}}(%rip), %ymm0, %ymm1
+; X64-AVX512VL-NEXT:    vpaddw {{.*}}(%rip), %ymm0, %ymm0
+; X64-AVX512VL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; X64-AVX512VL-NEXT:    retq
 ;
 ; X64-AVX512BWVL-LABEL: test_broadcast_16i16_32i16:
@@ -67,9 +68,10 @@ define <32 x i16> @test_broadcast_16i16_32i16(<16 x i16> *%p) nounwind {
 ;
 ; X64-AVX512DQVL-LABEL: test_broadcast_16i16_32i16:
 ; X64-AVX512DQVL:       ## %bb.0:
-; X64-AVX512DQVL-NEXT:    vmovdqa (%rdi), %ymm1
-; X64-AVX512DQVL-NEXT:    vpaddw {{.*}}(%rip), %ymm1, %ymm0
-; X64-AVX512DQVL-NEXT:    vpaddw {{.*}}(%rip), %ymm1, %ymm1
+; X64-AVX512DQVL-NEXT:    vmovdqa (%rdi), %ymm0
+; X64-AVX512DQVL-NEXT:    vpaddw {{.*}}(%rip), %ymm0, %ymm1
+; X64-AVX512DQVL-NEXT:    vpaddw {{.*}}(%rip), %ymm0, %ymm0
+; X64-AVX512DQVL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; X64-AVX512DQVL-NEXT:    retq
  %1 = load <16 x i16>, <16 x i16> *%p
  %2 = shufflevector <16 x i16> %1, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -80,9 +82,10 @@ define <32 x i16> @test_broadcast_16i16_32i16(<16 x i16> *%p) nounwind {
 define <64 x i8> @test_broadcast_32i8_64i8(<32 x i8> *%p) nounwind {
 ; X64-AVX512VL-LABEL: test_broadcast_32i8_64i8:
 ; X64-AVX512VL:       ## %bb.0:
-; X64-AVX512VL-NEXT:    vmovdqa (%rdi), %ymm1
-; X64-AVX512VL-NEXT:    vpaddb {{.*}}(%rip), %ymm1, %ymm0
-; X64-AVX512VL-NEXT:    vpaddb {{.*}}(%rip), %ymm1, %ymm1
+; X64-AVX512VL-NEXT:    vmovdqa (%rdi), %ymm0
+; X64-AVX512VL-NEXT:    vpaddb {{.*}}(%rip), %ymm0, %ymm1
+; X64-AVX512VL-NEXT:    vpaddb {{.*}}(%rip), %ymm0, %ymm0
+; X64-AVX512VL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; X64-AVX512VL-NEXT:    retq
 ;
 ; X64-AVX512BWVL-LABEL: test_broadcast_32i8_64i8:
@@ -93,9 +96,10 @@ define <64 x i8> @test_broadcast_32i8_64i8(<32 x i8> *%p) nounwind {
 ;
 ; X64-AVX512DQVL-LABEL: test_broadcast_32i8_64i8:
 ; X64-AVX512DQVL:       ## %bb.0:
-; X64-AVX512DQVL-NEXT:    vmovdqa (%rdi), %ymm1
-; X64-AVX512DQVL-NEXT:    vpaddb {{.*}}(%rip), %ymm1, %ymm0
-; X64-AVX512DQVL-NEXT:    vpaddb {{.*}}(%rip), %ymm1, %ymm1
+; X64-AVX512DQVL-NEXT:    vmovdqa (%rdi), %ymm0
+; X64-AVX512DQVL-NEXT:    vpaddb {{.*}}(%rip), %ymm0, %ymm1
+; X64-AVX512DQVL-NEXT:    vpaddb {{.*}}(%rip), %ymm0, %ymm0
+; X64-AVX512DQVL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; X64-AVX512DQVL-NEXT:    retq
  %1 = load <32 x i8>, <32 x i8> *%p
  %2 = shufflevector <32 x i8> %1, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
diff --git a/llvm/test/CodeGen/X86/avx512-vec-cmp.ll b/llvm/test/CodeGen/X86/avx512-vec-cmp.ll
index 1799ebd53..7abcabe9c 100644
--- a/llvm/test/CodeGen/X86/avx512-vec-cmp.ll
+++ b/llvm/test/CodeGen/X86/avx512-vec-cmp.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-apple-darwin -show-mc-encoding -mattr=+avx512f | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=KNL
-; RUN: llc < %s -mtriple=x86_64-apple-darwin -show-mc-encoding -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512BW
-; RUN: llc < %s -mtriple=x86_64-apple-darwin -show-mc-encoding -mattr=+avx512f,+avx512bw,+avx512vl,+avx512dq | FileCheck %s --check-prefix=CHECK --check-prefix=SKX
+; RUN: llc < %s -disable-peephole -mtriple=x86_64-apple-darwin -show-mc-encoding -mattr=+avx512f | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=KNL
+; RUN: llc < %s -disable-peephole -mtriple=x86_64-apple-darwin -show-mc-encoding -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512BW
+; RUN: llc < %s -disable-peephole -mtriple=x86_64-apple-darwin -show-mc-encoding -mattr=+avx512f,+avx512bw,+avx512vl,+avx512dq | FileCheck %s --check-prefix=CHECK --check-prefix=SKX
 
 define <16 x float> @test1(<16 x float> %x, <16 x float> %y) nounwind {
 ; CHECK-LABEL: test1:
@@ -614,6 +614,30 @@ define <2 x double> @test31(<2 x double> %x, <2 x double> %x1, <2 x double>* %yp
   ret <2 x double> %max
 }
 
+define <2 x double> @test31_commute(<2 x double> %x, <2 x double> %x1, <2 x double>* %yp) nounwind {
+; AVX512-LABEL: test31_commute:
+; AVX512:       ## %bb.0:
+; AVX512-NEXT:    ## kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512-NEXT:    ## kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512-NEXT:    vmovupd (%rdi), %xmm2 ## encoding: [0xc5,0xf9,0x10,0x17]
+; AVX512-NEXT:    vcmpltpd %zmm0, %zmm2, %k1 ## encoding: [0x62,0xf1,0xed,0x48,0xc2,0xc8,0x01]
+; AVX512-NEXT:    vblendmpd %zmm0, %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf2,0xf5,0x49,0x65,0xc0]
+; AVX512-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512-NEXT:    vzeroupper ## encoding: [0xc5,0xf8,0x77]
+; AVX512-NEXT:    retq ## encoding: [0xc3]
+;
+; SKX-LABEL: test31_commute:
+; SKX:       ## %bb.0:
+; SKX-NEXT:    vcmpgtpd (%rdi), %xmm0, %k1 ## encoding: [0x62,0xf1,0xfd,0x08,0xc2,0x0f,0x0e]
+; SKX-NEXT:    vblendmpd %xmm0, %xmm1, %xmm0 {%k1} ## encoding: [0x62,0xf2,0xf5,0x09,0x65,0xc0]
+; SKX-NEXT:    retq ## encoding: [0xc3]
+
+  %y = load <2 x double>, <2 x double>* %yp, align 4
+  %mask = fcmp olt <2 x double> %y, %x
+  %max = select <2 x i1> %mask, <2 x double> %x, <2 x double> %x1
+  ret <2 x double> %max
+}
+
 define <4 x double> @test32(<4 x double> %x, <4 x double> %x1, <4 x double>* %yp) nounwind {
 ; AVX512-LABEL: test32:
 ; AVX512:       ## %bb.0:
@@ -637,6 +661,29 @@ define <4 x double> @test32(<4 x double> %x, <4 x double> %x1, <4 x double>* %yp
   ret <4 x double> %max
 }
 
+define <4 x double> @test32_commute(<4 x double> %x, <4 x double> %x1, <4 x double>* %yp) nounwind {
+; AVX512-LABEL: test32_commute:
+; AVX512:       ## %bb.0:
+; AVX512-NEXT:    ## kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512-NEXT:    ## kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512-NEXT:    vmovupd (%rdi), %ymm2 ## encoding: [0xc5,0xfd,0x10,0x17]
+; AVX512-NEXT:    vcmpltpd %zmm0, %zmm2, %k1 ## encoding: [0x62,0xf1,0xed,0x48,0xc2,0xc8,0x01]
+; AVX512-NEXT:    vblendmpd %zmm0, %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf2,0xf5,0x49,0x65,0xc0]
+; AVX512-NEXT:    ## kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512-NEXT:    retq ## encoding: [0xc3]
+;
+; SKX-LABEL: test32_commute:
+; SKX:       ## %bb.0:
+; SKX-NEXT:    vcmpgtpd (%rdi), %ymm0, %k1 ## encoding: [0x62,0xf1,0xfd,0x28,0xc2,0x0f,0x0e]
+; SKX-NEXT:    vblendmpd %ymm0, %ymm1, %ymm0 {%k1} ## encoding: [0x62,0xf2,0xf5,0x29,0x65,0xc0]
+; SKX-NEXT:    retq ## encoding: [0xc3]
+
+  %y = load <4 x double>, <4 x double>* %yp, align 4
+  %mask = fcmp ogt <4 x double> %x, %y
+  %max = select <4 x i1> %mask, <4 x double> %x, <4 x double> %x1
+  ret <4 x double> %max
+}
+
 define <8 x double> @test33(<8 x double> %x, <8 x double> %x1, <8 x double>* %yp) nounwind {
 ; CHECK-LABEL: test33:
 ; CHECK:       ## %bb.0:
@@ -649,6 +696,18 @@ define <8 x double> @test33(<8 x double> %x, <8 x double> %x1, <8 x double>* %yp
   ret <8 x double> %max
 }
 
+define <8 x double> @test33_commute(<8 x double> %x, <8 x double> %x1, <8 x double>* %yp) nounwind {
+; CHECK-LABEL: test33_commute:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vcmpgtpd (%rdi), %zmm0, %k1 ## encoding: [0x62,0xf1,0xfd,0x48,0xc2,0x0f,0x0e]
+; CHECK-NEXT:    vblendmpd %zmm0, %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf2,0xf5,0x49,0x65,0xc0]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+  %y = load <8 x double>, <8 x double>* %yp, align 4
+  %mask = fcmp olt <8 x double> %y, %x
+  %max = select <8 x i1> %mask, <8 x double> %x, <8 x double> %x1
+  ret <8 x double> %max
+}
+
 define <4 x float> @test34(<4 x float> %x, <4 x float> %x1, <4 x float>* %yp) nounwind {
 ; AVX512-LABEL: test34:
 ; AVX512:       ## %bb.0:
@@ -672,6 +731,29 @@ define <4 x float> @test34(<4 x float> %x, <4 x float> %x1, <4 x float>* %yp) no
   ret <4 x float> %max
 }
 
+define <4 x float> @test34_commute(<4 x float> %x, <4 x float> %x1, <4 x float>* %yp) nounwind {
+; AVX512-LABEL: test34_commute:
+; AVX512:       ## %bb.0:
+; AVX512-NEXT:    ## kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512-NEXT:    ## kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512-NEXT:    vmovups (%rdi), %xmm2 ## encoding: [0xc5,0xf8,0x10,0x17]
+; AVX512-NEXT:    vcmpltps %zmm0, %zmm2, %k1 ## encoding: [0x62,0xf1,0x6c,0x48,0xc2,0xc8,0x01]
+; AVX512-NEXT:    vblendmps %zmm0, %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf2,0x75,0x49,0x65,0xc0]
+; AVX512-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512-NEXT:    vzeroupper ## encoding: [0xc5,0xf8,0x77]
+; AVX512-NEXT:    retq ## encoding: [0xc3]
+;
+; SKX-LABEL: test34_commute:
+; SKX:       ## %bb.0:
+; SKX-NEXT:    vcmpgtps (%rdi), %xmm0, %k1 ## encoding: [0x62,0xf1,0x7c,0x08,0xc2,0x0f,0x0e]
+; SKX-NEXT:    vblendmps %xmm0, %xmm1, %xmm0 {%k1} ## encoding: [0x62,0xf2,0x75,0x09,0x65,0xc0]
+; SKX-NEXT:    retq ## encoding: [0xc3]
+  %y = load <4 x float>, <4 x float>* %yp, align 4
+  %mask = fcmp olt <4 x float> %y, %x
+  %max = select <4 x i1> %mask, <4 x float> %x, <4 x float> %x1
+  ret <4 x float> %max
+}
+
 define <8 x float> @test35(<8 x float> %x, <8 x float> %x1, <8 x float>* %yp) nounwind {
 ; AVX512-LABEL: test35:
 ; AVX512:       ## %bb.0:
@@ -695,6 +777,29 @@ define <8 x float> @test35(<8 x float> %x, <8 x float> %x1, <8 x float>* %yp) no
   ret <8 x float> %max
 }
 
+define <8 x float> @test35_commute(<8 x float> %x, <8 x float> %x1, <8 x float>* %yp) nounwind {
+; AVX512-LABEL: test35_commute:
+; AVX512:       ## %bb.0:
+; AVX512-NEXT:    ## kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512-NEXT:    ## kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512-NEXT:    vmovups (%rdi), %ymm2 ## encoding: [0xc5,0xfc,0x10,0x17]
+; AVX512-NEXT:    vcmpltps %zmm0, %zmm2, %k1 ## encoding: [0x62,0xf1,0x6c,0x48,0xc2,0xc8,0x01]
+; AVX512-NEXT:    vblendmps %zmm0, %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf2,0x75,0x49,0x65,0xc0]
+; AVX512-NEXT:    ## kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512-NEXT:    retq ## encoding: [0xc3]
+;
+; SKX-LABEL: test35_commute:
+; SKX:       ## %bb.0:
+; SKX-NEXT:    vcmpgtps (%rdi), %ymm0, %k1 ## encoding: [0x62,0xf1,0x7c,0x28,0xc2,0x0f,0x0e]
+; SKX-NEXT:    vblendmps %ymm0, %ymm1, %ymm0 {%k1} ## encoding: [0x62,0xf2,0x75,0x29,0x65,0xc0]
+; SKX-NEXT:    retq ## encoding: [0xc3]
+
+  %y = load <8 x float>, <8 x float>* %yp, align 4
+  %mask = fcmp ogt <8 x float> %x, %y
+  %max = select <8 x i1> %mask, <8 x float> %x, <8 x float> %x1
+  ret <8 x float> %max
+}
+
 define <16 x float> @test36(<16 x float> %x, <16 x float> %x1, <16 x float>* %yp) nounwind {
 ; CHECK-LABEL: test36:
 ; CHECK:       ## %bb.0:
@@ -707,6 +812,18 @@ define <16 x float> @test36(<16 x float> %x, <16 x float> %x1, <16 x float>* %yp
   ret <16 x float> %max
 }
 
+define <16 x float> @test36_commute(<16 x float> %x, <16 x float> %x1, <16 x float>* %yp) nounwind {
+; CHECK-LABEL: test36_commute:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vcmpgtps (%rdi), %zmm0, %k1 ## encoding: [0x62,0xf1,0x7c,0x48,0xc2,0x0f,0x0e]
+; CHECK-NEXT:    vblendmps %zmm0, %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf2,0x75,0x49,0x65,0xc0]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+  %y = load <16 x float>, <16 x float>* %yp, align 4
+  %mask = fcmp olt <16 x float> %y, %x
+  %max = select <16 x i1> %mask, <16 x float> %x, <16 x float> %x1
+  ret <16 x float> %max
+}
+
 define <8 x double> @test37(<8 x double> %x, <8 x double> %x1, double* %ptr) nounwind {
 ; CHECK-LABEL: test37:
 ; CHECK:       ## %bb.0:
@@ -723,13 +840,28 @@ define <8 x double> @test37(<8 x double> %x, <8 x double> %x1, double* %ptr) nou
   ret <8 x double> %max
 }
 
+define <8 x double> @test37_commute(<8 x double> %x, <8 x double> %x1, double* %ptr) nounwind {
+; CHECK-LABEL: test37_commute:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vcmpgtpd (%rdi){1to8}, %zmm0, %k1 ## encoding: [0x62,0xf1,0xfd,0x58,0xc2,0x0f,0x0e]
+; CHECK-NEXT:    vblendmpd %zmm0, %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf2,0xf5,0x49,0x65,0xc0]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+
+  %a = load double, double* %ptr
+  %v = insertelement <8 x double> undef, double %a, i32 0
+  %shuffle = shufflevector <8 x double> %v, <8 x double> undef, <8 x i32> zeroinitializer
+
+  %mask = fcmp ogt <8 x double> %x, %shuffle
+  %max = select <8 x i1> %mask, <8 x double> %x, <8 x double> %x1
+  ret <8 x double> %max
+}
+
 define <4 x double> @test38(<4 x double> %x, <4 x double> %x1, double* %ptr) nounwind {
 ; AVX512-LABEL: test38:
 ; AVX512:       ## %bb.0:
 ; AVX512-NEXT:    ## kill: def $ymm1 killed $ymm1 def $zmm1
 ; AVX512-NEXT:    ## kill: def $ymm0 killed $ymm0 def $zmm0
-; AVX512-NEXT:    vbroadcastsd (%rdi), %ymm2 ## encoding: [0xc4,0xe2,0x7d,0x19,0x17]
-; AVX512-NEXT:    vcmpltpd %zmm2, %zmm0, %k1 ## encoding: [0x62,0xf1,0xfd,0x48,0xc2,0xca,0x01]
+; AVX512-NEXT:    vcmpltpd (%rdi){1to8}, %zmm0, %k1 ## encoding: [0x62,0xf1,0xfd,0x58,0xc2,0x0f,0x01]
 ; AVX512-NEXT:    vblendmpd %zmm0, %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf2,0xf5,0x49,0x65,0xc0]
 ; AVX512-NEXT:    ## kill: def $ymm0 killed $ymm0 killed $zmm0
 ; AVX512-NEXT:    retq ## encoding: [0xc3]
@@ -749,14 +881,37 @@ define <4 x double> @test38(<4 x double> %x, <4 x double> %x1, double* %ptr) nou
   ret <4 x double> %max
 }
 
+define <4 x double> @test38_commute(<4 x double> %x, <4 x double> %x1, double* %ptr) nounwind {
+; AVX512-LABEL: test38_commute:
+; AVX512:       ## %bb.0:
+; AVX512-NEXT:    ## kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512-NEXT:    ## kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512-NEXT:    vcmpgtpd (%rdi){1to8}, %zmm0, %k1 ## encoding: [0x62,0xf1,0xfd,0x58,0xc2,0x0f,0x0e]
+; AVX512-NEXT:    vblendmpd %zmm0, %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf2,0xf5,0x49,0x65,0xc0]
+; AVX512-NEXT:    ## kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512-NEXT:    retq ## encoding: [0xc3]
+;
+; SKX-LABEL: test38_commute:
+; SKX:       ## %bb.0:
+; SKX-NEXT:    vcmpgtpd (%rdi){1to4}, %ymm0, %k1 ## encoding: [0x62,0xf1,0xfd,0x38,0xc2,0x0f,0x0e]
+; SKX-NEXT:    vblendmpd %ymm0, %ymm1, %ymm0 {%k1} ## encoding: [0x62,0xf2,0xf5,0x29,0x65,0xc0]
+; SKX-NEXT:    retq ## encoding: [0xc3]
+
+  %a = load double, double* %ptr
+  %v = insertelement <4 x double> undef, double %a, i32 0
+  %shuffle = shufflevector <4 x double> %v, <4 x double> undef, <4 x i32> zeroinitializer
+
+  %mask = fcmp ogt <4 x double> %x, %shuffle
+  %max = select <4 x i1> %mask, <4 x double> %x, <4 x double> %x1
+  ret <4 x double> %max
+}
+
 define <2 x double> @test39(<2 x double> %x, <2 x double> %x1, double* %ptr) nounwind {
 ; AVX512-LABEL: test39:
 ; AVX512:       ## %bb.0:
 ; AVX512-NEXT:    ## kill: def $xmm1 killed $xmm1 def $zmm1
 ; AVX512-NEXT:    ## kill: def $xmm0 killed $xmm0 def $zmm0
-; AVX512-NEXT:    vmovddup (%rdi), %xmm2 ## encoding: [0xc5,0xfb,0x12,0x17]
-; AVX512-NEXT:    ## xmm2 = mem[0,0]
-; AVX512-NEXT:    vcmpltpd %zmm2, %zmm0, %k1 ## encoding: [0x62,0xf1,0xfd,0x48,0xc2,0xca,0x01]
+; AVX512-NEXT:    vcmpltpd (%rdi){1to8}, %zmm0, %k1 ## encoding: [0x62,0xf1,0xfd,0x58,0xc2,0x0f,0x01]
 ; AVX512-NEXT:    vblendmpd %zmm0, %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf2,0xf5,0x49,0x65,0xc0]
 ; AVX512-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $zmm0
 ; AVX512-NEXT:    vzeroupper ## encoding: [0xc5,0xf8,0x77]
@@ -777,6 +932,32 @@ define <2 x double> @test39(<2 x double> %x, <2 x double> %x1, double* %ptr) nou
   ret <2 x double> %max
 }
 
+define <2 x double> @test39_commute(<2 x double> %x, <2 x double> %x1, double* %ptr) nounwind {
+; AVX512-LABEL: test39_commute:
+; AVX512:       ## %bb.0:
+; AVX512-NEXT:    ## kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512-NEXT:    ## kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512-NEXT:    vcmpgtpd (%rdi){1to8}, %zmm0, %k1 ## encoding: [0x62,0xf1,0xfd,0x58,0xc2,0x0f,0x0e]
+; AVX512-NEXT:    vblendmpd %zmm0, %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf2,0xf5,0x49,0x65,0xc0]
+; AVX512-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512-NEXT:    vzeroupper ## encoding: [0xc5,0xf8,0x77]
+; AVX512-NEXT:    retq ## encoding: [0xc3]
+;
+; SKX-LABEL: test39_commute:
+; SKX:       ## %bb.0:
+; SKX-NEXT:    vcmpgtpd (%rdi){1to2}, %xmm0, %k1 ## encoding: [0x62,0xf1,0xfd,0x18,0xc2,0x0f,0x0e]
+; SKX-NEXT:    vblendmpd %xmm0, %xmm1, %xmm0 {%k1} ## encoding: [0x62,0xf2,0xf5,0x09,0x65,0xc0]
+; SKX-NEXT:    retq ## encoding: [0xc3]
+
+  %a = load double, double* %ptr
+  %v = insertelement <2 x double> undef, double %a, i32 0
+  %shuffle = shufflevector <2 x double> %v, <2 x double> undef, <2 x i32> <i32 0, i32 0>
+
+  %mask = fcmp ogt <2 x double> %x, %shuffle
+  %max = select <2 x i1> %mask, <2 x double> %x, <2 x double> %x1
+  ret <2 x double> %max
+}
+
 
 define <16  x float> @test40(<16  x float> %x, <16  x float> %x1, float* %ptr) nounwind {
 ; CHECK-LABEL: test40:
@@ -794,13 +975,28 @@ define <16  x float> @test40(<16  x float> %x, <16  x float> %x1, float* %ptr) n
   ret <16  x float> %max
 }
 
+define <16  x float> @test40_commute(<16  x float> %x, <16  x float> %x1, float* %ptr) nounwind {
+; CHECK-LABEL: test40_commute:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vcmpgtps (%rdi){1to16}, %zmm0, %k1 ## encoding: [0x62,0xf1,0x7c,0x58,0xc2,0x0f,0x0e]
+; CHECK-NEXT:    vblendmps %zmm0, %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf2,0x75,0x49,0x65,0xc0]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+
+  %a = load float, float* %ptr
+  %v = insertelement <16  x float> undef, float %a, i32 0
+  %shuffle = shufflevector <16  x float> %v, <16  x float> undef, <16 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
+
+  %mask = fcmp ogt <16  x float> %x, %shuffle
+  %max = select <16 x i1> %mask, <16  x float> %x, <16  x float> %x1
+  ret <16  x float> %max
+}
+
 define <8  x float> @test41(<8  x float> %x, <8  x float> %x1, float* %ptr) nounwind {
 ; AVX512-LABEL: test41:
 ; AVX512:       ## %bb.0:
 ; AVX512-NEXT:    ## kill: def $ymm1 killed $ymm1 def $zmm1
 ; AVX512-NEXT:    ## kill: def $ymm0 killed $ymm0 def $zmm0
-; AVX512-NEXT:    vbroadcastss (%rdi), %ymm2 ## encoding: [0xc4,0xe2,0x7d,0x18,0x17]
-; AVX512-NEXT:    vcmpltps %zmm2, %zmm0, %k1 ## encoding: [0x62,0xf1,0x7c,0x48,0xc2,0xca,0x01]
+; AVX512-NEXT:    vcmpltps (%rdi){1to16}, %zmm0, %k1 ## encoding: [0x62,0xf1,0x7c,0x58,0xc2,0x0f,0x01]
 ; AVX512-NEXT:    vblendmps %zmm0, %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf2,0x75,0x49,0x65,0xc0]
 ; AVX512-NEXT:    ## kill: def $ymm0 killed $ymm0 killed $zmm0
 ; AVX512-NEXT:    retq ## encoding: [0xc3]
@@ -820,13 +1016,37 @@ define <8  x float> @test41(<8  x float> %x, <8  x float> %x1, float* %ptr) noun
   ret <8  x float> %max
 }
 
+define <8  x float> @test41_commute(<8  x float> %x, <8  x float> %x1, float* %ptr) nounwind {
+; AVX512-LABEL: test41_commute:
+; AVX512:       ## %bb.0:
+; AVX512-NEXT:    ## kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512-NEXT:    ## kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512-NEXT:    vcmpgtps (%rdi){1to16}, %zmm0, %k1 ## encoding: [0x62,0xf1,0x7c,0x58,0xc2,0x0f,0x0e]
+; AVX512-NEXT:    vblendmps %zmm0, %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf2,0x75,0x49,0x65,0xc0]
+; AVX512-NEXT:    ## kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512-NEXT:    retq ## encoding: [0xc3]
+;
+; SKX-LABEL: test41_commute:
+; SKX:       ## %bb.0:
+; SKX-NEXT:    vcmpgtps (%rdi){1to8}, %ymm0, %k1 ## encoding: [0x62,0xf1,0x7c,0x38,0xc2,0x0f,0x0e]
+; SKX-NEXT:    vblendmps %ymm0, %ymm1, %ymm0 {%k1} ## encoding: [0x62,0xf2,0x75,0x29,0x65,0xc0]
+; SKX-NEXT:    retq ## encoding: [0xc3]
+
+  %a = load float, float* %ptr
+  %v = insertelement <8  x float> undef, float %a, i32 0
+  %shuffle = shufflevector <8  x float> %v, <8  x float> undef, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
+
+  %mask = fcmp ogt <8  x float> %x, %shuffle
+  %max = select <8 x i1> %mask, <8  x float> %x, <8  x float> %x1
+  ret <8  x float> %max
+}
+
 define <4  x float> @test42(<4  x float> %x, <4  x float> %x1, float* %ptr) nounwind {
 ; AVX512-LABEL: test42:
 ; AVX512:       ## %bb.0:
 ; AVX512-NEXT:    ## kill: def $xmm1 killed $xmm1 def $zmm1
 ; AVX512-NEXT:    ## kill: def $xmm0 killed $xmm0 def $zmm0
-; AVX512-NEXT:    vbroadcastss (%rdi), %xmm2 ## encoding: [0xc4,0xe2,0x79,0x18,0x17]
-; AVX512-NEXT:    vcmpltps %zmm2, %zmm0, %k1 ## encoding: [0x62,0xf1,0x7c,0x48,0xc2,0xca,0x01]
+; AVX512-NEXT:    vcmpltps (%rdi){1to16}, %zmm0, %k1 ## encoding: [0x62,0xf1,0x7c,0x58,0xc2,0x0f,0x01]
 ; AVX512-NEXT:    vblendmps %zmm0, %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf2,0x75,0x49,0x65,0xc0]
 ; AVX512-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $zmm0
 ; AVX512-NEXT:    vzeroupper ## encoding: [0xc5,0xf8,0x77]
@@ -847,6 +1067,32 @@ define <4  x float> @test42(<4  x float> %x, <4  x float> %x1, float* %ptr) noun
   ret <4  x float> %max
 }
 
+define <4  x float> @test42_commute(<4  x float> %x, <4  x float> %x1, float* %ptr) nounwind {
+; AVX512-LABEL: test42_commute:
+; AVX512:       ## %bb.0:
+; AVX512-NEXT:    ## kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512-NEXT:    ## kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512-NEXT:    vcmpgtps (%rdi){1to16}, %zmm0, %k1 ## encoding: [0x62,0xf1,0x7c,0x58,0xc2,0x0f,0x0e]
+; AVX512-NEXT:    vblendmps %zmm0, %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf2,0x75,0x49,0x65,0xc0]
+; AVX512-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512-NEXT:    vzeroupper ## encoding: [0xc5,0xf8,0x77]
+; AVX512-NEXT:    retq ## encoding: [0xc3]
+;
+; SKX-LABEL: test42_commute:
+; SKX:       ## %bb.0:
+; SKX-NEXT:    vcmpgtps (%rdi){1to4}, %xmm0, %k1 ## encoding: [0x62,0xf1,0x7c,0x18,0xc2,0x0f,0x0e]
+; SKX-NEXT:    vblendmps %xmm0, %xmm1, %xmm0 {%k1} ## encoding: [0x62,0xf2,0x75,0x09,0x65,0xc0]
+; SKX-NEXT:    retq ## encoding: [0xc3]
+
+  %a = load float, float* %ptr
+  %v = insertelement <4  x float> undef, float %a, i32 0
+  %shuffle = shufflevector <4  x float> %v, <4  x float> undef, <4 x i32> <i32 0, i32 0, i32 0, i32 0>
+
+  %mask = fcmp ogt <4  x float> %x, %shuffle
+  %max = select <4 x i1> %mask, <4  x float> %x, <4  x float> %x1
+  ret <4  x float> %max
+}
+
 define <8 x double> @test43(<8 x double> %x, <8 x double> %x1, double* %ptr,<8 x i1> %mask_in) nounwind {
 ; KNL-LABEL: test43:
 ; KNL:       ## %bb.0:
@@ -883,25 +1129,53 @@ define <8 x double> @test43(<8 x double> %x, <8 x double> %x1, double* %ptr,<8 x
   ret <8 x double> %max
 }
 
+define <8 x double> @test43_commute(<8 x double> %x, <8 x double> %x1, double* %ptr,<8 x i1> %mask_in) nounwind {
+; KNL-LABEL: test43_commute:
+; KNL:       ## %bb.0:
+; KNL-NEXT:    vpmovsxwq %xmm2, %zmm2 ## encoding: [0x62,0xf2,0x7d,0x48,0x24,0xd2]
+; KNL-NEXT:    vpsllq $63, %zmm2, %zmm2 ## encoding: [0x62,0xf1,0xed,0x48,0x73,0xf2,0x3f]
+; KNL-NEXT:    vcmpgtpd (%rdi){1to8}, %zmm0, %k1 ## encoding: [0x62,0xf1,0xfd,0x58,0xc2,0x0f,0x0e]
+; KNL-NEXT:    vptestmq %zmm2, %zmm2, %k1 {%k1} ## encoding: [0x62,0xf2,0xed,0x49,0x27,0xca]
+; KNL-NEXT:    vblendmpd %zmm0, %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf2,0xf5,0x49,0x65,0xc0]
+; KNL-NEXT:    retq ## encoding: [0xc3]
+;
+; AVX512BW-LABEL: test43_commute:
+; AVX512BW:       ## %bb.0:
+; AVX512BW-NEXT:    vpsllw $15, %xmm2, %xmm2 ## encoding: [0xc5,0xe9,0x71,0xf2,0x0f]
+; AVX512BW-NEXT:    vpmovw2m %zmm2, %k1 ## encoding: [0x62,0xf2,0xfe,0x48,0x29,0xca]
+; AVX512BW-NEXT:    vcmpgtpd (%rdi){1to8}, %zmm0, %k1 {%k1} ## encoding: [0x62,0xf1,0xfd,0x59,0xc2,0x0f,0x0e]
+; AVX512BW-NEXT:    vblendmpd %zmm0, %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf2,0xf5,0x49,0x65,0xc0]
+; AVX512BW-NEXT:    retq ## encoding: [0xc3]
+;
+; SKX-LABEL: test43_commute:
+; SKX:       ## %bb.0:
+; SKX-NEXT:    vpsllw $15, %xmm2, %xmm2 ## EVEX TO VEX Compression encoding: [0xc5,0xe9,0x71,0xf2,0x0f]
+; SKX-NEXT:    vpmovw2m %xmm2, %k1 ## encoding: [0x62,0xf2,0xfe,0x08,0x29,0xca]
+; SKX-NEXT:    vcmpgtpd (%rdi){1to8}, %zmm0, %k1 {%k1} ## encoding: [0x62,0xf1,0xfd,0x59,0xc2,0x0f,0x0e]
+; SKX-NEXT:    vblendmpd %zmm0, %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf2,0xf5,0x49,0x65,0xc0]
+; SKX-NEXT:    retq ## encoding: [0xc3]
+
+  %a = load double, double* %ptr
+  %v = insertelement <8 x double> undef, double %a, i32 0
+  %shuffle = shufflevector <8 x double> %v, <8 x double> undef, <8 x i32> zeroinitializer
+
+  %mask_cmp = fcmp ogt <8 x double> %x, %shuffle
+  %mask = and <8 x i1> %mask_cmp, %mask_in
+  %max = select <8 x i1> %mask, <8 x double> %x, <8 x double> %x1
+  ret <8 x double> %max
+}
+
 define <4 x i32> @test44(<4 x i16> %x, <4 x i16> %y) #0 {
 ; AVX512-LABEL: test44:
 ; AVX512:       ## %bb.0:
-; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2 ## encoding: [0xc5,0xe9,0xef,0xd2]
-; AVX512-NEXT:    vpblendw $170, %xmm2, %xmm1, %xmm1 ## encoding: [0xc4,0xe3,0x71,0x0e,0xca,0xaa]
-; AVX512-NEXT:    ## xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
-; AVX512-NEXT:    vpblendw $170, %xmm2, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x0e,0xc2,0xaa]
-; AVX512-NEXT:    ## xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3],xmm0[4],xmm2[5],xmm0[6],xmm2[7]
-; AVX512-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0x76,0xc1]
+; AVX512-NEXT:    vpcmpeqw %xmm1, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0x75,0xc1]
+; AVX512-NEXT:    vpmovsxwd %xmm0, %xmm0 ## encoding: [0xc4,0xe2,0x79,0x23,0xc0]
 ; AVX512-NEXT:    retq ## encoding: [0xc3]
 ;
 ; SKX-LABEL: test44:
 ; SKX:       ## %bb.0:
-; SKX-NEXT:    vpxor %xmm2, %xmm2, %xmm2 ## EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xd2]
-; SKX-NEXT:    vpblendw $170, %xmm2, %xmm1, %xmm1 ## encoding: [0xc4,0xe3,0x71,0x0e,0xca,0xaa]
-; SKX-NEXT:    ## xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
-; SKX-NEXT:    vpblendw $170, %xmm2, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x0e,0xc2,0xaa]
-; SKX-NEXT:    ## xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3],xmm0[4],xmm2[5],xmm0[6],xmm2[7]
-; SKX-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0x76,0xc1]
+; SKX-NEXT:    vpcmpeqw %xmm1, %xmm0, %k0 ## encoding: [0x62,0xf1,0x7d,0x08,0x75,0xc1]
+; SKX-NEXT:    vpmovm2d %k0, %xmm0 ## encoding: [0x62,0xf2,0x7e,0x08,0x38,0xc0]
 ; SKX-NEXT:    retq ## encoding: [0xc3]
   %mask = icmp eq <4 x i16> %x, %y
   %1 = sext <4 x i1> %mask to <4 x i32>
@@ -911,23 +1185,17 @@ define <4 x i32> @test44(<4 x i16> %x, <4 x i16> %y) #0 {
 define <2 x i64> @test45(<2 x i16> %x, <2 x i16> %y) #0 {
 ; AVX512-LABEL: test45:
 ; AVX512:       ## %bb.0:
-; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2 ## encoding: [0xc5,0xe9,0xef,0xd2]
-; AVX512-NEXT:    vpblendw $17, %xmm1, %xmm2, %xmm1 ## encoding: [0xc4,0xe3,0x69,0x0e,0xc9,0x11]
-; AVX512-NEXT:    ## xmm1 = xmm1[0],xmm2[1,2,3],xmm1[4],xmm2[5,6,7]
-; AVX512-NEXT:    vpblendw $17, %xmm0, %xmm2, %xmm0 ## encoding: [0xc4,0xe3,0x69,0x0e,0xc0,0x11]
-; AVX512-NEXT:    ## xmm0 = xmm0[0],xmm2[1,2,3],xmm0[4],xmm2[5,6,7]
-; AVX512-NEXT:    vpcmpeqq %xmm1, %xmm0, %xmm0 ## encoding: [0xc4,0xe2,0x79,0x29,0xc1]
-; AVX512-NEXT:    vpsrlq $63, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0x73,0xd0,0x3f]
+; AVX512-NEXT:    vpcmpeqw %xmm1, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0x75,0xc1]
+; AVX512-NEXT:    vpmovzxwq %xmm0, %xmm0 ## encoding: [0xc4,0xe2,0x79,0x34,0xc0]
+; AVX512-NEXT:    ## xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
+; AVX512-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xdb,0x05,A,A,A,A]
+; AVX512-NEXT:    ## fixup A - offset: 4, value: LCPI59_0-4, kind: reloc_riprel_4byte
 ; AVX512-NEXT:    retq ## encoding: [0xc3]
 ;
 ; SKX-LABEL: test45:
 ; SKX:       ## %bb.0:
-; SKX-NEXT:    vpxor %xmm2, %xmm2, %xmm2 ## EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xd2]
-; SKX-NEXT:    vpblendw $17, %xmm1, %xmm2, %xmm1 ## encoding: [0xc4,0xe3,0x69,0x0e,0xc9,0x11]
-; SKX-NEXT:    ## xmm1 = xmm1[0],xmm2[1,2,3],xmm1[4],xmm2[5,6,7]
-; SKX-NEXT:    vpblendw $17, %xmm0, %xmm2, %xmm0 ## encoding: [0xc4,0xe3,0x69,0x0e,0xc0,0x11]
-; SKX-NEXT:    ## xmm0 = xmm0[0],xmm2[1,2,3],xmm0[4],xmm2[5,6,7]
-; SKX-NEXT:    vpcmpeqq %xmm1, %xmm0, %xmm0 ## encoding: [0xc4,0xe2,0x79,0x29,0xc1]
+; SKX-NEXT:    vpcmpeqw %xmm1, %xmm0, %k0 ## encoding: [0x62,0xf1,0x7d,0x08,0x75,0xc1]
+; SKX-NEXT:    vpmovm2q %k0, %xmm0 ## encoding: [0x62,0xf2,0xfe,0x08,0x38,0xc0]
 ; SKX-NEXT:    vpsrlq $63, %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0x73,0xd0,0x3f]
 ; SKX-NEXT:    retq ## encoding: [0xc3]
   %mask = icmp eq <2 x i16> %x, %y
@@ -942,7 +1210,7 @@ define <2 x i64> @test46(<2 x float> %x, <2 x float> %y) #0 {
 ; AVX512-NEXT:    vpermilps $212, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x04,0xc0,0xd4]
 ; AVX512-NEXT:    ## xmm0 = xmm0[0,1,1,3]
 ; AVX512-NEXT:    vandps {{.*}}(%rip), %xmm0, %xmm0 ## encoding: [0xc5,0xf8,0x54,0x05,A,A,A,A]
-; AVX512-NEXT:    ## fixup A - offset: 4, value: LCPI47_0-4, kind: reloc_riprel_4byte
+; AVX512-NEXT:    ## fixup A - offset: 4, value: LCPI60_0-4, kind: reloc_riprel_4byte
 ; AVX512-NEXT:    retq ## encoding: [0xc3]
 ;
 ; SKX-LABEL: test46:
@@ -1126,3 +1394,182 @@ define <2 x i64> @PR41066(<2 x i64> %t0, <2 x double> %x, <2 x double> %y) {
   %t2 = select <2 x i1> %t1, <2 x i64> <i64 undef, i64 0>, <2 x i64> zeroinitializer
   ret <2 x i64> %t2
 }
+
+define <4 x i32> @zext_bool_logic(<4 x i64> %cond1, <4 x i64> %cond2, <4 x i32> %x) {
+; AVX512-LABEL: zext_bool_logic:
+; AVX512:       ## %bb.0:
+; AVX512-NEXT:    ## kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512-NEXT:    ## kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512-NEXT:    vptestnmq %zmm0, %zmm0, %k0 ## encoding: [0x62,0xf2,0xfe,0x48,0x27,0xc0]
+; AVX512-NEXT:    vptestnmq %zmm1, %zmm1, %k1 ## encoding: [0x62,0xf2,0xf6,0x48,0x27,0xc9]
+; AVX512-NEXT:    korw %k1, %k0, %k1 ## encoding: [0xc5,0xfc,0x45,0xc9]
+; AVX512-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf3,0x7d,0xc9,0x25,0xc0,0xff]
+; AVX512-NEXT:    vpsubd %xmm0, %xmm2, %xmm0 ## encoding: [0xc5,0xe9,0xfa,0xc0]
+; AVX512-NEXT:    vzeroupper ## encoding: [0xc5,0xf8,0x77]
+; AVX512-NEXT:    retq ## encoding: [0xc3]
+;
+; SKX-LABEL: zext_bool_logic:
+; SKX:       ## %bb.0:
+; SKX-NEXT:    vptestnmq %ymm0, %ymm0, %k0 ## encoding: [0x62,0xf2,0xfe,0x28,0x27,0xc0]
+; SKX-NEXT:    vptestnmq %ymm1, %ymm1, %k1 ## encoding: [0x62,0xf2,0xf6,0x28,0x27,0xc9]
+; SKX-NEXT:    korw %k1, %k0, %k0 ## encoding: [0xc5,0xfc,0x45,0xc1]
+; SKX-NEXT:    vpmovm2d %k0, %xmm0 ## encoding: [0x62,0xf2,0x7e,0x08,0x38,0xc0]
+; SKX-NEXT:    vpsubd %xmm0, %xmm2, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfa,0xc0]
+; SKX-NEXT:    vzeroupper ## encoding: [0xc5,0xf8,0x77]
+; SKX-NEXT:    retq ## encoding: [0xc3]
+  %a = icmp eq <4 x i64> %cond1, zeroinitializer
+  %b = icmp eq <4 x i64> %cond2, zeroinitializer
+  %c = or <4 x i1> %a, %b
+  %d = zext <4 x i1> %c to <4 x i32>
+  %e = add <4 x i32> %d, %x
+  ret <4 x i32> %e
+}
+
+; This used to crash in WidenVecRes_SETCC due to generating the wrong
+; result type.
+define void @half_vec_compare(<2 x half>* %x, <2 x i8>* %y) {
+; KNL-LABEL: half_vec_compare:
+; KNL:       ## %bb.0: ## %entry
+; KNL-NEXT:    movzwl (%rdi), %eax ## encoding: [0x0f,0xb7,0x07]
+; KNL-NEXT:    movzwl 2(%rdi), %ecx ## encoding: [0x0f,0xb7,0x4f,0x02]
+; KNL-NEXT:    cwtl ## encoding: [0x98]
+; KNL-NEXT:    vmovd %eax, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6e,0xc0]
+; KNL-NEXT:    vcvtph2ps %xmm0, %xmm0 ## encoding: [0xc4,0xe2,0x79,0x13,0xc0]
+; KNL-NEXT:    movswl %cx, %eax ## encoding: [0x0f,0xbf,0xc1]
+; KNL-NEXT:    vmovd %eax, %xmm1 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6e,0xc8]
+; KNL-NEXT:    vcvtph2ps %xmm1, %xmm1 ## encoding: [0xc4,0xe2,0x79,0x13,0xc9]
+; KNL-NEXT:    vxorps %xmm2, %xmm2, %xmm2 ## encoding: [0xc5,0xe8,0x57,0xd2]
+; KNL-NEXT:    vucomiss %xmm2, %xmm1 ## EVEX TO VEX Compression encoding: [0xc5,0xf8,0x2e,0xca]
+; KNL-NEXT:    setp %al ## encoding: [0x0f,0x9a,0xc0]
+; KNL-NEXT:    setne %cl ## encoding: [0x0f,0x95,0xc1]
+; KNL-NEXT:    orb %al, %cl ## encoding: [0x08,0xc1]
+; KNL-NEXT:    kmovw %ecx, %k0 ## encoding: [0xc5,0xf8,0x92,0xc1]
+; KNL-NEXT:    kshiftlw $1, %k0, %k0 ## encoding: [0xc4,0xe3,0xf9,0x32,0xc0,0x01]
+; KNL-NEXT:    vucomiss %xmm2, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf8,0x2e,0xc2]
+; KNL-NEXT:    setp %al ## encoding: [0x0f,0x9a,0xc0]
+; KNL-NEXT:    setne %cl ## encoding: [0x0f,0x95,0xc1]
+; KNL-NEXT:    orb %al, %cl ## encoding: [0x08,0xc1]
+; KNL-NEXT:    andl $1, %ecx ## encoding: [0x83,0xe1,0x01]
+; KNL-NEXT:    kmovw %ecx, %k1 ## encoding: [0xc5,0xf8,0x92,0xc9]
+; KNL-NEXT:    korw %k0, %k1, %k1 ## encoding: [0xc5,0xf4,0x45,0xc8]
+; KNL-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf3,0x7d,0xc9,0x25,0xc0,0xff]
+; KNL-NEXT:    vpmovdw %zmm0, %ymm0 ## encoding: [0x62,0xf2,0x7e,0x48,0x33,0xc0]
+; KNL-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0x63,0xc0]
+; KNL-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xdb,0x05,A,A,A,A]
+; KNL-NEXT:    ## fixup A - offset: 4, value: LCPI68_0-4, kind: reloc_riprel_4byte
+; KNL-NEXT:    vpextrw $0, %xmm0, (%rsi) ## encoding: [0xc4,0xe3,0x79,0x15,0x06,0x00]
+; KNL-NEXT:    vzeroupper ## encoding: [0xc5,0xf8,0x77]
+; KNL-NEXT:    retq ## encoding: [0xc3]
+;
+; AVX512BW-LABEL: half_vec_compare:
+; AVX512BW:       ## %bb.0: ## %entry
+; AVX512BW-NEXT:    movzwl (%rdi), %eax ## encoding: [0x0f,0xb7,0x07]
+; AVX512BW-NEXT:    movzwl 2(%rdi), %ecx ## encoding: [0x0f,0xb7,0x4f,0x02]
+; AVX512BW-NEXT:    cwtl ## encoding: [0x98]
+; AVX512BW-NEXT:    vmovd %eax, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6e,0xc0]
+; AVX512BW-NEXT:    vcvtph2ps %xmm0, %xmm0 ## encoding: [0xc4,0xe2,0x79,0x13,0xc0]
+; AVX512BW-NEXT:    movswl %cx, %eax ## encoding: [0x0f,0xbf,0xc1]
+; AVX512BW-NEXT:    vmovd %eax, %xmm1 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6e,0xc8]
+; AVX512BW-NEXT:    vcvtph2ps %xmm1, %xmm1 ## encoding: [0xc4,0xe2,0x79,0x13,0xc9]
+; AVX512BW-NEXT:    vxorps %xmm2, %xmm2, %xmm2 ## encoding: [0xc5,0xe8,0x57,0xd2]
+; AVX512BW-NEXT:    vucomiss %xmm2, %xmm1 ## EVEX TO VEX Compression encoding: [0xc5,0xf8,0x2e,0xca]
+; AVX512BW-NEXT:    setp %al ## encoding: [0x0f,0x9a,0xc0]
+; AVX512BW-NEXT:    setne %cl ## encoding: [0x0f,0x95,0xc1]
+; AVX512BW-NEXT:    orb %al, %cl ## encoding: [0x08,0xc1]
+; AVX512BW-NEXT:    kmovd %ecx, %k0 ## encoding: [0xc5,0xfb,0x92,0xc1]
+; AVX512BW-NEXT:    kshiftlw $1, %k0, %k0 ## encoding: [0xc4,0xe3,0xf9,0x32,0xc0,0x01]
+; AVX512BW-NEXT:    vucomiss %xmm2, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf8,0x2e,0xc2]
+; AVX512BW-NEXT:    setp %al ## encoding: [0x0f,0x9a,0xc0]
+; AVX512BW-NEXT:    setne %cl ## encoding: [0x0f,0x95,0xc1]
+; AVX512BW-NEXT:    orb %al, %cl ## encoding: [0x08,0xc1]
+; AVX512BW-NEXT:    andl $1, %ecx ## encoding: [0x83,0xe1,0x01]
+; AVX512BW-NEXT:    kmovw %ecx, %k1 ## encoding: [0xc5,0xf8,0x92,0xc9]
+; AVX512BW-NEXT:    korw %k0, %k1, %k0 ## encoding: [0xc5,0xf4,0x45,0xc0]
+; AVX512BW-NEXT:    vpmovm2w %k0, %zmm0 ## encoding: [0x62,0xf2,0xfe,0x48,0x28,0xc0]
+; AVX512BW-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0x63,0xc0]
+; AVX512BW-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xdb,0x05,A,A,A,A]
+; AVX512BW-NEXT:    ## fixup A - offset: 4, value: LCPI68_0-4, kind: reloc_riprel_4byte
+; AVX512BW-NEXT:    vpextrw $0, %xmm0, (%rsi) ## EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x15,0x06,0x00]
+; AVX512BW-NEXT:    vzeroupper ## encoding: [0xc5,0xf8,0x77]
+; AVX512BW-NEXT:    retq ## encoding: [0xc3]
+;
+; SKX-LABEL: half_vec_compare:
+; SKX:       ## %bb.0: ## %entry
+; SKX-NEXT:    movzwl (%rdi), %eax ## encoding: [0x0f,0xb7,0x07]
+; SKX-NEXT:    movzwl 2(%rdi), %ecx ## encoding: [0x0f,0xb7,0x4f,0x02]
+; SKX-NEXT:    cwtl ## encoding: [0x98]
+; SKX-NEXT:    vmovd %eax, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6e,0xc0]
+; SKX-NEXT:    vcvtph2ps %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x13,0xc0]
+; SKX-NEXT:    movswl %cx, %eax ## encoding: [0x0f,0xbf,0xc1]
+; SKX-NEXT:    vmovd %eax, %xmm1 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6e,0xc8]
+; SKX-NEXT:    vcvtph2ps %xmm1, %xmm1 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x13,0xc9]
+; SKX-NEXT:    vxorps %xmm2, %xmm2, %xmm2 ## EVEX TO VEX Compression encoding: [0xc5,0xe8,0x57,0xd2]
+; SKX-NEXT:    vucomiss %xmm2, %xmm1 ## EVEX TO VEX Compression encoding: [0xc5,0xf8,0x2e,0xca]
+; SKX-NEXT:    setp %al ## encoding: [0x0f,0x9a,0xc0]
+; SKX-NEXT:    setne %cl ## encoding: [0x0f,0x95,0xc1]
+; SKX-NEXT:    orb %al, %cl ## encoding: [0x08,0xc1]
+; SKX-NEXT:    kmovd %ecx, %k0 ## encoding: [0xc5,0xfb,0x92,0xc1]
+; SKX-NEXT:    kshiftlb $1, %k0, %k0 ## encoding: [0xc4,0xe3,0x79,0x32,0xc0,0x01]
+; SKX-NEXT:    vucomiss %xmm2, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf8,0x2e,0xc2]
+; SKX-NEXT:    setp %al ## encoding: [0x0f,0x9a,0xc0]
+; SKX-NEXT:    setne %cl ## encoding: [0x0f,0x95,0xc1]
+; SKX-NEXT:    orb %al, %cl ## encoding: [0x08,0xc1]
+; SKX-NEXT:    kmovd %ecx, %k1 ## encoding: [0xc5,0xfb,0x92,0xc9]
+; SKX-NEXT:    kshiftlb $7, %k1, %k1 ## encoding: [0xc4,0xe3,0x79,0x32,0xc9,0x07]
+; SKX-NEXT:    kshiftrb $7, %k1, %k1 ## encoding: [0xc4,0xe3,0x79,0x30,0xc9,0x07]
+; SKX-NEXT:    korw %k0, %k1, %k0 ## encoding: [0xc5,0xf4,0x45,0xc0]
+; SKX-NEXT:    vpmovm2w %k0, %xmm0 ## encoding: [0x62,0xf2,0xfe,0x08,0x28,0xc0]
+; SKX-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0x63,0xc0]
+; SKX-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0xdb,0x05,A,A,A,A]
+; SKX-NEXT:    ## fixup A - offset: 4, value: LCPI68_0-4, kind: reloc_riprel_4byte
+; SKX-NEXT:    vpextrw $0, %xmm0, (%rsi) ## EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x15,0x06,0x00]
+; SKX-NEXT:    retq ## encoding: [0xc3]
+entry:
+  %0 = load <2 x half>, <2 x half>* %x
+  %1 = fcmp une <2 x half> %0, zeroinitializer
+  %2 = zext <2 x i1> %1 to <2 x i8>
+  store <2 x i8> %2, <2 x i8>* %y
+  ret void
+}
+
+; This test used to end up with the vpcmpgtb on KNL having its operands in the wrong order.
+define <8 x i64> @cmp_swap_bug(<16 x i8>* %x, <8 x i64> %y, <8 x i64> %z) {
+; KNL-LABEL: cmp_swap_bug:
+; KNL:       ## %bb.0: ## %entry
+; KNL-NEXT:    vmovdqa (%rdi), %xmm2 ## encoding: [0xc5,0xf9,0x6f,0x17]
+; KNL-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
+; KNL-NEXT:    ## encoding: [0xc4,0xe2,0x69,0x00,0x15,A,A,A,A]
+; KNL-NEXT:    ## fixup A - offset: 5, value: LCPI69_0-4, kind: reloc_riprel_4byte
+; KNL-NEXT:    vpxor %xmm3, %xmm3, %xmm3 ## encoding: [0xc5,0xe1,0xef,0xdb]
+; KNL-NEXT:    vpcmpgtb %xmm2, %xmm3, %xmm2 ## encoding: [0xc5,0xe1,0x64,0xd2]
+; KNL-NEXT:    vpmovsxbd %xmm2, %zmm2 ## encoding: [0x62,0xf2,0x7d,0x48,0x21,0xd2]
+; KNL-NEXT:    vptestmd %zmm2, %zmm2, %k1 ## encoding: [0x62,0xf2,0x6d,0x48,0x27,0xca]
+; KNL-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf2,0xf5,0x49,0x64,0xc0]
+; KNL-NEXT:    retq ## encoding: [0xc3]
+;
+; AVX512BW-LABEL: cmp_swap_bug:
+; AVX512BW:       ## %bb.0: ## %entry
+; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm2 ## encoding: [0xc5,0xf9,0x6f,0x17]
+; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
+; AVX512BW-NEXT:    ## encoding: [0xc4,0xe2,0x69,0x00,0x15,A,A,A,A]
+; AVX512BW-NEXT:    ## fixup A - offset: 5, value: LCPI69_0-4, kind: reloc_riprel_4byte
+; AVX512BW-NEXT:    vpmovb2m %zmm2, %k1 ## encoding: [0x62,0xf2,0x7e,0x48,0x29,0xca]
+; AVX512BW-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf2,0xf5,0x49,0x64,0xc0]
+; AVX512BW-NEXT:    retq ## encoding: [0xc3]
+;
+; SKX-LABEL: cmp_swap_bug:
+; SKX:       ## %bb.0: ## %entry
+; SKX-NEXT:    vmovdqa (%rdi), %xmm2 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0x17]
+; SKX-NEXT:    vpshufb {{.*}}(%rip), %xmm2, %xmm2 ## EVEX TO VEX Compression xmm2 = xmm2[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
+; SKX-NEXT:    ## encoding: [0xc4,0xe2,0x69,0x00,0x15,A,A,A,A]
+; SKX-NEXT:    ## fixup A - offset: 5, value: LCPI69_0-4, kind: reloc_riprel_4byte
+; SKX-NEXT:    vpmovb2m %xmm2, %k1 ## encoding: [0x62,0xf2,0x7e,0x08,0x29,0xca]
+; SKX-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf2,0xf5,0x49,0x64,0xc0]
+; SKX-NEXT:    retq ## encoding: [0xc3]
+entry:
+  %0 = load <16 x i8>, <16 x i8>* %x
+  %1 = shufflevector <16 x i8> %0, <16 x i8> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+  %2 = icmp slt <8 x i8> %1, zeroinitializer
+  %3 = select <8 x i1> %2, <8 x i64> %y, <8 x i64> %z
+  ret <8 x i64> %3
+}
diff --git a/llvm/test/CodeGen/X86/avx512-vec3-crash.ll b/llvm/test/CodeGen/X86/avx512-vec3-crash.ll
index 7b2513ceb..5a3bc7f8b 100644
--- a/llvm/test/CodeGen/X86/avx512-vec3-crash.ll
+++ b/llvm/test/CodeGen/X86/avx512-vec3-crash.ll
@@ -6,19 +6,15 @@ define <3 x i8 > @foo(<3 x i8>%x, <3 x i8>%a, <3 x i8>%b) {
 ; CHECK-LABEL: foo:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vmovd %edi, %xmm0
-; CHECK-NEXT:    vpinsrd $1, %esi, %xmm0, %xmm0
-; CHECK-NEXT:    vpinsrd $2, %edx, %xmm0, %xmm0
-; CHECK-NEXT:    vpslld $24, %xmm0, %xmm0
+; CHECK-NEXT:    vpinsrb $1, %esi, %xmm0, %xmm0
+; CHECK-NEXT:    vpinsrb $2, %edx, %xmm0, %xmm0
 ; CHECK-NEXT:    vmovd %ecx, %xmm1
-; CHECK-NEXT:    vpinsrd $1, %r8d, %xmm1, %xmm1
-; CHECK-NEXT:    vpsrad $24, %xmm0, %xmm0
-; CHECK-NEXT:    vpinsrd $2, %r9d, %xmm1, %xmm1
-; CHECK-NEXT:    vpslld $24, %xmm1, %xmm1
-; CHECK-NEXT:    vpsrad $24, %xmm1, %xmm1
-; CHECK-NEXT:    vpcmpgtd %xmm0, %xmm1, %xmm0
+; CHECK-NEXT:    vpinsrb $1, %r8d, %xmm1, %xmm1
+; CHECK-NEXT:    vpinsrb $2, %r9d, %xmm1, %xmm1
+; CHECK-NEXT:    vpcmpgtb %xmm0, %xmm1, %xmm0
 ; CHECK-NEXT:    vpextrb $0, %xmm0, %eax
-; CHECK-NEXT:    vpextrb $4, %xmm0, %edx
-; CHECK-NEXT:    vpextrb $8, %xmm0, %ecx
+; CHECK-NEXT:    vpextrb $1, %xmm0, %edx
+; CHECK-NEXT:    vpextrb $2, %xmm0, %ecx
 ; CHECK-NEXT:    # kill: def $al killed $al killed $eax
 ; CHECK-NEXT:    # kill: def $dl killed $dl killed $edx
 ; CHECK-NEXT:    # kill: def $cl killed $cl killed $ecx
diff --git a/llvm/test/CodeGen/X86/avx512-vselect.ll b/llvm/test/CodeGen/X86/avx512-vselect.ll
index d61e4e13d..d2c9a8e42 100644
--- a/llvm/test/CodeGen/X86/avx512-vselect.ll
+++ b/llvm/test/CodeGen/X86/avx512-vselect.ll
@@ -47,3 +47,159 @@ entry:
   %ret = select <16 x i1> %m.or, <16 x double> %a, <16 x double> %b
   ret <16 x double> %ret
 }
+
+define <16 x i64> @test3(<16 x i8> %x, <16 x i64> %a, <16 x i64> %b) {
+; CHECK-SKX-LABEL: test3:
+; CHECK-SKX:       # %bb.0:
+; CHECK-SKX-NEXT:    vptestnmb %xmm0, %xmm0, %k1
+; CHECK-SKX-NEXT:    vpblendmq %zmm1, %zmm3, %zmm0 {%k1}
+; CHECK-SKX-NEXT:    kshiftrw $8, %k1, %k1
+; CHECK-SKX-NEXT:    vpblendmq %zmm2, %zmm4, %zmm1 {%k1}
+; CHECK-SKX-NEXT:    retq
+;
+; CHECK-KNL-LABEL: test3:
+; CHECK-KNL:       # %bb.0:
+; CHECK-KNL-NEXT:    vpxor %xmm5, %xmm5, %xmm5
+; CHECK-KNL-NEXT:    vpcmpeqb %xmm5, %xmm0, %xmm0
+; CHECK-KNL-NEXT:    vpmovsxbd %xmm0, %zmm0
+; CHECK-KNL-NEXT:    vptestmd %zmm0, %zmm0, %k1
+; CHECK-KNL-NEXT:    vpblendmq %zmm1, %zmm3, %zmm0 {%k1}
+; CHECK-KNL-NEXT:    kshiftrw $8, %k1, %k1
+; CHECK-KNL-NEXT:    vpblendmq %zmm2, %zmm4, %zmm1 {%k1}
+; CHECK-KNL-NEXT:    retq
+  %c = icmp eq <16 x i8> %x, zeroinitializer
+  %ret = select <16 x i1> %c, <16 x i64> %a, <16 x i64> %b
+  ret <16 x i64> %ret
+}
+
+define <16 x i64> @test4(<16 x i16> %x, <16 x i64> %a, <16 x i64> %b) {
+; CHECK-SKX-LABEL: test4:
+; CHECK-SKX:       # %bb.0:
+; CHECK-SKX-NEXT:    vptestnmw %ymm0, %ymm0, %k1
+; CHECK-SKX-NEXT:    vpblendmq %zmm1, %zmm3, %zmm0 {%k1}
+; CHECK-SKX-NEXT:    kshiftrw $8, %k1, %k1
+; CHECK-SKX-NEXT:    vpblendmq %zmm2, %zmm4, %zmm1 {%k1}
+; CHECK-SKX-NEXT:    retq
+;
+; CHECK-KNL-LABEL: test4:
+; CHECK-KNL:       # %bb.0:
+; CHECK-KNL-NEXT:    vpxor %xmm5, %xmm5, %xmm5
+; CHECK-KNL-NEXT:    vpcmpeqw %ymm5, %ymm0, %ymm0
+; CHECK-KNL-NEXT:    vpmovsxwd %ymm0, %zmm0
+; CHECK-KNL-NEXT:    vptestmd %zmm0, %zmm0, %k1
+; CHECK-KNL-NEXT:    vpblendmq %zmm1, %zmm3, %zmm0 {%k1}
+; CHECK-KNL-NEXT:    kshiftrw $8, %k1, %k1
+; CHECK-KNL-NEXT:    vpblendmq %zmm2, %zmm4, %zmm1 {%k1}
+; CHECK-KNL-NEXT:    retq
+  %c = icmp eq <16 x i16> %x, zeroinitializer
+  %ret = select <16 x i1> %c, <16 x i64> %a, <16 x i64> %b
+  ret <16 x i64> %ret
+}
+
+define <16 x i64> @test5(<16 x i32> %x, <16 x i64> %a, <16 x i64> %b) {
+; CHECK-LABEL: test5:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vptestnmd %zmm0, %zmm0, %k1
+; CHECK-NEXT:    vpblendmq %zmm1, %zmm3, %zmm0 {%k1}
+; CHECK-NEXT:    kshiftrw $8, %k1, %k1
+; CHECK-NEXT:    vpblendmq %zmm2, %zmm4, %zmm1 {%k1}
+; CHECK-NEXT:    retq
+  %c = icmp eq <16 x i32> %x, zeroinitializer
+  %ret = select <16 x i1> %c, <16 x i64> %a, <16 x i64> %b
+  ret <16 x i64> %ret
+}
+
+define <32 x i32> @test6(<32 x i8> %x, <32 x i32> %a, <32 x i32> %b) {
+; CHECK-SKX-LABEL: test6:
+; CHECK-SKX:       # %bb.0:
+; CHECK-SKX-NEXT:    vptestnmb %ymm0, %ymm0, %k1
+; CHECK-SKX-NEXT:    vpblendmd %zmm1, %zmm3, %zmm0 {%k1}
+; CHECK-SKX-NEXT:    kshiftrd $16, %k1, %k1
+; CHECK-SKX-NEXT:    vpblendmd %zmm2, %zmm4, %zmm1 {%k1}
+; CHECK-SKX-NEXT:    retq
+;
+; CHECK-KNL-LABEL: test6:
+; CHECK-KNL:       # %bb.0:
+; CHECK-KNL-NEXT:    vpxor %xmm5, %xmm5, %xmm5
+; CHECK-KNL-NEXT:    vpcmpeqb %ymm5, %ymm0, %ymm0
+; CHECK-KNL-NEXT:    vextracti128 $1, %ymm0, %xmm5
+; CHECK-KNL-NEXT:    vpmovsxbd %xmm5, %zmm5
+; CHECK-KNL-NEXT:    vptestmd %zmm5, %zmm5, %k1
+; CHECK-KNL-NEXT:    vpmovsxbd %xmm0, %zmm0
+; CHECK-KNL-NEXT:    vptestmd %zmm0, %zmm0, %k2
+; CHECK-KNL-NEXT:    vpblendmd %zmm1, %zmm3, %zmm0 {%k2}
+; CHECK-KNL-NEXT:    vpblendmd %zmm2, %zmm4, %zmm1 {%k1}
+; CHECK-KNL-NEXT:    retq
+  %c = icmp eq <32 x i8> %x, zeroinitializer
+  %ret = select <32 x i1> %c, <32 x i32> %a, <32 x i32> %b
+  ret <32 x i32> %ret
+}
+
+define <32 x i32> @test7(<32 x i16> %x, <32 x i32> %a, <32 x i32> %b) {
+; CHECK-SKX-LABEL: test7:
+; CHECK-SKX:       # %bb.0:
+; CHECK-SKX-NEXT:    vptestnmw %zmm0, %zmm0, %k1
+; CHECK-SKX-NEXT:    vpblendmd %zmm1, %zmm3, %zmm0 {%k1}
+; CHECK-SKX-NEXT:    kshiftrd $16, %k1, %k1
+; CHECK-SKX-NEXT:    vpblendmd %zmm2, %zmm4, %zmm1 {%k1}
+; CHECK-SKX-NEXT:    retq
+;
+; CHECK-KNL-LABEL: test7:
+; CHECK-KNL:       # %bb.0:
+; CHECK-KNL-NEXT:    vextracti64x4 $1, %zmm0, %ymm5
+; CHECK-KNL-NEXT:    vpxor %xmm6, %xmm6, %xmm6
+; CHECK-KNL-NEXT:    vpcmpeqw %ymm6, %ymm5, %ymm5
+; CHECK-KNL-NEXT:    vpmovsxwd %ymm5, %zmm5
+; CHECK-KNL-NEXT:    vptestmd %zmm5, %zmm5, %k1
+; CHECK-KNL-NEXT:    vpcmpeqw %ymm6, %ymm0, %ymm0
+; CHECK-KNL-NEXT:    vpmovsxwd %ymm0, %zmm0
+; CHECK-KNL-NEXT:    vptestmd %zmm0, %zmm0, %k2
+; CHECK-KNL-NEXT:    vpblendmd %zmm1, %zmm3, %zmm0 {%k2}
+; CHECK-KNL-NEXT:    vpblendmd %zmm2, %zmm4, %zmm1 {%k1}
+; CHECK-KNL-NEXT:    retq
+  %c = icmp eq <32 x i16> %x, zeroinitializer
+  %ret = select <32 x i1> %c, <32 x i32> %a, <32 x i32> %b
+  ret <32 x i32> %ret
+}
+
+define <64 x i16> @test8(<64 x i8> %x, <64 x i16> %a, <64 x i16> %b) {
+; CHECK-SKX-LABEL: test8:
+; CHECK-SKX:       # %bb.0:
+; CHECK-SKX-NEXT:    vptestnmb %zmm0, %zmm0, %k1
+; CHECK-SKX-NEXT:    vpblendmw %zmm1, %zmm3, %zmm0 {%k1}
+; CHECK-SKX-NEXT:    kshiftrq $32, %k1, %k1
+; CHECK-SKX-NEXT:    vpblendmw %zmm2, %zmm4, %zmm1 {%k1}
+; CHECK-SKX-NEXT:    retq
+;
+; CHECK-KNL-LABEL: test8:
+; CHECK-KNL:       # %bb.0:
+; CHECK-KNL-NEXT:    pushq %rbp
+; CHECK-KNL-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-KNL-NEXT:    .cfi_offset %rbp, -16
+; CHECK-KNL-NEXT:    movq %rsp, %rbp
+; CHECK-KNL-NEXT:    .cfi_def_cfa_register %rbp
+; CHECK-KNL-NEXT:    andq $-32, %rsp
+; CHECK-KNL-NEXT:    subq $32, %rsp
+; CHECK-KNL-NEXT:    vextracti64x4 $1, %zmm0, %ymm8
+; CHECK-KNL-NEXT:    vmovdqa 16(%rbp), %ymm9
+; CHECK-KNL-NEXT:    vpxor %xmm10, %xmm10, %xmm10
+; CHECK-KNL-NEXT:    vpcmpeqb %ymm0, %ymm10, %ymm11
+; CHECK-KNL-NEXT:    vpmovsxbw %xmm11, %ymm0
+; CHECK-KNL-NEXT:    vpblendvb %ymm0, %ymm1, %ymm5, %ymm0
+; CHECK-KNL-NEXT:    vextracti128 $1, %ymm11, %xmm1
+; CHECK-KNL-NEXT:    vpmovsxbw %xmm1, %ymm1
+; CHECK-KNL-NEXT:    vpblendvb %ymm1, %ymm2, %ymm6, %ymm1
+; CHECK-KNL-NEXT:    vpcmpeqb %ymm10, %ymm8, %ymm5
+; CHECK-KNL-NEXT:    vpmovsxbw %xmm5, %ymm2
+; CHECK-KNL-NEXT:    vpblendvb %ymm2, %ymm3, %ymm7, %ymm2
+; CHECK-KNL-NEXT:    vextracti128 $1, %ymm5, %xmm3
+; CHECK-KNL-NEXT:    vpmovsxbw %xmm3, %ymm3
+; CHECK-KNL-NEXT:    vpblendvb %ymm3, %ymm4, %ymm9, %ymm3
+; CHECK-KNL-NEXT:    movq %rbp, %rsp
+; CHECK-KNL-NEXT:    popq %rbp
+; CHECK-KNL-NEXT:    .cfi_def_cfa %rsp, 8
+; CHECK-KNL-NEXT:    retq
+  %c = icmp eq <64 x i8> %x, zeroinitializer
+  %ret = select <64 x i1> %c, <64 x i16> %a, <64 x i16> %b
+  ret <64 x i16> %ret
+}
diff --git a/llvm/test/CodeGen/X86/avx512bw-intrinsics-upgrade.ll b/llvm/test/CodeGen/X86/avx512bw-intrinsics-upgrade.ll
index beea392fc..0ca1f7be8 100644
--- a/llvm/test/CodeGen/X86/avx512bw-intrinsics-upgrade.ll
+++ b/llvm/test/CodeGen/X86/avx512bw-intrinsics-upgrade.ll
@@ -49,11 +49,10 @@ declare <64 x i8> @llvm.x86.avx512.mask.pbroadcast.b.gpr.512(i8, <64 x i8>, i64)
   define <64 x i8>@test_int_x86_avx512_mask_pbroadcast_b_gpr_512(i8 %x0, <64 x i8> %x1, i64 %mask) {
 ; X86-LABEL: test_int_x86_avx512_mask_pbroadcast_b_gpr_512:
 ; X86:       # %bb.0:
-; X86-NEXT:    movb {{[0-9]+}}(%esp), %al # encoding: [0x8a,0x44,0x24,0x04]
-; X86-NEXT:    vpbroadcastb %eax, %zmm1 # encoding: [0x62,0xf2,0x7d,0x48,0x7a,0xc8]
+; X86-NEXT:    vpbroadcastb {{[0-9]+}}(%esp), %zmm1 # encoding: [0x62,0xf2,0x7d,0x48,0x78,0x4c,0x24,0x04]
 ; X86-NEXT:    kmovq {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf8,0x90,0x4c,0x24,0x08]
-; X86-NEXT:    vpbroadcastb %eax, %zmm0 {%k1} # encoding: [0x62,0xf2,0x7d,0x49,0x7a,0xc0]
-; X86-NEXT:    vpbroadcastb %eax, %zmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xc9,0x7a,0xd0]
+; X86-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k1} # encoding: [0x62,0xf1,0x7f,0x49,0x6f,0xc1]
+; X86-NEXT:    vmovdqu8 %zmm1, %zmm2 {%k1} {z} # encoding: [0x62,0xf1,0x7f,0xc9,0x6f,0xd1]
 ; X86-NEXT:    vpaddb %zmm2, %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xfc,0xc2]
 ; X86-NEXT:    vpaddb %zmm0, %zmm1, %zmm0 # encoding: [0x62,0xf1,0x75,0x48,0xfc,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
@@ -79,11 +78,10 @@ declare <32 x i16> @llvm.x86.avx512.mask.pbroadcast.w.gpr.512(i16, <32 x i16>, i
   define <32 x i16>@test_int_x86_avx512_mask_pbroadcast_w_gpr_512(i16 %x0, <32 x i16> %x1, i32 %mask) {
 ; X86-LABEL: test_int_x86_avx512_mask_pbroadcast_w_gpr_512:
 ; X86:       # %bb.0:
-; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb7,0x44,0x24,0x04]
-; X86-NEXT:    vpbroadcastw %eax, %zmm1 # encoding: [0x62,0xf2,0x7d,0x48,0x7b,0xc8]
+; X86-NEXT:    vpbroadcastw {{[0-9]+}}(%esp), %zmm1 # encoding: [0x62,0xf2,0x7d,0x48,0x79,0x4c,0x24,0x02]
 ; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x08]
-; X86-NEXT:    vpbroadcastw %eax, %zmm0 {%k1} # encoding: [0x62,0xf2,0x7d,0x49,0x7b,0xc0]
-; X86-NEXT:    vpbroadcastw %eax, %zmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xc9,0x7b,0xd0]
+; X86-NEXT:    vmovdqu16 %zmm1, %zmm0 {%k1} # encoding: [0x62,0xf1,0xff,0x49,0x6f,0xc1]
+; X86-NEXT:    vmovdqu16 %zmm1, %zmm2 {%k1} {z} # encoding: [0x62,0xf1,0xff,0xc9,0x6f,0xd1]
 ; X86-NEXT:    vpaddw %zmm2, %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xfd,0xc2]
 ; X86-NEXT:    vpaddw %zmm0, %zmm1, %zmm0 # encoding: [0x62,0xf1,0x75,0x48,0xfd,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
diff --git a/llvm/test/CodeGen/X86/avx512bw-mask-op.ll b/llvm/test/CodeGen/X86/avx512bw-mask-op.ll
index c6a2da131..dcdbb1629 100644
--- a/llvm/test/CodeGen/X86/avx512bw-mask-op.ll
+++ b/llvm/test/CodeGen/X86/avx512bw-mask-op.ll
@@ -238,3 +238,51 @@ define i64 @test_v64i1_mul(i64 %x, i64 %y) {
   %ret = bitcast <64 x i1> %m2 to i64
   ret i64 %ret
 }
+
+define <32 x i1> @bitcast_f32_to_v32i1(float %x) {
+; CHECK-LABEL: bitcast_f32_to_v32i1:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vmovd %xmm0, %eax
+; CHECK-NEXT:    kmovd %eax, %k0
+; CHECK-NEXT:    vpmovm2b %k0, %ymm0
+; CHECK-NEXT:    retq
+  %a = bitcast float %x to <32 x i1>
+  ret <32 x i1> %a
+}
+
+define <64 x i1> @bitcast_f64_to_v64i1(double %x) {
+; CHECK-LABEL: bitcast_f64_to_v64i1:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vmovq %xmm0, %rax
+; CHECK-NEXT:    kmovq %rax, %k0
+; CHECK-NEXT:    vpmovm2b %k0, %zmm0
+; CHECK-NEXT:    retq
+  %a = bitcast double %x to <64 x i1>
+  ret <64 x i1> %a
+}
+
+define float @bitcast_v32i1_to_f32(<32 x i1> %x) {
+; CHECK-LABEL: bitcast_v32i1_to_f32:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vpsllw $7, %ymm0, %ymm0
+; CHECK-NEXT:    vpmovmskb %ymm0, %eax
+; CHECK-NEXT:    vmovd %eax, %xmm0
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %a = bitcast <32 x i1> %x to float
+  ret float %a
+}
+
+define double @bitcast_v64i1_to_f64(<64 x i1> %x) {
+; CHECK-LABEL: bitcast_v64i1_to_f64:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vpsllw $7, %zmm0, %zmm0
+; CHECK-NEXT:    vpmovb2m %zmm0, %k0
+; CHECK-NEXT:    kmovq %k0, %rax
+; CHECK-NEXT:    vmovq %rax, %xmm0
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %a = bitcast <64 x i1> %x to double
+  ret double %a
+}
+
diff --git a/llvm/test/CodeGen/X86/avx512bwvl-intrinsics-upgrade.ll b/llvm/test/CodeGen/X86/avx512bwvl-intrinsics-upgrade.ll
index 0c20b2679..8907578cd 100644
--- a/llvm/test/CodeGen/X86/avx512bwvl-intrinsics-upgrade.ll
+++ b/llvm/test/CodeGen/X86/avx512bwvl-intrinsics-upgrade.ll
@@ -7,11 +7,10 @@ declare <16 x i8> @llvm.x86.avx512.mask.pbroadcast.b.gpr.128(i8, <16 x i8>, i16)
 define <16 x i8>@test_int_x86_avx512_mask_pbroadcast_b_gpr_128(i8 %x0, <16 x i8> %x1, i16 %mask) {
 ; X86-LABEL: test_int_x86_avx512_mask_pbroadcast_b_gpr_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    movb {{[0-9]+}}(%esp), %al # encoding: [0x8a,0x44,0x24,0x04]
-; X86-NEXT:    vpbroadcastb %eax, %xmm1 # encoding: [0x62,0xf2,0x7d,0x08,0x7a,0xc8]
+; X86-NEXT:    vpbroadcastb {{[0-9]+}}(%esp), %xmm1 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x78,0x4c,0x24,0x04]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
-; X86-NEXT:    vpbroadcastb %eax, %xmm0 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x7a,0xc0]
-; X86-NEXT:    vpbroadcastb %eax, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0x89,0x7a,0xd0]
+; X86-NEXT:    vmovdqu8 %xmm1, %xmm0 {%k1} # encoding: [0x62,0xf1,0x7f,0x09,0x6f,0xc1]
+; X86-NEXT:    vmovdqu8 %xmm1, %xmm2 {%k1} {z} # encoding: [0x62,0xf1,0x7f,0x89,0x6f,0xd1]
 ; X86-NEXT:    vpaddb %xmm2, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfc,0xc2]
 ; X86-NEXT:    vpaddb %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfc,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
@@ -39,12 +38,11 @@ declare <8 x i16> @llvm.x86.avx512.mask.pbroadcast.w.gpr.128(i16, <8 x i16>, i8)
 define <8 x i16>@test_int_x86_avx512_mask_pbroadcast_w_gpr_128(i16 %x0, <8 x i16> %x1, i8 %mask) {
 ; X86-LABEL: test_int_x86_avx512_mask_pbroadcast_w_gpr_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb7,0x44,0x24,0x04]
-; X86-NEXT:    vpbroadcastw %eax, %xmm1 # encoding: [0x62,0xf2,0x7d,0x08,0x7b,0xc8]
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
-; X86-NEXT:    kmovd %ecx, %k1 # encoding: [0xc5,0xfb,0x92,0xc9]
-; X86-NEXT:    vpbroadcastw %eax, %xmm0 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x7b,0xc0]
-; X86-NEXT:    vpbroadcastw %eax, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0x89,0x7b,0xd0]
+; X86-NEXT:    vpbroadcastw {{[0-9]+}}(%esp), %xmm1 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x79,0x4c,0x24,0x04]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
+; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
+; X86-NEXT:    vmovdqu16 %xmm1, %xmm0 {%k1} # encoding: [0x62,0xf1,0xff,0x09,0x6f,0xc1]
+; X86-NEXT:    vmovdqu16 %xmm1, %xmm2 {%k1} {z} # encoding: [0x62,0xf1,0xff,0x89,0x6f,0xd1]
 ; X86-NEXT:    vpaddw %xmm2, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfd,0xc2]
 ; X86-NEXT:    vpaddw %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfd,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
@@ -72,11 +70,10 @@ define <8 x i16>@test_int_x86_avx512_mask_pbroadcast_w_gpr_128(i16 %x0, <8 x i16
   define <32 x i8>@test_int_x86_avx512_mask_pbroadcast_b_gpr_256(i8 %x0, <32 x i8> %x1, i32 %mask) {
 ; X86-LABEL: test_int_x86_avx512_mask_pbroadcast_b_gpr_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    movb {{[0-9]+}}(%esp), %al # encoding: [0x8a,0x44,0x24,0x04]
-; X86-NEXT:    vpbroadcastb %eax, %ymm1 # encoding: [0x62,0xf2,0x7d,0x28,0x7a,0xc8]
+; X86-NEXT:    vpbroadcastb {{[0-9]+}}(%esp), %ymm1 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x78,0x4c,0x24,0x04]
 ; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x08]
-; X86-NEXT:    vpbroadcastb %eax, %ymm0 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x7a,0xc0]
-; X86-NEXT:    vpbroadcastb %eax, %ymm2 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xa9,0x7a,0xd0]
+; X86-NEXT:    vmovdqu8 %ymm1, %ymm0 {%k1} # encoding: [0x62,0xf1,0x7f,0x29,0x6f,0xc1]
+; X86-NEXT:    vmovdqu8 %ymm1, %ymm2 {%k1} {z} # encoding: [0x62,0xf1,0x7f,0xa9,0x6f,0xd1]
 ; X86-NEXT:    vpaddb %ymm2, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfc,0xc2]
 ; X86-NEXT:    vpaddb %ymm0, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf5,0xfc,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
@@ -105,11 +102,10 @@ declare <16 x i16> @llvm.x86.avx512.mask.pbroadcast.w.gpr.256(i16, <16 x i16>, i
   define <16 x i16>@test_int_x86_avx512_mask_pbroadcast_w_gpr_256(i16 %x0, <16 x i16> %x1, i16 %mask) {
 ; X86-LABEL: test_int_x86_avx512_mask_pbroadcast_w_gpr_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb7,0x44,0x24,0x04]
-; X86-NEXT:    vpbroadcastw %eax, %ymm1 # encoding: [0x62,0xf2,0x7d,0x28,0x7b,0xc8]
+; X86-NEXT:    vpbroadcastw {{[0-9]+}}(%esp), %ymm1 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x79,0x4c,0x24,0x04]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
-; X86-NEXT:    vpbroadcastw %eax, %ymm0 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x7b,0xc0]
-; X86-NEXT:    vpbroadcastw %eax, %ymm2 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xa9,0x7b,0xd0]
+; X86-NEXT:    vmovdqu16 %ymm1, %ymm0 {%k1} # encoding: [0x62,0xf1,0xff,0x29,0x6f,0xc1]
+; X86-NEXT:    vmovdqu16 %ymm1, %ymm2 {%k1} {z} # encoding: [0x62,0xf1,0xff,0xa9,0x6f,0xd1]
 ; X86-NEXT:    vpaddw %ymm2, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfd,0xc2]
 ; X86-NEXT:    vpaddw %ymm0, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf5,0xfd,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
@@ -5133,19 +5129,19 @@ define <8 x i8> @test_cmp_w_128(<8 x i16> %a0, <8 x i16> %a1) {
 ; CHECK-NEXT:    vpcmpgtw %xmm1, %xmm0, %k5 # encoding: [0x62,0xf1,0x7d,0x08,0x65,0xe9]
 ; CHECK-NEXT:    kmovd %k0, %eax # encoding: [0xc5,0xfb,0x93,0xc0]
 ; CHECK-NEXT:    vpxor %xmm0, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xef,0xc0]
-; CHECK-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xc4,0xc0,0x00]
+; CHECK-NEXT:    vpinsrb $0, %eax, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x00]
 ; CHECK-NEXT:    kmovd %k1, %eax # encoding: [0xc5,0xfb,0x93,0xc1]
-; CHECK-NEXT:    vpinsrw $1, %eax, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xc4,0xc0,0x01]
+; CHECK-NEXT:    vpinsrb $1, %eax, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x01]
 ; CHECK-NEXT:    kmovd %k2, %eax # encoding: [0xc5,0xfb,0x93,0xc2]
-; CHECK-NEXT:    vpinsrw $2, %eax, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xc4,0xc0,0x02]
+; CHECK-NEXT:    vpinsrb $2, %eax, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x02]
 ; CHECK-NEXT:    kmovd %k3, %eax # encoding: [0xc5,0xfb,0x93,0xc3]
-; CHECK-NEXT:    vpinsrw $4, %eax, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xc4,0xc0,0x04]
+; CHECK-NEXT:    vpinsrb $4, %eax, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x04]
 ; CHECK-NEXT:    kmovd %k4, %eax # encoding: [0xc5,0xfb,0x93,0xc4]
-; CHECK-NEXT:    vpinsrw $5, %eax, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xc4,0xc0,0x05]
+; CHECK-NEXT:    vpinsrb $5, %eax, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x05]
 ; CHECK-NEXT:    kmovd %k5, %eax # encoding: [0xc5,0xfb,0x93,0xc5]
-; CHECK-NEXT:    vpinsrw $6, %eax, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xc4,0xc0,0x06]
+; CHECK-NEXT:    vpinsrb $6, %eax, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x06]
 ; CHECK-NEXT:    movl $255, %eax # encoding: [0xb8,0xff,0x00,0x00,0x00]
-; CHECK-NEXT:    vpinsrw $7, %eax, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xc4,0xc0,0x07]
+; CHECK-NEXT:    vpinsrb $7, %eax, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x07]
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res0 = call i8 @llvm.x86.avx512.mask.cmp.w.128(<8 x i16> %a0, <8 x i16> %a1, i32 0, i8 -1)
   %vec0 = insertelement <8 x i8> undef, i8 %res0, i32 0
@@ -5169,7 +5165,7 @@ define <8 x i8> @test_cmp_w_128(<8 x i16> %a0, <8 x i16> %a1) {
 define <8 x i8> @test_mask_cmp_w_128(<8 x i16> %a0, <8 x i16> %a1, i8 %mask) {
 ; X86-LABEL: test_mask_cmp_w_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb7,0x44,0x24,0x04]
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
 ; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    vpcmpeqw %xmm1, %xmm0, %k0 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0x75,0xc1]
 ; X86-NEXT:    vpcmpgtw %xmm0, %xmm1, %k2 {%k1} # encoding: [0x62,0xf1,0x75,0x09,0x65,0xd0]
@@ -5179,18 +5175,18 @@ define <8 x i8> @test_mask_cmp_w_128(<8 x i16> %a0, <8 x i16> %a1, i8 %mask) {
 ; X86-NEXT:    vpcmpgtw %xmm1, %xmm0, %k1 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0x65,0xc9]
 ; X86-NEXT:    kmovd %k0, %ecx # encoding: [0xc5,0xfb,0x93,0xc8]
 ; X86-NEXT:    vpxor %xmm0, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xef,0xc0]
-; X86-NEXT:    vpinsrw $0, %ecx, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xc4,0xc1,0x00]
+; X86-NEXT:    vpinsrb $0, %ecx, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x20,0xc1,0x00]
 ; X86-NEXT:    kmovd %k2, %ecx # encoding: [0xc5,0xfb,0x93,0xca]
-; X86-NEXT:    vpinsrw $1, %ecx, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xc4,0xc1,0x01]
+; X86-NEXT:    vpinsrb $1, %ecx, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x20,0xc1,0x01]
 ; X86-NEXT:    kmovd %k3, %ecx # encoding: [0xc5,0xfb,0x93,0xcb]
-; X86-NEXT:    vpinsrw $2, %ecx, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xc4,0xc1,0x02]
+; X86-NEXT:    vpinsrb $2, %ecx, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x20,0xc1,0x02]
 ; X86-NEXT:    kmovd %k4, %ecx # encoding: [0xc5,0xfb,0x93,0xcc]
-; X86-NEXT:    vpinsrw $4, %ecx, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xc4,0xc1,0x04]
+; X86-NEXT:    vpinsrb $4, %ecx, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x20,0xc1,0x04]
 ; X86-NEXT:    kmovd %k5, %ecx # encoding: [0xc5,0xfb,0x93,0xcd]
-; X86-NEXT:    vpinsrw $5, %ecx, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xc4,0xc1,0x05]
+; X86-NEXT:    vpinsrb $5, %ecx, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x20,0xc1,0x05]
 ; X86-NEXT:    kmovd %k1, %ecx # encoding: [0xc5,0xfb,0x93,0xc9]
-; X86-NEXT:    vpinsrw $6, %ecx, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xc4,0xc1,0x06]
-; X86-NEXT:    vpinsrw $7, %eax, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xc4,0xc0,0x07]
+; X86-NEXT:    vpinsrb $6, %ecx, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x20,0xc1,0x06]
+; X86-NEXT:    vpinsrb $7, %eax, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x07]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_mask_cmp_w_128:
@@ -5204,18 +5200,18 @@ define <8 x i8> @test_mask_cmp_w_128(<8 x i16> %a0, <8 x i16> %a1, i8 %mask) {
 ; X64-NEXT:    vpcmpgtw %xmm1, %xmm0, %k1 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0x65,0xc9]
 ; X64-NEXT:    kmovd %k0, %eax # encoding: [0xc5,0xfb,0x93,0xc0]
 ; X64-NEXT:    vpxor %xmm0, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xef,0xc0]
-; X64-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xc4,0xc0,0x00]
+; X64-NEXT:    vpinsrb $0, %eax, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x00]
 ; X64-NEXT:    kmovd %k2, %eax # encoding: [0xc5,0xfb,0x93,0xc2]
-; X64-NEXT:    vpinsrw $1, %eax, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xc4,0xc0,0x01]
+; X64-NEXT:    vpinsrb $1, %eax, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x01]
 ; X64-NEXT:    kmovd %k3, %eax # encoding: [0xc5,0xfb,0x93,0xc3]
-; X64-NEXT:    vpinsrw $2, %eax, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xc4,0xc0,0x02]
+; X64-NEXT:    vpinsrb $2, %eax, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x02]
 ; X64-NEXT:    kmovd %k4, %eax # encoding: [0xc5,0xfb,0x93,0xc4]
-; X64-NEXT:    vpinsrw $4, %eax, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xc4,0xc0,0x04]
+; X64-NEXT:    vpinsrb $4, %eax, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x04]
 ; X64-NEXT:    kmovd %k5, %eax # encoding: [0xc5,0xfb,0x93,0xc5]
-; X64-NEXT:    vpinsrw $5, %eax, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xc4,0xc0,0x05]
+; X64-NEXT:    vpinsrb $5, %eax, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x05]
 ; X64-NEXT:    kmovd %k1, %eax # encoding: [0xc5,0xfb,0x93,0xc1]
-; X64-NEXT:    vpinsrw $6, %eax, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xc4,0xc0,0x06]
-; X64-NEXT:    vpinsrw $7, %edi, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xc4,0xc7,0x07]
+; X64-NEXT:    vpinsrb $6, %eax, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x06]
+; X64-NEXT:    vpinsrb $7, %edi, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x20,0xc7,0x07]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res0 = call i8 @llvm.x86.avx512.mask.cmp.w.128(<8 x i16> %a0, <8 x i16> %a1, i32 0, i8 %mask)
   %vec0 = insertelement <8 x i8> undef, i8 %res0, i32 0
@@ -5249,19 +5245,19 @@ define <8 x i8> @test_ucmp_w_128(<8 x i16> %a0, <8 x i16> %a1) {
 ; CHECK-NEXT:    vpcmpnleuw %xmm1, %xmm0, %k5 # encoding: [0x62,0xf3,0xfd,0x08,0x3e,0xe9,0x06]
 ; CHECK-NEXT:    kmovd %k0, %eax # encoding: [0xc5,0xfb,0x93,0xc0]
 ; CHECK-NEXT:    vpxor %xmm0, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xef,0xc0]
-; CHECK-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xc4,0xc0,0x00]
+; CHECK-NEXT:    vpinsrb $0, %eax, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x00]
 ; CHECK-NEXT:    kmovd %k1, %eax # encoding: [0xc5,0xfb,0x93,0xc1]
-; CHECK-NEXT:    vpinsrw $1, %eax, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xc4,0xc0,0x01]
+; CHECK-NEXT:    vpinsrb $1, %eax, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x01]
 ; CHECK-NEXT:    kmovd %k2, %eax # encoding: [0xc5,0xfb,0x93,0xc2]
-; CHECK-NEXT:    vpinsrw $2, %eax, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xc4,0xc0,0x02]
+; CHECK-NEXT:    vpinsrb $2, %eax, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x02]
 ; CHECK-NEXT:    kmovd %k3, %eax # encoding: [0xc5,0xfb,0x93,0xc3]
-; CHECK-NEXT:    vpinsrw $4, %eax, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xc4,0xc0,0x04]
+; CHECK-NEXT:    vpinsrb $4, %eax, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x04]
 ; CHECK-NEXT:    kmovd %k4, %eax # encoding: [0xc5,0xfb,0x93,0xc4]
-; CHECK-NEXT:    vpinsrw $5, %eax, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xc4,0xc0,0x05]
+; CHECK-NEXT:    vpinsrb $5, %eax, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x05]
 ; CHECK-NEXT:    kmovd %k5, %eax # encoding: [0xc5,0xfb,0x93,0xc5]
-; CHECK-NEXT:    vpinsrw $6, %eax, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xc4,0xc0,0x06]
+; CHECK-NEXT:    vpinsrb $6, %eax, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x06]
 ; CHECK-NEXT:    movl $255, %eax # encoding: [0xb8,0xff,0x00,0x00,0x00]
-; CHECK-NEXT:    vpinsrw $7, %eax, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xc4,0xc0,0x07]
+; CHECK-NEXT:    vpinsrb $7, %eax, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x07]
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res0 = call i8 @llvm.x86.avx512.mask.ucmp.w.128(<8 x i16> %a0, <8 x i16> %a1, i32 0, i8 -1)
   %vec0 = insertelement <8 x i8> undef, i8 %res0, i32 0
@@ -5285,7 +5281,7 @@ define <8 x i8> @test_ucmp_w_128(<8 x i16> %a0, <8 x i16> %a1) {
 define <8 x i8> @test_mask_ucmp_w_128(<8 x i16> %a0, <8 x i16> %a1, i8 %mask) {
 ; X86-LABEL: test_mask_ucmp_w_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb7,0x44,0x24,0x04]
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
 ; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    vpcmpeqw %xmm1, %xmm0, %k0 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0x75,0xc1]
 ; X86-NEXT:    vpcmpltuw %xmm1, %xmm0, %k2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x3e,0xd1,0x01]
@@ -5295,18 +5291,18 @@ define <8 x i8> @test_mask_ucmp_w_128(<8 x i16> %a0, <8 x i16> %a1, i8 %mask) {
 ; X86-NEXT:    vpcmpnleuw %xmm1, %xmm0, %k1 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x3e,0xc9,0x06]
 ; X86-NEXT:    kmovd %k0, %ecx # encoding: [0xc5,0xfb,0x93,0xc8]
 ; X86-NEXT:    vpxor %xmm0, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xef,0xc0]
-; X86-NEXT:    vpinsrw $0, %ecx, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xc4,0xc1,0x00]
+; X86-NEXT:    vpinsrb $0, %ecx, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x20,0xc1,0x00]
 ; X86-NEXT:    kmovd %k2, %ecx # encoding: [0xc5,0xfb,0x93,0xca]
-; X86-NEXT:    vpinsrw $1, %ecx, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xc4,0xc1,0x01]
+; X86-NEXT:    vpinsrb $1, %ecx, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x20,0xc1,0x01]
 ; X86-NEXT:    kmovd %k3, %ecx # encoding: [0xc5,0xfb,0x93,0xcb]
-; X86-NEXT:    vpinsrw $2, %ecx, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xc4,0xc1,0x02]
+; X86-NEXT:    vpinsrb $2, %ecx, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x20,0xc1,0x02]
 ; X86-NEXT:    kmovd %k4, %ecx # encoding: [0xc5,0xfb,0x93,0xcc]
-; X86-NEXT:    vpinsrw $4, %ecx, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xc4,0xc1,0x04]
+; X86-NEXT:    vpinsrb $4, %ecx, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x20,0xc1,0x04]
 ; X86-NEXT:    kmovd %k5, %ecx # encoding: [0xc5,0xfb,0x93,0xcd]
-; X86-NEXT:    vpinsrw $5, %ecx, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xc4,0xc1,0x05]
+; X86-NEXT:    vpinsrb $5, %ecx, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x20,0xc1,0x05]
 ; X86-NEXT:    kmovd %k1, %ecx # encoding: [0xc5,0xfb,0x93,0xc9]
-; X86-NEXT:    vpinsrw $6, %ecx, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xc4,0xc1,0x06]
-; X86-NEXT:    vpinsrw $7, %eax, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xc4,0xc0,0x07]
+; X86-NEXT:    vpinsrb $6, %ecx, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x20,0xc1,0x06]
+; X86-NEXT:    vpinsrb $7, %eax, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x07]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_mask_ucmp_w_128:
@@ -5320,18 +5316,18 @@ define <8 x i8> @test_mask_ucmp_w_128(<8 x i16> %a0, <8 x i16> %a1, i8 %mask) {
 ; X64-NEXT:    vpcmpnleuw %xmm1, %xmm0, %k1 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x3e,0xc9,0x06]
 ; X64-NEXT:    kmovd %k0, %eax # encoding: [0xc5,0xfb,0x93,0xc0]
 ; X64-NEXT:    vpxor %xmm0, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xef,0xc0]
-; X64-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xc4,0xc0,0x00]
+; X64-NEXT:    vpinsrb $0, %eax, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x00]
 ; X64-NEXT:    kmovd %k2, %eax # encoding: [0xc5,0xfb,0x93,0xc2]
-; X64-NEXT:    vpinsrw $1, %eax, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xc4,0xc0,0x01]
+; X64-NEXT:    vpinsrb $1, %eax, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x01]
 ; X64-NEXT:    kmovd %k3, %eax # encoding: [0xc5,0xfb,0x93,0xc3]
-; X64-NEXT:    vpinsrw $2, %eax, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xc4,0xc0,0x02]
+; X64-NEXT:    vpinsrb $2, %eax, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x02]
 ; X64-NEXT:    kmovd %k4, %eax # encoding: [0xc5,0xfb,0x93,0xc4]
-; X64-NEXT:    vpinsrw $4, %eax, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xc4,0xc0,0x04]
+; X64-NEXT:    vpinsrb $4, %eax, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x04]
 ; X64-NEXT:    kmovd %k5, %eax # encoding: [0xc5,0xfb,0x93,0xc5]
-; X64-NEXT:    vpinsrw $5, %eax, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xc4,0xc0,0x05]
+; X64-NEXT:    vpinsrb $5, %eax, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x05]
 ; X64-NEXT:    kmovd %k1, %eax # encoding: [0xc5,0xfb,0x93,0xc1]
-; X64-NEXT:    vpinsrw $6, %eax, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xc4,0xc0,0x06]
-; X64-NEXT:    vpinsrw $7, %edi, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xc4,0xc7,0x07]
+; X64-NEXT:    vpinsrb $6, %eax, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x06]
+; X64-NEXT:    vpinsrb $7, %edi, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x20,0xc7,0x07]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res0 = call i8 @llvm.x86.avx512.mask.ucmp.w.128(<8 x i16> %a0, <8 x i16> %a1, i32 0, i8 %mask)
   %vec0 = insertelement <8 x i8> undef, i8 %res0, i32 0
diff --git a/llvm/test/CodeGen/X86/avx512dqvl-intrinsics-upgrade.ll b/llvm/test/CodeGen/X86/avx512dqvl-intrinsics-upgrade.ll
index 63402d801..b64509858 100644
--- a/llvm/test/CodeGen/X86/avx512dqvl-intrinsics-upgrade.ll
+++ b/llvm/test/CodeGen/X86/avx512dqvl-intrinsics-upgrade.ll
@@ -2011,8 +2011,7 @@ define <8 x i64> @test_mask_mullo_epi64_rmb_512(<8 x i64> %a, i64* %ptr_b) {
 ; X86-LABEL: test_mask_mullo_epi64_rmb_512:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vpbroadcastq (%eax), %zmm1 # encoding: [0x62,0xf2,0xfd,0x48,0x59,0x08]
-; X86-NEXT:    vpmullq %zmm1, %zmm0, %zmm0 # encoding: [0x62,0xf2,0xfd,0x48,0x40,0xc1]
+; X86-NEXT:    vpmullq (%eax){1to8}, %zmm0, %zmm0 # encoding: [0x62,0xf2,0xfd,0x58,0x40,0x00]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_mask_mullo_epi64_rmb_512:
@@ -2030,9 +2029,8 @@ define <8 x i64> @test_mask_mullo_epi64_rmbk_512(<8 x i64> %a, i64* %ptr_b, <8 x
 ; X86-LABEL: test_mask_mullo_epi64_rmbk_512:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vpbroadcastq (%eax), %zmm2 # encoding: [0x62,0xf2,0xfd,0x48,0x59,0x10]
 ; X86-NEXT:    kmovb {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf9,0x90,0x4c,0x24,0x08]
-; X86-NEXT:    vpmullq %zmm2, %zmm0, %zmm1 {%k1} # encoding: [0x62,0xf2,0xfd,0x49,0x40,0xca]
+; X86-NEXT:    vpmullq (%eax){1to8}, %zmm0, %zmm1 {%k1} # encoding: [0x62,0xf2,0xfd,0x59,0x40,0x08]
 ; X86-NEXT:    vmovdqa64 %zmm1, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc1]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -2053,9 +2051,8 @@ define <8 x i64> @test_mask_mullo_epi64_rmbkz_512(<8 x i64> %a, i64* %ptr_b, i8
 ; X86-LABEL: test_mask_mullo_epi64_rmbkz_512:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vpbroadcastq (%eax), %zmm1 # encoding: [0x62,0xf2,0xfd,0x48,0x59,0x08]
 ; X86-NEXT:    kmovb {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf9,0x90,0x4c,0x24,0x08]
-; X86-NEXT:    vpmullq %zmm1, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xc9,0x40,0xc1]
+; X86-NEXT:    vpmullq (%eax){1to8}, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xd9,0x40,0x00]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_mask_mullo_epi64_rmbkz_512:
@@ -2172,8 +2169,7 @@ define <4 x i64> @test_mask_mullo_epi64_rmb_256(<4 x i64> %a, i64* %ptr_b) {
 ; X86-LABEL: test_mask_mullo_epi64_rmb_256:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vpbroadcastq (%eax), %ymm1 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x59,0x08]
-; X86-NEXT:    vpmullq %ymm1, %ymm0, %ymm0 # encoding: [0x62,0xf2,0xfd,0x28,0x40,0xc1]
+; X86-NEXT:    vpmullq (%eax){1to4}, %ymm0, %ymm0 # encoding: [0x62,0xf2,0xfd,0x38,0x40,0x00]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_mask_mullo_epi64_rmb_256:
@@ -2191,9 +2187,8 @@ define <4 x i64> @test_mask_mullo_epi64_rmbk_256(<4 x i64> %a, i64* %ptr_b, <4 x
 ; X86-LABEL: test_mask_mullo_epi64_rmbk_256:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vpbroadcastq (%eax), %ymm2 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x59,0x10]
 ; X86-NEXT:    kmovb {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf9,0x90,0x4c,0x24,0x08]
-; X86-NEXT:    vpmullq %ymm2, %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x40,0xca]
+; X86-NEXT:    vpmullq (%eax){1to4}, %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf2,0xfd,0x39,0x40,0x08]
 ; X86-NEXT:    vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -2214,9 +2209,8 @@ define <4 x i64> @test_mask_mullo_epi64_rmbkz_256(<4 x i64> %a, i64* %ptr_b, i8
 ; X86-LABEL: test_mask_mullo_epi64_rmbkz_256:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vpbroadcastq (%eax), %ymm1 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x59,0x08]
 ; X86-NEXT:    kmovb {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf9,0x90,0x4c,0x24,0x08]
-; X86-NEXT:    vpmullq %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x40,0xc1]
+; X86-NEXT:    vpmullq (%eax){1to4}, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xb9,0x40,0x00]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_mask_mullo_epi64_rmbkz_256:
@@ -2334,8 +2328,7 @@ define <2 x i64> @test_mask_mullo_epi64_rmb_128(<2 x i64> %a, i64* %ptr_b) {
 ; X86-LABEL: test_mask_mullo_epi64_rmb_128:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vpbroadcastq (%eax), %xmm1 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x59,0x08]
-; X86-NEXT:    vpmullq %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf2,0xfd,0x08,0x40,0xc1]
+; X86-NEXT:    vpmullq (%eax){1to2}, %xmm0, %xmm0 # encoding: [0x62,0xf2,0xfd,0x18,0x40,0x00]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_mask_mullo_epi64_rmb_128:
@@ -2353,9 +2346,8 @@ define <2 x i64> @test_mask_mullo_epi64_rmbk_128(<2 x i64> %a, i64* %ptr_b, <2 x
 ; X86-LABEL: test_mask_mullo_epi64_rmbk_128:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vpbroadcastq (%eax), %xmm2 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x59,0x10]
 ; X86-NEXT:    kmovb {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf9,0x90,0x4c,0x24,0x08]
-; X86-NEXT:    vpmullq %xmm2, %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x40,0xca]
+; X86-NEXT:    vpmullq (%eax){1to2}, %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0xfd,0x19,0x40,0x08]
 ; X86-NEXT:    vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -2376,9 +2368,8 @@ define <2 x i64> @test_mask_mullo_epi64_rmbkz_128(<2 x i64> %a, i64* %ptr_b, i8
 ; X86-LABEL: test_mask_mullo_epi64_rmbkz_128:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vpbroadcastq (%eax), %xmm1 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x59,0x08]
 ; X86-NEXT:    kmovb {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf9,0x90,0x4c,0x24,0x08]
-; X86-NEXT:    vpmullq %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x40,0xc1]
+; X86-NEXT:    vpmullq (%eax){1to2}, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x99,0x40,0x00]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_mask_mullo_epi64_rmbkz_128:
diff --git a/llvm/test/CodeGen/X86/avx512f-vec-test-testn.ll b/llvm/test/CodeGen/X86/avx512f-vec-test-testn.ll
index 5b1c69b68..b76f99166 100644
--- a/llvm/test/CodeGen/X86/avx512f-vec-test-testn.ll
+++ b/llvm/test/CodeGen/X86/avx512f-vec-test-testn.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefix=CHECK --check-prefix=X64
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefix=CHECK --check-prefix=X86
 
 ; Function Attrs: norecurse nounwind readnone
 define zeroext i8 @TEST_mm512_test_epi64_mask(<8 x i64> %__A, <8 x i64> %__B) local_unnamed_addr #0 {
@@ -9,7 +10,7 @@ define zeroext i8 @TEST_mm512_test_epi64_mask(<8 x i64> %__A, <8 x i64> %__B) lo
 ; CHECK-NEXT:    kmovw %k0, %eax
 ; CHECK-NEXT:    # kill: def $al killed $al killed $eax
 ; CHECK-NEXT:    vzeroupper
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
 entry:
   %and.i.i = and <8 x i64> %__B, %__A
   %0 = icmp ne <8 x i64> %and.i.i, zeroinitializer
@@ -25,7 +26,7 @@ define zeroext i8 @TEST_mm512_test_epi64_mask_2(<8 x i64> %__A, <8 x i64> %__B)
 ; CHECK-NEXT:    kmovw %k0, %eax
 ; CHECK-NEXT:    # kill: def $al killed $al killed $eax
 ; CHECK-NEXT:    vzeroupper
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
 entry:
   %and.i.i = and <8 x i64> %__B, %__A
   %0 = icmp ne <8 x i64> zeroinitializer, %and.i.i
@@ -41,7 +42,7 @@ define zeroext i16 @TEST_mm512_test_epi32_mask(<8 x i64> %__A, <8 x i64> %__B) l
 ; CHECK-NEXT:    kmovw %k0, %eax
 ; CHECK-NEXT:    # kill: def $ax killed $ax killed $eax
 ; CHECK-NEXT:    vzeroupper
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
 entry:
   %and.i.i = and <8 x i64> %__B, %__A
   %0 = bitcast <8 x i64> %and.i.i to <16 x i32>
@@ -52,14 +53,23 @@ entry:
 
 ; Function Attrs: norecurse nounwind readnone
 define zeroext i8 @TEST_mm512_mask_test_epi64_mask(i8 %__U, <8 x i64> %__A, <8 x i64> %__B) local_unnamed_addr #0 {
-; CHECK-LABEL: TEST_mm512_mask_test_epi64_mask:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vptestmq %zmm0, %zmm1, %k0
-; CHECK-NEXT:    kmovw %k0, %eax
-; CHECK-NEXT:    andb %dil, %al
-; CHECK-NEXT:    # kill: def $al killed $al killed $eax
-; CHECK-NEXT:    vzeroupper
-; CHECK-NEXT:    retq
+; X64-LABEL: TEST_mm512_mask_test_epi64_mask:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    vptestmq %zmm0, %zmm1, %k0
+; X64-NEXT:    kmovw %k0, %eax
+; X64-NEXT:    andb %dil, %al
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+;
+; X86-LABEL: TEST_mm512_mask_test_epi64_mask:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    vptestmq %zmm0, %zmm1, %k0
+; X86-NEXT:    kmovw %k0, %eax
+; X86-NEXT:    andb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    vzeroupper
+; X86-NEXT:    retl
 entry:
   %and.i.i = and <8 x i64> %__B, %__A
   %0 = icmp ne <8 x i64> %and.i.i, zeroinitializer
@@ -71,14 +81,23 @@ entry:
 
 ; Function Attrs: norecurse nounwind readnone
 define zeroext i16 @TEST_mm512_mask_test_epi32_mask(i16 %__U, <8 x i64> %__A, <8 x i64> %__B) local_unnamed_addr #0 {
-; CHECK-LABEL: TEST_mm512_mask_test_epi32_mask:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vptestmd %zmm0, %zmm1, %k0
-; CHECK-NEXT:    kmovw %k0, %eax
-; CHECK-NEXT:    andl %edi, %eax
-; CHECK-NEXT:    # kill: def $ax killed $ax killed $eax
-; CHECK-NEXT:    vzeroupper
-; CHECK-NEXT:    retq
+; X64-LABEL: TEST_mm512_mask_test_epi32_mask:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    vptestmd %zmm0, %zmm1, %k0
+; X64-NEXT:    kmovw %k0, %eax
+; X64-NEXT:    andl %edi, %eax
+; X64-NEXT:    # kill: def $ax killed $ax killed $eax
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+;
+; X86-LABEL: TEST_mm512_mask_test_epi32_mask:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    vptestmd %zmm0, %zmm1, %k0
+; X86-NEXT:    kmovw %k0, %eax
+; X86-NEXT:    andw {{[0-9]+}}(%esp), %ax
+; X86-NEXT:    # kill: def $ax killed $ax killed $eax
+; X86-NEXT:    vzeroupper
+; X86-NEXT:    retl
 entry:
   %and.i.i = and <8 x i64> %__B, %__A
   %0 = bitcast <8 x i64> %and.i.i to <16 x i32>
@@ -97,7 +116,7 @@ define zeroext i8 @TEST_mm512_testn_epi64_mask(<8 x i64> %__A, <8 x i64> %__B) l
 ; CHECK-NEXT:    kmovw %k0, %eax
 ; CHECK-NEXT:    # kill: def $al killed $al killed $eax
 ; CHECK-NEXT:    vzeroupper
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
 entry:
   %and.i.i = and <8 x i64> %__B, %__A
   %0 = icmp eq <8 x i64> %and.i.i, zeroinitializer
@@ -113,7 +132,7 @@ define zeroext i8 @TEST_mm512_testn_epi64_mask_2(<8 x i64> %__A, <8 x i64> %__B)
 ; CHECK-NEXT:    kmovw %k0, %eax
 ; CHECK-NEXT:    # kill: def $al killed $al killed $eax
 ; CHECK-NEXT:    vzeroupper
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
 entry:
   %and.i.i = and <8 x i64> %__B, %__A
   %0 = icmp eq <8 x i64> zeroinitializer, %and.i.i
@@ -129,7 +148,7 @@ define zeroext i16 @TEST_mm512_testn_epi32_mask(<8 x i64> %__A, <8 x i64> %__B)
 ; CHECK-NEXT:    kmovw %k0, %eax
 ; CHECK-NEXT:    # kill: def $ax killed $ax killed $eax
 ; CHECK-NEXT:    vzeroupper
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
 entry:
   %and.i.i = and <8 x i64> %__B, %__A
   %0 = bitcast <8 x i64> %and.i.i to <16 x i32>
@@ -140,14 +159,23 @@ entry:
 
 ; Function Attrs: norecurse nounwind readnone
 define zeroext i8 @TEST_mm512_mask_testn_epi64_mask(i8 %__U, <8 x i64> %__A, <8 x i64> %__B) local_unnamed_addr #0 {
-; CHECK-LABEL: TEST_mm512_mask_testn_epi64_mask:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vptestnmq %zmm0, %zmm1, %k0
-; CHECK-NEXT:    kmovw %k0, %eax
-; CHECK-NEXT:    andb %dil, %al
-; CHECK-NEXT:    # kill: def $al killed $al killed $eax
-; CHECK-NEXT:    vzeroupper
-; CHECK-NEXT:    retq
+; X64-LABEL: TEST_mm512_mask_testn_epi64_mask:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    vptestnmq %zmm0, %zmm1, %k0
+; X64-NEXT:    kmovw %k0, %eax
+; X64-NEXT:    andb %dil, %al
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+;
+; X86-LABEL: TEST_mm512_mask_testn_epi64_mask:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    vptestnmq %zmm0, %zmm1, %k0
+; X86-NEXT:    kmovw %k0, %eax
+; X86-NEXT:    andb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    vzeroupper
+; X86-NEXT:    retl
 entry:
   %and.i.i = and <8 x i64> %__B, %__A
   %0 = icmp eq <8 x i64> %and.i.i, zeroinitializer
@@ -159,14 +187,23 @@ entry:
 
 ; Function Attrs: norecurse nounwind readnone
 define zeroext i16 @TEST_mm512_mask_testn_epi32_mask(i16 %__U, <8 x i64> %__A, <8 x i64> %__B) local_unnamed_addr #0 {
-; CHECK-LABEL: TEST_mm512_mask_testn_epi32_mask:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vptestnmd %zmm0, %zmm1, %k0
-; CHECK-NEXT:    kmovw %k0, %eax
-; CHECK-NEXT:    andl %edi, %eax
-; CHECK-NEXT:    # kill: def $ax killed $ax killed $eax
-; CHECK-NEXT:    vzeroupper
-; CHECK-NEXT:    retq
+; X64-LABEL: TEST_mm512_mask_testn_epi32_mask:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    vptestnmd %zmm0, %zmm1, %k0
+; X64-NEXT:    kmovw %k0, %eax
+; X64-NEXT:    andl %edi, %eax
+; X64-NEXT:    # kill: def $ax killed $ax killed $eax
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+;
+; X86-LABEL: TEST_mm512_mask_testn_epi32_mask:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    vptestnmd %zmm0, %zmm1, %k0
+; X86-NEXT:    kmovw %k0, %eax
+; X86-NEXT:    andw {{[0-9]+}}(%esp), %ax
+; X86-NEXT:    # kill: def $ax killed $ax killed $eax
+; X86-NEXT:    vzeroupper
+; X86-NEXT:    retl
 entry:
   %and.i.i = and <8 x i64> %__B, %__A
   %0 = bitcast <8 x i64> %and.i.i to <16 x i32>
@@ -177,3 +214,19 @@ entry:
   ret i16 %4
 }
 
+define <2 x i64> @setcc_commute(<2 x i64> %a) {
+; CHECK-LABEL: setcc_commute:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; CHECK-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vpsubq %xmm0, %xmm1, %xmm1
+; CHECK-NEXT:    vptestnmq %zmm0, %zmm0, %k1
+; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
+; CHECK-NEXT:    vmovdqa %xmm1, %xmm0
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    ret{{[l|q]}}
+  %1 = sub <2 x i64> zeroinitializer, %a
+  %2 = icmp eq <2 x i64> %a, zeroinitializer
+  %3 = select <2 x i1> %2, <2 x i64> %a, <2 x i64> %1
+  ret <2 x i64> %3
+}
diff --git a/llvm/test/CodeGen/X86/avx512ifma-intrinsics-upgrade.ll b/llvm/test/CodeGen/X86/avx512ifma-intrinsics-upgrade.ll
index 5b90bdb83..bf85814f3 100644
--- a/llvm/test/CodeGen/X86/avx512ifma-intrinsics-upgrade.ll
+++ b/llvm/test/CodeGen/X86/avx512ifma-intrinsics-upgrade.ll
@@ -199,8 +199,7 @@ define <8 x i64>@test_int_x86_avx512_vpmadd52h_uq_512_load_bcast(<8 x i64> %x0,
 ; X86-LABEL: test_int_x86_avx512_vpmadd52h_uq_512_load_bcast:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vpbroadcastq (%eax), %zmm2 # encoding: [0x62,0xf2,0xfd,0x48,0x59,0x10]
-; X86-NEXT:    vpmadd52huq %zmm2, %zmm1, %zmm0 # encoding: [0x62,0xf2,0xf5,0x48,0xb5,0xc2]
+; X86-NEXT:    vpmadd52huq (%eax){1to8}, %zmm1, %zmm0 # encoding: [0x62,0xf2,0xf5,0x58,0xb5,0x00]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_vpmadd52h_uq_512_load_bcast:
@@ -236,8 +235,7 @@ define <8 x i64>@test_int_x86_avx512_vpmadd52h_uq_512_load_commute_bcast(<8 x i6
 ; X86-LABEL: test_int_x86_avx512_vpmadd52h_uq_512_load_commute_bcast:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vpbroadcastq (%eax), %zmm2 # encoding: [0x62,0xf2,0xfd,0x48,0x59,0x10]
-; X86-NEXT:    vpmadd52huq %zmm1, %zmm2, %zmm0 # encoding: [0x62,0xf2,0xed,0x48,0xb5,0xc1]
+; X86-NEXT:    vpmadd52huq (%eax){1to8}, %zmm1, %zmm0 # encoding: [0x62,0xf2,0xf5,0x58,0xb5,0x00]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_vpmadd52h_uq_512_load_commute_bcast:
@@ -276,10 +274,9 @@ define <8 x i64>@test_int_x86_avx512_mask_vpmadd52h_uq_512_load_bcast(<8 x i64>
 ; X86-LABEL: test_int_x86_avx512_mask_vpmadd52h_uq_512_load_bcast:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vpbroadcastq (%eax), %zmm2 # encoding: [0x62,0xf2,0xfd,0x48,0x59,0x10]
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
-; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vpmadd52huq %zmm2, %zmm1, %zmm0 {%k1} # encoding: [0x62,0xf2,0xf5,0x49,0xb5,0xc2]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
+; X86-NEXT:    kmovw %ecx, %k1 # encoding: [0xc5,0xf8,0x92,0xc9]
+; X86-NEXT:    vpmadd52huq (%eax){1to8}, %zmm1, %zmm0 {%k1} # encoding: [0x62,0xf2,0xf5,0x59,0xb5,0x00]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpmadd52h_uq_512_load_bcast:
@@ -319,10 +316,9 @@ define <8 x i64>@test_int_x86_avx512_mask_vpmadd52h_uq_512_load_commute_bcast(<8
 ; X86-LABEL: test_int_x86_avx512_mask_vpmadd52h_uq_512_load_commute_bcast:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vpbroadcastq (%eax), %zmm2 # encoding: [0x62,0xf2,0xfd,0x48,0x59,0x10]
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
-; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vpmadd52huq %zmm1, %zmm2, %zmm0 {%k1} # encoding: [0x62,0xf2,0xed,0x49,0xb5,0xc1]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
+; X86-NEXT:    kmovw %ecx, %k1 # encoding: [0xc5,0xf8,0x92,0xc9]
+; X86-NEXT:    vpmadd52huq (%eax){1to8}, %zmm1, %zmm0 {%k1} # encoding: [0x62,0xf2,0xf5,0x59,0xb5,0x00]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpmadd52h_uq_512_load_commute_bcast:
@@ -362,10 +358,9 @@ define <8 x i64>@test_int_x86_avx512_maskz_vpmadd52h_uq_512_load_bcast(<8 x i64>
 ; X86-LABEL: test_int_x86_avx512_maskz_vpmadd52h_uq_512_load_bcast:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vpbroadcastq (%eax), %zmm2 # encoding: [0x62,0xf2,0xfd,0x48,0x59,0x10]
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
-; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vpmadd52huq %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0xb5,0xc2]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
+; X86-NEXT:    kmovw %ecx, %k1 # encoding: [0xc5,0xf8,0x92,0xc9]
+; X86-NEXT:    vpmadd52huq (%eax){1to8}, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xd9,0xb5,0x00]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_maskz_vpmadd52h_uq_512_load_bcast:
@@ -405,10 +400,9 @@ define <8 x i64>@test_int_x86_avx512_maskz_vpmadd52h_uq_512_load_commute_bcast(<
 ; X86-LABEL: test_int_x86_avx512_maskz_vpmadd52h_uq_512_load_commute_bcast:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vpbroadcastq (%eax), %zmm2 # encoding: [0x62,0xf2,0xfd,0x48,0x59,0x10]
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
-; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vpmadd52huq %zmm1, %zmm2, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xed,0xc9,0xb5,0xc1]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
+; X86-NEXT:    kmovw %ecx, %k1 # encoding: [0xc5,0xf8,0x92,0xc9]
+; X86-NEXT:    vpmadd52huq (%eax){1to8}, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xd9,0xb5,0x00]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_maskz_vpmadd52h_uq_512_load_commute_bcast:
diff --git a/llvm/test/CodeGen/X86/avx512ifma-intrinsics.ll b/llvm/test/CodeGen/X86/avx512ifma-intrinsics.ll
index 077269fde..6884666a2 100644
--- a/llvm/test/CodeGen/X86/avx512ifma-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/avx512ifma-intrinsics.ll
@@ -219,8 +219,7 @@ define <8 x i64>@test_int_x86_avx512_vpmadd52h_uq_512_load_bcast(<8 x i64> %x0,
 ; X86-LABEL: test_int_x86_avx512_vpmadd52h_uq_512_load_bcast:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vpbroadcastq (%eax), %zmm2 # encoding: [0x62,0xf2,0xfd,0x48,0x59,0x10]
-; X86-NEXT:    vpmadd52huq %zmm2, %zmm1, %zmm0 # encoding: [0x62,0xf2,0xf5,0x48,0xb5,0xc2]
+; X86-NEXT:    vpmadd52huq (%eax){1to8}, %zmm1, %zmm0 # encoding: [0x62,0xf2,0xf5,0x58,0xb5,0x00]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_vpmadd52h_uq_512_load_bcast:
@@ -256,8 +255,7 @@ define <8 x i64>@test_int_x86_avx512_vpmadd52h_uq_512_load_commute_bcast(<8 x i6
 ; X86-LABEL: test_int_x86_avx512_vpmadd52h_uq_512_load_commute_bcast:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vpbroadcastq (%eax), %zmm2 # encoding: [0x62,0xf2,0xfd,0x48,0x59,0x10]
-; X86-NEXT:    vpmadd52huq %zmm1, %zmm2, %zmm0 # encoding: [0x62,0xf2,0xed,0x48,0xb5,0xc1]
+; X86-NEXT:    vpmadd52huq (%eax){1to8}, %zmm1, %zmm0 # encoding: [0x62,0xf2,0xf5,0x58,0xb5,0x00]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_vpmadd52h_uq_512_load_commute_bcast:
@@ -298,10 +296,9 @@ define <8 x i64>@test_int_x86_avx512_mask_vpmadd52h_uq_512_load_bcast(<8 x i64>
 ; X86-LABEL: test_int_x86_avx512_mask_vpmadd52h_uq_512_load_bcast:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vpbroadcastq (%eax), %zmm2 # encoding: [0x62,0xf2,0xfd,0x48,0x59,0x10]
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
-; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vpmadd52huq %zmm2, %zmm1, %zmm0 {%k1} # encoding: [0x62,0xf2,0xf5,0x49,0xb5,0xc2]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
+; X86-NEXT:    kmovw %ecx, %k1 # encoding: [0xc5,0xf8,0x92,0xc9]
+; X86-NEXT:    vpmadd52huq (%eax){1to8}, %zmm1, %zmm0 {%k1} # encoding: [0x62,0xf2,0xf5,0x59,0xb5,0x00]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpmadd52h_uq_512_load_bcast:
@@ -345,10 +342,9 @@ define <8 x i64>@test_int_x86_avx512_mask_vpmadd52h_uq_512_load_commute_bcast(<8
 ; X86-LABEL: test_int_x86_avx512_mask_vpmadd52h_uq_512_load_commute_bcast:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vpbroadcastq (%eax), %zmm2 # encoding: [0x62,0xf2,0xfd,0x48,0x59,0x10]
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
-; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vpmadd52huq %zmm1, %zmm2, %zmm0 {%k1} # encoding: [0x62,0xf2,0xed,0x49,0xb5,0xc1]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
+; X86-NEXT:    kmovw %ecx, %k1 # encoding: [0xc5,0xf8,0x92,0xc9]
+; X86-NEXT:    vpmadd52huq (%eax){1to8}, %zmm1, %zmm0 {%k1} # encoding: [0x62,0xf2,0xf5,0x59,0xb5,0x00]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpmadd52h_uq_512_load_commute_bcast:
@@ -392,10 +388,9 @@ define <8 x i64>@test_int_x86_avx512_maskz_vpmadd52h_uq_512_load_bcast(<8 x i64>
 ; X86-LABEL: test_int_x86_avx512_maskz_vpmadd52h_uq_512_load_bcast:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vpbroadcastq (%eax), %zmm2 # encoding: [0x62,0xf2,0xfd,0x48,0x59,0x10]
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
-; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vpmadd52huq %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0xb5,0xc2]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
+; X86-NEXT:    kmovw %ecx, %k1 # encoding: [0xc5,0xf8,0x92,0xc9]
+; X86-NEXT:    vpmadd52huq (%eax){1to8}, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xd9,0xb5,0x00]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_maskz_vpmadd52h_uq_512_load_bcast:
@@ -439,10 +434,9 @@ define <8 x i64>@test_int_x86_avx512_maskz_vpmadd52h_uq_512_load_commute_bcast(<
 ; X86-LABEL: test_int_x86_avx512_maskz_vpmadd52h_uq_512_load_commute_bcast:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vpbroadcastq (%eax), %zmm2 # encoding: [0x62,0xf2,0xfd,0x48,0x59,0x10]
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
-; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vpmadd52huq %zmm1, %zmm2, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xed,0xc9,0xb5,0xc1]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
+; X86-NEXT:    kmovw %ecx, %k1 # encoding: [0xc5,0xf8,0x92,0xc9]
+; X86-NEXT:    vpmadd52huq (%eax){1to8}, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xd9,0xb5,0x00]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_maskz_vpmadd52h_uq_512_load_commute_bcast:
diff --git a/llvm/test/CodeGen/X86/avx512vbmi2-funnel-shifts.ll b/llvm/test/CodeGen/X86/avx512vbmi2-funnel-shifts.ll
new file mode 100644
index 000000000..6a65d941c
--- /dev/null
+++ b/llvm/test/CodeGen/X86/avx512vbmi2-funnel-shifts.ll
@@ -0,0 +1,104 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx512vbmi2 | FileCheck %s --check-prefixes=CHECK,X86
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vbmi2 -| FileCheck %s --check-prefixes=CHECK,X64
+
+define <8 x i64> @avx512_funnel_shift_q_512(<8 x i64> %a0, <8 x i64> %a1) {
+; X86-LABEL: avx512_funnel_shift_q_512:
+; X86:       # %bb.0:
+; X86-NEXT:    vpsllvq {{\.LCPI.*}}, %zmm0, %zmm0
+; X86-NEXT:    vpsrlvq {{\.LCPI.*}}, %zmm1, %zmm1
+; X86-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; X86-NEXT:    retl
+;
+; X64-LABEL: avx512_funnel_shift_q_512:
+; X64:       # %bb.0:
+; X64-NEXT:    vpsllvq {{.*}}(%rip), %zmm0, %zmm0
+; X64-NEXT:    vpsrlvq {{.*}}(%rip), %zmm1, %zmm1
+; X64-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; X64-NEXT:    retq
+  %1 =  shl <8 x i64> %a0, <i64 31, i64 33, i64 31, i64 33, i64 31, i64 33, i64 31, i64 33>
+  %2 = lshr <8 x i64> %a1, <i64 33, i64 31, i64 33, i64 31, i64 33, i64 31, i64 33, i64 31>
+  %3 = or <8 x i64> %1, %2
+  ret <8 x i64> %3
+}
+
+define <8 x i64> @avx512_funnel_shift_q_512_splat(<8 x i64> %a0, <8 x i64> %a1) {
+; CHECK-LABEL: avx512_funnel_shift_q_512_splat:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsllq $31, %zmm0, %zmm0
+; CHECK-NEXT:    vpsrlq $33, %zmm1, %zmm1
+; CHECK-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %1 =  shl <8 x i64> %a0, <i64 31, i64 31, i64 31, i64 31, i64 31, i64 31, i64 31, i64 31>
+  %2 = lshr <8 x i64> %a1, <i64 33, i64 33, i64 33, i64 33, i64 33, i64 33, i64 33, i64 33>
+  %3 = or <8 x i64> %1, %2
+  ret <8 x i64> %3
+}
+
+define <16 x i32> @avx512_funnel_shift_d_512(<16 x i32> %a0, <16 x i32> %a1) {
+; X86-LABEL: avx512_funnel_shift_d_512:
+; X86:       # %bb.0:
+; X86-NEXT:    vpsllvd {{\.LCPI.*}}, %zmm0, %zmm0
+; X86-NEXT:    vpsrlvd {{\.LCPI.*}}, %zmm1, %zmm1
+; X86-NEXT:    vpord %zmm1, %zmm0, %zmm0
+; X86-NEXT:    retl
+;
+; X64-LABEL: avx512_funnel_shift_d_512:
+; X64:       # %bb.0:
+; X64-NEXT:    vpsllvd {{.*}}(%rip), %zmm0, %zmm0
+; X64-NEXT:    vpsrlvd {{.*}}(%rip), %zmm1, %zmm1
+; X64-NEXT:    vpord %zmm1, %zmm0, %zmm0
+; X64-NEXT:    retq
+  %1 =  shl <16 x i32> %a0, <i32 15, i32 17, i32 15, i32 17, i32 15, i32 17, i32 15, i32 17, i32 15, i32 17, i32 15, i32 17, i32 15, i32 17, i32 15, i32 17>
+  %2 = lshr <16 x i32> %a1, <i32 17, i32 15, i32 17, i32 15, i32 17, i32 15, i32 17, i32 15, i32 17, i32 15, i32 17, i32 15, i32 17, i32 15, i32 17, i32 15>
+  %3 = or <16 x i32> %1, %2
+  ret <16 x i32> %3
+}
+
+define <16 x i32> @avx512_funnel_shift_d_512_splat(<16 x i32> %a0, <16 x i32> %a1) {
+; CHECK-LABEL: avx512_funnel_shift_d_512_splat:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpslld $15, %zmm0, %zmm0
+; CHECK-NEXT:    vpsrld $17, %zmm1, %zmm1
+; CHECK-NEXT:    vpord %zmm1, %zmm0, %zmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %1 =  shl <16 x i32> %a0, <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
+  %2 = lshr <16 x i32> %a1, <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
+  %3 = or <16 x i32> %1, %2
+  ret <16 x i32> %3
+}
+
+define <32 x i16> @avx512_funnel_shift_w_512(<32 x i16> %a0, <32 x i16> %a1) {
+; X86-LABEL: avx512_funnel_shift_w_512:
+; X86:       # %bb.0:
+; X86-NEXT:    vpsllvw {{\.LCPI.*}}, %zmm0, %zmm0
+; X86-NEXT:    vpsrlvw {{\.LCPI.*}}, %zmm1, %zmm1
+; X86-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; X86-NEXT:    retl
+;
+; X64-LABEL: avx512_funnel_shift_w_512:
+; X64:       # %bb.0:
+; X64-NEXT:    vpsllvw {{.*}}(%rip), %zmm0, %zmm0
+; X64-NEXT:    vpsrlvw {{.*}}(%rip), %zmm1, %zmm1
+; X64-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; X64-NEXT:    retq
+  %1 =  shl <32 x i16> %a0, <i16 7, i16 9, i16 7, i16 9, i16 7, i16 9, i16 7, i16 9, i16 7, i16 9, i16 7, i16 9, i16 7, i16 9, i16 7, i16 9, i16 7, i16 9, i16 7, i16 9, i16 7, i16 9, i16 7, i16 9, i16 7, i16 9, i16 7, i16 9, i16 7, i16 9, i16 7, i16 9>
+  %2 = lshr <32 x i16> %a1, <i16 9, i16 7, i16 9, i16 7, i16 9, i16 7, i16 9, i16 7, i16 9, i16 7, i16 9, i16 7, i16 9, i16 7, i16 9, i16 7, i16 9, i16 7, i16 9, i16 7, i16 9, i16 7, i16 9, i16 7, i16 9, i16 7, i16 9, i16 7, i16 9, i16 7, i16 9, i16 7>
+  %3 = or <32 x i16> %1, %2
+  ret <32 x i16> %3
+}
+
+define <32 x i16> @avx512_funnel_shift_w_512_splat(<32 x i16> %a0, <32 x i16> %a1) {
+; CHECK-LABEL: avx512_funnel_shift_w_512_splat:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsllw $7, %zmm0, %zmm0
+; CHECK-NEXT:    vpsrlw $9, %zmm1, %zmm1
+; CHECK-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %1 =  shl <32 x i16> %a0, <i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7>
+  %2 = lshr <32 x i16> %a1, <i16 9, i16 9, i16 9, i16 9, i16 9, i16 9, i16 9, i16 9, i16 9, i16 9, i16 9, i16 9, i16 9, i16 9, i16 9, i16 9, i16 9, i16 9, i16 9, i16 9, i16 9, i16 9, i16 9, i16 9, i16 9, i16 9, i16 9, i16 9, i16 9, i16 9, i16 9, i16 9>
+  %3 = or <32 x i16> %1, %2
+  ret <32 x i16> %3
+}
+
+
diff --git a/llvm/test/CodeGen/X86/avx512vbmi2vl-funnel-shifts.ll b/llvm/test/CodeGen/X86/avx512vbmi2vl-funnel-shifts.ll
new file mode 100644
index 000000000..ed5fb6f5e
--- /dev/null
+++ b/llvm/test/CodeGen/X86/avx512vbmi2vl-funnel-shifts.ll
@@ -0,0 +1,203 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx512vbmi2,+avx512vl | FileCheck %s --check-prefixes=CHECK,X86
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vbmi2,+avx512vl -| FileCheck %s --check-prefixes=CHECK,X64
+
+define <2 x i64> @avx512_funnel_shift_q_128(<2 x i64> %a0, <2 x i64> %a1) {
+; X86-LABEL: avx512_funnel_shift_q_128:
+; X86:       # %bb.0:
+; X86-NEXT:    vpsllvq {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-NEXT:    vpsrlvq {{\.LCPI.*}}, %xmm1, %xmm1
+; X86-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; X86-NEXT:    retl
+;
+; X64-LABEL: avx512_funnel_shift_q_128:
+; X64:       # %bb.0:
+; X64-NEXT:    vpsllvq {{.*}}(%rip), %xmm0, %xmm0
+; X64-NEXT:    vpsrlvq {{.*}}(%rip), %xmm1, %xmm1
+; X64-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; X64-NEXT:    retq
+  %1 =  shl <2 x i64> %a0, <i64 31, i64 33>
+  %2 = lshr <2 x i64> %a1, <i64 33, i64 31>
+  %3 = or <2 x i64> %1, %2
+  ret <2 x i64> %3
+}
+
+define <4 x i64> @avx512_funnel_shift_q_256(<4 x i64> %a0, <4 x i64> %a1) {
+; X86-LABEL: avx512_funnel_shift_q_256:
+; X86:       # %bb.0:
+; X86-NEXT:    vpsllvq {{\.LCPI.*}}, %ymm0, %ymm0
+; X86-NEXT:    vpsrlvq {{\.LCPI.*}}, %ymm1, %ymm1
+; X86-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; X86-NEXT:    retl
+;
+; X64-LABEL: avx512_funnel_shift_q_256:
+; X64:       # %bb.0:
+; X64-NEXT:    vpsllvq {{.*}}(%rip), %ymm0, %ymm0
+; X64-NEXT:    vpsrlvq {{.*}}(%rip), %ymm1, %ymm1
+; X64-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; X64-NEXT:    retq
+  %1 =  shl <4 x i64> %a0, <i64 31, i64 33, i64 31, i64 33>
+  %2 = lshr <4 x i64> %a1, <i64 33, i64 31, i64 33, i64 31>
+  %3 = or <4 x i64> %1, %2
+  ret <4 x i64> %3
+}
+
+define <2 x i64> @avx512_funnel_shift_q_128_splat(<2 x i64> %a0, <2 x i64> %a1) {
+; CHECK-LABEL: avx512_funnel_shift_q_128_splat:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsllq $31, %xmm0, %xmm0
+; CHECK-NEXT:    vpsrlq $33, %xmm1, %xmm1
+; CHECK-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %1 =  shl <2 x i64> %a0, <i64 31, i64 31>
+  %2 = lshr <2 x i64> %a1, <i64 33, i64 33>
+  %3 = or <2 x i64> %1, %2
+  ret <2 x i64> %3
+}
+
+define <4 x i64> @avx512_funnel_shift_q_256_splat(<4 x i64> %a0, <4 x i64> %a1) {
+; CHECK-LABEL: avx512_funnel_shift_q_256_splat:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsllq $31, %ymm0, %ymm0
+; CHECK-NEXT:    vpsrlq $33, %ymm1, %ymm1
+; CHECK-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %1 =  shl <4 x i64> %a0, <i64 31, i64 31, i64 31, i64 31>
+  %2 = lshr <4 x i64> %a1, <i64 33, i64 33, i64 33, i64 33>
+  %3 = or <4 x i64> %1, %2
+  ret <4 x i64> %3
+}
+
+define <4 x i32> @avx512_funnel_shift_d_128(<4 x i32> %a0, <4 x i32> %a1) {
+; X86-LABEL: avx512_funnel_shift_d_128:
+; X86:       # %bb.0:
+; X86-NEXT:    vpsllvd {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-NEXT:    vpsrlvd {{\.LCPI.*}}, %xmm1, %xmm1
+; X86-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; X86-NEXT:    retl
+;
+; X64-LABEL: avx512_funnel_shift_d_128:
+; X64:       # %bb.0:
+; X64-NEXT:    vpsllvd {{.*}}(%rip), %xmm0, %xmm0
+; X64-NEXT:    vpsrlvd {{.*}}(%rip), %xmm1, %xmm1
+; X64-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; X64-NEXT:    retq
+  %1 =  shl <4 x i32> %a0, <i32 15, i32 17, i32 15, i32 17>
+  %2 = lshr <4 x i32> %a1, <i32 17, i32 15, i32 17, i32 15>
+  %3 = or <4 x i32> %1, %2
+  ret <4 x i32> %3
+}
+
+define <8 x i32> @avx512_funnel_shift_d_256(<8 x i32> %a0, <8 x i32> %a1) {
+; X86-LABEL: avx512_funnel_shift_d_256:
+; X86:       # %bb.0:
+; X86-NEXT:    vpsllvd {{\.LCPI.*}}, %ymm0, %ymm0
+; X86-NEXT:    vpsrlvd {{\.LCPI.*}}, %ymm1, %ymm1
+; X86-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; X86-NEXT:    retl
+;
+; X64-LABEL: avx512_funnel_shift_d_256:
+; X64:       # %bb.0:
+; X64-NEXT:    vpsllvd {{.*}}(%rip), %ymm0, %ymm0
+; X64-NEXT:    vpsrlvd {{.*}}(%rip), %ymm1, %ymm1
+; X64-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; X64-NEXT:    retq
+  %1 =  shl <8 x i32> %a0, <i32 15, i32 17, i32 15, i32 17, i32 15, i32 17, i32 15, i32 17>
+  %2 = lshr <8 x i32> %a1, <i32 17, i32 15, i32 17, i32 15, i32 17, i32 15, i32 17, i32 15>
+  %3 = or <8 x i32> %1, %2
+  ret <8 x i32> %3
+}
+
+define <4 x i32> @avx512_funnel_shift_d_128_splat(<4 x i32> %a0, <4 x i32> %a1) {
+; CHECK-LABEL: avx512_funnel_shift_d_128_splat:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpslld $15, %xmm0, %xmm0
+; CHECK-NEXT:    vpsrld $17, %xmm1, %xmm1
+; CHECK-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %1 =  shl <4 x i32> %a0, <i32 15, i32 15, i32 15, i32 15>
+  %2 = lshr <4 x i32> %a1, <i32 17, i32 17, i32 17, i32 17>
+  %3 = or <4 x i32> %1, %2
+  ret <4 x i32> %3
+}
+
+define <8 x i32> @avx512_funnel_shift_d_256_splat(<8 x i32> %a0, <8 x i32> %a1) {
+; CHECK-LABEL: avx512_funnel_shift_d_256_splat:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpslld $15, %ymm0, %ymm0
+; CHECK-NEXT:    vpsrld $17, %ymm1, %ymm1
+; CHECK-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %1 =  shl <8 x i32> %a0, <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
+  %2 = lshr <8 x i32> %a1, <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
+  %3 = or <8 x i32> %1, %2
+  ret <8 x i32> %3
+}
+
+define <8 x i16> @avx512_funnel_shift_w_128(<8 x i16> %a0, <8 x i16> %a1) {
+; X86-LABEL: avx512_funnel_shift_w_128:
+; X86:       # %bb.0:
+; X86-NEXT:    vpsllvw {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-NEXT:    vpsrlvw {{\.LCPI.*}}, %xmm1, %xmm1
+; X86-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; X86-NEXT:    retl
+;
+; X64-LABEL: avx512_funnel_shift_w_128:
+; X64:       # %bb.0:
+; X64-NEXT:    vpsllvw {{.*}}(%rip), %xmm0, %xmm0
+; X64-NEXT:    vpsrlvw {{.*}}(%rip), %xmm1, %xmm1
+; X64-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; X64-NEXT:    retq
+  %1 =  shl <8 x i16> %a0, <i16 7, i16 9, i16 7, i16 9, i16 7, i16 9, i16 7, i16 9>
+  %2 = lshr <8 x i16> %a1, <i16 9, i16 7, i16 9, i16 7, i16 9, i16 7, i16 9, i16 7>
+  %3 = or <8 x i16> %1, %2
+  ret <8 x i16> %3
+}
+
+define <16 x i16> @avx512_funnel_shift_w_256(<16 x i16> %a0, <16 x i16> %a1) {
+; X86-LABEL: avx512_funnel_shift_w_256:
+; X86:       # %bb.0:
+; X86-NEXT:    vpsllvw {{\.LCPI.*}}, %ymm0, %ymm0
+; X86-NEXT:    vpsrlvw {{\.LCPI.*}}, %ymm1, %ymm1
+; X86-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; X86-NEXT:    retl
+;
+; X64-LABEL: avx512_funnel_shift_w_256:
+; X64:       # %bb.0:
+; X64-NEXT:    vpsllvw {{.*}}(%rip), %ymm0, %ymm0
+; X64-NEXT:    vpsrlvw {{.*}}(%rip), %ymm1, %ymm1
+; X64-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; X64-NEXT:    retq
+  %1 =  shl <16 x i16> %a0, <i16 7, i16 9, i16 7, i16 9, i16 7, i16 9, i16 7, i16 9, i16 7, i16 9, i16 7, i16 9, i16 7, i16 9, i16 7, i16 9>
+  %2 = lshr <16 x i16> %a1, <i16 9, i16 7, i16 9, i16 7, i16 9, i16 7, i16 9, i16 7, i16 9, i16 7, i16 9, i16 7, i16 9, i16 7, i16 9, i16 7>
+  %3 = or <16 x i16> %1, %2
+  ret <16 x i16> %3
+}
+
+define <8 x i16> @avx512_funnel_shift_w_128_splat(<8 x i16> %a0, <8 x i16> %a1) {
+; CHECK-LABEL: avx512_funnel_shift_w_128_splat:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsllw $7, %xmm0, %xmm0
+; CHECK-NEXT:    vpsrlw $9, %xmm1, %xmm1
+; CHECK-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %1 =  shl <8 x i16> %a0, <i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7>
+  %2 = lshr <8 x i16> %a1, <i16 9, i16 9, i16 9, i16 9, i16 9, i16 9, i16 9, i16 9>
+  %3 = or <8 x i16> %1, %2
+  ret <8 x i16> %3
+}
+
+define <16 x i16> @avx512_funnel_shift_w_256_splat(<16 x i16> %a0, <16 x i16> %a1) {
+; CHECK-LABEL: avx512_funnel_shift_w_256_splat:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsllw $7, %ymm0, %ymm0
+; CHECK-NEXT:    vpsrlw $9, %ymm1, %ymm1
+; CHECK-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %1 =  shl <16 x i16> %a0, <i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7>
+  %2 = lshr <16 x i16> %a1, <i16 9, i16 9, i16 9, i16 9, i16 9, i16 9, i16 9, i16 9, i16 9, i16 9, i16 9, i16 9, i16 9, i16 9, i16 9, i16 9>
+  %3 = or <16 x i16> %1, %2
+  ret <16 x i16> %3
+}
+
+
diff --git a/llvm/test/CodeGen/X86/avx512vl-intrinsics-fast-isel.ll b/llvm/test/CodeGen/X86/avx512vl-intrinsics-fast-isel.ll
index 4f1affb94..82a19ba41 100644
--- a/llvm/test/CodeGen/X86/avx512vl-intrinsics-fast-isel.ll
+++ b/llvm/test/CodeGen/X86/avx512vl-intrinsics-fast-isel.ll
@@ -3326,6 +3326,8 @@ define <2 x i64> @test_mm256_cvtepi64_epi8(<4 x i64> %__A) {
 ; CHECK-LABEL: test_mm256_cvtepi64_epi8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vpmovqb %ymm0, %xmm0
+; CHECK-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7]
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    ret{{[l|q]}}
 entry:
@@ -3339,6 +3341,7 @@ define <2 x i64> @test_mm256_cvtepi64_epi16(<4 x i64> %__A) {
 ; CHECK-LABEL: test_mm256_cvtepi64_epi16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vpmovqw %ymm0, %xmm0
+; CHECK-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    ret{{[l|q]}}
 entry:
@@ -3352,6 +3355,7 @@ define <2 x i64> @test_mm256_cvtepi32_epi8(<4 x i64> %__A) {
 ; CHECK-LABEL: test_mm256_cvtepi32_epi8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vpmovdb %ymm0, %xmm0
+; CHECK-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    ret{{[l|q]}}
 entry:
diff --git a/llvm/test/CodeGen/X86/avx512vl-intrinsics-upgrade.ll b/llvm/test/CodeGen/X86/avx512vl-intrinsics-upgrade.ll
index 8d0aafc56..28c411174 100644
--- a/llvm/test/CodeGen/X86/avx512vl-intrinsics-upgrade.ll
+++ b/llvm/test/CodeGen/X86/avx512vl-intrinsics-upgrade.ll
@@ -7,12 +7,11 @@ declare <4 x i32> @llvm.x86.avx512.mask.pbroadcast.d.gpr.128(i32, <4 x i32>, i8)
 define <4 x i32>@test_int_x86_avx512_mask_pbroadcast_d_gpr_128(i32 %x0, <4 x i32> %x1, i8 %mask) {
 ; X86-LABEL: test_int_x86_avx512_mask_pbroadcast_d_gpr_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vpbroadcastd %eax, %xmm1 # encoding: [0x62,0xf2,0x7d,0x08,0x7c,0xc8]
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
-; X86-NEXT:    kmovw %ecx, %k1 # encoding: [0xc5,0xf8,0x92,0xc9]
-; X86-NEXT:    vpbroadcastd %eax, %xmm0 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x7c,0xc0]
-; X86-NEXT:    vpbroadcastd %eax, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0x89,0x7c,0xd0]
+; X86-NEXT:    vpbroadcastd {{[0-9]+}}(%esp), %xmm1 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x58,0x4c,0x24,0x04]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vmovdqa32 %xmm1, %xmm0 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0x6f,0xc1]
+; X86-NEXT:    vmovdqa32 %xmm1, %xmm2 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0x6f,0xd1]
 ; X86-NEXT:    vpaddd %xmm2, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc2]
 ; X86-NEXT:    vpaddd %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
@@ -72,12 +71,11 @@ define <2 x i64>@test_int_x86_avx512_mask_pbroadcast_q_gpr_128(i64 %x0, <2 x i64
   define <8 x i32>@test_int_x86_avx512_mask_pbroadcast_d_gpr_256(i32 %x0, <8 x i32> %x1, i8 %mask) {
 ; X86-LABEL: test_int_x86_avx512_mask_pbroadcast_d_gpr_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vpbroadcastd %eax, %ymm1 # encoding: [0x62,0xf2,0x7d,0x28,0x7c,0xc8]
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
-; X86-NEXT:    kmovw %ecx, %k1 # encoding: [0xc5,0xf8,0x92,0xc9]
-; X86-NEXT:    vpbroadcastd %eax, %ymm0 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x7c,0xc0]
-; X86-NEXT:    vpbroadcastd %eax, %ymm2 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xa9,0x7c,0xd0]
+; X86-NEXT:    vpbroadcastd {{[0-9]+}}(%esp), %ymm1 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x58,0x4c,0x24,0x04]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vmovdqa32 %ymm1, %ymm0 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0x6f,0xc1]
+; X86-NEXT:    vmovdqa32 %ymm1, %ymm2 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0x6f,0xd1]
 ; X86-NEXT:    vpaddd %ymm2, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfe,0xc2]
 ; X86-NEXT:    vpaddd %ymm0, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf5,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
@@ -3924,10 +3922,9 @@ define <2 x i64> @test_mask_andnot_epi64_rmbk_128(<2 x i64> %a, i64* %ptr_b, <2
 ; X86-LABEL: test_mask_andnot_epi64_rmbk_128:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vpbroadcastq (%eax), %xmm2 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x59,0x10]
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
-; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vpandnq %xmm2, %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0xfd,0x09,0xdf,0xca]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
+; X86-NEXT:    kmovw %ecx, %k1 # encoding: [0xc5,0xf8,0x92,0xc9]
+; X86-NEXT:    vpandnq (%eax){1to2}, %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0xfd,0x19,0xdf,0x08]
 ; X86-NEXT:    vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -3948,10 +3945,9 @@ define <2 x i64> @test_mask_andnot_epi64_rmbkz_128(<2 x i64> %a, i64* %ptr_b, i8
 ; X86-LABEL: test_mask_andnot_epi64_rmbkz_128:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vpbroadcastq (%eax), %xmm1 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x59,0x08]
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
-; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vpandnq %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0x89,0xdf,0xc1]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
+; X86-NEXT:    kmovw %ecx, %k1 # encoding: [0xc5,0xf8,0x92,0xc9]
+; X86-NEXT:    vpandnq (%eax){1to2}, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0x99,0xdf,0x00]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_mask_andnot_epi64_rmbkz_128:
@@ -4091,10 +4087,9 @@ define <4 x i64> @test_mask_andnot_epi64_rmbk_256(<4 x i64> %a, i64* %ptr_b, <4
 ; X86-LABEL: test_mask_andnot_epi64_rmbk_256:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vpbroadcastq (%eax), %ymm2 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x59,0x10]
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
-; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vpandnq %ymm2, %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0xfd,0x29,0xdf,0xca]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
+; X86-NEXT:    kmovw %ecx, %k1 # encoding: [0xc5,0xf8,0x92,0xc9]
+; X86-NEXT:    vpandnq (%eax){1to4}, %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0xfd,0x39,0xdf,0x08]
 ; X86-NEXT:    vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -4115,10 +4110,9 @@ define <4 x i64> @test_mask_andnot_epi64_rmbkz_256(<4 x i64> %a, i64* %ptr_b, i8
 ; X86-LABEL: test_mask_andnot_epi64_rmbkz_256:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vpbroadcastq (%eax), %ymm1 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x59,0x08]
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
-; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vpandnq %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0xa9,0xdf,0xc1]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
+; X86-NEXT:    kmovw %ecx, %k1 # encoding: [0xc5,0xf8,0x92,0xc9]
+; X86-NEXT:    vpandnq (%eax){1to4}, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0xb9,0xdf,0x00]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_mask_andnot_epi64_rmbkz_256:
@@ -7865,7 +7859,6 @@ define <8 x i32>@test_int_x86_avx512_mask_inserti32x4_256(<8 x i32> %x0, <4 x i3
 ; X64-NEXT:    vpaddd %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xfe,0xc0]
 ; X64-NEXT:    vpaddd %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
-
   %res = call <8 x i32> @llvm.x86.avx512.mask.inserti32x4.256(<8 x i32> %x0, <4 x i32> %x1, i32 1, <8 x i32> %x3, i8 %x4)
   %res1 = call <8 x i32> @llvm.x86.avx512.mask.inserti32x4.256(<8 x i32> %x0, <4 x i32> %x1, i32 1, <8 x i32> %x3, i8 -1)
   %res2 = call <8 x i32> @llvm.x86.avx512.mask.inserti32x4.256(<8 x i32> %x0, <4 x i32> %x1, i32 1, <8 x i32> zeroinitializer, i8 %x4)
@@ -8069,19 +8062,19 @@ define <8 x i8> @test_cmp_d_256(<8 x i32> %a0, <8 x i32> %a1) {
 ; CHECK-NEXT:    vpcmpgtd %ymm1, %ymm0, %k5 # encoding: [0x62,0xf1,0x7d,0x28,0x66,0xe9]
 ; CHECK-NEXT:    kmovw %k0, %eax # encoding: [0xc5,0xf8,0x93,0xc0]
 ; CHECK-NEXT:    vpxor %xmm0, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xef,0xc0]
-; CHECK-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x00]
+; CHECK-NEXT:    vpinsrb $0, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x00]
 ; CHECK-NEXT:    kmovw %k1, %eax # encoding: [0xc5,0xf8,0x93,0xc1]
-; CHECK-NEXT:    vpinsrw $1, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x01]
+; CHECK-NEXT:    vpinsrb $1, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x01]
 ; CHECK-NEXT:    kmovw %k2, %eax # encoding: [0xc5,0xf8,0x93,0xc2]
-; CHECK-NEXT:    vpinsrw $2, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x02]
+; CHECK-NEXT:    vpinsrb $2, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x02]
 ; CHECK-NEXT:    kmovw %k3, %eax # encoding: [0xc5,0xf8,0x93,0xc3]
-; CHECK-NEXT:    vpinsrw $4, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x04]
+; CHECK-NEXT:    vpinsrb $4, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x04]
 ; CHECK-NEXT:    kmovw %k4, %eax # encoding: [0xc5,0xf8,0x93,0xc4]
-; CHECK-NEXT:    vpinsrw $5, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x05]
+; CHECK-NEXT:    vpinsrb $5, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x05]
 ; CHECK-NEXT:    kmovw %k5, %eax # encoding: [0xc5,0xf8,0x93,0xc5]
-; CHECK-NEXT:    vpinsrw $6, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x06]
+; CHECK-NEXT:    vpinsrb $6, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x06]
 ; CHECK-NEXT:    movl $255, %eax # encoding: [0xb8,0xff,0x00,0x00,0x00]
-; CHECK-NEXT:    vpinsrw $7, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x07]
+; CHECK-NEXT:    vpinsrb $7, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x07]
 ; CHECK-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res0 = call i8 @llvm.x86.avx512.mask.cmp.d.256(<8 x i32> %a0, <8 x i32> %a1, i32 0, i8 -1)
@@ -8106,7 +8099,7 @@ define <8 x i8> @test_cmp_d_256(<8 x i32> %a0, <8 x i32> %a1) {
 define <8 x i8> @test_mask_cmp_d_256(<8 x i32> %a0, <8 x i32> %a1, i8 %mask) {
 ; X86-LABEL: test_mask_cmp_d_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb7,0x44,0x24,0x04]
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpcmpeqd %ymm1, %ymm0, %k0 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0x76,0xc1]
 ; X86-NEXT:    vpcmpgtd %ymm0, %ymm1, %k2 {%k1} # encoding: [0x62,0xf1,0x75,0x29,0x66,0xd0]
@@ -8116,18 +8109,18 @@ define <8 x i8> @test_mask_cmp_d_256(<8 x i32> %a0, <8 x i32> %a1, i8 %mask) {
 ; X86-NEXT:    vpcmpgtd %ymm1, %ymm0, %k1 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0x66,0xc9]
 ; X86-NEXT:    kmovw %k0, %ecx # encoding: [0xc5,0xf8,0x93,0xc8]
 ; X86-NEXT:    vpxor %xmm0, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xef,0xc0]
-; X86-NEXT:    vpinsrw $0, %ecx, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc1,0x00]
+; X86-NEXT:    vpinsrb $0, %ecx, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc1,0x00]
 ; X86-NEXT:    kmovw %k2, %ecx # encoding: [0xc5,0xf8,0x93,0xca]
-; X86-NEXT:    vpinsrw $1, %ecx, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc1,0x01]
+; X86-NEXT:    vpinsrb $1, %ecx, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc1,0x01]
 ; X86-NEXT:    kmovw %k3, %ecx # encoding: [0xc5,0xf8,0x93,0xcb]
-; X86-NEXT:    vpinsrw $2, %ecx, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc1,0x02]
+; X86-NEXT:    vpinsrb $2, %ecx, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc1,0x02]
 ; X86-NEXT:    kmovw %k4, %ecx # encoding: [0xc5,0xf8,0x93,0xcc]
-; X86-NEXT:    vpinsrw $4, %ecx, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc1,0x04]
+; X86-NEXT:    vpinsrb $4, %ecx, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc1,0x04]
 ; X86-NEXT:    kmovw %k5, %ecx # encoding: [0xc5,0xf8,0x93,0xcd]
-; X86-NEXT:    vpinsrw $5, %ecx, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc1,0x05]
+; X86-NEXT:    vpinsrb $5, %ecx, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc1,0x05]
 ; X86-NEXT:    kmovw %k1, %ecx # encoding: [0xc5,0xf8,0x93,0xc9]
-; X86-NEXT:    vpinsrw $6, %ecx, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc1,0x06]
-; X86-NEXT:    vpinsrw $7, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x07]
+; X86-NEXT:    vpinsrb $6, %ecx, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc1,0x06]
+; X86-NEXT:    vpinsrb $7, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x07]
 ; X86-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -8142,18 +8135,18 @@ define <8 x i8> @test_mask_cmp_d_256(<8 x i32> %a0, <8 x i32> %a1, i8 %mask) {
 ; X64-NEXT:    vpcmpgtd %ymm1, %ymm0, %k1 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0x66,0xc9]
 ; X64-NEXT:    kmovw %k0, %eax # encoding: [0xc5,0xf8,0x93,0xc0]
 ; X64-NEXT:    vpxor %xmm0, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xef,0xc0]
-; X64-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x00]
+; X64-NEXT:    vpinsrb $0, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x00]
 ; X64-NEXT:    kmovw %k2, %eax # encoding: [0xc5,0xf8,0x93,0xc2]
-; X64-NEXT:    vpinsrw $1, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x01]
+; X64-NEXT:    vpinsrb $1, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x01]
 ; X64-NEXT:    kmovw %k3, %eax # encoding: [0xc5,0xf8,0x93,0xc3]
-; X64-NEXT:    vpinsrw $2, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x02]
+; X64-NEXT:    vpinsrb $2, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x02]
 ; X64-NEXT:    kmovw %k4, %eax # encoding: [0xc5,0xf8,0x93,0xc4]
-; X64-NEXT:    vpinsrw $4, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x04]
+; X64-NEXT:    vpinsrb $4, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x04]
 ; X64-NEXT:    kmovw %k5, %eax # encoding: [0xc5,0xf8,0x93,0xc5]
-; X64-NEXT:    vpinsrw $5, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x05]
+; X64-NEXT:    vpinsrb $5, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x05]
 ; X64-NEXT:    kmovw %k1, %eax # encoding: [0xc5,0xf8,0x93,0xc1]
-; X64-NEXT:    vpinsrw $6, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x06]
-; X64-NEXT:    vpinsrw $7, %edi, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc7,0x07]
+; X64-NEXT:    vpinsrb $6, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x06]
+; X64-NEXT:    vpinsrb $7, %edi, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc7,0x07]
 ; X64-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res0 = call i8 @llvm.x86.avx512.mask.cmp.d.256(<8 x i32> %a0, <8 x i32> %a1, i32 0, i8 %mask)
@@ -8188,19 +8181,19 @@ define <8 x i8> @test_ucmp_d_256(<8 x i32> %a0, <8 x i32> %a1) {
 ; CHECK-NEXT:    vpcmpnleud %ymm1, %ymm0, %k5 # encoding: [0x62,0xf3,0x7d,0x28,0x1e,0xe9,0x06]
 ; CHECK-NEXT:    kmovw %k0, %eax # encoding: [0xc5,0xf8,0x93,0xc0]
 ; CHECK-NEXT:    vpxor %xmm0, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xef,0xc0]
-; CHECK-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x00]
+; CHECK-NEXT:    vpinsrb $0, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x00]
 ; CHECK-NEXT:    kmovw %k1, %eax # encoding: [0xc5,0xf8,0x93,0xc1]
-; CHECK-NEXT:    vpinsrw $1, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x01]
+; CHECK-NEXT:    vpinsrb $1, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x01]
 ; CHECK-NEXT:    kmovw %k2, %eax # encoding: [0xc5,0xf8,0x93,0xc2]
-; CHECK-NEXT:    vpinsrw $2, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x02]
+; CHECK-NEXT:    vpinsrb $2, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x02]
 ; CHECK-NEXT:    kmovw %k3, %eax # encoding: [0xc5,0xf8,0x93,0xc3]
-; CHECK-NEXT:    vpinsrw $4, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x04]
+; CHECK-NEXT:    vpinsrb $4, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x04]
 ; CHECK-NEXT:    kmovw %k4, %eax # encoding: [0xc5,0xf8,0x93,0xc4]
-; CHECK-NEXT:    vpinsrw $5, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x05]
+; CHECK-NEXT:    vpinsrb $5, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x05]
 ; CHECK-NEXT:    kmovw %k5, %eax # encoding: [0xc5,0xf8,0x93,0xc5]
-; CHECK-NEXT:    vpinsrw $6, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x06]
+; CHECK-NEXT:    vpinsrb $6, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x06]
 ; CHECK-NEXT:    movl $255, %eax # encoding: [0xb8,0xff,0x00,0x00,0x00]
-; CHECK-NEXT:    vpinsrw $7, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x07]
+; CHECK-NEXT:    vpinsrb $7, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x07]
 ; CHECK-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res0 = call i8 @llvm.x86.avx512.mask.ucmp.d.256(<8 x i32> %a0, <8 x i32> %a1, i32 0, i8 -1)
@@ -8225,7 +8218,7 @@ define <8 x i8> @test_ucmp_d_256(<8 x i32> %a0, <8 x i32> %a1) {
 define <8 x i8> @test_mask_ucmp_d_256(<8 x i32> %a0, <8 x i32> %a1, i8 %mask) {
 ; X86-LABEL: test_mask_ucmp_d_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb7,0x44,0x24,0x04]
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpcmpeqd %ymm1, %ymm0, %k0 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0x76,0xc1]
 ; X86-NEXT:    vpcmpltud %ymm1, %ymm0, %k2 {%k1} # encoding: [0x62,0xf3,0x7d,0x29,0x1e,0xd1,0x01]
@@ -8235,18 +8228,18 @@ define <8 x i8> @test_mask_ucmp_d_256(<8 x i32> %a0, <8 x i32> %a1, i8 %mask) {
 ; X86-NEXT:    vpcmpnleud %ymm1, %ymm0, %k1 {%k1} # encoding: [0x62,0xf3,0x7d,0x29,0x1e,0xc9,0x06]
 ; X86-NEXT:    kmovw %k0, %ecx # encoding: [0xc5,0xf8,0x93,0xc8]
 ; X86-NEXT:    vpxor %xmm0, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xef,0xc0]
-; X86-NEXT:    vpinsrw $0, %ecx, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc1,0x00]
+; X86-NEXT:    vpinsrb $0, %ecx, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc1,0x00]
 ; X86-NEXT:    kmovw %k2, %ecx # encoding: [0xc5,0xf8,0x93,0xca]
-; X86-NEXT:    vpinsrw $1, %ecx, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc1,0x01]
+; X86-NEXT:    vpinsrb $1, %ecx, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc1,0x01]
 ; X86-NEXT:    kmovw %k3, %ecx # encoding: [0xc5,0xf8,0x93,0xcb]
-; X86-NEXT:    vpinsrw $2, %ecx, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc1,0x02]
+; X86-NEXT:    vpinsrb $2, %ecx, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc1,0x02]
 ; X86-NEXT:    kmovw %k4, %ecx # encoding: [0xc5,0xf8,0x93,0xcc]
-; X86-NEXT:    vpinsrw $4, %ecx, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc1,0x04]
+; X86-NEXT:    vpinsrb $4, %ecx, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc1,0x04]
 ; X86-NEXT:    kmovw %k5, %ecx # encoding: [0xc5,0xf8,0x93,0xcd]
-; X86-NEXT:    vpinsrw $5, %ecx, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc1,0x05]
+; X86-NEXT:    vpinsrb $5, %ecx, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc1,0x05]
 ; X86-NEXT:    kmovw %k1, %ecx # encoding: [0xc5,0xf8,0x93,0xc9]
-; X86-NEXT:    vpinsrw $6, %ecx, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc1,0x06]
-; X86-NEXT:    vpinsrw $7, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x07]
+; X86-NEXT:    vpinsrb $6, %ecx, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc1,0x06]
+; X86-NEXT:    vpinsrb $7, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x07]
 ; X86-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -8261,18 +8254,18 @@ define <8 x i8> @test_mask_ucmp_d_256(<8 x i32> %a0, <8 x i32> %a1, i8 %mask) {
 ; X64-NEXT:    vpcmpnleud %ymm1, %ymm0, %k1 {%k1} # encoding: [0x62,0xf3,0x7d,0x29,0x1e,0xc9,0x06]
 ; X64-NEXT:    kmovw %k0, %eax # encoding: [0xc5,0xf8,0x93,0xc0]
 ; X64-NEXT:    vpxor %xmm0, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xef,0xc0]
-; X64-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x00]
+; X64-NEXT:    vpinsrb $0, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x00]
 ; X64-NEXT:    kmovw %k2, %eax # encoding: [0xc5,0xf8,0x93,0xc2]
-; X64-NEXT:    vpinsrw $1, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x01]
+; X64-NEXT:    vpinsrb $1, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x01]
 ; X64-NEXT:    kmovw %k3, %eax # encoding: [0xc5,0xf8,0x93,0xc3]
-; X64-NEXT:    vpinsrw $2, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x02]
+; X64-NEXT:    vpinsrb $2, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x02]
 ; X64-NEXT:    kmovw %k4, %eax # encoding: [0xc5,0xf8,0x93,0xc4]
-; X64-NEXT:    vpinsrw $4, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x04]
+; X64-NEXT:    vpinsrb $4, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x04]
 ; X64-NEXT:    kmovw %k5, %eax # encoding: [0xc5,0xf8,0x93,0xc5]
-; X64-NEXT:    vpinsrw $5, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x05]
+; X64-NEXT:    vpinsrb $5, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x05]
 ; X64-NEXT:    kmovw %k1, %eax # encoding: [0xc5,0xf8,0x93,0xc1]
-; X64-NEXT:    vpinsrw $6, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x06]
-; X64-NEXT:    vpinsrw $7, %edi, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc7,0x07]
+; X64-NEXT:    vpinsrb $6, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x06]
+; X64-NEXT:    vpinsrb $7, %edi, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc7,0x07]
 ; X64-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res0 = call i8 @llvm.x86.avx512.mask.ucmp.d.256(<8 x i32> %a0, <8 x i32> %a1, i32 0, i8 %mask)
@@ -8307,19 +8300,19 @@ define <8 x i8> @test_cmp_q_256(<4 x i64> %a0, <4 x i64> %a1) {
 ; CHECK-NEXT:    vpcmpgtq %ymm1, %ymm0, %k5 # encoding: [0x62,0xf2,0xfd,0x28,0x37,0xe9]
 ; CHECK-NEXT:    kmovw %k0, %eax # encoding: [0xc5,0xf8,0x93,0xc0]
 ; CHECK-NEXT:    vpxor %xmm0, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xef,0xc0]
-; CHECK-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x00]
+; CHECK-NEXT:    vpinsrb $0, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x00]
 ; CHECK-NEXT:    kmovw %k1, %eax # encoding: [0xc5,0xf8,0x93,0xc1]
-; CHECK-NEXT:    vpinsrw $1, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x01]
+; CHECK-NEXT:    vpinsrb $1, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x01]
 ; CHECK-NEXT:    kmovw %k2, %eax # encoding: [0xc5,0xf8,0x93,0xc2]
-; CHECK-NEXT:    vpinsrw $2, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x02]
+; CHECK-NEXT:    vpinsrb $2, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x02]
 ; CHECK-NEXT:    kmovw %k3, %eax # encoding: [0xc5,0xf8,0x93,0xc3]
-; CHECK-NEXT:    vpinsrw $4, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x04]
+; CHECK-NEXT:    vpinsrb $4, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x04]
 ; CHECK-NEXT:    kmovw %k4, %eax # encoding: [0xc5,0xf8,0x93,0xc4]
-; CHECK-NEXT:    vpinsrw $5, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x05]
+; CHECK-NEXT:    vpinsrb $5, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x05]
 ; CHECK-NEXT:    kmovw %k5, %eax # encoding: [0xc5,0xf8,0x93,0xc5]
-; CHECK-NEXT:    vpinsrw $6, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x06]
+; CHECK-NEXT:    vpinsrb $6, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x06]
 ; CHECK-NEXT:    movl $15, %eax # encoding: [0xb8,0x0f,0x00,0x00,0x00]
-; CHECK-NEXT:    vpinsrw $7, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x07]
+; CHECK-NEXT:    vpinsrb $7, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x07]
 ; CHECK-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res0 = call i8 @llvm.x86.avx512.mask.cmp.q.256(<4 x i64> %a0, <4 x i64> %a1, i32 0, i8 -1)
@@ -8345,59 +8338,59 @@ define <8 x i8> @test_mask_cmp_q_256(<4 x i64> %a0, <4 x i64> %a1, i8 %mask) {
 ; X86-LABEL: test_mask_cmp_q_256:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
-; X86-NEXT:    kmovw %eax, %k2 # encoding: [0xc5,0xf8,0x92,0xd0]
-; X86-NEXT:    vpcmpeqq %ymm1, %ymm0, %k0 {%k2} # encoding: [0x62,0xf2,0xfd,0x2a,0x29,0xc1]
-; X86-NEXT:    vpcmpgtq %ymm0, %ymm1, %k1 {%k2} # encoding: [0x62,0xf2,0xf5,0x2a,0x37,0xc8]
-; X86-NEXT:    vpcmpleq %ymm1, %ymm0, %k3 {%k2} # encoding: [0x62,0xf3,0xfd,0x2a,0x1f,0xd9,0x02]
-; X86-NEXT:    vpcmpneqq %ymm1, %ymm0, %k4 {%k2} # encoding: [0x62,0xf3,0xfd,0x2a,0x1f,0xe1,0x04]
-; X86-NEXT:    vpcmpnltq %ymm1, %ymm0, %k5 {%k2} # encoding: [0x62,0xf3,0xfd,0x2a,0x1f,0xe9,0x05]
-; X86-NEXT:    vpcmpgtq %ymm1, %ymm0, %k6 {%k2} # encoding: [0x62,0xf2,0xfd,0x2a,0x37,0xf1]
-; X86-NEXT:    kshiftlw $12, %k2, %k2 # encoding: [0xc4,0xe3,0xf9,0x32,0xd2,0x0c]
-; X86-NEXT:    kshiftrw $12, %k2, %k2 # encoding: [0xc4,0xe3,0xf9,0x30,0xd2,0x0c]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpcmpeqq %ymm1, %ymm0, %k0 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x29,0xc1]
+; X86-NEXT:    vpcmpgtq %ymm0, %ymm1, %k2 {%k1} # encoding: [0x62,0xf2,0xf5,0x29,0x37,0xd0]
+; X86-NEXT:    vpcmpleq %ymm1, %ymm0, %k3 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x1f,0xd9,0x02]
+; X86-NEXT:    vpcmpneqq %ymm1, %ymm0, %k4 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x1f,0xe1,0x04]
+; X86-NEXT:    vpcmpnltq %ymm1, %ymm0, %k5 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x1f,0xe9,0x05]
+; X86-NEXT:    vpcmpgtq %ymm1, %ymm0, %k6 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x37,0xf1]
+; X86-NEXT:    kshiftlw $12, %k1, %k1 # encoding: [0xc4,0xe3,0xf9,0x32,0xc9,0x0c]
+; X86-NEXT:    kshiftrw $12, %k1, %k1 # encoding: [0xc4,0xe3,0xf9,0x30,0xc9,0x0c]
 ; X86-NEXT:    kmovw %k0, %eax # encoding: [0xc5,0xf8,0x93,0xc0]
 ; X86-NEXT:    vpxor %xmm0, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xef,0xc0]
-; X86-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x00]
-; X86-NEXT:    kmovw %k1, %eax # encoding: [0xc5,0xf8,0x93,0xc1]
-; X86-NEXT:    vpinsrw $1, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x01]
+; X86-NEXT:    vpinsrb $0, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x00]
+; X86-NEXT:    kmovw %k2, %eax # encoding: [0xc5,0xf8,0x93,0xc2]
+; X86-NEXT:    vpinsrb $1, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x01]
 ; X86-NEXT:    kmovw %k3, %eax # encoding: [0xc5,0xf8,0x93,0xc3]
-; X86-NEXT:    vpinsrw $2, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x02]
+; X86-NEXT:    vpinsrb $2, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x02]
 ; X86-NEXT:    kmovw %k4, %eax # encoding: [0xc5,0xf8,0x93,0xc4]
-; X86-NEXT:    vpinsrw $4, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x04]
+; X86-NEXT:    vpinsrb $4, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x04]
 ; X86-NEXT:    kmovw %k5, %eax # encoding: [0xc5,0xf8,0x93,0xc5]
-; X86-NEXT:    vpinsrw $5, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x05]
+; X86-NEXT:    vpinsrb $5, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x05]
 ; X86-NEXT:    kmovw %k6, %eax # encoding: [0xc5,0xf8,0x93,0xc6]
-; X86-NEXT:    vpinsrw $6, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x06]
-; X86-NEXT:    kmovw %k2, %eax # encoding: [0xc5,0xf8,0x93,0xc2]
-; X86-NEXT:    vpinsrw $7, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x07]
+; X86-NEXT:    vpinsrb $6, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x06]
+; X86-NEXT:    kmovw %k1, %eax # encoding: [0xc5,0xf8,0x93,0xc1]
+; X86-NEXT:    vpinsrb $7, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x07]
 ; X86-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_mask_cmp_q_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    kmovw %edi, %k2 # encoding: [0xc5,0xf8,0x92,0xd7]
-; X64-NEXT:    vpcmpeqq %ymm1, %ymm0, %k0 {%k2} # encoding: [0x62,0xf2,0xfd,0x2a,0x29,0xc1]
-; X64-NEXT:    vpcmpgtq %ymm0, %ymm1, %k1 {%k2} # encoding: [0x62,0xf2,0xf5,0x2a,0x37,0xc8]
-; X64-NEXT:    vpcmpleq %ymm1, %ymm0, %k3 {%k2} # encoding: [0x62,0xf3,0xfd,0x2a,0x1f,0xd9,0x02]
-; X64-NEXT:    vpcmpneqq %ymm1, %ymm0, %k4 {%k2} # encoding: [0x62,0xf3,0xfd,0x2a,0x1f,0xe1,0x04]
-; X64-NEXT:    vpcmpnltq %ymm1, %ymm0, %k5 {%k2} # encoding: [0x62,0xf3,0xfd,0x2a,0x1f,0xe9,0x05]
-; X64-NEXT:    vpcmpgtq %ymm1, %ymm0, %k6 {%k2} # encoding: [0x62,0xf2,0xfd,0x2a,0x37,0xf1]
-; X64-NEXT:    kshiftlw $12, %k2, %k2 # encoding: [0xc4,0xe3,0xf9,0x32,0xd2,0x0c]
-; X64-NEXT:    kshiftrw $12, %k2, %k2 # encoding: [0xc4,0xe3,0xf9,0x30,0xd2,0x0c]
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vpcmpeqq %ymm1, %ymm0, %k0 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x29,0xc1]
+; X64-NEXT:    vpcmpgtq %ymm0, %ymm1, %k2 {%k1} # encoding: [0x62,0xf2,0xf5,0x29,0x37,0xd0]
+; X64-NEXT:    vpcmpleq %ymm1, %ymm0, %k3 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x1f,0xd9,0x02]
+; X64-NEXT:    vpcmpneqq %ymm1, %ymm0, %k4 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x1f,0xe1,0x04]
+; X64-NEXT:    vpcmpnltq %ymm1, %ymm0, %k5 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x1f,0xe9,0x05]
+; X64-NEXT:    vpcmpgtq %ymm1, %ymm0, %k6 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x37,0xf1]
+; X64-NEXT:    kshiftlw $12, %k1, %k1 # encoding: [0xc4,0xe3,0xf9,0x32,0xc9,0x0c]
+; X64-NEXT:    kshiftrw $12, %k1, %k1 # encoding: [0xc4,0xe3,0xf9,0x30,0xc9,0x0c]
 ; X64-NEXT:    kmovw %k0, %eax # encoding: [0xc5,0xf8,0x93,0xc0]
 ; X64-NEXT:    vpxor %xmm0, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xef,0xc0]
-; X64-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x00]
-; X64-NEXT:    kmovw %k1, %eax # encoding: [0xc5,0xf8,0x93,0xc1]
-; X64-NEXT:    vpinsrw $1, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x01]
+; X64-NEXT:    vpinsrb $0, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x00]
+; X64-NEXT:    kmovw %k2, %eax # encoding: [0xc5,0xf8,0x93,0xc2]
+; X64-NEXT:    vpinsrb $1, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x01]
 ; X64-NEXT:    kmovw %k3, %eax # encoding: [0xc5,0xf8,0x93,0xc3]
-; X64-NEXT:    vpinsrw $2, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x02]
+; X64-NEXT:    vpinsrb $2, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x02]
 ; X64-NEXT:    kmovw %k4, %eax # encoding: [0xc5,0xf8,0x93,0xc4]
-; X64-NEXT:    vpinsrw $4, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x04]
+; X64-NEXT:    vpinsrb $4, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x04]
 ; X64-NEXT:    kmovw %k5, %eax # encoding: [0xc5,0xf8,0x93,0xc5]
-; X64-NEXT:    vpinsrw $5, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x05]
+; X64-NEXT:    vpinsrb $5, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x05]
 ; X64-NEXT:    kmovw %k6, %eax # encoding: [0xc5,0xf8,0x93,0xc6]
-; X64-NEXT:    vpinsrw $6, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x06]
-; X64-NEXT:    kmovw %k2, %eax # encoding: [0xc5,0xf8,0x93,0xc2]
-; X64-NEXT:    vpinsrw $7, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x07]
+; X64-NEXT:    vpinsrb $6, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x06]
+; X64-NEXT:    kmovw %k1, %eax # encoding: [0xc5,0xf8,0x93,0xc1]
+; X64-NEXT:    vpinsrb $7, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x07]
 ; X64-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res0 = call i8 @llvm.x86.avx512.mask.cmp.q.256(<4 x i64> %a0, <4 x i64> %a1, i32 0, i8 %mask)
@@ -8432,19 +8425,19 @@ define <8 x i8> @test_ucmp_q_256(<4 x i64> %a0, <4 x i64> %a1) {
 ; CHECK-NEXT:    vpcmpnleuq %ymm1, %ymm0, %k5 # encoding: [0x62,0xf3,0xfd,0x28,0x1e,0xe9,0x06]
 ; CHECK-NEXT:    kmovw %k0, %eax # encoding: [0xc5,0xf8,0x93,0xc0]
 ; CHECK-NEXT:    vpxor %xmm0, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xef,0xc0]
-; CHECK-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x00]
+; CHECK-NEXT:    vpinsrb $0, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x00]
 ; CHECK-NEXT:    kmovw %k1, %eax # encoding: [0xc5,0xf8,0x93,0xc1]
-; CHECK-NEXT:    vpinsrw $1, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x01]
+; CHECK-NEXT:    vpinsrb $1, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x01]
 ; CHECK-NEXT:    kmovw %k2, %eax # encoding: [0xc5,0xf8,0x93,0xc2]
-; CHECK-NEXT:    vpinsrw $2, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x02]
+; CHECK-NEXT:    vpinsrb $2, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x02]
 ; CHECK-NEXT:    kmovw %k3, %eax # encoding: [0xc5,0xf8,0x93,0xc3]
-; CHECK-NEXT:    vpinsrw $4, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x04]
+; CHECK-NEXT:    vpinsrb $4, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x04]
 ; CHECK-NEXT:    kmovw %k4, %eax # encoding: [0xc5,0xf8,0x93,0xc4]
-; CHECK-NEXT:    vpinsrw $5, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x05]
+; CHECK-NEXT:    vpinsrb $5, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x05]
 ; CHECK-NEXT:    kmovw %k5, %eax # encoding: [0xc5,0xf8,0x93,0xc5]
-; CHECK-NEXT:    vpinsrw $6, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x06]
+; CHECK-NEXT:    vpinsrb $6, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x06]
 ; CHECK-NEXT:    movl $15, %eax # encoding: [0xb8,0x0f,0x00,0x00,0x00]
-; CHECK-NEXT:    vpinsrw $7, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x07]
+; CHECK-NEXT:    vpinsrb $7, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x07]
 ; CHECK-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res0 = call i8 @llvm.x86.avx512.mask.ucmp.q.256(<4 x i64> %a0, <4 x i64> %a1, i32 0, i8 -1)
@@ -8470,59 +8463,59 @@ define <8 x i8> @test_mask_ucmp_q_256(<4 x i64> %a0, <4 x i64> %a1, i8 %mask) {
 ; X86-LABEL: test_mask_ucmp_q_256:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
-; X86-NEXT:    kmovw %eax, %k2 # encoding: [0xc5,0xf8,0x92,0xd0]
-; X86-NEXT:    vpcmpeqq %ymm1, %ymm0, %k0 {%k2} # encoding: [0x62,0xf2,0xfd,0x2a,0x29,0xc1]
-; X86-NEXT:    vpcmpltuq %ymm1, %ymm0, %k1 {%k2} # encoding: [0x62,0xf3,0xfd,0x2a,0x1e,0xc9,0x01]
-; X86-NEXT:    vpcmpleuq %ymm1, %ymm0, %k3 {%k2} # encoding: [0x62,0xf3,0xfd,0x2a,0x1e,0xd9,0x02]
-; X86-NEXT:    vpcmpneqq %ymm1, %ymm0, %k4 {%k2} # encoding: [0x62,0xf3,0xfd,0x2a,0x1f,0xe1,0x04]
-; X86-NEXT:    vpcmpnltuq %ymm1, %ymm0, %k5 {%k2} # encoding: [0x62,0xf3,0xfd,0x2a,0x1e,0xe9,0x05]
-; X86-NEXT:    vpcmpnleuq %ymm1, %ymm0, %k6 {%k2} # encoding: [0x62,0xf3,0xfd,0x2a,0x1e,0xf1,0x06]
-; X86-NEXT:    kshiftlw $12, %k2, %k2 # encoding: [0xc4,0xe3,0xf9,0x32,0xd2,0x0c]
-; X86-NEXT:    kshiftrw $12, %k2, %k2 # encoding: [0xc4,0xe3,0xf9,0x30,0xd2,0x0c]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpcmpeqq %ymm1, %ymm0, %k0 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x29,0xc1]
+; X86-NEXT:    vpcmpltuq %ymm1, %ymm0, %k2 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x1e,0xd1,0x01]
+; X86-NEXT:    vpcmpleuq %ymm1, %ymm0, %k3 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x1e,0xd9,0x02]
+; X86-NEXT:    vpcmpneqq %ymm1, %ymm0, %k4 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x1f,0xe1,0x04]
+; X86-NEXT:    vpcmpnltuq %ymm1, %ymm0, %k5 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x1e,0xe9,0x05]
+; X86-NEXT:    vpcmpnleuq %ymm1, %ymm0, %k6 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x1e,0xf1,0x06]
+; X86-NEXT:    kshiftlw $12, %k1, %k1 # encoding: [0xc4,0xe3,0xf9,0x32,0xc9,0x0c]
+; X86-NEXT:    kshiftrw $12, %k1, %k1 # encoding: [0xc4,0xe3,0xf9,0x30,0xc9,0x0c]
 ; X86-NEXT:    kmovw %k0, %eax # encoding: [0xc5,0xf8,0x93,0xc0]
 ; X86-NEXT:    vpxor %xmm0, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xef,0xc0]
-; X86-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x00]
-; X86-NEXT:    kmovw %k1, %eax # encoding: [0xc5,0xf8,0x93,0xc1]
-; X86-NEXT:    vpinsrw $1, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x01]
+; X86-NEXT:    vpinsrb $0, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x00]
+; X86-NEXT:    kmovw %k2, %eax # encoding: [0xc5,0xf8,0x93,0xc2]
+; X86-NEXT:    vpinsrb $1, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x01]
 ; X86-NEXT:    kmovw %k3, %eax # encoding: [0xc5,0xf8,0x93,0xc3]
-; X86-NEXT:    vpinsrw $2, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x02]
+; X86-NEXT:    vpinsrb $2, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x02]
 ; X86-NEXT:    kmovw %k4, %eax # encoding: [0xc5,0xf8,0x93,0xc4]
-; X86-NEXT:    vpinsrw $4, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x04]
+; X86-NEXT:    vpinsrb $4, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x04]
 ; X86-NEXT:    kmovw %k5, %eax # encoding: [0xc5,0xf8,0x93,0xc5]
-; X86-NEXT:    vpinsrw $5, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x05]
+; X86-NEXT:    vpinsrb $5, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x05]
 ; X86-NEXT:    kmovw %k6, %eax # encoding: [0xc5,0xf8,0x93,0xc6]
-; X86-NEXT:    vpinsrw $6, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x06]
-; X86-NEXT:    kmovw %k2, %eax # encoding: [0xc5,0xf8,0x93,0xc2]
-; X86-NEXT:    vpinsrw $7, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x07]
+; X86-NEXT:    vpinsrb $6, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x06]
+; X86-NEXT:    kmovw %k1, %eax # encoding: [0xc5,0xf8,0x93,0xc1]
+; X86-NEXT:    vpinsrb $7, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x07]
 ; X86-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_mask_ucmp_q_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    kmovw %edi, %k2 # encoding: [0xc5,0xf8,0x92,0xd7]
-; X64-NEXT:    vpcmpeqq %ymm1, %ymm0, %k0 {%k2} # encoding: [0x62,0xf2,0xfd,0x2a,0x29,0xc1]
-; X64-NEXT:    vpcmpltuq %ymm1, %ymm0, %k1 {%k2} # encoding: [0x62,0xf3,0xfd,0x2a,0x1e,0xc9,0x01]
-; X64-NEXT:    vpcmpleuq %ymm1, %ymm0, %k3 {%k2} # encoding: [0x62,0xf3,0xfd,0x2a,0x1e,0xd9,0x02]
-; X64-NEXT:    vpcmpneqq %ymm1, %ymm0, %k4 {%k2} # encoding: [0x62,0xf3,0xfd,0x2a,0x1f,0xe1,0x04]
-; X64-NEXT:    vpcmpnltuq %ymm1, %ymm0, %k5 {%k2} # encoding: [0x62,0xf3,0xfd,0x2a,0x1e,0xe9,0x05]
-; X64-NEXT:    vpcmpnleuq %ymm1, %ymm0, %k6 {%k2} # encoding: [0x62,0xf3,0xfd,0x2a,0x1e,0xf1,0x06]
-; X64-NEXT:    kshiftlw $12, %k2, %k2 # encoding: [0xc4,0xe3,0xf9,0x32,0xd2,0x0c]
-; X64-NEXT:    kshiftrw $12, %k2, %k2 # encoding: [0xc4,0xe3,0xf9,0x30,0xd2,0x0c]
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vpcmpeqq %ymm1, %ymm0, %k0 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x29,0xc1]
+; X64-NEXT:    vpcmpltuq %ymm1, %ymm0, %k2 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x1e,0xd1,0x01]
+; X64-NEXT:    vpcmpleuq %ymm1, %ymm0, %k3 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x1e,0xd9,0x02]
+; X64-NEXT:    vpcmpneqq %ymm1, %ymm0, %k4 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x1f,0xe1,0x04]
+; X64-NEXT:    vpcmpnltuq %ymm1, %ymm0, %k5 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x1e,0xe9,0x05]
+; X64-NEXT:    vpcmpnleuq %ymm1, %ymm0, %k6 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x1e,0xf1,0x06]
+; X64-NEXT:    kshiftlw $12, %k1, %k1 # encoding: [0xc4,0xe3,0xf9,0x32,0xc9,0x0c]
+; X64-NEXT:    kshiftrw $12, %k1, %k1 # encoding: [0xc4,0xe3,0xf9,0x30,0xc9,0x0c]
 ; X64-NEXT:    kmovw %k0, %eax # encoding: [0xc5,0xf8,0x93,0xc0]
 ; X64-NEXT:    vpxor %xmm0, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xef,0xc0]
-; X64-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x00]
-; X64-NEXT:    kmovw %k1, %eax # encoding: [0xc5,0xf8,0x93,0xc1]
-; X64-NEXT:    vpinsrw $1, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x01]
+; X64-NEXT:    vpinsrb $0, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x00]
+; X64-NEXT:    kmovw %k2, %eax # encoding: [0xc5,0xf8,0x93,0xc2]
+; X64-NEXT:    vpinsrb $1, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x01]
 ; X64-NEXT:    kmovw %k3, %eax # encoding: [0xc5,0xf8,0x93,0xc3]
-; X64-NEXT:    vpinsrw $2, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x02]
+; X64-NEXT:    vpinsrb $2, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x02]
 ; X64-NEXT:    kmovw %k4, %eax # encoding: [0xc5,0xf8,0x93,0xc4]
-; X64-NEXT:    vpinsrw $4, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x04]
+; X64-NEXT:    vpinsrb $4, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x04]
 ; X64-NEXT:    kmovw %k5, %eax # encoding: [0xc5,0xf8,0x93,0xc5]
-; X64-NEXT:    vpinsrw $5, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x05]
+; X64-NEXT:    vpinsrb $5, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x05]
 ; X64-NEXT:    kmovw %k6, %eax # encoding: [0xc5,0xf8,0x93,0xc6]
-; X64-NEXT:    vpinsrw $6, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x06]
-; X64-NEXT:    kmovw %k2, %eax # encoding: [0xc5,0xf8,0x93,0xc2]
-; X64-NEXT:    vpinsrw $7, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x07]
+; X64-NEXT:    vpinsrb $6, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x06]
+; X64-NEXT:    kmovw %k1, %eax # encoding: [0xc5,0xf8,0x93,0xc1]
+; X64-NEXT:    vpinsrb $7, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x07]
 ; X64-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res0 = call i8 @llvm.x86.avx512.mask.ucmp.q.256(<4 x i64> %a0, <4 x i64> %a1, i32 0, i8 %mask)
@@ -8557,19 +8550,19 @@ define <8 x i8> @test_cmp_d_128(<4 x i32> %a0, <4 x i32> %a1) {
 ; CHECK-NEXT:    vpcmpgtd %xmm1, %xmm0, %k5 # encoding: [0x62,0xf1,0x7d,0x08,0x66,0xe9]
 ; CHECK-NEXT:    kmovw %k0, %eax # encoding: [0xc5,0xf8,0x93,0xc0]
 ; CHECK-NEXT:    vpxor %xmm0, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xef,0xc0]
-; CHECK-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x00]
+; CHECK-NEXT:    vpinsrb $0, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x00]
 ; CHECK-NEXT:    kmovw %k1, %eax # encoding: [0xc5,0xf8,0x93,0xc1]
-; CHECK-NEXT:    vpinsrw $1, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x01]
+; CHECK-NEXT:    vpinsrb $1, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x01]
 ; CHECK-NEXT:    kmovw %k2, %eax # encoding: [0xc5,0xf8,0x93,0xc2]
-; CHECK-NEXT:    vpinsrw $2, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x02]
+; CHECK-NEXT:    vpinsrb $2, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x02]
 ; CHECK-NEXT:    kmovw %k3, %eax # encoding: [0xc5,0xf8,0x93,0xc3]
-; CHECK-NEXT:    vpinsrw $4, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x04]
+; CHECK-NEXT:    vpinsrb $4, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x04]
 ; CHECK-NEXT:    kmovw %k4, %eax # encoding: [0xc5,0xf8,0x93,0xc4]
-; CHECK-NEXT:    vpinsrw $5, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x05]
+; CHECK-NEXT:    vpinsrb $5, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x05]
 ; CHECK-NEXT:    kmovw %k5, %eax # encoding: [0xc5,0xf8,0x93,0xc5]
-; CHECK-NEXT:    vpinsrw $6, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x06]
+; CHECK-NEXT:    vpinsrb $6, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x06]
 ; CHECK-NEXT:    movl $15, %eax # encoding: [0xb8,0x0f,0x00,0x00,0x00]
-; CHECK-NEXT:    vpinsrw $7, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x07]
+; CHECK-NEXT:    vpinsrb $7, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x07]
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res0 = call i8 @llvm.x86.avx512.mask.cmp.d.128(<4 x i32> %a0, <4 x i32> %a1, i32 0, i8 -1)
   %vec0 = insertelement <8 x i8> undef, i8 %res0, i32 0
@@ -8594,58 +8587,58 @@ define <8 x i8> @test_mask_cmp_d_128(<4 x i32> %a0, <4 x i32> %a1, i8 %mask) {
 ; X86-LABEL: test_mask_cmp_d_128:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
-; X86-NEXT:    kmovw %eax, %k2 # encoding: [0xc5,0xf8,0x92,0xd0]
-; X86-NEXT:    vpcmpeqd %xmm1, %xmm0, %k0 {%k2} # encoding: [0x62,0xf1,0x7d,0x0a,0x76,0xc1]
-; X86-NEXT:    vpcmpgtd %xmm0, %xmm1, %k1 {%k2} # encoding: [0x62,0xf1,0x75,0x0a,0x66,0xc8]
-; X86-NEXT:    vpcmpled %xmm1, %xmm0, %k3 {%k2} # encoding: [0x62,0xf3,0x7d,0x0a,0x1f,0xd9,0x02]
-; X86-NEXT:    vpcmpneqd %xmm1, %xmm0, %k4 {%k2} # encoding: [0x62,0xf3,0x7d,0x0a,0x1f,0xe1,0x04]
-; X86-NEXT:    vpcmpnltd %xmm1, %xmm0, %k5 {%k2} # encoding: [0x62,0xf3,0x7d,0x0a,0x1f,0xe9,0x05]
-; X86-NEXT:    vpcmpgtd %xmm1, %xmm0, %k6 {%k2} # encoding: [0x62,0xf1,0x7d,0x0a,0x66,0xf1]
-; X86-NEXT:    kshiftlw $12, %k2, %k2 # encoding: [0xc4,0xe3,0xf9,0x32,0xd2,0x0c]
-; X86-NEXT:    kshiftrw $12, %k2, %k2 # encoding: [0xc4,0xe3,0xf9,0x30,0xd2,0x0c]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpcmpeqd %xmm1, %xmm0, %k0 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0x76,0xc1]
+; X86-NEXT:    vpcmpgtd %xmm0, %xmm1, %k2 {%k1} # encoding: [0x62,0xf1,0x75,0x09,0x66,0xd0]
+; X86-NEXT:    vpcmpled %xmm1, %xmm0, %k3 {%k1} # encoding: [0x62,0xf3,0x7d,0x09,0x1f,0xd9,0x02]
+; X86-NEXT:    vpcmpneqd %xmm1, %xmm0, %k4 {%k1} # encoding: [0x62,0xf3,0x7d,0x09,0x1f,0xe1,0x04]
+; X86-NEXT:    vpcmpnltd %xmm1, %xmm0, %k5 {%k1} # encoding: [0x62,0xf3,0x7d,0x09,0x1f,0xe9,0x05]
+; X86-NEXT:    vpcmpgtd %xmm1, %xmm0, %k6 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0x66,0xf1]
+; X86-NEXT:    kshiftlw $12, %k1, %k1 # encoding: [0xc4,0xe3,0xf9,0x32,0xc9,0x0c]
+; X86-NEXT:    kshiftrw $12, %k1, %k1 # encoding: [0xc4,0xe3,0xf9,0x30,0xc9,0x0c]
 ; X86-NEXT:    kmovw %k0, %eax # encoding: [0xc5,0xf8,0x93,0xc0]
 ; X86-NEXT:    vpxor %xmm0, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xef,0xc0]
-; X86-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x00]
-; X86-NEXT:    kmovw %k1, %eax # encoding: [0xc5,0xf8,0x93,0xc1]
-; X86-NEXT:    vpinsrw $1, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x01]
+; X86-NEXT:    vpinsrb $0, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x00]
+; X86-NEXT:    kmovw %k2, %eax # encoding: [0xc5,0xf8,0x93,0xc2]
+; X86-NEXT:    vpinsrb $1, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x01]
 ; X86-NEXT:    kmovw %k3, %eax # encoding: [0xc5,0xf8,0x93,0xc3]
-; X86-NEXT:    vpinsrw $2, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x02]
+; X86-NEXT:    vpinsrb $2, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x02]
 ; X86-NEXT:    kmovw %k4, %eax # encoding: [0xc5,0xf8,0x93,0xc4]
-; X86-NEXT:    vpinsrw $4, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x04]
+; X86-NEXT:    vpinsrb $4, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x04]
 ; X86-NEXT:    kmovw %k5, %eax # encoding: [0xc5,0xf8,0x93,0xc5]
-; X86-NEXT:    vpinsrw $5, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x05]
+; X86-NEXT:    vpinsrb $5, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x05]
 ; X86-NEXT:    kmovw %k6, %eax # encoding: [0xc5,0xf8,0x93,0xc6]
-; X86-NEXT:    vpinsrw $6, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x06]
-; X86-NEXT:    kmovw %k2, %eax # encoding: [0xc5,0xf8,0x93,0xc2]
-; X86-NEXT:    vpinsrw $7, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x07]
+; X86-NEXT:    vpinsrb $6, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x06]
+; X86-NEXT:    kmovw %k1, %eax # encoding: [0xc5,0xf8,0x93,0xc1]
+; X86-NEXT:    vpinsrb $7, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x07]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_mask_cmp_d_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    kmovw %edi, %k2 # encoding: [0xc5,0xf8,0x92,0xd7]
-; X64-NEXT:    vpcmpeqd %xmm1, %xmm0, %k0 {%k2} # encoding: [0x62,0xf1,0x7d,0x0a,0x76,0xc1]
-; X64-NEXT:    vpcmpgtd %xmm0, %xmm1, %k1 {%k2} # encoding: [0x62,0xf1,0x75,0x0a,0x66,0xc8]
-; X64-NEXT:    vpcmpled %xmm1, %xmm0, %k3 {%k2} # encoding: [0x62,0xf3,0x7d,0x0a,0x1f,0xd9,0x02]
-; X64-NEXT:    vpcmpneqd %xmm1, %xmm0, %k4 {%k2} # encoding: [0x62,0xf3,0x7d,0x0a,0x1f,0xe1,0x04]
-; X64-NEXT:    vpcmpnltd %xmm1, %xmm0, %k5 {%k2} # encoding: [0x62,0xf3,0x7d,0x0a,0x1f,0xe9,0x05]
-; X64-NEXT:    vpcmpgtd %xmm1, %xmm0, %k6 {%k2} # encoding: [0x62,0xf1,0x7d,0x0a,0x66,0xf1]
-; X64-NEXT:    kshiftlw $12, %k2, %k2 # encoding: [0xc4,0xe3,0xf9,0x32,0xd2,0x0c]
-; X64-NEXT:    kshiftrw $12, %k2, %k2 # encoding: [0xc4,0xe3,0xf9,0x30,0xd2,0x0c]
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vpcmpeqd %xmm1, %xmm0, %k0 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0x76,0xc1]
+; X64-NEXT:    vpcmpgtd %xmm0, %xmm1, %k2 {%k1} # encoding: [0x62,0xf1,0x75,0x09,0x66,0xd0]
+; X64-NEXT:    vpcmpled %xmm1, %xmm0, %k3 {%k1} # encoding: [0x62,0xf3,0x7d,0x09,0x1f,0xd9,0x02]
+; X64-NEXT:    vpcmpneqd %xmm1, %xmm0, %k4 {%k1} # encoding: [0x62,0xf3,0x7d,0x09,0x1f,0xe1,0x04]
+; X64-NEXT:    vpcmpnltd %xmm1, %xmm0, %k5 {%k1} # encoding: [0x62,0xf3,0x7d,0x09,0x1f,0xe9,0x05]
+; X64-NEXT:    vpcmpgtd %xmm1, %xmm0, %k6 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0x66,0xf1]
+; X64-NEXT:    kshiftlw $12, %k1, %k1 # encoding: [0xc4,0xe3,0xf9,0x32,0xc9,0x0c]
+; X64-NEXT:    kshiftrw $12, %k1, %k1 # encoding: [0xc4,0xe3,0xf9,0x30,0xc9,0x0c]
 ; X64-NEXT:    kmovw %k0, %eax # encoding: [0xc5,0xf8,0x93,0xc0]
 ; X64-NEXT:    vpxor %xmm0, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xef,0xc0]
-; X64-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x00]
-; X64-NEXT:    kmovw %k1, %eax # encoding: [0xc5,0xf8,0x93,0xc1]
-; X64-NEXT:    vpinsrw $1, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x01]
+; X64-NEXT:    vpinsrb $0, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x00]
+; X64-NEXT:    kmovw %k2, %eax # encoding: [0xc5,0xf8,0x93,0xc2]
+; X64-NEXT:    vpinsrb $1, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x01]
 ; X64-NEXT:    kmovw %k3, %eax # encoding: [0xc5,0xf8,0x93,0xc3]
-; X64-NEXT:    vpinsrw $2, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x02]
+; X64-NEXT:    vpinsrb $2, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x02]
 ; X64-NEXT:    kmovw %k4, %eax # encoding: [0xc5,0xf8,0x93,0xc4]
-; X64-NEXT:    vpinsrw $4, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x04]
+; X64-NEXT:    vpinsrb $4, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x04]
 ; X64-NEXT:    kmovw %k5, %eax # encoding: [0xc5,0xf8,0x93,0xc5]
-; X64-NEXT:    vpinsrw $5, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x05]
+; X64-NEXT:    vpinsrb $5, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x05]
 ; X64-NEXT:    kmovw %k6, %eax # encoding: [0xc5,0xf8,0x93,0xc6]
-; X64-NEXT:    vpinsrw $6, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x06]
-; X64-NEXT:    kmovw %k2, %eax # encoding: [0xc5,0xf8,0x93,0xc2]
-; X64-NEXT:    vpinsrw $7, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x07]
+; X64-NEXT:    vpinsrb $6, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x06]
+; X64-NEXT:    kmovw %k1, %eax # encoding: [0xc5,0xf8,0x93,0xc1]
+; X64-NEXT:    vpinsrb $7, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x07]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res0 = call i8 @llvm.x86.avx512.mask.cmp.d.128(<4 x i32> %a0, <4 x i32> %a1, i32 0, i8 %mask)
   %vec0 = insertelement <8 x i8> undef, i8 %res0, i32 0
@@ -8679,19 +8672,19 @@ define <8 x i8> @test_ucmp_d_128(<4 x i32> %a0, <4 x i32> %a1) {
 ; CHECK-NEXT:    vpcmpnleud %xmm1, %xmm0, %k5 # encoding: [0x62,0xf3,0x7d,0x08,0x1e,0xe9,0x06]
 ; CHECK-NEXT:    kmovw %k0, %eax # encoding: [0xc5,0xf8,0x93,0xc0]
 ; CHECK-NEXT:    vpxor %xmm0, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xef,0xc0]
-; CHECK-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x00]
+; CHECK-NEXT:    vpinsrb $0, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x00]
 ; CHECK-NEXT:    kmovw %k1, %eax # encoding: [0xc5,0xf8,0x93,0xc1]
-; CHECK-NEXT:    vpinsrw $1, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x01]
+; CHECK-NEXT:    vpinsrb $1, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x01]
 ; CHECK-NEXT:    kmovw %k2, %eax # encoding: [0xc5,0xf8,0x93,0xc2]
-; CHECK-NEXT:    vpinsrw $2, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x02]
+; CHECK-NEXT:    vpinsrb $2, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x02]
 ; CHECK-NEXT:    kmovw %k3, %eax # encoding: [0xc5,0xf8,0x93,0xc3]
-; CHECK-NEXT:    vpinsrw $4, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x04]
+; CHECK-NEXT:    vpinsrb $4, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x04]
 ; CHECK-NEXT:    kmovw %k4, %eax # encoding: [0xc5,0xf8,0x93,0xc4]
-; CHECK-NEXT:    vpinsrw $5, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x05]
+; CHECK-NEXT:    vpinsrb $5, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x05]
 ; CHECK-NEXT:    kmovw %k5, %eax # encoding: [0xc5,0xf8,0x93,0xc5]
-; CHECK-NEXT:    vpinsrw $6, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x06]
+; CHECK-NEXT:    vpinsrb $6, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x06]
 ; CHECK-NEXT:    movl $15, %eax # encoding: [0xb8,0x0f,0x00,0x00,0x00]
-; CHECK-NEXT:    vpinsrw $7, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x07]
+; CHECK-NEXT:    vpinsrb $7, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x07]
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res0 = call i8 @llvm.x86.avx512.mask.ucmp.d.128(<4 x i32> %a0, <4 x i32> %a1, i32 0, i8 -1)
   %vec0 = insertelement <8 x i8> undef, i8 %res0, i32 0
@@ -8716,58 +8709,58 @@ define <8 x i8> @test_mask_ucmp_d_128(<4 x i32> %a0, <4 x i32> %a1, i8 %mask) {
 ; X86-LABEL: test_mask_ucmp_d_128:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
-; X86-NEXT:    kmovw %eax, %k2 # encoding: [0xc5,0xf8,0x92,0xd0]
-; X86-NEXT:    vpcmpeqd %xmm1, %xmm0, %k0 {%k2} # encoding: [0x62,0xf1,0x7d,0x0a,0x76,0xc1]
-; X86-NEXT:    vpcmpltud %xmm1, %xmm0, %k1 {%k2} # encoding: [0x62,0xf3,0x7d,0x0a,0x1e,0xc9,0x01]
-; X86-NEXT:    vpcmpleud %xmm1, %xmm0, %k3 {%k2} # encoding: [0x62,0xf3,0x7d,0x0a,0x1e,0xd9,0x02]
-; X86-NEXT:    vpcmpneqd %xmm1, %xmm0, %k4 {%k2} # encoding: [0x62,0xf3,0x7d,0x0a,0x1f,0xe1,0x04]
-; X86-NEXT:    vpcmpnltud %xmm1, %xmm0, %k5 {%k2} # encoding: [0x62,0xf3,0x7d,0x0a,0x1e,0xe9,0x05]
-; X86-NEXT:    vpcmpnleud %xmm1, %xmm0, %k6 {%k2} # encoding: [0x62,0xf3,0x7d,0x0a,0x1e,0xf1,0x06]
-; X86-NEXT:    kshiftlw $12, %k2, %k2 # encoding: [0xc4,0xe3,0xf9,0x32,0xd2,0x0c]
-; X86-NEXT:    kshiftrw $12, %k2, %k2 # encoding: [0xc4,0xe3,0xf9,0x30,0xd2,0x0c]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpcmpeqd %xmm1, %xmm0, %k0 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0x76,0xc1]
+; X86-NEXT:    vpcmpltud %xmm1, %xmm0, %k2 {%k1} # encoding: [0x62,0xf3,0x7d,0x09,0x1e,0xd1,0x01]
+; X86-NEXT:    vpcmpleud %xmm1, %xmm0, %k3 {%k1} # encoding: [0x62,0xf3,0x7d,0x09,0x1e,0xd9,0x02]
+; X86-NEXT:    vpcmpneqd %xmm1, %xmm0, %k4 {%k1} # encoding: [0x62,0xf3,0x7d,0x09,0x1f,0xe1,0x04]
+; X86-NEXT:    vpcmpnltud %xmm1, %xmm0, %k5 {%k1} # encoding: [0x62,0xf3,0x7d,0x09,0x1e,0xe9,0x05]
+; X86-NEXT:    vpcmpnleud %xmm1, %xmm0, %k6 {%k1} # encoding: [0x62,0xf3,0x7d,0x09,0x1e,0xf1,0x06]
+; X86-NEXT:    kshiftlw $12, %k1, %k1 # encoding: [0xc4,0xe3,0xf9,0x32,0xc9,0x0c]
+; X86-NEXT:    kshiftrw $12, %k1, %k1 # encoding: [0xc4,0xe3,0xf9,0x30,0xc9,0x0c]
 ; X86-NEXT:    kmovw %k0, %eax # encoding: [0xc5,0xf8,0x93,0xc0]
 ; X86-NEXT:    vpxor %xmm0, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xef,0xc0]
-; X86-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x00]
-; X86-NEXT:    kmovw %k1, %eax # encoding: [0xc5,0xf8,0x93,0xc1]
-; X86-NEXT:    vpinsrw $1, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x01]
+; X86-NEXT:    vpinsrb $0, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x00]
+; X86-NEXT:    kmovw %k2, %eax # encoding: [0xc5,0xf8,0x93,0xc2]
+; X86-NEXT:    vpinsrb $1, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x01]
 ; X86-NEXT:    kmovw %k3, %eax # encoding: [0xc5,0xf8,0x93,0xc3]
-; X86-NEXT:    vpinsrw $2, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x02]
+; X86-NEXT:    vpinsrb $2, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x02]
 ; X86-NEXT:    kmovw %k4, %eax # encoding: [0xc5,0xf8,0x93,0xc4]
-; X86-NEXT:    vpinsrw $4, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x04]
+; X86-NEXT:    vpinsrb $4, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x04]
 ; X86-NEXT:    kmovw %k5, %eax # encoding: [0xc5,0xf8,0x93,0xc5]
-; X86-NEXT:    vpinsrw $5, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x05]
+; X86-NEXT:    vpinsrb $5, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x05]
 ; X86-NEXT:    kmovw %k6, %eax # encoding: [0xc5,0xf8,0x93,0xc6]
-; X86-NEXT:    vpinsrw $6, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x06]
-; X86-NEXT:    kmovw %k2, %eax # encoding: [0xc5,0xf8,0x93,0xc2]
-; X86-NEXT:    vpinsrw $7, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x07]
+; X86-NEXT:    vpinsrb $6, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x06]
+; X86-NEXT:    kmovw %k1, %eax # encoding: [0xc5,0xf8,0x93,0xc1]
+; X86-NEXT:    vpinsrb $7, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x07]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_mask_ucmp_d_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    kmovw %edi, %k2 # encoding: [0xc5,0xf8,0x92,0xd7]
-; X64-NEXT:    vpcmpeqd %xmm1, %xmm0, %k0 {%k2} # encoding: [0x62,0xf1,0x7d,0x0a,0x76,0xc1]
-; X64-NEXT:    vpcmpltud %xmm1, %xmm0, %k1 {%k2} # encoding: [0x62,0xf3,0x7d,0x0a,0x1e,0xc9,0x01]
-; X64-NEXT:    vpcmpleud %xmm1, %xmm0, %k3 {%k2} # encoding: [0x62,0xf3,0x7d,0x0a,0x1e,0xd9,0x02]
-; X64-NEXT:    vpcmpneqd %xmm1, %xmm0, %k4 {%k2} # encoding: [0x62,0xf3,0x7d,0x0a,0x1f,0xe1,0x04]
-; X64-NEXT:    vpcmpnltud %xmm1, %xmm0, %k5 {%k2} # encoding: [0x62,0xf3,0x7d,0x0a,0x1e,0xe9,0x05]
-; X64-NEXT:    vpcmpnleud %xmm1, %xmm0, %k6 {%k2} # encoding: [0x62,0xf3,0x7d,0x0a,0x1e,0xf1,0x06]
-; X64-NEXT:    kshiftlw $12, %k2, %k2 # encoding: [0xc4,0xe3,0xf9,0x32,0xd2,0x0c]
-; X64-NEXT:    kshiftrw $12, %k2, %k2 # encoding: [0xc4,0xe3,0xf9,0x30,0xd2,0x0c]
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vpcmpeqd %xmm1, %xmm0, %k0 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0x76,0xc1]
+; X64-NEXT:    vpcmpltud %xmm1, %xmm0, %k2 {%k1} # encoding: [0x62,0xf3,0x7d,0x09,0x1e,0xd1,0x01]
+; X64-NEXT:    vpcmpleud %xmm1, %xmm0, %k3 {%k1} # encoding: [0x62,0xf3,0x7d,0x09,0x1e,0xd9,0x02]
+; X64-NEXT:    vpcmpneqd %xmm1, %xmm0, %k4 {%k1} # encoding: [0x62,0xf3,0x7d,0x09,0x1f,0xe1,0x04]
+; X64-NEXT:    vpcmpnltud %xmm1, %xmm0, %k5 {%k1} # encoding: [0x62,0xf3,0x7d,0x09,0x1e,0xe9,0x05]
+; X64-NEXT:    vpcmpnleud %xmm1, %xmm0, %k6 {%k1} # encoding: [0x62,0xf3,0x7d,0x09,0x1e,0xf1,0x06]
+; X64-NEXT:    kshiftlw $12, %k1, %k1 # encoding: [0xc4,0xe3,0xf9,0x32,0xc9,0x0c]
+; X64-NEXT:    kshiftrw $12, %k1, %k1 # encoding: [0xc4,0xe3,0xf9,0x30,0xc9,0x0c]
 ; X64-NEXT:    kmovw %k0, %eax # encoding: [0xc5,0xf8,0x93,0xc0]
 ; X64-NEXT:    vpxor %xmm0, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xef,0xc0]
-; X64-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x00]
-; X64-NEXT:    kmovw %k1, %eax # encoding: [0xc5,0xf8,0x93,0xc1]
-; X64-NEXT:    vpinsrw $1, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x01]
+; X64-NEXT:    vpinsrb $0, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x00]
+; X64-NEXT:    kmovw %k2, %eax # encoding: [0xc5,0xf8,0x93,0xc2]
+; X64-NEXT:    vpinsrb $1, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x01]
 ; X64-NEXT:    kmovw %k3, %eax # encoding: [0xc5,0xf8,0x93,0xc3]
-; X64-NEXT:    vpinsrw $2, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x02]
+; X64-NEXT:    vpinsrb $2, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x02]
 ; X64-NEXT:    kmovw %k4, %eax # encoding: [0xc5,0xf8,0x93,0xc4]
-; X64-NEXT:    vpinsrw $4, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x04]
+; X64-NEXT:    vpinsrb $4, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x04]
 ; X64-NEXT:    kmovw %k5, %eax # encoding: [0xc5,0xf8,0x93,0xc5]
-; X64-NEXT:    vpinsrw $5, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x05]
+; X64-NEXT:    vpinsrb $5, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x05]
 ; X64-NEXT:    kmovw %k6, %eax # encoding: [0xc5,0xf8,0x93,0xc6]
-; X64-NEXT:    vpinsrw $6, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x06]
-; X64-NEXT:    kmovw %k2, %eax # encoding: [0xc5,0xf8,0x93,0xc2]
-; X64-NEXT:    vpinsrw $7, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x07]
+; X64-NEXT:    vpinsrb $6, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x06]
+; X64-NEXT:    kmovw %k1, %eax # encoding: [0xc5,0xf8,0x93,0xc1]
+; X64-NEXT:    vpinsrb $7, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x07]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res0 = call i8 @llvm.x86.avx512.mask.ucmp.d.128(<4 x i32> %a0, <4 x i32> %a1, i32 0, i8 %mask)
   %vec0 = insertelement <8 x i8> undef, i8 %res0, i32 0
@@ -8801,19 +8794,19 @@ define <8 x i8> @test_cmp_q_128(<2 x i64> %a0, <2 x i64> %a1) {
 ; CHECK-NEXT:    vpcmpgtq %xmm1, %xmm0, %k5 # encoding: [0x62,0xf2,0xfd,0x08,0x37,0xe9]
 ; CHECK-NEXT:    kmovw %k0, %eax # encoding: [0xc5,0xf8,0x93,0xc0]
 ; CHECK-NEXT:    vpxor %xmm0, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xef,0xc0]
-; CHECK-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x00]
+; CHECK-NEXT:    vpinsrb $0, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x00]
 ; CHECK-NEXT:    kmovw %k1, %eax # encoding: [0xc5,0xf8,0x93,0xc1]
-; CHECK-NEXT:    vpinsrw $1, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x01]
+; CHECK-NEXT:    vpinsrb $1, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x01]
 ; CHECK-NEXT:    kmovw %k2, %eax # encoding: [0xc5,0xf8,0x93,0xc2]
-; CHECK-NEXT:    vpinsrw $2, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x02]
+; CHECK-NEXT:    vpinsrb $2, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x02]
 ; CHECK-NEXT:    kmovw %k3, %eax # encoding: [0xc5,0xf8,0x93,0xc3]
-; CHECK-NEXT:    vpinsrw $4, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x04]
+; CHECK-NEXT:    vpinsrb $4, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x04]
 ; CHECK-NEXT:    kmovw %k4, %eax # encoding: [0xc5,0xf8,0x93,0xc4]
-; CHECK-NEXT:    vpinsrw $5, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x05]
+; CHECK-NEXT:    vpinsrb $5, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x05]
 ; CHECK-NEXT:    kmovw %k5, %eax # encoding: [0xc5,0xf8,0x93,0xc5]
-; CHECK-NEXT:    vpinsrw $6, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x06]
+; CHECK-NEXT:    vpinsrb $6, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x06]
 ; CHECK-NEXT:    movl $3, %eax # encoding: [0xb8,0x03,0x00,0x00,0x00]
-; CHECK-NEXT:    vpinsrw $7, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x07]
+; CHECK-NEXT:    vpinsrb $7, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x07]
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res0 = call i8 @llvm.x86.avx512.mask.cmp.q.128(<2 x i64> %a0, <2 x i64> %a1, i32 0, i8 -1)
   %vec0 = insertelement <8 x i8> undef, i8 %res0, i32 0
@@ -8838,58 +8831,58 @@ define <8 x i8> @test_mask_cmp_q_128(<2 x i64> %a0, <2 x i64> %a1, i8 %mask) {
 ; X86-LABEL: test_mask_cmp_q_128:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
-; X86-NEXT:    kmovw %eax, %k2 # encoding: [0xc5,0xf8,0x92,0xd0]
-; X86-NEXT:    vpcmpeqq %xmm1, %xmm0, %k0 {%k2} # encoding: [0x62,0xf2,0xfd,0x0a,0x29,0xc1]
-; X86-NEXT:    vpcmpgtq %xmm0, %xmm1, %k1 {%k2} # encoding: [0x62,0xf2,0xf5,0x0a,0x37,0xc8]
-; X86-NEXT:    vpcmpleq %xmm1, %xmm0, %k3 {%k2} # encoding: [0x62,0xf3,0xfd,0x0a,0x1f,0xd9,0x02]
-; X86-NEXT:    vpcmpneqq %xmm1, %xmm0, %k4 {%k2} # encoding: [0x62,0xf3,0xfd,0x0a,0x1f,0xe1,0x04]
-; X86-NEXT:    vpcmpnltq %xmm1, %xmm0, %k5 {%k2} # encoding: [0x62,0xf3,0xfd,0x0a,0x1f,0xe9,0x05]
-; X86-NEXT:    vpcmpgtq %xmm1, %xmm0, %k6 {%k2} # encoding: [0x62,0xf2,0xfd,0x0a,0x37,0xf1]
-; X86-NEXT:    kshiftlw $14, %k2, %k2 # encoding: [0xc4,0xe3,0xf9,0x32,0xd2,0x0e]
-; X86-NEXT:    kshiftrw $14, %k2, %k2 # encoding: [0xc4,0xe3,0xf9,0x30,0xd2,0x0e]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpcmpeqq %xmm1, %xmm0, %k0 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x29,0xc1]
+; X86-NEXT:    vpcmpgtq %xmm0, %xmm1, %k2 {%k1} # encoding: [0x62,0xf2,0xf5,0x09,0x37,0xd0]
+; X86-NEXT:    vpcmpleq %xmm1, %xmm0, %k3 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x1f,0xd9,0x02]
+; X86-NEXT:    vpcmpneqq %xmm1, %xmm0, %k4 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x1f,0xe1,0x04]
+; X86-NEXT:    vpcmpnltq %xmm1, %xmm0, %k5 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x1f,0xe9,0x05]
+; X86-NEXT:    vpcmpgtq %xmm1, %xmm0, %k6 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x37,0xf1]
+; X86-NEXT:    kshiftlw $14, %k1, %k1 # encoding: [0xc4,0xe3,0xf9,0x32,0xc9,0x0e]
+; X86-NEXT:    kshiftrw $14, %k1, %k1 # encoding: [0xc4,0xe3,0xf9,0x30,0xc9,0x0e]
 ; X86-NEXT:    kmovw %k0, %eax # encoding: [0xc5,0xf8,0x93,0xc0]
 ; X86-NEXT:    vpxor %xmm0, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xef,0xc0]
-; X86-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x00]
-; X86-NEXT:    kmovw %k1, %eax # encoding: [0xc5,0xf8,0x93,0xc1]
-; X86-NEXT:    vpinsrw $1, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x01]
+; X86-NEXT:    vpinsrb $0, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x00]
+; X86-NEXT:    kmovw %k2, %eax # encoding: [0xc5,0xf8,0x93,0xc2]
+; X86-NEXT:    vpinsrb $1, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x01]
 ; X86-NEXT:    kmovw %k3, %eax # encoding: [0xc5,0xf8,0x93,0xc3]
-; X86-NEXT:    vpinsrw $2, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x02]
+; X86-NEXT:    vpinsrb $2, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x02]
 ; X86-NEXT:    kmovw %k4, %eax # encoding: [0xc5,0xf8,0x93,0xc4]
-; X86-NEXT:    vpinsrw $4, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x04]
+; X86-NEXT:    vpinsrb $4, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x04]
 ; X86-NEXT:    kmovw %k5, %eax # encoding: [0xc5,0xf8,0x93,0xc5]
-; X86-NEXT:    vpinsrw $5, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x05]
+; X86-NEXT:    vpinsrb $5, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x05]
 ; X86-NEXT:    kmovw %k6, %eax # encoding: [0xc5,0xf8,0x93,0xc6]
-; X86-NEXT:    vpinsrw $6, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x06]
-; X86-NEXT:    kmovw %k2, %eax # encoding: [0xc5,0xf8,0x93,0xc2]
-; X86-NEXT:    vpinsrw $7, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x07]
+; X86-NEXT:    vpinsrb $6, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x06]
+; X86-NEXT:    kmovw %k1, %eax # encoding: [0xc5,0xf8,0x93,0xc1]
+; X86-NEXT:    vpinsrb $7, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x07]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_mask_cmp_q_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    kmovw %edi, %k2 # encoding: [0xc5,0xf8,0x92,0xd7]
-; X64-NEXT:    vpcmpeqq %xmm1, %xmm0, %k0 {%k2} # encoding: [0x62,0xf2,0xfd,0x0a,0x29,0xc1]
-; X64-NEXT:    vpcmpgtq %xmm0, %xmm1, %k1 {%k2} # encoding: [0x62,0xf2,0xf5,0x0a,0x37,0xc8]
-; X64-NEXT:    vpcmpleq %xmm1, %xmm0, %k3 {%k2} # encoding: [0x62,0xf3,0xfd,0x0a,0x1f,0xd9,0x02]
-; X64-NEXT:    vpcmpneqq %xmm1, %xmm0, %k4 {%k2} # encoding: [0x62,0xf3,0xfd,0x0a,0x1f,0xe1,0x04]
-; X64-NEXT:    vpcmpnltq %xmm1, %xmm0, %k5 {%k2} # encoding: [0x62,0xf3,0xfd,0x0a,0x1f,0xe9,0x05]
-; X64-NEXT:    vpcmpgtq %xmm1, %xmm0, %k6 {%k2} # encoding: [0x62,0xf2,0xfd,0x0a,0x37,0xf1]
-; X64-NEXT:    kshiftlw $14, %k2, %k2 # encoding: [0xc4,0xe3,0xf9,0x32,0xd2,0x0e]
-; X64-NEXT:    kshiftrw $14, %k2, %k2 # encoding: [0xc4,0xe3,0xf9,0x30,0xd2,0x0e]
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vpcmpeqq %xmm1, %xmm0, %k0 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x29,0xc1]
+; X64-NEXT:    vpcmpgtq %xmm0, %xmm1, %k2 {%k1} # encoding: [0x62,0xf2,0xf5,0x09,0x37,0xd0]
+; X64-NEXT:    vpcmpleq %xmm1, %xmm0, %k3 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x1f,0xd9,0x02]
+; X64-NEXT:    vpcmpneqq %xmm1, %xmm0, %k4 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x1f,0xe1,0x04]
+; X64-NEXT:    vpcmpnltq %xmm1, %xmm0, %k5 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x1f,0xe9,0x05]
+; X64-NEXT:    vpcmpgtq %xmm1, %xmm0, %k6 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x37,0xf1]
+; X64-NEXT:    kshiftlw $14, %k1, %k1 # encoding: [0xc4,0xe3,0xf9,0x32,0xc9,0x0e]
+; X64-NEXT:    kshiftrw $14, %k1, %k1 # encoding: [0xc4,0xe3,0xf9,0x30,0xc9,0x0e]
 ; X64-NEXT:    kmovw %k0, %eax # encoding: [0xc5,0xf8,0x93,0xc0]
 ; X64-NEXT:    vpxor %xmm0, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xef,0xc0]
-; X64-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x00]
-; X64-NEXT:    kmovw %k1, %eax # encoding: [0xc5,0xf8,0x93,0xc1]
-; X64-NEXT:    vpinsrw $1, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x01]
+; X64-NEXT:    vpinsrb $0, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x00]
+; X64-NEXT:    kmovw %k2, %eax # encoding: [0xc5,0xf8,0x93,0xc2]
+; X64-NEXT:    vpinsrb $1, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x01]
 ; X64-NEXT:    kmovw %k3, %eax # encoding: [0xc5,0xf8,0x93,0xc3]
-; X64-NEXT:    vpinsrw $2, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x02]
+; X64-NEXT:    vpinsrb $2, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x02]
 ; X64-NEXT:    kmovw %k4, %eax # encoding: [0xc5,0xf8,0x93,0xc4]
-; X64-NEXT:    vpinsrw $4, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x04]
+; X64-NEXT:    vpinsrb $4, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x04]
 ; X64-NEXT:    kmovw %k5, %eax # encoding: [0xc5,0xf8,0x93,0xc5]
-; X64-NEXT:    vpinsrw $5, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x05]
+; X64-NEXT:    vpinsrb $5, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x05]
 ; X64-NEXT:    kmovw %k6, %eax # encoding: [0xc5,0xf8,0x93,0xc6]
-; X64-NEXT:    vpinsrw $6, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x06]
-; X64-NEXT:    kmovw %k2, %eax # encoding: [0xc5,0xf8,0x93,0xc2]
-; X64-NEXT:    vpinsrw $7, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x07]
+; X64-NEXT:    vpinsrb $6, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x06]
+; X64-NEXT:    kmovw %k1, %eax # encoding: [0xc5,0xf8,0x93,0xc1]
+; X64-NEXT:    vpinsrb $7, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x07]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res0 = call i8 @llvm.x86.avx512.mask.cmp.q.128(<2 x i64> %a0, <2 x i64> %a1, i32 0, i8 %mask)
   %vec0 = insertelement <8 x i8> undef, i8 %res0, i32 0
@@ -8923,19 +8916,19 @@ define <8 x i8> @test_ucmp_q_128(<2 x i64> %a0, <2 x i64> %a1) {
 ; CHECK-NEXT:    vpcmpnleuq %xmm1, %xmm0, %k5 # encoding: [0x62,0xf3,0xfd,0x08,0x1e,0xe9,0x06]
 ; CHECK-NEXT:    kmovw %k0, %eax # encoding: [0xc5,0xf8,0x93,0xc0]
 ; CHECK-NEXT:    vpxor %xmm0, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xef,0xc0]
-; CHECK-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x00]
+; CHECK-NEXT:    vpinsrb $0, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x00]
 ; CHECK-NEXT:    kmovw %k1, %eax # encoding: [0xc5,0xf8,0x93,0xc1]
-; CHECK-NEXT:    vpinsrw $1, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x01]
+; CHECK-NEXT:    vpinsrb $1, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x01]
 ; CHECK-NEXT:    kmovw %k2, %eax # encoding: [0xc5,0xf8,0x93,0xc2]
-; CHECK-NEXT:    vpinsrw $2, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x02]
+; CHECK-NEXT:    vpinsrb $2, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x02]
 ; CHECK-NEXT:    kmovw %k3, %eax # encoding: [0xc5,0xf8,0x93,0xc3]
-; CHECK-NEXT:    vpinsrw $4, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x04]
+; CHECK-NEXT:    vpinsrb $4, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x04]
 ; CHECK-NEXT:    kmovw %k4, %eax # encoding: [0xc5,0xf8,0x93,0xc4]
-; CHECK-NEXT:    vpinsrw $5, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x05]
+; CHECK-NEXT:    vpinsrb $5, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x05]
 ; CHECK-NEXT:    kmovw %k5, %eax # encoding: [0xc5,0xf8,0x93,0xc5]
-; CHECK-NEXT:    vpinsrw $6, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x06]
+; CHECK-NEXT:    vpinsrb $6, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x06]
 ; CHECK-NEXT:    movl $3, %eax # encoding: [0xb8,0x03,0x00,0x00,0x00]
-; CHECK-NEXT:    vpinsrw $7, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x07]
+; CHECK-NEXT:    vpinsrb $7, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x07]
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res0 = call i8 @llvm.x86.avx512.mask.ucmp.q.128(<2 x i64> %a0, <2 x i64> %a1, i32 0, i8 -1)
   %vec0 = insertelement <8 x i8> undef, i8 %res0, i32 0
@@ -8960,58 +8953,58 @@ define <8 x i8> @test_mask_ucmp_q_128(<2 x i64> %a0, <2 x i64> %a1, i8 %mask) {
 ; X86-LABEL: test_mask_ucmp_q_128:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
-; X86-NEXT:    kmovw %eax, %k2 # encoding: [0xc5,0xf8,0x92,0xd0]
-; X86-NEXT:    vpcmpeqq %xmm1, %xmm0, %k0 {%k2} # encoding: [0x62,0xf2,0xfd,0x0a,0x29,0xc1]
-; X86-NEXT:    vpcmpltuq %xmm1, %xmm0, %k1 {%k2} # encoding: [0x62,0xf3,0xfd,0x0a,0x1e,0xc9,0x01]
-; X86-NEXT:    vpcmpleuq %xmm1, %xmm0, %k3 {%k2} # encoding: [0x62,0xf3,0xfd,0x0a,0x1e,0xd9,0x02]
-; X86-NEXT:    vpcmpneqq %xmm1, %xmm0, %k4 {%k2} # encoding: [0x62,0xf3,0xfd,0x0a,0x1f,0xe1,0x04]
-; X86-NEXT:    vpcmpnltuq %xmm1, %xmm0, %k5 {%k2} # encoding: [0x62,0xf3,0xfd,0x0a,0x1e,0xe9,0x05]
-; X86-NEXT:    vpcmpnleuq %xmm1, %xmm0, %k6 {%k2} # encoding: [0x62,0xf3,0xfd,0x0a,0x1e,0xf1,0x06]
-; X86-NEXT:    kshiftlw $14, %k2, %k2 # encoding: [0xc4,0xe3,0xf9,0x32,0xd2,0x0e]
-; X86-NEXT:    kshiftrw $14, %k2, %k2 # encoding: [0xc4,0xe3,0xf9,0x30,0xd2,0x0e]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpcmpeqq %xmm1, %xmm0, %k0 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x29,0xc1]
+; X86-NEXT:    vpcmpltuq %xmm1, %xmm0, %k2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x1e,0xd1,0x01]
+; X86-NEXT:    vpcmpleuq %xmm1, %xmm0, %k3 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x1e,0xd9,0x02]
+; X86-NEXT:    vpcmpneqq %xmm1, %xmm0, %k4 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x1f,0xe1,0x04]
+; X86-NEXT:    vpcmpnltuq %xmm1, %xmm0, %k5 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x1e,0xe9,0x05]
+; X86-NEXT:    vpcmpnleuq %xmm1, %xmm0, %k6 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x1e,0xf1,0x06]
+; X86-NEXT:    kshiftlw $14, %k1, %k1 # encoding: [0xc4,0xe3,0xf9,0x32,0xc9,0x0e]
+; X86-NEXT:    kshiftrw $14, %k1, %k1 # encoding: [0xc4,0xe3,0xf9,0x30,0xc9,0x0e]
 ; X86-NEXT:    kmovw %k0, %eax # encoding: [0xc5,0xf8,0x93,0xc0]
 ; X86-NEXT:    vpxor %xmm0, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xef,0xc0]
-; X86-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x00]
-; X86-NEXT:    kmovw %k1, %eax # encoding: [0xc5,0xf8,0x93,0xc1]
-; X86-NEXT:    vpinsrw $1, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x01]
+; X86-NEXT:    vpinsrb $0, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x00]
+; X86-NEXT:    kmovw %k2, %eax # encoding: [0xc5,0xf8,0x93,0xc2]
+; X86-NEXT:    vpinsrb $1, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x01]
 ; X86-NEXT:    kmovw %k3, %eax # encoding: [0xc5,0xf8,0x93,0xc3]
-; X86-NEXT:    vpinsrw $2, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x02]
+; X86-NEXT:    vpinsrb $2, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x02]
 ; X86-NEXT:    kmovw %k4, %eax # encoding: [0xc5,0xf8,0x93,0xc4]
-; X86-NEXT:    vpinsrw $4, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x04]
+; X86-NEXT:    vpinsrb $4, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x04]
 ; X86-NEXT:    kmovw %k5, %eax # encoding: [0xc5,0xf8,0x93,0xc5]
-; X86-NEXT:    vpinsrw $5, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x05]
+; X86-NEXT:    vpinsrb $5, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x05]
 ; X86-NEXT:    kmovw %k6, %eax # encoding: [0xc5,0xf8,0x93,0xc6]
-; X86-NEXT:    vpinsrw $6, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x06]
-; X86-NEXT:    kmovw %k2, %eax # encoding: [0xc5,0xf8,0x93,0xc2]
-; X86-NEXT:    vpinsrw $7, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x07]
+; X86-NEXT:    vpinsrb $6, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x06]
+; X86-NEXT:    kmovw %k1, %eax # encoding: [0xc5,0xf8,0x93,0xc1]
+; X86-NEXT:    vpinsrb $7, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x07]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_mask_ucmp_q_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    kmovw %edi, %k2 # encoding: [0xc5,0xf8,0x92,0xd7]
-; X64-NEXT:    vpcmpeqq %xmm1, %xmm0, %k0 {%k2} # encoding: [0x62,0xf2,0xfd,0x0a,0x29,0xc1]
-; X64-NEXT:    vpcmpltuq %xmm1, %xmm0, %k1 {%k2} # encoding: [0x62,0xf3,0xfd,0x0a,0x1e,0xc9,0x01]
-; X64-NEXT:    vpcmpleuq %xmm1, %xmm0, %k3 {%k2} # encoding: [0x62,0xf3,0xfd,0x0a,0x1e,0xd9,0x02]
-; X64-NEXT:    vpcmpneqq %xmm1, %xmm0, %k4 {%k2} # encoding: [0x62,0xf3,0xfd,0x0a,0x1f,0xe1,0x04]
-; X64-NEXT:    vpcmpnltuq %xmm1, %xmm0, %k5 {%k2} # encoding: [0x62,0xf3,0xfd,0x0a,0x1e,0xe9,0x05]
-; X64-NEXT:    vpcmpnleuq %xmm1, %xmm0, %k6 {%k2} # encoding: [0x62,0xf3,0xfd,0x0a,0x1e,0xf1,0x06]
-; X64-NEXT:    kshiftlw $14, %k2, %k2 # encoding: [0xc4,0xe3,0xf9,0x32,0xd2,0x0e]
-; X64-NEXT:    kshiftrw $14, %k2, %k2 # encoding: [0xc4,0xe3,0xf9,0x30,0xd2,0x0e]
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vpcmpeqq %xmm1, %xmm0, %k0 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x29,0xc1]
+; X64-NEXT:    vpcmpltuq %xmm1, %xmm0, %k2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x1e,0xd1,0x01]
+; X64-NEXT:    vpcmpleuq %xmm1, %xmm0, %k3 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x1e,0xd9,0x02]
+; X64-NEXT:    vpcmpneqq %xmm1, %xmm0, %k4 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x1f,0xe1,0x04]
+; X64-NEXT:    vpcmpnltuq %xmm1, %xmm0, %k5 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x1e,0xe9,0x05]
+; X64-NEXT:    vpcmpnleuq %xmm1, %xmm0, %k6 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x1e,0xf1,0x06]
+; X64-NEXT:    kshiftlw $14, %k1, %k1 # encoding: [0xc4,0xe3,0xf9,0x32,0xc9,0x0e]
+; X64-NEXT:    kshiftrw $14, %k1, %k1 # encoding: [0xc4,0xe3,0xf9,0x30,0xc9,0x0e]
 ; X64-NEXT:    kmovw %k0, %eax # encoding: [0xc5,0xf8,0x93,0xc0]
 ; X64-NEXT:    vpxor %xmm0, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xef,0xc0]
-; X64-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x00]
-; X64-NEXT:    kmovw %k1, %eax # encoding: [0xc5,0xf8,0x93,0xc1]
-; X64-NEXT:    vpinsrw $1, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x01]
+; X64-NEXT:    vpinsrb $0, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x00]
+; X64-NEXT:    kmovw %k2, %eax # encoding: [0xc5,0xf8,0x93,0xc2]
+; X64-NEXT:    vpinsrb $1, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x01]
 ; X64-NEXT:    kmovw %k3, %eax # encoding: [0xc5,0xf8,0x93,0xc3]
-; X64-NEXT:    vpinsrw $2, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x02]
+; X64-NEXT:    vpinsrb $2, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x02]
 ; X64-NEXT:    kmovw %k4, %eax # encoding: [0xc5,0xf8,0x93,0xc4]
-; X64-NEXT:    vpinsrw $4, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x04]
+; X64-NEXT:    vpinsrb $4, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x04]
 ; X64-NEXT:    kmovw %k5, %eax # encoding: [0xc5,0xf8,0x93,0xc5]
-; X64-NEXT:    vpinsrw $5, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x05]
+; X64-NEXT:    vpinsrb $5, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x05]
 ; X64-NEXT:    kmovw %k6, %eax # encoding: [0xc5,0xf8,0x93,0xc6]
-; X64-NEXT:    vpinsrw $6, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x06]
-; X64-NEXT:    kmovw %k2, %eax # encoding: [0xc5,0xf8,0x93,0xc2]
-; X64-NEXT:    vpinsrw $7, %eax, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xc4,0xc0,0x07]
+; X64-NEXT:    vpinsrb $6, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x06]
+; X64-NEXT:    kmovw %k1, %eax # encoding: [0xc5,0xf8,0x93,0xc1]
+; X64-NEXT:    vpinsrb $7, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x20,0xc0,0x07]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res0 = call i8 @llvm.x86.avx512.mask.ucmp.q.128(<2 x i64> %a0, <2 x i64> %a1, i32 0, i8 %mask)
   %vec0 = insertelement <8 x i8> undef, i8 %res0, i32 0
diff --git a/llvm/test/CodeGen/X86/avx512vl-nontemporal.ll b/llvm/test/CodeGen/X86/avx512vl-nontemporal.ll
index 683cae69b..48cb47947 100644
--- a/llvm/test/CodeGen/X86/avx512vl-nontemporal.ll
+++ b/llvm/test/CodeGen/X86/avx512vl-nontemporal.ll
@@ -1,17 +1,29 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=skx --show-mc-encoding | FileCheck %s
 
 define i32 @f256(<8 x float> %A, <8 x float> %AA, i8* %B, <4 x double> %C, <4 x double> %CC, i32 %D, <4 x i64> %E, <4 x i64> %EE, i32* %loadptr) {
-; CHECK: vmovntps %ymm{{.*}} ## EVEX TO VEX Compression encoding: [0xc5
+; CHECK-LABEL: f256:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    movl (%rdx), %eax ## encoding: [0x8b,0x02]
+; CHECK-NEXT:    vaddps %ymm1, %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfc,0x58,0xc1]
+; CHECK-NEXT:    vmovntps %ymm0, (%rdi) ## EVEX TO VEX Compression encoding: [0xc5,0xfc,0x2b,0x07]
+; CHECK-NEXT:    vpaddq %ymm5, %ymm4, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xdd,0xd4,0xc5]
+; CHECK-NEXT:    addl (%rdx), %eax ## encoding: [0x03,0x02]
+; CHECK-NEXT:    vmovntdq %ymm0, (%rdi) ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0xe7,0x07]
+; CHECK-NEXT:    vaddpd %ymm3, %ymm2, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xed,0x58,0xc3]
+; CHECK-NEXT:    addl (%rdx), %eax ## encoding: [0x03,0x02]
+; CHECK-NEXT:    vmovntpd %ymm0, (%rdi) ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0x2b,0x07]
+; CHECK-NEXT:    addl (%rdx), %eax ## encoding: [0x03,0x02]
+; CHECK-NEXT:    vzeroupper ## encoding: [0xc5,0xf8,0x77]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %v0 = load i32, i32* %loadptr, align 1
   %cast = bitcast i8* %B to <8 x float>*
   %A2 = fadd <8 x float> %A, %AA
   store <8 x float> %A2, <8 x float>* %cast, align 64, !nontemporal !0
-; CHECK: vmovntdq %ymm{{.*}} ## EVEX TO VEX Compression encoding: [0xc5
   %v1 = load i32, i32* %loadptr, align 1
   %cast1 = bitcast i8* %B to <4 x i64>*
   %E2 = add <4 x i64> %E, %EE
   store <4 x i64> %E2, <4 x i64>* %cast1, align 64, !nontemporal !0
-; CHECK: vmovntpd %ymm{{.*}} ## EVEX TO VEX Compression encoding: [0xc5
   %v2 = load i32, i32* %loadptr, align 1
   %cast2 = bitcast i8* %B to <4 x double>*
   %C2 = fadd <4 x double> %C, %CC
@@ -24,17 +36,27 @@ define i32 @f256(<8 x float> %A, <8 x float> %AA, i8* %B, <4 x double> %C, <4 x
 }
 
 define i32  @f128(<4 x float> %A, <4 x float> %AA, i8* %B, <2 x double> %C, <2 x double> %CC, i32 %D, <2 x i64> %E, <2 x i64> %EE, i32* %loadptr) {
+; CHECK-LABEL: f128:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    movl (%rdx), %eax ## encoding: [0x8b,0x02]
+; CHECK-NEXT:    vaddps %xmm1, %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf8,0x58,0xc1]
+; CHECK-NEXT:    vmovntps %xmm0, (%rdi) ## EVEX TO VEX Compression encoding: [0xc5,0xf8,0x2b,0x07]
+; CHECK-NEXT:    vpaddq %xmm5, %xmm4, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xd9,0xd4,0xc5]
+; CHECK-NEXT:    addl (%rdx), %eax ## encoding: [0x03,0x02]
+; CHECK-NEXT:    vmovntdq %xmm0, (%rdi) ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0xe7,0x07]
+; CHECK-NEXT:    vaddpd %xmm3, %xmm2, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xe9,0x58,0xc3]
+; CHECK-NEXT:    addl (%rdx), %eax ## encoding: [0x03,0x02]
+; CHECK-NEXT:    vmovntpd %xmm0, (%rdi) ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0x2b,0x07]
+; CHECK-NEXT:    addl (%rdx), %eax ## encoding: [0x03,0x02]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %v0 = load i32, i32* %loadptr, align 1
-; CHECK: vmovntps %xmm{{.*}} ## EVEX TO VEX Compression encoding: [0xc5
   %cast = bitcast i8* %B to <4 x float>*
   %A2 = fadd <4 x float> %A, %AA
   store <4 x float> %A2, <4 x float>* %cast, align 64, !nontemporal !0
-; CHECK: vmovntdq %xmm{{.*}} ## EVEX TO VEX Compression encoding: [0xc5
   %v1 = load i32, i32* %loadptr, align 1
   %cast1 = bitcast i8* %B to <2 x i64>*
   %E2 = add <2 x i64> %E, %EE
   store <2 x i64> %E2, <2 x i64>* %cast1, align 64, !nontemporal !0
-; CHECK: vmovntpd %xmm{{.*}} ## EVEX TO VEX Compression encoding: [0xc5
   %v2 = load i32, i32* %loadptr, align 1
   %cast2 = bitcast i8* %B to <2 x double>*
   %C2 = fadd <2 x double> %C, %CC
diff --git a/llvm/test/CodeGen/X86/avx512vl-vec-cmp.ll b/llvm/test/CodeGen/X86/avx512vl-vec-cmp.ll
index 5024e67fd..55e127e4f 100644
--- a/llvm/test/CodeGen/X86/avx512vl-vec-cmp.ll
+++ b/llvm/test/CodeGen/X86/avx512vl-vec-cmp.ll
@@ -378,8 +378,7 @@ define <4 x i64> @test256_13(<4 x i64> %x, <4 x i64> %x1, i64* %yb.ptr) nounwind
 ; NoVLX:       # %bb.0:
 ; NoVLX-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %ymm2
-; NoVLX-NEXT:    vpcmpeqq %zmm2, %zmm0, %k1
+; NoVLX-NEXT:    vpcmpeqq (%rdi){1to8}, %zmm0, %k1
 ; NoVLX-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; NoVLX-NEXT:    retq
@@ -402,8 +401,7 @@ define <8 x i32> @test256_14(<8 x i32> %x, i32* %yb.ptr, <8 x i32> %x1) nounwind
 ; NoVLX:       # %bb.0:
 ; NoVLX-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rdi), %ymm2
-; NoVLX-NEXT:    vpcmpled %zmm2, %zmm0, %k1
+; NoVLX-NEXT:    vpcmpled (%rdi){1to16}, %zmm0, %k1
 ; NoVLX-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; NoVLX-NEXT:    retq
@@ -429,8 +427,7 @@ define <8 x i32> @test256_15(<8 x i32> %x, i32* %yb.ptr, <8 x i32> %x1, <8 x i32
 ; NoVLX-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
 ; NoVLX-NEXT:    vpcmpnltd %zmm2, %zmm1, %k1
-; NoVLX-NEXT:    vpbroadcastd (%rdi), %ymm2
-; NoVLX-NEXT:    vpcmpgtd %zmm2, %zmm0, %k1 {%k1}
+; NoVLX-NEXT:    vpcmpgtd (%rdi){1to16}, %zmm0, %k1 {%k1}
 ; NoVLX-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; NoVLX-NEXT:    retq
@@ -458,8 +455,7 @@ define <4 x i64> @test256_16(<4 x i64> %x, i64* %yb.ptr, <4 x i64> %x1, <4 x i64
 ; NoVLX-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
 ; NoVLX-NEXT:    vpcmpnltq %zmm2, %zmm1, %k1
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %ymm2
-; NoVLX-NEXT:    vpcmpgtq %zmm2, %zmm0, %k1 {%k1}
+; NoVLX-NEXT:    vpcmpgtq (%rdi){1to8}, %zmm0, %k1 {%k1}
 ; NoVLX-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; NoVLX-NEXT:    retq
@@ -937,8 +933,7 @@ define <2 x i64> @test128_13(<2 x i64> %x, <2 x i64> %x1, i64* %yb.ptr) nounwind
 ; NoVLX:       # %bb.0:
 ; NoVLX-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %xmm2
-; NoVLX-NEXT:    vpcmpeqq %zmm2, %zmm0, %k1
+; NoVLX-NEXT:    vpcmpeqq (%rdi){1to8}, %zmm0, %k1
 ; NoVLX-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
 ; NoVLX-NEXT:    retq
@@ -961,8 +956,7 @@ define <4 x i32> @test128_14(<4 x i32> %x, i32* %yb.ptr, <4 x i32> %x1) nounwind
 ; NoVLX:       # %bb.0:
 ; NoVLX-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rdi), %xmm2
-; NoVLX-NEXT:    vpcmpled %zmm2, %zmm0, %k1
+; NoVLX-NEXT:    vpcmpled (%rdi){1to16}, %zmm0, %k1
 ; NoVLX-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
 ; NoVLX-NEXT:    retq
@@ -988,8 +982,7 @@ define <4 x i32> @test128_15(<4 x i32> %x, i32* %yb.ptr, <4 x i32> %x1, <4 x i32
 ; NoVLX-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; NoVLX-NEXT:    vpcmpnltd %zmm2, %zmm1, %k1
-; NoVLX-NEXT:    vpbroadcastd (%rdi), %xmm2
-; NoVLX-NEXT:    vpcmpgtd %zmm2, %zmm0, %k1 {%k1}
+; NoVLX-NEXT:    vpcmpgtd (%rdi){1to16}, %zmm0, %k1 {%k1}
 ; NoVLX-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
 ; NoVLX-NEXT:    retq
@@ -1017,8 +1010,7 @@ define <2 x i64> @test128_16(<2 x i64> %x, i64* %yb.ptr, <2 x i64> %x1, <2 x i64
 ; NoVLX-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; NoVLX-NEXT:    vpcmpnltq %zmm2, %zmm1, %k1
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %xmm2
-; NoVLX-NEXT:    vpcmpgtq %zmm2, %zmm0, %k1 {%k1}
+; NoVLX-NEXT:    vpcmpgtq (%rdi){1to8}, %zmm0, %k1 {%k1}
 ; NoVLX-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
 ; NoVLX-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/avx512vl-vec-masked-cmp.ll b/llvm/test/CodeGen/X86/avx512vl-vec-masked-cmp.ll
index 79de4aec4..f3f65c545 100644
--- a/llvm/test/CodeGen/X86/avx512vl-vec-masked-cmp.ll
+++ b/llvm/test/CodeGen/X86/avx512vl-vec-masked-cmp.ll
@@ -122,7 +122,6 @@ define zeroext i64 @test_vpcmpeqb_v16i1_v64i1_mask(<2 x i64> %__a, <2 x i64> %__
 ; NoVLX-NEXT:    vpmovsxbd %xmm0, %zmm0
 ; NoVLX-NEXT:    vptestmd %zmm0, %zmm0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -147,7 +146,6 @@ define zeroext i64 @test_vpcmpeqb_v16i1_v64i1_mask_mem(<2 x i64> %__a, <2 x i64>
 ; NoVLX-NEXT:    vpmovsxbd %xmm0, %zmm0
 ; NoVLX-NEXT:    vptestmd %zmm0, %zmm0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -594,7 +592,6 @@ define zeroext i64 @test_vpcmpeqw_v8i1_v64i1_mask(<2 x i64> %__a, <2 x i64> %__b
 ; NoVLX-NEXT:    vpmovsxwq %xmm0, %zmm0
 ; NoVLX-NEXT:    vptestmq %zmm0, %zmm0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -619,7 +616,6 @@ define zeroext i64 @test_vpcmpeqw_v8i1_v64i1_mask_mem(<2 x i64> %__a, <2 x i64>*
 ; NoVLX-NEXT:    vpmovsxwq %xmm0, %zmm0
 ; NoVLX-NEXT:    vptestmq %zmm0, %zmm0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -647,7 +643,6 @@ define zeroext i64 @test_masked_vpcmpeqw_v8i1_v64i1_mask(i8 zeroext %__u, <2 x i
 ; NoVLX-NEXT:    kmovw %edi, %k1
 ; NoVLX-NEXT:    vptestmq %zmm0, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -676,7 +671,6 @@ define zeroext i64 @test_masked_vpcmpeqw_v8i1_v64i1_mask_mem(i8 zeroext %__u, <2
 ; NoVLX-NEXT:    kmovw %edi, %k1
 ; NoVLX-NEXT:    vptestmq %zmm0, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -817,7 +811,6 @@ define zeroext i64 @test_vpcmpeqw_v16i1_v64i1_mask(<4 x i64> %__a, <4 x i64> %__
 ; NoVLX-NEXT:    vpmovsxwd %ymm0, %zmm0
 ; NoVLX-NEXT:    vptestmd %zmm0, %zmm0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -843,7 +836,6 @@ define zeroext i64 @test_vpcmpeqw_v16i1_v64i1_mask_mem(<4 x i64> %__a, <4 x i64>
 ; NoVLX-NEXT:    vpmovsxwd %ymm0, %zmm0
 ; NoVLX-NEXT:    vptestmd %zmm0, %zmm0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -1199,8 +1191,7 @@ define zeroext i8 @test_vpcmpeqd_v4i1_v8i1_mask_mem_b(<2 x i64> %__a, i32* %__b)
 ; NoVLX-LABEL: test_vpcmpeqd_v4i1_v8i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpeqd %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpeqd (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -1230,9 +1221,8 @@ define zeroext i8 @test_masked_vpcmpeqd_v4i1_v8i1_mask_mem_b(i8 zeroext %__u, <2
 ; NoVLX-LABEL: test_masked_vpcmpeqd_v4i1_v8i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpeqd %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpeqd (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -1390,8 +1380,7 @@ define zeroext i16 @test_vpcmpeqd_v4i1_v16i1_mask_mem_b(<2 x i64> %__a, i32* %__
 ; NoVLX-LABEL: test_vpcmpeqd_v4i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpeqd %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpeqd (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -1421,9 +1410,8 @@ define zeroext i16 @test_masked_vpcmpeqd_v4i1_v16i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpeqd_v4i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpeqd %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpeqd (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -1572,8 +1560,7 @@ define zeroext i32 @test_vpcmpeqd_v4i1_v32i1_mask_mem_b(<2 x i64> %__a, i32* %__
 ; NoVLX-LABEL: test_vpcmpeqd_v4i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpeqd %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpeqd (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -1601,9 +1588,8 @@ define zeroext i32 @test_masked_vpcmpeqd_v4i1_v32i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpeqd_v4i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpeqd %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpeqd (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -1639,7 +1625,6 @@ define zeroext i64 @test_vpcmpeqd_v4i1_v64i1_mask(<2 x i64> %__a, <2 x i64> %__b
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -1666,7 +1651,6 @@ define zeroext i64 @test_vpcmpeqd_v4i1_v64i1_mask_mem(<2 x i64> %__a, <2 x i64>*
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -1696,7 +1680,6 @@ define zeroext i64 @test_masked_vpcmpeqd_v4i1_v64i1_mask(i8 zeroext %__u, <2 x i
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -1728,7 +1711,6 @@ define zeroext i64 @test_masked_vpcmpeqd_v4i1_v64i1_mask_mem(i8 zeroext %__u, <2
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -1755,12 +1737,10 @@ define zeroext i64 @test_vpcmpeqd_v4i1_v64i1_mask_mem_b(<2 x i64> %__a, i32* %__
 ; NoVLX-LABEL: test_vpcmpeqd_v4i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpeqd %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpeqd (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -1785,13 +1765,11 @@ define zeroext i64 @test_masked_vpcmpeqd_v4i1_v64i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpeqd_v4i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpeqd %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpeqd (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -1948,8 +1926,7 @@ define zeroext i16 @test_vpcmpeqd_v8i1_v16i1_mask_mem_b(<4 x i64> %__a, i32* %__
 ; NoVLX-LABEL: test_vpcmpeqd_v8i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rdi), %ymm1
-; NoVLX-NEXT:    vpcmpeqd %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpeqd (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -1980,9 +1957,8 @@ define zeroext i16 @test_masked_vpcmpeqd_v8i1_v16i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpeqd_v8i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rsi), %ymm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpeqd %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpeqd (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -2133,8 +2109,7 @@ define zeroext i32 @test_vpcmpeqd_v8i1_v32i1_mask_mem_b(<4 x i64> %__a, i32* %__
 ; NoVLX-LABEL: test_vpcmpeqd_v8i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rdi), %ymm1
-; NoVLX-NEXT:    vpcmpeqd %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpeqd (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -2163,9 +2138,8 @@ define zeroext i32 @test_masked_vpcmpeqd_v8i1_v32i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpeqd_v8i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rsi), %ymm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpeqd %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpeqd (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -2201,7 +2175,6 @@ define zeroext i64 @test_vpcmpeqd_v8i1_v64i1_mask(<4 x i64> %__a, <4 x i64> %__b
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -2229,7 +2202,6 @@ define zeroext i64 @test_vpcmpeqd_v8i1_v64i1_mask_mem(<4 x i64> %__a, <4 x i64>*
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -2260,7 +2232,6 @@ define zeroext i64 @test_masked_vpcmpeqd_v8i1_v64i1_mask(i8 zeroext %__u, <4 x i
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -2292,7 +2263,6 @@ define zeroext i64 @test_masked_vpcmpeqd_v8i1_v64i1_mask_mem(i8 zeroext %__u, <4
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -2319,12 +2289,10 @@ define zeroext i64 @test_vpcmpeqd_v8i1_v64i1_mask_mem_b(<4 x i64> %__a, i32* %__
 ; NoVLX-LABEL: test_vpcmpeqd_v8i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rdi), %ymm1
-; NoVLX-NEXT:    vpcmpeqd %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpeqd (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -2350,13 +2318,11 @@ define zeroext i64 @test_masked_vpcmpeqd_v8i1_v64i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpeqd_v8i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rsi), %ymm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpeqd %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpeqd (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -2543,7 +2509,6 @@ define zeroext i64 @test_vpcmpeqd_v16i1_v64i1_mask(<8 x i64> %__a, <8 x i64> %__
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    vpcmpeqd %zmm1, %zmm0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -2567,7 +2532,6 @@ define zeroext i64 @test_vpcmpeqd_v16i1_v64i1_mask_mem(<8 x i64> %__a, <8 x i64>
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    vpcmpeqd (%rdi), %zmm0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -2648,7 +2612,6 @@ define zeroext i64 @test_vpcmpeqd_v16i1_v64i1_mask_mem_b(<8 x i64> %__a, i32* %_
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    vpcmpeqd (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -2707,7 +2670,6 @@ define zeroext i4 @test_vpcmpeqq_v2i1_v4i1_mask(<2 x i64> %__a, <2 x i64> %__b)
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    andl $3, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -2734,7 +2696,6 @@ define zeroext i4 @test_vpcmpeqq_v2i1_v4i1_mask_mem(<2 x i64> %__a, <2 x i64>* %
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    andl $3, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -2764,7 +2725,6 @@ define zeroext i4 @test_masked_vpcmpeqq_v2i1_v4i1_mask(i8 zeroext %__u, <2 x i64
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    andl $3, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -2796,7 +2756,6 @@ define zeroext i4 @test_masked_vpcmpeqq_v2i1_v4i1_mask_mem(i8 zeroext %__u, <2 x
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    andl $3, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -2823,12 +2782,10 @@ define zeroext i4 @test_vpcmpeqq_v2i1_v4i1_mask_mem_b(<2 x i64> %__a, i64* %__b)
 ; NoVLX-LABEL: test_vpcmpeqq_v2i1_v4i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpeqq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpeqq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    andl $3, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -2853,13 +2810,11 @@ define zeroext i4 @test_masked_vpcmpeqq_v2i1_v4i1_mask_mem_b(i8 zeroext %__u, <2
 ; NoVLX-LABEL: test_masked_vpcmpeqq_v2i1_v4i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpeqq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpeqq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    andl $3, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -3013,8 +2968,7 @@ define zeroext i8 @test_vpcmpeqq_v2i1_v8i1_mask_mem_b(<2 x i64> %__a, i64* %__b)
 ; NoVLX-LABEL: test_vpcmpeqq_v2i1_v8i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpeqq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpeqq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -3044,9 +2998,8 @@ define zeroext i8 @test_masked_vpcmpeqq_v2i1_v8i1_mask_mem_b(i8 zeroext %__u, <2
 ; NoVLX-LABEL: test_masked_vpcmpeqq_v2i1_v8i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpeqq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpeqq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -3204,8 +3157,7 @@ define zeroext i16 @test_vpcmpeqq_v2i1_v16i1_mask_mem_b(<2 x i64> %__a, i64* %__
 ; NoVLX-LABEL: test_vpcmpeqq_v2i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpeqq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpeqq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -3235,9 +3187,8 @@ define zeroext i16 @test_masked_vpcmpeqq_v2i1_v16i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpeqq_v2i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpeqq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpeqq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -3386,8 +3337,7 @@ define zeroext i32 @test_vpcmpeqq_v2i1_v32i1_mask_mem_b(<2 x i64> %__a, i64* %__
 ; NoVLX-LABEL: test_vpcmpeqq_v2i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpeqq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpeqq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -3415,9 +3365,8 @@ define zeroext i32 @test_masked_vpcmpeqq_v2i1_v32i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpeqq_v2i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpeqq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpeqq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -3453,7 +3402,6 @@ define zeroext i64 @test_vpcmpeqq_v2i1_v64i1_mask(<2 x i64> %__a, <2 x i64> %__b
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -3480,7 +3428,6 @@ define zeroext i64 @test_vpcmpeqq_v2i1_v64i1_mask_mem(<2 x i64> %__a, <2 x i64>*
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -3510,7 +3457,6 @@ define zeroext i64 @test_masked_vpcmpeqq_v2i1_v64i1_mask(i8 zeroext %__u, <2 x i
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -3542,7 +3488,6 @@ define zeroext i64 @test_masked_vpcmpeqq_v2i1_v64i1_mask_mem(i8 zeroext %__u, <2
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -3569,12 +3514,10 @@ define zeroext i64 @test_vpcmpeqq_v2i1_v64i1_mask_mem_b(<2 x i64> %__a, i64* %__
 ; NoVLX-LABEL: test_vpcmpeqq_v2i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpeqq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpeqq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -3599,13 +3542,11 @@ define zeroext i64 @test_masked_vpcmpeqq_v2i1_v64i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpeqq_v2i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpeqq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpeqq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -3764,8 +3705,7 @@ define zeroext i8 @test_vpcmpeqq_v4i1_v8i1_mask_mem_b(<4 x i64> %__a, i64* %__b)
 ; NoVLX-LABEL: test_vpcmpeqq_v4i1_v8i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %ymm1
-; NoVLX-NEXT:    vpcmpeqq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpeqq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -3796,9 +3736,8 @@ define zeroext i8 @test_masked_vpcmpeqq_v4i1_v8i1_mask_mem_b(i8 zeroext %__u, <4
 ; NoVLX-LABEL: test_masked_vpcmpeqq_v4i1_v8i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %ymm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpeqq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpeqq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -3961,8 +3900,7 @@ define zeroext i16 @test_vpcmpeqq_v4i1_v16i1_mask_mem_b(<4 x i64> %__a, i64* %__
 ; NoVLX-LABEL: test_vpcmpeqq_v4i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %ymm1
-; NoVLX-NEXT:    vpcmpeqq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpeqq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -3993,9 +3931,8 @@ define zeroext i16 @test_masked_vpcmpeqq_v4i1_v16i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpeqq_v4i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %ymm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpeqq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpeqq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -4149,8 +4086,7 @@ define zeroext i32 @test_vpcmpeqq_v4i1_v32i1_mask_mem_b(<4 x i64> %__a, i64* %__
 ; NoVLX-LABEL: test_vpcmpeqq_v4i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %ymm1
-; NoVLX-NEXT:    vpcmpeqq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpeqq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -4179,9 +4115,8 @@ define zeroext i32 @test_masked_vpcmpeqq_v4i1_v32i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpeqq_v4i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %ymm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpeqq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpeqq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -4218,7 +4153,6 @@ define zeroext i64 @test_vpcmpeqq_v4i1_v64i1_mask(<4 x i64> %__a, <4 x i64> %__b
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -4246,7 +4180,6 @@ define zeroext i64 @test_vpcmpeqq_v4i1_v64i1_mask_mem(<4 x i64> %__a, <4 x i64>*
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -4277,7 +4210,6 @@ define zeroext i64 @test_masked_vpcmpeqq_v4i1_v64i1_mask(i8 zeroext %__u, <4 x i
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -4310,7 +4242,6 @@ define zeroext i64 @test_masked_vpcmpeqq_v4i1_v64i1_mask_mem(i8 zeroext %__u, <4
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -4338,12 +4269,10 @@ define zeroext i64 @test_vpcmpeqq_v4i1_v64i1_mask_mem_b(<4 x i64> %__a, i64* %__
 ; NoVLX-LABEL: test_vpcmpeqq_v4i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %ymm1
-; NoVLX-NEXT:    vpcmpeqq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpeqq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -4369,13 +4298,11 @@ define zeroext i64 @test_masked_vpcmpeqq_v4i1_v64i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpeqq_v4i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %ymm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpeqq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpeqq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -4733,7 +4660,6 @@ define zeroext i64 @test_vpcmpeqq_v8i1_v64i1_mask(<8 x i64> %__a, <8 x i64> %__b
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    vpcmpeqq %zmm1, %zmm0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -4757,7 +4683,6 @@ define zeroext i64 @test_vpcmpeqq_v8i1_v64i1_mask_mem(<8 x i64> %__a, <8 x i64>*
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    vpcmpeqq (%rdi), %zmm0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -4784,7 +4709,6 @@ define zeroext i64 @test_masked_vpcmpeqq_v8i1_v64i1_mask(i8 zeroext %__u, <8 x i
 ; NoVLX-NEXT:    kmovw %edi, %k1
 ; NoVLX-NEXT:    vpcmpeqq %zmm1, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -4812,7 +4736,6 @@ define zeroext i64 @test_masked_vpcmpeqq_v8i1_v64i1_mask_mem(i8 zeroext %__u, <8
 ; NoVLX-NEXT:    kmovw %edi, %k1
 ; NoVLX-NEXT:    vpcmpeqq (%rsi), %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -4840,7 +4763,6 @@ define zeroext i64 @test_vpcmpeqq_v8i1_v64i1_mask_mem_b(<8 x i64> %__a, i64* %__
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    vpcmpeqq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -4868,7 +4790,6 @@ define zeroext i64 @test_masked_vpcmpeqq_v8i1_v64i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-NEXT:    kmovw %edi, %k1
 ; NoVLX-NEXT:    vpcmpeqq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -5005,7 +4926,6 @@ define zeroext i64 @test_vpcmpsgtb_v16i1_v64i1_mask(<2 x i64> %__a, <2 x i64> %_
 ; NoVLX-NEXT:    vpmovsxbd %xmm0, %zmm0
 ; NoVLX-NEXT:    vptestmd %zmm0, %zmm0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -5030,7 +4950,6 @@ define zeroext i64 @test_vpcmpsgtb_v16i1_v64i1_mask_mem(<2 x i64> %__a, <2 x i64
 ; NoVLX-NEXT:    vpmovsxbd %xmm0, %zmm0
 ; NoVLX-NEXT:    vptestmd %zmm0, %zmm0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -5477,7 +5396,6 @@ define zeroext i64 @test_vpcmpsgtw_v8i1_v64i1_mask(<2 x i64> %__a, <2 x i64> %__
 ; NoVLX-NEXT:    vpmovsxwq %xmm0, %zmm0
 ; NoVLX-NEXT:    vptestmq %zmm0, %zmm0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -5502,7 +5420,6 @@ define zeroext i64 @test_vpcmpsgtw_v8i1_v64i1_mask_mem(<2 x i64> %__a, <2 x i64>
 ; NoVLX-NEXT:    vpmovsxwq %xmm0, %zmm0
 ; NoVLX-NEXT:    vptestmq %zmm0, %zmm0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -5530,7 +5447,6 @@ define zeroext i64 @test_masked_vpcmpsgtw_v8i1_v64i1_mask(i8 zeroext %__u, <2 x
 ; NoVLX-NEXT:    kmovw %edi, %k1
 ; NoVLX-NEXT:    vptestmq %zmm0, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -5559,7 +5475,6 @@ define zeroext i64 @test_masked_vpcmpsgtw_v8i1_v64i1_mask_mem(i8 zeroext %__u, <
 ; NoVLX-NEXT:    kmovw %edi, %k1
 ; NoVLX-NEXT:    vptestmq %zmm0, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -5700,7 +5615,6 @@ define zeroext i64 @test_vpcmpsgtw_v16i1_v64i1_mask(<4 x i64> %__a, <4 x i64> %_
 ; NoVLX-NEXT:    vpmovsxwd %ymm0, %zmm0
 ; NoVLX-NEXT:    vptestmd %zmm0, %zmm0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -5726,7 +5640,6 @@ define zeroext i64 @test_vpcmpsgtw_v16i1_v64i1_mask_mem(<4 x i64> %__a, <4 x i64
 ; NoVLX-NEXT:    vpmovsxwd %ymm0, %zmm0
 ; NoVLX-NEXT:    vptestmd %zmm0, %zmm0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -6082,8 +5995,7 @@ define zeroext i8 @test_vpcmpsgtd_v4i1_v8i1_mask_mem_b(<2 x i64> %__a, i32* %__b
 ; NoVLX-LABEL: test_vpcmpsgtd_v4i1_v8i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpgtd %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpgtd (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -6113,9 +6025,8 @@ define zeroext i8 @test_masked_vpcmpsgtd_v4i1_v8i1_mask_mem_b(i8 zeroext %__u, <
 ; NoVLX-LABEL: test_masked_vpcmpsgtd_v4i1_v8i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpgtd %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpgtd (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -6273,8 +6184,7 @@ define zeroext i16 @test_vpcmpsgtd_v4i1_v16i1_mask_mem_b(<2 x i64> %__a, i32* %_
 ; NoVLX-LABEL: test_vpcmpsgtd_v4i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpgtd %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpgtd (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -6304,9 +6214,8 @@ define zeroext i16 @test_masked_vpcmpsgtd_v4i1_v16i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpsgtd_v4i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpgtd %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpgtd (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -6455,8 +6364,7 @@ define zeroext i32 @test_vpcmpsgtd_v4i1_v32i1_mask_mem_b(<2 x i64> %__a, i32* %_
 ; NoVLX-LABEL: test_vpcmpsgtd_v4i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpgtd %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpgtd (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -6484,9 +6392,8 @@ define zeroext i32 @test_masked_vpcmpsgtd_v4i1_v32i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpsgtd_v4i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpgtd %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpgtd (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -6522,7 +6429,6 @@ define zeroext i64 @test_vpcmpsgtd_v4i1_v64i1_mask(<2 x i64> %__a, <2 x i64> %__
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -6549,7 +6455,6 @@ define zeroext i64 @test_vpcmpsgtd_v4i1_v64i1_mask_mem(<2 x i64> %__a, <2 x i64>
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -6579,7 +6484,6 @@ define zeroext i64 @test_masked_vpcmpsgtd_v4i1_v64i1_mask(i8 zeroext %__u, <2 x
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -6611,7 +6515,6 @@ define zeroext i64 @test_masked_vpcmpsgtd_v4i1_v64i1_mask_mem(i8 zeroext %__u, <
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -6638,12 +6541,10 @@ define zeroext i64 @test_vpcmpsgtd_v4i1_v64i1_mask_mem_b(<2 x i64> %__a, i32* %_
 ; NoVLX-LABEL: test_vpcmpsgtd_v4i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpgtd %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpgtd (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -6668,13 +6569,11 @@ define zeroext i64 @test_masked_vpcmpsgtd_v4i1_v64i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpsgtd_v4i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpgtd %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpgtd (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -6831,8 +6730,7 @@ define zeroext i16 @test_vpcmpsgtd_v8i1_v16i1_mask_mem_b(<4 x i64> %__a, i32* %_
 ; NoVLX-LABEL: test_vpcmpsgtd_v8i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rdi), %ymm1
-; NoVLX-NEXT:    vpcmpgtd %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpgtd (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -6863,9 +6761,8 @@ define zeroext i16 @test_masked_vpcmpsgtd_v8i1_v16i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpsgtd_v8i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rsi), %ymm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpgtd %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpgtd (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -7016,8 +6913,7 @@ define zeroext i32 @test_vpcmpsgtd_v8i1_v32i1_mask_mem_b(<4 x i64> %__a, i32* %_
 ; NoVLX-LABEL: test_vpcmpsgtd_v8i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rdi), %ymm1
-; NoVLX-NEXT:    vpcmpgtd %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpgtd (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -7046,9 +6942,8 @@ define zeroext i32 @test_masked_vpcmpsgtd_v8i1_v32i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpsgtd_v8i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rsi), %ymm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpgtd %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpgtd (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -7084,7 +6979,6 @@ define zeroext i64 @test_vpcmpsgtd_v8i1_v64i1_mask(<4 x i64> %__a, <4 x i64> %__
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -7112,7 +7006,6 @@ define zeroext i64 @test_vpcmpsgtd_v8i1_v64i1_mask_mem(<4 x i64> %__a, <4 x i64>
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -7143,7 +7036,6 @@ define zeroext i64 @test_masked_vpcmpsgtd_v8i1_v64i1_mask(i8 zeroext %__u, <4 x
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -7175,7 +7067,6 @@ define zeroext i64 @test_masked_vpcmpsgtd_v8i1_v64i1_mask_mem(i8 zeroext %__u, <
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -7202,12 +7093,10 @@ define zeroext i64 @test_vpcmpsgtd_v8i1_v64i1_mask_mem_b(<4 x i64> %__a, i32* %_
 ; NoVLX-LABEL: test_vpcmpsgtd_v8i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rdi), %ymm1
-; NoVLX-NEXT:    vpcmpgtd %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpgtd (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -7233,13 +7122,11 @@ define zeroext i64 @test_masked_vpcmpsgtd_v8i1_v64i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpsgtd_v8i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rsi), %ymm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpgtd %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpgtd (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -7426,7 +7313,6 @@ define zeroext i64 @test_vpcmpsgtd_v16i1_v64i1_mask(<8 x i64> %__a, <8 x i64> %_
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    vpcmpgtd %zmm1, %zmm0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -7450,7 +7336,6 @@ define zeroext i64 @test_vpcmpsgtd_v16i1_v64i1_mask_mem(<8 x i64> %__a, <8 x i64
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    vpcmpgtd (%rdi), %zmm0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -7531,7 +7416,6 @@ define zeroext i64 @test_vpcmpsgtd_v16i1_v64i1_mask_mem_b(<8 x i64> %__a, i32* %
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    vpcmpgtd (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -7590,7 +7474,6 @@ define zeroext i4 @test_vpcmpsgtq_v2i1_v4i1_mask(<2 x i64> %__a, <2 x i64> %__b)
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    andl $3, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -7617,7 +7500,6 @@ define zeroext i4 @test_vpcmpsgtq_v2i1_v4i1_mask_mem(<2 x i64> %__a, <2 x i64>*
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    andl $3, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -7647,7 +7529,6 @@ define zeroext i4 @test_masked_vpcmpsgtq_v2i1_v4i1_mask(i8 zeroext %__u, <2 x i6
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    andl $3, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -7679,7 +7560,6 @@ define zeroext i4 @test_masked_vpcmpsgtq_v2i1_v4i1_mask_mem(i8 zeroext %__u, <2
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    andl $3, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -7706,12 +7586,10 @@ define zeroext i4 @test_vpcmpsgtq_v2i1_v4i1_mask_mem_b(<2 x i64> %__a, i64* %__b
 ; NoVLX-LABEL: test_vpcmpsgtq_v2i1_v4i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpgtq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpgtq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    andl $3, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -7736,13 +7614,11 @@ define zeroext i4 @test_masked_vpcmpsgtq_v2i1_v4i1_mask_mem_b(i8 zeroext %__u, <
 ; NoVLX-LABEL: test_masked_vpcmpsgtq_v2i1_v4i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpgtq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpgtq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    andl $3, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -7896,8 +7772,7 @@ define zeroext i8 @test_vpcmpsgtq_v2i1_v8i1_mask_mem_b(<2 x i64> %__a, i64* %__b
 ; NoVLX-LABEL: test_vpcmpsgtq_v2i1_v8i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpgtq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpgtq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -7927,9 +7802,8 @@ define zeroext i8 @test_masked_vpcmpsgtq_v2i1_v8i1_mask_mem_b(i8 zeroext %__u, <
 ; NoVLX-LABEL: test_masked_vpcmpsgtq_v2i1_v8i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpgtq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpgtq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -8087,8 +7961,7 @@ define zeroext i16 @test_vpcmpsgtq_v2i1_v16i1_mask_mem_b(<2 x i64> %__a, i64* %_
 ; NoVLX-LABEL: test_vpcmpsgtq_v2i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpgtq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpgtq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -8118,9 +7991,8 @@ define zeroext i16 @test_masked_vpcmpsgtq_v2i1_v16i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpsgtq_v2i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpgtq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpgtq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -8269,8 +8141,7 @@ define zeroext i32 @test_vpcmpsgtq_v2i1_v32i1_mask_mem_b(<2 x i64> %__a, i64* %_
 ; NoVLX-LABEL: test_vpcmpsgtq_v2i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpgtq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpgtq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -8298,9 +8169,8 @@ define zeroext i32 @test_masked_vpcmpsgtq_v2i1_v32i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpsgtq_v2i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpgtq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpgtq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -8336,7 +8206,6 @@ define zeroext i64 @test_vpcmpsgtq_v2i1_v64i1_mask(<2 x i64> %__a, <2 x i64> %__
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -8363,7 +8232,6 @@ define zeroext i64 @test_vpcmpsgtq_v2i1_v64i1_mask_mem(<2 x i64> %__a, <2 x i64>
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -8393,7 +8261,6 @@ define zeroext i64 @test_masked_vpcmpsgtq_v2i1_v64i1_mask(i8 zeroext %__u, <2 x
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -8425,7 +8292,6 @@ define zeroext i64 @test_masked_vpcmpsgtq_v2i1_v64i1_mask_mem(i8 zeroext %__u, <
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -8452,12 +8318,10 @@ define zeroext i64 @test_vpcmpsgtq_v2i1_v64i1_mask_mem_b(<2 x i64> %__a, i64* %_
 ; NoVLX-LABEL: test_vpcmpsgtq_v2i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpgtq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpgtq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -8482,13 +8346,11 @@ define zeroext i64 @test_masked_vpcmpsgtq_v2i1_v64i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpsgtq_v2i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpgtq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpgtq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -8647,8 +8509,7 @@ define zeroext i8 @test_vpcmpsgtq_v4i1_v8i1_mask_mem_b(<4 x i64> %__a, i64* %__b
 ; NoVLX-LABEL: test_vpcmpsgtq_v4i1_v8i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %ymm1
-; NoVLX-NEXT:    vpcmpgtq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpgtq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -8679,9 +8540,8 @@ define zeroext i8 @test_masked_vpcmpsgtq_v4i1_v8i1_mask_mem_b(i8 zeroext %__u, <
 ; NoVLX-LABEL: test_masked_vpcmpsgtq_v4i1_v8i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %ymm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpgtq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpgtq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -8844,8 +8704,7 @@ define zeroext i16 @test_vpcmpsgtq_v4i1_v16i1_mask_mem_b(<4 x i64> %__a, i64* %_
 ; NoVLX-LABEL: test_vpcmpsgtq_v4i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %ymm1
-; NoVLX-NEXT:    vpcmpgtq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpgtq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -8876,9 +8735,8 @@ define zeroext i16 @test_masked_vpcmpsgtq_v4i1_v16i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpsgtq_v4i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %ymm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpgtq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpgtq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -9032,8 +8890,7 @@ define zeroext i32 @test_vpcmpsgtq_v4i1_v32i1_mask_mem_b(<4 x i64> %__a, i64* %_
 ; NoVLX-LABEL: test_vpcmpsgtq_v4i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %ymm1
-; NoVLX-NEXT:    vpcmpgtq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpgtq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -9062,9 +8919,8 @@ define zeroext i32 @test_masked_vpcmpsgtq_v4i1_v32i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpsgtq_v4i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %ymm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpgtq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpgtq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -9101,7 +8957,6 @@ define zeroext i64 @test_vpcmpsgtq_v4i1_v64i1_mask(<4 x i64> %__a, <4 x i64> %__
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -9129,7 +8984,6 @@ define zeroext i64 @test_vpcmpsgtq_v4i1_v64i1_mask_mem(<4 x i64> %__a, <4 x i64>
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -9160,7 +9014,6 @@ define zeroext i64 @test_masked_vpcmpsgtq_v4i1_v64i1_mask(i8 zeroext %__u, <4 x
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -9193,7 +9046,6 @@ define zeroext i64 @test_masked_vpcmpsgtq_v4i1_v64i1_mask_mem(i8 zeroext %__u, <
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -9221,12 +9073,10 @@ define zeroext i64 @test_vpcmpsgtq_v4i1_v64i1_mask_mem_b(<4 x i64> %__a, i64* %_
 ; NoVLX-LABEL: test_vpcmpsgtq_v4i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %ymm1
-; NoVLX-NEXT:    vpcmpgtq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpgtq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -9252,13 +9102,11 @@ define zeroext i64 @test_masked_vpcmpsgtq_v4i1_v64i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpsgtq_v4i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %ymm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpgtq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpgtq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -9616,7 +9464,6 @@ define zeroext i64 @test_vpcmpsgtq_v8i1_v64i1_mask(<8 x i64> %__a, <8 x i64> %__
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    vpcmpgtq %zmm1, %zmm0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -9640,7 +9487,6 @@ define zeroext i64 @test_vpcmpsgtq_v8i1_v64i1_mask_mem(<8 x i64> %__a, <8 x i64>
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    vpcmpgtq (%rdi), %zmm0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -9667,7 +9513,6 @@ define zeroext i64 @test_masked_vpcmpsgtq_v8i1_v64i1_mask(i8 zeroext %__u, <8 x
 ; NoVLX-NEXT:    kmovw %edi, %k1
 ; NoVLX-NEXT:    vpcmpgtq %zmm1, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -9695,7 +9540,6 @@ define zeroext i64 @test_masked_vpcmpsgtq_v8i1_v64i1_mask_mem(i8 zeroext %__u, <
 ; NoVLX-NEXT:    kmovw %edi, %k1
 ; NoVLX-NEXT:    vpcmpgtq (%rsi), %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -9723,7 +9567,6 @@ define zeroext i64 @test_vpcmpsgtq_v8i1_v64i1_mask_mem_b(<8 x i64> %__a, i64* %_
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    vpcmpgtq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -9751,7 +9594,6 @@ define zeroext i64 @test_masked_vpcmpsgtq_v8i1_v64i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-NEXT:    kmovw %edi, %k1
 ; NoVLX-NEXT:    vpcmpgtq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -9895,7 +9737,6 @@ define zeroext i64 @test_vpcmpsgeb_v16i1_v64i1_mask(<2 x i64> %__a, <2 x i64> %_
 ; NoVLX-NEXT:    vpmovsxbd %xmm0, %zmm0
 ; NoVLX-NEXT:    vptestmd %zmm0, %zmm0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -9922,7 +9763,6 @@ define zeroext i64 @test_vpcmpsgeb_v16i1_v64i1_mask_mem(<2 x i64> %__a, <2 x i64
 ; NoVLX-NEXT:    vpmovsxbd %xmm0, %zmm0
 ; NoVLX-NEXT:    vptestmd %zmm0, %zmm0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -10391,7 +10231,6 @@ define zeroext i64 @test_vpcmpsgew_v8i1_v64i1_mask(<2 x i64> %__a, <2 x i64> %__
 ; NoVLX-NEXT:    vpmovsxwq %xmm0, %zmm0
 ; NoVLX-NEXT:    vptestmq %zmm0, %zmm0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -10418,7 +10257,6 @@ define zeroext i64 @test_vpcmpsgew_v8i1_v64i1_mask_mem(<2 x i64> %__a, <2 x i64>
 ; NoVLX-NEXT:    vpmovsxwq %xmm0, %zmm0
 ; NoVLX-NEXT:    vptestmq %zmm0, %zmm0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -10447,7 +10285,6 @@ define zeroext i64 @test_masked_vpcmpsgew_v8i1_v64i1_mask(i8 zeroext %__u, <2 x
 ; NoVLX-NEXT:    kmovw %edi, %k1
 ; NoVLX-NEXT:    vptestmq %zmm0, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -10478,7 +10315,6 @@ define zeroext i64 @test_masked_vpcmpsgew_v8i1_v64i1_mask_mem(i8 zeroext %__u, <
 ; NoVLX-NEXT:    kmovw %edi, %k1
 ; NoVLX-NEXT:    vptestmq %zmm0, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -10626,7 +10462,6 @@ define zeroext i64 @test_vpcmpsgew_v16i1_v64i1_mask(<4 x i64> %__a, <4 x i64> %_
 ; NoVLX-NEXT:    vpmovsxwd %ymm0, %zmm0
 ; NoVLX-NEXT:    vptestmd %zmm0, %zmm0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -10654,7 +10489,6 @@ define zeroext i64 @test_vpcmpsgew_v16i1_v64i1_mask_mem(<4 x i64> %__a, <4 x i64
 ; NoVLX-NEXT:    vpmovsxwd %ymm0, %zmm0
 ; NoVLX-NEXT:    vptestmd %zmm0, %zmm0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -11025,8 +10859,7 @@ define zeroext i8 @test_vpcmpsged_v4i1_v8i1_mask_mem_b(<2 x i64> %__a, i32* %__b
 ; NoVLX-LABEL: test_vpcmpsged_v4i1_v8i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpnltd %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpnltd (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -11056,9 +10889,8 @@ define zeroext i8 @test_masked_vpcmpsged_v4i1_v8i1_mask_mem_b(i8 zeroext %__u, <
 ; NoVLX-LABEL: test_masked_vpcmpsged_v4i1_v8i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpnltd %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpnltd (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -11216,8 +11048,7 @@ define zeroext i16 @test_vpcmpsged_v4i1_v16i1_mask_mem_b(<2 x i64> %__a, i32* %_
 ; NoVLX-LABEL: test_vpcmpsged_v4i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpnltd %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpnltd (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -11247,9 +11078,8 @@ define zeroext i16 @test_masked_vpcmpsged_v4i1_v16i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpsged_v4i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpnltd %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpnltd (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -11398,8 +11228,7 @@ define zeroext i32 @test_vpcmpsged_v4i1_v32i1_mask_mem_b(<2 x i64> %__a, i32* %_
 ; NoVLX-LABEL: test_vpcmpsged_v4i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpnltd %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpnltd (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -11427,9 +11256,8 @@ define zeroext i32 @test_masked_vpcmpsged_v4i1_v32i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpsged_v4i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpnltd %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpnltd (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -11465,7 +11293,6 @@ define zeroext i64 @test_vpcmpsged_v4i1_v64i1_mask(<2 x i64> %__a, <2 x i64> %__
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -11492,7 +11319,6 @@ define zeroext i64 @test_vpcmpsged_v4i1_v64i1_mask_mem(<2 x i64> %__a, <2 x i64>
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -11522,7 +11348,6 @@ define zeroext i64 @test_masked_vpcmpsged_v4i1_v64i1_mask(i8 zeroext %__u, <2 x
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -11554,7 +11379,6 @@ define zeroext i64 @test_masked_vpcmpsged_v4i1_v64i1_mask_mem(i8 zeroext %__u, <
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -11581,12 +11405,10 @@ define zeroext i64 @test_vpcmpsged_v4i1_v64i1_mask_mem_b(<2 x i64> %__a, i32* %_
 ; NoVLX-LABEL: test_vpcmpsged_v4i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpnltd %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpnltd (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -11611,13 +11433,11 @@ define zeroext i64 @test_masked_vpcmpsged_v4i1_v64i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpsged_v4i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpnltd %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpnltd (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -11774,8 +11594,7 @@ define zeroext i16 @test_vpcmpsged_v8i1_v16i1_mask_mem_b(<4 x i64> %__a, i32* %_
 ; NoVLX-LABEL: test_vpcmpsged_v8i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rdi), %ymm1
-; NoVLX-NEXT:    vpcmpnltd %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpnltd (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -11806,9 +11625,8 @@ define zeroext i16 @test_masked_vpcmpsged_v8i1_v16i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpsged_v8i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rsi), %ymm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpnltd %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpnltd (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -11959,8 +11777,7 @@ define zeroext i32 @test_vpcmpsged_v8i1_v32i1_mask_mem_b(<4 x i64> %__a, i32* %_
 ; NoVLX-LABEL: test_vpcmpsged_v8i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rdi), %ymm1
-; NoVLX-NEXT:    vpcmpnltd %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpnltd (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -11989,9 +11806,8 @@ define zeroext i32 @test_masked_vpcmpsged_v8i1_v32i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpsged_v8i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rsi), %ymm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpnltd %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpnltd (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -12027,7 +11843,6 @@ define zeroext i64 @test_vpcmpsged_v8i1_v64i1_mask(<4 x i64> %__a, <4 x i64> %__
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -12055,7 +11870,6 @@ define zeroext i64 @test_vpcmpsged_v8i1_v64i1_mask_mem(<4 x i64> %__a, <4 x i64>
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -12086,7 +11900,6 @@ define zeroext i64 @test_masked_vpcmpsged_v8i1_v64i1_mask(i8 zeroext %__u, <4 x
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -12118,7 +11931,6 @@ define zeroext i64 @test_masked_vpcmpsged_v8i1_v64i1_mask_mem(i8 zeroext %__u, <
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -12145,12 +11957,10 @@ define zeroext i64 @test_vpcmpsged_v8i1_v64i1_mask_mem_b(<4 x i64> %__a, i32* %_
 ; NoVLX-LABEL: test_vpcmpsged_v8i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rdi), %ymm1
-; NoVLX-NEXT:    vpcmpnltd %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpnltd (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -12176,13 +11986,11 @@ define zeroext i64 @test_masked_vpcmpsged_v8i1_v64i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpsged_v8i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rsi), %ymm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpnltd %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpnltd (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -12369,7 +12177,6 @@ define zeroext i64 @test_vpcmpsged_v16i1_v64i1_mask(<8 x i64> %__a, <8 x i64> %_
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    vpcmpnltd %zmm1, %zmm0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -12393,7 +12200,6 @@ define zeroext i64 @test_vpcmpsged_v16i1_v64i1_mask_mem(<8 x i64> %__a, <8 x i64
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    vpcmpnltd (%rdi), %zmm0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -12474,7 +12280,6 @@ define zeroext i64 @test_vpcmpsged_v16i1_v64i1_mask_mem_b(<8 x i64> %__a, i32* %
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    vpcmpnltd (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -12533,7 +12338,6 @@ define zeroext i4 @test_vpcmpsgeq_v2i1_v4i1_mask(<2 x i64> %__a, <2 x i64> %__b)
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    andl $3, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -12560,7 +12364,6 @@ define zeroext i4 @test_vpcmpsgeq_v2i1_v4i1_mask_mem(<2 x i64> %__a, <2 x i64>*
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    andl $3, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -12590,7 +12393,6 @@ define zeroext i4 @test_masked_vpcmpsgeq_v2i1_v4i1_mask(i8 zeroext %__u, <2 x i6
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    andl $3, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -12622,7 +12424,6 @@ define zeroext i4 @test_masked_vpcmpsgeq_v2i1_v4i1_mask_mem(i8 zeroext %__u, <2
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    andl $3, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -12649,12 +12450,10 @@ define zeroext i4 @test_vpcmpsgeq_v2i1_v4i1_mask_mem_b(<2 x i64> %__a, i64* %__b
 ; NoVLX-LABEL: test_vpcmpsgeq_v2i1_v4i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpnltq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpnltq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    andl $3, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -12679,13 +12478,11 @@ define zeroext i4 @test_masked_vpcmpsgeq_v2i1_v4i1_mask_mem_b(i8 zeroext %__u, <
 ; NoVLX-LABEL: test_masked_vpcmpsgeq_v2i1_v4i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpnltq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpnltq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    andl $3, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -12839,8 +12636,7 @@ define zeroext i8 @test_vpcmpsgeq_v2i1_v8i1_mask_mem_b(<2 x i64> %__a, i64* %__b
 ; NoVLX-LABEL: test_vpcmpsgeq_v2i1_v8i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpnltq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpnltq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -12870,9 +12666,8 @@ define zeroext i8 @test_masked_vpcmpsgeq_v2i1_v8i1_mask_mem_b(i8 zeroext %__u, <
 ; NoVLX-LABEL: test_masked_vpcmpsgeq_v2i1_v8i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpnltq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpnltq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -13030,8 +12825,7 @@ define zeroext i16 @test_vpcmpsgeq_v2i1_v16i1_mask_mem_b(<2 x i64> %__a, i64* %_
 ; NoVLX-LABEL: test_vpcmpsgeq_v2i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpnltq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpnltq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -13061,9 +12855,8 @@ define zeroext i16 @test_masked_vpcmpsgeq_v2i1_v16i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpsgeq_v2i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpnltq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpnltq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -13212,8 +13005,7 @@ define zeroext i32 @test_vpcmpsgeq_v2i1_v32i1_mask_mem_b(<2 x i64> %__a, i64* %_
 ; NoVLX-LABEL: test_vpcmpsgeq_v2i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpnltq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpnltq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -13241,9 +13033,8 @@ define zeroext i32 @test_masked_vpcmpsgeq_v2i1_v32i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpsgeq_v2i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpnltq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpnltq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -13279,7 +13070,6 @@ define zeroext i64 @test_vpcmpsgeq_v2i1_v64i1_mask(<2 x i64> %__a, <2 x i64> %__
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -13306,7 +13096,6 @@ define zeroext i64 @test_vpcmpsgeq_v2i1_v64i1_mask_mem(<2 x i64> %__a, <2 x i64>
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -13336,7 +13125,6 @@ define zeroext i64 @test_masked_vpcmpsgeq_v2i1_v64i1_mask(i8 zeroext %__u, <2 x
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -13368,7 +13156,6 @@ define zeroext i64 @test_masked_vpcmpsgeq_v2i1_v64i1_mask_mem(i8 zeroext %__u, <
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -13395,12 +13182,10 @@ define zeroext i64 @test_vpcmpsgeq_v2i1_v64i1_mask_mem_b(<2 x i64> %__a, i64* %_
 ; NoVLX-LABEL: test_vpcmpsgeq_v2i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpnltq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpnltq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -13425,13 +13210,11 @@ define zeroext i64 @test_masked_vpcmpsgeq_v2i1_v64i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpsgeq_v2i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpnltq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpnltq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -13590,8 +13373,7 @@ define zeroext i8 @test_vpcmpsgeq_v4i1_v8i1_mask_mem_b(<4 x i64> %__a, i64* %__b
 ; NoVLX-LABEL: test_vpcmpsgeq_v4i1_v8i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %ymm1
-; NoVLX-NEXT:    vpcmpnltq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpnltq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -13622,9 +13404,8 @@ define zeroext i8 @test_masked_vpcmpsgeq_v4i1_v8i1_mask_mem_b(i8 zeroext %__u, <
 ; NoVLX-LABEL: test_masked_vpcmpsgeq_v4i1_v8i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %ymm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpnltq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpnltq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -13787,8 +13568,7 @@ define zeroext i16 @test_vpcmpsgeq_v4i1_v16i1_mask_mem_b(<4 x i64> %__a, i64* %_
 ; NoVLX-LABEL: test_vpcmpsgeq_v4i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %ymm1
-; NoVLX-NEXT:    vpcmpnltq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpnltq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -13819,9 +13599,8 @@ define zeroext i16 @test_masked_vpcmpsgeq_v4i1_v16i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpsgeq_v4i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %ymm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpnltq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpnltq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -13975,8 +13754,7 @@ define zeroext i32 @test_vpcmpsgeq_v4i1_v32i1_mask_mem_b(<4 x i64> %__a, i64* %_
 ; NoVLX-LABEL: test_vpcmpsgeq_v4i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %ymm1
-; NoVLX-NEXT:    vpcmpnltq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpnltq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -14005,9 +13783,8 @@ define zeroext i32 @test_masked_vpcmpsgeq_v4i1_v32i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpsgeq_v4i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %ymm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpnltq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpnltq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -14044,7 +13821,6 @@ define zeroext i64 @test_vpcmpsgeq_v4i1_v64i1_mask(<4 x i64> %__a, <4 x i64> %__
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -14072,7 +13848,6 @@ define zeroext i64 @test_vpcmpsgeq_v4i1_v64i1_mask_mem(<4 x i64> %__a, <4 x i64>
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -14103,7 +13878,6 @@ define zeroext i64 @test_masked_vpcmpsgeq_v4i1_v64i1_mask(i8 zeroext %__u, <4 x
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -14136,7 +13910,6 @@ define zeroext i64 @test_masked_vpcmpsgeq_v4i1_v64i1_mask_mem(i8 zeroext %__u, <
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -14164,12 +13937,10 @@ define zeroext i64 @test_vpcmpsgeq_v4i1_v64i1_mask_mem_b(<4 x i64> %__a, i64* %_
 ; NoVLX-LABEL: test_vpcmpsgeq_v4i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %ymm1
-; NoVLX-NEXT:    vpcmpnltq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpnltq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -14195,13 +13966,11 @@ define zeroext i64 @test_masked_vpcmpsgeq_v4i1_v64i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpsgeq_v4i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %ymm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpnltq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpnltq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -14559,7 +14328,6 @@ define zeroext i64 @test_vpcmpsgeq_v8i1_v64i1_mask(<8 x i64> %__a, <8 x i64> %__
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    vpcmpnltq %zmm1, %zmm0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -14583,7 +14351,6 @@ define zeroext i64 @test_vpcmpsgeq_v8i1_v64i1_mask_mem(<8 x i64> %__a, <8 x i64>
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    vpcmpnltq (%rdi), %zmm0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -14610,7 +14377,6 @@ define zeroext i64 @test_masked_vpcmpsgeq_v8i1_v64i1_mask(i8 zeroext %__u, <8 x
 ; NoVLX-NEXT:    kmovw %edi, %k1
 ; NoVLX-NEXT:    vpcmpnltq %zmm1, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -14638,7 +14404,6 @@ define zeroext i64 @test_masked_vpcmpsgeq_v8i1_v64i1_mask_mem(i8 zeroext %__u, <
 ; NoVLX-NEXT:    kmovw %edi, %k1
 ; NoVLX-NEXT:    vpcmpnltq (%rsi), %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -14666,7 +14431,6 @@ define zeroext i64 @test_vpcmpsgeq_v8i1_v64i1_mask_mem_b(<8 x i64> %__a, i64* %_
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    vpcmpnltq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -14694,7 +14458,6 @@ define zeroext i64 @test_masked_vpcmpsgeq_v8i1_v64i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-NEXT:    kmovw %edi, %k1
 ; NoVLX-NEXT:    vpcmpnltq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -14841,7 +14604,6 @@ define zeroext i64 @test_vpcmpultb_v16i1_v64i1_mask(<2 x i64> %__a, <2 x i64> %_
 ; NoVLX-NEXT:    vpmovsxbd %xmm0, %zmm0
 ; NoVLX-NEXT:    vptestmd %zmm0, %zmm0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -14868,7 +14630,6 @@ define zeroext i64 @test_vpcmpultb_v16i1_v64i1_mask_mem(<2 x i64> %__a, <2 x i64
 ; NoVLX-NEXT:    vpmovsxbd %xmm0, %zmm0
 ; NoVLX-NEXT:    vptestmd %zmm0, %zmm0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -15345,7 +15106,6 @@ define zeroext i64 @test_vpcmpultw_v8i1_v64i1_mask(<2 x i64> %__a, <2 x i64> %__
 ; NoVLX-NEXT:    vpmovsxwq %xmm0, %zmm0
 ; NoVLX-NEXT:    vptestmq %zmm0, %zmm0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -15372,7 +15132,6 @@ define zeroext i64 @test_vpcmpultw_v8i1_v64i1_mask_mem(<2 x i64> %__a, <2 x i64>
 ; NoVLX-NEXT:    vpmovsxwq %xmm0, %zmm0
 ; NoVLX-NEXT:    vptestmq %zmm0, %zmm0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -15402,7 +15161,6 @@ define zeroext i64 @test_masked_vpcmpultw_v8i1_v64i1_mask(i8 zeroext %__u, <2 x
 ; NoVLX-NEXT:    kmovw %edi, %k1
 ; NoVLX-NEXT:    vptestmq %zmm0, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -15433,7 +15191,6 @@ define zeroext i64 @test_masked_vpcmpultw_v8i1_v64i1_mask_mem(i8 zeroext %__u, <
 ; NoVLX-NEXT:    kmovw %edi, %k1
 ; NoVLX-NEXT:    vptestmq %zmm0, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -15584,7 +15341,6 @@ define zeroext i64 @test_vpcmpultw_v16i1_v64i1_mask(<4 x i64> %__a, <4 x i64> %_
 ; NoVLX-NEXT:    vpmovsxwd %ymm0, %zmm0
 ; NoVLX-NEXT:    vptestmd %zmm0, %zmm0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -15612,7 +15368,6 @@ define zeroext i64 @test_vpcmpultw_v16i1_v64i1_mask_mem(<4 x i64> %__a, <4 x i64
 ; NoVLX-NEXT:    vpmovsxwd %ymm0, %zmm0
 ; NoVLX-NEXT:    vptestmd %zmm0, %zmm0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -15988,8 +15743,7 @@ define zeroext i8 @test_vpcmpultd_v4i1_v8i1_mask_mem_b(<2 x i64> %__a, i32* %__b
 ; NoVLX-LABEL: test_vpcmpultd_v4i1_v8i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpltud %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpltud (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -16019,9 +15773,8 @@ define zeroext i8 @test_masked_vpcmpultd_v4i1_v8i1_mask_mem_b(i8 zeroext %__u, <
 ; NoVLX-LABEL: test_masked_vpcmpultd_v4i1_v8i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpltud %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpltud (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -16179,8 +15932,7 @@ define zeroext i16 @test_vpcmpultd_v4i1_v16i1_mask_mem_b(<2 x i64> %__a, i32* %_
 ; NoVLX-LABEL: test_vpcmpultd_v4i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpltud %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpltud (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -16210,9 +15962,8 @@ define zeroext i16 @test_masked_vpcmpultd_v4i1_v16i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpultd_v4i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpltud %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpltud (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -16361,8 +16112,7 @@ define zeroext i32 @test_vpcmpultd_v4i1_v32i1_mask_mem_b(<2 x i64> %__a, i32* %_
 ; NoVLX-LABEL: test_vpcmpultd_v4i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpltud %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpltud (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -16390,9 +16140,8 @@ define zeroext i32 @test_masked_vpcmpultd_v4i1_v32i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpultd_v4i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpltud %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpltud (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -16428,7 +16177,6 @@ define zeroext i64 @test_vpcmpultd_v4i1_v64i1_mask(<2 x i64> %__a, <2 x i64> %__
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -16455,7 +16203,6 @@ define zeroext i64 @test_vpcmpultd_v4i1_v64i1_mask_mem(<2 x i64> %__a, <2 x i64>
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -16485,7 +16232,6 @@ define zeroext i64 @test_masked_vpcmpultd_v4i1_v64i1_mask(i8 zeroext %__u, <2 x
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -16517,7 +16263,6 @@ define zeroext i64 @test_masked_vpcmpultd_v4i1_v64i1_mask_mem(i8 zeroext %__u, <
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -16544,12 +16289,10 @@ define zeroext i64 @test_vpcmpultd_v4i1_v64i1_mask_mem_b(<2 x i64> %__a, i32* %_
 ; NoVLX-LABEL: test_vpcmpultd_v4i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpltud %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpltud (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -16574,13 +16317,11 @@ define zeroext i64 @test_masked_vpcmpultd_v4i1_v64i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpultd_v4i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpltud %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpltud (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -16737,8 +16478,7 @@ define zeroext i16 @test_vpcmpultd_v8i1_v16i1_mask_mem_b(<4 x i64> %__a, i32* %_
 ; NoVLX-LABEL: test_vpcmpultd_v8i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rdi), %ymm1
-; NoVLX-NEXT:    vpcmpltud %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpltud (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -16769,9 +16509,8 @@ define zeroext i16 @test_masked_vpcmpultd_v8i1_v16i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpultd_v8i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rsi), %ymm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpltud %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpltud (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -16922,8 +16661,7 @@ define zeroext i32 @test_vpcmpultd_v8i1_v32i1_mask_mem_b(<4 x i64> %__a, i32* %_
 ; NoVLX-LABEL: test_vpcmpultd_v8i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rdi), %ymm1
-; NoVLX-NEXT:    vpcmpltud %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpltud (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -16952,9 +16690,8 @@ define zeroext i32 @test_masked_vpcmpultd_v8i1_v32i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpultd_v8i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rsi), %ymm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpltud %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpltud (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -16990,7 +16727,6 @@ define zeroext i64 @test_vpcmpultd_v8i1_v64i1_mask(<4 x i64> %__a, <4 x i64> %__
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -17018,7 +16754,6 @@ define zeroext i64 @test_vpcmpultd_v8i1_v64i1_mask_mem(<4 x i64> %__a, <4 x i64>
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -17049,7 +16784,6 @@ define zeroext i64 @test_masked_vpcmpultd_v8i1_v64i1_mask(i8 zeroext %__u, <4 x
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -17081,7 +16815,6 @@ define zeroext i64 @test_masked_vpcmpultd_v8i1_v64i1_mask_mem(i8 zeroext %__u, <
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -17108,12 +16841,10 @@ define zeroext i64 @test_vpcmpultd_v8i1_v64i1_mask_mem_b(<4 x i64> %__a, i32* %_
 ; NoVLX-LABEL: test_vpcmpultd_v8i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rdi), %ymm1
-; NoVLX-NEXT:    vpcmpltud %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpltud (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -17139,13 +16870,11 @@ define zeroext i64 @test_masked_vpcmpultd_v8i1_v64i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpultd_v8i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rsi), %ymm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpltud %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpltud (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -17332,7 +17061,6 @@ define zeroext i64 @test_vpcmpultd_v16i1_v64i1_mask(<8 x i64> %__a, <8 x i64> %_
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    vpcmpltud %zmm1, %zmm0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -17356,7 +17084,6 @@ define zeroext i64 @test_vpcmpultd_v16i1_v64i1_mask_mem(<8 x i64> %__a, <8 x i64
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    vpcmpltud (%rdi), %zmm0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -17437,7 +17164,6 @@ define zeroext i64 @test_vpcmpultd_v16i1_v64i1_mask_mem_b(<8 x i64> %__a, i32* %
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    vpcmpltud (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -17496,7 +17222,6 @@ define zeroext i4 @test_vpcmpultq_v2i1_v4i1_mask(<2 x i64> %__a, <2 x i64> %__b)
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    andl $3, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -17523,7 +17248,6 @@ define zeroext i4 @test_vpcmpultq_v2i1_v4i1_mask_mem(<2 x i64> %__a, <2 x i64>*
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    andl $3, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -17553,7 +17277,6 @@ define zeroext i4 @test_masked_vpcmpultq_v2i1_v4i1_mask(i8 zeroext %__u, <2 x i6
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    andl $3, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -17585,7 +17308,6 @@ define zeroext i4 @test_masked_vpcmpultq_v2i1_v4i1_mask_mem(i8 zeroext %__u, <2
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    andl $3, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -17612,12 +17334,10 @@ define zeroext i4 @test_vpcmpultq_v2i1_v4i1_mask_mem_b(<2 x i64> %__a, i64* %__b
 ; NoVLX-LABEL: test_vpcmpultq_v2i1_v4i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpltuq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpltuq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    andl $3, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -17642,13 +17362,11 @@ define zeroext i4 @test_masked_vpcmpultq_v2i1_v4i1_mask_mem_b(i8 zeroext %__u, <
 ; NoVLX-LABEL: test_masked_vpcmpultq_v2i1_v4i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpltuq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpltuq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    andl $3, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -17802,8 +17520,7 @@ define zeroext i8 @test_vpcmpultq_v2i1_v8i1_mask_mem_b(<2 x i64> %__a, i64* %__b
 ; NoVLX-LABEL: test_vpcmpultq_v2i1_v8i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpltuq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpltuq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -17833,9 +17550,8 @@ define zeroext i8 @test_masked_vpcmpultq_v2i1_v8i1_mask_mem_b(i8 zeroext %__u, <
 ; NoVLX-LABEL: test_masked_vpcmpultq_v2i1_v8i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpltuq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpltuq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -17993,8 +17709,7 @@ define zeroext i16 @test_vpcmpultq_v2i1_v16i1_mask_mem_b(<2 x i64> %__a, i64* %_
 ; NoVLX-LABEL: test_vpcmpultq_v2i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpltuq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpltuq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -18024,9 +17739,8 @@ define zeroext i16 @test_masked_vpcmpultq_v2i1_v16i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpultq_v2i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpltuq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpltuq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -18175,8 +17889,7 @@ define zeroext i32 @test_vpcmpultq_v2i1_v32i1_mask_mem_b(<2 x i64> %__a, i64* %_
 ; NoVLX-LABEL: test_vpcmpultq_v2i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpltuq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpltuq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -18204,9 +17917,8 @@ define zeroext i32 @test_masked_vpcmpultq_v2i1_v32i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpultq_v2i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpltuq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpltuq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -18242,7 +17954,6 @@ define zeroext i64 @test_vpcmpultq_v2i1_v64i1_mask(<2 x i64> %__a, <2 x i64> %__
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -18269,7 +17980,6 @@ define zeroext i64 @test_vpcmpultq_v2i1_v64i1_mask_mem(<2 x i64> %__a, <2 x i64>
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -18299,7 +18009,6 @@ define zeroext i64 @test_masked_vpcmpultq_v2i1_v64i1_mask(i8 zeroext %__u, <2 x
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -18331,7 +18040,6 @@ define zeroext i64 @test_masked_vpcmpultq_v2i1_v64i1_mask_mem(i8 zeroext %__u, <
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -18358,12 +18066,10 @@ define zeroext i64 @test_vpcmpultq_v2i1_v64i1_mask_mem_b(<2 x i64> %__a, i64* %_
 ; NoVLX-LABEL: test_vpcmpultq_v2i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpltuq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpltuq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -18388,13 +18094,11 @@ define zeroext i64 @test_masked_vpcmpultq_v2i1_v64i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpultq_v2i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpltuq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpltuq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -18553,8 +18257,7 @@ define zeroext i8 @test_vpcmpultq_v4i1_v8i1_mask_mem_b(<4 x i64> %__a, i64* %__b
 ; NoVLX-LABEL: test_vpcmpultq_v4i1_v8i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %ymm1
-; NoVLX-NEXT:    vpcmpltuq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpltuq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -18585,9 +18288,8 @@ define zeroext i8 @test_masked_vpcmpultq_v4i1_v8i1_mask_mem_b(i8 zeroext %__u, <
 ; NoVLX-LABEL: test_masked_vpcmpultq_v4i1_v8i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %ymm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpltuq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpltuq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -18750,8 +18452,7 @@ define zeroext i16 @test_vpcmpultq_v4i1_v16i1_mask_mem_b(<4 x i64> %__a, i64* %_
 ; NoVLX-LABEL: test_vpcmpultq_v4i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %ymm1
-; NoVLX-NEXT:    vpcmpltuq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpltuq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -18782,9 +18483,8 @@ define zeroext i16 @test_masked_vpcmpultq_v4i1_v16i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpultq_v4i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %ymm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpltuq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpltuq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -18938,8 +18638,7 @@ define zeroext i32 @test_vpcmpultq_v4i1_v32i1_mask_mem_b(<4 x i64> %__a, i64* %_
 ; NoVLX-LABEL: test_vpcmpultq_v4i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %ymm1
-; NoVLX-NEXT:    vpcmpltuq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpltuq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -18968,9 +18667,8 @@ define zeroext i32 @test_masked_vpcmpultq_v4i1_v32i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpultq_v4i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %ymm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpltuq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpltuq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -19007,7 +18705,6 @@ define zeroext i64 @test_vpcmpultq_v4i1_v64i1_mask(<4 x i64> %__a, <4 x i64> %__
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -19035,7 +18732,6 @@ define zeroext i64 @test_vpcmpultq_v4i1_v64i1_mask_mem(<4 x i64> %__a, <4 x i64>
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -19066,7 +18762,6 @@ define zeroext i64 @test_masked_vpcmpultq_v4i1_v64i1_mask(i8 zeroext %__u, <4 x
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -19099,7 +18794,6 @@ define zeroext i64 @test_masked_vpcmpultq_v4i1_v64i1_mask_mem(i8 zeroext %__u, <
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -19127,12 +18821,10 @@ define zeroext i64 @test_vpcmpultq_v4i1_v64i1_mask_mem_b(<4 x i64> %__a, i64* %_
 ; NoVLX-LABEL: test_vpcmpultq_v4i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %ymm1
-; NoVLX-NEXT:    vpcmpltuq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpltuq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -19158,13 +18850,11 @@ define zeroext i64 @test_masked_vpcmpultq_v4i1_v64i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpultq_v4i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %ymm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpltuq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpltuq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -19522,7 +19212,6 @@ define zeroext i64 @test_vpcmpultq_v8i1_v64i1_mask(<8 x i64> %__a, <8 x i64> %__
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    vpcmpltuq %zmm1, %zmm0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -19546,7 +19235,6 @@ define zeroext i64 @test_vpcmpultq_v8i1_v64i1_mask_mem(<8 x i64> %__a, <8 x i64>
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    vpcmpltuq (%rdi), %zmm0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -19573,7 +19261,6 @@ define zeroext i64 @test_masked_vpcmpultq_v8i1_v64i1_mask(i8 zeroext %__u, <8 x
 ; NoVLX-NEXT:    kmovw %edi, %k1
 ; NoVLX-NEXT:    vpcmpltuq %zmm1, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -19601,7 +19288,6 @@ define zeroext i64 @test_masked_vpcmpultq_v8i1_v64i1_mask_mem(i8 zeroext %__u, <
 ; NoVLX-NEXT:    kmovw %edi, %k1
 ; NoVLX-NEXT:    vpcmpltuq (%rsi), %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -19629,7 +19315,6 @@ define zeroext i64 @test_vpcmpultq_v8i1_v64i1_mask_mem_b(<8 x i64> %__a, i64* %_
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    vpcmpltuq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -19657,7 +19342,6 @@ define zeroext i64 @test_masked_vpcmpultq_v8i1_v64i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-NEXT:    kmovw %edi, %k1
 ; NoVLX-NEXT:    vpcmpltuq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -19743,8 +19427,7 @@ define zeroext i8 @test_vcmpoeqps_v4i1_v8i1_mask_mem_b(<2 x i64> %__a, float* %_
 ; NoVLX-LABEL: test_vcmpoeqps_v4i1_v8i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vbroadcastss (%rdi), %xmm1
-; NoVLX-NEXT:    vcmpeqps %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vcmpeqps (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -19840,8 +19523,7 @@ define zeroext i8 @test_masked_vcmpoeqps_v4i1_v8i1_mask_mem_b(i4 zeroext %__u, <
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vbroadcastss (%rsi), %xmm1
-; NoVLX-NEXT:    vcmpeqps %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vcmpeqps (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -19931,8 +19613,7 @@ define zeroext i16 @test_vcmpoeqps_v4i1_v16i1_mask_mem_b(<2 x i64> %__a, float*
 ; NoVLX-LABEL: test_vcmpoeqps_v4i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vbroadcastss (%rdi), %xmm1
-; NoVLX-NEXT:    vcmpeqps %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vcmpeqps (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -20028,8 +19709,7 @@ define zeroext i16 @test_masked_vcmpoeqps_v4i1_v16i1_mask_mem_b(i4 zeroext %__u,
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vbroadcastss (%rsi), %xmm1
-; NoVLX-NEXT:    vcmpeqps %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vcmpeqps (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -20114,8 +19794,7 @@ define zeroext i32 @test_vcmpoeqps_v4i1_v32i1_mask_mem_b(<2 x i64> %__a, float*
 ; NoVLX-LABEL: test_vcmpoeqps_v4i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vbroadcastss (%rdi), %xmm1
-; NoVLX-NEXT:    vcmpeqps %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vcmpeqps (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -20205,8 +19884,7 @@ define zeroext i32 @test_masked_vcmpoeqps_v4i1_v32i1_mask_mem_b(i4 zeroext %__u,
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vbroadcastss (%rsi), %xmm1
-; NoVLX-NEXT:    vcmpeqps %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vcmpeqps (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -20242,7 +19920,6 @@ define zeroext i64 @test_vcmpoeqps_v4i1_v64i1_mask(<2 x i64> %__a, <2 x i64> %__
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -20269,7 +19946,6 @@ define zeroext i64 @test_vcmpoeqps_v4i1_v64i1_mask_mem(<2 x i64> %__a, <2 x i64>
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -20292,12 +19968,10 @@ define zeroext i64 @test_vcmpoeqps_v4i1_v64i1_mask_mem_b(<2 x i64> %__a, float*
 ; NoVLX-LABEL: test_vcmpoeqps_v4i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vbroadcastss (%rdi), %xmm1
-; NoVLX-NEXT:    vcmpeqps %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vcmpeqps (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -20328,7 +20002,6 @@ define zeroext i64 @test_masked_vcmpoeqps_v4i1_v64i1_mask(i4 zeroext %__u, <2 x
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -20359,7 +20032,6 @@ define zeroext i64 @test_masked_vcmpoeqps_v4i1_v64i1_mask_mem(i4 zeroext %__u, <
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -20386,12 +20058,10 @@ define zeroext i64 @test_masked_vcmpoeqps_v4i1_v64i1_mask_mem_b(i4 zeroext %__u,
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vbroadcastss (%rsi), %xmm1
-; NoVLX-NEXT:    vcmpeqps %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vcmpeqps (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -20480,8 +20150,7 @@ define zeroext i16 @test_vcmpoeqps_v8i1_v16i1_mask_mem_b(<4 x i64> %__a, float*
 ; NoVLX-LABEL: test_vcmpoeqps_v8i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vbroadcastss (%rdi), %ymm1
-; NoVLX-NEXT:    vcmpeqps %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vcmpeqps (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -20579,9 +20248,8 @@ define zeroext i16 @test_masked_vcmpoeqps_v8i1_v16i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vcmpoeqps_v8i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vbroadcastss (%rsi), %ymm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vcmpeqps %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vcmpeqps (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -20669,8 +20337,7 @@ define zeroext i32 @test_vcmpoeqps_v8i1_v32i1_mask_mem_b(<4 x i64> %__a, float*
 ; NoVLX-LABEL: test_vcmpoeqps_v8i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vbroadcastss (%rdi), %ymm1
-; NoVLX-NEXT:    vcmpeqps %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vcmpeqps (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -20762,9 +20429,8 @@ define zeroext i32 @test_masked_vcmpoeqps_v8i1_v32i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vcmpoeqps_v8i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vbroadcastss (%rsi), %ymm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vcmpeqps %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vcmpeqps (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -20801,7 +20467,6 @@ define zeroext i64 @test_vcmpoeqps_v8i1_v64i1_mask(<4 x i64> %__a, <4 x i64> %__
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -20829,7 +20494,6 @@ define zeroext i64 @test_vcmpoeqps_v8i1_v64i1_mask_mem(<4 x i64> %__a, <4 x i64>
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -20853,12 +20517,10 @@ define zeroext i64 @test_vcmpoeqps_v8i1_v64i1_mask_mem_b(<4 x i64> %__a, float*
 ; NoVLX-LABEL: test_vcmpoeqps_v8i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vbroadcastss (%rdi), %ymm1
-; NoVLX-NEXT:    vcmpeqps %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vcmpeqps (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -20890,7 +20552,6 @@ define zeroext i64 @test_masked_vcmpoeqps_v8i1_v64i1_mask(i8 zeroext %__u, <4 x
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -20922,7 +20583,6 @@ define zeroext i64 @test_masked_vcmpoeqps_v8i1_v64i1_mask_mem(i8 zeroext %__u, <
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -20949,13 +20609,11 @@ define zeroext i64 @test_masked_vcmpoeqps_v8i1_v64i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vcmpoeqps_v8i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vbroadcastss (%rsi), %ymm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vcmpeqps %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vcmpeqps (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -21188,7 +20846,6 @@ define zeroext i64 @test_vcmpoeqps_v16i1_v64i1_mask(<8 x i64> %__a, <8 x i64> %_
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    vcmpeqps %zmm1, %zmm0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -21212,7 +20869,6 @@ define zeroext i64 @test_vcmpoeqps_v16i1_v64i1_mask_mem(<8 x i64> %__a, <8 x i64
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    vcmpeqps (%rdi), %zmm0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -21237,7 +20893,6 @@ define zeroext i64 @test_vcmpoeqps_v16i1_v64i1_mask_mem_b(<8 x i64> %__a, float*
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    vcmpeqps (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -21338,21 +20993,12 @@ entry:
 
 
 define zeroext i64 @test_vcmpoeqps_v16i1_v64i1_sae_mask(<8 x i64> %__a, <8 x i64> %__b) local_unnamed_addr {
-; VLX-LABEL: test_vcmpoeqps_v16i1_v64i1_sae_mask:
-; VLX:       # %bb.0: # %entry
-; VLX-NEXT:    vcmpleps {sae}, %zmm1, %zmm0, %k0
-; VLX-NEXT:    kmovd %k0, %eax
-; VLX-NEXT:    movzwl %ax, %eax
-; VLX-NEXT:    vzeroupper
-; VLX-NEXT:    retq
-;
-; NoVLX-LABEL: test_vcmpoeqps_v16i1_v64i1_sae_mask:
-; NoVLX:       # %bb.0: # %entry
-; NoVLX-NEXT:    vcmpleps {sae}, %zmm1, %zmm0, %k0
-; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
-; NoVLX-NEXT:    vzeroupper
-; NoVLX-NEXT:    retq
+; CHECK-LABEL: test_vcmpoeqps_v16i1_v64i1_sae_mask:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vcmpleps {sae}, %zmm1, %zmm0, %k0
+; CHECK-NEXT:    kmovw %k0, %eax
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
 entry:
   %0 = bitcast <8 x i64> %__a to <16 x float>
   %1 = bitcast <8 x i64> %__b to <16 x float>
@@ -21407,7 +21053,6 @@ define zeroext i4 @test_vcmpoeqpd_v2i1_v4i1_mask(<2 x i64> %__a, <2 x i64> %__b)
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    andl $3, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -21434,7 +21079,6 @@ define zeroext i4 @test_vcmpoeqpd_v2i1_v4i1_mask_mem(<2 x i64> %__a, <2 x i64>*
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    andl $3, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -21457,12 +21101,10 @@ define zeroext i4 @test_vcmpoeqpd_v2i1_v4i1_mask_mem_b(<2 x i64> %__a, double* %
 ; NoVLX-LABEL: test_vcmpoeqpd_v2i1_v4i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vmovddup {{.*#+}} xmm1 = mem[0,0]
-; NoVLX-NEXT:    vcmpeqpd %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vcmpeqpd (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    andl $3, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -21493,7 +21135,6 @@ define zeroext i4 @test_masked_vcmpoeqpd_v2i1_v4i1_mask(i2 zeroext %__u, <2 x i6
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    andl $3, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -21524,7 +21165,6 @@ define zeroext i4 @test_masked_vcmpoeqpd_v2i1_v4i1_mask_mem(i2 zeroext %__u, <2
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    andl $3, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -21551,12 +21191,10 @@ define zeroext i4 @test_masked_vcmpoeqpd_v2i1_v4i1_mask_mem_b(i2 zeroext %__u, <
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vmovddup {{.*#+}} xmm1 = mem[0,0]
-; NoVLX-NEXT:    vcmpeqpd %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vcmpeqpd (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    andl $3, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -21642,8 +21280,7 @@ define zeroext i8 @test_vcmpoeqpd_v2i1_v8i1_mask_mem_b(<2 x i64> %__a, double* %
 ; NoVLX-LABEL: test_vcmpoeqpd_v2i1_v8i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vmovddup {{.*#+}} xmm1 = mem[0,0]
-; NoVLX-NEXT:    vcmpeqpd %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vcmpeqpd (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -21739,8 +21376,7 @@ define zeroext i8 @test_masked_vcmpoeqpd_v2i1_v8i1_mask_mem_b(i2 zeroext %__u, <
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vmovddup {{.*#+}} xmm1 = mem[0,0]
-; NoVLX-NEXT:    vcmpeqpd %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vcmpeqpd (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -21830,8 +21466,7 @@ define zeroext i16 @test_vcmpoeqpd_v2i1_v16i1_mask_mem_b(<2 x i64> %__a, double*
 ; NoVLX-LABEL: test_vcmpoeqpd_v2i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vmovddup {{.*#+}} xmm1 = mem[0,0]
-; NoVLX-NEXT:    vcmpeqpd %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vcmpeqpd (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -21927,8 +21562,7 @@ define zeroext i16 @test_masked_vcmpoeqpd_v2i1_v16i1_mask_mem_b(i2 zeroext %__u,
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vmovddup {{.*#+}} xmm1 = mem[0,0]
-; NoVLX-NEXT:    vcmpeqpd %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vcmpeqpd (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -22013,8 +21647,7 @@ define zeroext i32 @test_vcmpoeqpd_v2i1_v32i1_mask_mem_b(<2 x i64> %__a, double*
 ; NoVLX-LABEL: test_vcmpoeqpd_v2i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vmovddup {{.*#+}} xmm1 = mem[0,0]
-; NoVLX-NEXT:    vcmpeqpd %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vcmpeqpd (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -22104,8 +21737,7 @@ define zeroext i32 @test_masked_vcmpoeqpd_v2i1_v32i1_mask_mem_b(i2 zeroext %__u,
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vmovddup {{.*#+}} xmm1 = mem[0,0]
-; NoVLX-NEXT:    vcmpeqpd %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vcmpeqpd (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -22141,7 +21773,6 @@ define zeroext i64 @test_vcmpoeqpd_v2i1_v64i1_mask(<2 x i64> %__a, <2 x i64> %__
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -22168,7 +21799,6 @@ define zeroext i64 @test_vcmpoeqpd_v2i1_v64i1_mask_mem(<2 x i64> %__a, <2 x i64>
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -22191,12 +21821,10 @@ define zeroext i64 @test_vcmpoeqpd_v2i1_v64i1_mask_mem_b(<2 x i64> %__a, double*
 ; NoVLX-LABEL: test_vcmpoeqpd_v2i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vmovddup {{.*#+}} xmm1 = mem[0,0]
-; NoVLX-NEXT:    vcmpeqpd %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vcmpeqpd (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -22227,7 +21855,6 @@ define zeroext i64 @test_masked_vcmpoeqpd_v2i1_v64i1_mask(i2 zeroext %__u, <2 x
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -22258,7 +21885,6 @@ define zeroext i64 @test_masked_vcmpoeqpd_v2i1_v64i1_mask_mem(i2 zeroext %__u, <
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -22285,12 +21911,10 @@ define zeroext i64 @test_masked_vcmpoeqpd_v2i1_v64i1_mask_mem_b(i2 zeroext %__u,
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vmovddup {{.*#+}} xmm1 = mem[0,0]
-; NoVLX-NEXT:    vcmpeqpd %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vcmpeqpd (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -22379,8 +22003,7 @@ define zeroext i8 @test_vcmpoeqpd_v4i1_v8i1_mask_mem_b(<4 x i64> %__a, double* %
 ; NoVLX-LABEL: test_vcmpoeqpd_v4i1_v8i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vbroadcastsd (%rdi), %ymm1
-; NoVLX-NEXT:    vcmpeqpd %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vcmpeqpd (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -22479,8 +22102,7 @@ define zeroext i8 @test_masked_vcmpoeqpd_v4i1_v8i1_mask_mem_b(i4 zeroext %__u, <
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vbroadcastsd (%rsi), %ymm1
-; NoVLX-NEXT:    vcmpeqpd %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vcmpeqpd (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -22573,8 +22195,7 @@ define zeroext i16 @test_vcmpoeqpd_v4i1_v16i1_mask_mem_b(<4 x i64> %__a, double*
 ; NoVLX-LABEL: test_vcmpoeqpd_v4i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vbroadcastsd (%rdi), %ymm1
-; NoVLX-NEXT:    vcmpeqpd %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vcmpeqpd (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -22673,8 +22294,7 @@ define zeroext i16 @test_masked_vcmpoeqpd_v4i1_v16i1_mask_mem_b(i4 zeroext %__u,
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vbroadcastsd (%rsi), %ymm1
-; NoVLX-NEXT:    vcmpeqpd %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vcmpeqpd (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -22762,8 +22382,7 @@ define zeroext i32 @test_vcmpoeqpd_v4i1_v32i1_mask_mem_b(<4 x i64> %__a, double*
 ; NoVLX-LABEL: test_vcmpoeqpd_v4i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vbroadcastsd (%rdi), %ymm1
-; NoVLX-NEXT:    vcmpeqpd %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vcmpeqpd (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -22856,8 +22475,7 @@ define zeroext i32 @test_masked_vcmpoeqpd_v4i1_v32i1_mask_mem_b(i4 zeroext %__u,
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vbroadcastsd (%rsi), %ymm1
-; NoVLX-NEXT:    vcmpeqpd %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vcmpeqpd (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -22894,7 +22512,6 @@ define zeroext i64 @test_vcmpoeqpd_v4i1_v64i1_mask(<4 x i64> %__a, <4 x i64> %__
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -22922,7 +22539,6 @@ define zeroext i64 @test_vcmpoeqpd_v4i1_v64i1_mask_mem(<4 x i64> %__a, <4 x i64>
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -22946,12 +22562,10 @@ define zeroext i64 @test_vcmpoeqpd_v4i1_v64i1_mask_mem_b(<4 x i64> %__a, double*
 ; NoVLX-LABEL: test_vcmpoeqpd_v4i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vbroadcastsd (%rdi), %ymm1
-; NoVLX-NEXT:    vcmpeqpd %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vcmpeqpd (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -22983,7 +22597,6 @@ define zeroext i64 @test_masked_vcmpoeqpd_v4i1_v64i1_mask(i4 zeroext %__u, <4 x
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -23015,7 +22628,6 @@ define zeroext i64 @test_masked_vcmpoeqpd_v4i1_v64i1_mask_mem(i4 zeroext %__u, <
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -23043,12 +22655,10 @@ define zeroext i64 @test_masked_vcmpoeqpd_v4i1_v64i1_mask_mem_b(i4 zeroext %__u,
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vbroadcastsd (%rsi), %ymm1
-; NoVLX-NEXT:    vcmpeqpd %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vcmpeqpd (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -23521,7 +23131,6 @@ define zeroext i64 @test_vcmpoeqpd_v8i1_v64i1_mask(<8 x i64> %__a, <8 x i64> %__
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    vcmpeqpd %zmm1, %zmm0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -23545,7 +23154,6 @@ define zeroext i64 @test_vcmpoeqpd_v8i1_v64i1_mask_mem(<8 x i64> %__a, <8 x i64>
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    vcmpeqpd (%rdi), %zmm0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -23570,7 +23178,6 @@ define zeroext i64 @test_vcmpoeqpd_v8i1_v64i1_mask_mem_b(<8 x i64> %__a, double*
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    vcmpeqpd (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -23598,7 +23205,6 @@ define zeroext i64 @test_masked_vcmpoeqpd_v8i1_v64i1_mask(i8 zeroext %__u, <8 x
 ; NoVLX-NEXT:    kmovw %edi, %k1
 ; NoVLX-NEXT:    vcmpeqpd %zmm1, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -23626,7 +23232,6 @@ define zeroext i64 @test_masked_vcmpoeqpd_v8i1_v64i1_mask_mem(i8 zeroext %__u, <
 ; NoVLX-NEXT:    kmovw %edi, %k1
 ; NoVLX-NEXT:    vcmpeqpd (%rsi), %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -23655,7 +23260,6 @@ define zeroext i64 @test_masked_vcmpoeqpd_v8i1_v64i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-NEXT:    kmovw %edi, %k1
 ; NoVLX-NEXT:    vcmpeqpd (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kmovw %k0, %eax
-; NoVLX-NEXT:    movzwl %ax, %eax
 ; NoVLX-NEXT:    vzeroupper
 ; NoVLX-NEXT:    retq
 entry:
@@ -23677,8 +23281,7 @@ define zeroext i64 @test_vcmpoeqpd_v8i1_v64i1_sae_mask(<8 x i64> %__a, <8 x i64>
 ; VLX-LABEL: test_vcmpoeqpd_v8i1_v64i1_sae_mask:
 ; VLX:       # %bb.0: # %entry
 ; VLX-NEXT:    vcmplepd {sae}, %zmm1, %zmm0, %k0
-; VLX-NEXT:    kmovd %k0, %eax
-; VLX-NEXT:    movzbl %al, %eax
+; VLX-NEXT:    kmovb %k0, %eax
 ; VLX-NEXT:    vzeroupper
 ; VLX-NEXT:    retq
 ;
@@ -23763,8 +23366,7 @@ define i8 @mask_zero_lower(<4 x i32> %a) {
 ; NoVLX:       # %bb.0:
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; NoVLX-NEXT:    vptestmd %zmm0, %zmm0, %k0
-; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
-; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
+; NoVLX-NEXT:    kshiftlw $4, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
 ; NoVLX-NEXT:    # kill: def $al killed $al killed $eax
 ; NoVLX-NEXT:    vzeroupper
diff --git a/llvm/test/CodeGen/X86/avx512vl_vnni-intrinsics-upgrade.ll b/llvm/test/CodeGen/X86/avx512vl_vnni-intrinsics-upgrade.ll
index 40b1c0e29..4305259b0 100644
--- a/llvm/test/CodeGen/X86/avx512vl_vnni-intrinsics-upgrade.ll
+++ b/llvm/test/CodeGen/X86/avx512vl_vnni-intrinsics-upgrade.ll
@@ -11,9 +11,9 @@ define <8 x i32>@test_int_x86_avx512_mask_vpdpbusd_256(<8 x i32> %x0, <8 x i32>
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
 ; X86-NEXT:    kmovw %ecx, %k1 # encoding: [0xc5,0xf8,0x92,0xc9]
-; X86-NEXT:    vmovaps %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xd8]
+; X86-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
 ; X86-NEXT:    vpdpbusd (%eax), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0x75,0x29,0x50,0x18]
-; X86-NEXT:    vmovaps %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xe0]
+; X86-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
 ; X86-NEXT:    vpdpbusd %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0x75,0x28,0x50,0xe2]
 ; X86-NEXT:    vpdpbusd %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xa9,0x50,0xc2]
 ; X86-NEXT:    vpaddd %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xfe,0xc0]
@@ -23,9 +23,9 @@ define <8 x i32>@test_int_x86_avx512_mask_vpdpbusd_256(<8 x i32> %x0, <8 x i32>
 ; X64-LABEL: test_int_x86_avx512_mask_vpdpbusd_256:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
-; X64-NEXT:    vmovaps %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xd8]
+; X64-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
 ; X64-NEXT:    vpdpbusd (%rdi), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0x75,0x29,0x50,0x1f]
-; X64-NEXT:    vmovaps %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xe0]
+; X64-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
 ; X64-NEXT:    vpdpbusd %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0x75,0x28,0x50,0xe2]
 ; X64-NEXT:    vpdpbusd %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xa9,0x50,0xc2]
 ; X64-NEXT:    vpaddd %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xfe,0xc0]
@@ -49,9 +49,9 @@ define <4 x i32>@test_int_x86_avx512_mask_vpdpbusd_128(<4 x i32> %x0, <4 x i32>
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
 ; X86-NEXT:    kmovw %ecx, %k1 # encoding: [0xc5,0xf8,0x92,0xc9]
-; X86-NEXT:    vmovaps %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xd8]
+; X86-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
 ; X86-NEXT:    vpdpbusd (%eax), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x09,0x50,0x18]
-; X86-NEXT:    vmovaps %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xe0]
+; X86-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
 ; X86-NEXT:    vpdpbusd %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0x75,0x08,0x50,0xe2]
 ; X86-NEXT:    vpdpbusd %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0x89,0x50,0xc2]
 ; X86-NEXT:    vpaddd %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xfe,0xc0]
@@ -61,9 +61,9 @@ define <4 x i32>@test_int_x86_avx512_mask_vpdpbusd_128(<4 x i32> %x0, <4 x i32>
 ; X64-LABEL: test_int_x86_avx512_mask_vpdpbusd_128:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
-; X64-NEXT:    vmovaps %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xd8]
+; X64-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
 ; X64-NEXT:    vpdpbusd (%rdi), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x09,0x50,0x1f]
-; X64-NEXT:    vmovaps %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xe0]
+; X64-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
 ; X64-NEXT:    vpdpbusd %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0x75,0x08,0x50,0xe2]
 ; X64-NEXT:    vpdpbusd %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0x89,0x50,0xc2]
 ; X64-NEXT:    vpaddd %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xfe,0xc0]
@@ -87,9 +87,9 @@ define <8 x i32>@test_int_x86_avx512_mask_vpdpbusds_256(<8 x i32> %x0, <8 x i32>
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
 ; X86-NEXT:    kmovw %ecx, %k1 # encoding: [0xc5,0xf8,0x92,0xc9]
-; X86-NEXT:    vmovaps %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xd8]
+; X86-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
 ; X86-NEXT:    vpdpbusds (%eax), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0x75,0x29,0x51,0x18]
-; X86-NEXT:    vmovaps %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xe0]
+; X86-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
 ; X86-NEXT:    vpdpbusds %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0x75,0x28,0x51,0xe2]
 ; X86-NEXT:    vpdpbusds %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xa9,0x51,0xc2]
 ; X86-NEXT:    vpaddd %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xfe,0xc0]
@@ -99,9 +99,9 @@ define <8 x i32>@test_int_x86_avx512_mask_vpdpbusds_256(<8 x i32> %x0, <8 x i32>
 ; X64-LABEL: test_int_x86_avx512_mask_vpdpbusds_256:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
-; X64-NEXT:    vmovaps %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xd8]
+; X64-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
 ; X64-NEXT:    vpdpbusds (%rdi), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0x75,0x29,0x51,0x1f]
-; X64-NEXT:    vmovaps %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xe0]
+; X64-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
 ; X64-NEXT:    vpdpbusds %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0x75,0x28,0x51,0xe2]
 ; X64-NEXT:    vpdpbusds %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xa9,0x51,0xc2]
 ; X64-NEXT:    vpaddd %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xfe,0xc0]
@@ -125,9 +125,9 @@ define <4 x i32>@test_int_x86_avx512_mask_vpdpbusds_128(<4 x i32> %x0, <4 x i32>
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
 ; X86-NEXT:    kmovw %ecx, %k1 # encoding: [0xc5,0xf8,0x92,0xc9]
-; X86-NEXT:    vmovaps %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xd8]
+; X86-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
 ; X86-NEXT:    vpdpbusds (%eax), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x09,0x51,0x18]
-; X86-NEXT:    vmovaps %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xe0]
+; X86-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
 ; X86-NEXT:    vpdpbusds %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0x75,0x08,0x51,0xe2]
 ; X86-NEXT:    vpdpbusds %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0x89,0x51,0xc2]
 ; X86-NEXT:    vpaddd %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xfe,0xc0]
@@ -137,9 +137,9 @@ define <4 x i32>@test_int_x86_avx512_mask_vpdpbusds_128(<4 x i32> %x0, <4 x i32>
 ; X64-LABEL: test_int_x86_avx512_mask_vpdpbusds_128:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
-; X64-NEXT:    vmovaps %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xd8]
+; X64-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
 ; X64-NEXT:    vpdpbusds (%rdi), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x09,0x51,0x1f]
-; X64-NEXT:    vmovaps %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xe0]
+; X64-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
 ; X64-NEXT:    vpdpbusds %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0x75,0x08,0x51,0xe2]
 ; X64-NEXT:    vpdpbusds %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0x89,0x51,0xc2]
 ; X64-NEXT:    vpaddd %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xfe,0xc0]
@@ -163,9 +163,9 @@ define <8 x i32>@test_int_x86_avx512_mask_vpdpwssd_256(<8 x i32> %x0, <8 x i32>
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
 ; X86-NEXT:    kmovw %ecx, %k1 # encoding: [0xc5,0xf8,0x92,0xc9]
-; X86-NEXT:    vmovaps %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xd8]
+; X86-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
 ; X86-NEXT:    vpdpwssd (%eax), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0x75,0x29,0x52,0x18]
-; X86-NEXT:    vmovaps %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xe0]
+; X86-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
 ; X86-NEXT:    vpdpwssd %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0x75,0x28,0x52,0xe2]
 ; X86-NEXT:    vpdpwssd %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xa9,0x52,0xc2]
 ; X86-NEXT:    vpaddd %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xfe,0xc0]
@@ -175,9 +175,9 @@ define <8 x i32>@test_int_x86_avx512_mask_vpdpwssd_256(<8 x i32> %x0, <8 x i32>
 ; X64-LABEL: test_int_x86_avx512_mask_vpdpwssd_256:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
-; X64-NEXT:    vmovaps %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xd8]
+; X64-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
 ; X64-NEXT:    vpdpwssd (%rdi), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0x75,0x29,0x52,0x1f]
-; X64-NEXT:    vmovaps %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xe0]
+; X64-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
 ; X64-NEXT:    vpdpwssd %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0x75,0x28,0x52,0xe2]
 ; X64-NEXT:    vpdpwssd %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xa9,0x52,0xc2]
 ; X64-NEXT:    vpaddd %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xfe,0xc0]
@@ -201,9 +201,9 @@ define <4 x i32>@test_int_x86_avx512_mask_vpdpwssd_128(<4 x i32> %x0, <4 x i32>
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
 ; X86-NEXT:    kmovw %ecx, %k1 # encoding: [0xc5,0xf8,0x92,0xc9]
-; X86-NEXT:    vmovaps %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xd8]
+; X86-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
 ; X86-NEXT:    vpdpwssd (%eax), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x09,0x52,0x18]
-; X86-NEXT:    vmovaps %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xe0]
+; X86-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
 ; X86-NEXT:    vpdpwssd %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0x75,0x08,0x52,0xe2]
 ; X86-NEXT:    vpdpwssd %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0x89,0x52,0xc2]
 ; X86-NEXT:    vpaddd %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xfe,0xc0]
@@ -213,9 +213,9 @@ define <4 x i32>@test_int_x86_avx512_mask_vpdpwssd_128(<4 x i32> %x0, <4 x i32>
 ; X64-LABEL: test_int_x86_avx512_mask_vpdpwssd_128:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
-; X64-NEXT:    vmovaps %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xd8]
+; X64-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
 ; X64-NEXT:    vpdpwssd (%rdi), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x09,0x52,0x1f]
-; X64-NEXT:    vmovaps %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xe0]
+; X64-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
 ; X64-NEXT:    vpdpwssd %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0x75,0x08,0x52,0xe2]
 ; X64-NEXT:    vpdpwssd %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0x89,0x52,0xc2]
 ; X64-NEXT:    vpaddd %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xfe,0xc0]
@@ -240,9 +240,9 @@ define <8 x i32>@test_int_x86_avx512_mask_vpdpwssds_256(<8 x i32> %x0, <8 x i32>
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
 ; X86-NEXT:    kmovw %ecx, %k1 # encoding: [0xc5,0xf8,0x92,0xc9]
-; X86-NEXT:    vmovaps %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xd8]
+; X86-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
 ; X86-NEXT:    vpdpwssds (%eax), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0x75,0x29,0x53,0x18]
-; X86-NEXT:    vmovaps %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xe0]
+; X86-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
 ; X86-NEXT:    vpdpwssds %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0x75,0x28,0x53,0xe2]
 ; X86-NEXT:    vpdpwssds %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xa9,0x53,0xc2]
 ; X86-NEXT:    vpaddd %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xfe,0xc0]
@@ -252,9 +252,9 @@ define <8 x i32>@test_int_x86_avx512_mask_vpdpwssds_256(<8 x i32> %x0, <8 x i32>
 ; X64-LABEL: test_int_x86_avx512_mask_vpdpwssds_256:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
-; X64-NEXT:    vmovaps %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xd8]
+; X64-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
 ; X64-NEXT:    vpdpwssds (%rdi), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0x75,0x29,0x53,0x1f]
-; X64-NEXT:    vmovaps %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xe0]
+; X64-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
 ; X64-NEXT:    vpdpwssds %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0x75,0x28,0x53,0xe2]
 ; X64-NEXT:    vpdpwssds %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xa9,0x53,0xc2]
 ; X64-NEXT:    vpaddd %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xfe,0xc0]
@@ -278,9 +278,9 @@ define <4 x i32>@test_int_x86_avx512_mask_vpdpwssds_128(<4 x i32> %x0, <4 x i32>
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
 ; X86-NEXT:    kmovw %ecx, %k1 # encoding: [0xc5,0xf8,0x92,0xc9]
-; X86-NEXT:    vmovaps %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xd8]
+; X86-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
 ; X86-NEXT:    vpdpwssds (%eax), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x09,0x53,0x18]
-; X86-NEXT:    vmovaps %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xe0]
+; X86-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
 ; X86-NEXT:    vpdpwssds %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0x75,0x08,0x53,0xe2]
 ; X86-NEXT:    vpdpwssds %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0x89,0x53,0xc2]
 ; X86-NEXT:    vpaddd %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xfe,0xc0]
@@ -290,9 +290,9 @@ define <4 x i32>@test_int_x86_avx512_mask_vpdpwssds_128(<4 x i32> %x0, <4 x i32>
 ; X64-LABEL: test_int_x86_avx512_mask_vpdpwssds_128:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
-; X64-NEXT:    vmovaps %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xd8]
+; X64-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
 ; X64-NEXT:    vpdpwssds (%rdi), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x09,0x53,0x1f]
-; X64-NEXT:    vmovaps %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xe0]
+; X64-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
 ; X64-NEXT:    vpdpwssds %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0x75,0x08,0x53,0xe2]
 ; X64-NEXT:    vpdpwssds %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0x89,0x53,0xc2]
 ; X64-NEXT:    vpaddd %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xfe,0xc0]
diff --git a/llvm/test/CodeGen/X86/avx512vl_vnni-intrinsics.ll b/llvm/test/CodeGen/X86/avx512vl_vnni-intrinsics.ll
index 61879ae2f..68bda52ac 100644
--- a/llvm/test/CodeGen/X86/avx512vl_vnni-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/avx512vl_vnni-intrinsics.ll
@@ -10,9 +10,9 @@ define <8 x i32>@test_int_x86_avx512_mask_vpdpbusd_256(<8 x i32> %x0, <8 x i32>
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
 ; X86-NEXT:    kmovw %ecx, %k1 # encoding: [0xc5,0xf8,0x92,0xc9]
-; X86-NEXT:    vmovaps %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xd8]
+; X86-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
 ; X86-NEXT:    vpdpbusd (%eax), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0x75,0x29,0x50,0x18]
-; X86-NEXT:    vmovaps %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xe0]
+; X86-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
 ; X86-NEXT:    vpdpbusd %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0x75,0x28,0x50,0xe2]
 ; X86-NEXT:    vpdpbusd %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xa9,0x50,0xc2]
 ; X86-NEXT:    vpaddd %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xfe,0xc0]
@@ -22,9 +22,9 @@ define <8 x i32>@test_int_x86_avx512_mask_vpdpbusd_256(<8 x i32> %x0, <8 x i32>
 ; X64-LABEL: test_int_x86_avx512_mask_vpdpbusd_256:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
-; X64-NEXT:    vmovaps %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xd8]
+; X64-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
 ; X64-NEXT:    vpdpbusd (%rdi), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0x75,0x29,0x50,0x1f]
-; X64-NEXT:    vmovaps %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xe0]
+; X64-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
 ; X64-NEXT:    vpdpbusd %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0x75,0x28,0x50,0xe2]
 ; X64-NEXT:    vpdpbusd %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xa9,0x50,0xc2]
 ; X64-NEXT:    vpaddd %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xfe,0xc0]
@@ -51,9 +51,9 @@ define <4 x i32>@test_int_x86_avx512_mask_vpdpbusd_128(<4 x i32> %x0, <4 x i32>
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
 ; X86-NEXT:    kmovw %ecx, %k1 # encoding: [0xc5,0xf8,0x92,0xc9]
-; X86-NEXT:    vmovaps %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xd8]
+; X86-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
 ; X86-NEXT:    vpdpbusd (%eax), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x09,0x50,0x18]
-; X86-NEXT:    vmovaps %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xe0]
+; X86-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
 ; X86-NEXT:    vpdpbusd %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0x75,0x08,0x50,0xe2]
 ; X86-NEXT:    vpdpbusd %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0x89,0x50,0xc2]
 ; X86-NEXT:    vpaddd %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xfe,0xc0]
@@ -63,9 +63,9 @@ define <4 x i32>@test_int_x86_avx512_mask_vpdpbusd_128(<4 x i32> %x0, <4 x i32>
 ; X64-LABEL: test_int_x86_avx512_mask_vpdpbusd_128:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
-; X64-NEXT:    vmovaps %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xd8]
+; X64-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
 ; X64-NEXT:    vpdpbusd (%rdi), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x09,0x50,0x1f]
-; X64-NEXT:    vmovaps %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xe0]
+; X64-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
 ; X64-NEXT:    vpdpbusd %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0x75,0x08,0x50,0xe2]
 ; X64-NEXT:    vpdpbusd %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0x89,0x50,0xc2]
 ; X64-NEXT:    vpaddd %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xfe,0xc0]
@@ -94,9 +94,9 @@ define <8 x i32>@test_int_x86_avx512_mask_vpdpbusds_256(<8 x i32> %x0, <8 x i32>
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
 ; X86-NEXT:    kmovw %ecx, %k1 # encoding: [0xc5,0xf8,0x92,0xc9]
-; X86-NEXT:    vmovaps %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xd8]
+; X86-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
 ; X86-NEXT:    vpdpbusds (%eax), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0x75,0x29,0x51,0x18]
-; X86-NEXT:    vmovaps %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xe0]
+; X86-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
 ; X86-NEXT:    vpdpbusds %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0x75,0x28,0x51,0xe2]
 ; X86-NEXT:    vpdpbusds %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xa9,0x51,0xc2]
 ; X86-NEXT:    vpaddd %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xfe,0xc0]
@@ -106,9 +106,9 @@ define <8 x i32>@test_int_x86_avx512_mask_vpdpbusds_256(<8 x i32> %x0, <8 x i32>
 ; X64-LABEL: test_int_x86_avx512_mask_vpdpbusds_256:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
-; X64-NEXT:    vmovaps %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xd8]
+; X64-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
 ; X64-NEXT:    vpdpbusds (%rdi), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0x75,0x29,0x51,0x1f]
-; X64-NEXT:    vmovaps %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xe0]
+; X64-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
 ; X64-NEXT:    vpdpbusds %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0x75,0x28,0x51,0xe2]
 ; X64-NEXT:    vpdpbusds %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xa9,0x51,0xc2]
 ; X64-NEXT:    vpaddd %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xfe,0xc0]
@@ -135,9 +135,9 @@ define <4 x i32>@test_int_x86_avx512_mask_vpdpbusds_128(<4 x i32> %x0, <4 x i32>
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
 ; X86-NEXT:    kmovw %ecx, %k1 # encoding: [0xc5,0xf8,0x92,0xc9]
-; X86-NEXT:    vmovaps %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xd8]
+; X86-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
 ; X86-NEXT:    vpdpbusds (%eax), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x09,0x51,0x18]
-; X86-NEXT:    vmovaps %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xe0]
+; X86-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
 ; X86-NEXT:    vpdpbusds %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0x75,0x08,0x51,0xe2]
 ; X86-NEXT:    vpdpbusds %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0x89,0x51,0xc2]
 ; X86-NEXT:    vpaddd %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xfe,0xc0]
@@ -147,9 +147,9 @@ define <4 x i32>@test_int_x86_avx512_mask_vpdpbusds_128(<4 x i32> %x0, <4 x i32>
 ; X64-LABEL: test_int_x86_avx512_mask_vpdpbusds_128:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
-; X64-NEXT:    vmovaps %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xd8]
+; X64-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
 ; X64-NEXT:    vpdpbusds (%rdi), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x09,0x51,0x1f]
-; X64-NEXT:    vmovaps %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xe0]
+; X64-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
 ; X64-NEXT:    vpdpbusds %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0x75,0x08,0x51,0xe2]
 ; X64-NEXT:    vpdpbusds %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0x89,0x51,0xc2]
 ; X64-NEXT:    vpaddd %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xfe,0xc0]
@@ -178,9 +178,9 @@ define <8 x i32>@test_int_x86_avx512_mask_vpdpwssd_256(<8 x i32> %x0, <8 x i32>
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
 ; X86-NEXT:    kmovw %ecx, %k1 # encoding: [0xc5,0xf8,0x92,0xc9]
-; X86-NEXT:    vmovaps %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xd8]
+; X86-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
 ; X86-NEXT:    vpdpwssd (%eax), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0x75,0x29,0x52,0x18]
-; X86-NEXT:    vmovaps %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xe0]
+; X86-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
 ; X86-NEXT:    vpdpwssd %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0x75,0x28,0x52,0xe2]
 ; X86-NEXT:    vpdpwssd %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xa9,0x52,0xc2]
 ; X86-NEXT:    vpaddd %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xfe,0xc0]
@@ -190,9 +190,9 @@ define <8 x i32>@test_int_x86_avx512_mask_vpdpwssd_256(<8 x i32> %x0, <8 x i32>
 ; X64-LABEL: test_int_x86_avx512_mask_vpdpwssd_256:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
-; X64-NEXT:    vmovaps %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xd8]
+; X64-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
 ; X64-NEXT:    vpdpwssd (%rdi), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0x75,0x29,0x52,0x1f]
-; X64-NEXT:    vmovaps %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xe0]
+; X64-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
 ; X64-NEXT:    vpdpwssd %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0x75,0x28,0x52,0xe2]
 ; X64-NEXT:    vpdpwssd %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xa9,0x52,0xc2]
 ; X64-NEXT:    vpaddd %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xfe,0xc0]
@@ -219,9 +219,9 @@ define <4 x i32>@test_int_x86_avx512_mask_vpdpwssd_128(<4 x i32> %x0, <4 x i32>
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
 ; X86-NEXT:    kmovw %ecx, %k1 # encoding: [0xc5,0xf8,0x92,0xc9]
-; X86-NEXT:    vmovaps %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xd8]
+; X86-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
 ; X86-NEXT:    vpdpwssd (%eax), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x09,0x52,0x18]
-; X86-NEXT:    vmovaps %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xe0]
+; X86-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
 ; X86-NEXT:    vpdpwssd %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0x75,0x08,0x52,0xe2]
 ; X86-NEXT:    vpdpwssd %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0x89,0x52,0xc2]
 ; X86-NEXT:    vpaddd %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xfe,0xc0]
@@ -231,9 +231,9 @@ define <4 x i32>@test_int_x86_avx512_mask_vpdpwssd_128(<4 x i32> %x0, <4 x i32>
 ; X64-LABEL: test_int_x86_avx512_mask_vpdpwssd_128:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
-; X64-NEXT:    vmovaps %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xd8]
+; X64-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
 ; X64-NEXT:    vpdpwssd (%rdi), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x09,0x52,0x1f]
-; X64-NEXT:    vmovaps %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xe0]
+; X64-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
 ; X64-NEXT:    vpdpwssd %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0x75,0x08,0x52,0xe2]
 ; X64-NEXT:    vpdpwssd %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0x89,0x52,0xc2]
 ; X64-NEXT:    vpaddd %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xfe,0xc0]
@@ -262,9 +262,9 @@ define <8 x i32>@test_int_x86_avx512_mask_vpdpwssds_256(<8 x i32> %x0, <8 x i32>
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
 ; X86-NEXT:    kmovw %ecx, %k1 # encoding: [0xc5,0xf8,0x92,0xc9]
-; X86-NEXT:    vmovaps %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xd8]
+; X86-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
 ; X86-NEXT:    vpdpwssds (%eax), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0x75,0x29,0x53,0x18]
-; X86-NEXT:    vmovaps %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xe0]
+; X86-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
 ; X86-NEXT:    vpdpwssds %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0x75,0x28,0x53,0xe2]
 ; X86-NEXT:    vpdpwssds %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xa9,0x53,0xc2]
 ; X86-NEXT:    vpaddd %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xfe,0xc0]
@@ -274,9 +274,9 @@ define <8 x i32>@test_int_x86_avx512_mask_vpdpwssds_256(<8 x i32> %x0, <8 x i32>
 ; X64-LABEL: test_int_x86_avx512_mask_vpdpwssds_256:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
-; X64-NEXT:    vmovaps %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xd8]
+; X64-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
 ; X64-NEXT:    vpdpwssds (%rdi), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0x75,0x29,0x53,0x1f]
-; X64-NEXT:    vmovaps %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xe0]
+; X64-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
 ; X64-NEXT:    vpdpwssds %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0x75,0x28,0x53,0xe2]
 ; X64-NEXT:    vpdpwssds %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xa9,0x53,0xc2]
 ; X64-NEXT:    vpaddd %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xfe,0xc0]
@@ -303,9 +303,9 @@ define <4 x i32>@test_int_x86_avx512_mask_vpdpwssds_128(<4 x i32> %x0, <4 x i32>
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
 ; X86-NEXT:    kmovw %ecx, %k1 # encoding: [0xc5,0xf8,0x92,0xc9]
-; X86-NEXT:    vmovaps %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xd8]
+; X86-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
 ; X86-NEXT:    vpdpwssds (%eax), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x09,0x53,0x18]
-; X86-NEXT:    vmovaps %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xe0]
+; X86-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
 ; X86-NEXT:    vpdpwssds %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0x75,0x08,0x53,0xe2]
 ; X86-NEXT:    vpdpwssds %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0x89,0x53,0xc2]
 ; X86-NEXT:    vpaddd %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xfe,0xc0]
@@ -315,9 +315,9 @@ define <4 x i32>@test_int_x86_avx512_mask_vpdpwssds_128(<4 x i32> %x0, <4 x i32>
 ; X64-LABEL: test_int_x86_avx512_mask_vpdpwssds_128:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
-; X64-NEXT:    vmovaps %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xd8]
+; X64-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
 ; X64-NEXT:    vpdpwssds (%rdi), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x09,0x53,0x1f]
-; X64-NEXT:    vmovaps %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xe0]
+; X64-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
 ; X64-NEXT:    vpdpwssds %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0x75,0x08,0x53,0xe2]
 ; X64-NEXT:    vpdpwssds %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0x89,0x53,0xc2]
 ; X64-NEXT:    vpaddd %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xfe,0xc0]
diff --git a/llvm/test/CodeGen/X86/avx512vlcd-intrinsics-fast-isel.ll b/llvm/test/CodeGen/X86/avx512vlcd-intrinsics-fast-isel.ll
index 8b57c5720..a9fb97f70 100644
--- a/llvm/test/CodeGen/X86/avx512vlcd-intrinsics-fast-isel.ll
+++ b/llvm/test/CodeGen/X86/avx512vlcd-intrinsics-fast-isel.ll
@@ -7,7 +7,6 @@ define <2 x i64> @test_mm_broadcastmb_epi64(<2 x i64> %a, <2 x i64> %b) {
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    vpcmpeqd %xmm1, %xmm0, %k0
 ; X86-NEXT:    kmovw %k0, %eax
-; X86-NEXT:    movzbl %al, %eax
 ; X86-NEXT:    vmovd %eax, %xmm0
 ; X86-NEXT:    vpbroadcastq %xmm0, %xmm0
 ; X86-NEXT:    retl
@@ -34,7 +33,6 @@ define <4 x i64> @test_mm256_broadcastmb_epi64(<4 x i64> %a, <4 x i64> %b) {
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    vpcmpeqq %ymm1, %ymm0, %k0
 ; X86-NEXT:    kmovw %k0, %eax
-; X86-NEXT:    movzbl %al, %eax
 ; X86-NEXT:    vmovd %eax, %xmm0
 ; X86-NEXT:    vpbroadcastq %xmm0, %ymm0
 ; X86-NEXT:    retl
diff --git a/llvm/test/CodeGen/X86/avx512vlvp2intersect-intrinsics.ll b/llvm/test/CodeGen/X86/avx512vlvp2intersect-intrinsics.ll
index c4c70fcb2..fe3662d49 100644
--- a/llvm/test/CodeGen/X86/avx512vlvp2intersect-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/avx512vlvp2intersect-intrinsics.ll
@@ -229,8 +229,7 @@ define void @test_mm256_2intersect_epi64_b(i64* nocapture readonly %a, i64* noca
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx # encoding: [0x8b,0x54,0x24,0x0c]
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi # encoding: [0x8b,0x74,0x24,0x08]
 ; X86-NEXT:    vbroadcastsd (%esi), %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x19,0x06]
-; X86-NEXT:    vbroadcastsd (%edx), %ymm1 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x19,0x0a]
-; X86-NEXT:    vp2intersectq %ymm1, %ymm0, %k0 # encoding: [0x62,0xf2,0xff,0x28,0x68,0xc1]
+; X86-NEXT:    vp2intersectq (%edx){1to4}, %ymm0, %k0 # encoding: [0x62,0xf2,0xff,0x38,0x68,0x02]
 ; X86-NEXT:    kshiftlw $12, %k0, %k2 # encoding: [0xc4,0xe3,0xf9,0x32,0xd0,0x0c]
 ; X86-NEXT:    kshiftrw $12, %k2, %k2 # encoding: [0xc4,0xe3,0xf9,0x30,0xd2,0x0c]
 ; X86-NEXT:    kmovw %k2, %edx # encoding: [0xc5,0xf8,0x93,0xd2]
@@ -535,9 +534,7 @@ define void @test_mm_2intersect_epi64_b(i64* nocapture readonly %a, i64* nocaptu
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi # encoding: [0x8b,0x74,0x24,0x08]
 ; X86-NEXT:    vmovddup (%esi), %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xfb,0x12,0x06]
 ; X86-NEXT:    # xmm0 = mem[0,0]
-; X86-NEXT:    vmovddup (%edx), %xmm1 # EVEX TO VEX Compression encoding: [0xc5,0xfb,0x12,0x0a]
-; X86-NEXT:    # xmm1 = mem[0,0]
-; X86-NEXT:    vp2intersectq %xmm1, %xmm0, %k0 # encoding: [0x62,0xf2,0xff,0x08,0x68,0xc1]
+; X86-NEXT:    vp2intersectq (%edx){1to2}, %xmm0, %k0 # encoding: [0x62,0xf2,0xff,0x18,0x68,0x02]
 ; X86-NEXT:    kshiftlw $14, %k0, %k2 # encoding: [0xc4,0xe3,0xf9,0x32,0xd0,0x0e]
 ; X86-NEXT:    kshiftrw $14, %k2, %k2 # encoding: [0xc4,0xe3,0xf9,0x30,0xd2,0x0e]
 ; X86-NEXT:    kmovw %k2, %edx # encoding: [0xc5,0xf8,0x93,0xd2]
diff --git a/llvm/test/CodeGen/X86/avx512vnni-intrinsics-upgrade.ll b/llvm/test/CodeGen/X86/avx512vnni-intrinsics-upgrade.ll
index 1ed336f7f..bd3d40671 100644
--- a/llvm/test/CodeGen/X86/avx512vnni-intrinsics-upgrade.ll
+++ b/llvm/test/CodeGen/X86/avx512vnni-intrinsics-upgrade.ll
@@ -10,9 +10,9 @@ define <16 x i32>@test_int_x86_avx512_mask_vpdpbusd_512(<16 x i32> %x0, <16 x i3
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
-; X86-NEXT:    vmovaps %zmm0, %zmm3 # encoding: [0x62,0xf1,0x7c,0x48,0x28,0xd8]
+; X86-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
 ; X86-NEXT:    vpdpbusd (%eax), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x49,0x50,0x18]
-; X86-NEXT:    vmovaps %zmm0, %zmm4 # encoding: [0x62,0xf1,0x7c,0x48,0x28,0xe0]
+; X86-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
 ; X86-NEXT:    vpdpbusd %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0x75,0x48,0x50,0xe2]
 ; X86-NEXT:    vpdpbusd %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xc9,0x50,0xc2]
 ; X86-NEXT:    vpaddd %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0x5d,0x48,0xfe,0xc0]
@@ -22,9 +22,9 @@ define <16 x i32>@test_int_x86_avx512_mask_vpdpbusd_512(<16 x i32> %x0, <16 x i3
 ; X64-LABEL: test_int_x86_avx512_mask_vpdpbusd_512:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
-; X64-NEXT:    vmovaps %zmm0, %zmm3 # encoding: [0x62,0xf1,0x7c,0x48,0x28,0xd8]
+; X64-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
 ; X64-NEXT:    vpdpbusd (%rdi), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x49,0x50,0x1f]
-; X64-NEXT:    vmovaps %zmm0, %zmm4 # encoding: [0x62,0xf1,0x7c,0x48,0x28,0xe0]
+; X64-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
 ; X64-NEXT:    vpdpbusd %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0x75,0x48,0x50,0xe2]
 ; X64-NEXT:    vpdpbusd %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xc9,0x50,0xc2]
 ; X64-NEXT:    vpaddd %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0x5d,0x48,0xfe,0xc0]
@@ -47,9 +47,9 @@ define <16 x i32>@test_int_x86_avx512_mask_vpdpbusds_512(<16 x i32> %x0, <16 x i
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
-; X86-NEXT:    vmovaps %zmm0, %zmm3 # encoding: [0x62,0xf1,0x7c,0x48,0x28,0xd8]
+; X86-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
 ; X86-NEXT:    vpdpbusds (%eax), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x49,0x51,0x18]
-; X86-NEXT:    vmovaps %zmm0, %zmm4 # encoding: [0x62,0xf1,0x7c,0x48,0x28,0xe0]
+; X86-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
 ; X86-NEXT:    vpdpbusds %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0x75,0x48,0x51,0xe2]
 ; X86-NEXT:    vpdpbusds %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xc9,0x51,0xc2]
 ; X86-NEXT:    vpaddd %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0x5d,0x48,0xfe,0xc0]
@@ -59,9 +59,9 @@ define <16 x i32>@test_int_x86_avx512_mask_vpdpbusds_512(<16 x i32> %x0, <16 x i
 ; X64-LABEL: test_int_x86_avx512_mask_vpdpbusds_512:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
-; X64-NEXT:    vmovaps %zmm0, %zmm3 # encoding: [0x62,0xf1,0x7c,0x48,0x28,0xd8]
+; X64-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
 ; X64-NEXT:    vpdpbusds (%rdi), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x49,0x51,0x1f]
-; X64-NEXT:    vmovaps %zmm0, %zmm4 # encoding: [0x62,0xf1,0x7c,0x48,0x28,0xe0]
+; X64-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
 ; X64-NEXT:    vpdpbusds %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0x75,0x48,0x51,0xe2]
 ; X64-NEXT:    vpdpbusds %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xc9,0x51,0xc2]
 ; X64-NEXT:    vpaddd %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0x5d,0x48,0xfe,0xc0]
@@ -84,9 +84,9 @@ define <16 x i32>@test_int_x86_avx512_mask_vpdpwssd_512(<16 x i32> %x0, <16 x i3
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
-; X86-NEXT:    vmovaps %zmm0, %zmm3 # encoding: [0x62,0xf1,0x7c,0x48,0x28,0xd8]
+; X86-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
 ; X86-NEXT:    vpdpwssd (%eax), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x49,0x52,0x18]
-; X86-NEXT:    vmovaps %zmm0, %zmm4 # encoding: [0x62,0xf1,0x7c,0x48,0x28,0xe0]
+; X86-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
 ; X86-NEXT:    vpdpwssd %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0x75,0x48,0x52,0xe2]
 ; X86-NEXT:    vpdpwssd %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xc9,0x52,0xc2]
 ; X86-NEXT:    vpaddd %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0x5d,0x48,0xfe,0xc0]
@@ -96,9 +96,9 @@ define <16 x i32>@test_int_x86_avx512_mask_vpdpwssd_512(<16 x i32> %x0, <16 x i3
 ; X64-LABEL: test_int_x86_avx512_mask_vpdpwssd_512:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
-; X64-NEXT:    vmovaps %zmm0, %zmm3 # encoding: [0x62,0xf1,0x7c,0x48,0x28,0xd8]
+; X64-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
 ; X64-NEXT:    vpdpwssd (%rdi), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x49,0x52,0x1f]
-; X64-NEXT:    vmovaps %zmm0, %zmm4 # encoding: [0x62,0xf1,0x7c,0x48,0x28,0xe0]
+; X64-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
 ; X64-NEXT:    vpdpwssd %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0x75,0x48,0x52,0xe2]
 ; X64-NEXT:    vpdpwssd %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xc9,0x52,0xc2]
 ; X64-NEXT:    vpaddd %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0x5d,0x48,0xfe,0xc0]
@@ -121,9 +121,9 @@ define <16 x i32>@test_int_x86_avx512_mask_vpdpwssds_512(<16 x i32> %x0, <16 x i
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
-; X86-NEXT:    vmovaps %zmm0, %zmm3 # encoding: [0x62,0xf1,0x7c,0x48,0x28,0xd8]
+; X86-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
 ; X86-NEXT:    vpdpwssds (%eax), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x49,0x53,0x18]
-; X86-NEXT:    vmovaps %zmm0, %zmm4 # encoding: [0x62,0xf1,0x7c,0x48,0x28,0xe0]
+; X86-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
 ; X86-NEXT:    vpdpwssds %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0x75,0x48,0x53,0xe2]
 ; X86-NEXT:    vpdpwssds %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xc9,0x53,0xc2]
 ; X86-NEXT:    vpaddd %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0x5d,0x48,0xfe,0xc0]
@@ -133,9 +133,9 @@ define <16 x i32>@test_int_x86_avx512_mask_vpdpwssds_512(<16 x i32> %x0, <16 x i
 ; X64-LABEL: test_int_x86_avx512_mask_vpdpwssds_512:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
-; X64-NEXT:    vmovaps %zmm0, %zmm3 # encoding: [0x62,0xf1,0x7c,0x48,0x28,0xd8]
+; X64-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
 ; X64-NEXT:    vpdpwssds (%rdi), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x49,0x53,0x1f]
-; X64-NEXT:    vmovaps %zmm0, %zmm4 # encoding: [0x62,0xf1,0x7c,0x48,0x28,0xe0]
+; X64-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
 ; X64-NEXT:    vpdpwssds %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0x75,0x48,0x53,0xe2]
 ; X64-NEXT:    vpdpwssds %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xc9,0x53,0xc2]
 ; X64-NEXT:    vpaddd %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0x5d,0x48,0xfe,0xc0]
diff --git a/llvm/test/CodeGen/X86/avx512vnni-intrinsics.ll b/llvm/test/CodeGen/X86/avx512vnni-intrinsics.ll
index f6a42b945..47fce1801 100644
--- a/llvm/test/CodeGen/X86/avx512vnni-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/avx512vnni-intrinsics.ll
@@ -9,9 +9,9 @@ define <16 x i32>@test_int_x86_avx512_mask_vpdpbusd_512(<16 x i32> %x0, <16 x i3
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
-; X86-NEXT:    vmovaps %zmm0, %zmm3 # encoding: [0x62,0xf1,0x7c,0x48,0x28,0xd8]
+; X86-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
 ; X86-NEXT:    vpdpbusd (%eax), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x49,0x50,0x18]
-; X86-NEXT:    vmovaps %zmm0, %zmm4 # encoding: [0x62,0xf1,0x7c,0x48,0x28,0xe0]
+; X86-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
 ; X86-NEXT:    vpdpbusd %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0x75,0x48,0x50,0xe2]
 ; X86-NEXT:    vpdpbusd %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xc9,0x50,0xc2]
 ; X86-NEXT:    vpaddd %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0x5d,0x48,0xfe,0xc0]
@@ -21,9 +21,9 @@ define <16 x i32>@test_int_x86_avx512_mask_vpdpbusd_512(<16 x i32> %x0, <16 x i3
 ; X64-LABEL: test_int_x86_avx512_mask_vpdpbusd_512:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
-; X64-NEXT:    vmovaps %zmm0, %zmm3 # encoding: [0x62,0xf1,0x7c,0x48,0x28,0xd8]
+; X64-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
 ; X64-NEXT:    vpdpbusd (%rdi), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x49,0x50,0x1f]
-; X64-NEXT:    vmovaps %zmm0, %zmm4 # encoding: [0x62,0xf1,0x7c,0x48,0x28,0xe0]
+; X64-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
 ; X64-NEXT:    vpdpbusd %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0x75,0x48,0x50,0xe2]
 ; X64-NEXT:    vpdpbusd %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xc9,0x50,0xc2]
 ; X64-NEXT:    vpaddd %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0x5d,0x48,0xfe,0xc0]
@@ -49,9 +49,9 @@ define <16 x i32>@test_int_x86_avx512_mask_vpdpbusds_512(<16 x i32> %x0, <16 x i
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
-; X86-NEXT:    vmovaps %zmm0, %zmm3 # encoding: [0x62,0xf1,0x7c,0x48,0x28,0xd8]
+; X86-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
 ; X86-NEXT:    vpdpbusds (%eax), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x49,0x51,0x18]
-; X86-NEXT:    vmovaps %zmm0, %zmm4 # encoding: [0x62,0xf1,0x7c,0x48,0x28,0xe0]
+; X86-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
 ; X86-NEXT:    vpdpbusds %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0x75,0x48,0x51,0xe2]
 ; X86-NEXT:    vpdpbusds %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xc9,0x51,0xc2]
 ; X86-NEXT:    vpaddd %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0x5d,0x48,0xfe,0xc0]
@@ -61,9 +61,9 @@ define <16 x i32>@test_int_x86_avx512_mask_vpdpbusds_512(<16 x i32> %x0, <16 x i
 ; X64-LABEL: test_int_x86_avx512_mask_vpdpbusds_512:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
-; X64-NEXT:    vmovaps %zmm0, %zmm3 # encoding: [0x62,0xf1,0x7c,0x48,0x28,0xd8]
+; X64-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
 ; X64-NEXT:    vpdpbusds (%rdi), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x49,0x51,0x1f]
-; X64-NEXT:    vmovaps %zmm0, %zmm4 # encoding: [0x62,0xf1,0x7c,0x48,0x28,0xe0]
+; X64-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
 ; X64-NEXT:    vpdpbusds %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0x75,0x48,0x51,0xe2]
 ; X64-NEXT:    vpdpbusds %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xc9,0x51,0xc2]
 ; X64-NEXT:    vpaddd %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0x5d,0x48,0xfe,0xc0]
@@ -89,9 +89,9 @@ define <16 x i32>@test_int_x86_avx512_mask_vpdpwssd_512(<16 x i32> %x0, <16 x i3
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
-; X86-NEXT:    vmovaps %zmm0, %zmm3 # encoding: [0x62,0xf1,0x7c,0x48,0x28,0xd8]
+; X86-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
 ; X86-NEXT:    vpdpwssd (%eax), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x49,0x52,0x18]
-; X86-NEXT:    vmovaps %zmm0, %zmm4 # encoding: [0x62,0xf1,0x7c,0x48,0x28,0xe0]
+; X86-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
 ; X86-NEXT:    vpdpwssd %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0x75,0x48,0x52,0xe2]
 ; X86-NEXT:    vpdpwssd %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xc9,0x52,0xc2]
 ; X86-NEXT:    vpaddd %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0x5d,0x48,0xfe,0xc0]
@@ -101,9 +101,9 @@ define <16 x i32>@test_int_x86_avx512_mask_vpdpwssd_512(<16 x i32> %x0, <16 x i3
 ; X64-LABEL: test_int_x86_avx512_mask_vpdpwssd_512:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
-; X64-NEXT:    vmovaps %zmm0, %zmm3 # encoding: [0x62,0xf1,0x7c,0x48,0x28,0xd8]
+; X64-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
 ; X64-NEXT:    vpdpwssd (%rdi), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x49,0x52,0x1f]
-; X64-NEXT:    vmovaps %zmm0, %zmm4 # encoding: [0x62,0xf1,0x7c,0x48,0x28,0xe0]
+; X64-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
 ; X64-NEXT:    vpdpwssd %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0x75,0x48,0x52,0xe2]
 ; X64-NEXT:    vpdpwssd %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xc9,0x52,0xc2]
 ; X64-NEXT:    vpaddd %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0x5d,0x48,0xfe,0xc0]
@@ -129,9 +129,9 @@ define <16 x i32>@test_int_x86_avx512_mask_vpdpwssds_512(<16 x i32> %x0, <16 x i
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
-; X86-NEXT:    vmovaps %zmm0, %zmm3 # encoding: [0x62,0xf1,0x7c,0x48,0x28,0xd8]
+; X86-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
 ; X86-NEXT:    vpdpwssds (%eax), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x49,0x53,0x18]
-; X86-NEXT:    vmovaps %zmm0, %zmm4 # encoding: [0x62,0xf1,0x7c,0x48,0x28,0xe0]
+; X86-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
 ; X86-NEXT:    vpdpwssds %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0x75,0x48,0x53,0xe2]
 ; X86-NEXT:    vpdpwssds %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xc9,0x53,0xc2]
 ; X86-NEXT:    vpaddd %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0x5d,0x48,0xfe,0xc0]
@@ -141,9 +141,9 @@ define <16 x i32>@test_int_x86_avx512_mask_vpdpwssds_512(<16 x i32> %x0, <16 x i
 ; X64-LABEL: test_int_x86_avx512_mask_vpdpwssds_512:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
-; X64-NEXT:    vmovaps %zmm0, %zmm3 # encoding: [0x62,0xf1,0x7c,0x48,0x28,0xd8]
+; X64-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
 ; X64-NEXT:    vpdpwssds (%rdi), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x49,0x53,0x1f]
-; X64-NEXT:    vmovaps %zmm0, %zmm4 # encoding: [0x62,0xf1,0x7c,0x48,0x28,0xe0]
+; X64-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
 ; X64-NEXT:    vpdpwssds %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0x75,0x48,0x53,0xe2]
 ; X64-NEXT:    vpdpwssds %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xc9,0x53,0xc2]
 ; X64-NEXT:    vpaddd %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0x5d,0x48,0xfe,0xc0]
diff --git a/llvm/test/CodeGen/X86/avx512vnni.ll b/llvm/test/CodeGen/X86/avx512vnni.ll
new file mode 100644
index 000000000..2464a3e93
--- /dev/null
+++ b/llvm/test/CodeGen/X86/avx512vnni.ll
@@ -0,0 +1,198 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -disable-peephole -mtriple=x86_64-unknown-unknown -mattr=+avx512vnni,+avx512vl,+avx512bw | FileCheck %s --check-prefixes=CHECK
+
+define <4 x i32> @test_pmaddwd_v8i16_add_v4i32(<4 x i32> %a0, <8 x i16> %a1, <8 x i16> %a2) {
+; CHECK-LABEL: test_pmaddwd_v8i16_add_v4i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpdpwssd %xmm2, %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %1 = call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> %a1, <8 x i16> %a2)
+  %2 = add <4 x i32> %1, %a0
+  ret <4 x i32> %2
+}
+
+define <4 x i32> @test_pmaddwd_v8i16_add_v4i32_commute(<4 x i32> %a0, <8 x i16> %a1, <8 x i16> %a2) {
+; CHECK-LABEL: test_pmaddwd_v8i16_add_v4i32_commute:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpdpwssd %xmm2, %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %1 = call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> %a1, <8 x i16> %a2)
+  %2 = add <4 x i32> %a0, %1
+  ret <4 x i32> %2
+}
+
+define <4 x i32> @test_pmaddwd_v8i16_add_v4i32_load1(<4 x i32> %a0, <8 x i16>* %p1, <8 x i16> %a2) {
+; CHECK-LABEL: test_pmaddwd_v8i16_add_v4i32_load1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpdpwssd (%rdi), %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %a1 = load <8 x i16>, <8 x i16>* %p1
+  %1 = call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> %a1, <8 x i16> %a2)
+  %2 = add <4 x i32> %1, %a0
+  ret <4 x i32> %2
+}
+
+define <4 x i32> @test_pmaddwd_v8i16_add_v4i32_load2(<4 x i32> %a0, <8 x i16> %a1, <8 x i16>* %p2) {
+; CHECK-LABEL: test_pmaddwd_v8i16_add_v4i32_load2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpdpwssd (%rdi), %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %a2 = load <8 x i16>, <8 x i16>* %p2
+  %1 = call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> %a1, <8 x i16> %a2)
+  %2 = add <4 x i32> %1, %a0
+  ret <4 x i32> %2
+}
+
+define <4 x i32> @test_pmaddwd_v8i16_add_v4i32_commute_load1(<4 x i32> %a0, <8 x i16>* %p1, <8 x i16> %a2) {
+; CHECK-LABEL: test_pmaddwd_v8i16_add_v4i32_commute_load1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpdpwssd (%rdi), %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %a1 = load <8 x i16>, <8 x i16>* %p1
+  %1 = call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> %a1, <8 x i16> %a2)
+  %2 = add <4 x i32> %a0, %1
+  ret <4 x i32> %2
+}
+
+define <4 x i32> @test_pmaddwd_v8i16_add_v4i32_commute_load2(<4 x i32> %a0, <8 x i16> %a1, <8 x i16>* %p2) {
+; CHECK-LABEL: test_pmaddwd_v8i16_add_v4i32_commute_load2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpdpwssd (%rdi), %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %a2 = load <8 x i16>, <8 x i16>* %p2
+  %1 = call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> %a1, <8 x i16> %a2)
+  %2 = add <4 x i32> %a0, %1
+  ret <4 x i32> %2
+}
+
+define <8 x i32> @test_pmaddwd_v16i16_add_v8i32(<8 x i32> %a0, <16 x i16> %a1, <16 x i16> %a2) {
+; CHECK-LABEL: test_pmaddwd_v16i16_add_v8i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpdpwssd %ymm2, %ymm1, %ymm0
+; CHECK-NEXT:    retq
+  %1 = call <8 x i32> @llvm.x86.avx2.pmadd.wd(<16 x i16> %a1, <16 x i16> %a2)
+  %2 = add <8 x i32> %1, %a0
+  ret <8 x i32> %2
+}
+
+define <8 x i32> @test_pmaddwd_v16i16_add_v8i32_commute(<8 x i32> %a0, <16 x i16> %a1, <16 x i16> %a2) {
+; CHECK-LABEL: test_pmaddwd_v16i16_add_v8i32_commute:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpdpwssd %ymm2, %ymm1, %ymm0
+; CHECK-NEXT:    retq
+  %1 = call <8 x i32> @llvm.x86.avx2.pmadd.wd(<16 x i16> %a1, <16 x i16> %a2)
+  %2 = add <8 x i32> %a0, %1
+  ret <8 x i32> %2
+}
+
+define <8 x i32> @test_pmaddwd_v16i16_add_v8i32_load1(<8 x i32> %a0, <16 x i16>* %p1, <16 x i16> %a2) {
+; CHECK-LABEL: test_pmaddwd_v16i16_add_v8i32_load1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpdpwssd (%rdi), %ymm1, %ymm0
+; CHECK-NEXT:    retq
+  %a1 = load <16 x i16>, <16 x i16>* %p1
+  %1 = call <8 x i32> @llvm.x86.avx2.pmadd.wd(<16 x i16> %a1, <16 x i16> %a2)
+  %2 = add <8 x i32> %1, %a0
+  ret <8 x i32> %2
+}
+
+define <8 x i32> @test_pmaddwd_v16i16_add_v8i32_load2(<8 x i32> %a0, <16 x i16> %a1, <16 x i16>* %p2) {
+; CHECK-LABEL: test_pmaddwd_v16i16_add_v8i32_load2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpdpwssd (%rdi), %ymm1, %ymm0
+; CHECK-NEXT:    retq
+  %a2 = load <16 x i16>, <16 x i16>* %p2
+  %1 = call <8 x i32> @llvm.x86.avx2.pmadd.wd(<16 x i16> %a1, <16 x i16> %a2)
+  %2 = add <8 x i32> %1, %a0
+  ret <8 x i32> %2
+}
+
+define <8 x i32> @test_pmaddwd_v16i16_add_v8i32_commute_load1(<8 x i32> %a0, <16 x i16>* %p1, <16 x i16> %a2) {
+; CHECK-LABEL: test_pmaddwd_v16i16_add_v8i32_commute_load1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpdpwssd (%rdi), %ymm1, %ymm0
+; CHECK-NEXT:    retq
+  %a1 = load <16 x i16>, <16 x i16>* %p1
+  %1 = call <8 x i32> @llvm.x86.avx2.pmadd.wd(<16 x i16> %a1, <16 x i16> %a2)
+  %2 = add <8 x i32> %a0, %1
+  ret <8 x i32> %2
+}
+
+define <8 x i32> @test_pmaddwd_v16i16_add_v8i32_commute_load2(<8 x i32> %a0, <16 x i16> %a1, <16 x i16>* %p2) {
+; CHECK-LABEL: test_pmaddwd_v16i16_add_v8i32_commute_load2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpdpwssd (%rdi), %ymm1, %ymm0
+; CHECK-NEXT:    retq
+  %a2 = load <16 x i16>, <16 x i16>* %p2
+  %1 = call <8 x i32> @llvm.x86.avx2.pmadd.wd(<16 x i16> %a1, <16 x i16> %a2)
+  %2 = add <8 x i32> %a0, %1
+  ret <8 x i32> %2
+}
+
+define <16 x i32> @test_pmaddwd_v32i16_add_v16i32(<16 x i32> %a0, <32 x i16> %a1, <32 x i16> %a2) {
+; CHECK-LABEL: test_pmaddwd_v32i16_add_v16i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpdpwssd %zmm2, %zmm1, %zmm0
+; CHECK-NEXT:    retq
+  %1 = call <16 x i32> @llvm.x86.avx512.pmaddw.d.512(<32 x i16> %a1, <32 x i16> %a2)
+  %2 = add <16 x i32> %1, %a0
+  ret <16 x i32> %2
+}
+
+define <16 x i32> @test_pmaddwd_v32i16_add_v16i32_commute(<16 x i32> %a0, <32 x i16> %a1, <32 x i16> %a2) {
+; CHECK-LABEL: test_pmaddwd_v32i16_add_v16i32_commute:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpdpwssd %zmm2, %zmm1, %zmm0
+; CHECK-NEXT:    retq
+  %1 = call <16 x i32> @llvm.x86.avx512.pmaddw.d.512(<32 x i16> %a1, <32 x i16> %a2)
+  %2 = add <16 x i32> %a0, %1
+  ret <16 x i32> %2
+}
+
+define <16 x i32> @test_pmaddwd_v32i16_add_v16i32_load1(<16 x i32> %a0, <32 x i16>* %p1, <32 x i16> %a2) {
+; CHECK-LABEL: test_pmaddwd_v32i16_add_v16i32_load1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpdpwssd (%rdi), %zmm1, %zmm0
+; CHECK-NEXT:    retq
+  %a1 = load <32 x i16>, <32 x i16>* %p1
+  %1 = call <16 x i32> @llvm.x86.avx512.pmaddw.d.512(<32 x i16> %a1, <32 x i16> %a2)
+  %2 = add <16 x i32> %1, %a0
+  ret <16 x i32> %2
+}
+
+define <16 x i32> @test_pmaddwd_v32i16_add_v16i32_load2(<16 x i32> %a0, <32 x i16> %a1, <32 x i16>* %p2) {
+; CHECK-LABEL: test_pmaddwd_v32i16_add_v16i32_load2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpdpwssd (%rdi), %zmm1, %zmm0
+; CHECK-NEXT:    retq
+  %a2 = load <32 x i16>, <32 x i16>* %p2
+  %1 = call <16 x i32> @llvm.x86.avx512.pmaddw.d.512(<32 x i16> %a1, <32 x i16> %a2)
+  %2 = add <16 x i32> %1, %a0
+  ret <16 x i32> %2
+}
+
+define <16 x i32> @test_pmaddwd_v32i16_add_v16i32_commute_load1(<16 x i32> %a0, <32 x i16>* %p1, <32 x i16> %a2) {
+; CHECK-LABEL: test_pmaddwd_v32i16_add_v16i32_commute_load1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpdpwssd (%rdi), %zmm1, %zmm0
+; CHECK-NEXT:    retq
+  %a1 = load <32 x i16>, <32 x i16>* %p1
+  %1 = call <16 x i32> @llvm.x86.avx512.pmaddw.d.512(<32 x i16> %a1, <32 x i16> %a2)
+  %2 = add <16 x i32> %a0, %1
+  ret <16 x i32> %2
+}
+
+define <16 x i32> @test_pmaddwd_v32i16_add_v16i32_commute_load2(<16 x i32> %a0, <32 x i16> %a1, <32 x i16>* %p2) {
+; CHECK-LABEL: test_pmaddwd_v32i16_add_v16i32_commute_load2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpdpwssd (%rdi), %zmm1, %zmm0
+; CHECK-NEXT:    retq
+  %a2 = load <32 x i16>, <32 x i16>* %p2
+  %1 = call <16 x i32> @llvm.x86.avx512.pmaddw.d.512(<32 x i16> %a1, <32 x i16> %a2)
+  %2 = add <16 x i32> %a0, %1
+  ret <16 x i32> %2
+}
+
+declare <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16>, <8 x i16>)
+declare <8 x i32> @llvm.x86.avx2.pmadd.wd(<16 x i16>, <16 x i16>)
+declare <16 x i32> @llvm.x86.avx512.pmaddw.d.512(<32 x i16>, <32 x i16>)
diff --git a/llvm/test/CodeGen/X86/avx512vp2intersect-intrinsics.ll b/llvm/test/CodeGen/X86/avx512vp2intersect-intrinsics.ll
index 3e37c7c5b..7e7a46db7 100644
--- a/llvm/test/CodeGen/X86/avx512vp2intersect-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/avx512vp2intersect-intrinsics.ll
@@ -199,8 +199,7 @@ define void @test_mm512_2intersect_epi64_b(i64* nocapture readonly %a, i64* noca
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx # encoding: [0x8b,0x4c,0x24,0x08]
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx # encoding: [0x8b,0x54,0x24,0x04]
 ; X86-NEXT:    vbroadcastsd (%edx), %zmm0 # encoding: [0x62,0xf2,0xfd,0x48,0x19,0x02]
-; X86-NEXT:    vbroadcastsd (%ecx), %zmm1 # encoding: [0x62,0xf2,0xfd,0x48,0x19,0x09]
-; X86-NEXT:    vp2intersectq %zmm1, %zmm0, %k0 # encoding: [0x62,0xf2,0xff,0x48,0x68,0xc1]
+; X86-NEXT:    vp2intersectq (%ecx){1to8}, %zmm0, %k0 # encoding: [0x62,0xf2,0xff,0x58,0x68,0x01]
 ; X86-NEXT:    kmovw %k1, %ecx # encoding: [0xc5,0xf8,0x93,0xc9]
 ; X86-NEXT:    kmovw %k0, %edx # encoding: [0xc5,0xf8,0x93,0xd0]
 ; X86-NEXT:    movb %dl, (%eax) # encoding: [0x88,0x10]
diff --git a/llvm/test/CodeGen/X86/bit-piece-comment.ll b/llvm/test/CodeGen/X86/bit-piece-comment.ll
index a70b5253e..ee5eaf86a 100644
--- a/llvm/test/CodeGen/X86/bit-piece-comment.ll
+++ b/llvm/test/CodeGen/X86/bit-piece-comment.ll
@@ -32,7 +32,7 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.value(metadata, i64, metadata, metadata) #1
 
-attributes #0 = { norecurse nounwind optsize readnone uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { norecurse nounwind optsize readnone uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/CodeGen/X86/bit-test-shift.ll b/llvm/test/CodeGen/X86/bit-test-shift.ll
index 8970db402..b45f5f876 100644
--- a/llvm/test/CodeGen/X86/bit-test-shift.ll
+++ b/llvm/test/CodeGen/X86/bit-test-shift.ll
@@ -1,11 +1,16 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=i686-- | FileCheck %s
 ; <rdar://problem/8285015>
 
 define i32 @x(i32 %t) nounwind readnone ssp {
+; CHECK-LABEL: x:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    shll $23, %eax
+; CHECK-NEXT:    sarl $31, %eax
+; CHECK-NEXT:    andl $-26, %eax
+; CHECK-NEXT:    retl
 entry:
-; CHECK: shll	$23, %eax
-; CHECK: sarl	$31, %eax
-; CHECK: andl	$-26, %eax
   %and = and i32 %t, 256
   %tobool = icmp eq i32 %and, 0
   %retval.0 = select i1 %tobool, i32 0, i32 -26
diff --git a/llvm/test/CodeGen/X86/bitcast-and-setcc-128.ll b/llvm/test/CodeGen/X86/bitcast-and-setcc-128.ll
index 289ddcb19..7d5886d7e 100644
--- a/llvm/test/CodeGen/X86/bitcast-and-setcc-128.ll
+++ b/llvm/test/CodeGen/X86/bitcast-and-setcc-128.ll
@@ -178,144 +178,63 @@ define i16 @v16i8(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c, <16 x i8> %d) {
 }
 
 define i2 @v2i8(<2 x i8> %a, <2 x i8> %b, <2 x i8> %c, <2 x i8> %d) {
-; SSE2-SSSE3-LABEL: v2i8:
-; SSE2-SSSE3:       # %bb.0:
-; SSE2-SSSE3-NEXT:    psllq $56, %xmm2
-; SSE2-SSSE3-NEXT:    movdqa %xmm2, %xmm4
-; SSE2-SSSE3-NEXT:    psrad $31, %xmm4
-; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,3,2,3]
-; SSE2-SSSE3-NEXT:    psrad $24, %xmm2
-; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
-; SSE2-SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
-; SSE2-SSSE3-NEXT:    psllq $56, %xmm3
-; SSE2-SSSE3-NEXT:    movdqa %xmm3, %xmm4
-; SSE2-SSSE3-NEXT:    psrad $31, %xmm4
-; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,3,2,3]
-; SSE2-SSSE3-NEXT:    psrad $24, %xmm3
-; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,3,2,3]
-; SSE2-SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
-; SSE2-SSSE3-NEXT:    psllq $56, %xmm0
-; SSE2-SSSE3-NEXT:    movdqa %xmm0, %xmm4
-; SSE2-SSSE3-NEXT:    psrad $31, %xmm4
-; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,3,2,3]
-; SSE2-SSSE3-NEXT:    psrad $24, %xmm0
-; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
-; SSE2-SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
-; SSE2-SSSE3-NEXT:    psllq $56, %xmm1
-; SSE2-SSSE3-NEXT:    movdqa %xmm1, %xmm4
-; SSE2-SSSE3-NEXT:    psrad $31, %xmm4
-; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,3,2,3]
-; SSE2-SSSE3-NEXT:    psrad $24, %xmm1
-; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
-; SSE2-SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
-; SSE2-SSSE3-NEXT:    movdqa {{.*#+}} xmm4 = [2147483648,2147483648]
-; SSE2-SSSE3-NEXT:    pxor %xmm4, %xmm1
-; SSE2-SSSE3-NEXT:    pxor %xmm4, %xmm0
-; SSE2-SSSE3-NEXT:    movdqa %xmm0, %xmm5
-; SSE2-SSSE3-NEXT:    pcmpeqd %xmm1, %xmm5
-; SSE2-SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE2-SSSE3-NEXT:    pand %xmm5, %xmm1
-; SSE2-SSSE3-NEXT:    por %xmm0, %xmm1
-; SSE2-SSSE3-NEXT:    pxor %xmm4, %xmm3
-; SSE2-SSSE3-NEXT:    pxor %xmm4, %xmm2
-; SSE2-SSSE3-NEXT:    movdqa %xmm2, %xmm0
-; SSE2-SSSE3-NEXT:    pcmpeqd %xmm3, %xmm0
-; SSE2-SSSE3-NEXT:    pcmpgtd %xmm3, %xmm2
-; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
-; SSE2-SSSE3-NEXT:    pand %xmm0, %xmm3
-; SSE2-SSSE3-NEXT:    por %xmm2, %xmm3
-; SSE2-SSSE3-NEXT:    pand %xmm1, %xmm3
-; SSE2-SSSE3-NEXT:    movmskpd %xmm3, %eax
-; SSE2-SSSE3-NEXT:    # kill: def $al killed $al killed $eax
-; SSE2-SSSE3-NEXT:    retq
-;
-; AVX1-LABEL: v2i8:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpsllq $56, %xmm3, %xmm3
-; AVX1-NEXT:    vpsrad $31, %xmm3, %xmm4
-; AVX1-NEXT:    vpsrad $24, %xmm3, %xmm3
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1],xmm4[2,3],xmm3[4,5],xmm4[6,7]
-; AVX1-NEXT:    vpsllq $56, %xmm2, %xmm2
-; AVX1-NEXT:    vpsrad $31, %xmm2, %xmm4
-; AVX1-NEXT:    vpsrad $24, %xmm2, %xmm2
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1],xmm4[2,3],xmm2[4,5],xmm4[6,7]
-; AVX1-NEXT:    vpcmpgtq %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpsllq $56, %xmm1, %xmm1
-; AVX1-NEXT:    vpsrad $31, %xmm1, %xmm3
-; AVX1-NEXT:    vpsrad $24, %xmm1, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5],xmm3[6,7]
-; AVX1-NEXT:    vpsllq $56, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrad $31, %xmm0, %xmm3
-; AVX1-NEXT:    vpsrad $24, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm3[2,3],xmm0[4,5],xmm3[6,7]
-; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vmovmskpd %xmm0, %eax
-; AVX1-NEXT:    # kill: def $al killed $al killed $eax
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: v2i8:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpsllq $56, %xmm3, %xmm3
-; AVX2-NEXT:    vpsrad $31, %xmm3, %xmm4
-; AVX2-NEXT:    vpsrad $24, %xmm3, %xmm3
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm3 = xmm3[0],xmm4[1],xmm3[2],xmm4[3]
-; AVX2-NEXT:    vpsllq $56, %xmm2, %xmm2
-; AVX2-NEXT:    vpsrad $31, %xmm2, %xmm4
-; AVX2-NEXT:    vpsrad $24, %xmm2, %xmm2
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm2 = xmm2[0],xmm4[1],xmm2[2],xmm4[3]
-; AVX2-NEXT:    vpcmpgtq %xmm3, %xmm2, %xmm2
-; AVX2-NEXT:    vpsllq $56, %xmm1, %xmm1
-; AVX2-NEXT:    vpsrad $31, %xmm1, %xmm3
-; AVX2-NEXT:    vpsrad $24, %xmm1, %xmm1
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm3[1],xmm1[2],xmm3[3]
-; AVX2-NEXT:    vpsllq $56, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrad $31, %xmm0, %xmm3
-; AVX2-NEXT:    vpsrad $24, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm3[1],xmm0[2],xmm3[3]
-; AVX2-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; AVX2-NEXT:    vmovmskpd %xmm0, %eax
-; AVX2-NEXT:    # kill: def $al killed $al killed $eax
-; AVX2-NEXT:    retq
+; SSE2-LABEL: v2i8:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pcmpgtb %xmm1, %xmm0
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,1,1]
+; SSE2-NEXT:    pcmpgtb %xmm3, %xmm2
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,1,1]
+; SSE2-NEXT:    pand %xmm0, %xmm1
+; SSE2-NEXT:    movmskpd %xmm1, %eax
+; SSE2-NEXT:    # kill: def $al killed $al killed $eax
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: v2i8:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    pcmpgtb %xmm1, %xmm0
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = <u,u,u,0,u,u,u,0,u,u,u,1,u,u,u,1>
+; SSSE3-NEXT:    pshufb %xmm1, %xmm0
+; SSSE3-NEXT:    pcmpgtb %xmm3, %xmm2
+; SSSE3-NEXT:    pshufb %xmm1, %xmm2
+; SSSE3-NEXT:    pand %xmm0, %xmm2
+; SSSE3-NEXT:    movmskpd %xmm2, %eax
+; SSSE3-NEXT:    # kill: def $al killed $al killed $eax
+; SSSE3-NEXT:    retq
+;
+; AVX12-LABEL: v2i8:
+; AVX12:       # %bb.0:
+; AVX12-NEXT:    vpcmpgtb %xmm1, %xmm0, %xmm0
+; AVX12-NEXT:    vpmovsxbq %xmm0, %xmm0
+; AVX12-NEXT:    vpcmpgtb %xmm3, %xmm2, %xmm1
+; AVX12-NEXT:    vpmovsxbq %xmm1, %xmm1
+; AVX12-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; AVX12-NEXT:    vmovmskpd %xmm0, %eax
+; AVX12-NEXT:    # kill: def $al killed $al killed $eax
+; AVX12-NEXT:    retq
 ;
 ; AVX512F-LABEL: v2i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpsllq $56, %xmm3, %xmm3
-; AVX512F-NEXT:    vpsraq $56, %xmm3, %xmm3
-; AVX512F-NEXT:    vpsllq $56, %xmm2, %xmm2
-; AVX512F-NEXT:    vpsraq $56, %xmm2, %xmm2
-; AVX512F-NEXT:    vpsllq $56, %xmm1, %xmm1
-; AVX512F-NEXT:    vpsraq $56, %xmm1, %xmm1
-; AVX512F-NEXT:    vpsllq $56, %xmm0, %xmm0
-; AVX512F-NEXT:    vpsraq $56, %xmm0, %xmm0
-; AVX512F-NEXT:    vpcmpgtq %xmm1, %xmm0, %k1
-; AVX512F-NEXT:    vpcmpgtq %xmm3, %xmm2, %k0 {%k1}
+; AVX512F-NEXT:    vpcmpgtb %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    vpmovsxbd %xmm0, %zmm0
+; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k0
+; AVX512F-NEXT:    vpcmpgtb %xmm3, %xmm2, %xmm0
+; AVX512F-NEXT:    vpmovsxbd %xmm0, %zmm0
+; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k1
+; AVX512F-NEXT:    kandw %k1, %k0, %k0
 ; AVX512F-NEXT:    kmovw %k0, %eax
 ; AVX512F-NEXT:    # kill: def $al killed $al killed $eax
+; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: v2i8:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpsllq $56, %xmm3, %xmm3
-; AVX512BW-NEXT:    vpsraq $56, %xmm3, %xmm3
-; AVX512BW-NEXT:    vpsllq $56, %xmm2, %xmm2
-; AVX512BW-NEXT:    vpsraq $56, %xmm2, %xmm2
-; AVX512BW-NEXT:    vpsllq $56, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpsraq $56, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpsllq $56, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpsraq $56, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpcmpgtq %xmm1, %xmm0, %k1
-; AVX512BW-NEXT:    vpcmpgtq %xmm3, %xmm2, %k0 {%k1}
+; AVX512BW-NEXT:    vpcmpgtb %xmm1, %xmm0, %k0
+; AVX512BW-NEXT:    vpcmpgtb %xmm3, %xmm2, %k1
+; AVX512BW-NEXT:    kandw %k1, %k0, %k0
 ; AVX512BW-NEXT:    kmovd %k0, %eax
 ; AVX512BW-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512BW-NEXT:    retq
@@ -329,142 +248,47 @@ define i2 @v2i8(<2 x i8> %a, <2 x i8> %b, <2 x i8> %c, <2 x i8> %d) {
 define i2 @v2i16(<2 x i16> %a, <2 x i16> %b, <2 x i16> %c, <2 x i16> %d) {
 ; SSE2-SSSE3-LABEL: v2i16:
 ; SSE2-SSSE3:       # %bb.0:
-; SSE2-SSSE3-NEXT:    psllq $48, %xmm2
-; SSE2-SSSE3-NEXT:    movdqa %xmm2, %xmm4
-; SSE2-SSSE3-NEXT:    psrad $31, %xmm4
-; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,3,2,3]
-; SSE2-SSSE3-NEXT:    psrad $16, %xmm2
-; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
-; SSE2-SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
-; SSE2-SSSE3-NEXT:    psllq $48, %xmm3
-; SSE2-SSSE3-NEXT:    movdqa %xmm3, %xmm4
-; SSE2-SSSE3-NEXT:    psrad $31, %xmm4
-; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,3,2,3]
-; SSE2-SSSE3-NEXT:    psrad $16, %xmm3
-; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,3,2,3]
-; SSE2-SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
-; SSE2-SSSE3-NEXT:    psllq $48, %xmm0
-; SSE2-SSSE3-NEXT:    movdqa %xmm0, %xmm4
-; SSE2-SSSE3-NEXT:    psrad $31, %xmm4
-; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,3,2,3]
-; SSE2-SSSE3-NEXT:    psrad $16, %xmm0
-; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
-; SSE2-SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
-; SSE2-SSSE3-NEXT:    psllq $48, %xmm1
-; SSE2-SSSE3-NEXT:    movdqa %xmm1, %xmm4
-; SSE2-SSSE3-NEXT:    psrad $31, %xmm4
-; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,3,2,3]
-; SSE2-SSSE3-NEXT:    psrad $16, %xmm1
-; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
-; SSE2-SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
-; SSE2-SSSE3-NEXT:    movdqa {{.*#+}} xmm4 = [2147483648,2147483648]
-; SSE2-SSSE3-NEXT:    pxor %xmm4, %xmm1
-; SSE2-SSSE3-NEXT:    pxor %xmm4, %xmm0
-; SSE2-SSSE3-NEXT:    movdqa %xmm0, %xmm5
-; SSE2-SSSE3-NEXT:    pcmpeqd %xmm1, %xmm5
-; SSE2-SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE2-SSSE3-NEXT:    pand %xmm5, %xmm1
-; SSE2-SSSE3-NEXT:    por %xmm0, %xmm1
-; SSE2-SSSE3-NEXT:    pxor %xmm4, %xmm3
-; SSE2-SSSE3-NEXT:    pxor %xmm4, %xmm2
-; SSE2-SSSE3-NEXT:    movdqa %xmm2, %xmm0
-; SSE2-SSSE3-NEXT:    pcmpeqd %xmm3, %xmm0
-; SSE2-SSSE3-NEXT:    pcmpgtd %xmm3, %xmm2
-; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
-; SSE2-SSSE3-NEXT:    pand %xmm0, %xmm3
-; SSE2-SSSE3-NEXT:    por %xmm2, %xmm3
-; SSE2-SSSE3-NEXT:    pand %xmm1, %xmm3
-; SSE2-SSSE3-NEXT:    movmskpd %xmm3, %eax
+; SSE2-SSSE3-NEXT:    pcmpgtw %xmm1, %xmm0
+; SSE2-SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,1,1]
+; SSE2-SSSE3-NEXT:    pcmpgtw %xmm3, %xmm2
+; SSE2-SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,1,1]
+; SSE2-SSSE3-NEXT:    pand %xmm0, %xmm1
+; SSE2-SSSE3-NEXT:    movmskpd %xmm1, %eax
 ; SSE2-SSSE3-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE2-SSSE3-NEXT:    retq
 ;
-; AVX1-LABEL: v2i16:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpsllq $48, %xmm3, %xmm3
-; AVX1-NEXT:    vpsrad $31, %xmm3, %xmm4
-; AVX1-NEXT:    vpsrad $16, %xmm3, %xmm3
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1],xmm4[2,3],xmm3[4,5],xmm4[6,7]
-; AVX1-NEXT:    vpsllq $48, %xmm2, %xmm2
-; AVX1-NEXT:    vpsrad $31, %xmm2, %xmm4
-; AVX1-NEXT:    vpsrad $16, %xmm2, %xmm2
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1],xmm4[2,3],xmm2[4,5],xmm4[6,7]
-; AVX1-NEXT:    vpcmpgtq %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpsllq $48, %xmm1, %xmm1
-; AVX1-NEXT:    vpsrad $31, %xmm1, %xmm3
-; AVX1-NEXT:    vpsrad $16, %xmm1, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5],xmm3[6,7]
-; AVX1-NEXT:    vpsllq $48, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrad $31, %xmm0, %xmm3
-; AVX1-NEXT:    vpsrad $16, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm3[2,3],xmm0[4,5],xmm3[6,7]
-; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vmovmskpd %xmm0, %eax
-; AVX1-NEXT:    # kill: def $al killed $al killed $eax
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: v2i16:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpsllq $48, %xmm3, %xmm3
-; AVX2-NEXT:    vpsrad $31, %xmm3, %xmm4
-; AVX2-NEXT:    vpsrad $16, %xmm3, %xmm3
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm3 = xmm3[0],xmm4[1],xmm3[2],xmm4[3]
-; AVX2-NEXT:    vpsllq $48, %xmm2, %xmm2
-; AVX2-NEXT:    vpsrad $31, %xmm2, %xmm4
-; AVX2-NEXT:    vpsrad $16, %xmm2, %xmm2
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm2 = xmm2[0],xmm4[1],xmm2[2],xmm4[3]
-; AVX2-NEXT:    vpcmpgtq %xmm3, %xmm2, %xmm2
-; AVX2-NEXT:    vpsllq $48, %xmm1, %xmm1
-; AVX2-NEXT:    vpsrad $31, %xmm1, %xmm3
-; AVX2-NEXT:    vpsrad $16, %xmm1, %xmm1
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm3[1],xmm1[2],xmm3[3]
-; AVX2-NEXT:    vpsllq $48, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrad $31, %xmm0, %xmm3
-; AVX2-NEXT:    vpsrad $16, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm3[1],xmm0[2],xmm3[3]
-; AVX2-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; AVX2-NEXT:    vmovmskpd %xmm0, %eax
-; AVX2-NEXT:    # kill: def $al killed $al killed $eax
-; AVX2-NEXT:    retq
+; AVX12-LABEL: v2i16:
+; AVX12:       # %bb.0:
+; AVX12-NEXT:    vpcmpgtw %xmm1, %xmm0, %xmm0
+; AVX12-NEXT:    vpmovsxwq %xmm0, %xmm0
+; AVX12-NEXT:    vpcmpgtw %xmm3, %xmm2, %xmm1
+; AVX12-NEXT:    vpmovsxwq %xmm1, %xmm1
+; AVX12-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; AVX12-NEXT:    vmovmskpd %xmm0, %eax
+; AVX12-NEXT:    # kill: def $al killed $al killed $eax
+; AVX12-NEXT:    retq
 ;
 ; AVX512F-LABEL: v2i16:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpsllq $48, %xmm3, %xmm3
-; AVX512F-NEXT:    vpsraq $48, %xmm3, %xmm3
-; AVX512F-NEXT:    vpsllq $48, %xmm2, %xmm2
-; AVX512F-NEXT:    vpsraq $48, %xmm2, %xmm2
-; AVX512F-NEXT:    vpsllq $48, %xmm1, %xmm1
-; AVX512F-NEXT:    vpsraq $48, %xmm1, %xmm1
-; AVX512F-NEXT:    vpsllq $48, %xmm0, %xmm0
-; AVX512F-NEXT:    vpsraq $48, %xmm0, %xmm0
-; AVX512F-NEXT:    vpcmpgtq %xmm1, %xmm0, %k1
-; AVX512F-NEXT:    vpcmpgtq %xmm3, %xmm2, %k0 {%k1}
+; AVX512F-NEXT:    vpcmpgtw %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    vpmovsxwd %xmm0, %ymm0
+; AVX512F-NEXT:    vptestmd %ymm0, %ymm0, %k0
+; AVX512F-NEXT:    vpcmpgtw %xmm3, %xmm2, %xmm0
+; AVX512F-NEXT:    vpmovsxwd %xmm0, %ymm0
+; AVX512F-NEXT:    vptestmd %ymm0, %ymm0, %k1
+; AVX512F-NEXT:    kandw %k1, %k0, %k0
 ; AVX512F-NEXT:    kmovw %k0, %eax
 ; AVX512F-NEXT:    # kill: def $al killed $al killed $eax
+; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: v2i16:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpsllq $48, %xmm3, %xmm3
-; AVX512BW-NEXT:    vpsraq $48, %xmm3, %xmm3
-; AVX512BW-NEXT:    vpsllq $48, %xmm2, %xmm2
-; AVX512BW-NEXT:    vpsraq $48, %xmm2, %xmm2
-; AVX512BW-NEXT:    vpsllq $48, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpsraq $48, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpsllq $48, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpsraq $48, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpcmpgtq %xmm1, %xmm0, %k1
-; AVX512BW-NEXT:    vpcmpgtq %xmm3, %xmm2, %k0 {%k1}
+; AVX512BW-NEXT:    vpcmpgtw %xmm1, %xmm0, %k0
+; AVX512BW-NEXT:    vpcmpgtw %xmm3, %xmm2, %k1
+; AVX512BW-NEXT:    kandw %k1, %k0, %k0
 ; AVX512BW-NEXT:    kmovd %k0, %eax
 ; AVX512BW-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512BW-NEXT:    retq
@@ -478,118 +302,40 @@ define i2 @v2i16(<2 x i16> %a, <2 x i16> %b, <2 x i16> %c, <2 x i16> %d) {
 define i2 @v2i32(<2 x i32> %a, <2 x i32> %b, <2 x i32> %c, <2 x i32> %d) {
 ; SSE2-SSSE3-LABEL: v2i32:
 ; SSE2-SSSE3:       # %bb.0:
-; SSE2-SSSE3-NEXT:    psllq $32, %xmm2
-; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,3,2,3]
-; SSE2-SSSE3-NEXT:    psrad $31, %xmm2
-; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
-; SSE2-SSSE3-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
-; SSE2-SSSE3-NEXT:    psllq $32, %xmm3
-; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,3,2,3]
-; SSE2-SSSE3-NEXT:    psrad $31, %xmm3
-; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,3,2,3]
-; SSE2-SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; SSE2-SSSE3-NEXT:    psllq $32, %xmm0
-; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,3,2,3]
-; SSE2-SSSE3-NEXT:    psrad $31, %xmm0
-; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
-; SSE2-SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1]
-; SSE2-SSSE3-NEXT:    psllq $32, %xmm1
-; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,3,2,3]
-; SSE2-SSSE3-NEXT:    psrad $31, %xmm1
-; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
-; SSE2-SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; SSE2-SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [2147483648,2147483648]
-; SSE2-SSSE3-NEXT:    pxor %xmm1, %xmm0
-; SSE2-SSSE3-NEXT:    pxor %xmm1, %xmm3
-; SSE2-SSSE3-NEXT:    movdqa %xmm3, %xmm5
-; SSE2-SSSE3-NEXT:    pcmpeqd %xmm0, %xmm5
-; SSE2-SSSE3-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
-; SSE2-SSSE3-NEXT:    pand %xmm5, %xmm0
-; SSE2-SSSE3-NEXT:    por %xmm3, %xmm0
-; SSE2-SSSE3-NEXT:    pxor %xmm1, %xmm2
-; SSE2-SSSE3-NEXT:    pxor %xmm1, %xmm4
-; SSE2-SSSE3-NEXT:    movdqa %xmm4, %xmm1
-; SSE2-SSSE3-NEXT:    pcmpeqd %xmm2, %xmm1
-; SSE2-SSSE3-NEXT:    pcmpgtd %xmm2, %xmm4
-; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[0,0,2,2]
-; SSE2-SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSE2-SSSE3-NEXT:    por %xmm4, %xmm2
-; SSE2-SSSE3-NEXT:    pand %xmm0, %xmm2
-; SSE2-SSSE3-NEXT:    movmskpd %xmm2, %eax
+; SSE2-SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
+; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,1,1]
+; SSE2-SSSE3-NEXT:    pcmpgtd %xmm3, %xmm2
+; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,0,1,1]
+; SSE2-SSSE3-NEXT:    pand %xmm0, %xmm1
+; SSE2-SSSE3-NEXT:    movmskpd %xmm1, %eax
 ; SSE2-SSSE3-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE2-SSSE3-NEXT:    retq
 ;
-; AVX1-LABEL: v2i32:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpsllq $32, %xmm3, %xmm4
-; AVX1-NEXT:    vpsrad $31, %xmm4, %xmm4
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1],xmm4[2,3],xmm3[4,5],xmm4[6,7]
-; AVX1-NEXT:    vpsllq $32, %xmm2, %xmm4
-; AVX1-NEXT:    vpsrad $31, %xmm4, %xmm4
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1],xmm4[2,3],xmm2[4,5],xmm4[6,7]
-; AVX1-NEXT:    vpcmpgtq %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpsllq $32, %xmm1, %xmm3
-; AVX1-NEXT:    vpsrad $31, %xmm3, %xmm3
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5],xmm3[6,7]
-; AVX1-NEXT:    vpsllq $32, %xmm0, %xmm3
-; AVX1-NEXT:    vpsrad $31, %xmm3, %xmm3
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm3[2,3],xmm0[4,5],xmm3[6,7]
-; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vmovmskpd %xmm0, %eax
-; AVX1-NEXT:    # kill: def $al killed $al killed $eax
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: v2i32:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpsllq $32, %xmm3, %xmm4
-; AVX2-NEXT:    vpsrad $31, %xmm4, %xmm4
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm3 = xmm3[0],xmm4[1],xmm3[2],xmm4[3]
-; AVX2-NEXT:    vpsllq $32, %xmm2, %xmm4
-; AVX2-NEXT:    vpsrad $31, %xmm4, %xmm4
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm2 = xmm2[0],xmm4[1],xmm2[2],xmm4[3]
-; AVX2-NEXT:    vpcmpgtq %xmm3, %xmm2, %xmm2
-; AVX2-NEXT:    vpsllq $32, %xmm1, %xmm3
-; AVX2-NEXT:    vpsrad $31, %xmm3, %xmm3
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm3[1],xmm1[2],xmm3[3]
-; AVX2-NEXT:    vpsllq $32, %xmm0, %xmm3
-; AVX2-NEXT:    vpsrad $31, %xmm3, %xmm3
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm3[1],xmm0[2],xmm3[3]
-; AVX2-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; AVX2-NEXT:    vmovmskpd %xmm0, %eax
-; AVX2-NEXT:    # kill: def $al killed $al killed $eax
-; AVX2-NEXT:    retq
+; AVX12-LABEL: v2i32:
+; AVX12:       # %bb.0:
+; AVX12-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm0
+; AVX12-NEXT:    vpmovsxdq %xmm0, %xmm0
+; AVX12-NEXT:    vpcmpgtd %xmm3, %xmm2, %xmm1
+; AVX12-NEXT:    vpmovsxdq %xmm1, %xmm1
+; AVX12-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; AVX12-NEXT:    vmovmskpd %xmm0, %eax
+; AVX12-NEXT:    # kill: def $al killed $al killed $eax
+; AVX12-NEXT:    retq
 ;
 ; AVX512F-LABEL: v2i32:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpsllq $32, %xmm3, %xmm3
-; AVX512F-NEXT:    vpsraq $32, %xmm3, %xmm3
-; AVX512F-NEXT:    vpsllq $32, %xmm2, %xmm2
-; AVX512F-NEXT:    vpsraq $32, %xmm2, %xmm2
-; AVX512F-NEXT:    vpsllq $32, %xmm1, %xmm1
-; AVX512F-NEXT:    vpsraq $32, %xmm1, %xmm1
-; AVX512F-NEXT:    vpsllq $32, %xmm0, %xmm0
-; AVX512F-NEXT:    vpsraq $32, %xmm0, %xmm0
-; AVX512F-NEXT:    vpcmpgtq %xmm1, %xmm0, %k1
-; AVX512F-NEXT:    vpcmpgtq %xmm3, %xmm2, %k0 {%k1}
+; AVX512F-NEXT:    vpcmpgtd %xmm1, %xmm0, %k0
+; AVX512F-NEXT:    vpcmpgtd %xmm3, %xmm2, %k1
+; AVX512F-NEXT:    kandw %k1, %k0, %k0
 ; AVX512F-NEXT:    kmovw %k0, %eax
 ; AVX512F-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: v2i32:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpsllq $32, %xmm3, %xmm3
-; AVX512BW-NEXT:    vpsraq $32, %xmm3, %xmm3
-; AVX512BW-NEXT:    vpsllq $32, %xmm2, %xmm2
-; AVX512BW-NEXT:    vpsraq $32, %xmm2, %xmm2
-; AVX512BW-NEXT:    vpsllq $32, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpsraq $32, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpsllq $32, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpsraq $32, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpcmpgtq %xmm1, %xmm0, %k1
-; AVX512BW-NEXT:    vpcmpgtq %xmm3, %xmm2, %k0 {%k1}
+; AVX512BW-NEXT:    vpcmpgtd %xmm1, %xmm0, %k0
+; AVX512BW-NEXT:    vpcmpgtd %xmm3, %xmm2, %k1
+; AVX512BW-NEXT:    kandw %k1, %k0, %k0
 ; AVX512BW-NEXT:    kmovd %k0, %eax
 ; AVX512BW-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512BW-NEXT:    retq
@@ -700,66 +446,47 @@ define i2 @v2f64(<2 x double> %a, <2 x double> %b, <2 x double> %c, <2 x double>
 define i4 @v4i8(<4 x i8> %a, <4 x i8> %b, <4 x i8> %c, <4 x i8> %d) {
 ; SSE2-SSSE3-LABEL: v4i8:
 ; SSE2-SSSE3:       # %bb.0:
-; SSE2-SSSE3-NEXT:    pslld $24, %xmm3
-; SSE2-SSSE3-NEXT:    psrad $24, %xmm3
-; SSE2-SSSE3-NEXT:    pslld $24, %xmm2
-; SSE2-SSSE3-NEXT:    psrad $24, %xmm2
-; SSE2-SSSE3-NEXT:    pcmpgtd %xmm3, %xmm2
-; SSE2-SSSE3-NEXT:    pslld $24, %xmm1
-; SSE2-SSSE3-NEXT:    psrad $24, %xmm1
-; SSE2-SSSE3-NEXT:    pslld $24, %xmm0
-; SSE2-SSSE3-NEXT:    psrad $24, %xmm0
-; SSE2-SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-SSSE3-NEXT:    pand %xmm2, %xmm0
-; SSE2-SSSE3-NEXT:    movmskps %xmm0, %eax
+; SSE2-SSSE3-NEXT:    pcmpgtb %xmm1, %xmm0
+; SSE2-SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSE2-SSSE3-NEXT:    pcmpgtb %xmm3, %xmm2
+; SSE2-SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; SSE2-SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]
+; SSE2-SSSE3-NEXT:    pand %xmm0, %xmm1
+; SSE2-SSSE3-NEXT:    movmskps %xmm1, %eax
 ; SSE2-SSSE3-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE2-SSSE3-NEXT:    retq
 ;
 ; AVX12-LABEL: v4i8:
 ; AVX12:       # %bb.0:
-; AVX12-NEXT:    vpslld $24, %xmm3, %xmm3
-; AVX12-NEXT:    vpsrad $24, %xmm3, %xmm3
-; AVX12-NEXT:    vpslld $24, %xmm2, %xmm2
-; AVX12-NEXT:    vpsrad $24, %xmm2, %xmm2
-; AVX12-NEXT:    vpcmpgtd %xmm3, %xmm2, %xmm2
-; AVX12-NEXT:    vpslld $24, %xmm1, %xmm1
-; AVX12-NEXT:    vpsrad $24, %xmm1, %xmm1
-; AVX12-NEXT:    vpslld $24, %xmm0, %xmm0
-; AVX12-NEXT:    vpsrad $24, %xmm0, %xmm0
-; AVX12-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm0
-; AVX12-NEXT:    vpand %xmm2, %xmm0, %xmm0
+; AVX12-NEXT:    vpcmpgtb %xmm1, %xmm0, %xmm0
+; AVX12-NEXT:    vpmovsxbd %xmm0, %xmm0
+; AVX12-NEXT:    vpcmpgtb %xmm3, %xmm2, %xmm1
+; AVX12-NEXT:    vpmovsxbd %xmm1, %xmm1
+; AVX12-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX12-NEXT:    vmovmskps %xmm0, %eax
 ; AVX12-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX12-NEXT:    retq
 ;
 ; AVX512F-LABEL: v4i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpslld $24, %xmm3, %xmm3
-; AVX512F-NEXT:    vpsrad $24, %xmm3, %xmm3
-; AVX512F-NEXT:    vpslld $24, %xmm2, %xmm2
-; AVX512F-NEXT:    vpsrad $24, %xmm2, %xmm2
-; AVX512F-NEXT:    vpslld $24, %xmm1, %xmm1
-; AVX512F-NEXT:    vpsrad $24, %xmm1, %xmm1
-; AVX512F-NEXT:    vpslld $24, %xmm0, %xmm0
-; AVX512F-NEXT:    vpsrad $24, %xmm0, %xmm0
-; AVX512F-NEXT:    vpcmpgtd %xmm1, %xmm0, %k1
-; AVX512F-NEXT:    vpcmpgtd %xmm3, %xmm2, %k0 {%k1}
+; AVX512F-NEXT:    vpcmpgtb %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    vpmovsxbd %xmm0, %zmm0
+; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k0
+; AVX512F-NEXT:    vpcmpgtb %xmm3, %xmm2, %xmm0
+; AVX512F-NEXT:    vpmovsxbd %xmm0, %zmm0
+; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k1
+; AVX512F-NEXT:    kandw %k1, %k0, %k0
 ; AVX512F-NEXT:    kmovw %k0, %eax
 ; AVX512F-NEXT:    # kill: def $al killed $al killed $eax
+; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: v4i8:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpslld $24, %xmm3, %xmm3
-; AVX512BW-NEXT:    vpsrad $24, %xmm3, %xmm3
-; AVX512BW-NEXT:    vpslld $24, %xmm2, %xmm2
-; AVX512BW-NEXT:    vpsrad $24, %xmm2, %xmm2
-; AVX512BW-NEXT:    vpslld $24, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpsrad $24, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpslld $24, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpsrad $24, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpcmpgtd %xmm1, %xmm0, %k1
-; AVX512BW-NEXT:    vpcmpgtd %xmm3, %xmm2, %k0 {%k1}
+; AVX512BW-NEXT:    vpcmpgtb %xmm1, %xmm0, %k0
+; AVX512BW-NEXT:    vpcmpgtb %xmm3, %xmm2, %k1
+; AVX512BW-NEXT:    kandw %k1, %k0, %k0
 ; AVX512BW-NEXT:    kmovd %k0, %eax
 ; AVX512BW-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512BW-NEXT:    retq
@@ -773,66 +500,45 @@ define i4 @v4i8(<4 x i8> %a, <4 x i8> %b, <4 x i8> %c, <4 x i8> %d) {
 define i4 @v4i16(<4 x i16> %a, <4 x i16> %b, <4 x i16> %c, <4 x i16> %d) {
 ; SSE2-SSSE3-LABEL: v4i16:
 ; SSE2-SSSE3:       # %bb.0:
-; SSE2-SSSE3-NEXT:    pslld $16, %xmm3
-; SSE2-SSSE3-NEXT:    psrad $16, %xmm3
-; SSE2-SSSE3-NEXT:    pslld $16, %xmm2
-; SSE2-SSSE3-NEXT:    psrad $16, %xmm2
-; SSE2-SSSE3-NEXT:    pcmpgtd %xmm3, %xmm2
-; SSE2-SSSE3-NEXT:    pslld $16, %xmm1
-; SSE2-SSSE3-NEXT:    psrad $16, %xmm1
-; SSE2-SSSE3-NEXT:    pslld $16, %xmm0
-; SSE2-SSSE3-NEXT:    psrad $16, %xmm0
-; SSE2-SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-SSSE3-NEXT:    pand %xmm2, %xmm0
-; SSE2-SSSE3-NEXT:    movmskps %xmm0, %eax
+; SSE2-SSSE3-NEXT:    pcmpgtw %xmm1, %xmm0
+; SSE2-SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSE2-SSSE3-NEXT:    pcmpgtw %xmm3, %xmm2
+; SSE2-SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; SSE2-SSSE3-NEXT:    pand %xmm0, %xmm1
+; SSE2-SSSE3-NEXT:    movmskps %xmm1, %eax
 ; SSE2-SSSE3-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE2-SSSE3-NEXT:    retq
 ;
 ; AVX12-LABEL: v4i16:
 ; AVX12:       # %bb.0:
-; AVX12-NEXT:    vpslld $16, %xmm3, %xmm3
-; AVX12-NEXT:    vpsrad $16, %xmm3, %xmm3
-; AVX12-NEXT:    vpslld $16, %xmm2, %xmm2
-; AVX12-NEXT:    vpsrad $16, %xmm2, %xmm2
-; AVX12-NEXT:    vpcmpgtd %xmm3, %xmm2, %xmm2
-; AVX12-NEXT:    vpslld $16, %xmm1, %xmm1
-; AVX12-NEXT:    vpsrad $16, %xmm1, %xmm1
-; AVX12-NEXT:    vpslld $16, %xmm0, %xmm0
-; AVX12-NEXT:    vpsrad $16, %xmm0, %xmm0
-; AVX12-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm0
-; AVX12-NEXT:    vpand %xmm2, %xmm0, %xmm0
+; AVX12-NEXT:    vpcmpgtw %xmm1, %xmm0, %xmm0
+; AVX12-NEXT:    vpmovsxwd %xmm0, %xmm0
+; AVX12-NEXT:    vpcmpgtw %xmm3, %xmm2, %xmm1
+; AVX12-NEXT:    vpmovsxwd %xmm1, %xmm1
+; AVX12-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX12-NEXT:    vmovmskps %xmm0, %eax
 ; AVX12-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX12-NEXT:    retq
 ;
 ; AVX512F-LABEL: v4i16:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpslld $16, %xmm3, %xmm3
-; AVX512F-NEXT:    vpsrad $16, %xmm3, %xmm3
-; AVX512F-NEXT:    vpslld $16, %xmm2, %xmm2
-; AVX512F-NEXT:    vpsrad $16, %xmm2, %xmm2
-; AVX512F-NEXT:    vpslld $16, %xmm1, %xmm1
-; AVX512F-NEXT:    vpsrad $16, %xmm1, %xmm1
-; AVX512F-NEXT:    vpslld $16, %xmm0, %xmm0
-; AVX512F-NEXT:    vpsrad $16, %xmm0, %xmm0
-; AVX512F-NEXT:    vpcmpgtd %xmm1, %xmm0, %k1
-; AVX512F-NEXT:    vpcmpgtd %xmm3, %xmm2, %k0 {%k1}
+; AVX512F-NEXT:    vpcmpgtw %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    vpmovsxwd %xmm0, %ymm0
+; AVX512F-NEXT:    vptestmd %ymm0, %ymm0, %k0
+; AVX512F-NEXT:    vpcmpgtw %xmm3, %xmm2, %xmm0
+; AVX512F-NEXT:    vpmovsxwd %xmm0, %ymm0
+; AVX512F-NEXT:    vptestmd %ymm0, %ymm0, %k1
+; AVX512F-NEXT:    kandw %k1, %k0, %k0
 ; AVX512F-NEXT:    kmovw %k0, %eax
 ; AVX512F-NEXT:    # kill: def $al killed $al killed $eax
+; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: v4i16:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpslld $16, %xmm3, %xmm3
-; AVX512BW-NEXT:    vpsrad $16, %xmm3, %xmm3
-; AVX512BW-NEXT:    vpslld $16, %xmm2, %xmm2
-; AVX512BW-NEXT:    vpsrad $16, %xmm2, %xmm2
-; AVX512BW-NEXT:    vpslld $16, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpsrad $16, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpslld $16, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpsrad $16, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpcmpgtd %xmm1, %xmm0, %k1
-; AVX512BW-NEXT:    vpcmpgtd %xmm3, %xmm2, %k0 {%k1}
+; AVX512BW-NEXT:    vpcmpgtw %xmm1, %xmm0, %k0
+; AVX512BW-NEXT:    vpcmpgtw %xmm3, %xmm2, %k1
+; AVX512BW-NEXT:    kandw %k1, %k0, %k0
 ; AVX512BW-NEXT:    kmovd %k0, %eax
 ; AVX512BW-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512BW-NEXT:    retq
@@ -846,35 +552,23 @@ define i4 @v4i16(<4 x i16> %a, <4 x i16> %b, <4 x i16> %c, <4 x i16> %d) {
 define i8 @v8i8(<8 x i8> %a, <8 x i8> %b, <8 x i8> %c, <8 x i8> %d) {
 ; SSE2-SSSE3-LABEL: v8i8:
 ; SSE2-SSSE3:       # %bb.0:
-; SSE2-SSSE3-NEXT:    psllw $8, %xmm3
-; SSE2-SSSE3-NEXT:    psraw $8, %xmm3
-; SSE2-SSSE3-NEXT:    psllw $8, %xmm2
-; SSE2-SSSE3-NEXT:    psraw $8, %xmm2
-; SSE2-SSSE3-NEXT:    pcmpgtw %xmm3, %xmm2
-; SSE2-SSSE3-NEXT:    psllw $8, %xmm1
-; SSE2-SSSE3-NEXT:    psraw $8, %xmm1
-; SSE2-SSSE3-NEXT:    psllw $8, %xmm0
-; SSE2-SSSE3-NEXT:    psraw $8, %xmm0
-; SSE2-SSSE3-NEXT:    pcmpgtw %xmm1, %xmm0
-; SSE2-SSSE3-NEXT:    pand %xmm2, %xmm0
-; SSE2-SSSE3-NEXT:    packsswb %xmm0, %xmm0
-; SSE2-SSSE3-NEXT:    pmovmskb %xmm0, %eax
+; SSE2-SSSE3-NEXT:    pcmpgtb %xmm1, %xmm0
+; SSE2-SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-SSSE3-NEXT:    pcmpgtb %xmm3, %xmm2
+; SSE2-SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; SSE2-SSSE3-NEXT:    pand %xmm0, %xmm1
+; SSE2-SSSE3-NEXT:    packsswb %xmm0, %xmm1
+; SSE2-SSSE3-NEXT:    pmovmskb %xmm1, %eax
 ; SSE2-SSSE3-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE2-SSSE3-NEXT:    retq
 ;
 ; AVX12-LABEL: v8i8:
 ; AVX12:       # %bb.0:
-; AVX12-NEXT:    vpsllw $8, %xmm3, %xmm3
-; AVX12-NEXT:    vpsraw $8, %xmm3, %xmm3
-; AVX12-NEXT:    vpsllw $8, %xmm2, %xmm2
-; AVX12-NEXT:    vpsraw $8, %xmm2, %xmm2
-; AVX12-NEXT:    vpcmpgtw %xmm3, %xmm2, %xmm2
-; AVX12-NEXT:    vpsllw $8, %xmm1, %xmm1
-; AVX12-NEXT:    vpsraw $8, %xmm1, %xmm1
-; AVX12-NEXT:    vpsllw $8, %xmm0, %xmm0
-; AVX12-NEXT:    vpsraw $8, %xmm0, %xmm0
-; AVX12-NEXT:    vpcmpgtw %xmm1, %xmm0, %xmm0
-; AVX12-NEXT:    vpand %xmm2, %xmm0, %xmm0
+; AVX12-NEXT:    vpcmpgtb %xmm1, %xmm0, %xmm0
+; AVX12-NEXT:    vpmovsxbw %xmm0, %xmm0
+; AVX12-NEXT:    vpcmpgtb %xmm3, %xmm2, %xmm1
+; AVX12-NEXT:    vpmovsxbw %xmm1, %xmm1
+; AVX12-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX12-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
 ; AVX12-NEXT:    vpmovmskb %xmm0, %eax
 ; AVX12-NEXT:    # kill: def $al killed $al killed $eax
@@ -882,19 +576,13 @@ define i8 @v8i8(<8 x i8> %a, <8 x i8> %b, <8 x i8> %c, <8 x i8> %d) {
 ;
 ; AVX512F-LABEL: v8i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpsllw $8, %xmm3, %xmm3
-; AVX512F-NEXT:    vpsraw $8, %xmm3, %xmm3
-; AVX512F-NEXT:    vpsllw $8, %xmm2, %xmm2
-; AVX512F-NEXT:    vpsraw $8, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpgtw %xmm3, %xmm2, %xmm2
-; AVX512F-NEXT:    vpsllw $8, %xmm1, %xmm1
-; AVX512F-NEXT:    vpsraw $8, %xmm1, %xmm1
-; AVX512F-NEXT:    vpsllw $8, %xmm0, %xmm0
-; AVX512F-NEXT:    vpsraw $8, %xmm0, %xmm0
-; AVX512F-NEXT:    vpcmpgtw %xmm1, %xmm0, %xmm0
-; AVX512F-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; AVX512F-NEXT:    vpmovsxwd %xmm0, %ymm0
-; AVX512F-NEXT:    vptestmd %ymm0, %ymm0, %k0
+; AVX512F-NEXT:    vpcmpgtb %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    vpmovsxbd %xmm0, %zmm0
+; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k0
+; AVX512F-NEXT:    vpcmpgtb %xmm3, %xmm2, %xmm0
+; AVX512F-NEXT:    vpmovsxbd %xmm0, %zmm0
+; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k1
+; AVX512F-NEXT:    kandw %k1, %k0, %k0
 ; AVX512F-NEXT:    kmovw %k0, %eax
 ; AVX512F-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512F-NEXT:    vzeroupper
@@ -902,16 +590,9 @@ define i8 @v8i8(<8 x i8> %a, <8 x i8> %b, <8 x i8> %c, <8 x i8> %d) {
 ;
 ; AVX512BW-LABEL: v8i8:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpsllw $8, %xmm3, %xmm3
-; AVX512BW-NEXT:    vpsraw $8, %xmm3, %xmm3
-; AVX512BW-NEXT:    vpsllw $8, %xmm2, %xmm2
-; AVX512BW-NEXT:    vpsraw $8, %xmm2, %xmm2
-; AVX512BW-NEXT:    vpsllw $8, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpsraw $8, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpsllw $8, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpsraw $8, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpcmpgtw %xmm1, %xmm0, %k1
-; AVX512BW-NEXT:    vpcmpgtw %xmm3, %xmm2, %k0 {%k1}
+; AVX512BW-NEXT:    vpcmpgtb %xmm1, %xmm0, %k0
+; AVX512BW-NEXT:    vpcmpgtb %xmm3, %xmm2, %k1
+; AVX512BW-NEXT:    kandw %k1, %k0, %k0
 ; AVX512BW-NEXT:    kmovd %k0, %eax
 ; AVX512BW-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512BW-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/bitcast-and-setcc-256.ll b/llvm/test/CodeGen/X86/bitcast-and-setcc-256.ll
index b982cde2a..e50dca964 100644
--- a/llvm/test/CodeGen/X86/bitcast-and-setcc-256.ll
+++ b/llvm/test/CodeGen/X86/bitcast-and-setcc-256.ll
@@ -55,17 +55,17 @@ define i4 @v4i64(<4 x i64> %a, <4 x i64> %b, <4 x i64> %c, <4 x i64> %d) {
 ;
 ; AVX1-LABEL: v4i64:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm4
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm5
+; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm4
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm5
 ; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm5, %xmm4
-; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
-; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm1
-; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm4
-; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm4, %xmm1
 ; AVX1-NEXT:    vpcmpgtq %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
-; AVX1-NEXT:    vandpd %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm2, %ymm2
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
+; AVX1-NEXT:    vpcmpgtq %xmm3, %xmm4, %xmm3
+; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
+; AVX1-NEXT:    vandpd %ymm2, %ymm0, %ymm0
 ; AVX1-NEXT:    vmovmskpd %ymm0, %eax
 ; AVX1-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX1-NEXT:    vzeroupper
@@ -73,9 +73,9 @@ define i4 @v4i64(<4 x i64> %a, <4 x i64> %b, <4 x i64> %c, <4 x i64> %d) {
 ;
 ; AVX2-LABEL: v4i64:
 ; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpcmpgtq %ymm3, %ymm2, %ymm2
 ; AVX2-NEXT:    vpcmpgtq %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpcmpgtq %ymm3, %ymm2, %ymm1
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %ymm2, %ymm0, %ymm0
 ; AVX2-NEXT:    vmovmskpd %ymm0, %eax
 ; AVX2-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX2-NEXT:    vzeroupper
@@ -121,9 +121,9 @@ define i4 @v4f64(<4 x double> %a, <4 x double> %b, <4 x double> %c, <4 x double>
 ;
 ; AVX12-LABEL: v4f64:
 ; AVX12:       # %bb.0:
+; AVX12-NEXT:    vcmpltpd %ymm2, %ymm3, %ymm2
 ; AVX12-NEXT:    vcmpltpd %ymm0, %ymm1, %ymm0
-; AVX12-NEXT:    vcmpltpd %ymm2, %ymm3, %ymm1
-; AVX12-NEXT:    vandpd %ymm1, %ymm0, %ymm0
+; AVX12-NEXT:    vandpd %ymm2, %ymm0, %ymm0
 ; AVX12-NEXT:    vmovmskpd %ymm0, %eax
 ; AVX12-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX12-NEXT:    vzeroupper
@@ -241,32 +241,28 @@ define i8 @v8i32_and(<8 x i32> %a, <8 x i32> %b, <8 x i32> %c, <8 x i32> %d) {
 ;
 ; AVX1-LABEL: v8i32_and:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm4
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm1
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpgtd %xmm3, %xmm2, %xmm1
-; AVX1-NEXT:    vpand %xmm1, %xmm4, %xmm1
-; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm3
-; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm4
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm5
+; AVX1-NEXT:    vpcmpgtd %xmm4, %xmm5, %xmm4
 ; AVX1-NEXT:    vpcmpgtd %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm1, %xmm0
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
-; AVX1-NEXT:    vpmovmskb %xmm0, %eax
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm2, %ymm2
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
+; AVX1-NEXT:    vpcmpgtd %xmm3, %xmm4, %xmm3
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
+; AVX1-NEXT:    vandps %ymm2, %ymm0, %ymm0
+; AVX1-NEXT:    vmovmskps %ymm0, %eax
 ; AVX1-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: v8i32_and:
 ; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpcmpgtd %ymm3, %ymm2, %ymm2
 ; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpcmpgtd %ymm3, %ymm2, %ymm1
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
-; AVX2-NEXT:    vpmovmskb %xmm0, %eax
+; AVX2-NEXT:    vpand %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vmovmskps %ymm0, %eax
 ; AVX2-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -314,32 +310,28 @@ define i8 @v8i32_or(<8 x i32> %a, <8 x i32> %b, <8 x i32> %c, <8 x i32> %d) {
 ;
 ; AVX1-LABEL: v8i32_or:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm4
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm1
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpgtd %xmm3, %xmm2, %xmm1
-; AVX1-NEXT:    vpor %xmm1, %xmm4, %xmm1
-; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm3
-; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm4
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm5
+; AVX1-NEXT:    vpcmpgtd %xmm4, %xmm5, %xmm4
 ; AVX1-NEXT:    vpcmpgtd %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpor %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm1, %xmm0
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
-; AVX1-NEXT:    vpmovmskb %xmm0, %eax
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm2, %ymm2
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
+; AVX1-NEXT:    vpcmpgtd %xmm3, %xmm4, %xmm3
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
+; AVX1-NEXT:    vorps %ymm2, %ymm0, %ymm0
+; AVX1-NEXT:    vmovmskps %ymm0, %eax
 ; AVX1-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: v8i32_or:
 ; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpcmpgtd %ymm3, %ymm2, %ymm2
 ; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpcmpgtd %ymm3, %ymm2, %ymm1
-; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
-; AVX2-NEXT:    vpmovmskb %xmm0, %eax
+; AVX2-NEXT:    vpor %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vmovmskps %ymm0, %eax
 ; AVX2-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -395,42 +387,36 @@ define i8 @v8i32_or_and(<8 x i32> %a, <8 x i32> %b, <8 x i32> %c, <8 x i32> %d,
 ;
 ; AVX1-LABEL: v8i32_or_and:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm6
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm1
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm3, %xmm1
-; AVX1-NEXT:    vpor %xmm1, %xmm6, %xmm1
-; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm2
-; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm3
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm6
+; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm7
+; AVX1-NEXT:    vpcmpgtd %xmm6, %xmm7, %xmm6
 ; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm3, %xmm2
-; AVX1-NEXT:    vpor %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm2, %ymm2
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm6
+; AVX1-NEXT:    vpcmpgtd %xmm3, %xmm6, %xmm3
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
+; AVX1-NEXT:    vorps %ymm2, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm5, %xmm1
 ; AVX1-NEXT:    vextractf128 $1, %ymm4, %xmm2
 ; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm2, %xmm1
 ; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm4, %xmm2
-; AVX1-NEXT:    vpackssdw %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
-; AVX1-NEXT:    vpmovmskb %xmm0, %eax
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
+; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vmovmskps %ymm0, %eax
 ; AVX1-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: v8i32_or_and:
 ; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpcmpgtd %ymm2, %ymm3, %ymm2
 ; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpcmpgtd %ymm2, %ymm3, %ymm1
-; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %ymm2, %ymm0, %ymm0
 ; AVX2-NEXT:    vpcmpeqd %ymm5, %ymm4, %ymm1
-; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX2-NEXT:    vpackssdw %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm2
-; AVX2-NEXT:    vpackssdw %xmm2, %xmm0, %xmm0
-; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
-; AVX2-NEXT:    vpmovmskb %xmm0, %eax
+; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vmovmskps %ymm0, %eax
 ; AVX2-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -482,13 +468,10 @@ define i8 @v8f32_and(<8 x float> %a, <8 x float> %b, <8 x float> %c, <8 x float>
 ;
 ; AVX12-LABEL: v8f32_and:
 ; AVX12:       # %bb.0:
+; AVX12-NEXT:    vcmpltps %ymm2, %ymm3, %ymm2
 ; AVX12-NEXT:    vcmpltps %ymm0, %ymm1, %ymm0
-; AVX12-NEXT:    vcmpltps %ymm2, %ymm3, %ymm1
-; AVX12-NEXT:    vandps %ymm1, %ymm0, %ymm0
-; AVX12-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX12-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
-; AVX12-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
-; AVX12-NEXT:    vpmovmskb %xmm0, %eax
+; AVX12-NEXT:    vandps %ymm2, %ymm0, %ymm0
+; AVX12-NEXT:    vmovmskps %ymm0, %eax
 ; AVX12-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX12-NEXT:    vzeroupper
 ; AVX12-NEXT:    retq
@@ -536,13 +519,10 @@ define i8 @v8f32_xor(<8 x float> %a, <8 x float> %b, <8 x float> %c, <8 x float>
 ;
 ; AVX12-LABEL: v8f32_xor:
 ; AVX12:       # %bb.0:
+; AVX12-NEXT:    vcmpltps %ymm2, %ymm3, %ymm2
 ; AVX12-NEXT:    vcmpltps %ymm0, %ymm1, %ymm0
-; AVX12-NEXT:    vcmpltps %ymm2, %ymm3, %ymm1
-; AVX12-NEXT:    vxorps %ymm1, %ymm0, %ymm0
-; AVX12-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX12-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
-; AVX12-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
-; AVX12-NEXT:    vpmovmskb %xmm0, %eax
+; AVX12-NEXT:    vxorps %ymm2, %ymm0, %ymm0
+; AVX12-NEXT:    vmovmskps %ymm0, %eax
 ; AVX12-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX12-NEXT:    vzeroupper
 ; AVX12-NEXT:    retq
@@ -604,17 +584,12 @@ define i8 @v8f32_xor_and(<8 x float> %a, <8 x float> %b, <8 x float> %c, <8 x fl
 ;
 ; AVX12-LABEL: v8f32_xor_and:
 ; AVX12:       # %bb.0:
+; AVX12-NEXT:    vcmpeq_uqps %ymm3, %ymm2, %ymm2
 ; AVX12-NEXT:    vcmpnleps %ymm1, %ymm0, %ymm0
-; AVX12-NEXT:    vcmpeq_uqps %ymm3, %ymm2, %ymm1
-; AVX12-NEXT:    vxorps %ymm1, %ymm0, %ymm0
+; AVX12-NEXT:    vxorps %ymm2, %ymm0, %ymm0
 ; AVX12-NEXT:    vcmpltps %ymm4, %ymm5, %ymm1
-; AVX12-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX12-NEXT:    vpackssdw %xmm2, %xmm1, %xmm1
-; AVX12-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX12-NEXT:    vpackssdw %xmm2, %xmm0, %xmm0
-; AVX12-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX12-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
-; AVX12-NEXT:    vpmovmskb %xmm0, %eax
+; AVX12-NEXT:    vandps %ymm1, %ymm0, %ymm0
+; AVX12-NEXT:    vmovmskps %ymm0, %eax
 ; AVX12-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX12-NEXT:    vzeroupper
 ; AVX12-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/bitcast-and-setcc-512.ll b/llvm/test/CodeGen/X86/bitcast-and-setcc-512.ll
index 2abcdee7a..29499848e 100644
--- a/llvm/test/CodeGen/X86/bitcast-and-setcc-512.ll
+++ b/llvm/test/CodeGen/X86/bitcast-and-setcc-512.ll
@@ -50,56 +50,45 @@ define i8 @v8i64(<8 x i64> %a, <8 x i64> %b, <8 x i64> %c, <8 x i64> %d) {
 ;
 ; AVX1-LABEL: v8i64:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm8
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm9
+; AVX1-NEXT:    vextractf128 $1, %ymm7, %xmm8
+; AVX1-NEXT:    vextractf128 $1, %ymm5, %xmm9
 ; AVX1-NEXT:    vpcmpgtq %xmm8, %xmm9, %xmm8
+; AVX1-NEXT:    vpcmpgtq %xmm7, %xmm5, %xmm5
+; AVX1-NEXT:    vpackssdw %xmm8, %xmm5, %xmm8
+; AVX1-NEXT:    vextractf128 $1, %ymm6, %xmm7
+; AVX1-NEXT:    vextractf128 $1, %ymm4, %xmm5
+; AVX1-NEXT:    vpcmpgtq %xmm7, %xmm5, %xmm5
+; AVX1-NEXT:    vpcmpgtq %xmm6, %xmm4, %xmm4
+; AVX1-NEXT:    vpackssdw %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vinsertf128 $1, %xmm8, %ymm4, %ymm4
+; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm5
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm6
+; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm6, %xmm5
 ; AVX1-NEXT:    vpcmpgtq %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpackssdw %xmm8, %xmm1, %xmm8
+; AVX1-NEXT:    vpackssdw %xmm5, %xmm1, %xmm1
 ; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm3
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vpcmpgtq %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm5
+; AVX1-NEXT:    vpcmpgtq %xmm3, %xmm5, %xmm3
 ; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpackssdw %xmm8, %xmm0, %xmm0
-; AVX1-NEXT:    vextractf128 $1, %ymm7, %xmm1
-; AVX1-NEXT:    vextractf128 $1, %ymm5, %xmm2
-; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vpcmpgtq %xmm7, %xmm5, %xmm2
-; AVX1-NEXT:    vpackssdw %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vextractf128 $1, %ymm6, %xmm2
-; AVX1-NEXT:    vextractf128 $1, %ymm4, %xmm3
-; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm3, %xmm2
-; AVX1-NEXT:    vpcmpgtq %xmm6, %xmm4, %xmm3
-; AVX1-NEXT:    vpackssdw %xmm2, %xmm3, %xmm2
-; AVX1-NEXT:    vpackssdw %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsllw $15, %xmm0, %xmm0
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
-; AVX1-NEXT:    vpmovmskb %xmm0, %eax
+; AVX1-NEXT:    vpackssdw %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    vandps %ymm4, %ymm0, %ymm0
+; AVX1-NEXT:    vmovmskps %ymm0, %eax
 ; AVX1-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: v8i64:
 ; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpcmpgtq %ymm7, %ymm5, %ymm5
+; AVX2-NEXT:    vpcmpgtq %ymm6, %ymm4, %ymm4
+; AVX2-NEXT:    vpackssdw %ymm5, %ymm4, %ymm4
 ; AVX2-NEXT:    vpcmpgtq %ymm3, %ymm1, %ymm1
-; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm3
-; AVX2-NEXT:    vpackssdw %xmm3, %xmm1, %xmm1
 ; AVX2-NEXT:    vpcmpgtq %ymm2, %ymm0, %ymm0
-; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm2
-; AVX2-NEXT:    vpackssdw %xmm2, %xmm0, %xmm0
-; AVX2-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpcmpgtq %ymm7, %ymm5, %ymm1
-; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX2-NEXT:    vpackssdw %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpgtq %ymm6, %ymm4, %ymm2
-; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm3
-; AVX2-NEXT:    vpackssdw %xmm3, %xmm2, %xmm2
-; AVX2-NEXT:    vpackssdw %xmm1, %xmm2, %xmm1
-; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpsllw $15, %xmm0, %xmm0
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
-; AVX2-NEXT:    vpmovmskb %xmm0, %eax
+; AVX2-NEXT:    vpackssdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %ymm4, %ymm0, %ymm0
+; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2-NEXT:    vmovmskps %ymm0, %eax
 ; AVX2-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -171,29 +160,42 @@ define i8 @v8f64(<8 x double> %a, <8 x double> %b, <8 x double> %c, <8 x double>
 ; SSE-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE-NEXT:    retq
 ;
-; AVX12-LABEL: v8f64:
-; AVX12:       # %bb.0:
-; AVX12-NEXT:    vcmpltpd %ymm1, %ymm3, %ymm1
-; AVX12-NEXT:    vextractf128 $1, %ymm1, %xmm3
-; AVX12-NEXT:    vpackssdw %xmm3, %xmm1, %xmm1
-; AVX12-NEXT:    vcmpltpd %ymm0, %ymm2, %ymm0
-; AVX12-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX12-NEXT:    vpackssdw %xmm2, %xmm0, %xmm0
-; AVX12-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
-; AVX12-NEXT:    vcmpltpd %ymm5, %ymm7, %ymm1
-; AVX12-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX12-NEXT:    vpackssdw %xmm2, %xmm1, %xmm1
-; AVX12-NEXT:    vcmpltpd %ymm4, %ymm6, %ymm2
-; AVX12-NEXT:    vextractf128 $1, %ymm2, %xmm3
-; AVX12-NEXT:    vpackssdw %xmm3, %xmm2, %xmm2
-; AVX12-NEXT:    vpackssdw %xmm1, %xmm2, %xmm1
-; AVX12-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX12-NEXT:    vpsllw $15, %xmm0, %xmm0
-; AVX12-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
-; AVX12-NEXT:    vpmovmskb %xmm0, %eax
-; AVX12-NEXT:    # kill: def $al killed $al killed $eax
-; AVX12-NEXT:    vzeroupper
-; AVX12-NEXT:    retq
+; AVX1-LABEL: v8f64:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vcmpltpd %ymm5, %ymm7, %ymm5
+; AVX1-NEXT:    vextractf128 $1, %ymm5, %xmm7
+; AVX1-NEXT:    vpackssdw %xmm7, %xmm5, %xmm5
+; AVX1-NEXT:    vcmpltpd %ymm4, %ymm6, %ymm4
+; AVX1-NEXT:    vextractf128 $1, %ymm4, %xmm6
+; AVX1-NEXT:    vpackssdw %xmm6, %xmm4, %xmm4
+; AVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm4, %ymm4
+; AVX1-NEXT:    vcmpltpd %ymm1, %ymm3, %ymm1
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
+; AVX1-NEXT:    vpackssdw %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vcmpltpd %ymm0, %ymm2, %ymm0
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT:    vpackssdw %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    vandps %ymm4, %ymm0, %ymm0
+; AVX1-NEXT:    vmovmskps %ymm0, %eax
+; AVX1-NEXT:    # kill: def $al killed $al killed $eax
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: v8f64:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vcmpltpd %ymm5, %ymm7, %ymm5
+; AVX2-NEXT:    vcmpltpd %ymm4, %ymm6, %ymm4
+; AVX2-NEXT:    vpackssdw %ymm5, %ymm4, %ymm4
+; AVX2-NEXT:    vcmpltpd %ymm1, %ymm3, %ymm1
+; AVX2-NEXT:    vcmpltpd %ymm0, %ymm2, %ymm0
+; AVX2-NEXT:    vpackssdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %ymm4, %ymm0, %ymm0
+; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2-NEXT:    vmovmskps %ymm0, %eax
+; AVX2-NEXT:    # kill: def $al killed $al killed $eax
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: v8f64:
 ; AVX512F:       # %bb.0:
@@ -300,16 +302,20 @@ define i32 @v32i16(<32 x i16> %a, <32 x i16> %b, <32 x i16> %c, <32 x i16> %d) {
 ;
 ; AVX512F-LABEL: v32i16:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpcmpgtw %ymm3, %ymm1, %ymm1
-; AVX512F-NEXT:    vpcmpgtw %ymm2, %ymm0, %ymm0
-; AVX512F-NEXT:    vpcmpgtw %ymm7, %ymm5, %ymm2
-; AVX512F-NEXT:    vpand %ymm2, %ymm1, %ymm1
-; AVX512F-NEXT:    vpcmpgtw %ymm6, %ymm4, %ymm2
-; AVX512F-NEXT:    vpand %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm3, %ymm4
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm2, %ymm5
+; AVX512F-NEXT:    vpcmpgtw %ymm4, %ymm5, %ymm4
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm5
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm6
+; AVX512F-NEXT:    vpcmpgtw %ymm5, %ymm6, %ymm5
+; AVX512F-NEXT:    vpand %ymm4, %ymm5, %ymm4
+; AVX512F-NEXT:    vpcmpgtw %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vpcmpgtw %ymm3, %ymm2, %ymm1
+; AVX512F-NEXT:    vpand %ymm1, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpmovsxwd %ymm0, %zmm0
 ; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k0
 ; AVX512F-NEXT:    kmovw %k0, %ecx
-; AVX512F-NEXT:    vpmovsxwd %ymm1, %zmm0
+; AVX512F-NEXT:    vpmovsxwd %ymm4, %zmm0
 ; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k0
 ; AVX512F-NEXT:    kmovw %k0, %eax
 ; AVX512F-NEXT:    shll $16, %eax
@@ -608,30 +614,34 @@ define i64 @v64i8(<64 x i8> %a, <64 x i8> %b, <64 x i8> %c, <64 x i8> %d) {
 ;
 ; AVX512F-LABEL: v64i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpcmpgtb %ymm3, %ymm1, %ymm1
-; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm3
-; AVX512F-NEXT:    vpcmpgtb %ymm2, %ymm0, %ymm0
-; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm2
-; AVX512F-NEXT:    vpcmpgtb %ymm7, %ymm5, %ymm5
-; AVX512F-NEXT:    vextracti128 $1, %ymm5, %xmm7
-; AVX512F-NEXT:    vpand %xmm7, %xmm3, %xmm3
-; AVX512F-NEXT:    vpcmpgtb %ymm6, %ymm4, %ymm4
-; AVX512F-NEXT:    vextracti128 $1, %ymm4, %xmm6
-; AVX512F-NEXT:    vpand %xmm6, %xmm2, %xmm2
-; AVX512F-NEXT:    vpand %xmm4, %xmm0, %xmm0
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm3, %ymm4
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm2, %ymm5
+; AVX512F-NEXT:    vpcmpgtb %ymm4, %ymm5, %ymm4
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm5
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm6
+; AVX512F-NEXT:    vpcmpgtb %ymm5, %ymm6, %ymm5
+; AVX512F-NEXT:    vextracti128 $1, %ymm5, %xmm6
+; AVX512F-NEXT:    vpcmpgtb %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-NEXT:    vextracti128 $1, %ymm4, %xmm7
+; AVX512F-NEXT:    vpand %xmm7, %xmm6, %xmm6
+; AVX512F-NEXT:    vpcmpgtb %ymm3, %ymm2, %ymm2
+; AVX512F-NEXT:    vextracti128 $1, %ymm2, %xmm3
+; AVX512F-NEXT:    vpand %xmm3, %xmm1, %xmm1
+; AVX512F-NEXT:    vpand %xmm2, %xmm0, %xmm0
 ; AVX512F-NEXT:    vpmovsxbd %xmm0, %zmm0
 ; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k0
 ; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm0
+; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm0
 ; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k0
 ; AVX512F-NEXT:    kmovw %k0, %ecx
 ; AVX512F-NEXT:    shll $16, %ecx
 ; AVX512F-NEXT:    orl %eax, %ecx
-; AVX512F-NEXT:    vpand %xmm5, %xmm1, %xmm0
+; AVX512F-NEXT:    vpand %xmm4, %xmm5, %xmm0
 ; AVX512F-NEXT:    vpmovsxbd %xmm0, %zmm0
 ; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k0
 ; AVX512F-NEXT:    kmovw %k0, %edx
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm0
+; AVX512F-NEXT:    vpmovsxbd %xmm6, %zmm0
 ; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k0
 ; AVX512F-NEXT:    kmovw %k0, %eax
 ; AVX512F-NEXT:    shll $16, %eax
diff --git a/llvm/test/CodeGen/X86/bitcast-int-to-vector-bool-zext.ll b/llvm/test/CodeGen/X86/bitcast-int-to-vector-bool-zext.ll
index 00bbaf383..af1abe71e 100644
--- a/llvm/test/CodeGen/X86/bitcast-int-to-vector-bool-zext.ll
+++ b/llvm/test/CodeGen/X86/bitcast-int-to-vector-bool-zext.ll
@@ -496,12 +496,12 @@ define <32 x i8> @ext_i32_32i8(i32 %a0) {
 ; AVX512F-NEXT:    kmovw %edi, %k1
 ; AVX512F-NEXT:    shrl $16, %edi
 ; AVX512F-NEXT:    kmovw %edi, %k2
-; AVX512F-NEXT:    movl {{.*}}(%rip), %eax
-; AVX512F-NEXT:    vpbroadcastd %eax, %zmm0 {%k1} {z}
-; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
-; AVX512F-NEXT:    vpbroadcastd %eax, %zmm1 {%k2} {z}
+; AVX512F-NEXT:    vpbroadcastd {{.*#+}} zmm0 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; AVX512F-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1} {z}
 ; AVX512F-NEXT:    vpmovdb %zmm1, %xmm1
-; AVX512F-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vmovdqa32 %zmm0, %zmm0 {%k2} {z}
+; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512F-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: ext_i32_32i8:
@@ -776,12 +776,13 @@ define <32 x i16> @ext_i32_32i16(i32 %a0) {
 ; AVX512F-NEXT:    kmovw %edi, %k1
 ; AVX512F-NEXT:    shrl $16, %edi
 ; AVX512F-NEXT:    kmovw %edi, %k2
-; AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k2} {z}
 ; AVX512F-NEXT:    vpmovdw %zmm0, %ymm0
 ; AVX512F-NEXT:    vpsrlw $15, %ymm0, %ymm0
-; AVX512F-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k2} {z}
+; AVX512F-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k1} {z}
 ; AVX512F-NEXT:    vpmovdw %zmm1, %ymm1
 ; AVX512F-NEXT:    vpsrlw $15, %ymm1, %ymm1
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: ext_i32_32i16:
@@ -889,26 +890,26 @@ define <64 x i8> @ext_i64_64i8(i64 %a0) {
 ; AVX512F-LABEL: ext_i64_64i8:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    movq %rdi, %rax
-; AVX512F-NEXT:    movq %rdi, %rcx
+; AVX512F-NEXT:    movl %edi, %ecx
 ; AVX512F-NEXT:    kmovw %edi, %k1
-; AVX512F-NEXT:    movl %edi, %edx
-; AVX512F-NEXT:    shrl $16, %edx
-; AVX512F-NEXT:    shrq $32, %rax
-; AVX512F-NEXT:    shrq $48, %rcx
+; AVX512F-NEXT:    shrq $32, %rdi
+; AVX512F-NEXT:    shrq $48, %rax
+; AVX512F-NEXT:    shrl $16, %ecx
 ; AVX512F-NEXT:    kmovw %ecx, %k2
 ; AVX512F-NEXT:    kmovw %eax, %k3
-; AVX512F-NEXT:    kmovw %edx, %k4
-; AVX512F-NEXT:    movl {{.*}}(%rip), %eax
-; AVX512F-NEXT:    vpbroadcastd %eax, %zmm0 {%k1} {z}
-; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
-; AVX512F-NEXT:    vpbroadcastd %eax, %zmm1 {%k4} {z}
+; AVX512F-NEXT:    kmovw %edi, %k4
+; AVX512F-NEXT:    vpbroadcastd {{.*#+}} zmm0 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; AVX512F-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k4} {z}
 ; AVX512F-NEXT:    vpmovdb %zmm1, %xmm1
-; AVX512F-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX512F-NEXT:    vpbroadcastd %eax, %zmm1 {%k3} {z}
-; AVX512F-NEXT:    vpmovdb %zmm1, %xmm1
-; AVX512F-NEXT:    vpbroadcastd %eax, %zmm2 {%k2} {z}
+; AVX512F-NEXT:    vmovdqa32 %zmm0, %zmm2 {%k3} {z}
 ; AVX512F-NEXT:    vpmovdb %zmm2, %xmm2
 ; AVX512F-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512F-NEXT:    vmovdqa32 %zmm0, %zmm2 {%k1} {z}
+; AVX512F-NEXT:    vpmovdb %zmm2, %xmm2
+; AVX512F-NEXT:    vmovdqa32 %zmm0, %zmm0 {%k2} {z}
+; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512F-NEXT:    vinserti128 $1, %xmm0, %ymm2, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: ext_i64_64i8:
diff --git a/llvm/test/CodeGen/X86/bitcast-mmx.ll b/llvm/test/CodeGen/X86/bitcast-mmx.ll
index 343c23032..566cb1b85 100644
--- a/llvm/test/CodeGen/X86/bitcast-mmx.ll
+++ b/llvm/test/CodeGen/X86/bitcast-mmx.ll
@@ -34,10 +34,10 @@ define i64 @t1(i64 %x, i32 %n) nounwind {
 ; X86-NEXT:    movl %esp, %ebp
 ; X86-NEXT:    andl $-8, %esp
 ; X86-NEXT:    subl $8, %esp
-; X86-NEXT:    movd 16(%ebp), %mm0
-; X86-NEXT:    movq 8(%ebp), %mm1
-; X86-NEXT:    psllq %mm0, %mm1
-; X86-NEXT:    movq %mm1, (%esp)
+; X86-NEXT:    movq 8(%ebp), %mm0
+; X86-NEXT:    movd 16(%ebp), %mm1
+; X86-NEXT:    psllq %mm1, %mm0
+; X86-NEXT:    movq %mm0, (%esp)
 ; X86-NEXT:    movl (%esp), %eax
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-NEXT:    movl %ebp, %esp
@@ -46,10 +46,10 @@ define i64 @t1(i64 %x, i32 %n) nounwind {
 ;
 ; X64-LABEL: t1:
 ; X64:       # %bb.0: # %entry
-; X64-NEXT:    movd %esi, %mm0
-; X64-NEXT:    movq %rdi, %mm1
-; X64-NEXT:    psllq %mm0, %mm1
-; X64-NEXT:    movq %mm1, %rax
+; X64-NEXT:    movq %rdi, %mm0
+; X64-NEXT:    movd %esi, %mm1
+; X64-NEXT:    psllq %mm1, %mm0
+; X64-NEXT:    movq %mm0, %rax
 ; X64-NEXT:    retq
 entry:
   %0 = bitcast i64 %x to x86_mmx
@@ -65,11 +65,11 @@ define i64 @t2(i64 %x, i32 %n, i32 %w) nounwind {
 ; X86-NEXT:    movl %esp, %ebp
 ; X86-NEXT:    andl $-8, %esp
 ; X86-NEXT:    subl $8, %esp
-; X86-NEXT:    movd 16(%ebp), %mm0
-; X86-NEXT:    movd 20(%ebp), %mm1
-; X86-NEXT:    psllq %mm0, %mm1
-; X86-NEXT:    por 8(%ebp), %mm1
-; X86-NEXT:    movq %mm1, (%esp)
+; X86-NEXT:    movd 20(%ebp), %mm0
+; X86-NEXT:    movd 16(%ebp), %mm1
+; X86-NEXT:    psllq %mm1, %mm0
+; X86-NEXT:    por 8(%ebp), %mm0
+; X86-NEXT:    movq %mm0, (%esp)
 ; X86-NEXT:    movl (%esp), %eax
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-NEXT:    movl %ebp, %esp
@@ -78,12 +78,12 @@ define i64 @t2(i64 %x, i32 %n, i32 %w) nounwind {
 ;
 ; X64-LABEL: t2:
 ; X64:       # %bb.0: # %entry
-; X64-NEXT:    movd %esi, %mm0
-; X64-NEXT:    movd %edx, %mm1
-; X64-NEXT:    psllq %mm0, %mm1
-; X64-NEXT:    movq %rdi, %mm0
-; X64-NEXT:    por %mm1, %mm0
-; X64-NEXT:    movq %mm0, %rax
+; X64-NEXT:    movd %edx, %mm0
+; X64-NEXT:    movd %esi, %mm1
+; X64-NEXT:    psllq %mm1, %mm0
+; X64-NEXT:    movq %rdi, %mm1
+; X64-NEXT:    por %mm0, %mm1
+; X64-NEXT:    movq %mm1, %rax
 ; X64-NEXT:    retq
 entry:
   %0 = insertelement <2 x i32> undef, i32 %w, i32 0
diff --git a/llvm/test/CodeGen/X86/bitcast-setcc-128.ll b/llvm/test/CodeGen/X86/bitcast-setcc-128.ll
index 0a44e791b..d7871ff21 100644
--- a/llvm/test/CodeGen/X86/bitcast-setcc-128.ll
+++ b/llvm/test/CodeGen/X86/bitcast-setcc-128.ll
@@ -144,87 +144,45 @@ define i16 @v16i8(<16 x i8> %a, <16 x i8> %b) {
 }
 
 define i2 @v2i8(<2 x i8> %a, <2 x i8> %b) {
-; SSE2-SSSE3-LABEL: v2i8:
-; SSE2-SSSE3:       # %bb.0:
-; SSE2-SSSE3-NEXT:    psllq $56, %xmm0
-; SSE2-SSSE3-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-SSSE3-NEXT:    psrad $31, %xmm2
-; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
-; SSE2-SSSE3-NEXT:    psrad $24, %xmm0
-; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
-; SSE2-SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; SSE2-SSSE3-NEXT:    psllq $56, %xmm1
-; SSE2-SSSE3-NEXT:    movdqa %xmm1, %xmm2
-; SSE2-SSSE3-NEXT:    psrad $31, %xmm2
-; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
-; SSE2-SSSE3-NEXT:    psrad $24, %xmm1
-; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
-; SSE2-SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
-; SSE2-SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
-; SSE2-SSSE3-NEXT:    pxor %xmm2, %xmm1
-; SSE2-SSSE3-NEXT:    pxor %xmm2, %xmm0
-; SSE2-SSSE3-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-SSSE3-NEXT:    pcmpeqd %xmm1, %xmm2
-; SSE2-SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE2-SSSE3-NEXT:    pand %xmm2, %xmm1
-; SSE2-SSSE3-NEXT:    por %xmm0, %xmm1
-; SSE2-SSSE3-NEXT:    movmskpd %xmm1, %eax
-; SSE2-SSSE3-NEXT:    # kill: def $al killed $al killed $eax
-; SSE2-SSSE3-NEXT:    retq
-;
-; AVX1-LABEL: v2i8:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpsllq $56, %xmm1, %xmm1
-; AVX1-NEXT:    vpsrad $31, %xmm1, %xmm2
-; AVX1-NEXT:    vpsrad $24, %xmm1, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
-; AVX1-NEXT:    vpsllq $56, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrad $31, %xmm0, %xmm2
-; AVX1-NEXT:    vpsrad $24, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
-; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vmovmskpd %xmm0, %eax
-; AVX1-NEXT:    # kill: def $al killed $al killed $eax
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: v2i8:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpsllq $56, %xmm1, %xmm1
-; AVX2-NEXT:    vpsrad $31, %xmm1, %xmm2
-; AVX2-NEXT:    vpsrad $24, %xmm1, %xmm1
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; AVX2-NEXT:    vpsllq $56, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrad $31, %xmm0, %xmm2
-; AVX2-NEXT:    vpsrad $24, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
-; AVX2-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vmovmskpd %xmm0, %eax
-; AVX2-NEXT:    # kill: def $al killed $al killed $eax
-; AVX2-NEXT:    retq
+; SSE2-LABEL: v2i8:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pcmpgtb %xmm1, %xmm0
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,1,1]
+; SSE2-NEXT:    movmskpd %xmm0, %eax
+; SSE2-NEXT:    # kill: def $al killed $al killed $eax
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: v2i8:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    pcmpgtb %xmm1, %xmm0
+; SSSE3-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[u,u,u,0,u,u,u,0,u,u,u,1,u,u,u,1]
+; SSSE3-NEXT:    movmskpd %xmm0, %eax
+; SSSE3-NEXT:    # kill: def $al killed $al killed $eax
+; SSSE3-NEXT:    retq
+;
+; AVX12-LABEL: v2i8:
+; AVX12:       # %bb.0:
+; AVX12-NEXT:    vpcmpgtb %xmm1, %xmm0, %xmm0
+; AVX12-NEXT:    vpmovsxbq %xmm0, %xmm0
+; AVX12-NEXT:    vmovmskpd %xmm0, %eax
+; AVX12-NEXT:    # kill: def $al killed $al killed $eax
+; AVX12-NEXT:    retq
 ;
 ; AVX512F-LABEL: v2i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpsllq $56, %xmm1, %xmm1
-; AVX512F-NEXT:    vpsraq $56, %xmm1, %xmm1
-; AVX512F-NEXT:    vpsllq $56, %xmm0, %xmm0
-; AVX512F-NEXT:    vpsraq $56, %xmm0, %xmm0
-; AVX512F-NEXT:    vpcmpgtq %xmm1, %xmm0, %k0
+; AVX512F-NEXT:    vpcmpgtb %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    vpmovsxbd %xmm0, %zmm0
+; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k0
 ; AVX512F-NEXT:    kmovw %k0, %eax
 ; AVX512F-NEXT:    # kill: def $al killed $al killed $eax
+; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: v2i8:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpsllq $56, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpsraq $56, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpsllq $56, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpsraq $56, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpcmpgtq %xmm1, %xmm0, %k0
+; AVX512BW-NEXT:    vpcmpgtb %xmm1, %xmm0, %k0
 ; AVX512BW-NEXT:    kmovd %k0, %eax
 ; AVX512BW-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512BW-NEXT:    retq
@@ -236,85 +194,34 @@ define i2 @v2i8(<2 x i8> %a, <2 x i8> %b) {
 define i2 @v2i16(<2 x i16> %a, <2 x i16> %b) {
 ; SSE2-SSSE3-LABEL: v2i16:
 ; SSE2-SSSE3:       # %bb.0:
-; SSE2-SSSE3-NEXT:    psllq $48, %xmm0
-; SSE2-SSSE3-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-SSSE3-NEXT:    psrad $31, %xmm2
-; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
-; SSE2-SSSE3-NEXT:    psrad $16, %xmm0
-; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
-; SSE2-SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; SSE2-SSSE3-NEXT:    psllq $48, %xmm1
-; SSE2-SSSE3-NEXT:    movdqa %xmm1, %xmm2
-; SSE2-SSSE3-NEXT:    psrad $31, %xmm2
-; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
-; SSE2-SSSE3-NEXT:    psrad $16, %xmm1
-; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
-; SSE2-SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
-; SSE2-SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
-; SSE2-SSSE3-NEXT:    pxor %xmm2, %xmm1
-; SSE2-SSSE3-NEXT:    pxor %xmm2, %xmm0
-; SSE2-SSSE3-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-SSSE3-NEXT:    pcmpeqd %xmm1, %xmm2
-; SSE2-SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE2-SSSE3-NEXT:    pand %xmm2, %xmm1
-; SSE2-SSSE3-NEXT:    por %xmm0, %xmm1
-; SSE2-SSSE3-NEXT:    movmskpd %xmm1, %eax
+; SSE2-SSSE3-NEXT:    pcmpgtw %xmm1, %xmm0
+; SSE2-SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,1,1]
+; SSE2-SSSE3-NEXT:    movmskpd %xmm0, %eax
 ; SSE2-SSSE3-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE2-SSSE3-NEXT:    retq
 ;
-; AVX1-LABEL: v2i16:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpsllq $48, %xmm1, %xmm1
-; AVX1-NEXT:    vpsrad $31, %xmm1, %xmm2
-; AVX1-NEXT:    vpsrad $16, %xmm1, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
-; AVX1-NEXT:    vpsllq $48, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrad $31, %xmm0, %xmm2
-; AVX1-NEXT:    vpsrad $16, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
-; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vmovmskpd %xmm0, %eax
-; AVX1-NEXT:    # kill: def $al killed $al killed $eax
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: v2i16:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpsllq $48, %xmm1, %xmm1
-; AVX2-NEXT:    vpsrad $31, %xmm1, %xmm2
-; AVX2-NEXT:    vpsrad $16, %xmm1, %xmm1
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; AVX2-NEXT:    vpsllq $48, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrad $31, %xmm0, %xmm2
-; AVX2-NEXT:    vpsrad $16, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
-; AVX2-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vmovmskpd %xmm0, %eax
-; AVX2-NEXT:    # kill: def $al killed $al killed $eax
-; AVX2-NEXT:    retq
+; AVX12-LABEL: v2i16:
+; AVX12:       # %bb.0:
+; AVX12-NEXT:    vpcmpgtw %xmm1, %xmm0, %xmm0
+; AVX12-NEXT:    vpmovsxwq %xmm0, %xmm0
+; AVX12-NEXT:    vmovmskpd %xmm0, %eax
+; AVX12-NEXT:    # kill: def $al killed $al killed $eax
+; AVX12-NEXT:    retq
 ;
 ; AVX512F-LABEL: v2i16:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpsllq $48, %xmm1, %xmm1
-; AVX512F-NEXT:    vpsraq $48, %xmm1, %xmm1
-; AVX512F-NEXT:    vpsllq $48, %xmm0, %xmm0
-; AVX512F-NEXT:    vpsraq $48, %xmm0, %xmm0
-; AVX512F-NEXT:    vpcmpgtq %xmm1, %xmm0, %k0
+; AVX512F-NEXT:    vpcmpgtw %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    vpmovsxwd %xmm0, %ymm0
+; AVX512F-NEXT:    vptestmd %ymm0, %ymm0, %k0
 ; AVX512F-NEXT:    kmovw %k0, %eax
 ; AVX512F-NEXT:    # kill: def $al killed $al killed $eax
+; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: v2i16:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpsllq $48, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpsraq $48, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpsllq $48, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpsraq $48, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpcmpgtq %xmm1, %xmm0, %k0
+; AVX512BW-NEXT:    vpcmpgtw %xmm1, %xmm0, %k0
 ; AVX512BW-NEXT:    kmovd %k0, %eax
 ; AVX512BW-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512BW-NEXT:    retq
@@ -326,73 +233,30 @@ define i2 @v2i16(<2 x i16> %a, <2 x i16> %b) {
 define i2 @v2i32(<2 x i32> %a, <2 x i32> %b) {
 ; SSE2-SSSE3-LABEL: v2i32:
 ; SSE2-SSSE3:       # %bb.0:
-; SSE2-SSSE3-NEXT:    psllq $32, %xmm0
-; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,3,2,3]
-; SSE2-SSSE3-NEXT:    psrad $31, %xmm0
-; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
-; SSE2-SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; SSE2-SSSE3-NEXT:    psllq $32, %xmm1
-; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,3,2,3]
-; SSE2-SSSE3-NEXT:    psrad $31, %xmm1
-; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
-; SSE2-SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; SSE2-SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [2147483648,2147483648]
-; SSE2-SSSE3-NEXT:    pxor %xmm1, %xmm0
-; SSE2-SSSE3-NEXT:    pxor %xmm1, %xmm2
-; SSE2-SSSE3-NEXT:    movdqa %xmm2, %xmm1
-; SSE2-SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE2-SSSE3-NEXT:    pcmpgtd %xmm0, %xmm2
-; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,0,2,2]
-; SSE2-SSSE3-NEXT:    pand %xmm1, %xmm0
-; SSE2-SSSE3-NEXT:    por %xmm2, %xmm0
+; SSE2-SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
+; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,1,1]
 ; SSE2-SSSE3-NEXT:    movmskpd %xmm0, %eax
 ; SSE2-SSSE3-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE2-SSSE3-NEXT:    retq
 ;
-; AVX1-LABEL: v2i32:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpsllq $32, %xmm1, %xmm2
-; AVX1-NEXT:    vpsrad $31, %xmm2, %xmm2
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
-; AVX1-NEXT:    vpsllq $32, %xmm0, %xmm2
-; AVX1-NEXT:    vpsrad $31, %xmm2, %xmm2
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
-; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vmovmskpd %xmm0, %eax
-; AVX1-NEXT:    # kill: def $al killed $al killed $eax
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: v2i32:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpsllq $32, %xmm1, %xmm2
-; AVX2-NEXT:    vpsrad $31, %xmm2, %xmm2
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; AVX2-NEXT:    vpsllq $32, %xmm0, %xmm2
-; AVX2-NEXT:    vpsrad $31, %xmm2, %xmm2
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
-; AVX2-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vmovmskpd %xmm0, %eax
-; AVX2-NEXT:    # kill: def $al killed $al killed $eax
-; AVX2-NEXT:    retq
+; AVX12-LABEL: v2i32:
+; AVX12:       # %bb.0:
+; AVX12-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm0
+; AVX12-NEXT:    vpmovsxdq %xmm0, %xmm0
+; AVX12-NEXT:    vmovmskpd %xmm0, %eax
+; AVX12-NEXT:    # kill: def $al killed $al killed $eax
+; AVX12-NEXT:    retq
 ;
 ; AVX512F-LABEL: v2i32:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpsllq $32, %xmm1, %xmm1
-; AVX512F-NEXT:    vpsraq $32, %xmm1, %xmm1
-; AVX512F-NEXT:    vpsllq $32, %xmm0, %xmm0
-; AVX512F-NEXT:    vpsraq $32, %xmm0, %xmm0
-; AVX512F-NEXT:    vpcmpgtq %xmm1, %xmm0, %k0
+; AVX512F-NEXT:    vpcmpgtd %xmm1, %xmm0, %k0
 ; AVX512F-NEXT:    kmovw %k0, %eax
 ; AVX512F-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: v2i32:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpsllq $32, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpsraq $32, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpsllq $32, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpsraq $32, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpcmpgtq %xmm1, %xmm0, %k0
+; AVX512BW-NEXT:    vpcmpgtd %xmm1, %xmm0, %k0
 ; AVX512BW-NEXT:    kmovd %k0, %eax
 ; AVX512BW-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512BW-NEXT:    retq
@@ -478,44 +342,34 @@ define i2 @v2f64(<2 x double> %a, <2 x double> %b) {
 define i4 @v4i8(<4 x i8> %a, <4 x i8> %b) {
 ; SSE2-SSSE3-LABEL: v4i8:
 ; SSE2-SSSE3:       # %bb.0:
-; SSE2-SSSE3-NEXT:    pslld $24, %xmm1
-; SSE2-SSSE3-NEXT:    psrad $24, %xmm1
-; SSE2-SSSE3-NEXT:    pslld $24, %xmm0
-; SSE2-SSSE3-NEXT:    psrad $24, %xmm0
-; SSE2-SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
+; SSE2-SSSE3-NEXT:    pcmpgtb %xmm1, %xmm0
+; SSE2-SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
 ; SSE2-SSSE3-NEXT:    movmskps %xmm0, %eax
 ; SSE2-SSSE3-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE2-SSSE3-NEXT:    retq
 ;
 ; AVX12-LABEL: v4i8:
 ; AVX12:       # %bb.0:
-; AVX12-NEXT:    vpslld $24, %xmm1, %xmm1
-; AVX12-NEXT:    vpsrad $24, %xmm1, %xmm1
-; AVX12-NEXT:    vpslld $24, %xmm0, %xmm0
-; AVX12-NEXT:    vpsrad $24, %xmm0, %xmm0
-; AVX12-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm0
+; AVX12-NEXT:    vpcmpgtb %xmm1, %xmm0, %xmm0
+; AVX12-NEXT:    vpmovsxbd %xmm0, %xmm0
 ; AVX12-NEXT:    vmovmskps %xmm0, %eax
 ; AVX12-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX12-NEXT:    retq
 ;
 ; AVX512F-LABEL: v4i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpslld $24, %xmm1, %xmm1
-; AVX512F-NEXT:    vpsrad $24, %xmm1, %xmm1
-; AVX512F-NEXT:    vpslld $24, %xmm0, %xmm0
-; AVX512F-NEXT:    vpsrad $24, %xmm0, %xmm0
-; AVX512F-NEXT:    vpcmpgtd %xmm1, %xmm0, %k0
+; AVX512F-NEXT:    vpcmpgtb %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    vpmovsxbd %xmm0, %zmm0
+; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k0
 ; AVX512F-NEXT:    kmovw %k0, %eax
 ; AVX512F-NEXT:    # kill: def $al killed $al killed $eax
+; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: v4i8:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpslld $24, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpsrad $24, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpslld $24, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpsrad $24, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpcmpgtd %xmm1, %xmm0, %k0
+; AVX512BW-NEXT:    vpcmpgtb %xmm1, %xmm0, %k0
 ; AVX512BW-NEXT:    kmovd %k0, %eax
 ; AVX512BW-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512BW-NEXT:    retq
@@ -527,44 +381,33 @@ define i4 @v4i8(<4 x i8> %a, <4 x i8> %b) {
 define i4 @v4i16(<4 x i16> %a, <4 x i16> %b) {
 ; SSE2-SSSE3-LABEL: v4i16:
 ; SSE2-SSSE3:       # %bb.0:
-; SSE2-SSSE3-NEXT:    pslld $16, %xmm1
-; SSE2-SSSE3-NEXT:    psrad $16, %xmm1
-; SSE2-SSSE3-NEXT:    pslld $16, %xmm0
-; SSE2-SSSE3-NEXT:    psrad $16, %xmm0
-; SSE2-SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
+; SSE2-SSSE3-NEXT:    pcmpgtw %xmm1, %xmm0
+; SSE2-SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
 ; SSE2-SSSE3-NEXT:    movmskps %xmm0, %eax
 ; SSE2-SSSE3-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE2-SSSE3-NEXT:    retq
 ;
 ; AVX12-LABEL: v4i16:
 ; AVX12:       # %bb.0:
-; AVX12-NEXT:    vpslld $16, %xmm1, %xmm1
-; AVX12-NEXT:    vpsrad $16, %xmm1, %xmm1
-; AVX12-NEXT:    vpslld $16, %xmm0, %xmm0
-; AVX12-NEXT:    vpsrad $16, %xmm0, %xmm0
-; AVX12-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm0
+; AVX12-NEXT:    vpcmpgtw %xmm1, %xmm0, %xmm0
+; AVX12-NEXT:    vpmovsxwd %xmm0, %xmm0
 ; AVX12-NEXT:    vmovmskps %xmm0, %eax
 ; AVX12-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX12-NEXT:    retq
 ;
 ; AVX512F-LABEL: v4i16:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpslld $16, %xmm1, %xmm1
-; AVX512F-NEXT:    vpsrad $16, %xmm1, %xmm1
-; AVX512F-NEXT:    vpslld $16, %xmm0, %xmm0
-; AVX512F-NEXT:    vpsrad $16, %xmm0, %xmm0
-; AVX512F-NEXT:    vpcmpgtd %xmm1, %xmm0, %k0
+; AVX512F-NEXT:    vpcmpgtw %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    vpmovsxwd %xmm0, %ymm0
+; AVX512F-NEXT:    vptestmd %ymm0, %ymm0, %k0
 ; AVX512F-NEXT:    kmovw %k0, %eax
 ; AVX512F-NEXT:    # kill: def $al killed $al killed $eax
+; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: v4i16:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpslld $16, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpsrad $16, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpslld $16, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpsrad $16, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpcmpgtd %xmm1, %xmm0, %k0
+; AVX512BW-NEXT:    vpcmpgtw %xmm1, %xmm0, %k0
 ; AVX512BW-NEXT:    kmovd %k0, %eax
 ; AVX512BW-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512BW-NEXT:    retq
@@ -576,11 +419,8 @@ define i4 @v4i16(<4 x i16> %a, <4 x i16> %b) {
 define i8 @v8i8(<8 x i8> %a, <8 x i8> %b) {
 ; SSE2-SSSE3-LABEL: v8i8:
 ; SSE2-SSSE3:       # %bb.0:
-; SSE2-SSSE3-NEXT:    psllw $8, %xmm1
-; SSE2-SSSE3-NEXT:    psraw $8, %xmm1
-; SSE2-SSSE3-NEXT:    psllw $8, %xmm0
-; SSE2-SSSE3-NEXT:    psraw $8, %xmm0
-; SSE2-SSSE3-NEXT:    pcmpgtw %xmm1, %xmm0
+; SSE2-SSSE3-NEXT:    pcmpgtb %xmm1, %xmm0
+; SSE2-SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; SSE2-SSSE3-NEXT:    packsswb %xmm0, %xmm0
 ; SSE2-SSSE3-NEXT:    pmovmskb %xmm0, %eax
 ; SSE2-SSSE3-NEXT:    # kill: def $al killed $al killed $eax
@@ -588,11 +428,8 @@ define i8 @v8i8(<8 x i8> %a, <8 x i8> %b) {
 ;
 ; AVX12-LABEL: v8i8:
 ; AVX12:       # %bb.0:
-; AVX12-NEXT:    vpsllw $8, %xmm1, %xmm1
-; AVX12-NEXT:    vpsraw $8, %xmm1, %xmm1
-; AVX12-NEXT:    vpsllw $8, %xmm0, %xmm0
-; AVX12-NEXT:    vpsraw $8, %xmm0, %xmm0
-; AVX12-NEXT:    vpcmpgtw %xmm1, %xmm0, %xmm0
+; AVX12-NEXT:    vpcmpgtb %xmm1, %xmm0, %xmm0
+; AVX12-NEXT:    vpmovsxbw %xmm0, %xmm0
 ; AVX12-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
 ; AVX12-NEXT:    vpmovmskb %xmm0, %eax
 ; AVX12-NEXT:    # kill: def $al killed $al killed $eax
@@ -600,13 +437,9 @@ define i8 @v8i8(<8 x i8> %a, <8 x i8> %b) {
 ;
 ; AVX512F-LABEL: v8i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpsllw $8, %xmm1, %xmm1
-; AVX512F-NEXT:    vpsraw $8, %xmm1, %xmm1
-; AVX512F-NEXT:    vpsllw $8, %xmm0, %xmm0
-; AVX512F-NEXT:    vpsraw $8, %xmm0, %xmm0
-; AVX512F-NEXT:    vpcmpgtw %xmm1, %xmm0, %xmm0
-; AVX512F-NEXT:    vpmovsxwd %xmm0, %ymm0
-; AVX512F-NEXT:    vptestmd %ymm0, %ymm0, %k0
+; AVX512F-NEXT:    vpcmpgtb %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    vpmovsxbd %xmm0, %zmm0
+; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k0
 ; AVX512F-NEXT:    kmovw %k0, %eax
 ; AVX512F-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512F-NEXT:    vzeroupper
@@ -614,11 +447,7 @@ define i8 @v8i8(<8 x i8> %a, <8 x i8> %b) {
 ;
 ; AVX512BW-LABEL: v8i8:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpsllw $8, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpsraw $8, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpsllw $8, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpsraw $8, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpcmpgtw %xmm1, %xmm0, %k0
+; AVX512BW-NEXT:    vpcmpgtb %xmm1, %xmm0, %k0
 ; AVX512BW-NEXT:    kmovd %k0, %eax
 ; AVX512BW-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512BW-NEXT:    retq
@@ -653,7 +482,6 @@ define i64 @v16i8_widened_with_zeroes(<16 x i8> %a, <16 x i8> %b) {
 ; AVX512F-NEXT:    vpmovsxbd %xmm0, %zmm0
 ; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k0
 ; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    movzwl %ax, %eax
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/bitcast-setcc-512.ll b/llvm/test/CodeGen/X86/bitcast-setcc-512.ll
index 9f5097bab..09cfb8d10 100644
--- a/llvm/test/CodeGen/X86/bitcast-setcc-512.ll
+++ b/llvm/test/CodeGen/X86/bitcast-setcc-512.ll
@@ -51,12 +51,14 @@ define i32 @v32i16(<32 x i16> %a, <32 x i16> %b) {
 ;
 ; AVX512F-LABEL: v32i16:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpcmpgtw %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
+; AVX512F-NEXT:    vpcmpgtw %ymm2, %ymm3, %ymm2
+; AVX512F-NEXT:    vpcmpgtw %ymm1, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpmovsxwd %ymm0, %zmm0
 ; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k0
 ; AVX512F-NEXT:    kmovw %k0, %ecx
-; AVX512F-NEXT:    vpcmpgtw %ymm3, %ymm1, %ymm0
-; AVX512F-NEXT:    vpmovsxwd %ymm0, %zmm0
+; AVX512F-NEXT:    vpmovsxwd %ymm2, %zmm0
 ; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k0
 ; AVX512F-NEXT:    kmovw %k0, %eax
 ; AVX512F-NEXT:    shll $16, %eax
@@ -256,10 +258,12 @@ define i64 @v64i8(<64 x i8> %a, <64 x i8> %b) {
 ;
 ; AVX512F-LABEL: v64i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpcmpgtb %ymm3, %ymm1, %ymm1
-; AVX512F-NEXT:    vpcmpgtb %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
+; AVX512F-NEXT:    vpcmpgtb %ymm2, %ymm3, %ymm2
+; AVX512F-NEXT:    vpcmpgtb %ymm1, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpmovmskb %ymm0, %ecx
-; AVX512F-NEXT:    vpmovmskb %ymm1, %eax
+; AVX512F-NEXT:    vpmovmskb %ymm2, %eax
 ; AVX512F-NEXT:    shlq $32, %rax
 ; AVX512F-NEXT:    orq %rcx, %rax
 ; AVX512F-NEXT:    vzeroupper
@@ -446,6 +450,7 @@ define void @bitcast_64i8_store(i64* %p, <64 x i8> %a0) {
 ;
 ; AVX512F-LABEL: bitcast_64i8_store:
 ; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; AVX512F-NEXT:    vpcmpgtb %ymm0, %ymm2, %ymm0
 ; AVX512F-NEXT:    vpmovsxbd %xmm0, %zmm3
@@ -515,6 +520,7 @@ define void @bitcast_32i16_store(i32* %p, <32 x i16> %a0) {
 ;
 ; AVX512F-LABEL: bitcast_32i16_store:
 ; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; AVX512F-NEXT:    vpcmpgtw %ymm0, %ymm2, %ymm0
 ; AVX512F-NEXT:    vpmovsxwd %ymm0, %zmm0
diff --git a/llvm/test/CodeGen/X86/bitcast-vector-bool.ll b/llvm/test/CodeGen/X86/bitcast-vector-bool.ll
index 1e402ae0e..083d0c6f3 100644
--- a/llvm/test/CodeGen/X86/bitcast-vector-bool.ll
+++ b/llvm/test/CodeGen/X86/bitcast-vector-bool.ll
@@ -151,27 +151,14 @@ define i4 @bitcast_v8i16_to_v2i4(<8 x i16> %a0) nounwind {
 }
 
 define i8 @bitcast_v16i8_to_v2i8(<16 x i8> %a0) nounwind {
-; SSE2-LABEL: bitcast_v16i8_to_v2i8:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    pmovmskb %xmm0, %eax
-; SSE2-NEXT:    movd %eax, %xmm0
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,1,3]
-; SSE2-NEXT:    movdqa %xmm0, -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movb -{{[0-9]+}}(%rsp), %al
-; SSE2-NEXT:    addb -{{[0-9]+}}(%rsp), %al
-; SSE2-NEXT:    retq
-;
-; SSSE3-LABEL: bitcast_v16i8_to_v2i8:
-; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    pmovmskb %xmm0, %eax
-; SSSE3-NEXT:    movd %eax, %xmm0
-; SSSE3-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
-; SSSE3-NEXT:    movdqa %xmm0, -{{[0-9]+}}(%rsp)
-; SSSE3-NEXT:    movb -{{[0-9]+}}(%rsp), %al
-; SSSE3-NEXT:    addb -{{[0-9]+}}(%rsp), %al
-; SSSE3-NEXT:    retq
+; SSE2-SSSE3-LABEL: bitcast_v16i8_to_v2i8:
+; SSE2-SSSE3:       # %bb.0:
+; SSE2-SSSE3-NEXT:    pmovmskb %xmm0, %eax
+; SSE2-SSSE3-NEXT:    movd %eax, %xmm0
+; SSE2-SSSE3-NEXT:    movdqa %xmm0, -{{[0-9]+}}(%rsp)
+; SSE2-SSSE3-NEXT:    movb -{{[0-9]+}}(%rsp), %al
+; SSE2-SSSE3-NEXT:    addb -{{[0-9]+}}(%rsp), %al
+; SSE2-SSSE3-NEXT:    retq
 ;
 ; AVX12-LABEL: bitcast_v16i8_to_v2i8:
 ; AVX12:       # %bb.0:
@@ -187,7 +174,7 @@ define i8 @bitcast_v16i8_to_v2i8(<16 x i8> %a0) nounwind {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpmovb2m %xmm0, %k0
 ; AVX512-NEXT:    kmovw %k0, -{{[0-9]+}}(%rsp)
-; AVX512-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX512-NEXT:    vmovdqa -{{[0-9]+}}(%rsp), %xmm0
 ; AVX512-NEXT:    vpextrb $0, %xmm0, %ecx
 ; AVX512-NEXT:    vpextrb $1, %xmm0, %eax
 ; AVX512-NEXT:    addb %cl, %al
@@ -318,29 +305,15 @@ define i4 @bitcast_v8i32_to_v2i4(<8 x i32> %a0) nounwind {
 }
 
 define i8 @bitcast_v16i16_to_v2i8(<16 x i16> %a0) nounwind {
-; SSE2-LABEL: bitcast_v16i16_to_v2i8:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    packsswb %xmm1, %xmm0
-; SSE2-NEXT:    pmovmskb %xmm0, %eax
-; SSE2-NEXT:    movd %eax, %xmm0
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,1,3]
-; SSE2-NEXT:    movdqa %xmm0, -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movb -{{[0-9]+}}(%rsp), %al
-; SSE2-NEXT:    addb -{{[0-9]+}}(%rsp), %al
-; SSE2-NEXT:    retq
-;
-; SSSE3-LABEL: bitcast_v16i16_to_v2i8:
-; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    packsswb %xmm1, %xmm0
-; SSSE3-NEXT:    pmovmskb %xmm0, %eax
-; SSSE3-NEXT:    movd %eax, %xmm0
-; SSSE3-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
-; SSSE3-NEXT:    movdqa %xmm0, -{{[0-9]+}}(%rsp)
-; SSSE3-NEXT:    movb -{{[0-9]+}}(%rsp), %al
-; SSSE3-NEXT:    addb -{{[0-9]+}}(%rsp), %al
-; SSSE3-NEXT:    retq
+; SSE2-SSSE3-LABEL: bitcast_v16i16_to_v2i8:
+; SSE2-SSSE3:       # %bb.0:
+; SSE2-SSSE3-NEXT:    packsswb %xmm1, %xmm0
+; SSE2-SSSE3-NEXT:    pmovmskb %xmm0, %eax
+; SSE2-SSSE3-NEXT:    movd %eax, %xmm0
+; SSE2-SSSE3-NEXT:    movdqa %xmm0, -{{[0-9]+}}(%rsp)
+; SSE2-SSSE3-NEXT:    movb -{{[0-9]+}}(%rsp), %al
+; SSE2-SSSE3-NEXT:    addb -{{[0-9]+}}(%rsp), %al
+; SSE2-SSSE3-NEXT:    retq
 ;
 ; AVX1-LABEL: bitcast_v16i16_to_v2i8:
 ; AVX1:       # %bb.0:
@@ -374,7 +347,7 @@ define i8 @bitcast_v16i16_to_v2i8(<16 x i16> %a0) nounwind {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpmovw2m %ymm0, %k0
 ; AVX512-NEXT:    kmovw %k0, -{{[0-9]+}}(%rsp)
-; AVX512-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX512-NEXT:    vmovdqa -{{[0-9]+}}(%rsp), %xmm0
 ; AVX512-NEXT:    vpextrb $0, %xmm0, %ecx
 ; AVX512-NEXT:    vpextrb $1, %xmm0, %eax
 ; AVX512-NEXT:    addb %cl, %al
@@ -392,12 +365,10 @@ define i8 @bitcast_v16i16_to_v2i8(<16 x i16> %a0) nounwind {
 define i16 @bitcast_v32i8_to_v2i16(<32 x i8> %a0) nounwind {
 ; SSE2-SSSE3-LABEL: bitcast_v32i8_to_v2i16:
 ; SSE2-SSSE3:       # %bb.0:
-; SSE2-SSSE3-NEXT:    pmovmskb %xmm0, %eax
-; SSE2-SSSE3-NEXT:    pmovmskb %xmm1, %ecx
-; SSE2-SSSE3-NEXT:    shll $16, %ecx
-; SSE2-SSSE3-NEXT:    orl %eax, %ecx
-; SSE2-SSSE3-NEXT:    movd %ecx, %xmm0
-; SSE2-SSSE3-NEXT:    pextrw $0, %xmm0, %ecx
+; SSE2-SSSE3-NEXT:    pmovmskb %xmm0, %ecx
+; SSE2-SSSE3-NEXT:    pmovmskb %xmm1, %eax
+; SSE2-SSSE3-NEXT:    shll $16, %eax
+; SSE2-SSSE3-NEXT:    movd %eax, %xmm0
 ; SSE2-SSSE3-NEXT:    pextrw $1, %xmm0, %eax
 ; SSE2-SSSE3-NEXT:    addl %ecx, %eax
 ; SSE2-SSSE3-NEXT:    # kill: def $ax killed $ax killed $eax
@@ -405,13 +376,11 @@ define i16 @bitcast_v32i8_to_v2i16(<32 x i8> %a0) nounwind {
 ;
 ; AVX1-LABEL: bitcast_v32i8_to_v2i16:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpmovmskb %xmm0, %eax
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; AVX1-NEXT:    vpmovmskb %xmm0, %ecx
-; AVX1-NEXT:    shll $16, %ecx
-; AVX1-NEXT:    orl %eax, %ecx
-; AVX1-NEXT:    vmovd %ecx, %xmm0
-; AVX1-NEXT:    vpextrw $0, %xmm0, %ecx
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vpmovmskb %xmm0, %eax
+; AVX1-NEXT:    shll $16, %eax
+; AVX1-NEXT:    vmovd %eax, %xmm0
 ; AVX1-NEXT:    vpextrw $1, %xmm0, %eax
 ; AVX1-NEXT:    addl %ecx, %eax
 ; AVX1-NEXT:    # kill: def $ax killed $ax killed $eax
@@ -420,9 +389,8 @@ define i16 @bitcast_v32i8_to_v2i16(<32 x i8> %a0) nounwind {
 ;
 ; AVX2-LABEL: bitcast_v32i8_to_v2i16:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpmovmskb %ymm0, %eax
-; AVX2-NEXT:    vmovd %eax, %xmm0
-; AVX2-NEXT:    vpextrw $0, %xmm0, %ecx
+; AVX2-NEXT:    vpmovmskb %ymm0, %ecx
+; AVX2-NEXT:    vmovd %ecx, %xmm0
 ; AVX2-NEXT:    vpextrw $1, %xmm0, %eax
 ; AVX2-NEXT:    addl %ecx, %eax
 ; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax
@@ -437,8 +405,8 @@ define i16 @bitcast_v32i8_to_v2i16(<32 x i8> %a0) nounwind {
 ; AVX512-NEXT:    subq $32, %rsp
 ; AVX512-NEXT:    vpmovb2m %ymm0, %k0
 ; AVX512-NEXT:    kmovd %k0, (%rsp)
-; AVX512-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX512-NEXT:    vpextrw $0, %xmm0, %ecx
+; AVX512-NEXT:    vmovdqa (%rsp), %xmm0
+; AVX512-NEXT:    vmovd %xmm0, %ecx
 ; AVX512-NEXT:    vpextrw $1, %xmm0, %eax
 ; AVX512-NEXT:    addl %ecx, %eax
 ; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
@@ -461,48 +429,11 @@ define i16 @bitcast_v32i8_to_v2i16(<32 x i8> %a0) nounwind {
 define i4 @bitcast_v8i64_to_v2i4(<8 x i64> %a0) nounwind {
 ; SSE2-SSSE3-LABEL: bitcast_v8i64_to_v2i4:
 ; SSE2-SSSE3:       # %bb.0:
-; SSE2-SSSE3-NEXT:    movdqa {{.*#+}} xmm4 = [2147483648,2147483648]
-; SSE2-SSSE3-NEXT:    pxor %xmm4, %xmm3
-; SSE2-SSSE3-NEXT:    movdqa %xmm4, %xmm5
-; SSE2-SSSE3-NEXT:    pcmpgtd %xmm3, %xmm5
-; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSE2-SSSE3-NEXT:    pcmpeqd %xmm4, %xmm3
-; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE2-SSSE3-NEXT:    pand %xmm6, %xmm3
-; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE2-SSSE3-NEXT:    por %xmm3, %xmm5
-; SSE2-SSSE3-NEXT:    pxor %xmm4, %xmm2
-; SSE2-SSSE3-NEXT:    movdqa %xmm4, %xmm3
-; SSE2-SSSE3-NEXT:    pcmpgtd %xmm2, %xmm3
-; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[0,0,2,2]
-; SSE2-SSSE3-NEXT:    pcmpeqd %xmm4, %xmm2
-; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm2[1,1,3,3]
-; SSE2-SSSE3-NEXT:    pand %xmm6, %xmm7
-; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,1,3,3]
-; SSE2-SSSE3-NEXT:    por %xmm7, %xmm2
-; SSE2-SSSE3-NEXT:    packssdw %xmm5, %xmm2
-; SSE2-SSSE3-NEXT:    pxor %xmm4, %xmm1
-; SSE2-SSSE3-NEXT:    movdqa %xmm4, %xmm3
-; SSE2-SSSE3-NEXT:    pcmpgtd %xmm1, %xmm3
-; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[0,0,2,2]
-; SSE2-SSSE3-NEXT:    pcmpeqd %xmm4, %xmm1
-; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-SSSE3-NEXT:    pand %xmm5, %xmm1
-; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE2-SSSE3-NEXT:    por %xmm1, %xmm3
-; SSE2-SSSE3-NEXT:    pxor %xmm4, %xmm0
-; SSE2-SSSE3-NEXT:    movdqa %xmm4, %xmm1
-; SSE2-SSSE3-NEXT:    pcmpgtd %xmm0, %xmm1
-; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[0,0,2,2]
-; SSE2-SSSE3-NEXT:    pcmpeqd %xmm4, %xmm0
-; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-SSSE3-NEXT:    pand %xmm5, %xmm0
-; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-SSSE3-NEXT:    por %xmm0, %xmm1
-; SSE2-SSSE3-NEXT:    packssdw %xmm3, %xmm1
-; SSE2-SSSE3-NEXT:    packssdw %xmm2, %xmm1
-; SSE2-SSSE3-NEXT:    packsswb %xmm0, %xmm1
-; SSE2-SSSE3-NEXT:    pmovmskb %xmm1, %eax
+; SSE2-SSSE3-NEXT:    packssdw %xmm3, %xmm2
+; SSE2-SSSE3-NEXT:    packssdw %xmm1, %xmm0
+; SSE2-SSSE3-NEXT:    packssdw %xmm2, %xmm0
+; SSE2-SSSE3-NEXT:    packsswb %xmm0, %xmm0
+; SSE2-SSSE3-NEXT:    pmovmskb %xmm0, %eax
 ; SSE2-SSSE3-NEXT:    movzbl %al, %ecx
 ; SSE2-SSSE3-NEXT:    shrl $4, %ecx
 ; SSE2-SSSE3-NEXT:    movq %rcx, %xmm0
@@ -579,33 +510,17 @@ define i4 @bitcast_v8i64_to_v2i4(<8 x i64> %a0) nounwind {
 }
 
 define i8 @bitcast_v16i32_to_v2i8(<16 x i32> %a0) nounwind {
-; SSE2-LABEL: bitcast_v16i32_to_v2i8:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    packssdw %xmm3, %xmm2
-; SSE2-NEXT:    packssdw %xmm1, %xmm0
-; SSE2-NEXT:    packsswb %xmm2, %xmm0
-; SSE2-NEXT:    pmovmskb %xmm0, %eax
-; SSE2-NEXT:    movd %eax, %xmm0
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,1,3]
-; SSE2-NEXT:    movdqa %xmm0, -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movb -{{[0-9]+}}(%rsp), %al
-; SSE2-NEXT:    addb -{{[0-9]+}}(%rsp), %al
-; SSE2-NEXT:    retq
-;
-; SSSE3-LABEL: bitcast_v16i32_to_v2i8:
-; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    packssdw %xmm3, %xmm2
-; SSSE3-NEXT:    packssdw %xmm1, %xmm0
-; SSSE3-NEXT:    packsswb %xmm2, %xmm0
-; SSSE3-NEXT:    pmovmskb %xmm0, %eax
-; SSSE3-NEXT:    movd %eax, %xmm0
-; SSSE3-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
-; SSSE3-NEXT:    movdqa %xmm0, -{{[0-9]+}}(%rsp)
-; SSSE3-NEXT:    movb -{{[0-9]+}}(%rsp), %al
-; SSSE3-NEXT:    addb -{{[0-9]+}}(%rsp), %al
-; SSSE3-NEXT:    retq
+; SSE2-SSSE3-LABEL: bitcast_v16i32_to_v2i8:
+; SSE2-SSSE3:       # %bb.0:
+; SSE2-SSSE3-NEXT:    packssdw %xmm3, %xmm2
+; SSE2-SSSE3-NEXT:    packssdw %xmm1, %xmm0
+; SSE2-SSSE3-NEXT:    packsswb %xmm2, %xmm0
+; SSE2-SSSE3-NEXT:    pmovmskb %xmm0, %eax
+; SSE2-SSSE3-NEXT:    movd %eax, %xmm0
+; SSE2-SSSE3-NEXT:    movdqa %xmm0, -{{[0-9]+}}(%rsp)
+; SSE2-SSSE3-NEXT:    movb -{{[0-9]+}}(%rsp), %al
+; SSE2-SSSE3-NEXT:    addb -{{[0-9]+}}(%rsp), %al
+; SSE2-SSSE3-NEXT:    retq
 ;
 ; AVX1-LABEL: bitcast_v16i32_to_v2i8:
 ; AVX1:       # %bb.0:
@@ -646,7 +561,7 @@ define i8 @bitcast_v16i32_to_v2i8(<16 x i32> %a0) nounwind {
 ; AVX512-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX512-NEXT:    vpcmpgtd %zmm0, %zmm1, %k0
 ; AVX512-NEXT:    kmovw %k0, -{{[0-9]+}}(%rsp)
-; AVX512-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX512-NEXT:    vmovdqa -{{[0-9]+}}(%rsp), %xmm0
 ; AVX512-NEXT:    vpextrb $0, %xmm0, %ecx
 ; AVX512-NEXT:    vpextrb $1, %xmm0, %eax
 ; AVX512-NEXT:    addb %cl, %al
@@ -665,13 +580,11 @@ define i16 @bitcast_v32i16_to_v2i16(<32 x i16> %a0) nounwind {
 ; SSE2-SSSE3-LABEL: bitcast_v32i16_to_v2i16:
 ; SSE2-SSSE3:       # %bb.0:
 ; SSE2-SSSE3-NEXT:    packsswb %xmm1, %xmm0
-; SSE2-SSSE3-NEXT:    pmovmskb %xmm0, %eax
+; SSE2-SSSE3-NEXT:    pmovmskb %xmm0, %ecx
 ; SSE2-SSSE3-NEXT:    packsswb %xmm3, %xmm2
-; SSE2-SSSE3-NEXT:    pmovmskb %xmm2, %ecx
-; SSE2-SSSE3-NEXT:    shll $16, %ecx
-; SSE2-SSSE3-NEXT:    orl %eax, %ecx
-; SSE2-SSSE3-NEXT:    movd %ecx, %xmm0
-; SSE2-SSSE3-NEXT:    pextrw $0, %xmm0, %ecx
+; SSE2-SSSE3-NEXT:    pmovmskb %xmm2, %eax
+; SSE2-SSSE3-NEXT:    shll $16, %eax
+; SSE2-SSSE3-NEXT:    movd %eax, %xmm0
 ; SSE2-SSSE3-NEXT:    pextrw $1, %xmm0, %eax
 ; SSE2-SSSE3-NEXT:    addl %ecx, %eax
 ; SSE2-SSSE3-NEXT:    # kill: def $ax killed $ax killed $eax
@@ -681,14 +594,12 @@ define i16 @bitcast_v32i16_to_v2i16(<32 x i16> %a0) nounwind {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
 ; AVX1-NEXT:    vpacksswb %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpmovmskb %xmm0, %eax
+; AVX1-NEXT:    vpmovmskb %xmm0, %ecx
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm0
 ; AVX1-NEXT:    vpacksswb %xmm0, %xmm1, %xmm0
-; AVX1-NEXT:    vpmovmskb %xmm0, %ecx
-; AVX1-NEXT:    shll $16, %ecx
-; AVX1-NEXT:    orl %eax, %ecx
-; AVX1-NEXT:    vmovd %ecx, %xmm0
-; AVX1-NEXT:    vpextrw $0, %xmm0, %ecx
+; AVX1-NEXT:    vpmovmskb %xmm0, %eax
+; AVX1-NEXT:    shll $16, %eax
+; AVX1-NEXT:    vmovd %eax, %xmm0
 ; AVX1-NEXT:    vpextrw $1, %xmm0, %eax
 ; AVX1-NEXT:    addl %ecx, %eax
 ; AVX1-NEXT:    # kill: def $ax killed $ax killed $eax
@@ -699,9 +610,8 @@ define i16 @bitcast_v32i16_to_v2i16(<32 x i16> %a0) nounwind {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpacksswb %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX2-NEXT:    vpmovmskb %ymm0, %eax
-; AVX2-NEXT:    vmovd %eax, %xmm0
-; AVX2-NEXT:    vpextrw $0, %xmm0, %ecx
+; AVX2-NEXT:    vpmovmskb %ymm0, %ecx
+; AVX2-NEXT:    vmovd %ecx, %xmm0
 ; AVX2-NEXT:    vpextrw $1, %xmm0, %eax
 ; AVX2-NEXT:    addl %ecx, %eax
 ; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax
@@ -716,8 +626,8 @@ define i16 @bitcast_v32i16_to_v2i16(<32 x i16> %a0) nounwind {
 ; AVX512-NEXT:    subq $32, %rsp
 ; AVX512-NEXT:    vpmovw2m %zmm0, %k0
 ; AVX512-NEXT:    kmovd %k0, (%rsp)
-; AVX512-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX512-NEXT:    vpextrw $0, %xmm0, %ecx
+; AVX512-NEXT:    vmovdqa (%rsp), %xmm0
+; AVX512-NEXT:    vmovd %xmm0, %ecx
 ; AVX512-NEXT:    vpextrw $1, %xmm0, %eax
 ; AVX512-NEXT:    addl %ecx, %eax
 ; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
@@ -984,9 +894,9 @@ define i32 @bitcast_v64i8_to_v2i32(<64 x i8> %a0) nounwind {
 ; SSE2-SSSE3-NEXT:    orl %ecx, %edx
 ; SSE2-SSSE3-NEXT:    orl %eax, %edx
 ; SSE2-SSSE3-NEXT:    movw %dx, -{{[0-9]+}}(%rsp)
-; SSE2-SSSE3-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; SSE2-SSSE3-NEXT:    movdqa -{{[0-9]+}}(%rsp), %xmm0
 ; SSE2-SSSE3-NEXT:    movd %xmm0, %ecx
-; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,0,1]
+; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
 ; SSE2-SSSE3-NEXT:    movd %xmm0, %eax
 ; SSE2-SSSE3-NEXT:    addl %ecx, %eax
 ; SSE2-SSSE3-NEXT:    retq
@@ -1246,7 +1156,7 @@ define i32 @bitcast_v64i8_to_v2i32(<64 x i8> %a0) nounwind {
 ; AVX1-NEXT:    orl %ecx, %edx
 ; AVX1-NEXT:    orl %eax, %edx
 ; AVX1-NEXT:    movl %edx, -{{[0-9]+}}(%rsp)
-; AVX1-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
+; AVX1-NEXT:    vmovdqa -{{[0-9]+}}(%rsp), %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %ecx
 ; AVX1-NEXT:    vpextrd $1, %xmm0, %eax
 ; AVX1-NEXT:    addl %ecx, %eax
@@ -1506,7 +1416,7 @@ define i32 @bitcast_v64i8_to_v2i32(<64 x i8> %a0) nounwind {
 ; AVX2-NEXT:    orl %ecx, %edx
 ; AVX2-NEXT:    orl %eax, %edx
 ; AVX2-NEXT:    movl %edx, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
+; AVX2-NEXT:    vmovdqa -{{[0-9]+}}(%rsp), %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %ecx
 ; AVX2-NEXT:    vpextrd $1, %xmm0, %eax
 ; AVX2-NEXT:    addl %ecx, %eax
@@ -1517,7 +1427,7 @@ define i32 @bitcast_v64i8_to_v2i32(<64 x i8> %a0) nounwind {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpmovb2m %zmm0, %k0
 ; AVX512-NEXT:    kmovq %k0, -{{[0-9]+}}(%rsp)
-; AVX512-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
+; AVX512-NEXT:    vmovdqa -{{[0-9]+}}(%rsp), %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %ecx
 ; AVX512-NEXT:    vpextrd $1, %xmm0, %eax
 ; AVX512-NEXT:    addl %ecx, %eax
diff --git a/llvm/test/CodeGen/X86/bitcast2.ll b/llvm/test/CodeGen/X86/bitcast2.ll
index febd7ba12..381a75edf 100644
--- a/llvm/test/CodeGen/X86/bitcast2.ll
+++ b/llvm/test/CodeGen/X86/bitcast2.ll
@@ -1,12 +1,20 @@
-; RUN: llc < %s -mtriple=x86_64-- -mattr=-avx | grep movq | count 2
-; RUN: llc < %s -mtriple=x86_64-- -mattr=-avx | not grep rsp
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-- -mattr=-avx | FileCheck %s
 
 define i64 @test1(double %A) {
+; CHECK-LABEL: test1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %xmm0, %rax
+; CHECK-NEXT:    retq
    %B = bitcast double %A to i64
    ret i64 %B
 }
 
 define double @test2(i64 %A) {
+; CHECK-LABEL: test2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %rdi, %xmm0
+; CHECK-NEXT:    retq
    %B = bitcast i64 %A to double
    ret double %B
 }
diff --git a/llvm/test/CodeGen/X86/bitreverse.ll b/llvm/test/CodeGen/X86/bitreverse.ll
index 5dd2b36be..e2b58f9a2 100644
--- a/llvm/test/CodeGen/X86/bitreverse.ll
+++ b/llvm/test/CodeGen/X86/bitreverse.ll
@@ -55,13 +55,11 @@ define <2 x i16> @test_bitreverse_v2i16(<2 x i16> %a) nounwind {
 ; X64-NEXT:    pxor %xmm1, %xmm1
 ; X64-NEXT:    movdqa %xmm0, %xmm2
 ; X64-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
-; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
-; X64-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[3,2,1,0,4,5,6,7]
-; X64-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,6,5,4]
+; X64-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[1,0,3,2,4,5,6,7]
+; X64-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,4,7,6]
 ; X64-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
-; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; X64-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[3,2,1,0,4,5,6,7]
-; X64-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,6,5,4]
+; X64-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[1,0,3,2,4,5,6,7]
+; X64-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,4,7,6]
 ; X64-NEXT:    packuswb %xmm2, %xmm0
 ; X64-NEXT:    movdqa %xmm0, %xmm1
 ; X64-NEXT:    psllw $4, %xmm1
@@ -81,7 +79,6 @@ define <2 x i16> @test_bitreverse_v2i16(<2 x i16> %a) nounwind {
 ; X64-NEXT:    pand {{.*}}(%rip), %xmm0
 ; X64-NEXT:    psrlw $1, %xmm0
 ; X64-NEXT:    por %xmm1, %xmm0
-; X64-NEXT:    psrlq $48, %xmm0
 ; X64-NEXT:    retq
   %b = call <2 x i16> @llvm.bitreverse.v2i16(<2 x i16> %a)
   ret <2 x i16> %b
@@ -410,7 +407,7 @@ define <2 x i16> @fold_v2i16() {
 ;
 ; X64-LABEL: fold_v2i16:
 ; X64:       # %bb.0:
-; X64-NEXT:    movaps {{.*#+}} xmm0 = [61440,240]
+; X64-NEXT:    movaps {{.*#+}} xmm0 = <61440,240,u,u,u,u,u,u>
 ; X64-NEXT:    retq
   %b = call <2 x i16> @llvm.bitreverse.v2i16(<2 x i16> <i16 15, i16 3840>)
   ret <2 x i16> %b
diff --git a/llvm/test/CodeGen/X86/block-placement.ll b/llvm/test/CodeGen/X86/block-placement.ll
index acc4b7e13..258cc2031 100644
--- a/llvm/test/CodeGen/X86/block-placement.ll
+++ b/llvm/test/CodeGen/X86/block-placement.ll
@@ -358,11 +358,11 @@ define void @unnatural_cfg2(i32* %p0, i32 %a0) {
 ; CHECK: %loop.header
 ; CHECK: %loop.body1
 ; CHECK: %loop.body2
+; CHECK: %loop.body3
+; CHECK: %loop.inner1.begin
 ; CHECK: %loop.body4
 ; CHECK: %loop.inner2.begin
 ; CHECK: %loop.inner2.begin
-; CHECK: %loop.body3
-; CHECK: %loop.inner1.begin
 ; CHECK: %bail
 
 entry:
diff --git a/llvm/test/CodeGen/X86/bmi-x86_64.ll b/llvm/test/CodeGen/X86/bmi-x86_64.ll
index 81ac31329..7646fd6bb 100644
--- a/llvm/test/CodeGen/X86/bmi-x86_64.ll
+++ b/llvm/test/CodeGen/X86/bmi-x86_64.ll
@@ -86,9 +86,9 @@ define i64 @bextr64d(i64 %a) {
 ;
 ; BMI2-SLOW-LABEL: bextr64d:
 ; BMI2-SLOW:       # %bb.0: # %entry
-; BMI2-SLOW-NEXT:    shrq $2, %rdi
-; BMI2-SLOW-NEXT:    movb $33, %al
+; BMI2-SLOW-NEXT:    movl $35, %eax
 ; BMI2-SLOW-NEXT:    bzhiq %rax, %rdi, %rax
+; BMI2-SLOW-NEXT:    shrq $2, %rax
 ; BMI2-SLOW-NEXT:    retq
 ;
 ; BEXTR-FAST-LABEL: bextr64d:
@@ -102,6 +102,34 @@ entry:
   ret i64 %and
 }
 
+define i64 @bextr64d_load(i64* %aptr) {
+; BMI1-SLOW-LABEL: bextr64d_load:
+; BMI1-SLOW:       # %bb.0: # %entry
+; BMI1-SLOW-NEXT:    movq (%rdi), %rax
+; BMI1-SLOW-NEXT:    shrq $2, %rax
+; BMI1-SLOW-NEXT:    movl $8448, %ecx # imm = 0x2100
+; BMI1-SLOW-NEXT:    bextrq %rcx, %rax, %rax
+; BMI1-SLOW-NEXT:    retq
+;
+; BMI2-SLOW-LABEL: bextr64d_load:
+; BMI2-SLOW:       # %bb.0: # %entry
+; BMI2-SLOW-NEXT:    movl $35, %eax
+; BMI2-SLOW-NEXT:    bzhiq %rax, (%rdi), %rax
+; BMI2-SLOW-NEXT:    shrq $2, %rax
+; BMI2-SLOW-NEXT:    retq
+;
+; BEXTR-FAST-LABEL: bextr64d_load:
+; BEXTR-FAST:       # %bb.0: # %entry
+; BEXTR-FAST-NEXT:    movl $8450, %eax # imm = 0x2102
+; BEXTR-FAST-NEXT:    bextrq %rax, (%rdi), %rax
+; BEXTR-FAST-NEXT:    retq
+entry:
+  %a = load i64, i64* %aptr, align 8
+  %shr = lshr i64 %a, 2
+  %and = and i64 %shr, 8589934591
+  ret i64 %and
+}
+
 define i64 @non_bextr64(i64 %x) {
 ; CHECK-LABEL: non_bextr64:
 ; CHECK:       # %bb.0: # %entry
diff --git a/llvm/test/CodeGen/X86/bmi.ll b/llvm/test/CodeGen/X86/bmi.ll
index d5603431e..9c3a74fcc 100644
--- a/llvm/test/CodeGen/X86/bmi.ll
+++ b/llvm/test/CodeGen/X86/bmi.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+cmov,+bmi | FileCheck %s --check-prefixes=CHECK,X86,X86-SLOW-BEXTR
-; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+cmov,+bmi,+bmi2 | FileCheck %s --check-prefixes=CHECK,X86,X86-SLOW-BEXTR
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+cmov,+bmi,+bmi2 | FileCheck %s --check-prefixes=CHECK,X86,X86-SLOW-BEXTR,X86-SLOW-BEXTR-BMI2
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+bmi | FileCheck %s --check-prefixes=CHECK,X64,X64-SLOW-BEXTR
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+bmi,+bmi2 | FileCheck %s --check-prefixes=CHECK,X64,X64-SLOW-BEXTR
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+bmi,+bmi2 | FileCheck %s --check-prefixes=CHECK,X64,X64-SLOW-BEXTR,X64-SLOW-BEXTR-BMI2
 ; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+cmov,+bmi,+fast-bextr | FileCheck %s --check-prefixes=CHECK,X86,X86-FAST-BEXTR
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+bmi,+fast-bextr | FileCheck %s --check-prefixes=CHECK,X64,X64-FAST-BEXTR
 
diff --git a/llvm/test/CodeGen/X86/br-fold.ll b/llvm/test/CodeGen/X86/br-fold.ll
index bcd672eb5..6cd9b78c1 100644
--- a/llvm/test/CodeGen/X86/br-fold.ll
+++ b/llvm/test/CodeGen/X86/br-fold.ll
@@ -5,18 +5,18 @@
 ; RUN: llc -mtriple=x86_64-scei-ps4 < %s | FileCheck -check-prefix=PS4 %s
 
 ; X64_DARWIN: orq
-; X64-DARWIN-NEXT: ud2
+; X64_DARWIN-NEXT: ud2
 
 ; X64_LINUX: orq %rax, %rcx
 ; X64_LINUX-NEXT: jne
 ; X64_LINUX-NEXT: %bb8.i329
 
 ; X64_WINDOWS: orq %rax, %rcx
-; X64_WINDOWS-NEXT: ud2
+; X64_WINDOWS-NEXT: jne
 
 ; X64_WINDOWS_GNU: movq .refptr._ZN11xercesc_2_513SchemaSymbols21fgURI_SCHEMAFORSCHEMAE(%rip), %rax
 ; X64_WINDOWS_GNU: orq .refptr._ZN11xercesc_2_56XMLUni16fgNotationStringE(%rip), %rax
-; X64_WINDOWS_GNU-NEXT: ud2
+; X64_WINDOWS_GNU-NEXT: jne
 
 ; PS4: orq %rax, %rcx
 ; PS4-NEXT: ud2
diff --git a/llvm/test/CodeGen/X86/break-anti-dependencies.ll b/llvm/test/CodeGen/X86/break-anti-dependencies.ll
index e8e270924..4867acf67 100644
--- a/llvm/test/CodeGen/X86/break-anti-dependencies.ll
+++ b/llvm/test/CodeGen/X86/break-anti-dependencies.ll
@@ -1,14 +1,58 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; Without list-burr scheduling we may not see the difference in codegen here.
 ; Use a subtarget that has post-RA scheduling enabled because the anti-dependency
 ; breaker requires liveness information to be kept.
-; RUN: llc < %s -mtriple=x86_64-- -mcpu=atom -enable-misched=false -post-RA-scheduler -pre-RA-sched=list-burr -break-anti-dependencies=none > %t
-; RUN:   grep "%xmm0" %t | count 14
-; RUN:   not grep "%xmm1" %t
-; RUN: llc < %s -mtriple=x86_64-- -mcpu=atom -post-RA-scheduler -break-anti-dependencies=critical > %t
-; RUN:   grep "%xmm0" %t | count 7
-; RUN:   grep "%xmm1" %t | count 7
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=atom -enable-misched=false -post-RA-scheduler -pre-RA-sched=list-burr -break-anti-dependencies=none | FileCheck %s --check-prefix=none
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=atom -post-RA-scheduler -break-anti-dependencies=critical | FileCheck %s --check-prefix=critical
 
 define void @goo(double* %r, double* %p, double* %q) nounwind {
+; none-LABEL: goo:
+; none:       # %bb.0: # %entry
+; none-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; none-NEXT:    addsd {{.*}}(%rip), %xmm0
+; none-NEXT:    mulsd {{.*}}(%rip), %xmm0
+; none-NEXT:    addsd {{.*}}(%rip), %xmm0
+; none-NEXT:    mulsd {{.*}}(%rip), %xmm0
+; none-NEXT:    addsd {{.*}}(%rip), %xmm0
+; none-NEXT:    cvttsd2si %xmm0, %eax
+; none-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; none-NEXT:    addsd {{.*}}(%rip), %xmm0
+; none-NEXT:    mulsd {{.*}}(%rip), %xmm0
+; none-NEXT:    addsd {{.*}}(%rip), %xmm0
+; none-NEXT:    mulsd {{.*}}(%rip), %xmm0
+; none-NEXT:    addsd {{.*}}(%rip), %xmm0
+; none-NEXT:    cvttsd2si %xmm0, %ecx
+; none-NEXT:    cmpl %eax, %ecx
+; none-NEXT:    jge .LBB0_2
+; none-NEXT:  # %bb.1: # %bb
+; none-NEXT:    movabsq $4621425052621576602, %rax # imm = 0x402299999999999A
+; none-NEXT:    movq %rax, (%rdx)
+; none-NEXT:  .LBB0_2: # %return
+; none-NEXT:    retq
+;
+; critical-LABEL: goo:
+; critical:       # %bb.0: # %entry
+; critical-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; critical-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
+; critical-NEXT:    addsd {{.*}}(%rip), %xmm0
+; critical-NEXT:    addsd {{.*}}(%rip), %xmm1
+; critical-NEXT:    mulsd {{.*}}(%rip), %xmm0
+; critical-NEXT:    mulsd {{.*}}(%rip), %xmm1
+; critical-NEXT:    addsd {{.*}}(%rip), %xmm0
+; critical-NEXT:    addsd {{.*}}(%rip), %xmm1
+; critical-NEXT:    mulsd {{.*}}(%rip), %xmm0
+; critical-NEXT:    mulsd {{.*}}(%rip), %xmm1
+; critical-NEXT:    addsd {{.*}}(%rip), %xmm0
+; critical-NEXT:    addsd {{.*}}(%rip), %xmm1
+; critical-NEXT:    cvttsd2si %xmm0, %eax
+; critical-NEXT:    cvttsd2si %xmm1, %ecx
+; critical-NEXT:    cmpl %ecx, %eax
+; critical-NEXT:    jge .LBB0_2
+; critical-NEXT:  # %bb.1: # %bb
+; critical-NEXT:    movabsq $4621425052621576602, %rax # imm = 0x402299999999999A
+; critical-NEXT:    movq %rax, (%rdx)
+; critical-NEXT:  .LBB0_2: # %return
+; critical-NEXT:    retq
 entry:
 	%0 = load double, double* %p, align 8
 	%1 = fadd double %0, 1.100000e+00
diff --git a/llvm/test/CodeGen/X86/broadcast-elm-cross-splat-vec.ll b/llvm/test/CodeGen/X86/broadcast-elm-cross-splat-vec.ll
index e947deea5..d4726f50a 100644
--- a/llvm/test/CodeGen/X86/broadcast-elm-cross-splat-vec.ll
+++ b/llvm/test/CodeGen/X86/broadcast-elm-cross-splat-vec.ll
@@ -301,14 +301,14 @@ define <64 x i8> @f64xi8_i16(<64 x i8> %a) {
 ; AVX-NEXT:    vandps %ymm2, %ymm1, %ymm1
 ; AVX-NEXT:    retl
 ;
-; NO-AVX512BW-LABEL: f64xi8_i16:
-; NO-AVX512BW:       # %bb.0:
-; NO-AVX512BW-NEXT:    vpbroadcastw {{.*#+}} ymm2 = [256,256,256,256,256,256,256,256,256,256,256,256,256,256,256,256]
-; NO-AVX512BW-NEXT:    vpaddb %ymm2, %ymm1, %ymm1
-; NO-AVX512BW-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
-; NO-AVX512BW-NEXT:    vpand %ymm2, %ymm0, %ymm0
-; NO-AVX512BW-NEXT:    vpand %ymm2, %ymm1, %ymm1
-; NO-AVX512BW-NEXT:    retl
+; AVX2-LABEL: f64xi8_i16:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpbroadcastw {{.*#+}} ymm2 = [256,256,256,256,256,256,256,256,256,256,256,256,256,256,256,256]
+; AVX2-NEXT:    vpaddb %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    retl
 ;
 ; AVX512BW-LABEL: f64xi8_i16:
 ; AVX512BW:       # %bb.0:
@@ -333,14 +333,14 @@ define <64 x i8> @f64xi8_i16(<64 x i8> %a) {
 ; AVX-64-NEXT:    vandps %ymm2, %ymm1, %ymm1
 ; AVX-64-NEXT:    retq
 ;
-; NO-AVX512BW-64-LABEL: f64xi8_i16:
-; NO-AVX512BW-64:       # %bb.0:
-; NO-AVX512BW-64-NEXT:    vpbroadcastw {{.*#+}} ymm2 = [256,256,256,256,256,256,256,256,256,256,256,256,256,256,256,256]
-; NO-AVX512BW-64-NEXT:    vpaddb %ymm2, %ymm1, %ymm1
-; NO-AVX512BW-64-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
-; NO-AVX512BW-64-NEXT:    vpand %ymm2, %ymm0, %ymm0
-; NO-AVX512BW-64-NEXT:    vpand %ymm2, %ymm1, %ymm1
-; NO-AVX512BW-64-NEXT:    retq
+; AVX2-64-LABEL: f64xi8_i16:
+; AVX2-64:       # %bb.0:
+; AVX2-64-NEXT:    vpbroadcastw {{.*#+}} ymm2 = [256,256,256,256,256,256,256,256,256,256,256,256,256,256,256,256]
+; AVX2-64-NEXT:    vpaddb %ymm2, %ymm1, %ymm1
+; AVX2-64-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
+; AVX2-64-NEXT:    vpand %ymm2, %ymm0, %ymm0
+; AVX2-64-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX2-64-NEXT:    retq
 ;
 ; AVX512BW-64-LABEL: f64xi8_i16:
 ; AVX512BW-64:       # %bb.0:
@@ -370,14 +370,14 @@ define <64 x i8> @f64i8_i32(<64 x i8> %a) {
 ; AVX-NEXT:    vandps %ymm2, %ymm1, %ymm1
 ; AVX-NEXT:    retl
 ;
-; NO-AVX512BW-LABEL: f64i8_i32:
-; NO-AVX512BW:       # %bb.0:
-; NO-AVX512BW-NEXT:    vpbroadcastd {{.*#+}} ymm2 = [50462976,50462976,50462976,50462976,50462976,50462976,50462976,50462976]
-; NO-AVX512BW-NEXT:    vpaddb %ymm2, %ymm1, %ymm1
-; NO-AVX512BW-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
-; NO-AVX512BW-NEXT:    vpand %ymm2, %ymm0, %ymm0
-; NO-AVX512BW-NEXT:    vpand %ymm2, %ymm1, %ymm1
-; NO-AVX512BW-NEXT:    retl
+; AVX2-LABEL: f64i8_i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm2 = [50462976,50462976,50462976,50462976,50462976,50462976,50462976,50462976]
+; AVX2-NEXT:    vpaddb %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    retl
 ;
 ; AVX512BW-LABEL: f64i8_i32:
 ; AVX512BW:       # %bb.0:
@@ -401,14 +401,14 @@ define <64 x i8> @f64i8_i32(<64 x i8> %a) {
 ; AVX-64-NEXT:    vandps %ymm2, %ymm1, %ymm1
 ; AVX-64-NEXT:    retq
 ;
-; NO-AVX512BW-64-LABEL: f64i8_i32:
-; NO-AVX512BW-64:       # %bb.0:
-; NO-AVX512BW-64-NEXT:    vpbroadcastd {{.*#+}} ymm2 = [50462976,50462976,50462976,50462976,50462976,50462976,50462976,50462976]
-; NO-AVX512BW-64-NEXT:    vpaddb %ymm2, %ymm1, %ymm1
-; NO-AVX512BW-64-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
-; NO-AVX512BW-64-NEXT:    vpand %ymm2, %ymm0, %ymm0
-; NO-AVX512BW-64-NEXT:    vpand %ymm2, %ymm1, %ymm1
-; NO-AVX512BW-64-NEXT:    retq
+; AVX2-64-LABEL: f64i8_i32:
+; AVX2-64:       # %bb.0:
+; AVX2-64-NEXT:    vpbroadcastd {{.*#+}} ymm2 = [50462976,50462976,50462976,50462976,50462976,50462976,50462976,50462976]
+; AVX2-64-NEXT:    vpaddb %ymm2, %ymm1, %ymm1
+; AVX2-64-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
+; AVX2-64-NEXT:    vpand %ymm2, %ymm0, %ymm0
+; AVX2-64-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX2-64-NEXT:    retq
 ;
 ; AVX512BW-64-LABEL: f64i8_i32:
 ; AVX512BW-64:       # %bb.0:
@@ -438,14 +438,14 @@ define <64 x i8> @f64xi8_i64(<64 x i8> %a) {
 ; AVX-NEXT:    vandps %ymm2, %ymm1, %ymm1
 ; AVX-NEXT:    retl
 ;
-; NO-AVX512BW-LABEL: f64xi8_i64:
-; NO-AVX512BW:       # %bb.0:
-; NO-AVX512BW-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [7.9499288951273625E-275,7.9499288951273625E-275,7.9499288951273625E-275,7.9499288951273625E-275]
-; NO-AVX512BW-NEXT:    vpaddb %ymm2, %ymm1, %ymm1
-; NO-AVX512BW-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
-; NO-AVX512BW-NEXT:    vpand %ymm2, %ymm0, %ymm0
-; NO-AVX512BW-NEXT:    vpand %ymm2, %ymm1, %ymm1
-; NO-AVX512BW-NEXT:    retl
+; AVX2-LABEL: f64xi8_i64:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [7.9499288951273625E-275,7.9499288951273625E-275,7.9499288951273625E-275,7.9499288951273625E-275]
+; AVX2-NEXT:    vpaddb %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    retl
 ;
 ; AVX512BW-LABEL: f64xi8_i64:
 ; AVX512BW:       # %bb.0:
@@ -469,14 +469,14 @@ define <64 x i8> @f64xi8_i64(<64 x i8> %a) {
 ; AVX-64-NEXT:    vandps %ymm2, %ymm1, %ymm1
 ; AVX-64-NEXT:    retq
 ;
-; NO-AVX512BW-64-LABEL: f64xi8_i64:
-; NO-AVX512BW-64:       # %bb.0:
-; NO-AVX512BW-64-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [506097522914230528,506097522914230528,506097522914230528,506097522914230528]
-; NO-AVX512BW-64-NEXT:    vpaddb %ymm2, %ymm1, %ymm1
-; NO-AVX512BW-64-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
-; NO-AVX512BW-64-NEXT:    vpand %ymm2, %ymm0, %ymm0
-; NO-AVX512BW-64-NEXT:    vpand %ymm2, %ymm1, %ymm1
-; NO-AVX512BW-64-NEXT:    retq
+; AVX2-64-LABEL: f64xi8_i64:
+; AVX2-64:       # %bb.0:
+; AVX2-64-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [506097522914230528,506097522914230528,506097522914230528,506097522914230528]
+; AVX2-64-NEXT:    vpaddb %ymm2, %ymm1, %ymm1
+; AVX2-64-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
+; AVX2-64-NEXT:    vpand %ymm2, %ymm0, %ymm0
+; AVX2-64-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX2-64-NEXT:    retq
 ;
 ; AVX512BW-64-LABEL: f64xi8_i64:
 ; AVX512BW-64:       # %bb.0:
@@ -507,15 +507,15 @@ define <64 x i8> @f64xi8_i128(<64 x i8> %a) {
 ; AVX-NEXT:    vandps %ymm2, %ymm1, %ymm1
 ; AVX-NEXT:    retl
 ;
-; NO-AVX512BW-LABEL: f64xi8_i128:
-; NO-AVX512BW:       # %bb.0:
-; NO-AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15]
-; NO-AVX512BW-NEXT:    # ymm2 = mem[0,1,0,1]
-; NO-AVX512BW-NEXT:    vpaddb %ymm2, %ymm1, %ymm1
-; NO-AVX512BW-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
-; NO-AVX512BW-NEXT:    vpand %ymm2, %ymm0, %ymm0
-; NO-AVX512BW-NEXT:    vpand %ymm2, %ymm1, %ymm1
-; NO-AVX512BW-NEXT:    retl
+; AVX2-LABEL: f64xi8_i128:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15]
+; AVX2-NEXT:    # ymm2 = mem[0,1,0,1]
+; AVX2-NEXT:    vpaddb %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    retl
 ;
 ; AVX512BW-LABEL: f64xi8_i128:
 ; AVX512BW:       # %bb.0:
@@ -541,15 +541,15 @@ define <64 x i8> @f64xi8_i128(<64 x i8> %a) {
 ; AVX-64-NEXT:    vandps %ymm2, %ymm1, %ymm1
 ; AVX-64-NEXT:    retq
 ;
-; NO-AVX512BW-64-LABEL: f64xi8_i128:
-; NO-AVX512BW-64:       # %bb.0:
-; NO-AVX512BW-64-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15]
-; NO-AVX512BW-64-NEXT:    # ymm2 = mem[0,1,0,1]
-; NO-AVX512BW-64-NEXT:    vpaddb %ymm2, %ymm1, %ymm1
-; NO-AVX512BW-64-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
-; NO-AVX512BW-64-NEXT:    vpand %ymm2, %ymm0, %ymm0
-; NO-AVX512BW-64-NEXT:    vpand %ymm2, %ymm1, %ymm1
-; NO-AVX512BW-64-NEXT:    retq
+; AVX2-64-LABEL: f64xi8_i128:
+; AVX2-64:       # %bb.0:
+; AVX2-64-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15]
+; AVX2-64-NEXT:    # ymm2 = mem[0,1,0,1]
+; AVX2-64-NEXT:    vpaddb %ymm2, %ymm1, %ymm1
+; AVX2-64-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
+; AVX2-64-NEXT:    vpand %ymm2, %ymm0, %ymm0
+; AVX2-64-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX2-64-NEXT:    retq
 ;
 ; AVX512BW-64-LABEL: f64xi8_i128:
 ; AVX512BW-64:       # %bb.0:
@@ -582,14 +582,14 @@ define <64 x i8> @f64xi8_i256(<64 x i8> %a) {
 ; AVX-NEXT:    vandps %ymm2, %ymm1, %ymm1
 ; AVX-NEXT:    retl
 ;
-; NO-AVX512BW-LABEL: f64xi8_i256:
-; NO-AVX512BW:       # %bb.0:
-; NO-AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31]
-; NO-AVX512BW-NEXT:    vpaddb %ymm2, %ymm1, %ymm1
-; NO-AVX512BW-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
-; NO-AVX512BW-NEXT:    vpand %ymm2, %ymm0, %ymm0
-; NO-AVX512BW-NEXT:    vpand %ymm2, %ymm1, %ymm1
-; NO-AVX512BW-NEXT:    retl
+; AVX2-LABEL: f64xi8_i256:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31]
+; AVX2-NEXT:    vpaddb %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    retl
 ;
 ; AVX512BW-LABEL: f64xi8_i256:
 ; AVX512BW:       # %bb.0:
@@ -616,14 +616,14 @@ define <64 x i8> @f64xi8_i256(<64 x i8> %a) {
 ; AVX-64-NEXT:    vandps %ymm2, %ymm1, %ymm1
 ; AVX-64-NEXT:    retq
 ;
-; NO-AVX512BW-64-LABEL: f64xi8_i256:
-; NO-AVX512BW-64:       # %bb.0:
-; NO-AVX512BW-64-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31]
-; NO-AVX512BW-64-NEXT:    vpaddb %ymm2, %ymm1, %ymm1
-; NO-AVX512BW-64-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
-; NO-AVX512BW-64-NEXT:    vpand %ymm2, %ymm0, %ymm0
-; NO-AVX512BW-64-NEXT:    vpand %ymm2, %ymm1, %ymm1
-; NO-AVX512BW-64-NEXT:    retq
+; AVX2-64-LABEL: f64xi8_i256:
+; AVX2-64:       # %bb.0:
+; AVX2-64-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31]
+; AVX2-64-NEXT:    vpaddb %ymm2, %ymm1, %ymm1
+; AVX2-64-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
+; AVX2-64-NEXT:    vpand %ymm2, %ymm0, %ymm0
+; AVX2-64-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX2-64-NEXT:    retq
 ;
 ; AVX512BW-64-LABEL: f64xi8_i256:
 ; AVX512BW-64:       # %bb.0:
@@ -848,14 +848,14 @@ define <32 x i16> @f32xi16_i32(<32 x i16> %a) {
 ; AVX-NEXT:    vandps %ymm2, %ymm1, %ymm1
 ; AVX-NEXT:    retl
 ;
-; NO-AVX512BW-LABEL: f32xi16_i32:
-; NO-AVX512BW:       # %bb.0:
-; NO-AVX512BW-NEXT:    vpbroadcastd {{.*#+}} ymm2 = [65536,65536,65536,65536,65536,65536,65536,65536]
-; NO-AVX512BW-NEXT:    vpaddw %ymm2, %ymm1, %ymm1
-; NO-AVX512BW-NEXT:    vpaddw %ymm2, %ymm0, %ymm0
-; NO-AVX512BW-NEXT:    vpand %ymm2, %ymm0, %ymm0
-; NO-AVX512BW-NEXT:    vpand %ymm2, %ymm1, %ymm1
-; NO-AVX512BW-NEXT:    retl
+; AVX2-LABEL: f32xi16_i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm2 = [65536,65536,65536,65536,65536,65536,65536,65536]
+; AVX2-NEXT:    vpaddw %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    vpaddw %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    retl
 ;
 ; AVX512BW-LABEL: f32xi16_i32:
 ; AVX512BW:       # %bb.0:
@@ -879,14 +879,14 @@ define <32 x i16> @f32xi16_i32(<32 x i16> %a) {
 ; AVX-64-NEXT:    vandps %ymm2, %ymm1, %ymm1
 ; AVX-64-NEXT:    retq
 ;
-; NO-AVX512BW-64-LABEL: f32xi16_i32:
-; NO-AVX512BW-64:       # %bb.0:
-; NO-AVX512BW-64-NEXT:    vpbroadcastd {{.*#+}} ymm2 = [65536,65536,65536,65536,65536,65536,65536,65536]
-; NO-AVX512BW-64-NEXT:    vpaddw %ymm2, %ymm1, %ymm1
-; NO-AVX512BW-64-NEXT:    vpaddw %ymm2, %ymm0, %ymm0
-; NO-AVX512BW-64-NEXT:    vpand %ymm2, %ymm0, %ymm0
-; NO-AVX512BW-64-NEXT:    vpand %ymm2, %ymm1, %ymm1
-; NO-AVX512BW-64-NEXT:    retq
+; AVX2-64-LABEL: f32xi16_i32:
+; AVX2-64:       # %bb.0:
+; AVX2-64-NEXT:    vpbroadcastd {{.*#+}} ymm2 = [65536,65536,65536,65536,65536,65536,65536,65536]
+; AVX2-64-NEXT:    vpaddw %ymm2, %ymm1, %ymm1
+; AVX2-64-NEXT:    vpaddw %ymm2, %ymm0, %ymm0
+; AVX2-64-NEXT:    vpand %ymm2, %ymm0, %ymm0
+; AVX2-64-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX2-64-NEXT:    retq
 ;
 ; AVX512BW-64-LABEL: f32xi16_i32:
 ; AVX512BW-64:       # %bb.0:
@@ -916,14 +916,14 @@ define <32 x i16> @f32xi16_i64(<32 x i16> %a) {
 ; AVX-NEXT:    vandps %ymm2, %ymm1, %ymm1
 ; AVX-NEXT:    retl
 ;
-; NO-AVX512BW-LABEL: f32xi16_i64:
-; NO-AVX512BW:       # %bb.0:
-; NO-AVX512BW-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [4.1720559249406128E-309,4.1720559249406128E-309,4.1720559249406128E-309,4.1720559249406128E-309]
-; NO-AVX512BW-NEXT:    vpaddw %ymm2, %ymm1, %ymm1
-; NO-AVX512BW-NEXT:    vpaddw %ymm2, %ymm0, %ymm0
-; NO-AVX512BW-NEXT:    vpand %ymm2, %ymm0, %ymm0
-; NO-AVX512BW-NEXT:    vpand %ymm2, %ymm1, %ymm1
-; NO-AVX512BW-NEXT:    retl
+; AVX2-LABEL: f32xi16_i64:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [4.1720559249406128E-309,4.1720559249406128E-309,4.1720559249406128E-309,4.1720559249406128E-309]
+; AVX2-NEXT:    vpaddw %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    vpaddw %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    retl
 ;
 ; AVX512BW-LABEL: f32xi16_i64:
 ; AVX512BW:       # %bb.0:
@@ -947,14 +947,14 @@ define <32 x i16> @f32xi16_i64(<32 x i16> %a) {
 ; AVX-64-NEXT:    vandps %ymm2, %ymm1, %ymm1
 ; AVX-64-NEXT:    retq
 ;
-; NO-AVX512BW-64-LABEL: f32xi16_i64:
-; NO-AVX512BW-64:       # %bb.0:
-; NO-AVX512BW-64-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [844433520132096,844433520132096,844433520132096,844433520132096]
-; NO-AVX512BW-64-NEXT:    vpaddw %ymm2, %ymm1, %ymm1
-; NO-AVX512BW-64-NEXT:    vpaddw %ymm2, %ymm0, %ymm0
-; NO-AVX512BW-64-NEXT:    vpand %ymm2, %ymm0, %ymm0
-; NO-AVX512BW-64-NEXT:    vpand %ymm2, %ymm1, %ymm1
-; NO-AVX512BW-64-NEXT:    retq
+; AVX2-64-LABEL: f32xi16_i64:
+; AVX2-64:       # %bb.0:
+; AVX2-64-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [844433520132096,844433520132096,844433520132096,844433520132096]
+; AVX2-64-NEXT:    vpaddw %ymm2, %ymm1, %ymm1
+; AVX2-64-NEXT:    vpaddw %ymm2, %ymm0, %ymm0
+; AVX2-64-NEXT:    vpand %ymm2, %ymm0, %ymm0
+; AVX2-64-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX2-64-NEXT:    retq
 ;
 ; AVX512BW-64-LABEL: f32xi16_i64:
 ; AVX512BW-64:       # %bb.0:
@@ -985,15 +985,15 @@ define <32 x i16> @f32xi16_i128(<32 x i16> %a) {
 ; AVX-NEXT:    vandps %ymm2, %ymm1, %ymm1
 ; AVX-NEXT:    retl
 ;
-; NO-AVX512BW-LABEL: f32xi16_i128:
-; NO-AVX512BW:       # %bb.0:
-; NO-AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [0,1,2,3,4,5,6,7,0,1,2,3,4,5,6,7]
-; NO-AVX512BW-NEXT:    # ymm2 = mem[0,1,0,1]
-; NO-AVX512BW-NEXT:    vpaddw %ymm2, %ymm1, %ymm1
-; NO-AVX512BW-NEXT:    vpaddw %ymm2, %ymm0, %ymm0
-; NO-AVX512BW-NEXT:    vpand %ymm2, %ymm0, %ymm0
-; NO-AVX512BW-NEXT:    vpand %ymm2, %ymm1, %ymm1
-; NO-AVX512BW-NEXT:    retl
+; AVX2-LABEL: f32xi16_i128:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [0,1,2,3,4,5,6,7,0,1,2,3,4,5,6,7]
+; AVX2-NEXT:    # ymm2 = mem[0,1,0,1]
+; AVX2-NEXT:    vpaddw %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    vpaddw %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    retl
 ;
 ; AVX512BW-LABEL: f32xi16_i128:
 ; AVX512BW:       # %bb.0:
@@ -1019,15 +1019,15 @@ define <32 x i16> @f32xi16_i128(<32 x i16> %a) {
 ; AVX-64-NEXT:    vandps %ymm2, %ymm1, %ymm1
 ; AVX-64-NEXT:    retq
 ;
-; NO-AVX512BW-64-LABEL: f32xi16_i128:
-; NO-AVX512BW-64:       # %bb.0:
-; NO-AVX512BW-64-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [0,1,2,3,4,5,6,7,0,1,2,3,4,5,6,7]
-; NO-AVX512BW-64-NEXT:    # ymm2 = mem[0,1,0,1]
-; NO-AVX512BW-64-NEXT:    vpaddw %ymm2, %ymm1, %ymm1
-; NO-AVX512BW-64-NEXT:    vpaddw %ymm2, %ymm0, %ymm0
-; NO-AVX512BW-64-NEXT:    vpand %ymm2, %ymm0, %ymm0
-; NO-AVX512BW-64-NEXT:    vpand %ymm2, %ymm1, %ymm1
-; NO-AVX512BW-64-NEXT:    retq
+; AVX2-64-LABEL: f32xi16_i128:
+; AVX2-64:       # %bb.0:
+; AVX2-64-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [0,1,2,3,4,5,6,7,0,1,2,3,4,5,6,7]
+; AVX2-64-NEXT:    # ymm2 = mem[0,1,0,1]
+; AVX2-64-NEXT:    vpaddw %ymm2, %ymm1, %ymm1
+; AVX2-64-NEXT:    vpaddw %ymm2, %ymm0, %ymm0
+; AVX2-64-NEXT:    vpand %ymm2, %ymm0, %ymm0
+; AVX2-64-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX2-64-NEXT:    retq
 ;
 ; AVX512BW-64-LABEL: f32xi16_i128:
 ; AVX512BW-64:       # %bb.0:
@@ -1060,14 +1060,14 @@ define <32 x i16> @f32xi16_i256(<32 x i16> %a) {
 ; AVX-NEXT:    vandps %ymm2, %ymm1, %ymm1
 ; AVX-NEXT:    retl
 ;
-; NO-AVX512BW-LABEL: f32xi16_i256:
-; NO-AVX512BW:       # %bb.0:
-; NO-AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15]
-; NO-AVX512BW-NEXT:    vpaddw %ymm2, %ymm1, %ymm1
-; NO-AVX512BW-NEXT:    vpaddw %ymm2, %ymm0, %ymm0
-; NO-AVX512BW-NEXT:    vpand %ymm2, %ymm0, %ymm0
-; NO-AVX512BW-NEXT:    vpand %ymm2, %ymm1, %ymm1
-; NO-AVX512BW-NEXT:    retl
+; AVX2-LABEL: f32xi16_i256:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15]
+; AVX2-NEXT:    vpaddw %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    vpaddw %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    retl
 ;
 ; AVX512BW-LABEL: f32xi16_i256:
 ; AVX512BW:       # %bb.0:
@@ -1094,14 +1094,14 @@ define <32 x i16> @f32xi16_i256(<32 x i16> %a) {
 ; AVX-64-NEXT:    vandps %ymm2, %ymm1, %ymm1
 ; AVX-64-NEXT:    retq
 ;
-; NO-AVX512BW-64-LABEL: f32xi16_i256:
-; NO-AVX512BW-64:       # %bb.0:
-; NO-AVX512BW-64-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15]
-; NO-AVX512BW-64-NEXT:    vpaddw %ymm2, %ymm1, %ymm1
-; NO-AVX512BW-64-NEXT:    vpaddw %ymm2, %ymm0, %ymm0
-; NO-AVX512BW-64-NEXT:    vpand %ymm2, %ymm0, %ymm0
-; NO-AVX512BW-64-NEXT:    vpand %ymm2, %ymm1, %ymm1
-; NO-AVX512BW-64-NEXT:    retq
+; AVX2-64-LABEL: f32xi16_i256:
+; AVX2-64:       # %bb.0:
+; AVX2-64-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15]
+; AVX2-64-NEXT:    vpaddw %ymm2, %ymm1, %ymm1
+; AVX2-64-NEXT:    vpaddw %ymm2, %ymm0, %ymm0
+; AVX2-64-NEXT:    vpand %ymm2, %ymm0, %ymm0
+; AVX2-64-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX2-64-NEXT:    retq
 ;
 ; AVX512BW-64-LABEL: f32xi16_i256:
 ; AVX512BW-64:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/bswap-vector.ll b/llvm/test/CodeGen/X86/bswap-vector.ll
index cfbb5ecc5..4bb2b764c 100644
--- a/llvm/test/CodeGen/X86/bswap-vector.ll
+++ b/llvm/test/CodeGen/X86/bswap-vector.ll
@@ -2,7 +2,6 @@
 ; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu | FileCheck %s --check-prefix=CHECK-ALL --check-prefix=CHECK-SSE --check-prefix=CHECK-NOSSSE3
 ; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+ssse3 | FileCheck %s --check-prefix=CHECK-ALL --check-prefix=CHECK-SSE --check-prefix=CHECK-SSSE3
 ; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx2 | FileCheck %s --check-prefix=CHECK-ALL --check-prefix=CHECK-AVX --check-prefix=CHECK-AVX2
-; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx2 -x86-experimental-vector-widening-legalization | FileCheck %s --check-prefix=CHECK-ALL --check-prefix=CHECK-WIDE-AVX --check-prefix=CHECK-WIDE-AVX2
 
 declare <8 x i16> @llvm.bswap.v8i16(<8 x i16>)
 declare <4 x i32> @llvm.bswap.v4i32(<4 x i32>)
@@ -31,11 +30,6 @@ define <8 x i16> @test1(<8 x i16> %v) {
 ; CHECK-AVX:       # %bb.0: # %entry
 ; CHECK-AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[1,0,3,2,5,4,7,6,9,8,11,10,13,12,15,14]
 ; CHECK-AVX-NEXT:    retq
-;
-; CHECK-WIDE-AVX-LABEL: test1:
-; CHECK-WIDE-AVX:       # %bb.0: # %entry
-; CHECK-WIDE-AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[1,0,3,2,5,4,7,6,9,8,11,10,13,12,15,14]
-; CHECK-WIDE-AVX-NEXT:    retq
 entry:
   %r = call <8 x i16> @llvm.bswap.v8i16(<8 x i16> %v)
   ret <8 x i16> %r
@@ -64,11 +58,6 @@ define <4 x i32> @test2(<4 x i32> %v) {
 ; CHECK-AVX:       # %bb.0:
 ; CHECK-AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[3,2,1,0,7,6,5,4,11,10,9,8,15,14,13,12]
 ; CHECK-AVX-NEXT:    retq
-;
-; CHECK-WIDE-AVX-LABEL: test2:
-; CHECK-WIDE-AVX:       # %bb.0:
-; CHECK-WIDE-AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[3,2,1,0,7,6,5,4,11,10,9,8,15,14,13,12]
-; CHECK-WIDE-AVX-NEXT:    retq
   %r = call <4 x i32> @llvm.bswap.v4i32(<4 x i32> %v)
   ret <4 x i32> %r
 }
@@ -99,12 +88,6 @@ define <4 x i32> @or_bswap(<4 x i32> %x, <4 x i32> %y, <4 x i32>* %p1, <4 x i32>
 ; CHECK-AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; CHECK-AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[3,2,1,0,7,6,5,4,11,10,9,8,15,14,13,12]
 ; CHECK-AVX-NEXT:    retq
-;
-; CHECK-WIDE-AVX-LABEL: or_bswap:
-; CHECK-WIDE-AVX:       # %bb.0:
-; CHECK-WIDE-AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; CHECK-WIDE-AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[3,2,1,0,7,6,5,4,11,10,9,8,15,14,13,12]
-; CHECK-WIDE-AVX-NEXT:    retq
   %xt = call <4 x i32> @llvm.bswap.v4i32(<4 x i32> %x)
   %yt = call <4 x i32> @llvm.bswap.v4i32(<4 x i32> %y)
   %r = or <4 x i32> %xt, %yt
@@ -136,11 +119,6 @@ define <2 x i64> @test3(<2 x i64> %v) {
 ; CHECK-AVX:       # %bb.0: # %entry
 ; CHECK-AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[7,6,5,4,3,2,1,0,15,14,13,12,11,10,9,8]
 ; CHECK-AVX-NEXT:    retq
-;
-; CHECK-WIDE-AVX-LABEL: test3:
-; CHECK-WIDE-AVX:       # %bb.0: # %entry
-; CHECK-WIDE-AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[7,6,5,4,3,2,1,0,15,14,13,12,11,10,9,8]
-; CHECK-WIDE-AVX-NEXT:    retq
 entry:
   %r = call <2 x i64> @llvm.bswap.v2i64(<2 x i64> %v)
   ret <2 x i64> %r
@@ -183,11 +161,6 @@ define <16 x i16> @test4(<16 x i16> %v) {
 ; CHECK-AVX:       # %bb.0: # %entry
 ; CHECK-AVX-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[1,0,3,2,5,4,7,6,9,8,11,10,13,12,15,14,17,16,19,18,21,20,23,22,25,24,27,26,29,28,31,30]
 ; CHECK-AVX-NEXT:    retq
-;
-; CHECK-WIDE-AVX-LABEL: test4:
-; CHECK-WIDE-AVX:       # %bb.0: # %entry
-; CHECK-WIDE-AVX-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[1,0,3,2,5,4,7,6,9,8,11,10,13,12,15,14,17,16,19,18,21,20,23,22,25,24,27,26,29,28,31,30]
-; CHECK-WIDE-AVX-NEXT:    retq
 entry:
   %r = call <16 x i16> @llvm.bswap.v16i16(<16 x i16> %v)
   ret <16 x i16> %r
@@ -226,11 +199,6 @@ define <8 x i32> @test5(<8 x i32> %v) {
 ; CHECK-AVX:       # %bb.0: # %entry
 ; CHECK-AVX-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[3,2,1,0,7,6,5,4,11,10,9,8,15,14,13,12,19,18,17,16,23,22,21,20,27,26,25,24,31,30,29,28]
 ; CHECK-AVX-NEXT:    retq
-;
-; CHECK-WIDE-AVX-LABEL: test5:
-; CHECK-WIDE-AVX:       # %bb.0: # %entry
-; CHECK-WIDE-AVX-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[3,2,1,0,7,6,5,4,11,10,9,8,15,14,13,12,19,18,17,16,23,22,21,20,27,26,25,24,31,30,29,28]
-; CHECK-WIDE-AVX-NEXT:    retq
 entry:
   %r = call <8 x i32> @llvm.bswap.v8i32(<8 x i32> %v)
   ret <8 x i32> %r
@@ -273,11 +241,6 @@ define <4 x i64> @test6(<4 x i64> %v) {
 ; CHECK-AVX:       # %bb.0: # %entry
 ; CHECK-AVX-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[7,6,5,4,3,2,1,0,15,14,13,12,11,10,9,8,23,22,21,20,19,18,17,16,31,30,29,28,27,26,25,24]
 ; CHECK-AVX-NEXT:    retq
-;
-; CHECK-WIDE-AVX-LABEL: test6:
-; CHECK-WIDE-AVX:       # %bb.0: # %entry
-; CHECK-WIDE-AVX-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[7,6,5,4,3,2,1,0,15,14,13,12,11,10,9,8,23,22,21,20,19,18,17,16,31,30,29,28,27,26,25,24]
-; CHECK-WIDE-AVX-NEXT:    retq
 entry:
   %r = call <4 x i64> @llvm.bswap.v4i64(<4 x i64> %v)
   ret <4 x i64> %r
@@ -291,29 +254,23 @@ define <4 x i16> @test7(<4 x i16> %v) {
 ; CHECK-NOSSSE3-NEXT:    pxor %xmm1, %xmm1
 ; CHECK-NOSSSE3-NEXT:    movdqa %xmm0, %xmm2
 ; CHECK-NOSSSE3-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
-; CHECK-NOSSSE3-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[3,2,1,0,4,5,6,7]
-; CHECK-NOSSSE3-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,6,5,4]
+; CHECK-NOSSSE3-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[1,0,3,2,4,5,6,7]
+; CHECK-NOSSSE3-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,4,7,6]
 ; CHECK-NOSSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
-; CHECK-NOSSSE3-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[3,2,1,0,4,5,6,7]
-; CHECK-NOSSSE3-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,6,5,4]
+; CHECK-NOSSSE3-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[1,0,3,2,4,5,6,7]
+; CHECK-NOSSSE3-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,4,7,6]
 ; CHECK-NOSSSE3-NEXT:    packuswb %xmm2, %xmm0
-; CHECK-NOSSSE3-NEXT:    psrld $16, %xmm0
 ; CHECK-NOSSSE3-NEXT:    retq
 ;
 ; CHECK-SSSE3-LABEL: test7:
 ; CHECK-SSSE3:       # %bb.0: # %entry
-; CHECK-SSSE3-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[1,0],zero,zero,xmm0[5,4],zero,zero,xmm0[9,8],zero,zero,xmm0[13,12],zero,zero
+; CHECK-SSSE3-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[1,0,3,2,5,4,7,6,9,8,11,10,13,12,15,14]
 ; CHECK-SSSE3-NEXT:    retq
 ;
 ; CHECK-AVX-LABEL: test7:
 ; CHECK-AVX:       # %bb.0: # %entry
-; CHECK-AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[1,0],zero,zero,xmm0[5,4],zero,zero,xmm0[9,8],zero,zero,xmm0[13,12],zero,zero
+; CHECK-AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[1,0,3,2,5,4,7,6,9,8,11,10,13,12,15,14]
 ; CHECK-AVX-NEXT:    retq
-;
-; CHECK-WIDE-AVX-LABEL: test7:
-; CHECK-WIDE-AVX:       # %bb.0: # %entry
-; CHECK-WIDE-AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[1,0,3,2,5,4,7,6,9,8,11,10,13,12,15,14]
-; CHECK-WIDE-AVX-NEXT:    retq
 entry:
   %r = call <4 x i16> @llvm.bswap.v4i16(<4 x i16> %v)
   ret <4 x i16> %r
@@ -407,11 +364,6 @@ define <8 x i16> @fold_v8i16() {
 ; CHECK-AVX:       # %bb.0: # %entry
 ; CHECK-AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [0,256,65535,512,65023,1024,64511,1536]
 ; CHECK-AVX-NEXT:    retq
-;
-; CHECK-WIDE-AVX-LABEL: fold_v8i16:
-; CHECK-WIDE-AVX:       # %bb.0: # %entry
-; CHECK-WIDE-AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [0,256,65535,512,65023,1024,64511,1536]
-; CHECK-WIDE-AVX-NEXT:    retq
 entry:
   %r = call <8 x i16> @llvm.bswap.v8i16(<8 x i16> <i16 0, i16 1, i16 -1, i16 2, i16 -3, i16 4, i16 -5, i16 6>)
   ret <8 x i16> %r
@@ -427,11 +379,6 @@ define <4 x i32> @fold_v4i32() {
 ; CHECK-AVX:       # %bb.0: # %entry
 ; CHECK-AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [0,4294967295,33554432,4261412863]
 ; CHECK-AVX-NEXT:    retq
-;
-; CHECK-WIDE-AVX-LABEL: fold_v4i32:
-; CHECK-WIDE-AVX:       # %bb.0: # %entry
-; CHECK-WIDE-AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [0,4294967295,33554432,4261412863]
-; CHECK-WIDE-AVX-NEXT:    retq
 entry:
   %r = call <4 x i32> @llvm.bswap.v4i32(<4 x i32> <i32 0, i32 -1, i32 2, i32 -3>)
   ret <4 x i32> %r
@@ -447,11 +394,6 @@ define <2 x i64> @fold_v2i64() {
 ; CHECK-AVX:       # %bb.0: # %entry
 ; CHECK-AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [18374686479671623680,18446744073709551615]
 ; CHECK-AVX-NEXT:    retq
-;
-; CHECK-WIDE-AVX-LABEL: fold_v2i64:
-; CHECK-WIDE-AVX:       # %bb.0: # %entry
-; CHECK-WIDE-AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [18374686479671623680,18446744073709551615]
-; CHECK-WIDE-AVX-NEXT:    retq
 entry:
   %r = call <2 x i64> @llvm.bswap.v2i64(<2 x i64> <i64 255, i64 -1>)
   ret <2 x i64> %r
@@ -468,11 +410,6 @@ define <16 x i16> @fold_v16i16() {
 ; CHECK-AVX:       # %bb.0: # %entry
 ; CHECK-AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [0,256,65535,512,65023,1024,64511,1536,63999,2048,63487,2560,62975,3072,62463,3584]
 ; CHECK-AVX-NEXT:    retq
-;
-; CHECK-WIDE-AVX-LABEL: fold_v16i16:
-; CHECK-WIDE-AVX:       # %bb.0: # %entry
-; CHECK-WIDE-AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [0,256,65535,512,65023,1024,64511,1536,63999,2048,63487,2560,62975,3072,62463,3584]
-; CHECK-WIDE-AVX-NEXT:    retq
 entry:
   %r = call <16 x i16> @llvm.bswap.v16i16(<16 x i16> <i16 0, i16 1, i16 -1, i16 2, i16 -3, i16 4, i16 -5, i16 6, i16 -7, i16 8, i16 -9, i16 10, i16 -11, i16 12, i16 -13, i16 14>)
   ret <16 x i16> %r
@@ -489,11 +426,6 @@ define <8 x i32> @fold_v8i32() {
 ; CHECK-AVX:       # %bb.0: # %entry
 ; CHECK-AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [0,16777216,4294967295,33554432,4261412863,67108864,4227858431,100663296]
 ; CHECK-AVX-NEXT:    retq
-;
-; CHECK-WIDE-AVX-LABEL: fold_v8i32:
-; CHECK-WIDE-AVX:       # %bb.0: # %entry
-; CHECK-WIDE-AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [0,16777216,4294967295,33554432,4261412863,67108864,4227858431,100663296]
-; CHECK-WIDE-AVX-NEXT:    retq
 entry:
   %r = call <8 x i32> @llvm.bswap.v8i32(<8 x i32> <i32 0, i32 1, i32 -1, i32 2, i32 -3, i32 4, i32 -5, i32 6>)
   ret <8 x i32> %r
@@ -510,11 +442,6 @@ define <4 x i64> @fold_v4i64() {
 ; CHECK-AVX:       # %bb.0: # %entry
 ; CHECK-AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [18374686479671623680,18446744073709551615,18446462598732840960,72056494526300160]
 ; CHECK-AVX-NEXT:    retq
-;
-; CHECK-WIDE-AVX-LABEL: fold_v4i64:
-; CHECK-WIDE-AVX:       # %bb.0: # %entry
-; CHECK-WIDE-AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [18374686479671623680,18446744073709551615,18446462598732840960,72056494526300160]
-; CHECK-WIDE-AVX-NEXT:    retq
 entry:
   %r = call <4 x i64> @llvm.bswap.v4i64(<4 x i64> <i64 255, i64 -1, i64 65535, i64 16776960>)
   ret <4 x i64> %r
diff --git a/llvm/test/CodeGen/X86/bswap_tree.ll b/llvm/test/CodeGen/X86/bswap_tree.ll
index 537acdb24..b136263b1 100644
--- a/llvm/test/CodeGen/X86/bswap_tree.ll
+++ b/llvm/test/CodeGen/X86/bswap_tree.ll
@@ -70,3 +70,32 @@ define i32 @test2(i32 %x) nounwind {
   %result = or i32 %or0, %or1
   ret i32 %result
 }
+
+declare i32 @llvm.bswap.i32(i32)
+
+; Match a 32-bit packed halfword bswap, with some subtree
+; already converted to a bswap.
+define i32 @test3(i32 %x) nounwind {
+; CHECK-LABEL: test3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    bswapl %eax
+; CHECK-NEXT:    roll $16, %eax
+; CHECK-NEXT:    retl
+;
+; CHECK64-LABEL: test3:
+; CHECK64:       # %bb.0:
+; CHECK64-NEXT:    movl %edi, %eax
+; CHECK64-NEXT:    bswapl %eax
+; CHECK64-NEXT:    roll $16, %eax
+; CHECK64-NEXT:    retq
+  %byte2 = and i32 %x, 16711680   ; 0x00ff0000
+  %byte3 = and i32 %x, 4278190080 ; 0xff000000
+  %1 = shl  i32 %byte2, 8
+  %2 = lshr i32 %byte3, 8
+  %or = or i32 %1, %2
+  %bswap = call i32 @llvm.bswap.i32(i32 %x)
+  %3 = lshr i32 %bswap, 16
+  %result = or i32 %or, %3
+  ret i32 %result
+}
diff --git a/llvm/test/CodeGen/X86/bug26810.ll b/llvm/test/CodeGen/X86/bug26810.ll
index fbb873002..7146f4cc3 100644
--- a/llvm/test/CodeGen/X86/bug26810.ll
+++ b/llvm/test/CodeGen/X86/bug26810.ll
@@ -287,12 +287,12 @@ declare i64 @_time64(i64*) local_unnamed_addr #4
 ; Function Attrs: argmemonly nounwind
 declare void @llvm.memset.p0i8.i32(i8* nocapture writeonly, i8, i32, i1) #6
 
-attributes #0 = { norecurse "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #2 = { norecurse nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #3 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #4 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #5 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { norecurse "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { norecurse nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #4 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #5 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #6 = { argmemonly nounwind }
 attributes #7 = { nounwind }
 
diff --git a/llvm/test/CodeGen/X86/build-vector-512.ll b/llvm/test/CodeGen/X86/build-vector-512.ll
index aba8b13db..4bc731cbf 100644
--- a/llvm/test/CodeGen/X86/build-vector-512.ll
+++ b/llvm/test/CodeGen/X86/build-vector-512.ll
@@ -156,159 +156,83 @@ define <16 x i32> @test_buildvector_v16i32(i32 %a0, i32 %a1, i32 %a2, i32 %a3, i
 }
 
 define <32 x i16> @test_buildvector_v32i16(i16 %a0, i16 %a1, i16 %a2, i16 %a3, i16 %a4, i16 %a5, i16 %a6, i16 %a7, i16 %a8, i16 %a9, i16 %a10, i16 %a11, i16 %a12, i16 %a13, i16 %a14, i16 %a15, i16 %a16, i16 %a17, i16 %a18, i16 %a19, i16 %a20, i16 %a21, i16 %a22, i16 %a23, i16 %a24, i16 %a25, i16 %a26, i16 %a27, i16 %a28, i16 %a29, i16 %a30, i16 %a31) {
-; AVX512F-32-LABEL: test_buildvector_v32i16:
-; AVX512F-32:       # %bb.0:
-; AVX512F-32-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX512F-32-NEXT:    vpinsrw $1, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX512F-32-NEXT:    vpinsrw $2, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX512F-32-NEXT:    vpinsrw $3, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX512F-32-NEXT:    vpinsrw $4, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX512F-32-NEXT:    vpinsrw $5, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX512F-32-NEXT:    vpinsrw $6, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX512F-32-NEXT:    vpinsrw $7, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX512F-32-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX512F-32-NEXT:    vpinsrw $1, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512F-32-NEXT:    vpinsrw $2, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512F-32-NEXT:    vpinsrw $3, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512F-32-NEXT:    vpinsrw $4, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512F-32-NEXT:    vpinsrw $5, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512F-32-NEXT:    vpinsrw $6, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512F-32-NEXT:    vpinsrw $7, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512F-32-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX512F-32-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX512F-32-NEXT:    vpinsrw $1, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512F-32-NEXT:    vpinsrw $2, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512F-32-NEXT:    vpinsrw $3, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512F-32-NEXT:    vpinsrw $4, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512F-32-NEXT:    vpinsrw $5, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512F-32-NEXT:    vpinsrw $6, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512F-32-NEXT:    vpinsrw $7, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512F-32-NEXT:    vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; AVX512F-32-NEXT:    vpinsrw $1, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; AVX512F-32-NEXT:    vpinsrw $2, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; AVX512F-32-NEXT:    vpinsrw $3, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; AVX512F-32-NEXT:    vpinsrw $4, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; AVX512F-32-NEXT:    vpinsrw $5, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; AVX512F-32-NEXT:    vpinsrw $6, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; AVX512F-32-NEXT:    vpinsrw $7, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; AVX512F-32-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
-; AVX512F-32-NEXT:    retl
-;
-; AVX512F-64-LABEL: test_buildvector_v32i16:
-; AVX512F-64:       # %bb.0:
-; AVX512F-64-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX512F-64-NEXT:    vpinsrw $1, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512F-64-NEXT:    vpinsrw $2, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512F-64-NEXT:    vpinsrw $3, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512F-64-NEXT:    vpinsrw $4, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512F-64-NEXT:    vpinsrw $5, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512F-64-NEXT:    vpinsrw $6, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512F-64-NEXT:    vpinsrw $7, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512F-64-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX512F-64-NEXT:    vpinsrw $1, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX512F-64-NEXT:    vpinsrw $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX512F-64-NEXT:    vpinsrw $3, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX512F-64-NEXT:    vpinsrw $4, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX512F-64-NEXT:    vpinsrw $5, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX512F-64-NEXT:    vpinsrw $6, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX512F-64-NEXT:    vpinsrw $7, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX512F-64-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512F-64-NEXT:    vmovd %edi, %xmm0
-; AVX512F-64-NEXT:    vpinsrw $1, %esi, %xmm0, %xmm0
-; AVX512F-64-NEXT:    vpinsrw $2, %edx, %xmm0, %xmm0
-; AVX512F-64-NEXT:    vpinsrw $3, %ecx, %xmm0, %xmm0
-; AVX512F-64-NEXT:    vpinsrw $4, %r8d, %xmm0, %xmm0
-; AVX512F-64-NEXT:    vpinsrw $5, %r9d, %xmm0, %xmm0
-; AVX512F-64-NEXT:    vpinsrw $6, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512F-64-NEXT:    vpinsrw $7, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512F-64-NEXT:    vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; AVX512F-64-NEXT:    vpinsrw $1, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512F-64-NEXT:    vpinsrw $2, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512F-64-NEXT:    vpinsrw $3, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512F-64-NEXT:    vpinsrw $4, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512F-64-NEXT:    vpinsrw $5, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512F-64-NEXT:    vpinsrw $6, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512F-64-NEXT:    vpinsrw $7, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512F-64-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
-; AVX512F-64-NEXT:    retq
-;
-; AVX512BW-32-LABEL: test_buildvector_v32i16:
-; AVX512BW-32:       # %bb.0:
-; AVX512BW-32-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX512BW-32-NEXT:    vpinsrw $1, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX512BW-32-NEXT:    vpinsrw $2, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX512BW-32-NEXT:    vpinsrw $3, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX512BW-32-NEXT:    vpinsrw $4, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX512BW-32-NEXT:    vpinsrw $5, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX512BW-32-NEXT:    vpinsrw $6, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX512BW-32-NEXT:    vpinsrw $7, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX512BW-32-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX512BW-32-NEXT:    vpinsrw $1, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512BW-32-NEXT:    vpinsrw $2, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512BW-32-NEXT:    vpinsrw $3, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512BW-32-NEXT:    vpinsrw $4, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512BW-32-NEXT:    vpinsrw $5, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512BW-32-NEXT:    vpinsrw $6, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512BW-32-NEXT:    vpinsrw $7, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512BW-32-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX512BW-32-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX512BW-32-NEXT:    vpinsrw $1, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512BW-32-NEXT:    vpinsrw $2, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512BW-32-NEXT:    vpinsrw $3, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512BW-32-NEXT:    vpinsrw $4, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512BW-32-NEXT:    vpinsrw $5, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512BW-32-NEXT:    vpinsrw $6, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512BW-32-NEXT:    vpinsrw $7, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512BW-32-NEXT:    vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; AVX512BW-32-NEXT:    vpinsrw $1, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; AVX512BW-32-NEXT:    vpinsrw $2, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; AVX512BW-32-NEXT:    vpinsrw $3, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; AVX512BW-32-NEXT:    vpinsrw $4, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; AVX512BW-32-NEXT:    vpinsrw $5, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; AVX512BW-32-NEXT:    vpinsrw $6, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; AVX512BW-32-NEXT:    vpinsrw $7, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; AVX512BW-32-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
-; AVX512BW-32-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
-; AVX512BW-32-NEXT:    retl
+; AVX-32-LABEL: test_buildvector_v32i16:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vpinsrw $1, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrw $2, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrw $3, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrw $4, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrw $5, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrw $6, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrw $7, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vpinsrw $1, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX-32-NEXT:    vpinsrw $2, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX-32-NEXT:    vpinsrw $3, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX-32-NEXT:    vpinsrw $4, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX-32-NEXT:    vpinsrw $5, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX-32-NEXT:    vpinsrw $6, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX-32-NEXT:    vpinsrw $7, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX-32-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX-32-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vpinsrw $1, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX-32-NEXT:    vpinsrw $2, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX-32-NEXT:    vpinsrw $3, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX-32-NEXT:    vpinsrw $4, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX-32-NEXT:    vpinsrw $5, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX-32-NEXT:    vpinsrw $6, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX-32-NEXT:    vpinsrw $7, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX-32-NEXT:    vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vpinsrw $1, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX-32-NEXT:    vpinsrw $2, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX-32-NEXT:    vpinsrw $3, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX-32-NEXT:    vpinsrw $4, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX-32-NEXT:    vpinsrw $5, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX-32-NEXT:    vpinsrw $6, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX-32-NEXT:    vpinsrw $7, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX-32-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX-32-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX-32-NEXT:    retl
 ;
-; AVX512BW-64-LABEL: test_buildvector_v32i16:
-; AVX512BW-64:       # %bb.0:
-; AVX512BW-64-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX512BW-64-NEXT:    vpinsrw $1, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512BW-64-NEXT:    vpinsrw $2, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512BW-64-NEXT:    vpinsrw $3, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512BW-64-NEXT:    vpinsrw $4, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512BW-64-NEXT:    vpinsrw $5, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512BW-64-NEXT:    vpinsrw $6, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512BW-64-NEXT:    vpinsrw $7, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512BW-64-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX512BW-64-NEXT:    vpinsrw $1, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX512BW-64-NEXT:    vpinsrw $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX512BW-64-NEXT:    vpinsrw $3, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX512BW-64-NEXT:    vpinsrw $4, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX512BW-64-NEXT:    vpinsrw $5, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX512BW-64-NEXT:    vpinsrw $6, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX512BW-64-NEXT:    vpinsrw $7, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX512BW-64-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX512BW-64-NEXT:    vmovd %edi, %xmm1
-; AVX512BW-64-NEXT:    vpinsrw $1, %esi, %xmm1, %xmm1
-; AVX512BW-64-NEXT:    vpinsrw $2, %edx, %xmm1, %xmm1
-; AVX512BW-64-NEXT:    vpinsrw $3, %ecx, %xmm1, %xmm1
-; AVX512BW-64-NEXT:    vpinsrw $4, %r8d, %xmm1, %xmm1
-; AVX512BW-64-NEXT:    vpinsrw $5, %r9d, %xmm1, %xmm1
-; AVX512BW-64-NEXT:    vpinsrw $6, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX512BW-64-NEXT:    vpinsrw $7, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX512BW-64-NEXT:    vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; AVX512BW-64-NEXT:    vpinsrw $1, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512BW-64-NEXT:    vpinsrw $2, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512BW-64-NEXT:    vpinsrw $3, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512BW-64-NEXT:    vpinsrw $4, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512BW-64-NEXT:    vpinsrw $5, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512BW-64-NEXT:    vpinsrw $6, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512BW-64-NEXT:    vpinsrw $7, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512BW-64-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
-; AVX512BW-64-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
-; AVX512BW-64-NEXT:    retq
+; AVX-64-LABEL: test_buildvector_v32i16:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-64-NEXT:    vpinsrw $1, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX-64-NEXT:    vpinsrw $2, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX-64-NEXT:    vpinsrw $3, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX-64-NEXT:    vpinsrw $4, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX-64-NEXT:    vpinsrw $5, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX-64-NEXT:    vpinsrw $6, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX-64-NEXT:    vpinsrw $7, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX-64-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX-64-NEXT:    vpinsrw $1, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX-64-NEXT:    vpinsrw $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX-64-NEXT:    vpinsrw $3, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX-64-NEXT:    vpinsrw $4, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX-64-NEXT:    vpinsrw $5, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX-64-NEXT:    vpinsrw $6, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX-64-NEXT:    vpinsrw $7, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX-64-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX-64-NEXT:    vmovd %edi, %xmm1
+; AVX-64-NEXT:    vpinsrw $1, %esi, %xmm1, %xmm1
+; AVX-64-NEXT:    vpinsrw $2, %edx, %xmm1, %xmm1
+; AVX-64-NEXT:    vpinsrw $3, %ecx, %xmm1, %xmm1
+; AVX-64-NEXT:    vpinsrw $4, %r8d, %xmm1, %xmm1
+; AVX-64-NEXT:    vpinsrw $5, %r9d, %xmm1, %xmm1
+; AVX-64-NEXT:    vpinsrw $6, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX-64-NEXT:    vpinsrw $7, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX-64-NEXT:    vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; AVX-64-NEXT:    vpinsrw $1, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX-64-NEXT:    vpinsrw $2, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX-64-NEXT:    vpinsrw $3, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX-64-NEXT:    vpinsrw $4, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX-64-NEXT:    vpinsrw $5, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX-64-NEXT:    vpinsrw $6, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX-64-NEXT:    vpinsrw $7, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX-64-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX-64-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX-64-NEXT:    retq
   %ins0  = insertelement <32 x i16> undef,  i16 %a0,  i32 0
   %ins1  = insertelement <32 x i16> %ins0,  i16 %a1,  i32 1
   %ins2  = insertelement <32 x i16> %ins1,  i16 %a2,  i32 2
@@ -345,287 +269,147 @@ define <32 x i16> @test_buildvector_v32i16(i16 %a0, i16 %a1, i16 %a2, i16 %a3, i
 }
 
 define <64 x i8> @test_buildvector_v64i8(i8 %a0, i8 %a1, i8 %a2, i8 %a3, i8 %a4, i8 %a5, i8 %a6, i8 %a7, i8 %a8, i8 %a9, i8 %a10, i8 %a11, i8 %a12, i8 %a13, i8 %a14, i8 %a15, i8 %a16, i8 %a17, i8 %a18, i8 %a19, i8 %a20, i8 %a21, i8 %a22, i8 %a23, i8 %a24, i8 %a25, i8 %a26, i8 %a27, i8 %a28, i8 %a29, i8 %a30, i8 %a31, i8 %a32, i8 %a33, i8 %a34, i8 %a35, i8 %a36, i8 %a37, i8 %a38, i8 %a39, i8 %a40, i8 %a41, i8 %a42, i8 %a43, i8 %a44, i8 %a45, i8 %a46, i8 %a47, i8 %a48, i8 %a49, i8 %a50, i8 %a51, i8 %a52, i8 %a53, i8 %a54, i8 %a55, i8 %a56, i8 %a57, i8 %a58, i8 %a59, i8 %a60, i8 %a61, i8 %a62, i8 %a63) {
-; AVX512F-32-LABEL: test_buildvector_v64i8:
-; AVX512F-32:       # %bb.0:
-; AVX512F-32-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX512F-32-NEXT:    vpinsrb $1, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX512F-32-NEXT:    vpinsrb $2, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX512F-32-NEXT:    vpinsrb $3, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX512F-32-NEXT:    vpinsrb $4, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX512F-32-NEXT:    vpinsrb $5, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX512F-32-NEXT:    vpinsrb $6, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX512F-32-NEXT:    vpinsrb $7, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX512F-32-NEXT:    vpinsrb $8, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX512F-32-NEXT:    vpinsrb $9, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX512F-32-NEXT:    vpinsrb $10, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX512F-32-NEXT:    vpinsrb $11, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX512F-32-NEXT:    vpinsrb $12, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX512F-32-NEXT:    vpinsrb $13, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX512F-32-NEXT:    vpinsrb $14, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX512F-32-NEXT:    vpinsrb $15, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX512F-32-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX512F-32-NEXT:    vpinsrb $1, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512F-32-NEXT:    vpinsrb $2, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512F-32-NEXT:    vpinsrb $3, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512F-32-NEXT:    vpinsrb $4, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512F-32-NEXT:    vpinsrb $5, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512F-32-NEXT:    vpinsrb $6, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512F-32-NEXT:    vpinsrb $7, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512F-32-NEXT:    vpinsrb $8, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512F-32-NEXT:    vpinsrb $9, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512F-32-NEXT:    vpinsrb $10, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512F-32-NEXT:    vpinsrb $11, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512F-32-NEXT:    vpinsrb $12, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512F-32-NEXT:    vpinsrb $13, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512F-32-NEXT:    vpinsrb $14, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512F-32-NEXT:    vpinsrb $15, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512F-32-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX512F-32-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX512F-32-NEXT:    vpinsrb $1, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512F-32-NEXT:    vpinsrb $2, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512F-32-NEXT:    vpinsrb $3, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512F-32-NEXT:    vpinsrb $4, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512F-32-NEXT:    vpinsrb $5, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512F-32-NEXT:    vpinsrb $6, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512F-32-NEXT:    vpinsrb $7, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512F-32-NEXT:    vpinsrb $8, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512F-32-NEXT:    vpinsrb $9, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512F-32-NEXT:    vpinsrb $10, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512F-32-NEXT:    vpinsrb $11, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512F-32-NEXT:    vpinsrb $12, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512F-32-NEXT:    vpinsrb $13, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512F-32-NEXT:    vpinsrb $14, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512F-32-NEXT:    vpinsrb $15, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512F-32-NEXT:    vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; AVX512F-32-NEXT:    vpinsrb $1, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; AVX512F-32-NEXT:    vpinsrb $2, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; AVX512F-32-NEXT:    vpinsrb $3, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; AVX512F-32-NEXT:    vpinsrb $4, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; AVX512F-32-NEXT:    vpinsrb $5, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; AVX512F-32-NEXT:    vpinsrb $6, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; AVX512F-32-NEXT:    vpinsrb $7, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; AVX512F-32-NEXT:    vpinsrb $8, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; AVX512F-32-NEXT:    vpinsrb $9, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; AVX512F-32-NEXT:    vpinsrb $10, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; AVX512F-32-NEXT:    vpinsrb $11, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; AVX512F-32-NEXT:    vpinsrb $12, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; AVX512F-32-NEXT:    vpinsrb $13, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; AVX512F-32-NEXT:    vpinsrb $14, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; AVX512F-32-NEXT:    vpinsrb $15, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; AVX512F-32-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
-; AVX512F-32-NEXT:    retl
-;
-; AVX512F-64-LABEL: test_buildvector_v64i8:
-; AVX512F-64:       # %bb.0:
-; AVX512F-64-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX512F-64-NEXT:    vpinsrb $1, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512F-64-NEXT:    vpinsrb $2, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512F-64-NEXT:    vpinsrb $3, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512F-64-NEXT:    vpinsrb $4, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512F-64-NEXT:    vpinsrb $5, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512F-64-NEXT:    vpinsrb $6, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512F-64-NEXT:    vpinsrb $7, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512F-64-NEXT:    vpinsrb $8, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512F-64-NEXT:    vpinsrb $9, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512F-64-NEXT:    vpinsrb $10, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512F-64-NEXT:    vpinsrb $11, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512F-64-NEXT:    vpinsrb $12, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512F-64-NEXT:    vpinsrb $13, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512F-64-NEXT:    vpinsrb $14, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512F-64-NEXT:    vpinsrb $15, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512F-64-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX512F-64-NEXT:    vpinsrb $1, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX512F-64-NEXT:    vpinsrb $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX512F-64-NEXT:    vpinsrb $3, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX512F-64-NEXT:    vpinsrb $4, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX512F-64-NEXT:    vpinsrb $5, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX512F-64-NEXT:    vpinsrb $6, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX512F-64-NEXT:    vpinsrb $7, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX512F-64-NEXT:    vpinsrb $8, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX512F-64-NEXT:    vpinsrb $9, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX512F-64-NEXT:    vpinsrb $10, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX512F-64-NEXT:    vpinsrb $11, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX512F-64-NEXT:    vpinsrb $12, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX512F-64-NEXT:    vpinsrb $13, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX512F-64-NEXT:    vpinsrb $14, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX512F-64-NEXT:    vpinsrb $15, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX512F-64-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512F-64-NEXT:    vmovd %edi, %xmm0
-; AVX512F-64-NEXT:    vpinsrb $1, %esi, %xmm0, %xmm0
-; AVX512F-64-NEXT:    vpinsrb $2, %edx, %xmm0, %xmm0
-; AVX512F-64-NEXT:    vpinsrb $3, %ecx, %xmm0, %xmm0
-; AVX512F-64-NEXT:    vpinsrb $4, %r8d, %xmm0, %xmm0
-; AVX512F-64-NEXT:    vpinsrb $5, %r9d, %xmm0, %xmm0
-; AVX512F-64-NEXT:    vpinsrb $6, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512F-64-NEXT:    vpinsrb $7, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512F-64-NEXT:    vpinsrb $8, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512F-64-NEXT:    vpinsrb $9, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512F-64-NEXT:    vpinsrb $10, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512F-64-NEXT:    vpinsrb $11, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512F-64-NEXT:    vpinsrb $12, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512F-64-NEXT:    vpinsrb $13, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512F-64-NEXT:    vpinsrb $14, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512F-64-NEXT:    vpinsrb $15, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512F-64-NEXT:    vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; AVX512F-64-NEXT:    vpinsrb $1, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512F-64-NEXT:    vpinsrb $2, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512F-64-NEXT:    vpinsrb $3, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512F-64-NEXT:    vpinsrb $4, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512F-64-NEXT:    vpinsrb $5, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512F-64-NEXT:    vpinsrb $6, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512F-64-NEXT:    vpinsrb $7, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512F-64-NEXT:    vpinsrb $8, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512F-64-NEXT:    vpinsrb $9, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512F-64-NEXT:    vpinsrb $10, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512F-64-NEXT:    vpinsrb $11, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512F-64-NEXT:    vpinsrb $12, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512F-64-NEXT:    vpinsrb $13, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512F-64-NEXT:    vpinsrb $14, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512F-64-NEXT:    vpinsrb $15, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512F-64-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
-; AVX512F-64-NEXT:    retq
-;
-; AVX512BW-32-LABEL: test_buildvector_v64i8:
-; AVX512BW-32:       # %bb.0:
-; AVX512BW-32-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX512BW-32-NEXT:    vpinsrb $1, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX512BW-32-NEXT:    vpinsrb $2, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX512BW-32-NEXT:    vpinsrb $3, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX512BW-32-NEXT:    vpinsrb $4, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX512BW-32-NEXT:    vpinsrb $5, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX512BW-32-NEXT:    vpinsrb $6, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX512BW-32-NEXT:    vpinsrb $7, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX512BW-32-NEXT:    vpinsrb $8, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX512BW-32-NEXT:    vpinsrb $9, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX512BW-32-NEXT:    vpinsrb $10, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX512BW-32-NEXT:    vpinsrb $11, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX512BW-32-NEXT:    vpinsrb $12, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX512BW-32-NEXT:    vpinsrb $13, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX512BW-32-NEXT:    vpinsrb $14, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX512BW-32-NEXT:    vpinsrb $15, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX512BW-32-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX512BW-32-NEXT:    vpinsrb $1, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512BW-32-NEXT:    vpinsrb $2, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512BW-32-NEXT:    vpinsrb $3, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512BW-32-NEXT:    vpinsrb $4, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512BW-32-NEXT:    vpinsrb $5, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512BW-32-NEXT:    vpinsrb $6, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512BW-32-NEXT:    vpinsrb $7, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512BW-32-NEXT:    vpinsrb $8, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512BW-32-NEXT:    vpinsrb $9, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512BW-32-NEXT:    vpinsrb $10, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512BW-32-NEXT:    vpinsrb $11, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512BW-32-NEXT:    vpinsrb $12, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512BW-32-NEXT:    vpinsrb $13, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512BW-32-NEXT:    vpinsrb $14, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512BW-32-NEXT:    vpinsrb $15, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512BW-32-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX512BW-32-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX512BW-32-NEXT:    vpinsrb $1, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512BW-32-NEXT:    vpinsrb $2, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512BW-32-NEXT:    vpinsrb $3, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512BW-32-NEXT:    vpinsrb $4, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512BW-32-NEXT:    vpinsrb $5, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512BW-32-NEXT:    vpinsrb $6, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512BW-32-NEXT:    vpinsrb $7, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512BW-32-NEXT:    vpinsrb $8, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512BW-32-NEXT:    vpinsrb $9, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512BW-32-NEXT:    vpinsrb $10, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512BW-32-NEXT:    vpinsrb $11, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512BW-32-NEXT:    vpinsrb $12, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512BW-32-NEXT:    vpinsrb $13, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512BW-32-NEXT:    vpinsrb $14, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512BW-32-NEXT:    vpinsrb $15, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX512BW-32-NEXT:    vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; AVX512BW-32-NEXT:    vpinsrb $1, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; AVX512BW-32-NEXT:    vpinsrb $2, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; AVX512BW-32-NEXT:    vpinsrb $3, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; AVX512BW-32-NEXT:    vpinsrb $4, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; AVX512BW-32-NEXT:    vpinsrb $5, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; AVX512BW-32-NEXT:    vpinsrb $6, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; AVX512BW-32-NEXT:    vpinsrb $7, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; AVX512BW-32-NEXT:    vpinsrb $8, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; AVX512BW-32-NEXT:    vpinsrb $9, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; AVX512BW-32-NEXT:    vpinsrb $10, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; AVX512BW-32-NEXT:    vpinsrb $11, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; AVX512BW-32-NEXT:    vpinsrb $12, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; AVX512BW-32-NEXT:    vpinsrb $13, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; AVX512BW-32-NEXT:    vpinsrb $14, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; AVX512BW-32-NEXT:    vpinsrb $15, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; AVX512BW-32-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
-; AVX512BW-32-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
-; AVX512BW-32-NEXT:    retl
+; AVX-32-LABEL: test_buildvector_v64i8:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vpinsrb $1, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrb $2, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrb $3, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrb $4, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrb $5, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrb $6, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrb $7, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrb $8, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrb $9, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrb $10, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrb $11, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrb $12, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrb $13, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrb $14, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrb $15, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vpinsrb $1, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX-32-NEXT:    vpinsrb $2, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX-32-NEXT:    vpinsrb $3, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX-32-NEXT:    vpinsrb $4, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX-32-NEXT:    vpinsrb $5, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX-32-NEXT:    vpinsrb $6, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX-32-NEXT:    vpinsrb $7, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX-32-NEXT:    vpinsrb $8, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX-32-NEXT:    vpinsrb $9, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX-32-NEXT:    vpinsrb $10, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX-32-NEXT:    vpinsrb $11, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX-32-NEXT:    vpinsrb $12, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX-32-NEXT:    vpinsrb $13, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX-32-NEXT:    vpinsrb $14, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX-32-NEXT:    vpinsrb $15, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX-32-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX-32-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vpinsrb $1, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX-32-NEXT:    vpinsrb $2, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX-32-NEXT:    vpinsrb $3, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX-32-NEXT:    vpinsrb $4, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX-32-NEXT:    vpinsrb $5, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX-32-NEXT:    vpinsrb $6, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX-32-NEXT:    vpinsrb $7, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX-32-NEXT:    vpinsrb $8, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX-32-NEXT:    vpinsrb $9, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX-32-NEXT:    vpinsrb $10, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX-32-NEXT:    vpinsrb $11, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX-32-NEXT:    vpinsrb $12, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX-32-NEXT:    vpinsrb $13, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX-32-NEXT:    vpinsrb $14, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX-32-NEXT:    vpinsrb $15, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX-32-NEXT:    vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vpinsrb $1, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX-32-NEXT:    vpinsrb $2, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX-32-NEXT:    vpinsrb $3, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX-32-NEXT:    vpinsrb $4, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX-32-NEXT:    vpinsrb $5, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX-32-NEXT:    vpinsrb $6, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX-32-NEXT:    vpinsrb $7, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX-32-NEXT:    vpinsrb $8, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX-32-NEXT:    vpinsrb $9, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX-32-NEXT:    vpinsrb $10, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX-32-NEXT:    vpinsrb $11, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX-32-NEXT:    vpinsrb $12, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX-32-NEXT:    vpinsrb $13, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX-32-NEXT:    vpinsrb $14, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX-32-NEXT:    vpinsrb $15, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX-32-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX-32-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX-32-NEXT:    retl
 ;
-; AVX512BW-64-LABEL: test_buildvector_v64i8:
-; AVX512BW-64:       # %bb.0:
-; AVX512BW-64-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX512BW-64-NEXT:    vpinsrb $1, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512BW-64-NEXT:    vpinsrb $2, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512BW-64-NEXT:    vpinsrb $3, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512BW-64-NEXT:    vpinsrb $4, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512BW-64-NEXT:    vpinsrb $5, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512BW-64-NEXT:    vpinsrb $6, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512BW-64-NEXT:    vpinsrb $7, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512BW-64-NEXT:    vpinsrb $8, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512BW-64-NEXT:    vpinsrb $9, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512BW-64-NEXT:    vpinsrb $10, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512BW-64-NEXT:    vpinsrb $11, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512BW-64-NEXT:    vpinsrb $12, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512BW-64-NEXT:    vpinsrb $13, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512BW-64-NEXT:    vpinsrb $14, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512BW-64-NEXT:    vpinsrb $15, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512BW-64-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX512BW-64-NEXT:    vpinsrb $1, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX512BW-64-NEXT:    vpinsrb $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX512BW-64-NEXT:    vpinsrb $3, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX512BW-64-NEXT:    vpinsrb $4, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX512BW-64-NEXT:    vpinsrb $5, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX512BW-64-NEXT:    vpinsrb $6, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX512BW-64-NEXT:    vpinsrb $7, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX512BW-64-NEXT:    vpinsrb $8, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX512BW-64-NEXT:    vpinsrb $9, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX512BW-64-NEXT:    vpinsrb $10, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX512BW-64-NEXT:    vpinsrb $11, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX512BW-64-NEXT:    vpinsrb $12, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX512BW-64-NEXT:    vpinsrb $13, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX512BW-64-NEXT:    vpinsrb $14, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX512BW-64-NEXT:    vpinsrb $15, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX512BW-64-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX512BW-64-NEXT:    vmovd %edi, %xmm1
-; AVX512BW-64-NEXT:    vpinsrb $1, %esi, %xmm1, %xmm1
-; AVX512BW-64-NEXT:    vpinsrb $2, %edx, %xmm1, %xmm1
-; AVX512BW-64-NEXT:    vpinsrb $3, %ecx, %xmm1, %xmm1
-; AVX512BW-64-NEXT:    vpinsrb $4, %r8d, %xmm1, %xmm1
-; AVX512BW-64-NEXT:    vpinsrb $5, %r9d, %xmm1, %xmm1
-; AVX512BW-64-NEXT:    vpinsrb $6, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX512BW-64-NEXT:    vpinsrb $7, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX512BW-64-NEXT:    vpinsrb $8, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX512BW-64-NEXT:    vpinsrb $9, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX512BW-64-NEXT:    vpinsrb $10, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX512BW-64-NEXT:    vpinsrb $11, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX512BW-64-NEXT:    vpinsrb $12, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX512BW-64-NEXT:    vpinsrb $13, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX512BW-64-NEXT:    vpinsrb $14, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX512BW-64-NEXT:    vpinsrb $15, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX512BW-64-NEXT:    vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; AVX512BW-64-NEXT:    vpinsrb $1, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512BW-64-NEXT:    vpinsrb $2, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512BW-64-NEXT:    vpinsrb $3, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512BW-64-NEXT:    vpinsrb $4, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512BW-64-NEXT:    vpinsrb $5, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512BW-64-NEXT:    vpinsrb $6, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512BW-64-NEXT:    vpinsrb $7, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512BW-64-NEXT:    vpinsrb $8, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512BW-64-NEXT:    vpinsrb $9, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512BW-64-NEXT:    vpinsrb $10, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512BW-64-NEXT:    vpinsrb $11, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512BW-64-NEXT:    vpinsrb $12, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512BW-64-NEXT:    vpinsrb $13, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512BW-64-NEXT:    vpinsrb $14, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512BW-64-NEXT:    vpinsrb $15, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512BW-64-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
-; AVX512BW-64-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
-; AVX512BW-64-NEXT:    retq
+; AVX-64-LABEL: test_buildvector_v64i8:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-64-NEXT:    vpinsrb $1, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX-64-NEXT:    vpinsrb $2, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX-64-NEXT:    vpinsrb $3, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX-64-NEXT:    vpinsrb $4, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX-64-NEXT:    vpinsrb $5, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX-64-NEXT:    vpinsrb $6, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX-64-NEXT:    vpinsrb $7, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX-64-NEXT:    vpinsrb $8, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX-64-NEXT:    vpinsrb $9, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX-64-NEXT:    vpinsrb $10, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX-64-NEXT:    vpinsrb $11, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX-64-NEXT:    vpinsrb $12, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX-64-NEXT:    vpinsrb $13, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX-64-NEXT:    vpinsrb $14, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX-64-NEXT:    vpinsrb $15, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX-64-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX-64-NEXT:    vpinsrb $1, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX-64-NEXT:    vpinsrb $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX-64-NEXT:    vpinsrb $3, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX-64-NEXT:    vpinsrb $4, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX-64-NEXT:    vpinsrb $5, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX-64-NEXT:    vpinsrb $6, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX-64-NEXT:    vpinsrb $7, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX-64-NEXT:    vpinsrb $8, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX-64-NEXT:    vpinsrb $9, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX-64-NEXT:    vpinsrb $10, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX-64-NEXT:    vpinsrb $11, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX-64-NEXT:    vpinsrb $12, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX-64-NEXT:    vpinsrb $13, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX-64-NEXT:    vpinsrb $14, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX-64-NEXT:    vpinsrb $15, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX-64-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX-64-NEXT:    vmovd %edi, %xmm1
+; AVX-64-NEXT:    vpinsrb $1, %esi, %xmm1, %xmm1
+; AVX-64-NEXT:    vpinsrb $2, %edx, %xmm1, %xmm1
+; AVX-64-NEXT:    vpinsrb $3, %ecx, %xmm1, %xmm1
+; AVX-64-NEXT:    vpinsrb $4, %r8d, %xmm1, %xmm1
+; AVX-64-NEXT:    vpinsrb $5, %r9d, %xmm1, %xmm1
+; AVX-64-NEXT:    vpinsrb $6, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX-64-NEXT:    vpinsrb $7, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX-64-NEXT:    vpinsrb $8, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX-64-NEXT:    vpinsrb $9, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX-64-NEXT:    vpinsrb $10, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX-64-NEXT:    vpinsrb $11, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX-64-NEXT:    vpinsrb $12, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX-64-NEXT:    vpinsrb $13, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX-64-NEXT:    vpinsrb $14, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX-64-NEXT:    vpinsrb $15, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX-64-NEXT:    vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; AVX-64-NEXT:    vpinsrb $1, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX-64-NEXT:    vpinsrb $2, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX-64-NEXT:    vpinsrb $3, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX-64-NEXT:    vpinsrb $4, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX-64-NEXT:    vpinsrb $5, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX-64-NEXT:    vpinsrb $6, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX-64-NEXT:    vpinsrb $7, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX-64-NEXT:    vpinsrb $8, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX-64-NEXT:    vpinsrb $9, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX-64-NEXT:    vpinsrb $10, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX-64-NEXT:    vpinsrb $11, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX-64-NEXT:    vpinsrb $12, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX-64-NEXT:    vpinsrb $13, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX-64-NEXT:    vpinsrb $14, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX-64-NEXT:    vpinsrb $15, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX-64-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX-64-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX-64-NEXT:    retq
   %ins0  = insertelement <64 x i8> undef,  i8 %a0,  i32 0
   %ins1  = insertelement <64 x i8> %ins0,  i8 %a1,  i32 1
   %ins2  = insertelement <64 x i8> %ins1,  i8 %a2,  i32 2
diff --git a/llvm/test/CodeGen/X86/buildvec-insertvec.ll b/llvm/test/CodeGen/X86/buildvec-insertvec.ll
index 5261a2b63..7867e4494 100644
--- a/llvm/test/CodeGen/X86/buildvec-insertvec.ll
+++ b/llvm/test/CodeGen/X86/buildvec-insertvec.ll
@@ -6,22 +6,29 @@ define void @foo(<3 x float> %in, <4 x i8>* nocapture %out) nounwind {
 ; SSE2-LABEL: foo:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    cvttps2dq %xmm0, %xmm0
-; SSE2-NEXT:    movl $255, %eax
-; SSE2-NEXT:    movd %eax, %xmm1
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,0],xmm0[2,0]
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,0]
-; SSE2-NEXT:    andps {{.*}}(%rip), %xmm0
-; SSE2-NEXT:    packuswb %xmm0, %xmm0
-; SSE2-NEXT:    packuswb %xmm0, %xmm0
+; SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    movl -{{[0-9]+}}(%rsp), %ecx
+; SSE2-NEXT:    shll $8, %ecx
+; SSE2-NEXT:    orl %eax, %ecx
+; SSE2-NEXT:    movd %ecx, %xmm0
+; SSE2-NEXT:    movl $65280, %eax # imm = 0xFF00
+; SSE2-NEXT:    orl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    pinsrw $1, %eax, %xmm0
 ; SSE2-NEXT:    movd %xmm0, (%rdi)
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: foo:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    cvttps2dq %xmm0, %xmm0
+; SSE41-NEXT:    pextrb $8, %xmm0, %eax
+; SSE41-NEXT:    pextrb $4, %xmm0, %ecx
+; SSE41-NEXT:    pextrb $0, %xmm0, %edx
+; SSE41-NEXT:    movd %edx, %xmm0
+; SSE41-NEXT:    pinsrb $1, %ecx, %xmm0
+; SSE41-NEXT:    pinsrb $2, %eax, %xmm0
 ; SSE41-NEXT:    movl $255, %eax
-; SSE41-NEXT:    pinsrd $3, %eax, %xmm0
-; SSE41-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; SSE41-NEXT:    pinsrb $3, %eax, %xmm0
 ; SSE41-NEXT:    movd %xmm0, (%rdi)
 ; SSE41-NEXT:    retq
   %t0 = fptoui <3 x float> %in to <3 x i8>
diff --git a/llvm/test/CodeGen/X86/bypass-slow-division-32.ll b/llvm/test/CodeGen/X86/bypass-slow-division-32.ll
index 66aacf19c..988e4f9b9 100644
--- a/llvm/test/CodeGen/X86/bypass-slow-division-32.ll
+++ b/llvm/test/CodeGen/X86/bypass-slow-division-32.ll
@@ -17,7 +17,6 @@ define i32 @Test_get_quotient(i32 %a, i32 %b) nounwind {
 ; CHECK-NEXT:    retl
 ; CHECK-NEXT:  .LBB0_1:
 ; CHECK-NEXT:    movzbl %al, %eax
-; CHECK-NEXT:    # kill: def $eax killed $eax def $ax
 ; CHECK-NEXT:    divb %cl
 ; CHECK-NEXT:    movzbl %al, %eax
 ; CHECK-NEXT:    retl
@@ -41,7 +40,6 @@ define i32 @Test_get_remainder(i32 %a, i32 %b) nounwind {
 ; CHECK-NEXT:    retl
 ; CHECK-NEXT:  .LBB1_1:
 ; CHECK-NEXT:    movzbl %al, %eax
-; CHECK-NEXT:    # kill: def $eax killed $eax def $ax
 ; CHECK-NEXT:    divb %cl
 ; CHECK-NEXT:    movzbl %ah, %eax
 ; CHECK-NEXT:    retl
@@ -65,7 +63,6 @@ define i32 @Test_get_quotient_and_remainder(i32 %a, i32 %b) nounwind {
 ; CHECK-NEXT:    retl
 ; CHECK-NEXT:  .LBB2_1:
 ; CHECK-NEXT:    movzbl %al, %eax
-; CHECK-NEXT:    # kill: def $eax killed $eax def $ax
 ; CHECK-NEXT:    divb %cl
 ; CHECK-NEXT:    movzbl %ah, %edx
 ; CHECK-NEXT:    movzbl %al, %eax
@@ -103,14 +100,12 @@ define i32 @Test_use_div_and_idiv(i32 %a, i32 %b) nounwind {
 ; CHECK-NEXT:    jmp .LBB3_6
 ; CHECK-NEXT:  .LBB3_1:
 ; CHECK-NEXT:    movzbl %cl, %eax
-; CHECK-NEXT:    # kill: def $eax killed $eax def $ax
 ; CHECK-NEXT:    divb %bl
 ; CHECK-NEXT:    movzbl %al, %esi
 ; CHECK-NEXT:    testl $-256, %edi
 ; CHECK-NEXT:    jne .LBB3_5
 ; CHECK-NEXT:  .LBB3_4:
 ; CHECK-NEXT:    movzbl %cl, %eax
-; CHECK-NEXT:    # kill: def $eax killed $eax def $ax
 ; CHECK-NEXT:    divb %bl
 ; CHECK-NEXT:    movzbl %al, %eax
 ; CHECK-NEXT:  .LBB3_6:
@@ -208,7 +203,6 @@ define i32 @Test_use_div_imm_reg(i32 %a) nounwind {
 ; CHECK-NEXT:  .LBB8_1:
 ; CHECK-NEXT:    movb $4, %al
 ; CHECK-NEXT:    movzbl %al, %eax
-; CHECK-NEXT:    # kill: def $eax killed $eax def $ax
 ; CHECK-NEXT:    divb %cl
 ; CHECK-NEXT:    movzbl %al, %eax
 ; CHECK-NEXT:    retl
@@ -230,7 +224,6 @@ define i32 @Test_use_rem_imm_reg(i32 %a) nounwind {
 ; CHECK-NEXT:  .LBB9_1:
 ; CHECK-NEXT:    movb $4, %al
 ; CHECK-NEXT:    movzbl %al, %eax
-; CHECK-NEXT:    # kill: def $eax killed $eax def $ax
 ; CHECK-NEXT:    divb %cl
 ; CHECK-NEXT:    movzbl %al, %eax
 ; CHECK-NEXT:    retl
diff --git a/llvm/test/CodeGen/X86/bypass-slow-division-64.ll b/llvm/test/CodeGen/X86/bypass-slow-division-64.ll
index 11fc0df24..14a71050e 100644
--- a/llvm/test/CodeGen/X86/bypass-slow-division-64.ll
+++ b/llvm/test/CodeGen/X86/bypass-slow-division-64.ll
@@ -75,3 +75,13 @@ define i64 @Test_get_quotient_and_remainder(i64 %a, i64 %b) nounwind {
   %result = add i64 %resultdiv, %resultrem
   ret i64 %result
 }
+
+define void @PR43514(i32 %x, i32 %y) {
+; CHECK-LABEL: PR43514:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    retq
+  %z1 = zext i32 %x to i64
+  %z2 = zext i32 %y to i64
+  %s = srem i64 %z1, %z2
+  ret void
+}
diff --git a/llvm/test/CodeGen/X86/bypass-slow-division-tune.ll b/llvm/test/CodeGen/X86/bypass-slow-division-tune.ll
index a387d6873..84355448d 100644
--- a/llvm/test/CodeGen/X86/bypass-slow-division-tune.ll
+++ b/llvm/test/CodeGen/X86/bypass-slow-division-tune.ll
@@ -20,7 +20,6 @@ define i32 @div32(i32 %a, i32 %b) {
 ; ATOM-NEXT:    retq
 ; ATOM-NEXT:  .LBB0_1:
 ; ATOM-NEXT:    movzbl %dil, %eax
-; ATOM-NEXT:    # kill: def $eax killed $eax def $ax
 ; ATOM-NEXT:    divb %sil
 ; ATOM-NEXT:    movzbl %al, %eax
 ; ATOM-NEXT:    retq
@@ -131,6 +130,24 @@ define i64 @div64_optsize(i64 %a, i64 %b) optsize {
   ret i64 %div
 }
 
+define i64 @div64_pgso(i64 %a, i64 %b) !prof !15 {
+; CHECK-LABEL: div64_pgso:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    cqto
+; CHECK-NEXT:    idivq %rsi
+; CHECK-NEXT:    retq
+;
+; HUGEWS-LABEL: div64_pgso:
+; HUGEWS:       # %bb.0:
+; HUGEWS-NEXT:    movq %rdi, %rax
+; HUGEWS-NEXT:    cqto
+; HUGEWS-NEXT:    idivq %rsi
+; HUGEWS-NEXT:    retq
+  %div = sdiv i64 %a, %b
+  ret i64 %div
+}
+
 define i64 @div64_hugews(i64 %a, i64 %b) {
 ; ATOM-LABEL: div64_hugews:
 ; ATOM:       # %bb.0:
@@ -138,12 +155,12 @@ define i64 @div64_hugews(i64 %a, i64 %b) {
 ; ATOM-NEXT:    movq %rdi, %rax
 ; ATOM-NEXT:    orq %rsi, %rcx
 ; ATOM-NEXT:    shrq $32, %rcx
-; ATOM-NEXT:    je .LBB3_1
+; ATOM-NEXT:    je .LBB4_1
 ; ATOM-NEXT:  # %bb.2:
 ; ATOM-NEXT:    cqto
 ; ATOM-NEXT:    idivq %rsi
 ; ATOM-NEXT:    retq
-; ATOM-NEXT:  .LBB3_1:
+; ATOM-NEXT:  .LBB4_1:
 ; ATOM-NEXT:    # kill: def $eax killed $eax killed $rax
 ; ATOM-NEXT:    xorl %edx, %edx
 ; ATOM-NEXT:    divl %esi
@@ -156,12 +173,12 @@ define i64 @div64_hugews(i64 %a, i64 %b) {
 ; SLM-NEXT:    movq %rdi, %rax
 ; SLM-NEXT:    orq %rsi, %rcx
 ; SLM-NEXT:    shrq $32, %rcx
-; SLM-NEXT:    je .LBB3_1
+; SLM-NEXT:    je .LBB4_1
 ; SLM-NEXT:  # %bb.2:
 ; SLM-NEXT:    cqto
 ; SLM-NEXT:    idivq %rsi
 ; SLM-NEXT:    retq
-; SLM-NEXT:  .LBB3_1:
+; SLM-NEXT:  .LBB4_1:
 ; SLM-NEXT:    xorl %edx, %edx
 ; SLM-NEXT:    # kill: def $eax killed $eax killed $rax
 ; SLM-NEXT:    divl %esi
@@ -174,12 +191,12 @@ define i64 @div64_hugews(i64 %a, i64 %b) {
 ; SKL-NEXT:    movq %rdi, %rcx
 ; SKL-NEXT:    orq %rsi, %rcx
 ; SKL-NEXT:    shrq $32, %rcx
-; SKL-NEXT:    je .LBB3_1
+; SKL-NEXT:    je .LBB4_1
 ; SKL-NEXT:  # %bb.2:
 ; SKL-NEXT:    cqto
 ; SKL-NEXT:    idivq %rsi
 ; SKL-NEXT:    retq
-; SKL-NEXT:  .LBB3_1:
+; SKL-NEXT:  .LBB4_1:
 ; SKL-NEXT:    # kill: def $eax killed $eax killed $rax
 ; SKL-NEXT:    xorl %edx, %edx
 ; SKL-NEXT:    divl %esi
@@ -214,6 +231,24 @@ define i32 @div32_optsize(i32 %a, i32 %b) optsize {
   ret i32 %div
 }
 
+define i32 @div32_pgso(i32 %a, i32 %b) !prof !15 {
+; CHECK-LABEL: div32_pgso:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    cltd
+; CHECK-NEXT:    idivl %esi
+; CHECK-NEXT:    retq
+;
+; HUGEWS-LABEL: div32_pgso:
+; HUGEWS:       # %bb.0:
+; HUGEWS-NEXT:    movl %edi, %eax
+; HUGEWS-NEXT:    cltd
+; HUGEWS-NEXT:    idivl %esi
+; HUGEWS-NEXT:    retq
+  %div = sdiv i32 %a, %b
+  ret i32 %div
+}
+
 define i32 @div32_minsize(i32 %a, i32 %b) minsize {
 ; CHECK-LABEL: div32_minsize:
 ; CHECK:       # %bb.0:
@@ -247,3 +282,4 @@ define i32 @div32_minsize(i32 %a, i32 %b) minsize {
 !12 = !{i32 10000, i64 1000, i32 1}
 !13 = !{i32 999000, i64 1000, i32 3}
 !14 = !{i32 999999, i64 5, i32 3}
+!15 = !{!"function_entry_count", i64 0}
diff --git a/llvm/test/CodeGen/X86/byval6.ll b/llvm/test/CodeGen/X86/byval6.ll
index 21754a1ef..e7e29bf54 100644
--- a/llvm/test/CodeGen/X86/byval6.ll
+++ b/llvm/test/CodeGen/X86/byval6.ll
@@ -1,11 +1,50 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mcpu=generic -mtriple=i686-- | grep add | not grep 16
+; RUN: llc < %s -mcpu=generic -mtriple=i686-- | FileCheck %s
 
 	%struct.W = type { x86_fp80, x86_fp80 }
 @B = global %struct.W { x86_fp80 0xK4001A000000000000000, x86_fp80 0xK4001C000000000000000 }, align 32
 @.cpx = internal constant %struct.W { x86_fp80 0xK4001E000000000000000, x86_fp80 0xK40028000000000000000 }
 
 define i32 @main() nounwind  {
+; CHECK-LABEL: main:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushl %ebx
+; CHECK-NEXT:    pushl %edi
+; CHECK-NEXT:    pushl %esi
+; CHECK-NEXT:    movl .cpx+20, %eax
+; CHECK-NEXT:    movl .cpx+16, %ecx
+; CHECK-NEXT:    movl .cpx+12, %edx
+; CHECK-NEXT:    movl .cpx+8, %esi
+; CHECK-NEXT:    movl .cpx+4, %edi
+; CHECK-NEXT:    movl .cpx, %ebx
+; CHECK-NEXT:    pushl %eax
+; CHECK-NEXT:    pushl %ecx
+; CHECK-NEXT:    pushl %edx
+; CHECK-NEXT:    pushl %esi
+; CHECK-NEXT:    pushl %edi
+; CHECK-NEXT:    pushl %ebx
+; CHECK-NEXT:    pushl $3
+; CHECK-NEXT:    calll bar
+; CHECK-NEXT:    addl $28, %esp
+; CHECK-NEXT:    movl B+20, %eax
+; CHECK-NEXT:    movl B+16, %ecx
+; CHECK-NEXT:    movl B+12, %edx
+; CHECK-NEXT:    movl B+8, %esi
+; CHECK-NEXT:    movl B+4, %edi
+; CHECK-NEXT:    movl B, %ebx
+; CHECK-NEXT:    pushl %eax
+; CHECK-NEXT:    pushl %ecx
+; CHECK-NEXT:    pushl %edx
+; CHECK-NEXT:    pushl %esi
+; CHECK-NEXT:    pushl %edi
+; CHECK-NEXT:    pushl %ebx
+; CHECK-NEXT:    pushl $3
+; CHECK-NEXT:    calll baz
+; CHECK-NEXT:    addl $28, %esp
+; CHECK-NEXT:    popl %esi
+; CHECK-NEXT:    popl %edi
+; CHECK-NEXT:    popl %ebx
+; CHECK-NEXT:    retl
 entry:
 	tail call void (i32, ...) @bar( i32 3, %struct.W* byval  @.cpx ) nounwind
 	tail call void (i32, ...) @baz( i32 3, %struct.W* byval  @B ) nounwind
diff --git a/llvm/test/CodeGen/X86/callbr-asm.ll b/llvm/test/CodeGen/X86/callbr-asm.ll
index ed3c314ed..8938aefca 100644
--- a/llvm/test/CodeGen/X86/callbr-asm.ll
+++ b/llvm/test/CodeGen/X86/callbr-asm.ll
@@ -53,7 +53,6 @@ fail:
   ret i32 1
 }
 
-
 ; Test 3 - asm-goto implements a loop. The loop gets recognized, but many loop
 ; transforms fail due to canonicalization having callbr exceptions. Trivial
 ; blocks at labels 1 and 3 also don't get simplified due to callbr.
@@ -131,3 +130,32 @@ normal1:                                          ; preds = %normal0
   %1 = load i32, i32* %a.addr, align 4
   ret i32 %1
 }
+
+; Test 4 - asm-goto referenced with the 'l' (ell) modifier and not.
+define void @test4() {
+; CHECK-LABEL: test4:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    #APP
+; CHECK-NOT:     ja .Ltmp50
+; CHECK-NEXT:    ja .Ltmp5
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:  .LBB3_1:
+; CHECK-NEXT:    #APP
+; CHECK-NOT:     ja .Ltmp50
+; CHECK-NEXT:    ja .Ltmp5
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    jmp .LBB3_3
+entry:
+  callbr void asm sideeffect "ja $0", "X,~{dirflag},~{fpsr},~{flags}"(i8* blockaddress(@test4, %quux))
+          to label %asm.fallthrough [label %quux]
+
+asm.fallthrough:                                  ; preds = %entry
+  callbr void asm sideeffect "ja ${0:l}", "X,~{dirflag},~{fpsr},~{flags}"(i8* blockaddress(@test4, %quux))
+          to label %cleanup [label %quux]
+
+quux:                                             ; preds = %asm.fallthrough, %entry
+  br label %cleanup
+
+cleanup:                                          ; preds = %asm.fallthrough, %quux
+  ret void
+}
diff --git a/llvm/test/CodeGen/X86/catchpad-lifetime.ll b/llvm/test/CodeGen/X86/catchpad-lifetime.ll
index d85adec36..8a6d4cbd2 100644
--- a/llvm/test/CodeGen/X86/catchpad-lifetime.ll
+++ b/llvm/test/CodeGen/X86/catchpad-lifetime.ll
@@ -7,6 +7,8 @@ declare void @throw()
 
 declare i32 @__CxxFrameHandler3(...)
 
+declare void @llvm.trap()
+
 define void @test1() personality i32 (...)* @__CxxFrameHandler3 {
 entry:
   %alloca2 = alloca i8*, align 4
@@ -30,6 +32,7 @@ catch.pad:                                        ; preds = %catch.dispatch
   %bc2 = bitcast i8** %alloca2 to i8*
   call void @llvm.lifetime.start.p0i8(i64 4, i8* %bc2)
   store volatile i8* null, i8** %alloca1
+  call void @llvm.trap()
   unreachable
 
 ; CHECK-LABEL: "?catch$2@?0?test1@4HA"
@@ -67,6 +70,7 @@ catch.pad:                                        ; preds = %catch.dispatch
   %bc2 = bitcast i8** %alloca2 to i8*
   call void @llvm.lifetime.start.p0i8(i64 4, i8* %bc2)
   store volatile i8* null, i8** %alloca1
+  call void @llvm.trap()
   unreachable
 
 ; CHECK-LABEL: "?catch$2@?0?test2@4HA"
diff --git a/llvm/test/CodeGen/X86/catchpad-realign-savexmm.ll b/llvm/test/CodeGen/X86/catchpad-realign-savexmm.ll
index 116010179..e68294b83 100644
--- a/llvm/test/CodeGen/X86/catchpad-realign-savexmm.ll
+++ b/llvm/test/CodeGen/X86/catchpad-realign-savexmm.ll
@@ -29,13 +29,13 @@ catch:
 
 ; CHECK: f: # @f
 ; CHECK: pushq   %rbp
-; CHECK: .seh_pushreg 5
+; CHECK: .seh_pushreg %rbp
 ; CHECK: subq    $64, %rsp
 ; CHECK: .seh_stackalloc 64
 ; CHECK: leaq    64(%rsp), %rbp
-; CHECK: .seh_setframe 5, 64
+; CHECK: .seh_setframe %rbp, 64
 ; CHECK: movaps  %xmm6, -16(%rbp)        # 16-byte Spill
-; CHECK: .seh_savexmm 6, 48
+; CHECK: .seh_savexmm %xmm6, 48
 ; CHECK: .seh_endprologue
 ; CHECK: movq    $-2, -24(%rbp)
 ; CHECK: movsd   fp_global(%rip), %xmm6  # xmm6 = mem[0],zero
@@ -51,3 +51,18 @@ catch:
 ; CHECK: popq    %rbp
 ; CHECK: retq
 ; CHECK: .seh_handlerdata
+; CHECK: # %catch
+; CHECK: movq    %rdx, 16(%rsp)
+; CHECK: pushq   %rbp
+; CHECK: .seh_pushreg %rbp
+; CHECK: subq    $48, %rsp
+; CHECK: .seh_stackalloc 48
+; CHECK: leaq    64(%rdx), %rbp
+; CHECK: movapd  %xmm6, 32(%rsp)
+; CHECK: .seh_savexmm %xmm6, 32
+; CHECK: .seh_endprologue
+; CHECK: movapd  32(%rsp), %xmm6
+; CHECK: leaq    .LBB0_1(%rip), %rax
+; CHECK: addq    $48, %rsp
+; CHECK: popq    %rbp
+; CHECK: retq # CATCHRET
diff --git a/llvm/test/CodeGen/X86/catchpad-regmask.ll b/llvm/test/CodeGen/X86/catchpad-regmask.ll
index 0d436f6eb..969393f95 100644
--- a/llvm/test/CodeGen/X86/catchpad-regmask.ll
+++ b/llvm/test/CodeGen/X86/catchpad-regmask.ll
@@ -75,7 +75,7 @@ unreachable:                                      ; preds = %entry
 ; CHECK: popq %rbp
 ; CHECK: retq
 
-; CHECK: "?catch$2@?0?global_array@4HA":
+; CHECK: "?catch${{[0-9]+}}@?0?global_array@4HA":
 ; CHECK: pushq   %rbp
 ; CHECK: movslq  {{.*}}, %[[idx:[^ ]*]]
 ; CHECK: leaq    array(%rip), %[[base:[^ ]*]]
@@ -122,7 +122,7 @@ unreachable:                                      ; preds = %entry
 ; CHECK: popq %rbp
 ; CHECK: retq
 
-; CHECK: "?catch$2@?0?access_imported@4HA":
+; CHECK: "?catch${{[0-9]+}}@?0?access_imported@4HA":
 ; CHECK: pushq   %rbp
 ; CHECK: movq    __imp_imported(%rip), %[[base:[^ ]*]]
 ; CHECK: movl    $222, (%[[base]])
@@ -130,7 +130,7 @@ unreachable:                                      ; preds = %entry
 ; CHECK: retq                            # CATCHRET
 
 
-attributes #0 = { uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { noreturn }
 
 !llvm.module.flags = !{!0}
diff --git a/llvm/test/CodeGen/X86/catchpad-weight.ll b/llvm/test/CodeGen/X86/catchpad-weight.ll
index 116f2852d..c4968b1ec 100644
--- a/llvm/test/CodeGen/X86/catchpad-weight.ll
+++ b/llvm/test/CodeGen/X86/catchpad-weight.ll
@@ -75,8 +75,8 @@ declare void @"\01??1HasDtor@@QEAA@XZ"(%struct.HasDtor*) #3
 ; Function Attrs: nounwind argmemonly
 declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture) #1
 
-attributes #0 = { uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind argmemonly }
-attributes #2 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #3 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #4 = { nounwind }
diff --git a/llvm/test/CodeGen/X86/catchret-regmask.ll b/llvm/test/CodeGen/X86/catchret-regmask.ll
index 1231172a7..4af57e109 100644
--- a/llvm/test/CodeGen/X86/catchret-regmask.ll
+++ b/llvm/test/CodeGen/X86/catchret-regmask.ll
@@ -6,6 +6,7 @@ target triple = "x86_64-pc-windows-msvc"
 declare i32 @__CxxFrameHandler3(...)
 declare void @throw() noreturn uwtable
 declare i8* @getval()
+declare void @llvm.trap()
 
 define i8* @reload_out_of_pad(i8* %arg) #0 personality i32 (...)* @__CxxFrameHandler3 {
 assertPassed:
@@ -19,6 +20,7 @@ catch:
   ; This block *must* appear after the catchret to test the bug.
   ; FIXME: Make this an MIR test so we can control MBB layout.
 unreachable:
+  call void @llvm.trap()
   unreachable
 
 catch.dispatch:
@@ -35,7 +37,7 @@ return:
 ; CHECK: movq -[[arg_slot]](%rbp), %rax # 8-byte Reload
 ; CHECK: retq
 
-; CHECK: "?catch$3@?0?reload_out_of_pad@4HA":
+; CHECK: "?catch${{[0-9]+}}@?0?reload_out_of_pad@4HA":
 ; CHECK-NOT: Reload
 ; CHECK: retq
 
@@ -50,6 +52,7 @@ catch:
   catchret from %cp to label %return
 
 unreachable:
+  call void @llvm.trap()
   unreachable
 
 catch.dispatch:
@@ -65,7 +68,7 @@ return:
 ; CHECK: movq -[[val_slot:[0-9]+]](%rbp), %rax # 8-byte Reload
 ; CHECK: retq
 
-; CHECK: "?catch$3@?0?spill_in_pad@4HA":
+; CHECK: "?catch${{[0-9]+}}@?0?spill_in_pad@4HA":
 ; CHECK: callq getval
 ; CHECK: movq %rax, -[[val_slot]](%rbp) # 8-byte Spill
 ; CHECK: retq
diff --git a/llvm/test/CodeGen/X86/cfguard-checks.ll b/llvm/test/CodeGen/X86/cfguard-checks.ll
new file mode 100644
index 000000000..8da06680c
--- /dev/null
+++ b/llvm/test/CodeGen/X86/cfguard-checks.ll
@@ -0,0 +1,262 @@
+; RUN: llc < %s -mtriple=i686-pc-windows-msvc | FileCheck %s -check-prefix=X32
+; RUN: llc < %s -mtriple=x86_64-pc-windows-msvc | FileCheck %s -check-prefix=X64
+; Control Flow Guard is currently only available on Windows
+
+; Test that Control Flow Guard checks are correctly added when required.
+
+
+declare i32 @target_func()
+
+
+; Test that Control Flow Guard checks are not added on calls with the "guard_nocf" attribute.
+define i32 @func_guard_nocf() {
+entry:
+  %func_ptr = alloca i32 ()*, align 8
+  store i32 ()* @target_func, i32 ()** %func_ptr, align 8
+  %0 = load i32 ()*, i32 ()** %func_ptr, align 8
+  %1 = call i32 %0() #0
+  ret i32 %1
+
+  ; X32-LABEL: func_guard_nocf
+  ; X32: 	     movl  $_target_func, %eax
+  ; X32-NOT: __guard_check_icall_fptr
+	; X32: 	     calll *%eax
+
+  ; X64-LABEL: func_guard_nocf
+  ; X64:       leaq	target_func(%rip), %rax
+  ; X64-NOT: __guard_dispatch_icall_fptr
+  ; X64:       callq	*%rax
+}
+attributes #0 = { "guard_nocf" }
+
+
+; Test that Control Flow Guard checks are added even at -O0.
+; FIXME Ideally these checks should be added as a single call instruction, as in the optimized case.
+define i32 @func_optnone_cf() #1 {
+entry:
+  %func_ptr = alloca i32 ()*, align 8
+  store i32 ()* @target_func, i32 ()** %func_ptr, align 8
+  %0 = load i32 ()*, i32 ()** %func_ptr, align 8
+  %1 = call i32 %0()
+  ret i32 %1
+
+  ; On i686, the call to __guard_check_icall_fptr should come immediately before the call to the target function.
+  ; X32-LABEL: func_optnone_cf
+	; X32: 	     leal  _target_func, %eax
+	; X32: 	     movl  %eax, (%esp)
+	; X32: 	     movl  (%esp), %ecx
+	; X32: 	     movl ___guard_check_icall_fptr, %eax
+	; X32: 	     calll *%eax
+	; X32-NEXT:  calll *%ecx
+
+  ; On x86_64, __guard_dispatch_icall_fptr tail calls the function, so there should be only one call instruction.
+  ; X64-LABEL: func_optnone_cf
+  ; X64:       leaq	target_func(%rip), %rax
+  ; X64:       movq __guard_dispatch_icall_fptr(%rip), %rcx
+  ; X64:       callq *%rcx
+  ; X64-NOT:   callq
+}
+attributes #1 = { noinline optnone }
+
+
+; Test that Control Flow Guard checks are correctly added in optimized code (common case).
+define i32 @func_cf() {
+entry:
+  %func_ptr = alloca i32 ()*, align 8
+  store i32 ()* @target_func, i32 ()** %func_ptr, align 8
+  %0 = load i32 ()*, i32 ()** %func_ptr, align 8
+  %1 = call i32 %0()
+  ret i32 %1
+
+  ; On i686, the call to __guard_check_icall_fptr should come immediately before the call to the target function.
+  ; X32-LABEL: func_cf
+  ; X32: 	     movl  $_target_func, %esi
+	; X32: 	     movl  $_target_func, %ecx
+	; X32: 	     calll *___guard_check_icall_fptr
+	; X32-NEXT:  calll *%esi
+
+  ; On x86_64, __guard_dispatch_icall_fptr tail calls the function, so there should be only one call instruction.
+  ; X64-LABEL: func_cf
+  ; X64:       leaq	target_func(%rip), %rax
+  ; X64:       callq *__guard_dispatch_icall_fptr(%rip)
+  ; X64-NOT:   callq
+}
+
+
+; Test that Control Flow Guard checks are correctly added on invoke instructions.
+define i32 @func_cf_invoke() personality i8* bitcast (void ()* @h to i8*) {
+entry:
+  %0 = alloca i32, align 4
+  %func_ptr = alloca i32 ()*, align 8
+  store i32 ()* @target_func, i32 ()** %func_ptr, align 8
+  %1 = load i32 ()*, i32 ()** %func_ptr, align 8
+  %2 = invoke i32 %1()
+          to label %invoke.cont unwind label %lpad
+invoke.cont:                                      ; preds = %entry
+  ret i32 %2
+
+lpad:                                             ; preds = %entry
+  %tmp = landingpad { i8*, i32 }
+          catch i8* null
+  ret i32 -1
+
+  ; On i686, the call to __guard_check_icall_fptr should come immediately before the call to the target function.
+  ; X32-LABEL: func_cf_invoke
+  ; X32: 	     movl  $_target_func, %esi
+	; X32: 	     movl  $_target_func, %ecx
+	; X32: 	     calll *___guard_check_icall_fptr
+	; X32-NEXT:  calll *%esi
+  ; X32:       # %invoke.cont
+  ; X32:       # %lpad
+
+  ; On x86_64, __guard_dispatch_icall_fptr tail calls the function, so there should be only one call instruction.
+  ; X64-LABEL: func_cf_invoke
+  ; X64:       leaq	target_func(%rip), %rax
+  ; X64:       callq *__guard_dispatch_icall_fptr(%rip)
+  ; X64-NOT:   callq
+  ; X64:       # %invoke.cont
+  ; X64:       # %lpad
+}
+
+declare void @h()
+
+
+; Test that Control Flow Guard preserves floating point arguments.
+declare double @target_func_doubles(double, double, double, double)
+
+define double @func_cf_doubles() {
+entry:
+  %func_ptr = alloca double (double, double, double, double)*, align 8
+  store double (double, double, double, double)* @target_func_doubles, double (double, double, double, double)** %func_ptr, align 8
+  %0 = load double (double, double, double, double)*, double (double, double, double, double)** %func_ptr, align 8
+  %1 = call double %0(double 1.000000e+00, double 2.000000e+00, double 3.000000e+00, double 4.000000e+00)
+  ret double %1
+
+  ; On i686, the call to __guard_check_icall_fptr should come immediately before the call to the target function.
+  ; X32-LABEL: func_cf_doubles
+  ; X32: 	     movl  $_target_func_doubles, %esi
+	; X32: 	     movl  $_target_func_doubles, %ecx
+	; X32: 	     calll *___guard_check_icall_fptr
+	; X32:       calll *%esi
+
+
+  ; On x86_64, __guard_dispatch_icall_fptr tail calls the function, so there should be only one call instruction.
+  ; X64-LABEL: func_cf_doubles
+  ; X64:       leaq	target_func_doubles(%rip), %rax
+  ; X64:       movsd __real@3ff0000000000000(%rip), %xmm0
+  ; X64:       movsd __real@4000000000000000(%rip), %xmm1
+  ; X64:       movsd __real@4008000000000000(%rip), %xmm2
+  ; X64:       movsd __real@4010000000000000(%rip), %xmm3
+  ; X64:       callq *__guard_dispatch_icall_fptr(%rip)
+  ; X64-NOT:   callq
+}
+
+
+; Test that Control Flow Guard checks are correctly added for tail calls.
+define i32 @func_cf_tail() {
+entry:
+  %func_ptr = alloca i32 ()*, align 8
+  store i32 ()* @target_func, i32 ()** %func_ptr, align 8
+  %0 = load i32 ()*, i32 ()** %func_ptr, align 8
+  %1 = musttail call i32 %0()
+  ret i32 %1
+
+  ; On i686, the call to __guard_check_icall_fptr should come immediately before the call to the target function.
+  ; X32-LABEL: func_cf_tail
+	; X32: 	     movl  $_target_func, %ecx
+	; X32: 	     calll *___guard_check_icall_fptr
+  ; X32:       movl $_target_func, %eax
+	; X32:       jmpl	*%eax                  # TAILCALL
+  ; X32-NOT:   calll
+
+  ; X64-LABEL: func_cf_tail
+  ; X64:       leaq	target_func(%rip), %rax
+  ; X64:       rex64 jmpq *__guard_dispatch_icall_fptr(%rip)         # TAILCALL
+  ; X64-NOT:   callq
+}
+
+%struct.Foo = type { i32 (%struct.Foo*)** }
+
+; Test that Control Flow Guard checks are correctly added for variadic musttail
+; calls. These are used for MS C++ ABI virtual member pointer thunks.
+; PR44049
+define i32 @vmptr_thunk(%struct.Foo* inreg %p) {
+entry:
+  %vptr.addr = getelementptr inbounds %struct.Foo, %struct.Foo* %p, i32 0, i32 0
+  %vptr = load i32 (%struct.Foo*)**, i32 (%struct.Foo*)*** %vptr.addr
+  %slot = getelementptr inbounds i32 (%struct.Foo*)*, i32 (%struct.Foo*)** %vptr, i32 1
+  %vmethod = load i32 (%struct.Foo*)*, i32 (%struct.Foo*)** %slot
+  %rv = musttail call i32 %vmethod(%struct.Foo* inreg %p)
+  ret i32 %rv
+
+  ; On i686, the call to __guard_check_icall_fptr should come immediately before the call to the target function.
+  ; X32-LABEL: _vmptr_thunk:
+  ; X32:       movl %eax, %esi
+  ; X32:       movl (%eax), %eax
+  ; X32:       movl 4(%eax), %ecx
+  ; X32:       calll *___guard_check_icall_fptr
+  ; X32:       movl %esi, %eax
+  ; X32:       jmpl       *%ecx                  # TAILCALL
+  ; X32-NOT:   calll
+
+  ; Use NEXT here because we previously had an extra instruction in this sequence.
+  ; X64-LABEL: vmptr_thunk:
+  ; X64:            movq (%rcx), %rax
+  ; X64-NEXT:       movq 8(%rax), %rax
+  ; X64-NEXT:       movq __guard_dispatch_icall_fptr(%rip), %rdx
+  ; X64-NEXT:       rex64 jmpq *%rdx            # TAILCALL
+  ; X64-NOT:   callq
+}
+
+; Test that longjmp targets have public labels and are included in the .gljmp section.
+%struct._SETJMP_FLOAT128 = type { [2 x i64] }
+@buf1 = internal global [16 x %struct._SETJMP_FLOAT128] zeroinitializer, align 16
+
+define i32 @func_cf_setjmp() {
+  %1 = alloca i32, align 4
+  %2 = alloca i32, align 4
+  store i32 0, i32* %1, align 4
+  store i32 -1, i32* %2, align 4
+  %3 = call i8* @llvm.frameaddress(i32 0)
+  %4 = call i32 @_setjmp(i8* bitcast ([16 x %struct._SETJMP_FLOAT128]* @buf1 to i8*), i8* %3) #2
+
+  ; X32-LABEL: func_cf_setjmp
+  ; X32:       calll __setjmp
+  ; X32-NEXT:  $cfgsj_func_cf_setjmp0:
+
+  ; X64-LABEL: func_cf_setjmp
+  ; X64:       callq _setjmp
+  ; X64-NEXT:  $cfgsj_func_cf_setjmp0:
+
+  %5 = call i8* @llvm.frameaddress(i32 0)
+  %6 = call i32 @_setjmp(i8* bitcast ([16 x %struct._SETJMP_FLOAT128]* @buf1 to i8*), i8* %5) #2
+
+  ; X32:       calll __setjmp
+  ; X32-NEXT:  $cfgsj_func_cf_setjmp1:
+
+  ; X64:       callq _setjmp
+  ; X64-NEXT:  $cfgsj_func_cf_setjmp1:
+
+  store i32 1, i32* %2, align 4
+  %7 = load i32, i32* %2, align 4
+  ret i32 %7
+
+  ; X32:       .section .gljmp$y,"dr"
+  ; X32-NEXT:  .symidx $cfgsj_func_cf_setjmp0
+  ; X32-NEXT:  .symidx $cfgsj_func_cf_setjmp1
+
+  ; X64:       .section .gljmp$y,"dr"
+  ; X64-NEXT:  .symidx $cfgsj_func_cf_setjmp0
+  ; X64-NEXT:  .symidx $cfgsj_func_cf_setjmp1
+}
+
+declare i8* @llvm.frameaddress(i32)
+
+; Function Attrs: returns_twice
+declare dso_local i32 @_setjmp(i8*, i8*) #2
+
+attributes #2 = { returns_twice }
+
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 2, !"cfguard", i32 2}
diff --git a/llvm/test/CodeGen/X86/cfguard-module-flag.ll b/llvm/test/CodeGen/X86/cfguard-module-flag.ll
new file mode 100644
index 000000000..0ed58b392
--- /dev/null
+++ b/llvm/test/CodeGen/X86/cfguard-module-flag.ll
@@ -0,0 +1,26 @@
+
+; RUN: llc < %s -mtriple=i686-pc-windows-msvc | FileCheck %s -check-prefix=X32
+; RUN: llc < %s -mtriple=x86_64-pc-windows-msvc | FileCheck %s -check-prefix=X64
+; Control Flow Guard is currently only available on Windows
+
+; Test that Control Flow Guard checks are not added in modules with the
+; cfguard=1 flag (emit tables but no checks).
+
+
+declare void @target_func()
+
+define void @func_in_module_without_cfguard() #0 {
+entry:
+  %func_ptr = alloca void ()*, align 8
+  store void ()* @target_func, void ()** %func_ptr, align 8
+  %0 = load void ()*, void ()** %func_ptr, align 8
+
+  call void %0()
+  ret void
+
+  ; X32-NOT: __guard_check_icall_fptr
+  ; X64-NOT: __guard_dispatch_icall_fptr
+}
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 2, !"cfguard", i32 1}
diff --git a/llvm/test/CodeGen/X86/cfguard-x86-64-vectorcall.ll b/llvm/test/CodeGen/X86/cfguard-x86-64-vectorcall.ll
new file mode 100644
index 000000000..be7acc531
--- /dev/null
+++ b/llvm/test/CodeGen/X86/cfguard-x86-64-vectorcall.ll
@@ -0,0 +1,38 @@
+; RUN: llc < %s -mtriple=x86_64-pc-windows-msvc | FileCheck %s -check-prefix=X64
+; Control Flow Guard is currently only available on Windows
+
+
+; Test that Control Flow Guard checks are correctly added for x86_64 vector calls.
+define void @func_cf_vector_x64(void (%struct.HVA)* %0, %struct.HVA* %1) #0 {
+entry:
+  %2 = alloca %struct.HVA, align 8
+  %3 = bitcast %struct.HVA* %2 to i8*
+  %4 = bitcast %struct.HVA* %1 to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 8 %3, i8* align 8 %4, i64 32, i1 false)
+  %5 = load %struct.HVA, %struct.HVA* %2, align 8
+  call x86_vectorcallcc void %0(%struct.HVA inreg %5)
+  ret void
+
+  ; X64-LABEL: func_cf_vector_x64
+  ; X64:       movq	%rcx, %rax
+  ; X64:       movups (%rdx), %xmm0
+  ; X64:       movups 16(%rdx), %xmm1
+  ; X64:       movaps %xmm0, 32(%rsp)
+  ; X64:       movaps %xmm1, 48(%rsp)
+  ; X64:       movsd 32(%rsp), %xmm0         # xmm0 = mem[0],zero
+  ; X64:       movsd 40(%rsp), %xmm1         # xmm1 = mem[0],zero
+  ; X64:       movsd 48(%rsp), %xmm2         # xmm2 = mem[0],zero
+  ; X64:       movsd 56(%rsp), %xmm3         # xmm3 = mem[0],zero
+  ; X64:       callq *__guard_dispatch_icall_fptr(%rip)
+  ; X64-NOT:   callq
+}
+attributes #0 = { "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" }
+
+%struct.HVA = type { double, double, double, double }
+
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture writeonly, i8* nocapture readonly, i64, i1 immarg) #1
+attributes #1 = { argmemonly nounwind willreturn }
+
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 2, !"cfguard", i32 2}
diff --git a/llvm/test/CodeGen/X86/cfguard-x86-vectorcall.ll b/llvm/test/CodeGen/X86/cfguard-x86-vectorcall.ll
new file mode 100644
index 000000000..25e369191
--- /dev/null
+++ b/llvm/test/CodeGen/X86/cfguard-x86-vectorcall.ll
@@ -0,0 +1,43 @@
+; RUN: llc < %s -mtriple=i686-pc-windows-msvc | FileCheck %s -check-prefix=X32
+; Control Flow Guard is currently only available on Windows
+
+
+; Test that Control Flow Guard checks are correctly added for x86 vector calls.
+define void @func_cf_vector_x86(void (%struct.HVA)* %0, %struct.HVA* %1) #0 {
+entry:
+  %2 = alloca %struct.HVA, align 8
+  %3 = bitcast %struct.HVA* %2 to i8*
+  %4 = bitcast %struct.HVA* %1 to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 8 %3, i8* align 8 %4, i32 32, i1 false)
+  %5 = load %struct.HVA, %struct.HVA* %2, align 8
+  call x86_vectorcallcc void %0(%struct.HVA inreg %5)
+  ret void
+
+  ; X32-LABEL: func_cf_vector_x86
+  ; X32: 	     movl 12(%ebp), %eax
+  ; X32: 	     movl 8(%ebp), %ecx
+  ; X32: 	     movsd 24(%eax), %xmm4         # xmm4 = mem[0],zero
+  ; X32: 	     movsd %xmm4, 24(%esp)
+  ; X32: 	     movsd 16(%eax), %xmm5         # xmm5 = mem[0],zero
+  ; X32: 	     movsd %xmm5, 16(%esp)
+  ; X32: 	     movsd (%eax), %xmm6           # xmm6 = mem[0],zero
+  ; X32: 	     movsd 8(%eax), %xmm7          # xmm7 = mem[0],zero
+  ; X32: 	     movsd %xmm7, 8(%esp)
+  ; X32: 	     movsd %xmm6, (%esp)
+  ; X32: 	     calll *___guard_check_icall_fptr
+  ; X32: 	     movaps %xmm6, %xmm0
+  ; X32: 	     movaps %xmm7, %xmm1
+  ; X32: 	     movaps %xmm5, %xmm2
+  ; X32: 	     movaps %xmm4, %xmm3
+  ; X32: 	     calll  *%ecx
+}
+attributes #0 = { "target-cpu"="pentium4" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" }
+
+%struct.HVA = type { double, double, double, double }
+
+declare void @llvm.memcpy.p0i8.p0i8.i32(i8* nocapture writeonly, i8* nocapture readonly, i32, i1 immarg) #1
+attributes #1 = { argmemonly nounwind willreturn }
+
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 2, !"cfguard", i32 2}
diff --git a/llvm/test/CodeGen/X86/cleanuppad-inalloca.ll b/llvm/test/CodeGen/X86/cleanuppad-inalloca.ll
index c0660fee2..4e28b5f1a 100644
--- a/llvm/test/CodeGen/X86/cleanuppad-inalloca.ll
+++ b/llvm/test/CodeGen/X86/cleanuppad-inalloca.ll
@@ -65,4 +65,4 @@ declare i32 @__CxxFrameHandler3(...)
 
 declare x86_thiscallcc void @"\01??1A@@QAE@XZ"(%struct.A*) #0
 
-attributes #0 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/X86/cleanuppad-realign.ll b/llvm/test/CodeGen/X86/cleanuppad-realign.ll
index 314d5da07..33afa946c 100644
--- a/llvm/test/CodeGen/X86/cleanuppad-realign.ll
+++ b/llvm/test/CodeGen/X86/cleanuppad-realign.ll
@@ -47,13 +47,13 @@ ehcleanup:                                        ; preds = %entry
 
 ; X64-LABEL: realigned_cleanup: # @realigned_cleanup
 ; X64:         pushq   %rbp
-; X64:         .seh_pushreg 5
+; X64:         .seh_pushreg %rbp
 ; X64:         pushq   %rbx
-; X64:         .seh_pushreg 3
+; X64:         .seh_pushreg %rbx
 ; X64:         subq    $104, %rsp
 ; X64:         .seh_stackalloc 104
 ; X64:         leaq    96(%rsp), %rbp
-; X64:         .seh_setframe 5, 96
+; X64:         .seh_setframe %rbp, 96
 ; X64:         .seh_endprologue
 ; X64:         andq    $-32, %rsp
 ; X64:         movq    %rsp, %rbx
@@ -64,9 +64,9 @@ ehcleanup:                                        ; preds = %entry
 ; X64-LABEL: "?dtor$2@?0?realigned_cleanup@4HA":
 ; X64:         movq    %rdx, 16(%rsp)
 ; X64:         pushq   %rbp
-; X64:         .seh_pushreg 5
+; X64:         .seh_pushreg %rbp
 ; X64:         pushq   %rbx
-; X64:         .seh_pushreg 3
+; X64:         .seh_pushreg %rbx
 ; X64:         subq    $40, %rsp
 ; X64:         .seh_stackalloc 40
 ; X64:         leaq    96(%rdx), %rbp
diff --git a/llvm/test/CodeGen/X86/clear_upper_vector_element_bits.ll b/llvm/test/CodeGen/X86/clear_upper_vector_element_bits.ll
index 983c73426..78487bd16 100644
--- a/llvm/test/CodeGen/X86/clear_upper_vector_element_bits.ll
+++ b/llvm/test/CodeGen/X86/clear_upper_vector_element_bits.ll
@@ -985,99 +985,54 @@ define <32 x i8> @_clearupper32xi8b(<32 x i8>) nounwind {
 ; AVX1-LABEL: _clearupper32xi8b:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
-; AVX1-NEXT:    movq -{{[0-9]+}}(%rsp), %r9
-; AVX1-NEXT:    movq -{{[0-9]+}}(%rsp), %rcx
-; AVX1-NEXT:    movq %r9, %r8
-; AVX1-NEXT:    shrq $56, %r8
-; AVX1-NEXT:    andl $15, %r8d
-; AVX1-NEXT:    movq %rcx, %rsi
-; AVX1-NEXT:    movq %rcx, %rdi
-; AVX1-NEXT:    movq %rcx, %rdx
-; AVX1-NEXT:    movq %rcx, %rax
-; AVX1-NEXT:    shrq $32, %rax
-; AVX1-NEXT:    andl $15, %eax
-; AVX1-NEXT:    shlq $32, %rax
-; AVX1-NEXT:    andl $252645135, %ecx # imm = 0xF0F0F0F
-; AVX1-NEXT:    orq %rax, %rcx
-; AVX1-NEXT:    movq %r9, %rax
-; AVX1-NEXT:    shrq $48, %rax
-; AVX1-NEXT:    andl $15, %eax
-; AVX1-NEXT:    shrq $40, %rdx
-; AVX1-NEXT:    andl $15, %edx
-; AVX1-NEXT:    shlq $40, %rdx
-; AVX1-NEXT:    orq %rcx, %rdx
-; AVX1-NEXT:    movq %r9, %rcx
-; AVX1-NEXT:    shrq $40, %rcx
-; AVX1-NEXT:    andl $15, %ecx
-; AVX1-NEXT:    shrq $48, %rdi
-; AVX1-NEXT:    andl $15, %edi
-; AVX1-NEXT:    shlq $48, %rdi
-; AVX1-NEXT:    orq %rdx, %rdi
-; AVX1-NEXT:    movq %r9, %rdx
-; AVX1-NEXT:    shrq $32, %rdx
-; AVX1-NEXT:    andl $15, %edx
-; AVX1-NEXT:    shrq $56, %rsi
-; AVX1-NEXT:    andl $15, %esi
-; AVX1-NEXT:    shlq $56, %rsi
-; AVX1-NEXT:    orq %rdi, %rsi
-; AVX1-NEXT:    movq %rsi, -{{[0-9]+}}(%rsp)
-; AVX1-NEXT:    shlq $32, %rdx
-; AVX1-NEXT:    andl $252645135, %r9d # imm = 0xF0F0F0F
-; AVX1-NEXT:    orq %rdx, %r9
-; AVX1-NEXT:    shlq $40, %rcx
-; AVX1-NEXT:    orq %r9, %rcx
-; AVX1-NEXT:    shlq $48, %rax
-; AVX1-NEXT:    orq %rcx, %rax
-; AVX1-NEXT:    shlq $56, %r8
-; AVX1-NEXT:    orq %rax, %r8
-; AVX1-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT:    vmovq %xmm0, %rax
-; AVX1-NEXT:    movq %rax, %r8
-; AVX1-NEXT:    movq %rax, %r9
+; AVX1-NEXT:    movq -{{[0-9]+}}(%rsp), %rax
+; AVX1-NEXT:    movq %rax, %rcx
+; AVX1-NEXT:    movq %rax, %rdx
 ; AVX1-NEXT:    movq %rax, %rsi
 ; AVX1-NEXT:    movq %rax, %rdi
-; AVX1-NEXT:    movl %eax, %ecx
-; AVX1-NEXT:    movl %eax, %edx
-; AVX1-NEXT:    vmovd %eax, %xmm1
-; AVX1-NEXT:    shrl $8, %eax
-; AVX1-NEXT:    vpinsrb $1, %eax, %xmm1, %xmm1
-; AVX1-NEXT:    shrl $16, %edx
-; AVX1-NEXT:    vpinsrb $2, %edx, %xmm1, %xmm1
-; AVX1-NEXT:    shrl $24, %ecx
-; AVX1-NEXT:    vpinsrb $3, %ecx, %xmm1, %xmm1
 ; AVX1-NEXT:    shrq $32, %rdi
-; AVX1-NEXT:    vpinsrb $4, %edi, %xmm1, %xmm1
+; AVX1-NEXT:    andl $15, %edi
+; AVX1-NEXT:    shlq $32, %rdi
+; AVX1-NEXT:    andl $252645135, %eax # imm = 0xF0F0F0F
+; AVX1-NEXT:    orq %rdi, %rax
+; AVX1-NEXT:    movq -{{[0-9]+}}(%rsp), %rdi
 ; AVX1-NEXT:    shrq $40, %rsi
-; AVX1-NEXT:    vpinsrb $5, %esi, %xmm1, %xmm1
-; AVX1-NEXT:    vmovdqa -{{[0-9]+}}(%rsp), %xmm2
-; AVX1-NEXT:    shrq $48, %r9
-; AVX1-NEXT:    vpinsrb $6, %r9d, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX1-NEXT:    shrq $56, %r8
-; AVX1-NEXT:    vpinsrb $7, %r8d, %xmm1, %xmm0
-; AVX1-NEXT:    movl %eax, %ecx
-; AVX1-NEXT:    shrl $8, %ecx
-; AVX1-NEXT:    vpinsrb $8, %eax, %xmm0, %xmm0
-; AVX1-NEXT:    vpinsrb $9, %ecx, %xmm0, %xmm0
-; AVX1-NEXT:    movl %eax, %ecx
-; AVX1-NEXT:    shrl $16, %ecx
-; AVX1-NEXT:    vpinsrb $10, %ecx, %xmm0, %xmm0
-; AVX1-NEXT:    movl %eax, %ecx
-; AVX1-NEXT:    shrl $24, %ecx
-; AVX1-NEXT:    vpinsrb $11, %ecx, %xmm0, %xmm0
-; AVX1-NEXT:    movq %rax, %rcx
+; AVX1-NEXT:    andl $15, %esi
+; AVX1-NEXT:    shlq $40, %rsi
+; AVX1-NEXT:    orq %rax, %rsi
+; AVX1-NEXT:    movq %rdi, %rax
+; AVX1-NEXT:    shrq $48, %rdx
+; AVX1-NEXT:    andl $15, %edx
+; AVX1-NEXT:    shlq $48, %rdx
+; AVX1-NEXT:    orq %rsi, %rdx
+; AVX1-NEXT:    movq %rdi, %rsi
+; AVX1-NEXT:    shrq $56, %rcx
+; AVX1-NEXT:    andl $15, %ecx
+; AVX1-NEXT:    shlq $56, %rcx
+; AVX1-NEXT:    orq %rdx, %rcx
+; AVX1-NEXT:    movq %rdi, %rdx
+; AVX1-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
+; AVX1-NEXT:    movq %rdi, %rcx
 ; AVX1-NEXT:    shrq $32, %rcx
-; AVX1-NEXT:    vpinsrb $12, %ecx, %xmm0, %xmm0
-; AVX1-NEXT:    movq %rax, %rcx
-; AVX1-NEXT:    shrq $40, %rcx
-; AVX1-NEXT:    vpinsrb $13, %ecx, %xmm0, %xmm0
-; AVX1-NEXT:    movq %rax, %rcx
-; AVX1-NEXT:    shrq $48, %rcx
-; AVX1-NEXT:    vpinsrb $14, %ecx, %xmm0, %xmm0
-; AVX1-NEXT:    vmovq %xmm2, %rcx
+; AVX1-NEXT:    andl $15, %ecx
+; AVX1-NEXT:    shlq $32, %rcx
+; AVX1-NEXT:    andl $252645135, %edi # imm = 0xF0F0F0F
+; AVX1-NEXT:    orq %rcx, %rdi
+; AVX1-NEXT:    shrq $40, %rdx
+; AVX1-NEXT:    andl $15, %edx
+; AVX1-NEXT:    shlq $40, %rdx
+; AVX1-NEXT:    orq %rdi, %rdx
+; AVX1-NEXT:    shrq $48, %rsi
+; AVX1-NEXT:    andl $15, %esi
+; AVX1-NEXT:    shlq $48, %rsi
+; AVX1-NEXT:    orq %rdx, %rsi
 ; AVX1-NEXT:    shrq $56, %rax
-; AVX1-NEXT:    vpinsrb $15, %eax, %xmm0, %xmm0
+; AVX1-NEXT:    andl $15, %eax
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    shlq $56, %rax
+; AVX1-NEXT:    orq %rsi, %rax
+; AVX1-NEXT:    vmovq %xmm0, %rcx
+; AVX1-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
 ; AVX1-NEXT:    movl %ecx, %eax
 ; AVX1-NEXT:    shrl $8, %eax
 ; AVX1-NEXT:    vmovd %ecx, %xmm1
@@ -1097,129 +1052,85 @@ define <32 x i8> @_clearupper32xi8b(<32 x i8>) nounwind {
 ; AVX1-NEXT:    movq %rcx, %rax
 ; AVX1-NEXT:    shrq $48, %rax
 ; AVX1-NEXT:    vpinsrb $6, %eax, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrq $1, %xmm2, %rax
+; AVX1-NEXT:    vpextrq $1, %xmm0, %rax
 ; AVX1-NEXT:    shrq $56, %rcx
-; AVX1-NEXT:    vpinsrb $7, %ecx, %xmm1, %xmm1
+; AVX1-NEXT:    vpinsrb $7, %ecx, %xmm1, %xmm0
 ; AVX1-NEXT:    movl %eax, %ecx
 ; AVX1-NEXT:    shrl $8, %ecx
-; AVX1-NEXT:    vpinsrb $8, %eax, %xmm1, %xmm1
-; AVX1-NEXT:    vpinsrb $9, %ecx, %xmm1, %xmm1
+; AVX1-NEXT:    vpinsrb $8, %eax, %xmm0, %xmm0
+; AVX1-NEXT:    vpinsrb $9, %ecx, %xmm0, %xmm0
 ; AVX1-NEXT:    movl %eax, %ecx
 ; AVX1-NEXT:    shrl $16, %ecx
-; AVX1-NEXT:    vpinsrb $10, %ecx, %xmm1, %xmm1
+; AVX1-NEXT:    vpinsrb $10, %ecx, %xmm0, %xmm0
 ; AVX1-NEXT:    movl %eax, %ecx
 ; AVX1-NEXT:    shrl $24, %ecx
-; AVX1-NEXT:    vpinsrb $11, %ecx, %xmm1, %xmm1
+; AVX1-NEXT:    vpinsrb $11, %ecx, %xmm0, %xmm0
 ; AVX1-NEXT:    movq %rax, %rcx
 ; AVX1-NEXT:    shrq $32, %rcx
-; AVX1-NEXT:    vpinsrb $12, %ecx, %xmm1, %xmm1
+; AVX1-NEXT:    vpinsrb $12, %ecx, %xmm0, %xmm0
 ; AVX1-NEXT:    movq %rax, %rcx
 ; AVX1-NEXT:    shrq $40, %rcx
-; AVX1-NEXT:    vpinsrb $13, %ecx, %xmm1, %xmm1
+; AVX1-NEXT:    vpinsrb $13, %ecx, %xmm0, %xmm0
 ; AVX1-NEXT:    movq %rax, %rcx
 ; AVX1-NEXT:    shrq $48, %rcx
-; AVX1-NEXT:    vpinsrb $14, %ecx, %xmm1, %xmm1
+; AVX1-NEXT:    vpinsrb $14, %ecx, %xmm0, %xmm0
 ; AVX1-NEXT:    shrq $56, %rax
-; AVX1-NEXT:    vpinsrb $15, %eax, %xmm1, %xmm1
+; AVX1-NEXT:    vpinsrb $15, %eax, %xmm0, %xmm0
+; AVX1-NEXT:    vmovaps -{{[0-9]+}}(%rsp), %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: _clearupper32xi8b:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vmovdqa %xmm0, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT:    movq -{{[0-9]+}}(%rsp), %r9
-; AVX2-NEXT:    movq -{{[0-9]+}}(%rsp), %rcx
-; AVX2-NEXT:    movq %r9, %r8
-; AVX2-NEXT:    shrq $56, %r8
-; AVX2-NEXT:    andl $15, %r8d
-; AVX2-NEXT:    movq %rcx, %rsi
-; AVX2-NEXT:    movq %rcx, %rdi
-; AVX2-NEXT:    movq %rcx, %rdx
-; AVX2-NEXT:    movq %rcx, %rax
-; AVX2-NEXT:    shrq $32, %rax
-; AVX2-NEXT:    andl $15, %eax
-; AVX2-NEXT:    shlq $32, %rax
-; AVX2-NEXT:    andl $252645135, %ecx # imm = 0xF0F0F0F
-; AVX2-NEXT:    orq %rax, %rcx
-; AVX2-NEXT:    movq %r9, %rax
-; AVX2-NEXT:    shrq $48, %rax
-; AVX2-NEXT:    andl $15, %eax
-; AVX2-NEXT:    shrq $40, %rdx
-; AVX2-NEXT:    andl $15, %edx
-; AVX2-NEXT:    shlq $40, %rdx
-; AVX2-NEXT:    orq %rcx, %rdx
-; AVX2-NEXT:    movq %r9, %rcx
-; AVX2-NEXT:    shrq $40, %rcx
-; AVX2-NEXT:    andl $15, %ecx
-; AVX2-NEXT:    shrq $48, %rdi
-; AVX2-NEXT:    andl $15, %edi
-; AVX2-NEXT:    shlq $48, %rdi
-; AVX2-NEXT:    orq %rdx, %rdi
-; AVX2-NEXT:    movq %r9, %rdx
-; AVX2-NEXT:    shrq $32, %rdx
-; AVX2-NEXT:    andl $15, %edx
-; AVX2-NEXT:    shrq $56, %rsi
-; AVX2-NEXT:    andl $15, %esi
-; AVX2-NEXT:    shlq $56, %rsi
-; AVX2-NEXT:    orq %rdi, %rsi
-; AVX2-NEXT:    movq %rsi, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT:    shlq $32, %rdx
-; AVX2-NEXT:    andl $252645135, %r9d # imm = 0xF0F0F0F
-; AVX2-NEXT:    orq %rdx, %r9
-; AVX2-NEXT:    shlq $40, %rcx
-; AVX2-NEXT:    orq %r9, %rcx
-; AVX2-NEXT:    shlq $48, %rax
-; AVX2-NEXT:    orq %rcx, %rax
-; AVX2-NEXT:    shlq $56, %r8
-; AVX2-NEXT:    orq %rax, %r8
-; AVX2-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; AVX2-NEXT:    vmovq %xmm0, %rax
-; AVX2-NEXT:    movq %rax, %r8
-; AVX2-NEXT:    movq %rax, %r9
+; AVX2-NEXT:    movq -{{[0-9]+}}(%rsp), %rax
+; AVX2-NEXT:    movq %rax, %rcx
+; AVX2-NEXT:    movq %rax, %rdx
 ; AVX2-NEXT:    movq %rax, %rsi
 ; AVX2-NEXT:    movq %rax, %rdi
-; AVX2-NEXT:    movl %eax, %ecx
-; AVX2-NEXT:    movl %eax, %edx
-; AVX2-NEXT:    vmovd %eax, %xmm1
-; AVX2-NEXT:    shrl $8, %eax
-; AVX2-NEXT:    vpinsrb $1, %eax, %xmm1, %xmm1
-; AVX2-NEXT:    shrl $16, %edx
-; AVX2-NEXT:    vpinsrb $2, %edx, %xmm1, %xmm1
-; AVX2-NEXT:    shrl $24, %ecx
-; AVX2-NEXT:    vpinsrb $3, %ecx, %xmm1, %xmm1
 ; AVX2-NEXT:    shrq $32, %rdi
-; AVX2-NEXT:    vpinsrb $4, %edi, %xmm1, %xmm1
+; AVX2-NEXT:    andl $15, %edi
+; AVX2-NEXT:    shlq $32, %rdi
+; AVX2-NEXT:    andl $252645135, %eax # imm = 0xF0F0F0F
+; AVX2-NEXT:    orq %rdi, %rax
+; AVX2-NEXT:    movq -{{[0-9]+}}(%rsp), %rdi
 ; AVX2-NEXT:    shrq $40, %rsi
-; AVX2-NEXT:    vpinsrb $5, %esi, %xmm1, %xmm1
-; AVX2-NEXT:    vmovdqa -{{[0-9]+}}(%rsp), %xmm2
-; AVX2-NEXT:    shrq $48, %r9
-; AVX2-NEXT:    vpinsrb $6, %r9d, %xmm1, %xmm1
-; AVX2-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX2-NEXT:    shrq $56, %r8
-; AVX2-NEXT:    vpinsrb $7, %r8d, %xmm1, %xmm0
-; AVX2-NEXT:    movl %eax, %ecx
-; AVX2-NEXT:    shrl $8, %ecx
-; AVX2-NEXT:    vpinsrb $8, %eax, %xmm0, %xmm0
-; AVX2-NEXT:    vpinsrb $9, %ecx, %xmm0, %xmm0
-; AVX2-NEXT:    movl %eax, %ecx
-; AVX2-NEXT:    shrl $16, %ecx
-; AVX2-NEXT:    vpinsrb $10, %ecx, %xmm0, %xmm0
-; AVX2-NEXT:    movl %eax, %ecx
-; AVX2-NEXT:    shrl $24, %ecx
-; AVX2-NEXT:    vpinsrb $11, %ecx, %xmm0, %xmm0
-; AVX2-NEXT:    movq %rax, %rcx
+; AVX2-NEXT:    andl $15, %esi
+; AVX2-NEXT:    shlq $40, %rsi
+; AVX2-NEXT:    orq %rax, %rsi
+; AVX2-NEXT:    movq %rdi, %rax
+; AVX2-NEXT:    shrq $48, %rdx
+; AVX2-NEXT:    andl $15, %edx
+; AVX2-NEXT:    shlq $48, %rdx
+; AVX2-NEXT:    orq %rsi, %rdx
+; AVX2-NEXT:    movq %rdi, %rsi
+; AVX2-NEXT:    shrq $56, %rcx
+; AVX2-NEXT:    andl $15, %ecx
+; AVX2-NEXT:    shlq $56, %rcx
+; AVX2-NEXT:    orq %rdx, %rcx
+; AVX2-NEXT:    movq %rdi, %rdx
+; AVX2-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT:    movq %rdi, %rcx
 ; AVX2-NEXT:    shrq $32, %rcx
-; AVX2-NEXT:    vpinsrb $12, %ecx, %xmm0, %xmm0
-; AVX2-NEXT:    movq %rax, %rcx
-; AVX2-NEXT:    shrq $40, %rcx
-; AVX2-NEXT:    vpinsrb $13, %ecx, %xmm0, %xmm0
-; AVX2-NEXT:    movq %rax, %rcx
-; AVX2-NEXT:    shrq $48, %rcx
-; AVX2-NEXT:    vpinsrb $14, %ecx, %xmm0, %xmm0
-; AVX2-NEXT:    vmovq %xmm2, %rcx
+; AVX2-NEXT:    andl $15, %ecx
+; AVX2-NEXT:    shlq $32, %rcx
+; AVX2-NEXT:    andl $252645135, %edi # imm = 0xF0F0F0F
+; AVX2-NEXT:    orq %rcx, %rdi
+; AVX2-NEXT:    shrq $40, %rdx
+; AVX2-NEXT:    andl $15, %edx
+; AVX2-NEXT:    shlq $40, %rdx
+; AVX2-NEXT:    orq %rdi, %rdx
+; AVX2-NEXT:    shrq $48, %rsi
+; AVX2-NEXT:    andl $15, %esi
+; AVX2-NEXT:    shlq $48, %rsi
+; AVX2-NEXT:    orq %rdx, %rsi
 ; AVX2-NEXT:    shrq $56, %rax
-; AVX2-NEXT:    vpinsrb $15, %eax, %xmm0, %xmm0
+; AVX2-NEXT:    andl $15, %eax
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
+; AVX2-NEXT:    shlq $56, %rax
+; AVX2-NEXT:    orq %rsi, %rax
+; AVX2-NEXT:    vmovq %xmm0, %rcx
+; AVX2-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
 ; AVX2-NEXT:    movl %ecx, %eax
 ; AVX2-NEXT:    shrl $8, %eax
 ; AVX2-NEXT:    vmovd %ecx, %xmm1
@@ -1239,30 +1150,31 @@ define <32 x i8> @_clearupper32xi8b(<32 x i8>) nounwind {
 ; AVX2-NEXT:    movq %rcx, %rax
 ; AVX2-NEXT:    shrq $48, %rax
 ; AVX2-NEXT:    vpinsrb $6, %eax, %xmm1, %xmm1
-; AVX2-NEXT:    vpextrq $1, %xmm2, %rax
+; AVX2-NEXT:    vpextrq $1, %xmm0, %rax
 ; AVX2-NEXT:    shrq $56, %rcx
-; AVX2-NEXT:    vpinsrb $7, %ecx, %xmm1, %xmm1
+; AVX2-NEXT:    vpinsrb $7, %ecx, %xmm1, %xmm0
 ; AVX2-NEXT:    movl %eax, %ecx
 ; AVX2-NEXT:    shrl $8, %ecx
-; AVX2-NEXT:    vpinsrb $8, %eax, %xmm1, %xmm1
-; AVX2-NEXT:    vpinsrb $9, %ecx, %xmm1, %xmm1
+; AVX2-NEXT:    vpinsrb $8, %eax, %xmm0, %xmm0
+; AVX2-NEXT:    vpinsrb $9, %ecx, %xmm0, %xmm0
 ; AVX2-NEXT:    movl %eax, %ecx
 ; AVX2-NEXT:    shrl $16, %ecx
-; AVX2-NEXT:    vpinsrb $10, %ecx, %xmm1, %xmm1
+; AVX2-NEXT:    vpinsrb $10, %ecx, %xmm0, %xmm0
 ; AVX2-NEXT:    movl %eax, %ecx
 ; AVX2-NEXT:    shrl $24, %ecx
-; AVX2-NEXT:    vpinsrb $11, %ecx, %xmm1, %xmm1
+; AVX2-NEXT:    vpinsrb $11, %ecx, %xmm0, %xmm0
 ; AVX2-NEXT:    movq %rax, %rcx
 ; AVX2-NEXT:    shrq $32, %rcx
-; AVX2-NEXT:    vpinsrb $12, %ecx, %xmm1, %xmm1
+; AVX2-NEXT:    vpinsrb $12, %ecx, %xmm0, %xmm0
 ; AVX2-NEXT:    movq %rax, %rcx
 ; AVX2-NEXT:    shrq $40, %rcx
-; AVX2-NEXT:    vpinsrb $13, %ecx, %xmm1, %xmm1
+; AVX2-NEXT:    vpinsrb $13, %ecx, %xmm0, %xmm0
 ; AVX2-NEXT:    movq %rax, %rcx
 ; AVX2-NEXT:    shrq $48, %rcx
-; AVX2-NEXT:    vpinsrb $14, %ecx, %xmm1, %xmm1
+; AVX2-NEXT:    vpinsrb $14, %ecx, %xmm0, %xmm0
 ; AVX2-NEXT:    shrq $56, %rax
-; AVX2-NEXT:    vpinsrb $15, %eax, %xmm1, %xmm1
+; AVX2-NEXT:    vpinsrb $15, %eax, %xmm0, %xmm0
+; AVX2-NEXT:    vmovdqa -{{[0-9]+}}(%rsp), %xmm1
 ; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
 ; AVX2-NEXT:    retq
   %x4  = bitcast <32 x i8> %0 to <64 x i4>
diff --git a/llvm/test/CodeGen/X86/cmov-into-branch.ll b/llvm/test/CodeGen/X86/cmov-into-branch.ll
index 9ce99c789..d9ab2550f 100644
--- a/llvm/test/CodeGen/X86/cmov-into-branch.ll
+++ b/llvm/test/CodeGen/X86/cmov-into-branch.ll
@@ -88,7 +88,7 @@ define i32 @weighted_select1(i32 %a, i32 %b) {
 ; CHECK-NEXT:    cmovnel %edi, %eax
 ; CHECK-NEXT:    retq
   %cmp = icmp ne i32 %a, 0
-  %sel = select i1 %cmp, i32 %a, i32 %b, !prof !0
+  %sel = select i1 %cmp, i32 %a, i32 %b, !prof !15
   ret i32 %sel
 }
 
@@ -104,7 +104,7 @@ define i32 @weighted_select2(i32 %a, i32 %b) {
 ; CHECK-NEXT:  .LBB6_2: # %select.end
 ; CHECK-NEXT:    retq
   %cmp = icmp ne i32 %a, 0
-  %sel = select i1 %cmp, i32 %a, i32 %b, !prof !1
+  %sel = select i1 %cmp, i32 %a, i32 %b, !prof !16
   ret i32 %sel
 }
 
@@ -124,7 +124,7 @@ define i32 @weighted_select3(i32 %a, i32 %b) {
 ; CHECK-NEXT:    movl %esi, %eax
 ; CHECK-NEXT:    retq
   %cmp = icmp ne i32 %a, 0
-  %sel = select i1 %cmp, i32 %a, i32 %b, !prof !2
+  %sel = select i1 %cmp, i32 %a, i32 %b, !prof !17
   ret i32 %sel
 }
 
@@ -137,12 +137,76 @@ define i32 @unweighted_select(i32 %a, i32 %b) {
 ; CHECK-NEXT:    cmovnel %edi, %eax
 ; CHECK-NEXT:    retq
   %cmp = icmp ne i32 %a, 0
-  %sel = select i1 %cmp, i32 %a, i32 %b, !prof !3
+  %sel = select i1 %cmp, i32 %a, i32 %b, !prof !18
   ret i32 %sel
 }
 
-!0 = !{!"branch_weights", i32 1, i32 99}
-!1 = !{!"branch_weights", i32 1, i32 100}
-!2 = !{!"branch_weights", i32 100, i32 1}
-!3 = !{!"branch_weights", i32 0, i32 0}
+define i32 @weighted_select_optsize(i32 %a, i32 %b) optsize {
+; CHECK-LABEL: weighted_select_optsize:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %esi, %eax
+; CHECK-NEXT:    testl %edi, %edi
+; CHECK-NEXT:    cmovnel %edi, %eax
+; CHECK-NEXT:    retq
+  %cmp = icmp ne i32 %a, 0
+  %sel = select i1 %cmp, i32 %a, i32 %b, !prof !16
+  ret i32 %sel
+}
+
+define i32 @weighted_select_pgso(i32 %a, i32 %b) !prof !14 {
+; CHECK-LABEL: weighted_select_pgso:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %esi, %eax
+; CHECK-NEXT:    testl %edi, %edi
+; CHECK-NEXT:    cmovnel %edi, %eax
+; CHECK-NEXT:    retq
+  %cmp = icmp ne i32 %a, 0
+  %sel = select i1 %cmp, i32 %a, i32 %b, !prof !16
+  ret i32 %sel
+}
+
+; If two selects in a row are predictable, turn them into branches.
+define i32 @weighted_selects(i32 %a, i32 %b) !prof !19 {
+; CHECK-LABEL: weighted_selects:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %esi, %eax
+; CHECK-NEXT:    testl %edi, %edi
+; CHECK-NEXT:    movl %edi, %ecx
+; CHECK-NEXT:    jne .LBB11_2
+; CHECK-NEXT:  # %bb.1: # %select.false
+; CHECK-NEXT:    movl %eax, %ecx
+; CHECK-NEXT:  .LBB11_2: # %select.end
+; CHECK-NEXT:    testl %ecx, %ecx
+; CHECK-NEXT:    jne .LBB11_4
+; CHECK-NEXT:  # %bb.3: # %select.false2
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:  .LBB11_4: # %select.end1
+; CHECK-NEXT:    retq
+  %cmp = icmp ne i32 %a, 0
+  %sel = select i1 %cmp, i32 %a, i32 %b, !prof !16
+  %cmp1 = icmp ne i32 %sel, 0
+  %sel1 = select i1 %cmp1, i32 %b, i32 %a, !prof !16
+  ret i32 %sel1
+}
 
+!llvm.module.flags = !{!0}
+!0 = !{i32 1, !"ProfileSummary", !1}
+!1 = !{!2, !3, !4, !5, !6, !7, !8, !9}
+!2 = !{!"ProfileFormat", !"InstrProf"}
+!3 = !{!"TotalCount", i64 10000}
+!4 = !{!"MaxCount", i64 10}
+!5 = !{!"MaxInternalCount", i64 1}
+!6 = !{!"MaxFunctionCount", i64 1000}
+!7 = !{!"NumCounts", i64 3}
+!8 = !{!"NumFunctions", i64 3}
+!9 = !{!"DetailedSummary", !10}
+!10 = !{!11, !12, !13}
+!11 = !{i32 10000, i64 100, i32 1}
+!12 = !{i32 999000, i64 100, i32 1}
+!13 = !{i32 999999, i64 1, i32 2}
+!14 = !{!"function_entry_count", i64 0}
+!15 = !{!"branch_weights", i32 1, i32 99}
+!16 = !{!"branch_weights", i32 1, i32 100}
+!17 = !{!"branch_weights", i32 100, i32 1}
+!18 = !{!"branch_weights", i32 0, i32 0}
+!19 = !{!"function_entry_count", i64 100}
diff --git a/llvm/test/CodeGen/X86/cmov.ll b/llvm/test/CodeGen/X86/cmov.ll
index 4cf31f388..9960c25f7 100644
--- a/llvm/test/CodeGen/X86/cmov.ll
+++ b/llvm/test/CodeGen/X86/cmov.ll
@@ -186,8 +186,6 @@ entry:
 }
 
 
-; Don't try to use a 16-bit conditional move to do an 8-bit select,
-; because it isn't worth it. Just use a branch instead.
 define i8 @test7(i1 inreg %c, i8 inreg %a, i8 inreg %b) nounwind {
 ; CHECK-LABEL: test7:
 ; CHECK:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/cmp.ll b/llvm/test/CodeGen/X86/cmp.ll
index d13fecb8e..5f63223e6 100644
--- a/llvm/test/CodeGen/X86/cmp.ll
+++ b/llvm/test/CodeGen/X86/cmp.ll
@@ -184,7 +184,7 @@ define i32 @test8(i64 %res) nounwind {
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    shrq $32, %rdi # encoding: [0x48,0xc1,0xef,0x20]
 ; CHECK-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
-; CHECK-NEXT:    cmpq $3, %rdi # encoding: [0x48,0x83,0xff,0x03]
+; CHECK-NEXT:    cmpl $3, %edi # encoding: [0x83,0xff,0x03]
 ; CHECK-NEXT:    setb %al # encoding: [0x0f,0x92,0xc0]
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 entry:
@@ -224,7 +224,7 @@ define i32 @test11(i64 %l) nounwind {
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    shrq $47, %rdi # encoding: [0x48,0xc1,0xef,0x2f]
 ; CHECK-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
-; CHECK-NEXT:    cmpq $1, %rdi # encoding: [0x48,0x83,0xff,0x01]
+; CHECK-NEXT:    cmpl $1, %edi # encoding: [0x83,0xff,0x01]
 ; CHECK-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 entry:
diff --git a/llvm/test/CodeGen/X86/cmpxchg8b_alloca_regalloc_handling.ll b/llvm/test/CodeGen/X86/cmpxchg8b_alloca_regalloc_handling.ll
index b500484a4..364d78aa3 100644
--- a/llvm/test/CodeGen/X86/cmpxchg8b_alloca_regalloc_handling.ll
+++ b/llvm/test/CodeGen/X86/cmpxchg8b_alloca_regalloc_handling.ll
@@ -33,3 +33,64 @@ define void @foo_alloca_direct_address(i64* %addr, i32 %n) {
 ; CHECK-LABEL: foo_alloca_direct_address
 ; CHECK-NOT: leal    {{\(%e.*\)}}, [[REGISTER:%e.i]]
 ; CHECK: lock            cmpxchg8b       ([[REGISTER]])
+
+; We used to have a bug when combining:
+; - base pointer for stack frame (VLA + alignment)
+; - cmpxchg8b frameindex + index reg
+
+declare void @escape(i32*)
+
+define void @foo_alloca_index(i32 %i, i64 %val) {
+entry:
+  %Counters = alloca [19 x i64], align 32
+  %vla = alloca i32, i32 %i
+  call void @escape(i32* %vla)
+  br label %body
+
+body:
+  %p = getelementptr inbounds [19 x i64], [19 x i64]* %Counters, i32 0, i32 %i
+  %t2 = cmpxchg volatile i64* %p, i64 %val, i64 %val seq_cst seq_cst
+  %t3 = extractvalue { i64, i1 } %t2, 0
+  %cmp.i = icmp eq i64 %val, %t3
+  br i1 %cmp.i, label %done, label %body
+
+done:
+  ret void
+}
+
+; Check that we add a LEA
+; CHECK-LABEL: foo_alloca_index:
+; CHECK: leal    {{[0-9]*\(%e..,%e..,8\), %e..}}
+; CHECK: lock            cmpxchg8b       ({{%e..}})
+
+
+
+; We used to have a bug when combining:
+; - base pointer for stack frame (VLA + alignment)
+; - cmpxchg8b global + index reg
+
+@Counters = external global [19 x i64]
+
+define void @foo_alloca_index_global(i32 %i, i64 %val) {
+entry:
+  %aligner = alloca i32, align 32
+  call void @escape(i32* %aligner)
+  %vla = alloca i32, i32 %i
+  call void @escape(i32* %vla)
+  br label %body
+
+body:
+  %p = getelementptr inbounds [19 x i64], [19 x i64]* @Counters, i32 0, i32 %i
+  %t2 = cmpxchg volatile i64* %p, i64 %val, i64 %val seq_cst seq_cst
+  %t3 = extractvalue { i64, i1 } %t2, 0
+  %cmp.i = icmp eq i64 %val, %t3
+  br i1 %cmp.i, label %done, label %body
+
+done:
+  ret void
+}
+
+; Check that we add a LEA
+; CHECK-LABEL: foo_alloca_index_global:
+; CHECK: leal    {{Counters\(,%e..,8\), %e..}}
+; CHECK: lock            cmpxchg8b       ({{%e..}})
diff --git a/llvm/test/CodeGen/X86/coalesce-esp.ll b/llvm/test/CodeGen/X86/coalesce-esp.ll
index e0257e68f..6f2ea439b 100644
--- a/llvm/test/CodeGen/X86/coalesce-esp.ll
+++ b/llvm/test/CodeGen/X86/coalesce-esp.ll
@@ -1,4 +1,5 @@
-; RUN: llc < %s | grep "movl	%esp, %ebp"
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s | FileCheck %s
 ; PR4572
 
 ; Don't coalesce with %esp if it would end up putting %esp in
@@ -11,6 +12,33 @@ target triple = "i386-pc-mingw32"
 	%"struct.std::valarray<unsigned int>" = type { i32, i32* }
 
 define void @_ZSt17__gslice_to_indexjRKSt8valarrayIjES2_RS0_(i32 %__o, %"struct.std::valarray<unsigned int>"* nocapture %__l, %"struct.std::valarray<unsigned int>"* nocapture %__s, %"struct.std::valarray<unsigned int>"* nocapture %__i) nounwind {
+; CHECK-LABEL: _ZSt17__gslice_to_indexjRKSt8valarrayIjES2_RS0_:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushl %ebp
+; CHECK-NEXT:    movl %esp, %ebp
+; CHECK-NEXT:    movl %esp, %eax
+; CHECK-NEXT:    xorl %ecx, %ecx
+; CHECK-NEXT:    testb %cl, %cl
+; CHECK-NEXT:    je .LBB0_1
+; CHECK-NEXT:  # %bb.5: # %return
+; CHECK-NEXT:    movl %ebp, %esp
+; CHECK-NEXT:    popl %ebp
+; CHECK-NEXT:    retl
+; CHECK-NEXT:  .LBB0_1: # %bb4.preheader
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    jmp .LBB0_2
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB0_4: # %bb7.backedge
+; CHECK-NEXT:    # in Loop: Header=BB0_2 Depth=1
+; CHECK-NEXT:    addl $-4, %edx
+; CHECK-NEXT:  .LBB0_2: # %bb4
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    testb %cl, %cl
+; CHECK-NEXT:    jne .LBB0_4
+; CHECK-NEXT:  # %bb.3: # %bb5
+; CHECK-NEXT:    # in Loop: Header=BB0_2 Depth=1
+; CHECK-NEXT:    movl $0, (%eax,%edx)
+; CHECK-NEXT:    jmp .LBB0_4
 entry:
 	%0 = alloca i32, i32 undef, align 4		; <i32*> [#uses=1]
 	br i1 undef, label %return, label %bb4
diff --git a/llvm/test/CodeGen/X86/coalescer-commute1.ll b/llvm/test/CodeGen/X86/coalescer-commute1.ll
index dccf3b906..43ba2fc77 100644
--- a/llvm/test/CodeGen/X86/coalescer-commute1.ll
+++ b/llvm/test/CodeGen/X86/coalescer-commute1.ll
@@ -1,10 +1,30 @@
-; RUN: llc < %s -mtriple=i686-apple-darwin -mcpu=corei7-avx -mattr=+sse2 | not grep movaps
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-apple-darwin -mcpu=corei7-avx -mattr=+sse2 | FileCheck %s
 ; PR1877
 
 @NNTOT = weak global i32 0		; <i32*> [#uses=1]
 @G = weak global float 0.000000e+00		; <float*> [#uses=1]
 
 define void @runcont(i32* %source) nounwind  {
+; CHECK-LABEL: runcont:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    movl L_NNTOT$non_lazy_ptr, %ecx
+; CHECK-NEXT:    movl (%ecx), %ecx
+; CHECK-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  LBB0_1: ## %bb
+; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vcvtsi2ssl (%eax,%edx,4), %xmm2, %xmm1
+; CHECK-NEXT:    vaddss %xmm0, %xmm1, %xmm0
+; CHECK-NEXT:    incl %edx
+; CHECK-NEXT:    cmpl %edx, %ecx
+; CHECK-NEXT:    jne LBB0_1
+; CHECK-NEXT:  ## %bb.2: ## %bb13
+; CHECK-NEXT:    movl L_G$non_lazy_ptr, %eax
+; CHECK-NEXT:    vmovss %xmm0, (%eax)
+; CHECK-NEXT:    retl
 entry:
 	%tmp10 = load i32, i32* @NNTOT, align 4		; <i32> [#uses=1]
 	br label %bb
diff --git a/llvm/test/CodeGen/X86/coalescer-commute3.ll b/llvm/test/CodeGen/X86/coalescer-commute3.ll
index 9f22bf0e1..cb67050cf 100644
--- a/llvm/test/CodeGen/X86/coalescer-commute3.ll
+++ b/llvm/test/CodeGen/X86/coalescer-commute3.ll
@@ -1,8 +1,35 @@
-; RUN: llc < %s -mtriple=i686-apple-darwin -mattr=+sse2 -no-x86-call-frame-opt | grep mov | count 6
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-apple-darwin -mattr=+sse2 -no-x86-call-frame-opt | FileCheck %s
 
 	%struct.quad_struct = type { i32, i32, %struct.quad_struct*, %struct.quad_struct*, %struct.quad_struct*, %struct.quad_struct*, %struct.quad_struct* }
 
 define  i32 @perimeter(%struct.quad_struct* %tree, i32 %size) nounwind  {
+; CHECK-LABEL: perimeter:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    pushl %esi
+; CHECK-NEXT:    subl $8, %esp
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    testl %eax, %eax
+; CHECK-NEXT:    je LBB0_3
+; CHECK-NEXT:  ## %bb.1: ## %entry
+; CHECK-NEXT:    cmpl $2, %eax
+; CHECK-NEXT:    jne LBB0_3
+; CHECK-NEXT:  ## %bb.2: ## %bb
+; CHECK-NEXT:    movl $0, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movl $0, (%esp)
+; CHECK-NEXT:    calll _perimeter
+; CHECK-NEXT:    movl %eax, %esi
+; CHECK-NEXT:    movl $0, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movl $0, (%esp)
+; CHECK-NEXT:    calll _perimeter
+; CHECK-NEXT:    addl %esi, %eax
+; CHECK-NEXT:    jmp LBB0_4
+; CHECK-NEXT:  LBB0_3: ## %UnifiedReturnBlock
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:  LBB0_4: ## %UnifiedReturnBlock
+; CHECK-NEXT:    addl $8, %esp
+; CHECK-NEXT:    popl %esi
+; CHECK-NEXT:    retl
 entry:
 	switch i32 %size, label %UnifiedReturnBlock [
 		 i32 2, label %bb
diff --git a/llvm/test/CodeGen/X86/coalescer-commute4.ll b/llvm/test/CodeGen/X86/coalescer-commute4.ll
index d4af1a62d..01008fb40 100644
--- a/llvm/test/CodeGen/X86/coalescer-commute4.ll
+++ b/llvm/test/CodeGen/X86/coalescer-commute4.ll
@@ -1,7 +1,39 @@
-; RUN: llc < %s -mtriple=i686-apple-darwin -mattr=+sse2 | not grep movaps
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-apple-darwin -mattr=+sse2 | FileCheck %s
 ; PR1501
 
 define float @foo(i32* %x, float* %y, i32 %c) nounwind  {
+; CHECK-LABEL: foo:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    pushl %esi
+; CHECK-NEXT:    pushl %eax
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    testl %eax, %eax
+; CHECK-NEXT:    je LBB0_1
+; CHECK-NEXT:  ## %bb.2: ## %bb.preheader
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; CHECK-NEXT:    xorps %xmm0, %xmm0
+; CHECK-NEXT:    xorl %esi, %esi
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  LBB0_3: ## %bb
+; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    xorps %xmm1, %xmm1
+; CHECK-NEXT:    cvtsi2ssl (%edx,%esi,4), %xmm1
+; CHECK-NEXT:    mulss (%ecx,%esi,4), %xmm1
+; CHECK-NEXT:    addss %xmm1, %xmm0
+; CHECK-NEXT:    incl %esi
+; CHECK-NEXT:    cmpl %eax, %esi
+; CHECK-NEXT:    jb LBB0_3
+; CHECK-NEXT:    jmp LBB0_4
+; CHECK-NEXT:  LBB0_1:
+; CHECK-NEXT:    xorps %xmm0, %xmm0
+; CHECK-NEXT:  LBB0_4: ## %bb23
+; CHECK-NEXT:    movss %xmm0, (%esp)
+; CHECK-NEXT:    flds (%esp)
+; CHECK-NEXT:    addl $4, %esp
+; CHECK-NEXT:    popl %esi
+; CHECK-NEXT:    retl
 entry:
 	%tmp2132 = icmp eq i32 %c, 0		; <i1> [#uses=2]
 	br i1 %tmp2132, label %bb23, label %bb.preheader
diff --git a/llvm/test/CodeGen/X86/coalescer-commute5.ll b/llvm/test/CodeGen/X86/coalescer-commute5.ll
index 510d115f4..a571bb42a 100644
--- a/llvm/test/CodeGen/X86/coalescer-commute5.ll
+++ b/llvm/test/CodeGen/X86/coalescer-commute5.ll
@@ -1,6 +1,11 @@
-; RUN: llc < %s -mtriple=i686-apple-darwin -mattr=+sse2 | not grep movaps
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-apple-darwin -mattr=+sse2 | FileCheck %s
 
 define i32 @t() {
+; CHECK-LABEL: t:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    retl
 entry:
 	br i1 true, label %bb1664, label %bb1656
 bb1656:		; preds = %entry
@@ -18,4 +23,4 @@ bb5310:		; preds = %bb4743, %bb1664
 	ret i32 0
 }
 
-declare <16 x i8> @llvm.x86.sse2.pcmpeq.b(<16 x i8>, <16 x i8>) nounwind readnone 
+declare <16 x i8> @llvm.x86.sse2.pcmpeq.b(<16 x i8>, <16 x i8>) nounwind readnone
diff --git a/llvm/test/CodeGen/X86/coalescer-remat.ll b/llvm/test/CodeGen/X86/coalescer-remat.ll
index 62e0562fd..bad5cec33 100644
--- a/llvm/test/CodeGen/X86/coalescer-remat.ll
+++ b/llvm/test/CodeGen/X86/coalescer-remat.ll
@@ -1,9 +1,23 @@
-; RUN: llc < %s -mtriple=x86_64-apple-darwin | grep xor | count 3
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-apple-darwin | FileCheck %s
 
 @val = internal global i64 0
 @"\01LC" = internal constant [7 x i8] c"0x%lx\0A\00"
 
 define i32 @main() nounwind {
+; CHECK-LABEL: main:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    movl $1, %ecx
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    lock cmpxchgq %rcx, {{.*}}(%rip)
+; CHECK-NEXT:    leaq {{.*}}(%rip), %rdi
+; CHECK-NEXT:    movq %rax, %rsi
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    callq _printf
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    popq %rcx
+; CHECK-NEXT:    retq
 entry:
   %t0 = cmpxchg i64* @val, i64 0, i64 1 monotonic monotonic
   %0 = extractvalue { i64, i1 } %t0, 0
diff --git a/llvm/test/CodeGen/X86/codegen-prepare-collapse.ll b/llvm/test/CodeGen/X86/codegen-prepare-collapse.ll
new file mode 100644
index 000000000..18e3ef7af
--- /dev/null
+++ b/llvm/test/CodeGen/X86/codegen-prepare-collapse.ll
@@ -0,0 +1,18 @@
+; RUN: llc -fast-isel=true -O1 -mtriple=x86_64-unkown-linux-gnu -start-before=codegenprepare -stop-after=codegenprepare -o - < %s | FileCheck %s
+
+; CHECK-LABEL: @foo
+define void @foo() {
+top:
+; CHECK: br label %L34
+  br label %L34
+
+L34:                                              ; preds = %L34, %L34, %top
+  %.sroa.075.0 = phi i64 [ undef, %top ], [ undef, %L34 ], [ undef, %L34 ]
+  %0 = icmp sgt i8 undef, -1
+  %cond5896 = icmp eq i8 0, 2
+  %cond58 = and i1 %cond5896, %0
+; During codegenprepare such degenerate branches can occur and should not
+; lead to crashes.
+; CHECK: br label %L34
+  br i1 %cond58, label %L34, label %L34
+}
diff --git a/llvm/test/CodeGen/X86/coff-fp-section-name.ll b/llvm/test/CodeGen/X86/coff-fp-section-name.ll
new file mode 100644
index 000000000..5228596ef
--- /dev/null
+++ b/llvm/test/CodeGen/X86/coff-fp-section-name.ll
@@ -0,0 +1,97 @@
+; RUN: llc -O0 < %s | FileCheck %s
+
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc"
+
+define dso_local i32 @main() #0 {
+entry:
+  %retval = alloca i32, align 4
+  %a = alloca fp128, align 16
+  %b = alloca fp128, align 16
+  %c = alloca fp128, align 16
+  %d = alloca fp128, align 16
+  %e = alloca fp128, align 16
+  %f = alloca fp128, align 16
+  %g = alloca fp128, align 16
+  %h = alloca float, align 4
+  %i = alloca float, align 4
+  %j = alloca float, align 4
+  %k = alloca float, align 4
+  %l = alloca double, align 8
+  %m = alloca double, align 8
+  %n = alloca double, align 8
+  %o = alloca double, align 8
+  store i32 0, i32* %retval, align 4
+
+  store fp128 0xLBB2C11D0AE2E087D73E717A35985531C, fp128* %a, align 16
+  store fp128 0xLBB2C11D0AE2E087D73E717A35985531C, fp128* %b, align 16
+  store fp128 0xL00000000000000004002000000000000, fp128* %c, align 16
+  store fp128 0xL00000000000000007FFF800000000000, fp128* %d, align 16
+  store fp128 0xL00000000000000007FFF000000000000, fp128* %e, align 16
+  store fp128 0xL00000000000000007FFF000000000000, fp128* %f, align 16
+  store fp128 0xL10000000000000003F66244CE242C556, fp128* %g, align 16
+  store float 0x3E212E0BE0000000, float* %h, align 4
+  store float 8.000000e+00, float* %i, align 4
+  store float 0x7FF8000000000000, float* %j, align 4
+  store float 0x7FF0000000000000, float* %k, align 4
+  store double 1.000000e+00, double* %l, align 8
+  store double 8.000000e+00, double* %m, align 8
+  store double 0x7FF8000000000000, double* %n, align 8
+  store double 0x7FF0000000000000, double* %o, align 8
+
+  ret i32 0
+}
+
+attributes #0 = { "target-features"="+mmx" }
+
+; %o
+; CHECK: .globl	__real@7ff0000000000000
+; CHECK: .section	.rdata,"dr",discard,__real@7ff0000000000000
+
+; %n
+; CHECK: .globl	__real@7ff8000000000000
+; CHECK: .section	.rdata,"dr",discard,__real@7ff8000000000000
+
+; %m
+; CHECK: .globl	__real@4020000000000000
+; CHECK: .section	.rdata,"dr",discard,__real@4020000000000000
+
+; %l
+; CHECK: .globl	__real@3ff0000000000000
+; CHECK: .section	.rdata,"dr",discard,__real@3ff0000000000000
+
+; %j
+; CHECK: .globl	__real@7f800000
+; CHECK: .section	.rdata,"dr",discard,__real@7f800000
+
+; %k
+; CHECK: .globl	__real@7fc00000
+; CHECK: .section	.rdata,"dr",discard,__real@7fc00000
+
+; %i
+; CHECK: .globl	__real@41000000
+; CHECK: .section	.rdata,"dr",discard,__real@41000000
+
+; %h
+; CHECK: .globl	__real@3109705f
+; CHECK: .section	.rdata,"dr",discard,__real@3109705f
+
+; %a, %b
+; CHECK: .globl	__xmm@73e717a35985531cbb2c11d0ae2e087d
+; CHECK: .section	.rdata,"dr",discard,__xmm@73e717a35985531cbb2c11d0ae2e087d
+
+; %c
+; CHECK: .globl	__xmm@40020000000000000000000000000000
+; CHECK: .section	.rdata,"dr",discard,__xmm@40020000000000000000000000000000
+
+; %d
+; CHECK: .globl	__xmm@7fff8000000000000000000000000000
+; CHECK: .section	.rdata,"dr",discard,__xmm@7fff8000000000000000000000000000
+
+; %e, %f
+; CHECK: .globl	__xmm@7fff0000000000000000000000000000
+; CHECK: .section	.rdata,"dr",discard,__xmm@7fff0000000000000000000000000000
+
+; %g
+; CHECK: .globl	__xmm@3f66244ce242c5561000000000000000
+; CHECK: .section	.rdata,"dr",discard,__xmm@3f66244ce242c5561000000000000000
diff --git a/llvm/test/CodeGen/X86/combine-64bit-vec-binop.ll b/llvm/test/CodeGen/X86/combine-64bit-vec-binop.ll
index 4bd1ebbc9..387b770c2 100644
--- a/llvm/test/CodeGen/X86/combine-64bit-vec-binop.ll
+++ b/llvm/test/CodeGen/X86/combine-64bit-vec-binop.ll
@@ -101,9 +101,9 @@ define double @test2_mul(double %A, double %B) {
 define double @test3_mul(double %A, double %B) {
 ; SSE41-LABEL: test3_mul:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
-; SSE41-NEXT:    pmullw %xmm2, %xmm0
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; SSE41-NEXT:    pmullw %xmm1, %xmm0
 ; SSE41-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
 ; SSE41-NEXT:    retq
   %1 = bitcast double %A to <8 x i8>
diff --git a/llvm/test/CodeGen/X86/combine-and.ll b/llvm/test/CodeGen/X86/combine-and.ll
index 0be32a5bc..462374c48 100644
--- a/llvm/test/CodeGen/X86/combine-and.ll
+++ b/llvm/test/CodeGen/X86/combine-and.ll
@@ -163,6 +163,18 @@ define <4 x i32> @test14(<4 x i32> %A) {
   ret <4 x i32> %1
 }
 
+; X & undef must fold to 0. So lane 0 must choose from the zero vector.
+
+define <4 x i32> @undef_lane(<4 x i32> %x) {
+; CHECK-LABEL: undef_lane:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xorps %xmm1, %xmm1
+; CHECK-NEXT:    blendps {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2],xmm0[3]
+; CHECK-NEXT:    retq
+  %r = and <4 x i32> %x, <i32 undef, i32 4294967295, i32 0, i32 4294967295>
+  ret <4 x i32> %r
+}
+
 define <4 x i32> @test15(<4 x i32> %A, <4 x i32> %B) {
 ; CHECK-LABEL: test15:
 ; CHECK:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/combine-bitselect.ll b/llvm/test/CodeGen/X86/combine-bitselect.ll
index 8cb6a4dca..b65a7504b 100644
--- a/llvm/test/CodeGen/X86/combine-bitselect.ll
+++ b/llvm/test/CodeGen/X86/combine-bitselect.ll
@@ -329,9 +329,7 @@ define <8 x i64> @bitselect_v8i64_rr(<8 x i64>, <8 x i64>) {
 ;
 ; AVX512F-LABEL: bitselect_v8i64_rr:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
-; AVX512F-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
-; AVX512F-NEXT:    vporq %zmm0, %zmm1, %zmm0
+; AVX512F-NEXT:    vpternlogq $216, {{.*}}(%rip), %zmm1, %zmm0
 ; AVX512F-NEXT:    retq
   %3 = and <8 x i64> %0, <i64 4294967296, i64 12884901890, i64 12884901890, i64 12884901890, i64 4294967296, i64 12884901890, i64 12884901890, i64 12884901890>
   %4 = and <8 x i64> %1, <i64 -4294967297, i64 -12884901891, i64 -12884901891, i64 -12884901891, i64 -4294967297, i64 -12884901891, i64 -12884901891, i64 -12884901891>
@@ -405,9 +403,7 @@ define <8 x i64> @bitselect_v8i64_rm(<8 x i64>, <8 x i64>* nocapture readonly) {
 ; AVX512F-LABEL: bitselect_v8i64_rm:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovdqa64 (%rdi), %zmm1
-; AVX512F-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
-; AVX512F-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
-; AVX512F-NEXT:    vporq %zmm0, %zmm1, %zmm0
+; AVX512F-NEXT:    vpternlogq $216, {{.*}}(%rip), %zmm1, %zmm0
 ; AVX512F-NEXT:    retq
   %3 = load <8 x i64>, <8 x i64>* %1
   %4 = and <8 x i64> %0, <i64 8589934593, i64 3, i64 8589934593, i64 3, i64 8589934593, i64 3, i64 8589934593, i64 3>
@@ -482,9 +478,7 @@ define <8 x i64> @bitselect_v8i64_mr(<8 x i64>* nocapture readonly, <8 x i64>) {
 ; AVX512F-LABEL: bitselect_v8i64_mr:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovdqa64 (%rdi), %zmm1
-; AVX512F-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
-; AVX512F-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
-; AVX512F-NEXT:    vporq %zmm0, %zmm1, %zmm0
+; AVX512F-NEXT:    vpternlogq $216, {{.*}}(%rip), %zmm1, %zmm0
 ; AVX512F-NEXT:    retq
   %3 = load <8 x i64>, <8 x i64>* %0
   %4 = and <8 x i64> %3, <i64 12884901890, i64 4294967296, i64 12884901890, i64 4294967296, i64 12884901890, i64 4294967296, i64 12884901890, i64 4294967296>
@@ -554,11 +548,9 @@ define <8 x i64> @bitselect_v8i64_mm(<8 x i64>* nocapture readonly, <8 x i64>* n
 ;
 ; AVX512F-LABEL: bitselect_v8i64_mm:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa64 (%rdi), %zmm0
 ; AVX512F-NEXT:    vmovdqa64 (%rsi), %zmm1
-; AVX512F-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
-; AVX512F-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
-; AVX512F-NEXT:    vporq %zmm0, %zmm1, %zmm0
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [18446744073709551612,18446744065119617022,18446744073709551612,18446744065119617022,18446744073709551612,18446744065119617022,18446744073709551612,18446744065119617022]
+; AVX512F-NEXT:    vpternlogq $202, (%rdi), %zmm1, %zmm0
 ; AVX512F-NEXT:    retq
   %3 = load <8 x i64>, <8 x i64>* %0
   %4 = load <8 x i64>, <8 x i64>* %1
@@ -616,10 +608,8 @@ define <4 x i1> @bitselect_v4i1_loop(<4 x i32> %a0, <4 x i32> %a1) {
 ; AVX512F:       # %bb.0: # %bb
 ; AVX512F-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
 ; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; AVX512F-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [12,12,12,12]
-; AVX512F-NEXT:    vpcmpeqd %zmm2, %zmm1, %k1
-; AVX512F-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [15,15,15,15]
-; AVX512F-NEXT:    vpcmpeqd %zmm2, %zmm1, %k2
+; AVX512F-NEXT:    vpcmpeqd {{.*}}(%rip){1to16}, %zmm1, %k1
+; AVX512F-NEXT:    vpcmpeqd {{.*}}(%rip){1to16}, %zmm1, %k2
 ; AVX512F-NEXT:    vptestnmd %zmm0, %zmm0, %k0 {%k2}
 ; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k1 {%k1}
 ; AVX512F-NEXT:    korw %k0, %k1, %k1
diff --git a/llvm/test/CodeGen/X86/combine-bswap.ll b/llvm/test/CodeGen/X86/combine-bswap.ll
index 742ea2e02..c30360fd7 100644
--- a/llvm/test/CodeGen/X86/combine-bswap.ll
+++ b/llvm/test/CodeGen/X86/combine-bswap.ll
@@ -40,8 +40,7 @@ define i32 @test_bswap_bswap(i32 %a0) nounwind {
 define i32 @test_demandedbits_bswap(i32 %a0) nounwind {
 ; X86-LABEL: test_demandedbits_bswap:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl $-16777216, %eax # imm = 0xFF000000
-; X86-NEXT:    orl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    bswapl %eax
 ; X86-NEXT:    andl $-65536, %eax # imm = 0xFFFF0000
 ; X86-NEXT:    retl
@@ -49,7 +48,6 @@ define i32 @test_demandedbits_bswap(i32 %a0) nounwind {
 ; X64-LABEL: test_demandedbits_bswap:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl %edi, %eax
-; X64-NEXT:    orl $-16777216, %eax # imm = 0xFF000000
 ; X64-NEXT:    bswapl %eax
 ; X64-NEXT:    andl $-65536, %eax # imm = 0xFFFF0000
 ; X64-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/combine-fabs.ll b/llvm/test/CodeGen/X86/combine-fabs.ll
index b779c589c..163504005 100644
--- a/llvm/test/CodeGen/X86/combine-fabs.ll
+++ b/llvm/test/CodeGen/X86/combine-fabs.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=SSE
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=AVX
 
@@ -44,7 +45,7 @@ define float @combine_fabs_fabs(float %a) {
 ;
 ; AVX-LABEL: combine_fabs_fabs:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vbroadcastss {{.*}}(%rip), %xmm1
+; AVX-NEXT:    vbroadcastss {{.*#+}} xmm1 = [NaN,NaN,NaN,NaN]
 ; AVX-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    retq
   %1 = call float @llvm.fabs.f32(float %a)
@@ -60,7 +61,7 @@ define <4 x float> @combine_vec_fabs_fabs(<4 x float> %a) {
 ;
 ; AVX-LABEL: combine_vec_fabs_fabs:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vbroadcastss {{.*}}(%rip), %xmm1
+; AVX-NEXT:    vbroadcastss {{.*#+}} xmm1 = [NaN,NaN,NaN,NaN]
 ; AVX-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    retq
   %1 = call <4 x float> @llvm.fabs.v4f32(<4 x float> %a)
@@ -77,7 +78,7 @@ define float @combine_fabs_fneg(float %a) {
 ;
 ; AVX-LABEL: combine_fabs_fneg:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vbroadcastss {{.*}}(%rip), %xmm1
+; AVX-NEXT:    vbroadcastss {{.*#+}} xmm1 = [NaN,NaN,NaN,NaN]
 ; AVX-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    retq
   %1 = fsub float -0.0, %a
@@ -93,7 +94,7 @@ define <4 x float> @combine_vec_fabs_fneg(<4 x float> %a) {
 ;
 ; AVX-LABEL: combine_vec_fabs_fneg:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vbroadcastss {{.*}}(%rip), %xmm1
+; AVX-NEXT:    vbroadcastss {{.*#+}} xmm1 = [NaN,NaN,NaN,NaN]
 ; AVX-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    retq
   %1 = fsub <4 x float> <float -0.0, float -0.0, float -0.0, float -0.0>, %a
@@ -110,7 +111,7 @@ define float @combine_fabs_fcopysign(float %a, float %b) {
 ;
 ; AVX-LABEL: combine_fabs_fcopysign:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vbroadcastss {{.*}}(%rip), %xmm1
+; AVX-NEXT:    vbroadcastss {{.*#+}} xmm1 = [NaN,NaN,NaN,NaN]
 ; AVX-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    retq
   %1 = call float @llvm.copysign.f32(float %a, float %b)
@@ -126,7 +127,7 @@ define <4 x float> @combine_vec_fabs_fcopysign(<4 x float> %a, <4 x float> %b) {
 ;
 ; AVX-LABEL: combine_vec_fabs_fcopysign:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vbroadcastss {{.*}}(%rip), %xmm1
+; AVX-NEXT:    vbroadcastss {{.*#+}} xmm1 = [NaN,NaN,NaN,NaN]
 ; AVX-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    retq
   %1 = call <4 x float> @llvm.copysign.v4f32(<4 x float> %a, <4 x float> %b)
diff --git a/llvm/test/CodeGen/X86/combine-lds.ll b/llvm/test/CodeGen/X86/combine-lds.ll
index f2c81a495..62f3a1b1e 100644
--- a/llvm/test/CodeGen/X86/combine-lds.ll
+++ b/llvm/test/CodeGen/X86/combine-lds.ll
@@ -1,6 +1,11 @@
-; RUN: llc < %s -mtriple=i686-- -mattr=+sse2 | grep fldl | count 1
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- -mattr=+sse2 | FileCheck %s
 
 define double @doload64(i64 %x) nounwind  {
+; CHECK-LABEL: doload64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fldl {{[0-9]+}}(%esp)
+; CHECK-NEXT:    retl
 	%tmp717 = bitcast i64 %x to double
 	ret double %tmp717
 }
diff --git a/llvm/test/CodeGen/X86/combine-mulo.ll b/llvm/test/CodeGen/X86/combine-mulo.ll
index 99f69a9a1..0eddf2f81 100644
--- a/llvm/test/CodeGen/X86/combine-mulo.ll
+++ b/llvm/test/CodeGen/X86/combine-mulo.ll
@@ -34,30 +34,21 @@ define <4 x i32> @combine_vec_smul_two(<4 x i32> %a0, <4 x i32> %a1) {
 ; SSE-LABEL: combine_vec_smul_two:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    movdqa %xmm0, %xmm2
-; SSE-NEXT:    pxor %xmm0, %xmm0
-; SSE-NEXT:    pxor %xmm3, %xmm3
+; SSE-NEXT:    paddd %xmm0, %xmm2
+; SSE-NEXT:    movdqa %xmm0, %xmm3
 ; SSE-NEXT:    pcmpgtd %xmm2, %xmm3
-; SSE-NEXT:    pcmpeqd %xmm4, %xmm4
-; SSE-NEXT:    pxor %xmm4, %xmm3
-; SSE-NEXT:    paddd %xmm2, %xmm2
-; SSE-NEXT:    pcmpgtd %xmm2, %xmm0
-; SSE-NEXT:    pxor %xmm4, %xmm0
-; SSE-NEXT:    pcmpeqd %xmm3, %xmm0
-; SSE-NEXT:    blendvps %xmm0, %xmm2, %xmm1
-; SSE-NEXT:    movaps %xmm1, %xmm0
+; SSE-NEXT:    pxor %xmm0, %xmm3
+; SSE-NEXT:    movdqa %xmm3, %xmm0
+; SSE-NEXT:    blendvps %xmm0, %xmm1, %xmm2
+; SSE-NEXT:    movaps %xmm2, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: combine_vec_smul_two:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX-NEXT:    vpcmpgtd %xmm0, %xmm2, %xmm3
-; AVX-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX-NEXT:    vpxor %xmm4, %xmm3, %xmm3
-; AVX-NEXT:    vpaddd %xmm0, %xmm0, %xmm0
-; AVX-NEXT:    vpcmpgtd %xmm0, %xmm2, %xmm2
-; AVX-NEXT:    vpxor %xmm4, %xmm2, %xmm2
-; AVX-NEXT:    vpcmpeqd %xmm2, %xmm3, %xmm2
-; AVX-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    vpaddd %xmm0, %xmm0, %xmm2
+; AVX-NEXT:    vpcmpgtd %xmm2, %xmm0, %xmm3
+; AVX-NEXT:    vpxor %xmm3, %xmm0, %xmm0
+; AVX-NEXT:    vblendvps %xmm0, %xmm1, %xmm2, %xmm0
 ; AVX-NEXT:    retq
   %1 = call {<4 x i32>, <4 x i1>} @llvm.smul.with.overflow.v4i32(<4 x i32> %a0, <4 x i32> <i32 2, i32 2, i32 2, i32 2>)
   %2 = extractvalue {<4 x i32>, <4 x i1>} %1, 0
diff --git a/llvm/test/CodeGen/X86/combine-or.ll b/llvm/test/CodeGen/X86/combine-or.ll
index ab13a34bb..2d7ab3db8 100644
--- a/llvm/test/CodeGen/X86/combine-or.ll
+++ b/llvm/test/CodeGen/X86/combine-or.ll
@@ -362,7 +362,7 @@ define <4 x float> @test25(<4 x float> %a0) {
 define <4 x i8> @test_crash(<4 x i8> %a, <4 x i8> %b) {
 ; CHECK-LABEL: test_crash:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    blendps {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
+; CHECK-NEXT:    pblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2,3,4,5,6,7]
 ; CHECK-NEXT:    retq
   %shuf1 = shufflevector <4 x i8> %a, <4 x i8> zeroinitializer, <4 x i32><i32 4, i32 4, i32 2, i32 3>
   %shuf2 = shufflevector <4 x i8> %b, <4 x i8> zeroinitializer, <4 x i32><i32 0, i32 1, i32 4, i32 4>
diff --git a/llvm/test/CodeGen/X86/combine-sdiv.ll b/llvm/test/CodeGen/X86/combine-sdiv.ll
index 156cf3753..235a6f0f3 100644
--- a/llvm/test/CodeGen/X86/combine-sdiv.ll
+++ b/llvm/test/CodeGen/X86/combine-sdiv.ll
@@ -933,24 +933,26 @@ define <32 x i16> @combine_vec_sdiv_by_pow2b_v32i16(<32 x i16> %x) {
 ;
 ; AVX512F-LABEL: combine_vec_sdiv_by_pow2b_v32i16:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpsraw $15, %ymm0, %ymm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
+; AVX512F-NEXT:    vpsraw $15, %ymm1, %ymm2
 ; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [0,4,2,16,8,32,64,2,0,4,2,16,8,32,64,2]
 ; AVX512F-NEXT:    # ymm3 = mem[0,1,0,1]
 ; AVX512F-NEXT:    vpmulhuw %ymm3, %ymm2, %ymm2
-; AVX512F-NEXT:    vpaddw %ymm2, %ymm0, %ymm2
+; AVX512F-NEXT:    vpaddw %ymm2, %ymm1, %ymm2
 ; AVX512F-NEXT:    vpmovsxwd %ymm2, %zmm2
 ; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [0,2,1,4,3,5,6,1,0,2,1,4,3,5,6,1]
 ; AVX512F-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
 ; AVX512F-NEXT:    vpsravd %zmm4, %zmm2, %zmm2
 ; AVX512F-NEXT:    vpmovdw %zmm2, %ymm2
-; AVX512F-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm2[1,2,3,4,5,6,7],ymm0[8],ymm2[9,10,11,12,13,14,15]
-; AVX512F-NEXT:    vpsraw $15, %ymm1, %ymm2
+; AVX512F-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm2[1,2,3,4,5,6,7],ymm1[8],ymm2[9,10,11,12,13,14,15]
+; AVX512F-NEXT:    vpsraw $15, %ymm0, %ymm2
 ; AVX512F-NEXT:    vpmulhuw %ymm3, %ymm2, %ymm2
-; AVX512F-NEXT:    vpaddw %ymm2, %ymm1, %ymm2
+; AVX512F-NEXT:    vpaddw %ymm2, %ymm0, %ymm2
 ; AVX512F-NEXT:    vpmovsxwd %ymm2, %zmm2
 ; AVX512F-NEXT:    vpsravd %zmm4, %zmm2, %zmm2
 ; AVX512F-NEXT:    vpmovdw %zmm2, %ymm2
-; AVX512F-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm2[1,2,3,4,5,6,7],ymm1[8],ymm2[9,10,11,12,13,14,15]
+; AVX512F-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm2[1,2,3,4,5,6,7],ymm0[8],ymm2[9,10,11,12,13,14,15]
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: combine_vec_sdiv_by_pow2b_v32i16:
@@ -2102,46 +2104,32 @@ define <4 x i32> @combine_vec_sdiv_by_pow2b_undef3(<4 x i32> %x) {
 
 ; PR37119
 define <16 x i8> @non_splat_minus_one_divisor_0(<16 x i8> %A) {
-; SSE2-LABEL: non_splat_minus_one_divisor_0:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [0,0,255,0,0,0,255,0,0,255,255,255,255,255,255,255]
-; SSE2-NEXT:    pxor %xmm1, %xmm0
-; SSE2-NEXT:    psubb %xmm0, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm0
-; SSE2-NEXT:    retq
-;
-; SSE41-LABEL: non_splat_minus_one_divisor_0:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    pxor %xmm2, %xmm2
-; SSE41-NEXT:    psubb %xmm0, %xmm2
-; SSE41-NEXT:    movaps {{.*#+}} xmm0 = [0,0,255,0,0,0,255,0,0,255,255,255,255,255,255,255]
-; SSE41-NEXT:    pblendvb %xmm0, %xmm1, %xmm2
-; SSE41-NEXT:    movdqa %xmm2, %xmm0
-; SSE41-NEXT:    retq
+; SSE-LABEL: non_splat_minus_one_divisor_0:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,0,255,255,255,0,255,255,0,0,0,0,0,0,0]
+; SSE-NEXT:    pxor %xmm1, %xmm0
+; SSE-NEXT:    psubb %xmm1, %xmm0
+; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: non_splat_minus_one_divisor_0:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpsubb %xmm0, %xmm1, %xmm1
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,0,255,0,0,0,255,0,0,255,255,255,255,255,255,255]
-; AVX1-NEXT:    vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [255,255,0,255,255,255,0,255,255,0,0,0,0,0,0,0]
+; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: non_splat_minus_one_divisor_0:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpsubb %xmm0, %xmm1, %xmm1
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,0,255,0,0,0,255,0,0,255,255,255,255,255,255,255]
-; AVX2-NEXT:    vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [255,255,0,255,255,255,0,255,255,0,0,0,0,0,0,0]
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: non_splat_minus_one_divisor_0:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT:    vpsubb %xmm0, %xmm1, %xmm1
-; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,0,255,0,0,0,255,0,0,255,255,255,255,255,255,255]
-; AVX512F-NEXT:    vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
+; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm1 = [255,255,0,255,255,255,0,255,255,0,0,0,0,0,0,0]
+; AVX512F-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: non_splat_minus_one_divisor_0:
@@ -2154,10 +2142,9 @@ define <16 x i8> @non_splat_minus_one_divisor_0(<16 x i8> %A) {
 ;
 ; XOP-LABEL: non_splat_minus_one_divisor_0:
 ; XOP:       # %bb.0:
-; XOP-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; XOP-NEXT:    vpsubb %xmm0, %xmm1, %xmm1
-; XOP-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,0,255,0,0,0,255,0,0,255,255,255,255,255,255,255]
-; XOP-NEXT:    vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
+; XOP-NEXT:    vmovdqa {{.*#+}} xmm1 = [255,255,0,255,255,255,0,255,255,0,0,0,0,0,0,0]
+; XOP-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; XOP-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
 ; XOP-NEXT:    retq
   %div = sdiv <16 x i8> %A, <i8 -1, i8 -1, i8 1, i8 -1, i8 -1, i8 -1, i8 1, i8 -1, i8 -1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
   ret <16 x i8> %div
@@ -2201,38 +2188,38 @@ define <16 x i8> @non_splat_minus_one_divisor_1(<16 x i8> %A) {
 ; SSE41-LABEL: non_splat_minus_one_divisor_1:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    pxor %xmm2, %xmm2
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    pcmpgtb %xmm1, %xmm0
+; SSE41-NEXT:    pxor %xmm3, %xmm3
+; SSE41-NEXT:    pcmpgtb %xmm1, %xmm3
 ; SSE41-NEXT:    pxor %xmm4, %xmm4
-; SSE41-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3],xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; SSE41-NEXT:    psllw $1, %xmm3
-; SSE41-NEXT:    pblendw {{.*#+}} xmm3 = xmm4[0,1],xmm3[2],xmm4[3,4,5],xmm3[6],xmm4[7]
+; SSE41-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3],xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm2 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
+; SSE41-NEXT:    psllw $1, %xmm2
+; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm4[0,1],xmm2[2],xmm4[3,4,5],xmm2[6],xmm4[7]
+; SSE41-NEXT:    psrlw $8, %xmm2
+; SSE41-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm0[8],xmm3[9],xmm0[9],xmm3[10],xmm0[10],xmm3[11],xmm0[11],xmm3[12],xmm0[12],xmm3[13],xmm0[13],xmm3[14],xmm0[14],xmm3[15],xmm0[15]
+; SSE41-NEXT:    pmullw {{.*}}(%rip), %xmm3
 ; SSE41-NEXT:    psrlw $8, %xmm3
+; SSE41-NEXT:    packuswb %xmm3, %xmm2
+; SSE41-NEXT:    paddb %xmm1, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
 ; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15]
-; SSE41-NEXT:    pmullw {{.*}}(%rip), %xmm0
-; SSE41-NEXT:    psrlw $8, %xmm0
-; SSE41-NEXT:    packuswb %xmm0, %xmm3
-; SSE41-NEXT:    paddb %xmm1, %xmm3
-; SSE41-NEXT:    movdqa %xmm3, %xmm0
-; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm3[8],xmm0[9],xmm3[9],xmm0[10],xmm3[10],xmm0[11],xmm3[11],xmm0[12],xmm3[12],xmm0[13],xmm3[13],xmm0[14],xmm3[14],xmm0[15],xmm3[15]
 ; SSE41-NEXT:    psraw $8, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm4
-; SSE41-NEXT:    psllw $1, %xmm4
+; SSE41-NEXT:    movdqa %xmm0, %xmm3
+; SSE41-NEXT:    psllw $1, %xmm3
 ; SSE41-NEXT:    psllw $7, %xmm0
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4],xmm4[5],xmm0[6],xmm4[7]
+; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4],xmm3[5],xmm0[6],xmm3[7]
 ; SSE41-NEXT:    psrlw $8, %xmm0
-; SSE41-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE41-NEXT:    psraw $8, %xmm3
-; SSE41-NEXT:    psllw $7, %xmm3
-; SSE41-NEXT:    psrlw $8, %xmm3
-; SSE41-NEXT:    packuswb %xmm0, %xmm3
+; SSE41-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE41-NEXT:    psraw $8, %xmm2
+; SSE41-NEXT:    psllw $7, %xmm2
+; SSE41-NEXT:    psrlw $8, %xmm2
+; SSE41-NEXT:    packuswb %xmm0, %xmm2
 ; SSE41-NEXT:    movaps {{.*#+}} xmm0 = [0,0,255,0,0,0,255,0,0,255,255,255,255,255,255,255]
-; SSE41-NEXT:    pblendvb %xmm0, %xmm3, %xmm1
-; SSE41-NEXT:    psubb %xmm1, %xmm2
-; SSE41-NEXT:    movaps {{.*#+}} xmm0 = [255,255,0,255,255,255,0,255,255,0,0,0,0,255,0,255]
 ; SSE41-NEXT:    pblendvb %xmm0, %xmm2, %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,0,255,255,255,0,255,255,0,0,0,0,255,0,255]
+; SSE41-NEXT:    pxor %xmm0, %xmm1
+; SSE41-NEXT:    psubb %xmm0, %xmm1
 ; SSE41-NEXT:    movdqa %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
@@ -2245,68 +2232,68 @@ define <16 x i8> @non_splat_minus_one_divisor_1(<16 x i8> %A) {
 ; AVX1-NEXT:    vpsllw $1, %xmm4, %xmm4
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1],xmm4[2],xmm3[3,4,5],xmm4[6],xmm3[7]
 ; AVX1-NEXT:    vpsrlw $8, %xmm3, %xmm3
-; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
-; AVX1-NEXT:    vpmullw {{.*}}(%rip), %xmm2, %xmm2
-; AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
-; AVX1-NEXT:    vpackuswb %xmm2, %xmm3, %xmm2
-; AVX1-NEXT:    vpaddb %xmm2, %xmm0, %xmm2
-; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX1-NEXT:    vpsraw $8, %xmm3, %xmm3
-; AVX1-NEXT:    vpsllw $1, %xmm3, %xmm4
-; AVX1-NEXT:    vpsllw $7, %xmm3, %xmm3
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3,4],xmm4[5],xmm3[6],xmm4[7]
-; AVX1-NEXT:    vpsrlw $8, %xmm3, %xmm3
-; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
+; AVX1-NEXT:    vpmullw {{.*}}(%rip), %xmm1, %xmm1
+; AVX1-NEXT:    vpsrlw $8, %xmm1, %xmm1
+; AVX1-NEXT:    vpackuswb %xmm1, %xmm3, %xmm1
+; AVX1-NEXT:    vpaddb %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; AVX1-NEXT:    vpsraw $8, %xmm2, %xmm2
+; AVX1-NEXT:    vpsllw $1, %xmm2, %xmm3
 ; AVX1-NEXT:    vpsllw $7, %xmm2, %xmm2
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3,4],xmm3[5],xmm2[6],xmm3[7]
 ; AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
-; AVX1-NEXT:    vpackuswb %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,0,255,0,0,0,255,0,0,255,255,255,255,255,255,255]
-; AVX1-NEXT:    vpblendvb %xmm3, %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpsubb %xmm0, %xmm1, %xmm1
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [255,255,0,255,255,255,0,255,255,0,0,0,0,255,0,255]
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; AVX1-NEXT:    vpsraw $8, %xmm1, %xmm1
+; AVX1-NEXT:    vpsllw $7, %xmm1, %xmm1
+; AVX1-NEXT:    vpsrlw $8, %xmm1, %xmm1
+; AVX1-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,0,255,0,0,0,255,0,0,255,255,255,255,255,255,255]
 ; AVX1-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [255,255,0,255,255,255,0,255,255,0,0,0,0,255,0,255]
+; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: non_splat_minus_one_divisor_1:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpgtb %xmm0, %xmm1, %xmm2
-; AVX2-NEXT:    vpmovzxbw {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero,xmm2[8],zero,xmm2[9],zero,xmm2[10],zero,xmm2[11],zero,xmm2[12],zero,xmm2[13],zero,xmm2[14],zero,xmm2[15],zero
-; AVX2-NEXT:    vpmullw {{.*}}(%rip), %ymm2, %ymm2
-; AVX2-NEXT:    vpsrlw $8, %ymm2, %ymm2
-; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm3
-; AVX2-NEXT:    vpackuswb %xmm3, %xmm2, %xmm2
-; AVX2-NEXT:    vpaddb %xmm2, %xmm0, %xmm2
-; AVX2-NEXT:    vpmovsxbw %xmm2, %ymm2
-; AVX2-NEXT:    vpmullw {{.*}}(%rip), %ymm2, %ymm2
-; AVX2-NEXT:    vpsrlw $8, %ymm2, %ymm2
-; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm3
-; AVX2-NEXT:    vpackuswb %xmm3, %xmm2, %xmm2
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,0,255,0,0,0,255,0,0,255,255,255,255,255,255,255]
-; AVX2-NEXT:    vpblendvb %xmm3, %xmm2, %xmm0, %xmm0
-; AVX2-NEXT:    vpsubb %xmm0, %xmm1, %xmm1
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [255,255,0,255,255,255,0,255,255,0,0,0,0,255,0,255]
+; AVX2-NEXT:    vpcmpgtb %xmm0, %xmm1, %xmm1
+; AVX2-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX2-NEXT:    vpmullw {{.*}}(%rip), %ymm1, %ymm1
+; AVX2-NEXT:    vpsrlw $8, %ymm1, %ymm1
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
+; AVX2-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpaddb %xmm1, %xmm0, %xmm1
+; AVX2-NEXT:    vpmovsxbw %xmm1, %ymm1
+; AVX2-NEXT:    vpmullw {{.*}}(%rip), %ymm1, %ymm1
+; AVX2-NEXT:    vpsrlw $8, %ymm1, %ymm1
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
+; AVX2-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,0,255,0,0,0,255,0,0,255,255,255,255,255,255,255]
 ; AVX2-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [255,255,0,255,255,255,0,255,255,0,0,0,0,255,0,255]
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: non_splat_minus_one_divisor_1:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT:    vpcmpgtb %xmm0, %xmm1, %xmm2
-; AVX512F-NEXT:    vpmovzxbd {{.*#+}} zmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero
-; AVX512F-NEXT:    vpsrlvd {{.*}}(%rip), %zmm2, %zmm2
-; AVX512F-NEXT:    vpmovdb %zmm2, %xmm2
-; AVX512F-NEXT:    vpaddb %xmm2, %xmm0, %xmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vpsravd {{.*}}(%rip), %zmm2, %zmm2
-; AVX512F-NEXT:    vpmovdb %zmm2, %xmm2
-; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,0,255,0,0,0,255,0,0,255,255,255,255,255,255,255]
-; AVX512F-NEXT:    vpblendvb %xmm3, %xmm2, %xmm0, %xmm0
-; AVX512F-NEXT:    vpsubb %xmm0, %xmm1, %xmm1
-; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm2 = [255,255,0,255,255,255,0,255,255,0,0,0,0,255,0,255]
+; AVX512F-NEXT:    vpcmpgtb %xmm0, %xmm1, %xmm1
+; AVX512F-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; AVX512F-NEXT:    vpsrlvd {{.*}}(%rip), %zmm1, %zmm1
+; AVX512F-NEXT:    vpmovdb %zmm1, %xmm1
+; AVX512F-NEXT:    vpaddb %xmm1, %xmm0, %xmm1
+; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
+; AVX512F-NEXT:    vpsravd {{.*}}(%rip), %zmm1, %zmm1
+; AVX512F-NEXT:    vpmovdb %zmm1, %xmm1
+; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,0,255,0,0,0,255,0,0,255,255,255,255,255,255,255]
 ; AVX512F-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm1 = [255,255,0,255,255,255,0,255,255,0,0,0,0,255,0,255]
+; AVX512F-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
@@ -2334,15 +2321,15 @@ define <16 x i8> @non_splat_minus_one_divisor_1(<16 x i8> %A) {
 ; XOP-LABEL: non_splat_minus_one_divisor_1:
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; XOP-NEXT:    vpcmpgtb %xmm0, %xmm1, %xmm2
-; XOP-NEXT:    vpshlb {{.*}}(%rip), %xmm2, %xmm2
-; XOP-NEXT:    vpaddb %xmm2, %xmm0, %xmm2
-; XOP-NEXT:    vpshab {{.*}}(%rip), %xmm2, %xmm2
-; XOP-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,0,255,0,0,0,255,0,0,255,255,255,255,255,255,255]
-; XOP-NEXT:    vpblendvb %xmm3, %xmm2, %xmm0, %xmm0
-; XOP-NEXT:    vpsubb %xmm0, %xmm1, %xmm1
-; XOP-NEXT:    vmovdqa {{.*#+}} xmm2 = [255,255,0,255,255,255,0,255,255,0,0,0,0,255,0,255]
+; XOP-NEXT:    vpcmpgtb %xmm0, %xmm1, %xmm1
+; XOP-NEXT:    vpshlb {{.*}}(%rip), %xmm1, %xmm1
+; XOP-NEXT:    vpaddb %xmm1, %xmm0, %xmm1
+; XOP-NEXT:    vpshab {{.*}}(%rip), %xmm1, %xmm1
+; XOP-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,0,255,0,0,0,255,0,0,255,255,255,255,255,255,255]
 ; XOP-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
+; XOP-NEXT:    vmovdqa {{.*#+}} xmm1 = [255,255,0,255,255,255,0,255,255,0,0,0,0,255,0,255]
+; XOP-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; XOP-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
 ; XOP-NEXT:    retq
   %div = sdiv <16 x i8> %A, <i8 -1, i8 -1, i8 2, i8 -1, i8 -1, i8 -1, i8 2, i8 -1, i8 -1, i8 2, i8 2, i8 2, i8 2, i8 -128, i8 2, i8 -128>
   ret <16 x i8> %div
@@ -3104,3 +3091,144 @@ define <4 x i1> @boolvec_sdiv(<4 x i1> %x, <4 x i1> %y) {
   %r = sdiv <4 x i1> %x, %y
   ret <4 x i1> %r
 }
+
+define i32 @combine_sdiv_two(i32 %x) {
+; CHECK-LABEL: combine_sdiv_two:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    # kill: def $edi killed $edi def $rdi
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    shrl $31, %eax
+; CHECK-NEXT:    addl %edi, %eax
+; CHECK-NEXT:    sarl %eax
+; CHECK-NEXT:    retq
+  %1 = sdiv i32 %x, 2
+  ret i32 %1
+}
+
+define i32 @combine_sdiv_negtwo(i32 %x) {
+; CHECK-LABEL: combine_sdiv_negtwo:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    # kill: def $edi killed $edi def $rdi
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    shrl $31, %eax
+; CHECK-NEXT:    addl %edi, %eax
+; CHECK-NEXT:    sarl %eax
+; CHECK-NEXT:    negl %eax
+; CHECK-NEXT:    retq
+  %1 = sdiv i32 %x, -2
+  ret i32 %1
+}
+
+define i8 @combine_i8_sdiv_pow2(i8 %x) {
+; CHECK-LABEL: combine_i8_sdiv_pow2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    # kill: def $edi killed $edi def $rdi
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    sarb $7, %al
+; CHECK-NEXT:    shrb $4, %al
+; CHECK-NEXT:    addl %edi, %eax
+; CHECK-NEXT:    sarb $4, %al
+; CHECK-NEXT:    # kill: def $al killed $al killed $eax
+; CHECK-NEXT:    retq
+  %1 = sdiv i8 %x, 16
+  ret i8 %1
+}
+
+define i8 @combine_i8_sdiv_negpow2(i8 %x) {
+; CHECK-LABEL: combine_i8_sdiv_negpow2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    # kill: def $edi killed $edi def $rdi
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    sarb $7, %al
+; CHECK-NEXT:    shrb $2, %al
+; CHECK-NEXT:    addl %edi, %eax
+; CHECK-NEXT:    sarb $6, %al
+; CHECK-NEXT:    negb %al
+; CHECK-NEXT:    # kill: def $al killed $al killed $eax
+; CHECK-NEXT:    retq
+  %1 = sdiv i8 %x, -64
+  ret i8 %1
+}
+
+define i16 @combine_i16_sdiv_pow2(i16 %x) {
+; CHECK-LABEL: combine_i16_sdiv_pow2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    # kill: def $edi killed $edi def $rdi
+; CHECK-NEXT:    leal 15(%rdi), %eax
+; CHECK-NEXT:    testw %di, %di
+; CHECK-NEXT:    cmovnsl %edi, %eax
+; CHECK-NEXT:    cwtl
+; CHECK-NEXT:    shrl $4, %eax
+; CHECK-NEXT:    # kill: def $ax killed $ax killed $eax
+; CHECK-NEXT:    retq
+  %1 = sdiv i16 %x, 16
+  ret i16 %1
+}
+
+define i16 @combine_i16_sdiv_negpow2(i16 %x) {
+; CHECK-LABEL: combine_i16_sdiv_negpow2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    # kill: def $edi killed $edi def $rdi
+; CHECK-NEXT:    leal 255(%rdi), %eax
+; CHECK-NEXT:    testw %di, %di
+; CHECK-NEXT:    cmovnsl %edi, %eax
+; CHECK-NEXT:    cwtl
+; CHECK-NEXT:    sarl $8, %eax
+; CHECK-NEXT:    negl %eax
+; CHECK-NEXT:    # kill: def $ax killed $ax killed $eax
+; CHECK-NEXT:    retq
+  %1 = sdiv i16 %x, -256
+  ret i16 %1
+}
+
+define i32 @combine_i32_sdiv_pow2(i32 %x) {
+; CHECK-LABEL: combine_i32_sdiv_pow2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    # kill: def $edi killed $edi def $rdi
+; CHECK-NEXT:    leal 15(%rdi), %eax
+; CHECK-NEXT:    testl %edi, %edi
+; CHECK-NEXT:    cmovnsl %edi, %eax
+; CHECK-NEXT:    sarl $4, %eax
+; CHECK-NEXT:    retq
+  %1 = sdiv i32 %x, 16
+  ret i32 %1
+}
+
+define i32 @combine_i32_sdiv_negpow2(i32 %x) {
+; CHECK-LABEL: combine_i32_sdiv_negpow2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    # kill: def $edi killed $edi def $rdi
+; CHECK-NEXT:    leal 255(%rdi), %eax
+; CHECK-NEXT:    testl %edi, %edi
+; CHECK-NEXT:    cmovnsl %edi, %eax
+; CHECK-NEXT:    sarl $8, %eax
+; CHECK-NEXT:    negl %eax
+; CHECK-NEXT:    retq
+  %1 = sdiv i32 %x, -256
+  ret i32 %1
+}
+
+define i64 @combine_i64_sdiv_pow2(i64 %x) {
+; CHECK-LABEL: combine_i64_sdiv_pow2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    leaq 15(%rdi), %rax
+; CHECK-NEXT:    testq %rdi, %rdi
+; CHECK-NEXT:    cmovnsq %rdi, %rax
+; CHECK-NEXT:    sarq $4, %rax
+; CHECK-NEXT:    retq
+  %1 = sdiv i64 %x, 16
+  ret i64 %1
+}
+
+define i64 @combine_i64_sdiv_negpow2(i64 %x) {
+; CHECK-LABEL: combine_i64_sdiv_negpow2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    leaq 255(%rdi), %rax
+; CHECK-NEXT:    testq %rdi, %rdi
+; CHECK-NEXT:    cmovnsq %rdi, %rax
+; CHECK-NEXT:    sarq $8, %rax
+; CHECK-NEXT:    negq %rax
+; CHECK-NEXT:    retq
+  %1 = sdiv i64 %x, -256
+  ret i64 %1
+}
diff --git a/llvm/test/CodeGen/X86/combine-shl.ll b/llvm/test/CodeGen/X86/combine-shl.ll
index 55ad952c3..4d37f324c 100644
--- a/llvm/test/CodeGen/X86/combine-shl.ll
+++ b/llvm/test/CodeGen/X86/combine-shl.ll
@@ -832,3 +832,106 @@ define <4 x i32> @combine_vec_shl_mul1(<4 x i32> %x) {
   %2 = shl <4 x i32> %1, <i32 1, i32 2, i32 3, i32 4>
   ret <4 x i32> %2
 }
+
+; fold (add (shl x, c1), c2) -> (or (shl x, c1), c2)
+define <4 x i32> @combine_vec_add_shl_nonsplat(<4 x i32> %a0)  {
+; SSE2-LABEL: combine_vec_add_shl_nonsplat:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [4,8,16,32]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pmuludq %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pmuludq %xmm2, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT:    por {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: combine_vec_add_shl_nonsplat:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm0
+; SSE41-NEXT:    por {{.*}}(%rip), %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: combine_vec_add_shl_nonsplat:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpsllvd {{.*}}(%rip), %xmm0, %xmm0
+; AVX-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [3,3,3,3]
+; AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %1 = shl <4 x i32> %a0, <i32 2, i32 3, i32 4, i32 5>
+  %2 = add <4 x i32> %1, <i32 3, i32 3, i32 3, i32 3>
+  ret <4 x i32> %2
+}
+
+define <4 x i32> @combine_vec_add_shl_and_nonsplat(<4 x i32> %a0)  {
+; SSE2-LABEL: combine_vec_add_shl_and_nonsplat:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [4,8,16,32]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pmuludq %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pmuludq %xmm2, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT:    por {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: combine_vec_add_shl_and_nonsplat:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pxor %xmm1, %xmm1
+; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
+; SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm0
+; SSE41-NEXT:    por {{.*}}(%rip), %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: combine_vec_add_shl_and_nonsplat:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
+; AVX-NEXT:    vpsllvd {{.*}}(%rip), %xmm0, %xmm0
+; AVX-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [15,15,15,15]
+; AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %1 = and <4 x i32> %a0, <i32 4294901760, i32 4294901760, i32 4294901760, i32 4294901760>
+  %2 = shl <4 x i32> %1, <i32 2, i32 3, i32 4, i32 5>
+  %3 = add <4 x i32> %2, <i32 15, i32 15, i32 15, i32 15>
+  ret <4 x i32> %3
+}
+
+define <4 x i32> @combine_vec_add_shuffle_shl(<4 x i32> %a0)  {
+; SSE2-LABEL: combine_vec_add_shuffle_shl:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    pslld $3, %xmm1
+; SSE2-NEXT:    pslld $2, %xmm0
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,3,3,0]
+; SSE2-NEXT:    por {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: combine_vec_add_shuffle_shl:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    pslld $3, %xmm1
+; SSE41-NEXT:    pslld $2, %xmm0
+; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5,6,7]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,1,0]
+; SSE41-NEXT:    por {{.*}}(%rip), %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: combine_vec_add_shuffle_shl:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpsllvd {{.*}}(%rip), %xmm0, %xmm0
+; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,1,0]
+; AVX-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [3,3,3,3]
+; AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %1 = shl <4 x i32> %a0, <i32 2, i32 3, i32 0, i32 1>
+  %2 = shufflevector <4 x i32> %1, <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 1, i32 0>
+  %3 = add <4 x i32> %2, <i32 3, i32 3, i32 3, i32 3>
+  ret <4 x i32> %3
+}
diff --git a/llvm/test/CodeGen/X86/combine-srem.ll b/llvm/test/CodeGen/X86/combine-srem.ll
index ef338b371..b08bb5b4d 100644
--- a/llvm/test/CodeGen/X86/combine-srem.ll
+++ b/llvm/test/CodeGen/X86/combine-srem.ll
@@ -56,10 +56,9 @@ define i32 @combine_srem_by_minsigned(i32 %x) {
 ; CHECK-LABEL: combine_srem_by_minsigned:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    # kill: def $edi killed $edi def $rdi
-; CHECK-NEXT:    movl %edi, %eax
-; CHECK-NEXT:    sarl $31, %eax
-; CHECK-NEXT:    shrl %eax
-; CHECK-NEXT:    addl %edi, %eax
+; CHECK-NEXT:    leal 2147483647(%rdi), %eax
+; CHECK-NEXT:    testl %edi, %edi
+; CHECK-NEXT:    cmovnsl %edi, %eax
 ; CHECK-NEXT:    andl $-2147483648, %eax # imm = 0x80000000
 ; CHECK-NEXT:    addl %edi, %eax
 ; CHECK-NEXT:    retq
@@ -432,3 +431,135 @@ define <4 x i1> @boolvec_srem(<4 x i1> %x, <4 x i1> %y) {
   %r = srem <4 x i1> %x, %y
   ret <4 x i1> %r
 }
+
+define i32 @combine_srem_two(i32 %x) {
+; CHECK-LABEL: combine_srem_two:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    movl %edi, %ecx
+; CHECK-NEXT:    shrl $31, %ecx
+; CHECK-NEXT:    addl %edi, %ecx
+; CHECK-NEXT:    andl $-2, %ecx
+; CHECK-NEXT:    subl %ecx, %eax
+; CHECK-NEXT:    retq
+  %1 = srem i32 %x, 2
+  ret i32 %1
+}
+
+define i32 @combine_srem_negtwo(i32 %x) {
+; CHECK-LABEL: combine_srem_negtwo:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    movl %edi, %ecx
+; CHECK-NEXT:    shrl $31, %ecx
+; CHECK-NEXT:    addl %edi, %ecx
+; CHECK-NEXT:    andl $-2, %ecx
+; CHECK-NEXT:    subl %ecx, %eax
+; CHECK-NEXT:    retq
+  %1 = srem i32 %x, -2
+  ret i32 %1
+}
+
+define i8 @combine_i8_srem_negpow2(i8 %x) {
+; CHECK-LABEL: combine_i8_srem_negpow2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    movl %eax, %ecx
+; CHECK-NEXT:    sarb $7, %cl
+; CHECK-NEXT:    shrb $2, %cl
+; CHECK-NEXT:    addb %al, %cl
+; CHECK-NEXT:    andb $-64, %cl
+; CHECK-NEXT:    subb %cl, %al
+; CHECK-NEXT:    # kill: def $al killed $al killed $eax
+; CHECK-NEXT:    retq
+  %1 = srem i8 %x, -64
+  ret i8 %1
+}
+
+define i16 @combine_i16_srem_pow2(i16 %x) {
+; CHECK-LABEL: combine_i16_srem_pow2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    leal 15(%rax), %ecx
+; CHECK-NEXT:    testw %ax, %ax
+; CHECK-NEXT:    cmovnsl %edi, %ecx
+; CHECK-NEXT:    andl $-16, %ecx
+; CHECK-NEXT:    subl %ecx, %eax
+; CHECK-NEXT:    # kill: def $ax killed $ax killed $rax
+; CHECK-NEXT:    retq
+  %1 = srem i16 %x, 16
+  ret i16 %1
+}
+
+define i16 @combine_i16_srem_negpow2(i16 %x) {
+; CHECK-LABEL: combine_i16_srem_negpow2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    leal 255(%rax), %ecx
+; CHECK-NEXT:    testw %ax, %ax
+; CHECK-NEXT:    cmovnsl %edi, %ecx
+; CHECK-NEXT:    andl $-256, %ecx
+; CHECK-NEXT:    subl %ecx, %eax
+; CHECK-NEXT:    # kill: def $ax killed $ax killed $rax
+; CHECK-NEXT:    retq
+  %1 = srem i16 %x, -256
+  ret i16 %1
+}
+
+define i32 @combine_srem_pow2(i32 %x) {
+; CHECK-LABEL: combine_srem_pow2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    leal 15(%rax), %ecx
+; CHECK-NEXT:    testl %edi, %edi
+; CHECK-NEXT:    cmovnsl %edi, %ecx
+; CHECK-NEXT:    andl $-16, %ecx
+; CHECK-NEXT:    subl %ecx, %eax
+; CHECK-NEXT:    # kill: def $eax killed $eax killed $rax
+; CHECK-NEXT:    retq
+  %1 = srem i32 %x, 16
+  ret i32 %1
+}
+
+define i32 @combine_srem_negpow2(i32 %x) {
+; CHECK-LABEL: combine_srem_negpow2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    leal 255(%rax), %ecx
+; CHECK-NEXT:    testl %edi, %edi
+; CHECK-NEXT:    cmovnsl %edi, %ecx
+; CHECK-NEXT:    andl $-256, %ecx
+; CHECK-NEXT:    subl %ecx, %eax
+; CHECK-NEXT:    # kill: def $eax killed $eax killed $rax
+; CHECK-NEXT:    retq
+  %1 = srem i32 %x, -256
+  ret i32 %1
+}
+
+define i64 @combine_i64_srem_pow2(i64 %x) {
+; CHECK-LABEL: combine_i64_srem_pow2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    leaq 15(%rdi), %rcx
+; CHECK-NEXT:    testq %rdi, %rdi
+; CHECK-NEXT:    cmovnsq %rdi, %rcx
+; CHECK-NEXT:    andq $-16, %rcx
+; CHECK-NEXT:    subq %rcx, %rax
+; CHECK-NEXT:    retq
+  %1 = srem i64 %x, 16
+  ret i64 %1
+}
+
+define i64 @combine_i64_srem_negpow2(i64 %x) {
+; CHECK-LABEL: combine_i64_srem_negpow2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    leaq 255(%rdi), %rcx
+; CHECK-NEXT:    testq %rdi, %rdi
+; CHECK-NEXT:    cmovnsq %rdi, %rcx
+; CHECK-NEXT:    andq $-256, %rcx
+; CHECK-NEXT:    subq %rcx, %rax
+; CHECK-NEXT:    retq
+  %1 = srem i64 %x, -256
+  ret i64 %1
+}
diff --git a/llvm/test/CodeGen/X86/combine-sse41-intrinsics.ll b/llvm/test/CodeGen/X86/combine-sse41-intrinsics.ll
index 7944aec30..cc4dee33c 100644
--- a/llvm/test/CodeGen/X86/combine-sse41-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/combine-sse41-intrinsics.ll
@@ -127,8 +127,8 @@ define <16 x i8> @demandedelts_pblendvb(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>
   ret <16 x i8> %5
 }
 
-define <2 x i64> @demandedbits_blendpd(i64 %a0, i64 %a2, <2 x double> %a3) {
-; CHECK-LABEL: demandedbits_blendpd:
+define <2 x i64> @demandedbits_blendvpd(i64 %a0, i64 %a2, <2 x double> %a3) {
+; CHECK-LABEL: demandedbits_blendvpd:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    movq %rdi, %rax
 ; CHECK-NEXT:    orq $1, %rax
@@ -153,6 +153,49 @@ define <2 x i64> @demandedbits_blendpd(i64 %a0, i64 %a2, <2 x double> %a3) {
   ret <2 x i64> %9
 }
 
+define <16 x i8> @xor_pblendvb(<16 x i8> %a0, <16 x i8> %a1, <16 x i8> %a2) {
+; CHECK-LABEL: xor_pblendvb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movdqa %xmm0, %xmm3
+; CHECK-NEXT:    movaps %xmm2, %xmm0
+; CHECK-NEXT:    pblendvb %xmm0, %xmm3, %xmm1
+; CHECK-NEXT:    movdqa %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %1 = xor <16 x i8> %a2, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
+  %2 = tail call <16 x i8> @llvm.x86.sse41.pblendvb(<16 x i8> %a0, <16 x i8> %a1, <16 x i8> %1)
+  ret <16 x i8> %2
+}
+
+define <4 x float> @xor_blendvps(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) {
+; CHECK-LABEL: xor_blendvps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm0, %xmm3
+; CHECK-NEXT:    movaps %xmm2, %xmm0
+; CHECK-NEXT:    blendvps %xmm0, %xmm3, %xmm1
+; CHECK-NEXT:    movaps %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %1 = bitcast <4 x float> %a2 to <4 x i32>
+  %2 = xor <4 x i32> %1, <i32 -1, i32 -1, i32 -1, i32 -1>
+  %3 = bitcast <4 x i32> %2 to <4 x float>
+  %4 = tail call <4 x float> @llvm.x86.sse41.blendvps(<4 x float> %a0, <4 x float> %a1, <4 x float> %3)
+  ret <4 x float> %4
+}
+
+define <2 x double> @xor_blendvpd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) {
+; CHECK-LABEL: xor_blendvpd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movapd %xmm0, %xmm3
+; CHECK-NEXT:    movaps %xmm2, %xmm0
+; CHECK-NEXT:    blendvpd %xmm0, %xmm3, %xmm1
+; CHECK-NEXT:    movapd %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %1 = bitcast <2 x double> %a2 to <4 x i32>
+  %2 = xor <4 x i32> %1, <i32 -1, i32 -1, i32 -1, i32 -1>
+  %3 = bitcast <4 x i32> %2 to <2 x double>
+  %4 = tail call <2 x double> @llvm.x86.sse41.blendvpd(<2 x double> %a0, <2 x double> %a1, <2 x double> %3)
+  ret <2 x double> %4
+}
+
 declare <2 x double> @llvm.x86.sse41.blendpd(<2 x double>, <2 x double>, i32)
 declare <4 x float> @llvm.x86.sse41.blendps(<4 x float>, <4 x float>, i32)
 declare <8 x i16> @llvm.x86.sse41.pblendw(<8 x i16>, <8 x i16>, i32)
diff --git a/llvm/test/CodeGen/X86/combineIncDecVector-crash.ll b/llvm/test/CodeGen/X86/combineIncDecVector-crash.ll
new file mode 100644
index 000000000..8dea7a5fd
--- /dev/null
+++ b/llvm/test/CodeGen/X86/combineIncDecVector-crash.ll
@@ -0,0 +1,51 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s | FileCheck %s
+
+; This used to crash, just ensure that it doesn't.
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128-ni:1"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @TestvMeth(i32 %0, i64 %1) gc "statepoint-example" !prof !1 {
+; CHECK-LABEL: TestvMeth:
+; CHECK:       # %bb.0: # %bci_0
+; CHECK-NEXT:    subq $24, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    movq %rsi, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movl %edi, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movl $10, %esi
+; CHECK-NEXT:    movl $10, %edx
+; CHECK-NEXT:    movl $400, %ecx # imm = 0x190
+; CHECK-NEXT:    callq newarray
+; CHECK-NEXT:  .Ltmp0:
+; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; CHECK-NEXT:    addss (%rax), %xmm0
+; CHECK-NEXT:    movdqu (%rax), %xmm1
+; CHECK-NEXT:    pcmpeqd %xmm2, %xmm2
+; CHECK-NEXT:    psubd %xmm2, %xmm1
+; CHECK-NEXT:    movdqu %xmm1, (%rax)
+; CHECK-NEXT:    movss %xmm0, (%rax)
+bci_0:
+   %token418 = call token (i64, i32, i8 * (i64, i32, i32, i32)*, i32,
+i32, ...) @llvm.experimental.gc.statepoint.p0f_p1i8i64i32i32i32f(i64
+2882400000, i32 0, i8 * (i64, i32, i32, i32)* nonnull @newarray, i32 4,
+i32 0, i64 undef, i32 10, i32 10, i32 400, i32 0, i32 35, i32 0, i32 1,
+i32 0, i32 43, i32 1, i32 13, i32 0, i32 3, i32 400, i32 3, i32 %0, i32
+4, i64 %1, i32 7, i8* null, i32 3, i32 -11464, i32 7, i8* null, i32 3,
+i32 -243, i32 3, i32 14, i32 3, i32 117, i32 3, i32 -13, i32 3, i32 -15,
+i32 3, i32 -210, i32 3, i32 541, i32 7, i8* null)
+   %v2 = load atomic float, float * undef unordered, align 4
+   %v3 = load <4 x i32>, <4 x i32> * undef, align 4
+   %v4 = add <4 x i32> %v3, <i32 1, i32 1, i32 1, i32 1>
+   store <4 x i32> %v4, <4 x i32> * undef, align 4
+   %v5 = fadd float %v2, 1.500000e+01
+   store atomic float %v5, float * undef unordered, align 4
+   unreachable
+}
+
+declare i32* @personality_function()
+declare i8 * @newarray(i64, i32, i32, i32)
+declare token @llvm.experimental.gc.statepoint.p0f_p1i8i64i32i32i32f(i64
+immarg, i32 immarg, i8 * (i64, i32, i32, i32)*, i32 immarg, i32 immarg, ...)
+
+!1 = !{!"function_entry_count", i64 32768}
diff --git a/llvm/test/CodeGen/X86/commute-fcmp.ll b/llvm/test/CodeGen/X86/commute-fcmp.ll
index 4da8a4556..390cec3a5 100644
--- a/llvm/test/CodeGen/X86/commute-fcmp.ll
+++ b/llvm/test/CodeGen/X86/commute-fcmp.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+sse2 -disable-peephole | FileCheck %s --check-prefix=SSE
 ; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx2 -disable-peephole | FileCheck %s --check-prefix=AVX
-; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx512vl -disable-peephole | FileCheck %s --check-prefix=AVX512
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx512vl,+avx512dq -disable-peephole | FileCheck %s --check-prefix=AVX512
 
 ;
 ; Float Comparisons
@@ -788,3 +788,523 @@ define <4 x i64> @commute_cmppd_le_ymmm(<4 x double>* %a0, <4 x double> %a1) {
   %3 = sext <4 x i1> %2 to <4 x i64>
   ret <4 x i64> %3
 }
+
+define <16 x i32> @commute_cmpps_eq_zmm(<16 x float>* %a0, <16 x float> %a1) {
+; SSE-LABEL: commute_cmpps_eq_zmm:
+; SSE:       # %bb.0:
+; SSE-NEXT:    cmpeqps (%rdi), %xmm0
+; SSE-NEXT:    cmpeqps 16(%rdi), %xmm1
+; SSE-NEXT:    cmpeqps 32(%rdi), %xmm2
+; SSE-NEXT:    cmpeqps 48(%rdi), %xmm3
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: commute_cmpps_eq_zmm:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcmpeqps (%rdi), %ymm0, %ymm0
+; AVX-NEXT:    vcmpeqps 32(%rdi), %ymm1, %ymm1
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: commute_cmpps_eq_zmm:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vcmpeqps (%rdi), %zmm0, %k0
+; AVX512-NEXT:    vpmovm2d %k0, %zmm0
+; AVX512-NEXT:    retq
+  %1 = load <16 x float>, <16 x float>* %a0
+  %2 = fcmp oeq <16 x float> %1, %a1
+  %3 = sext <16 x i1> %2 to <16 x i32>
+  ret <16 x i32> %3
+}
+
+define <16 x i32> @commute_cmpps_ne_zmm(<16 x float>* %a0, <16 x float> %a1) {
+; SSE-LABEL: commute_cmpps_ne_zmm:
+; SSE:       # %bb.0:
+; SSE-NEXT:    cmpneqps (%rdi), %xmm0
+; SSE-NEXT:    cmpneqps 16(%rdi), %xmm1
+; SSE-NEXT:    cmpneqps 32(%rdi), %xmm2
+; SSE-NEXT:    cmpneqps 48(%rdi), %xmm3
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: commute_cmpps_ne_zmm:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcmpneqps (%rdi), %ymm0, %ymm0
+; AVX-NEXT:    vcmpneqps 32(%rdi), %ymm1, %ymm1
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: commute_cmpps_ne_zmm:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vcmpneqps (%rdi), %zmm0, %k0
+; AVX512-NEXT:    vpmovm2d %k0, %zmm0
+; AVX512-NEXT:    retq
+  %1 = load <16 x float>, <16 x float>* %a0
+  %2 = fcmp une <16 x float> %1, %a1
+  %3 = sext <16 x i1> %2 to <16 x i32>
+  ret <16 x i32> %3
+}
+
+define <16 x i32> @commute_cmpps_ord_zmm(<16 x float>* %a0, <16 x float> %a1) {
+; SSE-LABEL: commute_cmpps_ord_zmm:
+; SSE:       # %bb.0:
+; SSE-NEXT:    cmpordps (%rdi), %xmm0
+; SSE-NEXT:    cmpordps 16(%rdi), %xmm1
+; SSE-NEXT:    cmpordps 32(%rdi), %xmm2
+; SSE-NEXT:    cmpordps 48(%rdi), %xmm3
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: commute_cmpps_ord_zmm:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcmpordps (%rdi), %ymm0, %ymm0
+; AVX-NEXT:    vcmpordps 32(%rdi), %ymm1, %ymm1
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: commute_cmpps_ord_zmm:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vcmpordps (%rdi), %zmm0, %k0
+; AVX512-NEXT:    vpmovm2d %k0, %zmm0
+; AVX512-NEXT:    retq
+  %1 = load <16 x float>, <16 x float>* %a0
+  %2 = fcmp ord <16 x float> %1, %a1
+  %3 = sext <16 x i1> %2 to <16 x i32>
+  ret <16 x i32> %3
+}
+
+define <16 x i32> @commute_cmpps_uno_zmm(<16 x float>* %a0, <16 x float> %a1) {
+; SSE-LABEL: commute_cmpps_uno_zmm:
+; SSE:       # %bb.0:
+; SSE-NEXT:    cmpunordps (%rdi), %xmm0
+; SSE-NEXT:    cmpunordps 16(%rdi), %xmm1
+; SSE-NEXT:    cmpunordps 32(%rdi), %xmm2
+; SSE-NEXT:    cmpunordps 48(%rdi), %xmm3
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: commute_cmpps_uno_zmm:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcmpunordps (%rdi), %ymm0, %ymm0
+; AVX-NEXT:    vcmpunordps 32(%rdi), %ymm1, %ymm1
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: commute_cmpps_uno_zmm:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vcmpunordps (%rdi), %zmm0, %k0
+; AVX512-NEXT:    vpmovm2d %k0, %zmm0
+; AVX512-NEXT:    retq
+  %1 = load <16 x float>, <16 x float>* %a0
+  %2 = fcmp uno <16 x float> %1, %a1
+  %3 = sext <16 x i1> %2 to <16 x i32>
+  ret <16 x i32> %3
+}
+
+define <16 x i32> @commute_cmpps_ueq_zmm(<16 x float>* %a0, <16 x float> %a1) {
+; SSE-LABEL: commute_cmpps_ueq_zmm:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movaps (%rdi), %xmm7
+; SSE-NEXT:    movaps 16(%rdi), %xmm5
+; SSE-NEXT:    movaps 32(%rdi), %xmm6
+; SSE-NEXT:    movaps 48(%rdi), %xmm8
+; SSE-NEXT:    movaps %xmm7, %xmm4
+; SSE-NEXT:    cmpeqps %xmm0, %xmm4
+; SSE-NEXT:    cmpunordps %xmm7, %xmm0
+; SSE-NEXT:    orps %xmm4, %xmm0
+; SSE-NEXT:    movaps %xmm5, %xmm4
+; SSE-NEXT:    cmpeqps %xmm1, %xmm4
+; SSE-NEXT:    cmpunordps %xmm5, %xmm1
+; SSE-NEXT:    orps %xmm4, %xmm1
+; SSE-NEXT:    movaps %xmm6, %xmm4
+; SSE-NEXT:    cmpeqps %xmm2, %xmm4
+; SSE-NEXT:    cmpunordps %xmm6, %xmm2
+; SSE-NEXT:    orps %xmm4, %xmm2
+; SSE-NEXT:    movaps %xmm8, %xmm4
+; SSE-NEXT:    cmpeqps %xmm3, %xmm4
+; SSE-NEXT:    cmpunordps %xmm8, %xmm3
+; SSE-NEXT:    orps %xmm4, %xmm3
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: commute_cmpps_ueq_zmm:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcmpeq_uqps (%rdi), %ymm0, %ymm0
+; AVX-NEXT:    vcmpeq_uqps 32(%rdi), %ymm1, %ymm1
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: commute_cmpps_ueq_zmm:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vcmpeq_uqps (%rdi), %zmm0, %k0
+; AVX512-NEXT:    vpmovm2d %k0, %zmm0
+; AVX512-NEXT:    retq
+  %1 = load <16 x float>, <16 x float>* %a0
+  %2 = fcmp ueq <16 x float> %1, %a1
+  %3 = sext <16 x i1> %2 to <16 x i32>
+  ret <16 x i32> %3
+}
+
+define <16 x i32> @commute_cmpps_one_zmm(<16 x float>* %a0, <16 x float> %a1) {
+; SSE-LABEL: commute_cmpps_one_zmm:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movaps (%rdi), %xmm7
+; SSE-NEXT:    movaps 16(%rdi), %xmm5
+; SSE-NEXT:    movaps 32(%rdi), %xmm6
+; SSE-NEXT:    movaps 48(%rdi), %xmm8
+; SSE-NEXT:    movaps %xmm7, %xmm4
+; SSE-NEXT:    cmpneqps %xmm0, %xmm4
+; SSE-NEXT:    cmpordps %xmm7, %xmm0
+; SSE-NEXT:    andps %xmm4, %xmm0
+; SSE-NEXT:    movaps %xmm5, %xmm4
+; SSE-NEXT:    cmpneqps %xmm1, %xmm4
+; SSE-NEXT:    cmpordps %xmm5, %xmm1
+; SSE-NEXT:    andps %xmm4, %xmm1
+; SSE-NEXT:    movaps %xmm6, %xmm4
+; SSE-NEXT:    cmpneqps %xmm2, %xmm4
+; SSE-NEXT:    cmpordps %xmm6, %xmm2
+; SSE-NEXT:    andps %xmm4, %xmm2
+; SSE-NEXT:    movaps %xmm8, %xmm4
+; SSE-NEXT:    cmpneqps %xmm3, %xmm4
+; SSE-NEXT:    cmpordps %xmm8, %xmm3
+; SSE-NEXT:    andps %xmm4, %xmm3
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: commute_cmpps_one_zmm:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcmpneq_oqps (%rdi), %ymm0, %ymm0
+; AVX-NEXT:    vcmpneq_oqps 32(%rdi), %ymm1, %ymm1
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: commute_cmpps_one_zmm:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vcmpneq_oqps (%rdi), %zmm0, %k0
+; AVX512-NEXT:    vpmovm2d %k0, %zmm0
+; AVX512-NEXT:    retq
+  %1 = load <16 x float>, <16 x float>* %a0
+  %2 = fcmp one <16 x float> %1, %a1
+  %3 = sext <16 x i1> %2 to <16 x i32>
+  ret <16 x i32> %3
+}
+
+define <16 x i32> @commute_cmpps_lt_zmm(<16 x float>* %a0, <16 x float> %a1) {
+; SSE-LABEL: commute_cmpps_lt_zmm:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movaps (%rdi), %xmm4
+; SSE-NEXT:    movaps 16(%rdi), %xmm5
+; SSE-NEXT:    movaps 32(%rdi), %xmm6
+; SSE-NEXT:    movaps 48(%rdi), %xmm7
+; SSE-NEXT:    cmpltps %xmm0, %xmm4
+; SSE-NEXT:    cmpltps %xmm1, %xmm5
+; SSE-NEXT:    cmpltps %xmm2, %xmm6
+; SSE-NEXT:    cmpltps %xmm3, %xmm7
+; SSE-NEXT:    movaps %xmm4, %xmm0
+; SSE-NEXT:    movaps %xmm5, %xmm1
+; SSE-NEXT:    movaps %xmm6, %xmm2
+; SSE-NEXT:    movaps %xmm7, %xmm3
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: commute_cmpps_lt_zmm:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovaps (%rdi), %ymm2
+; AVX-NEXT:    vmovaps 32(%rdi), %ymm3
+; AVX-NEXT:    vcmpltps %ymm0, %ymm2, %ymm0
+; AVX-NEXT:    vcmpltps %ymm1, %ymm3, %ymm1
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: commute_cmpps_lt_zmm:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vcmpgtps (%rdi), %zmm0, %k0
+; AVX512-NEXT:    vpmovm2d %k0, %zmm0
+; AVX512-NEXT:    retq
+  %1 = load <16 x float>, <16 x float>* %a0
+  %2 = fcmp olt <16 x float> %1, %a1
+  %3 = sext <16 x i1> %2 to <16 x i32>
+  ret <16 x i32> %3
+}
+
+define <16 x i32> @commute_cmpps_le_zmm(<16 x float>* %a0, <16 x float> %a1) {
+; SSE-LABEL: commute_cmpps_le_zmm:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movaps (%rdi), %xmm4
+; SSE-NEXT:    movaps 16(%rdi), %xmm5
+; SSE-NEXT:    movaps 32(%rdi), %xmm6
+; SSE-NEXT:    movaps 48(%rdi), %xmm7
+; SSE-NEXT:    cmpleps %xmm0, %xmm4
+; SSE-NEXT:    cmpleps %xmm1, %xmm5
+; SSE-NEXT:    cmpleps %xmm2, %xmm6
+; SSE-NEXT:    cmpleps %xmm3, %xmm7
+; SSE-NEXT:    movaps %xmm4, %xmm0
+; SSE-NEXT:    movaps %xmm5, %xmm1
+; SSE-NEXT:    movaps %xmm6, %xmm2
+; SSE-NEXT:    movaps %xmm7, %xmm3
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: commute_cmpps_le_zmm:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovaps (%rdi), %ymm2
+; AVX-NEXT:    vmovaps 32(%rdi), %ymm3
+; AVX-NEXT:    vcmpleps %ymm0, %ymm2, %ymm0
+; AVX-NEXT:    vcmpleps %ymm1, %ymm3, %ymm1
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: commute_cmpps_le_zmm:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vcmpgeps (%rdi), %zmm0, %k0
+; AVX512-NEXT:    vpmovm2d %k0, %zmm0
+; AVX512-NEXT:    retq
+  %1 = load <16 x float>, <16 x float>* %a0
+  %2 = fcmp ole <16 x float> %1, %a1
+  %3 = sext <16 x i1> %2 to <16 x i32>
+  ret <16 x i32> %3
+}
+
+define <8 x i64> @commute_cmppd_eq_zmmm(<8 x double>* %a0, <8 x double> %a1) {
+; SSE-LABEL: commute_cmppd_eq_zmmm:
+; SSE:       # %bb.0:
+; SSE-NEXT:    cmpeqpd (%rdi), %xmm0
+; SSE-NEXT:    cmpeqpd 16(%rdi), %xmm1
+; SSE-NEXT:    cmpeqpd 32(%rdi), %xmm2
+; SSE-NEXT:    cmpeqpd 48(%rdi), %xmm3
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: commute_cmppd_eq_zmmm:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcmpeqpd (%rdi), %ymm0, %ymm0
+; AVX-NEXT:    vcmpeqpd 32(%rdi), %ymm1, %ymm1
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: commute_cmppd_eq_zmmm:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vcmpeqpd (%rdi), %zmm0, %k0
+; AVX512-NEXT:    vpmovm2q %k0, %zmm0
+; AVX512-NEXT:    retq
+  %1 = load <8 x double>, <8 x double>* %a0
+  %2 = fcmp oeq <8 x double> %1, %a1
+  %3 = sext <8 x i1> %2 to <8 x i64>
+  ret <8 x i64> %3
+}
+
+define <8 x i64> @commute_cmppd_ne_zmmm(<8 x double>* %a0, <8 x double> %a1) {
+; SSE-LABEL: commute_cmppd_ne_zmmm:
+; SSE:       # %bb.0:
+; SSE-NEXT:    cmpneqpd (%rdi), %xmm0
+; SSE-NEXT:    cmpneqpd 16(%rdi), %xmm1
+; SSE-NEXT:    cmpneqpd 32(%rdi), %xmm2
+; SSE-NEXT:    cmpneqpd 48(%rdi), %xmm3
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: commute_cmppd_ne_zmmm:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcmpneqpd (%rdi), %ymm0, %ymm0
+; AVX-NEXT:    vcmpneqpd 32(%rdi), %ymm1, %ymm1
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: commute_cmppd_ne_zmmm:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vcmpneqpd (%rdi), %zmm0, %k0
+; AVX512-NEXT:    vpmovm2q %k0, %zmm0
+; AVX512-NEXT:    retq
+  %1 = load <8 x double>, <8 x double>* %a0
+  %2 = fcmp une <8 x double> %1, %a1
+  %3 = sext <8 x i1> %2 to <8 x i64>
+  ret <8 x i64> %3
+}
+
+define <8 x i64> @commute_cmppd_ord_zmmm(<8 x double>* %a0, <8 x double> %a1) {
+; SSE-LABEL: commute_cmppd_ord_zmmm:
+; SSE:       # %bb.0:
+; SSE-NEXT:    cmpordpd (%rdi), %xmm0
+; SSE-NEXT:    cmpordpd 16(%rdi), %xmm1
+; SSE-NEXT:    cmpordpd 32(%rdi), %xmm2
+; SSE-NEXT:    cmpordpd 48(%rdi), %xmm3
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: commute_cmppd_ord_zmmm:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcmpordpd (%rdi), %ymm0, %ymm0
+; AVX-NEXT:    vcmpordpd 32(%rdi), %ymm1, %ymm1
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: commute_cmppd_ord_zmmm:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vcmpordpd (%rdi), %zmm0, %k0
+; AVX512-NEXT:    vpmovm2q %k0, %zmm0
+; AVX512-NEXT:    retq
+  %1 = load <8 x double>, <8 x double>* %a0
+  %2 = fcmp ord <8 x double> %1, %a1
+  %3 = sext <8 x i1> %2 to <8 x i64>
+  ret <8 x i64> %3
+}
+
+define <8 x i64> @commute_cmppd_uno_zmmm(<8 x double>* %a0, <8 x double> %a1) {
+; SSE-LABEL: commute_cmppd_uno_zmmm:
+; SSE:       # %bb.0:
+; SSE-NEXT:    cmpunordpd (%rdi), %xmm0
+; SSE-NEXT:    cmpunordpd 16(%rdi), %xmm1
+; SSE-NEXT:    cmpunordpd 32(%rdi), %xmm2
+; SSE-NEXT:    cmpunordpd 48(%rdi), %xmm3
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: commute_cmppd_uno_zmmm:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcmpunordpd (%rdi), %ymm0, %ymm0
+; AVX-NEXT:    vcmpunordpd 32(%rdi), %ymm1, %ymm1
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: commute_cmppd_uno_zmmm:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vcmpunordpd (%rdi), %zmm0, %k0
+; AVX512-NEXT:    vpmovm2q %k0, %zmm0
+; AVX512-NEXT:    retq
+  %1 = load <8 x double>, <8 x double>* %a0
+  %2 = fcmp uno <8 x double> %1, %a1
+  %3 = sext <8 x i1> %2 to <8 x i64>
+  ret <8 x i64> %3
+}
+
+define <8 x i64> @commute_cmppd_ueq_zmmm(<8 x double>* %a0, <8 x double> %a1) {
+; SSE-LABEL: commute_cmppd_ueq_zmmm:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movapd (%rdi), %xmm7
+; SSE-NEXT:    movapd 16(%rdi), %xmm5
+; SSE-NEXT:    movapd 32(%rdi), %xmm6
+; SSE-NEXT:    movapd 48(%rdi), %xmm8
+; SSE-NEXT:    movapd %xmm7, %xmm4
+; SSE-NEXT:    cmpeqpd %xmm0, %xmm4
+; SSE-NEXT:    cmpunordpd %xmm7, %xmm0
+; SSE-NEXT:    orpd %xmm4, %xmm0
+; SSE-NEXT:    movapd %xmm5, %xmm4
+; SSE-NEXT:    cmpeqpd %xmm1, %xmm4
+; SSE-NEXT:    cmpunordpd %xmm5, %xmm1
+; SSE-NEXT:    orpd %xmm4, %xmm1
+; SSE-NEXT:    movapd %xmm6, %xmm4
+; SSE-NEXT:    cmpeqpd %xmm2, %xmm4
+; SSE-NEXT:    cmpunordpd %xmm6, %xmm2
+; SSE-NEXT:    orpd %xmm4, %xmm2
+; SSE-NEXT:    movapd %xmm8, %xmm4
+; SSE-NEXT:    cmpeqpd %xmm3, %xmm4
+; SSE-NEXT:    cmpunordpd %xmm8, %xmm3
+; SSE-NEXT:    orpd %xmm4, %xmm3
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: commute_cmppd_ueq_zmmm:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcmpeq_uqpd (%rdi), %ymm0, %ymm0
+; AVX-NEXT:    vcmpeq_uqpd 32(%rdi), %ymm1, %ymm1
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: commute_cmppd_ueq_zmmm:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vcmpeq_uqpd (%rdi), %zmm0, %k0
+; AVX512-NEXT:    vpmovm2q %k0, %zmm0
+; AVX512-NEXT:    retq
+  %1 = load <8 x double>, <8 x double>* %a0
+  %2 = fcmp ueq <8 x double> %1, %a1
+  %3 = sext <8 x i1> %2 to <8 x i64>
+  ret <8 x i64> %3
+}
+
+define <8 x i64> @commute_cmppd_one_zmmm(<8 x double>* %a0, <8 x double> %a1) {
+; SSE-LABEL: commute_cmppd_one_zmmm:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movapd (%rdi), %xmm7
+; SSE-NEXT:    movapd 16(%rdi), %xmm5
+; SSE-NEXT:    movapd 32(%rdi), %xmm6
+; SSE-NEXT:    movapd 48(%rdi), %xmm8
+; SSE-NEXT:    movapd %xmm7, %xmm4
+; SSE-NEXT:    cmpneqpd %xmm0, %xmm4
+; SSE-NEXT:    cmpordpd %xmm7, %xmm0
+; SSE-NEXT:    andpd %xmm4, %xmm0
+; SSE-NEXT:    movapd %xmm5, %xmm4
+; SSE-NEXT:    cmpneqpd %xmm1, %xmm4
+; SSE-NEXT:    cmpordpd %xmm5, %xmm1
+; SSE-NEXT:    andpd %xmm4, %xmm1
+; SSE-NEXT:    movapd %xmm6, %xmm4
+; SSE-NEXT:    cmpneqpd %xmm2, %xmm4
+; SSE-NEXT:    cmpordpd %xmm6, %xmm2
+; SSE-NEXT:    andpd %xmm4, %xmm2
+; SSE-NEXT:    movapd %xmm8, %xmm4
+; SSE-NEXT:    cmpneqpd %xmm3, %xmm4
+; SSE-NEXT:    cmpordpd %xmm8, %xmm3
+; SSE-NEXT:    andpd %xmm4, %xmm3
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: commute_cmppd_one_zmmm:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcmpneq_oqpd (%rdi), %ymm0, %ymm0
+; AVX-NEXT:    vcmpneq_oqpd 32(%rdi), %ymm1, %ymm1
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: commute_cmppd_one_zmmm:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vcmpneq_oqpd (%rdi), %zmm0, %k0
+; AVX512-NEXT:    vpmovm2q %k0, %zmm0
+; AVX512-NEXT:    retq
+  %1 = load <8 x double>, <8 x double>* %a0
+  %2 = fcmp one <8 x double> %1, %a1
+  %3 = sext <8 x i1> %2 to <8 x i64>
+  ret <8 x i64> %3
+}
+
+define <8 x i64> @commute_cmppd_lt_zmmm(<8 x double>* %a0, <8 x double> %a1) {
+; SSE-LABEL: commute_cmppd_lt_zmmm:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movapd (%rdi), %xmm4
+; SSE-NEXT:    movapd 16(%rdi), %xmm5
+; SSE-NEXT:    movapd 32(%rdi), %xmm6
+; SSE-NEXT:    movapd 48(%rdi), %xmm7
+; SSE-NEXT:    cmpltpd %xmm0, %xmm4
+; SSE-NEXT:    cmpltpd %xmm1, %xmm5
+; SSE-NEXT:    cmpltpd %xmm2, %xmm6
+; SSE-NEXT:    cmpltpd %xmm3, %xmm7
+; SSE-NEXT:    movapd %xmm4, %xmm0
+; SSE-NEXT:    movapd %xmm5, %xmm1
+; SSE-NEXT:    movapd %xmm6, %xmm2
+; SSE-NEXT:    movapd %xmm7, %xmm3
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: commute_cmppd_lt_zmmm:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovapd (%rdi), %ymm2
+; AVX-NEXT:    vmovapd 32(%rdi), %ymm3
+; AVX-NEXT:    vcmpltpd %ymm0, %ymm2, %ymm0
+; AVX-NEXT:    vcmpltpd %ymm1, %ymm3, %ymm1
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: commute_cmppd_lt_zmmm:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vcmpgtpd (%rdi), %zmm0, %k0
+; AVX512-NEXT:    vpmovm2q %k0, %zmm0
+; AVX512-NEXT:    retq
+  %1 = load <8 x double>, <8 x double>* %a0
+  %2 = fcmp olt <8 x double> %1, %a1
+  %3 = sext <8 x i1> %2 to <8 x i64>
+  ret <8 x i64> %3
+}
+
+define <8 x i64> @commute_cmppd_le_zmmm(<8 x double>* %a0, <8 x double> %a1) {
+; SSE-LABEL: commute_cmppd_le_zmmm:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movapd (%rdi), %xmm4
+; SSE-NEXT:    movapd 16(%rdi), %xmm5
+; SSE-NEXT:    movapd 32(%rdi), %xmm6
+; SSE-NEXT:    movapd 48(%rdi), %xmm7
+; SSE-NEXT:    cmplepd %xmm0, %xmm4
+; SSE-NEXT:    cmplepd %xmm1, %xmm5
+; SSE-NEXT:    cmplepd %xmm2, %xmm6
+; SSE-NEXT:    cmplepd %xmm3, %xmm7
+; SSE-NEXT:    movapd %xmm4, %xmm0
+; SSE-NEXT:    movapd %xmm5, %xmm1
+; SSE-NEXT:    movapd %xmm6, %xmm2
+; SSE-NEXT:    movapd %xmm7, %xmm3
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: commute_cmppd_le_zmmm:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovapd (%rdi), %ymm2
+; AVX-NEXT:    vmovapd 32(%rdi), %ymm3
+; AVX-NEXT:    vcmplepd %ymm0, %ymm2, %ymm0
+; AVX-NEXT:    vcmplepd %ymm1, %ymm3, %ymm1
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: commute_cmppd_le_zmmm:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vcmpgepd (%rdi), %zmm0, %k0
+; AVX512-NEXT:    vpmovm2q %k0, %zmm0
+; AVX512-NEXT:    retq
+  %1 = load <8 x double>, <8 x double>* %a0
+  %2 = fcmp ole <8 x double> %1, %a1
+  %3 = sext <8 x i1> %2 to <8 x i64>
+  ret <8 x i64> %3
+}
diff --git a/llvm/test/CodeGen/X86/compare-add.ll b/llvm/test/CodeGen/X86/compare-add.ll
index 51e47ea4a..004520162 100644
--- a/llvm/test/CodeGen/X86/compare-add.ll
+++ b/llvm/test/CodeGen/X86/compare-add.ll
@@ -1,6 +1,12 @@
-; RUN: llc < %s -mtriple=i686-- | not grep add
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- | FileCheck %s
 
 define i1 @X(i32 %X) {
+; CHECK-LABEL: X:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    cmpl $12331, {{[0-9]+}}(%esp) # imm = 0x302B
+; CHECK-NEXT:    setne %al
+; CHECK-NEXT:    retl
         %Y = add i32 %X, 14             ; <i32> [#uses=1]
         %Z = icmp ne i32 %Y, 12345              ; <i1> [#uses=1]
         ret i1 %Z
diff --git a/llvm/test/CodeGen/X86/compare_folding.ll b/llvm/test/CodeGen/X86/compare_folding.ll
index 0f85d644a..b24217fe6 100644
--- a/llvm/test/CodeGen/X86/compare_folding.ll
+++ b/llvm/test/CodeGen/X86/compare_folding.ll
@@ -1,10 +1,14 @@
-; RUN: llc < %s -mtriple=i686-- -mcpu=yonah | \
-; RUN:   grep movsd | count 1
-; RUN: llc < %s -mtriple=i686-- -mcpu=yonah | \
-; RUN:   grep ucomisd
-declare i1 @llvm.isunordered.f64(double, double)
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- -mcpu=yonah | FileCheck %s
 
+declare i1 @llvm.isunordered.f64(double, double)
 define i1 @test1(double %X, double %Y) {
+; CHECK-LABEL: test1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    ucomisd {{[0-9]+}}(%esp), %xmm0
+; CHECK-NEXT:    setp %al
+; CHECK-NEXT:    retl
         %COM = fcmp uno double %X, %Y           ; <i1> [#uses=1]
         ret i1 %COM
 }
diff --git a/llvm/test/CodeGen/X86/complex-fastmath.ll b/llvm/test/CodeGen/X86/complex-fastmath.ll
index d61c1242b..4fef7c2a4 100644
--- a/llvm/test/CodeGen/X86/complex-fastmath.ll
+++ b/llvm/test/CodeGen/X86/complex-fastmath.ll
@@ -39,7 +39,7 @@ define <2 x float> @complex_square_f32(<2 x float>) #0 {
 ; FMA-NEXT:    vaddss %xmm0, %xmm0, %xmm2
 ; FMA-NEXT:    vmulss %xmm2, %xmm1, %xmm2
 ; FMA-NEXT:    vmulss %xmm1, %xmm1, %xmm1
-; FMA-NEXT:    vfmsub231ss %xmm0, %xmm0, %xmm1
+; FMA-NEXT:    vfmsub231ss {{.*#+}} xmm1 = (xmm0 * xmm0) - xmm1
 ; FMA-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm2[0],xmm1[2,3]
 ; FMA-NEXT:    retq
   %2 = extractelement <2 x float> %0, i32 0
@@ -85,7 +85,7 @@ define <2 x double> @complex_square_f64(<2 x double>) #0 {
 ; FMA-NEXT:    vaddsd %xmm0, %xmm0, %xmm2
 ; FMA-NEXT:    vmulsd %xmm2, %xmm1, %xmm2
 ; FMA-NEXT:    vmulsd %xmm1, %xmm1, %xmm1
-; FMA-NEXT:    vfmsub231sd %xmm0, %xmm0, %xmm1
+; FMA-NEXT:    vfmsub231sd {{.*#+}} xmm1 = (xmm0 * xmm0) - xmm1
 ; FMA-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm1[0],xmm2[0]
 ; FMA-NEXT:    retq
   %2 = extractelement <2 x double> %0, i32 0
@@ -137,9 +137,9 @@ define <2 x float> @complex_mul_f32(<2 x float>, <2 x float>) #0 {
 ; FMA-NEXT:    vmovshdup {{.*#+}} xmm2 = xmm0[1,1,3,3]
 ; FMA-NEXT:    vmovshdup {{.*#+}} xmm3 = xmm1[1,1,3,3]
 ; FMA-NEXT:    vmulss %xmm2, %xmm1, %xmm4
-; FMA-NEXT:    vfmadd231ss %xmm0, %xmm3, %xmm4
+; FMA-NEXT:    vfmadd231ss {{.*#+}} xmm4 = (xmm3 * xmm0) + xmm4
 ; FMA-NEXT:    vmulss %xmm2, %xmm3, %xmm2
-; FMA-NEXT:    vfmsub231ss %xmm0, %xmm1, %xmm2
+; FMA-NEXT:    vfmsub231ss {{.*#+}} xmm2 = (xmm1 * xmm0) - xmm2
 ; FMA-NEXT:    vinsertps {{.*#+}} xmm0 = xmm2[0],xmm4[0],xmm2[2,3]
 ; FMA-NEXT:    retq
   %3 = extractelement <2 x float> %0, i32 0
@@ -192,9 +192,9 @@ define <2 x double> @complex_mul_f64(<2 x double>, <2 x double>) #0 {
 ; FMA-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
 ; FMA-NEXT:    vpermilpd {{.*#+}} xmm3 = xmm1[1,0]
 ; FMA-NEXT:    vmulsd %xmm2, %xmm1, %xmm4
-; FMA-NEXT:    vfmadd231sd %xmm0, %xmm3, %xmm4
+; FMA-NEXT:    vfmadd231sd {{.*#+}} xmm4 = (xmm3 * xmm0) + xmm4
 ; FMA-NEXT:    vmulsd %xmm2, %xmm3, %xmm2
-; FMA-NEXT:    vfmsub231sd %xmm0, %xmm1, %xmm2
+; FMA-NEXT:    vfmsub231sd {{.*#+}} xmm2 = (xmm1 * xmm0) - xmm2
 ; FMA-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm2[0],xmm4[0]
 ; FMA-NEXT:    retq
   %3 = extractelement <2 x double> %0, i32 0
diff --git a/llvm/test/CodeGen/X86/conditional-tailcall-pgso.ll b/llvm/test/CodeGen/X86/conditional-tailcall-pgso.ll
new file mode 100644
index 000000000..65bd1dad2
--- /dev/null
+++ b/llvm/test/CodeGen/X86/conditional-tailcall-pgso.ll
@@ -0,0 +1,242 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-linux   -show-mc-encoding | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK32
+; RUN: llc < %s -mtriple=x86_64-linux -show-mc-encoding | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK64
+; RUN: llc < %s -mtriple=x86_64-win32 -show-mc-encoding | FileCheck %s --check-prefix=CHECK --check-prefix=WIN64
+
+declare void @foo()
+declare void @bar()
+
+define void @f(i32 %x, i32 %y) !prof !14 {
+; CHECK32-LABEL: f:
+; CHECK32:       # %bb.0: # %entry
+; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; CHECK32-NEXT:    cmpl {{[0-9]+}}(%esp), %eax # encoding: [0x3b,0x44,0x24,0x08]
+; CHECK32-NEXT:    jne bar # TAILCALL
+; CHECK32-NEXT:    # encoding: [0x75,A]
+; CHECK32-NEXT:    # fixup A - offset: 1, value: bar-1, kind: FK_PCRel_1
+; CHECK32-NEXT:  # %bb.1: # %bb1
+; CHECK32-NEXT:    jmp foo # TAILCALL
+; CHECK32-NEXT:    # encoding: [0xeb,A]
+; CHECK32-NEXT:    # fixup A - offset: 1, value: foo-1, kind: FK_PCRel_1
+;
+; CHECK64-LABEL: f:
+; CHECK64:       # %bb.0: # %entry
+; CHECK64-NEXT:    cmpl %esi, %edi # encoding: [0x39,0xf7]
+; CHECK64-NEXT:    jne bar # TAILCALL
+; CHECK64-NEXT:    # encoding: [0x75,A]
+; CHECK64-NEXT:    # fixup A - offset: 1, value: bar-1, kind: FK_PCRel_1
+; CHECK64-NEXT:  # %bb.1: # %bb1
+; CHECK64-NEXT:    jmp foo # TAILCALL
+; CHECK64-NEXT:    # encoding: [0xeb,A]
+; CHECK64-NEXT:    # fixup A - offset: 1, value: foo-1, kind: FK_PCRel_1
+;
+; WIN64-LABEL: f:
+; WIN64:       # %bb.0: # %entry
+; WIN64-NEXT:    cmpl %edx, %ecx # encoding: [0x39,0xd1]
+; WIN64-NEXT:    jne bar # TAILCALL
+; WIN64-NEXT:    # encoding: [0x75,A]
+; WIN64-NEXT:    # fixup A - offset: 1, value: bar-1, kind: FK_PCRel_1
+; WIN64-NEXT:  # %bb.1: # %bb1
+; WIN64-NEXT:    jmp foo # TAILCALL
+; WIN64-NEXT:    # encoding: [0xeb,A]
+; WIN64-NEXT:    # fixup A - offset: 1, value: foo-1, kind: FK_PCRel_1
+entry:
+	%p = icmp eq i32 %x, %y
+  br i1 %p, label %bb1, label %bb2
+bb1:
+  tail call void @foo()
+  ret void
+bb2:
+  tail call void @bar()
+  ret void
+
+; Check that the asm doesn't just look good, but uses the correct encoding.
+}
+
+define void @f_non_leaf(i32 %x, i32 %y) !prof !14 {
+; CHECK32-LABEL: f_non_leaf:
+; CHECK32:       # %bb.0: # %entry
+; CHECK32-NEXT:    pushl %ebx # encoding: [0x53]
+; CHECK32-NEXT:    .cfi_def_cfa_offset 8
+; CHECK32-NEXT:    .cfi_offset %ebx, -8
+; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x08]
+; CHECK32-NEXT:    #APP
+; CHECK32-NEXT:    #NO_APP
+; CHECK32-NEXT:    cmpl {{[0-9]+}}(%esp), %eax # encoding: [0x3b,0x44,0x24,0x0c]
+; CHECK32-NEXT:    jne .LBB1_2 # encoding: [0x75,A]
+; CHECK32-NEXT:    # fixup A - offset: 1, value: .LBB1_2-1, kind: FK_PCRel_1
+; CHECK32-NEXT:  # %bb.1: # %bb1
+; CHECK32-NEXT:    popl %ebx # encoding: [0x5b]
+; CHECK32-NEXT:    .cfi_def_cfa_offset 4
+; CHECK32-NEXT:    jmp foo # TAILCALL
+; CHECK32-NEXT:    # encoding: [0xeb,A]
+; CHECK32-NEXT:    # fixup A - offset: 1, value: foo-1, kind: FK_PCRel_1
+; CHECK32-NEXT:  .LBB1_2: # %bb2
+; CHECK32-NEXT:    .cfi_def_cfa_offset 8
+; CHECK32-NEXT:    popl %ebx # encoding: [0x5b]
+; CHECK32-NEXT:    .cfi_def_cfa_offset 4
+; CHECK32-NEXT:    jmp bar # TAILCALL
+; CHECK32-NEXT:    # encoding: [0xeb,A]
+; CHECK32-NEXT:    # fixup A - offset: 1, value: bar-1, kind: FK_PCRel_1
+;
+; CHECK64-LABEL: f_non_leaf:
+; CHECK64:       # %bb.0: # %entry
+; CHECK64-NEXT:    pushq %rbx # encoding: [0x53]
+; CHECK64-NEXT:    .cfi_def_cfa_offset 16
+; CHECK64-NEXT:    .cfi_offset %rbx, -16
+; CHECK64-NEXT:    #APP
+; CHECK64-NEXT:    #NO_APP
+; CHECK64-NEXT:    cmpl %esi, %edi # encoding: [0x39,0xf7]
+; CHECK64-NEXT:    jne .LBB1_2 # encoding: [0x75,A]
+; CHECK64-NEXT:    # fixup A - offset: 1, value: .LBB1_2-1, kind: FK_PCRel_1
+; CHECK64-NEXT:  # %bb.1: # %bb1
+; CHECK64-NEXT:    popq %rbx # encoding: [0x5b]
+; CHECK64-NEXT:    .cfi_def_cfa_offset 8
+; CHECK64-NEXT:    jmp foo # TAILCALL
+; CHECK64-NEXT:    # encoding: [0xeb,A]
+; CHECK64-NEXT:    # fixup A - offset: 1, value: foo-1, kind: FK_PCRel_1
+; CHECK64-NEXT:  .LBB1_2: # %bb2
+; CHECK64-NEXT:    .cfi_def_cfa_offset 16
+; CHECK64-NEXT:    popq %rbx # encoding: [0x5b]
+; CHECK64-NEXT:    .cfi_def_cfa_offset 8
+; CHECK64-NEXT:    jmp bar # TAILCALL
+; CHECK64-NEXT:    # encoding: [0xeb,A]
+; CHECK64-NEXT:    # fixup A - offset: 1, value: bar-1, kind: FK_PCRel_1
+;
+; WIN64-LABEL: f_non_leaf:
+; WIN64:       # %bb.0: # %entry
+; WIN64-NEXT:    pushq %rbx # encoding: [0x53]
+; WIN64-NEXT:    .seh_pushreg %rbx
+; WIN64-NEXT:    .seh_endprologue
+; WIN64-NEXT:    #APP
+; WIN64-NEXT:    #NO_APP
+; WIN64-NEXT:    cmpl %edx, %ecx # encoding: [0x39,0xd1]
+; WIN64-NEXT:    jne .LBB1_2 # encoding: [0x75,A]
+; WIN64-NEXT:    # fixup A - offset: 1, value: .LBB1_2-1, kind: FK_PCRel_1
+; WIN64-NEXT:  # %bb.1: # %bb1
+; WIN64-NEXT:    popq %rbx # encoding: [0x5b]
+; WIN64-NEXT:    jmp foo # TAILCALL
+; WIN64-NEXT:    # encoding: [0xeb,A]
+; WIN64-NEXT:    # fixup A - offset: 1, value: foo-1, kind: FK_PCRel_1
+; WIN64-NEXT:  .LBB1_2: # %bb2
+; WIN64-NEXT:    nop # encoding: [0x90]
+; WIN64-NEXT:    popq %rbx # encoding: [0x5b]
+; WIN64-NEXT:    jmp bar # TAILCALL
+; WIN64-NEXT:    # encoding: [0xeb,A]
+; WIN64-NEXT:    # fixup A - offset: 1, value: bar-1, kind: FK_PCRel_1
+; WIN64-NEXT:    .seh_handlerdata
+; WIN64-NEXT:    .text
+; WIN64-NEXT:    .seh_endproc
+entry:
+  ; Force %ebx to be spilled on the stack, turning this into
+  ; not a "leaf" function for Win64.
+  tail call void asm sideeffect "", "~{ebx}"()
+
+	%p = icmp eq i32 %x, %y
+  br i1 %p, label %bb1, label %bb2
+bb1:
+  tail call void @foo()
+  ret void
+bb2:
+  tail call void @bar()
+  ret void
+
+}
+
+declare x86_thiscallcc zeroext i1 @baz(i8*, i32)
+define x86_thiscallcc zeroext i1 @BlockPlacementTest(i8* %this, i32 %x) !prof !14 {
+; CHECK32-LABEL: BlockPlacementTest:
+; CHECK32:       # %bb.0: # %entry
+; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %edx # encoding: [0x8b,0x54,0x24,0x04]
+; CHECK32-NEXT:    testb $42, %dl # encoding: [0xf6,0xc2,0x2a]
+; CHECK32-NEXT:    je .LBB2_3 # encoding: [0x74,A]
+; CHECK32-NEXT:    # fixup A - offset: 1, value: .LBB2_3-1, kind: FK_PCRel_1
+; CHECK32-NEXT:  # %bb.1: # %land.rhs
+; CHECK32-NEXT:    movb $1, %al # encoding: [0xb0,0x01]
+; CHECK32-NEXT:    testb $44, %dl # encoding: [0xf6,0xc2,0x2c]
+; CHECK32-NEXT:    je baz # TAILCALL
+; CHECK32-NEXT:    # encoding: [0x74,A]
+; CHECK32-NEXT:    # fixup A - offset: 1, value: baz-1, kind: FK_PCRel_1
+; CHECK32-NEXT:  .LBB2_2: # %land.end
+; CHECK32-NEXT:    # kill: def $al killed $al killed $eax
+; CHECK32-NEXT:    retl $4 # encoding: [0xc2,0x04,0x00]
+; CHECK32-NEXT:  .LBB2_3:
+; CHECK32-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
+; CHECK32-NEXT:    jmp .LBB2_2 # encoding: [0xeb,A]
+; CHECK32-NEXT:    # fixup A - offset: 1, value: .LBB2_2-1, kind: FK_PCRel_1
+;
+; CHECK64-LABEL: BlockPlacementTest:
+; CHECK64:       # %bb.0: # %entry
+; CHECK64-NEXT:    testb $42, %sil # encoding: [0x40,0xf6,0xc6,0x2a]
+; CHECK64-NEXT:    je .LBB2_3 # encoding: [0x74,A]
+; CHECK64-NEXT:    # fixup A - offset: 1, value: .LBB2_3-1, kind: FK_PCRel_1
+; CHECK64-NEXT:  # %bb.1: # %land.rhs
+; CHECK64-NEXT:    movb $1, %al # encoding: [0xb0,0x01]
+; CHECK64-NEXT:    testb $44, %sil # encoding: [0x40,0xf6,0xc6,0x2c]
+; CHECK64-NEXT:    je baz # TAILCALL
+; CHECK64-NEXT:    # encoding: [0x74,A]
+; CHECK64-NEXT:    # fixup A - offset: 1, value: baz-1, kind: FK_PCRel_1
+; CHECK64-NEXT:  .LBB2_2: # %land.end
+; CHECK64-NEXT:    # kill: def $al killed $al killed $eax
+; CHECK64-NEXT:    retq # encoding: [0xc3]
+; CHECK64-NEXT:  .LBB2_3:
+; CHECK64-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
+; CHECK64-NEXT:    jmp .LBB2_2 # encoding: [0xeb,A]
+; CHECK64-NEXT:    # fixup A - offset: 1, value: .LBB2_2-1, kind: FK_PCRel_1
+;
+; WIN64-LABEL: BlockPlacementTest:
+; WIN64:       # %bb.0: # %entry
+; WIN64-NEXT:    testb $42, %dl # encoding: [0xf6,0xc2,0x2a]
+; WIN64-NEXT:    je .LBB2_3 # encoding: [0x74,A]
+; WIN64-NEXT:    # fixup A - offset: 1, value: .LBB2_3-1, kind: FK_PCRel_1
+; WIN64-NEXT:  # %bb.1: # %land.rhs
+; WIN64-NEXT:    movb $1, %al # encoding: [0xb0,0x01]
+; WIN64-NEXT:    testb $44, %dl # encoding: [0xf6,0xc2,0x2c]
+; WIN64-NEXT:    je baz # TAILCALL
+; WIN64-NEXT:    # encoding: [0x74,A]
+; WIN64-NEXT:    # fixup A - offset: 1, value: baz-1, kind: FK_PCRel_1
+; WIN64-NEXT:  .LBB2_2: # %land.end
+; WIN64-NEXT:    # kill: def $al killed $al killed $eax
+; WIN64-NEXT:    retq # encoding: [0xc3]
+; WIN64-NEXT:  .LBB2_3:
+; WIN64-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
+; WIN64-NEXT:    jmp .LBB2_2 # encoding: [0xeb,A]
+; WIN64-NEXT:    # fixup A - offset: 1, value: .LBB2_2-1, kind: FK_PCRel_1
+entry:
+  %and = and i32 %x, 42
+  %tobool = icmp eq i32 %and, 0
+  br i1 %tobool, label %land.end, label %land.rhs
+
+land.rhs:
+  %and6 = and i32 %x, 44
+  %tobool7 = icmp eq i32 %and6, 0
+  br i1 %tobool7, label %lor.rhs, label %land.end
+
+lor.rhs:
+  %call = tail call x86_thiscallcc zeroext i1 @baz(i8* %this, i32 %x) #2
+  br label %land.end
+
+land.end:
+  %0 = phi i1 [ false, %entry ], [ true, %land.rhs ], [ %call, %lor.rhs ]
+  ret i1 %0
+
+; Make sure machine block placement isn't confused by the conditional tail call,
+; but sees that it can fall through to the next block.
+}
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 1, !"ProfileSummary", !1}
+!1 = !{!2, !3, !4, !5, !6, !7, !8, !9}
+!2 = !{!"ProfileFormat", !"InstrProf"}
+!3 = !{!"TotalCount", i64 10000}
+!4 = !{!"MaxCount", i64 10}
+!5 = !{!"MaxInternalCount", i64 1}
+!6 = !{!"MaxFunctionCount", i64 1000}
+!7 = !{!"NumCounts", i64 3}
+!8 = !{!"NumFunctions", i64 3}
+!9 = !{!"DetailedSummary", !10}
+!10 = !{!11, !12, !13}
+!11 = !{i32 10000, i64 100, i32 1}
+!12 = !{i32 999000, i64 100, i32 1}
+!13 = !{i32 999999, i64 1, i32 2}
+!14 = !{!"function_entry_count", i64 0}
diff --git a/llvm/test/CodeGen/X86/conditional-tailcall.ll b/llvm/test/CodeGen/X86/conditional-tailcall.ll
index e1a1e7b77..17078413a 100644
--- a/llvm/test/CodeGen/X86/conditional-tailcall.ll
+++ b/llvm/test/CodeGen/X86/conditional-tailcall.ll
@@ -106,7 +106,7 @@ define void @f_non_leaf(i32 %x, i32 %y) optsize {
 ; WIN64-LABEL: f_non_leaf:
 ; WIN64:       # %bb.0: # %entry
 ; WIN64-NEXT:    pushq %rbx # encoding: [0x53]
-; WIN64-NEXT:    .seh_pushreg 3
+; WIN64-NEXT:    .seh_pushreg %rbx
 ; WIN64-NEXT:    .seh_endprologue
 ; WIN64-NEXT:    #APP
 ; WIN64-NEXT:    #NO_APP
diff --git a/llvm/test/CodeGen/X86/constant-combines.ll b/llvm/test/CodeGen/X86/constant-combines.ll
index 45bc635bb..736aea6b5 100644
--- a/llvm/test/CodeGen/X86/constant-combines.ll
+++ b/llvm/test/CodeGen/X86/constant-combines.ll
@@ -7,7 +7,8 @@ target triple = "x86_64-unknown-unknown"
 define void @bitstore_fold() {
 ; CHECK-LABEL: bitstore_fold:
 ; CHECK:       # %bb.0: # %BB
-; CHECK-NEXT:    movl $-2, 0
+; CHECK-NEXT:    movl $-1, 0
+; CHECK-NEXT:    movb $0, 0
 ; CHECK-NEXT:    retq
 BB:
    store i32 -1, i32* null
diff --git a/llvm/test/CodeGen/X86/constrained-fp80-trunc-ext.ll b/llvm/test/CodeGen/X86/constrained-fp80-trunc-ext.ll
index ae07f8434..9c408c70c 100644
--- a/llvm/test/CodeGen/X86/constrained-fp80-trunc-ext.ll
+++ b/llvm/test/CodeGen/X86/constrained-fp80-trunc-ext.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -O3 -mtriple=x86_64-gnu-linux < %s | FileCheck %s
 
-define x86_fp80 @constrained_fpext_f32_as_fp80(float %mem) {
+define x86_fp80 @constrained_fpext_f32_as_fp80(float %mem) #0 {
 ; CHECK-LABEL: constrained_fpext_f32_as_fp80:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    movss %xmm0, -{{[0-9]+}}(%rsp)
@@ -10,11 +10,11 @@ define x86_fp80 @constrained_fpext_f32_as_fp80(float %mem) {
 entry:
   %ext = call x86_fp80 @llvm.experimental.constrained.fpext.f80.f32(
             float %mem,
-            metadata !"fpexcept.strict")
+            metadata !"fpexcept.strict") #0
   ret x86_fp80 %ext
 }
 
-define float @constrained_fptrunc_f80_to_f32(x86_fp80 %reg) {
+define float @constrained_fptrunc_f80_to_f32(x86_fp80 %reg) #0 {
 ; CHECK-LABEL: constrained_fptrunc_f80_to_f32:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    fldt {{[0-9]+}}(%rsp)
@@ -24,11 +24,11 @@ define float @constrained_fptrunc_f80_to_f32(x86_fp80 %reg) {
   %trunc = call float @llvm.experimental.constrained.fptrunc.f32.f80(
              x86_fp80 %reg,
              metadata !"round.dynamic",
-             metadata !"fpexcept.strict")
+             metadata !"fpexcept.strict") #0
   ret float %trunc
 }
 
-define x86_fp80 @constrained_fpext_f64_to_f80(double %mem) {
+define x86_fp80 @constrained_fpext_f64_to_f80(double %mem) #0 {
 ; CHECK-LABEL: constrained_fpext_f64_to_f80:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    movsd %xmm0, -{{[0-9]+}}(%rsp)
@@ -37,11 +37,11 @@ define x86_fp80 @constrained_fpext_f64_to_f80(double %mem) {
 entry:
   %ext = call x86_fp80 @llvm.experimental.constrained.fpext.f80.f64(
             double %mem,
-            metadata !"fpexcept.strict")
+            metadata !"fpexcept.strict") #0
   ret x86_fp80 %ext
 }
 
-define double @constrained_fptrunc_f80_to_f64(x86_fp80 %reg) {
+define double @constrained_fptrunc_f80_to_f64(x86_fp80 %reg) #0 {
 ; CHECK-LABEL: constrained_fptrunc_f80_to_f64:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    fldt {{[0-9]+}}(%rsp)
@@ -51,10 +51,12 @@ define double @constrained_fptrunc_f80_to_f64(x86_fp80 %reg) {
   %trunc = call double @llvm.experimental.constrained.fptrunc.f64.f80(
              x86_fp80 %reg,
              metadata !"round.dynamic",
-             metadata !"fpexcept.strict")
+             metadata !"fpexcept.strict") #0
   ret double %trunc
 }
 
+attributes #0 = { strictfp }
+
 declare x86_fp80 @llvm.experimental.constrained.fpext.f80.f32(float, metadata)
 declare x86_fp80 @llvm.experimental.constrained.fpext.f80.f64(double, metadata)
 declare float @llvm.experimental.constrained.fptrunc.f32.f80(x86_fp80, metadata, metadata)
diff --git a/llvm/test/CodeGen/X86/copysign-constant-magnitude.ll b/llvm/test/CodeGen/X86/copysign-constant-magnitude.ll
index 2e39fb976..aca23b032 100644
--- a/llvm/test/CodeGen/X86/copysign-constant-magnitude.ll
+++ b/llvm/test/CodeGen/X86/copysign-constant-magnitude.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=x86_64-apple-macosx10.10.0 < %s | FileCheck %s
 
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
@@ -12,9 +13,8 @@ target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 define double @mag_pos0_double(double %x) nounwind {
 ; CHECK-LABEL: mag_pos0_double:
 ; CHECK:       ## %bb.0:
-; CHECK-NEXT:    andps [[SIGNMASK1]](%rip), %xmm0
+; CHECK-NEXT:    andps {{.*}}(%rip), %xmm0
 ; CHECK-NEXT:    retq
-;
   %y = call double @copysign(double 0.0, double %x)
   ret double %y
 }
@@ -25,10 +25,9 @@ define double @mag_pos0_double(double %x) nounwind {
 define double @mag_neg0_double(double %x) nounwind {
 ; CHECK-LABEL: mag_neg0_double:
 ; CHECK:       ## %bb.0:
-; CHECK-NEXT:    movsd [[SIGNMASK2]](%rip), %xmm1
+; CHECK-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
 ; CHECK-NEXT:    andps %xmm1, %xmm0
 ; CHECK-NEXT:    retq
-;
   %y = call double @copysign(double -0.0, double %x)
   ret double %y
 }
@@ -42,11 +41,10 @@ define double @mag_neg0_double(double %x) nounwind {
 define double @mag_pos1_double(double %x) nounwind {
 ; CHECK-LABEL: mag_pos1_double:
 ; CHECK:       ## %bb.0:
-; CHECK-NEXT:    andps [[SIGNMASK3]](%rip), %xmm0
-; CHECK-NEXT:    movsd [[ONE3]](%rip), %xmm1
+; CHECK-NEXT:    andps {{.*}}(%rip), %xmm0
+; CHECK-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
 ; CHECK-NEXT:    orps %xmm1, %xmm0
 ; CHECK-NEXT:    retq
-;
   %y = call double @copysign(double 1.0, double %x)
   ret double %y
 }
@@ -61,10 +59,9 @@ define double @mag_pos1_double(double %x) nounwind {
 define double @mag_neg1_double(double %x) nounwind {
 ; CHECK-LABEL: mag_neg1_double:
 ; CHECK:       ## %bb.0:
-; CHECK-NEXT:    andps [[SIGNMASK4]](%rip), %xmm0
-; CHECK-NEXT:    orps [[ONE4]](%rip), %xmm0
+; CHECK-NEXT:    andps {{.*}}(%rip), %xmm0
+; CHECK-NEXT:    orps {{.*}}(%rip), %xmm0
 ; CHECK-NEXT:    retq
-;
   %y = call double @copysign(double -1.0, double %x)
   ret double %y
 }
@@ -78,9 +75,8 @@ define double @mag_neg1_double(double %x) nounwind {
 define float @mag_pos0_float(float %x) nounwind {
 ; CHECK-LABEL: mag_pos0_float:
 ; CHECK:       ## %bb.0:
-; CHECK-NEXT:    andps [[SIGNMASK5]](%rip), %xmm0
+; CHECK-NEXT:    andps {{.*}}(%rip), %xmm0
 ; CHECK-NEXT:    retq
-;
   %y = call float @copysignf(float 0.0, float %x)
   ret float %y
 }
@@ -91,10 +87,9 @@ define float @mag_pos0_float(float %x) nounwind {
 define float @mag_neg0_float(float %x) nounwind {
 ; CHECK-LABEL: mag_neg0_float:
 ; CHECK:       ## %bb.0:
-; CHECK-NEXT:    movss [[SIGNMASK6]](%rip), %xmm1
+; CHECK-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; CHECK-NEXT:    andps %xmm1, %xmm0
 ; CHECK-NEXT:    retq
-;
   %y = call float @copysignf(float -0.0, float %x)
   ret float %y
 }
@@ -110,11 +105,10 @@ define float @mag_neg0_float(float %x) nounwind {
 define float @mag_pos1_float(float %x) nounwind {
 ; CHECK-LABEL: mag_pos1_float:
 ; CHECK:       ## %bb.0:
-; CHECK-NEXT:    andps [[SIGNMASK7]](%rip), %xmm0
-; CHECK-NEXT:    movss [[ONE7]](%rip), %xmm1
+; CHECK-NEXT:    andps {{.*}}(%rip), %xmm0
+; CHECK-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; CHECK-NEXT:    orps %xmm1, %xmm0
 ; CHECK-NEXT:    retq
-;
   %y = call float @copysignf(float 1.0, float %x)
   ret float %y
 }
@@ -133,10 +127,9 @@ define float @mag_pos1_float(float %x) nounwind {
 define float @mag_neg1_float(float %x) nounwind {
 ; CHECK-LABEL: mag_neg1_float:
 ; CHECK:       ## %bb.0:
-; CHECK-NEXT:    andps [[SIGNMASK8]](%rip), %xmm0
-; CHECK-NEXT:    orps [[ONE8]](%rip), %xmm0
+; CHECK-NEXT:    andps {{.*}}(%rip), %xmm0
+; CHECK-NEXT:    orps {{.*}}(%rip), %xmm0
 ; CHECK-NEXT:    retq
-;
   %y = call float @copysignf(float -1.0, float %x)
   ret float %y
 }
diff --git a/llvm/test/CodeGen/X86/cpus-intel.ll b/llvm/test/CodeGen/X86/cpus-intel.ll
index 1177ebcec..e0dd64740 100644
--- a/llvm/test/CodeGen/X86/cpus-intel.ll
+++ b/llvm/test/CodeGen/X86/cpus-intel.ll
@@ -39,6 +39,7 @@
 ; RUN: llc < %s -o /dev/null -mtriple=x86_64-unknown-unknown -mcpu=cannonlake 2>&1 | FileCheck %s --check-prefix=CHECK-NO-ERROR --allow-empty
 ; RUN: llc < %s -o /dev/null -mtriple=x86_64-unknown-unknown -mcpu=icelake-client 2>&1 | FileCheck %s --check-prefix=CHECK-NO-ERROR --allow-empty
 ; RUN: llc < %s -o /dev/null -mtriple=x86_64-unknown-unknown -mcpu=icelake-server 2>&1 | FileCheck %s --check-prefix=CHECK-NO-ERROR --allow-empty
+; RUN: llc < %s -o /dev/null -mtriple=x86_64-unknown-unknown -mcpu=tigerlake 2>&1 | FileCheck %s --check-prefix=CHECK-NO-ERROR --allow-empty
 ; RUN: llc < %s -o /dev/null -mtriple=x86_64-unknown-unknown -mcpu=atom 2>&1 | FileCheck %s --check-prefix=CHECK-NO-ERROR --allow-empty
 ; RUN: llc < %s -o /dev/null -mtriple=x86_64-unknown-unknown -mcpu=bonnell 2>&1 | FileCheck %s --check-prefix=CHECK-NO-ERROR --allow-empty
 ; RUN: llc < %s -o /dev/null -mtriple=x86_64-unknown-unknown -mcpu=silvermont 2>&1 | FileCheck %s --check-prefix=CHECK-NO-ERROR --allow-empty
diff --git a/llvm/test/CodeGen/X86/crash-lre-eliminate-dead-def.ll b/llvm/test/CodeGen/X86/crash-lre-eliminate-dead-def.ll
index b4c30a738..c9c511f93 100644
--- a/llvm/test/CodeGen/X86/crash-lre-eliminate-dead-def.ll
+++ b/llvm/test/CodeGen/X86/crash-lre-eliminate-dead-def.ll
@@ -264,5 +264,5 @@ unreachable:                                      ; preds = %cleanup100
 ; Function Attrs: nounwind
 declare void @printf(i8* nocapture readonly, ...) #1
 
-attributes #0 = { noreturn nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noreturn nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/X86/csr-split.ll b/llvm/test/CodeGen/X86/csr-split.ll
new file mode 100644
index 000000000..9bd480338
--- /dev/null
+++ b/llvm/test/CodeGen/X86/csr-split.ll
@@ -0,0 +1,230 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -verify-machineinstrs -mtriple=x86_64-unknown-linux < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -mtriple=i386-unknown-linux < %s | FileCheck %s --check-prefix=CHECK-32BIT
+
+; Check CSR split can work properly for tests below.
+
+@a = common dso_local local_unnamed_addr global i32 0, align 4
+
+define dso_local signext i32 @test1(i32* %b) local_unnamed_addr  {
+; CHECK-LABEL: test1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset %rbx, -16
+; CHECK-NEXT:    movslq {{.*}}(%rip), %rax
+; CHECK-NEXT:    cmpq %rdi, %rax
+; CHECK-NEXT:    je .LBB0_2
+; CHECK-NEXT:  # %bb.1: # %if.end
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  .LBB0_2: # %if.then
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    movq %rdi, %rbx
+; CHECK-NEXT:    callq callVoid
+; CHECK-NEXT:    movq %rbx, %rdi
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    jmp callNonVoid # TAILCALL
+;
+; CHECK-32BIT-LABEL: test1:
+; CHECK-32BIT:       # %bb.0: # %entry
+; CHECK-32BIT-NEXT:    subl $12, %esp
+; CHECK-32BIT-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-32BIT-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-32BIT-NEXT:    cmpl %eax, a
+; CHECK-32BIT-NEXT:    je .LBB0_2
+; CHECK-32BIT-NEXT:  # %bb.1: # %if.end
+; CHECK-32BIT-NEXT:    addl $12, %esp
+; CHECK-32BIT-NEXT:    .cfi_def_cfa_offset 4
+; CHECK-32BIT-NEXT:    retl
+; CHECK-32BIT-NEXT:  .LBB0_2: # %if.then
+; CHECK-32BIT-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-32BIT-NEXT:    calll callVoid
+; CHECK-32BIT-NEXT:    addl $12, %esp
+; CHECK-32BIT-NEXT:    .cfi_def_cfa_offset 4
+; CHECK-32BIT-NEXT:    jmp callNonVoid # TAILCALL
+entry:
+  %0 = load i32, i32* @a, align 4, !tbaa !2
+  %conv = sext i32 %0 to i64
+  %1 = inttoptr i64 %conv to i32*
+  %cmp = icmp eq i32* %1, %b
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %call = tail call signext i32 bitcast (i32 (...)* @callVoid to i32 ()*)()
+  %call2 = tail call signext i32 @callNonVoid(i32* %b)
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %entry
+  %retval.0 = phi i32 [ %call2, %if.then ], [ undef, %entry ]
+  ret i32 %retval.0
+}
+
+declare signext i32 @callVoid(...) local_unnamed_addr
+
+declare signext i32 @callNonVoid(i32*) local_unnamed_addr
+
+define dso_local signext i32 @test2(i32* %p1) local_unnamed_addr  {
+; CHECK-LABEL: test2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset %rbx, -16
+; CHECK-NEXT:    testq %rdi, %rdi
+; CHECK-NEXT:    je .LBB1_2
+; CHECK-NEXT:  # %bb.1: # %if.end
+; CHECK-NEXT:    movq %rdi, %rbx
+; CHECK-NEXT:    movslq {{.*}}(%rip), %rax
+; CHECK-NEXT:    cmpq %rdi, %rax
+; CHECK-NEXT:    je .LBB1_3
+; CHECK-NEXT:  .LBB1_2: # %return
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  .LBB1_3: # %if.then2
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    callq callVoid
+; CHECK-NEXT:    movq %rbx, %rdi
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    jmp callNonVoid # TAILCALL
+;
+; CHECK-32BIT-LABEL: test2:
+; CHECK-32BIT:       # %bb.0: # %entry
+; CHECK-32BIT-NEXT:    subl $12, %esp
+; CHECK-32BIT-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-32BIT-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-32BIT-NEXT:    testl %eax, %eax
+; CHECK-32BIT-NEXT:    je .LBB1_2
+; CHECK-32BIT-NEXT:  # %bb.1: # %if.end
+; CHECK-32BIT-NEXT:    cmpl %eax, a
+; CHECK-32BIT-NEXT:    je .LBB1_3
+; CHECK-32BIT-NEXT:  .LBB1_2: # %return
+; CHECK-32BIT-NEXT:    xorl %eax, %eax
+; CHECK-32BIT-NEXT:    addl $12, %esp
+; CHECK-32BIT-NEXT:    .cfi_def_cfa_offset 4
+; CHECK-32BIT-NEXT:    retl
+; CHECK-32BIT-NEXT:  .LBB1_3: # %if.then2
+; CHECK-32BIT-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-32BIT-NEXT:    calll callVoid
+; CHECK-32BIT-NEXT:    addl $12, %esp
+; CHECK-32BIT-NEXT:    .cfi_def_cfa_offset 4
+; CHECK-32BIT-NEXT:    jmp callNonVoid # TAILCALL
+entry:
+  %tobool = icmp eq i32* %p1, null
+  br i1 %tobool, label %return, label %if.end
+
+if.end:                                           ; preds = %entry
+  %0 = load i32, i32* @a, align 4, !tbaa !2
+  %conv = sext i32 %0 to i64
+  %1 = inttoptr i64 %conv to i32*
+  %cmp = icmp eq i32* %1, %p1
+  br i1 %cmp, label %if.then2, label %return
+
+if.then2:                                         ; preds = %if.end
+  %call = tail call signext i32 bitcast (i32 (...)* @callVoid to i32 ()*)()
+  %call3 = tail call signext i32 @callNonVoid(i32* nonnull %p1)
+  br label %return
+
+return:                                           ; preds = %if.end, %entry, %if.then2
+  %retval.0 = phi i32 [ %call3, %if.then2 ], [ 0, %entry ], [ 0, %if.end ]
+  ret i32 %retval.0
+}
+
+
+define dso_local i8* @test3(i8** nocapture %p1, i8 zeroext %p2) local_unnamed_addr  {
+; CHECK-LABEL: test3:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    .cfi_offset %rbx, -24
+; CHECK-NEXT:    .cfi_offset %r14, -16
+; CHECK-NEXT:    movq (%rdi), %rbx
+; CHECK-NEXT:    testq %rbx, %rbx
+; CHECK-NEXT:    je .LBB2_2
+; CHECK-NEXT:  # %bb.1: # %land.rhs
+; CHECK-NEXT:    movq %rdi, %r14
+; CHECK-NEXT:    movzbl %sil, %esi
+; CHECK-NEXT:    movq %rbx, %rdi
+; CHECK-NEXT:    callq bar
+; CHECK-NEXT:    movq %rax, (%r14)
+; CHECK-NEXT:  .LBB2_2: # %land.end
+; CHECK-NEXT:    movq %rbx, %rax
+; CHECK-NEXT:    addq $8, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
+;
+; CHECK-32BIT-LABEL: test3:
+; CHECK-32BIT:       # %bb.0: # %entry
+; CHECK-32BIT-NEXT:    pushl %edi
+; CHECK-32BIT-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-32BIT-NEXT:    pushl %esi
+; CHECK-32BIT-NEXT:    .cfi_def_cfa_offset 12
+; CHECK-32BIT-NEXT:    pushl %eax
+; CHECK-32BIT-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-32BIT-NEXT:    .cfi_offset %esi, -12
+; CHECK-32BIT-NEXT:    .cfi_offset %edi, -8
+; CHECK-32BIT-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; CHECK-32BIT-NEXT:    movl (%edi), %esi
+; CHECK-32BIT-NEXT:    testl %esi, %esi
+; CHECK-32BIT-NEXT:    je .LBB2_2
+; CHECK-32BIT-NEXT:  # %bb.1: # %land.rhs
+; CHECK-32BIT-NEXT:    subl $8, %esp
+; CHECK-32BIT-NEXT:    .cfi_adjust_cfa_offset 8
+; CHECK-32BIT-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-32BIT-NEXT:    pushl %eax
+; CHECK-32BIT-NEXT:    .cfi_adjust_cfa_offset 4
+; CHECK-32BIT-NEXT:    pushl %esi
+; CHECK-32BIT-NEXT:    .cfi_adjust_cfa_offset 4
+; CHECK-32BIT-NEXT:    calll bar
+; CHECK-32BIT-NEXT:    addl $16, %esp
+; CHECK-32BIT-NEXT:    .cfi_adjust_cfa_offset -16
+; CHECK-32BIT-NEXT:    movl %eax, (%edi)
+; CHECK-32BIT-NEXT:  .LBB2_2: # %land.end
+; CHECK-32BIT-NEXT:    movl %esi, %eax
+; CHECK-32BIT-NEXT:    addl $4, %esp
+; CHECK-32BIT-NEXT:    .cfi_def_cfa_offset 12
+; CHECK-32BIT-NEXT:    popl %esi
+; CHECK-32BIT-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-32BIT-NEXT:    popl %edi
+; CHECK-32BIT-NEXT:    .cfi_def_cfa_offset 4
+; CHECK-32BIT-NEXT:    retl
+entry:
+  %0 = load i8*, i8** %p1, align 8, !tbaa !6
+  %tobool = icmp eq i8* %0, null
+  br i1 %tobool, label %land.end, label %land.rhs
+
+land.rhs:                                         ; preds = %entry
+  %call = tail call i8* @bar(i8* nonnull %0, i8 zeroext %p2)
+  store i8* %call, i8** %p1, align 8, !tbaa !6
+  br label %land.end
+
+land.end:                                         ; preds = %entry, %land.rhs
+  ret i8* %0
+}
+
+declare i8* @bar(i8*, i8 zeroext) local_unnamed_addr
+
+
+!llvm.module.flags = !{!0}
+!llvm.ident = !{!1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{!"clang version 10.0.0 (trunk 367381) (llvm/trunk 367388)"}
+!2 = !{!3, !3, i64 0}
+!3 = !{!"int", !4, i64 0}
+!4 = !{!"omnipotent char", !5, i64 0}
+!5 = !{!"Simple C/C++ TBAA"}
+!6 = !{!7, !7, i64 0}
+!7 = !{!"any pointer", !4, i64 0}
diff --git a/llvm/test/CodeGen/X86/cstring.ll b/llvm/test/CodeGen/X86/cstring.ll
index 5b5a7662f..dd00d58dc 100644
--- a/llvm/test/CodeGen/X86/cstring.ll
+++ b/llvm/test/CodeGen/X86/cstring.ll
@@ -1,4 +1,5 @@
-; RUN: llc < %s -mtriple=i386-apple-darwin | not grep comm
+; RUN: llc < %s -mtriple=i386-apple-darwin | FileCheck %s
 ; rdar://6479858
 
+; CHECK-NOT: comm
 @str1 = internal constant [1 x i8] zeroinitializer
diff --git a/llvm/test/CodeGen/X86/ctpop-combine.ll b/llvm/test/CodeGen/X86/ctpop-combine.ll
index cdef5771f..e60935cc8 100644
--- a/llvm/test/CodeGen/X86/ctpop-combine.ll
+++ b/llvm/test/CodeGen/X86/ctpop-combine.ll
@@ -116,7 +116,7 @@ define i32 @ctpop_eq_one(i64 %x) nounwind readnone {
 ; POPCOUNT:       # %bb.0:
 ; POPCOUNT-NEXT:    popcntq %rdi, %rcx
 ; POPCOUNT-NEXT:    xorl %eax, %eax
-; POPCOUNT-NEXT:    cmpq $1, %rcx
+; POPCOUNT-NEXT:    cmpl $1, %ecx
 ; POPCOUNT-NEXT:    sete %al
 ; POPCOUNT-NEXT:    retq
 ;
@@ -141,7 +141,7 @@ define i32 @ctpop_ne_one(i64 %x) nounwind readnone {
 ; POPCOUNT:       # %bb.0:
 ; POPCOUNT-NEXT:    popcntq %rdi, %rcx
 ; POPCOUNT-NEXT:    xorl %eax, %eax
-; POPCOUNT-NEXT:    cmpq $1, %rcx
+; POPCOUNT-NEXT:    cmpl $1, %ecx
 ; POPCOUNT-NEXT:    setne %al
 ; POPCOUNT-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/cvtv2f32.ll b/llvm/test/CodeGen/X86/cvtv2f32.ll
index 8b39a5d53..a44f16509 100644
--- a/llvm/test/CodeGen/X86/cvtv2f32.ll
+++ b/llvm/test/CodeGen/X86/cvtv2f32.ll
@@ -42,11 +42,9 @@ define <2 x float> @uitofp_2i32_cvt_buildvector(i32 %x, i32 %y, <2 x float> %v)
 define <2 x float> @uitofp_2i32_buildvector_cvt(i32 %x, i32 %y, <2 x float> %v) {
 ; X32-LABEL: uitofp_2i32_buildvector_cvt:
 ; X32:       # %bb.0:
-; X32-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; X32-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; X32-NEXT:    unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm1[0]
-; X32-NEXT:    movapd {{.*#+}} xmm1 = [4.503599627370496E+15,4.503599627370496E+15]
-; X32-NEXT:    orpd %xmm1, %xmm2
+; X32-NEXT:    movdqa {{.*#+}} xmm1 = [4.503599627370496E+15,4.503599627370496E+15]
+; X32-NEXT:    pmovzxdq {{.*#+}} xmm2 = mem[0],zero,mem[1],zero
+; X32-NEXT:    por %xmm1, %xmm2
 ; X32-NEXT:    subpd %xmm1, %xmm2
 ; X32-NEXT:    cvtpd2ps %xmm2, %xmm1
 ; X32-NEXT:    mulps %xmm1, %xmm0
@@ -54,13 +52,13 @@ define <2 x float> @uitofp_2i32_buildvector_cvt(i32 %x, i32 %y, <2 x float> %v)
 ;
 ; X64-LABEL: uitofp_2i32_buildvector_cvt:
 ; X64:       # %bb.0:
-; X64-NEXT:    movd %esi, %xmm1
-; X64-NEXT:    movd %edi, %xmm2
-; X64-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm1[0]
-; X64-NEXT:    movdqa {{.*#+}} xmm1 = [4.503599627370496E+15,4.503599627370496E+15]
-; X64-NEXT:    por %xmm1, %xmm2
-; X64-NEXT:    subpd %xmm1, %xmm2
-; X64-NEXT:    cvtpd2ps %xmm2, %xmm1
+; X64-NEXT:    movd %edi, %xmm1
+; X64-NEXT:    pinsrd $1, %esi, %xmm1
+; X64-NEXT:    pmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
+; X64-NEXT:    movdqa {{.*#+}} xmm2 = [4.503599627370496E+15,4.503599627370496E+15]
+; X64-NEXT:    por %xmm2, %xmm1
+; X64-NEXT:    subpd %xmm2, %xmm1
+; X64-NEXT:    cvtpd2ps %xmm1, %xmm1
 ; X64-NEXT:    mulps %xmm1, %xmm0
 ; X64-NEXT:    retq
   %t1 = insertelement <2 x i32> undef, i32 %x, i32 0
@@ -73,23 +71,21 @@ define <2 x float> @uitofp_2i32_buildvector_cvt(i32 %x, i32 %y, <2 x float> %v)
 define <2 x float> @uitofp_2i32_legalized(<2 x i32> %in, <2 x float> %v) {
 ; X32-LABEL: uitofp_2i32_legalized:
 ; X32:       # %bb.0:
-; X32-NEXT:    xorps %xmm2, %xmm2
-; X32-NEXT:    blendps {{.*#+}} xmm2 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
-; X32-NEXT:    movaps {{.*#+}} xmm0 = [4.503599627370496E+15,4.503599627370496E+15]
-; X32-NEXT:    orps %xmm0, %xmm2
-; X32-NEXT:    subpd %xmm0, %xmm2
-; X32-NEXT:    cvtpd2ps %xmm2, %xmm0
+; X32-NEXT:    pmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
+; X32-NEXT:    movdqa {{.*#+}} xmm2 = [4.503599627370496E+15,4.503599627370496E+15]
+; X32-NEXT:    por %xmm2, %xmm0
+; X32-NEXT:    subpd %xmm2, %xmm0
+; X32-NEXT:    cvtpd2ps %xmm0, %xmm0
 ; X32-NEXT:    mulps %xmm1, %xmm0
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: uitofp_2i32_legalized:
 ; X64:       # %bb.0:
-; X64-NEXT:    xorps %xmm2, %xmm2
-; X64-NEXT:    blendps {{.*#+}} xmm2 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
-; X64-NEXT:    movaps {{.*#+}} xmm0 = [4.503599627370496E+15,4.503599627370496E+15]
-; X64-NEXT:    orps %xmm0, %xmm2
-; X64-NEXT:    subpd %xmm0, %xmm2
-; X64-NEXT:    cvtpd2ps %xmm2, %xmm0
+; X64-NEXT:    pmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
+; X64-NEXT:    movdqa {{.*#+}} xmm2 = [4.503599627370496E+15,4.503599627370496E+15]
+; X64-NEXT:    por %xmm2, %xmm0
+; X64-NEXT:    subpd %xmm2, %xmm0
+; X64-NEXT:    cvtpd2ps %xmm0, %xmm0
 ; X64-NEXT:    mulps %xmm1, %xmm0
 ; X64-NEXT:    retq
   %t1 = uitofp <2 x i32> %in to <2 x float>
diff --git a/llvm/test/CodeGen/X86/cxx_tlscc64.ll b/llvm/test/CodeGen/X86/cxx_tlscc64.ll
index ef947367c..b67344283 100644
--- a/llvm/test/CodeGen/X86/cxx_tlscc64.ll
+++ b/llvm/test/CodeGen/X86/cxx_tlscc64.ll
@@ -166,6 +166,6 @@ entry:
 define cxx_fast_tlscc nonnull i8* @test_ssp() #2 {
   ret i8* @ssp_var
 }
-attributes #0 = { nounwind "no-frame-pointer-elim"="true" }
+attributes #0 = { nounwind "frame-pointer"="all" }
 attributes #1 = { nounwind }
 attributes #2 = { nounwind sspreq }
diff --git a/llvm/test/CodeGen/X86/dagcombine-unsafe-math.ll b/llvm/test/CodeGen/X86/dagcombine-unsafe-math.ll
index 62e182b99..c2b6240c6 100644
--- a/llvm/test/CodeGen/X86/dagcombine-unsafe-math.ll
+++ b/llvm/test/CodeGen/X86/dagcombine-unsafe-math.ll
@@ -1,5 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -enable-unsafe-fp-math -mtriple=x86_64-apple-darwin -mcpu=corei7-avx | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=corei7-avx | FileCheck %s
 
 
 ; rdar://13126763
@@ -62,9 +61,9 @@ define float @test5(<4 x float> %x) {
   %splat = shufflevector <4 x float> %x, <4 x float> undef, <4 x i32> zeroinitializer
   %v1 = extractelement <4 x float> %splat, i32 1
   %v0 = extractelement <4 x float> %splat, i32 0
-  %add1 = fadd float %v0, %v1
+  %add1 = fadd reassoc nsz float %v0, %v1
   %v2 = extractelement <4 x float> %splat, i32 2
-  %add2 = fadd float %v2, %add1
+  %add2 = fadd reassoc nsz float %v2, %add1
   ret float %add2
 }
 
diff --git a/llvm/test/CodeGen/X86/dbg-changes-codegen-branch-folding.ll b/llvm/test/CodeGen/X86/dbg-changes-codegen-branch-folding.ll
index d3116f194..5a1a7f9a1 100644
--- a/llvm/test/CodeGen/X86/dbg-changes-codegen-branch-folding.ll
+++ b/llvm/test/CodeGen/X86/dbg-changes-codegen-branch-folding.ll
@@ -114,9 +114,9 @@ declare void @_Z3fooPcjPKc(i8*, i32, i8*) #2
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.value(metadata, i64, metadata, metadata) #3
 
-attributes #0 = { uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { argmemonly nounwind }
-attributes #2 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #3 = { nounwind readnone }
 attributes #4 = { nounwind }
 
diff --git a/llvm/test/CodeGen/X86/dbg-changes-codegen.ll b/llvm/test/CodeGen/X86/dbg-changes-codegen.ll
index 3d01e07d7..c16dce12d 100644
--- a/llvm/test/CodeGen/X86/dbg-changes-codegen.ll
+++ b/llvm/test/CodeGen/X86/dbg-changes-codegen.ll
@@ -71,8 +71,8 @@ _ZN7Flibble3barEP6Wibble.exit:                    ; preds = %entry, %if.then.i
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.value(metadata, i64, metadata, metadata) #2
 
-attributes #0 = { nounwind readonly uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind readonly uwtable "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind readnone }
 
 !1 = distinct !DISubprogram()
diff --git a/llvm/test/CodeGen/X86/dbg-combine.ll b/llvm/test/CodeGen/X86/dbg-combine.ll
index 5e67ad718..c2a7a587f 100644
--- a/llvm/test/CodeGen/X86/dbg-combine.ll
+++ b/llvm/test/CodeGen/X86/dbg-combine.ll
@@ -66,7 +66,7 @@ declare i8* @llvm.stacksave() #2
 ; Function Attrs: nounwind
 declare void @llvm.stackrestore(i8*) #2
 
-attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 attributes #2 = { nounwind }
 
diff --git a/llvm/test/CodeGen/X86/dbg-value-func-arg.ll b/llvm/test/CodeGen/X86/dbg-value-func-arg.ll
new file mode 100644
index 000000000..cdaf637a9
--- /dev/null
+++ b/llvm/test/CodeGen/X86/dbg-value-func-arg.ll
@@ -0,0 +1,86 @@
+; RUN: llc < %s -mtriple i386-unknown-unknown -O1 -stop-before=finalize-isel | FileCheck %s
+
+; Test case is designed to show the differences related to how implicit values
+; are handled by SelectionDAGBuilder::EmitFuncArgumentDbgValue.
+;
+; One purpose is to validate a bugfix in DIExpression::isImplicit() related
+; to DIExpressions containing both DW_OP_stack_value and DW_OP_LLVM_fragment.
+;
+; IR generated by:
+;
+;   // clang -S -target i386-- -Wall -g -O1 -mllvm -stop-before=finalize-isel
+;
+;   typedef struct {
+;       int x;
+;       int *y;
+;   } bar;
+;
+;   bar* func1(bar *p1, long p2, bar p3)
+;   {
+;      bar *foo1 = p1+18;
+;      p3.y = &((p1+18)->x);
+;      return p1;
+;   }
+
+
+target datalayout = "e-m:e-p:32:32-p270:32:32-p271:32:32-p272:64:64-f64:32:64-f80:32-n8:16:32-S128"
+target triple = "i386-unknown-unknown"
+
+%struct.bar = type { i32, i32* }
+
+; Function Attrs: norecurse nounwind readnone
+define dso_local %struct.bar* @func1(%struct.bar* readnone returned %0, i32 %1, i32 %2, i32* nocapture readnone %3) local_unnamed_addr #0 !dbg !8 {
+; CHECK-DAG: DBG_VALUE %fixed-stack.1, 0, {{.*}}, !DIExpression(DW_OP_LLVM_fragment, 0, 32),
+; CHECK-DAG: DBG_VALUE %fixed-stack.0, 0, {{.*}}, !DIExpression(DW_OP_LLVM_fragment, 32, 32),
+; CHECK-DAG: DBG_VALUE %fixed-stack.3, 0, {{.*}}, !DIExpression(),
+; CHECK-DAG: DBG_VALUE %fixed-stack.2, 0, {{.*}}, !DIExpression(),
+; CHECK-DAG: DBG_VALUE %fixed-stack.3, 0, {{.*}}, !DIExpression(DW_OP_plus_uconst, 144, DW_OP_stack_value),
+; CHECK-DAG: DBG_VALUE %fixed-stack.3, 0, {{.*}}, !DIExpression(DW_OP_plus_uconst, 144, DW_OP_stack_value, DW_OP_LLVM_fragment, 32, 32),
+
+  call void @llvm.dbg.value(metadata i32 %2, metadata !24, metadata !DIExpression(DW_OP_LLVM_fragment, 0, 32)), !dbg !26
+  call void @llvm.dbg.value(metadata i32* %3, metadata !24, metadata !DIExpression(DW_OP_LLVM_fragment, 32, 32)), !dbg !26
+  call void @llvm.dbg.value(metadata %struct.bar* %0, metadata !22, metadata !DIExpression()), !dbg !26
+  call void @llvm.dbg.value(metadata i32 %1, metadata !23, metadata !DIExpression()), !dbg !26
+  call void @llvm.dbg.value(metadata %struct.bar* %0, metadata !25, metadata !DIExpression(DW_OP_plus_uconst, 144, DW_OP_stack_value)), !dbg !26
+  call void @llvm.dbg.value(metadata %struct.bar* %0, metadata !24, metadata !DIExpression(DW_OP_plus_uconst, 144, DW_OP_stack_value, DW_OP_LLVM_fragment, 32, 32)), !dbg !26
+  ret %struct.bar* %0, !dbg !27
+}
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.value(metadata, metadata, metadata) #1
+
+attributes #0 = { norecurse nounwind readnone }
+attributes #1 = { nounwind readnone speculatable willreturn }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5, !6}
+!llvm.ident = !{!7}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 10.0.0 (trunk 375507)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "example.c", directory: "/home/ubuntu")
+!2 = !{}
+!3 = !{i32 1, !"NumRegisterParameters", i32 0}
+!4 = !{i32 2, !"Dwarf Version", i32 4}
+!5 = !{i32 2, !"Debug Info Version", i32 3}
+!6 = !{i32 1, !"wchar_size", i32 4}
+!7 = !{!"clang version 10.0.0 (trunk 375507)"}
+!8 = distinct !DISubprogram(name: "func1", scope: !9, file: !9, line: 6, type: !10, scopeLine: 7, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !21)
+!9 = !DIFile(filename: "./example.c", directory: "/home/ubuntu")
+!10 = !DISubroutineType(types: !11)
+!11 = !{!12, !12, !20, !13}
+!12 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !13, size: 32)
+!13 = !DIDerivedType(tag: DW_TAG_typedef, name: "bar", file: !9, line: 4, baseType: !14)
+!14 = distinct !DICompositeType(tag: DW_TAG_structure_type, file: !9, line: 1, size: 64, elements: !15)
+!15 = !{!16, !18}
+!16 = !DIDerivedType(tag: DW_TAG_member, name: "x", scope: !14, file: !9, line: 2, baseType: !17, size: 32)
+!17 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!18 = !DIDerivedType(tag: DW_TAG_member, name: "y", scope: !14, file: !9, line: 3, baseType: !19, size: 32, offset: 32)
+!19 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !17, size: 32)
+!20 = !DIBasicType(name: "long int", size: 32, encoding: DW_ATE_signed)
+!21 = !{!22, !23, !24, !25}
+!22 = !DILocalVariable(name: "p1", arg: 1, scope: !8, file: !9, line: 6, type: !12)
+!23 = !DILocalVariable(name: "p2", arg: 2, scope: !8, file: !9, line: 6, type: !20)
+!24 = !DILocalVariable(name: "p3", arg: 3, scope: !8, file: !9, line: 6, type: !13)
+!25 = !DILocalVariable(name: "foo1", scope: !8, file: !9, line: 8, type: !12)
+!26 = !DILocation(line: 0, scope: !8)
+!27 = !DILocation(line: 10, column: 4, scope: !8)
diff --git a/llvm/test/CodeGen/X86/debug-loclists-lto.ll b/llvm/test/CodeGen/X86/debug-loclists-lto.ll
new file mode 100644
index 000000000..7578e09c8
--- /dev/null
+++ b/llvm/test/CodeGen/X86/debug-loclists-lto.ll
@@ -0,0 +1,66 @@
+; RUN: llc -mtriple=x86_64-pc-linux -filetype=asm -function-sections < %s | FileCheck --implicit-check-not=loclists_table_base %s
+
+; CHECK: {{^}}.Lloclists_table_base0:
+; CHECK-NEXT: .long   .Ldebug_loc0-.Lloclists_table_base0
+; CHECK-NEXT: .long   .Ldebug_loc1-.Lloclists_table_base0
+; CHECK: .long   .Lloclists_table_base0  # DW_AT_loclists_base
+; CHECK: .long   .Lloclists_table_base0  # DW_AT_loclists_base
+
+; Function Attrs: uwtable
+define dso_local void @_Z2f2v() local_unnamed_addr #0 !dbg !15 {
+entry:
+  tail call void @_Z2f1v(), !dbg !19
+  call void @llvm.dbg.value(metadata i32 3, metadata !17, metadata !DIExpression()), !dbg !20
+  tail call void @_Z2f1v(), !dbg !21
+  ret void, !dbg !22
+}
+declare !dbg !4 dso_local void @_Z2f1v() local_unnamed_addr #1
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.value(metadata, metadata, metadata) #2
+; Function Attrs: uwtable
+define dso_local void @_Z2f3v() local_unnamed_addr #0 !dbg !23 {
+entry:
+  tail call void @_Z2f1v(), !dbg !26
+  call void @llvm.dbg.value(metadata i32 3, metadata !25, metadata !DIExpression()), !dbg !27
+  tail call void @_Z2f1v(), !dbg !28
+  ret void, !dbg !29
+}
+
+attributes #0 = { uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { nounwind readnone speculatable willreturn }
+
+!llvm.dbg.cu = !{!0, !7}
+!llvm.ident = !{!11, !11}
+!llvm.module.flags = !{!12, !13, !14}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 10.0.0 (git@github.com:llvm/llvm-project.git 9b962d83ece841e43fd2823375dc6ddc94c1b178)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !3, nameTableKind: None)
+!1 = !DIFile(filename: "loc1.cpp", directory: "/usr/local/google/home/blaikie/dev/scratch", checksumkind: CSK_MD5, checksum: "3c96069dc8a3a1e7868038213ed0364a")
+!2 = !{}
+!3 = !{!4}
+!4 = !DISubprogram(name: "f1", linkageName: "_Z2f1v", scope: !1, file: !1, line: 1, type: !5, flags: DIFlagPrototyped, spFlags: DISPFlagOptimized, retainedNodes: !2)
+!5 = !DISubroutineType(types: !6)
+!6 = !{null}
+!7 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !8, producer: "clang version 10.0.0 (git@github.com:llvm/llvm-project.git 9b962d83ece841e43fd2823375dc6ddc94c1b178)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !9, nameTableKind: None)
+!8 = !DIFile(filename: "loc2.cpp", directory: "/usr/local/google/home/blaikie/dev/scratch", checksumkind: CSK_MD5, checksum: "2d309df0c6f5d8ce7264cc7696738fa9")
+!9 = !{!10}
+!10 = !DISubprogram(name: "f1", linkageName: "_Z2f1v", scope: !8, file: !8, line: 1, type: !5, flags: DIFlagPrototyped, spFlags: DISPFlagOptimized, retainedNodes: !2)
+!11 = !{!"clang version 10.0.0 (git@github.com:llvm/llvm-project.git 9b962d83ece841e43fd2823375dc6ddc94c1b178)"}
+!12 = !{i32 7, !"Dwarf Version", i32 5}
+!13 = !{i32 2, !"Debug Info Version", i32 3}
+!14 = !{i32 1, !"wchar_size", i32 4}
+!15 = distinct !DISubprogram(name: "f2", linkageName: "_Z2f2v", scope: !1, file: !1, line: 2, type: !5, scopeLine: 2, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !16)
+!16 = !{!17}
+!17 = !DILocalVariable(name: "i", scope: !15, file: !1, line: 3, type: !18)
+!18 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!19 = !DILocation(line: 4, column: 3, scope: !15)
+!20 = !DILocation(line: 0, scope: !15)
+!21 = !DILocation(line: 6, column: 3, scope: !15)
+!22 = !DILocation(line: 7, column: 1, scope: !15)
+!23 = distinct !DISubprogram(name: "f3", linkageName: "_Z2f3v", scope: !8, file: !8, line: 2, type: !5, scopeLine: 2, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !7, retainedNodes: !24)
+!24 = !{!25}
+!25 = !DILocalVariable(name: "i", scope: !23, file: !8, line: 3, type: !18)
+!26 = !DILocation(line: 4, column: 3, scope: !23)
+!27 = !DILocation(line: 0, scope: !23)
+!28 = !DILocation(line: 6, column: 3, scope: !23)
+!29 = !DILocation(line: 7, column: 1, scope: !23)
diff --git a/llvm/test/CodeGen/X86/debug-loclists.ll b/llvm/test/CodeGen/X86/debug-loclists.ll
index 0c2ab3dfa..10cd877bf 100644
--- a/llvm/test/CodeGen/X86/debug-loclists.ll
+++ b/llvm/test/CodeGen/X86/debug-loclists.ll
@@ -1,142 +1,124 @@
-; RUN: llc -mtriple=x86_64-pc-linux -filetype=obj -o %t < %s
-; RUN: llvm-dwarfdump -v %t | FileCheck %s
-
-; CHECK:      0x00000033: DW_TAG_formal_parameter [3]
-; CHECK-NEXT:               DW_AT_location [DW_FORM_sec_offset]   (0x0000000c
-; CHECK-NEXT:                  [0x0000000000000000, 0x0000000000000004): DW_OP_breg5 RDI+0
-; CHECK-NEXT:                  [0x0000000000000004, 0x0000000000000012): DW_OP_breg3 RBX+0)
-; CHECK-NEXT:               DW_AT_name [DW_FORM_strx1]    (indexed (0000000e) string = "a")
-; CHECK-NEXT:               DW_AT_decl_file [DW_FORM_data1]       ("/home/folder{{\\|\/}}test.cc")
-; CHECK-NEXT:               DW_AT_decl_line [DW_FORM_data1]       (6)
-; CHECK-NEXT:               DW_AT_type [DW_FORM_ref4]     (cu + 0x0040 => {0x00000040} "A")
+; RUN: llc -mtriple=x86_64-pc-linux -filetype=obj -function-sections -o %t < %s
+; RUN: llvm-dwarfdump -v -debug-info -debug-loclists %t | FileCheck %s
+
+; RUN: llc -dwarf-version=5 -split-dwarf-file=foo.dwo -mtriple=x86_64-pc-linux -filetype=obj -function-sections -o %t < %s
+; RUN: llvm-dwarfdump -v -debug-info -debug-loclists %t | FileCheck %s --check-prefix=DWO
+
+; CHECK:      DW_TAG_variable
+; CHECK-NEXT:   DW_AT_location [DW_FORM_loclistx]   (indexed (0x0) loclist = 0x00000018:
+; CHECK-NEXT:     [0x0000000000000000, 0x0000000000000003) ".text._Z2f1ii": DW_OP_consts +3, DW_OP_stack_value
+; CHECK-NEXT:     [0x0000000000000003, 0x0000000000000004) ".text._Z2f1ii": DW_OP_consts +4, DW_OP_stack_value)
+; CHECK-NEXT:   DW_AT_name {{.*}} "y"
+
+; CHECK:      DW_TAG_variable
+; CHECK-NEXT:   DW_AT_location [DW_FORM_loclistx]   (indexed (0x1) loclist = 0x00000029:
+; CHECK-NEXT:     [0x0000000000000000, 0x0000000000000003) ".text._Z2f1ii": DW_OP_consts +5, DW_OP_stack_value)
+; CHECK-NEXT:   DW_AT_name {{.*}} "x"
+
+; CHECK:      DW_TAG_variable
+; CHECK-NEXT:   DW_AT_location [DW_FORM_loclistx]   (indexed (0x2) loclist = 0x00000031:
+; CHECK-NEXT:     [0x0000000000000003, 0x0000000000000004) ".text._Z2f1ii": DW_OP_reg0 RAX)
+; CHECK-NEXT:   DW_AT_name {{.*}} "r"
 
 ; CHECK:      .debug_loclists contents:
-; CHECK-NEXT: 0x00000000: locations list header: length = 0x00000015, version = 0x0005, addr_size = 0x08, seg_size = 0x00, offset_entry_count = 0x00000000
-; CHECK-NEXT: 0x00000000:
-; CHECK-NEXT:  [0x0000000000000000, 0x0000000000000004): DW_OP_breg5 RDI+0
-; CHECK-NEXT:  [0x0000000000000004, 0x0000000000000012): DW_OP_breg3 RBX+0
-
-; There is no way to use llvm-dwarfdump atm (2018, october) to verify the DW_LLE_* codes emited,
-; because dumper is not yet implements that. Use asm code to do this check instead.
-;
-; RUN: llc -mtriple=x86_64-pc-linux -filetype=asm < %s -o - | FileCheck %s --check-prefix=ASM
-; ASM:      .section .debug_loclists,"",@progbits
-; ASM-NEXT: .long .Ldebug_loclist_table_end0-.Ldebug_loclist_table_start0 # Length
-; ASM-NEXT: .Ldebug_loclist_table_start0:
-; ASM-NEXT:  .short 5                              # Version
-; ASM-NEXT:  .byte 8                               # Address size
-; ASM-NEXT:  .byte 0                               # Segment selector size
-; ASM-NEXT:  .long 0                               # Offset entry count
-; ASM-NEXT: .Lloclists_table_base0:                
-; ASM-NEXT: .Ldebug_loc0:
-; ASM-NEXT:  .byte 4                               # DW_LLE_offset_pair
-; ASM-NEXT:  .uleb128 .Lfunc_begin0-.Lfunc_begin0  # starting offset
-; ASM-NEXT:  .uleb128 .Ltmp0-.Lfunc_begin0         # ending offset
-; ASM-NEXT:  .byte 2                               # Loc expr size
-; ASM-NEXT:  .byte 117                             # DW_OP_breg5
-; ASM-NEXT:  .byte 0                               # 0
-; ASM-NEXT:  .byte 4                               # DW_LLE_offset_pair
-; ASM-NEXT:  .uleb128 .Ltmp0-.Lfunc_begin0         # starting offset
-; ASM-NEXT:  .uleb128 .Ltmp1-.Lfunc_begin0         # ending offset
-; ASM-NEXT:  .byte 2                               # Loc expr size
-; ASM-NEXT:  .byte 115                             # DW_OP_breg3
-; ASM-NEXT:  .byte 0                               # 0
-; ASM-NEXT:  .byte 0                               # DW_LLE_end_of_list
-; ASM-NEXT: .Ldebug_loclist_table_end0:
-
-; ModuleID = 'test.cc'
-source_filename = "test.cc"
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
-target triple = "x86_64-unknown-linux-gnu"
-
-%struct.A = type { i32 (...)** }
-
-@_ZTV1A = dso_local unnamed_addr constant { [4 x i8*] } { [4 x i8*] [i8* null, i8* bitcast ({ i8*, i8* }* @_ZTI1A to i8*), i8* bitcast (void (%struct.A*)* @_ZN1A3fooEv to i8*), i8* bitcast (void (%struct.A*)* @_ZN1A3barEv to i8*)] }, align 8
-@_ZTVN10__cxxabiv117__class_type_infoE = external dso_local global i8*
-@_ZTS1A = dso_local constant [3 x i8] c"1A\00", align 1
-@_ZTI1A = dso_local constant { i8*, i8* } { i8* bitcast (i8** getelementptr inbounds (i8*, i8** @_ZTVN10__cxxabiv117__class_type_infoE, i64 2) to i8*), i8* getelementptr inbounds ([3 x i8], [3 x i8]* @_ZTS1A, i32 0, i32 0) }, align 8
-
-; Function Attrs: noinline optnone uwtable
-define dso_local void @_Z3baz1A(%struct.A* %a) #0 !dbg !7 {
-entry:
-  call void @llvm.dbg.declare(metadata %struct.A* %a, metadata !23, metadata !DIExpression()), !dbg !24
-  call void @_ZN1A3fooEv(%struct.A* %a), !dbg !25
-  call void @_ZN1A3barEv(%struct.A* %a), !dbg !26
-  ret void, !dbg !27
-}
+; CHECK-NEXT: 0x00000000: locations list header: length = 0x00000035, version = 0x0005, addr_size = 0x08, seg_size = 0x00, offset_entry_count = 0x00000003
 
-; Function Attrs: nounwind readnone speculatable
-declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
+; DWO:      .debug_loclists.dwo contents:
+; DWO-NEXT: 0x00000000: locations list header: length = 0x00000035, version = 0x0005, addr_size = 0x08, seg_size = 0x00, offset_entry_count = 0x00000003
 
-; Function Attrs: noinline nounwind optnone uwtable
-define dso_local void @_ZN1A3fooEv(%struct.A* %this) unnamed_addr #2 align 2 !dbg !28 {
-entry:
-  %this.addr = alloca %struct.A*, align 8
-  store %struct.A* %this, %struct.A** %this.addr, align 8
-  call void @llvm.dbg.declare(metadata %struct.A** %this.addr, metadata !29, metadata !DIExpression()), !dbg !31
-  %this1 = load %struct.A*, %struct.A** %this.addr, align 8
-  ret void, !dbg !32
-}
+; CHECK-NEXT: offsets: [
+; CHECK-NEXT: 0x0000000c => 0x00000018
+; CHECK-NEXT: 0x0000001d => 0x00000029
+; CHECK-NEXT: 0x00000025 => 0x00000031
+; CHECK-NEXT: ]
+
+; Don't use startx_length if there's more than one entry, because the shared
+; base address will be useful for both the range that does start at the start of
+; the function, and the one that doesn't.
+
+; CHECK-NEXT: 0x00000018:
+; CHECK-NEXT:             DW_LLE_base_addressx (0x0000000000000000)
+; CHECK-NEXT:             DW_LLE_offset_pair   (0x0000000000000000, 0x0000000000000003): DW_OP_consts +3, DW_OP_stack_value
+; CHECK-NEXT:             DW_LLE_offset_pair   (0x0000000000000003, 0x0000000000000004): DW_OP_consts +4, DW_OP_stack_value
+; CHECK-NEXT:             DW_LLE_end_of_list   ()
+
+; Show that startx_length can be used when the address range starts at the start of the function.
+
+; CHECK:      0x00000029:
+; CHECK-NEXT:             DW_LLE_startx_length (0x0000000000000000, 0x0000000000000003): DW_OP_consts +5, DW_OP_stack_value
+; CHECK-NEXT:             DW_LLE_end_of_list   ()
+
+; And use a base address when the range doesn't start at an existing/useful
+; address in the pool.
+
+; CHECK:      0x00000031:
+; CHECK-NEXT:             DW_LLE_base_addressx (0x0000000000000000)
+; CHECK-NEXT:             DW_LLE_offset_pair   (0x0000000000000003, 0x0000000000000004): DW_OP_reg0 RAX
+; CHECK-NEXT:             DW_LLE_end_of_list   ()
+
+; Built with clang -O3 -ffunction-sections from source:
+; 
+; int f1(int i, int j) {
+;   int x = 5;
+;   int y = 3;
+;   int r = i + j;
+;   int undef;
+;   x = undef;
+;   y = 4;
+;   return r;
+; }
+; void f2() {
+; }
 
-; Function Attrs: noinline nounwind optnone uwtable
-define dso_local void @_ZN1A3barEv(%struct.A* %this) unnamed_addr #2 align 2 !dbg !33 {
+; Function Attrs: norecurse nounwind readnone uwtable
+define dso_local i32 @_Z2f1ii(i32 %i, i32 %j) local_unnamed_addr !dbg !7 {
 entry:
-  %this.addr = alloca %struct.A*, align 8
-  store %struct.A* %this, %struct.A** %this.addr, align 8
-  call void @llvm.dbg.declare(metadata %struct.A** %this.addr, metadata !34, metadata !DIExpression()), !dbg !35
-  %this1 = load %struct.A*, %struct.A** %this.addr, align 8
-  ret void, !dbg !36
+  call void @llvm.dbg.value(metadata i32 %i, metadata !12, metadata !DIExpression()), !dbg !18
+  call void @llvm.dbg.value(metadata i32 %j, metadata !13, metadata !DIExpression()), !dbg !18
+  call void @llvm.dbg.value(metadata i32 5, metadata !14, metadata !DIExpression()), !dbg !18
+  call void @llvm.dbg.value(metadata i32 3, metadata !15, metadata !DIExpression()), !dbg !18
+  %add = add nsw i32 %j, %i, !dbg !19
+  call void @llvm.dbg.value(metadata i32 %add, metadata !16, metadata !DIExpression()), !dbg !18
+  call void @llvm.dbg.value(metadata i32 undef, metadata !14, metadata !DIExpression()), !dbg !18
+  call void @llvm.dbg.value(metadata i32 4, metadata !15, metadata !DIExpression()), !dbg !18
+  ret i32 %add, !dbg !20
 }
 
-; Function Attrs: noinline norecurse nounwind optnone uwtable
-define dso_local i32 @main() #3 !dbg !37 {
+; Function Attrs: norecurse nounwind readnone uwtable
+define dso_local void @_Z2f2v() local_unnamed_addr !dbg !21 {
 entry:
-  %retval = alloca i32, align 4
-  store i32 0, i32* %retval, align 4
-  ret i32 0, !dbg !38
+  ret void, !dbg !24
 }
 
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.value(metadata, metadata, metadata)
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!3, !4, !5}
 !llvm.ident = !{!6}
 
-!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 8.0.0 (trunk 344035)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
-!1 = !DIFile(filename: "test.cc", directory: "/home/folder", checksumkind: CSK_MD5, checksum: "e0f357ad6dcb791a774a0dae55baf5e7")
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 10.0.0 (trunk 374581) (llvm/trunk 374579)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "loc2.cpp", directory: "/usr/local/google/home/blaikie/dev/scratch", checksumkind: CSK_MD5, checksum: "91e0069c680e2a63f4f885ec93f5d07e")
 !2 = !{}
 !3 = !{i32 2, !"Dwarf Version", i32 5}
 !4 = !{i32 2, !"Debug Info Version", i32 3}
 !5 = !{i32 1, !"wchar_size", i32 4}
-!6 = !{!"clang version 8.0.0 (trunk 344035)"}
-!7 = distinct !DISubprogram(name: "baz", linkageName: "_Z3baz1A", scope: !1, file: !1, line: 6, type: !8, isLocal: false, isDefinition: true, scopeLine: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !2)
+!6 = !{!"clang version 10.0.0 (trunk 374581) (llvm/trunk 374579)"}
+!7 = distinct !DISubprogram(name: "f1", linkageName: "_Z2f1ii", scope: !1, file: !1, line: 1, type: !8, scopeLine: 1, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !11)
 !8 = !DISubroutineType(types: !9)
-!9 = !{null, !10}
-!10 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "A", file: !1, line: 1, size: 64, flags: DIFlagTypePassByReference, elements: !11, vtableHolder: !10, identifier: "_ZTS1A")
-!11 = !{!12, !18, !22}
-!12 = !DIDerivedType(tag: DW_TAG_member, name: "_vptr$A", scope: !1, file: !1, baseType: !13, size: 64, flags: DIFlagArtificial)
-!13 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !14, size: 64)
-!14 = !DIDerivedType(tag: DW_TAG_pointer_type, name: "__vtbl_ptr_type", baseType: !15, size: 64)
-!15 = !DISubroutineType(types: !16)
-!16 = !{!17}
-!17 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
-!18 = !DISubprogram(name: "foo", linkageName: "_ZN1A3fooEv", scope: !10, file: !1, line: 2, type: !19, isLocal: false, isDefinition: false, scopeLine: 2, containingType: !10, virtuality: DW_VIRTUALITY_virtual, virtualIndex: 0, flags: DIFlagPrototyped, isOptimized: false)
-!19 = !DISubroutineType(types: !20)
-!20 = !{null, !21}
-!21 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !10, size: 64, flags: DIFlagArtificial | DIFlagObjectPointer)
-!22 = !DISubprogram(name: "bar", linkageName: "_ZN1A3barEv", scope: !10, file: !1, line: 3, type: !19, isLocal: false, isDefinition: false, scopeLine: 3, containingType: !10, virtuality: DW_VIRTUALITY_virtual, virtualIndex: 1, flags: DIFlagPrototyped, isOptimized: false)
-!23 = !DILocalVariable(name: "a", arg: 1, scope: !7, file: !1, line: 6, type: !10)
-!24 = !DILocation(line: 6, column: 19, scope: !7)
-!25 = !DILocation(line: 7, column: 6, scope: !7)
-!26 = !DILocation(line: 8, column: 6, scope: !7)
-!27 = !DILocation(line: 9, column: 1, scope: !7)
-!28 = distinct !DISubprogram(name: "foo", linkageName: "_ZN1A3fooEv", scope: !10, file: !1, line: 12, type: !19, isLocal: false, isDefinition: true, scopeLine: 12, flags: DIFlagPrototyped, isOptimized: false, unit: !0, declaration: !18, retainedNodes: !2)
-!29 = !DILocalVariable(name: "this", arg: 1, scope: !28, type: !30, flags: DIFlagArtificial | DIFlagObjectPointer)
-!30 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !10, size: 64)
-!31 = !DILocation(line: 0, scope: !28)
-!32 = !DILocation(line: 12, column: 16, scope: !28)
-!33 = distinct !DISubprogram(name: "bar", linkageName: "_ZN1A3barEv", scope: !10, file: !1, line: 13, type: !19, isLocal: false, isDefinition: true, scopeLine: 13, flags: DIFlagPrototyped, isOptimized: false, unit: !0, declaration: !22, retainedNodes: !2)
-!34 = !DILocalVariable(name: "this", arg: 1, scope: !33, type: !30, flags: DIFlagArtificial | DIFlagObjectPointer)
-!35 = !DILocation(line: 0, scope: !33)
-!36 = !DILocation(line: 13, column: 16, scope: !33)
-!37 = distinct !DISubprogram(name: "main", scope: !1, file: !1, line: 15, type: !15, isLocal: false, isDefinition: true, scopeLine: 15, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !2)
-!38 = !DILocation(line: 16, column: 3, scope: !37)
+!9 = !{!10, !10, !10}
+!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!11 = !{!12, !13, !14, !15, !16, !17}
+!12 = !DILocalVariable(name: "i", arg: 1, scope: !7, file: !1, line: 1, type: !10)
+!13 = !DILocalVariable(name: "j", arg: 2, scope: !7, file: !1, line: 1, type: !10)
+!14 = !DILocalVariable(name: "x", scope: !7, file: !1, line: 2, type: !10)
+!15 = !DILocalVariable(name: "y", scope: !7, file: !1, line: 3, type: !10)
+!16 = !DILocalVariable(name: "r", scope: !7, file: !1, line: 4, type: !10)
+!17 = !DILocalVariable(name: "undef", scope: !7, file: !1, line: 5, type: !10)
+!18 = !DILocation(line: 0, scope: !7)
+!19 = !DILocation(line: 4, column: 13, scope: !7)
+!20 = !DILocation(line: 8, column: 3, scope: !7)
+!21 = distinct !DISubprogram(name: "f2", linkageName: "_Z2f2v", scope: !1, file: !1, line: 10, type: !22, scopeLine: 10, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2)
+!22 = !DISubroutineType(types: !23)
+!23 = !{null}
+!24 = !DILocation(line: 11, column: 1, scope: !21)
diff --git a/llvm/test/CodeGen/X86/debugloc-argsize.ll b/llvm/test/CodeGen/X86/debugloc-argsize.ll
index a9cbadf3c..53ac263f0 100644
--- a/llvm/test/CodeGen/X86/debugloc-argsize.ll
+++ b/llvm/test/CodeGen/X86/debugloc-argsize.ll
@@ -30,7 +30,7 @@ declare i8* @__cxa_begin_catch(i8*)
 
 declare void @__cxa_end_catch()
 
-attributes #0 = { optsize "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { optsize "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { optsize }
 attributes #2 = { nounwind }
 
diff --git a/llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll b/llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll
new file mode 100644
index 000000000..0e8ba1b6d
--- /dev/null
+++ b/llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll
@@ -0,0 +1,916 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-unknown-unknown   -mattr=+sse,+sse2 | FileCheck %s --check-prefixes=ALL,X86
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse,+sse2 | FileCheck %s --check-prefixes=ALL,X64
+
+; If the target does not have a single div/rem operation,
+; -div-rem-pairs pass will decompose the remainder calculation as:
+;   X % Y --> X - ((X / Y) * Y)
+; But if the target does have a single div/rem operation,
+; the opposite transform is likely beneficial.
+
+define i8 @scalar_i8(i8 %x, i8 %y, i8* %divdst) nounwind {
+; X86-LABEL: scalar_i8:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %ch
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-NEXT:    movsbl %cl, %eax
+; X86-NEXT:    idivb %ch
+; X86-NEXT:    movb %al, (%edx)
+; X86-NEXT:    mulb %ch
+; X86-NEXT:    subb %al, %cl
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: scalar_i8:
+; X64:       # %bb.0:
+; X64-NEXT:    movsbl %dil, %ecx
+; X64-NEXT:    movl %ecx, %eax
+; X64-NEXT:    idivb %sil
+; X64-NEXT:    movb %al, (%rdx)
+; X64-NEXT:    mulb %sil
+; X64-NEXT:    subb %al, %cl
+; X64-NEXT:    movl %ecx, %eax
+; X64-NEXT:    retq
+  %div = sdiv i8 %x, %y
+  store i8 %div, i8* %divdst, align 4
+  %t1 = mul i8 %div, %y
+  %t2 = sub i8 %x, %t1
+  ret i8 %t2
+}
+
+define i16 @scalar_i16(i16 %x, i16 %y, i16* %divdst) nounwind {
+; X86-LABEL: scalar_i16:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    cwtd
+; X86-NEXT:    idivw %si
+; X86-NEXT:    # kill: def $ax killed $ax def $eax
+; X86-NEXT:    movw %ax, (%edi)
+; X86-NEXT:    imull %eax, %esi
+; X86-NEXT:    subl %esi, %ecx
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    retl
+;
+; X64-LABEL: scalar_i16:
+; X64:       # %bb.0:
+; X64-NEXT:    movq %rdx, %rcx
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    cwtd
+; X64-NEXT:    idivw %si
+; X64-NEXT:    # kill: def $ax killed $ax def $eax
+; X64-NEXT:    movw %ax, (%rcx)
+; X64-NEXT:    imull %eax, %esi
+; X64-NEXT:    subl %esi, %edi
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    retq
+  %div = sdiv i16 %x, %y
+  store i16 %div, i16* %divdst, align 4
+  %t1 = mul i16 %div, %y
+  %t2 = sub i16 %x, %t1
+  ret i16 %t2
+}
+
+define i32 @scalar_i32(i32 %x, i32 %y, i32* %divdst) nounwind {
+; X86-LABEL: scalar_i32:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    cltd
+; X86-NEXT:    idivl %edi
+; X86-NEXT:    movl %eax, (%esi)
+; X86-NEXT:    imull %edi, %eax
+; X86-NEXT:    subl %eax, %ecx
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    retl
+;
+; X64-LABEL: scalar_i32:
+; X64:       # %bb.0:
+; X64-NEXT:    movq %rdx, %rcx
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    cltd
+; X64-NEXT:    idivl %esi
+; X64-NEXT:    movl %eax, (%rcx)
+; X64-NEXT:    imull %esi, %eax
+; X64-NEXT:    subl %eax, %edi
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    retq
+  %div = sdiv i32 %x, %y
+  store i32 %div, i32* %divdst, align 4
+  %t1 = mul i32 %div, %y
+  %t2 = sub i32 %x, %t1
+  ret i32 %t2
+}
+
+define i64 @scalar_i64(i64 %x, i64 %y, i64* %divdst) nounwind {
+; X86-LABEL: scalar_i64:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    calll __divdi3
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    movl %edx, %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl %ecx, 4(%edx)
+; X86-NEXT:    movl %eax, (%edx)
+; X86-NEXT:    imull %eax, %ebx
+; X86-NEXT:    mull %ebp
+; X86-NEXT:    addl %ebx, %edx
+; X86-NEXT:    imull %ebp, %ecx
+; X86-NEXT:    addl %edx, %ecx
+; X86-NEXT:    subl %eax, %esi
+; X86-NEXT:    sbbl %ecx, %edi
+; X86-NEXT:    movl %esi, %eax
+; X86-NEXT:    movl %edi, %edx
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl
+;
+; X64-LABEL: scalar_i64:
+; X64:       # %bb.0:
+; X64-NEXT:    movq %rdx, %rcx
+; X64-NEXT:    movq %rdi, %rax
+; X64-NEXT:    cqto
+; X64-NEXT:    idivq %rsi
+; X64-NEXT:    movq %rax, (%rcx)
+; X64-NEXT:    imulq %rsi, %rax
+; X64-NEXT:    subq %rax, %rdi
+; X64-NEXT:    movq %rdi, %rax
+; X64-NEXT:    retq
+  %div = sdiv i64 %x, %y
+  store i64 %div, i64* %divdst, align 4
+  %t1 = mul i64 %div, %y
+  %t2 = sub i64 %x, %t1
+  ret i64 %t2
+}
+
+define <16 x i8> @vector_i128_i8(<16 x i8> %x, <16 x i8> %y, <16 x i8>* %divdst) nounwind {
+; X86-LABEL: vector_i128_i8:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    movl %esp, %ebp
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    andl $-16, %esp
+; X86-NEXT:    subl $48, %esp
+; X86-NEXT:    movdqa %xmm0, (%esp)
+; X86-NEXT:    movdqa %xmm1, {{[0-9]+}}(%esp)
+; X86-NEXT:    movsbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    idivb {{[0-9]+}}(%esp)
+; X86-NEXT:    movzbl %al, %eax
+; X86-NEXT:    movd %eax, %xmm2
+; X86-NEXT:    movsbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    idivb {{[0-9]+}}(%esp)
+; X86-NEXT:    movzbl %al, %eax
+; X86-NEXT:    movd %eax, %xmm3
+; X86-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; X86-NEXT:    movsbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    idivb {{[0-9]+}}(%esp)
+; X86-NEXT:    movzbl %al, %eax
+; X86-NEXT:    movd %eax, %xmm4
+; X86-NEXT:    movsbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    idivb {{[0-9]+}}(%esp)
+; X86-NEXT:    movzbl %al, %eax
+; X86-NEXT:    movd %eax, %xmm2
+; X86-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3],xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7]
+; X86-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
+; X86-NEXT:    movsbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    idivb {{[0-9]+}}(%esp)
+; X86-NEXT:    movzbl %al, %eax
+; X86-NEXT:    movd %eax, %xmm3
+; X86-NEXT:    movsbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    idivb {{[0-9]+}}(%esp)
+; X86-NEXT:    movzbl %al, %eax
+; X86-NEXT:    movd %eax, %xmm4
+; X86-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3],xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
+; X86-NEXT:    movsbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    idivb {{[0-9]+}}(%esp)
+; X86-NEXT:    movzbl %al, %eax
+; X86-NEXT:    movd %eax, %xmm5
+; X86-NEXT:    movsbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    idivb {{[0-9]+}}(%esp)
+; X86-NEXT:    movzbl %al, %eax
+; X86-NEXT:    movd %eax, %xmm3
+; X86-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3],xmm3[4],xmm5[4],xmm3[5],xmm5[5],xmm3[6],xmm5[6],xmm3[7],xmm5[7]
+; X86-NEXT:    movsbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    idivb {{[0-9]+}}(%esp)
+; X86-NEXT:    movzbl %al, %eax
+; X86-NEXT:    movd %eax, %xmm5
+; X86-NEXT:    movsbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    idivb {{[0-9]+}}(%esp)
+; X86-NEXT:    movzbl %al, %eax
+; X86-NEXT:    movd %eax, %xmm6
+; X86-NEXT:    movsbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    idivb {{[0-9]+}}(%esp)
+; X86-NEXT:    movzbl %al, %edx
+; X86-NEXT:    movsbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    idivb {{[0-9]+}}(%esp)
+; X86-NEXT:    movzbl %al, %esi
+; X86-NEXT:    movsbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    idivb {{[0-9]+}}(%esp)
+; X86-NEXT:    movzbl %al, %edi
+; X86-NEXT:    movsbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    idivb {{[0-9]+}}(%esp)
+; X86-NEXT:    movzbl %al, %ebx
+; X86-NEXT:    movsbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    idivb {{[0-9]+}}(%esp)
+; X86-NEXT:    movl %eax, %ecx
+; X86-NEXT:    movsbl (%esp), %eax
+; X86-NEXT:    idivb {{[0-9]+}}(%esp)
+; X86-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
+; X86-NEXT:    movd %edx, %xmm4
+; X86-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; X86-NEXT:    movd %esi, %xmm2
+; X86-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3],xmm6[4],xmm5[4],xmm6[5],xmm5[5],xmm6[6],xmm5[6],xmm6[7],xmm5[7]
+; X86-NEXT:    movd %edi, %xmm5
+; X86-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3],xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7]
+; X86-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm6[0],xmm2[1],xmm6[1],xmm2[2],xmm6[2],xmm2[3],xmm6[3]
+; X86-NEXT:    movd %ebx, %xmm4
+; X86-NEXT:    movzbl %cl, %ecx
+; X86-NEXT:    movd %ecx, %xmm6
+; X86-NEXT:    movl 8(%ebp), %ecx
+; X86-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3],xmm4[4],xmm5[4],xmm4[5],xmm5[5],xmm4[6],xmm5[6],xmm4[7],xmm5[7]
+; X86-NEXT:    movzbl %al, %eax
+; X86-NEXT:    movd %eax, %xmm5
+; X86-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1],xmm5[2],xmm6[2],xmm5[3],xmm6[3],xmm5[4],xmm6[4],xmm5[5],xmm6[5],xmm5[6],xmm6[6],xmm5[7],xmm6[7]
+; X86-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
+; X86-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1]
+; X86-NEXT:    movdqa %xmm5, %xmm2
+; X86-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; X86-NEXT:    movdqa %xmm2, (%ecx)
+; X86-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; X86-NEXT:    movdqa %xmm1, %xmm2
+; X86-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
+; X86-NEXT:    pmullw %xmm3, %xmm2
+; X86-NEXT:    movdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
+; X86-NEXT:    pand %xmm3, %xmm2
+; X86-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3],xmm5[4],xmm0[4],xmm5[5],xmm0[5],xmm5[6],xmm0[6],xmm5[7],xmm0[7]
+; X86-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; X86-NEXT:    pmullw %xmm5, %xmm1
+; X86-NEXT:    pand %xmm3, %xmm1
+; X86-NEXT:    packuswb %xmm2, %xmm1
+; X86-NEXT:    psubb %xmm1, %xmm0
+; X86-NEXT:    leal -12(%ebp), %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl
+;
+; X64-LABEL: vector_i128_i8:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rbp
+; X64-NEXT:    pushq %r15
+; X64-NEXT:    pushq %r14
+; X64-NEXT:    pushq %r13
+; X64-NEXT:    pushq %r12
+; X64-NEXT:    pushq %rbx
+; X64-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    movdqa %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movdqa %xmm1, -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
+; X64-NEXT:    idivb -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movzbl %al, %eax
+; X64-NEXT:    movd %eax, %xmm2
+; X64-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
+; X64-NEXT:    idivb -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movzbl %al, %r8d
+; X64-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
+; X64-NEXT:    idivb -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movzbl %al, %r9d
+; X64-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
+; X64-NEXT:    idivb -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movzbl %al, %r10d
+; X64-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
+; X64-NEXT:    idivb -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movzbl %al, %r11d
+; X64-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
+; X64-NEXT:    idivb -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movzbl %al, %r14d
+; X64-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
+; X64-NEXT:    idivb -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movzbl %al, %r15d
+; X64-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
+; X64-NEXT:    idivb -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movzbl %al, %r12d
+; X64-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
+; X64-NEXT:    idivb -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movzbl %al, %r13d
+; X64-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
+; X64-NEXT:    idivb -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movzbl %al, %edi
+; X64-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
+; X64-NEXT:    idivb -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movzbl %al, %esi
+; X64-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
+; X64-NEXT:    idivb -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movzbl %al, %ebx
+; X64-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
+; X64-NEXT:    idivb -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movzbl %al, %ebp
+; X64-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
+; X64-NEXT:    idivb -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movzbl %al, %edx
+; X64-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
+; X64-NEXT:    idivb -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movl %eax, %ecx
+; X64-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
+; X64-NEXT:    idivb -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movd %r8d, %xmm3
+; X64-NEXT:    movd %r9d, %xmm4
+; X64-NEXT:    movd %r10d, %xmm5
+; X64-NEXT:    movd %r11d, %xmm6
+; X64-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; X64-NEXT:    movd %r14d, %xmm2
+; X64-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3],xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]
+; X64-NEXT:    movd %r15d, %xmm4
+; X64-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
+; X64-NEXT:    movd %r12d, %xmm3
+; X64-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm6[0],xmm2[1],xmm6[1],xmm2[2],xmm6[2],xmm2[3],xmm6[3],xmm2[4],xmm6[4],xmm2[5],xmm6[5],xmm2[6],xmm6[6],xmm2[7],xmm6[7]
+; X64-NEXT:    movd %r13d, %xmm6
+; X64-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3],xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7]
+; X64-NEXT:    movd %edi, %xmm4
+; X64-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; X64-NEXT:    movd %esi, %xmm2
+; X64-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1]
+; X64-NEXT:    movd %ebx, %xmm5
+; X64-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm6[0],xmm4[1],xmm6[1],xmm4[2],xmm6[2],xmm4[3],xmm6[3],xmm4[4],xmm6[4],xmm4[5],xmm6[5],xmm4[6],xmm6[6],xmm4[7],xmm6[7]
+; X64-NEXT:    movd %ebp, %xmm6
+; X64-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3],xmm5[4],xmm2[4],xmm5[5],xmm2[5],xmm5[6],xmm2[6],xmm5[7],xmm2[7]
+; X64-NEXT:    movd %edx, %xmm2
+; X64-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
+; X64-NEXT:    movzbl %cl, %ecx
+; X64-NEXT:    movd %ecx, %xmm4
+; X64-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm6[0],xmm2[1],xmm6[1],xmm2[2],xmm6[2],xmm2[3],xmm6[3],xmm2[4],xmm6[4],xmm2[5],xmm6[5],xmm2[6],xmm6[6],xmm2[7],xmm6[7]
+; X64-NEXT:    movzbl %al, %eax
+; X64-NEXT:    movd %eax, %xmm6
+; X64-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm4[0],xmm6[1],xmm4[1],xmm6[2],xmm4[2],xmm6[3],xmm4[3],xmm6[4],xmm4[4],xmm6[5],xmm4[5],xmm6[6],xmm4[6],xmm6[7],xmm4[7]
+; X64-NEXT:    punpcklwd {{.*#+}} xmm6 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]
+; X64-NEXT:    punpckldq {{.*#+}} xmm6 = xmm6[0],xmm5[0],xmm6[1],xmm5[1]
+; X64-NEXT:    movdqa %xmm6, %xmm2
+; X64-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; X64-NEXT:    movdqa %xmm2, (%rax)
+; X64-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; X64-NEXT:    movdqa %xmm1, %xmm2
+; X64-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
+; X64-NEXT:    pmullw %xmm3, %xmm2
+; X64-NEXT:    movdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
+; X64-NEXT:    pand %xmm3, %xmm2
+; X64-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm0[0],xmm6[1],xmm0[1],xmm6[2],xmm0[2],xmm6[3],xmm0[3],xmm6[4],xmm0[4],xmm6[5],xmm0[5],xmm6[6],xmm0[6],xmm6[7],xmm0[7]
+; X64-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; X64-NEXT:    pmullw %xmm6, %xmm1
+; X64-NEXT:    pand %xmm3, %xmm1
+; X64-NEXT:    packuswb %xmm2, %xmm1
+; X64-NEXT:    psubb %xmm1, %xmm0
+; X64-NEXT:    popq %rbx
+; X64-NEXT:    popq %r12
+; X64-NEXT:    popq %r13
+; X64-NEXT:    popq %r14
+; X64-NEXT:    popq %r15
+; X64-NEXT:    popq %rbp
+; X64-NEXT:    retq
+  %div = sdiv <16 x i8> %x, %y
+  store <16 x i8> %div, <16 x i8>* %divdst, align 16
+  %t1 = mul <16 x i8> %div, %y
+  %t2 = sub <16 x i8> %x, %t1
+  ret <16 x i8> %t2
+}
+
+define <8 x i16> @vector_i128_i16(<8 x i16> %x, <8 x i16> %y, <8 x i16>* %divdst) nounwind {
+; X86-LABEL: vector_i128_i16:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    pextrw $7, %xmm0, %eax
+; X86-NEXT:    pextrw $7, %xmm1, %esi
+; X86-NEXT:    # kill: def $ax killed $ax killed $eax
+; X86-NEXT:    cwtd
+; X86-NEXT:    idivw %si
+; X86-NEXT:    # kill: def $ax killed $ax def $eax
+; X86-NEXT:    movd %eax, %xmm2
+; X86-NEXT:    pextrw $6, %xmm0, %eax
+; X86-NEXT:    pextrw $6, %xmm1, %esi
+; X86-NEXT:    # kill: def $ax killed $ax killed $eax
+; X86-NEXT:    cwtd
+; X86-NEXT:    idivw %si
+; X86-NEXT:    # kill: def $ax killed $ax def $eax
+; X86-NEXT:    movd %eax, %xmm3
+; X86-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; X86-NEXT:    pextrw $5, %xmm0, %eax
+; X86-NEXT:    pextrw $5, %xmm1, %esi
+; X86-NEXT:    # kill: def $ax killed $ax killed $eax
+; X86-NEXT:    cwtd
+; X86-NEXT:    idivw %si
+; X86-NEXT:    # kill: def $ax killed $ax def $eax
+; X86-NEXT:    movd %eax, %xmm4
+; X86-NEXT:    pextrw $4, %xmm0, %eax
+; X86-NEXT:    pextrw $4, %xmm1, %esi
+; X86-NEXT:    # kill: def $ax killed $ax killed $eax
+; X86-NEXT:    cwtd
+; X86-NEXT:    idivw %si
+; X86-NEXT:    # kill: def $ax killed $ax def $eax
+; X86-NEXT:    movd %eax, %xmm2
+; X86-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
+; X86-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; X86-NEXT:    pextrw $3, %xmm0, %eax
+; X86-NEXT:    pextrw $3, %xmm1, %esi
+; X86-NEXT:    # kill: def $ax killed $ax killed $eax
+; X86-NEXT:    cwtd
+; X86-NEXT:    idivw %si
+; X86-NEXT:    # kill: def $ax killed $ax def $eax
+; X86-NEXT:    movd %eax, %xmm3
+; X86-NEXT:    pextrw $2, %xmm0, %eax
+; X86-NEXT:    pextrw $2, %xmm1, %esi
+; X86-NEXT:    # kill: def $ax killed $ax killed $eax
+; X86-NEXT:    cwtd
+; X86-NEXT:    idivw %si
+; X86-NEXT:    # kill: def $ax killed $ax def $eax
+; X86-NEXT:    movd %eax, %xmm4
+; X86-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; X86-NEXT:    pextrw $1, %xmm0, %eax
+; X86-NEXT:    pextrw $1, %xmm1, %esi
+; X86-NEXT:    # kill: def $ax killed $ax killed $eax
+; X86-NEXT:    cwtd
+; X86-NEXT:    idivw %si
+; X86-NEXT:    # kill: def $ax killed $ax def $eax
+; X86-NEXT:    movd %eax, %xmm3
+; X86-NEXT:    movd %xmm0, %eax
+; X86-NEXT:    movd %xmm1, %esi
+; X86-NEXT:    # kill: def $ax killed $ax killed $eax
+; X86-NEXT:    cwtd
+; X86-NEXT:    idivw %si
+; X86-NEXT:    # kill: def $ax killed $ax def $eax
+; X86-NEXT:    movd %eax, %xmm5
+; X86-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
+; X86-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
+; X86-NEXT:    punpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm2[0]
+; X86-NEXT:    movdqa %xmm5, (%ecx)
+; X86-NEXT:    pmullw %xmm1, %xmm5
+; X86-NEXT:    psubw %xmm5, %xmm0
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
+;
+; X64-LABEL: vector_i128_i16:
+; X64:       # %bb.0:
+; X64-NEXT:    pextrw $7, %xmm0, %eax
+; X64-NEXT:    pextrw $7, %xmm1, %ecx
+; X64-NEXT:    # kill: def $ax killed $ax killed $eax
+; X64-NEXT:    cwtd
+; X64-NEXT:    idivw %cx
+; X64-NEXT:    # kill: def $ax killed $ax def $eax
+; X64-NEXT:    movd %eax, %xmm2
+; X64-NEXT:    pextrw $6, %xmm0, %eax
+; X64-NEXT:    pextrw $6, %xmm1, %ecx
+; X64-NEXT:    # kill: def $ax killed $ax killed $eax
+; X64-NEXT:    cwtd
+; X64-NEXT:    idivw %cx
+; X64-NEXT:    # kill: def $ax killed $ax def $eax
+; X64-NEXT:    movd %eax, %xmm3
+; X64-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; X64-NEXT:    pextrw $5, %xmm0, %eax
+; X64-NEXT:    pextrw $5, %xmm1, %ecx
+; X64-NEXT:    # kill: def $ax killed $ax killed $eax
+; X64-NEXT:    cwtd
+; X64-NEXT:    idivw %cx
+; X64-NEXT:    # kill: def $ax killed $ax def $eax
+; X64-NEXT:    movd %eax, %xmm4
+; X64-NEXT:    pextrw $4, %xmm0, %eax
+; X64-NEXT:    pextrw $4, %xmm1, %ecx
+; X64-NEXT:    # kill: def $ax killed $ax killed $eax
+; X64-NEXT:    cwtd
+; X64-NEXT:    idivw %cx
+; X64-NEXT:    # kill: def $ax killed $ax def $eax
+; X64-NEXT:    movd %eax, %xmm2
+; X64-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
+; X64-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; X64-NEXT:    pextrw $3, %xmm0, %eax
+; X64-NEXT:    pextrw $3, %xmm1, %ecx
+; X64-NEXT:    # kill: def $ax killed $ax killed $eax
+; X64-NEXT:    cwtd
+; X64-NEXT:    idivw %cx
+; X64-NEXT:    # kill: def $ax killed $ax def $eax
+; X64-NEXT:    movd %eax, %xmm3
+; X64-NEXT:    pextrw $2, %xmm0, %eax
+; X64-NEXT:    pextrw $2, %xmm1, %ecx
+; X64-NEXT:    # kill: def $ax killed $ax killed $eax
+; X64-NEXT:    cwtd
+; X64-NEXT:    idivw %cx
+; X64-NEXT:    # kill: def $ax killed $ax def $eax
+; X64-NEXT:    movd %eax, %xmm4
+; X64-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; X64-NEXT:    pextrw $1, %xmm0, %eax
+; X64-NEXT:    pextrw $1, %xmm1, %ecx
+; X64-NEXT:    # kill: def $ax killed $ax killed $eax
+; X64-NEXT:    cwtd
+; X64-NEXT:    idivw %cx
+; X64-NEXT:    # kill: def $ax killed $ax def $eax
+; X64-NEXT:    movd %eax, %xmm3
+; X64-NEXT:    movd %xmm0, %eax
+; X64-NEXT:    movd %xmm1, %ecx
+; X64-NEXT:    # kill: def $ax killed $ax killed $eax
+; X64-NEXT:    cwtd
+; X64-NEXT:    idivw %cx
+; X64-NEXT:    # kill: def $ax killed $ax def $eax
+; X64-NEXT:    movd %eax, %xmm5
+; X64-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
+; X64-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
+; X64-NEXT:    punpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm2[0]
+; X64-NEXT:    movdqa %xmm5, (%rdi)
+; X64-NEXT:    pmullw %xmm1, %xmm5
+; X64-NEXT:    psubw %xmm5, %xmm0
+; X64-NEXT:    retq
+  %div = sdiv <8 x i16> %x, %y
+  store <8 x i16> %div, <8 x i16>* %divdst, align 16
+  %t1 = mul <8 x i16> %div, %y
+  %t2 = sub <8 x i16> %x, %t1
+  ret <8 x i16> %t2
+}
+
+define <4 x i32> @vector_i128_i32(<4 x i32> %x, <4 x i32> %y, <4 x i32>* %divdst) nounwind {
+; X86-LABEL: vector_i128_i32:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,1,2,3]
+; X86-NEXT:    movd %xmm2, %eax
+; X86-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,1,2,3]
+; X86-NEXT:    movd %xmm2, %esi
+; X86-NEXT:    cltd
+; X86-NEXT:    idivl %esi
+; X86-NEXT:    movd %eax, %xmm2
+; X86-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,0,1]
+; X86-NEXT:    movd %xmm3, %eax
+; X86-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,0,1]
+; X86-NEXT:    movd %xmm3, %esi
+; X86-NEXT:    cltd
+; X86-NEXT:    idivl %esi
+; X86-NEXT:    movd %eax, %xmm3
+; X86-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; X86-NEXT:    movd %xmm0, %eax
+; X86-NEXT:    movd %xmm1, %esi
+; X86-NEXT:    cltd
+; X86-NEXT:    idivl %esi
+; X86-NEXT:    movd %eax, %xmm4
+; X86-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[1,1,2,3]
+; X86-NEXT:    movd %xmm5, %eax
+; X86-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[1,1,2,3]
+; X86-NEXT:    movd %xmm5, %esi
+; X86-NEXT:    cltd
+; X86-NEXT:    idivl %esi
+; X86-NEXT:    movd %eax, %xmm5
+; X86-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
+; X86-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm3[0]
+; X86-NEXT:    movdqa %xmm4, (%ecx)
+; X86-NEXT:    pmuludq %xmm1, %xmm4
+; X86-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[0,2,2,3]
+; X86-NEXT:    shufps {{.*#+}} xmm5 = xmm5[0,0],xmm2[0,0]
+; X86-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; X86-NEXT:    pmuludq %xmm5, %xmm1
+; X86-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; X86-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; X86-NEXT:    psubd %xmm3, %xmm0
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
+;
+; X64-LABEL: vector_i128_i32:
+; X64:       # %bb.0:
+; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,1,2,3]
+; X64-NEXT:    movd %xmm2, %eax
+; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,1,2,3]
+; X64-NEXT:    movd %xmm2, %ecx
+; X64-NEXT:    cltd
+; X64-NEXT:    idivl %ecx
+; X64-NEXT:    movd %eax, %xmm2
+; X64-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,0,1]
+; X64-NEXT:    movd %xmm3, %eax
+; X64-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,0,1]
+; X64-NEXT:    movd %xmm3, %ecx
+; X64-NEXT:    cltd
+; X64-NEXT:    idivl %ecx
+; X64-NEXT:    movd %eax, %xmm3
+; X64-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; X64-NEXT:    movd %xmm0, %eax
+; X64-NEXT:    movd %xmm1, %ecx
+; X64-NEXT:    cltd
+; X64-NEXT:    idivl %ecx
+; X64-NEXT:    movd %eax, %xmm4
+; X64-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[1,1,2,3]
+; X64-NEXT:    movd %xmm5, %eax
+; X64-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[1,1,2,3]
+; X64-NEXT:    movd %xmm5, %ecx
+; X64-NEXT:    cltd
+; X64-NEXT:    idivl %ecx
+; X64-NEXT:    movd %eax, %xmm5
+; X64-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
+; X64-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm3[0]
+; X64-NEXT:    movdqa %xmm4, (%rdi)
+; X64-NEXT:    pmuludq %xmm1, %xmm4
+; X64-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[0,2,2,3]
+; X64-NEXT:    shufps {{.*#+}} xmm5 = xmm5[0,0],xmm2[0,0]
+; X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; X64-NEXT:    pmuludq %xmm5, %xmm1
+; X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; X64-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; X64-NEXT:    psubd %xmm3, %xmm0
+; X64-NEXT:    retq
+  %div = sdiv <4 x i32> %x, %y
+  store <4 x i32> %div, <4 x i32>* %divdst, align 16
+  %t1 = mul <4 x i32> %div, %y
+  %t2 = sub <4 x i32> %x, %t1
+  ret <4 x i32> %t2
+}
+
+define <2 x i64> @vector_i128_i64(<2 x i64> %x, <2 x i64> %y, <2 x i64>* %divdst) nounwind {
+; X86-LABEL: vector_i128_i64:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $72, %esp
+; X86-NEXT:    movdqu %xmm1, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; X86-NEXT:    movdqu %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,1,2,3]
+; X86-NEXT:    movd %xmm2, {{[0-9]+}}(%esp)
+; X86-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; X86-NEXT:    movd %xmm2, {{[0-9]+}}(%esp)
+; X86-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,1,2,3]
+; X86-NEXT:    movd %xmm1, {{[0-9]+}}(%esp)
+; X86-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-NEXT:    movd %xmm1, (%esp)
+; X86-NEXT:    calll __divdi3
+; X86-NEXT:    movdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm1 # 16-byte Reload
+; X86-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X86-NEXT:    movd %xmm0, {{[0-9]+}}(%esp)
+; X86-NEXT:    movd %xmm1, {{[0-9]+}}(%esp)
+; X86-NEXT:    movdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm1 # 16-byte Reload
+; X86-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X86-NEXT:    movd %xmm0, {{[0-9]+}}(%esp)
+; X86-NEXT:    movd %xmm1, (%esp)
+; X86-NEXT:    movd %edx, %xmm0
+; X86-NEXT:    movd %eax, %xmm1
+; X86-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; X86-NEXT:    movdqu %xmm1, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; X86-NEXT:    calll __divdi3
+; X86-NEXT:    movd %edx, %xmm0
+; X86-NEXT:    movd %eax, %xmm1
+; X86-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; X86-NEXT:    movdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
+; X86-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; X86-NEXT:    movdqa %xmm1, (%esi)
+; X86-NEXT:    movdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm3 # 16-byte Reload
+; X86-NEXT:    movdqa %xmm3, %xmm0
+; X86-NEXT:    psrlq $32, %xmm0
+; X86-NEXT:    pmuludq %xmm1, %xmm0
+; X86-NEXT:    movdqa %xmm1, %xmm2
+; X86-NEXT:    psrlq $32, %xmm2
+; X86-NEXT:    pmuludq %xmm3, %xmm2
+; X86-NEXT:    paddq %xmm0, %xmm2
+; X86-NEXT:    psllq $32, %xmm2
+; X86-NEXT:    pmuludq %xmm3, %xmm1
+; X86-NEXT:    paddq %xmm2, %xmm1
+; X86-NEXT:    movdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
+; X86-NEXT:    psubq %xmm1, %xmm0
+; X86-NEXT:    addl $72, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
+;
+; X64-LABEL: vector_i128_i64:
+; X64:       # %bb.0:
+; X64-NEXT:    movq %xmm0, %rax
+; X64-NEXT:    movq %xmm1, %rcx
+; X64-NEXT:    cqto
+; X64-NEXT:    idivq %rcx
+; X64-NEXT:    movq %rax, %xmm2
+; X64-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,0,1]
+; X64-NEXT:    movq %xmm3, %rax
+; X64-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,0,1]
+; X64-NEXT:    movq %xmm3, %rcx
+; X64-NEXT:    cqto
+; X64-NEXT:    idivq %rcx
+; X64-NEXT:    movq %rax, %xmm3
+; X64-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; X64-NEXT:    movdqa %xmm2, (%rdi)
+; X64-NEXT:    movdqa %xmm1, %xmm3
+; X64-NEXT:    psrlq $32, %xmm3
+; X64-NEXT:    pmuludq %xmm2, %xmm3
+; X64-NEXT:    movdqa %xmm2, %xmm4
+; X64-NEXT:    psrlq $32, %xmm4
+; X64-NEXT:    pmuludq %xmm1, %xmm4
+; X64-NEXT:    paddq %xmm3, %xmm4
+; X64-NEXT:    psllq $32, %xmm4
+; X64-NEXT:    pmuludq %xmm1, %xmm2
+; X64-NEXT:    paddq %xmm4, %xmm2
+; X64-NEXT:    psubq %xmm2, %xmm0
+; X64-NEXT:    retq
+  %div = sdiv <2 x i64> %x, %y
+  store <2 x i64> %div, <2 x i64>* %divdst, align 16
+  %t1 = mul <2 x i64> %div, %y
+  %t2 = sub <2 x i64> %x, %t1
+  ret <2 x i64> %t2
+}
+
+; Special tests.
+
+define i32 @scalar_i32_commutative(i32 %x, i32* %ysrc, i32* %divdst) nounwind {
+; X86-LABEL: scalar_i32_commutative:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl (%eax), %edi
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    cltd
+; X86-NEXT:    idivl %edi
+; X86-NEXT:    movl %eax, (%esi)
+; X86-NEXT:    imull %eax, %edi
+; X86-NEXT:    subl %edi, %ecx
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    retl
+;
+; X64-LABEL: scalar_i32_commutative:
+; X64:       # %bb.0:
+; X64-NEXT:    movq %rdx, %rcx
+; X64-NEXT:    movl (%rsi), %esi
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    cltd
+; X64-NEXT:    idivl %esi
+; X64-NEXT:    movl %eax, (%rcx)
+; X64-NEXT:    imull %eax, %esi
+; X64-NEXT:    subl %esi, %edi
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    retq
+  %y = load i32, i32* %ysrc, align 4
+  %div = sdiv i32 %x, %y
+  store i32 %div, i32* %divdst, align 4
+  %t1 = mul i32 %y, %div ; commutative
+  %t2 = sub i32 %x, %t1
+  ret i32 %t2
+}
+
+; We do not care about extra uses.
+define i32 @extrause(i32 %x, i32 %y, i32* %divdst, i32* %t1dst) nounwind {
+; X86-LABEL: extrause:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    cltd
+; X86-NEXT:    idivl %ebx
+; X86-NEXT:    movl %eax, (%edi)
+; X86-NEXT:    imull %ebx, %eax
+; X86-NEXT:    movl %eax, (%esi)
+; X86-NEXT:    subl %eax, %ecx
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    retl
+;
+; X64-LABEL: extrause:
+; X64:       # %bb.0:
+; X64-NEXT:    movq %rdx, %r8
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    cltd
+; X64-NEXT:    idivl %esi
+; X64-NEXT:    movl %eax, (%r8)
+; X64-NEXT:    imull %esi, %eax
+; X64-NEXT:    movl %eax, (%rcx)
+; X64-NEXT:    subl %eax, %edi
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    retq
+  %div = sdiv i32 %x, %y
+  store i32 %div, i32* %divdst, align 4
+  %t1 = mul i32 %div, %y
+  store i32 %t1, i32* %t1dst, align 4
+  %t2 = sub i32 %x, %t1
+  ret i32 %t2
+}
+
+; 'rem' should appear next to 'div'.
+define i32 @multiple_bb(i32 %x, i32 %y, i32* %divdst, i1 zeroext %store_srem, i32* %sremdst) nounwind {
+; X86-LABEL: multiple_bb:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %bl
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    cltd
+; X86-NEXT:    idivl %esi
+; X86-NEXT:    movl %eax, (%edi)
+; X86-NEXT:    testb %bl, %bl
+; X86-NEXT:    je .LBB10_2
+; X86-NEXT:  # %bb.1: # %do_srem
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl %eax, %edi
+; X86-NEXT:    imull %esi, %edi
+; X86-NEXT:    subl %edi, %ecx
+; X86-NEXT:    movl %ecx, (%edx)
+; X86-NEXT:  .LBB10_2: # %end
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    retl
+;
+; X64-LABEL: multiple_bb:
+; X64:       # %bb.0:
+; X64-NEXT:    movq %rdx, %r9
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    cltd
+; X64-NEXT:    idivl %esi
+; X64-NEXT:    movl %eax, (%r9)
+; X64-NEXT:    testb %cl, %cl
+; X64-NEXT:    je .LBB10_2
+; X64-NEXT:  # %bb.1: # %do_srem
+; X64-NEXT:    movl %eax, %ecx
+; X64-NEXT:    imull %esi, %ecx
+; X64-NEXT:    subl %ecx, %edi
+; X64-NEXT:    movl %edi, (%r8)
+; X64-NEXT:  .LBB10_2: # %end
+; X64-NEXT:    retq
+  %div = sdiv i32 %x, %y
+  store i32 %div, i32* %divdst, align 4
+  br i1 %store_srem, label %do_srem, label %end
+do_srem:
+  %t1 = mul i32 %div, %y
+  %t2 = sub i32 %x, %t1
+  store i32 %t2, i32* %sremdst, align 4
+  br label %end
+end:
+  ret i32 %div
+}
+
+define i32 @negative_different_x(i32 %x0, i32 %x1, i32 %y, i32* %divdst) nounwind {
+; X86-LABEL: negative_different_x:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    cltd
+; X86-NEXT:    idivl %edi
+; X86-NEXT:    movl %eax, (%esi)
+; X86-NEXT:    imull %edi, %eax
+; X86-NEXT:    subl %eax, %ecx
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    retl
+;
+; X64-LABEL: negative_different_x:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edx, %r8d
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    cltd
+; X64-NEXT:    idivl %r8d
+; X64-NEXT:    movl %eax, (%rcx)
+; X64-NEXT:    imull %r8d, %eax
+; X64-NEXT:    subl %eax, %esi
+; X64-NEXT:    movl %esi, %eax
+; X64-NEXT:    retq
+  %div = sdiv i32 %x0, %y ; not %x1
+  store i32 %div, i32* %divdst, align 4
+  %t1 = mul i32 %div, %y
+  %t2 = sub i32 %x1, %t1 ; not %x0
+  ret i32 %t2
+}
diff --git a/llvm/test/CodeGen/X86/div-rem-pair-recomposition-unsigned.ll b/llvm/test/CodeGen/X86/div-rem-pair-recomposition-unsigned.ll
new file mode 100644
index 000000000..500c2110d
--- /dev/null
+++ b/llvm/test/CodeGen/X86/div-rem-pair-recomposition-unsigned.ll
@@ -0,0 +1,916 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-unknown-unknown   -mattr=+sse,+sse2 | FileCheck %s --check-prefixes=ALL,X86
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse,+sse2 | FileCheck %s --check-prefixes=ALL,X64
+
+; If the target does not have a single div/rem operation,
+; -div-rem-pairs pass will decompose the remainder calculation as:
+;   X % Y --> X - ((X / Y) * Y)
+; But if the target does have a single div/rem operation,
+; the opposite transform is likely beneficial.
+
+define i8 @scalar_i8(i8 %x, i8 %y, i8* %divdst) nounwind {
+; X86-LABEL: scalar_i8:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %ch
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-NEXT:    movzbl %cl, %eax
+; X86-NEXT:    divb %ch
+; X86-NEXT:    movb %al, (%edx)
+; X86-NEXT:    mulb %ch
+; X86-NEXT:    subb %al, %cl
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: scalar_i8:
+; X64:       # %bb.0:
+; X64-NEXT:    movzbl %dil, %ecx
+; X64-NEXT:    movl %ecx, %eax
+; X64-NEXT:    divb %sil
+; X64-NEXT:    movb %al, (%rdx)
+; X64-NEXT:    mulb %sil
+; X64-NEXT:    subb %al, %cl
+; X64-NEXT:    movl %ecx, %eax
+; X64-NEXT:    retq
+  %div = udiv i8 %x, %y
+  store i8 %div, i8* %divdst, align 4
+  %t1 = mul i8 %div, %y
+  %t2 = sub i8 %x, %t1
+  ret i8 %t2
+}
+
+define i16 @scalar_i16(i16 %x, i16 %y, i16* %divdst) nounwind {
+; X86-LABEL: scalar_i16:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    xorl %edx, %edx
+; X86-NEXT:    divw %si
+; X86-NEXT:    # kill: def $ax killed $ax def $eax
+; X86-NEXT:    movw %ax, (%edi)
+; X86-NEXT:    imull %eax, %esi
+; X86-NEXT:    subl %esi, %ecx
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    retl
+;
+; X64-LABEL: scalar_i16:
+; X64:       # %bb.0:
+; X64-NEXT:    movq %rdx, %rcx
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    xorl %edx, %edx
+; X64-NEXT:    divw %si
+; X64-NEXT:    # kill: def $ax killed $ax def $eax
+; X64-NEXT:    movw %ax, (%rcx)
+; X64-NEXT:    imull %eax, %esi
+; X64-NEXT:    subl %esi, %edi
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    retq
+  %div = udiv i16 %x, %y
+  store i16 %div, i16* %divdst, align 4
+  %t1 = mul i16 %div, %y
+  %t2 = sub i16 %x, %t1
+  ret i16 %t2
+}
+
+define i32 @scalar_i32(i32 %x, i32 %y, i32* %divdst) nounwind {
+; X86-LABEL: scalar_i32:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    xorl %edx, %edx
+; X86-NEXT:    divl %edi
+; X86-NEXT:    movl %eax, (%esi)
+; X86-NEXT:    imull %edi, %eax
+; X86-NEXT:    subl %eax, %ecx
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    retl
+;
+; X64-LABEL: scalar_i32:
+; X64:       # %bb.0:
+; X64-NEXT:    movq %rdx, %rcx
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    xorl %edx, %edx
+; X64-NEXT:    divl %esi
+; X64-NEXT:    movl %eax, (%rcx)
+; X64-NEXT:    imull %esi, %eax
+; X64-NEXT:    subl %eax, %edi
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    retq
+  %div = udiv i32 %x, %y
+  store i32 %div, i32* %divdst, align 4
+  %t1 = mul i32 %div, %y
+  %t2 = sub i32 %x, %t1
+  ret i32 %t2
+}
+
+define i64 @scalar_i64(i64 %x, i64 %y, i64* %divdst) nounwind {
+; X86-LABEL: scalar_i64:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    calll __udivdi3
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    movl %edx, %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl %ecx, 4(%edx)
+; X86-NEXT:    movl %eax, (%edx)
+; X86-NEXT:    imull %eax, %ebx
+; X86-NEXT:    mull %ebp
+; X86-NEXT:    addl %ebx, %edx
+; X86-NEXT:    imull %ebp, %ecx
+; X86-NEXT:    addl %edx, %ecx
+; X86-NEXT:    subl %eax, %esi
+; X86-NEXT:    sbbl %ecx, %edi
+; X86-NEXT:    movl %esi, %eax
+; X86-NEXT:    movl %edi, %edx
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl
+;
+; X64-LABEL: scalar_i64:
+; X64:       # %bb.0:
+; X64-NEXT:    movq %rdx, %rcx
+; X64-NEXT:    movq %rdi, %rax
+; X64-NEXT:    xorl %edx, %edx
+; X64-NEXT:    divq %rsi
+; X64-NEXT:    movq %rax, (%rcx)
+; X64-NEXT:    imulq %rsi, %rax
+; X64-NEXT:    subq %rax, %rdi
+; X64-NEXT:    movq %rdi, %rax
+; X64-NEXT:    retq
+  %div = udiv i64 %x, %y
+  store i64 %div, i64* %divdst, align 4
+  %t1 = mul i64 %div, %y
+  %t2 = sub i64 %x, %t1
+  ret i64 %t2
+}
+
+define <16 x i8> @vector_i128_i8(<16 x i8> %x, <16 x i8> %y, <16 x i8>* %divdst) nounwind {
+; X86-LABEL: vector_i128_i8:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    movl %esp, %ebp
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    andl $-16, %esp
+; X86-NEXT:    subl $48, %esp
+; X86-NEXT:    movdqa %xmm0, (%esp)
+; X86-NEXT:    movdqa %xmm1, {{[0-9]+}}(%esp)
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    divb {{[0-9]+}}(%esp)
+; X86-NEXT:    movzbl %al, %eax
+; X86-NEXT:    movd %eax, %xmm2
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    divb {{[0-9]+}}(%esp)
+; X86-NEXT:    movzbl %al, %eax
+; X86-NEXT:    movd %eax, %xmm3
+; X86-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    divb {{[0-9]+}}(%esp)
+; X86-NEXT:    movzbl %al, %eax
+; X86-NEXT:    movd %eax, %xmm4
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    divb {{[0-9]+}}(%esp)
+; X86-NEXT:    movzbl %al, %eax
+; X86-NEXT:    movd %eax, %xmm2
+; X86-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3],xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7]
+; X86-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    divb {{[0-9]+}}(%esp)
+; X86-NEXT:    movzbl %al, %eax
+; X86-NEXT:    movd %eax, %xmm3
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    divb {{[0-9]+}}(%esp)
+; X86-NEXT:    movzbl %al, %eax
+; X86-NEXT:    movd %eax, %xmm4
+; X86-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3],xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    divb {{[0-9]+}}(%esp)
+; X86-NEXT:    movzbl %al, %eax
+; X86-NEXT:    movd %eax, %xmm5
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    divb {{[0-9]+}}(%esp)
+; X86-NEXT:    movzbl %al, %eax
+; X86-NEXT:    movd %eax, %xmm3
+; X86-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3],xmm3[4],xmm5[4],xmm3[5],xmm5[5],xmm3[6],xmm5[6],xmm3[7],xmm5[7]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    divb {{[0-9]+}}(%esp)
+; X86-NEXT:    movzbl %al, %eax
+; X86-NEXT:    movd %eax, %xmm5
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    divb {{[0-9]+}}(%esp)
+; X86-NEXT:    movzbl %al, %eax
+; X86-NEXT:    movd %eax, %xmm6
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    divb {{[0-9]+}}(%esp)
+; X86-NEXT:    movzbl %al, %edx
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    divb {{[0-9]+}}(%esp)
+; X86-NEXT:    movzbl %al, %esi
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    divb {{[0-9]+}}(%esp)
+; X86-NEXT:    movzbl %al, %edi
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    divb {{[0-9]+}}(%esp)
+; X86-NEXT:    movzbl %al, %ebx
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    divb {{[0-9]+}}(%esp)
+; X86-NEXT:    movl %eax, %ecx
+; X86-NEXT:    movzbl (%esp), %eax
+; X86-NEXT:    divb {{[0-9]+}}(%esp)
+; X86-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
+; X86-NEXT:    movd %edx, %xmm4
+; X86-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; X86-NEXT:    movd %esi, %xmm2
+; X86-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3],xmm6[4],xmm5[4],xmm6[5],xmm5[5],xmm6[6],xmm5[6],xmm6[7],xmm5[7]
+; X86-NEXT:    movd %edi, %xmm5
+; X86-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3],xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7]
+; X86-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm6[0],xmm2[1],xmm6[1],xmm2[2],xmm6[2],xmm2[3],xmm6[3]
+; X86-NEXT:    movd %ebx, %xmm4
+; X86-NEXT:    movzbl %cl, %ecx
+; X86-NEXT:    movd %ecx, %xmm6
+; X86-NEXT:    movl 8(%ebp), %ecx
+; X86-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3],xmm4[4],xmm5[4],xmm4[5],xmm5[5],xmm4[6],xmm5[6],xmm4[7],xmm5[7]
+; X86-NEXT:    movzbl %al, %eax
+; X86-NEXT:    movd %eax, %xmm5
+; X86-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1],xmm5[2],xmm6[2],xmm5[3],xmm6[3],xmm5[4],xmm6[4],xmm5[5],xmm6[5],xmm5[6],xmm6[6],xmm5[7],xmm6[7]
+; X86-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
+; X86-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1]
+; X86-NEXT:    movdqa %xmm5, %xmm2
+; X86-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; X86-NEXT:    movdqa %xmm2, (%ecx)
+; X86-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; X86-NEXT:    movdqa %xmm1, %xmm2
+; X86-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
+; X86-NEXT:    pmullw %xmm3, %xmm2
+; X86-NEXT:    movdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
+; X86-NEXT:    pand %xmm3, %xmm2
+; X86-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3],xmm5[4],xmm0[4],xmm5[5],xmm0[5],xmm5[6],xmm0[6],xmm5[7],xmm0[7]
+; X86-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; X86-NEXT:    pmullw %xmm5, %xmm1
+; X86-NEXT:    pand %xmm3, %xmm1
+; X86-NEXT:    packuswb %xmm2, %xmm1
+; X86-NEXT:    psubb %xmm1, %xmm0
+; X86-NEXT:    leal -12(%ebp), %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl
+;
+; X64-LABEL: vector_i128_i8:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rbp
+; X64-NEXT:    pushq %r15
+; X64-NEXT:    pushq %r14
+; X64-NEXT:    pushq %r13
+; X64-NEXT:    pushq %r12
+; X64-NEXT:    pushq %rbx
+; X64-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    movdqa %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movdqa %xmm1, -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-NEXT:    divb -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movzbl %al, %eax
+; X64-NEXT:    movd %eax, %xmm2
+; X64-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-NEXT:    divb -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movzbl %al, %r8d
+; X64-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-NEXT:    divb -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movzbl %al, %r9d
+; X64-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-NEXT:    divb -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movzbl %al, %r10d
+; X64-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-NEXT:    divb -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movzbl %al, %r11d
+; X64-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-NEXT:    divb -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movzbl %al, %r14d
+; X64-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-NEXT:    divb -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movzbl %al, %r15d
+; X64-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-NEXT:    divb -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movzbl %al, %r12d
+; X64-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-NEXT:    divb -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movzbl %al, %r13d
+; X64-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-NEXT:    divb -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movzbl %al, %edi
+; X64-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-NEXT:    divb -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movzbl %al, %esi
+; X64-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-NEXT:    divb -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movzbl %al, %ebx
+; X64-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-NEXT:    divb -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movzbl %al, %ebp
+; X64-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-NEXT:    divb -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movzbl %al, %edx
+; X64-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-NEXT:    divb -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movl %eax, %ecx
+; X64-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-NEXT:    divb -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movd %r8d, %xmm3
+; X64-NEXT:    movd %r9d, %xmm4
+; X64-NEXT:    movd %r10d, %xmm5
+; X64-NEXT:    movd %r11d, %xmm6
+; X64-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; X64-NEXT:    movd %r14d, %xmm2
+; X64-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3],xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]
+; X64-NEXT:    movd %r15d, %xmm4
+; X64-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
+; X64-NEXT:    movd %r12d, %xmm3
+; X64-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm6[0],xmm2[1],xmm6[1],xmm2[2],xmm6[2],xmm2[3],xmm6[3],xmm2[4],xmm6[4],xmm2[5],xmm6[5],xmm2[6],xmm6[6],xmm2[7],xmm6[7]
+; X64-NEXT:    movd %r13d, %xmm6
+; X64-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3],xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7]
+; X64-NEXT:    movd %edi, %xmm4
+; X64-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; X64-NEXT:    movd %esi, %xmm2
+; X64-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1]
+; X64-NEXT:    movd %ebx, %xmm5
+; X64-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm6[0],xmm4[1],xmm6[1],xmm4[2],xmm6[2],xmm4[3],xmm6[3],xmm4[4],xmm6[4],xmm4[5],xmm6[5],xmm4[6],xmm6[6],xmm4[7],xmm6[7]
+; X64-NEXT:    movd %ebp, %xmm6
+; X64-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3],xmm5[4],xmm2[4],xmm5[5],xmm2[5],xmm5[6],xmm2[6],xmm5[7],xmm2[7]
+; X64-NEXT:    movd %edx, %xmm2
+; X64-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
+; X64-NEXT:    movzbl %cl, %ecx
+; X64-NEXT:    movd %ecx, %xmm4
+; X64-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm6[0],xmm2[1],xmm6[1],xmm2[2],xmm6[2],xmm2[3],xmm6[3],xmm2[4],xmm6[4],xmm2[5],xmm6[5],xmm2[6],xmm6[6],xmm2[7],xmm6[7]
+; X64-NEXT:    movzbl %al, %eax
+; X64-NEXT:    movd %eax, %xmm6
+; X64-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm4[0],xmm6[1],xmm4[1],xmm6[2],xmm4[2],xmm6[3],xmm4[3],xmm6[4],xmm4[4],xmm6[5],xmm4[5],xmm6[6],xmm4[6],xmm6[7],xmm4[7]
+; X64-NEXT:    punpcklwd {{.*#+}} xmm6 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]
+; X64-NEXT:    punpckldq {{.*#+}} xmm6 = xmm6[0],xmm5[0],xmm6[1],xmm5[1]
+; X64-NEXT:    movdqa %xmm6, %xmm2
+; X64-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; X64-NEXT:    movdqa %xmm2, (%rax)
+; X64-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; X64-NEXT:    movdqa %xmm1, %xmm2
+; X64-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
+; X64-NEXT:    pmullw %xmm3, %xmm2
+; X64-NEXT:    movdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
+; X64-NEXT:    pand %xmm3, %xmm2
+; X64-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm0[0],xmm6[1],xmm0[1],xmm6[2],xmm0[2],xmm6[3],xmm0[3],xmm6[4],xmm0[4],xmm6[5],xmm0[5],xmm6[6],xmm0[6],xmm6[7],xmm0[7]
+; X64-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; X64-NEXT:    pmullw %xmm6, %xmm1
+; X64-NEXT:    pand %xmm3, %xmm1
+; X64-NEXT:    packuswb %xmm2, %xmm1
+; X64-NEXT:    psubb %xmm1, %xmm0
+; X64-NEXT:    popq %rbx
+; X64-NEXT:    popq %r12
+; X64-NEXT:    popq %r13
+; X64-NEXT:    popq %r14
+; X64-NEXT:    popq %r15
+; X64-NEXT:    popq %rbp
+; X64-NEXT:    retq
+  %div = udiv <16 x i8> %x, %y
+  store <16 x i8> %div, <16 x i8>* %divdst, align 16
+  %t1 = mul <16 x i8> %div, %y
+  %t2 = sub <16 x i8> %x, %t1
+  ret <16 x i8> %t2
+}
+
+define <8 x i16> @vector_i128_i16(<8 x i16> %x, <8 x i16> %y, <8 x i16>* %divdst) nounwind {
+; X86-LABEL: vector_i128_i16:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    pextrw $7, %xmm0, %eax
+; X86-NEXT:    pextrw $7, %xmm1, %esi
+; X86-NEXT:    # kill: def $ax killed $ax killed $eax
+; X86-NEXT:    xorl %edx, %edx
+; X86-NEXT:    divw %si
+; X86-NEXT:    # kill: def $ax killed $ax def $eax
+; X86-NEXT:    movd %eax, %xmm2
+; X86-NEXT:    pextrw $6, %xmm0, %eax
+; X86-NEXT:    pextrw $6, %xmm1, %esi
+; X86-NEXT:    # kill: def $ax killed $ax killed $eax
+; X86-NEXT:    xorl %edx, %edx
+; X86-NEXT:    divw %si
+; X86-NEXT:    # kill: def $ax killed $ax def $eax
+; X86-NEXT:    movd %eax, %xmm3
+; X86-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; X86-NEXT:    pextrw $5, %xmm0, %eax
+; X86-NEXT:    pextrw $5, %xmm1, %esi
+; X86-NEXT:    # kill: def $ax killed $ax killed $eax
+; X86-NEXT:    xorl %edx, %edx
+; X86-NEXT:    divw %si
+; X86-NEXT:    # kill: def $ax killed $ax def $eax
+; X86-NEXT:    movd %eax, %xmm4
+; X86-NEXT:    pextrw $4, %xmm0, %eax
+; X86-NEXT:    pextrw $4, %xmm1, %esi
+; X86-NEXT:    # kill: def $ax killed $ax killed $eax
+; X86-NEXT:    xorl %edx, %edx
+; X86-NEXT:    divw %si
+; X86-NEXT:    # kill: def $ax killed $ax def $eax
+; X86-NEXT:    movd %eax, %xmm2
+; X86-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
+; X86-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; X86-NEXT:    pextrw $3, %xmm0, %eax
+; X86-NEXT:    pextrw $3, %xmm1, %esi
+; X86-NEXT:    # kill: def $ax killed $ax killed $eax
+; X86-NEXT:    xorl %edx, %edx
+; X86-NEXT:    divw %si
+; X86-NEXT:    # kill: def $ax killed $ax def $eax
+; X86-NEXT:    movd %eax, %xmm3
+; X86-NEXT:    pextrw $2, %xmm0, %eax
+; X86-NEXT:    pextrw $2, %xmm1, %esi
+; X86-NEXT:    # kill: def $ax killed $ax killed $eax
+; X86-NEXT:    xorl %edx, %edx
+; X86-NEXT:    divw %si
+; X86-NEXT:    # kill: def $ax killed $ax def $eax
+; X86-NEXT:    movd %eax, %xmm4
+; X86-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; X86-NEXT:    pextrw $1, %xmm0, %eax
+; X86-NEXT:    pextrw $1, %xmm1, %esi
+; X86-NEXT:    # kill: def $ax killed $ax killed $eax
+; X86-NEXT:    xorl %edx, %edx
+; X86-NEXT:    divw %si
+; X86-NEXT:    # kill: def $ax killed $ax def $eax
+; X86-NEXT:    movd %eax, %xmm3
+; X86-NEXT:    movd %xmm0, %eax
+; X86-NEXT:    movd %xmm1, %esi
+; X86-NEXT:    # kill: def $ax killed $ax killed $eax
+; X86-NEXT:    xorl %edx, %edx
+; X86-NEXT:    divw %si
+; X86-NEXT:    # kill: def $ax killed $ax def $eax
+; X86-NEXT:    movd %eax, %xmm5
+; X86-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
+; X86-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
+; X86-NEXT:    punpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm2[0]
+; X86-NEXT:    movdqa %xmm5, (%ecx)
+; X86-NEXT:    pmullw %xmm1, %xmm5
+; X86-NEXT:    psubw %xmm5, %xmm0
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
+;
+; X64-LABEL: vector_i128_i16:
+; X64:       # %bb.0:
+; X64-NEXT:    pextrw $7, %xmm0, %eax
+; X64-NEXT:    pextrw $7, %xmm1, %ecx
+; X64-NEXT:    # kill: def $ax killed $ax killed $eax
+; X64-NEXT:    xorl %edx, %edx
+; X64-NEXT:    divw %cx
+; X64-NEXT:    # kill: def $ax killed $ax def $eax
+; X64-NEXT:    movd %eax, %xmm2
+; X64-NEXT:    pextrw $6, %xmm0, %eax
+; X64-NEXT:    pextrw $6, %xmm1, %ecx
+; X64-NEXT:    # kill: def $ax killed $ax killed $eax
+; X64-NEXT:    xorl %edx, %edx
+; X64-NEXT:    divw %cx
+; X64-NEXT:    # kill: def $ax killed $ax def $eax
+; X64-NEXT:    movd %eax, %xmm3
+; X64-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; X64-NEXT:    pextrw $5, %xmm0, %eax
+; X64-NEXT:    pextrw $5, %xmm1, %ecx
+; X64-NEXT:    # kill: def $ax killed $ax killed $eax
+; X64-NEXT:    xorl %edx, %edx
+; X64-NEXT:    divw %cx
+; X64-NEXT:    # kill: def $ax killed $ax def $eax
+; X64-NEXT:    movd %eax, %xmm4
+; X64-NEXT:    pextrw $4, %xmm0, %eax
+; X64-NEXT:    pextrw $4, %xmm1, %ecx
+; X64-NEXT:    # kill: def $ax killed $ax killed $eax
+; X64-NEXT:    xorl %edx, %edx
+; X64-NEXT:    divw %cx
+; X64-NEXT:    # kill: def $ax killed $ax def $eax
+; X64-NEXT:    movd %eax, %xmm2
+; X64-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
+; X64-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; X64-NEXT:    pextrw $3, %xmm0, %eax
+; X64-NEXT:    pextrw $3, %xmm1, %ecx
+; X64-NEXT:    # kill: def $ax killed $ax killed $eax
+; X64-NEXT:    xorl %edx, %edx
+; X64-NEXT:    divw %cx
+; X64-NEXT:    # kill: def $ax killed $ax def $eax
+; X64-NEXT:    movd %eax, %xmm3
+; X64-NEXT:    pextrw $2, %xmm0, %eax
+; X64-NEXT:    pextrw $2, %xmm1, %ecx
+; X64-NEXT:    # kill: def $ax killed $ax killed $eax
+; X64-NEXT:    xorl %edx, %edx
+; X64-NEXT:    divw %cx
+; X64-NEXT:    # kill: def $ax killed $ax def $eax
+; X64-NEXT:    movd %eax, %xmm4
+; X64-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; X64-NEXT:    pextrw $1, %xmm0, %eax
+; X64-NEXT:    pextrw $1, %xmm1, %ecx
+; X64-NEXT:    # kill: def $ax killed $ax killed $eax
+; X64-NEXT:    xorl %edx, %edx
+; X64-NEXT:    divw %cx
+; X64-NEXT:    # kill: def $ax killed $ax def $eax
+; X64-NEXT:    movd %eax, %xmm3
+; X64-NEXT:    movd %xmm0, %eax
+; X64-NEXT:    movd %xmm1, %ecx
+; X64-NEXT:    # kill: def $ax killed $ax killed $eax
+; X64-NEXT:    xorl %edx, %edx
+; X64-NEXT:    divw %cx
+; X64-NEXT:    # kill: def $ax killed $ax def $eax
+; X64-NEXT:    movd %eax, %xmm5
+; X64-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
+; X64-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
+; X64-NEXT:    punpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm2[0]
+; X64-NEXT:    movdqa %xmm5, (%rdi)
+; X64-NEXT:    pmullw %xmm1, %xmm5
+; X64-NEXT:    psubw %xmm5, %xmm0
+; X64-NEXT:    retq
+  %div = udiv <8 x i16> %x, %y
+  store <8 x i16> %div, <8 x i16>* %divdst, align 16
+  %t1 = mul <8 x i16> %div, %y
+  %t2 = sub <8 x i16> %x, %t1
+  ret <8 x i16> %t2
+}
+
+define <4 x i32> @vector_i128_i32(<4 x i32> %x, <4 x i32> %y, <4 x i32>* %divdst) nounwind {
+; X86-LABEL: vector_i128_i32:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,1,2,3]
+; X86-NEXT:    movd %xmm2, %eax
+; X86-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,1,2,3]
+; X86-NEXT:    movd %xmm2, %esi
+; X86-NEXT:    xorl %edx, %edx
+; X86-NEXT:    divl %esi
+; X86-NEXT:    movd %eax, %xmm2
+; X86-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,0,1]
+; X86-NEXT:    movd %xmm3, %eax
+; X86-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,0,1]
+; X86-NEXT:    movd %xmm3, %esi
+; X86-NEXT:    xorl %edx, %edx
+; X86-NEXT:    divl %esi
+; X86-NEXT:    movd %eax, %xmm3
+; X86-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; X86-NEXT:    movd %xmm0, %eax
+; X86-NEXT:    movd %xmm1, %esi
+; X86-NEXT:    xorl %edx, %edx
+; X86-NEXT:    divl %esi
+; X86-NEXT:    movd %eax, %xmm4
+; X86-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[1,1,2,3]
+; X86-NEXT:    movd %xmm5, %eax
+; X86-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[1,1,2,3]
+; X86-NEXT:    movd %xmm5, %esi
+; X86-NEXT:    xorl %edx, %edx
+; X86-NEXT:    divl %esi
+; X86-NEXT:    movd %eax, %xmm5
+; X86-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
+; X86-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm3[0]
+; X86-NEXT:    movdqa %xmm4, (%ecx)
+; X86-NEXT:    pmuludq %xmm1, %xmm4
+; X86-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[0,2,2,3]
+; X86-NEXT:    shufps {{.*#+}} xmm5 = xmm5[0,0],xmm2[0,0]
+; X86-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; X86-NEXT:    pmuludq %xmm5, %xmm1
+; X86-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; X86-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; X86-NEXT:    psubd %xmm3, %xmm0
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
+;
+; X64-LABEL: vector_i128_i32:
+; X64:       # %bb.0:
+; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,1,2,3]
+; X64-NEXT:    movd %xmm2, %eax
+; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,1,2,3]
+; X64-NEXT:    movd %xmm2, %ecx
+; X64-NEXT:    xorl %edx, %edx
+; X64-NEXT:    divl %ecx
+; X64-NEXT:    movd %eax, %xmm2
+; X64-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,0,1]
+; X64-NEXT:    movd %xmm3, %eax
+; X64-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,0,1]
+; X64-NEXT:    movd %xmm3, %ecx
+; X64-NEXT:    xorl %edx, %edx
+; X64-NEXT:    divl %ecx
+; X64-NEXT:    movd %eax, %xmm3
+; X64-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; X64-NEXT:    movd %xmm0, %eax
+; X64-NEXT:    movd %xmm1, %ecx
+; X64-NEXT:    xorl %edx, %edx
+; X64-NEXT:    divl %ecx
+; X64-NEXT:    movd %eax, %xmm4
+; X64-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[1,1,2,3]
+; X64-NEXT:    movd %xmm5, %eax
+; X64-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[1,1,2,3]
+; X64-NEXT:    movd %xmm5, %ecx
+; X64-NEXT:    xorl %edx, %edx
+; X64-NEXT:    divl %ecx
+; X64-NEXT:    movd %eax, %xmm5
+; X64-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
+; X64-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm3[0]
+; X64-NEXT:    movdqa %xmm4, (%rdi)
+; X64-NEXT:    pmuludq %xmm1, %xmm4
+; X64-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[0,2,2,3]
+; X64-NEXT:    shufps {{.*#+}} xmm5 = xmm5[0,0],xmm2[0,0]
+; X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; X64-NEXT:    pmuludq %xmm5, %xmm1
+; X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; X64-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; X64-NEXT:    psubd %xmm3, %xmm0
+; X64-NEXT:    retq
+  %div = udiv <4 x i32> %x, %y
+  store <4 x i32> %div, <4 x i32>* %divdst, align 16
+  %t1 = mul <4 x i32> %div, %y
+  %t2 = sub <4 x i32> %x, %t1
+  ret <4 x i32> %t2
+}
+
+define <2 x i64> @vector_i128_i64(<2 x i64> %x, <2 x i64> %y, <2 x i64>* %divdst) nounwind {
+; X86-LABEL: vector_i128_i64:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $72, %esp
+; X86-NEXT:    movdqu %xmm1, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; X86-NEXT:    movdqu %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,1,2,3]
+; X86-NEXT:    movd %xmm2, {{[0-9]+}}(%esp)
+; X86-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; X86-NEXT:    movd %xmm2, {{[0-9]+}}(%esp)
+; X86-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,1,2,3]
+; X86-NEXT:    movd %xmm1, {{[0-9]+}}(%esp)
+; X86-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-NEXT:    movd %xmm1, (%esp)
+; X86-NEXT:    calll __udivdi3
+; X86-NEXT:    movdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm1 # 16-byte Reload
+; X86-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X86-NEXT:    movd %xmm0, {{[0-9]+}}(%esp)
+; X86-NEXT:    movd %xmm1, {{[0-9]+}}(%esp)
+; X86-NEXT:    movdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm1 # 16-byte Reload
+; X86-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X86-NEXT:    movd %xmm0, {{[0-9]+}}(%esp)
+; X86-NEXT:    movd %xmm1, (%esp)
+; X86-NEXT:    movd %edx, %xmm0
+; X86-NEXT:    movd %eax, %xmm1
+; X86-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; X86-NEXT:    movdqu %xmm1, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; X86-NEXT:    calll __udivdi3
+; X86-NEXT:    movd %edx, %xmm0
+; X86-NEXT:    movd %eax, %xmm1
+; X86-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; X86-NEXT:    movdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
+; X86-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; X86-NEXT:    movdqa %xmm1, (%esi)
+; X86-NEXT:    movdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm3 # 16-byte Reload
+; X86-NEXT:    movdqa %xmm3, %xmm0
+; X86-NEXT:    psrlq $32, %xmm0
+; X86-NEXT:    pmuludq %xmm1, %xmm0
+; X86-NEXT:    movdqa %xmm1, %xmm2
+; X86-NEXT:    psrlq $32, %xmm2
+; X86-NEXT:    pmuludq %xmm3, %xmm2
+; X86-NEXT:    paddq %xmm0, %xmm2
+; X86-NEXT:    psllq $32, %xmm2
+; X86-NEXT:    pmuludq %xmm3, %xmm1
+; X86-NEXT:    paddq %xmm2, %xmm1
+; X86-NEXT:    movdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
+; X86-NEXT:    psubq %xmm1, %xmm0
+; X86-NEXT:    addl $72, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
+;
+; X64-LABEL: vector_i128_i64:
+; X64:       # %bb.0:
+; X64-NEXT:    movq %xmm0, %rax
+; X64-NEXT:    movq %xmm1, %rcx
+; X64-NEXT:    xorl %edx, %edx
+; X64-NEXT:    divq %rcx
+; X64-NEXT:    movq %rax, %xmm2
+; X64-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,0,1]
+; X64-NEXT:    movq %xmm3, %rax
+; X64-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,0,1]
+; X64-NEXT:    movq %xmm3, %rcx
+; X64-NEXT:    xorl %edx, %edx
+; X64-NEXT:    divq %rcx
+; X64-NEXT:    movq %rax, %xmm3
+; X64-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; X64-NEXT:    movdqa %xmm2, (%rdi)
+; X64-NEXT:    movdqa %xmm1, %xmm3
+; X64-NEXT:    psrlq $32, %xmm3
+; X64-NEXT:    pmuludq %xmm2, %xmm3
+; X64-NEXT:    movdqa %xmm2, %xmm4
+; X64-NEXT:    psrlq $32, %xmm4
+; X64-NEXT:    pmuludq %xmm1, %xmm4
+; X64-NEXT:    paddq %xmm3, %xmm4
+; X64-NEXT:    psllq $32, %xmm4
+; X64-NEXT:    pmuludq %xmm1, %xmm2
+; X64-NEXT:    paddq %xmm4, %xmm2
+; X64-NEXT:    psubq %xmm2, %xmm0
+; X64-NEXT:    retq
+  %div = udiv <2 x i64> %x, %y
+  store <2 x i64> %div, <2 x i64>* %divdst, align 16
+  %t1 = mul <2 x i64> %div, %y
+  %t2 = sub <2 x i64> %x, %t1
+  ret <2 x i64> %t2
+}
+
+; Special tests.
+
+define i32 @scalar_i32_commutative(i32 %x, i32* %ysrc, i32* %divdst) nounwind {
+; X86-LABEL: scalar_i32_commutative:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl (%eax), %edi
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    xorl %edx, %edx
+; X86-NEXT:    divl %edi
+; X86-NEXT:    movl %eax, (%esi)
+; X86-NEXT:    imull %eax, %edi
+; X86-NEXT:    subl %edi, %ecx
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    retl
+;
+; X64-LABEL: scalar_i32_commutative:
+; X64:       # %bb.0:
+; X64-NEXT:    movq %rdx, %rcx
+; X64-NEXT:    movl (%rsi), %esi
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    xorl %edx, %edx
+; X64-NEXT:    divl %esi
+; X64-NEXT:    movl %eax, (%rcx)
+; X64-NEXT:    imull %eax, %esi
+; X64-NEXT:    subl %esi, %edi
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    retq
+  %y = load i32, i32* %ysrc, align 4
+  %div = udiv i32 %x, %y
+  store i32 %div, i32* %divdst, align 4
+  %t1 = mul i32 %y, %div ; commutative
+  %t2 = sub i32 %x, %t1
+  ret i32 %t2
+}
+
+; We do not care about extra uses.
+define i32 @extrause(i32 %x, i32 %y, i32* %divdst, i32* %t1dst) nounwind {
+; X86-LABEL: extrause:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    xorl %edx, %edx
+; X86-NEXT:    divl %ebx
+; X86-NEXT:    movl %eax, (%edi)
+; X86-NEXT:    imull %ebx, %eax
+; X86-NEXT:    movl %eax, (%esi)
+; X86-NEXT:    subl %eax, %ecx
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    retl
+;
+; X64-LABEL: extrause:
+; X64:       # %bb.0:
+; X64-NEXT:    movq %rdx, %r8
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    xorl %edx, %edx
+; X64-NEXT:    divl %esi
+; X64-NEXT:    movl %eax, (%r8)
+; X64-NEXT:    imull %esi, %eax
+; X64-NEXT:    movl %eax, (%rcx)
+; X64-NEXT:    subl %eax, %edi
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    retq
+  %div = udiv i32 %x, %y
+  store i32 %div, i32* %divdst, align 4
+  %t1 = mul i32 %div, %y
+  store i32 %t1, i32* %t1dst, align 4
+  %t2 = sub i32 %x, %t1
+  ret i32 %t2
+}
+
+; 'rem' should appear next to 'div'.
+define i32 @multiple_bb(i32 %x, i32 %y, i32* %divdst, i1 zeroext %store_urem, i32* %uremdst) nounwind {
+; X86-LABEL: multiple_bb:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %bl
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    xorl %edx, %edx
+; X86-NEXT:    divl %esi
+; X86-NEXT:    movl %eax, (%edi)
+; X86-NEXT:    testb %bl, %bl
+; X86-NEXT:    je .LBB10_2
+; X86-NEXT:  # %bb.1: # %do_urem
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl %eax, %edi
+; X86-NEXT:    imull %esi, %edi
+; X86-NEXT:    subl %edi, %ecx
+; X86-NEXT:    movl %ecx, (%edx)
+; X86-NEXT:  .LBB10_2: # %end
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    retl
+;
+; X64-LABEL: multiple_bb:
+; X64:       # %bb.0:
+; X64-NEXT:    movq %rdx, %r9
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    xorl %edx, %edx
+; X64-NEXT:    divl %esi
+; X64-NEXT:    movl %eax, (%r9)
+; X64-NEXT:    testb %cl, %cl
+; X64-NEXT:    je .LBB10_2
+; X64-NEXT:  # %bb.1: # %do_urem
+; X64-NEXT:    movl %eax, %ecx
+; X64-NEXT:    imull %esi, %ecx
+; X64-NEXT:    subl %ecx, %edi
+; X64-NEXT:    movl %edi, (%r8)
+; X64-NEXT:  .LBB10_2: # %end
+; X64-NEXT:    retq
+  %div = udiv i32 %x, %y
+  store i32 %div, i32* %divdst, align 4
+  br i1 %store_urem, label %do_urem, label %end
+do_urem:
+  %t1 = mul i32 %div, %y
+  %t2 = sub i32 %x, %t1
+  store i32 %t2, i32* %uremdst, align 4
+  br label %end
+end:
+  ret i32 %div
+}
+
+define i32 @negative_different_x(i32 %x0, i32 %x1, i32 %y, i32* %divdst) nounwind {
+; X86-LABEL: negative_different_x:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    xorl %edx, %edx
+; X86-NEXT:    divl %edi
+; X86-NEXT:    movl %eax, (%esi)
+; X86-NEXT:    imull %edi, %eax
+; X86-NEXT:    subl %eax, %ecx
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    retl
+;
+; X64-LABEL: negative_different_x:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edx, %r8d
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    xorl %edx, %edx
+; X64-NEXT:    divl %r8d
+; X64-NEXT:    movl %eax, (%rcx)
+; X64-NEXT:    imull %r8d, %eax
+; X64-NEXT:    subl %eax, %esi
+; X64-NEXT:    movl %esi, %eax
+; X64-NEXT:    retq
+  %div = udiv i32 %x0, %y ; not %x1
+  store i32 %div, i32* %divdst, align 4
+  %t1 = mul i32 %div, %y
+  %t2 = sub i32 %x1, %t1 ; not %x0
+  ret i32 %t2
+}
diff --git a/llvm/test/CodeGen/X86/divrem.ll b/llvm/test/CodeGen/X86/divrem.ll
index 0592dc9b3..b90ad30e5 100644
--- a/llvm/test/CodeGen/X86/divrem.ll
+++ b/llvm/test/CodeGen/X86/divrem.ll
@@ -118,8 +118,7 @@ define void @si8(i8 %x, i8 %y, i8* %p, i8* %q) nounwind {
 ; X32-LABEL: si8:
 ; X32:       # %bb.0:
 ; X32-NEXT:    pushl %ebx
-; X32-NEXT:    movb {{[0-9]+}}(%esp), %al
-; X32-NEXT:    cbtw
+; X32-NEXT:    movsbl {{[0-9]+}}(%esp), %eax
 ; X32-NEXT:    idivb {{[0-9]+}}(%esp)
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
@@ -131,9 +130,7 @@ define void @si8(i8 %x, i8 %y, i8* %p, i8* %q) nounwind {
 ;
 ; X64-LABEL: si8:
 ; X64:       # %bb.0:
-; X64-NEXT:    movl %edi, %eax
-; X64-NEXT:    # kill: def $al killed $al killed $eax
-; X64-NEXT:    cbtw
+; X64-NEXT:    movsbl %dil, %eax
 ; X64-NEXT:    idivb %sil
 ; X64-NEXT:    movsbl %ah, %esi
 ; X64-NEXT:    movb %al, (%rdx)
@@ -263,7 +260,6 @@ define void @ui8(i8 %x, i8 %y, i8* %p, i8* %q) nounwind {
 ; X32:       # %bb.0:
 ; X32-NEXT:    pushl %ebx
 ; X32-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    # kill: def $eax killed $eax def $ax
 ; X32-NEXT:    divb {{[0-9]+}}(%esp)
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
@@ -276,7 +272,6 @@ define void @ui8(i8 %x, i8 %y, i8* %p, i8* %q) nounwind {
 ; X64-LABEL: ui8:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movzbl %dil, %eax
-; X64-NEXT:    # kill: def $eax killed $eax def $ax
 ; X64-NEXT:    divb %sil
 ; X64-NEXT:    movzbl %ah, %esi
 ; X64-NEXT:    movb %al, (%rdx)
diff --git a/llvm/test/CodeGen/X86/divrem8_ext.ll b/llvm/test/CodeGen/X86/divrem8_ext.ll
index c141f914c..e54b357b4 100644
--- a/llvm/test/CodeGen/X86/divrem8_ext.ll
+++ b/llvm/test/CodeGen/X86/divrem8_ext.ll
@@ -6,7 +6,6 @@ define zeroext i8 @test_udivrem_zext_ah(i8 %x, i8 %y) {
 ; X32-LABEL: test_udivrem_zext_ah:
 ; X32:       # %bb.0:
 ; X32-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    # kill: def $eax killed $eax def $ax
 ; X32-NEXT:    divb {{[0-9]+}}(%esp)
 ; X32-NEXT:    movzbl %ah, %ecx
 ; X32-NEXT:    movb %al, z
@@ -16,7 +15,6 @@ define zeroext i8 @test_udivrem_zext_ah(i8 %x, i8 %y) {
 ; X64-LABEL: test_udivrem_zext_ah:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movzbl %dil, %eax
-; X64-NEXT:    # kill: def $eax killed $eax def $ax
 ; X64-NEXT:    divb %sil
 ; X64-NEXT:    movzbl %ah, %ecx
 ; X64-NEXT:    movb %al, {{.*}}(%rip)
@@ -32,7 +30,6 @@ define zeroext i8 @test_urem_zext_ah(i8 %x, i8 %y) {
 ; X32-LABEL: test_urem_zext_ah:
 ; X32:       # %bb.0:
 ; X32-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    # kill: def $eax killed $eax def $ax
 ; X32-NEXT:    divb {{[0-9]+}}(%esp)
 ; X32-NEXT:    movzbl %ah, %eax
 ; X32-NEXT:    # kill: def $al killed $al killed $eax
@@ -41,7 +38,6 @@ define zeroext i8 @test_urem_zext_ah(i8 %x, i8 %y) {
 ; X64-LABEL: test_urem_zext_ah:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movzbl %dil, %eax
-; X64-NEXT:    # kill: def $eax killed $eax def $ax
 ; X64-NEXT:    divb %sil
 ; X64-NEXT:    movzbl %ah, %eax
 ; X64-NEXT:    # kill: def $al killed $al killed $eax
@@ -55,7 +51,6 @@ define i8 @test_urem_noext_ah(i8 %x, i8 %y) {
 ; X32:       # %bb.0:
 ; X32-NEXT:    movb {{[0-9]+}}(%esp), %cl
 ; X32-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    # kill: def $eax killed $eax def $ax
 ; X32-NEXT:    divb %cl
 ; X32-NEXT:    movzbl %ah, %eax
 ; X32-NEXT:    addb %cl, %al
@@ -65,7 +60,6 @@ define i8 @test_urem_noext_ah(i8 %x, i8 %y) {
 ; X64-LABEL: test_urem_noext_ah:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movzbl %dil, %eax
-; X64-NEXT:    # kill: def $eax killed $eax def $ax
 ; X64-NEXT:    divb %sil
 ; X64-NEXT:    movzbl %ah, %eax
 ; X64-NEXT:    addb %sil, %al
@@ -80,7 +74,6 @@ define i64 @test_urem_zext64_ah(i8 %x, i8 %y) {
 ; X32-LABEL: test_urem_zext64_ah:
 ; X32:       # %bb.0:
 ; X32-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    # kill: def $eax killed $eax def $ax
 ; X32-NEXT:    divb {{[0-9]+}}(%esp)
 ; X32-NEXT:    movzbl %ah, %eax
 ; X32-NEXT:    xorl %edx, %edx
@@ -89,7 +82,6 @@ define i64 @test_urem_zext64_ah(i8 %x, i8 %y) {
 ; X64-LABEL: test_urem_zext64_ah:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movzbl %dil, %eax
-; X64-NEXT:    # kill: def $eax killed $eax def $ax
 ; X64-NEXT:    divb %sil
 ; X64-NEXT:    movzbl %ah, %eax
 ; X64-NEXT:    retq
@@ -101,8 +93,7 @@ define i64 @test_urem_zext64_ah(i8 %x, i8 %y) {
 define signext i8 @test_sdivrem_sext_ah(i8 %x, i8 %y) {
 ; X32-LABEL: test_sdivrem_sext_ah:
 ; X32:       # %bb.0:
-; X32-NEXT:    movb {{[0-9]+}}(%esp), %al
-; X32-NEXT:    cbtw
+; X32-NEXT:    movsbl {{[0-9]+}}(%esp), %eax
 ; X32-NEXT:    idivb {{[0-9]+}}(%esp)
 ; X32-NEXT:    movsbl %ah, %ecx
 ; X32-NEXT:    movb %al, z
@@ -111,9 +102,7 @@ define signext i8 @test_sdivrem_sext_ah(i8 %x, i8 %y) {
 ;
 ; X64-LABEL: test_sdivrem_sext_ah:
 ; X64:       # %bb.0:
-; X64-NEXT:    movl %edi, %eax
-; X64-NEXT:    # kill: def $al killed $al killed $eax
-; X64-NEXT:    cbtw
+; X64-NEXT:    movsbl %dil, %eax
 ; X64-NEXT:    idivb %sil
 ; X64-NEXT:    movsbl %ah, %ecx
 ; X64-NEXT:    movb %al, {{.*}}(%rip)
@@ -128,8 +117,7 @@ define signext i8 @test_sdivrem_sext_ah(i8 %x, i8 %y) {
 define signext i8 @test_srem_sext_ah(i8 %x, i8 %y) {
 ; X32-LABEL: test_srem_sext_ah:
 ; X32:       # %bb.0:
-; X32-NEXT:    movb {{[0-9]+}}(%esp), %al
-; X32-NEXT:    cbtw
+; X32-NEXT:    movsbl {{[0-9]+}}(%esp), %eax
 ; X32-NEXT:    idivb {{[0-9]+}}(%esp)
 ; X32-NEXT:    movsbl %ah, %eax
 ; X32-NEXT:    # kill: def $al killed $al killed $eax
@@ -137,9 +125,7 @@ define signext i8 @test_srem_sext_ah(i8 %x, i8 %y) {
 ;
 ; X64-LABEL: test_srem_sext_ah:
 ; X64:       # %bb.0:
-; X64-NEXT:    movl %edi, %eax
-; X64-NEXT:    # kill: def $al killed $al killed $eax
-; X64-NEXT:    cbtw
+; X64-NEXT:    movsbl %dil, %eax
 ; X64-NEXT:    idivb %sil
 ; X64-NEXT:    movsbl %ah, %eax
 ; X64-NEXT:    # kill: def $al killed $al killed $eax
@@ -151,9 +137,8 @@ define signext i8 @test_srem_sext_ah(i8 %x, i8 %y) {
 define i8 @test_srem_noext_ah(i8 %x, i8 %y) {
 ; X32-LABEL: test_srem_noext_ah:
 ; X32:       # %bb.0:
-; X32-NEXT:    movb {{[0-9]+}}(%esp), %al
-; X32-NEXT:    cbtw
 ; X32-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X32-NEXT:    movsbl {{[0-9]+}}(%esp), %eax
 ; X32-NEXT:    idivb %cl
 ; X32-NEXT:    movsbl %ah, %eax
 ; X32-NEXT:    addb %cl, %al
@@ -162,9 +147,7 @@ define i8 @test_srem_noext_ah(i8 %x, i8 %y) {
 ;
 ; X64-LABEL: test_srem_noext_ah:
 ; X64:       # %bb.0:
-; X64-NEXT:    movl %edi, %eax
-; X64-NEXT:    # kill: def $al killed $al killed $eax
-; X64-NEXT:    cbtw
+; X64-NEXT:    movsbl %dil, %eax
 ; X64-NEXT:    idivb %sil
 ; X64-NEXT:    movsbl %ah, %eax
 ; X64-NEXT:    addb %sil, %al
@@ -178,8 +161,7 @@ define i8 @test_srem_noext_ah(i8 %x, i8 %y) {
 define i64 @test_srem_sext64_ah(i8 %x, i8 %y) {
 ; X32-LABEL: test_srem_sext64_ah:
 ; X32:       # %bb.0:
-; X32-NEXT:    movb {{[0-9]+}}(%esp), %al
-; X32-NEXT:    cbtw
+; X32-NEXT:    movsbl {{[0-9]+}}(%esp), %eax
 ; X32-NEXT:    idivb {{[0-9]+}}(%esp)
 ; X32-NEXT:    movsbl %ah, %eax
 ; X32-NEXT:    movl %eax, %edx
@@ -188,9 +170,7 @@ define i64 @test_srem_sext64_ah(i8 %x, i8 %y) {
 ;
 ; X64-LABEL: test_srem_sext64_ah:
 ; X64:       # %bb.0:
-; X64-NEXT:    movl %edi, %eax
-; X64-NEXT:    # kill: def $al killed $al killed $eax
-; X64-NEXT:    cbtw
+; X64-NEXT:    movsbl %dil, %eax
 ; X64-NEXT:    idivb %sil
 ; X64-NEXT:    movsbl %ah, %eax
 ; X64-NEXT:    cltq
@@ -204,7 +184,6 @@ define i64 @pr25754(i8 %a, i8 %c) {
 ; X32-LABEL: pr25754:
 ; X32:       # %bb.0:
 ; X32-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    # kill: def $eax killed $eax def $ax
 ; X32-NEXT:    divb {{[0-9]+}}(%esp)
 ; X32-NEXT:    movzbl %ah, %ecx
 ; X32-NEXT:    movzbl %al, %eax
@@ -215,7 +194,6 @@ define i64 @pr25754(i8 %a, i8 %c) {
 ; X64-LABEL: pr25754:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movzbl %dil, %eax
-; X64-NEXT:    # kill: def $eax killed $eax def $ax
 ; X64-NEXT:    divb %sil
 ; X64-NEXT:    movzbl %ah, %ecx
 ; X64-NEXT:    movzbl %al, %eax
diff --git a/llvm/test/CodeGen/X86/dont-trunc-store-double-to-float.ll b/llvm/test/CodeGen/X86/dont-trunc-store-double-to-float.ll
index e9287b8b9..e6781f27f 100644
--- a/llvm/test/CodeGen/X86/dont-trunc-store-double-to-float.ll
+++ b/llvm/test/CodeGen/X86/dont-trunc-store-double-to-float.ll
@@ -1,10 +1,23 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=i686-- < %s | FileCheck %s
 
-; CHECK-LABEL: @bar
-; CHECK-DAG: movl $1074339512,
-; CHECK-DAG: movl $1374389535,
-; CHECK-DAG: movl $1078523331,
 define void @bar() unnamed_addr {
+; CHECK-LABEL: bar:
+; CHECK:       # %bb.0: # %entry-block
+; CHECK-NEXT:    pushl %ebp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    .cfi_offset %ebp, -8
+; CHECK-NEXT:    movl %esp, %ebp
+; CHECK-NEXT:    .cfi_def_cfa_register %ebp
+; CHECK-NEXT:    andl $-8, %esp
+; CHECK-NEXT:    subl $16, %esp
+; CHECK-NEXT:    movl $1074339512, {{[0-9]+}}(%esp) # imm = 0x40091EB8
+; CHECK-NEXT:    movl $1374389535, (%esp) # imm = 0x51EB851F
+; CHECK-NEXT:    movl $1078523331, {{[0-9]+}}(%esp) # imm = 0x4048F5C3
+; CHECK-NEXT:    movl %ebp, %esp
+; CHECK-NEXT:    popl %ebp
+; CHECK-NEXT:    .cfi_def_cfa %esp, 4
+; CHECK-NEXT:    retl
 entry-block:
   %a = alloca double
   %b = alloca float
diff --git a/llvm/test/CodeGen/X86/dtor-priority-coff.ll b/llvm/test/CodeGen/X86/dtor-priority-coff.ll
new file mode 100644
index 000000000..ad2c02764
--- /dev/null
+++ b/llvm/test/CodeGen/X86/dtor-priority-coff.ll
@@ -0,0 +1,70 @@
+; RUN: llc < %s | FileCheck %s
+
+; Check that we come up with appropriate section names that link.exe sorts
+; well.
+
+; CHECK: .section        .CRT$XTA00042,"dr"
+; CHECK: .p2align        3
+; CHECK: .quad   f
+; CHECK: .section        .CRT$XTT12345,"dr"
+; CHECK: .p2align        3
+; CHECK: .quad   g
+; CHECK: .section        .CRT$XTT23456,"dr",associative,h
+; CHECK: .p2align        3
+; CHECK: .quad   init_h
+; CHECK: .section        .CRT$XTX,"dr"
+; CHECK: .p2align        3
+; CHECK: .quad   str3_dtor
+
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc19.14.26433"
+
+$h = comdat any
+
+@h = linkonce_odr global i8 55, comdat, align 1
+
+@str0 = private dso_local unnamed_addr constant [6 x i8] c"later\00", align 1
+@str1 = private dso_local unnamed_addr constant [6 x i8] c"first\00", align 1
+@str2 = private dso_local unnamed_addr constant [5 x i8] c"main\00", align 1
+@str3 = private dso_local unnamed_addr constant [8 x i8] c"default\00", align 1
+
+@llvm.global_dtors = appending global [4 x { i32, void ()*, i8* }] [
+  { i32, void ()*, i8* } { i32 12345, void ()* @g, i8* null },
+  { i32, void ()*, i8* } { i32 42, void ()* @f, i8* null },
+  { i32, void ()*, i8* } { i32 23456, void ()* @init_h, i8* @h },
+  { i32, void ()*, i8* } { i32 65535, void ()* @str3_dtor, i8* null }
+]
+
+declare dso_local i32 @puts(i8* nocapture readonly) local_unnamed_addr
+
+define dso_local void @g() {
+entry:
+  %call = tail call i32 @puts(i8* getelementptr inbounds ([6 x i8], [6 x i8]* @str0, i64 0, i64 0))
+  ret void
+}
+
+define dso_local void @f() {
+entry:
+  %call = tail call i32 @puts(i8* getelementptr inbounds ([6 x i8], [6 x i8]* @str1, i64 0, i64 0))
+  ret void
+}
+
+define dso_local void @str3_dtor() {
+entry:
+  %call = tail call i32 @puts(i8* getelementptr inbounds ([8 x i8], [8 x i8]* @str3, i64 0, i64 0))
+  ret void
+}
+
+define dso_local void @init_h() {
+entry:
+  store i8 42, i8* @h
+  ret void
+}
+
+
+; Function Attrs: nounwind uwtable
+define dso_local i32 @main() local_unnamed_addr {
+entry:
+  %call = tail call i32 @puts(i8* getelementptr inbounds ([5 x i8], [5 x i8]* @str2, i64 0, i64 0))
+  ret i32 0
+}
diff --git a/llvm/test/CodeGen/X86/dwarf-headers.ll b/llvm/test/CodeGen/X86/dwarf-headers.ll
index ef626ad70..6159fc29f 100644
--- a/llvm/test/CodeGen/X86/dwarf-headers.ll
+++ b/llvm/test/CodeGen/X86/dwarf-headers.ll
@@ -75,7 +75,7 @@
 ; O-5: .debug_info contents:
 ; O-5: 0x00000000: Compile Unit: {{.*}} version = 0x0005 unit_type = DW_UT_skeleton abbr_offset
 ; O-5-SAME:        DWO_id = 0xccd7e58ef8bf4aa6
-; O-5: 0x00000014: DW_TAG_compile_unit
+; O-5: 0x00000014: DW_TAG_skeleton_unit 
 ;
 ; DWO-5: .debug_info.dwo contents:
 ; DWO-5: 0x00000000: Type Unit: {{.*}} version = 0x0005 unit_type = DW_UT_split_type abbr_offset
diff --git a/llvm/test/CodeGen/X86/early-cfi-sections.ll b/llvm/test/CodeGen/X86/early-cfi-sections.ll
index 4adfe8502..3a9e62a1c 100644
--- a/llvm/test/CodeGen/X86/early-cfi-sections.ll
+++ b/llvm/test/CodeGen/X86/early-cfi-sections.ll
@@ -12,7 +12,7 @@ entry:
   ret void, !dbg !8
 }
 
-attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!3, !4}
diff --git a/llvm/test/CodeGen/X86/eh-frame-unreachable.ll b/llvm/test/CodeGen/X86/eh-frame-unreachable.ll
index 7cdb06858..4558c7785 100644
--- a/llvm/test/CodeGen/X86/eh-frame-unreachable.ll
+++ b/llvm/test/CodeGen/X86/eh-frame-unreachable.ll
@@ -9,4 +9,4 @@
 define void @f() #0 {
   unreachable
 }
-attributes #0 = { "no-frame-pointer-elim"="true" }
+attributes #0 = { "frame-pointer"="all" }
diff --git a/llvm/test/CodeGen/X86/elf-associated.ll b/llvm/test/CodeGen/X86/elf-associated.ll
index 7d58c3437..e0e9e0058 100644
--- a/llvm/test/CodeGen/X86/elf-associated.ll
+++ b/llvm/test/CodeGen/X86/elf-associated.ll
@@ -33,7 +33,7 @@
 ; CHECK-DAG: .section	bbb,"awo",@progbits,h,unique,2
 ; CHECK-DAG: .section	.data.k,"awo",@progbits,h
 
-; Non-GlobalObject metadata.
+; Non-GlobalValue metadata.
 @l = global i32 1, section "ccc", !associated !5
 !5 = !{i32* null}
 ; CHECK-DAG: .section	ccc,"aw",@progbits
@@ -42,3 +42,9 @@
 @m = global i32 1, section "ddd", !associated !6
 !6 = distinct !{null}
 ; CHECK-DAG: .section	ddd,"aw",@progbits
+
+; Aliases are OK.
+@n = alias i32, i32* inttoptr (i64 add (i64 ptrtoint (i32* @a to i64), i64 1297036692682702848) to i32*)
+@o = global i32 1, section "eee", !associated !7
+!7 = !{i32* @n}
+; CHECK-DAG: .section	eee,"awo",@progbits,n,unique,3
diff --git a/llvm/test/CodeGen/X86/empty-function.ll b/llvm/test/CodeGen/X86/empty-function.ll
index 92bebd0ab..7d908311e 100644
--- a/llvm/test/CodeGen/X86/empty-function.ll
+++ b/llvm/test/CodeGen/X86/empty-function.ll
@@ -15,7 +15,7 @@ entry:
 
 ; CHECK-LABEL: f:
 ; WIN32: nop
-; WIN64: ud2
+; WIN64: nop
 ; LINUX-NOT: nop
 ; LINUX-NOT: ud2
 
diff --git a/llvm/test/CodeGen/X86/empty-struct-return-type.ll b/llvm/test/CodeGen/X86/empty-struct-return-type.ll
index 1bfc2f7f4..4bfc19ec4 100644
--- a/llvm/test/CodeGen/X86/empty-struct-return-type.ll
+++ b/llvm/test/CodeGen/X86/empty-struct-return-type.ll
@@ -1,4 +1,5 @@
-; RUN: llc < %s | grep call
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s | FileCheck %s
 ; PR4688
 
 ; Return types can be empty structs, which can be awkward.
@@ -7,6 +8,14 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3
 target triple = "x86_64-unknown-linux-gnu"
 
 define void @_ZN15QtSharedPointer22internalSafetyCheckAddEPVKv(i8* %ptr) {
+; CHECK-LABEL: _ZN15QtSharedPointer22internalSafetyCheckAddEPVKv:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    callq _ZNK5QHashIPv15QHashDummyValueE5valueERKS0_
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
 entry:
 	%0 = call { } @_ZNK5QHashIPv15QHashDummyValueE5valueERKS0_(i8** undef)		; <{ }> [#uses=0]
         ret void
diff --git a/llvm/test/CodeGen/X86/epilogue-cfi-fp.ll b/llvm/test/CodeGen/X86/epilogue-cfi-fp.ll
index 9ccedb97f..972db676c 100644
--- a/llvm/test/CodeGen/X86/epilogue-cfi-fp.ll
+++ b/llvm/test/CodeGen/X86/epilogue-cfi-fp.ll
@@ -25,5 +25,5 @@ entry:
   ret i32 0
 }
 
-attributes #0 = { "no-frame-pointer-elim"="true" }
+attributes #0 = { "frame-pointer"="all" }
 
diff --git a/llvm/test/CodeGen/X86/exedepsfix-broadcast.ll b/llvm/test/CodeGen/X86/exedepsfix-broadcast.ll
index 2fcbdd39f..83b1dbb7a 100644
--- a/llvm/test/CodeGen/X86/exedepsfix-broadcast.ll
+++ b/llvm/test/CodeGen/X86/exedepsfix-broadcast.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-apple-macosx -mattr=+avx2 -enable-unsafe-fp-math | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-apple-macosx -mattr=+avx2 -enable-no-signed-zeros-fp-math | FileCheck %s
 
 ; Check that the ExeDepsFix pass correctly fixes the domain for broadcast instructions.
 ; <rdar://problem/16354675>
diff --git a/llvm/test/CodeGen/X86/extend.ll b/llvm/test/CodeGen/X86/extend.ll
index 399d05eaa..0c1de7950 100644
--- a/llvm/test/CodeGen/X86/extend.ll
+++ b/llvm/test/CodeGen/X86/extend.ll
@@ -1,16 +1,26 @@
-; RUN: llc < %s -mtriple=i686-- -x86-asm-syntax=intel | grep movzx | count 1
-; RUN: llc < %s -mtriple=i686-- -x86-asm-syntax=intel | grep movsx | count 1
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- -x86-asm-syntax=intel | FileCheck %s
 
 @G1 = internal global i8 0              ; <i8*> [#uses=1]
 @G2 = internal global i8 0              ; <i8*> [#uses=1]
 
 define i16 @test1() {
+; CHECK-LABEL: test1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movzx eax, byte ptr [G1]
+; CHECK-NEXT:    # kill: def $ax killed $ax killed $eax
+; CHECK-NEXT:    ret
         %tmp.0 = load i8, i8* @G1           ; <i8> [#uses=1]
         %tmp.3 = zext i8 %tmp.0 to i16          ; <i16> [#uses=1]
         ret i16 %tmp.3
 }
 
 define i16 @test2() {
+; CHECK-LABEL: test2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movsx eax, byte ptr [G2]
+; CHECK-NEXT:    # kill: def $ax killed $ax killed $eax
+; CHECK-NEXT:    ret
         %tmp.0 = load i8, i8* @G2           ; <i8> [#uses=1]
         %tmp.3 = sext i8 %tmp.0 to i16          ; <i16> [#uses=1]
         ret i16 %tmp.3
diff --git a/llvm/test/CodeGen/X86/extmul128.ll b/llvm/test/CodeGen/X86/extmul128.ll
index 1ccdbb550..09ad58029 100644
--- a/llvm/test/CodeGen/X86/extmul128.ll
+++ b/llvm/test/CodeGen/X86/extmul128.ll
@@ -1,12 +1,23 @@
-; RUN: llc < %s -mtriple=x86_64-- | grep mul | count 2
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-- | FileCheck %s
 
 define i128 @i64_sext_i128(i64 %a, i64 %b) {
+; CHECK-LABEL: i64_sext_i128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    imulq %rsi
+; CHECK-NEXT:    retq
   %aa = sext i64 %a to i128
   %bb = sext i64 %b to i128
   %cc = mul i128 %aa, %bb
   ret i128 %cc
 }
 define i128 @i64_zext_i128(i64 %a, i64 %b) {
+; CHECK-LABEL: i64_zext_i128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    mulq %rsi
+; CHECK-NEXT:    retq
   %aa = zext i64 %a to i128
   %bb = zext i64 %b to i128
   %cc = mul i128 %aa, %bb
diff --git a/llvm/test/CodeGen/X86/extmul64.ll b/llvm/test/CodeGen/X86/extmul64.ll
index 7e3d2fca7..5cf0eee86 100644
--- a/llvm/test/CodeGen/X86/extmul64.ll
+++ b/llvm/test/CodeGen/X86/extmul64.ll
@@ -1,12 +1,23 @@
-; RUN: llc < %s -mtriple=i686-- | grep mul | count 2
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- | FileCheck %s
 
 define i64 @i32_sext_i64(i32 %a, i32 %b) {
+; CHECK-LABEL: i32_sext_i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    imull {{[0-9]+}}(%esp)
+; CHECK-NEXT:    retl
   %aa = sext i32 %a to i64
   %bb = sext i32 %b to i64
   %cc = mul i64 %aa, %bb
   ret i64 %cc
 }
 define i64 @i32_zext_i64(i32 %a, i32 %b) {
+; CHECK-LABEL: i32_zext_i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    mull {{[0-9]+}}(%esp)
+; CHECK-NEXT:    retl
   %aa = zext i32 %a to i64
   %bb = zext i32 %b to i64
   %cc = mul i64 %aa, %bb
diff --git a/llvm/test/CodeGen/X86/extract-combine.ll b/llvm/test/CodeGen/X86/extract-combine.ll
index 7b38a0157..78c2c1e2d 100644
--- a/llvm/test/CodeGen/X86/extract-combine.ll
+++ b/llvm/test/CodeGen/X86/extract-combine.ll
@@ -1,7 +1,11 @@
-; RUN: llc < %s -mtriple=x86_64-- -mcpu=core2 -o %t
-; RUN: not grep unpcklps %t
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=core2 | FileCheck %s --implicit-check-not unpcklps
 
 define i32 @foo() nounwind {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xorps %xmm0, %xmm0
+; CHECK-NEXT:    movaps %xmm0, 0
 entry:
 	%tmp74.i25762 = shufflevector <16 x float> zeroinitializer, <16 x float> zeroinitializer, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>		; <<16 x float>> [#uses=1]
 	%tmp518 = shufflevector <16 x float> %tmp74.i25762, <16 x float> undef, <4 x i32> <i32 12, i32 13, i32 14, i32 15>		; <<4 x float>> [#uses=1]
diff --git a/llvm/test/CodeGen/X86/extract-concat.ll b/llvm/test/CodeGen/X86/extract-concat.ll
index 029c69a34..cd8c4d1e2 100644
--- a/llvm/test/CodeGen/X86/extract-concat.ll
+++ b/llvm/test/CodeGen/X86/extract-concat.ll
@@ -1,15 +1,37 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mcpu=corei7 -mtriple=x86_64-unknown-linux-gnu | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-- -mattr=sse4.2  | FileCheck %s --check-prefixes=SSE42
+; RUN: llc < %s -mtriple=x86_64-- -mattr=avx     | FileCheck %s --check-prefixes=AVX,AVX1
+; RUN: llc < %s -mtriple=x86_64-- -mattr=avx2    | FileCheck %s --check-prefixes=AVX,AVX2
+; RUN: llc < %s -mtriple=x86_64-- -mattr=avx512f | FileCheck %s --check-prefixes=AVX,AVX512F
 
 define void @foo(<4 x float> %in, <4 x i8>* %out) {
-; CHECK-LABEL: foo:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    cvttps2dq %xmm0, %xmm0
-; CHECK-NEXT:    movl $255, %eax
-; CHECK-NEXT:    pinsrd $3, %eax, %xmm0
-; CHECK-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
-; CHECK-NEXT:    movd %xmm0, (%rdi)
-; CHECK-NEXT:    retq
+; SSE42-LABEL: foo:
+; SSE42:       # %bb.0:
+; SSE42-NEXT:    cvttps2dq %xmm0, %xmm0
+; SSE42-NEXT:    pextrb $8, %xmm0, %eax
+; SSE42-NEXT:    pextrb $4, %xmm0, %ecx
+; SSE42-NEXT:    pextrb $0, %xmm0, %edx
+; SSE42-NEXT:    movd %edx, %xmm0
+; SSE42-NEXT:    pinsrb $1, %ecx, %xmm0
+; SSE42-NEXT:    pinsrb $2, %eax, %xmm0
+; SSE42-NEXT:    movl $255, %eax
+; SSE42-NEXT:    pinsrb $3, %eax, %xmm0
+; SSE42-NEXT:    movd %xmm0, (%rdi)
+; SSE42-NEXT:    retq
+;
+; AVX-LABEL: foo:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX-NEXT:    vpextrb $8, %xmm0, %eax
+; AVX-NEXT:    vpextrb $4, %xmm0, %ecx
+; AVX-NEXT:    vpextrb $0, %xmm0, %edx
+; AVX-NEXT:    vmovd %edx, %xmm0
+; AVX-NEXT:    vpinsrb $1, %ecx, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $2, %eax, %xmm0, %xmm0
+; AVX-NEXT:    movl $255, %eax
+; AVX-NEXT:    vpinsrb $3, %eax, %xmm0, %xmm0
+; AVX-NEXT:    vmovd %xmm0, (%rdi)
+; AVX-NEXT:    retq
   %t0 = fptosi <4 x float> %in to <4 x i32>
   %t1 = trunc <4 x i32> %t0 to <4 x i16>
   %t2 = shufflevector <4 x i16> %t1, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -19,3 +41,131 @@ define void @foo(<4 x float> %in, <4 x i8>* %out) {
   store <4 x i8> %t5, <4 x i8>* %out
   ret void
 }
+
+define <16 x i64> @catcat(<4 x i64> %x) {
+; SSE42-LABEL: catcat:
+; SSE42:       # %bb.0:
+; SSE42-NEXT:    movq %rdi, %rax
+; SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,1,0,1]
+; SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; SSE42-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,1,0,1]
+; SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
+; SSE42-NEXT:    movdqa %xmm1, 112(%rdi)
+; SSE42-NEXT:    movdqa %xmm1, 96(%rdi)
+; SSE42-NEXT:    movdqa %xmm3, 80(%rdi)
+; SSE42-NEXT:    movdqa %xmm3, 64(%rdi)
+; SSE42-NEXT:    movdqa %xmm0, 48(%rdi)
+; SSE42-NEXT:    movdqa %xmm0, 32(%rdi)
+; SSE42-NEXT:    movdqa %xmm2, 16(%rdi)
+; SSE42-NEXT:    movdqa %xmm2, (%rdi)
+; SSE42-NEXT:    retq
+;
+; AVX1-LABEL: catcat:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vmovddup {{.*#+}} ymm1 = ymm0[0,0,2,2]
+; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm2 = ymm1[2,3,2,3]
+; AVX1-NEXT:    vpermilpd {{.*#+}} ymm1 = ymm0[1,1,3,3]
+; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm3 = ymm1[2,3,2,3]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[0,1,0,1]
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm1, %ymm4
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm1
+; AVX1-NEXT:    vmovaps %ymm4, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: catcat:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpermpd {{.*#+}} ymm1 = ymm0[1,1,1,1]
+; AVX2-NEXT:    vpermpd {{.*#+}} ymm2 = ymm0[2,2,2,2]
+; AVX2-NEXT:    vpermpd {{.*#+}} ymm3 = ymm0[3,3,3,3]
+; AVX2-NEXT:    vbroadcastsd %xmm0, %ymm0
+; AVX2-NEXT:    retq
+;
+; AVX512F-LABEL: catcat:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512F-NEXT:    vmovaps {{.*#+}} zmm1 = [0,0,0,0,1,1,1,1]
+; AVX512F-NEXT:    vpermpd %zmm0, %zmm1, %zmm2
+; AVX512F-NEXT:    vmovaps {{.*#+}} zmm1 = [2,2,2,2,3,3,3,3]
+; AVX512F-NEXT:    vpermpd %zmm0, %zmm1, %zmm1
+; AVX512F-NEXT:    vmovaps %zmm2, %zmm0
+; AVX512F-NEXT:    retq
+  %cat1 = shufflevector <4 x i64> %x, <4 x i64> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+  %cat2 = shufflevector <8 x i64> %cat1, <8 x i64> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %r = shufflevector <16 x i64> %cat2, <16 x i64> undef, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 1, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
+  ret  <16 x i64> %r
+}
+
+define <16 x i64> @load_catcat(<4 x i64>* %p) {
+; SSE42-LABEL: load_catcat:
+; SSE42:       # %bb.0:
+; SSE42-NEXT:    movq %rdi, %rax
+; SSE42-NEXT:    movdqa (%rsi), %xmm0
+; SSE42-NEXT:    movdqa 16(%rsi), %xmm1
+; SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,1,0,1]
+; SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; SSE42-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,1,0,1]
+; SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
+; SSE42-NEXT:    movdqa %xmm1, 112(%rdi)
+; SSE42-NEXT:    movdqa %xmm1, 96(%rdi)
+; SSE42-NEXT:    movdqa %xmm3, 80(%rdi)
+; SSE42-NEXT:    movdqa %xmm3, 64(%rdi)
+; SSE42-NEXT:    movdqa %xmm0, 48(%rdi)
+; SSE42-NEXT:    movdqa %xmm0, 32(%rdi)
+; SSE42-NEXT:    movdqa %xmm2, 16(%rdi)
+; SSE42-NEXT:    movdqa %xmm2, (%rdi)
+; SSE42-NEXT:    retq
+;
+; AVX1-LABEL: load_catcat:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vbroadcastsd (%rdi), %ymm0
+; AVX1-NEXT:    vbroadcastsd 8(%rdi), %ymm1
+; AVX1-NEXT:    vbroadcastsd 16(%rdi), %ymm2
+; AVX1-NEXT:    vbroadcastsd 24(%rdi), %ymm3
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: load_catcat:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vmovaps (%rdi), %ymm3
+; AVX2-NEXT:    vbroadcastsd (%rdi), %ymm0
+; AVX2-NEXT:    vpermpd {{.*#+}} ymm1 = ymm3[1,1,1,1]
+; AVX2-NEXT:    vpermpd {{.*#+}} ymm2 = ymm3[2,2,2,2]
+; AVX2-NEXT:    vpermpd {{.*#+}} ymm3 = ymm3[3,3,3,3]
+; AVX2-NEXT:    retq
+;
+; AVX512F-LABEL: load_catcat:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm1 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [0,4,0,4,1,5,1,5]
+; AVX512F-NEXT:    vpermq %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [2,6,2,6,3,7,3,7]
+; AVX512F-NEXT:    vpermq %zmm1, %zmm2, %zmm1
+; AVX512F-NEXT:    retq
+  %x = load <4 x i64>, <4 x i64>* %p
+  %cat1 = shufflevector <4 x i64> %x, <4 x i64> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+  %cat2 = shufflevector <8 x i64> %cat1, <8 x i64> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %r = shufflevector <16 x i64> %cat2, <16 x i64> undef, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 1, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
+  ret  <16 x i64> %r
+}
+
+; Use weird types to make sure we do not miscompile a case where
+; the source ops are not an even multiple size of the result.
+
+define <4 x i32> @cat_ext_straddle(<6 x i32>* %px, <6 x i32>* %py) {
+; SSE42-LABEL: cat_ext_straddle:
+; SSE42:       # %bb.0:
+; SSE42-NEXT:    movaps 16(%rdi), %xmm0
+; SSE42-NEXT:    unpcklpd {{.*#+}} xmm0 = xmm0[0],mem[0]
+; SSE42-NEXT:    retq
+;
+; AVX-LABEL: cat_ext_straddle:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovaps 16(%rdi), %xmm0
+; AVX-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],mem[0]
+; AVX-NEXT:    retq
+  %x = load <6 x i32>, <6 x i32>* %px
+  %y = load <6 x i32>, <6 x i32>* %py
+  %cat = shufflevector <6 x i32> %x, <6 x i32> %y, <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
+  %ext = shufflevector <12 x i32> %cat, <12 x i32> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  ret <4 x i32> %ext
+}
diff --git a/llvm/test/CodeGen/X86/extract-extract.ll b/llvm/test/CodeGen/X86/extract-extract.ll
index aeb3566cb..88c735e2a 100644
--- a/llvm/test/CodeGen/X86/extract-extract.ll
+++ b/llvm/test/CodeGen/X86/extract-extract.ll
@@ -1,4 +1,5 @@
-; RUN: llc < %s -mtriple=i686-- >/dev/null
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- | FileCheck %s
 ; PR4699
 
 ; Handle this extractvalue-of-extractvalue case without getting in
@@ -10,6 +11,9 @@
         %pp = type { %cc }
 
 define fastcc void @foo(%pp* nocapture byval %p_arg) {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    retl
 entry:
         %tmp2 = getelementptr %pp, %pp* %p_arg, i64 0, i32 0         ; <%cc*> [#uses=
         %tmp3 = load %cc, %cc* %tmp2         ; <%cc> [#uses=1]
diff --git a/llvm/test/CodeGen/X86/extract-insert.ll b/llvm/test/CodeGen/X86/extract-insert.ll
index 97ca884a5..0a35d78ef 100644
--- a/llvm/test/CodeGen/X86/extract-insert.ll
+++ b/llvm/test/CodeGen/X86/extract-insert.ll
@@ -31,12 +31,10 @@ define i8 @extractelt_bitcast(i32 %x) nounwind {
 define i8 @extractelt_bitcast_extra_use(i32 %x, <4 x i8>* %p) nounwind {
 ; X86-LABEL: extractelt_bitcast_extra_use:
 ; X86:       # %bb.0:
-; X86-NEXT:    pushl %eax
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl %eax, (%ecx)
 ; X86-NEXT:    # kill: def $al killed $al killed $eax
-; X86-NEXT:    popl %ecx
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: extractelt_bitcast_extra_use:
diff --git a/llvm/test/CodeGen/X86/extract-store.ll b/llvm/test/CodeGen/X86/extract-store.ll
index 4d557c942..c29fac61a 100644
--- a/llvm/test/CodeGen/X86/extract-store.ll
+++ b/llvm/test/CodeGen/X86/extract-store.ll
@@ -5,8 +5,8 @@
 ; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=X64 --check-prefix=SSE-X64 --check-prefix=SSE41-X64
 ; RUN: llc < %s -mtriple=i686-unknown -mattr=+avx      | FileCheck %s --check-prefix=X32 --check-prefix=AVX-X32
 ; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx    | FileCheck %s --check-prefix=X64 --check-prefix=AVX-X64
-; RUN: llc < %s -O2 -mtriple=x86_64-linux-android -mattr=+mmx -enable-legalize-types-checking | FileCheck %s --check-prefix=X64 --check-prefix=SSE-X64 --check-prefix=SSE-F128
-; RUN: llc < %s -O2 -mtriple=x86_64-linux-gnu -mattr=+mmx -enable-legalize-types-checking | FileCheck %s --check-prefix=X64 --check-prefix=SSE-X64 --check-prefix=SSE-F128
+; RUN: llc < %s -O2 -mtriple=x86_64-linux-android -mattr=+sse -enable-legalize-types-checking | FileCheck %s --check-prefix=X64 --check-prefix=SSE-X64 --check-prefix=SSE2-X64
+; RUN: llc < %s -O2 -mtriple=x86_64-linux-gnu -mattr=+sse -enable-legalize-types-checking | FileCheck %s --check-prefix=X64 --check-prefix=SSE-X64 --check-prefix=SSE2-X64
 
 define void @extract_i8_0(i8* nocapture %dst, <16 x i8> %foo) nounwind {
 ; SSE2-X32-LABEL: extract_i8_0:
@@ -43,12 +43,6 @@ define void @extract_i8_0(i8* nocapture %dst, <16 x i8> %foo) nounwind {
 ; AVX-X64:       # %bb.0:
 ; AVX-X64-NEXT:    vpextrb $0, %xmm0, (%rdi)
 ; AVX-X64-NEXT:    retq
-;
-; SSE-F128-LABEL: extract_i8_0:
-; SSE-F128:       # %bb.0:
-; SSE-F128-NEXT:    movd %xmm0, %eax
-; SSE-F128-NEXT:    movb %al, (%rdi)
-; SSE-F128-NEXT:    retq
   %vecext = extractelement <16 x i8> %foo, i32 0
   store i8 %vecext, i8* %dst, align 1
   ret void
@@ -91,13 +85,6 @@ define void @extract_i8_3(i8* nocapture %dst, <16 x i8> %foo) nounwind {
 ; AVX-X64:       # %bb.0:
 ; AVX-X64-NEXT:    vpextrb $3, %xmm0, (%rdi)
 ; AVX-X64-NEXT:    retq
-;
-; SSE-F128-LABEL: extract_i8_3:
-; SSE-F128:       # %bb.0:
-; SSE-F128-NEXT:    movd %xmm0, %eax
-; SSE-F128-NEXT:    shrl $24, %eax
-; SSE-F128-NEXT:    movb %al, (%rdi)
-; SSE-F128-NEXT:    retq
   %vecext = extractelement <16 x i8> %foo, i32 3
   store i8 %vecext, i8* %dst, align 1
   ret void
@@ -138,12 +125,6 @@ define void @extract_i8_15(i8* nocapture %dst, <16 x i8> %foo) nounwind {
 ; AVX-X64:       # %bb.0:
 ; AVX-X64-NEXT:    vpextrb $15, %xmm0, (%rdi)
 ; AVX-X64-NEXT:    retq
-;
-; SSE-F128-LABEL: extract_i8_15:
-; SSE-F128:       # %bb.0:
-; SSE-F128-NEXT:    pextrw $7, %xmm0, %eax
-; SSE-F128-NEXT:    movb %ah, (%rdi)
-; SSE-F128-NEXT:    retq
   %vecext = extractelement <16 x i8> %foo, i32 15
   store i8 %vecext, i8* %dst, align 1
   ret void
@@ -184,12 +165,6 @@ define void @extract_i16_0(i16* nocapture %dst, <8 x i16> %foo) nounwind {
 ; AVX-X64:       # %bb.0:
 ; AVX-X64-NEXT:    vpextrw $0, %xmm0, (%rdi)
 ; AVX-X64-NEXT:    retq
-;
-; SSE-F128-LABEL: extract_i16_0:
-; SSE-F128:       # %bb.0:
-; SSE-F128-NEXT:    movd %xmm0, %eax
-; SSE-F128-NEXT:    movw %ax, (%rdi)
-; SSE-F128-NEXT:    retq
   %vecext = extractelement <8 x i16> %foo, i32 0
   store i16 %vecext, i16* %dst, align 1
   ret void
@@ -230,12 +205,6 @@ define void @extract_i16_7(i16* nocapture %dst, <8 x i16> %foo) nounwind {
 ; AVX-X64:       # %bb.0:
 ; AVX-X64-NEXT:    vpextrw $7, %xmm0, (%rdi)
 ; AVX-X64-NEXT:    retq
-;
-; SSE-F128-LABEL: extract_i16_7:
-; SSE-F128:       # %bb.0:
-; SSE-F128-NEXT:    pextrw $7, %xmm0, %eax
-; SSE-F128-NEXT:    movw %ax, (%rdi)
-; SSE-F128-NEXT:    retq
   %vecext = extractelement <8 x i16> %foo, i32 7
   store i16 %vecext, i16* %dst, align 1
   ret void
@@ -303,12 +272,6 @@ define void @extract_i32_3(i32* nocapture %dst, <4 x i32> %foo) nounwind {
 ; AVX-X64:       # %bb.0:
 ; AVX-X64-NEXT:    vextractps $3, %xmm0, (%rdi)
 ; AVX-X64-NEXT:    retq
-;
-; SSE-F128-LABEL: extract_i32_3:
-; SSE-F128:       # %bb.0:
-; SSE-F128-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
-; SSE-F128-NEXT:    movd %xmm0, (%rdi)
-; SSE-F128-NEXT:    retq
   %vecext = extractelement <4 x i32> %foo, i32 3
   store i32 %vecext, i32* %dst, align 1
   ret void
@@ -371,12 +334,6 @@ define void @extract_i64_1(i64* nocapture %dst, <2 x i64> %foo) nounwind {
 ; AVX-X64:       # %bb.0:
 ; AVX-X64-NEXT:    vpextrq $1, %xmm0, (%rdi)
 ; AVX-X64-NEXT:    retq
-;
-; SSE-F128-LABEL: extract_i64_1:
-; SSE-F128:       # %bb.0:
-; SSE-F128-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; SSE-F128-NEXT:    movq %xmm0, (%rdi)
-; SSE-F128-NEXT:    retq
   %vecext = extractelement <2 x i64> %foo, i32 1
   store i64 %vecext, i64* %dst, align 1
   ret void
@@ -444,12 +401,6 @@ define void @extract_f32_3(float* nocapture %dst, <4 x float> %foo) nounwind {
 ; AVX-X64:       # %bb.0:
 ; AVX-X64-NEXT:    vextractps $3, %xmm0, (%rdi)
 ; AVX-X64-NEXT:    retq
-;
-; SSE-F128-LABEL: extract_f32_3:
-; SSE-F128:       # %bb.0:
-; SSE-F128-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
-; SSE-F128-NEXT:    movss %xmm0, (%rdi)
-; SSE-F128-NEXT:    retq
   %vecext = extractelement <4 x float> %foo, i32 3
   store float %vecext, float* %dst, align 1
   ret void
@@ -527,17 +478,10 @@ define void @extract_f128_0(fp128* nocapture %dst, <2 x fp128> %foo) nounwind {
 ; SSE-X32-NEXT:    popl %edi
 ; SSE-X32-NEXT:    retl
 ;
-; SSE2-X64-LABEL: extract_f128_0:
-; SSE2-X64:       # %bb.0:
-; SSE2-X64-NEXT:    movq %rdx, 8(%rdi)
-; SSE2-X64-NEXT:    movq %rsi, (%rdi)
-; SSE2-X64-NEXT:    retq
-;
-; SSE41-X64-LABEL: extract_f128_0:
-; SSE41-X64:       # %bb.0:
-; SSE41-X64-NEXT:    movq %rdx, 8(%rdi)
-; SSE41-X64-NEXT:    movq %rsi, (%rdi)
-; SSE41-X64-NEXT:    retq
+; SSE-X64-LABEL: extract_f128_0:
+; SSE-X64:       # %bb.0:
+; SSE-X64-NEXT:    movups %xmm0, (%rdi)
+; SSE-X64-NEXT:    retq
 ;
 ; AVX-X32-LABEL: extract_f128_0:
 ; AVX-X32:       # %bb.0:
@@ -548,14 +492,8 @@ define void @extract_f128_0(fp128* nocapture %dst, <2 x fp128> %foo) nounwind {
 ;
 ; AVX-X64-LABEL: extract_f128_0:
 ; AVX-X64:       # %bb.0:
-; AVX-X64-NEXT:    movq %rdx, 8(%rdi)
-; AVX-X64-NEXT:    movq %rsi, (%rdi)
+; AVX-X64-NEXT:    vmovups %xmm0, (%rdi)
 ; AVX-X64-NEXT:    retq
-;
-; SSE-F128-LABEL: extract_f128_0:
-; SSE-F128:       # %bb.0:
-; SSE-F128-NEXT:    movups %xmm0, (%rdi)
-; SSE-F128-NEXT:    retq
   %vecext = extractelement <2 x fp128> %foo, i32 0
   store fp128 %vecext, fp128* %dst, align 1
   ret void
@@ -579,17 +517,10 @@ define void @extract_f128_1(fp128* nocapture %dst, <2 x fp128> %foo) nounwind {
 ; SSE-X32-NEXT:    popl %edi
 ; SSE-X32-NEXT:    retl
 ;
-; SSE2-X64-LABEL: extract_f128_1:
-; SSE2-X64:       # %bb.0:
-; SSE2-X64-NEXT:    movq %r8, 8(%rdi)
-; SSE2-X64-NEXT:    movq %rcx, (%rdi)
-; SSE2-X64-NEXT:    retq
-;
-; SSE41-X64-LABEL: extract_f128_1:
-; SSE41-X64:       # %bb.0:
-; SSE41-X64-NEXT:    movq %r8, 8(%rdi)
-; SSE41-X64-NEXT:    movq %rcx, (%rdi)
-; SSE41-X64-NEXT:    retq
+; SSE-X64-LABEL: extract_f128_1:
+; SSE-X64:       # %bb.0:
+; SSE-X64-NEXT:    movups %xmm1, (%rdi)
+; SSE-X64-NEXT:    retq
 ;
 ; AVX-X32-LABEL: extract_f128_1:
 ; AVX-X32:       # %bb.0:
@@ -600,14 +531,8 @@ define void @extract_f128_1(fp128* nocapture %dst, <2 x fp128> %foo) nounwind {
 ;
 ; AVX-X64-LABEL: extract_f128_1:
 ; AVX-X64:       # %bb.0:
-; AVX-X64-NEXT:    movq %r8, 8(%rdi)
-; AVX-X64-NEXT:    movq %rcx, (%rdi)
+; AVX-X64-NEXT:    vmovups %xmm1, (%rdi)
 ; AVX-X64-NEXT:    retq
-;
-; SSE-F128-LABEL: extract_f128_1:
-; SSE-F128:       # %bb.0:
-; SSE-F128-NEXT:    movups %xmm1, (%rdi)
-; SSE-F128-NEXT:    retq
   %vecext = extractelement <2 x fp128> %foo, i32 1
   store fp128 %vecext, fp128* %dst, align 1
   ret void
diff --git a/llvm/test/CodeGen/X86/extractelement-index.ll b/llvm/test/CodeGen/X86/extractelement-index.ll
index a34ea7fc3..a95c4daf3 100644
--- a/llvm/test/CodeGen/X86/extractelement-index.ll
+++ b/llvm/test/CodeGen/X86/extractelement-index.ll
@@ -8,6 +8,30 @@
 ; ExtractElement - Constant Index
 ;
 
+define i64 @extract_undef_index_from_zero_vec() nounwind {
+; SSE-LABEL: extract_undef_index_from_zero_vec:
+; SSE:       # %bb.0:
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: extract_undef_index_from_zero_vec:
+; AVX:       # %bb.0:
+; AVX-NEXT:    retq
+  %E = extractelement <2 x i64> zeroinitializer, i64 undef
+  ret i64 %E
+}
+
+define i64 @extract_undef_index_from_nonzero_vec() nounwind {
+; SSE-LABEL: extract_undef_index_from_nonzero_vec:
+; SSE:       # %bb.0:
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: extract_undef_index_from_nonzero_vec:
+; AVX:       # %bb.0:
+; AVX-NEXT:    retq
+  %E = extractelement <2 x i64> <i64 -1, i64 -1>, i64 undef
+  ret i64 %E
+}
+
 define i8 @extractelement_v16i8_1(<16 x i8> %a) nounwind {
 ; SSE2-LABEL: extractelement_v16i8_1:
 ; SSE2:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/extractelement-load.ll b/llvm/test/CodeGen/X86/extractelement-load.ll
index 915b9bfa8..f499568ca 100644
--- a/llvm/test/CodeGen/X86/extractelement-load.ll
+++ b/llvm/test/CodeGen/X86/extractelement-load.ll
@@ -9,13 +9,20 @@ define i32 @t(<2 x i64>* %val) nounwind  {
 ; X32-SSE2-LABEL: t:
 ; X32-SSE2:       # %bb.0:
 ; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE2-NEXT:    movl 8(%eax), %eax
+; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = mem[2,3,0,1]
+; X32-SSE2-NEXT:    movd %xmm0, %eax
 ; X32-SSE2-NEXT:    retl
 ;
-; X64-LABEL: t:
-; X64:       # %bb.0:
-; X64-NEXT:    movl 8(%rdi), %eax
-; X64-NEXT:    retq
+; X64-SSSE3-LABEL: t:
+; X64-SSSE3:       # %bb.0:
+; X64-SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = mem[2,3,0,1]
+; X64-SSSE3-NEXT:    movd %xmm0, %eax
+; X64-SSSE3-NEXT:    retq
+;
+; X64-AVX-LABEL: t:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    movl 8(%rdi), %eax
+; X64-AVX-NEXT:    retq
   %tmp2 = load <2 x i64>, <2 x i64>* %val, align 16		; <<2 x i64>> [#uses=1]
   %tmp3 = bitcast <2 x i64> %tmp2 to <4 x i32>		; <<4 x i32>> [#uses=1]
   %tmp4 = extractelement <4 x i32> %tmp3, i32 2		; <i32> [#uses=1]
@@ -93,3 +100,168 @@ define i64 @t4(<2 x double>* %a) {
   ret i64 %e
 }
 
+; Don't extract from a volatile.
+define void @t5(<2 x double> *%a0, double *%a1) {
+; X32-SSE2-LABEL: t5:
+; X32-SSE2:       # %bb.0:
+; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-SSE2-NEXT:    movaps (%ecx), %xmm0
+; X32-SSE2-NEXT:    movhps %xmm0, (%eax)
+; X32-SSE2-NEXT:    retl
+;
+; X64-SSSE3-LABEL: t5:
+; X64-SSSE3:       # %bb.0:
+; X64-SSSE3-NEXT:    movaps (%rdi), %xmm0
+; X64-SSSE3-NEXT:    movhps %xmm0, (%rsi)
+; X64-SSSE3-NEXT:    retq
+;
+; X64-AVX-LABEL: t5:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    vmovaps (%rdi), %xmm0
+; X64-AVX-NEXT:    vmovhps %xmm0, (%rsi)
+; X64-AVX-NEXT:    retq
+  %vecload = load volatile <2 x double>, <2 x double>* %a0, align 16
+  %vecext = extractelement <2 x double> %vecload, i32 1
+  store volatile double %vecext, double* %a1, align 8
+  ret void
+}
+
+; Check for multiuse.
+define float @t6(<8 x float> *%a0) {
+; X32-SSE2-LABEL: t6:
+; X32-SSE2:       # %bb.0:
+; X32-SSE2-NEXT:    pushl %eax
+; X32-SSE2-NEXT:    .cfi_def_cfa_offset 8
+; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-SSE2-NEXT:    movaps (%eax), %xmm0
+; X32-SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; X32-SSE2-NEXT:    xorps %xmm1, %xmm1
+; X32-SSE2-NEXT:    cmpeqss %xmm0, %xmm1
+; X32-SSE2-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; X32-SSE2-NEXT:    andps %xmm1, %xmm2
+; X32-SSE2-NEXT:    andnps %xmm0, %xmm1
+; X32-SSE2-NEXT:    orps %xmm2, %xmm1
+; X32-SSE2-NEXT:    movss %xmm1, (%esp)
+; X32-SSE2-NEXT:    flds (%esp)
+; X32-SSE2-NEXT:    popl %eax
+; X32-SSE2-NEXT:    .cfi_def_cfa_offset 4
+; X32-SSE2-NEXT:    retl
+;
+; X64-SSSE3-LABEL: t6:
+; X64-SSSE3:       # %bb.0:
+; X64-SSSE3-NEXT:    movshdup {{.*#+}} xmm1 = mem[1,1,3,3]
+; X64-SSSE3-NEXT:    xorps %xmm0, %xmm0
+; X64-SSSE3-NEXT:    cmpeqss %xmm1, %xmm0
+; X64-SSSE3-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; X64-SSSE3-NEXT:    andps %xmm0, %xmm2
+; X64-SSSE3-NEXT:    andnps %xmm1, %xmm0
+; X64-SSSE3-NEXT:    orps %xmm2, %xmm0
+; X64-SSSE3-NEXT:    retq
+;
+; X64-AVX-LABEL: t6:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    vmovshdup {{.*#+}} xmm0 = mem[1,1,3,3]
+; X64-AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X64-AVX-NEXT:    vcmpeqss %xmm1, %xmm0, %xmm1
+; X64-AVX-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; X64-AVX-NEXT:    vblendvps %xmm1, %xmm2, %xmm0, %xmm0
+; X64-AVX-NEXT:    retq
+  %vecload = load <8 x float>, <8 x float>* %a0, align 32
+  %vecext = extractelement <8 x float> %vecload, i32 1
+  %cmp = fcmp oeq float %vecext, 0.000000e+00
+  %cond = select i1 %cmp, float 1.000000e+00, float %vecext
+  ret float %cond
+}
+
+define void @PR43971(<8 x float> *%a0, float *%a1) {
+; X32-SSE2-LABEL: PR43971:
+; X32-SSE2:       # %bb.0: # %entry
+; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-SSE2-NEXT:    movaps 16(%ecx), %xmm0
+; X32-SSE2-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
+; X32-SSE2-NEXT:    xorps %xmm1, %xmm1
+; X32-SSE2-NEXT:    cmpltss %xmm0, %xmm1
+; X32-SSE2-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; X32-SSE2-NEXT:    andps %xmm1, %xmm2
+; X32-SSE2-NEXT:    andnps %xmm0, %xmm1
+; X32-SSE2-NEXT:    orps %xmm2, %xmm1
+; X32-SSE2-NEXT:    movss %xmm1, (%eax)
+; X32-SSE2-NEXT:    retl
+;
+; X64-SSSE3-LABEL: PR43971:
+; X64-SSSE3:       # %bb.0: # %entry
+; X64-SSSE3-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X64-SSSE3-NEXT:    xorps %xmm1, %xmm1
+; X64-SSSE3-NEXT:    cmpltss %xmm0, %xmm1
+; X64-SSSE3-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; X64-SSSE3-NEXT:    andps %xmm1, %xmm2
+; X64-SSSE3-NEXT:    andnps %xmm0, %xmm1
+; X64-SSSE3-NEXT:    orps %xmm2, %xmm1
+; X64-SSSE3-NEXT:    movss %xmm1, (%rsi)
+; X64-SSSE3-NEXT:    retq
+;
+; X64-AVX-LABEL: PR43971:
+; X64-AVX:       # %bb.0: # %entry
+; X64-AVX-NEXT:    vpermilpd {{.*#+}} xmm0 = mem[1,0]
+; X64-AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X64-AVX-NEXT:    vcmpltss %xmm0, %xmm1, %xmm1
+; X64-AVX-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; X64-AVX-NEXT:    vblendvps %xmm1, %xmm2, %xmm0, %xmm0
+; X64-AVX-NEXT:    vmovss %xmm0, (%rsi)
+; X64-AVX-NEXT:    retq
+entry:
+  %0 = load <8 x float>, <8 x float>* %a0, align 32
+  %vecext = extractelement <8 x float> %0, i32 6
+  %cmp = fcmp ogt float %vecext, 0.000000e+00
+  %1 = load float, float* %a1, align 4
+  %cond = select i1 %cmp, float %1, float %vecext
+  store float %cond, float* %a1, align 4
+  ret void
+}
+
+define float @PR43971_1(<8 x float> *%a0) nounwind {
+; X32-SSE2-LABEL: PR43971_1:
+; X32-SSE2:       # %bb.0: # %entry
+; X32-SSE2-NEXT:    pushl %eax
+; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-SSE2-NEXT:    movaps (%eax), %xmm0
+; X32-SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; X32-SSE2-NEXT:    xorps %xmm1, %xmm1
+; X32-SSE2-NEXT:    cmpeqss %xmm0, %xmm1
+; X32-SSE2-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; X32-SSE2-NEXT:    andps %xmm1, %xmm2
+; X32-SSE2-NEXT:    andnps %xmm0, %xmm1
+; X32-SSE2-NEXT:    orps %xmm2, %xmm1
+; X32-SSE2-NEXT:    movss %xmm1, (%esp)
+; X32-SSE2-NEXT:    flds (%esp)
+; X32-SSE2-NEXT:    popl %eax
+; X32-SSE2-NEXT:    retl
+;
+; X64-SSSE3-LABEL: PR43971_1:
+; X64-SSSE3:       # %bb.0: # %entry
+; X64-SSSE3-NEXT:    movshdup {{.*#+}} xmm1 = mem[1,1,3,3]
+; X64-SSSE3-NEXT:    xorps %xmm0, %xmm0
+; X64-SSSE3-NEXT:    cmpeqss %xmm1, %xmm0
+; X64-SSSE3-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; X64-SSSE3-NEXT:    andps %xmm0, %xmm2
+; X64-SSSE3-NEXT:    andnps %xmm1, %xmm0
+; X64-SSSE3-NEXT:    orps %xmm2, %xmm0
+; X64-SSSE3-NEXT:    retq
+;
+; X64-AVX-LABEL: PR43971_1:
+; X64-AVX:       # %bb.0: # %entry
+; X64-AVX-NEXT:    vmovshdup {{.*#+}} xmm0 = mem[1,1,3,3]
+; X64-AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X64-AVX-NEXT:    vcmpeqss %xmm1, %xmm0, %xmm1
+; X64-AVX-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; X64-AVX-NEXT:    vblendvps %xmm1, %xmm2, %xmm0, %xmm0
+; X64-AVX-NEXT:    retq
+entry:
+  %0 = load <8 x float>, <8 x float>* %a0, align 32
+  %vecext = extractelement <8 x float> %0, i32 1
+  %cmp = fcmp oeq float %vecext, 0.000000e+00
+  %cond = select i1 %cmp, float 1.000000e+00, float %vecext
+  ret float %cond
+}
diff --git a/llvm/test/CodeGen/X86/extractps.ll b/llvm/test/CodeGen/X86/extractps.ll
index 586099d74..ca4a47a42 100644
--- a/llvm/test/CodeGen/X86/extractps.ll
+++ b/llvm/test/CodeGen/X86/extractps.ll
@@ -1,12 +1,16 @@
-; RUN: llc < %s -mtriple=i686-- -mcpu=penryn > %t
-; RUN: not grep movd %t
-; RUN: grep "movss	%xmm" %t | count 1
-; RUN: grep "extractps	\$1, %xmm0, " %t | count 1
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- -mcpu=penryn | FileCheck %s
 ; PR2647
 
 @0 = external global float, align 16         ; <float*>:0 [#uses=2]
 
-define internal void @""() nounwind {
+define internal void @a() nounwind {
+; CHECK-LABEL: a:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; CHECK-NEXT:    rsqrtss %xmm0, %xmm0
+; CHECK-NEXT:    movss %xmm0, __unnamed_1
+; CHECK-NEXT:    retl
         load float, float* @0, align 16                ; <float>:1 [#uses=1]
         insertelement <4 x float> undef, float %1, i32 0                ; <<4 x float>>:2 [#uses=1]
         call <4 x float> @llvm.x86.sse.rsqrt.ss( <4 x float> %2 )              ; <<4 x float>>:3 [#uses=1]
@@ -14,7 +18,14 @@ define internal void @""() nounwind {
         store float %4, float* @0, align 16
         ret void
 }
-define internal void @""() nounwind {
+define internal void @b() nounwind {
+; CHECK-LABEL: b:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; CHECK-NEXT:    movsldup {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; CHECK-NEXT:    rsqrtss %xmm0, %xmm0
+; CHECK-NEXT:    extractps $1, %xmm0, __unnamed_1
+; CHECK-NEXT:    retl
         load float, float* @0, align 16                ; <float>:1 [#uses=1]
         insertelement <4 x float> undef, float %1, i32 1                ; <<4 x float>>:2 [#uses=1]
         call <4 x float> @llvm.x86.sse.rsqrt.ss( <4 x float> %2 )              ; <<4 x float>>:3 [#uses=1]
diff --git a/llvm/test/CodeGen/X86/f16c-intrinsics.ll b/llvm/test/CodeGen/X86/f16c-intrinsics.ll
index 20ea67529..553cf3147 100644
--- a/llvm/test/CodeGen/X86/f16c-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/f16c-intrinsics.ll
@@ -268,18 +268,12 @@ define void @test_x86_vcvtps2ph_128_m(<4 x i16>* nocapture %d, <4 x float> %a) n
 ; X32-AVX512VL-LABEL: test_x86_vcvtps2ph_128_m:
 ; X32-AVX512VL:       # %bb.0: # %entry
 ; X32-AVX512VL-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X32-AVX512VL-NEXT:    vcvtps2ph $3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x1d,0xc0,0x03]
-; X32-AVX512VL-NEXT:    vpmovzxwd %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x33,0xc0]
-; X32-AVX512VL-NEXT:    # xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; X32-AVX512VL-NEXT:    vpmovdw %xmm0, (%eax) # encoding: [0x62,0xf2,0x7e,0x08,0x33,0x00]
+; X32-AVX512VL-NEXT:    vcvtps2ph $3, %xmm0, (%eax) # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x1d,0x00,0x03]
 ; X32-AVX512VL-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-AVX512VL-LABEL: test_x86_vcvtps2ph_128_m:
 ; X64-AVX512VL:       # %bb.0: # %entry
-; X64-AVX512VL-NEXT:    vcvtps2ph $3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x1d,0xc0,0x03]
-; X64-AVX512VL-NEXT:    vpmovzxwd %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x33,0xc0]
-; X64-AVX512VL-NEXT:    # xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; X64-AVX512VL-NEXT:    vpmovdw %xmm0, (%rdi) # encoding: [0x62,0xf2,0x7e,0x08,0x33,0x07]
+; X64-AVX512VL-NEXT:    vcvtps2ph $3, %xmm0, (%rdi) # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x1d,0x07,0x03]
 ; X64-AVX512VL-NEXT:    retq # encoding: [0xc3]
 entry:
   %0 = tail call <8 x i16> @llvm.x86.vcvtps2ph.128(<4 x float> %a, i32 3)
diff --git a/llvm/test/CodeGen/X86/fadd-combines.ll b/llvm/test/CodeGen/X86/fadd-combines.ll
index fdd83dacd..048e5c5d3 100644
--- a/llvm/test/CodeGen/X86/fadd-combines.ll
+++ b/llvm/test/CodeGen/X86/fadd-combines.ll
@@ -247,4 +247,32 @@ define float @fadd_const_multiuse_attr(float %x) #0 {
   ret float %a3
 }
 
+; PR32939 - https://bugs.llvm.org/show_bug.cgi?id=32939
+
+define double @fmul2_negated(double %a, double %b, double %c) {
+; CHECK-LABEL: fmul2_negated:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addsd %xmm1, %xmm1
+; CHECK-NEXT:    mulsd %xmm2, %xmm1
+; CHECK-NEXT:    subsd %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %mul = fmul double %b, 2.0
+  %mul1 = fmul double %mul, %c
+  %sub = fsub double %a, %mul1
+  ret double %sub
+}
+
+define <2 x double> @fmul2_negated_vec(<2 x double> %a, <2 x double> %b, <2 x double> %c) {
+; CHECK-LABEL: fmul2_negated_vec:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addpd %xmm1, %xmm1
+; CHECK-NEXT:    mulpd %xmm2, %xmm1
+; CHECK-NEXT:    subpd %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %mul = fmul <2 x double> %b, <double 2.0, double 2.0>
+  %mul1 = fmul <2 x double> %mul, %c
+  %sub = fsub <2 x double> %a, %mul1
+  ret <2 x double> %sub
+}
+
 attributes #0 = { "less-precise-fpmad"="true" "no-infs-fp-math"="true" "no-nans-fp-math"="true" "unsafe-fp-math"="true" "no-signed-zeros-fp-math"="true" }
diff --git a/llvm/test/CodeGen/X86/fast-isel-divrem.ll b/llvm/test/CodeGen/X86/fast-isel-divrem.ll
index 3e649ca9a..45b9d6a07 100644
--- a/llvm/test/CodeGen/X86/fast-isel-divrem.ll
+++ b/llvm/test/CodeGen/X86/fast-isel-divrem.ll
@@ -8,7 +8,7 @@ entry:
 }
 
 ; CHECK-LABEL: test_sdiv8:
-; CHECK: movsbw
+; CHECK: movsbl
 ; CHECK: idivb
 
 define i8 @test_srem8(i8 %dividend, i8 %divisor) nounwind {
@@ -18,7 +18,7 @@ entry:
 }
 
 ; CHECK-LABEL: test_srem8:
-; CHECK: movsbw
+; CHECK: movsbl
 ; CHECK: idivb
 
 define i8 @test_udiv8(i8 %dividend, i8 %divisor) nounwind {
@@ -28,7 +28,7 @@ entry:
 }
 
 ; CHECK-LABEL: test_udiv8:
-; CHECK: movzbw
+; CHECK: movzbl
 ; CHECK: divb
 
 define i8 @test_urem8(i8 %dividend, i8 %divisor) nounwind {
@@ -38,7 +38,7 @@ entry:
 }
 
 ; CHECK-LABEL: test_urem8:
-; CHECK: movzbw
+; CHECK: movzbl
 ; CHECK: divb
 
 define i16 @test_sdiv16(i16 %dividend, i16 %divisor) nounwind {
diff --git a/llvm/test/CodeGen/X86/fast-isel-nontemporal.ll b/llvm/test/CodeGen/X86/fast-isel-nontemporal.ll
index 37e380b2b..7345df7ae 100644
--- a/llvm/test/CodeGen/X86/fast-isel-nontemporal.ll
+++ b/llvm/test/CodeGen/X86/fast-isel-nontemporal.ll
@@ -891,6 +891,8 @@ define void @test_nt64xi8(<64 x i8>* nocapture %ptr, <64 x i8> %X) {
 ;
 ; AVX512VL-LABEL: test_nt64xi8:
 ; AVX512VL:       # %bb.0: # %entry
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
+; AVX512VL-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; AVX512VL-NEXT:    vmovntdq %ymm0, (%rdi)
 ; AVX512VL-NEXT:    vmovntdq %ymm1, 32(%rdi)
 ; AVX512VL-NEXT:    vzeroupper
@@ -898,6 +900,8 @@ define void @test_nt64xi8(<64 x i8>* nocapture %ptr, <64 x i8> %X) {
 ;
 ; AVX512F-LABEL: test_nt64xi8:
 ; AVX512F:       # %bb.0: # %entry
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
+; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; AVX512F-NEXT:    vmovntdq %ymm0, (%rdi)
 ; AVX512F-NEXT:    vmovntdq %ymm1, 32(%rdi)
 ; AVX512F-NEXT:    vzeroupper
@@ -931,6 +935,8 @@ define void @test_nt32xi16(<32 x i16>* nocapture %ptr, <32 x i16> %X) {
 ;
 ; AVX512VL-LABEL: test_nt32xi16:
 ; AVX512VL:       # %bb.0: # %entry
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
+; AVX512VL-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; AVX512VL-NEXT:    vmovntdq %ymm0, (%rdi)
 ; AVX512VL-NEXT:    vmovntdq %ymm1, 32(%rdi)
 ; AVX512VL-NEXT:    vzeroupper
@@ -938,6 +944,8 @@ define void @test_nt32xi16(<32 x i16>* nocapture %ptr, <32 x i16> %X) {
 ;
 ; AVX512F-LABEL: test_nt32xi16:
 ; AVX512F:       # %bb.0: # %entry
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
+; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; AVX512F-NEXT:    vmovntdq %ymm0, (%rdi)
 ; AVX512F-NEXT:    vmovntdq %ymm1, 32(%rdi)
 ; AVX512F-NEXT:    vzeroupper
@@ -1162,22 +1170,10 @@ define <64 x i8> @test_load_nt64xi8(<64 x i8>* nocapture %ptr) {
 ; AVX2-NEXT:    vmovntdqa 32(%rdi), %ymm1
 ; AVX2-NEXT:    retq
 ;
-; AVX512VL-LABEL: test_load_nt64xi8:
-; AVX512VL:       # %bb.0: # %entry
-; AVX512VL-NEXT:    vmovntdqa (%rdi), %ymm0
-; AVX512VL-NEXT:    vmovntdqa 32(%rdi), %ymm1
-; AVX512VL-NEXT:    retq
-;
-; AVX512F-LABEL: test_load_nt64xi8:
-; AVX512F:       # %bb.0: # %entry
-; AVX512F-NEXT:    vmovntdqa (%rdi), %ymm0
-; AVX512F-NEXT:    vmovntdqa 32(%rdi), %ymm1
-; AVX512F-NEXT:    retq
-;
-; AVX512BW-LABEL: test_load_nt64xi8:
-; AVX512BW:       # %bb.0: # %entry
-; AVX512BW-NEXT:    vmovntdqa (%rdi), %zmm0
-; AVX512BW-NEXT:    retq
+; AVX512-LABEL: test_load_nt64xi8:
+; AVX512:       # %bb.0: # %entry
+; AVX512-NEXT:    vmovntdqa (%rdi), %zmm0
+; AVX512-NEXT:    retq
 entry:
   %0 = load <64 x i8>, <64 x i8>* %ptr, align 64, !nontemporal !1
   ret <64 x i8> %0
@@ -1228,22 +1224,10 @@ define <32 x i16> @test_load_nt32xi16(<32 x i16>* nocapture %ptr) {
 ; AVX2-NEXT:    vmovntdqa 32(%rdi), %ymm1
 ; AVX2-NEXT:    retq
 ;
-; AVX512VL-LABEL: test_load_nt32xi16:
-; AVX512VL:       # %bb.0: # %entry
-; AVX512VL-NEXT:    vmovntdqa (%rdi), %ymm0
-; AVX512VL-NEXT:    vmovntdqa 32(%rdi), %ymm1
-; AVX512VL-NEXT:    retq
-;
-; AVX512F-LABEL: test_load_nt32xi16:
-; AVX512F:       # %bb.0: # %entry
-; AVX512F-NEXT:    vmovntdqa (%rdi), %ymm0
-; AVX512F-NEXT:    vmovntdqa 32(%rdi), %ymm1
-; AVX512F-NEXT:    retq
-;
-; AVX512BW-LABEL: test_load_nt32xi16:
-; AVX512BW:       # %bb.0: # %entry
-; AVX512BW-NEXT:    vmovntdqa (%rdi), %zmm0
-; AVX512BW-NEXT:    retq
+; AVX512-LABEL: test_load_nt32xi16:
+; AVX512:       # %bb.0: # %entry
+; AVX512-NEXT:    vmovntdqa (%rdi), %zmm0
+; AVX512-NEXT:    retq
 entry:
   %0 = load <32 x i16>, <32 x i16>* %ptr, align 64, !nontemporal !1
   ret <32 x i16> %0
diff --git a/llvm/test/CodeGen/X86/fast-isel-select-pseudo-cmov.ll b/llvm/test/CodeGen/X86/fast-isel-select-pseudo-cmov.ll
index aab74bbde..354c69565 100644
--- a/llvm/test/CodeGen/X86/fast-isel-select-pseudo-cmov.ll
+++ b/llvm/test/CodeGen/X86/fast-isel-select-pseudo-cmov.ll
@@ -27,7 +27,7 @@ define float @select_fcmp_one_f32(float %a, float %b, float %c, float %d) {
 ; AVX512-LABEL: select_fcmp_one_f32:
 ; AVX512:       ## %bb.0:
 ; AVX512-NEXT:    vcmpneq_oqss %xmm1, %xmm0, %k1
-; AVX512-NEXT:    vmovss %xmm2, %xmm0, %xmm3 {%k1}
+; AVX512-NEXT:    vmovss %xmm2, %xmm3, %xmm3 {%k1}
 ; AVX512-NEXT:    vmovaps %xmm3, %xmm0
 ; AVX512-NEXT:    retq
   %1 = fcmp one float %a, %b
@@ -55,7 +55,7 @@ define double @select_fcmp_one_f64(double %a, double %b, double %c, double %d) {
 ; AVX512-LABEL: select_fcmp_one_f64:
 ; AVX512:       ## %bb.0:
 ; AVX512-NEXT:    vcmpneq_oqsd %xmm1, %xmm0, %k1
-; AVX512-NEXT:    vmovsd %xmm2, %xmm0, %xmm3 {%k1}
+; AVX512-NEXT:    vmovsd %xmm2, %xmm3, %xmm3 {%k1}
 ; AVX512-NEXT:    vmovapd %xmm3, %xmm0
 ; AVX512-NEXT:    retq
   %1 = fcmp one double %a, %b
@@ -87,7 +87,7 @@ define float @select_icmp_eq_f32(i64 %a, i64 %b, float %c, float %d) {
 ; AVX512-ISEL-NEXT:    cmpq %rsi, %rdi
 ; AVX512-ISEL-NEXT:    sete %al
 ; AVX512-ISEL-NEXT:    kmovd %eax, %k1
-; AVX512-ISEL-NEXT:    vmovss %xmm0, %xmm0, %xmm1 {%k1}
+; AVX512-ISEL-NEXT:    vmovss %xmm0, %xmm1, %xmm1 {%k1}
 ; AVX512-ISEL-NEXT:    vmovaps %xmm1, %xmm0
 ; AVX512-ISEL-NEXT:    retq
 ;
@@ -128,7 +128,7 @@ define float @select_icmp_ne_f32(i64 %a, i64 %b, float %c, float %d) {
 ; AVX512-ISEL-NEXT:    cmpq %rsi, %rdi
 ; AVX512-ISEL-NEXT:    setne %al
 ; AVX512-ISEL-NEXT:    kmovd %eax, %k1
-; AVX512-ISEL-NEXT:    vmovss %xmm0, %xmm0, %xmm1 {%k1}
+; AVX512-ISEL-NEXT:    vmovss %xmm0, %xmm1, %xmm1 {%k1}
 ; AVX512-ISEL-NEXT:    vmovaps %xmm1, %xmm0
 ; AVX512-ISEL-NEXT:    retq
 ;
@@ -169,7 +169,7 @@ define float @select_icmp_ugt_f32(i64 %a, i64 %b, float %c, float %d) {
 ; AVX512-ISEL-NEXT:    cmpq %rsi, %rdi
 ; AVX512-ISEL-NEXT:    seta %al
 ; AVX512-ISEL-NEXT:    kmovd %eax, %k1
-; AVX512-ISEL-NEXT:    vmovss %xmm0, %xmm0, %xmm1 {%k1}
+; AVX512-ISEL-NEXT:    vmovss %xmm0, %xmm1, %xmm1 {%k1}
 ; AVX512-ISEL-NEXT:    vmovaps %xmm1, %xmm0
 ; AVX512-ISEL-NEXT:    retq
 ;
@@ -210,7 +210,7 @@ define float @select_icmp_uge_f32(i64 %a, i64 %b, float %c, float %d) {
 ; AVX512-ISEL-NEXT:    cmpq %rsi, %rdi
 ; AVX512-ISEL-NEXT:    setae %al
 ; AVX512-ISEL-NEXT:    kmovd %eax, %k1
-; AVX512-ISEL-NEXT:    vmovss %xmm0, %xmm0, %xmm1 {%k1}
+; AVX512-ISEL-NEXT:    vmovss %xmm0, %xmm1, %xmm1 {%k1}
 ; AVX512-ISEL-NEXT:    vmovaps %xmm1, %xmm0
 ; AVX512-ISEL-NEXT:    retq
 ;
@@ -251,7 +251,7 @@ define float @select_icmp_ult_f32(i64 %a, i64 %b, float %c, float %d) {
 ; AVX512-ISEL-NEXT:    cmpq %rsi, %rdi
 ; AVX512-ISEL-NEXT:    setb %al
 ; AVX512-ISEL-NEXT:    kmovd %eax, %k1
-; AVX512-ISEL-NEXT:    vmovss %xmm0, %xmm0, %xmm1 {%k1}
+; AVX512-ISEL-NEXT:    vmovss %xmm0, %xmm1, %xmm1 {%k1}
 ; AVX512-ISEL-NEXT:    vmovaps %xmm1, %xmm0
 ; AVX512-ISEL-NEXT:    retq
 ;
@@ -292,7 +292,7 @@ define float @select_icmp_ule_f32(i64 %a, i64 %b, float %c, float %d) {
 ; AVX512-ISEL-NEXT:    cmpq %rsi, %rdi
 ; AVX512-ISEL-NEXT:    setbe %al
 ; AVX512-ISEL-NEXT:    kmovd %eax, %k1
-; AVX512-ISEL-NEXT:    vmovss %xmm0, %xmm0, %xmm1 {%k1}
+; AVX512-ISEL-NEXT:    vmovss %xmm0, %xmm1, %xmm1 {%k1}
 ; AVX512-ISEL-NEXT:    vmovaps %xmm1, %xmm0
 ; AVX512-ISEL-NEXT:    retq
 ;
@@ -333,7 +333,7 @@ define float @select_icmp_sgt_f32(i64 %a, i64 %b, float %c, float %d) {
 ; AVX512-ISEL-NEXT:    cmpq %rsi, %rdi
 ; AVX512-ISEL-NEXT:    setg %al
 ; AVX512-ISEL-NEXT:    kmovd %eax, %k1
-; AVX512-ISEL-NEXT:    vmovss %xmm0, %xmm0, %xmm1 {%k1}
+; AVX512-ISEL-NEXT:    vmovss %xmm0, %xmm1, %xmm1 {%k1}
 ; AVX512-ISEL-NEXT:    vmovaps %xmm1, %xmm0
 ; AVX512-ISEL-NEXT:    retq
 ;
@@ -374,7 +374,7 @@ define float @select_icmp_sge_f32(i64 %a, i64 %b, float %c, float %d) {
 ; AVX512-ISEL-NEXT:    cmpq %rsi, %rdi
 ; AVX512-ISEL-NEXT:    setge %al
 ; AVX512-ISEL-NEXT:    kmovd %eax, %k1
-; AVX512-ISEL-NEXT:    vmovss %xmm0, %xmm0, %xmm1 {%k1}
+; AVX512-ISEL-NEXT:    vmovss %xmm0, %xmm1, %xmm1 {%k1}
 ; AVX512-ISEL-NEXT:    vmovaps %xmm1, %xmm0
 ; AVX512-ISEL-NEXT:    retq
 ;
@@ -415,7 +415,7 @@ define float @select_icmp_slt_f32(i64 %a, i64 %b, float %c, float %d) {
 ; AVX512-ISEL-NEXT:    cmpq %rsi, %rdi
 ; AVX512-ISEL-NEXT:    setl %al
 ; AVX512-ISEL-NEXT:    kmovd %eax, %k1
-; AVX512-ISEL-NEXT:    vmovss %xmm0, %xmm0, %xmm1 {%k1}
+; AVX512-ISEL-NEXT:    vmovss %xmm0, %xmm1, %xmm1 {%k1}
 ; AVX512-ISEL-NEXT:    vmovaps %xmm1, %xmm0
 ; AVX512-ISEL-NEXT:    retq
 ;
@@ -456,7 +456,7 @@ define float @select_icmp_sle_f32(i64 %a, i64 %b, float %c, float %d) {
 ; AVX512-ISEL-NEXT:    cmpq %rsi, %rdi
 ; AVX512-ISEL-NEXT:    setle %al
 ; AVX512-ISEL-NEXT:    kmovd %eax, %k1
-; AVX512-ISEL-NEXT:    vmovss %xmm0, %xmm0, %xmm1 {%k1}
+; AVX512-ISEL-NEXT:    vmovss %xmm0, %xmm1, %xmm1 {%k1}
 ; AVX512-ISEL-NEXT:    vmovaps %xmm1, %xmm0
 ; AVX512-ISEL-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/fast-isel-select-sse.ll b/llvm/test/CodeGen/X86/fast-isel-select-sse.ll
index e91b925a3..17d2803e9 100644
--- a/llvm/test/CodeGen/X86/fast-isel-select-sse.ll
+++ b/llvm/test/CodeGen/X86/fast-isel-select-sse.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -verify-machineinstrs                                          | FileCheck %s --check-prefix=SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -verify-machineinstrs -fast-isel -fast-isel-abort=1            | FileCheck %s --check-prefix=SSE
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -verify-machineinstrs -O0                                          | FileCheck %s --check-prefix=SSE
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -verify-machineinstrs -O0 -fast-isel -fast-isel-abort=1            | FileCheck %s --check-prefix=SSE
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -verify-machineinstrs                               -mattr=avx | FileCheck %s --check-prefix=AVX
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -verify-machineinstrs -fast-isel -fast-isel-abort=1 -mattr=avx | FileCheck %s --check-prefix=AVX
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -verify-machineinstrs                               -mattr=avx512f | FileCheck %s --check-prefix=AVX512
@@ -12,9 +12,10 @@ define float @select_fcmp_oeq_f32(float %a, float %b, float %c, float %d) {
 ; SSE-LABEL: select_fcmp_oeq_f32:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    cmpeqss %xmm1, %xmm0
-; SSE-NEXT:    andps %xmm0, %xmm2
+; SSE-NEXT:    movaps %xmm0, %xmm1
+; SSE-NEXT:    andps %xmm2, %xmm1
 ; SSE-NEXT:    andnps %xmm3, %xmm0
-; SSE-NEXT:    orps %xmm2, %xmm0
+; SSE-NEXT:    orps %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: select_fcmp_oeq_f32:
@@ -26,7 +27,7 @@ define float @select_fcmp_oeq_f32(float %a, float %b, float %c, float %d) {
 ; AVX512-LABEL: select_fcmp_oeq_f32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcmpeqss %xmm1, %xmm0, %k1
-; AVX512-NEXT:    vmovss %xmm2, %xmm0, %xmm3 {%k1}
+; AVX512-NEXT:    vmovss %xmm2, %xmm3, %xmm3 {%k1}
 ; AVX512-NEXT:    vmovaps %xmm3, %xmm0
 ; AVX512-NEXT:    retq
   %1 = fcmp oeq float %a, %b
@@ -38,9 +39,10 @@ define double @select_fcmp_oeq_f64(double %a, double %b, double %c, double %d) {
 ; SSE-LABEL: select_fcmp_oeq_f64:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    cmpeqsd %xmm1, %xmm0
-; SSE-NEXT:    andpd %xmm0, %xmm2
+; SSE-NEXT:    movaps %xmm0, %xmm1
+; SSE-NEXT:    andpd %xmm2, %xmm1
 ; SSE-NEXT:    andnpd %xmm3, %xmm0
-; SSE-NEXT:    orpd %xmm2, %xmm0
+; SSE-NEXT:    orpd %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: select_fcmp_oeq_f64:
@@ -52,7 +54,7 @@ define double @select_fcmp_oeq_f64(double %a, double %b, double %c, double %d) {
 ; AVX512-LABEL: select_fcmp_oeq_f64:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcmpeqsd %xmm1, %xmm0, %k1
-; AVX512-NEXT:    vmovsd %xmm2, %xmm0, %xmm3 {%k1}
+; AVX512-NEXT:    vmovsd %xmm2, %xmm3, %xmm3 {%k1}
 ; AVX512-NEXT:    vmovapd %xmm3, %xmm0
 ; AVX512-NEXT:    retq
   %1 = fcmp oeq double %a, %b
@@ -64,9 +66,10 @@ define float @select_fcmp_ogt_f32(float %a, float %b, float %c, float %d) {
 ; SSE-LABEL: select_fcmp_ogt_f32:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    cmpltss %xmm0, %xmm1
-; SSE-NEXT:    andps %xmm1, %xmm2
+; SSE-NEXT:    movaps %xmm1, %xmm0
+; SSE-NEXT:    andps %xmm2, %xmm0
 ; SSE-NEXT:    andnps %xmm3, %xmm1
-; SSE-NEXT:    orps %xmm2, %xmm1
+; SSE-NEXT:    orps %xmm0, %xmm1
 ; SSE-NEXT:    movaps %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
@@ -79,7 +82,7 @@ define float @select_fcmp_ogt_f32(float %a, float %b, float %c, float %d) {
 ; AVX512-LABEL: select_fcmp_ogt_f32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcmpltss %xmm0, %xmm1, %k1
-; AVX512-NEXT:    vmovss %xmm2, %xmm0, %xmm3 {%k1}
+; AVX512-NEXT:    vmovss %xmm2, %xmm3, %xmm3 {%k1}
 ; AVX512-NEXT:    vmovaps %xmm3, %xmm0
 ; AVX512-NEXT:    retq
   %1 = fcmp ogt float %a, %b
@@ -91,10 +94,11 @@ define double @select_fcmp_ogt_f64(double %a, double %b, double %c, double %d) {
 ; SSE-LABEL: select_fcmp_ogt_f64:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    cmpltsd %xmm0, %xmm1
-; SSE-NEXT:    andpd %xmm1, %xmm2
+; SSE-NEXT:    movaps %xmm1, %xmm0
+; SSE-NEXT:    andpd %xmm2, %xmm0
 ; SSE-NEXT:    andnpd %xmm3, %xmm1
-; SSE-NEXT:    orpd %xmm2, %xmm1
-; SSE-NEXT:    movapd %xmm1, %xmm0
+; SSE-NEXT:    orpd %xmm0, %xmm1
+; SSE-NEXT:    movaps %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: select_fcmp_ogt_f64:
@@ -106,7 +110,7 @@ define double @select_fcmp_ogt_f64(double %a, double %b, double %c, double %d) {
 ; AVX512-LABEL: select_fcmp_ogt_f64:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcmpltsd %xmm0, %xmm1, %k1
-; AVX512-NEXT:    vmovsd %xmm2, %xmm0, %xmm3 {%k1}
+; AVX512-NEXT:    vmovsd %xmm2, %xmm3, %xmm3 {%k1}
 ; AVX512-NEXT:    vmovapd %xmm3, %xmm0
 ; AVX512-NEXT:    retq
   %1 = fcmp ogt double %a, %b
@@ -118,9 +122,10 @@ define float @select_fcmp_oge_f32(float %a, float %b, float %c, float %d) {
 ; SSE-LABEL: select_fcmp_oge_f32:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    cmpless %xmm0, %xmm1
-; SSE-NEXT:    andps %xmm1, %xmm2
+; SSE-NEXT:    movaps %xmm1, %xmm0
+; SSE-NEXT:    andps %xmm2, %xmm0
 ; SSE-NEXT:    andnps %xmm3, %xmm1
-; SSE-NEXT:    orps %xmm2, %xmm1
+; SSE-NEXT:    orps %xmm0, %xmm1
 ; SSE-NEXT:    movaps %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
@@ -133,7 +138,7 @@ define float @select_fcmp_oge_f32(float %a, float %b, float %c, float %d) {
 ; AVX512-LABEL: select_fcmp_oge_f32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcmpless %xmm0, %xmm1, %k1
-; AVX512-NEXT:    vmovss %xmm2, %xmm0, %xmm3 {%k1}
+; AVX512-NEXT:    vmovss %xmm2, %xmm3, %xmm3 {%k1}
 ; AVX512-NEXT:    vmovaps %xmm3, %xmm0
 ; AVX512-NEXT:    retq
   %1 = fcmp oge float %a, %b
@@ -145,10 +150,11 @@ define double @select_fcmp_oge_f64(double %a, double %b, double %c, double %d) {
 ; SSE-LABEL: select_fcmp_oge_f64:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    cmplesd %xmm0, %xmm1
-; SSE-NEXT:    andpd %xmm1, %xmm2
+; SSE-NEXT:    movaps %xmm1, %xmm0
+; SSE-NEXT:    andpd %xmm2, %xmm0
 ; SSE-NEXT:    andnpd %xmm3, %xmm1
-; SSE-NEXT:    orpd %xmm2, %xmm1
-; SSE-NEXT:    movapd %xmm1, %xmm0
+; SSE-NEXT:    orpd %xmm0, %xmm1
+; SSE-NEXT:    movaps %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: select_fcmp_oge_f64:
@@ -160,7 +166,7 @@ define double @select_fcmp_oge_f64(double %a, double %b, double %c, double %d) {
 ; AVX512-LABEL: select_fcmp_oge_f64:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcmplesd %xmm0, %xmm1, %k1
-; AVX512-NEXT:    vmovsd %xmm2, %xmm0, %xmm3 {%k1}
+; AVX512-NEXT:    vmovsd %xmm2, %xmm3, %xmm3 {%k1}
 ; AVX512-NEXT:    vmovapd %xmm3, %xmm0
 ; AVX512-NEXT:    retq
   %1 = fcmp oge double %a, %b
@@ -172,9 +178,10 @@ define float @select_fcmp_olt_f32(float %a, float %b, float %c, float %d) {
 ; SSE-LABEL: select_fcmp_olt_f32:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    cmpltss %xmm1, %xmm0
-; SSE-NEXT:    andps %xmm0, %xmm2
+; SSE-NEXT:    movaps %xmm0, %xmm1
+; SSE-NEXT:    andps %xmm2, %xmm1
 ; SSE-NEXT:    andnps %xmm3, %xmm0
-; SSE-NEXT:    orps %xmm2, %xmm0
+; SSE-NEXT:    orps %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: select_fcmp_olt_f32:
@@ -186,7 +193,7 @@ define float @select_fcmp_olt_f32(float %a, float %b, float %c, float %d) {
 ; AVX512-LABEL: select_fcmp_olt_f32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcmpltss %xmm1, %xmm0, %k1
-; AVX512-NEXT:    vmovss %xmm2, %xmm0, %xmm3 {%k1}
+; AVX512-NEXT:    vmovss %xmm2, %xmm3, %xmm3 {%k1}
 ; AVX512-NEXT:    vmovaps %xmm3, %xmm0
 ; AVX512-NEXT:    retq
   %1 = fcmp olt float %a, %b
@@ -198,9 +205,10 @@ define double @select_fcmp_olt_f64(double %a, double %b, double %c, double %d) {
 ; SSE-LABEL: select_fcmp_olt_f64:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    cmpltsd %xmm1, %xmm0
-; SSE-NEXT:    andpd %xmm0, %xmm2
+; SSE-NEXT:    movaps %xmm0, %xmm1
+; SSE-NEXT:    andpd %xmm2, %xmm1
 ; SSE-NEXT:    andnpd %xmm3, %xmm0
-; SSE-NEXT:    orpd %xmm2, %xmm0
+; SSE-NEXT:    orpd %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: select_fcmp_olt_f64:
@@ -212,7 +220,7 @@ define double @select_fcmp_olt_f64(double %a, double %b, double %c, double %d) {
 ; AVX512-LABEL: select_fcmp_olt_f64:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcmpltsd %xmm1, %xmm0, %k1
-; AVX512-NEXT:    vmovsd %xmm2, %xmm0, %xmm3 {%k1}
+; AVX512-NEXT:    vmovsd %xmm2, %xmm3, %xmm3 {%k1}
 ; AVX512-NEXT:    vmovapd %xmm3, %xmm0
 ; AVX512-NEXT:    retq
   %1 = fcmp olt double %a, %b
@@ -224,9 +232,10 @@ define float @select_fcmp_ole_f32(float %a, float %b, float %c, float %d) {
 ; SSE-LABEL: select_fcmp_ole_f32:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    cmpless %xmm1, %xmm0
-; SSE-NEXT:    andps %xmm0, %xmm2
+; SSE-NEXT:    movaps %xmm0, %xmm1
+; SSE-NEXT:    andps %xmm2, %xmm1
 ; SSE-NEXT:    andnps %xmm3, %xmm0
-; SSE-NEXT:    orps %xmm2, %xmm0
+; SSE-NEXT:    orps %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: select_fcmp_ole_f32:
@@ -238,7 +247,7 @@ define float @select_fcmp_ole_f32(float %a, float %b, float %c, float %d) {
 ; AVX512-LABEL: select_fcmp_ole_f32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcmpless %xmm1, %xmm0, %k1
-; AVX512-NEXT:    vmovss %xmm2, %xmm0, %xmm3 {%k1}
+; AVX512-NEXT:    vmovss %xmm2, %xmm3, %xmm3 {%k1}
 ; AVX512-NEXT:    vmovaps %xmm3, %xmm0
 ; AVX512-NEXT:    retq
   %1 = fcmp ole float %a, %b
@@ -250,9 +259,10 @@ define double @select_fcmp_ole_f64(double %a, double %b, double %c, double %d) {
 ; SSE-LABEL: select_fcmp_ole_f64:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    cmplesd %xmm1, %xmm0
-; SSE-NEXT:    andpd %xmm0, %xmm2
+; SSE-NEXT:    movaps %xmm0, %xmm1
+; SSE-NEXT:    andpd %xmm2, %xmm1
 ; SSE-NEXT:    andnpd %xmm3, %xmm0
-; SSE-NEXT:    orpd %xmm2, %xmm0
+; SSE-NEXT:    orpd %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: select_fcmp_ole_f64:
@@ -264,7 +274,7 @@ define double @select_fcmp_ole_f64(double %a, double %b, double %c, double %d) {
 ; AVX512-LABEL: select_fcmp_ole_f64:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcmplesd %xmm1, %xmm0, %k1
-; AVX512-NEXT:    vmovsd %xmm2, %xmm0, %xmm3 {%k1}
+; AVX512-NEXT:    vmovsd %xmm2, %xmm3, %xmm3 {%k1}
 ; AVX512-NEXT:    vmovapd %xmm3, %xmm0
 ; AVX512-NEXT:    retq
   %1 = fcmp ole double %a, %b
@@ -276,9 +286,10 @@ define float @select_fcmp_ord_f32(float %a, float %b, float %c, float %d) {
 ; SSE-LABEL: select_fcmp_ord_f32:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    cmpordss %xmm1, %xmm0
-; SSE-NEXT:    andps %xmm0, %xmm2
+; SSE-NEXT:    movaps %xmm0, %xmm1
+; SSE-NEXT:    andps %xmm2, %xmm1
 ; SSE-NEXT:    andnps %xmm3, %xmm0
-; SSE-NEXT:    orps %xmm2, %xmm0
+; SSE-NEXT:    orps %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: select_fcmp_ord_f32:
@@ -290,7 +301,7 @@ define float @select_fcmp_ord_f32(float %a, float %b, float %c, float %d) {
 ; AVX512-LABEL: select_fcmp_ord_f32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcmpordss %xmm1, %xmm0, %k1
-; AVX512-NEXT:    vmovss %xmm2, %xmm0, %xmm3 {%k1}
+; AVX512-NEXT:    vmovss %xmm2, %xmm3, %xmm3 {%k1}
 ; AVX512-NEXT:    vmovaps %xmm3, %xmm0
 ; AVX512-NEXT:    retq
   %1 = fcmp ord float %a, %b
@@ -302,9 +313,10 @@ define double @select_fcmp_ord_f64(double %a, double %b, double %c, double %d) {
 ; SSE-LABEL: select_fcmp_ord_f64:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    cmpordsd %xmm1, %xmm0
-; SSE-NEXT:    andpd %xmm0, %xmm2
+; SSE-NEXT:    movaps %xmm0, %xmm1
+; SSE-NEXT:    andpd %xmm2, %xmm1
 ; SSE-NEXT:    andnpd %xmm3, %xmm0
-; SSE-NEXT:    orpd %xmm2, %xmm0
+; SSE-NEXT:    orpd %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: select_fcmp_ord_f64:
@@ -316,7 +328,7 @@ define double @select_fcmp_ord_f64(double %a, double %b, double %c, double %d) {
 ; AVX512-LABEL: select_fcmp_ord_f64:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcmpordsd %xmm1, %xmm0, %k1
-; AVX512-NEXT:    vmovsd %xmm2, %xmm0, %xmm3 {%k1}
+; AVX512-NEXT:    vmovsd %xmm2, %xmm3, %xmm3 {%k1}
 ; AVX512-NEXT:    vmovapd %xmm3, %xmm0
 ; AVX512-NEXT:    retq
   %1 = fcmp ord double %a, %b
@@ -328,9 +340,10 @@ define float @select_fcmp_uno_f32(float %a, float %b, float %c, float %d) {
 ; SSE-LABEL: select_fcmp_uno_f32:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    cmpunordss %xmm1, %xmm0
-; SSE-NEXT:    andps %xmm0, %xmm2
+; SSE-NEXT:    movaps %xmm0, %xmm1
+; SSE-NEXT:    andps %xmm2, %xmm1
 ; SSE-NEXT:    andnps %xmm3, %xmm0
-; SSE-NEXT:    orps %xmm2, %xmm0
+; SSE-NEXT:    orps %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: select_fcmp_uno_f32:
@@ -342,7 +355,7 @@ define float @select_fcmp_uno_f32(float %a, float %b, float %c, float %d) {
 ; AVX512-LABEL: select_fcmp_uno_f32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcmpunordss %xmm1, %xmm0, %k1
-; AVX512-NEXT:    vmovss %xmm2, %xmm0, %xmm3 {%k1}
+; AVX512-NEXT:    vmovss %xmm2, %xmm3, %xmm3 {%k1}
 ; AVX512-NEXT:    vmovaps %xmm3, %xmm0
 ; AVX512-NEXT:    retq
   %1 = fcmp uno float %a, %b
@@ -354,9 +367,10 @@ define double @select_fcmp_uno_f64(double %a, double %b, double %c, double %d) {
 ; SSE-LABEL: select_fcmp_uno_f64:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    cmpunordsd %xmm1, %xmm0
-; SSE-NEXT:    andpd %xmm0, %xmm2
+; SSE-NEXT:    movaps %xmm0, %xmm1
+; SSE-NEXT:    andpd %xmm2, %xmm1
 ; SSE-NEXT:    andnpd %xmm3, %xmm0
-; SSE-NEXT:    orpd %xmm2, %xmm0
+; SSE-NEXT:    orpd %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: select_fcmp_uno_f64:
@@ -368,7 +382,7 @@ define double @select_fcmp_uno_f64(double %a, double %b, double %c, double %d) {
 ; AVX512-LABEL: select_fcmp_uno_f64:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcmpunordsd %xmm1, %xmm0, %k1
-; AVX512-NEXT:    vmovsd %xmm2, %xmm0, %xmm3 {%k1}
+; AVX512-NEXT:    vmovsd %xmm2, %xmm3, %xmm3 {%k1}
 ; AVX512-NEXT:    vmovapd %xmm3, %xmm0
 ; AVX512-NEXT:    retq
   %1 = fcmp uno double %a, %b
@@ -380,9 +394,10 @@ define float @select_fcmp_ugt_f32(float %a, float %b, float %c, float %d) {
 ; SSE-LABEL: select_fcmp_ugt_f32:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    cmpnless %xmm1, %xmm0
-; SSE-NEXT:    andps %xmm0, %xmm2
+; SSE-NEXT:    movaps %xmm0, %xmm1
+; SSE-NEXT:    andps %xmm2, %xmm1
 ; SSE-NEXT:    andnps %xmm3, %xmm0
-; SSE-NEXT:    orps %xmm2, %xmm0
+; SSE-NEXT:    orps %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: select_fcmp_ugt_f32:
@@ -394,7 +409,7 @@ define float @select_fcmp_ugt_f32(float %a, float %b, float %c, float %d) {
 ; AVX512-LABEL: select_fcmp_ugt_f32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcmpnless %xmm1, %xmm0, %k1
-; AVX512-NEXT:    vmovss %xmm2, %xmm0, %xmm3 {%k1}
+; AVX512-NEXT:    vmovss %xmm2, %xmm3, %xmm3 {%k1}
 ; AVX512-NEXT:    vmovaps %xmm3, %xmm0
 ; AVX512-NEXT:    retq
   %1 = fcmp ugt float %a, %b
@@ -406,9 +421,10 @@ define double @select_fcmp_ugt_f64(double %a, double %b, double %c, double %d) {
 ; SSE-LABEL: select_fcmp_ugt_f64:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    cmpnlesd %xmm1, %xmm0
-; SSE-NEXT:    andpd %xmm0, %xmm2
+; SSE-NEXT:    movaps %xmm0, %xmm1
+; SSE-NEXT:    andpd %xmm2, %xmm1
 ; SSE-NEXT:    andnpd %xmm3, %xmm0
-; SSE-NEXT:    orpd %xmm2, %xmm0
+; SSE-NEXT:    orpd %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: select_fcmp_ugt_f64:
@@ -420,7 +436,7 @@ define double @select_fcmp_ugt_f64(double %a, double %b, double %c, double %d) {
 ; AVX512-LABEL: select_fcmp_ugt_f64:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcmpnlesd %xmm1, %xmm0, %k1
-; AVX512-NEXT:    vmovsd %xmm2, %xmm0, %xmm3 {%k1}
+; AVX512-NEXT:    vmovsd %xmm2, %xmm3, %xmm3 {%k1}
 ; AVX512-NEXT:    vmovapd %xmm3, %xmm0
 ; AVX512-NEXT:    retq
   %1 = fcmp ugt double %a, %b
@@ -432,9 +448,10 @@ define float @select_fcmp_uge_f32(float %a, float %b, float %c, float %d) {
 ; SSE-LABEL: select_fcmp_uge_f32:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    cmpnltss %xmm1, %xmm0
-; SSE-NEXT:    andps %xmm0, %xmm2
+; SSE-NEXT:    movaps %xmm0, %xmm1
+; SSE-NEXT:    andps %xmm2, %xmm1
 ; SSE-NEXT:    andnps %xmm3, %xmm0
-; SSE-NEXT:    orps %xmm2, %xmm0
+; SSE-NEXT:    orps %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: select_fcmp_uge_f32:
@@ -446,7 +463,7 @@ define float @select_fcmp_uge_f32(float %a, float %b, float %c, float %d) {
 ; AVX512-LABEL: select_fcmp_uge_f32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcmpnltss %xmm1, %xmm0, %k1
-; AVX512-NEXT:    vmovss %xmm2, %xmm0, %xmm3 {%k1}
+; AVX512-NEXT:    vmovss %xmm2, %xmm3, %xmm3 {%k1}
 ; AVX512-NEXT:    vmovaps %xmm3, %xmm0
 ; AVX512-NEXT:    retq
   %1 = fcmp uge float %a, %b
@@ -458,9 +475,10 @@ define double @select_fcmp_uge_f64(double %a, double %b, double %c, double %d) {
 ; SSE-LABEL: select_fcmp_uge_f64:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    cmpnltsd %xmm1, %xmm0
-; SSE-NEXT:    andpd %xmm0, %xmm2
+; SSE-NEXT:    movaps %xmm0, %xmm1
+; SSE-NEXT:    andpd %xmm2, %xmm1
 ; SSE-NEXT:    andnpd %xmm3, %xmm0
-; SSE-NEXT:    orpd %xmm2, %xmm0
+; SSE-NEXT:    orpd %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: select_fcmp_uge_f64:
@@ -472,7 +490,7 @@ define double @select_fcmp_uge_f64(double %a, double %b, double %c, double %d) {
 ; AVX512-LABEL: select_fcmp_uge_f64:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcmpnltsd %xmm1, %xmm0, %k1
-; AVX512-NEXT:    vmovsd %xmm2, %xmm0, %xmm3 {%k1}
+; AVX512-NEXT:    vmovsd %xmm2, %xmm3, %xmm3 {%k1}
 ; AVX512-NEXT:    vmovapd %xmm3, %xmm0
 ; AVX512-NEXT:    retq
   %1 = fcmp uge double %a, %b
@@ -484,9 +502,10 @@ define float @select_fcmp_ult_f32(float %a, float %b, float %c, float %d) {
 ; SSE-LABEL: select_fcmp_ult_f32:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    cmpnless %xmm0, %xmm1
-; SSE-NEXT:    andps %xmm1, %xmm2
+; SSE-NEXT:    movaps %xmm1, %xmm0
+; SSE-NEXT:    andps %xmm2, %xmm0
 ; SSE-NEXT:    andnps %xmm3, %xmm1
-; SSE-NEXT:    orps %xmm2, %xmm1
+; SSE-NEXT:    orps %xmm0, %xmm1
 ; SSE-NEXT:    movaps %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
@@ -499,7 +518,7 @@ define float @select_fcmp_ult_f32(float %a, float %b, float %c, float %d) {
 ; AVX512-LABEL: select_fcmp_ult_f32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcmpnless %xmm0, %xmm1, %k1
-; AVX512-NEXT:    vmovss %xmm2, %xmm0, %xmm3 {%k1}
+; AVX512-NEXT:    vmovss %xmm2, %xmm3, %xmm3 {%k1}
 ; AVX512-NEXT:    vmovaps %xmm3, %xmm0
 ; AVX512-NEXT:    retq
   %1 = fcmp ult float %a, %b
@@ -511,10 +530,11 @@ define double @select_fcmp_ult_f64(double %a, double %b, double %c, double %d) {
 ; SSE-LABEL: select_fcmp_ult_f64:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    cmpnlesd %xmm0, %xmm1
-; SSE-NEXT:    andpd %xmm1, %xmm2
+; SSE-NEXT:    movaps %xmm1, %xmm0
+; SSE-NEXT:    andpd %xmm2, %xmm0
 ; SSE-NEXT:    andnpd %xmm3, %xmm1
-; SSE-NEXT:    orpd %xmm2, %xmm1
-; SSE-NEXT:    movapd %xmm1, %xmm0
+; SSE-NEXT:    orpd %xmm0, %xmm1
+; SSE-NEXT:    movaps %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: select_fcmp_ult_f64:
@@ -526,7 +546,7 @@ define double @select_fcmp_ult_f64(double %a, double %b, double %c, double %d) {
 ; AVX512-LABEL: select_fcmp_ult_f64:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcmpnlesd %xmm0, %xmm1, %k1
-; AVX512-NEXT:    vmovsd %xmm2, %xmm0, %xmm3 {%k1}
+; AVX512-NEXT:    vmovsd %xmm2, %xmm3, %xmm3 {%k1}
 ; AVX512-NEXT:    vmovapd %xmm3, %xmm0
 ; AVX512-NEXT:    retq
   %1 = fcmp ult double %a, %b
@@ -538,9 +558,10 @@ define float @select_fcmp_ule_f32(float %a, float %b, float %c, float %d) {
 ; SSE-LABEL: select_fcmp_ule_f32:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    cmpnltss %xmm0, %xmm1
-; SSE-NEXT:    andps %xmm1, %xmm2
+; SSE-NEXT:    movaps %xmm1, %xmm0
+; SSE-NEXT:    andps %xmm2, %xmm0
 ; SSE-NEXT:    andnps %xmm3, %xmm1
-; SSE-NEXT:    orps %xmm2, %xmm1
+; SSE-NEXT:    orps %xmm0, %xmm1
 ; SSE-NEXT:    movaps %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
@@ -553,7 +574,7 @@ define float @select_fcmp_ule_f32(float %a, float %b, float %c, float %d) {
 ; AVX512-LABEL: select_fcmp_ule_f32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcmpnltss %xmm0, %xmm1, %k1
-; AVX512-NEXT:    vmovss %xmm2, %xmm0, %xmm3 {%k1}
+; AVX512-NEXT:    vmovss %xmm2, %xmm3, %xmm3 {%k1}
 ; AVX512-NEXT:    vmovaps %xmm3, %xmm0
 ; AVX512-NEXT:    retq
   %1 = fcmp ule float %a, %b
@@ -565,10 +586,11 @@ define double @select_fcmp_ule_f64(double %a, double %b, double %c, double %d) {
 ; SSE-LABEL: select_fcmp_ule_f64:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    cmpnltsd %xmm0, %xmm1
-; SSE-NEXT:    andpd %xmm1, %xmm2
+; SSE-NEXT:    movaps %xmm1, %xmm0
+; SSE-NEXT:    andpd %xmm2, %xmm0
 ; SSE-NEXT:    andnpd %xmm3, %xmm1
-; SSE-NEXT:    orpd %xmm2, %xmm1
-; SSE-NEXT:    movapd %xmm1, %xmm0
+; SSE-NEXT:    orpd %xmm0, %xmm1
+; SSE-NEXT:    movaps %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: select_fcmp_ule_f64:
@@ -580,7 +602,7 @@ define double @select_fcmp_ule_f64(double %a, double %b, double %c, double %d) {
 ; AVX512-LABEL: select_fcmp_ule_f64:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcmpnltsd %xmm0, %xmm1, %k1
-; AVX512-NEXT:    vmovsd %xmm2, %xmm0, %xmm3 {%k1}
+; AVX512-NEXT:    vmovsd %xmm2, %xmm3, %xmm3 {%k1}
 ; AVX512-NEXT:    vmovapd %xmm3, %xmm0
 ; AVX512-NEXT:    retq
   %1 = fcmp ule double %a, %b
@@ -592,9 +614,10 @@ define float @select_fcmp_une_f32(float %a, float %b, float %c, float %d) {
 ; SSE-LABEL: select_fcmp_une_f32:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    cmpneqss %xmm1, %xmm0
-; SSE-NEXT:    andps %xmm0, %xmm2
+; SSE-NEXT:    movaps %xmm0, %xmm1
+; SSE-NEXT:    andps %xmm2, %xmm1
 ; SSE-NEXT:    andnps %xmm3, %xmm0
-; SSE-NEXT:    orps %xmm2, %xmm0
+; SSE-NEXT:    orps %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: select_fcmp_une_f32:
@@ -606,7 +629,7 @@ define float @select_fcmp_une_f32(float %a, float %b, float %c, float %d) {
 ; AVX512-LABEL: select_fcmp_une_f32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcmpneqss %xmm1, %xmm0, %k1
-; AVX512-NEXT:    vmovss %xmm2, %xmm0, %xmm3 {%k1}
+; AVX512-NEXT:    vmovss %xmm2, %xmm3, %xmm3 {%k1}
 ; AVX512-NEXT:    vmovaps %xmm3, %xmm0
 ; AVX512-NEXT:    retq
   %1 = fcmp une float %a, %b
@@ -618,9 +641,10 @@ define double @select_fcmp_une_f64(double %a, double %b, double %c, double %d) {
 ; SSE-LABEL: select_fcmp_une_f64:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    cmpneqsd %xmm1, %xmm0
-; SSE-NEXT:    andpd %xmm0, %xmm2
+; SSE-NEXT:    movaps %xmm0, %xmm1
+; SSE-NEXT:    andpd %xmm2, %xmm1
 ; SSE-NEXT:    andnpd %xmm3, %xmm0
-; SSE-NEXT:    orpd %xmm2, %xmm0
+; SSE-NEXT:    orpd %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: select_fcmp_une_f64:
@@ -632,7 +656,7 @@ define double @select_fcmp_une_f64(double %a, double %b, double %c, double %d) {
 ; AVX512-LABEL: select_fcmp_une_f64:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcmpneqsd %xmm1, %xmm0, %k1
-; AVX512-NEXT:    vmovsd %xmm2, %xmm0, %xmm3 {%k1}
+; AVX512-NEXT:    vmovsd %xmm2, %xmm3, %xmm3 {%k1}
 ; AVX512-NEXT:    vmovapd %xmm3, %xmm0
 ; AVX512-NEXT:    retq
   %1 = fcmp une double %a, %b
diff --git a/llvm/test/CodeGen/X86/fast-isel-vecload.ll b/llvm/test/CodeGen/X86/fast-isel-vecload.ll
index 31730493f..1cdeae96b 100644
--- a/llvm/test/CodeGen/X86/fast-isel-vecload.ll
+++ b/llvm/test/CodeGen/X86/fast-isel-vecload.ll
@@ -684,16 +684,10 @@ define <64 x i8> @test_v64i8(<64 x i8>* %V) {
 ; AVXONLY-NEXT:    vmovaps 32(%rdi), %ymm1
 ; AVXONLY-NEXT:    retq
 ;
-; KNL-LABEL: test_v64i8:
-; KNL:       # %bb.0: # %entry
-; KNL-NEXT:    vmovaps (%rdi), %ymm0
-; KNL-NEXT:    vmovaps 32(%rdi), %ymm1
-; KNL-NEXT:    retq
-;
-; SKX-LABEL: test_v64i8:
-; SKX:       # %bb.0: # %entry
-; SKX-NEXT:    vmovdqa64 (%rdi), %zmm0
-; SKX-NEXT:    retq
+; AVX512-LABEL: test_v64i8:
+; AVX512:       # %bb.0: # %entry
+; AVX512-NEXT:    vmovdqa64 (%rdi), %zmm0
+; AVX512-NEXT:    retq
 entry:
   %0 = load <64 x i8>, <64 x i8>* %V, align 64
   ret <64 x i8> %0
@@ -714,16 +708,10 @@ define <32 x i16> @test_v32i16(<32 x i16>* %V) {
 ; AVXONLY-NEXT:    vmovaps 32(%rdi), %ymm1
 ; AVXONLY-NEXT:    retq
 ;
-; KNL-LABEL: test_v32i16:
-; KNL:       # %bb.0: # %entry
-; KNL-NEXT:    vmovaps (%rdi), %ymm0
-; KNL-NEXT:    vmovaps 32(%rdi), %ymm1
-; KNL-NEXT:    retq
-;
-; SKX-LABEL: test_v32i16:
-; SKX:       # %bb.0: # %entry
-; SKX-NEXT:    vmovdqa64 (%rdi), %zmm0
-; SKX-NEXT:    retq
+; AVX512-LABEL: test_v32i16:
+; AVX512:       # %bb.0: # %entry
+; AVX512-NEXT:    vmovdqa64 (%rdi), %zmm0
+; AVX512-NEXT:    retq
 entry:
   %0 = load <32 x i16>, <32 x i16>* %V, align 64
   ret <32 x i16> %0
@@ -792,16 +780,10 @@ define <64 x i8> @test_v64i8_unaligned(<64 x i8>* %V) {
 ; AVXONLY-NEXT:    vmovups 32(%rdi), %ymm1
 ; AVXONLY-NEXT:    retq
 ;
-; KNL-LABEL: test_v64i8_unaligned:
-; KNL:       # %bb.0: # %entry
-; KNL-NEXT:    vmovups (%rdi), %ymm0
-; KNL-NEXT:    vmovups 32(%rdi), %ymm1
-; KNL-NEXT:    retq
-;
-; SKX-LABEL: test_v64i8_unaligned:
-; SKX:       # %bb.0: # %entry
-; SKX-NEXT:    vmovdqu64 (%rdi), %zmm0
-; SKX-NEXT:    retq
+; AVX512-LABEL: test_v64i8_unaligned:
+; AVX512:       # %bb.0: # %entry
+; AVX512-NEXT:    vmovdqu64 (%rdi), %zmm0
+; AVX512-NEXT:    retq
 entry:
   %0 = load <64 x i8>, <64 x i8>* %V, align 4
   ret <64 x i8> %0
@@ -822,16 +804,10 @@ define <32 x i16> @test_v32i16_unaligned(<32 x i16>* %V) {
 ; AVXONLY-NEXT:    vmovups 32(%rdi), %ymm1
 ; AVXONLY-NEXT:    retq
 ;
-; KNL-LABEL: test_v32i16_unaligned:
-; KNL:       # %bb.0: # %entry
-; KNL-NEXT:    vmovups (%rdi), %ymm0
-; KNL-NEXT:    vmovups 32(%rdi), %ymm1
-; KNL-NEXT:    retq
-;
-; SKX-LABEL: test_v32i16_unaligned:
-; SKX:       # %bb.0: # %entry
-; SKX-NEXT:    vmovdqu64 (%rdi), %zmm0
-; SKX-NEXT:    retq
+; AVX512-LABEL: test_v32i16_unaligned:
+; AVX512:       # %bb.0: # %entry
+; AVX512-NEXT:    vmovdqu64 (%rdi), %zmm0
+; AVX512-NEXT:    retq
 entry:
   %0 = load <32 x i16>, <32 x i16>* %V, align 4
   ret <32 x i16> %0
diff --git a/llvm/test/CodeGen/X86/field-extract-use-trunc.ll b/llvm/test/CodeGen/X86/field-extract-use-trunc.ll
index e7d2f41f0..1a169a046 100644
--- a/llvm/test/CodeGen/X86/field-extract-use-trunc.ll
+++ b/llvm/test/CodeGen/X86/field-extract-use-trunc.ll
@@ -1,38 +1,107 @@
-; RUN: llc < %s -mtriple=i686-- | grep sar | count 1
-; RUN: llc < %s -mtriple=x86_64-- | not grep sar
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- | FileCheck %s --check-prefix=i686
+; RUN: llc < %s -mtriple=x86_64-- | FileCheck %s --check-prefix=x86_64
 
 define i32 @test(i32 %f12) nounwind {
-	%tmp7.25 = lshr i32 %f12, 16		
+; i686-LABEL: test:
+; i686:       # %bb.0:
+; i686-NEXT:    movsbl {{[0-9]+}}(%esp), %eax
+; i686-NEXT:    retl
+;
+; x86_64-LABEL: test:
+; x86_64:       # %bb.0:
+; x86_64-NEXT:    shrl $16, %edi
+; x86_64-NEXT:    movsbl %dil, %eax
+; x86_64-NEXT:    retq
+	%tmp7.25 = lshr i32 %f12, 16
 	%tmp7.26 = trunc i32 %tmp7.25 to i8
 	%tmp78.2 = sext i8 %tmp7.26 to i32
 	ret i32 %tmp78.2
 }
 
 define i32 @test2(i32 %f12) nounwind {
+; i686-LABEL: test2:
+; i686:       # %bb.0:
+; i686-NEXT:    movsbl {{[0-9]+}}(%esp), %eax
+; i686-NEXT:    retl
+;
+; x86_64-LABEL: test2:
+; x86_64:       # %bb.0:
+; x86_64-NEXT:    shrl $16, %edi
+; x86_64-NEXT:    movsbl %dil, %eax
+; x86_64-NEXT:    retq
 	%f11 = shl i32 %f12, 8
 	%tmp7.25 = ashr i32 %f11, 24
 	ret i32 %tmp7.25
 }
 
 define i32 @test3(i32 %f12) nounwind {
+; i686-LABEL: test3:
+; i686:       # %bb.0:
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; i686-NEXT:    shrl $11, %eax
+; i686-NEXT:    movsbl %al, %eax
+; i686-NEXT:    retl
+;
+; x86_64-LABEL: test3:
+; x86_64:       # %bb.0:
+; x86_64-NEXT:    shrl $11, %edi
+; x86_64-NEXT:    movsbl %dil, %eax
+; x86_64-NEXT:    retq
 	%f11 = shl i32 %f12, 13
 	%tmp7.25 = ashr i32 %f11, 24
 	ret i32 %tmp7.25
 }
 
 define i64 @test4(i64 %f12) nounwind {
+; i686-LABEL: test4:
+; i686:       # %bb.0:
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; i686-NEXT:    movl %eax, %edx
+; i686-NEXT:    sarl $31, %edx
+; i686-NEXT:    retl
+;
+; x86_64-LABEL: test4:
+; x86_64:       # %bb.0:
+; x86_64-NEXT:    movslq %edi, %rax
+; x86_64-NEXT:    retq
 	%f11 = shl i64 %f12, 32
 	%tmp7.25 = ashr i64 %f11, 32
 	ret i64 %tmp7.25
 }
 
 define i16 @test5(i16 %f12) nounwind {
+; i686-LABEL: test5:
+; i686:       # %bb.0:
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; i686-NEXT:    shrl $6, %eax
+; i686-NEXT:    movsbl %al, %eax
+; i686-NEXT:    # kill: def $ax killed $ax killed $eax
+; i686-NEXT:    retl
+;
+; x86_64-LABEL: test5:
+; x86_64:       # %bb.0:
+; x86_64-NEXT:    shrl $6, %edi
+; x86_64-NEXT:    movsbl %dil, %eax
+; x86_64-NEXT:    # kill: def $ax killed $ax killed $eax
+; x86_64-NEXT:    retq
 	%f11 = shl i16 %f12, 2
 	%tmp7.25 = ashr i16 %f11, 8
 	ret i16 %tmp7.25
 }
 
 define i16 @test6(i16 %f12) nounwind {
+; i686-LABEL: test6:
+; i686:       # %bb.0:
+; i686-NEXT:    movsbl {{[0-9]+}}(%esp), %eax
+; i686-NEXT:    # kill: def $ax killed $ax killed $eax
+; i686-NEXT:    retl
+;
+; x86_64-LABEL: test6:
+; x86_64:       # %bb.0:
+; x86_64-NEXT:    movsbl %dil, %eax
+; x86_64-NEXT:    # kill: def $ax killed $ax killed $eax
+; x86_64-NEXT:    retq
 	%f11 = shl i16 %f12, 8
 	%tmp7.25 = ashr i16 %f11, 8
 	ret i16 %tmp7.25
diff --git a/llvm/test/CodeGen/X86/fildll.ll b/llvm/test/CodeGen/X86/fildll.ll
index aeb753c06..a91974fbd 100644
--- a/llvm/test/CodeGen/X86/fildll.ll
+++ b/llvm/test/CodeGen/X86/fildll.ll
@@ -1,11 +1,48 @@
-; RUN: llc < %s -mtriple=i686-- -x86-asm-syntax=att -mattr=-sse2 | grep fildll | count 2
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- -x86-asm-syntax=att -mattr=-sse2 | FileCheck %s
 
 define fastcc double @sint64_to_fp(i64 %X) {
+; CHECK-LABEL: sint64_to_fp:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushl %ebp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    .cfi_offset %ebp, -8
+; CHECK-NEXT:    movl %esp, %ebp
+; CHECK-NEXT:    .cfi_def_cfa_register %ebp
+; CHECK-NEXT:    andl $-8, %esp
+; CHECK-NEXT:    subl $8, %esp
+; CHECK-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movl %ecx, (%esp)
+; CHECK-NEXT:    fildll (%esp)
+; CHECK-NEXT:    movl %ebp, %esp
+; CHECK-NEXT:    popl %ebp
+; CHECK-NEXT:    .cfi_def_cfa %esp, 4
+; CHECK-NEXT:    retl
         %R = sitofp i64 %X to double            ; <double> [#uses=1]
         ret double %R
 }
 
 define fastcc double @uint64_to_fp(i64 %X) {
+; CHECK-LABEL: uint64_to_fp:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushl %ebp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    .cfi_offset %ebp, -8
+; CHECK-NEXT:    movl %esp, %ebp
+; CHECK-NEXT:    .cfi_def_cfa_register %ebp
+; CHECK-NEXT:    andl $-8, %esp
+; CHECK-NEXT:    subl $16, %esp
+; CHECK-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movl %ecx, (%esp)
+; CHECK-NEXT:    shrl $31, %edx
+; CHECK-NEXT:    fildll (%esp)
+; CHECK-NEXT:    fadds {{\.LCPI.*}}(,%edx,4)
+; CHECK-NEXT:    fstpl {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fldl {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movl %ebp, %esp
+; CHECK-NEXT:    popl %ebp
+; CHECK-NEXT:    .cfi_def_cfa %esp, 4
+; CHECK-NEXT:    retl
         %R = uitofp i64 %X to double            ; <double> [#uses=1]
         ret double %R
 }
diff --git a/llvm/test/CodeGen/X86/finite-libcalls.ll b/llvm/test/CodeGen/X86/finite-libcalls.ll
index d54ee48ea..31fadfb0a 100644
--- a/llvm/test/CodeGen/X86/finite-libcalls.ll
+++ b/llvm/test/CodeGen/X86/finite-libcalls.ll
@@ -9,7 +9,7 @@
 define float @exp_f32(float %x) #0 {
 ; GNU-LABEL: exp_f32:
 ; GNU:       # %bb.0:
-; GNU-NEXT:    jmp __expf_finite # TAILCALL
+; GNU-NEXT:    jmp expf # TAILCALL
 ;
 ; WIN-LABEL: exp_f32:
 ; WIN:       # %bb.0:
@@ -25,7 +25,7 @@ define float @exp_f32(float %x) #0 {
 define double @exp_f64(double %x) #0 {
 ; GNU-LABEL: exp_f64:
 ; GNU:       # %bb.0:
-; GNU-NEXT:    jmp __exp_finite # TAILCALL
+; GNU-NEXT:    jmp exp # TAILCALL
 ;
 ; WIN-LABEL: exp_f64:
 ; WIN:       # %bb.0:
@@ -44,7 +44,7 @@ define x86_fp80 @exp_f80(x86_fp80 %x) #0 {
 ; GNU-NEXT:    subq $24, %rsp
 ; GNU-NEXT:    fldt {{[0-9]+}}(%rsp)
 ; GNU-NEXT:    fstpt (%rsp)
-; GNU-NEXT:    callq __expl_finite
+; GNU-NEXT:    callq expl
 ; GNU-NEXT:    addq $24, %rsp
 ; GNU-NEXT:    retq
 ;
@@ -80,7 +80,7 @@ define x86_fp80 @exp_f80(x86_fp80 %x) #0 {
 define float @exp2_f32(float %x) #0 {
 ; GNU-LABEL: exp2_f32:
 ; GNU:       # %bb.0:
-; GNU-NEXT:    jmp __exp2f_finite # TAILCALL
+; GNU-NEXT:    jmp exp2f # TAILCALL
 ;
 ; WIN-LABEL: exp2_f32:
 ; WIN:       # %bb.0:
@@ -96,7 +96,7 @@ define float @exp2_f32(float %x) #0 {
 define double @exp2_f64(double %x) #0 {
 ; GNU-LABEL: exp2_f64:
 ; GNU:       # %bb.0:
-; GNU-NEXT:    jmp __exp2_finite # TAILCALL
+; GNU-NEXT:    jmp exp2 # TAILCALL
 ;
 ; WIN-LABEL: exp2_f64:
 ; WIN:       # %bb.0:
@@ -115,7 +115,7 @@ define x86_fp80 @exp2_f80(x86_fp80 %x) #0 {
 ; GNU-NEXT:    subq $24, %rsp
 ; GNU-NEXT:    fldt {{[0-9]+}}(%rsp)
 ; GNU-NEXT:    fstpt (%rsp)
-; GNU-NEXT:    callq __exp2l_finite
+; GNU-NEXT:    callq exp2l
 ; GNU-NEXT:    addq $24, %rsp
 ; GNU-NEXT:    retq
 ;
@@ -151,7 +151,7 @@ define x86_fp80 @exp2_f80(x86_fp80 %x) #0 {
 define float @log_f32(float %x) #0 {
 ; GNU-LABEL: log_f32:
 ; GNU:       # %bb.0:
-; GNU-NEXT:    jmp __logf_finite # TAILCALL
+; GNU-NEXT:    jmp logf # TAILCALL
 ;
 ; WIN-LABEL: log_f32:
 ; WIN:       # %bb.0:
@@ -167,7 +167,7 @@ define float @log_f32(float %x) #0 {
 define double @log_f64(double %x) #0 {
 ; GNU-LABEL: log_f64:
 ; GNU:       # %bb.0:
-; GNU-NEXT:    jmp __log_finite # TAILCALL
+; GNU-NEXT:    jmp log # TAILCALL
 ;
 ; WIN-LABEL: log_f64:
 ; WIN:       # %bb.0:
@@ -186,7 +186,7 @@ define x86_fp80 @log_f80(x86_fp80 %x) #0 {
 ; GNU-NEXT:    subq $24, %rsp
 ; GNU-NEXT:    fldt {{[0-9]+}}(%rsp)
 ; GNU-NEXT:    fstpt (%rsp)
-; GNU-NEXT:    callq __logl_finite
+; GNU-NEXT:    callq logl
 ; GNU-NEXT:    addq $24, %rsp
 ; GNU-NEXT:    retq
 ;
@@ -222,7 +222,7 @@ define x86_fp80 @log_f80(x86_fp80 %x) #0 {
 define float @log2_f32(float %x) #0 {
 ; GNU-LABEL: log2_f32:
 ; GNU:       # %bb.0:
-; GNU-NEXT:    jmp __log2f_finite # TAILCALL
+; GNU-NEXT:    jmp log2f # TAILCALL
 ;
 ; WIN-LABEL: log2_f32:
 ; WIN:       # %bb.0:
@@ -238,7 +238,7 @@ define float @log2_f32(float %x) #0 {
 define double @log2_f64(double %x) #0 {
 ; GNU-LABEL: log2_f64:
 ; GNU:       # %bb.0:
-; GNU-NEXT:    jmp __log2_finite # TAILCALL
+; GNU-NEXT:    jmp log2 # TAILCALL
 ;
 ; WIN-LABEL: log2_f64:
 ; WIN:       # %bb.0:
@@ -257,7 +257,7 @@ define x86_fp80 @log2_f80(x86_fp80 %x) #0 {
 ; GNU-NEXT:    subq $24, %rsp
 ; GNU-NEXT:    fldt {{[0-9]+}}(%rsp)
 ; GNU-NEXT:    fstpt (%rsp)
-; GNU-NEXT:    callq __log2l_finite
+; GNU-NEXT:    callq log2l
 ; GNU-NEXT:    addq $24, %rsp
 ; GNU-NEXT:    retq
 ;
@@ -293,7 +293,7 @@ define x86_fp80 @log2_f80(x86_fp80 %x) #0 {
 define float @log10_f32(float %x) #0 {
 ; GNU-LABEL: log10_f32:
 ; GNU:       # %bb.0:
-; GNU-NEXT:    jmp __log10f_finite # TAILCALL
+; GNU-NEXT:    jmp log10f # TAILCALL
 ;
 ; WIN-LABEL: log10_f32:
 ; WIN:       # %bb.0:
@@ -309,7 +309,7 @@ define float @log10_f32(float %x) #0 {
 define double @log10_f64(double %x) #0 {
 ; GNU-LABEL: log10_f64:
 ; GNU:       # %bb.0:
-; GNU-NEXT:    jmp __log10_finite # TAILCALL
+; GNU-NEXT:    jmp log10 # TAILCALL
 ;
 ; WIN-LABEL: log10_f64:
 ; WIN:       # %bb.0:
@@ -328,7 +328,7 @@ define x86_fp80 @log10_f80(x86_fp80 %x) #0 {
 ; GNU-NEXT:    subq $24, %rsp
 ; GNU-NEXT:    fldt {{[0-9]+}}(%rsp)
 ; GNU-NEXT:    fstpt (%rsp)
-; GNU-NEXT:    callq __log10l_finite
+; GNU-NEXT:    callq log10l
 ; GNU-NEXT:    addq $24, %rsp
 ; GNU-NEXT:    retq
 ;
@@ -365,7 +365,7 @@ define float @pow_f32(float %x) #0 {
 ; GNU-LABEL: pow_f32:
 ; GNU:       # %bb.0:
 ; GNU-NEXT:    movaps %xmm0, %xmm1
-; GNU-NEXT:    jmp __powf_finite # TAILCALL
+; GNU-NEXT:    jmp powf # TAILCALL
 ;
 ; WIN-LABEL: pow_f32:
 ; WIN:       # %bb.0:
@@ -384,7 +384,7 @@ define double @pow_f64(double %x) #0 {
 ; GNU-LABEL: pow_f64:
 ; GNU:       # %bb.0:
 ; GNU-NEXT:    movaps %xmm0, %xmm1
-; GNU-NEXT:    jmp __pow_finite # TAILCALL
+; GNU-NEXT:    jmp pow # TAILCALL
 ;
 ; WIN-LABEL: pow_f64:
 ; WIN:       # %bb.0:
@@ -407,7 +407,7 @@ define x86_fp80 @pow_f80(x86_fp80 %x) #0 {
 ; GNU-NEXT:    fld %st(0)
 ; GNU-NEXT:    fstpt {{[0-9]+}}(%rsp)
 ; GNU-NEXT:    fstpt (%rsp)
-; GNU-NEXT:    callq __powl_finite
+; GNU-NEXT:    callq powl
 ; GNU-NEXT:    addq $40, %rsp
 ; GNU-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/fixup-bw-copy.ll b/llvm/test/CodeGen/X86/fixup-bw-copy.ll
index ed15ec3b8..16a70e077 100644
--- a/llvm/test/CodeGen/X86/fixup-bw-copy.ll
+++ b/llvm/test/CodeGen/X86/fixup-bw-copy.ll
@@ -52,7 +52,7 @@ define i8 @test_movb_hreg(i16 %a0) {
 ;
 ; X32-LABEL: test_movb_hreg:
 ; X32:       # %bb.0:
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
 ; X32-NEXT:    addb %al, %ah
 ; X32-NEXT:    movb %ah, %al
 ; X32-NEXT:    retl
diff --git a/llvm/test/CodeGen/X86/fixup-lea.ll b/llvm/test/CodeGen/X86/fixup-lea.ll
index 8d8a1cd19..3f661a8e9 100644
--- a/llvm/test/CodeGen/X86/fixup-lea.ll
+++ b/llvm/test/CodeGen/X86/fixup-lea.ll
@@ -108,17 +108,96 @@ for.end:
   ret void
 }
 
+define void @foo_pgso(i32 inreg %dns) !prof !14 {
+; SLOW-LABEL: foo_pgso:
+; SLOW:       # %bb.0: # %entry
+; SLOW-NEXT:    xorl %ecx, %ecx
+; SLOW-NEXT:    decl %ecx
+; SLOW-NEXT:  .LBB4_1: # %for.body
+; SLOW-NEXT:    # =>This Inner Loop Header: Depth=1
+; SLOW-NEXT:    movzwl %cx, %edx
+; SLOW-NEXT:    decl %ecx
+; SLOW-NEXT:    cmpl %eax, %edx
+; SLOW-NEXT:    jl .LBB4_1
+; SLOW-NEXT:  # %bb.2: # %for.end
+; SLOW-NEXT:    retl
+;
+; FAST-LABEL: foo_pgso:
+; FAST:       # %bb.0: # %entry
+; FAST-NEXT:    xorl %ecx, %ecx
+; FAST-NEXT:    decl %ecx
+; FAST-NEXT:  .LBB4_1: # %for.body
+; FAST-NEXT:    # =>This Inner Loop Header: Depth=1
+; FAST-NEXT:    movzwl %cx, %edx
+; FAST-NEXT:    addl $-1, %ecx
+; FAST-NEXT:    cmpl %eax, %edx
+; FAST-NEXT:    jl .LBB4_1
+; FAST-NEXT:  # %bb.2: # %for.end
+; FAST-NEXT:    retl
+entry:
+  br label %for.body
+
+for.body:
+  %i.05 = phi i16 [ %dec, %for.body ], [ 0, %entry ]
+  %dec = add i16 %i.05, -1
+  %conv = zext i16 %dec to i32
+  %cmp = icmp slt i32 %conv, %dns
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+define void @bar_pgso(i32 inreg %dns) !prof !14 {
+; SLOW-LABEL: bar_pgso:
+; SLOW:       # %bb.0: # %entry
+; SLOW-NEXT:    xorl %ecx, %ecx
+; SLOW-NEXT:    incl %ecx
+; SLOW-NEXT:  .LBB5_1: # %for.body
+; SLOW-NEXT:    # =>This Inner Loop Header: Depth=1
+; SLOW-NEXT:    movzwl %cx, %edx
+; SLOW-NEXT:    incl %ecx
+; SLOW-NEXT:    cmpl %eax, %edx
+; SLOW-NEXT:    jl .LBB5_1
+; SLOW-NEXT:  # %bb.2: # %for.end
+; SLOW-NEXT:    retl
+;
+; FAST-LABEL: bar_pgso:
+; FAST:       # %bb.0: # %entry
+; FAST-NEXT:    xorl %ecx, %ecx
+; FAST-NEXT:    incl %ecx
+; FAST-NEXT:  .LBB5_1: # %for.body
+; FAST-NEXT:    # =>This Inner Loop Header: Depth=1
+; FAST-NEXT:    movzwl %cx, %edx
+; FAST-NEXT:    addl $1, %ecx
+; FAST-NEXT:    cmpl %eax, %edx
+; FAST-NEXT:    jl .LBB5_1
+; FAST-NEXT:  # %bb.2: # %for.end
+; FAST-NEXT:    retl
+entry:
+  br label %for.body
+
+for.body:
+  %i.05 = phi i16 [ %inc, %for.body ], [ 0, %entry ]
+  %inc = add i16 %i.05, 1
+  %conv = zext i16 %inc to i32
+  %cmp = icmp slt i32 %conv, %dns
+  br i1 %cmp, label %for.body, label %for.end
+for.end:
+  ret void
+}
+
 define void @foo_nosize(i32 inreg %dns) {
 ; SLOW-LABEL: foo_nosize:
 ; SLOW:       # %bb.0: # %entry
 ; SLOW-NEXT:    movw $-1, %cx
 ; SLOW-NEXT:    .p2align 4, 0x90
-; SLOW-NEXT:  .LBB4_1: # %for.body
+; SLOW-NEXT:  .LBB6_1: # %for.body
 ; SLOW-NEXT:    # =>This Inner Loop Header: Depth=1
 ; SLOW-NEXT:    movzwl %cx, %edx
 ; SLOW-NEXT:    decl %ecx
 ; SLOW-NEXT:    cmpl %eax, %edx
-; SLOW-NEXT:    jl .LBB4_1
+; SLOW-NEXT:    jl .LBB6_1
 ; SLOW-NEXT:  # %bb.2: # %for.end
 ; SLOW-NEXT:    retl
 ;
@@ -126,12 +205,12 @@ define void @foo_nosize(i32 inreg %dns) {
 ; FAST:       # %bb.0: # %entry
 ; FAST-NEXT:    movw $-1, %cx
 ; FAST-NEXT:    .p2align 4, 0x90
-; FAST-NEXT:  .LBB4_1: # %for.body
+; FAST-NEXT:  .LBB6_1: # %for.body
 ; FAST-NEXT:    # =>This Inner Loop Header: Depth=1
 ; FAST-NEXT:    movzwl %cx, %edx
 ; FAST-NEXT:    addl $-1, %ecx
 ; FAST-NEXT:    cmpl %eax, %edx
-; FAST-NEXT:    jl .LBB4_1
+; FAST-NEXT:    jl .LBB6_1
 ; FAST-NEXT:  # %bb.2: # %for.end
 ; FAST-NEXT:    retl
 entry:
@@ -153,12 +232,12 @@ define void @bar_nosize(i32 inreg %dns) {
 ; SLOW:       # %bb.0: # %entry
 ; SLOW-NEXT:    movw $1, %cx
 ; SLOW-NEXT:    .p2align 4, 0x90
-; SLOW-NEXT:  .LBB5_1: # %for.body
+; SLOW-NEXT:  .LBB7_1: # %for.body
 ; SLOW-NEXT:    # =>This Inner Loop Header: Depth=1
 ; SLOW-NEXT:    movzwl %cx, %edx
 ; SLOW-NEXT:    incl %ecx
 ; SLOW-NEXT:    cmpl %eax, %edx
-; SLOW-NEXT:    jl .LBB5_1
+; SLOW-NEXT:    jl .LBB7_1
 ; SLOW-NEXT:  # %bb.2: # %for.end
 ; SLOW-NEXT:    retl
 ;
@@ -166,12 +245,12 @@ define void @bar_nosize(i32 inreg %dns) {
 ; FAST:       # %bb.0: # %entry
 ; FAST-NEXT:    movw $1, %cx
 ; FAST-NEXT:    .p2align 4, 0x90
-; FAST-NEXT:  .LBB5_1: # %for.body
+; FAST-NEXT:  .LBB7_1: # %for.body
 ; FAST-NEXT:    # =>This Inner Loop Header: Depth=1
 ; FAST-NEXT:    movzwl %cx, %edx
 ; FAST-NEXT:    addl $1, %ecx
 ; FAST-NEXT:    cmpl %eax, %edx
-; FAST-NEXT:    jl .LBB5_1
+; FAST-NEXT:    jl .LBB7_1
 ; FAST-NEXT:  # %bb.2: # %for.end
 ; FAST-NEXT:    retl
 entry:
@@ -186,3 +265,20 @@ for.body:
 for.end:
   ret void
 }
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 1, !"ProfileSummary", !1}
+!1 = !{!2, !3, !4, !5, !6, !7, !8, !9}
+!2 = !{!"ProfileFormat", !"InstrProf"}
+!3 = !{!"TotalCount", i64 10000}
+!4 = !{!"MaxCount", i64 10}
+!5 = !{!"MaxInternalCount", i64 1}
+!6 = !{!"MaxFunctionCount", i64 1000}
+!7 = !{!"NumCounts", i64 3}
+!8 = !{!"NumFunctions", i64 3}
+!9 = !{!"DetailedSummary", !10}
+!10 = !{!11, !12, !13}
+!11 = !{i32 10000, i64 100, i32 1}
+!12 = !{i32 999000, i64 100, i32 1}
+!13 = !{i32 999999, i64 1, i32 2}
+!14 = !{!"function_entry_count", i64 0}
diff --git a/llvm/test/CodeGen/X86/float-conv-elim.ll b/llvm/test/CodeGen/X86/float-conv-elim.ll
index 9ec189b05..4a651cf93 100644
--- a/llvm/test/CodeGen/X86/float-conv-elim.ll
+++ b/llvm/test/CodeGen/X86/float-conv-elim.ll
@@ -28,5 +28,5 @@ define zeroext i8 @bar(i8 zeroext %a) #0 {
   ret i8 %conv1
 }
 
-attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
diff --git a/llvm/test/CodeGen/X86/fma-fneg-combine-2.ll b/llvm/test/CodeGen/X86/fma-fneg-combine-2.ll
new file mode 100644
index 000000000..bc1e1beb4
--- /dev/null
+++ b/llvm/test/CodeGen/X86/fma-fneg-combine-2.ll
@@ -0,0 +1,109 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx,fma | FileCheck %s --check-prefixes=CHECK,FMA3
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx,fma4 | FileCheck %s --check-prefixes=CHECK,FMA4
+
+define float @test_fneg_fma_subx_y_negz_f32(float %w, float %x, float %y, float %z)  {
+; FMA3-LABEL: test_fneg_fma_subx_y_negz_f32:
+; FMA3:       # %bb.0: # %entry
+; FMA3-NEXT:    vsubss %xmm0, %xmm1, %xmm0
+; FMA3-NEXT:    vfmadd213ss {{.*#+}} xmm0 = (xmm2 * xmm0) + xmm3
+; FMA3-NEXT:    retq
+;
+; FMA4-LABEL: test_fneg_fma_subx_y_negz_f32:
+; FMA4:       # %bb.0: # %entry
+; FMA4-NEXT:    vsubss %xmm0, %xmm1, %xmm0
+; FMA4-NEXT:    vfmaddss %xmm3, %xmm2, %xmm0, %xmm0
+; FMA4-NEXT:    retq
+entry:
+  %subx = fsub nsz float %w, %x
+  %negz = fsub float -0.000000e+00, %z
+  %0 = tail call nsz float @llvm.fma.f32(float %subx, float %y, float %negz)
+  %1 = fsub float -0.000000e+00, %0
+  ret float %1
+}
+
+define float @test_fneg_fma_x_suby_negz_f32(float %w, float %x, float %y, float %z)  {
+; FMA3-LABEL: test_fneg_fma_x_suby_negz_f32:
+; FMA3:       # %bb.0: # %entry
+; FMA3-NEXT:    vsubss %xmm0, %xmm2, %xmm0
+; FMA3-NEXT:    vfmadd213ss {{.*#+}} xmm0 = (xmm1 * xmm0) + xmm3
+; FMA3-NEXT:    retq
+;
+; FMA4-LABEL: test_fneg_fma_x_suby_negz_f32:
+; FMA4:       # %bb.0: # %entry
+; FMA4-NEXT:    vsubss %xmm0, %xmm2, %xmm0
+; FMA4-NEXT:    vfmaddss %xmm3, %xmm0, %xmm1, %xmm0
+; FMA4-NEXT:    retq
+entry:
+  %suby = fsub nsz float %w, %y
+  %negz = fsub float -0.000000e+00, %z
+  %0 = tail call nsz float @llvm.fma.f32(float %x, float %suby, float %negz)
+  %1 = fsub float -0.000000e+00, %0
+  ret float %1
+}
+
+define float @test_fneg_fma_subx_suby_negz_f32(float %w, float %x, float %y, float %z)  {
+; FMA3-LABEL: test_fneg_fma_subx_suby_negz_f32:
+; FMA3:       # %bb.0: # %entry
+; FMA3-NEXT:    vsubss %xmm1, %xmm0, %xmm1
+; FMA3-NEXT:    vsubss %xmm0, %xmm2, %xmm0
+; FMA3-NEXT:    vfmadd213ss {{.*#+}} xmm0 = (xmm1 * xmm0) + xmm3
+; FMA3-NEXT:    retq
+;
+; FMA4-LABEL: test_fneg_fma_subx_suby_negz_f32:
+; FMA4:       # %bb.0: # %entry
+; FMA4-NEXT:    vsubss %xmm1, %xmm0, %xmm1
+; FMA4-NEXT:    vsubss %xmm0, %xmm2, %xmm0
+; FMA4-NEXT:    vfmaddss %xmm3, %xmm0, %xmm1, %xmm0
+; FMA4-NEXT:    retq
+entry:
+  %subx = fsub nsz float %w, %x
+  %suby = fsub nsz float %w, %y
+  %negz = fsub float -0.000000e+00, %z
+  %0 = tail call nsz float @llvm.fma.f32(float %subx, float %suby, float %negz)
+  %1 = fsub float -0.000000e+00, %0
+  ret float %1
+}
+
+define float @test_fneg_fma_subx_negy_negz_f32(float %w, float %x, float %y, float %z)  {
+; FMA3-LABEL: test_fneg_fma_subx_negy_negz_f32:
+; FMA3:       # %bb.0: # %entry
+; FMA3-NEXT:    vsubss %xmm1, %xmm0, %xmm0
+; FMA3-NEXT:    vfmadd213ss {{.*#+}} xmm0 = (xmm2 * xmm0) + xmm3
+; FMA3-NEXT:    retq
+;
+; FMA4-LABEL: test_fneg_fma_subx_negy_negz_f32:
+; FMA4:       # %bb.0: # %entry
+; FMA4-NEXT:    vsubss %xmm1, %xmm0, %xmm0
+; FMA4-NEXT:    vfmaddss %xmm3, %xmm2, %xmm0, %xmm0
+; FMA4-NEXT:    retq
+entry:
+  %subx = fsub nsz float %w, %x
+  %negy = fsub float -0.000000e+00, %y
+  %negz = fsub float -0.000000e+00, %z
+  %0 = tail call nsz float @llvm.fma.f32(float %subx, float %negy, float %negz)
+  %1 = fsub float -0.000000e+00, %0
+  ret float %1
+}
+
+; This would crash while trying getNegatedExpression().
+
+define float @negated_constant(float %x) {
+; FMA3-LABEL: negated_constant:
+; FMA3:       # %bb.0:
+; FMA3-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm1
+; FMA3-NEXT:    vfmadd132ss {{.*#+}} xmm0 = (xmm0 * mem) + xmm1
+; FMA3-NEXT:    retq
+;
+; FMA4-LABEL: negated_constant:
+; FMA4:       # %bb.0:
+; FMA4-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm1
+; FMA4-NEXT:    vfmaddss %xmm1, {{.*}}(%rip), %xmm0, %xmm0
+; FMA4-NEXT:    retq
+  %m = fmul float %x, 42.0
+  %fma = call nsz float @llvm.fma.f32(float %x, float -42.0, float %m)
+  %nfma = fneg float %fma
+  ret float %nfma
+}
+
+declare float @llvm.fma.f32(float, float, float)
diff --git a/llvm/test/CodeGen/X86/fmaxnum.ll b/llvm/test/CodeGen/X86/fmaxnum.ll
index e308412f7..23de2667a 100644
--- a/llvm/test/CodeGen/X86/fmaxnum.ll
+++ b/llvm/test/CodeGen/X86/fmaxnum.ll
@@ -44,7 +44,7 @@ define float @test_fmaxf(float %x, float %y) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vmaxss %xmm0, %xmm1, %xmm2
 ; AVX512-NEXT:    vcmpunordss %xmm0, %xmm0, %k1
-; AVX512-NEXT:    vmovss %xmm1, %xmm0, %xmm2 {%k1}
+; AVX512-NEXT:    vmovss %xmm1, %xmm2, %xmm2 {%k1}
 ; AVX512-NEXT:    vmovaps %xmm2, %xmm0
 ; AVX512-NEXT:    retq
   %z = call float @fmaxf(float %x, float %y) readnone
@@ -85,7 +85,7 @@ define double @test_fmax(double %x, double %y) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vmaxsd %xmm0, %xmm1, %xmm2
 ; AVX512-NEXT:    vcmpunordsd %xmm0, %xmm0, %k1
-; AVX512-NEXT:    vmovsd %xmm1, %xmm0, %xmm2 {%k1}
+; AVX512-NEXT:    vmovsd %xmm1, %xmm2, %xmm2 {%k1}
 ; AVX512-NEXT:    vmovapd %xmm2, %xmm0
 ; AVX512-NEXT:    retq
   %z = call double @fmax(double %x, double %y) readnone
@@ -133,7 +133,7 @@ define float @test_intrinsic_fmaxf(float %x, float %y) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vmaxss %xmm0, %xmm1, %xmm2
 ; AVX512-NEXT:    vcmpunordss %xmm0, %xmm0, %k1
-; AVX512-NEXT:    vmovss %xmm1, %xmm0, %xmm2 {%k1}
+; AVX512-NEXT:    vmovss %xmm1, %xmm2, %xmm2 {%k1}
 ; AVX512-NEXT:    vmovaps %xmm2, %xmm0
 ; AVX512-NEXT:    retq
   %z = call float @llvm.maxnum.f32(float %x, float %y) readnone
@@ -164,7 +164,7 @@ define double @test_intrinsic_fmax(double %x, double %y) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vmaxsd %xmm0, %xmm1, %xmm2
 ; AVX512-NEXT:    vcmpunordsd %xmm0, %xmm0, %k1
-; AVX512-NEXT:    vmovsd %xmm1, %xmm0, %xmm2 {%k1}
+; AVX512-NEXT:    vmovsd %xmm1, %xmm2, %xmm2 {%k1}
 ; AVX512-NEXT:    vmovapd %xmm2, %xmm0
 ; AVX512-NEXT:    retq
   %z = call double @llvm.maxnum.f64(double %x, double %y) readnone
diff --git a/llvm/test/CodeGen/X86/fmf-reduction.ll b/llvm/test/CodeGen/X86/fmf-reduction.ll
new file mode 100644
index 000000000..1d669d2a9
--- /dev/null
+++ b/llvm/test/CodeGen/X86/fmf-reduction.ll
@@ -0,0 +1,26 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-- -mattr=fma | FileCheck %s
+
+; Propagation of IR FMF should not drop flags when adding the DAG reduction flag.
+; This should include an FMA instruction, not separate FMUL/FADD.
+
+define double @julia_dotf(<4 x double> %x, <4 x double> %y, <4 x double> %z, i1 %t3) {
+; CHECK-LABEL: julia_dotf:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vfmadd213pd {{.*#+}} ymm0 = (ymm1 * ymm0) + ymm2
+; CHECK-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; CHECK-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
+; CHECK-NEXT:    vaddsd %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %t1 = fmul contract <4 x double> %x, %y
+  %t2 = fadd fast <4 x double> %z, %t1
+  %rdx.shuf = shufflevector <4 x double> %t2, <4 x double> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+  %bin.rdx22 = fadd fast <4 x double> %t2, %rdx.shuf
+  %rdx.shuf23 = shufflevector <4 x double> %bin.rdx22, <4 x double> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+  %bin.rdx24 = fadd fast <4 x double> %bin.rdx22, %rdx.shuf23
+  %t4 = extractelement <4 x double> %bin.rdx24, i32 0
+  ret double %t4
+}
+
diff --git a/llvm/test/CodeGen/X86/fminnum.ll b/llvm/test/CodeGen/X86/fminnum.ll
index 33accf2e4..1667a5cd0 100644
--- a/llvm/test/CodeGen/X86/fminnum.ll
+++ b/llvm/test/CodeGen/X86/fminnum.ll
@@ -44,7 +44,7 @@ define float @test_fminf(float %x, float %y) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vminss %xmm0, %xmm1, %xmm2
 ; AVX512-NEXT:    vcmpunordss %xmm0, %xmm0, %k1
-; AVX512-NEXT:    vmovss %xmm1, %xmm0, %xmm2 {%k1}
+; AVX512-NEXT:    vmovss %xmm1, %xmm2, %xmm2 {%k1}
 ; AVX512-NEXT:    vmovaps %xmm2, %xmm0
 ; AVX512-NEXT:    retq
   %z = call float @fminf(float %x, float %y) readnone
@@ -85,7 +85,7 @@ define double @test_fmin(double %x, double %y) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vminsd %xmm0, %xmm1, %xmm2
 ; AVX512-NEXT:    vcmpunordsd %xmm0, %xmm0, %k1
-; AVX512-NEXT:    vmovsd %xmm1, %xmm0, %xmm2 {%k1}
+; AVX512-NEXT:    vmovsd %xmm1, %xmm2, %xmm2 {%k1}
 ; AVX512-NEXT:    vmovapd %xmm2, %xmm0
 ; AVX512-NEXT:    retq
   %z = call double @fmin(double %x, double %y) readnone
@@ -133,7 +133,7 @@ define float @test_intrinsic_fminf(float %x, float %y) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vminss %xmm0, %xmm1, %xmm2
 ; AVX512-NEXT:    vcmpunordss %xmm0, %xmm0, %k1
-; AVX512-NEXT:    vmovss %xmm1, %xmm0, %xmm2 {%k1}
+; AVX512-NEXT:    vmovss %xmm1, %xmm2, %xmm2 {%k1}
 ; AVX512-NEXT:    vmovaps %xmm2, %xmm0
 ; AVX512-NEXT:    retq
   %z = call float @llvm.minnum.f32(float %x, float %y) readnone
@@ -164,7 +164,7 @@ define double @test_intrinsic_fmin(double %x, double %y) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vminsd %xmm0, %xmm1, %xmm2
 ; AVX512-NEXT:    vcmpunordsd %xmm0, %xmm0, %k1
-; AVX512-NEXT:    vmovsd %xmm1, %xmm0, %xmm2 {%k1}
+; AVX512-NEXT:    vmovsd %xmm1, %xmm2, %xmm2 {%k1}
 ; AVX512-NEXT:    vmovapd %xmm2, %xmm0
 ; AVX512-NEXT:    retq
   %z = call double @llvm.minnum.f64(double %x, double %y) readnone
diff --git a/llvm/test/CodeGen/X86/fmul-combines.ll b/llvm/test/CodeGen/X86/fmul-combines.ll
index 3a2b7bc4a..0c8a822e7 100644
--- a/llvm/test/CodeGen/X86/fmul-combines.ll
+++ b/llvm/test/CodeGen/X86/fmul-combines.ll
@@ -76,7 +76,7 @@ define <4 x float> @constant_fold_fmul_v4f32_undef(<4 x float> %x) {
   ret <4 x float> %y
 }
 
-define <4 x float> @fmul0_v4f32_nsz_nnan(<4 x float> %x) #0 {
+define <4 x float> @fmul0_v4f32_nsz_nnan(<4 x float> %x) {
 ; CHECK-LABEL: fmul0_v4f32_nsz_nnan:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    xorps %xmm0, %xmm0
@@ -85,7 +85,7 @@ define <4 x float> @fmul0_v4f32_nsz_nnan(<4 x float> %x) #0 {
   ret <4 x float> %y
 }
 
-define <4 x float> @fmul0_v4f32_undef(<4 x float> %x) #0 {
+define <4 x float> @fmul0_v4f32_undef(<4 x float> %x) {
 ; CHECK-LABEL: fmul0_v4f32_undef:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    xorps %xmm0, %xmm0
@@ -94,23 +94,23 @@ define <4 x float> @fmul0_v4f32_undef(<4 x float> %x) #0 {
   ret <4 x float> %y
 }
 
-define <4 x float> @fmul_c2_c4_v4f32(<4 x float> %x) #0 {
+define <4 x float> @fmul_c2_c4_v4f32(<4 x float> %x) {
 ; CHECK-LABEL: fmul_c2_c4_v4f32:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    mulps {{.*}}(%rip), %xmm0
 ; CHECK-NEXT:    retq
-  %y = fmul <4 x float> %x, <float 2.0, float 2.0, float 2.0, float 2.0>
-  %z = fmul <4 x float> %y, <float 4.0, float 4.0, float 4.0, float 4.0>
+  %y = fmul fast <4 x float> %x, <float 2.0, float 2.0, float 2.0, float 2.0>
+  %z = fmul fast <4 x float> %y, <float 4.0, float 4.0, float 4.0, float 4.0>
   ret <4 x float> %z
 }
 
-define <4 x float> @fmul_c3_c4_v4f32(<4 x float> %x) #0 {
+define <4 x float> @fmul_c3_c4_v4f32(<4 x float> %x) {
 ; CHECK-LABEL: fmul_c3_c4_v4f32:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    mulps {{.*}}(%rip), %xmm0
 ; CHECK-NEXT:    retq
-  %y = fmul <4 x float> %x, <float 3.0, float 3.0, float 3.0, float 3.0>
-  %z = fmul <4 x float> %y, <float 4.0, float 4.0, float 4.0, float 4.0>
+  %y = fmul fast <4 x float> %x, <float 3.0, float 3.0, float 3.0, float 3.0>
+  %z = fmul fast <4 x float> %y, <float 4.0, float 4.0, float 4.0, float 4.0>
   ret <4 x float> %z
 }
 
@@ -120,24 +120,24 @@ define <4 x float> @fmul_c3_c4_v4f32(<4 x float> %x) #0 {
 ; CHECK: float 32
 
 ; We should be able to pre-multiply the two constant vectors.
-define <4 x float> @fmul_v4f32_two_consts_no_splat(<4 x float> %x) #0 {
+define <4 x float> @fmul_v4f32_two_consts_no_splat(<4 x float> %x) {
 ; CHECK-LABEL: fmul_v4f32_two_consts_no_splat:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    mulps {{.*}}(%rip), %xmm0
 ; CHECK-NEXT:    retq
-  %y = fmul <4 x float> %x, <float 1.0, float 2.0, float 3.0, float 4.0>
-  %z = fmul <4 x float> %y, <float 5.0, float 6.0, float 7.0, float 8.0>
+  %y = fmul fast <4 x float> %x, <float 1.0, float 2.0, float 3.0, float 4.0>
+  %z = fmul fast <4 x float> %y, <float 5.0, float 6.0, float 7.0, float 8.0>
   ret <4 x float> %z
 }
 
 ; Same as above, but reverse operands to make sure non-canonical form is also handled.
-define <4 x float> @fmul_v4f32_two_consts_no_splat_non_canonical(<4 x float> %x) #0 {
+define <4 x float> @fmul_v4f32_two_consts_no_splat_non_canonical(<4 x float> %x) {
 ; CHECK-LABEL: fmul_v4f32_two_consts_no_splat_non_canonical:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    mulps {{.*}}(%rip), %xmm0
 ; CHECK-NEXT:    retq
-  %y = fmul <4 x float> <float 1.0, float 2.0, float 3.0, float 4.0>, %x
-  %z = fmul <4 x float> <float 5.0, float 6.0, float 7.0, float 8.0>, %y
+  %y = fmul fast <4 x float> <float 1.0, float 2.0, float 3.0, float 4.0>, %x
+  %z = fmul fast <4 x float> <float 5.0, float 6.0, float 7.0, float 8.0>, %y
   ret <4 x float> %z
 }
 
@@ -172,14 +172,14 @@ define <4 x float> @fmul_v4f32_two_consts_no_splat_reassoc_2(<4 x float> %x) {
 
 ; More than one use of a constant multiply should not inhibit the optimization.
 ; Instead of a chain of 2 dependent mults, this test will have 2 independent mults.
-define <4 x float> @fmul_v4f32_two_consts_no_splat_multiple_use(<4 x float> %x) #0 {
+define <4 x float> @fmul_v4f32_two_consts_no_splat_multiple_use(<4 x float> %x) {
 ; CHECK-LABEL: fmul_v4f32_two_consts_no_splat_multiple_use:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    mulps {{.*}}(%rip), %xmm0
 ; CHECK-NEXT:    retq
-  %y = fmul <4 x float> %x, <float 1.0, float 2.0, float 3.0, float 4.0>
-  %z = fmul <4 x float> %y, <float 5.0, float 6.0, float 7.0, float 8.0>
-  %a = fadd <4 x float> %y, %z
+  %y = fmul fast <4 x float> %x, <float 1.0, float 2.0, float 3.0, float 4.0>
+  %z = fmul fast <4 x float> %y, <float 5.0, float 6.0, float 7.0, float 8.0>
+  %a = fadd fast <4 x float> %y, %z
   ret <4 x float> %a
 }
 
@@ -191,7 +191,7 @@ define <4 x float> @fmul_v4f32_two_consts_no_splat_multiple_use(<4 x float> %x)
 ; CHECK: float 24
 ; CHECK: float 24
 
-define <4 x float> @PR22698_splats(<4 x float> %a) #0 {
+define <4 x float> @PR22698_splats(<4 x float> %a) {
 ; CHECK-LABEL: PR22698_splats:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    mulps {{.*}}(%rip), %xmm0
@@ -209,7 +209,7 @@ define <4 x float> @PR22698_splats(<4 x float> %a) #0 {
 ; CHECK: float 231
 ; CHECK: float 384
 
-define <4 x float> @PR22698_no_splats(<4 x float> %a) #0 {
+define <4 x float> @PR22698_no_splats(<4 x float> %a) {
 ; CHECK-LABEL: PR22698_no_splats:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    mulps {{.*}}(%rip), %xmm0
@@ -220,23 +220,23 @@ define <4 x float> @PR22698_no_splats(<4 x float> %a) #0 {
   ret <4 x float> %mul3
 }
 
-define float @fmul_c2_c4_f32(float %x) #0 {
+define float @fmul_c2_c4_f32(float %x) {
 ; CHECK-LABEL: fmul_c2_c4_f32:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    mulss {{.*}}(%rip), %xmm0
 ; CHECK-NEXT:    retq
-  %y = fmul float %x, 2.0
-  %z = fmul float %y, 4.0
+  %y = fmul fast float %x, 2.0
+  %z = fmul fast float %y, 4.0
   ret float %z
 }
 
-define float @fmul_c3_c4_f32(float %x) #0 {
+define float @fmul_c3_c4_f32(float %x) {
 ; CHECK-LABEL: fmul_c3_c4_f32:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    mulss {{.*}}(%rip), %xmm0
 ; CHECK-NEXT:    retq
-  %y = fmul float %x, 3.0
-  %z = fmul float %y, 4.0
+  %y = fmul fast float %x, 3.0
+  %z = fmul fast float %y, 4.0
   ret float %z
 }
 
@@ -261,5 +261,3 @@ define <4 x float> @fmul_fneg_fneg_v4f32(<4 x float> %x, <4 x float> %y) {
   %mul = fmul <4 x float> %x.neg, %y.neg
   ret <4 x float> %mul
 }
-
-attributes #0 = { "less-precise-fpmad"="true" "no-infs-fp-math"="true" "no-nans-fp-math"="true" "unsafe-fp-math"="true" }
diff --git a/llvm/test/CodeGen/X86/fold-call-2.ll b/llvm/test/CodeGen/X86/fold-call-2.ll
index b5e260641..e49de7c6b 100644
--- a/llvm/test/CodeGen/X86/fold-call-2.ll
+++ b/llvm/test/CodeGen/X86/fold-call-2.ll
@@ -1,8 +1,17 @@
-; RUN: llc < %s -mtriple=i386-apple-darwin | grep mov | count 1
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i386-apple-darwin | FileCheck %s
 
 @f = external global void ()*		; <void ()**> [#uses=1]
 
 define i32 @main() nounwind {
+; CHECK-LABEL: main:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    subl $12, %esp
+; CHECK-NEXT:    movl L_f$non_lazy_ptr, %eax
+; CHECK-NEXT:    calll *(%eax)
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    addl $12, %esp
+; CHECK-NEXT:    retl
 entry:
 	load void ()*, void ()** @f, align 8		; <void ()*>:0 [#uses=1]
 	tail call void %0( ) nounwind
diff --git a/llvm/test/CodeGen/X86/fold-call-3.ll b/llvm/test/CodeGen/X86/fold-call-3.ll
index 840a32447..d0f311d39 100644
--- a/llvm/test/CodeGen/X86/fold-call-3.ll
+++ b/llvm/test/CodeGen/X86/fold-call-3.ll
@@ -1,7 +1,8 @@
-; RUN: llc < %s -mtriple=x86_64-apple-darwin | grep call | grep 560
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-apple-darwin | FileCheck %s
 ; rdar://6522427
 ; This command line used to crash due to dangling nodes left after PreprocessISelDAG
-; RUN: llc < %s -mtriple=x86_64-apple-darwin -pre-RA-sched=linearize
+; RUN: llc < %s -mtriple=x86_64-apple-darwin -pre-RA-sched=linearize | FileCheck %s --check-prefix=pre-RA
 
 	%"struct.clang::Action" = type { %"struct.clang::ActionBase" }
 	%"struct.clang::ActionBase" = type { i32 (...)** }
@@ -10,6 +11,63 @@
 @llvm.used = appending global [1 x i8*] [ i8* bitcast (void (i8*, %"struct.clang::Action"*)* @_Z25RawPointerPerformanceTestPvRN5clang6ActionE to i8*) ], section "llvm.metadata"		; <[1 x i8*]*> [#uses=0]
 
 define void @_Z25RawPointerPerformanceTestPvRN5clang6ActionE(i8* %Val, %"struct.clang::Action"* %Actions) nounwind {
+; CHECK-LABEL: _Z25RawPointerPerformanceTestPvRN5clang6ActionE:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    subq $24, %rsp
+; CHECK-NEXT:    cmpl $0, {{.*}}(%rip)
+; CHECK-NEXT:    je LBB0_3
+; CHECK-NEXT:  ## %bb.1: ## %bb.nph
+; CHECK-NEXT:    movq %rsi, %rbx
+; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    xorl %ebp, %ebp
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  LBB0_2: ## %bb
+; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movq (%rbx), %rcx
+; CHECK-NEXT:    movq %rbx, %rdi
+; CHECK-NEXT:    movq %rax, %rsi
+; CHECK-NEXT:    callq *560(%rcx)
+; CHECK-NEXT:    movq %rax, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movq %rdx, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    incl %ebp
+; CHECK-NEXT:    cmpl {{.*}}(%rip), %ebp
+; CHECK-NEXT:    jb LBB0_2
+; CHECK-NEXT:  LBB0_3: ## %return
+; CHECK-NEXT:    addq $24, %rsp
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    retq
+;
+; pre-RA-LABEL: _Z25RawPointerPerformanceTestPvRN5clang6ActionE:
+; pre-RA:       ## %bb.0: ## %entry
+; pre-RA-NEXT:    pushq %rbp
+; pre-RA-NEXT:    pushq %rbx
+; pre-RA-NEXT:    subq $24, %rsp
+; pre-RA-NEXT:    cmpl $0, {{.*}}(%rip)
+; pre-RA-NEXT:    je LBB0_3
+; pre-RA-NEXT:  ## %bb.1: ## %bb.nph
+; pre-RA-NEXT:    movq %rsi, %rbx
+; pre-RA-NEXT:    movq %rdi, %rax
+; pre-RA-NEXT:    xorl %ebp, %ebp
+; pre-RA-NEXT:    .p2align 4, 0x90
+; pre-RA-NEXT:  LBB0_2: ## %bb
+; pre-RA-NEXT:    ## =>This Inner Loop Header: Depth=1
+; pre-RA-NEXT:    movq (%rbx), %rcx
+; pre-RA-NEXT:    movq %rbx, %rdi
+; pre-RA-NEXT:    movq %rax, %rsi
+; pre-RA-NEXT:    callq *560(%rcx)
+; pre-RA-NEXT:    incl %ebp
+; pre-RA-NEXT:    movq %rax, {{[0-9]+}}(%rsp)
+; pre-RA-NEXT:    movq %rdx, {{[0-9]+}}(%rsp)
+; pre-RA-NEXT:    cmpl {{.*}}(%rip), %ebp
+; pre-RA-NEXT:    jb LBB0_2
+; pre-RA-NEXT:  LBB0_3: ## %return
+; pre-RA-NEXT:    addq $24, %rsp
+; pre-RA-NEXT:    popq %rbx
+; pre-RA-NEXT:    popq %rbp
+; pre-RA-NEXT:    retq
 entry:
 	%0 = alloca %"struct.clang::ActionBase::ActionResult<0u>", align 8		; <%"struct.clang::ActionBase::ActionResult<0u>"*> [#uses=3]
 	%1 = load i32, i32* @NumTrials, align 4		; <i32> [#uses=1]
diff --git a/llvm/test/CodeGen/X86/fold-call-oper.ll b/llvm/test/CodeGen/X86/fold-call-oper.ll
index 1e3e58ddc..a88ddbb8b 100644
--- a/llvm/test/CodeGen/X86/fold-call-oper.ll
+++ b/llvm/test/CodeGen/X86/fold-call-oper.ll
@@ -45,4 +45,4 @@ for.inc8.i:                                       ; preds = %if.then.i, %for.bod
   br label %for.body3.i
 }
 
-attributes #0 = { noreturn uwtable "no-frame-pointer-elim"="true" }
+attributes #0 = { noreturn uwtable "frame-pointer"="all" }
diff --git a/llvm/test/CodeGen/X86/fold-load-unops.ll b/llvm/test/CodeGen/X86/fold-load-unops.ll
index aa6bc720f..b42b94a63 100644
--- a/llvm/test/CodeGen/X86/fold-load-unops.ll
+++ b/llvm/test/CodeGen/X86/fold-load-unops.ll
@@ -113,6 +113,38 @@ define <4 x float> @rcpss_full_size(<4 x float>* %a) optsize {
     ret <4 x float> %res
 }
 
+define float @rcpss_pgso(float* %a) !prof !14 {
+; SSE-LABEL: rcpss_pgso:
+; SSE:       # %bb.0:
+; SSE-NEXT:    rcpss (%rdi), %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: rcpss_pgso:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vrcpss (%rdi), %xmm0, %xmm0
+; AVX-NEXT:    retq
+    %ld = load float, float* %a
+    %ins = insertelement <4 x float> undef, float %ld, i32 0
+    %res = tail call <4 x float> @llvm.x86.sse.rcp.ss(<4 x float> %ins)
+    %ext = extractelement <4 x float> %res, i32 0
+    ret float %ext
+}
+
+define <4 x float> @rcpss_full_pgso(<4 x float>* %a) !prof !14 {
+; SSE-LABEL: rcpss_full_pgso:
+; SSE:       # %bb.0:
+; SSE-NEXT:    rcpss (%rdi), %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: rcpss_full_pgso:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vrcpss (%rdi), %xmm0, %xmm0
+; AVX-NEXT:    retq
+    %ld = load <4 x float>, <4 x float>* %a
+    %res = tail call <4 x float> @llvm.x86.sse.rcp.ss(<4 x float> %ld)
+    ret <4 x float> %res
+}
+
 define float @rsqrtss_size(float* %a) optsize {
 ; SSE-LABEL: rsqrtss_size:
 ; SSE:       # %bb.0:
@@ -145,6 +177,38 @@ define <4 x float> @rsqrtss_full_size(<4 x float>* %a) optsize {
     ret <4 x float> %res
 }
 
+define float @rsqrtss_pgso(float* %a) !prof !14 {
+; SSE-LABEL: rsqrtss_pgso:
+; SSE:       # %bb.0:
+; SSE-NEXT:    rsqrtss (%rdi), %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: rsqrtss_pgso:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vrsqrtss (%rdi), %xmm0, %xmm0
+; AVX-NEXT:    retq
+    %ld = load float, float* %a
+    %ins = insertelement <4 x float> undef, float %ld, i32 0
+    %res = tail call <4 x float> @llvm.x86.sse.rsqrt.ss(<4 x float> %ins)
+    %ext = extractelement <4 x float> %res, i32 0
+    ret float %ext
+}
+
+define <4 x float> @rsqrtss_full_pgso(<4 x float>* %a) !prof !14 {
+; SSE-LABEL: rsqrtss_full_pgso:
+; SSE:       # %bb.0:
+; SSE-NEXT:    rsqrtss (%rdi), %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: rsqrtss_full_pgso:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vrsqrtss (%rdi), %xmm0, %xmm0
+; AVX-NEXT:    retq
+    %ld = load <4 x float>, <4 x float>* %a
+    %res = tail call <4 x float> @llvm.x86.sse.rsqrt.ss(<4 x float> %ld)
+    ret <4 x float> %res
+}
+
 define float @sqrtss_size(float* %a) optsize{
 ; SSE-LABEL: sqrtss_size:
 ; SSE:       # %bb.0:
@@ -196,6 +260,57 @@ define <4 x float> @sqrtss_full_size_volatile(<4 x float>* %a) optsize{
     ret <4 x float> %res
 }
 
+define float @sqrtss_pgso(float* %a) !prof !14 {
+; SSE-LABEL: sqrtss_pgso:
+; SSE:       # %bb.0:
+; SSE-NEXT:    sqrtss (%rdi), %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: sqrtss_pgso:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vsqrtss (%rdi), %xmm0, %xmm0
+; AVX-NEXT:    retq
+    %ld = load float, float* %a
+    %ins = insertelement <4 x float> undef, float %ld, i32 0
+    %res = tail call <4 x float> @llvm.x86.sse.sqrt.ss(<4 x float> %ins)
+    %ext = extractelement <4 x float> %res, i32 0
+    ret float %ext
+}
+
+define <4 x float> @sqrtss_full_pgso(<4 x float>* %a) !prof !14 {
+; SSE-LABEL: sqrtss_full_pgso:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movaps (%rdi), %xmm0
+; SSE-NEXT:    sqrtss %xmm0, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: sqrtss_full_pgso:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovaps (%rdi), %xmm0
+; AVX-NEXT:    vsqrtss %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    retq
+    %ld = load <4 x float>, <4 x float>* %a
+    %res = tail call <4 x float> @llvm.x86.sse.sqrt.ss(<4 x float> %ld)
+    ret <4 x float> %res
+}
+
+define <4 x float> @sqrtss_full_pgso_volatile(<4 x float>* %a) !prof !14 {
+; SSE-LABEL: sqrtss_full_pgso_volatile:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movaps (%rdi), %xmm0
+; SSE-NEXT:    sqrtss %xmm0, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: sqrtss_full_pgso_volatile:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovaps (%rdi), %xmm0
+; AVX-NEXT:    vsqrtss %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    retq
+    %ld = load volatile <4 x float>, <4 x float>* %a
+    %res = tail call <4 x float> @llvm.x86.sse.sqrt.ss(<4 x float> %ld)
+    ret <4 x float> %res
+}
+
 define double @sqrtsd_size(double* %a) optsize {
 ; SSE-LABEL: sqrtsd_size:
 ; SSE:       # %bb.0:
@@ -247,7 +362,75 @@ define <2 x double> @sqrtsd_full_size_volatile(<2 x double>* %a) optsize {
     ret <2 x double> %res
 }
 
+define double @sqrtsd_pgso(double* %a) !prof !14 {
+; SSE-LABEL: sqrtsd_pgso:
+; SSE:       # %bb.0:
+; SSE-NEXT:    sqrtsd (%rdi), %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: sqrtsd_pgso:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vsqrtsd (%rdi), %xmm0, %xmm0
+; AVX-NEXT:    retq
+    %ld = load double, double* %a
+    %ins = insertelement <2 x double> undef, double %ld, i32 0
+    %res = tail call <2 x double> @llvm.x86.sse2.sqrt.sd(<2 x double> %ins)
+    %ext = extractelement <2 x double> %res, i32 0
+    ret double %ext
+}
+
+define <2 x double> @sqrtsd_full_pgso(<2 x double>* %a) !prof !14 {
+; SSE-LABEL: sqrtsd_full_pgso:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movapd (%rdi), %xmm0
+; SSE-NEXT:    sqrtsd %xmm0, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: sqrtsd_full_pgso:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovapd (%rdi), %xmm0
+; AVX-NEXT:    vsqrtsd %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    retq
+    %ld = load <2 x double>, <2 x double>* %a
+    %res = tail call <2 x double> @llvm.x86.sse2.sqrt.sd(<2 x double> %ld)
+    ret <2 x double> %res
+}
+
+define <2 x double> @sqrtsd_full_pgso_volatile(<2 x double>* %a) !prof !14 {
+; SSE-LABEL: sqrtsd_full_pgso_volatile:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movapd (%rdi), %xmm0
+; SSE-NEXT:    sqrtsd %xmm0, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: sqrtsd_full_pgso_volatile:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovapd (%rdi), %xmm0
+; AVX-NEXT:    vsqrtsd %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    retq
+    %ld = load volatile <2 x double>, <2 x double>* %a
+    %res = tail call <2 x double> @llvm.x86.sse2.sqrt.sd(<2 x double> %ld)
+    ret <2 x double> %res
+}
+
 declare <4 x float> @llvm.x86.sse.rcp.ss(<4 x float>) nounwind readnone
 declare <4 x float> @llvm.x86.sse.rsqrt.ss(<4 x float>) nounwind readnone
 declare <4 x float> @llvm.x86.sse.sqrt.ss(<4 x float>) nounwind readnone
 declare <2 x double> @llvm.x86.sse2.sqrt.sd(<2 x double>) nounwind readnone
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 1, !"ProfileSummary", !1}
+!1 = !{!2, !3, !4, !5, !6, !7, !8, !9}
+!2 = !{!"ProfileFormat", !"InstrProf"}
+!3 = !{!"TotalCount", i64 10000}
+!4 = !{!"MaxCount", i64 10}
+!5 = !{!"MaxInternalCount", i64 1}
+!6 = !{!"MaxFunctionCount", i64 1000}
+!7 = !{!"NumCounts", i64 3}
+!8 = !{!"NumFunctions", i64 3}
+!9 = !{!"DetailedSummary", !10}
+!10 = !{!11, !12, !13}
+!11 = !{i32 10000, i64 100, i32 1}
+!12 = !{i32 999000, i64 100, i32 1}
+!13 = !{i32 999999, i64 1, i32 2}
+!14 = !{!"function_entry_count", i64 0}
diff --git a/llvm/test/CodeGen/X86/fold-load-vec.ll b/llvm/test/CodeGen/X86/fold-load-vec.ll
index 115f2bf7a..e8dc8f26f 100644
--- a/llvm/test/CodeGen/X86/fold-load-vec.ll
+++ b/llvm/test/CodeGen/X86/fold-load-vec.ll
@@ -12,7 +12,7 @@ define void @sample_test(<4 x float>* %source, <2 x float>* %dest) nounwind {
 ; CHECK-NEXT:    movq %rsi, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    xorps %xmm0, %xmm0
 ; CHECK-NEXT:    movlps %xmm0, (%rsp)
-; CHECK-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[2,3]
+; CHECK-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
 ; CHECK-NEXT:    movlps %xmm0, (%rsp)
 ; CHECK-NEXT:    movlps %xmm0, (%rsi)
 ; CHECK-NEXT:    movq {{[0-9]+}}(%rsp), %rax
diff --git a/llvm/test/CodeGen/X86/fold-sext-trunc.ll b/llvm/test/CodeGen/X86/fold-sext-trunc.ll
index 475fbea42..b6af452b3 100644
--- a/llvm/test/CodeGen/X86/fold-sext-trunc.ll
+++ b/llvm/test/CodeGen/X86/fold-sext-trunc.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -mtriple=x86_64-- | grep movslq | count 1
+; RUN: llc < %s -mtriple=x86_64-- | FileCheck %s
 ; RUN: llc < %s -O0 -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -stop-after livedebugvalues -o - | FileCheck %s -check-prefix=MIR
 ; PR4050
 
@@ -9,6 +9,7 @@
 
 declare void @func_28(i64, i64)
 
+; CHECK: movslq  g_10+4(%rip), %rdi
 define void @int322(i32 %foo) !dbg !5 {
 entry:
   %val = load i64, i64* getelementptr (%0, %0* bitcast (%struct.S1* @g_10 to %0*), i32 0, i32 0), !dbg !16
diff --git a/llvm/test/CodeGen/X86/fold-tied-op.ll b/llvm/test/CodeGen/X86/fold-tied-op.ll
index eb06eb75a..4c8d30c41 100644
--- a/llvm/test/CodeGen/X86/fold-tied-op.ll
+++ b/llvm/test/CodeGen/X86/fold-tied-op.ll
@@ -63,7 +63,7 @@ if.end:                                           ; preds = %if.else, %if.then
   ret i64 undef
 }
 
-attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.ident = !{!0}
 
diff --git a/llvm/test/CodeGen/X86/fold-vector-sext-zext.ll b/llvm/test/CodeGen/X86/fold-vector-sext-zext.ll
index 465c7cebf..982b29c08 100644
--- a/llvm/test/CodeGen/X86/fold-vector-sext-zext.ll
+++ b/llvm/test/CodeGen/X86/fold-vector-sext-zext.ll
@@ -11,12 +11,12 @@
 define <4 x i16> @test_sext_4i8_4i16() {
 ; X32-LABEL: test_sext_4i8_4i16:
 ; X32:       # %bb.0:
-; X32-NEXT:    vmovaps {{.*#+}} xmm0 = [0,4294967295,2,4294967293]
+; X32-NEXT:    vmovaps {{.*#+}} xmm0 = <0,65535,2,65533,u,u,u,u>
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: test_sext_4i8_4i16:
 ; X64:       # %bb.0:
-; X64-NEXT:    vmovaps {{.*#+}} xmm0 = [0,4294967295,2,4294967293]
+; X64-NEXT:    vmovaps {{.*#+}} xmm0 = <0,65535,2,65533,u,u,u,u>
 ; X64-NEXT:    retq
   %1 = insertelement <4 x i8> undef, i8 0, i32 0
   %2 = insertelement <4 x i8> %1, i8 -1, i32 1
@@ -29,12 +29,12 @@ define <4 x i16> @test_sext_4i8_4i16() {
 define <4 x i16> @test_sext_4i8_4i16_undef() {
 ; X32-LABEL: test_sext_4i8_4i16_undef:
 ; X32:       # %bb.0:
-; X32-NEXT:    vmovaps {{.*#+}} xmm0 = <u,4294967295,u,4294967293>
+; X32-NEXT:    vmovaps {{.*#+}} xmm0 = <u,65535,u,65533,u,u,u,u>
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: test_sext_4i8_4i16_undef:
 ; X64:       # %bb.0:
-; X64-NEXT:    vmovaps {{.*#+}} xmm0 = <u,4294967295,u,4294967293>
+; X64-NEXT:    vmovaps {{.*#+}} xmm0 = <u,65535,u,65533,u,u,u,u>
 ; X64-NEXT:    retq
   %1 = insertelement <4 x i8> undef, i8 undef, i32 0
   %2 = insertelement <4 x i8> %1, i8 -1, i32 1
@@ -207,12 +207,12 @@ define <8 x i32> @test_sext_8i8_8i32_undef() {
 define <4 x i16> @test_zext_4i8_4i16() {
 ; X32-LABEL: test_zext_4i8_4i16:
 ; X32:       # %bb.0:
-; X32-NEXT:    vmovaps {{.*#+}} xmm0 = [0,255,2,253]
+; X32-NEXT:    vmovaps {{.*#+}} xmm0 = <0,255,2,253,u,u,u,u>
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: test_zext_4i8_4i16:
 ; X64:       # %bb.0:
-; X64-NEXT:    vmovaps {{.*#+}} xmm0 = [0,255,2,253]
+; X64-NEXT:    vmovaps {{.*#+}} xmm0 = <0,255,2,253,u,u,u,u>
 ; X64-NEXT:    retq
   %1 = insertelement <4 x i8> undef, i8 0, i32 0
   %2 = insertelement <4 x i8> %1, i8 -1, i32 1
@@ -261,12 +261,12 @@ define <4 x i64> @test_zext_4i8_4i64() {
 define <4 x i16> @test_zext_4i8_4i16_undef() {
 ; X32-LABEL: test_zext_4i8_4i16_undef:
 ; X32:       # %bb.0:
-; X32-NEXT:    vmovaps {{.*#+}} xmm0 = [0,255,0,253]
+; X32-NEXT:    vmovaps {{.*#+}} xmm0 = <0,255,0,253,u,u,u,u>
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: test_zext_4i8_4i16_undef:
 ; X64:       # %bb.0:
-; X64-NEXT:    vmovaps {{.*#+}} xmm0 = [0,255,0,253]
+; X64-NEXT:    vmovaps {{.*#+}} xmm0 = <0,255,0,253,u,u,u,u>
 ; X64-NEXT:    retq
   %1 = insertelement <4 x i8> undef, i8 undef, i32 0
   %2 = insertelement <4 x i8> %1, i8 -1, i32 1
diff --git a/llvm/test/CodeGen/X86/fp-arith.ll b/llvm/test/CodeGen/X86/fp-arith.ll
index 73a132da3..8f1ce43a8 100644
--- a/llvm/test/CodeGen/X86/fp-arith.ll
+++ b/llvm/test/CodeGen/X86/fp-arith.ll
@@ -22,9 +22,8 @@ define x86_fp80 @fiadd_fp80_i16(x86_fp80 %a0, i16 %a1) {
 ; X64-LABEL: fiadd_fp80_i16:
 ; X64:       # %bb.0:
 ; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
-; X64-NEXT:    movswl %di, %eax
-; X64-NEXT:    movl %eax, -{{[0-9]+}}(%rsp)
-; X64-NEXT:    fiaddl -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movw %di, -{{[0-9]+}}(%rsp)
+; X64-NEXT:    fiadds -{{[0-9]+}}(%rsp)
 ; X64-NEXT:    retq
   %1 = sitofp i16 %a1 to x86_fp80
   %2 = fadd x86_fp80 %a0, %1
@@ -48,9 +47,9 @@ define x86_fp80 @fiadd_fp80_i16_ld(x86_fp80 %a0, i16 *%a1) {
 ; X64-LABEL: fiadd_fp80_i16_ld:
 ; X64:       # %bb.0:
 ; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
-; X64-NEXT:    movswl (%rdi), %eax
-; X64-NEXT:    movl %eax, -{{[0-9]+}}(%rsp)
-; X64-NEXT:    fiaddl -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movzwl (%rdi), %eax
+; X64-NEXT:    movw %ax, -{{[0-9]+}}(%rsp)
+; X64-NEXT:    fiadds -{{[0-9]+}}(%rsp)
 ; X64-NEXT:    retq
   %1 = load i16, i16 *%a1
   %2 = sitofp i16 %1 to x86_fp80
@@ -129,9 +128,8 @@ define x86_fp80 @fisub_fp80_i16(x86_fp80 %a0, i16 %a1) {
 ; X64-LABEL: fisub_fp80_i16:
 ; X64:       # %bb.0:
 ; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
-; X64-NEXT:    movswl %di, %eax
-; X64-NEXT:    movl %eax, -{{[0-9]+}}(%rsp)
-; X64-NEXT:    fisubl -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movw %di, -{{[0-9]+}}(%rsp)
+; X64-NEXT:    fisubs -{{[0-9]+}}(%rsp)
 ; X64-NEXT:    retq
   %1 = sitofp i16 %a1 to x86_fp80
   %2 = fsub x86_fp80 %a0, %1
@@ -155,9 +153,9 @@ define x86_fp80 @fisub_fp80_i16_ld(x86_fp80 %a0, i16 *%a1) {
 ; X64-LABEL: fisub_fp80_i16_ld:
 ; X64:       # %bb.0:
 ; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
-; X64-NEXT:    movswl (%rdi), %eax
-; X64-NEXT:    movl %eax, -{{[0-9]+}}(%rsp)
-; X64-NEXT:    fisubl -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movzwl (%rdi), %eax
+; X64-NEXT:    movw %ax, -{{[0-9]+}}(%rsp)
+; X64-NEXT:    fisubs -{{[0-9]+}}(%rsp)
 ; X64-NEXT:    retq
   %1 = load i16, i16 *%a1
   %2 = sitofp i16 %1 to x86_fp80
@@ -236,9 +234,8 @@ define x86_fp80 @fisubr_fp80_i16(x86_fp80 %a0, i16 %a1) {
 ; X64-LABEL: fisubr_fp80_i16:
 ; X64:       # %bb.0:
 ; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
-; X64-NEXT:    movswl %di, %eax
-; X64-NEXT:    movl %eax, -{{[0-9]+}}(%rsp)
-; X64-NEXT:    fisubrl -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movw %di, -{{[0-9]+}}(%rsp)
+; X64-NEXT:    fisubrs -{{[0-9]+}}(%rsp)
 ; X64-NEXT:    retq
   %1 = sitofp i16 %a1 to x86_fp80
   %2 = fsub x86_fp80 %1, %a0
@@ -262,9 +259,9 @@ define x86_fp80 @fisubr_fp80_i16_ld(x86_fp80 %a0, i16 *%a1) {
 ; X64-LABEL: fisubr_fp80_i16_ld:
 ; X64:       # %bb.0:
 ; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
-; X64-NEXT:    movswl (%rdi), %eax
-; X64-NEXT:    movl %eax, -{{[0-9]+}}(%rsp)
-; X64-NEXT:    fisubrl -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movzwl (%rdi), %eax
+; X64-NEXT:    movw %ax, -{{[0-9]+}}(%rsp)
+; X64-NEXT:    fisubrs -{{[0-9]+}}(%rsp)
 ; X64-NEXT:    retq
   %1 = load i16, i16 *%a1
   %2 = sitofp i16 %1 to x86_fp80
@@ -343,9 +340,8 @@ define x86_fp80 @fimul_fp80_i16(x86_fp80 %a0, i16 %a1) {
 ; X64-LABEL: fimul_fp80_i16:
 ; X64:       # %bb.0:
 ; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
-; X64-NEXT:    movswl %di, %eax
-; X64-NEXT:    movl %eax, -{{[0-9]+}}(%rsp)
-; X64-NEXT:    fimull -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movw %di, -{{[0-9]+}}(%rsp)
+; X64-NEXT:    fimuls -{{[0-9]+}}(%rsp)
 ; X64-NEXT:    retq
   %1 = sitofp i16 %a1 to x86_fp80
   %2 = fmul x86_fp80 %a0, %1
@@ -369,9 +365,9 @@ define x86_fp80 @fimul_fp80_i16_ld(x86_fp80 %a0, i16 *%a1) {
 ; X64-LABEL: fimul_fp80_i16_ld:
 ; X64:       # %bb.0:
 ; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
-; X64-NEXT:    movswl (%rdi), %eax
-; X64-NEXT:    movl %eax, -{{[0-9]+}}(%rsp)
-; X64-NEXT:    fimull -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movzwl (%rdi), %eax
+; X64-NEXT:    movw %ax, -{{[0-9]+}}(%rsp)
+; X64-NEXT:    fimuls -{{[0-9]+}}(%rsp)
 ; X64-NEXT:    retq
   %1 = load i16, i16 *%a1
   %2 = sitofp i16 %1 to x86_fp80
@@ -450,9 +446,8 @@ define x86_fp80 @fidiv_fp80_i16(x86_fp80 %a0, i16 %a1) {
 ; X64-LABEL: fidiv_fp80_i16:
 ; X64:       # %bb.0:
 ; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
-; X64-NEXT:    movswl %di, %eax
-; X64-NEXT:    movl %eax, -{{[0-9]+}}(%rsp)
-; X64-NEXT:    fidivl -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movw %di, -{{[0-9]+}}(%rsp)
+; X64-NEXT:    fidivs -{{[0-9]+}}(%rsp)
 ; X64-NEXT:    retq
   %1 = sitofp i16 %a1 to x86_fp80
   %2 = fdiv x86_fp80 %a0, %1
@@ -476,9 +471,9 @@ define x86_fp80 @fidiv_fp80_i16_ld(x86_fp80 %a0, i16 *%a1) {
 ; X64-LABEL: fidiv_fp80_i16_ld:
 ; X64:       # %bb.0:
 ; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
-; X64-NEXT:    movswl (%rdi), %eax
-; X64-NEXT:    movl %eax, -{{[0-9]+}}(%rsp)
-; X64-NEXT:    fidivl -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movzwl (%rdi), %eax
+; X64-NEXT:    movw %ax, -{{[0-9]+}}(%rsp)
+; X64-NEXT:    fidivs -{{[0-9]+}}(%rsp)
 ; X64-NEXT:    retq
   %1 = load i16, i16 *%a1
   %2 = sitofp i16 %1 to x86_fp80
@@ -557,9 +552,8 @@ define x86_fp80 @fidivr_fp80_i16(x86_fp80 %a0, i16 %a1) {
 ; X64-LABEL: fidivr_fp80_i16:
 ; X64:       # %bb.0:
 ; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
-; X64-NEXT:    movswl %di, %eax
-; X64-NEXT:    movl %eax, -{{[0-9]+}}(%rsp)
-; X64-NEXT:    fidivrl -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movw %di, -{{[0-9]+}}(%rsp)
+; X64-NEXT:    fidivrs -{{[0-9]+}}(%rsp)
 ; X64-NEXT:    retq
   %1 = sitofp i16 %a1 to x86_fp80
   %2 = fdiv x86_fp80 %1, %a0
@@ -583,9 +577,9 @@ define x86_fp80 @fidivr_fp80_i16_ld(x86_fp80 %a0, i16 *%a1) {
 ; X64-LABEL: fidivr_fp80_i16_ld:
 ; X64:       # %bb.0:
 ; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
-; X64-NEXT:    movswl (%rdi), %eax
-; X64-NEXT:    movl %eax, -{{[0-9]+}}(%rsp)
-; X64-NEXT:    fidivrl -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movzwl (%rdi), %eax
+; X64-NEXT:    movw %ax, -{{[0-9]+}}(%rsp)
+; X64-NEXT:    fidivrs -{{[0-9]+}}(%rsp)
 ; X64-NEXT:    retq
   %1 = load i16, i16 *%a1
   %2 = sitofp i16 %1 to x86_fp80
diff --git a/llvm/test/CodeGen/X86/fp-cvt.ll b/llvm/test/CodeGen/X86/fp-cvt.ll
index 8164651f9..667c2d414 100644
--- a/llvm/test/CodeGen/X86/fp-cvt.ll
+++ b/llvm/test/CodeGen/X86/fp-cvt.ll
@@ -444,20 +444,21 @@ define i64 @fptoui_i64_fp80(x86_fp80 %a0) nounwind {
 ; X86-NEXT:    subl $16, %esp
 ; X86-NEXT:    fldt 8(%ebp)
 ; X86-NEXT:    flds {{\.LCPI.*}}
-; X86-NEXT:    fld %st(1)
-; X86-NEXT:    fsub %st(1), %st
-; X86-NEXT:    fxch %st(1)
-; X86-NEXT:    fucomp %st(2)
+; X86-NEXT:    fucom %st(1)
 ; X86-NEXT:    fnstsw %ax
+; X86-NEXT:    xorl %edx, %edx
 ; X86-NEXT:    # kill: def $ah killed $ah killed $ax
 ; X86-NEXT:    sahf
+; X86-NEXT:    setbe %al
+; X86-NEXT:    fldz
 ; X86-NEXT:    ja .LBB10_2
 ; X86-NEXT:  # %bb.1:
-; X86-NEXT:    fstp %st(1)
+; X86-NEXT:    fstp %st(0)
 ; X86-NEXT:    fldz
+; X86-NEXT:    fxch %st(1)
 ; X86-NEXT:  .LBB10_2:
-; X86-NEXT:    fstp %st(0)
-; X86-NEXT:    setbe %al
+; X86-NEXT:    fstp %st(1)
+; X86-NEXT:    fsubrp %st, %st(1)
 ; X86-NEXT:    fnstcw {{[0-9]+}}(%esp)
 ; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    orl $3072, %ecx # imm = 0xC00
@@ -465,7 +466,7 @@ define i64 @fptoui_i64_fp80(x86_fp80 %a0) nounwind {
 ; X86-NEXT:    fldcw {{[0-9]+}}(%esp)
 ; X86-NEXT:    fistpll {{[0-9]+}}(%esp)
 ; X86-NEXT:    fldcw {{[0-9]+}}(%esp)
-; X86-NEXT:    movzbl %al, %edx
+; X86-NEXT:    movb %al, %dl
 ; X86-NEXT:    shll $31, %edx
 ; X86-NEXT:    xorl {{[0-9]+}}(%esp), %edx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
@@ -477,14 +478,14 @@ define i64 @fptoui_i64_fp80(x86_fp80 %a0) nounwind {
 ; X64-X87:       # %bb.0:
 ; X64-X87-NEXT:    fldt {{[0-9]+}}(%rsp)
 ; X64-X87-NEXT:    flds {{.*}}(%rip)
-; X64-X87-NEXT:    fld %st(1)
-; X64-X87-NEXT:    fsub %st(1), %st
 ; X64-X87-NEXT:    xorl %eax, %eax
+; X64-X87-NEXT:    fucomi %st(1), %st
+; X64-X87-NEXT:    setbe %al
+; X64-X87-NEXT:    fldz
 ; X64-X87-NEXT:    fxch %st(1)
-; X64-X87-NEXT:    fucompi %st(2), %st
 ; X64-X87-NEXT:    fcmovnbe %st(1), %st
 ; X64-X87-NEXT:    fstp %st(1)
-; X64-X87-NEXT:    setbe %al
+; X64-X87-NEXT:    fsubrp %st, %st(1)
 ; X64-X87-NEXT:    fnstcw -{{[0-9]+}}(%rsp)
 ; X64-X87-NEXT:    movzwl -{{[0-9]+}}(%rsp), %ecx
 ; X64-X87-NEXT:    orl $3072, %ecx # imm = 0xC00
@@ -500,13 +501,13 @@ define i64 @fptoui_i64_fp80(x86_fp80 %a0) nounwind {
 ; X64-SSSE3:       # %bb.0:
 ; X64-SSSE3-NEXT:    fldt {{[0-9]+}}(%rsp)
 ; X64-SSSE3-NEXT:    flds {{.*}}(%rip)
-; X64-SSSE3-NEXT:    fld %st(1)
-; X64-SSSE3-NEXT:    fsub %st(1), %st
 ; X64-SSSE3-NEXT:    xorl %eax, %eax
+; X64-SSSE3-NEXT:    fucomi %st(1), %st
+; X64-SSSE3-NEXT:    fldz
 ; X64-SSSE3-NEXT:    fxch %st(1)
-; X64-SSSE3-NEXT:    fucompi %st(2), %st
 ; X64-SSSE3-NEXT:    fcmovnbe %st(1), %st
 ; X64-SSSE3-NEXT:    fstp %st(1)
+; X64-SSSE3-NEXT:    fsubrp %st, %st(1)
 ; X64-SSSE3-NEXT:    fisttpll -{{[0-9]+}}(%rsp)
 ; X64-SSSE3-NEXT:    setbe %al
 ; X64-SSSE3-NEXT:    shlq $63, %rax
@@ -526,20 +527,21 @@ define i64 @fptoui_i64_fp80_ld(x86_fp80 *%a0) nounwind {
 ; X86-NEXT:    movl 8(%ebp), %eax
 ; X86-NEXT:    fldt (%eax)
 ; X86-NEXT:    flds {{\.LCPI.*}}
-; X86-NEXT:    fld %st(1)
-; X86-NEXT:    fsub %st(1), %st
-; X86-NEXT:    fxch %st(1)
-; X86-NEXT:    fucomp %st(2)
+; X86-NEXT:    fucom %st(1)
 ; X86-NEXT:    fnstsw %ax
+; X86-NEXT:    xorl %edx, %edx
 ; X86-NEXT:    # kill: def $ah killed $ah killed $ax
 ; X86-NEXT:    sahf
+; X86-NEXT:    setbe %al
+; X86-NEXT:    fldz
 ; X86-NEXT:    ja .LBB11_2
 ; X86-NEXT:  # %bb.1:
-; X86-NEXT:    fstp %st(1)
+; X86-NEXT:    fstp %st(0)
 ; X86-NEXT:    fldz
+; X86-NEXT:    fxch %st(1)
 ; X86-NEXT:  .LBB11_2:
-; X86-NEXT:    fstp %st(0)
-; X86-NEXT:    setbe %al
+; X86-NEXT:    fstp %st(1)
+; X86-NEXT:    fsubrp %st, %st(1)
 ; X86-NEXT:    fnstcw {{[0-9]+}}(%esp)
 ; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    orl $3072, %ecx # imm = 0xC00
@@ -547,7 +549,7 @@ define i64 @fptoui_i64_fp80_ld(x86_fp80 *%a0) nounwind {
 ; X86-NEXT:    fldcw {{[0-9]+}}(%esp)
 ; X86-NEXT:    fistpll {{[0-9]+}}(%esp)
 ; X86-NEXT:    fldcw {{[0-9]+}}(%esp)
-; X86-NEXT:    movzbl %al, %edx
+; X86-NEXT:    movb %al, %dl
 ; X86-NEXT:    shll $31, %edx
 ; X86-NEXT:    xorl {{[0-9]+}}(%esp), %edx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
@@ -559,14 +561,14 @@ define i64 @fptoui_i64_fp80_ld(x86_fp80 *%a0) nounwind {
 ; X64-X87:       # %bb.0:
 ; X64-X87-NEXT:    fldt (%rdi)
 ; X64-X87-NEXT:    flds {{.*}}(%rip)
-; X64-X87-NEXT:    fld %st(1)
-; X64-X87-NEXT:    fsub %st(1), %st
 ; X64-X87-NEXT:    xorl %eax, %eax
+; X64-X87-NEXT:    fucomi %st(1), %st
+; X64-X87-NEXT:    setbe %al
+; X64-X87-NEXT:    fldz
 ; X64-X87-NEXT:    fxch %st(1)
-; X64-X87-NEXT:    fucompi %st(2), %st
 ; X64-X87-NEXT:    fcmovnbe %st(1), %st
 ; X64-X87-NEXT:    fstp %st(1)
-; X64-X87-NEXT:    setbe %al
+; X64-X87-NEXT:    fsubrp %st, %st(1)
 ; X64-X87-NEXT:    fnstcw -{{[0-9]+}}(%rsp)
 ; X64-X87-NEXT:    movzwl -{{[0-9]+}}(%rsp), %ecx
 ; X64-X87-NEXT:    orl $3072, %ecx # imm = 0xC00
@@ -582,13 +584,13 @@ define i64 @fptoui_i64_fp80_ld(x86_fp80 *%a0) nounwind {
 ; X64-SSSE3:       # %bb.0:
 ; X64-SSSE3-NEXT:    fldt (%rdi)
 ; X64-SSSE3-NEXT:    flds {{.*}}(%rip)
-; X64-SSSE3-NEXT:    fld %st(1)
-; X64-SSSE3-NEXT:    fsub %st(1), %st
 ; X64-SSSE3-NEXT:    xorl %eax, %eax
+; X64-SSSE3-NEXT:    fucomi %st(1), %st
+; X64-SSSE3-NEXT:    fldz
 ; X64-SSSE3-NEXT:    fxch %st(1)
-; X64-SSSE3-NEXT:    fucompi %st(2), %st
 ; X64-SSSE3-NEXT:    fcmovnbe %st(1), %st
 ; X64-SSSE3-NEXT:    fstp %st(1)
+; X64-SSSE3-NEXT:    fsubrp %st, %st(1)
 ; X64-SSSE3-NEXT:    fisttpll -{{[0-9]+}}(%rsp)
 ; X64-SSSE3-NEXT:    setbe %al
 ; X64-SSSE3-NEXT:    shlq $63, %rax
@@ -615,9 +617,8 @@ define x86_fp80 @sitofp_fp80_i16(i16 %a0) nounwind {
 ;
 ; X64-LABEL: sitofp_fp80_i16:
 ; X64:       # %bb.0:
-; X64-NEXT:    movswl %di, %eax
-; X64-NEXT:    movl %eax, -{{[0-9]+}}(%rsp)
-; X64-NEXT:    fildl -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movw %di, -{{[0-9]+}}(%rsp)
+; X64-NEXT:    filds -{{[0-9]+}}(%rsp)
 ; X64-NEXT:    retq
   %1 = sitofp i16 %a0 to x86_fp80
   ret x86_fp80 %1
@@ -636,9 +637,9 @@ define x86_fp80 @sitofp_fp80_i16_ld(i16 *%a0) nounwind {
 ;
 ; X64-LABEL: sitofp_fp80_i16_ld:
 ; X64:       # %bb.0:
-; X64-NEXT:    movswl (%rdi), %eax
-; X64-NEXT:    movl %eax, -{{[0-9]+}}(%rsp)
-; X64-NEXT:    fildl -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movzwl (%rdi), %eax
+; X64-NEXT:    movw %ax, -{{[0-9]+}}(%rsp)
+; X64-NEXT:    filds -{{[0-9]+}}(%rsp)
 ; X64-NEXT:    retq
   %1 = load i16, i16 *%a0
   %2 = sitofp i16 %1 to x86_fp80
@@ -828,11 +829,9 @@ define x86_fp80 @uitofp_fp80_i64(i64 %a0) nounwind {
 ; X86-NEXT:    movl 12(%ebp), %ecx
 ; X86-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
 ; X86-NEXT:    movl %eax, (%esp)
-; X86-NEXT:    xorl %eax, %eax
-; X86-NEXT:    testl %ecx, %ecx
-; X86-NEXT:    setns %al
+; X86-NEXT:    shrl $31, %ecx
 ; X86-NEXT:    fildll (%esp)
-; X86-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; X86-NEXT:    fadds {{\.LCPI.*}}(,%ecx,4)
 ; X86-NEXT:    movl %ebp, %esp
 ; X86-NEXT:    popl %ebp
 ; X86-NEXT:    retl
@@ -842,7 +841,7 @@ define x86_fp80 @uitofp_fp80_i64(i64 %a0) nounwind {
 ; X64-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
 ; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    testq %rdi, %rdi
-; X64-NEXT:    setns %al
+; X64-NEXT:    sets %al
 ; X64-NEXT:    fildll -{{[0-9]+}}(%rsp)
 ; X64-NEXT:    fadds {{\.LCPI.*}}(,%rax,4)
 ; X64-NEXT:    retq
@@ -862,11 +861,9 @@ define x86_fp80 @uitofp_fp80_i64_ld(i64 *%a0) nounwind {
 ; X86-NEXT:    movl 4(%eax), %eax
 ; X86-NEXT:    movl %eax, {{[0-9]+}}(%esp)
 ; X86-NEXT:    movl %ecx, (%esp)
-; X86-NEXT:    xorl %ecx, %ecx
-; X86-NEXT:    testl %eax, %eax
-; X86-NEXT:    setns %cl
+; X86-NEXT:    shrl $31, %eax
 ; X86-NEXT:    fildll (%esp)
-; X86-NEXT:    fadds {{\.LCPI.*}}(,%ecx,4)
+; X86-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
 ; X86-NEXT:    movl %ebp, %esp
 ; X86-NEXT:    popl %ebp
 ; X86-NEXT:    retl
@@ -877,7 +874,7 @@ define x86_fp80 @uitofp_fp80_i64_ld(i64 *%a0) nounwind {
 ; X64-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
 ; X64-NEXT:    xorl %ecx, %ecx
 ; X64-NEXT:    testq %rax, %rax
-; X64-NEXT:    setns %cl
+; X64-NEXT:    sets %cl
 ; X64-NEXT:    fildll -{{[0-9]+}}(%rsp)
 ; X64-NEXT:    fadds {{\.LCPI.*}}(,%rcx,4)
 ; X64-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/fp-elim-and-no-fp-elim.ll b/llvm/test/CodeGen/X86/fp-elim-and-no-fp-elim.ll
index c3b2dfb5d..0c9597b97 100644
--- a/llvm/test/CodeGen/X86/fp-elim-and-no-fp-elim.ll
+++ b/llvm/test/CodeGen/X86/fp-elim-and-no-fp-elim.ll
@@ -28,5 +28,5 @@ entry:
 
 declare void @foo(float, float)
 
-attributes #0 = { "no-frame-pointer-elim"="true" }
-attributes #1 = { "no-frame-pointer-elim"="false" }
+attributes #0 = { "frame-pointer"="all" }
+attributes #1 = { "frame-pointer"="none" }
diff --git a/llvm/test/CodeGen/X86/fp-elim.ll b/llvm/test/CodeGen/X86/fp-elim.ll
index 65764d144..b644631d1 100644
--- a/llvm/test/CodeGen/X86/fp-elim.ll
+++ b/llvm/test/CodeGen/X86/fp-elim.ll
@@ -31,7 +31,7 @@ entry:
   ret void
 }
 
-define i32 @t3() "no-frame-pointer-elim-non-leaf" nounwind readnone {
+define i32 @t3() "frame-pointer"="non-leaf" nounwind readnone {
 entry:
 ; FP-ELIM-LABEL:  t3:
 ; FP-ELIM-NEXT:     movl
@@ -44,7 +44,7 @@ entry:
   ret i32 10
 }
 
-define void @t4() "no-frame-pointer-elim-non-leaf" nounwind {
+define void @t4() "frame-pointer"="non-leaf" nounwind {
 entry:
 ; FP-ELIM-LABEL:  t4:
 ; FP-ELIM-NEXT:     pushl %ebp
diff --git a/llvm/test/CodeGen/X86/fp-fast.ll b/llvm/test/CodeGen/X86/fp-fast.ll
index 81dd983d9..fcf479cfe 100644
--- a/llvm/test/CodeGen/X86/fp-fast.ll
+++ b/llvm/test/CodeGen/X86/fp-fast.ll
@@ -1,106 +1,106 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=x86_64-unknown-unknown -mattr=avx -enable-unsafe-fp-math --enable-no-nans-fp-math < %s | FileCheck %s
+; RUN: llc -mtriple=x86_64-unknown-unknown -mattr=avx < %s | FileCheck %s
 
-define float @test1(float %a) {
+define float @test1(float %a) #0 {
 ; CHECK-LABEL: test1:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm0
 ; CHECK-NEXT:    retq
-  %t1 = fadd float %a, %a
-  %r = fadd float %t1, %t1
+  %t1 = fadd nnan reassoc nsz float %a, %a
+  %r = fadd nnan reassoc nsz float %t1, %t1
   ret float %r
 }
 
-define float @test2(float %a) {
+define float @test2(float %a) #0 {
 ; CHECK-LABEL: test2:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm0
 ; CHECK-NEXT:    retq
-  %t1 = fmul float 4.0, %a
-  %t2 = fadd float %a, %a
-  %r = fadd float %t1, %t2
+  %t1 = fmul nnan reassoc nsz float 4.0, %a
+  %t2 = fadd nnan reassoc nsz float %a, %a
+  %r = fadd nnan reassoc nsz float %t1, %t2
   ret float %r
 }
 
-define float @test3(float %a) {
+define float @test3(float %a) #0 {
 ; CHECK-LABEL: test3:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm0
 ; CHECK-NEXT:    retq
-  %t1 = fmul float %a, 4.0
-  %t2 = fadd float %a, %a
-  %r = fadd float %t1, %t2
+  %t1 = fmul nnan reassoc nsz float %a, 4.0
+  %t2 = fadd nnan reassoc nsz float %a, %a
+  %r = fadd nnan reassoc nsz float %t1, %t2
   ret float %r
 }
 
-define float @test4(float %a) {
+define float @test4(float %a) #0 {
 ; CHECK-LABEL: test4:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm0
 ; CHECK-NEXT:    retq
-  %t1 = fadd float %a, %a
-  %t2 = fmul float 4.0, %a
-  %r = fadd float %t1, %t2
+  %t1 = fadd nnan reassoc nsz float %a, %a
+  %t2 = fmul nnan reassoc nsz float 4.0, %a
+  %r = fadd nnan reassoc nsz float %t1, %t2
   ret float %r
 }
 
-define float @test5(float %a) {
+define float @test5(float %a) #0 {
 ; CHECK-LABEL: test5:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm0
 ; CHECK-NEXT:    retq
-  %t1 = fadd float %a, %a
-  %t2 = fmul float %a, 4.0
-  %r = fadd float %t1, %t2
+  %t1 = fadd nnan reassoc nsz float %a, %a
+  %t2 = fmul nnan reassoc nsz float %a, 4.0
+  %r = fadd nnan reassoc nsz float %t1, %t2
   ret float %r
 }
 
-define float @test6(float %a) {
+define float @test6(float %a) #0 {
 ; CHECK-LABEL: test6:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vxorps %xmm0, %xmm0, %xmm0
 ; CHECK-NEXT:    retq
-  %t1 = fmul float 2.0, %a
-  %t2 = fadd float %a, %a
-  %r = fsub float %t1, %t2
+  %t1 = fmul nnan reassoc nsz float 2.0, %a
+  %t2 = fadd nnan reassoc nsz float %a, %a
+  %r = fsub nnan reassoc nsz float %t1, %t2
   ret float %r
 }
 
-define float @test7(float %a) {
+define float @test7(float %a) #0 {
 ; CHECK-LABEL: test7:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vxorps %xmm0, %xmm0, %xmm0
 ; CHECK-NEXT:    retq
-  %t1 = fmul float %a, 2.0
-  %t2 = fadd float %a, %a
-  %r = fsub float %t1, %t2
+  %t1 = fmul nnan reassoc nsz float %a, 2.0
+  %t2 = fadd nnan reassoc nsz float %a, %a
+  %r = fsub nnan reassoc nsz float %t1, %t2
   ret float %r
 }
 
-define float @test8(float %a) {
+define float @test8(float %a) #0 {
 ; CHECK-LABEL: test8:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    retq
-  %t1 = fmul float %a, 0.0
-  %t2 = fadd float %a, %t1
+  %t1 = fmul nsz float %a, 0.0
+  %t2 = fadd nnan reassoc nsz float %a, %t1
   ret float %t2
 }
 
-define float @test9(float %a) {
+define float @test9(float %a) #0 {
 ; CHECK-LABEL: test9:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    retq
-  %t1 = fmul float 0.0, %a
-  %t2 = fadd float %t1, %a
+  %t1 = fmul nsz float 0.0, %a
+  %t2 = fadd nnan reassoc nsz float %t1, %a
   ret float %t2
 }
 
-define float @test10(float %a) {
+define float @test10(float %a) #0 {
 ; CHECK-LABEL: test10:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vxorps %xmm0, %xmm0, %xmm0
 ; CHECK-NEXT:    retq
-  %t1 = fsub float -0.0, %a
-  %t2 = fadd float %a, %t1
+  %t1 = fsub nsz float -0.0, %a
+  %t2 = fadd nnan reassoc nsz float %a, %t1
   ret float %t2
 }
+
diff --git a/llvm/test/CodeGen/X86/fp-fold.ll b/llvm/test/CodeGen/X86/fp-fold.ll
index e796e3561..74b5232a4 100644
--- a/llvm/test/CodeGen/X86/fp-fold.ll
+++ b/llvm/test/CodeGen/X86/fp-fold.ll
@@ -1,98 +1,92 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown                        | FileCheck %s --check-prefixes=ANY,STRICT
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -enable-unsafe-fp-math | FileCheck %s --check-prefixes=ANY,UNSAFE
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s
 
-define float @fadd_zero(float %x) {
-; STRICT-LABEL: fadd_zero:
-; STRICT:       # %bb.0:
-; STRICT-NEXT:    xorps %xmm1, %xmm1
-; STRICT-NEXT:    addss %xmm1, %xmm0
-; STRICT-NEXT:    retq
-;
-; UNSAFE-LABEL: fadd_zero:
-; UNSAFE:       # %bb.0:
-; UNSAFE-NEXT:    retq
+define float @fadd_zero_strict(float %x) {
+; CHECK-LABEL: fadd_zero_strict:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xorps %xmm1, %xmm1
+; CHECK-NEXT:    addss %xmm1, %xmm0
+; CHECK-NEXT:    retq
   %r = fadd float %x, 0.0
   ret float %r
 }
 
 define float @fadd_negzero(float %x) {
-; ANY-LABEL: fadd_negzero:
-; ANY:       # %bb.0:
-; ANY-NEXT:    retq
+; CHECK-LABEL: fadd_negzero:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    retq
   %r = fadd float %x, -0.0
   ret float %r
 }
 
 define float @fadd_produce_zero(float %x) {
-; ANY-LABEL: fadd_produce_zero:
-; ANY:       # %bb.0:
-; ANY-NEXT:    xorps %xmm0, %xmm0
-; ANY-NEXT:    retq
+; CHECK-LABEL: fadd_produce_zero:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xorps %xmm0, %xmm0
+; CHECK-NEXT:    retq
   %neg = fsub nsz float 0.0, %x
   %r = fadd nnan float %neg, %x
   ret float %r
 }
 
 define float @fadd_reassociate(float %x) {
-; ANY-LABEL: fadd_reassociate:
-; ANY:       # %bb.0:
-; ANY-NEXT:    addss {{.*}}(%rip), %xmm0
-; ANY-NEXT:    retq
+; CHECK-LABEL: fadd_reassociate:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addss {{.*}}(%rip), %xmm0
+; CHECK-NEXT:    retq
   %sum = fadd float %x, 8.0
   %r = fadd reassoc nsz float %sum, 12.0
   ret float %r
 }
 
 define float @fadd_negzero_nsz(float %x) {
-; ANY-LABEL: fadd_negzero_nsz:
-; ANY:       # %bb.0:
-; ANY-NEXT:    retq
+; CHECK-LABEL: fadd_negzero_nsz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    retq
   %r = fadd nsz float %x, -0.0
   ret float %r
 }
 
 define float @fadd_zero_nsz(float %x) {
-; ANY-LABEL: fadd_zero_nsz:
-; ANY:       # %bb.0:
-; ANY-NEXT:    retq
+; CHECK-LABEL: fadd_zero_nsz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    retq
   %r = fadd nsz float %x, 0.0
   ret float %r
 }
 
 define float @fsub_zero(float %x) {
-; ANY-LABEL: fsub_zero:
-; ANY:       # %bb.0:
-; ANY-NEXT:    retq
+; CHECK-LABEL: fsub_zero:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    retq
   %r = fsub float %x, 0.0
   ret float %r
 }
 
 define float @fsub_self(float %x) {
-; ANY-LABEL: fsub_self:
-; ANY:       # %bb.0:
-; ANY-NEXT:    xorps %xmm0, %xmm0
-; ANY-NEXT:    retq
+; CHECK-LABEL: fsub_self:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xorps %xmm0, %xmm0
+; CHECK-NEXT:    retq
   %r = fsub nnan float %x, %x
   ret float %r
 }
 
 define float @fsub_neg_x_y(float %x, float %y) {
-; ANY-LABEL: fsub_neg_x_y:
-; ANY:       # %bb.0:
-; ANY-NEXT:    subss %xmm0, %xmm1
-; ANY-NEXT:    movaps %xmm1, %xmm0
-; ANY-NEXT:    retq
+; CHECK-LABEL: fsub_neg_x_y:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subss %xmm0, %xmm1
+; CHECK-NEXT:    movaps %xmm1, %xmm0
+; CHECK-NEXT:    retq
   %neg = fsub nsz float 0.0, %x
   %r = fadd nsz float %neg, %y
   ret float %r
 }
 
 define float @fsub_neg_y(float %x, float %y) {
-; ANY-LABEL: fsub_neg_y:
-; ANY:       # %bb.0:
-; ANY-NEXT:    mulss {{.*}}(%rip), %xmm0
-; ANY-NEXT:    retq
+; CHECK-LABEL: fsub_neg_y:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mulss {{.*}}(%rip), %xmm0
+; CHECK-NEXT:    retq
   %mul = fmul float %x, 5.0
   %add = fadd float %mul, %y
   %r = fsub nsz reassoc float %y, %add
@@ -100,10 +94,10 @@ define float @fsub_neg_y(float %x, float %y) {
 }
 
 define <4 x float> @fsub_neg_y_vector(<4 x float> %x, <4 x float> %y) {
-; ANY-LABEL: fsub_neg_y_vector:
-; ANY:       # %bb.0:
-; ANY-NEXT:    mulps {{.*}}(%rip), %xmm0
-; ANY-NEXT:    retq
+; CHECK-LABEL: fsub_neg_y_vector:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mulps {{.*}}(%rip), %xmm0
+; CHECK-NEXT:    retq
   %mul = fmul <4 x float> %x, <float 5.0, float 5.0, float 5.0, float 5.0>
   %add = fadd <4 x float> %mul, %y
   %r = fsub nsz reassoc <4 x float> %y, %add
@@ -111,10 +105,10 @@ define <4 x float> @fsub_neg_y_vector(<4 x float> %x, <4 x float> %y) {
 }
 
 define <4 x float> @fsub_neg_y_vector_nonuniform(<4 x float> %x, <4 x float> %y) {
-; ANY-LABEL: fsub_neg_y_vector_nonuniform:
-; ANY:       # %bb.0:
-; ANY-NEXT:    mulps {{.*}}(%rip), %xmm0
-; ANY-NEXT:    retq
+; CHECK-LABEL: fsub_neg_y_vector_nonuniform:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mulps {{.*}}(%rip), %xmm0
+; CHECK-NEXT:    retq
   %mul = fmul <4 x float> %x, <float 5.0, float 6.0, float 7.0, float 8.0>
   %add = fadd <4 x float> %mul, %y
   %r = fsub nsz reassoc <4 x float> %y, %add
@@ -122,10 +116,10 @@ define <4 x float> @fsub_neg_y_vector_nonuniform(<4 x float> %x, <4 x float> %y)
 }
 
 define float @fsub_neg_y_commute(float %x, float %y) {
-; ANY-LABEL: fsub_neg_y_commute:
-; ANY:       # %bb.0:
-; ANY-NEXT:    mulss {{.*}}(%rip), %xmm0
-; ANY-NEXT:    retq
+; CHECK-LABEL: fsub_neg_y_commute:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mulss {{.*}}(%rip), %xmm0
+; CHECK-NEXT:    retq
   %mul = fmul float %x, 5.0
   %add = fadd float %y, %mul
   %r = fsub nsz reassoc float %y, %add
@@ -133,10 +127,10 @@ define float @fsub_neg_y_commute(float %x, float %y) {
 }
 
 define <4 x float> @fsub_neg_y_commute_vector(<4 x float> %x, <4 x float> %y) {
-; ANY-LABEL: fsub_neg_y_commute_vector:
-; ANY:       # %bb.0:
-; ANY-NEXT:    mulps {{.*}}(%rip), %xmm0
-; ANY-NEXT:    retq
+; CHECK-LABEL: fsub_neg_y_commute_vector:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mulps {{.*}}(%rip), %xmm0
+; CHECK-NEXT:    retq
   %mul = fmul <4 x float> %x, <float 5.0, float 5.0, float 5.0, float 5.0>
   %add = fadd <4 x float> %y, %mul
   %r = fsub nsz reassoc <4 x float> %y, %add
@@ -146,10 +140,10 @@ define <4 x float> @fsub_neg_y_commute_vector(<4 x float> %x, <4 x float> %y) {
 ; Y - (X + Y) --> -X
 
 define float @fsub_fadd_common_op_fneg(float %x, float %y) {
-; ANY-LABEL: fsub_fadd_common_op_fneg:
-; ANY:       # %bb.0:
-; ANY-NEXT:    xorps {{.*}}(%rip), %xmm0
-; ANY-NEXT:    retq
+; CHECK-LABEL: fsub_fadd_common_op_fneg:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xorps {{.*}}(%rip), %xmm0
+; CHECK-NEXT:    retq
   %a = fadd float %x, %y
   %r = fsub reassoc nsz float %y, %a
   ret float %r
@@ -158,10 +152,10 @@ define float @fsub_fadd_common_op_fneg(float %x, float %y) {
 ; Y - (X + Y) --> -X
 
 define <4 x float> @fsub_fadd_common_op_fneg_vec(<4 x float> %x, <4 x float> %y) {
-; ANY-LABEL: fsub_fadd_common_op_fneg_vec:
-; ANY:       # %bb.0:
-; ANY-NEXT:    xorps {{.*}}(%rip), %xmm0
-; ANY-NEXT:    retq
+; CHECK-LABEL: fsub_fadd_common_op_fneg_vec:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xorps {{.*}}(%rip), %xmm0
+; CHECK-NEXT:    retq
   %a = fadd <4 x float> %x, %y
   %r = fsub nsz reassoc <4 x float> %y, %a
   ret <4 x float> %r
@@ -171,10 +165,10 @@ define <4 x float> @fsub_fadd_common_op_fneg_vec(<4 x float> %x, <4 x float> %y)
 ; Commute operands of the 'add'.
 
 define float @fsub_fadd_common_op_fneg_commute(float %x, float %y) {
-; ANY-LABEL: fsub_fadd_common_op_fneg_commute:
-; ANY:       # %bb.0:
-; ANY-NEXT:    xorps {{.*}}(%rip), %xmm0
-; ANY-NEXT:    retq
+; CHECK-LABEL: fsub_fadd_common_op_fneg_commute:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xorps {{.*}}(%rip), %xmm0
+; CHECK-NEXT:    retq
   %a = fadd float %y, %x
   %r = fsub reassoc nsz float %y, %a
   ret float %r
@@ -183,90 +177,90 @@ define float @fsub_fadd_common_op_fneg_commute(float %x, float %y) {
 ; Y - (Y + X) --> -X
 
 define <4 x float> @fsub_fadd_common_op_fneg_commute_vec(<4 x float> %x, <4 x float> %y) {
-; ANY-LABEL: fsub_fadd_common_op_fneg_commute_vec:
-; ANY:       # %bb.0:
-; ANY-NEXT:    xorps {{.*}}(%rip), %xmm0
-; ANY-NEXT:    retq
+; CHECK-LABEL: fsub_fadd_common_op_fneg_commute_vec:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xorps {{.*}}(%rip), %xmm0
+; CHECK-NEXT:    retq
   %a = fadd <4 x float> %y, %x
   %r = fsub reassoc nsz <4 x float> %y, %a
   ret <4 x float> %r
 }
 
-define float @fsub_negzero(float %x) {
-; STRICT-LABEL: fsub_negzero:
-; STRICT:       # %bb.0:
-; STRICT-NEXT:    xorps %xmm1, %xmm1
-; STRICT-NEXT:    addss %xmm1, %xmm0
-; STRICT-NEXT:    retq
-;
-; UNSAFE-LABEL: fsub_negzero:
-; UNSAFE:       # %bb.0:
-; UNSAFE-NEXT:    retq
+define float @fsub_negzero_strict(float %x) {
+; CHECK-LABEL: fsub_negzero_strict:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xorps %xmm1, %xmm1
+; CHECK-NEXT:    addss %xmm1, %xmm0
+; CHECK-NEXT:    retq
   %r = fsub float %x, -0.0
   ret float %r
 }
 
-define <4 x float> @fsub_negzero_vector(<4 x float> %x) {
-; STRICT-LABEL: fsub_negzero_vector:
-; STRICT:       # %bb.0:
-; STRICT-NEXT:    xorps %xmm1, %xmm1
-; STRICT-NEXT:    addps %xmm1, %xmm0
-; STRICT-NEXT:    retq
-;
-; UNSAFE-LABEL: fsub_negzero_vector:
-; UNSAFE:       # %bb.0:
-; UNSAFE-NEXT:    retq
+define float @fsub_negzero_nsz(float %x) {
+; CHECK-LABEL: fsub_negzero_nsz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    retq
+  %r = fsub nsz float %x, -0.0
+  ret float %r
+}
+
+define <4 x float> @fsub_negzero_strict_vector(<4 x float> %x) {
+; CHECK-LABEL: fsub_negzero_strict_vector:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xorps %xmm1, %xmm1
+; CHECK-NEXT:    addps %xmm1, %xmm0
+; CHECK-NEXT:    retq
   %r = fsub <4 x float> %x, <float -0.0, float -0.0, float -0.0, float -0.0>
   ret <4 x float> %r
 }
 
+define <4 x float> @fsub_negzero_nsz_vector(<4 x float> %x) {
+; CHECK-LABEL: fsub_negzero_nsz_vector:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    retq
+  %r = fsub nsz <4 x float> %x, <float -0.0, float -0.0, float -0.0, float -0.0>
+  ret <4 x float> %r
+}
+
 define float @fsub_zero_nsz_1(float %x) {
-; ANY-LABEL: fsub_zero_nsz_1:
-; ANY:       # %bb.0:
-; ANY-NEXT:    retq
+; CHECK-LABEL: fsub_zero_nsz_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    retq
   %r = fsub nsz float %x, 0.0
   ret float %r
 }
 
 define float @fsub_zero_nsz_2(float %x) {
-; ANY-LABEL: fsub_zero_nsz_2:
-; ANY:       # %bb.0:
-; ANY-NEXT:    xorps {{.*}}(%rip), %xmm0
-; ANY-NEXT:    retq
+; CHECK-LABEL: fsub_zero_nsz_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xorps {{.*}}(%rip), %xmm0
+; CHECK-NEXT:    retq
   %r = fsub nsz float 0.0, %x
   ret float %r
 }
 
-define float @fsub_negzero_nsz(float %x) {
-; ANY-LABEL: fsub_negzero_nsz:
-; ANY:       # %bb.0:
-; ANY-NEXT:    retq
-  %r = fsub nsz float %x, -0.0
-  ret float %r
-}
-
 define float @fmul_zero(float %x) {
-; ANY-LABEL: fmul_zero:
-; ANY:       # %bb.0:
-; ANY-NEXT:    xorps %xmm0, %xmm0
-; ANY-NEXT:    retq
+; CHECK-LABEL: fmul_zero:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xorps %xmm0, %xmm0
+; CHECK-NEXT:    retq
   %r = fmul nnan nsz float %x, 0.0
   ret float %r
 }
 
 define float @fmul_one(float %x) {
-; ANY-LABEL: fmul_one:
-; ANY:       # %bb.0:
-; ANY-NEXT:    retq
+; CHECK-LABEL: fmul_one:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    retq
   %r = fmul float %x, 1.0
   ret float %r
 }
 
 define float @fmul_x_const_const(float %x) {
-; ANY-LABEL: fmul_x_const_const:
-; ANY:       # %bb.0:
-; ANY-NEXT:    mulss {{.*}}(%rip), %xmm0
-; ANY-NEXT:    retq
+; CHECK-LABEL: fmul_x_const_const:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mulss {{.*}}(%rip), %xmm0
+; CHECK-NEXT:    retq
   %mul = fmul reassoc float %x, 9.0
   %r = fmul reassoc float %mul, 4.0
   ret float %r
diff --git a/llvm/test/CodeGen/X86/fp-intrinsics-flags-x86_64.ll b/llvm/test/CodeGen/X86/fp-intrinsics-flags-x86_64.ll
new file mode 100644
index 000000000..c2228046d
--- /dev/null
+++ b/llvm/test/CodeGen/X86/fp-intrinsics-flags-x86_64.ll
@@ -0,0 +1,18 @@
+; RUN: llc -O3 -mtriple=x86_64-pc-linux < %s -stop-after=finalize-isel | FileCheck %s
+
+define i32 @f20u(double %x) #0 {
+; CHECK-LABEL: name: f20u
+; CHECK: liveins: $xmm0
+; CHECK: [[COPY:%[0-9]+]]:fr64 = COPY $xmm0
+; CHECK: [[CVTTSD2SI64rr:%[0-9]+]]:gr64 = CVTTSD2SI64rr [[COPY]], implicit $mxcsr
+; CHECK: [[COPY1:%[0-9]+]]:gr32 = COPY [[CVTTSD2SI64rr]].sub_32bit
+; CHECK: $eax = COPY [[COPY1]]
+; CHECK: RET 0, $eax
+entry:
+  %result = call i32 @llvm.experimental.constrained.fptoui.i32.f64(double %x, metadata !"fpexcept.strict") #0
+  ret i32 %result
+}
+
+attributes #0 = { strictfp }
+
+declare i32 @llvm.experimental.constrained.fptoui.i32.f64(double, metadata)
diff --git a/llvm/test/CodeGen/X86/fp-intrinsics-flags.ll b/llvm/test/CodeGen/X86/fp-intrinsics-flags.ll
new file mode 100644
index 000000000..75f5c0b93
--- /dev/null
+++ b/llvm/test/CodeGen/X86/fp-intrinsics-flags.ll
@@ -0,0 +1,110 @@
+; RUN: llc -O3 -mtriple=i686-pc-linux -mattr=sse2 -stop-after=finalize-isel < %s | FileCheck %s
+
+define double @sifdb(i8 %x) #0 {
+entry:
+; CHECK-LABEL: name: sifdb
+; CHECK: [[MOVSX32rm8_:%[0-9]+]]:gr32 = MOVSX32rm8 %fixed-stack.0, 1, $noreg, 0, $noreg :: (load 1 from %fixed-stack.0, align 16)
+; CHECK: [[CVTSI2SDrr:%[0-9]+]]:fr64 = CVTSI2SDrr killed [[MOVSX32rm8_]]
+; CHECK: MOVSDmr %stack.0, 1, $noreg, 0, $noreg, killed [[CVTSI2SDrr]] :: (store 8 into %stack.0, align 4)
+; CHECK: [[LD_Fp64m80_:%[0-9]+]]:rfp80 = nofpexcept LD_Fp64m80 %stack.0, 1, $noreg, 0, $noreg, implicit-def dead $fpsw, implicit $fpcw :: (load 8 from %stack.0, align 4)
+; CHECK: RET 0, killed [[LD_Fp64m80_]]
+  %result = call double @llvm.experimental.constrained.sitofp.f64.i8(i8 %x, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret double %result
+}
+
+define double @sifdw(i16 %x) #0 {
+entry:
+; CHECK-LABEL: name: sifdw
+; CHECK: [[MOVSX32rm16_:%[0-9]+]]:gr32 = MOVSX32rm16 %fixed-stack.0, 1, $noreg, 0, $noreg :: (load 2 from %fixed-stack.0, align 16)
+; CHECK: [[CVTSI2SDrr:%[0-9]+]]:fr64 = CVTSI2SDrr killed [[MOVSX32rm16_]]
+; CHECK: MOVSDmr %stack.0, 1, $noreg, 0, $noreg, killed [[CVTSI2SDrr]] :: (store 8 into %stack.0, align 4)
+; CHECK: [[LD_Fp64m80_:%[0-9]+]]:rfp80 = nofpexcept LD_Fp64m80 %stack.0, 1, $noreg, 0, $noreg, implicit-def dead $fpsw, implicit $fpcw :: (load 8 from %stack.0, align 4)
+; CHECK: RET 0, killed [[LD_Fp64m80_]]
+  %result = call double @llvm.experimental.constrained.sitofp.f64.i16(i16 %x, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret double %result
+}
+
+define i64 @f20u64(double %x) #0 {
+entry:
+; CHECK-LABEL: name: f20u64
+; CHECK: [[MOVSDrm_alt:%[0-9]+]]:fr64 = MOVSDrm_alt %fixed-stack.0, 1, $noreg, 0, $noreg :: (load 8 from %fixed-stack.0, align 16)
+; CHECK: [[MOVSDrm_alt1:%[0-9]+]]:fr64 = MOVSDrm_alt $noreg, 1, $noreg, %const.0, $noreg :: (load 8 from constant-pool)
+; CHECK: COMISDrr [[MOVSDrm_alt1]], [[MOVSDrm_alt]], implicit-def $eflags, implicit $mxcsr
+; CHECK: [[FsFLD0SD:%[0-9]+]]:fr64 = FsFLD0SD
+; CHECK: JCC_1
+; CHECK: [[PHI:%[0-9]+]]:fr64 = PHI [[MOVSDrm_alt1]], {{.*}}, [[FsFLD0SD]], {{.*}}
+; CHECK: [[SUBSDrr:%[0-9]+]]:fr64 = SUBSDrr [[MOVSDrm_alt]], killed [[PHI]], implicit $mxcsr
+; CHECK: MOVSDmr %stack.0, 1, $noreg, 0, $noreg, killed [[SUBSDrr]] :: (store 8 into %stack.0)
+; CHECK: [[SETCCr:%[0-9]+]]:gr8 = SETCCr 6, implicit $eflags
+; CHECK: [[LD_Fp64m:%[0-9]+]]:rfp64 = LD_Fp64m %stack.0, 1, $noreg, 0, $noreg, implicit-def dead $fpsw, implicit $fpcw :: (load 8 from %stack.0)
+; CHECK: FNSTCW16m %stack.1, 1, $noreg, 0, $noreg, implicit-def $fpsw, implicit $fpcw :: (store 2 into %stack.1)
+; CHECK: [[MOVZX32rm16_:%[0-9]+]]:gr32 = MOVZX32rm16 %stack.1, 1, $noreg, 0, $noreg :: (load 2 from %stack.1)
+; CHECK: [[OR32ri:%[0-9]+]]:gr32 = OR32ri killed [[MOVZX32rm16_]], 3072, implicit-def $eflags
+; CHECK: [[COPY3:%[0-9]+]]:gr16 = COPY killed [[OR32ri]].sub_16bit
+; CHECK: MOV16mr %stack.2, 1, $noreg, 0, $noreg, killed [[COPY3]] :: (store 2 into %stack.2)
+; CHECK: FLDCW16m %stack.2, 1, $noreg, 0, $noreg, implicit-def $fpsw, implicit-def $fpcw :: (load 2 from %stack.2)
+; CHECK: IST_Fp64m64 %stack.0, 1, $noreg, 0, $noreg, [[LD_Fp64m]], implicit-def $fpsw, implicit $fpcw
+; CHECK: FLDCW16m %stack.1, 1, $noreg, 0, $noreg, implicit-def $fpsw, implicit-def $fpcw :: (load 2 from %stack.1)
+; CHECK: [[MOVZX32rr8_:%[0-9]+]]:gr32 = MOVZX32rr8 killed [[SETCCr]]
+; CHECK: [[SHL32ri:%[0-9]+]]:gr32 = SHL32ri [[MOVZX32rr8_]], 31, implicit-def dead $eflags
+; CHECK: [[XOR32rm:%[0-9]+]]:gr32 = XOR32rm [[SHL32ri]], %stack.0, 1, $noreg, 4, $noreg, implicit-def dead $eflags :: (load 4 from %stack.0 + 4)
+; CHECK: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm %stack.0, 1, $noreg, 0, $noreg :: (load 4 from %stack.0, align 8)
+; CHECK: $eax = COPY [[MOV32rm]]
+; CHECK: $edx = COPY [[XOR32rm]]
+; CHECK: RET 0, $eax, $edx
+  %result = call i64 @llvm.experimental.constrained.fptoui.i64.f64(double %x, metadata !"fpexcept.strict") #0
+  ret i64 %result
+}
+
+define i8 @f20s8(double %x) #0 {
+entry:
+; CHECK-LABEL: name: f20s8
+; CHECK: [[CVTTSD2SIrm:%[0-9]+]]:gr32 = CVTTSD2SIrm %fixed-stack.0, 1, $noreg, 0, $noreg, implicit $mxcsr :: (load 8 from %fixed-stack.0, align 16)
+; CHECK: [[COPY:%[0-9]+]]:gr32_abcd = COPY [[CVTTSD2SIrm]]
+; CHECK: [[COPY1:%[0-9]+]]:gr8 = COPY [[COPY]].sub_8bit
+; CHECK: $al = COPY [[COPY1]]
+; CHECK: RET 0, $al
+  %result = call i8 @llvm.experimental.constrained.fptosi.i8.f64(double %x, metadata !"fpexcept.strict") #0
+  ret i8 %result
+}
+
+define i16 @f20s16(double %x) #0 {
+entry:
+; CHECK-LABEL: name: f20s16
+; CHECK: [[CVTTSD2SIrm:%[0-9]+]]:gr32 = CVTTSD2SIrm %fixed-stack.0, 1, $noreg, 0, $noreg, implicit $mxcsr :: (load 8 from %fixed-stack.0, align 16)
+; CHECK: [[COPY:%[0-9]+]]:gr16 = COPY [[CVTTSD2SIrm]].sub_16bit
+; CHECK: $ax = COPY [[COPY]]
+; CHECK: RET 0, $ax
+  %result = call i16 @llvm.experimental.constrained.fptosi.i16.f64(double %x, metadata !"fpexcept.strict") #0
+  ret i16 %result
+}
+
+define i32 @f20u(double %x) #0 {
+entry:
+; CHECK-LABEL: name: f20u
+; CHECK: [[MOVSDrm_alt:%[0-9]+]]:fr64 = MOVSDrm_alt %fixed-stack.0, 1, $noreg, 0, $noreg :: (load 8 from %fixed-stack.0, align 16)
+; CHECK: [[MOVSDrm_alt1:%[0-9]+]]:fr64 = MOVSDrm_alt $noreg, 1, $noreg, %const.0, $noreg :: (load 8 from constant-pool)
+; CHECK: COMISDrr [[MOVSDrm_alt1]], [[MOVSDrm_alt]], implicit-def $eflags, implicit $mxcsr
+; CHECK: [[FsFLD0SD:%[0-9]+]]:fr64 = FsFLD0SD
+; CHECK: JCC_1
+; CHECK: [[PHI:%[0-9]+]]:fr64 = PHI [[MOVSDrm_alt1]], {{.*}}, [[FsFLD0SD]], {{.*}}
+; CHECK: [[SETCCr:%[0-9]+]]:gr8 = SETCCr 6, implicit $eflags
+; CHECK: [[MOVZX32rr8_:%[0-9]+]]:gr32 = MOVZX32rr8 killed [[SETCCr]]
+; CHECK: [[SHL32ri:%[0-9]+]]:gr32 = SHL32ri [[MOVZX32rr8_]], 31, implicit-def dead $eflags
+; CHECK: [[SUBSDrr:%[0-9]+]]:fr64 = SUBSDrr [[MOVSDrm_alt]], killed [[PHI]], implicit $mxcsr
+; CHECK: [[CVTTSD2SIrr:%[0-9]+]]:gr32 = CVTTSD2SIrr killed [[SUBSDrr]], implicit $mxcsr
+; CHECK: [[XOR32rr:%[0-9]+]]:gr32 = XOR32rr [[CVTTSD2SIrr]], killed [[SHL32ri]], implicit-def dead $eflags
+; CHECK: $eax = COPY [[XOR32rr]]
+; CHECK: RET 0, $eax
+  %result = call i32 @llvm.experimental.constrained.fptoui.i32.f64(double %x, metadata !"fpexcept.strict") #0
+  ret i32 %result
+}
+
+attributes #0 = { strictfp }
+
+declare double @llvm.experimental.constrained.sitofp.f64.i8(i8, metadata, metadata)
+declare double @llvm.experimental.constrained.sitofp.f64.i16(i16, metadata, metadata)
+declare i32 @llvm.experimental.constrained.fptoui.i32.f64(double, metadata)
+declare i64 @llvm.experimental.constrained.fptoui.i64.f64(double, metadata)
+declare i8 @llvm.experimental.constrained.fptosi.i8.f64(double, metadata)
+declare i16 @llvm.experimental.constrained.fptosi.i16.f64(double, metadata)
diff --git a/llvm/test/CodeGen/X86/fp-intrinsics-fma.ll b/llvm/test/CodeGen/X86/fp-intrinsics-fma.ll
new file mode 100644
index 000000000..101c47fd0
--- /dev/null
+++ b/llvm/test/CodeGen/X86/fp-intrinsics-fma.ll
@@ -0,0 +1,69 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -O3 -mtriple=x86_64-pc-linux < %s | FileCheck %s --check-prefixes=COMMON,NOFMA
+; RUN: llc -O3 -mtriple=x86_64-pc-linux -mattr=+fma < %s | FileCheck %s --check-prefixes=COMMON,FMA
+; RUN: llc -O3 -mtriple=x86_64-pc-linux -mattr=+avx512f < %s | FileCheck %s --check-prefixes=COMMON,FMA
+
+; Verify that fma(3.5) isn't simplified when the rounding mode is
+; unknown.
+define float @f17() #0 {
+; NOFMA-LABEL: f17:
+; NOFMA:       # %bb.0: # %entry
+; NOFMA-NEXT:    pushq %rax
+; NOFMA-NEXT:    .cfi_def_cfa_offset 16
+; NOFMA-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; NOFMA-NEXT:    movaps %xmm0, %xmm1
+; NOFMA-NEXT:    movaps %xmm0, %xmm2
+; NOFMA-NEXT:    callq fmaf
+; NOFMA-NEXT:    popq %rax
+; NOFMA-NEXT:    .cfi_def_cfa_offset 8
+; NOFMA-NEXT:    retq
+;
+; FMA-LABEL: f17:
+; FMA:       # %bb.0: # %entry
+; FMA-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; FMA-NEXT:    vfmadd213ss {{.*#+}} xmm0 = (xmm0 * xmm0) + xmm0
+; FMA-NEXT:    retq
+entry:
+  %result = call float @llvm.experimental.constrained.fma.f32(
+                                               float 3.5,
+                                               float 3.5,
+                                               float 3.5,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret float %result
+}
+
+; Verify that fma(42.1) isn't simplified when the rounding mode is
+; unknown.
+define double @f18() #0 {
+; NOFMA-LABEL: f18:
+; NOFMA:       # %bb.0: # %entry
+; NOFMA-NEXT:    pushq %rax
+; NOFMA-NEXT:    .cfi_def_cfa_offset 16
+; NOFMA-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; NOFMA-NEXT:    movaps %xmm0, %xmm1
+; NOFMA-NEXT:    movaps %xmm0, %xmm2
+; NOFMA-NEXT:    callq fma
+; NOFMA-NEXT:    popq %rax
+; NOFMA-NEXT:    .cfi_def_cfa_offset 8
+; NOFMA-NEXT:    retq
+;
+; FMA-LABEL: f18:
+; FMA:       # %bb.0: # %entry
+; FMA-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; FMA-NEXT:    vfmadd213sd {{.*#+}} xmm0 = (xmm0 * xmm0) + xmm0
+; FMA-NEXT:    retq
+entry:
+  %result = call double @llvm.experimental.constrained.fma.f64(
+                                               double 42.1,
+                                               double 42.1,
+                                               double 42.1,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret double %result
+}
+
+attributes #0 = { strictfp }
+
+declare float @llvm.experimental.constrained.fma.f32(float, float, float, metadata, metadata)
+declare double @llvm.experimental.constrained.fma.f64(double, double, double, metadata, metadata)
diff --git a/llvm/test/CodeGen/X86/fp-intrinsics.ll b/llvm/test/CodeGen/X86/fp-intrinsics.ll
index ffcbdef3d..e00248b22 100644
--- a/llvm/test/CodeGen/X86/fp-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/fp-intrinsics.ll
@@ -1,5 +1,10 @@
-; RUN: llc -O3 -mtriple=x86_64-pc-linux < %s | FileCheck --check-prefix=COMMON --check-prefix=NO-FMA --check-prefix=FMACALL64 --check-prefix=FMACALL32 %s
-; RUN: llc -O3 -mtriple=x86_64-pc-linux -mattr=+fma < %s | FileCheck -check-prefix=COMMON --check-prefix=HAS-FMA --check-prefix=FMA64 --check-prefix=FMA32 %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -O3 -mtriple=i686-pc-linux -mattr=+cmov < %s | FileCheck %s --check-prefix=COMMON --check-prefix=X87
+; RUN: llc -O3 -mtriple=i686-pc-linux -mattr=sse2 < %s | FileCheck %s --check-prefix=COMMON --check-prefix=X86-SSE
+; RUN: llc -O3 -mtriple=x86_64-pc-linux < %s | FileCheck %s --check-prefix=COMMON --check-prefix=SSE
+; RUN: llc -O3 -mtriple=x86_64-pc-linux -mattr=+avx < %s | FileCheck %s --check-prefix=COMMON --check-prefix=AVX --check-prefix=AVX1
+; RUN: llc -O3 -mtriple=x86_64-pc-linux -mattr=+avx512f < %s | FileCheck %s --check-prefix=COMMON --check-prefix=AVX --check-prefix=AVX512 --check-prefix=AVX512F
+; RUN: llc -O3 -mtriple=x86_64-pc-linux -mattr=+avx512dq < %s | FileCheck %s --check-prefix=COMMON --check-prefix=AVX --check-prefix=AVX512 --check-prefix=AVX512DQ
 
 ; Verify that constants aren't folded to inexact results when the rounding mode
 ; is unknown.
@@ -9,15 +14,42 @@
 ;   return 1.0/10.0;
 ; }
 ;
-; CHECK-LABEL: f1
-; COMMON: divsd
-define double @f1() {
+define double @f1() #0 {
+; X87-LABEL: f1:
+; X87:       # %bb.0: # %entry
+; X87-NEXT:    fld1
+; X87-NEXT:    fdivs {{\.LCPI.*}}
+; X87-NEXT:    retl
+;
+; X86-SSE-LABEL: f1:
+; X86-SSE:       # %bb.0: # %entry
+; X86-SSE-NEXT:    subl $12, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 16
+; X86-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE-NEXT:    divsd {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    movsd %xmm0, (%esp)
+; X86-SSE-NEXT:    fldl (%esp)
+; X86-SSE-NEXT:    addl $12, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 4
+; X86-SSE-NEXT:    retl
+;
+; SSE-LABEL: f1:
+; SSE:       # %bb.0: # %entry
+; SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE-NEXT:    divsd {{.*}}(%rip), %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: f1:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-NEXT:    vdivsd {{.*}}(%rip), %xmm0, %xmm0
+; AVX-NEXT:    retq
 entry:
   %div = call double @llvm.experimental.constrained.fdiv.f64(
                                                double 1.000000e+00,
                                                double 1.000000e+01,
                                                metadata !"round.dynamic",
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
   ret double %div
 }
 
@@ -29,15 +61,43 @@ entry:
 ;   return a - 0;
 ; }
 ;
-; CHECK-LABEL: f2
-; COMMON:  subsd
-define double @f2(double %a) {
+define double @f2(double %a) #0 {
+; X87-LABEL: f2:
+; X87:       # %bb.0: # %entry
+; X87-NEXT:    fldz
+; X87-NEXT:    fsubrl {{[0-9]+}}(%esp)
+; X87-NEXT:    retl
+;
+; X86-SSE-LABEL: f2:
+; X86-SSE:       # %bb.0: # %entry
+; X86-SSE-NEXT:    subl $12, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 16
+; X86-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE-NEXT:    xorpd %xmm1, %xmm1
+; X86-SSE-NEXT:    subsd %xmm1, %xmm0
+; X86-SSE-NEXT:    movsd %xmm0, (%esp)
+; X86-SSE-NEXT:    fldl (%esp)
+; X86-SSE-NEXT:    addl $12, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 4
+; X86-SSE-NEXT:    retl
+;
+; SSE-LABEL: f2:
+; SSE:       # %bb.0: # %entry
+; SSE-NEXT:    xorpd %xmm1, %xmm1
+; SSE-NEXT:    subsd %xmm1, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: f2:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vsubsd %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
 entry:
   %sub = call double @llvm.experimental.constrained.fsub.f64(
                                                double %a,
                                                double 0.000000e+00,
                                                metadata !"round.dynamic",
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
   ret double %sub
 }
 
@@ -50,25 +110,63 @@ entry:
 ;   return -((-a)*b);
 ; }
 ;
-; CHECK-LABEL: f3:
-; COMMON:  subsd
-; COMMON:  mulsd
-; COMMON:  subsd
-define double @f3(double %a, double %b) {
+define double @f3(double %a, double %b) #0 {
+; X87-LABEL: f3:
+; X87:       # %bb.0: # %entry
+; X87-NEXT:    fldz
+; X87-NEXT:    fchs
+; X87-NEXT:    fld %st(0)
+; X87-NEXT:    fsubl {{[0-9]+}}(%esp)
+; X87-NEXT:    fmull {{[0-9]+}}(%esp)
+; X87-NEXT:    fsubrp %st, %st(1)
+; X87-NEXT:    retl
+;
+; X86-SSE-LABEL: f3:
+; X86-SSE:       # %bb.0: # %entry
+; X86-SSE-NEXT:    subl $12, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 16
+; X86-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE-NEXT:    movapd %xmm0, %xmm1
+; X86-SSE-NEXT:    subsd {{[0-9]+}}(%esp), %xmm1
+; X86-SSE-NEXT:    mulsd {{[0-9]+}}(%esp), %xmm1
+; X86-SSE-NEXT:    subsd %xmm1, %xmm0
+; X86-SSE-NEXT:    movsd %xmm0, (%esp)
+; X86-SSE-NEXT:    fldl (%esp)
+; X86-SSE-NEXT:    addl $12, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 4
+; X86-SSE-NEXT:    retl
+;
+; SSE-LABEL: f3:
+; SSE:       # %bb.0: # %entry
+; SSE-NEXT:    movsd {{.*#+}} xmm2 = mem[0],zero
+; SSE-NEXT:    movapd %xmm2, %xmm3
+; SSE-NEXT:    subsd %xmm0, %xmm3
+; SSE-NEXT:    mulsd %xmm1, %xmm3
+; SSE-NEXT:    subsd %xmm3, %xmm2
+; SSE-NEXT:    movapd %xmm2, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: f3:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    vmovsd {{.*#+}} xmm2 = mem[0],zero
+; AVX-NEXT:    vsubsd %xmm0, %xmm2, %xmm0
+; AVX-NEXT:    vmulsd %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vsubsd %xmm0, %xmm2, %xmm0
+; AVX-NEXT:    retq
 entry:
   %sub = call double @llvm.experimental.constrained.fsub.f64(
                                                double -0.000000e+00, double %a,
                                                metadata !"round.dynamic",
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
   %mul = call double @llvm.experimental.constrained.fmul.f64(
                                                double %sub, double %b,
                                                metadata !"round.dynamic",
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
   %ret = call double @llvm.experimental.constrained.fsub.f64(
                                                double -0.000000e+00,
                                                double %mul,
                                                metadata !"round.dynamic",
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
   ret double %ret
 }
 
@@ -83,11 +181,51 @@ entry:
 ; }
 ;
 ;
-; CHECK-LABEL: f4:
-; COMMON: testl
-; COMMON: jle
-; COMMON: addsd
-define double @f4(i32 %n, double %a) {
+define double @f4(i32 %n, double %a) #0 {
+; X87-LABEL: f4:
+; X87:       # %bb.0: # %entry
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    cmpl $0, {{[0-9]+}}(%esp)
+; X87-NEXT:    jle .LBB3_2
+; X87-NEXT:  # %bb.1: # %if.then
+; X87-NEXT:    fld1
+; X87-NEXT:    faddp %st, %st(1)
+; X87-NEXT:  .LBB3_2: # %if.end
+; X87-NEXT:    retl
+;
+; X86-SSE-LABEL: f4:
+; X86-SSE:       # %bb.0: # %entry
+; X86-SSE-NEXT:    subl $12, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 16
+; X86-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE-NEXT:    cmpl $0, {{[0-9]+}}(%esp)
+; X86-SSE-NEXT:    jle .LBB3_2
+; X86-SSE-NEXT:  # %bb.1: # %if.then
+; X86-SSE-NEXT:    addsd {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:  .LBB3_2: # %if.end
+; X86-SSE-NEXT:    movsd %xmm0, (%esp)
+; X86-SSE-NEXT:    fldl (%esp)
+; X86-SSE-NEXT:    addl $12, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 4
+; X86-SSE-NEXT:    retl
+;
+; SSE-LABEL: f4:
+; SSE:       # %bb.0: # %entry
+; SSE-NEXT:    testl %edi, %edi
+; SSE-NEXT:    jle .LBB3_2
+; SSE-NEXT:  # %bb.1: # %if.then
+; SSE-NEXT:    addsd {{.*}}(%rip), %xmm0
+; SSE-NEXT:  .LBB3_2: # %if.end
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: f4:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    testl %edi, %edi
+; AVX-NEXT:    jle .LBB3_2
+; AVX-NEXT:  # %bb.1: # %if.then
+; AVX-NEXT:    vaddsd {{.*}}(%rip), %xmm0, %xmm0
+; AVX-NEXT:  .LBB3_2: # %if.end
+; AVX-NEXT:    retq
 entry:
   %cmp = icmp sgt i32 %n, 0
   br i1 %cmp, label %if.then, label %if.end
@@ -96,7 +234,7 @@ if.then:
   %add = call double @llvm.experimental.constrained.fadd.f64(
                                                double 1.000000e+00, double %a,
                                                metadata !"round.dynamic",
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
   br label %if.end
 
 if.end:
@@ -105,210 +243,2445 @@ if.end:
 }
 
 ; Verify that sqrt(42.0) isn't simplified when the rounding mode is unknown.
-; CHECK-LABEL: f5
-; COMMON:  sqrtsd
-define double @f5() {
+define double @f5() #0 {
+; X87-LABEL: f5:
+; X87:       # %bb.0: # %entry
+; X87-NEXT:    flds {{\.LCPI.*}}
+; X87-NEXT:    fsqrt
+; X87-NEXT:    retl
+;
+; X86-SSE-LABEL: f5:
+; X86-SSE:       # %bb.0: # %entry
+; X86-SSE-NEXT:    subl $12, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 16
+; X86-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE-NEXT:    sqrtsd %xmm0, %xmm0
+; X86-SSE-NEXT:    movsd %xmm0, (%esp)
+; X86-SSE-NEXT:    fldl (%esp)
+; X86-SSE-NEXT:    addl $12, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 4
+; X86-SSE-NEXT:    retl
+;
+; SSE-LABEL: f5:
+; SSE:       # %bb.0: # %entry
+; SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE-NEXT:    sqrtsd %xmm0, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: f5:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-NEXT:    vsqrtsd %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    retq
 entry:
   %result = call double @llvm.experimental.constrained.sqrt.f64(double 42.0,
                                                metadata !"round.dynamic",
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
   ret double %result
 }
 
 ; Verify that pow(42.1, 3.0) isn't simplified when the rounding mode is unknown.
-; CHECK-LABEL: f6
-; COMMON:  pow
-define double @f6() {
+define double @f6() #0 {
+; X87-LABEL: f6:
+; X87:       # %bb.0: # %entry
+; X87-NEXT:    subl $28, %esp
+; X87-NEXT:    .cfi_def_cfa_offset 32
+; X87-NEXT:    flds {{\.LCPI.*}}
+; X87-NEXT:    fstpl {{[0-9]+}}(%esp)
+; X87-NEXT:    fldl {{\.LCPI.*}}
+; X87-NEXT:    fstpl (%esp)
+; X87-NEXT:    calll pow
+; X87-NEXT:    addl $28, %esp
+; X87-NEXT:    .cfi_def_cfa_offset 4
+; X87-NEXT:    retl
+;
+; X86-SSE-LABEL: f6:
+; X86-SSE:       # %bb.0: # %entry
+; X86-SSE-NEXT:    subl $28, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 32
+; X86-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE-NEXT:    movsd %xmm0, {{[0-9]+}}(%esp)
+; X86-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE-NEXT:    movsd %xmm0, (%esp)
+; X86-SSE-NEXT:    calll pow
+; X86-SSE-NEXT:    addl $28, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 4
+; X86-SSE-NEXT:    retl
+;
+; SSE-LABEL: f6:
+; SSE:       # %bb.0: # %entry
+; SSE-NEXT:    pushq %rax
+; SSE-NEXT:    .cfi_def_cfa_offset 16
+; SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
+; SSE-NEXT:    callq pow
+; SSE-NEXT:    popq %rax
+; SSE-NEXT:    .cfi_def_cfa_offset 8
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: f6:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    pushq %rax
+; AVX-NEXT:    .cfi_def_cfa_offset 16
+; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
+; AVX-NEXT:    callq pow
+; AVX-NEXT:    popq %rax
+; AVX-NEXT:    .cfi_def_cfa_offset 8
+; AVX-NEXT:    retq
 entry:
   %result = call double @llvm.experimental.constrained.pow.f64(double 42.1,
                                                double 3.0,
                                                metadata !"round.dynamic",
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
   ret double %result
 }
 
 ; Verify that powi(42.1, 3) isn't simplified when the rounding mode is unknown.
-; CHECK-LABEL: f7
-; COMMON:  powi
-define double @f7() {
+define double @f7() #0 {
+; X87-LABEL: f7:
+; X87:       # %bb.0: # %entry
+; X87-NEXT:    subl $12, %esp
+; X87-NEXT:    .cfi_def_cfa_offset 16
+; X87-NEXT:    fldl {{\.LCPI.*}}
+; X87-NEXT:    fstpl (%esp)
+; X87-NEXT:    movl $3, {{[0-9]+}}(%esp)
+; X87-NEXT:    calll __powidf2
+; X87-NEXT:    addl $12, %esp
+; X87-NEXT:    .cfi_def_cfa_offset 4
+; X87-NEXT:    retl
+;
+; X86-SSE-LABEL: f7:
+; X86-SSE:       # %bb.0: # %entry
+; X86-SSE-NEXT:    subl $12, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 16
+; X86-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE-NEXT:    movsd %xmm0, (%esp)
+; X86-SSE-NEXT:    movl $3, {{[0-9]+}}(%esp)
+; X86-SSE-NEXT:    calll __powidf2
+; X86-SSE-NEXT:    addl $12, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 4
+; X86-SSE-NEXT:    retl
+;
+; SSE-LABEL: f7:
+; SSE:       # %bb.0: # %entry
+; SSE-NEXT:    pushq %rax
+; SSE-NEXT:    .cfi_def_cfa_offset 16
+; SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE-NEXT:    movl $3, %edi
+; SSE-NEXT:    callq __powidf2
+; SSE-NEXT:    popq %rax
+; SSE-NEXT:    .cfi_def_cfa_offset 8
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: f7:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    pushq %rax
+; AVX-NEXT:    .cfi_def_cfa_offset 16
+; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-NEXT:    movl $3, %edi
+; AVX-NEXT:    callq __powidf2
+; AVX-NEXT:    popq %rax
+; AVX-NEXT:    .cfi_def_cfa_offset 8
+; AVX-NEXT:    retq
 entry:
   %result = call double @llvm.experimental.constrained.powi.f64(double 42.1,
                                                i32 3,
                                                metadata !"round.dynamic",
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
   ret double %result
 }
 
 ; Verify that sin(42.0) isn't simplified when the rounding mode is unknown.
-; CHECK-LABEL: f8
-; COMMON:  sin
-define double @f8() {
+define double @f8() #0 {
+; X87-LABEL: f8:
+; X87:       # %bb.0: # %entry
+; X87-NEXT:    subl $12, %esp
+; X87-NEXT:    .cfi_def_cfa_offset 16
+; X87-NEXT:    flds {{\.LCPI.*}}
+; X87-NEXT:    fstpl (%esp)
+; X87-NEXT:    calll sin
+; X87-NEXT:    addl $12, %esp
+; X87-NEXT:    .cfi_def_cfa_offset 4
+; X87-NEXT:    retl
+;
+; X86-SSE-LABEL: f8:
+; X86-SSE:       # %bb.0: # %entry
+; X86-SSE-NEXT:    subl $12, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 16
+; X86-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE-NEXT:    movsd %xmm0, (%esp)
+; X86-SSE-NEXT:    calll sin
+; X86-SSE-NEXT:    addl $12, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 4
+; X86-SSE-NEXT:    retl
+;
+; SSE-LABEL: f8:
+; SSE:       # %bb.0: # %entry
+; SSE-NEXT:    pushq %rax
+; SSE-NEXT:    .cfi_def_cfa_offset 16
+; SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE-NEXT:    callq sin
+; SSE-NEXT:    popq %rax
+; SSE-NEXT:    .cfi_def_cfa_offset 8
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: f8:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    pushq %rax
+; AVX-NEXT:    .cfi_def_cfa_offset 16
+; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-NEXT:    callq sin
+; AVX-NEXT:    popq %rax
+; AVX-NEXT:    .cfi_def_cfa_offset 8
+; AVX-NEXT:    retq
 entry:
   %result = call double @llvm.experimental.constrained.sin.f64(double 42.0,
                                                metadata !"round.dynamic",
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
   ret double %result
 }
 
 ; Verify that cos(42.0) isn't simplified when the rounding mode is unknown.
-; CHECK-LABEL: f9
-; COMMON:  cos
-define double @f9() {
+define double @f9() #0 {
+; X87-LABEL: f9:
+; X87:       # %bb.0: # %entry
+; X87-NEXT:    subl $12, %esp
+; X87-NEXT:    .cfi_def_cfa_offset 16
+; X87-NEXT:    flds {{\.LCPI.*}}
+; X87-NEXT:    fstpl (%esp)
+; X87-NEXT:    calll cos
+; X87-NEXT:    addl $12, %esp
+; X87-NEXT:    .cfi_def_cfa_offset 4
+; X87-NEXT:    retl
+;
+; X86-SSE-LABEL: f9:
+; X86-SSE:       # %bb.0: # %entry
+; X86-SSE-NEXT:    subl $12, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 16
+; X86-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE-NEXT:    movsd %xmm0, (%esp)
+; X86-SSE-NEXT:    calll cos
+; X86-SSE-NEXT:    addl $12, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 4
+; X86-SSE-NEXT:    retl
+;
+; SSE-LABEL: f9:
+; SSE:       # %bb.0: # %entry
+; SSE-NEXT:    pushq %rax
+; SSE-NEXT:    .cfi_def_cfa_offset 16
+; SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE-NEXT:    callq cos
+; SSE-NEXT:    popq %rax
+; SSE-NEXT:    .cfi_def_cfa_offset 8
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: f9:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    pushq %rax
+; AVX-NEXT:    .cfi_def_cfa_offset 16
+; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-NEXT:    callq cos
+; AVX-NEXT:    popq %rax
+; AVX-NEXT:    .cfi_def_cfa_offset 8
+; AVX-NEXT:    retq
 entry:
   %result = call double @llvm.experimental.constrained.cos.f64(double 42.0,
                                                metadata !"round.dynamic",
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
   ret double %result
 }
 
 ; Verify that exp(42.0) isn't simplified when the rounding mode is unknown.
-; CHECK-LABEL: f10
-; COMMON:  exp
-define double @f10() {
+define double @f10() #0 {
+; X87-LABEL: f10:
+; X87:       # %bb.0: # %entry
+; X87-NEXT:    subl $12, %esp
+; X87-NEXT:    .cfi_def_cfa_offset 16
+; X87-NEXT:    flds {{\.LCPI.*}}
+; X87-NEXT:    fstpl (%esp)
+; X87-NEXT:    calll exp
+; X87-NEXT:    addl $12, %esp
+; X87-NEXT:    .cfi_def_cfa_offset 4
+; X87-NEXT:    retl
+;
+; X86-SSE-LABEL: f10:
+; X86-SSE:       # %bb.0: # %entry
+; X86-SSE-NEXT:    subl $12, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 16
+; X86-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE-NEXT:    movsd %xmm0, (%esp)
+; X86-SSE-NEXT:    calll exp
+; X86-SSE-NEXT:    addl $12, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 4
+; X86-SSE-NEXT:    retl
+;
+; SSE-LABEL: f10:
+; SSE:       # %bb.0: # %entry
+; SSE-NEXT:    pushq %rax
+; SSE-NEXT:    .cfi_def_cfa_offset 16
+; SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE-NEXT:    callq exp
+; SSE-NEXT:    popq %rax
+; SSE-NEXT:    .cfi_def_cfa_offset 8
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: f10:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    pushq %rax
+; AVX-NEXT:    .cfi_def_cfa_offset 16
+; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-NEXT:    callq exp
+; AVX-NEXT:    popq %rax
+; AVX-NEXT:    .cfi_def_cfa_offset 8
+; AVX-NEXT:    retq
 entry:
   %result = call double @llvm.experimental.constrained.exp.f64(double 42.0,
                                                metadata !"round.dynamic",
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
   ret double %result
 }
 
 ; Verify that exp2(42.1) isn't simplified when the rounding mode is unknown.
-; CHECK-LABEL: f11
-; COMMON:  exp2
-define double @f11() {
+define double @f11() #0 {
+; X87-LABEL: f11:
+; X87:       # %bb.0: # %entry
+; X87-NEXT:    subl $12, %esp
+; X87-NEXT:    .cfi_def_cfa_offset 16
+; X87-NEXT:    fldl {{\.LCPI.*}}
+; X87-NEXT:    fstpl (%esp)
+; X87-NEXT:    calll exp2
+; X87-NEXT:    addl $12, %esp
+; X87-NEXT:    .cfi_def_cfa_offset 4
+; X87-NEXT:    retl
+;
+; X86-SSE-LABEL: f11:
+; X86-SSE:       # %bb.0: # %entry
+; X86-SSE-NEXT:    subl $12, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 16
+; X86-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE-NEXT:    movsd %xmm0, (%esp)
+; X86-SSE-NEXT:    calll exp2
+; X86-SSE-NEXT:    addl $12, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 4
+; X86-SSE-NEXT:    retl
+;
+; SSE-LABEL: f11:
+; SSE:       # %bb.0: # %entry
+; SSE-NEXT:    pushq %rax
+; SSE-NEXT:    .cfi_def_cfa_offset 16
+; SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE-NEXT:    callq exp2
+; SSE-NEXT:    popq %rax
+; SSE-NEXT:    .cfi_def_cfa_offset 8
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: f11:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    pushq %rax
+; AVX-NEXT:    .cfi_def_cfa_offset 16
+; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-NEXT:    callq exp2
+; AVX-NEXT:    popq %rax
+; AVX-NEXT:    .cfi_def_cfa_offset 8
+; AVX-NEXT:    retq
 entry:
   %result = call double @llvm.experimental.constrained.exp2.f64(double 42.1,
                                                metadata !"round.dynamic",
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
   ret double %result
 }
 
 ; Verify that log(42.0) isn't simplified when the rounding mode is unknown.
-; CHECK-LABEL: f12
-; COMMON:  log
-define double @f12() {
+define double @f12() #0 {
+; X87-LABEL: f12:
+; X87:       # %bb.0: # %entry
+; X87-NEXT:    subl $12, %esp
+; X87-NEXT:    .cfi_def_cfa_offset 16
+; X87-NEXT:    flds {{\.LCPI.*}}
+; X87-NEXT:    fstpl (%esp)
+; X87-NEXT:    calll log
+; X87-NEXT:    addl $12, %esp
+; X87-NEXT:    .cfi_def_cfa_offset 4
+; X87-NEXT:    retl
+;
+; X86-SSE-LABEL: f12:
+; X86-SSE:       # %bb.0: # %entry
+; X86-SSE-NEXT:    subl $12, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 16
+; X86-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE-NEXT:    movsd %xmm0, (%esp)
+; X86-SSE-NEXT:    calll log
+; X86-SSE-NEXT:    addl $12, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 4
+; X86-SSE-NEXT:    retl
+;
+; SSE-LABEL: f12:
+; SSE:       # %bb.0: # %entry
+; SSE-NEXT:    pushq %rax
+; SSE-NEXT:    .cfi_def_cfa_offset 16
+; SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE-NEXT:    callq log
+; SSE-NEXT:    popq %rax
+; SSE-NEXT:    .cfi_def_cfa_offset 8
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: f12:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    pushq %rax
+; AVX-NEXT:    .cfi_def_cfa_offset 16
+; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-NEXT:    callq log
+; AVX-NEXT:    popq %rax
+; AVX-NEXT:    .cfi_def_cfa_offset 8
+; AVX-NEXT:    retq
 entry:
   %result = call double @llvm.experimental.constrained.log.f64(double 42.0,
                                                metadata !"round.dynamic",
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
   ret double %result
 }
 
 ; Verify that log10(42.0) isn't simplified when the rounding mode is unknown.
-; CHECK-LABEL: f13
-; COMMON:  log10
-define double @f13() {
+define double @f13() #0 {
+; X87-LABEL: f13:
+; X87:       # %bb.0: # %entry
+; X87-NEXT:    subl $12, %esp
+; X87-NEXT:    .cfi_def_cfa_offset 16
+; X87-NEXT:    flds {{\.LCPI.*}}
+; X87-NEXT:    fstpl (%esp)
+; X87-NEXT:    calll log10
+; X87-NEXT:    addl $12, %esp
+; X87-NEXT:    .cfi_def_cfa_offset 4
+; X87-NEXT:    retl
+;
+; X86-SSE-LABEL: f13:
+; X86-SSE:       # %bb.0: # %entry
+; X86-SSE-NEXT:    subl $12, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 16
+; X86-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE-NEXT:    movsd %xmm0, (%esp)
+; X86-SSE-NEXT:    calll log10
+; X86-SSE-NEXT:    addl $12, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 4
+; X86-SSE-NEXT:    retl
+;
+; SSE-LABEL: f13:
+; SSE:       # %bb.0: # %entry
+; SSE-NEXT:    pushq %rax
+; SSE-NEXT:    .cfi_def_cfa_offset 16
+; SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE-NEXT:    callq log10
+; SSE-NEXT:    popq %rax
+; SSE-NEXT:    .cfi_def_cfa_offset 8
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: f13:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    pushq %rax
+; AVX-NEXT:    .cfi_def_cfa_offset 16
+; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-NEXT:    callq log10
+; AVX-NEXT:    popq %rax
+; AVX-NEXT:    .cfi_def_cfa_offset 8
+; AVX-NEXT:    retq
 entry:
   %result = call double @llvm.experimental.constrained.log10.f64(double 42.0,
                                                metadata !"round.dynamic",
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
   ret double %result
 }
 
 ; Verify that log2(42.0) isn't simplified when the rounding mode is unknown.
-; CHECK-LABEL: f14
-; COMMON:  log2
-define double @f14() {
+define double @f14() #0 {
+; X87-LABEL: f14:
+; X87:       # %bb.0: # %entry
+; X87-NEXT:    subl $12, %esp
+; X87-NEXT:    .cfi_def_cfa_offset 16
+; X87-NEXT:    flds {{\.LCPI.*}}
+; X87-NEXT:    fstpl (%esp)
+; X87-NEXT:    calll log2
+; X87-NEXT:    addl $12, %esp
+; X87-NEXT:    .cfi_def_cfa_offset 4
+; X87-NEXT:    retl
+;
+; X86-SSE-LABEL: f14:
+; X86-SSE:       # %bb.0: # %entry
+; X86-SSE-NEXT:    subl $12, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 16
+; X86-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE-NEXT:    movsd %xmm0, (%esp)
+; X86-SSE-NEXT:    calll log2
+; X86-SSE-NEXT:    addl $12, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 4
+; X86-SSE-NEXT:    retl
+;
+; SSE-LABEL: f14:
+; SSE:       # %bb.0: # %entry
+; SSE-NEXT:    pushq %rax
+; SSE-NEXT:    .cfi_def_cfa_offset 16
+; SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE-NEXT:    callq log2
+; SSE-NEXT:    popq %rax
+; SSE-NEXT:    .cfi_def_cfa_offset 8
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: f14:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    pushq %rax
+; AVX-NEXT:    .cfi_def_cfa_offset 16
+; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-NEXT:    callq log2
+; AVX-NEXT:    popq %rax
+; AVX-NEXT:    .cfi_def_cfa_offset 8
+; AVX-NEXT:    retq
 entry:
   %result = call double @llvm.experimental.constrained.log2.f64(double 42.0,
                                                metadata !"round.dynamic",
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
   ret double %result
 }
 
 ; Verify that rint(42.1) isn't simplified when the rounding mode is unknown.
-; CHECK-LABEL: f15
-; NO-FMA:  rint
-; HAS-FMA: vroundsd
-define double @f15() {
+define double @f15() #0 {
+; X87-LABEL: f15:
+; X87:       # %bb.0: # %entry
+; X87-NEXT:    subl $12, %esp
+; X87-NEXT:    .cfi_def_cfa_offset 16
+; X87-NEXT:    fldl {{\.LCPI.*}}
+; X87-NEXT:    fstpl (%esp)
+; X87-NEXT:    calll rint
+; X87-NEXT:    addl $12, %esp
+; X87-NEXT:    .cfi_def_cfa_offset 4
+; X87-NEXT:    retl
+;
+; X86-SSE-LABEL: f15:
+; X86-SSE:       # %bb.0: # %entry
+; X86-SSE-NEXT:    subl $12, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 16
+; X86-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE-NEXT:    movsd %xmm0, (%esp)
+; X86-SSE-NEXT:    calll rint
+; X86-SSE-NEXT:    addl $12, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 4
+; X86-SSE-NEXT:    retl
+;
+; SSE-LABEL: f15:
+; SSE:       # %bb.0: # %entry
+; SSE-NEXT:    pushq %rax
+; SSE-NEXT:    .cfi_def_cfa_offset 16
+; SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE-NEXT:    callq rint
+; SSE-NEXT:    popq %rax
+; SSE-NEXT:    .cfi_def_cfa_offset 8
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: f15:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-NEXT:    vroundsd $4, %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    retq
 entry:
   %result = call double @llvm.experimental.constrained.rint.f64(double 42.1,
                                                metadata !"round.dynamic",
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
   ret double %result
 }
 
 ; Verify that nearbyint(42.1) isn't simplified when the rounding mode is
 ; unknown.
-; CHECK-LABEL: f16
-; NO-FMA:  nearbyint
-; HAS-FMA: vroundsd
-define double @f16() {
+define double @f16() #0 {
+; X87-LABEL: f16:
+; X87:       # %bb.0: # %entry
+; X87-NEXT:    subl $12, %esp
+; X87-NEXT:    .cfi_def_cfa_offset 16
+; X87-NEXT:    fldl {{\.LCPI.*}}
+; X87-NEXT:    fstpl (%esp)
+; X87-NEXT:    calll nearbyint
+; X87-NEXT:    addl $12, %esp
+; X87-NEXT:    .cfi_def_cfa_offset 4
+; X87-NEXT:    retl
+;
+; X86-SSE-LABEL: f16:
+; X86-SSE:       # %bb.0: # %entry
+; X86-SSE-NEXT:    subl $12, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 16
+; X86-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE-NEXT:    movsd %xmm0, (%esp)
+; X86-SSE-NEXT:    calll nearbyint
+; X86-SSE-NEXT:    addl $12, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 4
+; X86-SSE-NEXT:    retl
+;
+; SSE-LABEL: f16:
+; SSE:       # %bb.0: # %entry
+; SSE-NEXT:    pushq %rax
+; SSE-NEXT:    .cfi_def_cfa_offset 16
+; SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE-NEXT:    callq nearbyint
+; SSE-NEXT:    popq %rax
+; SSE-NEXT:    .cfi_def_cfa_offset 8
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: f16:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-NEXT:    vroundsd $12, %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    retq
 entry:
   %result = call double @llvm.experimental.constrained.nearbyint.f64(
                                                double 42.1,
                                                metadata !"round.dynamic",
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
   ret double %result
 }
 
-; Verify that fma(3.5) isn't simplified when the rounding mode is
-; unknown.
-; CHECK-LABEL: f17
-; FMACALL32: jmp fmaf  # TAILCALL
-; FMA32: vfmadd213ss
-define float @f17() {
+define double @f19() #0 {
+; X87-LABEL: f19:
+; X87:       # %bb.0: # %entry
+; X87-NEXT:    subl $28, %esp
+; X87-NEXT:    .cfi_def_cfa_offset 32
+; X87-NEXT:    flds {{\.LCPI.*}}
+; X87-NEXT:    fstpl {{[0-9]+}}(%esp)
+; X87-NEXT:    movl $1072693248, {{[0-9]+}}(%esp) # imm = 0x3FF00000
+; X87-NEXT:    movl $0, (%esp)
+; X87-NEXT:    calll fmod
+; X87-NEXT:    addl $28, %esp
+; X87-NEXT:    .cfi_def_cfa_offset 4
+; X87-NEXT:    retl
+;
+; X86-SSE-LABEL: f19:
+; X86-SSE:       # %bb.0: # %entry
+; X86-SSE-NEXT:    subl $28, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 32
+; X86-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE-NEXT:    movsd %xmm0, {{[0-9]+}}(%esp)
+; X86-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE-NEXT:    movsd %xmm0, (%esp)
+; X86-SSE-NEXT:    calll fmod
+; X86-SSE-NEXT:    addl $28, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 4
+; X86-SSE-NEXT:    retl
+;
+; SSE-LABEL: f19:
+; SSE:       # %bb.0: # %entry
+; SSE-NEXT:    pushq %rax
+; SSE-NEXT:    .cfi_def_cfa_offset 16
+; SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
+; SSE-NEXT:    callq fmod
+; SSE-NEXT:    popq %rax
+; SSE-NEXT:    .cfi_def_cfa_offset 8
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: f19:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    pushq %rax
+; AVX-NEXT:    .cfi_def_cfa_offset 16
+; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
+; AVX-NEXT:    callq fmod
+; AVX-NEXT:    popq %rax
+; AVX-NEXT:    .cfi_def_cfa_offset 8
+; AVX-NEXT:    retq
 entry:
-  %result = call float @llvm.experimental.constrained.fma.f32(
-                                               float 3.5,
-                                               float 3.5,
-                                               float 3.5,
+  %rem = call double @llvm.experimental.constrained.frem.f64(
+                                               double 1.000000e+00,
+                                               double 1.000000e+01,
                                                metadata !"round.dynamic",
-                                               metadata !"fpexcept.strict")
-  ret float %result
+                                               metadata !"fpexcept.strict") #0
+  ret double %rem
 }
 
-; Verify that fma(42.1) isn't simplified when the rounding mode is
+; Verify that fptosi(%x) isn't simplified when the rounding mode is
 ; unknown.
-; CHECK-LABEL: f18
-; FMACALL64: jmp fma  # TAILCALL
-; FMA64: vfmadd213sd
-define double @f18() {
+; Verify that no gross errors happen.
+; FIXME: The SSE/AVX code does not raise an invalid exception for all values
+; that don't fit in i8.
+define i8 @f20s8(double %x) #0 {
+; X87-LABEL: f20s8:
+; X87:       # %bb.0: # %entry
+; X87-NEXT:    subl $8, %esp
+; X87-NEXT:    .cfi_def_cfa_offset 12
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fnstcw {{[0-9]+}}(%esp)
+; X87-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    orl $3072, %eax # imm = 0xC00
+; X87-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; X87-NEXT:    fldcw {{[0-9]+}}(%esp)
+; X87-NEXT:    fistps {{[0-9]+}}(%esp)
+; X87-NEXT:    fldcw {{[0-9]+}}(%esp)
+; X87-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X87-NEXT:    addl $8, %esp
+; X87-NEXT:    .cfi_def_cfa_offset 4
+; X87-NEXT:    retl
+;
+; X86-SSE-LABEL: f20s8:
+; X86-SSE:       # %bb.0: # %entry
+; X86-SSE-NEXT:    cvttsd2si {{[0-9]+}}(%esp), %eax
+; X86-SSE-NEXT:    # kill: def $al killed $al killed $eax
+; X86-SSE-NEXT:    retl
+;
+; SSE-LABEL: f20s8:
+; SSE:       # %bb.0: # %entry
+; SSE-NEXT:    cvttsd2si %xmm0, %eax
+; SSE-NEXT:    # kill: def $al killed $al killed $eax
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: f20s8:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    vcvttsd2si %xmm0, %eax
+; AVX-NEXT:    # kill: def $al killed $al killed $eax
+; AVX-NEXT:    retq
 entry:
-  %result = call double @llvm.experimental.constrained.fma.f64(
-                                               double 42.1,
-                                               double 42.1,
-                                               double 42.1,
-                                               metadata !"round.dynamic",
-                                               metadata !"fpexcept.strict")
-  ret double %result
+  %result = call i8 @llvm.experimental.constrained.fptosi.i8.f64(double %x,
+                                               metadata !"fpexcept.strict") #0
+  ret i8 %result
 }
 
-; CHECK-LABEL: f19
-; COMMON: fmod
-define double @f19() {
+; Verify that fptosi(%x) isn't simplified when the rounding mode is
+; unknown.
+; Verify that no gross errors happen.
+; FIXME: The SSE/AVX code does not raise an invalid exception for all values
+; that don't fit in i16.
+define i16 @f20s16(double %x) #0 {
+; X87-LABEL: f20s16:
+; X87:       # %bb.0: # %entry
+; X87-NEXT:    subl $8, %esp
+; X87-NEXT:    .cfi_def_cfa_offset 12
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fnstcw {{[0-9]+}}(%esp)
+; X87-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    orl $3072, %eax # imm = 0xC00
+; X87-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; X87-NEXT:    fldcw {{[0-9]+}}(%esp)
+; X87-NEXT:    fistps {{[0-9]+}}(%esp)
+; X87-NEXT:    fldcw {{[0-9]+}}(%esp)
+; X87-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    addl $8, %esp
+; X87-NEXT:    .cfi_def_cfa_offset 4
+; X87-NEXT:    retl
+;
+; X86-SSE-LABEL: f20s16:
+; X86-SSE:       # %bb.0: # %entry
+; X86-SSE-NEXT:    cvttsd2si {{[0-9]+}}(%esp), %eax
+; X86-SSE-NEXT:    # kill: def $ax killed $ax killed $eax
+; X86-SSE-NEXT:    retl
+;
+; SSE-LABEL: f20s16:
+; SSE:       # %bb.0: # %entry
+; SSE-NEXT:    cvttsd2si %xmm0, %eax
+; SSE-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: f20s16:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    vcvttsd2si %xmm0, %eax
+; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX-NEXT:    retq
 entry:
-  %rem = call double @llvm.experimental.constrained.frem.f64(
-                                               double 1.000000e+00,
-                                               double 1.000000e+01,
-                                               metadata !"round.dynamic",
-                                               metadata !"fpexcept.strict")
-  ret double %rem
+  %result = call i16 @llvm.experimental.constrained.fptosi.i16.f64(double %x,
+                                               metadata !"fpexcept.strict") #0
+  ret i16 %result
+}
+
+; Verify that fptosi(%x) isn't simplified when the rounding mode is
+; unknown.
+; Verify that no gross errors happen.
+define i32 @f20s(double %x) #0 {
+; X87-LABEL: f20s:
+; X87:       # %bb.0: # %entry
+; X87-NEXT:    subl $8, %esp
+; X87-NEXT:    .cfi_def_cfa_offset 12
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fnstcw (%esp)
+; X87-NEXT:    movzwl (%esp), %eax
+; X87-NEXT:    orl $3072, %eax # imm = 0xC00
+; X87-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; X87-NEXT:    fldcw {{[0-9]+}}(%esp)
+; X87-NEXT:    fistpl {{[0-9]+}}(%esp)
+; X87-NEXT:    fldcw (%esp)
+; X87-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    addl $8, %esp
+; X87-NEXT:    .cfi_def_cfa_offset 4
+; X87-NEXT:    retl
+;
+; X86-SSE-LABEL: f20s:
+; X86-SSE:       # %bb.0: # %entry
+; X86-SSE-NEXT:    cvttsd2si {{[0-9]+}}(%esp), %eax
+; X86-SSE-NEXT:    retl
+;
+; SSE-LABEL: f20s:
+; SSE:       # %bb.0: # %entry
+; SSE-NEXT:    cvttsd2si %xmm0, %eax
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: f20s:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    vcvttsd2si %xmm0, %eax
+; AVX-NEXT:    retq
+entry:
+  %result = call i32 @llvm.experimental.constrained.fptosi.i32.f64(double %x,
+                                               metadata !"fpexcept.strict") #0
+  ret i32 %result
+}
+
+; Verify that fptoui(%x) isn't simplified when the rounding mode is
+; unknown.
+; Verify that no gross errors happen.
+; FIXME: This code generates spurious inexact exceptions.
+define i64 @f20s64(double %x) #0 {
+; X87-LABEL: f20s64:
+; X87:       # %bb.0: # %entry
+; X87-NEXT:    subl $20, %esp
+; X87-NEXT:    .cfi_def_cfa_offset 24
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fnstcw {{[0-9]+}}(%esp)
+; X87-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    orl $3072, %eax # imm = 0xC00
+; X87-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; X87-NEXT:    fldcw {{[0-9]+}}(%esp)
+; X87-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X87-NEXT:    fldcw {{[0-9]+}}(%esp)
+; X87-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X87-NEXT:    addl $20, %esp
+; X87-NEXT:    .cfi_def_cfa_offset 4
+; X87-NEXT:    retl
+;
+; X86-SSE-LABEL: f20s64:
+; X86-SSE:       # %bb.0: # %entry
+; X86-SSE-NEXT:    subl $20, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 24
+; X86-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE-NEXT:    movsd %xmm0, {{[0-9]+}}(%esp)
+; X86-SSE-NEXT:    fldl {{[0-9]+}}(%esp)
+; X86-SSE-NEXT:    fnstcw {{[0-9]+}}(%esp)
+; X86-SSE-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-SSE-NEXT:    orl $3072, %eax # imm = 0xC00
+; X86-SSE-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; X86-SSE-NEXT:    fldcw {{[0-9]+}}(%esp)
+; X86-SSE-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-SSE-NEXT:    fldcw {{[0-9]+}}(%esp)
+; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-SSE-NEXT:    addl $20, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 4
+; X86-SSE-NEXT:    retl
+;
+; SSE-LABEL: f20s64:
+; SSE:       # %bb.0: # %entry
+; SSE-NEXT:    cvttsd2si %xmm0, %rax
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: f20s64:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    vcvttsd2si %xmm0, %rax
+; AVX-NEXT:    retq
+entry:
+  %result = call i64 @llvm.experimental.constrained.fptosi.i64.f64(double %x,
+                                               metadata !"fpexcept.strict") #0
+  ret i64 %result
+}
+
+; Verify that fptoui(%x) isn't simplified when the rounding mode is
+; unknown.
+; Verify that no gross errors happen.
+define i128 @f20s128(double %x) nounwind strictfp {
+; X87-LABEL: f20s128:
+; X87:       # %bb.0: # %entry
+; X87-NEXT:    pushl %edi
+; X87-NEXT:    pushl %esi
+; X87-NEXT:    subl $36, %esp
+; X87-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fstpl {{[0-9]+}}(%esp)
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl %eax, (%esp)
+; X87-NEXT:    calll __fixdfti
+; X87-NEXT:    subl $4, %esp
+; X87-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X87-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X87-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X87-NEXT:    movl %edi, 8(%esi)
+; X87-NEXT:    movl %edx, 12(%esi)
+; X87-NEXT:    movl %eax, (%esi)
+; X87-NEXT:    movl %ecx, 4(%esi)
+; X87-NEXT:    movl %esi, %eax
+; X87-NEXT:    addl $36, %esp
+; X87-NEXT:    popl %esi
+; X87-NEXT:    popl %edi
+; X87-NEXT:    retl $4
+;
+; X86-SSE-LABEL: f20s128:
+; X86-SSE:       # %bb.0: # %entry
+; X86-SSE-NEXT:    pushl %edi
+; X86-SSE-NEXT:    pushl %esi
+; X86-SSE-NEXT:    subl $36, %esp
+; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE-NEXT:    movsd %xmm0, {{[0-9]+}}(%esp)
+; X86-SSE-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-SSE-NEXT:    movl %eax, (%esp)
+; X86-SSE-NEXT:    calll __fixdfti
+; X86-SSE-NEXT:    subl $4, %esp
+; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-SSE-NEXT:    movl %edi, 8(%esi)
+; X86-SSE-NEXT:    movl %edx, 12(%esi)
+; X86-SSE-NEXT:    movl %eax, (%esi)
+; X86-SSE-NEXT:    movl %ecx, 4(%esi)
+; X86-SSE-NEXT:    movl %esi, %eax
+; X86-SSE-NEXT:    addl $36, %esp
+; X86-SSE-NEXT:    popl %esi
+; X86-SSE-NEXT:    popl %edi
+; X86-SSE-NEXT:    retl $4
+;
+; SSE-LABEL: f20s128:
+; SSE:       # %bb.0: # %entry
+; SSE-NEXT:    pushq %rax
+; SSE-NEXT:    callq __fixdfti
+; SSE-NEXT:    popq %rcx
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: f20s128:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    pushq %rax
+; AVX-NEXT:    callq __fixdfti
+; AVX-NEXT:    popq %rcx
+; AVX-NEXT:    retq
+entry:
+  %result = call i128 @llvm.experimental.constrained.fptosi.i128.f64(double %x,
+                                               metadata !"fpexcept.strict") #0
+  ret i128 %result
+}
+
+; Verify that fptoui(%x) isn't simplified when the rounding mode is
+; unknown.
+; Verify that no gross errors happen.
+; FIXME: The SSE/AVX code does not raise an invalid exception for all values
+; that don't fit in i8.
+define i8 @f20u8(double %x) #0 {
+; X87-LABEL: f20u8:
+; X87:       # %bb.0: # %entry
+; X87-NEXT:    subl $8, %esp
+; X87-NEXT:    .cfi_def_cfa_offset 12
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fnstcw {{[0-9]+}}(%esp)
+; X87-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    orl $3072, %eax # imm = 0xC00
+; X87-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; X87-NEXT:    fldcw {{[0-9]+}}(%esp)
+; X87-NEXT:    fistps {{[0-9]+}}(%esp)
+; X87-NEXT:    fldcw {{[0-9]+}}(%esp)
+; X87-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X87-NEXT:    addl $8, %esp
+; X87-NEXT:    .cfi_def_cfa_offset 4
+; X87-NEXT:    retl
+;
+; X86-SSE-LABEL: f20u8:
+; X86-SSE:       # %bb.0: # %entry
+; X86-SSE-NEXT:    cvttsd2si {{[0-9]+}}(%esp), %eax
+; X86-SSE-NEXT:    # kill: def $al killed $al killed $eax
+; X86-SSE-NEXT:    retl
+;
+; SSE-LABEL: f20u8:
+; SSE:       # %bb.0: # %entry
+; SSE-NEXT:    cvttsd2si %xmm0, %eax
+; SSE-NEXT:    # kill: def $al killed $al killed $eax
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: f20u8:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    vcvttsd2si %xmm0, %eax
+; AVX-NEXT:    # kill: def $al killed $al killed $eax
+; AVX-NEXT:    retq
+entry:
+  %result = call i8 @llvm.experimental.constrained.fptoui.i8.f64(double %x,
+                                               metadata !"fpexcept.strict") #0
+  ret i8 %result
+}
+; Verify that fptoui(%x) isn't simplified when the rounding mode is
+; unknown.
+; Verify that no gross errors happen.
+; FIXME: The SSE/AVX code does not raise an invalid exception for all values
+; that don't fit in i16.
+define i16 @f20u16(double %x) #0 {
+; X87-LABEL: f20u16:
+; X87:       # %bb.0: # %entry
+; X87-NEXT:    subl $8, %esp
+; X87-NEXT:    .cfi_def_cfa_offset 12
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fnstcw (%esp)
+; X87-NEXT:    movzwl (%esp), %eax
+; X87-NEXT:    orl $3072, %eax # imm = 0xC00
+; X87-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; X87-NEXT:    fldcw {{[0-9]+}}(%esp)
+; X87-NEXT:    fistpl {{[0-9]+}}(%esp)
+; X87-NEXT:    fldcw (%esp)
+; X87-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    # kill: def $ax killed $ax killed $eax
+; X87-NEXT:    addl $8, %esp
+; X87-NEXT:    .cfi_def_cfa_offset 4
+; X87-NEXT:    retl
+;
+; X86-SSE-LABEL: f20u16:
+; X86-SSE:       # %bb.0: # %entry
+; X86-SSE-NEXT:    cvttsd2si {{[0-9]+}}(%esp), %eax
+; X86-SSE-NEXT:    # kill: def $ax killed $ax killed $eax
+; X86-SSE-NEXT:    retl
+;
+; SSE-LABEL: f20u16:
+; SSE:       # %bb.0: # %entry
+; SSE-NEXT:    cvttsd2si %xmm0, %eax
+; SSE-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: f20u16:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    vcvttsd2si %xmm0, %eax
+; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX-NEXT:    retq
+entry:
+  %result = call i16 @llvm.experimental.constrained.fptoui.i16.f64(double %x,
+                                               metadata !"fpexcept.strict") #0
+  ret i16 %result
+}
+
+; Verify that fptoui(%x) isn't simplified when the rounding mode is
+; unknown.
+; Verify that no gross errors happen.
+; FIXME: The X87/SSE/AVX1 code does not raise an invalid exception for all
+; values that don't fit in i32. The AVX512 code does.
+define i32 @f20u(double %x) #0 {
+; X87-LABEL: f20u:
+; X87:       # %bb.0: # %entry
+; X87-NEXT:    subl $20, %esp
+; X87-NEXT:    .cfi_def_cfa_offset 24
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fnstcw {{[0-9]+}}(%esp)
+; X87-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    orl $3072, %eax # imm = 0xC00
+; X87-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; X87-NEXT:    fldcw {{[0-9]+}}(%esp)
+; X87-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X87-NEXT:    fldcw {{[0-9]+}}(%esp)
+; X87-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    addl $20, %esp
+; X87-NEXT:    .cfi_def_cfa_offset 4
+; X87-NEXT:    retl
+;
+; X86-SSE-LABEL: f20u:
+; X86-SSE:       # %bb.0: # %entry
+; X86-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE-NEXT:    movsd {{.*#+}} xmm2 = mem[0],zero
+; X86-SSE-NEXT:    comisd %xmm0, %xmm2
+; X86-SSE-NEXT:    xorpd %xmm1, %xmm1
+; X86-SSE-NEXT:    ja .LBB24_2
+; X86-SSE-NEXT:  # %bb.1: # %entry
+; X86-SSE-NEXT:    movapd %xmm2, %xmm1
+; X86-SSE-NEXT:  .LBB24_2: # %entry
+; X86-SSE-NEXT:    setbe %al
+; X86-SSE-NEXT:    movzbl %al, %ecx
+; X86-SSE-NEXT:    shll $31, %ecx
+; X86-SSE-NEXT:    subsd %xmm1, %xmm0
+; X86-SSE-NEXT:    cvttsd2si %xmm0, %eax
+; X86-SSE-NEXT:    xorl %ecx, %eax
+; X86-SSE-NEXT:    retl
+;
+; SSE-LABEL: f20u:
+; SSE:       # %bb.0: # %entry
+; SSE-NEXT:    cvttsd2si %xmm0, %rax
+; SSE-NEXT:    # kill: def $eax killed $eax killed $rax
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: f20u:
+; AVX1:       # %bb.0: # %entry
+; AVX1-NEXT:    vcvttsd2si %xmm0, %rax
+; AVX1-NEXT:    # kill: def $eax killed $eax killed $rax
+; AVX1-NEXT:    retq
+;
+; AVX512-LABEL: f20u:
+; AVX512:       # %bb.0: # %entry
+; AVX512-NEXT:    vcvttsd2usi %xmm0, %eax
+; AVX512-NEXT:    retq
+entry:
+  %result = call i32 @llvm.experimental.constrained.fptoui.i32.f64(double %x,
+                                               metadata !"fpexcept.strict") #0
+  ret i32 %result
+}
+
+; Verify that fptoui(%x) isn't simplified when the rounding mode is
+; unknown.
+; Verify that no gross errors happen.
+; FIXME: This code generates spurious inexact exceptions.
+define i64 @f20u64(double %x) #0 {
+; X87-LABEL: f20u64:
+; X87:       # %bb.0: # %entry
+; X87-NEXT:    subl $20, %esp
+; X87-NEXT:    .cfi_def_cfa_offset 24
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    flds {{\.LCPI.*}}
+; X87-NEXT:    xorl %edx, %edx
+; X87-NEXT:    fcomi %st(1), %st
+; X87-NEXT:    setbe %dl
+; X87-NEXT:    fldz
+; X87-NEXT:    fxch %st(1)
+; X87-NEXT:    fcmovnbe %st(1), %st
+; X87-NEXT:    fstp %st(1)
+; X87-NEXT:    fsubrp %st, %st(1)
+; X87-NEXT:    fnstcw {{[0-9]+}}(%esp)
+; X87-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    orl $3072, %eax # imm = 0xC00
+; X87-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; X87-NEXT:    fldcw {{[0-9]+}}(%esp)
+; X87-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X87-NEXT:    fldcw {{[0-9]+}}(%esp)
+; X87-NEXT:    shll $31, %edx
+; X87-NEXT:    xorl {{[0-9]+}}(%esp), %edx
+; X87-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    addl $20, %esp
+; X87-NEXT:    .cfi_def_cfa_offset 4
+; X87-NEXT:    retl
+;
+; X86-SSE-LABEL: f20u64:
+; X86-SSE:       # %bb.0: # %entry
+; X86-SSE-NEXT:    subl $20, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 24
+; X86-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE-NEXT:    movsd {{.*#+}} xmm2 = mem[0],zero
+; X86-SSE-NEXT:    comisd %xmm0, %xmm2
+; X86-SSE-NEXT:    xorpd %xmm1, %xmm1
+; X86-SSE-NEXT:    ja .LBB25_2
+; X86-SSE-NEXT:  # %bb.1: # %entry
+; X86-SSE-NEXT:    movapd %xmm2, %xmm1
+; X86-SSE-NEXT:  .LBB25_2: # %entry
+; X86-SSE-NEXT:    subsd %xmm1, %xmm0
+; X86-SSE-NEXT:    movsd %xmm0, {{[0-9]+}}(%esp)
+; X86-SSE-NEXT:    setbe %al
+; X86-SSE-NEXT:    fldl {{[0-9]+}}(%esp)
+; X86-SSE-NEXT:    fnstcw {{[0-9]+}}(%esp)
+; X86-SSE-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
+; X86-SSE-NEXT:    orl $3072, %ecx # imm = 0xC00
+; X86-SSE-NEXT:    movw %cx, {{[0-9]+}}(%esp)
+; X86-SSE-NEXT:    fldcw {{[0-9]+}}(%esp)
+; X86-SSE-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-SSE-NEXT:    fldcw {{[0-9]+}}(%esp)
+; X86-SSE-NEXT:    movzbl %al, %edx
+; X86-SSE-NEXT:    shll $31, %edx
+; X86-SSE-NEXT:    xorl {{[0-9]+}}(%esp), %edx
+; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE-NEXT:    addl $20, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 4
+; X86-SSE-NEXT:    retl
+;
+; SSE-LABEL: f20u64:
+; SSE:       # %bb.0: # %entry
+; SSE-NEXT:    movsd {{.*#+}} xmm2 = mem[0],zero
+; SSE-NEXT:    comisd %xmm2, %xmm0
+; SSE-NEXT:    xorpd %xmm1, %xmm1
+; SSE-NEXT:    jb .LBB25_2
+; SSE-NEXT:  # %bb.1: # %entry
+; SSE-NEXT:    movapd %xmm2, %xmm1
+; SSE-NEXT:  .LBB25_2: # %entry
+; SSE-NEXT:    subsd %xmm1, %xmm0
+; SSE-NEXT:    cvttsd2si %xmm0, %rcx
+; SSE-NEXT:    setae %al
+; SSE-NEXT:    movzbl %al, %eax
+; SSE-NEXT:    shlq $63, %rax
+; SSE-NEXT:    xorq %rcx, %rax
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: f20u64:
+; AVX1:       # %bb.0: # %entry
+; AVX1-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
+; AVX1-NEXT:    vcomisd %xmm1, %xmm0
+; AVX1-NEXT:    vxorpd %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    jb .LBB25_2
+; AVX1-NEXT:  # %bb.1: # %entry
+; AVX1-NEXT:    vmovapd %xmm1, %xmm2
+; AVX1-NEXT:  .LBB25_2: # %entry
+; AVX1-NEXT:    vsubsd %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vcvttsd2si %xmm0, %rcx
+; AVX1-NEXT:    setae %al
+; AVX1-NEXT:    movzbl %al, %eax
+; AVX1-NEXT:    shlq $63, %rax
+; AVX1-NEXT:    xorq %rcx, %rax
+; AVX1-NEXT:    retq
+;
+; AVX512-LABEL: f20u64:
+; AVX512:       # %bb.0: # %entry
+; AVX512-NEXT:    vcvttsd2usi %xmm0, %rax
+; AVX512-NEXT:    retq
+entry:
+  %result = call i64 @llvm.experimental.constrained.fptoui.i64.f64(double %x,
+                                               metadata !"fpexcept.strict") #0
+  ret i64 %result
+}
+
+
+; Verify that fptoui(%x) isn't simplified when the rounding mode is
+; unknown.
+; Verify that no gross errors happen.
+define i128 @f20u128(double %x) nounwind strictfp {
+; X87-LABEL: f20u128:
+; X87:       # %bb.0: # %entry
+; X87-NEXT:    pushl %edi
+; X87-NEXT:    pushl %esi
+; X87-NEXT:    subl $36, %esp
+; X87-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fstpl {{[0-9]+}}(%esp)
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl %eax, (%esp)
+; X87-NEXT:    calll __fixunsdfti
+; X87-NEXT:    subl $4, %esp
+; X87-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X87-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X87-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X87-NEXT:    movl %edi, 8(%esi)
+; X87-NEXT:    movl %edx, 12(%esi)
+; X87-NEXT:    movl %eax, (%esi)
+; X87-NEXT:    movl %ecx, 4(%esi)
+; X87-NEXT:    movl %esi, %eax
+; X87-NEXT:    addl $36, %esp
+; X87-NEXT:    popl %esi
+; X87-NEXT:    popl %edi
+; X87-NEXT:    retl $4
+;
+; X86-SSE-LABEL: f20u128:
+; X86-SSE:       # %bb.0: # %entry
+; X86-SSE-NEXT:    pushl %edi
+; X86-SSE-NEXT:    pushl %esi
+; X86-SSE-NEXT:    subl $36, %esp
+; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE-NEXT:    movsd %xmm0, {{[0-9]+}}(%esp)
+; X86-SSE-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-SSE-NEXT:    movl %eax, (%esp)
+; X86-SSE-NEXT:    calll __fixunsdfti
+; X86-SSE-NEXT:    subl $4, %esp
+; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-SSE-NEXT:    movl %edi, 8(%esi)
+; X86-SSE-NEXT:    movl %edx, 12(%esi)
+; X86-SSE-NEXT:    movl %eax, (%esi)
+; X86-SSE-NEXT:    movl %ecx, 4(%esi)
+; X86-SSE-NEXT:    movl %esi, %eax
+; X86-SSE-NEXT:    addl $36, %esp
+; X86-SSE-NEXT:    popl %esi
+; X86-SSE-NEXT:    popl %edi
+; X86-SSE-NEXT:    retl $4
+;
+; SSE-LABEL: f20u128:
+; SSE:       # %bb.0: # %entry
+; SSE-NEXT:    pushq %rax
+; SSE-NEXT:    callq __fixunsdfti
+; SSE-NEXT:    popq %rcx
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: f20u128:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    pushq %rax
+; AVX-NEXT:    callq __fixunsdfti
+; AVX-NEXT:    popq %rcx
+; AVX-NEXT:    retq
+entry:
+  %result = call i128 @llvm.experimental.constrained.fptoui.i128.f64(double %x,
+                                               metadata !"fpexcept.strict") #0
+  ret i128 %result
 }
 
 ; Verify that round(42.1) isn't simplified when the rounding mode is
 ; unknown.
 ; Verify that no gross errors happen.
-; CHECK-LABEL: @f21
-; COMMON: cvtsd2ss
-define float @f21() {
+define float @f21() #0 {
+; X87-LABEL: f21:
+; X87:       # %bb.0: # %entry
+; X87-NEXT:    pushl %eax
+; X87-NEXT:    .cfi_def_cfa_offset 8
+; X87-NEXT:    fldl {{\.LCPI.*}}
+; X87-NEXT:    fstps (%esp)
+; X87-NEXT:    flds (%esp)
+; X87-NEXT:    popl %eax
+; X87-NEXT:    .cfi_def_cfa_offset 4
+; X87-NEXT:    retl
+;
+; X86-SSE-LABEL: f21:
+; X86-SSE:       # %bb.0: # %entry
+; X86-SSE-NEXT:    pushl %eax
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 8
+; X86-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE-NEXT:    cvtsd2ss %xmm0, %xmm0
+; X86-SSE-NEXT:    movss %xmm0, (%esp)
+; X86-SSE-NEXT:    flds (%esp)
+; X86-SSE-NEXT:    popl %eax
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 4
+; X86-SSE-NEXT:    retl
+;
+; SSE-LABEL: f21:
+; SSE:       # %bb.0: # %entry
+; SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE-NEXT:    cvtsd2ss %xmm0, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: f21:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-NEXT:    vcvtsd2ss %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    retq
 entry:
   %result = call float @llvm.experimental.constrained.fptrunc.f32.f64(
                                                double 42.1,
                                                metadata !"round.dynamic",
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
   ret float %result
 }
 
-; CHECK-LABEL: @f22
-; COMMON: cvtss2sd
-define double @f22(float %x) {
+define double @f22(float %x) #0 {
+; X87-LABEL: f22:
+; X87:       # %bb.0: # %entry
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    retl
+;
+; X86-SSE-LABEL: f22:
+; X86-SSE:       # %bb.0: # %entry
+; X86-SSE-NEXT:    subl $12, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 16
+; X86-SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-SSE-NEXT:    cvtss2sd %xmm0, %xmm0
+; X86-SSE-NEXT:    movsd %xmm0, (%esp)
+; X86-SSE-NEXT:    fldl (%esp)
+; X86-SSE-NEXT:    addl $12, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 4
+; X86-SSE-NEXT:    retl
+;
+; SSE-LABEL: f22:
+; SSE:       # %bb.0: # %entry
+; SSE-NEXT:    cvtss2sd %xmm0, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: f22:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    retq
 entry:
   %result = call double @llvm.experimental.constrained.fpext.f64.f32(float %x,
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
+  ret double %result
+}
+
+define i32 @f23(double %x) #0 {
+; X87-LABEL: f23:
+; X87:       # %bb.0: # %entry
+; X87-NEXT:    subl $12, %esp
+; X87-NEXT:    .cfi_def_cfa_offset 16
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fstpl (%esp)
+; X87-NEXT:    calll lrint
+; X87-NEXT:    addl $12, %esp
+; X87-NEXT:    .cfi_def_cfa_offset 4
+; X87-NEXT:    retl
+;
+; X86-SSE-LABEL: f23:
+; X86-SSE:       # %bb.0: # %entry
+; X86-SSE-NEXT:    subl $12, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 16
+; X86-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE-NEXT:    movsd %xmm0, (%esp)
+; X86-SSE-NEXT:    calll lrint
+; X86-SSE-NEXT:    addl $12, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 4
+; X86-SSE-NEXT:    retl
+;
+; SSE-LABEL: f23:
+; SSE:       # %bb.0: # %entry
+; SSE-NEXT:    pushq %rax
+; SSE-NEXT:    .cfi_def_cfa_offset 16
+; SSE-NEXT:    callq lrint
+; SSE-NEXT:    popq %rcx
+; SSE-NEXT:    .cfi_def_cfa_offset 8
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: f23:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    pushq %rax
+; AVX-NEXT:    .cfi_def_cfa_offset 16
+; AVX-NEXT:    callq lrint
+; AVX-NEXT:    popq %rcx
+; AVX-NEXT:    .cfi_def_cfa_offset 8
+; AVX-NEXT:    retq
+entry:
+  %result = call i32 @llvm.experimental.constrained.lrint.i32.f64(double %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret i32 %result
+}
+
+define i32 @f24(float %x) #0 {
+; X87-LABEL: f24:
+; X87:       # %bb.0: # %entry
+; X87-NEXT:    subl $12, %esp
+; X87-NEXT:    .cfi_def_cfa_offset 16
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    fstps (%esp)
+; X87-NEXT:    calll lrintf
+; X87-NEXT:    addl $12, %esp
+; X87-NEXT:    .cfi_def_cfa_offset 4
+; X87-NEXT:    retl
+;
+; X86-SSE-LABEL: f24:
+; X86-SSE:       # %bb.0: # %entry
+; X86-SSE-NEXT:    subl $12, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 16
+; X86-SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-SSE-NEXT:    movss %xmm0, (%esp)
+; X86-SSE-NEXT:    calll lrintf
+; X86-SSE-NEXT:    addl $12, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 4
+; X86-SSE-NEXT:    retl
+;
+; SSE-LABEL: f24:
+; SSE:       # %bb.0: # %entry
+; SSE-NEXT:    pushq %rax
+; SSE-NEXT:    .cfi_def_cfa_offset 16
+; SSE-NEXT:    callq lrintf
+; SSE-NEXT:    popq %rcx
+; SSE-NEXT:    .cfi_def_cfa_offset 8
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: f24:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    pushq %rax
+; AVX-NEXT:    .cfi_def_cfa_offset 16
+; AVX-NEXT:    callq lrintf
+; AVX-NEXT:    popq %rcx
+; AVX-NEXT:    .cfi_def_cfa_offset 8
+; AVX-NEXT:    retq
+entry:
+  %result = call i32 @llvm.experimental.constrained.lrint.i32.f32(float %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret i32 %result
+}
+
+define i64 @f25(double %x) #0 {
+; X87-LABEL: f25:
+; X87:       # %bb.0: # %entry
+; X87-NEXT:    subl $12, %esp
+; X87-NEXT:    .cfi_def_cfa_offset 16
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fstpl (%esp)
+; X87-NEXT:    calll llrint
+; X87-NEXT:    addl $12, %esp
+; X87-NEXT:    .cfi_def_cfa_offset 4
+; X87-NEXT:    retl
+;
+; X86-SSE-LABEL: f25:
+; X86-SSE:       # %bb.0: # %entry
+; X86-SSE-NEXT:    subl $12, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 16
+; X86-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE-NEXT:    movsd %xmm0, (%esp)
+; X86-SSE-NEXT:    calll llrint
+; X86-SSE-NEXT:    addl $12, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 4
+; X86-SSE-NEXT:    retl
+;
+; SSE-LABEL: f25:
+; SSE:       # %bb.0: # %entry
+; SSE-NEXT:    pushq %rax
+; SSE-NEXT:    .cfi_def_cfa_offset 16
+; SSE-NEXT:    callq llrint
+; SSE-NEXT:    popq %rcx
+; SSE-NEXT:    .cfi_def_cfa_offset 8
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: f25:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    pushq %rax
+; AVX-NEXT:    .cfi_def_cfa_offset 16
+; AVX-NEXT:    callq llrint
+; AVX-NEXT:    popq %rcx
+; AVX-NEXT:    .cfi_def_cfa_offset 8
+; AVX-NEXT:    retq
+entry:
+  %result = call i64 @llvm.experimental.constrained.llrint.i64.f64(double %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret i64 %result
+}
+
+define i64 @f26(float %x) {
+; X87-LABEL: f26:
+; X87:       # %bb.0: # %entry
+; X87-NEXT:    subl $12, %esp
+; X87-NEXT:    .cfi_def_cfa_offset 16
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    fstps (%esp)
+; X87-NEXT:    calll llrintf
+; X87-NEXT:    addl $12, %esp
+; X87-NEXT:    .cfi_def_cfa_offset 4
+; X87-NEXT:    retl
+;
+; X86-SSE-LABEL: f26:
+; X86-SSE:       # %bb.0: # %entry
+; X86-SSE-NEXT:    subl $12, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 16
+; X86-SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-SSE-NEXT:    movss %xmm0, (%esp)
+; X86-SSE-NEXT:    calll llrintf
+; X86-SSE-NEXT:    addl $12, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 4
+; X86-SSE-NEXT:    retl
+;
+; SSE-LABEL: f26:
+; SSE:       # %bb.0: # %entry
+; SSE-NEXT:    pushq %rax
+; SSE-NEXT:    .cfi_def_cfa_offset 16
+; SSE-NEXT:    callq llrintf
+; SSE-NEXT:    popq %rcx
+; SSE-NEXT:    .cfi_def_cfa_offset 8
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: f26:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    pushq %rax
+; AVX-NEXT:    .cfi_def_cfa_offset 16
+; AVX-NEXT:    callq llrintf
+; AVX-NEXT:    popq %rcx
+; AVX-NEXT:    .cfi_def_cfa_offset 8
+; AVX-NEXT:    retq
+entry:
+  %result = call i64 @llvm.experimental.constrained.llrint.i64.f32(float %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret i64 %result
+}
+
+define i32 @f27(double %x) #0 {
+; X87-LABEL: f27:
+; X87:       # %bb.0: # %entry
+; X87-NEXT:    subl $12, %esp
+; X87-NEXT:    .cfi_def_cfa_offset 16
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fstpl (%esp)
+; X87-NEXT:    calll lround
+; X87-NEXT:    addl $12, %esp
+; X87-NEXT:    .cfi_def_cfa_offset 4
+; X87-NEXT:    retl
+;
+; X86-SSE-LABEL: f27:
+; X86-SSE:       # %bb.0: # %entry
+; X86-SSE-NEXT:    subl $12, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 16
+; X86-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE-NEXT:    movsd %xmm0, (%esp)
+; X86-SSE-NEXT:    calll lround
+; X86-SSE-NEXT:    addl $12, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 4
+; X86-SSE-NEXT:    retl
+;
+; SSE-LABEL: f27:
+; SSE:       # %bb.0: # %entry
+; SSE-NEXT:    pushq %rax
+; SSE-NEXT:    .cfi_def_cfa_offset 16
+; SSE-NEXT:    callq lround
+; SSE-NEXT:    popq %rcx
+; SSE-NEXT:    .cfi_def_cfa_offset 8
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: f27:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    pushq %rax
+; AVX-NEXT:    .cfi_def_cfa_offset 16
+; AVX-NEXT:    callq lround
+; AVX-NEXT:    popq %rcx
+; AVX-NEXT:    .cfi_def_cfa_offset 8
+; AVX-NEXT:    retq
+entry:
+  %result = call i32 @llvm.experimental.constrained.lround.i32.f64(double %x,
+                                               metadata !"fpexcept.strict") #0
+  ret i32 %result
+}
+
+define i32 @f28(float %x) #0 {
+; X87-LABEL: f28:
+; X87:       # %bb.0: # %entry
+; X87-NEXT:    subl $12, %esp
+; X87-NEXT:    .cfi_def_cfa_offset 16
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    fstps (%esp)
+; X87-NEXT:    calll lroundf
+; X87-NEXT:    addl $12, %esp
+; X87-NEXT:    .cfi_def_cfa_offset 4
+; X87-NEXT:    retl
+;
+; X86-SSE-LABEL: f28:
+; X86-SSE:       # %bb.0: # %entry
+; X86-SSE-NEXT:    subl $12, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 16
+; X86-SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-SSE-NEXT:    movss %xmm0, (%esp)
+; X86-SSE-NEXT:    calll lroundf
+; X86-SSE-NEXT:    addl $12, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 4
+; X86-SSE-NEXT:    retl
+;
+; SSE-LABEL: f28:
+; SSE:       # %bb.0: # %entry
+; SSE-NEXT:    pushq %rax
+; SSE-NEXT:    .cfi_def_cfa_offset 16
+; SSE-NEXT:    callq lroundf
+; SSE-NEXT:    popq %rcx
+; SSE-NEXT:    .cfi_def_cfa_offset 8
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: f28:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    pushq %rax
+; AVX-NEXT:    .cfi_def_cfa_offset 16
+; AVX-NEXT:    callq lroundf
+; AVX-NEXT:    popq %rcx
+; AVX-NEXT:    .cfi_def_cfa_offset 8
+; AVX-NEXT:    retq
+entry:
+  %result = call i32 @llvm.experimental.constrained.lround.i32.f32(float %x,
+                                               metadata !"fpexcept.strict") #0
+  ret i32 %result
+}
+
+define i64 @f29(double %x) #0 {
+; X87-LABEL: f29:
+; X87:       # %bb.0: # %entry
+; X87-NEXT:    subl $12, %esp
+; X87-NEXT:    .cfi_def_cfa_offset 16
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fstpl (%esp)
+; X87-NEXT:    calll llround
+; X87-NEXT:    addl $12, %esp
+; X87-NEXT:    .cfi_def_cfa_offset 4
+; X87-NEXT:    retl
+;
+; X86-SSE-LABEL: f29:
+; X86-SSE:       # %bb.0: # %entry
+; X86-SSE-NEXT:    subl $12, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 16
+; X86-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE-NEXT:    movsd %xmm0, (%esp)
+; X86-SSE-NEXT:    calll llround
+; X86-SSE-NEXT:    addl $12, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 4
+; X86-SSE-NEXT:    retl
+;
+; SSE-LABEL: f29:
+; SSE:       # %bb.0: # %entry
+; SSE-NEXT:    pushq %rax
+; SSE-NEXT:    .cfi_def_cfa_offset 16
+; SSE-NEXT:    callq llround
+; SSE-NEXT:    popq %rcx
+; SSE-NEXT:    .cfi_def_cfa_offset 8
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: f29:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    pushq %rax
+; AVX-NEXT:    .cfi_def_cfa_offset 16
+; AVX-NEXT:    callq llround
+; AVX-NEXT:    popq %rcx
+; AVX-NEXT:    .cfi_def_cfa_offset 8
+; AVX-NEXT:    retq
+entry:
+  %result = call i64 @llvm.experimental.constrained.llround.i64.f64(double %x,
+                                               metadata !"fpexcept.strict") #0
+  ret i64 %result
+}
+
+define i64 @f30(float %x) #0 {
+; X87-LABEL: f30:
+; X87:       # %bb.0: # %entry
+; X87-NEXT:    subl $12, %esp
+; X87-NEXT:    .cfi_def_cfa_offset 16
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    fstps (%esp)
+; X87-NEXT:    calll llroundf
+; X87-NEXT:    addl $12, %esp
+; X87-NEXT:    .cfi_def_cfa_offset 4
+; X87-NEXT:    retl
+;
+; X86-SSE-LABEL: f30:
+; X86-SSE:       # %bb.0: # %entry
+; X86-SSE-NEXT:    subl $12, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 16
+; X86-SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-SSE-NEXT:    movss %xmm0, (%esp)
+; X86-SSE-NEXT:    calll llroundf
+; X86-SSE-NEXT:    addl $12, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 4
+; X86-SSE-NEXT:    retl
+;
+; SSE-LABEL: f30:
+; SSE:       # %bb.0: # %entry
+; SSE-NEXT:    pushq %rax
+; SSE-NEXT:    .cfi_def_cfa_offset 16
+; SSE-NEXT:    callq llroundf
+; SSE-NEXT:    popq %rcx
+; SSE-NEXT:    .cfi_def_cfa_offset 8
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: f30:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    pushq %rax
+; AVX-NEXT:    .cfi_def_cfa_offset 16
+; AVX-NEXT:    callq llroundf
+; AVX-NEXT:    popq %rcx
+; AVX-NEXT:    .cfi_def_cfa_offset 8
+; AVX-NEXT:    retq
+entry:
+  %result = call i64 @llvm.experimental.constrained.llround.i64.f32(float %x,
+                                               metadata !"fpexcept.strict") #0
+  ret i64 %result
+}
+
+; Verify that sitofp(%x) isn't simplified when the rounding mode is
+; unknown.
+; Verify that no gross errors happen.
+define double @sifdb(i8 %x) #0 {
+; X87-LABEL: sifdb:
+; X87:       # %bb.0: # %entry
+; X87-NEXT:    pushl %eax
+; X87-NEXT:    .cfi_def_cfa_offset 8
+; X87-NEXT:    movsbl {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; X87-NEXT:    filds {{[0-9]+}}(%esp)
+; X87-NEXT:    popl %eax
+; X87-NEXT:    .cfi_def_cfa_offset 4
+; X87-NEXT:    retl
+;
+; X86-SSE-LABEL: sifdb:
+; X86-SSE:       # %bb.0: # %entry
+; X86-SSE-NEXT:    subl $12, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 16
+; X86-SSE-NEXT:    movsbl {{[0-9]+}}(%esp), %eax
+; X86-SSE-NEXT:    cvtsi2sd %eax, %xmm0
+; X86-SSE-NEXT:    movsd %xmm0, (%esp)
+; X86-SSE-NEXT:    fldl (%esp)
+; X86-SSE-NEXT:    addl $12, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 4
+; X86-SSE-NEXT:    retl
+;
+; SSE-LABEL: sifdb:
+; SSE:       # %bb.0: # %entry
+; SSE-NEXT:    movsbl %dil, %eax
+; SSE-NEXT:    cvtsi2sd %eax, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: sifdb:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    movsbl %dil, %eax
+; AVX-NEXT:    vcvtsi2sd %eax, %xmm0, %xmm0
+; AVX-NEXT:    retq
+entry:
+  %result = call double @llvm.experimental.constrained.sitofp.f64.i8(i8 %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
   ret double %result
 }
 
+define double @sifdw(i16 %x) #0 {
+; X87-LABEL: sifdw:
+; X87:       # %bb.0: # %entry
+; X87-NEXT:    pushl %eax
+; X87-NEXT:    .cfi_def_cfa_offset 8
+; X87-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; X87-NEXT:    filds {{[0-9]+}}(%esp)
+; X87-NEXT:    popl %eax
+; X87-NEXT:    .cfi_def_cfa_offset 4
+; X87-NEXT:    retl
+;
+; X86-SSE-LABEL: sifdw:
+; X86-SSE:       # %bb.0: # %entry
+; X86-SSE-NEXT:    subl $12, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 16
+; X86-SSE-NEXT:    movswl {{[0-9]+}}(%esp), %eax
+; X86-SSE-NEXT:    cvtsi2sd %eax, %xmm0
+; X86-SSE-NEXT:    movsd %xmm0, (%esp)
+; X86-SSE-NEXT:    fldl (%esp)
+; X86-SSE-NEXT:    addl $12, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 4
+; X86-SSE-NEXT:    retl
+;
+; SSE-LABEL: sifdw:
+; SSE:       # %bb.0: # %entry
+; SSE-NEXT:    movswl %di, %eax
+; SSE-NEXT:    cvtsi2sd %eax, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: sifdw:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    movswl %di, %eax
+; AVX-NEXT:    vcvtsi2sd %eax, %xmm0, %xmm0
+; AVX-NEXT:    retq
+entry:
+  %result = call double @llvm.experimental.constrained.sitofp.f64.i16(i16 %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret double %result
+}
+
+define double @sifdi(i32 %x) #0 {
+; X87-LABEL: sifdi:
+; X87:       # %bb.0: # %entry
+; X87-NEXT:    pushl %eax
+; X87-NEXT:    .cfi_def_cfa_offset 8
+; X87-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl %eax, (%esp)
+; X87-NEXT:    fildl (%esp)
+; X87-NEXT:    popl %eax
+; X87-NEXT:    .cfi_def_cfa_offset 4
+; X87-NEXT:    retl
+;
+; X86-SSE-LABEL: sifdi:
+; X86-SSE:       # %bb.0: # %entry
+; X86-SSE-NEXT:    subl $12, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 16
+; X86-SSE-NEXT:    cvtsi2sdl {{[0-9]+}}(%esp), %xmm0
+; X86-SSE-NEXT:    movsd %xmm0, (%esp)
+; X86-SSE-NEXT:    fldl (%esp)
+; X86-SSE-NEXT:    addl $12, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 4
+; X86-SSE-NEXT:    retl
+;
+; SSE-LABEL: sifdi:
+; SSE:       # %bb.0: # %entry
+; SSE-NEXT:    cvtsi2sd %edi, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: sifdi:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    vcvtsi2sd %edi, %xmm0, %xmm0
+; AVX-NEXT:    retq
+entry:
+  %result = call double @llvm.experimental.constrained.sitofp.f64.i32(i32 %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret double %result
+}
+
+define float @siffb(i8 %x) #0 {
+; X87-LABEL: siffb:
+; X87:       # %bb.0: # %entry
+; X87-NEXT:    pushl %eax
+; X87-NEXT:    .cfi_def_cfa_offset 8
+; X87-NEXT:    movsbl {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; X87-NEXT:    filds {{[0-9]+}}(%esp)
+; X87-NEXT:    popl %eax
+; X87-NEXT:    .cfi_def_cfa_offset 4
+; X87-NEXT:    retl
+;
+; X86-SSE-LABEL: siffb:
+; X86-SSE:       # %bb.0: # %entry
+; X86-SSE-NEXT:    pushl %eax
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 8
+; X86-SSE-NEXT:    movsbl {{[0-9]+}}(%esp), %eax
+; X86-SSE-NEXT:    cvtsi2ss %eax, %xmm0
+; X86-SSE-NEXT:    movss %xmm0, (%esp)
+; X86-SSE-NEXT:    flds (%esp)
+; X86-SSE-NEXT:    popl %eax
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 4
+; X86-SSE-NEXT:    retl
+;
+; SSE-LABEL: siffb:
+; SSE:       # %bb.0: # %entry
+; SSE-NEXT:    movsbl %dil, %eax
+; SSE-NEXT:    cvtsi2ss %eax, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: siffb:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    movsbl %dil, %eax
+; AVX-NEXT:    vcvtsi2ss %eax, %xmm0, %xmm0
+; AVX-NEXT:    retq
+entry:
+  %result = call float @llvm.experimental.constrained.sitofp.f32.i8(i8 %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret float %result
+}
+
+define float @siffw(i16 %x) #0 {
+; X87-LABEL: siffw:
+; X87:       # %bb.0: # %entry
+; X87-NEXT:    pushl %eax
+; X87-NEXT:    .cfi_def_cfa_offset 8
+; X87-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; X87-NEXT:    filds {{[0-9]+}}(%esp)
+; X87-NEXT:    popl %eax
+; X87-NEXT:    .cfi_def_cfa_offset 4
+; X87-NEXT:    retl
+;
+; X86-SSE-LABEL: siffw:
+; X86-SSE:       # %bb.0: # %entry
+; X86-SSE-NEXT:    pushl %eax
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 8
+; X86-SSE-NEXT:    movswl {{[0-9]+}}(%esp), %eax
+; X86-SSE-NEXT:    cvtsi2ss %eax, %xmm0
+; X86-SSE-NEXT:    movss %xmm0, (%esp)
+; X86-SSE-NEXT:    flds (%esp)
+; X86-SSE-NEXT:    popl %eax
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 4
+; X86-SSE-NEXT:    retl
+;
+; SSE-LABEL: siffw:
+; SSE:       # %bb.0: # %entry
+; SSE-NEXT:    movswl %di, %eax
+; SSE-NEXT:    cvtsi2ss %eax, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: siffw:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    movswl %di, %eax
+; AVX-NEXT:    vcvtsi2ss %eax, %xmm0, %xmm0
+; AVX-NEXT:    retq
+entry:
+  %result = call float @llvm.experimental.constrained.sitofp.f32.i16(i16 %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret float %result
+}
+
+define float @siffi(i32 %x) #0 {
+; X87-LABEL: siffi:
+; X87:       # %bb.0: # %entry
+; X87-NEXT:    pushl %eax
+; X87-NEXT:    .cfi_def_cfa_offset 8
+; X87-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl %eax, (%esp)
+; X87-NEXT:    fildl (%esp)
+; X87-NEXT:    popl %eax
+; X87-NEXT:    .cfi_def_cfa_offset 4
+; X87-NEXT:    retl
+;
+; X86-SSE-LABEL: siffi:
+; X86-SSE:       # %bb.0: # %entry
+; X86-SSE-NEXT:    pushl %eax
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 8
+; X86-SSE-NEXT:    cvtsi2ssl {{[0-9]+}}(%esp), %xmm0
+; X86-SSE-NEXT:    movss %xmm0, (%esp)
+; X86-SSE-NEXT:    flds (%esp)
+; X86-SSE-NEXT:    popl %eax
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 4
+; X86-SSE-NEXT:    retl
+;
+; SSE-LABEL: siffi:
+; SSE:       # %bb.0: # %entry
+; SSE-NEXT:    cvtsi2ss %edi, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: siffi:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    vcvtsi2ss %edi, %xmm0, %xmm0
+; AVX-NEXT:    retq
+entry:
+  %result = call float @llvm.experimental.constrained.sitofp.f32.i32(i32 %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret float %result
+}
+
+define double @sifdl(i64 %x) #0 {
+; X87-LABEL: sifdl:
+; X87:       # %bb.0: # %entry
+; X87-NEXT:    fildll {{[0-9]+}}(%esp)
+; X87-NEXT:    retl
+;
+; X86-SSE-LABEL: sifdl:
+; X86-SSE:       # %bb.0: # %entry
+; X86-SSE-NEXT:    subl $12, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 16
+; X86-SSE-NEXT:    fildll {{[0-9]+}}(%esp)
+; X86-SSE-NEXT:    fstpl (%esp)
+; X86-SSE-NEXT:    fldl (%esp)
+; X86-SSE-NEXT:    addl $12, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 4
+; X86-SSE-NEXT:    retl
+;
+; SSE-LABEL: sifdl:
+; SSE:       # %bb.0: # %entry
+; SSE-NEXT:    cvtsi2sd %rdi, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: sifdl:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    vcvtsi2sd %rdi, %xmm0, %xmm0
+; AVX-NEXT:    retq
+entry:
+  %result = call double @llvm.experimental.constrained.sitofp.f64.i64(i64 %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret double %result
+}
+
+define float @siffl(i64 %x) #0 {
+; X87-LABEL: siffl:
+; X87:       # %bb.0: # %entry
+; X87-NEXT:    fildll {{[0-9]+}}(%esp)
+; X87-NEXT:    retl
+;
+; X86-SSE-LABEL: siffl:
+; X86-SSE:       # %bb.0: # %entry
+; X86-SSE-NEXT:    pushl %eax
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 8
+; X86-SSE-NEXT:    fildll {{[0-9]+}}(%esp)
+; X86-SSE-NEXT:    fstps (%esp)
+; X86-SSE-NEXT:    flds (%esp)
+; X86-SSE-NEXT:    popl %eax
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 4
+; X86-SSE-NEXT:    retl
+;
+; SSE-LABEL: siffl:
+; SSE:       # %bb.0: # %entry
+; SSE-NEXT:    cvtsi2ss %rdi, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: siffl:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    vcvtsi2ss %rdi, %xmm0, %xmm0
+; AVX-NEXT:    retq
+entry:
+  %result = call float @llvm.experimental.constrained.sitofp.f32.i64(i64 %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret float %result
+}
+
+; Verify that uitofp(%x) isn't simplified when the rounding mode is
+; unknown.
+; Verify that no gross errors happen.
+define double @uifdb(i8 %x) #0 {
+; X87-LABEL: uifdb:
+; X87:       # %bb.0: # %entry
+; X87-NEXT:    pushl %eax
+; X87-NEXT:    .cfi_def_cfa_offset 8
+; X87-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; X87-NEXT:    filds {{[0-9]+}}(%esp)
+; X87-NEXT:    popl %eax
+; X87-NEXT:    .cfi_def_cfa_offset 4
+; X87-NEXT:    retl
+;
+; X86-SSE-LABEL: uifdb:
+; X86-SSE:       # %bb.0: # %entry
+; X86-SSE-NEXT:    subl $12, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 16
+; X86-SSE-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE-NEXT:    cvtsi2sd %eax, %xmm0
+; X86-SSE-NEXT:    movsd %xmm0, (%esp)
+; X86-SSE-NEXT:    fldl (%esp)
+; X86-SSE-NEXT:    addl $12, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 4
+; X86-SSE-NEXT:    retl
+;
+; SSE-LABEL: uifdb:
+; SSE:       # %bb.0: # %entry
+; SSE-NEXT:    movzbl %dil, %eax
+; SSE-NEXT:    cvtsi2sd %eax, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: uifdb:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    movzbl %dil, %eax
+; AVX-NEXT:    vcvtsi2sd %eax, %xmm0, %xmm0
+; AVX-NEXT:    retq
+entry:
+  %result = call double @llvm.experimental.constrained.uitofp.f64.i8(i8 %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret double %result
+}
+
+define double @uifdw(i16 %x) #0 {
+; X87-LABEL: uifdw:
+; X87:       # %bb.0: # %entry
+; X87-NEXT:    pushl %eax
+; X87-NEXT:    .cfi_def_cfa_offset 8
+; X87-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl %eax, (%esp)
+; X87-NEXT:    fildl (%esp)
+; X87-NEXT:    popl %eax
+; X87-NEXT:    .cfi_def_cfa_offset 4
+; X87-NEXT:    retl
+;
+; X86-SSE-LABEL: uifdw:
+; X86-SSE:       # %bb.0: # %entry
+; X86-SSE-NEXT:    subl $12, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 16
+; X86-SSE-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-SSE-NEXT:    cvtsi2sd %eax, %xmm0
+; X86-SSE-NEXT:    movsd %xmm0, (%esp)
+; X86-SSE-NEXT:    fldl (%esp)
+; X86-SSE-NEXT:    addl $12, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 4
+; X86-SSE-NEXT:    retl
+;
+; SSE-LABEL: uifdw:
+; SSE:       # %bb.0: # %entry
+; SSE-NEXT:    movzwl %di, %eax
+; SSE-NEXT:    cvtsi2sd %eax, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: uifdw:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    movzwl %di, %eax
+; AVX-NEXT:    vcvtsi2sd %eax, %xmm0, %xmm0
+; AVX-NEXT:    retq
+entry:
+  %result = call double @llvm.experimental.constrained.uitofp.f64.i16(i16 %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret double %result
+}
+
+define double @uifdi(i32 %x) #0 {
+; X87-LABEL: uifdi:
+; X87:       # %bb.0: # %entry
+; X87-NEXT:    subl $12, %esp
+; X87-NEXT:    .cfi_def_cfa_offset 16
+; X87-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl %eax, (%esp)
+; X87-NEXT:    movl $0, {{[0-9]+}}(%esp)
+; X87-NEXT:    fildll (%esp)
+; X87-NEXT:    addl $12, %esp
+; X87-NEXT:    .cfi_def_cfa_offset 4
+; X87-NEXT:    retl
+;
+; X86-SSE-LABEL: uifdi:
+; X86-SSE:       # %bb.0: # %entry
+; X86-SSE-NEXT:    subl $12, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 16
+; X86-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; X86-SSE-NEXT:    orpd %xmm0, %xmm1
+; X86-SSE-NEXT:    subsd %xmm0, %xmm1
+; X86-SSE-NEXT:    movsd %xmm1, (%esp)
+; X86-SSE-NEXT:    fldl (%esp)
+; X86-SSE-NEXT:    addl $12, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 4
+; X86-SSE-NEXT:    retl
+;
+; SSE-LABEL: uifdi:
+; SSE:       # %bb.0: # %entry
+; SSE-NEXT:    movl %edi, %eax
+; SSE-NEXT:    cvtsi2sd %rax, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: uifdi:
+; AVX1:       # %bb.0: # %entry
+; AVX1-NEXT:    movl %edi, %eax
+; AVX1-NEXT:    vcvtsi2sd %rax, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX512-LABEL: uifdi:
+; AVX512:       # %bb.0: # %entry
+; AVX512-NEXT:    vcvtusi2sd %edi, %xmm0, %xmm0
+; AVX512-NEXT:    retq
+entry:
+  %result = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret double %result
+}
+
+define double @uifdl(i64 %x) #0 {
+; X87-LABEL: uifdl:
+; X87:       # %bb.0: # %entry
+; X87-NEXT:    subl $20, %esp
+; X87-NEXT:    .cfi_def_cfa_offset 24
+; X87-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X87-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X87-NEXT:    movl %eax, (%esp)
+; X87-NEXT:    shrl $31, %ecx
+; X87-NEXT:    fildll (%esp)
+; X87-NEXT:    fadds {{\.LCPI.*}}(,%ecx,4)
+; X87-NEXT:    fstpl {{[0-9]+}}(%esp)
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    addl $20, %esp
+; X87-NEXT:    .cfi_def_cfa_offset 4
+; X87-NEXT:    retl
+;
+; X86-SSE-LABEL: uifdl:
+; X86-SSE:       # %bb.0: # %entry
+; X86-SSE-NEXT:    subl $12, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 16
+; X86-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
+; X86-SSE-NEXT:    subpd {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    movapd %xmm0, %xmm1
+; X86-SSE-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
+; X86-SSE-NEXT:    addpd %xmm0, %xmm1
+; X86-SSE-NEXT:    movlpd %xmm1, (%esp)
+; X86-SSE-NEXT:    fldl (%esp)
+; X86-SSE-NEXT:    addl $12, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 4
+; X86-SSE-NEXT:    retl
+;
+; SSE-LABEL: uifdl:
+; SSE:       # %bb.0: # %entry
+; SSE-NEXT:    movq %rdi, %xmm1
+; SSE-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]
+; SSE-NEXT:    subpd {{.*}}(%rip), %xmm1
+; SSE-NEXT:    movapd %xmm1, %xmm0
+; SSE-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1]
+; SSE-NEXT:    addpd %xmm1, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: uifdl:
+; AVX1:       # %bb.0: # %entry
+; AVX1-NEXT:    vmovq %rdi, %xmm0
+; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
+; AVX1-NEXT:    vsubpd {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
+; AVX1-NEXT:    vaddpd %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX512-LABEL: uifdl:
+; AVX512:       # %bb.0: # %entry
+; AVX512-NEXT:    vcvtusi2sd %rdi, %xmm0, %xmm0
+; AVX512-NEXT:    retq
+entry:
+  %result = call double @llvm.experimental.constrained.uitofp.f64.i64(i64 %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret double %result
+}
+
+define float @uiffb(i8 %x) #0 {
+; X87-LABEL: uiffb:
+; X87:       # %bb.0: # %entry
+; X87-NEXT:    pushl %eax
+; X87-NEXT:    .cfi_def_cfa_offset 8
+; X87-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; X87-NEXT:    filds {{[0-9]+}}(%esp)
+; X87-NEXT:    popl %eax
+; X87-NEXT:    .cfi_def_cfa_offset 4
+; X87-NEXT:    retl
+;
+; X86-SSE-LABEL: uiffb:
+; X86-SSE:       # %bb.0: # %entry
+; X86-SSE-NEXT:    pushl %eax
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 8
+; X86-SSE-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE-NEXT:    cvtsi2ss %eax, %xmm0
+; X86-SSE-NEXT:    movss %xmm0, (%esp)
+; X86-SSE-NEXT:    flds (%esp)
+; X86-SSE-NEXT:    popl %eax
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 4
+; X86-SSE-NEXT:    retl
+;
+; SSE-LABEL: uiffb:
+; SSE:       # %bb.0: # %entry
+; SSE-NEXT:    movzbl %dil, %eax
+; SSE-NEXT:    cvtsi2ss %eax, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: uiffb:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    movzbl %dil, %eax
+; AVX-NEXT:    vcvtsi2ss %eax, %xmm0, %xmm0
+; AVX-NEXT:    retq
+entry:
+  %result = call float @llvm.experimental.constrained.uitofp.f32.i8(i8 %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret float %result
+}
+
+define float @uiffw(i16 %x) #0 {
+; X87-LABEL: uiffw:
+; X87:       # %bb.0: # %entry
+; X87-NEXT:    pushl %eax
+; X87-NEXT:    .cfi_def_cfa_offset 8
+; X87-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl %eax, (%esp)
+; X87-NEXT:    fildl (%esp)
+; X87-NEXT:    popl %eax
+; X87-NEXT:    .cfi_def_cfa_offset 4
+; X87-NEXT:    retl
+;
+; X86-SSE-LABEL: uiffw:
+; X86-SSE:       # %bb.0: # %entry
+; X86-SSE-NEXT:    pushl %eax
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 8
+; X86-SSE-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-SSE-NEXT:    cvtsi2ss %eax, %xmm0
+; X86-SSE-NEXT:    movss %xmm0, (%esp)
+; X86-SSE-NEXT:    flds (%esp)
+; X86-SSE-NEXT:    popl %eax
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 4
+; X86-SSE-NEXT:    retl
+;
+; SSE-LABEL: uiffw:
+; SSE:       # %bb.0: # %entry
+; SSE-NEXT:    movzwl %di, %eax
+; SSE-NEXT:    cvtsi2ss %eax, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: uiffw:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    movzwl %di, %eax
+; AVX-NEXT:    vcvtsi2ss %eax, %xmm0, %xmm0
+; AVX-NEXT:    retq
+entry:
+  %result = call float @llvm.experimental.constrained.uitofp.f32.i16(i16 %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret float %result
+}
+
+define float @uiffi(i32 %x) #0 {
+; X87-LABEL: uiffi:
+; X87:       # %bb.0: # %entry
+; X87-NEXT:    subl $12, %esp
+; X87-NEXT:    .cfi_def_cfa_offset 16
+; X87-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl %eax, (%esp)
+; X87-NEXT:    movl $0, {{[0-9]+}}(%esp)
+; X87-NEXT:    fildll (%esp)
+; X87-NEXT:    addl $12, %esp
+; X87-NEXT:    .cfi_def_cfa_offset 4
+; X87-NEXT:    retl
+;
+; X86-SSE-LABEL: uiffi:
+; X86-SSE:       # %bb.0: # %entry
+; X86-SSE-NEXT:    pushl %eax
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 8
+; X86-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; X86-SSE-NEXT:    orpd %xmm0, %xmm1
+; X86-SSE-NEXT:    subsd %xmm0, %xmm1
+; X86-SSE-NEXT:    xorps %xmm0, %xmm0
+; X86-SSE-NEXT:    cvtsd2ss %xmm1, %xmm0
+; X86-SSE-NEXT:    movss %xmm0, (%esp)
+; X86-SSE-NEXT:    flds (%esp)
+; X86-SSE-NEXT:    popl %eax
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 4
+; X86-SSE-NEXT:    retl
+;
+; SSE-LABEL: uiffi:
+; SSE:       # %bb.0: # %entry
+; SSE-NEXT:    movl %edi, %eax
+; SSE-NEXT:    cvtsi2ss %rax, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: uiffi:
+; AVX1:       # %bb.0: # %entry
+; AVX1-NEXT:    movl %edi, %eax
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX512-LABEL: uiffi:
+; AVX512:       # %bb.0: # %entry
+; AVX512-NEXT:    vcvtusi2ss %edi, %xmm0, %xmm0
+; AVX512-NEXT:    retq
+entry:
+  %result = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret float %result
+}
+
+define float @uiffl(i64 %x) #0 {
+; X87-LABEL: uiffl:
+; X87:       # %bb.0: # %entry
+; X87-NEXT:    subl $20, %esp
+; X87-NEXT:    .cfi_def_cfa_offset 24
+; X87-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X87-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X87-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X87-NEXT:    shrl $31, %ecx
+; X87-NEXT:    fildll {{[0-9]+}}(%esp)
+; X87-NEXT:    fadds {{\.LCPI.*}}(,%ecx,4)
+; X87-NEXT:    fstps {{[0-9]+}}(%esp)
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    addl $20, %esp
+; X87-NEXT:    .cfi_def_cfa_offset 4
+; X87-NEXT:    retl
+;
+; X86-SSE-LABEL: uiffl:
+; X86-SSE:       # %bb.0: # %entry
+; X86-SSE-NEXT:    subl $20, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 24
+; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE-NEXT:    movlps %xmm0, {{[0-9]+}}(%esp)
+; X86-SSE-NEXT:    shrl $31, %eax
+; X86-SSE-NEXT:    fildll {{[0-9]+}}(%esp)
+; X86-SSE-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; X86-SSE-NEXT:    fstps {{[0-9]+}}(%esp)
+; X86-SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-SSE-NEXT:    movss %xmm0, (%esp)
+; X86-SSE-NEXT:    flds (%esp)
+; X86-SSE-NEXT:    addl $20, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 4
+; X86-SSE-NEXT:    retl
+;
+; SSE-LABEL: uiffl:
+; SSE:       # %bb.0: # %entry
+; SSE-NEXT:    movq %rdi, %rax
+; SSE-NEXT:    shrq %rax
+; SSE-NEXT:    movl %edi, %ecx
+; SSE-NEXT:    andl $1, %ecx
+; SSE-NEXT:    orq %rax, %rcx
+; SSE-NEXT:    testq %rdi, %rdi
+; SSE-NEXT:    cmovnsq %rdi, %rcx
+; SSE-NEXT:    cvtsi2ss %rcx, %xmm0
+; SSE-NEXT:    jns .LBB52_2
+; SSE-NEXT:  # %bb.1:
+; SSE-NEXT:    addss %xmm0, %xmm0
+; SSE-NEXT:  .LBB52_2: # %entry
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: uiffl:
+; AVX1:       # %bb.0: # %entry
+; AVX1-NEXT:    movq %rdi, %rax
+; AVX1-NEXT:    shrq %rax
+; AVX1-NEXT:    movl %edi, %ecx
+; AVX1-NEXT:    andl $1, %ecx
+; AVX1-NEXT:    orq %rax, %rcx
+; AVX1-NEXT:    testq %rdi, %rdi
+; AVX1-NEXT:    cmovnsq %rdi, %rcx
+; AVX1-NEXT:    vcvtsi2ss %rcx, %xmm0, %xmm0
+; AVX1-NEXT:    jns .LBB52_2
+; AVX1-NEXT:  # %bb.1:
+; AVX1-NEXT:    vaddss %xmm0, %xmm0, %xmm0
+; AVX1-NEXT:  .LBB52_2: # %entry
+; AVX1-NEXT:    retq
+;
+; AVX512-LABEL: uiffl:
+; AVX512:       # %bb.0: # %entry
+; AVX512-NEXT:    vcvtusi2ss %rdi, %xmm0, %xmm0
+; AVX512-NEXT:    retq
+entry:
+  %result = call float @llvm.experimental.constrained.uitofp.f32.i64(i64 %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret float %result
+}
+
+attributes #0 = { strictfp }
+
 @llvm.fp.env = thread_local global i8 zeroinitializer, section "llvm.metadata"
 declare double @llvm.experimental.constrained.fadd.f64(double, double, metadata, metadata)
 declare double @llvm.experimental.constrained.fsub.f64(double, double, metadata, metadata)
@@ -327,8 +2700,39 @@ declare double @llvm.experimental.constrained.log10.f64(double, metadata, metada
 declare double @llvm.experimental.constrained.log2.f64(double, metadata, metadata)
 declare double @llvm.experimental.constrained.rint.f64(double, metadata, metadata)
 declare double @llvm.experimental.constrained.nearbyint.f64(double, metadata, metadata)
-declare float @llvm.experimental.constrained.fma.f32(float, float, float, metadata, metadata)
-declare double @llvm.experimental.constrained.fma.f64(double, double, double, metadata, metadata)
+declare i8  @llvm.experimental.constrained.fptosi.i8.f64(double, metadata)
+declare i16 @llvm.experimental.constrained.fptosi.i16.f64(double, metadata)
+declare i32 @llvm.experimental.constrained.fptosi.i32.f64(double, metadata)
+declare i64 @llvm.experimental.constrained.fptosi.i64.f64(double, metadata)
+declare i128 @llvm.experimental.constrained.fptosi.i128.f64(double, metadata)
+declare i8  @llvm.experimental.constrained.fptoui.i8.f64(double, metadata)
+declare i16 @llvm.experimental.constrained.fptoui.i16.f64(double, metadata)
+declare i32 @llvm.experimental.constrained.fptoui.i32.f64(double, metadata)
+declare i64 @llvm.experimental.constrained.fptoui.i64.f64(double, metadata)
+declare i128 @llvm.experimental.constrained.fptoui.i128.f64(double, metadata)
 declare float @llvm.experimental.constrained.fptrunc.f32.f64(double, metadata, metadata)
 declare double @llvm.experimental.constrained.fpext.f64.f32(float, metadata)
-
+declare i32 @llvm.experimental.constrained.lrint.i32.f64(double, metadata, metadata)
+declare i32 @llvm.experimental.constrained.lrint.i32.f32(float, metadata, metadata)
+declare i64 @llvm.experimental.constrained.llrint.i64.f64(double, metadata, metadata)
+declare i64 @llvm.experimental.constrained.llrint.i64.f32(float, metadata, metadata)
+declare i32 @llvm.experimental.constrained.lround.i32.f64(double, metadata)
+declare i32 @llvm.experimental.constrained.lround.i32.f32(float, metadata)
+declare i64 @llvm.experimental.constrained.llround.i64.f64(double, metadata)
+declare i64 @llvm.experimental.constrained.llround.i64.f32(float, metadata)
+declare double @llvm.experimental.constrained.sitofp.f64.i8(i8, metadata, metadata)
+declare double @llvm.experimental.constrained.sitofp.f64.i16(i16, metadata, metadata)
+declare double @llvm.experimental.constrained.sitofp.f64.i32(i32, metadata, metadata)
+declare double @llvm.experimental.constrained.sitofp.f64.i64(i64, metadata, metadata)
+declare float @llvm.experimental.constrained.sitofp.f32.i8(i8, metadata, metadata)
+declare float @llvm.experimental.constrained.sitofp.f32.i16(i16, metadata, metadata)
+declare float @llvm.experimental.constrained.sitofp.f32.i32(i32, metadata, metadata)
+declare float @llvm.experimental.constrained.sitofp.f32.i64(i64, metadata, metadata)
+declare double @llvm.experimental.constrained.uitofp.f64.i8(i8, metadata, metadata)
+declare double @llvm.experimental.constrained.uitofp.f64.i16(i16, metadata, metadata)
+declare double @llvm.experimental.constrained.uitofp.f64.i32(i32, metadata, metadata)
+declare double @llvm.experimental.constrained.uitofp.f64.i64(i64, metadata, metadata)
+declare float @llvm.experimental.constrained.uitofp.f32.i8(i8, metadata, metadata)
+declare float @llvm.experimental.constrained.uitofp.f32.i16(i16, metadata, metadata)
+declare float @llvm.experimental.constrained.uitofp.f32.i32(i32, metadata, metadata)
+declare float @llvm.experimental.constrained.uitofp.f32.i64(i64, metadata, metadata)
diff --git a/llvm/test/CodeGen/X86/fp-stack-2results.ll b/llvm/test/CodeGen/X86/fp-stack-2results.ll
index c5a2d1af7..05a29a77d 100644
--- a/llvm/test/CodeGen/X86/fp-stack-2results.ll
+++ b/llvm/test/CodeGen/X86/fp-stack-2results.ll
@@ -1,11 +1,18 @@
-; RUN: llc < %s -mtriple=i686-- | grep fldz
-; RUN: llc < %s -mtriple=x86_64-- | grep fld1
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- | FileCheck %s --check-prefixes=ALL,i686
+; RUN: llc < %s -mtriple=x86_64-- | FileCheck %s -check-prefixes=ALL,x86_64
+
 
 %0 = type { x86_fp80, x86_fp80 }
 
 ; This is basically this code on x86-64:
 ; _Complex long double test() { return 1.0; }
 define %0 @test() {
+; ALL-LABEL: test:
+; ALL:       # %bb.0:
+; ALL-NEXT:    fldz
+; ALL-NEXT:    fld1
+; ALL-NEXT:    ret{{[l|q]}}
   %A = fpext double 1.0 to x86_fp80
   %B = fpext double 0.0 to x86_fp80
   %mrv = insertvalue %0 undef, x86_fp80 %A, 0
@@ -19,6 +26,11 @@ define %0 @test() {
 ;	fld	%st(0)
 ;	ret
 define %0 @test2() {
+; ALL-LABEL: test2:
+; ALL:       # %bb.0:
+; ALL-NEXT:    fld1
+; ALL-NEXT:    fld %st(0)
+; ALL-NEXT:    ret{{[l|q]}}
   %A = fpext double 1.0 to x86_fp80
   %mrv = insertvalue %0 undef, x86_fp80 %A, 0
   %mrv1 = insertvalue %0 %mrv, x86_fp80 %A, 1
@@ -27,6 +39,47 @@ define %0 @test2() {
 
 ; Uses both values.
 define void @call1(x86_fp80 *%P1, x86_fp80 *%P2) {
+; i686-LABEL: call1:
+; i686:       # %bb.0:
+; i686-NEXT:    pushl %edi
+; i686-NEXT:    .cfi_def_cfa_offset 8
+; i686-NEXT:    pushl %esi
+; i686-NEXT:    .cfi_def_cfa_offset 12
+; i686-NEXT:    .cfi_offset %esi, -12
+; i686-NEXT:    .cfi_offset %edi, -8
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; i686-NEXT:    calll test
+; i686-NEXT:    fstpt (%edi)
+; i686-NEXT:    fstpt (%esi)
+; i686-NEXT:    popl %esi
+; i686-NEXT:    .cfi_def_cfa_offset 8
+; i686-NEXT:    popl %edi
+; i686-NEXT:    .cfi_def_cfa_offset 4
+; i686-NEXT:    retl
+;
+; x86_64-LABEL: call1:
+; x86_64:       # %bb.0:
+; x86_64-NEXT:    pushq %r14
+; x86_64-NEXT:    .cfi_def_cfa_offset 16
+; x86_64-NEXT:    pushq %rbx
+; x86_64-NEXT:    .cfi_def_cfa_offset 24
+; x86_64-NEXT:    pushq %rax
+; x86_64-NEXT:    .cfi_def_cfa_offset 32
+; x86_64-NEXT:    .cfi_offset %rbx, -24
+; x86_64-NEXT:    .cfi_offset %r14, -16
+; x86_64-NEXT:    movq %rsi, %r14
+; x86_64-NEXT:    movq %rdi, %rbx
+; x86_64-NEXT:    callq test
+; x86_64-NEXT:    fstpt (%rbx)
+; x86_64-NEXT:    fstpt (%r14)
+; x86_64-NEXT:    addq $8, %rsp
+; x86_64-NEXT:    .cfi_def_cfa_offset 24
+; x86_64-NEXT:    popq %rbx
+; x86_64-NEXT:    .cfi_def_cfa_offset 16
+; x86_64-NEXT:    popq %r14
+; x86_64-NEXT:    .cfi_def_cfa_offset 8
+; x86_64-NEXT:    retq
   %a = call %0 @test()
   %b = extractvalue %0 %a, 0
   store x86_fp80 %b, x86_fp80* %P1
@@ -38,6 +91,49 @@ define void @call1(x86_fp80 *%P1, x86_fp80 *%P2) {
 
 ; Uses both values, requires fxch
 define void @call2(x86_fp80 *%P1, x86_fp80 *%P2) {
+; i686-LABEL: call2:
+; i686:       # %bb.0:
+; i686-NEXT:    pushl %edi
+; i686-NEXT:    .cfi_def_cfa_offset 8
+; i686-NEXT:    pushl %esi
+; i686-NEXT:    .cfi_def_cfa_offset 12
+; i686-NEXT:    .cfi_offset %esi, -12
+; i686-NEXT:    .cfi_offset %edi, -8
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; i686-NEXT:    calll test
+; i686-NEXT:    fxch %st(1)
+; i686-NEXT:    fstpt (%edi)
+; i686-NEXT:    fstpt (%esi)
+; i686-NEXT:    popl %esi
+; i686-NEXT:    .cfi_def_cfa_offset 8
+; i686-NEXT:    popl %edi
+; i686-NEXT:    .cfi_def_cfa_offset 4
+; i686-NEXT:    retl
+;
+; x86_64-LABEL: call2:
+; x86_64:       # %bb.0:
+; x86_64-NEXT:    pushq %r14
+; x86_64-NEXT:    .cfi_def_cfa_offset 16
+; x86_64-NEXT:    pushq %rbx
+; x86_64-NEXT:    .cfi_def_cfa_offset 24
+; x86_64-NEXT:    pushq %rax
+; x86_64-NEXT:    .cfi_def_cfa_offset 32
+; x86_64-NEXT:    .cfi_offset %rbx, -24
+; x86_64-NEXT:    .cfi_offset %r14, -16
+; x86_64-NEXT:    movq %rsi, %r14
+; x86_64-NEXT:    movq %rdi, %rbx
+; x86_64-NEXT:    callq test
+; x86_64-NEXT:    fxch %st(1)
+; x86_64-NEXT:    fstpt (%rbx)
+; x86_64-NEXT:    fstpt (%r14)
+; x86_64-NEXT:    addq $8, %rsp
+; x86_64-NEXT:    .cfi_def_cfa_offset 24
+; x86_64-NEXT:    popq %rbx
+; x86_64-NEXT:    .cfi_def_cfa_offset 16
+; x86_64-NEXT:    popq %r14
+; x86_64-NEXT:    .cfi_def_cfa_offset 8
+; x86_64-NEXT:    retq
   %a = call %0 @test()
   %b = extractvalue %0 %a, 1
   store x86_fp80 %b, x86_fp80* %P1
@@ -49,6 +145,31 @@ define void @call2(x86_fp80 *%P1, x86_fp80 *%P2) {
 
 ; Uses ST(0), ST(1) is dead but must be popped.
 define void @call3(x86_fp80 *%P1, x86_fp80 *%P2) {
+; i686-LABEL: call3:
+; i686:       # %bb.0:
+; i686-NEXT:    pushl %esi
+; i686-NEXT:    .cfi_def_cfa_offset 8
+; i686-NEXT:    .cfi_offset %esi, -8
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; i686-NEXT:    calll test
+; i686-NEXT:    fstp %st(1)
+; i686-NEXT:    fstpt (%esi)
+; i686-NEXT:    popl %esi
+; i686-NEXT:    .cfi_def_cfa_offset 4
+; i686-NEXT:    retl
+;
+; x86_64-LABEL: call3:
+; x86_64:       # %bb.0:
+; x86_64-NEXT:    pushq %rbx
+; x86_64-NEXT:    .cfi_def_cfa_offset 16
+; x86_64-NEXT:    .cfi_offset %rbx, -16
+; x86_64-NEXT:    movq %rdi, %rbx
+; x86_64-NEXT:    callq test
+; x86_64-NEXT:    fstp %st(1)
+; x86_64-NEXT:    fstpt (%rbx)
+; x86_64-NEXT:    popq %rbx
+; x86_64-NEXT:    .cfi_def_cfa_offset 8
+; x86_64-NEXT:    retq
   %a = call %0 @test()
   %b = extractvalue %0 %a, 0
   store x86_fp80 %b, x86_fp80* %P1
@@ -57,6 +178,31 @@ define void @call3(x86_fp80 *%P1, x86_fp80 *%P2) {
 
 ; Uses ST(1), ST(0) is dead and must be popped.
 define void @call4(x86_fp80 *%P1, x86_fp80 *%P2) {
+; i686-LABEL: call4:
+; i686:       # %bb.0:
+; i686-NEXT:    pushl %esi
+; i686-NEXT:    .cfi_def_cfa_offset 8
+; i686-NEXT:    .cfi_offset %esi, -8
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; i686-NEXT:    calll test
+; i686-NEXT:    fstp %st(0)
+; i686-NEXT:    fstpt (%esi)
+; i686-NEXT:    popl %esi
+; i686-NEXT:    .cfi_def_cfa_offset 4
+; i686-NEXT:    retl
+;
+; x86_64-LABEL: call4:
+; x86_64:       # %bb.0:
+; x86_64-NEXT:    pushq %rbx
+; x86_64-NEXT:    .cfi_def_cfa_offset 16
+; x86_64-NEXT:    .cfi_offset %rbx, -16
+; x86_64-NEXT:    movq %rsi, %rbx
+; x86_64-NEXT:    callq test
+; x86_64-NEXT:    fstp %st(0)
+; x86_64-NEXT:    fstpt (%rbx)
+; x86_64-NEXT:    popq %rbx
+; x86_64-NEXT:    .cfi_def_cfa_offset 8
+; x86_64-NEXT:    retq
   %a = call %0 @test()
 
   %c = extractvalue %0 %a, 1
diff --git a/llvm/test/CodeGen/X86/fp-stack-direct-ret.ll b/llvm/test/CodeGen/X86/fp-stack-direct-ret.ll
index b8bd22308..6e6eed39b 100644
--- a/llvm/test/CodeGen/X86/fp-stack-direct-ret.ll
+++ b/llvm/test/CodeGen/X86/fp-stack-direct-ret.ll
@@ -1,9 +1,13 @@
-; RUN: llc < %s -mtriple=i686-- | not grep fstp
-; RUN: llc < %s -mtriple=i686-- -mcpu=yonah | not grep movsd
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- | FileCheck %s --check-prefixes=ALL,GENERIC
+; RUN: llc < %s -mtriple=i686-- -mcpu=yonah | FileCheck %s --check-prefixes=ALL,YONAH
 
 declare double @foo()
 
 define double @bar() {
+; ALL-LABEL: bar:
+; ALL:       # %bb.0: # %entry
+; ALL-NEXT:    jmp foo # TAILCALL
 entry:
 	%tmp5 = tail call double @foo()
 	ret double %tmp5
diff --git a/llvm/test/CodeGen/X86/fp-stack-ret-conv.ll b/llvm/test/CodeGen/X86/fp-stack-ret-conv.ll
index 3e26141ec..25b3f49df 100644
--- a/llvm/test/CodeGen/X86/fp-stack-ret-conv.ll
+++ b/llvm/test/CodeGen/X86/fp-stack-ret-conv.ll
@@ -1,11 +1,26 @@
-; RUN: llc < %s -mcpu=yonah | grep cvtss2sd
-; RUN: llc < %s -mcpu=yonah | grep fstps
-; RUN: llc < %s -mcpu=yonah | not grep cvtsd2ss
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mcpu=yonah | FileCheck %s
 
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64"
 target triple = "i686-apple-darwin8"
 
 define void @test(double *%b) {
+; CHECK-LABEL: test:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    pushl %esi
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    subl $8, %esp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset %esi, -8
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; CHECK-NEXT:    calll _foo
+; CHECK-NEXT:    fstps {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; CHECK-NEXT:    cvtss2sd %xmm0, %xmm0
+; CHECK-NEXT:    movsd %xmm0, (%esi)
+; CHECK-NEXT:    addl $8, %esp
+; CHECK-NEXT:    popl %esi
+; CHECK-NEXT:    retl
 entry:
 	%tmp13 = tail call double @foo()
 	%tmp1314 = fptrunc double %tmp13 to float		; <float> [#uses=1]
diff --git a/llvm/test/CodeGen/X86/fp-stack-ret-store.ll b/llvm/test/CodeGen/X86/fp-stack-ret-store.ll
index c7cbb2a05..bf76f4bd3 100644
--- a/llvm/test/CodeGen/X86/fp-stack-ret-store.ll
+++ b/llvm/test/CodeGen/X86/fp-stack-ret-store.ll
@@ -1,4 +1,6 @@
-; RUN: llc < %s -mcpu=yonah | not grep movss
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mcpu=yonah | FileCheck %s
+
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
 target triple = "i686-apple-darwin8"
 
@@ -6,6 +8,19 @@ target triple = "i686-apple-darwin8"
 ; go through a stack slot to get there.
 
 define void @bar(double* %P) {
+; CHECK-LABEL: bar:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    pushl %esi
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    subl $8, %esp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset %esi, -8
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; CHECK-NEXT:    calll _foo
+; CHECK-NEXT:    fstpl (%esi)
+; CHECK-NEXT:    addl $8, %esp
+; CHECK-NEXT:    popl %esi
+; CHECK-NEXT:    retl
 entry:
 	%tmp = tail call double (...) @foo( )		; <double> [#uses=1]
 	store double %tmp, double* %P, align 8
@@ -15,6 +30,19 @@ entry:
 declare double @foo(...)
 
 define void @bar2(float* %P) {
+; CHECK-LABEL: bar2:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    pushl %esi
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    subl $8, %esp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset %esi, -8
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; CHECK-NEXT:    calll _foo2
+; CHECK-NEXT:    fstps (%esi)
+; CHECK-NEXT:    addl $8, %esp
+; CHECK-NEXT:    popl %esi
+; CHECK-NEXT:    retl
 entry:
 	%tmp = tail call double (...) @foo2( )		; <double> [#uses=1]
 	%tmp1 = fptrunc double %tmp to float		; <float> [#uses=1]
diff --git a/llvm/test/CodeGen/X86/fp-stack-retcopy.ll b/llvm/test/CodeGen/X86/fp-stack-retcopy.ll
index bc77f7963..c58b00cb0 100644
--- a/llvm/test/CodeGen/X86/fp-stack-retcopy.ll
+++ b/llvm/test/CodeGen/X86/fp-stack-retcopy.ll
@@ -1,10 +1,14 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; This should not copy the result of foo into an xmm register.
-; RUN: llc < %s -mcpu=yonah -mtriple=i686-apple-darwin9 | not grep xmm
+; RUN: llc < %s -mcpu=yonah -mtriple=i686-apple-darwin9 | FileCheck %s
 ; rdar://5689903
 
 declare double @foo()
 
 define double @carg({ double, double }* byval  %z) nounwind  {
+; CHECK-LABEL: carg:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    jmp _foo ## TAILCALL
 entry:
 	%tmp5 = tail call double @foo() nounwind 		; <double> [#uses=1]
 	ret double %tmp5
diff --git a/llvm/test/CodeGen/X86/fp-stack-set-st1.ll b/llvm/test/CodeGen/X86/fp-stack-set-st1.ll
index 45597bb78..065f84d34 100644
--- a/llvm/test/CodeGen/X86/fp-stack-set-st1.ll
+++ b/llvm/test/CodeGen/X86/fp-stack-set-st1.ll
@@ -1,6 +1,23 @@
-; RUN: llc < %s -mtriple=i686-- | grep fxch | count 2
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- | FileCheck %s
 
 define i32 @main() nounwind {
+; CHECK-LABEL: main:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fldl {{\.LCPI.*}}
+; CHECK-NEXT:    fldl {{\.LCPI.*}}
+; CHECK-NEXT:    fxch %st(1)
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    fmul %st(1), %st
+; CHECK-NEXT:    fst %st(1)
+; CHECK-NEXT:    frndint
+; CHECK-NEXT:    fxch %st(1)
+; CHECK-NEXT:    fsub %st(1), %st
+; CHECK-NEXT:    f2xm1
+; CHECK-EMPTY:
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    fstp %st(0)
+; CHECK-NEXT:    fstp %st(0)
 entry:
 	%asmtmp = tail call { double, double } asm sideeffect "fmul\09%st(1),%st\0A\09fst\09%st(1)\0A\09frndint\0A\09fxch  %st(1)\0A\09fsub\09%st(1),%st\0A\09f2xm1\0A\09", "={st},={st(1)},0,1,~{dirflag},~{fpsr},~{flags}"(double 0x4030FEFBD582097D, double 4.620000e+01) nounwind		; <{ double, double }> [#uses=0]
 	unreachable
diff --git a/llvm/test/CodeGen/X86/fp-strict-libcalls-msvc32.ll b/llvm/test/CodeGen/X86/fp-strict-libcalls-msvc32.ll
new file mode 100644
index 000000000..45d7d513e
--- /dev/null
+++ b/llvm/test/CodeGen/X86/fp-strict-libcalls-msvc32.ll
@@ -0,0 +1,155 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -O3 -mtriple=i686-pc-windows-msvc -mattr=+cmov < %s | FileCheck %s
+
+define float @ceil(float %x) #0 {
+; CHECK-LABEL: ceil:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subl $12, %esp
+; CHECK-NEXT:    flds {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fstpl (%esp)
+; CHECK-NEXT:    calll _ceil
+; CHECK-NEXT:    fstps {{[0-9]+}}(%esp)
+; CHECK-NEXT:    flds {{[0-9]+}}(%esp)
+; CHECK-NEXT:    addl $12, %esp
+; CHECK-NEXT:    retl
+  %result = call float @llvm.experimental.constrained.ceil.f32(float %x, metadata !"fpexcept.strict") #0
+  ret float %result
+}
+
+define float @cos(float %x) #0 {
+; CHECK-LABEL: cos:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subl $12, %esp
+; CHECK-NEXT:    flds {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fstpl (%esp)
+; CHECK-NEXT:    calll _cos
+; CHECK-NEXT:    fstps {{[0-9]+}}(%esp)
+; CHECK-NEXT:    flds {{[0-9]+}}(%esp)
+; CHECK-NEXT:    addl $12, %esp
+; CHECK-NEXT:    retl
+  %result = call float @llvm.experimental.constrained.cos.f32(float %x, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret float %result
+}
+
+define float @exp(float %x) #0 {
+; CHECK-LABEL: exp:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subl $12, %esp
+; CHECK-NEXT:    flds {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fstpl (%esp)
+; CHECK-NEXT:    calll _exp
+; CHECK-NEXT:    fstps {{[0-9]+}}(%esp)
+; CHECK-NEXT:    flds {{[0-9]+}}(%esp)
+; CHECK-NEXT:    addl $12, %esp
+; CHECK-NEXT:    retl
+  %result = call float @llvm.experimental.constrained.exp.f32(float %x, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret float %result
+}
+
+define float @floor(float %x) #0 {
+; CHECK-LABEL: floor:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subl $12, %esp
+; CHECK-NEXT:    flds {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fstpl (%esp)
+; CHECK-NEXT:    calll _floor
+; CHECK-NEXT:    fstps {{[0-9]+}}(%esp)
+; CHECK-NEXT:    flds {{[0-9]+}}(%esp)
+; CHECK-NEXT:    addl $12, %esp
+; CHECK-NEXT:    retl
+  %result = call float @llvm.experimental.constrained.floor.f32(float %x, metadata !"fpexcept.strict") #0
+  ret float %result
+}
+
+define float @frem(float %x, float %y) #0 {
+; CHECK-LABEL: frem:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subl $20, %esp
+; CHECK-NEXT:    flds {{[0-9]+}}(%esp)
+; CHECK-NEXT:    flds {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fxch %st(1)
+; CHECK-NEXT:    fstpl {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fstpl (%esp)
+; CHECK-NEXT:    calll _fmod
+; CHECK-NEXT:    fstps {{[0-9]+}}(%esp)
+; CHECK-NEXT:    flds {{[0-9]+}}(%esp)
+; CHECK-NEXT:    addl $20, %esp
+; CHECK-NEXT:    retl
+  %result = call float @llvm.experimental.constrained.frem.f32(float %x, float %y, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret float %result
+}
+
+define float @log(float %x) #0 {
+; CHECK-LABEL: log:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subl $12, %esp
+; CHECK-NEXT:    flds {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fstpl (%esp)
+; CHECK-NEXT:    calll _log
+; CHECK-NEXT:    fstps {{[0-9]+}}(%esp)
+; CHECK-NEXT:    flds {{[0-9]+}}(%esp)
+; CHECK-NEXT:    addl $12, %esp
+; CHECK-NEXT:    retl
+  %result = call float @llvm.experimental.constrained.log.f32(float %x, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret float %result
+}
+
+define float @log10(float %x) #0 {
+; CHECK-LABEL: log10:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subl $12, %esp
+; CHECK-NEXT:    flds {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fstpl (%esp)
+; CHECK-NEXT:    calll _log10
+; CHECK-NEXT:    fstps {{[0-9]+}}(%esp)
+; CHECK-NEXT:    flds {{[0-9]+}}(%esp)
+; CHECK-NEXT:    addl $12, %esp
+; CHECK-NEXT:    retl
+  %result = call float @llvm.experimental.constrained.log10.f32(float %x, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret float %result
+}
+
+define float @pow(float %x, float %y) #0 {
+; CHECK-LABEL: pow:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subl $20, %esp
+; CHECK-NEXT:    flds {{[0-9]+}}(%esp)
+; CHECK-NEXT:    flds {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fxch %st(1)
+; CHECK-NEXT:    fstpl {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fstpl (%esp)
+; CHECK-NEXT:    calll _pow
+; CHECK-NEXT:    fstps {{[0-9]+}}(%esp)
+; CHECK-NEXT:    flds {{[0-9]+}}(%esp)
+; CHECK-NEXT:    addl $20, %esp
+; CHECK-NEXT:    retl
+  %result = call float @llvm.experimental.constrained.pow.f32(float %x, float %y, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret float %result
+}
+
+define float @sin(float %x) #0 {
+; CHECK-LABEL: sin:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subl $12, %esp
+; CHECK-NEXT:    flds {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fstpl (%esp)
+; CHECK-NEXT:    calll _sin
+; CHECK-NEXT:    fstps {{[0-9]+}}(%esp)
+; CHECK-NEXT:    flds {{[0-9]+}}(%esp)
+; CHECK-NEXT:    addl $12, %esp
+; CHECK-NEXT:    retl
+  %result = call float @llvm.experimental.constrained.sin.f32(float %x, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret float %result
+}
+
+attributes #0 = { strictfp }
+
+declare float @llvm.experimental.constrained.ceil.f32(float, metadata)
+declare float @llvm.experimental.constrained.cos.f32(float, metadata, metadata)
+declare float @llvm.experimental.constrained.exp.f32(float, metadata, metadata)
+declare float @llvm.experimental.constrained.floor.f32(float, metadata)
+declare float @llvm.experimental.constrained.frem.f32(float, float, metadata, metadata)
+declare float @llvm.experimental.constrained.log.f32(float, metadata, metadata)
+declare float @llvm.experimental.constrained.log10.f32(float, metadata, metadata)
+declare float @llvm.experimental.constrained.pow.f32(float, float, metadata, metadata)
+declare float @llvm.experimental.constrained.sin.f32(float, metadata, metadata)
diff --git a/llvm/test/CodeGen/X86/fp-strict-scalar-cmp.ll b/llvm/test/CodeGen/X86/fp-strict-scalar-cmp.ll
new file mode 100644
index 000000000..f27076028
--- /dev/null
+++ b/llvm/test/CodeGen/X86/fp-strict-scalar-cmp.ll
@@ -0,0 +1,4032 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse2 -O3 | FileCheck %s --check-prefixes=CHECK-32,SSE-32
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 -O3 | FileCheck %s --check-prefixes=CHECK-64,SSE-64
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx -O3 | FileCheck %s --check-prefixes=CHECK-32,AVX-32
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx -O3 | FileCheck %s --check-prefixes=CHECK-64,AVX-64
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx512f -mattr=+avx512vl -O3 | FileCheck %s --check-prefixes=CHECK-32,AVX-32
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f -mattr=+avx512vl -O3 | FileCheck %s --check-prefixes=CHECK-64,AVX-64
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=-sse -O3 | FileCheck %s --check-prefixes=X87
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=-sse,+cmov -O3 | FileCheck %s --check-prefixes=X87-CMOV
+
+define i32 @test_f32_oeq_q(i32 %a, i32 %b, float %f1, float %f2) #0 {
+; SSE-32-LABEL: test_f32_oeq_q:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE-32-NEXT:    ucomiss {{[0-9]+}}(%esp), %xmm0
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; SSE-32-NEXT:    cmovnel %eax, %ecx
+; SSE-32-NEXT:    cmovpl %eax, %ecx
+; SSE-32-NEXT:    movl (%ecx), %eax
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_f32_oeq_q:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movl %edi, %eax
+; SSE-64-NEXT:    ucomiss %xmm1, %xmm0
+; SSE-64-NEXT:    cmovnel %esi, %eax
+; SSE-64-NEXT:    cmovpl %esi, %eax
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_f32_oeq_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vucomiss {{[0-9]+}}(%esp), %xmm0
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; AVX-32-NEXT:    cmovnel %eax, %ecx
+; AVX-32-NEXT:    cmovpl %eax, %ecx
+; AVX-32-NEXT:    movl (%ecx), %eax
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_f32_oeq_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    movl %edi, %eax
+; AVX-64-NEXT:    vucomiss %xmm1, %xmm0
+; AVX-64-NEXT:    cmovnel %esi, %eax
+; AVX-64-NEXT:    cmovpl %esi, %eax
+; AVX-64-NEXT:    retq
+;
+; X87-LABEL: test_f32_oeq_q:
+; X87:       # %bb.0:
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    fucompp
+; X87-NEXT:    fnstsw %ax
+; X87-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-NEXT:    sahf
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    jne .LBB0_3
+; X87-NEXT:  # %bb.1:
+; X87-NEXT:    jp .LBB0_3
+; X87-NEXT:  # %bb.2:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:  .LBB0_3:
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+;
+; X87-CMOV-LABEL: test_f32_oeq_q:
+; X87-CMOV:       # %bb.0:
+; X87-CMOV-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fucompi %st(1), %st
+; X87-CMOV-NEXT:    fstp %st(0)
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X87-CMOV-NEXT:    cmovnel %eax, %ecx
+; X87-CMOV-NEXT:    cmovpl %eax, %ecx
+; X87-CMOV-NEXT:    movl (%ecx), %eax
+; X87-CMOV-NEXT:    retl
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f32(
+                                               float %f1, float %f2, metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_f32_ogt_q(i32 %a, i32 %b, float %f1, float %f2) #0 {
+; SSE-32-LABEL: test_f32_ogt_q:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE-32-NEXT:    ucomiss {{[0-9]+}}(%esp), %xmm0
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; SSE-32-NEXT:    cmoval %eax, %ecx
+; SSE-32-NEXT:    movl (%ecx), %eax
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_f32_ogt_q:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movl %edi, %eax
+; SSE-64-NEXT:    ucomiss %xmm1, %xmm0
+; SSE-64-NEXT:    cmovbel %esi, %eax
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_f32_ogt_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vucomiss {{[0-9]+}}(%esp), %xmm0
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; AVX-32-NEXT:    cmoval %eax, %ecx
+; AVX-32-NEXT:    movl (%ecx), %eax
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_f32_ogt_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    movl %edi, %eax
+; AVX-64-NEXT:    vucomiss %xmm1, %xmm0
+; AVX-64-NEXT:    cmovbel %esi, %eax
+; AVX-64-NEXT:    retq
+;
+; X87-LABEL: test_f32_ogt_q:
+; X87:       # %bb.0:
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    fucompp
+; X87-NEXT:    fnstsw %ax
+; X87-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-NEXT:    sahf
+; X87-NEXT:    ja .LBB1_1
+; X87-NEXT:  # %bb.2:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+; X87-NEXT:  .LBB1_1:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+;
+; X87-CMOV-LABEL: test_f32_ogt_q:
+; X87-CMOV:       # %bb.0:
+; X87-CMOV-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fucompi %st(1), %st
+; X87-CMOV-NEXT:    fstp %st(0)
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X87-CMOV-NEXT:    cmoval %eax, %ecx
+; X87-CMOV-NEXT:    movl (%ecx), %eax
+; X87-CMOV-NEXT:    retl
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f32(
+                                               float %f1, float %f2, metadata !"ogt",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_f32_oge_q(i32 %a, i32 %b, float %f1, float %f2) #0 {
+; SSE-32-LABEL: test_f32_oge_q:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE-32-NEXT:    ucomiss {{[0-9]+}}(%esp), %xmm0
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; SSE-32-NEXT:    cmovael %eax, %ecx
+; SSE-32-NEXT:    movl (%ecx), %eax
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_f32_oge_q:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movl %edi, %eax
+; SSE-64-NEXT:    ucomiss %xmm1, %xmm0
+; SSE-64-NEXT:    cmovbl %esi, %eax
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_f32_oge_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vucomiss {{[0-9]+}}(%esp), %xmm0
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; AVX-32-NEXT:    cmovael %eax, %ecx
+; AVX-32-NEXT:    movl (%ecx), %eax
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_f32_oge_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    movl %edi, %eax
+; AVX-64-NEXT:    vucomiss %xmm1, %xmm0
+; AVX-64-NEXT:    cmovbl %esi, %eax
+; AVX-64-NEXT:    retq
+;
+; X87-LABEL: test_f32_oge_q:
+; X87:       # %bb.0:
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    fucompp
+; X87-NEXT:    fnstsw %ax
+; X87-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-NEXT:    sahf
+; X87-NEXT:    jae .LBB2_1
+; X87-NEXT:  # %bb.2:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+; X87-NEXT:  .LBB2_1:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+;
+; X87-CMOV-LABEL: test_f32_oge_q:
+; X87-CMOV:       # %bb.0:
+; X87-CMOV-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fucompi %st(1), %st
+; X87-CMOV-NEXT:    fstp %st(0)
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X87-CMOV-NEXT:    cmovael %eax, %ecx
+; X87-CMOV-NEXT:    movl (%ecx), %eax
+; X87-CMOV-NEXT:    retl
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f32(
+                                               float %f1, float %f2, metadata !"oge",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_f32_olt_q(i32 %a, i32 %b, float %f1, float %f2) #0 {
+; SSE-32-LABEL: test_f32_olt_q:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE-32-NEXT:    ucomiss {{[0-9]+}}(%esp), %xmm0
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; SSE-32-NEXT:    cmoval %eax, %ecx
+; SSE-32-NEXT:    movl (%ecx), %eax
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_f32_olt_q:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movl %edi, %eax
+; SSE-64-NEXT:    ucomiss %xmm0, %xmm1
+; SSE-64-NEXT:    cmovbel %esi, %eax
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_f32_olt_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vucomiss {{[0-9]+}}(%esp), %xmm0
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; AVX-32-NEXT:    cmoval %eax, %ecx
+; AVX-32-NEXT:    movl (%ecx), %eax
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_f32_olt_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    movl %edi, %eax
+; AVX-64-NEXT:    vucomiss %xmm0, %xmm1
+; AVX-64-NEXT:    cmovbel %esi, %eax
+; AVX-64-NEXT:    retq
+;
+; X87-LABEL: test_f32_olt_q:
+; X87:       # %bb.0:
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    fucompp
+; X87-NEXT:    fnstsw %ax
+; X87-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-NEXT:    sahf
+; X87-NEXT:    ja .LBB3_1
+; X87-NEXT:  # %bb.2:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+; X87-NEXT:  .LBB3_1:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+;
+; X87-CMOV-LABEL: test_f32_olt_q:
+; X87-CMOV:       # %bb.0:
+; X87-CMOV-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fucompi %st(1), %st
+; X87-CMOV-NEXT:    fstp %st(0)
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X87-CMOV-NEXT:    cmoval %eax, %ecx
+; X87-CMOV-NEXT:    movl (%ecx), %eax
+; X87-CMOV-NEXT:    retl
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f32(
+                                               float %f1, float %f2, metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_f32_ole_q(i32 %a, i32 %b, float %f1, float %f2) #0 {
+; SSE-32-LABEL: test_f32_ole_q:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE-32-NEXT:    ucomiss {{[0-9]+}}(%esp), %xmm0
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; SSE-32-NEXT:    cmovael %eax, %ecx
+; SSE-32-NEXT:    movl (%ecx), %eax
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_f32_ole_q:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movl %edi, %eax
+; SSE-64-NEXT:    ucomiss %xmm0, %xmm1
+; SSE-64-NEXT:    cmovbl %esi, %eax
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_f32_ole_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vucomiss {{[0-9]+}}(%esp), %xmm0
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; AVX-32-NEXT:    cmovael %eax, %ecx
+; AVX-32-NEXT:    movl (%ecx), %eax
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_f32_ole_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    movl %edi, %eax
+; AVX-64-NEXT:    vucomiss %xmm0, %xmm1
+; AVX-64-NEXT:    cmovbl %esi, %eax
+; AVX-64-NEXT:    retq
+;
+; X87-LABEL: test_f32_ole_q:
+; X87:       # %bb.0:
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    fucompp
+; X87-NEXT:    fnstsw %ax
+; X87-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-NEXT:    sahf
+; X87-NEXT:    jae .LBB4_1
+; X87-NEXT:  # %bb.2:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+; X87-NEXT:  .LBB4_1:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+;
+; X87-CMOV-LABEL: test_f32_ole_q:
+; X87-CMOV:       # %bb.0:
+; X87-CMOV-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fucompi %st(1), %st
+; X87-CMOV-NEXT:    fstp %st(0)
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X87-CMOV-NEXT:    cmovael %eax, %ecx
+; X87-CMOV-NEXT:    movl (%ecx), %eax
+; X87-CMOV-NEXT:    retl
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f32(
+                                               float %f1, float %f2, metadata !"ole",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_f32_one_q(i32 %a, i32 %b, float %f1, float %f2) #0 {
+; SSE-32-LABEL: test_f32_one_q:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE-32-NEXT:    ucomiss {{[0-9]+}}(%esp), %xmm0
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; SSE-32-NEXT:    cmovnel %eax, %ecx
+; SSE-32-NEXT:    movl (%ecx), %eax
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_f32_one_q:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movl %edi, %eax
+; SSE-64-NEXT:    ucomiss %xmm1, %xmm0
+; SSE-64-NEXT:    cmovel %esi, %eax
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_f32_one_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vucomiss {{[0-9]+}}(%esp), %xmm0
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; AVX-32-NEXT:    cmovnel %eax, %ecx
+; AVX-32-NEXT:    movl (%ecx), %eax
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_f32_one_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    movl %edi, %eax
+; AVX-64-NEXT:    vucomiss %xmm1, %xmm0
+; AVX-64-NEXT:    cmovel %esi, %eax
+; AVX-64-NEXT:    retq
+;
+; X87-LABEL: test_f32_one_q:
+; X87:       # %bb.0:
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    fucompp
+; X87-NEXT:    fnstsw %ax
+; X87-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-NEXT:    sahf
+; X87-NEXT:    jne .LBB5_1
+; X87-NEXT:  # %bb.2:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+; X87-NEXT:  .LBB5_1:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+;
+; X87-CMOV-LABEL: test_f32_one_q:
+; X87-CMOV:       # %bb.0:
+; X87-CMOV-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fucompi %st(1), %st
+; X87-CMOV-NEXT:    fstp %st(0)
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X87-CMOV-NEXT:    cmovnel %eax, %ecx
+; X87-CMOV-NEXT:    movl (%ecx), %eax
+; X87-CMOV-NEXT:    retl
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f32(
+                                               float %f1, float %f2, metadata !"one",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_f32_ord_q(i32 %a, i32 %b, float %f1, float %f2) #0 {
+; SSE-32-LABEL: test_f32_ord_q:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE-32-NEXT:    ucomiss {{[0-9]+}}(%esp), %xmm0
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; SSE-32-NEXT:    cmovnpl %eax, %ecx
+; SSE-32-NEXT:    movl (%ecx), %eax
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_f32_ord_q:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movl %edi, %eax
+; SSE-64-NEXT:    ucomiss %xmm1, %xmm0
+; SSE-64-NEXT:    cmovpl %esi, %eax
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_f32_ord_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vucomiss {{[0-9]+}}(%esp), %xmm0
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; AVX-32-NEXT:    cmovnpl %eax, %ecx
+; AVX-32-NEXT:    movl (%ecx), %eax
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_f32_ord_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    movl %edi, %eax
+; AVX-64-NEXT:    vucomiss %xmm1, %xmm0
+; AVX-64-NEXT:    cmovpl %esi, %eax
+; AVX-64-NEXT:    retq
+;
+; X87-LABEL: test_f32_ord_q:
+; X87:       # %bb.0:
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    fucompp
+; X87-NEXT:    fnstsw %ax
+; X87-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-NEXT:    sahf
+; X87-NEXT:    jnp .LBB6_1
+; X87-NEXT:  # %bb.2:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+; X87-NEXT:  .LBB6_1:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+;
+; X87-CMOV-LABEL: test_f32_ord_q:
+; X87-CMOV:       # %bb.0:
+; X87-CMOV-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fucompi %st(1), %st
+; X87-CMOV-NEXT:    fstp %st(0)
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X87-CMOV-NEXT:    cmovnpl %eax, %ecx
+; X87-CMOV-NEXT:    movl (%ecx), %eax
+; X87-CMOV-NEXT:    retl
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f32(
+                                               float %f1, float %f2, metadata !"ord",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_f32_ueq_q(i32 %a, i32 %b, float %f1, float %f2) #0 {
+; SSE-32-LABEL: test_f32_ueq_q:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE-32-NEXT:    ucomiss {{[0-9]+}}(%esp), %xmm0
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; SSE-32-NEXT:    cmovel %eax, %ecx
+; SSE-32-NEXT:    movl (%ecx), %eax
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_f32_ueq_q:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movl %edi, %eax
+; SSE-64-NEXT:    ucomiss %xmm1, %xmm0
+; SSE-64-NEXT:    cmovnel %esi, %eax
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_f32_ueq_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vucomiss {{[0-9]+}}(%esp), %xmm0
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; AVX-32-NEXT:    cmovel %eax, %ecx
+; AVX-32-NEXT:    movl (%ecx), %eax
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_f32_ueq_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    movl %edi, %eax
+; AVX-64-NEXT:    vucomiss %xmm1, %xmm0
+; AVX-64-NEXT:    cmovnel %esi, %eax
+; AVX-64-NEXT:    retq
+;
+; X87-LABEL: test_f32_ueq_q:
+; X87:       # %bb.0:
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    fucompp
+; X87-NEXT:    fnstsw %ax
+; X87-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-NEXT:    sahf
+; X87-NEXT:    je .LBB7_1
+; X87-NEXT:  # %bb.2:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+; X87-NEXT:  .LBB7_1:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+;
+; X87-CMOV-LABEL: test_f32_ueq_q:
+; X87-CMOV:       # %bb.0:
+; X87-CMOV-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fucompi %st(1), %st
+; X87-CMOV-NEXT:    fstp %st(0)
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X87-CMOV-NEXT:    cmovel %eax, %ecx
+; X87-CMOV-NEXT:    movl (%ecx), %eax
+; X87-CMOV-NEXT:    retl
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f32(
+                                               float %f1, float %f2, metadata !"ueq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_f32_ugt_q(i32 %a, i32 %b, float %f1, float %f2) #0 {
+; SSE-32-LABEL: test_f32_ugt_q:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE-32-NEXT:    ucomiss {{[0-9]+}}(%esp), %xmm0
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; SSE-32-NEXT:    cmovbl %eax, %ecx
+; SSE-32-NEXT:    movl (%ecx), %eax
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_f32_ugt_q:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movl %edi, %eax
+; SSE-64-NEXT:    ucomiss %xmm0, %xmm1
+; SSE-64-NEXT:    cmovael %esi, %eax
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_f32_ugt_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vucomiss {{[0-9]+}}(%esp), %xmm0
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; AVX-32-NEXT:    cmovbl %eax, %ecx
+; AVX-32-NEXT:    movl (%ecx), %eax
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_f32_ugt_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    movl %edi, %eax
+; AVX-64-NEXT:    vucomiss %xmm0, %xmm1
+; AVX-64-NEXT:    cmovael %esi, %eax
+; AVX-64-NEXT:    retq
+;
+; X87-LABEL: test_f32_ugt_q:
+; X87:       # %bb.0:
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    fucompp
+; X87-NEXT:    fnstsw %ax
+; X87-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-NEXT:    sahf
+; X87-NEXT:    jb .LBB8_1
+; X87-NEXT:  # %bb.2:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+; X87-NEXT:  .LBB8_1:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+;
+; X87-CMOV-LABEL: test_f32_ugt_q:
+; X87-CMOV:       # %bb.0:
+; X87-CMOV-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fucompi %st(1), %st
+; X87-CMOV-NEXT:    fstp %st(0)
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X87-CMOV-NEXT:    cmovbl %eax, %ecx
+; X87-CMOV-NEXT:    movl (%ecx), %eax
+; X87-CMOV-NEXT:    retl
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f32(
+                                               float %f1, float %f2, metadata !"ugt",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_f32_uge_q(i32 %a, i32 %b, float %f1, float %f2) #0 {
+; SSE-32-LABEL: test_f32_uge_q:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE-32-NEXT:    ucomiss {{[0-9]+}}(%esp), %xmm0
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; SSE-32-NEXT:    cmovbel %eax, %ecx
+; SSE-32-NEXT:    movl (%ecx), %eax
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_f32_uge_q:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movl %edi, %eax
+; SSE-64-NEXT:    ucomiss %xmm0, %xmm1
+; SSE-64-NEXT:    cmoval %esi, %eax
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_f32_uge_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vucomiss {{[0-9]+}}(%esp), %xmm0
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; AVX-32-NEXT:    cmovbel %eax, %ecx
+; AVX-32-NEXT:    movl (%ecx), %eax
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_f32_uge_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    movl %edi, %eax
+; AVX-64-NEXT:    vucomiss %xmm0, %xmm1
+; AVX-64-NEXT:    cmoval %esi, %eax
+; AVX-64-NEXT:    retq
+;
+; X87-LABEL: test_f32_uge_q:
+; X87:       # %bb.0:
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    fucompp
+; X87-NEXT:    fnstsw %ax
+; X87-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-NEXT:    sahf
+; X87-NEXT:    jbe .LBB9_1
+; X87-NEXT:  # %bb.2:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+; X87-NEXT:  .LBB9_1:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+;
+; X87-CMOV-LABEL: test_f32_uge_q:
+; X87-CMOV:       # %bb.0:
+; X87-CMOV-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fucompi %st(1), %st
+; X87-CMOV-NEXT:    fstp %st(0)
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X87-CMOV-NEXT:    cmovbel %eax, %ecx
+; X87-CMOV-NEXT:    movl (%ecx), %eax
+; X87-CMOV-NEXT:    retl
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f32(
+                                               float %f1, float %f2, metadata !"uge",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_f32_ult_q(i32 %a, i32 %b, float %f1, float %f2) #0 {
+; SSE-32-LABEL: test_f32_ult_q:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE-32-NEXT:    ucomiss {{[0-9]+}}(%esp), %xmm0
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; SSE-32-NEXT:    cmovbl %eax, %ecx
+; SSE-32-NEXT:    movl (%ecx), %eax
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_f32_ult_q:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movl %edi, %eax
+; SSE-64-NEXT:    ucomiss %xmm1, %xmm0
+; SSE-64-NEXT:    cmovael %esi, %eax
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_f32_ult_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vucomiss {{[0-9]+}}(%esp), %xmm0
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; AVX-32-NEXT:    cmovbl %eax, %ecx
+; AVX-32-NEXT:    movl (%ecx), %eax
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_f32_ult_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    movl %edi, %eax
+; AVX-64-NEXT:    vucomiss %xmm1, %xmm0
+; AVX-64-NEXT:    cmovael %esi, %eax
+; AVX-64-NEXT:    retq
+;
+; X87-LABEL: test_f32_ult_q:
+; X87:       # %bb.0:
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    fucompp
+; X87-NEXT:    fnstsw %ax
+; X87-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-NEXT:    sahf
+; X87-NEXT:    jb .LBB10_1
+; X87-NEXT:  # %bb.2:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+; X87-NEXT:  .LBB10_1:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+;
+; X87-CMOV-LABEL: test_f32_ult_q:
+; X87-CMOV:       # %bb.0:
+; X87-CMOV-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fucompi %st(1), %st
+; X87-CMOV-NEXT:    fstp %st(0)
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X87-CMOV-NEXT:    cmovbl %eax, %ecx
+; X87-CMOV-NEXT:    movl (%ecx), %eax
+; X87-CMOV-NEXT:    retl
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f32(
+                                               float %f1, float %f2, metadata !"ult",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_f32_ule_q(i32 %a, i32 %b, float %f1, float %f2) #0 {
+; SSE-32-LABEL: test_f32_ule_q:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE-32-NEXT:    ucomiss {{[0-9]+}}(%esp), %xmm0
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; SSE-32-NEXT:    cmovbel %eax, %ecx
+; SSE-32-NEXT:    movl (%ecx), %eax
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_f32_ule_q:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movl %edi, %eax
+; SSE-64-NEXT:    ucomiss %xmm1, %xmm0
+; SSE-64-NEXT:    cmoval %esi, %eax
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_f32_ule_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vucomiss {{[0-9]+}}(%esp), %xmm0
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; AVX-32-NEXT:    cmovbel %eax, %ecx
+; AVX-32-NEXT:    movl (%ecx), %eax
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_f32_ule_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    movl %edi, %eax
+; AVX-64-NEXT:    vucomiss %xmm1, %xmm0
+; AVX-64-NEXT:    cmoval %esi, %eax
+; AVX-64-NEXT:    retq
+;
+; X87-LABEL: test_f32_ule_q:
+; X87:       # %bb.0:
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    fucompp
+; X87-NEXT:    fnstsw %ax
+; X87-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-NEXT:    sahf
+; X87-NEXT:    jbe .LBB11_1
+; X87-NEXT:  # %bb.2:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+; X87-NEXT:  .LBB11_1:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+;
+; X87-CMOV-LABEL: test_f32_ule_q:
+; X87-CMOV:       # %bb.0:
+; X87-CMOV-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fucompi %st(1), %st
+; X87-CMOV-NEXT:    fstp %st(0)
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X87-CMOV-NEXT:    cmovbel %eax, %ecx
+; X87-CMOV-NEXT:    movl (%ecx), %eax
+; X87-CMOV-NEXT:    retl
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f32(
+                                               float %f1, float %f2, metadata !"ule",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_f32_une_q(i32 %a, i32 %b, float %f1, float %f2) #0 {
+; SSE-32-LABEL: test_f32_une_q:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE-32-NEXT:    ucomiss {{[0-9]+}}(%esp), %xmm0
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; SSE-32-NEXT:    cmovnel %eax, %ecx
+; SSE-32-NEXT:    cmovpl %eax, %ecx
+; SSE-32-NEXT:    movl (%ecx), %eax
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_f32_une_q:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movl %esi, %eax
+; SSE-64-NEXT:    ucomiss %xmm1, %xmm0
+; SSE-64-NEXT:    cmovnel %edi, %eax
+; SSE-64-NEXT:    cmovpl %edi, %eax
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_f32_une_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vucomiss {{[0-9]+}}(%esp), %xmm0
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; AVX-32-NEXT:    cmovnel %eax, %ecx
+; AVX-32-NEXT:    cmovpl %eax, %ecx
+; AVX-32-NEXT:    movl (%ecx), %eax
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_f32_une_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    movl %esi, %eax
+; AVX-64-NEXT:    vucomiss %xmm1, %xmm0
+; AVX-64-NEXT:    cmovnel %edi, %eax
+; AVX-64-NEXT:    cmovpl %edi, %eax
+; AVX-64-NEXT:    retq
+;
+; X87-LABEL: test_f32_une_q:
+; X87:       # %bb.0:
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    fucompp
+; X87-NEXT:    fnstsw %ax
+; X87-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-NEXT:    sahf
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    jne .LBB12_3
+; X87-NEXT:  # %bb.1:
+; X87-NEXT:    jp .LBB12_3
+; X87-NEXT:  # %bb.2:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:  .LBB12_3:
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+;
+; X87-CMOV-LABEL: test_f32_une_q:
+; X87-CMOV:       # %bb.0:
+; X87-CMOV-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fucompi %st(1), %st
+; X87-CMOV-NEXT:    fstp %st(0)
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X87-CMOV-NEXT:    cmovnel %eax, %ecx
+; X87-CMOV-NEXT:    cmovpl %eax, %ecx
+; X87-CMOV-NEXT:    movl (%ecx), %eax
+; X87-CMOV-NEXT:    retl
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f32(
+                                               float %f1, float %f2, metadata !"une",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_f32_uno_q(i32 %a, i32 %b, float %f1, float %f2) #0 {
+; SSE-32-LABEL: test_f32_uno_q:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE-32-NEXT:    ucomiss {{[0-9]+}}(%esp), %xmm0
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; SSE-32-NEXT:    cmovpl %eax, %ecx
+; SSE-32-NEXT:    movl (%ecx), %eax
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_f32_uno_q:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movl %edi, %eax
+; SSE-64-NEXT:    ucomiss %xmm1, %xmm0
+; SSE-64-NEXT:    cmovnpl %esi, %eax
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_f32_uno_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vucomiss {{[0-9]+}}(%esp), %xmm0
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; AVX-32-NEXT:    cmovpl %eax, %ecx
+; AVX-32-NEXT:    movl (%ecx), %eax
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_f32_uno_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    movl %edi, %eax
+; AVX-64-NEXT:    vucomiss %xmm1, %xmm0
+; AVX-64-NEXT:    cmovnpl %esi, %eax
+; AVX-64-NEXT:    retq
+;
+; X87-LABEL: test_f32_uno_q:
+; X87:       # %bb.0:
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    fucompp
+; X87-NEXT:    fnstsw %ax
+; X87-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-NEXT:    sahf
+; X87-NEXT:    jp .LBB13_1
+; X87-NEXT:  # %bb.2:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+; X87-NEXT:  .LBB13_1:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+;
+; X87-CMOV-LABEL: test_f32_uno_q:
+; X87-CMOV:       # %bb.0:
+; X87-CMOV-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fucompi %st(1), %st
+; X87-CMOV-NEXT:    fstp %st(0)
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X87-CMOV-NEXT:    cmovpl %eax, %ecx
+; X87-CMOV-NEXT:    movl (%ecx), %eax
+; X87-CMOV-NEXT:    retl
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f32(
+                                               float %f1, float %f2, metadata !"uno",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_f64_oeq_q(i32 %a, i32 %b, double %f1, double %f2) #0 {
+; SSE-32-LABEL: test_f64_oeq_q:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE-32-NEXT:    ucomisd {{[0-9]+}}(%esp), %xmm0
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; SSE-32-NEXT:    cmovnel %eax, %ecx
+; SSE-32-NEXT:    cmovpl %eax, %ecx
+; SSE-32-NEXT:    movl (%ecx), %eax
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_f64_oeq_q:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movl %edi, %eax
+; SSE-64-NEXT:    ucomisd %xmm1, %xmm0
+; SSE-64-NEXT:    cmovnel %esi, %eax
+; SSE-64-NEXT:    cmovpl %esi, %eax
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_f64_oeq_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-32-NEXT:    vucomisd {{[0-9]+}}(%esp), %xmm0
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; AVX-32-NEXT:    cmovnel %eax, %ecx
+; AVX-32-NEXT:    cmovpl %eax, %ecx
+; AVX-32-NEXT:    movl (%ecx), %eax
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_f64_oeq_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    movl %edi, %eax
+; AVX-64-NEXT:    vucomisd %xmm1, %xmm0
+; AVX-64-NEXT:    cmovnel %esi, %eax
+; AVX-64-NEXT:    cmovpl %esi, %eax
+; AVX-64-NEXT:    retq
+;
+; X87-LABEL: test_f64_oeq_q:
+; X87:       # %bb.0:
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fucompp
+; X87-NEXT:    fnstsw %ax
+; X87-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-NEXT:    sahf
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    jne .LBB14_3
+; X87-NEXT:  # %bb.1:
+; X87-NEXT:    jp .LBB14_3
+; X87-NEXT:  # %bb.2:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:  .LBB14_3:
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+;
+; X87-CMOV-LABEL: test_f64_oeq_q:
+; X87-CMOV:       # %bb.0:
+; X87-CMOV-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fucompi %st(1), %st
+; X87-CMOV-NEXT:    fstp %st(0)
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X87-CMOV-NEXT:    cmovnel %eax, %ecx
+; X87-CMOV-NEXT:    cmovpl %eax, %ecx
+; X87-CMOV-NEXT:    movl (%ecx), %eax
+; X87-CMOV-NEXT:    retl
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f64(
+                                               double %f1, double %f2, metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_f64_ogt_q(i32 %a, i32 %b, double %f1, double %f2) #0 {
+; SSE-32-LABEL: test_f64_ogt_q:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE-32-NEXT:    ucomisd {{[0-9]+}}(%esp), %xmm0
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; SSE-32-NEXT:    cmoval %eax, %ecx
+; SSE-32-NEXT:    movl (%ecx), %eax
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_f64_ogt_q:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movl %edi, %eax
+; SSE-64-NEXT:    ucomisd %xmm1, %xmm0
+; SSE-64-NEXT:    cmovbel %esi, %eax
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_f64_ogt_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-32-NEXT:    vucomisd {{[0-9]+}}(%esp), %xmm0
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; AVX-32-NEXT:    cmoval %eax, %ecx
+; AVX-32-NEXT:    movl (%ecx), %eax
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_f64_ogt_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    movl %edi, %eax
+; AVX-64-NEXT:    vucomisd %xmm1, %xmm0
+; AVX-64-NEXT:    cmovbel %esi, %eax
+; AVX-64-NEXT:    retq
+;
+; X87-LABEL: test_f64_ogt_q:
+; X87:       # %bb.0:
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fucompp
+; X87-NEXT:    fnstsw %ax
+; X87-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-NEXT:    sahf
+; X87-NEXT:    ja .LBB15_1
+; X87-NEXT:  # %bb.2:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+; X87-NEXT:  .LBB15_1:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+;
+; X87-CMOV-LABEL: test_f64_ogt_q:
+; X87-CMOV:       # %bb.0:
+; X87-CMOV-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fucompi %st(1), %st
+; X87-CMOV-NEXT:    fstp %st(0)
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X87-CMOV-NEXT:    cmoval %eax, %ecx
+; X87-CMOV-NEXT:    movl (%ecx), %eax
+; X87-CMOV-NEXT:    retl
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f64(
+                                               double %f1, double %f2, metadata !"ogt",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_f64_oge_q(i32 %a, i32 %b, double %f1, double %f2) #0 {
+; SSE-32-LABEL: test_f64_oge_q:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE-32-NEXT:    ucomisd {{[0-9]+}}(%esp), %xmm0
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; SSE-32-NEXT:    cmovael %eax, %ecx
+; SSE-32-NEXT:    movl (%ecx), %eax
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_f64_oge_q:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movl %edi, %eax
+; SSE-64-NEXT:    ucomisd %xmm1, %xmm0
+; SSE-64-NEXT:    cmovbl %esi, %eax
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_f64_oge_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-32-NEXT:    vucomisd {{[0-9]+}}(%esp), %xmm0
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; AVX-32-NEXT:    cmovael %eax, %ecx
+; AVX-32-NEXT:    movl (%ecx), %eax
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_f64_oge_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    movl %edi, %eax
+; AVX-64-NEXT:    vucomisd %xmm1, %xmm0
+; AVX-64-NEXT:    cmovbl %esi, %eax
+; AVX-64-NEXT:    retq
+;
+; X87-LABEL: test_f64_oge_q:
+; X87:       # %bb.0:
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fucompp
+; X87-NEXT:    fnstsw %ax
+; X87-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-NEXT:    sahf
+; X87-NEXT:    jae .LBB16_1
+; X87-NEXT:  # %bb.2:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+; X87-NEXT:  .LBB16_1:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+;
+; X87-CMOV-LABEL: test_f64_oge_q:
+; X87-CMOV:       # %bb.0:
+; X87-CMOV-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fucompi %st(1), %st
+; X87-CMOV-NEXT:    fstp %st(0)
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X87-CMOV-NEXT:    cmovael %eax, %ecx
+; X87-CMOV-NEXT:    movl (%ecx), %eax
+; X87-CMOV-NEXT:    retl
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f64(
+                                               double %f1, double %f2, metadata !"oge",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_f64_olt_q(i32 %a, i32 %b, double %f1, double %f2) #0 {
+; SSE-32-LABEL: test_f64_olt_q:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE-32-NEXT:    ucomisd {{[0-9]+}}(%esp), %xmm0
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; SSE-32-NEXT:    cmoval %eax, %ecx
+; SSE-32-NEXT:    movl (%ecx), %eax
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_f64_olt_q:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movl %edi, %eax
+; SSE-64-NEXT:    ucomisd %xmm0, %xmm1
+; SSE-64-NEXT:    cmovbel %esi, %eax
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_f64_olt_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-32-NEXT:    vucomisd {{[0-9]+}}(%esp), %xmm0
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; AVX-32-NEXT:    cmoval %eax, %ecx
+; AVX-32-NEXT:    movl (%ecx), %eax
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_f64_olt_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    movl %edi, %eax
+; AVX-64-NEXT:    vucomisd %xmm0, %xmm1
+; AVX-64-NEXT:    cmovbel %esi, %eax
+; AVX-64-NEXT:    retq
+;
+; X87-LABEL: test_f64_olt_q:
+; X87:       # %bb.0:
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fucompp
+; X87-NEXT:    fnstsw %ax
+; X87-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-NEXT:    sahf
+; X87-NEXT:    ja .LBB17_1
+; X87-NEXT:  # %bb.2:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+; X87-NEXT:  .LBB17_1:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+;
+; X87-CMOV-LABEL: test_f64_olt_q:
+; X87-CMOV:       # %bb.0:
+; X87-CMOV-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fucompi %st(1), %st
+; X87-CMOV-NEXT:    fstp %st(0)
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X87-CMOV-NEXT:    cmoval %eax, %ecx
+; X87-CMOV-NEXT:    movl (%ecx), %eax
+; X87-CMOV-NEXT:    retl
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f64(
+                                               double %f1, double %f2, metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_f64_ole_q(i32 %a, i32 %b, double %f1, double %f2) #0 {
+; SSE-32-LABEL: test_f64_ole_q:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE-32-NEXT:    ucomisd {{[0-9]+}}(%esp), %xmm0
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; SSE-32-NEXT:    cmovael %eax, %ecx
+; SSE-32-NEXT:    movl (%ecx), %eax
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_f64_ole_q:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movl %edi, %eax
+; SSE-64-NEXT:    ucomisd %xmm0, %xmm1
+; SSE-64-NEXT:    cmovbl %esi, %eax
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_f64_ole_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-32-NEXT:    vucomisd {{[0-9]+}}(%esp), %xmm0
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; AVX-32-NEXT:    cmovael %eax, %ecx
+; AVX-32-NEXT:    movl (%ecx), %eax
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_f64_ole_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    movl %edi, %eax
+; AVX-64-NEXT:    vucomisd %xmm0, %xmm1
+; AVX-64-NEXT:    cmovbl %esi, %eax
+; AVX-64-NEXT:    retq
+;
+; X87-LABEL: test_f64_ole_q:
+; X87:       # %bb.0:
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fucompp
+; X87-NEXT:    fnstsw %ax
+; X87-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-NEXT:    sahf
+; X87-NEXT:    jae .LBB18_1
+; X87-NEXT:  # %bb.2:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+; X87-NEXT:  .LBB18_1:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+;
+; X87-CMOV-LABEL: test_f64_ole_q:
+; X87-CMOV:       # %bb.0:
+; X87-CMOV-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fucompi %st(1), %st
+; X87-CMOV-NEXT:    fstp %st(0)
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X87-CMOV-NEXT:    cmovael %eax, %ecx
+; X87-CMOV-NEXT:    movl (%ecx), %eax
+; X87-CMOV-NEXT:    retl
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f64(
+                                               double %f1, double %f2, metadata !"ole",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_f64_one_q(i32 %a, i32 %b, double %f1, double %f2) #0 {
+; SSE-32-LABEL: test_f64_one_q:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE-32-NEXT:    ucomisd {{[0-9]+}}(%esp), %xmm0
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; SSE-32-NEXT:    cmovnel %eax, %ecx
+; SSE-32-NEXT:    movl (%ecx), %eax
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_f64_one_q:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movl %edi, %eax
+; SSE-64-NEXT:    ucomisd %xmm1, %xmm0
+; SSE-64-NEXT:    cmovel %esi, %eax
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_f64_one_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-32-NEXT:    vucomisd {{[0-9]+}}(%esp), %xmm0
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; AVX-32-NEXT:    cmovnel %eax, %ecx
+; AVX-32-NEXT:    movl (%ecx), %eax
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_f64_one_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    movl %edi, %eax
+; AVX-64-NEXT:    vucomisd %xmm1, %xmm0
+; AVX-64-NEXT:    cmovel %esi, %eax
+; AVX-64-NEXT:    retq
+;
+; X87-LABEL: test_f64_one_q:
+; X87:       # %bb.0:
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fucompp
+; X87-NEXT:    fnstsw %ax
+; X87-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-NEXT:    sahf
+; X87-NEXT:    jne .LBB19_1
+; X87-NEXT:  # %bb.2:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+; X87-NEXT:  .LBB19_1:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+;
+; X87-CMOV-LABEL: test_f64_one_q:
+; X87-CMOV:       # %bb.0:
+; X87-CMOV-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fucompi %st(1), %st
+; X87-CMOV-NEXT:    fstp %st(0)
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X87-CMOV-NEXT:    cmovnel %eax, %ecx
+; X87-CMOV-NEXT:    movl (%ecx), %eax
+; X87-CMOV-NEXT:    retl
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f64(
+                                               double %f1, double %f2, metadata !"one",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_f64_ord_q(i32 %a, i32 %b, double %f1, double %f2) #0 {
+; SSE-32-LABEL: test_f64_ord_q:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE-32-NEXT:    ucomisd {{[0-9]+}}(%esp), %xmm0
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; SSE-32-NEXT:    cmovnpl %eax, %ecx
+; SSE-32-NEXT:    movl (%ecx), %eax
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_f64_ord_q:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movl %edi, %eax
+; SSE-64-NEXT:    ucomisd %xmm1, %xmm0
+; SSE-64-NEXT:    cmovpl %esi, %eax
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_f64_ord_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-32-NEXT:    vucomisd {{[0-9]+}}(%esp), %xmm0
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; AVX-32-NEXT:    cmovnpl %eax, %ecx
+; AVX-32-NEXT:    movl (%ecx), %eax
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_f64_ord_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    movl %edi, %eax
+; AVX-64-NEXT:    vucomisd %xmm1, %xmm0
+; AVX-64-NEXT:    cmovpl %esi, %eax
+; AVX-64-NEXT:    retq
+;
+; X87-LABEL: test_f64_ord_q:
+; X87:       # %bb.0:
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fucompp
+; X87-NEXT:    fnstsw %ax
+; X87-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-NEXT:    sahf
+; X87-NEXT:    jnp .LBB20_1
+; X87-NEXT:  # %bb.2:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+; X87-NEXT:  .LBB20_1:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+;
+; X87-CMOV-LABEL: test_f64_ord_q:
+; X87-CMOV:       # %bb.0:
+; X87-CMOV-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fucompi %st(1), %st
+; X87-CMOV-NEXT:    fstp %st(0)
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X87-CMOV-NEXT:    cmovnpl %eax, %ecx
+; X87-CMOV-NEXT:    movl (%ecx), %eax
+; X87-CMOV-NEXT:    retl
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f64(
+                                               double %f1, double %f2, metadata !"ord",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_f64_ueq_q(i32 %a, i32 %b, double %f1, double %f2) #0 {
+; SSE-32-LABEL: test_f64_ueq_q:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE-32-NEXT:    ucomisd {{[0-9]+}}(%esp), %xmm0
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; SSE-32-NEXT:    cmovel %eax, %ecx
+; SSE-32-NEXT:    movl (%ecx), %eax
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_f64_ueq_q:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movl %edi, %eax
+; SSE-64-NEXT:    ucomisd %xmm1, %xmm0
+; SSE-64-NEXT:    cmovnel %esi, %eax
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_f64_ueq_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-32-NEXT:    vucomisd {{[0-9]+}}(%esp), %xmm0
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; AVX-32-NEXT:    cmovel %eax, %ecx
+; AVX-32-NEXT:    movl (%ecx), %eax
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_f64_ueq_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    movl %edi, %eax
+; AVX-64-NEXT:    vucomisd %xmm1, %xmm0
+; AVX-64-NEXT:    cmovnel %esi, %eax
+; AVX-64-NEXT:    retq
+;
+; X87-LABEL: test_f64_ueq_q:
+; X87:       # %bb.0:
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fucompp
+; X87-NEXT:    fnstsw %ax
+; X87-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-NEXT:    sahf
+; X87-NEXT:    je .LBB21_1
+; X87-NEXT:  # %bb.2:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+; X87-NEXT:  .LBB21_1:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+;
+; X87-CMOV-LABEL: test_f64_ueq_q:
+; X87-CMOV:       # %bb.0:
+; X87-CMOV-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fucompi %st(1), %st
+; X87-CMOV-NEXT:    fstp %st(0)
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X87-CMOV-NEXT:    cmovel %eax, %ecx
+; X87-CMOV-NEXT:    movl (%ecx), %eax
+; X87-CMOV-NEXT:    retl
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f64(
+                                               double %f1, double %f2, metadata !"ueq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_f64_ugt_q(i32 %a, i32 %b, double %f1, double %f2) #0 {
+; SSE-32-LABEL: test_f64_ugt_q:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE-32-NEXT:    ucomisd {{[0-9]+}}(%esp), %xmm0
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; SSE-32-NEXT:    cmovbl %eax, %ecx
+; SSE-32-NEXT:    movl (%ecx), %eax
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_f64_ugt_q:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movl %edi, %eax
+; SSE-64-NEXT:    ucomisd %xmm0, %xmm1
+; SSE-64-NEXT:    cmovael %esi, %eax
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_f64_ugt_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-32-NEXT:    vucomisd {{[0-9]+}}(%esp), %xmm0
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; AVX-32-NEXT:    cmovbl %eax, %ecx
+; AVX-32-NEXT:    movl (%ecx), %eax
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_f64_ugt_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    movl %edi, %eax
+; AVX-64-NEXT:    vucomisd %xmm0, %xmm1
+; AVX-64-NEXT:    cmovael %esi, %eax
+; AVX-64-NEXT:    retq
+;
+; X87-LABEL: test_f64_ugt_q:
+; X87:       # %bb.0:
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fucompp
+; X87-NEXT:    fnstsw %ax
+; X87-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-NEXT:    sahf
+; X87-NEXT:    jb .LBB22_1
+; X87-NEXT:  # %bb.2:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+; X87-NEXT:  .LBB22_1:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+;
+; X87-CMOV-LABEL: test_f64_ugt_q:
+; X87-CMOV:       # %bb.0:
+; X87-CMOV-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fucompi %st(1), %st
+; X87-CMOV-NEXT:    fstp %st(0)
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X87-CMOV-NEXT:    cmovbl %eax, %ecx
+; X87-CMOV-NEXT:    movl (%ecx), %eax
+; X87-CMOV-NEXT:    retl
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f64(
+                                               double %f1, double %f2, metadata !"ugt",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_f64_uge_q(i32 %a, i32 %b, double %f1, double %f2) #0 {
+; SSE-32-LABEL: test_f64_uge_q:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE-32-NEXT:    ucomisd {{[0-9]+}}(%esp), %xmm0
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; SSE-32-NEXT:    cmovbel %eax, %ecx
+; SSE-32-NEXT:    movl (%ecx), %eax
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_f64_uge_q:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movl %edi, %eax
+; SSE-64-NEXT:    ucomisd %xmm0, %xmm1
+; SSE-64-NEXT:    cmoval %esi, %eax
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_f64_uge_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-32-NEXT:    vucomisd {{[0-9]+}}(%esp), %xmm0
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; AVX-32-NEXT:    cmovbel %eax, %ecx
+; AVX-32-NEXT:    movl (%ecx), %eax
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_f64_uge_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    movl %edi, %eax
+; AVX-64-NEXT:    vucomisd %xmm0, %xmm1
+; AVX-64-NEXT:    cmoval %esi, %eax
+; AVX-64-NEXT:    retq
+;
+; X87-LABEL: test_f64_uge_q:
+; X87:       # %bb.0:
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fucompp
+; X87-NEXT:    fnstsw %ax
+; X87-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-NEXT:    sahf
+; X87-NEXT:    jbe .LBB23_1
+; X87-NEXT:  # %bb.2:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+; X87-NEXT:  .LBB23_1:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+;
+; X87-CMOV-LABEL: test_f64_uge_q:
+; X87-CMOV:       # %bb.0:
+; X87-CMOV-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fucompi %st(1), %st
+; X87-CMOV-NEXT:    fstp %st(0)
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X87-CMOV-NEXT:    cmovbel %eax, %ecx
+; X87-CMOV-NEXT:    movl (%ecx), %eax
+; X87-CMOV-NEXT:    retl
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f64(
+                                               double %f1, double %f2, metadata !"uge",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_f64_ult_q(i32 %a, i32 %b, double %f1, double %f2) #0 {
+; SSE-32-LABEL: test_f64_ult_q:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE-32-NEXT:    ucomisd {{[0-9]+}}(%esp), %xmm0
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; SSE-32-NEXT:    cmovbl %eax, %ecx
+; SSE-32-NEXT:    movl (%ecx), %eax
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_f64_ult_q:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movl %edi, %eax
+; SSE-64-NEXT:    ucomisd %xmm1, %xmm0
+; SSE-64-NEXT:    cmovael %esi, %eax
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_f64_ult_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-32-NEXT:    vucomisd {{[0-9]+}}(%esp), %xmm0
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; AVX-32-NEXT:    cmovbl %eax, %ecx
+; AVX-32-NEXT:    movl (%ecx), %eax
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_f64_ult_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    movl %edi, %eax
+; AVX-64-NEXT:    vucomisd %xmm1, %xmm0
+; AVX-64-NEXT:    cmovael %esi, %eax
+; AVX-64-NEXT:    retq
+;
+; X87-LABEL: test_f64_ult_q:
+; X87:       # %bb.0:
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fucompp
+; X87-NEXT:    fnstsw %ax
+; X87-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-NEXT:    sahf
+; X87-NEXT:    jb .LBB24_1
+; X87-NEXT:  # %bb.2:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+; X87-NEXT:  .LBB24_1:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+;
+; X87-CMOV-LABEL: test_f64_ult_q:
+; X87-CMOV:       # %bb.0:
+; X87-CMOV-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fucompi %st(1), %st
+; X87-CMOV-NEXT:    fstp %st(0)
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X87-CMOV-NEXT:    cmovbl %eax, %ecx
+; X87-CMOV-NEXT:    movl (%ecx), %eax
+; X87-CMOV-NEXT:    retl
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f64(
+                                               double %f1, double %f2, metadata !"ult",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_f64_ule_q(i32 %a, i32 %b, double %f1, double %f2) #0 {
+; SSE-32-LABEL: test_f64_ule_q:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE-32-NEXT:    ucomisd {{[0-9]+}}(%esp), %xmm0
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; SSE-32-NEXT:    cmovbel %eax, %ecx
+; SSE-32-NEXT:    movl (%ecx), %eax
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_f64_ule_q:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movl %edi, %eax
+; SSE-64-NEXT:    ucomisd %xmm1, %xmm0
+; SSE-64-NEXT:    cmoval %esi, %eax
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_f64_ule_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-32-NEXT:    vucomisd {{[0-9]+}}(%esp), %xmm0
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; AVX-32-NEXT:    cmovbel %eax, %ecx
+; AVX-32-NEXT:    movl (%ecx), %eax
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_f64_ule_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    movl %edi, %eax
+; AVX-64-NEXT:    vucomisd %xmm1, %xmm0
+; AVX-64-NEXT:    cmoval %esi, %eax
+; AVX-64-NEXT:    retq
+;
+; X87-LABEL: test_f64_ule_q:
+; X87:       # %bb.0:
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fucompp
+; X87-NEXT:    fnstsw %ax
+; X87-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-NEXT:    sahf
+; X87-NEXT:    jbe .LBB25_1
+; X87-NEXT:  # %bb.2:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+; X87-NEXT:  .LBB25_1:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+;
+; X87-CMOV-LABEL: test_f64_ule_q:
+; X87-CMOV:       # %bb.0:
+; X87-CMOV-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fucompi %st(1), %st
+; X87-CMOV-NEXT:    fstp %st(0)
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X87-CMOV-NEXT:    cmovbel %eax, %ecx
+; X87-CMOV-NEXT:    movl (%ecx), %eax
+; X87-CMOV-NEXT:    retl
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f64(
+                                               double %f1, double %f2, metadata !"ule",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_f64_une_q(i32 %a, i32 %b, double %f1, double %f2) #0 {
+; SSE-32-LABEL: test_f64_une_q:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE-32-NEXT:    ucomisd {{[0-9]+}}(%esp), %xmm0
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; SSE-32-NEXT:    cmovnel %eax, %ecx
+; SSE-32-NEXT:    cmovpl %eax, %ecx
+; SSE-32-NEXT:    movl (%ecx), %eax
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_f64_une_q:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movl %esi, %eax
+; SSE-64-NEXT:    ucomisd %xmm1, %xmm0
+; SSE-64-NEXT:    cmovnel %edi, %eax
+; SSE-64-NEXT:    cmovpl %edi, %eax
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_f64_une_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-32-NEXT:    vucomisd {{[0-9]+}}(%esp), %xmm0
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; AVX-32-NEXT:    cmovnel %eax, %ecx
+; AVX-32-NEXT:    cmovpl %eax, %ecx
+; AVX-32-NEXT:    movl (%ecx), %eax
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_f64_une_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    movl %esi, %eax
+; AVX-64-NEXT:    vucomisd %xmm1, %xmm0
+; AVX-64-NEXT:    cmovnel %edi, %eax
+; AVX-64-NEXT:    cmovpl %edi, %eax
+; AVX-64-NEXT:    retq
+;
+; X87-LABEL: test_f64_une_q:
+; X87:       # %bb.0:
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fucompp
+; X87-NEXT:    fnstsw %ax
+; X87-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-NEXT:    sahf
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    jne .LBB26_3
+; X87-NEXT:  # %bb.1:
+; X87-NEXT:    jp .LBB26_3
+; X87-NEXT:  # %bb.2:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:  .LBB26_3:
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+;
+; X87-CMOV-LABEL: test_f64_une_q:
+; X87-CMOV:       # %bb.0:
+; X87-CMOV-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fucompi %st(1), %st
+; X87-CMOV-NEXT:    fstp %st(0)
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X87-CMOV-NEXT:    cmovnel %eax, %ecx
+; X87-CMOV-NEXT:    cmovpl %eax, %ecx
+; X87-CMOV-NEXT:    movl (%ecx), %eax
+; X87-CMOV-NEXT:    retl
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f64(
+                                               double %f1, double %f2, metadata !"une",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_f64_uno_q(i32 %a, i32 %b, double %f1, double %f2) #0 {
+; SSE-32-LABEL: test_f64_uno_q:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE-32-NEXT:    ucomisd {{[0-9]+}}(%esp), %xmm0
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; SSE-32-NEXT:    cmovpl %eax, %ecx
+; SSE-32-NEXT:    movl (%ecx), %eax
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_f64_uno_q:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movl %edi, %eax
+; SSE-64-NEXT:    ucomisd %xmm1, %xmm0
+; SSE-64-NEXT:    cmovnpl %esi, %eax
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_f64_uno_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-32-NEXT:    vucomisd {{[0-9]+}}(%esp), %xmm0
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; AVX-32-NEXT:    cmovpl %eax, %ecx
+; AVX-32-NEXT:    movl (%ecx), %eax
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_f64_uno_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    movl %edi, %eax
+; AVX-64-NEXT:    vucomisd %xmm1, %xmm0
+; AVX-64-NEXT:    cmovnpl %esi, %eax
+; AVX-64-NEXT:    retq
+;
+; X87-LABEL: test_f64_uno_q:
+; X87:       # %bb.0:
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fucompp
+; X87-NEXT:    fnstsw %ax
+; X87-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-NEXT:    sahf
+; X87-NEXT:    jp .LBB27_1
+; X87-NEXT:  # %bb.2:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+; X87-NEXT:  .LBB27_1:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+;
+; X87-CMOV-LABEL: test_f64_uno_q:
+; X87-CMOV:       # %bb.0:
+; X87-CMOV-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fucompi %st(1), %st
+; X87-CMOV-NEXT:    fstp %st(0)
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X87-CMOV-NEXT:    cmovpl %eax, %ecx
+; X87-CMOV-NEXT:    movl (%ecx), %eax
+; X87-CMOV-NEXT:    retl
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f64(
+                                               double %f1, double %f2, metadata !"uno",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_f32_oeq_s(i32 %a, i32 %b, float %f1, float %f2) #0 {
+; SSE-32-LABEL: test_f32_oeq_s:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE-32-NEXT:    comiss {{[0-9]+}}(%esp), %xmm0
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; SSE-32-NEXT:    cmovnel %eax, %ecx
+; SSE-32-NEXT:    cmovpl %eax, %ecx
+; SSE-32-NEXT:    movl (%ecx), %eax
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_f32_oeq_s:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movl %edi, %eax
+; SSE-64-NEXT:    comiss %xmm1, %xmm0
+; SSE-64-NEXT:    cmovnel %esi, %eax
+; SSE-64-NEXT:    cmovpl %esi, %eax
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_f32_oeq_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vcomiss {{[0-9]+}}(%esp), %xmm0
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; AVX-32-NEXT:    cmovnel %eax, %ecx
+; AVX-32-NEXT:    cmovpl %eax, %ecx
+; AVX-32-NEXT:    movl (%ecx), %eax
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_f32_oeq_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    movl %edi, %eax
+; AVX-64-NEXT:    vcomiss %xmm1, %xmm0
+; AVX-64-NEXT:    cmovnel %esi, %eax
+; AVX-64-NEXT:    cmovpl %esi, %eax
+; AVX-64-NEXT:    retq
+;
+; X87-LABEL: test_f32_oeq_s:
+; X87:       # %bb.0:
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    fcompp
+; X87-NEXT:    fnstsw %ax
+; X87-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-NEXT:    sahf
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    jne .LBB28_3
+; X87-NEXT:  # %bb.1:
+; X87-NEXT:    jp .LBB28_3
+; X87-NEXT:  # %bb.2:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:  .LBB28_3:
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+;
+; X87-CMOV-LABEL: test_f32_oeq_s:
+; X87-CMOV:       # %bb.0:
+; X87-CMOV-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fcompi %st(1), %st
+; X87-CMOV-NEXT:    fstp %st(0)
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X87-CMOV-NEXT:    cmovnel %eax, %ecx
+; X87-CMOV-NEXT:    cmovpl %eax, %ecx
+; X87-CMOV-NEXT:    movl (%ecx), %eax
+; X87-CMOV-NEXT:    retl
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f32(
+                                               float %f1, float %f2, metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_f32_ogt_s(i32 %a, i32 %b, float %f1, float %f2) #0 {
+; SSE-32-LABEL: test_f32_ogt_s:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE-32-NEXT:    comiss {{[0-9]+}}(%esp), %xmm0
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; SSE-32-NEXT:    cmoval %eax, %ecx
+; SSE-32-NEXT:    movl (%ecx), %eax
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_f32_ogt_s:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movl %edi, %eax
+; SSE-64-NEXT:    comiss %xmm1, %xmm0
+; SSE-64-NEXT:    cmovbel %esi, %eax
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_f32_ogt_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vcomiss {{[0-9]+}}(%esp), %xmm0
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; AVX-32-NEXT:    cmoval %eax, %ecx
+; AVX-32-NEXT:    movl (%ecx), %eax
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_f32_ogt_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    movl %edi, %eax
+; AVX-64-NEXT:    vcomiss %xmm1, %xmm0
+; AVX-64-NEXT:    cmovbel %esi, %eax
+; AVX-64-NEXT:    retq
+;
+; X87-LABEL: test_f32_ogt_s:
+; X87:       # %bb.0:
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    fcompp
+; X87-NEXT:    fnstsw %ax
+; X87-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-NEXT:    sahf
+; X87-NEXT:    ja .LBB29_1
+; X87-NEXT:  # %bb.2:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+; X87-NEXT:  .LBB29_1:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+;
+; X87-CMOV-LABEL: test_f32_ogt_s:
+; X87-CMOV:       # %bb.0:
+; X87-CMOV-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fcompi %st(1), %st
+; X87-CMOV-NEXT:    fstp %st(0)
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X87-CMOV-NEXT:    cmoval %eax, %ecx
+; X87-CMOV-NEXT:    movl (%ecx), %eax
+; X87-CMOV-NEXT:    retl
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f32(
+                                               float %f1, float %f2, metadata !"ogt",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_f32_oge_s(i32 %a, i32 %b, float %f1, float %f2) #0 {
+; SSE-32-LABEL: test_f32_oge_s:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE-32-NEXT:    comiss {{[0-9]+}}(%esp), %xmm0
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; SSE-32-NEXT:    cmovael %eax, %ecx
+; SSE-32-NEXT:    movl (%ecx), %eax
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_f32_oge_s:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movl %edi, %eax
+; SSE-64-NEXT:    comiss %xmm1, %xmm0
+; SSE-64-NEXT:    cmovbl %esi, %eax
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_f32_oge_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vcomiss {{[0-9]+}}(%esp), %xmm0
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; AVX-32-NEXT:    cmovael %eax, %ecx
+; AVX-32-NEXT:    movl (%ecx), %eax
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_f32_oge_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    movl %edi, %eax
+; AVX-64-NEXT:    vcomiss %xmm1, %xmm0
+; AVX-64-NEXT:    cmovbl %esi, %eax
+; AVX-64-NEXT:    retq
+;
+; X87-LABEL: test_f32_oge_s:
+; X87:       # %bb.0:
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    fcompp
+; X87-NEXT:    fnstsw %ax
+; X87-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-NEXT:    sahf
+; X87-NEXT:    jae .LBB30_1
+; X87-NEXT:  # %bb.2:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+; X87-NEXT:  .LBB30_1:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+;
+; X87-CMOV-LABEL: test_f32_oge_s:
+; X87-CMOV:       # %bb.0:
+; X87-CMOV-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fcompi %st(1), %st
+; X87-CMOV-NEXT:    fstp %st(0)
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X87-CMOV-NEXT:    cmovael %eax, %ecx
+; X87-CMOV-NEXT:    movl (%ecx), %eax
+; X87-CMOV-NEXT:    retl
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f32(
+                                               float %f1, float %f2, metadata !"oge",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_f32_olt_s(i32 %a, i32 %b, float %f1, float %f2) #0 {
+; SSE-32-LABEL: test_f32_olt_s:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE-32-NEXT:    comiss {{[0-9]+}}(%esp), %xmm0
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; SSE-32-NEXT:    cmoval %eax, %ecx
+; SSE-32-NEXT:    movl (%ecx), %eax
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_f32_olt_s:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movl %edi, %eax
+; SSE-64-NEXT:    comiss %xmm0, %xmm1
+; SSE-64-NEXT:    cmovbel %esi, %eax
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_f32_olt_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vcomiss {{[0-9]+}}(%esp), %xmm0
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; AVX-32-NEXT:    cmoval %eax, %ecx
+; AVX-32-NEXT:    movl (%ecx), %eax
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_f32_olt_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    movl %edi, %eax
+; AVX-64-NEXT:    vcomiss %xmm0, %xmm1
+; AVX-64-NEXT:    cmovbel %esi, %eax
+; AVX-64-NEXT:    retq
+;
+; X87-LABEL: test_f32_olt_s:
+; X87:       # %bb.0:
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    fcompp
+; X87-NEXT:    fnstsw %ax
+; X87-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-NEXT:    sahf
+; X87-NEXT:    ja .LBB31_1
+; X87-NEXT:  # %bb.2:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+; X87-NEXT:  .LBB31_1:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+;
+; X87-CMOV-LABEL: test_f32_olt_s:
+; X87-CMOV:       # %bb.0:
+; X87-CMOV-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fcompi %st(1), %st
+; X87-CMOV-NEXT:    fstp %st(0)
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X87-CMOV-NEXT:    cmoval %eax, %ecx
+; X87-CMOV-NEXT:    movl (%ecx), %eax
+; X87-CMOV-NEXT:    retl
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f32(
+                                               float %f1, float %f2, metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_f32_ole_s(i32 %a, i32 %b, float %f1, float %f2) #0 {
+; SSE-32-LABEL: test_f32_ole_s:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE-32-NEXT:    comiss {{[0-9]+}}(%esp), %xmm0
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; SSE-32-NEXT:    cmovael %eax, %ecx
+; SSE-32-NEXT:    movl (%ecx), %eax
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_f32_ole_s:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movl %edi, %eax
+; SSE-64-NEXT:    comiss %xmm0, %xmm1
+; SSE-64-NEXT:    cmovbl %esi, %eax
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_f32_ole_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vcomiss {{[0-9]+}}(%esp), %xmm0
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; AVX-32-NEXT:    cmovael %eax, %ecx
+; AVX-32-NEXT:    movl (%ecx), %eax
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_f32_ole_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    movl %edi, %eax
+; AVX-64-NEXT:    vcomiss %xmm0, %xmm1
+; AVX-64-NEXT:    cmovbl %esi, %eax
+; AVX-64-NEXT:    retq
+;
+; X87-LABEL: test_f32_ole_s:
+; X87:       # %bb.0:
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    fcompp
+; X87-NEXT:    fnstsw %ax
+; X87-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-NEXT:    sahf
+; X87-NEXT:    jae .LBB32_1
+; X87-NEXT:  # %bb.2:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+; X87-NEXT:  .LBB32_1:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+;
+; X87-CMOV-LABEL: test_f32_ole_s:
+; X87-CMOV:       # %bb.0:
+; X87-CMOV-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fcompi %st(1), %st
+; X87-CMOV-NEXT:    fstp %st(0)
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X87-CMOV-NEXT:    cmovael %eax, %ecx
+; X87-CMOV-NEXT:    movl (%ecx), %eax
+; X87-CMOV-NEXT:    retl
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f32(
+                                               float %f1, float %f2, metadata !"ole",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_f32_one_s(i32 %a, i32 %b, float %f1, float %f2) #0 {
+; SSE-32-LABEL: test_f32_one_s:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE-32-NEXT:    comiss {{[0-9]+}}(%esp), %xmm0
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; SSE-32-NEXT:    cmovnel %eax, %ecx
+; SSE-32-NEXT:    movl (%ecx), %eax
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_f32_one_s:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movl %edi, %eax
+; SSE-64-NEXT:    comiss %xmm1, %xmm0
+; SSE-64-NEXT:    cmovel %esi, %eax
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_f32_one_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vcomiss {{[0-9]+}}(%esp), %xmm0
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; AVX-32-NEXT:    cmovnel %eax, %ecx
+; AVX-32-NEXT:    movl (%ecx), %eax
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_f32_one_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    movl %edi, %eax
+; AVX-64-NEXT:    vcomiss %xmm1, %xmm0
+; AVX-64-NEXT:    cmovel %esi, %eax
+; AVX-64-NEXT:    retq
+;
+; X87-LABEL: test_f32_one_s:
+; X87:       # %bb.0:
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    fcompp
+; X87-NEXT:    fnstsw %ax
+; X87-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-NEXT:    sahf
+; X87-NEXT:    jne .LBB33_1
+; X87-NEXT:  # %bb.2:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+; X87-NEXT:  .LBB33_1:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+;
+; X87-CMOV-LABEL: test_f32_one_s:
+; X87-CMOV:       # %bb.0:
+; X87-CMOV-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fcompi %st(1), %st
+; X87-CMOV-NEXT:    fstp %st(0)
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X87-CMOV-NEXT:    cmovnel %eax, %ecx
+; X87-CMOV-NEXT:    movl (%ecx), %eax
+; X87-CMOV-NEXT:    retl
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f32(
+                                               float %f1, float %f2, metadata !"one",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_f32_ord_s(i32 %a, i32 %b, float %f1, float %f2) #0 {
+; SSE-32-LABEL: test_f32_ord_s:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE-32-NEXT:    comiss {{[0-9]+}}(%esp), %xmm0
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; SSE-32-NEXT:    cmovnpl %eax, %ecx
+; SSE-32-NEXT:    movl (%ecx), %eax
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_f32_ord_s:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movl %edi, %eax
+; SSE-64-NEXT:    comiss %xmm1, %xmm0
+; SSE-64-NEXT:    cmovpl %esi, %eax
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_f32_ord_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vcomiss {{[0-9]+}}(%esp), %xmm0
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; AVX-32-NEXT:    cmovnpl %eax, %ecx
+; AVX-32-NEXT:    movl (%ecx), %eax
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_f32_ord_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    movl %edi, %eax
+; AVX-64-NEXT:    vcomiss %xmm1, %xmm0
+; AVX-64-NEXT:    cmovpl %esi, %eax
+; AVX-64-NEXT:    retq
+;
+; X87-LABEL: test_f32_ord_s:
+; X87:       # %bb.0:
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    fcompp
+; X87-NEXT:    fnstsw %ax
+; X87-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-NEXT:    sahf
+; X87-NEXT:    jnp .LBB34_1
+; X87-NEXT:  # %bb.2:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+; X87-NEXT:  .LBB34_1:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+;
+; X87-CMOV-LABEL: test_f32_ord_s:
+; X87-CMOV:       # %bb.0:
+; X87-CMOV-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fcompi %st(1), %st
+; X87-CMOV-NEXT:    fstp %st(0)
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X87-CMOV-NEXT:    cmovnpl %eax, %ecx
+; X87-CMOV-NEXT:    movl (%ecx), %eax
+; X87-CMOV-NEXT:    retl
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f32(
+                                               float %f1, float %f2, metadata !"ord",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_f32_ueq_s(i32 %a, i32 %b, float %f1, float %f2) #0 {
+; SSE-32-LABEL: test_f32_ueq_s:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE-32-NEXT:    comiss {{[0-9]+}}(%esp), %xmm0
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; SSE-32-NEXT:    cmovel %eax, %ecx
+; SSE-32-NEXT:    movl (%ecx), %eax
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_f32_ueq_s:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movl %edi, %eax
+; SSE-64-NEXT:    comiss %xmm1, %xmm0
+; SSE-64-NEXT:    cmovnel %esi, %eax
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_f32_ueq_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vcomiss {{[0-9]+}}(%esp), %xmm0
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; AVX-32-NEXT:    cmovel %eax, %ecx
+; AVX-32-NEXT:    movl (%ecx), %eax
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_f32_ueq_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    movl %edi, %eax
+; AVX-64-NEXT:    vcomiss %xmm1, %xmm0
+; AVX-64-NEXT:    cmovnel %esi, %eax
+; AVX-64-NEXT:    retq
+;
+; X87-LABEL: test_f32_ueq_s:
+; X87:       # %bb.0:
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    fcompp
+; X87-NEXT:    fnstsw %ax
+; X87-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-NEXT:    sahf
+; X87-NEXT:    je .LBB35_1
+; X87-NEXT:  # %bb.2:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+; X87-NEXT:  .LBB35_1:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+;
+; X87-CMOV-LABEL: test_f32_ueq_s:
+; X87-CMOV:       # %bb.0:
+; X87-CMOV-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fcompi %st(1), %st
+; X87-CMOV-NEXT:    fstp %st(0)
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X87-CMOV-NEXT:    cmovel %eax, %ecx
+; X87-CMOV-NEXT:    movl (%ecx), %eax
+; X87-CMOV-NEXT:    retl
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f32(
+                                               float %f1, float %f2, metadata !"ueq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_f32_ugt_s(i32 %a, i32 %b, float %f1, float %f2) #0 {
+; SSE-32-LABEL: test_f32_ugt_s:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE-32-NEXT:    comiss {{[0-9]+}}(%esp), %xmm0
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; SSE-32-NEXT:    cmovbl %eax, %ecx
+; SSE-32-NEXT:    movl (%ecx), %eax
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_f32_ugt_s:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movl %edi, %eax
+; SSE-64-NEXT:    comiss %xmm0, %xmm1
+; SSE-64-NEXT:    cmovael %esi, %eax
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_f32_ugt_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vcomiss {{[0-9]+}}(%esp), %xmm0
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; AVX-32-NEXT:    cmovbl %eax, %ecx
+; AVX-32-NEXT:    movl (%ecx), %eax
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_f32_ugt_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    movl %edi, %eax
+; AVX-64-NEXT:    vcomiss %xmm0, %xmm1
+; AVX-64-NEXT:    cmovael %esi, %eax
+; AVX-64-NEXT:    retq
+;
+; X87-LABEL: test_f32_ugt_s:
+; X87:       # %bb.0:
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    fcompp
+; X87-NEXT:    fnstsw %ax
+; X87-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-NEXT:    sahf
+; X87-NEXT:    jb .LBB36_1
+; X87-NEXT:  # %bb.2:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+; X87-NEXT:  .LBB36_1:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+;
+; X87-CMOV-LABEL: test_f32_ugt_s:
+; X87-CMOV:       # %bb.0:
+; X87-CMOV-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fcompi %st(1), %st
+; X87-CMOV-NEXT:    fstp %st(0)
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X87-CMOV-NEXT:    cmovbl %eax, %ecx
+; X87-CMOV-NEXT:    movl (%ecx), %eax
+; X87-CMOV-NEXT:    retl
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f32(
+                                               float %f1, float %f2, metadata !"ugt",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_f32_uge_s(i32 %a, i32 %b, float %f1, float %f2) #0 {
+; SSE-32-LABEL: test_f32_uge_s:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE-32-NEXT:    comiss {{[0-9]+}}(%esp), %xmm0
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; SSE-32-NEXT:    cmovbel %eax, %ecx
+; SSE-32-NEXT:    movl (%ecx), %eax
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_f32_uge_s:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movl %edi, %eax
+; SSE-64-NEXT:    comiss %xmm0, %xmm1
+; SSE-64-NEXT:    cmoval %esi, %eax
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_f32_uge_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vcomiss {{[0-9]+}}(%esp), %xmm0
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; AVX-32-NEXT:    cmovbel %eax, %ecx
+; AVX-32-NEXT:    movl (%ecx), %eax
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_f32_uge_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    movl %edi, %eax
+; AVX-64-NEXT:    vcomiss %xmm0, %xmm1
+; AVX-64-NEXT:    cmoval %esi, %eax
+; AVX-64-NEXT:    retq
+;
+; X87-LABEL: test_f32_uge_s:
+; X87:       # %bb.0:
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    fcompp
+; X87-NEXT:    fnstsw %ax
+; X87-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-NEXT:    sahf
+; X87-NEXT:    jbe .LBB37_1
+; X87-NEXT:  # %bb.2:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+; X87-NEXT:  .LBB37_1:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+;
+; X87-CMOV-LABEL: test_f32_uge_s:
+; X87-CMOV:       # %bb.0:
+; X87-CMOV-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fcompi %st(1), %st
+; X87-CMOV-NEXT:    fstp %st(0)
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X87-CMOV-NEXT:    cmovbel %eax, %ecx
+; X87-CMOV-NEXT:    movl (%ecx), %eax
+; X87-CMOV-NEXT:    retl
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f32(
+                                               float %f1, float %f2, metadata !"uge",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_f32_ult_s(i32 %a, i32 %b, float %f1, float %f2) #0 {
+; SSE-32-LABEL: test_f32_ult_s:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE-32-NEXT:    comiss {{[0-9]+}}(%esp), %xmm0
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; SSE-32-NEXT:    cmovbl %eax, %ecx
+; SSE-32-NEXT:    movl (%ecx), %eax
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_f32_ult_s:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movl %edi, %eax
+; SSE-64-NEXT:    comiss %xmm1, %xmm0
+; SSE-64-NEXT:    cmovael %esi, %eax
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_f32_ult_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vcomiss {{[0-9]+}}(%esp), %xmm0
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; AVX-32-NEXT:    cmovbl %eax, %ecx
+; AVX-32-NEXT:    movl (%ecx), %eax
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_f32_ult_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    movl %edi, %eax
+; AVX-64-NEXT:    vcomiss %xmm1, %xmm0
+; AVX-64-NEXT:    cmovael %esi, %eax
+; AVX-64-NEXT:    retq
+;
+; X87-LABEL: test_f32_ult_s:
+; X87:       # %bb.0:
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    fcompp
+; X87-NEXT:    fnstsw %ax
+; X87-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-NEXT:    sahf
+; X87-NEXT:    jb .LBB38_1
+; X87-NEXT:  # %bb.2:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+; X87-NEXT:  .LBB38_1:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+;
+; X87-CMOV-LABEL: test_f32_ult_s:
+; X87-CMOV:       # %bb.0:
+; X87-CMOV-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fcompi %st(1), %st
+; X87-CMOV-NEXT:    fstp %st(0)
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X87-CMOV-NEXT:    cmovbl %eax, %ecx
+; X87-CMOV-NEXT:    movl (%ecx), %eax
+; X87-CMOV-NEXT:    retl
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f32(
+                                               float %f1, float %f2, metadata !"ult",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_f32_ule_s(i32 %a, i32 %b, float %f1, float %f2) #0 {
+; SSE-32-LABEL: test_f32_ule_s:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE-32-NEXT:    comiss {{[0-9]+}}(%esp), %xmm0
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; SSE-32-NEXT:    cmovbel %eax, %ecx
+; SSE-32-NEXT:    movl (%ecx), %eax
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_f32_ule_s:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movl %edi, %eax
+; SSE-64-NEXT:    comiss %xmm1, %xmm0
+; SSE-64-NEXT:    cmoval %esi, %eax
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_f32_ule_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vcomiss {{[0-9]+}}(%esp), %xmm0
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; AVX-32-NEXT:    cmovbel %eax, %ecx
+; AVX-32-NEXT:    movl (%ecx), %eax
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_f32_ule_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    movl %edi, %eax
+; AVX-64-NEXT:    vcomiss %xmm1, %xmm0
+; AVX-64-NEXT:    cmoval %esi, %eax
+; AVX-64-NEXT:    retq
+;
+; X87-LABEL: test_f32_ule_s:
+; X87:       # %bb.0:
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    fcompp
+; X87-NEXT:    fnstsw %ax
+; X87-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-NEXT:    sahf
+; X87-NEXT:    jbe .LBB39_1
+; X87-NEXT:  # %bb.2:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+; X87-NEXT:  .LBB39_1:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+;
+; X87-CMOV-LABEL: test_f32_ule_s:
+; X87-CMOV:       # %bb.0:
+; X87-CMOV-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fcompi %st(1), %st
+; X87-CMOV-NEXT:    fstp %st(0)
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X87-CMOV-NEXT:    cmovbel %eax, %ecx
+; X87-CMOV-NEXT:    movl (%ecx), %eax
+; X87-CMOV-NEXT:    retl
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f32(
+                                               float %f1, float %f2, metadata !"ule",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_f32_une_s(i32 %a, i32 %b, float %f1, float %f2) #0 {
+; SSE-32-LABEL: test_f32_une_s:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE-32-NEXT:    comiss {{[0-9]+}}(%esp), %xmm0
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; SSE-32-NEXT:    cmovnel %eax, %ecx
+; SSE-32-NEXT:    cmovpl %eax, %ecx
+; SSE-32-NEXT:    movl (%ecx), %eax
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_f32_une_s:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movl %esi, %eax
+; SSE-64-NEXT:    comiss %xmm1, %xmm0
+; SSE-64-NEXT:    cmovnel %edi, %eax
+; SSE-64-NEXT:    cmovpl %edi, %eax
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_f32_une_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vcomiss {{[0-9]+}}(%esp), %xmm0
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; AVX-32-NEXT:    cmovnel %eax, %ecx
+; AVX-32-NEXT:    cmovpl %eax, %ecx
+; AVX-32-NEXT:    movl (%ecx), %eax
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_f32_une_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    movl %esi, %eax
+; AVX-64-NEXT:    vcomiss %xmm1, %xmm0
+; AVX-64-NEXT:    cmovnel %edi, %eax
+; AVX-64-NEXT:    cmovpl %edi, %eax
+; AVX-64-NEXT:    retq
+;
+; X87-LABEL: test_f32_une_s:
+; X87:       # %bb.0:
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    fcompp
+; X87-NEXT:    fnstsw %ax
+; X87-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-NEXT:    sahf
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    jne .LBB40_3
+; X87-NEXT:  # %bb.1:
+; X87-NEXT:    jp .LBB40_3
+; X87-NEXT:  # %bb.2:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:  .LBB40_3:
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+;
+; X87-CMOV-LABEL: test_f32_une_s:
+; X87-CMOV:       # %bb.0:
+; X87-CMOV-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fcompi %st(1), %st
+; X87-CMOV-NEXT:    fstp %st(0)
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X87-CMOV-NEXT:    cmovnel %eax, %ecx
+; X87-CMOV-NEXT:    cmovpl %eax, %ecx
+; X87-CMOV-NEXT:    movl (%ecx), %eax
+; X87-CMOV-NEXT:    retl
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f32(
+                                               float %f1, float %f2, metadata !"une",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_f32_uno_s(i32 %a, i32 %b, float %f1, float %f2) #0 {
+; SSE-32-LABEL: test_f32_uno_s:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE-32-NEXT:    comiss {{[0-9]+}}(%esp), %xmm0
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; SSE-32-NEXT:    cmovpl %eax, %ecx
+; SSE-32-NEXT:    movl (%ecx), %eax
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_f32_uno_s:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movl %edi, %eax
+; SSE-64-NEXT:    comiss %xmm1, %xmm0
+; SSE-64-NEXT:    cmovnpl %esi, %eax
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_f32_uno_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vcomiss {{[0-9]+}}(%esp), %xmm0
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; AVX-32-NEXT:    cmovpl %eax, %ecx
+; AVX-32-NEXT:    movl (%ecx), %eax
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_f32_uno_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    movl %edi, %eax
+; AVX-64-NEXT:    vcomiss %xmm1, %xmm0
+; AVX-64-NEXT:    cmovnpl %esi, %eax
+; AVX-64-NEXT:    retq
+;
+; X87-LABEL: test_f32_uno_s:
+; X87:       # %bb.0:
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    fcompp
+; X87-NEXT:    fnstsw %ax
+; X87-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-NEXT:    sahf
+; X87-NEXT:    jp .LBB41_1
+; X87-NEXT:  # %bb.2:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+; X87-NEXT:  .LBB41_1:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+;
+; X87-CMOV-LABEL: test_f32_uno_s:
+; X87-CMOV:       # %bb.0:
+; X87-CMOV-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fcompi %st(1), %st
+; X87-CMOV-NEXT:    fstp %st(0)
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X87-CMOV-NEXT:    cmovpl %eax, %ecx
+; X87-CMOV-NEXT:    movl (%ecx), %eax
+; X87-CMOV-NEXT:    retl
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f32(
+                                               float %f1, float %f2, metadata !"uno",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_f64_oeq_s(i32 %a, i32 %b, double %f1, double %f2) #0 {
+; SSE-32-LABEL: test_f64_oeq_s:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE-32-NEXT:    comisd {{[0-9]+}}(%esp), %xmm0
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; SSE-32-NEXT:    cmovnel %eax, %ecx
+; SSE-32-NEXT:    cmovpl %eax, %ecx
+; SSE-32-NEXT:    movl (%ecx), %eax
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_f64_oeq_s:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movl %edi, %eax
+; SSE-64-NEXT:    comisd %xmm1, %xmm0
+; SSE-64-NEXT:    cmovnel %esi, %eax
+; SSE-64-NEXT:    cmovpl %esi, %eax
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_f64_oeq_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-32-NEXT:    vcomisd {{[0-9]+}}(%esp), %xmm0
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; AVX-32-NEXT:    cmovnel %eax, %ecx
+; AVX-32-NEXT:    cmovpl %eax, %ecx
+; AVX-32-NEXT:    movl (%ecx), %eax
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_f64_oeq_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    movl %edi, %eax
+; AVX-64-NEXT:    vcomisd %xmm1, %xmm0
+; AVX-64-NEXT:    cmovnel %esi, %eax
+; AVX-64-NEXT:    cmovpl %esi, %eax
+; AVX-64-NEXT:    retq
+;
+; X87-LABEL: test_f64_oeq_s:
+; X87:       # %bb.0:
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fcompp
+; X87-NEXT:    fnstsw %ax
+; X87-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-NEXT:    sahf
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    jne .LBB42_3
+; X87-NEXT:  # %bb.1:
+; X87-NEXT:    jp .LBB42_3
+; X87-NEXT:  # %bb.2:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:  .LBB42_3:
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+;
+; X87-CMOV-LABEL: test_f64_oeq_s:
+; X87-CMOV:       # %bb.0:
+; X87-CMOV-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fcompi %st(1), %st
+; X87-CMOV-NEXT:    fstp %st(0)
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X87-CMOV-NEXT:    cmovnel %eax, %ecx
+; X87-CMOV-NEXT:    cmovpl %eax, %ecx
+; X87-CMOV-NEXT:    movl (%ecx), %eax
+; X87-CMOV-NEXT:    retl
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f64(
+                                               double %f1, double %f2, metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_f64_ogt_s(i32 %a, i32 %b, double %f1, double %f2) #0 {
+; SSE-32-LABEL: test_f64_ogt_s:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE-32-NEXT:    comisd {{[0-9]+}}(%esp), %xmm0
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; SSE-32-NEXT:    cmoval %eax, %ecx
+; SSE-32-NEXT:    movl (%ecx), %eax
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_f64_ogt_s:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movl %edi, %eax
+; SSE-64-NEXT:    comisd %xmm1, %xmm0
+; SSE-64-NEXT:    cmovbel %esi, %eax
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_f64_ogt_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-32-NEXT:    vcomisd {{[0-9]+}}(%esp), %xmm0
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; AVX-32-NEXT:    cmoval %eax, %ecx
+; AVX-32-NEXT:    movl (%ecx), %eax
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_f64_ogt_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    movl %edi, %eax
+; AVX-64-NEXT:    vcomisd %xmm1, %xmm0
+; AVX-64-NEXT:    cmovbel %esi, %eax
+; AVX-64-NEXT:    retq
+;
+; X87-LABEL: test_f64_ogt_s:
+; X87:       # %bb.0:
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fcompp
+; X87-NEXT:    fnstsw %ax
+; X87-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-NEXT:    sahf
+; X87-NEXT:    ja .LBB43_1
+; X87-NEXT:  # %bb.2:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+; X87-NEXT:  .LBB43_1:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+;
+; X87-CMOV-LABEL: test_f64_ogt_s:
+; X87-CMOV:       # %bb.0:
+; X87-CMOV-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fcompi %st(1), %st
+; X87-CMOV-NEXT:    fstp %st(0)
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X87-CMOV-NEXT:    cmoval %eax, %ecx
+; X87-CMOV-NEXT:    movl (%ecx), %eax
+; X87-CMOV-NEXT:    retl
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f64(
+                                               double %f1, double %f2, metadata !"ogt",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_f64_oge_s(i32 %a, i32 %b, double %f1, double %f2) #0 {
+; SSE-32-LABEL: test_f64_oge_s:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE-32-NEXT:    comisd {{[0-9]+}}(%esp), %xmm0
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; SSE-32-NEXT:    cmovael %eax, %ecx
+; SSE-32-NEXT:    movl (%ecx), %eax
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_f64_oge_s:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movl %edi, %eax
+; SSE-64-NEXT:    comisd %xmm1, %xmm0
+; SSE-64-NEXT:    cmovbl %esi, %eax
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_f64_oge_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-32-NEXT:    vcomisd {{[0-9]+}}(%esp), %xmm0
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; AVX-32-NEXT:    cmovael %eax, %ecx
+; AVX-32-NEXT:    movl (%ecx), %eax
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_f64_oge_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    movl %edi, %eax
+; AVX-64-NEXT:    vcomisd %xmm1, %xmm0
+; AVX-64-NEXT:    cmovbl %esi, %eax
+; AVX-64-NEXT:    retq
+;
+; X87-LABEL: test_f64_oge_s:
+; X87:       # %bb.0:
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fcompp
+; X87-NEXT:    fnstsw %ax
+; X87-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-NEXT:    sahf
+; X87-NEXT:    jae .LBB44_1
+; X87-NEXT:  # %bb.2:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+; X87-NEXT:  .LBB44_1:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+;
+; X87-CMOV-LABEL: test_f64_oge_s:
+; X87-CMOV:       # %bb.0:
+; X87-CMOV-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fcompi %st(1), %st
+; X87-CMOV-NEXT:    fstp %st(0)
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X87-CMOV-NEXT:    cmovael %eax, %ecx
+; X87-CMOV-NEXT:    movl (%ecx), %eax
+; X87-CMOV-NEXT:    retl
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f64(
+                                               double %f1, double %f2, metadata !"oge",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_f64_olt_s(i32 %a, i32 %b, double %f1, double %f2) #0 {
+; SSE-32-LABEL: test_f64_olt_s:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE-32-NEXT:    comisd {{[0-9]+}}(%esp), %xmm0
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; SSE-32-NEXT:    cmoval %eax, %ecx
+; SSE-32-NEXT:    movl (%ecx), %eax
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_f64_olt_s:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movl %edi, %eax
+; SSE-64-NEXT:    comisd %xmm0, %xmm1
+; SSE-64-NEXT:    cmovbel %esi, %eax
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_f64_olt_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-32-NEXT:    vcomisd {{[0-9]+}}(%esp), %xmm0
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; AVX-32-NEXT:    cmoval %eax, %ecx
+; AVX-32-NEXT:    movl (%ecx), %eax
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_f64_olt_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    movl %edi, %eax
+; AVX-64-NEXT:    vcomisd %xmm0, %xmm1
+; AVX-64-NEXT:    cmovbel %esi, %eax
+; AVX-64-NEXT:    retq
+;
+; X87-LABEL: test_f64_olt_s:
+; X87:       # %bb.0:
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fcompp
+; X87-NEXT:    fnstsw %ax
+; X87-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-NEXT:    sahf
+; X87-NEXT:    ja .LBB45_1
+; X87-NEXT:  # %bb.2:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+; X87-NEXT:  .LBB45_1:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+;
+; X87-CMOV-LABEL: test_f64_olt_s:
+; X87-CMOV:       # %bb.0:
+; X87-CMOV-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fcompi %st(1), %st
+; X87-CMOV-NEXT:    fstp %st(0)
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X87-CMOV-NEXT:    cmoval %eax, %ecx
+; X87-CMOV-NEXT:    movl (%ecx), %eax
+; X87-CMOV-NEXT:    retl
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f64(
+                                               double %f1, double %f2, metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_f64_ole_s(i32 %a, i32 %b, double %f1, double %f2) #0 {
+; SSE-32-LABEL: test_f64_ole_s:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE-32-NEXT:    comisd {{[0-9]+}}(%esp), %xmm0
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; SSE-32-NEXT:    cmovael %eax, %ecx
+; SSE-32-NEXT:    movl (%ecx), %eax
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_f64_ole_s:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movl %edi, %eax
+; SSE-64-NEXT:    comisd %xmm0, %xmm1
+; SSE-64-NEXT:    cmovbl %esi, %eax
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_f64_ole_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-32-NEXT:    vcomisd {{[0-9]+}}(%esp), %xmm0
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; AVX-32-NEXT:    cmovael %eax, %ecx
+; AVX-32-NEXT:    movl (%ecx), %eax
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_f64_ole_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    movl %edi, %eax
+; AVX-64-NEXT:    vcomisd %xmm0, %xmm1
+; AVX-64-NEXT:    cmovbl %esi, %eax
+; AVX-64-NEXT:    retq
+;
+; X87-LABEL: test_f64_ole_s:
+; X87:       # %bb.0:
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fcompp
+; X87-NEXT:    fnstsw %ax
+; X87-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-NEXT:    sahf
+; X87-NEXT:    jae .LBB46_1
+; X87-NEXT:  # %bb.2:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+; X87-NEXT:  .LBB46_1:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+;
+; X87-CMOV-LABEL: test_f64_ole_s:
+; X87-CMOV:       # %bb.0:
+; X87-CMOV-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fcompi %st(1), %st
+; X87-CMOV-NEXT:    fstp %st(0)
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X87-CMOV-NEXT:    cmovael %eax, %ecx
+; X87-CMOV-NEXT:    movl (%ecx), %eax
+; X87-CMOV-NEXT:    retl
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f64(
+                                               double %f1, double %f2, metadata !"ole",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_f64_one_s(i32 %a, i32 %b, double %f1, double %f2) #0 {
+; SSE-32-LABEL: test_f64_one_s:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE-32-NEXT:    comisd {{[0-9]+}}(%esp), %xmm0
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; SSE-32-NEXT:    cmovnel %eax, %ecx
+; SSE-32-NEXT:    movl (%ecx), %eax
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_f64_one_s:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movl %edi, %eax
+; SSE-64-NEXT:    comisd %xmm1, %xmm0
+; SSE-64-NEXT:    cmovel %esi, %eax
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_f64_one_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-32-NEXT:    vcomisd {{[0-9]+}}(%esp), %xmm0
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; AVX-32-NEXT:    cmovnel %eax, %ecx
+; AVX-32-NEXT:    movl (%ecx), %eax
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_f64_one_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    movl %edi, %eax
+; AVX-64-NEXT:    vcomisd %xmm1, %xmm0
+; AVX-64-NEXT:    cmovel %esi, %eax
+; AVX-64-NEXT:    retq
+;
+; X87-LABEL: test_f64_one_s:
+; X87:       # %bb.0:
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fcompp
+; X87-NEXT:    fnstsw %ax
+; X87-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-NEXT:    sahf
+; X87-NEXT:    jne .LBB47_1
+; X87-NEXT:  # %bb.2:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+; X87-NEXT:  .LBB47_1:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+;
+; X87-CMOV-LABEL: test_f64_one_s:
+; X87-CMOV:       # %bb.0:
+; X87-CMOV-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fcompi %st(1), %st
+; X87-CMOV-NEXT:    fstp %st(0)
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X87-CMOV-NEXT:    cmovnel %eax, %ecx
+; X87-CMOV-NEXT:    movl (%ecx), %eax
+; X87-CMOV-NEXT:    retl
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f64(
+                                               double %f1, double %f2, metadata !"one",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_f64_ord_s(i32 %a, i32 %b, double %f1, double %f2) #0 {
+; SSE-32-LABEL: test_f64_ord_s:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE-32-NEXT:    comisd {{[0-9]+}}(%esp), %xmm0
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; SSE-32-NEXT:    cmovnpl %eax, %ecx
+; SSE-32-NEXT:    movl (%ecx), %eax
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_f64_ord_s:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movl %edi, %eax
+; SSE-64-NEXT:    comisd %xmm1, %xmm0
+; SSE-64-NEXT:    cmovpl %esi, %eax
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_f64_ord_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-32-NEXT:    vcomisd {{[0-9]+}}(%esp), %xmm0
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; AVX-32-NEXT:    cmovnpl %eax, %ecx
+; AVX-32-NEXT:    movl (%ecx), %eax
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_f64_ord_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    movl %edi, %eax
+; AVX-64-NEXT:    vcomisd %xmm1, %xmm0
+; AVX-64-NEXT:    cmovpl %esi, %eax
+; AVX-64-NEXT:    retq
+;
+; X87-LABEL: test_f64_ord_s:
+; X87:       # %bb.0:
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fcompp
+; X87-NEXT:    fnstsw %ax
+; X87-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-NEXT:    sahf
+; X87-NEXT:    jnp .LBB48_1
+; X87-NEXT:  # %bb.2:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+; X87-NEXT:  .LBB48_1:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+;
+; X87-CMOV-LABEL: test_f64_ord_s:
+; X87-CMOV:       # %bb.0:
+; X87-CMOV-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fcompi %st(1), %st
+; X87-CMOV-NEXT:    fstp %st(0)
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X87-CMOV-NEXT:    cmovnpl %eax, %ecx
+; X87-CMOV-NEXT:    movl (%ecx), %eax
+; X87-CMOV-NEXT:    retl
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f64(
+                                               double %f1, double %f2, metadata !"ord",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_f64_ueq_s(i32 %a, i32 %b, double %f1, double %f2) #0 {
+; SSE-32-LABEL: test_f64_ueq_s:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE-32-NEXT:    comisd {{[0-9]+}}(%esp), %xmm0
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; SSE-32-NEXT:    cmovel %eax, %ecx
+; SSE-32-NEXT:    movl (%ecx), %eax
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_f64_ueq_s:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movl %edi, %eax
+; SSE-64-NEXT:    comisd %xmm1, %xmm0
+; SSE-64-NEXT:    cmovnel %esi, %eax
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_f64_ueq_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-32-NEXT:    vcomisd {{[0-9]+}}(%esp), %xmm0
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; AVX-32-NEXT:    cmovel %eax, %ecx
+; AVX-32-NEXT:    movl (%ecx), %eax
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_f64_ueq_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    movl %edi, %eax
+; AVX-64-NEXT:    vcomisd %xmm1, %xmm0
+; AVX-64-NEXT:    cmovnel %esi, %eax
+; AVX-64-NEXT:    retq
+;
+; X87-LABEL: test_f64_ueq_s:
+; X87:       # %bb.0:
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fcompp
+; X87-NEXT:    fnstsw %ax
+; X87-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-NEXT:    sahf
+; X87-NEXT:    je .LBB49_1
+; X87-NEXT:  # %bb.2:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+; X87-NEXT:  .LBB49_1:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+;
+; X87-CMOV-LABEL: test_f64_ueq_s:
+; X87-CMOV:       # %bb.0:
+; X87-CMOV-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fcompi %st(1), %st
+; X87-CMOV-NEXT:    fstp %st(0)
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X87-CMOV-NEXT:    cmovel %eax, %ecx
+; X87-CMOV-NEXT:    movl (%ecx), %eax
+; X87-CMOV-NEXT:    retl
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f64(
+                                               double %f1, double %f2, metadata !"ueq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_f64_ugt_s(i32 %a, i32 %b, double %f1, double %f2) #0 {
+; SSE-32-LABEL: test_f64_ugt_s:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE-32-NEXT:    comisd {{[0-9]+}}(%esp), %xmm0
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; SSE-32-NEXT:    cmovbl %eax, %ecx
+; SSE-32-NEXT:    movl (%ecx), %eax
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_f64_ugt_s:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movl %edi, %eax
+; SSE-64-NEXT:    comisd %xmm0, %xmm1
+; SSE-64-NEXT:    cmovael %esi, %eax
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_f64_ugt_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-32-NEXT:    vcomisd {{[0-9]+}}(%esp), %xmm0
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; AVX-32-NEXT:    cmovbl %eax, %ecx
+; AVX-32-NEXT:    movl (%ecx), %eax
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_f64_ugt_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    movl %edi, %eax
+; AVX-64-NEXT:    vcomisd %xmm0, %xmm1
+; AVX-64-NEXT:    cmovael %esi, %eax
+; AVX-64-NEXT:    retq
+;
+; X87-LABEL: test_f64_ugt_s:
+; X87:       # %bb.0:
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fcompp
+; X87-NEXT:    fnstsw %ax
+; X87-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-NEXT:    sahf
+; X87-NEXT:    jb .LBB50_1
+; X87-NEXT:  # %bb.2:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+; X87-NEXT:  .LBB50_1:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+;
+; X87-CMOV-LABEL: test_f64_ugt_s:
+; X87-CMOV:       # %bb.0:
+; X87-CMOV-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fcompi %st(1), %st
+; X87-CMOV-NEXT:    fstp %st(0)
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X87-CMOV-NEXT:    cmovbl %eax, %ecx
+; X87-CMOV-NEXT:    movl (%ecx), %eax
+; X87-CMOV-NEXT:    retl
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f64(
+                                               double %f1, double %f2, metadata !"ugt",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_f64_uge_s(i32 %a, i32 %b, double %f1, double %f2) #0 {
+; SSE-32-LABEL: test_f64_uge_s:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE-32-NEXT:    comisd {{[0-9]+}}(%esp), %xmm0
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; SSE-32-NEXT:    cmovbel %eax, %ecx
+; SSE-32-NEXT:    movl (%ecx), %eax
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_f64_uge_s:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movl %edi, %eax
+; SSE-64-NEXT:    comisd %xmm0, %xmm1
+; SSE-64-NEXT:    cmoval %esi, %eax
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_f64_uge_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-32-NEXT:    vcomisd {{[0-9]+}}(%esp), %xmm0
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; AVX-32-NEXT:    cmovbel %eax, %ecx
+; AVX-32-NEXT:    movl (%ecx), %eax
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_f64_uge_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    movl %edi, %eax
+; AVX-64-NEXT:    vcomisd %xmm0, %xmm1
+; AVX-64-NEXT:    cmoval %esi, %eax
+; AVX-64-NEXT:    retq
+;
+; X87-LABEL: test_f64_uge_s:
+; X87:       # %bb.0:
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fcompp
+; X87-NEXT:    fnstsw %ax
+; X87-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-NEXT:    sahf
+; X87-NEXT:    jbe .LBB51_1
+; X87-NEXT:  # %bb.2:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+; X87-NEXT:  .LBB51_1:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+;
+; X87-CMOV-LABEL: test_f64_uge_s:
+; X87-CMOV:       # %bb.0:
+; X87-CMOV-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fcompi %st(1), %st
+; X87-CMOV-NEXT:    fstp %st(0)
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X87-CMOV-NEXT:    cmovbel %eax, %ecx
+; X87-CMOV-NEXT:    movl (%ecx), %eax
+; X87-CMOV-NEXT:    retl
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f64(
+                                               double %f1, double %f2, metadata !"uge",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_f64_ult_s(i32 %a, i32 %b, double %f1, double %f2) #0 {
+; SSE-32-LABEL: test_f64_ult_s:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE-32-NEXT:    comisd {{[0-9]+}}(%esp), %xmm0
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; SSE-32-NEXT:    cmovbl %eax, %ecx
+; SSE-32-NEXT:    movl (%ecx), %eax
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_f64_ult_s:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movl %edi, %eax
+; SSE-64-NEXT:    comisd %xmm1, %xmm0
+; SSE-64-NEXT:    cmovael %esi, %eax
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_f64_ult_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-32-NEXT:    vcomisd {{[0-9]+}}(%esp), %xmm0
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; AVX-32-NEXT:    cmovbl %eax, %ecx
+; AVX-32-NEXT:    movl (%ecx), %eax
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_f64_ult_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    movl %edi, %eax
+; AVX-64-NEXT:    vcomisd %xmm1, %xmm0
+; AVX-64-NEXT:    cmovael %esi, %eax
+; AVX-64-NEXT:    retq
+;
+; X87-LABEL: test_f64_ult_s:
+; X87:       # %bb.0:
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fcompp
+; X87-NEXT:    fnstsw %ax
+; X87-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-NEXT:    sahf
+; X87-NEXT:    jb .LBB52_1
+; X87-NEXT:  # %bb.2:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+; X87-NEXT:  .LBB52_1:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+;
+; X87-CMOV-LABEL: test_f64_ult_s:
+; X87-CMOV:       # %bb.0:
+; X87-CMOV-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fcompi %st(1), %st
+; X87-CMOV-NEXT:    fstp %st(0)
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X87-CMOV-NEXT:    cmovbl %eax, %ecx
+; X87-CMOV-NEXT:    movl (%ecx), %eax
+; X87-CMOV-NEXT:    retl
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f64(
+                                               double %f1, double %f2, metadata !"ult",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_f64_ule_s(i32 %a, i32 %b, double %f1, double %f2) #0 {
+; SSE-32-LABEL: test_f64_ule_s:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE-32-NEXT:    comisd {{[0-9]+}}(%esp), %xmm0
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; SSE-32-NEXT:    cmovbel %eax, %ecx
+; SSE-32-NEXT:    movl (%ecx), %eax
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_f64_ule_s:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movl %edi, %eax
+; SSE-64-NEXT:    comisd %xmm1, %xmm0
+; SSE-64-NEXT:    cmoval %esi, %eax
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_f64_ule_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-32-NEXT:    vcomisd {{[0-9]+}}(%esp), %xmm0
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; AVX-32-NEXT:    cmovbel %eax, %ecx
+; AVX-32-NEXT:    movl (%ecx), %eax
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_f64_ule_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    movl %edi, %eax
+; AVX-64-NEXT:    vcomisd %xmm1, %xmm0
+; AVX-64-NEXT:    cmoval %esi, %eax
+; AVX-64-NEXT:    retq
+;
+; X87-LABEL: test_f64_ule_s:
+; X87:       # %bb.0:
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fcompp
+; X87-NEXT:    fnstsw %ax
+; X87-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-NEXT:    sahf
+; X87-NEXT:    jbe .LBB53_1
+; X87-NEXT:  # %bb.2:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+; X87-NEXT:  .LBB53_1:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+;
+; X87-CMOV-LABEL: test_f64_ule_s:
+; X87-CMOV:       # %bb.0:
+; X87-CMOV-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fcompi %st(1), %st
+; X87-CMOV-NEXT:    fstp %st(0)
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X87-CMOV-NEXT:    cmovbel %eax, %ecx
+; X87-CMOV-NEXT:    movl (%ecx), %eax
+; X87-CMOV-NEXT:    retl
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f64(
+                                               double %f1, double %f2, metadata !"ule",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_f64_une_s(i32 %a, i32 %b, double %f1, double %f2) #0 {
+; SSE-32-LABEL: test_f64_une_s:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE-32-NEXT:    comisd {{[0-9]+}}(%esp), %xmm0
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; SSE-32-NEXT:    cmovnel %eax, %ecx
+; SSE-32-NEXT:    cmovpl %eax, %ecx
+; SSE-32-NEXT:    movl (%ecx), %eax
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_f64_une_s:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movl %esi, %eax
+; SSE-64-NEXT:    comisd %xmm1, %xmm0
+; SSE-64-NEXT:    cmovnel %edi, %eax
+; SSE-64-NEXT:    cmovpl %edi, %eax
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_f64_une_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-32-NEXT:    vcomisd {{[0-9]+}}(%esp), %xmm0
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; AVX-32-NEXT:    cmovnel %eax, %ecx
+; AVX-32-NEXT:    cmovpl %eax, %ecx
+; AVX-32-NEXT:    movl (%ecx), %eax
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_f64_une_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    movl %esi, %eax
+; AVX-64-NEXT:    vcomisd %xmm1, %xmm0
+; AVX-64-NEXT:    cmovnel %edi, %eax
+; AVX-64-NEXT:    cmovpl %edi, %eax
+; AVX-64-NEXT:    retq
+;
+; X87-LABEL: test_f64_une_s:
+; X87:       # %bb.0:
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fcompp
+; X87-NEXT:    fnstsw %ax
+; X87-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-NEXT:    sahf
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    jne .LBB54_3
+; X87-NEXT:  # %bb.1:
+; X87-NEXT:    jp .LBB54_3
+; X87-NEXT:  # %bb.2:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:  .LBB54_3:
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+;
+; X87-CMOV-LABEL: test_f64_une_s:
+; X87-CMOV:       # %bb.0:
+; X87-CMOV-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fcompi %st(1), %st
+; X87-CMOV-NEXT:    fstp %st(0)
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X87-CMOV-NEXT:    cmovnel %eax, %ecx
+; X87-CMOV-NEXT:    cmovpl %eax, %ecx
+; X87-CMOV-NEXT:    movl (%ecx), %eax
+; X87-CMOV-NEXT:    retl
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f64(
+                                               double %f1, double %f2, metadata !"une",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_f64_uno_s(i32 %a, i32 %b, double %f1, double %f2) #0 {
+; SSE-32-LABEL: test_f64_uno_s:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE-32-NEXT:    comisd {{[0-9]+}}(%esp), %xmm0
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; SSE-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; SSE-32-NEXT:    cmovpl %eax, %ecx
+; SSE-32-NEXT:    movl (%ecx), %eax
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_f64_uno_s:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movl %edi, %eax
+; SSE-64-NEXT:    comisd %xmm1, %xmm0
+; SSE-64-NEXT:    cmovnpl %esi, %eax
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_f64_uno_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-32-NEXT:    vcomisd {{[0-9]+}}(%esp), %xmm0
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; AVX-32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; AVX-32-NEXT:    cmovpl %eax, %ecx
+; AVX-32-NEXT:    movl (%ecx), %eax
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_f64_uno_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    movl %edi, %eax
+; AVX-64-NEXT:    vcomisd %xmm1, %xmm0
+; AVX-64-NEXT:    cmovnpl %esi, %eax
+; AVX-64-NEXT:    retq
+;
+; X87-LABEL: test_f64_uno_s:
+; X87:       # %bb.0:
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fcompp
+; X87-NEXT:    fnstsw %ax
+; X87-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-NEXT:    sahf
+; X87-NEXT:    jp .LBB55_1
+; X87-NEXT:  # %bb.2:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+; X87-NEXT:  .LBB55_1:
+; X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl (%eax), %eax
+; X87-NEXT:    retl
+;
+; X87-CMOV-LABEL: test_f64_uno_s:
+; X87-CMOV:       # %bb.0:
+; X87-CMOV-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-CMOV-NEXT:    fcompi %st(1), %st
+; X87-CMOV-NEXT:    fstp %st(0)
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-CMOV-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X87-CMOV-NEXT:    cmovpl %eax, %ecx
+; X87-CMOV-NEXT:    movl (%ecx), %eax
+; X87-CMOV-NEXT:    retl
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f64(
+                                               double %f1, double %f2, metadata !"uno",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+attributes #0 = { strictfp }
+
+declare i1 @llvm.experimental.constrained.fcmp.f32(float, float, metadata, metadata)
+declare i1 @llvm.experimental.constrained.fcmp.f64(double, double, metadata, metadata)
+declare i1 @llvm.experimental.constrained.fcmps.f32(float, float, metadata, metadata)
+declare i1 @llvm.experimental.constrained.fcmps.f64(double, double, metadata, metadata)
diff --git a/llvm/test/CodeGen/X86/fp-strict-scalar-fptoint.ll b/llvm/test/CodeGen/X86/fp-strict-scalar-fptoint.ll
new file mode 100644
index 000000000..c50d092ab
--- /dev/null
+++ b/llvm/test/CodeGen/X86/fp-strict-scalar-fptoint.ll
@@ -0,0 +1,1336 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse2 -O3 | FileCheck %s --check-prefixes=SSE-X86
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 -O3 | FileCheck %s --check-prefixes=SSE-X64
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx -O3 | FileCheck %s --check-prefixes=AVX-X86,AVX1-X86
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx -O3 | FileCheck %s --check-prefixes=AVX-X64,AVX1-X64
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx512f -mattr=+avx512vl -O3 | FileCheck %s --check-prefixes=AVX-X86,AVX512-X86
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f -mattr=+avx512vl -O3 | FileCheck %s --check-prefixes=AVX-X64,AVX512-X64
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=-sse -O3 | FileCheck %s --check-prefixes=CHECK,X87
+
+declare i1  @llvm.experimental.constrained.fptosi.i1.f32(float, metadata)
+declare i8  @llvm.experimental.constrained.fptosi.i8.f32(float, metadata)
+declare i16 @llvm.experimental.constrained.fptosi.i16.f32(float, metadata)
+declare i32 @llvm.experimental.constrained.fptosi.i32.f32(float, metadata)
+declare i64 @llvm.experimental.constrained.fptosi.i64.f32(float, metadata)
+declare i1  @llvm.experimental.constrained.fptoui.i1.f32(float, metadata)
+declare i8  @llvm.experimental.constrained.fptoui.i8.f32(float, metadata)
+declare i16 @llvm.experimental.constrained.fptoui.i16.f32(float, metadata)
+declare i32 @llvm.experimental.constrained.fptoui.i32.f32(float, metadata)
+declare i64 @llvm.experimental.constrained.fptoui.i64.f32(float, metadata)
+
+declare i1  @llvm.experimental.constrained.fptosi.i1.f64(double, metadata)
+declare i8  @llvm.experimental.constrained.fptosi.i8.f64(double, metadata)
+declare i16 @llvm.experimental.constrained.fptosi.i16.f64(double, metadata)
+declare i32 @llvm.experimental.constrained.fptosi.i32.f64(double, metadata)
+declare i64 @llvm.experimental.constrained.fptosi.i64.f64(double, metadata)
+declare i1  @llvm.experimental.constrained.fptoui.i1.f64(double, metadata)
+declare i8  @llvm.experimental.constrained.fptoui.i8.f64(double, metadata)
+declare i16 @llvm.experimental.constrained.fptoui.i16.f64(double, metadata)
+declare i32 @llvm.experimental.constrained.fptoui.i32.f64(double, metadata)
+declare i64 @llvm.experimental.constrained.fptoui.i64.f64(double, metadata)
+
+define i1 @fptosi_f32toi1(float %x) #0 {
+; SSE-X86-LABEL: fptosi_f32toi1:
+; SSE-X86:       # %bb.0:
+; SSE-X86-NEXT:    cvttss2si {{[0-9]+}}(%esp), %eax
+; SSE-X86-NEXT:    # kill: def $al killed $al killed $eax
+; SSE-X86-NEXT:    retl
+;
+; SSE-X64-LABEL: fptosi_f32toi1:
+; SSE-X64:       # %bb.0:
+; SSE-X64-NEXT:    cvttss2si %xmm0, %eax
+; SSE-X64-NEXT:    # kill: def $al killed $al killed $eax
+; SSE-X64-NEXT:    retq
+;
+; AVX-X86-LABEL: fptosi_f32toi1:
+; AVX-X86:       # %bb.0:
+; AVX-X86-NEXT:    vcvttss2si {{[0-9]+}}(%esp), %eax
+; AVX-X86-NEXT:    # kill: def $al killed $al killed $eax
+; AVX-X86-NEXT:    retl
+;
+; AVX-X64-LABEL: fptosi_f32toi1:
+; AVX-X64:       # %bb.0:
+; AVX-X64-NEXT:    vcvttss2si %xmm0, %eax
+; AVX-X64-NEXT:    # kill: def $al killed $al killed $eax
+; AVX-X64-NEXT:    retq
+;
+; CHECK-LABEL: fptosi_f32toi1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subl $8, %esp
+; CHECK-NEXT:    .cfi_def_cfa_offset 12
+; CHECK-NEXT:    flds {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fnstcw {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    orl $3072, %eax # imm = 0xC00
+; CHECK-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fldcw {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fistps {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fldcw {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movb {{[0-9]+}}(%esp), %al
+; CHECK-NEXT:    addl $8, %esp
+; CHECK-NEXT:    .cfi_def_cfa_offset 4
+; CHECK-NEXT:    retl
+  %result = call i1 @llvm.experimental.constrained.fptosi.i1.f32(float %x,
+                                               metadata !"fpexcept.strict") #0
+  ret i1 %result
+}
+
+define i8 @fptosi_f32toi8(float %x) #0 {
+; SSE-X86-LABEL: fptosi_f32toi8:
+; SSE-X86:       # %bb.0:
+; SSE-X86-NEXT:    cvttss2si {{[0-9]+}}(%esp), %eax
+; SSE-X86-NEXT:    # kill: def $al killed $al killed $eax
+; SSE-X86-NEXT:    retl
+;
+; SSE-X64-LABEL: fptosi_f32toi8:
+; SSE-X64:       # %bb.0:
+; SSE-X64-NEXT:    cvttss2si %xmm0, %eax
+; SSE-X64-NEXT:    # kill: def $al killed $al killed $eax
+; SSE-X64-NEXT:    retq
+;
+; AVX-X86-LABEL: fptosi_f32toi8:
+; AVX-X86:       # %bb.0:
+; AVX-X86-NEXT:    vcvttss2si {{[0-9]+}}(%esp), %eax
+; AVX-X86-NEXT:    # kill: def $al killed $al killed $eax
+; AVX-X86-NEXT:    retl
+;
+; AVX-X64-LABEL: fptosi_f32toi8:
+; AVX-X64:       # %bb.0:
+; AVX-X64-NEXT:    vcvttss2si %xmm0, %eax
+; AVX-X64-NEXT:    # kill: def $al killed $al killed $eax
+; AVX-X64-NEXT:    retq
+;
+; CHECK-LABEL: fptosi_f32toi8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subl $8, %esp
+; CHECK-NEXT:    .cfi_def_cfa_offset 12
+; CHECK-NEXT:    flds {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fnstcw {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    orl $3072, %eax # imm = 0xC00
+; CHECK-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fldcw {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fistps {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fldcw {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movb {{[0-9]+}}(%esp), %al
+; CHECK-NEXT:    addl $8, %esp
+; CHECK-NEXT:    .cfi_def_cfa_offset 4
+; CHECK-NEXT:    retl
+  %result = call i8 @llvm.experimental.constrained.fptosi.i8.f32(float %x,
+                                               metadata !"fpexcept.strict") #0
+  ret i8 %result
+}
+
+define i16 @fptosi_f32toi16(float %x) #0 {
+; SSE-X86-LABEL: fptosi_f32toi16:
+; SSE-X86:       # %bb.0:
+; SSE-X86-NEXT:    cvttss2si {{[0-9]+}}(%esp), %eax
+; SSE-X86-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE-X86-NEXT:    retl
+;
+; SSE-X64-LABEL: fptosi_f32toi16:
+; SSE-X64:       # %bb.0:
+; SSE-X64-NEXT:    cvttss2si %xmm0, %eax
+; SSE-X64-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE-X64-NEXT:    retq
+;
+; AVX-X86-LABEL: fptosi_f32toi16:
+; AVX-X86:       # %bb.0:
+; AVX-X86-NEXT:    vcvttss2si {{[0-9]+}}(%esp), %eax
+; AVX-X86-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX-X86-NEXT:    retl
+;
+; AVX-X64-LABEL: fptosi_f32toi16:
+; AVX-X64:       # %bb.0:
+; AVX-X64-NEXT:    vcvttss2si %xmm0, %eax
+; AVX-X64-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX-X64-NEXT:    retq
+;
+; CHECK-LABEL: fptosi_f32toi16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subl $8, %esp
+; CHECK-NEXT:    .cfi_def_cfa_offset 12
+; CHECK-NEXT:    flds {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fnstcw {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    orl $3072, %eax # imm = 0xC00
+; CHECK-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fldcw {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fistps {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fldcw {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    addl $8, %esp
+; CHECK-NEXT:    .cfi_def_cfa_offset 4
+; CHECK-NEXT:    retl
+  %result = call i16 @llvm.experimental.constrained.fptosi.i16.f32(float %x,
+                                               metadata !"fpexcept.strict") #0
+  ret i16 %result
+}
+
+define i32 @fptosi_f32toi32(float %x) #0 {
+; SSE-X86-LABEL: fptosi_f32toi32:
+; SSE-X86:       # %bb.0:
+; SSE-X86-NEXT:    cvttss2si {{[0-9]+}}(%esp), %eax
+; SSE-X86-NEXT:    retl
+;
+; SSE-X64-LABEL: fptosi_f32toi32:
+; SSE-X64:       # %bb.0:
+; SSE-X64-NEXT:    cvttss2si %xmm0, %eax
+; SSE-X64-NEXT:    retq
+;
+; AVX-X86-LABEL: fptosi_f32toi32:
+; AVX-X86:       # %bb.0:
+; AVX-X86-NEXT:    vcvttss2si {{[0-9]+}}(%esp), %eax
+; AVX-X86-NEXT:    retl
+;
+; AVX-X64-LABEL: fptosi_f32toi32:
+; AVX-X64:       # %bb.0:
+; AVX-X64-NEXT:    vcvttss2si %xmm0, %eax
+; AVX-X64-NEXT:    retq
+;
+; CHECK-LABEL: fptosi_f32toi32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subl $8, %esp
+; CHECK-NEXT:    .cfi_def_cfa_offset 12
+; CHECK-NEXT:    flds {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fnstcw (%esp)
+; CHECK-NEXT:    movzwl (%esp), %eax
+; CHECK-NEXT:    orl $3072, %eax # imm = 0xC00
+; CHECK-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fldcw {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fistpl {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fldcw (%esp)
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    addl $8, %esp
+; CHECK-NEXT:    .cfi_def_cfa_offset 4
+; CHECK-NEXT:    retl
+  %result = call i32 @llvm.experimental.constrained.fptosi.i32.f32(float %x,
+                                               metadata !"fpexcept.strict") #0
+  ret i32 %result
+}
+
+define i64 @fptosi_f32toi64(float %x) #0 {
+; SSE-X86-LABEL: fptosi_f32toi64:
+; SSE-X86:       # %bb.0:
+; SSE-X86-NEXT:    pushl %ebp
+; SSE-X86-NEXT:    .cfi_def_cfa_offset 8
+; SSE-X86-NEXT:    .cfi_offset %ebp, -8
+; SSE-X86-NEXT:    movl %esp, %ebp
+; SSE-X86-NEXT:    .cfi_def_cfa_register %ebp
+; SSE-X86-NEXT:    andl $-8, %esp
+; SSE-X86-NEXT:    subl $16, %esp
+; SSE-X86-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE-X86-NEXT:    movss %xmm0, {{[0-9]+}}(%esp)
+; SSE-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; SSE-X86-NEXT:    fnstcw {{[0-9]+}}(%esp)
+; SSE-X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; SSE-X86-NEXT:    orl $3072, %eax # imm = 0xC00
+; SSE-X86-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; SSE-X86-NEXT:    fldcw {{[0-9]+}}(%esp)
+; SSE-X86-NEXT:    fistpll {{[0-9]+}}(%esp)
+; SSE-X86-NEXT:    fldcw {{[0-9]+}}(%esp)
+; SSE-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; SSE-X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; SSE-X86-NEXT:    movl %ebp, %esp
+; SSE-X86-NEXT:    popl %ebp
+; SSE-X86-NEXT:    .cfi_def_cfa %esp, 4
+; SSE-X86-NEXT:    retl
+;
+; SSE-X64-LABEL: fptosi_f32toi64:
+; SSE-X64:       # %bb.0:
+; SSE-X64-NEXT:    cvttss2si %xmm0, %rax
+; SSE-X64-NEXT:    retq
+;
+; AVX-X86-LABEL: fptosi_f32toi64:
+; AVX-X86:       # %bb.0:
+; AVX-X86-NEXT:    pushl %ebp
+; AVX-X86-NEXT:    .cfi_def_cfa_offset 8
+; AVX-X86-NEXT:    .cfi_offset %ebp, -8
+; AVX-X86-NEXT:    movl %esp, %ebp
+; AVX-X86-NEXT:    .cfi_def_cfa_register %ebp
+; AVX-X86-NEXT:    andl $-8, %esp
+; AVX-X86-NEXT:    subl $8, %esp
+; AVX-X86-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-X86-NEXT:    vmovss %xmm0, (%esp)
+; AVX-X86-NEXT:    flds (%esp)
+; AVX-X86-NEXT:    fisttpll (%esp)
+; AVX-X86-NEXT:    movl (%esp), %eax
+; AVX-X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; AVX-X86-NEXT:    movl %ebp, %esp
+; AVX-X86-NEXT:    popl %ebp
+; AVX-X86-NEXT:    .cfi_def_cfa %esp, 4
+; AVX-X86-NEXT:    retl
+;
+; AVX-X64-LABEL: fptosi_f32toi64:
+; AVX-X64:       # %bb.0:
+; AVX-X64-NEXT:    vcvttss2si %xmm0, %rax
+; AVX-X64-NEXT:    retq
+;
+; CHECK-LABEL: fptosi_f32toi64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushl %ebp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    .cfi_offset %ebp, -8
+; CHECK-NEXT:    movl %esp, %ebp
+; CHECK-NEXT:    .cfi_def_cfa_register %ebp
+; CHECK-NEXT:    andl $-8, %esp
+; CHECK-NEXT:    subl $16, %esp
+; CHECK-NEXT:    flds 8(%ebp)
+; CHECK-NEXT:    fnstcw {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    orl $3072, %eax # imm = 0xC00
+; CHECK-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fldcw {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fistpll {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fldcw {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; CHECK-NEXT:    movl %ebp, %esp
+; CHECK-NEXT:    popl %ebp
+; CHECK-NEXT:    .cfi_def_cfa %esp, 4
+; CHECK-NEXT:    retl
+  %result = call i64 @llvm.experimental.constrained.fptosi.i64.f32(float %x,
+                                               metadata !"fpexcept.strict") #0
+  ret i64 %result
+}
+
+define i1 @fptoui_f32toi1(float %x) #0 {
+; SSE-X86-LABEL: fptoui_f32toi1:
+; SSE-X86:       # %bb.0:
+; SSE-X86-NEXT:    cvttss2si {{[0-9]+}}(%esp), %eax
+; SSE-X86-NEXT:    # kill: def $al killed $al killed $eax
+; SSE-X86-NEXT:    retl
+;
+; SSE-X64-LABEL: fptoui_f32toi1:
+; SSE-X64:       # %bb.0:
+; SSE-X64-NEXT:    cvttss2si %xmm0, %eax
+; SSE-X64-NEXT:    # kill: def $al killed $al killed $eax
+; SSE-X64-NEXT:    retq
+;
+; AVX-X86-LABEL: fptoui_f32toi1:
+; AVX-X86:       # %bb.0:
+; AVX-X86-NEXT:    vcvttss2si {{[0-9]+}}(%esp), %eax
+; AVX-X86-NEXT:    # kill: def $al killed $al killed $eax
+; AVX-X86-NEXT:    retl
+;
+; AVX-X64-LABEL: fptoui_f32toi1:
+; AVX-X64:       # %bb.0:
+; AVX-X64-NEXT:    vcvttss2si %xmm0, %eax
+; AVX-X64-NEXT:    # kill: def $al killed $al killed $eax
+; AVX-X64-NEXT:    retq
+;
+; CHECK-LABEL: fptoui_f32toi1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subl $8, %esp
+; CHECK-NEXT:    .cfi_def_cfa_offset 12
+; CHECK-NEXT:    flds {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fnstcw {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    orl $3072, %eax # imm = 0xC00
+; CHECK-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fldcw {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fistps {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fldcw {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movb {{[0-9]+}}(%esp), %al
+; CHECK-NEXT:    addl $8, %esp
+; CHECK-NEXT:    .cfi_def_cfa_offset 4
+; CHECK-NEXT:    retl
+  %result = call i1 @llvm.experimental.constrained.fptoui.i1.f32(float %x,
+                                               metadata !"fpexcept.strict") #0
+  ret i1 %result
+}
+
+define i8 @fptoui_f32toi8(float %x) #0 {
+; SSE-X86-LABEL: fptoui_f32toi8:
+; SSE-X86:       # %bb.0:
+; SSE-X86-NEXT:    cvttss2si {{[0-9]+}}(%esp), %eax
+; SSE-X86-NEXT:    # kill: def $al killed $al killed $eax
+; SSE-X86-NEXT:    retl
+;
+; SSE-X64-LABEL: fptoui_f32toi8:
+; SSE-X64:       # %bb.0:
+; SSE-X64-NEXT:    cvttss2si %xmm0, %eax
+; SSE-X64-NEXT:    # kill: def $al killed $al killed $eax
+; SSE-X64-NEXT:    retq
+;
+; AVX-X86-LABEL: fptoui_f32toi8:
+; AVX-X86:       # %bb.0:
+; AVX-X86-NEXT:    vcvttss2si {{[0-9]+}}(%esp), %eax
+; AVX-X86-NEXT:    # kill: def $al killed $al killed $eax
+; AVX-X86-NEXT:    retl
+;
+; AVX-X64-LABEL: fptoui_f32toi8:
+; AVX-X64:       # %bb.0:
+; AVX-X64-NEXT:    vcvttss2si %xmm0, %eax
+; AVX-X64-NEXT:    # kill: def $al killed $al killed $eax
+; AVX-X64-NEXT:    retq
+;
+; CHECK-LABEL: fptoui_f32toi8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subl $8, %esp
+; CHECK-NEXT:    .cfi_def_cfa_offset 12
+; CHECK-NEXT:    flds {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fnstcw {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    orl $3072, %eax # imm = 0xC00
+; CHECK-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fldcw {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fistps {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fldcw {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movb {{[0-9]+}}(%esp), %al
+; CHECK-NEXT:    addl $8, %esp
+; CHECK-NEXT:    .cfi_def_cfa_offset 4
+; CHECK-NEXT:    retl
+  %result = call i8 @llvm.experimental.constrained.fptoui.i8.f32(float %x,
+                                               metadata !"fpexcept.strict") #0
+  ret i8 %result
+}
+
+define i16 @fptoui_f32toi16(float %x) #0 {
+; SSE-X86-LABEL: fptoui_f32toi16:
+; SSE-X86:       # %bb.0:
+; SSE-X86-NEXT:    cvttss2si {{[0-9]+}}(%esp), %eax
+; SSE-X86-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE-X86-NEXT:    retl
+;
+; SSE-X64-LABEL: fptoui_f32toi16:
+; SSE-X64:       # %bb.0:
+; SSE-X64-NEXT:    cvttss2si %xmm0, %eax
+; SSE-X64-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE-X64-NEXT:    retq
+;
+; AVX-X86-LABEL: fptoui_f32toi16:
+; AVX-X86:       # %bb.0:
+; AVX-X86-NEXT:    vcvttss2si {{[0-9]+}}(%esp), %eax
+; AVX-X86-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX-X86-NEXT:    retl
+;
+; AVX-X64-LABEL: fptoui_f32toi16:
+; AVX-X64:       # %bb.0:
+; AVX-X64-NEXT:    vcvttss2si %xmm0, %eax
+; AVX-X64-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX-X64-NEXT:    retq
+;
+; CHECK-LABEL: fptoui_f32toi16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subl $8, %esp
+; CHECK-NEXT:    .cfi_def_cfa_offset 12
+; CHECK-NEXT:    flds {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fnstcw (%esp)
+; CHECK-NEXT:    movzwl (%esp), %eax
+; CHECK-NEXT:    orl $3072, %eax # imm = 0xC00
+; CHECK-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fldcw {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fistpl {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fldcw (%esp)
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    # kill: def $ax killed $ax killed $eax
+; CHECK-NEXT:    addl $8, %esp
+; CHECK-NEXT:    .cfi_def_cfa_offset 4
+; CHECK-NEXT:    retl
+  %result = call i16 @llvm.experimental.constrained.fptoui.i16.f32(float %x,
+                                               metadata !"fpexcept.strict") #0
+  ret i16 %result
+}
+
+define i32 @fptoui_f32toi32(float %x) #0 {
+; SSE-X86-LABEL: fptoui_f32toi32:
+; SSE-X86:       # %bb.0:
+; SSE-X86-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE-X86-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE-X86-NEXT:    comiss %xmm0, %xmm2
+; SSE-X86-NEXT:    xorps %xmm1, %xmm1
+; SSE-X86-NEXT:    ja .LBB8_2
+; SSE-X86-NEXT:  # %bb.1:
+; SSE-X86-NEXT:    movaps %xmm2, %xmm1
+; SSE-X86-NEXT:  .LBB8_2:
+; SSE-X86-NEXT:    setbe %al
+; SSE-X86-NEXT:    movzbl %al, %ecx
+; SSE-X86-NEXT:    shll $31, %ecx
+; SSE-X86-NEXT:    subss %xmm1, %xmm0
+; SSE-X86-NEXT:    cvttss2si %xmm0, %eax
+; SSE-X86-NEXT:    xorl %ecx, %eax
+; SSE-X86-NEXT:    retl
+;
+; SSE-X64-LABEL: fptoui_f32toi32:
+; SSE-X64:       # %bb.0:
+; SSE-X64-NEXT:    cvttss2si %xmm0, %rax
+; SSE-X64-NEXT:    # kill: def $eax killed $eax killed $rax
+; SSE-X64-NEXT:    retq
+;
+; AVX1-X86-LABEL: fptoui_f32toi32:
+; AVX1-X86:       # %bb.0:
+; AVX1-X86-NEXT:    pushl %ebp
+; AVX1-X86-NEXT:    .cfi_def_cfa_offset 8
+; AVX1-X86-NEXT:    .cfi_offset %ebp, -8
+; AVX1-X86-NEXT:    movl %esp, %ebp
+; AVX1-X86-NEXT:    .cfi_def_cfa_register %ebp
+; AVX1-X86-NEXT:    andl $-8, %esp
+; AVX1-X86-NEXT:    subl $8, %esp
+; AVX1-X86-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX1-X86-NEXT:    vmovss %xmm0, (%esp)
+; AVX1-X86-NEXT:    flds (%esp)
+; AVX1-X86-NEXT:    fisttpll (%esp)
+; AVX1-X86-NEXT:    movl (%esp), %eax
+; AVX1-X86-NEXT:    movl %ebp, %esp
+; AVX1-X86-NEXT:    popl %ebp
+; AVX1-X86-NEXT:    .cfi_def_cfa %esp, 4
+; AVX1-X86-NEXT:    retl
+;
+; AVX1-X64-LABEL: fptoui_f32toi32:
+; AVX1-X64:       # %bb.0:
+; AVX1-X64-NEXT:    vcvttss2si %xmm0, %rax
+; AVX1-X64-NEXT:    # kill: def $eax killed $eax killed $rax
+; AVX1-X64-NEXT:    retq
+;
+; AVX512-X86-LABEL: fptoui_f32toi32:
+; AVX512-X86:       # %bb.0:
+; AVX512-X86-NEXT:    vcvttss2usi {{[0-9]+}}(%esp), %eax
+; AVX512-X86-NEXT:    retl
+;
+; AVX512-X64-LABEL: fptoui_f32toi32:
+; AVX512-X64:       # %bb.0:
+; AVX512-X64-NEXT:    vcvttss2usi %xmm0, %eax
+; AVX512-X64-NEXT:    retq
+;
+; CHECK-LABEL: fptoui_f32toi32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushl %ebp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    .cfi_offset %ebp, -8
+; CHECK-NEXT:    movl %esp, %ebp
+; CHECK-NEXT:    .cfi_def_cfa_register %ebp
+; CHECK-NEXT:    andl $-8, %esp
+; CHECK-NEXT:    subl $16, %esp
+; CHECK-NEXT:    flds 8(%ebp)
+; CHECK-NEXT:    fnstcw {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    orl $3072, %eax # imm = 0xC00
+; CHECK-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fldcw {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fistpll {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fldcw {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    movl %ebp, %esp
+; CHECK-NEXT:    popl %ebp
+; CHECK-NEXT:    .cfi_def_cfa %esp, 4
+; CHECK-NEXT:    retl
+  %result = call i32 @llvm.experimental.constrained.fptoui.i32.f32(float %x,
+                                               metadata !"fpexcept.strict") #0
+  ret i32 %result
+}
+
+define i64 @fptoui_f32toi64(float %x) #0 {
+; SSE-X86-LABEL: fptoui_f32toi64:
+; SSE-X86:       # %bb.0:
+; SSE-X86-NEXT:    pushl %ebp
+; SSE-X86-NEXT:    .cfi_def_cfa_offset 8
+; SSE-X86-NEXT:    .cfi_offset %ebp, -8
+; SSE-X86-NEXT:    movl %esp, %ebp
+; SSE-X86-NEXT:    .cfi_def_cfa_register %ebp
+; SSE-X86-NEXT:    andl $-8, %esp
+; SSE-X86-NEXT:    subl $16, %esp
+; SSE-X86-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE-X86-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE-X86-NEXT:    comiss %xmm0, %xmm2
+; SSE-X86-NEXT:    xorps %xmm1, %xmm1
+; SSE-X86-NEXT:    ja .LBB9_2
+; SSE-X86-NEXT:  # %bb.1:
+; SSE-X86-NEXT:    movaps %xmm2, %xmm1
+; SSE-X86-NEXT:  .LBB9_2:
+; SSE-X86-NEXT:    subss %xmm1, %xmm0
+; SSE-X86-NEXT:    movss %xmm0, {{[0-9]+}}(%esp)
+; SSE-X86-NEXT:    setbe %al
+; SSE-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; SSE-X86-NEXT:    fnstcw {{[0-9]+}}(%esp)
+; SSE-X86-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
+; SSE-X86-NEXT:    orl $3072, %ecx # imm = 0xC00
+; SSE-X86-NEXT:    movw %cx, {{[0-9]+}}(%esp)
+; SSE-X86-NEXT:    fldcw {{[0-9]+}}(%esp)
+; SSE-X86-NEXT:    fistpll {{[0-9]+}}(%esp)
+; SSE-X86-NEXT:    fldcw {{[0-9]+}}(%esp)
+; SSE-X86-NEXT:    movzbl %al, %edx
+; SSE-X86-NEXT:    shll $31, %edx
+; SSE-X86-NEXT:    xorl {{[0-9]+}}(%esp), %edx
+; SSE-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; SSE-X86-NEXT:    movl %ebp, %esp
+; SSE-X86-NEXT:    popl %ebp
+; SSE-X86-NEXT:    .cfi_def_cfa %esp, 4
+; SSE-X86-NEXT:    retl
+;
+; SSE-X64-LABEL: fptoui_f32toi64:
+; SSE-X64:       # %bb.0:
+; SSE-X64-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE-X64-NEXT:    comiss %xmm2, %xmm0
+; SSE-X64-NEXT:    xorps %xmm1, %xmm1
+; SSE-X64-NEXT:    jb .LBB9_2
+; SSE-X64-NEXT:  # %bb.1:
+; SSE-X64-NEXT:    movaps %xmm2, %xmm1
+; SSE-X64-NEXT:  .LBB9_2:
+; SSE-X64-NEXT:    subss %xmm1, %xmm0
+; SSE-X64-NEXT:    cvttss2si %xmm0, %rcx
+; SSE-X64-NEXT:    setae %al
+; SSE-X64-NEXT:    movzbl %al, %eax
+; SSE-X64-NEXT:    shlq $63, %rax
+; SSE-X64-NEXT:    xorq %rcx, %rax
+; SSE-X64-NEXT:    retq
+;
+; AVX1-X86-LABEL: fptoui_f32toi64:
+; AVX1-X86:       # %bb.0:
+; AVX1-X86-NEXT:    pushl %ebp
+; AVX1-X86-NEXT:    .cfi_def_cfa_offset 8
+; AVX1-X86-NEXT:    .cfi_offset %ebp, -8
+; AVX1-X86-NEXT:    movl %esp, %ebp
+; AVX1-X86-NEXT:    .cfi_def_cfa_register %ebp
+; AVX1-X86-NEXT:    andl $-8, %esp
+; AVX1-X86-NEXT:    subl $8, %esp
+; AVX1-X86-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX1-X86-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX1-X86-NEXT:    vcomiss %xmm0, %xmm1
+; AVX1-X86-NEXT:    vxorps %xmm2, %xmm2, %xmm2
+; AVX1-X86-NEXT:    ja .LBB9_2
+; AVX1-X86-NEXT:  # %bb.1:
+; AVX1-X86-NEXT:    vmovaps %xmm1, %xmm2
+; AVX1-X86-NEXT:  .LBB9_2:
+; AVX1-X86-NEXT:    vsubss %xmm2, %xmm0, %xmm0
+; AVX1-X86-NEXT:    vmovss %xmm0, (%esp)
+; AVX1-X86-NEXT:    flds (%esp)
+; AVX1-X86-NEXT:    fisttpll (%esp)
+; AVX1-X86-NEXT:    setbe %al
+; AVX1-X86-NEXT:    movzbl %al, %edx
+; AVX1-X86-NEXT:    shll $31, %edx
+; AVX1-X86-NEXT:    xorl {{[0-9]+}}(%esp), %edx
+; AVX1-X86-NEXT:    movl (%esp), %eax
+; AVX1-X86-NEXT:    movl %ebp, %esp
+; AVX1-X86-NEXT:    popl %ebp
+; AVX1-X86-NEXT:    .cfi_def_cfa %esp, 4
+; AVX1-X86-NEXT:    retl
+;
+; AVX1-X64-LABEL: fptoui_f32toi64:
+; AVX1-X64:       # %bb.0:
+; AVX1-X64-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX1-X64-NEXT:    vcomiss %xmm1, %xmm0
+; AVX1-X64-NEXT:    vxorps %xmm2, %xmm2, %xmm2
+; AVX1-X64-NEXT:    jb .LBB9_2
+; AVX1-X64-NEXT:  # %bb.1:
+; AVX1-X64-NEXT:    vmovaps %xmm1, %xmm2
+; AVX1-X64-NEXT:  .LBB9_2:
+; AVX1-X64-NEXT:    vsubss %xmm2, %xmm0, %xmm0
+; AVX1-X64-NEXT:    vcvttss2si %xmm0, %rcx
+; AVX1-X64-NEXT:    setae %al
+; AVX1-X64-NEXT:    movzbl %al, %eax
+; AVX1-X64-NEXT:    shlq $63, %rax
+; AVX1-X64-NEXT:    xorq %rcx, %rax
+; AVX1-X64-NEXT:    retq
+;
+; AVX512-X86-LABEL: fptoui_f32toi64:
+; AVX512-X86:       # %bb.0:
+; AVX512-X86-NEXT:    pushl %ebp
+; AVX512-X86-NEXT:    .cfi_def_cfa_offset 8
+; AVX512-X86-NEXT:    .cfi_offset %ebp, -8
+; AVX512-X86-NEXT:    movl %esp, %ebp
+; AVX512-X86-NEXT:    .cfi_def_cfa_register %ebp
+; AVX512-X86-NEXT:    andl $-8, %esp
+; AVX512-X86-NEXT:    subl $8, %esp
+; AVX512-X86-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX512-X86-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX512-X86-NEXT:    xorl %edx, %edx
+; AVX512-X86-NEXT:    vcomiss %xmm0, %xmm1
+; AVX512-X86-NEXT:    seta %al
+; AVX512-X86-NEXT:    kmovw %eax, %k1
+; AVX512-X86-NEXT:    vxorps %xmm2, %xmm2, %xmm2
+; AVX512-X86-NEXT:    vmovss %xmm2, %xmm1, %xmm1 {%k1}
+; AVX512-X86-NEXT:    vsubss %xmm1, %xmm0, %xmm0
+; AVX512-X86-NEXT:    vmovss %xmm0, (%esp)
+; AVX512-X86-NEXT:    flds (%esp)
+; AVX512-X86-NEXT:    fisttpll (%esp)
+; AVX512-X86-NEXT:    setbe %dl
+; AVX512-X86-NEXT:    shll $31, %edx
+; AVX512-X86-NEXT:    xorl {{[0-9]+}}(%esp), %edx
+; AVX512-X86-NEXT:    movl (%esp), %eax
+; AVX512-X86-NEXT:    movl %ebp, %esp
+; AVX512-X86-NEXT:    popl %ebp
+; AVX512-X86-NEXT:    .cfi_def_cfa %esp, 4
+; AVX512-X86-NEXT:    retl
+;
+; AVX512-X64-LABEL: fptoui_f32toi64:
+; AVX512-X64:       # %bb.0:
+; AVX512-X64-NEXT:    vcvttss2usi %xmm0, %rax
+; AVX512-X64-NEXT:    retq
+;
+; CHECK-LABEL: fptoui_f32toi64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushl %ebp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    .cfi_offset %ebp, -8
+; CHECK-NEXT:    movl %esp, %ebp
+; CHECK-NEXT:    .cfi_def_cfa_register %ebp
+; CHECK-NEXT:    andl $-8, %esp
+; CHECK-NEXT:    subl $16, %esp
+; CHECK-NEXT:    flds 8(%ebp)
+; CHECK-NEXT:    flds {{\.LCPI.*}}
+; CHECK-NEXT:    fcom %st(1)
+; CHECK-NEXT:    fnstsw %ax
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    # kill: def $ah killed $ah killed $ax
+; CHECK-NEXT:    sahf
+; CHECK-NEXT:    setbe %al
+; CHECK-NEXT:    fldz
+; CHECK-NEXT:    ja .LBB9_2
+; CHECK-NEXT:  # %bb.1:
+; CHECK-NEXT:    fstp %st(0)
+; CHECK-NEXT:    fldz
+; CHECK-NEXT:    fxch %st(1)
+; CHECK-NEXT:  .LBB9_2:
+; CHECK-NEXT:    fstp %st(1)
+; CHECK-NEXT:    fsubrp %st, %st(1)
+; CHECK-NEXT:    fnstcw {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT:    orl $3072, %ecx # imm = 0xC00
+; CHECK-NEXT:    movw %cx, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fldcw {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fistpll {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fldcw {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movb %al, %dl
+; CHECK-NEXT:    shll $31, %edx
+; CHECK-NEXT:    xorl {{[0-9]+}}(%esp), %edx
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    movl %ebp, %esp
+; CHECK-NEXT:    popl %ebp
+; CHECK-NEXT:    .cfi_def_cfa %esp, 4
+; CHECK-NEXT:    retl
+  %result = call i64 @llvm.experimental.constrained.fptoui.i64.f32(float %x,
+                                               metadata !"fpexcept.strict") #0
+  ret i64 %result
+}
+
+define i8 @fptosi_f64toi8(double %x) #0 {
+; SSE-X86-LABEL: fptosi_f64toi8:
+; SSE-X86:       # %bb.0:
+; SSE-X86-NEXT:    cvttsd2si {{[0-9]+}}(%esp), %eax
+; SSE-X86-NEXT:    # kill: def $al killed $al killed $eax
+; SSE-X86-NEXT:    retl
+;
+; SSE-X64-LABEL: fptosi_f64toi8:
+; SSE-X64:       # %bb.0:
+; SSE-X64-NEXT:    cvttsd2si %xmm0, %eax
+; SSE-X64-NEXT:    # kill: def $al killed $al killed $eax
+; SSE-X64-NEXT:    retq
+;
+; AVX-X86-LABEL: fptosi_f64toi8:
+; AVX-X86:       # %bb.0:
+; AVX-X86-NEXT:    vcvttsd2si {{[0-9]+}}(%esp), %eax
+; AVX-X86-NEXT:    # kill: def $al killed $al killed $eax
+; AVX-X86-NEXT:    retl
+;
+; AVX-X64-LABEL: fptosi_f64toi8:
+; AVX-X64:       # %bb.0:
+; AVX-X64-NEXT:    vcvttsd2si %xmm0, %eax
+; AVX-X64-NEXT:    # kill: def $al killed $al killed $eax
+; AVX-X64-NEXT:    retq
+;
+; CHECK-LABEL: fptosi_f64toi8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subl $8, %esp
+; CHECK-NEXT:    .cfi_def_cfa_offset 12
+; CHECK-NEXT:    fldl {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fnstcw {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    orl $3072, %eax # imm = 0xC00
+; CHECK-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fldcw {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fistps {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fldcw {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movb {{[0-9]+}}(%esp), %al
+; CHECK-NEXT:    addl $8, %esp
+; CHECK-NEXT:    .cfi_def_cfa_offset 4
+; CHECK-NEXT:    retl
+  %result = call i8 @llvm.experimental.constrained.fptosi.i8.f64(double %x,
+                                               metadata !"fpexcept.strict") #0
+  ret i8 %result
+}
+
+define i16 @fptosi_f64toi16(double %x) #0 {
+; SSE-X86-LABEL: fptosi_f64toi16:
+; SSE-X86:       # %bb.0:
+; SSE-X86-NEXT:    cvttsd2si {{[0-9]+}}(%esp), %eax
+; SSE-X86-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE-X86-NEXT:    retl
+;
+; SSE-X64-LABEL: fptosi_f64toi16:
+; SSE-X64:       # %bb.0:
+; SSE-X64-NEXT:    cvttsd2si %xmm0, %eax
+; SSE-X64-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE-X64-NEXT:    retq
+;
+; AVX-X86-LABEL: fptosi_f64toi16:
+; AVX-X86:       # %bb.0:
+; AVX-X86-NEXT:    vcvttsd2si {{[0-9]+}}(%esp), %eax
+; AVX-X86-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX-X86-NEXT:    retl
+;
+; AVX-X64-LABEL: fptosi_f64toi16:
+; AVX-X64:       # %bb.0:
+; AVX-X64-NEXT:    vcvttsd2si %xmm0, %eax
+; AVX-X64-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX-X64-NEXT:    retq
+;
+; CHECK-LABEL: fptosi_f64toi16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subl $8, %esp
+; CHECK-NEXT:    .cfi_def_cfa_offset 12
+; CHECK-NEXT:    fldl {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fnstcw {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    orl $3072, %eax # imm = 0xC00
+; CHECK-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fldcw {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fistps {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fldcw {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    addl $8, %esp
+; CHECK-NEXT:    .cfi_def_cfa_offset 4
+; CHECK-NEXT:    retl
+  %result = call i16 @llvm.experimental.constrained.fptosi.i16.f64(double %x,
+                                               metadata !"fpexcept.strict") #0
+  ret i16 %result
+}
+
+define i32 @fptosi_f64toi32(double %x) #0 {
+; SSE-X86-LABEL: fptosi_f64toi32:
+; SSE-X86:       # %bb.0:
+; SSE-X86-NEXT:    cvttsd2si {{[0-9]+}}(%esp), %eax
+; SSE-X86-NEXT:    retl
+;
+; SSE-X64-LABEL: fptosi_f64toi32:
+; SSE-X64:       # %bb.0:
+; SSE-X64-NEXT:    cvttsd2si %xmm0, %eax
+; SSE-X64-NEXT:    retq
+;
+; AVX-X86-LABEL: fptosi_f64toi32:
+; AVX-X86:       # %bb.0:
+; AVX-X86-NEXT:    vcvttsd2si {{[0-9]+}}(%esp), %eax
+; AVX-X86-NEXT:    retl
+;
+; AVX-X64-LABEL: fptosi_f64toi32:
+; AVX-X64:       # %bb.0:
+; AVX-X64-NEXT:    vcvttsd2si %xmm0, %eax
+; AVX-X64-NEXT:    retq
+;
+; CHECK-LABEL: fptosi_f64toi32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subl $8, %esp
+; CHECK-NEXT:    .cfi_def_cfa_offset 12
+; CHECK-NEXT:    fldl {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fnstcw (%esp)
+; CHECK-NEXT:    movzwl (%esp), %eax
+; CHECK-NEXT:    orl $3072, %eax # imm = 0xC00
+; CHECK-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fldcw {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fistpl {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fldcw (%esp)
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    addl $8, %esp
+; CHECK-NEXT:    .cfi_def_cfa_offset 4
+; CHECK-NEXT:    retl
+  %result = call i32 @llvm.experimental.constrained.fptosi.i32.f64(double %x,
+                                               metadata !"fpexcept.strict") #0
+  ret i32 %result
+}
+
+define i64 @fptosi_f64toi64(double %x) #0 {
+; SSE-X86-LABEL: fptosi_f64toi64:
+; SSE-X86:       # %bb.0:
+; SSE-X86-NEXT:    pushl %ebp
+; SSE-X86-NEXT:    .cfi_def_cfa_offset 8
+; SSE-X86-NEXT:    .cfi_offset %ebp, -8
+; SSE-X86-NEXT:    movl %esp, %ebp
+; SSE-X86-NEXT:    .cfi_def_cfa_register %ebp
+; SSE-X86-NEXT:    andl $-8, %esp
+; SSE-X86-NEXT:    subl $16, %esp
+; SSE-X86-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE-X86-NEXT:    movsd %xmm0, {{[0-9]+}}(%esp)
+; SSE-X86-NEXT:    fldl {{[0-9]+}}(%esp)
+; SSE-X86-NEXT:    fnstcw {{[0-9]+}}(%esp)
+; SSE-X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; SSE-X86-NEXT:    orl $3072, %eax # imm = 0xC00
+; SSE-X86-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; SSE-X86-NEXT:    fldcw {{[0-9]+}}(%esp)
+; SSE-X86-NEXT:    fistpll {{[0-9]+}}(%esp)
+; SSE-X86-NEXT:    fldcw {{[0-9]+}}(%esp)
+; SSE-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; SSE-X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; SSE-X86-NEXT:    movl %ebp, %esp
+; SSE-X86-NEXT:    popl %ebp
+; SSE-X86-NEXT:    .cfi_def_cfa %esp, 4
+; SSE-X86-NEXT:    retl
+;
+; SSE-X64-LABEL: fptosi_f64toi64:
+; SSE-X64:       # %bb.0:
+; SSE-X64-NEXT:    cvttsd2si %xmm0, %rax
+; SSE-X64-NEXT:    retq
+;
+; AVX-X86-LABEL: fptosi_f64toi64:
+; AVX-X86:       # %bb.0:
+; AVX-X86-NEXT:    pushl %ebp
+; AVX-X86-NEXT:    .cfi_def_cfa_offset 8
+; AVX-X86-NEXT:    .cfi_offset %ebp, -8
+; AVX-X86-NEXT:    movl %esp, %ebp
+; AVX-X86-NEXT:    .cfi_def_cfa_register %ebp
+; AVX-X86-NEXT:    andl $-8, %esp
+; AVX-X86-NEXT:    subl $8, %esp
+; AVX-X86-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-X86-NEXT:    vmovsd %xmm0, (%esp)
+; AVX-X86-NEXT:    fldl (%esp)
+; AVX-X86-NEXT:    fisttpll (%esp)
+; AVX-X86-NEXT:    movl (%esp), %eax
+; AVX-X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; AVX-X86-NEXT:    movl %ebp, %esp
+; AVX-X86-NEXT:    popl %ebp
+; AVX-X86-NEXT:    .cfi_def_cfa %esp, 4
+; AVX-X86-NEXT:    retl
+;
+; AVX-X64-LABEL: fptosi_f64toi64:
+; AVX-X64:       # %bb.0:
+; AVX-X64-NEXT:    vcvttsd2si %xmm0, %rax
+; AVX-X64-NEXT:    retq
+;
+; CHECK-LABEL: fptosi_f64toi64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushl %ebp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    .cfi_offset %ebp, -8
+; CHECK-NEXT:    movl %esp, %ebp
+; CHECK-NEXT:    .cfi_def_cfa_register %ebp
+; CHECK-NEXT:    andl $-8, %esp
+; CHECK-NEXT:    subl $16, %esp
+; CHECK-NEXT:    fldl 8(%ebp)
+; CHECK-NEXT:    fnstcw {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    orl $3072, %eax # imm = 0xC00
+; CHECK-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fldcw {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fistpll {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fldcw {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; CHECK-NEXT:    movl %ebp, %esp
+; CHECK-NEXT:    popl %ebp
+; CHECK-NEXT:    .cfi_def_cfa %esp, 4
+; CHECK-NEXT:    retl
+  %result = call i64 @llvm.experimental.constrained.fptosi.i64.f64(double %x,
+                                               metadata !"fpexcept.strict") #0
+  ret i64 %result
+}
+
+define i1 @fptoui_f64toi1(double %x) #0 {
+; SSE-X86-LABEL: fptoui_f64toi1:
+; SSE-X86:       # %bb.0:
+; SSE-X86-NEXT:    cvttsd2si {{[0-9]+}}(%esp), %eax
+; SSE-X86-NEXT:    # kill: def $al killed $al killed $eax
+; SSE-X86-NEXT:    retl
+;
+; SSE-X64-LABEL: fptoui_f64toi1:
+; SSE-X64:       # %bb.0:
+; SSE-X64-NEXT:    cvttsd2si %xmm0, %eax
+; SSE-X64-NEXT:    # kill: def $al killed $al killed $eax
+; SSE-X64-NEXT:    retq
+;
+; AVX-X86-LABEL: fptoui_f64toi1:
+; AVX-X86:       # %bb.0:
+; AVX-X86-NEXT:    vcvttsd2si {{[0-9]+}}(%esp), %eax
+; AVX-X86-NEXT:    # kill: def $al killed $al killed $eax
+; AVX-X86-NEXT:    retl
+;
+; AVX-X64-LABEL: fptoui_f64toi1:
+; AVX-X64:       # %bb.0:
+; AVX-X64-NEXT:    vcvttsd2si %xmm0, %eax
+; AVX-X64-NEXT:    # kill: def $al killed $al killed $eax
+; AVX-X64-NEXT:    retq
+;
+; CHECK-LABEL: fptoui_f64toi1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subl $8, %esp
+; CHECK-NEXT:    .cfi_def_cfa_offset 12
+; CHECK-NEXT:    fldl {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fnstcw {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    orl $3072, %eax # imm = 0xC00
+; CHECK-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fldcw {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fistps {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fldcw {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movb {{[0-9]+}}(%esp), %al
+; CHECK-NEXT:    addl $8, %esp
+; CHECK-NEXT:    .cfi_def_cfa_offset 4
+; CHECK-NEXT:    retl
+  %result = call i1 @llvm.experimental.constrained.fptoui.i1.f64(double %x,
+                                               metadata !"fpexcept.strict") #0
+  ret i1 %result
+}
+
+define i8 @fptoui_f64toi8(double %x) #0 {
+; SSE-X86-LABEL: fptoui_f64toi8:
+; SSE-X86:       # %bb.0:
+; SSE-X86-NEXT:    cvttsd2si {{[0-9]+}}(%esp), %eax
+; SSE-X86-NEXT:    # kill: def $al killed $al killed $eax
+; SSE-X86-NEXT:    retl
+;
+; SSE-X64-LABEL: fptoui_f64toi8:
+; SSE-X64:       # %bb.0:
+; SSE-X64-NEXT:    cvttsd2si %xmm0, %eax
+; SSE-X64-NEXT:    # kill: def $al killed $al killed $eax
+; SSE-X64-NEXT:    retq
+;
+; AVX-X86-LABEL: fptoui_f64toi8:
+; AVX-X86:       # %bb.0:
+; AVX-X86-NEXT:    vcvttsd2si {{[0-9]+}}(%esp), %eax
+; AVX-X86-NEXT:    # kill: def $al killed $al killed $eax
+; AVX-X86-NEXT:    retl
+;
+; AVX-X64-LABEL: fptoui_f64toi8:
+; AVX-X64:       # %bb.0:
+; AVX-X64-NEXT:    vcvttsd2si %xmm0, %eax
+; AVX-X64-NEXT:    # kill: def $al killed $al killed $eax
+; AVX-X64-NEXT:    retq
+;
+; CHECK-LABEL: fptoui_f64toi8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subl $8, %esp
+; CHECK-NEXT:    .cfi_def_cfa_offset 12
+; CHECK-NEXT:    fldl {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fnstcw {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    orl $3072, %eax # imm = 0xC00
+; CHECK-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fldcw {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fistps {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fldcw {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movb {{[0-9]+}}(%esp), %al
+; CHECK-NEXT:    addl $8, %esp
+; CHECK-NEXT:    .cfi_def_cfa_offset 4
+; CHECK-NEXT:    retl
+  %result = call i8 @llvm.experimental.constrained.fptoui.i8.f64(double %x,
+                                               metadata !"fpexcept.strict") #0
+  ret i8 %result
+}
+
+define i16 @fptoui_f64toi16(double %x) #0 {
+; SSE-X86-LABEL: fptoui_f64toi16:
+; SSE-X86:       # %bb.0:
+; SSE-X86-NEXT:    cvttsd2si {{[0-9]+}}(%esp), %eax
+; SSE-X86-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE-X86-NEXT:    retl
+;
+; SSE-X64-LABEL: fptoui_f64toi16:
+; SSE-X64:       # %bb.0:
+; SSE-X64-NEXT:    cvttsd2si %xmm0, %eax
+; SSE-X64-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE-X64-NEXT:    retq
+;
+; AVX-X86-LABEL: fptoui_f64toi16:
+; AVX-X86:       # %bb.0:
+; AVX-X86-NEXT:    vcvttsd2si {{[0-9]+}}(%esp), %eax
+; AVX-X86-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX-X86-NEXT:    retl
+;
+; AVX-X64-LABEL: fptoui_f64toi16:
+; AVX-X64:       # %bb.0:
+; AVX-X64-NEXT:    vcvttsd2si %xmm0, %eax
+; AVX-X64-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX-X64-NEXT:    retq
+;
+; CHECK-LABEL: fptoui_f64toi16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subl $8, %esp
+; CHECK-NEXT:    .cfi_def_cfa_offset 12
+; CHECK-NEXT:    fldl {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fnstcw (%esp)
+; CHECK-NEXT:    movzwl (%esp), %eax
+; CHECK-NEXT:    orl $3072, %eax # imm = 0xC00
+; CHECK-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fldcw {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fistpl {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fldcw (%esp)
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    # kill: def $ax killed $ax killed $eax
+; CHECK-NEXT:    addl $8, %esp
+; CHECK-NEXT:    .cfi_def_cfa_offset 4
+; CHECK-NEXT:    retl
+  %result = call i16 @llvm.experimental.constrained.fptoui.i16.f64(double %x,
+                                               metadata !"fpexcept.strict") #0
+  ret i16 %result
+}
+
+define i32 @fptoui_f64toi32(double %x) #0 {
+; SSE-X86-LABEL: fptoui_f64toi32:
+; SSE-X86:       # %bb.0:
+; SSE-X86-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE-X86-NEXT:    movsd {{.*#+}} xmm2 = mem[0],zero
+; SSE-X86-NEXT:    comisd %xmm0, %xmm2
+; SSE-X86-NEXT:    xorpd %xmm1, %xmm1
+; SSE-X86-NEXT:    ja .LBB17_2
+; SSE-X86-NEXT:  # %bb.1:
+; SSE-X86-NEXT:    movapd %xmm2, %xmm1
+; SSE-X86-NEXT:  .LBB17_2:
+; SSE-X86-NEXT:    setbe %al
+; SSE-X86-NEXT:    movzbl %al, %ecx
+; SSE-X86-NEXT:    shll $31, %ecx
+; SSE-X86-NEXT:    subsd %xmm1, %xmm0
+; SSE-X86-NEXT:    cvttsd2si %xmm0, %eax
+; SSE-X86-NEXT:    xorl %ecx, %eax
+; SSE-X86-NEXT:    retl
+;
+; SSE-X64-LABEL: fptoui_f64toi32:
+; SSE-X64:       # %bb.0:
+; SSE-X64-NEXT:    cvttsd2si %xmm0, %rax
+; SSE-X64-NEXT:    # kill: def $eax killed $eax killed $rax
+; SSE-X64-NEXT:    retq
+;
+; AVX1-X86-LABEL: fptoui_f64toi32:
+; AVX1-X86:       # %bb.0:
+; AVX1-X86-NEXT:    pushl %ebp
+; AVX1-X86-NEXT:    .cfi_def_cfa_offset 8
+; AVX1-X86-NEXT:    .cfi_offset %ebp, -8
+; AVX1-X86-NEXT:    movl %esp, %ebp
+; AVX1-X86-NEXT:    .cfi_def_cfa_register %ebp
+; AVX1-X86-NEXT:    andl $-8, %esp
+; AVX1-X86-NEXT:    subl $8, %esp
+; AVX1-X86-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX1-X86-NEXT:    vmovsd %xmm0, (%esp)
+; AVX1-X86-NEXT:    fldl (%esp)
+; AVX1-X86-NEXT:    fisttpll (%esp)
+; AVX1-X86-NEXT:    movl (%esp), %eax
+; AVX1-X86-NEXT:    movl %ebp, %esp
+; AVX1-X86-NEXT:    popl %ebp
+; AVX1-X86-NEXT:    .cfi_def_cfa %esp, 4
+; AVX1-X86-NEXT:    retl
+;
+; AVX1-X64-LABEL: fptoui_f64toi32:
+; AVX1-X64:       # %bb.0:
+; AVX1-X64-NEXT:    vcvttsd2si %xmm0, %rax
+; AVX1-X64-NEXT:    # kill: def $eax killed $eax killed $rax
+; AVX1-X64-NEXT:    retq
+;
+; AVX512-X86-LABEL: fptoui_f64toi32:
+; AVX512-X86:       # %bb.0:
+; AVX512-X86-NEXT:    vcvttsd2usi {{[0-9]+}}(%esp), %eax
+; AVX512-X86-NEXT:    retl
+;
+; AVX512-X64-LABEL: fptoui_f64toi32:
+; AVX512-X64:       # %bb.0:
+; AVX512-X64-NEXT:    vcvttsd2usi %xmm0, %eax
+; AVX512-X64-NEXT:    retq
+;
+; CHECK-LABEL: fptoui_f64toi32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushl %ebp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    .cfi_offset %ebp, -8
+; CHECK-NEXT:    movl %esp, %ebp
+; CHECK-NEXT:    .cfi_def_cfa_register %ebp
+; CHECK-NEXT:    andl $-8, %esp
+; CHECK-NEXT:    subl $16, %esp
+; CHECK-NEXT:    fldl 8(%ebp)
+; CHECK-NEXT:    fnstcw {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    orl $3072, %eax # imm = 0xC00
+; CHECK-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fldcw {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fistpll {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fldcw {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    movl %ebp, %esp
+; CHECK-NEXT:    popl %ebp
+; CHECK-NEXT:    .cfi_def_cfa %esp, 4
+; CHECK-NEXT:    retl
+  %result = call i32 @llvm.experimental.constrained.fptoui.i32.f64(double %x,
+                                               metadata !"fpexcept.strict") #0
+  ret i32 %result
+}
+
+define i64 @fptoui_f64toi64(double %x) #0 {
+; SSE-X86-LABEL: fptoui_f64toi64:
+; SSE-X86:       # %bb.0:
+; SSE-X86-NEXT:    pushl %ebp
+; SSE-X86-NEXT:    .cfi_def_cfa_offset 8
+; SSE-X86-NEXT:    .cfi_offset %ebp, -8
+; SSE-X86-NEXT:    movl %esp, %ebp
+; SSE-X86-NEXT:    .cfi_def_cfa_register %ebp
+; SSE-X86-NEXT:    andl $-8, %esp
+; SSE-X86-NEXT:    subl $16, %esp
+; SSE-X86-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE-X86-NEXT:    movsd {{.*#+}} xmm2 = mem[0],zero
+; SSE-X86-NEXT:    comisd %xmm0, %xmm2
+; SSE-X86-NEXT:    xorpd %xmm1, %xmm1
+; SSE-X86-NEXT:    ja .LBB18_2
+; SSE-X86-NEXT:  # %bb.1:
+; SSE-X86-NEXT:    movapd %xmm2, %xmm1
+; SSE-X86-NEXT:  .LBB18_2:
+; SSE-X86-NEXT:    subsd %xmm1, %xmm0
+; SSE-X86-NEXT:    movsd %xmm0, {{[0-9]+}}(%esp)
+; SSE-X86-NEXT:    setbe %al
+; SSE-X86-NEXT:    fldl {{[0-9]+}}(%esp)
+; SSE-X86-NEXT:    fnstcw {{[0-9]+}}(%esp)
+; SSE-X86-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
+; SSE-X86-NEXT:    orl $3072, %ecx # imm = 0xC00
+; SSE-X86-NEXT:    movw %cx, {{[0-9]+}}(%esp)
+; SSE-X86-NEXT:    fldcw {{[0-9]+}}(%esp)
+; SSE-X86-NEXT:    fistpll {{[0-9]+}}(%esp)
+; SSE-X86-NEXT:    fldcw {{[0-9]+}}(%esp)
+; SSE-X86-NEXT:    movzbl %al, %edx
+; SSE-X86-NEXT:    shll $31, %edx
+; SSE-X86-NEXT:    xorl {{[0-9]+}}(%esp), %edx
+; SSE-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; SSE-X86-NEXT:    movl %ebp, %esp
+; SSE-X86-NEXT:    popl %ebp
+; SSE-X86-NEXT:    .cfi_def_cfa %esp, 4
+; SSE-X86-NEXT:    retl
+;
+; SSE-X64-LABEL: fptoui_f64toi64:
+; SSE-X64:       # %bb.0:
+; SSE-X64-NEXT:    movsd {{.*#+}} xmm2 = mem[0],zero
+; SSE-X64-NEXT:    comisd %xmm2, %xmm0
+; SSE-X64-NEXT:    xorpd %xmm1, %xmm1
+; SSE-X64-NEXT:    jb .LBB18_2
+; SSE-X64-NEXT:  # %bb.1:
+; SSE-X64-NEXT:    movapd %xmm2, %xmm1
+; SSE-X64-NEXT:  .LBB18_2:
+; SSE-X64-NEXT:    subsd %xmm1, %xmm0
+; SSE-X64-NEXT:    cvttsd2si %xmm0, %rcx
+; SSE-X64-NEXT:    setae %al
+; SSE-X64-NEXT:    movzbl %al, %eax
+; SSE-X64-NEXT:    shlq $63, %rax
+; SSE-X64-NEXT:    xorq %rcx, %rax
+; SSE-X64-NEXT:    retq
+;
+; AVX1-X86-LABEL: fptoui_f64toi64:
+; AVX1-X86:       # %bb.0:
+; AVX1-X86-NEXT:    pushl %ebp
+; AVX1-X86-NEXT:    .cfi_def_cfa_offset 8
+; AVX1-X86-NEXT:    .cfi_offset %ebp, -8
+; AVX1-X86-NEXT:    movl %esp, %ebp
+; AVX1-X86-NEXT:    .cfi_def_cfa_register %ebp
+; AVX1-X86-NEXT:    andl $-8, %esp
+; AVX1-X86-NEXT:    subl $8, %esp
+; AVX1-X86-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX1-X86-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
+; AVX1-X86-NEXT:    vcomisd %xmm0, %xmm1
+; AVX1-X86-NEXT:    vxorpd %xmm2, %xmm2, %xmm2
+; AVX1-X86-NEXT:    ja .LBB18_2
+; AVX1-X86-NEXT:  # %bb.1:
+; AVX1-X86-NEXT:    vmovapd %xmm1, %xmm2
+; AVX1-X86-NEXT:  .LBB18_2:
+; AVX1-X86-NEXT:    vsubsd %xmm2, %xmm0, %xmm0
+; AVX1-X86-NEXT:    vmovsd %xmm0, (%esp)
+; AVX1-X86-NEXT:    fldl (%esp)
+; AVX1-X86-NEXT:    fisttpll (%esp)
+; AVX1-X86-NEXT:    setbe %al
+; AVX1-X86-NEXT:    movzbl %al, %edx
+; AVX1-X86-NEXT:    shll $31, %edx
+; AVX1-X86-NEXT:    xorl {{[0-9]+}}(%esp), %edx
+; AVX1-X86-NEXT:    movl (%esp), %eax
+; AVX1-X86-NEXT:    movl %ebp, %esp
+; AVX1-X86-NEXT:    popl %ebp
+; AVX1-X86-NEXT:    .cfi_def_cfa %esp, 4
+; AVX1-X86-NEXT:    retl
+;
+; AVX1-X64-LABEL: fptoui_f64toi64:
+; AVX1-X64:       # %bb.0:
+; AVX1-X64-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
+; AVX1-X64-NEXT:    vcomisd %xmm1, %xmm0
+; AVX1-X64-NEXT:    vxorpd %xmm2, %xmm2, %xmm2
+; AVX1-X64-NEXT:    jb .LBB18_2
+; AVX1-X64-NEXT:  # %bb.1:
+; AVX1-X64-NEXT:    vmovapd %xmm1, %xmm2
+; AVX1-X64-NEXT:  .LBB18_2:
+; AVX1-X64-NEXT:    vsubsd %xmm2, %xmm0, %xmm0
+; AVX1-X64-NEXT:    vcvttsd2si %xmm0, %rcx
+; AVX1-X64-NEXT:    setae %al
+; AVX1-X64-NEXT:    movzbl %al, %eax
+; AVX1-X64-NEXT:    shlq $63, %rax
+; AVX1-X64-NEXT:    xorq %rcx, %rax
+; AVX1-X64-NEXT:    retq
+;
+; AVX512-X86-LABEL: fptoui_f64toi64:
+; AVX512-X86:       # %bb.0:
+; AVX512-X86-NEXT:    pushl %ebp
+; AVX512-X86-NEXT:    .cfi_def_cfa_offset 8
+; AVX512-X86-NEXT:    .cfi_offset %ebp, -8
+; AVX512-X86-NEXT:    movl %esp, %ebp
+; AVX512-X86-NEXT:    .cfi_def_cfa_register %ebp
+; AVX512-X86-NEXT:    andl $-8, %esp
+; AVX512-X86-NEXT:    subl $8, %esp
+; AVX512-X86-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX512-X86-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
+; AVX512-X86-NEXT:    xorl %edx, %edx
+; AVX512-X86-NEXT:    vcomisd %xmm0, %xmm1
+; AVX512-X86-NEXT:    seta %al
+; AVX512-X86-NEXT:    kmovw %eax, %k1
+; AVX512-X86-NEXT:    vxorpd %xmm2, %xmm2, %xmm2
+; AVX512-X86-NEXT:    vmovsd %xmm2, %xmm1, %xmm1 {%k1}
+; AVX512-X86-NEXT:    vsubsd %xmm1, %xmm0, %xmm0
+; AVX512-X86-NEXT:    vmovsd %xmm0, (%esp)
+; AVX512-X86-NEXT:    fldl (%esp)
+; AVX512-X86-NEXT:    fisttpll (%esp)
+; AVX512-X86-NEXT:    setbe %dl
+; AVX512-X86-NEXT:    shll $31, %edx
+; AVX512-X86-NEXT:    xorl {{[0-9]+}}(%esp), %edx
+; AVX512-X86-NEXT:    movl (%esp), %eax
+; AVX512-X86-NEXT:    movl %ebp, %esp
+; AVX512-X86-NEXT:    popl %ebp
+; AVX512-X86-NEXT:    .cfi_def_cfa %esp, 4
+; AVX512-X86-NEXT:    retl
+;
+; AVX512-X64-LABEL: fptoui_f64toi64:
+; AVX512-X64:       # %bb.0:
+; AVX512-X64-NEXT:    vcvttsd2usi %xmm0, %rax
+; AVX512-X64-NEXT:    retq
+;
+; CHECK-LABEL: fptoui_f64toi64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushl %ebp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    .cfi_offset %ebp, -8
+; CHECK-NEXT:    movl %esp, %ebp
+; CHECK-NEXT:    .cfi_def_cfa_register %ebp
+; CHECK-NEXT:    andl $-8, %esp
+; CHECK-NEXT:    subl $16, %esp
+; CHECK-NEXT:    fldl 8(%ebp)
+; CHECK-NEXT:    flds {{\.LCPI.*}}
+; CHECK-NEXT:    fcom %st(1)
+; CHECK-NEXT:    fnstsw %ax
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    # kill: def $ah killed $ah killed $ax
+; CHECK-NEXT:    sahf
+; CHECK-NEXT:    setbe %al
+; CHECK-NEXT:    fldz
+; CHECK-NEXT:    ja .LBB18_2
+; CHECK-NEXT:  # %bb.1:
+; CHECK-NEXT:    fstp %st(0)
+; CHECK-NEXT:    fldz
+; CHECK-NEXT:    fxch %st(1)
+; CHECK-NEXT:  .LBB18_2:
+; CHECK-NEXT:    fstp %st(1)
+; CHECK-NEXT:    fsubrp %st, %st(1)
+; CHECK-NEXT:    fnstcw {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT:    orl $3072, %ecx # imm = 0xC00
+; CHECK-NEXT:    movw %cx, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fldcw {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fistpll {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fldcw {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movb %al, %dl
+; CHECK-NEXT:    shll $31, %edx
+; CHECK-NEXT:    xorl {{[0-9]+}}(%esp), %edx
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    movl %ebp, %esp
+; CHECK-NEXT:    popl %ebp
+; CHECK-NEXT:    .cfi_def_cfa %esp, 4
+; CHECK-NEXT:    retl
+  %result = call i64 @llvm.experimental.constrained.fptoui.i64.f64(double %x,
+                                               metadata !"fpexcept.strict") #0
+  ret i64 %result
+}
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/X86/fp-strict-scalar-inttofp.ll b/llvm/test/CodeGen/X86/fp-strict-scalar-inttofp.ll
new file mode 100644
index 000000000..887185af8
--- /dev/null
+++ b/llvm/test/CodeGen/X86/fp-strict-scalar-inttofp.ll
@@ -0,0 +1,1299 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse2 -O3 | FileCheck %s --check-prefixes=SSE-X86
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 -O3 | FileCheck %s --check-prefixes=SSE-X64
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx -O3 | FileCheck %s --check-prefixes=AVX-X86,AVX1-X86
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx -O3 | FileCheck %s --check-prefixes=AVX-X64,AVX1-X64
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx512f -mattr=+avx512vl -O3 | FileCheck %s --check-prefixes=AVX-X86,AVX512-X86
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f -mattr=+avx512vl -O3 | FileCheck %s --check-prefixes=AVX-X64,AVX512-X64
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=-sse -O3 | FileCheck %s --check-prefixes=X87
+
+declare float @llvm.experimental.constrained.sitofp.f32.i1(i1, metadata, metadata)
+declare float @llvm.experimental.constrained.sitofp.f32.i8(i8, metadata, metadata)
+declare float @llvm.experimental.constrained.sitofp.f32.i16(i16, metadata, metadata)
+declare float @llvm.experimental.constrained.sitofp.f32.i32(i32, metadata, metadata)
+declare float @llvm.experimental.constrained.sitofp.f32.i64(i64, metadata, metadata)
+declare float @llvm.experimental.constrained.uitofp.f32.i1(i1, metadata, metadata)
+declare float @llvm.experimental.constrained.uitofp.f32.i8(i8, metadata, metadata)
+declare float @llvm.experimental.constrained.uitofp.f32.i16(i16, metadata, metadata)
+declare float @llvm.experimental.constrained.uitofp.f32.i32(i32, metadata, metadata)
+declare float @llvm.experimental.constrained.uitofp.f32.i64(i64, metadata, metadata)
+
+declare double @llvm.experimental.constrained.sitofp.f64.i1(i1, metadata, metadata)
+declare double @llvm.experimental.constrained.sitofp.f64.i8(i8, metadata, metadata)
+declare double @llvm.experimental.constrained.sitofp.f64.i16(i16, metadata, metadata)
+declare double @llvm.experimental.constrained.sitofp.f64.i32(i32, metadata, metadata)
+declare double @llvm.experimental.constrained.sitofp.f64.i64(i64, metadata, metadata)
+declare double @llvm.experimental.constrained.uitofp.f64.i1(i1, metadata, metadata)
+declare double @llvm.experimental.constrained.uitofp.f64.i8(i8, metadata, metadata)
+declare double @llvm.experimental.constrained.uitofp.f64.i16(i16, metadata, metadata)
+declare double @llvm.experimental.constrained.uitofp.f64.i32(i32, metadata, metadata)
+declare double @llvm.experimental.constrained.uitofp.f64.i64(i64, metadata, metadata)
+
+define float @sitofp_i1tof32(i1 %x) #0 {
+; SSE-X86-LABEL: sitofp_i1tof32:
+; SSE-X86:       # %bb.0:
+; SSE-X86-NEXT:    pushl %eax
+; SSE-X86-NEXT:    .cfi_def_cfa_offset 8
+; SSE-X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; SSE-X86-NEXT:    andb $1, %al
+; SSE-X86-NEXT:    negb %al
+; SSE-X86-NEXT:    movsbl %al, %eax
+; SSE-X86-NEXT:    cvtsi2ss %eax, %xmm0
+; SSE-X86-NEXT:    movss %xmm0, (%esp)
+; SSE-X86-NEXT:    flds (%esp)
+; SSE-X86-NEXT:    popl %eax
+; SSE-X86-NEXT:    .cfi_def_cfa_offset 4
+; SSE-X86-NEXT:    retl
+;
+; SSE-X64-LABEL: sitofp_i1tof32:
+; SSE-X64:       # %bb.0:
+; SSE-X64-NEXT:    andb $1, %dil
+; SSE-X64-NEXT:    negb %dil
+; SSE-X64-NEXT:    movsbl %dil, %eax
+; SSE-X64-NEXT:    cvtsi2ss %eax, %xmm0
+; SSE-X64-NEXT:    retq
+;
+; AVX-X86-LABEL: sitofp_i1tof32:
+; AVX-X86:       # %bb.0:
+; AVX-X86-NEXT:    pushl %eax
+; AVX-X86-NEXT:    .cfi_def_cfa_offset 8
+; AVX-X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; AVX-X86-NEXT:    andb $1, %al
+; AVX-X86-NEXT:    negb %al
+; AVX-X86-NEXT:    movsbl %al, %eax
+; AVX-X86-NEXT:    vcvtsi2ss %eax, %xmm0, %xmm0
+; AVX-X86-NEXT:    vmovss %xmm0, (%esp)
+; AVX-X86-NEXT:    flds (%esp)
+; AVX-X86-NEXT:    popl %eax
+; AVX-X86-NEXT:    .cfi_def_cfa_offset 4
+; AVX-X86-NEXT:    retl
+;
+; AVX-X64-LABEL: sitofp_i1tof32:
+; AVX-X64:       # %bb.0:
+; AVX-X64-NEXT:    andb $1, %dil
+; AVX-X64-NEXT:    negb %dil
+; AVX-X64-NEXT:    movsbl %dil, %eax
+; AVX-X64-NEXT:    vcvtsi2ss %eax, %xmm0, %xmm0
+; AVX-X64-NEXT:    retq
+;
+; X87-LABEL: sitofp_i1tof32:
+; X87:       # %bb.0:
+; X87-NEXT:    pushl %eax
+; X87-NEXT:    .cfi_def_cfa_offset 8
+; X87-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X87-NEXT:    andb $1, %al
+; X87-NEXT:    negb %al
+; X87-NEXT:    movsbl %al, %eax
+; X87-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; X87-NEXT:    filds {{[0-9]+}}(%esp)
+; X87-NEXT:    popl %eax
+; X87-NEXT:    .cfi_def_cfa_offset 4
+; X87-NEXT:    retl
+  %result = call float @llvm.experimental.constrained.sitofp.f32.i1(i1 %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret float %result
+}
+
+define float @sitofp_i8tof32(i8 %x) #0 {
+; SSE-X86-LABEL: sitofp_i8tof32:
+; SSE-X86:       # %bb.0:
+; SSE-X86-NEXT:    pushl %eax
+; SSE-X86-NEXT:    .cfi_def_cfa_offset 8
+; SSE-X86-NEXT:    movsbl {{[0-9]+}}(%esp), %eax
+; SSE-X86-NEXT:    cvtsi2ss %eax, %xmm0
+; SSE-X86-NEXT:    movss %xmm0, (%esp)
+; SSE-X86-NEXT:    flds (%esp)
+; SSE-X86-NEXT:    popl %eax
+; SSE-X86-NEXT:    .cfi_def_cfa_offset 4
+; SSE-X86-NEXT:    retl
+;
+; SSE-X64-LABEL: sitofp_i8tof32:
+; SSE-X64:       # %bb.0:
+; SSE-X64-NEXT:    movsbl %dil, %eax
+; SSE-X64-NEXT:    cvtsi2ss %eax, %xmm0
+; SSE-X64-NEXT:    retq
+;
+; AVX-X86-LABEL: sitofp_i8tof32:
+; AVX-X86:       # %bb.0:
+; AVX-X86-NEXT:    pushl %eax
+; AVX-X86-NEXT:    .cfi_def_cfa_offset 8
+; AVX-X86-NEXT:    movsbl {{[0-9]+}}(%esp), %eax
+; AVX-X86-NEXT:    vcvtsi2ss %eax, %xmm0, %xmm0
+; AVX-X86-NEXT:    vmovss %xmm0, (%esp)
+; AVX-X86-NEXT:    flds (%esp)
+; AVX-X86-NEXT:    popl %eax
+; AVX-X86-NEXT:    .cfi_def_cfa_offset 4
+; AVX-X86-NEXT:    retl
+;
+; AVX-X64-LABEL: sitofp_i8tof32:
+; AVX-X64:       # %bb.0:
+; AVX-X64-NEXT:    movsbl %dil, %eax
+; AVX-X64-NEXT:    vcvtsi2ss %eax, %xmm0, %xmm0
+; AVX-X64-NEXT:    retq
+;
+; X87-LABEL: sitofp_i8tof32:
+; X87:       # %bb.0:
+; X87-NEXT:    pushl %eax
+; X87-NEXT:    .cfi_def_cfa_offset 8
+; X87-NEXT:    movsbl {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; X87-NEXT:    filds {{[0-9]+}}(%esp)
+; X87-NEXT:    popl %eax
+; X87-NEXT:    .cfi_def_cfa_offset 4
+; X87-NEXT:    retl
+  %result = call float @llvm.experimental.constrained.sitofp.f32.i8(i8 %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret float %result
+}
+
+define float @sitofp_i16tof32(i16 %x) #0 {
+; SSE-X86-LABEL: sitofp_i16tof32:
+; SSE-X86:       # %bb.0:
+; SSE-X86-NEXT:    pushl %eax
+; SSE-X86-NEXT:    .cfi_def_cfa_offset 8
+; SSE-X86-NEXT:    movswl {{[0-9]+}}(%esp), %eax
+; SSE-X86-NEXT:    cvtsi2ss %eax, %xmm0
+; SSE-X86-NEXT:    movss %xmm0, (%esp)
+; SSE-X86-NEXT:    flds (%esp)
+; SSE-X86-NEXT:    popl %eax
+; SSE-X86-NEXT:    .cfi_def_cfa_offset 4
+; SSE-X86-NEXT:    retl
+;
+; SSE-X64-LABEL: sitofp_i16tof32:
+; SSE-X64:       # %bb.0:
+; SSE-X64-NEXT:    movswl %di, %eax
+; SSE-X64-NEXT:    cvtsi2ss %eax, %xmm0
+; SSE-X64-NEXT:    retq
+;
+; AVX-X86-LABEL: sitofp_i16tof32:
+; AVX-X86:       # %bb.0:
+; AVX-X86-NEXT:    pushl %eax
+; AVX-X86-NEXT:    .cfi_def_cfa_offset 8
+; AVX-X86-NEXT:    movswl {{[0-9]+}}(%esp), %eax
+; AVX-X86-NEXT:    vcvtsi2ss %eax, %xmm0, %xmm0
+; AVX-X86-NEXT:    vmovss %xmm0, (%esp)
+; AVX-X86-NEXT:    flds (%esp)
+; AVX-X86-NEXT:    popl %eax
+; AVX-X86-NEXT:    .cfi_def_cfa_offset 4
+; AVX-X86-NEXT:    retl
+;
+; AVX-X64-LABEL: sitofp_i16tof32:
+; AVX-X64:       # %bb.0:
+; AVX-X64-NEXT:    movswl %di, %eax
+; AVX-X64-NEXT:    vcvtsi2ss %eax, %xmm0, %xmm0
+; AVX-X64-NEXT:    retq
+;
+; X87-LABEL: sitofp_i16tof32:
+; X87:       # %bb.0:
+; X87-NEXT:    pushl %eax
+; X87-NEXT:    .cfi_def_cfa_offset 8
+; X87-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; X87-NEXT:    filds {{[0-9]+}}(%esp)
+; X87-NEXT:    popl %eax
+; X87-NEXT:    .cfi_def_cfa_offset 4
+; X87-NEXT:    retl
+  %result = call float @llvm.experimental.constrained.sitofp.f32.i16(i16 %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret float %result
+}
+
+define float @sitofp_i32tof32(i32 %x) #0 {
+; SSE-X86-LABEL: sitofp_i32tof32:
+; SSE-X86:       # %bb.0:
+; SSE-X86-NEXT:    pushl %eax
+; SSE-X86-NEXT:    .cfi_def_cfa_offset 8
+; SSE-X86-NEXT:    cvtsi2ssl {{[0-9]+}}(%esp), %xmm0
+; SSE-X86-NEXT:    movss %xmm0, (%esp)
+; SSE-X86-NEXT:    flds (%esp)
+; SSE-X86-NEXT:    popl %eax
+; SSE-X86-NEXT:    .cfi_def_cfa_offset 4
+; SSE-X86-NEXT:    retl
+;
+; SSE-X64-LABEL: sitofp_i32tof32:
+; SSE-X64:       # %bb.0:
+; SSE-X64-NEXT:    cvtsi2ss %edi, %xmm0
+; SSE-X64-NEXT:    retq
+;
+; AVX-X86-LABEL: sitofp_i32tof32:
+; AVX-X86:       # %bb.0:
+; AVX-X86-NEXT:    pushl %eax
+; AVX-X86-NEXT:    .cfi_def_cfa_offset 8
+; AVX-X86-NEXT:    vcvtsi2ssl {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-X86-NEXT:    vmovss %xmm0, (%esp)
+; AVX-X86-NEXT:    flds (%esp)
+; AVX-X86-NEXT:    popl %eax
+; AVX-X86-NEXT:    .cfi_def_cfa_offset 4
+; AVX-X86-NEXT:    retl
+;
+; AVX-X64-LABEL: sitofp_i32tof32:
+; AVX-X64:       # %bb.0:
+; AVX-X64-NEXT:    vcvtsi2ss %edi, %xmm0, %xmm0
+; AVX-X64-NEXT:    retq
+;
+; X87-LABEL: sitofp_i32tof32:
+; X87:       # %bb.0:
+; X87-NEXT:    pushl %eax
+; X87-NEXT:    .cfi_def_cfa_offset 8
+; X87-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl %eax, (%esp)
+; X87-NEXT:    fildl (%esp)
+; X87-NEXT:    popl %eax
+; X87-NEXT:    .cfi_def_cfa_offset 4
+; X87-NEXT:    retl
+  %result = call float @llvm.experimental.constrained.sitofp.f32.i32(i32 %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret float %result
+}
+
+define float @sitofp_i64tof32(i64 %x) #0 {
+; SSE-X86-LABEL: sitofp_i64tof32:
+; SSE-X86:       # %bb.0:
+; SSE-X86-NEXT:    pushl %eax
+; SSE-X86-NEXT:    .cfi_def_cfa_offset 8
+; SSE-X86-NEXT:    fildll {{[0-9]+}}(%esp)
+; SSE-X86-NEXT:    fstps (%esp)
+; SSE-X86-NEXT:    flds (%esp)
+; SSE-X86-NEXT:    popl %eax
+; SSE-X86-NEXT:    .cfi_def_cfa_offset 4
+; SSE-X86-NEXT:    retl
+;
+; SSE-X64-LABEL: sitofp_i64tof32:
+; SSE-X64:       # %bb.0:
+; SSE-X64-NEXT:    cvtsi2ss %rdi, %xmm0
+; SSE-X64-NEXT:    retq
+;
+; AVX-X86-LABEL: sitofp_i64tof32:
+; AVX-X86:       # %bb.0:
+; AVX-X86-NEXT:    pushl %eax
+; AVX-X86-NEXT:    .cfi_def_cfa_offset 8
+; AVX-X86-NEXT:    fildll {{[0-9]+}}(%esp)
+; AVX-X86-NEXT:    fstps (%esp)
+; AVX-X86-NEXT:    flds (%esp)
+; AVX-X86-NEXT:    popl %eax
+; AVX-X86-NEXT:    .cfi_def_cfa_offset 4
+; AVX-X86-NEXT:    retl
+;
+; AVX-X64-LABEL: sitofp_i64tof32:
+; AVX-X64:       # %bb.0:
+; AVX-X64-NEXT:    vcvtsi2ss %rdi, %xmm0, %xmm0
+; AVX-X64-NEXT:    retq
+;
+; X87-LABEL: sitofp_i64tof32:
+; X87:       # %bb.0:
+; X87-NEXT:    fildll {{[0-9]+}}(%esp)
+; X87-NEXT:    retl
+  %result = call float @llvm.experimental.constrained.sitofp.f32.i64(i64 %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret float %result
+}
+
+define float @uitofp_i1tof32(i1 %x) #0 {
+; SSE-X86-LABEL: uitofp_i1tof32:
+; SSE-X86:       # %bb.0:
+; SSE-X86-NEXT:    pushl %eax
+; SSE-X86-NEXT:    .cfi_def_cfa_offset 8
+; SSE-X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; SSE-X86-NEXT:    andb $1, %al
+; SSE-X86-NEXT:    movzbl %al, %eax
+; SSE-X86-NEXT:    cvtsi2ss %eax, %xmm0
+; SSE-X86-NEXT:    movss %xmm0, (%esp)
+; SSE-X86-NEXT:    flds (%esp)
+; SSE-X86-NEXT:    popl %eax
+; SSE-X86-NEXT:    .cfi_def_cfa_offset 4
+; SSE-X86-NEXT:    retl
+;
+; SSE-X64-LABEL: uitofp_i1tof32:
+; SSE-X64:       # %bb.0:
+; SSE-X64-NEXT:    andl $1, %edi
+; SSE-X64-NEXT:    cvtsi2ss %edi, %xmm0
+; SSE-X64-NEXT:    retq
+;
+; AVX-X86-LABEL: uitofp_i1tof32:
+; AVX-X86:       # %bb.0:
+; AVX-X86-NEXT:    pushl %eax
+; AVX-X86-NEXT:    .cfi_def_cfa_offset 8
+; AVX-X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; AVX-X86-NEXT:    andb $1, %al
+; AVX-X86-NEXT:    movzbl %al, %eax
+; AVX-X86-NEXT:    vcvtsi2ss %eax, %xmm0, %xmm0
+; AVX-X86-NEXT:    vmovss %xmm0, (%esp)
+; AVX-X86-NEXT:    flds (%esp)
+; AVX-X86-NEXT:    popl %eax
+; AVX-X86-NEXT:    .cfi_def_cfa_offset 4
+; AVX-X86-NEXT:    retl
+;
+; AVX-X64-LABEL: uitofp_i1tof32:
+; AVX-X64:       # %bb.0:
+; AVX-X64-NEXT:    andl $1, %edi
+; AVX-X64-NEXT:    vcvtsi2ss %edi, %xmm0, %xmm0
+; AVX-X64-NEXT:    retq
+;
+; X87-LABEL: uitofp_i1tof32:
+; X87:       # %bb.0:
+; X87-NEXT:    pushl %eax
+; X87-NEXT:    .cfi_def_cfa_offset 8
+; X87-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X87-NEXT:    andb $1, %al
+; X87-NEXT:    movzbl %al, %eax
+; X87-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; X87-NEXT:    filds {{[0-9]+}}(%esp)
+; X87-NEXT:    popl %eax
+; X87-NEXT:    .cfi_def_cfa_offset 4
+; X87-NEXT:    retl
+  %result = call float @llvm.experimental.constrained.uitofp.f32.i1(i1 %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret float %result
+}
+
+define float @uitofp_i8tof32(i8 %x) #0 {
+; SSE-X86-LABEL: uitofp_i8tof32:
+; SSE-X86:       # %bb.0:
+; SSE-X86-NEXT:    pushl %eax
+; SSE-X86-NEXT:    .cfi_def_cfa_offset 8
+; SSE-X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; SSE-X86-NEXT:    cvtsi2ss %eax, %xmm0
+; SSE-X86-NEXT:    movss %xmm0, (%esp)
+; SSE-X86-NEXT:    flds (%esp)
+; SSE-X86-NEXT:    popl %eax
+; SSE-X86-NEXT:    .cfi_def_cfa_offset 4
+; SSE-X86-NEXT:    retl
+;
+; SSE-X64-LABEL: uitofp_i8tof32:
+; SSE-X64:       # %bb.0:
+; SSE-X64-NEXT:    movzbl %dil, %eax
+; SSE-X64-NEXT:    cvtsi2ss %eax, %xmm0
+; SSE-X64-NEXT:    retq
+;
+; AVX-X86-LABEL: uitofp_i8tof32:
+; AVX-X86:       # %bb.0:
+; AVX-X86-NEXT:    pushl %eax
+; AVX-X86-NEXT:    .cfi_def_cfa_offset 8
+; AVX-X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; AVX-X86-NEXT:    vcvtsi2ss %eax, %xmm0, %xmm0
+; AVX-X86-NEXT:    vmovss %xmm0, (%esp)
+; AVX-X86-NEXT:    flds (%esp)
+; AVX-X86-NEXT:    popl %eax
+; AVX-X86-NEXT:    .cfi_def_cfa_offset 4
+; AVX-X86-NEXT:    retl
+;
+; AVX-X64-LABEL: uitofp_i8tof32:
+; AVX-X64:       # %bb.0:
+; AVX-X64-NEXT:    movzbl %dil, %eax
+; AVX-X64-NEXT:    vcvtsi2ss %eax, %xmm0, %xmm0
+; AVX-X64-NEXT:    retq
+;
+; X87-LABEL: uitofp_i8tof32:
+; X87:       # %bb.0:
+; X87-NEXT:    pushl %eax
+; X87-NEXT:    .cfi_def_cfa_offset 8
+; X87-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; X87-NEXT:    filds {{[0-9]+}}(%esp)
+; X87-NEXT:    popl %eax
+; X87-NEXT:    .cfi_def_cfa_offset 4
+; X87-NEXT:    retl
+  %result = call float @llvm.experimental.constrained.uitofp.f32.i8(i8 %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret float %result
+}
+
+define float @uitofp_i16tof32(i16 %x) #0 {
+; SSE-X86-LABEL: uitofp_i16tof32:
+; SSE-X86:       # %bb.0:
+; SSE-X86-NEXT:    pushl %eax
+; SSE-X86-NEXT:    .cfi_def_cfa_offset 8
+; SSE-X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; SSE-X86-NEXT:    cvtsi2ss %eax, %xmm0
+; SSE-X86-NEXT:    movss %xmm0, (%esp)
+; SSE-X86-NEXT:    flds (%esp)
+; SSE-X86-NEXT:    popl %eax
+; SSE-X86-NEXT:    .cfi_def_cfa_offset 4
+; SSE-X86-NEXT:    retl
+;
+; SSE-X64-LABEL: uitofp_i16tof32:
+; SSE-X64:       # %bb.0:
+; SSE-X64-NEXT:    movzwl %di, %eax
+; SSE-X64-NEXT:    cvtsi2ss %eax, %xmm0
+; SSE-X64-NEXT:    retq
+;
+; AVX-X86-LABEL: uitofp_i16tof32:
+; AVX-X86:       # %bb.0:
+; AVX-X86-NEXT:    pushl %eax
+; AVX-X86-NEXT:    .cfi_def_cfa_offset 8
+; AVX-X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; AVX-X86-NEXT:    vcvtsi2ss %eax, %xmm0, %xmm0
+; AVX-X86-NEXT:    vmovss %xmm0, (%esp)
+; AVX-X86-NEXT:    flds (%esp)
+; AVX-X86-NEXT:    popl %eax
+; AVX-X86-NEXT:    .cfi_def_cfa_offset 4
+; AVX-X86-NEXT:    retl
+;
+; AVX-X64-LABEL: uitofp_i16tof32:
+; AVX-X64:       # %bb.0:
+; AVX-X64-NEXT:    movzwl %di, %eax
+; AVX-X64-NEXT:    vcvtsi2ss %eax, %xmm0, %xmm0
+; AVX-X64-NEXT:    retq
+;
+; X87-LABEL: uitofp_i16tof32:
+; X87:       # %bb.0:
+; X87-NEXT:    pushl %eax
+; X87-NEXT:    .cfi_def_cfa_offset 8
+; X87-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl %eax, (%esp)
+; X87-NEXT:    fildl (%esp)
+; X87-NEXT:    popl %eax
+; X87-NEXT:    .cfi_def_cfa_offset 4
+; X87-NEXT:    retl
+  %result = call float @llvm.experimental.constrained.uitofp.f32.i16(i16 %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret float %result
+}
+
+define float @uitofp_i32tof32(i32 %x) #0 {
+; SSE-X86-LABEL: uitofp_i32tof32:
+; SSE-X86:       # %bb.0:
+; SSE-X86-NEXT:    pushl %eax
+; SSE-X86-NEXT:    .cfi_def_cfa_offset 8
+; SSE-X86-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE-X86-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE-X86-NEXT:    orpd %xmm0, %xmm1
+; SSE-X86-NEXT:    subsd %xmm0, %xmm1
+; SSE-X86-NEXT:    xorps %xmm0, %xmm0
+; SSE-X86-NEXT:    cvtsd2ss %xmm1, %xmm0
+; SSE-X86-NEXT:    movss %xmm0, (%esp)
+; SSE-X86-NEXT:    flds (%esp)
+; SSE-X86-NEXT:    popl %eax
+; SSE-X86-NEXT:    .cfi_def_cfa_offset 4
+; SSE-X86-NEXT:    retl
+;
+; SSE-X64-LABEL: uitofp_i32tof32:
+; SSE-X64:       # %bb.0:
+; SSE-X64-NEXT:    movl %edi, %eax
+; SSE-X64-NEXT:    cvtsi2ss %rax, %xmm0
+; SSE-X64-NEXT:    retq
+;
+; AVX1-X86-LABEL: uitofp_i32tof32:
+; AVX1-X86:       # %bb.0:
+; AVX1-X86-NEXT:    pushl %eax
+; AVX1-X86-NEXT:    .cfi_def_cfa_offset 8
+; AVX1-X86-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX1-X86-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX1-X86-NEXT:    vorpd %xmm0, %xmm1, %xmm1
+; AVX1-X86-NEXT:    vsubsd %xmm0, %xmm1, %xmm0
+; AVX1-X86-NEXT:    vcvtsd2ss %xmm0, %xmm0, %xmm0
+; AVX1-X86-NEXT:    vmovss %xmm0, (%esp)
+; AVX1-X86-NEXT:    flds (%esp)
+; AVX1-X86-NEXT:    popl %eax
+; AVX1-X86-NEXT:    .cfi_def_cfa_offset 4
+; AVX1-X86-NEXT:    retl
+;
+; AVX1-X64-LABEL: uitofp_i32tof32:
+; AVX1-X64:       # %bb.0:
+; AVX1-X64-NEXT:    movl %edi, %eax
+; AVX1-X64-NEXT:    vcvtsi2ss %rax, %xmm0, %xmm0
+; AVX1-X64-NEXT:    retq
+;
+; AVX512-X86-LABEL: uitofp_i32tof32:
+; AVX512-X86:       # %bb.0:
+; AVX512-X86-NEXT:    pushl %eax
+; AVX512-X86-NEXT:    .cfi_def_cfa_offset 8
+; AVX512-X86-NEXT:    vcvtusi2ssl {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512-X86-NEXT:    vmovss %xmm0, (%esp)
+; AVX512-X86-NEXT:    flds (%esp)
+; AVX512-X86-NEXT:    popl %eax
+; AVX512-X86-NEXT:    .cfi_def_cfa_offset 4
+; AVX512-X86-NEXT:    retl
+;
+; AVX512-X64-LABEL: uitofp_i32tof32:
+; AVX512-X64:       # %bb.0:
+; AVX512-X64-NEXT:    vcvtusi2ss %edi, %xmm0, %xmm0
+; AVX512-X64-NEXT:    retq
+;
+; X87-LABEL: uitofp_i32tof32:
+; X87:       # %bb.0:
+; X87-NEXT:    pushl %ebp
+; X87-NEXT:    .cfi_def_cfa_offset 8
+; X87-NEXT:    .cfi_offset %ebp, -8
+; X87-NEXT:    movl %esp, %ebp
+; X87-NEXT:    .cfi_def_cfa_register %ebp
+; X87-NEXT:    andl $-8, %esp
+; X87-NEXT:    subl $8, %esp
+; X87-NEXT:    movl 8(%ebp), %eax
+; X87-NEXT:    movl %eax, (%esp)
+; X87-NEXT:    movl $0, {{[0-9]+}}(%esp)
+; X87-NEXT:    fildll (%esp)
+; X87-NEXT:    movl %ebp, %esp
+; X87-NEXT:    popl %ebp
+; X87-NEXT:    .cfi_def_cfa %esp, 4
+; X87-NEXT:    retl
+  %result = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret float %result
+}
+
+define float @uitofp_i64tof32(i64 %x) #0 {
+; SSE-X86-LABEL: uitofp_i64tof32:
+; SSE-X86:       # %bb.0:
+; SSE-X86-NEXT:    pushl %ebp
+; SSE-X86-NEXT:    .cfi_def_cfa_offset 8
+; SSE-X86-NEXT:    .cfi_offset %ebp, -8
+; SSE-X86-NEXT:    movl %esp, %ebp
+; SSE-X86-NEXT:    .cfi_def_cfa_register %ebp
+; SSE-X86-NEXT:    andl $-8, %esp
+; SSE-X86-NEXT:    subl $16, %esp
+; SSE-X86-NEXT:    movl 12(%ebp), %eax
+; SSE-X86-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE-X86-NEXT:    movlps %xmm0, {{[0-9]+}}(%esp)
+; SSE-X86-NEXT:    shrl $31, %eax
+; SSE-X86-NEXT:    fildll {{[0-9]+}}(%esp)
+; SSE-X86-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; SSE-X86-NEXT:    fstps {{[0-9]+}}(%esp)
+; SSE-X86-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE-X86-NEXT:    movss %xmm0, (%esp)
+; SSE-X86-NEXT:    flds (%esp)
+; SSE-X86-NEXT:    movl %ebp, %esp
+; SSE-X86-NEXT:    popl %ebp
+; SSE-X86-NEXT:    .cfi_def_cfa %esp, 4
+; SSE-X86-NEXT:    retl
+;
+; SSE-X64-LABEL: uitofp_i64tof32:
+; SSE-X64:       # %bb.0:
+; SSE-X64-NEXT:    movq %rdi, %rax
+; SSE-X64-NEXT:    shrq %rax
+; SSE-X64-NEXT:    movl %edi, %ecx
+; SSE-X64-NEXT:    andl $1, %ecx
+; SSE-X64-NEXT:    orq %rax, %rcx
+; SSE-X64-NEXT:    testq %rdi, %rdi
+; SSE-X64-NEXT:    cmovnsq %rdi, %rcx
+; SSE-X64-NEXT:    cvtsi2ss %rcx, %xmm0
+; SSE-X64-NEXT:    jns .LBB9_2
+; SSE-X64-NEXT:  # %bb.1:
+; SSE-X64-NEXT:    addss %xmm0, %xmm0
+; SSE-X64-NEXT:  .LBB9_2:
+; SSE-X64-NEXT:    retq
+;
+; AVX-X86-LABEL: uitofp_i64tof32:
+; AVX-X86:       # %bb.0:
+; AVX-X86-NEXT:    pushl %ebp
+; AVX-X86-NEXT:    .cfi_def_cfa_offset 8
+; AVX-X86-NEXT:    .cfi_offset %ebp, -8
+; AVX-X86-NEXT:    movl %esp, %ebp
+; AVX-X86-NEXT:    .cfi_def_cfa_register %ebp
+; AVX-X86-NEXT:    andl $-8, %esp
+; AVX-X86-NEXT:    subl $16, %esp
+; AVX-X86-NEXT:    movl 12(%ebp), %eax
+; AVX-X86-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-X86-NEXT:    vmovlps %xmm0, {{[0-9]+}}(%esp)
+; AVX-X86-NEXT:    shrl $31, %eax
+; AVX-X86-NEXT:    fildll {{[0-9]+}}(%esp)
+; AVX-X86-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; AVX-X86-NEXT:    fstps {{[0-9]+}}(%esp)
+; AVX-X86-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-X86-NEXT:    vmovss %xmm0, (%esp)
+; AVX-X86-NEXT:    flds (%esp)
+; AVX-X86-NEXT:    movl %ebp, %esp
+; AVX-X86-NEXT:    popl %ebp
+; AVX-X86-NEXT:    .cfi_def_cfa %esp, 4
+; AVX-X86-NEXT:    retl
+;
+; AVX1-X64-LABEL: uitofp_i64tof32:
+; AVX1-X64:       # %bb.0:
+; AVX1-X64-NEXT:    movq %rdi, %rax
+; AVX1-X64-NEXT:    shrq %rax
+; AVX1-X64-NEXT:    movl %edi, %ecx
+; AVX1-X64-NEXT:    andl $1, %ecx
+; AVX1-X64-NEXT:    orq %rax, %rcx
+; AVX1-X64-NEXT:    testq %rdi, %rdi
+; AVX1-X64-NEXT:    cmovnsq %rdi, %rcx
+; AVX1-X64-NEXT:    vcvtsi2ss %rcx, %xmm0, %xmm0
+; AVX1-X64-NEXT:    jns .LBB9_2
+; AVX1-X64-NEXT:  # %bb.1:
+; AVX1-X64-NEXT:    vaddss %xmm0, %xmm0, %xmm0
+; AVX1-X64-NEXT:  .LBB9_2:
+; AVX1-X64-NEXT:    retq
+;
+; AVX512-X64-LABEL: uitofp_i64tof32:
+; AVX512-X64:       # %bb.0:
+; AVX512-X64-NEXT:    vcvtusi2ss %rdi, %xmm0, %xmm0
+; AVX512-X64-NEXT:    retq
+;
+; X87-LABEL: uitofp_i64tof32:
+; X87:       # %bb.0:
+; X87-NEXT:    pushl %ebp
+; X87-NEXT:    .cfi_def_cfa_offset 8
+; X87-NEXT:    .cfi_offset %ebp, -8
+; X87-NEXT:    movl %esp, %ebp
+; X87-NEXT:    .cfi_def_cfa_register %ebp
+; X87-NEXT:    andl $-8, %esp
+; X87-NEXT:    subl $16, %esp
+; X87-NEXT:    movl 8(%ebp), %eax
+; X87-NEXT:    movl 12(%ebp), %ecx
+; X87-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X87-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X87-NEXT:    shrl $31, %ecx
+; X87-NEXT:    fildll {{[0-9]+}}(%esp)
+; X87-NEXT:    fadds {{\.LCPI.*}}(,%ecx,4)
+; X87-NEXT:    fstps {{[0-9]+}}(%esp)
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    movl %ebp, %esp
+; X87-NEXT:    popl %ebp
+; X87-NEXT:    .cfi_def_cfa %esp, 4
+; X87-NEXT:    retl
+  %result = call float @llvm.experimental.constrained.uitofp.f32.i64(i64 %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret float %result
+}
+
+define double @sitofp_i8tof64(i8 %x) #0 {
+; SSE-X86-LABEL: sitofp_i8tof64:
+; SSE-X86:       # %bb.0:
+; SSE-X86-NEXT:    pushl %ebp
+; SSE-X86-NEXT:    .cfi_def_cfa_offset 8
+; SSE-X86-NEXT:    .cfi_offset %ebp, -8
+; SSE-X86-NEXT:    movl %esp, %ebp
+; SSE-X86-NEXT:    .cfi_def_cfa_register %ebp
+; SSE-X86-NEXT:    andl $-8, %esp
+; SSE-X86-NEXT:    subl $8, %esp
+; SSE-X86-NEXT:    movsbl 8(%ebp), %eax
+; SSE-X86-NEXT:    cvtsi2sd %eax, %xmm0
+; SSE-X86-NEXT:    movsd %xmm0, (%esp)
+; SSE-X86-NEXT:    fldl (%esp)
+; SSE-X86-NEXT:    movl %ebp, %esp
+; SSE-X86-NEXT:    popl %ebp
+; SSE-X86-NEXT:    .cfi_def_cfa %esp, 4
+; SSE-X86-NEXT:    retl
+;
+; SSE-X64-LABEL: sitofp_i8tof64:
+; SSE-X64:       # %bb.0:
+; SSE-X64-NEXT:    movsbl %dil, %eax
+; SSE-X64-NEXT:    cvtsi2sd %eax, %xmm0
+; SSE-X64-NEXT:    retq
+;
+; AVX-X86-LABEL: sitofp_i8tof64:
+; AVX-X86:       # %bb.0:
+; AVX-X86-NEXT:    pushl %ebp
+; AVX-X86-NEXT:    .cfi_def_cfa_offset 8
+; AVX-X86-NEXT:    .cfi_offset %ebp, -8
+; AVX-X86-NEXT:    movl %esp, %ebp
+; AVX-X86-NEXT:    .cfi_def_cfa_register %ebp
+; AVX-X86-NEXT:    andl $-8, %esp
+; AVX-X86-NEXT:    subl $8, %esp
+; AVX-X86-NEXT:    movsbl 8(%ebp), %eax
+; AVX-X86-NEXT:    vcvtsi2sd %eax, %xmm0, %xmm0
+; AVX-X86-NEXT:    vmovsd %xmm0, (%esp)
+; AVX-X86-NEXT:    fldl (%esp)
+; AVX-X86-NEXT:    movl %ebp, %esp
+; AVX-X86-NEXT:    popl %ebp
+; AVX-X86-NEXT:    .cfi_def_cfa %esp, 4
+; AVX-X86-NEXT:    retl
+;
+; AVX-X64-LABEL: sitofp_i8tof64:
+; AVX-X64:       # %bb.0:
+; AVX-X64-NEXT:    movsbl %dil, %eax
+; AVX-X64-NEXT:    vcvtsi2sd %eax, %xmm0, %xmm0
+; AVX-X64-NEXT:    retq
+;
+; X87-LABEL: sitofp_i8tof64:
+; X87:       # %bb.0:
+; X87-NEXT:    pushl %eax
+; X87-NEXT:    .cfi_def_cfa_offset 8
+; X87-NEXT:    movsbl {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; X87-NEXT:    filds {{[0-9]+}}(%esp)
+; X87-NEXT:    popl %eax
+; X87-NEXT:    .cfi_def_cfa_offset 4
+; X87-NEXT:    retl
+  %result = call double @llvm.experimental.constrained.sitofp.f64.i8(i8 %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret double %result
+}
+
+define double @sitofp_i16tof64(i16 %x) #0 {
+; SSE-X86-LABEL: sitofp_i16tof64:
+; SSE-X86:       # %bb.0:
+; SSE-X86-NEXT:    pushl %ebp
+; SSE-X86-NEXT:    .cfi_def_cfa_offset 8
+; SSE-X86-NEXT:    .cfi_offset %ebp, -8
+; SSE-X86-NEXT:    movl %esp, %ebp
+; SSE-X86-NEXT:    .cfi_def_cfa_register %ebp
+; SSE-X86-NEXT:    andl $-8, %esp
+; SSE-X86-NEXT:    subl $8, %esp
+; SSE-X86-NEXT:    movswl 8(%ebp), %eax
+; SSE-X86-NEXT:    cvtsi2sd %eax, %xmm0
+; SSE-X86-NEXT:    movsd %xmm0, (%esp)
+; SSE-X86-NEXT:    fldl (%esp)
+; SSE-X86-NEXT:    movl %ebp, %esp
+; SSE-X86-NEXT:    popl %ebp
+; SSE-X86-NEXT:    .cfi_def_cfa %esp, 4
+; SSE-X86-NEXT:    retl
+;
+; SSE-X64-LABEL: sitofp_i16tof64:
+; SSE-X64:       # %bb.0:
+; SSE-X64-NEXT:    movswl %di, %eax
+; SSE-X64-NEXT:    cvtsi2sd %eax, %xmm0
+; SSE-X64-NEXT:    retq
+;
+; AVX-X86-LABEL: sitofp_i16tof64:
+; AVX-X86:       # %bb.0:
+; AVX-X86-NEXT:    pushl %ebp
+; AVX-X86-NEXT:    .cfi_def_cfa_offset 8
+; AVX-X86-NEXT:    .cfi_offset %ebp, -8
+; AVX-X86-NEXT:    movl %esp, %ebp
+; AVX-X86-NEXT:    .cfi_def_cfa_register %ebp
+; AVX-X86-NEXT:    andl $-8, %esp
+; AVX-X86-NEXT:    subl $8, %esp
+; AVX-X86-NEXT:    movswl 8(%ebp), %eax
+; AVX-X86-NEXT:    vcvtsi2sd %eax, %xmm0, %xmm0
+; AVX-X86-NEXT:    vmovsd %xmm0, (%esp)
+; AVX-X86-NEXT:    fldl (%esp)
+; AVX-X86-NEXT:    movl %ebp, %esp
+; AVX-X86-NEXT:    popl %ebp
+; AVX-X86-NEXT:    .cfi_def_cfa %esp, 4
+; AVX-X86-NEXT:    retl
+;
+; AVX-X64-LABEL: sitofp_i16tof64:
+; AVX-X64:       # %bb.0:
+; AVX-X64-NEXT:    movswl %di, %eax
+; AVX-X64-NEXT:    vcvtsi2sd %eax, %xmm0, %xmm0
+; AVX-X64-NEXT:    retq
+;
+; X87-LABEL: sitofp_i16tof64:
+; X87:       # %bb.0:
+; X87-NEXT:    pushl %eax
+; X87-NEXT:    .cfi_def_cfa_offset 8
+; X87-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; X87-NEXT:    filds {{[0-9]+}}(%esp)
+; X87-NEXT:    popl %eax
+; X87-NEXT:    .cfi_def_cfa_offset 4
+; X87-NEXT:    retl
+  %result = call double @llvm.experimental.constrained.sitofp.f64.i16(i16 %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret double %result
+}
+
+define double @sitofp_i32tof64(i32 %x) #0 {
+; SSE-X86-LABEL: sitofp_i32tof64:
+; SSE-X86:       # %bb.0:
+; SSE-X86-NEXT:    pushl %ebp
+; SSE-X86-NEXT:    .cfi_def_cfa_offset 8
+; SSE-X86-NEXT:    .cfi_offset %ebp, -8
+; SSE-X86-NEXT:    movl %esp, %ebp
+; SSE-X86-NEXT:    .cfi_def_cfa_register %ebp
+; SSE-X86-NEXT:    andl $-8, %esp
+; SSE-X86-NEXT:    subl $8, %esp
+; SSE-X86-NEXT:    cvtsi2sdl 8(%ebp), %xmm0
+; SSE-X86-NEXT:    movsd %xmm0, (%esp)
+; SSE-X86-NEXT:    fldl (%esp)
+; SSE-X86-NEXT:    movl %ebp, %esp
+; SSE-X86-NEXT:    popl %ebp
+; SSE-X86-NEXT:    .cfi_def_cfa %esp, 4
+; SSE-X86-NEXT:    retl
+;
+; SSE-X64-LABEL: sitofp_i32tof64:
+; SSE-X64:       # %bb.0:
+; SSE-X64-NEXT:    cvtsi2sd %edi, %xmm0
+; SSE-X64-NEXT:    retq
+;
+; AVX-X86-LABEL: sitofp_i32tof64:
+; AVX-X86:       # %bb.0:
+; AVX-X86-NEXT:    pushl %ebp
+; AVX-X86-NEXT:    .cfi_def_cfa_offset 8
+; AVX-X86-NEXT:    .cfi_offset %ebp, -8
+; AVX-X86-NEXT:    movl %esp, %ebp
+; AVX-X86-NEXT:    .cfi_def_cfa_register %ebp
+; AVX-X86-NEXT:    andl $-8, %esp
+; AVX-X86-NEXT:    subl $8, %esp
+; AVX-X86-NEXT:    vcvtsi2sdl 8(%ebp), %xmm0, %xmm0
+; AVX-X86-NEXT:    vmovsd %xmm0, (%esp)
+; AVX-X86-NEXT:    fldl (%esp)
+; AVX-X86-NEXT:    movl %ebp, %esp
+; AVX-X86-NEXT:    popl %ebp
+; AVX-X86-NEXT:    .cfi_def_cfa %esp, 4
+; AVX-X86-NEXT:    retl
+;
+; AVX-X64-LABEL: sitofp_i32tof64:
+; AVX-X64:       # %bb.0:
+; AVX-X64-NEXT:    vcvtsi2sd %edi, %xmm0, %xmm0
+; AVX-X64-NEXT:    retq
+;
+; X87-LABEL: sitofp_i32tof64:
+; X87:       # %bb.0:
+; X87-NEXT:    pushl %eax
+; X87-NEXT:    .cfi_def_cfa_offset 8
+; X87-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl %eax, (%esp)
+; X87-NEXT:    fildl (%esp)
+; X87-NEXT:    popl %eax
+; X87-NEXT:    .cfi_def_cfa_offset 4
+; X87-NEXT:    retl
+  %result = call double @llvm.experimental.constrained.sitofp.f64.i32(i32 %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret double %result
+}
+
+define double @sitofp_i64tof64(i64 %x) #0 {
+; SSE-X86-LABEL: sitofp_i64tof64:
+; SSE-X86:       # %bb.0:
+; SSE-X86-NEXT:    pushl %ebp
+; SSE-X86-NEXT:    .cfi_def_cfa_offset 8
+; SSE-X86-NEXT:    .cfi_offset %ebp, -8
+; SSE-X86-NEXT:    movl %esp, %ebp
+; SSE-X86-NEXT:    .cfi_def_cfa_register %ebp
+; SSE-X86-NEXT:    andl $-8, %esp
+; SSE-X86-NEXT:    subl $8, %esp
+; SSE-X86-NEXT:    fildll 8(%ebp)
+; SSE-X86-NEXT:    fstpl (%esp)
+; SSE-X86-NEXT:    fldl (%esp)
+; SSE-X86-NEXT:    movl %ebp, %esp
+; SSE-X86-NEXT:    popl %ebp
+; SSE-X86-NEXT:    .cfi_def_cfa %esp, 4
+; SSE-X86-NEXT:    retl
+;
+; SSE-X64-LABEL: sitofp_i64tof64:
+; SSE-X64:       # %bb.0:
+; SSE-X64-NEXT:    cvtsi2sd %rdi, %xmm0
+; SSE-X64-NEXT:    retq
+;
+; AVX-X86-LABEL: sitofp_i64tof64:
+; AVX-X86:       # %bb.0:
+; AVX-X86-NEXT:    pushl %ebp
+; AVX-X86-NEXT:    .cfi_def_cfa_offset 8
+; AVX-X86-NEXT:    .cfi_offset %ebp, -8
+; AVX-X86-NEXT:    movl %esp, %ebp
+; AVX-X86-NEXT:    .cfi_def_cfa_register %ebp
+; AVX-X86-NEXT:    andl $-8, %esp
+; AVX-X86-NEXT:    subl $8, %esp
+; AVX-X86-NEXT:    fildll 8(%ebp)
+; AVX-X86-NEXT:    fstpl (%esp)
+; AVX-X86-NEXT:    fldl (%esp)
+; AVX-X86-NEXT:    movl %ebp, %esp
+; AVX-X86-NEXT:    popl %ebp
+; AVX-X86-NEXT:    .cfi_def_cfa %esp, 4
+; AVX-X86-NEXT:    retl
+;
+; AVX-X64-LABEL: sitofp_i64tof64:
+; AVX-X64:       # %bb.0:
+; AVX-X64-NEXT:    vcvtsi2sd %rdi, %xmm0, %xmm0
+; AVX-X64-NEXT:    retq
+;
+; X87-LABEL: sitofp_i64tof64:
+; X87:       # %bb.0:
+; X87-NEXT:    fildll {{[0-9]+}}(%esp)
+; X87-NEXT:    retl
+  %result = call double @llvm.experimental.constrained.sitofp.f64.i64(i64 %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret double %result
+}
+
+define double @uitofp_i1tof64(i1 %x) #0 {
+; SSE-X86-LABEL: uitofp_i1tof64:
+; SSE-X86:       # %bb.0:
+; SSE-X86-NEXT:    pushl %ebp
+; SSE-X86-NEXT:    .cfi_def_cfa_offset 8
+; SSE-X86-NEXT:    .cfi_offset %ebp, -8
+; SSE-X86-NEXT:    movl %esp, %ebp
+; SSE-X86-NEXT:    .cfi_def_cfa_register %ebp
+; SSE-X86-NEXT:    andl $-8, %esp
+; SSE-X86-NEXT:    subl $8, %esp
+; SSE-X86-NEXT:    movb 8(%ebp), %al
+; SSE-X86-NEXT:    andb $1, %al
+; SSE-X86-NEXT:    movzbl %al, %eax
+; SSE-X86-NEXT:    cvtsi2sd %eax, %xmm0
+; SSE-X86-NEXT:    movsd %xmm0, (%esp)
+; SSE-X86-NEXT:    fldl (%esp)
+; SSE-X86-NEXT:    movl %ebp, %esp
+; SSE-X86-NEXT:    popl %ebp
+; SSE-X86-NEXT:    .cfi_def_cfa %esp, 4
+; SSE-X86-NEXT:    retl
+;
+; SSE-X64-LABEL: uitofp_i1tof64:
+; SSE-X64:       # %bb.0:
+; SSE-X64-NEXT:    andl $1, %edi
+; SSE-X64-NEXT:    cvtsi2sd %edi, %xmm0
+; SSE-X64-NEXT:    retq
+;
+; AVX-X86-LABEL: uitofp_i1tof64:
+; AVX-X86:       # %bb.0:
+; AVX-X86-NEXT:    pushl %ebp
+; AVX-X86-NEXT:    .cfi_def_cfa_offset 8
+; AVX-X86-NEXT:    .cfi_offset %ebp, -8
+; AVX-X86-NEXT:    movl %esp, %ebp
+; AVX-X86-NEXT:    .cfi_def_cfa_register %ebp
+; AVX-X86-NEXT:    andl $-8, %esp
+; AVX-X86-NEXT:    subl $8, %esp
+; AVX-X86-NEXT:    movb 8(%ebp), %al
+; AVX-X86-NEXT:    andb $1, %al
+; AVX-X86-NEXT:    movzbl %al, %eax
+; AVX-X86-NEXT:    vcvtsi2sd %eax, %xmm0, %xmm0
+; AVX-X86-NEXT:    vmovsd %xmm0, (%esp)
+; AVX-X86-NEXT:    fldl (%esp)
+; AVX-X86-NEXT:    movl %ebp, %esp
+; AVX-X86-NEXT:    popl %ebp
+; AVX-X86-NEXT:    .cfi_def_cfa %esp, 4
+; AVX-X86-NEXT:    retl
+;
+; AVX-X64-LABEL: uitofp_i1tof64:
+; AVX-X64:       # %bb.0:
+; AVX-X64-NEXT:    andl $1, %edi
+; AVX-X64-NEXT:    vcvtsi2sd %edi, %xmm0, %xmm0
+; AVX-X64-NEXT:    retq
+;
+; X87-LABEL: uitofp_i1tof64:
+; X87:       # %bb.0:
+; X87-NEXT:    pushl %eax
+; X87-NEXT:    .cfi_def_cfa_offset 8
+; X87-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X87-NEXT:    andb $1, %al
+; X87-NEXT:    movzbl %al, %eax
+; X87-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; X87-NEXT:    filds {{[0-9]+}}(%esp)
+; X87-NEXT:    popl %eax
+; X87-NEXT:    .cfi_def_cfa_offset 4
+; X87-NEXT:    retl
+  %result = call double @llvm.experimental.constrained.uitofp.f64.i1(i1 %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret double %result
+}
+
+define double @uitofp_i8tof64(i8 %x) #0 {
+; SSE-X86-LABEL: uitofp_i8tof64:
+; SSE-X86:       # %bb.0:
+; SSE-X86-NEXT:    pushl %ebp
+; SSE-X86-NEXT:    .cfi_def_cfa_offset 8
+; SSE-X86-NEXT:    .cfi_offset %ebp, -8
+; SSE-X86-NEXT:    movl %esp, %ebp
+; SSE-X86-NEXT:    .cfi_def_cfa_register %ebp
+; SSE-X86-NEXT:    andl $-8, %esp
+; SSE-X86-NEXT:    subl $8, %esp
+; SSE-X86-NEXT:    movzbl 8(%ebp), %eax
+; SSE-X86-NEXT:    cvtsi2sd %eax, %xmm0
+; SSE-X86-NEXT:    movsd %xmm0, (%esp)
+; SSE-X86-NEXT:    fldl (%esp)
+; SSE-X86-NEXT:    movl %ebp, %esp
+; SSE-X86-NEXT:    popl %ebp
+; SSE-X86-NEXT:    .cfi_def_cfa %esp, 4
+; SSE-X86-NEXT:    retl
+;
+; SSE-X64-LABEL: uitofp_i8tof64:
+; SSE-X64:       # %bb.0:
+; SSE-X64-NEXT:    movzbl %dil, %eax
+; SSE-X64-NEXT:    cvtsi2sd %eax, %xmm0
+; SSE-X64-NEXT:    retq
+;
+; AVX-X86-LABEL: uitofp_i8tof64:
+; AVX-X86:       # %bb.0:
+; AVX-X86-NEXT:    pushl %ebp
+; AVX-X86-NEXT:    .cfi_def_cfa_offset 8
+; AVX-X86-NEXT:    .cfi_offset %ebp, -8
+; AVX-X86-NEXT:    movl %esp, %ebp
+; AVX-X86-NEXT:    .cfi_def_cfa_register %ebp
+; AVX-X86-NEXT:    andl $-8, %esp
+; AVX-X86-NEXT:    subl $8, %esp
+; AVX-X86-NEXT:    movzbl 8(%ebp), %eax
+; AVX-X86-NEXT:    vcvtsi2sd %eax, %xmm0, %xmm0
+; AVX-X86-NEXT:    vmovsd %xmm0, (%esp)
+; AVX-X86-NEXT:    fldl (%esp)
+; AVX-X86-NEXT:    movl %ebp, %esp
+; AVX-X86-NEXT:    popl %ebp
+; AVX-X86-NEXT:    .cfi_def_cfa %esp, 4
+; AVX-X86-NEXT:    retl
+;
+; AVX-X64-LABEL: uitofp_i8tof64:
+; AVX-X64:       # %bb.0:
+; AVX-X64-NEXT:    movzbl %dil, %eax
+; AVX-X64-NEXT:    vcvtsi2sd %eax, %xmm0, %xmm0
+; AVX-X64-NEXT:    retq
+;
+; X87-LABEL: uitofp_i8tof64:
+; X87:       # %bb.0:
+; X87-NEXT:    pushl %eax
+; X87-NEXT:    .cfi_def_cfa_offset 8
+; X87-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; X87-NEXT:    filds {{[0-9]+}}(%esp)
+; X87-NEXT:    popl %eax
+; X87-NEXT:    .cfi_def_cfa_offset 4
+; X87-NEXT:    retl
+  %result = call double @llvm.experimental.constrained.uitofp.f64.i8(i8 %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret double %result
+}
+
+define double @uitofp_i16tof64(i16 %x) #0 {
+; SSE-X86-LABEL: uitofp_i16tof64:
+; SSE-X86:       # %bb.0:
+; SSE-X86-NEXT:    pushl %ebp
+; SSE-X86-NEXT:    .cfi_def_cfa_offset 8
+; SSE-X86-NEXT:    .cfi_offset %ebp, -8
+; SSE-X86-NEXT:    movl %esp, %ebp
+; SSE-X86-NEXT:    .cfi_def_cfa_register %ebp
+; SSE-X86-NEXT:    andl $-8, %esp
+; SSE-X86-NEXT:    subl $8, %esp
+; SSE-X86-NEXT:    movzwl 8(%ebp), %eax
+; SSE-X86-NEXT:    cvtsi2sd %eax, %xmm0
+; SSE-X86-NEXT:    movsd %xmm0, (%esp)
+; SSE-X86-NEXT:    fldl (%esp)
+; SSE-X86-NEXT:    movl %ebp, %esp
+; SSE-X86-NEXT:    popl %ebp
+; SSE-X86-NEXT:    .cfi_def_cfa %esp, 4
+; SSE-X86-NEXT:    retl
+;
+; SSE-X64-LABEL: uitofp_i16tof64:
+; SSE-X64:       # %bb.0:
+; SSE-X64-NEXT:    movzwl %di, %eax
+; SSE-X64-NEXT:    cvtsi2sd %eax, %xmm0
+; SSE-X64-NEXT:    retq
+;
+; AVX-X86-LABEL: uitofp_i16tof64:
+; AVX-X86:       # %bb.0:
+; AVX-X86-NEXT:    pushl %ebp
+; AVX-X86-NEXT:    .cfi_def_cfa_offset 8
+; AVX-X86-NEXT:    .cfi_offset %ebp, -8
+; AVX-X86-NEXT:    movl %esp, %ebp
+; AVX-X86-NEXT:    .cfi_def_cfa_register %ebp
+; AVX-X86-NEXT:    andl $-8, %esp
+; AVX-X86-NEXT:    subl $8, %esp
+; AVX-X86-NEXT:    movzwl 8(%ebp), %eax
+; AVX-X86-NEXT:    vcvtsi2sd %eax, %xmm0, %xmm0
+; AVX-X86-NEXT:    vmovsd %xmm0, (%esp)
+; AVX-X86-NEXT:    fldl (%esp)
+; AVX-X86-NEXT:    movl %ebp, %esp
+; AVX-X86-NEXT:    popl %ebp
+; AVX-X86-NEXT:    .cfi_def_cfa %esp, 4
+; AVX-X86-NEXT:    retl
+;
+; AVX-X64-LABEL: uitofp_i16tof64:
+; AVX-X64:       # %bb.0:
+; AVX-X64-NEXT:    movzwl %di, %eax
+; AVX-X64-NEXT:    vcvtsi2sd %eax, %xmm0, %xmm0
+; AVX-X64-NEXT:    retq
+;
+; X87-LABEL: uitofp_i16tof64:
+; X87:       # %bb.0:
+; X87-NEXT:    pushl %eax
+; X87-NEXT:    .cfi_def_cfa_offset 8
+; X87-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl %eax, (%esp)
+; X87-NEXT:    fildl (%esp)
+; X87-NEXT:    popl %eax
+; X87-NEXT:    .cfi_def_cfa_offset 4
+; X87-NEXT:    retl
+  %result = call double @llvm.experimental.constrained.uitofp.f64.i16(i16 %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret double %result
+}
+
+define double @uitofp_i32tof64(i32 %x) #0 {
+; SSE-X86-LABEL: uitofp_i32tof64:
+; SSE-X86:       # %bb.0:
+; SSE-X86-NEXT:    pushl %ebp
+; SSE-X86-NEXT:    .cfi_def_cfa_offset 8
+; SSE-X86-NEXT:    .cfi_offset %ebp, -8
+; SSE-X86-NEXT:    movl %esp, %ebp
+; SSE-X86-NEXT:    .cfi_def_cfa_register %ebp
+; SSE-X86-NEXT:    andl $-8, %esp
+; SSE-X86-NEXT:    subl $8, %esp
+; SSE-X86-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE-X86-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE-X86-NEXT:    orpd %xmm0, %xmm1
+; SSE-X86-NEXT:    subsd %xmm0, %xmm1
+; SSE-X86-NEXT:    movsd %xmm1, (%esp)
+; SSE-X86-NEXT:    fldl (%esp)
+; SSE-X86-NEXT:    movl %ebp, %esp
+; SSE-X86-NEXT:    popl %ebp
+; SSE-X86-NEXT:    .cfi_def_cfa %esp, 4
+; SSE-X86-NEXT:    retl
+;
+; SSE-X64-LABEL: uitofp_i32tof64:
+; SSE-X64:       # %bb.0:
+; SSE-X64-NEXT:    movl %edi, %eax
+; SSE-X64-NEXT:    cvtsi2sd %rax, %xmm0
+; SSE-X64-NEXT:    retq
+;
+; AVX1-X86-LABEL: uitofp_i32tof64:
+; AVX1-X86:       # %bb.0:
+; AVX1-X86-NEXT:    pushl %ebp
+; AVX1-X86-NEXT:    .cfi_def_cfa_offset 8
+; AVX1-X86-NEXT:    .cfi_offset %ebp, -8
+; AVX1-X86-NEXT:    movl %esp, %ebp
+; AVX1-X86-NEXT:    .cfi_def_cfa_register %ebp
+; AVX1-X86-NEXT:    andl $-8, %esp
+; AVX1-X86-NEXT:    subl $8, %esp
+; AVX1-X86-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX1-X86-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX1-X86-NEXT:    vorpd %xmm0, %xmm1, %xmm1
+; AVX1-X86-NEXT:    vsubsd %xmm0, %xmm1, %xmm0
+; AVX1-X86-NEXT:    vmovsd %xmm0, (%esp)
+; AVX1-X86-NEXT:    fldl (%esp)
+; AVX1-X86-NEXT:    movl %ebp, %esp
+; AVX1-X86-NEXT:    popl %ebp
+; AVX1-X86-NEXT:    .cfi_def_cfa %esp, 4
+; AVX1-X86-NEXT:    retl
+;
+; AVX1-X64-LABEL: uitofp_i32tof64:
+; AVX1-X64:       # %bb.0:
+; AVX1-X64-NEXT:    movl %edi, %eax
+; AVX1-X64-NEXT:    vcvtsi2sd %rax, %xmm0, %xmm0
+; AVX1-X64-NEXT:    retq
+;
+; AVX512-X86-LABEL: uitofp_i32tof64:
+; AVX512-X86:       # %bb.0:
+; AVX512-X86-NEXT:    pushl %ebp
+; AVX512-X86-NEXT:    .cfi_def_cfa_offset 8
+; AVX512-X86-NEXT:    .cfi_offset %ebp, -8
+; AVX512-X86-NEXT:    movl %esp, %ebp
+; AVX512-X86-NEXT:    .cfi_def_cfa_register %ebp
+; AVX512-X86-NEXT:    andl $-8, %esp
+; AVX512-X86-NEXT:    subl $8, %esp
+; AVX512-X86-NEXT:    vcvtusi2sdl 8(%ebp), %xmm0, %xmm0
+; AVX512-X86-NEXT:    vmovsd %xmm0, (%esp)
+; AVX512-X86-NEXT:    fldl (%esp)
+; AVX512-X86-NEXT:    movl %ebp, %esp
+; AVX512-X86-NEXT:    popl %ebp
+; AVX512-X86-NEXT:    .cfi_def_cfa %esp, 4
+; AVX512-X86-NEXT:    retl
+;
+; AVX512-X64-LABEL: uitofp_i32tof64:
+; AVX512-X64:       # %bb.0:
+; AVX512-X64-NEXT:    vcvtusi2sd %edi, %xmm0, %xmm0
+; AVX512-X64-NEXT:    retq
+;
+; X87-LABEL: uitofp_i32tof64:
+; X87:       # %bb.0:
+; X87-NEXT:    pushl %ebp
+; X87-NEXT:    .cfi_def_cfa_offset 8
+; X87-NEXT:    .cfi_offset %ebp, -8
+; X87-NEXT:    movl %esp, %ebp
+; X87-NEXT:    .cfi_def_cfa_register %ebp
+; X87-NEXT:    andl $-8, %esp
+; X87-NEXT:    subl $8, %esp
+; X87-NEXT:    movl 8(%ebp), %eax
+; X87-NEXT:    movl %eax, (%esp)
+; X87-NEXT:    movl $0, {{[0-9]+}}(%esp)
+; X87-NEXT:    fildll (%esp)
+; X87-NEXT:    movl %ebp, %esp
+; X87-NEXT:    popl %ebp
+; X87-NEXT:    .cfi_def_cfa %esp, 4
+; X87-NEXT:    retl
+  %result = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret double %result
+}
+
+define double @uitofp_i64tof64(i64 %x) #0 {
+; SSE-X86-LABEL: uitofp_i64tof64:
+; SSE-X86:       # %bb.0:
+; SSE-X86-NEXT:    pushl %ebp
+; SSE-X86-NEXT:    .cfi_def_cfa_offset 8
+; SSE-X86-NEXT:    .cfi_offset %ebp, -8
+; SSE-X86-NEXT:    movl %esp, %ebp
+; SSE-X86-NEXT:    .cfi_def_cfa_register %ebp
+; SSE-X86-NEXT:    andl $-8, %esp
+; SSE-X86-NEXT:    subl $8, %esp
+; SSE-X86-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE-X86-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
+; SSE-X86-NEXT:    subpd {{\.LCPI.*}}, %xmm0
+; SSE-X86-NEXT:    movapd %xmm0, %xmm1
+; SSE-X86-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
+; SSE-X86-NEXT:    addpd %xmm0, %xmm1
+; SSE-X86-NEXT:    movlpd %xmm1, (%esp)
+; SSE-X86-NEXT:    fldl (%esp)
+; SSE-X86-NEXT:    movl %ebp, %esp
+; SSE-X86-NEXT:    popl %ebp
+; SSE-X86-NEXT:    .cfi_def_cfa %esp, 4
+; SSE-X86-NEXT:    retl
+;
+; SSE-X64-LABEL: uitofp_i64tof64:
+; SSE-X64:       # %bb.0:
+; SSE-X64-NEXT:    movq %rdi, %xmm1
+; SSE-X64-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]
+; SSE-X64-NEXT:    subpd {{.*}}(%rip), %xmm1
+; SSE-X64-NEXT:    movapd %xmm1, %xmm0
+; SSE-X64-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1]
+; SSE-X64-NEXT:    addpd %xmm1, %xmm0
+; SSE-X64-NEXT:    retq
+;
+; AVX-X86-LABEL: uitofp_i64tof64:
+; AVX-X86:       # %bb.0:
+; AVX-X86-NEXT:    pushl %ebp
+; AVX-X86-NEXT:    .cfi_def_cfa_offset 8
+; AVX-X86-NEXT:    .cfi_offset %ebp, -8
+; AVX-X86-NEXT:    movl %esp, %ebp
+; AVX-X86-NEXT:    .cfi_def_cfa_register %ebp
+; AVX-X86-NEXT:    andl $-8, %esp
+; AVX-X86-NEXT:    subl $8, %esp
+; AVX-X86-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-X86-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
+; AVX-X86-NEXT:    vsubpd {{\.LCPI.*}}, %xmm0, %xmm0
+; AVX-X86-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
+; AVX-X86-NEXT:    vaddpd %xmm0, %xmm1, %xmm0
+; AVX-X86-NEXT:    vmovlpd %xmm0, (%esp)
+; AVX-X86-NEXT:    fldl (%esp)
+; AVX-X86-NEXT:    movl %ebp, %esp
+; AVX-X86-NEXT:    popl %ebp
+; AVX-X86-NEXT:    .cfi_def_cfa %esp, 4
+; AVX-X86-NEXT:    retl
+;
+; AVX1-X64-LABEL: uitofp_i64tof64:
+; AVX1-X64:       # %bb.0:
+; AVX1-X64-NEXT:    vmovq %rdi, %xmm0
+; AVX1-X64-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
+; AVX1-X64-NEXT:    vsubpd {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-X64-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
+; AVX1-X64-NEXT:    vaddpd %xmm0, %xmm1, %xmm0
+; AVX1-X64-NEXT:    retq
+;
+; AVX512-X64-LABEL: uitofp_i64tof64:
+; AVX512-X64:       # %bb.0:
+; AVX512-X64-NEXT:    vcvtusi2sd %rdi, %xmm0, %xmm0
+; AVX512-X64-NEXT:    retq
+;
+; X87-LABEL: uitofp_i64tof64:
+; X87:       # %bb.0:
+; X87-NEXT:    pushl %ebp
+; X87-NEXT:    .cfi_def_cfa_offset 8
+; X87-NEXT:    .cfi_offset %ebp, -8
+; X87-NEXT:    movl %esp, %ebp
+; X87-NEXT:    .cfi_def_cfa_register %ebp
+; X87-NEXT:    andl $-8, %esp
+; X87-NEXT:    subl $16, %esp
+; X87-NEXT:    movl 8(%ebp), %eax
+; X87-NEXT:    movl 12(%ebp), %ecx
+; X87-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X87-NEXT:    movl %eax, (%esp)
+; X87-NEXT:    shrl $31, %ecx
+; X87-NEXT:    fildll (%esp)
+; X87-NEXT:    fadds {{\.LCPI.*}}(,%ecx,4)
+; X87-NEXT:    fstpl {{[0-9]+}}(%esp)
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    movl %ebp, %esp
+; X87-NEXT:    popl %ebp
+; X87-NEXT:    .cfi_def_cfa %esp, 4
+; X87-NEXT:    retl
+  %result = call double @llvm.experimental.constrained.uitofp.f64.i64(i64 %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret double %result
+}
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/X86/fp-strict-scalar-round.ll b/llvm/test/CodeGen/X86/fp-strict-scalar-round.ll
new file mode 100644
index 000000000..966be0212
--- /dev/null
+++ b/llvm/test/CodeGen/X86/fp-strict-scalar-round.ll
@@ -0,0 +1,474 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse4.1 -O3 | FileCheck %s --check-prefixes=SSE41,SSE41-X86
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 -O3 | FileCheck %s --check-prefixes=SSE41,SSE41-X64
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx -O3 | FileCheck %s --check-prefixes=AVX-X86
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx -O3 | FileCheck %s --check-prefixes=AVX-X64
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx512f -O3 | FileCheck %s --check-prefixes=AVX-X86,AVX512-X86
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f -O3 | FileCheck %s --check-prefixes=AVX-X64,AVX512-X64
+
+declare float @llvm.experimental.constrained.ceil.f32(float, metadata)
+declare double @llvm.experimental.constrained.ceil.f64(double, metadata)
+declare float @llvm.experimental.constrained.floor.f32(float, metadata)
+declare double @llvm.experimental.constrained.floor.f64(double, metadata)
+declare float @llvm.experimental.constrained.trunc.f32(float, metadata)
+declare double @llvm.experimental.constrained.trunc.f64(double, metadata)
+declare float @llvm.experimental.constrained.rint.f32(float, metadata, metadata)
+declare double @llvm.experimental.constrained.rint.f64(double, metadata, metadata)
+declare float @llvm.experimental.constrained.nearbyint.f32(float, metadata, metadata)
+declare double @llvm.experimental.constrained.nearbyint.f64(double, metadata, metadata)
+
+define float @fceil32(float %f) #0 {
+; SSE41-X86-LABEL: fceil32:
+; SSE41-X86:       # %bb.0:
+; SSE41-X86-NEXT:    pushl %eax
+; SSE41-X86-NEXT:    .cfi_def_cfa_offset 8
+; SSE41-X86-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE41-X86-NEXT:    roundss $10, %xmm0, %xmm0
+; SSE41-X86-NEXT:    movss %xmm0, (%esp)
+; SSE41-X86-NEXT:    flds (%esp)
+; SSE41-X86-NEXT:    popl %eax
+; SSE41-X86-NEXT:    .cfi_def_cfa_offset 4
+; SSE41-X86-NEXT:    retl
+;
+; SSE41-X64-LABEL: fceil32:
+; SSE41-X64:       # %bb.0:
+; SSE41-X64-NEXT:    roundss $10, %xmm0, %xmm0
+; SSE41-X64-NEXT:    retq
+;
+; AVX-X86-LABEL: fceil32:
+; AVX-X86:       # %bb.0:
+; AVX-X86-NEXT:    pushl %eax
+; AVX-X86-NEXT:    .cfi_def_cfa_offset 8
+; AVX-X86-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-X86-NEXT:    vroundss $10, %xmm0, %xmm0, %xmm0
+; AVX-X86-NEXT:    vmovss %xmm0, (%esp)
+; AVX-X86-NEXT:    flds (%esp)
+; AVX-X86-NEXT:    popl %eax
+; AVX-X86-NEXT:    .cfi_def_cfa_offset 4
+; AVX-X86-NEXT:    retl
+;
+; AVX-X64-LABEL: fceil32:
+; AVX-X64:       # %bb.0:
+; AVX-X64-NEXT:    vroundss $10, %xmm0, %xmm0, %xmm0
+; AVX-X64-NEXT:    retq
+  %res = call float @llvm.experimental.constrained.ceil.f32(
+                        float %f, metadata !"fpexcept.strict")
+  ret float %res
+}
+
+define double @fceilf64(double %f) #0 {
+; SSE41-X86-LABEL: fceilf64:
+; SSE41-X86:       # %bb.0:
+; SSE41-X86-NEXT:    pushl %ebp
+; SSE41-X86-NEXT:    .cfi_def_cfa_offset 8
+; SSE41-X86-NEXT:    .cfi_offset %ebp, -8
+; SSE41-X86-NEXT:    movl %esp, %ebp
+; SSE41-X86-NEXT:    .cfi_def_cfa_register %ebp
+; SSE41-X86-NEXT:    andl $-8, %esp
+; SSE41-X86-NEXT:    subl $8, %esp
+; SSE41-X86-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE41-X86-NEXT:    roundsd $10, %xmm0, %xmm0
+; SSE41-X86-NEXT:    movsd %xmm0, (%esp)
+; SSE41-X86-NEXT:    fldl (%esp)
+; SSE41-X86-NEXT:    movl %ebp, %esp
+; SSE41-X86-NEXT:    popl %ebp
+; SSE41-X86-NEXT:    .cfi_def_cfa %esp, 4
+; SSE41-X86-NEXT:    retl
+;
+; SSE41-X64-LABEL: fceilf64:
+; SSE41-X64:       # %bb.0:
+; SSE41-X64-NEXT:    roundsd $10, %xmm0, %xmm0
+; SSE41-X64-NEXT:    retq
+;
+; AVX-X86-LABEL: fceilf64:
+; AVX-X86:       # %bb.0:
+; AVX-X86-NEXT:    pushl %ebp
+; AVX-X86-NEXT:    .cfi_def_cfa_offset 8
+; AVX-X86-NEXT:    .cfi_offset %ebp, -8
+; AVX-X86-NEXT:    movl %esp, %ebp
+; AVX-X86-NEXT:    .cfi_def_cfa_register %ebp
+; AVX-X86-NEXT:    andl $-8, %esp
+; AVX-X86-NEXT:    subl $8, %esp
+; AVX-X86-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-X86-NEXT:    vroundsd $10, %xmm0, %xmm0, %xmm0
+; AVX-X86-NEXT:    vmovsd %xmm0, (%esp)
+; AVX-X86-NEXT:    fldl (%esp)
+; AVX-X86-NEXT:    movl %ebp, %esp
+; AVX-X86-NEXT:    popl %ebp
+; AVX-X86-NEXT:    .cfi_def_cfa %esp, 4
+; AVX-X86-NEXT:    retl
+;
+; AVX-X64-LABEL: fceilf64:
+; AVX-X64:       # %bb.0:
+; AVX-X64-NEXT:    vroundsd $10, %xmm0, %xmm0, %xmm0
+; AVX-X64-NEXT:    retq
+  %res = call double @llvm.experimental.constrained.ceil.f64(
+                        double %f, metadata !"fpexcept.strict")
+  ret double %res
+}
+
+define float @ffloor32(float %f) #0 {
+; SSE41-X86-LABEL: ffloor32:
+; SSE41-X86:       # %bb.0:
+; SSE41-X86-NEXT:    pushl %eax
+; SSE41-X86-NEXT:    .cfi_def_cfa_offset 8
+; SSE41-X86-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE41-X86-NEXT:    roundss $9, %xmm0, %xmm0
+; SSE41-X86-NEXT:    movss %xmm0, (%esp)
+; SSE41-X86-NEXT:    flds (%esp)
+; SSE41-X86-NEXT:    popl %eax
+; SSE41-X86-NEXT:    .cfi_def_cfa_offset 4
+; SSE41-X86-NEXT:    retl
+;
+; SSE41-X64-LABEL: ffloor32:
+; SSE41-X64:       # %bb.0:
+; SSE41-X64-NEXT:    roundss $9, %xmm0, %xmm0
+; SSE41-X64-NEXT:    retq
+;
+; AVX-X86-LABEL: ffloor32:
+; AVX-X86:       # %bb.0:
+; AVX-X86-NEXT:    pushl %eax
+; AVX-X86-NEXT:    .cfi_def_cfa_offset 8
+; AVX-X86-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-X86-NEXT:    vroundss $9, %xmm0, %xmm0, %xmm0
+; AVX-X86-NEXT:    vmovss %xmm0, (%esp)
+; AVX-X86-NEXT:    flds (%esp)
+; AVX-X86-NEXT:    popl %eax
+; AVX-X86-NEXT:    .cfi_def_cfa_offset 4
+; AVX-X86-NEXT:    retl
+;
+; AVX-X64-LABEL: ffloor32:
+; AVX-X64:       # %bb.0:
+; AVX-X64-NEXT:    vroundss $9, %xmm0, %xmm0, %xmm0
+; AVX-X64-NEXT:    retq
+  %res = call float @llvm.experimental.constrained.floor.f32(
+                        float %f, metadata !"fpexcept.strict")
+  ret float %res
+}
+
+define double @ffloorf64(double %f) #0 {
+; SSE41-X86-LABEL: ffloorf64:
+; SSE41-X86:       # %bb.0:
+; SSE41-X86-NEXT:    pushl %ebp
+; SSE41-X86-NEXT:    .cfi_def_cfa_offset 8
+; SSE41-X86-NEXT:    .cfi_offset %ebp, -8
+; SSE41-X86-NEXT:    movl %esp, %ebp
+; SSE41-X86-NEXT:    .cfi_def_cfa_register %ebp
+; SSE41-X86-NEXT:    andl $-8, %esp
+; SSE41-X86-NEXT:    subl $8, %esp
+; SSE41-X86-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE41-X86-NEXT:    roundsd $9, %xmm0, %xmm0
+; SSE41-X86-NEXT:    movsd %xmm0, (%esp)
+; SSE41-X86-NEXT:    fldl (%esp)
+; SSE41-X86-NEXT:    movl %ebp, %esp
+; SSE41-X86-NEXT:    popl %ebp
+; SSE41-X86-NEXT:    .cfi_def_cfa %esp, 4
+; SSE41-X86-NEXT:    retl
+;
+; SSE41-X64-LABEL: ffloorf64:
+; SSE41-X64:       # %bb.0:
+; SSE41-X64-NEXT:    roundsd $9, %xmm0, %xmm0
+; SSE41-X64-NEXT:    retq
+;
+; AVX-X86-LABEL: ffloorf64:
+; AVX-X86:       # %bb.0:
+; AVX-X86-NEXT:    pushl %ebp
+; AVX-X86-NEXT:    .cfi_def_cfa_offset 8
+; AVX-X86-NEXT:    .cfi_offset %ebp, -8
+; AVX-X86-NEXT:    movl %esp, %ebp
+; AVX-X86-NEXT:    .cfi_def_cfa_register %ebp
+; AVX-X86-NEXT:    andl $-8, %esp
+; AVX-X86-NEXT:    subl $8, %esp
+; AVX-X86-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-X86-NEXT:    vroundsd $9, %xmm0, %xmm0, %xmm0
+; AVX-X86-NEXT:    vmovsd %xmm0, (%esp)
+; AVX-X86-NEXT:    fldl (%esp)
+; AVX-X86-NEXT:    movl %ebp, %esp
+; AVX-X86-NEXT:    popl %ebp
+; AVX-X86-NEXT:    .cfi_def_cfa %esp, 4
+; AVX-X86-NEXT:    retl
+;
+; AVX-X64-LABEL: ffloorf64:
+; AVX-X64:       # %bb.0:
+; AVX-X64-NEXT:    vroundsd $9, %xmm0, %xmm0, %xmm0
+; AVX-X64-NEXT:    retq
+  %res = call double @llvm.experimental.constrained.floor.f64(
+                        double %f, metadata !"fpexcept.strict")
+  ret double %res
+}
+
+define float @ftrunc32(float %f) #0 {
+; SSE41-X86-LABEL: ftrunc32:
+; SSE41-X86:       # %bb.0:
+; SSE41-X86-NEXT:    pushl %eax
+; SSE41-X86-NEXT:    .cfi_def_cfa_offset 8
+; SSE41-X86-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE41-X86-NEXT:    roundss $11, %xmm0, %xmm0
+; SSE41-X86-NEXT:    movss %xmm0, (%esp)
+; SSE41-X86-NEXT:    flds (%esp)
+; SSE41-X86-NEXT:    popl %eax
+; SSE41-X86-NEXT:    .cfi_def_cfa_offset 4
+; SSE41-X86-NEXT:    retl
+;
+; SSE41-X64-LABEL: ftrunc32:
+; SSE41-X64:       # %bb.0:
+; SSE41-X64-NEXT:    roundss $11, %xmm0, %xmm0
+; SSE41-X64-NEXT:    retq
+;
+; AVX-X86-LABEL: ftrunc32:
+; AVX-X86:       # %bb.0:
+; AVX-X86-NEXT:    pushl %eax
+; AVX-X86-NEXT:    .cfi_def_cfa_offset 8
+; AVX-X86-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-X86-NEXT:    vroundss $11, %xmm0, %xmm0, %xmm0
+; AVX-X86-NEXT:    vmovss %xmm0, (%esp)
+; AVX-X86-NEXT:    flds (%esp)
+; AVX-X86-NEXT:    popl %eax
+; AVX-X86-NEXT:    .cfi_def_cfa_offset 4
+; AVX-X86-NEXT:    retl
+;
+; AVX-X64-LABEL: ftrunc32:
+; AVX-X64:       # %bb.0:
+; AVX-X64-NEXT:    vroundss $11, %xmm0, %xmm0, %xmm0
+; AVX-X64-NEXT:    retq
+  %res = call float @llvm.experimental.constrained.trunc.f32(
+                        float %f, metadata !"fpexcept.strict")
+  ret float %res
+}
+
+define double @ftruncf64(double %f) #0 {
+; SSE41-X86-LABEL: ftruncf64:
+; SSE41-X86:       # %bb.0:
+; SSE41-X86-NEXT:    pushl %ebp
+; SSE41-X86-NEXT:    .cfi_def_cfa_offset 8
+; SSE41-X86-NEXT:    .cfi_offset %ebp, -8
+; SSE41-X86-NEXT:    movl %esp, %ebp
+; SSE41-X86-NEXT:    .cfi_def_cfa_register %ebp
+; SSE41-X86-NEXT:    andl $-8, %esp
+; SSE41-X86-NEXT:    subl $8, %esp
+; SSE41-X86-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE41-X86-NEXT:    roundsd $11, %xmm0, %xmm0
+; SSE41-X86-NEXT:    movsd %xmm0, (%esp)
+; SSE41-X86-NEXT:    fldl (%esp)
+; SSE41-X86-NEXT:    movl %ebp, %esp
+; SSE41-X86-NEXT:    popl %ebp
+; SSE41-X86-NEXT:    .cfi_def_cfa %esp, 4
+; SSE41-X86-NEXT:    retl
+;
+; SSE41-X64-LABEL: ftruncf64:
+; SSE41-X64:       # %bb.0:
+; SSE41-X64-NEXT:    roundsd $11, %xmm0, %xmm0
+; SSE41-X64-NEXT:    retq
+;
+; AVX-X86-LABEL: ftruncf64:
+; AVX-X86:       # %bb.0:
+; AVX-X86-NEXT:    pushl %ebp
+; AVX-X86-NEXT:    .cfi_def_cfa_offset 8
+; AVX-X86-NEXT:    .cfi_offset %ebp, -8
+; AVX-X86-NEXT:    movl %esp, %ebp
+; AVX-X86-NEXT:    .cfi_def_cfa_register %ebp
+; AVX-X86-NEXT:    andl $-8, %esp
+; AVX-X86-NEXT:    subl $8, %esp
+; AVX-X86-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-X86-NEXT:    vroundsd $11, %xmm0, %xmm0, %xmm0
+; AVX-X86-NEXT:    vmovsd %xmm0, (%esp)
+; AVX-X86-NEXT:    fldl (%esp)
+; AVX-X86-NEXT:    movl %ebp, %esp
+; AVX-X86-NEXT:    popl %ebp
+; AVX-X86-NEXT:    .cfi_def_cfa %esp, 4
+; AVX-X86-NEXT:    retl
+;
+; AVX-X64-LABEL: ftruncf64:
+; AVX-X64:       # %bb.0:
+; AVX-X64-NEXT:    vroundsd $11, %xmm0, %xmm0, %xmm0
+; AVX-X64-NEXT:    retq
+  %res = call double @llvm.experimental.constrained.trunc.f64(
+                        double %f, metadata !"fpexcept.strict")
+  ret double %res
+}
+
+define float @frint32(float %f) #0 {
+; SSE41-X86-LABEL: frint32:
+; SSE41-X86:       # %bb.0:
+; SSE41-X86-NEXT:    pushl %eax
+; SSE41-X86-NEXT:    .cfi_def_cfa_offset 8
+; SSE41-X86-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE41-X86-NEXT:    roundss $4, %xmm0, %xmm0
+; SSE41-X86-NEXT:    movss %xmm0, (%esp)
+; SSE41-X86-NEXT:    flds (%esp)
+; SSE41-X86-NEXT:    popl %eax
+; SSE41-X86-NEXT:    .cfi_def_cfa_offset 4
+; SSE41-X86-NEXT:    retl
+;
+; SSE41-X64-LABEL: frint32:
+; SSE41-X64:       # %bb.0:
+; SSE41-X64-NEXT:    roundss $4, %xmm0, %xmm0
+; SSE41-X64-NEXT:    retq
+;
+; AVX-X86-LABEL: frint32:
+; AVX-X86:       # %bb.0:
+; AVX-X86-NEXT:    pushl %eax
+; AVX-X86-NEXT:    .cfi_def_cfa_offset 8
+; AVX-X86-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-X86-NEXT:    vroundss $4, %xmm0, %xmm0, %xmm0
+; AVX-X86-NEXT:    vmovss %xmm0, (%esp)
+; AVX-X86-NEXT:    flds (%esp)
+; AVX-X86-NEXT:    popl %eax
+; AVX-X86-NEXT:    .cfi_def_cfa_offset 4
+; AVX-X86-NEXT:    retl
+;
+; AVX-X64-LABEL: frint32:
+; AVX-X64:       # %bb.0:
+; AVX-X64-NEXT:    vroundss $4, %xmm0, %xmm0, %xmm0
+; AVX-X64-NEXT:    retq
+  %res = call float @llvm.experimental.constrained.rint.f32(
+                        float %f,
+                        metadata !"round.dynamic", metadata !"fpexcept.strict")
+  ret float %res
+}
+
+define double @frintf64(double %f) #0 {
+; SSE41-X86-LABEL: frintf64:
+; SSE41-X86:       # %bb.0:
+; SSE41-X86-NEXT:    pushl %ebp
+; SSE41-X86-NEXT:    .cfi_def_cfa_offset 8
+; SSE41-X86-NEXT:    .cfi_offset %ebp, -8
+; SSE41-X86-NEXT:    movl %esp, %ebp
+; SSE41-X86-NEXT:    .cfi_def_cfa_register %ebp
+; SSE41-X86-NEXT:    andl $-8, %esp
+; SSE41-X86-NEXT:    subl $8, %esp
+; SSE41-X86-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE41-X86-NEXT:    roundsd $4, %xmm0, %xmm0
+; SSE41-X86-NEXT:    movsd %xmm0, (%esp)
+; SSE41-X86-NEXT:    fldl (%esp)
+; SSE41-X86-NEXT:    movl %ebp, %esp
+; SSE41-X86-NEXT:    popl %ebp
+; SSE41-X86-NEXT:    .cfi_def_cfa %esp, 4
+; SSE41-X86-NEXT:    retl
+;
+; SSE41-X64-LABEL: frintf64:
+; SSE41-X64:       # %bb.0:
+; SSE41-X64-NEXT:    roundsd $4, %xmm0, %xmm0
+; SSE41-X64-NEXT:    retq
+;
+; AVX-X86-LABEL: frintf64:
+; AVX-X86:       # %bb.0:
+; AVX-X86-NEXT:    pushl %ebp
+; AVX-X86-NEXT:    .cfi_def_cfa_offset 8
+; AVX-X86-NEXT:    .cfi_offset %ebp, -8
+; AVX-X86-NEXT:    movl %esp, %ebp
+; AVX-X86-NEXT:    .cfi_def_cfa_register %ebp
+; AVX-X86-NEXT:    andl $-8, %esp
+; AVX-X86-NEXT:    subl $8, %esp
+; AVX-X86-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-X86-NEXT:    vroundsd $4, %xmm0, %xmm0, %xmm0
+; AVX-X86-NEXT:    vmovsd %xmm0, (%esp)
+; AVX-X86-NEXT:    fldl (%esp)
+; AVX-X86-NEXT:    movl %ebp, %esp
+; AVX-X86-NEXT:    popl %ebp
+; AVX-X86-NEXT:    .cfi_def_cfa %esp, 4
+; AVX-X86-NEXT:    retl
+;
+; AVX-X64-LABEL: frintf64:
+; AVX-X64:       # %bb.0:
+; AVX-X64-NEXT:    vroundsd $4, %xmm0, %xmm0, %xmm0
+; AVX-X64-NEXT:    retq
+  %res = call double @llvm.experimental.constrained.rint.f64(
+                        double %f,
+                        metadata !"round.dynamic", metadata !"fpexcept.strict")
+  ret double %res
+}
+
+define float @fnearbyint32(float %f) #0 {
+; SSE41-X86-LABEL: fnearbyint32:
+; SSE41-X86:       # %bb.0:
+; SSE41-X86-NEXT:    pushl %eax
+; SSE41-X86-NEXT:    .cfi_def_cfa_offset 8
+; SSE41-X86-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE41-X86-NEXT:    roundss $12, %xmm0, %xmm0
+; SSE41-X86-NEXT:    movss %xmm0, (%esp)
+; SSE41-X86-NEXT:    flds (%esp)
+; SSE41-X86-NEXT:    popl %eax
+; SSE41-X86-NEXT:    .cfi_def_cfa_offset 4
+; SSE41-X86-NEXT:    retl
+;
+; SSE41-X64-LABEL: fnearbyint32:
+; SSE41-X64:       # %bb.0:
+; SSE41-X64-NEXT:    roundss $12, %xmm0, %xmm0
+; SSE41-X64-NEXT:    retq
+;
+; AVX-X86-LABEL: fnearbyint32:
+; AVX-X86:       # %bb.0:
+; AVX-X86-NEXT:    pushl %eax
+; AVX-X86-NEXT:    .cfi_def_cfa_offset 8
+; AVX-X86-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-X86-NEXT:    vroundss $12, %xmm0, %xmm0, %xmm0
+; AVX-X86-NEXT:    vmovss %xmm0, (%esp)
+; AVX-X86-NEXT:    flds (%esp)
+; AVX-X86-NEXT:    popl %eax
+; AVX-X86-NEXT:    .cfi_def_cfa_offset 4
+; AVX-X86-NEXT:    retl
+;
+; AVX-X64-LABEL: fnearbyint32:
+; AVX-X64:       # %bb.0:
+; AVX-X64-NEXT:    vroundss $12, %xmm0, %xmm0, %xmm0
+; AVX-X64-NEXT:    retq
+  %res = call float @llvm.experimental.constrained.nearbyint.f32(
+                        float %f,
+                        metadata !"round.dynamic", metadata !"fpexcept.strict")
+  ret float %res
+}
+
+define double @fnearbyintf64(double %f) #0 {
+; SSE41-X86-LABEL: fnearbyintf64:
+; SSE41-X86:       # %bb.0:
+; SSE41-X86-NEXT:    pushl %ebp
+; SSE41-X86-NEXT:    .cfi_def_cfa_offset 8
+; SSE41-X86-NEXT:    .cfi_offset %ebp, -8
+; SSE41-X86-NEXT:    movl %esp, %ebp
+; SSE41-X86-NEXT:    .cfi_def_cfa_register %ebp
+; SSE41-X86-NEXT:    andl $-8, %esp
+; SSE41-X86-NEXT:    subl $8, %esp
+; SSE41-X86-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE41-X86-NEXT:    roundsd $12, %xmm0, %xmm0
+; SSE41-X86-NEXT:    movsd %xmm0, (%esp)
+; SSE41-X86-NEXT:    fldl (%esp)
+; SSE41-X86-NEXT:    movl %ebp, %esp
+; SSE41-X86-NEXT:    popl %ebp
+; SSE41-X86-NEXT:    .cfi_def_cfa %esp, 4
+; SSE41-X86-NEXT:    retl
+;
+; SSE41-X64-LABEL: fnearbyintf64:
+; SSE41-X64:       # %bb.0:
+; SSE41-X64-NEXT:    roundsd $12, %xmm0, %xmm0
+; SSE41-X64-NEXT:    retq
+;
+; AVX-X86-LABEL: fnearbyintf64:
+; AVX-X86:       # %bb.0:
+; AVX-X86-NEXT:    pushl %ebp
+; AVX-X86-NEXT:    .cfi_def_cfa_offset 8
+; AVX-X86-NEXT:    .cfi_offset %ebp, -8
+; AVX-X86-NEXT:    movl %esp, %ebp
+; AVX-X86-NEXT:    .cfi_def_cfa_register %ebp
+; AVX-X86-NEXT:    andl $-8, %esp
+; AVX-X86-NEXT:    subl $8, %esp
+; AVX-X86-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-X86-NEXT:    vroundsd $12, %xmm0, %xmm0, %xmm0
+; AVX-X86-NEXT:    vmovsd %xmm0, (%esp)
+; AVX-X86-NEXT:    fldl (%esp)
+; AVX-X86-NEXT:    movl %ebp, %esp
+; AVX-X86-NEXT:    popl %ebp
+; AVX-X86-NEXT:    .cfi_def_cfa %esp, 4
+; AVX-X86-NEXT:    retl
+;
+; AVX-X64-LABEL: fnearbyintf64:
+; AVX-X64:       # %bb.0:
+; AVX-X64-NEXT:    vroundsd $12, %xmm0, %xmm0, %xmm0
+; AVX-X64-NEXT:    retq
+  %res = call double @llvm.experimental.constrained.nearbyint.f64(
+                        double %f,
+                        metadata !"round.dynamic", metadata !"fpexcept.strict")
+  ret double %res
+}
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/X86/fp-strict-scalar.ll b/llvm/test/CodeGen/X86/fp-strict-scalar.ll
new file mode 100644
index 000000000..c864acda3
--- /dev/null
+++ b/llvm/test/CodeGen/X86/fp-strict-scalar.ll
@@ -0,0 +1,698 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse2 -O3 | FileCheck %s --check-prefixes=SSE-X86
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 -O3 | FileCheck %s --check-prefixes=SSE-X64
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+fma -O3 | FileCheck %s --check-prefixes=AVX-X86
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+fma -O3 | FileCheck %s --check-prefixes=AVX-X64
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx512f -mattr=+avx512vl -O3 | FileCheck %s --check-prefixes=AVX-X86
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f -mattr=+avx512vl -O3 | FileCheck %s --check-prefixes=AVX-X64
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=-sse -O3 | FileCheck %s --check-prefixes=X87
+
+declare double @llvm.experimental.constrained.fadd.f64(double, double, metadata, metadata)
+declare float @llvm.experimental.constrained.fadd.f32(float, float, metadata, metadata)
+declare double @llvm.experimental.constrained.fsub.f64(double, double, metadata, metadata)
+declare float @llvm.experimental.constrained.fsub.f32(float, float, metadata, metadata)
+declare double @llvm.experimental.constrained.fmul.f64(double, double, metadata, metadata)
+declare float @llvm.experimental.constrained.fmul.f32(float, float, metadata, metadata)
+declare double @llvm.experimental.constrained.fdiv.f64(double, double, metadata, metadata)
+declare float @llvm.experimental.constrained.fdiv.f32(float, float, metadata, metadata)
+declare double @llvm.experimental.constrained.fpext.f64.f32(float, metadata)
+declare float @llvm.experimental.constrained.fptrunc.f32.f64(double, metadata, metadata)
+declare float @llvm.experimental.constrained.sqrt.f32(float, metadata, metadata)
+declare double @llvm.experimental.constrained.sqrt.f64(double, metadata, metadata)
+declare float @llvm.experimental.constrained.fma.f32(float, float, float, metadata, metadata)
+declare double @llvm.experimental.constrained.fma.f64(double, double, double, metadata, metadata)
+
+define double @fadd_f64(double %a, double %b) nounwind strictfp {
+; SSE-X86-LABEL: fadd_f64:
+; SSE-X86:       # %bb.0:
+; SSE-X86-NEXT:    pushl %ebp
+; SSE-X86-NEXT:    movl %esp, %ebp
+; SSE-X86-NEXT:    andl $-8, %esp
+; SSE-X86-NEXT:    subl $8, %esp
+; SSE-X86-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE-X86-NEXT:    addsd 16(%ebp), %xmm0
+; SSE-X86-NEXT:    movsd %xmm0, (%esp)
+; SSE-X86-NEXT:    fldl (%esp)
+; SSE-X86-NEXT:    movl %ebp, %esp
+; SSE-X86-NEXT:    popl %ebp
+; SSE-X86-NEXT:    retl
+;
+; SSE-X64-LABEL: fadd_f64:
+; SSE-X64:       # %bb.0:
+; SSE-X64-NEXT:    addsd %xmm1, %xmm0
+; SSE-X64-NEXT:    retq
+;
+; AVX-X86-LABEL: fadd_f64:
+; AVX-X86:       # %bb.0:
+; AVX-X86-NEXT:    pushl %ebp
+; AVX-X86-NEXT:    movl %esp, %ebp
+; AVX-X86-NEXT:    andl $-8, %esp
+; AVX-X86-NEXT:    subl $8, %esp
+; AVX-X86-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-X86-NEXT:    vaddsd 16(%ebp), %xmm0, %xmm0
+; AVX-X86-NEXT:    vmovsd %xmm0, (%esp)
+; AVX-X86-NEXT:    fldl (%esp)
+; AVX-X86-NEXT:    movl %ebp, %esp
+; AVX-X86-NEXT:    popl %ebp
+; AVX-X86-NEXT:    retl
+;
+; AVX-X64-LABEL: fadd_f64:
+; AVX-X64:       # %bb.0:
+; AVX-X64-NEXT:    vaddsd %xmm1, %xmm0, %xmm0
+; AVX-X64-NEXT:    retq
+;
+; X87-LABEL: fadd_f64:
+; X87:       # %bb.0:
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    faddl {{[0-9]+}}(%esp)
+; X87-NEXT:    retl
+  %ret = call double @llvm.experimental.constrained.fadd.f64(double %a, double %b,
+                                                             metadata !"round.dynamic",
+                                                             metadata !"fpexcept.strict") #0
+  ret double %ret
+}
+
+define float @fadd_f32(float %a, float %b) nounwind strictfp {
+; SSE-X86-LABEL: fadd_f32:
+; SSE-X86:       # %bb.0:
+; SSE-X86-NEXT:    pushl %eax
+; SSE-X86-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE-X86-NEXT:    addss {{[0-9]+}}(%esp), %xmm0
+; SSE-X86-NEXT:    movss %xmm0, (%esp)
+; SSE-X86-NEXT:    flds (%esp)
+; SSE-X86-NEXT:    popl %eax
+; SSE-X86-NEXT:    retl
+;
+; SSE-X64-LABEL: fadd_f32:
+; SSE-X64:       # %bb.0:
+; SSE-X64-NEXT:    addss %xmm1, %xmm0
+; SSE-X64-NEXT:    retq
+;
+; AVX-X86-LABEL: fadd_f32:
+; AVX-X86:       # %bb.0:
+; AVX-X86-NEXT:    pushl %eax
+; AVX-X86-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-X86-NEXT:    vaddss {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-X86-NEXT:    vmovss %xmm0, (%esp)
+; AVX-X86-NEXT:    flds (%esp)
+; AVX-X86-NEXT:    popl %eax
+; AVX-X86-NEXT:    retl
+;
+; AVX-X64-LABEL: fadd_f32:
+; AVX-X64:       # %bb.0:
+; AVX-X64-NEXT:    vaddss %xmm1, %xmm0, %xmm0
+; AVX-X64-NEXT:    retq
+;
+; X87-LABEL: fadd_f32:
+; X87:       # %bb.0:
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    fadds {{[0-9]+}}(%esp)
+; X87-NEXT:    retl
+  %ret = call float @llvm.experimental.constrained.fadd.f32(float %a, float %b,
+                                                            metadata !"round.dynamic",
+                                                            metadata !"fpexcept.strict") #0
+  ret float %ret
+}
+
+define double @fsub_f64(double %a, double %b) nounwind strictfp {
+; SSE-X86-LABEL: fsub_f64:
+; SSE-X86:       # %bb.0:
+; SSE-X86-NEXT:    pushl %ebp
+; SSE-X86-NEXT:    movl %esp, %ebp
+; SSE-X86-NEXT:    andl $-8, %esp
+; SSE-X86-NEXT:    subl $8, %esp
+; SSE-X86-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE-X86-NEXT:    subsd 16(%ebp), %xmm0
+; SSE-X86-NEXT:    movsd %xmm0, (%esp)
+; SSE-X86-NEXT:    fldl (%esp)
+; SSE-X86-NEXT:    movl %ebp, %esp
+; SSE-X86-NEXT:    popl %ebp
+; SSE-X86-NEXT:    retl
+;
+; SSE-X64-LABEL: fsub_f64:
+; SSE-X64:       # %bb.0:
+; SSE-X64-NEXT:    subsd %xmm1, %xmm0
+; SSE-X64-NEXT:    retq
+;
+; AVX-X86-LABEL: fsub_f64:
+; AVX-X86:       # %bb.0:
+; AVX-X86-NEXT:    pushl %ebp
+; AVX-X86-NEXT:    movl %esp, %ebp
+; AVX-X86-NEXT:    andl $-8, %esp
+; AVX-X86-NEXT:    subl $8, %esp
+; AVX-X86-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-X86-NEXT:    vsubsd 16(%ebp), %xmm0, %xmm0
+; AVX-X86-NEXT:    vmovsd %xmm0, (%esp)
+; AVX-X86-NEXT:    fldl (%esp)
+; AVX-X86-NEXT:    movl %ebp, %esp
+; AVX-X86-NEXT:    popl %ebp
+; AVX-X86-NEXT:    retl
+;
+; AVX-X64-LABEL: fsub_f64:
+; AVX-X64:       # %bb.0:
+; AVX-X64-NEXT:    vsubsd %xmm1, %xmm0, %xmm0
+; AVX-X64-NEXT:    retq
+;
+; X87-LABEL: fsub_f64:
+; X87:       # %bb.0:
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fsubl {{[0-9]+}}(%esp)
+; X87-NEXT:    retl
+  %ret = call double @llvm.experimental.constrained.fsub.f64(double %a, double %b,
+                                                             metadata !"round.dynamic",
+                                                             metadata !"fpexcept.strict") #0
+  ret double %ret
+}
+
+define float @fsub_f32(float %a, float %b) nounwind strictfp {
+; SSE-X86-LABEL: fsub_f32:
+; SSE-X86:       # %bb.0:
+; SSE-X86-NEXT:    pushl %eax
+; SSE-X86-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE-X86-NEXT:    subss {{[0-9]+}}(%esp), %xmm0
+; SSE-X86-NEXT:    movss %xmm0, (%esp)
+; SSE-X86-NEXT:    flds (%esp)
+; SSE-X86-NEXT:    popl %eax
+; SSE-X86-NEXT:    retl
+;
+; SSE-X64-LABEL: fsub_f32:
+; SSE-X64:       # %bb.0:
+; SSE-X64-NEXT:    subss %xmm1, %xmm0
+; SSE-X64-NEXT:    retq
+;
+; AVX-X86-LABEL: fsub_f32:
+; AVX-X86:       # %bb.0:
+; AVX-X86-NEXT:    pushl %eax
+; AVX-X86-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-X86-NEXT:    vsubss {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-X86-NEXT:    vmovss %xmm0, (%esp)
+; AVX-X86-NEXT:    flds (%esp)
+; AVX-X86-NEXT:    popl %eax
+; AVX-X86-NEXT:    retl
+;
+; AVX-X64-LABEL: fsub_f32:
+; AVX-X64:       # %bb.0:
+; AVX-X64-NEXT:    vsubss %xmm1, %xmm0, %xmm0
+; AVX-X64-NEXT:    retq
+;
+; X87-LABEL: fsub_f32:
+; X87:       # %bb.0:
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    fsubs {{[0-9]+}}(%esp)
+; X87-NEXT:    retl
+  %ret = call float @llvm.experimental.constrained.fsub.f32(float %a, float %b,
+                                                            metadata !"round.dynamic",
+                                                            metadata !"fpexcept.strict") #0
+  ret float %ret
+}
+
+define double @fmul_f64(double %a, double %b) nounwind strictfp {
+; SSE-X86-LABEL: fmul_f64:
+; SSE-X86:       # %bb.0:
+; SSE-X86-NEXT:    pushl %ebp
+; SSE-X86-NEXT:    movl %esp, %ebp
+; SSE-X86-NEXT:    andl $-8, %esp
+; SSE-X86-NEXT:    subl $8, %esp
+; SSE-X86-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE-X86-NEXT:    mulsd 16(%ebp), %xmm0
+; SSE-X86-NEXT:    movsd %xmm0, (%esp)
+; SSE-X86-NEXT:    fldl (%esp)
+; SSE-X86-NEXT:    movl %ebp, %esp
+; SSE-X86-NEXT:    popl %ebp
+; SSE-X86-NEXT:    retl
+;
+; SSE-X64-LABEL: fmul_f64:
+; SSE-X64:       # %bb.0:
+; SSE-X64-NEXT:    mulsd %xmm1, %xmm0
+; SSE-X64-NEXT:    retq
+;
+; AVX-X86-LABEL: fmul_f64:
+; AVX-X86:       # %bb.0:
+; AVX-X86-NEXT:    pushl %ebp
+; AVX-X86-NEXT:    movl %esp, %ebp
+; AVX-X86-NEXT:    andl $-8, %esp
+; AVX-X86-NEXT:    subl $8, %esp
+; AVX-X86-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-X86-NEXT:    vmulsd 16(%ebp), %xmm0, %xmm0
+; AVX-X86-NEXT:    vmovsd %xmm0, (%esp)
+; AVX-X86-NEXT:    fldl (%esp)
+; AVX-X86-NEXT:    movl %ebp, %esp
+; AVX-X86-NEXT:    popl %ebp
+; AVX-X86-NEXT:    retl
+;
+; AVX-X64-LABEL: fmul_f64:
+; AVX-X64:       # %bb.0:
+; AVX-X64-NEXT:    vmulsd %xmm1, %xmm0, %xmm0
+; AVX-X64-NEXT:    retq
+;
+; X87-LABEL: fmul_f64:
+; X87:       # %bb.0:
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fmull {{[0-9]+}}(%esp)
+; X87-NEXT:    retl
+  %ret = call double @llvm.experimental.constrained.fmul.f64(double %a, double %b,
+                                                             metadata !"round.dynamic",
+                                                             metadata !"fpexcept.strict") #0
+  ret double %ret
+}
+
+define float @fmul_f32(float %a, float %b) nounwind strictfp {
+; SSE-X86-LABEL: fmul_f32:
+; SSE-X86:       # %bb.0:
+; SSE-X86-NEXT:    pushl %eax
+; SSE-X86-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE-X86-NEXT:    mulss {{[0-9]+}}(%esp), %xmm0
+; SSE-X86-NEXT:    movss %xmm0, (%esp)
+; SSE-X86-NEXT:    flds (%esp)
+; SSE-X86-NEXT:    popl %eax
+; SSE-X86-NEXT:    retl
+;
+; SSE-X64-LABEL: fmul_f32:
+; SSE-X64:       # %bb.0:
+; SSE-X64-NEXT:    mulss %xmm1, %xmm0
+; SSE-X64-NEXT:    retq
+;
+; AVX-X86-LABEL: fmul_f32:
+; AVX-X86:       # %bb.0:
+; AVX-X86-NEXT:    pushl %eax
+; AVX-X86-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-X86-NEXT:    vmulss {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-X86-NEXT:    vmovss %xmm0, (%esp)
+; AVX-X86-NEXT:    flds (%esp)
+; AVX-X86-NEXT:    popl %eax
+; AVX-X86-NEXT:    retl
+;
+; AVX-X64-LABEL: fmul_f32:
+; AVX-X64:       # %bb.0:
+; AVX-X64-NEXT:    vmulss %xmm1, %xmm0, %xmm0
+; AVX-X64-NEXT:    retq
+;
+; X87-LABEL: fmul_f32:
+; X87:       # %bb.0:
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    fmuls {{[0-9]+}}(%esp)
+; X87-NEXT:    retl
+  %ret = call float @llvm.experimental.constrained.fmul.f32(float %a, float %b,
+                                                            metadata !"round.dynamic",
+                                                            metadata !"fpexcept.strict") #0
+  ret float %ret
+}
+
+define double @fdiv_f64(double %a, double %b) nounwind strictfp {
+; SSE-X86-LABEL: fdiv_f64:
+; SSE-X86:       # %bb.0:
+; SSE-X86-NEXT:    pushl %ebp
+; SSE-X86-NEXT:    movl %esp, %ebp
+; SSE-X86-NEXT:    andl $-8, %esp
+; SSE-X86-NEXT:    subl $8, %esp
+; SSE-X86-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE-X86-NEXT:    divsd 16(%ebp), %xmm0
+; SSE-X86-NEXT:    movsd %xmm0, (%esp)
+; SSE-X86-NEXT:    fldl (%esp)
+; SSE-X86-NEXT:    movl %ebp, %esp
+; SSE-X86-NEXT:    popl %ebp
+; SSE-X86-NEXT:    retl
+;
+; SSE-X64-LABEL: fdiv_f64:
+; SSE-X64:       # %bb.0:
+; SSE-X64-NEXT:    divsd %xmm1, %xmm0
+; SSE-X64-NEXT:    retq
+;
+; AVX-X86-LABEL: fdiv_f64:
+; AVX-X86:       # %bb.0:
+; AVX-X86-NEXT:    pushl %ebp
+; AVX-X86-NEXT:    movl %esp, %ebp
+; AVX-X86-NEXT:    andl $-8, %esp
+; AVX-X86-NEXT:    subl $8, %esp
+; AVX-X86-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-X86-NEXT:    vdivsd 16(%ebp), %xmm0, %xmm0
+; AVX-X86-NEXT:    vmovsd %xmm0, (%esp)
+; AVX-X86-NEXT:    fldl (%esp)
+; AVX-X86-NEXT:    movl %ebp, %esp
+; AVX-X86-NEXT:    popl %ebp
+; AVX-X86-NEXT:    retl
+;
+; AVX-X64-LABEL: fdiv_f64:
+; AVX-X64:       # %bb.0:
+; AVX-X64-NEXT:    vdivsd %xmm1, %xmm0, %xmm0
+; AVX-X64-NEXT:    retq
+;
+; X87-LABEL: fdiv_f64:
+; X87:       # %bb.0:
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fdivl {{[0-9]+}}(%esp)
+; X87-NEXT:    retl
+  %ret = call double @llvm.experimental.constrained.fdiv.f64(double %a, double %b,
+                                                             metadata !"round.dynamic",
+                                                             metadata !"fpexcept.strict") #0
+  ret double %ret
+}
+
+define float @fdiv_f32(float %a, float %b) nounwind strictfp {
+; SSE-X86-LABEL: fdiv_f32:
+; SSE-X86:       # %bb.0:
+; SSE-X86-NEXT:    pushl %eax
+; SSE-X86-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE-X86-NEXT:    divss {{[0-9]+}}(%esp), %xmm0
+; SSE-X86-NEXT:    movss %xmm0, (%esp)
+; SSE-X86-NEXT:    flds (%esp)
+; SSE-X86-NEXT:    popl %eax
+; SSE-X86-NEXT:    retl
+;
+; SSE-X64-LABEL: fdiv_f32:
+; SSE-X64:       # %bb.0:
+; SSE-X64-NEXT:    divss %xmm1, %xmm0
+; SSE-X64-NEXT:    retq
+;
+; AVX-X86-LABEL: fdiv_f32:
+; AVX-X86:       # %bb.0:
+; AVX-X86-NEXT:    pushl %eax
+; AVX-X86-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-X86-NEXT:    vdivss {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-X86-NEXT:    vmovss %xmm0, (%esp)
+; AVX-X86-NEXT:    flds (%esp)
+; AVX-X86-NEXT:    popl %eax
+; AVX-X86-NEXT:    retl
+;
+; AVX-X64-LABEL: fdiv_f32:
+; AVX-X64:       # %bb.0:
+; AVX-X64-NEXT:    vdivss %xmm1, %xmm0, %xmm0
+; AVX-X64-NEXT:    retq
+;
+; X87-LABEL: fdiv_f32:
+; X87:       # %bb.0:
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    fdivs {{[0-9]+}}(%esp)
+; X87-NEXT:    retl
+  %ret = call float @llvm.experimental.constrained.fdiv.f32(float %a, float %b,
+                                                            metadata !"round.dynamic",
+                                                            metadata !"fpexcept.strict") #0
+  ret float %ret
+}
+
+define void @fpext_f32_to_f64(float* %val, double* %ret) nounwind strictfp {
+; SSE-X86-LABEL: fpext_f32_to_f64:
+; SSE-X86:       # %bb.0:
+; SSE-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; SSE-X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; SSE-X86-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE-X86-NEXT:    cvtss2sd %xmm0, %xmm0
+; SSE-X86-NEXT:    movsd %xmm0, (%eax)
+; SSE-X86-NEXT:    retl
+;
+; SSE-X64-LABEL: fpext_f32_to_f64:
+; SSE-X64:       # %bb.0:
+; SSE-X64-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE-X64-NEXT:    cvtss2sd %xmm0, %xmm0
+; SSE-X64-NEXT:    movsd %xmm0, (%rsi)
+; SSE-X64-NEXT:    retq
+;
+; AVX-X86-LABEL: fpext_f32_to_f64:
+; AVX-X86:       # %bb.0:
+; AVX-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; AVX-X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; AVX-X86-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-X86-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
+; AVX-X86-NEXT:    vmovsd %xmm0, (%eax)
+; AVX-X86-NEXT:    retl
+;
+; AVX-X64-LABEL: fpext_f32_to_f64:
+; AVX-X64:       # %bb.0:
+; AVX-X64-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-X64-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
+; AVX-X64-NEXT:    vmovsd %xmm0, (%rsi)
+; AVX-X64-NEXT:    retq
+;
+; X87-LABEL: fpext_f32_to_f64:
+; X87:       # %bb.0:
+; X87-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X87-NEXT:    flds (%ecx)
+; X87-NEXT:    fstpl (%eax)
+; X87-NEXT:    retl
+  %1 = load float, float* %val, align 4
+  %res = call double @llvm.experimental.constrained.fpext.f64.f32(float %1,
+                                                                  metadata !"fpexcept.strict") #0
+  store double %res, double* %ret, align 8
+  ret void
+}
+
+define void @fptrunc_double_to_f32(double* %val, float *%ret) nounwind strictfp {
+; SSE-X86-LABEL: fptrunc_double_to_f32:
+; SSE-X86:       # %bb.0:
+; SSE-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; SSE-X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; SSE-X86-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE-X86-NEXT:    cvtsd2ss %xmm0, %xmm0
+; SSE-X86-NEXT:    movss %xmm0, (%eax)
+; SSE-X86-NEXT:    retl
+;
+; SSE-X64-LABEL: fptrunc_double_to_f32:
+; SSE-X64:       # %bb.0:
+; SSE-X64-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE-X64-NEXT:    cvtsd2ss %xmm0, %xmm0
+; SSE-X64-NEXT:    movss %xmm0, (%rsi)
+; SSE-X64-NEXT:    retq
+;
+; AVX-X86-LABEL: fptrunc_double_to_f32:
+; AVX-X86:       # %bb.0:
+; AVX-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; AVX-X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; AVX-X86-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-X86-NEXT:    vcvtsd2ss %xmm0, %xmm0, %xmm0
+; AVX-X86-NEXT:    vmovss %xmm0, (%eax)
+; AVX-X86-NEXT:    retl
+;
+; AVX-X64-LABEL: fptrunc_double_to_f32:
+; AVX-X64:       # %bb.0:
+; AVX-X64-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-X64-NEXT:    vcvtsd2ss %xmm0, %xmm0, %xmm0
+; AVX-X64-NEXT:    vmovss %xmm0, (%rsi)
+; AVX-X64-NEXT:    retq
+;
+; X87-LABEL: fptrunc_double_to_f32:
+; X87:       # %bb.0:
+; X87-NEXT:    pushl %eax
+; X87-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X87-NEXT:    fldl (%ecx)
+; X87-NEXT:    fstps (%esp)
+; X87-NEXT:    flds (%esp)
+; X87-NEXT:    fstps (%eax)
+; X87-NEXT:    popl %eax
+; X87-NEXT:    retl
+  %1 = load double, double* %val, align 8
+  %res = call float @llvm.experimental.constrained.fptrunc.f32.f64(double %1,
+                                                                   metadata !"round.dynamic",
+                                                                   metadata !"fpexcept.strict") #0
+  store float %res, float* %ret, align 4
+  ret void
+}
+
+define void @fsqrt_f64(double* %a) nounwind strictfp {
+; SSE-X86-LABEL: fsqrt_f64:
+; SSE-X86:       # %bb.0:
+; SSE-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; SSE-X86-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE-X86-NEXT:    sqrtsd %xmm0, %xmm0
+; SSE-X86-NEXT:    movsd %xmm0, (%eax)
+; SSE-X86-NEXT:    retl
+;
+; SSE-X64-LABEL: fsqrt_f64:
+; SSE-X64:       # %bb.0:
+; SSE-X64-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE-X64-NEXT:    sqrtsd %xmm0, %xmm0
+; SSE-X64-NEXT:    movsd %xmm0, (%rdi)
+; SSE-X64-NEXT:    retq
+;
+; AVX-X86-LABEL: fsqrt_f64:
+; AVX-X86:       # %bb.0:
+; AVX-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; AVX-X86-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-X86-NEXT:    vsqrtsd %xmm0, %xmm0, %xmm0
+; AVX-X86-NEXT:    vmovsd %xmm0, (%eax)
+; AVX-X86-NEXT:    retl
+;
+; AVX-X64-LABEL: fsqrt_f64:
+; AVX-X64:       # %bb.0:
+; AVX-X64-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-X64-NEXT:    vsqrtsd %xmm0, %xmm0, %xmm0
+; AVX-X64-NEXT:    vmovsd %xmm0, (%rdi)
+; AVX-X64-NEXT:    retq
+;
+; X87-LABEL: fsqrt_f64:
+; X87:       # %bb.0:
+; X87-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    fldl (%eax)
+; X87-NEXT:    fsqrt
+; X87-NEXT:    fstpl (%eax)
+; X87-NEXT:    retl
+  %1 = load double, double* %a, align 8
+  %res = call double @llvm.experimental.constrained.sqrt.f64(double %1,
+                                                             metadata !"round.dynamic",
+                                                             metadata !"fpexcept.strict") #0
+  store double %res, double* %a, align 8
+  ret void
+}
+
+define void @fsqrt_f32(float* %a) nounwind strictfp {
+; SSE-X86-LABEL: fsqrt_f32:
+; SSE-X86:       # %bb.0:
+; SSE-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; SSE-X86-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE-X86-NEXT:    sqrtss %xmm0, %xmm0
+; SSE-X86-NEXT:    movss %xmm0, (%eax)
+; SSE-X86-NEXT:    retl
+;
+; SSE-X64-LABEL: fsqrt_f32:
+; SSE-X64:       # %bb.0:
+; SSE-X64-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE-X64-NEXT:    sqrtss %xmm0, %xmm0
+; SSE-X64-NEXT:    movss %xmm0, (%rdi)
+; SSE-X64-NEXT:    retq
+;
+; AVX-X86-LABEL: fsqrt_f32:
+; AVX-X86:       # %bb.0:
+; AVX-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; AVX-X86-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-X86-NEXT:    vsqrtss %xmm0, %xmm0, %xmm0
+; AVX-X86-NEXT:    vmovss %xmm0, (%eax)
+; AVX-X86-NEXT:    retl
+;
+; AVX-X64-LABEL: fsqrt_f32:
+; AVX-X64:       # %bb.0:
+; AVX-X64-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-X64-NEXT:    vsqrtss %xmm0, %xmm0, %xmm0
+; AVX-X64-NEXT:    vmovss %xmm0, (%rdi)
+; AVX-X64-NEXT:    retq
+;
+; X87-LABEL: fsqrt_f32:
+; X87:       # %bb.0:
+; X87-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    flds (%eax)
+; X87-NEXT:    fsqrt
+; X87-NEXT:    fstps (%eax)
+; X87-NEXT:    retl
+  %1 = load float, float* %a, align 4
+  %res = call float @llvm.experimental.constrained.sqrt.f32(float %1,
+                                                            metadata !"round.dynamic",
+                                                            metadata !"fpexcept.strict") #0
+  store float %res, float* %a, align 4
+  ret void
+}
+
+define double @fma_f64(double %a, double %b, double %c) nounwind strictfp {
+; SSE-X86-LABEL: fma_f64:
+; SSE-X86:       # %bb.0:
+; SSE-X86-NEXT:    subl $24, %esp
+; SSE-X86-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE-X86-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
+; SSE-X86-NEXT:    movsd {{.*#+}} xmm2 = mem[0],zero
+; SSE-X86-NEXT:    movsd %xmm2, {{[0-9]+}}(%esp)
+; SSE-X86-NEXT:    movsd %xmm1, {{[0-9]+}}(%esp)
+; SSE-X86-NEXT:    movsd %xmm0, (%esp)
+; SSE-X86-NEXT:    calll fma
+; SSE-X86-NEXT:    addl $24, %esp
+; SSE-X86-NEXT:    retl
+;
+; SSE-X64-LABEL: fma_f64:
+; SSE-X64:       # %bb.0:
+; SSE-X64-NEXT:    pushq %rax
+; SSE-X64-NEXT:    callq fma
+; SSE-X64-NEXT:    popq %rax
+; SSE-X64-NEXT:    retq
+;
+; AVX-X86-LABEL: fma_f64:
+; AVX-X86:       # %bb.0:
+; AVX-X86-NEXT:    pushl %ebp
+; AVX-X86-NEXT:    movl %esp, %ebp
+; AVX-X86-NEXT:    andl $-8, %esp
+; AVX-X86-NEXT:    subl $8, %esp
+; AVX-X86-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-X86-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
+; AVX-X86-NEXT:    vfmadd213sd {{.*#+}} xmm1 = (xmm0 * xmm1) + mem
+; AVX-X86-NEXT:    vmovsd %xmm1, (%esp)
+; AVX-X86-NEXT:    fldl (%esp)
+; AVX-X86-NEXT:    movl %ebp, %esp
+; AVX-X86-NEXT:    popl %ebp
+; AVX-X86-NEXT:    retl
+;
+; AVX-X64-LABEL: fma_f64:
+; AVX-X64:       # %bb.0:
+; AVX-X64-NEXT:    vfmadd213sd {{.*#+}} xmm0 = (xmm1 * xmm0) + xmm2
+; AVX-X64-NEXT:    retq
+;
+; X87-LABEL: fma_f64:
+; X87:       # %bb.0:
+; X87-NEXT:    subl $24, %esp
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fstpl {{[0-9]+}}(%esp)
+; X87-NEXT:    fstpl {{[0-9]+}}(%esp)
+; X87-NEXT:    fstpl (%esp)
+; X87-NEXT:    calll fma
+; X87-NEXT:    addl $24, %esp
+; X87-NEXT:    retl
+  %res = call double @llvm.experimental.constrained.fma.f64(double %a, double %b, double %c,
+                                                            metadata !"round.dynamic",
+                                                            metadata !"fpexcept.strict") #0
+  ret double %res
+}
+
+define float @fma_f32(float %a, float %b, float %c) nounwind strictfp {
+; SSE-X86-LABEL: fma_f32:
+; SSE-X86:       # %bb.0:
+; SSE-X86-NEXT:    subl $12, %esp
+; SSE-X86-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE-X86-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE-X86-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE-X86-NEXT:    movss %xmm2, {{[0-9]+}}(%esp)
+; SSE-X86-NEXT:    movss %xmm1, {{[0-9]+}}(%esp)
+; SSE-X86-NEXT:    movss %xmm0, (%esp)
+; SSE-X86-NEXT:    calll fmaf
+; SSE-X86-NEXT:    addl $12, %esp
+; SSE-X86-NEXT:    retl
+;
+; SSE-X64-LABEL: fma_f32:
+; SSE-X64:       # %bb.0:
+; SSE-X64-NEXT:    pushq %rax
+; SSE-X64-NEXT:    callq fmaf
+; SSE-X64-NEXT:    popq %rax
+; SSE-X64-NEXT:    retq
+;
+; AVX-X86-LABEL: fma_f32:
+; AVX-X86:       # %bb.0:
+; AVX-X86-NEXT:    pushl %eax
+; AVX-X86-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-X86-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX-X86-NEXT:    vfmadd213ss {{.*#+}} xmm1 = (xmm0 * xmm1) + mem
+; AVX-X86-NEXT:    vmovss %xmm1, (%esp)
+; AVX-X86-NEXT:    flds (%esp)
+; AVX-X86-NEXT:    popl %eax
+; AVX-X86-NEXT:    retl
+;
+; AVX-X64-LABEL: fma_f32:
+; AVX-X64:       # %bb.0:
+; AVX-X64-NEXT:    vfmadd213ss {{.*#+}} xmm0 = (xmm1 * xmm0) + xmm2
+; AVX-X64-NEXT:    retq
+;
+; X87-LABEL: fma_f32:
+; X87:       # %bb.0:
+; X87-NEXT:    subl $12, %esp
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    fstps {{[0-9]+}}(%esp)
+; X87-NEXT:    fstps {{[0-9]+}}(%esp)
+; X87-NEXT:    fstps (%esp)
+; X87-NEXT:    calll fmaf
+; X87-NEXT:    addl $12, %esp
+; X87-NEXT:    retl
+  %res = call float @llvm.experimental.constrained.fma.f32(float %a, float %b, float %c,
+                                                           metadata !"round.dynamic",
+                                                           metadata !"fpexcept.strict") #0
+  ret float %res
+}
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/X86/fp128-cast-strict.ll b/llvm/test/CodeGen/X86/fp128-cast-strict.ll
new file mode 100644
index 000000000..048656f48
--- /dev/null
+++ b/llvm/test/CodeGen/X86/fp128-cast-strict.ll
@@ -0,0 +1,975 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-linux-android -mattr=+sse | FileCheck %s --check-prefixes=X64,X64-SSE
+; RUN: llc < %s -mtriple=x86_64-linux-gnu -mattr=+sse | FileCheck %s --check-prefixes=X64,X64-SSE
+; RUN: llc < %s -mtriple=x86_64-linux-android -mattr=+avx | FileCheck %s --check-prefixes=X64,X64-AVX
+; RUN: llc < %s -mtriple=x86_64-linux-gnu -mattr=+avx | FileCheck %s --check-prefixes=X64,X64-AVX
+; RUN: llc < %s -mtriple=x86_64-linux-android -mattr=+avx512f | FileCheck %s --check-prefixes=X64,X64-AVX
+; RUN: llc < %s -mtriple=x86_64-linux-gnu -mattr=+avx512f | FileCheck %s --check-prefixes=X64,X64-AVX
+; RUN: llc < %s -mtriple=i686-linux-gnu -mattr=-sse | FileCheck %s --check-prefixes=X86
+
+; Check soft floating point conversion function calls.
+
+@vf32 = common global float 0.000000e+00, align 4
+@vf64 = common global double 0.000000e+00, align 8
+@vf80 = common global x86_fp80 0xK00000000000000000000, align 8
+@vf128 = common global fp128 0xL00000000000000000000000000000000, align 16
+
+define void @TestFPExtF32_F128() nounwind strictfp {
+; X64-SSE-LABEL: TestFPExtF32_F128:
+; X64-SSE:       # %bb.0: # %entry
+; X64-SSE-NEXT:    pushq %rax
+; X64-SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X64-SSE-NEXT:    callq __extendsftf2
+; X64-SSE-NEXT:    movaps %xmm0, {{.*}}(%rip)
+; X64-SSE-NEXT:    popq %rax
+; X64-SSE-NEXT:    retq
+;
+; X64-AVX-LABEL: TestFPExtF32_F128:
+; X64-AVX:       # %bb.0: # %entry
+; X64-AVX-NEXT:    pushq %rax
+; X64-AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X64-AVX-NEXT:    callq __extendsftf2
+; X64-AVX-NEXT:    vmovaps %xmm0, {{.*}}(%rip)
+; X64-AVX-NEXT:    popq %rax
+; X64-AVX-NEXT:    retq
+;
+; X86-LABEL: TestFPExtF32_F128:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $24, %esp
+; X86-NEXT:    flds vf32
+; X86-NEXT:    fstps {{[0-9]+}}(%esp)
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl %eax, (%esp)
+; X86-NEXT:    calll __extendsftf2
+; X86-NEXT:    subl $4, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl %esi, vf128+12
+; X86-NEXT:    movl %edx, vf128+8
+; X86-NEXT:    movl %ecx, vf128+4
+; X86-NEXT:    movl %eax, vf128
+; X86-NEXT:    addl $24, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
+entry:
+  %0 = load float, float* @vf32, align 4
+  %conv = call fp128 @llvm.experimental.constrained.fpext.f128.f32(float %0, metadata !"fpexcept.strict") #0
+  store fp128 %conv, fp128* @vf128, align 16
+  ret void
+}
+
+define void @TestFPExtF64_F128() nounwind strictfp {
+; X64-SSE-LABEL: TestFPExtF64_F128:
+; X64-SSE:       # %bb.0: # %entry
+; X64-SSE-NEXT:    pushq %rax
+; X64-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X64-SSE-NEXT:    callq __extenddftf2
+; X64-SSE-NEXT:    movaps %xmm0, {{.*}}(%rip)
+; X64-SSE-NEXT:    popq %rax
+; X64-SSE-NEXT:    retq
+;
+; X64-AVX-LABEL: TestFPExtF64_F128:
+; X64-AVX:       # %bb.0: # %entry
+; X64-AVX-NEXT:    pushq %rax
+; X64-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X64-AVX-NEXT:    callq __extenddftf2
+; X64-AVX-NEXT:    vmovaps %xmm0, {{.*}}(%rip)
+; X64-AVX-NEXT:    popq %rax
+; X64-AVX-NEXT:    retq
+;
+; X86-LABEL: TestFPExtF64_F128:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $40, %esp
+; X86-NEXT:    fldl vf64
+; X86-NEXT:    fstpl {{[0-9]+}}(%esp)
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl %eax, (%esp)
+; X86-NEXT:    calll __extenddftf2
+; X86-NEXT:    subl $4, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl %esi, vf128+12
+; X86-NEXT:    movl %edx, vf128+8
+; X86-NEXT:    movl %ecx, vf128+4
+; X86-NEXT:    movl %eax, vf128
+; X86-NEXT:    addl $40, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
+entry:
+  %0 = load double, double* @vf64, align 8
+  %conv = call fp128 @llvm.experimental.constrained.fpext.f128.f64(double %0, metadata !"fpexcept.strict") #0
+  store fp128 %conv, fp128* @vf128, align 16
+  ret void
+}
+
+define void @TestFPExtF80_F128() nounwind strictfp {
+; X64-SSE-LABEL: TestFPExtF80_F128:
+; X64-SSE:       # %bb.0: # %entry
+; X64-SSE-NEXT:    subq $24, %rsp
+; X64-SSE-NEXT:    fldt {{.*}}(%rip)
+; X64-SSE-NEXT:    fstpt (%rsp)
+; X64-SSE-NEXT:    callq __extendxftf2
+; X64-SSE-NEXT:    movaps %xmm0, {{.*}}(%rip)
+; X64-SSE-NEXT:    addq $24, %rsp
+; X64-SSE-NEXT:    retq
+;
+; X64-AVX-LABEL: TestFPExtF80_F128:
+; X64-AVX:       # %bb.0: # %entry
+; X64-AVX-NEXT:    subq $24, %rsp
+; X64-AVX-NEXT:    fldt {{.*}}(%rip)
+; X64-AVX-NEXT:    fstpt (%rsp)
+; X64-AVX-NEXT:    callq __extendxftf2
+; X64-AVX-NEXT:    vmovaps %xmm0, {{.*}}(%rip)
+; X64-AVX-NEXT:    addq $24, %rsp
+; X64-AVX-NEXT:    retq
+;
+; X86-LABEL: TestFPExtF80_F128:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $40, %esp
+; X86-NEXT:    fldt vf80
+; X86-NEXT:    fstpt {{[0-9]+}}(%esp)
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl %eax, (%esp)
+; X86-NEXT:    calll __extendxftf2
+; X86-NEXT:    subl $4, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl %esi, vf128+12
+; X86-NEXT:    movl %edx, vf128+8
+; X86-NEXT:    movl %ecx, vf128+4
+; X86-NEXT:    movl %eax, vf128
+; X86-NEXT:    addl $40, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
+entry:
+  %0 = load x86_fp80, x86_fp80* @vf80, align 8
+  %conv = call fp128 @llvm.experimental.constrained.fpext.f128.f80(x86_fp80 %0, metadata !"fpexcept.strict") #0
+  store fp128 %conv, fp128* @vf128, align 16
+  ret void
+}
+
+define void @TestFPTruncF128_F32() nounwind strictfp {
+; X64-SSE-LABEL: TestFPTruncF128_F32:
+; X64-SSE:       # %bb.0: # %entry
+; X64-SSE-NEXT:    pushq %rax
+; X64-SSE-NEXT:    movaps {{.*}}(%rip), %xmm0
+; X64-SSE-NEXT:    callq __trunctfsf2
+; X64-SSE-NEXT:    movss %xmm0, {{.*}}(%rip)
+; X64-SSE-NEXT:    popq %rax
+; X64-SSE-NEXT:    retq
+;
+; X64-AVX-LABEL: TestFPTruncF128_F32:
+; X64-AVX:       # %bb.0: # %entry
+; X64-AVX-NEXT:    pushq %rax
+; X64-AVX-NEXT:    vmovaps {{.*}}(%rip), %xmm0
+; X64-AVX-NEXT:    callq __trunctfsf2
+; X64-AVX-NEXT:    vmovss %xmm0, {{.*}}(%rip)
+; X64-AVX-NEXT:    popq %rax
+; X64-AVX-NEXT:    retq
+;
+; X86-LABEL: TestFPTruncF128_F32:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    pushl vf128+12
+; X86-NEXT:    pushl vf128+8
+; X86-NEXT:    pushl vf128+4
+; X86-NEXT:    pushl vf128
+; X86-NEXT:    calll __trunctfsf2
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    fstps vf32
+; X86-NEXT:    addl $12, %esp
+; X86-NEXT:    retl
+entry:
+  %0 = load fp128, fp128* @vf128, align 16
+  %conv = call float @llvm.experimental.constrained.fptrunc.f32.f128(fp128 %0, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  store float %conv, float* @vf32, align 4
+  ret void
+}
+
+define void @TestFPTruncF128_F64() nounwind strictfp {
+; X64-SSE-LABEL: TestFPTruncF128_F64:
+; X64-SSE:       # %bb.0: # %entry
+; X64-SSE-NEXT:    pushq %rax
+; X64-SSE-NEXT:    movaps {{.*}}(%rip), %xmm0
+; X64-SSE-NEXT:    callq __trunctfdf2
+; X64-SSE-NEXT:    movsd %xmm0, {{.*}}(%rip)
+; X64-SSE-NEXT:    popq %rax
+; X64-SSE-NEXT:    retq
+;
+; X64-AVX-LABEL: TestFPTruncF128_F64:
+; X64-AVX:       # %bb.0: # %entry
+; X64-AVX-NEXT:    pushq %rax
+; X64-AVX-NEXT:    vmovaps {{.*}}(%rip), %xmm0
+; X64-AVX-NEXT:    callq __trunctfdf2
+; X64-AVX-NEXT:    vmovsd %xmm0, {{.*}}(%rip)
+; X64-AVX-NEXT:    popq %rax
+; X64-AVX-NEXT:    retq
+;
+; X86-LABEL: TestFPTruncF128_F64:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    pushl vf128+12
+; X86-NEXT:    pushl vf128+8
+; X86-NEXT:    pushl vf128+4
+; X86-NEXT:    pushl vf128
+; X86-NEXT:    calll __trunctfdf2
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    fstpl vf64
+; X86-NEXT:    addl $12, %esp
+; X86-NEXT:    retl
+entry:
+  %0 = load fp128, fp128* @vf128, align 16
+  %conv = call double @llvm.experimental.constrained.fptrunc.f64.f128(fp128 %0, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  store double %conv, double* @vf64, align 8
+  ret void
+}
+
+define void @TestFPTruncF128_F80() nounwind strictfp {
+; X64-SSE-LABEL: TestFPTruncF128_F80:
+; X64-SSE:       # %bb.0: # %entry
+; X64-SSE-NEXT:    pushq %rax
+; X64-SSE-NEXT:    movaps {{.*}}(%rip), %xmm0
+; X64-SSE-NEXT:    callq __trunctfxf2
+; X64-SSE-NEXT:    fstpt {{.*}}(%rip)
+; X64-SSE-NEXT:    popq %rax
+; X64-SSE-NEXT:    retq
+;
+; X64-AVX-LABEL: TestFPTruncF128_F80:
+; X64-AVX:       # %bb.0: # %entry
+; X64-AVX-NEXT:    pushq %rax
+; X64-AVX-NEXT:    vmovaps {{.*}}(%rip), %xmm0
+; X64-AVX-NEXT:    callq __trunctfxf2
+; X64-AVX-NEXT:    fstpt {{.*}}(%rip)
+; X64-AVX-NEXT:    popq %rax
+; X64-AVX-NEXT:    retq
+;
+; X86-LABEL: TestFPTruncF128_F80:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    pushl vf128+12
+; X86-NEXT:    pushl vf128+8
+; X86-NEXT:    pushl vf128+4
+; X86-NEXT:    pushl vf128
+; X86-NEXT:    calll __trunctfxf2
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    fstpt vf80
+; X86-NEXT:    addl $12, %esp
+; X86-NEXT:    retl
+entry:
+  %0 = load fp128, fp128* @vf128, align 16
+  %conv = call x86_fp80 @llvm.experimental.constrained.fptrunc.f80.f128(fp128 %0, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  store x86_fp80 %conv, x86_fp80* @vf80, align 8
+  ret void
+}
+
+define i8 @fptosi_i8(fp128 %x) nounwind strictfp {
+; X64-LABEL: fptosi_i8:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    callq __fixtfsi
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+;
+; X86-LABEL: fptosi_i8:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll __fixtfsi
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    addl $12, %esp
+; X86-NEXT:    retl
+entry:
+  %conv = call i8 @llvm.experimental.constrained.fptosi.i8.f128(fp128 %x, metadata !"fpexcept.strict") #0
+  ret i8 %conv
+}
+
+define i16 @fptosi_i16(fp128 %x) nounwind strictfp {
+; X64-LABEL: fptosi_i16:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    callq __fixtfsi
+; X64-NEXT:    # kill: def $ax killed $ax killed $eax
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+;
+; X86-LABEL: fptosi_i16:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll __fixtfsi
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    # kill: def $ax killed $ax killed $eax
+; X86-NEXT:    addl $12, %esp
+; X86-NEXT:    retl
+entry:
+  %conv = call i16 @llvm.experimental.constrained.fptosi.i16.f128(fp128 %x, metadata !"fpexcept.strict") #0
+  ret i16 %conv
+}
+
+define i32 @fptosi_i32(fp128 %x) nounwind strictfp {
+; X64-LABEL: fptosi_i32:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    callq __fixtfsi
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+;
+; X86-LABEL: fptosi_i32:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll __fixtfsi
+; X86-NEXT:    addl $28, %esp
+; X86-NEXT:    retl
+entry:
+  %conv = call i32 @llvm.experimental.constrained.fptosi.i32.f128(fp128 %x, metadata !"fpexcept.strict") #0
+  ret i32 %conv
+}
+
+define i64 @fptosi_i64(fp128 %x) nounwind strictfp {
+; X64-LABEL: fptosi_i64:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    callq __fixtfdi
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+;
+; X86-LABEL: fptosi_i64:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll __fixtfdi
+; X86-NEXT:    addl $28, %esp
+; X86-NEXT:    retl
+entry:
+  %conv = call i64 @llvm.experimental.constrained.fptosi.i64.f128(fp128 %x, metadata !"fpexcept.strict") #0
+  ret i64 %conv
+}
+
+define i128 @fptosi_i128(fp128 %x) nounwind strictfp {
+; X64-LABEL: fptosi_i128:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    callq __fixtfti
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+;
+; X86-LABEL: fptosi_i128:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $20, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    calll __fixtfti
+; X86-NEXT:    addl $28, %esp
+; X86-NEXT:    movl (%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl %edi, 8(%esi)
+; X86-NEXT:    movl %edx, 12(%esi)
+; X86-NEXT:    movl %eax, (%esi)
+; X86-NEXT:    movl %ecx, 4(%esi)
+; X86-NEXT:    movl %esi, %eax
+; X86-NEXT:    addl $20, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    retl $4
+entry:
+  %conv = call i128 @llvm.experimental.constrained.fptosi.i128.f128(fp128 %x, metadata !"fpexcept.strict") #0
+  ret i128 %conv
+}
+
+define i8 @fptoui_i8(fp128 %x) nounwind strictfp {
+; X64-LABEL: fptoui_i8:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    callq __fixtfsi
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+;
+; X86-LABEL: fptoui_i8:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll __fixunstfsi
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    addl $12, %esp
+; X86-NEXT:    retl
+entry:
+  %conv = call i8 @llvm.experimental.constrained.fptoui.i8.f128(fp128 %x, metadata !"fpexcept.strict") #0
+  ret i8 %conv
+}
+
+define i16 @fptoui_i16(fp128 %x) nounwind strictfp {
+; X64-LABEL: fptoui_i16:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    callq __fixtfsi
+; X64-NEXT:    # kill: def $ax killed $ax killed $eax
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+;
+; X86-LABEL: fptoui_i16:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll __fixunstfsi
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    # kill: def $ax killed $ax killed $eax
+; X86-NEXT:    addl $12, %esp
+; X86-NEXT:    retl
+entry:
+  %conv = call i16 @llvm.experimental.constrained.fptoui.i16.f128(fp128 %x, metadata !"fpexcept.strict") #0
+  ret i16 %conv
+}
+
+define i32 @fptoui_i32(fp128 %x) nounwind strictfp {
+; X64-LABEL: fptoui_i32:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    callq __fixunstfsi
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+;
+; X86-LABEL: fptoui_i32:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll __fixunstfsi
+; X86-NEXT:    addl $28, %esp
+; X86-NEXT:    retl
+entry:
+  %conv = call i32 @llvm.experimental.constrained.fptoui.i32.f128(fp128 %x, metadata !"fpexcept.strict") #0
+  ret i32 %conv
+}
+
+define i64 @fptoui_i64(fp128 %x) nounwind strictfp {
+; X64-LABEL: fptoui_i64:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    callq __fixunstfdi
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+;
+; X86-LABEL: fptoui_i64:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll __fixunstfdi
+; X86-NEXT:    addl $28, %esp
+; X86-NEXT:    retl
+entry:
+  %conv = call i64 @llvm.experimental.constrained.fptoui.i64.f128(fp128 %x, metadata !"fpexcept.strict") #0
+  ret i64 %conv
+}
+
+define i128 @fptoui_i128(fp128 %x) nounwind strictfp {
+; X64-LABEL: fptoui_i128:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    callq __fixunstfti
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+;
+; X86-LABEL: fptoui_i128:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $20, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    calll __fixunstfti
+; X86-NEXT:    addl $28, %esp
+; X86-NEXT:    movl (%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl %edi, 8(%esi)
+; X86-NEXT:    movl %edx, 12(%esi)
+; X86-NEXT:    movl %eax, (%esi)
+; X86-NEXT:    movl %ecx, 4(%esi)
+; X86-NEXT:    movl %esi, %eax
+; X86-NEXT:    addl $20, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    retl $4
+entry:
+  %conv = call i128 @llvm.experimental.constrained.fptoui.i128.f128(fp128 %x, metadata !"fpexcept.strict") #0
+  ret i128 %conv
+}
+
+define fp128 @sitofp_i8(i8 %x) nounwind strictfp {
+; X64-LABEL: sitofp_i8:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movsbl %dil, %edi
+; X64-NEXT:    callq __floatsitf
+; X64-NEXT:    popq %rax
+; X64-NEXT:    retq
+;
+; X86-LABEL: sitofp_i8:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $20, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movsbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    subl $8, %esp
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    pushl %ecx
+; X86-NEXT:    calll __floatsitf
+; X86-NEXT:    addl $12, %esp
+; X86-NEXT:    movl (%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl %edi, 8(%esi)
+; X86-NEXT:    movl %edx, 12(%esi)
+; X86-NEXT:    movl %eax, (%esi)
+; X86-NEXT:    movl %ecx, 4(%esi)
+; X86-NEXT:    movl %esi, %eax
+; X86-NEXT:    addl $20, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    retl $4
+entry:
+  %conv = call fp128 @llvm.experimental.constrained.sitofp.f128.i8(i8 %x, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret fp128 %conv
+}
+
+define fp128 @sitofp_i16(i16 %x) nounwind strictfp {
+; X64-LABEL: sitofp_i16:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movswl %di, %edi
+; X64-NEXT:    callq __floatsitf
+; X64-NEXT:    popq %rax
+; X64-NEXT:    retq
+;
+; X86-LABEL: sitofp_i16:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $20, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movswl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    subl $8, %esp
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    pushl %ecx
+; X86-NEXT:    calll __floatsitf
+; X86-NEXT:    addl $12, %esp
+; X86-NEXT:    movl (%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl %edi, 8(%esi)
+; X86-NEXT:    movl %edx, 12(%esi)
+; X86-NEXT:    movl %eax, (%esi)
+; X86-NEXT:    movl %ecx, 4(%esi)
+; X86-NEXT:    movl %esi, %eax
+; X86-NEXT:    addl $20, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    retl $4
+entry:
+  %conv = call fp128 @llvm.experimental.constrained.sitofp.f128.i16(i16 %x, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret fp128 %conv
+}
+
+define fp128 @sitofp_i32(i32 %x) nounwind strictfp {
+; X64-LABEL: sitofp_i32:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    callq __floatsitf
+; X64-NEXT:    popq %rax
+; X64-NEXT:    retq
+;
+; X86-LABEL: sitofp_i32:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $20, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    subl $8, %esp
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    calll __floatsitf
+; X86-NEXT:    addl $12, %esp
+; X86-NEXT:    movl (%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl %edi, 8(%esi)
+; X86-NEXT:    movl %edx, 12(%esi)
+; X86-NEXT:    movl %eax, (%esi)
+; X86-NEXT:    movl %ecx, 4(%esi)
+; X86-NEXT:    movl %esi, %eax
+; X86-NEXT:    addl $20, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    retl $4
+entry:
+  %conv = call fp128 @llvm.experimental.constrained.sitofp.f128.i32(i32 %x, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret fp128 %conv
+}
+
+define fp128 @sitofp_i64(i64 %x) nounwind strictfp {
+; X64-LABEL: sitofp_i64:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    callq __floatditf
+; X64-NEXT:    popq %rax
+; X64-NEXT:    retq
+;
+; X86-LABEL: sitofp_i64:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $20, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    subl $4, %esp
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    calll __floatditf
+; X86-NEXT:    addl $12, %esp
+; X86-NEXT:    movl (%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl %edi, 8(%esi)
+; X86-NEXT:    movl %edx, 12(%esi)
+; X86-NEXT:    movl %eax, (%esi)
+; X86-NEXT:    movl %ecx, 4(%esi)
+; X86-NEXT:    movl %esi, %eax
+; X86-NEXT:    addl $20, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    retl $4
+entry:
+  %conv = call fp128 @llvm.experimental.constrained.sitofp.f128.i64(i64 %x, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret fp128 %conv
+}
+
+define fp128 @sitofp_i128(i128 %x) nounwind strictfp {
+; X64-LABEL: sitofp_i128:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    callq __floattitf
+; X64-NEXT:    popq %rax
+; X64-NEXT:    retq
+;
+; X86-LABEL: sitofp_i128:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $20, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    calll __floattitf
+; X86-NEXT:    addl $28, %esp
+; X86-NEXT:    movl (%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl %edi, 8(%esi)
+; X86-NEXT:    movl %edx, 12(%esi)
+; X86-NEXT:    movl %eax, (%esi)
+; X86-NEXT:    movl %ecx, 4(%esi)
+; X86-NEXT:    movl %esi, %eax
+; X86-NEXT:    addl $20, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    retl $4
+entry:
+  %conv = call fp128 @llvm.experimental.constrained.sitofp.f128.i128(i128 %x, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret fp128 %conv
+}
+
+define fp128 @uitofp_i8(i8 %x) nounwind strictfp {
+; X64-LABEL: uitofp_i8:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movzbl %dil, %edi
+; X64-NEXT:    callq __floatsitf
+; X64-NEXT:    popq %rax
+; X64-NEXT:    retq
+;
+; X86-LABEL: uitofp_i8:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $20, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    subl $8, %esp
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    pushl %ecx
+; X86-NEXT:    calll __floatunsitf
+; X86-NEXT:    addl $12, %esp
+; X86-NEXT:    movl (%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl %edi, 8(%esi)
+; X86-NEXT:    movl %edx, 12(%esi)
+; X86-NEXT:    movl %eax, (%esi)
+; X86-NEXT:    movl %ecx, 4(%esi)
+; X86-NEXT:    movl %esi, %eax
+; X86-NEXT:    addl $20, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    retl $4
+entry:
+  %conv = call fp128 @llvm.experimental.constrained.uitofp.f128.i8(i8 %x, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret fp128 %conv
+}
+
+define fp128 @uitofp_i16(i16 %x) nounwind strictfp {
+; X64-LABEL: uitofp_i16:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movzwl %di, %edi
+; X64-NEXT:    callq __floatsitf
+; X64-NEXT:    popq %rax
+; X64-NEXT:    retq
+;
+; X86-LABEL: uitofp_i16:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $20, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    subl $8, %esp
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    pushl %ecx
+; X86-NEXT:    calll __floatunsitf
+; X86-NEXT:    addl $12, %esp
+; X86-NEXT:    movl (%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl %edi, 8(%esi)
+; X86-NEXT:    movl %edx, 12(%esi)
+; X86-NEXT:    movl %eax, (%esi)
+; X86-NEXT:    movl %ecx, 4(%esi)
+; X86-NEXT:    movl %esi, %eax
+; X86-NEXT:    addl $20, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    retl $4
+entry:
+  %conv = call fp128 @llvm.experimental.constrained.uitofp.f128.i16(i16 %x, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret fp128 %conv
+}
+
+define fp128 @uitofp_i32(i32 %x) nounwind strictfp {
+; X64-LABEL: uitofp_i32:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    callq __floatunsitf
+; X64-NEXT:    popq %rax
+; X64-NEXT:    retq
+;
+; X86-LABEL: uitofp_i32:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $20, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    subl $8, %esp
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    calll __floatunsitf
+; X86-NEXT:    addl $12, %esp
+; X86-NEXT:    movl (%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl %edi, 8(%esi)
+; X86-NEXT:    movl %edx, 12(%esi)
+; X86-NEXT:    movl %eax, (%esi)
+; X86-NEXT:    movl %ecx, 4(%esi)
+; X86-NEXT:    movl %esi, %eax
+; X86-NEXT:    addl $20, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    retl $4
+entry:
+  %conv = call fp128 @llvm.experimental.constrained.uitofp.f128.i32(i32 %x, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret fp128 %conv
+}
+
+define fp128 @uitofp_i64(i64 %x) nounwind strictfp {
+; X64-LABEL: uitofp_i64:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    callq __floatunditf
+; X64-NEXT:    popq %rax
+; X64-NEXT:    retq
+;
+; X86-LABEL: uitofp_i64:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $20, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    subl $4, %esp
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    calll __floatunditf
+; X86-NEXT:    addl $12, %esp
+; X86-NEXT:    movl (%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl %edi, 8(%esi)
+; X86-NEXT:    movl %edx, 12(%esi)
+; X86-NEXT:    movl %eax, (%esi)
+; X86-NEXT:    movl %ecx, 4(%esi)
+; X86-NEXT:    movl %esi, %eax
+; X86-NEXT:    addl $20, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    retl $4
+entry:
+  %conv = call fp128 @llvm.experimental.constrained.uitofp.f128.i64(i64 %x, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret fp128 %conv
+}
+
+define fp128 @uitofp_i128(i128 %x) nounwind strictfp {
+; X64-LABEL: uitofp_i128:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    callq __floatuntitf
+; X64-NEXT:    popq %rax
+; X64-NEXT:    retq
+;
+; X86-LABEL: uitofp_i128:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $20, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    calll __floatuntitf
+; X86-NEXT:    addl $28, %esp
+; X86-NEXT:    movl (%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl %edi, 8(%esi)
+; X86-NEXT:    movl %edx, 12(%esi)
+; X86-NEXT:    movl %eax, (%esi)
+; X86-NEXT:    movl %ecx, 4(%esi)
+; X86-NEXT:    movl %esi, %eax
+; X86-NEXT:    addl $20, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    retl $4
+entry:
+  %conv = call fp128 @llvm.experimental.constrained.uitofp.f128.i128(i128 %x, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret fp128 %conv
+}
+
+attributes #0 = { strictfp }
+
+declare float @llvm.experimental.constrained.fptrunc.f32.f128(fp128, metadata, metadata)
+declare double @llvm.experimental.constrained.fptrunc.f64.f128(fp128, metadata, metadata)
+declare x86_fp80 @llvm.experimental.constrained.fptrunc.f80.f128(fp128, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.fpext.f128.f32(float, metadata)
+declare fp128 @llvm.experimental.constrained.fpext.f128.f64(double, metadata)
+declare fp128 @llvm.experimental.constrained.fpext.f128.f80(x86_fp80, metadata)
+declare i8 @llvm.experimental.constrained.fptosi.i8.f128(fp128, metadata)
+declare i16 @llvm.experimental.constrained.fptosi.i16.f128(fp128, metadata)
+declare i32 @llvm.experimental.constrained.fptosi.i32.f128(fp128, metadata)
+declare i64 @llvm.experimental.constrained.fptosi.i64.f128(fp128, metadata)
+declare i128 @llvm.experimental.constrained.fptosi.i128.f128(fp128, metadata)
+declare i8 @llvm.experimental.constrained.fptoui.i8.f128(fp128, metadata)
+declare i16 @llvm.experimental.constrained.fptoui.i16.f128(fp128, metadata)
+declare i32 @llvm.experimental.constrained.fptoui.i32.f128(fp128, metadata)
+declare i64 @llvm.experimental.constrained.fptoui.i64.f128(fp128, metadata)
+declare i128 @llvm.experimental.constrained.fptoui.i128.f128(fp128, metadata)
+declare fp128 @llvm.experimental.constrained.sitofp.f128.i8(i8, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.sitofp.f128.i16(i16, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.sitofp.f128.i32(i32, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.sitofp.f128.i64(i64, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.sitofp.f128.i128(i128, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.uitofp.f128.i8(i8, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.uitofp.f128.i16(i16, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.uitofp.f128.i32(i32, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.uitofp.f128.i64(i64, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.uitofp.f128.i128(i128, metadata, metadata)
diff --git a/llvm/test/CodeGen/X86/fp128-cast.ll b/llvm/test/CodeGen/X86/fp128-cast.ll
index f58bee3e1..8878273ff 100644
--- a/llvm/test/CodeGen/X86/fp128-cast.ll
+++ b/llvm/test/CodeGen/X86/fp128-cast.ll
@@ -1,39 +1,35 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -O2 -mtriple=x86_64-linux-android -mattr=+mmx | FileCheck %s --check-prefix=X64
-; RUN: llc < %s -O2 -mtriple=x86_64-linux-gnu -mattr=+mmx | FileCheck %s --check-prefix=X64
-; RUN: llc < %s -O2 -mtriple=x86_64-linux-gnu -mattr=-mmx | FileCheck %s --check-prefix=X64_NO_MMX
+; RUN: llc < %s -O2 -mtriple=x86_64-linux-android -mattr=+sse | FileCheck %s --check-prefixes=X64,X64-SSE
+; RUN: llc < %s -O2 -mtriple=x86_64-linux-gnu -mattr=+sse | FileCheck %s --check-prefixes=X64,X64-SSE
 ; RUN: llc < %s -O2 -mtriple=i686-linux-gnu -mattr=+mmx | FileCheck %s --check-prefix=X32
+; RUN: llc < %s -O2 -mtriple=x86_64-linux-android -mattr=+avx | FileCheck %s --check-prefixes=X64,X64-AVX
+; RUN: llc < %s -O2 -mtriple=x86_64-linux-gnu -mattr=+avx | FileCheck %s --check-prefixes=X64,X64-AVX
+; RUN: llc < %s -O2 -mtriple=x86_64-linux-android -mattr=+avx512f | FileCheck %s --check-prefixes=X64,X64-AVX
+; RUN: llc < %s -O2 -mtriple=x86_64-linux-gnu -mattr=+avx512f | FileCheck %s --check-prefixes=X64,X64-AVX
 
 ; Check soft floating point conversion function calls.
 
+@vi16 = common global i16 0, align 2
 @vi32 = common global i32 0, align 4
 @vi64 = common global i64 0, align 8
+@vi128 = common global i128 0, align 16
 @vu32 = common global i32 0, align 4
 @vu64 = common global i64 0, align 8
+@vu128 = common global i128 0, align 16
 @vf32 = common global float 0.000000e+00, align 4
 @vf64 = common global double 0.000000e+00, align 8
 @vf80 = common global x86_fp80 0xK00000000000000000000, align 8
 @vf128 = common global fp128 0xL00000000000000000000000000000000, align 16
 
 define void @TestFPExtF32_F128() nounwind {
-; X64-LABEL: TestFPExtF32_F128:
-; X64:       # %bb.0: # %entry
-; X64-NEXT:    pushq %rax
-; X64-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X64-NEXT:    callq __extendsftf2
-; X64-NEXT:    movaps %xmm0, {{.*}}(%rip)
-; X64-NEXT:    popq %rax
-; X64-NEXT:    retq
-;
-; X64_NO_MMX-LABEL: TestFPExtF32_F128:
-; X64_NO_MMX:       # %bb.0: # %entry
-; X64_NO_MMX-NEXT:    pushq %rax
-; X64_NO_MMX-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X64_NO_MMX-NEXT:    callq __extendsftf2
-; X64_NO_MMX-NEXT:    movq %rdx, vf128+{{.*}}(%rip)
-; X64_NO_MMX-NEXT:    movq %rax, {{.*}}(%rip)
-; X64_NO_MMX-NEXT:    popq %rax
-; X64_NO_MMX-NEXT:    retq
+; X64-SSE-LABEL: TestFPExtF32_F128:
+; X64-SSE:       # %bb.0: # %entry
+; X64-SSE-NEXT:    pushq %rax
+; X64-SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X64-SSE-NEXT:    callq __extendsftf2
+; X64-SSE-NEXT:    movaps %xmm0, {{.*}}(%rip)
+; X64-SSE-NEXT:    popq %rax
+; X64-SSE-NEXT:    retq
 ;
 ; X32-LABEL: TestFPExtF32_F128:
 ; X32:       # %bb.0: # %entry
@@ -56,6 +52,15 @@ define void @TestFPExtF32_F128() nounwind {
 ; X32-NEXT:    addl $24, %esp
 ; X32-NEXT:    popl %esi
 ; X32-NEXT:    retl
+;
+; X64-AVX-LABEL: TestFPExtF32_F128:
+; X64-AVX:       # %bb.0: # %entry
+; X64-AVX-NEXT:    pushq %rax
+; X64-AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X64-AVX-NEXT:    callq __extendsftf2
+; X64-AVX-NEXT:    vmovaps %xmm0, {{.*}}(%rip)
+; X64-AVX-NEXT:    popq %rax
+; X64-AVX-NEXT:    retq
 entry:
   %0 = load float, float* @vf32, align 4
   %conv = fpext float %0 to fp128
@@ -64,24 +69,14 @@ entry:
 }
 
 define void @TestFPExtF64_F128() nounwind {
-; X64-LABEL: TestFPExtF64_F128:
-; X64:       # %bb.0: # %entry
-; X64-NEXT:    pushq %rax
-; X64-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; X64-NEXT:    callq __extenddftf2
-; X64-NEXT:    movaps %xmm0, {{.*}}(%rip)
-; X64-NEXT:    popq %rax
-; X64-NEXT:    retq
-;
-; X64_NO_MMX-LABEL: TestFPExtF64_F128:
-; X64_NO_MMX:       # %bb.0: # %entry
-; X64_NO_MMX-NEXT:    pushq %rax
-; X64_NO_MMX-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; X64_NO_MMX-NEXT:    callq __extenddftf2
-; X64_NO_MMX-NEXT:    movq %rdx, vf128+{{.*}}(%rip)
-; X64_NO_MMX-NEXT:    movq %rax, {{.*}}(%rip)
-; X64_NO_MMX-NEXT:    popq %rax
-; X64_NO_MMX-NEXT:    retq
+; X64-SSE-LABEL: TestFPExtF64_F128:
+; X64-SSE:       # %bb.0: # %entry
+; X64-SSE-NEXT:    pushq %rax
+; X64-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X64-SSE-NEXT:    callq __extenddftf2
+; X64-SSE-NEXT:    movaps %xmm0, {{.*}}(%rip)
+; X64-SSE-NEXT:    popq %rax
+; X64-SSE-NEXT:    retq
 ;
 ; X32-LABEL: TestFPExtF64_F128:
 ; X32:       # %bb.0: # %entry
@@ -104,6 +99,15 @@ define void @TestFPExtF64_F128() nounwind {
 ; X32-NEXT:    addl $40, %esp
 ; X32-NEXT:    popl %esi
 ; X32-NEXT:    retl
+;
+; X64-AVX-LABEL: TestFPExtF64_F128:
+; X64-AVX:       # %bb.0: # %entry
+; X64-AVX-NEXT:    pushq %rax
+; X64-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X64-AVX-NEXT:    callq __extenddftf2
+; X64-AVX-NEXT:    vmovaps %xmm0, {{.*}}(%rip)
+; X64-AVX-NEXT:    popq %rax
+; X64-AVX-NEXT:    retq
 entry:
   %0 = load double, double* @vf64, align 8
   %conv = fpext double %0 to fp128
@@ -112,26 +116,15 @@ entry:
 }
 
 define void @TestFPExtF80_F128() nounwind {
-; X64-LABEL: TestFPExtF80_F128:
-; X64:       # %bb.0: # %entry
-; X64-NEXT:    subq $24, %rsp
-; X64-NEXT:    fldt {{.*}}(%rip)
-; X64-NEXT:    fstpt (%rsp)
-; X64-NEXT:    callq __extendxftf2
-; X64-NEXT:    movaps %xmm0, {{.*}}(%rip)
-; X64-NEXT:    addq $24, %rsp
-; X64-NEXT:    retq
-;
-; X64_NO_MMX-LABEL: TestFPExtF80_F128:
-; X64_NO_MMX:       # %bb.0: # %entry
-; X64_NO_MMX-NEXT:    subq $24, %rsp
-; X64_NO_MMX-NEXT:    fldt {{.*}}(%rip)
-; X64_NO_MMX-NEXT:    fstpt (%rsp)
-; X64_NO_MMX-NEXT:    callq __extendxftf2
-; X64_NO_MMX-NEXT:    movq %rdx, vf128+{{.*}}(%rip)
-; X64_NO_MMX-NEXT:    movq %rax, {{.*}}(%rip)
-; X64_NO_MMX-NEXT:    addq $24, %rsp
-; X64_NO_MMX-NEXT:    retq
+; X64-SSE-LABEL: TestFPExtF80_F128:
+; X64-SSE:       # %bb.0: # %entry
+; X64-SSE-NEXT:    subq $24, %rsp
+; X64-SSE-NEXT:    fldt {{.*}}(%rip)
+; X64-SSE-NEXT:    fstpt (%rsp)
+; X64-SSE-NEXT:    callq __extendxftf2
+; X64-SSE-NEXT:    movaps %xmm0, {{.*}}(%rip)
+; X64-SSE-NEXT:    addq $24, %rsp
+; X64-SSE-NEXT:    retq
 ;
 ; X32-LABEL: TestFPExtF80_F128:
 ; X32:       # %bb.0: # %entry
@@ -154,6 +147,16 @@ define void @TestFPExtF80_F128() nounwind {
 ; X32-NEXT:    addl $40, %esp
 ; X32-NEXT:    popl %esi
 ; X32-NEXT:    retl
+;
+; X64-AVX-LABEL: TestFPExtF80_F128:
+; X64-AVX:       # %bb.0: # %entry
+; X64-AVX-NEXT:    subq $24, %rsp
+; X64-AVX-NEXT:    fldt {{.*}}(%rip)
+; X64-AVX-NEXT:    fstpt (%rsp)
+; X64-AVX-NEXT:    callq __extendxftf2
+; X64-AVX-NEXT:    vmovaps %xmm0, {{.*}}(%rip)
+; X64-AVX-NEXT:    addq $24, %rsp
+; X64-AVX-NEXT:    retq
 entry:
   %0 = load x86_fp80, x86_fp80* @vf80, align 8
   %conv = fpext x86_fp80 %0 to fp128
@@ -161,25 +164,91 @@ entry:
   ret void
 }
 
+define void @TestFPToSIF128_I16() nounwind {
+; X64-SSE-LABEL: TestFPToSIF128_I16:
+; X64-SSE:       # %bb.0: # %entry
+; X64-SSE-NEXT:    pushq %rax
+; X64-SSE-NEXT:    movaps {{.*}}(%rip), %xmm0
+; X64-SSE-NEXT:    callq __fixtfsi
+; X64-SSE-NEXT:    movw %ax, {{.*}}(%rip)
+; X64-SSE-NEXT:    popq %rax
+; X64-SSE-NEXT:    retq
+;
+; X32-LABEL: TestFPToSIF128_I16:
+; X32:       # %bb.0: # %entry
+; X32-NEXT:    subl $12, %esp
+; X32-NEXT:    pushl vf128+12
+; X32-NEXT:    pushl vf128+8
+; X32-NEXT:    pushl vf128+4
+; X32-NEXT:    pushl vf128
+; X32-NEXT:    calll __fixtfsi
+; X32-NEXT:    addl $16, %esp
+; X32-NEXT:    movw %ax, vi16
+; X32-NEXT:    addl $12, %esp
+; X32-NEXT:    retl
+;
+; X64-AVX-LABEL: TestFPToSIF128_I16:
+; X64-AVX:       # %bb.0: # %entry
+; X64-AVX-NEXT:    pushq %rax
+; X64-AVX-NEXT:    vmovaps {{.*}}(%rip), %xmm0
+; X64-AVX-NEXT:    callq __fixtfsi
+; X64-AVX-NEXT:    movw %ax, {{.*}}(%rip)
+; X64-AVX-NEXT:    popq %rax
+; X64-AVX-NEXT:    retq
+entry:
+  %0 = load fp128, fp128* @vf128, align 16
+  %conv = fptosi fp128 %0 to i16
+  store i16 %conv, i16* @vi16, align 2
+  ret void
+}
+
+define void @TestFPToUIF128_I16() nounwind {
+; X64-SSE-LABEL: TestFPToUIF128_I16:
+; X64-SSE:       # %bb.0: # %entry
+; X64-SSE-NEXT:    pushq %rax
+; X64-SSE-NEXT:    movaps {{.*}}(%rip), %xmm0
+; X64-SSE-NEXT:    callq __fixtfsi
+; X64-SSE-NEXT:    movw %ax, {{.*}}(%rip)
+; X64-SSE-NEXT:    popq %rax
+; X64-SSE-NEXT:    retq
+;
+; X32-LABEL: TestFPToUIF128_I16:
+; X32:       # %bb.0: # %entry
+; X32-NEXT:    subl $12, %esp
+; X32-NEXT:    pushl vf128+12
+; X32-NEXT:    pushl vf128+8
+; X32-NEXT:    pushl vf128+4
+; X32-NEXT:    pushl vf128
+; X32-NEXT:    calll __fixunstfsi
+; X32-NEXT:    addl $16, %esp
+; X32-NEXT:    movw %ax, vi16
+; X32-NEXT:    addl $12, %esp
+; X32-NEXT:    retl
+;
+; X64-AVX-LABEL: TestFPToUIF128_I16:
+; X64-AVX:       # %bb.0: # %entry
+; X64-AVX-NEXT:    pushq %rax
+; X64-AVX-NEXT:    vmovaps {{.*}}(%rip), %xmm0
+; X64-AVX-NEXT:    callq __fixtfsi
+; X64-AVX-NEXT:    movw %ax, {{.*}}(%rip)
+; X64-AVX-NEXT:    popq %rax
+; X64-AVX-NEXT:    retq
+entry:
+  %0 = load fp128, fp128* @vf128, align 16
+  %conv = fptoui fp128 %0 to i16
+  store i16 %conv, i16* @vi16, align 2
+  ret void
+}
+
 define void @TestFPToSIF128_I32() nounwind {
-; X64-LABEL: TestFPToSIF128_I32:
-; X64:       # %bb.0: # %entry
-; X64-NEXT:    pushq %rax
-; X64-NEXT:    movaps {{.*}}(%rip), %xmm0
-; X64-NEXT:    callq __fixtfsi
-; X64-NEXT:    movl %eax, {{.*}}(%rip)
-; X64-NEXT:    popq %rax
-; X64-NEXT:    retq
-;
-; X64_NO_MMX-LABEL: TestFPToSIF128_I32:
-; X64_NO_MMX:       # %bb.0: # %entry
-; X64_NO_MMX-NEXT:    pushq %rax
-; X64_NO_MMX-NEXT:    movq {{.*}}(%rip), %rdi
-; X64_NO_MMX-NEXT:    movq vf128+{{.*}}(%rip), %rsi
-; X64_NO_MMX-NEXT:    callq __fixtfsi
-; X64_NO_MMX-NEXT:    movl %eax, {{.*}}(%rip)
-; X64_NO_MMX-NEXT:    popq %rax
-; X64_NO_MMX-NEXT:    retq
+; X64-SSE-LABEL: TestFPToSIF128_I32:
+; X64-SSE:       # %bb.0: # %entry
+; X64-SSE-NEXT:    pushq %rax
+; X64-SSE-NEXT:    movaps {{.*}}(%rip), %xmm0
+; X64-SSE-NEXT:    callq __fixtfsi
+; X64-SSE-NEXT:    movl %eax, {{.*}}(%rip)
+; X64-SSE-NEXT:    popq %rax
+; X64-SSE-NEXT:    retq
 ;
 ; X32-LABEL: TestFPToSIF128_I32:
 ; X32:       # %bb.0: # %entry
@@ -193,6 +262,15 @@ define void @TestFPToSIF128_I32() nounwind {
 ; X32-NEXT:    movl %eax, vi32
 ; X32-NEXT:    addl $12, %esp
 ; X32-NEXT:    retl
+;
+; X64-AVX-LABEL: TestFPToSIF128_I32:
+; X64-AVX:       # %bb.0: # %entry
+; X64-AVX-NEXT:    pushq %rax
+; X64-AVX-NEXT:    vmovaps {{.*}}(%rip), %xmm0
+; X64-AVX-NEXT:    callq __fixtfsi
+; X64-AVX-NEXT:    movl %eax, {{.*}}(%rip)
+; X64-AVX-NEXT:    popq %rax
+; X64-AVX-NEXT:    retq
 entry:
   %0 = load fp128, fp128* @vf128, align 16
   %conv = fptosi fp128 %0 to i32
@@ -201,24 +279,14 @@ entry:
 }
 
 define void @TestFPToUIF128_U32() nounwind {
-; X64-LABEL: TestFPToUIF128_U32:
-; X64:       # %bb.0: # %entry
-; X64-NEXT:    pushq %rax
-; X64-NEXT:    movaps {{.*}}(%rip), %xmm0
-; X64-NEXT:    callq __fixunstfsi
-; X64-NEXT:    movl %eax, {{.*}}(%rip)
-; X64-NEXT:    popq %rax
-; X64-NEXT:    retq
-;
-; X64_NO_MMX-LABEL: TestFPToUIF128_U32:
-; X64_NO_MMX:       # %bb.0: # %entry
-; X64_NO_MMX-NEXT:    pushq %rax
-; X64_NO_MMX-NEXT:    movq {{.*}}(%rip), %rdi
-; X64_NO_MMX-NEXT:    movq vf128+{{.*}}(%rip), %rsi
-; X64_NO_MMX-NEXT:    callq __fixunstfsi
-; X64_NO_MMX-NEXT:    movl %eax, {{.*}}(%rip)
-; X64_NO_MMX-NEXT:    popq %rax
-; X64_NO_MMX-NEXT:    retq
+; X64-SSE-LABEL: TestFPToUIF128_U32:
+; X64-SSE:       # %bb.0: # %entry
+; X64-SSE-NEXT:    pushq %rax
+; X64-SSE-NEXT:    movaps {{.*}}(%rip), %xmm0
+; X64-SSE-NEXT:    callq __fixunstfsi
+; X64-SSE-NEXT:    movl %eax, {{.*}}(%rip)
+; X64-SSE-NEXT:    popq %rax
+; X64-SSE-NEXT:    retq
 ;
 ; X32-LABEL: TestFPToUIF128_U32:
 ; X32:       # %bb.0: # %entry
@@ -232,6 +300,15 @@ define void @TestFPToUIF128_U32() nounwind {
 ; X32-NEXT:    movl %eax, vu32
 ; X32-NEXT:    addl $12, %esp
 ; X32-NEXT:    retl
+;
+; X64-AVX-LABEL: TestFPToUIF128_U32:
+; X64-AVX:       # %bb.0: # %entry
+; X64-AVX-NEXT:    pushq %rax
+; X64-AVX-NEXT:    vmovaps {{.*}}(%rip), %xmm0
+; X64-AVX-NEXT:    callq __fixunstfsi
+; X64-AVX-NEXT:    movl %eax, {{.*}}(%rip)
+; X64-AVX-NEXT:    popq %rax
+; X64-AVX-NEXT:    retq
 entry:
   %0 = load fp128, fp128* @vf128, align 16
   %conv = fptoui fp128 %0 to i32
@@ -240,26 +317,15 @@ entry:
 }
 
 define void @TestFPToSIF128_I64() nounwind {
-; X64-LABEL: TestFPToSIF128_I64:
-; X64:       # %bb.0: # %entry
-; X64-NEXT:    pushq %rax
-; X64-NEXT:    movaps {{.*}}(%rip), %xmm0
-; X64-NEXT:    callq __fixtfsi
-; X64-NEXT:    cltq
-; X64-NEXT:    movq %rax, {{.*}}(%rip)
-; X64-NEXT:    popq %rax
-; X64-NEXT:    retq
-;
-; X64_NO_MMX-LABEL: TestFPToSIF128_I64:
-; X64_NO_MMX:       # %bb.0: # %entry
-; X64_NO_MMX-NEXT:    pushq %rax
-; X64_NO_MMX-NEXT:    movq {{.*}}(%rip), %rdi
-; X64_NO_MMX-NEXT:    movq vf128+{{.*}}(%rip), %rsi
-; X64_NO_MMX-NEXT:    callq __fixtfsi
-; X64_NO_MMX-NEXT:    cltq
-; X64_NO_MMX-NEXT:    movq %rax, {{.*}}(%rip)
-; X64_NO_MMX-NEXT:    popq %rax
-; X64_NO_MMX-NEXT:    retq
+; X64-SSE-LABEL: TestFPToSIF128_I64:
+; X64-SSE:       # %bb.0: # %entry
+; X64-SSE-NEXT:    pushq %rax
+; X64-SSE-NEXT:    movaps {{.*}}(%rip), %xmm0
+; X64-SSE-NEXT:    callq __fixtfsi
+; X64-SSE-NEXT:    cltq
+; X64-SSE-NEXT:    movq %rax, {{.*}}(%rip)
+; X64-SSE-NEXT:    popq %rax
+; X64-SSE-NEXT:    retq
 ;
 ; X32-LABEL: TestFPToSIF128_I64:
 ; X32:       # %bb.0: # %entry
@@ -275,6 +341,16 @@ define void @TestFPToSIF128_I64() nounwind {
 ; X32-NEXT:    movl %eax, vi64+4
 ; X32-NEXT:    addl $12, %esp
 ; X32-NEXT:    retl
+;
+; X64-AVX-LABEL: TestFPToSIF128_I64:
+; X64-AVX:       # %bb.0: # %entry
+; X64-AVX-NEXT:    pushq %rax
+; X64-AVX-NEXT:    vmovaps {{.*}}(%rip), %xmm0
+; X64-AVX-NEXT:    callq __fixtfsi
+; X64-AVX-NEXT:    cltq
+; X64-AVX-NEXT:    movq %rax, {{.*}}(%rip)
+; X64-AVX-NEXT:    popq %rax
+; X64-AVX-NEXT:    retq
 entry:
   %0 = load fp128, fp128* @vf128, align 16
   %conv = fptosi fp128 %0 to i32
@@ -284,26 +360,15 @@ entry:
 }
 
 define void @TestFPToUIF128_U64() nounwind {
-; X64-LABEL: TestFPToUIF128_U64:
-; X64:       # %bb.0: # %entry
-; X64-NEXT:    pushq %rax
-; X64-NEXT:    movaps {{.*}}(%rip), %xmm0
-; X64-NEXT:    callq __fixunstfsi
-; X64-NEXT:    movl %eax, %eax
-; X64-NEXT:    movq %rax, {{.*}}(%rip)
-; X64-NEXT:    popq %rax
-; X64-NEXT:    retq
-;
-; X64_NO_MMX-LABEL: TestFPToUIF128_U64:
-; X64_NO_MMX:       # %bb.0: # %entry
-; X64_NO_MMX-NEXT:    pushq %rax
-; X64_NO_MMX-NEXT:    movq {{.*}}(%rip), %rdi
-; X64_NO_MMX-NEXT:    movq vf128+{{.*}}(%rip), %rsi
-; X64_NO_MMX-NEXT:    callq __fixunstfsi
-; X64_NO_MMX-NEXT:    movl %eax, %eax
-; X64_NO_MMX-NEXT:    movq %rax, {{.*}}(%rip)
-; X64_NO_MMX-NEXT:    popq %rax
-; X64_NO_MMX-NEXT:    retq
+; X64-SSE-LABEL: TestFPToUIF128_U64:
+; X64-SSE:       # %bb.0: # %entry
+; X64-SSE-NEXT:    pushq %rax
+; X64-SSE-NEXT:    movaps {{.*}}(%rip), %xmm0
+; X64-SSE-NEXT:    callq __fixunstfsi
+; X64-SSE-NEXT:    movl %eax, %eax
+; X64-SSE-NEXT:    movq %rax, {{.*}}(%rip)
+; X64-SSE-NEXT:    popq %rax
+; X64-SSE-NEXT:    retq
 ;
 ; X32-LABEL: TestFPToUIF128_U64:
 ; X32:       # %bb.0: # %entry
@@ -318,6 +383,16 @@ define void @TestFPToUIF128_U64() nounwind {
 ; X32-NEXT:    movl $0, vu64+4
 ; X32-NEXT:    addl $12, %esp
 ; X32-NEXT:    retl
+;
+; X64-AVX-LABEL: TestFPToUIF128_U64:
+; X64-AVX:       # %bb.0: # %entry
+; X64-AVX-NEXT:    pushq %rax
+; X64-AVX-NEXT:    vmovaps {{.*}}(%rip), %xmm0
+; X64-AVX-NEXT:    callq __fixunstfsi
+; X64-AVX-NEXT:    movl %eax, %eax
+; X64-AVX-NEXT:    movq %rax, {{.*}}(%rip)
+; X64-AVX-NEXT:    popq %rax
+; X64-AVX-NEXT:    retq
 entry:
   %0 = load fp128, fp128* @vf128, align 16
   %conv = fptoui fp128 %0 to i32
@@ -326,25 +401,117 @@ entry:
   ret void
 }
 
+define void @TestFPToSIF128_I128() nounwind {
+; X64-SSE-LABEL: TestFPToSIF128_I128:
+; X64-SSE:       # %bb.0: # %entry
+; X64-SSE-NEXT:    pushq %rax
+; X64-SSE-NEXT:    movaps {{.*}}(%rip), %xmm0
+; X64-SSE-NEXT:    callq __fixtfti
+; X64-SSE-NEXT:    movq %rdx, vi128+{{.*}}(%rip)
+; X64-SSE-NEXT:    movq %rax, {{.*}}(%rip)
+; X64-SSE-NEXT:    popq %rax
+; X64-SSE-NEXT:    retq
+;
+; X32-LABEL: TestFPToSIF128_I128:
+; X32:       # %bb.0: # %entry
+; X32-NEXT:    pushl %esi
+; X32-NEXT:    subl $36, %esp
+; X32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    pushl vf128+12
+; X32-NEXT:    pushl vf128+8
+; X32-NEXT:    pushl vf128+4
+; X32-NEXT:    pushl vf128
+; X32-NEXT:    pushl %eax
+; X32-NEXT:    calll __fixtfti
+; X32-NEXT:    addl $28, %esp
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT:    movl %esi, vi128+12
+; X32-NEXT:    movl %edx, vi128+8
+; X32-NEXT:    movl %ecx, vi128+4
+; X32-NEXT:    movl %eax, vi128
+; X32-NEXT:    addl $24, %esp
+; X32-NEXT:    popl %esi
+; X32-NEXT:    retl
+;
+; X64-AVX-LABEL: TestFPToSIF128_I128:
+; X64-AVX:       # %bb.0: # %entry
+; X64-AVX-NEXT:    pushq %rax
+; X64-AVX-NEXT:    vmovaps {{.*}}(%rip), %xmm0
+; X64-AVX-NEXT:    callq __fixtfti
+; X64-AVX-NEXT:    movq %rdx, vi128+{{.*}}(%rip)
+; X64-AVX-NEXT:    movq %rax, {{.*}}(%rip)
+; X64-AVX-NEXT:    popq %rax
+; X64-AVX-NEXT:    retq
+entry:
+  %0 = load fp128, fp128* @vf128, align 16
+  %conv = fptosi fp128 %0 to i128
+  store i128 %conv, i128* @vi128, align 16
+  ret void
+}
+
+define void @TestFPToUIF128_U128() nounwind {
+; X64-SSE-LABEL: TestFPToUIF128_U128:
+; X64-SSE:       # %bb.0: # %entry
+; X64-SSE-NEXT:    pushq %rax
+; X64-SSE-NEXT:    movaps {{.*}}(%rip), %xmm0
+; X64-SSE-NEXT:    callq __fixunstfti
+; X64-SSE-NEXT:    movq %rdx, vu128+{{.*}}(%rip)
+; X64-SSE-NEXT:    movq %rax, {{.*}}(%rip)
+; X64-SSE-NEXT:    popq %rax
+; X64-SSE-NEXT:    retq
+;
+; X32-LABEL: TestFPToUIF128_U128:
+; X32:       # %bb.0: # %entry
+; X32-NEXT:    pushl %esi
+; X32-NEXT:    subl $36, %esp
+; X32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    pushl vf128+12
+; X32-NEXT:    pushl vf128+8
+; X32-NEXT:    pushl vf128+4
+; X32-NEXT:    pushl vf128
+; X32-NEXT:    pushl %eax
+; X32-NEXT:    calll __fixunstfti
+; X32-NEXT:    addl $28, %esp
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT:    movl %esi, vu128+12
+; X32-NEXT:    movl %edx, vu128+8
+; X32-NEXT:    movl %ecx, vu128+4
+; X32-NEXT:    movl %eax, vu128
+; X32-NEXT:    addl $24, %esp
+; X32-NEXT:    popl %esi
+; X32-NEXT:    retl
+;
+; X64-AVX-LABEL: TestFPToUIF128_U128:
+; X64-AVX:       # %bb.0: # %entry
+; X64-AVX-NEXT:    pushq %rax
+; X64-AVX-NEXT:    vmovaps {{.*}}(%rip), %xmm0
+; X64-AVX-NEXT:    callq __fixunstfti
+; X64-AVX-NEXT:    movq %rdx, vu128+{{.*}}(%rip)
+; X64-AVX-NEXT:    movq %rax, {{.*}}(%rip)
+; X64-AVX-NEXT:    popq %rax
+; X64-AVX-NEXT:    retq
+entry:
+  %0 = load fp128, fp128* @vf128, align 16
+  %conv = fptoui fp128 %0 to i128
+  store i128 %conv, i128* @vu128, align 16
+  ret void
+}
+
 define void @TestFPTruncF128_F32() nounwind {
-; X64-LABEL: TestFPTruncF128_F32:
-; X64:       # %bb.0: # %entry
-; X64-NEXT:    pushq %rax
-; X64-NEXT:    movaps {{.*}}(%rip), %xmm0
-; X64-NEXT:    callq __trunctfsf2
-; X64-NEXT:    movss %xmm0, {{.*}}(%rip)
-; X64-NEXT:    popq %rax
-; X64-NEXT:    retq
-;
-; X64_NO_MMX-LABEL: TestFPTruncF128_F32:
-; X64_NO_MMX:       # %bb.0: # %entry
-; X64_NO_MMX-NEXT:    pushq %rax
-; X64_NO_MMX-NEXT:    movq {{.*}}(%rip), %rdi
-; X64_NO_MMX-NEXT:    movq vf128+{{.*}}(%rip), %rsi
-; X64_NO_MMX-NEXT:    callq __trunctfsf2
-; X64_NO_MMX-NEXT:    movss %xmm0, {{.*}}(%rip)
-; X64_NO_MMX-NEXT:    popq %rax
-; X64_NO_MMX-NEXT:    retq
+; X64-SSE-LABEL: TestFPTruncF128_F32:
+; X64-SSE:       # %bb.0: # %entry
+; X64-SSE-NEXT:    pushq %rax
+; X64-SSE-NEXT:    movaps {{.*}}(%rip), %xmm0
+; X64-SSE-NEXT:    callq __trunctfsf2
+; X64-SSE-NEXT:    movss %xmm0, {{.*}}(%rip)
+; X64-SSE-NEXT:    popq %rax
+; X64-SSE-NEXT:    retq
 ;
 ; X32-LABEL: TestFPTruncF128_F32:
 ; X32:       # %bb.0: # %entry
@@ -358,6 +525,15 @@ define void @TestFPTruncF128_F32() nounwind {
 ; X32-NEXT:    fstps vf32
 ; X32-NEXT:    addl $12, %esp
 ; X32-NEXT:    retl
+;
+; X64-AVX-LABEL: TestFPTruncF128_F32:
+; X64-AVX:       # %bb.0: # %entry
+; X64-AVX-NEXT:    pushq %rax
+; X64-AVX-NEXT:    vmovaps {{.*}}(%rip), %xmm0
+; X64-AVX-NEXT:    callq __trunctfsf2
+; X64-AVX-NEXT:    vmovss %xmm0, {{.*}}(%rip)
+; X64-AVX-NEXT:    popq %rax
+; X64-AVX-NEXT:    retq
 entry:
   %0 = load fp128, fp128* @vf128, align 16
   %conv = fptrunc fp128 %0 to float
@@ -366,24 +542,14 @@ entry:
 }
 
 define void @TestFPTruncF128_F64() nounwind {
-; X64-LABEL: TestFPTruncF128_F64:
-; X64:       # %bb.0: # %entry
-; X64-NEXT:    pushq %rax
-; X64-NEXT:    movaps {{.*}}(%rip), %xmm0
-; X64-NEXT:    callq __trunctfdf2
-; X64-NEXT:    movsd %xmm0, {{.*}}(%rip)
-; X64-NEXT:    popq %rax
-; X64-NEXT:    retq
-;
-; X64_NO_MMX-LABEL: TestFPTruncF128_F64:
-; X64_NO_MMX:       # %bb.0: # %entry
-; X64_NO_MMX-NEXT:    pushq %rax
-; X64_NO_MMX-NEXT:    movq {{.*}}(%rip), %rdi
-; X64_NO_MMX-NEXT:    movq vf128+{{.*}}(%rip), %rsi
-; X64_NO_MMX-NEXT:    callq __trunctfdf2
-; X64_NO_MMX-NEXT:    movsd %xmm0, {{.*}}(%rip)
-; X64_NO_MMX-NEXT:    popq %rax
-; X64_NO_MMX-NEXT:    retq
+; X64-SSE-LABEL: TestFPTruncF128_F64:
+; X64-SSE:       # %bb.0: # %entry
+; X64-SSE-NEXT:    pushq %rax
+; X64-SSE-NEXT:    movaps {{.*}}(%rip), %xmm0
+; X64-SSE-NEXT:    callq __trunctfdf2
+; X64-SSE-NEXT:    movsd %xmm0, {{.*}}(%rip)
+; X64-SSE-NEXT:    popq %rax
+; X64-SSE-NEXT:    retq
 ;
 ; X32-LABEL: TestFPTruncF128_F64:
 ; X32:       # %bb.0: # %entry
@@ -397,6 +563,15 @@ define void @TestFPTruncF128_F64() nounwind {
 ; X32-NEXT:    fstpl vf64
 ; X32-NEXT:    addl $12, %esp
 ; X32-NEXT:    retl
+;
+; X64-AVX-LABEL: TestFPTruncF128_F64:
+; X64-AVX:       # %bb.0: # %entry
+; X64-AVX-NEXT:    pushq %rax
+; X64-AVX-NEXT:    vmovaps {{.*}}(%rip), %xmm0
+; X64-AVX-NEXT:    callq __trunctfdf2
+; X64-AVX-NEXT:    vmovsd %xmm0, {{.*}}(%rip)
+; X64-AVX-NEXT:    popq %rax
+; X64-AVX-NEXT:    retq
 entry:
   %0 = load fp128, fp128* @vf128, align 16
   %conv = fptrunc fp128 %0 to double
@@ -405,28 +580,14 @@ entry:
 }
 
 define void @TestFPTruncF128_F80() nounwind {
-; X64-LABEL: TestFPTruncF128_F80:
-; X64:       # %bb.0: # %entry
-; X64-NEXT:    subq $24, %rsp
-; X64-NEXT:    movaps {{.*}}(%rip), %xmm0
-; X64-NEXT:    callq __trunctfxf2
-; X64-NEXT:    fstpt (%rsp)
-; X64-NEXT:    movq (%rsp), %rax
-; X64-NEXT:    movq %rax, {{.*}}(%rip)
-; X64-NEXT:    movl {{[0-9]+}}(%rsp), %eax
-; X64-NEXT:    movw %ax, vf80+{{.*}}(%rip)
-; X64-NEXT:    addq $24, %rsp
-; X64-NEXT:    retq
-;
-; X64_NO_MMX-LABEL: TestFPTruncF128_F80:
-; X64_NO_MMX:       # %bb.0: # %entry
-; X64_NO_MMX-NEXT:    pushq %rax
-; X64_NO_MMX-NEXT:    movq {{.*}}(%rip), %rdi
-; X64_NO_MMX-NEXT:    movq vf128+{{.*}}(%rip), %rsi
-; X64_NO_MMX-NEXT:    callq __trunctfxf2
-; X64_NO_MMX-NEXT:    fstpt {{.*}}(%rip)
-; X64_NO_MMX-NEXT:    popq %rax
-; X64_NO_MMX-NEXT:    retq
+; X64-SSE-LABEL: TestFPTruncF128_F80:
+; X64-SSE:       # %bb.0: # %entry
+; X64-SSE-NEXT:    pushq %rax
+; X64-SSE-NEXT:    movaps {{.*}}(%rip), %xmm0
+; X64-SSE-NEXT:    callq __trunctfxf2
+; X64-SSE-NEXT:    fstpt {{.*}}(%rip)
+; X64-SSE-NEXT:    popq %rax
+; X64-SSE-NEXT:    retq
 ;
 ; X32-LABEL: TestFPTruncF128_F80:
 ; X32:       # %bb.0: # %entry
@@ -440,6 +601,15 @@ define void @TestFPTruncF128_F80() nounwind {
 ; X32-NEXT:    fstpt vf80
 ; X32-NEXT:    addl $12, %esp
 ; X32-NEXT:    retl
+;
+; X64-AVX-LABEL: TestFPTruncF128_F80:
+; X64-AVX:       # %bb.0: # %entry
+; X64-AVX-NEXT:    pushq %rax
+; X64-AVX-NEXT:    vmovaps {{.*}}(%rip), %xmm0
+; X64-AVX-NEXT:    callq __trunctfxf2
+; X64-AVX-NEXT:    fstpt {{.*}}(%rip)
+; X64-AVX-NEXT:    popq %rax
+; X64-AVX-NEXT:    retq
 entry:
   %0 = load fp128, fp128* @vf128, align 16
   %conv = fptrunc fp128 %0 to x86_fp80
@@ -447,25 +617,111 @@ entry:
   ret void
 }
 
+define void @TestSIToFPI16_F128() nounwind {
+; X64-SSE-LABEL: TestSIToFPI16_F128:
+; X64-SSE:       # %bb.0: # %entry
+; X64-SSE-NEXT:    pushq %rax
+; X64-SSE-NEXT:    movswl {{.*}}(%rip), %edi
+; X64-SSE-NEXT:    callq __floatsitf
+; X64-SSE-NEXT:    movaps %xmm0, {{.*}}(%rip)
+; X64-SSE-NEXT:    popq %rax
+; X64-SSE-NEXT:    retq
+;
+; X32-LABEL: TestSIToFPI16_F128:
+; X32:       # %bb.0: # %entry
+; X32-NEXT:    pushl %esi
+; X32-NEXT:    subl $24, %esp
+; X32-NEXT:    movswl vi16, %eax
+; X32-NEXT:    subl $8, %esp
+; X32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X32-NEXT:    pushl %eax
+; X32-NEXT:    pushl %ecx
+; X32-NEXT:    calll __floatsitf
+; X32-NEXT:    addl $12, %esp
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT:    movl %esi, vf128+12
+; X32-NEXT:    movl %edx, vf128+8
+; X32-NEXT:    movl %ecx, vf128+4
+; X32-NEXT:    movl %eax, vf128
+; X32-NEXT:    addl $24, %esp
+; X32-NEXT:    popl %esi
+; X32-NEXT:    retl
+;
+; X64-AVX-LABEL: TestSIToFPI16_F128:
+; X64-AVX:       # %bb.0: # %entry
+; X64-AVX-NEXT:    pushq %rax
+; X64-AVX-NEXT:    movswl {{.*}}(%rip), %edi
+; X64-AVX-NEXT:    callq __floatsitf
+; X64-AVX-NEXT:    vmovaps %xmm0, {{.*}}(%rip)
+; X64-AVX-NEXT:    popq %rax
+; X64-AVX-NEXT:    retq
+entry:
+  %0 = load i16, i16* @vi16, align 4
+  %conv = sitofp i16 %0 to fp128
+  store fp128 %conv, fp128* @vf128, align 16
+  ret void
+}
+
+define void @TestSIToFPU16_F128() nounwind {
+; X64-SSE-LABEL: TestSIToFPU16_F128:
+; X64-SSE:       # %bb.0: # %entry
+; X64-SSE-NEXT:    pushq %rax
+; X64-SSE-NEXT:    movzwl {{.*}}(%rip), %edi
+; X64-SSE-NEXT:    callq __floatsitf
+; X64-SSE-NEXT:    movaps %xmm0, {{.*}}(%rip)
+; X64-SSE-NEXT:    popq %rax
+; X64-SSE-NEXT:    retq
+;
+; X32-LABEL: TestSIToFPU16_F128:
+; X32:       # %bb.0: # %entry
+; X32-NEXT:    pushl %esi
+; X32-NEXT:    subl $24, %esp
+; X32-NEXT:    movzwl vi16, %eax
+; X32-NEXT:    subl $8, %esp
+; X32-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X32-NEXT:    pushl %eax
+; X32-NEXT:    pushl %ecx
+; X32-NEXT:    calll __floatunsitf
+; X32-NEXT:    addl $12, %esp
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT:    movl %esi, vf128+12
+; X32-NEXT:    movl %edx, vf128+8
+; X32-NEXT:    movl %ecx, vf128+4
+; X32-NEXT:    movl %eax, vf128
+; X32-NEXT:    addl $24, %esp
+; X32-NEXT:    popl %esi
+; X32-NEXT:    retl
+;
+; X64-AVX-LABEL: TestSIToFPU16_F128:
+; X64-AVX:       # %bb.0: # %entry
+; X64-AVX-NEXT:    pushq %rax
+; X64-AVX-NEXT:    movzwl {{.*}}(%rip), %edi
+; X64-AVX-NEXT:    callq __floatsitf
+; X64-AVX-NEXT:    vmovaps %xmm0, {{.*}}(%rip)
+; X64-AVX-NEXT:    popq %rax
+; X64-AVX-NEXT:    retq
+entry:
+  %0 = load i16, i16* @vi16, align 4
+  %conv = uitofp i16 %0 to fp128
+  store fp128 %conv, fp128* @vf128, align 16
+  ret void
+}
+
 define void @TestSIToFPI32_F128() nounwind {
-; X64-LABEL: TestSIToFPI32_F128:
-; X64:       # %bb.0: # %entry
-; X64-NEXT:    pushq %rax
-; X64-NEXT:    movl {{.*}}(%rip), %edi
-; X64-NEXT:    callq __floatsitf
-; X64-NEXT:    movaps %xmm0, {{.*}}(%rip)
-; X64-NEXT:    popq %rax
-; X64-NEXT:    retq
-;
-; X64_NO_MMX-LABEL: TestSIToFPI32_F128:
-; X64_NO_MMX:       # %bb.0: # %entry
-; X64_NO_MMX-NEXT:    pushq %rax
-; X64_NO_MMX-NEXT:    movl {{.*}}(%rip), %edi
-; X64_NO_MMX-NEXT:    callq __floatsitf
-; X64_NO_MMX-NEXT:    movq %rdx, vf128+{{.*}}(%rip)
-; X64_NO_MMX-NEXT:    movq %rax, {{.*}}(%rip)
-; X64_NO_MMX-NEXT:    popq %rax
-; X64_NO_MMX-NEXT:    retq
+; X64-SSE-LABEL: TestSIToFPI32_F128:
+; X64-SSE:       # %bb.0: # %entry
+; X64-SSE-NEXT:    pushq %rax
+; X64-SSE-NEXT:    movl {{.*}}(%rip), %edi
+; X64-SSE-NEXT:    callq __floatsitf
+; X64-SSE-NEXT:    movaps %xmm0, {{.*}}(%rip)
+; X64-SSE-NEXT:    popq %rax
+; X64-SSE-NEXT:    retq
 ;
 ; X32-LABEL: TestSIToFPI32_F128:
 ; X32:       # %bb.0: # %entry
@@ -487,6 +743,15 @@ define void @TestSIToFPI32_F128() nounwind {
 ; X32-NEXT:    addl $24, %esp
 ; X32-NEXT:    popl %esi
 ; X32-NEXT:    retl
+;
+; X64-AVX-LABEL: TestSIToFPI32_F128:
+; X64-AVX:       # %bb.0: # %entry
+; X64-AVX-NEXT:    pushq %rax
+; X64-AVX-NEXT:    movl {{.*}}(%rip), %edi
+; X64-AVX-NEXT:    callq __floatsitf
+; X64-AVX-NEXT:    vmovaps %xmm0, {{.*}}(%rip)
+; X64-AVX-NEXT:    popq %rax
+; X64-AVX-NEXT:    retq
 entry:
   %0 = load i32, i32* @vi32, align 4
   %conv = sitofp i32 %0 to fp128
@@ -495,24 +760,14 @@ entry:
 }
 
 define void @TestUIToFPU32_F128() #2 {
-; X64-LABEL: TestUIToFPU32_F128:
-; X64:       # %bb.0: # %entry
-; X64-NEXT:    pushq %rax
-; X64-NEXT:    movl {{.*}}(%rip), %edi
-; X64-NEXT:    callq __floatunsitf
-; X64-NEXT:    movaps %xmm0, {{.*}}(%rip)
-; X64-NEXT:    popq %rax
-; X64-NEXT:    retq
-;
-; X64_NO_MMX-LABEL: TestUIToFPU32_F128:
-; X64_NO_MMX:       # %bb.0: # %entry
-; X64_NO_MMX-NEXT:    pushq %rax
-; X64_NO_MMX-NEXT:    movl {{.*}}(%rip), %edi
-; X64_NO_MMX-NEXT:    callq __floatunsitf
-; X64_NO_MMX-NEXT:    movq %rdx, vf128+{{.*}}(%rip)
-; X64_NO_MMX-NEXT:    movq %rax, {{.*}}(%rip)
-; X64_NO_MMX-NEXT:    popq %rax
-; X64_NO_MMX-NEXT:    retq
+; X64-SSE-LABEL: TestUIToFPU32_F128:
+; X64-SSE:       # %bb.0: # %entry
+; X64-SSE-NEXT:    pushq %rax
+; X64-SSE-NEXT:    movl {{.*}}(%rip), %edi
+; X64-SSE-NEXT:    callq __floatunsitf
+; X64-SSE-NEXT:    movaps %xmm0, {{.*}}(%rip)
+; X64-SSE-NEXT:    popq %rax
+; X64-SSE-NEXT:    retq
 ;
 ; X32-LABEL: TestUIToFPU32_F128:
 ; X32:       # %bb.0: # %entry
@@ -534,6 +789,15 @@ define void @TestUIToFPU32_F128() #2 {
 ; X32-NEXT:    addl $24, %esp
 ; X32-NEXT:    popl %esi
 ; X32-NEXT:    retl
+;
+; X64-AVX-LABEL: TestUIToFPU32_F128:
+; X64-AVX:       # %bb.0: # %entry
+; X64-AVX-NEXT:    pushq %rax
+; X64-AVX-NEXT:    movl {{.*}}(%rip), %edi
+; X64-AVX-NEXT:    callq __floatunsitf
+; X64-AVX-NEXT:    vmovaps %xmm0, {{.*}}(%rip)
+; X64-AVX-NEXT:    popq %rax
+; X64-AVX-NEXT:    retq
 entry:
   %0 = load i32, i32* @vu32, align 4
   %conv = uitofp i32 %0 to fp128
@@ -542,24 +806,14 @@ entry:
 }
 
 define void @TestSIToFPI64_F128() nounwind {
-; X64-LABEL: TestSIToFPI64_F128:
-; X64:       # %bb.0: # %entry
-; X64-NEXT:    pushq %rax
-; X64-NEXT:    movq {{.*}}(%rip), %rdi
-; X64-NEXT:    callq __floatditf
-; X64-NEXT:    movaps %xmm0, {{.*}}(%rip)
-; X64-NEXT:    popq %rax
-; X64-NEXT:    retq
-;
-; X64_NO_MMX-LABEL: TestSIToFPI64_F128:
-; X64_NO_MMX:       # %bb.0: # %entry
-; X64_NO_MMX-NEXT:    pushq %rax
-; X64_NO_MMX-NEXT:    movq {{.*}}(%rip), %rdi
-; X64_NO_MMX-NEXT:    callq __floatditf
-; X64_NO_MMX-NEXT:    movq %rdx, vf128+{{.*}}(%rip)
-; X64_NO_MMX-NEXT:    movq %rax, {{.*}}(%rip)
-; X64_NO_MMX-NEXT:    popq %rax
-; X64_NO_MMX-NEXT:    retq
+; X64-SSE-LABEL: TestSIToFPI64_F128:
+; X64-SSE:       # %bb.0: # %entry
+; X64-SSE-NEXT:    pushq %rax
+; X64-SSE-NEXT:    movq {{.*}}(%rip), %rdi
+; X64-SSE-NEXT:    callq __floatditf
+; X64-SSE-NEXT:    movaps %xmm0, {{.*}}(%rip)
+; X64-SSE-NEXT:    popq %rax
+; X64-SSE-NEXT:    retq
 ;
 ; X32-LABEL: TestSIToFPI64_F128:
 ; X32:       # %bb.0: # %entry
@@ -582,6 +836,15 @@ define void @TestSIToFPI64_F128() nounwind {
 ; X32-NEXT:    addl $24, %esp
 ; X32-NEXT:    popl %esi
 ; X32-NEXT:    retl
+;
+; X64-AVX-LABEL: TestSIToFPI64_F128:
+; X64-AVX:       # %bb.0: # %entry
+; X64-AVX-NEXT:    pushq %rax
+; X64-AVX-NEXT:    movq {{.*}}(%rip), %rdi
+; X64-AVX-NEXT:    callq __floatditf
+; X64-AVX-NEXT:    vmovaps %xmm0, {{.*}}(%rip)
+; X64-AVX-NEXT:    popq %rax
+; X64-AVX-NEXT:    retq
 entry:
   %0 = load i64, i64* @vi64, align 8
   %conv = sitofp i64 %0 to fp128
@@ -590,24 +853,14 @@ entry:
 }
 
 define void @TestUIToFPU64_F128() #2 {
-; X64-LABEL: TestUIToFPU64_F128:
-; X64:       # %bb.0: # %entry
-; X64-NEXT:    pushq %rax
-; X64-NEXT:    movq {{.*}}(%rip), %rdi
-; X64-NEXT:    callq __floatunditf
-; X64-NEXT:    movaps %xmm0, {{.*}}(%rip)
-; X64-NEXT:    popq %rax
-; X64-NEXT:    retq
-;
-; X64_NO_MMX-LABEL: TestUIToFPU64_F128:
-; X64_NO_MMX:       # %bb.0: # %entry
-; X64_NO_MMX-NEXT:    pushq %rax
-; X64_NO_MMX-NEXT:    movq {{.*}}(%rip), %rdi
-; X64_NO_MMX-NEXT:    callq __floatunditf
-; X64_NO_MMX-NEXT:    movq %rdx, vf128+{{.*}}(%rip)
-; X64_NO_MMX-NEXT:    movq %rax, {{.*}}(%rip)
-; X64_NO_MMX-NEXT:    popq %rax
-; X64_NO_MMX-NEXT:    retq
+; X64-SSE-LABEL: TestUIToFPU64_F128:
+; X64-SSE:       # %bb.0: # %entry
+; X64-SSE-NEXT:    pushq %rax
+; X64-SSE-NEXT:    movq {{.*}}(%rip), %rdi
+; X64-SSE-NEXT:    callq __floatunditf
+; X64-SSE-NEXT:    movaps %xmm0, {{.*}}(%rip)
+; X64-SSE-NEXT:    popq %rax
+; X64-SSE-NEXT:    retq
 ;
 ; X32-LABEL: TestUIToFPU64_F128:
 ; X32:       # %bb.0: # %entry
@@ -630,6 +883,15 @@ define void @TestUIToFPU64_F128() #2 {
 ; X32-NEXT:    addl $24, %esp
 ; X32-NEXT:    popl %esi
 ; X32-NEXT:    retl
+;
+; X64-AVX-LABEL: TestUIToFPU64_F128:
+; X64-AVX:       # %bb.0: # %entry
+; X64-AVX-NEXT:    pushq %rax
+; X64-AVX-NEXT:    movq {{.*}}(%rip), %rdi
+; X64-AVX-NEXT:    callq __floatunditf
+; X64-AVX-NEXT:    vmovaps %xmm0, {{.*}}(%rip)
+; X64-AVX-NEXT:    popq %rax
+; X64-AVX-NEXT:    retq
 entry:
   %0 = load i64, i64* @vu64, align 8
   %conv = uitofp i64 %0 to fp128
@@ -637,31 +899,120 @@ entry:
   ret void
 }
 
+define void @TestSIToFPI128_F128() nounwind {
+; X64-SSE-LABEL: TestSIToFPI128_F128:
+; X64-SSE:       # %bb.0: # %entry
+; X64-SSE-NEXT:    pushq %rax
+; X64-SSE-NEXT:    movq {{.*}}(%rip), %rdi
+; X64-SSE-NEXT:    movq vi128+{{.*}}(%rip), %rsi
+; X64-SSE-NEXT:    callq __floattitf
+; X64-SSE-NEXT:    movaps %xmm0, {{.*}}(%rip)
+; X64-SSE-NEXT:    popq %rax
+; X64-SSE-NEXT:    retq
+;
+; X32-LABEL: TestSIToFPI128_F128:
+; X32:       # %bb.0: # %entry
+; X32-NEXT:    pushl %esi
+; X32-NEXT:    subl $36, %esp
+; X32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    pushl vi128+12
+; X32-NEXT:    pushl vi128+8
+; X32-NEXT:    pushl vi128+4
+; X32-NEXT:    pushl vi128
+; X32-NEXT:    pushl %eax
+; X32-NEXT:    calll __floattitf
+; X32-NEXT:    addl $28, %esp
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT:    movl %esi, vf128+12
+; X32-NEXT:    movl %edx, vf128+8
+; X32-NEXT:    movl %ecx, vf128+4
+; X32-NEXT:    movl %eax, vf128
+; X32-NEXT:    addl $24, %esp
+; X32-NEXT:    popl %esi
+; X32-NEXT:    retl
+;
+; X64-AVX-LABEL: TestSIToFPI128_F128:
+; X64-AVX:       # %bb.0: # %entry
+; X64-AVX-NEXT:    pushq %rax
+; X64-AVX-NEXT:    movq {{.*}}(%rip), %rdi
+; X64-AVX-NEXT:    movq vi128+{{.*}}(%rip), %rsi
+; X64-AVX-NEXT:    callq __floattitf
+; X64-AVX-NEXT:    vmovaps %xmm0, {{.*}}(%rip)
+; X64-AVX-NEXT:    popq %rax
+; X64-AVX-NEXT:    retq
+entry:
+  %0 = load i128, i128* @vi128, align 16
+  %conv = sitofp i128 %0 to fp128
+  store fp128 %conv, fp128* @vf128, align 16
+  ret void
+}
+
+define void @TestUIToFPU128_F128() #2 {
+; X64-SSE-LABEL: TestUIToFPU128_F128:
+; X64-SSE:       # %bb.0: # %entry
+; X64-SSE-NEXT:    pushq %rax
+; X64-SSE-NEXT:    movq {{.*}}(%rip), %rdi
+; X64-SSE-NEXT:    movq vu128+{{.*}}(%rip), %rsi
+; X64-SSE-NEXT:    callq __floatuntitf
+; X64-SSE-NEXT:    movaps %xmm0, {{.*}}(%rip)
+; X64-SSE-NEXT:    popq %rax
+; X64-SSE-NEXT:    retq
+;
+; X32-LABEL: TestUIToFPU128_F128:
+; X32:       # %bb.0: # %entry
+; X32-NEXT:    pushl %esi
+; X32-NEXT:    subl $36, %esp
+; X32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    pushl vu128+12
+; X32-NEXT:    pushl vu128+8
+; X32-NEXT:    pushl vu128+4
+; X32-NEXT:    pushl vu128
+; X32-NEXT:    pushl %eax
+; X32-NEXT:    calll __floatuntitf
+; X32-NEXT:    addl $28, %esp
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT:    movl %esi, vf128+12
+; X32-NEXT:    movl %edx, vf128+8
+; X32-NEXT:    movl %ecx, vf128+4
+; X32-NEXT:    movl %eax, vf128
+; X32-NEXT:    addl $24, %esp
+; X32-NEXT:    popl %esi
+; X32-NEXT:    retl
+;
+; X64-AVX-LABEL: TestUIToFPU128_F128:
+; X64-AVX:       # %bb.0: # %entry
+; X64-AVX-NEXT:    pushq %rax
+; X64-AVX-NEXT:    movq {{.*}}(%rip), %rdi
+; X64-AVX-NEXT:    movq vu128+{{.*}}(%rip), %rsi
+; X64-AVX-NEXT:    callq __floatuntitf
+; X64-AVX-NEXT:    vmovaps %xmm0, {{.*}}(%rip)
+; X64-AVX-NEXT:    popq %rax
+; X64-AVX-NEXT:    retq
+entry:
+  %0 = load i128, i128* @vu128, align 16
+  %conv = uitofp i128 %0 to fp128
+  store fp128 %conv, fp128* @vf128, align 16
+  ret void
+}
+
 define i32 @TestConst128(fp128 %v) nounwind {
-; X64-LABEL: TestConst128:
-; X64:       # %bb.0: # %entry
-; X64-NEXT:    pushq %rax
-; X64-NEXT:    movaps {{.*}}(%rip), %xmm1
-; X64-NEXT:    callq __gttf2
-; X64-NEXT:    xorl %ecx, %ecx
-; X64-NEXT:    testl %eax, %eax
-; X64-NEXT:    setg %cl
-; X64-NEXT:    movl %ecx, %eax
-; X64-NEXT:    popq %rcx
-; X64-NEXT:    retq
-;
-; X64_NO_MMX-LABEL: TestConst128:
-; X64_NO_MMX:       # %bb.0: # %entry
-; X64_NO_MMX-NEXT:    pushq %rax
-; X64_NO_MMX-NEXT:    movabsq $4611404543450677248, %rcx # imm = 0x3FFF000000000000
-; X64_NO_MMX-NEXT:    xorl %edx, %edx
-; X64_NO_MMX-NEXT:    callq __gttf2
-; X64_NO_MMX-NEXT:    xorl %ecx, %ecx
-; X64_NO_MMX-NEXT:    testl %eax, %eax
-; X64_NO_MMX-NEXT:    setg %cl
-; X64_NO_MMX-NEXT:    movl %ecx, %eax
-; X64_NO_MMX-NEXT:    popq %rcx
-; X64_NO_MMX-NEXT:    retq
+; X64-SSE-LABEL: TestConst128:
+; X64-SSE:       # %bb.0: # %entry
+; X64-SSE-NEXT:    pushq %rax
+; X64-SSE-NEXT:    movaps {{.*}}(%rip), %xmm1
+; X64-SSE-NEXT:    callq __gttf2
+; X64-SSE-NEXT:    xorl %ecx, %ecx
+; X64-SSE-NEXT:    testl %eax, %eax
+; X64-SSE-NEXT:    setg %cl
+; X64-SSE-NEXT:    movl %ecx, %eax
+; X64-SSE-NEXT:    popq %rcx
+; X64-SSE-NEXT:    retq
 ;
 ; X32-LABEL: TestConst128:
 ; X32:       # %bb.0: # %entry
@@ -682,12 +1033,75 @@ define i32 @TestConst128(fp128 %v) nounwind {
 ; X32-NEXT:    movl %ecx, %eax
 ; X32-NEXT:    addl $12, %esp
 ; X32-NEXT:    retl
+;
+; X64-AVX-LABEL: TestConst128:
+; X64-AVX:       # %bb.0: # %entry
+; X64-AVX-NEXT:    pushq %rax
+; X64-AVX-NEXT:    vmovaps {{.*}}(%rip), %xmm1
+; X64-AVX-NEXT:    callq __gttf2
+; X64-AVX-NEXT:    xorl %ecx, %ecx
+; X64-AVX-NEXT:    testl %eax, %eax
+; X64-AVX-NEXT:    setg %cl
+; X64-AVX-NEXT:    movl %ecx, %eax
+; X64-AVX-NEXT:    popq %rcx
+; X64-AVX-NEXT:    retq
 entry:
   %cmp = fcmp ogt fp128 %v, 0xL00000000000000003FFF000000000000
   %conv = zext i1 %cmp to i32
   ret i32 %conv
 }
 
+
+define i32 @TestConst128Zero(fp128 %v) nounwind {
+; X64-SSE-LABEL: TestConst128Zero:
+; X64-SSE:       # %bb.0: # %entry
+; X64-SSE-NEXT:    pushq %rax
+; X64-SSE-NEXT:    xorps %xmm1, %xmm1
+; X64-SSE-NEXT:    callq __gttf2
+; X64-SSE-NEXT:    xorl %ecx, %ecx
+; X64-SSE-NEXT:    testl %eax, %eax
+; X64-SSE-NEXT:    setg %cl
+; X64-SSE-NEXT:    movl %ecx, %eax
+; X64-SSE-NEXT:    popq %rcx
+; X64-SSE-NEXT:    retq
+;
+; X32-LABEL: TestConst128Zero:
+; X32:       # %bb.0: # %entry
+; X32-NEXT:    subl $12, %esp
+; X32-NEXT:    pushl $0
+; X32-NEXT:    pushl $0
+; X32-NEXT:    pushl $0
+; X32-NEXT:    pushl $0
+; X32-NEXT:    pushl {{[0-9]+}}(%esp)
+; X32-NEXT:    pushl {{[0-9]+}}(%esp)
+; X32-NEXT:    pushl {{[0-9]+}}(%esp)
+; X32-NEXT:    pushl {{[0-9]+}}(%esp)
+; X32-NEXT:    calll __gttf2
+; X32-NEXT:    addl $32, %esp
+; X32-NEXT:    xorl %ecx, %ecx
+; X32-NEXT:    testl %eax, %eax
+; X32-NEXT:    setg %cl
+; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    addl $12, %esp
+; X32-NEXT:    retl
+;
+; X64-AVX-LABEL: TestConst128Zero:
+; X64-AVX:       # %bb.0: # %entry
+; X64-AVX-NEXT:    pushq %rax
+; X64-AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X64-AVX-NEXT:    callq __gttf2
+; X64-AVX-NEXT:    xorl %ecx, %ecx
+; X64-AVX-NEXT:    testl %eax, %eax
+; X64-AVX-NEXT:    setg %cl
+; X64-AVX-NEXT:    movl %ecx, %eax
+; X64-AVX-NEXT:    popq %rcx
+; X64-AVX-NEXT:    retq
+entry:
+  %cmp = fcmp ogt fp128 %v, 0xL00000000000000000000000000000000
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
 ; C code:
 ;  struct TestBits_ieee_ext {
 ;    unsigned v1;
@@ -703,35 +1117,20 @@ entry:
 ;   return ((u.bits.v1 | u.bits.v2)  == 0);
 ; }
 define i32 @TestBits128(fp128 %ld) nounwind {
-; X64-LABEL: TestBits128:
-; X64:       # %bb.0: # %entry
-; X64-NEXT:    subq $24, %rsp
-; X64-NEXT:    movaps %xmm0, %xmm1
-; X64-NEXT:    callq __multf3
-; X64-NEXT:    movaps %xmm0, (%rsp)
-; X64-NEXT:    movq (%rsp), %rcx
-; X64-NEXT:    movq %rcx, %rdx
-; X64-NEXT:    shrq $32, %rdx
-; X64-NEXT:    xorl %eax, %eax
-; X64-NEXT:    orl %ecx, %edx
-; X64-NEXT:    sete %al
-; X64-NEXT:    addq $24, %rsp
-; X64-NEXT:    retq
-;
-; X64_NO_MMX-LABEL: TestBits128:
-; X64_NO_MMX:       # %bb.0: # %entry
-; X64_NO_MMX-NEXT:    pushq %rax
-; X64_NO_MMX-NEXT:    movq %rdi, %rdx
-; X64_NO_MMX-NEXT:    movq %rsi, %rcx
-; X64_NO_MMX-NEXT:    callq __multf3
-; X64_NO_MMX-NEXT:    movq %rax, %rdx
-; X64_NO_MMX-NEXT:    shrq $32, %rdx
-; X64_NO_MMX-NEXT:    xorl %ecx, %ecx
-; X64_NO_MMX-NEXT:    orl %eax, %edx
-; X64_NO_MMX-NEXT:    sete %cl
-; X64_NO_MMX-NEXT:    movl %ecx, %eax
-; X64_NO_MMX-NEXT:    popq %rcx
-; X64_NO_MMX-NEXT:    retq
+; X64-SSE-LABEL: TestBits128:
+; X64-SSE:       # %bb.0: # %entry
+; X64-SSE-NEXT:    subq $24, %rsp
+; X64-SSE-NEXT:    movaps %xmm0, %xmm1
+; X64-SSE-NEXT:    callq __multf3
+; X64-SSE-NEXT:    movaps %xmm0, (%rsp)
+; X64-SSE-NEXT:    movq (%rsp), %rcx
+; X64-SSE-NEXT:    movq %rcx, %rdx
+; X64-SSE-NEXT:    shrq $32, %rdx
+; X64-SSE-NEXT:    xorl %eax, %eax
+; X64-SSE-NEXT:    orl %ecx, %edx
+; X64-SSE-NEXT:    sete %al
+; X64-SSE-NEXT:    addq $24, %rsp
+; X64-SSE-NEXT:    retq
 ;
 ; X32-LABEL: TestBits128:
 ; X32:       # %bb.0: # %entry
@@ -763,6 +1162,21 @@ define i32 @TestBits128(fp128 %ld) nounwind {
 ; X32-NEXT:    popl %esi
 ; X32-NEXT:    popl %edi
 ; X32-NEXT:    retl
+;
+; X64-AVX-LABEL: TestBits128:
+; X64-AVX:       # %bb.0: # %entry
+; X64-AVX-NEXT:    subq $24, %rsp
+; X64-AVX-NEXT:    vmovaps %xmm0, %xmm1
+; X64-AVX-NEXT:    callq __multf3
+; X64-AVX-NEXT:    vmovaps %xmm0, (%rsp)
+; X64-AVX-NEXT:    movq (%rsp), %rcx
+; X64-AVX-NEXT:    movq %rcx, %rdx
+; X64-AVX-NEXT:    shrq $32, %rdx
+; X64-AVX-NEXT:    xorl %eax, %eax
+; X64-AVX-NEXT:    orl %ecx, %edx
+; X64-AVX-NEXT:    sete %al
+; X64-AVX-NEXT:    addq $24, %rsp
+; X64-AVX-NEXT:    retq
 entry:
   %mul = fmul fp128 %ld, %ld
   %0 = bitcast fp128 %mul to i128
@@ -789,22 +1203,14 @@ entry:
 ;   return *(__float128*)&n;
 ; }
 define fp128 @TestPair128(i64 %a, i64 %b) nounwind {
-; X64-LABEL: TestPair128:
-; X64:       # %bb.0: # %entry
-; X64-NEXT:    addq $3, %rsi
-; X64-NEXT:    movq %rsi, -{{[0-9]+}}(%rsp)
-; X64-NEXT:    adcq $0, %rdi
-; X64-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-NEXT:    movaps -{{[0-9]+}}(%rsp), %xmm0
-; X64-NEXT:    retq
-;
-; X64_NO_MMX-LABEL: TestPair128:
-; X64_NO_MMX:       # %bb.0: # %entry
-; X64_NO_MMX-NEXT:    movq %rsi, %rax
-; X64_NO_MMX-NEXT:    addq $3, %rax
-; X64_NO_MMX-NEXT:    adcq $0, %rdi
-; X64_NO_MMX-NEXT:    movq %rdi, %rdx
-; X64_NO_MMX-NEXT:    retq
+; X64-SSE-LABEL: TestPair128:
+; X64-SSE:       # %bb.0: # %entry
+; X64-SSE-NEXT:    addq $3, %rsi
+; X64-SSE-NEXT:    movq %rsi, -{{[0-9]+}}(%rsp)
+; X64-SSE-NEXT:    adcq $0, %rdi
+; X64-SSE-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-SSE-NEXT:    movaps -{{[0-9]+}}(%rsp), %xmm0
+; X64-SSE-NEXT:    retq
 ;
 ; X32-LABEL: TestPair128:
 ; X32:       # %bb.0: # %entry
@@ -826,6 +1232,15 @@ define fp128 @TestPair128(i64 %a, i64 %b) nounwind {
 ; X32-NEXT:    popl %esi
 ; X32-NEXT:    popl %edi
 ; X32-NEXT:    retl $4
+;
+; X64-AVX-LABEL: TestPair128:
+; X64-AVX:       # %bb.0: # %entry
+; X64-AVX-NEXT:    addq $3, %rsi
+; X64-AVX-NEXT:    movq %rsi, -{{[0-9]+}}(%rsp)
+; X64-AVX-NEXT:    adcq $0, %rdi
+; X64-AVX-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-AVX-NEXT:    vmovaps -{{[0-9]+}}(%rsp), %xmm0
+; X64-AVX-NEXT:    retq
 entry:
   %conv = zext i64 %a to i128
   %shl = shl nuw i128 %conv, 64
@@ -837,40 +1252,20 @@ entry:
 }
 
 define fp128 @TestTruncCopysign(fp128 %x, i32 %n) nounwind {
-; X64-LABEL: TestTruncCopysign:
-; X64:       # %bb.0: # %entry
-; X64-NEXT:    cmpl $50001, %edi # imm = 0xC351
-; X64-NEXT:    jl .LBB17_2
-; X64-NEXT:  # %bb.1: # %if.then
-; X64-NEXT:    pushq %rax
-; X64-NEXT:    callq __trunctfdf2
-; X64-NEXT:    andps {{.*}}(%rip), %xmm0
-; X64-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
-; X64-NEXT:    orps %xmm1, %xmm0
-; X64-NEXT:    callq __extenddftf2
-; X64-NEXT:    addq $8, %rsp
-; X64-NEXT:  .LBB17_2: # %cleanup
-; X64-NEXT:    retq
-;
-; X64_NO_MMX-LABEL: TestTruncCopysign:
-; X64_NO_MMX:       # %bb.0: # %entry
-; X64_NO_MMX-NEXT:    movl %edx, %ecx
-; X64_NO_MMX-NEXT:    movq %rsi, %rdx
-; X64_NO_MMX-NEXT:    movq %rdi, %rax
-; X64_NO_MMX-NEXT:    cmpl $50001, %ecx # imm = 0xC351
-; X64_NO_MMX-NEXT:    jl .LBB17_2
-; X64_NO_MMX-NEXT:  # %bb.1: # %if.then
-; X64_NO_MMX-NEXT:    pushq %rax
-; X64_NO_MMX-NEXT:    movq %rax, %rdi
-; X64_NO_MMX-NEXT:    movq %rdx, %rsi
-; X64_NO_MMX-NEXT:    callq __trunctfdf2
-; X64_NO_MMX-NEXT:    andps {{.*}}(%rip), %xmm0
-; X64_NO_MMX-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
-; X64_NO_MMX-NEXT:    orps %xmm1, %xmm0
-; X64_NO_MMX-NEXT:    callq __extenddftf2
-; X64_NO_MMX-NEXT:    addq $8, %rsp
-; X64_NO_MMX-NEXT:  .LBB17_2: # %cleanup
-; X64_NO_MMX-NEXT:    retq
+; X64-SSE-LABEL: TestTruncCopysign:
+; X64-SSE:       # %bb.0: # %entry
+; X64-SSE-NEXT:    cmpl $50001, %edi # imm = 0xC351
+; X64-SSE-NEXT:    jl .LBB26_2
+; X64-SSE-NEXT:  # %bb.1: # %if.then
+; X64-SSE-NEXT:    pushq %rax
+; X64-SSE-NEXT:    callq __trunctfdf2
+; X64-SSE-NEXT:    andps {{.*}}(%rip), %xmm0
+; X64-SSE-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
+; X64-SSE-NEXT:    orps %xmm1, %xmm0
+; X64-SSE-NEXT:    callq __extenddftf2
+; X64-SSE-NEXT:    addq $8, %rsp
+; X64-SSE-NEXT:  .LBB26_2: # %cleanup
+; X64-SSE-NEXT:    retq
 ;
 ; X32-LABEL: TestTruncCopysign:
 ; X32:       # %bb.0: # %entry
@@ -883,7 +1278,7 @@ define fp128 @TestTruncCopysign(fp128 %x, i32 %n) nounwind {
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %edi
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X32-NEXT:    cmpl $50001, {{[0-9]+}}(%esp) # imm = 0xC351
-; X32-NEXT:    jl .LBB17_4
+; X32-NEXT:    jl .LBB26_4
 ; X32-NEXT:  # %bb.1: # %if.then
 ; X32-NEXT:    pushl %eax
 ; X32-NEXT:    pushl %ecx
@@ -895,11 +1290,11 @@ define fp128 @TestTruncCopysign(fp128 %x, i32 %n) nounwind {
 ; X32-NEXT:    testb $-128, {{[0-9]+}}(%esp)
 ; X32-NEXT:    flds {{\.LCPI.*}}
 ; X32-NEXT:    flds {{\.LCPI.*}}
-; X32-NEXT:    jne .LBB17_3
+; X32-NEXT:    jne .LBB26_3
 ; X32-NEXT:  # %bb.2: # %if.then
 ; X32-NEXT:    fstp %st(1)
 ; X32-NEXT:    fldz
-; X32-NEXT:  .LBB17_3: # %if.then
+; X32-NEXT:  .LBB26_3: # %if.then
 ; X32-NEXT:    fstp %st(0)
 ; X32-NEXT:    subl $16, %esp
 ; X32-NEXT:    leal {{[0-9]+}}(%esp), %eax
@@ -911,7 +1306,7 @@ define fp128 @TestTruncCopysign(fp128 %x, i32 %n) nounwind {
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X32-NEXT:  .LBB17_4: # %cleanup
+; X32-NEXT:  .LBB26_4: # %cleanup
 ; X32-NEXT:    movl %edx, (%esi)
 ; X32-NEXT:    movl %edi, 4(%esi)
 ; X32-NEXT:    movl %ecx, 8(%esi)
@@ -921,6 +1316,22 @@ define fp128 @TestTruncCopysign(fp128 %x, i32 %n) nounwind {
 ; X32-NEXT:    popl %esi
 ; X32-NEXT:    popl %edi
 ; X32-NEXT:    retl $4
+;
+; X64-AVX-LABEL: TestTruncCopysign:
+; X64-AVX:       # %bb.0: # %entry
+; X64-AVX-NEXT:    cmpl $50001, %edi # imm = 0xC351
+; X64-AVX-NEXT:    jl .LBB26_2
+; X64-AVX-NEXT:  # %bb.1: # %if.then
+; X64-AVX-NEXT:    pushq %rax
+; X64-AVX-NEXT:    callq __trunctfdf2
+; X64-AVX-NEXT:    vandps {{.*}}(%rip), %xmm0, %xmm0
+; X64-AVX-NEXT:    vmovddup {{.*#+}} xmm1 = [+Inf,+Inf]
+; X64-AVX-NEXT:    # xmm1 = mem[0,0]
+; X64-AVX-NEXT:    vorps %xmm0, %xmm1, %xmm0
+; X64-AVX-NEXT:    callq __extenddftf2
+; X64-AVX-NEXT:    addq $8, %rsp
+; X64-AVX-NEXT:  .LBB26_2: # %cleanup
+; X64-AVX-NEXT:    retq
 entry:
   %cmp = icmp sgt i32 %n, 50000
   br i1 %cmp, label %if.then, label %cleanup
@@ -937,21 +1348,15 @@ cleanup:                                          ; preds = %entry, %if.then
 }
 
 define i1 @PR34866(i128 %x) nounwind {
-; X64-LABEL: PR34866:
-; X64:       # %bb.0:
-; X64-NEXT:    movaps {{.*}}(%rip), %xmm0
-; X64-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-NEXT:    xorq -{{[0-9]+}}(%rsp), %rsi
-; X64-NEXT:    xorq -{{[0-9]+}}(%rsp), %rdi
-; X64-NEXT:    orq %rsi, %rdi
-; X64-NEXT:    sete %al
-; X64-NEXT:    retq
-;
-; X64_NO_MMX-LABEL: PR34866:
-; X64_NO_MMX:       # %bb.0:
-; X64_NO_MMX-NEXT:    orq %rsi, %rdi
-; X64_NO_MMX-NEXT:    sete %al
-; X64_NO_MMX-NEXT:    retq
+; X64-SSE-LABEL: PR34866:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    xorps %xmm0, %xmm0
+; X64-SSE-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-SSE-NEXT:    xorq -{{[0-9]+}}(%rsp), %rsi
+; X64-SSE-NEXT:    xorq -{{[0-9]+}}(%rsp), %rdi
+; X64-SSE-NEXT:    orq %rsi, %rdi
+; X64-SSE-NEXT:    sete %al
+; X64-SSE-NEXT:    retq
 ;
 ; X32-LABEL: PR34866:
 ; X32:       # %bb.0:
@@ -962,27 +1367,31 @@ define i1 @PR34866(i128 %x) nounwind {
 ; X32-NEXT:    orl %ecx, %eax
 ; X32-NEXT:    sete %al
 ; X32-NEXT:    retl
+;
+; X64-AVX-LABEL: PR34866:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; X64-AVX-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-AVX-NEXT:    xorq -{{[0-9]+}}(%rsp), %rsi
+; X64-AVX-NEXT:    xorq -{{[0-9]+}}(%rsp), %rdi
+; X64-AVX-NEXT:    orq %rsi, %rdi
+; X64-AVX-NEXT:    sete %al
+; X64-AVX-NEXT:    retq
   %bc_mmx = bitcast fp128 0xL00000000000000000000000000000000 to i128
   %cmp = icmp eq i128 %bc_mmx, %x
   ret i1 %cmp
 }
 
 define i1 @PR34866_commute(i128 %x) nounwind {
-; X64-LABEL: PR34866_commute:
-; X64:       # %bb.0:
-; X64-NEXT:    movaps {{.*}}(%rip), %xmm0
-; X64-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-NEXT:    xorq -{{[0-9]+}}(%rsp), %rsi
-; X64-NEXT:    xorq -{{[0-9]+}}(%rsp), %rdi
-; X64-NEXT:    orq %rsi, %rdi
-; X64-NEXT:    sete %al
-; X64-NEXT:    retq
-;
-; X64_NO_MMX-LABEL: PR34866_commute:
-; X64_NO_MMX:       # %bb.0:
-; X64_NO_MMX-NEXT:    orq %rsi, %rdi
-; X64_NO_MMX-NEXT:    sete %al
-; X64_NO_MMX-NEXT:    retq
+; X64-SSE-LABEL: PR34866_commute:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    xorps %xmm0, %xmm0
+; X64-SSE-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-SSE-NEXT:    xorq -{{[0-9]+}}(%rsp), %rsi
+; X64-SSE-NEXT:    xorq -{{[0-9]+}}(%rsp), %rdi
+; X64-SSE-NEXT:    orq %rsi, %rdi
+; X64-SSE-NEXT:    sete %al
+; X64-SSE-NEXT:    retq
 ;
 ; X32-LABEL: PR34866_commute:
 ; X32:       # %bb.0:
@@ -993,6 +1402,16 @@ define i1 @PR34866_commute(i128 %x) nounwind {
 ; X32-NEXT:    orl %ecx, %eax
 ; X32-NEXT:    sete %al
 ; X32-NEXT:    retl
+;
+; X64-AVX-LABEL: PR34866_commute:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; X64-AVX-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-AVX-NEXT:    xorq -{{[0-9]+}}(%rsp), %rsi
+; X64-AVX-NEXT:    xorq -{{[0-9]+}}(%rsp), %rdi
+; X64-AVX-NEXT:    orq %rsi, %rdi
+; X64-AVX-NEXT:    sete %al
+; X64-AVX-NEXT:    retq
   %bc_mmx = bitcast fp128 0xL00000000000000000000000000000000 to i128
   %cmp = icmp eq i128 %x, %bc_mmx
   ret i1 %cmp
diff --git a/llvm/test/CodeGen/X86/fp128-compare.ll b/llvm/test/CodeGen/X86/fp128-compare.ll
index 6f2b0c514..8aa4e5544 100644
--- a/llvm/test/CodeGen/X86/fp128-compare.ll
+++ b/llvm/test/CodeGen/X86/fp128-compare.ll
@@ -48,7 +48,10 @@ define i32 @TestComp128LT(fp128 %d1, fp128 %d2) {
 ; CHECK-NEXT:    pushq %rax
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    callq __lttf2
-; CHECK-NEXT:    shrl $31, %eax
+; CHECK-NEXT:    xorl %ecx, %ecx
+; CHECK-NEXT:    testl %eax, %eax
+; CHECK-NEXT:    sets %cl
+; CHECK-NEXT:    movl %ecx, %eax
 ; CHECK-NEXT:    popq %rcx
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK-NEXT:    retq
@@ -56,9 +59,9 @@ entry:
   %cmp = fcmp olt fp128 %d1, %d2
   %conv = zext i1 %cmp to i32
   ret i32 %conv
-; The 'shrl' is a special optimization in llvm to combine
-; the effect of 'fcmp olt' and 'zext'. The main purpose is
-; to test soften call to __lttf2.
+; FIXME: This used to generate a shrl to move the sign bit of eax into bit 0.
+; This no longer happens with fp128 compares being expanded by LegalizeDAG.
+; We can add a new DAG combine for X86ISD::CMP/SETCC to restore this.
 }
 
 define i32 @TestComp128LE(fp128 %d1, fp128 %d2) {
@@ -118,6 +121,68 @@ entry:
   ret i32 %conv
 }
 
+define i32 @TestComp128UEQ(fp128 %d1, fp128 %d2) {
+; CHECK-LABEL: TestComp128UEQ:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    subq $32, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    .cfi_offset %rbx, -16
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
+; CHECK-NEXT:    callq __eqtf2
+; CHECK-NEXT:    testl %eax, %eax
+; CHECK-NEXT:    sete %bl
+; CHECK-NEXT:    movaps (%rsp), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    callq __unordtf2
+; CHECK-NEXT:    testl %eax, %eax
+; CHECK-NEXT:    setne %al
+; CHECK-NEXT:    orb %bl, %al
+; CHECK-NEXT:    movzbl %al, %eax
+; CHECK-NEXT:    addq $32, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
+entry:
+  %cmp = fcmp ueq fp128 %d1, %d2
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+define i32 @TestComp128ONE(fp128 %d1, fp128 %d2) {
+; CHECK-LABEL: TestComp128ONE:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    subq $32, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    .cfi_offset %rbx, -16
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
+; CHECK-NEXT:    callq __eqtf2
+; CHECK-NEXT:    testl %eax, %eax
+; CHECK-NEXT:    setne %bl
+; CHECK-NEXT:    movaps (%rsp), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    callq __unordtf2
+; CHECK-NEXT:    testl %eax, %eax
+; CHECK-NEXT:    sete %al
+; CHECK-NEXT:    andb %bl, %al
+; CHECK-NEXT:    movzbl %al, %eax
+; CHECK-NEXT:    addq $32, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
+entry:
+  %cmp = fcmp one fp128 %d1, %d2
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
 define fp128 @TestMax(fp128 %x, fp128 %y) {
 ; CHECK-LABEL: TestMax:
 ; CHECK:       # %bb.0: # %entry
@@ -128,10 +193,10 @@ define fp128 @TestMax(fp128 %x, fp128 %y) {
 ; CHECK-NEXT:    callq __gttf2
 ; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
 ; CHECK-NEXT:    testl %eax, %eax
-; CHECK-NEXT:    jg .LBB6_2
+; CHECK-NEXT:    jg .LBB8_2
 ; CHECK-NEXT:  # %bb.1: # %entry
 ; CHECK-NEXT:    movaps (%rsp), %xmm0 # 16-byte Reload
-; CHECK-NEXT:  .LBB6_2: # %entry
+; CHECK-NEXT:  .LBB8_2: # %entry
 ; CHECK-NEXT:    addq $40, %rsp
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/fp128-extract.ll b/llvm/test/CodeGen/X86/fp128-extract.ll
index 46dbaf292..ac4cb28b5 100644
--- a/llvm/test/CodeGen/X86/fp128-extract.ll
+++ b/llvm/test/CodeGen/X86/fp128-extract.ll
@@ -9,15 +9,14 @@ define fp128 @TestExtract(<2 x double> %x) nounwind {
 ; CHECK-NEXT:    subq $40, %rsp
 ; CHECK-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
 ; CHECK-NEXT:    callq __extenddftf2
-; CHECK-NEXT:    movaps %xmm0, {{[0-9]+}}(%rsp) # 16-byte Spill
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; CHECK-NEXT:    movaps (%rsp), %xmm0 # 16-byte Reload
 ; CHECK-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
 ; CHECK-NEXT:    callq __extenddftf2
 ; CHECK-NEXT:    movaps %xmm0, %xmm1
-; CHECK-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm0 # 16-byte Reload
-; CHECK-NEXT:    callq __multf3
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
 ; CHECK-NEXT:    addq $40, %rsp
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    jmp __multf3 # TAILCALL
 entry:
   ; Simplified instruction pattern from the output of llvm before r289042,
   ; for a boost function ...::insert<...>::traverse<...>().
diff --git a/llvm/test/CodeGen/X86/fp128-g.ll b/llvm/test/CodeGen/X86/fp128-g.ll
index 44011b169..8130e0bd1 100644
--- a/llvm/test/CodeGen/X86/fp128-g.ll
+++ b/llvm/test/CodeGen/X86/fp128-g.ll
@@ -54,9 +54,7 @@ entry:
 ; X64:       movaps     (%rdi), %xmm0
 ; X64:       .loc
 ; X64:       movaps	%xmm0, %xmm1
-; X64:       callq	__addtf3
-; X64:       .loc
-; X64:       retq
+; X64:       jmp	__addtf3
 }
 
 ; Function Attrs: norecurse nounwind readonly uwtable
@@ -102,16 +100,14 @@ entry:
 ; X64:       movaps	(%rax), %xmm0
 ; X64:       .loc
 ; X64:       movaps	%xmm0, %xmm1
-; X64:       callq	__addtf3
-; X64:       .loc
-; X64:       retq
+; X64:       jmp	__addtf3
 }
 
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.value(metadata, i64, metadata, metadata) #2
 
-attributes #0 = { nounwind readonly uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+popcnt,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { norecurse nounwind readonly uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+popcnt,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind readonly uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+popcnt,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { norecurse nounwind readonly uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+popcnt,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!2}
diff --git a/llvm/test/CodeGen/X86/fp128-i128.ll b/llvm/test/CodeGen/X86/fp128-i128.ll
index f18b3e46e..0c6b5b555 100644
--- a/llvm/test/CodeGen/X86/fp128-i128.ll
+++ b/llvm/test/CodeGen/X86/fp128-i128.ll
@@ -309,29 +309,23 @@ if.end:                                           ; preds = %if.then, %entry
 define fp128 @TestI128_4(fp128 %x) #0 {
 ; SSE-LABEL: TestI128_4:
 ; SSE:       # %bb.0: # %entry
-; SSE-NEXT:    subq $40, %rsp
 ; SSE-NEXT:    movaps %xmm0, %xmm1
-; SSE-NEXT:    movaps %xmm0, {{[0-9]+}}(%rsp)
-; SSE-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; SSE-NEXT:    movq %rax, {{[0-9]+}}(%rsp)
-; SSE-NEXT:    movq $0, (%rsp)
-; SSE-NEXT:    movaps (%rsp), %xmm0
-; SSE-NEXT:    callq __addtf3
-; SSE-NEXT:    addq $40, %rsp
-; SSE-NEXT:    retq
+; SSE-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; SSE-NEXT:    movq -{{[0-9]+}}(%rsp), %rax
+; SSE-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
+; SSE-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
+; SSE-NEXT:    movaps -{{[0-9]+}}(%rsp), %xmm0
+; SSE-NEXT:    jmp __addtf3 # TAILCALL
 ;
 ; AVX-LABEL: TestI128_4:
 ; AVX:       # %bb.0: # %entry
-; AVX-NEXT:    subq $40, %rsp
 ; AVX-NEXT:    vmovaps %xmm0, %xmm1
-; AVX-NEXT:    vmovaps %xmm0, {{[0-9]+}}(%rsp)
-; AVX-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX-NEXT:    movq %rax, {{[0-9]+}}(%rsp)
-; AVX-NEXT:    movq $0, (%rsp)
-; AVX-NEXT:    vmovaps (%rsp), %xmm0
-; AVX-NEXT:    callq __addtf3
-; AVX-NEXT:    addq $40, %rsp
-; AVX-NEXT:    retq
+; AVX-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; AVX-NEXT:    movq -{{[0-9]+}}(%rsp), %rax
+; AVX-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
+; AVX-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
+; AVX-NEXT:    vmovaps -{{[0-9]+}}(%rsp), %xmm0
+; AVX-NEXT:    jmp __addtf3 # TAILCALL
 entry:
   %0 = bitcast fp128 %x to i128
   %bf.clear = and i128 %0, -18446744073709551616
@@ -370,29 +364,23 @@ entry:
 define fp128 @acosl(fp128 %x) #0 {
 ; SSE-LABEL: acosl:
 ; SSE:       # %bb.0: # %entry
-; SSE-NEXT:    subq $40, %rsp
 ; SSE-NEXT:    movaps %xmm0, %xmm1
-; SSE-NEXT:    movaps %xmm0, {{[0-9]+}}(%rsp)
-; SSE-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; SSE-NEXT:    movq %rax, {{[0-9]+}}(%rsp)
-; SSE-NEXT:    movq $0, (%rsp)
-; SSE-NEXT:    movaps (%rsp), %xmm0
-; SSE-NEXT:    callq __addtf3
-; SSE-NEXT:    addq $40, %rsp
-; SSE-NEXT:    retq
+; SSE-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; SSE-NEXT:    movq -{{[0-9]+}}(%rsp), %rax
+; SSE-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
+; SSE-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
+; SSE-NEXT:    movaps -{{[0-9]+}}(%rsp), %xmm0
+; SSE-NEXT:    jmp __addtf3 # TAILCALL
 ;
 ; AVX-LABEL: acosl:
 ; AVX:       # %bb.0: # %entry
-; AVX-NEXT:    subq $40, %rsp
 ; AVX-NEXT:    vmovaps %xmm0, %xmm1
-; AVX-NEXT:    vmovaps %xmm0, {{[0-9]+}}(%rsp)
-; AVX-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX-NEXT:    movq %rax, {{[0-9]+}}(%rsp)
-; AVX-NEXT:    movq $0, (%rsp)
-; AVX-NEXT:    vmovaps (%rsp), %xmm0
-; AVX-NEXT:    callq __addtf3
-; AVX-NEXT:    addq $40, %rsp
-; AVX-NEXT:    retq
+; AVX-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; AVX-NEXT:    movq -{{[0-9]+}}(%rsp), %rax
+; AVX-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
+; AVX-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
+; AVX-NEXT:    vmovaps -{{[0-9]+}}(%rsp), %xmm0
+; AVX-NEXT:    jmp __addtf3 # TAILCALL
 entry:
   %0 = bitcast fp128 %x to i128
   %bf.clear = and i128 %0, -18446744073709551616
@@ -508,9 +496,8 @@ define void @TestCopySign({ fp128, fp128 }* noalias nocapture sret %agg.result,
 ; AVX-NEXT:    testl %ebp, %ebp
 ; AVX-NEXT:    jle .LBB10_1
 ; AVX-NEXT:  # %bb.2: # %if.then
-; AVX-NEXT:    vandps {{.*}}(%rip), %xmm0, %xmm1
+; AVX-NEXT:    vandps {{.*}}(%rip), %xmm0, %xmm2
 ; AVX-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
-; AVX-NEXT:    vmovaps %xmm1, %xmm2
 ; AVX-NEXT:    jmp .LBB10_3
 ; AVX-NEXT:  .LBB10_1:
 ; AVX-NEXT:    vmovaps (%rsp), %xmm2 # 16-byte Reload
@@ -551,6 +538,6 @@ cleanup:                                          ; preds = %entry, %if.then
 }
 
 
-attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+ssse3,+sse3,+popcnt,+sse,+sse2,+sse4.1,+sse4.2" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+ssse3,+sse3,+popcnt,+sse,+sse2,+sse4.1,+sse4.2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+ssse3,+sse3,+popcnt,+sse,+sse2,+sse4.1,+sse4.2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+ssse3,+sse3,+popcnt,+sse,+sse2,+sse4.1,+sse4.2" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind readnone }
diff --git a/llvm/test/CodeGen/X86/fp128-libcalls-strict.ll b/llvm/test/CodeGen/X86/fp128-libcalls-strict.ll
new file mode 100644
index 000000000..b705c7602
--- /dev/null
+++ b/llvm/test/CodeGen/X86/fp128-libcalls-strict.ll
@@ -0,0 +1,1420 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -O2 -mtriple=x86_64-linux-android \
+; RUN:     -enable-legalize-types-checking \
+; RUN:     | FileCheck %s
+; RUN: llc < %s -O2 -mtriple=x86_64-linux-gnu \
+; RUN:     -enable-legalize-types-checking \
+; RUN:     | FileCheck %s
+; RUN: llc < %s -O2 -mtriple=i686-linux-gnu -mattr=+sse2 \
+; RUN:     -enable-legalize-types-checking \
+; RUN:     | FileCheck %s --check-prefix=X86
+
+; Check all soft floating point library function calls.
+
+define fp128 @add(fp128 %x, fp128 %y) nounwind strictfp {
+; CHECK-LABEL: add:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    callq __addtf3
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    retq
+;
+; X86-LABEL: add:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $20, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    calll __addtf3
+; X86-NEXT:    addl $44, %esp
+; X86-NEXT:    movl (%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl %edi, 8(%esi)
+; X86-NEXT:    movl %edx, 12(%esi)
+; X86-NEXT:    movl %eax, (%esi)
+; X86-NEXT:    movl %ecx, 4(%esi)
+; X86-NEXT:    movl %esi, %eax
+; X86-NEXT:    addl $20, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    retl $4
+entry:
+  %add = call fp128 @llvm.experimental.constrained.fadd.f128(fp128 %x, fp128 %y,  metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret fp128 %add
+}
+
+define fp128 @sub(fp128 %x, fp128 %y) nounwind strictfp {
+; CHECK-LABEL: sub:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    callq __subtf3
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    retq
+;
+; X86-LABEL: sub:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $20, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    calll __subtf3
+; X86-NEXT:    addl $44, %esp
+; X86-NEXT:    movl (%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl %edi, 8(%esi)
+; X86-NEXT:    movl %edx, 12(%esi)
+; X86-NEXT:    movl %eax, (%esi)
+; X86-NEXT:    movl %ecx, 4(%esi)
+; X86-NEXT:    movl %esi, %eax
+; X86-NEXT:    addl $20, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    retl $4
+entry:
+  %sub = call fp128 @llvm.experimental.constrained.fsub.f128(fp128 %x, fp128 %y,  metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret fp128 %sub
+}
+
+define fp128 @mul(fp128 %x, fp128 %y) nounwind strictfp {
+; CHECK-LABEL: mul:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    callq __multf3
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    retq
+;
+; X86-LABEL: mul:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $20, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    calll __multf3
+; X86-NEXT:    addl $44, %esp
+; X86-NEXT:    movl (%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl %edi, 8(%esi)
+; X86-NEXT:    movl %edx, 12(%esi)
+; X86-NEXT:    movl %eax, (%esi)
+; X86-NEXT:    movl %ecx, 4(%esi)
+; X86-NEXT:    movl %esi, %eax
+; X86-NEXT:    addl $20, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    retl $4
+entry:
+  %mul = call fp128 @llvm.experimental.constrained.fmul.f128(fp128 %x, fp128 %y,  metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret fp128 %mul
+}
+
+define fp128 @div(fp128 %x, fp128 %y) nounwind strictfp {
+; CHECK-LABEL: div:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    callq __divtf3
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    retq
+;
+; X86-LABEL: div:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $20, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    calll __divtf3
+; X86-NEXT:    addl $44, %esp
+; X86-NEXT:    movl (%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl %edi, 8(%esi)
+; X86-NEXT:    movl %edx, 12(%esi)
+; X86-NEXT:    movl %eax, (%esi)
+; X86-NEXT:    movl %ecx, 4(%esi)
+; X86-NEXT:    movl %esi, %eax
+; X86-NEXT:    addl $20, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    retl $4
+entry:
+  %div = call fp128 @llvm.experimental.constrained.fdiv.f128(fp128 %x, fp128 %y,  metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret fp128 %div
+}
+
+define fp128 @fma(fp128 %x, fp128 %y, fp128 %z) nounwind strictfp {
+; CHECK-LABEL: fma:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    callq fmal
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    retq
+;
+; X86-LABEL: fma:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $20, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    calll fmal
+; X86-NEXT:    addl $60, %esp
+; X86-NEXT:    movl (%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl %edi, 8(%esi)
+; X86-NEXT:    movl %edx, 12(%esi)
+; X86-NEXT:    movl %eax, (%esi)
+; X86-NEXT:    movl %ecx, 4(%esi)
+; X86-NEXT:    movl %esi, %eax
+; X86-NEXT:    addl $20, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    retl $4
+entry:
+  %fma = call fp128 @llvm.experimental.constrained.fma.f128(fp128 %x, fp128 %y,  fp128 %z, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret fp128 %fma
+}
+
+define fp128 @frem(fp128 %x, fp128 %y) nounwind strictfp {
+; CHECK-LABEL: frem:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    callq fmodl
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    retq
+;
+; X86-LABEL: frem:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $20, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    calll fmodl
+; X86-NEXT:    addl $44, %esp
+; X86-NEXT:    movl (%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl %edi, 8(%esi)
+; X86-NEXT:    movl %edx, 12(%esi)
+; X86-NEXT:    movl %eax, (%esi)
+; X86-NEXT:    movl %ecx, 4(%esi)
+; X86-NEXT:    movl %esi, %eax
+; X86-NEXT:    addl $20, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    retl $4
+entry:
+  %div = call fp128 @llvm.experimental.constrained.frem.f128(fp128 %x, fp128 %y,  metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret fp128 %div
+}
+
+define fp128 @ceil(fp128 %x) nounwind strictfp {
+; CHECK-LABEL: ceil:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    callq ceill
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    retq
+;
+; X86-LABEL: ceil:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $20, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    calll ceill
+; X86-NEXT:    addl $28, %esp
+; X86-NEXT:    movl (%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl %edi, 8(%esi)
+; X86-NEXT:    movl %edx, 12(%esi)
+; X86-NEXT:    movl %eax, (%esi)
+; X86-NEXT:    movl %ecx, 4(%esi)
+; X86-NEXT:    movl %esi, %eax
+; X86-NEXT:    addl $20, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    retl $4
+entry:
+  %ceil = call fp128 @llvm.experimental.constrained.ceil.f128(fp128 %x, metadata !"fpexcept.strict") #0
+  ret fp128 %ceil
+}
+
+define fp128 @cos(fp128 %x) nounwind strictfp {
+; CHECK-LABEL: cos:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    callq cosl
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    retq
+;
+; X86-LABEL: cos:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $20, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    calll cosl
+; X86-NEXT:    addl $28, %esp
+; X86-NEXT:    movl (%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl %edi, 8(%esi)
+; X86-NEXT:    movl %edx, 12(%esi)
+; X86-NEXT:    movl %eax, (%esi)
+; X86-NEXT:    movl %ecx, 4(%esi)
+; X86-NEXT:    movl %esi, %eax
+; X86-NEXT:    addl $20, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    retl $4
+entry:
+  %cos = call fp128 @llvm.experimental.constrained.cos.f128(fp128 %x, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret fp128 %cos
+}
+
+define fp128 @exp(fp128 %x) nounwind strictfp {
+; CHECK-LABEL: exp:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    callq expl
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    retq
+;
+; X86-LABEL: exp:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $20, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    calll expl
+; X86-NEXT:    addl $28, %esp
+; X86-NEXT:    movl (%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl %edi, 8(%esi)
+; X86-NEXT:    movl %edx, 12(%esi)
+; X86-NEXT:    movl %eax, (%esi)
+; X86-NEXT:    movl %ecx, 4(%esi)
+; X86-NEXT:    movl %esi, %eax
+; X86-NEXT:    addl $20, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    retl $4
+entry:
+  %exp = call fp128 @llvm.experimental.constrained.exp.f128(fp128 %x, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret fp128 %exp
+}
+
+define fp128 @exp2(fp128 %x) nounwind strictfp {
+; CHECK-LABEL: exp2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    callq exp2l
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    retq
+;
+; X86-LABEL: exp2:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $20, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    calll exp2l
+; X86-NEXT:    addl $28, %esp
+; X86-NEXT:    movl (%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl %edi, 8(%esi)
+; X86-NEXT:    movl %edx, 12(%esi)
+; X86-NEXT:    movl %eax, (%esi)
+; X86-NEXT:    movl %ecx, 4(%esi)
+; X86-NEXT:    movl %esi, %eax
+; X86-NEXT:    addl $20, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    retl $4
+entry:
+  %exp2 = call fp128 @llvm.experimental.constrained.exp2.f128(fp128 %x, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret fp128 %exp2
+}
+
+define fp128 @floor(fp128 %x) nounwind strictfp {
+; CHECK-LABEL: floor:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    callq floorl
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    retq
+;
+; X86-LABEL: floor:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $20, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    calll floorl
+; X86-NEXT:    addl $28, %esp
+; X86-NEXT:    movl (%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl %edi, 8(%esi)
+; X86-NEXT:    movl %edx, 12(%esi)
+; X86-NEXT:    movl %eax, (%esi)
+; X86-NEXT:    movl %ecx, 4(%esi)
+; X86-NEXT:    movl %esi, %eax
+; X86-NEXT:    addl $20, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    retl $4
+entry:
+  %floor = call fp128 @llvm.experimental.constrained.floor.f128(fp128 %x, metadata !"fpexcept.strict") #0
+  ret fp128 %floor
+}
+
+define fp128 @log(fp128 %x) nounwind strictfp {
+; CHECK-LABEL: log:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    callq logl
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    retq
+;
+; X86-LABEL: log:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $20, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    calll logl
+; X86-NEXT:    addl $28, %esp
+; X86-NEXT:    movl (%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl %edi, 8(%esi)
+; X86-NEXT:    movl %edx, 12(%esi)
+; X86-NEXT:    movl %eax, (%esi)
+; X86-NEXT:    movl %ecx, 4(%esi)
+; X86-NEXT:    movl %esi, %eax
+; X86-NEXT:    addl $20, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    retl $4
+entry:
+  %log = call fp128 @llvm.experimental.constrained.log.f128(fp128 %x, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret fp128 %log
+}
+
+define fp128 @log10(fp128 %x) nounwind strictfp {
+; CHECK-LABEL: log10:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    callq log10l
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    retq
+;
+; X86-LABEL: log10:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $20, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    calll log10l
+; X86-NEXT:    addl $28, %esp
+; X86-NEXT:    movl (%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl %edi, 8(%esi)
+; X86-NEXT:    movl %edx, 12(%esi)
+; X86-NEXT:    movl %eax, (%esi)
+; X86-NEXT:    movl %ecx, 4(%esi)
+; X86-NEXT:    movl %esi, %eax
+; X86-NEXT:    addl $20, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    retl $4
+entry:
+  %log10 = call fp128 @llvm.experimental.constrained.log10.f128(fp128 %x, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret fp128 %log10
+}
+
+define fp128 @log2(fp128 %x) nounwind strictfp {
+; CHECK-LABEL: log2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    callq log2l
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    retq
+;
+; X86-LABEL: log2:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $20, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    calll log2l
+; X86-NEXT:    addl $28, %esp
+; X86-NEXT:    movl (%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl %edi, 8(%esi)
+; X86-NEXT:    movl %edx, 12(%esi)
+; X86-NEXT:    movl %eax, (%esi)
+; X86-NEXT:    movl %ecx, 4(%esi)
+; X86-NEXT:    movl %esi, %eax
+; X86-NEXT:    addl $20, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    retl $4
+entry:
+  %log2 = call fp128 @llvm.experimental.constrained.log2.f128(fp128 %x, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret fp128 %log2
+}
+
+define fp128 @maxnum(fp128 %x, fp128 %y) nounwind strictfp {
+; CHECK-LABEL: maxnum:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    callq fmaxl
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    retq
+;
+; X86-LABEL: maxnum:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $20, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    calll fmaxl
+; X86-NEXT:    addl $44, %esp
+; X86-NEXT:    movl (%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl %edi, 8(%esi)
+; X86-NEXT:    movl %edx, 12(%esi)
+; X86-NEXT:    movl %eax, (%esi)
+; X86-NEXT:    movl %ecx, 4(%esi)
+; X86-NEXT:    movl %esi, %eax
+; X86-NEXT:    addl $20, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    retl $4
+entry:
+  %maxnum = call fp128 @llvm.experimental.constrained.maxnum.f128(fp128 %x, fp128 %y, metadata !"fpexcept.strict") #0
+  ret fp128 %maxnum
+}
+
+define fp128 @minnum(fp128 %x, fp128 %y) nounwind strictfp {
+; CHECK-LABEL: minnum:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    callq fminl
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    retq
+;
+; X86-LABEL: minnum:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $20, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    calll fminl
+; X86-NEXT:    addl $44, %esp
+; X86-NEXT:    movl (%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl %edi, 8(%esi)
+; X86-NEXT:    movl %edx, 12(%esi)
+; X86-NEXT:    movl %eax, (%esi)
+; X86-NEXT:    movl %ecx, 4(%esi)
+; X86-NEXT:    movl %esi, %eax
+; X86-NEXT:    addl $20, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    retl $4
+entry:
+  %minnum = call fp128 @llvm.experimental.constrained.minnum.f128(fp128 %x, fp128 %y, metadata !"fpexcept.strict") #0
+  ret fp128 %minnum
+}
+
+define fp128 @nearbyint(fp128 %x) nounwind strictfp {
+; CHECK-LABEL: nearbyint:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    callq nearbyintl
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    retq
+;
+; X86-LABEL: nearbyint:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $20, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    calll nearbyintl
+; X86-NEXT:    addl $28, %esp
+; X86-NEXT:    movl (%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl %edi, 8(%esi)
+; X86-NEXT:    movl %edx, 12(%esi)
+; X86-NEXT:    movl %eax, (%esi)
+; X86-NEXT:    movl %ecx, 4(%esi)
+; X86-NEXT:    movl %esi, %eax
+; X86-NEXT:    addl $20, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    retl $4
+entry:
+  %nearbyint = call fp128 @llvm.experimental.constrained.nearbyint.f128(fp128 %x, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret fp128 %nearbyint
+}
+
+define fp128 @pow(fp128 %x, fp128 %y) nounwind strictfp {
+; CHECK-LABEL: pow:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    callq powl
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    retq
+;
+; X86-LABEL: pow:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $20, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    calll powl
+; X86-NEXT:    addl $44, %esp
+; X86-NEXT:    movl (%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl %edi, 8(%esi)
+; X86-NEXT:    movl %edx, 12(%esi)
+; X86-NEXT:    movl %eax, (%esi)
+; X86-NEXT:    movl %ecx, 4(%esi)
+; X86-NEXT:    movl %esi, %eax
+; X86-NEXT:    addl $20, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    retl $4
+entry:
+  %pow = call fp128 @llvm.experimental.constrained.pow.f128(fp128 %x, fp128 %y, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret fp128 %pow
+}
+
+define fp128 @powi(fp128 %x, i32 %y) nounwind strictfp {
+; CHECK-LABEL: powi:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    callq __powitf2
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    retq
+;
+; X86-LABEL: powi:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $20, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    subl $8, %esp
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    calll __powitf2
+; X86-NEXT:    addl $28, %esp
+; X86-NEXT:    movl (%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl %edi, 8(%esi)
+; X86-NEXT:    movl %edx, 12(%esi)
+; X86-NEXT:    movl %eax, (%esi)
+; X86-NEXT:    movl %ecx, 4(%esi)
+; X86-NEXT:    movl %esi, %eax
+; X86-NEXT:    addl $20, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    retl $4
+entry:
+  %powi = call fp128 @llvm.experimental.constrained.powi.f128(fp128 %x, i32 %y, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret fp128 %powi
+}
+
+define fp128 @rint(fp128 %x) nounwind strictfp {
+; CHECK-LABEL: rint:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    callq rintl
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    retq
+;
+; X86-LABEL: rint:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $20, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    calll rintl
+; X86-NEXT:    addl $28, %esp
+; X86-NEXT:    movl (%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl %edi, 8(%esi)
+; X86-NEXT:    movl %edx, 12(%esi)
+; X86-NEXT:    movl %eax, (%esi)
+; X86-NEXT:    movl %ecx, 4(%esi)
+; X86-NEXT:    movl %esi, %eax
+; X86-NEXT:    addl $20, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    retl $4
+entry:
+  %rint = call fp128 @llvm.experimental.constrained.rint.f128(fp128 %x, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret fp128 %rint
+}
+
+define fp128 @round(fp128 %x) nounwind strictfp {
+; CHECK-LABEL: round:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    callq roundl
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    retq
+;
+; X86-LABEL: round:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $20, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    calll roundl
+; X86-NEXT:    addl $28, %esp
+; X86-NEXT:    movl (%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl %edi, 8(%esi)
+; X86-NEXT:    movl %edx, 12(%esi)
+; X86-NEXT:    movl %eax, (%esi)
+; X86-NEXT:    movl %ecx, 4(%esi)
+; X86-NEXT:    movl %esi, %eax
+; X86-NEXT:    addl $20, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    retl $4
+entry:
+  %round = call fp128 @llvm.experimental.constrained.round.f128(fp128 %x, metadata !"fpexcept.strict") #0
+  ret fp128 %round
+}
+
+define fp128 @sin(fp128 %x) nounwind strictfp {
+; CHECK-LABEL: sin:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    callq sinl
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    retq
+;
+; X86-LABEL: sin:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $20, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    calll sinl
+; X86-NEXT:    addl $28, %esp
+; X86-NEXT:    movl (%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl %edi, 8(%esi)
+; X86-NEXT:    movl %edx, 12(%esi)
+; X86-NEXT:    movl %eax, (%esi)
+; X86-NEXT:    movl %ecx, 4(%esi)
+; X86-NEXT:    movl %esi, %eax
+; X86-NEXT:    addl $20, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    retl $4
+entry:
+  %sin = call fp128 @llvm.experimental.constrained.sin.f128(fp128 %x, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret fp128 %sin
+}
+
+define fp128 @sqrt(fp128 %x) nounwind strictfp {
+; CHECK-LABEL: sqrt:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    callq sqrtl
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    retq
+;
+; X86-LABEL: sqrt:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $20, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    calll sqrtl
+; X86-NEXT:    addl $28, %esp
+; X86-NEXT:    movl (%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl %edi, 8(%esi)
+; X86-NEXT:    movl %edx, 12(%esi)
+; X86-NEXT:    movl %eax, (%esi)
+; X86-NEXT:    movl %ecx, 4(%esi)
+; X86-NEXT:    movl %esi, %eax
+; X86-NEXT:    addl $20, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    retl $4
+entry:
+  %sqrt = call fp128 @llvm.experimental.constrained.sqrt.f128(fp128 %x, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret fp128 %sqrt
+}
+
+define fp128 @trunc(fp128 %x) nounwind strictfp {
+; CHECK-LABEL: trunc:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    callq truncl
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    retq
+;
+; X86-LABEL: trunc:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $20, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    calll truncl
+; X86-NEXT:    addl $28, %esp
+; X86-NEXT:    movl (%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl %edi, 8(%esi)
+; X86-NEXT:    movl %edx, 12(%esi)
+; X86-NEXT:    movl %eax, (%esi)
+; X86-NEXT:    movl %ecx, 4(%esi)
+; X86-NEXT:    movl %esi, %eax
+; X86-NEXT:    addl $20, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    retl $4
+entry:
+  %trunc = call fp128 @llvm.experimental.constrained.trunc.f128(fp128 %x, metadata !"fpexcept.strict") #0
+  ret fp128 %trunc
+}
+
+define i32 @lrint(fp128 %x) nounwind strictfp {
+; CHECK-LABEL: lrint:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    callq lrintl
+; CHECK-NEXT:    popq %rcx
+; CHECK-NEXT:    retq
+;
+; X86-LABEL: lrint:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll lrintl
+; X86-NEXT:    addl $28, %esp
+; X86-NEXT:    retl
+entry:
+  %rint = call i32 @llvm.experimental.constrained.lrint.i32.f128(fp128 %x, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret i32 %rint
+}
+
+define i64 @llrint(fp128 %x) nounwind strictfp {
+; CHECK-LABEL: llrint:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    callq llrintl
+; CHECK-NEXT:    popq %rcx
+; CHECK-NEXT:    retq
+;
+; X86-LABEL: llrint:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll llrintl
+; X86-NEXT:    addl $28, %esp
+; X86-NEXT:    retl
+entry:
+  %rint = call i64 @llvm.experimental.constrained.llrint.i64.f128(fp128 %x, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret i64 %rint
+}
+
+define i32 @lround(fp128 %x) nounwind strictfp {
+; CHECK-LABEL: lround:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    callq lroundl
+; CHECK-NEXT:    popq %rcx
+; CHECK-NEXT:    retq
+;
+; X86-LABEL: lround:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll lroundl
+; X86-NEXT:    addl $28, %esp
+; X86-NEXT:    retl
+entry:
+  %round = call i32 @llvm.experimental.constrained.lround.i32.f128(fp128 %x, metadata !"fpexcept.strict") #0
+  ret i32 %round
+}
+
+define i64 @llround(fp128 %x) nounwind strictfp {
+; CHECK-LABEL: llround:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    callq llroundl
+; CHECK-NEXT:    popq %rcx
+; CHECK-NEXT:    retq
+;
+; X86-LABEL: llround:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll llroundl
+; X86-NEXT:    addl $28, %esp
+; X86-NEXT:    retl
+entry:
+  %round = call i64 @llvm.experimental.constrained.llround.i64.f128(fp128 %x, metadata !"fpexcept.strict") #0
+  ret i64 %round
+}
+
+define i64 @cmp(i64 %a, i64 %b, fp128 %x, fp128 %y) #0 {
+; CHECK-LABEL: cmp:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    movq %rsi, %r14
+; CHECK-NEXT:    movq %rdi, %rbx
+; CHECK-NEXT:    callq __eqtf2
+; CHECK-NEXT:    testl %eax, %eax
+; CHECK-NEXT:    cmovneq %r14, %rbx
+; CHECK-NEXT:    movq %rbx, %rax
+; CHECK-NEXT:    addq $8, %rsp
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    retq
+;
+; X86-LABEL: cmp:
+; X86:       # %bb.0:
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll __eqtf2
+; X86-NEXT:    addl $32, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    cmovel %eax, %ecx
+; X86-NEXT:    movl (%ecx), %eax
+; X86-NEXT:    movl 4(%ecx), %edx
+; X86-NEXT:    addl $12, %esp
+; X86-NEXT:    retl
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f128(
+                                               fp128 %x, fp128 %y,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+define i64 @cmps(i64 %a, i64 %b, fp128 %x, fp128 %y) #0 {
+; CHECK-LABEL: cmps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    movq %rsi, %r14
+; CHECK-NEXT:    movq %rdi, %rbx
+; CHECK-NEXT:    callq __eqtf2
+; CHECK-NEXT:    testl %eax, %eax
+; CHECK-NEXT:    cmovneq %r14, %rbx
+; CHECK-NEXT:    movq %rbx, %rax
+; CHECK-NEXT:    addq $8, %rsp
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    retq
+;
+; X86-LABEL: cmps:
+; X86:       # %bb.0:
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll __eqtf2
+; X86-NEXT:    addl $32, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    cmovel %eax, %ecx
+; X86-NEXT:    movl (%ecx), %eax
+; X86-NEXT:    movl 4(%ecx), %edx
+; X86-NEXT:    addl $12, %esp
+; X86-NEXT:    retl
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f128(
+                                               fp128 %x, fp128 %y,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+define i64 @cmp_ueq_q(i64 %a, i64 %b, fp128 %x, fp128 %y) #0 {
+; CHECK-LABEL: cmp_ueq_q:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    subq $32, %rsp
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
+; CHECK-NEXT:    movq %rsi, %r14
+; CHECK-NEXT:    movq %rdi, %rbx
+; CHECK-NEXT:    callq __eqtf2
+; CHECK-NEXT:    testl %eax, %eax
+; CHECK-NEXT:    sete %bpl
+; CHECK-NEXT:    movaps (%rsp), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    callq __unordtf2
+; CHECK-NEXT:    testl %eax, %eax
+; CHECK-NEXT:    setne %al
+; CHECK-NEXT:    orb %bpl, %al
+; CHECK-NEXT:    cmoveq %r14, %rbx
+; CHECK-NEXT:    movq %rbx, %rax
+; CHECK-NEXT:    addq $32, %rsp
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    retq
+;
+; X86-LABEL: cmp_ueq_q:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    movl %ebx, %esi
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll __eqtf2
+; X86-NEXT:    addl $32, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    sete %bl
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll __unordtf2
+; X86-NEXT:    addl $32, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    setne %al
+; X86-NEXT:    orb %bl, %al
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    cmovnel %eax, %ecx
+; X86-NEXT:    movl (%ecx), %eax
+; X86-NEXT:    movl 4(%ecx), %edx
+; X86-NEXT:    addl $12, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f128(
+                                               fp128 %x, fp128 %y,
+                                               metadata !"ueq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+define i64 @cmp_one_q(i64 %a, i64 %b, fp128 %x, fp128 %y) #0 {
+; CHECK-LABEL: cmp_one_q:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    subq $32, %rsp
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
+; CHECK-NEXT:    movq %rsi, %r14
+; CHECK-NEXT:    movq %rdi, %rbx
+; CHECK-NEXT:    callq __eqtf2
+; CHECK-NEXT:    testl %eax, %eax
+; CHECK-NEXT:    setne %bpl
+; CHECK-NEXT:    movaps (%rsp), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    callq __unordtf2
+; CHECK-NEXT:    testl %eax, %eax
+; CHECK-NEXT:    sete %al
+; CHECK-NEXT:    testb %bpl, %al
+; CHECK-NEXT:    cmoveq %r14, %rbx
+; CHECK-NEXT:    movq %rbx, %rax
+; CHECK-NEXT:    addq $32, %rsp
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    retq
+;
+; X86-LABEL: cmp_one_q:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    movl %ebx, %esi
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll __eqtf2
+; X86-NEXT:    addl $32, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    setne %bl
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll __unordtf2
+; X86-NEXT:    addl $32, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    sete %al
+; X86-NEXT:    testb %bl, %al
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    cmovnel %eax, %ecx
+; X86-NEXT:    movl (%ecx), %eax
+; X86-NEXT:    movl 4(%ecx), %edx
+; X86-NEXT:    addl $12, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f128(
+                                               fp128 %x, fp128 %y,
+                                               metadata !"one",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+attributes #0 = { nounwind strictfp }
+
+declare fp128 @llvm.experimental.constrained.fadd.f128(fp128, fp128, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.fsub.f128(fp128, fp128, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.fmul.f128(fp128, fp128, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.fdiv.f128(fp128, fp128, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.fma.f128(fp128, fp128, fp128, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.frem.f128(fp128, fp128, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.ceil.f128(fp128, metadata)
+declare fp128 @llvm.experimental.constrained.cos.f128(fp128, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.exp.f128(fp128, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.exp2.f128(fp128, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.floor.f128(fp128, metadata)
+declare fp128 @llvm.experimental.constrained.log.f128(fp128, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.log10.f128(fp128, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.log2.f128(fp128, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.maxnum.f128(fp128, fp128, metadata)
+declare fp128 @llvm.experimental.constrained.minnum.f128(fp128, fp128, metadata)
+declare fp128 @llvm.experimental.constrained.nearbyint.f128(fp128, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.pow.f128(fp128, fp128, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.powi.f128(fp128, i32, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.rint.f128(fp128, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.round.f128(fp128, metadata)
+declare fp128 @llvm.experimental.constrained.sin.f128(fp128, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.sqrt.f128(fp128, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.trunc.f128(fp128, metadata)
+declare i32 @llvm.experimental.constrained.lrint.i32.f128(fp128, metadata, metadata)
+declare i64 @llvm.experimental.constrained.llrint.i64.f128(fp128, metadata, metadata)
+declare i32 @llvm.experimental.constrained.lround.i32.f128(fp128, metadata)
+declare i64 @llvm.experimental.constrained.llround.i64.f128(fp128, metadata)
+declare i1 @llvm.experimental.constrained.fcmp.f128(fp128, fp128, metadata, metadata)
+declare i1 @llvm.experimental.constrained.fcmps.f128(fp128, fp128, metadata, metadata)
diff --git a/llvm/test/CodeGen/X86/fp128-libcalls.ll b/llvm/test/CodeGen/X86/fp128-libcalls.ll
index 09bda890f..adc8dc2d3 100644
--- a/llvm/test/CodeGen/X86/fp128-libcalls.ll
+++ b/llvm/test/CodeGen/X86/fp128-libcalls.ll
@@ -1,109 +1,845 @@
-; RUN: llc < %s -O2 -mtriple=x86_64-linux-android -mattr=+mmx \
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -O2 -mtriple=x86_64-linux-android \
 ; RUN:     -enable-legalize-types-checking | FileCheck %s
-; RUN: llc < %s -O2 -mtriple=x86_64-linux-gnu -mattr=+mmx \
+; RUN: llc < %s -O2 -mtriple=x86_64-linux-gnu \
 ; RUN:     -enable-legalize-types-checking | FileCheck %s
+; RUN: llc < %s -O2 -mtriple=i686-linux-gnu -mattr=sse2 \
+; RUN:     -enable-legalize-types-checking | FileCheck %s --check-prefix=X86
 
 ; Check all soft floating point library function calls.
 
 @vf64 = common global double 0.000000e+00, align 8
 @vf128 = common global fp128 0xL00000000000000000000000000000000, align 16
 
-define void @Test128Add(fp128 %d1, fp128 %d2) {
+define void @Test128Add(fp128 %d1, fp128 %d2) nounwind {
+; CHECK-LABEL: Test128Add:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    callq __addtf3
+; CHECK-NEXT:    movaps %xmm0, {{.*}}(%rip)
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    retq
+;
+; X86-LABEL: Test128Add:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $36, %esp
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    calll __addtf3
+; X86-NEXT:    addl $44, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl %esi, vf128+12
+; X86-NEXT:    movl %edx, vf128+8
+; X86-NEXT:    movl %ecx, vf128+4
+; X86-NEXT:    movl %eax, vf128
+; X86-NEXT:    addl $24, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
 entry:
   %add = fadd fp128 %d1, %d2
   store fp128 %add, fp128* @vf128, align 16
   ret void
-; CHECK-LABEL: Test128Add:
-; CHECK:       callq __addtf3
-; CHECK-NEXT:  movaps %xmm0, vf128(%rip)
-; CHECK:       retq
 }
 
-define void @Test128_1Add(fp128 %d1){
+define void @Test128_1Add(fp128 %d1) nounwind {
+; CHECK-LABEL: Test128_1Add:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    movaps %xmm0, %xmm1
+; CHECK-NEXT:    movaps {{.*}}(%rip), %xmm0
+; CHECK-NEXT:    callq __addtf3
+; CHECK-NEXT:    movaps %xmm0, {{.*}}(%rip)
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    retq
+;
+; X86-LABEL: Test128_1Add:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $36, %esp
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl vf128+12
+; X86-NEXT:    pushl vf128+8
+; X86-NEXT:    pushl vf128+4
+; X86-NEXT:    pushl vf128
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    calll __addtf3
+; X86-NEXT:    addl $44, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl %esi, vf128+8
+; X86-NEXT:    movl %edx, vf128+12
+; X86-NEXT:    movl %eax, vf128
+; X86-NEXT:    movl %ecx, vf128+4
+; X86-NEXT:    addl $24, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
 entry:
   %0 = load fp128, fp128* @vf128, align 16
   %add = fadd fp128 %0, %d1
   store fp128 %add, fp128* @vf128, align 16
   ret void
-; CHECK-LABEL: Test128_1Add:
-; CHECK:       movaps  %xmm0, %xmm1
-; CHECK-NEXT:  movaps  vf128(%rip), %xmm0
-; CHECK-NEXT:  callq   __addtf3
-; CHECK-NEXT:  movaps  %xmm0, vf128(%rip)
-; CHECK:       retq
 }
 
-define void @Test128Sub(fp128 %d1, fp128 %d2){
+define void @Test128Sub(fp128 %d1, fp128 %d2) nounwind {
+; CHECK-LABEL: Test128Sub:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    callq __subtf3
+; CHECK-NEXT:    movaps %xmm0, {{.*}}(%rip)
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    retq
+;
+; X86-LABEL: Test128Sub:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $36, %esp
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    calll __subtf3
+; X86-NEXT:    addl $44, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl %esi, vf128+12
+; X86-NEXT:    movl %edx, vf128+8
+; X86-NEXT:    movl %ecx, vf128+4
+; X86-NEXT:    movl %eax, vf128
+; X86-NEXT:    addl $24, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
 entry:
   %sub = fsub fp128 %d1, %d2
   store fp128 %sub, fp128* @vf128, align 16
   ret void
-; CHECK-LABEL: Test128Sub:
-; CHECK:       callq __subtf3
-; CHECK-NEXT:  movaps %xmm0, vf128(%rip)
-; CHECK:       retq
 }
 
-define void @Test128_1Sub(fp128 %d1){
+define void @Test128_1Sub(fp128 %d1) nounwind {
+; CHECK-LABEL: Test128_1Sub:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    movaps %xmm0, %xmm1
+; CHECK-NEXT:    movaps {{.*}}(%rip), %xmm0
+; CHECK-NEXT:    callq __subtf3
+; CHECK-NEXT:    movaps %xmm0, {{.*}}(%rip)
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    retq
+;
+; X86-LABEL: Test128_1Sub:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $36, %esp
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl vf128+12
+; X86-NEXT:    pushl vf128+8
+; X86-NEXT:    pushl vf128+4
+; X86-NEXT:    pushl vf128
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    calll __subtf3
+; X86-NEXT:    addl $44, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl %esi, vf128+8
+; X86-NEXT:    movl %edx, vf128+12
+; X86-NEXT:    movl %eax, vf128
+; X86-NEXT:    movl %ecx, vf128+4
+; X86-NEXT:    addl $24, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
 entry:
   %0 = load fp128, fp128* @vf128, align 16
   %sub = fsub fp128 %0, %d1
   store fp128 %sub, fp128* @vf128, align 16
   ret void
-; CHECK-LABEL: Test128_1Sub:
-; CHECK:       movaps  %xmm0, %xmm1
-; CHECK-NEXT:  movaps  vf128(%rip), %xmm0
-; CHECK-NEXT:  callq   __subtf3
-; CHECK-NEXT:  movaps  %xmm0, vf128(%rip)
-; CHECK:       retq
 }
 
-define void @Test128Mul(fp128 %d1, fp128 %d2){
+define void @Test128Mul(fp128 %d1, fp128 %d2) nounwind {
+; CHECK-LABEL: Test128Mul:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    callq __multf3
+; CHECK-NEXT:    movaps %xmm0, {{.*}}(%rip)
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    retq
+;
+; X86-LABEL: Test128Mul:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $36, %esp
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    calll __multf3
+; X86-NEXT:    addl $44, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl %esi, vf128+12
+; X86-NEXT:    movl %edx, vf128+8
+; X86-NEXT:    movl %ecx, vf128+4
+; X86-NEXT:    movl %eax, vf128
+; X86-NEXT:    addl $24, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
 entry:
   %mul = fmul fp128 %d1, %d2
   store fp128 %mul, fp128* @vf128, align 16
   ret void
-; CHECK-LABEL: Test128Mul:
-; CHECK:       callq __multf3
-; CHECK-NEXT:  movaps %xmm0, vf128(%rip)
-; CHECK:       retq
 }
 
-define void @Test128_1Mul(fp128 %d1){
+define void @Test128_1Mul(fp128 %d1) nounwind {
+; CHECK-LABEL: Test128_1Mul:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    movaps %xmm0, %xmm1
+; CHECK-NEXT:    movaps {{.*}}(%rip), %xmm0
+; CHECK-NEXT:    callq __multf3
+; CHECK-NEXT:    movaps %xmm0, {{.*}}(%rip)
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    retq
+;
+; X86-LABEL: Test128_1Mul:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $36, %esp
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl vf128+12
+; X86-NEXT:    pushl vf128+8
+; X86-NEXT:    pushl vf128+4
+; X86-NEXT:    pushl vf128
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    calll __multf3
+; X86-NEXT:    addl $44, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl %esi, vf128+8
+; X86-NEXT:    movl %edx, vf128+12
+; X86-NEXT:    movl %eax, vf128
+; X86-NEXT:    movl %ecx, vf128+4
+; X86-NEXT:    addl $24, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
 entry:
   %0 = load fp128, fp128* @vf128, align 16
   %mul = fmul fp128 %0, %d1
   store fp128 %mul, fp128* @vf128, align 16
   ret void
-; CHECK-LABEL: Test128_1Mul:
-; CHECK:       movaps  %xmm0, %xmm1
-; CHECK-NEXT:  movaps  vf128(%rip), %xmm0
-; CHECK-NEXT:  callq   __multf3
-; CHECK-NEXT:  movaps  %xmm0, vf128(%rip)
-; CHECK:       retq
 }
 
-define void @Test128Div(fp128 %d1, fp128 %d2){
+define void @Test128Div(fp128 %d1, fp128 %d2) nounwind {
+; CHECK-LABEL: Test128Div:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    callq __divtf3
+; CHECK-NEXT:    movaps %xmm0, {{.*}}(%rip)
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    retq
+;
+; X86-LABEL: Test128Div:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $36, %esp
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    calll __divtf3
+; X86-NEXT:    addl $44, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl %esi, vf128+12
+; X86-NEXT:    movl %edx, vf128+8
+; X86-NEXT:    movl %ecx, vf128+4
+; X86-NEXT:    movl %eax, vf128
+; X86-NEXT:    addl $24, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
 entry:
   %div = fdiv fp128 %d1, %d2
   store fp128 %div, fp128* @vf128, align 16
   ret void
-; CHECK-LABEL: Test128Div:
-; CHECK:       callq __divtf3
-; CHECK-NEXT:  movaps %xmm0, vf128(%rip)
-; CHECK:       retq
 }
 
-define void @Test128_1Div(fp128 %d1){
+define void @Test128_1Div(fp128 %d1) nounwind {
+; CHECK-LABEL: Test128_1Div:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    movaps %xmm0, %xmm1
+; CHECK-NEXT:    movaps {{.*}}(%rip), %xmm0
+; CHECK-NEXT:    callq __divtf3
+; CHECK-NEXT:    movaps %xmm0, {{.*}}(%rip)
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    retq
+;
+; X86-LABEL: Test128_1Div:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $36, %esp
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl vf128+12
+; X86-NEXT:    pushl vf128+8
+; X86-NEXT:    pushl vf128+4
+; X86-NEXT:    pushl vf128
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    calll __divtf3
+; X86-NEXT:    addl $44, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl %esi, vf128+8
+; X86-NEXT:    movl %edx, vf128+12
+; X86-NEXT:    movl %eax, vf128
+; X86-NEXT:    movl %ecx, vf128+4
+; X86-NEXT:    addl $24, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
 entry:
   %0 = load fp128, fp128* @vf128, align 16
   %div = fdiv fp128 %0, %d1
   store fp128 %div, fp128* @vf128, align 16
   ret void
-; CHECK-LABEL: Test128_1Div:
-; CHECK:       movaps  %xmm0, %xmm1
-; CHECK-NEXT:  movaps  vf128(%rip), %xmm0
-; CHECK-NEXT:  callq   __divtf3
-; CHECK-NEXT:  movaps  %xmm0, vf128(%rip)
-; CHECK:       retq
 }
+
+define void @Test128Rem(fp128 %d1, fp128 %d2) nounwind {
+; CHECK-LABEL: Test128Rem:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    callq fmodl
+; CHECK-NEXT:    movaps %xmm0, {{.*}}(%rip)
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    retq
+;
+; X86-LABEL: Test128Rem:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $36, %esp
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    calll fmodl
+; X86-NEXT:    addl $44, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl %esi, vf128+12
+; X86-NEXT:    movl %edx, vf128+8
+; X86-NEXT:    movl %ecx, vf128+4
+; X86-NEXT:    movl %eax, vf128
+; X86-NEXT:    addl $24, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
+entry:
+  %div = frem fp128 %d1, %d2
+  store fp128 %div, fp128* @vf128, align 16
+  ret void
+}
+
+define void @Test128_1Rem(fp128 %d1) nounwind {
+; CHECK-LABEL: Test128_1Rem:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    movaps %xmm0, %xmm1
+; CHECK-NEXT:    movaps {{.*}}(%rip), %xmm0
+; CHECK-NEXT:    callq fmodl
+; CHECK-NEXT:    movaps %xmm0, {{.*}}(%rip)
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    retq
+;
+; X86-LABEL: Test128_1Rem:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $36, %esp
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl vf128+12
+; X86-NEXT:    pushl vf128+8
+; X86-NEXT:    pushl vf128+4
+; X86-NEXT:    pushl vf128
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    calll fmodl
+; X86-NEXT:    addl $44, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl %esi, vf128+8
+; X86-NEXT:    movl %edx, vf128+12
+; X86-NEXT:    movl %eax, vf128
+; X86-NEXT:    movl %ecx, vf128+4
+; X86-NEXT:    addl $24, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
+entry:
+  %0 = load fp128, fp128* @vf128, align 16
+  %div = frem fp128 %0, %d1
+  store fp128 %div, fp128* @vf128, align 16
+  ret void
+}
+
+define void @Test128Sqrt(fp128 %d1) nounwind {
+; CHECK-LABEL: Test128Sqrt:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    callq sqrtl
+; CHECK-NEXT:    movaps %xmm0, {{.*}}(%rip)
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    retq
+;
+; X86-LABEL: Test128Sqrt:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $36, %esp
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    calll sqrtl
+; X86-NEXT:    addl $28, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl %esi, vf128+12
+; X86-NEXT:    movl %edx, vf128+8
+; X86-NEXT:    movl %ecx, vf128+4
+; X86-NEXT:    movl %eax, vf128
+; X86-NEXT:    addl $24, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
+entry:
+  %sqrt = call fp128 @llvm.sqrt.f128(fp128 %d1)
+  store fp128 %sqrt, fp128* @vf128, align 16
+  ret void
+}
+declare fp128 @llvm.sqrt.f128(fp128)
+
+define void @Test128Sin(fp128 %d1) nounwind {
+; CHECK-LABEL: Test128Sin:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    callq sinl
+; CHECK-NEXT:    movaps %xmm0, {{.*}}(%rip)
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    retq
+;
+; X86-LABEL: Test128Sin:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $36, %esp
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    calll sinl
+; X86-NEXT:    addl $28, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl %esi, vf128+12
+; X86-NEXT:    movl %edx, vf128+8
+; X86-NEXT:    movl %ecx, vf128+4
+; X86-NEXT:    movl %eax, vf128
+; X86-NEXT:    addl $24, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
+entry:
+  %sqrt = call fp128 @llvm.sin.f128(fp128 %d1)
+  store fp128 %sqrt, fp128* @vf128, align 16
+  ret void
+}
+declare fp128 @llvm.sin.f128(fp128)
+
+define void @Test128Cos(fp128 %d1) nounwind {
+; CHECK-LABEL: Test128Cos:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    callq cosl
+; CHECK-NEXT:    movaps %xmm0, {{.*}}(%rip)
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    retq
+;
+; X86-LABEL: Test128Cos:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $36, %esp
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    calll cosl
+; X86-NEXT:    addl $28, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl %esi, vf128+12
+; X86-NEXT:    movl %edx, vf128+8
+; X86-NEXT:    movl %ecx, vf128+4
+; X86-NEXT:    movl %eax, vf128
+; X86-NEXT:    addl $24, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
+entry:
+  %sqrt = call fp128 @llvm.cos.f128(fp128 %d1)
+  store fp128 %sqrt, fp128* @vf128, align 16
+  ret void
+}
+declare fp128 @llvm.cos.f128(fp128)
+
+define void @Test128Ceil(fp128 %d1) nounwind {
+; CHECK-LABEL: Test128Ceil:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    callq ceill
+; CHECK-NEXT:    movaps %xmm0, {{.*}}(%rip)
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    retq
+;
+; X86-LABEL: Test128Ceil:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $36, %esp
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    calll ceill
+; X86-NEXT:    addl $28, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl %esi, vf128+12
+; X86-NEXT:    movl %edx, vf128+8
+; X86-NEXT:    movl %ecx, vf128+4
+; X86-NEXT:    movl %eax, vf128
+; X86-NEXT:    addl $24, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
+entry:
+  %sqrt = call fp128 @llvm.ceil.f128(fp128 %d1)
+  store fp128 %sqrt, fp128* @vf128, align 16
+  ret void
+}
+declare fp128 @llvm.ceil.f128(fp128)
+
+define void @Test128Floor(fp128 %d1) nounwind {
+; CHECK-LABEL: Test128Floor:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    callq floorl
+; CHECK-NEXT:    movaps %xmm0, {{.*}}(%rip)
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    retq
+;
+; X86-LABEL: Test128Floor:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $36, %esp
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    calll floorl
+; X86-NEXT:    addl $28, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl %esi, vf128+12
+; X86-NEXT:    movl %edx, vf128+8
+; X86-NEXT:    movl %ecx, vf128+4
+; X86-NEXT:    movl %eax, vf128
+; X86-NEXT:    addl $24, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
+entry:
+  %sqrt = call fp128 @llvm.floor.f128(fp128 %d1)
+  store fp128 %sqrt, fp128* @vf128, align 16
+  ret void
+}
+declare fp128 @llvm.floor.f128(fp128)
+
+define void @Test128Trunc(fp128 %d1) nounwind {
+; CHECK-LABEL: Test128Trunc:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    callq truncl
+; CHECK-NEXT:    movaps %xmm0, {{.*}}(%rip)
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    retq
+;
+; X86-LABEL: Test128Trunc:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $36, %esp
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    calll truncl
+; X86-NEXT:    addl $28, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl %esi, vf128+12
+; X86-NEXT:    movl %edx, vf128+8
+; X86-NEXT:    movl %ecx, vf128+4
+; X86-NEXT:    movl %eax, vf128
+; X86-NEXT:    addl $24, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
+entry:
+  %sqrt = call fp128 @llvm.trunc.f128(fp128 %d1)
+  store fp128 %sqrt, fp128* @vf128, align 16
+  ret void
+}
+declare fp128 @llvm.trunc.f128(fp128)
+
+define void @Test128Nearbyint(fp128 %d1) nounwind {
+; CHECK-LABEL: Test128Nearbyint:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    callq nearbyintl
+; CHECK-NEXT:    movaps %xmm0, {{.*}}(%rip)
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    retq
+;
+; X86-LABEL: Test128Nearbyint:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $36, %esp
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    calll nearbyintl
+; X86-NEXT:    addl $28, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl %esi, vf128+12
+; X86-NEXT:    movl %edx, vf128+8
+; X86-NEXT:    movl %ecx, vf128+4
+; X86-NEXT:    movl %eax, vf128
+; X86-NEXT:    addl $24, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
+entry:
+  %sqrt = call fp128 @llvm.nearbyint.f128(fp128 %d1)
+  store fp128 %sqrt, fp128* @vf128, align 16
+  ret void
+}
+declare fp128 @llvm.nearbyint.f128(fp128)
+
+define void @Test128Rint(fp128 %d1) nounwind {
+; CHECK-LABEL: Test128Rint:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    callq rintl
+; CHECK-NEXT:    movaps %xmm0, {{.*}}(%rip)
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    retq
+;
+; X86-LABEL: Test128Rint:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $36, %esp
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    calll rintl
+; X86-NEXT:    addl $28, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl %esi, vf128+12
+; X86-NEXT:    movl %edx, vf128+8
+; X86-NEXT:    movl %ecx, vf128+4
+; X86-NEXT:    movl %eax, vf128
+; X86-NEXT:    addl $24, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
+entry:
+  %sqrt = call fp128 @llvm.rint.f128(fp128 %d1)
+  store fp128 %sqrt, fp128* @vf128, align 16
+  ret void
+}
+declare fp128 @llvm.rint.f128(fp128)
+
+define void @Test128Round(fp128 %d1) nounwind {
+; CHECK-LABEL: Test128Round:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    callq roundl
+; CHECK-NEXT:    movaps %xmm0, {{.*}}(%rip)
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    retq
+;
+; X86-LABEL: Test128Round:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $36, %esp
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    calll roundl
+; X86-NEXT:    addl $28, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl %esi, vf128+12
+; X86-NEXT:    movl %edx, vf128+8
+; X86-NEXT:    movl %ecx, vf128+4
+; X86-NEXT:    movl %eax, vf128
+; X86-NEXT:    addl $24, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
+entry:
+  %sqrt = call fp128 @llvm.round.f128(fp128 %d1)
+  store fp128 %sqrt, fp128* @vf128, align 16
+  ret void
+}
+declare fp128 @llvm.round.f128(fp128)
+
+define fp128 @Test128FMA(fp128 %a, fp128 %b, fp128 %c) nounwind {
+; CHECK-LABEL: Test128FMA:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    jmp fmal # TAILCALL
+;
+; X86-LABEL: Test128FMA:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $20, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    calll fmal
+; X86-NEXT:    addl $60, %esp
+; X86-NEXT:    movl (%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl %edi, 12(%esi)
+; X86-NEXT:    movl %edx, 8(%esi)
+; X86-NEXT:    movl %ecx, 4(%esi)
+; X86-NEXT:    movl %eax, (%esi)
+; X86-NEXT:    movl %esi, %eax
+; X86-NEXT:    addl $20, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    retl $4
+entry:
+  %call = call fp128 @llvm.fma.f128(fp128 %a, fp128 %b, fp128 %c)
+  ret fp128 %call
+}
+declare fp128 @llvm.fma.f128(fp128, fp128, fp128)
diff --git a/llvm/test/CodeGen/X86/fp128-select.ll b/llvm/test/CodeGen/X86/fp128-select.ll
index 503c7a929..65e930e26 100644
--- a/llvm/test/CodeGen/X86/fp128-select.ll
+++ b/llvm/test/CodeGen/X86/fp128-select.ll
@@ -1,39 +1,137 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -O2 -mtriple=x86_64-linux-android -mattr=+mmx \
-; RUN:     -enable-legalize-types-checking | FileCheck %s --check-prefix=MMX
-; RUN: llc < %s -O2 -mtriple=x86_64-linux-gnu -mattr=+mmx \
-; RUN:     -enable-legalize-types-checking | FileCheck %s --check-prefix=MMX
-; RUN: llc < %s -O2 -mtriple=x86_64-linux-android \
-; RUN:     -enable-legalize-types-checking | FileCheck %s
-; RUN: llc < %s -O2 -mtriple=x86_64-linux-gnu \
-; RUN:     -enable-legalize-types-checking | FileCheck %s
+; RUN: llc < %s -O2 -mtriple=x86_64-linux-android -mattr=+sse \
+; RUN:     -enable-legalize-types-checking | FileCheck %s --check-prefix=SSE
+; RUN: llc < %s -O2 -mtriple=x86_64-linux-gnu -mattr=+sse \
+; RUN:     -enable-legalize-types-checking | FileCheck %s --check-prefix=SSE
+; RUN: llc < %s -O2 -mtriple=x86_64-linux-android -mattr=-sse \
+; RUN:     -enable-legalize-types-checking | FileCheck %s --check-prefix=NOSSE
+; RUN: llc < %s -O2 -mtriple=x86_64-linux-gnu -mattr=-sse \
+; RUN:     -enable-legalize-types-checking | FileCheck %s --check-prefix=NOSSE
 
 define void @test_select(fp128* %p, fp128* %q, i1 zeroext %c) {
-; MMX-LABEL: test_select:
-; MMX:       # %bb.0:
-; MMX-NEXT:    testl %edx, %edx
-; MMX-NEXT:    jne .LBB0_1
-; MMX-NEXT:  # %bb.2:
-; MMX-NEXT:    movaps {{.*}}(%rip), %xmm0
-; MMX-NEXT:    movaps %xmm0, (%rsi)
-; MMX-NEXT:    retq
-; MMX-NEXT:  .LBB0_1:
-; MMX-NEXT:    movups (%rdi), %xmm0
-; MMX-NEXT:    movaps %xmm0, (%rsi)
-; MMX-NEXT:    retq
+; SSE-LABEL: test_select:
+; SSE:       # %bb.0:
+; SSE-NEXT:    testl %edx, %edx
+; SSE-NEXT:    jne .LBB0_1
+; SSE-NEXT:  # %bb.2:
+; SSE-NEXT:    movaps {{.*}}(%rip), %xmm0
+; SSE-NEXT:    movaps %xmm0, (%rsi)
+; SSE-NEXT:    retq
+; SSE-NEXT:  .LBB0_1:
+; SSE-NEXT:    movups (%rdi), %xmm0
+; SSE-NEXT:    movaps %xmm0, (%rsi)
+; SSE-NEXT:    retq
 ;
-; CHECK-LABEL: test_select:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    testl %edx, %edx
-; CHECK-NEXT:    cmovneq (%rdi), %rax
-; CHECK-NEXT:    movabsq $9223231299366420480, %rcx # imm = 0x7FFF800000000000
-; CHECK-NEXT:    cmovneq 8(%rdi), %rcx
-; CHECK-NEXT:    movq %rcx, 8(%rsi)
-; CHECK-NEXT:    movq %rax, (%rsi)
-; CHECK-NEXT:    retq
+; NOSSE-LABEL: test_select:
+; NOSSE:       # %bb.0:
+; NOSSE-NEXT:    xorl %eax, %eax
+; NOSSE-NEXT:    testl %edx, %edx
+; NOSSE-NEXT:    cmovneq (%rdi), %rax
+; NOSSE-NEXT:    movabsq $9223231299366420480, %rcx # imm = 0x7FFF800000000000
+; NOSSE-NEXT:    cmovneq 8(%rdi), %rcx
+; NOSSE-NEXT:    movq %rcx, 8(%rsi)
+; NOSSE-NEXT:    movq %rax, (%rsi)
+; NOSSE-NEXT:    retq
   %a = load fp128, fp128* %p, align 2
   %r = select i1 %c, fp128 %a, fp128 0xL00000000000000007FFF800000000000
   store fp128 %r, fp128* %q
   ret void
 }
+
+; The uitofp will become a select_cc. This used to crash during type
+; legalization because we didn't expect the operands to need to be softened.
+define fp128 @test_select_cc(fp128, fp128) {
+; SSE-LABEL: test_select_cc:
+; SSE:       # %bb.0: # %BB0
+; SSE-NEXT:    pushq %rbx
+; SSE-NEXT:    .cfi_def_cfa_offset 16
+; SSE-NEXT:    subq $32, %rsp
+; SSE-NEXT:    .cfi_def_cfa_offset 48
+; SSE-NEXT:    .cfi_offset %rbx, -16
+; SSE-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
+; SSE-NEXT:    callq __netf2
+; SSE-NEXT:    movl %eax, %ebx
+; SSE-NEXT:    movaps (%rsp), %xmm0 # 16-byte Reload
+; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; SSE-NEXT:    callq __eqtf2
+; SSE-NEXT:    testl %eax, %eax
+; SSE-NEXT:    je .LBB1_1
+; SSE-NEXT:  # %bb.2: # %BB0
+; SSE-NEXT:    xorps %xmm1, %xmm1
+; SSE-NEXT:    jmp .LBB1_3
+; SSE-NEXT:  .LBB1_1:
+; SSE-NEXT:    movaps {{.*}}(%rip), %xmm1
+; SSE-NEXT:  .LBB1_3: # %BB0
+; SSE-NEXT:    testl %ebx, %ebx
+; SSE-NEXT:    movaps (%rsp), %xmm0 # 16-byte Reload
+; SSE-NEXT:    jne .LBB1_5
+; SSE-NEXT:  # %bb.4: # %BB1
+; SSE-NEXT:    movaps %xmm1, %xmm0
+; SSE-NEXT:  .LBB1_5: # %BB2
+; SSE-NEXT:    addq $32, %rsp
+; SSE-NEXT:    .cfi_def_cfa_offset 16
+; SSE-NEXT:    popq %rbx
+; SSE-NEXT:    .cfi_def_cfa_offset 8
+; SSE-NEXT:    retq
+;
+; NOSSE-LABEL: test_select_cc:
+; NOSSE:       # %bb.0: # %BB0
+; NOSSE-NEXT:    pushq %rbp
+; NOSSE-NEXT:    .cfi_def_cfa_offset 16
+; NOSSE-NEXT:    pushq %r15
+; NOSSE-NEXT:    .cfi_def_cfa_offset 24
+; NOSSE-NEXT:    pushq %r14
+; NOSSE-NEXT:    .cfi_def_cfa_offset 32
+; NOSSE-NEXT:    pushq %r12
+; NOSSE-NEXT:    .cfi_def_cfa_offset 40
+; NOSSE-NEXT:    pushq %rbx
+; NOSSE-NEXT:    .cfi_def_cfa_offset 48
+; NOSSE-NEXT:    .cfi_offset %rbx, -48
+; NOSSE-NEXT:    .cfi_offset %r12, -40
+; NOSSE-NEXT:    .cfi_offset %r14, -32
+; NOSSE-NEXT:    .cfi_offset %r15, -24
+; NOSSE-NEXT:    .cfi_offset %rbp, -16
+; NOSSE-NEXT:    movq %rcx, %r12
+; NOSSE-NEXT:    movq %rdx, %rbx
+; NOSSE-NEXT:    movq %rsi, %r14
+; NOSSE-NEXT:    movq %rdi, %r15
+; NOSSE-NEXT:    callq __netf2
+; NOSSE-NEXT:    movl %eax, %ebp
+; NOSSE-NEXT:    movq %r15, %rdi
+; NOSSE-NEXT:    movq %r14, %rsi
+; NOSSE-NEXT:    movq %rbx, %rdx
+; NOSSE-NEXT:    movq %r12, %rcx
+; NOSSE-NEXT:    callq __eqtf2
+; NOSSE-NEXT:    movl %eax, %ecx
+; NOSSE-NEXT:    xorl %eax, %eax
+; NOSSE-NEXT:    testl %ecx, %ecx
+; NOSSE-NEXT:    movabsq $4611404543450677248, %rdx # imm = 0x3FFF000000000000
+; NOSSE-NEXT:    cmovneq %rax, %rdx
+; NOSSE-NEXT:    testl %ebp, %ebp
+; NOSSE-NEXT:    je .LBB1_2
+; NOSSE-NEXT:  # %bb.1:
+; NOSSE-NEXT:    movq %r15, %rax
+; NOSSE-NEXT:    movq %r14, %rdx
+; NOSSE-NEXT:  .LBB1_2: # %BB2
+; NOSSE-NEXT:    popq %rbx
+; NOSSE-NEXT:    .cfi_def_cfa_offset 40
+; NOSSE-NEXT:    popq %r12
+; NOSSE-NEXT:    .cfi_def_cfa_offset 32
+; NOSSE-NEXT:    popq %r14
+; NOSSE-NEXT:    .cfi_def_cfa_offset 24
+; NOSSE-NEXT:    popq %r15
+; NOSSE-NEXT:    .cfi_def_cfa_offset 16
+; NOSSE-NEXT:    popq %rbp
+; NOSSE-NEXT:    .cfi_def_cfa_offset 8
+; NOSSE-NEXT:    retq
+BB0:
+  %a = fcmp oeq fp128 %0, %1
+  %b = uitofp i1 %a to fp128
+  br i1 %a, label %BB1, label %BB2
+BB1:
+  br label %BB2
+BB2:
+  %c = phi fp128 [ %0, %BB0 ], [ %b, %BB1 ]
+  ret fp128 %c
+}
diff --git a/llvm/test/CodeGen/X86/fp2sint.ll b/llvm/test/CodeGen/X86/fp2sint.ll
index de5fe4d83..e25649559 100644
--- a/llvm/test/CodeGen/X86/fp2sint.ll
+++ b/llvm/test/CodeGen/X86/fp2sint.ll
@@ -1,8 +1,15 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ;; LowerFP_TO_SINT should not create a stack object if it's not needed.
 
-; RUN: llc < %s -mtriple=i686-- -mattr=+sse2 | not grep add
+; RUN: llc < %s -mtriple=i686-- -mattr=+sse2 | FileCheck %s
 
 define i32 @main(i32 %argc, i8** %argv) {
+; CHECK-LABEL: main:
+; CHECK:       # %bb.0: # %cond_false.i.i.i
+; CHECK-NEXT:    cvttsd2si 0, %eax
+; CHECK-NEXT:    movl %eax, 0
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    retl
 cond_false.i.i.i:               ; preds = %bb.i5
        %tmp35.i = load double, double* null, align 8           ; <double> [#uses=1]
        %tmp3536.i = fptosi double %tmp35.i to i32              ; <i32> [#uses=1]
diff --git a/llvm/test/CodeGen/X86/fp80-strict-scalar-cmp.ll b/llvm/test/CodeGen/X86/fp80-strict-scalar-cmp.ll
new file mode 100644
index 000000000..7e5896f04
--- /dev/null
+++ b/llvm/test/CodeGen/X86/fp80-strict-scalar-cmp.ll
@@ -0,0 +1,992 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=-sse -O3 | FileCheck %s --check-prefixes=CHECK,X87-32
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -O3 | FileCheck %s --check-prefixes=CHECK,X87-64
+
+define i32 @test_oeq_q(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 {
+; X87-32-LABEL: test_oeq_q:
+; X87-32:       # %bb.0:
+; X87-32-NEXT:    fldt {{[0-9]+}}(%esp)
+; X87-32-NEXT:    fldt {{[0-9]+}}(%esp)
+; X87-32-NEXT:    fucompp
+; X87-32-NEXT:    fnstsw %ax
+; X87-32-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-32-NEXT:    sahf
+; X87-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-32-NEXT:    jne .LBB0_3
+; X87-32-NEXT:  # %bb.1:
+; X87-32-NEXT:    jp .LBB0_3
+; X87-32-NEXT:  # %bb.2:
+; X87-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-32-NEXT:  .LBB0_3:
+; X87-32-NEXT:    movl (%eax), %eax
+; X87-32-NEXT:    retl
+;
+; X87-64-LABEL: test_oeq_q:
+; X87-64:       # %bb.0:
+; X87-64-NEXT:    movl %edi, %eax
+; X87-64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X87-64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X87-64-NEXT:    fucompi %st(1), %st
+; X87-64-NEXT:    fstp %st(0)
+; X87-64-NEXT:    cmovnel %esi, %eax
+; X87-64-NEXT:    cmovpl %esi, %eax
+; X87-64-NEXT:    retq
+  %cond = call i1 @llvm.experimental.constrained.fcmp.x86_fp80(
+                                               x86_fp80 %f1, x86_fp80 %f2, metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_ogt_q(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 {
+; X87-32-LABEL: test_ogt_q:
+; X87-32:       # %bb.0:
+; X87-32-NEXT:    fldt {{[0-9]+}}(%esp)
+; X87-32-NEXT:    fldt {{[0-9]+}}(%esp)
+; X87-32-NEXT:    fucompp
+; X87-32-NEXT:    fnstsw %ax
+; X87-32-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-32-NEXT:    sahf
+; X87-32-NEXT:    ja .LBB1_1
+; X87-32-NEXT:  # %bb.2:
+; X87-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-32-NEXT:    movl (%eax), %eax
+; X87-32-NEXT:    retl
+; X87-32-NEXT:  .LBB1_1:
+; X87-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-32-NEXT:    movl (%eax), %eax
+; X87-32-NEXT:    retl
+;
+; X87-64-LABEL: test_ogt_q:
+; X87-64:       # %bb.0:
+; X87-64-NEXT:    movl %edi, %eax
+; X87-64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X87-64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X87-64-NEXT:    fucompi %st(1), %st
+; X87-64-NEXT:    fstp %st(0)
+; X87-64-NEXT:    cmovbel %esi, %eax
+; X87-64-NEXT:    retq
+  %cond = call i1 @llvm.experimental.constrained.fcmp.x86_fp80(
+                                               x86_fp80 %f1, x86_fp80 %f2, metadata !"ogt",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_oge_q(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 {
+; X87-32-LABEL: test_oge_q:
+; X87-32:       # %bb.0:
+; X87-32-NEXT:    fldt {{[0-9]+}}(%esp)
+; X87-32-NEXT:    fldt {{[0-9]+}}(%esp)
+; X87-32-NEXT:    fucompp
+; X87-32-NEXT:    fnstsw %ax
+; X87-32-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-32-NEXT:    sahf
+; X87-32-NEXT:    jae .LBB2_1
+; X87-32-NEXT:  # %bb.2:
+; X87-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-32-NEXT:    movl (%eax), %eax
+; X87-32-NEXT:    retl
+; X87-32-NEXT:  .LBB2_1:
+; X87-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-32-NEXT:    movl (%eax), %eax
+; X87-32-NEXT:    retl
+;
+; X87-64-LABEL: test_oge_q:
+; X87-64:       # %bb.0:
+; X87-64-NEXT:    movl %edi, %eax
+; X87-64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X87-64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X87-64-NEXT:    fucompi %st(1), %st
+; X87-64-NEXT:    fstp %st(0)
+; X87-64-NEXT:    cmovbl %esi, %eax
+; X87-64-NEXT:    retq
+  %cond = call i1 @llvm.experimental.constrained.fcmp.x86_fp80(
+                                               x86_fp80 %f1, x86_fp80 %f2, metadata !"oge",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_olt_q(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 {
+; X87-32-LABEL: test_olt_q:
+; X87-32:       # %bb.0:
+; X87-32-NEXT:    fldt {{[0-9]+}}(%esp)
+; X87-32-NEXT:    fldt {{[0-9]+}}(%esp)
+; X87-32-NEXT:    fucompp
+; X87-32-NEXT:    fnstsw %ax
+; X87-32-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-32-NEXT:    sahf
+; X87-32-NEXT:    ja .LBB3_1
+; X87-32-NEXT:  # %bb.2:
+; X87-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-32-NEXT:    movl (%eax), %eax
+; X87-32-NEXT:    retl
+; X87-32-NEXT:  .LBB3_1:
+; X87-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-32-NEXT:    movl (%eax), %eax
+; X87-32-NEXT:    retl
+;
+; X87-64-LABEL: test_olt_q:
+; X87-64:       # %bb.0:
+; X87-64-NEXT:    movl %edi, %eax
+; X87-64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X87-64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X87-64-NEXT:    fucompi %st(1), %st
+; X87-64-NEXT:    fstp %st(0)
+; X87-64-NEXT:    cmovbel %esi, %eax
+; X87-64-NEXT:    retq
+  %cond = call i1 @llvm.experimental.constrained.fcmp.x86_fp80(
+                                               x86_fp80 %f1, x86_fp80 %f2, metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_ole_q(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 {
+; X87-32-LABEL: test_ole_q:
+; X87-32:       # %bb.0:
+; X87-32-NEXT:    fldt {{[0-9]+}}(%esp)
+; X87-32-NEXT:    fldt {{[0-9]+}}(%esp)
+; X87-32-NEXT:    fucompp
+; X87-32-NEXT:    fnstsw %ax
+; X87-32-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-32-NEXT:    sahf
+; X87-32-NEXT:    jae .LBB4_1
+; X87-32-NEXT:  # %bb.2:
+; X87-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-32-NEXT:    movl (%eax), %eax
+; X87-32-NEXT:    retl
+; X87-32-NEXT:  .LBB4_1:
+; X87-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-32-NEXT:    movl (%eax), %eax
+; X87-32-NEXT:    retl
+;
+; X87-64-LABEL: test_ole_q:
+; X87-64:       # %bb.0:
+; X87-64-NEXT:    movl %edi, %eax
+; X87-64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X87-64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X87-64-NEXT:    fucompi %st(1), %st
+; X87-64-NEXT:    fstp %st(0)
+; X87-64-NEXT:    cmovbl %esi, %eax
+; X87-64-NEXT:    retq
+  %cond = call i1 @llvm.experimental.constrained.fcmp.x86_fp80(
+                                               x86_fp80 %f1, x86_fp80 %f2, metadata !"ole",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_one_q(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 {
+; X87-32-LABEL: test_one_q:
+; X87-32:       # %bb.0:
+; X87-32-NEXT:    fldt {{[0-9]+}}(%esp)
+; X87-32-NEXT:    fldt {{[0-9]+}}(%esp)
+; X87-32-NEXT:    fucompp
+; X87-32-NEXT:    fnstsw %ax
+; X87-32-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-32-NEXT:    sahf
+; X87-32-NEXT:    jne .LBB5_1
+; X87-32-NEXT:  # %bb.2:
+; X87-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-32-NEXT:    movl (%eax), %eax
+; X87-32-NEXT:    retl
+; X87-32-NEXT:  .LBB5_1:
+; X87-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-32-NEXT:    movl (%eax), %eax
+; X87-32-NEXT:    retl
+;
+; X87-64-LABEL: test_one_q:
+; X87-64:       # %bb.0:
+; X87-64-NEXT:    movl %edi, %eax
+; X87-64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X87-64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X87-64-NEXT:    fucompi %st(1), %st
+; X87-64-NEXT:    fstp %st(0)
+; X87-64-NEXT:    cmovel %esi, %eax
+; X87-64-NEXT:    retq
+  %cond = call i1 @llvm.experimental.constrained.fcmp.x86_fp80(
+                                               x86_fp80 %f1, x86_fp80 %f2, metadata !"one",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_ord_q(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 {
+; X87-32-LABEL: test_ord_q:
+; X87-32:       # %bb.0:
+; X87-32-NEXT:    fldt {{[0-9]+}}(%esp)
+; X87-32-NEXT:    fldt {{[0-9]+}}(%esp)
+; X87-32-NEXT:    fucompp
+; X87-32-NEXT:    fnstsw %ax
+; X87-32-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-32-NEXT:    sahf
+; X87-32-NEXT:    jnp .LBB6_1
+; X87-32-NEXT:  # %bb.2:
+; X87-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-32-NEXT:    movl (%eax), %eax
+; X87-32-NEXT:    retl
+; X87-32-NEXT:  .LBB6_1:
+; X87-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-32-NEXT:    movl (%eax), %eax
+; X87-32-NEXT:    retl
+;
+; X87-64-LABEL: test_ord_q:
+; X87-64:       # %bb.0:
+; X87-64-NEXT:    movl %edi, %eax
+; X87-64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X87-64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X87-64-NEXT:    fucompi %st(1), %st
+; X87-64-NEXT:    fstp %st(0)
+; X87-64-NEXT:    cmovpl %esi, %eax
+; X87-64-NEXT:    retq
+  %cond = call i1 @llvm.experimental.constrained.fcmp.x86_fp80(
+                                               x86_fp80 %f1, x86_fp80 %f2, metadata !"ord",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_ueq_q(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 {
+; X87-32-LABEL: test_ueq_q:
+; X87-32:       # %bb.0:
+; X87-32-NEXT:    fldt {{[0-9]+}}(%esp)
+; X87-32-NEXT:    fldt {{[0-9]+}}(%esp)
+; X87-32-NEXT:    fucompp
+; X87-32-NEXT:    fnstsw %ax
+; X87-32-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-32-NEXT:    sahf
+; X87-32-NEXT:    je .LBB7_1
+; X87-32-NEXT:  # %bb.2:
+; X87-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-32-NEXT:    movl (%eax), %eax
+; X87-32-NEXT:    retl
+; X87-32-NEXT:  .LBB7_1:
+; X87-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-32-NEXT:    movl (%eax), %eax
+; X87-32-NEXT:    retl
+;
+; X87-64-LABEL: test_ueq_q:
+; X87-64:       # %bb.0:
+; X87-64-NEXT:    movl %edi, %eax
+; X87-64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X87-64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X87-64-NEXT:    fucompi %st(1), %st
+; X87-64-NEXT:    fstp %st(0)
+; X87-64-NEXT:    cmovnel %esi, %eax
+; X87-64-NEXT:    retq
+  %cond = call i1 @llvm.experimental.constrained.fcmp.x86_fp80(
+                                               x86_fp80 %f1, x86_fp80 %f2, metadata !"ueq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_ugt_q(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 {
+; X87-32-LABEL: test_ugt_q:
+; X87-32:       # %bb.0:
+; X87-32-NEXT:    fldt {{[0-9]+}}(%esp)
+; X87-32-NEXT:    fldt {{[0-9]+}}(%esp)
+; X87-32-NEXT:    fucompp
+; X87-32-NEXT:    fnstsw %ax
+; X87-32-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-32-NEXT:    sahf
+; X87-32-NEXT:    jb .LBB8_1
+; X87-32-NEXT:  # %bb.2:
+; X87-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-32-NEXT:    movl (%eax), %eax
+; X87-32-NEXT:    retl
+; X87-32-NEXT:  .LBB8_1:
+; X87-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-32-NEXT:    movl (%eax), %eax
+; X87-32-NEXT:    retl
+;
+; X87-64-LABEL: test_ugt_q:
+; X87-64:       # %bb.0:
+; X87-64-NEXT:    movl %edi, %eax
+; X87-64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X87-64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X87-64-NEXT:    fucompi %st(1), %st
+; X87-64-NEXT:    fstp %st(0)
+; X87-64-NEXT:    cmovael %esi, %eax
+; X87-64-NEXT:    retq
+  %cond = call i1 @llvm.experimental.constrained.fcmp.x86_fp80(
+                                               x86_fp80 %f1, x86_fp80 %f2, metadata !"ugt",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_uge_q(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 {
+; X87-32-LABEL: test_uge_q:
+; X87-32:       # %bb.0:
+; X87-32-NEXT:    fldt {{[0-9]+}}(%esp)
+; X87-32-NEXT:    fldt {{[0-9]+}}(%esp)
+; X87-32-NEXT:    fucompp
+; X87-32-NEXT:    fnstsw %ax
+; X87-32-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-32-NEXT:    sahf
+; X87-32-NEXT:    jbe .LBB9_1
+; X87-32-NEXT:  # %bb.2:
+; X87-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-32-NEXT:    movl (%eax), %eax
+; X87-32-NEXT:    retl
+; X87-32-NEXT:  .LBB9_1:
+; X87-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-32-NEXT:    movl (%eax), %eax
+; X87-32-NEXT:    retl
+;
+; X87-64-LABEL: test_uge_q:
+; X87-64:       # %bb.0:
+; X87-64-NEXT:    movl %edi, %eax
+; X87-64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X87-64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X87-64-NEXT:    fucompi %st(1), %st
+; X87-64-NEXT:    fstp %st(0)
+; X87-64-NEXT:    cmoval %esi, %eax
+; X87-64-NEXT:    retq
+  %cond = call i1 @llvm.experimental.constrained.fcmp.x86_fp80(
+                                               x86_fp80 %f1, x86_fp80 %f2, metadata !"uge",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_ult_q(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 {
+; X87-32-LABEL: test_ult_q:
+; X87-32:       # %bb.0:
+; X87-32-NEXT:    fldt {{[0-9]+}}(%esp)
+; X87-32-NEXT:    fldt {{[0-9]+}}(%esp)
+; X87-32-NEXT:    fucompp
+; X87-32-NEXT:    fnstsw %ax
+; X87-32-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-32-NEXT:    sahf
+; X87-32-NEXT:    jb .LBB10_1
+; X87-32-NEXT:  # %bb.2:
+; X87-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-32-NEXT:    movl (%eax), %eax
+; X87-32-NEXT:    retl
+; X87-32-NEXT:  .LBB10_1:
+; X87-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-32-NEXT:    movl (%eax), %eax
+; X87-32-NEXT:    retl
+;
+; X87-64-LABEL: test_ult_q:
+; X87-64:       # %bb.0:
+; X87-64-NEXT:    movl %edi, %eax
+; X87-64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X87-64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X87-64-NEXT:    fucompi %st(1), %st
+; X87-64-NEXT:    fstp %st(0)
+; X87-64-NEXT:    cmovael %esi, %eax
+; X87-64-NEXT:    retq
+  %cond = call i1 @llvm.experimental.constrained.fcmp.x86_fp80(
+                                               x86_fp80 %f1, x86_fp80 %f2, metadata !"ult",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_ule_q(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 {
+; X87-32-LABEL: test_ule_q:
+; X87-32:       # %bb.0:
+; X87-32-NEXT:    fldt {{[0-9]+}}(%esp)
+; X87-32-NEXT:    fldt {{[0-9]+}}(%esp)
+; X87-32-NEXT:    fucompp
+; X87-32-NEXT:    fnstsw %ax
+; X87-32-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-32-NEXT:    sahf
+; X87-32-NEXT:    jbe .LBB11_1
+; X87-32-NEXT:  # %bb.2:
+; X87-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-32-NEXT:    movl (%eax), %eax
+; X87-32-NEXT:    retl
+; X87-32-NEXT:  .LBB11_1:
+; X87-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-32-NEXT:    movl (%eax), %eax
+; X87-32-NEXT:    retl
+;
+; X87-64-LABEL: test_ule_q:
+; X87-64:       # %bb.0:
+; X87-64-NEXT:    movl %edi, %eax
+; X87-64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X87-64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X87-64-NEXT:    fucompi %st(1), %st
+; X87-64-NEXT:    fstp %st(0)
+; X87-64-NEXT:    cmoval %esi, %eax
+; X87-64-NEXT:    retq
+  %cond = call i1 @llvm.experimental.constrained.fcmp.x86_fp80(
+                                               x86_fp80 %f1, x86_fp80 %f2, metadata !"ule",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_une_q(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 {
+; X87-32-LABEL: test_une_q:
+; X87-32:       # %bb.0:
+; X87-32-NEXT:    fldt {{[0-9]+}}(%esp)
+; X87-32-NEXT:    fldt {{[0-9]+}}(%esp)
+; X87-32-NEXT:    fucompp
+; X87-32-NEXT:    fnstsw %ax
+; X87-32-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-32-NEXT:    sahf
+; X87-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-32-NEXT:    jne .LBB12_3
+; X87-32-NEXT:  # %bb.1:
+; X87-32-NEXT:    jp .LBB12_3
+; X87-32-NEXT:  # %bb.2:
+; X87-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-32-NEXT:  .LBB12_3:
+; X87-32-NEXT:    movl (%eax), %eax
+; X87-32-NEXT:    retl
+;
+; X87-64-LABEL: test_une_q:
+; X87-64:       # %bb.0:
+; X87-64-NEXT:    movl %esi, %eax
+; X87-64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X87-64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X87-64-NEXT:    fucompi %st(1), %st
+; X87-64-NEXT:    fstp %st(0)
+; X87-64-NEXT:    cmovnel %edi, %eax
+; X87-64-NEXT:    cmovpl %edi, %eax
+; X87-64-NEXT:    retq
+  %cond = call i1 @llvm.experimental.constrained.fcmp.x86_fp80(
+                                               x86_fp80 %f1, x86_fp80 %f2, metadata !"une",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_uno_q(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 {
+; X87-32-LABEL: test_uno_q:
+; X87-32:       # %bb.0:
+; X87-32-NEXT:    fldt {{[0-9]+}}(%esp)
+; X87-32-NEXT:    fldt {{[0-9]+}}(%esp)
+; X87-32-NEXT:    fucompp
+; X87-32-NEXT:    fnstsw %ax
+; X87-32-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-32-NEXT:    sahf
+; X87-32-NEXT:    jp .LBB13_1
+; X87-32-NEXT:  # %bb.2:
+; X87-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-32-NEXT:    movl (%eax), %eax
+; X87-32-NEXT:    retl
+; X87-32-NEXT:  .LBB13_1:
+; X87-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-32-NEXT:    movl (%eax), %eax
+; X87-32-NEXT:    retl
+;
+; X87-64-LABEL: test_uno_q:
+; X87-64:       # %bb.0:
+; X87-64-NEXT:    movl %edi, %eax
+; X87-64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X87-64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X87-64-NEXT:    fucompi %st(1), %st
+; X87-64-NEXT:    fstp %st(0)
+; X87-64-NEXT:    cmovnpl %esi, %eax
+; X87-64-NEXT:    retq
+  %cond = call i1 @llvm.experimental.constrained.fcmp.x86_fp80(
+                                               x86_fp80 %f1, x86_fp80 %f2, metadata !"uno",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_oeq_s(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 {
+; X87-32-LABEL: test_oeq_s:
+; X87-32:       # %bb.0:
+; X87-32-NEXT:    fldt {{[0-9]+}}(%esp)
+; X87-32-NEXT:    fldt {{[0-9]+}}(%esp)
+; X87-32-NEXT:    fcompp
+; X87-32-NEXT:    fnstsw %ax
+; X87-32-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-32-NEXT:    sahf
+; X87-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-32-NEXT:    jne .LBB14_3
+; X87-32-NEXT:  # %bb.1:
+; X87-32-NEXT:    jp .LBB14_3
+; X87-32-NEXT:  # %bb.2:
+; X87-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-32-NEXT:  .LBB14_3:
+; X87-32-NEXT:    movl (%eax), %eax
+; X87-32-NEXT:    retl
+;
+; X87-64-LABEL: test_oeq_s:
+; X87-64:       # %bb.0:
+; X87-64-NEXT:    movl %edi, %eax
+; X87-64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X87-64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X87-64-NEXT:    fcompi %st(1), %st
+; X87-64-NEXT:    fstp %st(0)
+; X87-64-NEXT:    cmovnel %esi, %eax
+; X87-64-NEXT:    cmovpl %esi, %eax
+; X87-64-NEXT:    retq
+  %cond = call i1 @llvm.experimental.constrained.fcmps.x86_fp80(
+                                               x86_fp80 %f1, x86_fp80 %f2, metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_ogt_s(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 {
+; X87-32-LABEL: test_ogt_s:
+; X87-32:       # %bb.0:
+; X87-32-NEXT:    fldt {{[0-9]+}}(%esp)
+; X87-32-NEXT:    fldt {{[0-9]+}}(%esp)
+; X87-32-NEXT:    fcompp
+; X87-32-NEXT:    fnstsw %ax
+; X87-32-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-32-NEXT:    sahf
+; X87-32-NEXT:    ja .LBB15_1
+; X87-32-NEXT:  # %bb.2:
+; X87-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-32-NEXT:    movl (%eax), %eax
+; X87-32-NEXT:    retl
+; X87-32-NEXT:  .LBB15_1:
+; X87-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-32-NEXT:    movl (%eax), %eax
+; X87-32-NEXT:    retl
+;
+; X87-64-LABEL: test_ogt_s:
+; X87-64:       # %bb.0:
+; X87-64-NEXT:    movl %edi, %eax
+; X87-64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X87-64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X87-64-NEXT:    fcompi %st(1), %st
+; X87-64-NEXT:    fstp %st(0)
+; X87-64-NEXT:    cmovbel %esi, %eax
+; X87-64-NEXT:    retq
+  %cond = call i1 @llvm.experimental.constrained.fcmps.x86_fp80(
+                                               x86_fp80 %f1, x86_fp80 %f2, metadata !"ogt",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_oge_s(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 {
+; X87-32-LABEL: test_oge_s:
+; X87-32:       # %bb.0:
+; X87-32-NEXT:    fldt {{[0-9]+}}(%esp)
+; X87-32-NEXT:    fldt {{[0-9]+}}(%esp)
+; X87-32-NEXT:    fcompp
+; X87-32-NEXT:    fnstsw %ax
+; X87-32-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-32-NEXT:    sahf
+; X87-32-NEXT:    jae .LBB16_1
+; X87-32-NEXT:  # %bb.2:
+; X87-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-32-NEXT:    movl (%eax), %eax
+; X87-32-NEXT:    retl
+; X87-32-NEXT:  .LBB16_1:
+; X87-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-32-NEXT:    movl (%eax), %eax
+; X87-32-NEXT:    retl
+;
+; X87-64-LABEL: test_oge_s:
+; X87-64:       # %bb.0:
+; X87-64-NEXT:    movl %edi, %eax
+; X87-64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X87-64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X87-64-NEXT:    fcompi %st(1), %st
+; X87-64-NEXT:    fstp %st(0)
+; X87-64-NEXT:    cmovbl %esi, %eax
+; X87-64-NEXT:    retq
+  %cond = call i1 @llvm.experimental.constrained.fcmps.x86_fp80(
+                                               x86_fp80 %f1, x86_fp80 %f2, metadata !"oge",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_olt_s(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 {
+; X87-32-LABEL: test_olt_s:
+; X87-32:       # %bb.0:
+; X87-32-NEXT:    fldt {{[0-9]+}}(%esp)
+; X87-32-NEXT:    fldt {{[0-9]+}}(%esp)
+; X87-32-NEXT:    fcompp
+; X87-32-NEXT:    fnstsw %ax
+; X87-32-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-32-NEXT:    sahf
+; X87-32-NEXT:    ja .LBB17_1
+; X87-32-NEXT:  # %bb.2:
+; X87-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-32-NEXT:    movl (%eax), %eax
+; X87-32-NEXT:    retl
+; X87-32-NEXT:  .LBB17_1:
+; X87-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-32-NEXT:    movl (%eax), %eax
+; X87-32-NEXT:    retl
+;
+; X87-64-LABEL: test_olt_s:
+; X87-64:       # %bb.0:
+; X87-64-NEXT:    movl %edi, %eax
+; X87-64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X87-64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X87-64-NEXT:    fcompi %st(1), %st
+; X87-64-NEXT:    fstp %st(0)
+; X87-64-NEXT:    cmovbel %esi, %eax
+; X87-64-NEXT:    retq
+  %cond = call i1 @llvm.experimental.constrained.fcmps.x86_fp80(
+                                               x86_fp80 %f1, x86_fp80 %f2, metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_ole_s(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 {
+; X87-32-LABEL: test_ole_s:
+; X87-32:       # %bb.0:
+; X87-32-NEXT:    fldt {{[0-9]+}}(%esp)
+; X87-32-NEXT:    fldt {{[0-9]+}}(%esp)
+; X87-32-NEXT:    fcompp
+; X87-32-NEXT:    fnstsw %ax
+; X87-32-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-32-NEXT:    sahf
+; X87-32-NEXT:    jae .LBB18_1
+; X87-32-NEXT:  # %bb.2:
+; X87-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-32-NEXT:    movl (%eax), %eax
+; X87-32-NEXT:    retl
+; X87-32-NEXT:  .LBB18_1:
+; X87-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-32-NEXT:    movl (%eax), %eax
+; X87-32-NEXT:    retl
+;
+; X87-64-LABEL: test_ole_s:
+; X87-64:       # %bb.0:
+; X87-64-NEXT:    movl %edi, %eax
+; X87-64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X87-64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X87-64-NEXT:    fcompi %st(1), %st
+; X87-64-NEXT:    fstp %st(0)
+; X87-64-NEXT:    cmovbl %esi, %eax
+; X87-64-NEXT:    retq
+  %cond = call i1 @llvm.experimental.constrained.fcmps.x86_fp80(
+                                               x86_fp80 %f1, x86_fp80 %f2, metadata !"ole",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_one_s(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 {
+; X87-32-LABEL: test_one_s:
+; X87-32:       # %bb.0:
+; X87-32-NEXT:    fldt {{[0-9]+}}(%esp)
+; X87-32-NEXT:    fldt {{[0-9]+}}(%esp)
+; X87-32-NEXT:    fcompp
+; X87-32-NEXT:    fnstsw %ax
+; X87-32-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-32-NEXT:    sahf
+; X87-32-NEXT:    jne .LBB19_1
+; X87-32-NEXT:  # %bb.2:
+; X87-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-32-NEXT:    movl (%eax), %eax
+; X87-32-NEXT:    retl
+; X87-32-NEXT:  .LBB19_1:
+; X87-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-32-NEXT:    movl (%eax), %eax
+; X87-32-NEXT:    retl
+;
+; X87-64-LABEL: test_one_s:
+; X87-64:       # %bb.0:
+; X87-64-NEXT:    movl %edi, %eax
+; X87-64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X87-64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X87-64-NEXT:    fcompi %st(1), %st
+; X87-64-NEXT:    fstp %st(0)
+; X87-64-NEXT:    cmovel %esi, %eax
+; X87-64-NEXT:    retq
+  %cond = call i1 @llvm.experimental.constrained.fcmps.x86_fp80(
+                                               x86_fp80 %f1, x86_fp80 %f2, metadata !"one",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_ord_s(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 {
+; X87-32-LABEL: test_ord_s:
+; X87-32:       # %bb.0:
+; X87-32-NEXT:    fldt {{[0-9]+}}(%esp)
+; X87-32-NEXT:    fldt {{[0-9]+}}(%esp)
+; X87-32-NEXT:    fcompp
+; X87-32-NEXT:    fnstsw %ax
+; X87-32-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-32-NEXT:    sahf
+; X87-32-NEXT:    jnp .LBB20_1
+; X87-32-NEXT:  # %bb.2:
+; X87-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-32-NEXT:    movl (%eax), %eax
+; X87-32-NEXT:    retl
+; X87-32-NEXT:  .LBB20_1:
+; X87-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-32-NEXT:    movl (%eax), %eax
+; X87-32-NEXT:    retl
+;
+; X87-64-LABEL: test_ord_s:
+; X87-64:       # %bb.0:
+; X87-64-NEXT:    movl %edi, %eax
+; X87-64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X87-64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X87-64-NEXT:    fcompi %st(1), %st
+; X87-64-NEXT:    fstp %st(0)
+; X87-64-NEXT:    cmovpl %esi, %eax
+; X87-64-NEXT:    retq
+  %cond = call i1 @llvm.experimental.constrained.fcmps.x86_fp80(
+                                               x86_fp80 %f1, x86_fp80 %f2, metadata !"ord",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_ueq_s(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 {
+; X87-32-LABEL: test_ueq_s:
+; X87-32:       # %bb.0:
+; X87-32-NEXT:    fldt {{[0-9]+}}(%esp)
+; X87-32-NEXT:    fldt {{[0-9]+}}(%esp)
+; X87-32-NEXT:    fcompp
+; X87-32-NEXT:    fnstsw %ax
+; X87-32-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-32-NEXT:    sahf
+; X87-32-NEXT:    je .LBB21_1
+; X87-32-NEXT:  # %bb.2:
+; X87-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-32-NEXT:    movl (%eax), %eax
+; X87-32-NEXT:    retl
+; X87-32-NEXT:  .LBB21_1:
+; X87-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-32-NEXT:    movl (%eax), %eax
+; X87-32-NEXT:    retl
+;
+; X87-64-LABEL: test_ueq_s:
+; X87-64:       # %bb.0:
+; X87-64-NEXT:    movl %edi, %eax
+; X87-64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X87-64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X87-64-NEXT:    fcompi %st(1), %st
+; X87-64-NEXT:    fstp %st(0)
+; X87-64-NEXT:    cmovnel %esi, %eax
+; X87-64-NEXT:    retq
+  %cond = call i1 @llvm.experimental.constrained.fcmps.x86_fp80(
+                                               x86_fp80 %f1, x86_fp80 %f2, metadata !"ueq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_ugt_s(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 {
+; X87-32-LABEL: test_ugt_s:
+; X87-32:       # %bb.0:
+; X87-32-NEXT:    fldt {{[0-9]+}}(%esp)
+; X87-32-NEXT:    fldt {{[0-9]+}}(%esp)
+; X87-32-NEXT:    fcompp
+; X87-32-NEXT:    fnstsw %ax
+; X87-32-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-32-NEXT:    sahf
+; X87-32-NEXT:    jb .LBB22_1
+; X87-32-NEXT:  # %bb.2:
+; X87-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-32-NEXT:    movl (%eax), %eax
+; X87-32-NEXT:    retl
+; X87-32-NEXT:  .LBB22_1:
+; X87-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-32-NEXT:    movl (%eax), %eax
+; X87-32-NEXT:    retl
+;
+; X87-64-LABEL: test_ugt_s:
+; X87-64:       # %bb.0:
+; X87-64-NEXT:    movl %edi, %eax
+; X87-64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X87-64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X87-64-NEXT:    fcompi %st(1), %st
+; X87-64-NEXT:    fstp %st(0)
+; X87-64-NEXT:    cmovael %esi, %eax
+; X87-64-NEXT:    retq
+  %cond = call i1 @llvm.experimental.constrained.fcmps.x86_fp80(
+                                               x86_fp80 %f1, x86_fp80 %f2, metadata !"ugt",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_uge_s(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 {
+; X87-32-LABEL: test_uge_s:
+; X87-32:       # %bb.0:
+; X87-32-NEXT:    fldt {{[0-9]+}}(%esp)
+; X87-32-NEXT:    fldt {{[0-9]+}}(%esp)
+; X87-32-NEXT:    fcompp
+; X87-32-NEXT:    fnstsw %ax
+; X87-32-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-32-NEXT:    sahf
+; X87-32-NEXT:    jbe .LBB23_1
+; X87-32-NEXT:  # %bb.2:
+; X87-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-32-NEXT:    movl (%eax), %eax
+; X87-32-NEXT:    retl
+; X87-32-NEXT:  .LBB23_1:
+; X87-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-32-NEXT:    movl (%eax), %eax
+; X87-32-NEXT:    retl
+;
+; X87-64-LABEL: test_uge_s:
+; X87-64:       # %bb.0:
+; X87-64-NEXT:    movl %edi, %eax
+; X87-64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X87-64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X87-64-NEXT:    fcompi %st(1), %st
+; X87-64-NEXT:    fstp %st(0)
+; X87-64-NEXT:    cmoval %esi, %eax
+; X87-64-NEXT:    retq
+  %cond = call i1 @llvm.experimental.constrained.fcmps.x86_fp80(
+                                               x86_fp80 %f1, x86_fp80 %f2, metadata !"uge",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_ult_s(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 {
+; X87-32-LABEL: test_ult_s:
+; X87-32:       # %bb.0:
+; X87-32-NEXT:    fldt {{[0-9]+}}(%esp)
+; X87-32-NEXT:    fldt {{[0-9]+}}(%esp)
+; X87-32-NEXT:    fcompp
+; X87-32-NEXT:    fnstsw %ax
+; X87-32-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-32-NEXT:    sahf
+; X87-32-NEXT:    jb .LBB24_1
+; X87-32-NEXT:  # %bb.2:
+; X87-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-32-NEXT:    movl (%eax), %eax
+; X87-32-NEXT:    retl
+; X87-32-NEXT:  .LBB24_1:
+; X87-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-32-NEXT:    movl (%eax), %eax
+; X87-32-NEXT:    retl
+;
+; X87-64-LABEL: test_ult_s:
+; X87-64:       # %bb.0:
+; X87-64-NEXT:    movl %edi, %eax
+; X87-64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X87-64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X87-64-NEXT:    fcompi %st(1), %st
+; X87-64-NEXT:    fstp %st(0)
+; X87-64-NEXT:    cmovael %esi, %eax
+; X87-64-NEXT:    retq
+  %cond = call i1 @llvm.experimental.constrained.fcmps.x86_fp80(
+                                               x86_fp80 %f1, x86_fp80 %f2, metadata !"ult",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_ule_s(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 {
+; X87-32-LABEL: test_ule_s:
+; X87-32:       # %bb.0:
+; X87-32-NEXT:    fldt {{[0-9]+}}(%esp)
+; X87-32-NEXT:    fldt {{[0-9]+}}(%esp)
+; X87-32-NEXT:    fcompp
+; X87-32-NEXT:    fnstsw %ax
+; X87-32-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-32-NEXT:    sahf
+; X87-32-NEXT:    jbe .LBB25_1
+; X87-32-NEXT:  # %bb.2:
+; X87-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-32-NEXT:    movl (%eax), %eax
+; X87-32-NEXT:    retl
+; X87-32-NEXT:  .LBB25_1:
+; X87-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-32-NEXT:    movl (%eax), %eax
+; X87-32-NEXT:    retl
+;
+; X87-64-LABEL: test_ule_s:
+; X87-64:       # %bb.0:
+; X87-64-NEXT:    movl %edi, %eax
+; X87-64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X87-64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X87-64-NEXT:    fcompi %st(1), %st
+; X87-64-NEXT:    fstp %st(0)
+; X87-64-NEXT:    cmoval %esi, %eax
+; X87-64-NEXT:    retq
+  %cond = call i1 @llvm.experimental.constrained.fcmps.x86_fp80(
+                                               x86_fp80 %f1, x86_fp80 %f2, metadata !"ule",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_une_s(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 {
+; X87-32-LABEL: test_une_s:
+; X87-32:       # %bb.0:
+; X87-32-NEXT:    fldt {{[0-9]+}}(%esp)
+; X87-32-NEXT:    fldt {{[0-9]+}}(%esp)
+; X87-32-NEXT:    fcompp
+; X87-32-NEXT:    fnstsw %ax
+; X87-32-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-32-NEXT:    sahf
+; X87-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-32-NEXT:    jne .LBB26_3
+; X87-32-NEXT:  # %bb.1:
+; X87-32-NEXT:    jp .LBB26_3
+; X87-32-NEXT:  # %bb.2:
+; X87-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-32-NEXT:  .LBB26_3:
+; X87-32-NEXT:    movl (%eax), %eax
+; X87-32-NEXT:    retl
+;
+; X87-64-LABEL: test_une_s:
+; X87-64:       # %bb.0:
+; X87-64-NEXT:    movl %esi, %eax
+; X87-64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X87-64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X87-64-NEXT:    fcompi %st(1), %st
+; X87-64-NEXT:    fstp %st(0)
+; X87-64-NEXT:    cmovnel %edi, %eax
+; X87-64-NEXT:    cmovpl %edi, %eax
+; X87-64-NEXT:    retq
+  %cond = call i1 @llvm.experimental.constrained.fcmps.x86_fp80(
+                                               x86_fp80 %f1, x86_fp80 %f2, metadata !"une",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @test_uno_s(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 {
+; X87-32-LABEL: test_uno_s:
+; X87-32:       # %bb.0:
+; X87-32-NEXT:    fldt {{[0-9]+}}(%esp)
+; X87-32-NEXT:    fldt {{[0-9]+}}(%esp)
+; X87-32-NEXT:    fcompp
+; X87-32-NEXT:    fnstsw %ax
+; X87-32-NEXT:    # kill: def $ah killed $ah killed $ax
+; X87-32-NEXT:    sahf
+; X87-32-NEXT:    jp .LBB27_1
+; X87-32-NEXT:  # %bb.2:
+; X87-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-32-NEXT:    movl (%eax), %eax
+; X87-32-NEXT:    retl
+; X87-32-NEXT:  .LBB27_1:
+; X87-32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X87-32-NEXT:    movl (%eax), %eax
+; X87-32-NEXT:    retl
+;
+; X87-64-LABEL: test_uno_s:
+; X87-64:       # %bb.0:
+; X87-64-NEXT:    movl %edi, %eax
+; X87-64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X87-64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X87-64-NEXT:    fcompi %st(1), %st
+; X87-64-NEXT:    fstp %st(0)
+; X87-64-NEXT:    cmovnpl %esi, %eax
+; X87-64-NEXT:    retq
+  %cond = call i1 @llvm.experimental.constrained.fcmps.x86_fp80(
+                                               x86_fp80 %f1, x86_fp80 %f2, metadata !"uno",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+attributes #0 = { strictfp }
+
+declare i1 @llvm.experimental.constrained.fcmp.x86_fp80(x86_fp80, x86_fp80, metadata, metadata)
+declare i1 @llvm.experimental.constrained.fcmps.x86_fp80(x86_fp80, x86_fp80, metadata, metadata)
diff --git a/llvm/test/CodeGen/X86/fp80-strict-scalar.ll b/llvm/test/CodeGen/X86/fp80-strict-scalar.ll
new file mode 100644
index 000000000..5f03917f5
--- /dev/null
+++ b/llvm/test/CodeGen/X86/fp80-strict-scalar.ll
@@ -0,0 +1,872 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-unknown-unknown -O3 | FileCheck %s --check-prefixes=CHECK,X86
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -O3 | FileCheck %s --check-prefixes=CHECK,X64
+
+declare x86_fp80 @llvm.experimental.constrained.fadd.x86_fp80(x86_fp80, x86_fp80, metadata, metadata)
+declare x86_fp80 @llvm.experimental.constrained.fsub.x86_fp80(x86_fp80, x86_fp80, metadata, metadata)
+declare x86_fp80 @llvm.experimental.constrained.fmul.x86_fp80(x86_fp80, x86_fp80, metadata, metadata)
+declare x86_fp80 @llvm.experimental.constrained.fdiv.x86_fp80(x86_fp80, x86_fp80, metadata, metadata)
+declare x86_fp80 @llvm.experimental.constrained.fpext.x86_fp80.f32(float, metadata)
+declare x86_fp80 @llvm.experimental.constrained.fpext.x86_fp80.f64(double, metadata)
+declare x86_fp80 @llvm.experimental.constrained.sqrt.x86_fp80(x86_fp80, metadata, metadata)
+declare float @llvm.experimental.constrained.fptrunc.f32.x86_fp80(x86_fp80, metadata, metadata)
+declare double @llvm.experimental.constrained.fptrunc.f64.x86_fp80(x86_fp80, metadata, metadata)
+declare i1  @llvm.experimental.constrained.fptosi.i1.x86_fp80(x86_fp80, metadata)
+declare i8  @llvm.experimental.constrained.fptosi.i8.x86_fp80(x86_fp80, metadata)
+declare i16 @llvm.experimental.constrained.fptosi.i16.x86_fp80(x86_fp80, metadata)
+declare i32 @llvm.experimental.constrained.fptosi.i32.x86_fp80(x86_fp80, metadata)
+declare i64 @llvm.experimental.constrained.fptosi.i64.x86_fp80(x86_fp80, metadata)
+declare i1 @llvm.experimental.constrained.fptoui.i1.x86_fp80(x86_fp80, metadata)
+declare i8  @llvm.experimental.constrained.fptoui.i8.x86_fp80(x86_fp80, metadata)
+declare i16 @llvm.experimental.constrained.fptoui.i16.x86_fp80(x86_fp80, metadata)
+declare i32 @llvm.experimental.constrained.fptoui.i32.x86_fp80(x86_fp80, metadata)
+declare i64 @llvm.experimental.constrained.fptoui.i64.x86_fp80(x86_fp80, metadata)
+declare x86_fp80 @llvm.experimental.constrained.sitofp.x86_fp80.i1(i1, metadata, metadata)
+declare x86_fp80 @llvm.experimental.constrained.sitofp.x86_fp80.i8(i8, metadata, metadata)
+declare x86_fp80 @llvm.experimental.constrained.sitofp.x86_fp80.i16(i16, metadata, metadata)
+declare x86_fp80 @llvm.experimental.constrained.sitofp.x86_fp80.i32(i32, metadata, metadata)
+declare x86_fp80 @llvm.experimental.constrained.sitofp.x86_fp80.i64(i64, metadata, metadata)
+declare x86_fp80 @llvm.experimental.constrained.uitofp.x86_fp80.i1(i1, metadata, metadata)
+declare x86_fp80 @llvm.experimental.constrained.uitofp.x86_fp80.i8(i8, metadata, metadata)
+declare x86_fp80 @llvm.experimental.constrained.uitofp.x86_fp80.i16(i16, metadata, metadata)
+declare x86_fp80 @llvm.experimental.constrained.uitofp.x86_fp80.i32(i32, metadata, metadata)
+declare x86_fp80 @llvm.experimental.constrained.uitofp.x86_fp80.i64(i64, metadata, metadata)
+
+define x86_fp80 @fadd_fp80(x86_fp80 %a, x86_fp80 %b) nounwind strictfp {
+; X86-LABEL: fadd_fp80:
+; X86:       # %bb.0:
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    faddp %st, %st(1)
+; X86-NEXT:    retl
+;
+; X64-LABEL: fadd_fp80:
+; X64:       # %bb.0:
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    faddp %st, %st(1)
+; X64-NEXT:    retq
+  %ret = call x86_fp80 @llvm.experimental.constrained.fadd.x86_fp80(x86_fp80 %a, x86_fp80 %b,
+                                                                    metadata !"round.dynamic",
+                                                                    metadata !"fpexcept.strict") #0
+  ret x86_fp80 %ret
+}
+
+define x86_fp80 @fsub_fp80(x86_fp80 %a, x86_fp80 %b) nounwind strictfp {
+; X86-LABEL: fsub_fp80:
+; X86:       # %bb.0:
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fsubp %st, %st(1)
+; X86-NEXT:    retl
+;
+; X64-LABEL: fsub_fp80:
+; X64:       # %bb.0:
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fsubp %st, %st(1)
+; X64-NEXT:    retq
+  %ret = call x86_fp80 @llvm.experimental.constrained.fsub.x86_fp80(x86_fp80 %a, x86_fp80 %b,
+                                                                    metadata !"round.dynamic",
+                                                                    metadata !"fpexcept.strict") #0
+  ret x86_fp80 %ret
+}
+
+define x86_fp80 @fmul_fp80(x86_fp80 %a, x86_fp80 %b) nounwind strictfp {
+; X86-LABEL: fmul_fp80:
+; X86:       # %bb.0:
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fmulp %st, %st(1)
+; X86-NEXT:    retl
+;
+; X64-LABEL: fmul_fp80:
+; X64:       # %bb.0:
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fmulp %st, %st(1)
+; X64-NEXT:    retq
+  %ret = call x86_fp80 @llvm.experimental.constrained.fmul.x86_fp80(x86_fp80 %a, x86_fp80 %b,
+                                                                    metadata !"round.dynamic",
+                                                                    metadata !"fpexcept.strict") #0
+  ret x86_fp80 %ret
+}
+
+define x86_fp80 @fdiv_fp80(x86_fp80 %a, x86_fp80 %b) nounwind strictfp {
+; X86-LABEL: fdiv_fp80:
+; X86:       # %bb.0:
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fdivp %st, %st(1)
+; X86-NEXT:    retl
+;
+; X64-LABEL: fdiv_fp80:
+; X64:       # %bb.0:
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fdivp %st, %st(1)
+; X64-NEXT:    retq
+  %ret = call x86_fp80 @llvm.experimental.constrained.fdiv.x86_fp80(x86_fp80 %a, x86_fp80 %b,
+                                                                    metadata !"round.dynamic",
+                                                                    metadata !"fpexcept.strict") #0
+  ret x86_fp80 %ret
+}
+
+define x86_fp80 @fpext_f32_to_fp80(float %a) nounwind strictfp {
+; X86-LABEL: fpext_f32_to_fp80:
+; X86:       # %bb.0:
+; X86-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-NEXT:    retl
+;
+; X64-LABEL: fpext_f32_to_fp80:
+; X64:       # %bb.0:
+; X64-NEXT:    movss %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NEXT:    flds -{{[0-9]+}}(%rsp)
+; X64-NEXT:    retq
+  %ret = call x86_fp80 @llvm.experimental.constrained.fpext.x86_fp80.f32(float %a,
+                                                                         metadata !"fpexcept.strict") #0
+  ret x86_fp80 %ret
+}
+
+define x86_fp80 @fpext_f64_to_fp80(double %a) nounwind strictfp {
+; X86-LABEL: fpext_f64_to_fp80:
+; X86:       # %bb.0:
+; X86-NEXT:    fldl {{[0-9]+}}(%esp)
+; X86-NEXT:    retl
+;
+; X64-LABEL: fpext_f64_to_fp80:
+; X64:       # %bb.0:
+; X64-NEXT:    movsd %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NEXT:    fldl -{{[0-9]+}}(%rsp)
+; X64-NEXT:    retq
+  %ret = call x86_fp80 @llvm.experimental.constrained.fpext.x86_fp80.f64(double %a,
+                                                                         metadata !"fpexcept.strict") #0
+  ret x86_fp80 %ret
+}
+
+define float @fptrunc_fp80_to_f32(x86_fp80 %a) nounwind strictfp {
+; X86-LABEL: fptrunc_fp80_to_f32:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fstps (%esp)
+; X86-NEXT:    flds (%esp)
+; X86-NEXT:    popl %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: fptrunc_fp80_to_f32:
+; X64:       # %bb.0:
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fstps -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X64-NEXT:    retq
+  %ret = call float @llvm.experimental.constrained.fptrunc.f32.x86_fp80(x86_fp80 %a,
+                                                                        metadata !"round.dynamic",
+                                                                        metadata !"fpexcept.strict") #0
+  ret float %ret
+}
+
+define double @fptrunc_fp80_to_f64(x86_fp80 %a) nounwind strictfp {
+; X86-LABEL: fptrunc_fp80_to_f64:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    movl %esp, %ebp
+; X86-NEXT:    andl $-8, %esp
+; X86-NEXT:    subl $8, %esp
+; X86-NEXT:    fldt 8(%ebp)
+; X86-NEXT:    fstpl (%esp)
+; X86-NEXT:    fldl (%esp)
+; X86-NEXT:    movl %ebp, %esp
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl
+;
+; X64-LABEL: fptrunc_fp80_to_f64:
+; X64:       # %bb.0:
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fstpl -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X64-NEXT:    retq
+  %ret = call double @llvm.experimental.constrained.fptrunc.f64.x86_fp80(x86_fp80 %a,
+                                                                         metadata !"round.dynamic",
+                                                                         metadata !"fpexcept.strict") #0
+  ret double %ret
+}
+
+define x86_fp80 @fsqrt_fp80(x86_fp80 %a) nounwind strictfp {
+; X86-LABEL: fsqrt_fp80:
+; X86:       # %bb.0:
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fsqrt
+; X86-NEXT:    retl
+;
+; X64-LABEL: fsqrt_fp80:
+; X64:       # %bb.0:
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fsqrt
+; X64-NEXT:    retq
+  %ret = call x86_fp80 @llvm.experimental.constrained.sqrt.x86_fp80(x86_fp80 %a,
+                                                                    metadata !"round.dynamic",
+                                                                    metadata !"fpexcept.strict") #0
+  ret x86_fp80 %ret
+}
+
+define i1 @fp80_to_sint1(x86_fp80 %x) #0 {
+; X86-LABEL: fp80_to_sint1:
+; X86:       # %bb.0:
+; X86-NEXT:    subl $8, %esp
+; X86-NEXT:    .cfi_def_cfa_offset 12
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fnstcw {{[0-9]+}}(%esp)
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    orl $3072, %eax # imm = 0xC00
+; X86-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; X86-NEXT:    fldcw {{[0-9]+}}(%esp)
+; X86-NEXT:    fistps {{[0-9]+}}(%esp)
+; X86-NEXT:    fldcw {{[0-9]+}}(%esp)
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    addl $8, %esp
+; X86-NEXT:    .cfi_def_cfa_offset 4
+; X86-NEXT:    retl
+;
+; X64-LABEL: fp80_to_sint1:
+; X64:       # %bb.0:
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fnstcw -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movzwl -{{[0-9]+}}(%rsp), %eax
+; X64-NEXT:    orl $3072, %eax # imm = 0xC00
+; X64-NEXT:    movw %ax, -{{[0-9]+}}(%rsp)
+; X64-NEXT:    fldcw -{{[0-9]+}}(%rsp)
+; X64-NEXT:    fistps -{{[0-9]+}}(%rsp)
+; X64-NEXT:    fldcw -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movb -{{[0-9]+}}(%rsp), %al
+; X64-NEXT:    retq
+  %result = call i1 @llvm.experimental.constrained.fptosi.i1.x86_fp80(x86_fp80 %x,
+                                               metadata !"fpexcept.strict") #0
+  ret i1 %result
+}
+
+define i8 @fp80_to_sint8(x86_fp80 %x) #0 {
+; X86-LABEL: fp80_to_sint8:
+; X86:       # %bb.0:
+; X86-NEXT:    subl $8, %esp
+; X86-NEXT:    .cfi_def_cfa_offset 12
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fnstcw {{[0-9]+}}(%esp)
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    orl $3072, %eax # imm = 0xC00
+; X86-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; X86-NEXT:    fldcw {{[0-9]+}}(%esp)
+; X86-NEXT:    fistps {{[0-9]+}}(%esp)
+; X86-NEXT:    fldcw {{[0-9]+}}(%esp)
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    addl $8, %esp
+; X86-NEXT:    .cfi_def_cfa_offset 4
+; X86-NEXT:    retl
+;
+; X64-LABEL: fp80_to_sint8:
+; X64:       # %bb.0:
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fnstcw -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movzwl -{{[0-9]+}}(%rsp), %eax
+; X64-NEXT:    orl $3072, %eax # imm = 0xC00
+; X64-NEXT:    movw %ax, -{{[0-9]+}}(%rsp)
+; X64-NEXT:    fldcw -{{[0-9]+}}(%rsp)
+; X64-NEXT:    fistps -{{[0-9]+}}(%rsp)
+; X64-NEXT:    fldcw -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movb -{{[0-9]+}}(%rsp), %al
+; X64-NEXT:    retq
+  %result = call i8 @llvm.experimental.constrained.fptosi.i8.x86_fp80(x86_fp80 %x,
+                                               metadata !"fpexcept.strict") #0
+  ret i8 %result
+}
+
+define i16 @fp80_to_sint16(x86_fp80 %x) #0 {
+; X86-LABEL: fp80_to_sint16:
+; X86:       # %bb.0:
+; X86-NEXT:    subl $8, %esp
+; X86-NEXT:    .cfi_def_cfa_offset 12
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fnstcw {{[0-9]+}}(%esp)
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    orl $3072, %eax # imm = 0xC00
+; X86-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; X86-NEXT:    fldcw {{[0-9]+}}(%esp)
+; X86-NEXT:    fistps {{[0-9]+}}(%esp)
+; X86-NEXT:    fldcw {{[0-9]+}}(%esp)
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    addl $8, %esp
+; X86-NEXT:    .cfi_def_cfa_offset 4
+; X86-NEXT:    retl
+;
+; X64-LABEL: fp80_to_sint16:
+; X64:       # %bb.0:
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fnstcw -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movzwl -{{[0-9]+}}(%rsp), %eax
+; X64-NEXT:    orl $3072, %eax # imm = 0xC00
+; X64-NEXT:    movw %ax, -{{[0-9]+}}(%rsp)
+; X64-NEXT:    fldcw -{{[0-9]+}}(%rsp)
+; X64-NEXT:    fistps -{{[0-9]+}}(%rsp)
+; X64-NEXT:    fldcw -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movzwl -{{[0-9]+}}(%rsp), %eax
+; X64-NEXT:    retq
+  %result = call i16 @llvm.experimental.constrained.fptosi.i16.x86_fp80(x86_fp80 %x,
+                                               metadata !"fpexcept.strict") #0
+  ret i16 %result
+}
+
+define i32 @fp80_to_sint32(x86_fp80 %x) #0 {
+; X86-LABEL: fp80_to_sint32:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    subl $8, %esp
+; X86-NEXT:    .cfi_def_cfa_offset 12
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fnstcw (%esp)
+; X86-NEXT:    movzwl (%esp), %eax
+; X86-NEXT:    orl $3072, %eax # imm = 0xC00
+; X86-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; X86-NEXT:    fldcw {{[0-9]+}}(%esp)
+; X86-NEXT:    fistpl {{[0-9]+}}(%esp)
+; X86-NEXT:    fldcw (%esp)
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    addl $8, %esp
+; X86-NEXT:    .cfi_def_cfa_offset 4
+; X86-NEXT:    retl
+;
+; X64-LABEL: fp80_to_sint32:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fnstcw -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movzwl -{{[0-9]+}}(%rsp), %eax
+; X64-NEXT:    orl $3072, %eax # imm = 0xC00
+; X64-NEXT:    movw %ax, -{{[0-9]+}}(%rsp)
+; X64-NEXT:    fldcw -{{[0-9]+}}(%rsp)
+; X64-NEXT:    fistpl -{{[0-9]+}}(%rsp)
+; X64-NEXT:    fldcw -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movl -{{[0-9]+}}(%rsp), %eax
+; X64-NEXT:    retq
+entry:
+  %result = call i32 @llvm.experimental.constrained.fptosi.i32.x86_fp80(x86_fp80 %x,
+                                               metadata !"fpexcept.strict") #0
+  ret i32 %result
+}
+
+define i64 @fp80_to_sint64(x86_fp80 %x) #0 {
+; X86-LABEL: fp80_to_sint64:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    .cfi_def_cfa_offset 8
+; X86-NEXT:    .cfi_offset %ebp, -8
+; X86-NEXT:    movl %esp, %ebp
+; X86-NEXT:    .cfi_def_cfa_register %ebp
+; X86-NEXT:    andl $-8, %esp
+; X86-NEXT:    subl $16, %esp
+; X86-NEXT:    fldt 8(%ebp)
+; X86-NEXT:    fnstcw {{[0-9]+}}(%esp)
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    orl $3072, %eax # imm = 0xC00
+; X86-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; X86-NEXT:    fldcw {{[0-9]+}}(%esp)
+; X86-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-NEXT:    fldcw {{[0-9]+}}(%esp)
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl %ebp, %esp
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    .cfi_def_cfa %esp, 4
+; X86-NEXT:    retl
+;
+; X64-LABEL: fp80_to_sint64:
+; X64:       # %bb.0:
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fnstcw -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movzwl -{{[0-9]+}}(%rsp), %eax
+; X64-NEXT:    orl $3072, %eax # imm = 0xC00
+; X64-NEXT:    movw %ax, -{{[0-9]+}}(%rsp)
+; X64-NEXT:    fldcw -{{[0-9]+}}(%rsp)
+; X64-NEXT:    fistpll -{{[0-9]+}}(%rsp)
+; X64-NEXT:    fldcw -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movq -{{[0-9]+}}(%rsp), %rax
+; X64-NEXT:    retq
+  %result = call i64 @llvm.experimental.constrained.fptosi.i64.x86_fp80(x86_fp80 %x,
+                                               metadata !"fpexcept.strict") #0
+  ret i64 %result
+}
+
+define i1 @fp80_to_uint1(x86_fp80 %x) #0 {
+; X86-LABEL: fp80_to_uint1:
+; X86:       # %bb.0:
+; X86-NEXT:    subl $8, %esp
+; X86-NEXT:    .cfi_def_cfa_offset 12
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fnstcw {{[0-9]+}}(%esp)
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    orl $3072, %eax # imm = 0xC00
+; X86-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; X86-NEXT:    fldcw {{[0-9]+}}(%esp)
+; X86-NEXT:    fistps {{[0-9]+}}(%esp)
+; X86-NEXT:    fldcw {{[0-9]+}}(%esp)
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    addl $8, %esp
+; X86-NEXT:    .cfi_def_cfa_offset 4
+; X86-NEXT:    retl
+;
+; X64-LABEL: fp80_to_uint1:
+; X64:       # %bb.0:
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fnstcw -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movzwl -{{[0-9]+}}(%rsp), %eax
+; X64-NEXT:    orl $3072, %eax # imm = 0xC00
+; X64-NEXT:    movw %ax, -{{[0-9]+}}(%rsp)
+; X64-NEXT:    fldcw -{{[0-9]+}}(%rsp)
+; X64-NEXT:    fistps -{{[0-9]+}}(%rsp)
+; X64-NEXT:    fldcw -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movb -{{[0-9]+}}(%rsp), %al
+; X64-NEXT:    retq
+  %result = call i1 @llvm.experimental.constrained.fptoui.i1.x86_fp80(x86_fp80 %x,
+                                               metadata !"fpexcept.strict") #0
+  ret i1 %result
+}
+
+define i8 @fp80_to_uint8(x86_fp80 %x) #0 {
+; X86-LABEL: fp80_to_uint8:
+; X86:       # %bb.0:
+; X86-NEXT:    subl $8, %esp
+; X86-NEXT:    .cfi_def_cfa_offset 12
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fnstcw {{[0-9]+}}(%esp)
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    orl $3072, %eax # imm = 0xC00
+; X86-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; X86-NEXT:    fldcw {{[0-9]+}}(%esp)
+; X86-NEXT:    fistps {{[0-9]+}}(%esp)
+; X86-NEXT:    fldcw {{[0-9]+}}(%esp)
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    addl $8, %esp
+; X86-NEXT:    .cfi_def_cfa_offset 4
+; X86-NEXT:    retl
+;
+; X64-LABEL: fp80_to_uint8:
+; X64:       # %bb.0:
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fnstcw -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movzwl -{{[0-9]+}}(%rsp), %eax
+; X64-NEXT:    orl $3072, %eax # imm = 0xC00
+; X64-NEXT:    movw %ax, -{{[0-9]+}}(%rsp)
+; X64-NEXT:    fldcw -{{[0-9]+}}(%rsp)
+; X64-NEXT:    fistps -{{[0-9]+}}(%rsp)
+; X64-NEXT:    fldcw -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movb -{{[0-9]+}}(%rsp), %al
+; X64-NEXT:    retq
+  %result = call i8 @llvm.experimental.constrained.fptoui.i8.x86_fp80(x86_fp80 %x,
+                                               metadata !"fpexcept.strict") #0
+  ret i8 %result
+}
+
+define i16 @fp80_to_uint16(x86_fp80 %x) #0 {
+; X86-LABEL: fp80_to_uint16:
+; X86:       # %bb.0:
+; X86-NEXT:    subl $8, %esp
+; X86-NEXT:    .cfi_def_cfa_offset 12
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fnstcw (%esp)
+; X86-NEXT:    movzwl (%esp), %eax
+; X86-NEXT:    orl $3072, %eax # imm = 0xC00
+; X86-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; X86-NEXT:    fldcw {{[0-9]+}}(%esp)
+; X86-NEXT:    fistpl {{[0-9]+}}(%esp)
+; X86-NEXT:    fldcw (%esp)
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    # kill: def $ax killed $ax killed $eax
+; X86-NEXT:    addl $8, %esp
+; X86-NEXT:    .cfi_def_cfa_offset 4
+; X86-NEXT:    retl
+;
+; X64-LABEL: fp80_to_uint16:
+; X64:       # %bb.0:
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fnstcw -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movzwl -{{[0-9]+}}(%rsp), %eax
+; X64-NEXT:    orl $3072, %eax # imm = 0xC00
+; X64-NEXT:    movw %ax, -{{[0-9]+}}(%rsp)
+; X64-NEXT:    fldcw -{{[0-9]+}}(%rsp)
+; X64-NEXT:    fistpl -{{[0-9]+}}(%rsp)
+; X64-NEXT:    fldcw -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movl -{{[0-9]+}}(%rsp), %eax
+; X64-NEXT:    # kill: def $ax killed $ax killed $eax
+; X64-NEXT:    retq
+  %result = call i16 @llvm.experimental.constrained.fptoui.i16.x86_fp80(x86_fp80 %x,
+                                               metadata !"fpexcept.strict") #0
+  ret i16 %result
+}
+
+define i32 @fp80_to_uint32(x86_fp80 %x) #0 {
+; X86-LABEL: fp80_to_uint32:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    .cfi_def_cfa_offset 8
+; X86-NEXT:    .cfi_offset %ebp, -8
+; X86-NEXT:    movl %esp, %ebp
+; X86-NEXT:    .cfi_def_cfa_register %ebp
+; X86-NEXT:    andl $-8, %esp
+; X86-NEXT:    subl $16, %esp
+; X86-NEXT:    fldt 8(%ebp)
+; X86-NEXT:    fnstcw {{[0-9]+}}(%esp)
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    orl $3072, %eax # imm = 0xC00
+; X86-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; X86-NEXT:    fldcw {{[0-9]+}}(%esp)
+; X86-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-NEXT:    fldcw {{[0-9]+}}(%esp)
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl %ebp, %esp
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    .cfi_def_cfa %esp, 4
+; X86-NEXT:    retl
+;
+; X64-LABEL: fp80_to_uint32:
+; X64:       # %bb.0:
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fnstcw -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movzwl -{{[0-9]+}}(%rsp), %eax
+; X64-NEXT:    orl $3072, %eax # imm = 0xC00
+; X64-NEXT:    movw %ax, -{{[0-9]+}}(%rsp)
+; X64-NEXT:    fldcw -{{[0-9]+}}(%rsp)
+; X64-NEXT:    fistpll -{{[0-9]+}}(%rsp)
+; X64-NEXT:    fldcw -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movl -{{[0-9]+}}(%rsp), %eax
+; X64-NEXT:    retq
+  %result = call i32 @llvm.experimental.constrained.fptoui.i32.x86_fp80(x86_fp80 %x,
+                                               metadata !"fpexcept.strict") #0
+  ret i32 %result
+}
+
+define i64 @fp80_to_uint64(x86_fp80 %x) #0 {
+; X86-LABEL: fp80_to_uint64:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    .cfi_def_cfa_offset 8
+; X86-NEXT:    .cfi_offset %ebp, -8
+; X86-NEXT:    movl %esp, %ebp
+; X86-NEXT:    .cfi_def_cfa_register %ebp
+; X86-NEXT:    andl $-8, %esp
+; X86-NEXT:    subl $16, %esp
+; X86-NEXT:    fldt 8(%ebp)
+; X86-NEXT:    flds {{\.LCPI.*}}
+; X86-NEXT:    fcom %st(1)
+; X86-NEXT:    fnstsw %ax
+; X86-NEXT:    xorl %edx, %edx
+; X86-NEXT:    # kill: def $ah killed $ah killed $ax
+; X86-NEXT:    sahf
+; X86-NEXT:    setbe %al
+; X86-NEXT:    fldz
+; X86-NEXT:    ja .LBB18_2
+; X86-NEXT:  # %bb.1:
+; X86-NEXT:    fstp %st(0)
+; X86-NEXT:    fldz
+; X86-NEXT:    fxch %st(1)
+; X86-NEXT:  .LBB18_2:
+; X86-NEXT:    fstp %st(1)
+; X86-NEXT:    fsubrp %st, %st(1)
+; X86-NEXT:    fnstcw {{[0-9]+}}(%esp)
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    orl $3072, %ecx # imm = 0xC00
+; X86-NEXT:    movw %cx, {{[0-9]+}}(%esp)
+; X86-NEXT:    fldcw {{[0-9]+}}(%esp)
+; X86-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-NEXT:    fldcw {{[0-9]+}}(%esp)
+; X86-NEXT:    movb %al, %dl
+; X86-NEXT:    shll $31, %edx
+; X86-NEXT:    xorl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl %ebp, %esp
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    .cfi_def_cfa %esp, 4
+; X86-NEXT:    retl
+;
+; X64-LABEL: fp80_to_uint64:
+; X64:       # %bb.0:
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    flds {{.*}}(%rip)
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    fcomi %st(1), %st
+; X64-NEXT:    setbe %al
+; X64-NEXT:    fldz
+; X64-NEXT:    fxch %st(1)
+; X64-NEXT:    fcmovnbe %st(1), %st
+; X64-NEXT:    fstp %st(1)
+; X64-NEXT:    fsubrp %st, %st(1)
+; X64-NEXT:    fnstcw -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movzwl -{{[0-9]+}}(%rsp), %ecx
+; X64-NEXT:    orl $3072, %ecx # imm = 0xC00
+; X64-NEXT:    movw %cx, -{{[0-9]+}}(%rsp)
+; X64-NEXT:    fldcw -{{[0-9]+}}(%rsp)
+; X64-NEXT:    fistpll -{{[0-9]+}}(%rsp)
+; X64-NEXT:    fldcw -{{[0-9]+}}(%rsp)
+; X64-NEXT:    shlq $63, %rax
+; X64-NEXT:    xorq -{{[0-9]+}}(%rsp), %rax
+; X64-NEXT:    retq
+  %result = call i64 @llvm.experimental.constrained.fptoui.i64.x86_fp80(x86_fp80 %x,
+                                               metadata !"fpexcept.strict") #0
+  ret i64 %result
+}
+
+define x86_fp80 @sint1_to_fp80(i1 %x) #0 {
+; X86-LABEL: sint1_to_fp80:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    .cfi_def_cfa_offset 8
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    andb $1, %al
+; X86-NEXT:    negb %al
+; X86-NEXT:    movsbl %al, %eax
+; X86-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; X86-NEXT:    filds {{[0-9]+}}(%esp)
+; X86-NEXT:    popl %eax
+; X86-NEXT:    .cfi_def_cfa_offset 4
+; X86-NEXT:    retl
+;
+; X64-LABEL: sint1_to_fp80:
+; X64:       # %bb.0:
+; X64-NEXT:    andb $1, %dil
+; X64-NEXT:    negb %dil
+; X64-NEXT:    movsbl %dil, %eax
+; X64-NEXT:    movw %ax, -{{[0-9]+}}(%rsp)
+; X64-NEXT:    filds -{{[0-9]+}}(%rsp)
+; X64-NEXT:    retq
+  %result = call x86_fp80 @llvm.experimental.constrained.sitofp.x86_fp80.i1(i1 %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret x86_fp80 %result
+}
+
+define x86_fp80 @sint8_to_fp80(i8 %x) #0 {
+; X86-LABEL: sint8_to_fp80:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    .cfi_def_cfa_offset 8
+; X86-NEXT:    movsbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; X86-NEXT:    filds {{[0-9]+}}(%esp)
+; X86-NEXT:    popl %eax
+; X86-NEXT:    .cfi_def_cfa_offset 4
+; X86-NEXT:    retl
+;
+; X64-LABEL: sint8_to_fp80:
+; X64:       # %bb.0:
+; X64-NEXT:    movsbl %dil, %eax
+; X64-NEXT:    movw %ax, -{{[0-9]+}}(%rsp)
+; X64-NEXT:    filds -{{[0-9]+}}(%rsp)
+; X64-NEXT:    retq
+  %result = call x86_fp80 @llvm.experimental.constrained.sitofp.x86_fp80.i8(i8 %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret x86_fp80 %result
+}
+
+define x86_fp80 @sint16_to_fp80(i16 %x) #0 {
+; X86-LABEL: sint16_to_fp80:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    .cfi_def_cfa_offset 8
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; X86-NEXT:    filds {{[0-9]+}}(%esp)
+; X86-NEXT:    popl %eax
+; X86-NEXT:    .cfi_def_cfa_offset 4
+; X86-NEXT:    retl
+;
+; X64-LABEL: sint16_to_fp80:
+; X64:       # %bb.0:
+; X64-NEXT:    movw %di, -{{[0-9]+}}(%rsp)
+; X64-NEXT:    filds -{{[0-9]+}}(%rsp)
+; X64-NEXT:    retq
+  %result = call x86_fp80 @llvm.experimental.constrained.sitofp.x86_fp80.i16(i16 %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret x86_fp80 %result
+}
+
+define x86_fp80 @sint32_to_fp80(i32 %x) #0 {
+; X86-LABEL: sint32_to_fp80:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    .cfi_def_cfa_offset 8
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl %eax, (%esp)
+; X86-NEXT:    fildl (%esp)
+; X86-NEXT:    popl %eax
+; X86-NEXT:    .cfi_def_cfa_offset 4
+; X86-NEXT:    retl
+;
+; X64-LABEL: sint32_to_fp80:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, -{{[0-9]+}}(%rsp)
+; X64-NEXT:    fildl -{{[0-9]+}}(%rsp)
+; X64-NEXT:    retq
+  %result = call x86_fp80 @llvm.experimental.constrained.sitofp.x86_fp80.i32(i32 %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret x86_fp80 %result
+}
+
+define x86_fp80 @sint64_to_fp80(i64 %x) #0 {
+; X86-LABEL: sint64_to_fp80:
+; X86:       # %bb.0:
+; X86-NEXT:    fildll {{[0-9]+}}(%esp)
+; X86-NEXT:    retl
+;
+; X64-LABEL: sint64_to_fp80:
+; X64:       # %bb.0:
+; X64-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NEXT:    fildll -{{[0-9]+}}(%rsp)
+; X64-NEXT:    retq
+  %result = call x86_fp80 @llvm.experimental.constrained.sitofp.x86_fp80.i64(i64 %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret x86_fp80 %result
+}
+
+define x86_fp80 @uint1_to_fp80(i1 %x) #0 {
+; X86-LABEL: uint1_to_fp80:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    .cfi_def_cfa_offset 8
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    andb $1, %al
+; X86-NEXT:    movzbl %al, %eax
+; X86-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; X86-NEXT:    filds {{[0-9]+}}(%esp)
+; X86-NEXT:    popl %eax
+; X86-NEXT:    .cfi_def_cfa_offset 4
+; X86-NEXT:    retl
+;
+; X64-LABEL: uint1_to_fp80:
+; X64:       # %bb.0:
+; X64-NEXT:    andl $1, %edi
+; X64-NEXT:    movw %di, -{{[0-9]+}}(%rsp)
+; X64-NEXT:    filds -{{[0-9]+}}(%rsp)
+; X64-NEXT:    retq
+  %result = call x86_fp80 @llvm.experimental.constrained.uitofp.x86_fp80.i1(i1 %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret x86_fp80 %result
+}
+
+define x86_fp80 @uint8_to_fp80(i8 %x) #0 {
+; X86-LABEL: uint8_to_fp80:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    .cfi_def_cfa_offset 8
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; X86-NEXT:    filds {{[0-9]+}}(%esp)
+; X86-NEXT:    popl %eax
+; X86-NEXT:    .cfi_def_cfa_offset 4
+; X86-NEXT:    retl
+;
+; X64-LABEL: uint8_to_fp80:
+; X64:       # %bb.0:
+; X64-NEXT:    movzbl %dil, %eax
+; X64-NEXT:    movw %ax, -{{[0-9]+}}(%rsp)
+; X64-NEXT:    filds -{{[0-9]+}}(%rsp)
+; X64-NEXT:    retq
+  %result = call x86_fp80 @llvm.experimental.constrained.uitofp.x86_fp80.i8(i8 %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret x86_fp80 %result
+}
+
+define x86_fp80 @uint16_to_fp80(i16 %x) #0 {
+; X86-LABEL: uint16_to_fp80:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    .cfi_def_cfa_offset 8
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl %eax, (%esp)
+; X86-NEXT:    fildl (%esp)
+; X86-NEXT:    popl %eax
+; X86-NEXT:    .cfi_def_cfa_offset 4
+; X86-NEXT:    retl
+;
+; X64-LABEL: uint16_to_fp80:
+; X64:       # %bb.0:
+; X64-NEXT:    movzwl %di, %eax
+; X64-NEXT:    movl %eax, -{{[0-9]+}}(%rsp)
+; X64-NEXT:    fildl -{{[0-9]+}}(%rsp)
+; X64-NEXT:    retq
+  %result = call x86_fp80 @llvm.experimental.constrained.uitofp.x86_fp80.i16(i16 %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret x86_fp80 %result
+}
+
+define x86_fp80 @uint32_to_fp80(i32 %x) #0 {
+; X86-LABEL: uint32_to_fp80:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    .cfi_def_cfa_offset 8
+; X86-NEXT:    .cfi_offset %ebp, -8
+; X86-NEXT:    movl %esp, %ebp
+; X86-NEXT:    .cfi_def_cfa_register %ebp
+; X86-NEXT:    andl $-8, %esp
+; X86-NEXT:    subl $8, %esp
+; X86-NEXT:    movl 8(%ebp), %eax
+; X86-NEXT:    movl %eax, (%esp)
+; X86-NEXT:    movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT:    fildll (%esp)
+; X86-NEXT:    movl %ebp, %esp
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    .cfi_def_cfa %esp, 4
+; X86-NEXT:    retl
+;
+; X64-LABEL: uint32_to_fp80:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
+; X64-NEXT:    fildll -{{[0-9]+}}(%rsp)
+; X64-NEXT:    retq
+  %result = call x86_fp80 @llvm.experimental.constrained.uitofp.x86_fp80.i32(i32 %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret x86_fp80 %result
+}
+
+define x86_fp80 @uint64_to_fp80(i64 %x) #0 {
+; X86-LABEL: uint64_to_fp80:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    .cfi_def_cfa_offset 8
+; X86-NEXT:    .cfi_offset %ebp, -8
+; X86-NEXT:    movl %esp, %ebp
+; X86-NEXT:    .cfi_def_cfa_register %ebp
+; X86-NEXT:    andl $-8, %esp
+; X86-NEXT:    subl $8, %esp
+; X86-NEXT:    movl 8(%ebp), %eax
+; X86-NEXT:    movl 12(%ebp), %ecx
+; X86-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NEXT:    movl %eax, (%esp)
+; X86-NEXT:    shrl $31, %ecx
+; X86-NEXT:    fildll (%esp)
+; X86-NEXT:    fadds {{\.LCPI.*}}(,%ecx,4)
+; X86-NEXT:    movl %ebp, %esp
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    .cfi_def_cfa %esp, 4
+; X86-NEXT:    retl
+;
+; X64-LABEL: uint64_to_fp80:
+; X64:       # %bb.0:
+; X64-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    testq %rdi, %rdi
+; X64-NEXT:    sets %al
+; X64-NEXT:    fildll -{{[0-9]+}}(%rsp)
+; X64-NEXT:    fadds {{\.LCPI.*}}(,%rax,4)
+; X64-NEXT:    retq
+  %result = call x86_fp80 @llvm.experimental.constrained.uitofp.x86_fp80.i64(i64 %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret x86_fp80 %result
+}
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/X86/fp_load_fold.ll b/llvm/test/CodeGen/X86/fp_load_fold.ll
index 4600c4ebe..11eacc482 100644
--- a/llvm/test/CodeGen/X86/fp_load_fold.ll
+++ b/llvm/test/CodeGen/X86/fp_load_fold.ll
@@ -1,39 +1,75 @@
-; RUN: llc < %s -mtriple=i686-- -x86-asm-syntax=intel | \
-; RUN:   grep -i ST | not grep "fadd\|fsub\|fdiv\|fmul"
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- -x86-asm-syntax=intel | FileCheck %s
 
 ; Test that the load of the memory location is folded into the operation.
 
 define double @test_add(double %X, double* %P) {
+; CHECK-LABEL: test_add:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mov eax, dword ptr [esp + 12]
+; CHECK-NEXT:    fld qword ptr [esp + 4]
+; CHECK-NEXT:    fadd qword ptr [eax]
+; CHECK-NEXT:    ret
 	%Y = load double, double* %P		; <double> [#uses=1]
 	%R = fadd double %X, %Y		; <double> [#uses=1]
 	ret double %R
 }
 
 define double @test_mul(double %X, double* %P) {
+; CHECK-LABEL: test_mul:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mov eax, dword ptr [esp + 12]
+; CHECK-NEXT:    fld qword ptr [esp + 4]
+; CHECK-NEXT:    fmul qword ptr [eax]
+; CHECK-NEXT:    ret
 	%Y = load double, double* %P		; <double> [#uses=1]
 	%R = fmul double %X, %Y		; <double> [#uses=1]
 	ret double %R
 }
 
 define double @test_sub(double %X, double* %P) {
+; CHECK-LABEL: test_sub:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mov eax, dword ptr [esp + 12]
+; CHECK-NEXT:    fld qword ptr [esp + 4]
+; CHECK-NEXT:    fsub qword ptr [eax]
+; CHECK-NEXT:    ret
 	%Y = load double, double* %P		; <double> [#uses=1]
 	%R = fsub double %X, %Y		; <double> [#uses=1]
 	ret double %R
 }
 
 define double @test_subr(double %X, double* %P) {
+; CHECK-LABEL: test_subr:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mov eax, dword ptr [esp + 12]
+; CHECK-NEXT:    fld qword ptr [eax]
+; CHECK-NEXT:    fsub qword ptr [esp + 4]
+; CHECK-NEXT:    ret
 	%Y = load double, double* %P		; <double> [#uses=1]
 	%R = fsub double %Y, %X		; <double> [#uses=1]
 	ret double %R
 }
 
 define double @test_div(double %X, double* %P) {
+; CHECK-LABEL: test_div:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mov eax, dword ptr [esp + 12]
+; CHECK-NEXT:    fld qword ptr [esp + 4]
+; CHECK-NEXT:    fdiv qword ptr [eax]
+; CHECK-NEXT:    ret
 	%Y = load double, double* %P		; <double> [#uses=1]
 	%R = fdiv double %X, %Y		; <double> [#uses=1]
 	ret double %R
 }
 
 define double @test_divr(double %X, double* %P) {
+; CHECK-LABEL: test_divr:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mov eax, dword ptr [esp + 12]
+; CHECK-NEXT:    fld qword ptr [eax]
+; CHECK-NEXT:    fdiv qword ptr [esp + 4]
+; CHECK-NEXT:    ret
 	%Y = load double, double* %P		; <double> [#uses=1]
 	%R = fdiv double %Y, %X		; <double> [#uses=1]
 	ret double %R
diff --git a/llvm/test/CodeGen/X86/fpcmp-soft-fp.ll b/llvm/test/CodeGen/X86/fpcmp-soft-fp.ll
index f96bf65e4..e89acc6bb 100644
--- a/llvm/test/CodeGen/X86/fpcmp-soft-fp.ll
+++ b/llvm/test/CodeGen/X86/fpcmp-soft-fp.ll
@@ -100,6 +100,7 @@ entry:
 ; CHECK: sete
 ; CHECK: calll __unorddf2
 ; CHECK: setne
+; CHECK: or
 ; CHECK: retl
 
 define i1 @test11(double %d) #0 {
@@ -108,10 +109,11 @@ entry:
   ret i1 %cmp
 }
 ; CHECK-LABEL: test11:
-; CHECK: calll __gtdf2
-; CHECK: setg
-; CHECK: calll __ltdf2
-; CHECK: sets
+; CHECK: calll __eqdf2
+; CHECK: setne
+; CHECK: calll __unorddf2
+; CHECK: sete
+; CHECK: and
 ; CHECK: retl
 
 define i1 @test12(double %d) #0 {
diff --git a/llvm/test/CodeGen/X86/frame-order.ll b/llvm/test/CodeGen/X86/frame-order.ll
index 7ba63d0c4..29fc6779f 100644
--- a/llvm/test/CodeGen/X86/frame-order.ll
+++ b/llvm/test/CodeGen/X86/frame-order.ll
@@ -74,9 +74,9 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
 
 declare void @capture(i32*) #2
 
-attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
-attributes #2 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #3 = { nounwind }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/CodeGen/X86/frameregister.ll b/llvm/test/CodeGen/X86/frameregister.ll
index 826bb9d78..3d196a3c4 100644
--- a/llvm/test/CodeGen/X86/frameregister.ll
+++ b/llvm/test/CodeGen/X86/frameregister.ll
@@ -27,4 +27,4 @@ declare void @llvm.write_register.i64(metadata, i64) nounwind
 ; CHECK-NOT: .asciz  "rbp"
 !0 = !{!"rbp\00"}
 
-attributes #0 = { nounwind "no-frame-pointer-elim"="true" }
+attributes #0 = { nounwind "frame-pointer"="all" }
diff --git a/llvm/test/CodeGen/X86/fshl.ll b/llvm/test/CodeGen/X86/fshl.ll
index c7ca98ddf..522a03afa 100644
--- a/llvm/test/CodeGen/X86/fshl.ll
+++ b/llvm/test/CodeGen/X86/fshl.ll
@@ -196,6 +196,26 @@ define i32 @var_shift_i32_optsize(i32 %x, i32 %y, i32 %z) nounwind optsize {
   ret i32 %tmp
 }
 
+define i32 @var_shift_i32_pgso(i32 %x, i32 %y, i32 %z) nounwind !prof !14 {
+; X86-LABEL: var_shift_i32_pgso:
+; X86:       # %bb.0:
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    shldl %cl, %edx, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: var_shift_i32_pgso:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edx, %ecx
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-NEXT:    shldl %cl, %esi, %eax
+; X64-NEXT:    retq
+  %tmp = tail call i32 @llvm.fshl.i32(i32 %x, i32 %y, i32 %z)
+  ret i32 %tmp
+}
+
 define i64 @var_shift_i64(i64 %x, i64 %y, i64 %z) nounwind {
 ; X86-FAST-LABEL: var_shift_i64:
 ; X86-FAST:       # %bb.0:
@@ -216,36 +236,36 @@ define i64 @var_shift_i64(i64 %x, i64 %y, i64 %z) nounwind {
 ; X86-FAST-NEXT:    shll %cl, %edi
 ; X86-FAST-NEXT:    shldl %cl, %eax, %ebp
 ; X86-FAST-NEXT:    testb $32, %bl
-; X86-FAST-NEXT:    je .LBB4_2
+; X86-FAST-NEXT:    je .LBB5_2
 ; X86-FAST-NEXT:  # %bb.1:
 ; X86-FAST-NEXT:    movl %edi, %ebp
 ; X86-FAST-NEXT:    xorl %edi, %edi
-; X86-FAST-NEXT:  .LBB4_2:
+; X86-FAST-NEXT:  .LBB5_2:
 ; X86-FAST-NEXT:    movb $64, %cl
 ; X86-FAST-NEXT:    subb %bl, %cl
 ; X86-FAST-NEXT:    movl %edx, %esi
 ; X86-FAST-NEXT:    shrl %cl, %esi
 ; X86-FAST-NEXT:    shrdl %cl, %edx, (%esp) # 4-byte Folded Spill
 ; X86-FAST-NEXT:    testb $32, %cl
-; X86-FAST-NEXT:    jne .LBB4_3
+; X86-FAST-NEXT:    jne .LBB5_3
 ; X86-FAST-NEXT:  # %bb.4:
 ; X86-FAST-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-FAST-NEXT:    movl (%esp), %ecx # 4-byte Reload
 ; X86-FAST-NEXT:    testl %ebx, %ebx
-; X86-FAST-NEXT:    jne .LBB4_6
-; X86-FAST-NEXT:    jmp .LBB4_7
-; X86-FAST-NEXT:  .LBB4_3:
+; X86-FAST-NEXT:    jne .LBB5_6
+; X86-FAST-NEXT:    jmp .LBB5_7
+; X86-FAST-NEXT:  .LBB5_3:
 ; X86-FAST-NEXT:    movl %esi, %ecx
 ; X86-FAST-NEXT:    xorl %esi, %esi
 ; X86-FAST-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-FAST-NEXT:    testl %ebx, %ebx
-; X86-FAST-NEXT:    je .LBB4_7
-; X86-FAST-NEXT:  .LBB4_6:
+; X86-FAST-NEXT:    je .LBB5_7
+; X86-FAST-NEXT:  .LBB5_6:
 ; X86-FAST-NEXT:    orl %esi, %ebp
 ; X86-FAST-NEXT:    orl %ecx, %edi
 ; X86-FAST-NEXT:    movl %edi, %eax
 ; X86-FAST-NEXT:    movl %ebp, %edx
-; X86-FAST-NEXT:  .LBB4_7:
+; X86-FAST-NEXT:  .LBB5_7:
 ; X86-FAST-NEXT:    addl $4, %esp
 ; X86-FAST-NEXT:    popl %esi
 ; X86-FAST-NEXT:    popl %edi
@@ -279,11 +299,11 @@ define i64 @var_shift_i64(i64 %x, i64 %y, i64 %z) nounwind {
 ; X86-SLOW-NEXT:    testb %dl, %dl
 ; X86-SLOW-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-SLOW-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-SLOW-NEXT:    je .LBB4_2
+; X86-SLOW-NEXT:    je .LBB5_2
 ; X86-SLOW-NEXT:  # %bb.1:
 ; X86-SLOW-NEXT:    orl %eax, %ebp
 ; X86-SLOW-NEXT:    movl %ebp, (%esp) # 4-byte Spill
-; X86-SLOW-NEXT:  .LBB4_2:
+; X86-SLOW-NEXT:  .LBB5_2:
 ; X86-SLOW-NEXT:    movl {{[0-9]+}}(%esp), %ebp
 ; X86-SLOW-NEXT:    movl %ebp, %eax
 ; X86-SLOW-NEXT:    movl %ebx, %ecx
@@ -294,41 +314,41 @@ define i64 @var_shift_i64(i64 %x, i64 %y, i64 %z) nounwind {
 ; X86-SLOW-NEXT:    negb %cl
 ; X86-SLOW-NEXT:    shrl %cl, %edi
 ; X86-SLOW-NEXT:    testb %ch, %ch
-; X86-SLOW-NEXT:    je .LBB4_4
+; X86-SLOW-NEXT:    je .LBB5_4
 ; X86-SLOW-NEXT:  # %bb.3:
 ; X86-SLOW-NEXT:    orl %edi, %eax
 ; X86-SLOW-NEXT:    movl %eax, %ebp
-; X86-SLOW-NEXT:  .LBB4_4:
+; X86-SLOW-NEXT:  .LBB5_4:
 ; X86-SLOW-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-SLOW-NEXT:    movl %eax, %edi
 ; X86-SLOW-NEXT:    movl %ebx, %ecx
 ; X86-SLOW-NEXT:    shll %cl, %edi
 ; X86-SLOW-NEXT:    testb $32, %bl
-; X86-SLOW-NEXT:    je .LBB4_6
+; X86-SLOW-NEXT:    je .LBB5_6
 ; X86-SLOW-NEXT:  # %bb.5:
 ; X86-SLOW-NEXT:    movl %edi, %ebp
 ; X86-SLOW-NEXT:    xorl %edi, %edi
-; X86-SLOW-NEXT:  .LBB4_6:
+; X86-SLOW-NEXT:  .LBB5_6:
 ; X86-SLOW-NEXT:    movb %dh, %cl
 ; X86-SLOW-NEXT:    shrl %cl, %esi
 ; X86-SLOW-NEXT:    testb $32, %dh
-; X86-SLOW-NEXT:    jne .LBB4_7
+; X86-SLOW-NEXT:    jne .LBB5_7
 ; X86-SLOW-NEXT:  # %bb.8:
 ; X86-SLOW-NEXT:    movl (%esp), %ecx # 4-byte Reload
 ; X86-SLOW-NEXT:    testl %ebx, %ebx
-; X86-SLOW-NEXT:    jne .LBB4_10
-; X86-SLOW-NEXT:    jmp .LBB4_11
-; X86-SLOW-NEXT:  .LBB4_7:
+; X86-SLOW-NEXT:    jne .LBB5_10
+; X86-SLOW-NEXT:    jmp .LBB5_11
+; X86-SLOW-NEXT:  .LBB5_7:
 ; X86-SLOW-NEXT:    movl %esi, %ecx
 ; X86-SLOW-NEXT:    xorl %esi, %esi
 ; X86-SLOW-NEXT:    testl %ebx, %ebx
-; X86-SLOW-NEXT:    je .LBB4_11
-; X86-SLOW-NEXT:  .LBB4_10:
+; X86-SLOW-NEXT:    je .LBB5_11
+; X86-SLOW-NEXT:  .LBB5_10:
 ; X86-SLOW-NEXT:    orl %esi, %ebp
 ; X86-SLOW-NEXT:    orl %ecx, %edi
 ; X86-SLOW-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-SLOW-NEXT:    movl %edi, %eax
-; X86-SLOW-NEXT:  .LBB4_11:
+; X86-SLOW-NEXT:  .LBB5_11:
 ; X86-SLOW-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
 ; X86-SLOW-NEXT:    addl $8, %esp
 ; X86-SLOW-NEXT:    popl %esi
@@ -503,3 +523,20 @@ define i64 @const_shift_i64(i64 %x, i64 %y) nounwind {
   %tmp = tail call i64 @llvm.fshl.i64(i64 %x, i64 %y, i64 7)
   ret i64 %tmp
 }
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 1, !"ProfileSummary", !1}
+!1 = !{!2, !3, !4, !5, !6, !7, !8, !9}
+!2 = !{!"ProfileFormat", !"InstrProf"}
+!3 = !{!"TotalCount", i64 10000}
+!4 = !{!"MaxCount", i64 10}
+!5 = !{!"MaxInternalCount", i64 1}
+!6 = !{!"MaxFunctionCount", i64 1000}
+!7 = !{!"NumCounts", i64 3}
+!8 = !{!"NumFunctions", i64 3}
+!9 = !{!"DetailedSummary", !10}
+!10 = !{!11, !12, !13}
+!11 = !{i32 10000, i64 100, i32 1}
+!12 = !{i32 999000, i64 100, i32 1}
+!13 = !{i32 999999, i64 1, i32 2}
+!14 = !{!"function_entry_count", i64 0}
diff --git a/llvm/test/CodeGen/X86/fshr.ll b/llvm/test/CodeGen/X86/fshr.ll
index a655c5804..26e284f15 100644
--- a/llvm/test/CodeGen/X86/fshr.ll
+++ b/llvm/test/CodeGen/X86/fshr.ll
@@ -195,6 +195,26 @@ define i32 @var_shift_i32_optsize(i32 %x, i32 %y, i32 %z) nounwind optsize {
   ret i32 %tmp
 }
 
+define i32 @var_shift_i32_pgso(i32 %x, i32 %y, i32 %z) nounwind !prof !14 {
+; X86-LABEL: var_shift_i32_pgso:
+; X86:       # %bb.0:
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    shrdl %cl, %edx, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: var_shift_i32_pgso:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edx, %ecx
+; X64-NEXT:    movl %esi, %eax
+; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-NEXT:    shrdl %cl, %edi, %eax
+; X64-NEXT:    retq
+  %tmp = tail call i32 @llvm.fshr.i32(i32 %x, i32 %y, i32 %z)
+  ret i32 %tmp
+}
+
 define i64 @var_shift_i64(i64 %x, i64 %y, i64 %z) nounwind {
 ; X86-FAST-LABEL: var_shift_i64:
 ; X86-FAST:       # %bb.0:
@@ -216,30 +236,30 @@ define i64 @var_shift_i64(i64 %x, i64 %y, i64 %z) nounwind {
 ; X86-FAST-NEXT:    shll %cl, %edi
 ; X86-FAST-NEXT:    shldl %cl, %eax, %esi
 ; X86-FAST-NEXT:    testb $32, %cl
-; X86-FAST-NEXT:    je .LBB4_2
+; X86-FAST-NEXT:    je .LBB5_2
 ; X86-FAST-NEXT:  # %bb.1:
 ; X86-FAST-NEXT:    movl %edi, %esi
 ; X86-FAST-NEXT:    xorl %edi, %edi
-; X86-FAST-NEXT:  .LBB4_2:
+; X86-FAST-NEXT:  .LBB5_2:
 ; X86-FAST-NEXT:    movl %edx, %ebp
 ; X86-FAST-NEXT:    movl %ebx, %ecx
 ; X86-FAST-NEXT:    shrl %cl, %ebp
 ; X86-FAST-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-FAST-NEXT:    shrdl %cl, %edx, %eax
 ; X86-FAST-NEXT:    testb $32, %bl
-; X86-FAST-NEXT:    je .LBB4_4
+; X86-FAST-NEXT:    je .LBB5_4
 ; X86-FAST-NEXT:  # %bb.3:
 ; X86-FAST-NEXT:    movl %ebp, %eax
 ; X86-FAST-NEXT:    xorl %ebp, %ebp
-; X86-FAST-NEXT:  .LBB4_4:
+; X86-FAST-NEXT:  .LBB5_4:
 ; X86-FAST-NEXT:    testl %ebx, %ebx
-; X86-FAST-NEXT:    je .LBB4_6
+; X86-FAST-NEXT:    je .LBB5_6
 ; X86-FAST-NEXT:  # %bb.5:
 ; X86-FAST-NEXT:    orl %ebp, %esi
 ; X86-FAST-NEXT:    orl %eax, %edi
 ; X86-FAST-NEXT:    movl %edi, (%esp) # 4-byte Spill
 ; X86-FAST-NEXT:    movl %esi, %edx
-; X86-FAST-NEXT:  .LBB4_6:
+; X86-FAST-NEXT:  .LBB5_6:
 ; X86-FAST-NEXT:    movl (%esp), %eax # 4-byte Reload
 ; X86-FAST-NEXT:    addl $4, %esp
 ; X86-FAST-NEXT:    popl %esi
@@ -274,13 +294,12 @@ define i64 @var_shift_i64(i64 %x, i64 %y, i64 %z) nounwind {
 ; X86-SLOW-NEXT:    shrl %cl, %edi
 ; X86-SLOW-NEXT:    testb %ch, %ch
 ; X86-SLOW-NEXT:    movl {{[0-9]+}}(%esp), %ebp
-; X86-SLOW-NEXT:    je .LBB4_2
+; X86-SLOW-NEXT:    je .LBB5_2
 ; X86-SLOW-NEXT:  # %bb.1:
 ; X86-SLOW-NEXT:    orl %edi, %edx
 ; X86-SLOW-NEXT:    movl %edx, (%esp) # 4-byte Spill
-; X86-SLOW-NEXT:  .LBB4_2:
-; X86-SLOW-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-SLOW-NEXT:    movl %ecx, %edx
+; X86-SLOW-NEXT:  .LBB5_2:
+; X86-SLOW-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-SLOW-NEXT:    movl %ebx, %ecx
 ; X86-SLOW-NEXT:    shrl %cl, %edx
 ; X86-SLOW-NEXT:    movb %bl, %ah
@@ -291,41 +310,41 @@ define i64 @var_shift_i64(i64 %x, i64 %y, i64 %z) nounwind {
 ; X86-SLOW-NEXT:    shll %cl, %edi
 ; X86-SLOW-NEXT:    testb %ah, %ah
 ; X86-SLOW-NEXT:    movl {{[0-9]+}}(%esp), %ebp
-; X86-SLOW-NEXT:    je .LBB4_4
+; X86-SLOW-NEXT:    je .LBB5_4
 ; X86-SLOW-NEXT:  # %bb.3:
 ; X86-SLOW-NEXT:    orl %edx, %edi
 ; X86-SLOW-NEXT:    movl %edi, %ebp
-; X86-SLOW-NEXT:  .LBB4_4:
+; X86-SLOW-NEXT:  .LBB5_4:
 ; X86-SLOW-NEXT:    movl {{[0-9]+}}(%esp), %edi
 ; X86-SLOW-NEXT:    movl %ebx, %ecx
 ; X86-SLOW-NEXT:    shrl %cl, %edi
 ; X86-SLOW-NEXT:    testb $32, %bl
-; X86-SLOW-NEXT:    je .LBB4_6
+; X86-SLOW-NEXT:    je .LBB5_6
 ; X86-SLOW-NEXT:  # %bb.5:
 ; X86-SLOW-NEXT:    movl %edi, %ebp
 ; X86-SLOW-NEXT:    xorl %edi, %edi
-; X86-SLOW-NEXT:  .LBB4_6:
+; X86-SLOW-NEXT:  .LBB5_6:
 ; X86-SLOW-NEXT:    movl %eax, %ecx
 ; X86-SLOW-NEXT:    shll %cl, %esi
 ; X86-SLOW-NEXT:    testb $32, %al
 ; X86-SLOW-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-SLOW-NEXT:    jne .LBB4_7
+; X86-SLOW-NEXT:    jne .LBB5_7
 ; X86-SLOW-NEXT:  # %bb.8:
 ; X86-SLOW-NEXT:    movl (%esp), %eax # 4-byte Reload
 ; X86-SLOW-NEXT:    testl %ebx, %ebx
-; X86-SLOW-NEXT:    jne .LBB4_10
-; X86-SLOW-NEXT:    jmp .LBB4_11
-; X86-SLOW-NEXT:  .LBB4_7:
+; X86-SLOW-NEXT:    jne .LBB5_10
+; X86-SLOW-NEXT:    jmp .LBB5_11
+; X86-SLOW-NEXT:  .LBB5_7:
 ; X86-SLOW-NEXT:    movl %esi, %eax
 ; X86-SLOW-NEXT:    xorl %esi, %esi
 ; X86-SLOW-NEXT:    testl %ebx, %ebx
-; X86-SLOW-NEXT:    je .LBB4_11
-; X86-SLOW-NEXT:  .LBB4_10:
+; X86-SLOW-NEXT:    je .LBB5_11
+; X86-SLOW-NEXT:  .LBB5_10:
 ; X86-SLOW-NEXT:    orl %ebp, %esi
 ; X86-SLOW-NEXT:    orl %edi, %eax
 ; X86-SLOW-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-SLOW-NEXT:    movl %eax, %edx
-; X86-SLOW-NEXT:  .LBB4_11:
+; X86-SLOW-NEXT:  .LBB5_11:
 ; X86-SLOW-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X86-SLOW-NEXT:    addl $8, %esp
 ; X86-SLOW-NEXT:    popl %esi
@@ -499,3 +518,20 @@ define i64 @const_shift_i64(i64 %x, i64 %y) nounwind {
   %tmp = tail call i64 @llvm.fshr.i64(i64 %x, i64 %y, i64 7)
   ret i64 %tmp
 }
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 1, !"ProfileSummary", !1}
+!1 = !{!2, !3, !4, !5, !6, !7, !8, !9}
+!2 = !{!"ProfileFormat", !"InstrProf"}
+!3 = !{!"TotalCount", i64 10000}
+!4 = !{!"MaxCount", i64 10}
+!5 = !{!"MaxInternalCount", i64 1}
+!6 = !{!"MaxFunctionCount", i64 1000}
+!7 = !{!"NumCounts", i64 3}
+!8 = !{!"NumFunctions", i64 3}
+!9 = !{!"DetailedSummary", !10}
+!10 = !{!11, !12, !13}
+!11 = !{i32 10000, i64 100, i32 1}
+!12 = !{i32 999000, i64 100, i32 1}
+!13 = !{i32 999999, i64 1, i32 2}
+!14 = !{!"function_entry_count", i64 0}
diff --git a/llvm/test/CodeGen/X86/fsxor-alignment.ll b/llvm/test/CodeGen/X86/fsxor-alignment.ll
index 6f9738f50..c6ab325e0 100644
--- a/llvm/test/CodeGen/X86/fsxor-alignment.ll
+++ b/llvm/test/CodeGen/X86/fsxor-alignment.ll
@@ -1,11 +1,23 @@
-; RUN: llc < %s -mtriple=i686-- -mattr=+sse2 -enable-unsafe-fp-math | \
-; RUN:  grep -v sp | grep xorps | count 2
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- -mattr=+sse2 -enable-unsafe-fp-math | FileCheck %s
 
 ; Don't fold the incoming stack arguments into the xorps instructions used
 ; to do floating-point negations, because the arguments aren't vectors
 ; and aren't vector-aligned.
 
 define void @foo(float* %p, float* %q, float %s, float %y) {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; CHECK-NEXT:    movaps {{.*#+}} xmm1 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
+; CHECK-NEXT:    xorps %xmm1, %xmm0
+; CHECK-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; CHECK-NEXT:    xorps %xmm1, %xmm2
+; CHECK-NEXT:    movss %xmm0, (%ecx)
+; CHECK-NEXT:    movss %xmm2, (%eax)
+; CHECK-NEXT:    retl
   %ss = fsub float -0.0, %s
   %yy = fsub float -0.0, %y
   store float %ss, float* %p
diff --git a/llvm/test/CodeGen/X86/ftrunc.ll b/llvm/test/CodeGen/X86/ftrunc.ll
index 448c21d93..0a1c1e2a8 100644
--- a/llvm/test/CodeGen/X86/ftrunc.ll
+++ b/llvm/test/CodeGen/X86/ftrunc.ll
@@ -78,7 +78,7 @@ define <4 x float> @trunc_unsigned_v4f32(<4 x float> %x) #0 {
 ; SSE2-NEXT:    orps {{.*}}(%rip), %xmm0
 ; SSE2-NEXT:    psrld $16, %xmm1
 ; SSE2-NEXT:    por {{.*}}(%rip), %xmm1
-; SSE2-NEXT:    addps {{.*}}(%rip), %xmm1
+; SSE2-NEXT:    subps {{.*}}(%rip), %xmm1
 ; SSE2-NEXT:    addps %xmm0, %xmm1
 ; SSE2-NEXT:    movaps %xmm1, %xmm0
 ; SSE2-NEXT:    retq
@@ -289,12 +289,12 @@ define <2 x double> @trunc_signed_v2f64(<2 x double> %x) #0 {
 ; SSE2-LABEL: trunc_signed_v2f64:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    cvttsd2si %xmm0, %rax
-; SSE2-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
+; SSE2-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1,1]
 ; SSE2-NEXT:    cvttsd2si %xmm0, %rcx
 ; SSE2-NEXT:    xorps %xmm0, %xmm0
 ; SSE2-NEXT:    cvtsi2sd %rax, %xmm0
 ; SSE2-NEXT:    cvtsi2sd %rcx, %xmm1
-; SSE2-NEXT:    movlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE2-NEXT:    unpcklpd {{.*#+}} xmm0 = xmm0[0],xmm1[0]
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: trunc_signed_v2f64:
@@ -315,20 +315,20 @@ define <4 x double> @trunc_signed_v4f64(<4 x double> %x) #0 {
 ; SSE2-LABEL: trunc_signed_v4f64:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    cvttsd2si %xmm1, %rax
-; SSE2-NEXT:    movhlps {{.*#+}} xmm1 = xmm1[1,1]
+; SSE2-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1,1]
 ; SSE2-NEXT:    cvttsd2si %xmm1, %rcx
 ; SSE2-NEXT:    cvttsd2si %xmm0, %rdx
-; SSE2-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
+; SSE2-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1,1]
 ; SSE2-NEXT:    cvttsd2si %xmm0, %rsi
 ; SSE2-NEXT:    xorps %xmm0, %xmm0
 ; SSE2-NEXT:    cvtsi2sd %rdx, %xmm0
 ; SSE2-NEXT:    xorps %xmm1, %xmm1
 ; SSE2-NEXT:    cvtsi2sd %rsi, %xmm1
-; SSE2-NEXT:    movlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE2-NEXT:    unpcklpd {{.*#+}} xmm0 = xmm0[0],xmm1[0]
 ; SSE2-NEXT:    xorps %xmm1, %xmm1
 ; SSE2-NEXT:    cvtsi2sd %rax, %xmm1
 ; SSE2-NEXT:    cvtsi2sd %rcx, %xmm2
-; SSE2-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; SSE2-NEXT:    unpcklpd {{.*#+}} xmm1 = xmm1[0],xmm2[0]
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: trunc_signed_v4f64:
diff --git a/llvm/test/CodeGen/X86/funclet-layout.ll b/llvm/test/CodeGen/X86/funclet-layout.ll
index 0942645cf..72f60ea72 100644
--- a/llvm/test/CodeGen/X86/funclet-layout.ll
+++ b/llvm/test/CodeGen/X86/funclet-layout.ll
@@ -9,6 +9,8 @@ target triple = "x86_64-pc-windows-msvc"
 @"\01??_7type_info@@6B@" = external constant i8*
 @"\01??_R0H@8" = internal global %rtti.TypeDescriptor2 { i8** @"\01??_7type_info@@6B@", i8* null, [3 x i8] c".H\00" }
 
+declare void @llvm.trap()
+
 define void @test1(i1 %B) personality i32 (...)* @__CxxFrameHandler3 {
 entry:
   invoke void @g()
@@ -31,6 +33,7 @@ try.cont:
   ret void
 
 unreachable:
+  call void @llvm.trap()
   unreachable
 }
 
@@ -76,6 +79,7 @@ try.cont.5:                                       ; preds = %try.cont
   ret i32 0
 
 unreachable:                                      ; preds = %catch, %entry
+  call void @llvm.trap()
   unreachable
 }
 
@@ -125,11 +129,13 @@ try.cont:                                         ; preds = %entry
   br i1 %V, label %exit_one, label %exit_two
 
 exit_one:
-  tail call void @exit(i32 0)
+  tail call void @g()
+  call void @llvm.trap()
   unreachable
 
 exit_two:
-  tail call void @exit(i32 0)
+  tail call void @g()
+  call void @llvm.trap()
   unreachable
 }
 
@@ -138,7 +144,7 @@ exit_two:
 ; The entry funclet contains %entry and %try.cont
 ; CHECK: # %entry
 ; CHECK: # %try.cont
-; CHECK: callq exit
+; CHECK: callq g
 ; CHECK-NOT: # exit_one
 ; CHECK-NOT: # exit_two
 ; CHECK: ud2
@@ -146,12 +152,12 @@ exit_two:
 ; The catch(...) funclet contains %catch.2
 ; CHECK: # %catch.2{{$}}
 ; CHECK: callq exit
-; CHECK: ud2
+; CHECK-NEXT: int3
 
 ; The catch(int) funclet contains %catch
 ; CHECK: # %catch{{$}}
 ; CHECK: callq exit
-; CHECK: ud2
+; CHECK-NEXT: int3
 
 declare void @exit(i32) noreturn nounwind
 declare void @_CxxThrowException(i8*, %eh.ThrowInfo*)
diff --git a/llvm/test/CodeGen/X86/gcc_except_table.ll b/llvm/test/CodeGen/X86/gcc_except_table.ll
index 92ea539bc..a6e802a5b 100644
--- a/llvm/test/CodeGen/X86/gcc_except_table.ll
+++ b/llvm/test/CodeGen/X86/gcc_except_table.ll
@@ -15,7 +15,7 @@ define i32 @main() uwtable optsize ssp personality i8* bitcast (i32 (...)* @__gx
 
 ; MINGW64: .seh_proc
 ; MINGW64: .seh_handler __gxx_personality_v0
-; MINGW64: .seh_setframe 5, 32
+; MINGW64: .seh_setframe %rbp, 32
 ; MINGW64: callq _Unwind_Resume
 ; MINGW64: .seh_handlerdata
 ; MINGW64: .seh_endproc
diff --git a/llvm/test/CodeGen/X86/gpr-to-mask.ll b/llvm/test/CodeGen/X86/gpr-to-mask.ll
index ff2377997..8b6a60740 100644
--- a/llvm/test/CodeGen/X86/gpr-to-mask.ll
+++ b/llvm/test/CodeGen/X86/gpr-to-mask.ll
@@ -13,7 +13,7 @@ define void @test_fcmp_storefloat(i1 %cond, float* %fptr, float %f1, float %f2,
 ; X86-64-NEXT:  .LBB0_2: # %else
 ; X86-64-NEXT:    vcmpeqss %xmm5, %xmm4, %k1
 ; X86-64-NEXT:  .LBB0_3: # %exit
-; X86-64-NEXT:    vmovss %xmm0, %xmm0, %xmm1 {%k1}
+; X86-64-NEXT:    vmovss %xmm0, %xmm1, %xmm1 {%k1}
 ; X86-64-NEXT:    vmovss %xmm1, (%rsi)
 ; X86-64-NEXT:    retq
 ;
@@ -112,7 +112,7 @@ define void @test_load_add(i1 %cond, float* %fptr, i1* %iptr1, i1* %iptr2, float
 ; X86-64-NEXT:    movb (%rcx), %al
 ; X86-64-NEXT:  .LBB2_3: # %exit
 ; X86-64-NEXT:    kmovd %eax, %k1
-; X86-64-NEXT:    vmovss %xmm0, %xmm0, %xmm1 {%k1}
+; X86-64-NEXT:    vmovss %xmm0, %xmm1, %xmm1 {%k1}
 ; X86-64-NEXT:    vmovss %xmm1, (%rsi)
 ; X86-64-NEXT:    retq
 ;
@@ -167,7 +167,7 @@ define void @test_load_i1(i1 %cond, float* %fptr, i1* %iptr1, i1* %iptr2, float
 ; X86-64-NEXT:  .LBB3_2: # %else
 ; X86-64-NEXT:    kmovb (%rcx), %k1
 ; X86-64-NEXT:  .LBB3_3: # %exit
-; X86-64-NEXT:    vmovss %xmm0, %xmm0, %xmm1 {%k1}
+; X86-64-NEXT:    vmovss %xmm0, %xmm1, %xmm1 {%k1}
 ; X86-64-NEXT:    vmovss %xmm1, (%rsi)
 ; X86-64-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/greedy_regalloc_bad_eviction_sequence.ll b/llvm/test/CodeGen/X86/greedy_regalloc_bad_eviction_sequence.ll
index 790bab9f4..f300c8ce3 100644
--- a/llvm/test/CodeGen/X86/greedy_regalloc_bad_eviction_sequence.ll
+++ b/llvm/test/CodeGen/X86/greedy_regalloc_bad_eviction_sequence.ll
@@ -99,8 +99,8 @@ for.body:                                         ; preds = %entry, %for.body
   br i1 %exitcond, label %for.cond.cleanup, label %for.body, !llvm.loop !7
 }
 
-attributes #0 = { norecurse nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-features"="+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { norecurse nounwind readonly "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-features"="+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { norecurse nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-features"="+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { norecurse nounwind readonly "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-features"="+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.module.flags = !{!0, !1}
 !llvm.ident = !{!2}
diff --git a/llvm/test/CodeGen/X86/h-registers-0.ll b/llvm/test/CodeGen/X86/h-registers-0.ll
index dfd79f39b..ae7666669 100644
--- a/llvm/test/CodeGen/X86/h-registers-0.ll
+++ b/llvm/test/CodeGen/X86/h-registers-0.ll
@@ -1,5 +1,6 @@
-; RUN: llc < %s -mattr=-bmi -mtriple=x86_64-linux | FileCheck %s -check-prefix=X86-64
-; RUN: llc < %s -mattr=-bmi -mtriple=x86_64-linux-gnux32 | FileCheck %s -check-prefix=X86-64
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mattr=-bmi -mtriple=x86_64-linux | FileCheck %s -check-prefix=X86-64 -check-prefix=X64
+; RUN: llc < %s -mattr=-bmi -mtriple=x86_64-linux-gnux32 | FileCheck %s -check-prefix=X86-64 -check-prefix=X32
 ; RUN: llc < %s -mattr=-bmi -mtriple=x86_64-win32 | FileCheck %s -check-prefix=WIN64
 ; RUN: llc < %s -mattr=-bmi -mtriple=i686--    | FileCheck %s -check-prefix=X86-32
 
@@ -7,18 +8,36 @@
 ; of h registers yet, due to x86 encoding complications.
 
 define void @bar64(i64 inreg %x, i8* inreg %p) nounwind {
-; X86-64-LABEL: bar64:
-; X86-64: shrq $8, %rdi
-; X86-64: incb %dil
+; X64-LABEL: bar64:
+; X64:       # %bb.0:
+; X64-NEXT:    shrq $8, %rdi
+; X64-NEXT:    incb %dil
+; X64-NEXT:    movb %dil, (%rsi)
+; X64-NEXT:    retq
+;
+; X32-LABEL: bar64:
+; X32:       # %bb.0:
+; X32-NEXT:    shrq $8, %rdi
+; X32-NEXT:    incb %dil
+; X32-NEXT:    movb %dil, (%esi)
+; X32-NEXT:    retq
+;
+; WIN64-LABEL: bar64:
+; WIN64:       # %bb.0:
+; WIN64-NEXT:    shrq $8, %rcx
+; WIN64-NEXT:    incb %cl
+; WIN64-NEXT:    movb %cl, (%rdx)
+; WIN64-NEXT:    retq
+;
+; X86-32-LABEL: bar64:
+; X86-32:       # %bb.0:
+; X86-32-NEXT:    incb %ah
+; X86-32-NEXT:    movb %ah, (%ecx)
+; X86-32-NEXT:    retl
 
 ; See FIXME: on regclass GR8.
 ; It could be optimally transformed like; incb %ch; movb %ch, (%rdx)
-; WIN64-LABEL:  bar64:
-; WIN64:  shrq $8, %rcx
-; WIN64:  incb %cl
 
-; X86-32-LABEL: bar64:
-; X86-32: incb %ah
   %t0 = lshr i64 %x, 8
   %t1 = trunc i64 %t0 to i8
   %t2 = add i8 %t1, 1
@@ -27,16 +46,34 @@ define void @bar64(i64 inreg %x, i8* inreg %p) nounwind {
 }
 
 define void @bar32(i32 inreg %x, i8* inreg %p) nounwind {
-; X86-64-LABEL: bar32:
-; X86-64: shrl $8, %edi
-; X86-64: incb %dil
+; X64-LABEL: bar32:
+; X64:       # %bb.0:
+; X64-NEXT:    shrl $8, %edi
+; X64-NEXT:    incb %dil
+; X64-NEXT:    movb %dil, (%rsi)
+; X64-NEXT:    retq
+;
+; X32-LABEL: bar32:
+; X32:       # %bb.0:
+; X32-NEXT:    shrl $8, %edi
+; X32-NEXT:    incb %dil
+; X32-NEXT:    movb %dil, (%esi)
+; X32-NEXT:    retq
+;
+; WIN64-LABEL: bar32:
+; WIN64:       # %bb.0:
+; WIN64-NEXT:    shrl $8, %ecx
+; WIN64-NEXT:    incb %cl
+; WIN64-NEXT:    movb %cl, (%rdx)
+; WIN64-NEXT:    retq
+;
+; X86-32-LABEL: bar32:
+; X86-32:       # %bb.0:
+; X86-32-NEXT:    incb %ah
+; X86-32-NEXT:    movb %ah, (%edx)
+; X86-32-NEXT:    retl
 
-; WIN64-LABEL:  bar32:
-; WIN64:  shrl $8, %ecx
-; WIN64:  incb %cl
 
-; X86-32-LABEL: bar32:
-; X86-32: incb %ah
   %t0 = lshr i32 %x, 8
   %t1 = trunc i32 %t0 to i8
   %t2 = add i8 %t1, 1
@@ -45,16 +82,35 @@ define void @bar32(i32 inreg %x, i8* inreg %p) nounwind {
 }
 
 define void @bar16(i16 inreg %x, i8* inreg %p) nounwind {
-; X86-64-LABEL: bar16:
-; X86-64: shrl $8, %edi
-; X86-64: incb %dil
+; X64-LABEL: bar16:
+; X64:       # %bb.0:
+; X64-NEXT:    shrl $8, %edi
+; X64-NEXT:    incb %dil
+; X64-NEXT:    movb %dil, (%rsi)
+; X64-NEXT:    retq
+;
+; X32-LABEL: bar16:
+; X32:       # %bb.0:
+; X32-NEXT:    shrl $8, %edi
+; X32-NEXT:    incb %dil
+; X32-NEXT:    movb %dil, (%esi)
+; X32-NEXT:    retq
+;
+; WIN64-LABEL: bar16:
+; WIN64:       # %bb.0:
+; WIN64-NEXT:    # kill: def $cx killed $cx def $ecx
+; WIN64-NEXT:    shrl $8, %ecx
+; WIN64-NEXT:    incb %cl
+; WIN64-NEXT:    movb %cl, (%rdx)
+; WIN64-NEXT:    retq
+;
+; X86-32-LABEL: bar16:
+; X86-32:       # %bb.0:
+; X86-32-NEXT:    incb %ah
+; X86-32-NEXT:    movb %ah, (%edx)
+; X86-32-NEXT:    retl
 
-; WIN64-LABEL:  bar16:
-; WIN64:  shrl $8, %ecx
-; WIN64:  incb %cl
 
-; X86-32-LABEL: bar16:
-; X86-32: incb %ah
   %t0 = lshr i16 %x, 8
   %t1 = trunc i16 %t0 to i8
   %t2 = add i8 %t1, 1
@@ -64,14 +120,23 @@ define void @bar16(i16 inreg %x, i8* inreg %p) nounwind {
 
 define i64 @qux64(i64 inreg %x) nounwind {
 ; X86-64-LABEL: qux64:
-; X86-64: movq %rdi, %rax
-; X86-64: movzbl %ah, %eax
+; X86-64:       # %bb.0:
+; X86-64-NEXT:    movq %rdi, %rax
+; X86-64-NEXT:    movzbl %ah, %eax
+; X86-64-NEXT:    retq
+;
+; WIN64-LABEL: qux64:
+; WIN64:       # %bb.0:
+; WIN64-NEXT:    movzbl %ch, %eax
+; WIN64-NEXT:    retq
+;
+; X86-32-LABEL: qux64:
+; X86-32:       # %bb.0:
+; X86-32-NEXT:    movzbl %ah, %eax
+; X86-32-NEXT:    xorl %edx, %edx
+; X86-32-NEXT:    retl
 
-; WIN64-LABEL:  qux64:
-; WIN64:  movzbl %ch, %eax
 
-; X86-32-LABEL: qux64:
-; X86-32: movzbl %ah, %eax
   %t0 = lshr i64 %x, 8
   %t1 = and i64 %t0, 255
   ret i64 %t1
@@ -79,14 +144,22 @@ define i64 @qux64(i64 inreg %x) nounwind {
 
 define i32 @qux32(i32 inreg %x) nounwind {
 ; X86-64-LABEL: qux32:
-; X86-64: movl %edi, %eax
-; X86-64: movzbl %ah, %eax
+; X86-64:       # %bb.0:
+; X86-64-NEXT:    movl %edi, %eax
+; X86-64-NEXT:    movzbl %ah, %eax
+; X86-64-NEXT:    retq
+;
+; WIN64-LABEL: qux32:
+; WIN64:       # %bb.0:
+; WIN64-NEXT:    movzbl %ch, %eax
+; WIN64-NEXT:    retq
+;
+; X86-32-LABEL: qux32:
+; X86-32:       # %bb.0:
+; X86-32-NEXT:    movzbl %ah, %eax
+; X86-32-NEXT:    retl
 
-; WIN64-LABEL:  qux32:
-; WIN64:  movzbl %ch, %eax
 
-; X86-32-LABEL: qux32:
-; X86-32: movzbl %ah, %eax
   %t0 = lshr i32 %x, 8
   %t1 = and i32 %t0, 255
   ret i32 %t1
@@ -94,15 +167,26 @@ define i32 @qux32(i32 inreg %x) nounwind {
 
 define i16 @qux16(i16 inreg %x) nounwind {
 ; X86-64-LABEL: qux16:
-; X86-64: movl %edi, %eax
-; X86-64: movzbl %ah, %eax
+; X86-64:       # %bb.0:
+; X86-64-NEXT:    movl %edi, %eax
+; X86-64-NEXT:    movzbl %ah, %eax
+; X86-64-NEXT:    # kill: def $ax killed $ax killed $eax
+; X86-64-NEXT:    retq
+;
+; WIN64-LABEL: qux16:
+; WIN64:       # %bb.0:
+; WIN64-NEXT:    movzwl %cx, %eax
+; WIN64-NEXT:    shrl $8, %eax
+; WIN64-NEXT:    # kill: def $ax killed $ax killed $eax
+; WIN64-NEXT:    retq
+;
+; X86-32-LABEL: qux16:
+; X86-32:       # %bb.0:
+; X86-32-NEXT:    movzbl %ah, %eax
+; X86-32-NEXT:    # kill: def $ax killed $ax killed $eax
+; X86-32-NEXT:    retl
 
-; WIN64-LABEL:  qux16:
-; WIN64:  movzwl  %cx, %eax
-; WIN64:  shrl    $8, %eax
 
-; X86-32-LABEL: qux16:
-; X86-32: movzbl %ah, %eax
   %t0 = lshr i16 %x, 8
   ret i16 %t0
 }
diff --git a/llvm/test/CodeGen/X86/h-registers-2.ll b/llvm/test/CodeGen/X86/h-registers-2.ll
index e7aeb4adf..3d2ad2792 100644
--- a/llvm/test/CodeGen/X86/h-registers-2.ll
+++ b/llvm/test/CodeGen/X86/h-registers-2.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=i686-- | FileCheck %s
 
 ; Use an h register, but don't omit the explicit shift for
@@ -5,11 +6,13 @@
 
 define i32 @foo(i8* %x, i32 %y) nounwind {
 ; CHECK-LABEL: foo:
-; CHECK-NOT: ret
-; CHECK: movzbl %{{[abcd]h}},
-; CHECK-NOT: ret
-; CHECK: shll $3,
-; CHECK: ret
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    movzbl %ah, %eax
+; CHECK-NEXT:    movb $77, (%ecx,%eax,8)
+; CHECK-NEXT:    shll $3, %eax
+; CHECK-NEXT:    retl
 
 	%t0 = lshr i32 %y, 8		; <i32> [#uses=1]
 	%t1 = and i32 %t0, 255		; <i32> [#uses=2]
diff --git a/llvm/test/CodeGen/X86/h-registers-3.ll b/llvm/test/CodeGen/X86/h-registers-3.ll
index 819f21625..564b1e8e1 100644
--- a/llvm/test/CodeGen/X86/h-registers-3.ll
+++ b/llvm/test/CodeGen/X86/h-registers-3.ll
@@ -1,35 +1,46 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=i686-unknown-linux-gnu   | FileCheck %s -check-prefix=X86
 ; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu | FileCheck %s -check-prefix=X64
 ; RUN: llc < %s -mtriple=x86_64-linux-gnux32      | FileCheck %s -check-prefix=X32
 
 define zeroext i8 @foo() nounwind ssp {
+; X86-LABEL: foo:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    calll bar
+; X86-NEXT:    movb %ah, %al
+; X86-NEXT:    addl $12, %esp
+; X86-NEXT:    retl
+;
+; X64-LABEL: foo:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    callq bar
+; X64-NEXT:    # kill: def $ax killed $ax def $eax
+; X64-NEXT:    shrl $8, %eax
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+;
+; X32-LABEL: foo:
+; X32:       # %bb.0: # %entry
+; X32-NEXT:    pushq %rax
+; X32-NEXT:    xorl %eax, %eax
+; X32-NEXT:    callq bar
+; X32-NEXT:    # kill: def $ax killed $ax def $eax
+; X32-NEXT:    shrl $8, %eax
+; X32-NEXT:    # kill: def $al killed $al killed $eax
+; X32-NEXT:    popq %rcx
+; X32-NEXT:    retq
 entry:
   %0 = tail call zeroext i16 (...) @bar() nounwind
   %1 = lshr i16 %0, 8
   %2 = trunc i16 %1 to i8
   ret i8 %2
 
-; X86-LABEL: foo
-; X86: calll
-; X86-NEXT: movb %ah, %al
-; X86-NEXT: addl $12, %esp
-; X86-NEXT: retl
 
-; X64-LABEL: foo
-; X64: callq
-; X64-NEXT: # kill
-; X64-NEXT: shrl $8, %eax
-; X64-NEXT: # kill
-; X64-NEXT: popq
-; X64-NEXT: retq
 
-; X32-LABEL: foo
-; X32: callq
-; X32-NEXT: # kill
-; X32-NEXT: shrl $8, %eax
-; X32-NEXT: # kill
-; X32-NEXT: popq
-; X32-NEXT: retq
 }
 
 declare zeroext i16 @bar(...)
diff --git a/llvm/test/CodeGen/X86/haddsub-3.ll b/llvm/test/CodeGen/X86/haddsub-3.ll
index 29b9a626d..f603ace20 100644
--- a/llvm/test/CodeGen/X86/haddsub-3.ll
+++ b/llvm/test/CodeGen/X86/haddsub-3.ll
@@ -1,8 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=SSE,SSE2
-; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+ssse3 | FileCheck %s --check-prefixes=SSE,SSSE3
-; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx | FileCheck %s --check-prefixes=AVX,AVX1
-; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=AVX,AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+sse2            | FileCheck %s --check-prefixes=SSE,SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+ssse3           | FileCheck %s --check-prefixes=SSE,SSSE3,SSSE3-SLOW
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+ssse3,fast-hops | FileCheck %s --check-prefixes=SSE,SSSE3,SSSE3-FAST
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx             | FileCheck %s --check-prefixes=AVX,AVX1,AVX1-SLOW
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx,fast-hops   | FileCheck %s --check-prefixes=AVX,AVX1,AVX1-FAST
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx2            | FileCheck %s --check-prefixes=AVX,AVX2
 
 define float @pr26491(<4 x float> %a0) {
 ; SSE2-LABEL: pr26491:
@@ -58,37 +60,68 @@ define <4 x double> @PR41414(i64 %x, <4 x double> %y) {
 ; SSE2-NEXT:    addpd %xmm2, %xmm1
 ; SSE2-NEXT:    retq
 ;
-; SSSE3-LABEL: PR41414:
-; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    movq %rdi, %xmm2
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],mem[0],xmm2[1],mem[1]
-; SSSE3-NEXT:    subpd {{.*}}(%rip), %xmm2
-; SSSE3-NEXT:    haddpd %xmm2, %xmm2
-; SSSE3-NEXT:    divpd %xmm2, %xmm1
-; SSSE3-NEXT:    divpd %xmm2, %xmm0
-; SSSE3-NEXT:    xorpd %xmm2, %xmm2
-; SSSE3-NEXT:    addpd %xmm2, %xmm0
-; SSSE3-NEXT:    addpd %xmm2, %xmm1
-; SSSE3-NEXT:    retq
+; SSSE3-SLOW-LABEL: PR41414:
+; SSSE3-SLOW:       # %bb.0:
+; SSSE3-SLOW-NEXT:    movq %rdi, %xmm2
+; SSSE3-SLOW-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],mem[0],xmm2[1],mem[1]
+; SSSE3-SLOW-NEXT:    subpd {{.*}}(%rip), %xmm2
+; SSSE3-SLOW-NEXT:    movapd %xmm2, %xmm3
+; SSSE3-SLOW-NEXT:    unpckhpd {{.*#+}} xmm3 = xmm3[1],xmm2[1]
+; SSSE3-SLOW-NEXT:    addpd %xmm2, %xmm3
+; SSSE3-SLOW-NEXT:    movddup {{.*#+}} xmm2 = xmm3[0,0]
+; SSSE3-SLOW-NEXT:    divpd %xmm2, %xmm1
+; SSSE3-SLOW-NEXT:    divpd %xmm2, %xmm0
+; SSSE3-SLOW-NEXT:    xorpd %xmm2, %xmm2
+; SSSE3-SLOW-NEXT:    addpd %xmm2, %xmm0
+; SSSE3-SLOW-NEXT:    addpd %xmm2, %xmm1
+; SSSE3-SLOW-NEXT:    retq
+;
+; SSSE3-FAST-LABEL: PR41414:
+; SSSE3-FAST:       # %bb.0:
+; SSSE3-FAST-NEXT:    movq %rdi, %xmm2
+; SSSE3-FAST-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],mem[0],xmm2[1],mem[1]
+; SSSE3-FAST-NEXT:    subpd {{.*}}(%rip), %xmm2
+; SSSE3-FAST-NEXT:    haddpd %xmm2, %xmm2
+; SSSE3-FAST-NEXT:    divpd %xmm2, %xmm1
+; SSSE3-FAST-NEXT:    divpd %xmm2, %xmm0
+; SSSE3-FAST-NEXT:    xorpd %xmm2, %xmm2
+; SSSE3-FAST-NEXT:    addpd %xmm2, %xmm0
+; SSSE3-FAST-NEXT:    addpd %xmm2, %xmm1
+; SSSE3-FAST-NEXT:    retq
+;
+; AVX1-SLOW-LABEL: PR41414:
+; AVX1-SLOW:       # %bb.0:
+; AVX1-SLOW-NEXT:    vmovq %rdi, %xmm1
+; AVX1-SLOW-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]
+; AVX1-SLOW-NEXT:    vsubpd {{.*}}(%rip), %xmm1, %xmm1
+; AVX1-SLOW-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm1[1,0]
+; AVX1-SLOW-NEXT:    vaddpd %xmm1, %xmm2, %xmm1
+; AVX1-SLOW-NEXT:    vmovddup {{.*#+}} xmm1 = xmm1[0,0]
+; AVX1-SLOW-NEXT:    vinsertf128 $1, %xmm1, %ymm1, %ymm1
+; AVX1-SLOW-NEXT:    vdivpd %ymm1, %ymm0, %ymm0
+; AVX1-SLOW-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; AVX1-SLOW-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
+; AVX1-SLOW-NEXT:    retq
 ;
-; AVX1-LABEL: PR41414:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovq %rdi, %xmm1
-; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]
-; AVX1-NEXT:    vsubpd {{.*}}(%rip), %xmm1, %xmm1
-; AVX1-NEXT:    vhaddpd %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm1, %ymm1
-; AVX1-NEXT:    vdivpd %ymm1, %ymm0, %ymm0
-; AVX1-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
-; AVX1-NEXT:    retq
+; AVX1-FAST-LABEL: PR41414:
+; AVX1-FAST:       # %bb.0:
+; AVX1-FAST-NEXT:    vmovq %rdi, %xmm1
+; AVX1-FAST-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]
+; AVX1-FAST-NEXT:    vsubpd {{.*}}(%rip), %xmm1, %xmm1
+; AVX1-FAST-NEXT:    vhaddpd %xmm1, %xmm1, %xmm1
+; AVX1-FAST-NEXT:    vinsertf128 $1, %xmm1, %ymm1, %ymm1
+; AVX1-FAST-NEXT:    vdivpd %ymm1, %ymm0, %ymm0
+; AVX1-FAST-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; AVX1-FAST-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
+; AVX1-FAST-NEXT:    retq
 ;
 ; AVX2-LABEL: PR41414:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vmovq %rdi, %xmm1
 ; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]
 ; AVX2-NEXT:    vsubpd {{.*}}(%rip), %xmm1, %xmm1
-; AVX2-NEXT:    vhaddpd %xmm1, %xmm1, %xmm1
+; AVX2-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm1[1,0]
+; AVX2-NEXT:    vaddsd %xmm1, %xmm2, %xmm1
 ; AVX2-NEXT:    vbroadcastsd %xmm1, %ymm1
 ; AVX2-NEXT:    vdivpd %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
diff --git a/llvm/test/CodeGen/X86/haddsub-broadcast.ll b/llvm/test/CodeGen/X86/haddsub-broadcast.ll
new file mode 100644
index 000000000..ec617bb2b
--- /dev/null
+++ b/llvm/test/CodeGen/X86/haddsub-broadcast.ll
@@ -0,0 +1,21 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- -mattr=avx2  | FileCheck %s
+
+; The broadcast node takes a vector operand as input and changes its length.
+
+define <4 x double> @PR43402(i64 %x) {
+; CHECK-LABEL: PR43402:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
+; CHECK-NEXT:    vsubpd {{\.LCPI.*}}, %xmm0, %xmm0
+; CHECK-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
+; CHECK-NEXT:    vaddsd %xmm0, %xmm1, %xmm0
+; CHECK-NEXT:    vbroadcastsd %xmm0, %ymm0
+; CHECK-NEXT:    retl
+  %conv = uitofp i64 %x to double
+  %t2 = insertelement <4 x double> undef, double %conv, i32 0
+  %t3 = shufflevector <4 x double> %t2, <4 x double> undef, <4 x i32> zeroinitializer
+  ret <4 x double> %t3
+}
+
diff --git a/llvm/test/CodeGen/X86/haddsub.ll b/llvm/test/CodeGen/X86/haddsub.ll
index a3490658d..872cc4bf9 100644
--- a/llvm/test/CodeGen/X86/haddsub.ll
+++ b/llvm/test/CodeGen/X86/haddsub.ll
@@ -1645,10 +1645,8 @@ define float @fadd_reduce_v8f32(float %a0, <8 x float> %a1) {
 ;
 ; SSE3-FAST-LABEL: fadd_reduce_v8f32:
 ; SSE3-FAST:       # %bb.0:
-; SSE3-FAST-NEXT:    addps %xmm2, %xmm1
-; SSE3-FAST-NEXT:    movaps %xmm1, %xmm2
-; SSE3-FAST-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm1[1]
-; SSE3-FAST-NEXT:    addps %xmm1, %xmm2
+; SSE3-FAST-NEXT:    haddps %xmm1, %xmm2
+; SSE3-FAST-NEXT:    haddps %xmm2, %xmm2
 ; SSE3-FAST-NEXT:    haddps %xmm2, %xmm2
 ; SSE3-FAST-NEXT:    addss %xmm2, %xmm0
 ; SSE3-FAST-NEXT:    retq
@@ -1668,9 +1666,8 @@ define float @fadd_reduce_v8f32(float %a0, <8 x float> %a1) {
 ; AVX-FAST-LABEL: fadd_reduce_v8f32:
 ; AVX-FAST:       # %bb.0:
 ; AVX-FAST-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX-FAST-NEXT:    vaddps %xmm2, %xmm1, %xmm1
-; AVX-FAST-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm1[1,0]
-; AVX-FAST-NEXT:    vaddps %xmm2, %xmm1, %xmm1
+; AVX-FAST-NEXT:    vhaddps %xmm1, %xmm2, %xmm1
+; AVX-FAST-NEXT:    vhaddps %xmm1, %xmm1, %xmm1
 ; AVX-FAST-NEXT:    vhaddps %xmm1, %xmm1, %xmm1
 ; AVX-FAST-NEXT:    vaddss %xmm1, %xmm0, %xmm0
 ; AVX-FAST-NEXT:    vzeroupper
@@ -1691,9 +1688,9 @@ define double @fadd_reduce_v4f64(double %a0, <4 x double> %a1) {
 ;
 ; SSE3-FAST-LABEL: fadd_reduce_v4f64:
 ; SSE3-FAST:       # %bb.0:
-; SSE3-FAST-NEXT:    addpd %xmm2, %xmm1
-; SSE3-FAST-NEXT:    haddpd %xmm1, %xmm1
-; SSE3-FAST-NEXT:    addsd %xmm1, %xmm0
+; SSE3-FAST-NEXT:    haddpd %xmm1, %xmm2
+; SSE3-FAST-NEXT:    haddpd %xmm2, %xmm2
+; SSE3-FAST-NEXT:    addsd %xmm2, %xmm0
 ; SSE3-FAST-NEXT:    retq
 ;
 ; AVX-SLOW-LABEL: fadd_reduce_v4f64:
@@ -1709,7 +1706,7 @@ define double @fadd_reduce_v4f64(double %a0, <4 x double> %a1) {
 ; AVX-FAST-LABEL: fadd_reduce_v4f64:
 ; AVX-FAST:       # %bb.0:
 ; AVX-FAST-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX-FAST-NEXT:    vaddpd %xmm2, %xmm1, %xmm1
+; AVX-FAST-NEXT:    vhaddpd %xmm1, %xmm2, %xmm1
 ; AVX-FAST-NEXT:    vhaddpd %xmm1, %xmm1, %xmm1
 ; AVX-FAST-NEXT:    vaddsd %xmm1, %xmm0, %xmm0
 ; AVX-FAST-NEXT:    vzeroupper
@@ -1985,3 +1982,227 @@ define float @hadd32_16_optsize(<16 x float> %x225) optsize {
   %x230 = extractelement <16 x float> %x229, i32 0
   ret float %x230
 }
+
+define float @hadd32_4_pgso(<4 x float> %x225) !prof !14 {
+; SSE3-LABEL: hadd32_4_pgso:
+; SSE3:       # %bb.0:
+; SSE3-NEXT:    movaps %xmm0, %xmm1
+; SSE3-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
+; SSE3-NEXT:    addps %xmm0, %xmm1
+; SSE3-NEXT:    haddps %xmm1, %xmm1
+; SSE3-NEXT:    movaps %xmm1, %xmm0
+; SSE3-NEXT:    retq
+;
+; AVX-LABEL: hadd32_4_pgso:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
+; AVX-NEXT:    vaddps %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vhaddps %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %x226 = shufflevector <4 x float> %x225, <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+  %x227 = fadd <4 x float> %x225, %x226
+  %x228 = shufflevector <4 x float> %x227, <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+  %x229 = fadd <4 x float> %x227, %x228
+  %x230 = extractelement <4 x float> %x229, i32 0
+  ret float %x230
+}
+
+define float @hadd32_8_pgso(<8 x float> %x225) !prof !14 {
+; SSE3-LABEL: hadd32_8_pgso:
+; SSE3:       # %bb.0:
+; SSE3-NEXT:    movaps %xmm0, %xmm1
+; SSE3-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
+; SSE3-NEXT:    addps %xmm0, %xmm1
+; SSE3-NEXT:    haddps %xmm1, %xmm1
+; SSE3-NEXT:    movaps %xmm1, %xmm0
+; SSE3-NEXT:    retq
+;
+; AVX-LABEL: hadd32_8_pgso:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
+; AVX-NEXT:    vaddps %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vhaddps %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vzeroupper
+; AVX-NEXT:    retq
+  %x226 = shufflevector <8 x float> %x225, <8 x float> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %x227 = fadd <8 x float> %x225, %x226
+  %x228 = shufflevector <8 x float> %x227, <8 x float> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %x229 = fadd <8 x float> %x227, %x228
+  %x230 = extractelement <8 x float> %x229, i32 0
+  ret float %x230
+}
+
+define float @hadd32_16_pgso(<16 x float> %x225) !prof !14 {
+; SSE3-LABEL: hadd32_16_pgso:
+; SSE3:       # %bb.0:
+; SSE3-NEXT:    movaps %xmm0, %xmm1
+; SSE3-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
+; SSE3-NEXT:    addps %xmm0, %xmm1
+; SSE3-NEXT:    haddps %xmm1, %xmm1
+; SSE3-NEXT:    movaps %xmm1, %xmm0
+; SSE3-NEXT:    retq
+;
+; AVX-LABEL: hadd32_16_pgso:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
+; AVX-NEXT:    vaddps %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vhaddps %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vzeroupper
+; AVX-NEXT:    retq
+  %x226 = shufflevector <16 x float> %x225, <16 x float> undef, <16 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %x227 = fadd <16 x float> %x225, %x226
+  %x228 = shufflevector <16 x float> %x227, <16 x float> undef, <16 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %x229 = fadd <16 x float> %x227, %x228
+  %x230 = extractelement <16 x float> %x229, i32 0
+  ret float %x230
+}
+
+define float @partial_reduction_fadd_v8f32(<8 x float> %x) {
+; SSE3-SLOW-LABEL: partial_reduction_fadd_v8f32:
+; SSE3-SLOW:       # %bb.0:
+; SSE3-SLOW-NEXT:    movaps %xmm0, %xmm1
+; SSE3-SLOW-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
+; SSE3-SLOW-NEXT:    addps %xmm0, %xmm1
+; SSE3-SLOW-NEXT:    movshdup {{.*#+}} xmm0 = xmm1[1,1,3,3]
+; SSE3-SLOW-NEXT:    addss %xmm0, %xmm1
+; SSE3-SLOW-NEXT:    movaps %xmm1, %xmm0
+; SSE3-SLOW-NEXT:    retq
+;
+; SSE3-FAST-LABEL: partial_reduction_fadd_v8f32:
+; SSE3-FAST:       # %bb.0:
+; SSE3-FAST-NEXT:    movaps %xmm0, %xmm1
+; SSE3-FAST-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
+; SSE3-FAST-NEXT:    addps %xmm0, %xmm1
+; SSE3-FAST-NEXT:    haddps %xmm1, %xmm1
+; SSE3-FAST-NEXT:    movaps %xmm1, %xmm0
+; SSE3-FAST-NEXT:    retq
+;
+; AVX-SLOW-LABEL: partial_reduction_fadd_v8f32:
+; AVX-SLOW:       # %bb.0:
+; AVX-SLOW-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
+; AVX-SLOW-NEXT:    vaddps %xmm1, %xmm0, %xmm0
+; AVX-SLOW-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX-SLOW-NEXT:    vaddss %xmm1, %xmm0, %xmm0
+; AVX-SLOW-NEXT:    vzeroupper
+; AVX-SLOW-NEXT:    retq
+;
+; AVX-FAST-LABEL: partial_reduction_fadd_v8f32:
+; AVX-FAST:       # %bb.0:
+; AVX-FAST-NEXT:    vhaddps %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    vhaddps %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    vzeroupper
+; AVX-FAST-NEXT:    retq
+  %x23 = shufflevector <8 x float> %x, <8 x float> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %x0213 = fadd <8 x float> %x, %x23
+  %x13 = shufflevector <8 x float> %x0213, <8 x float> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %x0123 = fadd nsz reassoc <8 x float> %x0213, %x13
+  %r = extractelement <8 x float> %x0123, i32 0
+  ret float %r
+}
+
+; Negative test - only the flags on the final math op in the
+; sequence determine whether we can transform to horizontal ops.
+
+define float @partial_reduction_fadd_v8f32_wrong_flags(<8 x float> %x) {
+; SSE3-SLOW-LABEL: partial_reduction_fadd_v8f32_wrong_flags:
+; SSE3-SLOW:       # %bb.0:
+; SSE3-SLOW-NEXT:    movaps %xmm0, %xmm1
+; SSE3-SLOW-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
+; SSE3-SLOW-NEXT:    addps %xmm0, %xmm1
+; SSE3-SLOW-NEXT:    movshdup {{.*#+}} xmm0 = xmm1[1,1,3,3]
+; SSE3-SLOW-NEXT:    addss %xmm0, %xmm1
+; SSE3-SLOW-NEXT:    movaps %xmm1, %xmm0
+; SSE3-SLOW-NEXT:    retq
+;
+; SSE3-FAST-LABEL: partial_reduction_fadd_v8f32_wrong_flags:
+; SSE3-FAST:       # %bb.0:
+; SSE3-FAST-NEXT:    movaps %xmm0, %xmm1
+; SSE3-FAST-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
+; SSE3-FAST-NEXT:    addps %xmm0, %xmm1
+; SSE3-FAST-NEXT:    haddps %xmm1, %xmm1
+; SSE3-FAST-NEXT:    movaps %xmm1, %xmm0
+; SSE3-FAST-NEXT:    retq
+;
+; AVX-SLOW-LABEL: partial_reduction_fadd_v8f32_wrong_flags:
+; AVX-SLOW:       # %bb.0:
+; AVX-SLOW-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
+; AVX-SLOW-NEXT:    vaddps %xmm1, %xmm0, %xmm0
+; AVX-SLOW-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX-SLOW-NEXT:    vaddss %xmm1, %xmm0, %xmm0
+; AVX-SLOW-NEXT:    vzeroupper
+; AVX-SLOW-NEXT:    retq
+;
+; AVX-FAST-LABEL: partial_reduction_fadd_v8f32_wrong_flags:
+; AVX-FAST:       # %bb.0:
+; AVX-FAST-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
+; AVX-FAST-NEXT:    vaddps %xmm1, %xmm0, %xmm0
+; AVX-FAST-NEXT:    vhaddps %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    vzeroupper
+; AVX-FAST-NEXT:    retq
+  %x23 = shufflevector <8 x float> %x, <8 x float> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %x0213 = fadd fast <8 x float> %x, %x23
+  %x13 = shufflevector <8 x float> %x0213, <8 x float> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %x0123 = fadd ninf nnan <8 x float> %x0213, %x13
+  %r = extractelement <8 x float> %x0123, i32 0
+  ret float %r
+}
+
+define float @partial_reduction_fadd_v16f32(<16 x float> %x) {
+; SSE3-SLOW-LABEL: partial_reduction_fadd_v16f32:
+; SSE3-SLOW:       # %bb.0:
+; SSE3-SLOW-NEXT:    movaps %xmm0, %xmm1
+; SSE3-SLOW-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
+; SSE3-SLOW-NEXT:    addps %xmm0, %xmm1
+; SSE3-SLOW-NEXT:    movshdup {{.*#+}} xmm0 = xmm1[1,1,3,3]
+; SSE3-SLOW-NEXT:    addss %xmm0, %xmm1
+; SSE3-SLOW-NEXT:    movaps %xmm1, %xmm0
+; SSE3-SLOW-NEXT:    retq
+;
+; SSE3-FAST-LABEL: partial_reduction_fadd_v16f32:
+; SSE3-FAST:       # %bb.0:
+; SSE3-FAST-NEXT:    movaps %xmm0, %xmm1
+; SSE3-FAST-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
+; SSE3-FAST-NEXT:    addps %xmm0, %xmm1
+; SSE3-FAST-NEXT:    haddps %xmm1, %xmm1
+; SSE3-FAST-NEXT:    movaps %xmm1, %xmm0
+; SSE3-FAST-NEXT:    retq
+;
+; AVX-SLOW-LABEL: partial_reduction_fadd_v16f32:
+; AVX-SLOW:       # %bb.0:
+; AVX-SLOW-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
+; AVX-SLOW-NEXT:    vaddps %xmm1, %xmm0, %xmm0
+; AVX-SLOW-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX-SLOW-NEXT:    vaddss %xmm1, %xmm0, %xmm0
+; AVX-SLOW-NEXT:    vzeroupper
+; AVX-SLOW-NEXT:    retq
+;
+; AVX-FAST-LABEL: partial_reduction_fadd_v16f32:
+; AVX-FAST:       # %bb.0:
+; AVX-FAST-NEXT:    vhaddps %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    vhaddps %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    vzeroupper
+; AVX-FAST-NEXT:    retq
+  %x23 = shufflevector <16 x float> %x, <16 x float> undef, <16 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %x0213 = fadd <16 x float> %x, %x23
+  %x13 = shufflevector <16 x float> %x0213, <16 x float> undef, <16 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %x0123 = fadd reassoc nsz <16 x float> %x0213, %x13
+  %r = extractelement <16 x float> %x0123, i32 0
+  ret float %r
+}
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 1, !"ProfileSummary", !1}
+!1 = !{!2, !3, !4, !5, !6, !7, !8, !9}
+!2 = !{!"ProfileFormat", !"InstrProf"}
+!3 = !{!"TotalCount", i64 10000}
+!4 = !{!"MaxCount", i64 10}
+!5 = !{!"MaxInternalCount", i64 1}
+!6 = !{!"MaxFunctionCount", i64 1000}
+!7 = !{!"NumCounts", i64 3}
+!8 = !{!"NumFunctions", i64 3}
+!9 = !{!"DetailedSummary", !10}
+!10 = !{!11, !12, !13}
+!11 = !{i32 10000, i64 100, i32 1}
+!12 = !{i32 999000, i64 100, i32 1}
+!13 = !{i32 999999, i64 1, i32 2}
+!14 = !{!"function_entry_count", i64 0}
diff --git a/llvm/test/CodeGen/X86/half.ll b/llvm/test/CodeGen/X86/half.ll
index b862bf7a7..a6c92ecd7 100644
--- a/llvm/test/CodeGen/X86/half.ll
+++ b/llvm/test/CodeGen/X86/half.ll
@@ -362,11 +362,10 @@ define void @test_uitofp_i64(i64 %a, half* %p) #0 {
 ; CHECK-I686-NEXT:    pushl %esi
 ; CHECK-I686-NEXT:    subl $24, %esp
 ; CHECK-I686-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; CHECK-I686-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; CHECK-I686-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; CHECK-I686-NEXT:    movlps %xmm0, {{[0-9]+}}(%esp)
-; CHECK-I686-NEXT:    xorl %eax, %eax
-; CHECK-I686-NEXT:    cmpl $0, {{[0-9]+}}(%esp)
-; CHECK-I686-NEXT:    setns %al
+; CHECK-I686-NEXT:    shrl $31, %eax
 ; CHECK-I686-NEXT:    fildll {{[0-9]+}}(%esp)
 ; CHECK-I686-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
 ; CHECK-I686-NEXT:    fstps (%esp)
diff --git a/llvm/test/CodeGen/X86/hidden-vis-2.ll b/llvm/test/CodeGen/X86/hidden-vis-2.ll
index 62e143d61..a29445f41 100644
--- a/llvm/test/CodeGen/X86/hidden-vis-2.ll
+++ b/llvm/test/CodeGen/X86/hidden-vis-2.ll
@@ -1,9 +1,19 @@
-; RUN: llc < %s -mtriple=i386-apple-darwin9   | grep mov | count 1
-; RUN: llc < %s -mtriple=x86_64-apple-darwin9 | not grep GOT
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i386-apple-darwin9   | FileCheck %s --check-prefix=i386-darwin9
+; RUN: llc < %s -mtriple=x86_64-apple-darwin9 | FileCheck %s --check-prefix=x86_64-darwin9
 
 @x = weak hidden global i32 0		; <i32*> [#uses=1]
 
 define i32 @t() nounwind readonly {
+; i386-darwin9-LABEL: t:
+; i386-darwin9:       ## %bb.0: ## %entry
+; i386-darwin9-NEXT:    movl _x, %eax
+; i386-darwin9-NEXT:    retl
+;
+; x86_64-darwin9-LABEL: t:
+; x86_64-darwin9:       ## %bb.0: ## %entry
+; x86_64-darwin9-NEXT:    movl {{.*}}(%rip), %eax
+; x86_64-darwin9-NEXT:    retq
 entry:
 	%0 = load i32, i32* @x, align 4		; <i32> [#uses=1]
 	ret i32 %0
diff --git a/llvm/test/CodeGen/X86/hoist-and-by-const-from-lshr-in-eqcmp-zero.ll b/llvm/test/CodeGen/X86/hoist-and-by-const-from-lshr-in-eqcmp-zero.ll
index 88c939f52..6663459f4 100644
--- a/llvm/test/CodeGen/X86/hoist-and-by-const-from-lshr-in-eqcmp-zero.ll
+++ b/llvm/test/CodeGen/X86/hoist-and-by-const-from-lshr-in-eqcmp-zero.ll
@@ -23,19 +23,18 @@ define i1 @scalar_i8_signbit_eq(i8 %x, i8 %y) nounwind {
 ; X86-LABEL: scalar_i8_signbit_eq:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X86-NEXT:    movb $-128, %al
-; X86-NEXT:    shrb %cl, %al
-; X86-NEXT:    testb %al, {{[0-9]+}}(%esp)
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    shlb %cl, %al
+; X86-NEXT:    testb $-128, %al
 ; X86-NEXT:    sete %al
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: scalar_i8_signbit_eq:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl %esi, %ecx
-; X64-NEXT:    movb $-128, %al
 ; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
-; X64-NEXT:    shrb %cl, %al
-; X64-NEXT:    testb %dil, %al
+; X64-NEXT:    shlb %cl, %dil
+; X64-NEXT:    testb $-128, %dil
 ; X64-NEXT:    sete %al
 ; X64-NEXT:    retq
   %t0 = lshr i8 128, %y
@@ -48,19 +47,18 @@ define i1 @scalar_i8_lowestbit_eq(i8 %x, i8 %y) nounwind {
 ; X86-LABEL: scalar_i8_lowestbit_eq:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X86-NEXT:    movb $1, %al
-; X86-NEXT:    shrb %cl, %al
-; X86-NEXT:    testb %al, {{[0-9]+}}(%esp)
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    shlb %cl, %al
+; X86-NEXT:    testb $1, %al
 ; X86-NEXT:    sete %al
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: scalar_i8_lowestbit_eq:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl %esi, %ecx
-; X64-NEXT:    movb $1, %al
 ; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
-; X64-NEXT:    shrb %cl, %al
-; X64-NEXT:    testb %dil, %al
+; X64-NEXT:    shlb %cl, %dil
+; X64-NEXT:    testb $1, %dil
 ; X64-NEXT:    sete %al
 ; X64-NEXT:    retq
   %t0 = lshr i8 1, %y
@@ -73,19 +71,18 @@ define i1 @scalar_i8_bitsinmiddle_eq(i8 %x, i8 %y) nounwind {
 ; X86-LABEL: scalar_i8_bitsinmiddle_eq:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X86-NEXT:    movb $24, %al
-; X86-NEXT:    shrb %cl, %al
-; X86-NEXT:    testb %al, {{[0-9]+}}(%esp)
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    shlb %cl, %al
+; X86-NEXT:    testb $24, %al
 ; X86-NEXT:    sete %al
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: scalar_i8_bitsinmiddle_eq:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl %esi, %ecx
-; X64-NEXT:    movb $24, %al
 ; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
-; X64-NEXT:    shrb %cl, %al
-; X64-NEXT:    testb %dil, %al
+; X64-NEXT:    shlb %cl, %dil
+; X64-NEXT:    testb $24, %dil
 ; X64-NEXT:    sete %al
 ; X64-NEXT:    retq
   %t0 = lshr i8 24, %y
@@ -100,36 +97,33 @@ define i1 @scalar_i16_signbit_eq(i16 %x, i16 %y) nounwind {
 ; X86-BMI1-LABEL: scalar_i16_signbit_eq:
 ; X86-BMI1:       # %bb.0:
 ; X86-BMI1-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X86-BMI1-NEXT:    movl $32768, %eax # imm = 0x8000
-; X86-BMI1-NEXT:    shrl %cl, %eax
-; X86-BMI1-NEXT:    testw %ax, {{[0-9]+}}(%esp)
+; X86-BMI1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-BMI1-NEXT:    shll %cl, %eax
+; X86-BMI1-NEXT:    testl $32768, %eax # imm = 0x8000
 ; X86-BMI1-NEXT:    sete %al
 ; X86-BMI1-NEXT:    retl
 ;
 ; X86-BMI2-LABEL: scalar_i16_signbit_eq:
 ; X86-BMI2:       # %bb.0:
 ; X86-BMI2-NEXT:    movb {{[0-9]+}}(%esp), %al
-; X86-BMI2-NEXT:    movl $32768, %ecx # imm = 0x8000
-; X86-BMI2-NEXT:    shrxl %eax, %ecx, %eax
-; X86-BMI2-NEXT:    testw %ax, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT:    shlxl %eax, {{[0-9]+}}(%esp), %eax
+; X86-BMI2-NEXT:    testl $32768, %eax # imm = 0x8000
 ; X86-BMI2-NEXT:    sete %al
 ; X86-BMI2-NEXT:    retl
 ;
 ; X64-BMI1-LABEL: scalar_i16_signbit_eq:
 ; X64-BMI1:       # %bb.0:
 ; X64-BMI1-NEXT:    movl %esi, %ecx
-; X64-BMI1-NEXT:    movl $32768, %eax # imm = 0x8000
 ; X64-BMI1-NEXT:    # kill: def $cl killed $cl killed $ecx
-; X64-BMI1-NEXT:    shrl %cl, %eax
-; X64-BMI1-NEXT:    testw %di, %ax
+; X64-BMI1-NEXT:    shll %cl, %edi
+; X64-BMI1-NEXT:    testl $32768, %edi # imm = 0x8000
 ; X64-BMI1-NEXT:    sete %al
 ; X64-BMI1-NEXT:    retq
 ;
 ; X64-BMI2-LABEL: scalar_i16_signbit_eq:
 ; X64-BMI2:       # %bb.0:
-; X64-BMI2-NEXT:    movl $32768, %eax # imm = 0x8000
-; X64-BMI2-NEXT:    shrxl %esi, %eax, %eax
-; X64-BMI2-NEXT:    testw %di, %ax
+; X64-BMI2-NEXT:    shlxl %esi, %edi, %eax
+; X64-BMI2-NEXT:    testl $32768, %eax # imm = 0x8000
 ; X64-BMI2-NEXT:    sete %al
 ; X64-BMI2-NEXT:    retq
   %t0 = lshr i16 32768, %y
@@ -142,36 +136,33 @@ define i1 @scalar_i16_lowestbit_eq(i16 %x, i16 %y) nounwind {
 ; X86-BMI1-LABEL: scalar_i16_lowestbit_eq:
 ; X86-BMI1:       # %bb.0:
 ; X86-BMI1-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X86-BMI1-NEXT:    movl $1, %eax
-; X86-BMI1-NEXT:    shrl %cl, %eax
-; X86-BMI1-NEXT:    testw %ax, {{[0-9]+}}(%esp)
+; X86-BMI1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-BMI1-NEXT:    shll %cl, %eax
+; X86-BMI1-NEXT:    testb $1, %al
 ; X86-BMI1-NEXT:    sete %al
 ; X86-BMI1-NEXT:    retl
 ;
 ; X86-BMI2-LABEL: scalar_i16_lowestbit_eq:
 ; X86-BMI2:       # %bb.0:
 ; X86-BMI2-NEXT:    movb {{[0-9]+}}(%esp), %al
-; X86-BMI2-NEXT:    movl $1, %ecx
-; X86-BMI2-NEXT:    shrxl %eax, %ecx, %eax
-; X86-BMI2-NEXT:    testw %ax, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT:    shlxl %eax, {{[0-9]+}}(%esp), %eax
+; X86-BMI2-NEXT:    testb $1, %al
 ; X86-BMI2-NEXT:    sete %al
 ; X86-BMI2-NEXT:    retl
 ;
 ; X64-BMI1-LABEL: scalar_i16_lowestbit_eq:
 ; X64-BMI1:       # %bb.0:
 ; X64-BMI1-NEXT:    movl %esi, %ecx
-; X64-BMI1-NEXT:    movl $1, %eax
 ; X64-BMI1-NEXT:    # kill: def $cl killed $cl killed $ecx
-; X64-BMI1-NEXT:    shrl %cl, %eax
-; X64-BMI1-NEXT:    testw %di, %ax
+; X64-BMI1-NEXT:    shll %cl, %edi
+; X64-BMI1-NEXT:    testb $1, %dil
 ; X64-BMI1-NEXT:    sete %al
 ; X64-BMI1-NEXT:    retq
 ;
 ; X64-BMI2-LABEL: scalar_i16_lowestbit_eq:
 ; X64-BMI2:       # %bb.0:
-; X64-BMI2-NEXT:    movl $1, %eax
-; X64-BMI2-NEXT:    shrxl %esi, %eax, %eax
-; X64-BMI2-NEXT:    testw %di, %ax
+; X64-BMI2-NEXT:    shlxl %esi, %edi, %eax
+; X64-BMI2-NEXT:    testb $1, %al
 ; X64-BMI2-NEXT:    sete %al
 ; X64-BMI2-NEXT:    retq
   %t0 = lshr i16 1, %y
@@ -184,36 +175,33 @@ define i1 @scalar_i16_bitsinmiddle_eq(i16 %x, i16 %y) nounwind {
 ; X86-BMI1-LABEL: scalar_i16_bitsinmiddle_eq:
 ; X86-BMI1:       # %bb.0:
 ; X86-BMI1-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X86-BMI1-NEXT:    movl $4080, %eax # imm = 0xFF0
-; X86-BMI1-NEXT:    shrl %cl, %eax
-; X86-BMI1-NEXT:    testw %ax, {{[0-9]+}}(%esp)
+; X86-BMI1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-BMI1-NEXT:    shll %cl, %eax
+; X86-BMI1-NEXT:    testl $4080, %eax # imm = 0xFF0
 ; X86-BMI1-NEXT:    sete %al
 ; X86-BMI1-NEXT:    retl
 ;
 ; X86-BMI2-LABEL: scalar_i16_bitsinmiddle_eq:
 ; X86-BMI2:       # %bb.0:
 ; X86-BMI2-NEXT:    movb {{[0-9]+}}(%esp), %al
-; X86-BMI2-NEXT:    movl $4080, %ecx # imm = 0xFF0
-; X86-BMI2-NEXT:    shrxl %eax, %ecx, %eax
-; X86-BMI2-NEXT:    testw %ax, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT:    shlxl %eax, {{[0-9]+}}(%esp), %eax
+; X86-BMI2-NEXT:    testl $4080, %eax # imm = 0xFF0
 ; X86-BMI2-NEXT:    sete %al
 ; X86-BMI2-NEXT:    retl
 ;
 ; X64-BMI1-LABEL: scalar_i16_bitsinmiddle_eq:
 ; X64-BMI1:       # %bb.0:
 ; X64-BMI1-NEXT:    movl %esi, %ecx
-; X64-BMI1-NEXT:    movl $4080, %eax # imm = 0xFF0
 ; X64-BMI1-NEXT:    # kill: def $cl killed $cl killed $ecx
-; X64-BMI1-NEXT:    shrl %cl, %eax
-; X64-BMI1-NEXT:    testw %di, %ax
+; X64-BMI1-NEXT:    shll %cl, %edi
+; X64-BMI1-NEXT:    testl $4080, %edi # imm = 0xFF0
 ; X64-BMI1-NEXT:    sete %al
 ; X64-BMI1-NEXT:    retq
 ;
 ; X64-BMI2-LABEL: scalar_i16_bitsinmiddle_eq:
 ; X64-BMI2:       # %bb.0:
-; X64-BMI2-NEXT:    movl $4080, %eax # imm = 0xFF0
-; X64-BMI2-NEXT:    shrxl %esi, %eax, %eax
-; X64-BMI2-NEXT:    testw %di, %ax
+; X64-BMI2-NEXT:    shlxl %esi, %edi, %eax
+; X64-BMI2-NEXT:    testl $4080, %eax # imm = 0xFF0
 ; X64-BMI2-NEXT:    sete %al
 ; X64-BMI2-NEXT:    retq
   %t0 = lshr i16 4080, %y
@@ -228,36 +216,33 @@ define i1 @scalar_i32_signbit_eq(i32 %x, i32 %y) nounwind {
 ; X86-BMI1-LABEL: scalar_i32_signbit_eq:
 ; X86-BMI1:       # %bb.0:
 ; X86-BMI1-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X86-BMI1-NEXT:    movl $-2147483648, %eax # imm = 0x80000000
-; X86-BMI1-NEXT:    shrl %cl, %eax
-; X86-BMI1-NEXT:    testl %eax, {{[0-9]+}}(%esp)
+; X86-BMI1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-BMI1-NEXT:    shll %cl, %eax
+; X86-BMI1-NEXT:    testl $-2147483648, %eax # imm = 0x80000000
 ; X86-BMI1-NEXT:    sete %al
 ; X86-BMI1-NEXT:    retl
 ;
 ; X86-BMI2-LABEL: scalar_i32_signbit_eq:
 ; X86-BMI2:       # %bb.0:
 ; X86-BMI2-NEXT:    movb {{[0-9]+}}(%esp), %al
-; X86-BMI2-NEXT:    movl $-2147483648, %ecx # imm = 0x80000000
-; X86-BMI2-NEXT:    shrxl %eax, %ecx, %eax
-; X86-BMI2-NEXT:    testl %eax, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT:    shlxl %eax, {{[0-9]+}}(%esp), %eax
+; X86-BMI2-NEXT:    testl $-2147483648, %eax # imm = 0x80000000
 ; X86-BMI2-NEXT:    sete %al
 ; X86-BMI2-NEXT:    retl
 ;
 ; X64-BMI1-LABEL: scalar_i32_signbit_eq:
 ; X64-BMI1:       # %bb.0:
 ; X64-BMI1-NEXT:    movl %esi, %ecx
-; X64-BMI1-NEXT:    movl $-2147483648, %eax # imm = 0x80000000
 ; X64-BMI1-NEXT:    # kill: def $cl killed $cl killed $ecx
-; X64-BMI1-NEXT:    shrl %cl, %eax
-; X64-BMI1-NEXT:    testl %edi, %eax
+; X64-BMI1-NEXT:    shll %cl, %edi
+; X64-BMI1-NEXT:    testl $-2147483648, %edi # imm = 0x80000000
 ; X64-BMI1-NEXT:    sete %al
 ; X64-BMI1-NEXT:    retq
 ;
 ; X64-BMI2-LABEL: scalar_i32_signbit_eq:
 ; X64-BMI2:       # %bb.0:
-; X64-BMI2-NEXT:    movl $-2147483648, %eax # imm = 0x80000000
-; X64-BMI2-NEXT:    shrxl %esi, %eax, %eax
-; X64-BMI2-NEXT:    testl %edi, %eax
+; X64-BMI2-NEXT:    shlxl %esi, %edi, %eax
+; X64-BMI2-NEXT:    testl $-2147483648, %eax # imm = 0x80000000
 ; X64-BMI2-NEXT:    sete %al
 ; X64-BMI2-NEXT:    retq
   %t0 = lshr i32 2147483648, %y
@@ -270,36 +255,33 @@ define i1 @scalar_i32_lowestbit_eq(i32 %x, i32 %y) nounwind {
 ; X86-BMI1-LABEL: scalar_i32_lowestbit_eq:
 ; X86-BMI1:       # %bb.0:
 ; X86-BMI1-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X86-BMI1-NEXT:    movl $1, %eax
-; X86-BMI1-NEXT:    shrl %cl, %eax
-; X86-BMI1-NEXT:    testl %eax, {{[0-9]+}}(%esp)
+; X86-BMI1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-BMI1-NEXT:    shll %cl, %eax
+; X86-BMI1-NEXT:    testb $1, %al
 ; X86-BMI1-NEXT:    sete %al
 ; X86-BMI1-NEXT:    retl
 ;
 ; X86-BMI2-LABEL: scalar_i32_lowestbit_eq:
 ; X86-BMI2:       # %bb.0:
 ; X86-BMI2-NEXT:    movb {{[0-9]+}}(%esp), %al
-; X86-BMI2-NEXT:    movl $1, %ecx
-; X86-BMI2-NEXT:    shrxl %eax, %ecx, %eax
-; X86-BMI2-NEXT:    testl %eax, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT:    shlxl %eax, {{[0-9]+}}(%esp), %eax
+; X86-BMI2-NEXT:    testb $1, %al
 ; X86-BMI2-NEXT:    sete %al
 ; X86-BMI2-NEXT:    retl
 ;
 ; X64-BMI1-LABEL: scalar_i32_lowestbit_eq:
 ; X64-BMI1:       # %bb.0:
 ; X64-BMI1-NEXT:    movl %esi, %ecx
-; X64-BMI1-NEXT:    movl $1, %eax
 ; X64-BMI1-NEXT:    # kill: def $cl killed $cl killed $ecx
-; X64-BMI1-NEXT:    shrl %cl, %eax
-; X64-BMI1-NEXT:    testl %edi, %eax
+; X64-BMI1-NEXT:    shll %cl, %edi
+; X64-BMI1-NEXT:    testb $1, %dil
 ; X64-BMI1-NEXT:    sete %al
 ; X64-BMI1-NEXT:    retq
 ;
 ; X64-BMI2-LABEL: scalar_i32_lowestbit_eq:
 ; X64-BMI2:       # %bb.0:
-; X64-BMI2-NEXT:    movl $1, %eax
-; X64-BMI2-NEXT:    shrxl %esi, %eax, %eax
-; X64-BMI2-NEXT:    testl %edi, %eax
+; X64-BMI2-NEXT:    shlxl %esi, %edi, %eax
+; X64-BMI2-NEXT:    testb $1, %al
 ; X64-BMI2-NEXT:    sete %al
 ; X64-BMI2-NEXT:    retq
   %t0 = lshr i32 1, %y
@@ -312,36 +294,33 @@ define i1 @scalar_i32_bitsinmiddle_eq(i32 %x, i32 %y) nounwind {
 ; X86-BMI1-LABEL: scalar_i32_bitsinmiddle_eq:
 ; X86-BMI1:       # %bb.0:
 ; X86-BMI1-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X86-BMI1-NEXT:    movl $16776960, %eax # imm = 0xFFFF00
-; X86-BMI1-NEXT:    shrl %cl, %eax
-; X86-BMI1-NEXT:    testl %eax, {{[0-9]+}}(%esp)
+; X86-BMI1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-BMI1-NEXT:    shll %cl, %eax
+; X86-BMI1-NEXT:    testl $16776960, %eax # imm = 0xFFFF00
 ; X86-BMI1-NEXT:    sete %al
 ; X86-BMI1-NEXT:    retl
 ;
 ; X86-BMI2-LABEL: scalar_i32_bitsinmiddle_eq:
 ; X86-BMI2:       # %bb.0:
 ; X86-BMI2-NEXT:    movb {{[0-9]+}}(%esp), %al
-; X86-BMI2-NEXT:    movl $16776960, %ecx # imm = 0xFFFF00
-; X86-BMI2-NEXT:    shrxl %eax, %ecx, %eax
-; X86-BMI2-NEXT:    testl %eax, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT:    shlxl %eax, {{[0-9]+}}(%esp), %eax
+; X86-BMI2-NEXT:    testl $16776960, %eax # imm = 0xFFFF00
 ; X86-BMI2-NEXT:    sete %al
 ; X86-BMI2-NEXT:    retl
 ;
 ; X64-BMI1-LABEL: scalar_i32_bitsinmiddle_eq:
 ; X64-BMI1:       # %bb.0:
 ; X64-BMI1-NEXT:    movl %esi, %ecx
-; X64-BMI1-NEXT:    movl $16776960, %eax # imm = 0xFFFF00
 ; X64-BMI1-NEXT:    # kill: def $cl killed $cl killed $ecx
-; X64-BMI1-NEXT:    shrl %cl, %eax
-; X64-BMI1-NEXT:    testl %edi, %eax
+; X64-BMI1-NEXT:    shll %cl, %edi
+; X64-BMI1-NEXT:    testl $16776960, %edi # imm = 0xFFFF00
 ; X64-BMI1-NEXT:    sete %al
 ; X64-BMI1-NEXT:    retq
 ;
 ; X64-BMI2-LABEL: scalar_i32_bitsinmiddle_eq:
 ; X64-BMI2:       # %bb.0:
-; X64-BMI2-NEXT:    movl $16776960, %eax # imm = 0xFFFF00
-; X64-BMI2-NEXT:    shrxl %esi, %eax, %eax
-; X64-BMI2-NEXT:    testl %edi, %eax
+; X64-BMI2-NEXT:    shlxl %esi, %edi, %eax
+; X64-BMI2-NEXT:    testl $16776960, %eax # imm = 0xFFFF00
 ; X64-BMI2-NEXT:    sete %al
 ; X64-BMI2-NEXT:    retq
   %t0 = lshr i32 16776960, %y
@@ -357,55 +336,44 @@ define i1 @scalar_i64_signbit_eq(i64 %x, i64 %y) nounwind {
 ; X86-BMI1:       # %bb.0:
 ; X86-BMI1-NEXT:    pushl %esi
 ; X86-BMI1-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X86-BMI1-NEXT:    movl $-2147483648, %eax # imm = 0x80000000
-; X86-BMI1-NEXT:    xorl %edx, %edx
-; X86-BMI1-NEXT:    xorl %esi, %esi
-; X86-BMI1-NEXT:    shrdl %cl, %eax, %esi
-; X86-BMI1-NEXT:    shrl %cl, %eax
+; X86-BMI1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-BMI1-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-BMI1-NEXT:    movl %eax, %esi
+; X86-BMI1-NEXT:    shll %cl, %esi
+; X86-BMI1-NEXT:    shldl %cl, %eax, %edx
 ; X86-BMI1-NEXT:    testb $32, %cl
-; X86-BMI1-NEXT:    cmovnel %eax, %esi
-; X86-BMI1-NEXT:    cmovnel %edx, %eax
-; X86-BMI1-NEXT:    andl {{[0-9]+}}(%esp), %esi
-; X86-BMI1-NEXT:    andl {{[0-9]+}}(%esp), %eax
-; X86-BMI1-NEXT:    orl %esi, %eax
+; X86-BMI1-NEXT:    cmovnel %esi, %edx
+; X86-BMI1-NEXT:    testl $-2147483648, %edx # imm = 0x80000000
 ; X86-BMI1-NEXT:    sete %al
 ; X86-BMI1-NEXT:    popl %esi
 ; X86-BMI1-NEXT:    retl
 ;
 ; X86-BMI2-LABEL: scalar_i64_signbit_eq:
 ; X86-BMI2:       # %bb.0:
-; X86-BMI2-NEXT:    pushl %esi
 ; X86-BMI2-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X86-BMI2-NEXT:    movl $-2147483648, %eax # imm = 0x80000000
-; X86-BMI2-NEXT:    xorl %edx, %edx
-; X86-BMI2-NEXT:    xorl %esi, %esi
-; X86-BMI2-NEXT:    shrdl %cl, %eax, %esi
-; X86-BMI2-NEXT:    shrxl %ecx, %eax, %eax
+; X86-BMI2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-BMI2-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-BMI2-NEXT:    shldl %cl, %eax, %edx
+; X86-BMI2-NEXT:    shlxl %ecx, %eax, %eax
 ; X86-BMI2-NEXT:    testb $32, %cl
-; X86-BMI2-NEXT:    cmovnel %eax, %esi
-; X86-BMI2-NEXT:    cmovnel %edx, %eax
-; X86-BMI2-NEXT:    andl {{[0-9]+}}(%esp), %esi
-; X86-BMI2-NEXT:    andl {{[0-9]+}}(%esp), %eax
-; X86-BMI2-NEXT:    orl %esi, %eax
+; X86-BMI2-NEXT:    cmovel %edx, %eax
+; X86-BMI2-NEXT:    testl $-2147483648, %eax # imm = 0x80000000
 ; X86-BMI2-NEXT:    sete %al
-; X86-BMI2-NEXT:    popl %esi
 ; X86-BMI2-NEXT:    retl
 ;
 ; X64-BMI1-LABEL: scalar_i64_signbit_eq:
 ; X64-BMI1:       # %bb.0:
 ; X64-BMI1-NEXT:    movq %rsi, %rcx
-; X64-BMI1-NEXT:    movabsq $-9223372036854775808, %rax # imm = 0x8000000000000000
 ; X64-BMI1-NEXT:    # kill: def $cl killed $cl killed $rcx
-; X64-BMI1-NEXT:    shrq %cl, %rax
-; X64-BMI1-NEXT:    testq %rdi, %rax
+; X64-BMI1-NEXT:    shlq %cl, %rdi
+; X64-BMI1-NEXT:    shrq $63, %rdi
 ; X64-BMI1-NEXT:    sete %al
 ; X64-BMI1-NEXT:    retq
 ;
 ; X64-BMI2-LABEL: scalar_i64_signbit_eq:
 ; X64-BMI2:       # %bb.0:
-; X64-BMI2-NEXT:    movabsq $-9223372036854775808, %rax # imm = 0x8000000000000000
-; X64-BMI2-NEXT:    shrxq %rsi, %rax, %rax
-; X64-BMI2-NEXT:    testq %rdi, %rax
+; X64-BMI2-NEXT:    shlxq %rsi, %rdi, %rax
+; X64-BMI2-NEXT:    shrq $63, %rax
 ; X64-BMI2-NEXT:    sete %al
 ; X64-BMI2-NEXT:    retq
   %t0 = lshr i64 9223372036854775808, %y
@@ -415,34 +383,42 @@ define i1 @scalar_i64_signbit_eq(i64 %x, i64 %y) nounwind {
 }
 
 define i1 @scalar_i64_lowestbit_eq(i64 %x, i64 %y) nounwind {
-; X86-LABEL: scalar_i64_lowestbit_eq:
-; X86:       # %bb.0:
-; X86-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X86-NEXT:    xorl %eax, %eax
-; X86-NEXT:    movl $1, %edx
-; X86-NEXT:    shrdl %cl, %eax, %edx
-; X86-NEXT:    testb $32, %cl
-; X86-NEXT:    cmovnel %eax, %edx
-; X86-NEXT:    andl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    orl $0, %edx
-; X86-NEXT:    sete %al
-; X86-NEXT:    retl
+; X86-BMI1-LABEL: scalar_i64_lowestbit_eq:
+; X86-BMI1:       # %bb.0:
+; X86-BMI1-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-BMI1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-BMI1-NEXT:    shll %cl, %eax
+; X86-BMI1-NEXT:    xorl %edx, %edx
+; X86-BMI1-NEXT:    testb $32, %cl
+; X86-BMI1-NEXT:    cmovel %eax, %edx
+; X86-BMI1-NEXT:    testb $1, %dl
+; X86-BMI1-NEXT:    sete %al
+; X86-BMI1-NEXT:    retl
+;
+; X86-BMI2-LABEL: scalar_i64_lowestbit_eq:
+; X86-BMI2:       # %bb.0:
+; X86-BMI2-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-BMI2-NEXT:    shlxl %eax, {{[0-9]+}}(%esp), %ecx
+; X86-BMI2-NEXT:    xorl %edx, %edx
+; X86-BMI2-NEXT:    testb $32, %al
+; X86-BMI2-NEXT:    cmovel %ecx, %edx
+; X86-BMI2-NEXT:    testb $1, %dl
+; X86-BMI2-NEXT:    sete %al
+; X86-BMI2-NEXT:    retl
 ;
 ; X64-BMI1-LABEL: scalar_i64_lowestbit_eq:
 ; X64-BMI1:       # %bb.0:
 ; X64-BMI1-NEXT:    movq %rsi, %rcx
-; X64-BMI1-NEXT:    movl $1, %eax
 ; X64-BMI1-NEXT:    # kill: def $cl killed $cl killed $rcx
-; X64-BMI1-NEXT:    shrq %cl, %rax
-; X64-BMI1-NEXT:    testq %rdi, %rax
+; X64-BMI1-NEXT:    shlq %cl, %rdi
+; X64-BMI1-NEXT:    testb $1, %dil
 ; X64-BMI1-NEXT:    sete %al
 ; X64-BMI1-NEXT:    retq
 ;
 ; X64-BMI2-LABEL: scalar_i64_lowestbit_eq:
 ; X64-BMI2:       # %bb.0:
-; X64-BMI2-NEXT:    movl $1, %eax
-; X64-BMI2-NEXT:    shrxq %rsi, %rax, %rax
-; X64-BMI2-NEXT:    testq %rdi, %rax
+; X64-BMI2-NEXT:    shlxq %rsi, %rdi, %rax
+; X64-BMI2-NEXT:    testb $1, %al
 ; X64-BMI2-NEXT:    sete %al
 ; X64-BMI2-NEXT:    retq
   %t0 = lshr i64 1, %y
@@ -456,17 +432,18 @@ define i1 @scalar_i64_bitsinmiddle_eq(i64 %x, i64 %y) nounwind {
 ; X86-BMI1:       # %bb.0:
 ; X86-BMI1-NEXT:    pushl %esi
 ; X86-BMI1-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X86-BMI1-NEXT:    movl $65535, %eax # imm = 0xFFFF
-; X86-BMI1-NEXT:    movl $-65536, %edx # imm = 0xFFFF0000
-; X86-BMI1-NEXT:    shrdl %cl, %eax, %edx
-; X86-BMI1-NEXT:    shrl %cl, %eax
-; X86-BMI1-NEXT:    xorl %esi, %esi
+; X86-BMI1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-BMI1-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-BMI1-NEXT:    movl %eax, %esi
+; X86-BMI1-NEXT:    shll %cl, %esi
+; X86-BMI1-NEXT:    shldl %cl, %eax, %edx
+; X86-BMI1-NEXT:    xorl %eax, %eax
 ; X86-BMI1-NEXT:    testb $32, %cl
-; X86-BMI1-NEXT:    cmovnel %eax, %edx
-; X86-BMI1-NEXT:    cmovel %eax, %esi
-; X86-BMI1-NEXT:    andl {{[0-9]+}}(%esp), %edx
-; X86-BMI1-NEXT:    andl {{[0-9]+}}(%esp), %esi
-; X86-BMI1-NEXT:    orl %edx, %esi
+; X86-BMI1-NEXT:    cmovnel %esi, %edx
+; X86-BMI1-NEXT:    movzwl %dx, %ecx
+; X86-BMI1-NEXT:    cmovel %esi, %eax
+; X86-BMI1-NEXT:    andl $-65536, %eax # imm = 0xFFFF0000
+; X86-BMI1-NEXT:    orl %ecx, %eax
 ; X86-BMI1-NEXT:    sete %al
 ; X86-BMI1-NEXT:    popl %esi
 ; X86-BMI1-NEXT:    retl
@@ -475,17 +452,17 @@ define i1 @scalar_i64_bitsinmiddle_eq(i64 %x, i64 %y) nounwind {
 ; X86-BMI2:       # %bb.0:
 ; X86-BMI2-NEXT:    pushl %esi
 ; X86-BMI2-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X86-BMI2-NEXT:    movl $65535, %eax # imm = 0xFFFF
-; X86-BMI2-NEXT:    movl $-65536, %edx # imm = 0xFFFF0000
-; X86-BMI2-NEXT:    shrdl %cl, %eax, %edx
-; X86-BMI2-NEXT:    shrxl %ecx, %eax, %eax
+; X86-BMI2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-BMI2-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-BMI2-NEXT:    shldl %cl, %eax, %edx
+; X86-BMI2-NEXT:    shlxl %ecx, %eax, %eax
 ; X86-BMI2-NEXT:    xorl %esi, %esi
 ; X86-BMI2-NEXT:    testb $32, %cl
 ; X86-BMI2-NEXT:    cmovnel %eax, %edx
+; X86-BMI2-NEXT:    movzwl %dx, %ecx
 ; X86-BMI2-NEXT:    cmovel %eax, %esi
-; X86-BMI2-NEXT:    andl {{[0-9]+}}(%esp), %edx
-; X86-BMI2-NEXT:    andl {{[0-9]+}}(%esp), %esi
-; X86-BMI2-NEXT:    orl %edx, %esi
+; X86-BMI2-NEXT:    andl $-65536, %esi # imm = 0xFFFF0000
+; X86-BMI2-NEXT:    orl %ecx, %esi
 ; X86-BMI2-NEXT:    sete %al
 ; X86-BMI2-NEXT:    popl %esi
 ; X86-BMI2-NEXT:    retl
@@ -493,18 +470,18 @@ define i1 @scalar_i64_bitsinmiddle_eq(i64 %x, i64 %y) nounwind {
 ; X64-BMI1-LABEL: scalar_i64_bitsinmiddle_eq:
 ; X64-BMI1:       # %bb.0:
 ; X64-BMI1-NEXT:    movq %rsi, %rcx
-; X64-BMI1-NEXT:    movabsq $281474976645120, %rax # imm = 0xFFFFFFFF0000
 ; X64-BMI1-NEXT:    # kill: def $cl killed $cl killed $rcx
-; X64-BMI1-NEXT:    shrq %cl, %rax
-; X64-BMI1-NEXT:    testq %rdi, %rax
+; X64-BMI1-NEXT:    shlq %cl, %rdi
+; X64-BMI1-NEXT:    movabsq $281474976645120, %rax # imm = 0xFFFFFFFF0000
+; X64-BMI1-NEXT:    testq %rax, %rdi
 ; X64-BMI1-NEXT:    sete %al
 ; X64-BMI1-NEXT:    retq
 ;
 ; X64-BMI2-LABEL: scalar_i64_bitsinmiddle_eq:
 ; X64-BMI2:       # %bb.0:
-; X64-BMI2-NEXT:    movabsq $281474976645120, %rax # imm = 0xFFFFFFFF0000
-; X64-BMI2-NEXT:    shrxq %rsi, %rax, %rax
-; X64-BMI2-NEXT:    testq %rdi, %rax
+; X64-BMI2-NEXT:    shlxq %rsi, %rdi, %rax
+; X64-BMI2-NEXT:    movabsq $281474976645120, %rcx # imm = 0xFFFFFFFF0000
+; X64-BMI2-NEXT:    testq %rcx, %rax
 ; X64-BMI2-NEXT:    sete %al
 ; X64-BMI2-NEXT:    retq
   %t0 = lshr i64 281474976645120, %y
@@ -518,37 +495,48 @@ define i1 @scalar_i64_bitsinmiddle_eq(i64 %x, i64 %y) nounwind {
 ;------------------------------------------------------------------------------;
 
 define <4 x i1> @vec_4xi32_splat_eq(<4 x i32> %x, <4 x i32> %y) nounwind {
-; SSE2-LABEL: vec_4xi32_splat_eq:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm1[2,3,3,3,4,5,6,7]
-; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [1,1,1,1]
-; SSE2-NEXT:    movdqa %xmm3, %xmm4
-; SSE2-NEXT:    psrld %xmm2, %xmm4
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm1[0,1,1,1,4,5,6,7]
-; SSE2-NEXT:    movdqa %xmm3, %xmm5
-; SSE2-NEXT:    psrld %xmm2, %xmm5
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm4[0]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm1[2,3,3,3,4,5,6,7]
-; SSE2-NEXT:    movdqa %xmm3, %xmm4
-; SSE2-NEXT:    psrld %xmm2, %xmm4
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,1,1,1,4,5,6,7]
-; SSE2-NEXT:    psrld %xmm1, %xmm3
-; SSE2-NEXT:    punpckhqdq {{.*#+}} xmm3 = xmm3[1],xmm4[1]
-; SSE2-NEXT:    shufps {{.*#+}} xmm5 = xmm5[0,3],xmm3[0,3]
-; SSE2-NEXT:    andps %xmm5, %xmm0
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
-; SSE2-NEXT:    ret{{[l|q]}}
+; X86-SSE2-LABEL: vec_4xi32_splat_eq:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pxor %xmm2, %xmm2
+; X86-SSE2-NEXT:    pslld $23, %xmm1
+; X86-SSE2-NEXT:    paddd {{\.LCPI.*}}, %xmm1
+; X86-SSE2-NEXT:    cvttps2dq %xmm1, %xmm1
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; X86-SSE2-NEXT:    pmuludq %xmm1, %xmm0
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; X86-SSE2-NEXT:    pmuludq %xmm3, %xmm1
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; X86-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pcmpeqd %xmm2, %xmm0
+; X86-SSE2-NEXT:    retl
 ;
 ; AVX2-LABEL: vec_4xi32_splat_eq:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [1,1,1,1]
-; AVX2-NEXT:    vpsrlvd %xmm1, %xmm2, %xmm1
-; AVX2-NEXT:    vpand %xmm0, %xmm1, %xmm0
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX2-NEXT:    vpsllvd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpand %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm0, %xmm0
 ; AVX2-NEXT:    ret{{[l|q]}}
+;
+; X64-SSE2-LABEL: vec_4xi32_splat_eq:
+; X64-SSE2:       # %bb.0:
+; X64-SSE2-NEXT:    pxor %xmm2, %xmm2
+; X64-SSE2-NEXT:    pslld $23, %xmm1
+; X64-SSE2-NEXT:    paddd {{.*}}(%rip), %xmm1
+; X64-SSE2-NEXT:    cvttps2dq %xmm1, %xmm1
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; X64-SSE2-NEXT:    pmuludq %xmm1, %xmm0
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; X64-SSE2-NEXT:    pmuludq %xmm3, %xmm1
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; X64-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; X64-SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
+; X64-SSE2-NEXT:    pcmpeqd %xmm2, %xmm0
+; X64-SSE2-NEXT:    retq
   %t0 = lshr <4 x i32> <i32 1, i32 1, i32 1, i32 1>, %y
   %t1 = and <4 x i32> %t0, %x
   %res = icmp eq <4 x i32> %t1, <i32 0, i32 0, i32 0, i32 0>
@@ -594,37 +582,48 @@ define <4 x i1> @vec_4xi32_nonsplat_eq(<4 x i32> %x, <4 x i32> %y) nounwind {
 }
 
 define <4 x i1> @vec_4xi32_nonsplat_undef0_eq(<4 x i32> %x, <4 x i32> %y) nounwind {
-; SSE2-LABEL: vec_4xi32_nonsplat_undef0_eq:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm1[2,3,3,3,4,5,6,7]
-; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = <1,1,u,1>
-; SSE2-NEXT:    movdqa %xmm3, %xmm4
-; SSE2-NEXT:    psrld %xmm2, %xmm4
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm1[0,1,1,1,4,5,6,7]
-; SSE2-NEXT:    movdqa %xmm3, %xmm5
-; SSE2-NEXT:    psrld %xmm2, %xmm5
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm4[0]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm1[2,3,3,3,4,5,6,7]
-; SSE2-NEXT:    movdqa %xmm3, %xmm4
-; SSE2-NEXT:    psrld %xmm2, %xmm4
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,1,1,1,4,5,6,7]
-; SSE2-NEXT:    psrld %xmm1, %xmm3
-; SSE2-NEXT:    punpckhqdq {{.*#+}} xmm3 = xmm3[1],xmm4[1]
-; SSE2-NEXT:    shufps {{.*#+}} xmm5 = xmm5[0,3],xmm3[0,3]
-; SSE2-NEXT:    andps %xmm5, %xmm0
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
-; SSE2-NEXT:    ret{{[l|q]}}
+; X86-SSE2-LABEL: vec_4xi32_nonsplat_undef0_eq:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pxor %xmm2, %xmm2
+; X86-SSE2-NEXT:    pslld $23, %xmm1
+; X86-SSE2-NEXT:    paddd {{\.LCPI.*}}, %xmm1
+; X86-SSE2-NEXT:    cvttps2dq %xmm1, %xmm1
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; X86-SSE2-NEXT:    pmuludq %xmm1, %xmm0
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; X86-SSE2-NEXT:    pmuludq %xmm3, %xmm1
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; X86-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pcmpeqd %xmm2, %xmm0
+; X86-SSE2-NEXT:    retl
 ;
 ; AVX2-LABEL: vec_4xi32_nonsplat_undef0_eq:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [1,1,1,1]
-; AVX2-NEXT:    vpsrlvd %xmm1, %xmm2, %xmm1
-; AVX2-NEXT:    vpand %xmm0, %xmm1, %xmm0
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX2-NEXT:    vpsllvd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpand %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm0, %xmm0
 ; AVX2-NEXT:    ret{{[l|q]}}
+;
+; X64-SSE2-LABEL: vec_4xi32_nonsplat_undef0_eq:
+; X64-SSE2:       # %bb.0:
+; X64-SSE2-NEXT:    pxor %xmm2, %xmm2
+; X64-SSE2-NEXT:    pslld $23, %xmm1
+; X64-SSE2-NEXT:    paddd {{.*}}(%rip), %xmm1
+; X64-SSE2-NEXT:    cvttps2dq %xmm1, %xmm1
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; X64-SSE2-NEXT:    pmuludq %xmm1, %xmm0
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; X64-SSE2-NEXT:    pmuludq %xmm3, %xmm1
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; X64-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; X64-SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
+; X64-SSE2-NEXT:    pcmpeqd %xmm2, %xmm0
+; X64-SSE2-NEXT:    retq
   %t0 = lshr <4 x i32> <i32 1, i32 1, i32 undef, i32 1>, %y
   %t1 = and <4 x i32> %t0, %x
   %res = icmp eq <4 x i32> %t1, <i32 0, i32 0, i32 0, i32 0>
@@ -713,20 +712,19 @@ define i1 @scalar_i8_signbit_ne(i8 %x, i8 %y) nounwind {
 ; X86-LABEL: scalar_i8_signbit_ne:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X86-NEXT:    movb $-128, %al
-; X86-NEXT:    shrb %cl, %al
-; X86-NEXT:    testb %al, {{[0-9]+}}(%esp)
-; X86-NEXT:    setne %al
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    shlb %cl, %al
+; X86-NEXT:    shrb $7, %al
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: scalar_i8_signbit_ne:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl %esi, %ecx
-; X64-NEXT:    movb $-128, %al
+; X64-NEXT:    movl %edi, %eax
 ; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
-; X64-NEXT:    shrb %cl, %al
-; X64-NEXT:    testb %dil, %al
-; X64-NEXT:    setne %al
+; X64-NEXT:    shlb %cl, %al
+; X64-NEXT:    shrb $7, %al
+; X64-NEXT:    # kill: def $al killed $al killed $eax
 ; X64-NEXT:    retq
   %t0 = lshr i8 128, %y
   %t1 = and i8 %t0, %x
@@ -764,7 +762,7 @@ define i1 @scalar_i32_x_is_const2_eq(i32 %y) nounwind {
 ; X86-BMI1-NEXT:    movb {{[0-9]+}}(%esp), %cl
 ; X86-BMI1-NEXT:    movl $1, %eax
 ; X86-BMI1-NEXT:    shrl %cl, %eax
-; X86-BMI1-NEXT:    testl $-1437226411, %eax # imm = 0xAA55AA55
+; X86-BMI1-NEXT:    testb %al, %al
 ; X86-BMI1-NEXT:    sete %al
 ; X86-BMI1-NEXT:    retl
 ;
@@ -773,7 +771,7 @@ define i1 @scalar_i32_x_is_const2_eq(i32 %y) nounwind {
 ; X86-BMI2-NEXT:    movb {{[0-9]+}}(%esp), %al
 ; X86-BMI2-NEXT:    movl $1, %ecx
 ; X86-BMI2-NEXT:    shrxl %eax, %ecx, %eax
-; X86-BMI2-NEXT:    testl $-1437226411, %eax # imm = 0xAA55AA55
+; X86-BMI2-NEXT:    testb %al, %al
 ; X86-BMI2-NEXT:    sete %al
 ; X86-BMI2-NEXT:    retl
 ;
@@ -783,7 +781,7 @@ define i1 @scalar_i32_x_is_const2_eq(i32 %y) nounwind {
 ; X64-BMI1-NEXT:    movl $1, %eax
 ; X64-BMI1-NEXT:    # kill: def $cl killed $cl killed $ecx
 ; X64-BMI1-NEXT:    shrl %cl, %eax
-; X64-BMI1-NEXT:    testl $-1437226411, %eax # imm = 0xAA55AA55
+; X64-BMI1-NEXT:    testb %al, %al
 ; X64-BMI1-NEXT:    sete %al
 ; X64-BMI1-NEXT:    retq
 ;
@@ -791,7 +789,7 @@ define i1 @scalar_i32_x_is_const2_eq(i32 %y) nounwind {
 ; X64-BMI2:       # %bb.0:
 ; X64-BMI2-NEXT:    movl $1, %eax
 ; X64-BMI2-NEXT:    shrxl %edi, %eax, %eax
-; X64-BMI2-NEXT:    testl $-1437226411, %eax # imm = 0xAA55AA55
+; X64-BMI2-NEXT:    testb %al, %al
 ; X64-BMI2-NEXT:    sete %al
 ; X64-BMI2-NEXT:    retq
   %t0 = lshr i32 1, %y
@@ -805,24 +803,10 @@ define i1 @scalar_i32_x_is_const2_eq(i32 %y) nounwind {
 ;------------------------------------------------------------------------------;
 
 define i1 @negative_scalar_i8_bitsinmiddle_slt(i8 %x, i8 %y) nounwind {
-; X86-LABEL: negative_scalar_i8_bitsinmiddle_slt:
-; X86:       # %bb.0:
-; X86-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X86-NEXT:    movb $24, %al
-; X86-NEXT:    shrb %cl, %al
-; X86-NEXT:    andb {{[0-9]+}}(%esp), %al
-; X86-NEXT:    shrb $7, %al
-; X86-NEXT:    retl
-;
-; X64-LABEL: negative_scalar_i8_bitsinmiddle_slt:
-; X64:       # %bb.0:
-; X64-NEXT:    movl %esi, %ecx
-; X64-NEXT:    movb $24, %al
-; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
-; X64-NEXT:    shrb %cl, %al
-; X64-NEXT:    andb %dil, %al
-; X64-NEXT:    shrb $7, %al
-; X64-NEXT:    retq
+; CHECK-LABEL: negative_scalar_i8_bitsinmiddle_slt:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    ret{{[l|q]}}
   %t0 = lshr i8 24, %y
   %t1 = and i8 %t0, %x
   %res = icmp slt i8 %t1, 0
diff --git a/llvm/test/CodeGen/X86/hoist-and-by-const-from-shl-in-eqcmp-zero.ll b/llvm/test/CodeGen/X86/hoist-and-by-const-from-shl-in-eqcmp-zero.ll
index aa4a135fd..99799d17d 100644
--- a/llvm/test/CodeGen/X86/hoist-and-by-const-from-shl-in-eqcmp-zero.ll
+++ b/llvm/test/CodeGen/X86/hoist-and-by-const-from-shl-in-eqcmp-zero.ll
@@ -23,19 +23,18 @@ define i1 @scalar_i8_signbit_eq(i8 %x, i8 %y) nounwind {
 ; X86-LABEL: scalar_i8_signbit_eq:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X86-NEXT:    movb $-128, %al
-; X86-NEXT:    shlb %cl, %al
-; X86-NEXT:    testb %al, {{[0-9]+}}(%esp)
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    shrb %cl, %al
+; X86-NEXT:    testb $-128, %al
 ; X86-NEXT:    sete %al
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: scalar_i8_signbit_eq:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl %esi, %ecx
-; X64-NEXT:    movb $-128, %al
 ; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
-; X64-NEXT:    shlb %cl, %al
-; X64-NEXT:    testb %dil, %al
+; X64-NEXT:    shrb %cl, %dil
+; X64-NEXT:    testb $-128, %dil
 ; X64-NEXT:    sete %al
 ; X64-NEXT:    retq
   %t0 = shl i8 128, %y
@@ -68,19 +67,18 @@ define i1 @scalar_i8_bitsinmiddle_eq(i8 %x, i8 %y) nounwind {
 ; X86-LABEL: scalar_i8_bitsinmiddle_eq:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X86-NEXT:    movb $24, %al
-; X86-NEXT:    shlb %cl, %al
-; X86-NEXT:    testb %al, {{[0-9]+}}(%esp)
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    shrb %cl, %al
+; X86-NEXT:    testb $24, %al
 ; X86-NEXT:    sete %al
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: scalar_i8_bitsinmiddle_eq:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl %esi, %ecx
-; X64-NEXT:    movb $24, %al
 ; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
-; X64-NEXT:    shlb %cl, %al
-; X64-NEXT:    testb %dil, %al
+; X64-NEXT:    shrb %cl, %dil
+; X64-NEXT:    testb $24, %dil
 ; X64-NEXT:    sete %al
 ; X64-NEXT:    retq
   %t0 = shl i8 24, %y
@@ -95,36 +93,36 @@ define i1 @scalar_i16_signbit_eq(i16 %x, i16 %y) nounwind {
 ; X86-BMI1-LABEL: scalar_i16_signbit_eq:
 ; X86-BMI1:       # %bb.0:
 ; X86-BMI1-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X86-BMI1-NEXT:    movl $-32768, %eax # imm = 0x8000
-; X86-BMI1-NEXT:    shll %cl, %eax
-; X86-BMI1-NEXT:    testw %ax, {{[0-9]+}}(%esp)
+; X86-BMI1-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-BMI1-NEXT:    shrl %cl, %eax
+; X86-BMI1-NEXT:    testl $32768, %eax # imm = 0x8000
 ; X86-BMI1-NEXT:    sete %al
 ; X86-BMI1-NEXT:    retl
 ;
 ; X86-BMI2-LABEL: scalar_i16_signbit_eq:
 ; X86-BMI2:       # %bb.0:
-; X86-BMI2-NEXT:    movb {{[0-9]+}}(%esp), %al
-; X86-BMI2-NEXT:    movl $-32768, %ecx # imm = 0x8000
-; X86-BMI2-NEXT:    shlxl %eax, %ecx, %eax
-; X86-BMI2-NEXT:    testw %ax, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-BMI2-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-BMI2-NEXT:    shrxl %ecx, %eax, %eax
+; X86-BMI2-NEXT:    testl $32768, %eax # imm = 0x8000
 ; X86-BMI2-NEXT:    sete %al
 ; X86-BMI2-NEXT:    retl
 ;
 ; X64-BMI1-LABEL: scalar_i16_signbit_eq:
 ; X64-BMI1:       # %bb.0:
 ; X64-BMI1-NEXT:    movl %esi, %ecx
-; X64-BMI1-NEXT:    movl $-32768, %eax # imm = 0x8000
+; X64-BMI1-NEXT:    movzwl %di, %eax
 ; X64-BMI1-NEXT:    # kill: def $cl killed $cl killed $ecx
-; X64-BMI1-NEXT:    shll %cl, %eax
-; X64-BMI1-NEXT:    testw %di, %ax
+; X64-BMI1-NEXT:    shrl %cl, %eax
+; X64-BMI1-NEXT:    testl $32768, %eax # imm = 0x8000
 ; X64-BMI1-NEXT:    sete %al
 ; X64-BMI1-NEXT:    retq
 ;
 ; X64-BMI2-LABEL: scalar_i16_signbit_eq:
 ; X64-BMI2:       # %bb.0:
-; X64-BMI2-NEXT:    movl $-32768, %eax # imm = 0x8000
-; X64-BMI2-NEXT:    shlxl %esi, %eax, %eax
-; X64-BMI2-NEXT:    testw %di, %ax
+; X64-BMI2-NEXT:    movzwl %di, %eax
+; X64-BMI2-NEXT:    shrxl %esi, %eax, %eax
+; X64-BMI2-NEXT:    testl $32768, %eax # imm = 0x8000
 ; X64-BMI2-NEXT:    sete %al
 ; X64-BMI2-NEXT:    retq
   %t0 = shl i16 32768, %y
@@ -157,36 +155,36 @@ define i1 @scalar_i16_bitsinmiddle_eq(i16 %x, i16 %y) nounwind {
 ; X86-BMI1-LABEL: scalar_i16_bitsinmiddle_eq:
 ; X86-BMI1:       # %bb.0:
 ; X86-BMI1-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X86-BMI1-NEXT:    movl $4080, %eax # imm = 0xFF0
-; X86-BMI1-NEXT:    shll %cl, %eax
-; X86-BMI1-NEXT:    testw %ax, {{[0-9]+}}(%esp)
+; X86-BMI1-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-BMI1-NEXT:    shrl %cl, %eax
+; X86-BMI1-NEXT:    testl $4080, %eax # imm = 0xFF0
 ; X86-BMI1-NEXT:    sete %al
 ; X86-BMI1-NEXT:    retl
 ;
 ; X86-BMI2-LABEL: scalar_i16_bitsinmiddle_eq:
 ; X86-BMI2:       # %bb.0:
-; X86-BMI2-NEXT:    movb {{[0-9]+}}(%esp), %al
-; X86-BMI2-NEXT:    movl $4080, %ecx # imm = 0xFF0
-; X86-BMI2-NEXT:    shlxl %eax, %ecx, %eax
-; X86-BMI2-NEXT:    testw %ax, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-BMI2-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-BMI2-NEXT:    shrxl %ecx, %eax, %eax
+; X86-BMI2-NEXT:    testl $4080, %eax # imm = 0xFF0
 ; X86-BMI2-NEXT:    sete %al
 ; X86-BMI2-NEXT:    retl
 ;
 ; X64-BMI1-LABEL: scalar_i16_bitsinmiddle_eq:
 ; X64-BMI1:       # %bb.0:
 ; X64-BMI1-NEXT:    movl %esi, %ecx
-; X64-BMI1-NEXT:    movl $4080, %eax # imm = 0xFF0
+; X64-BMI1-NEXT:    movzwl %di, %eax
 ; X64-BMI1-NEXT:    # kill: def $cl killed $cl killed $ecx
-; X64-BMI1-NEXT:    shll %cl, %eax
-; X64-BMI1-NEXT:    testw %di, %ax
+; X64-BMI1-NEXT:    shrl %cl, %eax
+; X64-BMI1-NEXT:    testl $4080, %eax # imm = 0xFF0
 ; X64-BMI1-NEXT:    sete %al
 ; X64-BMI1-NEXT:    retq
 ;
 ; X64-BMI2-LABEL: scalar_i16_bitsinmiddle_eq:
 ; X64-BMI2:       # %bb.0:
-; X64-BMI2-NEXT:    movl $4080, %eax # imm = 0xFF0
-; X64-BMI2-NEXT:    shlxl %esi, %eax, %eax
-; X64-BMI2-NEXT:    testw %di, %ax
+; X64-BMI2-NEXT:    movzwl %di, %eax
+; X64-BMI2-NEXT:    shrxl %esi, %eax, %eax
+; X64-BMI2-NEXT:    testl $4080, %eax # imm = 0xFF0
 ; X64-BMI2-NEXT:    sete %al
 ; X64-BMI2-NEXT:    retq
   %t0 = shl i16 4080, %y
@@ -201,36 +199,33 @@ define i1 @scalar_i32_signbit_eq(i32 %x, i32 %y) nounwind {
 ; X86-BMI1-LABEL: scalar_i32_signbit_eq:
 ; X86-BMI1:       # %bb.0:
 ; X86-BMI1-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X86-BMI1-NEXT:    movl $-2147483648, %eax # imm = 0x80000000
-; X86-BMI1-NEXT:    shll %cl, %eax
-; X86-BMI1-NEXT:    testl %eax, {{[0-9]+}}(%esp)
+; X86-BMI1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-BMI1-NEXT:    shrl %cl, %eax
+; X86-BMI1-NEXT:    testl $-2147483648, %eax # imm = 0x80000000
 ; X86-BMI1-NEXT:    sete %al
 ; X86-BMI1-NEXT:    retl
 ;
 ; X86-BMI2-LABEL: scalar_i32_signbit_eq:
 ; X86-BMI2:       # %bb.0:
 ; X86-BMI2-NEXT:    movb {{[0-9]+}}(%esp), %al
-; X86-BMI2-NEXT:    movl $-2147483648, %ecx # imm = 0x80000000
-; X86-BMI2-NEXT:    shlxl %eax, %ecx, %eax
-; X86-BMI2-NEXT:    testl %eax, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT:    shrxl %eax, {{[0-9]+}}(%esp), %eax
+; X86-BMI2-NEXT:    testl $-2147483648, %eax # imm = 0x80000000
 ; X86-BMI2-NEXT:    sete %al
 ; X86-BMI2-NEXT:    retl
 ;
 ; X64-BMI1-LABEL: scalar_i32_signbit_eq:
 ; X64-BMI1:       # %bb.0:
 ; X64-BMI1-NEXT:    movl %esi, %ecx
-; X64-BMI1-NEXT:    movl $-2147483648, %eax # imm = 0x80000000
 ; X64-BMI1-NEXT:    # kill: def $cl killed $cl killed $ecx
-; X64-BMI1-NEXT:    shll %cl, %eax
-; X64-BMI1-NEXT:    testl %edi, %eax
+; X64-BMI1-NEXT:    shrl %cl, %edi
+; X64-BMI1-NEXT:    testl $-2147483648, %edi # imm = 0x80000000
 ; X64-BMI1-NEXT:    sete %al
 ; X64-BMI1-NEXT:    retq
 ;
 ; X64-BMI2-LABEL: scalar_i32_signbit_eq:
 ; X64-BMI2:       # %bb.0:
-; X64-BMI2-NEXT:    movl $-2147483648, %eax # imm = 0x80000000
-; X64-BMI2-NEXT:    shlxl %esi, %eax, %eax
-; X64-BMI2-NEXT:    testl %edi, %eax
+; X64-BMI2-NEXT:    shrxl %esi, %edi, %eax
+; X64-BMI2-NEXT:    testl $-2147483648, %eax # imm = 0x80000000
 ; X64-BMI2-NEXT:    sete %al
 ; X64-BMI2-NEXT:    retq
   %t0 = shl i32 2147483648, %y
@@ -263,36 +258,33 @@ define i1 @scalar_i32_bitsinmiddle_eq(i32 %x, i32 %y) nounwind {
 ; X86-BMI1-LABEL: scalar_i32_bitsinmiddle_eq:
 ; X86-BMI1:       # %bb.0:
 ; X86-BMI1-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X86-BMI1-NEXT:    movl $16776960, %eax # imm = 0xFFFF00
-; X86-BMI1-NEXT:    shll %cl, %eax
-; X86-BMI1-NEXT:    testl %eax, {{[0-9]+}}(%esp)
+; X86-BMI1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-BMI1-NEXT:    shrl %cl, %eax
+; X86-BMI1-NEXT:    testl $16776960, %eax # imm = 0xFFFF00
 ; X86-BMI1-NEXT:    sete %al
 ; X86-BMI1-NEXT:    retl
 ;
 ; X86-BMI2-LABEL: scalar_i32_bitsinmiddle_eq:
 ; X86-BMI2:       # %bb.0:
 ; X86-BMI2-NEXT:    movb {{[0-9]+}}(%esp), %al
-; X86-BMI2-NEXT:    movl $16776960, %ecx # imm = 0xFFFF00
-; X86-BMI2-NEXT:    shlxl %eax, %ecx, %eax
-; X86-BMI2-NEXT:    testl %eax, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT:    shrxl %eax, {{[0-9]+}}(%esp), %eax
+; X86-BMI2-NEXT:    testl $16776960, %eax # imm = 0xFFFF00
 ; X86-BMI2-NEXT:    sete %al
 ; X86-BMI2-NEXT:    retl
 ;
 ; X64-BMI1-LABEL: scalar_i32_bitsinmiddle_eq:
 ; X64-BMI1:       # %bb.0:
 ; X64-BMI1-NEXT:    movl %esi, %ecx
-; X64-BMI1-NEXT:    movl $16776960, %eax # imm = 0xFFFF00
 ; X64-BMI1-NEXT:    # kill: def $cl killed $cl killed $ecx
-; X64-BMI1-NEXT:    shll %cl, %eax
-; X64-BMI1-NEXT:    testl %edi, %eax
+; X64-BMI1-NEXT:    shrl %cl, %edi
+; X64-BMI1-NEXT:    testl $16776960, %edi # imm = 0xFFFF00
 ; X64-BMI1-NEXT:    sete %al
 ; X64-BMI1-NEXT:    retq
 ;
 ; X64-BMI2-LABEL: scalar_i32_bitsinmiddle_eq:
 ; X64-BMI2:       # %bb.0:
-; X64-BMI2-NEXT:    movl $16776960, %eax # imm = 0xFFFF00
-; X64-BMI2-NEXT:    shlxl %esi, %eax, %eax
-; X64-BMI2-NEXT:    testl %edi, %eax
+; X64-BMI2-NEXT:    shrxl %esi, %edi, %eax
+; X64-BMI2-NEXT:    testl $16776960, %eax # imm = 0xFFFF00
 ; X64-BMI2-NEXT:    sete %al
 ; X64-BMI2-NEXT:    retq
   %t0 = shl i32 16776960, %y
@@ -304,35 +296,43 @@ define i1 @scalar_i32_bitsinmiddle_eq(i32 %x, i32 %y) nounwind {
 ; i64 scalar
 
 define i1 @scalar_i64_signbit_eq(i64 %x, i64 %y) nounwind {
-; X86-LABEL: scalar_i64_signbit_eq:
-; X86:       # %bb.0:
-; X86-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X86-NEXT:    xorl %eax, %eax
-; X86-NEXT:    movl $-2147483648, %edx # imm = 0x80000000
-; X86-NEXT:    shldl %cl, %eax, %edx
-; X86-NEXT:    testb $32, %cl
-; X86-NEXT:    cmovnel %eax, %edx
-; X86-NEXT:    andl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    orl $0, %edx
-; X86-NEXT:    sete %al
-; X86-NEXT:    retl
+; X86-BMI1-LABEL: scalar_i64_signbit_eq:
+; X86-BMI1:       # %bb.0:
+; X86-BMI1-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-BMI1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-BMI1-NEXT:    shrl %cl, %eax
+; X86-BMI1-NEXT:    xorl %edx, %edx
+; X86-BMI1-NEXT:    testb $32, %cl
+; X86-BMI1-NEXT:    cmovel %eax, %edx
+; X86-BMI1-NEXT:    testl $-2147483648, %edx # imm = 0x80000000
+; X86-BMI1-NEXT:    sete %al
+; X86-BMI1-NEXT:    retl
+;
+; X86-BMI2-LABEL: scalar_i64_signbit_eq:
+; X86-BMI2:       # %bb.0:
+; X86-BMI2-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-BMI2-NEXT:    shrxl %eax, {{[0-9]+}}(%esp), %ecx
+; X86-BMI2-NEXT:    xorl %edx, %edx
+; X86-BMI2-NEXT:    testb $32, %al
+; X86-BMI2-NEXT:    cmovel %ecx, %edx
+; X86-BMI2-NEXT:    testl $-2147483648, %edx # imm = 0x80000000
+; X86-BMI2-NEXT:    sete %al
+; X86-BMI2-NEXT:    retl
 ;
 ; X64-BMI1-LABEL: scalar_i64_signbit_eq:
 ; X64-BMI1:       # %bb.0:
 ; X64-BMI1-NEXT:    movq %rsi, %rcx
-; X64-BMI1-NEXT:    movabsq $-9223372036854775808, %rax # imm = 0x8000000000000000
 ; X64-BMI1-NEXT:    # kill: def $cl killed $cl killed $rcx
-; X64-BMI1-NEXT:    shlq %cl, %rax
-; X64-BMI1-NEXT:    testq %rdi, %rax
-; X64-BMI1-NEXT:    sete %al
+; X64-BMI1-NEXT:    shrq %cl, %rdi
+; X64-BMI1-NEXT:    btq $63, %rdi
+; X64-BMI1-NEXT:    setae %al
 ; X64-BMI1-NEXT:    retq
 ;
 ; X64-BMI2-LABEL: scalar_i64_signbit_eq:
 ; X64-BMI2:       # %bb.0:
-; X64-BMI2-NEXT:    movabsq $-9223372036854775808, %rax # imm = 0x8000000000000000
-; X64-BMI2-NEXT:    shlxq %rsi, %rax, %rax
-; X64-BMI2-NEXT:    testq %rdi, %rax
-; X64-BMI2-NEXT:    sete %al
+; X64-BMI2-NEXT:    shrxq %rsi, %rdi, %rax
+; X64-BMI2-NEXT:    btq $63, %rax
+; X64-BMI2-NEXT:    setae %al
 ; X64-BMI2-NEXT:    retq
   %t0 = shl i64 9223372036854775808, %y
   %t1 = and i64 %t0, %x
@@ -395,17 +395,18 @@ define i1 @scalar_i64_bitsinmiddle_eq(i64 %x, i64 %y) nounwind {
 ; X86-BMI1:       # %bb.0:
 ; X86-BMI1-NEXT:    pushl %esi
 ; X86-BMI1-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X86-BMI1-NEXT:    movl $-65536, %eax # imm = 0xFFFF0000
-; X86-BMI1-NEXT:    movl $65535, %edx # imm = 0xFFFF
-; X86-BMI1-NEXT:    shldl %cl, %eax, %edx
-; X86-BMI1-NEXT:    shll %cl, %eax
-; X86-BMI1-NEXT:    xorl %esi, %esi
+; X86-BMI1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-BMI1-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-BMI1-NEXT:    movl %edx, %esi
+; X86-BMI1-NEXT:    shrl %cl, %esi
+; X86-BMI1-NEXT:    shrdl %cl, %edx, %eax
+; X86-BMI1-NEXT:    xorl %edx, %edx
 ; X86-BMI1-NEXT:    testb $32, %cl
-; X86-BMI1-NEXT:    cmovnel %eax, %edx
-; X86-BMI1-NEXT:    cmovel %eax, %esi
-; X86-BMI1-NEXT:    andl {{[0-9]+}}(%esp), %edx
-; X86-BMI1-NEXT:    andl {{[0-9]+}}(%esp), %esi
-; X86-BMI1-NEXT:    orl %edx, %esi
+; X86-BMI1-NEXT:    cmovnel %esi, %eax
+; X86-BMI1-NEXT:    cmovel %esi, %edx
+; X86-BMI1-NEXT:    andl $-65536, %eax # imm = 0xFFFF0000
+; X86-BMI1-NEXT:    movzwl %dx, %ecx
+; X86-BMI1-NEXT:    orl %eax, %ecx
 ; X86-BMI1-NEXT:    sete %al
 ; X86-BMI1-NEXT:    popl %esi
 ; X86-BMI1-NEXT:    retl
@@ -414,17 +415,17 @@ define i1 @scalar_i64_bitsinmiddle_eq(i64 %x, i64 %y) nounwind {
 ; X86-BMI2:       # %bb.0:
 ; X86-BMI2-NEXT:    pushl %esi
 ; X86-BMI2-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X86-BMI2-NEXT:    movl $-65536, %eax # imm = 0xFFFF0000
-; X86-BMI2-NEXT:    movl $65535, %edx # imm = 0xFFFF
-; X86-BMI2-NEXT:    shldl %cl, %eax, %edx
-; X86-BMI2-NEXT:    shlxl %ecx, %eax, %eax
+; X86-BMI2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-BMI2-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-BMI2-NEXT:    shrdl %cl, %edx, %eax
+; X86-BMI2-NEXT:    shrxl %ecx, %edx, %edx
 ; X86-BMI2-NEXT:    xorl %esi, %esi
 ; X86-BMI2-NEXT:    testb $32, %cl
-; X86-BMI2-NEXT:    cmovnel %eax, %edx
-; X86-BMI2-NEXT:    cmovel %eax, %esi
-; X86-BMI2-NEXT:    andl {{[0-9]+}}(%esp), %edx
-; X86-BMI2-NEXT:    andl {{[0-9]+}}(%esp), %esi
-; X86-BMI2-NEXT:    orl %edx, %esi
+; X86-BMI2-NEXT:    cmovnel %edx, %eax
+; X86-BMI2-NEXT:    cmovel %edx, %esi
+; X86-BMI2-NEXT:    andl $-65536, %eax # imm = 0xFFFF0000
+; X86-BMI2-NEXT:    movzwl %si, %ecx
+; X86-BMI2-NEXT:    orl %eax, %ecx
 ; X86-BMI2-NEXT:    sete %al
 ; X86-BMI2-NEXT:    popl %esi
 ; X86-BMI2-NEXT:    retl
@@ -432,18 +433,18 @@ define i1 @scalar_i64_bitsinmiddle_eq(i64 %x, i64 %y) nounwind {
 ; X64-BMI1-LABEL: scalar_i64_bitsinmiddle_eq:
 ; X64-BMI1:       # %bb.0:
 ; X64-BMI1-NEXT:    movq %rsi, %rcx
-; X64-BMI1-NEXT:    movabsq $281474976645120, %rax # imm = 0xFFFFFFFF0000
 ; X64-BMI1-NEXT:    # kill: def $cl killed $cl killed $rcx
-; X64-BMI1-NEXT:    shlq %cl, %rax
-; X64-BMI1-NEXT:    testq %rdi, %rax
+; X64-BMI1-NEXT:    shrq %cl, %rdi
+; X64-BMI1-NEXT:    movabsq $281474976645120, %rax # imm = 0xFFFFFFFF0000
+; X64-BMI1-NEXT:    testq %rax, %rdi
 ; X64-BMI1-NEXT:    sete %al
 ; X64-BMI1-NEXT:    retq
 ;
 ; X64-BMI2-LABEL: scalar_i64_bitsinmiddle_eq:
 ; X64-BMI2:       # %bb.0:
-; X64-BMI2-NEXT:    movabsq $281474976645120, %rax # imm = 0xFFFFFFFF0000
-; X64-BMI2-NEXT:    shlxq %rsi, %rax, %rax
-; X64-BMI2-NEXT:    testq %rdi, %rax
+; X64-BMI2-NEXT:    shrxq %rsi, %rdi, %rax
+; X64-BMI2-NEXT:    movabsq $281474976645120, %rcx # imm = 0xFFFFFFFF0000
+; X64-BMI2-NEXT:    testq %rcx, %rax
 ; X64-BMI2-NEXT:    sete %al
 ; X64-BMI2-NEXT:    retq
   %t0 = shl i64 281474976645120, %y
@@ -477,10 +478,10 @@ define <4 x i1> @vec_4xi32_splat_eq(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; AVX2-LABEL: vec_4xi32_splat_eq:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [1,1,1,1]
-; AVX2-NEXT:    vpsllvd %xmm1, %xmm2, %xmm1
-; AVX2-NEXT:    vpand %xmm0, %xmm1, %xmm0
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX2-NEXT:    vpsrlvd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpand %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm0, %xmm0
 ; AVX2-NEXT:    ret{{[l|q]}}
 ;
 ; X64-SSE2-LABEL: vec_4xi32_splat_eq:
@@ -577,10 +578,10 @@ define <4 x i1> @vec_4xi32_nonsplat_undef0_eq(<4 x i32> %x, <4 x i32> %y) nounwi
 ; AVX2-LABEL: vec_4xi32_nonsplat_undef0_eq:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [1,1,1,1]
-; AVX2-NEXT:    vpsllvd %xmm1, %xmm2, %xmm1
-; AVX2-NEXT:    vpand %xmm0, %xmm1, %xmm0
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX2-NEXT:    vpsrlvd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpand %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm0, %xmm0
 ; AVX2-NEXT:    ret{{[l|q]}}
 ;
 ; X64-SSE2-LABEL: vec_4xi32_nonsplat_undef0_eq:
@@ -709,20 +710,19 @@ define i1 @scalar_i8_signbit_ne(i8 %x, i8 %y) nounwind {
 ; X86-LABEL: scalar_i8_signbit_ne:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X86-NEXT:    movb $-128, %al
-; X86-NEXT:    shlb %cl, %al
-; X86-NEXT:    testb %al, {{[0-9]+}}(%esp)
-; X86-NEXT:    setne %al
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    shrb %cl, %al
+; X86-NEXT:    shrb $7, %al
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: scalar_i8_signbit_ne:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl %esi, %ecx
-; X64-NEXT:    movb $-128, %al
+; X64-NEXT:    movl %edi, %eax
 ; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
-; X64-NEXT:    shlb %cl, %al
-; X64-NEXT:    testb %dil, %al
-; X64-NEXT:    setne %al
+; X64-NEXT:    shrb %cl, %al
+; X64-NEXT:    shrb $7, %al
+; X64-NEXT:    # kill: def $al killed $al killed $eax
 ; X64-NEXT:    retq
   %t0 = shl i8 128, %y
   %t1 = and i8 %t0, %x
diff --git a/llvm/test/CodeGen/X86/horizontal-reduce-smax.ll b/llvm/test/CodeGen/X86/horizontal-reduce-smax.ll
index c08f08f38..9419aff77 100644
--- a/llvm/test/CodeGen/X86/horizontal-reduce-smax.ll
+++ b/llvm/test/CodeGen/X86/horizontal-reduce-smax.ll
@@ -1829,3 +1829,417 @@ define i8 @test_reduce_v64i8(<64 x i8> %a0) {
   %19 = extractelement <64 x i8> %18, i32 0
   ret i8 %19
 }
+
+;
+; Partial Vector Reductions
+;
+
+define i16 @test_reduce_v16i16_v8i16(<16 x i16> %a0) {
+; X86-SSE2-LABEL: test_reduce_v16i16_v8i16:
+; X86-SSE2:       ## %bb.0:
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE2-NEXT:    pmaxsw %xmm0, %xmm1
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X86-SSE2-NEXT:    pmaxsw %xmm1, %xmm0
+; X86-SSE2-NEXT:    movdqa %xmm0, %xmm1
+; X86-SSE2-NEXT:    psrld $16, %xmm1
+; X86-SSE2-NEXT:    pmaxsw %xmm0, %xmm1
+; X86-SSE2-NEXT:    movd %xmm1, %eax
+; X86-SSE2-NEXT:    ## kill: def $ax killed $ax killed $eax
+; X86-SSE2-NEXT:    retl
+;
+; X86-SSE42-LABEL: test_reduce_v16i16_v8i16:
+; X86-SSE42:       ## %bb.0:
+; X86-SSE42-NEXT:    pxor LCPI12_0, %xmm0
+; X86-SSE42-NEXT:    phminposuw %xmm0, %xmm0
+; X86-SSE42-NEXT:    movd %xmm0, %eax
+; X86-SSE42-NEXT:    xorl $32767, %eax ## imm = 0x7FFF
+; X86-SSE42-NEXT:    ## kill: def $ax killed $ax killed $eax
+; X86-SSE42-NEXT:    retl
+;
+; X86-AVX-LABEL: test_reduce_v16i16_v8i16:
+; X86-AVX:       ## %bb.0:
+; X86-AVX-NEXT:    vpxor LCPI12_0, %xmm0, %xmm0
+; X86-AVX-NEXT:    vphminposuw %xmm0, %xmm0
+; X86-AVX-NEXT:    vmovd %xmm0, %eax
+; X86-AVX-NEXT:    xorl $32767, %eax ## imm = 0x7FFF
+; X86-AVX-NEXT:    ## kill: def $ax killed $ax killed $eax
+; X86-AVX-NEXT:    vzeroupper
+; X86-AVX-NEXT:    retl
+;
+; X64-SSE2-LABEL: test_reduce_v16i16_v8i16:
+; X64-SSE2:       ## %bb.0:
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-SSE2-NEXT:    pmaxsw %xmm0, %xmm1
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X64-SSE2-NEXT:    pmaxsw %xmm1, %xmm0
+; X64-SSE2-NEXT:    movdqa %xmm0, %xmm1
+; X64-SSE2-NEXT:    psrld $16, %xmm1
+; X64-SSE2-NEXT:    pmaxsw %xmm0, %xmm1
+; X64-SSE2-NEXT:    movd %xmm1, %eax
+; X64-SSE2-NEXT:    ## kill: def $ax killed $ax killed $eax
+; X64-SSE2-NEXT:    retq
+;
+; X64-SSE42-LABEL: test_reduce_v16i16_v8i16:
+; X64-SSE42:       ## %bb.0:
+; X64-SSE42-NEXT:    pxor {{.*}}(%rip), %xmm0
+; X64-SSE42-NEXT:    phminposuw %xmm0, %xmm0
+; X64-SSE42-NEXT:    movd %xmm0, %eax
+; X64-SSE42-NEXT:    xorl $32767, %eax ## imm = 0x7FFF
+; X64-SSE42-NEXT:    ## kill: def $ax killed $ax killed $eax
+; X64-SSE42-NEXT:    retq
+;
+; X64-AVX-LABEL: test_reduce_v16i16_v8i16:
+; X64-AVX:       ## %bb.0:
+; X64-AVX-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; X64-AVX-NEXT:    vphminposuw %xmm0, %xmm0
+; X64-AVX-NEXT:    vmovd %xmm0, %eax
+; X64-AVX-NEXT:    xorl $32767, %eax ## imm = 0x7FFF
+; X64-AVX-NEXT:    ## kill: def $ax killed $ax killed $eax
+; X64-AVX-NEXT:    vzeroupper
+; X64-AVX-NEXT:    retq
+  %1  = shufflevector <16 x i16> %a0, <16 x i16> undef, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %2  = icmp sgt <16 x i16> %a0, %1
+  %3  = select <16 x i1> %2, <16 x i16> %a0, <16 x i16> %1
+  %4  = shufflevector <16 x i16> %3, <16 x i16> undef, <16 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %5  = icmp sgt <16 x i16> %3, %4
+  %6  = select <16 x i1> %5, <16 x i16> %3, <16 x i16> %4
+  %7  = shufflevector <16 x i16> %6, <16 x i16> undef, <16 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %8  = icmp sgt <16 x i16> %6, %7
+  %9  = select <16 x i1> %8, <16 x i16> %6, <16 x i16> %7
+  %10 = extractelement <16 x i16> %9, i32 0
+  ret i16 %10
+}
+
+define i16 @test_reduce_v32i16_v8i16(<32 x i16> %a0) {
+; X86-SSE2-LABEL: test_reduce_v32i16_v8i16:
+; X86-SSE2:       ## %bb.0:
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE2-NEXT:    pmaxsw %xmm0, %xmm1
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X86-SSE2-NEXT:    pmaxsw %xmm1, %xmm0
+; X86-SSE2-NEXT:    movdqa %xmm0, %xmm1
+; X86-SSE2-NEXT:    psrld $16, %xmm1
+; X86-SSE2-NEXT:    pmaxsw %xmm0, %xmm1
+; X86-SSE2-NEXT:    movd %xmm1, %eax
+; X86-SSE2-NEXT:    ## kill: def $ax killed $ax killed $eax
+; X86-SSE2-NEXT:    retl
+;
+; X86-SSE42-LABEL: test_reduce_v32i16_v8i16:
+; X86-SSE42:       ## %bb.0:
+; X86-SSE42-NEXT:    pxor LCPI13_0, %xmm0
+; X86-SSE42-NEXT:    phminposuw %xmm0, %xmm0
+; X86-SSE42-NEXT:    movd %xmm0, %eax
+; X86-SSE42-NEXT:    xorl $32767, %eax ## imm = 0x7FFF
+; X86-SSE42-NEXT:    ## kill: def $ax killed $ax killed $eax
+; X86-SSE42-NEXT:    retl
+;
+; X86-AVX-LABEL: test_reduce_v32i16_v8i16:
+; X86-AVX:       ## %bb.0:
+; X86-AVX-NEXT:    vpxor LCPI13_0, %xmm0, %xmm0
+; X86-AVX-NEXT:    vphminposuw %xmm0, %xmm0
+; X86-AVX-NEXT:    vmovd %xmm0, %eax
+; X86-AVX-NEXT:    xorl $32767, %eax ## imm = 0x7FFF
+; X86-AVX-NEXT:    ## kill: def $ax killed $ax killed $eax
+; X86-AVX-NEXT:    vzeroupper
+; X86-AVX-NEXT:    retl
+;
+; X64-SSE2-LABEL: test_reduce_v32i16_v8i16:
+; X64-SSE2:       ## %bb.0:
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-SSE2-NEXT:    pmaxsw %xmm0, %xmm1
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X64-SSE2-NEXT:    pmaxsw %xmm1, %xmm0
+; X64-SSE2-NEXT:    movdqa %xmm0, %xmm1
+; X64-SSE2-NEXT:    psrld $16, %xmm1
+; X64-SSE2-NEXT:    pmaxsw %xmm0, %xmm1
+; X64-SSE2-NEXT:    movd %xmm1, %eax
+; X64-SSE2-NEXT:    ## kill: def $ax killed $ax killed $eax
+; X64-SSE2-NEXT:    retq
+;
+; X64-SSE42-LABEL: test_reduce_v32i16_v8i16:
+; X64-SSE42:       ## %bb.0:
+; X64-SSE42-NEXT:    pxor {{.*}}(%rip), %xmm0
+; X64-SSE42-NEXT:    phminposuw %xmm0, %xmm0
+; X64-SSE42-NEXT:    movd %xmm0, %eax
+; X64-SSE42-NEXT:    xorl $32767, %eax ## imm = 0x7FFF
+; X64-SSE42-NEXT:    ## kill: def $ax killed $ax killed $eax
+; X64-SSE42-NEXT:    retq
+;
+; X64-AVX-LABEL: test_reduce_v32i16_v8i16:
+; X64-AVX:       ## %bb.0:
+; X64-AVX-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; X64-AVX-NEXT:    vphminposuw %xmm0, %xmm0
+; X64-AVX-NEXT:    vmovd %xmm0, %eax
+; X64-AVX-NEXT:    xorl $32767, %eax ## imm = 0x7FFF
+; X64-AVX-NEXT:    ## kill: def $ax killed $ax killed $eax
+; X64-AVX-NEXT:    vzeroupper
+; X64-AVX-NEXT:    retq
+  %1  = shufflevector <32 x i16> %a0, <32 x i16> undef, <32 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %2  = icmp sgt <32 x i16> %a0, %1
+  %3  = select <32 x i1> %2, <32 x i16> %a0, <32 x i16> %1
+  %4  = shufflevector <32 x i16> %3, <32 x i16> undef, <32 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %5  = icmp sgt <32 x i16> %3, %4
+  %6  = select <32 x i1> %5, <32 x i16> %3, <32 x i16> %4
+  %7  = shufflevector <32 x i16> %6, <32 x i16> undef, <32 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %8  = icmp sgt <32 x i16> %6, %7
+  %9  = select <32 x i1> %8, <32 x i16> %6, <32 x i16> %7
+  %10 = extractelement <32 x i16> %9, i32 0
+  ret i16 %10
+}
+
+define i8 @test_reduce_v32i8_v16i8(<32 x i8> %a0) {
+; X86-SSE2-LABEL: test_reduce_v32i8_v16i8:
+; X86-SSE2:       ## %bb.0:
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE2-NEXT:    movdqa %xmm0, %xmm2
+; X86-SSE2-NEXT:    pcmpgtb %xmm1, %xmm2
+; X86-SSE2-NEXT:    pand %xmm2, %xmm0
+; X86-SSE2-NEXT:    pandn %xmm1, %xmm2
+; X86-SSE2-NEXT:    por %xmm0, %xmm2
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
+; X86-SSE2-NEXT:    movdqa %xmm2, %xmm1
+; X86-SSE2-NEXT:    pcmpgtb %xmm0, %xmm1
+; X86-SSE2-NEXT:    pand %xmm1, %xmm2
+; X86-SSE2-NEXT:    pandn %xmm0, %xmm1
+; X86-SSE2-NEXT:    por %xmm2, %xmm1
+; X86-SSE2-NEXT:    movdqa %xmm1, %xmm0
+; X86-SSE2-NEXT:    psrld $16, %xmm0
+; X86-SSE2-NEXT:    movdqa %xmm1, %xmm2
+; X86-SSE2-NEXT:    pcmpgtb %xmm0, %xmm2
+; X86-SSE2-NEXT:    pand %xmm2, %xmm1
+; X86-SSE2-NEXT:    pandn %xmm0, %xmm2
+; X86-SSE2-NEXT:    por %xmm1, %xmm2
+; X86-SSE2-NEXT:    movdqa %xmm2, %xmm0
+; X86-SSE2-NEXT:    psrlw $8, %xmm0
+; X86-SSE2-NEXT:    movdqa %xmm2, %xmm1
+; X86-SSE2-NEXT:    pcmpgtb %xmm0, %xmm1
+; X86-SSE2-NEXT:    pand %xmm1, %xmm2
+; X86-SSE2-NEXT:    pandn %xmm0, %xmm1
+; X86-SSE2-NEXT:    por %xmm2, %xmm1
+; X86-SSE2-NEXT:    movd %xmm1, %eax
+; X86-SSE2-NEXT:    ## kill: def $al killed $al killed $eax
+; X86-SSE2-NEXT:    retl
+;
+; X86-SSE42-LABEL: test_reduce_v32i8_v16i8:
+; X86-SSE42:       ## %bb.0:
+; X86-SSE42-NEXT:    pxor LCPI14_0, %xmm0
+; X86-SSE42-NEXT:    movdqa %xmm0, %xmm1
+; X86-SSE42-NEXT:    psrlw $8, %xmm1
+; X86-SSE42-NEXT:    pminub %xmm0, %xmm1
+; X86-SSE42-NEXT:    phminposuw %xmm1, %xmm0
+; X86-SSE42-NEXT:    pextrb $0, %xmm0, %eax
+; X86-SSE42-NEXT:    xorb $127, %al
+; X86-SSE42-NEXT:    ## kill: def $al killed $al killed $eax
+; X86-SSE42-NEXT:    retl
+;
+; X86-AVX-LABEL: test_reduce_v32i8_v16i8:
+; X86-AVX:       ## %bb.0:
+; X86-AVX-NEXT:    vpxor LCPI14_0, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; X86-AVX-NEXT:    vpminub %xmm1, %xmm0, %xmm0
+; X86-AVX-NEXT:    vphminposuw %xmm0, %xmm0
+; X86-AVX-NEXT:    vpextrb $0, %xmm0, %eax
+; X86-AVX-NEXT:    xorb $127, %al
+; X86-AVX-NEXT:    ## kill: def $al killed $al killed $eax
+; X86-AVX-NEXT:    vzeroupper
+; X86-AVX-NEXT:    retl
+;
+; X64-SSE2-LABEL: test_reduce_v32i8_v16i8:
+; X64-SSE2:       ## %bb.0:
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-SSE2-NEXT:    movdqa %xmm0, %xmm2
+; X64-SSE2-NEXT:    pcmpgtb %xmm1, %xmm2
+; X64-SSE2-NEXT:    pand %xmm2, %xmm0
+; X64-SSE2-NEXT:    pandn %xmm1, %xmm2
+; X64-SSE2-NEXT:    por %xmm0, %xmm2
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
+; X64-SSE2-NEXT:    movdqa %xmm2, %xmm1
+; X64-SSE2-NEXT:    pcmpgtb %xmm0, %xmm1
+; X64-SSE2-NEXT:    pand %xmm1, %xmm2
+; X64-SSE2-NEXT:    pandn %xmm0, %xmm1
+; X64-SSE2-NEXT:    por %xmm2, %xmm1
+; X64-SSE2-NEXT:    movdqa %xmm1, %xmm0
+; X64-SSE2-NEXT:    psrld $16, %xmm0
+; X64-SSE2-NEXT:    movdqa %xmm1, %xmm2
+; X64-SSE2-NEXT:    pcmpgtb %xmm0, %xmm2
+; X64-SSE2-NEXT:    pand %xmm2, %xmm1
+; X64-SSE2-NEXT:    pandn %xmm0, %xmm2
+; X64-SSE2-NEXT:    por %xmm1, %xmm2
+; X64-SSE2-NEXT:    movdqa %xmm2, %xmm0
+; X64-SSE2-NEXT:    psrlw $8, %xmm0
+; X64-SSE2-NEXT:    movdqa %xmm2, %xmm1
+; X64-SSE2-NEXT:    pcmpgtb %xmm0, %xmm1
+; X64-SSE2-NEXT:    pand %xmm1, %xmm2
+; X64-SSE2-NEXT:    pandn %xmm0, %xmm1
+; X64-SSE2-NEXT:    por %xmm2, %xmm1
+; X64-SSE2-NEXT:    movd %xmm1, %eax
+; X64-SSE2-NEXT:    ## kill: def $al killed $al killed $eax
+; X64-SSE2-NEXT:    retq
+;
+; X64-SSE42-LABEL: test_reduce_v32i8_v16i8:
+; X64-SSE42:       ## %bb.0:
+; X64-SSE42-NEXT:    pxor {{.*}}(%rip), %xmm0
+; X64-SSE42-NEXT:    movdqa %xmm0, %xmm1
+; X64-SSE42-NEXT:    psrlw $8, %xmm1
+; X64-SSE42-NEXT:    pminub %xmm0, %xmm1
+; X64-SSE42-NEXT:    phminposuw %xmm1, %xmm0
+; X64-SSE42-NEXT:    pextrb $0, %xmm0, %eax
+; X64-SSE42-NEXT:    xorb $127, %al
+; X64-SSE42-NEXT:    ## kill: def $al killed $al killed $eax
+; X64-SSE42-NEXT:    retq
+;
+; X64-AVX-LABEL: test_reduce_v32i8_v16i8:
+; X64-AVX:       ## %bb.0:
+; X64-AVX-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; X64-AVX-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; X64-AVX-NEXT:    vpminub %xmm1, %xmm0, %xmm0
+; X64-AVX-NEXT:    vphminposuw %xmm0, %xmm0
+; X64-AVX-NEXT:    vpextrb $0, %xmm0, %eax
+; X64-AVX-NEXT:    xorb $127, %al
+; X64-AVX-NEXT:    ## kill: def $al killed $al killed $eax
+; X64-AVX-NEXT:    vzeroupper
+; X64-AVX-NEXT:    retq
+  %1  = shufflevector <32 x i8> %a0, <32 x i8> undef, <32 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %2  = icmp sgt <32 x i8> %a0, %1
+  %3  = select <32 x i1> %2, <32 x i8> %a0, <32 x i8> %1
+  %4  = shufflevector <32 x i8> %3, <32 x i8> undef, <32 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %5  = icmp sgt <32 x i8> %3, %4
+  %6  = select <32 x i1> %5, <32 x i8> %3, <32 x i8> %4
+  %7  = shufflevector <32 x i8> %6, <32 x i8> undef, <32 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %8  = icmp sgt <32 x i8> %6, %7
+  %9  = select <32 x i1> %8, <32 x i8> %6, <32 x i8> %7
+  %10 = shufflevector <32 x i8> %9, <32 x i8> undef, <32 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %11 = icmp sgt <32 x i8> %9, %10
+  %12 = select <32 x i1> %11, <32 x i8> %9, <32 x i8> %10
+  %13 = extractelement <32 x i8> %12, i32 0
+  ret i8 %13
+}
+
+define i8 @test_reduce_v64i8_v16i8(<64 x i8> %a0) {
+; X86-SSE2-LABEL: test_reduce_v64i8_v16i8:
+; X86-SSE2:       ## %bb.0:
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE2-NEXT:    movdqa %xmm0, %xmm2
+; X86-SSE2-NEXT:    pcmpgtb %xmm1, %xmm2
+; X86-SSE2-NEXT:    pand %xmm2, %xmm0
+; X86-SSE2-NEXT:    pandn %xmm1, %xmm2
+; X86-SSE2-NEXT:    por %xmm0, %xmm2
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
+; X86-SSE2-NEXT:    movdqa %xmm2, %xmm1
+; X86-SSE2-NEXT:    pcmpgtb %xmm0, %xmm1
+; X86-SSE2-NEXT:    pand %xmm1, %xmm2
+; X86-SSE2-NEXT:    pandn %xmm0, %xmm1
+; X86-SSE2-NEXT:    por %xmm2, %xmm1
+; X86-SSE2-NEXT:    movdqa %xmm1, %xmm0
+; X86-SSE2-NEXT:    psrld $16, %xmm0
+; X86-SSE2-NEXT:    movdqa %xmm1, %xmm2
+; X86-SSE2-NEXT:    pcmpgtb %xmm0, %xmm2
+; X86-SSE2-NEXT:    pand %xmm2, %xmm1
+; X86-SSE2-NEXT:    pandn %xmm0, %xmm2
+; X86-SSE2-NEXT:    por %xmm1, %xmm2
+; X86-SSE2-NEXT:    movdqa %xmm2, %xmm0
+; X86-SSE2-NEXT:    psrlw $8, %xmm0
+; X86-SSE2-NEXT:    movdqa %xmm2, %xmm1
+; X86-SSE2-NEXT:    pcmpgtb %xmm0, %xmm1
+; X86-SSE2-NEXT:    pand %xmm1, %xmm2
+; X86-SSE2-NEXT:    pandn %xmm0, %xmm1
+; X86-SSE2-NEXT:    por %xmm2, %xmm1
+; X86-SSE2-NEXT:    movd %xmm1, %eax
+; X86-SSE2-NEXT:    ## kill: def $al killed $al killed $eax
+; X86-SSE2-NEXT:    retl
+;
+; X86-SSE42-LABEL: test_reduce_v64i8_v16i8:
+; X86-SSE42:       ## %bb.0:
+; X86-SSE42-NEXT:    pxor LCPI15_0, %xmm0
+; X86-SSE42-NEXT:    movdqa %xmm0, %xmm1
+; X86-SSE42-NEXT:    psrlw $8, %xmm1
+; X86-SSE42-NEXT:    pminub %xmm0, %xmm1
+; X86-SSE42-NEXT:    phminposuw %xmm1, %xmm0
+; X86-SSE42-NEXT:    pextrb $0, %xmm0, %eax
+; X86-SSE42-NEXT:    xorb $127, %al
+; X86-SSE42-NEXT:    ## kill: def $al killed $al killed $eax
+; X86-SSE42-NEXT:    retl
+;
+; X86-AVX-LABEL: test_reduce_v64i8_v16i8:
+; X86-AVX:       ## %bb.0:
+; X86-AVX-NEXT:    vpxor LCPI15_0, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; X86-AVX-NEXT:    vpminub %xmm1, %xmm0, %xmm0
+; X86-AVX-NEXT:    vphminposuw %xmm0, %xmm0
+; X86-AVX-NEXT:    vpextrb $0, %xmm0, %eax
+; X86-AVX-NEXT:    xorb $127, %al
+; X86-AVX-NEXT:    ## kill: def $al killed $al killed $eax
+; X86-AVX-NEXT:    vzeroupper
+; X86-AVX-NEXT:    retl
+;
+; X64-SSE2-LABEL: test_reduce_v64i8_v16i8:
+; X64-SSE2:       ## %bb.0:
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-SSE2-NEXT:    movdqa %xmm0, %xmm2
+; X64-SSE2-NEXT:    pcmpgtb %xmm1, %xmm2
+; X64-SSE2-NEXT:    pand %xmm2, %xmm0
+; X64-SSE2-NEXT:    pandn %xmm1, %xmm2
+; X64-SSE2-NEXT:    por %xmm0, %xmm2
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
+; X64-SSE2-NEXT:    movdqa %xmm2, %xmm1
+; X64-SSE2-NEXT:    pcmpgtb %xmm0, %xmm1
+; X64-SSE2-NEXT:    pand %xmm1, %xmm2
+; X64-SSE2-NEXT:    pandn %xmm0, %xmm1
+; X64-SSE2-NEXT:    por %xmm2, %xmm1
+; X64-SSE2-NEXT:    movdqa %xmm1, %xmm0
+; X64-SSE2-NEXT:    psrld $16, %xmm0
+; X64-SSE2-NEXT:    movdqa %xmm1, %xmm2
+; X64-SSE2-NEXT:    pcmpgtb %xmm0, %xmm2
+; X64-SSE2-NEXT:    pand %xmm2, %xmm1
+; X64-SSE2-NEXT:    pandn %xmm0, %xmm2
+; X64-SSE2-NEXT:    por %xmm1, %xmm2
+; X64-SSE2-NEXT:    movdqa %xmm2, %xmm0
+; X64-SSE2-NEXT:    psrlw $8, %xmm0
+; X64-SSE2-NEXT:    movdqa %xmm2, %xmm1
+; X64-SSE2-NEXT:    pcmpgtb %xmm0, %xmm1
+; X64-SSE2-NEXT:    pand %xmm1, %xmm2
+; X64-SSE2-NEXT:    pandn %xmm0, %xmm1
+; X64-SSE2-NEXT:    por %xmm2, %xmm1
+; X64-SSE2-NEXT:    movd %xmm1, %eax
+; X64-SSE2-NEXT:    ## kill: def $al killed $al killed $eax
+; X64-SSE2-NEXT:    retq
+;
+; X64-SSE42-LABEL: test_reduce_v64i8_v16i8:
+; X64-SSE42:       ## %bb.0:
+; X64-SSE42-NEXT:    pxor {{.*}}(%rip), %xmm0
+; X64-SSE42-NEXT:    movdqa %xmm0, %xmm1
+; X64-SSE42-NEXT:    psrlw $8, %xmm1
+; X64-SSE42-NEXT:    pminub %xmm0, %xmm1
+; X64-SSE42-NEXT:    phminposuw %xmm1, %xmm0
+; X64-SSE42-NEXT:    pextrb $0, %xmm0, %eax
+; X64-SSE42-NEXT:    xorb $127, %al
+; X64-SSE42-NEXT:    ## kill: def $al killed $al killed $eax
+; X64-SSE42-NEXT:    retq
+;
+; X64-AVX-LABEL: test_reduce_v64i8_v16i8:
+; X64-AVX:       ## %bb.0:
+; X64-AVX-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; X64-AVX-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; X64-AVX-NEXT:    vpminub %xmm1, %xmm0, %xmm0
+; X64-AVX-NEXT:    vphminposuw %xmm0, %xmm0
+; X64-AVX-NEXT:    vpextrb $0, %xmm0, %eax
+; X64-AVX-NEXT:    xorb $127, %al
+; X64-AVX-NEXT:    ## kill: def $al killed $al killed $eax
+; X64-AVX-NEXT:    vzeroupper
+; X64-AVX-NEXT:    retq
+  %1  = shufflevector <64 x i8> %a0, <64 x i8> undef, <64 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %2  = icmp sgt <64 x i8> %a0, %1
+  %3  = select <64 x i1> %2, <64 x i8> %a0, <64 x i8> %1
+  %4  = shufflevector <64 x i8> %3, <64 x i8> undef, <64 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %5  = icmp sgt <64 x i8> %3, %4
+  %6  = select <64 x i1> %5, <64 x i8> %3, <64 x i8> %4
+  %7  = shufflevector <64 x i8> %6, <64 x i8> undef, <64 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %8  = icmp sgt <64 x i8> %6, %7
+  %9  = select <64 x i1> %8, <64 x i8> %6, <64 x i8> %7
+  %10 = shufflevector <64 x i8> %9, <64 x i8> undef, <64 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %11 = icmp sgt <64 x i8> %9, %10
+  %12 = select <64 x i1> %11, <64 x i8> %9, <64 x i8> %10
+  %13 = extractelement <64 x i8> %12, i32 0
+  ret i8 %13
+}
diff --git a/llvm/test/CodeGen/X86/horizontal-reduce-smin.ll b/llvm/test/CodeGen/X86/horizontal-reduce-smin.ll
index c526cb8f9..df7aaf640 100644
--- a/llvm/test/CodeGen/X86/horizontal-reduce-smin.ll
+++ b/llvm/test/CodeGen/X86/horizontal-reduce-smin.ll
@@ -1833,3 +1833,417 @@ define i8 @test_reduce_v64i8(<64 x i8> %a0) {
   %19 = extractelement <64 x i8> %18, i32 0
   ret i8 %19
 }
+
+;
+; Partial Vector Reductions
+;
+
+define i16 @test_reduce_v16i16_v8i16(<16 x i16> %a0) {
+; X86-SSE2-LABEL: test_reduce_v16i16_v8i16:
+; X86-SSE2:       ## %bb.0:
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE2-NEXT:    pminsw %xmm0, %xmm1
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X86-SSE2-NEXT:    pminsw %xmm1, %xmm0
+; X86-SSE2-NEXT:    movdqa %xmm0, %xmm1
+; X86-SSE2-NEXT:    psrld $16, %xmm1
+; X86-SSE2-NEXT:    pminsw %xmm0, %xmm1
+; X86-SSE2-NEXT:    movd %xmm1, %eax
+; X86-SSE2-NEXT:    ## kill: def $ax killed $ax killed $eax
+; X86-SSE2-NEXT:    retl
+;
+; X86-SSE42-LABEL: test_reduce_v16i16_v8i16:
+; X86-SSE42:       ## %bb.0:
+; X86-SSE42-NEXT:    pxor LCPI12_0, %xmm0
+; X86-SSE42-NEXT:    phminposuw %xmm0, %xmm0
+; X86-SSE42-NEXT:    movd %xmm0, %eax
+; X86-SSE42-NEXT:    xorl $32768, %eax ## imm = 0x8000
+; X86-SSE42-NEXT:    ## kill: def $ax killed $ax killed $eax
+; X86-SSE42-NEXT:    retl
+;
+; X86-AVX-LABEL: test_reduce_v16i16_v8i16:
+; X86-AVX:       ## %bb.0:
+; X86-AVX-NEXT:    vpxor LCPI12_0, %xmm0, %xmm0
+; X86-AVX-NEXT:    vphminposuw %xmm0, %xmm0
+; X86-AVX-NEXT:    vmovd %xmm0, %eax
+; X86-AVX-NEXT:    xorl $32768, %eax ## imm = 0x8000
+; X86-AVX-NEXT:    ## kill: def $ax killed $ax killed $eax
+; X86-AVX-NEXT:    vzeroupper
+; X86-AVX-NEXT:    retl
+;
+; X64-SSE2-LABEL: test_reduce_v16i16_v8i16:
+; X64-SSE2:       ## %bb.0:
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-SSE2-NEXT:    pminsw %xmm0, %xmm1
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X64-SSE2-NEXT:    pminsw %xmm1, %xmm0
+; X64-SSE2-NEXT:    movdqa %xmm0, %xmm1
+; X64-SSE2-NEXT:    psrld $16, %xmm1
+; X64-SSE2-NEXT:    pminsw %xmm0, %xmm1
+; X64-SSE2-NEXT:    movd %xmm1, %eax
+; X64-SSE2-NEXT:    ## kill: def $ax killed $ax killed $eax
+; X64-SSE2-NEXT:    retq
+;
+; X64-SSE42-LABEL: test_reduce_v16i16_v8i16:
+; X64-SSE42:       ## %bb.0:
+; X64-SSE42-NEXT:    pxor {{.*}}(%rip), %xmm0
+; X64-SSE42-NEXT:    phminposuw %xmm0, %xmm0
+; X64-SSE42-NEXT:    movd %xmm0, %eax
+; X64-SSE42-NEXT:    xorl $32768, %eax ## imm = 0x8000
+; X64-SSE42-NEXT:    ## kill: def $ax killed $ax killed $eax
+; X64-SSE42-NEXT:    retq
+;
+; X64-AVX-LABEL: test_reduce_v16i16_v8i16:
+; X64-AVX:       ## %bb.0:
+; X64-AVX-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; X64-AVX-NEXT:    vphminposuw %xmm0, %xmm0
+; X64-AVX-NEXT:    vmovd %xmm0, %eax
+; X64-AVX-NEXT:    xorl $32768, %eax ## imm = 0x8000
+; X64-AVX-NEXT:    ## kill: def $ax killed $ax killed $eax
+; X64-AVX-NEXT:    vzeroupper
+; X64-AVX-NEXT:    retq
+  %1  = shufflevector <16 x i16> %a0, <16 x i16> undef, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %2  = icmp slt <16 x i16> %a0, %1
+  %3  = select <16 x i1> %2, <16 x i16> %a0, <16 x i16> %1
+  %4  = shufflevector <16 x i16> %3, <16 x i16> undef, <16 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %5  = icmp slt <16 x i16> %3, %4
+  %6  = select <16 x i1> %5, <16 x i16> %3, <16 x i16> %4
+  %7  = shufflevector <16 x i16> %6, <16 x i16> undef, <16 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %8  = icmp slt <16 x i16> %6, %7
+  %9  = select <16 x i1> %8, <16 x i16> %6, <16 x i16> %7
+  %10 = extractelement <16 x i16> %9, i32 0
+  ret i16 %10
+}
+
+define i16 @test_reduce_v32i16_v8i16(<32 x i16> %a0) {
+; X86-SSE2-LABEL: test_reduce_v32i16_v8i16:
+; X86-SSE2:       ## %bb.0:
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE2-NEXT:    pminsw %xmm0, %xmm1
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X86-SSE2-NEXT:    pminsw %xmm1, %xmm0
+; X86-SSE2-NEXT:    movdqa %xmm0, %xmm1
+; X86-SSE2-NEXT:    psrld $16, %xmm1
+; X86-SSE2-NEXT:    pminsw %xmm0, %xmm1
+; X86-SSE2-NEXT:    movd %xmm1, %eax
+; X86-SSE2-NEXT:    ## kill: def $ax killed $ax killed $eax
+; X86-SSE2-NEXT:    retl
+;
+; X86-SSE42-LABEL: test_reduce_v32i16_v8i16:
+; X86-SSE42:       ## %bb.0:
+; X86-SSE42-NEXT:    pxor LCPI13_0, %xmm0
+; X86-SSE42-NEXT:    phminposuw %xmm0, %xmm0
+; X86-SSE42-NEXT:    movd %xmm0, %eax
+; X86-SSE42-NEXT:    xorl $32768, %eax ## imm = 0x8000
+; X86-SSE42-NEXT:    ## kill: def $ax killed $ax killed $eax
+; X86-SSE42-NEXT:    retl
+;
+; X86-AVX-LABEL: test_reduce_v32i16_v8i16:
+; X86-AVX:       ## %bb.0:
+; X86-AVX-NEXT:    vpxor LCPI13_0, %xmm0, %xmm0
+; X86-AVX-NEXT:    vphminposuw %xmm0, %xmm0
+; X86-AVX-NEXT:    vmovd %xmm0, %eax
+; X86-AVX-NEXT:    xorl $32768, %eax ## imm = 0x8000
+; X86-AVX-NEXT:    ## kill: def $ax killed $ax killed $eax
+; X86-AVX-NEXT:    vzeroupper
+; X86-AVX-NEXT:    retl
+;
+; X64-SSE2-LABEL: test_reduce_v32i16_v8i16:
+; X64-SSE2:       ## %bb.0:
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-SSE2-NEXT:    pminsw %xmm0, %xmm1
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X64-SSE2-NEXT:    pminsw %xmm1, %xmm0
+; X64-SSE2-NEXT:    movdqa %xmm0, %xmm1
+; X64-SSE2-NEXT:    psrld $16, %xmm1
+; X64-SSE2-NEXT:    pminsw %xmm0, %xmm1
+; X64-SSE2-NEXT:    movd %xmm1, %eax
+; X64-SSE2-NEXT:    ## kill: def $ax killed $ax killed $eax
+; X64-SSE2-NEXT:    retq
+;
+; X64-SSE42-LABEL: test_reduce_v32i16_v8i16:
+; X64-SSE42:       ## %bb.0:
+; X64-SSE42-NEXT:    pxor {{.*}}(%rip), %xmm0
+; X64-SSE42-NEXT:    phminposuw %xmm0, %xmm0
+; X64-SSE42-NEXT:    movd %xmm0, %eax
+; X64-SSE42-NEXT:    xorl $32768, %eax ## imm = 0x8000
+; X64-SSE42-NEXT:    ## kill: def $ax killed $ax killed $eax
+; X64-SSE42-NEXT:    retq
+;
+; X64-AVX-LABEL: test_reduce_v32i16_v8i16:
+; X64-AVX:       ## %bb.0:
+; X64-AVX-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; X64-AVX-NEXT:    vphminposuw %xmm0, %xmm0
+; X64-AVX-NEXT:    vmovd %xmm0, %eax
+; X64-AVX-NEXT:    xorl $32768, %eax ## imm = 0x8000
+; X64-AVX-NEXT:    ## kill: def $ax killed $ax killed $eax
+; X64-AVX-NEXT:    vzeroupper
+; X64-AVX-NEXT:    retq
+  %1  = shufflevector <32 x i16> %a0, <32 x i16> undef, <32 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %2  = icmp slt <32 x i16> %a0, %1
+  %3  = select <32 x i1> %2, <32 x i16> %a0, <32 x i16> %1
+  %4  = shufflevector <32 x i16> %3, <32 x i16> undef, <32 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %5  = icmp slt <32 x i16> %3, %4
+  %6  = select <32 x i1> %5, <32 x i16> %3, <32 x i16> %4
+  %7  = shufflevector <32 x i16> %6, <32 x i16> undef, <32 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %8  = icmp slt <32 x i16> %6, %7
+  %9  = select <32 x i1> %8, <32 x i16> %6, <32 x i16> %7
+  %10 = extractelement <32 x i16> %9, i32 0
+  ret i16 %10
+}
+
+define i8 @test_reduce_v32i8_v16i8(<32 x i8> %a0) {
+; X86-SSE2-LABEL: test_reduce_v32i8_v16i8:
+; X86-SSE2:       ## %bb.0:
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE2-NEXT:    movdqa %xmm1, %xmm2
+; X86-SSE2-NEXT:    pcmpgtb %xmm0, %xmm2
+; X86-SSE2-NEXT:    pand %xmm2, %xmm0
+; X86-SSE2-NEXT:    pandn %xmm1, %xmm2
+; X86-SSE2-NEXT:    por %xmm0, %xmm2
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
+; X86-SSE2-NEXT:    movdqa %xmm0, %xmm1
+; X86-SSE2-NEXT:    pcmpgtb %xmm2, %xmm1
+; X86-SSE2-NEXT:    pand %xmm1, %xmm2
+; X86-SSE2-NEXT:    pandn %xmm0, %xmm1
+; X86-SSE2-NEXT:    por %xmm2, %xmm1
+; X86-SSE2-NEXT:    movdqa %xmm1, %xmm0
+; X86-SSE2-NEXT:    psrld $16, %xmm0
+; X86-SSE2-NEXT:    movdqa %xmm0, %xmm2
+; X86-SSE2-NEXT:    pcmpgtb %xmm1, %xmm2
+; X86-SSE2-NEXT:    pand %xmm2, %xmm1
+; X86-SSE2-NEXT:    pandn %xmm0, %xmm2
+; X86-SSE2-NEXT:    por %xmm1, %xmm2
+; X86-SSE2-NEXT:    movdqa %xmm2, %xmm0
+; X86-SSE2-NEXT:    psrlw $8, %xmm0
+; X86-SSE2-NEXT:    movdqa %xmm0, %xmm1
+; X86-SSE2-NEXT:    pcmpgtb %xmm2, %xmm1
+; X86-SSE2-NEXT:    pand %xmm1, %xmm2
+; X86-SSE2-NEXT:    pandn %xmm0, %xmm1
+; X86-SSE2-NEXT:    por %xmm2, %xmm1
+; X86-SSE2-NEXT:    movd %xmm1, %eax
+; X86-SSE2-NEXT:    ## kill: def $al killed $al killed $eax
+; X86-SSE2-NEXT:    retl
+;
+; X86-SSE42-LABEL: test_reduce_v32i8_v16i8:
+; X86-SSE42:       ## %bb.0:
+; X86-SSE42-NEXT:    pxor LCPI14_0, %xmm0
+; X86-SSE42-NEXT:    movdqa %xmm0, %xmm1
+; X86-SSE42-NEXT:    psrlw $8, %xmm1
+; X86-SSE42-NEXT:    pminub %xmm0, %xmm1
+; X86-SSE42-NEXT:    phminposuw %xmm1, %xmm0
+; X86-SSE42-NEXT:    pextrb $0, %xmm0, %eax
+; X86-SSE42-NEXT:    xorb $-128, %al
+; X86-SSE42-NEXT:    ## kill: def $al killed $al killed $eax
+; X86-SSE42-NEXT:    retl
+;
+; X86-AVX-LABEL: test_reduce_v32i8_v16i8:
+; X86-AVX:       ## %bb.0:
+; X86-AVX-NEXT:    vpxor LCPI14_0, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; X86-AVX-NEXT:    vpminub %xmm1, %xmm0, %xmm0
+; X86-AVX-NEXT:    vphminposuw %xmm0, %xmm0
+; X86-AVX-NEXT:    vpextrb $0, %xmm0, %eax
+; X86-AVX-NEXT:    xorb $-128, %al
+; X86-AVX-NEXT:    ## kill: def $al killed $al killed $eax
+; X86-AVX-NEXT:    vzeroupper
+; X86-AVX-NEXT:    retl
+;
+; X64-SSE2-LABEL: test_reduce_v32i8_v16i8:
+; X64-SSE2:       ## %bb.0:
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-SSE2-NEXT:    movdqa %xmm1, %xmm2
+; X64-SSE2-NEXT:    pcmpgtb %xmm0, %xmm2
+; X64-SSE2-NEXT:    pand %xmm2, %xmm0
+; X64-SSE2-NEXT:    pandn %xmm1, %xmm2
+; X64-SSE2-NEXT:    por %xmm0, %xmm2
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
+; X64-SSE2-NEXT:    movdqa %xmm0, %xmm1
+; X64-SSE2-NEXT:    pcmpgtb %xmm2, %xmm1
+; X64-SSE2-NEXT:    pand %xmm1, %xmm2
+; X64-SSE2-NEXT:    pandn %xmm0, %xmm1
+; X64-SSE2-NEXT:    por %xmm2, %xmm1
+; X64-SSE2-NEXT:    movdqa %xmm1, %xmm0
+; X64-SSE2-NEXT:    psrld $16, %xmm0
+; X64-SSE2-NEXT:    movdqa %xmm0, %xmm2
+; X64-SSE2-NEXT:    pcmpgtb %xmm1, %xmm2
+; X64-SSE2-NEXT:    pand %xmm2, %xmm1
+; X64-SSE2-NEXT:    pandn %xmm0, %xmm2
+; X64-SSE2-NEXT:    por %xmm1, %xmm2
+; X64-SSE2-NEXT:    movdqa %xmm2, %xmm0
+; X64-SSE2-NEXT:    psrlw $8, %xmm0
+; X64-SSE2-NEXT:    movdqa %xmm0, %xmm1
+; X64-SSE2-NEXT:    pcmpgtb %xmm2, %xmm1
+; X64-SSE2-NEXT:    pand %xmm1, %xmm2
+; X64-SSE2-NEXT:    pandn %xmm0, %xmm1
+; X64-SSE2-NEXT:    por %xmm2, %xmm1
+; X64-SSE2-NEXT:    movd %xmm1, %eax
+; X64-SSE2-NEXT:    ## kill: def $al killed $al killed $eax
+; X64-SSE2-NEXT:    retq
+;
+; X64-SSE42-LABEL: test_reduce_v32i8_v16i8:
+; X64-SSE42:       ## %bb.0:
+; X64-SSE42-NEXT:    pxor {{.*}}(%rip), %xmm0
+; X64-SSE42-NEXT:    movdqa %xmm0, %xmm1
+; X64-SSE42-NEXT:    psrlw $8, %xmm1
+; X64-SSE42-NEXT:    pminub %xmm0, %xmm1
+; X64-SSE42-NEXT:    phminposuw %xmm1, %xmm0
+; X64-SSE42-NEXT:    pextrb $0, %xmm0, %eax
+; X64-SSE42-NEXT:    xorb $-128, %al
+; X64-SSE42-NEXT:    ## kill: def $al killed $al killed $eax
+; X64-SSE42-NEXT:    retq
+;
+; X64-AVX-LABEL: test_reduce_v32i8_v16i8:
+; X64-AVX:       ## %bb.0:
+; X64-AVX-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; X64-AVX-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; X64-AVX-NEXT:    vpminub %xmm1, %xmm0, %xmm0
+; X64-AVX-NEXT:    vphminposuw %xmm0, %xmm0
+; X64-AVX-NEXT:    vpextrb $0, %xmm0, %eax
+; X64-AVX-NEXT:    xorb $-128, %al
+; X64-AVX-NEXT:    ## kill: def $al killed $al killed $eax
+; X64-AVX-NEXT:    vzeroupper
+; X64-AVX-NEXT:    retq
+  %1  = shufflevector <32 x i8> %a0, <32 x i8> undef, <32 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %2  = icmp slt <32 x i8> %a0, %1
+  %3  = select <32 x i1> %2, <32 x i8> %a0, <32 x i8> %1
+  %4  = shufflevector <32 x i8> %3, <32 x i8> undef, <32 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %5  = icmp slt <32 x i8> %3, %4
+  %6  = select <32 x i1> %5, <32 x i8> %3, <32 x i8> %4
+  %7  = shufflevector <32 x i8> %6, <32 x i8> undef, <32 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %8  = icmp slt <32 x i8> %6, %7
+  %9  = select <32 x i1> %8, <32 x i8> %6, <32 x i8> %7
+  %10 = shufflevector <32 x i8> %9, <32 x i8> undef, <32 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %11 = icmp slt <32 x i8> %9, %10
+  %12 = select <32 x i1> %11, <32 x i8> %9, <32 x i8> %10
+  %13 = extractelement <32 x i8> %12, i32 0
+  ret i8 %13
+}
+
+define i8 @test_reduce_v64i8_v16i8(<64 x i8> %a0) {
+; X86-SSE2-LABEL: test_reduce_v64i8_v16i8:
+; X86-SSE2:       ## %bb.0:
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE2-NEXT:    movdqa %xmm1, %xmm2
+; X86-SSE2-NEXT:    pcmpgtb %xmm0, %xmm2
+; X86-SSE2-NEXT:    pand %xmm2, %xmm0
+; X86-SSE2-NEXT:    pandn %xmm1, %xmm2
+; X86-SSE2-NEXT:    por %xmm0, %xmm2
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
+; X86-SSE2-NEXT:    movdqa %xmm0, %xmm1
+; X86-SSE2-NEXT:    pcmpgtb %xmm2, %xmm1
+; X86-SSE2-NEXT:    pand %xmm1, %xmm2
+; X86-SSE2-NEXT:    pandn %xmm0, %xmm1
+; X86-SSE2-NEXT:    por %xmm2, %xmm1
+; X86-SSE2-NEXT:    movdqa %xmm1, %xmm0
+; X86-SSE2-NEXT:    psrld $16, %xmm0
+; X86-SSE2-NEXT:    movdqa %xmm0, %xmm2
+; X86-SSE2-NEXT:    pcmpgtb %xmm1, %xmm2
+; X86-SSE2-NEXT:    pand %xmm2, %xmm1
+; X86-SSE2-NEXT:    pandn %xmm0, %xmm2
+; X86-SSE2-NEXT:    por %xmm1, %xmm2
+; X86-SSE2-NEXT:    movdqa %xmm2, %xmm0
+; X86-SSE2-NEXT:    psrlw $8, %xmm0
+; X86-SSE2-NEXT:    movdqa %xmm0, %xmm1
+; X86-SSE2-NEXT:    pcmpgtb %xmm2, %xmm1
+; X86-SSE2-NEXT:    pand %xmm1, %xmm2
+; X86-SSE2-NEXT:    pandn %xmm0, %xmm1
+; X86-SSE2-NEXT:    por %xmm2, %xmm1
+; X86-SSE2-NEXT:    movd %xmm1, %eax
+; X86-SSE2-NEXT:    ## kill: def $al killed $al killed $eax
+; X86-SSE2-NEXT:    retl
+;
+; X86-SSE42-LABEL: test_reduce_v64i8_v16i8:
+; X86-SSE42:       ## %bb.0:
+; X86-SSE42-NEXT:    pxor LCPI15_0, %xmm0
+; X86-SSE42-NEXT:    movdqa %xmm0, %xmm1
+; X86-SSE42-NEXT:    psrlw $8, %xmm1
+; X86-SSE42-NEXT:    pminub %xmm0, %xmm1
+; X86-SSE42-NEXT:    phminposuw %xmm1, %xmm0
+; X86-SSE42-NEXT:    pextrb $0, %xmm0, %eax
+; X86-SSE42-NEXT:    xorb $-128, %al
+; X86-SSE42-NEXT:    ## kill: def $al killed $al killed $eax
+; X86-SSE42-NEXT:    retl
+;
+; X86-AVX-LABEL: test_reduce_v64i8_v16i8:
+; X86-AVX:       ## %bb.0:
+; X86-AVX-NEXT:    vpxor LCPI15_0, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; X86-AVX-NEXT:    vpminub %xmm1, %xmm0, %xmm0
+; X86-AVX-NEXT:    vphminposuw %xmm0, %xmm0
+; X86-AVX-NEXT:    vpextrb $0, %xmm0, %eax
+; X86-AVX-NEXT:    xorb $-128, %al
+; X86-AVX-NEXT:    ## kill: def $al killed $al killed $eax
+; X86-AVX-NEXT:    vzeroupper
+; X86-AVX-NEXT:    retl
+;
+; X64-SSE2-LABEL: test_reduce_v64i8_v16i8:
+; X64-SSE2:       ## %bb.0:
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-SSE2-NEXT:    movdqa %xmm1, %xmm2
+; X64-SSE2-NEXT:    pcmpgtb %xmm0, %xmm2
+; X64-SSE2-NEXT:    pand %xmm2, %xmm0
+; X64-SSE2-NEXT:    pandn %xmm1, %xmm2
+; X64-SSE2-NEXT:    por %xmm0, %xmm2
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
+; X64-SSE2-NEXT:    movdqa %xmm0, %xmm1
+; X64-SSE2-NEXT:    pcmpgtb %xmm2, %xmm1
+; X64-SSE2-NEXT:    pand %xmm1, %xmm2
+; X64-SSE2-NEXT:    pandn %xmm0, %xmm1
+; X64-SSE2-NEXT:    por %xmm2, %xmm1
+; X64-SSE2-NEXT:    movdqa %xmm1, %xmm0
+; X64-SSE2-NEXT:    psrld $16, %xmm0
+; X64-SSE2-NEXT:    movdqa %xmm0, %xmm2
+; X64-SSE2-NEXT:    pcmpgtb %xmm1, %xmm2
+; X64-SSE2-NEXT:    pand %xmm2, %xmm1
+; X64-SSE2-NEXT:    pandn %xmm0, %xmm2
+; X64-SSE2-NEXT:    por %xmm1, %xmm2
+; X64-SSE2-NEXT:    movdqa %xmm2, %xmm0
+; X64-SSE2-NEXT:    psrlw $8, %xmm0
+; X64-SSE2-NEXT:    movdqa %xmm0, %xmm1
+; X64-SSE2-NEXT:    pcmpgtb %xmm2, %xmm1
+; X64-SSE2-NEXT:    pand %xmm1, %xmm2
+; X64-SSE2-NEXT:    pandn %xmm0, %xmm1
+; X64-SSE2-NEXT:    por %xmm2, %xmm1
+; X64-SSE2-NEXT:    movd %xmm1, %eax
+; X64-SSE2-NEXT:    ## kill: def $al killed $al killed $eax
+; X64-SSE2-NEXT:    retq
+;
+; X64-SSE42-LABEL: test_reduce_v64i8_v16i8:
+; X64-SSE42:       ## %bb.0:
+; X64-SSE42-NEXT:    pxor {{.*}}(%rip), %xmm0
+; X64-SSE42-NEXT:    movdqa %xmm0, %xmm1
+; X64-SSE42-NEXT:    psrlw $8, %xmm1
+; X64-SSE42-NEXT:    pminub %xmm0, %xmm1
+; X64-SSE42-NEXT:    phminposuw %xmm1, %xmm0
+; X64-SSE42-NEXT:    pextrb $0, %xmm0, %eax
+; X64-SSE42-NEXT:    xorb $-128, %al
+; X64-SSE42-NEXT:    ## kill: def $al killed $al killed $eax
+; X64-SSE42-NEXT:    retq
+;
+; X64-AVX-LABEL: test_reduce_v64i8_v16i8:
+; X64-AVX:       ## %bb.0:
+; X64-AVX-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; X64-AVX-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; X64-AVX-NEXT:    vpminub %xmm1, %xmm0, %xmm0
+; X64-AVX-NEXT:    vphminposuw %xmm0, %xmm0
+; X64-AVX-NEXT:    vpextrb $0, %xmm0, %eax
+; X64-AVX-NEXT:    xorb $-128, %al
+; X64-AVX-NEXT:    ## kill: def $al killed $al killed $eax
+; X64-AVX-NEXT:    vzeroupper
+; X64-AVX-NEXT:    retq
+  %1  = shufflevector <64 x i8> %a0, <64 x i8> undef, <64 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %2  = icmp slt <64 x i8> %a0, %1
+  %3  = select <64 x i1> %2, <64 x i8> %a0, <64 x i8> %1
+  %4  = shufflevector <64 x i8> %3, <64 x i8> undef, <64 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %5  = icmp slt <64 x i8> %3, %4
+  %6  = select <64 x i1> %5, <64 x i8> %3, <64 x i8> %4
+  %7  = shufflevector <64 x i8> %6, <64 x i8> undef, <64 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %8  = icmp slt <64 x i8> %6, %7
+  %9  = select <64 x i1> %8, <64 x i8> %6, <64 x i8> %7
+  %10 = shufflevector <64 x i8> %9, <64 x i8> undef, <64 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %11 = icmp slt <64 x i8> %9, %10
+  %12 = select <64 x i1> %11, <64 x i8> %9, <64 x i8> %10
+  %13 = extractelement <64 x i8> %12, i32 0
+  ret i8 %13
+}
diff --git a/llvm/test/CodeGen/X86/horizontal-reduce-umax.ll b/llvm/test/CodeGen/X86/horizontal-reduce-umax.ll
index 99038d7f3..b26bc4c02 100644
--- a/llvm/test/CodeGen/X86/horizontal-reduce-umax.ll
+++ b/llvm/test/CodeGen/X86/horizontal-reduce-umax.ll
@@ -2021,3 +2021,497 @@ define i8 @test_reduce_v64i8(<64 x i8> %a0) {
   %19 = extractelement <64 x i8> %18, i32 0
   ret i8 %19
 }
+
+;
+; Partial Vector Reductions
+;
+
+define i16 @test_reduce_v16i16_v8i16(<16 x i16> %a0) {
+; X86-SSE2-LABEL: test_reduce_v16i16_v8i16:
+; X86-SSE2:       ## %bb.0:
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [32768,32768,32768,32768,32768,32768,32768,32768]
+; X86-SSE2-NEXT:    pxor %xmm2, %xmm0
+; X86-SSE2-NEXT:    pxor %xmm2, %xmm1
+; X86-SSE2-NEXT:    pmaxsw %xmm0, %xmm1
+; X86-SSE2-NEXT:    movdqa %xmm1, %xmm0
+; X86-SSE2-NEXT:    pxor %xmm2, %xmm0
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; X86-SSE2-NEXT:    pxor %xmm2, %xmm0
+; X86-SSE2-NEXT:    pmaxsw %xmm1, %xmm0
+; X86-SSE2-NEXT:    movdqa %xmm0, %xmm1
+; X86-SSE2-NEXT:    pxor %xmm2, %xmm1
+; X86-SSE2-NEXT:    psrld $16, %xmm1
+; X86-SSE2-NEXT:    pxor %xmm2, %xmm1
+; X86-SSE2-NEXT:    pmaxsw %xmm0, %xmm1
+; X86-SSE2-NEXT:    movd %xmm1, %eax
+; X86-SSE2-NEXT:    xorl $32768, %eax ## imm = 0x8000
+; X86-SSE2-NEXT:    ## kill: def $ax killed $ax killed $eax
+; X86-SSE2-NEXT:    retl
+;
+; X86-SSE42-LABEL: test_reduce_v16i16_v8i16:
+; X86-SSE42:       ## %bb.0:
+; X86-SSE42-NEXT:    pcmpeqd %xmm1, %xmm1
+; X86-SSE42-NEXT:    pxor %xmm0, %xmm1
+; X86-SSE42-NEXT:    phminposuw %xmm1, %xmm0
+; X86-SSE42-NEXT:    movd %xmm0, %eax
+; X86-SSE42-NEXT:    notl %eax
+; X86-SSE42-NEXT:    ## kill: def $ax killed $ax killed $eax
+; X86-SSE42-NEXT:    retl
+;
+; X86-AVX-LABEL: test_reduce_v16i16_v8i16:
+; X86-AVX:       ## %bb.0:
+; X86-AVX-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
+; X86-AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; X86-AVX-NEXT:    vphminposuw %xmm0, %xmm0
+; X86-AVX-NEXT:    vmovd %xmm0, %eax
+; X86-AVX-NEXT:    notl %eax
+; X86-AVX-NEXT:    ## kill: def $ax killed $ax killed $eax
+; X86-AVX-NEXT:    vzeroupper
+; X86-AVX-NEXT:    retl
+;
+; X64-SSE2-LABEL: test_reduce_v16i16_v8i16:
+; X64-SSE2:       ## %bb.0:
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [32768,32768,32768,32768,32768,32768,32768,32768]
+; X64-SSE2-NEXT:    pxor %xmm2, %xmm0
+; X64-SSE2-NEXT:    pxor %xmm2, %xmm1
+; X64-SSE2-NEXT:    pmaxsw %xmm0, %xmm1
+; X64-SSE2-NEXT:    movdqa %xmm1, %xmm0
+; X64-SSE2-NEXT:    pxor %xmm2, %xmm0
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; X64-SSE2-NEXT:    pxor %xmm2, %xmm0
+; X64-SSE2-NEXT:    pmaxsw %xmm1, %xmm0
+; X64-SSE2-NEXT:    movdqa %xmm0, %xmm1
+; X64-SSE2-NEXT:    pxor %xmm2, %xmm1
+; X64-SSE2-NEXT:    psrld $16, %xmm1
+; X64-SSE2-NEXT:    pxor %xmm2, %xmm1
+; X64-SSE2-NEXT:    pmaxsw %xmm0, %xmm1
+; X64-SSE2-NEXT:    movd %xmm1, %eax
+; X64-SSE2-NEXT:    xorl $32768, %eax ## imm = 0x8000
+; X64-SSE2-NEXT:    ## kill: def $ax killed $ax killed $eax
+; X64-SSE2-NEXT:    retq
+;
+; X64-SSE42-LABEL: test_reduce_v16i16_v8i16:
+; X64-SSE42:       ## %bb.0:
+; X64-SSE42-NEXT:    pcmpeqd %xmm1, %xmm1
+; X64-SSE42-NEXT:    pxor %xmm0, %xmm1
+; X64-SSE42-NEXT:    phminposuw %xmm1, %xmm0
+; X64-SSE42-NEXT:    movd %xmm0, %eax
+; X64-SSE42-NEXT:    notl %eax
+; X64-SSE42-NEXT:    ## kill: def $ax killed $ax killed $eax
+; X64-SSE42-NEXT:    retq
+;
+; X64-AVX1-LABEL: test_reduce_v16i16_v8i16:
+; X64-AVX1:       ## %bb.0:
+; X64-AVX1-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
+; X64-AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT:    vphminposuw %xmm0, %xmm0
+; X64-AVX1-NEXT:    vmovd %xmm0, %eax
+; X64-AVX1-NEXT:    notl %eax
+; X64-AVX1-NEXT:    ## kill: def $ax killed $ax killed $eax
+; X64-AVX1-NEXT:    vzeroupper
+; X64-AVX1-NEXT:    retq
+;
+; X64-AVX2-LABEL: test_reduce_v16i16_v8i16:
+; X64-AVX2:       ## %bb.0:
+; X64-AVX2-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
+; X64-AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT:    vphminposuw %xmm0, %xmm0
+; X64-AVX2-NEXT:    vmovd %xmm0, %eax
+; X64-AVX2-NEXT:    notl %eax
+; X64-AVX2-NEXT:    ## kill: def $ax killed $ax killed $eax
+; X64-AVX2-NEXT:    vzeroupper
+; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512-LABEL: test_reduce_v16i16_v8i16:
+; X64-AVX512:       ## %bb.0:
+; X64-AVX512-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
+; X64-AVX512-NEXT:    vphminposuw %xmm0, %xmm0
+; X64-AVX512-NEXT:    vmovd %xmm0, %eax
+; X64-AVX512-NEXT:    notl %eax
+; X64-AVX512-NEXT:    ## kill: def $ax killed $ax killed $eax
+; X64-AVX512-NEXT:    vzeroupper
+; X64-AVX512-NEXT:    retq
+  %1  = shufflevector <16 x i16> %a0, <16 x i16> undef, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %2  = icmp ugt <16 x i16> %a0, %1
+  %3  = select <16 x i1> %2, <16 x i16> %a0, <16 x i16> %1
+  %4  = shufflevector <16 x i16> %3, <16 x i16> undef, <16 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %5  = icmp ugt <16 x i16> %3, %4
+  %6  = select <16 x i1> %5, <16 x i16> %3, <16 x i16> %4
+  %7  = shufflevector <16 x i16> %6, <16 x i16> undef, <16 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %8  = icmp ugt <16 x i16> %6, %7
+  %9  = select <16 x i1> %8, <16 x i16> %6, <16 x i16> %7
+  %10 = extractelement <16 x i16> %9, i32 0
+  ret i16 %10
+}
+
+define i16 @test_reduce_v32i16_v8i16(<32 x i16> %a0) {
+; X86-SSE2-LABEL: test_reduce_v32i16_v8i16:
+; X86-SSE2:       ## %bb.0:
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [32768,32768,32768,32768,32768,32768,32768,32768]
+; X86-SSE2-NEXT:    pxor %xmm2, %xmm0
+; X86-SSE2-NEXT:    pxor %xmm2, %xmm1
+; X86-SSE2-NEXT:    pmaxsw %xmm0, %xmm1
+; X86-SSE2-NEXT:    movdqa %xmm1, %xmm0
+; X86-SSE2-NEXT:    pxor %xmm2, %xmm0
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; X86-SSE2-NEXT:    pxor %xmm2, %xmm0
+; X86-SSE2-NEXT:    pmaxsw %xmm1, %xmm0
+; X86-SSE2-NEXT:    movdqa %xmm0, %xmm1
+; X86-SSE2-NEXT:    pxor %xmm2, %xmm1
+; X86-SSE2-NEXT:    psrld $16, %xmm1
+; X86-SSE2-NEXT:    pxor %xmm2, %xmm1
+; X86-SSE2-NEXT:    pmaxsw %xmm0, %xmm1
+; X86-SSE2-NEXT:    movd %xmm1, %eax
+; X86-SSE2-NEXT:    xorl $32768, %eax ## imm = 0x8000
+; X86-SSE2-NEXT:    ## kill: def $ax killed $ax killed $eax
+; X86-SSE2-NEXT:    retl
+;
+; X86-SSE42-LABEL: test_reduce_v32i16_v8i16:
+; X86-SSE42:       ## %bb.0:
+; X86-SSE42-NEXT:    pcmpeqd %xmm1, %xmm1
+; X86-SSE42-NEXT:    pxor %xmm0, %xmm1
+; X86-SSE42-NEXT:    phminposuw %xmm1, %xmm0
+; X86-SSE42-NEXT:    movd %xmm0, %eax
+; X86-SSE42-NEXT:    notl %eax
+; X86-SSE42-NEXT:    ## kill: def $ax killed $ax killed $eax
+; X86-SSE42-NEXT:    retl
+;
+; X86-AVX-LABEL: test_reduce_v32i16_v8i16:
+; X86-AVX:       ## %bb.0:
+; X86-AVX-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
+; X86-AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; X86-AVX-NEXT:    vphminposuw %xmm0, %xmm0
+; X86-AVX-NEXT:    vmovd %xmm0, %eax
+; X86-AVX-NEXT:    notl %eax
+; X86-AVX-NEXT:    ## kill: def $ax killed $ax killed $eax
+; X86-AVX-NEXT:    vzeroupper
+; X86-AVX-NEXT:    retl
+;
+; X64-SSE2-LABEL: test_reduce_v32i16_v8i16:
+; X64-SSE2:       ## %bb.0:
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [32768,32768,32768,32768,32768,32768,32768,32768]
+; X64-SSE2-NEXT:    pxor %xmm2, %xmm0
+; X64-SSE2-NEXT:    pxor %xmm2, %xmm1
+; X64-SSE2-NEXT:    pmaxsw %xmm0, %xmm1
+; X64-SSE2-NEXT:    movdqa %xmm1, %xmm0
+; X64-SSE2-NEXT:    pxor %xmm2, %xmm0
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; X64-SSE2-NEXT:    pxor %xmm2, %xmm0
+; X64-SSE2-NEXT:    pmaxsw %xmm1, %xmm0
+; X64-SSE2-NEXT:    movdqa %xmm0, %xmm1
+; X64-SSE2-NEXT:    pxor %xmm2, %xmm1
+; X64-SSE2-NEXT:    psrld $16, %xmm1
+; X64-SSE2-NEXT:    pxor %xmm2, %xmm1
+; X64-SSE2-NEXT:    pmaxsw %xmm0, %xmm1
+; X64-SSE2-NEXT:    movd %xmm1, %eax
+; X64-SSE2-NEXT:    xorl $32768, %eax ## imm = 0x8000
+; X64-SSE2-NEXT:    ## kill: def $ax killed $ax killed $eax
+; X64-SSE2-NEXT:    retq
+;
+; X64-SSE42-LABEL: test_reduce_v32i16_v8i16:
+; X64-SSE42:       ## %bb.0:
+; X64-SSE42-NEXT:    pcmpeqd %xmm1, %xmm1
+; X64-SSE42-NEXT:    pxor %xmm0, %xmm1
+; X64-SSE42-NEXT:    phminposuw %xmm1, %xmm0
+; X64-SSE42-NEXT:    movd %xmm0, %eax
+; X64-SSE42-NEXT:    notl %eax
+; X64-SSE42-NEXT:    ## kill: def $ax killed $ax killed $eax
+; X64-SSE42-NEXT:    retq
+;
+; X64-AVX1-LABEL: test_reduce_v32i16_v8i16:
+; X64-AVX1:       ## %bb.0:
+; X64-AVX1-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
+; X64-AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT:    vphminposuw %xmm0, %xmm0
+; X64-AVX1-NEXT:    vmovd %xmm0, %eax
+; X64-AVX1-NEXT:    notl %eax
+; X64-AVX1-NEXT:    ## kill: def $ax killed $ax killed $eax
+; X64-AVX1-NEXT:    vzeroupper
+; X64-AVX1-NEXT:    retq
+;
+; X64-AVX2-LABEL: test_reduce_v32i16_v8i16:
+; X64-AVX2:       ## %bb.0:
+; X64-AVX2-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
+; X64-AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT:    vphminposuw %xmm0, %xmm0
+; X64-AVX2-NEXT:    vmovd %xmm0, %eax
+; X64-AVX2-NEXT:    notl %eax
+; X64-AVX2-NEXT:    ## kill: def $ax killed $ax killed $eax
+; X64-AVX2-NEXT:    vzeroupper
+; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512-LABEL: test_reduce_v32i16_v8i16:
+; X64-AVX512:       ## %bb.0:
+; X64-AVX512-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
+; X64-AVX512-NEXT:    vphminposuw %xmm0, %xmm0
+; X64-AVX512-NEXT:    vmovd %xmm0, %eax
+; X64-AVX512-NEXT:    notl %eax
+; X64-AVX512-NEXT:    ## kill: def $ax killed $ax killed $eax
+; X64-AVX512-NEXT:    vzeroupper
+; X64-AVX512-NEXT:    retq
+  %1  = shufflevector <32 x i16> %a0, <32 x i16> undef, <32 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %2  = icmp ugt <32 x i16> %a0, %1
+  %3  = select <32 x i1> %2, <32 x i16> %a0, <32 x i16> %1
+  %4  = shufflevector <32 x i16> %3, <32 x i16> undef, <32 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %5  = icmp ugt <32 x i16> %3, %4
+  %6  = select <32 x i1> %5, <32 x i16> %3, <32 x i16> %4
+  %7  = shufflevector <32 x i16> %6, <32 x i16> undef, <32 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %8  = icmp ugt <32 x i16> %6, %7
+  %9  = select <32 x i1> %8, <32 x i16> %6, <32 x i16> %7
+  %10 = extractelement <32 x i16> %9, i32 0
+  ret i16 %10
+}
+
+define i8 @test_reduce_v32i8_v16i8(<32 x i8> %a0) {
+; X86-SSE2-LABEL: test_reduce_v32i8_v16i8:
+; X86-SSE2:       ## %bb.0:
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE2-NEXT:    pmaxub %xmm0, %xmm1
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X86-SSE2-NEXT:    pmaxub %xmm1, %xmm0
+; X86-SSE2-NEXT:    movdqa %xmm0, %xmm1
+; X86-SSE2-NEXT:    psrld $16, %xmm1
+; X86-SSE2-NEXT:    pmaxub %xmm0, %xmm1
+; X86-SSE2-NEXT:    movdqa %xmm1, %xmm0
+; X86-SSE2-NEXT:    psrlw $8, %xmm0
+; X86-SSE2-NEXT:    pmaxub %xmm1, %xmm0
+; X86-SSE2-NEXT:    movd %xmm0, %eax
+; X86-SSE2-NEXT:    ## kill: def $al killed $al killed $eax
+; X86-SSE2-NEXT:    retl
+;
+; X86-SSE42-LABEL: test_reduce_v32i8_v16i8:
+; X86-SSE42:       ## %bb.0:
+; X86-SSE42-NEXT:    pcmpeqd %xmm1, %xmm1
+; X86-SSE42-NEXT:    pxor %xmm0, %xmm1
+; X86-SSE42-NEXT:    movdqa %xmm1, %xmm0
+; X86-SSE42-NEXT:    psrlw $8, %xmm0
+; X86-SSE42-NEXT:    pminub %xmm1, %xmm0
+; X86-SSE42-NEXT:    phminposuw %xmm0, %xmm0
+; X86-SSE42-NEXT:    pextrb $0, %xmm0, %eax
+; X86-SSE42-NEXT:    notb %al
+; X86-SSE42-NEXT:    ## kill: def $al killed $al killed $eax
+; X86-SSE42-NEXT:    retl
+;
+; X86-AVX-LABEL: test_reduce_v32i8_v16i8:
+; X86-AVX:       ## %bb.0:
+; X86-AVX-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
+; X86-AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; X86-AVX-NEXT:    vpminub %xmm1, %xmm0, %xmm0
+; X86-AVX-NEXT:    vphminposuw %xmm0, %xmm0
+; X86-AVX-NEXT:    vpextrb $0, %xmm0, %eax
+; X86-AVX-NEXT:    notb %al
+; X86-AVX-NEXT:    ## kill: def $al killed $al killed $eax
+; X86-AVX-NEXT:    vzeroupper
+; X86-AVX-NEXT:    retl
+;
+; X64-SSE2-LABEL: test_reduce_v32i8_v16i8:
+; X64-SSE2:       ## %bb.0:
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-SSE2-NEXT:    pmaxub %xmm0, %xmm1
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X64-SSE2-NEXT:    pmaxub %xmm1, %xmm0
+; X64-SSE2-NEXT:    movdqa %xmm0, %xmm1
+; X64-SSE2-NEXT:    psrld $16, %xmm1
+; X64-SSE2-NEXT:    pmaxub %xmm0, %xmm1
+; X64-SSE2-NEXT:    movdqa %xmm1, %xmm0
+; X64-SSE2-NEXT:    psrlw $8, %xmm0
+; X64-SSE2-NEXT:    pmaxub %xmm1, %xmm0
+; X64-SSE2-NEXT:    movd %xmm0, %eax
+; X64-SSE2-NEXT:    ## kill: def $al killed $al killed $eax
+; X64-SSE2-NEXT:    retq
+;
+; X64-SSE42-LABEL: test_reduce_v32i8_v16i8:
+; X64-SSE42:       ## %bb.0:
+; X64-SSE42-NEXT:    pcmpeqd %xmm1, %xmm1
+; X64-SSE42-NEXT:    pxor %xmm0, %xmm1
+; X64-SSE42-NEXT:    movdqa %xmm1, %xmm0
+; X64-SSE42-NEXT:    psrlw $8, %xmm0
+; X64-SSE42-NEXT:    pminub %xmm1, %xmm0
+; X64-SSE42-NEXT:    phminposuw %xmm0, %xmm0
+; X64-SSE42-NEXT:    pextrb $0, %xmm0, %eax
+; X64-SSE42-NEXT:    notb %al
+; X64-SSE42-NEXT:    ## kill: def $al killed $al killed $eax
+; X64-SSE42-NEXT:    retq
+;
+; X64-AVX1-LABEL: test_reduce_v32i8_v16i8:
+; X64-AVX1:       ## %bb.0:
+; X64-AVX1-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
+; X64-AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; X64-AVX1-NEXT:    vpminub %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT:    vphminposuw %xmm0, %xmm0
+; X64-AVX1-NEXT:    vpextrb $0, %xmm0, %eax
+; X64-AVX1-NEXT:    notb %al
+; X64-AVX1-NEXT:    ## kill: def $al killed $al killed $eax
+; X64-AVX1-NEXT:    vzeroupper
+; X64-AVX1-NEXT:    retq
+;
+; X64-AVX2-LABEL: test_reduce_v32i8_v16i8:
+; X64-AVX2:       ## %bb.0:
+; X64-AVX2-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
+; X64-AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; X64-AVX2-NEXT:    vpminub %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT:    vphminposuw %xmm0, %xmm0
+; X64-AVX2-NEXT:    vpextrb $0, %xmm0, %eax
+; X64-AVX2-NEXT:    notb %al
+; X64-AVX2-NEXT:    ## kill: def $al killed $al killed $eax
+; X64-AVX2-NEXT:    vzeroupper
+; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512-LABEL: test_reduce_v32i8_v16i8:
+; X64-AVX512:       ## %bb.0:
+; X64-AVX512-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
+; X64-AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; X64-AVX512-NEXT:    vpminub %xmm1, %xmm0, %xmm0
+; X64-AVX512-NEXT:    vphminposuw %xmm0, %xmm0
+; X64-AVX512-NEXT:    vpextrb $0, %xmm0, %eax
+; X64-AVX512-NEXT:    notb %al
+; X64-AVX512-NEXT:    ## kill: def $al killed $al killed $eax
+; X64-AVX512-NEXT:    vzeroupper
+; X64-AVX512-NEXT:    retq
+  %1  = shufflevector <32 x i8> %a0, <32 x i8> undef, <32 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %2  = icmp ugt <32 x i8> %a0, %1
+  %3  = select <32 x i1> %2, <32 x i8> %a0, <32 x i8> %1
+  %4  = shufflevector <32 x i8> %3, <32 x i8> undef, <32 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %5  = icmp ugt <32 x i8> %3, %4
+  %6  = select <32 x i1> %5, <32 x i8> %3, <32 x i8> %4
+  %7  = shufflevector <32 x i8> %6, <32 x i8> undef, <32 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %8  = icmp ugt <32 x i8> %6, %7
+  %9  = select <32 x i1> %8, <32 x i8> %6, <32 x i8> %7
+  %10 = shufflevector <32 x i8> %9, <32 x i8> undef, <32 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %11 = icmp ugt <32 x i8> %9, %10
+  %12 = select <32 x i1> %11, <32 x i8> %9, <32 x i8> %10
+  %13 = extractelement <32 x i8> %12, i32 0
+  ret i8 %13
+}
+
+define i8 @test_reduce_v64i8_v16i8(<64 x i8> %a0) {
+; X86-SSE2-LABEL: test_reduce_v64i8_v16i8:
+; X86-SSE2:       ## %bb.0:
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE2-NEXT:    pmaxub %xmm0, %xmm1
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X86-SSE2-NEXT:    pmaxub %xmm1, %xmm0
+; X86-SSE2-NEXT:    movdqa %xmm0, %xmm1
+; X86-SSE2-NEXT:    psrld $16, %xmm1
+; X86-SSE2-NEXT:    pmaxub %xmm0, %xmm1
+; X86-SSE2-NEXT:    movdqa %xmm1, %xmm0
+; X86-SSE2-NEXT:    psrlw $8, %xmm0
+; X86-SSE2-NEXT:    pmaxub %xmm1, %xmm0
+; X86-SSE2-NEXT:    movd %xmm0, %eax
+; X86-SSE2-NEXT:    ## kill: def $al killed $al killed $eax
+; X86-SSE2-NEXT:    retl
+;
+; X86-SSE42-LABEL: test_reduce_v64i8_v16i8:
+; X86-SSE42:       ## %bb.0:
+; X86-SSE42-NEXT:    pcmpeqd %xmm1, %xmm1
+; X86-SSE42-NEXT:    pxor %xmm0, %xmm1
+; X86-SSE42-NEXT:    movdqa %xmm1, %xmm0
+; X86-SSE42-NEXT:    psrlw $8, %xmm0
+; X86-SSE42-NEXT:    pminub %xmm1, %xmm0
+; X86-SSE42-NEXT:    phminposuw %xmm0, %xmm0
+; X86-SSE42-NEXT:    pextrb $0, %xmm0, %eax
+; X86-SSE42-NEXT:    notb %al
+; X86-SSE42-NEXT:    ## kill: def $al killed $al killed $eax
+; X86-SSE42-NEXT:    retl
+;
+; X86-AVX-LABEL: test_reduce_v64i8_v16i8:
+; X86-AVX:       ## %bb.0:
+; X86-AVX-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
+; X86-AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; X86-AVX-NEXT:    vpminub %xmm1, %xmm0, %xmm0
+; X86-AVX-NEXT:    vphminposuw %xmm0, %xmm0
+; X86-AVX-NEXT:    vpextrb $0, %xmm0, %eax
+; X86-AVX-NEXT:    notb %al
+; X86-AVX-NEXT:    ## kill: def $al killed $al killed $eax
+; X86-AVX-NEXT:    vzeroupper
+; X86-AVX-NEXT:    retl
+;
+; X64-SSE2-LABEL: test_reduce_v64i8_v16i8:
+; X64-SSE2:       ## %bb.0:
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-SSE2-NEXT:    pmaxub %xmm0, %xmm1
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X64-SSE2-NEXT:    pmaxub %xmm1, %xmm0
+; X64-SSE2-NEXT:    movdqa %xmm0, %xmm1
+; X64-SSE2-NEXT:    psrld $16, %xmm1
+; X64-SSE2-NEXT:    pmaxub %xmm0, %xmm1
+; X64-SSE2-NEXT:    movdqa %xmm1, %xmm0
+; X64-SSE2-NEXT:    psrlw $8, %xmm0
+; X64-SSE2-NEXT:    pmaxub %xmm1, %xmm0
+; X64-SSE2-NEXT:    movd %xmm0, %eax
+; X64-SSE2-NEXT:    ## kill: def $al killed $al killed $eax
+; X64-SSE2-NEXT:    retq
+;
+; X64-SSE42-LABEL: test_reduce_v64i8_v16i8:
+; X64-SSE42:       ## %bb.0:
+; X64-SSE42-NEXT:    pcmpeqd %xmm1, %xmm1
+; X64-SSE42-NEXT:    pxor %xmm0, %xmm1
+; X64-SSE42-NEXT:    movdqa %xmm1, %xmm0
+; X64-SSE42-NEXT:    psrlw $8, %xmm0
+; X64-SSE42-NEXT:    pminub %xmm1, %xmm0
+; X64-SSE42-NEXT:    phminposuw %xmm0, %xmm0
+; X64-SSE42-NEXT:    pextrb $0, %xmm0, %eax
+; X64-SSE42-NEXT:    notb %al
+; X64-SSE42-NEXT:    ## kill: def $al killed $al killed $eax
+; X64-SSE42-NEXT:    retq
+;
+; X64-AVX1-LABEL: test_reduce_v64i8_v16i8:
+; X64-AVX1:       ## %bb.0:
+; X64-AVX1-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
+; X64-AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; X64-AVX1-NEXT:    vpminub %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT:    vphminposuw %xmm0, %xmm0
+; X64-AVX1-NEXT:    vpextrb $0, %xmm0, %eax
+; X64-AVX1-NEXT:    notb %al
+; X64-AVX1-NEXT:    ## kill: def $al killed $al killed $eax
+; X64-AVX1-NEXT:    vzeroupper
+; X64-AVX1-NEXT:    retq
+;
+; X64-AVX2-LABEL: test_reduce_v64i8_v16i8:
+; X64-AVX2:       ## %bb.0:
+; X64-AVX2-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
+; X64-AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; X64-AVX2-NEXT:    vpminub %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT:    vphminposuw %xmm0, %xmm0
+; X64-AVX2-NEXT:    vpextrb $0, %xmm0, %eax
+; X64-AVX2-NEXT:    notb %al
+; X64-AVX2-NEXT:    ## kill: def $al killed $al killed $eax
+; X64-AVX2-NEXT:    vzeroupper
+; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512-LABEL: test_reduce_v64i8_v16i8:
+; X64-AVX512:       ## %bb.0:
+; X64-AVX512-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
+; X64-AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; X64-AVX512-NEXT:    vpminub %xmm1, %xmm0, %xmm0
+; X64-AVX512-NEXT:    vphminposuw %xmm0, %xmm0
+; X64-AVX512-NEXT:    vpextrb $0, %xmm0, %eax
+; X64-AVX512-NEXT:    notb %al
+; X64-AVX512-NEXT:    ## kill: def $al killed $al killed $eax
+; X64-AVX512-NEXT:    vzeroupper
+; X64-AVX512-NEXT:    retq
+  %1  = shufflevector <64 x i8> %a0, <64 x i8> undef, <64 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %2  = icmp ugt <64 x i8> %a0, %1
+  %3  = select <64 x i1> %2, <64 x i8> %a0, <64 x i8> %1
+  %4  = shufflevector <64 x i8> %3, <64 x i8> undef, <64 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %5  = icmp ugt <64 x i8> %3, %4
+  %6  = select <64 x i1> %5, <64 x i8> %3, <64 x i8> %4
+  %7  = shufflevector <64 x i8> %6, <64 x i8> undef, <64 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %8  = icmp ugt <64 x i8> %6, %7
+  %9  = select <64 x i1> %8, <64 x i8> %6, <64 x i8> %7
+  %10 = shufflevector <64 x i8> %9, <64 x i8> undef, <64 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %11 = icmp ugt <64 x i8> %9, %10
+  %12 = select <64 x i1> %11, <64 x i8> %9, <64 x i8> %10
+  %13 = extractelement <64 x i8> %12, i32 0
+  ret i8 %13
+}
diff --git a/llvm/test/CodeGen/X86/horizontal-reduce-umin.ll b/llvm/test/CodeGen/X86/horizontal-reduce-umin.ll
index 5e4d83046..5b4d3165e 100644
--- a/llvm/test/CodeGen/X86/horizontal-reduce-umin.ll
+++ b/llvm/test/CodeGen/X86/horizontal-reduce-umin.ll
@@ -1885,3 +1885,357 @@ define i8 @test_reduce_v64i8(<64 x i8> %a0) {
   %19 = extractelement <64 x i8> %18, i32 0
   ret i8 %19
 }
+
+;
+; Partial Vector Reductions
+;
+
+define i16 @test_reduce_v16i16_v8i16(<16 x i16> %a0) {
+; X86-SSE2-LABEL: test_reduce_v16i16_v8i16:
+; X86-SSE2:       ## %bb.0:
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [32768,32768,32768,32768,32768,32768,32768,32768]
+; X86-SSE2-NEXT:    pxor %xmm2, %xmm0
+; X86-SSE2-NEXT:    pxor %xmm2, %xmm1
+; X86-SSE2-NEXT:    pminsw %xmm0, %xmm1
+; X86-SSE2-NEXT:    movdqa %xmm1, %xmm0
+; X86-SSE2-NEXT:    pxor %xmm2, %xmm0
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; X86-SSE2-NEXT:    pxor %xmm2, %xmm0
+; X86-SSE2-NEXT:    pminsw %xmm1, %xmm0
+; X86-SSE2-NEXT:    movdqa %xmm0, %xmm1
+; X86-SSE2-NEXT:    pxor %xmm2, %xmm1
+; X86-SSE2-NEXT:    psrld $16, %xmm1
+; X86-SSE2-NEXT:    pxor %xmm2, %xmm1
+; X86-SSE2-NEXT:    pminsw %xmm0, %xmm1
+; X86-SSE2-NEXT:    movd %xmm1, %eax
+; X86-SSE2-NEXT:    xorl $32768, %eax ## imm = 0x8000
+; X86-SSE2-NEXT:    ## kill: def $ax killed $ax killed $eax
+; X86-SSE2-NEXT:    retl
+;
+; X86-SSE42-LABEL: test_reduce_v16i16_v8i16:
+; X86-SSE42:       ## %bb.0:
+; X86-SSE42-NEXT:    phminposuw %xmm0, %xmm0
+; X86-SSE42-NEXT:    movd %xmm0, %eax
+; X86-SSE42-NEXT:    ## kill: def $ax killed $ax killed $eax
+; X86-SSE42-NEXT:    retl
+;
+; X86-AVX-LABEL: test_reduce_v16i16_v8i16:
+; X86-AVX:       ## %bb.0:
+; X86-AVX-NEXT:    vphminposuw %xmm0, %xmm0
+; X86-AVX-NEXT:    vmovd %xmm0, %eax
+; X86-AVX-NEXT:    ## kill: def $ax killed $ax killed $eax
+; X86-AVX-NEXT:    vzeroupper
+; X86-AVX-NEXT:    retl
+;
+; X64-SSE2-LABEL: test_reduce_v16i16_v8i16:
+; X64-SSE2:       ## %bb.0:
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [32768,32768,32768,32768,32768,32768,32768,32768]
+; X64-SSE2-NEXT:    pxor %xmm2, %xmm0
+; X64-SSE2-NEXT:    pxor %xmm2, %xmm1
+; X64-SSE2-NEXT:    pminsw %xmm0, %xmm1
+; X64-SSE2-NEXT:    movdqa %xmm1, %xmm0
+; X64-SSE2-NEXT:    pxor %xmm2, %xmm0
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; X64-SSE2-NEXT:    pxor %xmm2, %xmm0
+; X64-SSE2-NEXT:    pminsw %xmm1, %xmm0
+; X64-SSE2-NEXT:    movdqa %xmm0, %xmm1
+; X64-SSE2-NEXT:    pxor %xmm2, %xmm1
+; X64-SSE2-NEXT:    psrld $16, %xmm1
+; X64-SSE2-NEXT:    pxor %xmm2, %xmm1
+; X64-SSE2-NEXT:    pminsw %xmm0, %xmm1
+; X64-SSE2-NEXT:    movd %xmm1, %eax
+; X64-SSE2-NEXT:    xorl $32768, %eax ## imm = 0x8000
+; X64-SSE2-NEXT:    ## kill: def $ax killed $ax killed $eax
+; X64-SSE2-NEXT:    retq
+;
+; X64-SSE42-LABEL: test_reduce_v16i16_v8i16:
+; X64-SSE42:       ## %bb.0:
+; X64-SSE42-NEXT:    phminposuw %xmm0, %xmm0
+; X64-SSE42-NEXT:    movd %xmm0, %eax
+; X64-SSE42-NEXT:    ## kill: def $ax killed $ax killed $eax
+; X64-SSE42-NEXT:    retq
+;
+; X64-AVX-LABEL: test_reduce_v16i16_v8i16:
+; X64-AVX:       ## %bb.0:
+; X64-AVX-NEXT:    vphminposuw %xmm0, %xmm0
+; X64-AVX-NEXT:    vmovd %xmm0, %eax
+; X64-AVX-NEXT:    ## kill: def $ax killed $ax killed $eax
+; X64-AVX-NEXT:    vzeroupper
+; X64-AVX-NEXT:    retq
+  %1  = shufflevector <16 x i16> %a0, <16 x i16> undef, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %2  = icmp ult <16 x i16> %a0, %1
+  %3  = select <16 x i1> %2, <16 x i16> %a0, <16 x i16> %1
+  %4  = shufflevector <16 x i16> %3, <16 x i16> undef, <16 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %5  = icmp ult <16 x i16> %3, %4
+  %6  = select <16 x i1> %5, <16 x i16> %3, <16 x i16> %4
+  %7  = shufflevector <16 x i16> %6, <16 x i16> undef, <16 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %8  = icmp ult <16 x i16> %6, %7
+  %9  = select <16 x i1> %8, <16 x i16> %6, <16 x i16> %7
+  %10 = extractelement <16 x i16> %9, i32 0
+  ret i16 %10
+}
+
+define i16 @test_reduce_v32i16_v8i16(<32 x i16> %a0) {
+; X86-SSE2-LABEL: test_reduce_v32i16_v8i16:
+; X86-SSE2:       ## %bb.0:
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [32768,32768,32768,32768,32768,32768,32768,32768]
+; X86-SSE2-NEXT:    pxor %xmm2, %xmm0
+; X86-SSE2-NEXT:    pxor %xmm2, %xmm1
+; X86-SSE2-NEXT:    pminsw %xmm0, %xmm1
+; X86-SSE2-NEXT:    movdqa %xmm1, %xmm0
+; X86-SSE2-NEXT:    pxor %xmm2, %xmm0
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; X86-SSE2-NEXT:    pxor %xmm2, %xmm0
+; X86-SSE2-NEXT:    pminsw %xmm1, %xmm0
+; X86-SSE2-NEXT:    movdqa %xmm0, %xmm1
+; X86-SSE2-NEXT:    pxor %xmm2, %xmm1
+; X86-SSE2-NEXT:    psrld $16, %xmm1
+; X86-SSE2-NEXT:    pxor %xmm2, %xmm1
+; X86-SSE2-NEXT:    pminsw %xmm0, %xmm1
+; X86-SSE2-NEXT:    movd %xmm1, %eax
+; X86-SSE2-NEXT:    xorl $32768, %eax ## imm = 0x8000
+; X86-SSE2-NEXT:    ## kill: def $ax killed $ax killed $eax
+; X86-SSE2-NEXT:    retl
+;
+; X86-SSE42-LABEL: test_reduce_v32i16_v8i16:
+; X86-SSE42:       ## %bb.0:
+; X86-SSE42-NEXT:    phminposuw %xmm0, %xmm0
+; X86-SSE42-NEXT:    movd %xmm0, %eax
+; X86-SSE42-NEXT:    ## kill: def $ax killed $ax killed $eax
+; X86-SSE42-NEXT:    retl
+;
+; X86-AVX-LABEL: test_reduce_v32i16_v8i16:
+; X86-AVX:       ## %bb.0:
+; X86-AVX-NEXT:    vphminposuw %xmm0, %xmm0
+; X86-AVX-NEXT:    vmovd %xmm0, %eax
+; X86-AVX-NEXT:    ## kill: def $ax killed $ax killed $eax
+; X86-AVX-NEXT:    vzeroupper
+; X86-AVX-NEXT:    retl
+;
+; X64-SSE2-LABEL: test_reduce_v32i16_v8i16:
+; X64-SSE2:       ## %bb.0:
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [32768,32768,32768,32768,32768,32768,32768,32768]
+; X64-SSE2-NEXT:    pxor %xmm2, %xmm0
+; X64-SSE2-NEXT:    pxor %xmm2, %xmm1
+; X64-SSE2-NEXT:    pminsw %xmm0, %xmm1
+; X64-SSE2-NEXT:    movdqa %xmm1, %xmm0
+; X64-SSE2-NEXT:    pxor %xmm2, %xmm0
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; X64-SSE2-NEXT:    pxor %xmm2, %xmm0
+; X64-SSE2-NEXT:    pminsw %xmm1, %xmm0
+; X64-SSE2-NEXT:    movdqa %xmm0, %xmm1
+; X64-SSE2-NEXT:    pxor %xmm2, %xmm1
+; X64-SSE2-NEXT:    psrld $16, %xmm1
+; X64-SSE2-NEXT:    pxor %xmm2, %xmm1
+; X64-SSE2-NEXT:    pminsw %xmm0, %xmm1
+; X64-SSE2-NEXT:    movd %xmm1, %eax
+; X64-SSE2-NEXT:    xorl $32768, %eax ## imm = 0x8000
+; X64-SSE2-NEXT:    ## kill: def $ax killed $ax killed $eax
+; X64-SSE2-NEXT:    retq
+;
+; X64-SSE42-LABEL: test_reduce_v32i16_v8i16:
+; X64-SSE42:       ## %bb.0:
+; X64-SSE42-NEXT:    phminposuw %xmm0, %xmm0
+; X64-SSE42-NEXT:    movd %xmm0, %eax
+; X64-SSE42-NEXT:    ## kill: def $ax killed $ax killed $eax
+; X64-SSE42-NEXT:    retq
+;
+; X64-AVX-LABEL: test_reduce_v32i16_v8i16:
+; X64-AVX:       ## %bb.0:
+; X64-AVX-NEXT:    vphminposuw %xmm0, %xmm0
+; X64-AVX-NEXT:    vmovd %xmm0, %eax
+; X64-AVX-NEXT:    ## kill: def $ax killed $ax killed $eax
+; X64-AVX-NEXT:    vzeroupper
+; X64-AVX-NEXT:    retq
+  %1  = shufflevector <32 x i16> %a0, <32 x i16> undef, <32 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %2  = icmp ult <32 x i16> %a0, %1
+  %3  = select <32 x i1> %2, <32 x i16> %a0, <32 x i16> %1
+  %4  = shufflevector <32 x i16> %3, <32 x i16> undef, <32 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %5  = icmp ult <32 x i16> %3, %4
+  %6  = select <32 x i1> %5, <32 x i16> %3, <32 x i16> %4
+  %7  = shufflevector <32 x i16> %6, <32 x i16> undef, <32 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %8  = icmp ult <32 x i16> %6, %7
+  %9  = select <32 x i1> %8, <32 x i16> %6, <32 x i16> %7
+  %10 = extractelement <32 x i16> %9, i32 0
+  ret i16 %10
+}
+
+define i8 @test_reduce_v32i8_v16i8(<32 x i8> %a0) {
+; X86-SSE2-LABEL: test_reduce_v32i8_v16i8:
+; X86-SSE2:       ## %bb.0:
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE2-NEXT:    pminub %xmm0, %xmm1
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X86-SSE2-NEXT:    pminub %xmm1, %xmm0
+; X86-SSE2-NEXT:    movdqa %xmm0, %xmm1
+; X86-SSE2-NEXT:    psrld $16, %xmm1
+; X86-SSE2-NEXT:    pminub %xmm0, %xmm1
+; X86-SSE2-NEXT:    movdqa %xmm1, %xmm0
+; X86-SSE2-NEXT:    psrlw $8, %xmm0
+; X86-SSE2-NEXT:    pminub %xmm1, %xmm0
+; X86-SSE2-NEXT:    movd %xmm0, %eax
+; X86-SSE2-NEXT:    ## kill: def $al killed $al killed $eax
+; X86-SSE2-NEXT:    retl
+;
+; X86-SSE42-LABEL: test_reduce_v32i8_v16i8:
+; X86-SSE42:       ## %bb.0:
+; X86-SSE42-NEXT:    movdqa %xmm0, %xmm1
+; X86-SSE42-NEXT:    psrlw $8, %xmm1
+; X86-SSE42-NEXT:    pminub %xmm0, %xmm1
+; X86-SSE42-NEXT:    phminposuw %xmm1, %xmm0
+; X86-SSE42-NEXT:    pextrb $0, %xmm0, %eax
+; X86-SSE42-NEXT:    ## kill: def $al killed $al killed $eax
+; X86-SSE42-NEXT:    retl
+;
+; X86-AVX-LABEL: test_reduce_v32i8_v16i8:
+; X86-AVX:       ## %bb.0:
+; X86-AVX-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; X86-AVX-NEXT:    vpminub %xmm1, %xmm0, %xmm0
+; X86-AVX-NEXT:    vphminposuw %xmm0, %xmm0
+; X86-AVX-NEXT:    vpextrb $0, %xmm0, %eax
+; X86-AVX-NEXT:    ## kill: def $al killed $al killed $eax
+; X86-AVX-NEXT:    vzeroupper
+; X86-AVX-NEXT:    retl
+;
+; X64-SSE2-LABEL: test_reduce_v32i8_v16i8:
+; X64-SSE2:       ## %bb.0:
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-SSE2-NEXT:    pminub %xmm0, %xmm1
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X64-SSE2-NEXT:    pminub %xmm1, %xmm0
+; X64-SSE2-NEXT:    movdqa %xmm0, %xmm1
+; X64-SSE2-NEXT:    psrld $16, %xmm1
+; X64-SSE2-NEXT:    pminub %xmm0, %xmm1
+; X64-SSE2-NEXT:    movdqa %xmm1, %xmm0
+; X64-SSE2-NEXT:    psrlw $8, %xmm0
+; X64-SSE2-NEXT:    pminub %xmm1, %xmm0
+; X64-SSE2-NEXT:    movd %xmm0, %eax
+; X64-SSE2-NEXT:    ## kill: def $al killed $al killed $eax
+; X64-SSE2-NEXT:    retq
+;
+; X64-SSE42-LABEL: test_reduce_v32i8_v16i8:
+; X64-SSE42:       ## %bb.0:
+; X64-SSE42-NEXT:    movdqa %xmm0, %xmm1
+; X64-SSE42-NEXT:    psrlw $8, %xmm1
+; X64-SSE42-NEXT:    pminub %xmm0, %xmm1
+; X64-SSE42-NEXT:    phminposuw %xmm1, %xmm0
+; X64-SSE42-NEXT:    pextrb $0, %xmm0, %eax
+; X64-SSE42-NEXT:    ## kill: def $al killed $al killed $eax
+; X64-SSE42-NEXT:    retq
+;
+; X64-AVX-LABEL: test_reduce_v32i8_v16i8:
+; X64-AVX:       ## %bb.0:
+; X64-AVX-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; X64-AVX-NEXT:    vpminub %xmm1, %xmm0, %xmm0
+; X64-AVX-NEXT:    vphminposuw %xmm0, %xmm0
+; X64-AVX-NEXT:    vpextrb $0, %xmm0, %eax
+; X64-AVX-NEXT:    ## kill: def $al killed $al killed $eax
+; X64-AVX-NEXT:    vzeroupper
+; X64-AVX-NEXT:    retq
+  %1  = shufflevector <32 x i8> %a0, <32 x i8> undef, <32 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %2  = icmp ult <32 x i8> %a0, %1
+  %3  = select <32 x i1> %2, <32 x i8> %a0, <32 x i8> %1
+  %4  = shufflevector <32 x i8> %3, <32 x i8> undef, <32 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %5  = icmp ult <32 x i8> %3, %4
+  %6  = select <32 x i1> %5, <32 x i8> %3, <32 x i8> %4
+  %7  = shufflevector <32 x i8> %6, <32 x i8> undef, <32 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %8  = icmp ult <32 x i8> %6, %7
+  %9  = select <32 x i1> %8, <32 x i8> %6, <32 x i8> %7
+  %10 = shufflevector <32 x i8> %9, <32 x i8> undef, <32 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %11 = icmp ult <32 x i8> %9, %10
+  %12 = select <32 x i1> %11, <32 x i8> %9, <32 x i8> %10
+  %13 = extractelement <32 x i8> %12, i32 0
+  ret i8 %13
+}
+
+define i8 @test_reduce_v64i8_v16i8(<64 x i8> %a0) {
+; X86-SSE2-LABEL: test_reduce_v64i8_v16i8:
+; X86-SSE2:       ## %bb.0:
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE2-NEXT:    pminub %xmm0, %xmm1
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X86-SSE2-NEXT:    pminub %xmm1, %xmm0
+; X86-SSE2-NEXT:    movdqa %xmm0, %xmm1
+; X86-SSE2-NEXT:    psrld $16, %xmm1
+; X86-SSE2-NEXT:    pminub %xmm0, %xmm1
+; X86-SSE2-NEXT:    movdqa %xmm1, %xmm0
+; X86-SSE2-NEXT:    psrlw $8, %xmm0
+; X86-SSE2-NEXT:    pminub %xmm1, %xmm0
+; X86-SSE2-NEXT:    movd %xmm0, %eax
+; X86-SSE2-NEXT:    ## kill: def $al killed $al killed $eax
+; X86-SSE2-NEXT:    retl
+;
+; X86-SSE42-LABEL: test_reduce_v64i8_v16i8:
+; X86-SSE42:       ## %bb.0:
+; X86-SSE42-NEXT:    movdqa %xmm0, %xmm1
+; X86-SSE42-NEXT:    psrlw $8, %xmm1
+; X86-SSE42-NEXT:    pminub %xmm0, %xmm1
+; X86-SSE42-NEXT:    phminposuw %xmm1, %xmm0
+; X86-SSE42-NEXT:    pextrb $0, %xmm0, %eax
+; X86-SSE42-NEXT:    ## kill: def $al killed $al killed $eax
+; X86-SSE42-NEXT:    retl
+;
+; X86-AVX-LABEL: test_reduce_v64i8_v16i8:
+; X86-AVX:       ## %bb.0:
+; X86-AVX-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; X86-AVX-NEXT:    vpminub %xmm1, %xmm0, %xmm0
+; X86-AVX-NEXT:    vphminposuw %xmm0, %xmm0
+; X86-AVX-NEXT:    vpextrb $0, %xmm0, %eax
+; X86-AVX-NEXT:    ## kill: def $al killed $al killed $eax
+; X86-AVX-NEXT:    vzeroupper
+; X86-AVX-NEXT:    retl
+;
+; X64-SSE2-LABEL: test_reduce_v64i8_v16i8:
+; X64-SSE2:       ## %bb.0:
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-SSE2-NEXT:    pminub %xmm0, %xmm1
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X64-SSE2-NEXT:    pminub %xmm1, %xmm0
+; X64-SSE2-NEXT:    movdqa %xmm0, %xmm1
+; X64-SSE2-NEXT:    psrld $16, %xmm1
+; X64-SSE2-NEXT:    pminub %xmm0, %xmm1
+; X64-SSE2-NEXT:    movdqa %xmm1, %xmm0
+; X64-SSE2-NEXT:    psrlw $8, %xmm0
+; X64-SSE2-NEXT:    pminub %xmm1, %xmm0
+; X64-SSE2-NEXT:    movd %xmm0, %eax
+; X64-SSE2-NEXT:    ## kill: def $al killed $al killed $eax
+; X64-SSE2-NEXT:    retq
+;
+; X64-SSE42-LABEL: test_reduce_v64i8_v16i8:
+; X64-SSE42:       ## %bb.0:
+; X64-SSE42-NEXT:    movdqa %xmm0, %xmm1
+; X64-SSE42-NEXT:    psrlw $8, %xmm1
+; X64-SSE42-NEXT:    pminub %xmm0, %xmm1
+; X64-SSE42-NEXT:    phminposuw %xmm1, %xmm0
+; X64-SSE42-NEXT:    pextrb $0, %xmm0, %eax
+; X64-SSE42-NEXT:    ## kill: def $al killed $al killed $eax
+; X64-SSE42-NEXT:    retq
+;
+; X64-AVX-LABEL: test_reduce_v64i8_v16i8:
+; X64-AVX:       ## %bb.0:
+; X64-AVX-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; X64-AVX-NEXT:    vpminub %xmm1, %xmm0, %xmm0
+; X64-AVX-NEXT:    vphminposuw %xmm0, %xmm0
+; X64-AVX-NEXT:    vpextrb $0, %xmm0, %eax
+; X64-AVX-NEXT:    ## kill: def $al killed $al killed $eax
+; X64-AVX-NEXT:    vzeroupper
+; X64-AVX-NEXT:    retq
+  %1  = shufflevector <64 x i8> %a0, <64 x i8> undef, <64 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %2  = icmp ult <64 x i8> %a0, %1
+  %3  = select <64 x i1> %2, <64 x i8> %a0, <64 x i8> %1
+  %4  = shufflevector <64 x i8> %3, <64 x i8> undef, <64 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %5  = icmp ult <64 x i8> %3, %4
+  %6  = select <64 x i1> %5, <64 x i8> %3, <64 x i8> %4
+  %7  = shufflevector <64 x i8> %6, <64 x i8> undef, <64 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %8  = icmp ult <64 x i8> %6, %7
+  %9  = select <64 x i1> %8, <64 x i8> %6, <64 x i8> %7
+  %10 = shufflevector <64 x i8> %9, <64 x i8> undef, <64 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %11 = icmp ult <64 x i8> %9, %10
+  %12 = select <64 x i1> %11, <64 x i8> %9, <64 x i8> %10
+  %13 = extractelement <64 x i8> %12, i32 0
+  ret i8 %13
+}
diff --git a/llvm/test/CodeGen/X86/i128-add.ll b/llvm/test/CodeGen/X86/i128-add.ll
index 182ada4c4..8e75c4a57 100644
--- a/llvm/test/CodeGen/X86/i128-add.ll
+++ b/llvm/test/CodeGen/X86/i128-add.ll
@@ -57,10 +57,10 @@ define <1 x i128> @add_v1i128(<1 x i128> %x, <1 x i128> %y) nounwind {
 ; X86-NEXT:    adcl {{[0-9]+}}(%esp), %edi
 ; X86-NEXT:    adcl {{[0-9]+}}(%esp), %edx
 ; X86-NEXT:    adcl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    subl $-1, %esi
-; X86-NEXT:    sbbl $-1, %edi
-; X86-NEXT:    sbbl $-1, %edx
-; X86-NEXT:    sbbl $-1, %ecx
+; X86-NEXT:    addl $1, %esi
+; X86-NEXT:    adcl $0, %edi
+; X86-NEXT:    adcl $0, %edx
+; X86-NEXT:    adcl $0, %ecx
 ; X86-NEXT:    movl %esi, (%eax)
 ; X86-NEXT:    movl %edi, 4(%eax)
 ; X86-NEXT:    movl %edx, 8(%eax)
diff --git a/llvm/test/CodeGen/X86/i128-and-beyond.ll b/llvm/test/CodeGen/X86/i128-and-beyond.ll
index 5018900d0..9a1386c26 100644
--- a/llvm/test/CodeGen/X86/i128-and-beyond.ll
+++ b/llvm/test/CodeGen/X86/i128-and-beyond.ll
@@ -1,8 +1,21 @@
-; RUN: llc < %s -mtriple=i686-pc-linux-gnu | grep -- -1 | count 14
+; RUN: llc < %s -mtriple=i686-pc-linux-gnu | FileCheck %s
 
 ; These static initializers are too big to hand off to assemblers
 ; as monolithic blobs.
 
+; CHECK: -1
+; CHECK-NEXT: -1
+; CHECK: -1
+; CHECK-NEXT: -1
+; CHECK-NEXT: -1
+; CHECK-NEXT: -1
+; CHECK: -1
+; CHECK-NEXT: -1
+; CHECK-NEXT: -1
+; CHECK-NEXT: -1
+; CHECK-NEXT: -1
+; CHECK-NEXT: -1
+; CHECK-NEXT: -1
 @x = global i128 -1
 @y = global i256 -1
 @z = global i512 -1
diff --git a/llvm/test/CodeGen/X86/i128-immediate.ll b/llvm/test/CodeGen/X86/i128-immediate.ll
index 999076cdd..96c05a3a3 100644
--- a/llvm/test/CodeGen/X86/i128-immediate.ll
+++ b/llvm/test/CodeGen/X86/i128-immediate.ll
@@ -1,5 +1,11 @@
-; RUN: llc < %s -mtriple=x86_64-- | grep movq | count 2
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-- | FileCheck %s
 
 define i128 @__addvti3() {
+; CHECK-LABEL: __addvti3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq $-1, %rax
+; CHECK-NEXT:    movq $-1, %rdx
+; CHECK-NEXT:    retq
           ret i128 -1
 }
diff --git a/llvm/test/CodeGen/X86/i128-mul.ll b/llvm/test/CodeGen/X86/i128-mul.ll
index 3b74e041f..e40f10a67 100644
--- a/llvm/test/CodeGen/X86/i128-mul.ll
+++ b/llvm/test/CodeGen/X86/i128-mul.ll
@@ -14,42 +14,29 @@ define i64 @foo(i64 %x, i64 %y) nounwind {
 ; X86-NEXT:    pushl %edi
 ; X86-NEXT:    pushl %esi
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebp
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-NEXT:    movl %ecx, %eax
-; X86-NEXT:    mull %ebp
+; X86-NEXT:    mull %ebx
+; X86-NEXT:    movl %edx, %edi
+; X86-NEXT:    movl %ebp, %eax
+; X86-NEXT:    mull %ebx
 ; X86-NEXT:    movl %edx, %ebx
-; X86-NEXT:    movl %esi, %eax
-; X86-NEXT:    mull %ebp
-; X86-NEXT:    movl %edx, %ebp
-; X86-NEXT:    movl %eax, %esi
-; X86-NEXT:    addl %ebx, %esi
-; X86-NEXT:    adcl $0, %ebp
+; X86-NEXT:    movl %eax, %ebp
+; X86-NEXT:    addl %edi, %ebp
+; X86-NEXT:    adcl $0, %ebx
 ; X86-NEXT:    movl %ecx, %eax
-; X86-NEXT:    mull %edi
-; X86-NEXT:    movl %edx, %ebx
-; X86-NEXT:    addl %esi, %eax
-; X86-NEXT:    adcl %ebp, %ebx
+; X86-NEXT:    mull %esi
+; X86-NEXT:    movl %edx, %ecx
+; X86-NEXT:    addl %ebp, %eax
+; X86-NEXT:    adcl %ebx, %ecx
 ; X86-NEXT:    setb %al
-; X86-NEXT:    movzbl %al, %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    mull %edi
-; X86-NEXT:    movl %edx, %esi
-; X86-NEXT:    movl %eax, %ebp
-; X86-NEXT:    addl %ebx, %ebp
-; X86-NEXT:    adcl %ecx, %esi
-; X86-NEXT:    xorl %ecx, %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    mull %ecx
-; X86-NEXT:    movl %edx, %edi
-; X86-NEXT:    movl %eax, %ebx
+; X86-NEXT:    movzbl %al, %edi
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    mull %ecx
-; X86-NEXT:    addl %ebx, %eax
+; X86-NEXT:    mull %esi
+; X86-NEXT:    addl %ecx, %eax
 ; X86-NEXT:    adcl %edi, %edx
-; X86-NEXT:    addl %ebp, %eax
-; X86-NEXT:    adcl %esi, %edx
 ; X86-NEXT:    popl %esi
 ; X86-NEXT:    popl %edi
 ; X86-NEXT:    popl %ebx
@@ -81,7 +68,7 @@ define i64 @mul1(i64 %n, i64* nocapture %z, i64* nocapture %x, i64 %y) nounwind
 ; X86-NEXT:    pushl %ebx
 ; X86-NEXT:    pushl %edi
 ; X86-NEXT:    pushl %esi
-; X86-NEXT:    subl $28, %esp
+; X86-NEXT:    subl $24, %esp
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    orl %ecx, %eax
@@ -89,81 +76,63 @@ define i64 @mul1(i64 %n, i64* nocapture %z, i64* nocapture %x, i64 %y) nounwind
 ; X86-NEXT:  # %bb.1: # %for.body.preheader
 ; X86-NEXT:    xorl %eax, %eax
 ; X86-NEXT:    xorl %edx, %edx
-; X86-NEXT:    xorl %ebx, %ebx
-; X86-NEXT:    movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT:    xorl %ebp, %ebp
+; X86-NEXT:    movl $0, (%esp) # 4-byte Folded Spill
 ; X86-NEXT:    .p2align 4, 0x90
 ; X86-NEXT:  .LBB1_2: # %for.body
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl %eax, %ecx
-; X86-NEXT:    movl (%eax,%ebx,8), %ebp
-; X86-NEXT:    movl 4(%eax,%ebx,8), %esi
-; X86-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    movl %ebp, %eax
-; X86-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    mull %ecx
-; X86-NEXT:    movl %edx, %edi
-; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl (%eax,%ebp,8), %esi
+; X86-NEXT:    movl 4(%eax,%ebp,8), %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    movl %esi, %eax
-; X86-NEXT:    mull %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    mull %edi
+; X86-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    mull %edi
 ; X86-NEXT:    movl %edx, %ecx
-; X86-NEXT:    movl %eax, %esi
-; X86-NEXT:    addl %edi, %esi
+; X86-NEXT:    movl %eax, %ebx
+; X86-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
 ; X86-NEXT:    adcl $0, %ecx
-; X86-NEXT:    movl %ebp, %eax
+; X86-NEXT:    movl %esi, %eax
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-NEXT:    mull %edx
-; X86-NEXT:    movl %edx, %ebp
+; X86-NEXT:    movl %edx, %esi
 ; X86-NEXT:    movl %eax, %edi
-; X86-NEXT:    addl %esi, %edi
-; X86-NEXT:    adcl %ecx, %ebp
-; X86-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X86-NEXT:    addl %ebx, %edi
+; X86-NEXT:    adcl %ecx, %esi
+; X86-NEXT:    setb %bl
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X86-NEXT:    mull {{[0-9]+}}(%esp)
-; X86-NEXT:    movl %edx, %ecx
-; X86-NEXT:    movl %eax, %esi
-; X86-NEXT:    addl %ebp, %esi
-; X86-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
-; X86-NEXT:    adcl %eax, %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    xorl %edx, %edx
-; X86-NEXT:    mull %edx
-; X86-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    movl %eax, %ebp
-; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT:    xorl %edx, %edx
-; X86-NEXT:    mull %edx
-; X86-NEXT:    addl %ebp, %eax
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebp
-; X86-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
 ; X86-NEXT:    addl %esi, %eax
-; X86-NEXT:    adcl %ecx, %edx
-; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X86-NEXT:    movzbl %bl, %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT:    adcl %esi, %edx
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
 ; X86-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
 ; X86-NEXT:    adcl $0, %eax
 ; X86-NEXT:    adcl $0, %edx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl %esi, (%ecx,%ebx,8)
-; X86-NEXT:    movl %edi, 4(%ecx,%ebx,8)
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl %ecx, %edi
-; X86-NEXT:    addl $1, %ebx
-; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT:    adcl $0, %esi
-; X86-NEXT:    movl %ebx, %ecx
-; X86-NEXT:    xorl %ebp, %ecx
-; X86-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    xorl %edi, %esi
-; X86-NEXT:    orl %ecx, %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl %ecx, (%esi,%ebp,8)
+; X86-NEXT:    movl %edi, 4(%esi,%ebp,8)
+; X86-NEXT:    addl $1, %ebp
+; X86-NEXT:    movl (%esp), %edi # 4-byte Reload
+; X86-NEXT:    adcl $0, %edi
+; X86-NEXT:    movl %ebp, %esi
+; X86-NEXT:    xorl %ebx, %esi
+; X86-NEXT:    movl %edi, (%esp) # 4-byte Spill
+; X86-NEXT:    xorl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    orl %esi, %edi
 ; X86-NEXT:    jne .LBB1_2
 ; X86-NEXT:  .LBB1_3: # %for.end
 ; X86-NEXT:    xorl %eax, %eax
 ; X86-NEXT:    xorl %edx, %edx
-; X86-NEXT:    addl $28, %esp
+; X86-NEXT:    addl $24, %esp
 ; X86-NEXT:    popl %esi
 ; X86-NEXT:    popl %edi
 ; X86-NEXT:    popl %ebx
diff --git a/llvm/test/CodeGen/X86/i386-shrink-wrapping.ll b/llvm/test/CodeGen/X86/i386-shrink-wrapping.ll
index 95107fb52..f89963cf7 100644
--- a/llvm/test/CodeGen/X86/i386-shrink-wrapping.ll
+++ b/llvm/test/CodeGen/X86/i386-shrink-wrapping.ll
@@ -130,5 +130,5 @@ for.end:                                          ; preds = %for.cond.preheader
 ; Function Attrs: nounwind
 declare i32 @varfunc(i8* nocapture readonly, ...) #0
 
-attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-features"="+mmx,+sse" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-features"="+mmx,+sse" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind }
diff --git a/llvm/test/CodeGen/X86/ifunc-asm.ll b/llvm/test/CodeGen/X86/ifunc-asm.ll
index ef3f5b32a..c1604882f 100644
--- a/llvm/test/CodeGen/X86/ifunc-asm.ll
+++ b/llvm/test/CodeGen/X86/ifunc-asm.ll
@@ -10,6 +10,5 @@ entry:
 ; CHECK-NEXT: foo_ifunc:
 
 @foo = ifunc i32 (i32), i64 ()* @foo_ifunc
-; CHECK: .type foo,@function
-; CHECK-NEXT: .type foo,@gnu_indirect_function
+; CHECK:      .type foo,@gnu_indirect_function
 ; CHECK-NEXT: .set foo, foo_ifunc
diff --git a/llvm/test/CodeGen/X86/illegal-vector-args-return.ll b/llvm/test/CodeGen/X86/illegal-vector-args-return.ll
index ec7547f5f..a31e94aff 100644
--- a/llvm/test/CodeGen/X86/illegal-vector-args-return.ll
+++ b/llvm/test/CodeGen/X86/illegal-vector-args-return.ll
@@ -1,16 +1,24 @@
-; RUN: llc < %s -mattr=+sse2 -mcpu=nehalem | grep "mulpd	%xmm3, %xmm1"
-; RUN: llc < %s -mattr=+sse2 -mcpu=nehalem | grep "mulpd	%xmm2, %xmm0"
-; RUN: llc < %s -mattr=+sse2 -mcpu=nehalem | grep "addps	%xmm3, %xmm1"
-; RUN: llc < %s -mattr=+sse2 -mcpu=nehalem | grep "addps	%xmm2, %xmm0"
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mattr=+sse2 -mcpu=nehalem | FileCheck %s
 
 target triple = "i686-apple-darwin8"
 
 define <4 x double> @foo(<4 x double> %x, <4 x double> %z) {
+; CHECK-LABEL: foo:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    mulpd %xmm2, %xmm0
+; CHECK-NEXT:    mulpd %xmm3, %xmm1
+; CHECK-NEXT:    retl
   %y = fmul <4 x double> %x, %z
   ret <4 x double> %y
 }
 
 define <8 x float> @bar(<8 x float> %x, <8 x float> %z) {
+; CHECK-LABEL: bar:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    addps %xmm2, %xmm0
+; CHECK-NEXT:    addps %xmm3, %xmm1
+; CHECK-NEXT:    retl
   %y = fadd <8 x float> %x, %z
   ret <8 x float> %y
 }
diff --git a/llvm/test/CodeGen/X86/immediate_merging.ll b/llvm/test/CodeGen/X86/immediate_merging.ll
index a6e36c734..1bed1014f 100644
--- a/llvm/test/CodeGen/X86/immediate_merging.ll
+++ b/llvm/test/CodeGen/X86/immediate_merging.ll
@@ -73,6 +73,68 @@ if.end:                                           ; preds = %if.then, %entry
   ret i32 0
 }
 
+; Test PGSO to make sure immediates with multiple users don't get pulled in to
+; instructions.
+define i32 @foo_pgso() !prof !14 {
+; X86-LABEL: foo_pgso:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    movl $1234, %eax # imm = 0x4D2
+; X86-NEXT:    movl %eax, a
+; X86-NEXT:    movl %eax, b
+; X86-NEXT:    movl $12, %eax
+; X86-NEXT:    movl %eax, c
+; X86-NEXT:    cmpl %eax, e
+; X86-NEXT:    jne .LBB1_2
+; X86-NEXT:  # %bb.1: # %if.then
+; X86-NEXT:    movl $1, x
+; X86-NEXT:  .LBB1_2: # %if.end
+; X86-NEXT:    movl $1234, f # imm = 0x4D2
+; X86-NEXT:    movl $555, %eax # imm = 0x22B
+; X86-NEXT:    movl %eax, h
+; X86-NEXT:    addl %eax, i
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: foo_pgso:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    movl $1234, %eax # imm = 0x4D2
+; X64-NEXT:    movl %eax, {{.*}}(%rip)
+; X64-NEXT:    movl %eax, {{.*}}(%rip)
+; X64-NEXT:    movl $12, %eax
+; X64-NEXT:    movl %eax, {{.*}}(%rip)
+; X64-NEXT:    cmpl %eax, {{.*}}(%rip)
+; X64-NEXT:    jne .LBB1_2
+; X64-NEXT:  # %bb.1: # %if.then
+; X64-NEXT:    movl $1, {{.*}}(%rip)
+; X64-NEXT:  .LBB1_2: # %if.end
+; X64-NEXT:    movl $1234, {{.*}}(%rip) # imm = 0x4D2
+; X64-NEXT:    movl $555, %eax # imm = 0x22B
+; X64-NEXT:    movl %eax, {{.*}}(%rip)
+; X64-NEXT:    addl %eax, {{.*}}(%rip)
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    retq
+entry:
+  store i32 1234, i32* @a
+  store i32 1234, i32* @b
+  store i32 12, i32* @c
+  %0 = load i32, i32* @e
+  %cmp = icmp eq i32 %0, 12
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  store i32 1, i32* @x
+  br label %if.end
+
+; New block.. Make sure 1234 isn't live across basic blocks from before.
+if.end:                                           ; preds = %if.then, %entry
+  store i32 1234, i32* @f
+  store i32 555, i32* @h
+  %1 = load i32, i32* @i
+  %add1 = add nsw i32 %1, 555
+  store i32 %add1, i32* @i
+  ret i32 0
+}
+
 ; Test -O2 to make sure that all immediates get pulled in to their users.
 define i32 @foo2() {
 ; X86-LABEL: foo2:
@@ -124,3 +186,47 @@ entry:
   call void @llvm.memset.p0i8.i32(i8* getelementptr inbounds ([100 x i8], [100 x i8]* @AA, i32 0, i32 0), i8 33, i32 24, i1 false)
   ret void
 }
+
+; memset gets lowered in DAG. Constant merging should hoist all the
+; immediates used to store to the individual memory locations. Make
+; sure we don't directly store the immediates.
+define void @foomemset_pgso() !prof !14 {
+; X86-LABEL: foomemset_pgso:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    movl $555819297, %eax # imm = 0x21212121
+; X86-NEXT:    movl %eax, AA+20
+; X86-NEXT:    movl %eax, AA+16
+; X86-NEXT:    movl %eax, AA+12
+; X86-NEXT:    movl %eax, AA+8
+; X86-NEXT:    movl %eax, AA+4
+; X86-NEXT:    movl %eax, AA
+; X86-NEXT:    retl
+;
+; X64-LABEL: foomemset_pgso:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    movabsq $2387225703656530209, %rax # imm = 0x2121212121212121
+; X64-NEXT:    movq %rax, AA+{{.*}}(%rip)
+; X64-NEXT:    movq %rax, AA+{{.*}}(%rip)
+; X64-NEXT:    movq %rax, {{.*}}(%rip)
+; X64-NEXT:    retq
+entry:
+  call void @llvm.memset.p0i8.i32(i8* getelementptr inbounds ([100 x i8], [100 x i8]* @AA, i32 0, i32 0), i8 33, i32 24, i1 false)
+  ret void
+}
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 1, !"ProfileSummary", !1}
+!1 = !{!2, !3, !4, !5, !6, !7, !8, !9}
+!2 = !{!"ProfileFormat", !"InstrProf"}
+!3 = !{!"TotalCount", i64 10000}
+!4 = !{!"MaxCount", i64 10}
+!5 = !{!"MaxInternalCount", i64 1}
+!6 = !{!"MaxFunctionCount", i64 1000}
+!7 = !{!"NumCounts", i64 3}
+!8 = !{!"NumFunctions", i64 3}
+!9 = !{!"DetailedSummary", !10}
+!10 = !{!11, !12, !13}
+!11 = !{i32 10000, i64 100, i32 1}
+!12 = !{i32 999000, i64 100, i32 1}
+!13 = !{i32 999999, i64 1, i32 2}
+!14 = !{!"function_entry_count", i64 0}
diff --git a/llvm/test/CodeGen/X86/immediate_merging64.ll b/llvm/test/CodeGen/X86/immediate_merging64.ll
index 12be8bdff..a807a119e 100644
--- a/llvm/test/CodeGen/X86/immediate_merging64.ll
+++ b/llvm/test/CodeGen/X86/immediate_merging64.ll
@@ -19,6 +19,19 @@ define i1 @imm_multiple_users(i64 %a, i64* %b) optsize {
   ret i1 %cmp
 }
 
+define i1 @imm_multiple_users_pgso(i64 %a, i64* %b) !prof !14 {
+; CHECK-LABEL: imm_multiple_users_pgso:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq $-1, %rax
+; CHECK-NEXT:    movq %rax, (%rsi)
+; CHECK-NEXT:    cmpq %rax, %rdi
+; CHECK-NEXT:    sete %al
+; CHECK-NEXT:    retq
+  store i64 -1, i64* %b, align 8
+  %cmp = icmp eq i64 %a, -1
+  ret i1 %cmp
+}
+
 declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i1)
 
 ; Inlined memsets requiring multiple same-sized stores should be lowered using
@@ -34,3 +47,31 @@ define void @memset_zero(i8* noalias nocapture %D) optsize {
   tail call void @llvm.memset.p0i8.i64(i8* %D, i8 0, i64 15, i1 false)
   ret void
 }
+
+define void @memset_zero_pgso(i8* noalias nocapture %D) !prof !14 {
+; CHECK-LABEL: memset_zero_pgso:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    movq %rax, 7(%rdi)
+; CHECK-NEXT:    movq %rax, (%rdi)
+; CHECK-NEXT:    retq
+  tail call void @llvm.memset.p0i8.i64(i8* %D, i8 0, i64 15, i1 false)
+  ret void
+}
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 1, !"ProfileSummary", !1}
+!1 = !{!2, !3, !4, !5, !6, !7, !8, !9}
+!2 = !{!"ProfileFormat", !"InstrProf"}
+!3 = !{!"TotalCount", i64 10000}
+!4 = !{!"MaxCount", i64 10}
+!5 = !{!"MaxInternalCount", i64 1}
+!6 = !{!"MaxFunctionCount", i64 1000}
+!7 = !{!"NumCounts", i64 3}
+!8 = !{!"NumFunctions", i64 3}
+!9 = !{!"DetailedSummary", !10}
+!10 = !{!11, !12, !13}
+!11 = !{i32 10000, i64 100, i32 1}
+!12 = !{i32 999000, i64 100, i32 1}
+!13 = !{i32 999999, i64 1, i32 2}
+!14 = !{!"function_entry_count", i64 0}
diff --git a/llvm/test/CodeGen/X86/implicit-null-check.ll b/llvm/test/CodeGen/X86/implicit-null-check.ll
index 5b0790f69..6d6b31f86 100644
--- a/llvm/test/CodeGen/X86/implicit-null-check.ll
+++ b/llvm/test/CodeGen/X86/implicit-null-check.ll
@@ -432,14 +432,70 @@ define i32 @imp_null_check_gep_load_with_use_dep(i32* %x, i32 %a) {
   ret i32 %z
 }
 
+;; TODO: We could handle this case as we can lift the fence into the
+;; previous block before the conditional without changing behavior.
+define i32 @imp_null_check_load_fence1(i32* %x) {
+; CHECK-LABEL: imp_null_check_load_fence1:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    testq %rdi, %rdi
+; CHECK-NEXT:    je LBB16_1
+; CHECK-NEXT:  ## %bb.2: ## %not_null
+; CHECK-NEXT:    ##MEMBARRIER
+; CHECK-NEXT:    movl (%rdi), %eax
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  LBB16_1: ## %is_null
+; CHECK-NEXT:    movl $42, %eax
+; CHECK-NEXT:    retq
+
+entry:
+  %c = icmp eq i32* %x, null
+  br i1 %c, label %is_null, label %not_null, !make.implicit !0
+
+is_null:
+  ret i32 42
+
+not_null:
+  fence acquire
+  %t = load i32, i32* %x
+  ret i32 %t
+}
+
+;; TODO: We could handle this case as we can lift the fence into the
+;; previous block before the conditional without changing behavior.
+define i32 @imp_null_check_load_fence2(i32* %x) {
+; CHECK-LABEL: imp_null_check_load_fence2:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    testq %rdi, %rdi
+; CHECK-NEXT:    je LBB17_1
+; CHECK-NEXT:  ## %bb.2: ## %not_null
+; CHECK-NEXT:    mfence
+; CHECK-NEXT:    movl (%rdi), %eax
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  LBB17_1: ## %is_null
+; CHECK-NEXT:    movl $42, %eax
+; CHECK-NEXT:    retq
+
+entry:
+  %c = icmp eq i32* %x, null
+  br i1 %c, label %is_null, label %not_null, !make.implicit !0
+
+is_null:
+  ret i32 42
+
+not_null:
+  fence seq_cst
+  %t = load i32, i32* %x
+  ret i32 %t
+}
+
 define void @imp_null_check_store(i32* %x) {
 ; CHECK-LABEL: imp_null_check_store:
 ; CHECK:       ## %bb.0: ## %entry
 ; CHECK-NEXT:  Ltmp14:
-; CHECK-NEXT:    movl $1, (%rdi) ## on-fault: LBB16_1
+; CHECK-NEXT:    movl $1, (%rdi) ## on-fault: LBB18_1
 ; CHECK-NEXT:  ## %bb.2: ## %not_null
 ; CHECK-NEXT:    retq
-; CHECK-NEXT:  LBB16_1: ## %is_null
+; CHECK-NEXT:  LBB18_1: ## %is_null
 ; CHECK-NEXT:    retq
 
  entry:
@@ -459,10 +515,10 @@ define void @imp_null_check_unordered_store(i32* %x) {
 ; CHECK-LABEL: imp_null_check_unordered_store:
 ; CHECK:       ## %bb.0: ## %entry
 ; CHECK-NEXT:  Ltmp15:
-; CHECK-NEXT:    movl $1, (%rdi) ## on-fault: LBB17_1
+; CHECK-NEXT:    movl $1, (%rdi) ## on-fault: LBB19_1
 ; CHECK-NEXT:  ## %bb.2: ## %not_null
 ; CHECK-NEXT:    retq
-; CHECK-NEXT:  LBB17_1: ## %is_null
+; CHECK-NEXT:  LBB19_1: ## %is_null
 ; CHECK-NEXT:    retq
 
  entry:
@@ -481,10 +537,10 @@ define i32 @imp_null_check_neg_gep_load(i32* %x) {
 ; CHECK-LABEL: imp_null_check_neg_gep_load:
 ; CHECK:       ## %bb.0: ## %entry
 ; CHECK-NEXT:  Ltmp16:
-; CHECK-NEXT:    movl -128(%rdi), %eax ## on-fault: LBB18_1
+; CHECK-NEXT:    movl -128(%rdi), %eax ## on-fault: LBB20_1
 ; CHECK-NEXT:  ## %bb.2: ## %not_null
 ; CHECK-NEXT:    retq
-; CHECK-NEXT:  LBB18_1: ## %is_null
+; CHECK-NEXT:  LBB20_1: ## %is_null
 ; CHECK-NEXT:    movl $42, %eax
 ; CHECK-NEXT:    retq
 
diff --git a/llvm/test/CodeGen/X86/imul.ll b/llvm/test/CodeGen/X86/imul.ll
index 450b19142..e3f94c9b9 100644
--- a/llvm/test/CodeGen/X86/imul.ll
+++ b/llvm/test/CodeGen/X86/imul.ll
@@ -539,12 +539,9 @@ define i64 @testNegOverflow(i64 %a) {
 ;
 ; X86-LABEL: testNegOverflow:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl $1, %edx
-; X86-NEXT:    movl %ecx, %eax
-; X86-NEXT:    mull %edx
-; X86-NEXT:    shll $31, %ecx
-; X86-NEXT:    addl %ecx, %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl %eax, %edx
+; X86-NEXT:    shll $31, %edx
 ; X86-NEXT:    addl {{[0-9]+}}(%esp), %edx
 ; X86-NEXT:    retl
 entry:
diff --git a/llvm/test/CodeGen/X86/inline-asm-A-constraint.ll b/llvm/test/CodeGen/X86/inline-asm-A-constraint.ll
index 2ad011e88..d8e44470f 100644
--- a/llvm/test/CodeGen/X86/inline-asm-A-constraint.ll
+++ b/llvm/test/CodeGen/X86/inline-asm-A-constraint.ll
@@ -22,7 +22,7 @@ entry:
 ; CHECK: lock
 ; CHECK-NEXT: cmpxchg16b
 
-attributes #0 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind }
 
 !llvm.ident = !{!0}
diff --git a/llvm/test/CodeGen/X86/inline-asm-modifier-n.ll b/llvm/test/CodeGen/X86/inline-asm-modifier-n.ll
index 7e48dd2dd..3df7112c8 100644
--- a/llvm/test/CodeGen/X86/inline-asm-modifier-n.ll
+++ b/llvm/test/CodeGen/X86/inline-asm-modifier-n.ll
@@ -1,7 +1,14 @@
-; RUN: llc < %s -mtriple=i686-- -no-integrated-as | grep " 37"
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- -no-integrated-as | FileCheck %s
 ; rdar://7008959
 
 define void @bork() nounwind {
+; CHECK-LABEL: bork:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    BORK 37
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    retl
 entry:
 	tail call void asm sideeffect "BORK ${0:n}", "i,~{dirflag},~{fpsr},~{flags}"(i32 -37) nounwind
 	ret void
diff --git a/llvm/test/CodeGen/X86/inline-asm-mrv.ll b/llvm/test/CodeGen/X86/inline-asm-mrv.ll
index 41a97e3dc..676c145e7 100644
--- a/llvm/test/CodeGen/X86/inline-asm-mrv.ll
+++ b/llvm/test/CodeGen/X86/inline-asm-mrv.ll
@@ -1,13 +1,21 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -no-integrated-as | FileCheck %s
 ; PR2094
-; RUN: llc < %s -no-integrated-as | grep movslq
-; RUN: llc < %s -no-integrated-as | grep addps
-; RUN: llc < %s -no-integrated-as | grep paddd
-; RUN: llc < %s -no-integrated-as | not grep movq
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
 target triple = "x86_64-apple-darwin8"
 
 define i32 @test1(i8* %v, i8* %blk2, i8* %blk1, i32 %stride, i32 %h) nounwind  {
+; CHECK-LABEL: test1:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    movslq %ecx, %rax
+; CHECK-NEXT:    ## InlineAsm Start
+; CHECK-NEXT:    %eax %rcx %rdx %rax %r8d %rdx %rsi
+; CHECK-NEXT:    ## InlineAsm End
+; CHECK-NEXT:    ## InlineAsm Start
+; CHECK-NEXT:    set %eax
+; CHECK-NEXT:    ## InlineAsm End
+; CHECK-NEXT:    retq
 	%tmp12 = sext i32 %stride to i64		; <i64> [#uses=1]
 	%mrv = call {i32, i8*, i8*} asm sideeffect "$0 $1 $2 $3 $4 $5 $6",
          "=r,=r,=r,r,r,r,r"( i64 %tmp12, i32 %h, i8* %blk1, i8* %blk2 ) nounwind
@@ -18,6 +26,13 @@ define i32 @test1(i8* %v, i8* %blk2, i8* %blk1, i32 %stride, i32 %h) nounwind  {
 }
 
 define <4 x float> @test2() nounwind {
+; CHECK-LABEL: test2:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    ## InlineAsm Start
+; CHECK-NEXT:    set %xmm0, %xmm1
+; CHECK-NEXT:    ## InlineAsm End
+; CHECK-NEXT:    addps %xmm1, %xmm0
+; CHECK-NEXT:    retq
 	%mrv = call {<4 x float>, <4 x float>} asm "set $0, $1", "=x,=x"()
 	%a = extractvalue {<4 x float>, <4 x float>} %mrv, 0
 	%b = extractvalue {<4 x float>, <4 x float>} %mrv, 1
@@ -26,6 +41,13 @@ define <4 x float> @test2() nounwind {
 }
 
 define <4 x i32> @test3() nounwind {
+; CHECK-LABEL: test3:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    ## InlineAsm Start
+; CHECK-NEXT:    set %xmm0, %xmm1
+; CHECK-NEXT:    ## InlineAsm End
+; CHECK-NEXT:    paddd %xmm1, %xmm0
+; CHECK-NEXT:    retq
 	%mrv = call {<4 x i32>, <4 x i32>} asm "set $0, $1", "=x,=x"()
 	%a = extractvalue {<4 x i32>, <4 x i32>} %mrv, 0
 	%b = extractvalue {<4 x i32>, <4 x i32>} %mrv, 1
diff --git a/llvm/test/CodeGen/X86/inline-asm-pic.ll b/llvm/test/CodeGen/X86/inline-asm-pic.ll
index 0b5ff08c3..a079748c5 100644
--- a/llvm/test/CodeGen/X86/inline-asm-pic.ll
+++ b/llvm/test/CodeGen/X86/inline-asm-pic.ll
@@ -1,9 +1,19 @@
-; RUN: llc < %s -mtriple=i386-apple-darwin -relocation-model=pic | grep lea
-; RUN: llc < %s -mtriple=i386-apple-darwin -relocation-model=pic | grep call
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i386-apple-darwin -relocation-model=pic | FileCheck %s
 
 @main_q = internal global i8* null		; <i8**> [#uses=1]
 
 define void @func2() nounwind {
+; CHECK-LABEL: func2:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    calll L0$pb
+; CHECK-NEXT:  L0$pb:
+; CHECK-NEXT:    popl %eax
+; CHECK-NEXT:    leal _main_q-L0$pb(%eax), %eax
+; CHECK-NEXT:    ## InlineAsm Start
+; CHECK-NEXT:    movl %eax, %gs:152
+; CHECK-NEXT:    ## InlineAsm End
+; CHECK-NEXT:    retl
 entry:
 	tail call void asm "mov $1,%gs:$0", "=*m,ri,~{dirflag},~{fpsr},~{flags}"(i8** inttoptr (i32 152 to i8**), i8* bitcast (i8** @main_q to i8*)) nounwind
 	ret void
diff --git a/llvm/test/CodeGen/X86/inlineasm-sched-bug.ll b/llvm/test/CodeGen/X86/inlineasm-sched-bug.ll
index 25bf5e07c..b89349696 100644
--- a/llvm/test/CodeGen/X86/inlineasm-sched-bug.ll
+++ b/llvm/test/CodeGen/X86/inlineasm-sched-bug.ll
@@ -1,7 +1,9 @@
 ; PR13504
 ; RUN: llc -mtriple=i686-- -mcpu=atom < %s | FileCheck %s
+; Check that treemap is read before the asm statement.
+; CHECK: movl 8(%{{esp|ebp}})
 ; CHECK: bsfl
-; CHECK-NOT: movl
+; CHECK-NOT: movl 8(%{{esp|ebp}})
 
 define i32 @foo(i32 %treemap) nounwind uwtable {
 entry:
diff --git a/llvm/test/CodeGen/X86/ins_subreg_coalesce-2.ll b/llvm/test/CodeGen/X86/ins_subreg_coalesce-2.ll
index ff8190c45..1866796c7 100644
--- a/llvm/test/CodeGen/X86/ins_subreg_coalesce-2.ll
+++ b/llvm/test/CodeGen/X86/ins_subreg_coalesce-2.ll
@@ -1,6 +1,13 @@
-; RUN: llc < %s -mtriple=x86_64-- | not grep movw
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-- | FileCheck %s
 
 define i16 @test5(i16 %f12) nounwind {
+; CHECK-LABEL: test5:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    shrl $6, %edi
+; CHECK-NEXT:    movsbl %dil, %eax
+; CHECK-NEXT:    # kill: def $ax killed $ax killed $eax
+; CHECK-NEXT:    retq
 	%f11 = shl i16 %f12, 2		; <i16> [#uses=1]
 	%tmp7.25 = ashr i16 %f11, 8		; <i16> [#uses=1]
 	ret i16 %tmp7.25
diff --git a/llvm/test/CodeGen/X86/ins_subreg_coalesce-3.ll b/llvm/test/CodeGen/X86/ins_subreg_coalesce-3.ll
index 31f554d4a..50e3c5c88 100644
--- a/llvm/test/CodeGen/X86/ins_subreg_coalesce-3.ll
+++ b/llvm/test/CodeGen/X86/ins_subreg_coalesce-3.ll
@@ -1,4 +1,5 @@
-; RUN: llc < %s -mtriple=x86_64-- | grep mov | count 3
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-- | FileCheck %s
 
 	%struct.COMPOSITE = type { i8, i16, i16 }
 	%struct.FILE = type { i8*, i32, i32, i16, i16, %struct.__sbuf, i32, i8*, i32 (i8*)*, i32 (i8*, i8*, i32)*, i64 (i8*, i64, i32)*, i32 (i8*, i8*, i32)*, %struct.__sbuf, %struct.__sFILEX*, i32, [3 x i8], [1 x i8], %struct.__sbuf, i32, i64 }
@@ -18,6 +19,44 @@
 	%struct.rec = type { %struct.head_type }
 
 define void @FontChange(i1 %foo) nounwind {
+; CHECK-LABEL: FontChange:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    testb $1, %dil
+; CHECK-NEXT:    je .LBB0_9
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB0_1: # %bb366
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    testb $1, %dil
+; CHECK-NEXT:    jne .LBB0_1
+; CHECK-NEXT:  # %bb.2: # %bb428
+; CHECK-NEXT:    testb $1, %dil
+; CHECK-NEXT:    je .LBB0_9
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB0_3: # %bb650
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    cmpb $0, 0
+; CHECK-NEXT:    je .LBB0_3
+; CHECK-NEXT:  # %bb.4: # %bb662
+; CHECK-NEXT:    movl 0, %eax
+; CHECK-NEXT:    movl %eax, %ecx
+; CHECK-NEXT:    andl $57344, %ecx # imm = 0xE000
+; CHECK-NEXT:    cmpl $8192, %ecx # imm = 0x2000
+; CHECK-NEXT:    jne .LBB0_9
+; CHECK-NEXT:  # %bb.5: # %bb4884
+; CHECK-NEXT:    andl $7168, %eax # imm = 0x1C00
+; CHECK-NEXT:    cmpl $1024, %eax # imm = 0x400
+; CHECK-NEXT:    jne .LBB0_9
+; CHECK-NEXT:  # %bb.6: # %bb4932
+; CHECK-NEXT:    testb $1, %dil
+; CHECK-NEXT:    jne .LBB0_9
+; CHECK-NEXT:  # %bb.7: # %bb4940
+; CHECK-NEXT:    movl 0, %eax
+; CHECK-NEXT:    cmpl $160, %eax
+; CHECK-NEXT:    je .LBB0_9
+; CHECK-NEXT:  # %bb.8: # %bb4940
+; CHECK-NEXT:    cmpl $159, %eax
+; CHECK-NEXT:  .LBB0_9: # %bb4897
+; CHECK-NEXT:    retq
 entry:
 	br i1 %foo, label %bb298, label %bb49
 bb49:		; preds = %entry
diff --git a/llvm/test/CodeGen/X86/insertelement-copytoregs.ll b/llvm/test/CodeGen/X86/insertelement-copytoregs.ll
index 83f0bd2ba..d05fd2d34 100644
--- a/llvm/test/CodeGen/X86/insertelement-copytoregs.ll
+++ b/llvm/test/CodeGen/X86/insertelement-copytoregs.ll
@@ -1,7 +1,12 @@
-; RUN: llc < %s -mtriple=x86_64-- | FileCheck %s
-; CHECK-NOT: IMPLICIT_DEF
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-- | FileCheck %s --implicit-check-not IMPLICIT_DEF
 
 define void @foo(<2 x float>* %p) {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xorps %xmm0, %xmm0
+; CHECK-NEXT:    movlps %xmm0, (%rdi)
+; CHECK-NEXT:    retq
   %t = insertelement <2 x float> undef, float 0.0, i32 0
   %v = insertelement <2 x float> %t,   float 0.0, i32 1
   br label %bb8
diff --git a/llvm/test/CodeGen/X86/insertelement-legalize.ll b/llvm/test/CodeGen/X86/insertelement-legalize.ll
index 8adc3f7e2..1cbbd6121 100644
--- a/llvm/test/CodeGen/X86/insertelement-legalize.ll
+++ b/llvm/test/CodeGen/X86/insertelement-legalize.ll
@@ -1,7 +1,28 @@
-; RUN: llc < %s -mtriple=i686--
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- | FileCheck %s
 
 ; Test to check that we properly legalize an insert vector element
 define void @test(<2 x i64> %val, <2 x i64>* %dst, i64 %x) nounwind {
+; CHECK-LABEL: test:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushl %edi
+; CHECK-NEXT:    pushl %esi
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; CHECK-NEXT:    addl {{[0-9]+}}(%esp), %esi
+; CHECK-NEXT:    adcl {{[0-9]+}}(%esp), %edi
+; CHECK-NEXT:    addl %ecx, %ecx
+; CHECK-NEXT:    adcl %edx, %edx
+; CHECK-NEXT:    movl %ecx, 8(%eax)
+; CHECK-NEXT:    movl %esi, (%eax)
+; CHECK-NEXT:    movl %edx, 12(%eax)
+; CHECK-NEXT:    movl %edi, 4(%eax)
+; CHECK-NEXT:    popl %esi
+; CHECK-NEXT:    popl %edi
+; CHECK-NEXT:    retl
 entry:
 	%tmp4 = insertelement <2 x i64> %val, i64 %x, i32 0		; <<2 x i64>> [#uses=1]
 	%add = add <2 x i64> %tmp4, %val		; <<2 x i64>> [#uses=1]
diff --git a/llvm/test/CodeGen/X86/insertelement-shuffle.ll b/llvm/test/CodeGen/X86/insertelement-shuffle.ll
index 5b4433778..a2b8e2dac 100644
--- a/llvm/test/CodeGen/X86/insertelement-shuffle.ll
+++ b/llvm/test/CodeGen/X86/insertelement-shuffle.ll
@@ -9,7 +9,7 @@ define <8 x float> @insert_subvector_256(i16 %x0, i16 %x1, <8 x float> %v) nounw
 ; X86:       # %bb.0:
 ; X86-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; X86-NEXT:    vpinsrw $1, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; X86-NEXT:    vpbroadcastd %xmm1, %xmm1
+; X86-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,1,1]
 ; X86-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3,4,5,6,7]
 ; X86-NEXT:    retl
 ;
@@ -17,7 +17,7 @@ define <8 x float> @insert_subvector_256(i16 %x0, i16 %x1, <8 x float> %v) nounw
 ; X64:       # %bb.0:
 ; X64-NEXT:    vmovd %edi, %xmm1
 ; X64-NEXT:    vpinsrw $1, %esi, %xmm1, %xmm1
-; X64-NEXT:    vpbroadcastd %xmm1, %xmm1
+; X64-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,1,1]
 ; X64-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3,4,5,6,7]
 ; X64-NEXT:    retq
   %ins1 = insertelement <2 x i16> undef, i16 %x0, i32 0
@@ -30,18 +30,10 @@ define <8 x float> @insert_subvector_256(i16 %x0, i16 %x1, <8 x float> %v) nounw
 define <8 x i64> @insert_subvector_512(i32 %x0, i32 %x1, <8 x i64> %v) nounwind {
 ; X86_AVX256-LABEL: insert_subvector_512:
 ; X86_AVX256:       # %bb.0:
-; X86_AVX256-NEXT:    pushl %ebp
-; X86_AVX256-NEXT:    movl %esp, %ebp
-; X86_AVX256-NEXT:    andl $-8, %esp
-; X86_AVX256-NEXT:    subl $8, %esp
-; X86_AVX256-NEXT:    vmovsd {{.*#+}} xmm2 = mem[0],zero
-; X86_AVX256-NEXT:    vmovlps %xmm2, (%esp)
 ; X86_AVX256-NEXT:    vextracti128 $1, %ymm0, %xmm2
-; X86_AVX256-NEXT:    vpinsrd $0, (%esp), %xmm2, %xmm2
+; X86_AVX256-NEXT:    vpinsrd $0, {{[0-9]+}}(%esp), %xmm2, %xmm2
 ; X86_AVX256-NEXT:    vpinsrd $1, {{[0-9]+}}(%esp), %xmm2, %xmm2
 ; X86_AVX256-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
-; X86_AVX256-NEXT:    movl %ebp, %esp
-; X86_AVX256-NEXT:    popl %ebp
 ; X86_AVX256-NEXT:    retl
 ;
 ; X64_AVX256-LABEL: insert_subvector_512:
diff --git a/llvm/test/CodeGen/X86/insertelement-var-index.ll b/llvm/test/CodeGen/X86/insertelement-var-index.ll
index cbb29202c..a37fe6394 100644
--- a/llvm/test/CodeGen/X86/insertelement-var-index.ll
+++ b/llvm/test/CodeGen/X86/insertelement-var-index.ll
@@ -3,6 +3,22 @@
 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx    | FileCheck %s --check-prefixes=ALL,AVX,AVX1
 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2   | FileCheck %s --check-prefixes=ALL,AVX,AVX2
 
+define <16 x i8> @undef_index(i8 %x) nounwind {
+; ALL-LABEL: undef_index:
+; ALL:       # %bb.0:
+; ALL-NEXT:    retq
+  %ins = insertelement <16 x i8> undef, i8 %x, i64 undef
+  ret <16 x i8> %ins
+}
+
+define <16 x i8> @undef_scalar(<16 x i8> %x, i32 %index) nounwind {
+; ALL-LABEL: undef_scalar:
+; ALL:       # %bb.0:
+; ALL-NEXT:    retq
+  %ins = insertelement <16 x i8> %x, i8 undef, i32 %index
+  ret <16 x i8> %ins
+}
+
 define <16 x i8> @arg_i8_v16i8(i8 %x, i32 %y) nounwind {
 ; SSE-LABEL: arg_i8_v16i8:
 ; SSE:       # %bb.0:
@@ -607,3 +623,31 @@ define <4 x double> @load_f64_v4f64(double* %p, i32 %y) nounwind {
   ret <4 x double> %ins
 }
 
+; Don't die trying to insert to an invalid index.
+
+define i32 @PR44139(<16 x i64>* %p) {
+; ALL-LABEL: PR44139:
+; ALL:       # %bb.0:
+; ALL-NEXT:    movl (%rdi), %eax
+; ALL-NEXT:    leal 2147483647(%rax), %ecx
+; ALL-NEXT:    testl %eax, %eax
+; ALL-NEXT:    cmovnsl %eax, %ecx
+; ALL-NEXT:    andl $-2147483648, %ecx # imm = 0x80000000
+; ALL-NEXT:    addl %eax, %ecx
+; ALL-NEXT:    # kill: def $eax killed $eax killed $rax
+; ALL-NEXT:    xorl %edx, %edx
+; ALL-NEXT:    divl %ecx
+; ALL-NEXT:    retq
+  %L = load <16 x i64>, <16 x i64>* %p
+  %E1 = extractelement <16 x i64> %L, i64 0
+  %tempvector = insertelement <16 x i64> undef, i64 %E1, i32 0
+  %vector = shufflevector <16 x i64> %tempvector, <16 x i64> undef, <16 x i32> zeroinitializer
+  %C3 = icmp sgt i64 9223372036854775807, -9223372036854775808
+  %t0 = trunc <16 x i64> %vector to <16 x i32>
+  %I4 = insertelement <16 x i64> %vector, i64 %E1, i1 %C3
+  store <16 x i64> %I4, <16 x i64>* %p
+  %elt = extractelement <16 x i32> %t0, i32 0
+  %B = srem i32 %elt, -2147483648
+  %B9 = udiv i32 %elt, %B
+  ret i32 %B9
+}
diff --git a/llvm/test/CodeGen/X86/insertps-combine.ll b/llvm/test/CodeGen/X86/insertps-combine.ll
index 6bef76ee9..c98553db6 100644
--- a/llvm/test/CodeGen/X86/insertps-combine.ll
+++ b/llvm/test/CodeGen/X86/insertps-combine.ll
@@ -269,12 +269,12 @@ define float @extract_zero_insertps_z0z7(<4 x float> %a0, <4 x float> %a1) {
 define float @extract_lane_insertps_5123(<4 x float> %a0, <4 x float> *%p1) {
 ; SSE-LABEL: extract_lane_insertps_5123:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE-NEXT:    movshdup {{.*#+}} xmm0 = mem[1,1,3,3]
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: extract_lane_insertps_5123:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-NEXT:    vmovshdup {{.*#+}} xmm0 = mem[1,1,3,3]
 ; AVX-NEXT:    retq
   %a1 = load <4 x float>, <4 x float> *%p1
   %res = call <4 x float> @llvm.x86.sse41.insertps(<4 x float> %a0, <4 x float> %a1, i8 64)
@@ -285,12 +285,13 @@ define float @extract_lane_insertps_5123(<4 x float> %a0, <4 x float> *%p1) {
 define float @extract_lane_insertps_6123(<4 x float> %a0, <4 x float> *%p1) {
 ; SSE-LABEL: extract_lane_insertps_6123:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE-NEXT:    movaps (%rdi), %xmm0
+; SSE-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: extract_lane_insertps_6123:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-NEXT:    vpermilpd {{.*#+}} xmm0 = mem[1,0]
 ; AVX-NEXT:    retq
   %a1 = load <4 x float>, <4 x float> *%p1
   %res = call <4 x float> @llvm.x86.sse41.insertps(<4 x float> %a0, <4 x float> %a1, i8 128)
diff --git a/llvm/test/CodeGen/X86/ipra-reg-usage.ll b/llvm/test/CodeGen/X86/ipra-reg-usage.ll
index 3a877bfb3..7d0560040 100644
--- a/llvm/test/CodeGen/X86/ipra-reg-usage.ll
+++ b/llvm/test/CodeGen/X86/ipra-reg-usage.ll
@@ -3,7 +3,7 @@
 target triple = "x86_64-unknown-unknown"
 declare void @bar1()
 define preserve_allcc void @foo()#0 {
-; CHECK: foo Clobbered Registers: $cs $df $ds $eflags $eip $eiz $es $fpcw $fpsw $fs $gs $hip $ip $rip $riz $ss $ssp $bnd0 $bnd1 $bnd2 $bnd3 $cr0 $cr1 $cr2 $cr3 $cr4 $cr5 $cr6 $cr7 $cr8 $cr9 $cr10 $cr11 $cr12 $cr13 $cr14 $cr15 $dr0 $dr1 $dr2 $dr3 $dr4 $dr5 $dr6 $dr7 $dr8 $dr9 $dr10 $dr11 $dr12 $dr13 $dr14 $dr15 $fp0 $fp1 $fp2 $fp3 $fp4 $fp5 $fp6 $fp7 $k0 $k1 $k2 $k3 $k4 $k5 $k6 $k7 $mm0 $mm1 $mm2 $mm3 $mm4 $mm5 $mm6 $mm7 $r11 $st0 $st1 $st2 $st3 $st4 $st5 $st6 $st7 $xmm16 $xmm17 $xmm18 $xmm19 $xmm20 $xmm21 $xmm22 $xmm23 $xmm24 $xmm25 $xmm26 $xmm27 $xmm28 $xmm29 $xmm30 $xmm31 $ymm0 $ymm1 $ymm2 $ymm3 $ymm4 $ymm5 $ymm6 $ymm7 $ymm8 $ymm9 $ymm10 $ymm11 $ymm12 $ymm13 $ymm14 $ymm15 $ymm16 $ymm17 $ymm18 $ymm19 $ymm20 $ymm21 $ymm22 $ymm23 $ymm24 $ymm25 $ymm26 $ymm27 $ymm28 $ymm29 $ymm30 $ymm31 $zmm0 $zmm1 $zmm2 $zmm3 $zmm4 $zmm5 $zmm6 $zmm7 $zmm8 $zmm9 $zmm10 $zmm11 $zmm12 $zmm13 $zmm14 $zmm15 $zmm16 $zmm17 $zmm18 $zmm19 $zmm20 $zmm21 $zmm22 $zmm23 $zmm24 $zmm25 $zmm26 $zmm27 $zmm28 $zmm29 $zmm30 $zmm31 $r11b $r11bh $r11d $r11w $r11wh
+; CHECK: foo Clobbered Registers: $cs $df $ds $eflags $eip $eiz $es $fpcw $fpsw $fs $gs $hip $ip $mxcsr $rip $riz $ss $ssp $bnd0 $bnd1 $bnd2 $bnd3 $cr0 $cr1 $cr2 $cr3 $cr4 $cr5 $cr6 $cr7 $cr8 $cr9 $cr10 $cr11 $cr12 $cr13 $cr14 $cr15 $dr0 $dr1 $dr2 $dr3 $dr4 $dr5 $dr6 $dr7 $dr8 $dr9 $dr10 $dr11 $dr12 $dr13 $dr14 $dr15 $fp0 $fp1 $fp2 $fp3 $fp4 $fp5 $fp6 $fp7 $k0 $k1 $k2 $k3 $k4 $k5 $k6 $k7 $mm0 $mm1 $mm2 $mm3 $mm4 $mm5 $mm6 $mm7 $r11 $st0 $st1 $st2 $st3 $st4 $st5 $st6 $st7 $xmm16 $xmm17 $xmm18 $xmm19 $xmm20 $xmm21 $xmm22 $xmm23 $xmm24 $xmm25 $xmm26 $xmm27 $xmm28 $xmm29 $xmm30 $xmm31 $ymm0 $ymm1 $ymm2 $ymm3 $ymm4 $ymm5 $ymm6 $ymm7 $ymm8 $ymm9 $ymm10 $ymm11 $ymm12 $ymm13 $ymm14 $ymm15 $ymm16 $ymm17 $ymm18 $ymm19 $ymm20 $ymm21 $ymm22 $ymm23 $ymm24 $ymm25 $ymm26 $ymm27 $ymm28 $ymm29 $ymm30 $ymm31 $zmm0 $zmm1 $zmm2 $zmm3 $zmm4 $zmm5 $zmm6 $zmm7 $zmm8 $zmm9 $zmm10 $zmm11 $zmm12 $zmm13 $zmm14 $zmm15 $zmm16 $zmm17 $zmm18 $zmm19 $zmm20 $zmm21 $zmm22 $zmm23 $zmm24 $zmm25 $zmm26 $zmm27 $zmm28 $zmm29 $zmm30 $zmm31 $r11b $r11bh $r11d $r11w $r11wh
   call void @bar1()
   call void @bar2()
   ret void
diff --git a/llvm/test/CodeGen/X86/isel-blendi-gettargetconstant.ll b/llvm/test/CodeGen/X86/isel-blendi-gettargetconstant.ll
new file mode 100644
index 000000000..b43e2e5a5
--- /dev/null
+++ b/llvm/test/CodeGen/X86/isel-blendi-gettargetconstant.ll
@@ -0,0 +1,19 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-linux-gnu -mattr=sse4.2 | FileCheck %s
+
+define void @csrot_(float* %0) {
+; CHECK-LABEL: csrot_:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; CHECK-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; CHECK-NEXT:    xorps %xmm0, %xmm1
+; CHECK-NEXT:    blendps {{.*#+}} xmm1 = xmm1[0],mem[1,2,3]
+; CHECK-NEXT:    movlps %xmm1, (%rax)
+; CHECK-NEXT:    retq
+1:
+  %2 = load float, float* %0, align 4
+  %3 = fsub float -0.000000e+00, %2
+  %4 = insertelement <2 x float> <float undef, float -0.000000e+00>, float %3, i32 0
+  store <2 x float> %4, <2 x float>* undef, align 8
+  ret void
+}
diff --git a/llvm/test/CodeGen/X86/isel-sink2.ll b/llvm/test/CodeGen/X86/isel-sink2.ll
index e7236336b..5fe40982f 100644
--- a/llvm/test/CodeGen/X86/isel-sink2.ll
+++ b/llvm/test/CodeGen/X86/isel-sink2.ll
@@ -1,8 +1,18 @@
-; RUN: llc < %s -mtriple=i686-- > %t
-; RUN: grep "movb.7(%...)" %t
-; RUN: not grep leal %t
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- | FileCheck %s
 
 define i8 @test(i32 *%P) nounwind {
+; CHECK-LABEL: test:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    cmpb $0, 4(%eax)
+; CHECK-NEXT:    je .LBB0_1
+; CHECK-NEXT:  # %bb.2: # %F
+; CHECK-NEXT:    movb 7(%eax), %al
+; CHECK-NEXT:    retl
+; CHECK-NEXT:  .LBB0_1: # %TB
+; CHECK-NEXT:    movb $4, %al
+; CHECK-NEXT:    retl
   %Q = getelementptr i32, i32* %P, i32 1
   %R = bitcast i32* %Q to i8*
   %S = load i8, i8* %R
diff --git a/llvm/test/CodeGen/X86/isnan.ll b/llvm/test/CodeGen/X86/isnan.ll
index 98766838f..d46aa279e 100644
--- a/llvm/test/CodeGen/X86/isnan.ll
+++ b/llvm/test/CodeGen/X86/isnan.ll
@@ -1,8 +1,18 @@
-; RUN: llc < %s -mtriple=i686-- | not grep call
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- | FileCheck %s
 
 declare i1 @llvm.isunordered.f64(double)
 
 define i1 @test_isnan(double %X) {
+; CHECK-LABEL: test_isnan:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fldl {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fucomp %st(0)
+; CHECK-NEXT:    fnstsw %ax
+; CHECK-NEXT:    # kill: def $ah killed $ah killed $ax
+; CHECK-NEXT:    sahf
+; CHECK-NEXT:    setp %al
+; CHECK-NEXT:    retl
         %R = fcmp uno double %X, %X             ; <i1> [#uses=1]
         ret i1 %R
 }
diff --git a/llvm/test/CodeGen/X86/isnan2.ll b/llvm/test/CodeGen/X86/isnan2.ll
index e28f8450a..fd8229d6c 100644
--- a/llvm/test/CodeGen/X86/isnan2.ll
+++ b/llvm/test/CodeGen/X86/isnan2.ll
@@ -1,8 +1,16 @@
-; RUN: llc < %s -mtriple=i686-- -mcpu=yonah | not grep pxor
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- -mcpu=yonah | FileCheck %s
 
 ; This should not need to materialize 0.0 to evaluate the condition.
 
 define i32 @test(double %X) nounwind  {
+; CHECK-LABEL: test:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    ucomisd %xmm0, %xmm0
+; CHECK-NEXT:    setp %al
+; CHECK-NEXT:    retl
 entry:
 	%tmp6 = fcmp uno double %X, 0.000000e+00		; <i1> [#uses=1]
 	%tmp67 = zext i1 %tmp6 to i32		; <i32> [#uses=1]
diff --git a/llvm/test/CodeGen/X86/ispositive.ll b/llvm/test/CodeGen/X86/ispositive.ll
index ac9bd4383..412954662 100644
--- a/llvm/test/CodeGen/X86/ispositive.ll
+++ b/llvm/test/CodeGen/X86/ispositive.ll
@@ -1,6 +1,12 @@
-; RUN: llc < %s -mtriple=i686-- | grep "shrl.*31"
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- | FileCheck %s
 
 define i32 @test1(i32 %X) {
+; CHECK-LABEL: test1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    shrl $31, %eax
+; CHECK-NEXT:    retl
 entry:
         icmp slt i32 %X, 0              ; <i1>:0 [#uses=1]
         zext i1 %0 to i32               ; <i32>:1 [#uses=1]
diff --git a/llvm/test/CodeGen/X86/known-bits-vector.ll b/llvm/test/CodeGen/X86/known-bits-vector.ll
index 067ac9a6f..a1606e93e 100644
--- a/llvm/test/CodeGen/X86/known-bits-vector.ll
+++ b/llvm/test/CodeGen/X86/known-bits-vector.ll
@@ -438,7 +438,7 @@ define <4 x float> @knownbits_smax_smin_shuffle_uitofp(<4 x i32> %a0) {
 ; X32-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],mem[1],xmm0[2],mem[3],xmm0[4],mem[5],xmm0[6],mem[7]
 ; X32-NEXT:    vpsrld $16, %xmm0, %xmm0
 ; X32-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],mem[1],xmm0[2],mem[3],xmm0[4],mem[5],xmm0[6],mem[7]
-; X32-NEXT:    vaddps {{\.LCPI.*}}, %xmm0, %xmm0
+; X32-NEXT:    vsubps {{\.LCPI.*}}, %xmm0, %xmm0
 ; X32-NEXT:    vaddps %xmm0, %xmm1, %xmm0
 ; X32-NEXT:    retl
 ;
@@ -450,7 +450,7 @@ define <4 x float> @knownbits_smax_smin_shuffle_uitofp(<4 x i32> %a0) {
 ; X64-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],mem[1],xmm0[2],mem[3],xmm0[4],mem[5],xmm0[6],mem[7]
 ; X64-NEXT:    vpsrld $16, %xmm0, %xmm0
 ; X64-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],mem[1],xmm0[2],mem[3],xmm0[4],mem[5],xmm0[6],mem[7]
-; X64-NEXT:    vaddps {{.*}}(%rip), %xmm0, %xmm0
+; X64-NEXT:    vsubps {{.*}}(%rip), %xmm0, %xmm0
 ; X64-NEXT:    vaddps %xmm0, %xmm1, %xmm0
 ; X64-NEXT:    retq
   %1 = call <4 x i32> @llvm.x86.sse41.pminsd(<4 x i32> %a0, <4 x i32> <i32 0, i32 -65535, i32 -65535, i32 0>)
@@ -548,7 +548,7 @@ define <4 x float> @knownbits_abs_uitofp(<4 x i32> %a0) {
 ; X32-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],mem[1],xmm0[2],mem[3],xmm0[4],mem[5],xmm0[6],mem[7]
 ; X32-NEXT:    vpsrld $16, %xmm0, %xmm0
 ; X32-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],mem[1],xmm0[2],mem[3],xmm0[4],mem[5],xmm0[6],mem[7]
-; X32-NEXT:    vaddps {{\.LCPI.*}}, %xmm0, %xmm0
+; X32-NEXT:    vsubps {{\.LCPI.*}}, %xmm0, %xmm0
 ; X32-NEXT:    vaddps %xmm0, %xmm1, %xmm0
 ; X32-NEXT:    retl
 ;
@@ -558,7 +558,7 @@ define <4 x float> @knownbits_abs_uitofp(<4 x i32> %a0) {
 ; X64-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],mem[1],xmm0[2],mem[3],xmm0[4],mem[5],xmm0[6],mem[7]
 ; X64-NEXT:    vpsrld $16, %xmm0, %xmm0
 ; X64-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],mem[1],xmm0[2],mem[3],xmm0[4],mem[5],xmm0[6],mem[7]
-; X64-NEXT:    vaddps {{.*}}(%rip), %xmm0, %xmm0
+; X64-NEXT:    vsubps {{.*}}(%rip), %xmm0, %xmm0
 ; X64-NEXT:    vaddps %xmm0, %xmm1, %xmm0
 ; X64-NEXT:    retq
   %1 = sub <4 x i32> zeroinitializer, %a0
@@ -664,3 +664,25 @@ define <4 x float> @knownbits_lshr_and_select_shuffle_uitofp(<4 x i32> %a0, <4 x
   %6 = uitofp <4 x i32> %5 to <4 x float>
   ret <4 x float> %6
 }
+
+define <2 x double> @knownbits_lshr_subvector_uitofp(<4 x i32> %x)  {
+; X32-LABEL: knownbits_lshr_subvector_uitofp:
+; X32:       # %bb.0:
+; X32-NEXT:    vpsrld $2, %xmm0, %xmm1
+; X32-NEXT:    vpsrld $1, %xmm0, %xmm0
+; X32-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5,6,7]
+; X32-NEXT:    vcvtdq2pd %xmm0, %xmm0
+; X32-NEXT:    retl
+;
+; X64-LABEL: knownbits_lshr_subvector_uitofp:
+; X64:       # %bb.0:
+; X64-NEXT:    vpsrld $2, %xmm0, %xmm1
+; X64-NEXT:    vpsrld $1, %xmm0, %xmm0
+; X64-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5,6,7]
+; X64-NEXT:    vcvtdq2pd %xmm0, %xmm0
+; X64-NEXT:    retq
+  %1 = lshr <4 x i32> %x, <i32 1, i32 2, i32 0, i32 0>
+  %2 = shufflevector <4 x i32> %1, <4 x i32> undef, <2 x i32> <i32 0, i32 1>
+  %3 = uitofp <2 x i32> %2 to <2 x double>
+  ret <2 x double> %3
+}
diff --git a/llvm/test/CodeGen/X86/known-bits.ll b/llvm/test/CodeGen/X86/known-bits.ll
index 8f3b98325..8e2e9c05f 100644
--- a/llvm/test/CodeGen/X86/known-bits.ll
+++ b/llvm/test/CodeGen/X86/known-bits.ll
@@ -5,100 +5,44 @@
 define void @knownbits_zext_in_reg(i8*) nounwind {
 ; X32-LABEL: knownbits_zext_in_reg:
 ; X32:       # %bb.0: # %BB
-; X32-NEXT:    pushl %ebp
 ; X32-NEXT:    pushl %ebx
-; X32-NEXT:    pushl %edi
-; X32-NEXT:    pushl %esi
-; X32-NEXT:    subl $16, %esp
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-NEXT:    movzbl (%eax), %ecx
 ; X32-NEXT:    imull $101, %ecx, %eax
 ; X32-NEXT:    shrl $14, %eax
-; X32-NEXT:    imull $177, %ecx, %ecx
-; X32-NEXT:    shrl $14, %ecx
-; X32-NEXT:    movzbl %al, %eax
-; X32-NEXT:    vpxor %xmm0, %xmm0, %xmm0
-; X32-NEXT:    vpinsrd $1, %eax, %xmm0, %xmm1
-; X32-NEXT:    vbroadcastss {{.*#+}} xmm2 = [3.57331108E-43,3.57331108E-43,3.57331108E-43,3.57331108E-43]
-; X32-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; X32-NEXT:    movzbl %cl, %eax
-; X32-NEXT:    vpinsrd $1, %eax, %xmm0, %xmm0
-; X32-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; X32-NEXT:    vpextrd $1, %xmm1, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    vpextrd $1, %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    xorl %ecx, %ecx
-; X32-NEXT:    vmovd %xmm1, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    vmovd %xmm0, (%esp) # 4-byte Folded Spill
-; X32-NEXT:    vpextrd $2, %xmm1, %edi
-; X32-NEXT:    vpextrd $2, %xmm0, %esi
-; X32-NEXT:    vpextrd $3, %xmm1, %ebx
-; X32-NEXT:    vpextrd $3, %xmm0, %ebp
+; X32-NEXT:    imull $177, %ecx, %edx
+; X32-NEXT:    shrl $14, %edx
+; X32-NEXT:    movzbl %al, %ecx
+; X32-NEXT:    xorl %ebx, %ebx
 ; X32-NEXT:    .p2align 4, 0x90
 ; X32-NEXT:  .LBB0_1: # %CF
 ; X32-NEXT:    # =>This Loop Header: Depth=1
 ; X32-NEXT:    # Child Loop BB0_2 Depth 2
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    xorl %edx, %edx
-; X32-NEXT:    divl {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    xorl %edx, %edx
-; X32-NEXT:    divl (%esp) # 4-byte Folded Reload
-; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    xorl %edx, %edx
-; X32-NEXT:    divl %esi
-; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    xorl %edx, %edx
-; X32-NEXT:    divl %ebp
+; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    divb %dl
 ; X32-NEXT:    .p2align 4, 0x90
 ; X32-NEXT:  .LBB0_2: # %CF237
 ; X32-NEXT:    # Parent Loop BB0_1 Depth=1
 ; X32-NEXT:    # => This Inner Loop Header: Depth=2
-; X32-NEXT:    testb %cl, %cl
+; X32-NEXT:    testb %bl, %bl
 ; X32-NEXT:    jne .LBB0_2
 ; X32-NEXT:    jmp .LBB0_1
 ;
 ; X64-LABEL: knownbits_zext_in_reg:
 ; X64:       # %bb.0: # %BB
-; X64-NEXT:    pushq %rbp
-; X64-NEXT:    pushq %rbx
 ; X64-NEXT:    movzbl (%rdi), %eax
 ; X64-NEXT:    imull $101, %eax, %ecx
 ; X64-NEXT:    shrl $14, %ecx
-; X64-NEXT:    imull $177, %eax, %eax
-; X64-NEXT:    shrl $14, %eax
+; X64-NEXT:    imull $177, %eax, %edx
+; X64-NEXT:    shrl $14, %edx
 ; X64-NEXT:    movzbl %cl, %ecx
-; X64-NEXT:    vpxor %xmm0, %xmm0, %xmm0
-; X64-NEXT:    vpinsrd $1, %ecx, %xmm0, %xmm1
-; X64-NEXT:    vbroadcastss {{.*#+}} xmm2 = [3.57331108E-43,3.57331108E-43,3.57331108E-43,3.57331108E-43]
-; X64-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; X64-NEXT:    movzbl %al, %eax
-; X64-NEXT:    vpinsrd $1, %eax, %xmm0, %xmm0
-; X64-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; X64-NEXT:    vpextrd $1, %xmm1, %r8d
-; X64-NEXT:    vpextrd $1, %xmm0, %r9d
 ; X64-NEXT:    xorl %esi, %esi
-; X64-NEXT:    vmovd %xmm1, %r10d
-; X64-NEXT:    vmovd %xmm0, %r11d
-; X64-NEXT:    vpextrd $2, %xmm1, %edi
-; X64-NEXT:    vpextrd $2, %xmm0, %ebx
-; X64-NEXT:    vpextrd $3, %xmm1, %ecx
-; X64-NEXT:    vpextrd $3, %xmm0, %ebp
 ; X64-NEXT:    .p2align 4, 0x90
 ; X64-NEXT:  .LBB0_1: # %CF
 ; X64-NEXT:    # =>This Loop Header: Depth=1
 ; X64-NEXT:    # Child Loop BB0_2 Depth 2
-; X64-NEXT:    movl %r8d, %eax
-; X64-NEXT:    xorl %edx, %edx
-; X64-NEXT:    divl %r9d
-; X64-NEXT:    movl %r10d, %eax
-; X64-NEXT:    xorl %edx, %edx
-; X64-NEXT:    divl %r11d
-; X64-NEXT:    movl %edi, %eax
-; X64-NEXT:    xorl %edx, %edx
-; X64-NEXT:    divl %ebx
 ; X64-NEXT:    movl %ecx, %eax
-; X64-NEXT:    xorl %edx, %edx
-; X64-NEXT:    divl %ebp
+; X64-NEXT:    divb %dl
 ; X64-NEXT:    .p2align 4, 0x90
 ; X64-NEXT:  .LBB0_2: # %CF237
 ; X64-NEXT:    # Parent Loop BB0_1 Depth=1
@@ -246,26 +190,22 @@ define {i32, i1} @knownbits_uaddo_saddo(i64 %a0, i64 %a1) nounwind {
 define {i32, i1} @knownbits_usubo_ssubo(i64 %a0, i64 %a1) nounwind {
 ; X32-LABEL: knownbits_usubo_ssubo:
 ; X32:       # %bb.0:
-; X32-NEXT:    pushl %ebx
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    movl %ecx, %edx
-; X32-NEXT:    subl %eax, %edx
-; X32-NEXT:    setb %bl
-; X32-NEXT:    testl %eax, %eax
-; X32-NEXT:    setns %al
+; X32-NEXT:    cmpl %eax, %ecx
+; X32-NEXT:    setb %dh
+; X32-NEXT:    setns %dl
 ; X32-NEXT:    testl %ecx, %ecx
 ; X32-NEXT:    setns %cl
-; X32-NEXT:    cmpb %al, %cl
-; X32-NEXT:    setne %al
-; X32-NEXT:    testl %edx, %edx
-; X32-NEXT:    setns %dl
 ; X32-NEXT:    cmpb %dl, %cl
+; X32-NEXT:    setne %ch
+; X32-NEXT:    testl %eax, %eax
+; X32-NEXT:    setns %al
+; X32-NEXT:    cmpb %al, %cl
 ; X32-NEXT:    setne %dl
-; X32-NEXT:    andb %al, %dl
-; X32-NEXT:    orb %bl, %dl
+; X32-NEXT:    andb %ch, %dl
+; X32-NEXT:    orb %dh, %dl
 ; X32-NEXT:    xorl %eax, %eax
-; X32-NEXT:    popl %ebx
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: knownbits_usubo_ssubo:
diff --git a/llvm/test/CodeGen/X86/known-signbits-vector.ll b/llvm/test/CodeGen/X86/known-signbits-vector.ll
index 846d97d55..1d2afebe2 100644
--- a/llvm/test/CodeGen/X86/known-signbits-vector.ll
+++ b/llvm/test/CodeGen/X86/known-signbits-vector.ll
@@ -1,12 +1,13 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=X32
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=X64
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=CHECK,X86
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=CHECK,X64,X64-AVX1
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=CHECK,X64,X64-AVX2
 
 define <2 x double> @signbits_sext_v2i64_sitofp_v2f64(i32 %a0, i32 %a1) nounwind {
-; X32-LABEL: signbits_sext_v2i64_sitofp_v2f64:
-; X32:       # %bb.0:
-; X32-NEXT:    vcvtdq2pd {{[0-9]+}}(%esp), %xmm0
-; X32-NEXT:    retl
+; X86-LABEL: signbits_sext_v2i64_sitofp_v2f64:
+; X86:       # %bb.0:
+; X86-NEXT:    vcvtdq2pd {{[0-9]+}}(%esp), %xmm0
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: signbits_sext_v2i64_sitofp_v2f64:
 ; X64:       # %bb.0:
@@ -23,16 +24,16 @@ define <2 x double> @signbits_sext_v2i64_sitofp_v2f64(i32 %a0, i32 %a1) nounwind
 }
 
 define <4 x float> @signbits_sext_v4i64_sitofp_v4f32(i8 signext %a0, i16 signext %a1, i32 %a2, i32 %a3) nounwind {
-; X32-LABEL: signbits_sext_v4i64_sitofp_v4f32:
-; X32:       # %bb.0:
-; X32-NEXT:    movswl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movsbl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    vmovd %ecx, %xmm0
-; X32-NEXT:    vpinsrd $1, %eax, %xmm0, %xmm0
-; X32-NEXT:    vpinsrd $2, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; X32-NEXT:    vpinsrd $3, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; X32-NEXT:    vcvtdq2ps %xmm0, %xmm0
-; X32-NEXT:    retl
+; X86-LABEL: signbits_sext_v4i64_sitofp_v4f32:
+; X86:       # %bb.0:
+; X86-NEXT:    movswl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movsbl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    vmovd %ecx, %xmm0
+; X86-NEXT:    vpinsrd $1, %eax, %xmm0, %xmm0
+; X86-NEXT:    vpinsrd $2, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; X86-NEXT:    vpinsrd $3, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; X86-NEXT:    vcvtdq2ps %xmm0, %xmm0
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: signbits_sext_v4i64_sitofp_v4f32:
 ; X64:       # %bb.0:
@@ -54,16 +55,70 @@ define <4 x float> @signbits_sext_v4i64_sitofp_v4f32(i8 signext %a0, i16 signext
   ret <4 x float> %9
 }
 
+define <4 x double> @signbits_ashr_sitofp(<4 x i64> %a0) nounwind {
+; X86-LABEL: signbits_ashr_sitofp:
+; X86:       # %bb.0:
+; X86-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; X86-NEXT:    vpsrlq $36, %xmm1, %xmm2
+; X86-NEXT:    vpsrlq $35, %xmm1, %xmm1
+; X86-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm2[4,5,6,7]
+; X86-NEXT:    vmovdqa {{.*#+}} xmm2 = [268435456,0,134217728,0]
+; X86-NEXT:    vpxor %xmm2, %xmm1, %xmm1
+; X86-NEXT:    vpsubq %xmm2, %xmm1, %xmm1
+; X86-NEXT:    vpsrlq $34, %xmm0, %xmm2
+; X86-NEXT:    vpsrlq $33, %xmm0, %xmm0
+; X86-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm2[4,5,6,7]
+; X86-NEXT:    vmovdqa {{.*#+}} xmm2 = [1073741824,0,536870912,0]
+; X86-NEXT:    vpxor %xmm2, %xmm0, %xmm0
+; X86-NEXT:    vpsubq %xmm2, %xmm0, %xmm0
+; X86-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
+; X86-NEXT:    vcvtdq2pd %xmm0, %ymm0
+; X86-NEXT:    retl
+;
+; X64-AVX1-LABEL: signbits_ashr_sitofp:
+; X64-AVX1:       # %bb.0:
+; X64-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; X64-AVX1-NEXT:    vpsrlq $36, %xmm1, %xmm2
+; X64-AVX1-NEXT:    vpsrlq $35, %xmm1, %xmm1
+; X64-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm2[4,5,6,7]
+; X64-AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [268435456,134217728]
+; X64-AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm1
+; X64-AVX1-NEXT:    vpsubq %xmm2, %xmm1, %xmm1
+; X64-AVX1-NEXT:    vpsrlq $34, %xmm0, %xmm2
+; X64-AVX1-NEXT:    vpsrlq $33, %xmm0, %xmm0
+; X64-AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm2[4,5,6,7]
+; X64-AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [1073741824,536870912]
+; X64-AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
+; X64-AVX1-NEXT:    vpsubq %xmm2, %xmm0, %xmm0
+; X64-AVX1-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
+; X64-AVX1-NEXT:    vcvtdq2pd %xmm0, %ymm0
+; X64-AVX1-NEXT:    retq
+;
+; X64-AVX2-LABEL: signbits_ashr_sitofp:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    vpsrlvq {{.*}}(%rip), %ymm0, %ymm0
+; X64-AVX2-NEXT:    vmovdqa {{.*#+}} ymm1 = [1073741824,536870912,268435456,134217728]
+; X64-AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; X64-AVX2-NEXT:    vpsubq %ymm1, %ymm0, %ymm0
+; X64-AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; X64-AVX2-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
+; X64-AVX2-NEXT:    vcvtdq2pd %xmm0, %ymm0
+; X64-AVX2-NEXT:    retq
+  %1 = ashr <4 x i64> %a0, <i64 33, i64 34, i64 35, i64 36>
+  %2 = sitofp <4 x i64> %1 to <4 x double>
+  ret <4 x double> %2
+}
+
 define float @signbits_ashr_extract_sitofp_0(<2 x i64> %a0) nounwind {
-; X32-LABEL: signbits_ashr_extract_sitofp_0:
-; X32:       # %bb.0:
-; X32-NEXT:    pushl %eax
-; X32-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; X32-NEXT:    vcvtdq2ps %xmm0, %xmm0
-; X32-NEXT:    vmovss %xmm0, (%esp)
-; X32-NEXT:    flds (%esp)
-; X32-NEXT:    popl %eax
-; X32-NEXT:    retl
+; X86-LABEL: signbits_ashr_extract_sitofp_0:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; X86-NEXT:    vcvtdq2ps %xmm0, %xmm0
+; X86-NEXT:    vmovss %xmm0, (%esp)
+; X86-NEXT:    flds (%esp)
+; X86-NEXT:    popl %eax
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: signbits_ashr_extract_sitofp_0:
 ; X64:       # %bb.0:
@@ -77,15 +132,15 @@ define float @signbits_ashr_extract_sitofp_0(<2 x i64> %a0) nounwind {
 }
 
 define float @signbits_ashr_extract_sitofp_1(<2 x i64> %a0) nounwind {
-; X32-LABEL: signbits_ashr_extract_sitofp_1:
-; X32:       # %bb.0:
-; X32-NEXT:    pushl %eax
-; X32-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; X32-NEXT:    vcvtdq2ps %xmm0, %xmm0
-; X32-NEXT:    vmovss %xmm0, (%esp)
-; X32-NEXT:    flds (%esp)
-; X32-NEXT:    popl %eax
-; X32-NEXT:    retl
+; X86-LABEL: signbits_ashr_extract_sitofp_1:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; X86-NEXT:    vcvtdq2ps %xmm0, %xmm0
+; X86-NEXT:    vmovss %xmm0, (%esp)
+; X86-NEXT:    flds (%esp)
+; X86-NEXT:    popl %eax
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: signbits_ashr_extract_sitofp_1:
 ; X64:       # %bb.0:
@@ -99,17 +154,17 @@ define float @signbits_ashr_extract_sitofp_1(<2 x i64> %a0) nounwind {
 }
 
 define float @signbits_ashr_shl_extract_sitofp(<2 x i64> %a0) nounwind {
-; X32-LABEL: signbits_ashr_shl_extract_sitofp:
-; X32:       # %bb.0:
-; X32-NEXT:    pushl %eax
-; X32-NEXT:    vpsrad $29, %xmm0, %xmm0
-; X32-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; X32-NEXT:    vpsllq $20, %xmm0, %xmm0
-; X32-NEXT:    vcvtdq2ps %xmm0, %xmm0
-; X32-NEXT:    vmovss %xmm0, (%esp)
-; X32-NEXT:    flds (%esp)
-; X32-NEXT:    popl %eax
-; X32-NEXT:    retl
+; X86-LABEL: signbits_ashr_shl_extract_sitofp:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    vpsrad $29, %xmm0, %xmm0
+; X86-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; X86-NEXT:    vpsllq $20, %xmm0, %xmm0
+; X86-NEXT:    vcvtdq2ps %xmm0, %xmm0
+; X86-NEXT:    vmovss %xmm0, (%esp)
+; X86-NEXT:    flds (%esp)
+; X86-NEXT:    popl %eax
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: signbits_ashr_shl_extract_sitofp:
 ; X64:       # %bb.0:
@@ -126,21 +181,21 @@ define float @signbits_ashr_shl_extract_sitofp(<2 x i64> %a0) nounwind {
 }
 
 define float @signbits_ashr_insert_ashr_extract_sitofp(i64 %a0, i64 %a1) nounwind {
-; X32-LABEL: signbits_ashr_insert_ashr_extract_sitofp:
-; X32:       # %bb.0:
-; X32-NEXT:    pushl %eax
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl %eax, %ecx
-; X32-NEXT:    sarl $30, %ecx
-; X32-NEXT:    shll $2, %eax
-; X32-NEXT:    vmovd %eax, %xmm0
-; X32-NEXT:    vpinsrd $1, %ecx, %xmm0, %xmm0
-; X32-NEXT:    vpsrlq $3, %xmm0, %xmm0
-; X32-NEXT:    vcvtdq2ps %xmm0, %xmm0
-; X32-NEXT:    vmovss %xmm0, (%esp)
-; X32-NEXT:    flds (%esp)
-; X32-NEXT:    popl %eax
-; X32-NEXT:    retl
+; X86-LABEL: signbits_ashr_insert_ashr_extract_sitofp:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl %eax, %ecx
+; X86-NEXT:    sarl $30, %ecx
+; X86-NEXT:    shll $2, %eax
+; X86-NEXT:    vmovd %eax, %xmm0
+; X86-NEXT:    vpinsrd $1, %ecx, %xmm0, %xmm0
+; X86-NEXT:    vpsrlq $3, %xmm0, %xmm0
+; X86-NEXT:    vcvtdq2ps %xmm0, %xmm0
+; X86-NEXT:    vmovss %xmm0, (%esp)
+; X86-NEXT:    flds (%esp)
+; X86-NEXT:    popl %eax
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: signbits_ashr_insert_ashr_extract_sitofp:
 ; X64:       # %bb.0:
@@ -159,44 +214,87 @@ define float @signbits_ashr_insert_ashr_extract_sitofp(i64 %a0, i64 %a1) nounwin
 }
 
 define <4 x double> @signbits_sext_shuffle_sitofp(<4 x i32> %a0, <4 x i64> %a1) nounwind {
-; X32-LABEL: signbits_sext_shuffle_sitofp:
-; X32:       # %bb.0:
-; X32-NEXT:    vpmovsxdq %xmm0, %xmm1
-; X32-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; X32-NEXT:    vpmovsxdq %xmm0, %xmm0
-; X32-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; X32-NEXT:    vpermilpd {{.*#+}} ymm0 = ymm0[1,0,3,2]
-; X32-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,0,1]
-; X32-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; X32-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
-; X32-NEXT:    vcvtdq2pd %xmm0, %ymm0
-; X32-NEXT:    retl
+; X86-LABEL: signbits_sext_shuffle_sitofp:
+; X86:       # %bb.0:
+; X86-NEXT:    vpmovsxdq %xmm0, %xmm1
+; X86-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; X86-NEXT:    vpmovsxdq %xmm0, %xmm0
+; X86-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; X86-NEXT:    vpermilpd {{.*#+}} ymm0 = ymm0[1,0,3,2]
+; X86-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,0,1]
+; X86-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; X86-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
+; X86-NEXT:    vcvtdq2pd %xmm0, %ymm0
+; X86-NEXT:    retl
 ;
-; X64-LABEL: signbits_sext_shuffle_sitofp:
-; X64:       # %bb.0:
-; X64-NEXT:    vpmovsxdq %xmm0, %xmm1
-; X64-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; X64-NEXT:    vpmovsxdq %xmm0, %xmm0
-; X64-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; X64-NEXT:    vpermilpd {{.*#+}} ymm0 = ymm0[1,0,3,2]
-; X64-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,0,1]
-; X64-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; X64-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
-; X64-NEXT:    vcvtdq2pd %xmm0, %ymm0
-; X64-NEXT:    retq
+; X64-AVX1-LABEL: signbits_sext_shuffle_sitofp:
+; X64-AVX1:       # %bb.0:
+; X64-AVX1-NEXT:    vpmovsxdq %xmm0, %xmm1
+; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; X64-AVX1-NEXT:    vpmovsxdq %xmm0, %xmm0
+; X64-AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; X64-AVX1-NEXT:    vpermilpd {{.*#+}} ymm0 = ymm0[1,0,3,2]
+; X64-AVX1-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,0,1]
+; X64-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; X64-AVX1-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
+; X64-AVX1-NEXT:    vcvtdq2pd %xmm0, %ymm0
+; X64-AVX1-NEXT:    retq
+;
+; X64-AVX2-LABEL: signbits_sext_shuffle_sitofp:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    vpmovsxdq %xmm0, %ymm0
+; X64-AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[3,2,1,0]
+; X64-AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; X64-AVX2-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
+; X64-AVX2-NEXT:    vcvtdq2pd %xmm0, %ymm0
+; X64-AVX2-NEXT:    retq
   %1 = sext <4 x i32> %a0 to <4 x i64>
   %2 = shufflevector <4 x i64> %1, <4 x i64>%a1, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
   %3 = sitofp <4 x i64> %2 to <4 x double>
   ret <4 x double> %3
 }
 
+define <2 x double> @signbits_sext_shl_sitofp(<2 x i16> %a0) nounwind {
+; X86-LABEL: signbits_sext_shl_sitofp:
+; X86:       # %bb.0:
+; X86-NEXT:    vpmovsxwq %xmm0, %xmm0
+; X86-NEXT:    vpsllq $5, %xmm0, %xmm1
+; X86-NEXT:    vpsllq $11, %xmm0, %xmm0
+; X86-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4,5,6,7]
+; X86-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X86-NEXT:    vcvtdq2pd %xmm0, %xmm0
+; X86-NEXT:    retl
+;
+; X64-AVX1-LABEL: signbits_sext_shl_sitofp:
+; X64-AVX1:       # %bb.0:
+; X64-AVX1-NEXT:    vpmovsxwq %xmm0, %xmm0
+; X64-AVX1-NEXT:    vpsllq $5, %xmm0, %xmm1
+; X64-AVX1-NEXT:    vpsllq $11, %xmm0, %xmm0
+; X64-AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4,5,6,7]
+; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X64-AVX1-NEXT:    vcvtdq2pd %xmm0, %xmm0
+; X64-AVX1-NEXT:    retq
+;
+; X64-AVX2-LABEL: signbits_sext_shl_sitofp:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    vpmovsxwq %xmm0, %xmm0
+; X64-AVX2-NEXT:    vpsllvq {{.*}}(%rip), %xmm0, %xmm0
+; X64-AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X64-AVX2-NEXT:    vcvtdq2pd %xmm0, %xmm0
+; X64-AVX2-NEXT:    retq
+  %1 = sext <2 x i16> %a0 to <2 x i64>
+  %2 = shl <2 x i64> %1, <i64 11, i64 5>
+  %3 = sitofp <2 x i64> %2 to <2 x double>
+  ret <2 x double> %3
+}
+
 ; TODO: Fix vpshufd+vpsrlq -> vpshufd/vpermilps
 define <2 x double> @signbits_ashr_concat_ashr_extract_sitofp(<2 x i64> %a0, <4 x i64> %a1) nounwind {
-; X32-LABEL: signbits_ashr_concat_ashr_extract_sitofp:
-; X32:       # %bb.0:
-; X32-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[1,3,2,3]
-; X32-NEXT:    vcvtdq2pd %xmm0, %xmm0
-; X32-NEXT:    retl
+; X86-LABEL: signbits_ashr_concat_ashr_extract_sitofp:
+; X86:       # %bb.0:
+; X86-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[1,3,2,3]
+; X86-NEXT:    vcvtdq2pd %xmm0, %xmm0
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: signbits_ashr_concat_ashr_extract_sitofp:
 ; X64:       # %bb.0:
@@ -214,18 +312,18 @@ define <2 x double> @signbits_ashr_concat_ashr_extract_sitofp(<2 x i64> %a0, <4
 }
 
 define float @signbits_ashr_sext_sextinreg_and_extract_sitofp(<2 x i64> %a0, <2 x i64> %a1, i32 %a2) nounwind {
-; X32-LABEL: signbits_ashr_sext_sextinreg_and_extract_sitofp:
-; X32:       # %bb.0:
-; X32-NEXT:    pushl %eax
-; X32-NEXT:    vpsrad $29, %xmm0, %xmm0
-; X32-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; X32-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; X32-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; X32-NEXT:    vcvtdq2ps %xmm0, %xmm0
-; X32-NEXT:    vmovss %xmm0, (%esp)
-; X32-NEXT:    flds (%esp)
-; X32-NEXT:    popl %eax
-; X32-NEXT:    retl
+; X86-LABEL: signbits_ashr_sext_sextinreg_and_extract_sitofp:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    vpsrad $29, %xmm0, %xmm0
+; X86-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; X86-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; X86-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; X86-NEXT:    vcvtdq2ps %xmm0, %xmm0
+; X86-NEXT:    vmovss %xmm0, (%esp)
+; X86-NEXT:    flds (%esp)
+; X86-NEXT:    popl %eax
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: signbits_ashr_sext_sextinreg_and_extract_sitofp:
 ; X64:       # %bb.0:
@@ -247,39 +345,52 @@ define float @signbits_ashr_sext_sextinreg_and_extract_sitofp(<2 x i64> %a0, <2
 }
 
 define float @signbits_ashr_sextvecinreg_bitops_extract_sitofp(<2 x i64> %a0, <4 x i32> %a1) nounwind {
-; X32-LABEL: signbits_ashr_sextvecinreg_bitops_extract_sitofp:
-; X32:       # %bb.0:
-; X32-NEXT:    pushl %eax
-; X32-NEXT:    vpsrlq $60, %xmm0, %xmm2
-; X32-NEXT:    vpsrlq $61, %xmm0, %xmm0
-; X32-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm2[4,5,6,7]
-; X32-NEXT:    vmovdqa {{.*#+}} xmm2 = [4,0,8,0]
-; X32-NEXT:    vpxor %xmm2, %xmm0, %xmm0
-; X32-NEXT:    vpsubq %xmm2, %xmm0, %xmm0
-; X32-NEXT:    vpmovsxdq %xmm1, %xmm1
-; X32-NEXT:    vpand %xmm1, %xmm0, %xmm2
-; X32-NEXT:    vpor %xmm1, %xmm2, %xmm1
-; X32-NEXT:    vpxor %xmm0, %xmm1, %xmm0
-; X32-NEXT:    vcvtdq2ps %xmm0, %xmm0
-; X32-NEXT:    vmovss %xmm0, (%esp)
-; X32-NEXT:    flds (%esp)
-; X32-NEXT:    popl %eax
-; X32-NEXT:    retl
+; X86-LABEL: signbits_ashr_sextvecinreg_bitops_extract_sitofp:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    vpsrlq $60, %xmm0, %xmm2
+; X86-NEXT:    vpsrlq $61, %xmm0, %xmm0
+; X86-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm2[4,5,6,7]
+; X86-NEXT:    vmovdqa {{.*#+}} xmm2 = [4,0,8,0]
+; X86-NEXT:    vpxor %xmm2, %xmm0, %xmm0
+; X86-NEXT:    vpsubq %xmm2, %xmm0, %xmm0
+; X86-NEXT:    vpmovsxdq %xmm1, %xmm1
+; X86-NEXT:    vpand %xmm1, %xmm0, %xmm2
+; X86-NEXT:    vpor %xmm1, %xmm2, %xmm1
+; X86-NEXT:    vpxor %xmm0, %xmm1, %xmm0
+; X86-NEXT:    vcvtdq2ps %xmm0, %xmm0
+; X86-NEXT:    vmovss %xmm0, (%esp)
+; X86-NEXT:    flds (%esp)
+; X86-NEXT:    popl %eax
+; X86-NEXT:    retl
 ;
-; X64-LABEL: signbits_ashr_sextvecinreg_bitops_extract_sitofp:
-; X64:       # %bb.0:
-; X64-NEXT:    vpsrlq $60, %xmm0, %xmm2
-; X64-NEXT:    vpsrlq $61, %xmm0, %xmm0
-; X64-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm2[4,5,6,7]
-; X64-NEXT:    vmovdqa {{.*#+}} xmm2 = [4,8]
-; X64-NEXT:    vpxor %xmm2, %xmm0, %xmm0
-; X64-NEXT:    vpsubq %xmm2, %xmm0, %xmm0
-; X64-NEXT:    vpmovsxdq %xmm1, %xmm1
-; X64-NEXT:    vpand %xmm1, %xmm0, %xmm2
-; X64-NEXT:    vpor %xmm1, %xmm2, %xmm1
-; X64-NEXT:    vpxor %xmm0, %xmm1, %xmm0
-; X64-NEXT:    vcvtdq2ps %xmm0, %xmm0
-; X64-NEXT:    retq
+; X64-AVX1-LABEL: signbits_ashr_sextvecinreg_bitops_extract_sitofp:
+; X64-AVX1:       # %bb.0:
+; X64-AVX1-NEXT:    vpsrlq $60, %xmm0, %xmm2
+; X64-AVX1-NEXT:    vpsrlq $61, %xmm0, %xmm0
+; X64-AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm2[4,5,6,7]
+; X64-AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [4,8]
+; X64-AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
+; X64-AVX1-NEXT:    vpsubq %xmm2, %xmm0, %xmm0
+; X64-AVX1-NEXT:    vpmovsxdq %xmm1, %xmm1
+; X64-AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm2
+; X64-AVX1-NEXT:    vpor %xmm1, %xmm2, %xmm1
+; X64-AVX1-NEXT:    vpxor %xmm0, %xmm1, %xmm0
+; X64-AVX1-NEXT:    vcvtdq2ps %xmm0, %xmm0
+; X64-AVX1-NEXT:    retq
+;
+; X64-AVX2-LABEL: signbits_ashr_sextvecinreg_bitops_extract_sitofp:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    vpsrlvq {{.*}}(%rip), %xmm0, %xmm0
+; X64-AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [4,8]
+; X64-AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm0
+; X64-AVX2-NEXT:    vpsubq %xmm2, %xmm0, %xmm0
+; X64-AVX2-NEXT:    vpmovsxdq %xmm1, %xmm1
+; X64-AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm2
+; X64-AVX2-NEXT:    vpor %xmm1, %xmm2, %xmm1
+; X64-AVX2-NEXT:    vpxor %xmm0, %xmm1, %xmm0
+; X64-AVX2-NEXT:    vcvtdq2ps %xmm0, %xmm0
+; X64-AVX2-NEXT:    retq
   %1 = ashr <2 x i64> %a0, <i64 61, i64 60>
   %2 = shufflevector <4 x i32> %a1, <4 x i32> undef, <2 x i32> <i32 0, i32 1>
   %3 = sext <2 x i32> %2 to <2 x i64>
@@ -292,66 +403,82 @@ define float @signbits_ashr_sextvecinreg_bitops_extract_sitofp(<2 x i64> %a0, <4
 }
 
 define <4 x float> @signbits_ashr_sext_select_shuffle_sitofp(<4 x i64> %a0, <4 x i64> %a1, <4 x i64> %a2, <4 x i32> %a3) nounwind {
-; X32-LABEL: signbits_ashr_sext_select_shuffle_sitofp:
-; X32:       # %bb.0:
-; X32-NEXT:    pushl %ebp
-; X32-NEXT:    movl %esp, %ebp
-; X32-NEXT:    andl $-16, %esp
-; X32-NEXT:    subl $16, %esp
-; X32-NEXT:    vpmovsxdq 8(%ebp), %xmm3
-; X32-NEXT:    vpmovsxdq 16(%ebp), %xmm4
-; X32-NEXT:    vpsrad $31, %xmm2, %xmm5
-; X32-NEXT:    vpsrad $1, %xmm2, %xmm6
-; X32-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; X32-NEXT:    vpblendw {{.*#+}} xmm5 = xmm6[0,1],xmm5[2,3],xmm6[4,5],xmm5[6,7]
-; X32-NEXT:    vextractf128 $1, %ymm2, %xmm2
-; X32-NEXT:    vpsrad $31, %xmm2, %xmm6
-; X32-NEXT:    vpsrad $1, %xmm2, %xmm2
-; X32-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; X32-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1],xmm6[2,3],xmm2[4,5],xmm6[6,7]
-; X32-NEXT:    vpcmpeqq %xmm1, %xmm0, %xmm6
-; X32-NEXT:    vblendvpd %xmm6, %xmm5, %xmm3, %xmm3
-; X32-NEXT:    vextractf128 $1, %ymm1, %xmm1
-; X32-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; X32-NEXT:    vpcmpeqq %xmm1, %xmm0, %xmm0
-; X32-NEXT:    vblendvpd %xmm0, %xmm2, %xmm4, %xmm0
-; X32-NEXT:    vinsertf128 $1, %xmm0, %ymm3, %ymm0
-; X32-NEXT:    vmovddup {{.*#+}} ymm0 = ymm0[0,0,2,2]
-; X32-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; X32-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
-; X32-NEXT:    vcvtdq2ps %xmm0, %xmm0
-; X32-NEXT:    movl %ebp, %esp
-; X32-NEXT:    popl %ebp
-; X32-NEXT:    vzeroupper
-; X32-NEXT:    retl
+; X86-LABEL: signbits_ashr_sext_select_shuffle_sitofp:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    movl %esp, %ebp
+; X86-NEXT:    andl $-16, %esp
+; X86-NEXT:    subl $16, %esp
+; X86-NEXT:    vpmovsxdq 8(%ebp), %xmm3
+; X86-NEXT:    vpmovsxdq 16(%ebp), %xmm4
+; X86-NEXT:    vpsrad $31, %xmm2, %xmm5
+; X86-NEXT:    vpsrad $1, %xmm2, %xmm6
+; X86-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; X86-NEXT:    vpblendw {{.*#+}} xmm5 = xmm6[0,1],xmm5[2,3],xmm6[4,5],xmm5[6,7]
+; X86-NEXT:    vextractf128 $1, %ymm2, %xmm2
+; X86-NEXT:    vpsrad $31, %xmm2, %xmm6
+; X86-NEXT:    vpsrad $1, %xmm2, %xmm2
+; X86-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; X86-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1],xmm6[2,3],xmm2[4,5],xmm6[6,7]
+; X86-NEXT:    vpcmpeqq %xmm1, %xmm0, %xmm6
+; X86-NEXT:    vblendvpd %xmm6, %xmm5, %xmm3, %xmm3
+; X86-NEXT:    vextractf128 $1, %ymm1, %xmm1
+; X86-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; X86-NEXT:    vpcmpeqq %xmm1, %xmm0, %xmm0
+; X86-NEXT:    vblendvpd %xmm0, %xmm2, %xmm4, %xmm0
+; X86-NEXT:    vinsertf128 $1, %xmm0, %ymm3, %ymm0
+; X86-NEXT:    vmovddup {{.*#+}} ymm0 = ymm0[0,0,2,2]
+; X86-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; X86-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
+; X86-NEXT:    vcvtdq2ps %xmm0, %xmm0
+; X86-NEXT:    movl %ebp, %esp
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    vzeroupper
+; X86-NEXT:    retl
 ;
-; X64-LABEL: signbits_ashr_sext_select_shuffle_sitofp:
-; X64:       # %bb.0:
-; X64-NEXT:    vpsrad $31, %xmm2, %xmm4
-; X64-NEXT:    vpsrad $1, %xmm2, %xmm5
-; X64-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; X64-NEXT:    vpblendw {{.*#+}} xmm4 = xmm5[0,1],xmm4[2,3],xmm5[4,5],xmm4[6,7]
-; X64-NEXT:    vextractf128 $1, %ymm2, %xmm2
-; X64-NEXT:    vpsrad $31, %xmm2, %xmm5
-; X64-NEXT:    vpsrad $1, %xmm2, %xmm2
-; X64-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; X64-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1],xmm5[2,3],xmm2[4,5],xmm5[6,7]
-; X64-NEXT:    vpmovsxdq %xmm3, %xmm5
-; X64-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,3,0,1]
-; X64-NEXT:    vpmovsxdq %xmm3, %xmm3
-; X64-NEXT:    vpcmpeqq %xmm1, %xmm0, %xmm6
-; X64-NEXT:    vblendvpd %xmm6, %xmm4, %xmm5, %xmm4
-; X64-NEXT:    vextractf128 $1, %ymm1, %xmm1
-; X64-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; X64-NEXT:    vpcmpeqq %xmm1, %xmm0, %xmm0
-; X64-NEXT:    vblendvpd %xmm0, %xmm2, %xmm3, %xmm0
-; X64-NEXT:    vinsertf128 $1, %xmm0, %ymm4, %ymm0
-; X64-NEXT:    vmovddup {{.*#+}} ymm0 = ymm0[0,0,2,2]
-; X64-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; X64-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
-; X64-NEXT:    vcvtdq2ps %xmm0, %xmm0
-; X64-NEXT:    vzeroupper
-; X64-NEXT:    retq
+; X64-AVX1-LABEL: signbits_ashr_sext_select_shuffle_sitofp:
+; X64-AVX1:       # %bb.0:
+; X64-AVX1-NEXT:    vpsrad $31, %xmm2, %xmm4
+; X64-AVX1-NEXT:    vpsrad $1, %xmm2, %xmm5
+; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; X64-AVX1-NEXT:    vpblendw {{.*#+}} xmm4 = xmm5[0,1],xmm4[2,3],xmm5[4,5],xmm4[6,7]
+; X64-AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm2
+; X64-AVX1-NEXT:    vpsrad $31, %xmm2, %xmm5
+; X64-AVX1-NEXT:    vpsrad $1, %xmm2, %xmm2
+; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; X64-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1],xmm5[2,3],xmm2[4,5],xmm5[6,7]
+; X64-AVX1-NEXT:    vpmovsxdq %xmm3, %xmm5
+; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,3,0,1]
+; X64-AVX1-NEXT:    vpmovsxdq %xmm3, %xmm3
+; X64-AVX1-NEXT:    vpcmpeqq %xmm1, %xmm0, %xmm6
+; X64-AVX1-NEXT:    vblendvpd %xmm6, %xmm4, %xmm5, %xmm4
+; X64-AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm1
+; X64-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; X64-AVX1-NEXT:    vpcmpeqq %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT:    vblendvpd %xmm0, %xmm2, %xmm3, %xmm0
+; X64-AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm4, %ymm0
+; X64-AVX1-NEXT:    vmovddup {{.*#+}} ymm0 = ymm0[0,0,2,2]
+; X64-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; X64-AVX1-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
+; X64-AVX1-NEXT:    vcvtdq2ps %xmm0, %xmm0
+; X64-AVX1-NEXT:    vzeroupper
+; X64-AVX1-NEXT:    retq
+;
+; X64-AVX2-LABEL: signbits_ashr_sext_select_shuffle_sitofp:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    vpsrad $31, %ymm2, %ymm4
+; X64-AVX2-NEXT:    vpsrad $1, %ymm2, %ymm2
+; X64-AVX2-NEXT:    vpshufd {{.*#+}} ymm2 = ymm2[1,1,3,3,5,5,7,7]
+; X64-AVX2-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0],ymm4[1],ymm2[2],ymm4[3],ymm2[4],ymm4[5],ymm2[6],ymm4[7]
+; X64-AVX2-NEXT:    vpmovsxdq %xmm3, %ymm3
+; X64-AVX2-NEXT:    vpcmpeqq %ymm1, %ymm0, %ymm0
+; X64-AVX2-NEXT:    vblendvpd %ymm0, %ymm2, %ymm3, %ymm0
+; X64-AVX2-NEXT:    vpermilps {{.*#+}} ymm0 = ymm0[0,1,0,1,4,5,4,5]
+; X64-AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; X64-AVX2-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
+; X64-AVX2-NEXT:    vcvtdq2ps %xmm0, %xmm0
+; X64-AVX2-NEXT:    vzeroupper
+; X64-AVX2-NEXT:    retq
   %1 = ashr <4 x i64> %a2, <i64 33, i64 63, i64 33, i64 63>
   %2 = sext <4 x i32> %a3 to <4 x i64>
   %3 = icmp eq <4 x i64> %a0, %a1
@@ -366,34 +493,43 @@ define <4 x float> @signbits_ashr_sext_select_shuffle_sitofp(<4 x i64> %a0, <4 x
 ; v32i1->v32i8 promotion and the splitting of v32i8 into 2xv16i8. This requires
 ; ComputeNumSignBits handling for insert_subvector.
 define void @cross_bb_signbits_insert_subvec(<32 x i8>* %ptr, <32 x i8> %x, <32 x i8> %z) {
-; X32-LABEL: cross_bb_signbits_insert_subvec:
-; X32:       # %bb.0:
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; X32-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; X32-NEXT:    vpcmpeqb %xmm3, %xmm2, %xmm2
-; X32-NEXT:    vpcmpeqb %xmm3, %xmm0, %xmm0
-; X32-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; X32-NEXT:    vandnps %ymm1, %ymm0, %ymm1
-; X32-NEXT:    vandps {{\.LCPI.*}}, %ymm0, %ymm0
-; X32-NEXT:    vorps %ymm1, %ymm0, %ymm0
-; X32-NEXT:    vmovaps %ymm0, (%eax)
-; X32-NEXT:    vzeroupper
-; X32-NEXT:    retl
+; X86-LABEL: cross_bb_signbits_insert_subvec:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; X86-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; X86-NEXT:    vpcmpeqb %xmm3, %xmm2, %xmm2
+; X86-NEXT:    vpcmpeqb %xmm3, %xmm0, %xmm0
+; X86-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; X86-NEXT:    vandnps %ymm1, %ymm0, %ymm1
+; X86-NEXT:    vandps {{\.LCPI.*}}, %ymm0, %ymm0
+; X86-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; X86-NEXT:    vmovaps %ymm0, (%eax)
+; X86-NEXT:    vzeroupper
+; X86-NEXT:    retl
 ;
-; X64-LABEL: cross_bb_signbits_insert_subvec:
-; X64:       # %bb.0:
-; X64-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; X64-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; X64-NEXT:    vpcmpeqb %xmm3, %xmm2, %xmm2
-; X64-NEXT:    vpcmpeqb %xmm3, %xmm0, %xmm0
-; X64-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; X64-NEXT:    vandnps %ymm1, %ymm0, %ymm1
-; X64-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
-; X64-NEXT:    vorps %ymm1, %ymm0, %ymm0
-; X64-NEXT:    vmovaps %ymm0, (%rdi)
-; X64-NEXT:    vzeroupper
-; X64-NEXT:    retq
+; X64-AVX1-LABEL: cross_bb_signbits_insert_subvec:
+; X64-AVX1:       # %bb.0:
+; X64-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; X64-AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; X64-AVX1-NEXT:    vpcmpeqb %xmm3, %xmm2, %xmm2
+; X64-AVX1-NEXT:    vpcmpeqb %xmm3, %xmm0, %xmm0
+; X64-AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; X64-AVX1-NEXT:    vandnps %ymm1, %ymm0, %ymm1
+; X64-AVX1-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
+; X64-AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; X64-AVX1-NEXT:    vmovaps %ymm0, (%rdi)
+; X64-AVX1-NEXT:    vzeroupper
+; X64-AVX1-NEXT:    retq
+;
+; X64-AVX2-LABEL: cross_bb_signbits_insert_subvec:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; X64-AVX2-NEXT:    vpcmpeqb %ymm2, %ymm0, %ymm0
+; X64-AVX2-NEXT:    vpblendvb %ymm0, {{.*}}(%rip), %ymm1, %ymm0
+; X64-AVX2-NEXT:    vmovdqa %ymm0, (%rdi)
+; X64-AVX2-NEXT:    vzeroupper
+; X64-AVX2-NEXT:    retq
   %a = icmp eq <32 x i8> %x, zeroinitializer
   %b = icmp eq <32 x i8> %x, zeroinitializer
   %c = and <32 x i1> %a, %b
diff --git a/llvm/test/CodeGen/X86/kshift.ll b/llvm/test/CodeGen/X86/kshift.ll
index 245ba5c29..2da2f184f 100644
--- a/llvm/test/CodeGen/X86/kshift.ll
+++ b/llvm/test/CodeGen/X86/kshift.ll
@@ -5,13 +5,9 @@
 define i8 @kshiftl_v8i1_1(<8 x i64> %x, <8 x i64> %y) {
 ; KNL-LABEL: kshiftl_v8i1_1:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    vptestnmq %zmm0, %zmm0, %k1
-; KNL-NEXT:    vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
-; KNL-NEXT:    movb $-2, %al
-; KNL-NEXT:    kmovw %eax, %k1
-; KNL-NEXT:    vpexpandq %zmm0, %zmm0 {%k1} {z}
-; KNL-NEXT:    vptestnmq %zmm1, %zmm1, %k1
-; KNL-NEXT:    vptestmq %zmm0, %zmm0, %k0 {%k1}
+; KNL-NEXT:    vptestnmq %zmm0, %zmm0, %k0
+; KNL-NEXT:    kshiftlw $1, %k0, %k1
+; KNL-NEXT:    vptestnmq %zmm1, %zmm1, %k0 {%k1}
 ; KNL-NEXT:    kmovw %k0, %eax
 ; KNL-NEXT:    # kill: def $al killed $al killed $eax
 ; KNL-NEXT:    vzeroupper
@@ -65,21 +61,23 @@ define i16 @kshiftl_v16i1_1(<16 x i32> %x, <16 x i32> %y) {
 define i32 @kshiftl_v32i1_1(<32 x i16> %x, <32 x i16> %y) {
 ; KNL-LABEL: kshiftl_v32i1_1:
 ; KNL:       # %bb.0:
+; KNL-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
+; KNL-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
 ; KNL-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; KNL-NEXT:    vpcmpeqw %ymm4, %ymm1, %ymm1
-; KNL-NEXT:    vpmovsxwd %ymm1, %zmm1
-; KNL-NEXT:    vptestmd %zmm1, %zmm1, %k1
+; KNL-NEXT:    vpcmpeqw %ymm4, %ymm3, %ymm3
+; KNL-NEXT:    vpmovsxwd %ymm3, %zmm3
+; KNL-NEXT:    vptestmd %zmm3, %zmm3, %k1
 ; KNL-NEXT:    vpcmpeqw %ymm4, %ymm0, %ymm0
 ; KNL-NEXT:    vpmovsxwd %ymm0, %zmm0
 ; KNL-NEXT:    vptestmd %zmm0, %zmm0, %k2
 ; KNL-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k2} {z}
-; KNL-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k1} {z}
-; KNL-NEXT:    valignd {{.*#+}} zmm0 = zmm0[15],zmm1[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14]
+; KNL-NEXT:    vpternlogd $255, %zmm3, %zmm3, %zmm3 {%k1} {z}
+; KNL-NEXT:    valignd {{.*#+}} zmm0 = zmm0[15],zmm3[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14]
 ; KNL-NEXT:    kshiftlw $1, %k2, %k1
-; KNL-NEXT:    vpcmpeqw %ymm4, %ymm3, %ymm1
-; KNL-NEXT:    vpmovsxwd %ymm1, %zmm1
-; KNL-NEXT:    vptestmd %zmm1, %zmm1, %k2
-; KNL-NEXT:    vpcmpeqw %ymm4, %ymm2, %ymm1
+; KNL-NEXT:    vpcmpeqw %ymm4, %ymm2, %ymm2
+; KNL-NEXT:    vpmovsxwd %ymm2, %zmm2
+; KNL-NEXT:    vptestmd %zmm2, %zmm2, %k2
+; KNL-NEXT:    vpcmpeqw %ymm4, %ymm1, %ymm1
 ; KNL-NEXT:    vpmovsxwd %ymm1, %zmm1
 ; KNL-NEXT:    vptestmd %zmm1, %zmm1, %k0 {%k1}
 ; KNL-NEXT:    kmovw %k0, %ecx
@@ -109,6 +107,8 @@ define i32 @kshiftl_v32i1_1(<32 x i16> %x, <32 x i16> %y) {
 define i64 @kshiftl_v64i1_1(<64 x i8> %x, <64 x i8> %y) {
 ; KNL-LABEL: kshiftl_v64i1_1:
 ; KNL:       # %bb.0:
+; KNL-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
+; KNL-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
 ; KNL-NEXT:    vpxor %xmm4, %xmm4, %xmm4
 ; KNL-NEXT:    vpcmpeqb %ymm4, %ymm0, %ymm0
 ; KNL-NEXT:    vpmovsxbd %xmm0, %zmm5
@@ -116,32 +116,32 @@ define i64 @kshiftl_v64i1_1(<64 x i8> %x, <64 x i8> %y) {
 ; KNL-NEXT:    vextracti128 $1, %ymm0, %xmm0
 ; KNL-NEXT:    vpmovsxbd %xmm0, %zmm0
 ; KNL-NEXT:    vptestmd %zmm0, %zmm0, %k2
-; KNL-NEXT:    vpcmpeqb %ymm4, %ymm1, %ymm0
-; KNL-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; KNL-NEXT:    vpmovsxbd %xmm1, %zmm1
-; KNL-NEXT:    vptestmd %zmm1, %zmm1, %k3
+; KNL-NEXT:    vpcmpeqb %ymm4, %ymm3, %ymm0
+; KNL-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; KNL-NEXT:    vpmovsxbd %xmm3, %zmm3
+; KNL-NEXT:    vptestmd %zmm3, %zmm3, %k3
 ; KNL-NEXT:    vpmovsxbd %xmm0, %zmm0
 ; KNL-NEXT:    vptestmd %zmm0, %zmm0, %k4
 ; KNL-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k4} {z}
-; KNL-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k3} {z}
-; KNL-NEXT:    valignd {{.*#+}} zmm1 = zmm0[15],zmm1[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14]
+; KNL-NEXT:    vpternlogd $255, %zmm3, %zmm3, %zmm3 {%k3} {z}
+; KNL-NEXT:    valignd {{.*#+}} zmm3 = zmm0[15],zmm3[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14]
 ; KNL-NEXT:    vpternlogd $255, %zmm5, %zmm5, %zmm5 {%k2} {z}
 ; KNL-NEXT:    valignd {{.*#+}} zmm0 = zmm5[15],zmm0[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14]
 ; KNL-NEXT:    vpternlogd $255, %zmm6, %zmm6, %zmm6 {%k1} {z}
 ; KNL-NEXT:    valignd {{.*#+}} zmm5 = zmm6[15],zmm5[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14]
 ; KNL-NEXT:    kshiftlw $1, %k1, %k3
-; KNL-NEXT:    vpcmpeqb %ymm4, %ymm3, %ymm3
-; KNL-NEXT:    vextracti128 $1, %ymm3, %xmm6
+; KNL-NEXT:    vpcmpeqb %ymm4, %ymm2, %ymm2
+; KNL-NEXT:    vextracti128 $1, %ymm2, %xmm6
 ; KNL-NEXT:    vpmovsxbd %xmm6, %zmm6
 ; KNL-NEXT:    vptestmd %zmm6, %zmm6, %k1
-; KNL-NEXT:    vpmovsxbd %xmm3, %zmm3
-; KNL-NEXT:    vptestmd %zmm3, %zmm3, %k2
-; KNL-NEXT:    vpcmpeqb %ymm4, %ymm2, %ymm2
-; KNL-NEXT:    vextracti128 $1, %ymm2, %xmm3
-; KNL-NEXT:    vpmovsxbd %xmm3, %zmm3
-; KNL-NEXT:    vptestmd %zmm3, %zmm3, %k4
 ; KNL-NEXT:    vpmovsxbd %xmm2, %zmm2
-; KNL-NEXT:    vptestmd %zmm2, %zmm2, %k0 {%k3}
+; KNL-NEXT:    vptestmd %zmm2, %zmm2, %k2
+; KNL-NEXT:    vpcmpeqb %ymm4, %ymm1, %ymm1
+; KNL-NEXT:    vextracti128 $1, %ymm1, %xmm2
+; KNL-NEXT:    vpmovsxbd %xmm2, %zmm2
+; KNL-NEXT:    vptestmd %zmm2, %zmm2, %k4
+; KNL-NEXT:    vpmovsxbd %xmm1, %zmm1
+; KNL-NEXT:    vptestmd %zmm1, %zmm1, %k0 {%k3}
 ; KNL-NEXT:    kmovw %k0, %eax
 ; KNL-NEXT:    vptestmd %zmm5, %zmm5, %k0 {%k4}
 ; KNL-NEXT:    kmovw %k0, %ecx
@@ -149,7 +149,7 @@ define i64 @kshiftl_v64i1_1(<64 x i8> %x, <64 x i8> %y) {
 ; KNL-NEXT:    orl %eax, %ecx
 ; KNL-NEXT:    vptestmd %zmm0, %zmm0, %k0 {%k2}
 ; KNL-NEXT:    kmovw %k0, %edx
-; KNL-NEXT:    vptestmd %zmm1, %zmm1, %k0 {%k1}
+; KNL-NEXT:    vptestmd %zmm3, %zmm3, %k0 {%k1}
 ; KNL-NEXT:    kmovw %k0, %eax
 ; KNL-NEXT:    shll $16, %eax
 ; KNL-NEXT:    orl %edx, %eax
@@ -177,13 +177,9 @@ define i64 @kshiftl_v64i1_1(<64 x i8> %x, <64 x i8> %y) {
 define i8 @kshiftl_v8i1_7(<8 x i64> %x, <8 x i64> %y) {
 ; KNL-LABEL: kshiftl_v8i1_7:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    vptestnmq %zmm0, %zmm0, %k1
-; KNL-NEXT:    vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
-; KNL-NEXT:    movb $-128, %al
-; KNL-NEXT:    kmovw %eax, %k1
-; KNL-NEXT:    vpexpandq %zmm0, %zmm0 {%k1} {z}
-; KNL-NEXT:    vptestnmq %zmm1, %zmm1, %k1
-; KNL-NEXT:    vptestmq %zmm0, %zmm0, %k0 {%k1}
+; KNL-NEXT:    vptestnmq %zmm0, %zmm0, %k0
+; KNL-NEXT:    kshiftlw $7, %k0, %k1
+; KNL-NEXT:    vptestnmq %zmm1, %zmm1, %k0 {%k1}
 ; KNL-NEXT:    kmovw %k0, %eax
 ; KNL-NEXT:    # kill: def $al killed $al killed $eax
 ; KNL-NEXT:    vzeroupper
@@ -237,12 +233,13 @@ define i16 @kshiftl_v16i1_15(<16 x i32> %x, <16 x i32> %y) {
 define i32 @kshiftl_v32i1_31(<32 x i16> %x, <32 x i16> %y) {
 ; KNL-LABEL: kshiftl_v32i1_31:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; KNL-NEXT:    vpcmpeqw %ymm1, %ymm0, %ymm0
+; KNL-NEXT:    vextracti64x4 $1, %zmm1, %ymm1
+; KNL-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; KNL-NEXT:    vpcmpeqw %ymm2, %ymm0, %ymm0
 ; KNL-NEXT:    vpmovsxwd %ymm0, %zmm0
 ; KNL-NEXT:    vptestmd %zmm0, %zmm0, %k0
 ; KNL-NEXT:    kshiftlw $15, %k0, %k1
-; KNL-NEXT:    vpcmpeqw %ymm1, %ymm3, %ymm0
+; KNL-NEXT:    vpcmpeqw %ymm2, %ymm1, %ymm0
 ; KNL-NEXT:    vpmovsxwd %ymm0, %zmm0
 ; KNL-NEXT:    vptestmd %zmm0, %zmm0, %k0 {%k1}
 ; KNL-NEXT:    kmovw %k0, %eax
@@ -269,13 +266,14 @@ define i32 @kshiftl_v32i1_31(<32 x i16> %x, <32 x i16> %y) {
 define i64 @kshiftl_v64i1_63(<64 x i8> %x, <64 x i8> %y) {
 ; KNL-LABEL: kshiftl_v64i1_63:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; KNL-NEXT:    vpcmpeqb %xmm1, %xmm0, %xmm0
+; KNL-NEXT:    vextracti64x4 $1, %zmm1, %ymm1
+; KNL-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; KNL-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm0
 ; KNL-NEXT:    vpmovsxbd %xmm0, %zmm0
 ; KNL-NEXT:    vptestmd %zmm0, %zmm0, %k0
 ; KNL-NEXT:    kshiftlw $15, %k0, %k1
-; KNL-NEXT:    vextracti128 $1, %ymm3, %xmm0
-; KNL-NEXT:    vpcmpeqb %xmm1, %xmm0, %xmm0
+; KNL-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; KNL-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm0
 ; KNL-NEXT:    vpmovsxbd %xmm0, %zmm0
 ; KNL-NEXT:    vptestmd %zmm0, %zmm0, %k0 {%k1}
 ; KNL-NEXT:    kmovw %k0, %eax
@@ -303,13 +301,10 @@ define i64 @kshiftl_v64i1_63(<64 x i8> %x, <64 x i8> %y) {
 define i8 @kshiftr_v8i1_1(<8 x i64> %x, <8 x i64> %y) {
 ; KNL-LABEL: kshiftr_v8i1_1:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    vptestnmq %zmm0, %zmm0, %k1
-; KNL-NEXT:    vpxor %xmm0, %xmm0, %xmm0
-; KNL-NEXT:    vpternlogq $255, %zmm2, %zmm2, %zmm2 {%k1} {z}
-; KNL-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [1,2,3,4,5,6,7,15]
-; KNL-NEXT:    vpermi2q %zmm0, %zmm2, %zmm3
-; KNL-NEXT:    vptestnmq %zmm1, %zmm1, %k1
-; KNL-NEXT:    vptestmq %zmm3, %zmm3, %k0 {%k1}
+; KNL-NEXT:    vptestnmq %zmm0, %zmm0, %k0
+; KNL-NEXT:    kshiftlw $8, %k0, %k0
+; KNL-NEXT:    kshiftrw $9, %k0, %k1
+; KNL-NEXT:    vptestnmq %zmm1, %zmm1, %k0 {%k1}
 ; KNL-NEXT:    kmovw %k0, %eax
 ; KNL-NEXT:    # kill: def $al killed $al killed $eax
 ; KNL-NEXT:    vzeroupper
@@ -363,21 +358,23 @@ define i16 @kshiftr_v16i1_1(<16 x i32> %x, <16 x i32> %y) {
 define i32 @kshiftr_v32i1_1(<32 x i16> %x, <32 x i16> %y) {
 ; KNL-LABEL: kshiftr_v32i1_1:
 ; KNL:       # %bb.0:
+; KNL-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
+; KNL-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
 ; KNL-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; KNL-NEXT:    vpcmpeqw %ymm4, %ymm1, %ymm1
-; KNL-NEXT:    vpmovsxwd %ymm1, %zmm1
-; KNL-NEXT:    vptestmd %zmm1, %zmm1, %k1
+; KNL-NEXT:    vpcmpeqw %ymm4, %ymm3, %ymm3
+; KNL-NEXT:    vpmovsxwd %ymm3, %zmm3
+; KNL-NEXT:    vptestmd %zmm3, %zmm3, %k1
 ; KNL-NEXT:    vpcmpeqw %ymm4, %ymm0, %ymm0
 ; KNL-NEXT:    vpmovsxwd %ymm0, %zmm0
 ; KNL-NEXT:    vptestmd %zmm0, %zmm0, %k2
 ; KNL-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k2} {z}
-; KNL-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k1} {z}
-; KNL-NEXT:    valignd {{.*#+}} zmm0 = zmm0[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],zmm1[0]
+; KNL-NEXT:    vpternlogd $255, %zmm3, %zmm3, %zmm3 {%k1} {z}
+; KNL-NEXT:    valignd {{.*#+}} zmm0 = zmm0[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],zmm3[0]
 ; KNL-NEXT:    kshiftrw $1, %k1, %k1
-; KNL-NEXT:    vpcmpeqw %ymm4, %ymm2, %ymm1
+; KNL-NEXT:    vpcmpeqw %ymm4, %ymm1, %ymm1
 ; KNL-NEXT:    vpmovsxwd %ymm1, %zmm1
 ; KNL-NEXT:    vptestmd %zmm1, %zmm1, %k2
-; KNL-NEXT:    vpcmpeqw %ymm4, %ymm3, %ymm1
+; KNL-NEXT:    vpcmpeqw %ymm4, %ymm2, %ymm1
 ; KNL-NEXT:    vpmovsxwd %ymm1, %zmm1
 ; KNL-NEXT:    vptestmd %zmm1, %zmm1, %k0 {%k1}
 ; KNL-NEXT:    kmovw %k0, %ecx
@@ -407,42 +404,44 @@ define i32 @kshiftr_v32i1_1(<32 x i16> %x, <32 x i16> %y) {
 define i64 @kshiftr_v64i1_1(<64 x i8> %x, <64 x i8> %y) {
 ; KNL-LABEL: kshiftr_v64i1_1:
 ; KNL:       # %bb.0:
+; KNL-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
+; KNL-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
 ; KNL-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; KNL-NEXT:    vpcmpeqb %ymm4, %ymm1, %ymm1
-; KNL-NEXT:    vextracti128 $1, %ymm1, %xmm5
+; KNL-NEXT:    vpcmpeqb %ymm4, %ymm3, %ymm3
+; KNL-NEXT:    vextracti128 $1, %ymm3, %xmm5
 ; KNL-NEXT:    vpmovsxbd %xmm5, %zmm5
 ; KNL-NEXT:    vptestmd %zmm5, %zmm5, %k1
 ; KNL-NEXT:    vpcmpeqb %ymm4, %ymm0, %ymm0
 ; KNL-NEXT:    vpmovsxbd %xmm0, %zmm5
 ; KNL-NEXT:    vptestmd %zmm5, %zmm5, %k2
-; KNL-NEXT:    vpmovsxbd %xmm1, %zmm1
-; KNL-NEXT:    vptestmd %zmm1, %zmm1, %k3
+; KNL-NEXT:    vpmovsxbd %xmm3, %zmm3
+; KNL-NEXT:    vptestmd %zmm3, %zmm3, %k3
 ; KNL-NEXT:    vextracti128 $1, %ymm0, %xmm0
 ; KNL-NEXT:    vpmovsxbd %xmm0, %zmm0
 ; KNL-NEXT:    vptestmd %zmm0, %zmm0, %k4
 ; KNL-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k4} {z}
-; KNL-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k3} {z}
-; KNL-NEXT:    valignd {{.*#+}} zmm5 = zmm0[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],zmm1[0]
+; KNL-NEXT:    vpternlogd $255, %zmm3, %zmm3, %zmm3 {%k3} {z}
+; KNL-NEXT:    valignd {{.*#+}} zmm5 = zmm0[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],zmm3[0]
 ; KNL-NEXT:    vpternlogd $255, %zmm6, %zmm6, %zmm6 {%k2} {z}
 ; KNL-NEXT:    valignd {{.*#+}} zmm0 = zmm6[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],zmm0[0]
 ; KNL-NEXT:    vpternlogd $255, %zmm6, %zmm6, %zmm6 {%k1} {z}
-; KNL-NEXT:    valignd {{.*#+}} zmm1 = zmm1[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],zmm6[0]
+; KNL-NEXT:    valignd {{.*#+}} zmm3 = zmm3[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],zmm6[0]
 ; KNL-NEXT:    kshiftrw $1, %k1, %k3
-; KNL-NEXT:    vpcmpeqb %ymm4, %ymm2, %ymm2
-; KNL-NEXT:    vextracti128 $1, %ymm2, %xmm6
+; KNL-NEXT:    vpcmpeqb %ymm4, %ymm1, %ymm1
+; KNL-NEXT:    vextracti128 $1, %ymm1, %xmm6
 ; KNL-NEXT:    vpmovsxbd %xmm6, %zmm6
 ; KNL-NEXT:    vptestmd %zmm6, %zmm6, %k1
-; KNL-NEXT:    vpmovsxbd %xmm2, %zmm2
-; KNL-NEXT:    vptestmd %zmm2, %zmm2, %k2
-; KNL-NEXT:    vpcmpeqb %ymm4, %ymm3, %ymm2
-; KNL-NEXT:    vpmovsxbd %xmm2, %zmm3
-; KNL-NEXT:    vptestmd %zmm3, %zmm3, %k4
-; KNL-NEXT:    vextracti128 $1, %ymm2, %xmm2
-; KNL-NEXT:    vpmovsxbd %xmm2, %zmm2
-; KNL-NEXT:    vptestmd %zmm2, %zmm2, %k0 {%k3}
+; KNL-NEXT:    vpmovsxbd %xmm1, %zmm1
+; KNL-NEXT:    vptestmd %zmm1, %zmm1, %k2
+; KNL-NEXT:    vpcmpeqb %ymm4, %ymm2, %ymm1
+; KNL-NEXT:    vpmovsxbd %xmm1, %zmm2
+; KNL-NEXT:    vptestmd %zmm2, %zmm2, %k4
+; KNL-NEXT:    vextracti128 $1, %ymm1, %xmm1
+; KNL-NEXT:    vpmovsxbd %xmm1, %zmm1
+; KNL-NEXT:    vptestmd %zmm1, %zmm1, %k0 {%k3}
 ; KNL-NEXT:    kmovw %k0, %eax
 ; KNL-NEXT:    shll $16, %eax
-; KNL-NEXT:    vptestmd %zmm1, %zmm1, %k0 {%k4}
+; KNL-NEXT:    vptestmd %zmm3, %zmm3, %k0 {%k4}
 ; KNL-NEXT:    kmovw %k0, %ecx
 ; KNL-NEXT:    orl %eax, %ecx
 ; KNL-NEXT:    shlq $32, %rcx
@@ -475,13 +474,9 @@ define i64 @kshiftr_v64i1_1(<64 x i8> %x, <64 x i8> %y) {
 define i8 @kshiftr_v8i1_7(<8 x i64> %x, <8 x i64> %y) {
 ; KNL-LABEL: kshiftr_v8i1_7:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    vptestnmq %zmm0, %zmm0, %k1
-; KNL-NEXT:    vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
-; KNL-NEXT:    movb $-2, %al
-; KNL-NEXT:    kmovw %eax, %k1
-; KNL-NEXT:    vpexpandq %zmm0, %zmm0 {%k1} {z}
-; KNL-NEXT:    vptestnmq %zmm1, %zmm1, %k1
-; KNL-NEXT:    vptestmq %zmm0, %zmm0, %k0 {%k1}
+; KNL-NEXT:    vptestnmq %zmm0, %zmm0, %k0
+; KNL-NEXT:    kshiftlw $1, %k0, %k1
+; KNL-NEXT:    vptestnmq %zmm1, %zmm1, %k0 {%k1}
 ; KNL-NEXT:    kmovw %k0, %eax
 ; KNL-NEXT:    # kill: def $al killed $al killed $eax
 ; KNL-NEXT:    vzeroupper
@@ -535,12 +530,13 @@ define i16 @kshiftr_v16i1_15(<16 x i32> %x, <16 x i32> %y) {
 define i32 @kshiftr_v32i1_31(<32 x i16> %x, <32 x i16> %y) {
 ; KNL-LABEL: kshiftr_v32i1_31:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    vpxor %xmm0, %xmm0, %xmm0
-; KNL-NEXT:    vpcmpeqw %ymm0, %ymm1, %ymm1
-; KNL-NEXT:    vpmovsxwd %ymm1, %zmm1
-; KNL-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; KNL-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
+; KNL-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; KNL-NEXT:    vpcmpeqw %ymm2, %ymm0, %ymm0
+; KNL-NEXT:    vpmovsxwd %ymm0, %zmm0
+; KNL-NEXT:    vptestmd %zmm0, %zmm0, %k0
 ; KNL-NEXT:    kshiftrw $15, %k0, %k1
-; KNL-NEXT:    vpcmpeqw %ymm0, %ymm2, %ymm0
+; KNL-NEXT:    vpcmpeqw %ymm2, %ymm1, %ymm0
 ; KNL-NEXT:    vpmovsxwd %ymm0, %zmm0
 ; KNL-NEXT:    vptestmd %zmm0, %zmm0, %k0 {%k1}
 ; KNL-NEXT:    kmovw %k0, %eax
@@ -566,17 +562,17 @@ define i32 @kshiftr_v32i1_31(<32 x i16> %x, <32 x i16> %y) {
 define i64 @kshiftr_v64i1_63(<64 x i8> %x, <64 x i8> %y) {
 ; KNL-LABEL: kshiftr_v64i1_63:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    vextracti128 $1, %ymm1, %xmm0
-; KNL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; KNL-NEXT:    vpcmpeqb %xmm1, %xmm0, %xmm0
+; KNL-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
+; KNL-NEXT:    vextracti128 $1, %ymm0, %xmm0
+; KNL-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; KNL-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm0
 ; KNL-NEXT:    vpmovsxbd %xmm0, %zmm0
 ; KNL-NEXT:    vptestmd %zmm0, %zmm0, %k0
 ; KNL-NEXT:    kshiftrw $15, %k0, %k1
-; KNL-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm0
+; KNL-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm0
 ; KNL-NEXT:    vpmovsxbd %xmm0, %zmm0
 ; KNL-NEXT:    vptestmd %zmm0, %zmm0, %k0 {%k1}
 ; KNL-NEXT:    kmovw %k0, %eax
-; KNL-NEXT:    movzwl %ax, %eax
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    retq
 ;
@@ -599,14 +595,9 @@ define i64 @kshiftr_v64i1_63(<64 x i8> %x, <64 x i8> %y) {
 define i8 @kshiftl_v8i1_zu123u56(<8 x i64> %x, <8 x i64> %y) {
 ; KNL-LABEL: kshiftl_v8i1_zu123u56:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    vptestnmq %zmm0, %zmm0, %k1
-; KNL-NEXT:    vpxor %xmm0, %xmm0, %xmm0
-; KNL-NEXT:    vpternlogq $255, %zmm2, %zmm2, %zmm2 {%k1} {z}
-; KNL-NEXT:    vmovdqa64 {{.*#+}} zmm3 = <8,u,1,2,3,u,5,6>
-; KNL-NEXT:    vpermi2q %zmm0, %zmm2, %zmm3
-; KNL-NEXT:    vpsllq $63, %zmm3, %zmm0
-; KNL-NEXT:    vptestnmq %zmm1, %zmm1, %k1
-; KNL-NEXT:    vptestmq %zmm0, %zmm0, %k0 {%k1}
+; KNL-NEXT:    vptestnmq %zmm0, %zmm0, %k0
+; KNL-NEXT:    kshiftlw $1, %k0, %k1
+; KNL-NEXT:    vptestnmq %zmm1, %zmm1, %k0 {%k1}
 ; KNL-NEXT:    kmovw %k0, %eax
 ; KNL-NEXT:    # kill: def $al killed $al killed $eax
 ; KNL-NEXT:    vzeroupper
@@ -632,12 +623,9 @@ define i8 @kshiftl_v8i1_zu123u56(<8 x i64> %x, <8 x i64> %y) {
 define i8 @kshiftl_v8i1_u0123456(<8 x i64> %x, <8 x i64> %y) {
 ; KNL-LABEL: kshiftl_v8i1_u0123456:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    vptestnmq %zmm0, %zmm0, %k1
-; KNL-NEXT:    vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
-; KNL-NEXT:    valignq {{.*#+}} zmm0 = zmm0[7,0,1,2,3,4,5,6]
-; KNL-NEXT:    vpsllq $63, %zmm0, %zmm0
-; KNL-NEXT:    vptestnmq %zmm1, %zmm1, %k1
-; KNL-NEXT:    vptestmq %zmm0, %zmm0, %k0 {%k1}
+; KNL-NEXT:    vptestnmq %zmm0, %zmm0, %k0
+; KNL-NEXT:    kshiftlw $1, %k0, %k1
+; KNL-NEXT:    vptestnmq %zmm1, %zmm1, %k0 {%k1}
 ; KNL-NEXT:    kmovw %k0, %eax
 ; KNL-NEXT:    # kill: def $al killed $al killed $eax
 ; KNL-NEXT:    vzeroupper
@@ -663,14 +651,10 @@ define i8 @kshiftl_v8i1_u0123456(<8 x i64> %x, <8 x i64> %y) {
 define i8 @kshiftr_v8i1_1u3u567z(<8 x i64> %x, <8 x i64> %y) {
 ; KNL-LABEL: kshiftr_v8i1_1u3u567z:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    vptestnmq %zmm0, %zmm0, %k1
-; KNL-NEXT:    vpxor %xmm0, %xmm0, %xmm0
-; KNL-NEXT:    vpternlogq $255, %zmm2, %zmm2, %zmm2 {%k1} {z}
-; KNL-NEXT:    vmovdqa64 {{.*#+}} zmm3 = <1,u,3,u,5,6,7,15>
-; KNL-NEXT:    vpermi2q %zmm0, %zmm2, %zmm3
-; KNL-NEXT:    vpsllq $63, %zmm3, %zmm0
-; KNL-NEXT:    vptestnmq %zmm1, %zmm1, %k1
-; KNL-NEXT:    vptestmq %zmm0, %zmm0, %k0 {%k1}
+; KNL-NEXT:    vptestnmq %zmm0, %zmm0, %k0
+; KNL-NEXT:    kshiftlw $8, %k0, %k0
+; KNL-NEXT:    kshiftrw $9, %k0, %k1
+; KNL-NEXT:    vptestnmq %zmm1, %zmm1, %k0 {%k1}
 ; KNL-NEXT:    kmovw %k0, %eax
 ; KNL-NEXT:    # kill: def $al killed $al killed $eax
 ; KNL-NEXT:    vzeroupper
@@ -696,12 +680,9 @@ define i8 @kshiftr_v8i1_1u3u567z(<8 x i64> %x, <8 x i64> %y) {
 define i8 @kshiftr_v8i1_234567uu(<8 x i64> %x, <8 x i64> %y) {
 ; KNL-LABEL: kshiftr_v8i1_234567uu:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    vptestnmq %zmm0, %zmm0, %k1
-; KNL-NEXT:    vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
-; KNL-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[2,3,4,5,6,7,0,1]
-; KNL-NEXT:    vpsllq $63, %zmm0, %zmm0
-; KNL-NEXT:    vptestnmq %zmm1, %zmm1, %k1
-; KNL-NEXT:    vptestmq %zmm0, %zmm0, %k0 {%k1}
+; KNL-NEXT:    vptestnmq %zmm0, %zmm0, %k0
+; KNL-NEXT:    kshiftrw $2, %k0, %k1
+; KNL-NEXT:    vptestnmq %zmm1, %zmm1, %k0 {%k1}
 ; KNL-NEXT:    kmovw %k0, %eax
 ; KNL-NEXT:    # kill: def $al killed $al killed $eax
 ; KNL-NEXT:    vzeroupper
diff --git a/llvm/test/CodeGen/X86/label-annotation.ll b/llvm/test/CodeGen/X86/label-annotation.ll
index 8ed1fd470..626040c7a 100644
--- a/llvm/test/CodeGen/X86/label-annotation.ll
+++ b/llvm/test/CodeGen/X86/label-annotation.ll
@@ -77,8 +77,8 @@ entry:
 
 
 
-attributes #0 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { inaccessiblememonly noduplicate nounwind }
 attributes #3 = { nounwind }
 
diff --git a/llvm/test/CodeGen/X86/label-heapallocsite.ll b/llvm/test/CodeGen/X86/label-heapallocsite.ll
index deb74c2ea..e76e097d5 100644
--- a/llvm/test/CodeGen/X86/label-heapallocsite.ll
+++ b/llvm/test/CodeGen/X86/label-heapallocsite.ll
@@ -1,5 +1,5 @@
-; RUN: llc < %s | FileCheck --check-prefixes=DAG,CHECK %s
-; RUN: llc -O0 < %s | FileCheck --check-prefixes=FAST,CHECK %s
+; RUN: llc < %s | FileCheck --check-prefixes=CHECK %s
+; RUN: llc -O0 < %s | FileCheck --check-prefixes=CHECK %s
 
 ; Source to regenerate:
 ; $ clang -cc1 -triple x86_64-windows-msvc t.cpp -debug-info-kind=limited \
@@ -71,42 +71,33 @@ declare void @llvm.dbg.value(metadata, metadata, metadata) #2
 
 ; Don't emit metadata for tail calls.
 ; CHECK-LABEL: call_tail:         # @call_tail
-; CHECK-NOT: .Lheapallocsite
 ; CHECK: jmp alloc_foo
 
 ; CHECK-LABEL: call_virtual:      # @call_virtual
-; CHECK: .Lheapallocsite0:
-; CHECK: callq *{{.*}}%rax{{.*}}
-; CHECK: .Lheapallocsite1:
+; CHECK:       callq *{{.*}}%rax{{.*}}
+; CHECK-NEXT:  [[LABEL1:.Ltmp[0-9]+]]:
 
 ; CHECK-LABEL: call_multiple:     # @call_multiple
-; FastISel emits instructions in a different order.
-; DAG:   .Lheapallocsite2:
-; FAST:  .Lheapallocsite4:
-; CHECK: callq alloc_foo
-; DAG:   .Lheapallocsite3:
-; FAST:  .Lheapallocsite5:
-; DAG:   .Lheapallocsite4:
-; FAST:  .Lheapallocsite2:
-; CHECK: callq alloc_foo
-; DAG:   .Lheapallocsite5:
-; FAST:  .Lheapallocsite3:
+; CHECK:       callq alloc_foo
+; CHECK-NEXT:  [[LABEL3:.Ltmp[0-9]+]]:
+; CHECK:       callq alloc_foo
+; CHECK-NEXT:  [[LABEL5:.Ltmp[0-9]+]]:
 
 ; CHECK-LABEL: .short  4423                    # Record kind: S_GPROC32_ID
 ; CHECK:       .short  4446                    # Record kind: S_HEAPALLOCSITE
-; CHECK-NEXT:  .secrel32 .Lheapallocsite0
-; CHECK-NEXT:  .secidx .Lheapallocsite0
-; CHECK-NEXT:  .short .Lheapallocsite1-.Lheapallocsite0
+; CHECK-NEXT:  .secrel32 [[LABEL0:.Ltmp[0-9]+]]
+; CHECK-NEXT:  .secidx [[LABEL0]]
+; CHECK-NEXT:  .short [[LABEL1]]-[[LABEL0]]
 ; CHECK-NEXT:  .long 3
 ; CHECK:       .short  4446                    # Record kind: S_HEAPALLOCSITE
-; CHECK-NEXT:  .secrel32 .Lheapallocsite2
-; CHECK-NEXT:  .secidx .Lheapallocsite2
-; CHECK-NEXT:  .short .Lheapallocsite3-.Lheapallocsite2
+; CHECK-NEXT:  .secrel32 [[LABEL2:.Ltmp[0-9]+]]
+; CHECK-NEXT:  .secidx [[LABEL2]]
+; CHECK-NEXT:  .short [[LABEL3]]-[[LABEL2]]
 ; CHECK-NEXT:  .long 4096
 ; CHECK:       .short  4446                    # Record kind: S_HEAPALLOCSITE
-; CHECK-NEXT:  .secrel32 .Lheapallocsite4
-; CHECK-NEXT:  .secidx .Lheapallocsite4
-; CHECK-NEXT:  .short .Lheapallocsite5-.Lheapallocsite4
+; CHECK-NEXT:  .secrel32 [[LABEL4:.Ltmp[0-9]+]]
+; CHECK-NEXT:  .secidx [[LABEL4]]
+; CHECK-NEXT:  .short [[LABEL5]]-[[LABEL4]]
 ; CHECK-NEXT:  .long 4096
 
 attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-features"="+cx8,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/X86/large-constants.ll b/llvm/test/CodeGen/X86/large-constants.ll
index 945d633ee..be7e598fd 100644
--- a/llvm/test/CodeGen/X86/large-constants.ll
+++ b/llvm/test/CodeGen/X86/large-constants.ll
@@ -1,6 +1,35 @@
-; RUN: llc < %s -mtriple=x86_64-darwin -mcpu=corei7 | grep movabsq | count 3
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-darwin -mcpu=corei7 | FileCheck %s
 
 define i64 @constant_hoisting(i64 %o0, i64 %o1, i64 %o2, i64 %o3, i64 %o4, i64 %o5) {
+; CHECK-LABEL: constant_hoisting:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    movabsq $-281474976710654, %rax ## imm = 0xFFFF000000000002
+; CHECK-NEXT:    testq %rax, %rdi
+; CHECK-NEXT:    jne LBB0_7
+; CHECK-NEXT:  ## %bb.1: ## %bb1
+; CHECK-NEXT:    testq %rax, %rsi
+; CHECK-NEXT:    jne LBB0_7
+; CHECK-NEXT:  ## %bb.2: ## %bb2
+; CHECK-NEXT:    testq %rax, %rdx
+; CHECK-NEXT:    jne LBB0_7
+; CHECK-NEXT:  ## %bb.3: ## %bb3
+; CHECK-NEXT:    testq %rax, %rcx
+; CHECK-NEXT:    jne LBB0_7
+; CHECK-NEXT:  ## %bb.4: ## %bb4
+; CHECK-NEXT:    leaq 1(%rax), %rcx
+; CHECK-NEXT:    testq %rcx, %r8
+; CHECK-NEXT:    jne LBB0_7
+; CHECK-NEXT:  ## %bb.5: ## %bb5
+; CHECK-NEXT:    addq $2, %rax
+; CHECK-NEXT:    andq %rax, %r9
+; CHECK-NEXT:    je LBB0_6
+; CHECK-NEXT:  LBB0_7: ## %fail
+; CHECK-NEXT:    movq $-1, %rax
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  LBB0_6: ## %bb6
+; CHECK-NEXT:    movq %r9, %rax
+; CHECK-NEXT:    retq
 entry:
   %l0 = and i64 %o0, -281474976710654
   %c0 = icmp ne i64 %l0, 0
@@ -39,6 +68,16 @@ fail:
 }
 
 define void @constant_expressions() {
+; CHECK-LABEL: constant_expressions:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    movabsq $51250129900, %rax ## imm = 0xBEEBEEBEC
+; CHECK-NEXT:    movq (%rax), %rcx
+; CHECK-NEXT:    movq 16(%rax), %rdx
+; CHECK-NEXT:    addq 8(%rax), %rcx
+; CHECK-NEXT:    addq 24(%rax), %rdx
+; CHECK-NEXT:    addq %rcx, %rdx
+; CHECK-NEXT:    movq %rdx, (%rax)
+; CHECK-NEXT:    retq
 entry:
   %0 = load i64, i64* inttoptr (i64 add (i64 51250129900, i64 0) to i64*)
   %1 = load i64, i64* inttoptr (i64 add (i64 51250129900, i64 8) to i64*)
@@ -53,6 +92,16 @@ entry:
 
 
 define void @constant_expressions2() {
+; CHECK-LABEL: constant_expressions2:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    movabsq $51250129900, %rax ## imm = 0xBEEBEEBEC
+; CHECK-NEXT:    movq (%rax), %rcx
+; CHECK-NEXT:    movq 16(%rax), %rdx
+; CHECK-NEXT:    addq 8(%rax), %rcx
+; CHECK-NEXT:    addq 24(%rax), %rdx
+; CHECK-NEXT:    addq %rcx, %rdx
+; CHECK-NEXT:    movq %rdx, (%rax)
+; CHECK-NEXT:    retq
 entry:
   %0 = load i64, i64* inttoptr (i64 51250129900 to i64*)
   %1 = load i64, i64* inttoptr (i64 51250129908 to i64*)
diff --git a/llvm/test/CodeGen/X86/lea-opt-memop-check-1.ll b/llvm/test/CodeGen/X86/lea-opt-memop-check-1.ll
index 47ea1ace1..f292203d3 100644
--- a/llvm/test/CodeGen/X86/lea-opt-memop-check-1.ll
+++ b/llvm/test/CodeGen/X86/lea-opt-memop-check-1.ll
@@ -96,4 +96,4 @@ if.end:
 ; CHECK:	pushl ([[REG3]])
 }
 
-attributes #0 = { nounwind optsize "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-features"="+mmx,+pclmul,+popcnt,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind optsize "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-features"="+mmx,+pclmul,+popcnt,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/X86/lea-recursion.ll b/llvm/test/CodeGen/X86/lea-recursion.ll
index 5bba1141e..9bc50f60c 100644
--- a/llvm/test/CodeGen/X86/lea-recursion.ll
+++ b/llvm/test/CodeGen/X86/lea-recursion.ll
@@ -1,4 +1,5 @@
-; RUN: llc < %s -mtriple=x86_64-- | grep lea | count 13
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-- | FileCheck %s
 
 ; This testcase was written to demonstrate an instruction-selection problem,
 ; however it also happens to expose a limitation in the DAGCombiner's
@@ -12,6 +13,37 @@
 @g1 = weak global [1000 x i32] zeroinitializer, align 32		; <[1000 x i32]*> [#uses=7]
 
 define void @foo() {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movl {{.*}}(%rip), %eax
+; CHECK-NEXT:    movl {{.*}}(%rip), %ecx
+; CHECK-NEXT:    leal (%rax,%rcx), %edx
+; CHECK-NEXT:    leal 1(%rax,%rcx), %eax
+; CHECK-NEXT:    movl %eax, g0+{{.*}}(%rip)
+; CHECK-NEXT:    movl g1+{{.*}}(%rip), %eax
+; CHECK-NEXT:    leal 1(%rax,%rdx), %ecx
+; CHECK-NEXT:    leal 2(%rax,%rdx), %eax
+; CHECK-NEXT:    movl %eax, g0+{{.*}}(%rip)
+; CHECK-NEXT:    movl g1+{{.*}}(%rip), %eax
+; CHECK-NEXT:    leal 1(%rax,%rcx), %edx
+; CHECK-NEXT:    leal 2(%rax,%rcx), %eax
+; CHECK-NEXT:    movl %eax, g0+{{.*}}(%rip)
+; CHECK-NEXT:    movl g1+{{.*}}(%rip), %eax
+; CHECK-NEXT:    leal 1(%rax,%rdx), %ecx
+; CHECK-NEXT:    leal 2(%rax,%rdx), %eax
+; CHECK-NEXT:    movl %eax, g0+{{.*}}(%rip)
+; CHECK-NEXT:    movl g1+{{.*}}(%rip), %eax
+; CHECK-NEXT:    leal 1(%rax,%rcx), %edx
+; CHECK-NEXT:    leal 2(%rax,%rcx), %eax
+; CHECK-NEXT:    movl %eax, g0+{{.*}}(%rip)
+; CHECK-NEXT:    movl g1+{{.*}}(%rip), %eax
+; CHECK-NEXT:    leal 1(%rax,%rdx), %ecx
+; CHECK-NEXT:    leal 2(%rax,%rdx), %eax
+; CHECK-NEXT:    movl %eax, g0+{{.*}}(%rip)
+; CHECK-NEXT:    movl g1+{{.*}}(%rip), %eax
+; CHECK-NEXT:    leal 2(%rax,%rcx), %eax
+; CHECK-NEXT:    movl %eax, g0+{{.*}}(%rip)
+; CHECK-NEXT:    retq
 entry:
 	%tmp4 = load i32, i32* getelementptr ([1000 x i32], [1000 x i32]* @g0, i32 0, i32 0)		; <i32> [#uses=1]
 	%tmp8 = load i32, i32* getelementptr ([1000 x i32], [1000 x i32]* @g1, i32 0, i32 0)		; <i32> [#uses=1]
diff --git a/llvm/test/CodeGen/X86/lea.ll b/llvm/test/CodeGen/X86/lea.ll
index a326b1ef4..33d121f68 100644
--- a/llvm/test/CodeGen/X86/lea.ll
+++ b/llvm/test/CodeGen/X86/lea.ll
@@ -23,7 +23,7 @@ define i32 @test1(i32 %x) nounwind {
 
 
 ; ISel the add of -4 with a neg and use an lea for the rest of the
-; arithemtic.
+; arithmetic.
 define i32 @test2(i32 %x_offs) nounwind readnone {
 ; LINUX-LABEL: test2:
 ; LINUX:       # %bb.0: # %entry
diff --git a/llvm/test/CodeGen/X86/leaf-fp-elim.ll b/llvm/test/CodeGen/X86/leaf-fp-elim.ll
index dfbfbf7d1..4a4e9b444 100644
--- a/llvm/test/CodeGen/X86/leaf-fp-elim.ll
+++ b/llvm/test/CodeGen/X86/leaf-fp-elim.ll
@@ -6,7 +6,7 @@ target triple = "x86_64-apple-darwin11.0"
 @msg = internal global i8* null                   ; <i8**> [#uses=1]
 @.str = private constant [2 x i8] c"x\00", align 1 ; <[2 x i8]*> [#uses=1]
 
-define void @test(i8* %p) "no-frame-pointer-elim-non-leaf" nounwind optsize ssp {
+define void @test(i8* %p) "frame-pointer"="non-leaf" nounwind optsize ssp {
 
 ; No stack frame, please.
 ; CHECK:     _test
diff --git a/llvm/test/CodeGen/X86/lifetime-alias.ll b/llvm/test/CodeGen/X86/lifetime-alias.ll
index 7631766c2..010dc33b5 100644
--- a/llvm/test/CodeGen/X86/lifetime-alias.ll
+++ b/llvm/test/CodeGen/X86/lifetime-alias.ll
@@ -23,7 +23,7 @@ target triple = "x86_64-unknown-linux-gnu"
 @__PRETTY_FUNCTION__.main = private unnamed_addr constant [11 x i8] c"int main()\00", align 1
 
 ; Function Attrs: norecurse uwtable
-define dso_local i8 @main() local_unnamed_addr #0 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+define i8 @main() local_unnamed_addr #0 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 ; CHECK-LABEL: main:
 ; CHECK:       # %bb.0: # %_ZNSt3__312basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEED2Ev.exit50
 ; CHECK-NEXT:    pushq %rax
@@ -35,6 +35,8 @@ define dso_local i8 @main() local_unnamed_addr #0 personality i8* bitcast (i32 (
 ; CHECK-NEXT:    movw $5632, {{[0-9]+}}(%rsp) # imm = 0x1600
 ; CHECK-NEXT:    xorps %xmm0, %xmm0
 ; CHECK-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movb $11, -{{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movabsq $8389209137051166804, %rax # imm = 0x746C754320656854
 ; CHECK-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movl $1701999988, -{{[0-9]+}}(%rsp) # imm = 0x65727574
@@ -47,7 +49,7 @@ define dso_local i8 @main() local_unnamed_addr #0 personality i8* bitcast (i32 (
 ; CHECK-NEXT:    movups {{.*}}(%rip), %xmm1
 ; CHECK-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movb $0, -{{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movabsq $792633534417207296, %rax # imm = 0xB00000000000000
+; CHECK-NEXT:    movq -{{[0-9]+}}(%rsp), %rax
 ; CHECK-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movq -{{[0-9]+}}(%rsp), %rax
 ; CHECK-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
@@ -68,9 +70,9 @@ define dso_local i8 @main() local_unnamed_addr #0 personality i8* bitcast (i32 (
 ; CHECK-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    leaq -{{[0-9]+}}(%rsp), %rax
-; CHECK-NEXT:    movq %rax, {{.*}}(%rip)
+; CHECK-NEXT:    movq %rax, do_not_optimize{{.*}}(%rip)
 ; CHECK-NEXT:    leaq -{{[0-9]+}}(%rsp), %rax
-; CHECK-NEXT:    movq %rax, {{.*}}(%rip)
+; CHECK-NEXT:    movq %rax, do_not_optimize{{.*}}(%rip)
 ; CHECK-NEXT:    cmpb $0, -{{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    jns .LBB0_1
 ; CHECK-NEXT:  # %bb.2: # %_ZNSt3__312basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEED2Ev.exit50
@@ -148,10 +150,10 @@ declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture writeonly, i8* nocapture r
 ; Function Attrs: argmemonly nounwind
 declare void @llvm.memset.p0i8.i64(i8* nocapture writeonly, i8, i64, i1 immarg) #1
 
-attributes #0 = { norecurse uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { norecurse uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { argmemonly nounwind }
-attributes #2 = { noreturn nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #3 = { nobuiltin nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { noreturn nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { nobuiltin nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #4 = { nounwind }
 attributes #5 = { noreturn nounwind }
 attributes #6 = { builtin nounwind }
diff --git a/llvm/test/CodeGen/X86/limited-prec.ll b/llvm/test/CodeGen/X86/limited-prec.ll
index 07291f854..515a0bdf6 100644
--- a/llvm/test/CodeGen/X86/limited-prec.ll
+++ b/llvm/test/CodeGen/X86/limited-prec.ll
@@ -1,11 +1,103 @@
-; RUN: llc < %s -limit-float-precision=6 -mtriple=i686-- | \
-; RUN:    not grep exp | not grep log | not grep pow
-; RUN: llc < %s -limit-float-precision=12 -mtriple=i686-- | \
-; RUN:    not grep exp | not grep log | not grep pow
-; RUN: llc < %s -limit-float-precision=18 -mtriple=i686-- | \
-; RUN:    not grep exp | not grep log | not grep pow
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -limit-float-precision=6 -mtriple=i686-- | FileCheck %s --check-prefixes=ALL,precision6
+; RUN: llc < %s -limit-float-precision=12 -mtriple=i686-- | FileCheck %s --check-prefixes=ALL,precision12
+; RUN: llc < %s -limit-float-precision=18 -mtriple=i686-- | FileCheck %s --check-prefixes=ALL,precision18
 
 define float @f1(float %x) nounwind noinline {
+; precision6-LABEL: f1:
+; precision6:       # %bb.0: # %entry
+; precision6-NEXT:    subl $20, %esp
+; precision6-NEXT:    flds {{[0-9]+}}(%esp)
+; precision6-NEXT:    fmuls {{\.LCPI.*}}
+; precision6-NEXT:    fnstcw (%esp)
+; precision6-NEXT:    movzwl (%esp), %eax
+; precision6-NEXT:    orl $3072, %eax # imm = 0xC00
+; precision6-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; precision6-NEXT:    fldcw {{[0-9]+}}(%esp)
+; precision6-NEXT:    fistl {{[0-9]+}}(%esp)
+; precision6-NEXT:    fldcw (%esp)
+; precision6-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; precision6-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; precision6-NEXT:    fisubl {{[0-9]+}}(%esp)
+; precision6-NEXT:    fld %st(0)
+; precision6-NEXT:    fmuls {{\.LCPI.*}}
+; precision6-NEXT:    fadds {{\.LCPI.*}}
+; precision6-NEXT:    fmulp %st, %st(1)
+; precision6-NEXT:    fadds {{\.LCPI.*}}
+; precision6-NEXT:    fstps {{[0-9]+}}(%esp)
+; precision6-NEXT:    shll $23, %eax
+; precision6-NEXT:    addl {{[0-9]+}}(%esp), %eax
+; precision6-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; precision6-NEXT:    flds {{[0-9]+}}(%esp)
+; precision6-NEXT:    addl $20, %esp
+; precision6-NEXT:    retl
+;
+; precision12-LABEL: f1:
+; precision12:       # %bb.0: # %entry
+; precision12-NEXT:    subl $20, %esp
+; precision12-NEXT:    flds {{[0-9]+}}(%esp)
+; precision12-NEXT:    fmuls {{\.LCPI.*}}
+; precision12-NEXT:    fnstcw (%esp)
+; precision12-NEXT:    movzwl (%esp), %eax
+; precision12-NEXT:    orl $3072, %eax # imm = 0xC00
+; precision12-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; precision12-NEXT:    fldcw {{[0-9]+}}(%esp)
+; precision12-NEXT:    fistl {{[0-9]+}}(%esp)
+; precision12-NEXT:    fldcw (%esp)
+; precision12-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; precision12-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; precision12-NEXT:    fisubl {{[0-9]+}}(%esp)
+; precision12-NEXT:    fld %st(0)
+; precision12-NEXT:    fmuls {{\.LCPI.*}}
+; precision12-NEXT:    fadds {{\.LCPI.*}}
+; precision12-NEXT:    fmul %st(1), %st
+; precision12-NEXT:    fadds {{\.LCPI.*}}
+; precision12-NEXT:    fmulp %st, %st(1)
+; precision12-NEXT:    fadds {{\.LCPI.*}}
+; precision12-NEXT:    fstps {{[0-9]+}}(%esp)
+; precision12-NEXT:    shll $23, %eax
+; precision12-NEXT:    addl {{[0-9]+}}(%esp), %eax
+; precision12-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; precision12-NEXT:    flds {{[0-9]+}}(%esp)
+; precision12-NEXT:    addl $20, %esp
+; precision12-NEXT:    retl
+;
+; precision18-LABEL: f1:
+; precision18:       # %bb.0: # %entry
+; precision18-NEXT:    subl $20, %esp
+; precision18-NEXT:    flds {{[0-9]+}}(%esp)
+; precision18-NEXT:    fmuls {{\.LCPI.*}}
+; precision18-NEXT:    fnstcw (%esp)
+; precision18-NEXT:    movzwl (%esp), %eax
+; precision18-NEXT:    orl $3072, %eax # imm = 0xC00
+; precision18-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; precision18-NEXT:    fldcw {{[0-9]+}}(%esp)
+; precision18-NEXT:    fistl {{[0-9]+}}(%esp)
+; precision18-NEXT:    fldcw (%esp)
+; precision18-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; precision18-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; precision18-NEXT:    fisubl {{[0-9]+}}(%esp)
+; precision18-NEXT:    fld %st(0)
+; precision18-NEXT:    fmuls {{\.LCPI.*}}
+; precision18-NEXT:    fadds {{\.LCPI.*}}
+; precision18-NEXT:    fmul %st(1), %st
+; precision18-NEXT:    fadds {{\.LCPI.*}}
+; precision18-NEXT:    fmul %st(1), %st
+; precision18-NEXT:    fadds {{\.LCPI.*}}
+; precision18-NEXT:    fmul %st(1), %st
+; precision18-NEXT:    fadds {{\.LCPI.*}}
+; precision18-NEXT:    fmul %st(1), %st
+; precision18-NEXT:    fadds {{\.LCPI.*}}
+; precision18-NEXT:    fmulp %st, %st(1)
+; precision18-NEXT:    fld1
+; precision18-NEXT:    faddp %st, %st(1)
+; precision18-NEXT:    fstps {{[0-9]+}}(%esp)
+; precision18-NEXT:    shll $23, %eax
+; precision18-NEXT:    addl {{[0-9]+}}(%esp), %eax
+; precision18-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; precision18-NEXT:    flds {{[0-9]+}}(%esp)
+; precision18-NEXT:    addl $20, %esp
+; precision18-NEXT:    retl
 entry:
 	%"alloca point" = bitcast i32 0 to i32		; <i32> [#uses=0]
 	%0 = call float @llvm.exp.f32(float %x)		; <float> [#uses=1]
@@ -15,6 +107,97 @@ entry:
 declare float @llvm.exp.f32(float) nounwind readonly
 
 define float @f2(float %x) nounwind noinline {
+; precision6-LABEL: f2:
+; precision6:       # %bb.0: # %entry
+; precision6-NEXT:    subl $20, %esp
+; precision6-NEXT:    flds {{[0-9]+}}(%esp)
+; precision6-NEXT:    fnstcw (%esp)
+; precision6-NEXT:    movzwl (%esp), %eax
+; precision6-NEXT:    orl $3072, %eax # imm = 0xC00
+; precision6-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; precision6-NEXT:    fldcw {{[0-9]+}}(%esp)
+; precision6-NEXT:    fistl {{[0-9]+}}(%esp)
+; precision6-NEXT:    fldcw (%esp)
+; precision6-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; precision6-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; precision6-NEXT:    fisubl {{[0-9]+}}(%esp)
+; precision6-NEXT:    fld %st(0)
+; precision6-NEXT:    fmuls {{\.LCPI.*}}
+; precision6-NEXT:    fadds {{\.LCPI.*}}
+; precision6-NEXT:    fmulp %st, %st(1)
+; precision6-NEXT:    fadds {{\.LCPI.*}}
+; precision6-NEXT:    fstps {{[0-9]+}}(%esp)
+; precision6-NEXT:    shll $23, %eax
+; precision6-NEXT:    addl {{[0-9]+}}(%esp), %eax
+; precision6-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; precision6-NEXT:    flds {{[0-9]+}}(%esp)
+; precision6-NEXT:    addl $20, %esp
+; precision6-NEXT:    retl
+;
+; precision12-LABEL: f2:
+; precision12:       # %bb.0: # %entry
+; precision12-NEXT:    subl $20, %esp
+; precision12-NEXT:    flds {{[0-9]+}}(%esp)
+; precision12-NEXT:    fnstcw (%esp)
+; precision12-NEXT:    movzwl (%esp), %eax
+; precision12-NEXT:    orl $3072, %eax # imm = 0xC00
+; precision12-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; precision12-NEXT:    fldcw {{[0-9]+}}(%esp)
+; precision12-NEXT:    fistl {{[0-9]+}}(%esp)
+; precision12-NEXT:    fldcw (%esp)
+; precision12-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; precision12-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; precision12-NEXT:    fisubl {{[0-9]+}}(%esp)
+; precision12-NEXT:    fld %st(0)
+; precision12-NEXT:    fmuls {{\.LCPI.*}}
+; precision12-NEXT:    fadds {{\.LCPI.*}}
+; precision12-NEXT:    fmul %st(1), %st
+; precision12-NEXT:    fadds {{\.LCPI.*}}
+; precision12-NEXT:    fmulp %st, %st(1)
+; precision12-NEXT:    fadds {{\.LCPI.*}}
+; precision12-NEXT:    fstps {{[0-9]+}}(%esp)
+; precision12-NEXT:    shll $23, %eax
+; precision12-NEXT:    addl {{[0-9]+}}(%esp), %eax
+; precision12-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; precision12-NEXT:    flds {{[0-9]+}}(%esp)
+; precision12-NEXT:    addl $20, %esp
+; precision12-NEXT:    retl
+;
+; precision18-LABEL: f2:
+; precision18:       # %bb.0: # %entry
+; precision18-NEXT:    subl $20, %esp
+; precision18-NEXT:    flds {{[0-9]+}}(%esp)
+; precision18-NEXT:    fnstcw (%esp)
+; precision18-NEXT:    movzwl (%esp), %eax
+; precision18-NEXT:    orl $3072, %eax # imm = 0xC00
+; precision18-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; precision18-NEXT:    fldcw {{[0-9]+}}(%esp)
+; precision18-NEXT:    fistl {{[0-9]+}}(%esp)
+; precision18-NEXT:    fldcw (%esp)
+; precision18-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; precision18-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; precision18-NEXT:    fisubl {{[0-9]+}}(%esp)
+; precision18-NEXT:    fld %st(0)
+; precision18-NEXT:    fmuls {{\.LCPI.*}}
+; precision18-NEXT:    fadds {{\.LCPI.*}}
+; precision18-NEXT:    fmul %st(1), %st
+; precision18-NEXT:    fadds {{\.LCPI.*}}
+; precision18-NEXT:    fmul %st(1), %st
+; precision18-NEXT:    fadds {{\.LCPI.*}}
+; precision18-NEXT:    fmul %st(1), %st
+; precision18-NEXT:    fadds {{\.LCPI.*}}
+; precision18-NEXT:    fmul %st(1), %st
+; precision18-NEXT:    fadds {{\.LCPI.*}}
+; precision18-NEXT:    fmulp %st, %st(1)
+; precision18-NEXT:    fld1
+; precision18-NEXT:    faddp %st, %st(1)
+; precision18-NEXT:    fstps {{[0-9]+}}(%esp)
+; precision18-NEXT:    shll $23, %eax
+; precision18-NEXT:    addl {{[0-9]+}}(%esp), %eax
+; precision18-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; precision18-NEXT:    flds {{[0-9]+}}(%esp)
+; precision18-NEXT:    addl $20, %esp
+; precision18-NEXT:    retl
 entry:
 	%"alloca point" = bitcast i32 0 to i32		; <i32> [#uses=0]
 	%0 = call float @llvm.exp2.f32(float %x)		; <float> [#uses=1]
@@ -24,6 +207,100 @@ entry:
 declare float @llvm.exp2.f32(float) nounwind readonly
 
 define float @f3(float %x) nounwind noinline {
+; precision6-LABEL: f3:
+; precision6:       # %bb.0: # %entry
+; precision6-NEXT:    subl $20, %esp
+; precision6-NEXT:    flds {{[0-9]+}}(%esp)
+; precision6-NEXT:    fmuls {{\.LCPI.*}}
+; precision6-NEXT:    fnstcw (%esp)
+; precision6-NEXT:    movzwl (%esp), %eax
+; precision6-NEXT:    orl $3072, %eax # imm = 0xC00
+; precision6-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; precision6-NEXT:    fldcw {{[0-9]+}}(%esp)
+; precision6-NEXT:    fistl {{[0-9]+}}(%esp)
+; precision6-NEXT:    fldcw (%esp)
+; precision6-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; precision6-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; precision6-NEXT:    fisubl {{[0-9]+}}(%esp)
+; precision6-NEXT:    fld %st(0)
+; precision6-NEXT:    fmuls {{\.LCPI.*}}
+; precision6-NEXT:    fadds {{\.LCPI.*}}
+; precision6-NEXT:    fmulp %st, %st(1)
+; precision6-NEXT:    fadds {{\.LCPI.*}}
+; precision6-NEXT:    fstps {{[0-9]+}}(%esp)
+; precision6-NEXT:    shll $23, %eax
+; precision6-NEXT:    addl {{[0-9]+}}(%esp), %eax
+; precision6-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; precision6-NEXT:    flds {{[0-9]+}}(%esp)
+; precision6-NEXT:    addl $20, %esp
+; precision6-NEXT:    retl
+;
+; precision12-LABEL: f3:
+; precision12:       # %bb.0: # %entry
+; precision12-NEXT:    subl $20, %esp
+; precision12-NEXT:    flds {{[0-9]+}}(%esp)
+; precision12-NEXT:    fmuls {{\.LCPI.*}}
+; precision12-NEXT:    fnstcw (%esp)
+; precision12-NEXT:    movzwl (%esp), %eax
+; precision12-NEXT:    orl $3072, %eax # imm = 0xC00
+; precision12-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; precision12-NEXT:    fldcw {{[0-9]+}}(%esp)
+; precision12-NEXT:    fistl {{[0-9]+}}(%esp)
+; precision12-NEXT:    fldcw (%esp)
+; precision12-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; precision12-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; precision12-NEXT:    fisubl {{[0-9]+}}(%esp)
+; precision12-NEXT:    fld %st(0)
+; precision12-NEXT:    fmuls {{\.LCPI.*}}
+; precision12-NEXT:    fadds {{\.LCPI.*}}
+; precision12-NEXT:    fmul %st(1), %st
+; precision12-NEXT:    fadds {{\.LCPI.*}}
+; precision12-NEXT:    fmulp %st, %st(1)
+; precision12-NEXT:    fadds {{\.LCPI.*}}
+; precision12-NEXT:    fstps {{[0-9]+}}(%esp)
+; precision12-NEXT:    shll $23, %eax
+; precision12-NEXT:    addl {{[0-9]+}}(%esp), %eax
+; precision12-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; precision12-NEXT:    flds {{[0-9]+}}(%esp)
+; precision12-NEXT:    addl $20, %esp
+; precision12-NEXT:    retl
+;
+; precision18-LABEL: f3:
+; precision18:       # %bb.0: # %entry
+; precision18-NEXT:    subl $20, %esp
+; precision18-NEXT:    flds {{[0-9]+}}(%esp)
+; precision18-NEXT:    fmuls {{\.LCPI.*}}
+; precision18-NEXT:    fnstcw (%esp)
+; precision18-NEXT:    movzwl (%esp), %eax
+; precision18-NEXT:    orl $3072, %eax # imm = 0xC00
+; precision18-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; precision18-NEXT:    fldcw {{[0-9]+}}(%esp)
+; precision18-NEXT:    fistl {{[0-9]+}}(%esp)
+; precision18-NEXT:    fldcw (%esp)
+; precision18-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; precision18-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; precision18-NEXT:    fisubl {{[0-9]+}}(%esp)
+; precision18-NEXT:    fld %st(0)
+; precision18-NEXT:    fmuls {{\.LCPI.*}}
+; precision18-NEXT:    fadds {{\.LCPI.*}}
+; precision18-NEXT:    fmul %st(1), %st
+; precision18-NEXT:    fadds {{\.LCPI.*}}
+; precision18-NEXT:    fmul %st(1), %st
+; precision18-NEXT:    fadds {{\.LCPI.*}}
+; precision18-NEXT:    fmul %st(1), %st
+; precision18-NEXT:    fadds {{\.LCPI.*}}
+; precision18-NEXT:    fmul %st(1), %st
+; precision18-NEXT:    fadds {{\.LCPI.*}}
+; precision18-NEXT:    fmulp %st, %st(1)
+; precision18-NEXT:    fld1
+; precision18-NEXT:    faddp %st, %st(1)
+; precision18-NEXT:    fstps {{[0-9]+}}(%esp)
+; precision18-NEXT:    shll $23, %eax
+; precision18-NEXT:    addl {{[0-9]+}}(%esp), %eax
+; precision18-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; precision18-NEXT:    flds {{[0-9]+}}(%esp)
+; precision18-NEXT:    addl $20, %esp
+; precision18-NEXT:    retl
 entry:
 	%"alloca point" = bitcast i32 0 to i32		; <i32> [#uses=0]
 	%0 = call float @llvm.pow.f32(float 1.000000e+01, float %x)		; <float> [#uses=1]
@@ -33,6 +310,89 @@ entry:
 declare float @llvm.pow.f32(float, float) nounwind readonly
 
 define float @f4(float %x) nounwind noinline {
+; precision6-LABEL: f4:
+; precision6:       # %bb.0: # %entry
+; precision6-NEXT:    subl $8, %esp
+; precision6-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; precision6-NEXT:    movl %eax, %ecx
+; precision6-NEXT:    andl $8388607, %ecx # imm = 0x7FFFFF
+; precision6-NEXT:    orl $1065353216, %ecx # imm = 0x3F800000
+; precision6-NEXT:    movl %ecx, (%esp)
+; precision6-NEXT:    andl $2139095040, %eax # imm = 0x7F800000
+; precision6-NEXT:    shrl $23, %eax
+; precision6-NEXT:    addl $-127, %eax
+; precision6-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; precision6-NEXT:    flds (%esp)
+; precision6-NEXT:    fld %st(0)
+; precision6-NEXT:    fmuls {{\.LCPI.*}}
+; precision6-NEXT:    fadds {{\.LCPI.*}}
+; precision6-NEXT:    fmulp %st, %st(1)
+; precision6-NEXT:    fadds {{\.LCPI.*}}
+; precision6-NEXT:    fildl {{[0-9]+}}(%esp)
+; precision6-NEXT:    fmuls {{\.LCPI.*}}
+; precision6-NEXT:    faddp %st, %st(1)
+; precision6-NEXT:    addl $8, %esp
+; precision6-NEXT:    retl
+;
+; precision12-LABEL: f4:
+; precision12:       # %bb.0: # %entry
+; precision12-NEXT:    subl $8, %esp
+; precision12-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; precision12-NEXT:    movl %eax, %ecx
+; precision12-NEXT:    andl $8388607, %ecx # imm = 0x7FFFFF
+; precision12-NEXT:    orl $1065353216, %ecx # imm = 0x3F800000
+; precision12-NEXT:    movl %ecx, (%esp)
+; precision12-NEXT:    andl $2139095040, %eax # imm = 0x7F800000
+; precision12-NEXT:    shrl $23, %eax
+; precision12-NEXT:    addl $-127, %eax
+; precision12-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; precision12-NEXT:    flds (%esp)
+; precision12-NEXT:    fld %st(0)
+; precision12-NEXT:    fmuls {{\.LCPI.*}}
+; precision12-NEXT:    fadds {{\.LCPI.*}}
+; precision12-NEXT:    fmul %st(1), %st
+; precision12-NEXT:    fadds {{\.LCPI.*}}
+; precision12-NEXT:    fmul %st(1), %st
+; precision12-NEXT:    fadds {{\.LCPI.*}}
+; precision12-NEXT:    fmulp %st, %st(1)
+; precision12-NEXT:    fadds {{\.LCPI.*}}
+; precision12-NEXT:    fildl {{[0-9]+}}(%esp)
+; precision12-NEXT:    fmuls {{\.LCPI.*}}
+; precision12-NEXT:    faddp %st, %st(1)
+; precision12-NEXT:    addl $8, %esp
+; precision12-NEXT:    retl
+;
+; precision18-LABEL: f4:
+; precision18:       # %bb.0: # %entry
+; precision18-NEXT:    subl $8, %esp
+; precision18-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; precision18-NEXT:    movl %eax, %ecx
+; precision18-NEXT:    andl $8388607, %ecx # imm = 0x7FFFFF
+; precision18-NEXT:    orl $1065353216, %ecx # imm = 0x3F800000
+; precision18-NEXT:    movl %ecx, (%esp)
+; precision18-NEXT:    andl $2139095040, %eax # imm = 0x7F800000
+; precision18-NEXT:    shrl $23, %eax
+; precision18-NEXT:    addl $-127, %eax
+; precision18-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; precision18-NEXT:    flds (%esp)
+; precision18-NEXT:    fld %st(0)
+; precision18-NEXT:    fmuls {{\.LCPI.*}}
+; precision18-NEXT:    fadds {{\.LCPI.*}}
+; precision18-NEXT:    fmul %st(1), %st
+; precision18-NEXT:    fadds {{\.LCPI.*}}
+; precision18-NEXT:    fmul %st(1), %st
+; precision18-NEXT:    fadds {{\.LCPI.*}}
+; precision18-NEXT:    fmul %st(1), %st
+; precision18-NEXT:    fadds {{\.LCPI.*}}
+; precision18-NEXT:    fmul %st(1), %st
+; precision18-NEXT:    fadds {{\.LCPI.*}}
+; precision18-NEXT:    fmulp %st, %st(1)
+; precision18-NEXT:    fadds {{\.LCPI.*}}
+; precision18-NEXT:    fildl {{[0-9]+}}(%esp)
+; precision18-NEXT:    fmuls {{\.LCPI.*}}
+; precision18-NEXT:    faddp %st, %st(1)
+; precision18-NEXT:    addl $8, %esp
+; precision18-NEXT:    retl
 entry:
 	%"alloca point" = bitcast i32 0 to i32		; <i32> [#uses=0]
 	%0 = call float @llvm.log.f32(float %x)		; <float> [#uses=1]
@@ -42,6 +402,83 @@ entry:
 declare float @llvm.log.f32(float) nounwind readonly
 
 define float @f5(float %x) nounwind noinline {
+; precision6-LABEL: f5:
+; precision6:       # %bb.0: # %entry
+; precision6-NEXT:    subl $8, %esp
+; precision6-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; precision6-NEXT:    movl %eax, %ecx
+; precision6-NEXT:    andl $8388607, %ecx # imm = 0x7FFFFF
+; precision6-NEXT:    orl $1065353216, %ecx # imm = 0x3F800000
+; precision6-NEXT:    movl %ecx, (%esp)
+; precision6-NEXT:    andl $2139095040, %eax # imm = 0x7F800000
+; precision6-NEXT:    shrl $23, %eax
+; precision6-NEXT:    addl $-127, %eax
+; precision6-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; precision6-NEXT:    flds (%esp)
+; precision6-NEXT:    fld %st(0)
+; precision6-NEXT:    fmuls {{\.LCPI.*}}
+; precision6-NEXT:    fadds {{\.LCPI.*}}
+; precision6-NEXT:    fmulp %st, %st(1)
+; precision6-NEXT:    fadds {{\.LCPI.*}}
+; precision6-NEXT:    fiaddl {{[0-9]+}}(%esp)
+; precision6-NEXT:    addl $8, %esp
+; precision6-NEXT:    retl
+;
+; precision12-LABEL: f5:
+; precision12:       # %bb.0: # %entry
+; precision12-NEXT:    subl $8, %esp
+; precision12-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; precision12-NEXT:    movl %eax, %ecx
+; precision12-NEXT:    andl $8388607, %ecx # imm = 0x7FFFFF
+; precision12-NEXT:    orl $1065353216, %ecx # imm = 0x3F800000
+; precision12-NEXT:    movl %ecx, (%esp)
+; precision12-NEXT:    andl $2139095040, %eax # imm = 0x7F800000
+; precision12-NEXT:    shrl $23, %eax
+; precision12-NEXT:    addl $-127, %eax
+; precision12-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; precision12-NEXT:    flds (%esp)
+; precision12-NEXT:    fld %st(0)
+; precision12-NEXT:    fmuls {{\.LCPI.*}}
+; precision12-NEXT:    fadds {{\.LCPI.*}}
+; precision12-NEXT:    fmul %st(1), %st
+; precision12-NEXT:    fadds {{\.LCPI.*}}
+; precision12-NEXT:    fmul %st(1), %st
+; precision12-NEXT:    fadds {{\.LCPI.*}}
+; precision12-NEXT:    fmulp %st, %st(1)
+; precision12-NEXT:    fadds {{\.LCPI.*}}
+; precision12-NEXT:    fiaddl {{[0-9]+}}(%esp)
+; precision12-NEXT:    addl $8, %esp
+; precision12-NEXT:    retl
+;
+; precision18-LABEL: f5:
+; precision18:       # %bb.0: # %entry
+; precision18-NEXT:    subl $8, %esp
+; precision18-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; precision18-NEXT:    movl %eax, %ecx
+; precision18-NEXT:    andl $8388607, %ecx # imm = 0x7FFFFF
+; precision18-NEXT:    orl $1065353216, %ecx # imm = 0x3F800000
+; precision18-NEXT:    movl %ecx, (%esp)
+; precision18-NEXT:    andl $2139095040, %eax # imm = 0x7F800000
+; precision18-NEXT:    shrl $23, %eax
+; precision18-NEXT:    addl $-127, %eax
+; precision18-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; precision18-NEXT:    flds (%esp)
+; precision18-NEXT:    fld %st(0)
+; precision18-NEXT:    fmuls {{\.LCPI.*}}
+; precision18-NEXT:    fadds {{\.LCPI.*}}
+; precision18-NEXT:    fmul %st(1), %st
+; precision18-NEXT:    fadds {{\.LCPI.*}}
+; precision18-NEXT:    fmul %st(1), %st
+; precision18-NEXT:    fadds {{\.LCPI.*}}
+; precision18-NEXT:    fmul %st(1), %st
+; precision18-NEXT:    fadds {{\.LCPI.*}}
+; precision18-NEXT:    fmul %st(1), %st
+; precision18-NEXT:    fadds {{\.LCPI.*}}
+; precision18-NEXT:    fmulp %st, %st(1)
+; precision18-NEXT:    fadds {{\.LCPI.*}}
+; precision18-NEXT:    fiaddl {{[0-9]+}}(%esp)
+; precision18-NEXT:    addl $8, %esp
+; precision18-NEXT:    retl
 entry:
 	%"alloca point" = bitcast i32 0 to i32		; <i32> [#uses=0]
 	%0 = call float @llvm.log2.f32(float %x)		; <float> [#uses=1]
@@ -51,6 +488,85 @@ entry:
 declare float @llvm.log2.f32(float) nounwind readonly
 
 define float @f6(float %x) nounwind noinline {
+; precision6-LABEL: f6:
+; precision6:       # %bb.0: # %entry
+; precision6-NEXT:    subl $8, %esp
+; precision6-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; precision6-NEXT:    movl %eax, %ecx
+; precision6-NEXT:    andl $8388607, %ecx # imm = 0x7FFFFF
+; precision6-NEXT:    orl $1065353216, %ecx # imm = 0x3F800000
+; precision6-NEXT:    movl %ecx, (%esp)
+; precision6-NEXT:    andl $2139095040, %eax # imm = 0x7F800000
+; precision6-NEXT:    shrl $23, %eax
+; precision6-NEXT:    addl $-127, %eax
+; precision6-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; precision6-NEXT:    flds (%esp)
+; precision6-NEXT:    fld %st(0)
+; precision6-NEXT:    fmuls {{\.LCPI.*}}
+; precision6-NEXT:    fadds {{\.LCPI.*}}
+; precision6-NEXT:    fmulp %st, %st(1)
+; precision6-NEXT:    fadds {{\.LCPI.*}}
+; precision6-NEXT:    fildl {{[0-9]+}}(%esp)
+; precision6-NEXT:    fmuls {{\.LCPI.*}}
+; precision6-NEXT:    faddp %st, %st(1)
+; precision6-NEXT:    addl $8, %esp
+; precision6-NEXT:    retl
+;
+; precision12-LABEL: f6:
+; precision12:       # %bb.0: # %entry
+; precision12-NEXT:    subl $8, %esp
+; precision12-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; precision12-NEXT:    movl %eax, %ecx
+; precision12-NEXT:    andl $8388607, %ecx # imm = 0x7FFFFF
+; precision12-NEXT:    orl $1065353216, %ecx # imm = 0x3F800000
+; precision12-NEXT:    movl %ecx, (%esp)
+; precision12-NEXT:    andl $2139095040, %eax # imm = 0x7F800000
+; precision12-NEXT:    shrl $23, %eax
+; precision12-NEXT:    addl $-127, %eax
+; precision12-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; precision12-NEXT:    flds (%esp)
+; precision12-NEXT:    fld %st(0)
+; precision12-NEXT:    fmuls {{\.LCPI.*}}
+; precision12-NEXT:    fadds {{\.LCPI.*}}
+; precision12-NEXT:    fmul %st(1), %st
+; precision12-NEXT:    fadds {{\.LCPI.*}}
+; precision12-NEXT:    fmulp %st, %st(1)
+; precision12-NEXT:    fadds {{\.LCPI.*}}
+; precision12-NEXT:    fildl {{[0-9]+}}(%esp)
+; precision12-NEXT:    fmuls {{\.LCPI.*}}
+; precision12-NEXT:    faddp %st, %st(1)
+; precision12-NEXT:    addl $8, %esp
+; precision12-NEXT:    retl
+;
+; precision18-LABEL: f6:
+; precision18:       # %bb.0: # %entry
+; precision18-NEXT:    subl $8, %esp
+; precision18-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; precision18-NEXT:    movl %eax, %ecx
+; precision18-NEXT:    andl $8388607, %ecx # imm = 0x7FFFFF
+; precision18-NEXT:    orl $1065353216, %ecx # imm = 0x3F800000
+; precision18-NEXT:    movl %ecx, (%esp)
+; precision18-NEXT:    andl $2139095040, %eax # imm = 0x7F800000
+; precision18-NEXT:    shrl $23, %eax
+; precision18-NEXT:    addl $-127, %eax
+; precision18-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; precision18-NEXT:    flds (%esp)
+; precision18-NEXT:    fld %st(0)
+; precision18-NEXT:    fmuls {{\.LCPI.*}}
+; precision18-NEXT:    fadds {{\.LCPI.*}}
+; precision18-NEXT:    fmul %st(1), %st
+; precision18-NEXT:    fadds {{\.LCPI.*}}
+; precision18-NEXT:    fmul %st(1), %st
+; precision18-NEXT:    fadds {{\.LCPI.*}}
+; precision18-NEXT:    fmul %st(1), %st
+; precision18-NEXT:    fadds {{\.LCPI.*}}
+; precision18-NEXT:    fmulp %st, %st(1)
+; precision18-NEXT:    fadds {{\.LCPI.*}}
+; precision18-NEXT:    fildl {{[0-9]+}}(%esp)
+; precision18-NEXT:    fmuls {{\.LCPI.*}}
+; precision18-NEXT:    faddp %st, %st(1)
+; precision18-NEXT:    addl $8, %esp
+; precision18-NEXT:    retl
 entry:
 	%"alloca point" = bitcast i32 0 to i32		; <i32> [#uses=0]
 	%0 = call float @llvm.log10.f32(float %x)		; <float> [#uses=1]
diff --git a/llvm/test/CodeGen/X86/llc-override-mcpu-mattr.ll b/llvm/test/CodeGen/X86/llc-override-mcpu-mattr.ll
index 293ceee3b..228f2031b 100644
--- a/llvm/test/CodeGen/X86/llc-override-mcpu-mattr.ll
+++ b/llvm/test/CodeGen/X86/llc-override-mcpu-mattr.ll
@@ -1,12 +1,12 @@
 ; RUN: llc < %s -mtriple=x86_64-- -mcpu=broadwell | FileCheck %s
 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2 | FileCheck %s
 
-; Check that llc can overide function attributes target-cpu and target-features
+; Check that llc can set function attributes target-cpu and target-features
 ; using command line options -mcpu and -mattr.
 
 ; CHECK: vpsadbw (%r{{si|dx}}), %ymm{{[0-9]+}}, %ymm{{[0-9]+}}
 
-define <4 x i64> @foo1(<4 x i64>* %s1, <4 x i64>* %s2) #0 {
+define <4 x i64> @foo1(<4 x i64>* %s1, <4 x i64>* %s2) {
 entry:
   %ps1 = load <4 x i64>, <4 x i64>* %s1
   %ps2 = load <4 x i64>, <4 x i64>* %s2
@@ -17,5 +17,3 @@ entry:
 }
 
 declare <4 x i64> @llvm.x86.avx2.psad.bw(<32 x i8>, <32 x i8>)
-
-attributes #0 = { "target-cpu"="core2" "target-features"="+ssse3,+cx16,+sse4.2,+sse4.1,+sse,+sse2,+sse3,+avx,+popcnt" }
diff --git a/llvm/test/CodeGen/X86/load-combine-dbg.ll b/llvm/test/CodeGen/X86/load-combine-dbg.ll
index 59e6e1ac3..5982589df 100644
--- a/llvm/test/CodeGen/X86/load-combine-dbg.ll
+++ b/llvm/test/CodeGen/X86/load-combine-dbg.ll
@@ -25,7 +25,7 @@ define i32 @zext_load(i32* %arg) !dbg !30 {
 
 !0 = !{i32 2, !"Debug Info Version", i32 3}
 !1 = distinct !DICompileUnit(language: DW_LANG_Swift, file: !3, isOptimized: false, emissionKind: FullDebug)
-!2 = !DIModule(scope: null, name: "test", includePath: "", isysroot: "/")
+!2 = !DIModule(scope: null, name: "test", includePath: "", sysroot: "/")
 !3 = !DIFile(filename: "main.swift", directory: "/")
 
 !30 = distinct !DISubprogram(name: "main", scope: !2, file: !3, line: 1, type: !31, isLocal: false, isDefinition: true, isOptimized: false, unit: !1)
diff --git a/llvm/test/CodeGen/X86/load-combine.ll b/llvm/test/CodeGen/X86/load-combine.ll
index 99e444eeb..5184e99d0 100644
--- a/llvm/test/CodeGen/X86/load-combine.ll
+++ b/llvm/test/CodeGen/X86/load-combine.ll
@@ -1016,7 +1016,7 @@ define i32 @load_i32_by_i8_zaext_loads(i8* %arg, i32 %arg1) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; CHECK-NEXT:    movl 12(%ecx,%eax), %eax
+; CHECK-NEXT:    movl 12(%eax,%ecx), %eax
 ; CHECK-NEXT:    retl
 ;
 ; CHECK64-LABEL: load_i32_by_i8_zaext_loads:
@@ -1072,7 +1072,7 @@ define i32 @load_i32_by_i8_zsext_loads(i8* %arg, i32 %arg1) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; CHECK-NEXT:    movl 12(%ecx,%eax), %eax
+; CHECK-NEXT:    movl 12(%eax,%ecx), %eax
 ; CHECK-NEXT:    retl
 ;
 ; CHECK64-LABEL: load_i32_by_i8_zsext_loads:
@@ -1119,18 +1119,12 @@ define i32 @zext_load_i32_by_i8(i32* %arg) {
 ; CHECK-LABEL: zext_load_i32_by_i8:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; CHECK-NEXT:    movzbl (%eax), %ecx
-; CHECK-NEXT:    movzbl 1(%eax), %eax
-; CHECK-NEXT:    shll $8, %eax
-; CHECK-NEXT:    orl %ecx, %eax
+; CHECK-NEXT:    movzwl (%eax), %eax
 ; CHECK-NEXT:    retl
 ;
 ; CHECK64-LABEL: zext_load_i32_by_i8:
 ; CHECK64:       # %bb.0:
-; CHECK64-NEXT:    movzbl (%rdi), %ecx
-; CHECK64-NEXT:    movzbl 1(%rdi), %eax
-; CHECK64-NEXT:    shll $8, %eax
-; CHECK64-NEXT:    orl %ecx, %eax
+; CHECK64-NEXT:    movzwl (%rdi), %eax
 ; CHECK64-NEXT:    retq
   %tmp = bitcast i32* %arg to i8*
   %tmp1 = getelementptr inbounds i8, i8* %tmp, i32 0
@@ -1218,18 +1212,16 @@ define i32 @zext_load_i32_by_i8_bswap(i32* %arg) {
 ; CHECK-LABEL: zext_load_i32_by_i8_bswap:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; CHECK-NEXT:    movzbl 1(%eax), %ecx
-; CHECK-NEXT:    movzbl (%eax), %eax
-; CHECK-NEXT:    shll $8, %eax
-; CHECK-NEXT:    orl %ecx, %eax
+; CHECK-NEXT:    movzwl (%eax), %eax
+; CHECK-NEXT:    shll $16, %eax
+; CHECK-NEXT:    bswapl %eax
 ; CHECK-NEXT:    retl
 ;
 ; CHECK64-LABEL: zext_load_i32_by_i8_bswap:
 ; CHECK64:       # %bb.0:
-; CHECK64-NEXT:    movzbl 1(%rdi), %ecx
-; CHECK64-NEXT:    movzbl (%rdi), %eax
-; CHECK64-NEXT:    shll $8, %eax
-; CHECK64-NEXT:    orl %ecx, %eax
+; CHECK64-NEXT:    movzwl (%rdi), %eax
+; CHECK64-NEXT:    shll $16, %eax
+; CHECK64-NEXT:    bswapl %eax
 ; CHECK64-NEXT:    retq
   %tmp = bitcast i32* %arg to i8*
   %tmp1 = getelementptr inbounds i8, i8* %tmp, i32 1
diff --git a/llvm/test/CodeGen/X86/load-local-v3i1.ll b/llvm/test/CodeGen/X86/load-local-v3i1.ll
index 88b87c273..6be3cc1b7 100644
--- a/llvm/test/CodeGen/X86/load-local-v3i1.ll
+++ b/llvm/test/CodeGen/X86/load-local-v3i1.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.2 | FileCheck %s
 
 ; widen a v3i1 to v4i1 to do a vector load/store. We would previously
@@ -12,6 +13,37 @@ declare <3 x i32> @llvm.masked.load.v3i32.p1v3i32(<3 x i32> addrspace(1)*, i32,
 declare void @llvm.masked.store.v3i32.p1v3i32(<3 x i32>, <3 x i32> addrspace(1)*, i32, <3 x i1>)
 
 define  <3 x i32> @masked_load_v3(i32 addrspace(1)*, <3 x i1>) {
+; CHECK-LABEL: masked_load_v3:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    andb $1, %sil
+; CHECK-NEXT:    andb $1, %dl
+; CHECK-NEXT:    addb %dl, %dl
+; CHECK-NEXT:    orb %sil, %dl
+; CHECK-NEXT:    andb $1, %cl
+; CHECK-NEXT:    shlb $2, %cl
+; CHECK-NEXT:    orb %dl, %cl
+; CHECK-NEXT:    testb $1, %cl
+; CHECK-NEXT:    # implicit-def: $xmm0
+; CHECK-NEXT:    jne .LBB0_1
+; CHECK-NEXT:  # %bb.2: # %else
+; CHECK-NEXT:    testb $2, %cl
+; CHECK-NEXT:    jne .LBB0_3
+; CHECK-NEXT:  .LBB0_4: # %else2
+; CHECK-NEXT:    testb $4, %cl
+; CHECK-NEXT:    jne .LBB0_5
+; CHECK-NEXT:  .LBB0_6: # %else5
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  .LBB0_1: # %cond.load
+; CHECK-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; CHECK-NEXT:    testb $2, %cl
+; CHECK-NEXT:    je .LBB0_4
+; CHECK-NEXT:  .LBB0_3: # %cond.load1
+; CHECK-NEXT:    pinsrd $1, 4(%rdi), %xmm0
+; CHECK-NEXT:    testb $4, %cl
+; CHECK-NEXT:    je .LBB0_6
+; CHECK-NEXT:  .LBB0_5: # %cond.load4
+; CHECK-NEXT:    pinsrd $2, 8(%rdi), %xmm0
+; CHECK-NEXT:    retq
 entry:
   %2 = bitcast i32 addrspace(1)* %0 to <3 x i32> addrspace(1)*
   %3 = call <3 x i32> @llvm.masked.load.v3i32.p1v3i32(<3 x i32> addrspace(1)* %2, i32 4, <3 x i1> %1, <3 x i32> undef)
@@ -19,6 +51,36 @@ entry:
 }
 
 define void @masked_store4_v3(<3 x i32>, i32 addrspace(1)*, <3 x i1>) {
+; CHECK-LABEL: masked_store4_v3:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    andb $1, %sil
+; CHECK-NEXT:    andb $1, %dl
+; CHECK-NEXT:    addb %dl, %dl
+; CHECK-NEXT:    orb %sil, %dl
+; CHECK-NEXT:    andb $1, %cl
+; CHECK-NEXT:    shlb $2, %cl
+; CHECK-NEXT:    orb %dl, %cl
+; CHECK-NEXT:    testb $1, %cl
+; CHECK-NEXT:    jne .LBB1_1
+; CHECK-NEXT:  # %bb.2: # %else
+; CHECK-NEXT:    testb $2, %cl
+; CHECK-NEXT:    jne .LBB1_3
+; CHECK-NEXT:  .LBB1_4: # %else2
+; CHECK-NEXT:    testb $4, %cl
+; CHECK-NEXT:    jne .LBB1_5
+; CHECK-NEXT:  .LBB1_6: # %else4
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  .LBB1_1: # %cond.store
+; CHECK-NEXT:    movss %xmm0, (%rdi)
+; CHECK-NEXT:    testb $2, %cl
+; CHECK-NEXT:    je .LBB1_4
+; CHECK-NEXT:  .LBB1_3: # %cond.store1
+; CHECK-NEXT:    extractps $1, %xmm0, 4(%rdi)
+; CHECK-NEXT:    testb $4, %cl
+; CHECK-NEXT:    je .LBB1_6
+; CHECK-NEXT:  .LBB1_5: # %cond.store3
+; CHECK-NEXT:    extractps $2, %xmm0, 8(%rdi)
+; CHECK-NEXT:    retq
 entry:
   %3 = bitcast i32 addrspace(1)* %1 to <3 x i32> addrspace(1)*
   call void @llvm.masked.store.v3i32.p1v3i32(<3 x i32> %0, <3 x i32> addrspace(1)* %3, i32 4, <3 x i1> %2)
@@ -28,41 +90,41 @@ entry:
 define void @local_load_v3i1(i32 addrspace(1)* %out, i32 addrspace(1)* %in, <3 x i1>* %predicate_ptr) nounwind {
 ; CHECK-LABEL: local_load_v3i1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT: pushq   %rbp
-; CHECK-NEXT: pushq   %r15
-; CHECK-NEXT: pushq   %r14
-; CHECK-NEXT: pushq   %rbx
-; CHECK-NEXT: pushq   %rax
-; CHECK-NEXT: movq    %rdi, %r14
-; CHECK-NEXT: movzbl  (%rdx), %ebp
-; CHECK-NEXT: movl    %ebp, %eax
-; CHECK-NEXT: shrl    %eax
-; CHECK-NEXT: andl    $1, %eax
-; CHECK-NEXT: movl    %ebp, %ecx
-; CHECK-NEXT: andl    $1, %ecx
-; CHECK-NEXT: movd    %ecx, %xmm0
-; CHECK-NEXT: pinsrd  $1, %eax, %xmm0
-; CHECK-NEXT: shrl    $2, %ebp
-; CHECK-NEXT: andl    $1, %ebp
-; CHECK-NEXT: pinsrd  $2, %ebp, %xmm0
-; CHECK-NEXT: movd    %xmm0, %ebx
-; CHECK-NEXT: pextrd  $1, %xmm0, %r15d
-; CHECK-NEXT: movq    %rsi, %rdi
-; CHECK-NEXT: movl    %ebx, %esi
-; CHECK-NEXT: movl    %r15d, %edx
-; CHECK-NEXT: movl    %ebp, %ecx
-; CHECK-NEXT: callq   masked_load_v3
-; CHECK-NEXT: movq    %r14, %rdi
-; CHECK-NEXT: movl    %ebx, %esi
-; CHECK-NEXT: movl    %r15d, %edx
-; CHECK-NEXT: movl    %ebp, %ecx
-; CHECK-NEXT: callq   masked_store4_v3
-; CHECK-NEXT: addq    $8, %rsp
-; CHECK-NEXT: popq    %rbx
-; CHECK-NEXT: popq    %r14
-; CHECK-NEXT: popq    %r15
-; CHECK-NEXT: popq    %rbp
-; CHECK-NEXT: retq
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    movq %rdi, %r14
+; CHECK-NEXT:    movzbl (%rdx), %ebp
+; CHECK-NEXT:    movl %ebp, %eax
+; CHECK-NEXT:    shrl %eax
+; CHECK-NEXT:    andl $1, %eax
+; CHECK-NEXT:    movl %ebp, %ecx
+; CHECK-NEXT:    andl $1, %ecx
+; CHECK-NEXT:    movd %ecx, %xmm0
+; CHECK-NEXT:    pinsrd $1, %eax, %xmm0
+; CHECK-NEXT:    shrl $2, %ebp
+; CHECK-NEXT:    andl $1, %ebp
+; CHECK-NEXT:    pinsrd $2, %ebp, %xmm0
+; CHECK-NEXT:    movd %xmm0, %ebx
+; CHECK-NEXT:    pextrd $1, %xmm0, %r15d
+; CHECK-NEXT:    movq %rsi, %rdi
+; CHECK-NEXT:    movl %ebx, %esi
+; CHECK-NEXT:    movl %r15d, %edx
+; CHECK-NEXT:    movl %ebp, %ecx
+; CHECK-NEXT:    callq masked_load_v3
+; CHECK-NEXT:    movq %r14, %rdi
+; CHECK-NEXT:    movl %ebx, %esi
+; CHECK-NEXT:    movl %r15d, %edx
+; CHECK-NEXT:    movl %ebp, %ecx
+; CHECK-NEXT:    callq masked_store4_v3
+; CHECK-NEXT:    addq $8, %rsp
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    retq
   %predicate = load <3 x i1>, <3 x i1>* %predicate_ptr
   %load1 = call <3 x i32> @masked_load_v3(i32 addrspace(1)* %in, <3 x i1> %predicate)
   call void @masked_store4_v3(<3 x i32> %load1, i32 addrspace(1)* %out, <3 x i1> %predicate)
diff --git a/llvm/test/CodeGen/X86/load-partial.ll b/llvm/test/CodeGen/X86/load-partial.ll
index a0a94b96f..4e1014fa2 100644
--- a/llvm/test/CodeGen/X86/load-partial.ll
+++ b/llvm/test/CodeGen/X86/load-partial.ll
@@ -31,6 +31,33 @@ define <4 x float> @load_float4_float3(<4 x float>* nocapture readonly dereferen
   ret <4 x float> %r2
 }
 
+define <4 x float> @load_float4_float3_0122(<4 x float>* nocapture readonly dereferenceable(16)) {
+; SSE-LABEL: load_float4_float3_0122:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE-NEXT:    movups (%rdi), %xmm0
+; SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm1[0,0]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: load_float4_float3_0122:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-NEXT:    vmovups (%rdi), %xmm1
+; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0,0]
+; AVX-NEXT:    retq
+  %p0 = getelementptr inbounds <4 x float>, <4 x float>* %0, i64 0, i64 0
+  %p1 = getelementptr inbounds <4 x float>, <4 x float>* %0, i64 0, i64 1
+  %p2 = getelementptr inbounds <4 x float>, <4 x float>* %0, i64 0, i64 2
+  %ld0 = load float, float* %p0, align 4
+  %ld1 = load float, float* %p1, align 4
+  %ld2 = load float, float* %p2, align 4
+  %r0 = insertelement <4 x float> undef, float %ld0, i32 0
+  %r1 = insertelement <4 x float> %r0,   float %ld1, i32 1
+  %r2 = insertelement <4 x float> %r1,   float %ld2, i32 2
+  %r3 = insertelement <4 x float> %r2,   float %ld2, i32 3
+  ret <4 x float> %r3
+}
+
 define <8 x float> @load_float8_float3(<4 x float>* nocapture readonly dereferenceable(16)) {
 ; SSE-LABEL: load_float8_float3:
 ; SSE:       # %bb.0:
@@ -53,33 +80,42 @@ define <8 x float> @load_float8_float3(<4 x float>* nocapture readonly dereferen
   ret <8 x float> %r2
 }
 
-define <4 x float> @load_float4_float3_as_float2_float(<4 x float>* nocapture readonly dereferenceable(16)) {
-; SSE2-LABEL: load_float4_float3_as_float2_float:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; SSE2-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,0],xmm0[3,0]
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm1[0,2]
-; SSE2-NEXT:    retq
-;
-; SSSE3-LABEL: load_float4_float3_as_float2_float:
-; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; SSSE3-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; SSSE3-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,0],xmm0[3,0]
-; SSSE3-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm1[0,2]
-; SSSE3-NEXT:    retq
+define <8 x float> @load_float8_float3_0122(<4 x float>* nocapture readonly dereferenceable(16)) {
+; SSE-LABEL: load_float8_float3_0122:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE-NEXT:    movups (%rdi), %xmm0
+; SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm1[0,0]
+; SSE-NEXT:    retq
 ;
-; SSE41-LABEL: load_float4_float3_as_float2_float:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; SSE41-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0,1],mem[0],xmm0[3]
-; SSE41-NEXT:    retq
+; AVX-LABEL: load_float8_float3_0122:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-NEXT:    vmovups (%rdi), %xmm1
+; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0,0]
+; AVX-NEXT:    retq
+  %p0 = getelementptr inbounds <4 x float>, <4 x float>* %0, i64 0, i64 0
+  %p1 = getelementptr inbounds <4 x float>, <4 x float>* %0, i64 0, i64 1
+  %p2 = getelementptr inbounds <4 x float>, <4 x float>* %0, i64 0, i64 2
+  %ld0 = load float, float* %p0, align 4
+  %ld1 = load float, float* %p1, align 4
+  %ld2 = load float, float* %p2, align 4
+  %r0 = insertelement <8 x float> undef, float %ld0, i32 0
+  %r1 = insertelement <8 x float> %r0,   float %ld1, i32 1
+  %r2 = insertelement <8 x float> %r1,   float %ld2, i32 2
+  %r3 = insertelement <8 x float> %r2,   float %ld2, i32 3
+  ret <8 x float> %r3
+}
+
+define <4 x float> @load_float4_float3_as_float2_float(<4 x float>* nocapture readonly dereferenceable(16)) {
+; SSE-LABEL: load_float4_float3_as_float2_float:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movups (%rdi), %xmm0
+; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: load_float4_float3_as_float2_float:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1],mem[0],xmm0[3]
+; AVX-NEXT:    vmovups (%rdi), %xmm0
 ; AVX-NEXT:    retq
   %2 = bitcast <4 x float>* %0 to <2 x float>*
   %3 = load <2 x float>, <2 x float>* %2, align 4
@@ -93,37 +129,119 @@ define <4 x float> @load_float4_float3_as_float2_float(<4 x float>* nocapture re
   ret <4 x float> %10
 }
 
+define <4 x float> @load_float4_float3_as_float2_float_0122(<4 x float>* nocapture readonly dereferenceable(16)) {
+; SSE-LABEL: load_float4_float3_as_float2_float_0122:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm1[0,0]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: load_float4_float3_as_float2_float_0122:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,1],xmm1[0,0]
+; AVX-NEXT:    retq
+  %2 = bitcast <4 x float>* %0 to <2 x float>*
+  %3 = load <2 x float>, <2 x float>* %2, align 4
+  %4 = extractelement <2 x float> %3, i32 0
+  %5 = insertelement <4 x float> undef, float %4, i32 0
+  %6 = extractelement <2 x float> %3, i32 1
+  %7 = insertelement <4 x float> %5, float %6, i32 1
+  %8 = getelementptr inbounds <4 x float>, <4 x float>* %0, i64 0, i64 2
+  %9 = load float, float* %8, align 4
+  %10 = insertelement <4 x float> %7, float %9, i32 2
+  %11 = insertelement <4 x float> %10, float %9, i32 3
+  ret <4 x float> %11
+}
+
 define <4 x float> @load_float4_float3_trunc(<4 x float>* nocapture readonly dereferenceable(16)) {
-; SSE2-LABEL: load_float4_float3_trunc:
+; SSE-LABEL: load_float4_float3_trunc:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movaps (%rdi), %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: load_float4_float3_trunc:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovaps (%rdi), %xmm0
+; AVX-NEXT:    retq
+  %2 = bitcast <4 x float>* %0 to i64*
+  %3 = load i64, i64* %2, align 16
+  %4 = getelementptr inbounds <4 x float>, <4 x float>* %0, i64 0, i64 2
+  %5 = bitcast float* %4 to i64*
+  %6 = load i64, i64* %5, align 8
+  %7 = trunc i64 %3 to i32
+  %8 = bitcast i32 %7 to float
+  %9 = insertelement <4 x float> undef, float %8, i32 0
+  %10 = lshr i64 %3, 32
+  %11 = trunc i64 %10 to i32
+  %12 = bitcast i32 %11 to float
+  %13 = insertelement <4 x float> %9, float %12, i32 1
+  %14 = trunc i64 %6 to i32
+  %15 = bitcast i32 %14 to float
+  %16 = insertelement <4 x float> %13, float %15, i32 2
+  ret <4 x float> %16
+}
+
+define <4 x float> @load_float4_float3_trunc_0122(<4 x float>* nocapture readonly dereferenceable(16)) {
+; SSE-LABEL: load_float4_float3_trunc_0122:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE-NEXT:    movaps (%rdi), %xmm0
+; SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm1[0,0]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: load_float4_float3_trunc_0122:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-NEXT:    vmovaps (%rdi), %xmm1
+; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0,0]
+; AVX-NEXT:    retq
+  %2 = bitcast <4 x float>* %0 to i64*
+  %3 = load i64, i64* %2, align 16
+  %4 = getelementptr inbounds <4 x float>, <4 x float>* %0, i64 0, i64 2
+  %5 = bitcast float* %4 to i64*
+  %6 = load i64, i64* %5, align 8
+  %7 = trunc i64 %3 to i32
+  %8 = bitcast i32 %7 to float
+  %9 = insertelement <4 x float> undef, float %8, i32 0
+  %10 = lshr i64 %3, 32
+  %11 = trunc i64 %10 to i32
+  %12 = bitcast i32 %11 to float
+  %13 = insertelement <4 x float> %9, float %12, i32 1
+  %14 = trunc i64 %6 to i32
+  %15 = bitcast i32 %14 to float
+  %16 = insertelement <4 x float> %13, float %15, i32 2
+  %17 = insertelement <4 x float> %16, float %15, i32 3
+  ret <4 x float> %17
+}
+
+define <4 x float> @load_float4_float3_trunc_0123(<4 x float>* nocapture readonly dereferenceable(16)) {
+; SSE2-LABEL: load_float4_float3_trunc_0123:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE2-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; SSE2-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; SSE2-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; SSE2-NEXT:    movlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE2-NEXT:    movaps (%rdi), %xmm0
+; SSE2-NEXT:    movhps {{.*#+}} xmm0 = xmm0[0,1],mem[0,1]
 ; SSE2-NEXT:    retq
 ;
-; SSSE3-LABEL: load_float4_float3_trunc:
+; SSSE3-LABEL: load_float4_float3_trunc_0123:
 ; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSSE3-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; SSSE3-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; SSSE3-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; SSSE3-NEXT:    movlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSSE3-NEXT:    movaps (%rdi), %xmm0
+; SSSE3-NEXT:    movhps {{.*#+}} xmm0 = xmm0[0,1],mem[0,1]
 ; SSSE3-NEXT:    retq
 ;
-; SSE41-LABEL: load_float4_float3_trunc:
+; SSE41-LABEL: load_float4_float3_trunc_0123:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE41-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[2,3]
+; SSE41-NEXT:    movaps (%rdi), %xmm0
 ; SSE41-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0,1],mem[0],xmm0[3]
+; SSE41-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0,1,2],mem[0]
 ; SSE41-NEXT:    retq
 ;
-; AVX-LABEL: load_float4_float3_trunc:
+; AVX-LABEL: load_float4_float3_trunc_0123:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[2,3]
+; AVX-NEXT:    vmovaps (%rdi), %xmm0
 ; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1],mem[0],xmm0[3]
+; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],mem[0]
 ; AVX-NEXT:    retq
   %2 = bitcast <4 x float>* %0 to i64*
   %3 = load i64, i64* %2, align 16
@@ -140,7 +258,11 @@ define <4 x float> @load_float4_float3_trunc(<4 x float>* nocapture readonly der
   %14 = trunc i64 %6 to i32
   %15 = bitcast i32 %14 to float
   %16 = insertelement <4 x float> %13, float %15, i32 2
-  ret <4 x float> %16
+  %17 = lshr i64 %6, 32
+  %18 = trunc i64 %17 to i32
+  %19 = bitcast i32 %18 to float
+  %20 = insertelement <4 x float> %16, float %19, i32 3
+  ret <4 x float> %20
 }
 
 ; PR21780
@@ -177,3 +299,111 @@ define <4 x double> @load_double4_0u2u(double* nocapture readonly dereferenceabl
   %7 = shufflevector <4 x double> %6, <4 x double> undef, <4 x i32> <i32 0, i32 0, i32 2, i32 2>
   ret <4 x double> %7
 }
+
+; Test case identified in rL366501
+@h = local_unnamed_addr global i8 0, align 1
+define i32 @load_partial_illegal_type() {
+; SSE2-LABEL: load_partial_illegal_type:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movzwl {{.*}}(%rip), %eax
+; SSE2-NEXT:    movd %eax, %xmm0
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    por {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: load_partial_illegal_type:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movzwl {{.*}}(%rip), %eax
+; SSSE3-NEXT:    movd %eax, %xmm0
+; SSSE3-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,1],zero,xmm0[3,4,5,6,7,8,9,10,11,12,13,14,15]
+; SSSE3-NEXT:    por {{.*}}(%rip), %xmm0
+; SSSE3-NEXT:    movd %xmm0, %eax
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: load_partial_illegal_type:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movzwl {{.*}}(%rip), %eax
+; SSE41-NEXT:    movd %eax, %xmm0
+; SSE41-NEXT:    movl $2, %eax
+; SSE41-NEXT:    pinsrb $2, %eax, %xmm0
+; SSE41-NEXT:    movd %xmm0, %eax
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: load_partial_illegal_type:
+; AVX:       # %bb.0:
+; AVX-NEXT:    movzwl {{.*}}(%rip), %eax
+; AVX-NEXT:    vmovd %eax, %xmm0
+; AVX-NEXT:    movl $2, %eax
+; AVX-NEXT:    vpinsrb $2, %eax, %xmm0, %xmm0
+; AVX-NEXT:    vmovd %xmm0, %eax
+; AVX-NEXT:    retq
+  %1 = load <2 x i8>, <2 x i8>* bitcast (i8* @h to <2 x i8>*), align 1
+  %2 = shufflevector <2 x i8> %1, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+  %3 = insertelement <4 x i8> %2, i8 2, i32 2
+  %4 = bitcast <4 x i8> %3 to i32
+  ret i32 %4
+}
+
+define void @PR43227(i32* %explicit_0, <8 x i32>* %explicit_1) {
+; SSE2-LABEL: PR43227:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE2-NEXT:    xorps %xmm1, %xmm1
+; SSE2-NEXT:    xorps %xmm2, %xmm2
+; SSE2-NEXT:    movss {{.*#+}} xmm2 = xmm0[0],xmm2[1,2,3]
+; SSE2-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT:    movlhps {{.*#+}} xmm2 = xmm2[0],xmm0[0]
+; SSE2-NEXT:    movaps %xmm1, 672(%rsi)
+; SSE2-NEXT:    movaps %xmm2, 688(%rsi)
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: PR43227:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSSE3-NEXT:    xorps %xmm1, %xmm1
+; SSSE3-NEXT:    xorps %xmm2, %xmm2
+; SSSE3-NEXT:    movss {{.*#+}} xmm2 = xmm0[0],xmm2[1,2,3]
+; SSSE3-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSSE3-NEXT:    movlhps {{.*#+}} xmm2 = xmm2[0],xmm0[0]
+; SSSE3-NEXT:    movaps %xmm1, 672(%rsi)
+; SSSE3-NEXT:    movaps %xmm2, 688(%rsi)
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: PR43227:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE41-NEXT:    pxor %xmm1, %xmm1
+; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7]
+; SSE41-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE41-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SSE41-NEXT:    movdqa %xmm1, 672(%rsi)
+; SSE41-NEXT:    movdqa %xmm0, 688(%rsi)
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: PR43227:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX-NEXT:    vmovaps %ymm0, 672(%rsi)
+; AVX-NEXT:    vzeroupper
+; AVX-NEXT:    retq
+  %1 = getelementptr i32, i32* %explicit_0, i64 63
+  %2 = bitcast i32* %1 to <3 x i32>*
+  %3 = load <3 x i32>, <3 x i32>* %2, align 1
+  %4 = shufflevector <3 x i32> %3, <3 x i32> undef, <2 x i32> <i32 1, i32 2>
+  %5 = shufflevector <2 x i32> %4, <2 x i32> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %6 = shufflevector <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 undef, i32 0, i32 undef, i32 0>, <8 x i32> %5, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 5, i32 9, i32 7>
+  %7 = getelementptr inbounds <8 x i32>, <8 x i32>* %explicit_1, i64 21
+  store <8 x i32> %6, <8 x i32>* %7, align 32
+  ret void
+}
diff --git a/llvm/test/CodeGen/X86/localescape.ll b/llvm/test/CodeGen/X86/localescape.ll
index 2fba9b286..4593845f6 100644
--- a/llvm/test/CodeGen/X86/localescape.ll
+++ b/llvm/test/CodeGen/X86/localescape.ll
@@ -77,7 +77,7 @@ define void @alloc_func(i32 %n) {
 ; X64: subq    $16, %rsp
 ; X64: .seh_stackalloc 16
 ; X64: leaq    16(%rsp), %rbp
-; X64: .seh_setframe 5, 16
+; X64: .seh_setframe %rbp, 16
 ; X64: .set .Lalloc_func$frame_escape_0, -4
 ; X64: .set .Lalloc_func$frame_escape_1, -12
 ; X64: movl $42, -4(%rbp)
diff --git a/llvm/test/CodeGen/X86/loop-blocks.ll b/llvm/test/CodeGen/X86/loop-blocks.ll
index a5e806d93..e970061c0 100644
--- a/llvm/test/CodeGen/X86/loop-blocks.ll
+++ b/llvm/test/CodeGen/X86/loop-blocks.ll
@@ -269,6 +269,35 @@ exit:
 
 attributes #0 = { minsize norecurse nounwind optsize readnone uwtable }
 
+; CHECK-LABEL: slightly_more_involved_2_pgso:
+; CHECK-NOT:      jmp .LBB6_1
+; CHECK:          .LBB6_1:
+; CHECK-NEXT:     callq body
+
+define void @slightly_more_involved_2_pgso() norecurse nounwind readnone uwtable !prof !14 {
+entry:
+  br label %loop
+
+loop:
+  call void @body()
+  %t0 = call i32 @get()
+  %t1 = icmp slt i32 %t0, 2
+  br i1 %t1, label %block_a, label %bb
+
+bb:
+  %t2 = call i32 @get()
+  %t3 = icmp slt i32 %t2, 99
+  br i1 %t3, label %exit, label %loop
+
+block_a:
+  call void @bar99()
+  br label %loop
+
+exit:
+  call void @exit()
+  ret void
+}
+
 declare void @bar99() nounwind
 declare void @bar100() nounwind
 declare void @bar101() nounwind
@@ -281,3 +310,20 @@ declare i32 @get() nounwind
 declare void @block_a_true_func() nounwind
 declare void @block_a_false_func() nounwind
 declare void @block_a_merge_func() nounwind
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 1, !"ProfileSummary", !1}
+!1 = !{!2, !3, !4, !5, !6, !7, !8, !9}
+!2 = !{!"ProfileFormat", !"InstrProf"}
+!3 = !{!"TotalCount", i64 10000}
+!4 = !{!"MaxCount", i64 10}
+!5 = !{!"MaxInternalCount", i64 1}
+!6 = !{!"MaxFunctionCount", i64 1000}
+!7 = !{!"NumCounts", i64 3}
+!8 = !{!"NumFunctions", i64 3}
+!9 = !{!"DetailedSummary", !10}
+!10 = !{!11, !12, !13}
+!11 = !{i32 10000, i64 100, i32 1}
+!12 = !{i32 999000, i64 100, i32 1}
+!13 = !{i32 999999, i64 1, i32 2}
+!14 = !{!"function_entry_count", i64 0}
diff --git a/llvm/test/CodeGen/X86/loop-strength-reduce5.ll b/llvm/test/CodeGen/X86/loop-strength-reduce5.ll
index 2e4a58387..502faac27 100644
--- a/llvm/test/CodeGen/X86/loop-strength-reduce5.ll
+++ b/llvm/test/CodeGen/X86/loop-strength-reduce5.ll
@@ -1,9 +1,29 @@
-; RUN: llc < %s -mtriple=i686-- | grep inc | count 1
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- | FileCheck %s
 
 @X = weak global i16 0		; <i16*> [#uses=1]
 @Y = weak global i16 0		; <i16*> [#uses=1]
 
 define void @foo(i32 %N) nounwind {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    testl %eax, %eax
+; CHECK-NEXT:    jle .LBB0_3
+; CHECK-NEXT:  # %bb.1: # %bb.preheader
+; CHECK-NEXT:    xorl %ecx, %ecx
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB0_2: # %bb
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movw %dx, X
+; CHECK-NEXT:    movw %cx, Y
+; CHECK-NEXT:    incl %edx
+; CHECK-NEXT:    addl $4, %ecx
+; CHECK-NEXT:    cmpl %edx, %eax
+; CHECK-NEXT:    jne .LBB0_2
+; CHECK-NEXT:  .LBB0_3: # %return
+; CHECK-NEXT:    retl
 entry:
 	%tmp1019 = icmp sgt i32 %N, 0		; <i1> [#uses=1]
 	br i1 %tmp1019, label %bb, label %return
diff --git a/llvm/test/CodeGen/X86/loop-strength-reduce6.ll b/llvm/test/CodeGen/X86/loop-strength-reduce6.ll
index 326a73949..17636121e 100644
--- a/llvm/test/CodeGen/X86/loop-strength-reduce6.ll
+++ b/llvm/test/CodeGen/X86/loop-strength-reduce6.ll
@@ -1,6 +1,11 @@
-; RUN: llc < %s -mtriple=x86_64-- | not grep inc
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-- | FileCheck %s
 
 define fastcc i32 @decodeMP3(i32 %isize, i32* %done) nounwind {
+; CHECK-LABEL: decodeMP3:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    retq
 entry:
 	br label %cond_true189
 
diff --git a/llvm/test/CodeGen/X86/loop-strength-reduce7.ll b/llvm/test/CodeGen/X86/loop-strength-reduce7.ll
index 7a467d311..47b114b4d 100644
--- a/llvm/test/CodeGen/X86/loop-strength-reduce7.ll
+++ b/llvm/test/CodeGen/X86/loop-strength-reduce7.ll
@@ -1,4 +1,5 @@
-; RUN: llc < %s | not grep imul
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s | FileCheck %s
 
 target triple = "i386-apple-darwin9.6"
 	%struct.III_psy_xmin = type { [22 x double], [13 x [3 x double]] }
@@ -7,6 +8,30 @@ target triple = "i386-apple-darwin9.6"
 	%struct.lame_global_flags = type { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i8*, i8*, i32, i32, float, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, float, i32, i32, i32, float, float, float, float, i32, i32, i32, i32, i32, i32, i32, i32 }
 
 define fastcc void @outer_loop(%struct.lame_global_flags* nocapture %gfp, double* nocapture %xr, i32 %targ_bits, double* nocapture %best_noise, %struct.III_psy_xmin* nocapture %l3_xmin, i32* nocapture %l3_enc, %struct.III_scalefac_t* nocapture %scalefac, %struct.gr_info* nocapture %cod_info, i32 %ch) nounwind {
+; CHECK-LABEL: outer_loop:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    pushl %esi
+; CHECK-NEXT:    movl $88, %eax
+; CHECK-NEXT:    movl $168, %ecx
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  LBB0_2: ## %bb28.i37
+; CHECK-NEXT:    ## =>This Loop Header: Depth=1
+; CHECK-NEXT:    ## Child Loop BB0_3 Depth 2
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    movl %eax, %esi
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  LBB0_3: ## %bb29.i38
+; CHECK-NEXT:    ## Parent Loop BB0_2 Depth=1
+; CHECK-NEXT:    ## => This Inner Loop Header: Depth=2
+; CHECK-NEXT:    incl %edx
+; CHECK-NEXT:    addl $12, %esi
+; CHECK-NEXT:    cmpl $11, %edx
+; CHECK-NEXT:    jbe LBB0_3
+; CHECK-NEXT:  ## %bb.1: ## %bb28.i37.loopexit
+; CHECK-NEXT:    ## in Loop: Header=BB0_2 Depth=1
+; CHECK-NEXT:    addl $4, %eax
+; CHECK-NEXT:    addl $168, %ecx
+; CHECK-NEXT:    jmp LBB0_2
 entry:
 	br label %bb4
 
diff --git a/llvm/test/CodeGen/X86/lower-bitcast.ll b/llvm/test/CodeGen/X86/lower-bitcast.ll
index 11271f155..d41d2ea83 100644
--- a/llvm/test/CodeGen/X86/lower-bitcast.ll
+++ b/llvm/test/CodeGen/X86/lower-bitcast.ll
@@ -1,6 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-unknown -mcpu=core2 -mattr=+sse2 | FileCheck %s
-; RUN: llc < %s -mtriple=x86_64-unknown -mcpu=core2 -mattr=+sse2 -x86-experimental-vector-widening-legalization | FileCheck %s --check-prefix=CHECK-WIDE
 
 ; FIXME: Ideally we should be able to fold the entire body of @test1 into a
 ; single paddd instruction. At the moment we produce the sequence
@@ -9,15 +8,8 @@
 define double @test1(double %A) {
 ; CHECK-LABEL: test1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1,1,3]
 ; CHECK-NEXT:    paddd {{.*}}(%rip), %xmm0
-; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; CHECK-NEXT:    retq
-;
-; CHECK-WIDE-LABEL: test1:
-; CHECK-WIDE:       # %bb.0:
-; CHECK-WIDE-NEXT:    paddd {{.*}}(%rip), %xmm0
-; CHECK-WIDE-NEXT:    retq
   %1 = bitcast double %A to <2 x i32>
   %add = add <2 x i32> %1, <i32 3, i32 5>
   %2 = bitcast <2 x i32> %add to double
@@ -29,11 +21,6 @@ define double @test2(double %A, double %B) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    paddd %xmm1, %xmm0
 ; CHECK-NEXT:    retq
-;
-; CHECK-WIDE-LABEL: test2:
-; CHECK-WIDE:       # %bb.0:
-; CHECK-WIDE-NEXT:    paddd %xmm1, %xmm0
-; CHECK-WIDE-NEXT:    retq
   %1 = bitcast double %A to <2 x i32>
   %2 = bitcast double %B to <2 x i32>
   %add = add <2 x i32> %1, %2
@@ -48,13 +35,6 @@ define i64 @test3(i64 %A) {
 ; CHECK-NEXT:    addps {{.*}}(%rip), %xmm0
 ; CHECK-NEXT:    movq %xmm0, %rax
 ; CHECK-NEXT:    retq
-;
-; CHECK-WIDE-LABEL: test3:
-; CHECK-WIDE:       # %bb.0:
-; CHECK-WIDE-NEXT:    movq %rdi, %xmm0
-; CHECK-WIDE-NEXT:    addps {{.*}}(%rip), %xmm0
-; CHECK-WIDE-NEXT:    movq %xmm0, %rax
-; CHECK-WIDE-NEXT:    retq
   %1 = bitcast i64 %A to <2 x float>
   %add = fadd <2 x float> %1, <float 3.0, float 5.0>
   %2 = bitcast <2 x float> %add to i64
@@ -68,18 +48,9 @@ define i64 @test4(i64 %A) {
 ; CHECK-LABEL: test4:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    movq %rdi, %xmm0
-; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,1,3]
 ; CHECK-NEXT:    paddd {{.*}}(%rip), %xmm0
-; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; CHECK-NEXT:    movq %xmm0, %rax
 ; CHECK-NEXT:    retq
-;
-; CHECK-WIDE-LABEL: test4:
-; CHECK-WIDE:       # %bb.0:
-; CHECK-WIDE-NEXT:    movq %rdi, %xmm0
-; CHECK-WIDE-NEXT:    paddd {{.*}}(%rip), %xmm0
-; CHECK-WIDE-NEXT:    movq %xmm0, %rax
-; CHECK-WIDE-NEXT:    retq
   %1 = bitcast i64 %A to <2 x i32>
   %add = add <2 x i32> %1, <i32 3, i32 5>
   %2 = bitcast <2 x i32> %add to i64
@@ -91,11 +62,6 @@ define double @test5(double %A) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addps {{.*}}(%rip), %xmm0
 ; CHECK-NEXT:    retq
-;
-; CHECK-WIDE-LABEL: test5:
-; CHECK-WIDE:       # %bb.0:
-; CHECK-WIDE-NEXT:    addps {{.*}}(%rip), %xmm0
-; CHECK-WIDE-NEXT:    retq
   %1 = bitcast double %A to <2 x float>
   %add = fadd <2 x float> %1, <float 3.0, float 5.0>
   %2 = bitcast <2 x float> %add to double
@@ -108,15 +74,8 @@ define double @test5(double %A) {
 define double @test6(double %A) {
 ; CHECK-LABEL: test6:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
 ; CHECK-NEXT:    paddw {{.*}}(%rip), %xmm0
-; CHECK-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
 ; CHECK-NEXT:    retq
-;
-; CHECK-WIDE-LABEL: test6:
-; CHECK-WIDE:       # %bb.0:
-; CHECK-WIDE-NEXT:    paddw {{.*}}(%rip), %xmm0
-; CHECK-WIDE-NEXT:    retq
   %1 = bitcast double %A to <4 x i16>
   %add = add <4 x i16> %1, <i16 3, i16 4, i16 5, i16 6>
   %2 = bitcast <4 x i16> %add to double
@@ -128,11 +87,6 @@ define double @test7(double %A, double %B) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    paddw %xmm1, %xmm0
 ; CHECK-NEXT:    retq
-;
-; CHECK-WIDE-LABEL: test7:
-; CHECK-WIDE:       # %bb.0:
-; CHECK-WIDE-NEXT:    paddw %xmm1, %xmm0
-; CHECK-WIDE-NEXT:    retq
   %1 = bitcast double %A to <4 x i16>
   %2 = bitcast double %B to <4 x i16>
   %add = add <4 x i16> %1, %2
@@ -147,15 +101,8 @@ define double @test7(double %A, double %B) {
 define double @test8(double %A) {
 ; CHECK-LABEL: test8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; CHECK-NEXT:    paddb {{.*}}(%rip), %xmm0
-; CHECK-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
 ; CHECK-NEXT:    retq
-;
-; CHECK-WIDE-LABEL: test8:
-; CHECK-WIDE:       # %bb.0:
-; CHECK-WIDE-NEXT:    paddb {{.*}}(%rip), %xmm0
-; CHECK-WIDE-NEXT:    retq
   %1 = bitcast double %A to <8 x i8>
   %add = add <8 x i8> %1, <i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10>
   %2 = bitcast <8 x i8> %add to double
@@ -167,11 +114,6 @@ define double @test9(double %A, double %B) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    paddb %xmm1, %xmm0
 ; CHECK-NEXT:    retq
-;
-; CHECK-WIDE-LABEL: test9:
-; CHECK-WIDE:       # %bb.0:
-; CHECK-WIDE-NEXT:    paddb %xmm1, %xmm0
-; CHECK-WIDE-NEXT:    retq
   %1 = bitcast double %A to <8 x i8>
   %2 = bitcast double %B to <8 x i8>
   %add = add <8 x i8> %1, %2
diff --git a/llvm/test/CodeGen/X86/lower-ptrmask.ll b/llvm/test/CodeGen/X86/lower-ptrmask.ll
new file mode 100644
index 000000000..367353e7e
--- /dev/null
+++ b/llvm/test/CodeGen/X86/lower-ptrmask.ll
@@ -0,0 +1,31 @@
+; RUN: llc -mtriple=x86_64-apple-macosx -stop-after=finalize-isel %s -o - | FileCheck %s
+
+declare i8* @llvm.ptrmask.p0i8.i64(i8* , i64)
+
+; CHECK-LABEL: name: test1
+; CHECK:         %0:gr64 = COPY $rdi
+; CHECK-NEXT:    %1:gr64 = MOV64ri 72057594037927928
+; CHECK-NEXT:    %2:gr64 = AND64rr %0, killed %1, implicit-def dead $eflags
+; CHECK-NEXT:    $rax = COPY %2
+; CHECK-NEXT:    RET 0, $rax
+
+define i8* @test1(i8* %src) {
+  %ptr = call i8* @llvm.ptrmask.p0i8.i64(i8* %src, i64 72057594037927928)
+  ret i8* %ptr
+}
+
+declare i8* @llvm.ptrmask.p0i8.i32(i8*, i32)
+
+; CHECK-LABEL: name: test2
+; CHECK:         %0:gr64 = COPY $rdi
+; CHECK-NEXT:    %1:gr32 = COPY %0.sub_32bit
+; CHECK-NEXT:    %2:gr32 = AND32ri %1, 10000, implicit-def dead $eflags
+; CHECK-NEXT:    %3:gr64 = SUBREG_TO_REG 0, killed %2, %subreg.sub_32bit
+; CHECK-NEXT:    $rax = COPY %3
+; CHECK-NEXT:    RET 0, $rax
+
+
+define i8* @test2(i8* %src) {
+  %ptr = call i8* @llvm.ptrmask.p0i8.i32(i8* %src, i32 10000)
+  ret i8* %ptr
+}
diff --git a/llvm/test/CodeGen/X86/lrshrink.ll b/llvm/test/CodeGen/X86/lrshrink.ll
index a9cf086db..f1997eb70 100644
--- a/llvm/test/CodeGen/X86/lrshrink.ll
+++ b/llvm/test/CodeGen/X86/lrshrink.ll
@@ -5,6 +5,46 @@
 ; to minimize live-range.
 
 define i64 @test(i1 %a, i64 %r1, i64 %r2, i64 %s1, i64 %s2, i64 %t1, i64 %t2) {
+; CHECK-LABEL: test:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    .cfi_offset %rbx, -32
+; CHECK-NEXT:    .cfi_offset %r14, -24
+; CHECK-NEXT:    .cfi_offset %r15, -16
+; CHECK-NEXT:    movq %rcx, %r14
+; CHECK-NEXT:    movl $4, %r15d
+; CHECK-NEXT:    testb $1, %dil
+; CHECK-NEXT:    je .LBB0_2
+; CHECK-NEXT:  # %bb.1: # %then
+; CHECK-NEXT:    movq {{[0-9]+}}(%rsp), %r9
+; CHECK-NEXT:    movl $10, %r15d
+; CHECK-NEXT:    movq %rdx, %rsi
+; CHECK-NEXT:    movq %r8, %r14
+; CHECK-NEXT:  .LBB0_2: # %else
+; CHECK-NEXT:    addq %r9, %r14
+; CHECK-NEXT:    addq %rsi, %r15
+; CHECK-NEXT:    callq _Z3foov
+; CHECK-NEXT:    movl %eax, %ebx
+; CHECK-NEXT:    addq %r15, %rbx
+; CHECK-NEXT:    callq _Z3foov
+; CHECK-NEXT:    movl %eax, %r15d
+; CHECK-NEXT:    addq %rbx, %r15
+; CHECK-NEXT:    callq _Z3foov
+; CHECK-NEXT:    movl %eax, %eax
+; CHECK-NEXT:    addq %r15, %rax
+; CHECK-NEXT:    addq %r14, %rax
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
 entry:
   br i1 %a, label %then, label %else
 
@@ -16,15 +56,6 @@ else:
   %r = phi i64 [ %r1, %entry ], [ %r2, %then ]
   %s = phi i64 [ %s1, %entry ], [ %s2, %then ]
   %t = phi i64 [ %t1, %entry ], [ %t2, %then ]
-; CHECK-LABEL: test:
-; CHECK: add
-; CHECK: add
-; CHECK: call
-; CHECK: add
-; CHECK: call
-; CHECK: add
-; CHECK: call
-; CHECK: add
   %1 = tail call i32 @_Z3foov()
   %2 = zext i32 %1 to i64
   %3 = tail call i32 @_Z3foov()
diff --git a/llvm/test/CodeGen/X86/lsr-negative-stride.ll b/llvm/test/CodeGen/X86/lsr-negative-stride.ll
index 19d3422f4..26c6128ab 100644
--- a/llvm/test/CodeGen/X86/lsr-negative-stride.ll
+++ b/llvm/test/CodeGen/X86/lsr-negative-stride.ll
@@ -1,8 +1,5 @@
-; RUN: llc < %s -mtriple=i686-- > %t
-; RUN: not grep neg %t
-; RUN: not grep sub.*esp %t
-; RUN: not grep esi %t
-; RUN: not grep push %t
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- | FileCheck %s
 
 ; This corresponds to:
 ;int t(int a, int b) {
@@ -17,6 +14,41 @@
 
 
 define i32 @t(i32 %a, i32 %b) nounwind {
+; CHECK-LABEL: t:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; CHECK-NEXT:    cmpl %ecx, %edx
+; CHECK-NEXT:    jne .LBB0_2
+; CHECK-NEXT:  # %bb.1:
+; CHECK-NEXT:    movl %edx, %eax
+; CHECK-NEXT:    retl
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB0_2: # %bb.outer
+; CHECK-NEXT:    # =>This Loop Header: Depth=1
+; CHECK-NEXT:    # Child Loop BB0_3 Depth 2
+; CHECK-NEXT:    movl %edx, %eax
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB0_3: # %bb
+; CHECK-NEXT:    # Parent Loop BB0_2 Depth=1
+; CHECK-NEXT:    # => This Inner Loop Header: Depth=2
+; CHECK-NEXT:    subl %ecx, %eax
+; CHECK-NEXT:    jle .LBB0_5
+; CHECK-NEXT:  # %bb.4: # %cond_true
+; CHECK-NEXT:    # in Loop: Header=BB0_3 Depth=2
+; CHECK-NEXT:    cmpl %eax, %ecx
+; CHECK-NEXT:    movl %eax, %edx
+; CHECK-NEXT:    jne .LBB0_3
+; CHECK-NEXT:    jmp .LBB0_6
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB0_5: # %cond_false
+; CHECK-NEXT:    # in Loop: Header=BB0_2 Depth=1
+; CHECK-NEXT:    subl %edx, %ecx
+; CHECK-NEXT:    cmpl %edx, %ecx
+; CHECK-NEXT:    movl %edx, %eax
+; CHECK-NEXT:    jne .LBB0_2
+; CHECK-NEXT:  .LBB0_6: # %bb17
+; CHECK-NEXT:    retl
 entry:
 	%tmp1434 = icmp eq i32 %a, %b		; <i1> [#uses=1]
 	br i1 %tmp1434, label %bb17, label %bb.outer
diff --git a/llvm/test/CodeGen/X86/lsr-sort.ll b/llvm/test/CodeGen/X86/lsr-sort.ll
index 5aca606c1..e831e9b20 100644
--- a/llvm/test/CodeGen/X86/lsr-sort.ll
+++ b/llvm/test/CodeGen/X86/lsr-sort.ll
@@ -1,10 +1,23 @@
-; RUN: llc < %s -mtriple=x86_64-- > %t
-; RUN: grep inc %t | count 1
-; RUN: not grep incw %t
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-- | FileCheck %s
 
 @X = common global i16 0		; <i16*> [#uses=1]
 
 define i32 @foo(i32 %N) nounwind {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    testl %edi, %edi
+; CHECK-NEXT:    jle .LBB0_2
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB0_1: # %bb
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movw %ax, {{.*}}(%rip)
+; CHECK-NEXT:    incl %eax
+; CHECK-NEXT:    cmpl %eax, %edi
+; CHECK-NEXT:    jne .LBB0_1
+; CHECK-NEXT:  .LBB0_2: # %return
+; CHECK-NEXT:    retq
 entry:
 	%0 = icmp sgt i32 %N, 0		; <i1> [#uses=1]
 	br i1 %0, label %bb, label %return
diff --git a/llvm/test/CodeGen/X86/lzcnt-zext-cmp.ll b/llvm/test/CodeGen/X86/lzcnt-zext-cmp.ll
index 8d43a1b73..980956bda 100644
--- a/llvm/test/CodeGen/X86/lzcnt-zext-cmp.ll
+++ b/llvm/test/CodeGen/X86/lzcnt-zext-cmp.ll
@@ -321,7 +321,7 @@ define i32 @test_zext_cmp11(double %a, double %b) "no-nans-fp-math"="true" {
 ;
 ; ALL-LABEL: test_zext_cmp11:
 ; ALL:       # %bb.0: # %entry
-; ALL-NEXT:    vxorps %xmm2, %xmm2, %xmm2
+; ALL-NEXT:    vxorpd %xmm2, %xmm2, %xmm2
 ; ALL-NEXT:    vucomisd %xmm2, %xmm0
 ; ALL-NEXT:    sete %al
 ; ALL-NEXT:    vucomisd %xmm2, %xmm1
diff --git a/llvm/test/CodeGen/X86/machine-combiner.ll b/llvm/test/CodeGen/X86/machine-combiner.ll
index 162cda8c8..22da63a08 100644
--- a/llvm/test/CodeGen/X86/machine-combiner.ll
+++ b/llvm/test/CodeGen/X86/machine-combiner.ll
@@ -1,13 +1,13 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=sse -enable-unsafe-fp-math -machine-combiner-verify-pattern-order=true < %s | FileCheck %s --check-prefix=SSE
-; RUN: llc -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=avx -enable-unsafe-fp-math -machine-combiner-verify-pattern-order=true < %s | FileCheck %s --check-prefixes=AVX,AVX1
-; RUN: llc -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=avx512vl -enable-unsafe-fp-math -machine-combiner-verify-pattern-order=true < %s | FileCheck %s --check-prefixes=AVX,AVX512
+; RUN: llc -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=sse -enable-unsafe-fp-math -enable-no-nans-fp-math -enable-no-signed-zeros-fp-math -machine-combiner-verify-pattern-order=true < %s | FileCheck %s --check-prefix=SSE
+; RUN: llc -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=avx -enable-unsafe-fp-math -enable-no-nans-fp-math -enable-no-signed-zeros-fp-math -machine-combiner-verify-pattern-order=true < %s | FileCheck %s --check-prefixes=AVX,AVX1
+; RUN: llc -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=avx512vl -enable-unsafe-fp-math -enable-no-nans-fp-math -enable-no-signed-zeros-fp-math -machine-combiner-verify-pattern-order=true < %s | FileCheck %s --check-prefixes=AVX,AVX512
 
 ; Incremental updates of the instruction depths should be enough for this test
 ; case.
-; RUN: llc -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=sse -enable-unsafe-fp-math -machine-combiner-inc-threshold=0 < %s | FileCheck %s --check-prefix=SSE
-; RUN: llc -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=avx -enable-unsafe-fp-math -machine-combiner-inc-threshold=0 < %s | FileCheck %s --check-prefixes=AVX,AVX1
-; RUN: llc -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=avx512vl -enable-unsafe-fp-math -machine-combiner-inc-threshold=0 < %s | FileCheck %s --check-prefixes=AVX,AVX512
+; RUN: llc -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=sse -enable-unsafe-fp-math -enable-no-nans-fp-math -enable-no-signed-zeros-fp-math -machine-combiner-inc-threshold=0 < %s | FileCheck %s --check-prefix=SSE
+; RUN: llc -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=avx -enable-unsafe-fp-math -enable-no-nans-fp-math -enable-no-signed-zeros-fp-math -machine-combiner-inc-threshold=0 < %s | FileCheck %s --check-prefixes=AVX,AVX1
+; RUN: llc -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=avx512vl -enable-unsafe-fp-math -enable-no-nans-fp-math -enable-no-signed-zeros-fp-math -machine-combiner-inc-threshold=0 < %s | FileCheck %s --check-prefixes=AVX,AVX512
 
 ; Verify that the first two adds are independent regardless of how the inputs are
 ; commuted. The destination registers are used as source registers for the third add.
diff --git a/llvm/test/CodeGen/X86/machine-outliner-debuginfo.ll b/llvm/test/CodeGen/X86/machine-outliner-debuginfo.ll
index 709971cd1..b1bd68531 100644
--- a/llvm/test/CodeGen/X86/machine-outliner-debuginfo.ll
+++ b/llvm/test/CodeGen/X86/machine-outliner-debuginfo.ll
@@ -44,7 +44,7 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
 
 declare void @llvm.dbg.value(metadata, i64, metadata, metadata) #1
 
-attributes #0 = { noredzone nounwind ssp uwtable "no-frame-pointer-elim"="true" }
+attributes #0 = { noredzone nounwind ssp uwtable "frame-pointer"="all" }
 
 !llvm.dbg.cu = !{!2}
 !llvm.module.flags = !{!7, !8, !9}
diff --git a/llvm/test/CodeGen/X86/machine-outliner-disubprogram.ll b/llvm/test/CodeGen/X86/machine-outliner-disubprogram.ll
index 41570a2de..990fdb358 100644
--- a/llvm/test/CodeGen/X86/machine-outliner-disubprogram.ll
+++ b/llvm/test/CodeGen/X86/machine-outliner-disubprogram.ll
@@ -130,7 +130,7 @@ entry:
 ; CHECK-SAME: unit: !0,
 ; CHECK-SAME: retainedNodes: [[VARS]]
 
-attributes #0 = { noinline noredzone nounwind optnone ssp uwtable "no-frame-pointer-elim"="true"  }
+attributes #0 = { noinline noredzone nounwind optnone ssp uwtable "frame-pointer"="all"  }
 attributes #1 = { nounwind readnone speculatable }
 attributes #2 = { noredzone }
 
diff --git a/llvm/test/CodeGen/X86/machine-outliner-noredzone.ll b/llvm/test/CodeGen/X86/machine-outliner-noredzone.ll
index 94271e30b..0891352c3 100644
--- a/llvm/test/CodeGen/X86/machine-outliner-noredzone.ll
+++ b/llvm/test/CodeGen/X86/machine-outliner-noredzone.ll
@@ -66,5 +66,5 @@ define void @shibe(i32) #0 {
   ret void
 }
 
-attributes #0 = { noinline nounwind optnone ssp uwtable "no-frame-pointer-elim"="true" }
+attributes #0 = { noinline nounwind optnone ssp uwtable "frame-pointer"="all" }
 attributes #1 = { nounwind }
\ No newline at end of file
diff --git a/llvm/test/CodeGen/X86/machine-outliner-tailcalls.ll b/llvm/test/CodeGen/X86/machine-outliner-tailcalls.ll
index 71ebade62..2906b875d 100644
--- a/llvm/test/CodeGen/X86/machine-outliner-tailcalls.ll
+++ b/llvm/test/CodeGen/X86/machine-outliner-tailcalls.ll
@@ -24,7 +24,7 @@ define i32 @foo1(i32) local_unnamed_addr #0 {
   ret i32 undef
 }
 
-attributes #0 = { noredzone nounwind ssp uwtable "no-frame-pointer-elim"="false" }
+attributes #0 = { noredzone nounwind ssp uwtable "frame-pointer"="none" }
 
 !2 = !{!3, !3, i64 0}
 !3 = !{!"int", !4, i64 0}
diff --git a/llvm/test/CodeGen/X86/machine-outliner.ll b/llvm/test/CodeGen/X86/machine-outliner.ll
index b4c50e691..d480f5198 100644
--- a/llvm/test/CodeGen/X86/machine-outliner.ll
+++ b/llvm/test/CodeGen/X86/machine-outliner.ll
@@ -98,7 +98,7 @@ define i32 @main() #0 {
   ret i32 0
 }
 
-attributes #0 = { noredzone nounwind ssp uwtable "no-frame-pointer-elim"="true" }
+attributes #0 = { noredzone nounwind ssp uwtable "frame-pointer"="all" }
 
 ; CHECK: OUTLINED_FUNCTION_{{[0-9]+}}:
 ; CHECK-DAG:      movl  $1, -{{[0-9]+}}(%rbp)
diff --git a/llvm/test/CodeGen/X86/madd.ll b/llvm/test/CodeGen/X86/madd.ll
index 41f841a8e..117565742 100644
--- a/llvm/test/CodeGen/X86/madd.ll
+++ b/llvm/test/CodeGen/X86/madd.ll
@@ -1675,11 +1675,11 @@ define i32 @test_unsigned_short_1024(i16* nocapture readonly, i16* nocapture rea
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm2
 ; AVX1-NEXT:    vextractf128 $1, %ymm8, %xmm0
 ; AVX1-NEXT:    vpaddd %xmm0, %xmm5, %xmm0
-; AVX1-NEXT:    vpaddd %xmm8, %xmm6, %xmm1
+; AVX1-NEXT:    vpaddd %xmm6, %xmm8, %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm8
 ; AVX1-NEXT:    vextractf128 $1, %ymm9, %xmm0
 ; AVX1-NEXT:    vpaddd %xmm0, %xmm13, %xmm0
-; AVX1-NEXT:    vpaddd %xmm9, %xmm7, %xmm1
+; AVX1-NEXT:    vpaddd %xmm7, %xmm9, %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm9
 ; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm0
 ; AVX1-NEXT:    vpaddd %xmm0, %xmm10, %xmm0
@@ -1876,26 +1876,12 @@ define <4 x i32> @larger_mul(<16 x i16> %A, <16 x i16> %B) {
 ;
 ; AVX1-LABEL: larger_mul:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm2
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm0
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm2, %xmm0
-; AVX1-NEXT:    vpmovsxwd %xmm1, %xmm2
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
-; AVX1-NEXT:    vpmovsxwd %xmm1, %xmm1
-; AVX1-NEXT:    vpackssdw %xmm1, %xmm2, %xmm1
 ; AVX1-NEXT:    vpmaddwd %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: larger_mul:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
-; AVX2-NEXT:    vpmovsxwd %xmm1, %ymm1
-; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX2-NEXT:    vpackssdw %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm2
-; AVX2-NEXT:    vpackssdw %xmm2, %xmm0, %xmm0
 ; AVX2-NEXT:    vpmaddwd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -1912,9 +1898,7 @@ define <4 x i32> @larger_mul(<16 x i16> %A, <16 x i16> %B) {
 ; AVX512-NEXT:    vpinsrd $2, %eax, %xmm1, %xmm1
 ; AVX512-NEXT:    vpextrd $2, %xmm2, %eax
 ; AVX512-NEXT:    vpinsrd $3, %eax, %xmm1, %xmm1
-; AVX512-NEXT:    vpextrd $3, %xmm0, %eax
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpinsrd $1, %eax, %xmm0, %xmm0
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
 ; AVX512-NEXT:    vpextrd $1, %xmm2, %eax
 ; AVX512-NEXT:    vpinsrd $2, %eax, %xmm0, %xmm0
 ; AVX512-NEXT:    vpextrd $3, %xmm2, %eax
@@ -1991,9 +1975,11 @@ define <16 x i32> @pmaddwd_32(<32 x i16> %A, <32 x i16> %B) {
 ;
 ; AVX512F-LABEL: pmaddwd_32:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpmaddwd %ymm3, %ymm1, %ymm1
-; AVX512F-NEXT:    vpmaddwd %ymm2, %ymm0, %ymm0
-; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
+; AVX512F-NEXT:    vpmaddwd %ymm3, %ymm2, %ymm2
+; AVX512F-NEXT:    vpmaddwd %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: pmaddwd_32:
@@ -2202,9 +2188,11 @@ define <16 x i32> @jumbled_indices16(<32 x i16> %A, <32 x i16> %B) {
 ;
 ; AVX512F-LABEL: jumbled_indices16:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpmaddwd %ymm3, %ymm1, %ymm1
-; AVX512F-NEXT:    vpmaddwd %ymm2, %ymm0, %ymm0
-; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
+; AVX512F-NEXT:    vpmaddwd %ymm3, %ymm2, %ymm2
+; AVX512F-NEXT:    vpmaddwd %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: jumbled_indices16:
@@ -2597,29 +2585,29 @@ define <4 x i32> @pmaddwd_bad_indices(<8 x i16>* %Aptr, <8 x i16>* %Bptr) {
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movdqa (%rdi), %xmm0
 ; SSE2-NEXT:    movdqa (%rsi), %xmm1
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm1[0,2,2,3,4,5,6,7]
-; SSE2-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,6,6,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm0[2,1,2,3,4,5,6,7]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,6,5,6,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm0[2,1,2,3,4,5,6,7]
-; SSE2-NEXT:    pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,6,5,6,7]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm3[1,0,3,2,4,5,6,7]
-; SSE2-NEXT:    movdqa %xmm3, %xmm4
-; SSE2-NEXT:    pmulhw %xmm2, %xmm4
-; SSE2-NEXT:    pmullw %xmm2, %xmm3
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[1,0,3,2,4,5,6,7]
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,3,2,3,4,5,6,7]
 ; SSE2-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,7,6,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm1[0,2,2,3,4,5,6,7]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,6,6,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[3,1,2,3,4,5,6,7]
 ; SSE2-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,7,5,6,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[1,0,3,2,4,5,6,7]
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    pmulhw %xmm1, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm4
+; SSE2-NEXT:    pmulhw %xmm3, %xmm4
+; SSE2-NEXT:    pmullw %xmm3, %xmm2
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
+; SSE2-NEXT:    movdqa %xmm0, %xmm3
+; SSE2-NEXT:    pmulhw %xmm1, %xmm3
 ; SSE2-NEXT:    pmullw %xmm1, %xmm0
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
-; SSE2-NEXT:    paddd %xmm3, %xmm0
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; SSE2-NEXT:    paddd %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; AVX-LABEL: pmaddwd_bad_indices:
@@ -2627,13 +2615,13 @@ define <4 x i32> @pmaddwd_bad_indices(<8 x i16>* %Aptr, <8 x i16>* %Bptr) {
 ; AVX-NEXT:    vmovdqa (%rdi), %xmm0
 ; AVX-NEXT:    vmovdqa (%rsi), %xmm1
 ; AVX-NEXT:    vpshufb {{.*#+}} xmm2 = xmm0[2,3,4,5,10,11,12,13,12,13,10,11,12,13,14,15]
-; AVX-NEXT:    vpmovsxwd %xmm2, %xmm2
+; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,6,7,8,9,14,15,8,9,14,15,12,13,14,15]
 ; AVX-NEXT:    vpshufb {{.*#+}} xmm3 = xmm1[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
+; AVX-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[2,3,6,7,10,11,14,15,14,15,10,11,12,13,14,15]
+; AVX-NEXT:    vpmovsxwd %xmm2, %xmm2
 ; AVX-NEXT:    vpmovsxwd %xmm3, %xmm3
 ; AVX-NEXT:    vpmulld %xmm3, %xmm2, %xmm2
-; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,6,7,8,9,14,15,8,9,14,15,12,13,14,15]
 ; AVX-NEXT:    vpmovsxwd %xmm0, %xmm0
-; AVX-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[2,3,6,7,10,11,14,15,14,15,10,11,12,13,14,15]
 ; AVX-NEXT:    vpmovsxwd %xmm1, %xmm1
 ; AVX-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpaddd %xmm0, %xmm2, %xmm0
@@ -2677,9 +2665,9 @@ define i32 @madd_double_reduction(<8 x i16>* %arg, <8 x i16>* %arg1, <8 x i16>*
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vmovdqu (%rdi), %xmm0
 ; AVX-NEXT:    vmovdqu (%rdx), %xmm1
-; AVX-NEXT:    vpmaddwd (%rsi), %xmm0, %xmm0
 ; AVX-NEXT:    vpmaddwd (%rcx), %xmm1, %xmm1
-; AVX-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    vpmaddwd (%rsi), %xmm0, %xmm0
+; AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
@@ -2706,3 +2694,219 @@ define i32 @madd_double_reduction(<8 x i16>* %arg, <8 x i16>* %arg1, <8 x i16>*
   %tmp35 = extractelement <8 x i32> %tmp34, i64 0
   ret i32 %tmp35
 }
+
+define i32 @madd_quad_reduction(<8 x i16>* %arg, <8 x i16>* %arg1, <8 x i16>* %arg2, <8 x i16>* %arg3, <8 x i16>* %arg4, <8 x i16>* %arg5, <8 x i16>* %arg6, <8 x i16>* %arg7) {
+; SSE2-LABEL: madd_quad_reduction:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movq {{[0-9]+}}(%rsp), %r10
+; SSE2-NEXT:    movq {{[0-9]+}}(%rsp), %rax
+; SSE2-NEXT:    movdqu (%rdi), %xmm0
+; SSE2-NEXT:    movdqu (%rsi), %xmm1
+; SSE2-NEXT:    pmaddwd %xmm0, %xmm1
+; SSE2-NEXT:    movdqu (%rdx), %xmm0
+; SSE2-NEXT:    movdqu (%rcx), %xmm2
+; SSE2-NEXT:    pmaddwd %xmm0, %xmm2
+; SSE2-NEXT:    movdqu (%r8), %xmm0
+; SSE2-NEXT:    movdqu (%r9), %xmm3
+; SSE2-NEXT:    pmaddwd %xmm0, %xmm3
+; SSE2-NEXT:    paddd %xmm2, %xmm3
+; SSE2-NEXT:    movdqu (%rax), %xmm0
+; SSE2-NEXT:    movdqu (%r10), %xmm2
+; SSE2-NEXT:    pmaddwd %xmm0, %xmm2
+; SSE2-NEXT:    paddd %xmm3, %xmm2
+; SSE2-NEXT:    paddd %xmm1, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE2-NEXT:    paddd %xmm2, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE2-NEXT:    paddd %xmm0, %xmm1
+; SSE2-NEXT:    movd %xmm1, %eax
+; SSE2-NEXT:    retq
+;
+; AVX-LABEL: madd_quad_reduction:
+; AVX:       # %bb.0:
+; AVX-NEXT:    movq {{[0-9]+}}(%rsp), %r10
+; AVX-NEXT:    movq {{[0-9]+}}(%rsp), %rax
+; AVX-NEXT:    vmovdqu (%rdi), %xmm0
+; AVX-NEXT:    vmovdqu (%rdx), %xmm1
+; AVX-NEXT:    vpmaddwd (%rcx), %xmm1, %xmm1
+; AVX-NEXT:    vpmaddwd (%rsi), %xmm0, %xmm0
+; AVX-NEXT:    vmovdqu (%r8), %xmm2
+; AVX-NEXT:    vpmaddwd (%r9), %xmm2, %xmm2
+; AVX-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; AVX-NEXT:    vmovdqu (%rax), %xmm2
+; AVX-NEXT:    vpmaddwd (%r10), %xmm2, %xmm2
+; AVX-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vmovd %xmm0, %eax
+; AVX-NEXT:    retq
+  %tmp = load <8 x i16>, <8 x i16>* %arg, align 1
+  %tmp6 = load <8 x i16>, <8 x i16>* %arg1, align 1
+  %tmp7 = sext <8 x i16> %tmp to <8 x i32>
+  %tmp17 = sext <8 x i16> %tmp6 to <8 x i32>
+  %tmp19 = mul nsw <8 x i32> %tmp7, %tmp17
+  %tmp20 = load <8 x i16>, <8 x i16>* %arg2, align 1
+  %tmp21 = load <8 x i16>, <8 x i16>* %arg3, align 1
+  %tmp22 = sext <8 x i16> %tmp20 to <8 x i32>
+  %tmp23 = sext <8 x i16> %tmp21 to <8 x i32>
+  %tmp25 = mul nsw <8 x i32> %tmp22, %tmp23
+  %tmp26 = add nuw nsw <8 x i32> %tmp25, %tmp19
+
+  %tmp40 = load <8 x i16>, <8 x i16>* %arg4, align 1
+  %tmp41 = load <8 x i16>, <8 x i16>* %arg5, align 1
+  %tmp42 = sext <8 x i16> %tmp40 to <8 x i32>
+  %tmp43 = sext <8 x i16> %tmp41 to <8 x i32>
+  %tmp45 = mul nsw <8 x i32> %tmp42, %tmp43
+  %tmp56 = add nuw nsw <8 x i32> %tmp26, %tmp45
+
+  %tmp50 = load <8 x i16>, <8 x i16>* %arg6, align 1
+  %tmp51 = load <8 x i16>, <8 x i16>* %arg7, align 1
+  %tmp52 = sext <8 x i16> %tmp50 to <8 x i32>
+  %tmp53 = sext <8 x i16> %tmp51 to <8 x i32>
+  %tmp55 = mul nsw <8 x i32> %tmp52, %tmp53
+  %tmp57 = add nuw nsw <8 x i32> %tmp55, %tmp56
+
+  %tmp29 = shufflevector <8 x i32> %tmp57, <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+  %tmp30 = add <8 x i32> %tmp57, %tmp29
+  %tmp31 = shufflevector <8 x i32> %tmp30, <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %tmp32 = add <8 x i32> %tmp30, %tmp31
+  %tmp33 = shufflevector <8 x i32> %tmp32, <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %tmp34 = add <8 x i32> %tmp32, %tmp33
+  %tmp35 = extractelement <8 x i32> %tmp34, i64 0
+  ret i32 %tmp35
+}
+
+define i64 @sum_and_sum_of_squares(i8* %a, i32 %n) {
+; SSE2-LABEL: sum_and_sum_of_squares:
+; SSE2:       # %bb.0: # %entry
+; SSE2-NEXT:    movl %esi, %eax
+; SSE2-NEXT:    pxor %xmm0, %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    .p2align 4, 0x90
+; SSE2-NEXT:  .LBB33_1: # %vector.body
+; SSE2-NEXT:    # =>This Inner Loop Header: Depth=1
+; SSE2-NEXT:    movq {{.*#+}} xmm4 = mem[0],zero
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3],xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
+; SSE2-NEXT:    movdqa %xmm4, %xmm5
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3]
+; SSE2-NEXT:    paddd %xmm5, %xmm2
+; SSE2-NEXT:    movdqa %xmm4, %xmm5
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm0[4],xmm5[5],xmm0[5],xmm5[6],xmm0[6],xmm5[7],xmm0[7]
+; SSE2-NEXT:    paddd %xmm5, %xmm3
+; SSE2-NEXT:    pmaddwd %xmm4, %xmm4
+; SSE2-NEXT:    paddd %xmm4, %xmm1
+; SSE2-NEXT:    addq $8, %rdi
+; SSE2-NEXT:    addq $-8, %rax
+; SSE2-NEXT:    jne .LBB33_1
+; SSE2-NEXT:  # %bb.2: # %middle.block
+; SSE2-NEXT:    paddd %xmm0, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE2-NEXT:    paddd %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE2-NEXT:    paddd %xmm0, %xmm1
+; SSE2-NEXT:    movd %xmm1, %eax
+; SSE2-NEXT:    retq
+;
+; AVX1-LABEL: sum_and_sum_of_squares:
+; AVX1:       # %bb.0: # %entry
+; AVX1-NEXT:    movl %esi, %eax
+; AVX1-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    .p2align 4, 0x90
+; AVX1-NEXT:  .LBB33_1: # %vector.body
+; AVX1-NEXT:    # =>This Inner Loop Header: Depth=1
+; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm2 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
+; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm3 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm4
+; AVX1-NEXT:    vpaddd %xmm4, %xmm3, %xmm4
+; AVX1-NEXT:    vpaddd %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm1, %ymm1
+; AVX1-NEXT:    vpmaddwd %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpmaddwd %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
+; AVX1-NEXT:    vpaddd %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpaddd %xmm0, %xmm2, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
+; AVX1-NEXT:    addq $8, %rdi
+; AVX1-NEXT:    addq $-8, %rax
+; AVX1-NEXT:    jne .LBB33_1
+; AVX1-NEXT:  # %bb.2: # %middle.block
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vmovd %xmm0, %eax
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX256-LABEL: sum_and_sum_of_squares:
+; AVX256:       # %bb.0: # %entry
+; AVX256-NEXT:    movl %esi, %eax
+; AVX256-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; AVX256-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX256-NEXT:    .p2align 4, 0x90
+; AVX256-NEXT:  .LBB33_1: # %vector.body
+; AVX256-NEXT:    # =>This Inner Loop Header: Depth=1
+; AVX256-NEXT:    vpmovzxbd {{.*#+}} ymm2 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
+; AVX256-NEXT:    vpaddd %ymm1, %ymm2, %ymm1
+; AVX256-NEXT:    vpmaddwd %ymm2, %ymm2, %ymm2
+; AVX256-NEXT:    vpaddd %ymm0, %ymm2, %ymm0
+; AVX256-NEXT:    addq $8, %rdi
+; AVX256-NEXT:    addq $-8, %rax
+; AVX256-NEXT:    jne .LBB33_1
+; AVX256-NEXT:  # %bb.2: # %middle.block
+; AVX256-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX256-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX256-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX256-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX256-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX256-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX256-NEXT:    vmovd %xmm0, %eax
+; AVX256-NEXT:    vzeroupper
+; AVX256-NEXT:    retq
+entry:
+  %0 = zext i32 %n to i64
+  br label %vector.body
+
+vector.body:
+  %index = phi i64 [ %index.next, %vector.body ], [ 0, %entry ]
+  %vec.phi = phi <8 x i32> [ %6, %vector.body ], [ zeroinitializer, %entry ]
+  %sum.phi = phi <8 x i32> [ %4, %vector.body ], [ zeroinitializer, %entry ]
+  %1 = getelementptr inbounds i8, i8* %a, i64 %index
+  %2 = bitcast i8* %1 to <8 x i8>*
+  %wide.load = load <8 x i8>, <8 x i8>* %2, align 1
+  %3 = zext <8 x i8> %wide.load to <8 x i32>
+  %4 = add nsw <8 x i32> %3, %sum.phi
+  %5 = mul nsw <8 x i32> %3, %3
+  %6 = add nsw <8 x i32> %5, %vec.phi
+  %index.next = add i64 %index, 8
+  %7 = icmp eq i64 %index.next, %0
+  br i1 %7, label %middle.block, label %vector.body
+
+middle.block:
+  %rdx.shuf35 = shufflevector <8 x i32> %4, <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+  %bin.rdx36 = add <8 x i32> %4, %rdx.shuf35
+  %rdx.shuf37 = shufflevector <8 x i32> %bin.rdx36, <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %bin.rdx38 = add <8 x i32> %bin.rdx36, %rdx.shuf37
+  %rdx.shuf39 = shufflevector <8 x i32> %bin.rdx38, <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %bin.rdx40 = add <8 x i32> %bin.rdx38, %rdx.shuf39
+  %8 = extractelement <8 x i32> %bin.rdx40, i32 0
+  %rdx.shuf = shufflevector <8 x i32> %6, <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+  %bin.rdx = add <8 x i32> %6, %rdx.shuf
+  %rdx.shuf31 = shufflevector <8 x i32> %bin.rdx, <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %bin.rdx32 = add <8 x i32> %bin.rdx, %rdx.shuf31
+  %rdx.shuf33 = shufflevector <8 x i32> %bin.rdx32, <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %bin.rdx34 = add <8 x i32> %bin.rdx32, %rdx.shuf33
+  %9 = extractelement <8 x i32> %bin.rdx34, i32 0
+  %tmp = zext i32 %8 to i64
+  %tmp28 = shl nuw i64 %tmp, 32
+  %tmp29 = zext i32 %9 to i64
+  ret i64 %tmp29
+}
diff --git a/llvm/test/CodeGen/X86/masked-iv-safe.ll b/llvm/test/CodeGen/X86/masked-iv-safe.ll
index aca02a94d..2e86f9f23 100644
--- a/llvm/test/CodeGen/X86/masked-iv-safe.ll
+++ b/llvm/test/CodeGen/X86/masked-iv-safe.ll
@@ -1,14 +1,28 @@
-; RUN: llc < %s -mcpu=generic -mtriple=x86_64-- | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mcpu=generic -mtriple=x86_64-- | FileCheck %s --implicit-check-not '{{and|movz|sar|shl}}'
 
 ; Optimize away zext-inreg and sext-inreg on the loop induction
 ; variable using trip-count information.
 
-; CHECK-LABEL: count_up
-; CHECK-NOT: {{and|movz|sar|shl}}
-; CHECK: addq $8
-; CHECK-NOT: {{and|movz|sar|shl}}
-; CHECK: jne
 define void @count_up(double* %d, i64 %n) nounwind {
+; CHECK-LABEL: count_up:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movq $-80, %rax
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
+; CHECK-NEXT:    movsd {{.*#+}} xmm2 = mem[0],zero
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB0_1: # %loop
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
+; CHECK-NEXT:    mulsd %xmm0, %xmm3
+; CHECK-NEXT:    mulsd %xmm1, %xmm3
+; CHECK-NEXT:    mulsd %xmm2, %xmm3
+; CHECK-NEXT:    movsd %xmm3, 80(%rdi,%rax)
+; CHECK-NEXT:    addq $8, %rax
+; CHECK-NEXT:    jne .LBB0_1
+; CHECK-NEXT:  # %bb.2: # %return
+; CHECK-NEXT:    retq
 entry:
 	br label %loop
 
@@ -36,12 +50,25 @@ return:
 	ret void
 }
 
-; CHECK-LABEL: count_down
-; CHECK-NOT: {{and|movz|sar|shl}}
-; CHECK: addq $-8
-; CHECK-NOT: {{and|movz|sar|shl}}
-; CHECK: jne
 define void @count_down(double* %d, i64 %n) nounwind {
+; CHECK-LABEL: count_down:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movl $80, %eax
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
+; CHECK-NEXT:    movsd {{.*#+}} xmm2 = mem[0],zero
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB1_1: # %loop
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
+; CHECK-NEXT:    mulsd %xmm0, %xmm3
+; CHECK-NEXT:    mulsd %xmm1, %xmm3
+; CHECK-NEXT:    mulsd %xmm2, %xmm3
+; CHECK-NEXT:    movsd %xmm3, (%rdi,%rax)
+; CHECK-NEXT:    addq $-8, %rax
+; CHECK-NEXT:    jne .LBB1_1
+; CHECK-NEXT:  # %bb.2: # %return
+; CHECK-NEXT:    retq
 entry:
 	br label %loop
 
@@ -69,12 +96,25 @@ return:
 	ret void
 }
 
-; CHECK-LABEL: count_up_signed
-; CHECK-NOT: {{and|movz|sar|shl}}
-; CHECK: addq $8
-; CHECK-NOT: {{and|movz|sar|shl}}
-; CHECK: jne
 define void @count_up_signed(double* %d, i64 %n) nounwind {
+; CHECK-LABEL: count_up_signed:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movq $-80, %rax
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
+; CHECK-NEXT:    movsd {{.*#+}} xmm2 = mem[0],zero
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB2_1: # %loop
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
+; CHECK-NEXT:    mulsd %xmm0, %xmm3
+; CHECK-NEXT:    mulsd %xmm1, %xmm3
+; CHECK-NEXT:    mulsd %xmm2, %xmm3
+; CHECK-NEXT:    movsd %xmm3, 80(%rdi,%rax)
+; CHECK-NEXT:    addq $8, %rax
+; CHECK-NEXT:    jne .LBB2_1
+; CHECK-NEXT:  # %bb.2: # %return
+; CHECK-NEXT:    retq
 entry:
 	br label %loop
 
@@ -104,12 +144,25 @@ return:
 	ret void
 }
 
-; CHECK-LABEL: count_down_signed
-; CHECK-NOT: {{and|movz|sar|shl}}
-; CHECK: addq $-8
-; CHECK-NOT: {{and|movz|sar|shl}}
-; CHECK: jne
 define void @count_down_signed(double* %d, i64 %n) nounwind {
+; CHECK-LABEL: count_down_signed:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movl $80, %eax
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
+; CHECK-NEXT:    movsd {{.*#+}} xmm2 = mem[0],zero
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB3_1: # %loop
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
+; CHECK-NEXT:    mulsd %xmm0, %xmm3
+; CHECK-NEXT:    mulsd %xmm1, %xmm3
+; CHECK-NEXT:    mulsd %xmm2, %xmm3
+; CHECK-NEXT:    movsd %xmm3, (%rdi,%rax)
+; CHECK-NEXT:    addq $-8, %rax
+; CHECK-NEXT:    jne .LBB3_1
+; CHECK-NEXT:  # %bb.2: # %return
+; CHECK-NEXT:    retq
 entry:
 	br label %loop
 
@@ -139,12 +192,29 @@ return:
 	ret void
 }
 
-; CHECK-LABEL: another_count_up
-; CHECK-NOT: {{and|movz|sar|shl}}
-; CHECK: addq $8
-; CHECK-NOT: {{and|movz|sar|shl}}
-; CHECK: jne
 define void @another_count_up(double* %d, i64 %n) nounwind {
+; CHECK-LABEL: another_count_up:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movq $-8, %rax
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
+; CHECK-NEXT:    movsd {{.*#+}} xmm2 = mem[0],zero
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB4_1: # %loop
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
+; CHECK-NEXT:    mulsd %xmm0, %xmm3
+; CHECK-NEXT:    movsd %xmm3, 2048(%rdi,%rax)
+; CHECK-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
+; CHECK-NEXT:    mulsd %xmm1, %xmm3
+; CHECK-NEXT:    movsd %xmm3, 134217728(%rdi,%rax)
+; CHECK-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
+; CHECK-NEXT:    mulsd %xmm2, %xmm3
+; CHECK-NEXT:    movsd %xmm3, (%rdi,%rax)
+; CHECK-NEXT:    addq $8, %rax
+; CHECK-NEXT:    jne .LBB4_1
+; CHECK-NEXT:  # %bb.2: # %return
+; CHECK-NEXT:    retq
 entry:
 	br label %loop
 
@@ -172,12 +242,33 @@ return:
 	ret void
 }
 
-; CHECK-LABEL: another_count_down
-; CHECK-NOT: {{and|movz|sar|shl}}
-; CHECK: addq $-8
-; CHECK-NOT: {{and|movz|sar|shl}}
-; CHECK: jne
 define void @another_count_down(double* %d, i64 %n) nounwind {
+; CHECK-LABEL: another_count_down:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movq $-2040, %rax # imm = 0xF808
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
+; CHECK-NEXT:    movsd {{.*#+}} xmm2 = mem[0],zero
+; CHECK-NEXT:    movq %rdi, %rcx
+; CHECK-NEXT:    movq %rdi, %rdx
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB5_1: # %loop
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
+; CHECK-NEXT:    mulsd %xmm0, %xmm3
+; CHECK-NEXT:    movsd %xmm3, 2040(%rdi,%rax)
+; CHECK-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
+; CHECK-NEXT:    divsd %xmm1, %xmm3
+; CHECK-NEXT:    movsd %xmm3, (%rcx)
+; CHECK-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
+; CHECK-NEXT:    mulsd %xmm2, %xmm3
+; CHECK-NEXT:    movsd %xmm3, (%rdx)
+; CHECK-NEXT:    addq $-8, %rdx
+; CHECK-NEXT:    addq $134217720, %rcx # imm = 0x7FFFFF8
+; CHECK-NEXT:    addq $2040, %rax # imm = 0x7F8
+; CHECK-NEXT:    jne .LBB5_1
+; CHECK-NEXT:  # %bb.2: # %return
+; CHECK-NEXT:    retq
 entry:
 	br label %loop
 
@@ -205,12 +296,25 @@ return:
 	ret void
 }
 
-; CHECK-LABEL: another_count_up_signed
-; CHECK-NOT: {{and|movz|sar|shl}}
-; CHECK: addq $8
-; CHECK-NOT: {{and|movz|sar|shl}}
-; CHECK: jne
 define void @another_count_up_signed(double* %d, i64 %n) nounwind {
+; CHECK-LABEL: another_count_up_signed:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movq $-8, %rax
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
+; CHECK-NEXT:    movsd {{.*#+}} xmm2 = mem[0],zero
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB6_1: # %loop
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
+; CHECK-NEXT:    mulsd %xmm0, %xmm3
+; CHECK-NEXT:    divsd %xmm1, %xmm3
+; CHECK-NEXT:    mulsd %xmm2, %xmm3
+; CHECK-NEXT:    movsd %xmm3, (%rdi,%rax)
+; CHECK-NEXT:    addq $8, %rax
+; CHECK-NEXT:    jne .LBB6_1
+; CHECK-NEXT:  # %bb.2: # %return
+; CHECK-NEXT:    retq
 entry:
 	br label %loop
 
@@ -240,12 +344,25 @@ return:
 	ret void
 }
 
-; CHECK-LABEL: another_count_down_signed
-; CHECK-NOT: {{and|movz|sar|shl}}
-; CHECK: addq $-8
-; CHECK-NOT: {{and|movz|sar|shl}}
-; CHECK: jne
 define void @another_count_down_signed(double* %d, i64 %n) nounwind {
+; CHECK-LABEL: another_count_down_signed:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movl $8, %eax
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
+; CHECK-NEXT:    movsd {{.*#+}} xmm2 = mem[0],zero
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB7_1: # %loop
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
+; CHECK-NEXT:    mulsd %xmm0, %xmm3
+; CHECK-NEXT:    divsd %xmm1, %xmm3
+; CHECK-NEXT:    mulsd %xmm2, %xmm3
+; CHECK-NEXT:    movsd %xmm3, -8(%rdi,%rax)
+; CHECK-NEXT:    addq $-8, %rax
+; CHECK-NEXT:    jne .LBB7_1
+; CHECK-NEXT:  # %bb.2: # %return
+; CHECK-NEXT:    retq
 entry:
 	br label %loop
 
diff --git a/llvm/test/CodeGen/X86/masked-iv-unsafe.ll b/llvm/test/CodeGen/X86/masked-iv-unsafe.ll
index 53a1f0619..76f2ad22b 100644
--- a/llvm/test/CodeGen/X86/masked-iv-unsafe.ll
+++ b/llvm/test/CodeGen/X86/masked-iv-unsafe.ll
@@ -1,12 +1,35 @@
-; RUN: llc < %s -mtriple=x86_64-- > %t
-; RUN: grep and %t | count 6
-; RUN: grep movzb %t | count 6
-; RUN: grep sar %t | count 12
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-- | FileCheck %s
 
 ; Don't optimize away zext-inreg and sext-inreg on the loop induction
 ; variable, because it isn't safe to do so in these cases.
 
 define void @count_up(double* %d, i64 %n) nounwind {
+; CHECK-LABEL: count_up:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movl $10, %eax
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
+; CHECK-NEXT:    movsd {{.*#+}} xmm2 = mem[0],zero
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB0_1: # %loop
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movzbl %al, %ecx
+; CHECK-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
+; CHECK-NEXT:    mulsd %xmm0, %xmm3
+; CHECK-NEXT:    movsd %xmm3, (%rdi,%rcx,8)
+; CHECK-NEXT:    movl %eax, %ecx
+; CHECK-NEXT:    andl $16777215, %ecx # imm = 0xFFFFFF
+; CHECK-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
+; CHECK-NEXT:    mulsd %xmm1, %xmm3
+; CHECK-NEXT:    movsd %xmm3, (%rdi,%rcx,8)
+; CHECK-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
+; CHECK-NEXT:    mulsd %xmm2, %xmm3
+; CHECK-NEXT:    movsd %xmm3, (%rdi,%rax,8)
+; CHECK-NEXT:    incq %rax
+; CHECK-NEXT:    jne .LBB0_1
+; CHECK-NEXT:  # %bb.2: # %return
+; CHECK-NEXT:    retq
 entry:
 	br label %loop
 
@@ -35,6 +58,32 @@ return:
 }
 
 define void @count_down(double* %d, i64 %n) nounwind {
+; CHECK-LABEL: count_down:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movl $10, %eax
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
+; CHECK-NEXT:    movsd {{.*#+}} xmm2 = mem[0],zero
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB1_1: # %loop
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movzbl %al, %ecx
+; CHECK-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
+; CHECK-NEXT:    mulsd %xmm0, %xmm3
+; CHECK-NEXT:    movsd %xmm3, (%rdi,%rcx,8)
+; CHECK-NEXT:    movl %eax, %ecx
+; CHECK-NEXT:    andl $16777215, %ecx # imm = 0xFFFFFF
+; CHECK-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
+; CHECK-NEXT:    mulsd %xmm1, %xmm3
+; CHECK-NEXT:    movsd %xmm3, (%rdi,%rcx,8)
+; CHECK-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
+; CHECK-NEXT:    mulsd %xmm2, %xmm3
+; CHECK-NEXT:    movsd %xmm3, (%rdi,%rax,8)
+; CHECK-NEXT:    decq %rax
+; CHECK-NEXT:    cmpq $20, %rax
+; CHECK-NEXT:    jne .LBB1_1
+; CHECK-NEXT:  # %bb.2: # %return
+; CHECK-NEXT:    retq
 entry:
 	br label %loop
 
@@ -63,6 +112,36 @@ return:
 }
 
 define void @count_up_signed(double* %d, i64 %n) nounwind {
+; CHECK-LABEL: count_up_signed:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movl $10, %eax
+; CHECK-NEXT:    movl $167772160, %ecx # imm = 0xA000000
+; CHECK-NEXT:    movl $2560, %edx # imm = 0xA00
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
+; CHECK-NEXT:    movsd {{.*#+}} xmm2 = mem[0],zero
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB2_1: # %loop
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movq %rdx, %rsi
+; CHECK-NEXT:    sarq $8, %rsi
+; CHECK-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
+; CHECK-NEXT:    mulsd %xmm0, %xmm3
+; CHECK-NEXT:    movsd %xmm3, (%rdi,%rsi,8)
+; CHECK-NEXT:    movq %rcx, %rsi
+; CHECK-NEXT:    sarq $24, %rsi
+; CHECK-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
+; CHECK-NEXT:    mulsd %xmm1, %xmm3
+; CHECK-NEXT:    movsd %xmm3, (%rdi,%rsi,8)
+; CHECK-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
+; CHECK-NEXT:    mulsd %xmm2, %xmm3
+; CHECK-NEXT:    movsd %xmm3, (%rdi,%rax,8)
+; CHECK-NEXT:    addq $16777216, %rcx # imm = 0x1000000
+; CHECK-NEXT:    addq $256, %rdx # imm = 0x100
+; CHECK-NEXT:    incq %rax
+; CHECK-NEXT:    jne .LBB2_1
+; CHECK-NEXT:  # %bb.2: # %return
+; CHECK-NEXT:    retq
 entry:
 	br label %loop
 
@@ -93,6 +172,36 @@ return:
 }
 
 define void @count_down_signed(double* %d, i64 %n) nounwind {
+; CHECK-LABEL: count_down_signed:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movq $-10, %rax
+; CHECK-NEXT:    movl $167772160, %ecx # imm = 0xA000000
+; CHECK-NEXT:    movl $2560, %edx # imm = 0xA00
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
+; CHECK-NEXT:    movsd {{.*#+}} xmm2 = mem[0],zero
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB3_1: # %loop
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movq %rdx, %rsi
+; CHECK-NEXT:    sarq $8, %rsi
+; CHECK-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
+; CHECK-NEXT:    mulsd %xmm0, %xmm3
+; CHECK-NEXT:    movsd %xmm3, (%rdi,%rsi,8)
+; CHECK-NEXT:    movq %rcx, %rsi
+; CHECK-NEXT:    sarq $24, %rsi
+; CHECK-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
+; CHECK-NEXT:    mulsd %xmm1, %xmm3
+; CHECK-NEXT:    movsd %xmm3, (%rdi,%rsi,8)
+; CHECK-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
+; CHECK-NEXT:    mulsd %xmm2, %xmm3
+; CHECK-NEXT:    movsd %xmm3, 160(%rdi,%rax,8)
+; CHECK-NEXT:    addq $-16777216, %rcx # imm = 0xFF000000
+; CHECK-NEXT:    addq $-256, %rdx
+; CHECK-NEXT:    decq %rax
+; CHECK-NEXT:    jne .LBB3_1
+; CHECK-NEXT:  # %bb.2: # %return
+; CHECK-NEXT:    retq
 entry:
 	br label %loop
 
@@ -123,6 +232,32 @@ return:
 }
 
 define void @another_count_up(double* %d, i64 %n) nounwind {
+; CHECK-LABEL: another_count_up:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
+; CHECK-NEXT:    movsd {{.*#+}} xmm2 = mem[0],zero
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB4_1: # %loop
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movzbl %al, %ecx
+; CHECK-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
+; CHECK-NEXT:    mulsd %xmm0, %xmm3
+; CHECK-NEXT:    movsd %xmm3, (%rdi,%rcx,8)
+; CHECK-NEXT:    movl %eax, %ecx
+; CHECK-NEXT:    andl $16777215, %ecx # imm = 0xFFFFFF
+; CHECK-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
+; CHECK-NEXT:    mulsd %xmm1, %xmm3
+; CHECK-NEXT:    movsd %xmm3, (%rdi,%rcx,8)
+; CHECK-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
+; CHECK-NEXT:    mulsd %xmm2, %xmm3
+; CHECK-NEXT:    movsd %xmm3, (%rdi,%rax,8)
+; CHECK-NEXT:    incq %rax
+; CHECK-NEXT:    cmpq %rax, %rsi
+; CHECK-NEXT:    jne .LBB4_1
+; CHECK-NEXT:  # %bb.2: # %return
+; CHECK-NEXT:    retq
 entry:
         br label %loop
 
@@ -151,6 +286,31 @@ return:
 }
 
 define void @another_count_down(double* %d, i64 %n) nounwind {
+; CHECK-LABEL: another_count_down:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
+; CHECK-NEXT:    movsd {{.*#+}} xmm2 = mem[0],zero
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB5_1: # %loop
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movzbl %sil, %eax
+; CHECK-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
+; CHECK-NEXT:    mulsd %xmm0, %xmm3
+; CHECK-NEXT:    movsd %xmm3, (%rdi,%rax,8)
+; CHECK-NEXT:    movl %esi, %eax
+; CHECK-NEXT:    andl $16777215, %eax # imm = 0xFFFFFF
+; CHECK-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
+; CHECK-NEXT:    mulsd %xmm1, %xmm3
+; CHECK-NEXT:    movsd %xmm3, (%rdi,%rax,8)
+; CHECK-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
+; CHECK-NEXT:    mulsd %xmm2, %xmm3
+; CHECK-NEXT:    movsd %xmm3, (%rdi,%rsi,8)
+; CHECK-NEXT:    decq %rsi
+; CHECK-NEXT:    cmpq $10, %rsi
+; CHECK-NEXT:    jne .LBB5_1
+; CHECK-NEXT:  # %bb.2: # %return
+; CHECK-NEXT:    retq
 entry:
         br label %loop
 
@@ -179,6 +339,37 @@ return:
 }
 
 define void @another_count_up_signed(double* %d, i64 %n) nounwind {
+; CHECK-LABEL: another_count_up_signed:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xorl %r8d, %r8d
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
+; CHECK-NEXT:    movsd {{.*#+}} xmm2 = mem[0],zero
+; CHECK-NEXT:    xorl %ecx, %ecx
+; CHECK-NEXT:    movq %rdi, %rdx
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB6_1: # %loop
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movq %r8, %rax
+; CHECK-NEXT:    sarq $8, %rax
+; CHECK-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
+; CHECK-NEXT:    mulsd %xmm0, %xmm3
+; CHECK-NEXT:    movsd %xmm3, (%rdi,%rax,8)
+; CHECK-NEXT:    movq %rcx, %rax
+; CHECK-NEXT:    sarq $24, %rax
+; CHECK-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
+; CHECK-NEXT:    mulsd %xmm1, %xmm3
+; CHECK-NEXT:    movsd %xmm3, (%rdi,%rax,8)
+; CHECK-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
+; CHECK-NEXT:    mulsd %xmm2, %xmm3
+; CHECK-NEXT:    movsd %xmm3, (%rdx)
+; CHECK-NEXT:    addq $8, %rdx
+; CHECK-NEXT:    addq $16777216, %rcx # imm = 0x1000000
+; CHECK-NEXT:    addq $256, %r8 # imm = 0x100
+; CHECK-NEXT:    decq %rsi
+; CHECK-NEXT:    jne .LBB6_1
+; CHECK-NEXT:  # %bb.2: # %return
+; CHECK-NEXT:    retq
 entry:
         br label %loop
 
@@ -209,6 +400,37 @@ return:
 }
 
 define void @another_count_down_signed(double* %d, i64 %n) nounwind {
+; CHECK-LABEL: another_count_down_signed:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movq %rsi, %rax
+; CHECK-NEXT:    shlq $24, %rax
+; CHECK-NEXT:    leaq -10(%rsi), %rcx
+; CHECK-NEXT:    shlq $8, %rsi
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
+; CHECK-NEXT:    movsd {{.*#+}} xmm2 = mem[0],zero
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB7_1: # %loop
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movq %rsi, %rdx
+; CHECK-NEXT:    sarq $8, %rdx
+; CHECK-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
+; CHECK-NEXT:    mulsd %xmm0, %xmm3
+; CHECK-NEXT:    movsd %xmm3, (%rdi,%rdx,8)
+; CHECK-NEXT:    movq %rax, %rdx
+; CHECK-NEXT:    sarq $24, %rdx
+; CHECK-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
+; CHECK-NEXT:    mulsd %xmm1, %xmm3
+; CHECK-NEXT:    movsd %xmm3, (%rdi,%rdx,8)
+; CHECK-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
+; CHECK-NEXT:    mulsd %xmm2, %xmm3
+; CHECK-NEXT:    movsd %xmm3, 80(%rdi,%rcx,8)
+; CHECK-NEXT:    addq $-16777216, %rax # imm = 0xFF000000
+; CHECK-NEXT:    addq $-256, %rsi
+; CHECK-NEXT:    decq %rcx
+; CHECK-NEXT:    jne .LBB7_1
+; CHECK-NEXT:  # %bb.2: # %return
+; CHECK-NEXT:    retq
 entry:
         br label %loop
 
@@ -239,6 +461,32 @@ return:
 }
 
 define void @yet_another_count_down(double* %d, i64 %n) nounwind {
+; CHECK-LABEL: yet_another_count_down:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movq $-2040, %rax # imm = 0xF808
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
+; CHECK-NEXT:    movsd {{.*#+}} xmm2 = mem[0],zero
+; CHECK-NEXT:    movq %rdi, %rcx
+; CHECK-NEXT:    movq %rdi, %rdx
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB8_1: # %loop
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
+; CHECK-NEXT:    mulsd %xmm0, %xmm3
+; CHECK-NEXT:    movsd %xmm3, 2040(%rdi,%rax)
+; CHECK-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
+; CHECK-NEXT:    mulsd %xmm1, %xmm3
+; CHECK-NEXT:    movsd %xmm3, (%rcx)
+; CHECK-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
+; CHECK-NEXT:    mulsd %xmm2, %xmm3
+; CHECK-NEXT:    movsd %xmm3, (%rdx)
+; CHECK-NEXT:    addq $-8, %rdx
+; CHECK-NEXT:    addq $134217720, %rcx # imm = 0x7FFFFF8
+; CHECK-NEXT:    addq $2040, %rax # imm = 0x7F8
+; CHECK-NEXT:    jne .LBB8_1
+; CHECK-NEXT:  # %bb.2: # %return
+; CHECK-NEXT:    retq
 entry:
 	br label %loop
 
@@ -267,6 +515,32 @@ return:
 }
 
 define void @yet_another_count_up(double* %d, i64 %n) nounwind {
+; CHECK-LABEL: yet_another_count_up:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
+; CHECK-NEXT:    movsd {{.*#+}} xmm2 = mem[0],zero
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB9_1: # %loop
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movzbl %al, %ecx
+; CHECK-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
+; CHECK-NEXT:    mulsd %xmm0, %xmm3
+; CHECK-NEXT:    movsd %xmm3, (%rdi,%rcx,8)
+; CHECK-NEXT:    movl %eax, %ecx
+; CHECK-NEXT:    andl $16777215, %ecx # imm = 0xFFFFFF
+; CHECK-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
+; CHECK-NEXT:    mulsd %xmm1, %xmm3
+; CHECK-NEXT:    movsd %xmm3, (%rdi,%rcx,8)
+; CHECK-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
+; CHECK-NEXT:    mulsd %xmm2, %xmm3
+; CHECK-NEXT:    movsd %xmm3, (%rdi,%rax,8)
+; CHECK-NEXT:    addq $3, %rax
+; CHECK-NEXT:    cmpq $10, %rax
+; CHECK-NEXT:    jne .LBB9_1
+; CHECK-NEXT:  # %bb.2: # %return
+; CHECK-NEXT:    retq
 entry:
         br label %loop
 
@@ -295,6 +569,31 @@ return:
 }
 
 define void @still_another_count_down(double* %d, i64 %n) nounwind {
+; CHECK-LABEL: still_another_count_down:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movl $10, %eax
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
+; CHECK-NEXT:    movsd {{.*#+}} xmm2 = mem[0],zero
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB10_1: # %loop
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movzbl %al, %ecx
+; CHECK-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
+; CHECK-NEXT:    mulsd %xmm0, %xmm3
+; CHECK-NEXT:    movsd %xmm3, (%rdi,%rcx,8)
+; CHECK-NEXT:    movl %eax, %ecx
+; CHECK-NEXT:    andl $16777215, %ecx # imm = 0xFFFFFF
+; CHECK-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
+; CHECK-NEXT:    mulsd %xmm1, %xmm3
+; CHECK-NEXT:    movsd %xmm3, (%rdi,%rcx,8)
+; CHECK-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
+; CHECK-NEXT:    mulsd %xmm2, %xmm3
+; CHECK-NEXT:    movsd %xmm3, (%rdi,%rax,8)
+; CHECK-NEXT:    addq $-3, %rax
+; CHECK-NEXT:    jne .LBB10_1
+; CHECK-NEXT:  # %bb.2: # %return
+; CHECK-NEXT:    retq
 entry:
         br label %loop
 
@@ -323,6 +622,36 @@ return:
 }
 
 define void @yet_another_count_up_signed(double* %d, i64 %n) nounwind {
+; CHECK-LABEL: yet_another_count_up_signed:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movq $-10, %rax
+; CHECK-NEXT:    xorl %ecx, %ecx
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
+; CHECK-NEXT:    movsd {{.*#+}} xmm2 = mem[0],zero
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB11_1: # %loop
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movq %rcx, %rsi
+; CHECK-NEXT:    sarq $8, %rsi
+; CHECK-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
+; CHECK-NEXT:    mulsd %xmm0, %xmm3
+; CHECK-NEXT:    movsd %xmm3, (%rdi,%rsi,8)
+; CHECK-NEXT:    movq %rdx, %rsi
+; CHECK-NEXT:    sarq $24, %rsi
+; CHECK-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
+; CHECK-NEXT:    mulsd %xmm1, %xmm3
+; CHECK-NEXT:    movsd %xmm3, (%rdi,%rsi,8)
+; CHECK-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
+; CHECK-NEXT:    mulsd %xmm2, %xmm3
+; CHECK-NEXT:    movsd %xmm3, 80(%rdi,%rax,8)
+; CHECK-NEXT:    addq $50331648, %rdx # imm = 0x3000000
+; CHECK-NEXT:    addq $768, %rcx # imm = 0x300
+; CHECK-NEXT:    addq $3, %rax
+; CHECK-NEXT:    jne .LBB11_1
+; CHECK-NEXT:  # %bb.2: # %return
+; CHECK-NEXT:    retq
 entry:
         br label %loop
 
@@ -353,6 +682,36 @@ return:
 }
 
 define void @yet_another_count_down_signed(double* %d, i64 %n) nounwind {
+; CHECK-LABEL: yet_another_count_down_signed:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movl $10, %eax
+; CHECK-NEXT:    movl $167772160, %ecx # imm = 0xA000000
+; CHECK-NEXT:    movl $2560, %edx # imm = 0xA00
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
+; CHECK-NEXT:    movsd {{.*#+}} xmm2 = mem[0],zero
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB12_1: # %loop
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movq %rdx, %rsi
+; CHECK-NEXT:    sarq $8, %rsi
+; CHECK-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
+; CHECK-NEXT:    mulsd %xmm0, %xmm3
+; CHECK-NEXT:    movsd %xmm3, (%rdi,%rsi,8)
+; CHECK-NEXT:    movq %rcx, %rsi
+; CHECK-NEXT:    sarq $24, %rsi
+; CHECK-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
+; CHECK-NEXT:    mulsd %xmm1, %xmm3
+; CHECK-NEXT:    movsd %xmm3, (%rdi,%rsi,8)
+; CHECK-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
+; CHECK-NEXT:    mulsd %xmm2, %xmm3
+; CHECK-NEXT:    movsd %xmm3, (%rdi,%rax,8)
+; CHECK-NEXT:    addq $-50331648, %rcx # imm = 0xFD000000
+; CHECK-NEXT:    addq $-768, %rdx # imm = 0xFD00
+; CHECK-NEXT:    addq $-3, %rax
+; CHECK-NEXT:    jne .LBB12_1
+; CHECK-NEXT:  # %bb.2: # %return
+; CHECK-NEXT:    retq
 entry:
         br label %loop
 
diff --git a/llvm/test/CodeGen/X86/masked_compressstore.ll b/llvm/test/CodeGen/X86/masked_compressstore.ll
index 687ea286b..c275b4c9a 100644
--- a/llvm/test/CodeGen/X86/masked_compressstore.ll
+++ b/llvm/test/CodeGen/X86/masked_compressstore.ll
@@ -12,185 +12,210 @@
 ;
 
 define void @compressstore_v8f64_v8i1(double* %base, <8 x double> %V, <8 x i1> %mask) {
-; SSE2-LABEL: compressstore_v8f64_v8i1:
-; SSE2:       ## %bb.0:
-; SSE2-NEXT:    movd %xmm4, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB0_2
-; SSE2-NEXT:  ## %bb.1: ## %cond.store
-; SSE2-NEXT:    movlps %xmm0, (%rdi)
-; SSE2-NEXT:    addq $8, %rdi
-; SSE2-NEXT:  LBB0_2: ## %else
-; SSE2-NEXT:    shrl $16, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB0_4
-; SSE2-NEXT:  ## %bb.3: ## %cond.store1
-; SSE2-NEXT:    movhps %xmm0, (%rdi)
-; SSE2-NEXT:    addq $8, %rdi
-; SSE2-NEXT:  LBB0_4: ## %else2
-; SSE2-NEXT:    pextrw $2, %xmm4, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB0_6
-; SSE2-NEXT:  ## %bb.5: ## %cond.store4
-; SSE2-NEXT:    movlps %xmm1, (%rdi)
-; SSE2-NEXT:    addq $8, %rdi
-; SSE2-NEXT:  LBB0_6: ## %else5
-; SSE2-NEXT:    pextrw $3, %xmm4, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB0_8
-; SSE2-NEXT:  ## %bb.7: ## %cond.store7
-; SSE2-NEXT:    movhps %xmm1, (%rdi)
-; SSE2-NEXT:    addq $8, %rdi
-; SSE2-NEXT:  LBB0_8: ## %else8
-; SSE2-NEXT:    pextrw $4, %xmm4, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB0_10
-; SSE2-NEXT:  ## %bb.9: ## %cond.store10
-; SSE2-NEXT:    movlps %xmm2, (%rdi)
-; SSE2-NEXT:    addq $8, %rdi
-; SSE2-NEXT:  LBB0_10: ## %else11
-; SSE2-NEXT:    pextrw $5, %xmm4, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB0_12
-; SSE2-NEXT:  ## %bb.11: ## %cond.store13
-; SSE2-NEXT:    movhps %xmm2, (%rdi)
-; SSE2-NEXT:    addq $8, %rdi
-; SSE2-NEXT:  LBB0_12: ## %else14
-; SSE2-NEXT:    pextrw $6, %xmm4, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB0_14
-; SSE2-NEXT:  ## %bb.13: ## %cond.store16
-; SSE2-NEXT:    movlps %xmm3, (%rdi)
-; SSE2-NEXT:    addq $8, %rdi
-; SSE2-NEXT:  LBB0_14: ## %else17
-; SSE2-NEXT:    pextrw $7, %xmm4, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB0_16
-; SSE2-NEXT:  ## %bb.15: ## %cond.store19
-; SSE2-NEXT:    movhps %xmm3, (%rdi)
-; SSE2-NEXT:  LBB0_16: ## %else20
-; SSE2-NEXT:    retq
+; SSE-LABEL: compressstore_v8f64_v8i1:
+; SSE:       ## %bb.0:
+; SSE-NEXT:    psllw $15, %xmm4
+; SSE-NEXT:    packsswb %xmm0, %xmm4
+; SSE-NEXT:    pmovmskb %xmm4, %eax
+; SSE-NEXT:    testb $1, %al
+; SSE-NEXT:    jne LBB0_1
+; SSE-NEXT:  ## %bb.2: ## %else
+; SSE-NEXT:    testb $2, %al
+; SSE-NEXT:    jne LBB0_3
+; SSE-NEXT:  LBB0_4: ## %else2
+; SSE-NEXT:    testb $4, %al
+; SSE-NEXT:    jne LBB0_5
+; SSE-NEXT:  LBB0_6: ## %else5
+; SSE-NEXT:    testb $8, %al
+; SSE-NEXT:    jne LBB0_7
+; SSE-NEXT:  LBB0_8: ## %else8
+; SSE-NEXT:    testb $16, %al
+; SSE-NEXT:    jne LBB0_9
+; SSE-NEXT:  LBB0_10: ## %else11
+; SSE-NEXT:    testb $32, %al
+; SSE-NEXT:    jne LBB0_11
+; SSE-NEXT:  LBB0_12: ## %else14
+; SSE-NEXT:    testb $64, %al
+; SSE-NEXT:    jne LBB0_13
+; SSE-NEXT:  LBB0_14: ## %else17
+; SSE-NEXT:    testb $-128, %al
+; SSE-NEXT:    jne LBB0_15
+; SSE-NEXT:  LBB0_16: ## %else20
+; SSE-NEXT:    retq
+; SSE-NEXT:  LBB0_1: ## %cond.store
+; SSE-NEXT:    movq %xmm0, (%rdi)
+; SSE-NEXT:    addq $8, %rdi
+; SSE-NEXT:    testb $2, %al
+; SSE-NEXT:    je LBB0_4
+; SSE-NEXT:  LBB0_3: ## %cond.store1
+; SSE-NEXT:    movhpd %xmm0, (%rdi)
+; SSE-NEXT:    addq $8, %rdi
+; SSE-NEXT:    testb $4, %al
+; SSE-NEXT:    je LBB0_6
+; SSE-NEXT:  LBB0_5: ## %cond.store4
+; SSE-NEXT:    movlps %xmm1, (%rdi)
+; SSE-NEXT:    addq $8, %rdi
+; SSE-NEXT:    testb $8, %al
+; SSE-NEXT:    je LBB0_8
+; SSE-NEXT:  LBB0_7: ## %cond.store7
+; SSE-NEXT:    movhps %xmm1, (%rdi)
+; SSE-NEXT:    addq $8, %rdi
+; SSE-NEXT:    testb $16, %al
+; SSE-NEXT:    je LBB0_10
+; SSE-NEXT:  LBB0_9: ## %cond.store10
+; SSE-NEXT:    movlps %xmm2, (%rdi)
+; SSE-NEXT:    addq $8, %rdi
+; SSE-NEXT:    testb $32, %al
+; SSE-NEXT:    je LBB0_12
+; SSE-NEXT:  LBB0_11: ## %cond.store13
+; SSE-NEXT:    movhps %xmm2, (%rdi)
+; SSE-NEXT:    addq $8, %rdi
+; SSE-NEXT:    testb $64, %al
+; SSE-NEXT:    je LBB0_14
+; SSE-NEXT:  LBB0_13: ## %cond.store16
+; SSE-NEXT:    movlps %xmm3, (%rdi)
+; SSE-NEXT:    addq $8, %rdi
+; SSE-NEXT:    testb $-128, %al
+; SSE-NEXT:    je LBB0_16
+; SSE-NEXT:  LBB0_15: ## %cond.store19
+; SSE-NEXT:    movhps %xmm3, (%rdi)
+; SSE-NEXT:    retq
 ;
-; SSE42-LABEL: compressstore_v8f64_v8i1:
-; SSE42:       ## %bb.0:
-; SSE42-NEXT:    pextrb $0, %xmm4, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB0_2
-; SSE42-NEXT:  ## %bb.1: ## %cond.store
-; SSE42-NEXT:    movlps %xmm0, (%rdi)
-; SSE42-NEXT:    addq $8, %rdi
-; SSE42-NEXT:  LBB0_2: ## %else
-; SSE42-NEXT:    pextrb $2, %xmm4, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB0_4
-; SSE42-NEXT:  ## %bb.3: ## %cond.store1
-; SSE42-NEXT:    movhps %xmm0, (%rdi)
-; SSE42-NEXT:    addq $8, %rdi
-; SSE42-NEXT:  LBB0_4: ## %else2
-; SSE42-NEXT:    pextrb $4, %xmm4, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB0_6
-; SSE42-NEXT:  ## %bb.5: ## %cond.store4
-; SSE42-NEXT:    movlps %xmm1, (%rdi)
-; SSE42-NEXT:    addq $8, %rdi
-; SSE42-NEXT:  LBB0_6: ## %else5
-; SSE42-NEXT:    pextrb $6, %xmm4, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB0_8
-; SSE42-NEXT:  ## %bb.7: ## %cond.store7
-; SSE42-NEXT:    movhps %xmm1, (%rdi)
-; SSE42-NEXT:    addq $8, %rdi
-; SSE42-NEXT:  LBB0_8: ## %else8
-; SSE42-NEXT:    pextrb $8, %xmm4, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB0_10
-; SSE42-NEXT:  ## %bb.9: ## %cond.store10
-; SSE42-NEXT:    movlps %xmm2, (%rdi)
-; SSE42-NEXT:    addq $8, %rdi
-; SSE42-NEXT:  LBB0_10: ## %else11
-; SSE42-NEXT:    pextrb $10, %xmm4, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB0_12
-; SSE42-NEXT:  ## %bb.11: ## %cond.store13
-; SSE42-NEXT:    movhps %xmm2, (%rdi)
-; SSE42-NEXT:    addq $8, %rdi
-; SSE42-NEXT:  LBB0_12: ## %else14
-; SSE42-NEXT:    pextrb $12, %xmm4, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB0_14
-; SSE42-NEXT:  ## %bb.13: ## %cond.store16
-; SSE42-NEXT:    movlps %xmm3, (%rdi)
-; SSE42-NEXT:    addq $8, %rdi
-; SSE42-NEXT:  LBB0_14: ## %else17
-; SSE42-NEXT:    pextrb $14, %xmm4, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB0_16
-; SSE42-NEXT:  ## %bb.15: ## %cond.store19
-; SSE42-NEXT:    movhps %xmm3, (%rdi)
-; SSE42-NEXT:  LBB0_16: ## %else20
-; SSE42-NEXT:    retq
+; AVX1-LABEL: compressstore_v8f64_v8i1:
+; AVX1:       ## %bb.0:
+; AVX1-NEXT:    vpsllw $15, %xmm2, %xmm2
+; AVX1-NEXT:    vpacksswb %xmm0, %xmm2, %xmm2
+; AVX1-NEXT:    vpmovmskb %xmm2, %eax
+; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    je LBB0_2
+; AVX1-NEXT:  ## %bb.1: ## %cond.store
+; AVX1-NEXT:    vmovq %xmm0, (%rdi)
+; AVX1-NEXT:    addq $8, %rdi
+; AVX1-NEXT:  LBB0_2: ## %else
+; AVX1-NEXT:    testb $2, %al
+; AVX1-NEXT:    je LBB0_4
+; AVX1-NEXT:  ## %bb.3: ## %cond.store1
+; AVX1-NEXT:    vmovhpd %xmm0, (%rdi)
+; AVX1-NEXT:    addq $8, %rdi
+; AVX1-NEXT:  LBB0_4: ## %else2
+; AVX1-NEXT:    testb $4, %al
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    jne LBB0_5
+; AVX1-NEXT:  ## %bb.6: ## %else5
+; AVX1-NEXT:    testb $8, %al
+; AVX1-NEXT:    jne LBB0_7
+; AVX1-NEXT:  LBB0_8: ## %else8
+; AVX1-NEXT:    testb $16, %al
+; AVX1-NEXT:    jne LBB0_9
+; AVX1-NEXT:  LBB0_10: ## %else11
+; AVX1-NEXT:    testb $32, %al
+; AVX1-NEXT:    je LBB0_12
+; AVX1-NEXT:  LBB0_11: ## %cond.store13
+; AVX1-NEXT:    vmovhps %xmm1, (%rdi)
+; AVX1-NEXT:    addq $8, %rdi
+; AVX1-NEXT:  LBB0_12: ## %else14
+; AVX1-NEXT:    testb $64, %al
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm0
+; AVX1-NEXT:    jne LBB0_13
+; AVX1-NEXT:  ## %bb.14: ## %else17
+; AVX1-NEXT:    testb $-128, %al
+; AVX1-NEXT:    jne LBB0_15
+; AVX1-NEXT:  LBB0_16: ## %else20
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+; AVX1-NEXT:  LBB0_5: ## %cond.store4
+; AVX1-NEXT:    vmovlps %xmm0, (%rdi)
+; AVX1-NEXT:    addq $8, %rdi
+; AVX1-NEXT:    testb $8, %al
+; AVX1-NEXT:    je LBB0_8
+; AVX1-NEXT:  LBB0_7: ## %cond.store7
+; AVX1-NEXT:    vmovhps %xmm0, (%rdi)
+; AVX1-NEXT:    addq $8, %rdi
+; AVX1-NEXT:    testb $16, %al
+; AVX1-NEXT:    je LBB0_10
+; AVX1-NEXT:  LBB0_9: ## %cond.store10
+; AVX1-NEXT:    vmovlps %xmm1, (%rdi)
+; AVX1-NEXT:    addq $8, %rdi
+; AVX1-NEXT:    testb $32, %al
+; AVX1-NEXT:    jne LBB0_11
+; AVX1-NEXT:    jmp LBB0_12
+; AVX1-NEXT:  LBB0_13: ## %cond.store16
+; AVX1-NEXT:    vmovlps %xmm0, (%rdi)
+; AVX1-NEXT:    addq $8, %rdi
+; AVX1-NEXT:    testb $-128, %al
+; AVX1-NEXT:    je LBB0_16
+; AVX1-NEXT:  LBB0_15: ## %cond.store19
+; AVX1-NEXT:    vmovhps %xmm0, (%rdi)
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
 ;
-; AVX1OR2-LABEL: compressstore_v8f64_v8i1:
-; AVX1OR2:       ## %bb.0:
-; AVX1OR2-NEXT:    vpextrb $0, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
-; AVX1OR2-NEXT:    je LBB0_2
-; AVX1OR2-NEXT:  ## %bb.1: ## %cond.store
-; AVX1OR2-NEXT:    vmovlps %xmm0, (%rdi)
-; AVX1OR2-NEXT:    addq $8, %rdi
-; AVX1OR2-NEXT:  LBB0_2: ## %else
-; AVX1OR2-NEXT:    vpextrb $2, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
-; AVX1OR2-NEXT:    je LBB0_4
-; AVX1OR2-NEXT:  ## %bb.3: ## %cond.store1
-; AVX1OR2-NEXT:    vmovhps %xmm0, (%rdi)
-; AVX1OR2-NEXT:    addq $8, %rdi
-; AVX1OR2-NEXT:  LBB0_4: ## %else2
-; AVX1OR2-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
-; AVX1OR2-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX1OR2-NEXT:    je LBB0_6
-; AVX1OR2-NEXT:  ## %bb.5: ## %cond.store4
-; AVX1OR2-NEXT:    vmovlps %xmm0, (%rdi)
-; AVX1OR2-NEXT:    addq $8, %rdi
-; AVX1OR2-NEXT:  LBB0_6: ## %else5
-; AVX1OR2-NEXT:    vpextrb $6, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
-; AVX1OR2-NEXT:    je LBB0_8
-; AVX1OR2-NEXT:  ## %bb.7: ## %cond.store7
-; AVX1OR2-NEXT:    vmovhps %xmm0, (%rdi)
-; AVX1OR2-NEXT:    addq $8, %rdi
-; AVX1OR2-NEXT:  LBB0_8: ## %else8
-; AVX1OR2-NEXT:    vpextrb $8, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
-; AVX1OR2-NEXT:    je LBB0_10
-; AVX1OR2-NEXT:  ## %bb.9: ## %cond.store10
-; AVX1OR2-NEXT:    vmovlps %xmm1, (%rdi)
-; AVX1OR2-NEXT:    addq $8, %rdi
-; AVX1OR2-NEXT:  LBB0_10: ## %else11
-; AVX1OR2-NEXT:    vpextrb $10, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
-; AVX1OR2-NEXT:    je LBB0_12
-; AVX1OR2-NEXT:  ## %bb.11: ## %cond.store13
-; AVX1OR2-NEXT:    vmovhps %xmm1, (%rdi)
-; AVX1OR2-NEXT:    addq $8, %rdi
-; AVX1OR2-NEXT:  LBB0_12: ## %else14
-; AVX1OR2-NEXT:    vpextrb $12, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
-; AVX1OR2-NEXT:    vextractf128 $1, %ymm1, %xmm0
-; AVX1OR2-NEXT:    je LBB0_14
-; AVX1OR2-NEXT:  ## %bb.13: ## %cond.store16
-; AVX1OR2-NEXT:    vmovlps %xmm0, (%rdi)
-; AVX1OR2-NEXT:    addq $8, %rdi
-; AVX1OR2-NEXT:  LBB0_14: ## %else17
-; AVX1OR2-NEXT:    vpextrb $14, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
-; AVX1OR2-NEXT:    je LBB0_16
-; AVX1OR2-NEXT:  ## %bb.15: ## %cond.store19
-; AVX1OR2-NEXT:    vmovhps %xmm0, (%rdi)
-; AVX1OR2-NEXT:  LBB0_16: ## %else20
-; AVX1OR2-NEXT:    vzeroupper
-; AVX1OR2-NEXT:    retq
+; AVX2-LABEL: compressstore_v8f64_v8i1:
+; AVX2:       ## %bb.0:
+; AVX2-NEXT:    vpsllw $15, %xmm2, %xmm2
+; AVX2-NEXT:    vpacksswb %xmm0, %xmm2, %xmm2
+; AVX2-NEXT:    vpmovmskb %xmm2, %eax
+; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    je LBB0_2
+; AVX2-NEXT:  ## %bb.1: ## %cond.store
+; AVX2-NEXT:    vmovq %xmm0, (%rdi)
+; AVX2-NEXT:    addq $8, %rdi
+; AVX2-NEXT:  LBB0_2: ## %else
+; AVX2-NEXT:    testb $2, %al
+; AVX2-NEXT:    je LBB0_4
+; AVX2-NEXT:  ## %bb.3: ## %cond.store1
+; AVX2-NEXT:    vmovhpd %xmm0, (%rdi)
+; AVX2-NEXT:    addq $8, %rdi
+; AVX2-NEXT:  LBB0_4: ## %else2
+; AVX2-NEXT:    testb $4, %al
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
+; AVX2-NEXT:    jne LBB0_5
+; AVX2-NEXT:  ## %bb.6: ## %else5
+; AVX2-NEXT:    testb $8, %al
+; AVX2-NEXT:    jne LBB0_7
+; AVX2-NEXT:  LBB0_8: ## %else8
+; AVX2-NEXT:    testb $16, %al
+; AVX2-NEXT:    jne LBB0_9
+; AVX2-NEXT:  LBB0_10: ## %else11
+; AVX2-NEXT:    testb $32, %al
+; AVX2-NEXT:    je LBB0_12
+; AVX2-NEXT:  LBB0_11: ## %cond.store13
+; AVX2-NEXT:    vmovhps %xmm1, (%rdi)
+; AVX2-NEXT:    addq $8, %rdi
+; AVX2-NEXT:  LBB0_12: ## %else14
+; AVX2-NEXT:    testb $64, %al
+; AVX2-NEXT:    vextractf128 $1, %ymm1, %xmm0
+; AVX2-NEXT:    jne LBB0_13
+; AVX2-NEXT:  ## %bb.14: ## %else17
+; AVX2-NEXT:    testb $-128, %al
+; AVX2-NEXT:    jne LBB0_15
+; AVX2-NEXT:  LBB0_16: ## %else20
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+; AVX2-NEXT:  LBB0_5: ## %cond.store4
+; AVX2-NEXT:    vmovq %xmm0, (%rdi)
+; AVX2-NEXT:    addq $8, %rdi
+; AVX2-NEXT:    testb $8, %al
+; AVX2-NEXT:    je LBB0_8
+; AVX2-NEXT:  LBB0_7: ## %cond.store7
+; AVX2-NEXT:    vmovhpd %xmm0, (%rdi)
+; AVX2-NEXT:    addq $8, %rdi
+; AVX2-NEXT:    testb $16, %al
+; AVX2-NEXT:    je LBB0_10
+; AVX2-NEXT:  LBB0_9: ## %cond.store10
+; AVX2-NEXT:    vmovlps %xmm1, (%rdi)
+; AVX2-NEXT:    addq $8, %rdi
+; AVX2-NEXT:    testb $32, %al
+; AVX2-NEXT:    jne LBB0_11
+; AVX2-NEXT:    jmp LBB0_12
+; AVX2-NEXT:  LBB0_13: ## %cond.store16
+; AVX2-NEXT:    vmovlps %xmm0, (%rdi)
+; AVX2-NEXT:    addq $8, %rdi
+; AVX2-NEXT:    testb $-128, %al
+; AVX2-NEXT:    je LBB0_16
+; AVX2-NEXT:  LBB0_15: ## %cond.store19
+; AVX2-NEXT:    vmovhps %xmm0, (%rdi)
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: compressstore_v8f64_v8i1:
 ; AVX512F:       ## %bb.0:
@@ -222,358 +247,267 @@ define void @compressstore_v8f64_v8i1(double* %base, <8 x double> %V, <8 x i1> %
 }
 
 define void @compressstore_v16f64_v16i1(double* %base, <16 x double> %V, <16 x i1> %mask) {
-; SSE2-LABEL: compressstore_v16f64_v16i1:
-; SSE2:       ## %bb.0:
-; SSE2-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm8
-; SSE2-NEXT:    movdqa %xmm8, -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    testb $1, -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    je LBB1_2
-; SSE2-NEXT:  ## %bb.1: ## %cond.store
-; SSE2-NEXT:    movlps %xmm0, (%rdi)
-; SSE2-NEXT:    addq $8, %rdi
-; SSE2-NEXT:  LBB1_2: ## %else
-; SSE2-NEXT:    movd %xmm8, %eax
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    shrl $8, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    je LBB1_4
-; SSE2-NEXT:  ## %bb.3: ## %cond.store1
-; SSE2-NEXT:    movhps %xmm0, (%rdi)
-; SSE2-NEXT:    addq $8, %rdi
-; SSE2-NEXT:  LBB1_4: ## %else2
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    shrl $16, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    je LBB1_6
-; SSE2-NEXT:  ## %bb.5: ## %cond.store4
-; SSE2-NEXT:    movlps %xmm1, (%rdi)
-; SSE2-NEXT:    addq $8, %rdi
-; SSE2-NEXT:  LBB1_6: ## %else5
-; SSE2-NEXT:    shrl $24, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB1_8
-; SSE2-NEXT:  ## %bb.7: ## %cond.store7
-; SSE2-NEXT:    movhps %xmm1, (%rdi)
-; SSE2-NEXT:    addq $8, %rdi
-; SSE2-NEXT:  LBB1_8: ## %else8
-; SSE2-NEXT:    pextrw $2, %xmm8, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB1_10
-; SSE2-NEXT:  ## %bb.9: ## %cond.store10
-; SSE2-NEXT:    movlps %xmm2, (%rdi)
-; SSE2-NEXT:    addq $8, %rdi
-; SSE2-NEXT:  LBB1_10: ## %else11
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB1_12
-; SSE2-NEXT:  ## %bb.11: ## %cond.store13
-; SSE2-NEXT:    movhps %xmm2, (%rdi)
-; SSE2-NEXT:    addq $8, %rdi
-; SSE2-NEXT:  LBB1_12: ## %else14
-; SSE2-NEXT:    pextrw $3, %xmm8, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB1_14
-; SSE2-NEXT:  ## %bb.13: ## %cond.store16
-; SSE2-NEXT:    movlps %xmm3, (%rdi)
-; SSE2-NEXT:    addq $8, %rdi
-; SSE2-NEXT:  LBB1_14: ## %else17
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB1_16
-; SSE2-NEXT:  ## %bb.15: ## %cond.store19
-; SSE2-NEXT:    movhps %xmm3, (%rdi)
-; SSE2-NEXT:    addq $8, %rdi
-; SSE2-NEXT:  LBB1_16: ## %else20
-; SSE2-NEXT:    pextrw $4, %xmm8, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB1_18
-; SSE2-NEXT:  ## %bb.17: ## %cond.store22
-; SSE2-NEXT:    movlps %xmm4, (%rdi)
-; SSE2-NEXT:    addq $8, %rdi
-; SSE2-NEXT:  LBB1_18: ## %else23
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB1_20
-; SSE2-NEXT:  ## %bb.19: ## %cond.store25
-; SSE2-NEXT:    movhps %xmm4, (%rdi)
-; SSE2-NEXT:    addq $8, %rdi
-; SSE2-NEXT:  LBB1_20: ## %else26
-; SSE2-NEXT:    pextrw $5, %xmm8, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB1_22
-; SSE2-NEXT:  ## %bb.21: ## %cond.store28
-; SSE2-NEXT:    movlps %xmm5, (%rdi)
-; SSE2-NEXT:    addq $8, %rdi
-; SSE2-NEXT:  LBB1_22: ## %else29
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB1_24
-; SSE2-NEXT:  ## %bb.23: ## %cond.store31
-; SSE2-NEXT:    movhps %xmm5, (%rdi)
-; SSE2-NEXT:    addq $8, %rdi
-; SSE2-NEXT:  LBB1_24: ## %else32
-; SSE2-NEXT:    pextrw $6, %xmm8, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB1_26
-; SSE2-NEXT:  ## %bb.25: ## %cond.store34
-; SSE2-NEXT:    movlps %xmm6, (%rdi)
-; SSE2-NEXT:    addq $8, %rdi
-; SSE2-NEXT:  LBB1_26: ## %else35
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB1_28
-; SSE2-NEXT:  ## %bb.27: ## %cond.store37
-; SSE2-NEXT:    movhps %xmm6, (%rdi)
-; SSE2-NEXT:    addq $8, %rdi
-; SSE2-NEXT:  LBB1_28: ## %else38
-; SSE2-NEXT:    pextrw $7, %xmm8, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB1_30
-; SSE2-NEXT:  ## %bb.29: ## %cond.store40
-; SSE2-NEXT:    movlps %xmm7, (%rdi)
-; SSE2-NEXT:    addq $8, %rdi
-; SSE2-NEXT:  LBB1_30: ## %else41
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB1_32
-; SSE2-NEXT:  ## %bb.31: ## %cond.store43
-; SSE2-NEXT:    movhps %xmm7, (%rdi)
-; SSE2-NEXT:  LBB1_32: ## %else44
-; SSE2-NEXT:    retq
-;
-; SSE42-LABEL: compressstore_v16f64_v16i1:
-; SSE42:       ## %bb.0:
-; SSE42-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm8
-; SSE42-NEXT:    pextrb $0, %xmm8, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB1_2
-; SSE42-NEXT:  ## %bb.1: ## %cond.store
-; SSE42-NEXT:    movlps %xmm0, (%rdi)
-; SSE42-NEXT:    addq $8, %rdi
-; SSE42-NEXT:  LBB1_2: ## %else
-; SSE42-NEXT:    pextrb $1, %xmm8, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB1_4
-; SSE42-NEXT:  ## %bb.3: ## %cond.store1
-; SSE42-NEXT:    movhps %xmm0, (%rdi)
-; SSE42-NEXT:    addq $8, %rdi
-; SSE42-NEXT:  LBB1_4: ## %else2
-; SSE42-NEXT:    pextrb $2, %xmm8, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB1_6
-; SSE42-NEXT:  ## %bb.5: ## %cond.store4
-; SSE42-NEXT:    movlps %xmm1, (%rdi)
-; SSE42-NEXT:    addq $8, %rdi
-; SSE42-NEXT:  LBB1_6: ## %else5
-; SSE42-NEXT:    pextrb $3, %xmm8, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB1_8
-; SSE42-NEXT:  ## %bb.7: ## %cond.store7
-; SSE42-NEXT:    movhps %xmm1, (%rdi)
-; SSE42-NEXT:    addq $8, %rdi
-; SSE42-NEXT:  LBB1_8: ## %else8
-; SSE42-NEXT:    pextrb $4, %xmm8, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB1_10
-; SSE42-NEXT:  ## %bb.9: ## %cond.store10
-; SSE42-NEXT:    movlps %xmm2, (%rdi)
-; SSE42-NEXT:    addq $8, %rdi
-; SSE42-NEXT:  LBB1_10: ## %else11
-; SSE42-NEXT:    pextrb $5, %xmm8, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB1_12
-; SSE42-NEXT:  ## %bb.11: ## %cond.store13
-; SSE42-NEXT:    movhps %xmm2, (%rdi)
-; SSE42-NEXT:    addq $8, %rdi
-; SSE42-NEXT:  LBB1_12: ## %else14
-; SSE42-NEXT:    pextrb $6, %xmm8, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB1_14
-; SSE42-NEXT:  ## %bb.13: ## %cond.store16
-; SSE42-NEXT:    movlps %xmm3, (%rdi)
-; SSE42-NEXT:    addq $8, %rdi
-; SSE42-NEXT:  LBB1_14: ## %else17
-; SSE42-NEXT:    pextrb $7, %xmm8, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB1_16
-; SSE42-NEXT:  ## %bb.15: ## %cond.store19
-; SSE42-NEXT:    movhps %xmm3, (%rdi)
-; SSE42-NEXT:    addq $8, %rdi
-; SSE42-NEXT:  LBB1_16: ## %else20
-; SSE42-NEXT:    pextrb $8, %xmm8, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB1_18
-; SSE42-NEXT:  ## %bb.17: ## %cond.store22
-; SSE42-NEXT:    movlps %xmm4, (%rdi)
-; SSE42-NEXT:    addq $8, %rdi
-; SSE42-NEXT:  LBB1_18: ## %else23
-; SSE42-NEXT:    pextrb $9, %xmm8, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB1_20
-; SSE42-NEXT:  ## %bb.19: ## %cond.store25
-; SSE42-NEXT:    movhps %xmm4, (%rdi)
-; SSE42-NEXT:    addq $8, %rdi
-; SSE42-NEXT:  LBB1_20: ## %else26
-; SSE42-NEXT:    pextrb $10, %xmm8, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB1_22
-; SSE42-NEXT:  ## %bb.21: ## %cond.store28
-; SSE42-NEXT:    movlps %xmm5, (%rdi)
-; SSE42-NEXT:    addq $8, %rdi
-; SSE42-NEXT:  LBB1_22: ## %else29
-; SSE42-NEXT:    pextrb $11, %xmm8, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB1_24
-; SSE42-NEXT:  ## %bb.23: ## %cond.store31
-; SSE42-NEXT:    movhps %xmm5, (%rdi)
-; SSE42-NEXT:    addq $8, %rdi
-; SSE42-NEXT:  LBB1_24: ## %else32
-; SSE42-NEXT:    pextrb $12, %xmm8, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB1_26
-; SSE42-NEXT:  ## %bb.25: ## %cond.store34
-; SSE42-NEXT:    movlps %xmm6, (%rdi)
-; SSE42-NEXT:    addq $8, %rdi
-; SSE42-NEXT:  LBB1_26: ## %else35
-; SSE42-NEXT:    pextrb $13, %xmm8, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB1_28
-; SSE42-NEXT:  ## %bb.27: ## %cond.store37
-; SSE42-NEXT:    movhps %xmm6, (%rdi)
-; SSE42-NEXT:    addq $8, %rdi
-; SSE42-NEXT:  LBB1_28: ## %else38
-; SSE42-NEXT:    pextrb $14, %xmm8, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB1_30
-; SSE42-NEXT:  ## %bb.29: ## %cond.store40
-; SSE42-NEXT:    movlps %xmm7, (%rdi)
-; SSE42-NEXT:    addq $8, %rdi
-; SSE42-NEXT:  LBB1_30: ## %else41
-; SSE42-NEXT:    pextrb $15, %xmm8, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB1_32
-; SSE42-NEXT:  ## %bb.31: ## %cond.store43
-; SSE42-NEXT:    movhps %xmm7, (%rdi)
-; SSE42-NEXT:  LBB1_32: ## %else44
-; SSE42-NEXT:    retq
+; SSE-LABEL: compressstore_v16f64_v16i1:
+; SSE:       ## %bb.0:
+; SSE-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm8
+; SSE-NEXT:    psllw $7, %xmm8
+; SSE-NEXT:    pmovmskb %xmm8, %eax
+; SSE-NEXT:    testb $1, %al
+; SSE-NEXT:    jne LBB1_1
+; SSE-NEXT:  ## %bb.2: ## %else
+; SSE-NEXT:    testb $2, %al
+; SSE-NEXT:    jne LBB1_3
+; SSE-NEXT:  LBB1_4: ## %else2
+; SSE-NEXT:    testb $4, %al
+; SSE-NEXT:    jne LBB1_5
+; SSE-NEXT:  LBB1_6: ## %else5
+; SSE-NEXT:    testb $8, %al
+; SSE-NEXT:    jne LBB1_7
+; SSE-NEXT:  LBB1_8: ## %else8
+; SSE-NEXT:    testb $16, %al
+; SSE-NEXT:    jne LBB1_9
+; SSE-NEXT:  LBB1_10: ## %else11
+; SSE-NEXT:    testb $32, %al
+; SSE-NEXT:    jne LBB1_11
+; SSE-NEXT:  LBB1_12: ## %else14
+; SSE-NEXT:    testb $64, %al
+; SSE-NEXT:    jne LBB1_13
+; SSE-NEXT:  LBB1_14: ## %else17
+; SSE-NEXT:    testb $-128, %al
+; SSE-NEXT:    jne LBB1_15
+; SSE-NEXT:  LBB1_16: ## %else20
+; SSE-NEXT:    testl $256, %eax ## imm = 0x100
+; SSE-NEXT:    jne LBB1_17
+; SSE-NEXT:  LBB1_18: ## %else23
+; SSE-NEXT:    testl $512, %eax ## imm = 0x200
+; SSE-NEXT:    jne LBB1_19
+; SSE-NEXT:  LBB1_20: ## %else26
+; SSE-NEXT:    testl $1024, %eax ## imm = 0x400
+; SSE-NEXT:    jne LBB1_21
+; SSE-NEXT:  LBB1_22: ## %else29
+; SSE-NEXT:    testl $2048, %eax ## imm = 0x800
+; SSE-NEXT:    jne LBB1_23
+; SSE-NEXT:  LBB1_24: ## %else32
+; SSE-NEXT:    testl $4096, %eax ## imm = 0x1000
+; SSE-NEXT:    jne LBB1_25
+; SSE-NEXT:  LBB1_26: ## %else35
+; SSE-NEXT:    testl $8192, %eax ## imm = 0x2000
+; SSE-NEXT:    jne LBB1_27
+; SSE-NEXT:  LBB1_28: ## %else38
+; SSE-NEXT:    testl $16384, %eax ## imm = 0x4000
+; SSE-NEXT:    jne LBB1_29
+; SSE-NEXT:  LBB1_30: ## %else41
+; SSE-NEXT:    testl $32768, %eax ## imm = 0x8000
+; SSE-NEXT:    jne LBB1_31
+; SSE-NEXT:  LBB1_32: ## %else44
+; SSE-NEXT:    retq
+; SSE-NEXT:  LBB1_1: ## %cond.store
+; SSE-NEXT:    movlps %xmm0, (%rdi)
+; SSE-NEXT:    addq $8, %rdi
+; SSE-NEXT:    testb $2, %al
+; SSE-NEXT:    je LBB1_4
+; SSE-NEXT:  LBB1_3: ## %cond.store1
+; SSE-NEXT:    movhps %xmm0, (%rdi)
+; SSE-NEXT:    addq $8, %rdi
+; SSE-NEXT:    testb $4, %al
+; SSE-NEXT:    je LBB1_6
+; SSE-NEXT:  LBB1_5: ## %cond.store4
+; SSE-NEXT:    movlps %xmm1, (%rdi)
+; SSE-NEXT:    addq $8, %rdi
+; SSE-NEXT:    testb $8, %al
+; SSE-NEXT:    je LBB1_8
+; SSE-NEXT:  LBB1_7: ## %cond.store7
+; SSE-NEXT:    movhps %xmm1, (%rdi)
+; SSE-NEXT:    addq $8, %rdi
+; SSE-NEXT:    testb $16, %al
+; SSE-NEXT:    je LBB1_10
+; SSE-NEXT:  LBB1_9: ## %cond.store10
+; SSE-NEXT:    movlps %xmm2, (%rdi)
+; SSE-NEXT:    addq $8, %rdi
+; SSE-NEXT:    testb $32, %al
+; SSE-NEXT:    je LBB1_12
+; SSE-NEXT:  LBB1_11: ## %cond.store13
+; SSE-NEXT:    movhps %xmm2, (%rdi)
+; SSE-NEXT:    addq $8, %rdi
+; SSE-NEXT:    testb $64, %al
+; SSE-NEXT:    je LBB1_14
+; SSE-NEXT:  LBB1_13: ## %cond.store16
+; SSE-NEXT:    movlps %xmm3, (%rdi)
+; SSE-NEXT:    addq $8, %rdi
+; SSE-NEXT:    testb $-128, %al
+; SSE-NEXT:    je LBB1_16
+; SSE-NEXT:  LBB1_15: ## %cond.store19
+; SSE-NEXT:    movhps %xmm3, (%rdi)
+; SSE-NEXT:    addq $8, %rdi
+; SSE-NEXT:    testl $256, %eax ## imm = 0x100
+; SSE-NEXT:    je LBB1_18
+; SSE-NEXT:  LBB1_17: ## %cond.store22
+; SSE-NEXT:    movlps %xmm4, (%rdi)
+; SSE-NEXT:    addq $8, %rdi
+; SSE-NEXT:    testl $512, %eax ## imm = 0x200
+; SSE-NEXT:    je LBB1_20
+; SSE-NEXT:  LBB1_19: ## %cond.store25
+; SSE-NEXT:    movhps %xmm4, (%rdi)
+; SSE-NEXT:    addq $8, %rdi
+; SSE-NEXT:    testl $1024, %eax ## imm = 0x400
+; SSE-NEXT:    je LBB1_22
+; SSE-NEXT:  LBB1_21: ## %cond.store28
+; SSE-NEXT:    movlps %xmm5, (%rdi)
+; SSE-NEXT:    addq $8, %rdi
+; SSE-NEXT:    testl $2048, %eax ## imm = 0x800
+; SSE-NEXT:    je LBB1_24
+; SSE-NEXT:  LBB1_23: ## %cond.store31
+; SSE-NEXT:    movhps %xmm5, (%rdi)
+; SSE-NEXT:    addq $8, %rdi
+; SSE-NEXT:    testl $4096, %eax ## imm = 0x1000
+; SSE-NEXT:    je LBB1_26
+; SSE-NEXT:  LBB1_25: ## %cond.store34
+; SSE-NEXT:    movlps %xmm6, (%rdi)
+; SSE-NEXT:    addq $8, %rdi
+; SSE-NEXT:    testl $8192, %eax ## imm = 0x2000
+; SSE-NEXT:    je LBB1_28
+; SSE-NEXT:  LBB1_27: ## %cond.store37
+; SSE-NEXT:    movhps %xmm6, (%rdi)
+; SSE-NEXT:    addq $8, %rdi
+; SSE-NEXT:    testl $16384, %eax ## imm = 0x4000
+; SSE-NEXT:    je LBB1_30
+; SSE-NEXT:  LBB1_29: ## %cond.store40
+; SSE-NEXT:    movlps %xmm7, (%rdi)
+; SSE-NEXT:    addq $8, %rdi
+; SSE-NEXT:    testl $32768, %eax ## imm = 0x8000
+; SSE-NEXT:    je LBB1_32
+; SSE-NEXT:  LBB1_31: ## %cond.store43
+; SSE-NEXT:    movhps %xmm7, (%rdi)
+; SSE-NEXT:    retq
 ;
 ; AVX1OR2-LABEL: compressstore_v16f64_v16i1:
 ; AVX1OR2:       ## %bb.0:
-; AVX1OR2-NEXT:    vpextrb $0, %xmm4, %eax
+; AVX1OR2-NEXT:    vpsllw $7, %xmm4, %xmm4
+; AVX1OR2-NEXT:    vpmovmskb %xmm4, %eax
 ; AVX1OR2-NEXT:    testb $1, %al
 ; AVX1OR2-NEXT:    je LBB1_2
 ; AVX1OR2-NEXT:  ## %bb.1: ## %cond.store
 ; AVX1OR2-NEXT:    vmovlps %xmm0, (%rdi)
 ; AVX1OR2-NEXT:    addq $8, %rdi
 ; AVX1OR2-NEXT:  LBB1_2: ## %else
-; AVX1OR2-NEXT:    vpextrb $1, %xmm4, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $2, %al
 ; AVX1OR2-NEXT:    je LBB1_4
 ; AVX1OR2-NEXT:  ## %bb.3: ## %cond.store1
 ; AVX1OR2-NEXT:    vmovhps %xmm0, (%rdi)
 ; AVX1OR2-NEXT:    addq $8, %rdi
 ; AVX1OR2-NEXT:  LBB1_4: ## %else2
-; AVX1OR2-NEXT:    vpextrb $2, %xmm4, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $4, %al
 ; AVX1OR2-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX1OR2-NEXT:    je LBB1_6
-; AVX1OR2-NEXT:  ## %bb.5: ## %cond.store4
+; AVX1OR2-NEXT:    jne LBB1_5
+; AVX1OR2-NEXT:  ## %bb.6: ## %else5
+; AVX1OR2-NEXT:    testb $8, %al
+; AVX1OR2-NEXT:    jne LBB1_7
+; AVX1OR2-NEXT:  LBB1_8: ## %else8
+; AVX1OR2-NEXT:    testb $16, %al
+; AVX1OR2-NEXT:    jne LBB1_9
+; AVX1OR2-NEXT:  LBB1_10: ## %else11
+; AVX1OR2-NEXT:    testb $32, %al
+; AVX1OR2-NEXT:    je LBB1_12
+; AVX1OR2-NEXT:  LBB1_11: ## %cond.store13
+; AVX1OR2-NEXT:    vmovhps %xmm1, (%rdi)
+; AVX1OR2-NEXT:    addq $8, %rdi
+; AVX1OR2-NEXT:  LBB1_12: ## %else14
+; AVX1OR2-NEXT:    testb $64, %al
+; AVX1OR2-NEXT:    vextractf128 $1, %ymm1, %xmm0
+; AVX1OR2-NEXT:    jne LBB1_13
+; AVX1OR2-NEXT:  ## %bb.14: ## %else17
+; AVX1OR2-NEXT:    testb $-128, %al
+; AVX1OR2-NEXT:    jne LBB1_15
+; AVX1OR2-NEXT:  LBB1_16: ## %else20
+; AVX1OR2-NEXT:    testl $256, %eax ## imm = 0x100
+; AVX1OR2-NEXT:    jne LBB1_17
+; AVX1OR2-NEXT:  LBB1_18: ## %else23
+; AVX1OR2-NEXT:    testl $512, %eax ## imm = 0x200
+; AVX1OR2-NEXT:    je LBB1_20
+; AVX1OR2-NEXT:  LBB1_19: ## %cond.store25
+; AVX1OR2-NEXT:    vmovhps %xmm2, (%rdi)
+; AVX1OR2-NEXT:    addq $8, %rdi
+; AVX1OR2-NEXT:  LBB1_20: ## %else26
+; AVX1OR2-NEXT:    testl $1024, %eax ## imm = 0x400
+; AVX1OR2-NEXT:    vextractf128 $1, %ymm2, %xmm0
+; AVX1OR2-NEXT:    jne LBB1_21
+; AVX1OR2-NEXT:  ## %bb.22: ## %else29
+; AVX1OR2-NEXT:    testl $2048, %eax ## imm = 0x800
+; AVX1OR2-NEXT:    jne LBB1_23
+; AVX1OR2-NEXT:  LBB1_24: ## %else32
+; AVX1OR2-NEXT:    testl $4096, %eax ## imm = 0x1000
+; AVX1OR2-NEXT:    jne LBB1_25
+; AVX1OR2-NEXT:  LBB1_26: ## %else35
+; AVX1OR2-NEXT:    testl $8192, %eax ## imm = 0x2000
+; AVX1OR2-NEXT:    je LBB1_28
+; AVX1OR2-NEXT:  LBB1_27: ## %cond.store37
+; AVX1OR2-NEXT:    vmovhps %xmm3, (%rdi)
+; AVX1OR2-NEXT:    addq $8, %rdi
+; AVX1OR2-NEXT:  LBB1_28: ## %else38
+; AVX1OR2-NEXT:    testl $16384, %eax ## imm = 0x4000
+; AVX1OR2-NEXT:    vextractf128 $1, %ymm3, %xmm0
+; AVX1OR2-NEXT:    jne LBB1_29
+; AVX1OR2-NEXT:  ## %bb.30: ## %else41
+; AVX1OR2-NEXT:    testl $32768, %eax ## imm = 0x8000
+; AVX1OR2-NEXT:    jne LBB1_31
+; AVX1OR2-NEXT:  LBB1_32: ## %else44
+; AVX1OR2-NEXT:    vzeroupper
+; AVX1OR2-NEXT:    retq
+; AVX1OR2-NEXT:  LBB1_5: ## %cond.store4
 ; AVX1OR2-NEXT:    vmovlps %xmm0, (%rdi)
 ; AVX1OR2-NEXT:    addq $8, %rdi
-; AVX1OR2-NEXT:  LBB1_6: ## %else5
-; AVX1OR2-NEXT:    vpextrb $3, %xmm4, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $8, %al
 ; AVX1OR2-NEXT:    je LBB1_8
-; AVX1OR2-NEXT:  ## %bb.7: ## %cond.store7
+; AVX1OR2-NEXT:  LBB1_7: ## %cond.store7
 ; AVX1OR2-NEXT:    vmovhps %xmm0, (%rdi)
 ; AVX1OR2-NEXT:    addq $8, %rdi
-; AVX1OR2-NEXT:  LBB1_8: ## %else8
-; AVX1OR2-NEXT:    vpextrb $4, %xmm4, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $16, %al
 ; AVX1OR2-NEXT:    je LBB1_10
-; AVX1OR2-NEXT:  ## %bb.9: ## %cond.store10
+; AVX1OR2-NEXT:  LBB1_9: ## %cond.store10
 ; AVX1OR2-NEXT:    vmovlps %xmm1, (%rdi)
 ; AVX1OR2-NEXT:    addq $8, %rdi
-; AVX1OR2-NEXT:  LBB1_10: ## %else11
-; AVX1OR2-NEXT:    vpextrb $5, %xmm4, %eax
-; AVX1OR2-NEXT:    testb $1, %al
-; AVX1OR2-NEXT:    je LBB1_12
-; AVX1OR2-NEXT:  ## %bb.11: ## %cond.store13
-; AVX1OR2-NEXT:    vmovhps %xmm1, (%rdi)
-; AVX1OR2-NEXT:    addq $8, %rdi
-; AVX1OR2-NEXT:  LBB1_12: ## %else14
-; AVX1OR2-NEXT:    vpextrb $6, %xmm4, %eax
-; AVX1OR2-NEXT:    testb $1, %al
-; AVX1OR2-NEXT:    vextractf128 $1, %ymm1, %xmm0
-; AVX1OR2-NEXT:    je LBB1_14
-; AVX1OR2-NEXT:  ## %bb.13: ## %cond.store16
+; AVX1OR2-NEXT:    testb $32, %al
+; AVX1OR2-NEXT:    jne LBB1_11
+; AVX1OR2-NEXT:    jmp LBB1_12
+; AVX1OR2-NEXT:  LBB1_13: ## %cond.store16
 ; AVX1OR2-NEXT:    vmovlps %xmm0, (%rdi)
 ; AVX1OR2-NEXT:    addq $8, %rdi
-; AVX1OR2-NEXT:  LBB1_14: ## %else17
-; AVX1OR2-NEXT:    vpextrb $7, %xmm4, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $-128, %al
 ; AVX1OR2-NEXT:    je LBB1_16
-; AVX1OR2-NEXT:  ## %bb.15: ## %cond.store19
+; AVX1OR2-NEXT:  LBB1_15: ## %cond.store19
 ; AVX1OR2-NEXT:    vmovhps %xmm0, (%rdi)
 ; AVX1OR2-NEXT:    addq $8, %rdi
-; AVX1OR2-NEXT:  LBB1_16: ## %else20
-; AVX1OR2-NEXT:    vpextrb $8, %xmm4, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testl $256, %eax ## imm = 0x100
 ; AVX1OR2-NEXT:    je LBB1_18
-; AVX1OR2-NEXT:  ## %bb.17: ## %cond.store22
+; AVX1OR2-NEXT:  LBB1_17: ## %cond.store22
 ; AVX1OR2-NEXT:    vmovlps %xmm2, (%rdi)
 ; AVX1OR2-NEXT:    addq $8, %rdi
-; AVX1OR2-NEXT:  LBB1_18: ## %else23
-; AVX1OR2-NEXT:    vpextrb $9, %xmm4, %eax
-; AVX1OR2-NEXT:    testb $1, %al
-; AVX1OR2-NEXT:    je LBB1_20
-; AVX1OR2-NEXT:  ## %bb.19: ## %cond.store25
-; AVX1OR2-NEXT:    vmovhps %xmm2, (%rdi)
-; AVX1OR2-NEXT:    addq $8, %rdi
-; AVX1OR2-NEXT:  LBB1_20: ## %else26
-; AVX1OR2-NEXT:    vpextrb $10, %xmm4, %eax
-; AVX1OR2-NEXT:    testb $1, %al
-; AVX1OR2-NEXT:    vextractf128 $1, %ymm2, %xmm0
-; AVX1OR2-NEXT:    je LBB1_22
-; AVX1OR2-NEXT:  ## %bb.21: ## %cond.store28
+; AVX1OR2-NEXT:    testl $512, %eax ## imm = 0x200
+; AVX1OR2-NEXT:    jne LBB1_19
+; AVX1OR2-NEXT:    jmp LBB1_20
+; AVX1OR2-NEXT:  LBB1_21: ## %cond.store28
 ; AVX1OR2-NEXT:    vmovlps %xmm0, (%rdi)
 ; AVX1OR2-NEXT:    addq $8, %rdi
-; AVX1OR2-NEXT:  LBB1_22: ## %else29
-; AVX1OR2-NEXT:    vpextrb $11, %xmm4, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testl $2048, %eax ## imm = 0x800
 ; AVX1OR2-NEXT:    je LBB1_24
-; AVX1OR2-NEXT:  ## %bb.23: ## %cond.store31
+; AVX1OR2-NEXT:  LBB1_23: ## %cond.store31
 ; AVX1OR2-NEXT:    vmovhps %xmm0, (%rdi)
 ; AVX1OR2-NEXT:    addq $8, %rdi
-; AVX1OR2-NEXT:  LBB1_24: ## %else32
-; AVX1OR2-NEXT:    vpextrb $12, %xmm4, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testl $4096, %eax ## imm = 0x1000
 ; AVX1OR2-NEXT:    je LBB1_26
-; AVX1OR2-NEXT:  ## %bb.25: ## %cond.store34
+; AVX1OR2-NEXT:  LBB1_25: ## %cond.store34
 ; AVX1OR2-NEXT:    vmovlps %xmm3, (%rdi)
 ; AVX1OR2-NEXT:    addq $8, %rdi
-; AVX1OR2-NEXT:  LBB1_26: ## %else35
-; AVX1OR2-NEXT:    vpextrb $13, %xmm4, %eax
-; AVX1OR2-NEXT:    testb $1, %al
-; AVX1OR2-NEXT:    je LBB1_28
-; AVX1OR2-NEXT:  ## %bb.27: ## %cond.store37
-; AVX1OR2-NEXT:    vmovhps %xmm3, (%rdi)
-; AVX1OR2-NEXT:    addq $8, %rdi
-; AVX1OR2-NEXT:  LBB1_28: ## %else38
-; AVX1OR2-NEXT:    vpextrb $14, %xmm4, %eax
-; AVX1OR2-NEXT:    testb $1, %al
-; AVX1OR2-NEXT:    vextractf128 $1, %ymm3, %xmm0
-; AVX1OR2-NEXT:    je LBB1_30
-; AVX1OR2-NEXT:  ## %bb.29: ## %cond.store40
+; AVX1OR2-NEXT:    testl $8192, %eax ## imm = 0x2000
+; AVX1OR2-NEXT:    jne LBB1_27
+; AVX1OR2-NEXT:    jmp LBB1_28
+; AVX1OR2-NEXT:  LBB1_29: ## %cond.store40
 ; AVX1OR2-NEXT:    vmovlps %xmm0, (%rdi)
 ; AVX1OR2-NEXT:    addq $8, %rdi
-; AVX1OR2-NEXT:  LBB1_30: ## %else41
-; AVX1OR2-NEXT:    vpextrb $15, %xmm4, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testl $32768, %eax ## imm = 0x8000
 ; AVX1OR2-NEXT:    je LBB1_32
-; AVX1OR2-NEXT:  ## %bb.31: ## %cond.store43
+; AVX1OR2-NEXT:  LBB1_31: ## %cond.store43
 ; AVX1OR2-NEXT:    vmovhps %xmm0, (%rdi)
-; AVX1OR2-NEXT:  LBB1_32: ## %else44
 ; AVX1OR2-NEXT:    vzeroupper
 ; AVX1OR2-NEXT:    retq
 ;
@@ -669,106 +603,97 @@ define void @compressstore_v16f64_v16i1(double* %base, <16 x double> %V, <16 x i
 define void @compressstore_v2f32_v2i32(float* %base, <2 x float> %V, <2 x i32> %trigger) {
 ; SSE2-LABEL: compressstore_v2f32_v2i32:
 ; SSE2:       ## %bb.0:
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
 ; SSE2-NEXT:    pcmpeqd %xmm1, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,0,3,2]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    movd %xmm1, %eax
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,0,1,1]
+; SSE2-NEXT:    movmskpd %xmm1, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB2_2
-; SSE2-NEXT:  ## %bb.1: ## %cond.store
+; SSE2-NEXT:    jne LBB2_1
+; SSE2-NEXT:  ## %bb.2: ## %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne LBB2_3
+; SSE2-NEXT:  LBB2_4: ## %else2
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  LBB2_1: ## %cond.store
 ; SSE2-NEXT:    movss %xmm0, (%rdi)
 ; SSE2-NEXT:    addq $4, %rdi
-; SSE2-NEXT:  LBB2_2: ## %else
-; SSE2-NEXT:    pextrw $4, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $2, %al
 ; SSE2-NEXT:    je LBB2_4
-; SSE2-NEXT:  ## %bb.3: ## %cond.store1
+; SSE2-NEXT:  LBB2_3: ## %cond.store1
 ; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
 ; SSE2-NEXT:    movss %xmm0, (%rdi)
-; SSE2-NEXT:  LBB2_4: ## %else2
 ; SSE2-NEXT:    retq
 ;
 ; SSE42-LABEL: compressstore_v2f32_v2i32:
 ; SSE42:       ## %bb.0:
 ; SSE42-NEXT:    pxor %xmm2, %xmm2
-; SSE42-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
-; SSE42-NEXT:    pcmpeqq %xmm2, %xmm1
-; SSE42-NEXT:    pextrb $0, %xmm1, %eax
+; SSE42-NEXT:    pcmpeqd %xmm1, %xmm2
+; SSE42-NEXT:    pmovsxdq %xmm2, %xmm1
+; SSE42-NEXT:    movmskpd %xmm1, %eax
 ; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB2_2
-; SSE42-NEXT:  ## %bb.1: ## %cond.store
+; SSE42-NEXT:    jne LBB2_1
+; SSE42-NEXT:  ## %bb.2: ## %else
+; SSE42-NEXT:    testb $2, %al
+; SSE42-NEXT:    jne LBB2_3
+; SSE42-NEXT:  LBB2_4: ## %else2
+; SSE42-NEXT:    retq
+; SSE42-NEXT:  LBB2_1: ## %cond.store
 ; SSE42-NEXT:    movss %xmm0, (%rdi)
 ; SSE42-NEXT:    addq $4, %rdi
-; SSE42-NEXT:  LBB2_2: ## %else
-; SSE42-NEXT:    pextrb $8, %xmm1, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $2, %al
 ; SSE42-NEXT:    je LBB2_4
-; SSE42-NEXT:  ## %bb.3: ## %cond.store1
+; SSE42-NEXT:  LBB2_3: ## %cond.store1
 ; SSE42-NEXT:    extractps $1, %xmm0, (%rdi)
-; SSE42-NEXT:  LBB2_4: ## %else2
 ; SSE42-NEXT:    retq
 ;
-; AVX1-LABEL: compressstore_v2f32_v2i32:
-; AVX1:       ## %bb.0:
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
-; AVX1-NEXT:    vpcmpeqq %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $0, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je LBB2_2
-; AVX1-NEXT:  ## %bb.1: ## %cond.store
-; AVX1-NEXT:    vmovss %xmm0, (%rdi)
-; AVX1-NEXT:    addq $4, %rdi
-; AVX1-NEXT:  LBB2_2: ## %else
-; AVX1-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je LBB2_4
-; AVX1-NEXT:  ## %bb.3: ## %cond.store1
-; AVX1-NEXT:    vextractps $1, %xmm0, (%rdi)
-; AVX1-NEXT:  LBB2_4: ## %else2
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: compressstore_v2f32_v2i32:
-; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; AVX2-NEXT:    vpcmpeqq %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpextrb $0, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je LBB2_2
-; AVX2-NEXT:  ## %bb.1: ## %cond.store
-; AVX2-NEXT:    vmovss %xmm0, (%rdi)
-; AVX2-NEXT:    addq $4, %rdi
-; AVX2-NEXT:  LBB2_2: ## %else
-; AVX2-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je LBB2_4
-; AVX2-NEXT:  ## %bb.3: ## %cond.store1
-; AVX2-NEXT:    vextractps $1, %xmm0, (%rdi)
-; AVX2-NEXT:  LBB2_4: ## %else2
-; AVX2-NEXT:    retq
+; AVX1OR2-LABEL: compressstore_v2f32_v2i32:
+; AVX1OR2:       ## %bb.0:
+; AVX1OR2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX1OR2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
+; AVX1OR2-NEXT:    vpmovsxdq %xmm1, %xmm1
+; AVX1OR2-NEXT:    vmovmskpd %xmm1, %eax
+; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    jne LBB2_1
+; AVX1OR2-NEXT:  ## %bb.2: ## %else
+; AVX1OR2-NEXT:    testb $2, %al
+; AVX1OR2-NEXT:    jne LBB2_3
+; AVX1OR2-NEXT:  LBB2_4: ## %else2
+; AVX1OR2-NEXT:    retq
+; AVX1OR2-NEXT:  LBB2_1: ## %cond.store
+; AVX1OR2-NEXT:    vmovss %xmm0, (%rdi)
+; AVX1OR2-NEXT:    addq $4, %rdi
+; AVX1OR2-NEXT:    testb $2, %al
+; AVX1OR2-NEXT:    je LBB2_4
+; AVX1OR2-NEXT:  LBB2_3: ## %cond.store1
+; AVX1OR2-NEXT:    vextractps $1, %xmm0, (%rdi)
+; AVX1OR2-NEXT:    retq
 ;
 ; AVX512F-LABEL: compressstore_v2f32_v2i32:
 ; AVX512F:       ## %bb.0:
+; AVX512F-NEXT:    ## kill: def $xmm1 killed $xmm1 def $zmm1
 ; AVX512F-NEXT:    ## kill: def $xmm0 killed $xmm0 def $zmm0
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; AVX512F-NEXT:    vptestnmq %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    vptestnmd %zmm1, %zmm1, %k0
 ; AVX512F-NEXT:    kshiftlw $14, %k0, %k0
 ; AVX512F-NEXT:    kshiftrw $14, %k0, %k1
 ; AVX512F-NEXT:    vcompressps %zmm0, (%rdi) {%k1}
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
-; AVX512VL-LABEL: compressstore_v2f32_v2i32:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VL-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; AVX512VL-NEXT:    vptestnmq %xmm1, %xmm1, %k1
-; AVX512VL-NEXT:    vcompressps %xmm0, (%rdi) {%k1}
-; AVX512VL-NEXT:    retq
+; AVX512VLDQ-LABEL: compressstore_v2f32_v2i32:
+; AVX512VLDQ:       ## %bb.0:
+; AVX512VLDQ-NEXT:    vptestnmd %xmm1, %xmm1, %k0
+; AVX512VLDQ-NEXT:    kshiftlb $6, %k0, %k0
+; AVX512VLDQ-NEXT:    kshiftrb $6, %k0, %k1
+; AVX512VLDQ-NEXT:    vcompressps %xmm0, (%rdi) {%k1}
+; AVX512VLDQ-NEXT:    retq
+;
+; AVX512VLBW-LABEL: compressstore_v2f32_v2i32:
+; AVX512VLBW:       ## %bb.0:
+; AVX512VLBW-NEXT:    vptestnmd %xmm1, %xmm1, %k0
+; AVX512VLBW-NEXT:    kshiftlw $14, %k0, %k0
+; AVX512VLBW-NEXT:    kshiftrw $14, %k0, %k1
+; AVX512VLBW-NEXT:    vcompressps %xmm0, (%rdi) {%k1}
+; AVX512VLBW-NEXT:    retq
   %mask = icmp eq <2 x i32> %trigger, zeroinitializer
   call void @llvm.masked.compressstore.v2f32(<2 x float> %V, float* %base, <2 x i1> %mask)
   ret void
@@ -777,100 +702,115 @@ define void @compressstore_v2f32_v2i32(float* %base, <2 x float> %V, <2 x i32> %
 define void @compressstore_v4f32_v4i1(float* %base, <4 x float> %V, <4 x i1> %mask) {
 ; SSE2-LABEL: compressstore_v4f32_v4i1:
 ; SSE2:       ## %bb.0:
-; SSE2-NEXT:    movd %xmm1, %eax
+; SSE2-NEXT:    pslld $31, %xmm1
+; SSE2-NEXT:    movmskps %xmm1, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB3_2
-; SSE2-NEXT:  ## %bb.1: ## %cond.store
+; SSE2-NEXT:    jne LBB3_1
+; SSE2-NEXT:  ## %bb.2: ## %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne LBB3_3
+; SSE2-NEXT:  LBB3_4: ## %else2
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne LBB3_5
+; SSE2-NEXT:  LBB3_6: ## %else5
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne LBB3_7
+; SSE2-NEXT:  LBB3_8: ## %else8
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  LBB3_1: ## %cond.store
 ; SSE2-NEXT:    movss %xmm0, (%rdi)
 ; SSE2-NEXT:    addq $4, %rdi
-; SSE2-NEXT:  LBB3_2: ## %else
-; SSE2-NEXT:    pextrw $2, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $2, %al
 ; SSE2-NEXT:    je LBB3_4
-; SSE2-NEXT:  ## %bb.3: ## %cond.store1
-; SSE2-NEXT:    movaps %xmm0, %xmm2
-; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1],xmm0[2,3]
-; SSE2-NEXT:    movss %xmm2, (%rdi)
+; SSE2-NEXT:  LBB3_3: ## %cond.store1
+; SSE2-NEXT:    movaps %xmm0, %xmm1
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[2,3]
+; SSE2-NEXT:    movss %xmm1, (%rdi)
 ; SSE2-NEXT:    addq $4, %rdi
-; SSE2-NEXT:  LBB3_4: ## %else2
-; SSE2-NEXT:    pextrw $4, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $4, %al
 ; SSE2-NEXT:    je LBB3_6
-; SSE2-NEXT:  ## %bb.5: ## %cond.store4
-; SSE2-NEXT:    movaps %xmm0, %xmm2
-; SSE2-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm0[1]
-; SSE2-NEXT:    movss %xmm2, (%rdi)
+; SSE2-NEXT:  LBB3_5: ## %cond.store4
+; SSE2-NEXT:    movaps %xmm0, %xmm1
+; SSE2-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
+; SSE2-NEXT:    movss %xmm1, (%rdi)
 ; SSE2-NEXT:    addq $4, %rdi
-; SSE2-NEXT:  LBB3_6: ## %else5
-; SSE2-NEXT:    pextrw $6, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je LBB3_8
-; SSE2-NEXT:  ## %bb.7: ## %cond.store7
+; SSE2-NEXT:  LBB3_7: ## %cond.store7
 ; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
 ; SSE2-NEXT:    movss %xmm0, (%rdi)
-; SSE2-NEXT:  LBB3_8: ## %else8
 ; SSE2-NEXT:    retq
 ;
 ; SSE42-LABEL: compressstore_v4f32_v4i1:
 ; SSE42:       ## %bb.0:
-; SSE42-NEXT:    pextrb $0, %xmm1, %eax
+; SSE42-NEXT:    pslld $31, %xmm1
+; SSE42-NEXT:    movmskps %xmm1, %eax
 ; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB3_2
-; SSE42-NEXT:  ## %bb.1: ## %cond.store
+; SSE42-NEXT:    jne LBB3_1
+; SSE42-NEXT:  ## %bb.2: ## %else
+; SSE42-NEXT:    testb $2, %al
+; SSE42-NEXT:    jne LBB3_3
+; SSE42-NEXT:  LBB3_4: ## %else2
+; SSE42-NEXT:    testb $4, %al
+; SSE42-NEXT:    jne LBB3_5
+; SSE42-NEXT:  LBB3_6: ## %else5
+; SSE42-NEXT:    testb $8, %al
+; SSE42-NEXT:    jne LBB3_7
+; SSE42-NEXT:  LBB3_8: ## %else8
+; SSE42-NEXT:    retq
+; SSE42-NEXT:  LBB3_1: ## %cond.store
 ; SSE42-NEXT:    movss %xmm0, (%rdi)
 ; SSE42-NEXT:    addq $4, %rdi
-; SSE42-NEXT:  LBB3_2: ## %else
-; SSE42-NEXT:    pextrb $4, %xmm1, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $2, %al
 ; SSE42-NEXT:    je LBB3_4
-; SSE42-NEXT:  ## %bb.3: ## %cond.store1
+; SSE42-NEXT:  LBB3_3: ## %cond.store1
 ; SSE42-NEXT:    extractps $1, %xmm0, (%rdi)
 ; SSE42-NEXT:    addq $4, %rdi
-; SSE42-NEXT:  LBB3_4: ## %else2
-; SSE42-NEXT:    pextrb $8, %xmm1, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $4, %al
 ; SSE42-NEXT:    je LBB3_6
-; SSE42-NEXT:  ## %bb.5: ## %cond.store4
+; SSE42-NEXT:  LBB3_5: ## %cond.store4
 ; SSE42-NEXT:    extractps $2, %xmm0, (%rdi)
 ; SSE42-NEXT:    addq $4, %rdi
-; SSE42-NEXT:  LBB3_6: ## %else5
-; SSE42-NEXT:    pextrb $12, %xmm1, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $8, %al
 ; SSE42-NEXT:    je LBB3_8
-; SSE42-NEXT:  ## %bb.7: ## %cond.store7
+; SSE42-NEXT:  LBB3_7: ## %cond.store7
 ; SSE42-NEXT:    extractps $3, %xmm0, (%rdi)
-; SSE42-NEXT:  LBB3_8: ## %else8
 ; SSE42-NEXT:    retq
 ;
 ; AVX1OR2-LABEL: compressstore_v4f32_v4i1:
 ; AVX1OR2:       ## %bb.0:
-; AVX1OR2-NEXT:    vpextrb $0, %xmm1, %eax
+; AVX1OR2-NEXT:    vpslld $31, %xmm1, %xmm1
+; AVX1OR2-NEXT:    vmovmskps %xmm1, %eax
 ; AVX1OR2-NEXT:    testb $1, %al
-; AVX1OR2-NEXT:    je LBB3_2
-; AVX1OR2-NEXT:  ## %bb.1: ## %cond.store
+; AVX1OR2-NEXT:    jne LBB3_1
+; AVX1OR2-NEXT:  ## %bb.2: ## %else
+; AVX1OR2-NEXT:    testb $2, %al
+; AVX1OR2-NEXT:    jne LBB3_3
+; AVX1OR2-NEXT:  LBB3_4: ## %else2
+; AVX1OR2-NEXT:    testb $4, %al
+; AVX1OR2-NEXT:    jne LBB3_5
+; AVX1OR2-NEXT:  LBB3_6: ## %else5
+; AVX1OR2-NEXT:    testb $8, %al
+; AVX1OR2-NEXT:    jne LBB3_7
+; AVX1OR2-NEXT:  LBB3_8: ## %else8
+; AVX1OR2-NEXT:    retq
+; AVX1OR2-NEXT:  LBB3_1: ## %cond.store
 ; AVX1OR2-NEXT:    vmovss %xmm0, (%rdi)
 ; AVX1OR2-NEXT:    addq $4, %rdi
-; AVX1OR2-NEXT:  LBB3_2: ## %else
-; AVX1OR2-NEXT:    vpextrb $4, %xmm1, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $2, %al
 ; AVX1OR2-NEXT:    je LBB3_4
-; AVX1OR2-NEXT:  ## %bb.3: ## %cond.store1
+; AVX1OR2-NEXT:  LBB3_3: ## %cond.store1
 ; AVX1OR2-NEXT:    vextractps $1, %xmm0, (%rdi)
 ; AVX1OR2-NEXT:    addq $4, %rdi
-; AVX1OR2-NEXT:  LBB3_4: ## %else2
-; AVX1OR2-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $4, %al
 ; AVX1OR2-NEXT:    je LBB3_6
-; AVX1OR2-NEXT:  ## %bb.5: ## %cond.store4
+; AVX1OR2-NEXT:  LBB3_5: ## %cond.store4
 ; AVX1OR2-NEXT:    vextractps $2, %xmm0, (%rdi)
 ; AVX1OR2-NEXT:    addq $4, %rdi
-; AVX1OR2-NEXT:  LBB3_6: ## %else5
-; AVX1OR2-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $8, %al
 ; AVX1OR2-NEXT:    je LBB3_8
-; AVX1OR2-NEXT:  ## %bb.7: ## %cond.store7
+; AVX1OR2-NEXT:  LBB3_7: ## %cond.store7
 ; AVX1OR2-NEXT:    vextractps $3, %xmm0, (%rdi)
-; AVX1OR2-NEXT:  LBB3_8: ## %else8
 ; AVX1OR2-NEXT:    retq
 ;
 ; AVX512F-LABEL: compressstore_v4f32_v4i1:
@@ -904,192 +844,293 @@ define void @compressstore_v4f32_v4i1(float* %base, <4 x float> %V, <4 x i1> %ma
 define void @compressstore_v8f32_v8i1(float* %base, <8 x float> %V, <8 x i1> %mask) {
 ; SSE2-LABEL: compressstore_v8f32_v8i1:
 ; SSE2:       ## %bb.0:
-; SSE2-NEXT:    movd %xmm2, %eax
+; SSE2-NEXT:    psllw $15, %xmm2
+; SSE2-NEXT:    packsswb %xmm0, %xmm2
+; SSE2-NEXT:    pmovmskb %xmm2, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB4_2
-; SSE2-NEXT:  ## %bb.1: ## %cond.store
-; SSE2-NEXT:    movss %xmm0, (%rdi)
+; SSE2-NEXT:    jne LBB4_1
+; SSE2-NEXT:  ## %bb.2: ## %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne LBB4_3
+; SSE2-NEXT:  LBB4_4: ## %else2
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne LBB4_5
+; SSE2-NEXT:  LBB4_6: ## %else5
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne LBB4_7
+; SSE2-NEXT:  LBB4_8: ## %else8
+; SSE2-NEXT:    testb $16, %al
+; SSE2-NEXT:    jne LBB4_9
+; SSE2-NEXT:  LBB4_10: ## %else11
+; SSE2-NEXT:    testb $32, %al
+; SSE2-NEXT:    jne LBB4_11
+; SSE2-NEXT:  LBB4_12: ## %else14
+; SSE2-NEXT:    testb $64, %al
+; SSE2-NEXT:    jne LBB4_13
+; SSE2-NEXT:  LBB4_14: ## %else17
+; SSE2-NEXT:    testb $-128, %al
+; SSE2-NEXT:    jne LBB4_15
+; SSE2-NEXT:  LBB4_16: ## %else20
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  LBB4_1: ## %cond.store
+; SSE2-NEXT:    movd %xmm0, (%rdi)
 ; SSE2-NEXT:    addq $4, %rdi
-; SSE2-NEXT:  LBB4_2: ## %else
-; SSE2-NEXT:    shrl $16, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $2, %al
 ; SSE2-NEXT:    je LBB4_4
-; SSE2-NEXT:  ## %bb.3: ## %cond.store1
-; SSE2-NEXT:    movaps %xmm0, %xmm3
-; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm0[2,3]
-; SSE2-NEXT:    movss %xmm3, (%rdi)
+; SSE2-NEXT:  LBB4_3: ## %cond.store1
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1],xmm0[2,3]
+; SSE2-NEXT:    movss %xmm2, (%rdi)
 ; SSE2-NEXT:    addq $4, %rdi
-; SSE2-NEXT:  LBB4_4: ## %else2
-; SSE2-NEXT:    pextrw $2, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $4, %al
 ; SSE2-NEXT:    je LBB4_6
-; SSE2-NEXT:  ## %bb.5: ## %cond.store4
-; SSE2-NEXT:    movaps %xmm0, %xmm3
-; SSE2-NEXT:    unpckhpd {{.*#+}} xmm3 = xmm3[1],xmm0[1]
-; SSE2-NEXT:    movss %xmm3, (%rdi)
+; SSE2-NEXT:  LBB4_5: ## %cond.store4
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    punpckhqdq {{.*#+}} xmm2 = xmm2[1],xmm0[1]
+; SSE2-NEXT:    movd %xmm2, (%rdi)
 ; SSE2-NEXT:    addq $4, %rdi
-; SSE2-NEXT:  LBB4_6: ## %else5
-; SSE2-NEXT:    pextrw $3, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je LBB4_8
-; SSE2-NEXT:  ## %bb.7: ## %cond.store7
+; SSE2-NEXT:  LBB4_7: ## %cond.store7
 ; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
 ; SSE2-NEXT:    movss %xmm0, (%rdi)
 ; SSE2-NEXT:    addq $4, %rdi
-; SSE2-NEXT:  LBB4_8: ## %else8
-; SSE2-NEXT:    pextrw $4, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $16, %al
 ; SSE2-NEXT:    je LBB4_10
-; SSE2-NEXT:  ## %bb.9: ## %cond.store10
+; SSE2-NEXT:  LBB4_9: ## %cond.store10
 ; SSE2-NEXT:    movss %xmm1, (%rdi)
 ; SSE2-NEXT:    addq $4, %rdi
-; SSE2-NEXT:  LBB4_10: ## %else11
-; SSE2-NEXT:    pextrw $5, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $32, %al
 ; SSE2-NEXT:    je LBB4_12
-; SSE2-NEXT:  ## %bb.11: ## %cond.store13
+; SSE2-NEXT:  LBB4_11: ## %cond.store13
 ; SSE2-NEXT:    movaps %xmm1, %xmm0
 ; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm1[2,3]
 ; SSE2-NEXT:    movss %xmm0, (%rdi)
 ; SSE2-NEXT:    addq $4, %rdi
-; SSE2-NEXT:  LBB4_12: ## %else14
-; SSE2-NEXT:    pextrw $6, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $64, %al
 ; SSE2-NEXT:    je LBB4_14
-; SSE2-NEXT:  ## %bb.13: ## %cond.store16
+; SSE2-NEXT:  LBB4_13: ## %cond.store16
 ; SSE2-NEXT:    movaps %xmm1, %xmm0
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1]
 ; SSE2-NEXT:    movss %xmm0, (%rdi)
 ; SSE2-NEXT:    addq $4, %rdi
-; SSE2-NEXT:  LBB4_14: ## %else17
-; SSE2-NEXT:    pextrw $7, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $-128, %al
 ; SSE2-NEXT:    je LBB4_16
-; SSE2-NEXT:  ## %bb.15: ## %cond.store19
+; SSE2-NEXT:  LBB4_15: ## %cond.store19
 ; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,1,2,3]
 ; SSE2-NEXT:    movss %xmm1, (%rdi)
-; SSE2-NEXT:  LBB4_16: ## %else20
 ; SSE2-NEXT:    retq
 ;
 ; SSE42-LABEL: compressstore_v8f32_v8i1:
 ; SSE42:       ## %bb.0:
-; SSE42-NEXT:    pextrb $0, %xmm2, %eax
+; SSE42-NEXT:    psllw $15, %xmm2
+; SSE42-NEXT:    packsswb %xmm0, %xmm2
+; SSE42-NEXT:    pmovmskb %xmm2, %eax
 ; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB4_2
-; SSE42-NEXT:  ## %bb.1: ## %cond.store
-; SSE42-NEXT:    movss %xmm0, (%rdi)
+; SSE42-NEXT:    jne LBB4_1
+; SSE42-NEXT:  ## %bb.2: ## %else
+; SSE42-NEXT:    testb $2, %al
+; SSE42-NEXT:    jne LBB4_3
+; SSE42-NEXT:  LBB4_4: ## %else2
+; SSE42-NEXT:    testb $4, %al
+; SSE42-NEXT:    jne LBB4_5
+; SSE42-NEXT:  LBB4_6: ## %else5
+; SSE42-NEXT:    testb $8, %al
+; SSE42-NEXT:    jne LBB4_7
+; SSE42-NEXT:  LBB4_8: ## %else8
+; SSE42-NEXT:    testb $16, %al
+; SSE42-NEXT:    jne LBB4_9
+; SSE42-NEXT:  LBB4_10: ## %else11
+; SSE42-NEXT:    testb $32, %al
+; SSE42-NEXT:    jne LBB4_11
+; SSE42-NEXT:  LBB4_12: ## %else14
+; SSE42-NEXT:    testb $64, %al
+; SSE42-NEXT:    jne LBB4_13
+; SSE42-NEXT:  LBB4_14: ## %else17
+; SSE42-NEXT:    testb $-128, %al
+; SSE42-NEXT:    jne LBB4_15
+; SSE42-NEXT:  LBB4_16: ## %else20
+; SSE42-NEXT:    retq
+; SSE42-NEXT:  LBB4_1: ## %cond.store
+; SSE42-NEXT:    movd %xmm0, (%rdi)
 ; SSE42-NEXT:    addq $4, %rdi
-; SSE42-NEXT:  LBB4_2: ## %else
-; SSE42-NEXT:    pextrb $2, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $2, %al
 ; SSE42-NEXT:    je LBB4_4
-; SSE42-NEXT:  ## %bb.3: ## %cond.store1
-; SSE42-NEXT:    extractps $1, %xmm0, (%rdi)
+; SSE42-NEXT:  LBB4_3: ## %cond.store1
+; SSE42-NEXT:    pextrd $1, %xmm0, (%rdi)
 ; SSE42-NEXT:    addq $4, %rdi
-; SSE42-NEXT:  LBB4_4: ## %else2
-; SSE42-NEXT:    pextrb $4, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $4, %al
 ; SSE42-NEXT:    je LBB4_6
-; SSE42-NEXT:  ## %bb.5: ## %cond.store4
-; SSE42-NEXT:    extractps $2, %xmm0, (%rdi)
+; SSE42-NEXT:  LBB4_5: ## %cond.store4
+; SSE42-NEXT:    pextrd $2, %xmm0, (%rdi)
 ; SSE42-NEXT:    addq $4, %rdi
-; SSE42-NEXT:  LBB4_6: ## %else5
-; SSE42-NEXT:    pextrb $6, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $8, %al
 ; SSE42-NEXT:    je LBB4_8
-; SSE42-NEXT:  ## %bb.7: ## %cond.store7
-; SSE42-NEXT:    extractps $3, %xmm0, (%rdi)
+; SSE42-NEXT:  LBB4_7: ## %cond.store7
+; SSE42-NEXT:    pextrd $3, %xmm0, (%rdi)
 ; SSE42-NEXT:    addq $4, %rdi
-; SSE42-NEXT:  LBB4_8: ## %else8
-; SSE42-NEXT:    pextrb $8, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $16, %al
 ; SSE42-NEXT:    je LBB4_10
-; SSE42-NEXT:  ## %bb.9: ## %cond.store10
+; SSE42-NEXT:  LBB4_9: ## %cond.store10
 ; SSE42-NEXT:    movss %xmm1, (%rdi)
 ; SSE42-NEXT:    addq $4, %rdi
-; SSE42-NEXT:  LBB4_10: ## %else11
-; SSE42-NEXT:    pextrb $10, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $32, %al
 ; SSE42-NEXT:    je LBB4_12
-; SSE42-NEXT:  ## %bb.11: ## %cond.store13
+; SSE42-NEXT:  LBB4_11: ## %cond.store13
 ; SSE42-NEXT:    extractps $1, %xmm1, (%rdi)
 ; SSE42-NEXT:    addq $4, %rdi
-; SSE42-NEXT:  LBB4_12: ## %else14
-; SSE42-NEXT:    pextrb $12, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $64, %al
 ; SSE42-NEXT:    je LBB4_14
-; SSE42-NEXT:  ## %bb.13: ## %cond.store16
+; SSE42-NEXT:  LBB4_13: ## %cond.store16
 ; SSE42-NEXT:    extractps $2, %xmm1, (%rdi)
 ; SSE42-NEXT:    addq $4, %rdi
-; SSE42-NEXT:  LBB4_14: ## %else17
-; SSE42-NEXT:    pextrb $14, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $-128, %al
 ; SSE42-NEXT:    je LBB4_16
-; SSE42-NEXT:  ## %bb.15: ## %cond.store19
+; SSE42-NEXT:  LBB4_15: ## %cond.store19
 ; SSE42-NEXT:    extractps $3, %xmm1, (%rdi)
-; SSE42-NEXT:  LBB4_16: ## %else20
 ; SSE42-NEXT:    retq
 ;
-; AVX1OR2-LABEL: compressstore_v8f32_v8i1:
-; AVX1OR2:       ## %bb.0:
-; AVX1OR2-NEXT:    vpextrb $0, %xmm1, %eax
-; AVX1OR2-NEXT:    testb $1, %al
-; AVX1OR2-NEXT:    je LBB4_2
-; AVX1OR2-NEXT:  ## %bb.1: ## %cond.store
-; AVX1OR2-NEXT:    vmovss %xmm0, (%rdi)
-; AVX1OR2-NEXT:    addq $4, %rdi
-; AVX1OR2-NEXT:  LBB4_2: ## %else
-; AVX1OR2-NEXT:    vpextrb $2, %xmm1, %eax
-; AVX1OR2-NEXT:    testb $1, %al
-; AVX1OR2-NEXT:    je LBB4_4
-; AVX1OR2-NEXT:  ## %bb.3: ## %cond.store1
-; AVX1OR2-NEXT:    vextractps $1, %xmm0, (%rdi)
-; AVX1OR2-NEXT:    addq $4, %rdi
-; AVX1OR2-NEXT:  LBB4_4: ## %else2
-; AVX1OR2-NEXT:    vpextrb $4, %xmm1, %eax
-; AVX1OR2-NEXT:    testb $1, %al
-; AVX1OR2-NEXT:    je LBB4_6
-; AVX1OR2-NEXT:  ## %bb.5: ## %cond.store4
-; AVX1OR2-NEXT:    vextractps $2, %xmm0, (%rdi)
-; AVX1OR2-NEXT:    addq $4, %rdi
-; AVX1OR2-NEXT:  LBB4_6: ## %else5
-; AVX1OR2-NEXT:    vpextrb $6, %xmm1, %eax
-; AVX1OR2-NEXT:    testb $1, %al
-; AVX1OR2-NEXT:    je LBB4_8
-; AVX1OR2-NEXT:  ## %bb.7: ## %cond.store7
-; AVX1OR2-NEXT:    vextractps $3, %xmm0, (%rdi)
-; AVX1OR2-NEXT:    addq $4, %rdi
-; AVX1OR2-NEXT:  LBB4_8: ## %else8
-; AVX1OR2-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX1OR2-NEXT:    testb $1, %al
-; AVX1OR2-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX1OR2-NEXT:    je LBB4_10
-; AVX1OR2-NEXT:  ## %bb.9: ## %cond.store10
-; AVX1OR2-NEXT:    vmovss %xmm0, (%rdi)
-; AVX1OR2-NEXT:    addq $4, %rdi
-; AVX1OR2-NEXT:  LBB4_10: ## %else11
-; AVX1OR2-NEXT:    vpextrb $10, %xmm1, %eax
-; AVX1OR2-NEXT:    testb $1, %al
-; AVX1OR2-NEXT:    je LBB4_12
-; AVX1OR2-NEXT:  ## %bb.11: ## %cond.store13
-; AVX1OR2-NEXT:    vextractps $1, %xmm0, (%rdi)
-; AVX1OR2-NEXT:    addq $4, %rdi
-; AVX1OR2-NEXT:  LBB4_12: ## %else14
-; AVX1OR2-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX1OR2-NEXT:    testb $1, %al
-; AVX1OR2-NEXT:    je LBB4_14
-; AVX1OR2-NEXT:  ## %bb.13: ## %cond.store16
-; AVX1OR2-NEXT:    vextractps $2, %xmm0, (%rdi)
-; AVX1OR2-NEXT:    addq $4, %rdi
-; AVX1OR2-NEXT:  LBB4_14: ## %else17
-; AVX1OR2-NEXT:    vpextrb $14, %xmm1, %eax
-; AVX1OR2-NEXT:    testb $1, %al
-; AVX1OR2-NEXT:    je LBB4_16
-; AVX1OR2-NEXT:  ## %bb.15: ## %cond.store19
-; AVX1OR2-NEXT:    vextractps $3, %xmm0, (%rdi)
-; AVX1OR2-NEXT:  LBB4_16: ## %else20
-; AVX1OR2-NEXT:    vzeroupper
-; AVX1OR2-NEXT:    retq
+; AVX1-LABEL: compressstore_v8f32_v8i1:
+; AVX1:       ## %bb.0:
+; AVX1-NEXT:    vpsllw $15, %xmm1, %xmm1
+; AVX1-NEXT:    vpacksswb %xmm0, %xmm1, %xmm1
+; AVX1-NEXT:    vpmovmskb %xmm1, %eax
+; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    jne LBB4_1
+; AVX1-NEXT:  ## %bb.2: ## %else
+; AVX1-NEXT:    testb $2, %al
+; AVX1-NEXT:    jne LBB4_3
+; AVX1-NEXT:  LBB4_4: ## %else2
+; AVX1-NEXT:    testb $4, %al
+; AVX1-NEXT:    jne LBB4_5
+; AVX1-NEXT:  LBB4_6: ## %else5
+; AVX1-NEXT:    testb $8, %al
+; AVX1-NEXT:    je LBB4_8
+; AVX1-NEXT:  LBB4_7: ## %cond.store7
+; AVX1-NEXT:    vpextrd $3, %xmm0, (%rdi)
+; AVX1-NEXT:    addq $4, %rdi
+; AVX1-NEXT:  LBB4_8: ## %else8
+; AVX1-NEXT:    testb $16, %al
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    jne LBB4_9
+; AVX1-NEXT:  ## %bb.10: ## %else11
+; AVX1-NEXT:    testb $32, %al
+; AVX1-NEXT:    jne LBB4_11
+; AVX1-NEXT:  LBB4_12: ## %else14
+; AVX1-NEXT:    testb $64, %al
+; AVX1-NEXT:    jne LBB4_13
+; AVX1-NEXT:  LBB4_14: ## %else17
+; AVX1-NEXT:    testb $-128, %al
+; AVX1-NEXT:    jne LBB4_15
+; AVX1-NEXT:  LBB4_16: ## %else20
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+; AVX1-NEXT:  LBB4_1: ## %cond.store
+; AVX1-NEXT:    vmovd %xmm0, (%rdi)
+; AVX1-NEXT:    addq $4, %rdi
+; AVX1-NEXT:    testb $2, %al
+; AVX1-NEXT:    je LBB4_4
+; AVX1-NEXT:  LBB4_3: ## %cond.store1
+; AVX1-NEXT:    vpextrd $1, %xmm0, (%rdi)
+; AVX1-NEXT:    addq $4, %rdi
+; AVX1-NEXT:    testb $4, %al
+; AVX1-NEXT:    je LBB4_6
+; AVX1-NEXT:  LBB4_5: ## %cond.store4
+; AVX1-NEXT:    vpextrd $2, %xmm0, (%rdi)
+; AVX1-NEXT:    addq $4, %rdi
+; AVX1-NEXT:    testb $8, %al
+; AVX1-NEXT:    jne LBB4_7
+; AVX1-NEXT:    jmp LBB4_8
+; AVX1-NEXT:  LBB4_9: ## %cond.store10
+; AVX1-NEXT:    vmovss %xmm0, (%rdi)
+; AVX1-NEXT:    addq $4, %rdi
+; AVX1-NEXT:    testb $32, %al
+; AVX1-NEXT:    je LBB4_12
+; AVX1-NEXT:  LBB4_11: ## %cond.store13
+; AVX1-NEXT:    vextractps $1, %xmm0, (%rdi)
+; AVX1-NEXT:    addq $4, %rdi
+; AVX1-NEXT:    testb $64, %al
+; AVX1-NEXT:    je LBB4_14
+; AVX1-NEXT:  LBB4_13: ## %cond.store16
+; AVX1-NEXT:    vextractps $2, %xmm0, (%rdi)
+; AVX1-NEXT:    addq $4, %rdi
+; AVX1-NEXT:    testb $-128, %al
+; AVX1-NEXT:    je LBB4_16
+; AVX1-NEXT:  LBB4_15: ## %cond.store19
+; AVX1-NEXT:    vextractps $3, %xmm0, (%rdi)
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: compressstore_v8f32_v8i1:
+; AVX2:       ## %bb.0:
+; AVX2-NEXT:    vpsllw $15, %xmm1, %xmm1
+; AVX2-NEXT:    vpacksswb %xmm0, %xmm1, %xmm1
+; AVX2-NEXT:    vpmovmskb %xmm1, %eax
+; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    jne LBB4_1
+; AVX2-NEXT:  ## %bb.2: ## %else
+; AVX2-NEXT:    testb $2, %al
+; AVX2-NEXT:    jne LBB4_3
+; AVX2-NEXT:  LBB4_4: ## %else2
+; AVX2-NEXT:    testb $4, %al
+; AVX2-NEXT:    jne LBB4_5
+; AVX2-NEXT:  LBB4_6: ## %else5
+; AVX2-NEXT:    testb $8, %al
+; AVX2-NEXT:    je LBB4_8
+; AVX2-NEXT:  LBB4_7: ## %cond.store7
+; AVX2-NEXT:    vpextrd $3, %xmm0, (%rdi)
+; AVX2-NEXT:    addq $4, %rdi
+; AVX2-NEXT:  LBB4_8: ## %else8
+; AVX2-NEXT:    testb $16, %al
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
+; AVX2-NEXT:    jne LBB4_9
+; AVX2-NEXT:  ## %bb.10: ## %else11
+; AVX2-NEXT:    testb $32, %al
+; AVX2-NEXT:    jne LBB4_11
+; AVX2-NEXT:  LBB4_12: ## %else14
+; AVX2-NEXT:    testb $64, %al
+; AVX2-NEXT:    jne LBB4_13
+; AVX2-NEXT:  LBB4_14: ## %else17
+; AVX2-NEXT:    testb $-128, %al
+; AVX2-NEXT:    jne LBB4_15
+; AVX2-NEXT:  LBB4_16: ## %else20
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+; AVX2-NEXT:  LBB4_1: ## %cond.store
+; AVX2-NEXT:    vmovd %xmm0, (%rdi)
+; AVX2-NEXT:    addq $4, %rdi
+; AVX2-NEXT:    testb $2, %al
+; AVX2-NEXT:    je LBB4_4
+; AVX2-NEXT:  LBB4_3: ## %cond.store1
+; AVX2-NEXT:    vpextrd $1, %xmm0, (%rdi)
+; AVX2-NEXT:    addq $4, %rdi
+; AVX2-NEXT:    testb $4, %al
+; AVX2-NEXT:    je LBB4_6
+; AVX2-NEXT:  LBB4_5: ## %cond.store4
+; AVX2-NEXT:    vpextrd $2, %xmm0, (%rdi)
+; AVX2-NEXT:    addq $4, %rdi
+; AVX2-NEXT:    testb $8, %al
+; AVX2-NEXT:    jne LBB4_7
+; AVX2-NEXT:    jmp LBB4_8
+; AVX2-NEXT:  LBB4_9: ## %cond.store10
+; AVX2-NEXT:    vmovd %xmm0, (%rdi)
+; AVX2-NEXT:    addq $4, %rdi
+; AVX2-NEXT:    testb $32, %al
+; AVX2-NEXT:    je LBB4_12
+; AVX2-NEXT:  LBB4_11: ## %cond.store13
+; AVX2-NEXT:    vpextrd $1, %xmm0, (%rdi)
+; AVX2-NEXT:    addq $4, %rdi
+; AVX2-NEXT:    testb $64, %al
+; AVX2-NEXT:    je LBB4_14
+; AVX2-NEXT:  LBB4_13: ## %cond.store16
+; AVX2-NEXT:    vpextrd $2, %xmm0, (%rdi)
+; AVX2-NEXT:    addq $4, %rdi
+; AVX2-NEXT:    testb $-128, %al
+; AVX2-NEXT:    je LBB4_16
+; AVX2-NEXT:  LBB4_15: ## %cond.store19
+; AVX2-NEXT:    vpextrd $3, %xmm0, (%rdi)
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: compressstore_v8f32_v8i1:
 ; AVX512F:       ## %bb.0:
@@ -1123,7 +1164,7 @@ define void @compressstore_v8f32_v8i1(float* %base, <8 x float> %V, <8 x i1> %ma
 
 define void @compressstore_v16f32_const(float* %base, <16 x float> %V) {
 ; SSE2-LABEL: compressstore_v16f32_const:
-; SSE2:       ## %bb.0: ## %cond.store
+; SSE2:       ## %bb.0:
 ; SSE2-NEXT:    movss %xmm0, (%rdi)
 ; SSE2-NEXT:    movaps %xmm0, %xmm4
 ; SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[1,1],xmm0[2,3]
@@ -1160,7 +1201,7 @@ define void @compressstore_v16f32_const(float* %base, <16 x float> %V) {
 ; SSE2-NEXT:    retq
 ;
 ; SSE42-LABEL: compressstore_v16f32_const:
-; SSE42:       ## %bb.0: ## %cond.store
+; SSE42:       ## %bb.0:
 ; SSE42-NEXT:    movups %xmm0, (%rdi)
 ; SSE42-NEXT:    movups %xmm1, 16(%rdi)
 ; SSE42-NEXT:    insertps {{.*#+}} xmm2 = xmm2[0,1,2],xmm3[0]
@@ -1171,7 +1212,7 @@ define void @compressstore_v16f32_const(float* %base, <16 x float> %V) {
 ; SSE42-NEXT:    retq
 ;
 ; AVX1-LABEL: compressstore_v16f32_const:
-; AVX1:       ## %bb.0: ## %cond.store
+; AVX1:       ## %bb.0:
 ; AVX1-NEXT:    vmovups %ymm0, (%rdi)
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm0
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm0[0]
@@ -1183,7 +1224,7 @@ define void @compressstore_v16f32_const(float* %base, <16 x float> %V) {
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: compressstore_v16f32_const:
-; AVX2:       ## %bb.0: ## %cond.store
+; AVX2:       ## %bb.0:
 ; AVX2-NEXT:    vmovups %ymm0, (%rdi)
 ; AVX2-NEXT:    vmovaps {{.*#+}} xmm0 = [0,1,2,4]
 ; AVX2-NEXT:    vpermps %ymm1, %ymm0, %ymm0
@@ -1225,1255 +1266,1164 @@ define void @compressstore_v16f32_const(float* %base, <16 x float> %V) {
 define void @compressstore_v32f32_v32i32(float* %base, <32 x float> %V, <32 x i32> %trigger) {
 ; SSE2-LABEL: compressstore_v32f32_v32i32:
 ; SSE2:       ## %bb.0:
-; SSE2-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm9
 ; SSE2-NEXT:    pxor %xmm8, %xmm8
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm8
-; SSE2-NEXT:    movd %xmm8, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB6_2
-; SSE2-NEXT:  ## %bb.1: ## %cond.store
-; SSE2-NEXT:    movss %xmm0, (%rdi)
-; SSE2-NEXT:    addq $4, %rdi
-; SSE2-NEXT:  LBB6_2: ## %else
-; SSE2-NEXT:    pextrw $2, %xmm8, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB6_4
-; SSE2-NEXT:  ## %bb.3: ## %cond.store1
-; SSE2-NEXT:    movaps %xmm0, %xmm8
-; SSE2-NEXT:    shufps {{.*#+}} xmm8 = xmm8[1,1],xmm0[2,3]
-; SSE2-NEXT:    movss %xmm8, (%rdi)
-; SSE2-NEXT:    addq $4, %rdi
-; SSE2-NEXT:  LBB6_4: ## %else2
-; SSE2-NEXT:    xorps %xmm8, %xmm8
+; SSE2-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm9
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm9
+; SSE2-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm10
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm10
+; SSE2-NEXT:    packssdw %xmm9, %xmm10
+; SSE2-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm9
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm9
+; SSE2-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm11
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm11
+; SSE2-NEXT:    packssdw %xmm9, %xmm11
+; SSE2-NEXT:    packsswb %xmm10, %xmm11
+; SSE2-NEXT:    pmovmskb %xmm11, %ecx
+; SSE2-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm9
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm9
+; SSE2-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm10
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm10
+; SSE2-NEXT:    packssdw %xmm9, %xmm10
+; SSE2-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm9
 ; SSE2-NEXT:    pcmpeqd %xmm8, %xmm9
-; SSE2-NEXT:    pextrw $4, %xmm9, %eax
+; SSE2-NEXT:    pcmpeqd {{[0-9]+}}(%rsp), %xmm8
+; SSE2-NEXT:    packssdw %xmm9, %xmm8
+; SSE2-NEXT:    packsswb %xmm10, %xmm8
+; SSE2-NEXT:    pmovmskb %xmm8, %eax
+; SSE2-NEXT:    shll $16, %eax
+; SSE2-NEXT:    orl %ecx, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB6_6
-; SSE2-NEXT:  ## %bb.5: ## %cond.store4
-; SSE2-NEXT:    movaps %xmm0, %xmm8
-; SSE2-NEXT:    unpckhpd {{.*#+}} xmm8 = xmm8[1],xmm0[1]
-; SSE2-NEXT:    movss %xmm8, (%rdi)
-; SSE2-NEXT:    addq $4, %rdi
+; SSE2-NEXT:    jne LBB6_1
+; SSE2-NEXT:  ## %bb.2: ## %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne LBB6_3
+; SSE2-NEXT:  LBB6_4: ## %else2
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne LBB6_5
 ; SSE2-NEXT:  LBB6_6: ## %else5
-; SSE2-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm8
-; SSE2-NEXT:    pextrw $6, %xmm9, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB6_8
-; SSE2-NEXT:  ## %bb.7: ## %cond.store7
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
-; SSE2-NEXT:    movss %xmm0, (%rdi)
-; SSE2-NEXT:    addq $4, %rdi
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne LBB6_7
 ; SSE2-NEXT:  LBB6_8: ## %else8
-; SSE2-NEXT:    xorps %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqd %xmm8, %xmm0
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB6_10
-; SSE2-NEXT:  ## %bb.9: ## %cond.store10
-; SSE2-NEXT:    movss %xmm1, (%rdi)
-; SSE2-NEXT:    addq $4, %rdi
+; SSE2-NEXT:    testb $16, %al
+; SSE2-NEXT:    jne LBB6_9
 ; SSE2-NEXT:  LBB6_10: ## %else11
-; SSE2-NEXT:    pextrw $2, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB6_12
-; SSE2-NEXT:  ## %bb.11: ## %cond.store13
-; SSE2-NEXT:    movaps %xmm1, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm1[2,3]
-; SSE2-NEXT:    movss %xmm0, (%rdi)
-; SSE2-NEXT:    addq $4, %rdi
+; SSE2-NEXT:    testb $32, %al
+; SSE2-NEXT:    jne LBB6_11
 ; SSE2-NEXT:  LBB6_12: ## %else14
-; SSE2-NEXT:    xorps %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm8
-; SSE2-NEXT:    pextrw $4, %xmm8, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $64, %al
+; SSE2-NEXT:    jne LBB6_13
+; SSE2-NEXT:  LBB6_14: ## %else17
+; SSE2-NEXT:    testb $-128, %al
+; SSE2-NEXT:    jne LBB6_15
+; SSE2-NEXT:  LBB6_16: ## %else20
+; SSE2-NEXT:    testl $256, %eax ## imm = 0x100
+; SSE2-NEXT:    jne LBB6_17
+; SSE2-NEXT:  LBB6_18: ## %else23
+; SSE2-NEXT:    testl $512, %eax ## imm = 0x200
+; SSE2-NEXT:    jne LBB6_19
+; SSE2-NEXT:  LBB6_20: ## %else26
+; SSE2-NEXT:    testl $1024, %eax ## imm = 0x400
+; SSE2-NEXT:    jne LBB6_21
+; SSE2-NEXT:  LBB6_22: ## %else29
+; SSE2-NEXT:    testl $2048, %eax ## imm = 0x800
+; SSE2-NEXT:    jne LBB6_23
+; SSE2-NEXT:  LBB6_24: ## %else32
+; SSE2-NEXT:    testl $4096, %eax ## imm = 0x1000
+; SSE2-NEXT:    jne LBB6_25
+; SSE2-NEXT:  LBB6_26: ## %else35
+; SSE2-NEXT:    testl $8192, %eax ## imm = 0x2000
+; SSE2-NEXT:    jne LBB6_27
+; SSE2-NEXT:  LBB6_28: ## %else38
+; SSE2-NEXT:    testl $16384, %eax ## imm = 0x4000
+; SSE2-NEXT:    jne LBB6_29
+; SSE2-NEXT:  LBB6_30: ## %else41
+; SSE2-NEXT:    testl $32768, %eax ## imm = 0x8000
+; SSE2-NEXT:    jne LBB6_31
+; SSE2-NEXT:  LBB6_32: ## %else44
+; SSE2-NEXT:    testl $65536, %eax ## imm = 0x10000
+; SSE2-NEXT:    jne LBB6_33
+; SSE2-NEXT:  LBB6_34: ## %else47
+; SSE2-NEXT:    testl $131072, %eax ## imm = 0x20000
+; SSE2-NEXT:    jne LBB6_35
+; SSE2-NEXT:  LBB6_36: ## %else50
+; SSE2-NEXT:    testl $262144, %eax ## imm = 0x40000
+; SSE2-NEXT:    jne LBB6_37
+; SSE2-NEXT:  LBB6_38: ## %else53
+; SSE2-NEXT:    testl $524288, %eax ## imm = 0x80000
+; SSE2-NEXT:    jne LBB6_39
+; SSE2-NEXT:  LBB6_40: ## %else56
+; SSE2-NEXT:    testl $1048576, %eax ## imm = 0x100000
+; SSE2-NEXT:    jne LBB6_41
+; SSE2-NEXT:  LBB6_42: ## %else59
+; SSE2-NEXT:    testl $2097152, %eax ## imm = 0x200000
+; SSE2-NEXT:    jne LBB6_43
+; SSE2-NEXT:  LBB6_44: ## %else62
+; SSE2-NEXT:    testl $4194304, %eax ## imm = 0x400000
+; SSE2-NEXT:    jne LBB6_45
+; SSE2-NEXT:  LBB6_46: ## %else65
+; SSE2-NEXT:    testl $8388608, %eax ## imm = 0x800000
+; SSE2-NEXT:    jne LBB6_47
+; SSE2-NEXT:  LBB6_48: ## %else68
+; SSE2-NEXT:    testl $16777216, %eax ## imm = 0x1000000
+; SSE2-NEXT:    jne LBB6_49
+; SSE2-NEXT:  LBB6_50: ## %else71
+; SSE2-NEXT:    testl $33554432, %eax ## imm = 0x2000000
+; SSE2-NEXT:    jne LBB6_51
+; SSE2-NEXT:  LBB6_52: ## %else74
+; SSE2-NEXT:    testl $67108864, %eax ## imm = 0x4000000
+; SSE2-NEXT:    jne LBB6_53
+; SSE2-NEXT:  LBB6_54: ## %else77
+; SSE2-NEXT:    testl $134217728, %eax ## imm = 0x8000000
+; SSE2-NEXT:    jne LBB6_55
+; SSE2-NEXT:  LBB6_56: ## %else80
+; SSE2-NEXT:    testl $268435456, %eax ## imm = 0x10000000
+; SSE2-NEXT:    jne LBB6_57
+; SSE2-NEXT:  LBB6_58: ## %else83
+; SSE2-NEXT:    testl $536870912, %eax ## imm = 0x20000000
+; SSE2-NEXT:    jne LBB6_59
+; SSE2-NEXT:  LBB6_60: ## %else86
+; SSE2-NEXT:    testl $1073741824, %eax ## imm = 0x40000000
+; SSE2-NEXT:    jne LBB6_61
+; SSE2-NEXT:  LBB6_62: ## %else89
+; SSE2-NEXT:    testl $-2147483648, %eax ## imm = 0x80000000
+; SSE2-NEXT:    jne LBB6_63
+; SSE2-NEXT:  LBB6_64: ## %else92
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  LBB6_1: ## %cond.store
+; SSE2-NEXT:    movss %xmm0, (%rdi)
+; SSE2-NEXT:    addq $4, %rdi
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    je LBB6_4
+; SSE2-NEXT:  LBB6_3: ## %cond.store1
+; SSE2-NEXT:    movaps %xmm0, %xmm8
+; SSE2-NEXT:    shufps {{.*#+}} xmm8 = xmm8[1,1],xmm0[2,3]
+; SSE2-NEXT:    movss %xmm8, (%rdi)
+; SSE2-NEXT:    addq $4, %rdi
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    je LBB6_6
+; SSE2-NEXT:  LBB6_5: ## %cond.store4
+; SSE2-NEXT:    movaps %xmm0, %xmm8
+; SSE2-NEXT:    unpckhpd {{.*#+}} xmm8 = xmm8[1],xmm0[1]
+; SSE2-NEXT:    movss %xmm8, (%rdi)
+; SSE2-NEXT:    addq $4, %rdi
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    je LBB6_8
+; SSE2-NEXT:  LBB6_7: ## %cond.store7
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE2-NEXT:    movss %xmm0, (%rdi)
+; SSE2-NEXT:    addq $4, %rdi
+; SSE2-NEXT:    testb $16, %al
+; SSE2-NEXT:    je LBB6_10
+; SSE2-NEXT:  LBB6_9: ## %cond.store10
+; SSE2-NEXT:    movss %xmm1, (%rdi)
+; SSE2-NEXT:    addq $4, %rdi
+; SSE2-NEXT:    testb $32, %al
+; SSE2-NEXT:    je LBB6_12
+; SSE2-NEXT:  LBB6_11: ## %cond.store13
+; SSE2-NEXT:    movaps %xmm1, %xmm0
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm1[2,3]
+; SSE2-NEXT:    movss %xmm0, (%rdi)
+; SSE2-NEXT:    addq $4, %rdi
+; SSE2-NEXT:    testb $64, %al
 ; SSE2-NEXT:    je LBB6_14
-; SSE2-NEXT:  ## %bb.13: ## %cond.store16
+; SSE2-NEXT:  LBB6_13: ## %cond.store16
 ; SSE2-NEXT:    movaps %xmm1, %xmm0
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1]
 ; SSE2-NEXT:    movss %xmm0, (%rdi)
 ; SSE2-NEXT:    addq $4, %rdi
-; SSE2-NEXT:  LBB6_14: ## %else17
-; SSE2-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm0
-; SSE2-NEXT:    pextrw $6, %xmm8, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $-128, %al
 ; SSE2-NEXT:    je LBB6_16
-; SSE2-NEXT:  ## %bb.15: ## %cond.store19
+; SSE2-NEXT:  LBB6_15: ## %cond.store19
 ; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,1,2,3]
 ; SSE2-NEXT:    movss %xmm1, (%rdi)
 ; SSE2-NEXT:    addq $4, %rdi
-; SSE2-NEXT:  LBB6_16: ## %else20
-; SSE2-NEXT:    xorps %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE2-NEXT:    movd %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $256, %eax ## imm = 0x100
 ; SSE2-NEXT:    je LBB6_18
-; SSE2-NEXT:  ## %bb.17: ## %cond.store22
+; SSE2-NEXT:  LBB6_17: ## %cond.store22
 ; SSE2-NEXT:    movss %xmm2, (%rdi)
 ; SSE2-NEXT:    addq $4, %rdi
-; SSE2-NEXT:  LBB6_18: ## %else23
-; SSE2-NEXT:    pextrw $2, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $512, %eax ## imm = 0x200
 ; SSE2-NEXT:    je LBB6_20
-; SSE2-NEXT:  ## %bb.19: ## %cond.store25
-; SSE2-NEXT:    movaps %xmm2, %xmm1
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm2[2,3]
-; SSE2-NEXT:    movss %xmm1, (%rdi)
+; SSE2-NEXT:  LBB6_19: ## %cond.store25
+; SSE2-NEXT:    movaps %xmm2, %xmm0
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm2[2,3]
+; SSE2-NEXT:    movss %xmm0, (%rdi)
 ; SSE2-NEXT:    addq $4, %rdi
-; SSE2-NEXT:  LBB6_20: ## %else26
-; SSE2-NEXT:    xorps %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $1024, %eax ## imm = 0x400
 ; SSE2-NEXT:    je LBB6_22
-; SSE2-NEXT:  ## %bb.21: ## %cond.store28
-; SSE2-NEXT:    movaps %xmm2, %xmm1
-; SSE2-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm2[1]
-; SSE2-NEXT:    movss %xmm1, (%rdi)
+; SSE2-NEXT:  LBB6_21: ## %cond.store28
+; SSE2-NEXT:    movaps %xmm2, %xmm0
+; SSE2-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1],xmm2[1]
+; SSE2-NEXT:    movss %xmm0, (%rdi)
 ; SSE2-NEXT:    addq $4, %rdi
-; SSE2-NEXT:  LBB6_22: ## %else29
-; SSE2-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm1
-; SSE2-NEXT:    pextrw $6, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $2048, %eax ## imm = 0x800
 ; SSE2-NEXT:    je LBB6_24
-; SSE2-NEXT:  ## %bb.23: ## %cond.store31
+; SSE2-NEXT:  LBB6_23: ## %cond.store31
 ; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[3,1,2,3]
 ; SSE2-NEXT:    movss %xmm2, (%rdi)
 ; SSE2-NEXT:    addq $4, %rdi
-; SSE2-NEXT:  LBB6_24: ## %else32
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $4096, %eax ## imm = 0x1000
 ; SSE2-NEXT:    je LBB6_26
-; SSE2-NEXT:  ## %bb.25: ## %cond.store34
+; SSE2-NEXT:  LBB6_25: ## %cond.store34
 ; SSE2-NEXT:    movss %xmm3, (%rdi)
 ; SSE2-NEXT:    addq $4, %rdi
-; SSE2-NEXT:  LBB6_26: ## %else35
-; SSE2-NEXT:    pextrw $2, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $8192, %eax ## imm = 0x2000
 ; SSE2-NEXT:    je LBB6_28
-; SSE2-NEXT:  ## %bb.27: ## %cond.store37
+; SSE2-NEXT:  LBB6_27: ## %cond.store37
 ; SSE2-NEXT:    movaps %xmm3, %xmm0
 ; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm3[2,3]
 ; SSE2-NEXT:    movss %xmm0, (%rdi)
 ; SSE2-NEXT:    addq $4, %rdi
-; SSE2-NEXT:  LBB6_28: ## %else38
-; SSE2-NEXT:    xorps %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE2-NEXT:    pextrw $4, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $16384, %eax ## imm = 0x4000
 ; SSE2-NEXT:    je LBB6_30
-; SSE2-NEXT:  ## %bb.29: ## %cond.store40
+; SSE2-NEXT:  LBB6_29: ## %cond.store40
 ; SSE2-NEXT:    movaps %xmm3, %xmm0
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1],xmm3[1]
 ; SSE2-NEXT:    movss %xmm0, (%rdi)
 ; SSE2-NEXT:    addq $4, %rdi
-; SSE2-NEXT:  LBB6_30: ## %else41
-; SSE2-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm0
-; SSE2-NEXT:    pextrw $6, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $32768, %eax ## imm = 0x8000
 ; SSE2-NEXT:    je LBB6_32
-; SSE2-NEXT:  ## %bb.31: ## %cond.store43
+; SSE2-NEXT:  LBB6_31: ## %cond.store43
 ; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,1,2,3]
 ; SSE2-NEXT:    movss %xmm3, (%rdi)
 ; SSE2-NEXT:    addq $4, %rdi
-; SSE2-NEXT:  LBB6_32: ## %else44
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE2-NEXT:    movd %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $65536, %eax ## imm = 0x10000
 ; SSE2-NEXT:    je LBB6_34
-; SSE2-NEXT:  ## %bb.33: ## %cond.store46
+; SSE2-NEXT:  LBB6_33: ## %cond.store46
 ; SSE2-NEXT:    movss %xmm4, (%rdi)
 ; SSE2-NEXT:    addq $4, %rdi
-; SSE2-NEXT:  LBB6_34: ## %else47
-; SSE2-NEXT:    pextrw $2, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $131072, %eax ## imm = 0x20000
 ; SSE2-NEXT:    je LBB6_36
-; SSE2-NEXT:  ## %bb.35: ## %cond.store49
-; SSE2-NEXT:    movaps %xmm4, %xmm1
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm4[2,3]
-; SSE2-NEXT:    movss %xmm1, (%rdi)
+; SSE2-NEXT:  LBB6_35: ## %cond.store49
+; SSE2-NEXT:    movaps %xmm4, %xmm0
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm4[2,3]
+; SSE2-NEXT:    movss %xmm0, (%rdi)
 ; SSE2-NEXT:    addq $4, %rdi
-; SSE2-NEXT:  LBB6_36: ## %else50
-; SSE2-NEXT:    xorps %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $262144, %eax ## imm = 0x40000
 ; SSE2-NEXT:    je LBB6_38
-; SSE2-NEXT:  ## %bb.37: ## %cond.store52
-; SSE2-NEXT:    movaps %xmm4, %xmm1
-; SSE2-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm4[1]
-; SSE2-NEXT:    movss %xmm1, (%rdi)
+; SSE2-NEXT:  LBB6_37: ## %cond.store52
+; SSE2-NEXT:    movaps %xmm4, %xmm0
+; SSE2-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1],xmm4[1]
+; SSE2-NEXT:    movss %xmm0, (%rdi)
 ; SSE2-NEXT:    addq $4, %rdi
-; SSE2-NEXT:  LBB6_38: ## %else53
-; SSE2-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm1
-; SSE2-NEXT:    pextrw $6, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $524288, %eax ## imm = 0x80000
 ; SSE2-NEXT:    je LBB6_40
-; SSE2-NEXT:  ## %bb.39: ## %cond.store55
+; SSE2-NEXT:  LBB6_39: ## %cond.store55
 ; SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[3,1,2,3]
 ; SSE2-NEXT:    movss %xmm4, (%rdi)
 ; SSE2-NEXT:    addq $4, %rdi
-; SSE2-NEXT:  LBB6_40: ## %else56
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $1048576, %eax ## imm = 0x100000
 ; SSE2-NEXT:    je LBB6_42
-; SSE2-NEXT:  ## %bb.41: ## %cond.store58
+; SSE2-NEXT:  LBB6_41: ## %cond.store58
 ; SSE2-NEXT:    movss %xmm5, (%rdi)
 ; SSE2-NEXT:    addq $4, %rdi
-; SSE2-NEXT:  LBB6_42: ## %else59
-; SSE2-NEXT:    pextrw $2, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $2097152, %eax ## imm = 0x200000
 ; SSE2-NEXT:    je LBB6_44
-; SSE2-NEXT:  ## %bb.43: ## %cond.store61
+; SSE2-NEXT:  LBB6_43: ## %cond.store61
 ; SSE2-NEXT:    movaps %xmm5, %xmm0
 ; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm5[2,3]
 ; SSE2-NEXT:    movss %xmm0, (%rdi)
 ; SSE2-NEXT:    addq $4, %rdi
-; SSE2-NEXT:  LBB6_44: ## %else62
-; SSE2-NEXT:    xorps %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE2-NEXT:    pextrw $4, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $4194304, %eax ## imm = 0x400000
 ; SSE2-NEXT:    je LBB6_46
-; SSE2-NEXT:  ## %bb.45: ## %cond.store64
+; SSE2-NEXT:  LBB6_45: ## %cond.store64
 ; SSE2-NEXT:    movaps %xmm5, %xmm0
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1],xmm5[1]
 ; SSE2-NEXT:    movss %xmm0, (%rdi)
 ; SSE2-NEXT:    addq $4, %rdi
-; SSE2-NEXT:  LBB6_46: ## %else65
-; SSE2-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm0
-; SSE2-NEXT:    pextrw $6, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $8388608, %eax ## imm = 0x800000
 ; SSE2-NEXT:    je LBB6_48
-; SSE2-NEXT:  ## %bb.47: ## %cond.store67
+; SSE2-NEXT:  LBB6_47: ## %cond.store67
 ; SSE2-NEXT:    shufps {{.*#+}} xmm5 = xmm5[3,1,2,3]
 ; SSE2-NEXT:    movss %xmm5, (%rdi)
 ; SSE2-NEXT:    addq $4, %rdi
-; SSE2-NEXT:  LBB6_48: ## %else68
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE2-NEXT:    movd %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $16777216, %eax ## imm = 0x1000000
 ; SSE2-NEXT:    je LBB6_50
-; SSE2-NEXT:  ## %bb.49: ## %cond.store70
+; SSE2-NEXT:  LBB6_49: ## %cond.store70
 ; SSE2-NEXT:    movss %xmm6, (%rdi)
 ; SSE2-NEXT:    addq $4, %rdi
-; SSE2-NEXT:  LBB6_50: ## %else71
-; SSE2-NEXT:    pextrw $2, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $33554432, %eax ## imm = 0x2000000
 ; SSE2-NEXT:    je LBB6_52
-; SSE2-NEXT:  ## %bb.51: ## %cond.store73
-; SSE2-NEXT:    movaps %xmm6, %xmm1
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm6[2,3]
-; SSE2-NEXT:    movss %xmm1, (%rdi)
+; SSE2-NEXT:  LBB6_51: ## %cond.store73
+; SSE2-NEXT:    movaps %xmm6, %xmm0
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm6[2,3]
+; SSE2-NEXT:    movss %xmm0, (%rdi)
 ; SSE2-NEXT:    addq $4, %rdi
-; SSE2-NEXT:  LBB6_52: ## %else74
-; SSE2-NEXT:    xorps %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $67108864, %eax ## imm = 0x4000000
 ; SSE2-NEXT:    je LBB6_54
-; SSE2-NEXT:  ## %bb.53: ## %cond.store76
-; SSE2-NEXT:    movaps %xmm6, %xmm1
-; SSE2-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm6[1]
-; SSE2-NEXT:    movss %xmm1, (%rdi)
+; SSE2-NEXT:  LBB6_53: ## %cond.store76
+; SSE2-NEXT:    movaps %xmm6, %xmm0
+; SSE2-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1],xmm6[1]
+; SSE2-NEXT:    movss %xmm0, (%rdi)
 ; SSE2-NEXT:    addq $4, %rdi
-; SSE2-NEXT:  LBB6_54: ## %else77
-; SSE2-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm1
-; SSE2-NEXT:    pextrw $6, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $134217728, %eax ## imm = 0x8000000
 ; SSE2-NEXT:    je LBB6_56
-; SSE2-NEXT:  ## %bb.55: ## %cond.store79
+; SSE2-NEXT:  LBB6_55: ## %cond.store79
 ; SSE2-NEXT:    shufps {{.*#+}} xmm6 = xmm6[3,1,2,3]
 ; SSE2-NEXT:    movss %xmm6, (%rdi)
 ; SSE2-NEXT:    addq $4, %rdi
-; SSE2-NEXT:  LBB6_56: ## %else80
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $268435456, %eax ## imm = 0x10000000
 ; SSE2-NEXT:    je LBB6_58
-; SSE2-NEXT:  ## %bb.57: ## %cond.store82
+; SSE2-NEXT:  LBB6_57: ## %cond.store82
 ; SSE2-NEXT:    movss %xmm7, (%rdi)
 ; SSE2-NEXT:    addq $4, %rdi
-; SSE2-NEXT:  LBB6_58: ## %else83
-; SSE2-NEXT:    pextrw $2, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $536870912, %eax ## imm = 0x20000000
 ; SSE2-NEXT:    je LBB6_60
-; SSE2-NEXT:  ## %bb.59: ## %cond.store85
+; SSE2-NEXT:  LBB6_59: ## %cond.store85
 ; SSE2-NEXT:    movaps %xmm7, %xmm0
 ; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm7[2,3]
 ; SSE2-NEXT:    movss %xmm0, (%rdi)
 ; SSE2-NEXT:    addq $4, %rdi
-; SSE2-NEXT:  LBB6_60: ## %else86
-; SSE2-NEXT:    xorps %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE2-NEXT:    pextrw $4, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $1073741824, %eax ## imm = 0x40000000
 ; SSE2-NEXT:    je LBB6_62
-; SSE2-NEXT:  ## %bb.61: ## %cond.store88
+; SSE2-NEXT:  LBB6_61: ## %cond.store88
 ; SSE2-NEXT:    movaps %xmm7, %xmm0
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1],xmm7[1]
 ; SSE2-NEXT:    movss %xmm0, (%rdi)
 ; SSE2-NEXT:    addq $4, %rdi
-; SSE2-NEXT:  LBB6_62: ## %else89
-; SSE2-NEXT:    pextrw $6, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $-2147483648, %eax ## imm = 0x80000000
 ; SSE2-NEXT:    je LBB6_64
-; SSE2-NEXT:  ## %bb.63: ## %cond.store91
+; SSE2-NEXT:  LBB6_63: ## %cond.store91
 ; SSE2-NEXT:    shufps {{.*#+}} xmm7 = xmm7[3,1,2,3]
 ; SSE2-NEXT:    movss %xmm7, (%rdi)
-; SSE2-NEXT:  LBB6_64: ## %else92
 ; SSE2-NEXT:    retq
 ;
 ; SSE42-LABEL: compressstore_v32f32_v32i32:
 ; SSE42:       ## %bb.0:
-; SSE42-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm9
 ; SSE42-NEXT:    pxor %xmm8, %xmm8
-; SSE42-NEXT:    pcmpeqd %xmm9, %xmm8
-; SSE42-NEXT:    pextrb $0, %xmm8, %eax
+; SSE42-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm9
+; SSE42-NEXT:    pcmpeqd %xmm8, %xmm9
+; SSE42-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm10
+; SSE42-NEXT:    pcmpeqd %xmm8, %xmm10
+; SSE42-NEXT:    packssdw %xmm9, %xmm10
+; SSE42-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm9
+; SSE42-NEXT:    pcmpeqd %xmm8, %xmm9
+; SSE42-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm11
+; SSE42-NEXT:    pcmpeqd %xmm8, %xmm11
+; SSE42-NEXT:    packssdw %xmm9, %xmm11
+; SSE42-NEXT:    packsswb %xmm10, %xmm11
+; SSE42-NEXT:    pmovmskb %xmm11, %ecx
+; SSE42-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm9
+; SSE42-NEXT:    pcmpeqd %xmm8, %xmm9
+; SSE42-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm10
+; SSE42-NEXT:    pcmpeqd %xmm8, %xmm10
+; SSE42-NEXT:    packssdw %xmm9, %xmm10
+; SSE42-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm9
+; SSE42-NEXT:    pcmpeqd %xmm8, %xmm9
+; SSE42-NEXT:    pcmpeqd {{[0-9]+}}(%rsp), %xmm8
+; SSE42-NEXT:    packssdw %xmm9, %xmm8
+; SSE42-NEXT:    packsswb %xmm10, %xmm8
+; SSE42-NEXT:    pmovmskb %xmm8, %eax
+; SSE42-NEXT:    shll $16, %eax
+; SSE42-NEXT:    orl %ecx, %eax
 ; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB6_2
-; SSE42-NEXT:  ## %bb.1: ## %cond.store
-; SSE42-NEXT:    movd %xmm0, (%rdi)
+; SSE42-NEXT:    jne LBB6_1
+; SSE42-NEXT:  ## %bb.2: ## %else
+; SSE42-NEXT:    testb $2, %al
+; SSE42-NEXT:    jne LBB6_3
+; SSE42-NEXT:  LBB6_4: ## %else2
+; SSE42-NEXT:    testb $4, %al
+; SSE42-NEXT:    jne LBB6_5
+; SSE42-NEXT:  LBB6_6: ## %else5
+; SSE42-NEXT:    testb $8, %al
+; SSE42-NEXT:    jne LBB6_7
+; SSE42-NEXT:  LBB6_8: ## %else8
+; SSE42-NEXT:    testb $16, %al
+; SSE42-NEXT:    jne LBB6_9
+; SSE42-NEXT:  LBB6_10: ## %else11
+; SSE42-NEXT:    testb $32, %al
+; SSE42-NEXT:    jne LBB6_11
+; SSE42-NEXT:  LBB6_12: ## %else14
+; SSE42-NEXT:    testb $64, %al
+; SSE42-NEXT:    jne LBB6_13
+; SSE42-NEXT:  LBB6_14: ## %else17
+; SSE42-NEXT:    testb $-128, %al
+; SSE42-NEXT:    jne LBB6_15
+; SSE42-NEXT:  LBB6_16: ## %else20
+; SSE42-NEXT:    testl $256, %eax ## imm = 0x100
+; SSE42-NEXT:    jne LBB6_17
+; SSE42-NEXT:  LBB6_18: ## %else23
+; SSE42-NEXT:    testl $512, %eax ## imm = 0x200
+; SSE42-NEXT:    jne LBB6_19
+; SSE42-NEXT:  LBB6_20: ## %else26
+; SSE42-NEXT:    testl $1024, %eax ## imm = 0x400
+; SSE42-NEXT:    jne LBB6_21
+; SSE42-NEXT:  LBB6_22: ## %else29
+; SSE42-NEXT:    testl $2048, %eax ## imm = 0x800
+; SSE42-NEXT:    jne LBB6_23
+; SSE42-NEXT:  LBB6_24: ## %else32
+; SSE42-NEXT:    testl $4096, %eax ## imm = 0x1000
+; SSE42-NEXT:    jne LBB6_25
+; SSE42-NEXT:  LBB6_26: ## %else35
+; SSE42-NEXT:    testl $8192, %eax ## imm = 0x2000
+; SSE42-NEXT:    jne LBB6_27
+; SSE42-NEXT:  LBB6_28: ## %else38
+; SSE42-NEXT:    testl $16384, %eax ## imm = 0x4000
+; SSE42-NEXT:    jne LBB6_29
+; SSE42-NEXT:  LBB6_30: ## %else41
+; SSE42-NEXT:    testl $32768, %eax ## imm = 0x8000
+; SSE42-NEXT:    jne LBB6_31
+; SSE42-NEXT:  LBB6_32: ## %else44
+; SSE42-NEXT:    testl $65536, %eax ## imm = 0x10000
+; SSE42-NEXT:    jne LBB6_33
+; SSE42-NEXT:  LBB6_34: ## %else47
+; SSE42-NEXT:    testl $131072, %eax ## imm = 0x20000
+; SSE42-NEXT:    jne LBB6_35
+; SSE42-NEXT:  LBB6_36: ## %else50
+; SSE42-NEXT:    testl $262144, %eax ## imm = 0x40000
+; SSE42-NEXT:    jne LBB6_37
+; SSE42-NEXT:  LBB6_38: ## %else53
+; SSE42-NEXT:    testl $524288, %eax ## imm = 0x80000
+; SSE42-NEXT:    jne LBB6_39
+; SSE42-NEXT:  LBB6_40: ## %else56
+; SSE42-NEXT:    testl $1048576, %eax ## imm = 0x100000
+; SSE42-NEXT:    jne LBB6_41
+; SSE42-NEXT:  LBB6_42: ## %else59
+; SSE42-NEXT:    testl $2097152, %eax ## imm = 0x200000
+; SSE42-NEXT:    jne LBB6_43
+; SSE42-NEXT:  LBB6_44: ## %else62
+; SSE42-NEXT:    testl $4194304, %eax ## imm = 0x400000
+; SSE42-NEXT:    jne LBB6_45
+; SSE42-NEXT:  LBB6_46: ## %else65
+; SSE42-NEXT:    testl $8388608, %eax ## imm = 0x800000
+; SSE42-NEXT:    jne LBB6_47
+; SSE42-NEXT:  LBB6_48: ## %else68
+; SSE42-NEXT:    testl $16777216, %eax ## imm = 0x1000000
+; SSE42-NEXT:    jne LBB6_49
+; SSE42-NEXT:  LBB6_50: ## %else71
+; SSE42-NEXT:    testl $33554432, %eax ## imm = 0x2000000
+; SSE42-NEXT:    jne LBB6_51
+; SSE42-NEXT:  LBB6_52: ## %else74
+; SSE42-NEXT:    testl $67108864, %eax ## imm = 0x4000000
+; SSE42-NEXT:    jne LBB6_53
+; SSE42-NEXT:  LBB6_54: ## %else77
+; SSE42-NEXT:    testl $134217728, %eax ## imm = 0x8000000
+; SSE42-NEXT:    jne LBB6_55
+; SSE42-NEXT:  LBB6_56: ## %else80
+; SSE42-NEXT:    testl $268435456, %eax ## imm = 0x10000000
+; SSE42-NEXT:    jne LBB6_57
+; SSE42-NEXT:  LBB6_58: ## %else83
+; SSE42-NEXT:    testl $536870912, %eax ## imm = 0x20000000
+; SSE42-NEXT:    jne LBB6_59
+; SSE42-NEXT:  LBB6_60: ## %else86
+; SSE42-NEXT:    testl $1073741824, %eax ## imm = 0x40000000
+; SSE42-NEXT:    jne LBB6_61
+; SSE42-NEXT:  LBB6_62: ## %else89
+; SSE42-NEXT:    testl $-2147483648, %eax ## imm = 0x80000000
+; SSE42-NEXT:    jne LBB6_63
+; SSE42-NEXT:  LBB6_64: ## %else92
+; SSE42-NEXT:    retq
+; SSE42-NEXT:  LBB6_1: ## %cond.store
+; SSE42-NEXT:    movss %xmm0, (%rdi)
 ; SSE42-NEXT:    addq $4, %rdi
-; SSE42-NEXT:  LBB6_2: ## %else
-; SSE42-NEXT:    pextrb $4, %xmm8, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $2, %al
 ; SSE42-NEXT:    je LBB6_4
-; SSE42-NEXT:  ## %bb.3: ## %cond.store1
-; SSE42-NEXT:    pextrd $1, %xmm0, (%rdi)
+; SSE42-NEXT:  LBB6_3: ## %cond.store1
+; SSE42-NEXT:    extractps $1, %xmm0, (%rdi)
 ; SSE42-NEXT:    addq $4, %rdi
-; SSE42-NEXT:  LBB6_4: ## %else2
-; SSE42-NEXT:    pxor %xmm8, %xmm8
-; SSE42-NEXT:    pcmpeqd %xmm8, %xmm9
-; SSE42-NEXT:    pextrb $8, %xmm9, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $4, %al
 ; SSE42-NEXT:    je LBB6_6
-; SSE42-NEXT:  ## %bb.5: ## %cond.store4
-; SSE42-NEXT:    pextrd $2, %xmm0, (%rdi)
+; SSE42-NEXT:  LBB6_5: ## %cond.store4
+; SSE42-NEXT:    extractps $2, %xmm0, (%rdi)
 ; SSE42-NEXT:    addq $4, %rdi
-; SSE42-NEXT:  LBB6_6: ## %else5
-; SSE42-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm8
-; SSE42-NEXT:    pextrb $12, %xmm9, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $8, %al
 ; SSE42-NEXT:    je LBB6_8
-; SSE42-NEXT:  ## %bb.7: ## %cond.store7
-; SSE42-NEXT:    pextrd $3, %xmm0, (%rdi)
+; SSE42-NEXT:  LBB6_7: ## %cond.store7
+; SSE42-NEXT:    extractps $3, %xmm0, (%rdi)
 ; SSE42-NEXT:    addq $4, %rdi
-; SSE42-NEXT:  LBB6_8: ## %else8
-; SSE42-NEXT:    pxor %xmm0, %xmm0
-; SSE42-NEXT:    pcmpeqd %xmm8, %xmm0
-; SSE42-NEXT:    pextrb $0, %xmm0, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $16, %al
 ; SSE42-NEXT:    je LBB6_10
-; SSE42-NEXT:  ## %bb.9: ## %cond.store10
-; SSE42-NEXT:    movd %xmm1, (%rdi)
+; SSE42-NEXT:  LBB6_9: ## %cond.store10
+; SSE42-NEXT:    movss %xmm1, (%rdi)
 ; SSE42-NEXT:    addq $4, %rdi
-; SSE42-NEXT:  LBB6_10: ## %else11
-; SSE42-NEXT:    pextrb $4, %xmm0, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $32, %al
 ; SSE42-NEXT:    je LBB6_12
-; SSE42-NEXT:  ## %bb.11: ## %cond.store13
-; SSE42-NEXT:    pextrd $1, %xmm1, (%rdi)
+; SSE42-NEXT:  LBB6_11: ## %cond.store13
+; SSE42-NEXT:    extractps $1, %xmm1, (%rdi)
 ; SSE42-NEXT:    addq $4, %rdi
-; SSE42-NEXT:  LBB6_12: ## %else14
-; SSE42-NEXT:    pxor %xmm0, %xmm0
-; SSE42-NEXT:    pcmpeqd %xmm0, %xmm8
-; SSE42-NEXT:    pextrb $8, %xmm8, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $64, %al
 ; SSE42-NEXT:    je LBB6_14
-; SSE42-NEXT:  ## %bb.13: ## %cond.store16
-; SSE42-NEXT:    pextrd $2, %xmm1, (%rdi)
+; SSE42-NEXT:  LBB6_13: ## %cond.store16
+; SSE42-NEXT:    extractps $2, %xmm1, (%rdi)
 ; SSE42-NEXT:    addq $4, %rdi
-; SSE42-NEXT:  LBB6_14: ## %else17
-; SSE42-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm0
-; SSE42-NEXT:    pextrb $12, %xmm8, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $-128, %al
 ; SSE42-NEXT:    je LBB6_16
-; SSE42-NEXT:  ## %bb.15: ## %cond.store19
-; SSE42-NEXT:    pextrd $3, %xmm1, (%rdi)
+; SSE42-NEXT:  LBB6_15: ## %cond.store19
+; SSE42-NEXT:    extractps $3, %xmm1, (%rdi)
 ; SSE42-NEXT:    addq $4, %rdi
-; SSE42-NEXT:  LBB6_16: ## %else20
-; SSE42-NEXT:    pxor %xmm1, %xmm1
-; SSE42-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE42-NEXT:    pextrb $0, %xmm1, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $256, %eax ## imm = 0x100
 ; SSE42-NEXT:    je LBB6_18
-; SSE42-NEXT:  ## %bb.17: ## %cond.store22
+; SSE42-NEXT:  LBB6_17: ## %cond.store22
 ; SSE42-NEXT:    movss %xmm2, (%rdi)
 ; SSE42-NEXT:    addq $4, %rdi
-; SSE42-NEXT:  LBB6_18: ## %else23
-; SSE42-NEXT:    pextrb $4, %xmm1, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $512, %eax ## imm = 0x200
 ; SSE42-NEXT:    je LBB6_20
-; SSE42-NEXT:  ## %bb.19: ## %cond.store25
+; SSE42-NEXT:  LBB6_19: ## %cond.store25
 ; SSE42-NEXT:    extractps $1, %xmm2, (%rdi)
 ; SSE42-NEXT:    addq $4, %rdi
-; SSE42-NEXT:  LBB6_20: ## %else26
-; SSE42-NEXT:    pxor %xmm1, %xmm1
-; SSE42-NEXT:    pcmpeqd %xmm1, %xmm0
-; SSE42-NEXT:    pextrb $8, %xmm0, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $1024, %eax ## imm = 0x400
 ; SSE42-NEXT:    je LBB6_22
-; SSE42-NEXT:  ## %bb.21: ## %cond.store28
+; SSE42-NEXT:  LBB6_21: ## %cond.store28
 ; SSE42-NEXT:    extractps $2, %xmm2, (%rdi)
 ; SSE42-NEXT:    addq $4, %rdi
-; SSE42-NEXT:  LBB6_22: ## %else29
-; SSE42-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm1
-; SSE42-NEXT:    pextrb $12, %xmm0, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $2048, %eax ## imm = 0x800
 ; SSE42-NEXT:    je LBB6_24
-; SSE42-NEXT:  ## %bb.23: ## %cond.store31
+; SSE42-NEXT:  LBB6_23: ## %cond.store31
 ; SSE42-NEXT:    extractps $3, %xmm2, (%rdi)
 ; SSE42-NEXT:    addq $4, %rdi
-; SSE42-NEXT:  LBB6_24: ## %else32
-; SSE42-NEXT:    pxor %xmm0, %xmm0
-; SSE42-NEXT:    pcmpeqd %xmm1, %xmm0
-; SSE42-NEXT:    pextrb $0, %xmm0, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $4096, %eax ## imm = 0x1000
 ; SSE42-NEXT:    je LBB6_26
-; SSE42-NEXT:  ## %bb.25: ## %cond.store34
+; SSE42-NEXT:  LBB6_25: ## %cond.store34
 ; SSE42-NEXT:    movss %xmm3, (%rdi)
 ; SSE42-NEXT:    addq $4, %rdi
-; SSE42-NEXT:  LBB6_26: ## %else35
-; SSE42-NEXT:    pextrb $4, %xmm0, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $8192, %eax ## imm = 0x2000
 ; SSE42-NEXT:    je LBB6_28
-; SSE42-NEXT:  ## %bb.27: ## %cond.store37
+; SSE42-NEXT:  LBB6_27: ## %cond.store37
 ; SSE42-NEXT:    extractps $1, %xmm3, (%rdi)
 ; SSE42-NEXT:    addq $4, %rdi
-; SSE42-NEXT:  LBB6_28: ## %else38
-; SSE42-NEXT:    pxor %xmm0, %xmm0
-; SSE42-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE42-NEXT:    pextrb $8, %xmm1, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $16384, %eax ## imm = 0x4000
 ; SSE42-NEXT:    je LBB6_30
-; SSE42-NEXT:  ## %bb.29: ## %cond.store40
+; SSE42-NEXT:  LBB6_29: ## %cond.store40
 ; SSE42-NEXT:    extractps $2, %xmm3, (%rdi)
 ; SSE42-NEXT:    addq $4, %rdi
-; SSE42-NEXT:  LBB6_30: ## %else41
-; SSE42-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm0
-; SSE42-NEXT:    pextrb $12, %xmm1, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $32768, %eax ## imm = 0x8000
 ; SSE42-NEXT:    je LBB6_32
-; SSE42-NEXT:  ## %bb.31: ## %cond.store43
+; SSE42-NEXT:  LBB6_31: ## %cond.store43
 ; SSE42-NEXT:    extractps $3, %xmm3, (%rdi)
 ; SSE42-NEXT:    addq $4, %rdi
-; SSE42-NEXT:  LBB6_32: ## %else44
-; SSE42-NEXT:    pxor %xmm1, %xmm1
-; SSE42-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE42-NEXT:    pextrb $0, %xmm1, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $65536, %eax ## imm = 0x10000
 ; SSE42-NEXT:    je LBB6_34
-; SSE42-NEXT:  ## %bb.33: ## %cond.store46
+; SSE42-NEXT:  LBB6_33: ## %cond.store46
 ; SSE42-NEXT:    movss %xmm4, (%rdi)
 ; SSE42-NEXT:    addq $4, %rdi
-; SSE42-NEXT:  LBB6_34: ## %else47
-; SSE42-NEXT:    pextrb $4, %xmm1, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $131072, %eax ## imm = 0x20000
 ; SSE42-NEXT:    je LBB6_36
-; SSE42-NEXT:  ## %bb.35: ## %cond.store49
+; SSE42-NEXT:  LBB6_35: ## %cond.store49
 ; SSE42-NEXT:    extractps $1, %xmm4, (%rdi)
 ; SSE42-NEXT:    addq $4, %rdi
-; SSE42-NEXT:  LBB6_36: ## %else50
-; SSE42-NEXT:    pxor %xmm1, %xmm1
-; SSE42-NEXT:    pcmpeqd %xmm1, %xmm0
-; SSE42-NEXT:    pextrb $8, %xmm0, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $262144, %eax ## imm = 0x40000
 ; SSE42-NEXT:    je LBB6_38
-; SSE42-NEXT:  ## %bb.37: ## %cond.store52
+; SSE42-NEXT:  LBB6_37: ## %cond.store52
 ; SSE42-NEXT:    extractps $2, %xmm4, (%rdi)
 ; SSE42-NEXT:    addq $4, %rdi
-; SSE42-NEXT:  LBB6_38: ## %else53
-; SSE42-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm1
-; SSE42-NEXT:    pextrb $12, %xmm0, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $524288, %eax ## imm = 0x80000
 ; SSE42-NEXT:    je LBB6_40
-; SSE42-NEXT:  ## %bb.39: ## %cond.store55
+; SSE42-NEXT:  LBB6_39: ## %cond.store55
 ; SSE42-NEXT:    extractps $3, %xmm4, (%rdi)
 ; SSE42-NEXT:    addq $4, %rdi
-; SSE42-NEXT:  LBB6_40: ## %else56
-; SSE42-NEXT:    pxor %xmm0, %xmm0
-; SSE42-NEXT:    pcmpeqd %xmm1, %xmm0
-; SSE42-NEXT:    pextrb $0, %xmm0, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $1048576, %eax ## imm = 0x100000
 ; SSE42-NEXT:    je LBB6_42
-; SSE42-NEXT:  ## %bb.41: ## %cond.store58
+; SSE42-NEXT:  LBB6_41: ## %cond.store58
 ; SSE42-NEXT:    movss %xmm5, (%rdi)
 ; SSE42-NEXT:    addq $4, %rdi
-; SSE42-NEXT:  LBB6_42: ## %else59
-; SSE42-NEXT:    pextrb $4, %xmm0, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $2097152, %eax ## imm = 0x200000
 ; SSE42-NEXT:    je LBB6_44
-; SSE42-NEXT:  ## %bb.43: ## %cond.store61
+; SSE42-NEXT:  LBB6_43: ## %cond.store61
 ; SSE42-NEXT:    extractps $1, %xmm5, (%rdi)
 ; SSE42-NEXT:    addq $4, %rdi
-; SSE42-NEXT:  LBB6_44: ## %else62
-; SSE42-NEXT:    pxor %xmm0, %xmm0
-; SSE42-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE42-NEXT:    pextrb $8, %xmm1, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $4194304, %eax ## imm = 0x400000
 ; SSE42-NEXT:    je LBB6_46
-; SSE42-NEXT:  ## %bb.45: ## %cond.store64
+; SSE42-NEXT:  LBB6_45: ## %cond.store64
 ; SSE42-NEXT:    extractps $2, %xmm5, (%rdi)
 ; SSE42-NEXT:    addq $4, %rdi
-; SSE42-NEXT:  LBB6_46: ## %else65
-; SSE42-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm0
-; SSE42-NEXT:    pextrb $12, %xmm1, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $8388608, %eax ## imm = 0x800000
 ; SSE42-NEXT:    je LBB6_48
-; SSE42-NEXT:  ## %bb.47: ## %cond.store67
+; SSE42-NEXT:  LBB6_47: ## %cond.store67
 ; SSE42-NEXT:    extractps $3, %xmm5, (%rdi)
 ; SSE42-NEXT:    addq $4, %rdi
-; SSE42-NEXT:  LBB6_48: ## %else68
-; SSE42-NEXT:    pxor %xmm1, %xmm1
-; SSE42-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE42-NEXT:    pextrb $0, %xmm1, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $16777216, %eax ## imm = 0x1000000
 ; SSE42-NEXT:    je LBB6_50
-; SSE42-NEXT:  ## %bb.49: ## %cond.store70
+; SSE42-NEXT:  LBB6_49: ## %cond.store70
 ; SSE42-NEXT:    movss %xmm6, (%rdi)
 ; SSE42-NEXT:    addq $4, %rdi
-; SSE42-NEXT:  LBB6_50: ## %else71
-; SSE42-NEXT:    pextrb $4, %xmm1, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $33554432, %eax ## imm = 0x2000000
 ; SSE42-NEXT:    je LBB6_52
-; SSE42-NEXT:  ## %bb.51: ## %cond.store73
+; SSE42-NEXT:  LBB6_51: ## %cond.store73
 ; SSE42-NEXT:    extractps $1, %xmm6, (%rdi)
 ; SSE42-NEXT:    addq $4, %rdi
-; SSE42-NEXT:  LBB6_52: ## %else74
-; SSE42-NEXT:    pxor %xmm1, %xmm1
-; SSE42-NEXT:    pcmpeqd %xmm1, %xmm0
-; SSE42-NEXT:    pextrb $8, %xmm0, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $67108864, %eax ## imm = 0x4000000
 ; SSE42-NEXT:    je LBB6_54
-; SSE42-NEXT:  ## %bb.53: ## %cond.store76
+; SSE42-NEXT:  LBB6_53: ## %cond.store76
 ; SSE42-NEXT:    extractps $2, %xmm6, (%rdi)
 ; SSE42-NEXT:    addq $4, %rdi
-; SSE42-NEXT:  LBB6_54: ## %else77
-; SSE42-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm1
-; SSE42-NEXT:    pextrb $12, %xmm0, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $134217728, %eax ## imm = 0x8000000
 ; SSE42-NEXT:    je LBB6_56
-; SSE42-NEXT:  ## %bb.55: ## %cond.store79
+; SSE42-NEXT:  LBB6_55: ## %cond.store79
 ; SSE42-NEXT:    extractps $3, %xmm6, (%rdi)
 ; SSE42-NEXT:    addq $4, %rdi
-; SSE42-NEXT:  LBB6_56: ## %else80
-; SSE42-NEXT:    pxor %xmm0, %xmm0
-; SSE42-NEXT:    pcmpeqd %xmm1, %xmm0
-; SSE42-NEXT:    pextrb $0, %xmm0, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $268435456, %eax ## imm = 0x10000000
 ; SSE42-NEXT:    je LBB6_58
-; SSE42-NEXT:  ## %bb.57: ## %cond.store82
+; SSE42-NEXT:  LBB6_57: ## %cond.store82
 ; SSE42-NEXT:    movss %xmm7, (%rdi)
 ; SSE42-NEXT:    addq $4, %rdi
-; SSE42-NEXT:  LBB6_58: ## %else83
-; SSE42-NEXT:    pextrb $4, %xmm0, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $536870912, %eax ## imm = 0x20000000
 ; SSE42-NEXT:    je LBB6_60
-; SSE42-NEXT:  ## %bb.59: ## %cond.store85
+; SSE42-NEXT:  LBB6_59: ## %cond.store85
 ; SSE42-NEXT:    extractps $1, %xmm7, (%rdi)
 ; SSE42-NEXT:    addq $4, %rdi
-; SSE42-NEXT:  LBB6_60: ## %else86
-; SSE42-NEXT:    pxor %xmm0, %xmm0
-; SSE42-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE42-NEXT:    pextrb $8, %xmm1, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $1073741824, %eax ## imm = 0x40000000
 ; SSE42-NEXT:    je LBB6_62
-; SSE42-NEXT:  ## %bb.61: ## %cond.store88
+; SSE42-NEXT:  LBB6_61: ## %cond.store88
 ; SSE42-NEXT:    extractps $2, %xmm7, (%rdi)
 ; SSE42-NEXT:    addq $4, %rdi
-; SSE42-NEXT:  LBB6_62: ## %else89
-; SSE42-NEXT:    pextrb $12, %xmm1, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $-2147483648, %eax ## imm = 0x80000000
 ; SSE42-NEXT:    je LBB6_64
-; SSE42-NEXT:  ## %bb.63: ## %cond.store91
+; SSE42-NEXT:  LBB6_63: ## %cond.store91
 ; SSE42-NEXT:    extractps $3, %xmm7, (%rdi)
-; SSE42-NEXT:  LBB6_64: ## %else92
 ; SSE42-NEXT:    retq
 ;
 ; AVX1-LABEL: compressstore_v32f32_v32i32:
 ; AVX1:       ## %bb.0:
-; AVX1-NEXT:    vpxor %xmm8, %xmm8, %xmm8
-; AVX1-NEXT:    vpcmpeqd %xmm8, %xmm4, %xmm9
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm9, %xmm9
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm9, %xmm9
-; AVX1-NEXT:    vpextrb $0, %xmm9, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je LBB6_2
-; AVX1-NEXT:  ## %bb.1: ## %cond.store
-; AVX1-NEXT:    vmovd %xmm0, (%rdi)
-; AVX1-NEXT:    addq $4, %rdi
-; AVX1-NEXT:  LBB6_2: ## %else
-; AVX1-NEXT:    vpcmpeqd %xmm8, %xmm4, %xmm8
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm8, %xmm8
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm8, %xmm8
-; AVX1-NEXT:    vpextrb $1, %xmm8, %eax
+; AVX1-NEXT:    vextractf128 $1, %ymm5, %xmm8
+; AVX1-NEXT:    vpxor %xmm9, %xmm9, %xmm9
+; AVX1-NEXT:    vpcmpeqd %xmm9, %xmm8, %xmm8
+; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm9, %xmm5
+; AVX1-NEXT:    vpackssdw %xmm8, %xmm5, %xmm8
+; AVX1-NEXT:    vextractf128 $1, %ymm4, %xmm5
+; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm9, %xmm5
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm9, %xmm4
+; AVX1-NEXT:    vpackssdw %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vpacksswb %xmm8, %xmm4, %xmm4
+; AVX1-NEXT:    vpmovmskb %xmm4, %ecx
+; AVX1-NEXT:    vextractf128 $1, %ymm7, %xmm4
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm9, %xmm4
+; AVX1-NEXT:    vpcmpeqd %xmm7, %xmm9, %xmm5
+; AVX1-NEXT:    vpackssdw %xmm4, %xmm5, %xmm4
+; AVX1-NEXT:    vextractf128 $1, %ymm6, %xmm5
+; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm9, %xmm5
+; AVX1-NEXT:    vpcmpeqd %xmm6, %xmm9, %xmm6
+; AVX1-NEXT:    vpackssdw %xmm5, %xmm6, %xmm5
+; AVX1-NEXT:    vpacksswb %xmm4, %xmm5, %xmm4
+; AVX1-NEXT:    vpmovmskb %xmm4, %eax
+; AVX1-NEXT:    shll $16, %eax
+; AVX1-NEXT:    orl %ecx, %eax
 ; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je LBB6_4
-; AVX1-NEXT:  ## %bb.3: ## %cond.store1
-; AVX1-NEXT:    vpextrd $1, %xmm0, (%rdi)
-; AVX1-NEXT:    addq $4, %rdi
+; AVX1-NEXT:    jne LBB6_1
+; AVX1-NEXT:  ## %bb.2: ## %else
+; AVX1-NEXT:    testb $2, %al
+; AVX1-NEXT:    jne LBB6_3
 ; AVX1-NEXT:  LBB6_4: ## %else2
-; AVX1-NEXT:    vpxor %xmm8, %xmm8, %xmm8
-; AVX1-NEXT:    vpcmpeqd %xmm8, %xmm4, %xmm9
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm9, %xmm9
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm9, %xmm9
-; AVX1-NEXT:    vpextrb $2, %xmm9, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je LBB6_6
-; AVX1-NEXT:  ## %bb.5: ## %cond.store4
-; AVX1-NEXT:    vpextrd $2, %xmm0, (%rdi)
-; AVX1-NEXT:    addq $4, %rdi
+; AVX1-NEXT:    testb $4, %al
+; AVX1-NEXT:    jne LBB6_5
 ; AVX1-NEXT:  LBB6_6: ## %else5
-; AVX1-NEXT:    vpcmpeqd %xmm8, %xmm4, %xmm8
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm8, %xmm8
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm8, %xmm8
-; AVX1-NEXT:    vpextrb $3, %xmm8, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $8, %al
 ; AVX1-NEXT:    je LBB6_8
-; AVX1-NEXT:  ## %bb.7: ## %cond.store7
-; AVX1-NEXT:    vpextrd $3, %xmm0, (%rdi)
+; AVX1-NEXT:  LBB6_7: ## %cond.store7
+; AVX1-NEXT:    vextractps $3, %xmm0, (%rdi)
 ; AVX1-NEXT:    addq $4, %rdi
 ; AVX1-NEXT:  LBB6_8: ## %else8
-; AVX1-NEXT:    vextractf128 $1, %ymm4, %xmm8
-; AVX1-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm8, %xmm9
-; AVX1-NEXT:    vpackssdw %xmm9, %xmm0, %xmm4
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm4, %xmm4
-; AVX1-NEXT:    vpextrb $4, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm10
-; AVX1-NEXT:    je LBB6_10
-; AVX1-NEXT:  ## %bb.9: ## %cond.store10
-; AVX1-NEXT:    vmovss %xmm10, (%rdi)
+; AVX1-NEXT:    testb $16, %al
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    jne LBB6_9
+; AVX1-NEXT:  ## %bb.10: ## %else11
+; AVX1-NEXT:    testb $32, %al
+; AVX1-NEXT:    jne LBB6_11
+; AVX1-NEXT:  LBB6_12: ## %else14
+; AVX1-NEXT:    testb $64, %al
+; AVX1-NEXT:    jne LBB6_13
+; AVX1-NEXT:  LBB6_14: ## %else17
+; AVX1-NEXT:    testb $-128, %al
+; AVX1-NEXT:    jne LBB6_15
+; AVX1-NEXT:  LBB6_16: ## %else20
+; AVX1-NEXT:    testl $256, %eax ## imm = 0x100
+; AVX1-NEXT:    jne LBB6_17
+; AVX1-NEXT:  LBB6_18: ## %else23
+; AVX1-NEXT:    testl $512, %eax ## imm = 0x200
+; AVX1-NEXT:    jne LBB6_19
+; AVX1-NEXT:  LBB6_20: ## %else26
+; AVX1-NEXT:    testl $1024, %eax ## imm = 0x400
+; AVX1-NEXT:    jne LBB6_21
+; AVX1-NEXT:  LBB6_22: ## %else29
+; AVX1-NEXT:    testl $2048, %eax ## imm = 0x800
+; AVX1-NEXT:    je LBB6_24
+; AVX1-NEXT:  LBB6_23: ## %cond.store31
+; AVX1-NEXT:    vextractps $3, %xmm1, (%rdi)
 ; AVX1-NEXT:    addq $4, %rdi
-; AVX1-NEXT:  LBB6_10: ## %else11
-; AVX1-NEXT:    vpackssdw %xmm9, %xmm0, %xmm4
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm4, %xmm4
-; AVX1-NEXT:    vpextrb $5, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:  LBB6_24: ## %else32
+; AVX1-NEXT:    testl $4096, %eax ## imm = 0x1000
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm0
+; AVX1-NEXT:    jne LBB6_25
+; AVX1-NEXT:  ## %bb.26: ## %else35
+; AVX1-NEXT:    testl $8192, %eax ## imm = 0x2000
+; AVX1-NEXT:    jne LBB6_27
+; AVX1-NEXT:  LBB6_28: ## %else38
+; AVX1-NEXT:    testl $16384, %eax ## imm = 0x4000
+; AVX1-NEXT:    jne LBB6_29
+; AVX1-NEXT:  LBB6_30: ## %else41
+; AVX1-NEXT:    testl $32768, %eax ## imm = 0x8000
+; AVX1-NEXT:    jne LBB6_31
+; AVX1-NEXT:  LBB6_32: ## %else44
+; AVX1-NEXT:    testl $65536, %eax ## imm = 0x10000
+; AVX1-NEXT:    jne LBB6_33
+; AVX1-NEXT:  LBB6_34: ## %else47
+; AVX1-NEXT:    testl $131072, %eax ## imm = 0x20000
+; AVX1-NEXT:    jne LBB6_35
+; AVX1-NEXT:  LBB6_36: ## %else50
+; AVX1-NEXT:    testl $262144, %eax ## imm = 0x40000
+; AVX1-NEXT:    jne LBB6_37
+; AVX1-NEXT:  LBB6_38: ## %else53
+; AVX1-NEXT:    testl $524288, %eax ## imm = 0x80000
+; AVX1-NEXT:    je LBB6_40
+; AVX1-NEXT:  LBB6_39: ## %cond.store55
+; AVX1-NEXT:    vextractps $3, %xmm2, (%rdi)
+; AVX1-NEXT:    addq $4, %rdi
+; AVX1-NEXT:  LBB6_40: ## %else56
+; AVX1-NEXT:    testl $1048576, %eax ## imm = 0x100000
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm0
+; AVX1-NEXT:    jne LBB6_41
+; AVX1-NEXT:  ## %bb.42: ## %else59
+; AVX1-NEXT:    testl $2097152, %eax ## imm = 0x200000
+; AVX1-NEXT:    jne LBB6_43
+; AVX1-NEXT:  LBB6_44: ## %else62
+; AVX1-NEXT:    testl $4194304, %eax ## imm = 0x400000
+; AVX1-NEXT:    jne LBB6_45
+; AVX1-NEXT:  LBB6_46: ## %else65
+; AVX1-NEXT:    testl $8388608, %eax ## imm = 0x800000
+; AVX1-NEXT:    jne LBB6_47
+; AVX1-NEXT:  LBB6_48: ## %else68
+; AVX1-NEXT:    testl $16777216, %eax ## imm = 0x1000000
+; AVX1-NEXT:    jne LBB6_49
+; AVX1-NEXT:  LBB6_50: ## %else71
+; AVX1-NEXT:    testl $33554432, %eax ## imm = 0x2000000
+; AVX1-NEXT:    jne LBB6_51
+; AVX1-NEXT:  LBB6_52: ## %else74
+; AVX1-NEXT:    testl $67108864, %eax ## imm = 0x4000000
+; AVX1-NEXT:    jne LBB6_53
+; AVX1-NEXT:  LBB6_54: ## %else77
+; AVX1-NEXT:    testl $134217728, %eax ## imm = 0x8000000
+; AVX1-NEXT:    je LBB6_56
+; AVX1-NEXT:  LBB6_55: ## %cond.store79
+; AVX1-NEXT:    vextractps $3, %xmm3, (%rdi)
+; AVX1-NEXT:    addq $4, %rdi
+; AVX1-NEXT:  LBB6_56: ## %else80
+; AVX1-NEXT:    testl $268435456, %eax ## imm = 0x10000000
+; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm0
+; AVX1-NEXT:    jne LBB6_57
+; AVX1-NEXT:  ## %bb.58: ## %else83
+; AVX1-NEXT:    testl $536870912, %eax ## imm = 0x20000000
+; AVX1-NEXT:    jne LBB6_59
+; AVX1-NEXT:  LBB6_60: ## %else86
+; AVX1-NEXT:    testl $1073741824, %eax ## imm = 0x40000000
+; AVX1-NEXT:    jne LBB6_61
+; AVX1-NEXT:  LBB6_62: ## %else89
+; AVX1-NEXT:    testl $-2147483648, %eax ## imm = 0x80000000
+; AVX1-NEXT:    jne LBB6_63
+; AVX1-NEXT:  LBB6_64: ## %else92
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+; AVX1-NEXT:  LBB6_1: ## %cond.store
+; AVX1-NEXT:    vmovss %xmm0, (%rdi)
+; AVX1-NEXT:    addq $4, %rdi
+; AVX1-NEXT:    testb $2, %al
+; AVX1-NEXT:    je LBB6_4
+; AVX1-NEXT:  LBB6_3: ## %cond.store1
+; AVX1-NEXT:    vextractps $1, %xmm0, (%rdi)
+; AVX1-NEXT:    addq $4, %rdi
+; AVX1-NEXT:    testb $4, %al
+; AVX1-NEXT:    je LBB6_6
+; AVX1-NEXT:  LBB6_5: ## %cond.store4
+; AVX1-NEXT:    vextractps $2, %xmm0, (%rdi)
+; AVX1-NEXT:    addq $4, %rdi
+; AVX1-NEXT:    testb $8, %al
+; AVX1-NEXT:    jne LBB6_7
+; AVX1-NEXT:    jmp LBB6_8
+; AVX1-NEXT:  LBB6_9: ## %cond.store10
+; AVX1-NEXT:    vmovss %xmm0, (%rdi)
+; AVX1-NEXT:    addq $4, %rdi
+; AVX1-NEXT:    testb $32, %al
 ; AVX1-NEXT:    je LBB6_12
-; AVX1-NEXT:  ## %bb.11: ## %cond.store13
-; AVX1-NEXT:    vextractps $1, %xmm10, (%rdi)
+; AVX1-NEXT:  LBB6_11: ## %cond.store13
+; AVX1-NEXT:    vextractps $1, %xmm0, (%rdi)
 ; AVX1-NEXT:    addq $4, %rdi
-; AVX1-NEXT:  LBB6_12: ## %else14
-; AVX1-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm8, %xmm4
-; AVX1-NEXT:    vpackssdw %xmm4, %xmm0, %xmm0
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
-; AVX1-NEXT:    vpextrb $6, %xmm0, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $64, %al
 ; AVX1-NEXT:    je LBB6_14
-; AVX1-NEXT:  ## %bb.13: ## %cond.store16
-; AVX1-NEXT:    vextractps $2, %xmm10, (%rdi)
+; AVX1-NEXT:  LBB6_13: ## %cond.store16
+; AVX1-NEXT:    vextractps $2, %xmm0, (%rdi)
 ; AVX1-NEXT:    addq $4, %rdi
-; AVX1-NEXT:  LBB6_14: ## %else17
-; AVX1-NEXT:    vpackssdw %xmm4, %xmm0, %xmm0
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
-; AVX1-NEXT:    vpextrb $7, %xmm0, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $-128, %al
 ; AVX1-NEXT:    je LBB6_16
-; AVX1-NEXT:  ## %bb.15: ## %cond.store19
-; AVX1-NEXT:    vextractps $3, %xmm10, (%rdi)
+; AVX1-NEXT:  LBB6_15: ## %cond.store19
+; AVX1-NEXT:    vextractps $3, %xmm0, (%rdi)
 ; AVX1-NEXT:    addq $4, %rdi
-; AVX1-NEXT:  LBB6_16: ## %else20
-; AVX1-NEXT:    vpxor %xmm0, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm5, %xmm4
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm4, %xmm4
-; AVX1-NEXT:    vpacksswb %xmm4, %xmm0, %xmm4
-; AVX1-NEXT:    vpextrb $8, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $256, %eax ## imm = 0x100
 ; AVX1-NEXT:    je LBB6_18
-; AVX1-NEXT:  ## %bb.17: ## %cond.store22
+; AVX1-NEXT:  LBB6_17: ## %cond.store22
 ; AVX1-NEXT:    vmovss %xmm1, (%rdi)
 ; AVX1-NEXT:    addq $4, %rdi
-; AVX1-NEXT:  LBB6_18: ## %else23
-; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm5, %xmm0
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm0, %xmm0
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
-; AVX1-NEXT:    vpextrb $9, %xmm0, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $512, %eax ## imm = 0x200
 ; AVX1-NEXT:    je LBB6_20
-; AVX1-NEXT:  ## %bb.19: ## %cond.store25
+; AVX1-NEXT:  LBB6_19: ## %cond.store25
 ; AVX1-NEXT:    vextractps $1, %xmm1, (%rdi)
 ; AVX1-NEXT:    addq $4, %rdi
-; AVX1-NEXT:  LBB6_20: ## %else26
-; AVX1-NEXT:    vpxor %xmm0, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm5, %xmm4
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm4, %xmm4
-; AVX1-NEXT:    vpacksswb %xmm4, %xmm0, %xmm4
-; AVX1-NEXT:    vpextrb $10, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $1024, %eax ## imm = 0x400
 ; AVX1-NEXT:    je LBB6_22
-; AVX1-NEXT:  ## %bb.21: ## %cond.store28
+; AVX1-NEXT:  LBB6_21: ## %cond.store28
 ; AVX1-NEXT:    vextractps $2, %xmm1, (%rdi)
 ; AVX1-NEXT:    addq $4, %rdi
-; AVX1-NEXT:  LBB6_22: ## %else29
-; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm5, %xmm0
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm0, %xmm0
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
-; AVX1-NEXT:    vpextrb $11, %xmm0, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je LBB6_24
-; AVX1-NEXT:  ## %bb.23: ## %cond.store31
-; AVX1-NEXT:    vextractps $3, %xmm1, (%rdi)
-; AVX1-NEXT:    addq $4, %rdi
-; AVX1-NEXT:  LBB6_24: ## %else32
-; AVX1-NEXT:    vextractf128 $1, %ymm5, %xmm4
-; AVX1-NEXT:    vpxor %xmm0, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm4, %xmm5
-; AVX1-NEXT:    vpackssdw %xmm5, %xmm0, %xmm0
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
-; AVX1-NEXT:    vpextrb $12, %xmm0, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm0
-; AVX1-NEXT:    je LBB6_26
-; AVX1-NEXT:  ## %bb.25: ## %cond.store34
-; AVX1-NEXT:    vmovd %xmm0, (%rdi)
+; AVX1-NEXT:    testl $2048, %eax ## imm = 0x800
+; AVX1-NEXT:    jne LBB6_23
+; AVX1-NEXT:    jmp LBB6_24
+; AVX1-NEXT:  LBB6_25: ## %cond.store34
+; AVX1-NEXT:    vmovss %xmm0, (%rdi)
 ; AVX1-NEXT:    addq $4, %rdi
-; AVX1-NEXT:  LBB6_26: ## %else35
-; AVX1-NEXT:    vpackssdw %xmm5, %xmm0, %xmm1
-; AVX1-NEXT:    vpacksswb %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpextrb $13, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $8192, %eax ## imm = 0x2000
 ; AVX1-NEXT:    je LBB6_28
-; AVX1-NEXT:  ## %bb.27: ## %cond.store37
-; AVX1-NEXT:    vpextrd $1, %xmm0, (%rdi)
+; AVX1-NEXT:  LBB6_27: ## %cond.store37
+; AVX1-NEXT:    vextractps $1, %xmm0, (%rdi)
 ; AVX1-NEXT:    addq $4, %rdi
-; AVX1-NEXT:  LBB6_28: ## %else38
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm4, %xmm1
-; AVX1-NEXT:    vpackssdw %xmm1, %xmm0, %xmm4
-; AVX1-NEXT:    vpacksswb %xmm4, %xmm0, %xmm4
-; AVX1-NEXT:    vpextrb $14, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $16384, %eax ## imm = 0x4000
 ; AVX1-NEXT:    je LBB6_30
-; AVX1-NEXT:  ## %bb.29: ## %cond.store40
-; AVX1-NEXT:    vpextrd $2, %xmm0, (%rdi)
+; AVX1-NEXT:  LBB6_29: ## %cond.store40
+; AVX1-NEXT:    vextractps $2, %xmm0, (%rdi)
 ; AVX1-NEXT:    addq $4, %rdi
-; AVX1-NEXT:  LBB6_30: ## %else41
-; AVX1-NEXT:    vpackssdw %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpacksswb %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpextrb $15, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $32768, %eax ## imm = 0x8000
 ; AVX1-NEXT:    je LBB6_32
-; AVX1-NEXT:  ## %bb.31: ## %cond.store43
-; AVX1-NEXT:    vpextrd $3, %xmm0, (%rdi)
+; AVX1-NEXT:  LBB6_31: ## %cond.store43
+; AVX1-NEXT:    vextractps $3, %xmm0, (%rdi)
 ; AVX1-NEXT:    addq $4, %rdi
-; AVX1-NEXT:  LBB6_32: ## %else44
-; AVX1-NEXT:    vpxor %xmm0, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm6, %xmm1
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm1, %xmm1
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $0, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $65536, %eax ## imm = 0x10000
 ; AVX1-NEXT:    je LBB6_34
-; AVX1-NEXT:  ## %bb.33: ## %cond.store46
+; AVX1-NEXT:  LBB6_33: ## %cond.store46
 ; AVX1-NEXT:    vmovss %xmm2, (%rdi)
 ; AVX1-NEXT:    addq $4, %rdi
-; AVX1-NEXT:  LBB6_34: ## %else47
-; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm6, %xmm0
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm0, %xmm0
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
-; AVX1-NEXT:    vpextrb $1, %xmm0, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $131072, %eax ## imm = 0x20000
 ; AVX1-NEXT:    je LBB6_36
-; AVX1-NEXT:  ## %bb.35: ## %cond.store49
+; AVX1-NEXT:  LBB6_35: ## %cond.store49
 ; AVX1-NEXT:    vextractps $1, %xmm2, (%rdi)
 ; AVX1-NEXT:    addq $4, %rdi
-; AVX1-NEXT:  LBB6_36: ## %else50
-; AVX1-NEXT:    vpxor %xmm0, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm6, %xmm1
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm1, %xmm1
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $2, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $262144, %eax ## imm = 0x40000
 ; AVX1-NEXT:    je LBB6_38
-; AVX1-NEXT:  ## %bb.37: ## %cond.store52
+; AVX1-NEXT:  LBB6_37: ## %cond.store52
 ; AVX1-NEXT:    vextractps $2, %xmm2, (%rdi)
 ; AVX1-NEXT:    addq $4, %rdi
-; AVX1-NEXT:  LBB6_38: ## %else53
-; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm6, %xmm0
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm0, %xmm0
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
-; AVX1-NEXT:    vpextrb $3, %xmm0, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je LBB6_40
-; AVX1-NEXT:  ## %bb.39: ## %cond.store55
-; AVX1-NEXT:    vextractps $3, %xmm2, (%rdi)
-; AVX1-NEXT:    addq $4, %rdi
-; AVX1-NEXT:  LBB6_40: ## %else56
-; AVX1-NEXT:    vextractf128 $1, %ymm6, %xmm1
-; AVX1-NEXT:    vpxor %xmm0, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm1, %xmm4
-; AVX1-NEXT:    vpackssdw %xmm4, %xmm0, %xmm0
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
-; AVX1-NEXT:    vpextrb $4, %xmm0, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm0
-; AVX1-NEXT:    je LBB6_42
-; AVX1-NEXT:  ## %bb.41: ## %cond.store58
-; AVX1-NEXT:    vmovd %xmm0, (%rdi)
+; AVX1-NEXT:    testl $524288, %eax ## imm = 0x80000
+; AVX1-NEXT:    jne LBB6_39
+; AVX1-NEXT:    jmp LBB6_40
+; AVX1-NEXT:  LBB6_41: ## %cond.store58
+; AVX1-NEXT:    vmovss %xmm0, (%rdi)
 ; AVX1-NEXT:    addq $4, %rdi
-; AVX1-NEXT:  LBB6_42: ## %else59
-; AVX1-NEXT:    vpackssdw %xmm4, %xmm0, %xmm2
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $5, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $2097152, %eax ## imm = 0x200000
 ; AVX1-NEXT:    je LBB6_44
-; AVX1-NEXT:  ## %bb.43: ## %cond.store61
-; AVX1-NEXT:    vpextrd $1, %xmm0, (%rdi)
+; AVX1-NEXT:  LBB6_43: ## %cond.store61
+; AVX1-NEXT:    vextractps $1, %xmm0, (%rdi)
 ; AVX1-NEXT:    addq $4, %rdi
-; AVX1-NEXT:  LBB6_44: ## %else62
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpackssdw %xmm1, %xmm0, %xmm2
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $6, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $4194304, %eax ## imm = 0x400000
 ; AVX1-NEXT:    je LBB6_46
-; AVX1-NEXT:  ## %bb.45: ## %cond.store64
-; AVX1-NEXT:    vpextrd $2, %xmm0, (%rdi)
+; AVX1-NEXT:  LBB6_45: ## %cond.store64
+; AVX1-NEXT:    vextractps $2, %xmm0, (%rdi)
 ; AVX1-NEXT:    addq $4, %rdi
-; AVX1-NEXT:  LBB6_46: ## %else65
-; AVX1-NEXT:    vpackssdw %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $7, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $8388608, %eax ## imm = 0x800000
 ; AVX1-NEXT:    je LBB6_48
-; AVX1-NEXT:  ## %bb.47: ## %cond.store67
-; AVX1-NEXT:    vpextrd $3, %xmm0, (%rdi)
+; AVX1-NEXT:  LBB6_47: ## %cond.store67
+; AVX1-NEXT:    vextractps $3, %xmm0, (%rdi)
 ; AVX1-NEXT:    addq $4, %rdi
-; AVX1-NEXT:  LBB6_48: ## %else68
-; AVX1-NEXT:    vpxor %xmm0, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm7, %xmm1
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm1, %xmm1
-; AVX1-NEXT:    vpacksswb %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $16777216, %eax ## imm = 0x1000000
 ; AVX1-NEXT:    je LBB6_50
-; AVX1-NEXT:  ## %bb.49: ## %cond.store70
+; AVX1-NEXT:  LBB6_49: ## %cond.store70
 ; AVX1-NEXT:    vmovss %xmm3, (%rdi)
 ; AVX1-NEXT:    addq $4, %rdi
-; AVX1-NEXT:  LBB6_50: ## %else71
-; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm7, %xmm0
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm0, %xmm0
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
-; AVX1-NEXT:    vpextrb $9, %xmm0, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $33554432, %eax ## imm = 0x2000000
 ; AVX1-NEXT:    je LBB6_52
-; AVX1-NEXT:  ## %bb.51: ## %cond.store73
+; AVX1-NEXT:  LBB6_51: ## %cond.store73
 ; AVX1-NEXT:    vextractps $1, %xmm3, (%rdi)
 ; AVX1-NEXT:    addq $4, %rdi
-; AVX1-NEXT:  LBB6_52: ## %else74
-; AVX1-NEXT:    vpxor %xmm0, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm7, %xmm1
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm1, %xmm1
-; AVX1-NEXT:    vpacksswb %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpextrb $10, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $67108864, %eax ## imm = 0x4000000
 ; AVX1-NEXT:    je LBB6_54
-; AVX1-NEXT:  ## %bb.53: ## %cond.store76
+; AVX1-NEXT:  LBB6_53: ## %cond.store76
 ; AVX1-NEXT:    vextractps $2, %xmm3, (%rdi)
 ; AVX1-NEXT:    addq $4, %rdi
-; AVX1-NEXT:  LBB6_54: ## %else77
-; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm7, %xmm0
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm0, %xmm0
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
-; AVX1-NEXT:    vpextrb $11, %xmm0, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je LBB6_56
-; AVX1-NEXT:  ## %bb.55: ## %cond.store79
-; AVX1-NEXT:    vextractps $3, %xmm3, (%rdi)
-; AVX1-NEXT:    addq $4, %rdi
-; AVX1-NEXT:  LBB6_56: ## %else80
-; AVX1-NEXT:    vextractf128 $1, %ymm7, %xmm1
-; AVX1-NEXT:    vpxor %xmm0, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm1, %xmm2
-; AVX1-NEXT:    vpackssdw %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
-; AVX1-NEXT:    vpextrb $12, %xmm0, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm0
-; AVX1-NEXT:    je LBB6_58
-; AVX1-NEXT:  ## %bb.57: ## %cond.store82
-; AVX1-NEXT:    vmovd %xmm0, (%rdi)
+; AVX1-NEXT:    testl $134217728, %eax ## imm = 0x8000000
+; AVX1-NEXT:    jne LBB6_55
+; AVX1-NEXT:    jmp LBB6_56
+; AVX1-NEXT:  LBB6_57: ## %cond.store82
+; AVX1-NEXT:    vmovss %xmm0, (%rdi)
 ; AVX1-NEXT:    addq $4, %rdi
-; AVX1-NEXT:  LBB6_58: ## %else83
-; AVX1-NEXT:    vpackssdw %xmm2, %xmm0, %xmm2
-; AVX1-NEXT:    vpacksswb %xmm2, %xmm0, %xmm2
-; AVX1-NEXT:    vpextrb $13, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $536870912, %eax ## imm = 0x20000000
 ; AVX1-NEXT:    je LBB6_60
-; AVX1-NEXT:  ## %bb.59: ## %cond.store85
-; AVX1-NEXT:    vpextrd $1, %xmm0, (%rdi)
+; AVX1-NEXT:  LBB6_59: ## %cond.store85
+; AVX1-NEXT:    vextractps $1, %xmm0, (%rdi)
 ; AVX1-NEXT:    addq $4, %rdi
-; AVX1-NEXT:  LBB6_60: ## %else86
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpackssdw %xmm1, %xmm0, %xmm2
-; AVX1-NEXT:    vpacksswb %xmm2, %xmm0, %xmm2
-; AVX1-NEXT:    vpextrb $14, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $1073741824, %eax ## imm = 0x40000000
 ; AVX1-NEXT:    je LBB6_62
-; AVX1-NEXT:  ## %bb.61: ## %cond.store88
-; AVX1-NEXT:    vpextrd $2, %xmm0, (%rdi)
+; AVX1-NEXT:  LBB6_61: ## %cond.store88
+; AVX1-NEXT:    vextractps $2, %xmm0, (%rdi)
 ; AVX1-NEXT:    addq $4, %rdi
-; AVX1-NEXT:  LBB6_62: ## %else89
-; AVX1-NEXT:    vpackssdw %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpacksswb %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpextrb $15, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $-2147483648, %eax ## imm = 0x80000000
 ; AVX1-NEXT:    je LBB6_64
-; AVX1-NEXT:  ## %bb.63: ## %cond.store91
-; AVX1-NEXT:    vpextrd $3, %xmm0, (%rdi)
-; AVX1-NEXT:  LBB6_64: ## %else92
+; AVX1-NEXT:  LBB6_63: ## %cond.store91
+; AVX1-NEXT:    vextractps $3, %xmm0, (%rdi)
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: compressstore_v32f32_v32i32:
 ; AVX2:       ## %bb.0:
 ; AVX2-NEXT:    vpxor %xmm8, %xmm8, %xmm8
-; AVX2-NEXT:    vpcmpeqd %xmm8, %xmm4, %xmm9
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm9, %xmm9
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm9, %xmm9
-; AVX2-NEXT:    vpextrb $0, %xmm9, %eax
+; AVX2-NEXT:    vpcmpeqd %ymm7, %ymm8, %ymm7
+; AVX2-NEXT:    vpcmpeqd %ymm6, %ymm8, %ymm6
+; AVX2-NEXT:    vpackssdw %ymm7, %ymm6, %ymm6
+; AVX2-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[0,2,1,3]
+; AVX2-NEXT:    vpcmpeqd %ymm5, %ymm8, %ymm5
+; AVX2-NEXT:    vpcmpeqd %ymm4, %ymm8, %ymm4
+; AVX2-NEXT:    vpackssdw %ymm5, %ymm4, %ymm4
+; AVX2-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[0,2,1,3]
+; AVX2-NEXT:    vpacksswb %ymm6, %ymm4, %ymm4
+; AVX2-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[0,2,1,3]
+; AVX2-NEXT:    vpmovmskb %ymm4, %eax
 ; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je LBB6_2
-; AVX2-NEXT:  ## %bb.1: ## %cond.store
-; AVX2-NEXT:    vmovd %xmm0, (%rdi)
-; AVX2-NEXT:    addq $4, %rdi
-; AVX2-NEXT:  LBB6_2: ## %else
-; AVX2-NEXT:    vpcmpeqd %xmm8, %xmm4, %xmm8
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm8, %xmm8
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm8, %xmm8
-; AVX2-NEXT:    vpextrb $1, %xmm8, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je LBB6_4
-; AVX2-NEXT:  ## %bb.3: ## %cond.store1
-; AVX2-NEXT:    vpextrd $1, %xmm0, (%rdi)
-; AVX2-NEXT:    addq $4, %rdi
+; AVX2-NEXT:    jne LBB6_1
+; AVX2-NEXT:  ## %bb.2: ## %else
+; AVX2-NEXT:    testb $2, %al
+; AVX2-NEXT:    jne LBB6_3
 ; AVX2-NEXT:  LBB6_4: ## %else2
-; AVX2-NEXT:    vpxor %xmm8, %xmm8, %xmm8
-; AVX2-NEXT:    vpcmpeqd %xmm8, %xmm4, %xmm9
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm9, %xmm9
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm9, %xmm9
-; AVX2-NEXT:    vpextrb $2, %xmm9, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je LBB6_6
-; AVX2-NEXT:  ## %bb.5: ## %cond.store4
-; AVX2-NEXT:    vpextrd $2, %xmm0, (%rdi)
-; AVX2-NEXT:    addq $4, %rdi
+; AVX2-NEXT:    testb $4, %al
+; AVX2-NEXT:    jne LBB6_5
 ; AVX2-NEXT:  LBB6_6: ## %else5
-; AVX2-NEXT:    vpcmpeqd %xmm8, %xmm4, %xmm8
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm8, %xmm8
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm8, %xmm8
-; AVX2-NEXT:    vpextrb $3, %xmm8, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $8, %al
 ; AVX2-NEXT:    je LBB6_8
-; AVX2-NEXT:  ## %bb.7: ## %cond.store7
-; AVX2-NEXT:    vpextrd $3, %xmm0, (%rdi)
+; AVX2-NEXT:  LBB6_7: ## %cond.store7
+; AVX2-NEXT:    vextractps $3, %xmm0, (%rdi)
 ; AVX2-NEXT:    addq $4, %rdi
 ; AVX2-NEXT:  LBB6_8: ## %else8
-; AVX2-NEXT:    vextracti128 $1, %ymm4, %xmm8
-; AVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm8, %xmm9
-; AVX2-NEXT:    vpackssdw %xmm9, %xmm0, %xmm4
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm4, %xmm4
-; AVX2-NEXT:    vpextrb $4, %xmm4, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm10
-; AVX2-NEXT:    je LBB6_10
-; AVX2-NEXT:  ## %bb.9: ## %cond.store10
-; AVX2-NEXT:    vmovd %xmm10, (%rdi)
+; AVX2-NEXT:    testb $16, %al
+; AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX2-NEXT:    jne LBB6_9
+; AVX2-NEXT:  ## %bb.10: ## %else11
+; AVX2-NEXT:    testb $32, %al
+; AVX2-NEXT:    jne LBB6_11
+; AVX2-NEXT:  LBB6_12: ## %else14
+; AVX2-NEXT:    testb $64, %al
+; AVX2-NEXT:    jne LBB6_13
+; AVX2-NEXT:  LBB6_14: ## %else17
+; AVX2-NEXT:    testb $-128, %al
+; AVX2-NEXT:    jne LBB6_15
+; AVX2-NEXT:  LBB6_16: ## %else20
+; AVX2-NEXT:    testl $256, %eax ## imm = 0x100
+; AVX2-NEXT:    jne LBB6_17
+; AVX2-NEXT:  LBB6_18: ## %else23
+; AVX2-NEXT:    testl $512, %eax ## imm = 0x200
+; AVX2-NEXT:    jne LBB6_19
+; AVX2-NEXT:  LBB6_20: ## %else26
+; AVX2-NEXT:    testl $1024, %eax ## imm = 0x400
+; AVX2-NEXT:    jne LBB6_21
+; AVX2-NEXT:  LBB6_22: ## %else29
+; AVX2-NEXT:    testl $2048, %eax ## imm = 0x800
+; AVX2-NEXT:    je LBB6_24
+; AVX2-NEXT:  LBB6_23: ## %cond.store31
+; AVX2-NEXT:    vextractps $3, %xmm1, (%rdi)
 ; AVX2-NEXT:    addq $4, %rdi
-; AVX2-NEXT:  LBB6_10: ## %else11
-; AVX2-NEXT:    vpackssdw %xmm9, %xmm0, %xmm4
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm4, %xmm4
-; AVX2-NEXT:    vpextrb $5, %xmm4, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:  LBB6_24: ## %else32
+; AVX2-NEXT:    testl $4096, %eax ## imm = 0x1000
+; AVX2-NEXT:    vextractf128 $1, %ymm1, %xmm0
+; AVX2-NEXT:    jne LBB6_25
+; AVX2-NEXT:  ## %bb.26: ## %else35
+; AVX2-NEXT:    testl $8192, %eax ## imm = 0x2000
+; AVX2-NEXT:    jne LBB6_27
+; AVX2-NEXT:  LBB6_28: ## %else38
+; AVX2-NEXT:    testl $16384, %eax ## imm = 0x4000
+; AVX2-NEXT:    jne LBB6_29
+; AVX2-NEXT:  LBB6_30: ## %else41
+; AVX2-NEXT:    testl $32768, %eax ## imm = 0x8000
+; AVX2-NEXT:    jne LBB6_31
+; AVX2-NEXT:  LBB6_32: ## %else44
+; AVX2-NEXT:    testl $65536, %eax ## imm = 0x10000
+; AVX2-NEXT:    jne LBB6_33
+; AVX2-NEXT:  LBB6_34: ## %else47
+; AVX2-NEXT:    testl $131072, %eax ## imm = 0x20000
+; AVX2-NEXT:    jne LBB6_35
+; AVX2-NEXT:  LBB6_36: ## %else50
+; AVX2-NEXT:    testl $262144, %eax ## imm = 0x40000
+; AVX2-NEXT:    jne LBB6_37
+; AVX2-NEXT:  LBB6_38: ## %else53
+; AVX2-NEXT:    testl $524288, %eax ## imm = 0x80000
+; AVX2-NEXT:    je LBB6_40
+; AVX2-NEXT:  LBB6_39: ## %cond.store55
+; AVX2-NEXT:    vextractps $3, %xmm2, (%rdi)
+; AVX2-NEXT:    addq $4, %rdi
+; AVX2-NEXT:  LBB6_40: ## %else56
+; AVX2-NEXT:    testl $1048576, %eax ## imm = 0x100000
+; AVX2-NEXT:    vextractf128 $1, %ymm2, %xmm0
+; AVX2-NEXT:    jne LBB6_41
+; AVX2-NEXT:  ## %bb.42: ## %else59
+; AVX2-NEXT:    testl $2097152, %eax ## imm = 0x200000
+; AVX2-NEXT:    jne LBB6_43
+; AVX2-NEXT:  LBB6_44: ## %else62
+; AVX2-NEXT:    testl $4194304, %eax ## imm = 0x400000
+; AVX2-NEXT:    jne LBB6_45
+; AVX2-NEXT:  LBB6_46: ## %else65
+; AVX2-NEXT:    testl $8388608, %eax ## imm = 0x800000
+; AVX2-NEXT:    jne LBB6_47
+; AVX2-NEXT:  LBB6_48: ## %else68
+; AVX2-NEXT:    testl $16777216, %eax ## imm = 0x1000000
+; AVX2-NEXT:    jne LBB6_49
+; AVX2-NEXT:  LBB6_50: ## %else71
+; AVX2-NEXT:    testl $33554432, %eax ## imm = 0x2000000
+; AVX2-NEXT:    jne LBB6_51
+; AVX2-NEXT:  LBB6_52: ## %else74
+; AVX2-NEXT:    testl $67108864, %eax ## imm = 0x4000000
+; AVX2-NEXT:    jne LBB6_53
+; AVX2-NEXT:  LBB6_54: ## %else77
+; AVX2-NEXT:    testl $134217728, %eax ## imm = 0x8000000
+; AVX2-NEXT:    je LBB6_56
+; AVX2-NEXT:  LBB6_55: ## %cond.store79
+; AVX2-NEXT:    vextractps $3, %xmm3, (%rdi)
+; AVX2-NEXT:    addq $4, %rdi
+; AVX2-NEXT:  LBB6_56: ## %else80
+; AVX2-NEXT:    testl $268435456, %eax ## imm = 0x10000000
+; AVX2-NEXT:    vextractf128 $1, %ymm3, %xmm0
+; AVX2-NEXT:    jne LBB6_57
+; AVX2-NEXT:  ## %bb.58: ## %else83
+; AVX2-NEXT:    testl $536870912, %eax ## imm = 0x20000000
+; AVX2-NEXT:    jne LBB6_59
+; AVX2-NEXT:  LBB6_60: ## %else86
+; AVX2-NEXT:    testl $1073741824, %eax ## imm = 0x40000000
+; AVX2-NEXT:    jne LBB6_61
+; AVX2-NEXT:  LBB6_62: ## %else89
+; AVX2-NEXT:    testl $-2147483648, %eax ## imm = 0x80000000
+; AVX2-NEXT:    jne LBB6_63
+; AVX2-NEXT:  LBB6_64: ## %else92
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+; AVX2-NEXT:  LBB6_1: ## %cond.store
+; AVX2-NEXT:    vmovss %xmm0, (%rdi)
+; AVX2-NEXT:    addq $4, %rdi
+; AVX2-NEXT:    testb $2, %al
+; AVX2-NEXT:    je LBB6_4
+; AVX2-NEXT:  LBB6_3: ## %cond.store1
+; AVX2-NEXT:    vextractps $1, %xmm0, (%rdi)
+; AVX2-NEXT:    addq $4, %rdi
+; AVX2-NEXT:    testb $4, %al
+; AVX2-NEXT:    je LBB6_6
+; AVX2-NEXT:  LBB6_5: ## %cond.store4
+; AVX2-NEXT:    vextractps $2, %xmm0, (%rdi)
+; AVX2-NEXT:    addq $4, %rdi
+; AVX2-NEXT:    testb $8, %al
+; AVX2-NEXT:    jne LBB6_7
+; AVX2-NEXT:    jmp LBB6_8
+; AVX2-NEXT:  LBB6_9: ## %cond.store10
+; AVX2-NEXT:    vmovss %xmm0, (%rdi)
+; AVX2-NEXT:    addq $4, %rdi
+; AVX2-NEXT:    testb $32, %al
 ; AVX2-NEXT:    je LBB6_12
-; AVX2-NEXT:  ## %bb.11: ## %cond.store13
-; AVX2-NEXT:    vpextrd $1, %xmm10, (%rdi)
+; AVX2-NEXT:  LBB6_11: ## %cond.store13
+; AVX2-NEXT:    vextractps $1, %xmm0, (%rdi)
 ; AVX2-NEXT:    addq $4, %rdi
-; AVX2-NEXT:  LBB6_12: ## %else14
-; AVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm8, %xmm4
-; AVX2-NEXT:    vpackssdw %xmm4, %xmm0, %xmm0
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
-; AVX2-NEXT:    vpextrb $6, %xmm0, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $64, %al
 ; AVX2-NEXT:    je LBB6_14
-; AVX2-NEXT:  ## %bb.13: ## %cond.store16
-; AVX2-NEXT:    vpextrd $2, %xmm10, (%rdi)
+; AVX2-NEXT:  LBB6_13: ## %cond.store16
+; AVX2-NEXT:    vextractps $2, %xmm0, (%rdi)
 ; AVX2-NEXT:    addq $4, %rdi
-; AVX2-NEXT:  LBB6_14: ## %else17
-; AVX2-NEXT:    vpackssdw %xmm4, %xmm0, %xmm0
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
-; AVX2-NEXT:    vpextrb $7, %xmm0, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $-128, %al
 ; AVX2-NEXT:    je LBB6_16
-; AVX2-NEXT:  ## %bb.15: ## %cond.store19
-; AVX2-NEXT:    vpextrd $3, %xmm10, (%rdi)
+; AVX2-NEXT:  LBB6_15: ## %cond.store19
+; AVX2-NEXT:    vextractps $3, %xmm0, (%rdi)
 ; AVX2-NEXT:    addq $4, %rdi
-; AVX2-NEXT:  LBB6_16: ## %else20
-; AVX2-NEXT:    vpxor %xmm0, %xmm0, %xmm0
-; AVX2-NEXT:    vpcmpeqd %xmm0, %xmm5, %xmm4
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm4, %xmm4
-; AVX2-NEXT:    vpacksswb %xmm4, %xmm0, %xmm4
-; AVX2-NEXT:    vpextrb $8, %xmm4, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $256, %eax ## imm = 0x100
 ; AVX2-NEXT:    je LBB6_18
-; AVX2-NEXT:  ## %bb.17: ## %cond.store22
-; AVX2-NEXT:    vmovd %xmm1, (%rdi)
+; AVX2-NEXT:  LBB6_17: ## %cond.store22
+; AVX2-NEXT:    vmovss %xmm1, (%rdi)
 ; AVX2-NEXT:    addq $4, %rdi
-; AVX2-NEXT:  LBB6_18: ## %else23
-; AVX2-NEXT:    vpcmpeqd %xmm0, %xmm5, %xmm0
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm0, %xmm0
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
-; AVX2-NEXT:    vpextrb $9, %xmm0, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $512, %eax ## imm = 0x200
 ; AVX2-NEXT:    je LBB6_20
-; AVX2-NEXT:  ## %bb.19: ## %cond.store25
-; AVX2-NEXT:    vpextrd $1, %xmm1, (%rdi)
+; AVX2-NEXT:  LBB6_19: ## %cond.store25
+; AVX2-NEXT:    vextractps $1, %xmm1, (%rdi)
 ; AVX2-NEXT:    addq $4, %rdi
-; AVX2-NEXT:  LBB6_20: ## %else26
-; AVX2-NEXT:    vpxor %xmm0, %xmm0, %xmm0
-; AVX2-NEXT:    vpcmpeqd %xmm0, %xmm5, %xmm4
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm4, %xmm4
-; AVX2-NEXT:    vpacksswb %xmm4, %xmm0, %xmm4
-; AVX2-NEXT:    vpextrb $10, %xmm4, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $1024, %eax ## imm = 0x400
 ; AVX2-NEXT:    je LBB6_22
-; AVX2-NEXT:  ## %bb.21: ## %cond.store28
-; AVX2-NEXT:    vpextrd $2, %xmm1, (%rdi)
+; AVX2-NEXT:  LBB6_21: ## %cond.store28
+; AVX2-NEXT:    vextractps $2, %xmm1, (%rdi)
 ; AVX2-NEXT:    addq $4, %rdi
-; AVX2-NEXT:  LBB6_22: ## %else29
-; AVX2-NEXT:    vpcmpeqd %xmm0, %xmm5, %xmm0
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm0, %xmm0
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
-; AVX2-NEXT:    vpextrb $11, %xmm0, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je LBB6_24
-; AVX2-NEXT:  ## %bb.23: ## %cond.store31
-; AVX2-NEXT:    vpextrd $3, %xmm1, (%rdi)
-; AVX2-NEXT:    addq $4, %rdi
-; AVX2-NEXT:  LBB6_24: ## %else32
-; AVX2-NEXT:    vextracti128 $1, %ymm5, %xmm4
-; AVX2-NEXT:    vpxor %xmm0, %xmm0, %xmm0
-; AVX2-NEXT:    vpcmpeqd %xmm0, %xmm4, %xmm5
-; AVX2-NEXT:    vpackssdw %xmm5, %xmm0, %xmm0
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
-; AVX2-NEXT:    vpextrb $12, %xmm0, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm0
-; AVX2-NEXT:    je LBB6_26
-; AVX2-NEXT:  ## %bb.25: ## %cond.store34
-; AVX2-NEXT:    vmovd %xmm0, (%rdi)
+; AVX2-NEXT:    testl $2048, %eax ## imm = 0x800
+; AVX2-NEXT:    jne LBB6_23
+; AVX2-NEXT:    jmp LBB6_24
+; AVX2-NEXT:  LBB6_25: ## %cond.store34
+; AVX2-NEXT:    vmovss %xmm0, (%rdi)
 ; AVX2-NEXT:    addq $4, %rdi
-; AVX2-NEXT:  LBB6_26: ## %else35
-; AVX2-NEXT:    vpackssdw %xmm5, %xmm0, %xmm1
-; AVX2-NEXT:    vpacksswb %xmm1, %xmm0, %xmm1
-; AVX2-NEXT:    vpextrb $13, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $8192, %eax ## imm = 0x2000
 ; AVX2-NEXT:    je LBB6_28
-; AVX2-NEXT:  ## %bb.27: ## %cond.store37
-; AVX2-NEXT:    vpextrd $1, %xmm0, (%rdi)
+; AVX2-NEXT:  LBB6_27: ## %cond.store37
+; AVX2-NEXT:    vextractps $1, %xmm0, (%rdi)
 ; AVX2-NEXT:    addq $4, %rdi
-; AVX2-NEXT:  LBB6_28: ## %else38
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm4, %xmm1
-; AVX2-NEXT:    vpackssdw %xmm1, %xmm0, %xmm4
-; AVX2-NEXT:    vpacksswb %xmm4, %xmm0, %xmm4
-; AVX2-NEXT:    vpextrb $14, %xmm4, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $16384, %eax ## imm = 0x4000
 ; AVX2-NEXT:    je LBB6_30
-; AVX2-NEXT:  ## %bb.29: ## %cond.store40
-; AVX2-NEXT:    vpextrd $2, %xmm0, (%rdi)
+; AVX2-NEXT:  LBB6_29: ## %cond.store40
+; AVX2-NEXT:    vextractps $2, %xmm0, (%rdi)
 ; AVX2-NEXT:    addq $4, %rdi
-; AVX2-NEXT:  LBB6_30: ## %else41
-; AVX2-NEXT:    vpackssdw %xmm1, %xmm0, %xmm1
-; AVX2-NEXT:    vpacksswb %xmm1, %xmm0, %xmm1
-; AVX2-NEXT:    vpextrb $15, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $32768, %eax ## imm = 0x8000
 ; AVX2-NEXT:    je LBB6_32
-; AVX2-NEXT:  ## %bb.31: ## %cond.store43
-; AVX2-NEXT:    vpextrd $3, %xmm0, (%rdi)
+; AVX2-NEXT:  LBB6_31: ## %cond.store43
+; AVX2-NEXT:    vextractps $3, %xmm0, (%rdi)
 ; AVX2-NEXT:    addq $4, %rdi
-; AVX2-NEXT:  LBB6_32: ## %else44
-; AVX2-NEXT:    vpxor %xmm0, %xmm0, %xmm0
-; AVX2-NEXT:    vpcmpeqd %xmm0, %xmm6, %xmm1
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm1, %xmm1
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm1, %xmm1
-; AVX2-NEXT:    vpextrb $0, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $65536, %eax ## imm = 0x10000
 ; AVX2-NEXT:    je LBB6_34
-; AVX2-NEXT:  ## %bb.33: ## %cond.store46
-; AVX2-NEXT:    vmovd %xmm2, (%rdi)
+; AVX2-NEXT:  LBB6_33: ## %cond.store46
+; AVX2-NEXT:    vmovss %xmm2, (%rdi)
 ; AVX2-NEXT:    addq $4, %rdi
-; AVX2-NEXT:  LBB6_34: ## %else47
-; AVX2-NEXT:    vpcmpeqd %xmm0, %xmm6, %xmm0
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm0, %xmm0
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
-; AVX2-NEXT:    vpextrb $1, %xmm0, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $131072, %eax ## imm = 0x20000
 ; AVX2-NEXT:    je LBB6_36
-; AVX2-NEXT:  ## %bb.35: ## %cond.store49
-; AVX2-NEXT:    vpextrd $1, %xmm2, (%rdi)
+; AVX2-NEXT:  LBB6_35: ## %cond.store49
+; AVX2-NEXT:    vextractps $1, %xmm2, (%rdi)
 ; AVX2-NEXT:    addq $4, %rdi
-; AVX2-NEXT:  LBB6_36: ## %else50
-; AVX2-NEXT:    vpxor %xmm0, %xmm0, %xmm0
-; AVX2-NEXT:    vpcmpeqd %xmm0, %xmm6, %xmm1
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm1, %xmm1
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm1, %xmm1
-; AVX2-NEXT:    vpextrb $2, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $262144, %eax ## imm = 0x40000
 ; AVX2-NEXT:    je LBB6_38
-; AVX2-NEXT:  ## %bb.37: ## %cond.store52
-; AVX2-NEXT:    vpextrd $2, %xmm2, (%rdi)
-; AVX2-NEXT:    addq $4, %rdi
-; AVX2-NEXT:  LBB6_38: ## %else53
-; AVX2-NEXT:    vpcmpeqd %xmm0, %xmm6, %xmm0
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm0, %xmm0
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
-; AVX2-NEXT:    vpextrb $3, %xmm0, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je LBB6_40
-; AVX2-NEXT:  ## %bb.39: ## %cond.store55
-; AVX2-NEXT:    vpextrd $3, %xmm2, (%rdi)
+; AVX2-NEXT:  LBB6_37: ## %cond.store52
+; AVX2-NEXT:    vextractps $2, %xmm2, (%rdi)
 ; AVX2-NEXT:    addq $4, %rdi
-; AVX2-NEXT:  LBB6_40: ## %else56
-; AVX2-NEXT:    vextracti128 $1, %ymm6, %xmm1
-; AVX2-NEXT:    vpxor %xmm0, %xmm0, %xmm0
-; AVX2-NEXT:    vpcmpeqd %xmm0, %xmm1, %xmm4
-; AVX2-NEXT:    vpackssdw %xmm4, %xmm0, %xmm0
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
-; AVX2-NEXT:    vpextrb $4, %xmm0, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm0
-; AVX2-NEXT:    je LBB6_42
-; AVX2-NEXT:  ## %bb.41: ## %cond.store58
-; AVX2-NEXT:    vmovd %xmm0, (%rdi)
+; AVX2-NEXT:    testl $524288, %eax ## imm = 0x80000
+; AVX2-NEXT:    jne LBB6_39
+; AVX2-NEXT:    jmp LBB6_40
+; AVX2-NEXT:  LBB6_41: ## %cond.store58
+; AVX2-NEXT:    vmovss %xmm0, (%rdi)
 ; AVX2-NEXT:    addq $4, %rdi
-; AVX2-NEXT:  LBB6_42: ## %else59
-; AVX2-NEXT:    vpackssdw %xmm4, %xmm0, %xmm2
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrb $5, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $2097152, %eax ## imm = 0x200000
 ; AVX2-NEXT:    je LBB6_44
-; AVX2-NEXT:  ## %bb.43: ## %cond.store61
-; AVX2-NEXT:    vpextrd $1, %xmm0, (%rdi)
+; AVX2-NEXT:  LBB6_43: ## %cond.store61
+; AVX2-NEXT:    vextractps $1, %xmm0, (%rdi)
 ; AVX2-NEXT:    addq $4, %rdi
-; AVX2-NEXT:  LBB6_44: ## %else62
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpackssdw %xmm1, %xmm0, %xmm2
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrb $6, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $4194304, %eax ## imm = 0x400000
 ; AVX2-NEXT:    je LBB6_46
-; AVX2-NEXT:  ## %bb.45: ## %cond.store64
-; AVX2-NEXT:    vpextrd $2, %xmm0, (%rdi)
+; AVX2-NEXT:  LBB6_45: ## %cond.store64
+; AVX2-NEXT:    vextractps $2, %xmm0, (%rdi)
 ; AVX2-NEXT:    addq $4, %rdi
-; AVX2-NEXT:  LBB6_46: ## %else65
-; AVX2-NEXT:    vpackssdw %xmm1, %xmm0, %xmm1
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm1, %xmm1
-; AVX2-NEXT:    vpextrb $7, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $8388608, %eax ## imm = 0x800000
 ; AVX2-NEXT:    je LBB6_48
-; AVX2-NEXT:  ## %bb.47: ## %cond.store67
-; AVX2-NEXT:    vpextrd $3, %xmm0, (%rdi)
+; AVX2-NEXT:  LBB6_47: ## %cond.store67
+; AVX2-NEXT:    vextractps $3, %xmm0, (%rdi)
 ; AVX2-NEXT:    addq $4, %rdi
-; AVX2-NEXT:  LBB6_48: ## %else68
-; AVX2-NEXT:    vpxor %xmm0, %xmm0, %xmm0
-; AVX2-NEXT:    vpcmpeqd %xmm0, %xmm7, %xmm1
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm1, %xmm1
-; AVX2-NEXT:    vpacksswb %xmm1, %xmm0, %xmm1
-; AVX2-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $16777216, %eax ## imm = 0x1000000
 ; AVX2-NEXT:    je LBB6_50
-; AVX2-NEXT:  ## %bb.49: ## %cond.store70
-; AVX2-NEXT:    vmovd %xmm3, (%rdi)
+; AVX2-NEXT:  LBB6_49: ## %cond.store70
+; AVX2-NEXT:    vmovss %xmm3, (%rdi)
 ; AVX2-NEXT:    addq $4, %rdi
-; AVX2-NEXT:  LBB6_50: ## %else71
-; AVX2-NEXT:    vpcmpeqd %xmm0, %xmm7, %xmm0
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm0, %xmm0
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
-; AVX2-NEXT:    vpextrb $9, %xmm0, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $33554432, %eax ## imm = 0x2000000
 ; AVX2-NEXT:    je LBB6_52
-; AVX2-NEXT:  ## %bb.51: ## %cond.store73
-; AVX2-NEXT:    vpextrd $1, %xmm3, (%rdi)
+; AVX2-NEXT:  LBB6_51: ## %cond.store73
+; AVX2-NEXT:    vextractps $1, %xmm3, (%rdi)
 ; AVX2-NEXT:    addq $4, %rdi
-; AVX2-NEXT:  LBB6_52: ## %else74
-; AVX2-NEXT:    vpxor %xmm0, %xmm0, %xmm0
-; AVX2-NEXT:    vpcmpeqd %xmm0, %xmm7, %xmm1
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm1, %xmm1
-; AVX2-NEXT:    vpacksswb %xmm1, %xmm0, %xmm1
-; AVX2-NEXT:    vpextrb $10, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $67108864, %eax ## imm = 0x4000000
 ; AVX2-NEXT:    je LBB6_54
-; AVX2-NEXT:  ## %bb.53: ## %cond.store76
-; AVX2-NEXT:    vpextrd $2, %xmm3, (%rdi)
-; AVX2-NEXT:    addq $4, %rdi
-; AVX2-NEXT:  LBB6_54: ## %else77
-; AVX2-NEXT:    vpcmpeqd %xmm0, %xmm7, %xmm0
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm0, %xmm0
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
-; AVX2-NEXT:    vpextrb $11, %xmm0, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je LBB6_56
-; AVX2-NEXT:  ## %bb.55: ## %cond.store79
-; AVX2-NEXT:    vpextrd $3, %xmm3, (%rdi)
+; AVX2-NEXT:  LBB6_53: ## %cond.store76
+; AVX2-NEXT:    vextractps $2, %xmm3, (%rdi)
 ; AVX2-NEXT:    addq $4, %rdi
-; AVX2-NEXT:  LBB6_56: ## %else80
-; AVX2-NEXT:    vextracti128 $1, %ymm7, %xmm1
-; AVX2-NEXT:    vpxor %xmm0, %xmm0, %xmm0
-; AVX2-NEXT:    vpcmpeqd %xmm0, %xmm1, %xmm2
-; AVX2-NEXT:    vpackssdw %xmm2, %xmm0, %xmm0
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
-; AVX2-NEXT:    vpextrb $12, %xmm0, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    vextracti128 $1, %ymm3, %xmm0
-; AVX2-NEXT:    je LBB6_58
-; AVX2-NEXT:  ## %bb.57: ## %cond.store82
-; AVX2-NEXT:    vmovd %xmm0, (%rdi)
+; AVX2-NEXT:    testl $134217728, %eax ## imm = 0x8000000
+; AVX2-NEXT:    jne LBB6_55
+; AVX2-NEXT:    jmp LBB6_56
+; AVX2-NEXT:  LBB6_57: ## %cond.store82
+; AVX2-NEXT:    vmovss %xmm0, (%rdi)
 ; AVX2-NEXT:    addq $4, %rdi
-; AVX2-NEXT:  LBB6_58: ## %else83
-; AVX2-NEXT:    vpackssdw %xmm2, %xmm0, %xmm2
-; AVX2-NEXT:    vpacksswb %xmm2, %xmm0, %xmm2
-; AVX2-NEXT:    vpextrb $13, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $536870912, %eax ## imm = 0x20000000
 ; AVX2-NEXT:    je LBB6_60
-; AVX2-NEXT:  ## %bb.59: ## %cond.store85
-; AVX2-NEXT:    vpextrd $1, %xmm0, (%rdi)
+; AVX2-NEXT:  LBB6_59: ## %cond.store85
+; AVX2-NEXT:    vextractps $1, %xmm0, (%rdi)
 ; AVX2-NEXT:    addq $4, %rdi
-; AVX2-NEXT:  LBB6_60: ## %else86
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpackssdw %xmm1, %xmm0, %xmm2
-; AVX2-NEXT:    vpacksswb %xmm2, %xmm0, %xmm2
-; AVX2-NEXT:    vpextrb $14, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $1073741824, %eax ## imm = 0x40000000
 ; AVX2-NEXT:    je LBB6_62
-; AVX2-NEXT:  ## %bb.61: ## %cond.store88
-; AVX2-NEXT:    vpextrd $2, %xmm0, (%rdi)
+; AVX2-NEXT:  LBB6_61: ## %cond.store88
+; AVX2-NEXT:    vextractps $2, %xmm0, (%rdi)
 ; AVX2-NEXT:    addq $4, %rdi
-; AVX2-NEXT:  LBB6_62: ## %else89
-; AVX2-NEXT:    vpackssdw %xmm1, %xmm0, %xmm1
-; AVX2-NEXT:    vpacksswb %xmm1, %xmm0, %xmm1
-; AVX2-NEXT:    vpextrb $15, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $-2147483648, %eax ## imm = 0x80000000
 ; AVX2-NEXT:    je LBB6_64
-; AVX2-NEXT:  ## %bb.63: ## %cond.store91
-; AVX2-NEXT:    vpextrd $3, %xmm0, (%rdi)
-; AVX2-NEXT:  LBB6_64: ## %else92
+; AVX2-NEXT:  LBB6_63: ## %cond.store91
+; AVX2-NEXT:    vextractps $3, %xmm0, (%rdi)
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
@@ -2513,54 +2463,63 @@ define void @compressstore_v32f32_v32i32(float* %base, <32 x float> %V, <32 x i3
 define void @compressstore_v2i64_v2i1(i64* %base, <2 x i64> %V, <2 x i1> %mask) {
 ; SSE2-LABEL: compressstore_v2i64_v2i1:
 ; SSE2:       ## %bb.0:
-; SSE2-NEXT:    movd %xmm1, %eax
+; SSE2-NEXT:    psllq $63, %xmm1
+; SSE2-NEXT:    movmskpd %xmm1, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB7_2
-; SSE2-NEXT:  ## %bb.1: ## %cond.store
+; SSE2-NEXT:    jne LBB7_1
+; SSE2-NEXT:  ## %bb.2: ## %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne LBB7_3
+; SSE2-NEXT:  LBB7_4: ## %else2
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  LBB7_1: ## %cond.store
 ; SSE2-NEXT:    movq %xmm0, (%rdi)
 ; SSE2-NEXT:    addq $8, %rdi
-; SSE2-NEXT:  LBB7_2: ## %else
-; SSE2-NEXT:    pextrw $4, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $2, %al
 ; SSE2-NEXT:    je LBB7_4
-; SSE2-NEXT:  ## %bb.3: ## %cond.store1
+; SSE2-NEXT:  LBB7_3: ## %cond.store1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
 ; SSE2-NEXT:    movq %xmm0, (%rdi)
-; SSE2-NEXT:  LBB7_4: ## %else2
 ; SSE2-NEXT:    retq
 ;
 ; SSE42-LABEL: compressstore_v2i64_v2i1:
 ; SSE42:       ## %bb.0:
-; SSE42-NEXT:    pextrb $0, %xmm1, %eax
+; SSE42-NEXT:    psllq $63, %xmm1
+; SSE42-NEXT:    movmskpd %xmm1, %eax
 ; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB7_2
-; SSE42-NEXT:  ## %bb.1: ## %cond.store
+; SSE42-NEXT:    jne LBB7_1
+; SSE42-NEXT:  ## %bb.2: ## %else
+; SSE42-NEXT:    testb $2, %al
+; SSE42-NEXT:    jne LBB7_3
+; SSE42-NEXT:  LBB7_4: ## %else2
+; SSE42-NEXT:    retq
+; SSE42-NEXT:  LBB7_1: ## %cond.store
 ; SSE42-NEXT:    movq %xmm0, (%rdi)
 ; SSE42-NEXT:    addq $8, %rdi
-; SSE42-NEXT:  LBB7_2: ## %else
-; SSE42-NEXT:    pextrb $8, %xmm1, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $2, %al
 ; SSE42-NEXT:    je LBB7_4
-; SSE42-NEXT:  ## %bb.3: ## %cond.store1
+; SSE42-NEXT:  LBB7_3: ## %cond.store1
 ; SSE42-NEXT:    pextrq $1, %xmm0, (%rdi)
-; SSE42-NEXT:  LBB7_4: ## %else2
 ; SSE42-NEXT:    retq
 ;
 ; AVX1OR2-LABEL: compressstore_v2i64_v2i1:
 ; AVX1OR2:       ## %bb.0:
-; AVX1OR2-NEXT:    vpextrb $0, %xmm1, %eax
+; AVX1OR2-NEXT:    vpsllq $63, %xmm1, %xmm1
+; AVX1OR2-NEXT:    vmovmskpd %xmm1, %eax
 ; AVX1OR2-NEXT:    testb $1, %al
-; AVX1OR2-NEXT:    je LBB7_2
-; AVX1OR2-NEXT:  ## %bb.1: ## %cond.store
+; AVX1OR2-NEXT:    jne LBB7_1
+; AVX1OR2-NEXT:  ## %bb.2: ## %else
+; AVX1OR2-NEXT:    testb $2, %al
+; AVX1OR2-NEXT:    jne LBB7_3
+; AVX1OR2-NEXT:  LBB7_4: ## %else2
+; AVX1OR2-NEXT:    retq
+; AVX1OR2-NEXT:  LBB7_1: ## %cond.store
 ; AVX1OR2-NEXT:    vmovq %xmm0, (%rdi)
 ; AVX1OR2-NEXT:    addq $8, %rdi
-; AVX1OR2-NEXT:  LBB7_2: ## %else
-; AVX1OR2-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $2, %al
 ; AVX1OR2-NEXT:    je LBB7_4
-; AVX1OR2-NEXT:  ## %bb.3: ## %cond.store1
+; AVX1OR2-NEXT:  LBB7_3: ## %cond.store1
 ; AVX1OR2-NEXT:    vpextrq $1, %xmm0, (%rdi)
-; AVX1OR2-NEXT:  LBB7_4: ## %else2
 ; AVX1OR2-NEXT:    retq
 ;
 ; AVX512F-LABEL: compressstore_v2i64_v2i1:
@@ -2594,131 +2553,145 @@ define void @compressstore_v2i64_v2i1(i64* %base, <2 x i64> %V, <2 x i1> %mask)
 define void @compressstore_v4i64_v4i1(i64* %base, <4 x i64> %V, <4 x i1> %mask) {
 ; SSE2-LABEL: compressstore_v4i64_v4i1:
 ; SSE2:       ## %bb.0:
-; SSE2-NEXT:    movd %xmm2, %eax
+; SSE2-NEXT:    pslld $31, %xmm2
+; SSE2-NEXT:    movmskps %xmm2, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB8_2
-; SSE2-NEXT:  ## %bb.1: ## %cond.store
+; SSE2-NEXT:    jne LBB8_1
+; SSE2-NEXT:  ## %bb.2: ## %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne LBB8_3
+; SSE2-NEXT:  LBB8_4: ## %else2
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne LBB8_5
+; SSE2-NEXT:  LBB8_6: ## %else5
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne LBB8_7
+; SSE2-NEXT:  LBB8_8: ## %else8
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  LBB8_1: ## %cond.store
 ; SSE2-NEXT:    movq %xmm0, (%rdi)
 ; SSE2-NEXT:    addq $8, %rdi
-; SSE2-NEXT:  LBB8_2: ## %else
-; SSE2-NEXT:    pextrw $2, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $2, %al
 ; SSE2-NEXT:    je LBB8_4
-; SSE2-NEXT:  ## %bb.3: ## %cond.store1
+; SSE2-NEXT:  LBB8_3: ## %cond.store1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
 ; SSE2-NEXT:    movq %xmm0, (%rdi)
 ; SSE2-NEXT:    addq $8, %rdi
-; SSE2-NEXT:  LBB8_4: ## %else2
-; SSE2-NEXT:    pextrw $4, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $4, %al
 ; SSE2-NEXT:    je LBB8_6
-; SSE2-NEXT:  ## %bb.5: ## %cond.store4
+; SSE2-NEXT:  LBB8_5: ## %cond.store4
 ; SSE2-NEXT:    movq %xmm1, (%rdi)
 ; SSE2-NEXT:    addq $8, %rdi
-; SSE2-NEXT:  LBB8_6: ## %else5
-; SSE2-NEXT:    pextrw $6, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je LBB8_8
-; SSE2-NEXT:  ## %bb.7: ## %cond.store7
+; SSE2-NEXT:  LBB8_7: ## %cond.store7
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
 ; SSE2-NEXT:    movq %xmm0, (%rdi)
-; SSE2-NEXT:  LBB8_8: ## %else8
 ; SSE2-NEXT:    retq
 ;
 ; SSE42-LABEL: compressstore_v4i64_v4i1:
 ; SSE42:       ## %bb.0:
-; SSE42-NEXT:    pextrb $0, %xmm2, %eax
+; SSE42-NEXT:    pslld $31, %xmm2
+; SSE42-NEXT:    movmskps %xmm2, %eax
 ; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB8_2
-; SSE42-NEXT:  ## %bb.1: ## %cond.store
+; SSE42-NEXT:    jne LBB8_1
+; SSE42-NEXT:  ## %bb.2: ## %else
+; SSE42-NEXT:    testb $2, %al
+; SSE42-NEXT:    jne LBB8_3
+; SSE42-NEXT:  LBB8_4: ## %else2
+; SSE42-NEXT:    testb $4, %al
+; SSE42-NEXT:    jne LBB8_5
+; SSE42-NEXT:  LBB8_6: ## %else5
+; SSE42-NEXT:    testb $8, %al
+; SSE42-NEXT:    jne LBB8_7
+; SSE42-NEXT:  LBB8_8: ## %else8
+; SSE42-NEXT:    retq
+; SSE42-NEXT:  LBB8_1: ## %cond.store
 ; SSE42-NEXT:    movq %xmm0, (%rdi)
 ; SSE42-NEXT:    addq $8, %rdi
-; SSE42-NEXT:  LBB8_2: ## %else
-; SSE42-NEXT:    pextrb $4, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $2, %al
 ; SSE42-NEXT:    je LBB8_4
-; SSE42-NEXT:  ## %bb.3: ## %cond.store1
+; SSE42-NEXT:  LBB8_3: ## %cond.store1
 ; SSE42-NEXT:    pextrq $1, %xmm0, (%rdi)
 ; SSE42-NEXT:    addq $8, %rdi
-; SSE42-NEXT:  LBB8_4: ## %else2
-; SSE42-NEXT:    pextrb $8, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $4, %al
 ; SSE42-NEXT:    je LBB8_6
-; SSE42-NEXT:  ## %bb.5: ## %cond.store4
+; SSE42-NEXT:  LBB8_5: ## %cond.store4
 ; SSE42-NEXT:    movq %xmm1, (%rdi)
 ; SSE42-NEXT:    addq $8, %rdi
-; SSE42-NEXT:  LBB8_6: ## %else5
-; SSE42-NEXT:    pextrb $12, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $8, %al
 ; SSE42-NEXT:    je LBB8_8
-; SSE42-NEXT:  ## %bb.7: ## %cond.store7
+; SSE42-NEXT:  LBB8_7: ## %cond.store7
 ; SSE42-NEXT:    pextrq $1, %xmm1, (%rdi)
-; SSE42-NEXT:  LBB8_8: ## %else8
 ; SSE42-NEXT:    retq
 ;
 ; AVX1-LABEL: compressstore_v4i64_v4i1:
 ; AVX1:       ## %bb.0:
-; AVX1-NEXT:    vpextrb $0, %xmm1, %eax
+; AVX1-NEXT:    vpslld $31, %xmm1, %xmm1
+; AVX1-NEXT:    vmovmskps %xmm1, %eax
 ; AVX1-NEXT:    testb $1, %al
 ; AVX1-NEXT:    je LBB8_2
 ; AVX1-NEXT:  ## %bb.1: ## %cond.store
 ; AVX1-NEXT:    vmovq %xmm0, (%rdi)
 ; AVX1-NEXT:    addq $8, %rdi
 ; AVX1-NEXT:  LBB8_2: ## %else
-; AVX1-NEXT:    vpextrb $4, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $2, %al
 ; AVX1-NEXT:    je LBB8_4
 ; AVX1-NEXT:  ## %bb.3: ## %cond.store1
 ; AVX1-NEXT:    vpextrq $1, %xmm0, (%rdi)
 ; AVX1-NEXT:    addq $8, %rdi
 ; AVX1-NEXT:  LBB8_4: ## %else2
-; AVX1-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $4, %al
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT:    je LBB8_6
-; AVX1-NEXT:  ## %bb.5: ## %cond.store4
+; AVX1-NEXT:    jne LBB8_5
+; AVX1-NEXT:  ## %bb.6: ## %else5
+; AVX1-NEXT:    testb $8, %al
+; AVX1-NEXT:    jne LBB8_7
+; AVX1-NEXT:  LBB8_8: ## %else8
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+; AVX1-NEXT:  LBB8_5: ## %cond.store4
 ; AVX1-NEXT:    vmovq %xmm0, (%rdi)
 ; AVX1-NEXT:    addq $8, %rdi
-; AVX1-NEXT:  LBB8_6: ## %else5
-; AVX1-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $8, %al
 ; AVX1-NEXT:    je LBB8_8
-; AVX1-NEXT:  ## %bb.7: ## %cond.store7
+; AVX1-NEXT:  LBB8_7: ## %cond.store7
 ; AVX1-NEXT:    vpextrq $1, %xmm0, (%rdi)
-; AVX1-NEXT:  LBB8_8: ## %else8
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: compressstore_v4i64_v4i1:
 ; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpextrb $0, %xmm1, %eax
+; AVX2-NEXT:    vpslld $31, %xmm1, %xmm1
+; AVX2-NEXT:    vmovmskps %xmm1, %eax
 ; AVX2-NEXT:    testb $1, %al
 ; AVX2-NEXT:    je LBB8_2
 ; AVX2-NEXT:  ## %bb.1: ## %cond.store
 ; AVX2-NEXT:    vmovq %xmm0, (%rdi)
 ; AVX2-NEXT:    addq $8, %rdi
 ; AVX2-NEXT:  LBB8_2: ## %else
-; AVX2-NEXT:    vpextrb $4, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $2, %al
 ; AVX2-NEXT:    je LBB8_4
 ; AVX2-NEXT:  ## %bb.3: ## %cond.store1
 ; AVX2-NEXT:    vpextrq $1, %xmm0, (%rdi)
 ; AVX2-NEXT:    addq $8, %rdi
 ; AVX2-NEXT:  LBB8_4: ## %else2
-; AVX2-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $4, %al
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; AVX2-NEXT:    je LBB8_6
-; AVX2-NEXT:  ## %bb.5: ## %cond.store4
+; AVX2-NEXT:    jne LBB8_5
+; AVX2-NEXT:  ## %bb.6: ## %else5
+; AVX2-NEXT:    testb $8, %al
+; AVX2-NEXT:    jne LBB8_7
+; AVX2-NEXT:  LBB8_8: ## %else8
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+; AVX2-NEXT:  LBB8_5: ## %cond.store4
 ; AVX2-NEXT:    vmovq %xmm0, (%rdi)
 ; AVX2-NEXT:    addq $8, %rdi
-; AVX2-NEXT:  LBB8_6: ## %else5
-; AVX2-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $8, %al
 ; AVX2-NEXT:    je LBB8_8
-; AVX2-NEXT:  ## %bb.7: ## %cond.store7
+; AVX2-NEXT:  LBB8_7: ## %cond.store7
 ; AVX2-NEXT:    vpextrq $1, %xmm0, (%rdi)
-; AVX2-NEXT:  LBB8_8: ## %else8
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
@@ -2755,247 +2728,279 @@ define void @compressstore_v4i64_v4i1(i64* %base, <4 x i64> %V, <4 x i1> %mask)
 define void @compressstore_v8i64_v8i1(i64* %base, <8 x i64> %V, <8 x i1> %mask) {
 ; SSE2-LABEL: compressstore_v8i64_v8i1:
 ; SSE2:       ## %bb.0:
-; SSE2-NEXT:    movd %xmm4, %eax
+; SSE2-NEXT:    psllw $15, %xmm4
+; SSE2-NEXT:    packsswb %xmm0, %xmm4
+; SSE2-NEXT:    pmovmskb %xmm4, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB9_2
-; SSE2-NEXT:  ## %bb.1: ## %cond.store
+; SSE2-NEXT:    jne LBB9_1
+; SSE2-NEXT:  ## %bb.2: ## %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne LBB9_3
+; SSE2-NEXT:  LBB9_4: ## %else2
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne LBB9_5
+; SSE2-NEXT:  LBB9_6: ## %else5
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne LBB9_7
+; SSE2-NEXT:  LBB9_8: ## %else8
+; SSE2-NEXT:    testb $16, %al
+; SSE2-NEXT:    jne LBB9_9
+; SSE2-NEXT:  LBB9_10: ## %else11
+; SSE2-NEXT:    testb $32, %al
+; SSE2-NEXT:    jne LBB9_11
+; SSE2-NEXT:  LBB9_12: ## %else14
+; SSE2-NEXT:    testb $64, %al
+; SSE2-NEXT:    jne LBB9_13
+; SSE2-NEXT:  LBB9_14: ## %else17
+; SSE2-NEXT:    testb $-128, %al
+; SSE2-NEXT:    jne LBB9_15
+; SSE2-NEXT:  LBB9_16: ## %else20
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  LBB9_1: ## %cond.store
 ; SSE2-NEXT:    movq %xmm0, (%rdi)
 ; SSE2-NEXT:    addq $8, %rdi
-; SSE2-NEXT:  LBB9_2: ## %else
-; SSE2-NEXT:    shrl $16, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $2, %al
 ; SSE2-NEXT:    je LBB9_4
-; SSE2-NEXT:  ## %bb.3: ## %cond.store1
+; SSE2-NEXT:  LBB9_3: ## %cond.store1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
 ; SSE2-NEXT:    movq %xmm0, (%rdi)
 ; SSE2-NEXT:    addq $8, %rdi
-; SSE2-NEXT:  LBB9_4: ## %else2
-; SSE2-NEXT:    pextrw $2, %xmm4, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $4, %al
 ; SSE2-NEXT:    je LBB9_6
-; SSE2-NEXT:  ## %bb.5: ## %cond.store4
+; SSE2-NEXT:  LBB9_5: ## %cond.store4
 ; SSE2-NEXT:    movq %xmm1, (%rdi)
 ; SSE2-NEXT:    addq $8, %rdi
-; SSE2-NEXT:  LBB9_6: ## %else5
-; SSE2-NEXT:    pextrw $3, %xmm4, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je LBB9_8
-; SSE2-NEXT:  ## %bb.7: ## %cond.store7
+; SSE2-NEXT:  LBB9_7: ## %cond.store7
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
 ; SSE2-NEXT:    movq %xmm0, (%rdi)
 ; SSE2-NEXT:    addq $8, %rdi
-; SSE2-NEXT:  LBB9_8: ## %else8
-; SSE2-NEXT:    pextrw $4, %xmm4, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $16, %al
 ; SSE2-NEXT:    je LBB9_10
-; SSE2-NEXT:  ## %bb.9: ## %cond.store10
+; SSE2-NEXT:  LBB9_9: ## %cond.store10
 ; SSE2-NEXT:    movq %xmm2, (%rdi)
 ; SSE2-NEXT:    addq $8, %rdi
-; SSE2-NEXT:  LBB9_10: ## %else11
-; SSE2-NEXT:    pextrw $5, %xmm4, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $32, %al
 ; SSE2-NEXT:    je LBB9_12
-; SSE2-NEXT:  ## %bb.11: ## %cond.store13
+; SSE2-NEXT:  LBB9_11: ## %cond.store13
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
 ; SSE2-NEXT:    movq %xmm0, (%rdi)
 ; SSE2-NEXT:    addq $8, %rdi
-; SSE2-NEXT:  LBB9_12: ## %else14
-; SSE2-NEXT:    pextrw $6, %xmm4, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $64, %al
 ; SSE2-NEXT:    je LBB9_14
-; SSE2-NEXT:  ## %bb.13: ## %cond.store16
+; SSE2-NEXT:  LBB9_13: ## %cond.store16
 ; SSE2-NEXT:    movq %xmm3, (%rdi)
 ; SSE2-NEXT:    addq $8, %rdi
-; SSE2-NEXT:  LBB9_14: ## %else17
-; SSE2-NEXT:    pextrw $7, %xmm4, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $-128, %al
 ; SSE2-NEXT:    je LBB9_16
-; SSE2-NEXT:  ## %bb.15: ## %cond.store19
+; SSE2-NEXT:  LBB9_15: ## %cond.store19
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[2,3,0,1]
 ; SSE2-NEXT:    movq %xmm0, (%rdi)
-; SSE2-NEXT:  LBB9_16: ## %else20
 ; SSE2-NEXT:    retq
 ;
 ; SSE42-LABEL: compressstore_v8i64_v8i1:
 ; SSE42:       ## %bb.0:
-; SSE42-NEXT:    pextrb $0, %xmm4, %eax
+; SSE42-NEXT:    psllw $15, %xmm4
+; SSE42-NEXT:    packsswb %xmm0, %xmm4
+; SSE42-NEXT:    pmovmskb %xmm4, %eax
 ; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB9_2
-; SSE42-NEXT:  ## %bb.1: ## %cond.store
+; SSE42-NEXT:    jne LBB9_1
+; SSE42-NEXT:  ## %bb.2: ## %else
+; SSE42-NEXT:    testb $2, %al
+; SSE42-NEXT:    jne LBB9_3
+; SSE42-NEXT:  LBB9_4: ## %else2
+; SSE42-NEXT:    testb $4, %al
+; SSE42-NEXT:    jne LBB9_5
+; SSE42-NEXT:  LBB9_6: ## %else5
+; SSE42-NEXT:    testb $8, %al
+; SSE42-NEXT:    jne LBB9_7
+; SSE42-NEXT:  LBB9_8: ## %else8
+; SSE42-NEXT:    testb $16, %al
+; SSE42-NEXT:    jne LBB9_9
+; SSE42-NEXT:  LBB9_10: ## %else11
+; SSE42-NEXT:    testb $32, %al
+; SSE42-NEXT:    jne LBB9_11
+; SSE42-NEXT:  LBB9_12: ## %else14
+; SSE42-NEXT:    testb $64, %al
+; SSE42-NEXT:    jne LBB9_13
+; SSE42-NEXT:  LBB9_14: ## %else17
+; SSE42-NEXT:    testb $-128, %al
+; SSE42-NEXT:    jne LBB9_15
+; SSE42-NEXT:  LBB9_16: ## %else20
+; SSE42-NEXT:    retq
+; SSE42-NEXT:  LBB9_1: ## %cond.store
 ; SSE42-NEXT:    movq %xmm0, (%rdi)
 ; SSE42-NEXT:    addq $8, %rdi
-; SSE42-NEXT:  LBB9_2: ## %else
-; SSE42-NEXT:    pextrb $2, %xmm4, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $2, %al
 ; SSE42-NEXT:    je LBB9_4
-; SSE42-NEXT:  ## %bb.3: ## %cond.store1
+; SSE42-NEXT:  LBB9_3: ## %cond.store1
 ; SSE42-NEXT:    pextrq $1, %xmm0, (%rdi)
 ; SSE42-NEXT:    addq $8, %rdi
-; SSE42-NEXT:  LBB9_4: ## %else2
-; SSE42-NEXT:    pextrb $4, %xmm4, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $4, %al
 ; SSE42-NEXT:    je LBB9_6
-; SSE42-NEXT:  ## %bb.5: ## %cond.store4
+; SSE42-NEXT:  LBB9_5: ## %cond.store4
 ; SSE42-NEXT:    movq %xmm1, (%rdi)
 ; SSE42-NEXT:    addq $8, %rdi
-; SSE42-NEXT:  LBB9_6: ## %else5
-; SSE42-NEXT:    pextrb $6, %xmm4, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $8, %al
 ; SSE42-NEXT:    je LBB9_8
-; SSE42-NEXT:  ## %bb.7: ## %cond.store7
+; SSE42-NEXT:  LBB9_7: ## %cond.store7
 ; SSE42-NEXT:    pextrq $1, %xmm1, (%rdi)
 ; SSE42-NEXT:    addq $8, %rdi
-; SSE42-NEXT:  LBB9_8: ## %else8
-; SSE42-NEXT:    pextrb $8, %xmm4, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $16, %al
 ; SSE42-NEXT:    je LBB9_10
-; SSE42-NEXT:  ## %bb.9: ## %cond.store10
+; SSE42-NEXT:  LBB9_9: ## %cond.store10
 ; SSE42-NEXT:    movq %xmm2, (%rdi)
 ; SSE42-NEXT:    addq $8, %rdi
-; SSE42-NEXT:  LBB9_10: ## %else11
-; SSE42-NEXT:    pextrb $10, %xmm4, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $32, %al
 ; SSE42-NEXT:    je LBB9_12
-; SSE42-NEXT:  ## %bb.11: ## %cond.store13
+; SSE42-NEXT:  LBB9_11: ## %cond.store13
 ; SSE42-NEXT:    pextrq $1, %xmm2, (%rdi)
 ; SSE42-NEXT:    addq $8, %rdi
-; SSE42-NEXT:  LBB9_12: ## %else14
-; SSE42-NEXT:    pextrb $12, %xmm4, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $64, %al
 ; SSE42-NEXT:    je LBB9_14
-; SSE42-NEXT:  ## %bb.13: ## %cond.store16
+; SSE42-NEXT:  LBB9_13: ## %cond.store16
 ; SSE42-NEXT:    movq %xmm3, (%rdi)
 ; SSE42-NEXT:    addq $8, %rdi
-; SSE42-NEXT:  LBB9_14: ## %else17
-; SSE42-NEXT:    pextrb $14, %xmm4, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $-128, %al
 ; SSE42-NEXT:    je LBB9_16
-; SSE42-NEXT:  ## %bb.15: ## %cond.store19
+; SSE42-NEXT:  LBB9_15: ## %cond.store19
 ; SSE42-NEXT:    pextrq $1, %xmm3, (%rdi)
-; SSE42-NEXT:  LBB9_16: ## %else20
 ; SSE42-NEXT:    retq
 ;
 ; AVX1-LABEL: compressstore_v8i64_v8i1:
 ; AVX1:       ## %bb.0:
-; AVX1-NEXT:    vpextrb $0, %xmm2, %eax
+; AVX1-NEXT:    vpsllw $15, %xmm2, %xmm2
+; AVX1-NEXT:    vpacksswb %xmm0, %xmm2, %xmm2
+; AVX1-NEXT:    vpmovmskb %xmm2, %eax
 ; AVX1-NEXT:    testb $1, %al
 ; AVX1-NEXT:    je LBB9_2
 ; AVX1-NEXT:  ## %bb.1: ## %cond.store
 ; AVX1-NEXT:    vmovq %xmm0, (%rdi)
 ; AVX1-NEXT:    addq $8, %rdi
 ; AVX1-NEXT:  LBB9_2: ## %else
-; AVX1-NEXT:    vpextrb $2, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $2, %al
 ; AVX1-NEXT:    je LBB9_4
 ; AVX1-NEXT:  ## %bb.3: ## %cond.store1
 ; AVX1-NEXT:    vpextrq $1, %xmm0, (%rdi)
 ; AVX1-NEXT:    addq $8, %rdi
 ; AVX1-NEXT:  LBB9_4: ## %else2
-; AVX1-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $4, %al
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT:    je LBB9_6
-; AVX1-NEXT:  ## %bb.5: ## %cond.store4
+; AVX1-NEXT:    jne LBB9_5
+; AVX1-NEXT:  ## %bb.6: ## %else5
+; AVX1-NEXT:    testb $8, %al
+; AVX1-NEXT:    jne LBB9_7
+; AVX1-NEXT:  LBB9_8: ## %else8
+; AVX1-NEXT:    testb $16, %al
+; AVX1-NEXT:    jne LBB9_9
+; AVX1-NEXT:  LBB9_10: ## %else11
+; AVX1-NEXT:    testb $32, %al
+; AVX1-NEXT:    je LBB9_12
+; AVX1-NEXT:  LBB9_11: ## %cond.store13
+; AVX1-NEXT:    vpextrq $1, %xmm1, (%rdi)
+; AVX1-NEXT:    addq $8, %rdi
+; AVX1-NEXT:  LBB9_12: ## %else14
+; AVX1-NEXT:    testb $64, %al
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm0
+; AVX1-NEXT:    jne LBB9_13
+; AVX1-NEXT:  ## %bb.14: ## %else17
+; AVX1-NEXT:    testb $-128, %al
+; AVX1-NEXT:    jne LBB9_15
+; AVX1-NEXT:  LBB9_16: ## %else20
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+; AVX1-NEXT:  LBB9_5: ## %cond.store4
 ; AVX1-NEXT:    vmovq %xmm0, (%rdi)
 ; AVX1-NEXT:    addq $8, %rdi
-; AVX1-NEXT:  LBB9_6: ## %else5
-; AVX1-NEXT:    vpextrb $6, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $8, %al
 ; AVX1-NEXT:    je LBB9_8
-; AVX1-NEXT:  ## %bb.7: ## %cond.store7
+; AVX1-NEXT:  LBB9_7: ## %cond.store7
 ; AVX1-NEXT:    vpextrq $1, %xmm0, (%rdi)
 ; AVX1-NEXT:    addq $8, %rdi
-; AVX1-NEXT:  LBB9_8: ## %else8
-; AVX1-NEXT:    vpextrb $8, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $16, %al
 ; AVX1-NEXT:    je LBB9_10
-; AVX1-NEXT:  ## %bb.9: ## %cond.store10
+; AVX1-NEXT:  LBB9_9: ## %cond.store10
 ; AVX1-NEXT:    vmovq %xmm1, (%rdi)
 ; AVX1-NEXT:    addq $8, %rdi
-; AVX1-NEXT:  LBB9_10: ## %else11
-; AVX1-NEXT:    vpextrb $10, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je LBB9_12
-; AVX1-NEXT:  ## %bb.11: ## %cond.store13
-; AVX1-NEXT:    vpextrq $1, %xmm1, (%rdi)
-; AVX1-NEXT:    addq $8, %rdi
-; AVX1-NEXT:  LBB9_12: ## %else14
-; AVX1-NEXT:    vpextrb $12, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm0
-; AVX1-NEXT:    je LBB9_14
-; AVX1-NEXT:  ## %bb.13: ## %cond.store16
+; AVX1-NEXT:    testb $32, %al
+; AVX1-NEXT:    jne LBB9_11
+; AVX1-NEXT:    jmp LBB9_12
+; AVX1-NEXT:  LBB9_13: ## %cond.store16
 ; AVX1-NEXT:    vmovq %xmm0, (%rdi)
 ; AVX1-NEXT:    addq $8, %rdi
-; AVX1-NEXT:  LBB9_14: ## %else17
-; AVX1-NEXT:    vpextrb $14, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $-128, %al
 ; AVX1-NEXT:    je LBB9_16
-; AVX1-NEXT:  ## %bb.15: ## %cond.store19
+; AVX1-NEXT:  LBB9_15: ## %cond.store19
 ; AVX1-NEXT:    vpextrq $1, %xmm0, (%rdi)
-; AVX1-NEXT:  LBB9_16: ## %else20
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: compressstore_v8i64_v8i1:
 ; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpextrb $0, %xmm2, %eax
+; AVX2-NEXT:    vpsllw $15, %xmm2, %xmm2
+; AVX2-NEXT:    vpacksswb %xmm0, %xmm2, %xmm2
+; AVX2-NEXT:    vpmovmskb %xmm2, %eax
 ; AVX2-NEXT:    testb $1, %al
 ; AVX2-NEXT:    je LBB9_2
 ; AVX2-NEXT:  ## %bb.1: ## %cond.store
 ; AVX2-NEXT:    vmovq %xmm0, (%rdi)
 ; AVX2-NEXT:    addq $8, %rdi
 ; AVX2-NEXT:  LBB9_2: ## %else
-; AVX2-NEXT:    vpextrb $2, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $2, %al
 ; AVX2-NEXT:    je LBB9_4
 ; AVX2-NEXT:  ## %bb.3: ## %cond.store1
 ; AVX2-NEXT:    vpextrq $1, %xmm0, (%rdi)
 ; AVX2-NEXT:    addq $8, %rdi
 ; AVX2-NEXT:  LBB9_4: ## %else2
-; AVX2-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $4, %al
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; AVX2-NEXT:    je LBB9_6
-; AVX2-NEXT:  ## %bb.5: ## %cond.store4
+; AVX2-NEXT:    jne LBB9_5
+; AVX2-NEXT:  ## %bb.6: ## %else5
+; AVX2-NEXT:    testb $8, %al
+; AVX2-NEXT:    jne LBB9_7
+; AVX2-NEXT:  LBB9_8: ## %else8
+; AVX2-NEXT:    testb $16, %al
+; AVX2-NEXT:    jne LBB9_9
+; AVX2-NEXT:  LBB9_10: ## %else11
+; AVX2-NEXT:    testb $32, %al
+; AVX2-NEXT:    je LBB9_12
+; AVX2-NEXT:  LBB9_11: ## %cond.store13
+; AVX2-NEXT:    vpextrq $1, %xmm1, (%rdi)
+; AVX2-NEXT:    addq $8, %rdi
+; AVX2-NEXT:  LBB9_12: ## %else14
+; AVX2-NEXT:    testb $64, %al
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX2-NEXT:    jne LBB9_13
+; AVX2-NEXT:  ## %bb.14: ## %else17
+; AVX2-NEXT:    testb $-128, %al
+; AVX2-NEXT:    jne LBB9_15
+; AVX2-NEXT:  LBB9_16: ## %else20
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+; AVX2-NEXT:  LBB9_5: ## %cond.store4
 ; AVX2-NEXT:    vmovq %xmm0, (%rdi)
 ; AVX2-NEXT:    addq $8, %rdi
-; AVX2-NEXT:  LBB9_6: ## %else5
-; AVX2-NEXT:    vpextrb $6, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $8, %al
 ; AVX2-NEXT:    je LBB9_8
-; AVX2-NEXT:  ## %bb.7: ## %cond.store7
+; AVX2-NEXT:  LBB9_7: ## %cond.store7
 ; AVX2-NEXT:    vpextrq $1, %xmm0, (%rdi)
 ; AVX2-NEXT:    addq $8, %rdi
-; AVX2-NEXT:  LBB9_8: ## %else8
-; AVX2-NEXT:    vpextrb $8, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $16, %al
 ; AVX2-NEXT:    je LBB9_10
-; AVX2-NEXT:  ## %bb.9: ## %cond.store10
+; AVX2-NEXT:  LBB9_9: ## %cond.store10
 ; AVX2-NEXT:    vmovq %xmm1, (%rdi)
 ; AVX2-NEXT:    addq $8, %rdi
-; AVX2-NEXT:  LBB9_10: ## %else11
-; AVX2-NEXT:    vpextrb $10, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je LBB9_12
-; AVX2-NEXT:  ## %bb.11: ## %cond.store13
-; AVX2-NEXT:    vpextrq $1, %xmm1, (%rdi)
-; AVX2-NEXT:    addq $8, %rdi
-; AVX2-NEXT:  LBB9_12: ## %else14
-; AVX2-NEXT:    vpextrb $12, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm0
-; AVX2-NEXT:    je LBB9_14
-; AVX2-NEXT:  ## %bb.13: ## %cond.store16
+; AVX2-NEXT:    testb $32, %al
+; AVX2-NEXT:    jne LBB9_11
+; AVX2-NEXT:    jmp LBB9_12
+; AVX2-NEXT:  LBB9_13: ## %cond.store16
 ; AVX2-NEXT:    vmovq %xmm0, (%rdi)
 ; AVX2-NEXT:    addq $8, %rdi
-; AVX2-NEXT:  LBB9_14: ## %else17
-; AVX2-NEXT:    vpextrb $14, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $-128, %al
 ; AVX2-NEXT:    je LBB9_16
-; AVX2-NEXT:  ## %bb.15: ## %cond.store19
+; AVX2-NEXT:  LBB9_15: ## %cond.store19
 ; AVX2-NEXT:    vpextrq $1, %xmm0, (%rdi)
-; AVX2-NEXT:  LBB9_16: ## %else20
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
@@ -3037,108 +3042,114 @@ define void @compressstore_v4i32_v4i32(i32* %base, <4 x i32> %V, <4 x i32> %trig
 ; SSE2:       ## %bb.0:
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
 ; SSE2-NEXT:    pcmpeqd %xmm1, %xmm2
-; SSE2-NEXT:    movd %xmm2, %eax
+; SSE2-NEXT:    movmskps %xmm2, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB10_2
-; SSE2-NEXT:  ## %bb.1: ## %cond.store
+; SSE2-NEXT:    jne LBB10_1
+; SSE2-NEXT:  ## %bb.2: ## %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne LBB10_3
+; SSE2-NEXT:  LBB10_4: ## %else2
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne LBB10_5
+; SSE2-NEXT:  LBB10_6: ## %else5
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne LBB10_7
+; SSE2-NEXT:  LBB10_8: ## %else8
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  LBB10_1: ## %cond.store
 ; SSE2-NEXT:    movd %xmm0, (%rdi)
 ; SSE2-NEXT:    addq $4, %rdi
-; SSE2-NEXT:  LBB10_2: ## %else
-; SSE2-NEXT:    pextrw $2, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $2, %al
 ; SSE2-NEXT:    je LBB10_4
-; SSE2-NEXT:  ## %bb.3: ## %cond.store1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
-; SSE2-NEXT:    movd %xmm2, (%rdi)
+; SSE2-NEXT:  LBB10_3: ## %cond.store1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE2-NEXT:    movd %xmm1, (%rdi)
 ; SSE2-NEXT:    addq $4, %rdi
-; SSE2-NEXT:  LBB10_4: ## %else2
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm1
-; SSE2-NEXT:    pextrw $4, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $4, %al
 ; SSE2-NEXT:    je LBB10_6
-; SSE2-NEXT:  ## %bb.5: ## %cond.store4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
-; SSE2-NEXT:    movd %xmm2, (%rdi)
+; SSE2-NEXT:  LBB10_5: ## %cond.store4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    movd %xmm1, (%rdi)
 ; SSE2-NEXT:    addq $4, %rdi
-; SSE2-NEXT:  LBB10_6: ## %else5
-; SSE2-NEXT:    pextrw $6, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je LBB10_8
-; SSE2-NEXT:  ## %bb.7: ## %cond.store7
+; SSE2-NEXT:  LBB10_7: ## %cond.store7
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
 ; SSE2-NEXT:    movd %xmm0, (%rdi)
-; SSE2-NEXT:  LBB10_8: ## %else8
 ; SSE2-NEXT:    retq
 ;
 ; SSE42-LABEL: compressstore_v4i32_v4i32:
 ; SSE42:       ## %bb.0:
 ; SSE42-NEXT:    pxor %xmm2, %xmm2
 ; SSE42-NEXT:    pcmpeqd %xmm1, %xmm2
-; SSE42-NEXT:    pextrb $0, %xmm2, %eax
+; SSE42-NEXT:    movmskps %xmm2, %eax
 ; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB10_2
-; SSE42-NEXT:  ## %bb.1: ## %cond.store
+; SSE42-NEXT:    jne LBB10_1
+; SSE42-NEXT:  ## %bb.2: ## %else
+; SSE42-NEXT:    testb $2, %al
+; SSE42-NEXT:    jne LBB10_3
+; SSE42-NEXT:  LBB10_4: ## %else2
+; SSE42-NEXT:    testb $4, %al
+; SSE42-NEXT:    jne LBB10_5
+; SSE42-NEXT:  LBB10_6: ## %else5
+; SSE42-NEXT:    testb $8, %al
+; SSE42-NEXT:    jne LBB10_7
+; SSE42-NEXT:  LBB10_8: ## %else8
+; SSE42-NEXT:    retq
+; SSE42-NEXT:  LBB10_1: ## %cond.store
 ; SSE42-NEXT:    movss %xmm0, (%rdi)
 ; SSE42-NEXT:    addq $4, %rdi
-; SSE42-NEXT:  LBB10_2: ## %else
-; SSE42-NEXT:    pextrb $4, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $2, %al
 ; SSE42-NEXT:    je LBB10_4
-; SSE42-NEXT:  ## %bb.3: ## %cond.store1
+; SSE42-NEXT:  LBB10_3: ## %cond.store1
 ; SSE42-NEXT:    extractps $1, %xmm0, (%rdi)
 ; SSE42-NEXT:    addq $4, %rdi
-; SSE42-NEXT:  LBB10_4: ## %else2
-; SSE42-NEXT:    pxor %xmm2, %xmm2
-; SSE42-NEXT:    pcmpeqd %xmm2, %xmm1
-; SSE42-NEXT:    pextrb $8, %xmm1, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $4, %al
 ; SSE42-NEXT:    je LBB10_6
-; SSE42-NEXT:  ## %bb.5: ## %cond.store4
+; SSE42-NEXT:  LBB10_5: ## %cond.store4
 ; SSE42-NEXT:    extractps $2, %xmm0, (%rdi)
 ; SSE42-NEXT:    addq $4, %rdi
-; SSE42-NEXT:  LBB10_6: ## %else5
-; SSE42-NEXT:    pextrb $12, %xmm1, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $8, %al
 ; SSE42-NEXT:    je LBB10_8
-; SSE42-NEXT:  ## %bb.7: ## %cond.store7
+; SSE42-NEXT:  LBB10_7: ## %cond.store7
 ; SSE42-NEXT:    extractps $3, %xmm0, (%rdi)
-; SSE42-NEXT:  LBB10_8: ## %else8
 ; SSE42-NEXT:    retq
 ;
 ; AVX1OR2-LABEL: compressstore_v4i32_v4i32:
 ; AVX1OR2:       ## %bb.0:
 ; AVX1OR2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1OR2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX1OR2-NEXT:    vpextrb $0, %xmm2, %eax
+; AVX1OR2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
+; AVX1OR2-NEXT:    vmovmskps %xmm1, %eax
 ; AVX1OR2-NEXT:    testb $1, %al
-; AVX1OR2-NEXT:    je LBB10_2
-; AVX1OR2-NEXT:  ## %bb.1: ## %cond.store
+; AVX1OR2-NEXT:    jne LBB10_1
+; AVX1OR2-NEXT:  ## %bb.2: ## %else
+; AVX1OR2-NEXT:    testb $2, %al
+; AVX1OR2-NEXT:    jne LBB10_3
+; AVX1OR2-NEXT:  LBB10_4: ## %else2
+; AVX1OR2-NEXT:    testb $4, %al
+; AVX1OR2-NEXT:    jne LBB10_5
+; AVX1OR2-NEXT:  LBB10_6: ## %else5
+; AVX1OR2-NEXT:    testb $8, %al
+; AVX1OR2-NEXT:    jne LBB10_7
+; AVX1OR2-NEXT:  LBB10_8: ## %else8
+; AVX1OR2-NEXT:    retq
+; AVX1OR2-NEXT:  LBB10_1: ## %cond.store
 ; AVX1OR2-NEXT:    vmovss %xmm0, (%rdi)
 ; AVX1OR2-NEXT:    addq $4, %rdi
-; AVX1OR2-NEXT:  LBB10_2: ## %else
-; AVX1OR2-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $2, %al
 ; AVX1OR2-NEXT:    je LBB10_4
-; AVX1OR2-NEXT:  ## %bb.3: ## %cond.store1
+; AVX1OR2-NEXT:  LBB10_3: ## %cond.store1
 ; AVX1OR2-NEXT:    vextractps $1, %xmm0, (%rdi)
 ; AVX1OR2-NEXT:    addq $4, %rdi
-; AVX1OR2-NEXT:  LBB10_4: ## %else2
-; AVX1OR2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1OR2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX1OR2-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $4, %al
 ; AVX1OR2-NEXT:    je LBB10_6
-; AVX1OR2-NEXT:  ## %bb.5: ## %cond.store4
+; AVX1OR2-NEXT:  LBB10_5: ## %cond.store4
 ; AVX1OR2-NEXT:    vextractps $2, %xmm0, (%rdi)
 ; AVX1OR2-NEXT:    addq $4, %rdi
-; AVX1OR2-NEXT:  LBB10_6: ## %else5
-; AVX1OR2-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $8, %al
 ; AVX1OR2-NEXT:    je LBB10_8
-; AVX1OR2-NEXT:  ## %bb.7: ## %cond.store7
+; AVX1OR2-NEXT:  LBB10_7: ## %cond.store7
 ; AVX1OR2-NEXT:    vextractps $3, %xmm0, (%rdi)
-; AVX1OR2-NEXT:  LBB10_8: ## %else8
 ; AVX1OR2-NEXT:    retq
 ;
 ; AVX512F-LABEL: compressstore_v4i32_v4i32:
@@ -3171,374 +3182,363 @@ define void @compressstore_v8i16_v8i16(i16* %base, <8 x i16> %V, <8 x i16> %trig
 ; SSE2:       ## %bb.0:
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
 ; SSE2-NEXT:    pcmpeqw %xmm1, %xmm2
-; SSE2-NEXT:    movd %xmm2, %eax
+; SSE2-NEXT:    packsswb %xmm0, %xmm2
+; SSE2-NEXT:    pmovmskb %xmm2, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB11_2
-; SSE2-NEXT:  ## %bb.1: ## %cond.store
+; SSE2-NEXT:    jne LBB11_1
+; SSE2-NEXT:  ## %bb.2: ## %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne LBB11_3
+; SSE2-NEXT:  LBB11_4: ## %else2
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne LBB11_5
+; SSE2-NEXT:  LBB11_6: ## %else5
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne LBB11_7
+; SSE2-NEXT:  LBB11_8: ## %else8
+; SSE2-NEXT:    testb $16, %al
+; SSE2-NEXT:    jne LBB11_9
+; SSE2-NEXT:  LBB11_10: ## %else11
+; SSE2-NEXT:    testb $32, %al
+; SSE2-NEXT:    jne LBB11_11
+; SSE2-NEXT:  LBB11_12: ## %else14
+; SSE2-NEXT:    testb $64, %al
+; SSE2-NEXT:    jne LBB11_13
+; SSE2-NEXT:  LBB11_14: ## %else17
+; SSE2-NEXT:    testb $-128, %al
+; SSE2-NEXT:    jne LBB11_15
+; SSE2-NEXT:  LBB11_16: ## %else20
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  LBB11_1: ## %cond.store
 ; SSE2-NEXT:    movd %xmm0, %ecx
 ; SSE2-NEXT:    movw %cx, (%rdi)
 ; SSE2-NEXT:    addq $2, %rdi
-; SSE2-NEXT:  LBB11_2: ## %else
-; SSE2-NEXT:    shrl $16, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $2, %al
 ; SSE2-NEXT:    je LBB11_4
-; SSE2-NEXT:  ## %bb.3: ## %cond.store1
-; SSE2-NEXT:    pextrw $1, %xmm0, %eax
-; SSE2-NEXT:    movw %ax, (%rdi)
+; SSE2-NEXT:  LBB11_3: ## %cond.store1
+; SSE2-NEXT:    pextrw $1, %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, (%rdi)
 ; SSE2-NEXT:    addq $2, %rdi
-; SSE2-NEXT:  LBB11_4: ## %else2
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpeqw %xmm1, %xmm2
-; SSE2-NEXT:    pextrw $2, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $4, %al
 ; SSE2-NEXT:    je LBB11_6
-; SSE2-NEXT:  ## %bb.5: ## %cond.store4
-; SSE2-NEXT:    pextrw $2, %xmm0, %eax
-; SSE2-NEXT:    movw %ax, (%rdi)
+; SSE2-NEXT:  LBB11_5: ## %cond.store4
+; SSE2-NEXT:    pextrw $2, %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, (%rdi)
 ; SSE2-NEXT:    addq $2, %rdi
-; SSE2-NEXT:  LBB11_6: ## %else5
-; SSE2-NEXT:    pextrw $3, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je LBB11_8
-; SSE2-NEXT:  ## %bb.7: ## %cond.store7
-; SSE2-NEXT:    pextrw $3, %xmm0, %eax
-; SSE2-NEXT:    movw %ax, (%rdi)
+; SSE2-NEXT:  LBB11_7: ## %cond.store7
+; SSE2-NEXT:    pextrw $3, %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, (%rdi)
 ; SSE2-NEXT:    addq $2, %rdi
-; SSE2-NEXT:  LBB11_8: ## %else8
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpeqw %xmm1, %xmm2
-; SSE2-NEXT:    pextrw $4, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $16, %al
 ; SSE2-NEXT:    je LBB11_10
-; SSE2-NEXT:  ## %bb.9: ## %cond.store10
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    movw %ax, (%rdi)
+; SSE2-NEXT:  LBB11_9: ## %cond.store10
+; SSE2-NEXT:    pextrw $4, %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, (%rdi)
 ; SSE2-NEXT:    addq $2, %rdi
-; SSE2-NEXT:  LBB11_10: ## %else11
-; SSE2-NEXT:    pextrw $5, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $32, %al
 ; SSE2-NEXT:    je LBB11_12
-; SSE2-NEXT:  ## %bb.11: ## %cond.store13
-; SSE2-NEXT:    pextrw $5, %xmm0, %eax
-; SSE2-NEXT:    movw %ax, (%rdi)
+; SSE2-NEXT:  LBB11_11: ## %cond.store13
+; SSE2-NEXT:    pextrw $5, %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, (%rdi)
 ; SSE2-NEXT:    addq $2, %rdi
-; SSE2-NEXT:  LBB11_12: ## %else14
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpeqw %xmm2, %xmm1
-; SSE2-NEXT:    pextrw $6, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $64, %al
 ; SSE2-NEXT:    je LBB11_14
-; SSE2-NEXT:  ## %bb.13: ## %cond.store16
-; SSE2-NEXT:    pextrw $6, %xmm0, %eax
-; SSE2-NEXT:    movw %ax, (%rdi)
+; SSE2-NEXT:  LBB11_13: ## %cond.store16
+; SSE2-NEXT:    pextrw $6, %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, (%rdi)
 ; SSE2-NEXT:    addq $2, %rdi
-; SSE2-NEXT:  LBB11_14: ## %else17
-; SSE2-NEXT:    pextrw $7, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $-128, %al
 ; SSE2-NEXT:    je LBB11_16
-; SSE2-NEXT:  ## %bb.15: ## %cond.store19
+; SSE2-NEXT:  LBB11_15: ## %cond.store19
 ; SSE2-NEXT:    pextrw $7, %xmm0, %eax
 ; SSE2-NEXT:    movw %ax, (%rdi)
-; SSE2-NEXT:  LBB11_16: ## %else20
 ; SSE2-NEXT:    retq
 ;
 ; SSE42-LABEL: compressstore_v8i16_v8i16:
 ; SSE42:       ## %bb.0:
 ; SSE42-NEXT:    pxor %xmm2, %xmm2
 ; SSE42-NEXT:    pcmpeqw %xmm1, %xmm2
-; SSE42-NEXT:    pextrb $0, %xmm2, %eax
+; SSE42-NEXT:    packsswb %xmm0, %xmm2
+; SSE42-NEXT:    pmovmskb %xmm2, %eax
 ; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB11_2
-; SSE42-NEXT:  ## %bb.1: ## %cond.store
+; SSE42-NEXT:    jne LBB11_1
+; SSE42-NEXT:  ## %bb.2: ## %else
+; SSE42-NEXT:    testb $2, %al
+; SSE42-NEXT:    jne LBB11_3
+; SSE42-NEXT:  LBB11_4: ## %else2
+; SSE42-NEXT:    testb $4, %al
+; SSE42-NEXT:    jne LBB11_5
+; SSE42-NEXT:  LBB11_6: ## %else5
+; SSE42-NEXT:    testb $8, %al
+; SSE42-NEXT:    jne LBB11_7
+; SSE42-NEXT:  LBB11_8: ## %else8
+; SSE42-NEXT:    testb $16, %al
+; SSE42-NEXT:    jne LBB11_9
+; SSE42-NEXT:  LBB11_10: ## %else11
+; SSE42-NEXT:    testb $32, %al
+; SSE42-NEXT:    jne LBB11_11
+; SSE42-NEXT:  LBB11_12: ## %else14
+; SSE42-NEXT:    testb $64, %al
+; SSE42-NEXT:    jne LBB11_13
+; SSE42-NEXT:  LBB11_14: ## %else17
+; SSE42-NEXT:    testb $-128, %al
+; SSE42-NEXT:    jne LBB11_15
+; SSE42-NEXT:  LBB11_16: ## %else20
+; SSE42-NEXT:    retq
+; SSE42-NEXT:  LBB11_1: ## %cond.store
 ; SSE42-NEXT:    pextrw $0, %xmm0, (%rdi)
 ; SSE42-NEXT:    addq $2, %rdi
-; SSE42-NEXT:  LBB11_2: ## %else
-; SSE42-NEXT:    pextrb $2, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $2, %al
 ; SSE42-NEXT:    je LBB11_4
-; SSE42-NEXT:  ## %bb.3: ## %cond.store1
+; SSE42-NEXT:  LBB11_3: ## %cond.store1
 ; SSE42-NEXT:    pextrw $1, %xmm0, (%rdi)
 ; SSE42-NEXT:    addq $2, %rdi
-; SSE42-NEXT:  LBB11_4: ## %else2
-; SSE42-NEXT:    pxor %xmm2, %xmm2
-; SSE42-NEXT:    pcmpeqw %xmm1, %xmm2
-; SSE42-NEXT:    pextrb $4, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $4, %al
 ; SSE42-NEXT:    je LBB11_6
-; SSE42-NEXT:  ## %bb.5: ## %cond.store4
+; SSE42-NEXT:  LBB11_5: ## %cond.store4
 ; SSE42-NEXT:    pextrw $2, %xmm0, (%rdi)
 ; SSE42-NEXT:    addq $2, %rdi
-; SSE42-NEXT:  LBB11_6: ## %else5
-; SSE42-NEXT:    pextrb $6, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $8, %al
 ; SSE42-NEXT:    je LBB11_8
-; SSE42-NEXT:  ## %bb.7: ## %cond.store7
+; SSE42-NEXT:  LBB11_7: ## %cond.store7
 ; SSE42-NEXT:    pextrw $3, %xmm0, (%rdi)
 ; SSE42-NEXT:    addq $2, %rdi
-; SSE42-NEXT:  LBB11_8: ## %else8
-; SSE42-NEXT:    pxor %xmm2, %xmm2
-; SSE42-NEXT:    pcmpeqw %xmm1, %xmm2
-; SSE42-NEXT:    pextrb $8, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $16, %al
 ; SSE42-NEXT:    je LBB11_10
-; SSE42-NEXT:  ## %bb.9: ## %cond.store10
+; SSE42-NEXT:  LBB11_9: ## %cond.store10
 ; SSE42-NEXT:    pextrw $4, %xmm0, (%rdi)
 ; SSE42-NEXT:    addq $2, %rdi
-; SSE42-NEXT:  LBB11_10: ## %else11
-; SSE42-NEXT:    pextrb $10, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $32, %al
 ; SSE42-NEXT:    je LBB11_12
-; SSE42-NEXT:  ## %bb.11: ## %cond.store13
+; SSE42-NEXT:  LBB11_11: ## %cond.store13
 ; SSE42-NEXT:    pextrw $5, %xmm0, (%rdi)
 ; SSE42-NEXT:    addq $2, %rdi
-; SSE42-NEXT:  LBB11_12: ## %else14
-; SSE42-NEXT:    pxor %xmm2, %xmm2
-; SSE42-NEXT:    pcmpeqw %xmm2, %xmm1
-; SSE42-NEXT:    pextrb $12, %xmm1, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $64, %al
 ; SSE42-NEXT:    je LBB11_14
-; SSE42-NEXT:  ## %bb.13: ## %cond.store16
+; SSE42-NEXT:  LBB11_13: ## %cond.store16
 ; SSE42-NEXT:    pextrw $6, %xmm0, (%rdi)
 ; SSE42-NEXT:    addq $2, %rdi
-; SSE42-NEXT:  LBB11_14: ## %else17
-; SSE42-NEXT:    pextrb $14, %xmm1, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $-128, %al
 ; SSE42-NEXT:    je LBB11_16
-; SSE42-NEXT:  ## %bb.15: ## %cond.store19
+; SSE42-NEXT:  LBB11_15: ## %cond.store19
 ; SSE42-NEXT:    pextrw $7, %xmm0, (%rdi)
-; SSE42-NEXT:  LBB11_16: ## %else20
 ; SSE42-NEXT:    retq
 ;
 ; AVX1OR2-LABEL: compressstore_v8i16_v8i16:
 ; AVX1OR2:       ## %bb.0:
 ; AVX1OR2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1OR2-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm2
-; AVX1OR2-NEXT:    vpextrb $0, %xmm2, %eax
+; AVX1OR2-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm1
+; AVX1OR2-NEXT:    vpacksswb %xmm0, %xmm1, %xmm1
+; AVX1OR2-NEXT:    vpmovmskb %xmm1, %eax
 ; AVX1OR2-NEXT:    testb $1, %al
-; AVX1OR2-NEXT:    je LBB11_2
-; AVX1OR2-NEXT:  ## %bb.1: ## %cond.store
+; AVX1OR2-NEXT:    jne LBB11_1
+; AVX1OR2-NEXT:  ## %bb.2: ## %else
+; AVX1OR2-NEXT:    testb $2, %al
+; AVX1OR2-NEXT:    jne LBB11_3
+; AVX1OR2-NEXT:  LBB11_4: ## %else2
+; AVX1OR2-NEXT:    testb $4, %al
+; AVX1OR2-NEXT:    jne LBB11_5
+; AVX1OR2-NEXT:  LBB11_6: ## %else5
+; AVX1OR2-NEXT:    testb $8, %al
+; AVX1OR2-NEXT:    jne LBB11_7
+; AVX1OR2-NEXT:  LBB11_8: ## %else8
+; AVX1OR2-NEXT:    testb $16, %al
+; AVX1OR2-NEXT:    jne LBB11_9
+; AVX1OR2-NEXT:  LBB11_10: ## %else11
+; AVX1OR2-NEXT:    testb $32, %al
+; AVX1OR2-NEXT:    jne LBB11_11
+; AVX1OR2-NEXT:  LBB11_12: ## %else14
+; AVX1OR2-NEXT:    testb $64, %al
+; AVX1OR2-NEXT:    jne LBB11_13
+; AVX1OR2-NEXT:  LBB11_14: ## %else17
+; AVX1OR2-NEXT:    testb $-128, %al
+; AVX1OR2-NEXT:    jne LBB11_15
+; AVX1OR2-NEXT:  LBB11_16: ## %else20
+; AVX1OR2-NEXT:    retq
+; AVX1OR2-NEXT:  LBB11_1: ## %cond.store
 ; AVX1OR2-NEXT:    vpextrw $0, %xmm0, (%rdi)
 ; AVX1OR2-NEXT:    addq $2, %rdi
-; AVX1OR2-NEXT:  LBB11_2: ## %else
-; AVX1OR2-NEXT:    vpextrb $2, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $2, %al
 ; AVX1OR2-NEXT:    je LBB11_4
-; AVX1OR2-NEXT:  ## %bb.3: ## %cond.store1
+; AVX1OR2-NEXT:  LBB11_3: ## %cond.store1
 ; AVX1OR2-NEXT:    vpextrw $1, %xmm0, (%rdi)
 ; AVX1OR2-NEXT:    addq $2, %rdi
-; AVX1OR2-NEXT:  LBB11_4: ## %else2
-; AVX1OR2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1OR2-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm2
-; AVX1OR2-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $4, %al
 ; AVX1OR2-NEXT:    je LBB11_6
-; AVX1OR2-NEXT:  ## %bb.5: ## %cond.store4
+; AVX1OR2-NEXT:  LBB11_5: ## %cond.store4
 ; AVX1OR2-NEXT:    vpextrw $2, %xmm0, (%rdi)
 ; AVX1OR2-NEXT:    addq $2, %rdi
-; AVX1OR2-NEXT:  LBB11_6: ## %else5
-; AVX1OR2-NEXT:    vpextrb $6, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $8, %al
 ; AVX1OR2-NEXT:    je LBB11_8
-; AVX1OR2-NEXT:  ## %bb.7: ## %cond.store7
+; AVX1OR2-NEXT:  LBB11_7: ## %cond.store7
 ; AVX1OR2-NEXT:    vpextrw $3, %xmm0, (%rdi)
 ; AVX1OR2-NEXT:    addq $2, %rdi
-; AVX1OR2-NEXT:  LBB11_8: ## %else8
-; AVX1OR2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1OR2-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm2
-; AVX1OR2-NEXT:    vpextrb $8, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $16, %al
 ; AVX1OR2-NEXT:    je LBB11_10
-; AVX1OR2-NEXT:  ## %bb.9: ## %cond.store10
+; AVX1OR2-NEXT:  LBB11_9: ## %cond.store10
 ; AVX1OR2-NEXT:    vpextrw $4, %xmm0, (%rdi)
 ; AVX1OR2-NEXT:    addq $2, %rdi
-; AVX1OR2-NEXT:  LBB11_10: ## %else11
-; AVX1OR2-NEXT:    vpextrb $10, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $32, %al
 ; AVX1OR2-NEXT:    je LBB11_12
-; AVX1OR2-NEXT:  ## %bb.11: ## %cond.store13
+; AVX1OR2-NEXT:  LBB11_11: ## %cond.store13
 ; AVX1OR2-NEXT:    vpextrw $5, %xmm0, (%rdi)
 ; AVX1OR2-NEXT:    addq $2, %rdi
-; AVX1OR2-NEXT:  LBB11_12: ## %else14
-; AVX1OR2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1OR2-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm1
-; AVX1OR2-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $64, %al
 ; AVX1OR2-NEXT:    je LBB11_14
-; AVX1OR2-NEXT:  ## %bb.13: ## %cond.store16
+; AVX1OR2-NEXT:  LBB11_13: ## %cond.store16
 ; AVX1OR2-NEXT:    vpextrw $6, %xmm0, (%rdi)
 ; AVX1OR2-NEXT:    addq $2, %rdi
-; AVX1OR2-NEXT:  LBB11_14: ## %else17
-; AVX1OR2-NEXT:    vpextrb $14, %xmm1, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $-128, %al
 ; AVX1OR2-NEXT:    je LBB11_16
-; AVX1OR2-NEXT:  ## %bb.15: ## %cond.store19
+; AVX1OR2-NEXT:  LBB11_15: ## %cond.store19
 ; AVX1OR2-NEXT:    vpextrw $7, %xmm0, (%rdi)
-; AVX1OR2-NEXT:  LBB11_16: ## %else20
 ; AVX1OR2-NEXT:    retq
 ;
 ; AVX512F-LABEL: compressstore_v8i16_v8i16:
 ; AVX512F:       ## %bb.0:
 ; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vpmovsxwq %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmq %zmm2, %zmm2, %k0
+; AVX512F-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm1
+; AVX512F-NEXT:    vpmovsxwq %xmm1, %zmm1
+; AVX512F-NEXT:    vptestmq %zmm1, %zmm1, %k0
 ; AVX512F-NEXT:    kmovw %k0, %eax
 ; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je LBB11_2
-; AVX512F-NEXT:  ## %bb.1: ## %cond.store
+; AVX512F-NEXT:    jne LBB11_1
+; AVX512F-NEXT:  ## %bb.2: ## %else
+; AVX512F-NEXT:    testb $2, %al
+; AVX512F-NEXT:    jne LBB11_3
+; AVX512F-NEXT:  LBB11_4: ## %else2
+; AVX512F-NEXT:    testb $4, %al
+; AVX512F-NEXT:    jne LBB11_5
+; AVX512F-NEXT:  LBB11_6: ## %else5
+; AVX512F-NEXT:    testb $8, %al
+; AVX512F-NEXT:    jne LBB11_7
+; AVX512F-NEXT:  LBB11_8: ## %else8
+; AVX512F-NEXT:    testb $16, %al
+; AVX512F-NEXT:    jne LBB11_9
+; AVX512F-NEXT:  LBB11_10: ## %else11
+; AVX512F-NEXT:    testb $32, %al
+; AVX512F-NEXT:    jne LBB11_11
+; AVX512F-NEXT:  LBB11_12: ## %else14
+; AVX512F-NEXT:    testb $64, %al
+; AVX512F-NEXT:    jne LBB11_13
+; AVX512F-NEXT:  LBB11_14: ## %else17
+; AVX512F-NEXT:    testb $-128, %al
+; AVX512F-NEXT:    jne LBB11_15
+; AVX512F-NEXT:  LBB11_16: ## %else20
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+; AVX512F-NEXT:  LBB11_1: ## %cond.store
 ; AVX512F-NEXT:    vpextrw $0, %xmm0, (%rdi)
 ; AVX512F-NEXT:    addq $2, %rdi
-; AVX512F-NEXT:  LBB11_2: ## %else
-; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $2, %al
 ; AVX512F-NEXT:    je LBB11_4
-; AVX512F-NEXT:  ## %bb.3: ## %cond.store1
+; AVX512F-NEXT:  LBB11_3: ## %cond.store1
 ; AVX512F-NEXT:    vpextrw $1, %xmm0, (%rdi)
 ; AVX512F-NEXT:    addq $2, %rdi
-; AVX512F-NEXT:  LBB11_4: ## %else2
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vpmovsxwq %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmq %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $2, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $4, %al
 ; AVX512F-NEXT:    je LBB11_6
-; AVX512F-NEXT:  ## %bb.5: ## %cond.store4
+; AVX512F-NEXT:  LBB11_5: ## %cond.store4
 ; AVX512F-NEXT:    vpextrw $2, %xmm0, (%rdi)
 ; AVX512F-NEXT:    addq $2, %rdi
-; AVX512F-NEXT:  LBB11_6: ## %else5
-; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $8, %al
 ; AVX512F-NEXT:    je LBB11_8
-; AVX512F-NEXT:  ## %bb.7: ## %cond.store7
+; AVX512F-NEXT:  LBB11_7: ## %cond.store7
 ; AVX512F-NEXT:    vpextrw $3, %xmm0, (%rdi)
 ; AVX512F-NEXT:    addq $2, %rdi
-; AVX512F-NEXT:  LBB11_8: ## %else8
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vpmovsxwq %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmq %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $4, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $16, %al
 ; AVX512F-NEXT:    je LBB11_10
-; AVX512F-NEXT:  ## %bb.9: ## %cond.store10
+; AVX512F-NEXT:  LBB11_9: ## %cond.store10
 ; AVX512F-NEXT:    vpextrw $4, %xmm0, (%rdi)
 ; AVX512F-NEXT:    addq $2, %rdi
-; AVX512F-NEXT:  LBB11_10: ## %else11
-; AVX512F-NEXT:    kshiftrw $5, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $32, %al
 ; AVX512F-NEXT:    je LBB11_12
-; AVX512F-NEXT:  ## %bb.11: ## %cond.store13
+; AVX512F-NEXT:  LBB11_11: ## %cond.store13
 ; AVX512F-NEXT:    vpextrw $5, %xmm0, (%rdi)
 ; AVX512F-NEXT:    addq $2, %rdi
-; AVX512F-NEXT:  LBB11_12: ## %else14
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm1
-; AVX512F-NEXT:    vpmovsxwq %xmm1, %zmm1
-; AVX512F-NEXT:    vptestmq %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $6, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $64, %al
 ; AVX512F-NEXT:    je LBB11_14
-; AVX512F-NEXT:  ## %bb.13: ## %cond.store16
+; AVX512F-NEXT:  LBB11_13: ## %cond.store16
 ; AVX512F-NEXT:    vpextrw $6, %xmm0, (%rdi)
 ; AVX512F-NEXT:    addq $2, %rdi
-; AVX512F-NEXT:  LBB11_14: ## %else17
-; AVX512F-NEXT:    kshiftrw $7, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $-128, %al
 ; AVX512F-NEXT:    je LBB11_16
-; AVX512F-NEXT:  ## %bb.15: ## %cond.store19
+; AVX512F-NEXT:  LBB11_15: ## %cond.store19
 ; AVX512F-NEXT:    vpextrw $7, %xmm0, (%rdi)
-; AVX512F-NEXT:  LBB11_16: ## %else20
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VLDQ-LABEL: compressstore_v8i16_v8i16:
 ; AVX512VLDQ:       ## %bb.0:
 ; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm2
-; AVX512VLDQ-NEXT:    vpmovsxwd %xmm2, %ymm2
-; AVX512VLDQ-NEXT:    vpmovd2m %ymm2, %k0
+; AVX512VLDQ-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm1
+; AVX512VLDQ-NEXT:    vpmovsxwd %xmm1, %ymm1
+; AVX512VLDQ-NEXT:    vpmovd2m %ymm1, %k0
 ; AVX512VLDQ-NEXT:    kmovw %k0, %eax
 ; AVX512VLDQ-NEXT:    testb $1, %al
-; AVX512VLDQ-NEXT:    je LBB11_2
-; AVX512VLDQ-NEXT:  ## %bb.1: ## %cond.store
+; AVX512VLDQ-NEXT:    jne LBB11_1
+; AVX512VLDQ-NEXT:  ## %bb.2: ## %else
+; AVX512VLDQ-NEXT:    testb $2, %al
+; AVX512VLDQ-NEXT:    jne LBB11_3
+; AVX512VLDQ-NEXT:  LBB11_4: ## %else2
+; AVX512VLDQ-NEXT:    testb $4, %al
+; AVX512VLDQ-NEXT:    jne LBB11_5
+; AVX512VLDQ-NEXT:  LBB11_6: ## %else5
+; AVX512VLDQ-NEXT:    testb $8, %al
+; AVX512VLDQ-NEXT:    jne LBB11_7
+; AVX512VLDQ-NEXT:  LBB11_8: ## %else8
+; AVX512VLDQ-NEXT:    testb $16, %al
+; AVX512VLDQ-NEXT:    jne LBB11_9
+; AVX512VLDQ-NEXT:  LBB11_10: ## %else11
+; AVX512VLDQ-NEXT:    testb $32, %al
+; AVX512VLDQ-NEXT:    jne LBB11_11
+; AVX512VLDQ-NEXT:  LBB11_12: ## %else14
+; AVX512VLDQ-NEXT:    testb $64, %al
+; AVX512VLDQ-NEXT:    jne LBB11_13
+; AVX512VLDQ-NEXT:  LBB11_14: ## %else17
+; AVX512VLDQ-NEXT:    testb $-128, %al
+; AVX512VLDQ-NEXT:    jne LBB11_15
+; AVX512VLDQ-NEXT:  LBB11_16: ## %else20
+; AVX512VLDQ-NEXT:    vzeroupper
+; AVX512VLDQ-NEXT:    retq
+; AVX512VLDQ-NEXT:  LBB11_1: ## %cond.store
 ; AVX512VLDQ-NEXT:    vpextrw $0, %xmm0, (%rdi)
 ; AVX512VLDQ-NEXT:    addq $2, %rdi
-; AVX512VLDQ-NEXT:  LBB11_2: ## %else
-; AVX512VLDQ-NEXT:    kshiftrb $1, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $2, %al
 ; AVX512VLDQ-NEXT:    je LBB11_4
-; AVX512VLDQ-NEXT:  ## %bb.3: ## %cond.store1
+; AVX512VLDQ-NEXT:  LBB11_3: ## %cond.store1
 ; AVX512VLDQ-NEXT:    vpextrw $1, %xmm0, (%rdi)
 ; AVX512VLDQ-NEXT:    addq $2, %rdi
-; AVX512VLDQ-NEXT:  LBB11_4: ## %else2
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm2
-; AVX512VLDQ-NEXT:    vpmovsxwd %xmm2, %ymm2
-; AVX512VLDQ-NEXT:    vpmovd2m %ymm2, %k0
-; AVX512VLDQ-NEXT:    kshiftrb $2, %k0, %k1
-; AVX512VLDQ-NEXT:    kmovw %k1, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $4, %al
 ; AVX512VLDQ-NEXT:    je LBB11_6
-; AVX512VLDQ-NEXT:  ## %bb.5: ## %cond.store4
+; AVX512VLDQ-NEXT:  LBB11_5: ## %cond.store4
 ; AVX512VLDQ-NEXT:    vpextrw $2, %xmm0, (%rdi)
 ; AVX512VLDQ-NEXT:    addq $2, %rdi
-; AVX512VLDQ-NEXT:  LBB11_6: ## %else5
-; AVX512VLDQ-NEXT:    kshiftrb $3, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $8, %al
 ; AVX512VLDQ-NEXT:    je LBB11_8
-; AVX512VLDQ-NEXT:  ## %bb.7: ## %cond.store7
+; AVX512VLDQ-NEXT:  LBB11_7: ## %cond.store7
 ; AVX512VLDQ-NEXT:    vpextrw $3, %xmm0, (%rdi)
 ; AVX512VLDQ-NEXT:    addq $2, %rdi
-; AVX512VLDQ-NEXT:  LBB11_8: ## %else8
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm2
-; AVX512VLDQ-NEXT:    vpmovsxwd %xmm2, %ymm2
-; AVX512VLDQ-NEXT:    vpmovd2m %ymm2, %k0
-; AVX512VLDQ-NEXT:    kshiftrb $4, %k0, %k1
-; AVX512VLDQ-NEXT:    kmovw %k1, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $16, %al
 ; AVX512VLDQ-NEXT:    je LBB11_10
-; AVX512VLDQ-NEXT:  ## %bb.9: ## %cond.store10
+; AVX512VLDQ-NEXT:  LBB11_9: ## %cond.store10
 ; AVX512VLDQ-NEXT:    vpextrw $4, %xmm0, (%rdi)
 ; AVX512VLDQ-NEXT:    addq $2, %rdi
-; AVX512VLDQ-NEXT:  LBB11_10: ## %else11
-; AVX512VLDQ-NEXT:    kshiftrb $5, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $32, %al
 ; AVX512VLDQ-NEXT:    je LBB11_12
-; AVX512VLDQ-NEXT:  ## %bb.11: ## %cond.store13
+; AVX512VLDQ-NEXT:  LBB11_11: ## %cond.store13
 ; AVX512VLDQ-NEXT:    vpextrw $5, %xmm0, (%rdi)
 ; AVX512VLDQ-NEXT:    addq $2, %rdi
-; AVX512VLDQ-NEXT:  LBB11_12: ## %else14
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm1
-; AVX512VLDQ-NEXT:    vpmovsxwd %xmm1, %ymm1
-; AVX512VLDQ-NEXT:    vpmovd2m %ymm1, %k0
-; AVX512VLDQ-NEXT:    kshiftrb $6, %k0, %k1
-; AVX512VLDQ-NEXT:    kmovw %k1, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $64, %al
 ; AVX512VLDQ-NEXT:    je LBB11_14
-; AVX512VLDQ-NEXT:  ## %bb.13: ## %cond.store16
+; AVX512VLDQ-NEXT:  LBB11_13: ## %cond.store16
 ; AVX512VLDQ-NEXT:    vpextrw $6, %xmm0, (%rdi)
 ; AVX512VLDQ-NEXT:    addq $2, %rdi
-; AVX512VLDQ-NEXT:  LBB11_14: ## %else17
-; AVX512VLDQ-NEXT:    kshiftrb $7, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $-128, %al
 ; AVX512VLDQ-NEXT:    je LBB11_16
-; AVX512VLDQ-NEXT:  ## %bb.15: ## %cond.store19
+; AVX512VLDQ-NEXT:  LBB11_15: ## %cond.store19
 ; AVX512VLDQ-NEXT:    vpextrw $7, %xmm0, (%rdi)
-; AVX512VLDQ-NEXT:  LBB11_16: ## %else20
 ; AVX512VLDQ-NEXT:    vzeroupper
 ; AVX512VLDQ-NEXT:    retq
 ;
@@ -3547,66 +3547,67 @@ define void @compressstore_v8i16_v8i16(i16* %base, <8 x i16> %V, <8 x i16> %trig
 ; AVX512VLBW-NEXT:    vptestnmw %xmm1, %xmm1, %k0
 ; AVX512VLBW-NEXT:    kmovd %k0, %eax
 ; AVX512VLBW-NEXT:    testb $1, %al
-; AVX512VLBW-NEXT:    je LBB11_2
-; AVX512VLBW-NEXT:  ## %bb.1: ## %cond.store
+; AVX512VLBW-NEXT:    jne LBB11_1
+; AVX512VLBW-NEXT:  ## %bb.2: ## %else
+; AVX512VLBW-NEXT:    testb $2, %al
+; AVX512VLBW-NEXT:    jne LBB11_3
+; AVX512VLBW-NEXT:  LBB11_4: ## %else2
+; AVX512VLBW-NEXT:    testb $4, %al
+; AVX512VLBW-NEXT:    jne LBB11_5
+; AVX512VLBW-NEXT:  LBB11_6: ## %else5
+; AVX512VLBW-NEXT:    testb $8, %al
+; AVX512VLBW-NEXT:    jne LBB11_7
+; AVX512VLBW-NEXT:  LBB11_8: ## %else8
+; AVX512VLBW-NEXT:    testb $16, %al
+; AVX512VLBW-NEXT:    jne LBB11_9
+; AVX512VLBW-NEXT:  LBB11_10: ## %else11
+; AVX512VLBW-NEXT:    testb $32, %al
+; AVX512VLBW-NEXT:    jne LBB11_11
+; AVX512VLBW-NEXT:  LBB11_12: ## %else14
+; AVX512VLBW-NEXT:    testb $64, %al
+; AVX512VLBW-NEXT:    jne LBB11_13
+; AVX512VLBW-NEXT:  LBB11_14: ## %else17
+; AVX512VLBW-NEXT:    testb $-128, %al
+; AVX512VLBW-NEXT:    jne LBB11_15
+; AVX512VLBW-NEXT:  LBB11_16: ## %else20
+; AVX512VLBW-NEXT:    retq
+; AVX512VLBW-NEXT:  LBB11_1: ## %cond.store
 ; AVX512VLBW-NEXT:    vpextrw $0, %xmm0, (%rdi)
 ; AVX512VLBW-NEXT:    addq $2, %rdi
-; AVX512VLBW-NEXT:  LBB11_2: ## %else
-; AVX512VLBW-NEXT:    kshiftrw $1, %k0, %k1
-; AVX512VLBW-NEXT:    kmovd %k1, %eax
-; AVX512VLBW-NEXT:    testb $1, %al
+; AVX512VLBW-NEXT:    testb $2, %al
 ; AVX512VLBW-NEXT:    je LBB11_4
-; AVX512VLBW-NEXT:  ## %bb.3: ## %cond.store1
+; AVX512VLBW-NEXT:  LBB11_3: ## %cond.store1
 ; AVX512VLBW-NEXT:    vpextrw $1, %xmm0, (%rdi)
 ; AVX512VLBW-NEXT:    addq $2, %rdi
-; AVX512VLBW-NEXT:  LBB11_4: ## %else2
-; AVX512VLBW-NEXT:    kshiftrw $2, %k0, %k1
-; AVX512VLBW-NEXT:    kmovd %k1, %eax
-; AVX512VLBW-NEXT:    testb $1, %al
+; AVX512VLBW-NEXT:    testb $4, %al
 ; AVX512VLBW-NEXT:    je LBB11_6
-; AVX512VLBW-NEXT:  ## %bb.5: ## %cond.store4
+; AVX512VLBW-NEXT:  LBB11_5: ## %cond.store4
 ; AVX512VLBW-NEXT:    vpextrw $2, %xmm0, (%rdi)
 ; AVX512VLBW-NEXT:    addq $2, %rdi
-; AVX512VLBW-NEXT:  LBB11_6: ## %else5
-; AVX512VLBW-NEXT:    kshiftrw $3, %k0, %k1
-; AVX512VLBW-NEXT:    kmovd %k1, %eax
-; AVX512VLBW-NEXT:    testb $1, %al
+; AVX512VLBW-NEXT:    testb $8, %al
 ; AVX512VLBW-NEXT:    je LBB11_8
-; AVX512VLBW-NEXT:  ## %bb.7: ## %cond.store7
+; AVX512VLBW-NEXT:  LBB11_7: ## %cond.store7
 ; AVX512VLBW-NEXT:    vpextrw $3, %xmm0, (%rdi)
 ; AVX512VLBW-NEXT:    addq $2, %rdi
-; AVX512VLBW-NEXT:  LBB11_8: ## %else8
-; AVX512VLBW-NEXT:    kshiftrw $4, %k0, %k1
-; AVX512VLBW-NEXT:    kmovd %k1, %eax
-; AVX512VLBW-NEXT:    testb $1, %al
+; AVX512VLBW-NEXT:    testb $16, %al
 ; AVX512VLBW-NEXT:    je LBB11_10
-; AVX512VLBW-NEXT:  ## %bb.9: ## %cond.store10
+; AVX512VLBW-NEXT:  LBB11_9: ## %cond.store10
 ; AVX512VLBW-NEXT:    vpextrw $4, %xmm0, (%rdi)
 ; AVX512VLBW-NEXT:    addq $2, %rdi
-; AVX512VLBW-NEXT:  LBB11_10: ## %else11
-; AVX512VLBW-NEXT:    kshiftrw $5, %k0, %k1
-; AVX512VLBW-NEXT:    kmovd %k1, %eax
-; AVX512VLBW-NEXT:    testb $1, %al
+; AVX512VLBW-NEXT:    testb $32, %al
 ; AVX512VLBW-NEXT:    je LBB11_12
-; AVX512VLBW-NEXT:  ## %bb.11: ## %cond.store13
+; AVX512VLBW-NEXT:  LBB11_11: ## %cond.store13
 ; AVX512VLBW-NEXT:    vpextrw $5, %xmm0, (%rdi)
 ; AVX512VLBW-NEXT:    addq $2, %rdi
-; AVX512VLBW-NEXT:  LBB11_12: ## %else14
-; AVX512VLBW-NEXT:    kshiftrw $6, %k0, %k1
-; AVX512VLBW-NEXT:    kmovd %k1, %eax
-; AVX512VLBW-NEXT:    testb $1, %al
+; AVX512VLBW-NEXT:    testb $64, %al
 ; AVX512VLBW-NEXT:    je LBB11_14
-; AVX512VLBW-NEXT:  ## %bb.13: ## %cond.store16
+; AVX512VLBW-NEXT:  LBB11_13: ## %cond.store16
 ; AVX512VLBW-NEXT:    vpextrw $6, %xmm0, (%rdi)
 ; AVX512VLBW-NEXT:    addq $2, %rdi
-; AVX512VLBW-NEXT:  LBB11_14: ## %else17
-; AVX512VLBW-NEXT:    kshiftrw $7, %k0, %k0
-; AVX512VLBW-NEXT:    kmovd %k0, %eax
-; AVX512VLBW-NEXT:    testb $1, %al
+; AVX512VLBW-NEXT:    testb $-128, %al
 ; AVX512VLBW-NEXT:    je LBB11_16
-; AVX512VLBW-NEXT:  ## %bb.15: ## %cond.store19
+; AVX512VLBW-NEXT:  LBB11_15: ## %cond.store19
 ; AVX512VLBW-NEXT:    vpextrw $7, %xmm0, (%rdi)
-; AVX512VLBW-NEXT:  LBB11_16: ## %else20
 ; AVX512VLBW-NEXT:    retq
   %mask = icmp eq <8 x i16> %trigger, zeroinitializer
   call void @llvm.masked.compressstore.v8i16(<8 x i16> %V, i16* %base, <8 x i1> %mask)
@@ -3622,731 +3623,654 @@ define void @compressstore_v16i8_v16i8(i8* %base, <16 x i8> %V, <16 x i8> %trigg
 ; SSE2:       ## %bb.0:
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
 ; SSE2-NEXT:    pcmpeqb %xmm1, %xmm2
-; SSE2-NEXT:    movd %xmm2, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    je LBB12_2
-; SSE2-NEXT:  ## %bb.1: ## %cond.store
-; SSE2-NEXT:    movb %al, (%rdi)
-; SSE2-NEXT:    incq %rdi
-; SSE2-NEXT:  LBB12_2: ## %else
-; SSE2-NEXT:    shrl $8, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    je LBB12_4
-; SSE2-NEXT:  ## %bb.3: ## %cond.store1
-; SSE2-NEXT:    movb %ah, (%rdi)
-; SSE2-NEXT:    incq %rdi
+; SSE2-NEXT:    pmovmskb %xmm2, %eax
+; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    movd %xmm0, %ecx
+; SSE2-NEXT:    jne LBB12_1
+; SSE2-NEXT:  ## %bb.2: ## %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne LBB12_3
 ; SSE2-NEXT:  LBB12_4: ## %else2
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpeqb %xmm1, %xmm2
-; SSE2-NEXT:    movd %xmm2, %ecx
-; SSE2-NEXT:    movl %ecx, %edx
-; SSE2-NEXT:    shrl $16, %edx
-; SSE2-NEXT:    testb $1, %dl
-; SSE2-NEXT:    je LBB12_6
-; SSE2-NEXT:  ## %bb.5: ## %cond.store4
-; SSE2-NEXT:    movl %eax, %edx
-; SSE2-NEXT:    shrl $16, %edx
-; SSE2-NEXT:    movb %dl, (%rdi)
-; SSE2-NEXT:    incq %rdi
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne LBB12_5
 ; SSE2-NEXT:  LBB12_6: ## %else5
-; SSE2-NEXT:    shrl $24, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je LBB12_8
-; SSE2-NEXT:  ## %bb.7: ## %cond.store7
-; SSE2-NEXT:    shrl $24, %eax
-; SSE2-NEXT:    movb %al, (%rdi)
+; SSE2-NEXT:  LBB12_7: ## %cond.store7
+; SSE2-NEXT:    shrl $24, %ecx
+; SSE2-NEXT:    movb %cl, (%rdi)
 ; SSE2-NEXT:    incq %rdi
 ; SSE2-NEXT:  LBB12_8: ## %else8
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpeqb %xmm1, %xmm2
-; SSE2-NEXT:    pextrw $2, %xmm2, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    pextrw $2, %xmm0, %eax
+; SSE2-NEXT:    testb $16, %al
+; SSE2-NEXT:    pextrw $2, %xmm0, %ecx
 ; SSE2-NEXT:    je LBB12_10
 ; SSE2-NEXT:  ## %bb.9: ## %cond.store10
-; SSE2-NEXT:    movb %al, (%rdi)
+; SSE2-NEXT:    movb %cl, (%rdi)
 ; SSE2-NEXT:    incq %rdi
 ; SSE2-NEXT:  LBB12_10: ## %else11
-; SSE2-NEXT:    shrl $8, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testb $32, %al
 ; SSE2-NEXT:    je LBB12_12
 ; SSE2-NEXT:  ## %bb.11: ## %cond.store13
-; SSE2-NEXT:    movb %ah, (%rdi)
+; SSE2-NEXT:    movb %ch, (%rdi)
 ; SSE2-NEXT:    incq %rdi
 ; SSE2-NEXT:  LBB12_12: ## %else14
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpeqb %xmm1, %xmm2
-; SSE2-NEXT:    pextrw $3, %xmm2, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    pextrw $3, %xmm0, %eax
+; SSE2-NEXT:    testb $64, %al
+; SSE2-NEXT:    pextrw $3, %xmm0, %ecx
 ; SSE2-NEXT:    je LBB12_14
 ; SSE2-NEXT:  ## %bb.13: ## %cond.store16
-; SSE2-NEXT:    movb %al, (%rdi)
+; SSE2-NEXT:    movb %cl, (%rdi)
 ; SSE2-NEXT:    incq %rdi
 ; SSE2-NEXT:  LBB12_14: ## %else17
-; SSE2-NEXT:    shrl $8, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testb $-128, %al
 ; SSE2-NEXT:    je LBB12_16
 ; SSE2-NEXT:  ## %bb.15: ## %cond.store19
-; SSE2-NEXT:    movb %ah, (%rdi)
+; SSE2-NEXT:    movb %ch, (%rdi)
 ; SSE2-NEXT:    incq %rdi
 ; SSE2-NEXT:  LBB12_16: ## %else20
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpeqb %xmm1, %xmm2
-; SSE2-NEXT:    pextrw $4, %xmm2, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
+; SSE2-NEXT:    testl $256, %eax ## imm = 0x100
+; SSE2-NEXT:    pextrw $4, %xmm0, %ecx
 ; SSE2-NEXT:    je LBB12_18
 ; SSE2-NEXT:  ## %bb.17: ## %cond.store22
-; SSE2-NEXT:    movb %al, (%rdi)
+; SSE2-NEXT:    movb %cl, (%rdi)
 ; SSE2-NEXT:    incq %rdi
 ; SSE2-NEXT:  LBB12_18: ## %else23
-; SSE2-NEXT:    shrl $8, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $512, %eax ## imm = 0x200
 ; SSE2-NEXT:    je LBB12_20
 ; SSE2-NEXT:  ## %bb.19: ## %cond.store25
-; SSE2-NEXT:    movb %ah, (%rdi)
+; SSE2-NEXT:    movb %ch, (%rdi)
 ; SSE2-NEXT:    incq %rdi
 ; SSE2-NEXT:  LBB12_20: ## %else26
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpeqb %xmm1, %xmm2
-; SSE2-NEXT:    pextrw $5, %xmm2, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    pextrw $5, %xmm0, %eax
+; SSE2-NEXT:    testl $1024, %eax ## imm = 0x400
+; SSE2-NEXT:    pextrw $5, %xmm0, %ecx
 ; SSE2-NEXT:    je LBB12_22
 ; SSE2-NEXT:  ## %bb.21: ## %cond.store28
-; SSE2-NEXT:    movb %al, (%rdi)
+; SSE2-NEXT:    movb %cl, (%rdi)
 ; SSE2-NEXT:    incq %rdi
 ; SSE2-NEXT:  LBB12_22: ## %else29
-; SSE2-NEXT:    shrl $8, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $2048, %eax ## imm = 0x800
 ; SSE2-NEXT:    je LBB12_24
 ; SSE2-NEXT:  ## %bb.23: ## %cond.store31
-; SSE2-NEXT:    movb %ah, (%rdi)
+; SSE2-NEXT:    movb %ch, (%rdi)
 ; SSE2-NEXT:    incq %rdi
 ; SSE2-NEXT:  LBB12_24: ## %else32
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpeqb %xmm1, %xmm2
-; SSE2-NEXT:    pextrw $6, %xmm2, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    pextrw $6, %xmm0, %eax
+; SSE2-NEXT:    testl $4096, %eax ## imm = 0x1000
+; SSE2-NEXT:    pextrw $6, %xmm0, %ecx
 ; SSE2-NEXT:    je LBB12_26
 ; SSE2-NEXT:  ## %bb.25: ## %cond.store34
-; SSE2-NEXT:    movb %al, (%rdi)
+; SSE2-NEXT:    movb %cl, (%rdi)
 ; SSE2-NEXT:    incq %rdi
 ; SSE2-NEXT:  LBB12_26: ## %else35
-; SSE2-NEXT:    shrl $8, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $8192, %eax ## imm = 0x2000
 ; SSE2-NEXT:    je LBB12_28
 ; SSE2-NEXT:  ## %bb.27: ## %cond.store37
-; SSE2-NEXT:    movb %ah, (%rdi)
+; SSE2-NEXT:    movb %ch, (%rdi)
 ; SSE2-NEXT:    incq %rdi
 ; SSE2-NEXT:  LBB12_28: ## %else38
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpeqb %xmm2, %xmm1
-; SSE2-NEXT:    pextrw $7, %xmm1, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    pextrw $7, %xmm0, %eax
-; SSE2-NEXT:    je LBB12_30
-; SSE2-NEXT:  ## %bb.29: ## %cond.store40
-; SSE2-NEXT:    movb %al, (%rdi)
+; SSE2-NEXT:    testl $16384, %eax ## imm = 0x4000
+; SSE2-NEXT:    pextrw $7, %xmm0, %ecx
+; SSE2-NEXT:    jne LBB12_29
+; SSE2-NEXT:  ## %bb.30: ## %else41
+; SSE2-NEXT:    testl $32768, %eax ## imm = 0x8000
+; SSE2-NEXT:    jne LBB12_31
+; SSE2-NEXT:  LBB12_32: ## %else44
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  LBB12_1: ## %cond.store
+; SSE2-NEXT:    movb %cl, (%rdi)
+; SSE2-NEXT:    incq %rdi
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    je LBB12_4
+; SSE2-NEXT:  LBB12_3: ## %cond.store1
+; SSE2-NEXT:    movb %ch, (%rdi)
+; SSE2-NEXT:    incq %rdi
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    je LBB12_6
+; SSE2-NEXT:  LBB12_5: ## %cond.store4
+; SSE2-NEXT:    movl %ecx, %edx
+; SSE2-NEXT:    shrl $16, %edx
+; SSE2-NEXT:    movb %dl, (%rdi)
 ; SSE2-NEXT:    incq %rdi
-; SSE2-NEXT:  LBB12_30: ## %else41
-; SSE2-NEXT:    shrl $8, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne LBB12_7
+; SSE2-NEXT:    jmp LBB12_8
+; SSE2-NEXT:  LBB12_29: ## %cond.store40
+; SSE2-NEXT:    movb %cl, (%rdi)
+; SSE2-NEXT:    incq %rdi
+; SSE2-NEXT:    testl $32768, %eax ## imm = 0x8000
 ; SSE2-NEXT:    je LBB12_32
-; SSE2-NEXT:  ## %bb.31: ## %cond.store43
-; SSE2-NEXT:    movb %ah, (%rdi)
-; SSE2-NEXT:  LBB12_32: ## %else44
+; SSE2-NEXT:  LBB12_31: ## %cond.store43
+; SSE2-NEXT:    movb %ch, (%rdi)
 ; SSE2-NEXT:    retq
 ;
 ; SSE42-LABEL: compressstore_v16i8_v16i8:
 ; SSE42:       ## %bb.0:
 ; SSE42-NEXT:    pxor %xmm2, %xmm2
 ; SSE42-NEXT:    pcmpeqb %xmm1, %xmm2
-; SSE42-NEXT:    pextrb $0, %xmm2, %eax
+; SSE42-NEXT:    pmovmskb %xmm2, %eax
 ; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB12_2
-; SSE42-NEXT:  ## %bb.1: ## %cond.store
+; SSE42-NEXT:    jne LBB12_1
+; SSE42-NEXT:  ## %bb.2: ## %else
+; SSE42-NEXT:    testb $2, %al
+; SSE42-NEXT:    jne LBB12_3
+; SSE42-NEXT:  LBB12_4: ## %else2
+; SSE42-NEXT:    testb $4, %al
+; SSE42-NEXT:    jne LBB12_5
+; SSE42-NEXT:  LBB12_6: ## %else5
+; SSE42-NEXT:    testb $8, %al
+; SSE42-NEXT:    jne LBB12_7
+; SSE42-NEXT:  LBB12_8: ## %else8
+; SSE42-NEXT:    testb $16, %al
+; SSE42-NEXT:    jne LBB12_9
+; SSE42-NEXT:  LBB12_10: ## %else11
+; SSE42-NEXT:    testb $32, %al
+; SSE42-NEXT:    jne LBB12_11
+; SSE42-NEXT:  LBB12_12: ## %else14
+; SSE42-NEXT:    testb $64, %al
+; SSE42-NEXT:    jne LBB12_13
+; SSE42-NEXT:  LBB12_14: ## %else17
+; SSE42-NEXT:    testb $-128, %al
+; SSE42-NEXT:    jne LBB12_15
+; SSE42-NEXT:  LBB12_16: ## %else20
+; SSE42-NEXT:    testl $256, %eax ## imm = 0x100
+; SSE42-NEXT:    jne LBB12_17
+; SSE42-NEXT:  LBB12_18: ## %else23
+; SSE42-NEXT:    testl $512, %eax ## imm = 0x200
+; SSE42-NEXT:    jne LBB12_19
+; SSE42-NEXT:  LBB12_20: ## %else26
+; SSE42-NEXT:    testl $1024, %eax ## imm = 0x400
+; SSE42-NEXT:    jne LBB12_21
+; SSE42-NEXT:  LBB12_22: ## %else29
+; SSE42-NEXT:    testl $2048, %eax ## imm = 0x800
+; SSE42-NEXT:    jne LBB12_23
+; SSE42-NEXT:  LBB12_24: ## %else32
+; SSE42-NEXT:    testl $4096, %eax ## imm = 0x1000
+; SSE42-NEXT:    jne LBB12_25
+; SSE42-NEXT:  LBB12_26: ## %else35
+; SSE42-NEXT:    testl $8192, %eax ## imm = 0x2000
+; SSE42-NEXT:    jne LBB12_27
+; SSE42-NEXT:  LBB12_28: ## %else38
+; SSE42-NEXT:    testl $16384, %eax ## imm = 0x4000
+; SSE42-NEXT:    jne LBB12_29
+; SSE42-NEXT:  LBB12_30: ## %else41
+; SSE42-NEXT:    testl $32768, %eax ## imm = 0x8000
+; SSE42-NEXT:    jne LBB12_31
+; SSE42-NEXT:  LBB12_32: ## %else44
+; SSE42-NEXT:    retq
+; SSE42-NEXT:  LBB12_1: ## %cond.store
 ; SSE42-NEXT:    pextrb $0, %xmm0, (%rdi)
 ; SSE42-NEXT:    incq %rdi
-; SSE42-NEXT:  LBB12_2: ## %else
-; SSE42-NEXT:    pextrb $1, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $2, %al
 ; SSE42-NEXT:    je LBB12_4
-; SSE42-NEXT:  ## %bb.3: ## %cond.store1
+; SSE42-NEXT:  LBB12_3: ## %cond.store1
 ; SSE42-NEXT:    pextrb $1, %xmm0, (%rdi)
 ; SSE42-NEXT:    incq %rdi
-; SSE42-NEXT:  LBB12_4: ## %else2
-; SSE42-NEXT:    pxor %xmm2, %xmm2
-; SSE42-NEXT:    pcmpeqb %xmm1, %xmm2
-; SSE42-NEXT:    pextrb $2, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $4, %al
 ; SSE42-NEXT:    je LBB12_6
-; SSE42-NEXT:  ## %bb.5: ## %cond.store4
+; SSE42-NEXT:  LBB12_5: ## %cond.store4
 ; SSE42-NEXT:    pextrb $2, %xmm0, (%rdi)
 ; SSE42-NEXT:    incq %rdi
-; SSE42-NEXT:  LBB12_6: ## %else5
-; SSE42-NEXT:    pextrb $3, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $8, %al
 ; SSE42-NEXT:    je LBB12_8
-; SSE42-NEXT:  ## %bb.7: ## %cond.store7
+; SSE42-NEXT:  LBB12_7: ## %cond.store7
 ; SSE42-NEXT:    pextrb $3, %xmm0, (%rdi)
 ; SSE42-NEXT:    incq %rdi
-; SSE42-NEXT:  LBB12_8: ## %else8
-; SSE42-NEXT:    pxor %xmm2, %xmm2
-; SSE42-NEXT:    pcmpeqb %xmm1, %xmm2
-; SSE42-NEXT:    pextrb $4, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $16, %al
 ; SSE42-NEXT:    je LBB12_10
-; SSE42-NEXT:  ## %bb.9: ## %cond.store10
+; SSE42-NEXT:  LBB12_9: ## %cond.store10
 ; SSE42-NEXT:    pextrb $4, %xmm0, (%rdi)
 ; SSE42-NEXT:    incq %rdi
-; SSE42-NEXT:  LBB12_10: ## %else11
-; SSE42-NEXT:    pextrb $5, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $32, %al
 ; SSE42-NEXT:    je LBB12_12
-; SSE42-NEXT:  ## %bb.11: ## %cond.store13
+; SSE42-NEXT:  LBB12_11: ## %cond.store13
 ; SSE42-NEXT:    pextrb $5, %xmm0, (%rdi)
 ; SSE42-NEXT:    incq %rdi
-; SSE42-NEXT:  LBB12_12: ## %else14
-; SSE42-NEXT:    pxor %xmm2, %xmm2
-; SSE42-NEXT:    pcmpeqb %xmm1, %xmm2
-; SSE42-NEXT:    pextrb $6, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $64, %al
 ; SSE42-NEXT:    je LBB12_14
-; SSE42-NEXT:  ## %bb.13: ## %cond.store16
+; SSE42-NEXT:  LBB12_13: ## %cond.store16
 ; SSE42-NEXT:    pextrb $6, %xmm0, (%rdi)
 ; SSE42-NEXT:    incq %rdi
-; SSE42-NEXT:  LBB12_14: ## %else17
-; SSE42-NEXT:    pextrb $7, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $-128, %al
 ; SSE42-NEXT:    je LBB12_16
-; SSE42-NEXT:  ## %bb.15: ## %cond.store19
+; SSE42-NEXT:  LBB12_15: ## %cond.store19
 ; SSE42-NEXT:    pextrb $7, %xmm0, (%rdi)
 ; SSE42-NEXT:    incq %rdi
-; SSE42-NEXT:  LBB12_16: ## %else20
-; SSE42-NEXT:    pxor %xmm2, %xmm2
-; SSE42-NEXT:    pcmpeqb %xmm1, %xmm2
-; SSE42-NEXT:    pextrb $8, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $256, %eax ## imm = 0x100
 ; SSE42-NEXT:    je LBB12_18
-; SSE42-NEXT:  ## %bb.17: ## %cond.store22
+; SSE42-NEXT:  LBB12_17: ## %cond.store22
 ; SSE42-NEXT:    pextrb $8, %xmm0, (%rdi)
 ; SSE42-NEXT:    incq %rdi
-; SSE42-NEXT:  LBB12_18: ## %else23
-; SSE42-NEXT:    pextrb $9, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $512, %eax ## imm = 0x200
 ; SSE42-NEXT:    je LBB12_20
-; SSE42-NEXT:  ## %bb.19: ## %cond.store25
+; SSE42-NEXT:  LBB12_19: ## %cond.store25
 ; SSE42-NEXT:    pextrb $9, %xmm0, (%rdi)
 ; SSE42-NEXT:    incq %rdi
-; SSE42-NEXT:  LBB12_20: ## %else26
-; SSE42-NEXT:    pxor %xmm2, %xmm2
-; SSE42-NEXT:    pcmpeqb %xmm1, %xmm2
-; SSE42-NEXT:    pextrb $10, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $1024, %eax ## imm = 0x400
 ; SSE42-NEXT:    je LBB12_22
-; SSE42-NEXT:  ## %bb.21: ## %cond.store28
+; SSE42-NEXT:  LBB12_21: ## %cond.store28
 ; SSE42-NEXT:    pextrb $10, %xmm0, (%rdi)
 ; SSE42-NEXT:    incq %rdi
-; SSE42-NEXT:  LBB12_22: ## %else29
-; SSE42-NEXT:    pextrb $11, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $2048, %eax ## imm = 0x800
 ; SSE42-NEXT:    je LBB12_24
-; SSE42-NEXT:  ## %bb.23: ## %cond.store31
+; SSE42-NEXT:  LBB12_23: ## %cond.store31
 ; SSE42-NEXT:    pextrb $11, %xmm0, (%rdi)
 ; SSE42-NEXT:    incq %rdi
-; SSE42-NEXT:  LBB12_24: ## %else32
-; SSE42-NEXT:    pxor %xmm2, %xmm2
-; SSE42-NEXT:    pcmpeqb %xmm1, %xmm2
-; SSE42-NEXT:    pextrb $12, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $4096, %eax ## imm = 0x1000
 ; SSE42-NEXT:    je LBB12_26
-; SSE42-NEXT:  ## %bb.25: ## %cond.store34
+; SSE42-NEXT:  LBB12_25: ## %cond.store34
 ; SSE42-NEXT:    pextrb $12, %xmm0, (%rdi)
 ; SSE42-NEXT:    incq %rdi
-; SSE42-NEXT:  LBB12_26: ## %else35
-; SSE42-NEXT:    pextrb $13, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $8192, %eax ## imm = 0x2000
 ; SSE42-NEXT:    je LBB12_28
-; SSE42-NEXT:  ## %bb.27: ## %cond.store37
+; SSE42-NEXT:  LBB12_27: ## %cond.store37
 ; SSE42-NEXT:    pextrb $13, %xmm0, (%rdi)
 ; SSE42-NEXT:    incq %rdi
-; SSE42-NEXT:  LBB12_28: ## %else38
-; SSE42-NEXT:    pxor %xmm2, %xmm2
-; SSE42-NEXT:    pcmpeqb %xmm2, %xmm1
-; SSE42-NEXT:    pextrb $14, %xmm1, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $16384, %eax ## imm = 0x4000
 ; SSE42-NEXT:    je LBB12_30
-; SSE42-NEXT:  ## %bb.29: ## %cond.store40
+; SSE42-NEXT:  LBB12_29: ## %cond.store40
 ; SSE42-NEXT:    pextrb $14, %xmm0, (%rdi)
 ; SSE42-NEXT:    incq %rdi
-; SSE42-NEXT:  LBB12_30: ## %else41
-; SSE42-NEXT:    pextrb $15, %xmm1, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $32768, %eax ## imm = 0x8000
 ; SSE42-NEXT:    je LBB12_32
-; SSE42-NEXT:  ## %bb.31: ## %cond.store43
+; SSE42-NEXT:  LBB12_31: ## %cond.store43
 ; SSE42-NEXT:    pextrb $15, %xmm0, (%rdi)
-; SSE42-NEXT:  LBB12_32: ## %else44
 ; SSE42-NEXT:    retq
 ;
 ; AVX1OR2-LABEL: compressstore_v16i8_v16i8:
 ; AVX1OR2:       ## %bb.0:
 ; AVX1OR2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1OR2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX1OR2-NEXT:    vpextrb $0, %xmm2, %eax
+; AVX1OR2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm1
+; AVX1OR2-NEXT:    vpmovmskb %xmm1, %eax
 ; AVX1OR2-NEXT:    testb $1, %al
-; AVX1OR2-NEXT:    je LBB12_2
-; AVX1OR2-NEXT:  ## %bb.1: ## %cond.store
+; AVX1OR2-NEXT:    jne LBB12_1
+; AVX1OR2-NEXT:  ## %bb.2: ## %else
+; AVX1OR2-NEXT:    testb $2, %al
+; AVX1OR2-NEXT:    jne LBB12_3
+; AVX1OR2-NEXT:  LBB12_4: ## %else2
+; AVX1OR2-NEXT:    testb $4, %al
+; AVX1OR2-NEXT:    jne LBB12_5
+; AVX1OR2-NEXT:  LBB12_6: ## %else5
+; AVX1OR2-NEXT:    testb $8, %al
+; AVX1OR2-NEXT:    jne LBB12_7
+; AVX1OR2-NEXT:  LBB12_8: ## %else8
+; AVX1OR2-NEXT:    testb $16, %al
+; AVX1OR2-NEXT:    jne LBB12_9
+; AVX1OR2-NEXT:  LBB12_10: ## %else11
+; AVX1OR2-NEXT:    testb $32, %al
+; AVX1OR2-NEXT:    jne LBB12_11
+; AVX1OR2-NEXT:  LBB12_12: ## %else14
+; AVX1OR2-NEXT:    testb $64, %al
+; AVX1OR2-NEXT:    jne LBB12_13
+; AVX1OR2-NEXT:  LBB12_14: ## %else17
+; AVX1OR2-NEXT:    testb $-128, %al
+; AVX1OR2-NEXT:    jne LBB12_15
+; AVX1OR2-NEXT:  LBB12_16: ## %else20
+; AVX1OR2-NEXT:    testl $256, %eax ## imm = 0x100
+; AVX1OR2-NEXT:    jne LBB12_17
+; AVX1OR2-NEXT:  LBB12_18: ## %else23
+; AVX1OR2-NEXT:    testl $512, %eax ## imm = 0x200
+; AVX1OR2-NEXT:    jne LBB12_19
+; AVX1OR2-NEXT:  LBB12_20: ## %else26
+; AVX1OR2-NEXT:    testl $1024, %eax ## imm = 0x400
+; AVX1OR2-NEXT:    jne LBB12_21
+; AVX1OR2-NEXT:  LBB12_22: ## %else29
+; AVX1OR2-NEXT:    testl $2048, %eax ## imm = 0x800
+; AVX1OR2-NEXT:    jne LBB12_23
+; AVX1OR2-NEXT:  LBB12_24: ## %else32
+; AVX1OR2-NEXT:    testl $4096, %eax ## imm = 0x1000
+; AVX1OR2-NEXT:    jne LBB12_25
+; AVX1OR2-NEXT:  LBB12_26: ## %else35
+; AVX1OR2-NEXT:    testl $8192, %eax ## imm = 0x2000
+; AVX1OR2-NEXT:    jne LBB12_27
+; AVX1OR2-NEXT:  LBB12_28: ## %else38
+; AVX1OR2-NEXT:    testl $16384, %eax ## imm = 0x4000
+; AVX1OR2-NEXT:    jne LBB12_29
+; AVX1OR2-NEXT:  LBB12_30: ## %else41
+; AVX1OR2-NEXT:    testl $32768, %eax ## imm = 0x8000
+; AVX1OR2-NEXT:    jne LBB12_31
+; AVX1OR2-NEXT:  LBB12_32: ## %else44
+; AVX1OR2-NEXT:    retq
+; AVX1OR2-NEXT:  LBB12_1: ## %cond.store
 ; AVX1OR2-NEXT:    vpextrb $0, %xmm0, (%rdi)
 ; AVX1OR2-NEXT:    incq %rdi
-; AVX1OR2-NEXT:  LBB12_2: ## %else
-; AVX1OR2-NEXT:    vpextrb $1, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $2, %al
 ; AVX1OR2-NEXT:    je LBB12_4
-; AVX1OR2-NEXT:  ## %bb.3: ## %cond.store1
+; AVX1OR2-NEXT:  LBB12_3: ## %cond.store1
 ; AVX1OR2-NEXT:    vpextrb $1, %xmm0, (%rdi)
 ; AVX1OR2-NEXT:    incq %rdi
-; AVX1OR2-NEXT:  LBB12_4: ## %else2
-; AVX1OR2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1OR2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX1OR2-NEXT:    vpextrb $2, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $4, %al
 ; AVX1OR2-NEXT:    je LBB12_6
-; AVX1OR2-NEXT:  ## %bb.5: ## %cond.store4
+; AVX1OR2-NEXT:  LBB12_5: ## %cond.store4
 ; AVX1OR2-NEXT:    vpextrb $2, %xmm0, (%rdi)
 ; AVX1OR2-NEXT:    incq %rdi
-; AVX1OR2-NEXT:  LBB12_6: ## %else5
-; AVX1OR2-NEXT:    vpextrb $3, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $8, %al
 ; AVX1OR2-NEXT:    je LBB12_8
-; AVX1OR2-NEXT:  ## %bb.7: ## %cond.store7
+; AVX1OR2-NEXT:  LBB12_7: ## %cond.store7
 ; AVX1OR2-NEXT:    vpextrb $3, %xmm0, (%rdi)
 ; AVX1OR2-NEXT:    incq %rdi
-; AVX1OR2-NEXT:  LBB12_8: ## %else8
-; AVX1OR2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1OR2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX1OR2-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $16, %al
 ; AVX1OR2-NEXT:    je LBB12_10
-; AVX1OR2-NEXT:  ## %bb.9: ## %cond.store10
+; AVX1OR2-NEXT:  LBB12_9: ## %cond.store10
 ; AVX1OR2-NEXT:    vpextrb $4, %xmm0, (%rdi)
 ; AVX1OR2-NEXT:    incq %rdi
-; AVX1OR2-NEXT:  LBB12_10: ## %else11
-; AVX1OR2-NEXT:    vpextrb $5, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $32, %al
 ; AVX1OR2-NEXT:    je LBB12_12
-; AVX1OR2-NEXT:  ## %bb.11: ## %cond.store13
+; AVX1OR2-NEXT:  LBB12_11: ## %cond.store13
 ; AVX1OR2-NEXT:    vpextrb $5, %xmm0, (%rdi)
 ; AVX1OR2-NEXT:    incq %rdi
-; AVX1OR2-NEXT:  LBB12_12: ## %else14
-; AVX1OR2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1OR2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX1OR2-NEXT:    vpextrb $6, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $64, %al
 ; AVX1OR2-NEXT:    je LBB12_14
-; AVX1OR2-NEXT:  ## %bb.13: ## %cond.store16
+; AVX1OR2-NEXT:  LBB12_13: ## %cond.store16
 ; AVX1OR2-NEXT:    vpextrb $6, %xmm0, (%rdi)
 ; AVX1OR2-NEXT:    incq %rdi
-; AVX1OR2-NEXT:  LBB12_14: ## %else17
-; AVX1OR2-NEXT:    vpextrb $7, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $-128, %al
 ; AVX1OR2-NEXT:    je LBB12_16
-; AVX1OR2-NEXT:  ## %bb.15: ## %cond.store19
+; AVX1OR2-NEXT:  LBB12_15: ## %cond.store19
 ; AVX1OR2-NEXT:    vpextrb $7, %xmm0, (%rdi)
 ; AVX1OR2-NEXT:    incq %rdi
-; AVX1OR2-NEXT:  LBB12_16: ## %else20
-; AVX1OR2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1OR2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX1OR2-NEXT:    vpextrb $8, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testl $256, %eax ## imm = 0x100
 ; AVX1OR2-NEXT:    je LBB12_18
-; AVX1OR2-NEXT:  ## %bb.17: ## %cond.store22
+; AVX1OR2-NEXT:  LBB12_17: ## %cond.store22
 ; AVX1OR2-NEXT:    vpextrb $8, %xmm0, (%rdi)
 ; AVX1OR2-NEXT:    incq %rdi
-; AVX1OR2-NEXT:  LBB12_18: ## %else23
-; AVX1OR2-NEXT:    vpextrb $9, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testl $512, %eax ## imm = 0x200
 ; AVX1OR2-NEXT:    je LBB12_20
-; AVX1OR2-NEXT:  ## %bb.19: ## %cond.store25
+; AVX1OR2-NEXT:  LBB12_19: ## %cond.store25
 ; AVX1OR2-NEXT:    vpextrb $9, %xmm0, (%rdi)
 ; AVX1OR2-NEXT:    incq %rdi
-; AVX1OR2-NEXT:  LBB12_20: ## %else26
-; AVX1OR2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1OR2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX1OR2-NEXT:    vpextrb $10, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testl $1024, %eax ## imm = 0x400
 ; AVX1OR2-NEXT:    je LBB12_22
-; AVX1OR2-NEXT:  ## %bb.21: ## %cond.store28
+; AVX1OR2-NEXT:  LBB12_21: ## %cond.store28
 ; AVX1OR2-NEXT:    vpextrb $10, %xmm0, (%rdi)
 ; AVX1OR2-NEXT:    incq %rdi
-; AVX1OR2-NEXT:  LBB12_22: ## %else29
-; AVX1OR2-NEXT:    vpextrb $11, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testl $2048, %eax ## imm = 0x800
 ; AVX1OR2-NEXT:    je LBB12_24
-; AVX1OR2-NEXT:  ## %bb.23: ## %cond.store31
+; AVX1OR2-NEXT:  LBB12_23: ## %cond.store31
 ; AVX1OR2-NEXT:    vpextrb $11, %xmm0, (%rdi)
 ; AVX1OR2-NEXT:    incq %rdi
-; AVX1OR2-NEXT:  LBB12_24: ## %else32
-; AVX1OR2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1OR2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX1OR2-NEXT:    vpextrb $12, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testl $4096, %eax ## imm = 0x1000
 ; AVX1OR2-NEXT:    je LBB12_26
-; AVX1OR2-NEXT:  ## %bb.25: ## %cond.store34
+; AVX1OR2-NEXT:  LBB12_25: ## %cond.store34
 ; AVX1OR2-NEXT:    vpextrb $12, %xmm0, (%rdi)
 ; AVX1OR2-NEXT:    incq %rdi
-; AVX1OR2-NEXT:  LBB12_26: ## %else35
-; AVX1OR2-NEXT:    vpextrb $13, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testl $8192, %eax ## imm = 0x2000
 ; AVX1OR2-NEXT:    je LBB12_28
-; AVX1OR2-NEXT:  ## %bb.27: ## %cond.store37
+; AVX1OR2-NEXT:  LBB12_27: ## %cond.store37
 ; AVX1OR2-NEXT:    vpextrb $13, %xmm0, (%rdi)
 ; AVX1OR2-NEXT:    incq %rdi
-; AVX1OR2-NEXT:  LBB12_28: ## %else38
-; AVX1OR2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1OR2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm1
-; AVX1OR2-NEXT:    vpextrb $14, %xmm1, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testl $16384, %eax ## imm = 0x4000
 ; AVX1OR2-NEXT:    je LBB12_30
-; AVX1OR2-NEXT:  ## %bb.29: ## %cond.store40
+; AVX1OR2-NEXT:  LBB12_29: ## %cond.store40
 ; AVX1OR2-NEXT:    vpextrb $14, %xmm0, (%rdi)
 ; AVX1OR2-NEXT:    incq %rdi
-; AVX1OR2-NEXT:  LBB12_30: ## %else41
-; AVX1OR2-NEXT:    vpextrb $15, %xmm1, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testl $32768, %eax ## imm = 0x8000
 ; AVX1OR2-NEXT:    je LBB12_32
-; AVX1OR2-NEXT:  ## %bb.31: ## %cond.store43
+; AVX1OR2-NEXT:  LBB12_31: ## %cond.store43
 ; AVX1OR2-NEXT:    vpextrb $15, %xmm0, (%rdi)
-; AVX1OR2-NEXT:  LBB12_32: ## %else44
 ; AVX1OR2-NEXT:    retq
 ;
 ; AVX512F-LABEL: compressstore_v16i8_v16i8:
 ; AVX512F:       ## %bb.0:
 ; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm1
+; AVX512F-NEXT:    vpmovmskb %xmm1, %eax
 ; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je LBB12_2
-; AVX512F-NEXT:  ## %bb.1: ## %cond.store
+; AVX512F-NEXT:    jne LBB12_1
+; AVX512F-NEXT:  ## %bb.2: ## %else
+; AVX512F-NEXT:    testb $2, %al
+; AVX512F-NEXT:    jne LBB12_3
+; AVX512F-NEXT:  LBB12_4: ## %else2
+; AVX512F-NEXT:    testb $4, %al
+; AVX512F-NEXT:    jne LBB12_5
+; AVX512F-NEXT:  LBB12_6: ## %else5
+; AVX512F-NEXT:    testb $8, %al
+; AVX512F-NEXT:    jne LBB12_7
+; AVX512F-NEXT:  LBB12_8: ## %else8
+; AVX512F-NEXT:    testb $16, %al
+; AVX512F-NEXT:    jne LBB12_9
+; AVX512F-NEXT:  LBB12_10: ## %else11
+; AVX512F-NEXT:    testb $32, %al
+; AVX512F-NEXT:    jne LBB12_11
+; AVX512F-NEXT:  LBB12_12: ## %else14
+; AVX512F-NEXT:    testb $64, %al
+; AVX512F-NEXT:    jne LBB12_13
+; AVX512F-NEXT:  LBB12_14: ## %else17
+; AVX512F-NEXT:    testb $-128, %al
+; AVX512F-NEXT:    jne LBB12_15
+; AVX512F-NEXT:  LBB12_16: ## %else20
+; AVX512F-NEXT:    testl $256, %eax ## imm = 0x100
+; AVX512F-NEXT:    jne LBB12_17
+; AVX512F-NEXT:  LBB12_18: ## %else23
+; AVX512F-NEXT:    testl $512, %eax ## imm = 0x200
+; AVX512F-NEXT:    jne LBB12_19
+; AVX512F-NEXT:  LBB12_20: ## %else26
+; AVX512F-NEXT:    testl $1024, %eax ## imm = 0x400
+; AVX512F-NEXT:    jne LBB12_21
+; AVX512F-NEXT:  LBB12_22: ## %else29
+; AVX512F-NEXT:    testl $2048, %eax ## imm = 0x800
+; AVX512F-NEXT:    jne LBB12_23
+; AVX512F-NEXT:  LBB12_24: ## %else32
+; AVX512F-NEXT:    testl $4096, %eax ## imm = 0x1000
+; AVX512F-NEXT:    jne LBB12_25
+; AVX512F-NEXT:  LBB12_26: ## %else35
+; AVX512F-NEXT:    testl $8192, %eax ## imm = 0x2000
+; AVX512F-NEXT:    jne LBB12_27
+; AVX512F-NEXT:  LBB12_28: ## %else38
+; AVX512F-NEXT:    testl $16384, %eax ## imm = 0x4000
+; AVX512F-NEXT:    jne LBB12_29
+; AVX512F-NEXT:  LBB12_30: ## %else41
+; AVX512F-NEXT:    testl $32768, %eax ## imm = 0x8000
+; AVX512F-NEXT:    jne LBB12_31
+; AVX512F-NEXT:  LBB12_32: ## %else44
+; AVX512F-NEXT:    retq
+; AVX512F-NEXT:  LBB12_1: ## %cond.store
 ; AVX512F-NEXT:    vpextrb $0, %xmm0, (%rdi)
 ; AVX512F-NEXT:    incq %rdi
-; AVX512F-NEXT:  LBB12_2: ## %else
-; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $2, %al
 ; AVX512F-NEXT:    je LBB12_4
-; AVX512F-NEXT:  ## %bb.3: ## %cond.store1
+; AVX512F-NEXT:  LBB12_3: ## %cond.store1
 ; AVX512F-NEXT:    vpextrb $1, %xmm0, (%rdi)
 ; AVX512F-NEXT:    incq %rdi
-; AVX512F-NEXT:  LBB12_4: ## %else2
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $2, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $4, %al
 ; AVX512F-NEXT:    je LBB12_6
-; AVX512F-NEXT:  ## %bb.5: ## %cond.store4
+; AVX512F-NEXT:  LBB12_5: ## %cond.store4
 ; AVX512F-NEXT:    vpextrb $2, %xmm0, (%rdi)
 ; AVX512F-NEXT:    incq %rdi
-; AVX512F-NEXT:  LBB12_6: ## %else5
-; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $8, %al
 ; AVX512F-NEXT:    je LBB12_8
-; AVX512F-NEXT:  ## %bb.7: ## %cond.store7
+; AVX512F-NEXT:  LBB12_7: ## %cond.store7
 ; AVX512F-NEXT:    vpextrb $3, %xmm0, (%rdi)
 ; AVX512F-NEXT:    incq %rdi
-; AVX512F-NEXT:  LBB12_8: ## %else8
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $4, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $16, %al
 ; AVX512F-NEXT:    je LBB12_10
-; AVX512F-NEXT:  ## %bb.9: ## %cond.store10
+; AVX512F-NEXT:  LBB12_9: ## %cond.store10
 ; AVX512F-NEXT:    vpextrb $4, %xmm0, (%rdi)
 ; AVX512F-NEXT:    incq %rdi
-; AVX512F-NEXT:  LBB12_10: ## %else11
-; AVX512F-NEXT:    kshiftrw $5, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $32, %al
 ; AVX512F-NEXT:    je LBB12_12
-; AVX512F-NEXT:  ## %bb.11: ## %cond.store13
+; AVX512F-NEXT:  LBB12_11: ## %cond.store13
 ; AVX512F-NEXT:    vpextrb $5, %xmm0, (%rdi)
 ; AVX512F-NEXT:    incq %rdi
-; AVX512F-NEXT:  LBB12_12: ## %else14
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $6, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $64, %al
 ; AVX512F-NEXT:    je LBB12_14
-; AVX512F-NEXT:  ## %bb.13: ## %cond.store16
+; AVX512F-NEXT:  LBB12_13: ## %cond.store16
 ; AVX512F-NEXT:    vpextrb $6, %xmm0, (%rdi)
 ; AVX512F-NEXT:    incq %rdi
-; AVX512F-NEXT:  LBB12_14: ## %else17
-; AVX512F-NEXT:    kshiftrw $7, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $-128, %al
 ; AVX512F-NEXT:    je LBB12_16
-; AVX512F-NEXT:  ## %bb.15: ## %cond.store19
+; AVX512F-NEXT:  LBB12_15: ## %cond.store19
 ; AVX512F-NEXT:    vpextrb $7, %xmm0, (%rdi)
 ; AVX512F-NEXT:    incq %rdi
-; AVX512F-NEXT:  LBB12_16: ## %else20
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $8, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $256, %eax ## imm = 0x100
 ; AVX512F-NEXT:    je LBB12_18
-; AVX512F-NEXT:  ## %bb.17: ## %cond.store22
+; AVX512F-NEXT:  LBB12_17: ## %cond.store22
 ; AVX512F-NEXT:    vpextrb $8, %xmm0, (%rdi)
 ; AVX512F-NEXT:    incq %rdi
-; AVX512F-NEXT:  LBB12_18: ## %else23
-; AVX512F-NEXT:    kshiftrw $9, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $512, %eax ## imm = 0x200
 ; AVX512F-NEXT:    je LBB12_20
-; AVX512F-NEXT:  ## %bb.19: ## %cond.store25
+; AVX512F-NEXT:  LBB12_19: ## %cond.store25
 ; AVX512F-NEXT:    vpextrb $9, %xmm0, (%rdi)
 ; AVX512F-NEXT:    incq %rdi
-; AVX512F-NEXT:  LBB12_20: ## %else26
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $10, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $1024, %eax ## imm = 0x400
 ; AVX512F-NEXT:    je LBB12_22
-; AVX512F-NEXT:  ## %bb.21: ## %cond.store28
+; AVX512F-NEXT:  LBB12_21: ## %cond.store28
 ; AVX512F-NEXT:    vpextrb $10, %xmm0, (%rdi)
 ; AVX512F-NEXT:    incq %rdi
-; AVX512F-NEXT:  LBB12_22: ## %else29
-; AVX512F-NEXT:    kshiftrw $11, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $2048, %eax ## imm = 0x800
 ; AVX512F-NEXT:    je LBB12_24
-; AVX512F-NEXT:  ## %bb.23: ## %cond.store31
+; AVX512F-NEXT:  LBB12_23: ## %cond.store31
 ; AVX512F-NEXT:    vpextrb $11, %xmm0, (%rdi)
 ; AVX512F-NEXT:    incq %rdi
-; AVX512F-NEXT:  LBB12_24: ## %else32
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $12, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $4096, %eax ## imm = 0x1000
 ; AVX512F-NEXT:    je LBB12_26
-; AVX512F-NEXT:  ## %bb.25: ## %cond.store34
+; AVX512F-NEXT:  LBB12_25: ## %cond.store34
 ; AVX512F-NEXT:    vpextrb $12, %xmm0, (%rdi)
 ; AVX512F-NEXT:    incq %rdi
-; AVX512F-NEXT:  LBB12_26: ## %else35
-; AVX512F-NEXT:    kshiftrw $13, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $8192, %eax ## imm = 0x2000
 ; AVX512F-NEXT:    je LBB12_28
-; AVX512F-NEXT:  ## %bb.27: ## %cond.store37
+; AVX512F-NEXT:  LBB12_27: ## %cond.store37
 ; AVX512F-NEXT:    vpextrb $13, %xmm0, (%rdi)
 ; AVX512F-NEXT:    incq %rdi
-; AVX512F-NEXT:  LBB12_28: ## %else38
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm1
-; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $14, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $16384, %eax ## imm = 0x4000
 ; AVX512F-NEXT:    je LBB12_30
-; AVX512F-NEXT:  ## %bb.29: ## %cond.store40
+; AVX512F-NEXT:  LBB12_29: ## %cond.store40
 ; AVX512F-NEXT:    vpextrb $14, %xmm0, (%rdi)
 ; AVX512F-NEXT:    incq %rdi
-; AVX512F-NEXT:  LBB12_30: ## %else41
-; AVX512F-NEXT:    kshiftrw $15, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $32768, %eax ## imm = 0x8000
 ; AVX512F-NEXT:    je LBB12_32
-; AVX512F-NEXT:  ## %bb.31: ## %cond.store43
+; AVX512F-NEXT:  LBB12_31: ## %cond.store43
 ; AVX512F-NEXT:    vpextrb $15, %xmm0, (%rdi)
-; AVX512F-NEXT:  LBB12_32: ## %else44
-; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VLDQ-LABEL: compressstore_v16i8_v16i8:
 ; AVX512VLDQ:       ## %bb.0:
 ; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm2, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
+; AVX512VLDQ-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm1
+; AVX512VLDQ-NEXT:    vpmovmskb %xmm1, %eax
 ; AVX512VLDQ-NEXT:    testb $1, %al
-; AVX512VLDQ-NEXT:    je LBB12_2
-; AVX512VLDQ-NEXT:  ## %bb.1: ## %cond.store
+; AVX512VLDQ-NEXT:    jne LBB12_1
+; AVX512VLDQ-NEXT:  ## %bb.2: ## %else
+; AVX512VLDQ-NEXT:    testb $2, %al
+; AVX512VLDQ-NEXT:    jne LBB12_3
+; AVX512VLDQ-NEXT:  LBB12_4: ## %else2
+; AVX512VLDQ-NEXT:    testb $4, %al
+; AVX512VLDQ-NEXT:    jne LBB12_5
+; AVX512VLDQ-NEXT:  LBB12_6: ## %else5
+; AVX512VLDQ-NEXT:    testb $8, %al
+; AVX512VLDQ-NEXT:    jne LBB12_7
+; AVX512VLDQ-NEXT:  LBB12_8: ## %else8
+; AVX512VLDQ-NEXT:    testb $16, %al
+; AVX512VLDQ-NEXT:    jne LBB12_9
+; AVX512VLDQ-NEXT:  LBB12_10: ## %else11
+; AVX512VLDQ-NEXT:    testb $32, %al
+; AVX512VLDQ-NEXT:    jne LBB12_11
+; AVX512VLDQ-NEXT:  LBB12_12: ## %else14
+; AVX512VLDQ-NEXT:    testb $64, %al
+; AVX512VLDQ-NEXT:    jne LBB12_13
+; AVX512VLDQ-NEXT:  LBB12_14: ## %else17
+; AVX512VLDQ-NEXT:    testb $-128, %al
+; AVX512VLDQ-NEXT:    jne LBB12_15
+; AVX512VLDQ-NEXT:  LBB12_16: ## %else20
+; AVX512VLDQ-NEXT:    testl $256, %eax ## imm = 0x100
+; AVX512VLDQ-NEXT:    jne LBB12_17
+; AVX512VLDQ-NEXT:  LBB12_18: ## %else23
+; AVX512VLDQ-NEXT:    testl $512, %eax ## imm = 0x200
+; AVX512VLDQ-NEXT:    jne LBB12_19
+; AVX512VLDQ-NEXT:  LBB12_20: ## %else26
+; AVX512VLDQ-NEXT:    testl $1024, %eax ## imm = 0x400
+; AVX512VLDQ-NEXT:    jne LBB12_21
+; AVX512VLDQ-NEXT:  LBB12_22: ## %else29
+; AVX512VLDQ-NEXT:    testl $2048, %eax ## imm = 0x800
+; AVX512VLDQ-NEXT:    jne LBB12_23
+; AVX512VLDQ-NEXT:  LBB12_24: ## %else32
+; AVX512VLDQ-NEXT:    testl $4096, %eax ## imm = 0x1000
+; AVX512VLDQ-NEXT:    jne LBB12_25
+; AVX512VLDQ-NEXT:  LBB12_26: ## %else35
+; AVX512VLDQ-NEXT:    testl $8192, %eax ## imm = 0x2000
+; AVX512VLDQ-NEXT:    jne LBB12_27
+; AVX512VLDQ-NEXT:  LBB12_28: ## %else38
+; AVX512VLDQ-NEXT:    testl $16384, %eax ## imm = 0x4000
+; AVX512VLDQ-NEXT:    jne LBB12_29
+; AVX512VLDQ-NEXT:  LBB12_30: ## %else41
+; AVX512VLDQ-NEXT:    testl $32768, %eax ## imm = 0x8000
+; AVX512VLDQ-NEXT:    jne LBB12_31
+; AVX512VLDQ-NEXT:  LBB12_32: ## %else44
+; AVX512VLDQ-NEXT:    retq
+; AVX512VLDQ-NEXT:  LBB12_1: ## %cond.store
 ; AVX512VLDQ-NEXT:    vpextrb $0, %xmm0, (%rdi)
 ; AVX512VLDQ-NEXT:    incq %rdi
-; AVX512VLDQ-NEXT:  LBB12_2: ## %else
-; AVX512VLDQ-NEXT:    kshiftrw $1, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $2, %al
 ; AVX512VLDQ-NEXT:    je LBB12_4
-; AVX512VLDQ-NEXT:  ## %bb.3: ## %cond.store1
+; AVX512VLDQ-NEXT:  LBB12_3: ## %cond.store1
 ; AVX512VLDQ-NEXT:    vpextrb $1, %xmm0, (%rdi)
 ; AVX512VLDQ-NEXT:    incq %rdi
-; AVX512VLDQ-NEXT:  LBB12_4: ## %else2
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm2, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $2, %k0, %k1
-; AVX512VLDQ-NEXT:    kmovw %k1, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $4, %al
 ; AVX512VLDQ-NEXT:    je LBB12_6
-; AVX512VLDQ-NEXT:  ## %bb.5: ## %cond.store4
+; AVX512VLDQ-NEXT:  LBB12_5: ## %cond.store4
 ; AVX512VLDQ-NEXT:    vpextrb $2, %xmm0, (%rdi)
 ; AVX512VLDQ-NEXT:    incq %rdi
-; AVX512VLDQ-NEXT:  LBB12_6: ## %else5
-; AVX512VLDQ-NEXT:    kshiftrw $3, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $8, %al
 ; AVX512VLDQ-NEXT:    je LBB12_8
-; AVX512VLDQ-NEXT:  ## %bb.7: ## %cond.store7
+; AVX512VLDQ-NEXT:  LBB12_7: ## %cond.store7
 ; AVX512VLDQ-NEXT:    vpextrb $3, %xmm0, (%rdi)
 ; AVX512VLDQ-NEXT:    incq %rdi
-; AVX512VLDQ-NEXT:  LBB12_8: ## %else8
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm2, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $4, %k0, %k1
-; AVX512VLDQ-NEXT:    kmovw %k1, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $16, %al
 ; AVX512VLDQ-NEXT:    je LBB12_10
-; AVX512VLDQ-NEXT:  ## %bb.9: ## %cond.store10
+; AVX512VLDQ-NEXT:  LBB12_9: ## %cond.store10
 ; AVX512VLDQ-NEXT:    vpextrb $4, %xmm0, (%rdi)
 ; AVX512VLDQ-NEXT:    incq %rdi
-; AVX512VLDQ-NEXT:  LBB12_10: ## %else11
-; AVX512VLDQ-NEXT:    kshiftrw $5, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $32, %al
 ; AVX512VLDQ-NEXT:    je LBB12_12
-; AVX512VLDQ-NEXT:  ## %bb.11: ## %cond.store13
+; AVX512VLDQ-NEXT:  LBB12_11: ## %cond.store13
 ; AVX512VLDQ-NEXT:    vpextrb $5, %xmm0, (%rdi)
 ; AVX512VLDQ-NEXT:    incq %rdi
-; AVX512VLDQ-NEXT:  LBB12_12: ## %else14
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm2, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $6, %k0, %k1
-; AVX512VLDQ-NEXT:    kmovw %k1, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $64, %al
 ; AVX512VLDQ-NEXT:    je LBB12_14
-; AVX512VLDQ-NEXT:  ## %bb.13: ## %cond.store16
+; AVX512VLDQ-NEXT:  LBB12_13: ## %cond.store16
 ; AVX512VLDQ-NEXT:    vpextrb $6, %xmm0, (%rdi)
 ; AVX512VLDQ-NEXT:    incq %rdi
-; AVX512VLDQ-NEXT:  LBB12_14: ## %else17
-; AVX512VLDQ-NEXT:    kshiftrw $7, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $-128, %al
 ; AVX512VLDQ-NEXT:    je LBB12_16
-; AVX512VLDQ-NEXT:  ## %bb.15: ## %cond.store19
+; AVX512VLDQ-NEXT:  LBB12_15: ## %cond.store19
 ; AVX512VLDQ-NEXT:    vpextrb $7, %xmm0, (%rdi)
 ; AVX512VLDQ-NEXT:    incq %rdi
-; AVX512VLDQ-NEXT:  LBB12_16: ## %else20
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm2, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $8, %k0, %k1
-; AVX512VLDQ-NEXT:    kmovw %k1, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testl $256, %eax ## imm = 0x100
 ; AVX512VLDQ-NEXT:    je LBB12_18
-; AVX512VLDQ-NEXT:  ## %bb.17: ## %cond.store22
+; AVX512VLDQ-NEXT:  LBB12_17: ## %cond.store22
 ; AVX512VLDQ-NEXT:    vpextrb $8, %xmm0, (%rdi)
 ; AVX512VLDQ-NEXT:    incq %rdi
-; AVX512VLDQ-NEXT:  LBB12_18: ## %else23
-; AVX512VLDQ-NEXT:    kshiftrw $9, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testl $512, %eax ## imm = 0x200
 ; AVX512VLDQ-NEXT:    je LBB12_20
-; AVX512VLDQ-NEXT:  ## %bb.19: ## %cond.store25
+; AVX512VLDQ-NEXT:  LBB12_19: ## %cond.store25
 ; AVX512VLDQ-NEXT:    vpextrb $9, %xmm0, (%rdi)
 ; AVX512VLDQ-NEXT:    incq %rdi
-; AVX512VLDQ-NEXT:  LBB12_20: ## %else26
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm2, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $10, %k0, %k1
-; AVX512VLDQ-NEXT:    kmovw %k1, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testl $1024, %eax ## imm = 0x400
 ; AVX512VLDQ-NEXT:    je LBB12_22
-; AVX512VLDQ-NEXT:  ## %bb.21: ## %cond.store28
+; AVX512VLDQ-NEXT:  LBB12_21: ## %cond.store28
 ; AVX512VLDQ-NEXT:    vpextrb $10, %xmm0, (%rdi)
 ; AVX512VLDQ-NEXT:    incq %rdi
-; AVX512VLDQ-NEXT:  LBB12_22: ## %else29
-; AVX512VLDQ-NEXT:    kshiftrw $11, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testl $2048, %eax ## imm = 0x800
 ; AVX512VLDQ-NEXT:    je LBB12_24
-; AVX512VLDQ-NEXT:  ## %bb.23: ## %cond.store31
+; AVX512VLDQ-NEXT:  LBB12_23: ## %cond.store31
 ; AVX512VLDQ-NEXT:    vpextrb $11, %xmm0, (%rdi)
 ; AVX512VLDQ-NEXT:    incq %rdi
-; AVX512VLDQ-NEXT:  LBB12_24: ## %else32
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm2, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $12, %k0, %k1
-; AVX512VLDQ-NEXT:    kmovw %k1, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testl $4096, %eax ## imm = 0x1000
 ; AVX512VLDQ-NEXT:    je LBB12_26
-; AVX512VLDQ-NEXT:  ## %bb.25: ## %cond.store34
+; AVX512VLDQ-NEXT:  LBB12_25: ## %cond.store34
 ; AVX512VLDQ-NEXT:    vpextrb $12, %xmm0, (%rdi)
 ; AVX512VLDQ-NEXT:    incq %rdi
-; AVX512VLDQ-NEXT:  LBB12_26: ## %else35
-; AVX512VLDQ-NEXT:    kshiftrw $13, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testl $8192, %eax ## imm = 0x2000
 ; AVX512VLDQ-NEXT:    je LBB12_28
-; AVX512VLDQ-NEXT:  ## %bb.27: ## %cond.store37
+; AVX512VLDQ-NEXT:  LBB12_27: ## %cond.store37
 ; AVX512VLDQ-NEXT:    vpextrb $13, %xmm0, (%rdi)
 ; AVX512VLDQ-NEXT:    incq %rdi
-; AVX512VLDQ-NEXT:  LBB12_28: ## %else38
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm1
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm1, %zmm1
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm1, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $14, %k0, %k1
-; AVX512VLDQ-NEXT:    kmovw %k1, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testl $16384, %eax ## imm = 0x4000
 ; AVX512VLDQ-NEXT:    je LBB12_30
-; AVX512VLDQ-NEXT:  ## %bb.29: ## %cond.store40
+; AVX512VLDQ-NEXT:  LBB12_29: ## %cond.store40
 ; AVX512VLDQ-NEXT:    vpextrb $14, %xmm0, (%rdi)
 ; AVX512VLDQ-NEXT:    incq %rdi
-; AVX512VLDQ-NEXT:  LBB12_30: ## %else41
-; AVX512VLDQ-NEXT:    kshiftrw $15, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testl $32768, %eax ## imm = 0x8000
 ; AVX512VLDQ-NEXT:    je LBB12_32
-; AVX512VLDQ-NEXT:  ## %bb.31: ## %cond.store43
+; AVX512VLDQ-NEXT:  LBB12_31: ## %cond.store43
 ; AVX512VLDQ-NEXT:    vpextrb $15, %xmm0, (%rdi)
-; AVX512VLDQ-NEXT:  LBB12_32: ## %else44
-; AVX512VLDQ-NEXT:    vzeroupper
 ; AVX512VLDQ-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: compressstore_v16i8_v16i8:
@@ -4354,130 +4278,131 @@ define void @compressstore_v16i8_v16i8(i8* %base, <16 x i8> %V, <16 x i8> %trigg
 ; AVX512VLBW-NEXT:    vptestnmb %xmm1, %xmm1, %k0
 ; AVX512VLBW-NEXT:    kmovd %k0, %eax
 ; AVX512VLBW-NEXT:    testb $1, %al
-; AVX512VLBW-NEXT:    je LBB12_2
-; AVX512VLBW-NEXT:  ## %bb.1: ## %cond.store
+; AVX512VLBW-NEXT:    jne LBB12_1
+; AVX512VLBW-NEXT:  ## %bb.2: ## %else
+; AVX512VLBW-NEXT:    testb $2, %al
+; AVX512VLBW-NEXT:    jne LBB12_3
+; AVX512VLBW-NEXT:  LBB12_4: ## %else2
+; AVX512VLBW-NEXT:    testb $4, %al
+; AVX512VLBW-NEXT:    jne LBB12_5
+; AVX512VLBW-NEXT:  LBB12_6: ## %else5
+; AVX512VLBW-NEXT:    testb $8, %al
+; AVX512VLBW-NEXT:    jne LBB12_7
+; AVX512VLBW-NEXT:  LBB12_8: ## %else8
+; AVX512VLBW-NEXT:    testb $16, %al
+; AVX512VLBW-NEXT:    jne LBB12_9
+; AVX512VLBW-NEXT:  LBB12_10: ## %else11
+; AVX512VLBW-NEXT:    testb $32, %al
+; AVX512VLBW-NEXT:    jne LBB12_11
+; AVX512VLBW-NEXT:  LBB12_12: ## %else14
+; AVX512VLBW-NEXT:    testb $64, %al
+; AVX512VLBW-NEXT:    jne LBB12_13
+; AVX512VLBW-NEXT:  LBB12_14: ## %else17
+; AVX512VLBW-NEXT:    testb $-128, %al
+; AVX512VLBW-NEXT:    jne LBB12_15
+; AVX512VLBW-NEXT:  LBB12_16: ## %else20
+; AVX512VLBW-NEXT:    testl $256, %eax ## imm = 0x100
+; AVX512VLBW-NEXT:    jne LBB12_17
+; AVX512VLBW-NEXT:  LBB12_18: ## %else23
+; AVX512VLBW-NEXT:    testl $512, %eax ## imm = 0x200
+; AVX512VLBW-NEXT:    jne LBB12_19
+; AVX512VLBW-NEXT:  LBB12_20: ## %else26
+; AVX512VLBW-NEXT:    testl $1024, %eax ## imm = 0x400
+; AVX512VLBW-NEXT:    jne LBB12_21
+; AVX512VLBW-NEXT:  LBB12_22: ## %else29
+; AVX512VLBW-NEXT:    testl $2048, %eax ## imm = 0x800
+; AVX512VLBW-NEXT:    jne LBB12_23
+; AVX512VLBW-NEXT:  LBB12_24: ## %else32
+; AVX512VLBW-NEXT:    testl $4096, %eax ## imm = 0x1000
+; AVX512VLBW-NEXT:    jne LBB12_25
+; AVX512VLBW-NEXT:  LBB12_26: ## %else35
+; AVX512VLBW-NEXT:    testl $8192, %eax ## imm = 0x2000
+; AVX512VLBW-NEXT:    jne LBB12_27
+; AVX512VLBW-NEXT:  LBB12_28: ## %else38
+; AVX512VLBW-NEXT:    testl $16384, %eax ## imm = 0x4000
+; AVX512VLBW-NEXT:    jne LBB12_29
+; AVX512VLBW-NEXT:  LBB12_30: ## %else41
+; AVX512VLBW-NEXT:    testl $32768, %eax ## imm = 0x8000
+; AVX512VLBW-NEXT:    jne LBB12_31
+; AVX512VLBW-NEXT:  LBB12_32: ## %else44
+; AVX512VLBW-NEXT:    retq
+; AVX512VLBW-NEXT:  LBB12_1: ## %cond.store
 ; AVX512VLBW-NEXT:    vpextrb $0, %xmm0, (%rdi)
 ; AVX512VLBW-NEXT:    incq %rdi
-; AVX512VLBW-NEXT:  LBB12_2: ## %else
-; AVX512VLBW-NEXT:    kshiftrw $1, %k0, %k1
-; AVX512VLBW-NEXT:    kmovd %k1, %eax
-; AVX512VLBW-NEXT:    testb $1, %al
+; AVX512VLBW-NEXT:    testb $2, %al
 ; AVX512VLBW-NEXT:    je LBB12_4
-; AVX512VLBW-NEXT:  ## %bb.3: ## %cond.store1
+; AVX512VLBW-NEXT:  LBB12_3: ## %cond.store1
 ; AVX512VLBW-NEXT:    vpextrb $1, %xmm0, (%rdi)
 ; AVX512VLBW-NEXT:    incq %rdi
-; AVX512VLBW-NEXT:  LBB12_4: ## %else2
-; AVX512VLBW-NEXT:    kshiftrw $2, %k0, %k1
-; AVX512VLBW-NEXT:    kmovd %k1, %eax
-; AVX512VLBW-NEXT:    testb $1, %al
+; AVX512VLBW-NEXT:    testb $4, %al
 ; AVX512VLBW-NEXT:    je LBB12_6
-; AVX512VLBW-NEXT:  ## %bb.5: ## %cond.store4
+; AVX512VLBW-NEXT:  LBB12_5: ## %cond.store4
 ; AVX512VLBW-NEXT:    vpextrb $2, %xmm0, (%rdi)
 ; AVX512VLBW-NEXT:    incq %rdi
-; AVX512VLBW-NEXT:  LBB12_6: ## %else5
-; AVX512VLBW-NEXT:    kshiftrw $3, %k0, %k1
-; AVX512VLBW-NEXT:    kmovd %k1, %eax
-; AVX512VLBW-NEXT:    testb $1, %al
+; AVX512VLBW-NEXT:    testb $8, %al
 ; AVX512VLBW-NEXT:    je LBB12_8
-; AVX512VLBW-NEXT:  ## %bb.7: ## %cond.store7
+; AVX512VLBW-NEXT:  LBB12_7: ## %cond.store7
 ; AVX512VLBW-NEXT:    vpextrb $3, %xmm0, (%rdi)
 ; AVX512VLBW-NEXT:    incq %rdi
-; AVX512VLBW-NEXT:  LBB12_8: ## %else8
-; AVX512VLBW-NEXT:    kshiftrw $4, %k0, %k1
-; AVX512VLBW-NEXT:    kmovd %k1, %eax
-; AVX512VLBW-NEXT:    testb $1, %al
+; AVX512VLBW-NEXT:    testb $16, %al
 ; AVX512VLBW-NEXT:    je LBB12_10
-; AVX512VLBW-NEXT:  ## %bb.9: ## %cond.store10
+; AVX512VLBW-NEXT:  LBB12_9: ## %cond.store10
 ; AVX512VLBW-NEXT:    vpextrb $4, %xmm0, (%rdi)
 ; AVX512VLBW-NEXT:    incq %rdi
-; AVX512VLBW-NEXT:  LBB12_10: ## %else11
-; AVX512VLBW-NEXT:    kshiftrw $5, %k0, %k1
-; AVX512VLBW-NEXT:    kmovd %k1, %eax
-; AVX512VLBW-NEXT:    testb $1, %al
+; AVX512VLBW-NEXT:    testb $32, %al
 ; AVX512VLBW-NEXT:    je LBB12_12
-; AVX512VLBW-NEXT:  ## %bb.11: ## %cond.store13
+; AVX512VLBW-NEXT:  LBB12_11: ## %cond.store13
 ; AVX512VLBW-NEXT:    vpextrb $5, %xmm0, (%rdi)
 ; AVX512VLBW-NEXT:    incq %rdi
-; AVX512VLBW-NEXT:  LBB12_12: ## %else14
-; AVX512VLBW-NEXT:    kshiftrw $6, %k0, %k1
-; AVX512VLBW-NEXT:    kmovd %k1, %eax
-; AVX512VLBW-NEXT:    testb $1, %al
+; AVX512VLBW-NEXT:    testb $64, %al
 ; AVX512VLBW-NEXT:    je LBB12_14
-; AVX512VLBW-NEXT:  ## %bb.13: ## %cond.store16
+; AVX512VLBW-NEXT:  LBB12_13: ## %cond.store16
 ; AVX512VLBW-NEXT:    vpextrb $6, %xmm0, (%rdi)
 ; AVX512VLBW-NEXT:    incq %rdi
-; AVX512VLBW-NEXT:  LBB12_14: ## %else17
-; AVX512VLBW-NEXT:    kshiftrw $7, %k0, %k1
-; AVX512VLBW-NEXT:    kmovd %k1, %eax
-; AVX512VLBW-NEXT:    testb $1, %al
+; AVX512VLBW-NEXT:    testb $-128, %al
 ; AVX512VLBW-NEXT:    je LBB12_16
-; AVX512VLBW-NEXT:  ## %bb.15: ## %cond.store19
+; AVX512VLBW-NEXT:  LBB12_15: ## %cond.store19
 ; AVX512VLBW-NEXT:    vpextrb $7, %xmm0, (%rdi)
 ; AVX512VLBW-NEXT:    incq %rdi
-; AVX512VLBW-NEXT:  LBB12_16: ## %else20
-; AVX512VLBW-NEXT:    kshiftrw $8, %k0, %k1
-; AVX512VLBW-NEXT:    kmovd %k1, %eax
-; AVX512VLBW-NEXT:    testb $1, %al
+; AVX512VLBW-NEXT:    testl $256, %eax ## imm = 0x100
 ; AVX512VLBW-NEXT:    je LBB12_18
-; AVX512VLBW-NEXT:  ## %bb.17: ## %cond.store22
+; AVX512VLBW-NEXT:  LBB12_17: ## %cond.store22
 ; AVX512VLBW-NEXT:    vpextrb $8, %xmm0, (%rdi)
 ; AVX512VLBW-NEXT:    incq %rdi
-; AVX512VLBW-NEXT:  LBB12_18: ## %else23
-; AVX512VLBW-NEXT:    kshiftrw $9, %k0, %k1
-; AVX512VLBW-NEXT:    kmovd %k1, %eax
-; AVX512VLBW-NEXT:    testb $1, %al
+; AVX512VLBW-NEXT:    testl $512, %eax ## imm = 0x200
 ; AVX512VLBW-NEXT:    je LBB12_20
-; AVX512VLBW-NEXT:  ## %bb.19: ## %cond.store25
+; AVX512VLBW-NEXT:  LBB12_19: ## %cond.store25
 ; AVX512VLBW-NEXT:    vpextrb $9, %xmm0, (%rdi)
 ; AVX512VLBW-NEXT:    incq %rdi
-; AVX512VLBW-NEXT:  LBB12_20: ## %else26
-; AVX512VLBW-NEXT:    kshiftrw $10, %k0, %k1
-; AVX512VLBW-NEXT:    kmovd %k1, %eax
-; AVX512VLBW-NEXT:    testb $1, %al
+; AVX512VLBW-NEXT:    testl $1024, %eax ## imm = 0x400
 ; AVX512VLBW-NEXT:    je LBB12_22
-; AVX512VLBW-NEXT:  ## %bb.21: ## %cond.store28
+; AVX512VLBW-NEXT:  LBB12_21: ## %cond.store28
 ; AVX512VLBW-NEXT:    vpextrb $10, %xmm0, (%rdi)
 ; AVX512VLBW-NEXT:    incq %rdi
-; AVX512VLBW-NEXT:  LBB12_22: ## %else29
-; AVX512VLBW-NEXT:    kshiftrw $11, %k0, %k1
-; AVX512VLBW-NEXT:    kmovd %k1, %eax
-; AVX512VLBW-NEXT:    testb $1, %al
+; AVX512VLBW-NEXT:    testl $2048, %eax ## imm = 0x800
 ; AVX512VLBW-NEXT:    je LBB12_24
-; AVX512VLBW-NEXT:  ## %bb.23: ## %cond.store31
+; AVX512VLBW-NEXT:  LBB12_23: ## %cond.store31
 ; AVX512VLBW-NEXT:    vpextrb $11, %xmm0, (%rdi)
 ; AVX512VLBW-NEXT:    incq %rdi
-; AVX512VLBW-NEXT:  LBB12_24: ## %else32
-; AVX512VLBW-NEXT:    kshiftrw $12, %k0, %k1
-; AVX512VLBW-NEXT:    kmovd %k1, %eax
-; AVX512VLBW-NEXT:    testb $1, %al
+; AVX512VLBW-NEXT:    testl $4096, %eax ## imm = 0x1000
 ; AVX512VLBW-NEXT:    je LBB12_26
-; AVX512VLBW-NEXT:  ## %bb.25: ## %cond.store34
+; AVX512VLBW-NEXT:  LBB12_25: ## %cond.store34
 ; AVX512VLBW-NEXT:    vpextrb $12, %xmm0, (%rdi)
 ; AVX512VLBW-NEXT:    incq %rdi
-; AVX512VLBW-NEXT:  LBB12_26: ## %else35
-; AVX512VLBW-NEXT:    kshiftrw $13, %k0, %k1
-; AVX512VLBW-NEXT:    kmovd %k1, %eax
-; AVX512VLBW-NEXT:    testb $1, %al
+; AVX512VLBW-NEXT:    testl $8192, %eax ## imm = 0x2000
 ; AVX512VLBW-NEXT:    je LBB12_28
-; AVX512VLBW-NEXT:  ## %bb.27: ## %cond.store37
+; AVX512VLBW-NEXT:  LBB12_27: ## %cond.store37
 ; AVX512VLBW-NEXT:    vpextrb $13, %xmm0, (%rdi)
 ; AVX512VLBW-NEXT:    incq %rdi
-; AVX512VLBW-NEXT:  LBB12_28: ## %else38
-; AVX512VLBW-NEXT:    kshiftrw $14, %k0, %k1
-; AVX512VLBW-NEXT:    kmovd %k1, %eax
-; AVX512VLBW-NEXT:    testb $1, %al
+; AVX512VLBW-NEXT:    testl $16384, %eax ## imm = 0x4000
 ; AVX512VLBW-NEXT:    je LBB12_30
-; AVX512VLBW-NEXT:  ## %bb.29: ## %cond.store40
+; AVX512VLBW-NEXT:  LBB12_29: ## %cond.store40
 ; AVX512VLBW-NEXT:    vpextrb $14, %xmm0, (%rdi)
 ; AVX512VLBW-NEXT:    incq %rdi
-; AVX512VLBW-NEXT:  LBB12_30: ## %else41
-; AVX512VLBW-NEXT:    kshiftrw $15, %k0, %k0
-; AVX512VLBW-NEXT:    kmovd %k0, %eax
-; AVX512VLBW-NEXT:    testb $1, %al
+; AVX512VLBW-NEXT:    testl $32768, %eax ## imm = 0x8000
 ; AVX512VLBW-NEXT:    je LBB12_32
-; AVX512VLBW-NEXT:  ## %bb.31: ## %cond.store43
+; AVX512VLBW-NEXT:  LBB12_31: ## %cond.store43
 ; AVX512VLBW-NEXT:    vpextrb $15, %xmm0, (%rdi)
-; AVX512VLBW-NEXT:  LBB12_32: ## %else44
 ; AVX512VLBW-NEXT:    retq
   %mask = icmp eq <16 x i8> %trigger, zeroinitializer
   call void @llvm.masked.compressstore.v16i8(<16 x i8> %V, i8* %base, <16 x i1> %mask)
diff --git a/llvm/test/CodeGen/X86/masked_expandload.ll b/llvm/test/CodeGen/X86/masked_expandload.ll
index 3d996b03b..12cfb932a 100644
--- a/llvm/test/CodeGen/X86/masked_expandload.ll
+++ b/llvm/test/CodeGen/X86/masked_expandload.ll
@@ -18,57 +18,63 @@ define <2 x double> @expandload_v2f64_v2i64(double* %base, <2 x double> %src0, <
 ; SSE2-NEXT:    pcmpeqd %xmm1, %xmm2
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,0,3,2]
 ; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    movd %xmm1, %eax
+; SSE2-NEXT:    movmskpd %xmm1, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB0_2
-; SSE2-NEXT:  ## %bb.1: ## %cond.load
+; SSE2-NEXT:    jne LBB0_1
+; SSE2-NEXT:  ## %bb.2: ## %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne LBB0_3
+; SSE2-NEXT:  LBB0_4: ## %else2
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  LBB0_1: ## %cond.load
 ; SSE2-NEXT:    movlps {{.*#+}} xmm0 = mem[0,1],xmm0[2,3]
 ; SSE2-NEXT:    addq $8, %rdi
-; SSE2-NEXT:  LBB0_2: ## %else
-; SSE2-NEXT:    pextrw $4, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $2, %al
 ; SSE2-NEXT:    je LBB0_4
-; SSE2-NEXT:  ## %bb.3: ## %cond.load1
+; SSE2-NEXT:  LBB0_3: ## %cond.load1
 ; SSE2-NEXT:    movhps {{.*#+}} xmm0 = xmm0[0,1],mem[0,1]
-; SSE2-NEXT:  LBB0_4: ## %else2
 ; SSE2-NEXT:    retq
 ;
 ; SSE42-LABEL: expandload_v2f64_v2i64:
 ; SSE42:       ## %bb.0:
 ; SSE42-NEXT:    pxor %xmm2, %xmm2
 ; SSE42-NEXT:    pcmpeqq %xmm1, %xmm2
-; SSE42-NEXT:    pextrb $0, %xmm2, %eax
+; SSE42-NEXT:    movmskpd %xmm2, %eax
 ; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB0_2
-; SSE42-NEXT:  ## %bb.1: ## %cond.load
+; SSE42-NEXT:    jne LBB0_1
+; SSE42-NEXT:  ## %bb.2: ## %else
+; SSE42-NEXT:    testb $2, %al
+; SSE42-NEXT:    jne LBB0_3
+; SSE42-NEXT:  LBB0_4: ## %else2
+; SSE42-NEXT:    retq
+; SSE42-NEXT:  LBB0_1: ## %cond.load
 ; SSE42-NEXT:    movlps {{.*#+}} xmm0 = mem[0,1],xmm0[2,3]
 ; SSE42-NEXT:    addq $8, %rdi
-; SSE42-NEXT:  LBB0_2: ## %else
-; SSE42-NEXT:    pextrb $8, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $2, %al
 ; SSE42-NEXT:    je LBB0_4
-; SSE42-NEXT:  ## %bb.3: ## %cond.load1
+; SSE42-NEXT:  LBB0_3: ## %cond.load1
 ; SSE42-NEXT:    movhps {{.*#+}} xmm0 = xmm0[0,1],mem[0,1]
-; SSE42-NEXT:  LBB0_4: ## %else2
 ; SSE42-NEXT:    retq
 ;
 ; AVX1OR2-LABEL: expandload_v2f64_v2i64:
 ; AVX1OR2:       ## %bb.0:
 ; AVX1OR2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; AVX1OR2-NEXT:    vpcmpeqq %xmm2, %xmm1, %xmm1
-; AVX1OR2-NEXT:    vpextrb $0, %xmm1, %eax
+; AVX1OR2-NEXT:    vmovmskpd %xmm1, %eax
 ; AVX1OR2-NEXT:    testb $1, %al
-; AVX1OR2-NEXT:    je LBB0_2
-; AVX1OR2-NEXT:  ## %bb.1: ## %cond.load
+; AVX1OR2-NEXT:    jne LBB0_1
+; AVX1OR2-NEXT:  ## %bb.2: ## %else
+; AVX1OR2-NEXT:    testb $2, %al
+; AVX1OR2-NEXT:    jne LBB0_3
+; AVX1OR2-NEXT:  LBB0_4: ## %else2
+; AVX1OR2-NEXT:    retq
+; AVX1OR2-NEXT:  LBB0_1: ## %cond.load
 ; AVX1OR2-NEXT:    vmovlps {{.*#+}} xmm0 = mem[0,1],xmm0[2,3]
 ; AVX1OR2-NEXT:    addq $8, %rdi
-; AVX1OR2-NEXT:  LBB0_2: ## %else
-; AVX1OR2-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $2, %al
 ; AVX1OR2-NEXT:    je LBB0_4
-; AVX1OR2-NEXT:  ## %bb.3: ## %cond.load1
+; AVX1OR2-NEXT:  LBB0_3: ## %cond.load1
 ; AVX1OR2-NEXT:    vmovhps {{.*#+}} xmm0 = xmm0[0,1],mem[0,1]
-; AVX1OR2-NEXT:  LBB0_4: ## %else2
 ; AVX1OR2-NEXT:    retq
 ;
 ; AVX512F-LABEL: expandload_v2f64_v2i64:
@@ -97,161 +103,172 @@ define <4 x double> @expandload_v4f64_v4i64(double* %base, <4 x double> %src0, <
 ; SSE2-LABEL: expandload_v4f64_v4i64:
 ; SSE2:       ## %bb.0:
 ; SSE2-NEXT:    pxor %xmm4, %xmm4
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,0,3,2]
-; SSE2-NEXT:    pand %xmm4, %xmm2
-; SSE2-NEXT:    movd %xmm2, %eax
+; SSE2-NEXT:    pcmpeqd %xmm4, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[1,0,3,2]
+; SSE2-NEXT:    pand %xmm3, %xmm5
+; SSE2-NEXT:    pcmpeqd %xmm4, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,0,3,2]
+; SSE2-NEXT:    pand %xmm2, %xmm3
+; SSE2-NEXT:    packssdw %xmm5, %xmm3
+; SSE2-NEXT:    movmskps %xmm3, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB1_2
-; SSE2-NEXT:  ## %bb.1: ## %cond.load
+; SSE2-NEXT:    jne LBB1_1
+; SSE2-NEXT:  ## %bb.2: ## %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne LBB1_3
+; SSE2-NEXT:  LBB1_4: ## %else2
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne LBB1_5
+; SSE2-NEXT:  LBB1_6: ## %else6
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne LBB1_7
+; SSE2-NEXT:  LBB1_8: ## %else10
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  LBB1_1: ## %cond.load
 ; SSE2-NEXT:    movlps {{.*#+}} xmm0 = mem[0,1],xmm0[2,3]
 ; SSE2-NEXT:    addq $8, %rdi
-; SSE2-NEXT:  LBB1_2: ## %else
-; SSE2-NEXT:    pextrw $4, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $2, %al
 ; SSE2-NEXT:    je LBB1_4
-; SSE2-NEXT:  ## %bb.3: ## %cond.load1
+; SSE2-NEXT:  LBB1_3: ## %cond.load1
 ; SSE2-NEXT:    movhps {{.*#+}} xmm0 = xmm0[0,1],mem[0,1]
 ; SSE2-NEXT:    addq $8, %rdi
-; SSE2-NEXT:  LBB1_4: ## %else2
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,0,3,2]
-; SSE2-NEXT:    pand %xmm3, %xmm2
-; SSE2-NEXT:    pextrw $0, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $4, %al
 ; SSE2-NEXT:    je LBB1_6
-; SSE2-NEXT:  ## %bb.5: ## %cond.load5
+; SSE2-NEXT:  LBB1_5: ## %cond.load5
 ; SSE2-NEXT:    movlps {{.*#+}} xmm1 = mem[0,1],xmm1[2,3]
 ; SSE2-NEXT:    addq $8, %rdi
-; SSE2-NEXT:  LBB1_6: ## %else6
-; SSE2-NEXT:    pextrw $4, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je LBB1_8
-; SSE2-NEXT:  ## %bb.7: ## %cond.load9
+; SSE2-NEXT:  LBB1_7: ## %cond.load9
 ; SSE2-NEXT:    movhps {{.*#+}} xmm1 = xmm1[0,1],mem[0,1]
-; SSE2-NEXT:  LBB1_8: ## %else10
 ; SSE2-NEXT:    retq
 ;
 ; SSE42-LABEL: expandload_v4f64_v4i64:
 ; SSE42:       ## %bb.0:
 ; SSE42-NEXT:    pxor %xmm4, %xmm4
-; SSE42-NEXT:    pcmpeqq %xmm2, %xmm4
-; SSE42-NEXT:    pextrb $0, %xmm4, %eax
+; SSE42-NEXT:    pcmpeqq %xmm4, %xmm3
+; SSE42-NEXT:    pcmpeqq %xmm4, %xmm2
+; SSE42-NEXT:    packssdw %xmm3, %xmm2
+; SSE42-NEXT:    movmskps %xmm2, %eax
 ; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB1_2
-; SSE42-NEXT:  ## %bb.1: ## %cond.load
+; SSE42-NEXT:    jne LBB1_1
+; SSE42-NEXT:  ## %bb.2: ## %else
+; SSE42-NEXT:    testb $2, %al
+; SSE42-NEXT:    jne LBB1_3
+; SSE42-NEXT:  LBB1_4: ## %else2
+; SSE42-NEXT:    testb $4, %al
+; SSE42-NEXT:    jne LBB1_5
+; SSE42-NEXT:  LBB1_6: ## %else6
+; SSE42-NEXT:    testb $8, %al
+; SSE42-NEXT:    jne LBB1_7
+; SSE42-NEXT:  LBB1_8: ## %else10
+; SSE42-NEXT:    retq
+; SSE42-NEXT:  LBB1_1: ## %cond.load
 ; SSE42-NEXT:    movlps {{.*#+}} xmm0 = mem[0,1],xmm0[2,3]
 ; SSE42-NEXT:    addq $8, %rdi
-; SSE42-NEXT:  LBB1_2: ## %else
-; SSE42-NEXT:    pextrb $8, %xmm4, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $2, %al
 ; SSE42-NEXT:    je LBB1_4
-; SSE42-NEXT:  ## %bb.3: ## %cond.load1
+; SSE42-NEXT:  LBB1_3: ## %cond.load1
 ; SSE42-NEXT:    movhps {{.*#+}} xmm0 = xmm0[0,1],mem[0,1]
 ; SSE42-NEXT:    addq $8, %rdi
-; SSE42-NEXT:  LBB1_4: ## %else2
-; SSE42-NEXT:    pxor %xmm2, %xmm2
-; SSE42-NEXT:    pcmpeqq %xmm2, %xmm3
-; SSE42-NEXT:    pextrb $0, %xmm3, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $4, %al
 ; SSE42-NEXT:    je LBB1_6
-; SSE42-NEXT:  ## %bb.5: ## %cond.load5
+; SSE42-NEXT:  LBB1_5: ## %cond.load5
 ; SSE42-NEXT:    movlps {{.*#+}} xmm1 = mem[0,1],xmm1[2,3]
 ; SSE42-NEXT:    addq $8, %rdi
-; SSE42-NEXT:  LBB1_6: ## %else6
-; SSE42-NEXT:    pextrb $8, %xmm3, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $8, %al
 ; SSE42-NEXT:    je LBB1_8
-; SSE42-NEXT:  ## %bb.7: ## %cond.load9
+; SSE42-NEXT:  LBB1_7: ## %cond.load9
 ; SSE42-NEXT:    movhps {{.*#+}} xmm1 = xmm1[0,1],mem[0,1]
-; SSE42-NEXT:  LBB1_8: ## %else10
 ; SSE42-NEXT:    retq
 ;
 ; AVX1-LABEL: expandload_v4f64_v4i64:
 ; AVX1:       ## %bb.0:
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqq %xmm2, %xmm1, %xmm3
-; AVX1-NEXT:    vpextrb $0, %xmm3, %eax
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpeqq %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpcmpeqq %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; AVX1-NEXT:    vmovmskpd %ymm1, %eax
 ; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je LBB1_2
-; AVX1-NEXT:  ## %bb.1: ## %cond.load
-; AVX1-NEXT:    vmovsd {{.*#+}} xmm3 = mem[0],zero
-; AVX1-NEXT:    vblendps {{.*#+}} ymm0 = ymm3[0,1],ymm0[2,3,4,5,6,7]
+; AVX1-NEXT:    jne LBB1_1
+; AVX1-NEXT:  ## %bb.2: ## %else
+; AVX1-NEXT:    testb $2, %al
+; AVX1-NEXT:    jne LBB1_3
+; AVX1-NEXT:  LBB1_4: ## %else2
+; AVX1-NEXT:    testb $4, %al
+; AVX1-NEXT:    jne LBB1_5
+; AVX1-NEXT:  LBB1_6: ## %else6
+; AVX1-NEXT:    testb $8, %al
+; AVX1-NEXT:    jne LBB1_7
+; AVX1-NEXT:  LBB1_8: ## %else10
+; AVX1-NEXT:    retq
+; AVX1-NEXT:  LBB1_1: ## %cond.load
+; AVX1-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
+; AVX1-NEXT:    vblendpd {{.*#+}} ymm0 = ymm1[0],ymm0[1,2,3]
 ; AVX1-NEXT:    addq $8, %rdi
-; AVX1-NEXT:  LBB1_2: ## %else
-; AVX1-NEXT:    vpcmpeqq %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpextrb $8, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $2, %al
 ; AVX1-NEXT:    je LBB1_4
-; AVX1-NEXT:  ## %bb.3: ## %cond.load1
-; AVX1-NEXT:    vmovhps {{.*#+}} xmm2 = xmm0[0,1],mem[0,1]
-; AVX1-NEXT:    vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
+; AVX1-NEXT:  LBB1_3: ## %cond.load1
+; AVX1-NEXT:    vmovhpd {{.*#+}} xmm1 = xmm0[0],mem[0]
+; AVX1-NEXT:    vblendpd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3]
 ; AVX1-NEXT:    addq $8, %rdi
-; AVX1-NEXT:  LBB1_4: ## %else2
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm1
-; AVX1-NEXT:    vxorps %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqq %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $0, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $4, %al
 ; AVX1-NEXT:    je LBB1_6
-; AVX1-NEXT:  ## %bb.5: ## %cond.load5
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX1-NEXT:    vmovlps {{.*#+}} xmm2 = mem[0,1],xmm2[2,3]
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:  LBB1_5: ## %cond.load5
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vmovlpd {{.*#+}} xmm1 = mem[0],xmm1[1]
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX1-NEXT:    addq $8, %rdi
-; AVX1-NEXT:  LBB1_6: ## %else6
-; AVX1-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $8, %al
 ; AVX1-NEXT:    je LBB1_8
-; AVX1-NEXT:  ## %bb.7: ## %cond.load9
+; AVX1-NEXT:  LBB1_7: ## %cond.load9
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vmovhps {{.*#+}} xmm1 = xmm1[0,1],mem[0,1]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX1-NEXT:  LBB1_8: ## %else10
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: expandload_v4f64_v4i64:
 ; AVX2:       ## %bb.0:
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqq %xmm2, %xmm1, %xmm3
-; AVX2-NEXT:    vpextrb $0, %xmm3, %eax
+; AVX2-NEXT:    vpcmpeqq %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    vmovmskpd %ymm1, %eax
 ; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je LBB1_2
-; AVX2-NEXT:  ## %bb.1: ## %cond.load
-; AVX2-NEXT:    vmovq {{.*#+}} xmm3 = mem[0],zero
-; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm3[0,1],ymm0[2,3,4,5,6,7]
+; AVX2-NEXT:    jne LBB1_1
+; AVX2-NEXT:  ## %bb.2: ## %else
+; AVX2-NEXT:    testb $2, %al
+; AVX2-NEXT:    jne LBB1_3
+; AVX2-NEXT:  LBB1_4: ## %else2
+; AVX2-NEXT:    testb $4, %al
+; AVX2-NEXT:    jne LBB1_5
+; AVX2-NEXT:  LBB1_6: ## %else6
+; AVX2-NEXT:    testb $8, %al
+; AVX2-NEXT:    jne LBB1_7
+; AVX2-NEXT:  LBB1_8: ## %else10
+; AVX2-NEXT:    retq
+; AVX2-NEXT:  LBB1_1: ## %cond.load
+; AVX2-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
+; AVX2-NEXT:    vblendpd {{.*#+}} ymm0 = ymm1[0],ymm0[1,2,3]
 ; AVX2-NEXT:    addq $8, %rdi
-; AVX2-NEXT:  LBB1_2: ## %else
-; AVX2-NEXT:    vpcmpeqq %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpextrb $8, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $2, %al
 ; AVX2-NEXT:    je LBB1_4
-; AVX2-NEXT:  ## %bb.3: ## %cond.load1
-; AVX2-NEXT:    vmovhps {{.*#+}} xmm2 = xmm0[0,1],mem[0,1]
-; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
+; AVX2-NEXT:  LBB1_3: ## %cond.load1
+; AVX2-NEXT:    vmovhpd {{.*#+}} xmm1 = xmm0[0],mem[0]
+; AVX2-NEXT:    vblendpd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3]
 ; AVX2-NEXT:    addq $8, %rdi
-; AVX2-NEXT:  LBB1_4: ## %else2
-; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm1
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqq %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpextrb $0, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $4, %al
 ; AVX2-NEXT:    je LBB1_6
-; AVX2-NEXT:  ## %bb.5: ## %cond.load5
-; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm2
-; AVX2-NEXT:    vmovlps {{.*#+}} xmm2 = mem[0,1],xmm2[2,3]
-; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX2-NEXT:  LBB1_5: ## %cond.load5
+; AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vmovlpd {{.*#+}} xmm1 = mem[0],xmm1[1]
+; AVX2-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX2-NEXT:    addq $8, %rdi
-; AVX2-NEXT:  LBB1_6: ## %else6
-; AVX2-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $8, %al
 ; AVX2-NEXT:    je LBB1_8
-; AVX2-NEXT:  ## %bb.7: ## %cond.load9
-; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vmovhps {{.*#+}} xmm1 = xmm1[0,1],mem[0,1]
-; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-NEXT:  LBB1_8: ## %else10
+; AVX2-NEXT:  LBB1_7: ## %cond.load9
+; AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vmovhpd {{.*#+}} xmm1 = xmm1[0],mem[0]
+; AVX2-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: expandload_v4f64_v4i64:
@@ -276,194 +293,236 @@ define <4 x double> @expandload_v4f64_v4i64(double* %base, <4 x double> %src0, <
 }
 
 define <8 x double> @expandload_v8f64_v8i1(double* %base, <8 x double> %src0, <8 x i1> %mask) {
-; SSE2-LABEL: expandload_v8f64_v8i1:
-; SSE2:       ## %bb.0:
-; SSE2-NEXT:    movd %xmm4, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB2_2
-; SSE2-NEXT:  ## %bb.1: ## %cond.load
-; SSE2-NEXT:    movlps {{.*#+}} xmm0 = mem[0,1],xmm0[2,3]
-; SSE2-NEXT:    addq $8, %rdi
-; SSE2-NEXT:  LBB2_2: ## %else
-; SSE2-NEXT:    shrl $16, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB2_4
-; SSE2-NEXT:  ## %bb.3: ## %cond.load1
-; SSE2-NEXT:    movhps {{.*#+}} xmm0 = xmm0[0,1],mem[0,1]
-; SSE2-NEXT:    addq $8, %rdi
-; SSE2-NEXT:  LBB2_4: ## %else2
-; SSE2-NEXT:    pextrw $2, %xmm4, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB2_6
-; SSE2-NEXT:  ## %bb.5: ## %cond.load5
-; SSE2-NEXT:    movlps {{.*#+}} xmm1 = mem[0,1],xmm1[2,3]
-; SSE2-NEXT:    addq $8, %rdi
-; SSE2-NEXT:  LBB2_6: ## %else6
-; SSE2-NEXT:    pextrw $3, %xmm4, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB2_8
-; SSE2-NEXT:  ## %bb.7: ## %cond.load9
-; SSE2-NEXT:    movhps {{.*#+}} xmm1 = xmm1[0,1],mem[0,1]
-; SSE2-NEXT:    addq $8, %rdi
-; SSE2-NEXT:  LBB2_8: ## %else10
-; SSE2-NEXT:    pextrw $4, %xmm4, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB2_10
-; SSE2-NEXT:  ## %bb.9: ## %cond.load13
-; SSE2-NEXT:    movlps {{.*#+}} xmm2 = mem[0,1],xmm2[2,3]
-; SSE2-NEXT:    addq $8, %rdi
-; SSE2-NEXT:  LBB2_10: ## %else14
-; SSE2-NEXT:    pextrw $5, %xmm4, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB2_12
-; SSE2-NEXT:  ## %bb.11: ## %cond.load17
-; SSE2-NEXT:    movhps {{.*#+}} xmm2 = xmm2[0,1],mem[0,1]
-; SSE2-NEXT:    addq $8, %rdi
-; SSE2-NEXT:  LBB2_12: ## %else18
-; SSE2-NEXT:    pextrw $6, %xmm4, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB2_14
-; SSE2-NEXT:  ## %bb.13: ## %cond.load21
-; SSE2-NEXT:    movlps {{.*#+}} xmm3 = mem[0,1],xmm3[2,3]
-; SSE2-NEXT:    addq $8, %rdi
-; SSE2-NEXT:  LBB2_14: ## %else22
-; SSE2-NEXT:    pextrw $7, %xmm4, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB2_16
-; SSE2-NEXT:  ## %bb.15: ## %cond.load25
-; SSE2-NEXT:    movhps {{.*#+}} xmm3 = xmm3[0,1],mem[0,1]
-; SSE2-NEXT:  LBB2_16: ## %else26
-; SSE2-NEXT:    retq
+; SSE-LABEL: expandload_v8f64_v8i1:
+; SSE:       ## %bb.0:
+; SSE-NEXT:    psllw $15, %xmm4
+; SSE-NEXT:    packsswb %xmm0, %xmm4
+; SSE-NEXT:    pmovmskb %xmm4, %eax
+; SSE-NEXT:    testb $1, %al
+; SSE-NEXT:    jne LBB2_1
+; SSE-NEXT:  ## %bb.2: ## %else
+; SSE-NEXT:    testb $2, %al
+; SSE-NEXT:    jne LBB2_3
+; SSE-NEXT:  LBB2_4: ## %else2
+; SSE-NEXT:    testb $4, %al
+; SSE-NEXT:    jne LBB2_5
+; SSE-NEXT:  LBB2_6: ## %else6
+; SSE-NEXT:    testb $8, %al
+; SSE-NEXT:    jne LBB2_7
+; SSE-NEXT:  LBB2_8: ## %else10
+; SSE-NEXT:    testb $16, %al
+; SSE-NEXT:    jne LBB2_9
+; SSE-NEXT:  LBB2_10: ## %else14
+; SSE-NEXT:    testb $32, %al
+; SSE-NEXT:    jne LBB2_11
+; SSE-NEXT:  LBB2_12: ## %else18
+; SSE-NEXT:    testb $64, %al
+; SSE-NEXT:    jne LBB2_13
+; SSE-NEXT:  LBB2_14: ## %else22
+; SSE-NEXT:    testb $-128, %al
+; SSE-NEXT:    jne LBB2_15
+; SSE-NEXT:  LBB2_16: ## %else26
+; SSE-NEXT:    retq
+; SSE-NEXT:  LBB2_1: ## %cond.load
+; SSE-NEXT:    movlps {{.*#+}} xmm0 = mem[0,1],xmm0[2,3]
+; SSE-NEXT:    addq $8, %rdi
+; SSE-NEXT:    testb $2, %al
+; SSE-NEXT:    je LBB2_4
+; SSE-NEXT:  LBB2_3: ## %cond.load1
+; SSE-NEXT:    movhps {{.*#+}} xmm0 = xmm0[0,1],mem[0,1]
+; SSE-NEXT:    addq $8, %rdi
+; SSE-NEXT:    testb $4, %al
+; SSE-NEXT:    je LBB2_6
+; SSE-NEXT:  LBB2_5: ## %cond.load5
+; SSE-NEXT:    movlps {{.*#+}} xmm1 = mem[0,1],xmm1[2,3]
+; SSE-NEXT:    addq $8, %rdi
+; SSE-NEXT:    testb $8, %al
+; SSE-NEXT:    je LBB2_8
+; SSE-NEXT:  LBB2_7: ## %cond.load9
+; SSE-NEXT:    movhps {{.*#+}} xmm1 = xmm1[0,1],mem[0,1]
+; SSE-NEXT:    addq $8, %rdi
+; SSE-NEXT:    testb $16, %al
+; SSE-NEXT:    je LBB2_10
+; SSE-NEXT:  LBB2_9: ## %cond.load13
+; SSE-NEXT:    movlps {{.*#+}} xmm2 = mem[0,1],xmm2[2,3]
+; SSE-NEXT:    addq $8, %rdi
+; SSE-NEXT:    testb $32, %al
+; SSE-NEXT:    je LBB2_12
+; SSE-NEXT:  LBB2_11: ## %cond.load17
+; SSE-NEXT:    movhps {{.*#+}} xmm2 = xmm2[0,1],mem[0,1]
+; SSE-NEXT:    addq $8, %rdi
+; SSE-NEXT:    testb $64, %al
+; SSE-NEXT:    je LBB2_14
+; SSE-NEXT:  LBB2_13: ## %cond.load21
+; SSE-NEXT:    movlps {{.*#+}} xmm3 = mem[0,1],xmm3[2,3]
+; SSE-NEXT:    addq $8, %rdi
+; SSE-NEXT:    testb $-128, %al
+; SSE-NEXT:    je LBB2_16
+; SSE-NEXT:  LBB2_15: ## %cond.load25
+; SSE-NEXT:    movhps {{.*#+}} xmm3 = xmm3[0,1],mem[0,1]
+; SSE-NEXT:    retq
 ;
-; SSE42-LABEL: expandload_v8f64_v8i1:
-; SSE42:       ## %bb.0:
-; SSE42-NEXT:    pextrb $0, %xmm4, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB2_2
-; SSE42-NEXT:  ## %bb.1: ## %cond.load
-; SSE42-NEXT:    movlps {{.*#+}} xmm0 = mem[0,1],xmm0[2,3]
-; SSE42-NEXT:    addq $8, %rdi
-; SSE42-NEXT:  LBB2_2: ## %else
-; SSE42-NEXT:    pextrb $2, %xmm4, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB2_4
-; SSE42-NEXT:  ## %bb.3: ## %cond.load1
-; SSE42-NEXT:    movhps {{.*#+}} xmm0 = xmm0[0,1],mem[0,1]
-; SSE42-NEXT:    addq $8, %rdi
-; SSE42-NEXT:  LBB2_4: ## %else2
-; SSE42-NEXT:    pextrb $4, %xmm4, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB2_6
-; SSE42-NEXT:  ## %bb.5: ## %cond.load5
-; SSE42-NEXT:    movlps {{.*#+}} xmm1 = mem[0,1],xmm1[2,3]
-; SSE42-NEXT:    addq $8, %rdi
-; SSE42-NEXT:  LBB2_6: ## %else6
-; SSE42-NEXT:    pextrb $6, %xmm4, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB2_8
-; SSE42-NEXT:  ## %bb.7: ## %cond.load9
-; SSE42-NEXT:    movhps {{.*#+}} xmm1 = xmm1[0,1],mem[0,1]
-; SSE42-NEXT:    addq $8, %rdi
-; SSE42-NEXT:  LBB2_8: ## %else10
-; SSE42-NEXT:    pextrb $8, %xmm4, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB2_10
-; SSE42-NEXT:  ## %bb.9: ## %cond.load13
-; SSE42-NEXT:    movlps {{.*#+}} xmm2 = mem[0,1],xmm2[2,3]
-; SSE42-NEXT:    addq $8, %rdi
-; SSE42-NEXT:  LBB2_10: ## %else14
-; SSE42-NEXT:    pextrb $10, %xmm4, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB2_12
-; SSE42-NEXT:  ## %bb.11: ## %cond.load17
-; SSE42-NEXT:    movhps {{.*#+}} xmm2 = xmm2[0,1],mem[0,1]
-; SSE42-NEXT:    addq $8, %rdi
-; SSE42-NEXT:  LBB2_12: ## %else18
-; SSE42-NEXT:    pextrb $12, %xmm4, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB2_14
-; SSE42-NEXT:  ## %bb.13: ## %cond.load21
-; SSE42-NEXT:    movlps {{.*#+}} xmm3 = mem[0,1],xmm3[2,3]
-; SSE42-NEXT:    addq $8, %rdi
-; SSE42-NEXT:  LBB2_14: ## %else22
-; SSE42-NEXT:    pextrb $14, %xmm4, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB2_16
-; SSE42-NEXT:  ## %bb.15: ## %cond.load25
-; SSE42-NEXT:    movhps {{.*#+}} xmm3 = xmm3[0,1],mem[0,1]
-; SSE42-NEXT:  LBB2_16: ## %else26
-; SSE42-NEXT:    retq
+; AVX1-LABEL: expandload_v8f64_v8i1:
+; AVX1:       ## %bb.0:
+; AVX1-NEXT:    vpsllw $15, %xmm2, %xmm2
+; AVX1-NEXT:    vpacksswb %xmm0, %xmm2, %xmm2
+; AVX1-NEXT:    vpmovmskb %xmm2, %eax
+; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    jne LBB2_1
+; AVX1-NEXT:  ## %bb.2: ## %else
+; AVX1-NEXT:    testb $2, %al
+; AVX1-NEXT:    jne LBB2_3
+; AVX1-NEXT:  LBB2_4: ## %else2
+; AVX1-NEXT:    testb $4, %al
+; AVX1-NEXT:    jne LBB2_5
+; AVX1-NEXT:  LBB2_6: ## %else6
+; AVX1-NEXT:    testb $8, %al
+; AVX1-NEXT:    jne LBB2_7
+; AVX1-NEXT:  LBB2_8: ## %else10
+; AVX1-NEXT:    testb $16, %al
+; AVX1-NEXT:    jne LBB2_9
+; AVX1-NEXT:  LBB2_10: ## %else14
+; AVX1-NEXT:    testb $32, %al
+; AVX1-NEXT:    jne LBB2_11
+; AVX1-NEXT:  LBB2_12: ## %else18
+; AVX1-NEXT:    testb $64, %al
+; AVX1-NEXT:    jne LBB2_13
+; AVX1-NEXT:  LBB2_14: ## %else22
+; AVX1-NEXT:    testb $-128, %al
+; AVX1-NEXT:    jne LBB2_15
+; AVX1-NEXT:  LBB2_16: ## %else26
+; AVX1-NEXT:    retq
+; AVX1-NEXT:  LBB2_1: ## %cond.load
+; AVX1-NEXT:    vmovsd {{.*#+}} xmm2 = mem[0],zero
+; AVX1-NEXT:    vblendps {{.*#+}} ymm0 = ymm2[0,1],ymm0[2,3,4,5,6,7]
+; AVX1-NEXT:    addq $8, %rdi
+; AVX1-NEXT:    testb $2, %al
+; AVX1-NEXT:    je LBB2_4
+; AVX1-NEXT:  LBB2_3: ## %cond.load1
+; AVX1-NEXT:    vmovhps {{.*#+}} xmm2 = xmm0[0,1],mem[0,1]
+; AVX1-NEXT:    vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
+; AVX1-NEXT:    addq $8, %rdi
+; AVX1-NEXT:    testb $4, %al
+; AVX1-NEXT:    je LBB2_6
+; AVX1-NEXT:  LBB2_5: ## %cond.load5
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT:    vmovlps {{.*#+}} xmm2 = mem[0,1],xmm2[2,3]
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    addq $8, %rdi
+; AVX1-NEXT:    testb $8, %al
+; AVX1-NEXT:    je LBB2_8
+; AVX1-NEXT:  LBB2_7: ## %cond.load9
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT:    vmovhps {{.*#+}} xmm2 = xmm2[0,1],mem[0,1]
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    addq $8, %rdi
+; AVX1-NEXT:    testb $16, %al
+; AVX1-NEXT:    je LBB2_10
+; AVX1-NEXT:  LBB2_9: ## %cond.load13
+; AVX1-NEXT:    vmovsd {{.*#+}} xmm2 = mem[0],zero
+; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1],ymm1[2,3,4,5,6,7]
+; AVX1-NEXT:    addq $8, %rdi
+; AVX1-NEXT:    testb $32, %al
+; AVX1-NEXT:    je LBB2_12
+; AVX1-NEXT:  LBB2_11: ## %cond.load17
+; AVX1-NEXT:    vmovhps {{.*#+}} xmm2 = xmm1[0,1],mem[0,1]
+; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX1-NEXT:    addq $8, %rdi
+; AVX1-NEXT:    testb $64, %al
+; AVX1-NEXT:    je LBB2_14
+; AVX1-NEXT:  LBB2_13: ## %cond.load21
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT:    vmovlps {{.*#+}} xmm2 = mem[0,1],xmm2[2,3]
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; AVX1-NEXT:    addq $8, %rdi
+; AVX1-NEXT:    testb $-128, %al
+; AVX1-NEXT:    je LBB2_16
+; AVX1-NEXT:  LBB2_15: ## %cond.load25
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT:    vmovhps {{.*#+}} xmm2 = xmm2[0,1],mem[0,1]
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; AVX1-NEXT:    retq
 ;
-; AVX1OR2-LABEL: expandload_v8f64_v8i1:
-; AVX1OR2:       ## %bb.0:
-; AVX1OR2-NEXT:    vpextrb $0, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
-; AVX1OR2-NEXT:    je LBB2_2
-; AVX1OR2-NEXT:  ## %bb.1: ## %cond.load
-; AVX1OR2-NEXT:    vmovsd {{.*#+}} xmm3 = mem[0],zero
-; AVX1OR2-NEXT:    vblendps {{.*#+}} ymm0 = ymm3[0,1],ymm0[2,3,4,5,6,7]
-; AVX1OR2-NEXT:    addq $8, %rdi
-; AVX1OR2-NEXT:  LBB2_2: ## %else
-; AVX1OR2-NEXT:    vpextrb $2, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
-; AVX1OR2-NEXT:    je LBB2_4
-; AVX1OR2-NEXT:  ## %bb.3: ## %cond.load1
-; AVX1OR2-NEXT:    vmovhps {{.*#+}} xmm3 = xmm0[0,1],mem[0,1]
-; AVX1OR2-NEXT:    vblendps {{.*#+}} ymm0 = ymm3[0,1,2,3],ymm0[4,5,6,7]
-; AVX1OR2-NEXT:    addq $8, %rdi
-; AVX1OR2-NEXT:  LBB2_4: ## %else2
-; AVX1OR2-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
-; AVX1OR2-NEXT:    je LBB2_6
-; AVX1OR2-NEXT:  ## %bb.5: ## %cond.load5
-; AVX1OR2-NEXT:    vextractf128 $1, %ymm0, %xmm3
-; AVX1OR2-NEXT:    vmovlps {{.*#+}} xmm3 = mem[0,1],xmm3[2,3]
-; AVX1OR2-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
-; AVX1OR2-NEXT:    addq $8, %rdi
-; AVX1OR2-NEXT:  LBB2_6: ## %else6
-; AVX1OR2-NEXT:    vpextrb $6, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
-; AVX1OR2-NEXT:    je LBB2_8
-; AVX1OR2-NEXT:  ## %bb.7: ## %cond.load9
-; AVX1OR2-NEXT:    vextractf128 $1, %ymm0, %xmm3
-; AVX1OR2-NEXT:    vmovhps {{.*#+}} xmm3 = xmm3[0,1],mem[0,1]
-; AVX1OR2-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
-; AVX1OR2-NEXT:    addq $8, %rdi
-; AVX1OR2-NEXT:  LBB2_8: ## %else10
-; AVX1OR2-NEXT:    vpextrb $8, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
-; AVX1OR2-NEXT:    je LBB2_10
-; AVX1OR2-NEXT:  ## %bb.9: ## %cond.load13
-; AVX1OR2-NEXT:    vmovsd {{.*#+}} xmm3 = mem[0],zero
-; AVX1OR2-NEXT:    vblendps {{.*#+}} ymm1 = ymm3[0,1],ymm1[2,3,4,5,6,7]
-; AVX1OR2-NEXT:    addq $8, %rdi
-; AVX1OR2-NEXT:  LBB2_10: ## %else14
-; AVX1OR2-NEXT:    vpextrb $10, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
-; AVX1OR2-NEXT:    je LBB2_12
-; AVX1OR2-NEXT:  ## %bb.11: ## %cond.load17
-; AVX1OR2-NEXT:    vmovhps {{.*#+}} xmm3 = xmm1[0,1],mem[0,1]
-; AVX1OR2-NEXT:    vblendps {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
-; AVX1OR2-NEXT:    addq $8, %rdi
-; AVX1OR2-NEXT:  LBB2_12: ## %else18
-; AVX1OR2-NEXT:    vpextrb $12, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
-; AVX1OR2-NEXT:    je LBB2_14
-; AVX1OR2-NEXT:  ## %bb.13: ## %cond.load21
-; AVX1OR2-NEXT:    vextractf128 $1, %ymm1, %xmm3
-; AVX1OR2-NEXT:    vmovlps {{.*#+}} xmm3 = mem[0,1],xmm3[2,3]
-; AVX1OR2-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
-; AVX1OR2-NEXT:    addq $8, %rdi
-; AVX1OR2-NEXT:  LBB2_14: ## %else22
-; AVX1OR2-NEXT:    vpextrb $14, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
-; AVX1OR2-NEXT:    je LBB2_16
-; AVX1OR2-NEXT:  ## %bb.15: ## %cond.load25
-; AVX1OR2-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX1OR2-NEXT:    vmovhps {{.*#+}} xmm2 = xmm2[0,1],mem[0,1]
-; AVX1OR2-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
-; AVX1OR2-NEXT:  LBB2_16: ## %else26
-; AVX1OR2-NEXT:    retq
+; AVX2-LABEL: expandload_v8f64_v8i1:
+; AVX2:       ## %bb.0:
+; AVX2-NEXT:    vpsllw $15, %xmm2, %xmm2
+; AVX2-NEXT:    vpacksswb %xmm0, %xmm2, %xmm2
+; AVX2-NEXT:    vpmovmskb %xmm2, %eax
+; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    jne LBB2_1
+; AVX2-NEXT:  ## %bb.2: ## %else
+; AVX2-NEXT:    testb $2, %al
+; AVX2-NEXT:    jne LBB2_3
+; AVX2-NEXT:  LBB2_4: ## %else2
+; AVX2-NEXT:    testb $4, %al
+; AVX2-NEXT:    jne LBB2_5
+; AVX2-NEXT:  LBB2_6: ## %else6
+; AVX2-NEXT:    testb $8, %al
+; AVX2-NEXT:    jne LBB2_7
+; AVX2-NEXT:  LBB2_8: ## %else10
+; AVX2-NEXT:    testb $16, %al
+; AVX2-NEXT:    jne LBB2_9
+; AVX2-NEXT:  LBB2_10: ## %else14
+; AVX2-NEXT:    testb $32, %al
+; AVX2-NEXT:    jne LBB2_11
+; AVX2-NEXT:  LBB2_12: ## %else18
+; AVX2-NEXT:    testb $64, %al
+; AVX2-NEXT:    jne LBB2_13
+; AVX2-NEXT:  LBB2_14: ## %else22
+; AVX2-NEXT:    testb $-128, %al
+; AVX2-NEXT:    jne LBB2_15
+; AVX2-NEXT:  LBB2_16: ## %else26
+; AVX2-NEXT:    retq
+; AVX2-NEXT:  LBB2_1: ## %cond.load
+; AVX2-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1],ymm0[2,3,4,5,6,7]
+; AVX2-NEXT:    addq $8, %rdi
+; AVX2-NEXT:    testb $2, %al
+; AVX2-NEXT:    je LBB2_4
+; AVX2-NEXT:  LBB2_3: ## %cond.load1
+; AVX2-NEXT:    vmovhps {{.*#+}} xmm2 = xmm0[0,1],mem[0,1]
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
+; AVX2-NEXT:    addq $8, %rdi
+; AVX2-NEXT:    testb $4, %al
+; AVX2-NEXT:    je LBB2_6
+; AVX2-NEXT:  LBB2_5: ## %cond.load5
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm2
+; AVX2-NEXT:    vmovlps {{.*#+}} xmm2 = mem[0,1],xmm2[2,3]
+; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX2-NEXT:    addq $8, %rdi
+; AVX2-NEXT:    testb $8, %al
+; AVX2-NEXT:    je LBB2_8
+; AVX2-NEXT:  LBB2_7: ## %cond.load9
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm2
+; AVX2-NEXT:    vmovhps {{.*#+}} xmm2 = xmm2[0,1],mem[0,1]
+; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX2-NEXT:    addq $8, %rdi
+; AVX2-NEXT:    testb $16, %al
+; AVX2-NEXT:    je LBB2_10
+; AVX2-NEXT:  LBB2_9: ## %cond.load13
+; AVX2-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1],ymm1[2,3,4,5,6,7]
+; AVX2-NEXT:    addq $8, %rdi
+; AVX2-NEXT:    testb $32, %al
+; AVX2-NEXT:    je LBB2_12
+; AVX2-NEXT:  LBB2_11: ## %cond.load17
+; AVX2-NEXT:    vmovhps {{.*#+}} xmm2 = xmm1[0,1],mem[0,1]
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-NEXT:    addq $8, %rdi
+; AVX2-NEXT:    testb $64, %al
+; AVX2-NEXT:    je LBB2_14
+; AVX2-NEXT:  LBB2_13: ## %cond.load21
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
+; AVX2-NEXT:    vmovlps {{.*#+}} xmm2 = mem[0,1],xmm2[2,3]
+; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX2-NEXT:    addq $8, %rdi
+; AVX2-NEXT:    testb $-128, %al
+; AVX2-NEXT:    je LBB2_16
+; AVX2-NEXT:  LBB2_15: ## %cond.load25
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
+; AVX2-NEXT:    vmovhps {{.*#+}} xmm2 = xmm2[0,1],mem[0,1]
+; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: expandload_v8f64_v8i1:
 ; AVX512F:       ## %bb.0:
@@ -492,678 +551,482 @@ define <8 x double> @expandload_v8f64_v8i1(double* %base, <8 x double> %src0, <8
 }
 
 define <16 x double> @expandload_v16f64_v16i32(double* %base, <16 x double> %src0, <16 x i32> %trigger) {
-; SSE2-LABEL: expandload_v16f64_v16i32:
-; SSE2:       ## %bb.0:
-; SSE2-NEXT:    movq %rdi, %rax
-; SSE2-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm9
-; SSE2-NEXT:    pxor %xmm8, %xmm8
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm8
-; SSE2-NEXT:    movd %xmm8, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    je LBB3_2
-; SSE2-NEXT:  ## %bb.1: ## %cond.load
-; SSE2-NEXT:    movlps {{.*#+}} xmm0 = mem[0,1],xmm0[2,3]
-; SSE2-NEXT:    addq $8, %rsi
-; SSE2-NEXT:  LBB3_2: ## %else
-; SSE2-NEXT:    pextrw $2, %xmm8, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    je LBB3_4
-; SSE2-NEXT:  ## %bb.3: ## %cond.load1
-; SSE2-NEXT:    movhps {{.*#+}} xmm0 = xmm0[0,1],mem[0,1]
-; SSE2-NEXT:    addq $8, %rsi
-; SSE2-NEXT:  LBB3_4: ## %else2
-; SSE2-NEXT:    pxor %xmm8, %xmm8
-; SSE2-NEXT:    pcmpeqd %xmm8, %xmm9
-; SSE2-NEXT:    pextrw $4, %xmm9, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    je LBB3_6
-; SSE2-NEXT:  ## %bb.5: ## %cond.load5
-; SSE2-NEXT:    movlps {{.*#+}} xmm1 = mem[0,1],xmm1[2,3]
-; SSE2-NEXT:    addq $8, %rsi
-; SSE2-NEXT:  LBB3_6: ## %else6
-; SSE2-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm8
-; SSE2-NEXT:    pextrw $6, %xmm9, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    je LBB3_8
-; SSE2-NEXT:  ## %bb.7: ## %cond.load9
-; SSE2-NEXT:    movhps {{.*#+}} xmm1 = xmm1[0,1],mem[0,1]
-; SSE2-NEXT:    addq $8, %rsi
-; SSE2-NEXT:  LBB3_8: ## %else10
-; SSE2-NEXT:    pxor %xmm9, %xmm9
-; SSE2-NEXT:    pcmpeqd %xmm8, %xmm9
-; SSE2-NEXT:    movd %xmm9, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    je LBB3_10
-; SSE2-NEXT:  ## %bb.9: ## %cond.load13
-; SSE2-NEXT:    movlps {{.*#+}} xmm2 = mem[0,1],xmm2[2,3]
-; SSE2-NEXT:    addq $8, %rsi
-; SSE2-NEXT:  LBB3_10: ## %else14
-; SSE2-NEXT:    pextrw $2, %xmm9, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    je LBB3_12
-; SSE2-NEXT:  ## %bb.11: ## %cond.load17
-; SSE2-NEXT:    movhps {{.*#+}} xmm2 = xmm2[0,1],mem[0,1]
-; SSE2-NEXT:    addq $8, %rsi
-; SSE2-NEXT:  LBB3_12: ## %else18
-; SSE2-NEXT:    pxor %xmm9, %xmm9
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm8
-; SSE2-NEXT:    pextrw $4, %xmm8, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    je LBB3_14
-; SSE2-NEXT:  ## %bb.13: ## %cond.load21
-; SSE2-NEXT:    movlps {{.*#+}} xmm3 = mem[0,1],xmm3[2,3]
-; SSE2-NEXT:    addq $8, %rsi
-; SSE2-NEXT:  LBB3_14: ## %else22
-; SSE2-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm9
-; SSE2-NEXT:    pextrw $6, %xmm8, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    je LBB3_16
-; SSE2-NEXT:  ## %bb.15: ## %cond.load25
-; SSE2-NEXT:    movhps {{.*#+}} xmm3 = xmm3[0,1],mem[0,1]
-; SSE2-NEXT:    addq $8, %rsi
-; SSE2-NEXT:  LBB3_16: ## %else26
-; SSE2-NEXT:    pxor %xmm8, %xmm8
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm8
-; SSE2-NEXT:    movd %xmm8, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    je LBB3_18
-; SSE2-NEXT:  ## %bb.17: ## %cond.load29
-; SSE2-NEXT:    movlps {{.*#+}} xmm4 = mem[0,1],xmm4[2,3]
-; SSE2-NEXT:    addq $8, %rsi
-; SSE2-NEXT:  LBB3_18: ## %else30
-; SSE2-NEXT:    pextrw $2, %xmm8, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    je LBB3_20
-; SSE2-NEXT:  ## %bb.19: ## %cond.load33
-; SSE2-NEXT:    movhps {{.*#+}} xmm4 = xmm4[0,1],mem[0,1]
-; SSE2-NEXT:    addq $8, %rsi
-; SSE2-NEXT:  LBB3_20: ## %else34
-; SSE2-NEXT:    pxor %xmm8, %xmm8
-; SSE2-NEXT:    pcmpeqd %xmm8, %xmm9
-; SSE2-NEXT:    pextrw $4, %xmm9, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    je LBB3_22
-; SSE2-NEXT:  ## %bb.21: ## %cond.load37
-; SSE2-NEXT:    movlps {{.*#+}} xmm5 = mem[0,1],xmm5[2,3]
-; SSE2-NEXT:    addq $8, %rsi
-; SSE2-NEXT:  LBB3_22: ## %else38
-; SSE2-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm8
-; SSE2-NEXT:    pextrw $6, %xmm9, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    je LBB3_24
-; SSE2-NEXT:  ## %bb.23: ## %cond.load41
-; SSE2-NEXT:    movhps {{.*#+}} xmm5 = xmm5[0,1],mem[0,1]
-; SSE2-NEXT:    addq $8, %rsi
-; SSE2-NEXT:  LBB3_24: ## %else42
-; SSE2-NEXT:    pxor %xmm9, %xmm9
-; SSE2-NEXT:    pcmpeqd %xmm8, %xmm9
-; SSE2-NEXT:    movd %xmm9, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    je LBB3_26
-; SSE2-NEXT:  ## %bb.25: ## %cond.load45
-; SSE2-NEXT:    movlps {{.*#+}} xmm6 = mem[0,1],xmm6[2,3]
-; SSE2-NEXT:    addq $8, %rsi
-; SSE2-NEXT:  LBB3_26: ## %else46
-; SSE2-NEXT:    pextrw $2, %xmm9, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    je LBB3_28
-; SSE2-NEXT:  ## %bb.27: ## %cond.load49
-; SSE2-NEXT:    movhps {{.*#+}} xmm6 = xmm6[0,1],mem[0,1]
-; SSE2-NEXT:    addq $8, %rsi
-; SSE2-NEXT:  LBB3_28: ## %else50
-; SSE2-NEXT:    pxor %xmm9, %xmm9
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm8
-; SSE2-NEXT:    pextrw $4, %xmm8, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    je LBB3_30
-; SSE2-NEXT:  ## %bb.29: ## %cond.load53
-; SSE2-NEXT:    movlps {{.*#+}} xmm7 = mem[0,1],xmm7[2,3]
-; SSE2-NEXT:    addq $8, %rsi
-; SSE2-NEXT:  LBB3_30: ## %else54
-; SSE2-NEXT:    pextrw $6, %xmm8, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    je LBB3_32
-; SSE2-NEXT:  ## %bb.31: ## %cond.load57
-; SSE2-NEXT:    movhps {{.*#+}} xmm7 = xmm7[0,1],mem[0,1]
-; SSE2-NEXT:  LBB3_32: ## %else58
-; SSE2-NEXT:    movaps %xmm0, (%rax)
-; SSE2-NEXT:    movaps %xmm1, 16(%rax)
-; SSE2-NEXT:    movaps %xmm2, 32(%rax)
-; SSE2-NEXT:    movaps %xmm3, 48(%rax)
-; SSE2-NEXT:    movaps %xmm4, 64(%rax)
-; SSE2-NEXT:    movaps %xmm5, 80(%rax)
-; SSE2-NEXT:    movaps %xmm6, 96(%rax)
-; SSE2-NEXT:    movaps %xmm7, 112(%rax)
-; SSE2-NEXT:    retq
-;
-; SSE42-LABEL: expandload_v16f64_v16i32:
-; SSE42:       ## %bb.0:
-; SSE42-NEXT:    movq %rdi, %rax
-; SSE42-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm9
-; SSE42-NEXT:    pxor %xmm8, %xmm8
-; SSE42-NEXT:    pcmpeqd %xmm9, %xmm8
-; SSE42-NEXT:    pextrb $0, %xmm8, %ecx
-; SSE42-NEXT:    testb $1, %cl
-; SSE42-NEXT:    je LBB3_2
-; SSE42-NEXT:  ## %bb.1: ## %cond.load
-; SSE42-NEXT:    movlps {{.*#+}} xmm0 = mem[0,1],xmm0[2,3]
-; SSE42-NEXT:    addq $8, %rsi
-; SSE42-NEXT:  LBB3_2: ## %else
-; SSE42-NEXT:    pextrb $4, %xmm8, %ecx
-; SSE42-NEXT:    testb $1, %cl
-; SSE42-NEXT:    je LBB3_4
-; SSE42-NEXT:  ## %bb.3: ## %cond.load1
-; SSE42-NEXT:    movhps {{.*#+}} xmm0 = xmm0[0,1],mem[0,1]
-; SSE42-NEXT:    addq $8, %rsi
-; SSE42-NEXT:  LBB3_4: ## %else2
-; SSE42-NEXT:    pxor %xmm8, %xmm8
-; SSE42-NEXT:    pcmpeqd %xmm8, %xmm9
-; SSE42-NEXT:    pextrb $8, %xmm9, %ecx
-; SSE42-NEXT:    testb $1, %cl
-; SSE42-NEXT:    je LBB3_6
-; SSE42-NEXT:  ## %bb.5: ## %cond.load5
-; SSE42-NEXT:    movlps {{.*#+}} xmm1 = mem[0,1],xmm1[2,3]
-; SSE42-NEXT:    addq $8, %rsi
-; SSE42-NEXT:  LBB3_6: ## %else6
-; SSE42-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm8
-; SSE42-NEXT:    pextrb $12, %xmm9, %ecx
-; SSE42-NEXT:    testb $1, %cl
-; SSE42-NEXT:    je LBB3_8
-; SSE42-NEXT:  ## %bb.7: ## %cond.load9
-; SSE42-NEXT:    movhps {{.*#+}} xmm1 = xmm1[0,1],mem[0,1]
-; SSE42-NEXT:    addq $8, %rsi
-; SSE42-NEXT:  LBB3_8: ## %else10
-; SSE42-NEXT:    pxor %xmm9, %xmm9
-; SSE42-NEXT:    pcmpeqd %xmm8, %xmm9
-; SSE42-NEXT:    pextrb $0, %xmm9, %ecx
-; SSE42-NEXT:    testb $1, %cl
-; SSE42-NEXT:    je LBB3_10
-; SSE42-NEXT:  ## %bb.9: ## %cond.load13
-; SSE42-NEXT:    movlps {{.*#+}} xmm2 = mem[0,1],xmm2[2,3]
-; SSE42-NEXT:    addq $8, %rsi
-; SSE42-NEXT:  LBB3_10: ## %else14
-; SSE42-NEXT:    pextrb $4, %xmm9, %ecx
-; SSE42-NEXT:    testb $1, %cl
-; SSE42-NEXT:    je LBB3_12
-; SSE42-NEXT:  ## %bb.11: ## %cond.load17
-; SSE42-NEXT:    movhps {{.*#+}} xmm2 = xmm2[0,1],mem[0,1]
-; SSE42-NEXT:    addq $8, %rsi
-; SSE42-NEXT:  LBB3_12: ## %else18
-; SSE42-NEXT:    pxor %xmm9, %xmm9
-; SSE42-NEXT:    pcmpeqd %xmm9, %xmm8
-; SSE42-NEXT:    pextrb $8, %xmm8, %ecx
-; SSE42-NEXT:    testb $1, %cl
-; SSE42-NEXT:    je LBB3_14
-; SSE42-NEXT:  ## %bb.13: ## %cond.load21
-; SSE42-NEXT:    movlps {{.*#+}} xmm3 = mem[0,1],xmm3[2,3]
-; SSE42-NEXT:    addq $8, %rsi
-; SSE42-NEXT:  LBB3_14: ## %else22
-; SSE42-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm9
-; SSE42-NEXT:    pextrb $12, %xmm8, %ecx
-; SSE42-NEXT:    testb $1, %cl
-; SSE42-NEXT:    je LBB3_16
-; SSE42-NEXT:  ## %bb.15: ## %cond.load25
-; SSE42-NEXT:    movhps {{.*#+}} xmm3 = xmm3[0,1],mem[0,1]
-; SSE42-NEXT:    addq $8, %rsi
-; SSE42-NEXT:  LBB3_16: ## %else26
-; SSE42-NEXT:    pxor %xmm8, %xmm8
-; SSE42-NEXT:    pcmpeqd %xmm9, %xmm8
-; SSE42-NEXT:    pextrb $0, %xmm8, %ecx
-; SSE42-NEXT:    testb $1, %cl
-; SSE42-NEXT:    je LBB3_18
-; SSE42-NEXT:  ## %bb.17: ## %cond.load29
-; SSE42-NEXT:    movlps {{.*#+}} xmm4 = mem[0,1],xmm4[2,3]
-; SSE42-NEXT:    addq $8, %rsi
-; SSE42-NEXT:  LBB3_18: ## %else30
-; SSE42-NEXT:    pextrb $4, %xmm8, %ecx
-; SSE42-NEXT:    testb $1, %cl
-; SSE42-NEXT:    je LBB3_20
-; SSE42-NEXT:  ## %bb.19: ## %cond.load33
-; SSE42-NEXT:    movhps {{.*#+}} xmm4 = xmm4[0,1],mem[0,1]
-; SSE42-NEXT:    addq $8, %rsi
-; SSE42-NEXT:  LBB3_20: ## %else34
-; SSE42-NEXT:    pxor %xmm8, %xmm8
-; SSE42-NEXT:    pcmpeqd %xmm8, %xmm9
-; SSE42-NEXT:    pextrb $8, %xmm9, %ecx
-; SSE42-NEXT:    testb $1, %cl
-; SSE42-NEXT:    je LBB3_22
-; SSE42-NEXT:  ## %bb.21: ## %cond.load37
-; SSE42-NEXT:    movlps {{.*#+}} xmm5 = mem[0,1],xmm5[2,3]
-; SSE42-NEXT:    addq $8, %rsi
-; SSE42-NEXT:  LBB3_22: ## %else38
-; SSE42-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm8
-; SSE42-NEXT:    pextrb $12, %xmm9, %ecx
-; SSE42-NEXT:    testb $1, %cl
-; SSE42-NEXT:    je LBB3_24
-; SSE42-NEXT:  ## %bb.23: ## %cond.load41
-; SSE42-NEXT:    movhps {{.*#+}} xmm5 = xmm5[0,1],mem[0,1]
-; SSE42-NEXT:    addq $8, %rsi
-; SSE42-NEXT:  LBB3_24: ## %else42
-; SSE42-NEXT:    pxor %xmm9, %xmm9
-; SSE42-NEXT:    pcmpeqd %xmm8, %xmm9
-; SSE42-NEXT:    pextrb $0, %xmm9, %ecx
-; SSE42-NEXT:    testb $1, %cl
-; SSE42-NEXT:    je LBB3_26
-; SSE42-NEXT:  ## %bb.25: ## %cond.load45
-; SSE42-NEXT:    movlps {{.*#+}} xmm6 = mem[0,1],xmm6[2,3]
-; SSE42-NEXT:    addq $8, %rsi
-; SSE42-NEXT:  LBB3_26: ## %else46
-; SSE42-NEXT:    pextrb $4, %xmm9, %ecx
-; SSE42-NEXT:    testb $1, %cl
-; SSE42-NEXT:    je LBB3_28
-; SSE42-NEXT:  ## %bb.27: ## %cond.load49
-; SSE42-NEXT:    movhps {{.*#+}} xmm6 = xmm6[0,1],mem[0,1]
-; SSE42-NEXT:    addq $8, %rsi
-; SSE42-NEXT:  LBB3_28: ## %else50
-; SSE42-NEXT:    pxor %xmm9, %xmm9
-; SSE42-NEXT:    pcmpeqd %xmm9, %xmm8
-; SSE42-NEXT:    pextrb $8, %xmm8, %ecx
-; SSE42-NEXT:    testb $1, %cl
-; SSE42-NEXT:    je LBB3_30
-; SSE42-NEXT:  ## %bb.29: ## %cond.load53
-; SSE42-NEXT:    movlps {{.*#+}} xmm7 = mem[0,1],xmm7[2,3]
-; SSE42-NEXT:    addq $8, %rsi
-; SSE42-NEXT:  LBB3_30: ## %else54
-; SSE42-NEXT:    pextrb $12, %xmm8, %ecx
-; SSE42-NEXT:    testb $1, %cl
-; SSE42-NEXT:    je LBB3_32
-; SSE42-NEXT:  ## %bb.31: ## %cond.load57
-; SSE42-NEXT:    movhps {{.*#+}} xmm7 = xmm7[0,1],mem[0,1]
-; SSE42-NEXT:  LBB3_32: ## %else58
-; SSE42-NEXT:    movaps %xmm0, (%rax)
-; SSE42-NEXT:    movaps %xmm1, 16(%rax)
-; SSE42-NEXT:    movaps %xmm2, 32(%rax)
-; SSE42-NEXT:    movaps %xmm3, 48(%rax)
-; SSE42-NEXT:    movaps %xmm4, 64(%rax)
-; SSE42-NEXT:    movaps %xmm5, 80(%rax)
-; SSE42-NEXT:    movaps %xmm6, 96(%rax)
-; SSE42-NEXT:    movaps %xmm7, 112(%rax)
-; SSE42-NEXT:    retq
+; SSE-LABEL: expandload_v16f64_v16i32:
+; SSE:       ## %bb.0:
+; SSE-NEXT:    movq %rdi, %rax
+; SSE-NEXT:    pxor %xmm8, %xmm8
+; SSE-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm9
+; SSE-NEXT:    pcmpeqd %xmm8, %xmm9
+; SSE-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm10
+; SSE-NEXT:    pcmpeqd %xmm8, %xmm10
+; SSE-NEXT:    packssdw %xmm9, %xmm10
+; SSE-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm9
+; SSE-NEXT:    pcmpeqd %xmm8, %xmm9
+; SSE-NEXT:    pcmpeqd {{[0-9]+}}(%rsp), %xmm8
+; SSE-NEXT:    packssdw %xmm9, %xmm8
+; SSE-NEXT:    packsswb %xmm10, %xmm8
+; SSE-NEXT:    pmovmskb %xmm8, %ecx
+; SSE-NEXT:    testb $1, %cl
+; SSE-NEXT:    jne LBB3_1
+; SSE-NEXT:  ## %bb.2: ## %else
+; SSE-NEXT:    testb $2, %cl
+; SSE-NEXT:    jne LBB3_3
+; SSE-NEXT:  LBB3_4: ## %else2
+; SSE-NEXT:    testb $4, %cl
+; SSE-NEXT:    jne LBB3_5
+; SSE-NEXT:  LBB3_6: ## %else6
+; SSE-NEXT:    testb $8, %cl
+; SSE-NEXT:    jne LBB3_7
+; SSE-NEXT:  LBB3_8: ## %else10
+; SSE-NEXT:    testb $16, %cl
+; SSE-NEXT:    jne LBB3_9
+; SSE-NEXT:  LBB3_10: ## %else14
+; SSE-NEXT:    testb $32, %cl
+; SSE-NEXT:    jne LBB3_11
+; SSE-NEXT:  LBB3_12: ## %else18
+; SSE-NEXT:    testb $64, %cl
+; SSE-NEXT:    jne LBB3_13
+; SSE-NEXT:  LBB3_14: ## %else22
+; SSE-NEXT:    testb $-128, %cl
+; SSE-NEXT:    jne LBB3_15
+; SSE-NEXT:  LBB3_16: ## %else26
+; SSE-NEXT:    testl $256, %ecx ## imm = 0x100
+; SSE-NEXT:    jne LBB3_17
+; SSE-NEXT:  LBB3_18: ## %else30
+; SSE-NEXT:    testl $512, %ecx ## imm = 0x200
+; SSE-NEXT:    jne LBB3_19
+; SSE-NEXT:  LBB3_20: ## %else34
+; SSE-NEXT:    testl $1024, %ecx ## imm = 0x400
+; SSE-NEXT:    jne LBB3_21
+; SSE-NEXT:  LBB3_22: ## %else38
+; SSE-NEXT:    testl $2048, %ecx ## imm = 0x800
+; SSE-NEXT:    jne LBB3_23
+; SSE-NEXT:  LBB3_24: ## %else42
+; SSE-NEXT:    testl $4096, %ecx ## imm = 0x1000
+; SSE-NEXT:    jne LBB3_25
+; SSE-NEXT:  LBB3_26: ## %else46
+; SSE-NEXT:    testl $8192, %ecx ## imm = 0x2000
+; SSE-NEXT:    jne LBB3_27
+; SSE-NEXT:  LBB3_28: ## %else50
+; SSE-NEXT:    testl $16384, %ecx ## imm = 0x4000
+; SSE-NEXT:    jne LBB3_29
+; SSE-NEXT:  LBB3_30: ## %else54
+; SSE-NEXT:    testl $32768, %ecx ## imm = 0x8000
+; SSE-NEXT:    je LBB3_32
+; SSE-NEXT:  LBB3_31: ## %cond.load57
+; SSE-NEXT:    movhps {{.*#+}} xmm7 = xmm7[0,1],mem[0,1]
+; SSE-NEXT:  LBB3_32: ## %else58
+; SSE-NEXT:    movaps %xmm0, (%rax)
+; SSE-NEXT:    movaps %xmm1, 16(%rax)
+; SSE-NEXT:    movaps %xmm2, 32(%rax)
+; SSE-NEXT:    movaps %xmm3, 48(%rax)
+; SSE-NEXT:    movaps %xmm4, 64(%rax)
+; SSE-NEXT:    movaps %xmm5, 80(%rax)
+; SSE-NEXT:    movaps %xmm6, 96(%rax)
+; SSE-NEXT:    movaps %xmm7, 112(%rax)
+; SSE-NEXT:    retq
+; SSE-NEXT:  LBB3_1: ## %cond.load
+; SSE-NEXT:    movlps {{.*#+}} xmm0 = mem[0,1],xmm0[2,3]
+; SSE-NEXT:    addq $8, %rsi
+; SSE-NEXT:    testb $2, %cl
+; SSE-NEXT:    je LBB3_4
+; SSE-NEXT:  LBB3_3: ## %cond.load1
+; SSE-NEXT:    movhps {{.*#+}} xmm0 = xmm0[0,1],mem[0,1]
+; SSE-NEXT:    addq $8, %rsi
+; SSE-NEXT:    testb $4, %cl
+; SSE-NEXT:    je LBB3_6
+; SSE-NEXT:  LBB3_5: ## %cond.load5
+; SSE-NEXT:    movlps {{.*#+}} xmm1 = mem[0,1],xmm1[2,3]
+; SSE-NEXT:    addq $8, %rsi
+; SSE-NEXT:    testb $8, %cl
+; SSE-NEXT:    je LBB3_8
+; SSE-NEXT:  LBB3_7: ## %cond.load9
+; SSE-NEXT:    movhps {{.*#+}} xmm1 = xmm1[0,1],mem[0,1]
+; SSE-NEXT:    addq $8, %rsi
+; SSE-NEXT:    testb $16, %cl
+; SSE-NEXT:    je LBB3_10
+; SSE-NEXT:  LBB3_9: ## %cond.load13
+; SSE-NEXT:    movlps {{.*#+}} xmm2 = mem[0,1],xmm2[2,3]
+; SSE-NEXT:    addq $8, %rsi
+; SSE-NEXT:    testb $32, %cl
+; SSE-NEXT:    je LBB3_12
+; SSE-NEXT:  LBB3_11: ## %cond.load17
+; SSE-NEXT:    movhps {{.*#+}} xmm2 = xmm2[0,1],mem[0,1]
+; SSE-NEXT:    addq $8, %rsi
+; SSE-NEXT:    testb $64, %cl
+; SSE-NEXT:    je LBB3_14
+; SSE-NEXT:  LBB3_13: ## %cond.load21
+; SSE-NEXT:    movlps {{.*#+}} xmm3 = mem[0,1],xmm3[2,3]
+; SSE-NEXT:    addq $8, %rsi
+; SSE-NEXT:    testb $-128, %cl
+; SSE-NEXT:    je LBB3_16
+; SSE-NEXT:  LBB3_15: ## %cond.load25
+; SSE-NEXT:    movhps {{.*#+}} xmm3 = xmm3[0,1],mem[0,1]
+; SSE-NEXT:    addq $8, %rsi
+; SSE-NEXT:    testl $256, %ecx ## imm = 0x100
+; SSE-NEXT:    je LBB3_18
+; SSE-NEXT:  LBB3_17: ## %cond.load29
+; SSE-NEXT:    movlps {{.*#+}} xmm4 = mem[0,1],xmm4[2,3]
+; SSE-NEXT:    addq $8, %rsi
+; SSE-NEXT:    testl $512, %ecx ## imm = 0x200
+; SSE-NEXT:    je LBB3_20
+; SSE-NEXT:  LBB3_19: ## %cond.load33
+; SSE-NEXT:    movhps {{.*#+}} xmm4 = xmm4[0,1],mem[0,1]
+; SSE-NEXT:    addq $8, %rsi
+; SSE-NEXT:    testl $1024, %ecx ## imm = 0x400
+; SSE-NEXT:    je LBB3_22
+; SSE-NEXT:  LBB3_21: ## %cond.load37
+; SSE-NEXT:    movlps {{.*#+}} xmm5 = mem[0,1],xmm5[2,3]
+; SSE-NEXT:    addq $8, %rsi
+; SSE-NEXT:    testl $2048, %ecx ## imm = 0x800
+; SSE-NEXT:    je LBB3_24
+; SSE-NEXT:  LBB3_23: ## %cond.load41
+; SSE-NEXT:    movhps {{.*#+}} xmm5 = xmm5[0,1],mem[0,1]
+; SSE-NEXT:    addq $8, %rsi
+; SSE-NEXT:    testl $4096, %ecx ## imm = 0x1000
+; SSE-NEXT:    je LBB3_26
+; SSE-NEXT:  LBB3_25: ## %cond.load45
+; SSE-NEXT:    movlps {{.*#+}} xmm6 = mem[0,1],xmm6[2,3]
+; SSE-NEXT:    addq $8, %rsi
+; SSE-NEXT:    testl $8192, %ecx ## imm = 0x2000
+; SSE-NEXT:    je LBB3_28
+; SSE-NEXT:  LBB3_27: ## %cond.load49
+; SSE-NEXT:    movhps {{.*#+}} xmm6 = xmm6[0,1],mem[0,1]
+; SSE-NEXT:    addq $8, %rsi
+; SSE-NEXT:    testl $16384, %ecx ## imm = 0x4000
+; SSE-NEXT:    je LBB3_30
+; SSE-NEXT:  LBB3_29: ## %cond.load53
+; SSE-NEXT:    movlps {{.*#+}} xmm7 = mem[0,1],xmm7[2,3]
+; SSE-NEXT:    addq $8, %rsi
+; SSE-NEXT:    testl $32768, %ecx ## imm = 0x8000
+; SSE-NEXT:    jne LBB3_31
+; SSE-NEXT:    jmp LBB3_32
 ;
 ; AVX1-LABEL: expandload_v16f64_v16i32:
 ; AVX1:       ## %bb.0:
-; AVX1-NEXT:    vpxor %xmm6, %xmm6, %xmm6
-; AVX1-NEXT:    vpcmpeqd %xmm6, %xmm4, %xmm7
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm7, %xmm7
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm7, %xmm7
-; AVX1-NEXT:    vpextrb $0, %xmm7, %eax
+; AVX1-NEXT:    vextractf128 $1, %ymm5, %xmm6
+; AVX1-NEXT:    vpxor %xmm7, %xmm7, %xmm7
+; AVX1-NEXT:    vpcmpeqd %xmm7, %xmm6, %xmm6
+; AVX1-NEXT:    vpcmpeqd %xmm7, %xmm5, %xmm5
+; AVX1-NEXT:    vpackssdw %xmm6, %xmm5, %xmm5
+; AVX1-NEXT:    vextractf128 $1, %ymm4, %xmm6
+; AVX1-NEXT:    vpcmpeqd %xmm7, %xmm6, %xmm6
+; AVX1-NEXT:    vpcmpeqd %xmm7, %xmm4, %xmm4
+; AVX1-NEXT:    vpackssdw %xmm6, %xmm4, %xmm4
+; AVX1-NEXT:    vpacksswb %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vpmovmskb %xmm4, %eax
 ; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je LBB3_2
-; AVX1-NEXT:  ## %bb.1: ## %cond.load
-; AVX1-NEXT:    vmovsd {{.*#+}} xmm7 = mem[0],zero
-; AVX1-NEXT:    vblendps {{.*#+}} ymm0 = ymm7[0,1],ymm0[2,3,4,5,6,7]
+; AVX1-NEXT:    jne LBB3_1
+; AVX1-NEXT:  ## %bb.2: ## %else
+; AVX1-NEXT:    testb $2, %al
+; AVX1-NEXT:    jne LBB3_3
+; AVX1-NEXT:  LBB3_4: ## %else2
+; AVX1-NEXT:    testb $4, %al
+; AVX1-NEXT:    jne LBB3_5
+; AVX1-NEXT:  LBB3_6: ## %else6
+; AVX1-NEXT:    testb $8, %al
+; AVX1-NEXT:    jne LBB3_7
+; AVX1-NEXT:  LBB3_8: ## %else10
+; AVX1-NEXT:    testb $16, %al
+; AVX1-NEXT:    jne LBB3_9
+; AVX1-NEXT:  LBB3_10: ## %else14
+; AVX1-NEXT:    testb $32, %al
+; AVX1-NEXT:    jne LBB3_11
+; AVX1-NEXT:  LBB3_12: ## %else18
+; AVX1-NEXT:    testb $64, %al
+; AVX1-NEXT:    jne LBB3_13
+; AVX1-NEXT:  LBB3_14: ## %else22
+; AVX1-NEXT:    testb $-128, %al
+; AVX1-NEXT:    jne LBB3_15
+; AVX1-NEXT:  LBB3_16: ## %else26
+; AVX1-NEXT:    testl $256, %eax ## imm = 0x100
+; AVX1-NEXT:    jne LBB3_17
+; AVX1-NEXT:  LBB3_18: ## %else30
+; AVX1-NEXT:    testl $512, %eax ## imm = 0x200
+; AVX1-NEXT:    jne LBB3_19
+; AVX1-NEXT:  LBB3_20: ## %else34
+; AVX1-NEXT:    testl $1024, %eax ## imm = 0x400
+; AVX1-NEXT:    jne LBB3_21
+; AVX1-NEXT:  LBB3_22: ## %else38
+; AVX1-NEXT:    testl $2048, %eax ## imm = 0x800
+; AVX1-NEXT:    jne LBB3_23
+; AVX1-NEXT:  LBB3_24: ## %else42
+; AVX1-NEXT:    testl $4096, %eax ## imm = 0x1000
+; AVX1-NEXT:    jne LBB3_25
+; AVX1-NEXT:  LBB3_26: ## %else46
+; AVX1-NEXT:    testl $8192, %eax ## imm = 0x2000
+; AVX1-NEXT:    jne LBB3_27
+; AVX1-NEXT:  LBB3_28: ## %else50
+; AVX1-NEXT:    testl $16384, %eax ## imm = 0x4000
+; AVX1-NEXT:    jne LBB3_29
+; AVX1-NEXT:  LBB3_30: ## %else54
+; AVX1-NEXT:    testl $32768, %eax ## imm = 0x8000
+; AVX1-NEXT:    jne LBB3_31
+; AVX1-NEXT:  LBB3_32: ## %else58
+; AVX1-NEXT:    retq
+; AVX1-NEXT:  LBB3_1: ## %cond.load
+; AVX1-NEXT:    vmovsd {{.*#+}} xmm4 = mem[0],zero
+; AVX1-NEXT:    vblendps {{.*#+}} ymm0 = ymm4[0,1],ymm0[2,3,4,5,6,7]
 ; AVX1-NEXT:    addq $8, %rdi
-; AVX1-NEXT:  LBB3_2: ## %else
-; AVX1-NEXT:    vpcmpeqd %xmm6, %xmm4, %xmm6
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm6, %xmm6
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm6, %xmm6
-; AVX1-NEXT:    vpextrb $1, %xmm6, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $2, %al
 ; AVX1-NEXT:    je LBB3_4
-; AVX1-NEXT:  ## %bb.3: ## %cond.load1
-; AVX1-NEXT:    vmovhps {{.*#+}} xmm6 = xmm0[0,1],mem[0,1]
-; AVX1-NEXT:    vblendps {{.*#+}} ymm0 = ymm6[0,1,2,3],ymm0[4,5,6,7]
+; AVX1-NEXT:  LBB3_3: ## %cond.load1
+; AVX1-NEXT:    vmovhps {{.*#+}} xmm4 = xmm0[0,1],mem[0,1]
+; AVX1-NEXT:    vblendps {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm0[4,5,6,7]
 ; AVX1-NEXT:    addq $8, %rdi
-; AVX1-NEXT:  LBB3_4: ## %else2
-; AVX1-NEXT:    vxorps %xmm6, %xmm6, %xmm6
-; AVX1-NEXT:    vpcmpeqd %xmm6, %xmm4, %xmm7
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm7, %xmm7
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm7, %xmm7
-; AVX1-NEXT:    vpextrb $2, %xmm7, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $4, %al
 ; AVX1-NEXT:    je LBB3_6
-; AVX1-NEXT:  ## %bb.5: ## %cond.load5
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm7
-; AVX1-NEXT:    vmovlps {{.*#+}} xmm7 = mem[0,1],xmm7[2,3]
-; AVX1-NEXT:    vinsertf128 $1, %xmm7, %ymm0, %ymm0
+; AVX1-NEXT:  LBB3_5: ## %cond.load5
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
+; AVX1-NEXT:    vmovlps {{.*#+}} xmm4 = mem[0,1],xmm4[2,3]
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
 ; AVX1-NEXT:    addq $8, %rdi
-; AVX1-NEXT:  LBB3_6: ## %else6
-; AVX1-NEXT:    vpcmpeqd %xmm6, %xmm4, %xmm6
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm6, %xmm6
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm6, %xmm6
-; AVX1-NEXT:    vpextrb $3, %xmm6, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $8, %al
 ; AVX1-NEXT:    je LBB3_8
-; AVX1-NEXT:  ## %bb.7: ## %cond.load9
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm6
-; AVX1-NEXT:    vmovhps {{.*#+}} xmm6 = xmm6[0,1],mem[0,1]
-; AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm0, %ymm0
+; AVX1-NEXT:  LBB3_7: ## %cond.load9
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
+; AVX1-NEXT:    vmovhps {{.*#+}} xmm4 = xmm4[0,1],mem[0,1]
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
 ; AVX1-NEXT:    addq $8, %rdi
-; AVX1-NEXT:  LBB3_8: ## %else10
-; AVX1-NEXT:    vextractf128 $1, %ymm4, %xmm4
-; AVX1-NEXT:    vpxor %xmm6, %xmm6, %xmm6
-; AVX1-NEXT:    vpcmpeqd %xmm6, %xmm4, %xmm6
-; AVX1-NEXT:    vpackssdw %xmm6, %xmm0, %xmm7
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm7, %xmm7
-; AVX1-NEXT:    vpextrb $4, %xmm7, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $16, %al
 ; AVX1-NEXT:    je LBB3_10
-; AVX1-NEXT:  ## %bb.9: ## %cond.load13
-; AVX1-NEXT:    vmovsd {{.*#+}} xmm7 = mem[0],zero
-; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm7[0,1],ymm1[2,3,4,5,6,7]
+; AVX1-NEXT:  LBB3_9: ## %cond.load13
+; AVX1-NEXT:    vmovsd {{.*#+}} xmm4 = mem[0],zero
+; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm4[0,1],ymm1[2,3,4,5,6,7]
 ; AVX1-NEXT:    addq $8, %rdi
-; AVX1-NEXT:  LBB3_10: ## %else14
-; AVX1-NEXT:    vpackssdw %xmm6, %xmm0, %xmm6
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm6, %xmm6
-; AVX1-NEXT:    vpextrb $5, %xmm6, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $32, %al
 ; AVX1-NEXT:    je LBB3_12
-; AVX1-NEXT:  ## %bb.11: ## %cond.load17
-; AVX1-NEXT:    vmovhps {{.*#+}} xmm6 = xmm1[0,1],mem[0,1]
-; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm6[0,1,2,3],ymm1[4,5,6,7]
+; AVX1-NEXT:  LBB3_11: ## %cond.load17
+; AVX1-NEXT:    vmovhps {{.*#+}} xmm4 = xmm1[0,1],mem[0,1]
+; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm4[0,1,2,3],ymm1[4,5,6,7]
 ; AVX1-NEXT:    addq $8, %rdi
-; AVX1-NEXT:  LBB3_12: ## %else18
-; AVX1-NEXT:    vxorps %xmm6, %xmm6, %xmm6
-; AVX1-NEXT:    vpcmpeqd %xmm6, %xmm4, %xmm4
-; AVX1-NEXT:    vpackssdw %xmm4, %xmm0, %xmm6
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm6, %xmm6
-; AVX1-NEXT:    vpextrb $6, %xmm6, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $64, %al
 ; AVX1-NEXT:    je LBB3_14
-; AVX1-NEXT:  ## %bb.13: ## %cond.load21
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm6
-; AVX1-NEXT:    vmovlps {{.*#+}} xmm6 = mem[0,1],xmm6[2,3]
-; AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm1, %ymm1
+; AVX1-NEXT:  LBB3_13: ## %cond.load21
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm4
+; AVX1-NEXT:    vmovlps {{.*#+}} xmm4 = mem[0,1],xmm4[2,3]
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm1, %ymm1
 ; AVX1-NEXT:    addq $8, %rdi
-; AVX1-NEXT:  LBB3_14: ## %else22
-; AVX1-NEXT:    vpackssdw %xmm4, %xmm0, %xmm4
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm4, %xmm4
-; AVX1-NEXT:    vpextrb $7, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $-128, %al
 ; AVX1-NEXT:    je LBB3_16
-; AVX1-NEXT:  ## %bb.15: ## %cond.load25
+; AVX1-NEXT:  LBB3_15: ## %cond.load25
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm4
 ; AVX1-NEXT:    vmovhps {{.*#+}} xmm4 = xmm4[0,1],mem[0,1]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm1, %ymm1
 ; AVX1-NEXT:    addq $8, %rdi
-; AVX1-NEXT:  LBB3_16: ## %else26
-; AVX1-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm5, %xmm6
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm6, %xmm6
-; AVX1-NEXT:    vpacksswb %xmm6, %xmm0, %xmm6
-; AVX1-NEXT:    vpextrb $8, %xmm6, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $256, %eax ## imm = 0x100
 ; AVX1-NEXT:    je LBB3_18
-; AVX1-NEXT:  ## %bb.17: ## %cond.load29
-; AVX1-NEXT:    vmovsd {{.*#+}} xmm6 = mem[0],zero
-; AVX1-NEXT:    vblendps {{.*#+}} ymm2 = ymm6[0,1],ymm2[2,3,4,5,6,7]
+; AVX1-NEXT:  LBB3_17: ## %cond.load29
+; AVX1-NEXT:    vmovsd {{.*#+}} xmm4 = mem[0],zero
+; AVX1-NEXT:    vblendps {{.*#+}} ymm2 = ymm4[0,1],ymm2[2,3,4,5,6,7]
 ; AVX1-NEXT:    addq $8, %rdi
-; AVX1-NEXT:  LBB3_18: ## %else30
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm5, %xmm4
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm4, %xmm4
-; AVX1-NEXT:    vpacksswb %xmm4, %xmm0, %xmm4
-; AVX1-NEXT:    vpextrb $9, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $512, %eax ## imm = 0x200
 ; AVX1-NEXT:    je LBB3_20
-; AVX1-NEXT:  ## %bb.19: ## %cond.load33
+; AVX1-NEXT:  LBB3_19: ## %cond.load33
 ; AVX1-NEXT:    vmovhps {{.*#+}} xmm4 = xmm2[0,1],mem[0,1]
 ; AVX1-NEXT:    vblendps {{.*#+}} ymm2 = ymm4[0,1,2,3],ymm2[4,5,6,7]
 ; AVX1-NEXT:    addq $8, %rdi
-; AVX1-NEXT:  LBB3_20: ## %else34
-; AVX1-NEXT:    vxorps %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm5, %xmm6
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm6, %xmm6
-; AVX1-NEXT:    vpacksswb %xmm6, %xmm0, %xmm6
-; AVX1-NEXT:    vpextrb $10, %xmm6, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $1024, %eax ## imm = 0x400
 ; AVX1-NEXT:    je LBB3_22
-; AVX1-NEXT:  ## %bb.21: ## %cond.load37
-; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm6
-; AVX1-NEXT:    vmovlps {{.*#+}} xmm6 = mem[0,1],xmm6[2,3]
-; AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm2, %ymm2
+; AVX1-NEXT:  LBB3_21: ## %cond.load37
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm4
+; AVX1-NEXT:    vmovlps {{.*#+}} xmm4 = mem[0,1],xmm4[2,3]
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm2, %ymm2
 ; AVX1-NEXT:    addq $8, %rdi
-; AVX1-NEXT:  LBB3_22: ## %else38
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm5, %xmm4
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm4, %xmm4
-; AVX1-NEXT:    vpacksswb %xmm4, %xmm0, %xmm4
-; AVX1-NEXT:    vpextrb $11, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $2048, %eax ## imm = 0x800
 ; AVX1-NEXT:    je LBB3_24
-; AVX1-NEXT:  ## %bb.23: ## %cond.load41
+; AVX1-NEXT:  LBB3_23: ## %cond.load41
 ; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm4
 ; AVX1-NEXT:    vmovhps {{.*#+}} xmm4 = xmm4[0,1],mem[0,1]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm2, %ymm2
 ; AVX1-NEXT:    addq $8, %rdi
-; AVX1-NEXT:  LBB3_24: ## %else42
-; AVX1-NEXT:    vextractf128 $1, %ymm5, %xmm4
-; AVX1-NEXT:    vpxor %xmm5, %xmm5, %xmm5
-; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm4, %xmm5
-; AVX1-NEXT:    vpackssdw %xmm5, %xmm0, %xmm6
-; AVX1-NEXT:    vpacksswb %xmm6, %xmm0, %xmm6
-; AVX1-NEXT:    vpextrb $12, %xmm6, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $4096, %eax ## imm = 0x1000
 ; AVX1-NEXT:    je LBB3_26
-; AVX1-NEXT:  ## %bb.25: ## %cond.load45
-; AVX1-NEXT:    vmovsd {{.*#+}} xmm6 = mem[0],zero
-; AVX1-NEXT:    vblendps {{.*#+}} ymm3 = ymm6[0,1],ymm3[2,3,4,5,6,7]
+; AVX1-NEXT:  LBB3_25: ## %cond.load45
+; AVX1-NEXT:    vmovsd {{.*#+}} xmm4 = mem[0],zero
+; AVX1-NEXT:    vblendps {{.*#+}} ymm3 = ymm4[0,1],ymm3[2,3,4,5,6,7]
 ; AVX1-NEXT:    addq $8, %rdi
-; AVX1-NEXT:  LBB3_26: ## %else46
-; AVX1-NEXT:    vpackssdw %xmm5, %xmm0, %xmm5
-; AVX1-NEXT:    vpacksswb %xmm5, %xmm0, %xmm5
-; AVX1-NEXT:    vpextrb $13, %xmm5, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $8192, %eax ## imm = 0x2000
 ; AVX1-NEXT:    je LBB3_28
-; AVX1-NEXT:  ## %bb.27: ## %cond.load49
-; AVX1-NEXT:    vmovhps {{.*#+}} xmm5 = xmm3[0,1],mem[0,1]
-; AVX1-NEXT:    vblendps {{.*#+}} ymm3 = ymm5[0,1,2,3],ymm3[4,5,6,7]
+; AVX1-NEXT:  LBB3_27: ## %cond.load49
+; AVX1-NEXT:    vmovhps {{.*#+}} xmm4 = xmm3[0,1],mem[0,1]
+; AVX1-NEXT:    vblendps {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
 ; AVX1-NEXT:    addq $8, %rdi
-; AVX1-NEXT:  LBB3_28: ## %else50
-; AVX1-NEXT:    vxorps %xmm5, %xmm5, %xmm5
-; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vpackssdw %xmm4, %xmm0, %xmm5
-; AVX1-NEXT:    vpacksswb %xmm5, %xmm0, %xmm5
-; AVX1-NEXT:    vpextrb $14, %xmm5, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $16384, %eax ## imm = 0x4000
 ; AVX1-NEXT:    je LBB3_30
-; AVX1-NEXT:  ## %bb.29: ## %cond.load53
-; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm5
-; AVX1-NEXT:    vmovlps {{.*#+}} xmm5 = mem[0,1],xmm5[2,3]
-; AVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm3, %ymm3
+; AVX1-NEXT:  LBB3_29: ## %cond.load53
+; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm4
+; AVX1-NEXT:    vmovlps {{.*#+}} xmm4 = mem[0,1],xmm4[2,3]
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
 ; AVX1-NEXT:    addq $8, %rdi
-; AVX1-NEXT:  LBB3_30: ## %else54
-; AVX1-NEXT:    vpackssdw %xmm4, %xmm0, %xmm4
-; AVX1-NEXT:    vpacksswb %xmm4, %xmm0, %xmm4
-; AVX1-NEXT:    vpextrb $15, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $32768, %eax ## imm = 0x8000
 ; AVX1-NEXT:    je LBB3_32
-; AVX1-NEXT:  ## %bb.31: ## %cond.load57
+; AVX1-NEXT:  LBB3_31: ## %cond.load57
 ; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm4
 ; AVX1-NEXT:    vmovhps {{.*#+}} xmm4 = xmm4[0,1],mem[0,1]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
-; AVX1-NEXT:  LBB3_32: ## %else58
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: expandload_v16f64_v16i32:
 ; AVX2:       ## %bb.0:
 ; AVX2-NEXT:    vpxor %xmm6, %xmm6, %xmm6
-; AVX2-NEXT:    vpcmpeqd %xmm6, %xmm4, %xmm7
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm7, %xmm7
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm7, %xmm7
-; AVX2-NEXT:    vpextrb $0, %xmm7, %eax
+; AVX2-NEXT:    vpcmpeqd %ymm6, %ymm5, %ymm5
+; AVX2-NEXT:    vpcmpeqd %ymm6, %ymm4, %ymm4
+; AVX2-NEXT:    vpackssdw %ymm5, %ymm4, %ymm4
+; AVX2-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[0,2,1,3]
+; AVX2-NEXT:    vextracti128 $1, %ymm4, %xmm5
+; AVX2-NEXT:    vpacksswb %xmm5, %xmm4, %xmm4
+; AVX2-NEXT:    vpmovmskb %xmm4, %eax
 ; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je LBB3_2
-; AVX2-NEXT:  ## %bb.1: ## %cond.load
-; AVX2-NEXT:    vmovq {{.*#+}} xmm7 = mem[0],zero
-; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm7[0,1],ymm0[2,3,4,5,6,7]
+; AVX2-NEXT:    jne LBB3_1
+; AVX2-NEXT:  ## %bb.2: ## %else
+; AVX2-NEXT:    testb $2, %al
+; AVX2-NEXT:    jne LBB3_3
+; AVX2-NEXT:  LBB3_4: ## %else2
+; AVX2-NEXT:    testb $4, %al
+; AVX2-NEXT:    jne LBB3_5
+; AVX2-NEXT:  LBB3_6: ## %else6
+; AVX2-NEXT:    testb $8, %al
+; AVX2-NEXT:    jne LBB3_7
+; AVX2-NEXT:  LBB3_8: ## %else10
+; AVX2-NEXT:    testb $16, %al
+; AVX2-NEXT:    jne LBB3_9
+; AVX2-NEXT:  LBB3_10: ## %else14
+; AVX2-NEXT:    testb $32, %al
+; AVX2-NEXT:    jne LBB3_11
+; AVX2-NEXT:  LBB3_12: ## %else18
+; AVX2-NEXT:    testb $64, %al
+; AVX2-NEXT:    jne LBB3_13
+; AVX2-NEXT:  LBB3_14: ## %else22
+; AVX2-NEXT:    testb $-128, %al
+; AVX2-NEXT:    jne LBB3_15
+; AVX2-NEXT:  LBB3_16: ## %else26
+; AVX2-NEXT:    testl $256, %eax ## imm = 0x100
+; AVX2-NEXT:    jne LBB3_17
+; AVX2-NEXT:  LBB3_18: ## %else30
+; AVX2-NEXT:    testl $512, %eax ## imm = 0x200
+; AVX2-NEXT:    jne LBB3_19
+; AVX2-NEXT:  LBB3_20: ## %else34
+; AVX2-NEXT:    testl $1024, %eax ## imm = 0x400
+; AVX2-NEXT:    jne LBB3_21
+; AVX2-NEXT:  LBB3_22: ## %else38
+; AVX2-NEXT:    testl $2048, %eax ## imm = 0x800
+; AVX2-NEXT:    jne LBB3_23
+; AVX2-NEXT:  LBB3_24: ## %else42
+; AVX2-NEXT:    testl $4096, %eax ## imm = 0x1000
+; AVX2-NEXT:    jne LBB3_25
+; AVX2-NEXT:  LBB3_26: ## %else46
+; AVX2-NEXT:    testl $8192, %eax ## imm = 0x2000
+; AVX2-NEXT:    jne LBB3_27
+; AVX2-NEXT:  LBB3_28: ## %else50
+; AVX2-NEXT:    testl $16384, %eax ## imm = 0x4000
+; AVX2-NEXT:    jne LBB3_29
+; AVX2-NEXT:  LBB3_30: ## %else54
+; AVX2-NEXT:    testl $32768, %eax ## imm = 0x8000
+; AVX2-NEXT:    jne LBB3_31
+; AVX2-NEXT:  LBB3_32: ## %else58
+; AVX2-NEXT:    retq
+; AVX2-NEXT:  LBB3_1: ## %cond.load
+; AVX2-NEXT:    vmovq {{.*#+}} xmm4 = mem[0],zero
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm4[0,1],ymm0[2,3,4,5,6,7]
 ; AVX2-NEXT:    addq $8, %rdi
-; AVX2-NEXT:  LBB3_2: ## %else
-; AVX2-NEXT:    vpcmpeqd %xmm6, %xmm4, %xmm6
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm6, %xmm6
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm6, %xmm6
-; AVX2-NEXT:    vpextrb $1, %xmm6, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $2, %al
 ; AVX2-NEXT:    je LBB3_4
-; AVX2-NEXT:  ## %bb.3: ## %cond.load1
-; AVX2-NEXT:    vmovhps {{.*#+}} xmm6 = xmm0[0,1],mem[0,1]
-; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm6[0,1,2,3],ymm0[4,5,6,7]
+; AVX2-NEXT:  LBB3_3: ## %cond.load1
+; AVX2-NEXT:    vmovhps {{.*#+}} xmm4 = xmm0[0,1],mem[0,1]
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm0[4,5,6,7]
 ; AVX2-NEXT:    addq $8, %rdi
-; AVX2-NEXT:  LBB3_4: ## %else2
-; AVX2-NEXT:    vpxor %xmm6, %xmm6, %xmm6
-; AVX2-NEXT:    vpcmpeqd %xmm6, %xmm4, %xmm7
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm7, %xmm7
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm7, %xmm7
-; AVX2-NEXT:    vpextrb $2, %xmm7, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $4, %al
 ; AVX2-NEXT:    je LBB3_6
-; AVX2-NEXT:  ## %bb.5: ## %cond.load5
-; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm7
-; AVX2-NEXT:    vmovlps {{.*#+}} xmm7 = mem[0,1],xmm7[2,3]
-; AVX2-NEXT:    vinserti128 $1, %xmm7, %ymm0, %ymm0
+; AVX2-NEXT:  LBB3_5: ## %cond.load5
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm4
+; AVX2-NEXT:    vmovlps {{.*#+}} xmm4 = mem[0,1],xmm4[2,3]
+; AVX2-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm0
 ; AVX2-NEXT:    addq $8, %rdi
-; AVX2-NEXT:  LBB3_6: ## %else6
-; AVX2-NEXT:    vpcmpeqd %xmm6, %xmm4, %xmm6
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm6, %xmm6
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm6, %xmm6
-; AVX2-NEXT:    vpextrb $3, %xmm6, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $8, %al
 ; AVX2-NEXT:    je LBB3_8
-; AVX2-NEXT:  ## %bb.7: ## %cond.load9
-; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm6
-; AVX2-NEXT:    vmovhps {{.*#+}} xmm6 = xmm6[0,1],mem[0,1]
-; AVX2-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm0
+; AVX2-NEXT:  LBB3_7: ## %cond.load9
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm4
+; AVX2-NEXT:    vmovhps {{.*#+}} xmm4 = xmm4[0,1],mem[0,1]
+; AVX2-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm0
 ; AVX2-NEXT:    addq $8, %rdi
-; AVX2-NEXT:  LBB3_8: ## %else10
-; AVX2-NEXT:    vextracti128 $1, %ymm4, %xmm4
-; AVX2-NEXT:    vpxor %xmm6, %xmm6, %xmm6
-; AVX2-NEXT:    vpcmpeqd %xmm6, %xmm4, %xmm6
-; AVX2-NEXT:    vpackssdw %xmm6, %xmm0, %xmm7
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm7, %xmm7
-; AVX2-NEXT:    vpextrb $4, %xmm7, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $16, %al
 ; AVX2-NEXT:    je LBB3_10
-; AVX2-NEXT:  ## %bb.9: ## %cond.load13
-; AVX2-NEXT:    vmovq {{.*#+}} xmm7 = mem[0],zero
-; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm7[0,1],ymm1[2,3,4,5,6,7]
+; AVX2-NEXT:  LBB3_9: ## %cond.load13
+; AVX2-NEXT:    vmovq {{.*#+}} xmm4 = mem[0],zero
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm4[0,1],ymm1[2,3,4,5,6,7]
 ; AVX2-NEXT:    addq $8, %rdi
-; AVX2-NEXT:  LBB3_10: ## %else14
-; AVX2-NEXT:    vpackssdw %xmm6, %xmm0, %xmm6
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm6, %xmm6
-; AVX2-NEXT:    vpextrb $5, %xmm6, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $32, %al
 ; AVX2-NEXT:    je LBB3_12
-; AVX2-NEXT:  ## %bb.11: ## %cond.load17
-; AVX2-NEXT:    vmovhps {{.*#+}} xmm6 = xmm1[0,1],mem[0,1]
-; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm6[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-NEXT:  LBB3_11: ## %cond.load17
+; AVX2-NEXT:    vmovhps {{.*#+}} xmm4 = xmm1[0,1],mem[0,1]
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3],ymm1[4,5,6,7]
 ; AVX2-NEXT:    addq $8, %rdi
-; AVX2-NEXT:  LBB3_12: ## %else18
-; AVX2-NEXT:    vpxor %xmm6, %xmm6, %xmm6
-; AVX2-NEXT:    vpcmpeqd %xmm6, %xmm4, %xmm4
-; AVX2-NEXT:    vpackssdw %xmm4, %xmm0, %xmm6
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm6, %xmm6
-; AVX2-NEXT:    vpextrb $6, %xmm6, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $64, %al
 ; AVX2-NEXT:    je LBB3_14
-; AVX2-NEXT:  ## %bb.13: ## %cond.load21
-; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm6
-; AVX2-NEXT:    vmovlps {{.*#+}} xmm6 = mem[0,1],xmm6[2,3]
-; AVX2-NEXT:    vinserti128 $1, %xmm6, %ymm1, %ymm1
+; AVX2-NEXT:  LBB3_13: ## %cond.load21
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm4
+; AVX2-NEXT:    vmovlps {{.*#+}} xmm4 = mem[0,1],xmm4[2,3]
+; AVX2-NEXT:    vinserti128 $1, %xmm4, %ymm1, %ymm1
 ; AVX2-NEXT:    addq $8, %rdi
-; AVX2-NEXT:  LBB3_14: ## %else22
-; AVX2-NEXT:    vpackssdw %xmm4, %xmm0, %xmm4
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm4, %xmm4
-; AVX2-NEXT:    vpextrb $7, %xmm4, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $-128, %al
 ; AVX2-NEXT:    je LBB3_16
-; AVX2-NEXT:  ## %bb.15: ## %cond.load25
+; AVX2-NEXT:  LBB3_15: ## %cond.load25
 ; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm4
 ; AVX2-NEXT:    vmovhps {{.*#+}} xmm4 = xmm4[0,1],mem[0,1]
 ; AVX2-NEXT:    vinserti128 $1, %xmm4, %ymm1, %ymm1
 ; AVX2-NEXT:    addq $8, %rdi
-; AVX2-NEXT:  LBB3_16: ## %else26
-; AVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm5, %xmm6
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm6, %xmm6
-; AVX2-NEXT:    vpacksswb %xmm6, %xmm0, %xmm6
-; AVX2-NEXT:    vpextrb $8, %xmm6, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $256, %eax ## imm = 0x100
 ; AVX2-NEXT:    je LBB3_18
-; AVX2-NEXT:  ## %bb.17: ## %cond.load29
-; AVX2-NEXT:    vmovq {{.*#+}} xmm6 = mem[0],zero
-; AVX2-NEXT:    vpblendd {{.*#+}} ymm2 = ymm6[0,1],ymm2[2,3,4,5,6,7]
+; AVX2-NEXT:  LBB3_17: ## %cond.load29
+; AVX2-NEXT:    vmovq {{.*#+}} xmm4 = mem[0],zero
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm2 = ymm4[0,1],ymm2[2,3,4,5,6,7]
 ; AVX2-NEXT:    addq $8, %rdi
-; AVX2-NEXT:  LBB3_18: ## %else30
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm5, %xmm4
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm4, %xmm4
-; AVX2-NEXT:    vpacksswb %xmm4, %xmm0, %xmm4
-; AVX2-NEXT:    vpextrb $9, %xmm4, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $512, %eax ## imm = 0x200
 ; AVX2-NEXT:    je LBB3_20
-; AVX2-NEXT:  ## %bb.19: ## %cond.load33
+; AVX2-NEXT:  LBB3_19: ## %cond.load33
 ; AVX2-NEXT:    vmovhps {{.*#+}} xmm4 = xmm2[0,1],mem[0,1]
 ; AVX2-NEXT:    vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3],ymm2[4,5,6,7]
 ; AVX2-NEXT:    addq $8, %rdi
-; AVX2-NEXT:  LBB3_20: ## %else34
-; AVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm5, %xmm6
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm6, %xmm6
-; AVX2-NEXT:    vpacksswb %xmm6, %xmm0, %xmm6
-; AVX2-NEXT:    vpextrb $10, %xmm6, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $1024, %eax ## imm = 0x400
 ; AVX2-NEXT:    je LBB3_22
-; AVX2-NEXT:  ## %bb.21: ## %cond.load37
-; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm6
-; AVX2-NEXT:    vmovlps {{.*#+}} xmm6 = mem[0,1],xmm6[2,3]
-; AVX2-NEXT:    vinserti128 $1, %xmm6, %ymm2, %ymm2
+; AVX2-NEXT:  LBB3_21: ## %cond.load37
+; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm4
+; AVX2-NEXT:    vmovlps {{.*#+}} xmm4 = mem[0,1],xmm4[2,3]
+; AVX2-NEXT:    vinserti128 $1, %xmm4, %ymm2, %ymm2
 ; AVX2-NEXT:    addq $8, %rdi
-; AVX2-NEXT:  LBB3_22: ## %else38
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm5, %xmm4
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm4, %xmm4
-; AVX2-NEXT:    vpacksswb %xmm4, %xmm0, %xmm4
-; AVX2-NEXT:    vpextrb $11, %xmm4, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $2048, %eax ## imm = 0x800
 ; AVX2-NEXT:    je LBB3_24
-; AVX2-NEXT:  ## %bb.23: ## %cond.load41
+; AVX2-NEXT:  LBB3_23: ## %cond.load41
 ; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm4
 ; AVX2-NEXT:    vmovhps {{.*#+}} xmm4 = xmm4[0,1],mem[0,1]
 ; AVX2-NEXT:    vinserti128 $1, %xmm4, %ymm2, %ymm2
 ; AVX2-NEXT:    addq $8, %rdi
-; AVX2-NEXT:  LBB3_24: ## %else42
-; AVX2-NEXT:    vextracti128 $1, %ymm5, %xmm4
-; AVX2-NEXT:    vpxor %xmm5, %xmm5, %xmm5
-; AVX2-NEXT:    vpcmpeqd %xmm5, %xmm4, %xmm5
-; AVX2-NEXT:    vpackssdw %xmm5, %xmm0, %xmm6
-; AVX2-NEXT:    vpacksswb %xmm6, %xmm0, %xmm6
-; AVX2-NEXT:    vpextrb $12, %xmm6, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $4096, %eax ## imm = 0x1000
 ; AVX2-NEXT:    je LBB3_26
-; AVX2-NEXT:  ## %bb.25: ## %cond.load45
-; AVX2-NEXT:    vmovq {{.*#+}} xmm6 = mem[0],zero
-; AVX2-NEXT:    vpblendd {{.*#+}} ymm3 = ymm6[0,1],ymm3[2,3,4,5,6,7]
+; AVX2-NEXT:  LBB3_25: ## %cond.load45
+; AVX2-NEXT:    vmovq {{.*#+}} xmm4 = mem[0],zero
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1],ymm3[2,3,4,5,6,7]
 ; AVX2-NEXT:    addq $8, %rdi
-; AVX2-NEXT:  LBB3_26: ## %else46
-; AVX2-NEXT:    vpackssdw %xmm5, %xmm0, %xmm5
-; AVX2-NEXT:    vpacksswb %xmm5, %xmm0, %xmm5
-; AVX2-NEXT:    vpextrb $13, %xmm5, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $8192, %eax ## imm = 0x2000
 ; AVX2-NEXT:    je LBB3_28
-; AVX2-NEXT:  ## %bb.27: ## %cond.load49
-; AVX2-NEXT:    vmovhps {{.*#+}} xmm5 = xmm3[0,1],mem[0,1]
-; AVX2-NEXT:    vpblendd {{.*#+}} ymm3 = ymm5[0,1,2,3],ymm3[4,5,6,7]
+; AVX2-NEXT:  LBB3_27: ## %cond.load49
+; AVX2-NEXT:    vmovhps {{.*#+}} xmm4 = xmm3[0,1],mem[0,1]
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
 ; AVX2-NEXT:    addq $8, %rdi
-; AVX2-NEXT:  LBB3_28: ## %else50
-; AVX2-NEXT:    vpxor %xmm5, %xmm5, %xmm5
-; AVX2-NEXT:    vpcmpeqd %xmm5, %xmm4, %xmm4
-; AVX2-NEXT:    vpackssdw %xmm4, %xmm0, %xmm5
-; AVX2-NEXT:    vpacksswb %xmm5, %xmm0, %xmm5
-; AVX2-NEXT:    vpextrb $14, %xmm5, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $16384, %eax ## imm = 0x4000
 ; AVX2-NEXT:    je LBB3_30
-; AVX2-NEXT:  ## %bb.29: ## %cond.load53
-; AVX2-NEXT:    vextracti128 $1, %ymm3, %xmm5
-; AVX2-NEXT:    vmovlps {{.*#+}} xmm5 = mem[0,1],xmm5[2,3]
-; AVX2-NEXT:    vinserti128 $1, %xmm5, %ymm3, %ymm3
+; AVX2-NEXT:  LBB3_29: ## %cond.load53
+; AVX2-NEXT:    vextracti128 $1, %ymm3, %xmm4
+; AVX2-NEXT:    vmovlps {{.*#+}} xmm4 = mem[0,1],xmm4[2,3]
+; AVX2-NEXT:    vinserti128 $1, %xmm4, %ymm3, %ymm3
 ; AVX2-NEXT:    addq $8, %rdi
-; AVX2-NEXT:  LBB3_30: ## %else54
-; AVX2-NEXT:    vpackssdw %xmm4, %xmm0, %xmm4
-; AVX2-NEXT:    vpacksswb %xmm4, %xmm0, %xmm4
-; AVX2-NEXT:    vpextrb $15, %xmm4, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $32768, %eax ## imm = 0x8000
 ; AVX2-NEXT:    je LBB3_32
-; AVX2-NEXT:  ## %bb.31: ## %cond.load57
+; AVX2-NEXT:  LBB3_31: ## %cond.load57
 ; AVX2-NEXT:    vextracti128 $1, %ymm3, %xmm4
 ; AVX2-NEXT:    vmovhps {{.*#+}} xmm4 = xmm4[0,1],mem[0,1]
 ; AVX2-NEXT:    vinserti128 $1, %xmm4, %ymm3, %ymm3
-; AVX2-NEXT:  LBB3_32: ## %else58
 ; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: expandload_v16f64_v16i32:
@@ -1254,99 +1117,81 @@ define <16 x double> @expandload_v16f64_v16i32(double* %base, <16 x double> %src
 define <2 x float> @expandload_v2f32_v2i1(float* %base, <2 x float> %src0, <2 x i32> %trigger) {
 ; SSE2-LABEL: expandload_v2f32_v2i1:
 ; SSE2:       ## %bb.0:
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
 ; SSE2-NEXT:    pcmpeqd %xmm1, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,0,3,2]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    movd %xmm1, %eax
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,0,1,1]
+; SSE2-NEXT:    movmskpd %xmm1, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB4_2
-; SSE2-NEXT:  ## %bb.1: ## %cond.load
-; SSE2-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; SSE2-NEXT:    movss {{.*#+}} xmm0 = xmm2[0],xmm0[1,2,3]
+; SSE2-NEXT:    jne LBB4_1
+; SSE2-NEXT:  ## %bb.2: ## %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne LBB4_3
+; SSE2-NEXT:  LBB4_4: ## %else2
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  LBB4_1: ## %cond.load
+; SSE2-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE2-NEXT:    movss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
 ; SSE2-NEXT:    addq $4, %rdi
-; SSE2-NEXT:  LBB4_2: ## %else
-; SSE2-NEXT:    pextrw $4, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $2, %al
 ; SSE2-NEXT:    je LBB4_4
-; SSE2-NEXT:  ## %bb.3: ## %cond.load1
+; SSE2-NEXT:  LBB4_3: ## %cond.load1
 ; SSE2-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,0],xmm0[0,0]
 ; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[2,0],xmm0[2,3]
 ; SSE2-NEXT:    movaps %xmm1, %xmm0
-; SSE2-NEXT:  LBB4_4: ## %else2
 ; SSE2-NEXT:    retq
 ;
 ; SSE42-LABEL: expandload_v2f32_v2i1:
 ; SSE42:       ## %bb.0:
 ; SSE42-NEXT:    pxor %xmm2, %xmm2
-; SSE42-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
-; SSE42-NEXT:    pcmpeqq %xmm2, %xmm1
-; SSE42-NEXT:    pextrb $0, %xmm1, %eax
+; SSE42-NEXT:    pcmpeqd %xmm1, %xmm2
+; SSE42-NEXT:    pmovsxdq %xmm2, %xmm1
+; SSE42-NEXT:    movmskpd %xmm1, %eax
 ; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB4_2
-; SSE42-NEXT:  ## %bb.1: ## %cond.load
-; SSE42-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; SSE42-NEXT:    pblendw {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3,4,5,6,7]
+; SSE42-NEXT:    jne LBB4_1
+; SSE42-NEXT:  ## %bb.2: ## %else
+; SSE42-NEXT:    testb $2, %al
+; SSE42-NEXT:    jne LBB4_3
+; SSE42-NEXT:  LBB4_4: ## %else2
+; SSE42-NEXT:    retq
+; SSE42-NEXT:  LBB4_1: ## %cond.load
+; SSE42-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE42-NEXT:    blendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
 ; SSE42-NEXT:    addq $4, %rdi
-; SSE42-NEXT:  LBB4_2: ## %else
-; SSE42-NEXT:    pextrb $8, %xmm1, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $2, %al
 ; SSE42-NEXT:    je LBB4_4
-; SSE42-NEXT:  ## %bb.3: ## %cond.load1
+; SSE42-NEXT:  LBB4_3: ## %cond.load1
 ; SSE42-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[2,3]
-; SSE42-NEXT:  LBB4_4: ## %else2
 ; SSE42-NEXT:    retq
 ;
-; AVX1-LABEL: expandload_v2f32_v2i1:
-; AVX1:       ## %bb.0:
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
-; AVX1-NEXT:    vpcmpeqq %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $0, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je LBB4_2
-; AVX1-NEXT:  ## %bb.1: ## %cond.load
-; AVX1-NEXT:    vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3,4,5,6,7]
-; AVX1-NEXT:    addq $4, %rdi
-; AVX1-NEXT:  LBB4_2: ## %else
-; AVX1-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je LBB4_4
-; AVX1-NEXT:  ## %bb.3: ## %cond.load1
-; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[2,3]
-; AVX1-NEXT:  LBB4_4: ## %else2
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: expandload_v2f32_v2i1:
-; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; AVX2-NEXT:    vpcmpeqq %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpextrb $0, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je LBB4_2
-; AVX2-NEXT:  ## %bb.1: ## %cond.load
-; AVX2-NEXT:    vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm2[0],xmm0[1,2,3]
-; AVX2-NEXT:    addq $4, %rdi
-; AVX2-NEXT:  LBB4_2: ## %else
-; AVX2-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je LBB4_4
-; AVX2-NEXT:  ## %bb.3: ## %cond.load1
-; AVX2-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[2,3]
-; AVX2-NEXT:  LBB4_4: ## %else2
-; AVX2-NEXT:    retq
+; AVX1OR2-LABEL: expandload_v2f32_v2i1:
+; AVX1OR2:       ## %bb.0:
+; AVX1OR2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX1OR2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
+; AVX1OR2-NEXT:    vpmovsxdq %xmm1, %xmm1
+; AVX1OR2-NEXT:    vmovmskpd %xmm1, %eax
+; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    jne LBB4_1
+; AVX1OR2-NEXT:  ## %bb.2: ## %else
+; AVX1OR2-NEXT:    testb $2, %al
+; AVX1OR2-NEXT:    jne LBB4_3
+; AVX1OR2-NEXT:  LBB4_4: ## %else2
+; AVX1OR2-NEXT:    retq
+; AVX1OR2-NEXT:  LBB4_1: ## %cond.load
+; AVX1OR2-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX1OR2-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
+; AVX1OR2-NEXT:    addq $4, %rdi
+; AVX1OR2-NEXT:    testb $2, %al
+; AVX1OR2-NEXT:    je LBB4_4
+; AVX1OR2-NEXT:  LBB4_3: ## %cond.load1
+; AVX1OR2-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[2,3]
+; AVX1OR2-NEXT:    retq
 ;
 ; AVX512F-LABEL: expandload_v2f32_v2i1:
 ; AVX512F:       ## %bb.0:
+; AVX512F-NEXT:    ## kill: def $xmm1 killed $xmm1 def $zmm1
 ; AVX512F-NEXT:    ## kill: def $xmm0 killed $xmm0 def $zmm0
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; AVX512F-NEXT:    vptestnmq %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    vptestnmd %zmm1, %zmm1, %k0
 ; AVX512F-NEXT:    kshiftlw $14, %k0, %k0
 ; AVX512F-NEXT:    kshiftrw $14, %k0, %k1
 ; AVX512F-NEXT:    vexpandps (%rdi), %zmm0 {%k1}
@@ -1354,13 +1199,21 @@ define <2 x float> @expandload_v2f32_v2i1(float* %base, <2 x float> %src0, <2 x
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
-; AVX512VL-LABEL: expandload_v2f32_v2i1:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VL-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; AVX512VL-NEXT:    vptestnmq %xmm1, %xmm1, %k1
-; AVX512VL-NEXT:    vexpandps (%rdi), %xmm0 {%k1}
-; AVX512VL-NEXT:    retq
+; AVX512VLDQ-LABEL: expandload_v2f32_v2i1:
+; AVX512VLDQ:       ## %bb.0:
+; AVX512VLDQ-NEXT:    vptestnmd %xmm1, %xmm1, %k0
+; AVX512VLDQ-NEXT:    kshiftlb $6, %k0, %k0
+; AVX512VLDQ-NEXT:    kshiftrb $6, %k0, %k1
+; AVX512VLDQ-NEXT:    vexpandps (%rdi), %xmm0 {%k1}
+; AVX512VLDQ-NEXT:    retq
+;
+; AVX512VLBW-LABEL: expandload_v2f32_v2i1:
+; AVX512VLBW:       ## %bb.0:
+; AVX512VLBW-NEXT:    vptestnmd %xmm1, %xmm1, %k0
+; AVX512VLBW-NEXT:    kshiftlw $14, %k0, %k0
+; AVX512VLBW-NEXT:    kshiftrw $14, %k0, %k1
+; AVX512VLBW-NEXT:    vexpandps (%rdi), %xmm0 {%k1}
+; AVX512VLBW-NEXT:    retq
   %mask = icmp eq <2 x i32> %trigger, zeroinitializer
   %res = call <2 x float> @llvm.masked.expandload.v2f32(float* %base, <2 x i1> %mask, <2 x float> %src0)
   ret <2 x float> %res
@@ -1368,7 +1221,7 @@ define <2 x float> @expandload_v2f32_v2i1(float* %base, <2 x float> %src0, <2 x
 
 define <4 x float> @expandload_v4f32_const(float* %base, <4 x float> %src0) {
 ; SSE2-LABEL: expandload_v4f32_const:
-; SSE2:       ## %bb.0: ## %cond.load
+; SSE2:       ## %bb.0:
 ; SSE2-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
 ; SSE2-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,0],xmm2[0,0]
@@ -1379,7 +1232,7 @@ define <4 x float> @expandload_v4f32_const(float* %base, <4 x float> %src0) {
 ; SSE2-NEXT:    retq
 ;
 ; SSE42-LABEL: expandload_v4f32_const:
-; SSE42:       ## %bb.0: ## %cond.load
+; SSE42:       ## %bb.0:
 ; SSE42-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; SSE42-NEXT:    blendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
 ; SSE42-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[2,3]
@@ -1387,7 +1240,7 @@ define <4 x float> @expandload_v4f32_const(float* %base, <4 x float> %src0) {
 ; SSE42-NEXT:    retq
 ;
 ; AVX1OR2-LABEL: expandload_v4f32_const:
-; AVX1OR2:       ## %bb.0: ## %cond.load
+; AVX1OR2:       ## %bb.0:
 ; AVX1OR2-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; AVX1OR2-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
 ; AVX1OR2-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[2,3]
@@ -1423,7 +1276,7 @@ define <4 x float> @expandload_v4f32_const(float* %base, <4 x float> %src0) {
 
 define <16 x float> @expandload_v16f32_const(float* %base, <16 x float> %src0) {
 ; SSE2-LABEL: expandload_v16f32_const:
-; SSE2:       ## %bb.0: ## %cond.load
+; SSE2:       ## %bb.0:
 ; SSE2-NEXT:    movups (%rdi), %xmm0
 ; SSE2-NEXT:    movups 16(%rdi), %xmm1
 ; SSE2-NEXT:    movss {{.*#+}} xmm5 = mem[0],zero,zero,zero
@@ -1443,7 +1296,7 @@ define <16 x float> @expandload_v16f32_const(float* %base, <16 x float> %src0) {
 ; SSE2-NEXT:    retq
 ;
 ; SSE42-LABEL: expandload_v16f32_const:
-; SSE42:       ## %bb.0: ## %cond.load
+; SSE42:       ## %bb.0:
 ; SSE42-NEXT:    movups (%rdi), %xmm0
 ; SSE42-NEXT:    movups 16(%rdi), %xmm1
 ; SSE42-NEXT:    movss {{.*#+}} xmm4 = mem[0],zero,zero,zero
@@ -1457,13 +1310,12 @@ define <16 x float> @expandload_v16f32_const(float* %base, <16 x float> %src0) {
 ; SSE42-NEXT:    retq
 ;
 ; AVX1OR2-LABEL: expandload_v16f32_const:
-; AVX1OR2:       ## %bb.0: ## %cond.load
+; AVX1OR2:       ## %bb.0:
 ; AVX1OR2-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
 ; AVX1OR2-NEXT:    vblendps {{.*#+}} ymm0 = ymm2[0],ymm0[1,2,3,4,5,6,7]
-; AVX1OR2-NEXT:    vinsertps {{.*#+}} xmm2 = xmm0[0],mem[0],xmm0[2,3]
-; AVX1OR2-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1],mem[0],xmm2[3]
-; AVX1OR2-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1,2],mem[0]
-; AVX1OR2-NEXT:    vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
+; AVX1OR2-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[2,3]
+; AVX1OR2-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1],mem[0],xmm0[3]
+; AVX1OR2-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],mem[0]
 ; AVX1OR2-NEXT:    vmovsd {{.*#+}} xmm2 = mem[0],zero
 ; AVX1OR2-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1],mem[0],xmm2[3]
 ; AVX1OR2-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1,2],mem[0]
@@ -1472,7 +1324,6 @@ define <16 x float> @expandload_v16f32_const(float* %base, <16 x float> %src0) {
 ; AVX1OR2-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0],ymm1[1,2,3,4,5,6,7]
 ; AVX1OR2-NEXT:    vinsertps {{.*#+}} xmm2 = xmm1[0],mem[0],xmm1[2,3]
 ; AVX1OR2-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1],mem[0],xmm2[3]
-; AVX1OR2-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm1[4,5,6,7]
 ; AVX1OR2-NEXT:    vextractf128 $1, %ymm1, %xmm1
 ; AVX1OR2-NEXT:    vmovss {{.*#+}} xmm3 = mem[0],zero,zero,zero
 ; AVX1OR2-NEXT:    vblendps {{.*#+}} xmm1 = xmm3[0],xmm1[1,2,3]
@@ -1507,7 +1358,7 @@ define <16 x float> @expandload_v16f32_const(float* %base, <16 x float> %src0) {
 
 define <16 x float> @expandload_v16f32_const_undef(float* %base) {
 ; SSE2-LABEL: expandload_v16f32_const_undef:
-; SSE2:       ## %bb.0: ## %cond.load
+; SSE2:       ## %bb.0:
 ; SSE2-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; SSE2-NEXT:    movsd {{.*#+}} xmm2 = mem[0],zero
 ; SSE2-NEXT:    movlhps {{.*#+}} xmm2 = xmm2[0],xmm0[0]
@@ -1517,7 +1368,7 @@ define <16 x float> @expandload_v16f32_const_undef(float* %base) {
 ; SSE2-NEXT:    retq
 ;
 ; SSE42-LABEL: expandload_v16f32_const_undef:
-; SSE42:       ## %bb.0: ## %cond.load
+; SSE42:       ## %bb.0:
 ; SSE42-NEXT:    movsd {{.*#+}} xmm2 = mem[0],zero
 ; SSE42-NEXT:    insertps {{.*#+}} xmm2 = xmm2[0,1],mem[0],xmm2[3]
 ; SSE42-NEXT:    movups (%rdi), %xmm0
@@ -1526,7 +1377,7 @@ define <16 x float> @expandload_v16f32_const_undef(float* %base) {
 ; SSE42-NEXT:    retq
 ;
 ; AVX1OR2-LABEL: expandload_v16f32_const_undef:
-; AVX1OR2:       ## %bb.0: ## %cond.load
+; AVX1OR2:       ## %bb.0:
 ; AVX1OR2-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; AVX1OR2-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1],mem[0],xmm0[3]
 ; AVX1OR2-NEXT:    vinsertf128 $1, 44(%rdi), %ymm0, %ymm1
@@ -1562,1399 +1413,1314 @@ define <32 x float> @expandload_v32f32_v32i32(float* %base, <32 x float> %src0,
 ; SSE2-LABEL: expandload_v32f32_v32i32:
 ; SSE2:       ## %bb.0:
 ; SSE2-NEXT:    movq %rdi, %rax
-; SSE2-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm9
 ; SSE2-NEXT:    pxor %xmm8, %xmm8
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm8
-; SSE2-NEXT:    movd %xmm8, %ecx
+; SSE2-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm9
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm9
+; SSE2-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm10
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm10
+; SSE2-NEXT:    packssdw %xmm9, %xmm10
+; SSE2-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm9
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm9
+; SSE2-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm11
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm11
+; SSE2-NEXT:    packssdw %xmm9, %xmm11
+; SSE2-NEXT:    packsswb %xmm10, %xmm11
+; SSE2-NEXT:    pmovmskb %xmm11, %edx
+; SSE2-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm9
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm9
+; SSE2-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm10
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm10
+; SSE2-NEXT:    packssdw %xmm9, %xmm10
+; SSE2-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm9
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm9
+; SSE2-NEXT:    pcmpeqd {{[0-9]+}}(%rsp), %xmm8
+; SSE2-NEXT:    packssdw %xmm9, %xmm8
+; SSE2-NEXT:    packsswb %xmm10, %xmm8
+; SSE2-NEXT:    pmovmskb %xmm8, %ecx
+; SSE2-NEXT:    shll $16, %ecx
+; SSE2-NEXT:    orl %edx, %ecx
 ; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    je LBB8_2
-; SSE2-NEXT:  ## %bb.1: ## %cond.load
-; SSE2-NEXT:    movss {{.*#+}} xmm10 = mem[0],zero,zero,zero
-; SSE2-NEXT:    movss {{.*#+}} xmm0 = xmm10[0],xmm0[1,2,3]
+; SSE2-NEXT:    jne LBB8_1
+; SSE2-NEXT:  ## %bb.2: ## %else
+; SSE2-NEXT:    testb $2, %cl
+; SSE2-NEXT:    jne LBB8_3
+; SSE2-NEXT:  LBB8_4: ## %else2
+; SSE2-NEXT:    testb $4, %cl
+; SSE2-NEXT:    jne LBB8_5
+; SSE2-NEXT:  LBB8_6: ## %else6
+; SSE2-NEXT:    testb $8, %cl
+; SSE2-NEXT:    jne LBB8_7
+; SSE2-NEXT:  LBB8_8: ## %else10
+; SSE2-NEXT:    testb $16, %cl
+; SSE2-NEXT:    jne LBB8_9
+; SSE2-NEXT:  LBB8_10: ## %else14
+; SSE2-NEXT:    testb $32, %cl
+; SSE2-NEXT:    jne LBB8_11
+; SSE2-NEXT:  LBB8_12: ## %else18
+; SSE2-NEXT:    testb $64, %cl
+; SSE2-NEXT:    jne LBB8_13
+; SSE2-NEXT:  LBB8_14: ## %else22
+; SSE2-NEXT:    testb $-128, %cl
+; SSE2-NEXT:    jne LBB8_15
+; SSE2-NEXT:  LBB8_16: ## %else26
+; SSE2-NEXT:    testl $256, %ecx ## imm = 0x100
+; SSE2-NEXT:    jne LBB8_17
+; SSE2-NEXT:  LBB8_18: ## %else30
+; SSE2-NEXT:    testl $512, %ecx ## imm = 0x200
+; SSE2-NEXT:    jne LBB8_19
+; SSE2-NEXT:  LBB8_20: ## %else34
+; SSE2-NEXT:    testl $1024, %ecx ## imm = 0x400
+; SSE2-NEXT:    jne LBB8_21
+; SSE2-NEXT:  LBB8_22: ## %else38
+; SSE2-NEXT:    testl $2048, %ecx ## imm = 0x800
+; SSE2-NEXT:    jne LBB8_23
+; SSE2-NEXT:  LBB8_24: ## %else42
+; SSE2-NEXT:    testl $4096, %ecx ## imm = 0x1000
+; SSE2-NEXT:    jne LBB8_25
+; SSE2-NEXT:  LBB8_26: ## %else46
+; SSE2-NEXT:    testl $8192, %ecx ## imm = 0x2000
+; SSE2-NEXT:    jne LBB8_27
+; SSE2-NEXT:  LBB8_28: ## %else50
+; SSE2-NEXT:    testl $16384, %ecx ## imm = 0x4000
+; SSE2-NEXT:    jne LBB8_29
+; SSE2-NEXT:  LBB8_30: ## %else54
+; SSE2-NEXT:    testl $32768, %ecx ## imm = 0x8000
+; SSE2-NEXT:    jne LBB8_31
+; SSE2-NEXT:  LBB8_32: ## %else58
+; SSE2-NEXT:    testl $65536, %ecx ## imm = 0x10000
+; SSE2-NEXT:    jne LBB8_33
+; SSE2-NEXT:  LBB8_34: ## %else62
+; SSE2-NEXT:    testl $131072, %ecx ## imm = 0x20000
+; SSE2-NEXT:    jne LBB8_35
+; SSE2-NEXT:  LBB8_36: ## %else66
+; SSE2-NEXT:    testl $262144, %ecx ## imm = 0x40000
+; SSE2-NEXT:    jne LBB8_37
+; SSE2-NEXT:  LBB8_38: ## %else70
+; SSE2-NEXT:    testl $524288, %ecx ## imm = 0x80000
+; SSE2-NEXT:    jne LBB8_39
+; SSE2-NEXT:  LBB8_40: ## %else74
+; SSE2-NEXT:    testl $1048576, %ecx ## imm = 0x100000
+; SSE2-NEXT:    jne LBB8_41
+; SSE2-NEXT:  LBB8_42: ## %else78
+; SSE2-NEXT:    testl $2097152, %ecx ## imm = 0x200000
+; SSE2-NEXT:    jne LBB8_43
+; SSE2-NEXT:  LBB8_44: ## %else82
+; SSE2-NEXT:    testl $4194304, %ecx ## imm = 0x400000
+; SSE2-NEXT:    jne LBB8_45
+; SSE2-NEXT:  LBB8_46: ## %else86
+; SSE2-NEXT:    testl $8388608, %ecx ## imm = 0x800000
+; SSE2-NEXT:    jne LBB8_47
+; SSE2-NEXT:  LBB8_48: ## %else90
+; SSE2-NEXT:    testl $16777216, %ecx ## imm = 0x1000000
+; SSE2-NEXT:    jne LBB8_49
+; SSE2-NEXT:  LBB8_50: ## %else94
+; SSE2-NEXT:    testl $33554432, %ecx ## imm = 0x2000000
+; SSE2-NEXT:    jne LBB8_51
+; SSE2-NEXT:  LBB8_52: ## %else98
+; SSE2-NEXT:    testl $67108864, %ecx ## imm = 0x4000000
+; SSE2-NEXT:    jne LBB8_53
+; SSE2-NEXT:  LBB8_54: ## %else102
+; SSE2-NEXT:    testl $134217728, %ecx ## imm = 0x8000000
+; SSE2-NEXT:    jne LBB8_55
+; SSE2-NEXT:  LBB8_56: ## %else106
+; SSE2-NEXT:    testl $268435456, %ecx ## imm = 0x10000000
+; SSE2-NEXT:    jne LBB8_57
+; SSE2-NEXT:  LBB8_58: ## %else110
+; SSE2-NEXT:    testl $536870912, %ecx ## imm = 0x20000000
+; SSE2-NEXT:    jne LBB8_59
+; SSE2-NEXT:  LBB8_60: ## %else114
+; SSE2-NEXT:    testl $1073741824, %ecx ## imm = 0x40000000
+; SSE2-NEXT:    jne LBB8_61
+; SSE2-NEXT:  LBB8_62: ## %else118
+; SSE2-NEXT:    testl $-2147483648, %ecx ## imm = 0x80000000
+; SSE2-NEXT:    je LBB8_64
+; SSE2-NEXT:  LBB8_63: ## %cond.load121
+; SSE2-NEXT:    movss {{.*#+}} xmm8 = mem[0],zero,zero,zero
+; SSE2-NEXT:    shufps {{.*#+}} xmm8 = xmm8[0,0],xmm7[2,0]
+; SSE2-NEXT:    shufps {{.*#+}} xmm7 = xmm7[0,1],xmm8[2,0]
+; SSE2-NEXT:  LBB8_64: ## %else122
+; SSE2-NEXT:    movaps %xmm0, (%rax)
+; SSE2-NEXT:    movaps %xmm1, 16(%rax)
+; SSE2-NEXT:    movaps %xmm2, 32(%rax)
+; SSE2-NEXT:    movaps %xmm3, 48(%rax)
+; SSE2-NEXT:    movaps %xmm4, 64(%rax)
+; SSE2-NEXT:    movaps %xmm5, 80(%rax)
+; SSE2-NEXT:    movaps %xmm6, 96(%rax)
+; SSE2-NEXT:    movaps %xmm7, 112(%rax)
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  LBB8_1: ## %cond.load
+; SSE2-NEXT:    movss {{.*#+}} xmm8 = mem[0],zero,zero,zero
+; SSE2-NEXT:    movss {{.*#+}} xmm0 = xmm8[0],xmm0[1,2,3]
 ; SSE2-NEXT:    addq $4, %rsi
-; SSE2-NEXT:  LBB8_2: ## %else
-; SSE2-NEXT:    pextrw $2, %xmm8, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testb $2, %cl
 ; SSE2-NEXT:    je LBB8_4
-; SSE2-NEXT:  ## %bb.3: ## %cond.load1
+; SSE2-NEXT:  LBB8_3: ## %cond.load1
 ; SSE2-NEXT:    movss {{.*#+}} xmm8 = mem[0],zero,zero,zero
 ; SSE2-NEXT:    shufps {{.*#+}} xmm8 = xmm8[0,0],xmm0[0,0]
 ; SSE2-NEXT:    shufps {{.*#+}} xmm8 = xmm8[2,0],xmm0[2,3]
 ; SSE2-NEXT:    addq $4, %rsi
 ; SSE2-NEXT:    movaps %xmm8, %xmm0
-; SSE2-NEXT:  LBB8_4: ## %else2
-; SSE2-NEXT:    xorps %xmm8, %xmm8
-; SSE2-NEXT:    pcmpeqd %xmm8, %xmm9
-; SSE2-NEXT:    pextrw $4, %xmm9, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testb $4, %cl
 ; SSE2-NEXT:    je LBB8_6
-; SSE2-NEXT:  ## %bb.5: ## %cond.load5
+; SSE2-NEXT:  LBB8_5: ## %cond.load5
 ; SSE2-NEXT:    movss {{.*#+}} xmm8 = mem[0],zero,zero,zero
 ; SSE2-NEXT:    shufps {{.*#+}} xmm8 = xmm8[0,0],xmm0[3,0]
 ; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm8[0,2]
 ; SSE2-NEXT:    addq $4, %rsi
-; SSE2-NEXT:  LBB8_6: ## %else6
-; SSE2-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm8
-; SSE2-NEXT:    pextrw $6, %xmm9, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testb $8, %cl
 ; SSE2-NEXT:    je LBB8_8
-; SSE2-NEXT:  ## %bb.7: ## %cond.load9
-; SSE2-NEXT:    movss {{.*#+}} xmm9 = mem[0],zero,zero,zero
-; SSE2-NEXT:    shufps {{.*#+}} xmm9 = xmm9[0,0],xmm0[2,0]
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm9[2,0]
+; SSE2-NEXT:  LBB8_7: ## %cond.load9
+; SSE2-NEXT:    movss {{.*#+}} xmm8 = mem[0],zero,zero,zero
+; SSE2-NEXT:    shufps {{.*#+}} xmm8 = xmm8[0,0],xmm0[2,0]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm8[2,0]
 ; SSE2-NEXT:    addq $4, %rsi
-; SSE2-NEXT:  LBB8_8: ## %else10
-; SSE2-NEXT:    xorps %xmm9, %xmm9
-; SSE2-NEXT:    pcmpeqd %xmm8, %xmm9
-; SSE2-NEXT:    movd %xmm9, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testb $16, %cl
 ; SSE2-NEXT:    je LBB8_10
-; SSE2-NEXT:  ## %bb.9: ## %cond.load13
-; SSE2-NEXT:    movss {{.*#+}} xmm10 = mem[0],zero,zero,zero
-; SSE2-NEXT:    movss {{.*#+}} xmm1 = xmm10[0],xmm1[1,2,3]
+; SSE2-NEXT:  LBB8_9: ## %cond.load13
+; SSE2-NEXT:    movss {{.*#+}} xmm8 = mem[0],zero,zero,zero
+; SSE2-NEXT:    movss {{.*#+}} xmm1 = xmm8[0],xmm1[1,2,3]
 ; SSE2-NEXT:    addq $4, %rsi
-; SSE2-NEXT:  LBB8_10: ## %else14
-; SSE2-NEXT:    pextrw $2, %xmm9, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testb $32, %cl
 ; SSE2-NEXT:    je LBB8_12
-; SSE2-NEXT:  ## %bb.11: ## %cond.load17
-; SSE2-NEXT:    movss {{.*#+}} xmm9 = mem[0],zero,zero,zero
-; SSE2-NEXT:    shufps {{.*#+}} xmm9 = xmm9[0,0],xmm1[0,0]
-; SSE2-NEXT:    shufps {{.*#+}} xmm9 = xmm9[2,0],xmm1[2,3]
+; SSE2-NEXT:  LBB8_11: ## %cond.load17
+; SSE2-NEXT:    movss {{.*#+}} xmm8 = mem[0],zero,zero,zero
+; SSE2-NEXT:    shufps {{.*#+}} xmm8 = xmm8[0,0],xmm1[0,0]
+; SSE2-NEXT:    shufps {{.*#+}} xmm8 = xmm8[2,0],xmm1[2,3]
 ; SSE2-NEXT:    addq $4, %rsi
-; SSE2-NEXT:    movaps %xmm9, %xmm1
-; SSE2-NEXT:  LBB8_12: ## %else18
-; SSE2-NEXT:    xorps %xmm9, %xmm9
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm8
-; SSE2-NEXT:    pextrw $4, %xmm8, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    movaps %xmm8, %xmm1
+; SSE2-NEXT:    testb $64, %cl
 ; SSE2-NEXT:    je LBB8_14
-; SSE2-NEXT:  ## %bb.13: ## %cond.load21
-; SSE2-NEXT:    movss {{.*#+}} xmm9 = mem[0],zero,zero,zero
-; SSE2-NEXT:    shufps {{.*#+}} xmm9 = xmm9[0,0],xmm1[3,0]
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,1],xmm9[0,2]
+; SSE2-NEXT:  LBB8_13: ## %cond.load21
+; SSE2-NEXT:    movss {{.*#+}} xmm8 = mem[0],zero,zero,zero
+; SSE2-NEXT:    shufps {{.*#+}} xmm8 = xmm8[0,0],xmm1[3,0]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,1],xmm8[0,2]
 ; SSE2-NEXT:    addq $4, %rsi
-; SSE2-NEXT:  LBB8_14: ## %else22
-; SSE2-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm9
-; SSE2-NEXT:    pextrw $6, %xmm8, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testb $-128, %cl
 ; SSE2-NEXT:    je LBB8_16
-; SSE2-NEXT:  ## %bb.15: ## %cond.load25
+; SSE2-NEXT:  LBB8_15: ## %cond.load25
 ; SSE2-NEXT:    movss {{.*#+}} xmm8 = mem[0],zero,zero,zero
 ; SSE2-NEXT:    shufps {{.*#+}} xmm8 = xmm8[0,0],xmm1[2,0]
 ; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,1],xmm8[2,0]
 ; SSE2-NEXT:    addq $4, %rsi
-; SSE2-NEXT:  LBB8_16: ## %else26
-; SSE2-NEXT:    xorps %xmm8, %xmm8
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm8
-; SSE2-NEXT:    movd %xmm8, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $256, %ecx ## imm = 0x100
 ; SSE2-NEXT:    je LBB8_18
-; SSE2-NEXT:  ## %bb.17: ## %cond.load29
-; SSE2-NEXT:    movss {{.*#+}} xmm10 = mem[0],zero,zero,zero
-; SSE2-NEXT:    movss {{.*#+}} xmm2 = xmm10[0],xmm2[1,2,3]
+; SSE2-NEXT:  LBB8_17: ## %cond.load29
+; SSE2-NEXT:    movss {{.*#+}} xmm8 = mem[0],zero,zero,zero
+; SSE2-NEXT:    movss {{.*#+}} xmm2 = xmm8[0],xmm2[1,2,3]
 ; SSE2-NEXT:    addq $4, %rsi
-; SSE2-NEXT:  LBB8_18: ## %else30
-; SSE2-NEXT:    pextrw $2, %xmm8, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $512, %ecx ## imm = 0x200
 ; SSE2-NEXT:    je LBB8_20
-; SSE2-NEXT:  ## %bb.19: ## %cond.load33
+; SSE2-NEXT:  LBB8_19: ## %cond.load33
 ; SSE2-NEXT:    movss {{.*#+}} xmm8 = mem[0],zero,zero,zero
 ; SSE2-NEXT:    shufps {{.*#+}} xmm8 = xmm8[0,0],xmm2[0,0]
 ; SSE2-NEXT:    shufps {{.*#+}} xmm8 = xmm8[2,0],xmm2[2,3]
 ; SSE2-NEXT:    addq $4, %rsi
 ; SSE2-NEXT:    movaps %xmm8, %xmm2
-; SSE2-NEXT:  LBB8_20: ## %else34
-; SSE2-NEXT:    xorps %xmm8, %xmm8
-; SSE2-NEXT:    pcmpeqd %xmm8, %xmm9
-; SSE2-NEXT:    pextrw $4, %xmm9, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $1024, %ecx ## imm = 0x400
 ; SSE2-NEXT:    je LBB8_22
-; SSE2-NEXT:  ## %bb.21: ## %cond.load37
+; SSE2-NEXT:  LBB8_21: ## %cond.load37
 ; SSE2-NEXT:    movss {{.*#+}} xmm8 = mem[0],zero,zero,zero
 ; SSE2-NEXT:    shufps {{.*#+}} xmm8 = xmm8[0,0],xmm2[3,0]
 ; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,1],xmm8[0,2]
 ; SSE2-NEXT:    addq $4, %rsi
-; SSE2-NEXT:  LBB8_22: ## %else38
-; SSE2-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm8
-; SSE2-NEXT:    pextrw $6, %xmm9, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $2048, %ecx ## imm = 0x800
 ; SSE2-NEXT:    je LBB8_24
-; SSE2-NEXT:  ## %bb.23: ## %cond.load41
-; SSE2-NEXT:    movss {{.*#+}} xmm9 = mem[0],zero,zero,zero
-; SSE2-NEXT:    shufps {{.*#+}} xmm9 = xmm9[0,0],xmm2[2,0]
-; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,1],xmm9[2,0]
+; SSE2-NEXT:  LBB8_23: ## %cond.load41
+; SSE2-NEXT:    movss {{.*#+}} xmm8 = mem[0],zero,zero,zero
+; SSE2-NEXT:    shufps {{.*#+}} xmm8 = xmm8[0,0],xmm2[2,0]
+; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,1],xmm8[2,0]
 ; SSE2-NEXT:    addq $4, %rsi
-; SSE2-NEXT:  LBB8_24: ## %else42
-; SSE2-NEXT:    xorps %xmm9, %xmm9
-; SSE2-NEXT:    pcmpeqd %xmm8, %xmm9
-; SSE2-NEXT:    movd %xmm9, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $4096, %ecx ## imm = 0x1000
 ; SSE2-NEXT:    je LBB8_26
-; SSE2-NEXT:  ## %bb.25: ## %cond.load45
-; SSE2-NEXT:    movss {{.*#+}} xmm10 = mem[0],zero,zero,zero
-; SSE2-NEXT:    movss {{.*#+}} xmm3 = xmm10[0],xmm3[1,2,3]
+; SSE2-NEXT:  LBB8_25: ## %cond.load45
+; SSE2-NEXT:    movss {{.*#+}} xmm8 = mem[0],zero,zero,zero
+; SSE2-NEXT:    movss {{.*#+}} xmm3 = xmm8[0],xmm3[1,2,3]
 ; SSE2-NEXT:    addq $4, %rsi
-; SSE2-NEXT:  LBB8_26: ## %else46
-; SSE2-NEXT:    pextrw $2, %xmm9, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $8192, %ecx ## imm = 0x2000
 ; SSE2-NEXT:    je LBB8_28
-; SSE2-NEXT:  ## %bb.27: ## %cond.load49
-; SSE2-NEXT:    movss {{.*#+}} xmm9 = mem[0],zero,zero,zero
-; SSE2-NEXT:    shufps {{.*#+}} xmm9 = xmm9[0,0],xmm3[0,0]
-; SSE2-NEXT:    shufps {{.*#+}} xmm9 = xmm9[2,0],xmm3[2,3]
+; SSE2-NEXT:  LBB8_27: ## %cond.load49
+; SSE2-NEXT:    movss {{.*#+}} xmm8 = mem[0],zero,zero,zero
+; SSE2-NEXT:    shufps {{.*#+}} xmm8 = xmm8[0,0],xmm3[0,0]
+; SSE2-NEXT:    shufps {{.*#+}} xmm8 = xmm8[2,0],xmm3[2,3]
 ; SSE2-NEXT:    addq $4, %rsi
-; SSE2-NEXT:    movaps %xmm9, %xmm3
-; SSE2-NEXT:  LBB8_28: ## %else50
-; SSE2-NEXT:    xorps %xmm9, %xmm9
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm8
-; SSE2-NEXT:    pextrw $4, %xmm8, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    movaps %xmm8, %xmm3
+; SSE2-NEXT:    testl $16384, %ecx ## imm = 0x4000
 ; SSE2-NEXT:    je LBB8_30
-; SSE2-NEXT:  ## %bb.29: ## %cond.load53
-; SSE2-NEXT:    movss {{.*#+}} xmm9 = mem[0],zero,zero,zero
-; SSE2-NEXT:    shufps {{.*#+}} xmm9 = xmm9[0,0],xmm3[3,0]
-; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,1],xmm9[0,2]
+; SSE2-NEXT:  LBB8_29: ## %cond.load53
+; SSE2-NEXT:    movss {{.*#+}} xmm8 = mem[0],zero,zero,zero
+; SSE2-NEXT:    shufps {{.*#+}} xmm8 = xmm8[0,0],xmm3[3,0]
+; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,1],xmm8[0,2]
 ; SSE2-NEXT:    addq $4, %rsi
-; SSE2-NEXT:  LBB8_30: ## %else54
-; SSE2-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm9
-; SSE2-NEXT:    pextrw $6, %xmm8, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $32768, %ecx ## imm = 0x8000
 ; SSE2-NEXT:    je LBB8_32
-; SSE2-NEXT:  ## %bb.31: ## %cond.load57
+; SSE2-NEXT:  LBB8_31: ## %cond.load57
 ; SSE2-NEXT:    movss {{.*#+}} xmm8 = mem[0],zero,zero,zero
 ; SSE2-NEXT:    shufps {{.*#+}} xmm8 = xmm8[0,0],xmm3[2,0]
 ; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,1],xmm8[2,0]
 ; SSE2-NEXT:    addq $4, %rsi
-; SSE2-NEXT:  LBB8_32: ## %else58
-; SSE2-NEXT:    xorps %xmm8, %xmm8
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm8
-; SSE2-NEXT:    movd %xmm8, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $65536, %ecx ## imm = 0x10000
 ; SSE2-NEXT:    je LBB8_34
-; SSE2-NEXT:  ## %bb.33: ## %cond.load61
-; SSE2-NEXT:    movss {{.*#+}} xmm10 = mem[0],zero,zero,zero
-; SSE2-NEXT:    movss {{.*#+}} xmm4 = xmm10[0],xmm4[1,2,3]
+; SSE2-NEXT:  LBB8_33: ## %cond.load61
+; SSE2-NEXT:    movss {{.*#+}} xmm8 = mem[0],zero,zero,zero
+; SSE2-NEXT:    movss {{.*#+}} xmm4 = xmm8[0],xmm4[1,2,3]
 ; SSE2-NEXT:    addq $4, %rsi
-; SSE2-NEXT:  LBB8_34: ## %else62
-; SSE2-NEXT:    pextrw $2, %xmm8, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $131072, %ecx ## imm = 0x20000
 ; SSE2-NEXT:    je LBB8_36
-; SSE2-NEXT:  ## %bb.35: ## %cond.load65
+; SSE2-NEXT:  LBB8_35: ## %cond.load65
 ; SSE2-NEXT:    movss {{.*#+}} xmm8 = mem[0],zero,zero,zero
 ; SSE2-NEXT:    shufps {{.*#+}} xmm8 = xmm8[0,0],xmm4[0,0]
 ; SSE2-NEXT:    shufps {{.*#+}} xmm8 = xmm8[2,0],xmm4[2,3]
 ; SSE2-NEXT:    addq $4, %rsi
 ; SSE2-NEXT:    movaps %xmm8, %xmm4
-; SSE2-NEXT:  LBB8_36: ## %else66
-; SSE2-NEXT:    xorps %xmm8, %xmm8
-; SSE2-NEXT:    pcmpeqd %xmm8, %xmm9
-; SSE2-NEXT:    pextrw $4, %xmm9, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $262144, %ecx ## imm = 0x40000
 ; SSE2-NEXT:    je LBB8_38
-; SSE2-NEXT:  ## %bb.37: ## %cond.load69
+; SSE2-NEXT:  LBB8_37: ## %cond.load69
 ; SSE2-NEXT:    movss {{.*#+}} xmm8 = mem[0],zero,zero,zero
 ; SSE2-NEXT:    shufps {{.*#+}} xmm8 = xmm8[0,0],xmm4[3,0]
 ; SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[0,1],xmm8[0,2]
 ; SSE2-NEXT:    addq $4, %rsi
-; SSE2-NEXT:  LBB8_38: ## %else70
-; SSE2-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm8
-; SSE2-NEXT:    pextrw $6, %xmm9, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $524288, %ecx ## imm = 0x80000
 ; SSE2-NEXT:    je LBB8_40
-; SSE2-NEXT:  ## %bb.39: ## %cond.load73
-; SSE2-NEXT:    movss {{.*#+}} xmm9 = mem[0],zero,zero,zero
-; SSE2-NEXT:    shufps {{.*#+}} xmm9 = xmm9[0,0],xmm4[2,0]
-; SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[0,1],xmm9[2,0]
+; SSE2-NEXT:  LBB8_39: ## %cond.load73
+; SSE2-NEXT:    movss {{.*#+}} xmm8 = mem[0],zero,zero,zero
+; SSE2-NEXT:    shufps {{.*#+}} xmm8 = xmm8[0,0],xmm4[2,0]
+; SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[0,1],xmm8[2,0]
 ; SSE2-NEXT:    addq $4, %rsi
-; SSE2-NEXT:  LBB8_40: ## %else74
-; SSE2-NEXT:    xorps %xmm9, %xmm9
-; SSE2-NEXT:    pcmpeqd %xmm8, %xmm9
-; SSE2-NEXT:    movd %xmm9, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $1048576, %ecx ## imm = 0x100000
 ; SSE2-NEXT:    je LBB8_42
-; SSE2-NEXT:  ## %bb.41: ## %cond.load77
-; SSE2-NEXT:    movss {{.*#+}} xmm10 = mem[0],zero,zero,zero
-; SSE2-NEXT:    movss {{.*#+}} xmm5 = xmm10[0],xmm5[1,2,3]
+; SSE2-NEXT:  LBB8_41: ## %cond.load77
+; SSE2-NEXT:    movss {{.*#+}} xmm8 = mem[0],zero,zero,zero
+; SSE2-NEXT:    movss {{.*#+}} xmm5 = xmm8[0],xmm5[1,2,3]
 ; SSE2-NEXT:    addq $4, %rsi
-; SSE2-NEXT:  LBB8_42: ## %else78
-; SSE2-NEXT:    pextrw $2, %xmm9, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $2097152, %ecx ## imm = 0x200000
 ; SSE2-NEXT:    je LBB8_44
-; SSE2-NEXT:  ## %bb.43: ## %cond.load81
-; SSE2-NEXT:    movss {{.*#+}} xmm9 = mem[0],zero,zero,zero
-; SSE2-NEXT:    shufps {{.*#+}} xmm9 = xmm9[0,0],xmm5[0,0]
-; SSE2-NEXT:    shufps {{.*#+}} xmm9 = xmm9[2,0],xmm5[2,3]
+; SSE2-NEXT:  LBB8_43: ## %cond.load81
+; SSE2-NEXT:    movss {{.*#+}} xmm8 = mem[0],zero,zero,zero
+; SSE2-NEXT:    shufps {{.*#+}} xmm8 = xmm8[0,0],xmm5[0,0]
+; SSE2-NEXT:    shufps {{.*#+}} xmm8 = xmm8[2,0],xmm5[2,3]
 ; SSE2-NEXT:    addq $4, %rsi
-; SSE2-NEXT:    movaps %xmm9, %xmm5
-; SSE2-NEXT:  LBB8_44: ## %else82
-; SSE2-NEXT:    xorps %xmm9, %xmm9
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm8
-; SSE2-NEXT:    pextrw $4, %xmm8, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    movaps %xmm8, %xmm5
+; SSE2-NEXT:    testl $4194304, %ecx ## imm = 0x400000
 ; SSE2-NEXT:    je LBB8_46
-; SSE2-NEXT:  ## %bb.45: ## %cond.load85
-; SSE2-NEXT:    movss {{.*#+}} xmm9 = mem[0],zero,zero,zero
-; SSE2-NEXT:    shufps {{.*#+}} xmm9 = xmm9[0,0],xmm5[3,0]
-; SSE2-NEXT:    shufps {{.*#+}} xmm5 = xmm5[0,1],xmm9[0,2]
+; SSE2-NEXT:  LBB8_45: ## %cond.load85
+; SSE2-NEXT:    movss {{.*#+}} xmm8 = mem[0],zero,zero,zero
+; SSE2-NEXT:    shufps {{.*#+}} xmm8 = xmm8[0,0],xmm5[3,0]
+; SSE2-NEXT:    shufps {{.*#+}} xmm5 = xmm5[0,1],xmm8[0,2]
 ; SSE2-NEXT:    addq $4, %rsi
-; SSE2-NEXT:  LBB8_46: ## %else86
-; SSE2-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm9
-; SSE2-NEXT:    pextrw $6, %xmm8, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $8388608, %ecx ## imm = 0x800000
 ; SSE2-NEXT:    je LBB8_48
-; SSE2-NEXT:  ## %bb.47: ## %cond.load89
+; SSE2-NEXT:  LBB8_47: ## %cond.load89
 ; SSE2-NEXT:    movss {{.*#+}} xmm8 = mem[0],zero,zero,zero
 ; SSE2-NEXT:    shufps {{.*#+}} xmm8 = xmm8[0,0],xmm5[2,0]
 ; SSE2-NEXT:    shufps {{.*#+}} xmm5 = xmm5[0,1],xmm8[2,0]
 ; SSE2-NEXT:    addq $4, %rsi
-; SSE2-NEXT:  LBB8_48: ## %else90
-; SSE2-NEXT:    xorps %xmm8, %xmm8
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm8
-; SSE2-NEXT:    movd %xmm8, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $16777216, %ecx ## imm = 0x1000000
 ; SSE2-NEXT:    je LBB8_50
-; SSE2-NEXT:  ## %bb.49: ## %cond.load93
-; SSE2-NEXT:    movss {{.*#+}} xmm10 = mem[0],zero,zero,zero
-; SSE2-NEXT:    movss {{.*#+}} xmm6 = xmm10[0],xmm6[1,2,3]
+; SSE2-NEXT:  LBB8_49: ## %cond.load93
+; SSE2-NEXT:    movss {{.*#+}} xmm8 = mem[0],zero,zero,zero
+; SSE2-NEXT:    movss {{.*#+}} xmm6 = xmm8[0],xmm6[1,2,3]
 ; SSE2-NEXT:    addq $4, %rsi
-; SSE2-NEXT:  LBB8_50: ## %else94
-; SSE2-NEXT:    pextrw $2, %xmm8, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $33554432, %ecx ## imm = 0x2000000
 ; SSE2-NEXT:    je LBB8_52
-; SSE2-NEXT:  ## %bb.51: ## %cond.load97
+; SSE2-NEXT:  LBB8_51: ## %cond.load97
 ; SSE2-NEXT:    movss {{.*#+}} xmm8 = mem[0],zero,zero,zero
 ; SSE2-NEXT:    shufps {{.*#+}} xmm8 = xmm8[0,0],xmm6[0,0]
 ; SSE2-NEXT:    shufps {{.*#+}} xmm8 = xmm8[2,0],xmm6[2,3]
 ; SSE2-NEXT:    addq $4, %rsi
 ; SSE2-NEXT:    movaps %xmm8, %xmm6
-; SSE2-NEXT:  LBB8_52: ## %else98
-; SSE2-NEXT:    xorps %xmm8, %xmm8
-; SSE2-NEXT:    pcmpeqd %xmm8, %xmm9
-; SSE2-NEXT:    pextrw $4, %xmm9, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $67108864, %ecx ## imm = 0x4000000
 ; SSE2-NEXT:    je LBB8_54
-; SSE2-NEXT:  ## %bb.53: ## %cond.load101
+; SSE2-NEXT:  LBB8_53: ## %cond.load101
 ; SSE2-NEXT:    movss {{.*#+}} xmm8 = mem[0],zero,zero,zero
 ; SSE2-NEXT:    shufps {{.*#+}} xmm8 = xmm8[0,0],xmm6[3,0]
 ; SSE2-NEXT:    shufps {{.*#+}} xmm6 = xmm6[0,1],xmm8[0,2]
 ; SSE2-NEXT:    addq $4, %rsi
-; SSE2-NEXT:  LBB8_54: ## %else102
-; SSE2-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm8
-; SSE2-NEXT:    pextrw $6, %xmm9, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $134217728, %ecx ## imm = 0x8000000
 ; SSE2-NEXT:    je LBB8_56
-; SSE2-NEXT:  ## %bb.55: ## %cond.load105
-; SSE2-NEXT:    movss {{.*#+}} xmm9 = mem[0],zero,zero,zero
-; SSE2-NEXT:    shufps {{.*#+}} xmm9 = xmm9[0,0],xmm6[2,0]
-; SSE2-NEXT:    shufps {{.*#+}} xmm6 = xmm6[0,1],xmm9[2,0]
+; SSE2-NEXT:  LBB8_55: ## %cond.load105
+; SSE2-NEXT:    movss {{.*#+}} xmm8 = mem[0],zero,zero,zero
+; SSE2-NEXT:    shufps {{.*#+}} xmm8 = xmm8[0,0],xmm6[2,0]
+; SSE2-NEXT:    shufps {{.*#+}} xmm6 = xmm6[0,1],xmm8[2,0]
 ; SSE2-NEXT:    addq $4, %rsi
-; SSE2-NEXT:  LBB8_56: ## %else106
-; SSE2-NEXT:    xorps %xmm9, %xmm9
-; SSE2-NEXT:    pcmpeqd %xmm8, %xmm9
-; SSE2-NEXT:    movd %xmm9, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $268435456, %ecx ## imm = 0x10000000
 ; SSE2-NEXT:    je LBB8_58
-; SSE2-NEXT:  ## %bb.57: ## %cond.load109
-; SSE2-NEXT:    movss {{.*#+}} xmm10 = mem[0],zero,zero,zero
-; SSE2-NEXT:    movss {{.*#+}} xmm7 = xmm10[0],xmm7[1,2,3]
-; SSE2-NEXT:    addq $4, %rsi
-; SSE2-NEXT:  LBB8_58: ## %else110
-; SSE2-NEXT:    pextrw $2, %xmm9, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    je LBB8_60
-; SSE2-NEXT:  ## %bb.59: ## %cond.load113
-; SSE2-NEXT:    movss {{.*#+}} xmm9 = mem[0],zero,zero,zero
-; SSE2-NEXT:    shufps {{.*#+}} xmm9 = xmm9[0,0],xmm7[0,0]
-; SSE2-NEXT:    shufps {{.*#+}} xmm9 = xmm9[2,0],xmm7[2,3]
-; SSE2-NEXT:    addq $4, %rsi
-; SSE2-NEXT:    movaps %xmm9, %xmm7
-; SSE2-NEXT:  LBB8_60: ## %else114
-; SSE2-NEXT:    xorps %xmm9, %xmm9
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm8
-; SSE2-NEXT:    pextrw $4, %xmm8, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    je LBB8_62
-; SSE2-NEXT:  ## %bb.61: ## %cond.load117
-; SSE2-NEXT:    movss {{.*#+}} xmm9 = mem[0],zero,zero,zero
-; SSE2-NEXT:    shufps {{.*#+}} xmm9 = xmm9[0,0],xmm7[3,0]
-; SSE2-NEXT:    shufps {{.*#+}} xmm7 = xmm7[0,1],xmm9[0,2]
-; SSE2-NEXT:    addq $4, %rsi
-; SSE2-NEXT:  LBB8_62: ## %else118
-; SSE2-NEXT:    pextrw $6, %xmm8, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    je LBB8_64
-; SSE2-NEXT:  ## %bb.63: ## %cond.load121
+; SSE2-NEXT:  LBB8_57: ## %cond.load109
 ; SSE2-NEXT:    movss {{.*#+}} xmm8 = mem[0],zero,zero,zero
-; SSE2-NEXT:    shufps {{.*#+}} xmm8 = xmm8[0,0],xmm7[2,0]
-; SSE2-NEXT:    shufps {{.*#+}} xmm7 = xmm7[0,1],xmm8[2,0]
-; SSE2-NEXT:  LBB8_64: ## %else122
-; SSE2-NEXT:    movaps %xmm0, (%rax)
-; SSE2-NEXT:    movaps %xmm1, 16(%rax)
-; SSE2-NEXT:    movaps %xmm2, 32(%rax)
-; SSE2-NEXT:    movaps %xmm3, 48(%rax)
-; SSE2-NEXT:    movaps %xmm4, 64(%rax)
-; SSE2-NEXT:    movaps %xmm5, 80(%rax)
-; SSE2-NEXT:    movaps %xmm6, 96(%rax)
-; SSE2-NEXT:    movaps %xmm7, 112(%rax)
-; SSE2-NEXT:    retq
+; SSE2-NEXT:    movss {{.*#+}} xmm7 = xmm8[0],xmm7[1,2,3]
+; SSE2-NEXT:    addq $4, %rsi
+; SSE2-NEXT:    testl $536870912, %ecx ## imm = 0x20000000
+; SSE2-NEXT:    je LBB8_60
+; SSE2-NEXT:  LBB8_59: ## %cond.load113
+; SSE2-NEXT:    movss {{.*#+}} xmm8 = mem[0],zero,zero,zero
+; SSE2-NEXT:    shufps {{.*#+}} xmm8 = xmm8[0,0],xmm7[0,0]
+; SSE2-NEXT:    shufps {{.*#+}} xmm8 = xmm8[2,0],xmm7[2,3]
+; SSE2-NEXT:    addq $4, %rsi
+; SSE2-NEXT:    movaps %xmm8, %xmm7
+; SSE2-NEXT:    testl $1073741824, %ecx ## imm = 0x40000000
+; SSE2-NEXT:    je LBB8_62
+; SSE2-NEXT:  LBB8_61: ## %cond.load117
+; SSE2-NEXT:    movss {{.*#+}} xmm8 = mem[0],zero,zero,zero
+; SSE2-NEXT:    shufps {{.*#+}} xmm8 = xmm8[0,0],xmm7[3,0]
+; SSE2-NEXT:    shufps {{.*#+}} xmm7 = xmm7[0,1],xmm8[0,2]
+; SSE2-NEXT:    addq $4, %rsi
+; SSE2-NEXT:    testl $-2147483648, %ecx ## imm = 0x80000000
+; SSE2-NEXT:    jne LBB8_63
+; SSE2-NEXT:    jmp LBB8_64
 ;
 ; SSE42-LABEL: expandload_v32f32_v32i32:
 ; SSE42:       ## %bb.0:
 ; SSE42-NEXT:    movq %rdi, %rax
-; SSE42-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm9
 ; SSE42-NEXT:    pxor %xmm8, %xmm8
-; SSE42-NEXT:    pcmpeqd %xmm9, %xmm8
-; SSE42-NEXT:    pextrb $0, %xmm8, %ecx
+; SSE42-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm9
+; SSE42-NEXT:    pcmpeqd %xmm8, %xmm9
+; SSE42-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm10
+; SSE42-NEXT:    pcmpeqd %xmm8, %xmm10
+; SSE42-NEXT:    packssdw %xmm9, %xmm10
+; SSE42-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm9
+; SSE42-NEXT:    pcmpeqd %xmm8, %xmm9
+; SSE42-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm11
+; SSE42-NEXT:    pcmpeqd %xmm8, %xmm11
+; SSE42-NEXT:    packssdw %xmm9, %xmm11
+; SSE42-NEXT:    packsswb %xmm10, %xmm11
+; SSE42-NEXT:    pmovmskb %xmm11, %edx
+; SSE42-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm9
+; SSE42-NEXT:    pcmpeqd %xmm8, %xmm9
+; SSE42-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm10
+; SSE42-NEXT:    pcmpeqd %xmm8, %xmm10
+; SSE42-NEXT:    packssdw %xmm9, %xmm10
+; SSE42-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm9
+; SSE42-NEXT:    pcmpeqd %xmm8, %xmm9
+; SSE42-NEXT:    pcmpeqd {{[0-9]+}}(%rsp), %xmm8
+; SSE42-NEXT:    packssdw %xmm9, %xmm8
+; SSE42-NEXT:    packsswb %xmm10, %xmm8
+; SSE42-NEXT:    pmovmskb %xmm8, %ecx
+; SSE42-NEXT:    shll $16, %ecx
+; SSE42-NEXT:    orl %edx, %ecx
 ; SSE42-NEXT:    testb $1, %cl
-; SSE42-NEXT:    je LBB8_2
-; SSE42-NEXT:  ## %bb.1: ## %cond.load
-; SSE42-NEXT:    movss {{.*#+}} xmm10 = mem[0],zero,zero,zero
-; SSE42-NEXT:    blendps {{.*#+}} xmm0 = xmm10[0],xmm0[1,2,3]
+; SSE42-NEXT:    jne LBB8_1
+; SSE42-NEXT:  ## %bb.2: ## %else
+; SSE42-NEXT:    testb $2, %cl
+; SSE42-NEXT:    jne LBB8_3
+; SSE42-NEXT:  LBB8_4: ## %else2
+; SSE42-NEXT:    testb $4, %cl
+; SSE42-NEXT:    jne LBB8_5
+; SSE42-NEXT:  LBB8_6: ## %else6
+; SSE42-NEXT:    testb $8, %cl
+; SSE42-NEXT:    jne LBB8_7
+; SSE42-NEXT:  LBB8_8: ## %else10
+; SSE42-NEXT:    testb $16, %cl
+; SSE42-NEXT:    jne LBB8_9
+; SSE42-NEXT:  LBB8_10: ## %else14
+; SSE42-NEXT:    testb $32, %cl
+; SSE42-NEXT:    jne LBB8_11
+; SSE42-NEXT:  LBB8_12: ## %else18
+; SSE42-NEXT:    testb $64, %cl
+; SSE42-NEXT:    jne LBB8_13
+; SSE42-NEXT:  LBB8_14: ## %else22
+; SSE42-NEXT:    testb $-128, %cl
+; SSE42-NEXT:    jne LBB8_15
+; SSE42-NEXT:  LBB8_16: ## %else26
+; SSE42-NEXT:    testl $256, %ecx ## imm = 0x100
+; SSE42-NEXT:    jne LBB8_17
+; SSE42-NEXT:  LBB8_18: ## %else30
+; SSE42-NEXT:    testl $512, %ecx ## imm = 0x200
+; SSE42-NEXT:    jne LBB8_19
+; SSE42-NEXT:  LBB8_20: ## %else34
+; SSE42-NEXT:    testl $1024, %ecx ## imm = 0x400
+; SSE42-NEXT:    jne LBB8_21
+; SSE42-NEXT:  LBB8_22: ## %else38
+; SSE42-NEXT:    testl $2048, %ecx ## imm = 0x800
+; SSE42-NEXT:    jne LBB8_23
+; SSE42-NEXT:  LBB8_24: ## %else42
+; SSE42-NEXT:    testl $4096, %ecx ## imm = 0x1000
+; SSE42-NEXT:    jne LBB8_25
+; SSE42-NEXT:  LBB8_26: ## %else46
+; SSE42-NEXT:    testl $8192, %ecx ## imm = 0x2000
+; SSE42-NEXT:    jne LBB8_27
+; SSE42-NEXT:  LBB8_28: ## %else50
+; SSE42-NEXT:    testl $16384, %ecx ## imm = 0x4000
+; SSE42-NEXT:    jne LBB8_29
+; SSE42-NEXT:  LBB8_30: ## %else54
+; SSE42-NEXT:    testl $32768, %ecx ## imm = 0x8000
+; SSE42-NEXT:    jne LBB8_31
+; SSE42-NEXT:  LBB8_32: ## %else58
+; SSE42-NEXT:    testl $65536, %ecx ## imm = 0x10000
+; SSE42-NEXT:    jne LBB8_33
+; SSE42-NEXT:  LBB8_34: ## %else62
+; SSE42-NEXT:    testl $131072, %ecx ## imm = 0x20000
+; SSE42-NEXT:    jne LBB8_35
+; SSE42-NEXT:  LBB8_36: ## %else66
+; SSE42-NEXT:    testl $262144, %ecx ## imm = 0x40000
+; SSE42-NEXT:    jne LBB8_37
+; SSE42-NEXT:  LBB8_38: ## %else70
+; SSE42-NEXT:    testl $524288, %ecx ## imm = 0x80000
+; SSE42-NEXT:    jne LBB8_39
+; SSE42-NEXT:  LBB8_40: ## %else74
+; SSE42-NEXT:    testl $1048576, %ecx ## imm = 0x100000
+; SSE42-NEXT:    jne LBB8_41
+; SSE42-NEXT:  LBB8_42: ## %else78
+; SSE42-NEXT:    testl $2097152, %ecx ## imm = 0x200000
+; SSE42-NEXT:    jne LBB8_43
+; SSE42-NEXT:  LBB8_44: ## %else82
+; SSE42-NEXT:    testl $4194304, %ecx ## imm = 0x400000
+; SSE42-NEXT:    jne LBB8_45
+; SSE42-NEXT:  LBB8_46: ## %else86
+; SSE42-NEXT:    testl $8388608, %ecx ## imm = 0x800000
+; SSE42-NEXT:    jne LBB8_47
+; SSE42-NEXT:  LBB8_48: ## %else90
+; SSE42-NEXT:    testl $16777216, %ecx ## imm = 0x1000000
+; SSE42-NEXT:    jne LBB8_49
+; SSE42-NEXT:  LBB8_50: ## %else94
+; SSE42-NEXT:    testl $33554432, %ecx ## imm = 0x2000000
+; SSE42-NEXT:    jne LBB8_51
+; SSE42-NEXT:  LBB8_52: ## %else98
+; SSE42-NEXT:    testl $67108864, %ecx ## imm = 0x4000000
+; SSE42-NEXT:    jne LBB8_53
+; SSE42-NEXT:  LBB8_54: ## %else102
+; SSE42-NEXT:    testl $134217728, %ecx ## imm = 0x8000000
+; SSE42-NEXT:    jne LBB8_55
+; SSE42-NEXT:  LBB8_56: ## %else106
+; SSE42-NEXT:    testl $268435456, %ecx ## imm = 0x10000000
+; SSE42-NEXT:    jne LBB8_57
+; SSE42-NEXT:  LBB8_58: ## %else110
+; SSE42-NEXT:    testl $536870912, %ecx ## imm = 0x20000000
+; SSE42-NEXT:    jne LBB8_59
+; SSE42-NEXT:  LBB8_60: ## %else114
+; SSE42-NEXT:    testl $1073741824, %ecx ## imm = 0x40000000
+; SSE42-NEXT:    jne LBB8_61
+; SSE42-NEXT:  LBB8_62: ## %else118
+; SSE42-NEXT:    testl $-2147483648, %ecx ## imm = 0x80000000
+; SSE42-NEXT:    je LBB8_64
+; SSE42-NEXT:  LBB8_63: ## %cond.load121
+; SSE42-NEXT:    insertps {{.*#+}} xmm7 = xmm7[0,1,2],mem[0]
+; SSE42-NEXT:  LBB8_64: ## %else122
+; SSE42-NEXT:    movaps %xmm0, (%rax)
+; SSE42-NEXT:    movaps %xmm1, 16(%rax)
+; SSE42-NEXT:    movaps %xmm2, 32(%rax)
+; SSE42-NEXT:    movaps %xmm3, 48(%rax)
+; SSE42-NEXT:    movaps %xmm4, 64(%rax)
+; SSE42-NEXT:    movaps %xmm5, 80(%rax)
+; SSE42-NEXT:    movaps %xmm6, 96(%rax)
+; SSE42-NEXT:    movaps %xmm7, 112(%rax)
+; SSE42-NEXT:    retq
+; SSE42-NEXT:  LBB8_1: ## %cond.load
+; SSE42-NEXT:    movd {{.*#+}} xmm8 = mem[0],zero,zero,zero
+; SSE42-NEXT:    pblendw {{.*#+}} xmm0 = xmm8[0,1],xmm0[2,3,4,5,6,7]
 ; SSE42-NEXT:    addq $4, %rsi
-; SSE42-NEXT:  LBB8_2: ## %else
-; SSE42-NEXT:    pextrb $4, %xmm8, %ecx
-; SSE42-NEXT:    testb $1, %cl
+; SSE42-NEXT:    testb $2, %cl
 ; SSE42-NEXT:    je LBB8_4
-; SSE42-NEXT:  ## %bb.3: ## %cond.load1
+; SSE42-NEXT:  LBB8_3: ## %cond.load1
 ; SSE42-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[2,3]
 ; SSE42-NEXT:    addq $4, %rsi
-; SSE42-NEXT:  LBB8_4: ## %else2
-; SSE42-NEXT:    pxor %xmm8, %xmm8
-; SSE42-NEXT:    pcmpeqd %xmm8, %xmm9
-; SSE42-NEXT:    pextrb $8, %xmm9, %ecx
-; SSE42-NEXT:    testb $1, %cl
+; SSE42-NEXT:    testb $4, %cl
 ; SSE42-NEXT:    je LBB8_6
-; SSE42-NEXT:  ## %bb.5: ## %cond.load5
+; SSE42-NEXT:  LBB8_5: ## %cond.load5
 ; SSE42-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0,1],mem[0],xmm0[3]
 ; SSE42-NEXT:    addq $4, %rsi
-; SSE42-NEXT:  LBB8_6: ## %else6
-; SSE42-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm8
-; SSE42-NEXT:    pextrb $12, %xmm9, %ecx
-; SSE42-NEXT:    testb $1, %cl
+; SSE42-NEXT:    testb $8, %cl
 ; SSE42-NEXT:    je LBB8_8
-; SSE42-NEXT:  ## %bb.7: ## %cond.load9
+; SSE42-NEXT:  LBB8_7: ## %cond.load9
 ; SSE42-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0,1,2],mem[0]
 ; SSE42-NEXT:    addq $4, %rsi
-; SSE42-NEXT:  LBB8_8: ## %else10
-; SSE42-NEXT:    pxor %xmm9, %xmm9
-; SSE42-NEXT:    pcmpeqd %xmm8, %xmm9
-; SSE42-NEXT:    pextrb $0, %xmm9, %ecx
-; SSE42-NEXT:    testb $1, %cl
+; SSE42-NEXT:    testb $16, %cl
 ; SSE42-NEXT:    je LBB8_10
-; SSE42-NEXT:  ## %bb.9: ## %cond.load13
-; SSE42-NEXT:    movss {{.*#+}} xmm10 = mem[0],zero,zero,zero
-; SSE42-NEXT:    blendps {{.*#+}} xmm1 = xmm10[0],xmm1[1,2,3]
+; SSE42-NEXT:  LBB8_9: ## %cond.load13
+; SSE42-NEXT:    movd {{.*#+}} xmm8 = mem[0],zero,zero,zero
+; SSE42-NEXT:    pblendw {{.*#+}} xmm1 = xmm8[0,1],xmm1[2,3,4,5,6,7]
 ; SSE42-NEXT:    addq $4, %rsi
-; SSE42-NEXT:  LBB8_10: ## %else14
-; SSE42-NEXT:    pextrb $4, %xmm9, %ecx
-; SSE42-NEXT:    testb $1, %cl
+; SSE42-NEXT:    testb $32, %cl
 ; SSE42-NEXT:    je LBB8_12
-; SSE42-NEXT:  ## %bb.11: ## %cond.load17
+; SSE42-NEXT:  LBB8_11: ## %cond.load17
 ; SSE42-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[2,3]
 ; SSE42-NEXT:    addq $4, %rsi
-; SSE42-NEXT:  LBB8_12: ## %else18
-; SSE42-NEXT:    pxor %xmm9, %xmm9
-; SSE42-NEXT:    pcmpeqd %xmm9, %xmm8
-; SSE42-NEXT:    pextrb $8, %xmm8, %ecx
-; SSE42-NEXT:    testb $1, %cl
+; SSE42-NEXT:    testb $64, %cl
 ; SSE42-NEXT:    je LBB8_14
-; SSE42-NEXT:  ## %bb.13: ## %cond.load21
+; SSE42-NEXT:  LBB8_13: ## %cond.load21
 ; SSE42-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0,1],mem[0],xmm1[3]
 ; SSE42-NEXT:    addq $4, %rsi
-; SSE42-NEXT:  LBB8_14: ## %else22
-; SSE42-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm9
-; SSE42-NEXT:    pextrb $12, %xmm8, %ecx
-; SSE42-NEXT:    testb $1, %cl
+; SSE42-NEXT:    testb $-128, %cl
 ; SSE42-NEXT:    je LBB8_16
-; SSE42-NEXT:  ## %bb.15: ## %cond.load25
+; SSE42-NEXT:  LBB8_15: ## %cond.load25
 ; SSE42-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0,1,2],mem[0]
 ; SSE42-NEXT:    addq $4, %rsi
-; SSE42-NEXT:  LBB8_16: ## %else26
-; SSE42-NEXT:    pxor %xmm8, %xmm8
-; SSE42-NEXT:    pcmpeqd %xmm9, %xmm8
-; SSE42-NEXT:    pextrb $0, %xmm8, %ecx
-; SSE42-NEXT:    testb $1, %cl
+; SSE42-NEXT:    testl $256, %ecx ## imm = 0x100
 ; SSE42-NEXT:    je LBB8_18
-; SSE42-NEXT:  ## %bb.17: ## %cond.load29
-; SSE42-NEXT:    movss {{.*#+}} xmm10 = mem[0],zero,zero,zero
-; SSE42-NEXT:    blendps {{.*#+}} xmm2 = xmm10[0],xmm2[1,2,3]
+; SSE42-NEXT:  LBB8_17: ## %cond.load29
+; SSE42-NEXT:    movd {{.*#+}} xmm8 = mem[0],zero,zero,zero
+; SSE42-NEXT:    pblendw {{.*#+}} xmm2 = xmm8[0,1],xmm2[2,3,4,5,6,7]
 ; SSE42-NEXT:    addq $4, %rsi
-; SSE42-NEXT:  LBB8_18: ## %else30
-; SSE42-NEXT:    pextrb $4, %xmm8, %ecx
-; SSE42-NEXT:    testb $1, %cl
+; SSE42-NEXT:    testl $512, %ecx ## imm = 0x200
 ; SSE42-NEXT:    je LBB8_20
-; SSE42-NEXT:  ## %bb.19: ## %cond.load33
+; SSE42-NEXT:  LBB8_19: ## %cond.load33
 ; SSE42-NEXT:    insertps {{.*#+}} xmm2 = xmm2[0],mem[0],xmm2[2,3]
 ; SSE42-NEXT:    addq $4, %rsi
-; SSE42-NEXT:  LBB8_20: ## %else34
-; SSE42-NEXT:    pxor %xmm8, %xmm8
-; SSE42-NEXT:    pcmpeqd %xmm8, %xmm9
-; SSE42-NEXT:    pextrb $8, %xmm9, %ecx
-; SSE42-NEXT:    testb $1, %cl
+; SSE42-NEXT:    testl $1024, %ecx ## imm = 0x400
 ; SSE42-NEXT:    je LBB8_22
-; SSE42-NEXT:  ## %bb.21: ## %cond.load37
+; SSE42-NEXT:  LBB8_21: ## %cond.load37
 ; SSE42-NEXT:    insertps {{.*#+}} xmm2 = xmm2[0,1],mem[0],xmm2[3]
 ; SSE42-NEXT:    addq $4, %rsi
-; SSE42-NEXT:  LBB8_22: ## %else38
-; SSE42-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm8
-; SSE42-NEXT:    pextrb $12, %xmm9, %ecx
-; SSE42-NEXT:    testb $1, %cl
+; SSE42-NEXT:    testl $2048, %ecx ## imm = 0x800
 ; SSE42-NEXT:    je LBB8_24
-; SSE42-NEXT:  ## %bb.23: ## %cond.load41
+; SSE42-NEXT:  LBB8_23: ## %cond.load41
 ; SSE42-NEXT:    insertps {{.*#+}} xmm2 = xmm2[0,1,2],mem[0]
 ; SSE42-NEXT:    addq $4, %rsi
-; SSE42-NEXT:  LBB8_24: ## %else42
-; SSE42-NEXT:    pxor %xmm9, %xmm9
-; SSE42-NEXT:    pcmpeqd %xmm8, %xmm9
-; SSE42-NEXT:    pextrb $0, %xmm9, %ecx
-; SSE42-NEXT:    testb $1, %cl
+; SSE42-NEXT:    testl $4096, %ecx ## imm = 0x1000
 ; SSE42-NEXT:    je LBB8_26
-; SSE42-NEXT:  ## %bb.25: ## %cond.load45
-; SSE42-NEXT:    movss {{.*#+}} xmm10 = mem[0],zero,zero,zero
-; SSE42-NEXT:    blendps {{.*#+}} xmm3 = xmm10[0],xmm3[1,2,3]
+; SSE42-NEXT:  LBB8_25: ## %cond.load45
+; SSE42-NEXT:    movd {{.*#+}} xmm8 = mem[0],zero,zero,zero
+; SSE42-NEXT:    pblendw {{.*#+}} xmm3 = xmm8[0,1],xmm3[2,3,4,5,6,7]
 ; SSE42-NEXT:    addq $4, %rsi
-; SSE42-NEXT:  LBB8_26: ## %else46
-; SSE42-NEXT:    pextrb $4, %xmm9, %ecx
-; SSE42-NEXT:    testb $1, %cl
+; SSE42-NEXT:    testl $8192, %ecx ## imm = 0x2000
 ; SSE42-NEXT:    je LBB8_28
-; SSE42-NEXT:  ## %bb.27: ## %cond.load49
+; SSE42-NEXT:  LBB8_27: ## %cond.load49
 ; SSE42-NEXT:    insertps {{.*#+}} xmm3 = xmm3[0],mem[0],xmm3[2,3]
 ; SSE42-NEXT:    addq $4, %rsi
-; SSE42-NEXT:  LBB8_28: ## %else50
-; SSE42-NEXT:    pxor %xmm9, %xmm9
-; SSE42-NEXT:    pcmpeqd %xmm9, %xmm8
-; SSE42-NEXT:    pextrb $8, %xmm8, %ecx
-; SSE42-NEXT:    testb $1, %cl
+; SSE42-NEXT:    testl $16384, %ecx ## imm = 0x4000
 ; SSE42-NEXT:    je LBB8_30
-; SSE42-NEXT:  ## %bb.29: ## %cond.load53
+; SSE42-NEXT:  LBB8_29: ## %cond.load53
 ; SSE42-NEXT:    insertps {{.*#+}} xmm3 = xmm3[0,1],mem[0],xmm3[3]
 ; SSE42-NEXT:    addq $4, %rsi
-; SSE42-NEXT:  LBB8_30: ## %else54
-; SSE42-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm9
-; SSE42-NEXT:    pextrb $12, %xmm8, %ecx
-; SSE42-NEXT:    testb $1, %cl
+; SSE42-NEXT:    testl $32768, %ecx ## imm = 0x8000
 ; SSE42-NEXT:    je LBB8_32
-; SSE42-NEXT:  ## %bb.31: ## %cond.load57
+; SSE42-NEXT:  LBB8_31: ## %cond.load57
 ; SSE42-NEXT:    insertps {{.*#+}} xmm3 = xmm3[0,1,2],mem[0]
 ; SSE42-NEXT:    addq $4, %rsi
-; SSE42-NEXT:  LBB8_32: ## %else58
-; SSE42-NEXT:    pxor %xmm8, %xmm8
-; SSE42-NEXT:    pcmpeqd %xmm9, %xmm8
-; SSE42-NEXT:    pextrb $0, %xmm8, %ecx
-; SSE42-NEXT:    testb $1, %cl
+; SSE42-NEXT:    testl $65536, %ecx ## imm = 0x10000
 ; SSE42-NEXT:    je LBB8_34
-; SSE42-NEXT:  ## %bb.33: ## %cond.load61
-; SSE42-NEXT:    movss {{.*#+}} xmm10 = mem[0],zero,zero,zero
-; SSE42-NEXT:    blendps {{.*#+}} xmm4 = xmm10[0],xmm4[1,2,3]
+; SSE42-NEXT:  LBB8_33: ## %cond.load61
+; SSE42-NEXT:    movd {{.*#+}} xmm8 = mem[0],zero,zero,zero
+; SSE42-NEXT:    pblendw {{.*#+}} xmm4 = xmm8[0,1],xmm4[2,3,4,5,6,7]
 ; SSE42-NEXT:    addq $4, %rsi
-; SSE42-NEXT:  LBB8_34: ## %else62
-; SSE42-NEXT:    pextrb $4, %xmm8, %ecx
-; SSE42-NEXT:    testb $1, %cl
+; SSE42-NEXT:    testl $131072, %ecx ## imm = 0x20000
 ; SSE42-NEXT:    je LBB8_36
-; SSE42-NEXT:  ## %bb.35: ## %cond.load65
+; SSE42-NEXT:  LBB8_35: ## %cond.load65
 ; SSE42-NEXT:    insertps {{.*#+}} xmm4 = xmm4[0],mem[0],xmm4[2,3]
 ; SSE42-NEXT:    addq $4, %rsi
-; SSE42-NEXT:  LBB8_36: ## %else66
-; SSE42-NEXT:    pxor %xmm8, %xmm8
-; SSE42-NEXT:    pcmpeqd %xmm8, %xmm9
-; SSE42-NEXT:    pextrb $8, %xmm9, %ecx
-; SSE42-NEXT:    testb $1, %cl
+; SSE42-NEXT:    testl $262144, %ecx ## imm = 0x40000
 ; SSE42-NEXT:    je LBB8_38
-; SSE42-NEXT:  ## %bb.37: ## %cond.load69
+; SSE42-NEXT:  LBB8_37: ## %cond.load69
 ; SSE42-NEXT:    insertps {{.*#+}} xmm4 = xmm4[0,1],mem[0],xmm4[3]
 ; SSE42-NEXT:    addq $4, %rsi
-; SSE42-NEXT:  LBB8_38: ## %else70
-; SSE42-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm8
-; SSE42-NEXT:    pextrb $12, %xmm9, %ecx
-; SSE42-NEXT:    testb $1, %cl
+; SSE42-NEXT:    testl $524288, %ecx ## imm = 0x80000
 ; SSE42-NEXT:    je LBB8_40
-; SSE42-NEXT:  ## %bb.39: ## %cond.load73
+; SSE42-NEXT:  LBB8_39: ## %cond.load73
 ; SSE42-NEXT:    insertps {{.*#+}} xmm4 = xmm4[0,1,2],mem[0]
 ; SSE42-NEXT:    addq $4, %rsi
-; SSE42-NEXT:  LBB8_40: ## %else74
-; SSE42-NEXT:    pxor %xmm9, %xmm9
-; SSE42-NEXT:    pcmpeqd %xmm8, %xmm9
-; SSE42-NEXT:    pextrb $0, %xmm9, %ecx
-; SSE42-NEXT:    testb $1, %cl
+; SSE42-NEXT:    testl $1048576, %ecx ## imm = 0x100000
 ; SSE42-NEXT:    je LBB8_42
-; SSE42-NEXT:  ## %bb.41: ## %cond.load77
-; SSE42-NEXT:    movss {{.*#+}} xmm10 = mem[0],zero,zero,zero
-; SSE42-NEXT:    blendps {{.*#+}} xmm5 = xmm10[0],xmm5[1,2,3]
+; SSE42-NEXT:  LBB8_41: ## %cond.load77
+; SSE42-NEXT:    movd {{.*#+}} xmm8 = mem[0],zero,zero,zero
+; SSE42-NEXT:    pblendw {{.*#+}} xmm5 = xmm8[0,1],xmm5[2,3,4,5,6,7]
 ; SSE42-NEXT:    addq $4, %rsi
-; SSE42-NEXT:  LBB8_42: ## %else78
-; SSE42-NEXT:    pextrb $4, %xmm9, %ecx
-; SSE42-NEXT:    testb $1, %cl
+; SSE42-NEXT:    testl $2097152, %ecx ## imm = 0x200000
 ; SSE42-NEXT:    je LBB8_44
-; SSE42-NEXT:  ## %bb.43: ## %cond.load81
+; SSE42-NEXT:  LBB8_43: ## %cond.load81
 ; SSE42-NEXT:    insertps {{.*#+}} xmm5 = xmm5[0],mem[0],xmm5[2,3]
 ; SSE42-NEXT:    addq $4, %rsi
-; SSE42-NEXT:  LBB8_44: ## %else82
-; SSE42-NEXT:    pxor %xmm9, %xmm9
-; SSE42-NEXT:    pcmpeqd %xmm9, %xmm8
-; SSE42-NEXT:    pextrb $8, %xmm8, %ecx
-; SSE42-NEXT:    testb $1, %cl
+; SSE42-NEXT:    testl $4194304, %ecx ## imm = 0x400000
 ; SSE42-NEXT:    je LBB8_46
-; SSE42-NEXT:  ## %bb.45: ## %cond.load85
+; SSE42-NEXT:  LBB8_45: ## %cond.load85
 ; SSE42-NEXT:    insertps {{.*#+}} xmm5 = xmm5[0,1],mem[0],xmm5[3]
 ; SSE42-NEXT:    addq $4, %rsi
-; SSE42-NEXT:  LBB8_46: ## %else86
-; SSE42-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm9
-; SSE42-NEXT:    pextrb $12, %xmm8, %ecx
-; SSE42-NEXT:    testb $1, %cl
+; SSE42-NEXT:    testl $8388608, %ecx ## imm = 0x800000
 ; SSE42-NEXT:    je LBB8_48
-; SSE42-NEXT:  ## %bb.47: ## %cond.load89
+; SSE42-NEXT:  LBB8_47: ## %cond.load89
 ; SSE42-NEXT:    insertps {{.*#+}} xmm5 = xmm5[0,1,2],mem[0]
 ; SSE42-NEXT:    addq $4, %rsi
-; SSE42-NEXT:  LBB8_48: ## %else90
-; SSE42-NEXT:    pxor %xmm8, %xmm8
-; SSE42-NEXT:    pcmpeqd %xmm9, %xmm8
-; SSE42-NEXT:    pextrb $0, %xmm8, %ecx
-; SSE42-NEXT:    testb $1, %cl
+; SSE42-NEXT:    testl $16777216, %ecx ## imm = 0x1000000
 ; SSE42-NEXT:    je LBB8_50
-; SSE42-NEXT:  ## %bb.49: ## %cond.load93
-; SSE42-NEXT:    movss {{.*#+}} xmm10 = mem[0],zero,zero,zero
-; SSE42-NEXT:    blendps {{.*#+}} xmm6 = xmm10[0],xmm6[1,2,3]
+; SSE42-NEXT:  LBB8_49: ## %cond.load93
+; SSE42-NEXT:    movd {{.*#+}} xmm8 = mem[0],zero,zero,zero
+; SSE42-NEXT:    pblendw {{.*#+}} xmm6 = xmm8[0,1],xmm6[2,3,4,5,6,7]
 ; SSE42-NEXT:    addq $4, %rsi
-; SSE42-NEXT:  LBB8_50: ## %else94
-; SSE42-NEXT:    pextrb $4, %xmm8, %ecx
-; SSE42-NEXT:    testb $1, %cl
+; SSE42-NEXT:    testl $33554432, %ecx ## imm = 0x2000000
 ; SSE42-NEXT:    je LBB8_52
-; SSE42-NEXT:  ## %bb.51: ## %cond.load97
+; SSE42-NEXT:  LBB8_51: ## %cond.load97
 ; SSE42-NEXT:    insertps {{.*#+}} xmm6 = xmm6[0],mem[0],xmm6[2,3]
 ; SSE42-NEXT:    addq $4, %rsi
-; SSE42-NEXT:  LBB8_52: ## %else98
-; SSE42-NEXT:    pxor %xmm8, %xmm8
-; SSE42-NEXT:    pcmpeqd %xmm8, %xmm9
-; SSE42-NEXT:    pextrb $8, %xmm9, %ecx
-; SSE42-NEXT:    testb $1, %cl
+; SSE42-NEXT:    testl $67108864, %ecx ## imm = 0x4000000
 ; SSE42-NEXT:    je LBB8_54
-; SSE42-NEXT:  ## %bb.53: ## %cond.load101
+; SSE42-NEXT:  LBB8_53: ## %cond.load101
 ; SSE42-NEXT:    insertps {{.*#+}} xmm6 = xmm6[0,1],mem[0],xmm6[3]
 ; SSE42-NEXT:    addq $4, %rsi
-; SSE42-NEXT:  LBB8_54: ## %else102
-; SSE42-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm8
-; SSE42-NEXT:    pextrb $12, %xmm9, %ecx
-; SSE42-NEXT:    testb $1, %cl
+; SSE42-NEXT:    testl $134217728, %ecx ## imm = 0x8000000
 ; SSE42-NEXT:    je LBB8_56
-; SSE42-NEXT:  ## %bb.55: ## %cond.load105
+; SSE42-NEXT:  LBB8_55: ## %cond.load105
 ; SSE42-NEXT:    insertps {{.*#+}} xmm6 = xmm6[0,1,2],mem[0]
 ; SSE42-NEXT:    addq $4, %rsi
-; SSE42-NEXT:  LBB8_56: ## %else106
-; SSE42-NEXT:    pxor %xmm9, %xmm9
-; SSE42-NEXT:    pcmpeqd %xmm8, %xmm9
-; SSE42-NEXT:    pextrb $0, %xmm9, %ecx
-; SSE42-NEXT:    testb $1, %cl
+; SSE42-NEXT:    testl $268435456, %ecx ## imm = 0x10000000
 ; SSE42-NEXT:    je LBB8_58
-; SSE42-NEXT:  ## %bb.57: ## %cond.load109
-; SSE42-NEXT:    movss {{.*#+}} xmm10 = mem[0],zero,zero,zero
-; SSE42-NEXT:    blendps {{.*#+}} xmm7 = xmm10[0],xmm7[1,2,3]
+; SSE42-NEXT:  LBB8_57: ## %cond.load109
+; SSE42-NEXT:    movd {{.*#+}} xmm8 = mem[0],zero,zero,zero
+; SSE42-NEXT:    pblendw {{.*#+}} xmm7 = xmm8[0,1],xmm7[2,3,4,5,6,7]
 ; SSE42-NEXT:    addq $4, %rsi
-; SSE42-NEXT:  LBB8_58: ## %else110
-; SSE42-NEXT:    pextrb $4, %xmm9, %ecx
-; SSE42-NEXT:    testb $1, %cl
+; SSE42-NEXT:    testl $536870912, %ecx ## imm = 0x20000000
 ; SSE42-NEXT:    je LBB8_60
-; SSE42-NEXT:  ## %bb.59: ## %cond.load113
+; SSE42-NEXT:  LBB8_59: ## %cond.load113
 ; SSE42-NEXT:    insertps {{.*#+}} xmm7 = xmm7[0],mem[0],xmm7[2,3]
 ; SSE42-NEXT:    addq $4, %rsi
-; SSE42-NEXT:  LBB8_60: ## %else114
-; SSE42-NEXT:    pxor %xmm9, %xmm9
-; SSE42-NEXT:    pcmpeqd %xmm9, %xmm8
-; SSE42-NEXT:    pextrb $8, %xmm8, %ecx
-; SSE42-NEXT:    testb $1, %cl
+; SSE42-NEXT:    testl $1073741824, %ecx ## imm = 0x40000000
 ; SSE42-NEXT:    je LBB8_62
-; SSE42-NEXT:  ## %bb.61: ## %cond.load117
+; SSE42-NEXT:  LBB8_61: ## %cond.load117
 ; SSE42-NEXT:    insertps {{.*#+}} xmm7 = xmm7[0,1],mem[0],xmm7[3]
 ; SSE42-NEXT:    addq $4, %rsi
-; SSE42-NEXT:  LBB8_62: ## %else118
-; SSE42-NEXT:    pextrb $12, %xmm8, %ecx
-; SSE42-NEXT:    testb $1, %cl
-; SSE42-NEXT:    je LBB8_64
-; SSE42-NEXT:  ## %bb.63: ## %cond.load121
-; SSE42-NEXT:    insertps {{.*#+}} xmm7 = xmm7[0,1,2],mem[0]
-; SSE42-NEXT:  LBB8_64: ## %else122
-; SSE42-NEXT:    movaps %xmm0, (%rax)
-; SSE42-NEXT:    movaps %xmm1, 16(%rax)
-; SSE42-NEXT:    movaps %xmm2, 32(%rax)
-; SSE42-NEXT:    movaps %xmm3, 48(%rax)
-; SSE42-NEXT:    movaps %xmm4, 64(%rax)
-; SSE42-NEXT:    movaps %xmm5, 80(%rax)
-; SSE42-NEXT:    movaps %xmm6, 96(%rax)
-; SSE42-NEXT:    movaps %xmm7, 112(%rax)
-; SSE42-NEXT:    retq
+; SSE42-NEXT:    testl $-2147483648, %ecx ## imm = 0x80000000
+; SSE42-NEXT:    jne LBB8_63
+; SSE42-NEXT:    jmp LBB8_64
 ;
 ; AVX1-LABEL: expandload_v32f32_v32i32:
 ; AVX1:       ## %bb.0:
-; AVX1-NEXT:    vpxor %xmm8, %xmm8, %xmm8
-; AVX1-NEXT:    vpcmpeqd %xmm8, %xmm4, %xmm9
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm9, %xmm9
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm9, %xmm9
-; AVX1-NEXT:    vpextrb $0, %xmm9, %eax
+; AVX1-NEXT:    vextractf128 $1, %ymm5, %xmm8
+; AVX1-NEXT:    vpxor %xmm9, %xmm9, %xmm9
+; AVX1-NEXT:    vpcmpeqd %xmm9, %xmm8, %xmm8
+; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm9, %xmm5
+; AVX1-NEXT:    vpackssdw %xmm8, %xmm5, %xmm8
+; AVX1-NEXT:    vextractf128 $1, %ymm4, %xmm5
+; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm9, %xmm5
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm9, %xmm4
+; AVX1-NEXT:    vpackssdw %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vpacksswb %xmm8, %xmm4, %xmm4
+; AVX1-NEXT:    vpmovmskb %xmm4, %ecx
+; AVX1-NEXT:    vextractf128 $1, %ymm7, %xmm4
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm9, %xmm4
+; AVX1-NEXT:    vpcmpeqd %xmm7, %xmm9, %xmm5
+; AVX1-NEXT:    vpackssdw %xmm4, %xmm5, %xmm4
+; AVX1-NEXT:    vextractf128 $1, %ymm6, %xmm5
+; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm9, %xmm5
+; AVX1-NEXT:    vpcmpeqd %xmm6, %xmm9, %xmm6
+; AVX1-NEXT:    vpackssdw %xmm5, %xmm6, %xmm5
+; AVX1-NEXT:    vpacksswb %xmm4, %xmm5, %xmm4
+; AVX1-NEXT:    vpmovmskb %xmm4, %eax
+; AVX1-NEXT:    shll $16, %eax
+; AVX1-NEXT:    orl %ecx, %eax
 ; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je LBB8_2
-; AVX1-NEXT:  ## %bb.1: ## %cond.load
-; AVX1-NEXT:    vmovss {{.*#+}} xmm9 = mem[0],zero,zero,zero
-; AVX1-NEXT:    vblendps {{.*#+}} ymm0 = ymm9[0],ymm0[1,2,3,4,5,6,7]
+; AVX1-NEXT:    jne LBB8_1
+; AVX1-NEXT:  ## %bb.2: ## %else
+; AVX1-NEXT:    testb $2, %al
+; AVX1-NEXT:    jne LBB8_3
+; AVX1-NEXT:  LBB8_4: ## %else2
+; AVX1-NEXT:    testb $4, %al
+; AVX1-NEXT:    jne LBB8_5
+; AVX1-NEXT:  LBB8_6: ## %else6
+; AVX1-NEXT:    testb $8, %al
+; AVX1-NEXT:    jne LBB8_7
+; AVX1-NEXT:  LBB8_8: ## %else10
+; AVX1-NEXT:    testb $16, %al
+; AVX1-NEXT:    jne LBB8_9
+; AVX1-NEXT:  LBB8_10: ## %else14
+; AVX1-NEXT:    testb $32, %al
+; AVX1-NEXT:    jne LBB8_11
+; AVX1-NEXT:  LBB8_12: ## %else18
+; AVX1-NEXT:    testb $64, %al
+; AVX1-NEXT:    jne LBB8_13
+; AVX1-NEXT:  LBB8_14: ## %else22
+; AVX1-NEXT:    testb $-128, %al
+; AVX1-NEXT:    jne LBB8_15
+; AVX1-NEXT:  LBB8_16: ## %else26
+; AVX1-NEXT:    testl $256, %eax ## imm = 0x100
+; AVX1-NEXT:    jne LBB8_17
+; AVX1-NEXT:  LBB8_18: ## %else30
+; AVX1-NEXT:    testl $512, %eax ## imm = 0x200
+; AVX1-NEXT:    jne LBB8_19
+; AVX1-NEXT:  LBB8_20: ## %else34
+; AVX1-NEXT:    testl $1024, %eax ## imm = 0x400
+; AVX1-NEXT:    jne LBB8_21
+; AVX1-NEXT:  LBB8_22: ## %else38
+; AVX1-NEXT:    testl $2048, %eax ## imm = 0x800
+; AVX1-NEXT:    jne LBB8_23
+; AVX1-NEXT:  LBB8_24: ## %else42
+; AVX1-NEXT:    testl $4096, %eax ## imm = 0x1000
+; AVX1-NEXT:    jne LBB8_25
+; AVX1-NEXT:  LBB8_26: ## %else46
+; AVX1-NEXT:    testl $8192, %eax ## imm = 0x2000
+; AVX1-NEXT:    jne LBB8_27
+; AVX1-NEXT:  LBB8_28: ## %else50
+; AVX1-NEXT:    testl $16384, %eax ## imm = 0x4000
+; AVX1-NEXT:    jne LBB8_29
+; AVX1-NEXT:  LBB8_30: ## %else54
+; AVX1-NEXT:    testl $32768, %eax ## imm = 0x8000
+; AVX1-NEXT:    jne LBB8_31
+; AVX1-NEXT:  LBB8_32: ## %else58
+; AVX1-NEXT:    testl $65536, %eax ## imm = 0x10000
+; AVX1-NEXT:    jne LBB8_33
+; AVX1-NEXT:  LBB8_34: ## %else62
+; AVX1-NEXT:    testl $131072, %eax ## imm = 0x20000
+; AVX1-NEXT:    jne LBB8_35
+; AVX1-NEXT:  LBB8_36: ## %else66
+; AVX1-NEXT:    testl $262144, %eax ## imm = 0x40000
+; AVX1-NEXT:    jne LBB8_37
+; AVX1-NEXT:  LBB8_38: ## %else70
+; AVX1-NEXT:    testl $524288, %eax ## imm = 0x80000
+; AVX1-NEXT:    jne LBB8_39
+; AVX1-NEXT:  LBB8_40: ## %else74
+; AVX1-NEXT:    testl $1048576, %eax ## imm = 0x100000
+; AVX1-NEXT:    jne LBB8_41
+; AVX1-NEXT:  LBB8_42: ## %else78
+; AVX1-NEXT:    testl $2097152, %eax ## imm = 0x200000
+; AVX1-NEXT:    jne LBB8_43
+; AVX1-NEXT:  LBB8_44: ## %else82
+; AVX1-NEXT:    testl $4194304, %eax ## imm = 0x400000
+; AVX1-NEXT:    jne LBB8_45
+; AVX1-NEXT:  LBB8_46: ## %else86
+; AVX1-NEXT:    testl $8388608, %eax ## imm = 0x800000
+; AVX1-NEXT:    jne LBB8_47
+; AVX1-NEXT:  LBB8_48: ## %else90
+; AVX1-NEXT:    testl $16777216, %eax ## imm = 0x1000000
+; AVX1-NEXT:    jne LBB8_49
+; AVX1-NEXT:  LBB8_50: ## %else94
+; AVX1-NEXT:    testl $33554432, %eax ## imm = 0x2000000
+; AVX1-NEXT:    jne LBB8_51
+; AVX1-NEXT:  LBB8_52: ## %else98
+; AVX1-NEXT:    testl $67108864, %eax ## imm = 0x4000000
+; AVX1-NEXT:    jne LBB8_53
+; AVX1-NEXT:  LBB8_54: ## %else102
+; AVX1-NEXT:    testl $134217728, %eax ## imm = 0x8000000
+; AVX1-NEXT:    jne LBB8_55
+; AVX1-NEXT:  LBB8_56: ## %else106
+; AVX1-NEXT:    testl $268435456, %eax ## imm = 0x10000000
+; AVX1-NEXT:    jne LBB8_57
+; AVX1-NEXT:  LBB8_58: ## %else110
+; AVX1-NEXT:    testl $536870912, %eax ## imm = 0x20000000
+; AVX1-NEXT:    jne LBB8_59
+; AVX1-NEXT:  LBB8_60: ## %else114
+; AVX1-NEXT:    testl $1073741824, %eax ## imm = 0x40000000
+; AVX1-NEXT:    jne LBB8_61
+; AVX1-NEXT:  LBB8_62: ## %else118
+; AVX1-NEXT:    testl $-2147483648, %eax ## imm = 0x80000000
+; AVX1-NEXT:    jne LBB8_63
+; AVX1-NEXT:  LBB8_64: ## %else122
+; AVX1-NEXT:    retq
+; AVX1-NEXT:  LBB8_1: ## %cond.load
+; AVX1-NEXT:    vmovss {{.*#+}} xmm4 = mem[0],zero,zero,zero
+; AVX1-NEXT:    vblendps {{.*#+}} ymm0 = ymm4[0],ymm0[1,2,3,4,5,6,7]
 ; AVX1-NEXT:    addq $4, %rdi
-; AVX1-NEXT:  LBB8_2: ## %else
-; AVX1-NEXT:    vpcmpeqd %xmm8, %xmm4, %xmm8
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm8, %xmm8
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm8, %xmm8
-; AVX1-NEXT:    vpextrb $1, %xmm8, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $2, %al
 ; AVX1-NEXT:    je LBB8_4
-; AVX1-NEXT:  ## %bb.3: ## %cond.load1
-; AVX1-NEXT:    vinsertps {{.*#+}} xmm8 = xmm0[0],mem[0],xmm0[2,3]
-; AVX1-NEXT:    vblendps {{.*#+}} ymm0 = ymm8[0,1,2,3],ymm0[4,5,6,7]
+; AVX1-NEXT:  LBB8_3: ## %cond.load1
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm4 = xmm0[0],mem[0],xmm0[2,3]
+; AVX1-NEXT:    vblendps {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm0[4,5,6,7]
 ; AVX1-NEXT:    addq $4, %rdi
-; AVX1-NEXT:  LBB8_4: ## %else2
-; AVX1-NEXT:    vxorps %xmm8, %xmm8, %xmm8
-; AVX1-NEXT:    vpcmpeqd %xmm8, %xmm4, %xmm9
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm9, %xmm9
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm9, %xmm9
-; AVX1-NEXT:    vpextrb $2, %xmm9, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $4, %al
 ; AVX1-NEXT:    je LBB8_6
-; AVX1-NEXT:  ## %bb.5: ## %cond.load5
-; AVX1-NEXT:    vinsertps {{.*#+}} xmm9 = xmm0[0,1],mem[0],xmm0[3]
-; AVX1-NEXT:    vblendps {{.*#+}} ymm0 = ymm9[0,1,2,3],ymm0[4,5,6,7]
+; AVX1-NEXT:  LBB8_5: ## %cond.load5
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm4 = xmm0[0,1],mem[0],xmm0[3]
+; AVX1-NEXT:    vblendps {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm0[4,5,6,7]
 ; AVX1-NEXT:    addq $4, %rdi
-; AVX1-NEXT:  LBB8_6: ## %else6
-; AVX1-NEXT:    vpcmpeqd %xmm8, %xmm4, %xmm8
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm8, %xmm8
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm8, %xmm8
-; AVX1-NEXT:    vpextrb $3, %xmm8, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $8, %al
 ; AVX1-NEXT:    je LBB8_8
-; AVX1-NEXT:  ## %bb.7: ## %cond.load9
-; AVX1-NEXT:    vinsertps {{.*#+}} xmm8 = xmm0[0,1,2],mem[0]
-; AVX1-NEXT:    vblendps {{.*#+}} ymm0 = ymm8[0,1,2,3],ymm0[4,5,6,7]
+; AVX1-NEXT:  LBB8_7: ## %cond.load9
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm4 = xmm0[0,1,2],mem[0]
+; AVX1-NEXT:    vblendps {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm0[4,5,6,7]
 ; AVX1-NEXT:    addq $4, %rdi
-; AVX1-NEXT:  LBB8_8: ## %else10
-; AVX1-NEXT:    vextractf128 $1, %ymm4, %xmm8
-; AVX1-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm8, %xmm9
-; AVX1-NEXT:    vpackssdw %xmm9, %xmm0, %xmm4
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm4, %xmm4
-; AVX1-NEXT:    vpextrb $4, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $16, %al
 ; AVX1-NEXT:    je LBB8_10
-; AVX1-NEXT:  ## %bb.9: ## %cond.load13
-; AVX1-NEXT:    vmovd {{.*#+}} xmm10 = mem[0],zero,zero,zero
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm4 = xmm10[0,1],xmm4[2,3,4,5,6,7]
+; AVX1-NEXT:  LBB8_9: ## %cond.load13
+; AVX1-NEXT:    vmovd {{.*#+}} xmm4 = mem[0],zero,zero,zero
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm5
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1],xmm5[2,3,4,5,6,7]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
 ; AVX1-NEXT:    addq $4, %rdi
-; AVX1-NEXT:  LBB8_10: ## %else14
-; AVX1-NEXT:    vpackssdw %xmm9, %xmm0, %xmm4
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm4, %xmm4
-; AVX1-NEXT:    vpextrb $5, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $32, %al
 ; AVX1-NEXT:    je LBB8_12
-; AVX1-NEXT:  ## %bb.11: ## %cond.load17
+; AVX1-NEXT:  LBB8_11: ## %cond.load17
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm4 = xmm4[0],mem[0],xmm4[2,3]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
 ; AVX1-NEXT:    addq $4, %rdi
-; AVX1-NEXT:  LBB8_12: ## %else18
-; AVX1-NEXT:    vxorps %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm8, %xmm8
-; AVX1-NEXT:    vpackssdw %xmm8, %xmm0, %xmm4
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm4, %xmm4
-; AVX1-NEXT:    vpextrb $6, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $64, %al
 ; AVX1-NEXT:    je LBB8_14
-; AVX1-NEXT:  ## %bb.13: ## %cond.load21
+; AVX1-NEXT:  LBB8_13: ## %cond.load21
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm4 = xmm4[0,1],mem[0],xmm4[3]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
 ; AVX1-NEXT:    addq $4, %rdi
-; AVX1-NEXT:  LBB8_14: ## %else22
-; AVX1-NEXT:    vpackssdw %xmm8, %xmm0, %xmm4
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm4, %xmm4
-; AVX1-NEXT:    vpextrb $7, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $-128, %al
 ; AVX1-NEXT:    je LBB8_16
-; AVX1-NEXT:  ## %bb.15: ## %cond.load25
+; AVX1-NEXT:  LBB8_15: ## %cond.load25
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm4 = xmm4[0,1,2],mem[0]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
 ; AVX1-NEXT:    addq $4, %rdi
-; AVX1-NEXT:  LBB8_16: ## %else26
-; AVX1-NEXT:    vpxor %xmm8, %xmm8, %xmm8
-; AVX1-NEXT:    vpcmpeqd %xmm8, %xmm5, %xmm4
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm4, %xmm4
-; AVX1-NEXT:    vpacksswb %xmm4, %xmm0, %xmm4
-; AVX1-NEXT:    vpextrb $8, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $256, %eax ## imm = 0x100
 ; AVX1-NEXT:    je LBB8_18
-; AVX1-NEXT:  ## %bb.17: ## %cond.load29
+; AVX1-NEXT:  LBB8_17: ## %cond.load29
 ; AVX1-NEXT:    vmovss {{.*#+}} xmm4 = mem[0],zero,zero,zero
 ; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm4[0],ymm1[1,2,3,4,5,6,7]
 ; AVX1-NEXT:    addq $4, %rdi
-; AVX1-NEXT:  LBB8_18: ## %else30
-; AVX1-NEXT:    vpcmpeqd %xmm8, %xmm5, %xmm4
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm4, %xmm4
-; AVX1-NEXT:    vpacksswb %xmm4, %xmm0, %xmm4
-; AVX1-NEXT:    vpextrb $9, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $512, %eax ## imm = 0x200
 ; AVX1-NEXT:    je LBB8_20
-; AVX1-NEXT:  ## %bb.19: ## %cond.load33
+; AVX1-NEXT:  LBB8_19: ## %cond.load33
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm4 = xmm1[0],mem[0],xmm1[2,3]
 ; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm4[0,1,2,3],ymm1[4,5,6,7]
 ; AVX1-NEXT:    addq $4, %rdi
-; AVX1-NEXT:  LBB8_20: ## %else34
-; AVX1-NEXT:    vpxor %xmm8, %xmm8, %xmm8
-; AVX1-NEXT:    vpcmpeqd %xmm8, %xmm5, %xmm4
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm4, %xmm4
-; AVX1-NEXT:    vpacksswb %xmm4, %xmm0, %xmm4
-; AVX1-NEXT:    vpextrb $10, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $1024, %eax ## imm = 0x400
 ; AVX1-NEXT:    je LBB8_22
-; AVX1-NEXT:  ## %bb.21: ## %cond.load37
+; AVX1-NEXT:  LBB8_21: ## %cond.load37
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm4 = xmm1[0,1],mem[0],xmm1[3]
 ; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm4[0,1,2,3],ymm1[4,5,6,7]
 ; AVX1-NEXT:    addq $4, %rdi
-; AVX1-NEXT:  LBB8_22: ## %else38
-; AVX1-NEXT:    vpcmpeqd %xmm8, %xmm5, %xmm4
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm4, %xmm4
-; AVX1-NEXT:    vpacksswb %xmm4, %xmm0, %xmm4
-; AVX1-NEXT:    vpextrb $11, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $2048, %eax ## imm = 0x800
 ; AVX1-NEXT:    je LBB8_24
-; AVX1-NEXT:  ## %bb.23: ## %cond.load41
+; AVX1-NEXT:  LBB8_23: ## %cond.load41
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm4 = xmm1[0,1,2],mem[0]
 ; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm4[0,1,2,3],ymm1[4,5,6,7]
 ; AVX1-NEXT:    addq $4, %rdi
-; AVX1-NEXT:  LBB8_24: ## %else42
-; AVX1-NEXT:    vextractf128 $1, %ymm5, %xmm8
-; AVX1-NEXT:    vpxor %xmm5, %xmm5, %xmm5
-; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm8, %xmm5
-; AVX1-NEXT:    vpackssdw %xmm5, %xmm0, %xmm4
-; AVX1-NEXT:    vpacksswb %xmm4, %xmm0, %xmm4
-; AVX1-NEXT:    vpextrb $12, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $4096, %eax ## imm = 0x1000
 ; AVX1-NEXT:    je LBB8_26
-; AVX1-NEXT:  ## %bb.25: ## %cond.load45
-; AVX1-NEXT:    vmovd {{.*#+}} xmm9 = mem[0],zero,zero,zero
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm4
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm4 = xmm9[0,1],xmm4[2,3,4,5,6,7]
+; AVX1-NEXT:  LBB8_25: ## %cond.load45
+; AVX1-NEXT:    vmovd {{.*#+}} xmm4 = mem[0],zero,zero,zero
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm5
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1],xmm5[2,3,4,5,6,7]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm1, %ymm1
 ; AVX1-NEXT:    addq $4, %rdi
-; AVX1-NEXT:  LBB8_26: ## %else46
-; AVX1-NEXT:    vpackssdw %xmm5, %xmm0, %xmm4
-; AVX1-NEXT:    vpacksswb %xmm4, %xmm0, %xmm4
-; AVX1-NEXT:    vpextrb $13, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $8192, %eax ## imm = 0x2000
 ; AVX1-NEXT:    je LBB8_28
-; AVX1-NEXT:  ## %bb.27: ## %cond.load49
+; AVX1-NEXT:  LBB8_27: ## %cond.load49
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm4
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm4 = xmm4[0],mem[0],xmm4[2,3]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm1, %ymm1
 ; AVX1-NEXT:    addq $4, %rdi
-; AVX1-NEXT:  LBB8_28: ## %else50
-; AVX1-NEXT:    vxorps %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm8, %xmm4
-; AVX1-NEXT:    vpackssdw %xmm4, %xmm0, %xmm5
-; AVX1-NEXT:    vpacksswb %xmm5, %xmm0, %xmm5
-; AVX1-NEXT:    vpextrb $14, %xmm5, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $16384, %eax ## imm = 0x4000
 ; AVX1-NEXT:    je LBB8_30
-; AVX1-NEXT:  ## %bb.29: ## %cond.load53
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm5
-; AVX1-NEXT:    vinsertps {{.*#+}} xmm5 = xmm5[0,1],mem[0],xmm5[3]
-; AVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm1, %ymm1
+; AVX1-NEXT:  LBB8_29: ## %cond.load53
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm4
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm4 = xmm4[0,1],mem[0],xmm4[3]
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm1, %ymm1
 ; AVX1-NEXT:    addq $4, %rdi
-; AVX1-NEXT:  LBB8_30: ## %else54
-; AVX1-NEXT:    vpackssdw %xmm4, %xmm0, %xmm4
-; AVX1-NEXT:    vpacksswb %xmm4, %xmm0, %xmm4
-; AVX1-NEXT:    vpextrb $15, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $32768, %eax ## imm = 0x8000
 ; AVX1-NEXT:    je LBB8_32
-; AVX1-NEXT:  ## %bb.31: ## %cond.load57
+; AVX1-NEXT:  LBB8_31: ## %cond.load57
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm4
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm4 = xmm4[0,1,2],mem[0]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm1, %ymm1
 ; AVX1-NEXT:    addq $4, %rdi
-; AVX1-NEXT:  LBB8_32: ## %else58
-; AVX1-NEXT:    vxorps %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm6, %xmm5
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm5, %xmm5
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm5, %xmm5
-; AVX1-NEXT:    vpextrb $0, %xmm5, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $65536, %eax ## imm = 0x10000
 ; AVX1-NEXT:    je LBB8_34
-; AVX1-NEXT:  ## %bb.33: ## %cond.load61
-; AVX1-NEXT:    vmovss {{.*#+}} xmm5 = mem[0],zero,zero,zero
-; AVX1-NEXT:    vblendps {{.*#+}} ymm2 = ymm5[0],ymm2[1,2,3,4,5,6,7]
+; AVX1-NEXT:  LBB8_33: ## %cond.load61
+; AVX1-NEXT:    vmovss {{.*#+}} xmm4 = mem[0],zero,zero,zero
+; AVX1-NEXT:    vblendps {{.*#+}} ymm2 = ymm4[0],ymm2[1,2,3,4,5,6,7]
 ; AVX1-NEXT:    addq $4, %rdi
-; AVX1-NEXT:  LBB8_34: ## %else62
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm6, %xmm4
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm4, %xmm4
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm4, %xmm4
-; AVX1-NEXT:    vpextrb $1, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $131072, %eax ## imm = 0x20000
 ; AVX1-NEXT:    je LBB8_36
-; AVX1-NEXT:  ## %bb.35: ## %cond.load65
+; AVX1-NEXT:  LBB8_35: ## %cond.load65
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm4 = xmm2[0],mem[0],xmm2[2,3]
 ; AVX1-NEXT:    vblendps {{.*#+}} ymm2 = ymm4[0,1,2,3],ymm2[4,5,6,7]
 ; AVX1-NEXT:    addq $4, %rdi
-; AVX1-NEXT:  LBB8_36: ## %else66
-; AVX1-NEXT:    vxorps %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm6, %xmm5
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm5, %xmm5
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm5, %xmm5
-; AVX1-NEXT:    vpextrb $2, %xmm5, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $262144, %eax ## imm = 0x40000
 ; AVX1-NEXT:    je LBB8_38
-; AVX1-NEXT:  ## %bb.37: ## %cond.load69
-; AVX1-NEXT:    vinsertps {{.*#+}} xmm5 = xmm2[0,1],mem[0],xmm2[3]
-; AVX1-NEXT:    vblendps {{.*#+}} ymm2 = ymm5[0,1,2,3],ymm2[4,5,6,7]
+; AVX1-NEXT:  LBB8_37: ## %cond.load69
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm4 = xmm2[0,1],mem[0],xmm2[3]
+; AVX1-NEXT:    vblendps {{.*#+}} ymm2 = ymm4[0,1,2,3],ymm2[4,5,6,7]
 ; AVX1-NEXT:    addq $4, %rdi
-; AVX1-NEXT:  LBB8_38: ## %else70
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm6, %xmm4
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm4, %xmm4
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm4, %xmm4
-; AVX1-NEXT:    vpextrb $3, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $524288, %eax ## imm = 0x80000
 ; AVX1-NEXT:    je LBB8_40
-; AVX1-NEXT:  ## %bb.39: ## %cond.load73
+; AVX1-NEXT:  LBB8_39: ## %cond.load73
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm4 = xmm2[0,1,2],mem[0]
 ; AVX1-NEXT:    vblendps {{.*#+}} ymm2 = ymm4[0,1,2,3],ymm2[4,5,6,7]
 ; AVX1-NEXT:    addq $4, %rdi
-; AVX1-NEXT:  LBB8_40: ## %else74
-; AVX1-NEXT:    vextractf128 $1, %ymm6, %xmm8
-; AVX1-NEXT:    vxorps %xmm5, %xmm5, %xmm5
-; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm8, %xmm5
-; AVX1-NEXT:    vpackssdw %xmm5, %xmm0, %xmm6
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm6, %xmm6
-; AVX1-NEXT:    vpextrb $4, %xmm6, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $1048576, %eax ## imm = 0x100000
 ; AVX1-NEXT:    je LBB8_42
-; AVX1-NEXT:  ## %bb.41: ## %cond.load77
-; AVX1-NEXT:    vmovss {{.*#+}} xmm6 = mem[0],zero,zero,zero
-; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm4
-; AVX1-NEXT:    vblendps {{.*#+}} xmm4 = xmm6[0],xmm4[1,2,3]
+; AVX1-NEXT:  LBB8_41: ## %cond.load77
+; AVX1-NEXT:    vmovd {{.*#+}} xmm4 = mem[0],zero,zero,zero
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm5
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1],xmm5[2,3,4,5,6,7]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm2, %ymm2
 ; AVX1-NEXT:    addq $4, %rdi
-; AVX1-NEXT:  LBB8_42: ## %else78
-; AVX1-NEXT:    vpackssdw %xmm5, %xmm0, %xmm4
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm4, %xmm4
-; AVX1-NEXT:    vpextrb $5, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $2097152, %eax ## imm = 0x200000
 ; AVX1-NEXT:    je LBB8_44
-; AVX1-NEXT:  ## %bb.43: ## %cond.load81
+; AVX1-NEXT:  LBB8_43: ## %cond.load81
 ; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm4
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm4 = xmm4[0],mem[0],xmm4[2,3]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm2, %ymm2
 ; AVX1-NEXT:    addq $4, %rdi
-; AVX1-NEXT:  LBB8_44: ## %else82
-; AVX1-NEXT:    vxorps %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm8, %xmm4
-; AVX1-NEXT:    vpackssdw %xmm4, %xmm0, %xmm5
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm5, %xmm5
-; AVX1-NEXT:    vpextrb $6, %xmm5, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $4194304, %eax ## imm = 0x400000
 ; AVX1-NEXT:    je LBB8_46
-; AVX1-NEXT:  ## %bb.45: ## %cond.load85
-; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm5
-; AVX1-NEXT:    vinsertps {{.*#+}} xmm5 = xmm5[0,1],mem[0],xmm5[3]
-; AVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm2, %ymm2
+; AVX1-NEXT:  LBB8_45: ## %cond.load85
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm4
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm4 = xmm4[0,1],mem[0],xmm4[3]
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm2, %ymm2
 ; AVX1-NEXT:    addq $4, %rdi
-; AVX1-NEXT:  LBB8_46: ## %else86
-; AVX1-NEXT:    vpackssdw %xmm4, %xmm0, %xmm4
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm4, %xmm4
-; AVX1-NEXT:    vpextrb $7, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $8388608, %eax ## imm = 0x800000
 ; AVX1-NEXT:    je LBB8_48
-; AVX1-NEXT:  ## %bb.47: ## %cond.load89
+; AVX1-NEXT:  LBB8_47: ## %cond.load89
 ; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm4
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm4 = xmm4[0,1,2],mem[0]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm2, %ymm2
 ; AVX1-NEXT:    addq $4, %rdi
-; AVX1-NEXT:  LBB8_48: ## %else90
-; AVX1-NEXT:    vxorps %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm7, %xmm5
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm5, %xmm5
-; AVX1-NEXT:    vpacksswb %xmm5, %xmm0, %xmm5
-; AVX1-NEXT:    vpextrb $8, %xmm5, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $16777216, %eax ## imm = 0x1000000
 ; AVX1-NEXT:    je LBB8_50
-; AVX1-NEXT:  ## %bb.49: ## %cond.load93
-; AVX1-NEXT:    vmovss {{.*#+}} xmm5 = mem[0],zero,zero,zero
-; AVX1-NEXT:    vblendps {{.*#+}} ymm3 = ymm5[0],ymm3[1,2,3,4,5,6,7]
+; AVX1-NEXT:  LBB8_49: ## %cond.load93
+; AVX1-NEXT:    vmovss {{.*#+}} xmm4 = mem[0],zero,zero,zero
+; AVX1-NEXT:    vblendps {{.*#+}} ymm3 = ymm4[0],ymm3[1,2,3,4,5,6,7]
 ; AVX1-NEXT:    addq $4, %rdi
-; AVX1-NEXT:  LBB8_50: ## %else94
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm7, %xmm4
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm4, %xmm4
-; AVX1-NEXT:    vpacksswb %xmm4, %xmm0, %xmm4
-; AVX1-NEXT:    vpextrb $9, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $33554432, %eax ## imm = 0x2000000
 ; AVX1-NEXT:    je LBB8_52
-; AVX1-NEXT:  ## %bb.51: ## %cond.load97
+; AVX1-NEXT:  LBB8_51: ## %cond.load97
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm4 = xmm3[0],mem[0],xmm3[2,3]
 ; AVX1-NEXT:    vblendps {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
 ; AVX1-NEXT:    addq $4, %rdi
-; AVX1-NEXT:  LBB8_52: ## %else98
-; AVX1-NEXT:    vxorps %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm7, %xmm5
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm5, %xmm5
-; AVX1-NEXT:    vpacksswb %xmm5, %xmm0, %xmm5
-; AVX1-NEXT:    vpextrb $10, %xmm5, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $67108864, %eax ## imm = 0x4000000
 ; AVX1-NEXT:    je LBB8_54
-; AVX1-NEXT:  ## %bb.53: ## %cond.load101
-; AVX1-NEXT:    vinsertps {{.*#+}} xmm5 = xmm3[0,1],mem[0],xmm3[3]
-; AVX1-NEXT:    vblendps {{.*#+}} ymm3 = ymm5[0,1,2,3],ymm3[4,5,6,7]
-; AVX1-NEXT:    addq $4, %rdi
-; AVX1-NEXT:  LBB8_54: ## %else102
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm7, %xmm4
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm4, %xmm4
-; AVX1-NEXT:    vpacksswb %xmm4, %xmm0, %xmm4
-; AVX1-NEXT:    vpextrb $11, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:  LBB8_53: ## %cond.load101
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm4 = xmm3[0,1],mem[0],xmm3[3]
+; AVX1-NEXT:    vblendps {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
+; AVX1-NEXT:    addq $4, %rdi
+; AVX1-NEXT:    testl $134217728, %eax ## imm = 0x8000000
 ; AVX1-NEXT:    je LBB8_56
-; AVX1-NEXT:  ## %bb.55: ## %cond.load105
+; AVX1-NEXT:  LBB8_55: ## %cond.load105
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm4 = xmm3[0,1,2],mem[0]
 ; AVX1-NEXT:    vblendps {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
 ; AVX1-NEXT:    addq $4, %rdi
-; AVX1-NEXT:  LBB8_56: ## %else106
-; AVX1-NEXT:    vextractf128 $1, %ymm7, %xmm4
-; AVX1-NEXT:    vxorps %xmm5, %xmm5, %xmm5
-; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm4, %xmm5
-; AVX1-NEXT:    vpackssdw %xmm5, %xmm0, %xmm6
-; AVX1-NEXT:    vpacksswb %xmm6, %xmm0, %xmm6
-; AVX1-NEXT:    vpextrb $12, %xmm6, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $268435456, %eax ## imm = 0x10000000
 ; AVX1-NEXT:    je LBB8_58
-; AVX1-NEXT:  ## %bb.57: ## %cond.load109
-; AVX1-NEXT:    vmovd {{.*#+}} xmm6 = mem[0],zero,zero,zero
-; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm7
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm6 = xmm6[0,1],xmm7[2,3,4,5,6,7]
-; AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm3, %ymm3
+; AVX1-NEXT:  LBB8_57: ## %cond.load109
+; AVX1-NEXT:    vmovd {{.*#+}} xmm4 = mem[0],zero,zero,zero
+; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm5
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1],xmm5[2,3,4,5,6,7]
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
 ; AVX1-NEXT:    addq $4, %rdi
-; AVX1-NEXT:  LBB8_58: ## %else110
-; AVX1-NEXT:    vpackssdw %xmm5, %xmm0, %xmm5
-; AVX1-NEXT:    vpacksswb %xmm5, %xmm0, %xmm5
-; AVX1-NEXT:    vpextrb $13, %xmm5, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $536870912, %eax ## imm = 0x20000000
 ; AVX1-NEXT:    je LBB8_60
-; AVX1-NEXT:  ## %bb.59: ## %cond.load113
-; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm5
-; AVX1-NEXT:    vinsertps {{.*#+}} xmm5 = xmm5[0],mem[0],xmm5[2,3]
-; AVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm3, %ymm3
+; AVX1-NEXT:  LBB8_59: ## %cond.load113
+; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm4
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm4 = xmm4[0],mem[0],xmm4[2,3]
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
 ; AVX1-NEXT:    addq $4, %rdi
-; AVX1-NEXT:  LBB8_60: ## %else114
-; AVX1-NEXT:    vxorps %xmm5, %xmm5, %xmm5
-; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vpackssdw %xmm4, %xmm0, %xmm5
-; AVX1-NEXT:    vpacksswb %xmm5, %xmm0, %xmm5
-; AVX1-NEXT:    vpextrb $14, %xmm5, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $1073741824, %eax ## imm = 0x40000000
 ; AVX1-NEXT:    je LBB8_62
-; AVX1-NEXT:  ## %bb.61: ## %cond.load117
-; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm5
-; AVX1-NEXT:    vinsertps {{.*#+}} xmm5 = xmm5[0,1],mem[0],xmm5[3]
-; AVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm3, %ymm3
+; AVX1-NEXT:  LBB8_61: ## %cond.load117
+; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm4
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm4 = xmm4[0,1],mem[0],xmm4[3]
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
 ; AVX1-NEXT:    addq $4, %rdi
-; AVX1-NEXT:  LBB8_62: ## %else118
-; AVX1-NEXT:    vpackssdw %xmm4, %xmm0, %xmm4
-; AVX1-NEXT:    vpacksswb %xmm4, %xmm0, %xmm4
-; AVX1-NEXT:    vpextrb $15, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $-2147483648, %eax ## imm = 0x80000000
 ; AVX1-NEXT:    je LBB8_64
-; AVX1-NEXT:  ## %bb.63: ## %cond.load121
+; AVX1-NEXT:  LBB8_63: ## %cond.load121
 ; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm4
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm4 = xmm4[0,1,2],mem[0]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
-; AVX1-NEXT:  LBB8_64: ## %else122
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: expandload_v32f32_v32i32:
 ; AVX2:       ## %bb.0:
 ; AVX2-NEXT:    vpxor %xmm8, %xmm8, %xmm8
-; AVX2-NEXT:    vpcmpeqd %xmm8, %xmm4, %xmm9
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm9, %xmm9
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm9, %xmm9
-; AVX2-NEXT:    vpextrb $0, %xmm9, %eax
+; AVX2-NEXT:    vpcmpeqd %ymm7, %ymm8, %ymm7
+; AVX2-NEXT:    vpcmpeqd %ymm6, %ymm8, %ymm6
+; AVX2-NEXT:    vpackssdw %ymm7, %ymm6, %ymm6
+; AVX2-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[0,2,1,3]
+; AVX2-NEXT:    vpcmpeqd %ymm5, %ymm8, %ymm5
+; AVX2-NEXT:    vpcmpeqd %ymm4, %ymm8, %ymm4
+; AVX2-NEXT:    vpackssdw %ymm5, %ymm4, %ymm4
+; AVX2-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[0,2,1,3]
+; AVX2-NEXT:    vpacksswb %ymm6, %ymm4, %ymm4
+; AVX2-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[0,2,1,3]
+; AVX2-NEXT:    vpmovmskb %ymm4, %eax
 ; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je LBB8_2
-; AVX2-NEXT:  ## %bb.1: ## %cond.load
-; AVX2-NEXT:    vmovd {{.*#+}} xmm9 = mem[0],zero,zero,zero
-; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm9[0],ymm0[1,2,3,4,5,6,7]
+; AVX2-NEXT:    jne LBB8_1
+; AVX2-NEXT:  ## %bb.2: ## %else
+; AVX2-NEXT:    testb $2, %al
+; AVX2-NEXT:    jne LBB8_3
+; AVX2-NEXT:  LBB8_4: ## %else2
+; AVX2-NEXT:    testb $4, %al
+; AVX2-NEXT:    jne LBB8_5
+; AVX2-NEXT:  LBB8_6: ## %else6
+; AVX2-NEXT:    testb $8, %al
+; AVX2-NEXT:    jne LBB8_7
+; AVX2-NEXT:  LBB8_8: ## %else10
+; AVX2-NEXT:    testb $16, %al
+; AVX2-NEXT:    jne LBB8_9
+; AVX2-NEXT:  LBB8_10: ## %else14
+; AVX2-NEXT:    testb $32, %al
+; AVX2-NEXT:    jne LBB8_11
+; AVX2-NEXT:  LBB8_12: ## %else18
+; AVX2-NEXT:    testb $64, %al
+; AVX2-NEXT:    jne LBB8_13
+; AVX2-NEXT:  LBB8_14: ## %else22
+; AVX2-NEXT:    testb $-128, %al
+; AVX2-NEXT:    jne LBB8_15
+; AVX2-NEXT:  LBB8_16: ## %else26
+; AVX2-NEXT:    testl $256, %eax ## imm = 0x100
+; AVX2-NEXT:    jne LBB8_17
+; AVX2-NEXT:  LBB8_18: ## %else30
+; AVX2-NEXT:    testl $512, %eax ## imm = 0x200
+; AVX2-NEXT:    jne LBB8_19
+; AVX2-NEXT:  LBB8_20: ## %else34
+; AVX2-NEXT:    testl $1024, %eax ## imm = 0x400
+; AVX2-NEXT:    jne LBB8_21
+; AVX2-NEXT:  LBB8_22: ## %else38
+; AVX2-NEXT:    testl $2048, %eax ## imm = 0x800
+; AVX2-NEXT:    jne LBB8_23
+; AVX2-NEXT:  LBB8_24: ## %else42
+; AVX2-NEXT:    testl $4096, %eax ## imm = 0x1000
+; AVX2-NEXT:    jne LBB8_25
+; AVX2-NEXT:  LBB8_26: ## %else46
+; AVX2-NEXT:    testl $8192, %eax ## imm = 0x2000
+; AVX2-NEXT:    jne LBB8_27
+; AVX2-NEXT:  LBB8_28: ## %else50
+; AVX2-NEXT:    testl $16384, %eax ## imm = 0x4000
+; AVX2-NEXT:    jne LBB8_29
+; AVX2-NEXT:  LBB8_30: ## %else54
+; AVX2-NEXT:    testl $32768, %eax ## imm = 0x8000
+; AVX2-NEXT:    jne LBB8_31
+; AVX2-NEXT:  LBB8_32: ## %else58
+; AVX2-NEXT:    testl $65536, %eax ## imm = 0x10000
+; AVX2-NEXT:    jne LBB8_33
+; AVX2-NEXT:  LBB8_34: ## %else62
+; AVX2-NEXT:    testl $131072, %eax ## imm = 0x20000
+; AVX2-NEXT:    jne LBB8_35
+; AVX2-NEXT:  LBB8_36: ## %else66
+; AVX2-NEXT:    testl $262144, %eax ## imm = 0x40000
+; AVX2-NEXT:    jne LBB8_37
+; AVX2-NEXT:  LBB8_38: ## %else70
+; AVX2-NEXT:    testl $524288, %eax ## imm = 0x80000
+; AVX2-NEXT:    jne LBB8_39
+; AVX2-NEXT:  LBB8_40: ## %else74
+; AVX2-NEXT:    testl $1048576, %eax ## imm = 0x100000
+; AVX2-NEXT:    jne LBB8_41
+; AVX2-NEXT:  LBB8_42: ## %else78
+; AVX2-NEXT:    testl $2097152, %eax ## imm = 0x200000
+; AVX2-NEXT:    jne LBB8_43
+; AVX2-NEXT:  LBB8_44: ## %else82
+; AVX2-NEXT:    testl $4194304, %eax ## imm = 0x400000
+; AVX2-NEXT:    jne LBB8_45
+; AVX2-NEXT:  LBB8_46: ## %else86
+; AVX2-NEXT:    testl $8388608, %eax ## imm = 0x800000
+; AVX2-NEXT:    jne LBB8_47
+; AVX2-NEXT:  LBB8_48: ## %else90
+; AVX2-NEXT:    testl $16777216, %eax ## imm = 0x1000000
+; AVX2-NEXT:    jne LBB8_49
+; AVX2-NEXT:  LBB8_50: ## %else94
+; AVX2-NEXT:    testl $33554432, %eax ## imm = 0x2000000
+; AVX2-NEXT:    jne LBB8_51
+; AVX2-NEXT:  LBB8_52: ## %else98
+; AVX2-NEXT:    testl $67108864, %eax ## imm = 0x4000000
+; AVX2-NEXT:    jne LBB8_53
+; AVX2-NEXT:  LBB8_54: ## %else102
+; AVX2-NEXT:    testl $134217728, %eax ## imm = 0x8000000
+; AVX2-NEXT:    jne LBB8_55
+; AVX2-NEXT:  LBB8_56: ## %else106
+; AVX2-NEXT:    testl $268435456, %eax ## imm = 0x10000000
+; AVX2-NEXT:    jne LBB8_57
+; AVX2-NEXT:  LBB8_58: ## %else110
+; AVX2-NEXT:    testl $536870912, %eax ## imm = 0x20000000
+; AVX2-NEXT:    jne LBB8_59
+; AVX2-NEXT:  LBB8_60: ## %else114
+; AVX2-NEXT:    testl $1073741824, %eax ## imm = 0x40000000
+; AVX2-NEXT:    jne LBB8_61
+; AVX2-NEXT:  LBB8_62: ## %else118
+; AVX2-NEXT:    testl $-2147483648, %eax ## imm = 0x80000000
+; AVX2-NEXT:    jne LBB8_63
+; AVX2-NEXT:  LBB8_64: ## %else122
+; AVX2-NEXT:    retq
+; AVX2-NEXT:  LBB8_1: ## %cond.load
+; AVX2-NEXT:    vmovd {{.*#+}} xmm4 = mem[0],zero,zero,zero
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm4[0],ymm0[1,2,3,4,5,6,7]
 ; AVX2-NEXT:    addq $4, %rdi
-; AVX2-NEXT:  LBB8_2: ## %else
-; AVX2-NEXT:    vpcmpeqd %xmm8, %xmm4, %xmm8
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm8, %xmm8
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm8, %xmm8
-; AVX2-NEXT:    vpextrb $1, %xmm8, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $2, %al
 ; AVX2-NEXT:    je LBB8_4
-; AVX2-NEXT:  ## %bb.3: ## %cond.load1
-; AVX2-NEXT:    vinsertps {{.*#+}} xmm8 = xmm0[0],mem[0],xmm0[2,3]
-; AVX2-NEXT:    vblendps {{.*#+}} ymm0 = ymm8[0,1,2,3],ymm0[4,5,6,7]
+; AVX2-NEXT:  LBB8_3: ## %cond.load1
+; AVX2-NEXT:    vinsertps {{.*#+}} xmm4 = xmm0[0],mem[0],xmm0[2,3]
+; AVX2-NEXT:    vblendps {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm0[4,5,6,7]
 ; AVX2-NEXT:    addq $4, %rdi
-; AVX2-NEXT:  LBB8_4: ## %else2
-; AVX2-NEXT:    vxorps %xmm8, %xmm8, %xmm8
-; AVX2-NEXT:    vpcmpeqd %xmm8, %xmm4, %xmm9
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm9, %xmm9
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm9, %xmm9
-; AVX2-NEXT:    vpextrb $2, %xmm9, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $4, %al
 ; AVX2-NEXT:    je LBB8_6
-; AVX2-NEXT:  ## %bb.5: ## %cond.load5
-; AVX2-NEXT:    vinsertps {{.*#+}} xmm9 = xmm0[0,1],mem[0],xmm0[3]
-; AVX2-NEXT:    vblendps {{.*#+}} ymm0 = ymm9[0,1,2,3],ymm0[4,5,6,7]
+; AVX2-NEXT:  LBB8_5: ## %cond.load5
+; AVX2-NEXT:    vinsertps {{.*#+}} xmm4 = xmm0[0,1],mem[0],xmm0[3]
+; AVX2-NEXT:    vblendps {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm0[4,5,6,7]
 ; AVX2-NEXT:    addq $4, %rdi
-; AVX2-NEXT:  LBB8_6: ## %else6
-; AVX2-NEXT:    vpcmpeqd %xmm8, %xmm4, %xmm8
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm8, %xmm8
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm8, %xmm8
-; AVX2-NEXT:    vpextrb $3, %xmm8, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $8, %al
 ; AVX2-NEXT:    je LBB8_8
-; AVX2-NEXT:  ## %bb.7: ## %cond.load9
-; AVX2-NEXT:    vinsertps {{.*#+}} xmm8 = xmm0[0,1,2],mem[0]
-; AVX2-NEXT:    vblendps {{.*#+}} ymm0 = ymm8[0,1,2,3],ymm0[4,5,6,7]
+; AVX2-NEXT:  LBB8_7: ## %cond.load9
+; AVX2-NEXT:    vinsertps {{.*#+}} xmm4 = xmm0[0,1,2],mem[0]
+; AVX2-NEXT:    vblendps {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm0[4,5,6,7]
 ; AVX2-NEXT:    addq $4, %rdi
-; AVX2-NEXT:  LBB8_8: ## %else10
-; AVX2-NEXT:    vextracti128 $1, %ymm4, %xmm8
-; AVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm8, %xmm9
-; AVX2-NEXT:    vpackssdw %xmm9, %xmm0, %xmm4
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm4, %xmm4
-; AVX2-NEXT:    vpextrb $4, %xmm4, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $16, %al
 ; AVX2-NEXT:    je LBB8_10
-; AVX2-NEXT:  ## %bb.9: ## %cond.load13
-; AVX2-NEXT:    vmovss {{.*#+}} xmm10 = mem[0],zero,zero,zero
-; AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm4
-; AVX2-NEXT:    vblendps {{.*#+}} xmm4 = xmm10[0],xmm4[1,2,3]
+; AVX2-NEXT:  LBB8_9: ## %cond.load13
+; AVX2-NEXT:    vmovss {{.*#+}} xmm4 = mem[0],zero,zero,zero
+; AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm5
+; AVX2-NEXT:    vblendps {{.*#+}} xmm4 = xmm4[0],xmm5[1,2,3]
 ; AVX2-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
 ; AVX2-NEXT:    addq $4, %rdi
-; AVX2-NEXT:  LBB8_10: ## %else14
-; AVX2-NEXT:    vpackssdw %xmm9, %xmm0, %xmm4
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm4, %xmm4
-; AVX2-NEXT:    vpextrb $5, %xmm4, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $32, %al
 ; AVX2-NEXT:    je LBB8_12
-; AVX2-NEXT:  ## %bb.11: ## %cond.load17
+; AVX2-NEXT:  LBB8_11: ## %cond.load17
 ; AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm4
 ; AVX2-NEXT:    vinsertps {{.*#+}} xmm4 = xmm4[0],mem[0],xmm4[2,3]
 ; AVX2-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
 ; AVX2-NEXT:    addq $4, %rdi
-; AVX2-NEXT:  LBB8_12: ## %else18
-; AVX2-NEXT:    vxorps %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm8, %xmm8
-; AVX2-NEXT:    vpackssdw %xmm8, %xmm0, %xmm4
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm4, %xmm4
-; AVX2-NEXT:    vpextrb $6, %xmm4, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $64, %al
 ; AVX2-NEXT:    je LBB8_14
-; AVX2-NEXT:  ## %bb.13: ## %cond.load21
+; AVX2-NEXT:  LBB8_13: ## %cond.load21
 ; AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm4
 ; AVX2-NEXT:    vinsertps {{.*#+}} xmm4 = xmm4[0,1],mem[0],xmm4[3]
 ; AVX2-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
 ; AVX2-NEXT:    addq $4, %rdi
-; AVX2-NEXT:  LBB8_14: ## %else22
-; AVX2-NEXT:    vpackssdw %xmm8, %xmm0, %xmm4
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm4, %xmm4
-; AVX2-NEXT:    vpextrb $7, %xmm4, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $-128, %al
 ; AVX2-NEXT:    je LBB8_16
-; AVX2-NEXT:  ## %bb.15: ## %cond.load25
+; AVX2-NEXT:  LBB8_15: ## %cond.load25
 ; AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm4
 ; AVX2-NEXT:    vinsertps {{.*#+}} xmm4 = xmm4[0,1,2],mem[0]
 ; AVX2-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
 ; AVX2-NEXT:    addq $4, %rdi
-; AVX2-NEXT:  LBB8_16: ## %else26
-; AVX2-NEXT:    vpxor %xmm8, %xmm8, %xmm8
-; AVX2-NEXT:    vpcmpeqd %xmm8, %xmm5, %xmm4
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm4, %xmm4
-; AVX2-NEXT:    vpacksswb %xmm4, %xmm0, %xmm4
-; AVX2-NEXT:    vpextrb $8, %xmm4, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $256, %eax ## imm = 0x100
 ; AVX2-NEXT:    je LBB8_18
-; AVX2-NEXT:  ## %bb.17: ## %cond.load29
+; AVX2-NEXT:  LBB8_17: ## %cond.load29
 ; AVX2-NEXT:    vmovd {{.*#+}} xmm4 = mem[0],zero,zero,zero
 ; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm4[0],ymm1[1,2,3,4,5,6,7]
 ; AVX2-NEXT:    addq $4, %rdi
-; AVX2-NEXT:  LBB8_18: ## %else30
-; AVX2-NEXT:    vpcmpeqd %xmm8, %xmm5, %xmm4
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm4, %xmm4
-; AVX2-NEXT:    vpacksswb %xmm4, %xmm0, %xmm4
-; AVX2-NEXT:    vpextrb $9, %xmm4, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $512, %eax ## imm = 0x200
 ; AVX2-NEXT:    je LBB8_20
-; AVX2-NEXT:  ## %bb.19: ## %cond.load33
+; AVX2-NEXT:  LBB8_19: ## %cond.load33
 ; AVX2-NEXT:    vinsertps {{.*#+}} xmm4 = xmm1[0],mem[0],xmm1[2,3]
 ; AVX2-NEXT:    vblendps {{.*#+}} ymm1 = ymm4[0,1,2,3],ymm1[4,5,6,7]
 ; AVX2-NEXT:    addq $4, %rdi
-; AVX2-NEXT:  LBB8_20: ## %else34
-; AVX2-NEXT:    vpxor %xmm8, %xmm8, %xmm8
-; AVX2-NEXT:    vpcmpeqd %xmm8, %xmm5, %xmm4
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm4, %xmm4
-; AVX2-NEXT:    vpacksswb %xmm4, %xmm0, %xmm4
-; AVX2-NEXT:    vpextrb $10, %xmm4, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $1024, %eax ## imm = 0x400
 ; AVX2-NEXT:    je LBB8_22
-; AVX2-NEXT:  ## %bb.21: ## %cond.load37
+; AVX2-NEXT:  LBB8_21: ## %cond.load37
 ; AVX2-NEXT:    vinsertps {{.*#+}} xmm4 = xmm1[0,1],mem[0],xmm1[3]
 ; AVX2-NEXT:    vblendps {{.*#+}} ymm1 = ymm4[0,1,2,3],ymm1[4,5,6,7]
 ; AVX2-NEXT:    addq $4, %rdi
-; AVX2-NEXT:  LBB8_22: ## %else38
-; AVX2-NEXT:    vpcmpeqd %xmm8, %xmm5, %xmm4
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm4, %xmm4
-; AVX2-NEXT:    vpacksswb %xmm4, %xmm0, %xmm4
-; AVX2-NEXT:    vpextrb $11, %xmm4, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $2048, %eax ## imm = 0x800
 ; AVX2-NEXT:    je LBB8_24
-; AVX2-NEXT:  ## %bb.23: ## %cond.load41
+; AVX2-NEXT:  LBB8_23: ## %cond.load41
 ; AVX2-NEXT:    vinsertps {{.*#+}} xmm4 = xmm1[0,1,2],mem[0]
 ; AVX2-NEXT:    vblendps {{.*#+}} ymm1 = ymm4[0,1,2,3],ymm1[4,5,6,7]
 ; AVX2-NEXT:    addq $4, %rdi
-; AVX2-NEXT:  LBB8_24: ## %else42
-; AVX2-NEXT:    vextracti128 $1, %ymm5, %xmm8
-; AVX2-NEXT:    vpxor %xmm5, %xmm5, %xmm5
-; AVX2-NEXT:    vpcmpeqd %xmm5, %xmm8, %xmm5
-; AVX2-NEXT:    vpackssdw %xmm5, %xmm0, %xmm4
-; AVX2-NEXT:    vpacksswb %xmm4, %xmm0, %xmm4
-; AVX2-NEXT:    vpextrb $12, %xmm4, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $4096, %eax ## imm = 0x1000
 ; AVX2-NEXT:    je LBB8_26
-; AVX2-NEXT:  ## %bb.25: ## %cond.load45
-; AVX2-NEXT:    vmovss {{.*#+}} xmm9 = mem[0],zero,zero,zero
-; AVX2-NEXT:    vextractf128 $1, %ymm1, %xmm4
-; AVX2-NEXT:    vblendps {{.*#+}} xmm4 = xmm9[0],xmm4[1,2,3]
+; AVX2-NEXT:  LBB8_25: ## %cond.load45
+; AVX2-NEXT:    vmovss {{.*#+}} xmm4 = mem[0],zero,zero,zero
+; AVX2-NEXT:    vextractf128 $1, %ymm1, %xmm5
+; AVX2-NEXT:    vblendps {{.*#+}} xmm4 = xmm4[0],xmm5[1,2,3]
 ; AVX2-NEXT:    vinsertf128 $1, %xmm4, %ymm1, %ymm1
 ; AVX2-NEXT:    addq $4, %rdi
-; AVX2-NEXT:  LBB8_26: ## %else46
-; AVX2-NEXT:    vpackssdw %xmm5, %xmm0, %xmm4
-; AVX2-NEXT:    vpacksswb %xmm4, %xmm0, %xmm4
-; AVX2-NEXT:    vpextrb $13, %xmm4, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $8192, %eax ## imm = 0x2000
 ; AVX2-NEXT:    je LBB8_28
-; AVX2-NEXT:  ## %bb.27: ## %cond.load49
+; AVX2-NEXT:  LBB8_27: ## %cond.load49
 ; AVX2-NEXT:    vextractf128 $1, %ymm1, %xmm4
 ; AVX2-NEXT:    vinsertps {{.*#+}} xmm4 = xmm4[0],mem[0],xmm4[2,3]
 ; AVX2-NEXT:    vinsertf128 $1, %xmm4, %ymm1, %ymm1
 ; AVX2-NEXT:    addq $4, %rdi
-; AVX2-NEXT:  LBB8_28: ## %else50
-; AVX2-NEXT:    vxorps %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm8, %xmm4
-; AVX2-NEXT:    vpackssdw %xmm4, %xmm0, %xmm5
-; AVX2-NEXT:    vpacksswb %xmm5, %xmm0, %xmm5
-; AVX2-NEXT:    vpextrb $14, %xmm5, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $16384, %eax ## imm = 0x4000
 ; AVX2-NEXT:    je LBB8_30
-; AVX2-NEXT:  ## %bb.29: ## %cond.load53
-; AVX2-NEXT:    vextractf128 $1, %ymm1, %xmm5
-; AVX2-NEXT:    vinsertps {{.*#+}} xmm5 = xmm5[0,1],mem[0],xmm5[3]
-; AVX2-NEXT:    vinsertf128 $1, %xmm5, %ymm1, %ymm1
+; AVX2-NEXT:  LBB8_29: ## %cond.load53
+; AVX2-NEXT:    vextractf128 $1, %ymm1, %xmm4
+; AVX2-NEXT:    vinsertps {{.*#+}} xmm4 = xmm4[0,1],mem[0],xmm4[3]
+; AVX2-NEXT:    vinsertf128 $1, %xmm4, %ymm1, %ymm1
 ; AVX2-NEXT:    addq $4, %rdi
-; AVX2-NEXT:  LBB8_30: ## %else54
-; AVX2-NEXT:    vpackssdw %xmm4, %xmm0, %xmm4
-; AVX2-NEXT:    vpacksswb %xmm4, %xmm0, %xmm4
-; AVX2-NEXT:    vpextrb $15, %xmm4, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $32768, %eax ## imm = 0x8000
 ; AVX2-NEXT:    je LBB8_32
-; AVX2-NEXT:  ## %bb.31: ## %cond.load57
+; AVX2-NEXT:  LBB8_31: ## %cond.load57
 ; AVX2-NEXT:    vextractf128 $1, %ymm1, %xmm4
 ; AVX2-NEXT:    vinsertps {{.*#+}} xmm4 = xmm4[0,1,2],mem[0]
 ; AVX2-NEXT:    vinsertf128 $1, %xmm4, %ymm1, %ymm1
 ; AVX2-NEXT:    addq $4, %rdi
-; AVX2-NEXT:  LBB8_32: ## %else58
-; AVX2-NEXT:    vxorps %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm6, %xmm5
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm5, %xmm5
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm5, %xmm5
-; AVX2-NEXT:    vpextrb $0, %xmm5, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $65536, %eax ## imm = 0x10000
 ; AVX2-NEXT:    je LBB8_34
-; AVX2-NEXT:  ## %bb.33: ## %cond.load61
-; AVX2-NEXT:    vmovd {{.*#+}} xmm5 = mem[0],zero,zero,zero
-; AVX2-NEXT:    vpblendd {{.*#+}} ymm2 = ymm5[0],ymm2[1,2,3,4,5,6,7]
+; AVX2-NEXT:  LBB8_33: ## %cond.load61
+; AVX2-NEXT:    vmovd {{.*#+}} xmm4 = mem[0],zero,zero,zero
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm2 = ymm4[0],ymm2[1,2,3,4,5,6,7]
 ; AVX2-NEXT:    addq $4, %rdi
-; AVX2-NEXT:  LBB8_34: ## %else62
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm6, %xmm4
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm4, %xmm4
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm4, %xmm4
-; AVX2-NEXT:    vpextrb $1, %xmm4, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $131072, %eax ## imm = 0x20000
 ; AVX2-NEXT:    je LBB8_36
-; AVX2-NEXT:  ## %bb.35: ## %cond.load65
+; AVX2-NEXT:  LBB8_35: ## %cond.load65
 ; AVX2-NEXT:    vinsertps {{.*#+}} xmm4 = xmm2[0],mem[0],xmm2[2,3]
 ; AVX2-NEXT:    vblendps {{.*#+}} ymm2 = ymm4[0,1,2,3],ymm2[4,5,6,7]
 ; AVX2-NEXT:    addq $4, %rdi
-; AVX2-NEXT:  LBB8_36: ## %else66
-; AVX2-NEXT:    vxorps %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm6, %xmm5
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm5, %xmm5
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm5, %xmm5
-; AVX2-NEXT:    vpextrb $2, %xmm5, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $262144, %eax ## imm = 0x40000
 ; AVX2-NEXT:    je LBB8_38
-; AVX2-NEXT:  ## %bb.37: ## %cond.load69
-; AVX2-NEXT:    vinsertps {{.*#+}} xmm5 = xmm2[0,1],mem[0],xmm2[3]
-; AVX2-NEXT:    vblendps {{.*#+}} ymm2 = ymm5[0,1,2,3],ymm2[4,5,6,7]
+; AVX2-NEXT:  LBB8_37: ## %cond.load69
+; AVX2-NEXT:    vinsertps {{.*#+}} xmm4 = xmm2[0,1],mem[0],xmm2[3]
+; AVX2-NEXT:    vblendps {{.*#+}} ymm2 = ymm4[0,1,2,3],ymm2[4,5,6,7]
 ; AVX2-NEXT:    addq $4, %rdi
-; AVX2-NEXT:  LBB8_38: ## %else70
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm6, %xmm4
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm4, %xmm4
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm4, %xmm4
-; AVX2-NEXT:    vpextrb $3, %xmm4, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $524288, %eax ## imm = 0x80000
 ; AVX2-NEXT:    je LBB8_40
-; AVX2-NEXT:  ## %bb.39: ## %cond.load73
+; AVX2-NEXT:  LBB8_39: ## %cond.load73
 ; AVX2-NEXT:    vinsertps {{.*#+}} xmm4 = xmm2[0,1,2],mem[0]
 ; AVX2-NEXT:    vblendps {{.*#+}} ymm2 = ymm4[0,1,2,3],ymm2[4,5,6,7]
 ; AVX2-NEXT:    addq $4, %rdi
-; AVX2-NEXT:  LBB8_40: ## %else74
-; AVX2-NEXT:    vextracti128 $1, %ymm6, %xmm8
-; AVX2-NEXT:    vxorps %xmm5, %xmm5, %xmm5
-; AVX2-NEXT:    vpcmpeqd %xmm5, %xmm8, %xmm5
-; AVX2-NEXT:    vpackssdw %xmm5, %xmm0, %xmm6
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm6, %xmm6
-; AVX2-NEXT:    vpextrb $4, %xmm6, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $1048576, %eax ## imm = 0x100000
 ; AVX2-NEXT:    je LBB8_42
-; AVX2-NEXT:  ## %bb.41: ## %cond.load77
-; AVX2-NEXT:    vmovss {{.*#+}} xmm6 = mem[0],zero,zero,zero
-; AVX2-NEXT:    vextractf128 $1, %ymm2, %xmm4
-; AVX2-NEXT:    vblendps {{.*#+}} xmm4 = xmm6[0],xmm4[1,2,3]
+; AVX2-NEXT:  LBB8_41: ## %cond.load77
+; AVX2-NEXT:    vmovss {{.*#+}} xmm4 = mem[0],zero,zero,zero
+; AVX2-NEXT:    vextractf128 $1, %ymm2, %xmm5
+; AVX2-NEXT:    vblendps {{.*#+}} xmm4 = xmm4[0],xmm5[1,2,3]
 ; AVX2-NEXT:    vinsertf128 $1, %xmm4, %ymm2, %ymm2
 ; AVX2-NEXT:    addq $4, %rdi
-; AVX2-NEXT:  LBB8_42: ## %else78
-; AVX2-NEXT:    vpackssdw %xmm5, %xmm0, %xmm4
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm4, %xmm4
-; AVX2-NEXT:    vpextrb $5, %xmm4, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $2097152, %eax ## imm = 0x200000
 ; AVX2-NEXT:    je LBB8_44
-; AVX2-NEXT:  ## %bb.43: ## %cond.load81
+; AVX2-NEXT:  LBB8_43: ## %cond.load81
 ; AVX2-NEXT:    vextractf128 $1, %ymm2, %xmm4
 ; AVX2-NEXT:    vinsertps {{.*#+}} xmm4 = xmm4[0],mem[0],xmm4[2,3]
 ; AVX2-NEXT:    vinsertf128 $1, %xmm4, %ymm2, %ymm2
 ; AVX2-NEXT:    addq $4, %rdi
-; AVX2-NEXT:  LBB8_44: ## %else82
-; AVX2-NEXT:    vxorps %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm8, %xmm4
-; AVX2-NEXT:    vpackssdw %xmm4, %xmm0, %xmm5
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm5, %xmm5
-; AVX2-NEXT:    vpextrb $6, %xmm5, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $4194304, %eax ## imm = 0x400000
 ; AVX2-NEXT:    je LBB8_46
-; AVX2-NEXT:  ## %bb.45: ## %cond.load85
-; AVX2-NEXT:    vextractf128 $1, %ymm2, %xmm5
-; AVX2-NEXT:    vinsertps {{.*#+}} xmm5 = xmm5[0,1],mem[0],xmm5[3]
-; AVX2-NEXT:    vinsertf128 $1, %xmm5, %ymm2, %ymm2
+; AVX2-NEXT:  LBB8_45: ## %cond.load85
+; AVX2-NEXT:    vextractf128 $1, %ymm2, %xmm4
+; AVX2-NEXT:    vinsertps {{.*#+}} xmm4 = xmm4[0,1],mem[0],xmm4[3]
+; AVX2-NEXT:    vinsertf128 $1, %xmm4, %ymm2, %ymm2
 ; AVX2-NEXT:    addq $4, %rdi
-; AVX2-NEXT:  LBB8_46: ## %else86
-; AVX2-NEXT:    vpackssdw %xmm4, %xmm0, %xmm4
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm4, %xmm4
-; AVX2-NEXT:    vpextrb $7, %xmm4, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $8388608, %eax ## imm = 0x800000
 ; AVX2-NEXT:    je LBB8_48
-; AVX2-NEXT:  ## %bb.47: ## %cond.load89
+; AVX2-NEXT:  LBB8_47: ## %cond.load89
 ; AVX2-NEXT:    vextractf128 $1, %ymm2, %xmm4
 ; AVX2-NEXT:    vinsertps {{.*#+}} xmm4 = xmm4[0,1,2],mem[0]
 ; AVX2-NEXT:    vinsertf128 $1, %xmm4, %ymm2, %ymm2
 ; AVX2-NEXT:    addq $4, %rdi
-; AVX2-NEXT:  LBB8_48: ## %else90
-; AVX2-NEXT:    vxorps %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm7, %xmm5
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm5, %xmm5
-; AVX2-NEXT:    vpacksswb %xmm5, %xmm0, %xmm5
-; AVX2-NEXT:    vpextrb $8, %xmm5, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $16777216, %eax ## imm = 0x1000000
 ; AVX2-NEXT:    je LBB8_50
-; AVX2-NEXT:  ## %bb.49: ## %cond.load93
-; AVX2-NEXT:    vmovd {{.*#+}} xmm5 = mem[0],zero,zero,zero
-; AVX2-NEXT:    vpblendd {{.*#+}} ymm3 = ymm5[0],ymm3[1,2,3,4,5,6,7]
+; AVX2-NEXT:  LBB8_49: ## %cond.load93
+; AVX2-NEXT:    vmovd {{.*#+}} xmm4 = mem[0],zero,zero,zero
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0],ymm3[1,2,3,4,5,6,7]
 ; AVX2-NEXT:    addq $4, %rdi
-; AVX2-NEXT:  LBB8_50: ## %else94
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm7, %xmm4
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm4, %xmm4
-; AVX2-NEXT:    vpacksswb %xmm4, %xmm0, %xmm4
-; AVX2-NEXT:    vpextrb $9, %xmm4, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $33554432, %eax ## imm = 0x2000000
 ; AVX2-NEXT:    je LBB8_52
-; AVX2-NEXT:  ## %bb.51: ## %cond.load97
+; AVX2-NEXT:  LBB8_51: ## %cond.load97
 ; AVX2-NEXT:    vinsertps {{.*#+}} xmm4 = xmm3[0],mem[0],xmm3[2,3]
 ; AVX2-NEXT:    vblendps {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
 ; AVX2-NEXT:    addq $4, %rdi
-; AVX2-NEXT:  LBB8_52: ## %else98
-; AVX2-NEXT:    vxorps %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm7, %xmm5
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm5, %xmm5
-; AVX2-NEXT:    vpacksswb %xmm5, %xmm0, %xmm5
-; AVX2-NEXT:    vpextrb $10, %xmm5, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $67108864, %eax ## imm = 0x4000000
 ; AVX2-NEXT:    je LBB8_54
-; AVX2-NEXT:  ## %bb.53: ## %cond.load101
-; AVX2-NEXT:    vinsertps {{.*#+}} xmm5 = xmm3[0,1],mem[0],xmm3[3]
-; AVX2-NEXT:    vblendps {{.*#+}} ymm3 = ymm5[0,1,2,3],ymm3[4,5,6,7]
+; AVX2-NEXT:  LBB8_53: ## %cond.load101
+; AVX2-NEXT:    vinsertps {{.*#+}} xmm4 = xmm3[0,1],mem[0],xmm3[3]
+; AVX2-NEXT:    vblendps {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
 ; AVX2-NEXT:    addq $4, %rdi
-; AVX2-NEXT:  LBB8_54: ## %else102
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm7, %xmm4
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm4, %xmm4
-; AVX2-NEXT:    vpacksswb %xmm4, %xmm0, %xmm4
-; AVX2-NEXT:    vpextrb $11, %xmm4, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $134217728, %eax ## imm = 0x8000000
 ; AVX2-NEXT:    je LBB8_56
-; AVX2-NEXT:  ## %bb.55: ## %cond.load105
+; AVX2-NEXT:  LBB8_55: ## %cond.load105
 ; AVX2-NEXT:    vinsertps {{.*#+}} xmm4 = xmm3[0,1,2],mem[0]
 ; AVX2-NEXT:    vblendps {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
 ; AVX2-NEXT:    addq $4, %rdi
-; AVX2-NEXT:  LBB8_56: ## %else106
-; AVX2-NEXT:    vextracti128 $1, %ymm7, %xmm4
-; AVX2-NEXT:    vxorps %xmm5, %xmm5, %xmm5
-; AVX2-NEXT:    vpcmpeqd %xmm5, %xmm4, %xmm5
-; AVX2-NEXT:    vpackssdw %xmm5, %xmm0, %xmm6
-; AVX2-NEXT:    vpacksswb %xmm6, %xmm0, %xmm6
-; AVX2-NEXT:    vpextrb $12, %xmm6, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $268435456, %eax ## imm = 0x10000000
 ; AVX2-NEXT:    je LBB8_58
-; AVX2-NEXT:  ## %bb.57: ## %cond.load109
-; AVX2-NEXT:    vmovss {{.*#+}} xmm6 = mem[0],zero,zero,zero
-; AVX2-NEXT:    vextractf128 $1, %ymm3, %xmm7
-; AVX2-NEXT:    vblendps {{.*#+}} xmm6 = xmm6[0],xmm7[1,2,3]
-; AVX2-NEXT:    vinsertf128 $1, %xmm6, %ymm3, %ymm3
+; AVX2-NEXT:  LBB8_57: ## %cond.load109
+; AVX2-NEXT:    vmovss {{.*#+}} xmm4 = mem[0],zero,zero,zero
+; AVX2-NEXT:    vextractf128 $1, %ymm3, %xmm5
+; AVX2-NEXT:    vblendps {{.*#+}} xmm4 = xmm4[0],xmm5[1,2,3]
+; AVX2-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
 ; AVX2-NEXT:    addq $4, %rdi
-; AVX2-NEXT:  LBB8_58: ## %else110
-; AVX2-NEXT:    vpackssdw %xmm5, %xmm0, %xmm5
-; AVX2-NEXT:    vpacksswb %xmm5, %xmm0, %xmm5
-; AVX2-NEXT:    vpextrb $13, %xmm5, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $536870912, %eax ## imm = 0x20000000
 ; AVX2-NEXT:    je LBB8_60
-; AVX2-NEXT:  ## %bb.59: ## %cond.load113
-; AVX2-NEXT:    vextractf128 $1, %ymm3, %xmm5
-; AVX2-NEXT:    vinsertps {{.*#+}} xmm5 = xmm5[0],mem[0],xmm5[2,3]
-; AVX2-NEXT:    vinsertf128 $1, %xmm5, %ymm3, %ymm3
+; AVX2-NEXT:  LBB8_59: ## %cond.load113
+; AVX2-NEXT:    vextractf128 $1, %ymm3, %xmm4
+; AVX2-NEXT:    vinsertps {{.*#+}} xmm4 = xmm4[0],mem[0],xmm4[2,3]
+; AVX2-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
 ; AVX2-NEXT:    addq $4, %rdi
-; AVX2-NEXT:  LBB8_60: ## %else114
-; AVX2-NEXT:    vxorps %xmm5, %xmm5, %xmm5
-; AVX2-NEXT:    vpcmpeqd %xmm5, %xmm4, %xmm4
-; AVX2-NEXT:    vpackssdw %xmm4, %xmm0, %xmm5
-; AVX2-NEXT:    vpacksswb %xmm5, %xmm0, %xmm5
-; AVX2-NEXT:    vpextrb $14, %xmm5, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $1073741824, %eax ## imm = 0x40000000
 ; AVX2-NEXT:    je LBB8_62
-; AVX2-NEXT:  ## %bb.61: ## %cond.load117
-; AVX2-NEXT:    vextractf128 $1, %ymm3, %xmm5
-; AVX2-NEXT:    vinsertps {{.*#+}} xmm5 = xmm5[0,1],mem[0],xmm5[3]
-; AVX2-NEXT:    vinsertf128 $1, %xmm5, %ymm3, %ymm3
+; AVX2-NEXT:  LBB8_61: ## %cond.load117
+; AVX2-NEXT:    vextractf128 $1, %ymm3, %xmm4
+; AVX2-NEXT:    vinsertps {{.*#+}} xmm4 = xmm4[0,1],mem[0],xmm4[3]
+; AVX2-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
 ; AVX2-NEXT:    addq $4, %rdi
-; AVX2-NEXT:  LBB8_62: ## %else118
-; AVX2-NEXT:    vpackssdw %xmm4, %xmm0, %xmm4
-; AVX2-NEXT:    vpacksswb %xmm4, %xmm0, %xmm4
-; AVX2-NEXT:    vpextrb $15, %xmm4, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $-2147483648, %eax ## imm = 0x80000000
 ; AVX2-NEXT:    je LBB8_64
-; AVX2-NEXT:  ## %bb.63: ## %cond.load121
+; AVX2-NEXT:  LBB8_63: ## %cond.load121
 ; AVX2-NEXT:    vextractf128 $1, %ymm3, %xmm4
 ; AVX2-NEXT:    vinsertps {{.*#+}} xmm4 = xmm4[0,1,2],mem[0]
 ; AVX2-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
-; AVX2-NEXT:  LBB8_64: ## %else122
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: expandload_v32f32_v32i32:
@@ -2991,18 +2757,18 @@ define <32 x float> @expandload_v32f32_v32i32(float* %base, <32 x float> %src0,
 
 define <2 x i64> @expandload_v2i64_const(i64* %base, <2 x i64> %src0) {
 ; SSE2-LABEL: expandload_v2i64_const:
-; SSE2:       ## %bb.0: ## %else
+; SSE2:       ## %bb.0:
 ; SSE2-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
 ; SSE2-NEXT:    movlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
 ; SSE2-NEXT:    retq
 ;
 ; SSE42-LABEL: expandload_v2i64_const:
-; SSE42:       ## %bb.0: ## %else
+; SSE42:       ## %bb.0:
 ; SSE42-NEXT:    pinsrq $1, (%rdi), %xmm0
 ; SSE42-NEXT:    retq
 ;
 ; AVX1OR2-LABEL: expandload_v2i64_const:
-; AVX1OR2:       ## %bb.0: ## %else
+; AVX1OR2:       ## %bb.0:
 ; AVX1OR2-NEXT:    vpinsrq $1, (%rdi), %xmm0, %xmm0
 ; AVX1OR2-NEXT:    retq
 ;
@@ -3042,113 +2808,119 @@ define <4 x i32> @expandload_v4i32_v4i32(i32* %base, <4 x i32> %src0, <4 x i32>
 ; SSE2:       ## %bb.0:
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
 ; SSE2-NEXT:    pcmpeqd %xmm1, %xmm2
-; SSE2-NEXT:    movd %xmm2, %eax
+; SSE2-NEXT:    movmskps %xmm2, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB10_2
-; SSE2-NEXT:  ## %bb.1: ## %cond.load
-; SSE2-NEXT:    movss {{.*#+}} xmm3 = mem[0],zero,zero,zero
-; SSE2-NEXT:    movss {{.*#+}} xmm0 = xmm3[0],xmm0[1,2,3]
+; SSE2-NEXT:    jne LBB10_1
+; SSE2-NEXT:  ## %bb.2: ## %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne LBB10_3
+; SSE2-NEXT:  LBB10_4: ## %else2
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne LBB10_5
+; SSE2-NEXT:  LBB10_6: ## %else6
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne LBB10_7
+; SSE2-NEXT:  LBB10_8: ## %else10
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  LBB10_1: ## %cond.load
+; SSE2-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE2-NEXT:    movss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
 ; SSE2-NEXT:    addq $4, %rdi
-; SSE2-NEXT:  LBB10_2: ## %else
-; SSE2-NEXT:    pextrw $2, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $2, %al
 ; SSE2-NEXT:    je LBB10_4
-; SSE2-NEXT:  ## %bb.3: ## %cond.load1
-; SSE2-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,0],xmm0[0,0]
-; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[2,0],xmm0[2,3]
+; SSE2-NEXT:  LBB10_3: ## %cond.load1
+; SSE2-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,0],xmm0[0,0]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[2,0],xmm0[2,3]
 ; SSE2-NEXT:    addq $4, %rdi
-; SSE2-NEXT:    movaps %xmm2, %xmm0
-; SSE2-NEXT:  LBB10_4: ## %else2
-; SSE2-NEXT:    xorps %xmm2, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm1
-; SSE2-NEXT:    pextrw $4, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    movaps %xmm1, %xmm0
+; SSE2-NEXT:    testb $4, %al
 ; SSE2-NEXT:    je LBB10_6
-; SSE2-NEXT:  ## %bb.5: ## %cond.load5
-; SSE2-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,0],xmm0[3,0]
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm2[0,2]
+; SSE2-NEXT:  LBB10_5: ## %cond.load5
+; SSE2-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,0],xmm0[3,0]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm1[0,2]
 ; SSE2-NEXT:    addq $4, %rdi
-; SSE2-NEXT:  LBB10_6: ## %else6
-; SSE2-NEXT:    pextrw $6, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je LBB10_8
-; SSE2-NEXT:  ## %bb.7: ## %cond.load9
+; SSE2-NEXT:  LBB10_7: ## %cond.load9
 ; SSE2-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,0],xmm0[2,0]
 ; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,0]
-; SSE2-NEXT:  LBB10_8: ## %else10
 ; SSE2-NEXT:    retq
 ;
 ; SSE42-LABEL: expandload_v4i32_v4i32:
 ; SSE42:       ## %bb.0:
 ; SSE42-NEXT:    pxor %xmm2, %xmm2
 ; SSE42-NEXT:    pcmpeqd %xmm1, %xmm2
-; SSE42-NEXT:    pextrb $0, %xmm2, %eax
+; SSE42-NEXT:    movmskps %xmm2, %eax
 ; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB10_2
-; SSE42-NEXT:  ## %bb.1: ## %cond.load
+; SSE42-NEXT:    jne LBB10_1
+; SSE42-NEXT:  ## %bb.2: ## %else
+; SSE42-NEXT:    testb $2, %al
+; SSE42-NEXT:    jne LBB10_3
+; SSE42-NEXT:  LBB10_4: ## %else2
+; SSE42-NEXT:    testb $4, %al
+; SSE42-NEXT:    jne LBB10_5
+; SSE42-NEXT:  LBB10_6: ## %else6
+; SSE42-NEXT:    testb $8, %al
+; SSE42-NEXT:    jne LBB10_7
+; SSE42-NEXT:  LBB10_8: ## %else10
+; SSE42-NEXT:    retq
+; SSE42-NEXT:  LBB10_1: ## %cond.load
 ; SSE42-NEXT:    pinsrd $0, (%rdi), %xmm0
 ; SSE42-NEXT:    addq $4, %rdi
-; SSE42-NEXT:  LBB10_2: ## %else
-; SSE42-NEXT:    pextrb $4, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $2, %al
 ; SSE42-NEXT:    je LBB10_4
-; SSE42-NEXT:  ## %bb.3: ## %cond.load1
+; SSE42-NEXT:  LBB10_3: ## %cond.load1
 ; SSE42-NEXT:    pinsrd $1, (%rdi), %xmm0
 ; SSE42-NEXT:    addq $4, %rdi
-; SSE42-NEXT:  LBB10_4: ## %else2
-; SSE42-NEXT:    pxor %xmm2, %xmm2
-; SSE42-NEXT:    pcmpeqd %xmm2, %xmm1
-; SSE42-NEXT:    pextrb $8, %xmm1, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $4, %al
 ; SSE42-NEXT:    je LBB10_6
-; SSE42-NEXT:  ## %bb.5: ## %cond.load5
+; SSE42-NEXT:  LBB10_5: ## %cond.load5
 ; SSE42-NEXT:    pinsrd $2, (%rdi), %xmm0
 ; SSE42-NEXT:    addq $4, %rdi
-; SSE42-NEXT:  LBB10_6: ## %else6
-; SSE42-NEXT:    pextrb $12, %xmm1, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $8, %al
 ; SSE42-NEXT:    je LBB10_8
-; SSE42-NEXT:  ## %bb.7: ## %cond.load9
+; SSE42-NEXT:  LBB10_7: ## %cond.load9
 ; SSE42-NEXT:    pinsrd $3, (%rdi), %xmm0
-; SSE42-NEXT:  LBB10_8: ## %else10
 ; SSE42-NEXT:    retq
 ;
 ; AVX1OR2-LABEL: expandload_v4i32_v4i32:
 ; AVX1OR2:       ## %bb.0:
 ; AVX1OR2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1OR2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX1OR2-NEXT:    vpextrb $0, %xmm2, %eax
+; AVX1OR2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
+; AVX1OR2-NEXT:    vmovmskps %xmm1, %eax
 ; AVX1OR2-NEXT:    testb $1, %al
-; AVX1OR2-NEXT:    je LBB10_2
-; AVX1OR2-NEXT:  ## %bb.1: ## %cond.load
+; AVX1OR2-NEXT:    jne LBB10_1
+; AVX1OR2-NEXT:  ## %bb.2: ## %else
+; AVX1OR2-NEXT:    testb $2, %al
+; AVX1OR2-NEXT:    jne LBB10_3
+; AVX1OR2-NEXT:  LBB10_4: ## %else2
+; AVX1OR2-NEXT:    testb $4, %al
+; AVX1OR2-NEXT:    jne LBB10_5
+; AVX1OR2-NEXT:  LBB10_6: ## %else6
+; AVX1OR2-NEXT:    testb $8, %al
+; AVX1OR2-NEXT:    jne LBB10_7
+; AVX1OR2-NEXT:  LBB10_8: ## %else10
+; AVX1OR2-NEXT:    retq
+; AVX1OR2-NEXT:  LBB10_1: ## %cond.load
 ; AVX1OR2-NEXT:    vpinsrd $0, (%rdi), %xmm0, %xmm0
 ; AVX1OR2-NEXT:    addq $4, %rdi
-; AVX1OR2-NEXT:  LBB10_2: ## %else
-; AVX1OR2-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $2, %al
 ; AVX1OR2-NEXT:    je LBB10_4
-; AVX1OR2-NEXT:  ## %bb.3: ## %cond.load1
+; AVX1OR2-NEXT:  LBB10_3: ## %cond.load1
 ; AVX1OR2-NEXT:    vpinsrd $1, (%rdi), %xmm0, %xmm0
 ; AVX1OR2-NEXT:    addq $4, %rdi
-; AVX1OR2-NEXT:  LBB10_4: ## %else2
-; AVX1OR2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1OR2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX1OR2-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $4, %al
 ; AVX1OR2-NEXT:    je LBB10_6
-; AVX1OR2-NEXT:  ## %bb.5: ## %cond.load5
+; AVX1OR2-NEXT:  LBB10_5: ## %cond.load5
 ; AVX1OR2-NEXT:    vpinsrd $2, (%rdi), %xmm0, %xmm0
 ; AVX1OR2-NEXT:    addq $4, %rdi
-; AVX1OR2-NEXT:  LBB10_6: ## %else6
-; AVX1OR2-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $8, %al
 ; AVX1OR2-NEXT:    je LBB10_8
-; AVX1OR2-NEXT:  ## %bb.7: ## %cond.load9
+; AVX1OR2-NEXT:  LBB10_7: ## %cond.load9
 ; AVX1OR2-NEXT:    vpinsrd $3, (%rdi), %xmm0, %xmm0
-; AVX1OR2-NEXT:  LBB10_8: ## %else10
 ; AVX1OR2-NEXT:    retq
 ;
 ; AVX512F-LABEL: expandload_v4i32_v4i32:
@@ -3178,370 +2950,289 @@ define <4 x i32> @expandload_v4i32_v4i32(i32* %base, <4 x i32> %src0, <4 x i32>
 ;
 
 define <8 x i16> @expandload_v8i16_v8i16(i16* %base, <8 x i16> %src0, <8 x i16> %trigger) {
-; SSE2-LABEL: expandload_v8i16_v8i16:
-; SSE2:       ## %bb.0:
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpeqw %xmm1, %xmm2
-; SSE2-NEXT:    movd %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB11_2
-; SSE2-NEXT:  ## %bb.1: ## %cond.load
-; SSE2-NEXT:    pinsrw $0, (%rdi), %xmm0
-; SSE2-NEXT:    addq $2, %rdi
-; SSE2-NEXT:  LBB11_2: ## %else
-; SSE2-NEXT:    shrl $16, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB11_4
-; SSE2-NEXT:  ## %bb.3: ## %cond.load1
-; SSE2-NEXT:    pinsrw $1, (%rdi), %xmm0
-; SSE2-NEXT:    addq $2, %rdi
-; SSE2-NEXT:  LBB11_4: ## %else2
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpeqw %xmm1, %xmm2
-; SSE2-NEXT:    pextrw $2, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB11_6
-; SSE2-NEXT:  ## %bb.5: ## %cond.load5
-; SSE2-NEXT:    pinsrw $2, (%rdi), %xmm0
-; SSE2-NEXT:    addq $2, %rdi
-; SSE2-NEXT:  LBB11_6: ## %else6
-; SSE2-NEXT:    pextrw $3, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB11_8
-; SSE2-NEXT:  ## %bb.7: ## %cond.load9
-; SSE2-NEXT:    pinsrw $3, (%rdi), %xmm0
-; SSE2-NEXT:    addq $2, %rdi
-; SSE2-NEXT:  LBB11_8: ## %else10
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpeqw %xmm1, %xmm2
-; SSE2-NEXT:    pextrw $4, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB11_10
-; SSE2-NEXT:  ## %bb.9: ## %cond.load13
-; SSE2-NEXT:    pinsrw $4, (%rdi), %xmm0
-; SSE2-NEXT:    addq $2, %rdi
-; SSE2-NEXT:  LBB11_10: ## %else14
-; SSE2-NEXT:    pextrw $5, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB11_12
-; SSE2-NEXT:  ## %bb.11: ## %cond.load17
-; SSE2-NEXT:    pinsrw $5, (%rdi), %xmm0
-; SSE2-NEXT:    addq $2, %rdi
-; SSE2-NEXT:  LBB11_12: ## %else18
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpeqw %xmm2, %xmm1
-; SSE2-NEXT:    pextrw $6, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB11_14
-; SSE2-NEXT:  ## %bb.13: ## %cond.load21
-; SSE2-NEXT:    pinsrw $6, (%rdi), %xmm0
-; SSE2-NEXT:    addq $2, %rdi
-; SSE2-NEXT:  LBB11_14: ## %else22
-; SSE2-NEXT:    pextrw $7, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB11_16
-; SSE2-NEXT:  ## %bb.15: ## %cond.load25
-; SSE2-NEXT:    pinsrw $7, (%rdi), %xmm0
-; SSE2-NEXT:  LBB11_16: ## %else26
-; SSE2-NEXT:    retq
-;
-; SSE42-LABEL: expandload_v8i16_v8i16:
-; SSE42:       ## %bb.0:
-; SSE42-NEXT:    pxor %xmm2, %xmm2
-; SSE42-NEXT:    pcmpeqw %xmm1, %xmm2
-; SSE42-NEXT:    pextrb $0, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB11_2
-; SSE42-NEXT:  ## %bb.1: ## %cond.load
-; SSE42-NEXT:    pinsrw $0, (%rdi), %xmm0
-; SSE42-NEXT:    addq $2, %rdi
-; SSE42-NEXT:  LBB11_2: ## %else
-; SSE42-NEXT:    pextrb $2, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB11_4
-; SSE42-NEXT:  ## %bb.3: ## %cond.load1
-; SSE42-NEXT:    pinsrw $1, (%rdi), %xmm0
-; SSE42-NEXT:    addq $2, %rdi
-; SSE42-NEXT:  LBB11_4: ## %else2
-; SSE42-NEXT:    pxor %xmm2, %xmm2
-; SSE42-NEXT:    pcmpeqw %xmm1, %xmm2
-; SSE42-NEXT:    pextrb $4, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB11_6
-; SSE42-NEXT:  ## %bb.5: ## %cond.load5
-; SSE42-NEXT:    pinsrw $2, (%rdi), %xmm0
-; SSE42-NEXT:    addq $2, %rdi
-; SSE42-NEXT:  LBB11_6: ## %else6
-; SSE42-NEXT:    pextrb $6, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB11_8
-; SSE42-NEXT:  ## %bb.7: ## %cond.load9
-; SSE42-NEXT:    pinsrw $3, (%rdi), %xmm0
-; SSE42-NEXT:    addq $2, %rdi
-; SSE42-NEXT:  LBB11_8: ## %else10
-; SSE42-NEXT:    pxor %xmm2, %xmm2
-; SSE42-NEXT:    pcmpeqw %xmm1, %xmm2
-; SSE42-NEXT:    pextrb $8, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB11_10
-; SSE42-NEXT:  ## %bb.9: ## %cond.load13
-; SSE42-NEXT:    pinsrw $4, (%rdi), %xmm0
-; SSE42-NEXT:    addq $2, %rdi
-; SSE42-NEXT:  LBB11_10: ## %else14
-; SSE42-NEXT:    pextrb $10, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB11_12
-; SSE42-NEXT:  ## %bb.11: ## %cond.load17
-; SSE42-NEXT:    pinsrw $5, (%rdi), %xmm0
-; SSE42-NEXT:    addq $2, %rdi
-; SSE42-NEXT:  LBB11_12: ## %else18
-; SSE42-NEXT:    pxor %xmm2, %xmm2
-; SSE42-NEXT:    pcmpeqw %xmm2, %xmm1
-; SSE42-NEXT:    pextrb $12, %xmm1, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB11_14
-; SSE42-NEXT:  ## %bb.13: ## %cond.load21
-; SSE42-NEXT:    pinsrw $6, (%rdi), %xmm0
-; SSE42-NEXT:    addq $2, %rdi
-; SSE42-NEXT:  LBB11_14: ## %else22
-; SSE42-NEXT:    pextrb $14, %xmm1, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB11_16
-; SSE42-NEXT:  ## %bb.15: ## %cond.load25
-; SSE42-NEXT:    pinsrw $7, (%rdi), %xmm0
-; SSE42-NEXT:  LBB11_16: ## %else26
-; SSE42-NEXT:    retq
+; SSE-LABEL: expandload_v8i16_v8i16:
+; SSE:       ## %bb.0:
+; SSE-NEXT:    pxor %xmm2, %xmm2
+; SSE-NEXT:    pcmpeqw %xmm1, %xmm2
+; SSE-NEXT:    packsswb %xmm0, %xmm2
+; SSE-NEXT:    pmovmskb %xmm2, %eax
+; SSE-NEXT:    testb $1, %al
+; SSE-NEXT:    jne LBB11_1
+; SSE-NEXT:  ## %bb.2: ## %else
+; SSE-NEXT:    testb $2, %al
+; SSE-NEXT:    jne LBB11_3
+; SSE-NEXT:  LBB11_4: ## %else2
+; SSE-NEXT:    testb $4, %al
+; SSE-NEXT:    jne LBB11_5
+; SSE-NEXT:  LBB11_6: ## %else6
+; SSE-NEXT:    testb $8, %al
+; SSE-NEXT:    jne LBB11_7
+; SSE-NEXT:  LBB11_8: ## %else10
+; SSE-NEXT:    testb $16, %al
+; SSE-NEXT:    jne LBB11_9
+; SSE-NEXT:  LBB11_10: ## %else14
+; SSE-NEXT:    testb $32, %al
+; SSE-NEXT:    jne LBB11_11
+; SSE-NEXT:  LBB11_12: ## %else18
+; SSE-NEXT:    testb $64, %al
+; SSE-NEXT:    jne LBB11_13
+; SSE-NEXT:  LBB11_14: ## %else22
+; SSE-NEXT:    testb $-128, %al
+; SSE-NEXT:    jne LBB11_15
+; SSE-NEXT:  LBB11_16: ## %else26
+; SSE-NEXT:    retq
+; SSE-NEXT:  LBB11_1: ## %cond.load
+; SSE-NEXT:    pinsrw $0, (%rdi), %xmm0
+; SSE-NEXT:    addq $2, %rdi
+; SSE-NEXT:    testb $2, %al
+; SSE-NEXT:    je LBB11_4
+; SSE-NEXT:  LBB11_3: ## %cond.load1
+; SSE-NEXT:    pinsrw $1, (%rdi), %xmm0
+; SSE-NEXT:    addq $2, %rdi
+; SSE-NEXT:    testb $4, %al
+; SSE-NEXT:    je LBB11_6
+; SSE-NEXT:  LBB11_5: ## %cond.load5
+; SSE-NEXT:    pinsrw $2, (%rdi), %xmm0
+; SSE-NEXT:    addq $2, %rdi
+; SSE-NEXT:    testb $8, %al
+; SSE-NEXT:    je LBB11_8
+; SSE-NEXT:  LBB11_7: ## %cond.load9
+; SSE-NEXT:    pinsrw $3, (%rdi), %xmm0
+; SSE-NEXT:    addq $2, %rdi
+; SSE-NEXT:    testb $16, %al
+; SSE-NEXT:    je LBB11_10
+; SSE-NEXT:  LBB11_9: ## %cond.load13
+; SSE-NEXT:    pinsrw $4, (%rdi), %xmm0
+; SSE-NEXT:    addq $2, %rdi
+; SSE-NEXT:    testb $32, %al
+; SSE-NEXT:    je LBB11_12
+; SSE-NEXT:  LBB11_11: ## %cond.load17
+; SSE-NEXT:    pinsrw $5, (%rdi), %xmm0
+; SSE-NEXT:    addq $2, %rdi
+; SSE-NEXT:    testb $64, %al
+; SSE-NEXT:    je LBB11_14
+; SSE-NEXT:  LBB11_13: ## %cond.load21
+; SSE-NEXT:    pinsrw $6, (%rdi), %xmm0
+; SSE-NEXT:    addq $2, %rdi
+; SSE-NEXT:    testb $-128, %al
+; SSE-NEXT:    je LBB11_16
+; SSE-NEXT:  LBB11_15: ## %cond.load25
+; SSE-NEXT:    pinsrw $7, (%rdi), %xmm0
+; SSE-NEXT:    retq
 ;
 ; AVX1OR2-LABEL: expandload_v8i16_v8i16:
 ; AVX1OR2:       ## %bb.0:
 ; AVX1OR2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1OR2-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm2
-; AVX1OR2-NEXT:    vpextrb $0, %xmm2, %eax
+; AVX1OR2-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm1
+; AVX1OR2-NEXT:    vpacksswb %xmm0, %xmm1, %xmm1
+; AVX1OR2-NEXT:    vpmovmskb %xmm1, %eax
 ; AVX1OR2-NEXT:    testb $1, %al
-; AVX1OR2-NEXT:    je LBB11_2
-; AVX1OR2-NEXT:  ## %bb.1: ## %cond.load
+; AVX1OR2-NEXT:    jne LBB11_1
+; AVX1OR2-NEXT:  ## %bb.2: ## %else
+; AVX1OR2-NEXT:    testb $2, %al
+; AVX1OR2-NEXT:    jne LBB11_3
+; AVX1OR2-NEXT:  LBB11_4: ## %else2
+; AVX1OR2-NEXT:    testb $4, %al
+; AVX1OR2-NEXT:    jne LBB11_5
+; AVX1OR2-NEXT:  LBB11_6: ## %else6
+; AVX1OR2-NEXT:    testb $8, %al
+; AVX1OR2-NEXT:    jne LBB11_7
+; AVX1OR2-NEXT:  LBB11_8: ## %else10
+; AVX1OR2-NEXT:    testb $16, %al
+; AVX1OR2-NEXT:    jne LBB11_9
+; AVX1OR2-NEXT:  LBB11_10: ## %else14
+; AVX1OR2-NEXT:    testb $32, %al
+; AVX1OR2-NEXT:    jne LBB11_11
+; AVX1OR2-NEXT:  LBB11_12: ## %else18
+; AVX1OR2-NEXT:    testb $64, %al
+; AVX1OR2-NEXT:    jne LBB11_13
+; AVX1OR2-NEXT:  LBB11_14: ## %else22
+; AVX1OR2-NEXT:    testb $-128, %al
+; AVX1OR2-NEXT:    jne LBB11_15
+; AVX1OR2-NEXT:  LBB11_16: ## %else26
+; AVX1OR2-NEXT:    retq
+; AVX1OR2-NEXT:  LBB11_1: ## %cond.load
 ; AVX1OR2-NEXT:    vpinsrw $0, (%rdi), %xmm0, %xmm0
 ; AVX1OR2-NEXT:    addq $2, %rdi
-; AVX1OR2-NEXT:  LBB11_2: ## %else
-; AVX1OR2-NEXT:    vpextrb $2, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $2, %al
 ; AVX1OR2-NEXT:    je LBB11_4
-; AVX1OR2-NEXT:  ## %bb.3: ## %cond.load1
+; AVX1OR2-NEXT:  LBB11_3: ## %cond.load1
 ; AVX1OR2-NEXT:    vpinsrw $1, (%rdi), %xmm0, %xmm0
 ; AVX1OR2-NEXT:    addq $2, %rdi
-; AVX1OR2-NEXT:  LBB11_4: ## %else2
-; AVX1OR2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1OR2-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm2
-; AVX1OR2-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $4, %al
 ; AVX1OR2-NEXT:    je LBB11_6
-; AVX1OR2-NEXT:  ## %bb.5: ## %cond.load5
+; AVX1OR2-NEXT:  LBB11_5: ## %cond.load5
 ; AVX1OR2-NEXT:    vpinsrw $2, (%rdi), %xmm0, %xmm0
 ; AVX1OR2-NEXT:    addq $2, %rdi
-; AVX1OR2-NEXT:  LBB11_6: ## %else6
-; AVX1OR2-NEXT:    vpextrb $6, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $8, %al
 ; AVX1OR2-NEXT:    je LBB11_8
-; AVX1OR2-NEXT:  ## %bb.7: ## %cond.load9
+; AVX1OR2-NEXT:  LBB11_7: ## %cond.load9
 ; AVX1OR2-NEXT:    vpinsrw $3, (%rdi), %xmm0, %xmm0
 ; AVX1OR2-NEXT:    addq $2, %rdi
-; AVX1OR2-NEXT:  LBB11_8: ## %else10
-; AVX1OR2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1OR2-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm2
-; AVX1OR2-NEXT:    vpextrb $8, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $16, %al
 ; AVX1OR2-NEXT:    je LBB11_10
-; AVX1OR2-NEXT:  ## %bb.9: ## %cond.load13
+; AVX1OR2-NEXT:  LBB11_9: ## %cond.load13
 ; AVX1OR2-NEXT:    vpinsrw $4, (%rdi), %xmm0, %xmm0
 ; AVX1OR2-NEXT:    addq $2, %rdi
-; AVX1OR2-NEXT:  LBB11_10: ## %else14
-; AVX1OR2-NEXT:    vpextrb $10, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $32, %al
 ; AVX1OR2-NEXT:    je LBB11_12
-; AVX1OR2-NEXT:  ## %bb.11: ## %cond.load17
+; AVX1OR2-NEXT:  LBB11_11: ## %cond.load17
 ; AVX1OR2-NEXT:    vpinsrw $5, (%rdi), %xmm0, %xmm0
 ; AVX1OR2-NEXT:    addq $2, %rdi
-; AVX1OR2-NEXT:  LBB11_12: ## %else18
-; AVX1OR2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1OR2-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm1
-; AVX1OR2-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $64, %al
 ; AVX1OR2-NEXT:    je LBB11_14
-; AVX1OR2-NEXT:  ## %bb.13: ## %cond.load21
+; AVX1OR2-NEXT:  LBB11_13: ## %cond.load21
 ; AVX1OR2-NEXT:    vpinsrw $6, (%rdi), %xmm0, %xmm0
 ; AVX1OR2-NEXT:    addq $2, %rdi
-; AVX1OR2-NEXT:  LBB11_14: ## %else22
-; AVX1OR2-NEXT:    vpextrb $14, %xmm1, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $-128, %al
 ; AVX1OR2-NEXT:    je LBB11_16
-; AVX1OR2-NEXT:  ## %bb.15: ## %cond.load25
+; AVX1OR2-NEXT:  LBB11_15: ## %cond.load25
 ; AVX1OR2-NEXT:    vpinsrw $7, (%rdi), %xmm0, %xmm0
-; AVX1OR2-NEXT:  LBB11_16: ## %else26
 ; AVX1OR2-NEXT:    retq
 ;
 ; AVX512F-LABEL: expandload_v8i16_v8i16:
 ; AVX512F:       ## %bb.0:
 ; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vpmovsxwq %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmq %zmm2, %zmm2, %k0
+; AVX512F-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm1
+; AVX512F-NEXT:    vpmovsxwq %xmm1, %zmm1
+; AVX512F-NEXT:    vptestmq %zmm1, %zmm1, %k0
 ; AVX512F-NEXT:    kmovw %k0, %eax
 ; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je LBB11_2
-; AVX512F-NEXT:  ## %bb.1: ## %cond.load
+; AVX512F-NEXT:    jne LBB11_1
+; AVX512F-NEXT:  ## %bb.2: ## %else
+; AVX512F-NEXT:    testb $2, %al
+; AVX512F-NEXT:    jne LBB11_3
+; AVX512F-NEXT:  LBB11_4: ## %else2
+; AVX512F-NEXT:    testb $4, %al
+; AVX512F-NEXT:    jne LBB11_5
+; AVX512F-NEXT:  LBB11_6: ## %else6
+; AVX512F-NEXT:    testb $8, %al
+; AVX512F-NEXT:    jne LBB11_7
+; AVX512F-NEXT:  LBB11_8: ## %else10
+; AVX512F-NEXT:    testb $16, %al
+; AVX512F-NEXT:    jne LBB11_9
+; AVX512F-NEXT:  LBB11_10: ## %else14
+; AVX512F-NEXT:    testb $32, %al
+; AVX512F-NEXT:    jne LBB11_11
+; AVX512F-NEXT:  LBB11_12: ## %else18
+; AVX512F-NEXT:    testb $64, %al
+; AVX512F-NEXT:    jne LBB11_13
+; AVX512F-NEXT:  LBB11_14: ## %else22
+; AVX512F-NEXT:    testb $-128, %al
+; AVX512F-NEXT:    jne LBB11_15
+; AVX512F-NEXT:  LBB11_16: ## %else26
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+; AVX512F-NEXT:  LBB11_1: ## %cond.load
 ; AVX512F-NEXT:    vpinsrw $0, (%rdi), %xmm0, %xmm0
 ; AVX512F-NEXT:    addq $2, %rdi
-; AVX512F-NEXT:  LBB11_2: ## %else
-; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $2, %al
 ; AVX512F-NEXT:    je LBB11_4
-; AVX512F-NEXT:  ## %bb.3: ## %cond.load1
+; AVX512F-NEXT:  LBB11_3: ## %cond.load1
 ; AVX512F-NEXT:    vpinsrw $1, (%rdi), %xmm0, %xmm0
 ; AVX512F-NEXT:    addq $2, %rdi
-; AVX512F-NEXT:  LBB11_4: ## %else2
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vpmovsxwq %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmq %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $2, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $4, %al
 ; AVX512F-NEXT:    je LBB11_6
-; AVX512F-NEXT:  ## %bb.5: ## %cond.load5
+; AVX512F-NEXT:  LBB11_5: ## %cond.load5
 ; AVX512F-NEXT:    vpinsrw $2, (%rdi), %xmm0, %xmm0
 ; AVX512F-NEXT:    addq $2, %rdi
-; AVX512F-NEXT:  LBB11_6: ## %else6
-; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $8, %al
 ; AVX512F-NEXT:    je LBB11_8
-; AVX512F-NEXT:  ## %bb.7: ## %cond.load9
+; AVX512F-NEXT:  LBB11_7: ## %cond.load9
 ; AVX512F-NEXT:    vpinsrw $3, (%rdi), %xmm0, %xmm0
 ; AVX512F-NEXT:    addq $2, %rdi
-; AVX512F-NEXT:  LBB11_8: ## %else10
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vpmovsxwq %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmq %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $4, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $16, %al
 ; AVX512F-NEXT:    je LBB11_10
-; AVX512F-NEXT:  ## %bb.9: ## %cond.load13
+; AVX512F-NEXT:  LBB11_9: ## %cond.load13
 ; AVX512F-NEXT:    vpinsrw $4, (%rdi), %xmm0, %xmm0
 ; AVX512F-NEXT:    addq $2, %rdi
-; AVX512F-NEXT:  LBB11_10: ## %else14
-; AVX512F-NEXT:    kshiftrw $5, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $32, %al
 ; AVX512F-NEXT:    je LBB11_12
-; AVX512F-NEXT:  ## %bb.11: ## %cond.load17
+; AVX512F-NEXT:  LBB11_11: ## %cond.load17
 ; AVX512F-NEXT:    vpinsrw $5, (%rdi), %xmm0, %xmm0
 ; AVX512F-NEXT:    addq $2, %rdi
-; AVX512F-NEXT:  LBB11_12: ## %else18
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm1
-; AVX512F-NEXT:    vpmovsxwq %xmm1, %zmm1
-; AVX512F-NEXT:    vptestmq %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $6, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $64, %al
 ; AVX512F-NEXT:    je LBB11_14
-; AVX512F-NEXT:  ## %bb.13: ## %cond.load21
+; AVX512F-NEXT:  LBB11_13: ## %cond.load21
 ; AVX512F-NEXT:    vpinsrw $6, (%rdi), %xmm0, %xmm0
 ; AVX512F-NEXT:    addq $2, %rdi
-; AVX512F-NEXT:  LBB11_14: ## %else22
-; AVX512F-NEXT:    kshiftrw $7, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $-128, %al
 ; AVX512F-NEXT:    je LBB11_16
-; AVX512F-NEXT:  ## %bb.15: ## %cond.load25
+; AVX512F-NEXT:  LBB11_15: ## %cond.load25
 ; AVX512F-NEXT:    vpinsrw $7, (%rdi), %xmm0, %xmm0
-; AVX512F-NEXT:  LBB11_16: ## %else26
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VLDQ-LABEL: expandload_v8i16_v8i16:
 ; AVX512VLDQ:       ## %bb.0:
 ; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm2
-; AVX512VLDQ-NEXT:    vpmovsxwd %xmm2, %ymm2
-; AVX512VLDQ-NEXT:    vpmovd2m %ymm2, %k0
+; AVX512VLDQ-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm1
+; AVX512VLDQ-NEXT:    vpmovsxwd %xmm1, %ymm1
+; AVX512VLDQ-NEXT:    vpmovd2m %ymm1, %k0
 ; AVX512VLDQ-NEXT:    kmovw %k0, %eax
 ; AVX512VLDQ-NEXT:    testb $1, %al
-; AVX512VLDQ-NEXT:    je LBB11_2
-; AVX512VLDQ-NEXT:  ## %bb.1: ## %cond.load
+; AVX512VLDQ-NEXT:    jne LBB11_1
+; AVX512VLDQ-NEXT:  ## %bb.2: ## %else
+; AVX512VLDQ-NEXT:    testb $2, %al
+; AVX512VLDQ-NEXT:    jne LBB11_3
+; AVX512VLDQ-NEXT:  LBB11_4: ## %else2
+; AVX512VLDQ-NEXT:    testb $4, %al
+; AVX512VLDQ-NEXT:    jne LBB11_5
+; AVX512VLDQ-NEXT:  LBB11_6: ## %else6
+; AVX512VLDQ-NEXT:    testb $8, %al
+; AVX512VLDQ-NEXT:    jne LBB11_7
+; AVX512VLDQ-NEXT:  LBB11_8: ## %else10
+; AVX512VLDQ-NEXT:    testb $16, %al
+; AVX512VLDQ-NEXT:    jne LBB11_9
+; AVX512VLDQ-NEXT:  LBB11_10: ## %else14
+; AVX512VLDQ-NEXT:    testb $32, %al
+; AVX512VLDQ-NEXT:    jne LBB11_11
+; AVX512VLDQ-NEXT:  LBB11_12: ## %else18
+; AVX512VLDQ-NEXT:    testb $64, %al
+; AVX512VLDQ-NEXT:    jne LBB11_13
+; AVX512VLDQ-NEXT:  LBB11_14: ## %else22
+; AVX512VLDQ-NEXT:    testb $-128, %al
+; AVX512VLDQ-NEXT:    jne LBB11_15
+; AVX512VLDQ-NEXT:  LBB11_16: ## %else26
+; AVX512VLDQ-NEXT:    vzeroupper
+; AVX512VLDQ-NEXT:    retq
+; AVX512VLDQ-NEXT:  LBB11_1: ## %cond.load
 ; AVX512VLDQ-NEXT:    vpinsrw $0, (%rdi), %xmm0, %xmm0
 ; AVX512VLDQ-NEXT:    addq $2, %rdi
-; AVX512VLDQ-NEXT:  LBB11_2: ## %else
-; AVX512VLDQ-NEXT:    kshiftrb $1, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $2, %al
 ; AVX512VLDQ-NEXT:    je LBB11_4
-; AVX512VLDQ-NEXT:  ## %bb.3: ## %cond.load1
+; AVX512VLDQ-NEXT:  LBB11_3: ## %cond.load1
 ; AVX512VLDQ-NEXT:    vpinsrw $1, (%rdi), %xmm0, %xmm0
 ; AVX512VLDQ-NEXT:    addq $2, %rdi
-; AVX512VLDQ-NEXT:  LBB11_4: ## %else2
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm2
-; AVX512VLDQ-NEXT:    vpmovsxwd %xmm2, %ymm2
-; AVX512VLDQ-NEXT:    vpmovd2m %ymm2, %k0
-; AVX512VLDQ-NEXT:    kshiftrb $2, %k0, %k1
-; AVX512VLDQ-NEXT:    kmovw %k1, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $4, %al
 ; AVX512VLDQ-NEXT:    je LBB11_6
-; AVX512VLDQ-NEXT:  ## %bb.5: ## %cond.load5
+; AVX512VLDQ-NEXT:  LBB11_5: ## %cond.load5
 ; AVX512VLDQ-NEXT:    vpinsrw $2, (%rdi), %xmm0, %xmm0
 ; AVX512VLDQ-NEXT:    addq $2, %rdi
-; AVX512VLDQ-NEXT:  LBB11_6: ## %else6
-; AVX512VLDQ-NEXT:    kshiftrb $3, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $8, %al
 ; AVX512VLDQ-NEXT:    je LBB11_8
-; AVX512VLDQ-NEXT:  ## %bb.7: ## %cond.load9
+; AVX512VLDQ-NEXT:  LBB11_7: ## %cond.load9
 ; AVX512VLDQ-NEXT:    vpinsrw $3, (%rdi), %xmm0, %xmm0
 ; AVX512VLDQ-NEXT:    addq $2, %rdi
-; AVX512VLDQ-NEXT:  LBB11_8: ## %else10
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm2
-; AVX512VLDQ-NEXT:    vpmovsxwd %xmm2, %ymm2
-; AVX512VLDQ-NEXT:    vpmovd2m %ymm2, %k0
-; AVX512VLDQ-NEXT:    kshiftrb $4, %k0, %k1
-; AVX512VLDQ-NEXT:    kmovw %k1, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $16, %al
 ; AVX512VLDQ-NEXT:    je LBB11_10
-; AVX512VLDQ-NEXT:  ## %bb.9: ## %cond.load13
+; AVX512VLDQ-NEXT:  LBB11_9: ## %cond.load13
 ; AVX512VLDQ-NEXT:    vpinsrw $4, (%rdi), %xmm0, %xmm0
 ; AVX512VLDQ-NEXT:    addq $2, %rdi
-; AVX512VLDQ-NEXT:  LBB11_10: ## %else14
-; AVX512VLDQ-NEXT:    kshiftrb $5, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $32, %al
 ; AVX512VLDQ-NEXT:    je LBB11_12
-; AVX512VLDQ-NEXT:  ## %bb.11: ## %cond.load17
+; AVX512VLDQ-NEXT:  LBB11_11: ## %cond.load17
 ; AVX512VLDQ-NEXT:    vpinsrw $5, (%rdi), %xmm0, %xmm0
 ; AVX512VLDQ-NEXT:    addq $2, %rdi
-; AVX512VLDQ-NEXT:  LBB11_12: ## %else18
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm1
-; AVX512VLDQ-NEXT:    vpmovsxwd %xmm1, %ymm1
-; AVX512VLDQ-NEXT:    vpmovd2m %ymm1, %k0
-; AVX512VLDQ-NEXT:    kshiftrb $6, %k0, %k1
-; AVX512VLDQ-NEXT:    kmovw %k1, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $64, %al
 ; AVX512VLDQ-NEXT:    je LBB11_14
-; AVX512VLDQ-NEXT:  ## %bb.13: ## %cond.load21
+; AVX512VLDQ-NEXT:  LBB11_13: ## %cond.load21
 ; AVX512VLDQ-NEXT:    vpinsrw $6, (%rdi), %xmm0, %xmm0
 ; AVX512VLDQ-NEXT:    addq $2, %rdi
-; AVX512VLDQ-NEXT:  LBB11_14: ## %else22
-; AVX512VLDQ-NEXT:    kshiftrb $7, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $-128, %al
 ; AVX512VLDQ-NEXT:    je LBB11_16
-; AVX512VLDQ-NEXT:  ## %bb.15: ## %cond.load25
+; AVX512VLDQ-NEXT:  LBB11_15: ## %cond.load25
 ; AVX512VLDQ-NEXT:    vpinsrw $7, (%rdi), %xmm0, %xmm0
-; AVX512VLDQ-NEXT:  LBB11_16: ## %else26
 ; AVX512VLDQ-NEXT:    vzeroupper
 ; AVX512VLDQ-NEXT:    retq
 ;
@@ -3550,66 +3241,67 @@ define <8 x i16> @expandload_v8i16_v8i16(i16* %base, <8 x i16> %src0, <8 x i16>
 ; AVX512VLBW-NEXT:    vptestnmw %xmm1, %xmm1, %k0
 ; AVX512VLBW-NEXT:    kmovd %k0, %eax
 ; AVX512VLBW-NEXT:    testb $1, %al
-; AVX512VLBW-NEXT:    je LBB11_2
-; AVX512VLBW-NEXT:  ## %bb.1: ## %cond.load
+; AVX512VLBW-NEXT:    jne LBB11_1
+; AVX512VLBW-NEXT:  ## %bb.2: ## %else
+; AVX512VLBW-NEXT:    testb $2, %al
+; AVX512VLBW-NEXT:    jne LBB11_3
+; AVX512VLBW-NEXT:  LBB11_4: ## %else2
+; AVX512VLBW-NEXT:    testb $4, %al
+; AVX512VLBW-NEXT:    jne LBB11_5
+; AVX512VLBW-NEXT:  LBB11_6: ## %else6
+; AVX512VLBW-NEXT:    testb $8, %al
+; AVX512VLBW-NEXT:    jne LBB11_7
+; AVX512VLBW-NEXT:  LBB11_8: ## %else10
+; AVX512VLBW-NEXT:    testb $16, %al
+; AVX512VLBW-NEXT:    jne LBB11_9
+; AVX512VLBW-NEXT:  LBB11_10: ## %else14
+; AVX512VLBW-NEXT:    testb $32, %al
+; AVX512VLBW-NEXT:    jne LBB11_11
+; AVX512VLBW-NEXT:  LBB11_12: ## %else18
+; AVX512VLBW-NEXT:    testb $64, %al
+; AVX512VLBW-NEXT:    jne LBB11_13
+; AVX512VLBW-NEXT:  LBB11_14: ## %else22
+; AVX512VLBW-NEXT:    testb $-128, %al
+; AVX512VLBW-NEXT:    jne LBB11_15
+; AVX512VLBW-NEXT:  LBB11_16: ## %else26
+; AVX512VLBW-NEXT:    retq
+; AVX512VLBW-NEXT:  LBB11_1: ## %cond.load
 ; AVX512VLBW-NEXT:    vpinsrw $0, (%rdi), %xmm0, %xmm0
 ; AVX512VLBW-NEXT:    addq $2, %rdi
-; AVX512VLBW-NEXT:  LBB11_2: ## %else
-; AVX512VLBW-NEXT:    kshiftrw $1, %k0, %k1
-; AVX512VLBW-NEXT:    kmovd %k1, %eax
-; AVX512VLBW-NEXT:    testb $1, %al
+; AVX512VLBW-NEXT:    testb $2, %al
 ; AVX512VLBW-NEXT:    je LBB11_4
-; AVX512VLBW-NEXT:  ## %bb.3: ## %cond.load1
+; AVX512VLBW-NEXT:  LBB11_3: ## %cond.load1
 ; AVX512VLBW-NEXT:    vpinsrw $1, (%rdi), %xmm0, %xmm0
 ; AVX512VLBW-NEXT:    addq $2, %rdi
-; AVX512VLBW-NEXT:  LBB11_4: ## %else2
-; AVX512VLBW-NEXT:    kshiftrw $2, %k0, %k1
-; AVX512VLBW-NEXT:    kmovd %k1, %eax
-; AVX512VLBW-NEXT:    testb $1, %al
+; AVX512VLBW-NEXT:    testb $4, %al
 ; AVX512VLBW-NEXT:    je LBB11_6
-; AVX512VLBW-NEXT:  ## %bb.5: ## %cond.load5
+; AVX512VLBW-NEXT:  LBB11_5: ## %cond.load5
 ; AVX512VLBW-NEXT:    vpinsrw $2, (%rdi), %xmm0, %xmm0
 ; AVX512VLBW-NEXT:    addq $2, %rdi
-; AVX512VLBW-NEXT:  LBB11_6: ## %else6
-; AVX512VLBW-NEXT:    kshiftrw $3, %k0, %k1
-; AVX512VLBW-NEXT:    kmovd %k1, %eax
-; AVX512VLBW-NEXT:    testb $1, %al
+; AVX512VLBW-NEXT:    testb $8, %al
 ; AVX512VLBW-NEXT:    je LBB11_8
-; AVX512VLBW-NEXT:  ## %bb.7: ## %cond.load9
+; AVX512VLBW-NEXT:  LBB11_7: ## %cond.load9
 ; AVX512VLBW-NEXT:    vpinsrw $3, (%rdi), %xmm0, %xmm0
 ; AVX512VLBW-NEXT:    addq $2, %rdi
-; AVX512VLBW-NEXT:  LBB11_8: ## %else10
-; AVX512VLBW-NEXT:    kshiftrw $4, %k0, %k1
-; AVX512VLBW-NEXT:    kmovd %k1, %eax
-; AVX512VLBW-NEXT:    testb $1, %al
+; AVX512VLBW-NEXT:    testb $16, %al
 ; AVX512VLBW-NEXT:    je LBB11_10
-; AVX512VLBW-NEXT:  ## %bb.9: ## %cond.load13
+; AVX512VLBW-NEXT:  LBB11_9: ## %cond.load13
 ; AVX512VLBW-NEXT:    vpinsrw $4, (%rdi), %xmm0, %xmm0
 ; AVX512VLBW-NEXT:    addq $2, %rdi
-; AVX512VLBW-NEXT:  LBB11_10: ## %else14
-; AVX512VLBW-NEXT:    kshiftrw $5, %k0, %k1
-; AVX512VLBW-NEXT:    kmovd %k1, %eax
-; AVX512VLBW-NEXT:    testb $1, %al
+; AVX512VLBW-NEXT:    testb $32, %al
 ; AVX512VLBW-NEXT:    je LBB11_12
-; AVX512VLBW-NEXT:  ## %bb.11: ## %cond.load17
+; AVX512VLBW-NEXT:  LBB11_11: ## %cond.load17
 ; AVX512VLBW-NEXT:    vpinsrw $5, (%rdi), %xmm0, %xmm0
 ; AVX512VLBW-NEXT:    addq $2, %rdi
-; AVX512VLBW-NEXT:  LBB11_12: ## %else18
-; AVX512VLBW-NEXT:    kshiftrw $6, %k0, %k1
-; AVX512VLBW-NEXT:    kmovd %k1, %eax
-; AVX512VLBW-NEXT:    testb $1, %al
+; AVX512VLBW-NEXT:    testb $64, %al
 ; AVX512VLBW-NEXT:    je LBB11_14
-; AVX512VLBW-NEXT:  ## %bb.13: ## %cond.load21
+; AVX512VLBW-NEXT:  LBB11_13: ## %cond.load21
 ; AVX512VLBW-NEXT:    vpinsrw $6, (%rdi), %xmm0, %xmm0
 ; AVX512VLBW-NEXT:    addq $2, %rdi
-; AVX512VLBW-NEXT:  LBB11_14: ## %else22
-; AVX512VLBW-NEXT:    kshiftrw $7, %k0, %k0
-; AVX512VLBW-NEXT:    kmovd %k0, %eax
-; AVX512VLBW-NEXT:    testb $1, %al
+; AVX512VLBW-NEXT:    testb $-128, %al
 ; AVX512VLBW-NEXT:    je LBB11_16
-; AVX512VLBW-NEXT:  ## %bb.15: ## %cond.load25
+; AVX512VLBW-NEXT:  LBB11_15: ## %cond.load25
 ; AVX512VLBW-NEXT:    vpinsrw $7, (%rdi), %xmm0, %xmm0
-; AVX512VLBW-NEXT:  LBB11_16: ## %else26
 ; AVX512VLBW-NEXT:    retq
   %mask = icmp eq <8 x i16> %trigger, zeroinitializer
   %res = call <8 x i16> @llvm.masked.expandload.v8i16(i16* %base, <8 x i1> %mask, <8 x i16> %src0)
@@ -3625,207 +3317,210 @@ define <16 x i8> @expandload_v16i8_v16i8(i8* %base, <16 x i8> %src0, <16 x i8> %
 ; SSE2:       ## %bb.0:
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
 ; SSE2-NEXT:    pcmpeqb %xmm1, %xmm2
-; SSE2-NEXT:    movd %xmm2, %eax
+; SSE2-NEXT:    pmovmskb %xmm2, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB12_2
-; SSE2-NEXT:  ## %bb.1: ## %cond.load
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    jne LBB12_1
+; SSE2-NEXT:  ## %bb.2: ## %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne LBB12_3
+; SSE2-NEXT:  LBB12_4: ## %else2
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne LBB12_5
+; SSE2-NEXT:  LBB12_6: ## %else6
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne LBB12_7
+; SSE2-NEXT:  LBB12_8: ## %else10
+; SSE2-NEXT:    testb $16, %al
+; SSE2-NEXT:    jne LBB12_9
+; SSE2-NEXT:  LBB12_10: ## %else14
+; SSE2-NEXT:    testb $32, %al
+; SSE2-NEXT:    jne LBB12_11
+; SSE2-NEXT:  LBB12_12: ## %else18
+; SSE2-NEXT:    testb $64, %al
+; SSE2-NEXT:    jne LBB12_13
+; SSE2-NEXT:  LBB12_14: ## %else22
+; SSE2-NEXT:    testb $-128, %al
+; SSE2-NEXT:    jne LBB12_15
+; SSE2-NEXT:  LBB12_16: ## %else26
+; SSE2-NEXT:    testl $256, %eax ## imm = 0x100
+; SSE2-NEXT:    jne LBB12_17
+; SSE2-NEXT:  LBB12_18: ## %else30
+; SSE2-NEXT:    testl $512, %eax ## imm = 0x200
+; SSE2-NEXT:    jne LBB12_19
+; SSE2-NEXT:  LBB12_20: ## %else34
+; SSE2-NEXT:    testl $1024, %eax ## imm = 0x400
+; SSE2-NEXT:    jne LBB12_21
+; SSE2-NEXT:  LBB12_22: ## %else38
+; SSE2-NEXT:    testl $2048, %eax ## imm = 0x800
+; SSE2-NEXT:    jne LBB12_23
+; SSE2-NEXT:  LBB12_24: ## %else42
+; SSE2-NEXT:    testl $4096, %eax ## imm = 0x1000
+; SSE2-NEXT:    jne LBB12_25
+; SSE2-NEXT:  LBB12_26: ## %else46
+; SSE2-NEXT:    testl $8192, %eax ## imm = 0x2000
+; SSE2-NEXT:    jne LBB12_27
+; SSE2-NEXT:  LBB12_28: ## %else50
+; SSE2-NEXT:    testl $16384, %eax ## imm = 0x4000
+; SSE2-NEXT:    jne LBB12_29
+; SSE2-NEXT:  LBB12_30: ## %else54
+; SSE2-NEXT:    testl $32768, %eax ## imm = 0x8000
+; SSE2-NEXT:    jne LBB12_31
+; SSE2-NEXT:  LBB12_32: ## %else58
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  LBB12_1: ## %cond.load
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; SSE2-NEXT:    pand %xmm1, %xmm0
 ; SSE2-NEXT:    movzbl (%rdi), %ecx
-; SSE2-NEXT:    movd %ecx, %xmm3
-; SSE2-NEXT:    pandn %xmm3, %xmm2
-; SSE2-NEXT:    por %xmm2, %xmm0
+; SSE2-NEXT:    movd %ecx, %xmm2
+; SSE2-NEXT:    pandn %xmm2, %xmm1
+; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    incq %rdi
-; SSE2-NEXT:  LBB12_2: ## %else
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $2, %al
 ; SSE2-NEXT:    je LBB12_4
-; SSE2-NEXT:  ## %bb.3: ## %cond.load1
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    movzbl (%rdi), %eax
-; SSE2-NEXT:    movd %eax, %xmm3
-; SSE2-NEXT:    psllw $8, %xmm3
-; SSE2-NEXT:    pandn %xmm3, %xmm2
-; SSE2-NEXT:    por %xmm2, %xmm0
+; SSE2-NEXT:  LBB12_3: ## %cond.load1
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; SSE2-NEXT:    pand %xmm1, %xmm0
+; SSE2-NEXT:    movzbl (%rdi), %ecx
+; SSE2-NEXT:    movd %ecx, %xmm2
+; SSE2-NEXT:    psllw $8, %xmm2
+; SSE2-NEXT:    pandn %xmm2, %xmm1
+; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    incq %rdi
-; SSE2-NEXT:  LBB12_4: ## %else2
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpeqb %xmm1, %xmm2
-; SSE2-NEXT:    movd %xmm2, %eax
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    shrl $16, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testb $4, %al
 ; SSE2-NEXT:    je LBB12_6
-; SSE2-NEXT:  ## %bb.5: ## %cond.load5
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:  LBB12_5: ## %cond.load5
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; SSE2-NEXT:    pand %xmm1, %xmm0
 ; SSE2-NEXT:    movzbl (%rdi), %ecx
-; SSE2-NEXT:    movd %ecx, %xmm3
-; SSE2-NEXT:    pslld $16, %xmm3
-; SSE2-NEXT:    pandn %xmm3, %xmm2
-; SSE2-NEXT:    por %xmm2, %xmm0
+; SSE2-NEXT:    movd %ecx, %xmm2
+; SSE2-NEXT:    pslld $16, %xmm2
+; SSE2-NEXT:    pandn %xmm2, %xmm1
+; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    incq %rdi
-; SSE2-NEXT:  LBB12_6: ## %else6
-; SSE2-NEXT:    shrl $24, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je LBB12_8
-; SSE2-NEXT:  ## %bb.7: ## %cond.load9
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255]
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    movzbl (%rdi), %eax
-; SSE2-NEXT:    movd %eax, %xmm3
-; SSE2-NEXT:    pslld $24, %xmm3
-; SSE2-NEXT:    pandn %xmm3, %xmm2
-; SSE2-NEXT:    por %xmm2, %xmm0
+; SSE2-NEXT:  LBB12_7: ## %cond.load9
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255]
+; SSE2-NEXT:    pand %xmm1, %xmm0
+; SSE2-NEXT:    movzbl (%rdi), %ecx
+; SSE2-NEXT:    movd %ecx, %xmm2
+; SSE2-NEXT:    pslld $24, %xmm2
+; SSE2-NEXT:    pandn %xmm2, %xmm1
+; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    incq %rdi
-; SSE2-NEXT:  LBB12_8: ## %else10
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpeqb %xmm1, %xmm2
-; SSE2-NEXT:    pextrw $2, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $16, %al
 ; SSE2-NEXT:    je LBB12_10
-; SSE2-NEXT:  ## %bb.9: ## %cond.load13
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255]
-; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:  LBB12_9: ## %cond.load13
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255]
+; SSE2-NEXT:    pand %xmm1, %xmm0
 ; SSE2-NEXT:    movzbl (%rdi), %ecx
-; SSE2-NEXT:    movd %ecx, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,0,1,1]
-; SSE2-NEXT:    pandn %xmm3, %xmm2
-; SSE2-NEXT:    por %xmm2, %xmm0
+; SSE2-NEXT:    movd %ecx, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,0,1,1]
+; SSE2-NEXT:    pandn %xmm2, %xmm1
+; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    incq %rdi
-; SSE2-NEXT:  LBB12_10: ## %else14
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $32, %al
 ; SSE2-NEXT:    je LBB12_12
-; SSE2-NEXT:  ## %bb.11: ## %cond.load17
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255]
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    movzbl (%rdi), %eax
-; SSE2-NEXT:    movd %eax, %xmm3
-; SSE2-NEXT:    psllq $40, %xmm3
-; SSE2-NEXT:    pandn %xmm3, %xmm2
-; SSE2-NEXT:    por %xmm2, %xmm0
+; SSE2-NEXT:  LBB12_11: ## %cond.load17
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255]
+; SSE2-NEXT:    pand %xmm1, %xmm0
+; SSE2-NEXT:    movzbl (%rdi), %ecx
+; SSE2-NEXT:    movd %ecx, %xmm2
+; SSE2-NEXT:    psllq $40, %xmm2
+; SSE2-NEXT:    pandn %xmm2, %xmm1
+; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    incq %rdi
-; SSE2-NEXT:  LBB12_12: ## %else18
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpeqb %xmm1, %xmm2
-; SSE2-NEXT:    pextrw $3, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $64, %al
 ; SSE2-NEXT:    je LBB12_14
-; SSE2-NEXT:  ## %bb.13: ## %cond.load21
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255]
-; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:  LBB12_13: ## %cond.load21
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255]
+; SSE2-NEXT:    pand %xmm1, %xmm0
 ; SSE2-NEXT:    movzbl (%rdi), %ecx
-; SSE2-NEXT:    movd %ecx, %xmm3
-; SSE2-NEXT:    psllq $48, %xmm3
-; SSE2-NEXT:    pandn %xmm3, %xmm2
-; SSE2-NEXT:    por %xmm2, %xmm0
+; SSE2-NEXT:    movd %ecx, %xmm2
+; SSE2-NEXT:    psllq $48, %xmm2
+; SSE2-NEXT:    pandn %xmm2, %xmm1
+; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    incq %rdi
-; SSE2-NEXT:  LBB12_14: ## %else22
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $-128, %al
 ; SSE2-NEXT:    je LBB12_16
-; SSE2-NEXT:  ## %bb.15: ## %cond.load25
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255]
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    movzbl (%rdi), %eax
-; SSE2-NEXT:    movd %eax, %xmm3
-; SSE2-NEXT:    psllq $56, %xmm3
-; SSE2-NEXT:    pandn %xmm3, %xmm2
-; SSE2-NEXT:    por %xmm2, %xmm0
+; SSE2-NEXT:  LBB12_15: ## %cond.load25
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255]
+; SSE2-NEXT:    pand %xmm1, %xmm0
+; SSE2-NEXT:    movzbl (%rdi), %ecx
+; SSE2-NEXT:    movd %ecx, %xmm2
+; SSE2-NEXT:    psllq $56, %xmm2
+; SSE2-NEXT:    pandn %xmm2, %xmm1
+; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    incq %rdi
-; SSE2-NEXT:  LBB12_16: ## %else26
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpeqb %xmm1, %xmm2
-; SSE2-NEXT:    pextrw $4, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $256, %eax ## imm = 0x100
 ; SSE2-NEXT:    je LBB12_18
-; SSE2-NEXT:  ## %bb.17: ## %cond.load29
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255]
-; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:  LBB12_17: ## %cond.load29
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255]
+; SSE2-NEXT:    pand %xmm1, %xmm0
 ; SSE2-NEXT:    movzbl (%rdi), %ecx
-; SSE2-NEXT:    movd %ecx, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,1,0,1]
-; SSE2-NEXT:    pandn %xmm3, %xmm2
-; SSE2-NEXT:    por %xmm2, %xmm0
+; SSE2-NEXT:    movd %ecx, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,1,0,1]
+; SSE2-NEXT:    pandn %xmm2, %xmm1
+; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    incq %rdi
-; SSE2-NEXT:  LBB12_18: ## %else30
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $512, %eax ## imm = 0x200
 ; SSE2-NEXT:    je LBB12_20
-; SSE2-NEXT:  ## %bb.19: ## %cond.load33
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255]
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    movzbl (%rdi), %eax
-; SSE2-NEXT:    movd %eax, %xmm3
-; SSE2-NEXT:    pslldq {{.*#+}} xmm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm3[0,1,2,3,4,5,6]
-; SSE2-NEXT:    pandn %xmm3, %xmm2
-; SSE2-NEXT:    por %xmm2, %xmm0
+; SSE2-NEXT:  LBB12_19: ## %cond.load33
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255]
+; SSE2-NEXT:    pand %xmm1, %xmm0
+; SSE2-NEXT:    movzbl (%rdi), %ecx
+; SSE2-NEXT:    movd %ecx, %xmm2
+; SSE2-NEXT:    pslldq {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm2[0,1,2,3,4,5,6]
+; SSE2-NEXT:    pandn %xmm2, %xmm1
+; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    incq %rdi
-; SSE2-NEXT:  LBB12_20: ## %else34
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpeqb %xmm1, %xmm2
-; SSE2-NEXT:    pextrw $5, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $1024, %eax ## imm = 0x400
 ; SSE2-NEXT:    je LBB12_22
-; SSE2-NEXT:  ## %bb.21: ## %cond.load37
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255]
-; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:  LBB12_21: ## %cond.load37
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255]
+; SSE2-NEXT:    pand %xmm1, %xmm0
 ; SSE2-NEXT:    movzbl (%rdi), %ecx
-; SSE2-NEXT:    movd %ecx, %xmm3
-; SSE2-NEXT:    pslldq {{.*#+}} xmm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm3[0,1,2,3,4,5]
-; SSE2-NEXT:    pandn %xmm3, %xmm2
-; SSE2-NEXT:    por %xmm2, %xmm0
+; SSE2-NEXT:    movd %ecx, %xmm2
+; SSE2-NEXT:    pslldq {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm2[0,1,2,3,4,5]
+; SSE2-NEXT:    pandn %xmm2, %xmm1
+; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    incq %rdi
-; SSE2-NEXT:  LBB12_22: ## %else38
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $2048, %eax ## imm = 0x800
 ; SSE2-NEXT:    je LBB12_24
-; SSE2-NEXT:  ## %bb.23: ## %cond.load41
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255]
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    movzbl (%rdi), %eax
-; SSE2-NEXT:    movd %eax, %xmm3
-; SSE2-NEXT:    pslldq {{.*#+}} xmm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm3[0,1,2,3,4]
-; SSE2-NEXT:    pandn %xmm3, %xmm2
-; SSE2-NEXT:    por %xmm2, %xmm0
+; SSE2-NEXT:  LBB12_23: ## %cond.load41
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255]
+; SSE2-NEXT:    pand %xmm1, %xmm0
+; SSE2-NEXT:    movzbl (%rdi), %ecx
+; SSE2-NEXT:    movd %ecx, %xmm2
+; SSE2-NEXT:    pslldq {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm2[0,1,2,3,4]
+; SSE2-NEXT:    pandn %xmm2, %xmm1
+; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    incq %rdi
-; SSE2-NEXT:  LBB12_24: ## %else42
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpeqb %xmm1, %xmm2
-; SSE2-NEXT:    pextrw $6, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $4096, %eax ## imm = 0x1000
 ; SSE2-NEXT:    je LBB12_26
-; SSE2-NEXT:  ## %bb.25: ## %cond.load45
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255]
-; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:  LBB12_25: ## %cond.load45
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255]
+; SSE2-NEXT:    pand %xmm1, %xmm0
 ; SSE2-NEXT:    movzbl (%rdi), %ecx
-; SSE2-NEXT:    movd %ecx, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,1,2,0]
-; SSE2-NEXT:    pandn %xmm3, %xmm2
-; SSE2-NEXT:    por %xmm2, %xmm0
+; SSE2-NEXT:    movd %ecx, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,1,2,0]
+; SSE2-NEXT:    pandn %xmm2, %xmm1
+; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    incq %rdi
-; SSE2-NEXT:  LBB12_26: ## %else46
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $8192, %eax ## imm = 0x2000
 ; SSE2-NEXT:    je LBB12_28
-; SSE2-NEXT:  ## %bb.27: ## %cond.load49
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255]
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    movzbl (%rdi), %eax
-; SSE2-NEXT:    movd %eax, %xmm3
-; SSE2-NEXT:    pslldq {{.*#+}} xmm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm3[0,1,2]
-; SSE2-NEXT:    pandn %xmm3, %xmm2
-; SSE2-NEXT:    por %xmm2, %xmm0
+; SSE2-NEXT:  LBB12_27: ## %cond.load49
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255]
+; SSE2-NEXT:    pand %xmm1, %xmm0
+; SSE2-NEXT:    movzbl (%rdi), %ecx
+; SSE2-NEXT:    movd %ecx, %xmm2
+; SSE2-NEXT:    pslldq {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm2[0,1,2]
+; SSE2-NEXT:    pandn %xmm2, %xmm1
+; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    incq %rdi
-; SSE2-NEXT:  LBB12_28: ## %else50
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpeqb %xmm2, %xmm1
-; SSE2-NEXT:    pextrw $7, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $16384, %eax ## imm = 0x4000
 ; SSE2-NEXT:    je LBB12_30
-; SSE2-NEXT:  ## %bb.29: ## %cond.load53
+; SSE2-NEXT:  LBB12_29: ## %cond.load53
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255]
 ; SSE2-NEXT:    pand %xmm1, %xmm0
 ; SSE2-NEXT:    movzbl (%rdi), %ecx
@@ -3834,605 +3529,546 @@ define <16 x i8> @expandload_v16i8_v16i8(i8* %base, <16 x i8> %src0, <16 x i8> %
 ; SSE2-NEXT:    pandn %xmm2, %xmm1
 ; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    incq %rdi
-; SSE2-NEXT:  LBB12_30: ## %else54
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $32768, %eax ## imm = 0x8000
 ; SSE2-NEXT:    je LBB12_32
-; SSE2-NEXT:  ## %bb.31: ## %cond.load57
+; SSE2-NEXT:  LBB12_31: ## %cond.load57
 ; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
 ; SSE2-NEXT:    movzbl (%rdi), %eax
 ; SSE2-NEXT:    movd %eax, %xmm1
 ; SSE2-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0]
 ; SSE2-NEXT:    por %xmm1, %xmm0
-; SSE2-NEXT:  LBB12_32: ## %else58
 ; SSE2-NEXT:    retq
 ;
 ; SSE42-LABEL: expandload_v16i8_v16i8:
 ; SSE42:       ## %bb.0:
 ; SSE42-NEXT:    pxor %xmm2, %xmm2
 ; SSE42-NEXT:    pcmpeqb %xmm1, %xmm2
-; SSE42-NEXT:    pextrb $0, %xmm2, %eax
+; SSE42-NEXT:    pmovmskb %xmm2, %eax
 ; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB12_2
-; SSE42-NEXT:  ## %bb.1: ## %cond.load
+; SSE42-NEXT:    jne LBB12_1
+; SSE42-NEXT:  ## %bb.2: ## %else
+; SSE42-NEXT:    testb $2, %al
+; SSE42-NEXT:    jne LBB12_3
+; SSE42-NEXT:  LBB12_4: ## %else2
+; SSE42-NEXT:    testb $4, %al
+; SSE42-NEXT:    jne LBB12_5
+; SSE42-NEXT:  LBB12_6: ## %else6
+; SSE42-NEXT:    testb $8, %al
+; SSE42-NEXT:    jne LBB12_7
+; SSE42-NEXT:  LBB12_8: ## %else10
+; SSE42-NEXT:    testb $16, %al
+; SSE42-NEXT:    jne LBB12_9
+; SSE42-NEXT:  LBB12_10: ## %else14
+; SSE42-NEXT:    testb $32, %al
+; SSE42-NEXT:    jne LBB12_11
+; SSE42-NEXT:  LBB12_12: ## %else18
+; SSE42-NEXT:    testb $64, %al
+; SSE42-NEXT:    jne LBB12_13
+; SSE42-NEXT:  LBB12_14: ## %else22
+; SSE42-NEXT:    testb $-128, %al
+; SSE42-NEXT:    jne LBB12_15
+; SSE42-NEXT:  LBB12_16: ## %else26
+; SSE42-NEXT:    testl $256, %eax ## imm = 0x100
+; SSE42-NEXT:    jne LBB12_17
+; SSE42-NEXT:  LBB12_18: ## %else30
+; SSE42-NEXT:    testl $512, %eax ## imm = 0x200
+; SSE42-NEXT:    jne LBB12_19
+; SSE42-NEXT:  LBB12_20: ## %else34
+; SSE42-NEXT:    testl $1024, %eax ## imm = 0x400
+; SSE42-NEXT:    jne LBB12_21
+; SSE42-NEXT:  LBB12_22: ## %else38
+; SSE42-NEXT:    testl $2048, %eax ## imm = 0x800
+; SSE42-NEXT:    jne LBB12_23
+; SSE42-NEXT:  LBB12_24: ## %else42
+; SSE42-NEXT:    testl $4096, %eax ## imm = 0x1000
+; SSE42-NEXT:    jne LBB12_25
+; SSE42-NEXT:  LBB12_26: ## %else46
+; SSE42-NEXT:    testl $8192, %eax ## imm = 0x2000
+; SSE42-NEXT:    jne LBB12_27
+; SSE42-NEXT:  LBB12_28: ## %else50
+; SSE42-NEXT:    testl $16384, %eax ## imm = 0x4000
+; SSE42-NEXT:    jne LBB12_29
+; SSE42-NEXT:  LBB12_30: ## %else54
+; SSE42-NEXT:    testl $32768, %eax ## imm = 0x8000
+; SSE42-NEXT:    jne LBB12_31
+; SSE42-NEXT:  LBB12_32: ## %else58
+; SSE42-NEXT:    retq
+; SSE42-NEXT:  LBB12_1: ## %cond.load
 ; SSE42-NEXT:    pinsrb $0, (%rdi), %xmm0
 ; SSE42-NEXT:    incq %rdi
-; SSE42-NEXT:  LBB12_2: ## %else
-; SSE42-NEXT:    pextrb $1, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $2, %al
 ; SSE42-NEXT:    je LBB12_4
-; SSE42-NEXT:  ## %bb.3: ## %cond.load1
+; SSE42-NEXT:  LBB12_3: ## %cond.load1
 ; SSE42-NEXT:    pinsrb $1, (%rdi), %xmm0
 ; SSE42-NEXT:    incq %rdi
-; SSE42-NEXT:  LBB12_4: ## %else2
-; SSE42-NEXT:    pxor %xmm2, %xmm2
-; SSE42-NEXT:    pcmpeqb %xmm1, %xmm2
-; SSE42-NEXT:    pextrb $2, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $4, %al
 ; SSE42-NEXT:    je LBB12_6
-; SSE42-NEXT:  ## %bb.5: ## %cond.load5
+; SSE42-NEXT:  LBB12_5: ## %cond.load5
 ; SSE42-NEXT:    pinsrb $2, (%rdi), %xmm0
 ; SSE42-NEXT:    incq %rdi
-; SSE42-NEXT:  LBB12_6: ## %else6
-; SSE42-NEXT:    pextrb $3, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $8, %al
 ; SSE42-NEXT:    je LBB12_8
-; SSE42-NEXT:  ## %bb.7: ## %cond.load9
+; SSE42-NEXT:  LBB12_7: ## %cond.load9
 ; SSE42-NEXT:    pinsrb $3, (%rdi), %xmm0
 ; SSE42-NEXT:    incq %rdi
-; SSE42-NEXT:  LBB12_8: ## %else10
-; SSE42-NEXT:    pxor %xmm2, %xmm2
-; SSE42-NEXT:    pcmpeqb %xmm1, %xmm2
-; SSE42-NEXT:    pextrb $4, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $16, %al
 ; SSE42-NEXT:    je LBB12_10
-; SSE42-NEXT:  ## %bb.9: ## %cond.load13
+; SSE42-NEXT:  LBB12_9: ## %cond.load13
 ; SSE42-NEXT:    pinsrb $4, (%rdi), %xmm0
 ; SSE42-NEXT:    incq %rdi
-; SSE42-NEXT:  LBB12_10: ## %else14
-; SSE42-NEXT:    pextrb $5, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $32, %al
 ; SSE42-NEXT:    je LBB12_12
-; SSE42-NEXT:  ## %bb.11: ## %cond.load17
+; SSE42-NEXT:  LBB12_11: ## %cond.load17
 ; SSE42-NEXT:    pinsrb $5, (%rdi), %xmm0
 ; SSE42-NEXT:    incq %rdi
-; SSE42-NEXT:  LBB12_12: ## %else18
-; SSE42-NEXT:    pxor %xmm2, %xmm2
-; SSE42-NEXT:    pcmpeqb %xmm1, %xmm2
-; SSE42-NEXT:    pextrb $6, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $64, %al
 ; SSE42-NEXT:    je LBB12_14
-; SSE42-NEXT:  ## %bb.13: ## %cond.load21
+; SSE42-NEXT:  LBB12_13: ## %cond.load21
 ; SSE42-NEXT:    pinsrb $6, (%rdi), %xmm0
 ; SSE42-NEXT:    incq %rdi
-; SSE42-NEXT:  LBB12_14: ## %else22
-; SSE42-NEXT:    pextrb $7, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $-128, %al
 ; SSE42-NEXT:    je LBB12_16
-; SSE42-NEXT:  ## %bb.15: ## %cond.load25
+; SSE42-NEXT:  LBB12_15: ## %cond.load25
 ; SSE42-NEXT:    pinsrb $7, (%rdi), %xmm0
-; SSE42-NEXT:    incq %rdi
-; SSE42-NEXT:  LBB12_16: ## %else26
-; SSE42-NEXT:    pxor %xmm2, %xmm2
-; SSE42-NEXT:    pcmpeqb %xmm1, %xmm2
-; SSE42-NEXT:    pextrb $8, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    incq %rdi
+; SSE42-NEXT:    testl $256, %eax ## imm = 0x100
 ; SSE42-NEXT:    je LBB12_18
-; SSE42-NEXT:  ## %bb.17: ## %cond.load29
+; SSE42-NEXT:  LBB12_17: ## %cond.load29
 ; SSE42-NEXT:    pinsrb $8, (%rdi), %xmm0
 ; SSE42-NEXT:    incq %rdi
-; SSE42-NEXT:  LBB12_18: ## %else30
-; SSE42-NEXT:    pextrb $9, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $512, %eax ## imm = 0x200
 ; SSE42-NEXT:    je LBB12_20
-; SSE42-NEXT:  ## %bb.19: ## %cond.load33
+; SSE42-NEXT:  LBB12_19: ## %cond.load33
 ; SSE42-NEXT:    pinsrb $9, (%rdi), %xmm0
 ; SSE42-NEXT:    incq %rdi
-; SSE42-NEXT:  LBB12_20: ## %else34
-; SSE42-NEXT:    pxor %xmm2, %xmm2
-; SSE42-NEXT:    pcmpeqb %xmm1, %xmm2
-; SSE42-NEXT:    pextrb $10, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $1024, %eax ## imm = 0x400
 ; SSE42-NEXT:    je LBB12_22
-; SSE42-NEXT:  ## %bb.21: ## %cond.load37
+; SSE42-NEXT:  LBB12_21: ## %cond.load37
 ; SSE42-NEXT:    pinsrb $10, (%rdi), %xmm0
 ; SSE42-NEXT:    incq %rdi
-; SSE42-NEXT:  LBB12_22: ## %else38
-; SSE42-NEXT:    pextrb $11, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $2048, %eax ## imm = 0x800
 ; SSE42-NEXT:    je LBB12_24
-; SSE42-NEXT:  ## %bb.23: ## %cond.load41
+; SSE42-NEXT:  LBB12_23: ## %cond.load41
 ; SSE42-NEXT:    pinsrb $11, (%rdi), %xmm0
 ; SSE42-NEXT:    incq %rdi
-; SSE42-NEXT:  LBB12_24: ## %else42
-; SSE42-NEXT:    pxor %xmm2, %xmm2
-; SSE42-NEXT:    pcmpeqb %xmm1, %xmm2
-; SSE42-NEXT:    pextrb $12, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $4096, %eax ## imm = 0x1000
 ; SSE42-NEXT:    je LBB12_26
-; SSE42-NEXT:  ## %bb.25: ## %cond.load45
+; SSE42-NEXT:  LBB12_25: ## %cond.load45
 ; SSE42-NEXT:    pinsrb $12, (%rdi), %xmm0
 ; SSE42-NEXT:    incq %rdi
-; SSE42-NEXT:  LBB12_26: ## %else46
-; SSE42-NEXT:    pextrb $13, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $8192, %eax ## imm = 0x2000
 ; SSE42-NEXT:    je LBB12_28
-; SSE42-NEXT:  ## %bb.27: ## %cond.load49
+; SSE42-NEXT:  LBB12_27: ## %cond.load49
 ; SSE42-NEXT:    pinsrb $13, (%rdi), %xmm0
 ; SSE42-NEXT:    incq %rdi
-; SSE42-NEXT:  LBB12_28: ## %else50
-; SSE42-NEXT:    pxor %xmm2, %xmm2
-; SSE42-NEXT:    pcmpeqb %xmm2, %xmm1
-; SSE42-NEXT:    pextrb $14, %xmm1, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $16384, %eax ## imm = 0x4000
 ; SSE42-NEXT:    je LBB12_30
-; SSE42-NEXT:  ## %bb.29: ## %cond.load53
+; SSE42-NEXT:  LBB12_29: ## %cond.load53
 ; SSE42-NEXT:    pinsrb $14, (%rdi), %xmm0
 ; SSE42-NEXT:    incq %rdi
-; SSE42-NEXT:  LBB12_30: ## %else54
-; SSE42-NEXT:    pextrb $15, %xmm1, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $32768, %eax ## imm = 0x8000
 ; SSE42-NEXT:    je LBB12_32
-; SSE42-NEXT:  ## %bb.31: ## %cond.load57
+; SSE42-NEXT:  LBB12_31: ## %cond.load57
 ; SSE42-NEXT:    pinsrb $15, (%rdi), %xmm0
-; SSE42-NEXT:  LBB12_32: ## %else58
 ; SSE42-NEXT:    retq
 ;
 ; AVX1OR2-LABEL: expandload_v16i8_v16i8:
 ; AVX1OR2:       ## %bb.0:
 ; AVX1OR2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1OR2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX1OR2-NEXT:    vpextrb $0, %xmm2, %eax
+; AVX1OR2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm1
+; AVX1OR2-NEXT:    vpmovmskb %xmm1, %eax
 ; AVX1OR2-NEXT:    testb $1, %al
-; AVX1OR2-NEXT:    je LBB12_2
-; AVX1OR2-NEXT:  ## %bb.1: ## %cond.load
+; AVX1OR2-NEXT:    jne LBB12_1
+; AVX1OR2-NEXT:  ## %bb.2: ## %else
+; AVX1OR2-NEXT:    testb $2, %al
+; AVX1OR2-NEXT:    jne LBB12_3
+; AVX1OR2-NEXT:  LBB12_4: ## %else2
+; AVX1OR2-NEXT:    testb $4, %al
+; AVX1OR2-NEXT:    jne LBB12_5
+; AVX1OR2-NEXT:  LBB12_6: ## %else6
+; AVX1OR2-NEXT:    testb $8, %al
+; AVX1OR2-NEXT:    jne LBB12_7
+; AVX1OR2-NEXT:  LBB12_8: ## %else10
+; AVX1OR2-NEXT:    testb $16, %al
+; AVX1OR2-NEXT:    jne LBB12_9
+; AVX1OR2-NEXT:  LBB12_10: ## %else14
+; AVX1OR2-NEXT:    testb $32, %al
+; AVX1OR2-NEXT:    jne LBB12_11
+; AVX1OR2-NEXT:  LBB12_12: ## %else18
+; AVX1OR2-NEXT:    testb $64, %al
+; AVX1OR2-NEXT:    jne LBB12_13
+; AVX1OR2-NEXT:  LBB12_14: ## %else22
+; AVX1OR2-NEXT:    testb $-128, %al
+; AVX1OR2-NEXT:    jne LBB12_15
+; AVX1OR2-NEXT:  LBB12_16: ## %else26
+; AVX1OR2-NEXT:    testl $256, %eax ## imm = 0x100
+; AVX1OR2-NEXT:    jne LBB12_17
+; AVX1OR2-NEXT:  LBB12_18: ## %else30
+; AVX1OR2-NEXT:    testl $512, %eax ## imm = 0x200
+; AVX1OR2-NEXT:    jne LBB12_19
+; AVX1OR2-NEXT:  LBB12_20: ## %else34
+; AVX1OR2-NEXT:    testl $1024, %eax ## imm = 0x400
+; AVX1OR2-NEXT:    jne LBB12_21
+; AVX1OR2-NEXT:  LBB12_22: ## %else38
+; AVX1OR2-NEXT:    testl $2048, %eax ## imm = 0x800
+; AVX1OR2-NEXT:    jne LBB12_23
+; AVX1OR2-NEXT:  LBB12_24: ## %else42
+; AVX1OR2-NEXT:    testl $4096, %eax ## imm = 0x1000
+; AVX1OR2-NEXT:    jne LBB12_25
+; AVX1OR2-NEXT:  LBB12_26: ## %else46
+; AVX1OR2-NEXT:    testl $8192, %eax ## imm = 0x2000
+; AVX1OR2-NEXT:    jne LBB12_27
+; AVX1OR2-NEXT:  LBB12_28: ## %else50
+; AVX1OR2-NEXT:    testl $16384, %eax ## imm = 0x4000
+; AVX1OR2-NEXT:    jne LBB12_29
+; AVX1OR2-NEXT:  LBB12_30: ## %else54
+; AVX1OR2-NEXT:    testl $32768, %eax ## imm = 0x8000
+; AVX1OR2-NEXT:    jne LBB12_31
+; AVX1OR2-NEXT:  LBB12_32: ## %else58
+; AVX1OR2-NEXT:    retq
+; AVX1OR2-NEXT:  LBB12_1: ## %cond.load
 ; AVX1OR2-NEXT:    vpinsrb $0, (%rdi), %xmm0, %xmm0
 ; AVX1OR2-NEXT:    incq %rdi
-; AVX1OR2-NEXT:  LBB12_2: ## %else
-; AVX1OR2-NEXT:    vpextrb $1, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $2, %al
 ; AVX1OR2-NEXT:    je LBB12_4
-; AVX1OR2-NEXT:  ## %bb.3: ## %cond.load1
+; AVX1OR2-NEXT:  LBB12_3: ## %cond.load1
 ; AVX1OR2-NEXT:    vpinsrb $1, (%rdi), %xmm0, %xmm0
 ; AVX1OR2-NEXT:    incq %rdi
-; AVX1OR2-NEXT:  LBB12_4: ## %else2
-; AVX1OR2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1OR2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX1OR2-NEXT:    vpextrb $2, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $4, %al
 ; AVX1OR2-NEXT:    je LBB12_6
-; AVX1OR2-NEXT:  ## %bb.5: ## %cond.load5
+; AVX1OR2-NEXT:  LBB12_5: ## %cond.load5
 ; AVX1OR2-NEXT:    vpinsrb $2, (%rdi), %xmm0, %xmm0
 ; AVX1OR2-NEXT:    incq %rdi
-; AVX1OR2-NEXT:  LBB12_6: ## %else6
-; AVX1OR2-NEXT:    vpextrb $3, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $8, %al
 ; AVX1OR2-NEXT:    je LBB12_8
-; AVX1OR2-NEXT:  ## %bb.7: ## %cond.load9
+; AVX1OR2-NEXT:  LBB12_7: ## %cond.load9
 ; AVX1OR2-NEXT:    vpinsrb $3, (%rdi), %xmm0, %xmm0
 ; AVX1OR2-NEXT:    incq %rdi
-; AVX1OR2-NEXT:  LBB12_8: ## %else10
-; AVX1OR2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1OR2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX1OR2-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $16, %al
 ; AVX1OR2-NEXT:    je LBB12_10
-; AVX1OR2-NEXT:  ## %bb.9: ## %cond.load13
+; AVX1OR2-NEXT:  LBB12_9: ## %cond.load13
 ; AVX1OR2-NEXT:    vpinsrb $4, (%rdi), %xmm0, %xmm0
 ; AVX1OR2-NEXT:    incq %rdi
-; AVX1OR2-NEXT:  LBB12_10: ## %else14
-; AVX1OR2-NEXT:    vpextrb $5, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $32, %al
 ; AVX1OR2-NEXT:    je LBB12_12
-; AVX1OR2-NEXT:  ## %bb.11: ## %cond.load17
+; AVX1OR2-NEXT:  LBB12_11: ## %cond.load17
 ; AVX1OR2-NEXT:    vpinsrb $5, (%rdi), %xmm0, %xmm0
 ; AVX1OR2-NEXT:    incq %rdi
-; AVX1OR2-NEXT:  LBB12_12: ## %else18
-; AVX1OR2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1OR2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX1OR2-NEXT:    vpextrb $6, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $64, %al
 ; AVX1OR2-NEXT:    je LBB12_14
-; AVX1OR2-NEXT:  ## %bb.13: ## %cond.load21
+; AVX1OR2-NEXT:  LBB12_13: ## %cond.load21
 ; AVX1OR2-NEXT:    vpinsrb $6, (%rdi), %xmm0, %xmm0
 ; AVX1OR2-NEXT:    incq %rdi
-; AVX1OR2-NEXT:  LBB12_14: ## %else22
-; AVX1OR2-NEXT:    vpextrb $7, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $-128, %al
 ; AVX1OR2-NEXT:    je LBB12_16
-; AVX1OR2-NEXT:  ## %bb.15: ## %cond.load25
+; AVX1OR2-NEXT:  LBB12_15: ## %cond.load25
 ; AVX1OR2-NEXT:    vpinsrb $7, (%rdi), %xmm0, %xmm0
 ; AVX1OR2-NEXT:    incq %rdi
-; AVX1OR2-NEXT:  LBB12_16: ## %else26
-; AVX1OR2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1OR2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX1OR2-NEXT:    vpextrb $8, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testl $256, %eax ## imm = 0x100
 ; AVX1OR2-NEXT:    je LBB12_18
-; AVX1OR2-NEXT:  ## %bb.17: ## %cond.load29
+; AVX1OR2-NEXT:  LBB12_17: ## %cond.load29
 ; AVX1OR2-NEXT:    vpinsrb $8, (%rdi), %xmm0, %xmm0
 ; AVX1OR2-NEXT:    incq %rdi
-; AVX1OR2-NEXT:  LBB12_18: ## %else30
-; AVX1OR2-NEXT:    vpextrb $9, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testl $512, %eax ## imm = 0x200
 ; AVX1OR2-NEXT:    je LBB12_20
-; AVX1OR2-NEXT:  ## %bb.19: ## %cond.load33
+; AVX1OR2-NEXT:  LBB12_19: ## %cond.load33
 ; AVX1OR2-NEXT:    vpinsrb $9, (%rdi), %xmm0, %xmm0
 ; AVX1OR2-NEXT:    incq %rdi
-; AVX1OR2-NEXT:  LBB12_20: ## %else34
-; AVX1OR2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1OR2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX1OR2-NEXT:    vpextrb $10, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testl $1024, %eax ## imm = 0x400
 ; AVX1OR2-NEXT:    je LBB12_22
-; AVX1OR2-NEXT:  ## %bb.21: ## %cond.load37
+; AVX1OR2-NEXT:  LBB12_21: ## %cond.load37
 ; AVX1OR2-NEXT:    vpinsrb $10, (%rdi), %xmm0, %xmm0
 ; AVX1OR2-NEXT:    incq %rdi
-; AVX1OR2-NEXT:  LBB12_22: ## %else38
-; AVX1OR2-NEXT:    vpextrb $11, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testl $2048, %eax ## imm = 0x800
 ; AVX1OR2-NEXT:    je LBB12_24
-; AVX1OR2-NEXT:  ## %bb.23: ## %cond.load41
+; AVX1OR2-NEXT:  LBB12_23: ## %cond.load41
 ; AVX1OR2-NEXT:    vpinsrb $11, (%rdi), %xmm0, %xmm0
 ; AVX1OR2-NEXT:    incq %rdi
-; AVX1OR2-NEXT:  LBB12_24: ## %else42
-; AVX1OR2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1OR2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX1OR2-NEXT:    vpextrb $12, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testl $4096, %eax ## imm = 0x1000
 ; AVX1OR2-NEXT:    je LBB12_26
-; AVX1OR2-NEXT:  ## %bb.25: ## %cond.load45
+; AVX1OR2-NEXT:  LBB12_25: ## %cond.load45
 ; AVX1OR2-NEXT:    vpinsrb $12, (%rdi), %xmm0, %xmm0
 ; AVX1OR2-NEXT:    incq %rdi
-; AVX1OR2-NEXT:  LBB12_26: ## %else46
-; AVX1OR2-NEXT:    vpextrb $13, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testl $8192, %eax ## imm = 0x2000
 ; AVX1OR2-NEXT:    je LBB12_28
-; AVX1OR2-NEXT:  ## %bb.27: ## %cond.load49
+; AVX1OR2-NEXT:  LBB12_27: ## %cond.load49
 ; AVX1OR2-NEXT:    vpinsrb $13, (%rdi), %xmm0, %xmm0
 ; AVX1OR2-NEXT:    incq %rdi
-; AVX1OR2-NEXT:  LBB12_28: ## %else50
-; AVX1OR2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1OR2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm1
-; AVX1OR2-NEXT:    vpextrb $14, %xmm1, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testl $16384, %eax ## imm = 0x4000
 ; AVX1OR2-NEXT:    je LBB12_30
-; AVX1OR2-NEXT:  ## %bb.29: ## %cond.load53
+; AVX1OR2-NEXT:  LBB12_29: ## %cond.load53
 ; AVX1OR2-NEXT:    vpinsrb $14, (%rdi), %xmm0, %xmm0
 ; AVX1OR2-NEXT:    incq %rdi
-; AVX1OR2-NEXT:  LBB12_30: ## %else54
-; AVX1OR2-NEXT:    vpextrb $15, %xmm1, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testl $32768, %eax ## imm = 0x8000
 ; AVX1OR2-NEXT:    je LBB12_32
-; AVX1OR2-NEXT:  ## %bb.31: ## %cond.load57
+; AVX1OR2-NEXT:  LBB12_31: ## %cond.load57
 ; AVX1OR2-NEXT:    vpinsrb $15, (%rdi), %xmm0, %xmm0
-; AVX1OR2-NEXT:  LBB12_32: ## %else58
 ; AVX1OR2-NEXT:    retq
 ;
 ; AVX512F-LABEL: expandload_v16i8_v16i8:
 ; AVX512F:       ## %bb.0:
 ; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm1
+; AVX512F-NEXT:    vpmovmskb %xmm1, %eax
 ; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je LBB12_2
-; AVX512F-NEXT:  ## %bb.1: ## %cond.load
+; AVX512F-NEXT:    jne LBB12_1
+; AVX512F-NEXT:  ## %bb.2: ## %else
+; AVX512F-NEXT:    testb $2, %al
+; AVX512F-NEXT:    jne LBB12_3
+; AVX512F-NEXT:  LBB12_4: ## %else2
+; AVX512F-NEXT:    testb $4, %al
+; AVX512F-NEXT:    jne LBB12_5
+; AVX512F-NEXT:  LBB12_6: ## %else6
+; AVX512F-NEXT:    testb $8, %al
+; AVX512F-NEXT:    jne LBB12_7
+; AVX512F-NEXT:  LBB12_8: ## %else10
+; AVX512F-NEXT:    testb $16, %al
+; AVX512F-NEXT:    jne LBB12_9
+; AVX512F-NEXT:  LBB12_10: ## %else14
+; AVX512F-NEXT:    testb $32, %al
+; AVX512F-NEXT:    jne LBB12_11
+; AVX512F-NEXT:  LBB12_12: ## %else18
+; AVX512F-NEXT:    testb $64, %al
+; AVX512F-NEXT:    jne LBB12_13
+; AVX512F-NEXT:  LBB12_14: ## %else22
+; AVX512F-NEXT:    testb $-128, %al
+; AVX512F-NEXT:    jne LBB12_15
+; AVX512F-NEXT:  LBB12_16: ## %else26
+; AVX512F-NEXT:    testl $256, %eax ## imm = 0x100
+; AVX512F-NEXT:    jne LBB12_17
+; AVX512F-NEXT:  LBB12_18: ## %else30
+; AVX512F-NEXT:    testl $512, %eax ## imm = 0x200
+; AVX512F-NEXT:    jne LBB12_19
+; AVX512F-NEXT:  LBB12_20: ## %else34
+; AVX512F-NEXT:    testl $1024, %eax ## imm = 0x400
+; AVX512F-NEXT:    jne LBB12_21
+; AVX512F-NEXT:  LBB12_22: ## %else38
+; AVX512F-NEXT:    testl $2048, %eax ## imm = 0x800
+; AVX512F-NEXT:    jne LBB12_23
+; AVX512F-NEXT:  LBB12_24: ## %else42
+; AVX512F-NEXT:    testl $4096, %eax ## imm = 0x1000
+; AVX512F-NEXT:    jne LBB12_25
+; AVX512F-NEXT:  LBB12_26: ## %else46
+; AVX512F-NEXT:    testl $8192, %eax ## imm = 0x2000
+; AVX512F-NEXT:    jne LBB12_27
+; AVX512F-NEXT:  LBB12_28: ## %else50
+; AVX512F-NEXT:    testl $16384, %eax ## imm = 0x4000
+; AVX512F-NEXT:    jne LBB12_29
+; AVX512F-NEXT:  LBB12_30: ## %else54
+; AVX512F-NEXT:    testl $32768, %eax ## imm = 0x8000
+; AVX512F-NEXT:    jne LBB12_31
+; AVX512F-NEXT:  LBB12_32: ## %else58
+; AVX512F-NEXT:    retq
+; AVX512F-NEXT:  LBB12_1: ## %cond.load
 ; AVX512F-NEXT:    vpinsrb $0, (%rdi), %xmm0, %xmm0
 ; AVX512F-NEXT:    incq %rdi
-; AVX512F-NEXT:  LBB12_2: ## %else
-; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $2, %al
 ; AVX512F-NEXT:    je LBB12_4
-; AVX512F-NEXT:  ## %bb.3: ## %cond.load1
+; AVX512F-NEXT:  LBB12_3: ## %cond.load1
 ; AVX512F-NEXT:    vpinsrb $1, (%rdi), %xmm0, %xmm0
 ; AVX512F-NEXT:    incq %rdi
-; AVX512F-NEXT:  LBB12_4: ## %else2
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $2, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $4, %al
 ; AVX512F-NEXT:    je LBB12_6
-; AVX512F-NEXT:  ## %bb.5: ## %cond.load5
+; AVX512F-NEXT:  LBB12_5: ## %cond.load5
 ; AVX512F-NEXT:    vpinsrb $2, (%rdi), %xmm0, %xmm0
 ; AVX512F-NEXT:    incq %rdi
-; AVX512F-NEXT:  LBB12_6: ## %else6
-; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $8, %al
 ; AVX512F-NEXT:    je LBB12_8
-; AVX512F-NEXT:  ## %bb.7: ## %cond.load9
+; AVX512F-NEXT:  LBB12_7: ## %cond.load9
 ; AVX512F-NEXT:    vpinsrb $3, (%rdi), %xmm0, %xmm0
 ; AVX512F-NEXT:    incq %rdi
-; AVX512F-NEXT:  LBB12_8: ## %else10
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $4, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $16, %al
 ; AVX512F-NEXT:    je LBB12_10
-; AVX512F-NEXT:  ## %bb.9: ## %cond.load13
+; AVX512F-NEXT:  LBB12_9: ## %cond.load13
 ; AVX512F-NEXT:    vpinsrb $4, (%rdi), %xmm0, %xmm0
 ; AVX512F-NEXT:    incq %rdi
-; AVX512F-NEXT:  LBB12_10: ## %else14
-; AVX512F-NEXT:    kshiftrw $5, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $32, %al
 ; AVX512F-NEXT:    je LBB12_12
-; AVX512F-NEXT:  ## %bb.11: ## %cond.load17
+; AVX512F-NEXT:  LBB12_11: ## %cond.load17
 ; AVX512F-NEXT:    vpinsrb $5, (%rdi), %xmm0, %xmm0
 ; AVX512F-NEXT:    incq %rdi
-; AVX512F-NEXT:  LBB12_12: ## %else18
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $6, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $64, %al
 ; AVX512F-NEXT:    je LBB12_14
-; AVX512F-NEXT:  ## %bb.13: ## %cond.load21
+; AVX512F-NEXT:  LBB12_13: ## %cond.load21
 ; AVX512F-NEXT:    vpinsrb $6, (%rdi), %xmm0, %xmm0
 ; AVX512F-NEXT:    incq %rdi
-; AVX512F-NEXT:  LBB12_14: ## %else22
-; AVX512F-NEXT:    kshiftrw $7, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $-128, %al
 ; AVX512F-NEXT:    je LBB12_16
-; AVX512F-NEXT:  ## %bb.15: ## %cond.load25
+; AVX512F-NEXT:  LBB12_15: ## %cond.load25
 ; AVX512F-NEXT:    vpinsrb $7, (%rdi), %xmm0, %xmm0
 ; AVX512F-NEXT:    incq %rdi
-; AVX512F-NEXT:  LBB12_16: ## %else26
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $8, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $256, %eax ## imm = 0x100
 ; AVX512F-NEXT:    je LBB12_18
-; AVX512F-NEXT:  ## %bb.17: ## %cond.load29
+; AVX512F-NEXT:  LBB12_17: ## %cond.load29
 ; AVX512F-NEXT:    vpinsrb $8, (%rdi), %xmm0, %xmm0
 ; AVX512F-NEXT:    incq %rdi
-; AVX512F-NEXT:  LBB12_18: ## %else30
-; AVX512F-NEXT:    kshiftrw $9, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $512, %eax ## imm = 0x200
 ; AVX512F-NEXT:    je LBB12_20
-; AVX512F-NEXT:  ## %bb.19: ## %cond.load33
+; AVX512F-NEXT:  LBB12_19: ## %cond.load33
 ; AVX512F-NEXT:    vpinsrb $9, (%rdi), %xmm0, %xmm0
 ; AVX512F-NEXT:    incq %rdi
-; AVX512F-NEXT:  LBB12_20: ## %else34
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $10, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $1024, %eax ## imm = 0x400
 ; AVX512F-NEXT:    je LBB12_22
-; AVX512F-NEXT:  ## %bb.21: ## %cond.load37
+; AVX512F-NEXT:  LBB12_21: ## %cond.load37
 ; AVX512F-NEXT:    vpinsrb $10, (%rdi), %xmm0, %xmm0
 ; AVX512F-NEXT:    incq %rdi
-; AVX512F-NEXT:  LBB12_22: ## %else38
-; AVX512F-NEXT:    kshiftrw $11, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $2048, %eax ## imm = 0x800
 ; AVX512F-NEXT:    je LBB12_24
-; AVX512F-NEXT:  ## %bb.23: ## %cond.load41
+; AVX512F-NEXT:  LBB12_23: ## %cond.load41
 ; AVX512F-NEXT:    vpinsrb $11, (%rdi), %xmm0, %xmm0
 ; AVX512F-NEXT:    incq %rdi
-; AVX512F-NEXT:  LBB12_24: ## %else42
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $12, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $4096, %eax ## imm = 0x1000
 ; AVX512F-NEXT:    je LBB12_26
-; AVX512F-NEXT:  ## %bb.25: ## %cond.load45
+; AVX512F-NEXT:  LBB12_25: ## %cond.load45
 ; AVX512F-NEXT:    vpinsrb $12, (%rdi), %xmm0, %xmm0
 ; AVX512F-NEXT:    incq %rdi
-; AVX512F-NEXT:  LBB12_26: ## %else46
-; AVX512F-NEXT:    kshiftrw $13, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $8192, %eax ## imm = 0x2000
 ; AVX512F-NEXT:    je LBB12_28
-; AVX512F-NEXT:  ## %bb.27: ## %cond.load49
+; AVX512F-NEXT:  LBB12_27: ## %cond.load49
 ; AVX512F-NEXT:    vpinsrb $13, (%rdi), %xmm0, %xmm0
 ; AVX512F-NEXT:    incq %rdi
-; AVX512F-NEXT:  LBB12_28: ## %else50
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm1
-; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $14, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $16384, %eax ## imm = 0x4000
 ; AVX512F-NEXT:    je LBB12_30
-; AVX512F-NEXT:  ## %bb.29: ## %cond.load53
+; AVX512F-NEXT:  LBB12_29: ## %cond.load53
 ; AVX512F-NEXT:    vpinsrb $14, (%rdi), %xmm0, %xmm0
 ; AVX512F-NEXT:    incq %rdi
-; AVX512F-NEXT:  LBB12_30: ## %else54
-; AVX512F-NEXT:    kshiftrw $15, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $32768, %eax ## imm = 0x8000
 ; AVX512F-NEXT:    je LBB12_32
-; AVX512F-NEXT:  ## %bb.31: ## %cond.load57
+; AVX512F-NEXT:  LBB12_31: ## %cond.load57
 ; AVX512F-NEXT:    vpinsrb $15, (%rdi), %xmm0, %xmm0
-; AVX512F-NEXT:  LBB12_32: ## %else58
-; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VLDQ-LABEL: expandload_v16i8_v16i8:
 ; AVX512VLDQ:       ## %bb.0:
 ; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm2, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
+; AVX512VLDQ-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm1
+; AVX512VLDQ-NEXT:    vpmovmskb %xmm1, %eax
 ; AVX512VLDQ-NEXT:    testb $1, %al
-; AVX512VLDQ-NEXT:    je LBB12_2
-; AVX512VLDQ-NEXT:  ## %bb.1: ## %cond.load
+; AVX512VLDQ-NEXT:    jne LBB12_1
+; AVX512VLDQ-NEXT:  ## %bb.2: ## %else
+; AVX512VLDQ-NEXT:    testb $2, %al
+; AVX512VLDQ-NEXT:    jne LBB12_3
+; AVX512VLDQ-NEXT:  LBB12_4: ## %else2
+; AVX512VLDQ-NEXT:    testb $4, %al
+; AVX512VLDQ-NEXT:    jne LBB12_5
+; AVX512VLDQ-NEXT:  LBB12_6: ## %else6
+; AVX512VLDQ-NEXT:    testb $8, %al
+; AVX512VLDQ-NEXT:    jne LBB12_7
+; AVX512VLDQ-NEXT:  LBB12_8: ## %else10
+; AVX512VLDQ-NEXT:    testb $16, %al
+; AVX512VLDQ-NEXT:    jne LBB12_9
+; AVX512VLDQ-NEXT:  LBB12_10: ## %else14
+; AVX512VLDQ-NEXT:    testb $32, %al
+; AVX512VLDQ-NEXT:    jne LBB12_11
+; AVX512VLDQ-NEXT:  LBB12_12: ## %else18
+; AVX512VLDQ-NEXT:    testb $64, %al
+; AVX512VLDQ-NEXT:    jne LBB12_13
+; AVX512VLDQ-NEXT:  LBB12_14: ## %else22
+; AVX512VLDQ-NEXT:    testb $-128, %al
+; AVX512VLDQ-NEXT:    jne LBB12_15
+; AVX512VLDQ-NEXT:  LBB12_16: ## %else26
+; AVX512VLDQ-NEXT:    testl $256, %eax ## imm = 0x100
+; AVX512VLDQ-NEXT:    jne LBB12_17
+; AVX512VLDQ-NEXT:  LBB12_18: ## %else30
+; AVX512VLDQ-NEXT:    testl $512, %eax ## imm = 0x200
+; AVX512VLDQ-NEXT:    jne LBB12_19
+; AVX512VLDQ-NEXT:  LBB12_20: ## %else34
+; AVX512VLDQ-NEXT:    testl $1024, %eax ## imm = 0x400
+; AVX512VLDQ-NEXT:    jne LBB12_21
+; AVX512VLDQ-NEXT:  LBB12_22: ## %else38
+; AVX512VLDQ-NEXT:    testl $2048, %eax ## imm = 0x800
+; AVX512VLDQ-NEXT:    jne LBB12_23
+; AVX512VLDQ-NEXT:  LBB12_24: ## %else42
+; AVX512VLDQ-NEXT:    testl $4096, %eax ## imm = 0x1000
+; AVX512VLDQ-NEXT:    jne LBB12_25
+; AVX512VLDQ-NEXT:  LBB12_26: ## %else46
+; AVX512VLDQ-NEXT:    testl $8192, %eax ## imm = 0x2000
+; AVX512VLDQ-NEXT:    jne LBB12_27
+; AVX512VLDQ-NEXT:  LBB12_28: ## %else50
+; AVX512VLDQ-NEXT:    testl $16384, %eax ## imm = 0x4000
+; AVX512VLDQ-NEXT:    jne LBB12_29
+; AVX512VLDQ-NEXT:  LBB12_30: ## %else54
+; AVX512VLDQ-NEXT:    testl $32768, %eax ## imm = 0x8000
+; AVX512VLDQ-NEXT:    jne LBB12_31
+; AVX512VLDQ-NEXT:  LBB12_32: ## %else58
+; AVX512VLDQ-NEXT:    retq
+; AVX512VLDQ-NEXT:  LBB12_1: ## %cond.load
 ; AVX512VLDQ-NEXT:    vpinsrb $0, (%rdi), %xmm0, %xmm0
 ; AVX512VLDQ-NEXT:    incq %rdi
-; AVX512VLDQ-NEXT:  LBB12_2: ## %else
-; AVX512VLDQ-NEXT:    kshiftrw $1, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $2, %al
 ; AVX512VLDQ-NEXT:    je LBB12_4
-; AVX512VLDQ-NEXT:  ## %bb.3: ## %cond.load1
+; AVX512VLDQ-NEXT:  LBB12_3: ## %cond.load1
 ; AVX512VLDQ-NEXT:    vpinsrb $1, (%rdi), %xmm0, %xmm0
 ; AVX512VLDQ-NEXT:    incq %rdi
-; AVX512VLDQ-NEXT:  LBB12_4: ## %else2
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm2, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $2, %k0, %k1
-; AVX512VLDQ-NEXT:    kmovw %k1, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $4, %al
 ; AVX512VLDQ-NEXT:    je LBB12_6
-; AVX512VLDQ-NEXT:  ## %bb.5: ## %cond.load5
+; AVX512VLDQ-NEXT:  LBB12_5: ## %cond.load5
 ; AVX512VLDQ-NEXT:    vpinsrb $2, (%rdi), %xmm0, %xmm0
 ; AVX512VLDQ-NEXT:    incq %rdi
-; AVX512VLDQ-NEXT:  LBB12_6: ## %else6
-; AVX512VLDQ-NEXT:    kshiftrw $3, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $8, %al
 ; AVX512VLDQ-NEXT:    je LBB12_8
-; AVX512VLDQ-NEXT:  ## %bb.7: ## %cond.load9
+; AVX512VLDQ-NEXT:  LBB12_7: ## %cond.load9
 ; AVX512VLDQ-NEXT:    vpinsrb $3, (%rdi), %xmm0, %xmm0
 ; AVX512VLDQ-NEXT:    incq %rdi
-; AVX512VLDQ-NEXT:  LBB12_8: ## %else10
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm2, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $4, %k0, %k1
-; AVX512VLDQ-NEXT:    kmovw %k1, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $16, %al
 ; AVX512VLDQ-NEXT:    je LBB12_10
-; AVX512VLDQ-NEXT:  ## %bb.9: ## %cond.load13
+; AVX512VLDQ-NEXT:  LBB12_9: ## %cond.load13
 ; AVX512VLDQ-NEXT:    vpinsrb $4, (%rdi), %xmm0, %xmm0
 ; AVX512VLDQ-NEXT:    incq %rdi
-; AVX512VLDQ-NEXT:  LBB12_10: ## %else14
-; AVX512VLDQ-NEXT:    kshiftrw $5, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $32, %al
 ; AVX512VLDQ-NEXT:    je LBB12_12
-; AVX512VLDQ-NEXT:  ## %bb.11: ## %cond.load17
+; AVX512VLDQ-NEXT:  LBB12_11: ## %cond.load17
 ; AVX512VLDQ-NEXT:    vpinsrb $5, (%rdi), %xmm0, %xmm0
 ; AVX512VLDQ-NEXT:    incq %rdi
-; AVX512VLDQ-NEXT:  LBB12_12: ## %else18
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm2, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $6, %k0, %k1
-; AVX512VLDQ-NEXT:    kmovw %k1, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $64, %al
 ; AVX512VLDQ-NEXT:    je LBB12_14
-; AVX512VLDQ-NEXT:  ## %bb.13: ## %cond.load21
+; AVX512VLDQ-NEXT:  LBB12_13: ## %cond.load21
 ; AVX512VLDQ-NEXT:    vpinsrb $6, (%rdi), %xmm0, %xmm0
 ; AVX512VLDQ-NEXT:    incq %rdi
-; AVX512VLDQ-NEXT:  LBB12_14: ## %else22
-; AVX512VLDQ-NEXT:    kshiftrw $7, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $-128, %al
 ; AVX512VLDQ-NEXT:    je LBB12_16
-; AVX512VLDQ-NEXT:  ## %bb.15: ## %cond.load25
+; AVX512VLDQ-NEXT:  LBB12_15: ## %cond.load25
 ; AVX512VLDQ-NEXT:    vpinsrb $7, (%rdi), %xmm0, %xmm0
 ; AVX512VLDQ-NEXT:    incq %rdi
-; AVX512VLDQ-NEXT:  LBB12_16: ## %else26
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm2, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $8, %k0, %k1
-; AVX512VLDQ-NEXT:    kmovw %k1, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testl $256, %eax ## imm = 0x100
 ; AVX512VLDQ-NEXT:    je LBB12_18
-; AVX512VLDQ-NEXT:  ## %bb.17: ## %cond.load29
+; AVX512VLDQ-NEXT:  LBB12_17: ## %cond.load29
 ; AVX512VLDQ-NEXT:    vpinsrb $8, (%rdi), %xmm0, %xmm0
 ; AVX512VLDQ-NEXT:    incq %rdi
-; AVX512VLDQ-NEXT:  LBB12_18: ## %else30
-; AVX512VLDQ-NEXT:    kshiftrw $9, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testl $512, %eax ## imm = 0x200
 ; AVX512VLDQ-NEXT:    je LBB12_20
-; AVX512VLDQ-NEXT:  ## %bb.19: ## %cond.load33
+; AVX512VLDQ-NEXT:  LBB12_19: ## %cond.load33
 ; AVX512VLDQ-NEXT:    vpinsrb $9, (%rdi), %xmm0, %xmm0
 ; AVX512VLDQ-NEXT:    incq %rdi
-; AVX512VLDQ-NEXT:  LBB12_20: ## %else34
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm2, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $10, %k0, %k1
-; AVX512VLDQ-NEXT:    kmovw %k1, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testl $1024, %eax ## imm = 0x400
 ; AVX512VLDQ-NEXT:    je LBB12_22
-; AVX512VLDQ-NEXT:  ## %bb.21: ## %cond.load37
+; AVX512VLDQ-NEXT:  LBB12_21: ## %cond.load37
 ; AVX512VLDQ-NEXT:    vpinsrb $10, (%rdi), %xmm0, %xmm0
 ; AVX512VLDQ-NEXT:    incq %rdi
-; AVX512VLDQ-NEXT:  LBB12_22: ## %else38
-; AVX512VLDQ-NEXT:    kshiftrw $11, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testl $2048, %eax ## imm = 0x800
 ; AVX512VLDQ-NEXT:    je LBB12_24
-; AVX512VLDQ-NEXT:  ## %bb.23: ## %cond.load41
+; AVX512VLDQ-NEXT:  LBB12_23: ## %cond.load41
 ; AVX512VLDQ-NEXT:    vpinsrb $11, (%rdi), %xmm0, %xmm0
 ; AVX512VLDQ-NEXT:    incq %rdi
-; AVX512VLDQ-NEXT:  LBB12_24: ## %else42
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm2, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $12, %k0, %k1
-; AVX512VLDQ-NEXT:    kmovw %k1, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testl $4096, %eax ## imm = 0x1000
 ; AVX512VLDQ-NEXT:    je LBB12_26
-; AVX512VLDQ-NEXT:  ## %bb.25: ## %cond.load45
+; AVX512VLDQ-NEXT:  LBB12_25: ## %cond.load45
 ; AVX512VLDQ-NEXT:    vpinsrb $12, (%rdi), %xmm0, %xmm0
 ; AVX512VLDQ-NEXT:    incq %rdi
-; AVX512VLDQ-NEXT:  LBB12_26: ## %else46
-; AVX512VLDQ-NEXT:    kshiftrw $13, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testl $8192, %eax ## imm = 0x2000
 ; AVX512VLDQ-NEXT:    je LBB12_28
-; AVX512VLDQ-NEXT:  ## %bb.27: ## %cond.load49
+; AVX512VLDQ-NEXT:  LBB12_27: ## %cond.load49
 ; AVX512VLDQ-NEXT:    vpinsrb $13, (%rdi), %xmm0, %xmm0
 ; AVX512VLDQ-NEXT:    incq %rdi
-; AVX512VLDQ-NEXT:  LBB12_28: ## %else50
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm1
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm1, %zmm1
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm1, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $14, %k0, %k1
-; AVX512VLDQ-NEXT:    kmovw %k1, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testl $16384, %eax ## imm = 0x4000
 ; AVX512VLDQ-NEXT:    je LBB12_30
-; AVX512VLDQ-NEXT:  ## %bb.29: ## %cond.load53
+; AVX512VLDQ-NEXT:  LBB12_29: ## %cond.load53
 ; AVX512VLDQ-NEXT:    vpinsrb $14, (%rdi), %xmm0, %xmm0
 ; AVX512VLDQ-NEXT:    incq %rdi
-; AVX512VLDQ-NEXT:  LBB12_30: ## %else54
-; AVX512VLDQ-NEXT:    kshiftrw $15, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testl $32768, %eax ## imm = 0x8000
 ; AVX512VLDQ-NEXT:    je LBB12_32
-; AVX512VLDQ-NEXT:  ## %bb.31: ## %cond.load57
+; AVX512VLDQ-NEXT:  LBB12_31: ## %cond.load57
 ; AVX512VLDQ-NEXT:    vpinsrb $15, (%rdi), %xmm0, %xmm0
-; AVX512VLDQ-NEXT:  LBB12_32: ## %else58
-; AVX512VLDQ-NEXT:    vzeroupper
 ; AVX512VLDQ-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: expandload_v16i8_v16i8:
@@ -4440,130 +4076,131 @@ define <16 x i8> @expandload_v16i8_v16i8(i8* %base, <16 x i8> %src0, <16 x i8> %
 ; AVX512VLBW-NEXT:    vptestnmb %xmm1, %xmm1, %k0
 ; AVX512VLBW-NEXT:    kmovd %k0, %eax
 ; AVX512VLBW-NEXT:    testb $1, %al
-; AVX512VLBW-NEXT:    je LBB12_2
-; AVX512VLBW-NEXT:  ## %bb.1: ## %cond.load
+; AVX512VLBW-NEXT:    jne LBB12_1
+; AVX512VLBW-NEXT:  ## %bb.2: ## %else
+; AVX512VLBW-NEXT:    testb $2, %al
+; AVX512VLBW-NEXT:    jne LBB12_3
+; AVX512VLBW-NEXT:  LBB12_4: ## %else2
+; AVX512VLBW-NEXT:    testb $4, %al
+; AVX512VLBW-NEXT:    jne LBB12_5
+; AVX512VLBW-NEXT:  LBB12_6: ## %else6
+; AVX512VLBW-NEXT:    testb $8, %al
+; AVX512VLBW-NEXT:    jne LBB12_7
+; AVX512VLBW-NEXT:  LBB12_8: ## %else10
+; AVX512VLBW-NEXT:    testb $16, %al
+; AVX512VLBW-NEXT:    jne LBB12_9
+; AVX512VLBW-NEXT:  LBB12_10: ## %else14
+; AVX512VLBW-NEXT:    testb $32, %al
+; AVX512VLBW-NEXT:    jne LBB12_11
+; AVX512VLBW-NEXT:  LBB12_12: ## %else18
+; AVX512VLBW-NEXT:    testb $64, %al
+; AVX512VLBW-NEXT:    jne LBB12_13
+; AVX512VLBW-NEXT:  LBB12_14: ## %else22
+; AVX512VLBW-NEXT:    testb $-128, %al
+; AVX512VLBW-NEXT:    jne LBB12_15
+; AVX512VLBW-NEXT:  LBB12_16: ## %else26
+; AVX512VLBW-NEXT:    testl $256, %eax ## imm = 0x100
+; AVX512VLBW-NEXT:    jne LBB12_17
+; AVX512VLBW-NEXT:  LBB12_18: ## %else30
+; AVX512VLBW-NEXT:    testl $512, %eax ## imm = 0x200
+; AVX512VLBW-NEXT:    jne LBB12_19
+; AVX512VLBW-NEXT:  LBB12_20: ## %else34
+; AVX512VLBW-NEXT:    testl $1024, %eax ## imm = 0x400
+; AVX512VLBW-NEXT:    jne LBB12_21
+; AVX512VLBW-NEXT:  LBB12_22: ## %else38
+; AVX512VLBW-NEXT:    testl $2048, %eax ## imm = 0x800
+; AVX512VLBW-NEXT:    jne LBB12_23
+; AVX512VLBW-NEXT:  LBB12_24: ## %else42
+; AVX512VLBW-NEXT:    testl $4096, %eax ## imm = 0x1000
+; AVX512VLBW-NEXT:    jne LBB12_25
+; AVX512VLBW-NEXT:  LBB12_26: ## %else46
+; AVX512VLBW-NEXT:    testl $8192, %eax ## imm = 0x2000
+; AVX512VLBW-NEXT:    jne LBB12_27
+; AVX512VLBW-NEXT:  LBB12_28: ## %else50
+; AVX512VLBW-NEXT:    testl $16384, %eax ## imm = 0x4000
+; AVX512VLBW-NEXT:    jne LBB12_29
+; AVX512VLBW-NEXT:  LBB12_30: ## %else54
+; AVX512VLBW-NEXT:    testl $32768, %eax ## imm = 0x8000
+; AVX512VLBW-NEXT:    jne LBB12_31
+; AVX512VLBW-NEXT:  LBB12_32: ## %else58
+; AVX512VLBW-NEXT:    retq
+; AVX512VLBW-NEXT:  LBB12_1: ## %cond.load
 ; AVX512VLBW-NEXT:    vpinsrb $0, (%rdi), %xmm0, %xmm0
 ; AVX512VLBW-NEXT:    incq %rdi
-; AVX512VLBW-NEXT:  LBB12_2: ## %else
-; AVX512VLBW-NEXT:    kshiftrw $1, %k0, %k1
-; AVX512VLBW-NEXT:    kmovd %k1, %eax
-; AVX512VLBW-NEXT:    testb $1, %al
+; AVX512VLBW-NEXT:    testb $2, %al
 ; AVX512VLBW-NEXT:    je LBB12_4
-; AVX512VLBW-NEXT:  ## %bb.3: ## %cond.load1
+; AVX512VLBW-NEXT:  LBB12_3: ## %cond.load1
 ; AVX512VLBW-NEXT:    vpinsrb $1, (%rdi), %xmm0, %xmm0
 ; AVX512VLBW-NEXT:    incq %rdi
-; AVX512VLBW-NEXT:  LBB12_4: ## %else2
-; AVX512VLBW-NEXT:    kshiftrw $2, %k0, %k1
-; AVX512VLBW-NEXT:    kmovd %k1, %eax
-; AVX512VLBW-NEXT:    testb $1, %al
+; AVX512VLBW-NEXT:    testb $4, %al
 ; AVX512VLBW-NEXT:    je LBB12_6
-; AVX512VLBW-NEXT:  ## %bb.5: ## %cond.load5
+; AVX512VLBW-NEXT:  LBB12_5: ## %cond.load5
 ; AVX512VLBW-NEXT:    vpinsrb $2, (%rdi), %xmm0, %xmm0
 ; AVX512VLBW-NEXT:    incq %rdi
-; AVX512VLBW-NEXT:  LBB12_6: ## %else6
-; AVX512VLBW-NEXT:    kshiftrw $3, %k0, %k1
-; AVX512VLBW-NEXT:    kmovd %k1, %eax
-; AVX512VLBW-NEXT:    testb $1, %al
+; AVX512VLBW-NEXT:    testb $8, %al
 ; AVX512VLBW-NEXT:    je LBB12_8
-; AVX512VLBW-NEXT:  ## %bb.7: ## %cond.load9
+; AVX512VLBW-NEXT:  LBB12_7: ## %cond.load9
 ; AVX512VLBW-NEXT:    vpinsrb $3, (%rdi), %xmm0, %xmm0
 ; AVX512VLBW-NEXT:    incq %rdi
-; AVX512VLBW-NEXT:  LBB12_8: ## %else10
-; AVX512VLBW-NEXT:    kshiftrw $4, %k0, %k1
-; AVX512VLBW-NEXT:    kmovd %k1, %eax
-; AVX512VLBW-NEXT:    testb $1, %al
+; AVX512VLBW-NEXT:    testb $16, %al
 ; AVX512VLBW-NEXT:    je LBB12_10
-; AVX512VLBW-NEXT:  ## %bb.9: ## %cond.load13
+; AVX512VLBW-NEXT:  LBB12_9: ## %cond.load13
 ; AVX512VLBW-NEXT:    vpinsrb $4, (%rdi), %xmm0, %xmm0
 ; AVX512VLBW-NEXT:    incq %rdi
-; AVX512VLBW-NEXT:  LBB12_10: ## %else14
-; AVX512VLBW-NEXT:    kshiftrw $5, %k0, %k1
-; AVX512VLBW-NEXT:    kmovd %k1, %eax
-; AVX512VLBW-NEXT:    testb $1, %al
+; AVX512VLBW-NEXT:    testb $32, %al
 ; AVX512VLBW-NEXT:    je LBB12_12
-; AVX512VLBW-NEXT:  ## %bb.11: ## %cond.load17
+; AVX512VLBW-NEXT:  LBB12_11: ## %cond.load17
 ; AVX512VLBW-NEXT:    vpinsrb $5, (%rdi), %xmm0, %xmm0
 ; AVX512VLBW-NEXT:    incq %rdi
-; AVX512VLBW-NEXT:  LBB12_12: ## %else18
-; AVX512VLBW-NEXT:    kshiftrw $6, %k0, %k1
-; AVX512VLBW-NEXT:    kmovd %k1, %eax
-; AVX512VLBW-NEXT:    testb $1, %al
+; AVX512VLBW-NEXT:    testb $64, %al
 ; AVX512VLBW-NEXT:    je LBB12_14
-; AVX512VLBW-NEXT:  ## %bb.13: ## %cond.load21
+; AVX512VLBW-NEXT:  LBB12_13: ## %cond.load21
 ; AVX512VLBW-NEXT:    vpinsrb $6, (%rdi), %xmm0, %xmm0
 ; AVX512VLBW-NEXT:    incq %rdi
-; AVX512VLBW-NEXT:  LBB12_14: ## %else22
-; AVX512VLBW-NEXT:    kshiftrw $7, %k0, %k1
-; AVX512VLBW-NEXT:    kmovd %k1, %eax
-; AVX512VLBW-NEXT:    testb $1, %al
+; AVX512VLBW-NEXT:    testb $-128, %al
 ; AVX512VLBW-NEXT:    je LBB12_16
-; AVX512VLBW-NEXT:  ## %bb.15: ## %cond.load25
+; AVX512VLBW-NEXT:  LBB12_15: ## %cond.load25
 ; AVX512VLBW-NEXT:    vpinsrb $7, (%rdi), %xmm0, %xmm0
 ; AVX512VLBW-NEXT:    incq %rdi
-; AVX512VLBW-NEXT:  LBB12_16: ## %else26
-; AVX512VLBW-NEXT:    kshiftrw $8, %k0, %k1
-; AVX512VLBW-NEXT:    kmovd %k1, %eax
-; AVX512VLBW-NEXT:    testb $1, %al
+; AVX512VLBW-NEXT:    testl $256, %eax ## imm = 0x100
 ; AVX512VLBW-NEXT:    je LBB12_18
-; AVX512VLBW-NEXT:  ## %bb.17: ## %cond.load29
+; AVX512VLBW-NEXT:  LBB12_17: ## %cond.load29
 ; AVX512VLBW-NEXT:    vpinsrb $8, (%rdi), %xmm0, %xmm0
 ; AVX512VLBW-NEXT:    incq %rdi
-; AVX512VLBW-NEXT:  LBB12_18: ## %else30
-; AVX512VLBW-NEXT:    kshiftrw $9, %k0, %k1
-; AVX512VLBW-NEXT:    kmovd %k1, %eax
-; AVX512VLBW-NEXT:    testb $1, %al
+; AVX512VLBW-NEXT:    testl $512, %eax ## imm = 0x200
 ; AVX512VLBW-NEXT:    je LBB12_20
-; AVX512VLBW-NEXT:  ## %bb.19: ## %cond.load33
+; AVX512VLBW-NEXT:  LBB12_19: ## %cond.load33
 ; AVX512VLBW-NEXT:    vpinsrb $9, (%rdi), %xmm0, %xmm0
 ; AVX512VLBW-NEXT:    incq %rdi
-; AVX512VLBW-NEXT:  LBB12_20: ## %else34
-; AVX512VLBW-NEXT:    kshiftrw $10, %k0, %k1
-; AVX512VLBW-NEXT:    kmovd %k1, %eax
-; AVX512VLBW-NEXT:    testb $1, %al
+; AVX512VLBW-NEXT:    testl $1024, %eax ## imm = 0x400
 ; AVX512VLBW-NEXT:    je LBB12_22
-; AVX512VLBW-NEXT:  ## %bb.21: ## %cond.load37
+; AVX512VLBW-NEXT:  LBB12_21: ## %cond.load37
 ; AVX512VLBW-NEXT:    vpinsrb $10, (%rdi), %xmm0, %xmm0
 ; AVX512VLBW-NEXT:    incq %rdi
-; AVX512VLBW-NEXT:  LBB12_22: ## %else38
-; AVX512VLBW-NEXT:    kshiftrw $11, %k0, %k1
-; AVX512VLBW-NEXT:    kmovd %k1, %eax
-; AVX512VLBW-NEXT:    testb $1, %al
+; AVX512VLBW-NEXT:    testl $2048, %eax ## imm = 0x800
 ; AVX512VLBW-NEXT:    je LBB12_24
-; AVX512VLBW-NEXT:  ## %bb.23: ## %cond.load41
+; AVX512VLBW-NEXT:  LBB12_23: ## %cond.load41
 ; AVX512VLBW-NEXT:    vpinsrb $11, (%rdi), %xmm0, %xmm0
 ; AVX512VLBW-NEXT:    incq %rdi
-; AVX512VLBW-NEXT:  LBB12_24: ## %else42
-; AVX512VLBW-NEXT:    kshiftrw $12, %k0, %k1
-; AVX512VLBW-NEXT:    kmovd %k1, %eax
-; AVX512VLBW-NEXT:    testb $1, %al
+; AVX512VLBW-NEXT:    testl $4096, %eax ## imm = 0x1000
 ; AVX512VLBW-NEXT:    je LBB12_26
-; AVX512VLBW-NEXT:  ## %bb.25: ## %cond.load45
+; AVX512VLBW-NEXT:  LBB12_25: ## %cond.load45
 ; AVX512VLBW-NEXT:    vpinsrb $12, (%rdi), %xmm0, %xmm0
 ; AVX512VLBW-NEXT:    incq %rdi
-; AVX512VLBW-NEXT:  LBB12_26: ## %else46
-; AVX512VLBW-NEXT:    kshiftrw $13, %k0, %k1
-; AVX512VLBW-NEXT:    kmovd %k1, %eax
-; AVX512VLBW-NEXT:    testb $1, %al
+; AVX512VLBW-NEXT:    testl $8192, %eax ## imm = 0x2000
 ; AVX512VLBW-NEXT:    je LBB12_28
-; AVX512VLBW-NEXT:  ## %bb.27: ## %cond.load49
+; AVX512VLBW-NEXT:  LBB12_27: ## %cond.load49
 ; AVX512VLBW-NEXT:    vpinsrb $13, (%rdi), %xmm0, %xmm0
 ; AVX512VLBW-NEXT:    incq %rdi
-; AVX512VLBW-NEXT:  LBB12_28: ## %else50
-; AVX512VLBW-NEXT:    kshiftrw $14, %k0, %k1
-; AVX512VLBW-NEXT:    kmovd %k1, %eax
-; AVX512VLBW-NEXT:    testb $1, %al
+; AVX512VLBW-NEXT:    testl $16384, %eax ## imm = 0x4000
 ; AVX512VLBW-NEXT:    je LBB12_30
-; AVX512VLBW-NEXT:  ## %bb.29: ## %cond.load53
+; AVX512VLBW-NEXT:  LBB12_29: ## %cond.load53
 ; AVX512VLBW-NEXT:    vpinsrb $14, (%rdi), %xmm0, %xmm0
 ; AVX512VLBW-NEXT:    incq %rdi
-; AVX512VLBW-NEXT:  LBB12_30: ## %else54
-; AVX512VLBW-NEXT:    kshiftrw $15, %k0, %k0
-; AVX512VLBW-NEXT:    kmovd %k0, %eax
-; AVX512VLBW-NEXT:    testb $1, %al
+; AVX512VLBW-NEXT:    testl $32768, %eax ## imm = 0x8000
 ; AVX512VLBW-NEXT:    je LBB12_32
-; AVX512VLBW-NEXT:  ## %bb.31: ## %cond.load57
+; AVX512VLBW-NEXT:  LBB12_31: ## %cond.load57
 ; AVX512VLBW-NEXT:    vpinsrb $15, (%rdi), %xmm0, %xmm0
-; AVX512VLBW-NEXT:  LBB12_32: ## %else58
 ; AVX512VLBW-NEXT:    retq
   %mask = icmp eq <16 x i8> %trigger, zeroinitializer
   %res = call <16 x i8> @llvm.masked.expandload.v16i8(i8* %base, <16 x i1> %mask, <16 x i8> %src0)
diff --git a/llvm/test/CodeGen/X86/masked_gather.ll b/llvm/test/CodeGen/X86/masked_gather.ll
index cee52a0d0..7ea2cacd7 100644
--- a/llvm/test/CodeGen/X86/masked_gather.ll
+++ b/llvm/test/CodeGen/X86/masked_gather.ll
@@ -8,81 +8,88 @@
 ; vXf32
 ;
 
+%struct.a = type { [4 x i32], [4 x i8], %struct.b, i32 }
+%struct.b = type { i32, i32 }
+@c = external dso_local global %struct.a, align 4
+
 define <4 x float> @gather_v4f32_ptr_v4i32(<4 x float*> %ptr, <4 x i32> %trigger, <4 x float> %passthru) {
 ; SSE-LABEL: gather_v4f32_ptr_v4i32:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pxor %xmm4, %xmm4
 ; SSE-NEXT:    pcmpeqd %xmm2, %xmm4
-; SSE-NEXT:    pextrb $0, %xmm4, %eax
-; SSE-NEXT:    testb $1, %al
-; SSE-NEXT:    je .LBB0_2
-; SSE-NEXT:  # %bb.1: # %cond.load
-; SSE-NEXT:    movq %xmm0, %rax
-; SSE-NEXT:    movss {{.*#+}} xmm5 = mem[0],zero,zero,zero
-; SSE-NEXT:    blendps {{.*#+}} xmm3 = xmm5[0],xmm3[1,2,3]
-; SSE-NEXT:  .LBB0_2: # %else
-; SSE-NEXT:    pextrb $4, %xmm4, %eax
+; SSE-NEXT:    movmskps %xmm4, %eax
 ; SSE-NEXT:    testb $1, %al
+; SSE-NEXT:    jne .LBB0_1
+; SSE-NEXT:  # %bb.2: # %else
+; SSE-NEXT:    testb $2, %al
+; SSE-NEXT:    jne .LBB0_3
+; SSE-NEXT:  .LBB0_4: # %else2
+; SSE-NEXT:    testb $4, %al
+; SSE-NEXT:    jne .LBB0_5
+; SSE-NEXT:  .LBB0_6: # %else5
+; SSE-NEXT:    testb $8, %al
+; SSE-NEXT:    jne .LBB0_7
+; SSE-NEXT:  .LBB0_8: # %else8
+; SSE-NEXT:    movaps %xmm3, %xmm0
+; SSE-NEXT:    retq
+; SSE-NEXT:  .LBB0_1: # %cond.load
+; SSE-NEXT:    movq %xmm0, %rcx
+; SSE-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE-NEXT:    pblendw {{.*#+}} xmm3 = xmm2[0,1],xmm3[2,3,4,5,6,7]
+; SSE-NEXT:    testb $2, %al
 ; SSE-NEXT:    je .LBB0_4
-; SSE-NEXT:  # %bb.3: # %cond.load1
-; SSE-NEXT:    pextrq $1, %xmm0, %rax
+; SSE-NEXT:  .LBB0_3: # %cond.load1
+; SSE-NEXT:    pextrq $1, %xmm0, %rcx
 ; SSE-NEXT:    insertps {{.*#+}} xmm3 = xmm3[0],mem[0],xmm3[2,3]
-; SSE-NEXT:  .LBB0_4: # %else2
-; SSE-NEXT:    pxor %xmm0, %xmm0
-; SSE-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE-NEXT:    pextrb $8, %xmm2, %eax
-; SSE-NEXT:    testb $1, %al
+; SSE-NEXT:    testb $4, %al
 ; SSE-NEXT:    je .LBB0_6
-; SSE-NEXT:  # %bb.5: # %cond.load4
-; SSE-NEXT:    movq %xmm1, %rax
+; SSE-NEXT:  .LBB0_5: # %cond.load4
+; SSE-NEXT:    movq %xmm1, %rcx
 ; SSE-NEXT:    insertps {{.*#+}} xmm3 = xmm3[0,1],mem[0],xmm3[3]
-; SSE-NEXT:  .LBB0_6: # %else5
-; SSE-NEXT:    pextrb $12, %xmm2, %eax
-; SSE-NEXT:    testb $1, %al
+; SSE-NEXT:    testb $8, %al
 ; SSE-NEXT:    je .LBB0_8
-; SSE-NEXT:  # %bb.7: # %cond.load7
+; SSE-NEXT:  .LBB0_7: # %cond.load7
 ; SSE-NEXT:    pextrq $1, %xmm1, %rax
 ; SSE-NEXT:    insertps {{.*#+}} xmm3 = xmm3[0,1,2],mem[0]
-; SSE-NEXT:  .LBB0_8: # %else8
 ; SSE-NEXT:    movaps %xmm3, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: gather_v4f32_ptr_v4i32:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm1, %xmm3
-; AVX1-NEXT:    vpextrb $0, %xmm3, %eax
+; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vmovmskps %xmm1, %eax
 ; AVX1-NEXT:    testb $1, %al
 ; AVX1-NEXT:    je .LBB0_2
 ; AVX1-NEXT:  # %bb.1: # %cond.load
-; AVX1-NEXT:    vmovq %xmm0, %rax
-; AVX1-NEXT:    vmovss {{.*#+}} xmm4 = mem[0],zero,zero,zero
-; AVX1-NEXT:    vblendps {{.*#+}} xmm2 = xmm4[0],xmm2[1,2,3]
+; AVX1-NEXT:    vmovq %xmm0, %rcx
+; AVX1-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX1-NEXT:    vblendps {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
 ; AVX1-NEXT:  .LBB0_2: # %else
-; AVX1-NEXT:    vpextrb $4, %xmm3, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $2, %al
 ; AVX1-NEXT:    je .LBB0_4
 ; AVX1-NEXT:  # %bb.3: # %cond.load1
-; AVX1-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX1-NEXT:    vpextrq $1, %xmm0, %rcx
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0],mem[0],xmm2[2,3]
 ; AVX1-NEXT:  .LBB0_4: # %else2
-; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $4, %al
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT:    je .LBB0_6
-; AVX1-NEXT:  # %bb.5: # %cond.load4
-; AVX1-NEXT:    vmovq %xmm0, %rax
+; AVX1-NEXT:    jne .LBB0_5
+; AVX1-NEXT:  # %bb.6: # %else5
+; AVX1-NEXT:    testb $8, %al
+; AVX1-NEXT:    jne .LBB0_7
+; AVX1-NEXT:  .LBB0_8: # %else8
+; AVX1-NEXT:    vmovaps %xmm2, %xmm0
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+; AVX1-NEXT:  .LBB0_5: # %cond.load4
+; AVX1-NEXT:    vmovq %xmm0, %rcx
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1],mem[0],xmm2[3]
-; AVX1-NEXT:  .LBB0_6: # %else5
-; AVX1-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $8, %al
 ; AVX1-NEXT:    je .LBB0_8
-; AVX1-NEXT:  # %bb.7: # %cond.load7
+; AVX1-NEXT:  .LBB0_7: # %cond.load7
 ; AVX1-NEXT:    vpextrq $1, %xmm0, %rax
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1,2],mem[0]
-; AVX1-NEXT:  .LBB0_8: # %else8
 ; AVX1-NEXT:    vmovaps %xmm2, %xmm0
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -90,39 +97,39 @@ define <4 x float> @gather_v4f32_ptr_v4i32(<4 x float*> %ptr, <4 x i32> %trigger
 ; AVX2-LABEL: gather_v4f32_ptr_v4i32:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm1, %xmm3
-; AVX2-NEXT:    vpextrb $0, %xmm3, %eax
+; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm1, %xmm1
+; AVX2-NEXT:    vmovmskps %xmm1, %eax
 ; AVX2-NEXT:    testb $1, %al
 ; AVX2-NEXT:    je .LBB0_2
 ; AVX2-NEXT:  # %bb.1: # %cond.load
-; AVX2-NEXT:    vmovq %xmm0, %rax
-; AVX2-NEXT:    vmovss {{.*#+}} xmm4 = mem[0],zero,zero,zero
-; AVX2-NEXT:    vblendps {{.*#+}} xmm2 = xmm4[0],xmm2[1,2,3]
+; AVX2-NEXT:    vmovq %xmm0, %rcx
+; AVX2-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX2-NEXT:    vblendps {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
 ; AVX2-NEXT:  .LBB0_2: # %else
-; AVX2-NEXT:    vpextrb $4, %xmm3, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $2, %al
 ; AVX2-NEXT:    je .LBB0_4
 ; AVX2-NEXT:  # %bb.3: # %cond.load1
-; AVX2-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX2-NEXT:    vpextrq $1, %xmm0, %rcx
 ; AVX2-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0],mem[0],xmm2[2,3]
 ; AVX2-NEXT:  .LBB0_4: # %else2
-; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm1, %xmm1
-; AVX2-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $4, %al
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; AVX2-NEXT:    je .LBB0_6
-; AVX2-NEXT:  # %bb.5: # %cond.load4
-; AVX2-NEXT:    vmovq %xmm0, %rax
+; AVX2-NEXT:    jne .LBB0_5
+; AVX2-NEXT:  # %bb.6: # %else5
+; AVX2-NEXT:    testb $8, %al
+; AVX2-NEXT:    jne .LBB0_7
+; AVX2-NEXT:  .LBB0_8: # %else8
+; AVX2-NEXT:    vmovaps %xmm2, %xmm0
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+; AVX2-NEXT:  .LBB0_5: # %cond.load4
+; AVX2-NEXT:    vmovq %xmm0, %rcx
 ; AVX2-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1],mem[0],xmm2[3]
-; AVX2-NEXT:  .LBB0_6: # %else5
-; AVX2-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $8, %al
 ; AVX2-NEXT:    je .LBB0_8
-; AVX2-NEXT:  # %bb.7: # %cond.load7
+; AVX2-NEXT:  .LBB0_7: # %cond.load7
 ; AVX2-NEXT:    vpextrq $1, %xmm0, %rax
 ; AVX2-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1,2],mem[0]
-; AVX2-NEXT:  .LBB0_8: # %else8
 ; AVX2-NEXT:    vmovaps %xmm2, %xmm0
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -156,39 +163,38 @@ define <4 x float> @gather_v4f32_v4i32_v4i32(float* %base, <4 x i32> %idx, <4 x
 ; SSE-NEXT:    pmovsxdq %xmm0, %xmm0
 ; SSE-NEXT:    pxor %xmm5, %xmm5
 ; SSE-NEXT:    pcmpeqd %xmm1, %xmm5
-; SSE-NEXT:    pextrb $0, %xmm5, %eax
+; SSE-NEXT:    movmskps %xmm5, %eax
 ; SSE-NEXT:    testb $1, %al
 ; SSE-NEXT:    je .LBB1_2
 ; SSE-NEXT:  # %bb.1: # %cond.load
-; SSE-NEXT:    movq %xmm4, %rax
-; SSE-NEXT:    movss {{.*#+}} xmm6 = mem[0],zero,zero,zero
-; SSE-NEXT:    blendps {{.*#+}} xmm2 = xmm6[0],xmm2[1,2,3]
+; SSE-NEXT:    movq %xmm4, %rcx
+; SSE-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE-NEXT:    pblendw {{.*#+}} xmm2 = xmm1[0,1],xmm2[2,3,4,5,6,7]
 ; SSE-NEXT:  .LBB1_2: # %else
 ; SSE-NEXT:    psllq $2, %xmm0
-; SSE-NEXT:    pextrb $4, %xmm5, %eax
-; SSE-NEXT:    testb $1, %al
+; SSE-NEXT:    testb $2, %al
 ; SSE-NEXT:    je .LBB1_4
 ; SSE-NEXT:  # %bb.3: # %cond.load1
-; SSE-NEXT:    pextrq $1, %xmm4, %rax
+; SSE-NEXT:    pextrq $1, %xmm4, %rcx
 ; SSE-NEXT:    insertps {{.*#+}} xmm2 = xmm2[0],mem[0],xmm2[2,3]
 ; SSE-NEXT:  .LBB1_4: # %else2
 ; SSE-NEXT:    paddq %xmm0, %xmm3
-; SSE-NEXT:    pxor %xmm0, %xmm0
-; SSE-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE-NEXT:    pextrb $8, %xmm1, %eax
-; SSE-NEXT:    testb $1, %al
-; SSE-NEXT:    je .LBB1_6
-; SSE-NEXT:  # %bb.5: # %cond.load4
-; SSE-NEXT:    movq %xmm3, %rax
+; SSE-NEXT:    testb $4, %al
+; SSE-NEXT:    jne .LBB1_5
+; SSE-NEXT:  # %bb.6: # %else5
+; SSE-NEXT:    testb $8, %al
+; SSE-NEXT:    jne .LBB1_7
+; SSE-NEXT:  .LBB1_8: # %else8
+; SSE-NEXT:    movaps %xmm2, %xmm0
+; SSE-NEXT:    retq
+; SSE-NEXT:  .LBB1_5: # %cond.load4
+; SSE-NEXT:    movq %xmm3, %rcx
 ; SSE-NEXT:    insertps {{.*#+}} xmm2 = xmm2[0,1],mem[0],xmm2[3]
-; SSE-NEXT:  .LBB1_6: # %else5
-; SSE-NEXT:    pextrb $12, %xmm1, %eax
-; SSE-NEXT:    testb $1, %al
+; SSE-NEXT:    testb $8, %al
 ; SSE-NEXT:    je .LBB1_8
-; SSE-NEXT:  # %bb.7: # %cond.load7
+; SSE-NEXT:  .LBB1_7: # %cond.load7
 ; SSE-NEXT:    pextrq $1, %xmm3, %rax
 ; SSE-NEXT:    insertps {{.*#+}} xmm2 = xmm2[0,1,2],mem[0]
-; SSE-NEXT:  .LBB1_8: # %else8
 ; SSE-NEXT:    movaps %xmm2, %xmm0
 ; SSE-NEXT:    retq
 ;
@@ -205,39 +211,39 @@ define <4 x float> @gather_v4f32_v4i32_v4i32(float* %base, <4 x i32> %idx, <4 x
 ; AVX1-NEXT:    vpaddq %xmm0, %xmm3, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
 ; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm1, %xmm3
-; AVX1-NEXT:    vpextrb $0, %xmm3, %eax
+; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vmovmskps %xmm1, %eax
 ; AVX1-NEXT:    testb $1, %al
 ; AVX1-NEXT:    je .LBB1_2
 ; AVX1-NEXT:  # %bb.1: # %cond.load
-; AVX1-NEXT:    vmovq %xmm0, %rax
-; AVX1-NEXT:    vmovd {{.*#+}} xmm4 = mem[0],zero,zero,zero
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm4[0,1],xmm2[2,3,4,5,6,7]
+; AVX1-NEXT:    vmovq %xmm0, %rcx
+; AVX1-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX1-NEXT:    vblendps {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
 ; AVX1-NEXT:  .LBB1_2: # %else
-; AVX1-NEXT:    vpextrb $4, %xmm3, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $2, %al
 ; AVX1-NEXT:    je .LBB1_4
 ; AVX1-NEXT:  # %bb.3: # %cond.load1
-; AVX1-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX1-NEXT:    vpextrq $1, %xmm0, %rcx
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0],mem[0],xmm2[2,3]
 ; AVX1-NEXT:  .LBB1_4: # %else2
-; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $4, %al
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT:    je .LBB1_6
-; AVX1-NEXT:  # %bb.5: # %cond.load4
-; AVX1-NEXT:    vmovq %xmm0, %rax
+; AVX1-NEXT:    jne .LBB1_5
+; AVX1-NEXT:  # %bb.6: # %else5
+; AVX1-NEXT:    testb $8, %al
+; AVX1-NEXT:    jne .LBB1_7
+; AVX1-NEXT:  .LBB1_8: # %else8
+; AVX1-NEXT:    vmovaps %xmm2, %xmm0
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+; AVX1-NEXT:  .LBB1_5: # %cond.load4
+; AVX1-NEXT:    vmovq %xmm0, %rcx
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1],mem[0],xmm2[3]
-; AVX1-NEXT:  .LBB1_6: # %else5
-; AVX1-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $8, %al
 ; AVX1-NEXT:    je .LBB1_8
-; AVX1-NEXT:  # %bb.7: # %cond.load7
+; AVX1-NEXT:  .LBB1_7: # %cond.load7
 ; AVX1-NEXT:    vpextrq $1, %xmm0, %rax
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1,2],mem[0]
-; AVX1-NEXT:  .LBB1_8: # %else8
 ; AVX1-NEXT:    vmovaps %xmm2, %xmm0
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -250,39 +256,39 @@ define <4 x float> @gather_v4f32_v4i32_v4i32(float* %base, <4 x i32> %idx, <4 x
 ; AVX2-NEXT:    vpsllq $2, %ymm0, %ymm0
 ; AVX2-NEXT:    vpaddq %ymm0, %ymm3, %ymm0
 ; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm1, %xmm3
-; AVX2-NEXT:    vpextrb $0, %xmm3, %eax
+; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm1, %xmm1
+; AVX2-NEXT:    vmovmskps %xmm1, %eax
 ; AVX2-NEXT:    testb $1, %al
 ; AVX2-NEXT:    je .LBB1_2
 ; AVX2-NEXT:  # %bb.1: # %cond.load
-; AVX2-NEXT:    vmovq %xmm0, %rax
-; AVX2-NEXT:    vmovss {{.*#+}} xmm4 = mem[0],zero,zero,zero
-; AVX2-NEXT:    vblendps {{.*#+}} xmm2 = xmm4[0],xmm2[1,2,3]
+; AVX2-NEXT:    vmovq %xmm0, %rcx
+; AVX2-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX2-NEXT:    vblendps {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
 ; AVX2-NEXT:  .LBB1_2: # %else
-; AVX2-NEXT:    vpextrb $4, %xmm3, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $2, %al
 ; AVX2-NEXT:    je .LBB1_4
 ; AVX2-NEXT:  # %bb.3: # %cond.load1
-; AVX2-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX2-NEXT:    vpextrq $1, %xmm0, %rcx
 ; AVX2-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0],mem[0],xmm2[2,3]
 ; AVX2-NEXT:  .LBB1_4: # %else2
-; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm1, %xmm1
-; AVX2-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $4, %al
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; AVX2-NEXT:    je .LBB1_6
-; AVX2-NEXT:  # %bb.5: # %cond.load4
-; AVX2-NEXT:    vmovq %xmm0, %rax
+; AVX2-NEXT:    jne .LBB1_5
+; AVX2-NEXT:  # %bb.6: # %else5
+; AVX2-NEXT:    testb $8, %al
+; AVX2-NEXT:    jne .LBB1_7
+; AVX2-NEXT:  .LBB1_8: # %else8
+; AVX2-NEXT:    vmovaps %xmm2, %xmm0
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+; AVX2-NEXT:  .LBB1_5: # %cond.load4
+; AVX2-NEXT:    vmovq %xmm0, %rcx
 ; AVX2-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1],mem[0],xmm2[3]
-; AVX2-NEXT:  .LBB1_6: # %else5
-; AVX2-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $8, %al
 ; AVX2-NEXT:    je .LBB1_8
-; AVX2-NEXT:  # %bb.7: # %cond.load7
+; AVX2-NEXT:  .LBB1_7: # %cond.load7
 ; AVX2-NEXT:    vpextrq $1, %xmm0, %rax
 ; AVX2-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1,2],mem[0]
-; AVX2-NEXT:  .LBB1_8: # %else8
 ; AVX2-NEXT:    vmovaps %xmm2, %xmm0
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -317,39 +323,38 @@ define <4 x float> @gather_v4f32_v4i64_v4i32(float* %base, <4 x i64> %idx, <4 x
 ; SSE-NEXT:    paddq %xmm4, %xmm0
 ; SSE-NEXT:    pxor %xmm5, %xmm5
 ; SSE-NEXT:    pcmpeqd %xmm2, %xmm5
-; SSE-NEXT:    pextrb $0, %xmm5, %eax
+; SSE-NEXT:    movmskps %xmm5, %eax
 ; SSE-NEXT:    testb $1, %al
 ; SSE-NEXT:    je .LBB2_2
 ; SSE-NEXT:  # %bb.1: # %cond.load
-; SSE-NEXT:    movq %xmm0, %rax
-; SSE-NEXT:    movss {{.*#+}} xmm6 = mem[0],zero,zero,zero
-; SSE-NEXT:    blendps {{.*#+}} xmm3 = xmm6[0],xmm3[1,2,3]
+; SSE-NEXT:    movq %xmm0, %rcx
+; SSE-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE-NEXT:    pblendw {{.*#+}} xmm3 = xmm2[0,1],xmm3[2,3,4,5,6,7]
 ; SSE-NEXT:  .LBB2_2: # %else
 ; SSE-NEXT:    psllq $2, %xmm1
-; SSE-NEXT:    pextrb $4, %xmm5, %eax
-; SSE-NEXT:    testb $1, %al
+; SSE-NEXT:    testb $2, %al
 ; SSE-NEXT:    je .LBB2_4
 ; SSE-NEXT:  # %bb.3: # %cond.load1
-; SSE-NEXT:    pextrq $1, %xmm0, %rax
+; SSE-NEXT:    pextrq $1, %xmm0, %rcx
 ; SSE-NEXT:    insertps {{.*#+}} xmm3 = xmm3[0],mem[0],xmm3[2,3]
 ; SSE-NEXT:  .LBB2_4: # %else2
 ; SSE-NEXT:    paddq %xmm1, %xmm4
-; SSE-NEXT:    pxor %xmm0, %xmm0
-; SSE-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE-NEXT:    pextrb $8, %xmm2, %eax
-; SSE-NEXT:    testb $1, %al
-; SSE-NEXT:    je .LBB2_6
-; SSE-NEXT:  # %bb.5: # %cond.load4
-; SSE-NEXT:    movq %xmm4, %rax
+; SSE-NEXT:    testb $4, %al
+; SSE-NEXT:    jne .LBB2_5
+; SSE-NEXT:  # %bb.6: # %else5
+; SSE-NEXT:    testb $8, %al
+; SSE-NEXT:    jne .LBB2_7
+; SSE-NEXT:  .LBB2_8: # %else8
+; SSE-NEXT:    movaps %xmm3, %xmm0
+; SSE-NEXT:    retq
+; SSE-NEXT:  .LBB2_5: # %cond.load4
+; SSE-NEXT:    movq %xmm4, %rcx
 ; SSE-NEXT:    insertps {{.*#+}} xmm3 = xmm3[0,1],mem[0],xmm3[3]
-; SSE-NEXT:  .LBB2_6: # %else5
-; SSE-NEXT:    pextrb $12, %xmm2, %eax
-; SSE-NEXT:    testb $1, %al
+; SSE-NEXT:    testb $8, %al
 ; SSE-NEXT:    je .LBB2_8
-; SSE-NEXT:  # %bb.7: # %cond.load7
+; SSE-NEXT:  .LBB2_7: # %cond.load7
 ; SSE-NEXT:    pextrq $1, %xmm4, %rax
 ; SSE-NEXT:    insertps {{.*#+}} xmm3 = xmm3[0,1,2],mem[0]
-; SSE-NEXT:  .LBB2_8: # %else8
 ; SSE-NEXT:    movaps %xmm3, %xmm0
 ; SSE-NEXT:    retq
 ;
@@ -364,39 +369,39 @@ define <4 x float> @gather_v4f32_v4i64_v4i32(float* %base, <4 x i64> %idx, <4 x
 ; AVX1-NEXT:    vpaddq %xmm0, %xmm4, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
 ; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm1, %xmm3
-; AVX1-NEXT:    vpextrb $0, %xmm3, %eax
+; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vmovmskps %xmm1, %eax
 ; AVX1-NEXT:    testb $1, %al
 ; AVX1-NEXT:    je .LBB2_2
 ; AVX1-NEXT:  # %bb.1: # %cond.load
-; AVX1-NEXT:    vmovq %xmm0, %rax
-; AVX1-NEXT:    vmovd {{.*#+}} xmm4 = mem[0],zero,zero,zero
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm4[0,1],xmm2[2,3,4,5,6,7]
+; AVX1-NEXT:    vmovq %xmm0, %rcx
+; AVX1-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX1-NEXT:    vblendps {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
 ; AVX1-NEXT:  .LBB2_2: # %else
-; AVX1-NEXT:    vpextrb $4, %xmm3, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $2, %al
 ; AVX1-NEXT:    je .LBB2_4
 ; AVX1-NEXT:  # %bb.3: # %cond.load1
-; AVX1-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX1-NEXT:    vpextrq $1, %xmm0, %rcx
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0],mem[0],xmm2[2,3]
 ; AVX1-NEXT:  .LBB2_4: # %else2
-; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $4, %al
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT:    je .LBB2_6
-; AVX1-NEXT:  # %bb.5: # %cond.load4
-; AVX1-NEXT:    vmovq %xmm0, %rax
+; AVX1-NEXT:    jne .LBB2_5
+; AVX1-NEXT:  # %bb.6: # %else5
+; AVX1-NEXT:    testb $8, %al
+; AVX1-NEXT:    jne .LBB2_7
+; AVX1-NEXT:  .LBB2_8: # %else8
+; AVX1-NEXT:    vmovaps %xmm2, %xmm0
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+; AVX1-NEXT:  .LBB2_5: # %cond.load4
+; AVX1-NEXT:    vmovq %xmm0, %rcx
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1],mem[0],xmm2[3]
-; AVX1-NEXT:  .LBB2_6: # %else5
-; AVX1-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $8, %al
 ; AVX1-NEXT:    je .LBB2_8
-; AVX1-NEXT:  # %bb.7: # %cond.load7
+; AVX1-NEXT:  .LBB2_7: # %cond.load7
 ; AVX1-NEXT:    vpextrq $1, %xmm0, %rax
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1,2],mem[0]
-; AVX1-NEXT:  .LBB2_8: # %else8
 ; AVX1-NEXT:    vmovaps %xmm2, %xmm0
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -408,39 +413,39 @@ define <4 x float> @gather_v4f32_v4i64_v4i32(float* %base, <4 x i64> %idx, <4 x
 ; AVX2-NEXT:    vpsllq $2, %ymm0, %ymm0
 ; AVX2-NEXT:    vpaddq %ymm0, %ymm3, %ymm0
 ; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm1, %xmm3
-; AVX2-NEXT:    vpextrb $0, %xmm3, %eax
+; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm1, %xmm1
+; AVX2-NEXT:    vmovmskps %xmm1, %eax
 ; AVX2-NEXT:    testb $1, %al
 ; AVX2-NEXT:    je .LBB2_2
 ; AVX2-NEXT:  # %bb.1: # %cond.load
-; AVX2-NEXT:    vmovq %xmm0, %rax
-; AVX2-NEXT:    vmovss {{.*#+}} xmm4 = mem[0],zero,zero,zero
-; AVX2-NEXT:    vblendps {{.*#+}} xmm2 = xmm4[0],xmm2[1,2,3]
+; AVX2-NEXT:    vmovq %xmm0, %rcx
+; AVX2-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX2-NEXT:    vblendps {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
 ; AVX2-NEXT:  .LBB2_2: # %else
-; AVX2-NEXT:    vpextrb $4, %xmm3, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $2, %al
 ; AVX2-NEXT:    je .LBB2_4
 ; AVX2-NEXT:  # %bb.3: # %cond.load1
-; AVX2-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX2-NEXT:    vpextrq $1, %xmm0, %rcx
 ; AVX2-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0],mem[0],xmm2[2,3]
 ; AVX2-NEXT:  .LBB2_4: # %else2
-; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm1, %xmm1
-; AVX2-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $4, %al
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; AVX2-NEXT:    je .LBB2_6
-; AVX2-NEXT:  # %bb.5: # %cond.load4
-; AVX2-NEXT:    vmovq %xmm0, %rax
+; AVX2-NEXT:    jne .LBB2_5
+; AVX2-NEXT:  # %bb.6: # %else5
+; AVX2-NEXT:    testb $8, %al
+; AVX2-NEXT:    jne .LBB2_7
+; AVX2-NEXT:  .LBB2_8: # %else8
+; AVX2-NEXT:    vmovaps %xmm2, %xmm0
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+; AVX2-NEXT:  .LBB2_5: # %cond.load4
+; AVX2-NEXT:    vmovq %xmm0, %rcx
 ; AVX2-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1],mem[0],xmm2[3]
-; AVX2-NEXT:  .LBB2_6: # %else5
-; AVX2-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $8, %al
 ; AVX2-NEXT:    je .LBB2_8
-; AVX2-NEXT:  # %bb.7: # %cond.load7
+; AVX2-NEXT:  .LBB2_7: # %cond.load7
 ; AVX2-NEXT:    vpextrq $1, %xmm0, %rax
 ; AVX2-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1,2],mem[0]
-; AVX2-NEXT:  .LBB2_8: # %else8
 ; AVX2-NEXT:    vmovaps %xmm2, %xmm0
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -480,149 +485,124 @@ define <16 x i8> @gather_v16i8_v16i32_v16i8(i8* %base, <16 x i32> %idx, <16 x i8
 ; SSE-NEXT:    paddq %xmm8, %xmm0
 ; SSE-NEXT:    pxor %xmm6, %xmm6
 ; SSE-NEXT:    pcmpeqb %xmm4, %xmm6
-; SSE-NEXT:    pextrb $0, %xmm6, %eax
+; SSE-NEXT:    pmovmskb %xmm6, %eax
 ; SSE-NEXT:    testb $1, %al
 ; SSE-NEXT:    je .LBB3_2
 ; SSE-NEXT:  # %bb.1: # %cond.load
-; SSE-NEXT:    movq %xmm0, %rax
-; SSE-NEXT:    pinsrb $0, (%rax), %xmm5
+; SSE-NEXT:    movq %xmm0, %rcx
+; SSE-NEXT:    pinsrb $0, (%rcx), %xmm5
 ; SSE-NEXT:  .LBB3_2: # %else
-; SSE-NEXT:    pmovsxdq %xmm7, %xmm7
-; SSE-NEXT:    pextrb $1, %xmm6, %eax
-; SSE-NEXT:    testb $1, %al
+; SSE-NEXT:    pmovsxdq %xmm7, %xmm4
+; SSE-NEXT:    testb $2, %al
 ; SSE-NEXT:    je .LBB3_4
 ; SSE-NEXT:  # %bb.3: # %cond.load1
-; SSE-NEXT:    pextrq $1, %xmm0, %rax
-; SSE-NEXT:    pinsrb $1, (%rax), %xmm5
+; SSE-NEXT:    pextrq $1, %xmm0, %rcx
+; SSE-NEXT:    pinsrb $1, (%rcx), %xmm5
 ; SSE-NEXT:  .LBB3_4: # %else2
-; SSE-NEXT:    paddq %xmm8, %xmm7
-; SSE-NEXT:    pxor %xmm6, %xmm6
-; SSE-NEXT:    pcmpeqb %xmm4, %xmm6
-; SSE-NEXT:    pextrb $2, %xmm6, %eax
-; SSE-NEXT:    testb $1, %al
+; SSE-NEXT:    paddq %xmm8, %xmm4
+; SSE-NEXT:    testb $4, %al
 ; SSE-NEXT:    je .LBB3_6
 ; SSE-NEXT:  # %bb.5: # %cond.load4
-; SSE-NEXT:    movq %xmm7, %rax
-; SSE-NEXT:    pinsrb $2, (%rax), %xmm5
+; SSE-NEXT:    movq %xmm4, %rcx
+; SSE-NEXT:    pinsrb $2, (%rcx), %xmm5
 ; SSE-NEXT:  .LBB3_6: # %else5
 ; SSE-NEXT:    pmovsxdq %xmm1, %xmm0
-; SSE-NEXT:    pextrb $3, %xmm6, %eax
-; SSE-NEXT:    testb $1, %al
+; SSE-NEXT:    testb $8, %al
 ; SSE-NEXT:    je .LBB3_8
 ; SSE-NEXT:  # %bb.7: # %cond.load7
-; SSE-NEXT:    pextrq $1, %xmm7, %rax
-; SSE-NEXT:    pinsrb $3, (%rax), %xmm5
+; SSE-NEXT:    pextrq $1, %xmm4, %rcx
+; SSE-NEXT:    pinsrb $3, (%rcx), %xmm5
 ; SSE-NEXT:  .LBB3_8: # %else8
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
 ; SSE-NEXT:    paddq %xmm8, %xmm0
-; SSE-NEXT:    pxor %xmm6, %xmm6
-; SSE-NEXT:    pcmpeqb %xmm4, %xmm6
-; SSE-NEXT:    pextrb $4, %xmm6, %eax
-; SSE-NEXT:    testb $1, %al
+; SSE-NEXT:    testb $16, %al
 ; SSE-NEXT:    je .LBB3_10
 ; SSE-NEXT:  # %bb.9: # %cond.load10
-; SSE-NEXT:    movq %xmm0, %rax
-; SSE-NEXT:    pinsrb $4, (%rax), %xmm5
+; SSE-NEXT:    movq %xmm0, %rcx
+; SSE-NEXT:    pinsrb $4, (%rcx), %xmm5
 ; SSE-NEXT:  .LBB3_10: # %else11
 ; SSE-NEXT:    pmovsxdq %xmm1, %xmm1
-; SSE-NEXT:    pextrb $5, %xmm6, %eax
-; SSE-NEXT:    testb $1, %al
+; SSE-NEXT:    testb $32, %al
 ; SSE-NEXT:    je .LBB3_12
 ; SSE-NEXT:  # %bb.11: # %cond.load13
-; SSE-NEXT:    pextrq $1, %xmm0, %rax
-; SSE-NEXT:    pinsrb $5, (%rax), %xmm5
+; SSE-NEXT:    pextrq $1, %xmm0, %rcx
+; SSE-NEXT:    pinsrb $5, (%rcx), %xmm5
 ; SSE-NEXT:  .LBB3_12: # %else14
 ; SSE-NEXT:    paddq %xmm8, %xmm1
-; SSE-NEXT:    pxor %xmm6, %xmm6
-; SSE-NEXT:    pcmpeqb %xmm4, %xmm6
-; SSE-NEXT:    pextrb $6, %xmm6, %eax
-; SSE-NEXT:    testb $1, %al
+; SSE-NEXT:    testb $64, %al
 ; SSE-NEXT:    je .LBB3_14
 ; SSE-NEXT:  # %bb.13: # %cond.load16
-; SSE-NEXT:    movq %xmm1, %rax
-; SSE-NEXT:    pinsrb $6, (%rax), %xmm5
+; SSE-NEXT:    movq %xmm1, %rcx
+; SSE-NEXT:    pinsrb $6, (%rcx), %xmm5
 ; SSE-NEXT:  .LBB3_14: # %else17
 ; SSE-NEXT:    pmovsxdq %xmm2, %xmm0
-; SSE-NEXT:    pextrb $7, %xmm6, %eax
-; SSE-NEXT:    testb $1, %al
+; SSE-NEXT:    testb $-128, %al
 ; SSE-NEXT:    je .LBB3_16
 ; SSE-NEXT:  # %bb.15: # %cond.load19
-; SSE-NEXT:    pextrq $1, %xmm1, %rax
-; SSE-NEXT:    pinsrb $7, (%rax), %xmm5
+; SSE-NEXT:    pextrq $1, %xmm1, %rcx
+; SSE-NEXT:    pinsrb $7, (%rcx), %xmm5
 ; SSE-NEXT:  .LBB3_16: # %else20
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,3,0,1]
 ; SSE-NEXT:    paddq %xmm8, %xmm0
-; SSE-NEXT:    pxor %xmm2, %xmm2
-; SSE-NEXT:    pcmpeqb %xmm4, %xmm2
-; SSE-NEXT:    pextrb $8, %xmm2, %eax
-; SSE-NEXT:    testb $1, %al
+; SSE-NEXT:    testl $256, %eax # imm = 0x100
 ; SSE-NEXT:    je .LBB3_18
 ; SSE-NEXT:  # %bb.17: # %cond.load22
-; SSE-NEXT:    movq %xmm0, %rax
-; SSE-NEXT:    pinsrb $8, (%rax), %xmm5
+; SSE-NEXT:    movq %xmm0, %rcx
+; SSE-NEXT:    pinsrb $8, (%rcx), %xmm5
 ; SSE-NEXT:  .LBB3_18: # %else23
 ; SSE-NEXT:    pmovsxdq %xmm1, %xmm1
-; SSE-NEXT:    pextrb $9, %xmm2, %eax
-; SSE-NEXT:    testb $1, %al
+; SSE-NEXT:    testl $512, %eax # imm = 0x200
 ; SSE-NEXT:    je .LBB3_20
 ; SSE-NEXT:  # %bb.19: # %cond.load25
-; SSE-NEXT:    pextrq $1, %xmm0, %rax
-; SSE-NEXT:    pinsrb $9, (%rax), %xmm5
+; SSE-NEXT:    pextrq $1, %xmm0, %rcx
+; SSE-NEXT:    pinsrb $9, (%rcx), %xmm5
 ; SSE-NEXT:  .LBB3_20: # %else26
 ; SSE-NEXT:    paddq %xmm8, %xmm1
-; SSE-NEXT:    pxor %xmm2, %xmm2
-; SSE-NEXT:    pcmpeqb %xmm4, %xmm2
-; SSE-NEXT:    pextrb $10, %xmm2, %eax
-; SSE-NEXT:    testb $1, %al
+; SSE-NEXT:    testl $1024, %eax # imm = 0x400
 ; SSE-NEXT:    je .LBB3_22
 ; SSE-NEXT:  # %bb.21: # %cond.load28
-; SSE-NEXT:    movq %xmm1, %rax
-; SSE-NEXT:    pinsrb $10, (%rax), %xmm5
+; SSE-NEXT:    movq %xmm1, %rcx
+; SSE-NEXT:    pinsrb $10, (%rcx), %xmm5
 ; SSE-NEXT:  .LBB3_22: # %else29
 ; SSE-NEXT:    pmovsxdq %xmm3, %xmm0
-; SSE-NEXT:    pextrb $11, %xmm2, %eax
-; SSE-NEXT:    testb $1, %al
+; SSE-NEXT:    testl $2048, %eax # imm = 0x800
 ; SSE-NEXT:    je .LBB3_24
 ; SSE-NEXT:  # %bb.23: # %cond.load31
-; SSE-NEXT:    pextrq $1, %xmm1, %rax
-; SSE-NEXT:    pinsrb $11, (%rax), %xmm5
+; SSE-NEXT:    pextrq $1, %xmm1, %rcx
+; SSE-NEXT:    pinsrb $11, (%rcx), %xmm5
 ; SSE-NEXT:  .LBB3_24: # %else32
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[2,3,0,1]
 ; SSE-NEXT:    paddq %xmm8, %xmm0
-; SSE-NEXT:    pxor %xmm2, %xmm2
-; SSE-NEXT:    pcmpeqb %xmm4, %xmm2
-; SSE-NEXT:    pextrb $12, %xmm2, %eax
-; SSE-NEXT:    testb $1, %al
+; SSE-NEXT:    testl $4096, %eax # imm = 0x1000
 ; SSE-NEXT:    je .LBB3_26
 ; SSE-NEXT:  # %bb.25: # %cond.load34
-; SSE-NEXT:    movq %xmm0, %rax
-; SSE-NEXT:    pinsrb $12, (%rax), %xmm5
+; SSE-NEXT:    movq %xmm0, %rcx
+; SSE-NEXT:    pinsrb $12, (%rcx), %xmm5
 ; SSE-NEXT:  .LBB3_26: # %else35
 ; SSE-NEXT:    pmovsxdq %xmm1, %xmm1
-; SSE-NEXT:    pextrb $13, %xmm2, %eax
-; SSE-NEXT:    testb $1, %al
+; SSE-NEXT:    testl $8192, %eax # imm = 0x2000
 ; SSE-NEXT:    je .LBB3_28
 ; SSE-NEXT:  # %bb.27: # %cond.load37
-; SSE-NEXT:    pextrq $1, %xmm0, %rax
-; SSE-NEXT:    pinsrb $13, (%rax), %xmm5
+; SSE-NEXT:    pextrq $1, %xmm0, %rcx
+; SSE-NEXT:    pinsrb $13, (%rcx), %xmm5
 ; SSE-NEXT:  .LBB3_28: # %else38
 ; SSE-NEXT:    paddq %xmm1, %xmm8
-; SSE-NEXT:    pxor %xmm0, %xmm0
-; SSE-NEXT:    pcmpeqb %xmm0, %xmm4
-; SSE-NEXT:    pextrb $14, %xmm4, %eax
-; SSE-NEXT:    testb $1, %al
-; SSE-NEXT:    je .LBB3_30
-; SSE-NEXT:  # %bb.29: # %cond.load40
-; SSE-NEXT:    movq %xmm8, %rax
-; SSE-NEXT:    pinsrb $14, (%rax), %xmm5
-; SSE-NEXT:  .LBB3_30: # %else41
-; SSE-NEXT:    pextrb $15, %xmm4, %eax
-; SSE-NEXT:    testb $1, %al
+; SSE-NEXT:    testl $16384, %eax # imm = 0x4000
+; SSE-NEXT:    jne .LBB3_29
+; SSE-NEXT:  # %bb.30: # %else41
+; SSE-NEXT:    testl $32768, %eax # imm = 0x8000
+; SSE-NEXT:    jne .LBB3_31
+; SSE-NEXT:  .LBB3_32: # %else44
+; SSE-NEXT:    movdqa %xmm5, %xmm0
+; SSE-NEXT:    retq
+; SSE-NEXT:  .LBB3_29: # %cond.load40
+; SSE-NEXT:    movq %xmm8, %rcx
+; SSE-NEXT:    pinsrb $14, (%rcx), %xmm5
+; SSE-NEXT:    testl $32768, %eax # imm = 0x8000
 ; SSE-NEXT:    je .LBB3_32
-; SSE-NEXT:  # %bb.31: # %cond.load43
+; SSE-NEXT:  .LBB3_31: # %cond.load43
 ; SSE-NEXT:    pextrq $1, %xmm8, %rax
 ; SSE-NEXT:    pinsrb $15, (%rax), %xmm5
-; SSE-NEXT:  .LBB3_32: # %else44
 ; SSE-NEXT:    movdqa %xmm5, %xmm0
 ; SSE-NEXT:    retq
 ;
@@ -638,156 +618,132 @@ define <16 x i8> @gather_v16i8_v16i32_v16i8(i8* %base, <16 x i32> %idx, <16 x i8
 ; AVX1-NEXT:    vpaddq %xmm0, %xmm4, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm0, %ymm0
 ; AVX1-NEXT:    vpxor %xmm6, %xmm6, %xmm6
-; AVX1-NEXT:    vpcmpeqb %xmm6, %xmm2, %xmm6
-; AVX1-NEXT:    vpextrb $0, %xmm6, %eax
+; AVX1-NEXT:    vpcmpeqb %xmm6, %xmm2, %xmm2
+; AVX1-NEXT:    vpmovmskb %xmm2, %eax
 ; AVX1-NEXT:    testb $1, %al
 ; AVX1-NEXT:    je .LBB3_2
 ; AVX1-NEXT:  # %bb.1: # %cond.load
-; AVX1-NEXT:    vmovq %xmm0, %rax
-; AVX1-NEXT:    vpinsrb $0, (%rax), %xmm3, %xmm3
+; AVX1-NEXT:    vmovq %xmm0, %rcx
+; AVX1-NEXT:    vpinsrb $0, (%rcx), %xmm3, %xmm3
 ; AVX1-NEXT:  .LBB3_2: # %else
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm5[2,3,0,1]
-; AVX1-NEXT:    vpmovsxdq %xmm5, %xmm5
-; AVX1-NEXT:    vpextrb $1, %xmm6, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm5[2,3,0,1]
+; AVX1-NEXT:    vpmovsxdq %xmm5, %xmm6
+; AVX1-NEXT:    testb $2, %al
 ; AVX1-NEXT:    je .LBB3_4
 ; AVX1-NEXT:  # %bb.3: # %cond.load1
-; AVX1-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX1-NEXT:    vpinsrb $1, (%rax), %xmm3, %xmm3
+; AVX1-NEXT:    vpextrq $1, %xmm0, %rcx
+; AVX1-NEXT:    vpinsrb $1, (%rcx), %xmm3, %xmm3
 ; AVX1-NEXT:  .LBB3_4: # %else2
-; AVX1-NEXT:    vpmovsxdq %xmm7, %xmm6
-; AVX1-NEXT:    vpaddq %xmm5, %xmm4, %xmm5
-; AVX1-NEXT:    vpxor %xmm7, %xmm7, %xmm7
-; AVX1-NEXT:    vpcmpeqb %xmm7, %xmm2, %xmm7
-; AVX1-NEXT:    vpextrb $2, %xmm7, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    vpmovsxdq %xmm2, %xmm5
+; AVX1-NEXT:    vpaddq %xmm6, %xmm4, %xmm2
+; AVX1-NEXT:    testb $4, %al
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; AVX1-NEXT:    je .LBB3_6
 ; AVX1-NEXT:  # %bb.5: # %cond.load4
-; AVX1-NEXT:    vmovq %xmm0, %rax
-; AVX1-NEXT:    vpinsrb $2, (%rax), %xmm3, %xmm3
+; AVX1-NEXT:    vmovq %xmm0, %rcx
+; AVX1-NEXT:    vpinsrb $2, (%rcx), %xmm3, %xmm3
 ; AVX1-NEXT:  .LBB3_6: # %else5
-; AVX1-NEXT:    vpaddq %xmm6, %xmm4, %xmm6
-; AVX1-NEXT:    vpextrb $3, %xmm7, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    vpaddq %xmm5, %xmm4, %xmm5
+; AVX1-NEXT:    testb $8, %al
 ; AVX1-NEXT:    je .LBB3_8
 ; AVX1-NEXT:  # %bb.7: # %cond.load7
-; AVX1-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX1-NEXT:    vpinsrb $3, (%rax), %xmm3, %xmm3
+; AVX1-NEXT:    vpextrq $1, %xmm0, %rcx
+; AVX1-NEXT:    vpinsrb $3, (%rcx), %xmm3, %xmm3
 ; AVX1-NEXT:  .LBB3_8: # %else8
-; AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm5, %ymm0
-; AVX1-NEXT:    vpxor %xmm5, %xmm5, %xmm5
-; AVX1-NEXT:    vpcmpeqb %xmm5, %xmm2, %xmm5
-; AVX1-NEXT:    vpextrb $4, %xmm5, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm2, %ymm0
+; AVX1-NEXT:    testb $16, %al
 ; AVX1-NEXT:    je .LBB3_10
 ; AVX1-NEXT:  # %bb.9: # %cond.load10
-; AVX1-NEXT:    vmovq %xmm0, %rax
-; AVX1-NEXT:    vpinsrb $4, (%rax), %xmm3, %xmm3
+; AVX1-NEXT:    vmovq %xmm0, %rcx
+; AVX1-NEXT:    vpinsrb $4, (%rcx), %xmm3, %xmm3
 ; AVX1-NEXT:  .LBB3_10: # %else11
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm1[2,3,0,1]
-; AVX1-NEXT:    vpmovsxdq %xmm1, %xmm7
-; AVX1-NEXT:    vpextrb $5, %xmm5, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; AVX1-NEXT:    vpmovsxdq %xmm1, %xmm6
+; AVX1-NEXT:    testb $32, %al
 ; AVX1-NEXT:    je .LBB3_12
 ; AVX1-NEXT:  # %bb.11: # %cond.load13
-; AVX1-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX1-NEXT:    vpinsrb $5, (%rax), %xmm3, %xmm3
+; AVX1-NEXT:    vpextrq $1, %xmm0, %rcx
+; AVX1-NEXT:    vpinsrb $5, (%rcx), %xmm3, %xmm3
 ; AVX1-NEXT:  .LBB3_12: # %else14
-; AVX1-NEXT:    vpmovsxdq %xmm6, %xmm6
-; AVX1-NEXT:    vpaddq %xmm7, %xmm4, %xmm5
-; AVX1-NEXT:    vpxor %xmm7, %xmm7, %xmm7
-; AVX1-NEXT:    vpcmpeqb %xmm7, %xmm2, %xmm7
-; AVX1-NEXT:    vpextrb $6, %xmm7, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    vpmovsxdq %xmm2, %xmm5
+; AVX1-NEXT:    vpaddq %xmm6, %xmm4, %xmm2
+; AVX1-NEXT:    testb $64, %al
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; AVX1-NEXT:    je .LBB3_14
 ; AVX1-NEXT:  # %bb.13: # %cond.load16
-; AVX1-NEXT:    vmovq %xmm0, %rax
-; AVX1-NEXT:    vpinsrb $6, (%rax), %xmm3, %xmm3
+; AVX1-NEXT:    vmovq %xmm0, %rcx
+; AVX1-NEXT:    vpinsrb $6, (%rcx), %xmm3, %xmm3
 ; AVX1-NEXT:  .LBB3_14: # %else17
-; AVX1-NEXT:    vpaddq %xmm6, %xmm4, %xmm6
-; AVX1-NEXT:    vpextrb $7, %xmm7, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    vpaddq %xmm5, %xmm4, %xmm5
+; AVX1-NEXT:    testb $-128, %al
 ; AVX1-NEXT:    je .LBB3_16
 ; AVX1-NEXT:  # %bb.15: # %cond.load19
-; AVX1-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX1-NEXT:    vpinsrb $7, (%rax), %xmm3, %xmm3
+; AVX1-NEXT:    vpextrq $1, %xmm0, %rcx
+; AVX1-NEXT:    vpinsrb $7, (%rcx), %xmm3, %xmm3
 ; AVX1-NEXT:  .LBB3_16: # %else20
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm1
-; AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm5, %ymm0
-; AVX1-NEXT:    vpxor %xmm5, %xmm5, %xmm5
-; AVX1-NEXT:    vpcmpeqb %xmm5, %xmm2, %xmm5
-; AVX1-NEXT:    vpextrb $8, %xmm5, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm2, %ymm0
+; AVX1-NEXT:    testl $256, %eax # imm = 0x100
 ; AVX1-NEXT:    je .LBB3_18
 ; AVX1-NEXT:  # %bb.17: # %cond.load22
-; AVX1-NEXT:    vmovq %xmm0, %rax
-; AVX1-NEXT:    vpinsrb $8, (%rax), %xmm3, %xmm3
+; AVX1-NEXT:    vmovq %xmm0, %rcx
+; AVX1-NEXT:    vpinsrb $8, (%rcx), %xmm3, %xmm3
 ; AVX1-NEXT:  .LBB3_18: # %else23
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm1[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
 ; AVX1-NEXT:    vpmovsxdq %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $9, %xmm5, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $512, %eax # imm = 0x200
 ; AVX1-NEXT:    je .LBB3_20
 ; AVX1-NEXT:  # %bb.19: # %cond.load25
-; AVX1-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX1-NEXT:    vpinsrb $9, (%rax), %xmm3, %xmm3
+; AVX1-NEXT:    vpextrq $1, %xmm0, %rcx
+; AVX1-NEXT:    vpinsrb $9, (%rcx), %xmm3, %xmm3
 ; AVX1-NEXT:  .LBB3_20: # %else26
-; AVX1-NEXT:    vpmovsxdq %xmm6, %xmm5
+; AVX1-NEXT:    vpmovsxdq %xmm2, %xmm2
 ; AVX1-NEXT:    vpaddq %xmm1, %xmm4, %xmm1
-; AVX1-NEXT:    vpxor %xmm6, %xmm6, %xmm6
-; AVX1-NEXT:    vpcmpeqb %xmm6, %xmm2, %xmm6
-; AVX1-NEXT:    vpextrb $10, %xmm6, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $1024, %eax # imm = 0x400
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; AVX1-NEXT:    je .LBB3_22
 ; AVX1-NEXT:  # %bb.21: # %cond.load28
-; AVX1-NEXT:    vmovq %xmm0, %rax
-; AVX1-NEXT:    vpinsrb $10, (%rax), %xmm3, %xmm3
+; AVX1-NEXT:    vmovq %xmm0, %rcx
+; AVX1-NEXT:    vpinsrb $10, (%rcx), %xmm3, %xmm3
 ; AVX1-NEXT:  .LBB3_22: # %else29
-; AVX1-NEXT:    vpaddq %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vpextrb $11, %xmm6, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    vpaddq %xmm2, %xmm4, %xmm2
+; AVX1-NEXT:    testl $2048, %eax # imm = 0x800
 ; AVX1-NEXT:    je .LBB3_24
 ; AVX1-NEXT:  # %bb.23: # %cond.load31
-; AVX1-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX1-NEXT:    vpinsrb $11, (%rax), %xmm3, %xmm3
+; AVX1-NEXT:    vpextrq $1, %xmm0, %rcx
+; AVX1-NEXT:    vpinsrb $11, (%rcx), %xmm3, %xmm3
 ; AVX1-NEXT:  .LBB3_24: # %else32
-; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm1, %ymm0
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm0
+; AVX1-NEXT:    testl $4096, %eax # imm = 0x1000
 ; AVX1-NEXT:    je .LBB3_26
 ; AVX1-NEXT:  # %bb.25: # %cond.load34
-; AVX1-NEXT:    vmovq %xmm0, %rax
-; AVX1-NEXT:    vpinsrb $12, (%rax), %xmm3, %xmm3
+; AVX1-NEXT:    vmovq %xmm0, %rcx
+; AVX1-NEXT:    vpinsrb $12, (%rcx), %xmm3, %xmm3
 ; AVX1-NEXT:  .LBB3_26: # %else35
-; AVX1-NEXT:    vpextrb $13, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $8192, %eax # imm = 0x2000
 ; AVX1-NEXT:    je .LBB3_28
 ; AVX1-NEXT:  # %bb.27: # %cond.load37
-; AVX1-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX1-NEXT:    vpinsrb $13, (%rax), %xmm3, %xmm3
+; AVX1-NEXT:    vpextrq $1, %xmm0, %rcx
+; AVX1-NEXT:    vpinsrb $13, (%rcx), %xmm3, %xmm3
 ; AVX1-NEXT:  .LBB3_28: # %else38
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vpextrb $14, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $16384, %eax # imm = 0x4000
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT:    je .LBB3_30
-; AVX1-NEXT:  # %bb.29: # %cond.load40
-; AVX1-NEXT:    vmovq %xmm0, %rax
-; AVX1-NEXT:    vpinsrb $14, (%rax), %xmm3, %xmm3
-; AVX1-NEXT:  .LBB3_30: # %else41
-; AVX1-NEXT:    vpextrb $15, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    jne .LBB3_29
+; AVX1-NEXT:  # %bb.30: # %else41
+; AVX1-NEXT:    testl $32768, %eax # imm = 0x8000
+; AVX1-NEXT:    jne .LBB3_31
+; AVX1-NEXT:  .LBB3_32: # %else44
+; AVX1-NEXT:    vmovdqa %xmm3, %xmm0
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+; AVX1-NEXT:  .LBB3_29: # %cond.load40
+; AVX1-NEXT:    vmovq %xmm0, %rcx
+; AVX1-NEXT:    vpinsrb $14, (%rcx), %xmm3, %xmm3
+; AVX1-NEXT:    testl $32768, %eax # imm = 0x8000
 ; AVX1-NEXT:    je .LBB3_32
-; AVX1-NEXT:  # %bb.31: # %cond.load43
+; AVX1-NEXT:  .LBB3_31: # %cond.load43
 ; AVX1-NEXT:    vpextrq $1, %xmm0, %rax
 ; AVX1-NEXT:    vpinsrb $15, (%rax), %xmm3, %xmm3
-; AVX1-NEXT:  .LBB3_32: # %else44
 ; AVX1-NEXT:    vmovdqa %xmm3, %xmm0
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -799,145 +755,121 @@ define <16 x i8> @gather_v16i8_v16i32_v16i8(i8* %base, <16 x i32> %idx, <16 x i8
 ; AVX2-NEXT:    vpmovsxdq %xmm0, %ymm5
 ; AVX2-NEXT:    vpaddq %ymm5, %ymm4, %ymm5
 ; AVX2-NEXT:    vpxor %xmm6, %xmm6, %xmm6
-; AVX2-NEXT:    vpcmpeqb %xmm6, %xmm2, %xmm6
-; AVX2-NEXT:    vpextrb $0, %xmm6, %eax
+; AVX2-NEXT:    vpcmpeqb %xmm6, %xmm2, %xmm2
+; AVX2-NEXT:    vpmovmskb %xmm2, %eax
 ; AVX2-NEXT:    testb $1, %al
 ; AVX2-NEXT:    je .LBB3_2
 ; AVX2-NEXT:  # %bb.1: # %cond.load
-; AVX2-NEXT:    vmovq %xmm5, %rax
-; AVX2-NEXT:    vpinsrb $0, (%rax), %xmm3, %xmm3
+; AVX2-NEXT:    vmovq %xmm5, %rcx
+; AVX2-NEXT:    vpinsrb $0, (%rcx), %xmm3, %xmm3
 ; AVX2-NEXT:  .LBB3_2: # %else
-; AVX2-NEXT:    vpextrb $1, %xmm6, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $2, %al
 ; AVX2-NEXT:    je .LBB3_4
 ; AVX2-NEXT:  # %bb.3: # %cond.load1
-; AVX2-NEXT:    vpextrq $1, %xmm5, %rax
-; AVX2-NEXT:    vpinsrb $1, (%rax), %xmm3, %xmm3
+; AVX2-NEXT:    vpextrq $1, %xmm5, %rcx
+; AVX2-NEXT:    vpinsrb $1, (%rcx), %xmm3, %xmm3
 ; AVX2-NEXT:  .LBB3_4: # %else2
-; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; AVX2-NEXT:    vpxor %xmm6, %xmm6, %xmm6
-; AVX2-NEXT:    vpcmpeqb %xmm6, %xmm2, %xmm6
-; AVX2-NEXT:    vpextrb $2, %xmm6, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    vextracti128 $1, %ymm5, %xmm5
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm2
+; AVX2-NEXT:    testb $4, %al
+; AVX2-NEXT:    vextracti128 $1, %ymm5, %xmm0
 ; AVX2-NEXT:    je .LBB3_6
 ; AVX2-NEXT:  # %bb.5: # %cond.load4
-; AVX2-NEXT:    vmovq %xmm5, %rax
-; AVX2-NEXT:    vpinsrb $2, (%rax), %xmm3, %xmm3
+; AVX2-NEXT:    vmovq %xmm0, %rcx
+; AVX2-NEXT:    vpinsrb $2, (%rcx), %xmm3, %xmm3
 ; AVX2-NEXT:  .LBB3_6: # %else5
-; AVX2-NEXT:    vpmovsxdq %xmm0, %ymm0
-; AVX2-NEXT:    vpextrb $3, %xmm6, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    vpmovsxdq %xmm2, %ymm2
+; AVX2-NEXT:    testb $8, %al
 ; AVX2-NEXT:    je .LBB3_8
 ; AVX2-NEXT:  # %bb.7: # %cond.load7
-; AVX2-NEXT:    vpextrq $1, %xmm5, %rax
-; AVX2-NEXT:    vpinsrb $3, (%rax), %xmm3, %xmm3
+; AVX2-NEXT:    vpextrq $1, %xmm0, %rcx
+; AVX2-NEXT:    vpinsrb $3, (%rcx), %xmm3, %xmm3
 ; AVX2-NEXT:  .LBB3_8: # %else8
-; AVX2-NEXT:    vpaddq %ymm0, %ymm4, %ymm0
-; AVX2-NEXT:    vpxor %xmm5, %xmm5, %xmm5
-; AVX2-NEXT:    vpcmpeqb %xmm5, %xmm2, %xmm5
-; AVX2-NEXT:    vpextrb $4, %xmm5, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    vpaddq %ymm2, %ymm4, %ymm0
+; AVX2-NEXT:    testb $16, %al
 ; AVX2-NEXT:    je .LBB3_10
 ; AVX2-NEXT:  # %bb.9: # %cond.load10
-; AVX2-NEXT:    vmovq %xmm0, %rax
-; AVX2-NEXT:    vpinsrb $4, (%rax), %xmm3, %xmm3
+; AVX2-NEXT:    vmovq %xmm0, %rcx
+; AVX2-NEXT:    vpinsrb $4, (%rcx), %xmm3, %xmm3
 ; AVX2-NEXT:  .LBB3_10: # %else11
-; AVX2-NEXT:    vpextrb $5, %xmm5, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $32, %al
 ; AVX2-NEXT:    je .LBB3_12
 ; AVX2-NEXT:  # %bb.11: # %cond.load13
-; AVX2-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX2-NEXT:    vpinsrb $5, (%rax), %xmm3, %xmm3
+; AVX2-NEXT:    vpextrq $1, %xmm0, %rcx
+; AVX2-NEXT:    vpinsrb $5, (%rcx), %xmm3, %xmm3
 ; AVX2-NEXT:  .LBB3_12: # %else14
-; AVX2-NEXT:    vpxor %xmm5, %xmm5, %xmm5
-; AVX2-NEXT:    vpcmpeqb %xmm5, %xmm2, %xmm5
-; AVX2-NEXT:    vpextrb $6, %xmm5, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $64, %al
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
 ; AVX2-NEXT:    je .LBB3_14
 ; AVX2-NEXT:  # %bb.13: # %cond.load16
-; AVX2-NEXT:    vmovq %xmm0, %rax
-; AVX2-NEXT:    vpinsrb $6, (%rax), %xmm3, %xmm3
+; AVX2-NEXT:    vmovq %xmm0, %rcx
+; AVX2-NEXT:    vpinsrb $6, (%rcx), %xmm3, %xmm3
 ; AVX2-NEXT:  .LBB3_14: # %else17
-; AVX2-NEXT:    vpmovsxdq %xmm1, %ymm6
-; AVX2-NEXT:    vpextrb $7, %xmm5, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    vpmovsxdq %xmm1, %ymm2
+; AVX2-NEXT:    testb $-128, %al
 ; AVX2-NEXT:    je .LBB3_16
 ; AVX2-NEXT:  # %bb.15: # %cond.load19
-; AVX2-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX2-NEXT:    vpinsrb $7, (%rax), %xmm3, %xmm3
+; AVX2-NEXT:    vpextrq $1, %xmm0, %rcx
+; AVX2-NEXT:    vpinsrb $7, (%rcx), %xmm3, %xmm3
 ; AVX2-NEXT:  .LBB3_16: # %else20
-; AVX2-NEXT:    vpaddq %ymm6, %ymm4, %ymm0
-; AVX2-NEXT:    vpxor %xmm5, %xmm5, %xmm5
-; AVX2-NEXT:    vpcmpeqb %xmm5, %xmm2, %xmm5
-; AVX2-NEXT:    vpextrb $8, %xmm5, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    vpaddq %ymm2, %ymm4, %ymm0
+; AVX2-NEXT:    testl $256, %eax # imm = 0x100
 ; AVX2-NEXT:    je .LBB3_18
 ; AVX2-NEXT:  # %bb.17: # %cond.load22
-; AVX2-NEXT:    vmovq %xmm0, %rax
-; AVX2-NEXT:    vpinsrb $8, (%rax), %xmm3, %xmm3
+; AVX2-NEXT:    vmovq %xmm0, %rcx
+; AVX2-NEXT:    vpinsrb $8, (%rcx), %xmm3, %xmm3
 ; AVX2-NEXT:  .LBB3_18: # %else23
-; AVX2-NEXT:    vpextrb $9, %xmm5, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $512, %eax # imm = 0x200
 ; AVX2-NEXT:    je .LBB3_20
 ; AVX2-NEXT:  # %bb.19: # %cond.load25
-; AVX2-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX2-NEXT:    vpinsrb $9, (%rax), %xmm3, %xmm3
+; AVX2-NEXT:    vpextrq $1, %xmm0, %rcx
+; AVX2-NEXT:    vpinsrb $9, (%rcx), %xmm3, %xmm3
 ; AVX2-NEXT:  .LBB3_20: # %else26
 ; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm1
-; AVX2-NEXT:    vpxor %xmm5, %xmm5, %xmm5
-; AVX2-NEXT:    vpcmpeqb %xmm5, %xmm2, %xmm5
-; AVX2-NEXT:    vpextrb $10, %xmm5, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $1024, %eax # imm = 0x400
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
 ; AVX2-NEXT:    je .LBB3_22
 ; AVX2-NEXT:  # %bb.21: # %cond.load28
-; AVX2-NEXT:    vmovq %xmm0, %rax
-; AVX2-NEXT:    vpinsrb $10, (%rax), %xmm3, %xmm3
+; AVX2-NEXT:    vmovq %xmm0, %rcx
+; AVX2-NEXT:    vpinsrb $10, (%rcx), %xmm3, %xmm3
 ; AVX2-NEXT:  .LBB3_22: # %else29
 ; AVX2-NEXT:    vpmovsxdq %xmm1, %ymm1
-; AVX2-NEXT:    vpextrb $11, %xmm5, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $2048, %eax # imm = 0x800
 ; AVX2-NEXT:    je .LBB3_24
 ; AVX2-NEXT:  # %bb.23: # %cond.load31
-; AVX2-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX2-NEXT:    vpinsrb $11, (%rax), %xmm3, %xmm3
+; AVX2-NEXT:    vpextrq $1, %xmm0, %rcx
+; AVX2-NEXT:    vpinsrb $11, (%rcx), %xmm3, %xmm3
 ; AVX2-NEXT:  .LBB3_24: # %else32
 ; AVX2-NEXT:    vpaddq %ymm1, %ymm4, %ymm0
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX2-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $4096, %eax # imm = 0x1000
 ; AVX2-NEXT:    je .LBB3_26
 ; AVX2-NEXT:  # %bb.25: # %cond.load34
-; AVX2-NEXT:    vmovq %xmm0, %rax
-; AVX2-NEXT:    vpinsrb $12, (%rax), %xmm3, %xmm3
+; AVX2-NEXT:    vmovq %xmm0, %rcx
+; AVX2-NEXT:    vpinsrb $12, (%rcx), %xmm3, %xmm3
 ; AVX2-NEXT:  .LBB3_26: # %else35
-; AVX2-NEXT:    vpextrb $13, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $8192, %eax # imm = 0x2000
 ; AVX2-NEXT:    je .LBB3_28
 ; AVX2-NEXT:  # %bb.27: # %cond.load37
-; AVX2-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX2-NEXT:    vpinsrb $13, (%rax), %xmm3, %xmm3
+; AVX2-NEXT:    vpextrq $1, %xmm0, %rcx
+; AVX2-NEXT:    vpinsrb $13, (%rcx), %xmm3, %xmm3
 ; AVX2-NEXT:  .LBB3_28: # %else38
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX2-NEXT:    vpextrb $14, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $16384, %eax # imm = 0x4000
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; AVX2-NEXT:    je .LBB3_30
-; AVX2-NEXT:  # %bb.29: # %cond.load40
-; AVX2-NEXT:    vmovq %xmm0, %rax
-; AVX2-NEXT:    vpinsrb $14, (%rax), %xmm3, %xmm3
-; AVX2-NEXT:  .LBB3_30: # %else41
-; AVX2-NEXT:    vpextrb $15, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    jne .LBB3_29
+; AVX2-NEXT:  # %bb.30: # %else41
+; AVX2-NEXT:    testl $32768, %eax # imm = 0x8000
+; AVX2-NEXT:    jne .LBB3_31
+; AVX2-NEXT:  .LBB3_32: # %else44
+; AVX2-NEXT:    vmovdqa %xmm3, %xmm0
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+; AVX2-NEXT:  .LBB3_29: # %cond.load40
+; AVX2-NEXT:    vmovq %xmm0, %rcx
+; AVX2-NEXT:    vpinsrb $14, (%rcx), %xmm3, %xmm3
+; AVX2-NEXT:    testl $32768, %eax # imm = 0x8000
 ; AVX2-NEXT:    je .LBB3_32
-; AVX2-NEXT:  # %bb.31: # %cond.load43
+; AVX2-NEXT:  .LBB3_31: # %cond.load43
 ; AVX2-NEXT:    vpextrq $1, %xmm0, %rax
 ; AVX2-NEXT:    vpinsrb $15, (%rax), %xmm3, %xmm3
-; AVX2-NEXT:  .LBB3_32: # %else44
 ; AVX2-NEXT:    vmovdqa %xmm3, %xmm0
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -948,175 +880,134 @@ define <16 x i8> @gather_v16i8_v16i32_v16i8(i8* %base, <16 x i32> %idx, <16 x i8
 ; AVX512-NEXT:    vpmovsxdq %ymm0, %zmm4
 ; AVX512-NEXT:    vpaddq %zmm4, %zmm3, %zmm4
 ; AVX512-NEXT:    vpxor %xmm5, %xmm5, %xmm5
-; AVX512-NEXT:    vpcmpeqb %xmm5, %xmm1, %xmm5
-; AVX512-NEXT:    vpmovsxbd %xmm5, %zmm5
-; AVX512-NEXT:    vptestmd %zmm5, %zmm5, %k0
-; AVX512-NEXT:    kmovw %k0, %eax
-; AVX512-NEXT:    testb $1, %al
-; AVX512-NEXT:    je .LBB3_2
-; AVX512-NEXT:  # %bb.1: # %cond.load
-; AVX512-NEXT:    vmovq %xmm4, %rax
-; AVX512-NEXT:    vpinsrb $0, (%rax), %xmm2, %xmm2
-; AVX512-NEXT:  .LBB3_2: # %else
-; AVX512-NEXT:    kshiftrw $1, %k0, %k0
-; AVX512-NEXT:    kmovw %k0, %eax
+; AVX512-NEXT:    vpcmpeqb %xmm5, %xmm1, %xmm1
+; AVX512-NEXT:    vpmovmskb %xmm1, %eax
 ; AVX512-NEXT:    testb $1, %al
-; AVX512-NEXT:    je .LBB3_4
-; AVX512-NEXT:  # %bb.3: # %cond.load1
-; AVX512-NEXT:    vpextrq $1, %xmm4, %rax
-; AVX512-NEXT:    vpinsrb $1, (%rax), %xmm2, %xmm2
+; AVX512-NEXT:    jne .LBB3_1
+; AVX512-NEXT:  # %bb.2: # %else
+; AVX512-NEXT:    testb $2, %al
+; AVX512-NEXT:    jne .LBB3_3
 ; AVX512-NEXT:  .LBB3_4: # %else2
-; AVX512-NEXT:    vpxor %xmm5, %xmm5, %xmm5
-; AVX512-NEXT:    vpcmpeqb %xmm5, %xmm1, %xmm5
-; AVX512-NEXT:    vpmovsxbd %xmm5, %zmm5
-; AVX512-NEXT:    vptestmd %zmm5, %zmm5, %k0
-; AVX512-NEXT:    kshiftrw $2, %k0, %k1
-; AVX512-NEXT:    kmovw %k1, %eax
-; AVX512-NEXT:    testb $1, %al
-; AVX512-NEXT:    je .LBB3_6
-; AVX512-NEXT:  # %bb.5: # %cond.load4
-; AVX512-NEXT:    vextracti128 $1, %ymm4, %xmm5
-; AVX512-NEXT:    vmovq %xmm5, %rax
-; AVX512-NEXT:    vpinsrb $2, (%rax), %xmm2, %xmm2
+; AVX512-NEXT:    testb $4, %al
+; AVX512-NEXT:    jne .LBB3_5
 ; AVX512-NEXT:  .LBB3_6: # %else5
-; AVX512-NEXT:    kshiftrw $3, %k0, %k0
-; AVX512-NEXT:    kmovw %k0, %eax
-; AVX512-NEXT:    testb $1, %al
+; AVX512-NEXT:    testb $8, %al
 ; AVX512-NEXT:    je .LBB3_8
-; AVX512-NEXT:  # %bb.7: # %cond.load7
-; AVX512-NEXT:    vextracti128 $1, %ymm4, %xmm5
-; AVX512-NEXT:    vpextrq $1, %xmm5, %rax
-; AVX512-NEXT:    vpinsrb $3, (%rax), %xmm2, %xmm2
+; AVX512-NEXT:  .LBB3_7: # %cond.load7
+; AVX512-NEXT:    vextracti128 $1, %ymm4, %xmm1
+; AVX512-NEXT:    vpextrq $1, %xmm1, %rcx
+; AVX512-NEXT:    vpinsrb $3, (%rcx), %xmm2, %xmm2
 ; AVX512-NEXT:  .LBB3_8: # %else8
-; AVX512-NEXT:    vpxor %xmm5, %xmm5, %xmm5
-; AVX512-NEXT:    vpcmpeqb %xmm5, %xmm1, %xmm5
-; AVX512-NEXT:    vpmovsxbd %xmm5, %zmm5
-; AVX512-NEXT:    vptestmd %zmm5, %zmm5, %k0
-; AVX512-NEXT:    kshiftrw $4, %k0, %k1
-; AVX512-NEXT:    kmovw %k1, %eax
-; AVX512-NEXT:    testb $1, %al
-; AVX512-NEXT:    vextracti32x4 $2, %zmm4, %xmm5
+; AVX512-NEXT:    testb $16, %al
+; AVX512-NEXT:    vextracti32x4 $2, %zmm4, %xmm1
 ; AVX512-NEXT:    je .LBB3_10
 ; AVX512-NEXT:  # %bb.9: # %cond.load10
-; AVX512-NEXT:    vmovq %xmm5, %rax
-; AVX512-NEXT:    vpinsrb $4, (%rax), %xmm2, %xmm2
+; AVX512-NEXT:    vmovq %xmm1, %rcx
+; AVX512-NEXT:    vpinsrb $4, (%rcx), %xmm2, %xmm2
 ; AVX512-NEXT:  .LBB3_10: # %else11
-; AVX512-NEXT:    kshiftrw $5, %k0, %k0
-; AVX512-NEXT:    kmovw %k0, %eax
-; AVX512-NEXT:    testb $1, %al
+; AVX512-NEXT:    testb $32, %al
 ; AVX512-NEXT:    je .LBB3_12
 ; AVX512-NEXT:  # %bb.11: # %cond.load13
-; AVX512-NEXT:    vpextrq $1, %xmm5, %rax
-; AVX512-NEXT:    vpinsrb $5, (%rax), %xmm2, %xmm2
+; AVX512-NEXT:    vpextrq $1, %xmm1, %rcx
+; AVX512-NEXT:    vpinsrb $5, (%rcx), %xmm2, %xmm2
 ; AVX512-NEXT:  .LBB3_12: # %else14
-; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
-; AVX512-NEXT:    vpxor %xmm5, %xmm5, %xmm5
-; AVX512-NEXT:    vpcmpeqb %xmm5, %xmm1, %xmm5
-; AVX512-NEXT:    vpmovsxbd %xmm5, %zmm5
-; AVX512-NEXT:    vptestmd %zmm5, %zmm5, %k0
-; AVX512-NEXT:    kshiftrw $6, %k0, %k1
-; AVX512-NEXT:    kmovw %k1, %eax
-; AVX512-NEXT:    testb $1, %al
-; AVX512-NEXT:    vextracti32x4 $3, %zmm4, %xmm4
+; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
+; AVX512-NEXT:    testb $64, %al
+; AVX512-NEXT:    vextracti32x4 $3, %zmm4, %xmm0
 ; AVX512-NEXT:    je .LBB3_14
 ; AVX512-NEXT:  # %bb.13: # %cond.load16
-; AVX512-NEXT:    vmovq %xmm4, %rax
-; AVX512-NEXT:    vpinsrb $6, (%rax), %xmm2, %xmm2
+; AVX512-NEXT:    vmovq %xmm0, %rcx
+; AVX512-NEXT:    vpinsrb $6, (%rcx), %xmm2, %xmm2
 ; AVX512-NEXT:  .LBB3_14: # %else17
-; AVX512-NEXT:    vpmovsxdq %ymm0, %zmm0
-; AVX512-NEXT:    kshiftrw $7, %k0, %k0
-; AVX512-NEXT:    kmovw %k0, %eax
-; AVX512-NEXT:    testb $1, %al
+; AVX512-NEXT:    vpmovsxdq %ymm1, %zmm1
+; AVX512-NEXT:    testb $-128, %al
 ; AVX512-NEXT:    je .LBB3_16
 ; AVX512-NEXT:  # %bb.15: # %cond.load19
-; AVX512-NEXT:    vpextrq $1, %xmm4, %rax
-; AVX512-NEXT:    vpinsrb $7, (%rax), %xmm2, %xmm2
+; AVX512-NEXT:    vpextrq $1, %xmm0, %rcx
+; AVX512-NEXT:    vpinsrb $7, (%rcx), %xmm2, %xmm2
 ; AVX512-NEXT:  .LBB3_16: # %else20
-; AVX512-NEXT:    vpaddq %zmm0, %zmm3, %zmm0
-; AVX512-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX512-NEXT:    vpcmpeqb %xmm3, %xmm1, %xmm3
-; AVX512-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512-NEXT:    kshiftrw $8, %k0, %k1
-; AVX512-NEXT:    kmovw %k1, %eax
-; AVX512-NEXT:    testb $1, %al
-; AVX512-NEXT:    je .LBB3_18
-; AVX512-NEXT:  # %bb.17: # %cond.load22
-; AVX512-NEXT:    vmovq %xmm0, %rax
-; AVX512-NEXT:    vpinsrb $8, (%rax), %xmm2, %xmm2
-; AVX512-NEXT:  .LBB3_18: # %else23
-; AVX512-NEXT:    kshiftrw $9, %k0, %k0
-; AVX512-NEXT:    kmovw %k0, %eax
-; AVX512-NEXT:    testb $1, %al
-; AVX512-NEXT:    je .LBB3_20
-; AVX512-NEXT:  # %bb.19: # %cond.load25
-; AVX512-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX512-NEXT:    vpinsrb $9, (%rax), %xmm2, %xmm2
+; AVX512-NEXT:    vpaddq %zmm1, %zmm3, %zmm0
+; AVX512-NEXT:    testl $256, %eax # imm = 0x100
+; AVX512-NEXT:    jne .LBB3_17
+; AVX512-NEXT:  # %bb.18: # %else23
+; AVX512-NEXT:    testl $512, %eax # imm = 0x200
+; AVX512-NEXT:    jne .LBB3_19
 ; AVX512-NEXT:  .LBB3_20: # %else26
-; AVX512-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX512-NEXT:    vpcmpeqb %xmm3, %xmm1, %xmm3
-; AVX512-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512-NEXT:    kshiftrw $10, %k0, %k1
-; AVX512-NEXT:    kmovw %k1, %eax
-; AVX512-NEXT:    testb $1, %al
-; AVX512-NEXT:    je .LBB3_22
-; AVX512-NEXT:  # %bb.21: # %cond.load28
-; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm3
-; AVX512-NEXT:    vmovq %xmm3, %rax
-; AVX512-NEXT:    vpinsrb $10, (%rax), %xmm2, %xmm2
+; AVX512-NEXT:    testl $1024, %eax # imm = 0x400
+; AVX512-NEXT:    jne .LBB3_21
 ; AVX512-NEXT:  .LBB3_22: # %else29
-; AVX512-NEXT:    kshiftrw $11, %k0, %k0
-; AVX512-NEXT:    kmovw %k0, %eax
-; AVX512-NEXT:    testb $1, %al
+; AVX512-NEXT:    testl $2048, %eax # imm = 0x800
 ; AVX512-NEXT:    je .LBB3_24
-; AVX512-NEXT:  # %bb.23: # %cond.load31
-; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm3
-; AVX512-NEXT:    vpextrq $1, %xmm3, %rax
-; AVX512-NEXT:    vpinsrb $11, (%rax), %xmm2, %xmm2
+; AVX512-NEXT:  .LBB3_23: # %cond.load31
+; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512-NEXT:    vpextrq $1, %xmm1, %rcx
+; AVX512-NEXT:    vpinsrb $11, (%rcx), %xmm2, %xmm2
 ; AVX512-NEXT:  .LBB3_24: # %else32
-; AVX512-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX512-NEXT:    vpcmpeqb %xmm3, %xmm1, %xmm3
-; AVX512-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512-NEXT:    kshiftrw $12, %k0, %k1
-; AVX512-NEXT:    kmovw %k1, %eax
-; AVX512-NEXT:    testb $1, %al
-; AVX512-NEXT:    vextracti32x4 $2, %zmm0, %xmm3
+; AVX512-NEXT:    testl $4096, %eax # imm = 0x1000
+; AVX512-NEXT:    vextracti32x4 $2, %zmm0, %xmm1
 ; AVX512-NEXT:    je .LBB3_26
 ; AVX512-NEXT:  # %bb.25: # %cond.load34
-; AVX512-NEXT:    vmovq %xmm3, %rax
-; AVX512-NEXT:    vpinsrb $12, (%rax), %xmm2, %xmm2
+; AVX512-NEXT:    vmovq %xmm1, %rcx
+; AVX512-NEXT:    vpinsrb $12, (%rcx), %xmm2, %xmm2
 ; AVX512-NEXT:  .LBB3_26: # %else35
-; AVX512-NEXT:    kshiftrw $13, %k0, %k0
-; AVX512-NEXT:    kmovw %k0, %eax
-; AVX512-NEXT:    testb $1, %al
+; AVX512-NEXT:    testl $8192, %eax # imm = 0x2000
 ; AVX512-NEXT:    je .LBB3_28
 ; AVX512-NEXT:  # %bb.27: # %cond.load37
-; AVX512-NEXT:    vpextrq $1, %xmm3, %rax
-; AVX512-NEXT:    vpinsrb $13, (%rax), %xmm2, %xmm2
+; AVX512-NEXT:    vpextrq $1, %xmm1, %rcx
+; AVX512-NEXT:    vpinsrb $13, (%rcx), %xmm2, %xmm2
 ; AVX512-NEXT:  .LBB3_28: # %else38
-; AVX512-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX512-NEXT:    vpcmpeqb %xmm3, %xmm1, %xmm1
-; AVX512-NEXT:    vpmovsxbd %xmm1, %zmm1
-; AVX512-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512-NEXT:    kshiftrw $14, %k0, %k1
-; AVX512-NEXT:    kmovw %k1, %eax
-; AVX512-NEXT:    testb $1, %al
+; AVX512-NEXT:    testl $16384, %eax # imm = 0x4000
 ; AVX512-NEXT:    vextracti32x4 $3, %zmm0, %xmm0
-; AVX512-NEXT:    je .LBB3_30
-; AVX512-NEXT:  # %bb.29: # %cond.load40
-; AVX512-NEXT:    vmovq %xmm0, %rax
-; AVX512-NEXT:    vpinsrb $14, (%rax), %xmm2, %xmm2
-; AVX512-NEXT:  .LBB3_30: # %else41
-; AVX512-NEXT:    kshiftrw $15, %k0, %k0
-; AVX512-NEXT:    kmovw %k0, %eax
-; AVX512-NEXT:    testb $1, %al
+; AVX512-NEXT:    jne .LBB3_29
+; AVX512-NEXT:  # %bb.30: # %else41
+; AVX512-NEXT:    testl $32768, %eax # imm = 0x8000
+; AVX512-NEXT:    jne .LBB3_31
+; AVX512-NEXT:  .LBB3_32: # %else44
+; AVX512-NEXT:    vmovdqa %xmm2, %xmm0
+; AVX512-NEXT:    vzeroupper
+; AVX512-NEXT:    retq
+; AVX512-NEXT:  .LBB3_1: # %cond.load
+; AVX512-NEXT:    vmovq %xmm4, %rcx
+; AVX512-NEXT:    vpinsrb $0, (%rcx), %xmm2, %xmm2
+; AVX512-NEXT:    testb $2, %al
+; AVX512-NEXT:    je .LBB3_4
+; AVX512-NEXT:  .LBB3_3: # %cond.load1
+; AVX512-NEXT:    vpextrq $1, %xmm4, %rcx
+; AVX512-NEXT:    vpinsrb $1, (%rcx), %xmm2, %xmm2
+; AVX512-NEXT:    testb $4, %al
+; AVX512-NEXT:    je .LBB3_6
+; AVX512-NEXT:  .LBB3_5: # %cond.load4
+; AVX512-NEXT:    vextracti128 $1, %ymm4, %xmm1
+; AVX512-NEXT:    vmovq %xmm1, %rcx
+; AVX512-NEXT:    vpinsrb $2, (%rcx), %xmm2, %xmm2
+; AVX512-NEXT:    testb $8, %al
+; AVX512-NEXT:    jne .LBB3_7
+; AVX512-NEXT:    jmp .LBB3_8
+; AVX512-NEXT:  .LBB3_17: # %cond.load22
+; AVX512-NEXT:    vmovq %xmm0, %rcx
+; AVX512-NEXT:    vpinsrb $8, (%rcx), %xmm2, %xmm2
+; AVX512-NEXT:    testl $512, %eax # imm = 0x200
+; AVX512-NEXT:    je .LBB3_20
+; AVX512-NEXT:  .LBB3_19: # %cond.load25
+; AVX512-NEXT:    vpextrq $1, %xmm0, %rcx
+; AVX512-NEXT:    vpinsrb $9, (%rcx), %xmm2, %xmm2
+; AVX512-NEXT:    testl $1024, %eax # imm = 0x400
+; AVX512-NEXT:    je .LBB3_22
+; AVX512-NEXT:  .LBB3_21: # %cond.load28
+; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512-NEXT:    vmovq %xmm1, %rcx
+; AVX512-NEXT:    vpinsrb $10, (%rcx), %xmm2, %xmm2
+; AVX512-NEXT:    testl $2048, %eax # imm = 0x800
+; AVX512-NEXT:    jne .LBB3_23
+; AVX512-NEXT:    jmp .LBB3_24
+; AVX512-NEXT:  .LBB3_29: # %cond.load40
+; AVX512-NEXT:    vmovq %xmm0, %rcx
+; AVX512-NEXT:    vpinsrb $14, (%rcx), %xmm2, %xmm2
+; AVX512-NEXT:    testl $32768, %eax # imm = 0x8000
 ; AVX512-NEXT:    je .LBB3_32
-; AVX512-NEXT:  # %bb.31: # %cond.load43
+; AVX512-NEXT:  .LBB3_31: # %cond.load43
 ; AVX512-NEXT:    vpextrq $1, %xmm0, %rax
 ; AVX512-NEXT:    vpinsrb $15, (%rax), %xmm2, %xmm2
-; AVX512-NEXT:  .LBB3_32: # %else44
 ; AVX512-NEXT:    vmovdqa %xmm2, %xmm0
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
@@ -1129,6 +1020,736 @@ define <16 x i8> @gather_v16i8_v16i32_v16i8(i8* %base, <16 x i32> %idx, <16 x i8
   ret <16 x i8> %res
 }
 
+define <8 x i32> @gather_v8i32_v8i32(<8 x i32> %trigger) {
+; SSE-LABEL: gather_v8i32_v8i32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqa %xmm1, %xmm3
+; SSE-NEXT:    movdqa %xmm0, %xmm2
+; SSE-NEXT:    movl $c, %eax
+; SSE-NEXT:    movq %rax, %xmm0
+; SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[0,1,0,1]
+; SSE-NEXT:    movdqa {{.*#+}} xmm5 = [12,12]
+; SSE-NEXT:    paddq %xmm4, %xmm5
+; SSE-NEXT:    pxor %xmm0, %xmm0
+; SSE-NEXT:    pcmpeqd %xmm0, %xmm1
+; SSE-NEXT:    pcmpeqd %xmm2, %xmm0
+; SSE-NEXT:    packssdw %xmm1, %xmm0
+; SSE-NEXT:    packsswb %xmm0, %xmm0
+; SSE-NEXT:    pmovmskb %xmm0, %eax
+; SSE-NEXT:    testb $1, %al
+; SSE-NEXT:    je .LBB4_1
+; SSE-NEXT:  # %bb.2: # %cond.load
+; SSE-NEXT:    movq %xmm5, %rcx
+; SSE-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE-NEXT:    testb $2, %al
+; SSE-NEXT:    jne .LBB4_4
+; SSE-NEXT:    jmp .LBB4_5
+; SSE-NEXT:  .LBB4_1:
+; SSE-NEXT:    # implicit-def: $xmm0
+; SSE-NEXT:    testb $2, %al
+; SSE-NEXT:    je .LBB4_5
+; SSE-NEXT:  .LBB4_4: # %cond.load1
+; SSE-NEXT:    pextrq $1, %xmm5, %rcx
+; SSE-NEXT:    pinsrd $1, (%rcx), %xmm0
+; SSE-NEXT:  .LBB4_5: # %else2
+; SSE-NEXT:    testb $4, %al
+; SSE-NEXT:    jne .LBB4_6
+; SSE-NEXT:  # %bb.7: # %else5
+; SSE-NEXT:    testb $8, %al
+; SSE-NEXT:    jne .LBB4_8
+; SSE-NEXT:  .LBB4_9: # %else8
+; SSE-NEXT:    testb $16, %al
+; SSE-NEXT:    je .LBB4_10
+; SSE-NEXT:  .LBB4_11: # %cond.load10
+; SSE-NEXT:    movq %xmm5, %rcx
+; SSE-NEXT:    pinsrd $0, (%rcx), %xmm1
+; SSE-NEXT:    testb $32, %al
+; SSE-NEXT:    jne .LBB4_13
+; SSE-NEXT:    jmp .LBB4_14
+; SSE-NEXT:  .LBB4_6: # %cond.load4
+; SSE-NEXT:    movq %xmm5, %rcx
+; SSE-NEXT:    pinsrd $2, (%rcx), %xmm0
+; SSE-NEXT:    testb $8, %al
+; SSE-NEXT:    je .LBB4_9
+; SSE-NEXT:  .LBB4_8: # %cond.load7
+; SSE-NEXT:    pextrq $1, %xmm5, %rcx
+; SSE-NEXT:    pinsrd $3, (%rcx), %xmm0
+; SSE-NEXT:    testb $16, %al
+; SSE-NEXT:    jne .LBB4_11
+; SSE-NEXT:  .LBB4_10:
+; SSE-NEXT:    # implicit-def: $xmm1
+; SSE-NEXT:    testb $32, %al
+; SSE-NEXT:    je .LBB4_14
+; SSE-NEXT:  .LBB4_13: # %cond.load13
+; SSE-NEXT:    pextrq $1, %xmm5, %rcx
+; SSE-NEXT:    pinsrd $1, (%rcx), %xmm1
+; SSE-NEXT:  .LBB4_14: # %else14
+; SSE-NEXT:    testb $64, %al
+; SSE-NEXT:    jne .LBB4_15
+; SSE-NEXT:  # %bb.16: # %else17
+; SSE-NEXT:    testb $-128, %al
+; SSE-NEXT:    je .LBB4_18
+; SSE-NEXT:  .LBB4_17: # %cond.load19
+; SSE-NEXT:    pextrq $1, %xmm5, %rax
+; SSE-NEXT:    pinsrd $3, (%rax), %xmm1
+; SSE-NEXT:  .LBB4_18: # %else20
+; SSE-NEXT:    pxor %xmm5, %xmm5
+; SSE-NEXT:    paddq {{.*}}(%rip), %xmm4
+; SSE-NEXT:    movdqa %xmm2, %xmm6
+; SSE-NEXT:    pcmpeqd %xmm5, %xmm6
+; SSE-NEXT:    pcmpeqd %xmm3, %xmm5
+; SSE-NEXT:    packssdw %xmm5, %xmm6
+; SSE-NEXT:    packsswb %xmm0, %xmm6
+; SSE-NEXT:    pmovmskb %xmm6, %eax
+; SSE-NEXT:    testb $1, %al
+; SSE-NEXT:    je .LBB4_19
+; SSE-NEXT:  # %bb.20: # %cond.load23
+; SSE-NEXT:    movq %xmm4, %rcx
+; SSE-NEXT:    movd {{.*#+}} xmm5 = mem[0],zero,zero,zero
+; SSE-NEXT:    testb $2, %al
+; SSE-NEXT:    jne .LBB4_22
+; SSE-NEXT:    jmp .LBB4_23
+; SSE-NEXT:  .LBB4_15: # %cond.load16
+; SSE-NEXT:    movq %xmm5, %rcx
+; SSE-NEXT:    pinsrd $2, (%rcx), %xmm1
+; SSE-NEXT:    testb $-128, %al
+; SSE-NEXT:    jne .LBB4_17
+; SSE-NEXT:    jmp .LBB4_18
+; SSE-NEXT:  .LBB4_19:
+; SSE-NEXT:    # implicit-def: $xmm5
+; SSE-NEXT:    testb $2, %al
+; SSE-NEXT:    je .LBB4_23
+; SSE-NEXT:  .LBB4_22: # %cond.load29
+; SSE-NEXT:    pextrq $1, %xmm4, %rcx
+; SSE-NEXT:    pinsrd $1, (%rcx), %xmm5
+; SSE-NEXT:  .LBB4_23: # %else33
+; SSE-NEXT:    testb $4, %al
+; SSE-NEXT:    jne .LBB4_24
+; SSE-NEXT:  # %bb.25: # %else39
+; SSE-NEXT:    testb $8, %al
+; SSE-NEXT:    jne .LBB4_26
+; SSE-NEXT:  .LBB4_27: # %else45
+; SSE-NEXT:    testb $16, %al
+; SSE-NEXT:    je .LBB4_28
+; SSE-NEXT:  .LBB4_29: # %cond.load47
+; SSE-NEXT:    movq %xmm4, %rcx
+; SSE-NEXT:    pinsrd $0, (%rcx), %xmm6
+; SSE-NEXT:    testb $32, %al
+; SSE-NEXT:    jne .LBB4_31
+; SSE-NEXT:    jmp .LBB4_32
+; SSE-NEXT:  .LBB4_24: # %cond.load35
+; SSE-NEXT:    movq %xmm4, %rcx
+; SSE-NEXT:    pinsrd $2, (%rcx), %xmm5
+; SSE-NEXT:    testb $8, %al
+; SSE-NEXT:    je .LBB4_27
+; SSE-NEXT:  .LBB4_26: # %cond.load41
+; SSE-NEXT:    pextrq $1, %xmm4, %rcx
+; SSE-NEXT:    pinsrd $3, (%rcx), %xmm5
+; SSE-NEXT:    testb $16, %al
+; SSE-NEXT:    jne .LBB4_29
+; SSE-NEXT:  .LBB4_28:
+; SSE-NEXT:    # implicit-def: $xmm6
+; SSE-NEXT:    testb $32, %al
+; SSE-NEXT:    je .LBB4_32
+; SSE-NEXT:  .LBB4_31: # %cond.load53
+; SSE-NEXT:    pextrq $1, %xmm4, %rcx
+; SSE-NEXT:    pinsrd $1, (%rcx), %xmm6
+; SSE-NEXT:  .LBB4_32: # %else57
+; SSE-NEXT:    testb $64, %al
+; SSE-NEXT:    jne .LBB4_33
+; SSE-NEXT:  # %bb.34: # %else63
+; SSE-NEXT:    testb $-128, %al
+; SSE-NEXT:    je .LBB4_36
+; SSE-NEXT:  .LBB4_35: # %cond.load65
+; SSE-NEXT:    pextrq $1, %xmm4, %rax
+; SSE-NEXT:    pinsrd $3, (%rax), %xmm6
+; SSE-NEXT:  .LBB4_36: # %else69
+; SSE-NEXT:    pxor %xmm7, %xmm7
+; SSE-NEXT:    pcmpeqd %xmm7, %xmm2
+; SSE-NEXT:    pcmpeqd %xmm7, %xmm3
+; SSE-NEXT:    packssdw %xmm3, %xmm2
+; SSE-NEXT:    packsswb %xmm0, %xmm2
+; SSE-NEXT:    pmovmskb %xmm2, %eax
+; SSE-NEXT:    testb $1, %al
+; SSE-NEXT:    je .LBB4_37
+; SSE-NEXT:  # %bb.38: # %cond.load72
+; SSE-NEXT:    movq %xmm4, %rcx
+; SSE-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE-NEXT:    testb $2, %al
+; SSE-NEXT:    jne .LBB4_40
+; SSE-NEXT:    jmp .LBB4_41
+; SSE-NEXT:  .LBB4_33: # %cond.load59
+; SSE-NEXT:    movq %xmm4, %rcx
+; SSE-NEXT:    pinsrd $2, (%rcx), %xmm6
+; SSE-NEXT:    testb $-128, %al
+; SSE-NEXT:    jne .LBB4_35
+; SSE-NEXT:    jmp .LBB4_36
+; SSE-NEXT:  .LBB4_37:
+; SSE-NEXT:    # implicit-def: $xmm2
+; SSE-NEXT:    testb $2, %al
+; SSE-NEXT:    je .LBB4_41
+; SSE-NEXT:  .LBB4_40: # %cond.load78
+; SSE-NEXT:    pextrq $1, %xmm4, %rcx
+; SSE-NEXT:    pinsrd $1, (%rcx), %xmm2
+; SSE-NEXT:  .LBB4_41: # %else82
+; SSE-NEXT:    testb $4, %al
+; SSE-NEXT:    jne .LBB4_42
+; SSE-NEXT:  # %bb.43: # %else88
+; SSE-NEXT:    testb $8, %al
+; SSE-NEXT:    jne .LBB4_44
+; SSE-NEXT:  .LBB4_45: # %else94
+; SSE-NEXT:    testb $16, %al
+; SSE-NEXT:    je .LBB4_46
+; SSE-NEXT:  .LBB4_47: # %cond.load96
+; SSE-NEXT:    movq %xmm4, %rcx
+; SSE-NEXT:    pinsrd $0, (%rcx), %xmm3
+; SSE-NEXT:    testb $32, %al
+; SSE-NEXT:    jne .LBB4_49
+; SSE-NEXT:    jmp .LBB4_50
+; SSE-NEXT:  .LBB4_42: # %cond.load84
+; SSE-NEXT:    movq %xmm4, %rcx
+; SSE-NEXT:    pinsrd $2, (%rcx), %xmm2
+; SSE-NEXT:    testb $8, %al
+; SSE-NEXT:    je .LBB4_45
+; SSE-NEXT:  .LBB4_44: # %cond.load90
+; SSE-NEXT:    pextrq $1, %xmm4, %rcx
+; SSE-NEXT:    pinsrd $3, (%rcx), %xmm2
+; SSE-NEXT:    testb $16, %al
+; SSE-NEXT:    jne .LBB4_47
+; SSE-NEXT:  .LBB4_46:
+; SSE-NEXT:    # implicit-def: $xmm3
+; SSE-NEXT:    testb $32, %al
+; SSE-NEXT:    je .LBB4_50
+; SSE-NEXT:  .LBB4_49: # %cond.load102
+; SSE-NEXT:    pextrq $1, %xmm4, %rcx
+; SSE-NEXT:    pinsrd $1, (%rcx), %xmm3
+; SSE-NEXT:  .LBB4_50: # %else106
+; SSE-NEXT:    testb $64, %al
+; SSE-NEXT:    je .LBB4_52
+; SSE-NEXT:  # %bb.51: # %cond.load108
+; SSE-NEXT:    movq %xmm4, %rcx
+; SSE-NEXT:    pinsrd $2, (%rcx), %xmm3
+; SSE-NEXT:  .LBB4_52: # %else112
+; SSE-NEXT:    paddd %xmm5, %xmm0
+; SSE-NEXT:    paddd %xmm6, %xmm1
+; SSE-NEXT:    testb $-128, %al
+; SSE-NEXT:    je .LBB4_54
+; SSE-NEXT:  # %bb.53: # %cond.load114
+; SSE-NEXT:    pextrq $1, %xmm4, %rax
+; SSE-NEXT:    pinsrd $3, (%rax), %xmm3
+; SSE-NEXT:  .LBB4_54: # %else118
+; SSE-NEXT:    paddd %xmm3, %xmm1
+; SSE-NEXT:    paddd %xmm2, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: gather_v8i32_v8i32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    movl $c, %eax
+; AVX1-NEXT:    vmovq %rax, %xmm1
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[0,1,0,1]
+; AVX1-NEXT:    vpaddq {{.*}}(%rip), %xmm3, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm1, %ymm4
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm9
+; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm9, %xmm5
+; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm1, %ymm1
+; AVX1-NEXT:    vmovmskps %ymm1, %eax
+; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    # implicit-def: $ymm1
+; AVX1-NEXT:    je .LBB4_2
+; AVX1-NEXT:  # %bb.1: # %cond.load
+; AVX1-NEXT:    vmovq %xmm4, %rcx
+; AVX1-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX1-NEXT:  .LBB4_2: # %else
+; AVX1-NEXT:    testb $2, %al
+; AVX1-NEXT:    je .LBB4_4
+; AVX1-NEXT:  # %bb.3: # %cond.load1
+; AVX1-NEXT:    vpextrq $1, %xmm4, %rcx
+; AVX1-NEXT:    vpinsrd $1, (%rcx), %xmm1, %xmm5
+; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm5[0,1,2,3],ymm1[4,5,6,7]
+; AVX1-NEXT:  .LBB4_4: # %else2
+; AVX1-NEXT:    testb $4, %al
+; AVX1-NEXT:    vextractf128 $1, %ymm4, %xmm6
+; AVX1-NEXT:    jne .LBB4_5
+; AVX1-NEXT:  # %bb.6: # %else5
+; AVX1-NEXT:    testb $8, %al
+; AVX1-NEXT:    jne .LBB4_7
+; AVX1-NEXT:  .LBB4_8: # %else8
+; AVX1-NEXT:    testb $16, %al
+; AVX1-NEXT:    jne .LBB4_9
+; AVX1-NEXT:  .LBB4_10: # %else11
+; AVX1-NEXT:    testb $32, %al
+; AVX1-NEXT:    jne .LBB4_11
+; AVX1-NEXT:  .LBB4_12: # %else14
+; AVX1-NEXT:    testb $64, %al
+; AVX1-NEXT:    je .LBB4_14
+; AVX1-NEXT:  .LBB4_13: # %cond.load16
+; AVX1-NEXT:    vmovq %xmm6, %rcx
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm4
+; AVX1-NEXT:    vpinsrd $2, (%rcx), %xmm4, %xmm4
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm1, %ymm1
+; AVX1-NEXT:  .LBB4_14: # %else17
+; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm3, %ymm5
+; AVX1-NEXT:    testb $-128, %al
+; AVX1-NEXT:    je .LBB4_16
+; AVX1-NEXT:  # %bb.15: # %cond.load19
+; AVX1-NEXT:    vpextrq $1, %xmm6, %rax
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
+; AVX1-NEXT:    vpinsrd $3, (%rax), %xmm3, %xmm3
+; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
+; AVX1-NEXT:  .LBB4_16: # %else20
+; AVX1-NEXT:    vextractf128 $1, %ymm5, %xmm3
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm10 = [28,28]
+; AVX1-NEXT:    vpaddq %xmm3, %xmm10, %xmm3
+; AVX1-NEXT:    vpaddq %xmm5, %xmm10, %xmm4
+; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm4, %ymm7
+; AVX1-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm9, %xmm2
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm0, %xmm4
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm4, %ymm2
+; AVX1-NEXT:    vmovmskps %ymm2, %eax
+; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    # implicit-def: $ymm4
+; AVX1-NEXT:    je .LBB4_18
+; AVX1-NEXT:  # %bb.17: # %cond.load23
+; AVX1-NEXT:    vmovq %xmm7, %rcx
+; AVX1-NEXT:    vmovd {{.*#+}} xmm4 = mem[0],zero,zero,zero
+; AVX1-NEXT:  .LBB4_18: # %else27
+; AVX1-NEXT:    testb $2, %al
+; AVX1-NEXT:    je .LBB4_20
+; AVX1-NEXT:  # %bb.19: # %cond.load29
+; AVX1-NEXT:    vpextrq $1, %xmm7, %rcx
+; AVX1-NEXT:    vpinsrd $1, (%rcx), %xmm4, %xmm2
+; AVX1-NEXT:    vblendps {{.*#+}} ymm4 = ymm2[0,1,2,3],ymm4[4,5,6,7]
+; AVX1-NEXT:  .LBB4_20: # %else33
+; AVX1-NEXT:    vpaddq %xmm5, %xmm10, %xmm8
+; AVX1-NEXT:    testb $4, %al
+; AVX1-NEXT:    vextractf128 $1, %ymm7, %xmm7
+; AVX1-NEXT:    je .LBB4_22
+; AVX1-NEXT:  # %bb.21: # %cond.load35
+; AVX1-NEXT:    vmovq %xmm7, %rcx
+; AVX1-NEXT:    vpinsrd $2, (%rcx), %xmm4, %xmm2
+; AVX1-NEXT:    vblendps {{.*#+}} ymm4 = ymm2[0,1,2,3],ymm4[4,5,6,7]
+; AVX1-NEXT:  .LBB4_22: # %else39
+; AVX1-NEXT:    testb $8, %al
+; AVX1-NEXT:    je .LBB4_24
+; AVX1-NEXT:  # %bb.23: # %cond.load41
+; AVX1-NEXT:    vpextrq $1, %xmm7, %rcx
+; AVX1-NEXT:    vpinsrd $3, (%rcx), %xmm4, %xmm2
+; AVX1-NEXT:    vblendps {{.*#+}} ymm4 = ymm2[0,1,2,3],ymm4[4,5,6,7]
+; AVX1-NEXT:  .LBB4_24: # %else45
+; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm8, %ymm7
+; AVX1-NEXT:    testb $16, %al
+; AVX1-NEXT:    je .LBB4_26
+; AVX1-NEXT:  # %bb.25: # %cond.load47
+; AVX1-NEXT:    vmovq %xmm7, %rcx
+; AVX1-NEXT:    vextractf128 $1, %ymm4, %xmm2
+; AVX1-NEXT:    vpinsrd $0, (%rcx), %xmm2, %xmm2
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm4, %ymm4
+; AVX1-NEXT:  .LBB4_26: # %else51
+; AVX1-NEXT:    testb $32, %al
+; AVX1-NEXT:    je .LBB4_28
+; AVX1-NEXT:  # %bb.27: # %cond.load53
+; AVX1-NEXT:    vpextrq $1, %xmm7, %rcx
+; AVX1-NEXT:    vextractf128 $1, %ymm4, %xmm2
+; AVX1-NEXT:    vpinsrd $1, (%rcx), %xmm2, %xmm2
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm4, %ymm4
+; AVX1-NEXT:  .LBB4_28: # %else57
+; AVX1-NEXT:    testb $64, %al
+; AVX1-NEXT:    vextractf128 $1, %ymm7, %xmm7
+; AVX1-NEXT:    je .LBB4_30
+; AVX1-NEXT:  # %bb.29: # %cond.load59
+; AVX1-NEXT:    vmovq %xmm7, %rcx
+; AVX1-NEXT:    vextractf128 $1, %ymm4, %xmm2
+; AVX1-NEXT:    vpinsrd $2, (%rcx), %xmm2, %xmm2
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm4, %ymm4
+; AVX1-NEXT:  .LBB4_30: # %else63
+; AVX1-NEXT:    testb $-128, %al
+; AVX1-NEXT:    je .LBB4_32
+; AVX1-NEXT:  # %bb.31: # %cond.load65
+; AVX1-NEXT:    vpextrq $1, %xmm7, %rax
+; AVX1-NEXT:    vextractf128 $1, %ymm4, %xmm2
+; AVX1-NEXT:    vpinsrd $3, (%rax), %xmm2, %xmm2
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm4, %ymm4
+; AVX1-NEXT:  .LBB4_32: # %else69
+; AVX1-NEXT:    vpaddq %xmm5, %xmm10, %xmm2
+; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm7
+; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm9, %xmm6
+; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm0, %ymm0
+; AVX1-NEXT:    vmovmskps %ymm0, %eax
+; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    # implicit-def: $ymm0
+; AVX1-NEXT:    je .LBB4_34
+; AVX1-NEXT:  # %bb.33: # %cond.load72
+; AVX1-NEXT:    vmovq %xmm7, %rcx
+; AVX1-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX1-NEXT:  .LBB4_34: # %else76
+; AVX1-NEXT:    testb $2, %al
+; AVX1-NEXT:    je .LBB4_36
+; AVX1-NEXT:  # %bb.35: # %cond.load78
+; AVX1-NEXT:    vpextrq $1, %xmm7, %rcx
+; AVX1-NEXT:    vpinsrd $1, (%rcx), %xmm0, %xmm2
+; AVX1-NEXT:    vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
+; AVX1-NEXT:  .LBB4_36: # %else82
+; AVX1-NEXT:    vpaddq %xmm5, %xmm10, %xmm2
+; AVX1-NEXT:    testb $4, %al
+; AVX1-NEXT:    vextractf128 $1, %ymm7, %xmm5
+; AVX1-NEXT:    je .LBB4_38
+; AVX1-NEXT:  # %bb.37: # %cond.load84
+; AVX1-NEXT:    vmovq %xmm5, %rcx
+; AVX1-NEXT:    vpinsrd $2, (%rcx), %xmm0, %xmm6
+; AVX1-NEXT:    vblendps {{.*#+}} ymm0 = ymm6[0,1,2,3],ymm0[4,5,6,7]
+; AVX1-NEXT:  .LBB4_38: # %else88
+; AVX1-NEXT:    testb $8, %al
+; AVX1-NEXT:    je .LBB4_40
+; AVX1-NEXT:  # %bb.39: # %cond.load90
+; AVX1-NEXT:    vpextrq $1, %xmm5, %rcx
+; AVX1-NEXT:    vpinsrd $3, (%rcx), %xmm0, %xmm5
+; AVX1-NEXT:    vblendps {{.*#+}} ymm0 = ymm5[0,1,2,3],ymm0[4,5,6,7]
+; AVX1-NEXT:  .LBB4_40: # %else94
+; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
+; AVX1-NEXT:    testb $16, %al
+; AVX1-NEXT:    je .LBB4_42
+; AVX1-NEXT:  # %bb.41: # %cond.load96
+; AVX1-NEXT:    vmovq %xmm2, %rcx
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; AVX1-NEXT:    vpinsrd $0, (%rcx), %xmm3, %xmm3
+; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
+; AVX1-NEXT:  .LBB4_42: # %else100
+; AVX1-NEXT:    vextractf128 $1, %ymm4, %xmm3
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm5
+; AVX1-NEXT:    vpaddd %xmm4, %xmm1, %xmm1
+; AVX1-NEXT:    testb $32, %al
+; AVX1-NEXT:    je .LBB4_44
+; AVX1-NEXT:  # %bb.43: # %cond.load102
+; AVX1-NEXT:    vpextrq $1, %xmm2, %rcx
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
+; AVX1-NEXT:    vpinsrd $1, (%rcx), %xmm4, %xmm4
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
+; AVX1-NEXT:  .LBB4_44: # %else106
+; AVX1-NEXT:    vpaddd %xmm3, %xmm5, %xmm3
+; AVX1-NEXT:    testb $64, %al
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm2
+; AVX1-NEXT:    je .LBB4_46
+; AVX1-NEXT:  # %bb.45: # %cond.load108
+; AVX1-NEXT:    vmovq %xmm2, %rcx
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
+; AVX1-NEXT:    vpinsrd $2, (%rcx), %xmm4, %xmm4
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
+; AVX1-NEXT:  .LBB4_46: # %else112
+; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
+; AVX1-NEXT:    testb $-128, %al
+; AVX1-NEXT:    je .LBB4_48
+; AVX1-NEXT:  # %bb.47: # %cond.load114
+; AVX1-NEXT:    vpextrq $1, %xmm2, %rax
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT:    vpinsrd $3, (%rax), %xmm2, %xmm2
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:  .LBB4_48: # %else118
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
+; AVX1-NEXT:    vpaddd %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+; AVX1-NEXT:  .LBB4_5: # %cond.load4
+; AVX1-NEXT:    vmovq %xmm6, %rcx
+; AVX1-NEXT:    vpinsrd $2, (%rcx), %xmm1, %xmm5
+; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm5[0,1,2,3],ymm1[4,5,6,7]
+; AVX1-NEXT:    testb $8, %al
+; AVX1-NEXT:    je .LBB4_8
+; AVX1-NEXT:  .LBB4_7: # %cond.load7
+; AVX1-NEXT:    vpextrq $1, %xmm6, %rcx
+; AVX1-NEXT:    vpinsrd $3, (%rcx), %xmm1, %xmm5
+; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm5[0,1,2,3],ymm1[4,5,6,7]
+; AVX1-NEXT:    testb $16, %al
+; AVX1-NEXT:    je .LBB4_10
+; AVX1-NEXT:  .LBB4_9: # %cond.load10
+; AVX1-NEXT:    vmovq %xmm4, %rcx
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm5
+; AVX1-NEXT:    vpinsrd $0, (%rcx), %xmm5, %xmm5
+; AVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm1, %ymm1
+; AVX1-NEXT:    testb $32, %al
+; AVX1-NEXT:    je .LBB4_12
+; AVX1-NEXT:  .LBB4_11: # %cond.load13
+; AVX1-NEXT:    vpextrq $1, %xmm4, %rcx
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm4
+; AVX1-NEXT:    vpinsrd $1, (%rcx), %xmm4, %xmm4
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm1, %ymm1
+; AVX1-NEXT:    testb $64, %al
+; AVX1-NEXT:    jne .LBB4_13
+; AVX1-NEXT:    jmp .LBB4_14
+;
+; AVX2-LABEL: gather_v8i32_v8i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    movl $c, %eax
+; AVX2-NEXT:    vmovq %rax, %xmm1
+; AVX2-NEXT:    vpbroadcastq %xmm1, %ymm2
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [12,12,12,12]
+; AVX2-NEXT:    vpaddq %ymm1, %ymm2, %ymm3
+; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT:    vpcmpeqd %ymm1, %ymm0, %ymm1
+; AVX2-NEXT:    vmovmskps %ymm1, %eax
+; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    # implicit-def: $ymm1
+; AVX2-NEXT:    je .LBB4_2
+; AVX2-NEXT:  # %bb.1: # %cond.load
+; AVX2-NEXT:    vmovq %xmm3, %rcx
+; AVX2-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX2-NEXT:  .LBB4_2: # %else
+; AVX2-NEXT:    testb $2, %al
+; AVX2-NEXT:    je .LBB4_4
+; AVX2-NEXT:  # %bb.3: # %cond.load1
+; AVX2-NEXT:    vpextrq $1, %xmm3, %rcx
+; AVX2-NEXT:    vpinsrd $1, (%rcx), %xmm1, %xmm4
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-NEXT:  .LBB4_4: # %else2
+; AVX2-NEXT:    testb $4, %al
+; AVX2-NEXT:    vextracti128 $1, %ymm3, %xmm4
+; AVX2-NEXT:    jne .LBB4_5
+; AVX2-NEXT:  # %bb.6: # %else5
+; AVX2-NEXT:    testb $8, %al
+; AVX2-NEXT:    jne .LBB4_7
+; AVX2-NEXT:  .LBB4_8: # %else8
+; AVX2-NEXT:    testb $16, %al
+; AVX2-NEXT:    jne .LBB4_9
+; AVX2-NEXT:  .LBB4_10: # %else11
+; AVX2-NEXT:    testb $32, %al
+; AVX2-NEXT:    jne .LBB4_11
+; AVX2-NEXT:  .LBB4_12: # %else14
+; AVX2-NEXT:    testb $64, %al
+; AVX2-NEXT:    jne .LBB4_13
+; AVX2-NEXT:  .LBB4_14: # %else17
+; AVX2-NEXT:    testb $-128, %al
+; AVX2-NEXT:    je .LBB4_16
+; AVX2-NEXT:  .LBB4_15: # %cond.load19
+; AVX2-NEXT:    vpextrq $1, %xmm4, %rax
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm3
+; AVX2-NEXT:    vpinsrd $3, (%rax), %xmm3, %xmm3
+; AVX2-NEXT:    vinserti128 $1, %xmm3, %ymm1, %ymm1
+; AVX2-NEXT:  .LBB4_16: # %else20
+; AVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [28,28,28,28]
+; AVX2-NEXT:    vpaddq %ymm3, %ymm2, %ymm3
+; AVX2-NEXT:    vpcmpeqd %ymm4, %ymm0, %ymm2
+; AVX2-NEXT:    vmovmskps %ymm2, %eax
+; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    # implicit-def: $ymm2
+; AVX2-NEXT:    je .LBB4_18
+; AVX2-NEXT:  # %bb.17: # %cond.load23
+; AVX2-NEXT:    vmovq %xmm3, %rcx
+; AVX2-NEXT:    vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; AVX2-NEXT:  .LBB4_18: # %else27
+; AVX2-NEXT:    testb $2, %al
+; AVX2-NEXT:    je .LBB4_20
+; AVX2-NEXT:  # %bb.19: # %cond.load29
+; AVX2-NEXT:    vpextrq $1, %xmm3, %rcx
+; AVX2-NEXT:    vpinsrd $1, (%rcx), %xmm2, %xmm4
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3],ymm2[4,5,6,7]
+; AVX2-NEXT:  .LBB4_20: # %else33
+; AVX2-NEXT:    testb $4, %al
+; AVX2-NEXT:    vextracti128 $1, %ymm3, %xmm4
+; AVX2-NEXT:    jne .LBB4_21
+; AVX2-NEXT:  # %bb.22: # %else39
+; AVX2-NEXT:    testb $8, %al
+; AVX2-NEXT:    jne .LBB4_23
+; AVX2-NEXT:  .LBB4_24: # %else45
+; AVX2-NEXT:    testb $16, %al
+; AVX2-NEXT:    jne .LBB4_25
+; AVX2-NEXT:  .LBB4_26: # %else51
+; AVX2-NEXT:    testb $32, %al
+; AVX2-NEXT:    jne .LBB4_27
+; AVX2-NEXT:  .LBB4_28: # %else57
+; AVX2-NEXT:    testb $64, %al
+; AVX2-NEXT:    jne .LBB4_29
+; AVX2-NEXT:  .LBB4_30: # %else63
+; AVX2-NEXT:    testb $-128, %al
+; AVX2-NEXT:    je .LBB4_32
+; AVX2-NEXT:  .LBB4_31: # %cond.load65
+; AVX2-NEXT:    vpextrq $1, %xmm4, %rax
+; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm5
+; AVX2-NEXT:    vpinsrd $3, (%rax), %xmm5, %xmm5
+; AVX2-NEXT:    vinserti128 $1, %xmm5, %ymm2, %ymm2
+; AVX2-NEXT:  .LBB4_32: # %else69
+; AVX2-NEXT:    vpxor %xmm5, %xmm5, %xmm5
+; AVX2-NEXT:    vpcmpeqd %ymm5, %ymm0, %ymm0
+; AVX2-NEXT:    vmovmskps %ymm0, %eax
+; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    # implicit-def: $ymm0
+; AVX2-NEXT:    jne .LBB4_33
+; AVX2-NEXT:  # %bb.34: # %else76
+; AVX2-NEXT:    testb $2, %al
+; AVX2-NEXT:    jne .LBB4_35
+; AVX2-NEXT:  .LBB4_36: # %else82
+; AVX2-NEXT:    testb $4, %al
+; AVX2-NEXT:    jne .LBB4_37
+; AVX2-NEXT:  .LBB4_38: # %else88
+; AVX2-NEXT:    testb $8, %al
+; AVX2-NEXT:    jne .LBB4_39
+; AVX2-NEXT:  .LBB4_40: # %else94
+; AVX2-NEXT:    testb $16, %al
+; AVX2-NEXT:    jne .LBB4_41
+; AVX2-NEXT:  .LBB4_42: # %else100
+; AVX2-NEXT:    testb $32, %al
+; AVX2-NEXT:    jne .LBB4_43
+; AVX2-NEXT:  .LBB4_44: # %else106
+; AVX2-NEXT:    testb $64, %al
+; AVX2-NEXT:    je .LBB4_46
+; AVX2-NEXT:  .LBB4_45: # %cond.load108
+; AVX2-NEXT:    vmovq %xmm4, %rcx
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; AVX2-NEXT:    vpinsrd $2, (%rcx), %xmm3, %xmm3
+; AVX2-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm0
+; AVX2-NEXT:  .LBB4_46: # %else112
+; AVX2-NEXT:    vpaddd %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    testb $-128, %al
+; AVX2-NEXT:    je .LBB4_48
+; AVX2-NEXT:  # %bb.47: # %cond.load114
+; AVX2-NEXT:    vpextrq $1, %xmm4, %rax
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm2
+; AVX2-NEXT:    vpinsrd $3, (%rax), %xmm2, %xmm2
+; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX2-NEXT:  .LBB4_48: # %else118
+; AVX2-NEXT:    vpaddd %ymm0, %ymm1, %ymm0
+; AVX2-NEXT:    retq
+; AVX2-NEXT:  .LBB4_5: # %cond.load4
+; AVX2-NEXT:    vmovq %xmm4, %rcx
+; AVX2-NEXT:    vpinsrd $2, (%rcx), %xmm1, %xmm5
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm5[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-NEXT:    testb $8, %al
+; AVX2-NEXT:    je .LBB4_8
+; AVX2-NEXT:  .LBB4_7: # %cond.load7
+; AVX2-NEXT:    vpextrq $1, %xmm4, %rcx
+; AVX2-NEXT:    vpinsrd $3, (%rcx), %xmm1, %xmm5
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm5[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-NEXT:    testb $16, %al
+; AVX2-NEXT:    je .LBB4_10
+; AVX2-NEXT:  .LBB4_9: # %cond.load10
+; AVX2-NEXT:    vmovq %xmm3, %rcx
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm5
+; AVX2-NEXT:    vpinsrd $0, (%rcx), %xmm5, %xmm5
+; AVX2-NEXT:    vinserti128 $1, %xmm5, %ymm1, %ymm1
+; AVX2-NEXT:    testb $32, %al
+; AVX2-NEXT:    je .LBB4_12
+; AVX2-NEXT:  .LBB4_11: # %cond.load13
+; AVX2-NEXT:    vpextrq $1, %xmm3, %rcx
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm3
+; AVX2-NEXT:    vpinsrd $1, (%rcx), %xmm3, %xmm3
+; AVX2-NEXT:    vinserti128 $1, %xmm3, %ymm1, %ymm1
+; AVX2-NEXT:    testb $64, %al
+; AVX2-NEXT:    je .LBB4_14
+; AVX2-NEXT:  .LBB4_13: # %cond.load16
+; AVX2-NEXT:    vmovq %xmm4, %rcx
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm3
+; AVX2-NEXT:    vpinsrd $2, (%rcx), %xmm3, %xmm3
+; AVX2-NEXT:    vinserti128 $1, %xmm3, %ymm1, %ymm1
+; AVX2-NEXT:    testb $-128, %al
+; AVX2-NEXT:    jne .LBB4_15
+; AVX2-NEXT:    jmp .LBB4_16
+; AVX2-NEXT:  .LBB4_21: # %cond.load35
+; AVX2-NEXT:    vmovq %xmm4, %rcx
+; AVX2-NEXT:    vpinsrd $2, (%rcx), %xmm2, %xmm5
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm2 = ymm5[0,1,2,3],ymm2[4,5,6,7]
+; AVX2-NEXT:    testb $8, %al
+; AVX2-NEXT:    je .LBB4_24
+; AVX2-NEXT:  .LBB4_23: # %cond.load41
+; AVX2-NEXT:    vpextrq $1, %xmm4, %rcx
+; AVX2-NEXT:    vpinsrd $3, (%rcx), %xmm2, %xmm5
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm2 = ymm5[0,1,2,3],ymm2[4,5,6,7]
+; AVX2-NEXT:    testb $16, %al
+; AVX2-NEXT:    je .LBB4_26
+; AVX2-NEXT:  .LBB4_25: # %cond.load47
+; AVX2-NEXT:    vmovq %xmm3, %rcx
+; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm5
+; AVX2-NEXT:    vpinsrd $0, (%rcx), %xmm5, %xmm5
+; AVX2-NEXT:    vinserti128 $1, %xmm5, %ymm2, %ymm2
+; AVX2-NEXT:    testb $32, %al
+; AVX2-NEXT:    je .LBB4_28
+; AVX2-NEXT:  .LBB4_27: # %cond.load53
+; AVX2-NEXT:    vpextrq $1, %xmm3, %rcx
+; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm5
+; AVX2-NEXT:    vpinsrd $1, (%rcx), %xmm5, %xmm5
+; AVX2-NEXT:    vinserti128 $1, %xmm5, %ymm2, %ymm2
+; AVX2-NEXT:    testb $64, %al
+; AVX2-NEXT:    je .LBB4_30
+; AVX2-NEXT:  .LBB4_29: # %cond.load59
+; AVX2-NEXT:    vmovq %xmm4, %rcx
+; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm5
+; AVX2-NEXT:    vpinsrd $2, (%rcx), %xmm5, %xmm5
+; AVX2-NEXT:    vinserti128 $1, %xmm5, %ymm2, %ymm2
+; AVX2-NEXT:    testb $-128, %al
+; AVX2-NEXT:    jne .LBB4_31
+; AVX2-NEXT:    jmp .LBB4_32
+; AVX2-NEXT:  .LBB4_33: # %cond.load72
+; AVX2-NEXT:    vmovq %xmm3, %rcx
+; AVX2-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX2-NEXT:    testb $2, %al
+; AVX2-NEXT:    je .LBB4_36
+; AVX2-NEXT:  .LBB4_35: # %cond.load78
+; AVX2-NEXT:    vpextrq $1, %xmm3, %rcx
+; AVX2-NEXT:    vpinsrd $1, (%rcx), %xmm0, %xmm5
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm5[0,1,2,3],ymm0[4,5,6,7]
+; AVX2-NEXT:    testb $4, %al
+; AVX2-NEXT:    je .LBB4_38
+; AVX2-NEXT:  .LBB4_37: # %cond.load84
+; AVX2-NEXT:    vmovq %xmm4, %rcx
+; AVX2-NEXT:    vpinsrd $2, (%rcx), %xmm0, %xmm5
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm5[0,1,2,3],ymm0[4,5,6,7]
+; AVX2-NEXT:    testb $8, %al
+; AVX2-NEXT:    je .LBB4_40
+; AVX2-NEXT:  .LBB4_39: # %cond.load90
+; AVX2-NEXT:    vpextrq $1, %xmm4, %rcx
+; AVX2-NEXT:    vpinsrd $3, (%rcx), %xmm0, %xmm5
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm5[0,1,2,3],ymm0[4,5,6,7]
+; AVX2-NEXT:    testb $16, %al
+; AVX2-NEXT:    je .LBB4_42
+; AVX2-NEXT:  .LBB4_41: # %cond.load96
+; AVX2-NEXT:    vmovq %xmm3, %rcx
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm5
+; AVX2-NEXT:    vpinsrd $0, (%rcx), %xmm5, %xmm5
+; AVX2-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm0
+; AVX2-NEXT:    testb $32, %al
+; AVX2-NEXT:    je .LBB4_44
+; AVX2-NEXT:  .LBB4_43: # %cond.load102
+; AVX2-NEXT:    vpextrq $1, %xmm3, %rcx
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; AVX2-NEXT:    vpinsrd $1, (%rcx), %xmm3, %xmm3
+; AVX2-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm0
+; AVX2-NEXT:    testb $64, %al
+; AVX2-NEXT:    jne .LBB4_45
+; AVX2-NEXT:    jmp .LBB4_46
+;
+; AVX512-LABEL: gather_v8i32_v8i32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512-NEXT:    vptestnmd %zmm0, %zmm0, %k0
+; AVX512-NEXT:    kshiftlw $8, %k0, %k0
+; AVX512-NEXT:    kshiftrw $8, %k0, %k1
+; AVX512-NEXT:    vpbroadcastd {{.*#+}} zmm0 = [3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3]
+; AVX512-NEXT:    kmovw %k1, %k2
+; AVX512-NEXT:    vpgatherdd c(,%zmm0,4), %zmm1 {%k2}
+; AVX512-NEXT:    vpbroadcastd {{.*#+}} zmm0 = [28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28]
+; AVX512-NEXT:    vpgatherdd c(,%zmm0), %zmm2 {%k1}
+; AVX512-NEXT:    vpaddd %ymm2, %ymm2, %ymm0
+; AVX512-NEXT:    vpaddd %ymm0, %ymm1, %ymm0
+; AVX512-NEXT:    retq
+  %1 = insertelement <8 x %struct.a*> undef, %struct.a* @c, i32 0
+  %2 = shufflevector <8 x %struct.a*> %1, <8 x %struct.a*> undef, <8 x i32> zeroinitializer
+  %3 = getelementptr %struct.a, <8 x %struct.a*> %2, <8 x i32> zeroinitializer, i32 0, i32 3
+  %4 = icmp eq <8 x i32> %trigger, zeroinitializer
+  %5 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0i32(<8 x i32*> %3, i32 4, <8 x i1> %4, <8 x i32> undef)
+  %6 = getelementptr %struct.a, <8 x %struct.a*> %2, <8 x i32> zeroinitializer, i32 3
+  %7 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0i32(<8 x i32*> %6, i32 4, <8 x i1> %4, <8 x i32> undef)
+  %8 = add <8 x i32> %5, %7
+  %9 = getelementptr %struct.a, <8 x %struct.a*> %2, i32 0, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %10 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0i32(<8 x i32*> %9, i32 4, <8 x i1> %4, <8 x i32> undef)
+  %11 = add <8 x i32> %8, %10
+  ret <8 x i32> %11
+}
+
 declare <2 x double> @llvm.masked.gather.v2f64.v2p0f64(<2 x double*>, i32, <2 x i1>, <2 x double>)
 declare <4 x double> @llvm.masked.gather.v4f64.v4p0f64(<4 x double*>, i32, <4 x i1>, <4 x double>)
 
@@ -1136,3 +1757,5 @@ declare <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*>, i32, <4 x i1
 declare <8 x float> @llvm.masked.gather.v8f32.v8p0f32(<8 x float*>, i32, <8 x i1>, <8 x float>)
 
 declare <16 x i8> @llvm.masked.gather.v16i8.v16p0i8(<16 x i8*>, i32, <16 x i1>, <16 x i8>)
+
+declare <8 x i32> @llvm.masked.gather.v8i32.v8p0i32(<8 x i32*>, i32, <8 x i1>, <8 x i32>)
diff --git a/llvm/test/CodeGen/X86/masked_gather_scatter.ll b/llvm/test/CodeGen/X86/masked_gather_scatter.ll
index 610fca094..12309044e 100644
--- a/llvm/test/CodeGen/X86/masked_gather_scatter.ll
+++ b/llvm/test/CodeGen/X86/masked_gather_scatter.ll
@@ -69,8 +69,9 @@ declare <8 x i32> @llvm.masked.gather.v8i32.v8p0i32(<8 x i32*> , i32, <8 x i1> ,
 ; SCALAR-NEXT: br label %else
 ; SCALAR: else:
 ; SCALAR-NEXT:  %res.phi.else = phi
-; SCALAR-NEXT:  %Mask1 = extractelement <16 x i1> %imask, i64 1
-; SCALAR-NEXT:  br i1 %Mask1, label %cond.load1, label %else2
+; SCALAR-NEXT:  and i16 %{{.*}}, 2
+; SCALAR-NEXT:  icmp ne i16 %{{.*}}, 0
+; SCALAR-NEXT:  br i1 %{{.*}}, label %cond.load1, label %else2
 
 define <16 x float> @test2(float* %base, <16 x i32> %ind, i16 %mask) {
 ; KNL_64-LABEL: test2:
@@ -211,16 +212,18 @@ define <16 x i32> @test4(i32* %base, <16 x i32> %ind, i16 %mask) {
 
 
 ; SCALAR-LABEL: test5
-; SCALAR:        %Mask0 = extractelement <16 x i1> %imask, i64 0
-; SCALAR-NEXT:   br i1 %Mask0, label %cond.store, label %else
+; SCALAR:        and i16 %scalar_mask, 1
+; SCALAR-NEXT:   icmp ne i16 %{{.*}}, 0
+; SCALAR-NEXT:   br i1 %{{.*}}, label %cond.store, label %else
 ; SCALAR: cond.store:
 ; SCALAR-NEXT:  %Elt0 = extractelement <16 x i32> %val, i64 0
 ; SCALAR-NEXT:  %Ptr0 = extractelement <16 x i32*> %gep.random, i64 0
 ; SCALAR-NEXT:  store i32 %Elt0, i32* %Ptr0, align 4
 ; SCALAR-NEXT:  br label %else
 ; SCALAR: else:
-; SCALAR-NEXT: %Mask1 = extractelement <16 x i1> %imask, i64 1
-; SCALAR-NEXT:  br i1 %Mask1, label %cond.store1, label %else2
+; SCALAR-NEXT:   and i16 %scalar_mask, 2
+; SCALAR-NEXT:   icmp ne i16 %{{.*}}, 0
+; SCALAR-NEXT:  br i1 %{{.*}}, label %cond.store1, label %else2
 
 define void @test5(i32* %base, <16 x i32> %ind, i16 %mask, <16 x i32>%val) {
 ; KNL_64-LABEL: test5:
@@ -455,7 +458,7 @@ define <16 x i32> @test8(<16 x i32*> %ptr.random, <16 x i32> %ind, i16 %mask) {
 %struct.RT = type { i8, [10 x [20 x i32]], i8 }
 %struct.ST = type { i32, double, %struct.RT }
 
-; Masked gather for agregate types
+; Masked gather for aggregate types
 ; Test9 and Test10 should give the same result (scalar and vector indices in GEP)
 
 
@@ -912,13 +915,12 @@ define <2 x double> @test17(double* %base, <2 x i32> %ind, <2 x i1> %mask, <2 x
 ; KNL_64-LABEL: test17:
 ; KNL_64:       # %bb.0:
 ; KNL_64-NEXT:    # kill: def $xmm2 killed $xmm2 def $zmm2
-; KNL_64-NEXT:    vpsllq $32, %xmm0, %xmm0
-; KNL_64-NEXT:    vpsraq $32, %zmm0, %zmm0
+; KNL_64-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
 ; KNL_64-NEXT:    vpsllq $63, %xmm1, %xmm1
 ; KNL_64-NEXT:    vptestmq %zmm1, %zmm1, %k0
 ; KNL_64-NEXT:    kshiftlw $14, %k0, %k0
 ; KNL_64-NEXT:    kshiftrw $14, %k0, %k1
-; KNL_64-NEXT:    vgatherqpd (%rdi,%zmm0,8), %zmm2 {%k1}
+; KNL_64-NEXT:    vgatherdpd (%rdi,%ymm0,8), %zmm2 {%k1}
 ; KNL_64-NEXT:    vmovapd %xmm2, %xmm0
 ; KNL_64-NEXT:    vzeroupper
 ; KNL_64-NEXT:    retq
@@ -926,36 +928,31 @@ define <2 x double> @test17(double* %base, <2 x i32> %ind, <2 x i1> %mask, <2 x
 ; KNL_32-LABEL: test17:
 ; KNL_32:       # %bb.0:
 ; KNL_32-NEXT:    # kill: def $xmm2 killed $xmm2 def $zmm2
-; KNL_32-NEXT:    vpsllq $32, %xmm0, %xmm0
-; KNL_32-NEXT:    vpsraq $32, %zmm0, %zmm0
+; KNL_32-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
 ; KNL_32-NEXT:    vpsllq $63, %xmm1, %xmm1
 ; KNL_32-NEXT:    vptestmq %zmm1, %zmm1, %k0
 ; KNL_32-NEXT:    kshiftlw $14, %k0, %k0
 ; KNL_32-NEXT:    kshiftrw $14, %k0, %k1
 ; KNL_32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; KNL_32-NEXT:    vgatherqpd (%eax,%zmm0,8), %zmm2 {%k1}
+; KNL_32-NEXT:    vgatherdpd (%eax,%ymm0,8), %zmm2 {%k1}
 ; KNL_32-NEXT:    vmovapd %xmm2, %xmm0
 ; KNL_32-NEXT:    vzeroupper
 ; KNL_32-NEXT:    retl
 ;
 ; SKX-LABEL: test17:
 ; SKX:       # %bb.0:
-; SKX-NEXT:    vpsllq $32, %xmm0, %xmm0
-; SKX-NEXT:    vpsraq $32, %xmm0, %xmm0
 ; SKX-NEXT:    vpsllq $63, %xmm1, %xmm1
 ; SKX-NEXT:    vpmovq2m %xmm1, %k1
-; SKX-NEXT:    vgatherqpd (%rdi,%xmm0,8), %xmm2 {%k1}
+; SKX-NEXT:    vgatherdpd (%rdi,%xmm0,8), %xmm2 {%k1}
 ; SKX-NEXT:    vmovapd %xmm2, %xmm0
 ; SKX-NEXT:    retq
 ;
 ; SKX_32-LABEL: test17:
 ; SKX_32:       # %bb.0:
-; SKX_32-NEXT:    vpsllq $32, %xmm0, %xmm0
-; SKX_32-NEXT:    vpsraq $32, %xmm0, %xmm0
 ; SKX_32-NEXT:    vpsllq $63, %xmm1, %xmm1
 ; SKX_32-NEXT:    vpmovq2m %xmm1, %k1
 ; SKX_32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; SKX_32-NEXT:    vgatherqpd (%eax,%xmm0,8), %xmm2 {%k1}
+; SKX_32-NEXT:    vgatherdpd (%eax,%xmm0,8), %xmm2 {%k1}
 ; SKX_32-NEXT:    vmovapd %xmm2, %xmm0
 ; SKX_32-NEXT:    retl
 
@@ -1077,8 +1074,8 @@ define void @test20(<2 x float>%a1, <2 x float*> %ptr, <2 x i1> %mask) {
 ;
 ; KNL_32-LABEL: test20:
 ; KNL_32:       # %bb.0:
+; KNL_32-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
 ; KNL_32-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; KNL_32-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[0,2,2,3]
 ; KNL_32-NEXT:    vpsllq $63, %xmm2, %xmm2
 ; KNL_32-NEXT:    vptestmq %zmm2, %zmm2, %k0
 ; KNL_32-NEXT:    kshiftlw $14, %k0, %k0
@@ -1096,7 +1093,6 @@ define void @test20(<2 x float>%a1, <2 x float*> %ptr, <2 x i1> %mask) {
 ;
 ; SKX_32-LABEL: test20:
 ; SKX_32:       # %bb.0:
-; SKX_32-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[0,2,2,3]
 ; SKX_32-NEXT:    vpsllq $63, %xmm2, %xmm2
 ; SKX_32-NEXT:    vpmovq2m %xmm2, %k1
 ; SKX_32-NEXT:    vscatterdps %xmm0, (,%xmm1) {%k1}
@@ -1110,9 +1106,9 @@ define void @test21(<2 x i32>%a1, <2 x i32*> %ptr, <2 x i1>%mask) {
 ; KNL_64-LABEL: test21:
 ; KNL_64:       # %bb.0:
 ; KNL_64-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; KNL_64-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
 ; KNL_64-NEXT:    vpsllq $63, %xmm2, %xmm2
 ; KNL_64-NEXT:    vptestmq %zmm2, %zmm2, %k0
-; KNL_64-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; KNL_64-NEXT:    kshiftlw $14, %k0, %k0
 ; KNL_64-NEXT:    kshiftrw $14, %k0, %k1
 ; KNL_64-NEXT:    vpscatterqd %ymm0, (,%zmm1) {%k1}
@@ -1121,10 +1117,10 @@ define void @test21(<2 x i32>%a1, <2 x i32*> %ptr, <2 x i1>%mask) {
 ;
 ; KNL_32-LABEL: test21:
 ; KNL_32:       # %bb.0:
+; KNL_32-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; KNL_32-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; KNL_32-NEXT:    vpsllq $63, %xmm2, %xmm2
 ; KNL_32-NEXT:    vptestmq %zmm2, %zmm2, %k0
-; KNL_32-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; KNL_32-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
 ; KNL_32-NEXT:    kshiftlw $14, %k0, %k0
 ; KNL_32-NEXT:    kshiftrw $14, %k0, %k1
 ; KNL_32-NEXT:    vpscatterdd %zmm0, (,%zmm1) {%k1}
@@ -1135,7 +1131,6 @@ define void @test21(<2 x i32>%a1, <2 x i32*> %ptr, <2 x i1>%mask) {
 ; SKX:       # %bb.0:
 ; SKX-NEXT:    vpsllq $63, %xmm2, %xmm2
 ; SKX-NEXT:    vpmovq2m %xmm2, %k1
-; SKX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; SKX-NEXT:    vpscatterqd %xmm0, (,%xmm1) {%k1}
 ; SKX-NEXT:    retq
 ;
@@ -1143,8 +1138,6 @@ define void @test21(<2 x i32>%a1, <2 x i32*> %ptr, <2 x i1>%mask) {
 ; SKX_32:       # %bb.0:
 ; SKX_32-NEXT:    vpsllq $63, %xmm2, %xmm2
 ; SKX_32-NEXT:    vpmovq2m %xmm2, %k1
-; SKX_32-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; SKX_32-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
 ; SKX_32-NEXT:    vpscatterdd %xmm0, (,%xmm1) {%k1}
 ; SKX_32-NEXT:    retl
   call void @llvm.masked.scatter.v2i32.v2p0i32(<2 x i32> %a1, <2 x i32*> %ptr, i32 4, <2 x i1> %mask)
@@ -1158,7 +1151,7 @@ define <2 x float> @test22(float* %base, <2 x i32> %ind, <2 x i1> %mask, <2 x fl
 ; KNL_64-LABEL: test22:
 ; KNL_64:       # %bb.0:
 ; KNL_64-NEXT:    # kill: def $xmm2 killed $xmm2 def $zmm2
-; KNL_64-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; KNL_64-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; KNL_64-NEXT:    vpsllq $63, %xmm1, %xmm1
 ; KNL_64-NEXT:    vptestmq %zmm1, %zmm1, %k0
 ; KNL_64-NEXT:    kshiftlw $14, %k0, %k0
@@ -1171,7 +1164,7 @@ define <2 x float> @test22(float* %base, <2 x i32> %ind, <2 x i1> %mask, <2 x fl
 ; KNL_32-LABEL: test22:
 ; KNL_32:       # %bb.0:
 ; KNL_32-NEXT:    # kill: def $xmm2 killed $xmm2 def $zmm2
-; KNL_32-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; KNL_32-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; KNL_32-NEXT:    vpsllq $63, %xmm1, %xmm1
 ; KNL_32-NEXT:    vptestmq %zmm1, %zmm1, %k0
 ; KNL_32-NEXT:    kshiftlw $14, %k0, %k0
@@ -1184,7 +1177,6 @@ define <2 x float> @test22(float* %base, <2 x i32> %ind, <2 x i1> %mask, <2 x fl
 ;
 ; SKX-LABEL: test22:
 ; SKX:       # %bb.0:
-; SKX-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; SKX-NEXT:    vpsllq $63, %xmm1, %xmm1
 ; SKX-NEXT:    vpmovq2m %xmm1, %k1
 ; SKX-NEXT:    vgatherdps (%rdi,%xmm0,4), %xmm2 {%k1}
@@ -1193,7 +1185,6 @@ define <2 x float> @test22(float* %base, <2 x i32> %ind, <2 x i1> %mask, <2 x fl
 ;
 ; SKX_32-LABEL: test22:
 ; SKX_32:       # %bb.0:
-; SKX_32-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; SKX_32-NEXT:    vpsllq $63, %xmm1, %xmm1
 ; SKX_32-NEXT:    vpmovq2m %xmm1, %k1
 ; SKX_32-NEXT:    movl {{[0-9]+}}(%esp), %eax
@@ -1261,28 +1252,28 @@ declare <2 x i64> @llvm.masked.gather.v2i64.v2p0i64(<2 x i64*>, i32, <2 x i1>, <
 define <2 x i32> @test23(i32* %base, <2 x i32> %ind, <2 x i1> %mask, <2 x i32> %src0) {
 ; KNL_64-LABEL: test23:
 ; KNL_64:       # %bb.0:
+; KNL_64-NEXT:    # kill: def $xmm2 killed $xmm2 def $zmm2
+; KNL_64-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; KNL_64-NEXT:    vpsllq $63, %xmm1, %xmm1
 ; KNL_64-NEXT:    vptestmq %zmm1, %zmm1, %k0
-; KNL_64-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; KNL_64-NEXT:    vpshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
 ; KNL_64-NEXT:    kshiftlw $14, %k0, %k0
 ; KNL_64-NEXT:    kshiftrw $14, %k0, %k1
-; KNL_64-NEXT:    vpgatherdd (%rdi,%zmm0,4), %zmm1 {%k1}
-; KNL_64-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero
+; KNL_64-NEXT:    vpgatherdd (%rdi,%zmm0,4), %zmm2 {%k1}
+; KNL_64-NEXT:    vmovdqa %xmm2, %xmm0
 ; KNL_64-NEXT:    vzeroupper
 ; KNL_64-NEXT:    retq
 ;
 ; KNL_32-LABEL: test23:
 ; KNL_32:       # %bb.0:
+; KNL_32-NEXT:    # kill: def $xmm2 killed $xmm2 def $zmm2
+; KNL_32-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; KNL_32-NEXT:    vpsllq $63, %xmm1, %xmm1
 ; KNL_32-NEXT:    vptestmq %zmm1, %zmm1, %k0
-; KNL_32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; KNL_32-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; KNL_32-NEXT:    vpshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
 ; KNL_32-NEXT:    kshiftlw $14, %k0, %k0
 ; KNL_32-NEXT:    kshiftrw $14, %k0, %k1
-; KNL_32-NEXT:    vpgatherdd (%eax,%zmm0,4), %zmm1 {%k1}
-; KNL_32-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero
+; KNL_32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; KNL_32-NEXT:    vpgatherdd (%eax,%zmm0,4), %zmm2 {%k1}
+; KNL_32-NEXT:    vmovdqa %xmm2, %xmm0
 ; KNL_32-NEXT:    vzeroupper
 ; KNL_32-NEXT:    retl
 ;
@@ -1290,10 +1281,8 @@ define <2 x i32> @test23(i32* %base, <2 x i32> %ind, <2 x i1> %mask, <2 x i32> %
 ; SKX:       # %bb.0:
 ; SKX-NEXT:    vpsllq $63, %xmm1, %xmm1
 ; SKX-NEXT:    vpmovq2m %xmm1, %k1
-; SKX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; SKX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
-; SKX-NEXT:    vpgatherdd (%rdi,%xmm0,4), %xmm1 {%k1}
-; SKX-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero
+; SKX-NEXT:    vpgatherdd (%rdi,%xmm0,4), %xmm2 {%k1}
+; SKX-NEXT:    vmovdqa %xmm2, %xmm0
 ; SKX-NEXT:    retq
 ;
 ; SKX_32-LABEL: test23:
@@ -1301,10 +1290,8 @@ define <2 x i32> @test23(i32* %base, <2 x i32> %ind, <2 x i1> %mask, <2 x i32> %
 ; SKX_32-NEXT:    vpsllq $63, %xmm1, %xmm1
 ; SKX_32-NEXT:    vpmovq2m %xmm1, %k1
 ; SKX_32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; SKX_32-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; SKX_32-NEXT:    vpshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
-; SKX_32-NEXT:    vpgatherdd (%eax,%xmm0,4), %xmm1 {%k1}
-; SKX_32-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero
+; SKX_32-NEXT:    vpgatherdd (%eax,%xmm0,4), %xmm2 {%k1}
+; SKX_32-NEXT:    vmovdqa %xmm2, %xmm0
 ; SKX_32-NEXT:    retl
   %sext_ind = sext <2 x i32> %ind to <2 x i64>
   %gep.random = getelementptr i32, i32* %base, <2 x i64> %sext_ind
@@ -1315,28 +1302,28 @@ define <2 x i32> @test23(i32* %base, <2 x i32> %ind, <2 x i1> %mask, <2 x i32> %
 define <2 x i32> @test23b(i32* %base, <2 x i64> %ind, <2 x i1> %mask, <2 x i32> %src0) {
 ; KNL_64-LABEL: test23b:
 ; KNL_64:       # %bb.0:
+; KNL_64-NEXT:    # kill: def $xmm2 killed $xmm2 def $ymm2
 ; KNL_64-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; KNL_64-NEXT:    vpsllq $63, %xmm1, %xmm1
 ; KNL_64-NEXT:    vptestmq %zmm1, %zmm1, %k0
-; KNL_64-NEXT:    vpshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
 ; KNL_64-NEXT:    kshiftlw $14, %k0, %k0
 ; KNL_64-NEXT:    kshiftrw $14, %k0, %k1
-; KNL_64-NEXT:    vpgatherqd (%rdi,%zmm0,4), %ymm1 {%k1}
-; KNL_64-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero
+; KNL_64-NEXT:    vpgatherqd (%rdi,%zmm0,4), %ymm2 {%k1}
+; KNL_64-NEXT:    vmovdqa %xmm2, %xmm0
 ; KNL_64-NEXT:    vzeroupper
 ; KNL_64-NEXT:    retq
 ;
 ; KNL_32-LABEL: test23b:
 ; KNL_32:       # %bb.0:
+; KNL_32-NEXT:    # kill: def $xmm2 killed $xmm2 def $ymm2
 ; KNL_32-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; KNL_32-NEXT:    vpsllq $63, %xmm1, %xmm1
 ; KNL_32-NEXT:    vptestmq %zmm1, %zmm1, %k0
-; KNL_32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; KNL_32-NEXT:    vpshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
 ; KNL_32-NEXT:    kshiftlw $14, %k0, %k0
 ; KNL_32-NEXT:    kshiftrw $14, %k0, %k1
-; KNL_32-NEXT:    vpgatherqd (%eax,%zmm0,4), %ymm1 {%k1}
-; KNL_32-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero
+; KNL_32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; KNL_32-NEXT:    vpgatherqd (%eax,%zmm0,4), %ymm2 {%k1}
+; KNL_32-NEXT:    vmovdqa %xmm2, %xmm0
 ; KNL_32-NEXT:    vzeroupper
 ; KNL_32-NEXT:    retl
 ;
@@ -1344,9 +1331,8 @@ define <2 x i32> @test23b(i32* %base, <2 x i64> %ind, <2 x i1> %mask, <2 x i32>
 ; SKX:       # %bb.0:
 ; SKX-NEXT:    vpsllq $63, %xmm1, %xmm1
 ; SKX-NEXT:    vpmovq2m %xmm1, %k1
-; SKX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
-; SKX-NEXT:    vpgatherqd (%rdi,%xmm0,4), %xmm1 {%k1}
-; SKX-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero
+; SKX-NEXT:    vpgatherqd (%rdi,%xmm0,4), %xmm2 {%k1}
+; SKX-NEXT:    vmovdqa %xmm2, %xmm0
 ; SKX-NEXT:    retq
 ;
 ; SKX_32-LABEL: test23b:
@@ -1354,9 +1340,8 @@ define <2 x i32> @test23b(i32* %base, <2 x i64> %ind, <2 x i1> %mask, <2 x i32>
 ; SKX_32-NEXT:    vpsllq $63, %xmm1, %xmm1
 ; SKX_32-NEXT:    vpmovq2m %xmm1, %k1
 ; SKX_32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; SKX_32-NEXT:    vpshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
-; SKX_32-NEXT:    vpgatherqd (%eax,%xmm0,4), %xmm1 {%k1}
-; SKX_32-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero
+; SKX_32-NEXT:    vpgatherqd (%eax,%xmm0,4), %xmm2 {%k1}
+; SKX_32-NEXT:    vmovdqa %xmm2, %xmm0
 ; SKX_32-NEXT:    retl
   %gep.random = getelementptr i32, i32* %base, <2 x i64> %ind
   %res = call <2 x i32> @llvm.masked.gather.v2i32.v2p0i32(<2 x i32*> %gep.random, i32 4, <2 x i1> %mask, <2 x i32> %src0)
@@ -1366,22 +1351,22 @@ define <2 x i32> @test23b(i32* %base, <2 x i64> %ind, <2 x i1> %mask, <2 x i32>
 define <2 x i32> @test24(i32* %base, <2 x i32> %ind) {
 ; KNL_64-LABEL: test24:
 ; KNL_64:       # %bb.0:
-; KNL_64-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; KNL_64-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; KNL_64-NEXT:    movw $3, %ax
 ; KNL_64-NEXT:    kmovw %eax, %k1
 ; KNL_64-NEXT:    vpgatherdd (%rdi,%zmm0,4), %zmm1 {%k1}
-; KNL_64-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero
+; KNL_64-NEXT:    vmovdqa %xmm1, %xmm0
 ; KNL_64-NEXT:    vzeroupper
 ; KNL_64-NEXT:    retq
 ;
 ; KNL_32-LABEL: test24:
 ; KNL_32:       # %bb.0:
+; KNL_32-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; KNL_32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; KNL_32-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; KNL_32-NEXT:    movw $3, %cx
 ; KNL_32-NEXT:    kmovw %ecx, %k1
 ; KNL_32-NEXT:    vpgatherdd (%eax,%zmm0,4), %zmm1 {%k1}
-; KNL_32-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero
+; KNL_32-NEXT:    vmovdqa %xmm1, %xmm0
 ; KNL_32-NEXT:    vzeroupper
 ; KNL_32-NEXT:    retl
 ;
@@ -1389,9 +1374,8 @@ define <2 x i32> @test24(i32* %base, <2 x i32> %ind) {
 ; SKX:       # %bb.0:
 ; SKX-NEXT:    movb $3, %al
 ; SKX-NEXT:    kmovw %eax, %k1
-; SKX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; SKX-NEXT:    vpgatherdd (%rdi,%xmm0,4), %xmm1 {%k1}
-; SKX-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero
+; SKX-NEXT:    vmovdqa %xmm1, %xmm0
 ; SKX-NEXT:    retq
 ;
 ; SKX_32-LABEL: test24:
@@ -1399,9 +1383,8 @@ define <2 x i32> @test24(i32* %base, <2 x i32> %ind) {
 ; SKX_32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; SKX_32-NEXT:    movb $3, %cl
 ; SKX_32-NEXT:    kmovw %ecx, %k1
-; SKX_32-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; SKX_32-NEXT:    vpgatherdd (%eax,%xmm0,4), %xmm1 {%k1}
-; SKX_32-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero
+; SKX_32-NEXT:    vmovdqa %xmm1, %xmm0
 ; SKX_32-NEXT:    retl
   %sext_ind = sext <2 x i32> %ind to <2 x i64>
   %gep.random = getelementptr i32, i32* %base, <2 x i64> %sext_ind
@@ -1413,13 +1396,12 @@ define <2 x i64> @test25(i64* %base, <2 x i32> %ind, <2 x i1> %mask, <2 x i64> %
 ; KNL_64-LABEL: test25:
 ; KNL_64:       # %bb.0:
 ; KNL_64-NEXT:    # kill: def $xmm2 killed $xmm2 def $zmm2
-; KNL_64-NEXT:    vpsllq $32, %xmm0, %xmm0
-; KNL_64-NEXT:    vpsraq $32, %zmm0, %zmm0
+; KNL_64-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
 ; KNL_64-NEXT:    vpsllq $63, %xmm1, %xmm1
 ; KNL_64-NEXT:    vptestmq %zmm1, %zmm1, %k0
 ; KNL_64-NEXT:    kshiftlw $14, %k0, %k0
 ; KNL_64-NEXT:    kshiftrw $14, %k0, %k1
-; KNL_64-NEXT:    vpgatherqq (%rdi,%zmm0,8), %zmm2 {%k1}
+; KNL_64-NEXT:    vpgatherdq (%rdi,%ymm0,8), %zmm2 {%k1}
 ; KNL_64-NEXT:    vmovdqa %xmm2, %xmm0
 ; KNL_64-NEXT:    vzeroupper
 ; KNL_64-NEXT:    retq
@@ -1427,36 +1409,31 @@ define <2 x i64> @test25(i64* %base, <2 x i32> %ind, <2 x i1> %mask, <2 x i64> %
 ; KNL_32-LABEL: test25:
 ; KNL_32:       # %bb.0:
 ; KNL_32-NEXT:    # kill: def $xmm2 killed $xmm2 def $zmm2
-; KNL_32-NEXT:    vpsllq $32, %xmm0, %xmm0
-; KNL_32-NEXT:    vpsraq $32, %zmm0, %zmm0
+; KNL_32-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
 ; KNL_32-NEXT:    vpsllq $63, %xmm1, %xmm1
 ; KNL_32-NEXT:    vptestmq %zmm1, %zmm1, %k0
 ; KNL_32-NEXT:    kshiftlw $14, %k0, %k0
 ; KNL_32-NEXT:    kshiftrw $14, %k0, %k1
 ; KNL_32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; KNL_32-NEXT:    vpgatherqq (%eax,%zmm0,8), %zmm2 {%k1}
+; KNL_32-NEXT:    vpgatherdq (%eax,%ymm0,8), %zmm2 {%k1}
 ; KNL_32-NEXT:    vmovdqa %xmm2, %xmm0
 ; KNL_32-NEXT:    vzeroupper
 ; KNL_32-NEXT:    retl
 ;
 ; SKX-LABEL: test25:
 ; SKX:       # %bb.0:
-; SKX-NEXT:    vpsllq $32, %xmm0, %xmm0
-; SKX-NEXT:    vpsraq $32, %xmm0, %xmm0
 ; SKX-NEXT:    vpsllq $63, %xmm1, %xmm1
 ; SKX-NEXT:    vpmovq2m %xmm1, %k1
-; SKX-NEXT:    vpgatherqq (%rdi,%xmm0,8), %xmm2 {%k1}
+; SKX-NEXT:    vpgatherdq (%rdi,%xmm0,8), %xmm2 {%k1}
 ; SKX-NEXT:    vmovdqa %xmm2, %xmm0
 ; SKX-NEXT:    retq
 ;
 ; SKX_32-LABEL: test25:
 ; SKX_32:       # %bb.0:
-; SKX_32-NEXT:    vpsllq $32, %xmm0, %xmm0
-; SKX_32-NEXT:    vpsraq $32, %xmm0, %xmm0
 ; SKX_32-NEXT:    vpsllq $63, %xmm1, %xmm1
 ; SKX_32-NEXT:    vpmovq2m %xmm1, %k1
 ; SKX_32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; SKX_32-NEXT:    vpgatherqq (%eax,%xmm0,8), %xmm2 {%k1}
+; SKX_32-NEXT:    vpgatherdq (%eax,%xmm0,8), %xmm2 {%k1}
 ; SKX_32-NEXT:    vmovdqa %xmm2, %xmm0
 ; SKX_32-NEXT:    retl
   %sext_ind = sext <2 x i32> %ind to <2 x i64>
@@ -1469,11 +1446,10 @@ define <2 x i64> @test26(i64* %base, <2 x i32> %ind, <2 x i64> %src0) {
 ; KNL_64-LABEL: test26:
 ; KNL_64:       # %bb.0:
 ; KNL_64-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
-; KNL_64-NEXT:    vpsllq $32, %xmm0, %xmm0
-; KNL_64-NEXT:    vpsraq $32, %zmm0, %zmm0
+; KNL_64-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
 ; KNL_64-NEXT:    movb $3, %al
 ; KNL_64-NEXT:    kmovw %eax, %k1
-; KNL_64-NEXT:    vpgatherqq (%rdi,%zmm0,8), %zmm1 {%k1}
+; KNL_64-NEXT:    vpgatherdq (%rdi,%ymm0,8), %zmm1 {%k1}
 ; KNL_64-NEXT:    vmovdqa %xmm1, %xmm0
 ; KNL_64-NEXT:    vzeroupper
 ; KNL_64-NEXT:    retq
@@ -1481,32 +1457,27 @@ define <2 x i64> @test26(i64* %base, <2 x i32> %ind, <2 x i64> %src0) {
 ; KNL_32-LABEL: test26:
 ; KNL_32:       # %bb.0:
 ; KNL_32-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
-; KNL_32-NEXT:    vpsllq $32, %xmm0, %xmm0
-; KNL_32-NEXT:    vpsraq $32, %zmm0, %zmm0
+; KNL_32-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
 ; KNL_32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; KNL_32-NEXT:    movb $3, %cl
 ; KNL_32-NEXT:    kmovw %ecx, %k1
-; KNL_32-NEXT:    vpgatherqq (%eax,%zmm0,8), %zmm1 {%k1}
+; KNL_32-NEXT:    vpgatherdq (%eax,%ymm0,8), %zmm1 {%k1}
 ; KNL_32-NEXT:    vmovdqa %xmm1, %xmm0
 ; KNL_32-NEXT:    vzeroupper
 ; KNL_32-NEXT:    retl
 ;
 ; SKX-LABEL: test26:
 ; SKX:       # %bb.0:
-; SKX-NEXT:    vpsllq $32, %xmm0, %xmm0
-; SKX-NEXT:    vpsraq $32, %xmm0, %xmm0
 ; SKX-NEXT:    kxnorw %k0, %k0, %k1
-; SKX-NEXT:    vpgatherqq (%rdi,%xmm0,8), %xmm1 {%k1}
+; SKX-NEXT:    vpgatherdq (%rdi,%xmm0,8), %xmm1 {%k1}
 ; SKX-NEXT:    vmovdqa %xmm1, %xmm0
 ; SKX-NEXT:    retq
 ;
 ; SKX_32-LABEL: test26:
 ; SKX_32:       # %bb.0:
-; SKX_32-NEXT:    vpsllq $32, %xmm0, %xmm0
-; SKX_32-NEXT:    vpsraq $32, %xmm0, %xmm0
 ; SKX_32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; SKX_32-NEXT:    kxnorw %k0, %k0, %k1
-; SKX_32-NEXT:    vpgatherqq (%eax,%xmm0,8), %xmm1 {%k1}
+; SKX_32-NEXT:    vpgatherdq (%eax,%xmm0,8), %xmm1 {%k1}
 ; SKX_32-NEXT:    vmovdqa %xmm1, %xmm0
 ; SKX_32-NEXT:    retl
   %sext_ind = sext <2 x i32> %ind to <2 x i64>
@@ -1519,40 +1490,40 @@ define <2 x i64> @test26(i64* %base, <2 x i32> %ind, <2 x i64> %src0) {
 define <2 x float> @test27(float* %base, <2 x i32> %ind) {
 ; KNL_64-LABEL: test27:
 ; KNL_64:       # %bb.0:
-; KNL_64-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[0,2,2,3]
+; KNL_64-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; KNL_64-NEXT:    movw $3, %ax
 ; KNL_64-NEXT:    kmovw %eax, %k1
-; KNL_64-NEXT:    vgatherdps (%rdi,%zmm1,4), %zmm0 {%k1}
-; KNL_64-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; KNL_64-NEXT:    vgatherdps (%rdi,%zmm0,4), %zmm1 {%k1}
+; KNL_64-NEXT:    vmovaps %xmm1, %xmm0
 ; KNL_64-NEXT:    vzeroupper
 ; KNL_64-NEXT:    retq
 ;
 ; KNL_32-LABEL: test27:
 ; KNL_32:       # %bb.0:
-; KNL_32-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[0,2,2,3]
+; KNL_32-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; KNL_32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; KNL_32-NEXT:    movw $3, %cx
 ; KNL_32-NEXT:    kmovw %ecx, %k1
-; KNL_32-NEXT:    vgatherdps (%eax,%zmm1,4), %zmm0 {%k1}
-; KNL_32-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; KNL_32-NEXT:    vgatherdps (%eax,%zmm0,4), %zmm1 {%k1}
+; KNL_32-NEXT:    vmovaps %xmm1, %xmm0
 ; KNL_32-NEXT:    vzeroupper
 ; KNL_32-NEXT:    retl
 ;
 ; SKX-LABEL: test27:
 ; SKX:       # %bb.0:
-; SKX-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[0,2,2,3]
 ; SKX-NEXT:    movb $3, %al
 ; SKX-NEXT:    kmovw %eax, %k1
-; SKX-NEXT:    vgatherdps (%rdi,%xmm1,4), %xmm0 {%k1}
+; SKX-NEXT:    vgatherdps (%rdi,%xmm0,4), %xmm1 {%k1}
+; SKX-NEXT:    vmovaps %xmm1, %xmm0
 ; SKX-NEXT:    retq
 ;
 ; SKX_32-LABEL: test27:
 ; SKX_32:       # %bb.0:
-; SKX_32-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[0,2,2,3]
 ; SKX_32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; SKX_32-NEXT:    movb $3, %cl
 ; SKX_32-NEXT:    kmovw %ecx, %k1
-; SKX_32-NEXT:    vgatherdps (%eax,%xmm1,4), %xmm0 {%k1}
+; SKX_32-NEXT:    vgatherdps (%eax,%xmm0,4), %xmm1 {%k1}
+; SKX_32-NEXT:    vmovaps %xmm1, %xmm0
 ; SKX_32-NEXT:    retl
   %sext_ind = sext <2 x i32> %ind to <2 x i64>
   %gep.random = getelementptr float, float* %base, <2 x i64> %sext_ind
@@ -1565,7 +1536,7 @@ define void @test28(<2 x i32>%a1, <2 x i32*> %ptr) {
 ; KNL_64-LABEL: test28:
 ; KNL_64:       # %bb.0:
 ; KNL_64-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
-; KNL_64-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; KNL_64-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
 ; KNL_64-NEXT:    movb $3, %al
 ; KNL_64-NEXT:    kmovw %eax, %k1
 ; KNL_64-NEXT:    vpscatterqd %ymm0, (,%zmm1) {%k1}
@@ -1574,8 +1545,8 @@ define void @test28(<2 x i32>%a1, <2 x i32*> %ptr) {
 ;
 ; KNL_32-LABEL: test28:
 ; KNL_32:       # %bb.0:
-; KNL_32-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; KNL_32-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; KNL_32-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; KNL_32-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; KNL_32-NEXT:    movw $3, %ax
 ; KNL_32-NEXT:    kmovw %eax, %k1
 ; KNL_32-NEXT:    vpscatterdd %zmm0, (,%zmm1) {%k1}
@@ -1584,7 +1555,6 @@ define void @test28(<2 x i32>%a1, <2 x i32*> %ptr) {
 ;
 ; SKX-LABEL: test28:
 ; SKX:       # %bb.0:
-; SKX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; SKX-NEXT:    kxnorw %k0, %k0, %k1
 ; SKX-NEXT:    vpscatterqd %xmm0, (,%xmm1) {%k1}
 ; SKX-NEXT:    retq
@@ -1593,8 +1563,6 @@ define void @test28(<2 x i32>%a1, <2 x i32*> %ptr) {
 ; SKX_32:       # %bb.0:
 ; SKX_32-NEXT:    movb $3, %al
 ; SKX_32-NEXT:    kmovw %eax, %k1
-; SKX_32-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; SKX_32-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
 ; SKX_32-NEXT:    vpscatterdd %xmm0, (,%xmm1) {%k1}
 ; SKX_32-NEXT:    retl
   call void @llvm.masked.scatter.v2i32.v2p0i32(<2 x i32> %a1, <2 x i32*> %ptr, i32 4, <2 x i1> <i1 true, i1 true>)
@@ -1658,148 +1626,177 @@ declare <3 x i32> @llvm.masked.gather.v3i32.v3p0i32(<3 x i32*>, i32, <3 x i1>, <
 define <3 x i32> @test30(<3 x i32*> %base, <3 x i32> %ind, <3 x i1> %mask, <3 x i32> %src0) {
 ; KNL_64-LABEL: test30:
 ; KNL_64:       # %bb.0:
-; KNL_64-NEXT:    vpslld $31, %xmm2, %xmm2
-; KNL_64-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; KNL_64-NEXT:    kmovw %k0, %eax
+; KNL_64-NEXT:    andb $1, %dil
+; KNL_64-NEXT:    andb $1, %sil
+; KNL_64-NEXT:    addb %sil, %sil
+; KNL_64-NEXT:    orb %dil, %sil
+; KNL_64-NEXT:    andb $1, %dl
+; KNL_64-NEXT:    shlb $2, %dl
+; KNL_64-NEXT:    orb %sil, %dl
 ; KNL_64-NEXT:    vpmovsxdq %xmm1, %ymm1
 ; KNL_64-NEXT:    vpsllq $2, %ymm1, %ymm1
 ; KNL_64-NEXT:    vpaddq %ymm1, %ymm0, %ymm0
-; KNL_64-NEXT:    testb $1, %al
-; KNL_64-NEXT:    je .LBB31_2
-; KNL_64-NEXT:  # %bb.1: # %cond.load
+; KNL_64-NEXT:    testb $1, %dl
+; KNL_64-NEXT:    jne .LBB31_1
+; KNL_64-NEXT:  # %bb.2: # %else
+; KNL_64-NEXT:    testb $2, %dl
+; KNL_64-NEXT:    jne .LBB31_3
+; KNL_64-NEXT:  .LBB31_4: # %else2
+; KNL_64-NEXT:    testb $4, %dl
+; KNL_64-NEXT:    jne .LBB31_5
+; KNL_64-NEXT:  .LBB31_6: # %else5
+; KNL_64-NEXT:    vmovdqa %xmm2, %xmm0
+; KNL_64-NEXT:    vzeroupper
+; KNL_64-NEXT:    retq
+; KNL_64-NEXT:  .LBB31_1: # %cond.load
 ; KNL_64-NEXT:    vmovq %xmm0, %rax
-; KNL_64-NEXT:    vpinsrd $0, (%rax), %xmm3, %xmm3
-; KNL_64-NEXT:  .LBB31_2: # %else
-; KNL_64-NEXT:    kshiftrw $1, %k0, %k1
-; KNL_64-NEXT:    kmovw %k1, %eax
-; KNL_64-NEXT:    testb $1, %al
+; KNL_64-NEXT:    vpinsrd $0, (%rax), %xmm2, %xmm2
+; KNL_64-NEXT:    testb $2, %dl
 ; KNL_64-NEXT:    je .LBB31_4
-; KNL_64-NEXT:  # %bb.3: # %cond.load1
+; KNL_64-NEXT:  .LBB31_3: # %cond.load1
 ; KNL_64-NEXT:    vpextrq $1, %xmm0, %rax
-; KNL_64-NEXT:    vpinsrd $1, (%rax), %xmm3, %xmm3
-; KNL_64-NEXT:  .LBB31_4: # %else2
-; KNL_64-NEXT:    kshiftrw $2, %k0, %k0
-; KNL_64-NEXT:    kmovw %k0, %eax
-; KNL_64-NEXT:    testb $1, %al
+; KNL_64-NEXT:    vpinsrd $1, (%rax), %xmm2, %xmm2
+; KNL_64-NEXT:    testb $4, %dl
 ; KNL_64-NEXT:    je .LBB31_6
-; KNL_64-NEXT:  # %bb.5: # %cond.load4
+; KNL_64-NEXT:  .LBB31_5: # %cond.load4
 ; KNL_64-NEXT:    vextracti128 $1, %ymm0, %xmm0
 ; KNL_64-NEXT:    vmovq %xmm0, %rax
-; KNL_64-NEXT:    vpinsrd $2, (%rax), %xmm3, %xmm3
-; KNL_64-NEXT:  .LBB31_6: # %else5
-; KNL_64-NEXT:    vmovdqa %xmm3, %xmm0
+; KNL_64-NEXT:    vpinsrd $2, (%rax), %xmm2, %xmm2
+; KNL_64-NEXT:    vmovdqa %xmm2, %xmm0
 ; KNL_64-NEXT:    vzeroupper
 ; KNL_64-NEXT:    retq
 ;
 ; KNL_32-LABEL: test30:
 ; KNL_32:       # %bb.0:
-; KNL_32-NEXT:    subl $12, %esp
-; KNL_32-NEXT:    .cfi_def_cfa_offset 16
-; KNL_32-NEXT:    vmovdqa %xmm0, %xmm3
-; KNL_32-NEXT:    vpslld $31, %xmm2, %xmm0
-; KNL_32-NEXT:    vptestmd %zmm0, %zmm0, %k0
-; KNL_32-NEXT:    kmovw %k0, %eax
-; KNL_32-NEXT:    vmovdqa {{[0-9]+}}(%esp), %xmm0
+; KNL_32-NEXT:    pushl %eax
+; KNL_32-NEXT:    .cfi_def_cfa_offset 8
+; KNL_32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_32-NEXT:    andb $1, %al
+; KNL_32-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; KNL_32-NEXT:    andb $1, %cl
+; KNL_32-NEXT:    addb %cl, %cl
+; KNL_32-NEXT:    orb %al, %cl
+; KNL_32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_32-NEXT:    andb $1, %al
+; KNL_32-NEXT:    shlb $2, %al
+; KNL_32-NEXT:    orb %cl, %al
 ; KNL_32-NEXT:    vpslld $2, %xmm1, %xmm1
-; KNL_32-NEXT:    vpaddd %xmm1, %xmm3, %xmm1
-; KNL_32-NEXT:    testb $1, %al
-; KNL_32-NEXT:    je .LBB31_2
-; KNL_32-NEXT:  # %bb.1: # %cond.load
-; KNL_32-NEXT:    vmovd %xmm1, %eax
-; KNL_32-NEXT:    vpinsrd $0, (%eax), %xmm0, %xmm0
-; KNL_32-NEXT:  .LBB31_2: # %else
-; KNL_32-NEXT:    kshiftrw $1, %k0, %k1
-; KNL_32-NEXT:    kmovw %k1, %eax
+; KNL_32-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; KNL_32-NEXT:    testb $1, %al
-; KNL_32-NEXT:    je .LBB31_4
-; KNL_32-NEXT:  # %bb.3: # %cond.load1
-; KNL_32-NEXT:    vpextrd $1, %xmm1, %eax
-; KNL_32-NEXT:    vpinsrd $1, (%eax), %xmm0, %xmm0
+; KNL_32-NEXT:    jne .LBB31_1
+; KNL_32-NEXT:  # %bb.2: # %else
+; KNL_32-NEXT:    testb $2, %al
+; KNL_32-NEXT:    jne .LBB31_3
 ; KNL_32-NEXT:  .LBB31_4: # %else2
-; KNL_32-NEXT:    kshiftrw $2, %k0, %k0
-; KNL_32-NEXT:    kmovw %k0, %eax
-; KNL_32-NEXT:    testb $1, %al
+; KNL_32-NEXT:    testb $4, %al
 ; KNL_32-NEXT:    je .LBB31_6
-; KNL_32-NEXT:  # %bb.5: # %cond.load4
-; KNL_32-NEXT:    vpextrd $2, %xmm1, %eax
-; KNL_32-NEXT:    vpinsrd $2, (%eax), %xmm0, %xmm0
+; KNL_32-NEXT:  .LBB31_5: # %cond.load4
+; KNL_32-NEXT:    vpextrd $2, %xmm0, %eax
+; KNL_32-NEXT:    vpinsrd $2, (%eax), %xmm2, %xmm2
 ; KNL_32-NEXT:  .LBB31_6: # %else5
-; KNL_32-NEXT:    addl $12, %esp
+; KNL_32-NEXT:    vmovdqa %xmm2, %xmm0
+; KNL_32-NEXT:    popl %eax
 ; KNL_32-NEXT:    .cfi_def_cfa_offset 4
-; KNL_32-NEXT:    vzeroupper
 ; KNL_32-NEXT:    retl
+; KNL_32-NEXT:  .LBB31_1: # %cond.load
+; KNL_32-NEXT:    .cfi_def_cfa_offset 8
+; KNL_32-NEXT:    vmovd %xmm0, %ecx
+; KNL_32-NEXT:    vpinsrd $0, (%ecx), %xmm2, %xmm2
+; KNL_32-NEXT:    testb $2, %al
+; KNL_32-NEXT:    je .LBB31_4
+; KNL_32-NEXT:  .LBB31_3: # %cond.load1
+; KNL_32-NEXT:    vpextrd $1, %xmm0, %ecx
+; KNL_32-NEXT:    vpinsrd $1, (%ecx), %xmm2, %xmm2
+; KNL_32-NEXT:    testb $4, %al
+; KNL_32-NEXT:    jne .LBB31_5
+; KNL_32-NEXT:    jmp .LBB31_6
 ;
 ; SKX-LABEL: test30:
 ; SKX:       # %bb.0:
-; SKX-NEXT:    vpslld $31, %xmm2, %xmm2
-; SKX-NEXT:    vpmovd2m %xmm2, %k0
-; SKX-NEXT:    kmovw %k0, %eax
+; SKX-NEXT:    andb $1, %dil
+; SKX-NEXT:    andb $1, %sil
+; SKX-NEXT:    addb %sil, %sil
+; SKX-NEXT:    orb %dil, %sil
+; SKX-NEXT:    andb $1, %dl
+; SKX-NEXT:    shlb $2, %dl
+; SKX-NEXT:    orb %sil, %dl
 ; SKX-NEXT:    vpmovsxdq %xmm1, %ymm1
 ; SKX-NEXT:    vpsllq $2, %ymm1, %ymm1
 ; SKX-NEXT:    vpaddq %ymm1, %ymm0, %ymm0
-; SKX-NEXT:    testb $1, %al
-; SKX-NEXT:    je .LBB31_2
-; SKX-NEXT:  # %bb.1: # %cond.load
+; SKX-NEXT:    testb $1, %dl
+; SKX-NEXT:    jne .LBB31_1
+; SKX-NEXT:  # %bb.2: # %else
+; SKX-NEXT:    testb $2, %dl
+; SKX-NEXT:    jne .LBB31_3
+; SKX-NEXT:  .LBB31_4: # %else2
+; SKX-NEXT:    testb $4, %dl
+; SKX-NEXT:    jne .LBB31_5
+; SKX-NEXT:  .LBB31_6: # %else5
+; SKX-NEXT:    vmovdqa %xmm2, %xmm0
+; SKX-NEXT:    vzeroupper
+; SKX-NEXT:    retq
+; SKX-NEXT:  .LBB31_1: # %cond.load
 ; SKX-NEXT:    vmovq %xmm0, %rax
-; SKX-NEXT:    vpinsrd $0, (%rax), %xmm3, %xmm3
-; SKX-NEXT:  .LBB31_2: # %else
-; SKX-NEXT:    kshiftrb $1, %k0, %k1
-; SKX-NEXT:    kmovw %k1, %eax
-; SKX-NEXT:    testb $1, %al
+; SKX-NEXT:    vpinsrd $0, (%rax), %xmm2, %xmm2
+; SKX-NEXT:    testb $2, %dl
 ; SKX-NEXT:    je .LBB31_4
-; SKX-NEXT:  # %bb.3: # %cond.load1
+; SKX-NEXT:  .LBB31_3: # %cond.load1
 ; SKX-NEXT:    vpextrq $1, %xmm0, %rax
-; SKX-NEXT:    vpinsrd $1, (%rax), %xmm3, %xmm3
-; SKX-NEXT:  .LBB31_4: # %else2
-; SKX-NEXT:    kshiftrb $2, %k0, %k0
-; SKX-NEXT:    kmovw %k0, %eax
-; SKX-NEXT:    testb $1, %al
+; SKX-NEXT:    vpinsrd $1, (%rax), %xmm2, %xmm2
+; SKX-NEXT:    testb $4, %dl
 ; SKX-NEXT:    je .LBB31_6
-; SKX-NEXT:  # %bb.5: # %cond.load4
+; SKX-NEXT:  .LBB31_5: # %cond.load4
 ; SKX-NEXT:    vextracti128 $1, %ymm0, %xmm0
 ; SKX-NEXT:    vmovq %xmm0, %rax
-; SKX-NEXT:    vpinsrd $2, (%rax), %xmm3, %xmm3
-; SKX-NEXT:  .LBB31_6: # %else5
-; SKX-NEXT:    vmovdqa %xmm3, %xmm0
+; SKX-NEXT:    vpinsrd $2, (%rax), %xmm2, %xmm2
+; SKX-NEXT:    vmovdqa %xmm2, %xmm0
 ; SKX-NEXT:    vzeroupper
 ; SKX-NEXT:    retq
 ;
 ; SKX_32-LABEL: test30:
 ; SKX_32:       # %bb.0:
-; SKX_32-NEXT:    subl $12, %esp
-; SKX_32-NEXT:    .cfi_def_cfa_offset 16
-; SKX_32-NEXT:    vmovdqa %xmm0, %xmm3
-; SKX_32-NEXT:    vpslld $31, %xmm2, %xmm0
-; SKX_32-NEXT:    vpmovd2m %xmm0, %k0
-; SKX_32-NEXT:    kmovw %k0, %eax
-; SKX_32-NEXT:    vmovdqa {{[0-9]+}}(%esp), %xmm0
+; SKX_32-NEXT:    pushl %eax
+; SKX_32-NEXT:    .cfi_def_cfa_offset 8
+; SKX_32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; SKX_32-NEXT:    andb $1, %al
+; SKX_32-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; SKX_32-NEXT:    andb $1, %cl
+; SKX_32-NEXT:    addb %cl, %cl
+; SKX_32-NEXT:    orb %al, %cl
+; SKX_32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; SKX_32-NEXT:    andb $1, %al
+; SKX_32-NEXT:    shlb $2, %al
+; SKX_32-NEXT:    orb %cl, %al
 ; SKX_32-NEXT:    vpslld $2, %xmm1, %xmm1
-; SKX_32-NEXT:    vpaddd %xmm1, %xmm3, %xmm1
+; SKX_32-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; SKX_32-NEXT:    testb $1, %al
-; SKX_32-NEXT:    je .LBB31_2
-; SKX_32-NEXT:  # %bb.1: # %cond.load
-; SKX_32-NEXT:    vmovd %xmm1, %eax
-; SKX_32-NEXT:    vpinsrd $0, (%eax), %xmm0, %xmm0
-; SKX_32-NEXT:  .LBB31_2: # %else
-; SKX_32-NEXT:    kshiftrb $1, %k0, %k1
-; SKX_32-NEXT:    kmovw %k1, %eax
-; SKX_32-NEXT:    testb $1, %al
-; SKX_32-NEXT:    je .LBB31_4
-; SKX_32-NEXT:  # %bb.3: # %cond.load1
-; SKX_32-NEXT:    vpextrd $1, %xmm1, %eax
-; SKX_32-NEXT:    vpinsrd $1, (%eax), %xmm0, %xmm0
+; SKX_32-NEXT:    jne .LBB31_1
+; SKX_32-NEXT:  # %bb.2: # %else
+; SKX_32-NEXT:    testb $2, %al
+; SKX_32-NEXT:    jne .LBB31_3
 ; SKX_32-NEXT:  .LBB31_4: # %else2
-; SKX_32-NEXT:    kshiftrb $2, %k0, %k0
-; SKX_32-NEXT:    kmovw %k0, %eax
-; SKX_32-NEXT:    testb $1, %al
+; SKX_32-NEXT:    testb $4, %al
 ; SKX_32-NEXT:    je .LBB31_6
-; SKX_32-NEXT:  # %bb.5: # %cond.load4
-; SKX_32-NEXT:    vpextrd $2, %xmm1, %eax
-; SKX_32-NEXT:    vpinsrd $2, (%eax), %xmm0, %xmm0
+; SKX_32-NEXT:  .LBB31_5: # %cond.load4
+; SKX_32-NEXT:    vpextrd $2, %xmm0, %eax
+; SKX_32-NEXT:    vpinsrd $2, (%eax), %xmm2, %xmm2
 ; SKX_32-NEXT:  .LBB31_6: # %else5
-; SKX_32-NEXT:    addl $12, %esp
+; SKX_32-NEXT:    vmovdqa %xmm2, %xmm0
+; SKX_32-NEXT:    popl %eax
 ; SKX_32-NEXT:    .cfi_def_cfa_offset 4
 ; SKX_32-NEXT:    retl
+; SKX_32-NEXT:  .LBB31_1: # %cond.load
+; SKX_32-NEXT:    .cfi_def_cfa_offset 8
+; SKX_32-NEXT:    vmovd %xmm0, %ecx
+; SKX_32-NEXT:    vpinsrd $0, (%ecx), %xmm2, %xmm2
+; SKX_32-NEXT:    testb $2, %al
+; SKX_32-NEXT:    je .LBB31_4
+; SKX_32-NEXT:  .LBB31_3: # %cond.load1
+; SKX_32-NEXT:    vpextrd $1, %xmm0, %ecx
+; SKX_32-NEXT:    vpinsrd $1, (%ecx), %xmm2, %xmm2
+; SKX_32-NEXT:    testb $4, %al
+; SKX_32-NEXT:    jne .LBB31_5
+; SKX_32-NEXT:    jmp .LBB31_6
 
   %sext_ind = sext <3 x i32> %ind to <3 x i64>
   %gep.random = getelementptr i32, <3 x i32*> %base, <3 x i64> %sext_ind
@@ -2433,45 +2430,86 @@ define <8 x i32> @test_global_array(<8 x i64> %indxs) {
   ret <8 x i32> %g
 }
 
+define <8 x i32> @test_global_array_zeroinitializer_index(<8 x i64> %indxs) {
+; KNL_64-LABEL: test_global_array_zeroinitializer_index:
+; KNL_64:       # %bb.0:
+; KNL_64-NEXT:    kxnorw %k0, %k0, %k1
+; KNL_64-NEXT:    vpgatherqd glob_array(,%zmm0,4), %ymm1 {%k1}
+; KNL_64-NEXT:    vmovdqa %ymm1, %ymm0
+; KNL_64-NEXT:    retq
+;
+; KNL_32-LABEL: test_global_array_zeroinitializer_index:
+; KNL_32:       # %bb.0:
+; KNL_32-NEXT:    kxnorw %k0, %k0, %k1
+; KNL_32-NEXT:    vpgatherqd glob_array(,%zmm0,4), %ymm1 {%k1}
+; KNL_32-NEXT:    vmovdqa %ymm1, %ymm0
+; KNL_32-NEXT:    retl
+;
+; SKX_SMALL-LABEL: test_global_array_zeroinitializer_index:
+; SKX_SMALL:       # %bb.0:
+; SKX_SMALL-NEXT:    kxnorw %k0, %k0, %k1
+; SKX_SMALL-NEXT:    vpgatherqd glob_array(,%zmm0,4), %ymm1 {%k1}
+; SKX_SMALL-NEXT:    vmovdqa %ymm1, %ymm0
+; SKX_SMALL-NEXT:    retq
+;
+; SKX_LARGE-LABEL: test_global_array_zeroinitializer_index:
+; SKX_LARGE:       # %bb.0:
+; SKX_LARGE-NEXT:    movabsq $glob_array, %rax
+; SKX_LARGE-NEXT:    kxnorw %k0, %k0, %k1
+; SKX_LARGE-NEXT:    vpgatherqd (%rax,%zmm0,4), %ymm1 {%k1}
+; SKX_LARGE-NEXT:    vmovdqa %ymm1, %ymm0
+; SKX_LARGE-NEXT:    retq
+;
+; SKX_32-LABEL: test_global_array_zeroinitializer_index:
+; SKX_32:       # %bb.0:
+; SKX_32-NEXT:    kxnorw %k0, %k0, %k1
+; SKX_32-NEXT:    vpgatherqd glob_array(,%zmm0,4), %ymm1 {%k1}
+; SKX_32-NEXT:    vmovdqa %ymm1, %ymm0
+; SKX_32-NEXT:    retl
+  %p = getelementptr inbounds [16 x i32], [16 x i32]* @glob_array, <8 x i64> zeroinitializer, <8 x i64> %indxs
+  %g = call <8 x i32> @llvm.masked.gather.v8i32.v8p0i32(<8 x i32*> %p, i32 8, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i32> undef)
+  ret <8 x i32> %g
+}
+
 define void @v1_scatter(<1 x i32>%a1, <1 x i32*> %ptr, <1 x i1> %mask) {
 ; KNL_64-LABEL: v1_scatter:
 ; KNL_64:       # %bb.0:
 ; KNL_64-NEXT:    testb $1, %dl
-; KNL_64-NEXT:    je .LBB43_2
+; KNL_64-NEXT:    je .LBB44_2
 ; KNL_64-NEXT:  # %bb.1: # %cond.store
 ; KNL_64-NEXT:    movl %edi, (%rsi)
-; KNL_64-NEXT:  .LBB43_2: # %else
+; KNL_64-NEXT:  .LBB44_2: # %else
 ; KNL_64-NEXT:    retq
 ;
 ; KNL_32-LABEL: v1_scatter:
 ; KNL_32:       # %bb.0:
 ; KNL_32-NEXT:    testb $1, {{[0-9]+}}(%esp)
-; KNL_32-NEXT:    je .LBB43_2
+; KNL_32-NEXT:    je .LBB44_2
 ; KNL_32-NEXT:  # %bb.1: # %cond.store
 ; KNL_32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; KNL_32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; KNL_32-NEXT:    movl %ecx, (%eax)
-; KNL_32-NEXT:  .LBB43_2: # %else
+; KNL_32-NEXT:  .LBB44_2: # %else
 ; KNL_32-NEXT:    retl
 ;
 ; SKX-LABEL: v1_scatter:
 ; SKX:       # %bb.0:
 ; SKX-NEXT:    testb $1, %dl
-; SKX-NEXT:    je .LBB43_2
+; SKX-NEXT:    je .LBB44_2
 ; SKX-NEXT:  # %bb.1: # %cond.store
 ; SKX-NEXT:    movl %edi, (%rsi)
-; SKX-NEXT:  .LBB43_2: # %else
+; SKX-NEXT:  .LBB44_2: # %else
 ; SKX-NEXT:    retq
 ;
 ; SKX_32-LABEL: v1_scatter:
 ; SKX_32:       # %bb.0:
 ; SKX_32-NEXT:    testb $1, {{[0-9]+}}(%esp)
-; SKX_32-NEXT:    je .LBB43_2
+; SKX_32-NEXT:    je .LBB44_2
 ; SKX_32-NEXT:  # %bb.1: # %cond.store
 ; SKX_32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; SKX_32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; SKX_32-NEXT:    movl %ecx, (%eax)
-; SKX_32-NEXT:  .LBB43_2: # %else
+; SKX_32-NEXT:  .LBB44_2: # %else
 ; SKX_32-NEXT:    retl
   call void @llvm.masked.scatter.v1i32.v1p0i32(<1 x i32> %a1, <1 x i32*> %ptr, i32 4, <1 x i1> %mask)
   ret void
@@ -2611,9 +2649,7 @@ define <16 x float> @sext_i8_index(float* %base, <16 x i8> %ind) {
 define <8 x float> @sext_v8i8_index(float* %base, <8 x i8> %ind) {
 ; KNL_64-LABEL: sext_v8i8_index:
 ; KNL_64:       # %bb.0:
-; KNL_64-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; KNL_64-NEXT:    vpslld $24, %ymm0, %ymm0
-; KNL_64-NEXT:    vpsrad $24, %ymm0, %ymm1
+; KNL_64-NEXT:    vpmovsxbd %xmm0, %ymm1
 ; KNL_64-NEXT:    movw $255, %ax
 ; KNL_64-NEXT:    kmovw %eax, %k1
 ; KNL_64-NEXT:    vgatherdps (%rdi,%zmm1,4), %zmm0 {%k1}
@@ -2622,10 +2658,8 @@ define <8 x float> @sext_v8i8_index(float* %base, <8 x i8> %ind) {
 ;
 ; KNL_32-LABEL: sext_v8i8_index:
 ; KNL_32:       # %bb.0:
-; KNL_32-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; KNL_32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; KNL_32-NEXT:    vpslld $24, %ymm0, %ymm0
-; KNL_32-NEXT:    vpsrad $24, %ymm0, %ymm1
+; KNL_32-NEXT:    vpmovsxbd %xmm0, %ymm1
 ; KNL_32-NEXT:    movw $255, %cx
 ; KNL_32-NEXT:    kmovw %ecx, %k1
 ; KNL_32-NEXT:    vgatherdps (%eax,%zmm1,4), %zmm0 {%k1}
@@ -2634,20 +2668,16 @@ define <8 x float> @sext_v8i8_index(float* %base, <8 x i8> %ind) {
 ;
 ; SKX-LABEL: sext_v8i8_index:
 ; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; SKX-NEXT:    vpmovsxbd %xmm0, %ymm1
 ; SKX-NEXT:    kxnorw %k0, %k0, %k1
-; SKX-NEXT:    vpslld $24, %ymm0, %ymm0
-; SKX-NEXT:    vpsrad $24, %ymm0, %ymm1
 ; SKX-NEXT:    vgatherdps (%rdi,%ymm1,4), %ymm0 {%k1}
 ; SKX-NEXT:    retq
 ;
 ; SKX_32-LABEL: sext_v8i8_index:
 ; SKX_32:       # %bb.0:
-; SKX_32-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; SKX_32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; SKX_32-NEXT:    vpmovsxbd %xmm0, %ymm1
 ; SKX_32-NEXT:    kxnorw %k0, %k0, %k1
-; SKX_32-NEXT:    vpslld $24, %ymm0, %ymm0
-; SKX_32-NEXT:    vpsrad $24, %ymm0, %ymm1
 ; SKX_32-NEXT:    vgatherdps (%eax,%ymm1,4), %ymm0 {%k1}
 ; SKX_32-NEXT:    retl
 
@@ -2659,32 +2689,112 @@ define <8 x float> @sext_v8i8_index(float* %base, <8 x i8> %ind) {
 }
 declare <8 x float> @llvm.masked.gather.v8f32.v8p0f32(<8 x float*>, i32, <8 x i1>, <8 x float>)
 
+; Make sure we also allow index to be zero extended from a smaller than i32 element size.
+define <16 x float> @zext_i8_index(float* %base, <16 x i8> %ind) {
+; KNL_64-LABEL: zext_i8_index:
+; KNL_64:       # %bb.0:
+; KNL_64-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; KNL_64-NEXT:    kxnorw %k0, %k0, %k1
+; KNL_64-NEXT:    vgatherdps (%rdi,%zmm1,4), %zmm0 {%k1}
+; KNL_64-NEXT:    retq
+;
+; KNL_32-LABEL: zext_i8_index:
+; KNL_32:       # %bb.0:
+; KNL_32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; KNL_32-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; KNL_32-NEXT:    kxnorw %k0, %k0, %k1
+; KNL_32-NEXT:    vgatherdps (%eax,%zmm1,4), %zmm0 {%k1}
+; KNL_32-NEXT:    retl
+;
+; SKX-LABEL: zext_i8_index:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; SKX-NEXT:    kxnorw %k0, %k0, %k1
+; SKX-NEXT:    vgatherdps (%rdi,%zmm1,4), %zmm0 {%k1}
+; SKX-NEXT:    retq
+;
+; SKX_32-LABEL: zext_i8_index:
+; SKX_32:       # %bb.0:
+; SKX_32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; SKX_32-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; SKX_32-NEXT:    kxnorw %k0, %k0, %k1
+; SKX_32-NEXT:    vgatherdps (%eax,%zmm1,4), %zmm0 {%k1}
+; SKX_32-NEXT:    retl
+
+  %zext_ind = zext <16 x i8> %ind to <16 x i64>
+  %gep.random = getelementptr float, float *%base, <16 x i64> %zext_ind
+
+  %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %gep.random, i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x float> undef)
+  ret <16 x float>%res
+}
+
+; Make sure we also allow index to be zero extended from a smaller than i32 element size.
+define <8 x float> @zext_v8i8_index(float* %base, <8 x i8> %ind) {
+; KNL_64-LABEL: zext_v8i8_index:
+; KNL_64:       # %bb.0:
+; KNL_64-NEXT:    vpmovzxbd {{.*#+}} ymm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
+; KNL_64-NEXT:    movw $255, %ax
+; KNL_64-NEXT:    kmovw %eax, %k1
+; KNL_64-NEXT:    vgatherdps (%rdi,%zmm1,4), %zmm0 {%k1}
+; KNL_64-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; KNL_64-NEXT:    retq
+;
+; KNL_32-LABEL: zext_v8i8_index:
+; KNL_32:       # %bb.0:
+; KNL_32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; KNL_32-NEXT:    vpmovzxbd {{.*#+}} ymm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
+; KNL_32-NEXT:    movw $255, %cx
+; KNL_32-NEXT:    kmovw %ecx, %k1
+; KNL_32-NEXT:    vgatherdps (%eax,%zmm1,4), %zmm0 {%k1}
+; KNL_32-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; KNL_32-NEXT:    retl
+;
+; SKX-LABEL: zext_v8i8_index:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpmovzxbd {{.*#+}} ymm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
+; SKX-NEXT:    kxnorw %k0, %k0, %k1
+; SKX-NEXT:    vgatherdps (%rdi,%ymm1,4), %ymm0 {%k1}
+; SKX-NEXT:    retq
+;
+; SKX_32-LABEL: zext_v8i8_index:
+; SKX_32:       # %bb.0:
+; SKX_32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; SKX_32-NEXT:    vpmovzxbd {{.*#+}} ymm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
+; SKX_32-NEXT:    kxnorw %k0, %k0, %k1
+; SKX_32-NEXT:    vgatherdps (%eax,%ymm1,4), %ymm0 {%k1}
+; SKX_32-NEXT:    retl
+
+  %zext_ind = zext <8 x i8> %ind to <8 x i64>
+  %gep.random = getelementptr float, float *%base, <8 x i64> %zext_ind
+
+  %res = call <8 x float> @llvm.masked.gather.v8f32.v8p0f32(<8 x float*> %gep.random, i32 4, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x float> undef)
+  ret <8 x float>%res
+}
+
 ; Index requires promotion
 define void @test_scatter_2i32_index(<2 x double> %a1, double* %base, <2 x i32> %ind, <2 x i1> %mask) {
 ; KNL_64-LABEL: test_scatter_2i32_index:
 ; KNL_64:       # %bb.0:
+; KNL_64-NEXT:    # kill: def $xmm1 killed $xmm1 def $ymm1
 ; KNL_64-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; KNL_64-NEXT:    vpsllq $32, %xmm1, %xmm1
-; KNL_64-NEXT:    vpsraq $32, %zmm1, %zmm1
 ; KNL_64-NEXT:    vpsllq $63, %xmm2, %xmm2
 ; KNL_64-NEXT:    vptestmq %zmm2, %zmm2, %k0
 ; KNL_64-NEXT:    kshiftlw $14, %k0, %k0
 ; KNL_64-NEXT:    kshiftrw $14, %k0, %k1
-; KNL_64-NEXT:    vscatterqpd %zmm0, (%rdi,%zmm1,8) {%k1}
+; KNL_64-NEXT:    vscatterdpd %zmm0, (%rdi,%ymm1,8) {%k1}
 ; KNL_64-NEXT:    vzeroupper
 ; KNL_64-NEXT:    retq
 ;
 ; KNL_32-LABEL: test_scatter_2i32_index:
 ; KNL_32:       # %bb.0:
+; KNL_32-NEXT:    # kill: def $xmm1 killed $xmm1 def $ymm1
 ; KNL_32-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; KNL_32-NEXT:    vpsllq $32, %xmm1, %xmm1
-; KNL_32-NEXT:    vpsraq $32, %zmm1, %zmm1
 ; KNL_32-NEXT:    vpsllq $63, %xmm2, %xmm2
 ; KNL_32-NEXT:    vptestmq %zmm2, %zmm2, %k0
 ; KNL_32-NEXT:    kshiftlw $14, %k0, %k0
 ; KNL_32-NEXT:    kshiftrw $14, %k0, %k1
 ; KNL_32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; KNL_32-NEXT:    vscatterqpd %zmm0, (%eax,%zmm1,8) {%k1}
+; KNL_32-NEXT:    vscatterdpd %zmm0, (%eax,%ymm1,8) {%k1}
 ; KNL_32-NEXT:    vzeroupper
 ; KNL_32-NEXT:    retl
 ;
@@ -2692,19 +2802,15 @@ define void @test_scatter_2i32_index(<2 x double> %a1, double* %base, <2 x i32>
 ; SKX:       # %bb.0:
 ; SKX-NEXT:    vpsllq $63, %xmm2, %xmm2
 ; SKX-NEXT:    vpmovq2m %xmm2, %k1
-; SKX-NEXT:    vpsllq $32, %xmm1, %xmm1
-; SKX-NEXT:    vpsraq $32, %xmm1, %xmm1
-; SKX-NEXT:    vscatterqpd %xmm0, (%rdi,%xmm1,8) {%k1}
+; SKX-NEXT:    vscatterdpd %xmm0, (%rdi,%xmm1,8) {%k1}
 ; SKX-NEXT:    retq
 ;
 ; SKX_32-LABEL: test_scatter_2i32_index:
 ; SKX_32:       # %bb.0:
 ; SKX_32-NEXT:    vpsllq $63, %xmm2, %xmm2
 ; SKX_32-NEXT:    vpmovq2m %xmm2, %k1
-; SKX_32-NEXT:    vpsllq $32, %xmm1, %xmm1
-; SKX_32-NEXT:    vpsraq $32, %xmm1, %xmm1
 ; SKX_32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; SKX_32-NEXT:    vscatterqpd %xmm0, (%eax,%xmm1,8) {%k1}
+; SKX_32-NEXT:    vscatterdpd %xmm0, (%eax,%xmm1,8) {%k1}
 ; SKX_32-NEXT:    retl
   %gep = getelementptr double, double *%base, <2 x i32> %ind
   call void @llvm.masked.scatter.v2f64.v2p0f64(<2 x double> %a1, <2 x double*> %gep, i32 4, <2 x i1> %mask)
@@ -2761,12 +2867,12 @@ define <16 x float> @zext_index(float* %base, <16 x i32> %ind) {
 define <16 x double> @test_gather_setcc_split(double* %base, <16 x i32> %ind, <16 x i32> %cmp, <16 x double> %passthru) {
 ; KNL_64-LABEL: test_gather_setcc_split:
 ; KNL_64:       # %bb.0:
-; KNL_64-NEXT:    vextractf64x4 $1, %zmm0, %ymm4
-; KNL_64-NEXT:    vextracti64x4 $1, %zmm1, %ymm5
-; KNL_64-NEXT:    vptestnmd %zmm5, %zmm5, %k1
+; KNL_64-NEXT:    vextracti64x4 $1, %zmm1, %ymm4
+; KNL_64-NEXT:    vptestnmd %zmm4, %zmm4, %k1
 ; KNL_64-NEXT:    vptestnmd %zmm1, %zmm1, %k2
 ; KNL_64-NEXT:    vgatherdpd (%rdi,%ymm0,8), %zmm2 {%k2}
-; KNL_64-NEXT:    vgatherdpd (%rdi,%ymm4,8), %zmm3 {%k1}
+; KNL_64-NEXT:    vextractf64x4 $1, %zmm0, %ymm0
+; KNL_64-NEXT:    vgatherdpd (%rdi,%ymm0,8), %zmm3 {%k1}
 ; KNL_64-NEXT:    vmovapd %zmm2, %zmm0
 ; KNL_64-NEXT:    vmovapd %zmm3, %zmm1
 ; KNL_64-NEXT:    retq
@@ -2782,12 +2888,12 @@ define <16 x double> @test_gather_setcc_split(double* %base, <16 x i32> %ind, <1
 ; KNL_32-NEXT:    subl $64, %esp
 ; KNL_32-NEXT:    vmovapd 72(%ebp), %zmm3
 ; KNL_32-NEXT:    movl 8(%ebp), %eax
-; KNL_32-NEXT:    vextractf64x4 $1, %zmm0, %ymm4
-; KNL_32-NEXT:    vextracti64x4 $1, %zmm1, %ymm5
-; KNL_32-NEXT:    vptestnmd %zmm5, %zmm5, %k1
+; KNL_32-NEXT:    vextracti64x4 $1, %zmm1, %ymm4
+; KNL_32-NEXT:    vptestnmd %zmm4, %zmm4, %k1
 ; KNL_32-NEXT:    vptestnmd %zmm1, %zmm1, %k2
 ; KNL_32-NEXT:    vgatherdpd (%eax,%ymm0,8), %zmm2 {%k2}
-; KNL_32-NEXT:    vgatherdpd (%eax,%ymm4,8), %zmm3 {%k1}
+; KNL_32-NEXT:    vextractf64x4 $1, %zmm0, %ymm0
+; KNL_32-NEXT:    vgatherdpd (%eax,%ymm0,8), %zmm3 {%k1}
 ; KNL_32-NEXT:    vmovapd %zmm2, %zmm0
 ; KNL_32-NEXT:    vmovapd %zmm3, %zmm1
 ; KNL_32-NEXT:    movl %ebp, %esp
@@ -2797,12 +2903,12 @@ define <16 x double> @test_gather_setcc_split(double* %base, <16 x i32> %ind, <1
 ;
 ; SKX-LABEL: test_gather_setcc_split:
 ; SKX:       # %bb.0:
-; SKX-NEXT:    vextractf64x4 $1, %zmm0, %ymm4
-; SKX-NEXT:    vextracti64x4 $1, %zmm1, %ymm5
-; SKX-NEXT:    vptestnmd %ymm5, %ymm5, %k1
+; SKX-NEXT:    vextracti64x4 $1, %zmm1, %ymm4
+; SKX-NEXT:    vptestnmd %ymm4, %ymm4, %k1
 ; SKX-NEXT:    vptestnmd %ymm1, %ymm1, %k2
 ; SKX-NEXT:    vgatherdpd (%rdi,%ymm0,8), %zmm2 {%k2}
-; SKX-NEXT:    vgatherdpd (%rdi,%ymm4,8), %zmm3 {%k1}
+; SKX-NEXT:    vextractf64x4 $1, %zmm0, %ymm0
+; SKX-NEXT:    vgatherdpd (%rdi,%ymm0,8), %zmm3 {%k1}
 ; SKX-NEXT:    vmovapd %zmm2, %zmm0
 ; SKX-NEXT:    vmovapd %zmm3, %zmm1
 ; SKX-NEXT:    retq
@@ -2818,12 +2924,12 @@ define <16 x double> @test_gather_setcc_split(double* %base, <16 x i32> %ind, <1
 ; SKX_32-NEXT:    subl $64, %esp
 ; SKX_32-NEXT:    vmovapd 72(%ebp), %zmm3
 ; SKX_32-NEXT:    movl 8(%ebp), %eax
-; SKX_32-NEXT:    vextractf64x4 $1, %zmm0, %ymm4
-; SKX_32-NEXT:    vextracti64x4 $1, %zmm1, %ymm5
-; SKX_32-NEXT:    vptestnmd %ymm5, %ymm5, %k1
+; SKX_32-NEXT:    vextracti64x4 $1, %zmm1, %ymm4
+; SKX_32-NEXT:    vptestnmd %ymm4, %ymm4, %k1
 ; SKX_32-NEXT:    vptestnmd %ymm1, %ymm1, %k2
 ; SKX_32-NEXT:    vgatherdpd (%eax,%ymm0,8), %zmm2 {%k2}
-; SKX_32-NEXT:    vgatherdpd (%eax,%ymm4,8), %zmm3 {%k1}
+; SKX_32-NEXT:    vextractf64x4 $1, %zmm0, %ymm0
+; SKX_32-NEXT:    vgatherdpd (%eax,%ymm0,8), %zmm3 {%k1}
 ; SKX_32-NEXT:    vmovapd %zmm2, %zmm0
 ; SKX_32-NEXT:    vmovapd %zmm3, %zmm1
 ; SKX_32-NEXT:    movl %ebp, %esp
@@ -2841,12 +2947,12 @@ define <16 x double> @test_gather_setcc_split(double* %base, <16 x i32> %ind, <1
 define void @test_scatter_setcc_split(double* %base, <16 x i32> %ind, <16 x i32> %cmp, <16 x double> %src0)  {
 ; KNL_64-LABEL: test_scatter_setcc_split:
 ; KNL_64:       # %bb.0:
-; KNL_64-NEXT:    vextractf64x4 $1, %zmm0, %ymm4
-; KNL_64-NEXT:    vextracti64x4 $1, %zmm1, %ymm5
-; KNL_64-NEXT:    vptestnmd %zmm5, %zmm5, %k1
+; KNL_64-NEXT:    vextracti64x4 $1, %zmm1, %ymm4
+; KNL_64-NEXT:    vptestnmd %zmm4, %zmm4, %k1
 ; KNL_64-NEXT:    vptestnmd %zmm1, %zmm1, %k2
 ; KNL_64-NEXT:    vscatterdpd %zmm2, (%rdi,%ymm0,8) {%k2}
-; KNL_64-NEXT:    vscatterdpd %zmm3, (%rdi,%ymm4,8) {%k1}
+; KNL_64-NEXT:    vextractf64x4 $1, %zmm0, %ymm0
+; KNL_64-NEXT:    vscatterdpd %zmm3, (%rdi,%ymm0,8) {%k1}
 ; KNL_64-NEXT:    vzeroupper
 ; KNL_64-NEXT:    retq
 ;
@@ -2861,12 +2967,12 @@ define void @test_scatter_setcc_split(double* %base, <16 x i32> %ind, <16 x i32>
 ; KNL_32-NEXT:    subl $64, %esp
 ; KNL_32-NEXT:    vmovapd 72(%ebp), %zmm3
 ; KNL_32-NEXT:    movl 8(%ebp), %eax
-; KNL_32-NEXT:    vextractf64x4 $1, %zmm0, %ymm4
-; KNL_32-NEXT:    vextracti64x4 $1, %zmm1, %ymm5
-; KNL_32-NEXT:    vptestnmd %zmm5, %zmm5, %k1
+; KNL_32-NEXT:    vextracti64x4 $1, %zmm1, %ymm4
+; KNL_32-NEXT:    vptestnmd %zmm4, %zmm4, %k1
 ; KNL_32-NEXT:    vptestnmd %zmm1, %zmm1, %k2
 ; KNL_32-NEXT:    vscatterdpd %zmm2, (%eax,%ymm0,8) {%k2}
-; KNL_32-NEXT:    vscatterdpd %zmm3, (%eax,%ymm4,8) {%k1}
+; KNL_32-NEXT:    vextractf64x4 $1, %zmm0, %ymm0
+; KNL_32-NEXT:    vscatterdpd %zmm3, (%eax,%ymm0,8) {%k1}
 ; KNL_32-NEXT:    movl %ebp, %esp
 ; KNL_32-NEXT:    popl %ebp
 ; KNL_32-NEXT:    .cfi_def_cfa %esp, 4
@@ -2875,12 +2981,12 @@ define void @test_scatter_setcc_split(double* %base, <16 x i32> %ind, <16 x i32>
 ;
 ; SKX-LABEL: test_scatter_setcc_split:
 ; SKX:       # %bb.0:
-; SKX-NEXT:    vextractf64x4 $1, %zmm0, %ymm4
-; SKX-NEXT:    vextracti64x4 $1, %zmm1, %ymm5
-; SKX-NEXT:    vptestnmd %ymm5, %ymm5, %k1
+; SKX-NEXT:    vextracti64x4 $1, %zmm1, %ymm4
+; SKX-NEXT:    vptestnmd %ymm4, %ymm4, %k1
 ; SKX-NEXT:    vptestnmd %ymm1, %ymm1, %k2
 ; SKX-NEXT:    vscatterdpd %zmm2, (%rdi,%ymm0,8) {%k2}
-; SKX-NEXT:    vscatterdpd %zmm3, (%rdi,%ymm4,8) {%k1}
+; SKX-NEXT:    vextractf64x4 $1, %zmm0, %ymm0
+; SKX-NEXT:    vscatterdpd %zmm3, (%rdi,%ymm0,8) {%k1}
 ; SKX-NEXT:    vzeroupper
 ; SKX-NEXT:    retq
 ;
@@ -2895,12 +3001,12 @@ define void @test_scatter_setcc_split(double* %base, <16 x i32> %ind, <16 x i32>
 ; SKX_32-NEXT:    subl $64, %esp
 ; SKX_32-NEXT:    vmovapd 72(%ebp), %zmm3
 ; SKX_32-NEXT:    movl 8(%ebp), %eax
-; SKX_32-NEXT:    vextractf64x4 $1, %zmm0, %ymm4
-; SKX_32-NEXT:    vextracti64x4 $1, %zmm1, %ymm5
-; SKX_32-NEXT:    vptestnmd %ymm5, %ymm5, %k1
+; SKX_32-NEXT:    vextracti64x4 $1, %zmm1, %ymm4
+; SKX_32-NEXT:    vptestnmd %ymm4, %ymm4, %k1
 ; SKX_32-NEXT:    vptestnmd %ymm1, %ymm1, %k2
 ; SKX_32-NEXT:    vscatterdpd %zmm2, (%eax,%ymm0,8) {%k2}
-; SKX_32-NEXT:    vscatterdpd %zmm3, (%eax,%ymm4,8) {%k1}
+; SKX_32-NEXT:    vextractf64x4 $1, %zmm0, %ymm0
+; SKX_32-NEXT:    vscatterdpd %zmm3, (%eax,%ymm0,8) {%k1}
 ; SKX_32-NEXT:    movl %ebp, %esp
 ; SKX_32-NEXT:    popl %ebp
 ; SKX_32-NEXT:    .cfi_def_cfa %esp, 4
@@ -2972,3 +3078,236 @@ define void @zero_mask(<2 x double>%a1, <2 x double*> %ptr) {
   call void @llvm.masked.scatter.v2f64.v2p0f64(<2 x double> %a1, <2 x double*> %ptr, i32 4, <2 x i1> zeroinitializer)
   ret void
 }
+
+define <2 x i64> @gather_2i64_constant_indices(i64* %ptr, <2 x i1> %mask) {
+; KNL_64-LABEL: gather_2i64_constant_indices:
+; KNL_64:       # %bb.0:
+; KNL_64-NEXT:    vpsllq $63, %xmm0, %xmm0
+; KNL_64-NEXT:    vptestmq %zmm0, %zmm0, %k0
+; KNL_64-NEXT:    kshiftlw $14, %k0, %k0
+; KNL_64-NEXT:    kshiftrw $14, %k0, %k1
+; KNL_64-NEXT:    vmovdqa {{.*#+}} ymm1 = <0,4294967294,u,u,u,u,u,u>
+; KNL_64-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; KNL_64-NEXT:    vpgatherdq (%rdi,%ymm1,8), %zmm0 {%k1}
+; KNL_64-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; KNL_64-NEXT:    vzeroupper
+; KNL_64-NEXT:    retq
+;
+; KNL_32-LABEL: gather_2i64_constant_indices:
+; KNL_32:       # %bb.0:
+; KNL_32-NEXT:    vpsllq $63, %xmm0, %xmm0
+; KNL_32-NEXT:    vptestmq %zmm0, %zmm0, %k0
+; KNL_32-NEXT:    kshiftlw $14, %k0, %k0
+; KNL_32-NEXT:    kshiftrw $14, %k0, %k1
+; KNL_32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; KNL_32-NEXT:    vmovdqa {{.*#+}} ymm1 = <0,4294967294,u,u,u,u,u,u>
+; KNL_32-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; KNL_32-NEXT:    vpgatherdq (%eax,%ymm1,8), %zmm0 {%k1}
+; KNL_32-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; KNL_32-NEXT:    vzeroupper
+; KNL_32-NEXT:    retl
+;
+; SKX_SMALL-LABEL: gather_2i64_constant_indices:
+; SKX_SMALL:       # %bb.0:
+; SKX_SMALL-NEXT:    vpsllq $63, %xmm0, %xmm0
+; SKX_SMALL-NEXT:    vpmovq2m %xmm0, %k1
+; SKX_SMALL-NEXT:    vmovdqa {{.*#+}} xmm1 = <0,4294967294,u,u>
+; SKX_SMALL-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; SKX_SMALL-NEXT:    vpgatherdq (%rdi,%xmm1,8), %xmm0 {%k1}
+; SKX_SMALL-NEXT:    retq
+;
+; SKX_LARGE-LABEL: gather_2i64_constant_indices:
+; SKX_LARGE:       # %bb.0:
+; SKX_LARGE-NEXT:    vpsllq $63, %xmm0, %xmm0
+; SKX_LARGE-NEXT:    vpmovq2m %xmm0, %k1
+; SKX_LARGE-NEXT:    movabsq ${{\.LCPI.*}}, %rax
+; SKX_LARGE-NEXT:    vmovdqa (%rax), %xmm1
+; SKX_LARGE-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; SKX_LARGE-NEXT:    vpgatherdq (%rdi,%xmm1,8), %xmm0 {%k1}
+; SKX_LARGE-NEXT:    retq
+;
+; SKX_32-LABEL: gather_2i64_constant_indices:
+; SKX_32:       # %bb.0:
+; SKX_32-NEXT:    vpsllq $63, %xmm0, %xmm0
+; SKX_32-NEXT:    vpmovq2m %xmm0, %k1
+; SKX_32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; SKX_32-NEXT:    vmovdqa {{.*#+}} xmm1 = <0,4294967294,u,u>
+; SKX_32-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; SKX_32-NEXT:    vpgatherdq (%eax,%xmm1,8), %xmm0 {%k1}
+; SKX_32-NEXT:    retl
+  %gep = getelementptr i64, i64* %ptr, <2 x i64> <i64 0, i64 -2>
+  %res = tail call <2 x i64> @llvm.masked.gather.v2i64.v2p0i64(<2 x i64*> %gep, i32 8, <2 x i1> %mask, <2 x i64> zeroinitializer) #1
+  ret <2 x i64> %res
+}
+
+define <16 x i32> @gather_16i64_constant_indices(i32* %ptr, <16 x i1> %mask) {
+; KNL_64-LABEL: gather_16i64_constant_indices:
+; KNL_64:       # %bb.0:
+; KNL_64-NEXT:    vpmovsxbd %xmm0, %zmm0
+; KNL_64-NEXT:    vpslld $31, %zmm0, %zmm0
+; KNL_64-NEXT:    vptestmd %zmm0, %zmm0, %k1
+; KNL_64-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,4294967294,1,4294967288,10,20,50,65536,16777215,2147483647,100,4294965296,2147483648,76897723,7,4227069609]
+; KNL_64-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; KNL_64-NEXT:    vpgatherdd (%rdi,%zmm1,4), %zmm0 {%k1}
+; KNL_64-NEXT:    retq
+;
+; KNL_32-LABEL: gather_16i64_constant_indices:
+; KNL_32:       # %bb.0:
+; KNL_32-NEXT:    vpmovsxbd %xmm0, %zmm0
+; KNL_32-NEXT:    vpslld $31, %zmm0, %zmm0
+; KNL_32-NEXT:    vptestmd %zmm0, %zmm0, %k1
+; KNL_32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; KNL_32-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,4294967294,1,4294967288,10,20,50,65536,16777215,2147483647,100,4294965296,2147483648,76897723,7,4227069609]
+; KNL_32-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; KNL_32-NEXT:    vpgatherdd (%eax,%zmm1,4), %zmm0 {%k1}
+; KNL_32-NEXT:    retl
+;
+; SKX_SMALL-LABEL: gather_16i64_constant_indices:
+; SKX_SMALL:       # %bb.0:
+; SKX_SMALL-NEXT:    vpmovsxbd %xmm0, %zmm0
+; SKX_SMALL-NEXT:    vpslld $31, %zmm0, %zmm0
+; SKX_SMALL-NEXT:    vpmovd2m %zmm0, %k1
+; SKX_SMALL-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,4294967294,1,4294967288,10,20,50,65536,16777215,2147483647,100,4294965296,2147483648,76897723,7,4227069609]
+; SKX_SMALL-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; SKX_SMALL-NEXT:    vpgatherdd (%rdi,%zmm1,4), %zmm0 {%k1}
+; SKX_SMALL-NEXT:    retq
+;
+; SKX_LARGE-LABEL: gather_16i64_constant_indices:
+; SKX_LARGE:       # %bb.0:
+; SKX_LARGE-NEXT:    vpmovsxbd %xmm0, %zmm0
+; SKX_LARGE-NEXT:    vpslld $31, %zmm0, %zmm0
+; SKX_LARGE-NEXT:    vpmovd2m %zmm0, %k1
+; SKX_LARGE-NEXT:    movabsq ${{\.LCPI.*}}, %rax
+; SKX_LARGE-NEXT:    vmovdqa64 (%rax), %zmm1
+; SKX_LARGE-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; SKX_LARGE-NEXT:    vpgatherdd (%rdi,%zmm1,4), %zmm0 {%k1}
+; SKX_LARGE-NEXT:    retq
+;
+; SKX_32-LABEL: gather_16i64_constant_indices:
+; SKX_32:       # %bb.0:
+; SKX_32-NEXT:    vpmovsxbd %xmm0, %zmm0
+; SKX_32-NEXT:    vpslld $31, %zmm0, %zmm0
+; SKX_32-NEXT:    vpmovd2m %zmm0, %k1
+; SKX_32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; SKX_32-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,4294967294,1,4294967288,10,20,50,65536,16777215,2147483647,100,4294965296,2147483648,76897723,7,4227069609]
+; SKX_32-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; SKX_32-NEXT:    vpgatherdd (%eax,%zmm1,4), %zmm0 {%k1}
+; SKX_32-NEXT:    retl
+  %gep = getelementptr i32, i32* %ptr, <16 x i64> <i64 0, i64 -2, i64 1, i64 -8, i64 10, i64 20, i64 50, i64 65536, i64 16777215, i64 2147483647, i64 100, i64 -2000, i64 -2147483648, i64 76897723, i64 7, i64 -67897687>
+  %res = tail call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> %gep, i32 4, <16 x i1> %mask, <16 x i32> zeroinitializer) #1
+  ret <16 x i32> %res
+}
+
+define void @scatter_2i64_constant_indices(i32* %ptr, <2 x i1> %mask, <2 x i32> %src0)  {
+; KNL_64-LABEL: scatter_2i64_constant_indices:
+; KNL_64:       # %bb.0:
+; KNL_64-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; KNL_64-NEXT:    vpsllq $63, %xmm0, %xmm0
+; KNL_64-NEXT:    vptestmq %zmm0, %zmm0, %k0
+; KNL_64-NEXT:    kshiftlw $14, %k0, %k0
+; KNL_64-NEXT:    kshiftrw $14, %k0, %k1
+; KNL_64-NEXT:    vmovdqa64 {{.*#+}} zmm0 = <0,4294967294,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; KNL_64-NEXT:    vpscatterdd %zmm1, (%rdi,%zmm0,4) {%k1}
+; KNL_64-NEXT:    vzeroupper
+; KNL_64-NEXT:    retq
+;
+; KNL_32-LABEL: scatter_2i64_constant_indices:
+; KNL_32:       # %bb.0:
+; KNL_32-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; KNL_32-NEXT:    vpsllq $63, %xmm0, %xmm0
+; KNL_32-NEXT:    vptestmq %zmm0, %zmm0, %k0
+; KNL_32-NEXT:    kshiftlw $14, %k0, %k0
+; KNL_32-NEXT:    kshiftrw $14, %k0, %k1
+; KNL_32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; KNL_32-NEXT:    vmovdqa64 {{.*#+}} zmm0 = <0,4294967294,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; KNL_32-NEXT:    vpscatterdd %zmm1, (%eax,%zmm0,4) {%k1}
+; KNL_32-NEXT:    vzeroupper
+; KNL_32-NEXT:    retl
+;
+; SKX_SMALL-LABEL: scatter_2i64_constant_indices:
+; SKX_SMALL:       # %bb.0:
+; SKX_SMALL-NEXT:    vpsllq $63, %xmm0, %xmm0
+; SKX_SMALL-NEXT:    vpmovq2m %xmm0, %k1
+; SKX_SMALL-NEXT:    vmovdqa {{.*#+}} xmm0 = <0,4294967294,u,u>
+; SKX_SMALL-NEXT:    vpscatterdd %xmm1, (%rdi,%xmm0,4) {%k1}
+; SKX_SMALL-NEXT:    retq
+;
+; SKX_LARGE-LABEL: scatter_2i64_constant_indices:
+; SKX_LARGE:       # %bb.0:
+; SKX_LARGE-NEXT:    vpsllq $63, %xmm0, %xmm0
+; SKX_LARGE-NEXT:    vpmovq2m %xmm0, %k1
+; SKX_LARGE-NEXT:    movabsq ${{\.LCPI.*}}, %rax
+; SKX_LARGE-NEXT:    vmovdqa (%rax), %xmm0
+; SKX_LARGE-NEXT:    vpscatterdd %xmm1, (%rdi,%xmm0,4) {%k1}
+; SKX_LARGE-NEXT:    retq
+;
+; SKX_32-LABEL: scatter_2i64_constant_indices:
+; SKX_32:       # %bb.0:
+; SKX_32-NEXT:    vpsllq $63, %xmm0, %xmm0
+; SKX_32-NEXT:    vpmovq2m %xmm0, %k1
+; SKX_32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; SKX_32-NEXT:    vmovdqa {{.*#+}} xmm0 = <0,4294967294,u,u>
+; SKX_32-NEXT:    vpscatterdd %xmm1, (%eax,%xmm0,4) {%k1}
+; SKX_32-NEXT:    retl
+  %gep = getelementptr i32, i32* %ptr, <2 x i64> <i64 0, i64 -2>
+  call void @llvm.masked.scatter.v2i32.v2p0i32(<2 x i32> %src0, <2 x i32*> %gep, i32 4, <2 x i1> %mask)
+  ret void
+}
+
+define void @scatter_16i64_constant_indices(i32* %ptr, <16 x i1> %mask, <16 x i32> %src0)  {
+; KNL_64-LABEL: scatter_16i64_constant_indices:
+; KNL_64:       # %bb.0:
+; KNL_64-NEXT:    vpmovsxbd %xmm0, %zmm0
+; KNL_64-NEXT:    vpslld $31, %zmm0, %zmm0
+; KNL_64-NEXT:    vptestmd %zmm0, %zmm0, %k1
+; KNL_64-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [0,4294967294,1,4294967288,10,20,50,65536,16777215,2147483647,100,4294965296,2147483648,76897723,7,4227069609]
+; KNL_64-NEXT:    vpscatterdd %zmm1, (%rdi,%zmm0,4) {%k1}
+; KNL_64-NEXT:    vzeroupper
+; KNL_64-NEXT:    retq
+;
+; KNL_32-LABEL: scatter_16i64_constant_indices:
+; KNL_32:       # %bb.0:
+; KNL_32-NEXT:    vpmovsxbd %xmm0, %zmm0
+; KNL_32-NEXT:    vpslld $31, %zmm0, %zmm0
+; KNL_32-NEXT:    vptestmd %zmm0, %zmm0, %k1
+; KNL_32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; KNL_32-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [0,4294967294,1,4294967288,10,20,50,65536,16777215,2147483647,100,4294965296,2147483648,76897723,7,4227069609]
+; KNL_32-NEXT:    vpscatterdd %zmm1, (%eax,%zmm0,4) {%k1}
+; KNL_32-NEXT:    vzeroupper
+; KNL_32-NEXT:    retl
+;
+; SKX_SMALL-LABEL: scatter_16i64_constant_indices:
+; SKX_SMALL:       # %bb.0:
+; SKX_SMALL-NEXT:    vpmovsxbd %xmm0, %zmm0
+; SKX_SMALL-NEXT:    vpslld $31, %zmm0, %zmm0
+; SKX_SMALL-NEXT:    vpmovd2m %zmm0, %k1
+; SKX_SMALL-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [0,4294967294,1,4294967288,10,20,50,65536,16777215,2147483647,100,4294965296,2147483648,76897723,7,4227069609]
+; SKX_SMALL-NEXT:    vpscatterdd %zmm1, (%rdi,%zmm0,4) {%k1}
+; SKX_SMALL-NEXT:    vzeroupper
+; SKX_SMALL-NEXT:    retq
+;
+; SKX_LARGE-LABEL: scatter_16i64_constant_indices:
+; SKX_LARGE:       # %bb.0:
+; SKX_LARGE-NEXT:    vpmovsxbd %xmm0, %zmm0
+; SKX_LARGE-NEXT:    vpslld $31, %zmm0, %zmm0
+; SKX_LARGE-NEXT:    vpmovd2m %zmm0, %k1
+; SKX_LARGE-NEXT:    movabsq ${{\.LCPI.*}}, %rax
+; SKX_LARGE-NEXT:    vmovdqa64 (%rax), %zmm0
+; SKX_LARGE-NEXT:    vpscatterdd %zmm1, (%rdi,%zmm0,4) {%k1}
+; SKX_LARGE-NEXT:    vzeroupper
+; SKX_LARGE-NEXT:    retq
+;
+; SKX_32-LABEL: scatter_16i64_constant_indices:
+; SKX_32:       # %bb.0:
+; SKX_32-NEXT:    vpmovsxbd %xmm0, %zmm0
+; SKX_32-NEXT:    vpslld $31, %zmm0, %zmm0
+; SKX_32-NEXT:    vpmovd2m %zmm0, %k1
+; SKX_32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; SKX_32-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [0,4294967294,1,4294967288,10,20,50,65536,16777215,2147483647,100,4294965296,2147483648,76897723,7,4227069609]
+; SKX_32-NEXT:    vpscatterdd %zmm1, (%eax,%zmm0,4) {%k1}
+; SKX_32-NEXT:    vzeroupper
+; SKX_32-NEXT:    retl
+  %gep = getelementptr i32, i32* %ptr, <16 x i64> <i64 0, i64 -2, i64 1, i64 -8, i64 10, i64 20, i64 50, i64 65536, i64 16777215, i64 2147483647, i64 100, i64 -2000, i64 -2147483648, i64 76897723, i64 7, i64 -67897687>
+  call void @llvm.masked.scatter.v16i32.v16p0i32(<16 x i32> %src0, <16 x i32*> %gep, i32 4, <16 x i1> %mask)
+  ret void
+}
diff --git a/llvm/test/CodeGen/X86/masked_gather_scatter_widen.ll b/llvm/test/CodeGen/X86/masked_gather_scatter_widen.ll
index 03491a8ad..fc6341640 100644
--- a/llvm/test/CodeGen/X86/masked_gather_scatter_widen.ll
+++ b/llvm/test/CodeGen/X86/masked_gather_scatter_widen.ll
@@ -1,10 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512vl -mattr=+avx512dq -x86-experimental-vector-widening-legalization < %s | FileCheck %s --check-prefix=CHECK --check-prefix=WIDEN --check-prefix=WIDEN_SKX
-; RUN: llc -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512f -x86-experimental-vector-widening-legalization < %s | FileCheck %s --check-prefix=CHECK --check-prefix=WIDEN --check-prefix=WIDEN_KNL
-; RUN: llc -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512vl -mattr=+avx512dq < %s | FileCheck %s --check-prefix=CHECK --check-prefix=PROMOTE --check-prefix=PROMOTE_SKX
-; RUN: llc -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512f < %s | FileCheck %s --check-prefix=CHECK --check-prefix=PROMOTE --check-prefix=PROMOTE_KNL
-; RUN: llc -mtriple=x86_64-unknown-linux-gnu -mcpu=skylake -x86-experimental-vector-widening-legalization < %s | FileCheck %s --check-prefix=WIDEN_AVX2
-; RUN: llc -mtriple=x86_64-unknown-linux-gnu -mcpu=skylake < %s | FileCheck %s --check-prefix=PROMOTE_AVX2
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512vl -mattr=+avx512dq < %s | FileCheck %s --check-prefix=CHECK --check-prefix=WIDEN --check-prefix=WIDEN_SKX
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512f < %s | FileCheck %s --check-prefix=CHECK --check-prefix=WIDEN --check-prefix=WIDEN_KNL
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu -mcpu=skylake < %s | FileCheck %s --check-prefix=WIDEN_AVX2
 
 define <2 x double> @test_gather_v2i32_index(double* %base, <2 x i32> %ind, <2 x i1> %mask, <2 x double> %src0) {
 ; WIDEN_SKX-LABEL: test_gather_v2i32_index:
@@ -28,46 +25,12 @@ define <2 x double> @test_gather_v2i32_index(double* %base, <2 x i32> %ind, <2 x
 ; WIDEN_KNL-NEXT:    vzeroupper
 ; WIDEN_KNL-NEXT:    retq
 ;
-; PROMOTE_SKX-LABEL: test_gather_v2i32_index:
-; PROMOTE_SKX:       # %bb.0:
-; PROMOTE_SKX-NEXT:    vpsllq $32, %xmm0, %xmm0
-; PROMOTE_SKX-NEXT:    vpsraq $32, %xmm0, %xmm0
-; PROMOTE_SKX-NEXT:    vpsllq $63, %xmm1, %xmm1
-; PROMOTE_SKX-NEXT:    vpmovq2m %xmm1, %k1
-; PROMOTE_SKX-NEXT:    vgatherqpd (%rdi,%xmm0,8), %xmm2 {%k1}
-; PROMOTE_SKX-NEXT:    vmovapd %xmm2, %xmm0
-; PROMOTE_SKX-NEXT:    retq
-;
-; PROMOTE_KNL-LABEL: test_gather_v2i32_index:
-; PROMOTE_KNL:       # %bb.0:
-; PROMOTE_KNL-NEXT:    # kill: def $xmm2 killed $xmm2 def $zmm2
-; PROMOTE_KNL-NEXT:    vpsllq $32, %xmm0, %xmm0
-; PROMOTE_KNL-NEXT:    vpsraq $32, %zmm0, %zmm0
-; PROMOTE_KNL-NEXT:    vpsllq $63, %xmm1, %xmm1
-; PROMOTE_KNL-NEXT:    vptestmq %zmm1, %zmm1, %k0
-; PROMOTE_KNL-NEXT:    kshiftlw $14, %k0, %k0
-; PROMOTE_KNL-NEXT:    kshiftrw $14, %k0, %k1
-; PROMOTE_KNL-NEXT:    vgatherqpd (%rdi,%zmm0,8), %zmm2 {%k1}
-; PROMOTE_KNL-NEXT:    vmovapd %xmm2, %xmm0
-; PROMOTE_KNL-NEXT:    vzeroupper
-; PROMOTE_KNL-NEXT:    retq
-;
 ; WIDEN_AVX2-LABEL: test_gather_v2i32_index:
 ; WIDEN_AVX2:       # %bb.0:
 ; WIDEN_AVX2-NEXT:    vpsllq $63, %xmm1, %xmm1
 ; WIDEN_AVX2-NEXT:    vgatherdpd %xmm1, (%rdi,%xmm0,8), %xmm2
 ; WIDEN_AVX2-NEXT:    vmovapd %xmm2, %xmm0
 ; WIDEN_AVX2-NEXT:    retq
-;
-; PROMOTE_AVX2-LABEL: test_gather_v2i32_index:
-; PROMOTE_AVX2:       # %bb.0:
-; PROMOTE_AVX2-NEXT:    vpsllq $32, %xmm0, %xmm3
-; PROMOTE_AVX2-NEXT:    vpsrad $31, %xmm3, %xmm3
-; PROMOTE_AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm3[1],xmm0[2],xmm3[3]
-; PROMOTE_AVX2-NEXT:    vpsllq $63, %xmm1, %xmm1
-; PROMOTE_AVX2-NEXT:    vgatherqpd %xmm1, (%rdi,%xmm0,8), %xmm2
-; PROMOTE_AVX2-NEXT:    vmovapd %xmm2, %xmm0
-; PROMOTE_AVX2-NEXT:    retq
   %gep.random = getelementptr double, double* %base, <2 x i32> %ind
   %res = call <2 x double> @llvm.masked.gather.v2f64.v2p0f64(<2 x double*> %gep.random, i32 4, <2 x i1> %mask, <2 x double> %src0)
   ret <2 x double> %res
@@ -93,28 +56,6 @@ define void @test_scatter_v2i32_index(<2 x double> %a1, double* %base, <2 x i32>
 ; WIDEN_KNL-NEXT:    vzeroupper
 ; WIDEN_KNL-NEXT:    retq
 ;
-; PROMOTE_SKX-LABEL: test_scatter_v2i32_index:
-; PROMOTE_SKX:       # %bb.0:
-; PROMOTE_SKX-NEXT:    vpsllq $63, %xmm2, %xmm2
-; PROMOTE_SKX-NEXT:    vpmovq2m %xmm2, %k1
-; PROMOTE_SKX-NEXT:    vpsllq $32, %xmm1, %xmm1
-; PROMOTE_SKX-NEXT:    vpsraq $32, %xmm1, %xmm1
-; PROMOTE_SKX-NEXT:    vscatterqpd %xmm0, (%rdi,%xmm1,8) {%k1}
-; PROMOTE_SKX-NEXT:    retq
-;
-; PROMOTE_KNL-LABEL: test_scatter_v2i32_index:
-; PROMOTE_KNL:       # %bb.0:
-; PROMOTE_KNL-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; PROMOTE_KNL-NEXT:    vpsllq $32, %xmm1, %xmm1
-; PROMOTE_KNL-NEXT:    vpsraq $32, %zmm1, %zmm1
-; PROMOTE_KNL-NEXT:    vpsllq $63, %xmm2, %xmm2
-; PROMOTE_KNL-NEXT:    vptestmq %zmm2, %zmm2, %k0
-; PROMOTE_KNL-NEXT:    kshiftlw $14, %k0, %k0
-; PROMOTE_KNL-NEXT:    kshiftrw $14, %k0, %k1
-; PROMOTE_KNL-NEXT:    vscatterqpd %zmm0, (%rdi,%zmm1,8) {%k1}
-; PROMOTE_KNL-NEXT:    vzeroupper
-; PROMOTE_KNL-NEXT:    retq
-;
 ; WIDEN_AVX2-LABEL: test_scatter_v2i32_index:
 ; WIDEN_AVX2:       # %bb.0:
 ; WIDEN_AVX2-NEXT:    vpmovsxdq %xmm1, %xmm1
@@ -122,46 +63,24 @@ define void @test_scatter_v2i32_index(<2 x double> %a1, double* %base, <2 x i32>
 ; WIDEN_AVX2-NEXT:    vmovq %rdi, %xmm3
 ; WIDEN_AVX2-NEXT:    vpbroadcastq %xmm3, %xmm3
 ; WIDEN_AVX2-NEXT:    vpaddq %xmm1, %xmm3, %xmm1
-; WIDEN_AVX2-NEXT:    vpextrb $0, %xmm2, %eax
-; WIDEN_AVX2-NEXT:    testb $1, %al
-; WIDEN_AVX2-NEXT:    je .LBB1_2
-; WIDEN_AVX2-NEXT:  # %bb.1: # %cond.store
-; WIDEN_AVX2-NEXT:    vmovq %xmm1, %rax
-; WIDEN_AVX2-NEXT:    vmovlps %xmm0, (%rax)
-; WIDEN_AVX2-NEXT:  .LBB1_2: # %else
-; WIDEN_AVX2-NEXT:    vpextrb $8, %xmm2, %eax
+; WIDEN_AVX2-NEXT:    vpsllq $63, %xmm2, %xmm2
+; WIDEN_AVX2-NEXT:    vmovmskpd %xmm2, %eax
 ; WIDEN_AVX2-NEXT:    testb $1, %al
+; WIDEN_AVX2-NEXT:    jne .LBB1_1
+; WIDEN_AVX2-NEXT:  # %bb.2: # %else
+; WIDEN_AVX2-NEXT:    testb $2, %al
+; WIDEN_AVX2-NEXT:    jne .LBB1_3
+; WIDEN_AVX2-NEXT:  .LBB1_4: # %else2
+; WIDEN_AVX2-NEXT:    retq
+; WIDEN_AVX2-NEXT:  .LBB1_1: # %cond.store
+; WIDEN_AVX2-NEXT:    vmovq %xmm1, %rcx
+; WIDEN_AVX2-NEXT:    vmovlps %xmm0, (%rcx)
+; WIDEN_AVX2-NEXT:    testb $2, %al
 ; WIDEN_AVX2-NEXT:    je .LBB1_4
-; WIDEN_AVX2-NEXT:  # %bb.3: # %cond.store1
+; WIDEN_AVX2-NEXT:  .LBB1_3: # %cond.store1
 ; WIDEN_AVX2-NEXT:    vpextrq $1, %xmm1, %rax
 ; WIDEN_AVX2-NEXT:    vmovhps %xmm0, (%rax)
-; WIDEN_AVX2-NEXT:  .LBB1_4: # %else2
 ; WIDEN_AVX2-NEXT:    retq
-;
-; PROMOTE_AVX2-LABEL: test_scatter_v2i32_index:
-; PROMOTE_AVX2:       # %bb.0:
-; PROMOTE_AVX2-NEXT:    vpsllq $32, %xmm1, %xmm3
-; PROMOTE_AVX2-NEXT:    vpsrad $31, %xmm3, %xmm3
-; PROMOTE_AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm3[1],xmm1[2],xmm3[3]
-; PROMOTE_AVX2-NEXT:    vpsllq $3, %xmm1, %xmm1
-; PROMOTE_AVX2-NEXT:    vmovq %rdi, %xmm3
-; PROMOTE_AVX2-NEXT:    vpbroadcastq %xmm3, %xmm3
-; PROMOTE_AVX2-NEXT:    vpaddq %xmm1, %xmm3, %xmm1
-; PROMOTE_AVX2-NEXT:    vpextrb $0, %xmm2, %eax
-; PROMOTE_AVX2-NEXT:    testb $1, %al
-; PROMOTE_AVX2-NEXT:    je .LBB1_2
-; PROMOTE_AVX2-NEXT:  # %bb.1: # %cond.store
-; PROMOTE_AVX2-NEXT:    vmovq %xmm1, %rax
-; PROMOTE_AVX2-NEXT:    vmovlps %xmm0, (%rax)
-; PROMOTE_AVX2-NEXT:  .LBB1_2: # %else
-; PROMOTE_AVX2-NEXT:    vpextrb $8, %xmm2, %eax
-; PROMOTE_AVX2-NEXT:    testb $1, %al
-; PROMOTE_AVX2-NEXT:    je .LBB1_4
-; PROMOTE_AVX2-NEXT:  # %bb.3: # %cond.store1
-; PROMOTE_AVX2-NEXT:    vpextrq $1, %xmm1, %rax
-; PROMOTE_AVX2-NEXT:    vmovhps %xmm0, (%rax)
-; PROMOTE_AVX2-NEXT:  .LBB1_4: # %else2
-; PROMOTE_AVX2-NEXT:    retq
   %gep = getelementptr double, double *%base, <2 x i32> %ind
   call void @llvm.masked.scatter.v2f64.v2p0f64(<2 x double> %a1, <2 x double*> %gep, i32 4, <2 x i1> %mask)
   ret void
@@ -189,28 +108,6 @@ define <2 x i32> @test_gather_v2i32_data(<2 x i32*> %ptr, <2 x i1> %mask, <2 x i
 ; WIDEN_KNL-NEXT:    vzeroupper
 ; WIDEN_KNL-NEXT:    retq
 ;
-; PROMOTE_SKX-LABEL: test_gather_v2i32_data:
-; PROMOTE_SKX:       # %bb.0:
-; PROMOTE_SKX-NEXT:    vpsllq $63, %xmm1, %xmm1
-; PROMOTE_SKX-NEXT:    vpmovq2m %xmm1, %k1
-; PROMOTE_SKX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
-; PROMOTE_SKX-NEXT:    vpgatherqd (,%xmm0), %xmm1 {%k1}
-; PROMOTE_SKX-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero
-; PROMOTE_SKX-NEXT:    retq
-;
-; PROMOTE_KNL-LABEL: test_gather_v2i32_data:
-; PROMOTE_KNL:       # %bb.0:
-; PROMOTE_KNL-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; PROMOTE_KNL-NEXT:    vpsllq $63, %xmm1, %xmm1
-; PROMOTE_KNL-NEXT:    vptestmq %zmm1, %zmm1, %k0
-; PROMOTE_KNL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
-; PROMOTE_KNL-NEXT:    kshiftlw $14, %k0, %k0
-; PROMOTE_KNL-NEXT:    kshiftrw $14, %k0, %k1
-; PROMOTE_KNL-NEXT:    vpgatherqd (,%zmm0), %ymm1 {%k1}
-; PROMOTE_KNL-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero
-; PROMOTE_KNL-NEXT:    vzeroupper
-; PROMOTE_KNL-NEXT:    retq
-;
 ; WIDEN_AVX2-LABEL: test_gather_v2i32_data:
 ; WIDEN_AVX2:       # %bb.0:
 ; WIDEN_AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
@@ -218,15 +115,6 @@ define <2 x i32> @test_gather_v2i32_data(<2 x i32*> %ptr, <2 x i1> %mask, <2 x i
 ; WIDEN_AVX2-NEXT:    vpgatherqd %xmm1, (,%xmm0), %xmm2
 ; WIDEN_AVX2-NEXT:    vmovdqa %xmm2, %xmm0
 ; WIDEN_AVX2-NEXT:    retq
-;
-; PROMOTE_AVX2-LABEL: test_gather_v2i32_data:
-; PROMOTE_AVX2:       # %bb.0:
-; PROMOTE_AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
-; PROMOTE_AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; PROMOTE_AVX2-NEXT:    vpslld $31, %xmm1, %xmm1
-; PROMOTE_AVX2-NEXT:    vpgatherqd %xmm1, (,%xmm0), %xmm2
-; PROMOTE_AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm2[0],zero,xmm2[1],zero
-; PROMOTE_AVX2-NEXT:    retq
   %res = call <2 x i32> @llvm.masked.gather.v2i32.v2p0i32(<2 x i32*> %ptr, i32 4, <2 x i1> %mask, <2 x i32> %src0)
   ret <2 x i32>%res
 }
@@ -251,61 +139,26 @@ define void @test_scatter_v2i32_data(<2 x i32>%a1, <2 x i32*> %ptr, <2 x i1>%mas
 ; WIDEN_KNL-NEXT:    vzeroupper
 ; WIDEN_KNL-NEXT:    retq
 ;
-; PROMOTE_SKX-LABEL: test_scatter_v2i32_data:
-; PROMOTE_SKX:       # %bb.0:
-; PROMOTE_SKX-NEXT:    vpsllq $63, %xmm2, %xmm2
-; PROMOTE_SKX-NEXT:    vpmovq2m %xmm2, %k1
-; PROMOTE_SKX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; PROMOTE_SKX-NEXT:    vpscatterqd %xmm0, (,%xmm1) {%k1}
-; PROMOTE_SKX-NEXT:    retq
-;
-; PROMOTE_KNL-LABEL: test_scatter_v2i32_data:
-; PROMOTE_KNL:       # %bb.0:
-; PROMOTE_KNL-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
-; PROMOTE_KNL-NEXT:    vpsllq $63, %xmm2, %xmm2
-; PROMOTE_KNL-NEXT:    vptestmq %zmm2, %zmm2, %k0
-; PROMOTE_KNL-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; PROMOTE_KNL-NEXT:    kshiftlw $14, %k0, %k0
-; PROMOTE_KNL-NEXT:    kshiftrw $14, %k0, %k1
-; PROMOTE_KNL-NEXT:    vpscatterqd %ymm0, (,%zmm1) {%k1}
-; PROMOTE_KNL-NEXT:    vzeroupper
-; PROMOTE_KNL-NEXT:    retq
-;
 ; WIDEN_AVX2-LABEL: test_scatter_v2i32_data:
 ; WIDEN_AVX2:       # %bb.0:
-; WIDEN_AVX2-NEXT:    vpextrb $0, %xmm2, %eax
-; WIDEN_AVX2-NEXT:    testb $1, %al
-; WIDEN_AVX2-NEXT:    je .LBB3_2
-; WIDEN_AVX2-NEXT:  # %bb.1: # %cond.store
-; WIDEN_AVX2-NEXT:    vmovq %xmm1, %rax
-; WIDEN_AVX2-NEXT:    vmovss %xmm0, (%rax)
-; WIDEN_AVX2-NEXT:  .LBB3_2: # %else
-; WIDEN_AVX2-NEXT:    vpextrb $8, %xmm2, %eax
+; WIDEN_AVX2-NEXT:    vpsllq $63, %xmm2, %xmm2
+; WIDEN_AVX2-NEXT:    vmovmskpd %xmm2, %eax
 ; WIDEN_AVX2-NEXT:    testb $1, %al
+; WIDEN_AVX2-NEXT:    jne .LBB3_1
+; WIDEN_AVX2-NEXT:  # %bb.2: # %else
+; WIDEN_AVX2-NEXT:    testb $2, %al
+; WIDEN_AVX2-NEXT:    jne .LBB3_3
+; WIDEN_AVX2-NEXT:  .LBB3_4: # %else2
+; WIDEN_AVX2-NEXT:    retq
+; WIDEN_AVX2-NEXT:  .LBB3_1: # %cond.store
+; WIDEN_AVX2-NEXT:    vmovq %xmm1, %rcx
+; WIDEN_AVX2-NEXT:    vmovss %xmm0, (%rcx)
+; WIDEN_AVX2-NEXT:    testb $2, %al
 ; WIDEN_AVX2-NEXT:    je .LBB3_4
-; WIDEN_AVX2-NEXT:  # %bb.3: # %cond.store1
+; WIDEN_AVX2-NEXT:  .LBB3_3: # %cond.store1
 ; WIDEN_AVX2-NEXT:    vpextrq $1, %xmm1, %rax
 ; WIDEN_AVX2-NEXT:    vextractps $1, %xmm0, (%rax)
-; WIDEN_AVX2-NEXT:  .LBB3_4: # %else2
 ; WIDEN_AVX2-NEXT:    retq
-;
-; PROMOTE_AVX2-LABEL: test_scatter_v2i32_data:
-; PROMOTE_AVX2:       # %bb.0:
-; PROMOTE_AVX2-NEXT:    vpextrb $0, %xmm2, %eax
-; PROMOTE_AVX2-NEXT:    testb $1, %al
-; PROMOTE_AVX2-NEXT:    je .LBB3_2
-; PROMOTE_AVX2-NEXT:  # %bb.1: # %cond.store
-; PROMOTE_AVX2-NEXT:    vmovq %xmm1, %rax
-; PROMOTE_AVX2-NEXT:    vmovss %xmm0, (%rax)
-; PROMOTE_AVX2-NEXT:  .LBB3_2: # %else
-; PROMOTE_AVX2-NEXT:    vpextrb $8, %xmm2, %eax
-; PROMOTE_AVX2-NEXT:    testb $1, %al
-; PROMOTE_AVX2-NEXT:    je .LBB3_4
-; PROMOTE_AVX2-NEXT:  # %bb.3: # %cond.store1
-; PROMOTE_AVX2-NEXT:    vpextrq $1, %xmm1, %rax
-; PROMOTE_AVX2-NEXT:    vextractps $2, %xmm0, (%rax)
-; PROMOTE_AVX2-NEXT:  .LBB3_4: # %else2
-; PROMOTE_AVX2-NEXT:    retq
   call void @llvm.masked.scatter.v2i32.v2p0i32(<2 x i32> %a1, <2 x i32*> %ptr, i32 4, <2 x i1> %mask)
   ret void
 }
@@ -332,29 +185,6 @@ define <2 x i32> @test_gather_v2i32_data_index(i32* %base, <2 x i32> %ind, <2 x
 ; WIDEN_KNL-NEXT:    vzeroupper
 ; WIDEN_KNL-NEXT:    retq
 ;
-; PROMOTE_SKX-LABEL: test_gather_v2i32_data_index:
-; PROMOTE_SKX:       # %bb.0:
-; PROMOTE_SKX-NEXT:    vpsllq $63, %xmm1, %xmm1
-; PROMOTE_SKX-NEXT:    vpmovq2m %xmm1, %k1
-; PROMOTE_SKX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; PROMOTE_SKX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
-; PROMOTE_SKX-NEXT:    vpgatherdd (%rdi,%xmm0,4), %xmm1 {%k1}
-; PROMOTE_SKX-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero
-; PROMOTE_SKX-NEXT:    retq
-;
-; PROMOTE_KNL-LABEL: test_gather_v2i32_data_index:
-; PROMOTE_KNL:       # %bb.0:
-; PROMOTE_KNL-NEXT:    vpsllq $63, %xmm1, %xmm1
-; PROMOTE_KNL-NEXT:    vptestmq %zmm1, %zmm1, %k0
-; PROMOTE_KNL-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; PROMOTE_KNL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
-; PROMOTE_KNL-NEXT:    kshiftlw $14, %k0, %k0
-; PROMOTE_KNL-NEXT:    kshiftrw $14, %k0, %k1
-; PROMOTE_KNL-NEXT:    vpgatherdd (%rdi,%zmm0,4), %zmm1 {%k1}
-; PROMOTE_KNL-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero
-; PROMOTE_KNL-NEXT:    vzeroupper
-; PROMOTE_KNL-NEXT:    retq
-;
 ; WIDEN_AVX2-LABEL: test_gather_v2i32_data_index:
 ; WIDEN_AVX2:       # %bb.0:
 ; WIDEN_AVX2-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,2],zero,zero
@@ -362,16 +192,6 @@ define <2 x i32> @test_gather_v2i32_data_index(i32* %base, <2 x i32> %ind, <2 x
 ; WIDEN_AVX2-NEXT:    vpgatherdd %xmm1, (%rdi,%xmm0,4), %xmm2
 ; WIDEN_AVX2-NEXT:    vmovdqa %xmm2, %xmm0
 ; WIDEN_AVX2-NEXT:    retq
-;
-; PROMOTE_AVX2-LABEL: test_gather_v2i32_data_index:
-; PROMOTE_AVX2:       # %bb.0:
-; PROMOTE_AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; PROMOTE_AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
-; PROMOTE_AVX2-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,2],zero,zero
-; PROMOTE_AVX2-NEXT:    vpslld $31, %xmm1, %xmm1
-; PROMOTE_AVX2-NEXT:    vpgatherdd %xmm1, (%rdi,%xmm0,4), %xmm2
-; PROMOTE_AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm2[0],zero,xmm2[1],zero
-; PROMOTE_AVX2-NEXT:    retq
   %gep.random = getelementptr i32, i32* %base, <2 x i32> %ind
   %res = call <2 x i32> @llvm.masked.gather.v2i32.v2p0i32(<2 x i32*> %gep.random, i32 4, <2 x i1> %mask, <2 x i32> %src0)
   ret <2 x i32> %res
@@ -397,27 +217,6 @@ define void @test_scatter_v2i32_data_index(<2 x i32> %a1, i32* %base, <2 x i32>
 ; WIDEN_KNL-NEXT:    vzeroupper
 ; WIDEN_KNL-NEXT:    retq
 ;
-; PROMOTE_SKX-LABEL: test_scatter_v2i32_data_index:
-; PROMOTE_SKX:       # %bb.0:
-; PROMOTE_SKX-NEXT:    vpsllq $63, %xmm2, %xmm2
-; PROMOTE_SKX-NEXT:    vpmovq2m %xmm2, %k1
-; PROMOTE_SKX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; PROMOTE_SKX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; PROMOTE_SKX-NEXT:    vpscatterdd %xmm0, (%rdi,%xmm1,4) {%k1}
-; PROMOTE_SKX-NEXT:    retq
-;
-; PROMOTE_KNL-LABEL: test_scatter_v2i32_data_index:
-; PROMOTE_KNL:       # %bb.0:
-; PROMOTE_KNL-NEXT:    vpsllq $63, %xmm2, %xmm2
-; PROMOTE_KNL-NEXT:    vptestmq %zmm2, %zmm2, %k0
-; PROMOTE_KNL-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; PROMOTE_KNL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; PROMOTE_KNL-NEXT:    kshiftlw $14, %k0, %k0
-; PROMOTE_KNL-NEXT:    kshiftrw $14, %k0, %k1
-; PROMOTE_KNL-NEXT:    vpscatterdd %zmm0, (%rdi,%zmm1,4) {%k1}
-; PROMOTE_KNL-NEXT:    vzeroupper
-; PROMOTE_KNL-NEXT:    retq
-;
 ; WIDEN_AVX2-LABEL: test_scatter_v2i32_data_index:
 ; WIDEN_AVX2:       # %bb.0:
 ; WIDEN_AVX2-NEXT:    vpmovsxdq %xmm1, %xmm1
@@ -425,46 +224,24 @@ define void @test_scatter_v2i32_data_index(<2 x i32> %a1, i32* %base, <2 x i32>
 ; WIDEN_AVX2-NEXT:    vmovq %rdi, %xmm3
 ; WIDEN_AVX2-NEXT:    vpbroadcastq %xmm3, %xmm3
 ; WIDEN_AVX2-NEXT:    vpaddq %xmm1, %xmm3, %xmm1
-; WIDEN_AVX2-NEXT:    vpextrb $0, %xmm2, %eax
-; WIDEN_AVX2-NEXT:    testb $1, %al
-; WIDEN_AVX2-NEXT:    je .LBB5_2
-; WIDEN_AVX2-NEXT:  # %bb.1: # %cond.store
-; WIDEN_AVX2-NEXT:    vmovq %xmm1, %rax
-; WIDEN_AVX2-NEXT:    vmovss %xmm0, (%rax)
-; WIDEN_AVX2-NEXT:  .LBB5_2: # %else
-; WIDEN_AVX2-NEXT:    vpextrb $8, %xmm2, %eax
+; WIDEN_AVX2-NEXT:    vpsllq $63, %xmm2, %xmm2
+; WIDEN_AVX2-NEXT:    vmovmskpd %xmm2, %eax
 ; WIDEN_AVX2-NEXT:    testb $1, %al
+; WIDEN_AVX2-NEXT:    jne .LBB5_1
+; WIDEN_AVX2-NEXT:  # %bb.2: # %else
+; WIDEN_AVX2-NEXT:    testb $2, %al
+; WIDEN_AVX2-NEXT:    jne .LBB5_3
+; WIDEN_AVX2-NEXT:  .LBB5_4: # %else2
+; WIDEN_AVX2-NEXT:    retq
+; WIDEN_AVX2-NEXT:  .LBB5_1: # %cond.store
+; WIDEN_AVX2-NEXT:    vmovq %xmm1, %rcx
+; WIDEN_AVX2-NEXT:    vmovss %xmm0, (%rcx)
+; WIDEN_AVX2-NEXT:    testb $2, %al
 ; WIDEN_AVX2-NEXT:    je .LBB5_4
-; WIDEN_AVX2-NEXT:  # %bb.3: # %cond.store1
+; WIDEN_AVX2-NEXT:  .LBB5_3: # %cond.store1
 ; WIDEN_AVX2-NEXT:    vpextrq $1, %xmm1, %rax
 ; WIDEN_AVX2-NEXT:    vextractps $1, %xmm0, (%rax)
-; WIDEN_AVX2-NEXT:  .LBB5_4: # %else2
 ; WIDEN_AVX2-NEXT:    retq
-;
-; PROMOTE_AVX2-LABEL: test_scatter_v2i32_data_index:
-; PROMOTE_AVX2:       # %bb.0:
-; PROMOTE_AVX2-NEXT:    vpsllq $32, %xmm1, %xmm3
-; PROMOTE_AVX2-NEXT:    vpsrad $31, %xmm3, %xmm3
-; PROMOTE_AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm3[1],xmm1[2],xmm3[3]
-; PROMOTE_AVX2-NEXT:    vpsllq $2, %xmm1, %xmm1
-; PROMOTE_AVX2-NEXT:    vmovq %rdi, %xmm3
-; PROMOTE_AVX2-NEXT:    vpbroadcastq %xmm3, %xmm3
-; PROMOTE_AVX2-NEXT:    vpaddq %xmm1, %xmm3, %xmm1
-; PROMOTE_AVX2-NEXT:    vpextrb $0, %xmm2, %eax
-; PROMOTE_AVX2-NEXT:    testb $1, %al
-; PROMOTE_AVX2-NEXT:    je .LBB5_2
-; PROMOTE_AVX2-NEXT:  # %bb.1: # %cond.store
-; PROMOTE_AVX2-NEXT:    vmovq %xmm1, %rax
-; PROMOTE_AVX2-NEXT:    vmovss %xmm0, (%rax)
-; PROMOTE_AVX2-NEXT:  .LBB5_2: # %else
-; PROMOTE_AVX2-NEXT:    vpextrb $8, %xmm2, %eax
-; PROMOTE_AVX2-NEXT:    testb $1, %al
-; PROMOTE_AVX2-NEXT:    je .LBB5_4
-; PROMOTE_AVX2-NEXT:  # %bb.3: # %cond.store1
-; PROMOTE_AVX2-NEXT:    vpextrq $1, %xmm1, %rax
-; PROMOTE_AVX2-NEXT:    vextractps $2, %xmm0, (%rax)
-; PROMOTE_AVX2-NEXT:  .LBB5_4: # %else2
-; PROMOTE_AVX2-NEXT:    retq
   %gep = getelementptr i32, i32 *%base, <2 x i32> %ind
   call void @llvm.masked.scatter.v2i32.v2p0i32(<2 x i32> %a1, <2 x i32*> %gep, i32 4, <2 x i1> %mask)
   ret void
diff --git a/llvm/test/CodeGen/X86/masked_load.ll b/llvm/test/CodeGen/X86/masked_load.ll
index 17f8f0a1b..cecdcec01 100644
--- a/llvm/test/CodeGen/X86/masked_load.ll
+++ b/llvm/test/CodeGen/X86/masked_load.ll
@@ -41,18 +41,21 @@ define <2 x double> @load_v2f64_v2i64(<2 x i64> %trigger, <2 x double>* %addr, <
 ; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,0,3,2]
 ; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    movmskpd %xmm0, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB1_2
-; SSE2-NEXT:  ## %bb.1: ## %cond.load
+; SSE2-NEXT:    jne LBB1_1
+; SSE2-NEXT:  ## %bb.2: ## %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne LBB1_3
+; SSE2-NEXT:  LBB1_4: ## %else2
+; SSE2-NEXT:    movaps %xmm1, %xmm0
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  LBB1_1: ## %cond.load
 ; SSE2-NEXT:    movlps {{.*#+}} xmm1 = mem[0,1],xmm1[2,3]
-; SSE2-NEXT:  LBB1_2: ## %else
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $2, %al
 ; SSE2-NEXT:    je LBB1_4
-; SSE2-NEXT:  ## %bb.3: ## %cond.load1
+; SSE2-NEXT:  LBB1_3: ## %cond.load1
 ; SSE2-NEXT:    movhps {{.*#+}} xmm1 = xmm1[0,1],mem[0,1]
-; SSE2-NEXT:  LBB1_4: ## %else2
 ; SSE2-NEXT:    movaps %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
@@ -60,18 +63,21 @@ define <2 x double> @load_v2f64_v2i64(<2 x i64> %trigger, <2 x double>* %addr, <
 ; SSE42:       ## %bb.0:
 ; SSE42-NEXT:    pxor %xmm2, %xmm2
 ; SSE42-NEXT:    pcmpeqq %xmm0, %xmm2
-; SSE42-NEXT:    pextrb $0, %xmm2, %eax
+; SSE42-NEXT:    movmskpd %xmm2, %eax
 ; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB1_2
-; SSE42-NEXT:  ## %bb.1: ## %cond.load
+; SSE42-NEXT:    jne LBB1_1
+; SSE42-NEXT:  ## %bb.2: ## %else
+; SSE42-NEXT:    testb $2, %al
+; SSE42-NEXT:    jne LBB1_3
+; SSE42-NEXT:  LBB1_4: ## %else2
+; SSE42-NEXT:    movaps %xmm1, %xmm0
+; SSE42-NEXT:    retq
+; SSE42-NEXT:  LBB1_1: ## %cond.load
 ; SSE42-NEXT:    movlps {{.*#+}} xmm1 = mem[0,1],xmm1[2,3]
-; SSE42-NEXT:  LBB1_2: ## %else
-; SSE42-NEXT:    pextrb $8, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $2, %al
 ; SSE42-NEXT:    je LBB1_4
-; SSE42-NEXT:  ## %bb.3: ## %cond.load1
+; SSE42-NEXT:  LBB1_3: ## %cond.load1
 ; SSE42-NEXT:    movhps {{.*#+}} xmm1 = xmm1[0,1],mem[0,1]
-; SSE42-NEXT:  LBB1_4: ## %else2
 ; SSE42-NEXT:    movaps %xmm1, %xmm0
 ; SSE42-NEXT:    retq
 ;
@@ -106,73 +112,41 @@ define <2 x double> @load_v2f64_v2i64(<2 x i64> %trigger, <2 x double>* %addr, <
 }
 
 define <4 x double> @load_v4f64_v4i32(<4 x i32> %trigger, <4 x double>* %addr, <4 x double> %dst) {
-; SSE2-LABEL: load_v4f64_v4i32:
-; SSE2:       ## %bb.0:
-; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm3
-; SSE2-NEXT:    movd %xmm3, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB2_2
-; SSE2-NEXT:  ## %bb.1: ## %cond.load
-; SSE2-NEXT:    movlps {{.*#+}} xmm1 = mem[0,1],xmm1[2,3]
-; SSE2-NEXT:  LBB2_2: ## %else
-; SSE2-NEXT:    pextrw $2, %xmm3, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB2_4
-; SSE2-NEXT:  ## %bb.3: ## %cond.load1
-; SSE2-NEXT:    movhps {{.*#+}} xmm1 = xmm1[0,1],mem[0,1]
-; SSE2-NEXT:  LBB2_4: ## %else2
-; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm0
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB2_6
-; SSE2-NEXT:  ## %bb.5: ## %cond.load4
-; SSE2-NEXT:    movlps {{.*#+}} xmm2 = mem[0,1],xmm2[2,3]
-; SSE2-NEXT:  LBB2_6: ## %else5
-; SSE2-NEXT:    pextrw $6, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB2_8
-; SSE2-NEXT:  ## %bb.7: ## %cond.load7
-; SSE2-NEXT:    movhps {{.*#+}} xmm2 = xmm2[0,1],mem[0,1]
-; SSE2-NEXT:  LBB2_8: ## %else8
-; SSE2-NEXT:    movaps %xmm1, %xmm0
-; SSE2-NEXT:    movaps %xmm2, %xmm1
-; SSE2-NEXT:    retq
-;
-; SSE42-LABEL: load_v4f64_v4i32:
-; SSE42:       ## %bb.0:
-; SSE42-NEXT:    pxor %xmm3, %xmm3
-; SSE42-NEXT:    pcmpeqd %xmm0, %xmm3
-; SSE42-NEXT:    pextrb $0, %xmm3, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB2_2
-; SSE42-NEXT:  ## %bb.1: ## %cond.load
-; SSE42-NEXT:    movlps {{.*#+}} xmm1 = mem[0,1],xmm1[2,3]
-; SSE42-NEXT:  LBB2_2: ## %else
-; SSE42-NEXT:    pextrb $4, %xmm3, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB2_4
-; SSE42-NEXT:  ## %bb.3: ## %cond.load1
-; SSE42-NEXT:    movhps {{.*#+}} xmm1 = xmm1[0,1],mem[0,1]
-; SSE42-NEXT:  LBB2_4: ## %else2
-; SSE42-NEXT:    pxor %xmm3, %xmm3
-; SSE42-NEXT:    pcmpeqd %xmm3, %xmm0
-; SSE42-NEXT:    pextrb $8, %xmm0, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB2_6
-; SSE42-NEXT:  ## %bb.5: ## %cond.load4
-; SSE42-NEXT:    movlps {{.*#+}} xmm2 = mem[0,1],xmm2[2,3]
-; SSE42-NEXT:  LBB2_6: ## %else5
-; SSE42-NEXT:    pextrb $12, %xmm0, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB2_8
-; SSE42-NEXT:  ## %bb.7: ## %cond.load7
-; SSE42-NEXT:    movhps {{.*#+}} xmm2 = xmm2[0,1],mem[0,1]
-; SSE42-NEXT:  LBB2_8: ## %else8
-; SSE42-NEXT:    movaps %xmm1, %xmm0
-; SSE42-NEXT:    movaps %xmm2, %xmm1
-; SSE42-NEXT:    retq
+; SSE-LABEL: load_v4f64_v4i32:
+; SSE:       ## %bb.0:
+; SSE-NEXT:    pxor %xmm3, %xmm3
+; SSE-NEXT:    pcmpeqd %xmm0, %xmm3
+; SSE-NEXT:    movmskps %xmm3, %eax
+; SSE-NEXT:    testb $1, %al
+; SSE-NEXT:    jne LBB2_1
+; SSE-NEXT:  ## %bb.2: ## %else
+; SSE-NEXT:    testb $2, %al
+; SSE-NEXT:    jne LBB2_3
+; SSE-NEXT:  LBB2_4: ## %else2
+; SSE-NEXT:    testb $4, %al
+; SSE-NEXT:    jne LBB2_5
+; SSE-NEXT:  LBB2_6: ## %else5
+; SSE-NEXT:    testb $8, %al
+; SSE-NEXT:    je LBB2_8
+; SSE-NEXT:  LBB2_7: ## %cond.load7
+; SSE-NEXT:    movhps {{.*#+}} xmm2 = xmm2[0,1],mem[0,1]
+; SSE-NEXT:  LBB2_8: ## %else8
+; SSE-NEXT:    movaps %xmm1, %xmm0
+; SSE-NEXT:    movaps %xmm2, %xmm1
+; SSE-NEXT:    retq
+; SSE-NEXT:  LBB2_1: ## %cond.load
+; SSE-NEXT:    movlps {{.*#+}} xmm1 = mem[0,1],xmm1[2,3]
+; SSE-NEXT:    testb $2, %al
+; SSE-NEXT:    je LBB2_4
+; SSE-NEXT:  LBB2_3: ## %cond.load1
+; SSE-NEXT:    movhps {{.*#+}} xmm1 = xmm1[0,1],mem[0,1]
+; SSE-NEXT:    testb $4, %al
+; SSE-NEXT:    je LBB2_6
+; SSE-NEXT:  LBB2_5: ## %cond.load4
+; SSE-NEXT:    movlps {{.*#+}} xmm2 = mem[0,1],xmm2[2,3]
+; SSE-NEXT:    testb $8, %al
+; SSE-NEXT:    jne LBB2_7
+; SSE-NEXT:    jmp LBB2_8
 ;
 ; AVX1-LABEL: load_v4f64_v4i32:
 ; AVX1:       ## %bb.0:
@@ -217,77 +191,42 @@ define <4 x double> @load_v4f64_v4i32(<4 x i32> %trigger, <4 x double>* %addr, <
 }
 
 define <4 x double> @load_v4f64_v4i32_zero(<4 x i32> %trigger, <4 x double>* %addr) {
-; SSE2-LABEL: load_v4f64_v4i32_zero:
-; SSE2:       ## %bb.0:
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    movdqa %xmm2, %xmm3
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm3
-; SSE2-NEXT:    movd %xmm3, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    xorps %xmm1, %xmm1
-; SSE2-NEXT:    je LBB3_2
-; SSE2-NEXT:  ## %bb.1: ## %cond.load
-; SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; SSE2-NEXT:    xorps %xmm1, %xmm1
-; SSE2-NEXT:  LBB3_2: ## %else
-; SSE2-NEXT:    pextrw $2, %xmm3, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB3_4
-; SSE2-NEXT:  ## %bb.3: ## %cond.load1
-; SSE2-NEXT:    movhps {{.*#+}} xmm0 = xmm0[0,1],mem[0,1]
-; SSE2-NEXT:  LBB3_4: ## %else2
-; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm2
-; SSE2-NEXT:    pextrw $4, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB3_6
-; SSE2-NEXT:  ## %bb.5: ## %cond.load4
-; SSE2-NEXT:    movlps {{.*#+}} xmm1 = mem[0,1],xmm1[2,3]
-; SSE2-NEXT:  LBB3_6: ## %else5
-; SSE2-NEXT:    pextrw $6, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB3_8
-; SSE2-NEXT:  ## %bb.7: ## %cond.load7
-; SSE2-NEXT:    movhps {{.*#+}} xmm1 = xmm1[0,1],mem[0,1]
-; SSE2-NEXT:  LBB3_8: ## %else8
-; SSE2-NEXT:    retq
-;
-; SSE42-LABEL: load_v4f64_v4i32_zero:
-; SSE42:       ## %bb.0:
-; SSE42-NEXT:    movdqa %xmm0, %xmm2
-; SSE42-NEXT:    pxor %xmm0, %xmm0
-; SSE42-NEXT:    movdqa %xmm2, %xmm3
-; SSE42-NEXT:    pcmpeqd %xmm0, %xmm3
-; SSE42-NEXT:    pextrb $0, %xmm3, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    xorps %xmm1, %xmm1
-; SSE42-NEXT:    je LBB3_2
-; SSE42-NEXT:  ## %bb.1: ## %cond.load
-; SSE42-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; SSE42-NEXT:    xorps %xmm1, %xmm1
-; SSE42-NEXT:  LBB3_2: ## %else
-; SSE42-NEXT:    pextrb $4, %xmm3, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB3_4
-; SSE42-NEXT:  ## %bb.3: ## %cond.load1
-; SSE42-NEXT:    movhps {{.*#+}} xmm0 = xmm0[0,1],mem[0,1]
-; SSE42-NEXT:  LBB3_4: ## %else2
-; SSE42-NEXT:    pxor %xmm3, %xmm3
-; SSE42-NEXT:    pcmpeqd %xmm3, %xmm2
-; SSE42-NEXT:    pextrb $8, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB3_6
-; SSE42-NEXT:  ## %bb.5: ## %cond.load4
-; SSE42-NEXT:    movlps {{.*#+}} xmm1 = mem[0,1],xmm1[2,3]
-; SSE42-NEXT:  LBB3_6: ## %else5
-; SSE42-NEXT:    pextrb $12, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB3_8
-; SSE42-NEXT:  ## %bb.7: ## %cond.load7
-; SSE42-NEXT:    movhps {{.*#+}} xmm1 = xmm1[0,1],mem[0,1]
-; SSE42-NEXT:  LBB3_8: ## %else8
-; SSE42-NEXT:    retq
+; SSE-LABEL: load_v4f64_v4i32_zero:
+; SSE:       ## %bb.0:
+; SSE-NEXT:    movdqa %xmm0, %xmm1
+; SSE-NEXT:    pxor %xmm0, %xmm0
+; SSE-NEXT:    pcmpeqd %xmm0, %xmm1
+; SSE-NEXT:    movmskps %xmm1, %eax
+; SSE-NEXT:    testb $1, %al
+; SSE-NEXT:    xorps %xmm1, %xmm1
+; SSE-NEXT:    jne LBB3_1
+; SSE-NEXT:  ## %bb.2: ## %else
+; SSE-NEXT:    testb $2, %al
+; SSE-NEXT:    jne LBB3_3
+; SSE-NEXT:  LBB3_4: ## %else2
+; SSE-NEXT:    testb $4, %al
+; SSE-NEXT:    jne LBB3_5
+; SSE-NEXT:  LBB3_6: ## %else5
+; SSE-NEXT:    testb $8, %al
+; SSE-NEXT:    jne LBB3_7
+; SSE-NEXT:  LBB3_8: ## %else8
+; SSE-NEXT:    retq
+; SSE-NEXT:  LBB3_1: ## %cond.load
+; SSE-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; SSE-NEXT:    xorps %xmm1, %xmm1
+; SSE-NEXT:    testb $2, %al
+; SSE-NEXT:    je LBB3_4
+; SSE-NEXT:  LBB3_3: ## %cond.load1
+; SSE-NEXT:    movhps {{.*#+}} xmm0 = xmm0[0,1],mem[0,1]
+; SSE-NEXT:    testb $4, %al
+; SSE-NEXT:    je LBB3_6
+; SSE-NEXT:  LBB3_5: ## %cond.load4
+; SSE-NEXT:    movlps {{.*#+}} xmm1 = mem[0,1],xmm1[2,3]
+; SSE-NEXT:    testb $8, %al
+; SSE-NEXT:    je LBB3_8
+; SSE-NEXT:  LBB3_7: ## %cond.load7
+; SSE-NEXT:    movhps {{.*#+}} xmm1 = xmm1[0,1],mem[0,1]
+; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: load_v4f64_v4i32_zero:
 ; AVX1:       ## %bb.0:
@@ -332,74 +271,82 @@ define <4 x double> @load_v4f64_v4i64(<4 x i64> %trigger, <4 x double>* %addr, <
 ; SSE2-LABEL: load_v4f64_v4i64:
 ; SSE2:       ## %bb.0:
 ; SSE2-NEXT:    pxor %xmm4, %xmm4
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[1,0,3,2]
-; SSE2-NEXT:    pand %xmm4, %xmm0
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB4_2
-; SSE2-NEXT:  ## %bb.1: ## %cond.load
-; SSE2-NEXT:    movlps {{.*#+}} xmm2 = mem[0,1],xmm2[2,3]
-; SSE2-NEXT:  LBB4_2: ## %else
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
+; SSE2-NEXT:    pcmpeqd %xmm4, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[1,0,3,2]
+; SSE2-NEXT:    pand %xmm1, %xmm5
+; SSE2-NEXT:    pcmpeqd %xmm4, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,0,3,2]
+; SSE2-NEXT:    pand %xmm0, %xmm1
+; SSE2-NEXT:    packssdw %xmm5, %xmm1
+; SSE2-NEXT:    movmskps %xmm1, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB4_4
-; SSE2-NEXT:  ## %bb.3: ## %cond.load1
-; SSE2-NEXT:    movhps {{.*#+}} xmm2 = xmm2[0,1],mem[0,1]
+; SSE2-NEXT:    jne LBB4_1
+; SSE2-NEXT:  ## %bb.2: ## %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne LBB4_3
 ; SSE2-NEXT:  LBB4_4: ## %else2
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,0,3,2]
-; SSE2-NEXT:    pand %xmm1, %xmm0
-; SSE2-NEXT:    pextrw $0, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB4_6
-; SSE2-NEXT:  ## %bb.5: ## %cond.load4
-; SSE2-NEXT:    movlps {{.*#+}} xmm3 = mem[0,1],xmm3[2,3]
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne LBB4_5
 ; SSE2-NEXT:  LBB4_6: ## %else5
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je LBB4_8
-; SSE2-NEXT:  ## %bb.7: ## %cond.load7
+; SSE2-NEXT:  LBB4_7: ## %cond.load7
 ; SSE2-NEXT:    movhps {{.*#+}} xmm3 = xmm3[0,1],mem[0,1]
 ; SSE2-NEXT:  LBB4_8: ## %else8
 ; SSE2-NEXT:    movaps %xmm2, %xmm0
 ; SSE2-NEXT:    movaps %xmm3, %xmm1
 ; SSE2-NEXT:    retq
+; SSE2-NEXT:  LBB4_1: ## %cond.load
+; SSE2-NEXT:    movlps {{.*#+}} xmm2 = mem[0,1],xmm2[2,3]
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    je LBB4_4
+; SSE2-NEXT:  LBB4_3: ## %cond.load1
+; SSE2-NEXT:    movhps {{.*#+}} xmm2 = xmm2[0,1],mem[0,1]
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    je LBB4_6
+; SSE2-NEXT:  LBB4_5: ## %cond.load4
+; SSE2-NEXT:    movlps {{.*#+}} xmm3 = mem[0,1],xmm3[2,3]
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne LBB4_7
+; SSE2-NEXT:    jmp LBB4_8
 ;
 ; SSE42-LABEL: load_v4f64_v4i64:
 ; SSE42:       ## %bb.0:
 ; SSE42-NEXT:    pxor %xmm4, %xmm4
-; SSE42-NEXT:    pcmpeqq %xmm0, %xmm4
-; SSE42-NEXT:    pextrb $0, %xmm4, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB4_2
-; SSE42-NEXT:  ## %bb.1: ## %cond.load
-; SSE42-NEXT:    movlps {{.*#+}} xmm2 = mem[0,1],xmm2[2,3]
-; SSE42-NEXT:  LBB4_2: ## %else
-; SSE42-NEXT:    pextrb $8, %xmm4, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB4_4
-; SSE42-NEXT:  ## %bb.3: ## %cond.load1
-; SSE42-NEXT:    movhps {{.*#+}} xmm2 = xmm2[0,1],mem[0,1]
+; SSE42-NEXT:    pcmpeqq %xmm4, %xmm1
+; SSE42-NEXT:    pcmpeqq %xmm4, %xmm0
+; SSE42-NEXT:    packssdw %xmm1, %xmm0
+; SSE42-NEXT:    movmskps %xmm0, %eax
+; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    jne LBB4_1
+; SSE42-NEXT:  ## %bb.2: ## %else
+; SSE42-NEXT:    testb $2, %al
+; SSE42-NEXT:    jne LBB4_3
 ; SSE42-NEXT:  LBB4_4: ## %else2
-; SSE42-NEXT:    pxor %xmm0, %xmm0
-; SSE42-NEXT:    pcmpeqq %xmm0, %xmm1
-; SSE42-NEXT:    pextrb $0, %xmm1, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB4_6
-; SSE42-NEXT:  ## %bb.5: ## %cond.load4
-; SSE42-NEXT:    movlps {{.*#+}} xmm3 = mem[0,1],xmm3[2,3]
+; SSE42-NEXT:    testb $4, %al
+; SSE42-NEXT:    jne LBB4_5
 ; SSE42-NEXT:  LBB4_6: ## %else5
-; SSE42-NEXT:    pextrb $8, %xmm1, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $8, %al
 ; SSE42-NEXT:    je LBB4_8
-; SSE42-NEXT:  ## %bb.7: ## %cond.load7
+; SSE42-NEXT:  LBB4_7: ## %cond.load7
 ; SSE42-NEXT:    movhps {{.*#+}} xmm3 = xmm3[0,1],mem[0,1]
 ; SSE42-NEXT:  LBB4_8: ## %else8
 ; SSE42-NEXT:    movaps %xmm2, %xmm0
 ; SSE42-NEXT:    movaps %xmm3, %xmm1
 ; SSE42-NEXT:    retq
+; SSE42-NEXT:  LBB4_1: ## %cond.load
+; SSE42-NEXT:    movlps {{.*#+}} xmm2 = mem[0,1],xmm2[2,3]
+; SSE42-NEXT:    testb $2, %al
+; SSE42-NEXT:    je LBB4_4
+; SSE42-NEXT:  LBB4_3: ## %cond.load1
+; SSE42-NEXT:    movhps {{.*#+}} xmm2 = xmm2[0,1],mem[0,1]
+; SSE42-NEXT:    testb $4, %al
+; SSE42-NEXT:    je LBB4_6
+; SSE42-NEXT:  LBB4_5: ## %cond.load4
+; SSE42-NEXT:    movlps {{.*#+}} xmm3 = mem[0,1],xmm3[2,3]
+; SSE42-NEXT:    testb $8, %al
+; SSE42-NEXT:    jne LBB4_7
+; SSE42-NEXT:    jmp LBB4_8
 ;
 ; AVX1-LABEL: load_v4f64_v4i64:
 ; AVX1:       ## %bb.0:
@@ -442,168 +389,107 @@ define <4 x double> @load_v4f64_v4i64(<4 x i64> %trigger, <4 x double>* %addr, <
 }
 
 define <8 x double> @load_v8f64_v8i16(<8 x i16> %trigger, <8 x double>* %addr, <8 x double> %dst) {
-; SSE2-LABEL: load_v8f64_v8i16:
-; SSE2:       ## %bb.0:
-; SSE2-NEXT:    pxor %xmm5, %xmm5
-; SSE2-NEXT:    pcmpeqw %xmm0, %xmm5
-; SSE2-NEXT:    movd %xmm5, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB5_2
-; SSE2-NEXT:  ## %bb.1: ## %cond.load
-; SSE2-NEXT:    movlps {{.*#+}} xmm1 = mem[0,1],xmm1[2,3]
-; SSE2-NEXT:  LBB5_2: ## %else
-; SSE2-NEXT:    shrl $16, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB5_4
-; SSE2-NEXT:  ## %bb.3: ## %cond.load1
-; SSE2-NEXT:    movhps {{.*#+}} xmm1 = xmm1[0,1],mem[0,1]
-; SSE2-NEXT:  LBB5_4: ## %else2
-; SSE2-NEXT:    pxor %xmm5, %xmm5
-; SSE2-NEXT:    pcmpeqw %xmm0, %xmm5
-; SSE2-NEXT:    pextrw $2, %xmm5, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB5_6
-; SSE2-NEXT:  ## %bb.5: ## %cond.load4
-; SSE2-NEXT:    movlps {{.*#+}} xmm2 = mem[0,1],xmm2[2,3]
-; SSE2-NEXT:  LBB5_6: ## %else5
-; SSE2-NEXT:    pextrw $3, %xmm5, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB5_8
-; SSE2-NEXT:  ## %bb.7: ## %cond.load7
-; SSE2-NEXT:    movhps {{.*#+}} xmm2 = xmm2[0,1],mem[0,1]
-; SSE2-NEXT:  LBB5_8: ## %else8
-; SSE2-NEXT:    pxor %xmm5, %xmm5
-; SSE2-NEXT:    pcmpeqw %xmm0, %xmm5
-; SSE2-NEXT:    pextrw $4, %xmm5, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB5_10
-; SSE2-NEXT:  ## %bb.9: ## %cond.load10
-; SSE2-NEXT:    movlps {{.*#+}} xmm3 = mem[0,1],xmm3[2,3]
-; SSE2-NEXT:  LBB5_10: ## %else11
-; SSE2-NEXT:    pextrw $5, %xmm5, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB5_12
-; SSE2-NEXT:  ## %bb.11: ## %cond.load13
-; SSE2-NEXT:    movhps {{.*#+}} xmm3 = xmm3[0,1],mem[0,1]
-; SSE2-NEXT:  LBB5_12: ## %else14
-; SSE2-NEXT:    pxor %xmm5, %xmm5
-; SSE2-NEXT:    pcmpeqw %xmm5, %xmm0
-; SSE2-NEXT:    pextrw $6, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB5_14
-; SSE2-NEXT:  ## %bb.13: ## %cond.load16
-; SSE2-NEXT:    movlps {{.*#+}} xmm4 = mem[0,1],xmm4[2,3]
-; SSE2-NEXT:  LBB5_14: ## %else17
-; SSE2-NEXT:    pextrw $7, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB5_16
-; SSE2-NEXT:  ## %bb.15: ## %cond.load19
-; SSE2-NEXT:    movhps {{.*#+}} xmm4 = xmm4[0,1],mem[0,1]
-; SSE2-NEXT:  LBB5_16: ## %else20
-; SSE2-NEXT:    movaps %xmm1, %xmm0
-; SSE2-NEXT:    movaps %xmm2, %xmm1
-; SSE2-NEXT:    movaps %xmm3, %xmm2
-; SSE2-NEXT:    movaps %xmm4, %xmm3
-; SSE2-NEXT:    retq
-;
-; SSE42-LABEL: load_v8f64_v8i16:
-; SSE42:       ## %bb.0:
-; SSE42-NEXT:    pxor %xmm5, %xmm5
-; SSE42-NEXT:    pcmpeqw %xmm0, %xmm5
-; SSE42-NEXT:    pextrb $0, %xmm5, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB5_2
-; SSE42-NEXT:  ## %bb.1: ## %cond.load
-; SSE42-NEXT:    movlps {{.*#+}} xmm1 = mem[0,1],xmm1[2,3]
-; SSE42-NEXT:  LBB5_2: ## %else
-; SSE42-NEXT:    pextrb $2, %xmm5, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB5_4
-; SSE42-NEXT:  ## %bb.3: ## %cond.load1
-; SSE42-NEXT:    movhps {{.*#+}} xmm1 = xmm1[0,1],mem[0,1]
-; SSE42-NEXT:  LBB5_4: ## %else2
-; SSE42-NEXT:    pxor %xmm5, %xmm5
-; SSE42-NEXT:    pcmpeqw %xmm0, %xmm5
-; SSE42-NEXT:    pextrb $4, %xmm5, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB5_6
-; SSE42-NEXT:  ## %bb.5: ## %cond.load4
-; SSE42-NEXT:    movlps {{.*#+}} xmm2 = mem[0,1],xmm2[2,3]
-; SSE42-NEXT:  LBB5_6: ## %else5
-; SSE42-NEXT:    pextrb $6, %xmm5, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB5_8
-; SSE42-NEXT:  ## %bb.7: ## %cond.load7
-; SSE42-NEXT:    movhps {{.*#+}} xmm2 = xmm2[0,1],mem[0,1]
-; SSE42-NEXT:  LBB5_8: ## %else8
-; SSE42-NEXT:    pxor %xmm5, %xmm5
-; SSE42-NEXT:    pcmpeqw %xmm0, %xmm5
-; SSE42-NEXT:    pextrb $8, %xmm5, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB5_10
-; SSE42-NEXT:  ## %bb.9: ## %cond.load10
-; SSE42-NEXT:    movlps {{.*#+}} xmm3 = mem[0,1],xmm3[2,3]
-; SSE42-NEXT:  LBB5_10: ## %else11
-; SSE42-NEXT:    pextrb $10, %xmm5, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB5_12
-; SSE42-NEXT:  ## %bb.11: ## %cond.load13
-; SSE42-NEXT:    movhps {{.*#+}} xmm3 = xmm3[0,1],mem[0,1]
-; SSE42-NEXT:  LBB5_12: ## %else14
-; SSE42-NEXT:    pxor %xmm5, %xmm5
-; SSE42-NEXT:    pcmpeqw %xmm5, %xmm0
-; SSE42-NEXT:    pextrb $12, %xmm0, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB5_14
-; SSE42-NEXT:  ## %bb.13: ## %cond.load16
-; SSE42-NEXT:    movlps {{.*#+}} xmm4 = mem[0,1],xmm4[2,3]
-; SSE42-NEXT:  LBB5_14: ## %else17
-; SSE42-NEXT:    pextrb $14, %xmm0, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB5_16
-; SSE42-NEXT:  ## %bb.15: ## %cond.load19
-; SSE42-NEXT:    movhps {{.*#+}} xmm4 = xmm4[0,1],mem[0,1]
-; SSE42-NEXT:  LBB5_16: ## %else20
-; SSE42-NEXT:    movaps %xmm1, %xmm0
-; SSE42-NEXT:    movaps %xmm2, %xmm1
-; SSE42-NEXT:    movaps %xmm3, %xmm2
-; SSE42-NEXT:    movaps %xmm4, %xmm3
-; SSE42-NEXT:    retq
+; SSE-LABEL: load_v8f64_v8i16:
+; SSE:       ## %bb.0:
+; SSE-NEXT:    pxor %xmm5, %xmm5
+; SSE-NEXT:    pcmpeqw %xmm0, %xmm5
+; SSE-NEXT:    packsswb %xmm0, %xmm5
+; SSE-NEXT:    pmovmskb %xmm5, %eax
+; SSE-NEXT:    testb $1, %al
+; SSE-NEXT:    jne LBB5_1
+; SSE-NEXT:  ## %bb.2: ## %else
+; SSE-NEXT:    testb $2, %al
+; SSE-NEXT:    jne LBB5_3
+; SSE-NEXT:  LBB5_4: ## %else2
+; SSE-NEXT:    testb $4, %al
+; SSE-NEXT:    jne LBB5_5
+; SSE-NEXT:  LBB5_6: ## %else5
+; SSE-NEXT:    testb $8, %al
+; SSE-NEXT:    jne LBB5_7
+; SSE-NEXT:  LBB5_8: ## %else8
+; SSE-NEXT:    testb $16, %al
+; SSE-NEXT:    jne LBB5_9
+; SSE-NEXT:  LBB5_10: ## %else11
+; SSE-NEXT:    testb $32, %al
+; SSE-NEXT:    jne LBB5_11
+; SSE-NEXT:  LBB5_12: ## %else14
+; SSE-NEXT:    testb $64, %al
+; SSE-NEXT:    jne LBB5_13
+; SSE-NEXT:  LBB5_14: ## %else17
+; SSE-NEXT:    testb $-128, %al
+; SSE-NEXT:    je LBB5_16
+; SSE-NEXT:  LBB5_15: ## %cond.load19
+; SSE-NEXT:    movhps {{.*#+}} xmm4 = xmm4[0,1],mem[0,1]
+; SSE-NEXT:  LBB5_16: ## %else20
+; SSE-NEXT:    movaps %xmm1, %xmm0
+; SSE-NEXT:    movaps %xmm2, %xmm1
+; SSE-NEXT:    movaps %xmm3, %xmm2
+; SSE-NEXT:    movaps %xmm4, %xmm3
+; SSE-NEXT:    retq
+; SSE-NEXT:  LBB5_1: ## %cond.load
+; SSE-NEXT:    movlps {{.*#+}} xmm1 = mem[0,1],xmm1[2,3]
+; SSE-NEXT:    testb $2, %al
+; SSE-NEXT:    je LBB5_4
+; SSE-NEXT:  LBB5_3: ## %cond.load1
+; SSE-NEXT:    movhps {{.*#+}} xmm1 = xmm1[0,1],mem[0,1]
+; SSE-NEXT:    testb $4, %al
+; SSE-NEXT:    je LBB5_6
+; SSE-NEXT:  LBB5_5: ## %cond.load4
+; SSE-NEXT:    movlps {{.*#+}} xmm2 = mem[0,1],xmm2[2,3]
+; SSE-NEXT:    testb $8, %al
+; SSE-NEXT:    je LBB5_8
+; SSE-NEXT:  LBB5_7: ## %cond.load7
+; SSE-NEXT:    movhps {{.*#+}} xmm2 = xmm2[0,1],mem[0,1]
+; SSE-NEXT:    testb $16, %al
+; SSE-NEXT:    je LBB5_10
+; SSE-NEXT:  LBB5_9: ## %cond.load10
+; SSE-NEXT:    movlps {{.*#+}} xmm3 = mem[0,1],xmm3[2,3]
+; SSE-NEXT:    testb $32, %al
+; SSE-NEXT:    je LBB5_12
+; SSE-NEXT:  LBB5_11: ## %cond.load13
+; SSE-NEXT:    movhps {{.*#+}} xmm3 = xmm3[0,1],mem[0,1]
+; SSE-NEXT:    testb $64, %al
+; SSE-NEXT:    je LBB5_14
+; SSE-NEXT:  LBB5_13: ## %cond.load16
+; SSE-NEXT:    movlps {{.*#+}} xmm4 = mem[0,1],xmm4[2,3]
+; SSE-NEXT:    testb $-128, %al
+; SSE-NEXT:    jne LBB5_15
+; SSE-NEXT:    jmp LBB5_16
 ;
 ; AVX1-LABEL: load_v8f64_v8i16:
 ; AVX1:       ## %bb.0:
-; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7]
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm4, %xmm4
-; AVX1-NEXT:    vpmovsxdq %xmm4, %xmm5
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[2,3,0,1]
-; AVX1-NEXT:    vpmovsxdq %xmm4, %xmm4
-; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm5, %ymm4
-; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm0, %xmm0
-; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm3
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm3, %ymm0
-; AVX1-NEXT:    vmaskmovpd (%rdi), %ymm0, %ymm3
-; AVX1-NEXT:    vblendvpd %ymm0, %ymm3, %ymm1, %ymm0
-; AVX1-NEXT:    vmaskmovpd 32(%rdi), %ymm4, %ymm1
-; AVX1-NEXT:    vblendvpd %ymm4, %ymm1, %ymm2, %ymm1
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; AVX1-NEXT:    vpcmpeqw %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpmovsxwd %xmm3, %xmm3
+; AVX1-NEXT:    vpmovsxdq %xmm3, %xmm5
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,3,0,1]
+; AVX1-NEXT:    vpmovsxdq %xmm3, %xmm3
+; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm5, %ymm3
+; AVX1-NEXT:    vpcmpeqw %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vpmovsxwq %xmm0, %xmm4
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpmovsxwq %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm4, %ymm0
+; AVX1-NEXT:    vmaskmovpd (%rdi), %ymm0, %ymm4
+; AVX1-NEXT:    vblendvpd %ymm0, %ymm4, %ymm1, %ymm0
+; AVX1-NEXT:    vmaskmovpd 32(%rdi), %ymm3, %ymm1
+; AVX1-NEXT:    vblendvpd %ymm3, %ymm1, %ymm2, %ymm1
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: load_v8f64_v8i16:
 ; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7]
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm4, %xmm4
-; AVX2-NEXT:    vpmovsxdq %xmm4, %ymm4
-; AVX2-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm0, %xmm0
-; AVX2-NEXT:    vpmovsxdq %xmm0, %ymm0
-; AVX2-NEXT:    vmaskmovpd (%rdi), %ymm0, %ymm3
-; AVX2-NEXT:    vblendvpd %ymm0, %ymm3, %ymm1, %ymm0
-; AVX2-NEXT:    vmaskmovpd 32(%rdi), %ymm4, %ymm1
-; AVX2-NEXT:    vblendvpd %ymm4, %ymm1, %ymm2, %ymm1
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; AVX2-NEXT:    vpcmpeqw %xmm4, %xmm3, %xmm3
+; AVX2-NEXT:    vpmovsxwd %xmm3, %xmm3
+; AVX2-NEXT:    vpmovsxdq %xmm3, %ymm3
+; AVX2-NEXT:    vpcmpeqw %xmm4, %xmm0, %xmm0
+; AVX2-NEXT:    vpmovsxwq %xmm0, %ymm0
+; AVX2-NEXT:    vmaskmovpd (%rdi), %ymm0, %ymm4
+; AVX2-NEXT:    vblendvpd %ymm0, %ymm4, %ymm1, %ymm0
+; AVX2-NEXT:    vmaskmovpd 32(%rdi), %ymm3, %ymm1
+; AVX2-NEXT:    vblendvpd %ymm3, %ymm1, %ymm2, %ymm1
 ; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: load_v8f64_v8i16:
@@ -638,135 +524,123 @@ define <8 x double> @load_v8f64_v8i64(<8 x i64> %trigger, <8 x double>* %addr, <
 ; SSE2-LABEL: load_v8f64_v8i64:
 ; SSE2:       ## %bb.0:
 ; SSE2-NEXT:    movdqa %xmm7, %xmm8
+; SSE2-NEXT:    movaps %xmm6, %xmm9
 ; SSE2-NEXT:    pxor %xmm7, %xmm7
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[1,0,3,2]
-; SSE2-NEXT:    pand %xmm7, %xmm0
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB6_2
-; SSE2-NEXT:  ## %bb.1: ## %cond.load
-; SSE2-NEXT:    movlps {{.*#+}} xmm4 = mem[0,1],xmm4[2,3]
-; SSE2-NEXT:  LBB6_2: ## %else
-; SSE2-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    shrl $16, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB6_4
-; SSE2-NEXT:  ## %bb.3: ## %cond.load1
-; SSE2-NEXT:    movhps {{.*#+}} xmm4 = xmm4[0,1],mem[0,1]
+; SSE2-NEXT:    pcmpeqd %xmm7, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[1,0,3,2]
+; SSE2-NEXT:    pand %xmm3, %xmm6
+; SSE2-NEXT:    pcmpeqd %xmm7, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,0,3,2]
+; SSE2-NEXT:    pand %xmm2, %xmm3
+; SSE2-NEXT:    packssdw %xmm6, %xmm3
+; SSE2-NEXT:    pcmpeqd %xmm7, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,0,3,2]
+; SSE2-NEXT:    pand %xmm1, %xmm2
+; SSE2-NEXT:    pcmpeqd %xmm7, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,0,3,2]
+; SSE2-NEXT:    pand %xmm0, %xmm1
+; SSE2-NEXT:    packssdw %xmm2, %xmm1
+; SSE2-NEXT:    packssdw %xmm3, %xmm1
+; SSE2-NEXT:    packsswb %xmm0, %xmm1
+; SSE2-NEXT:    pmovmskb %xmm1, %eax
+; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    jne LBB6_1
+; SSE2-NEXT:  ## %bb.2: ## %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne LBB6_3
 ; SSE2-NEXT:  LBB6_4: ## %else2
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,0,3,2]
-; SSE2-NEXT:    pand %xmm1, %xmm0
-; SSE2-NEXT:    pextrw $0, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB6_6
-; SSE2-NEXT:  ## %bb.5: ## %cond.load4
-; SSE2-NEXT:    movlps {{.*#+}} xmm5 = mem[0,1],xmm5[2,3]
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne LBB6_5
 ; SSE2-NEXT:  LBB6_6: ## %else5
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB6_8
-; SSE2-NEXT:  ## %bb.7: ## %cond.load7
-; SSE2-NEXT:    movhps {{.*#+}} xmm5 = xmm5[0,1],mem[0,1]
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne LBB6_7
 ; SSE2-NEXT:  LBB6_8: ## %else8
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,0,3,2]
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    pextrw $0, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB6_10
-; SSE2-NEXT:  ## %bb.9: ## %cond.load10
-; SSE2-NEXT:    movlps {{.*#+}} xmm6 = mem[0,1],xmm6[2,3]
+; SSE2-NEXT:    testb $16, %al
+; SSE2-NEXT:    jne LBB6_9
 ; SSE2-NEXT:  LBB6_10: ## %else11
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB6_12
-; SSE2-NEXT:  ## %bb.11: ## %cond.load13
-; SSE2-NEXT:    movhps {{.*#+}} xmm6 = xmm6[0,1],mem[0,1]
+; SSE2-NEXT:    testb $32, %al
+; SSE2-NEXT:    jne LBB6_11
 ; SSE2-NEXT:  LBB6_12: ## %else14
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,0,3,2]
-; SSE2-NEXT:    pand %xmm3, %xmm0
-; SSE2-NEXT:    pextrw $0, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB6_14
-; SSE2-NEXT:  ## %bb.13: ## %cond.load16
-; SSE2-NEXT:    movlps {{.*#+}} xmm8 = mem[0,1],xmm8[2,3]
+; SSE2-NEXT:    testb $64, %al
+; SSE2-NEXT:    jne LBB6_13
 ; SSE2-NEXT:  LBB6_14: ## %else17
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $-128, %al
 ; SSE2-NEXT:    je LBB6_16
-; SSE2-NEXT:  ## %bb.15: ## %cond.load19
+; SSE2-NEXT:  LBB6_15: ## %cond.load19
 ; SSE2-NEXT:    movhps {{.*#+}} xmm8 = xmm8[0,1],mem[0,1]
 ; SSE2-NEXT:  LBB6_16: ## %else20
 ; SSE2-NEXT:    movaps %xmm4, %xmm0
 ; SSE2-NEXT:    movaps %xmm5, %xmm1
-; SSE2-NEXT:    movaps %xmm6, %xmm2
+; SSE2-NEXT:    movaps %xmm9, %xmm2
 ; SSE2-NEXT:    movaps %xmm8, %xmm3
 ; SSE2-NEXT:    retq
+; SSE2-NEXT:  LBB6_1: ## %cond.load
+; SSE2-NEXT:    movlps {{.*#+}} xmm4 = mem[0,1],xmm4[2,3]
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    je LBB6_4
+; SSE2-NEXT:  LBB6_3: ## %cond.load1
+; SSE2-NEXT:    movhps {{.*#+}} xmm4 = xmm4[0,1],mem[0,1]
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    je LBB6_6
+; SSE2-NEXT:  LBB6_5: ## %cond.load4
+; SSE2-NEXT:    movlps {{.*#+}} xmm5 = mem[0,1],xmm5[2,3]
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    je LBB6_8
+; SSE2-NEXT:  LBB6_7: ## %cond.load7
+; SSE2-NEXT:    movhps {{.*#+}} xmm5 = xmm5[0,1],mem[0,1]
+; SSE2-NEXT:    testb $16, %al
+; SSE2-NEXT:    je LBB6_10
+; SSE2-NEXT:  LBB6_9: ## %cond.load10
+; SSE2-NEXT:    movlps {{.*#+}} xmm9 = mem[0,1],xmm9[2,3]
+; SSE2-NEXT:    testb $32, %al
+; SSE2-NEXT:    je LBB6_12
+; SSE2-NEXT:  LBB6_11: ## %cond.load13
+; SSE2-NEXT:    movhps {{.*#+}} xmm9 = xmm9[0,1],mem[0,1]
+; SSE2-NEXT:    testb $64, %al
+; SSE2-NEXT:    je LBB6_14
+; SSE2-NEXT:  LBB6_13: ## %cond.load16
+; SSE2-NEXT:    movlps {{.*#+}} xmm8 = mem[0,1],xmm8[2,3]
+; SSE2-NEXT:    testb $-128, %al
+; SSE2-NEXT:    jne LBB6_15
+; SSE2-NEXT:    jmp LBB6_16
 ;
 ; SSE42-LABEL: load_v8f64_v8i64:
 ; SSE42:       ## %bb.0:
 ; SSE42-NEXT:    movdqa %xmm7, %xmm8
 ; SSE42-NEXT:    pxor %xmm7, %xmm7
-; SSE42-NEXT:    pcmpeqq %xmm0, %xmm7
-; SSE42-NEXT:    pextrb $0, %xmm7, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB6_2
-; SSE42-NEXT:  ## %bb.1: ## %cond.load
-; SSE42-NEXT:    movlps {{.*#+}} xmm4 = mem[0,1],xmm4[2,3]
-; SSE42-NEXT:  LBB6_2: ## %else
-; SSE42-NEXT:    pextrb $8, %xmm7, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB6_4
-; SSE42-NEXT:  ## %bb.3: ## %cond.load1
-; SSE42-NEXT:    movhps {{.*#+}} xmm4 = xmm4[0,1],mem[0,1]
+; SSE42-NEXT:    pcmpeqq %xmm7, %xmm3
+; SSE42-NEXT:    pcmpeqq %xmm7, %xmm2
+; SSE42-NEXT:    packssdw %xmm3, %xmm2
+; SSE42-NEXT:    pcmpeqq %xmm7, %xmm1
+; SSE42-NEXT:    pcmpeqq %xmm7, %xmm0
+; SSE42-NEXT:    packssdw %xmm1, %xmm0
+; SSE42-NEXT:    packssdw %xmm2, %xmm0
+; SSE42-NEXT:    packsswb %xmm0, %xmm0
+; SSE42-NEXT:    pmovmskb %xmm0, %eax
+; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    jne LBB6_1
+; SSE42-NEXT:  ## %bb.2: ## %else
+; SSE42-NEXT:    testb $2, %al
+; SSE42-NEXT:    jne LBB6_3
 ; SSE42-NEXT:  LBB6_4: ## %else2
-; SSE42-NEXT:    pxor %xmm0, %xmm0
-; SSE42-NEXT:    pcmpeqq %xmm0, %xmm1
-; SSE42-NEXT:    pextrb $0, %xmm1, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB6_6
-; SSE42-NEXT:  ## %bb.5: ## %cond.load4
-; SSE42-NEXT:    movlps {{.*#+}} xmm5 = mem[0,1],xmm5[2,3]
+; SSE42-NEXT:    testb $4, %al
+; SSE42-NEXT:    jne LBB6_5
 ; SSE42-NEXT:  LBB6_6: ## %else5
-; SSE42-NEXT:    pextrb $8, %xmm1, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB6_8
-; SSE42-NEXT:  ## %bb.7: ## %cond.load7
-; SSE42-NEXT:    movhps {{.*#+}} xmm5 = xmm5[0,1],mem[0,1]
+; SSE42-NEXT:    testb $8, %al
+; SSE42-NEXT:    jne LBB6_7
 ; SSE42-NEXT:  LBB6_8: ## %else8
-; SSE42-NEXT:    pxor %xmm0, %xmm0
-; SSE42-NEXT:    pcmpeqq %xmm0, %xmm2
-; SSE42-NEXT:    pextrb $0, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB6_10
-; SSE42-NEXT:  ## %bb.9: ## %cond.load10
-; SSE42-NEXT:    movlps {{.*#+}} xmm6 = mem[0,1],xmm6[2,3]
+; SSE42-NEXT:    testb $16, %al
+; SSE42-NEXT:    jne LBB6_9
 ; SSE42-NEXT:  LBB6_10: ## %else11
-; SSE42-NEXT:    pextrb $8, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB6_12
-; SSE42-NEXT:  ## %bb.11: ## %cond.load13
-; SSE42-NEXT:    movhps {{.*#+}} xmm6 = xmm6[0,1],mem[0,1]
+; SSE42-NEXT:    testb $32, %al
+; SSE42-NEXT:    jne LBB6_11
 ; SSE42-NEXT:  LBB6_12: ## %else14
-; SSE42-NEXT:    pxor %xmm0, %xmm0
-; SSE42-NEXT:    pcmpeqq %xmm0, %xmm3
-; SSE42-NEXT:    pextrb $0, %xmm3, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB6_14
-; SSE42-NEXT:  ## %bb.13: ## %cond.load16
-; SSE42-NEXT:    movlps {{.*#+}} xmm8 = mem[0,1],xmm8[2,3]
+; SSE42-NEXT:    testb $64, %al
+; SSE42-NEXT:    jne LBB6_13
 ; SSE42-NEXT:  LBB6_14: ## %else17
-; SSE42-NEXT:    pextrb $8, %xmm3, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $-128, %al
 ; SSE42-NEXT:    je LBB6_16
-; SSE42-NEXT:  ## %bb.15: ## %cond.load19
+; SSE42-NEXT:  LBB6_15: ## %cond.load19
 ; SSE42-NEXT:    movhps {{.*#+}} xmm8 = xmm8[0,1],mem[0,1]
 ; SSE42-NEXT:  LBB6_16: ## %else20
 ; SSE42-NEXT:    movaps %xmm4, %xmm0
@@ -774,6 +648,35 @@ define <8 x double> @load_v8f64_v8i64(<8 x i64> %trigger, <8 x double>* %addr, <
 ; SSE42-NEXT:    movaps %xmm6, %xmm2
 ; SSE42-NEXT:    movaps %xmm8, %xmm3
 ; SSE42-NEXT:    retq
+; SSE42-NEXT:  LBB6_1: ## %cond.load
+; SSE42-NEXT:    movlps {{.*#+}} xmm4 = mem[0,1],xmm4[2,3]
+; SSE42-NEXT:    testb $2, %al
+; SSE42-NEXT:    je LBB6_4
+; SSE42-NEXT:  LBB6_3: ## %cond.load1
+; SSE42-NEXT:    movhps {{.*#+}} xmm4 = xmm4[0,1],mem[0,1]
+; SSE42-NEXT:    testb $4, %al
+; SSE42-NEXT:    je LBB6_6
+; SSE42-NEXT:  LBB6_5: ## %cond.load4
+; SSE42-NEXT:    movlps {{.*#+}} xmm5 = mem[0,1],xmm5[2,3]
+; SSE42-NEXT:    testb $8, %al
+; SSE42-NEXT:    je LBB6_8
+; SSE42-NEXT:  LBB6_7: ## %cond.load7
+; SSE42-NEXT:    movhps {{.*#+}} xmm5 = xmm5[0,1],mem[0,1]
+; SSE42-NEXT:    testb $16, %al
+; SSE42-NEXT:    je LBB6_10
+; SSE42-NEXT:  LBB6_9: ## %cond.load10
+; SSE42-NEXT:    movlps {{.*#+}} xmm6 = mem[0,1],xmm6[2,3]
+; SSE42-NEXT:    testb $32, %al
+; SSE42-NEXT:    je LBB6_12
+; SSE42-NEXT:  LBB6_11: ## %cond.load13
+; SSE42-NEXT:    movhps {{.*#+}} xmm6 = xmm6[0,1],mem[0,1]
+; SSE42-NEXT:    testb $64, %al
+; SSE42-NEXT:    je LBB6_14
+; SSE42-NEXT:  LBB6_13: ## %cond.load16
+; SSE42-NEXT:    movlps {{.*#+}} xmm8 = mem[0,1],xmm8[2,3]
+; SSE42-NEXT:    testb $-128, %al
+; SSE42-NEXT:    jne LBB6_15
+; SSE42-NEXT:    jmp LBB6_16
 ;
 ; AVX1-LABEL: load_v8f64_v8i64:
 ; AVX1:       ## %bb.0:
@@ -820,77 +723,69 @@ define <8 x double> @load_v8f64_v8i64(<8 x i64> %trigger, <8 x double>* %addr, <
 define <2 x float> @load_v2f32_v2i32(<2 x i32> %trigger, <2 x float>* %addr, <2 x float> %dst) {
 ; SSE2-LABEL: load_v2f32_v2i32:
 ; SSE2:       ## %bb.0:
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
 ; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,0,3,2]
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB7_2
-; SSE2-NEXT:  ## %bb.1: ## %cond.load
-; SSE2-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; SSE2-NEXT:    movss {{.*#+}} xmm1 = xmm2[0],xmm1[1,2,3]
-; SSE2-NEXT:  LBB7_2: ## %else
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,0,1,1]
+; SSE2-NEXT:    movmskpd %xmm0, %eax
 ; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    jne LBB7_1
+; SSE2-NEXT:  ## %bb.2: ## %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne LBB7_3
+; SSE2-NEXT:  LBB7_4: ## %else2
+; SSE2-NEXT:    movaps %xmm1, %xmm0
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  LBB7_1: ## %cond.load
+; SSE2-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT:    movss {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3]
+; SSE2-NEXT:    testb $2, %al
 ; SSE2-NEXT:    je LBB7_4
-; SSE2-NEXT:  ## %bb.3: ## %cond.load1
+; SSE2-NEXT:  LBB7_3: ## %cond.load1
 ; SSE2-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,0],xmm1[0,0]
 ; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[2,0],xmm1[2,3]
 ; SSE2-NEXT:    movaps %xmm0, %xmm1
-; SSE2-NEXT:  LBB7_4: ## %else2
 ; SSE2-NEXT:    movaps %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE42-LABEL: load_v2f32_v2i32:
 ; SSE42:       ## %bb.0:
 ; SSE42-NEXT:    pxor %xmm2, %xmm2
-; SSE42-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
-; SSE42-NEXT:    pcmpeqq %xmm2, %xmm0
-; SSE42-NEXT:    pextrb $0, %xmm0, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB7_2
-; SSE42-NEXT:  ## %bb.1: ## %cond.load
-; SSE42-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; SSE42-NEXT:    pblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3,4,5,6,7]
-; SSE42-NEXT:  LBB7_2: ## %else
-; SSE42-NEXT:    pextrb $8, %xmm0, %eax
+; SSE42-NEXT:    pcmpeqd %xmm0, %xmm2
+; SSE42-NEXT:    pmovsxdq %xmm2, %xmm0
+; SSE42-NEXT:    movmskpd %xmm0, %eax
 ; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    jne LBB7_1
+; SSE42-NEXT:  ## %bb.2: ## %else
+; SSE42-NEXT:    testb $2, %al
+; SSE42-NEXT:    jne LBB7_3
+; SSE42-NEXT:  LBB7_4: ## %else2
+; SSE42-NEXT:    movaps %xmm1, %xmm0
+; SSE42-NEXT:    retq
+; SSE42-NEXT:  LBB7_1: ## %cond.load
+; SSE42-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE42-NEXT:    blendps {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3]
+; SSE42-NEXT:    testb $2, %al
 ; SSE42-NEXT:    je LBB7_4
-; SSE42-NEXT:  ## %bb.3: ## %cond.load1
+; SSE42-NEXT:  LBB7_3: ## %cond.load1
 ; SSE42-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[2,3]
-; SSE42-NEXT:  LBB7_4: ## %else2
 ; SSE42-NEXT:    movaps %xmm1, %xmm0
 ; SSE42-NEXT:    retq
 ;
-; AVX1-LABEL: load_v2f32_v2i32:
-; AVX1:       ## %bb.0:
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
-; AVX1-NEXT:    vpcmpeqq %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,2],zero,zero
-; AVX1-NEXT:    vmaskmovps (%rdi), %xmm0, %xmm2
-; AVX1-NEXT:    vblendvps %xmm0, %xmm2, %xmm1, %xmm0
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: load_v2f32_v2i32:
-; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
-; AVX2-NEXT:    vpcmpeqq %xmm2, %xmm0, %xmm0
-; AVX2-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,2],zero,zero
-; AVX2-NEXT:    vmaskmovps (%rdi), %xmm0, %xmm2
-; AVX2-NEXT:    vblendvps %xmm0, %xmm2, %xmm1, %xmm0
-; AVX2-NEXT:    retq
+; AVX1OR2-LABEL: load_v2f32_v2i32:
+; AVX1OR2:       ## %bb.0:
+; AVX1OR2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX1OR2-NEXT:    vpcmpeqd %xmm2, %xmm0, %xmm0
+; AVX1OR2-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
+; AVX1OR2-NEXT:    vmaskmovps (%rdi), %xmm0, %xmm2
+; AVX1OR2-NEXT:    vblendvps %xmm0, %xmm2, %xmm1, %xmm0
+; AVX1OR2-NEXT:    retq
 ;
 ; AVX512F-LABEL: load_v2f32_v2i32:
 ; AVX512F:       ## %bb.0:
 ; AVX512F-NEXT:    ## kill: def $xmm1 killed $xmm1 def $zmm1
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
-; AVX512F-NEXT:    vptestnmq %zmm0, %zmm0, %k0
+; AVX512F-NEXT:    ## kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512F-NEXT:    vptestnmd %zmm0, %zmm0, %k0
 ; AVX512F-NEXT:    kshiftlw $14, %k0, %k0
 ; AVX512F-NEXT:    kshiftrw $14, %k0, %k1
 ; AVX512F-NEXT:    vblendmps (%rdi), %zmm1, %zmm0 {%k1}
@@ -898,13 +793,21 @@ define <2 x float> @load_v2f32_v2i32(<2 x i32> %trigger, <2 x float>* %addr, <2
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
-; AVX512VL-LABEL: load_v2f32_v2i32:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VL-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
-; AVX512VL-NEXT:    vptestnmq %xmm0, %xmm0, %k1
-; AVX512VL-NEXT:    vblendmps (%rdi), %xmm1, %xmm0 {%k1}
-; AVX512VL-NEXT:    retq
+; AVX512VLDQ-LABEL: load_v2f32_v2i32:
+; AVX512VLDQ:       ## %bb.0:
+; AVX512VLDQ-NEXT:    vptestnmd %xmm0, %xmm0, %k0
+; AVX512VLDQ-NEXT:    kshiftlb $6, %k0, %k0
+; AVX512VLDQ-NEXT:    kshiftrb $6, %k0, %k1
+; AVX512VLDQ-NEXT:    vblendmps (%rdi), %xmm1, %xmm0 {%k1}
+; AVX512VLDQ-NEXT:    retq
+;
+; AVX512VLBW-LABEL: load_v2f32_v2i32:
+; AVX512VLBW:       ## %bb.0:
+; AVX512VLBW-NEXT:    vptestnmd %xmm0, %xmm0, %k0
+; AVX512VLBW-NEXT:    kshiftlw $14, %k0, %k0
+; AVX512VLBW-NEXT:    kshiftrw $14, %k0, %k1
+; AVX512VLBW-NEXT:    vblendmps (%rdi), %xmm1, %xmm0 {%k1}
+; AVX512VLBW-NEXT:    retq
   %mask = icmp eq <2 x i32> %trigger, zeroinitializer
   %res = call <2 x float> @llvm.masked.load.v2f32.p0v2f32(<2 x float>* %addr, i32 4, <2 x i1> %mask, <2 x float> %dst)
   ret <2 x float> %res
@@ -913,73 +816,63 @@ define <2 x float> @load_v2f32_v2i32(<2 x i32> %trigger, <2 x float>* %addr, <2
 define <2 x float> @load_v2f32_v2i32_undef(<2 x i32> %trigger, <2 x float>* %addr) {
 ; SSE2-LABEL: load_v2f32_v2i32_undef:
 ; SSE2:       ## %bb.0:
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,0,3,2]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    movd %xmm1, %eax
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    pcmpeqd %xmm0, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,0,1,1]
+; SSE2-NEXT:    movmskpd %xmm0, %eax
 ; SSE2-NEXT:    testb $1, %al
 ; SSE2-NEXT:    ## implicit-def: $xmm0
-; SSE2-NEXT:    je LBB8_2
-; SSE2-NEXT:  ## %bb.1: ## %cond.load
+; SSE2-NEXT:    jne LBB8_1
+; SSE2-NEXT:  ## %bb.2: ## %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne LBB8_3
+; SSE2-NEXT:  LBB8_4: ## %else2
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  LBB8_1: ## %cond.load
 ; SSE2-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE2-NEXT:  LBB8_2: ## %else
-; SSE2-NEXT:    pextrw $4, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $2, %al
 ; SSE2-NEXT:    je LBB8_4
-; SSE2-NEXT:  ## %bb.3: ## %cond.load1
+; SSE2-NEXT:  LBB8_3: ## %cond.load1
 ; SSE2-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,0],xmm0[0,0]
 ; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[2,0],xmm0[2,3]
 ; SSE2-NEXT:    movaps %xmm1, %xmm0
-; SSE2-NEXT:  LBB8_4: ## %else2
 ; SSE2-NEXT:    retq
 ;
 ; SSE42-LABEL: load_v2f32_v2i32_undef:
 ; SSE42:       ## %bb.0:
-; SSE42-NEXT:    movdqa %xmm0, %xmm1
-; SSE42-NEXT:    pxor %xmm0, %xmm0
-; SSE42-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm0[2,3],xmm1[4,5],xmm0[6,7]
-; SSE42-NEXT:    pcmpeqq %xmm0, %xmm1
-; SSE42-NEXT:    pextrb $0, %xmm1, %eax
+; SSE42-NEXT:    pxor %xmm1, %xmm1
+; SSE42-NEXT:    pcmpeqd %xmm0, %xmm1
+; SSE42-NEXT:    pmovsxdq %xmm1, %xmm0
+; SSE42-NEXT:    movmskpd %xmm0, %eax
 ; SSE42-NEXT:    testb $1, %al
 ; SSE42-NEXT:    ## implicit-def: $xmm0
-; SSE42-NEXT:    je LBB8_2
-; SSE42-NEXT:  ## %bb.1: ## %cond.load
+; SSE42-NEXT:    jne LBB8_1
+; SSE42-NEXT:  ## %bb.2: ## %else
+; SSE42-NEXT:    testb $2, %al
+; SSE42-NEXT:    jne LBB8_3
+; SSE42-NEXT:  LBB8_4: ## %else2
+; SSE42-NEXT:    retq
+; SSE42-NEXT:  LBB8_1: ## %cond.load
 ; SSE42-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE42-NEXT:  LBB8_2: ## %else
-; SSE42-NEXT:    pextrb $8, %xmm1, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $2, %al
 ; SSE42-NEXT:    je LBB8_4
-; SSE42-NEXT:  ## %bb.3: ## %cond.load1
+; SSE42-NEXT:  LBB8_3: ## %cond.load1
 ; SSE42-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[2,3]
-; SSE42-NEXT:  LBB8_4: ## %else2
 ; SSE42-NEXT:    retq
 ;
-; AVX1-LABEL: load_v2f32_v2i32_undef:
-; AVX1:       ## %bb.0:
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
-; AVX1-NEXT:    vpcmpeqq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,2],zero,zero
-; AVX1-NEXT:    vmaskmovps (%rdi), %xmm0, %xmm0
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: load_v2f32_v2i32_undef:
-; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
-; AVX2-NEXT:    vpcmpeqq %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,2],zero,zero
-; AVX2-NEXT:    vmaskmovps (%rdi), %xmm0, %xmm0
-; AVX2-NEXT:    retq
+; AVX1OR2-LABEL: load_v2f32_v2i32_undef:
+; AVX1OR2:       ## %bb.0:
+; AVX1OR2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX1OR2-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; AVX1OR2-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
+; AVX1OR2-NEXT:    vmaskmovps (%rdi), %xmm0, %xmm0
+; AVX1OR2-NEXT:    retq
 ;
 ; AVX512F-LABEL: load_v2f32_v2i32_undef:
 ; AVX512F:       ## %bb.0:
-; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
-; AVX512F-NEXT:    vptestnmq %zmm0, %zmm0, %k0
+; AVX512F-NEXT:    ## kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512F-NEXT:    vptestnmd %zmm0, %zmm0, %k0
 ; AVX512F-NEXT:    kshiftlw $14, %k0, %k0
 ; AVX512F-NEXT:    kshiftrw $14, %k0, %k1
 ; AVX512F-NEXT:    vmovups (%rdi), %zmm0 {%k1} {z}
@@ -987,13 +880,21 @@ define <2 x float> @load_v2f32_v2i32_undef(<2 x i32> %trigger, <2 x float>* %add
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
-; AVX512VL-LABEL: load_v2f32_v2i32_undef:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
-; AVX512VL-NEXT:    vptestnmq %xmm0, %xmm0, %k1
-; AVX512VL-NEXT:    vmovups (%rdi), %xmm0 {%k1} {z}
-; AVX512VL-NEXT:    retq
+; AVX512VLDQ-LABEL: load_v2f32_v2i32_undef:
+; AVX512VLDQ:       ## %bb.0:
+; AVX512VLDQ-NEXT:    vptestnmd %xmm0, %xmm0, %k0
+; AVX512VLDQ-NEXT:    kshiftlb $6, %k0, %k0
+; AVX512VLDQ-NEXT:    kshiftrb $6, %k0, %k1
+; AVX512VLDQ-NEXT:    vmovups (%rdi), %xmm0 {%k1} {z}
+; AVX512VLDQ-NEXT:    retq
+;
+; AVX512VLBW-LABEL: load_v2f32_v2i32_undef:
+; AVX512VLBW:       ## %bb.0:
+; AVX512VLBW-NEXT:    vptestnmd %xmm0, %xmm0, %k0
+; AVX512VLBW-NEXT:    kshiftlw $14, %k0, %k0
+; AVX512VLBW-NEXT:    kshiftrw $14, %k0, %k1
+; AVX512VLBW-NEXT:    vmovups (%rdi), %xmm0 {%k1} {z}
+; AVX512VLBW-NEXT:    retq
   %mask = icmp eq <2 x i32> %trigger, zeroinitializer
   %res = call <2 x float> @llvm.masked.load.v2f32.p0v2f32(<2 x float>* %addr, i32 4, <2 x i1> %mask, <2 x float>undef)
   ret <2 x float> %res
@@ -1004,40 +905,43 @@ define <4 x float> @load_v4f32_v4i32(<4 x i32> %trigger, <4 x float>* %addr, <4
 ; SSE2:       ## %bb.0:
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
 ; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    movd %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB9_2
-; SSE2-NEXT:  ## %bb.1: ## %cond.load
-; SSE2-NEXT:    movss {{.*#+}} xmm3 = mem[0],zero,zero,zero
-; SSE2-NEXT:    movss {{.*#+}} xmm1 = xmm3[0],xmm1[1,2,3]
-; SSE2-NEXT:  LBB9_2: ## %else
-; SSE2-NEXT:    pextrw $2, %xmm2, %eax
+; SSE2-NEXT:    movmskps %xmm2, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB9_4
-; SSE2-NEXT:  ## %bb.3: ## %cond.load1
-; SSE2-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,0],xmm1[0,0]
-; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[2,0],xmm1[2,3]
-; SSE2-NEXT:    movaps %xmm2, %xmm1
+; SSE2-NEXT:    jne LBB9_1
+; SSE2-NEXT:  ## %bb.2: ## %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne LBB9_3
 ; SSE2-NEXT:  LBB9_4: ## %else2
-; SSE2-NEXT:    xorps %xmm2, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm0
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB9_6
-; SSE2-NEXT:  ## %bb.5: ## %cond.load4
-; SSE2-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,0],xmm1[3,0]
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0,2]
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne LBB9_5
 ; SSE2-NEXT:  LBB9_6: ## %else5
-; SSE2-NEXT:    pextrw $6, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne LBB9_7
+; SSE2-NEXT:  LBB9_8: ## %else8
+; SSE2-NEXT:    movaps %xmm1, %xmm0
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  LBB9_1: ## %cond.load
+; SSE2-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT:    movss {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3]
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    je LBB9_4
+; SSE2-NEXT:  LBB9_3: ## %cond.load1
+; SSE2-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,0],xmm1[0,0]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[2,0],xmm1[2,3]
+; SSE2-NEXT:    movaps %xmm0, %xmm1
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    je LBB9_6
+; SSE2-NEXT:  LBB9_5: ## %cond.load4
+; SSE2-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,0],xmm1[3,0]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,1],xmm0[0,2]
+; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je LBB9_8
-; SSE2-NEXT:  ## %bb.7: ## %cond.load7
+; SSE2-NEXT:  LBB9_7: ## %cond.load7
 ; SSE2-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,0],xmm1[2,0]
 ; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,1],xmm0[2,0]
-; SSE2-NEXT:  LBB9_8: ## %else8
 ; SSE2-NEXT:    movaps %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
@@ -1045,33 +949,36 @@ define <4 x float> @load_v4f32_v4i32(<4 x i32> %trigger, <4 x float>* %addr, <4
 ; SSE42:       ## %bb.0:
 ; SSE42-NEXT:    pxor %xmm2, %xmm2
 ; SSE42-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE42-NEXT:    pextrb $0, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB9_2
-; SSE42-NEXT:  ## %bb.1: ## %cond.load
-; SSE42-NEXT:    movss {{.*#+}} xmm3 = mem[0],zero,zero,zero
-; SSE42-NEXT:    blendps {{.*#+}} xmm1 = xmm3[0],xmm1[1,2,3]
-; SSE42-NEXT:  LBB9_2: ## %else
-; SSE42-NEXT:    pextrb $4, %xmm2, %eax
+; SSE42-NEXT:    movmskps %xmm2, %eax
 ; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    jne LBB9_1
+; SSE42-NEXT:  ## %bb.2: ## %else
+; SSE42-NEXT:    testb $2, %al
+; SSE42-NEXT:    jne LBB9_3
+; SSE42-NEXT:  LBB9_4: ## %else2
+; SSE42-NEXT:    testb $4, %al
+; SSE42-NEXT:    jne LBB9_5
+; SSE42-NEXT:  LBB9_6: ## %else5
+; SSE42-NEXT:    testb $8, %al
+; SSE42-NEXT:    jne LBB9_7
+; SSE42-NEXT:  LBB9_8: ## %else8
+; SSE42-NEXT:    movaps %xmm1, %xmm0
+; SSE42-NEXT:    retq
+; SSE42-NEXT:  LBB9_1: ## %cond.load
+; SSE42-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE42-NEXT:    pblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3,4,5,6,7]
+; SSE42-NEXT:    testb $2, %al
 ; SSE42-NEXT:    je LBB9_4
-; SSE42-NEXT:  ## %bb.3: ## %cond.load1
+; SSE42-NEXT:  LBB9_3: ## %cond.load1
 ; SSE42-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[2,3]
-; SSE42-NEXT:  LBB9_4: ## %else2
-; SSE42-NEXT:    pxor %xmm2, %xmm2
-; SSE42-NEXT:    pcmpeqd %xmm2, %xmm0
-; SSE42-NEXT:    pextrb $8, %xmm0, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $4, %al
 ; SSE42-NEXT:    je LBB9_6
-; SSE42-NEXT:  ## %bb.5: ## %cond.load4
+; SSE42-NEXT:  LBB9_5: ## %cond.load4
 ; SSE42-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0,1],mem[0],xmm1[3]
-; SSE42-NEXT:  LBB9_6: ## %else5
-; SSE42-NEXT:    pextrb $12, %xmm0, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $8, %al
 ; SSE42-NEXT:    je LBB9_8
-; SSE42-NEXT:  ## %bb.7: ## %cond.load7
+; SSE42-NEXT:  LBB9_7: ## %cond.load7
 ; SSE42-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0,1,2],mem[0]
-; SSE42-NEXT:  LBB9_8: ## %else8
 ; SSE42-NEXT:    movaps %xmm1, %xmm0
 ; SSE42-NEXT:    retq
 ;
@@ -1108,130 +1015,148 @@ define <4 x float> @load_v4f32_v4i32(<4 x i32> %trigger, <4 x float>* %addr, <4
 define <8 x float> @load_v8f32_v8i1_zero(<8 x i1> %mask, <8 x float>* %addr) {
 ; SSE2-LABEL: load_v8f32_v8i1_zero:
 ; SSE2:       ## %bb.0:
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    psllw $15, %xmm0
+; SSE2-NEXT:    packsswb %xmm0, %xmm0
+; SSE2-NEXT:    pmovmskb %xmm0, %eax
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
 ; SSE2-NEXT:    testb $1, %al
 ; SSE2-NEXT:    xorps %xmm1, %xmm1
-; SSE2-NEXT:    je LBB10_2
-; SSE2-NEXT:  ## %bb.1: ## %cond.load
+; SSE2-NEXT:    jne LBB10_1
+; SSE2-NEXT:  ## %bb.2: ## %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne LBB10_3
+; SSE2-NEXT:  LBB10_4: ## %else2
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne LBB10_5
+; SSE2-NEXT:  LBB10_6: ## %else5
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne LBB10_7
+; SSE2-NEXT:  LBB10_8: ## %else8
+; SSE2-NEXT:    testb $16, %al
+; SSE2-NEXT:    jne LBB10_9
+; SSE2-NEXT:  LBB10_10: ## %else11
+; SSE2-NEXT:    testb $32, %al
+; SSE2-NEXT:    jne LBB10_11
+; SSE2-NEXT:  LBB10_12: ## %else14
+; SSE2-NEXT:    testb $64, %al
+; SSE2-NEXT:    jne LBB10_13
+; SSE2-NEXT:  LBB10_14: ## %else17
+; SSE2-NEXT:    testb $-128, %al
+; SSE2-NEXT:    jne LBB10_15
+; SSE2-NEXT:  LBB10_16: ## %else20
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  LBB10_1: ## %cond.load
 ; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; SSE2-NEXT:    xorps %xmm1, %xmm1
-; SSE2-NEXT:  LBB10_2: ## %else
-; SSE2-NEXT:    shrl $16, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $2, %al
 ; SSE2-NEXT:    je LBB10_4
-; SSE2-NEXT:  ## %bb.3: ## %cond.load1
-; SSE2-NEXT:    movss {{.*#+}} xmm3 = mem[0],zero,zero,zero
-; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,0],xmm0[0,0]
-; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[2,0],xmm0[2,3]
-; SSE2-NEXT:    movaps %xmm3, %xmm0
-; SSE2-NEXT:  LBB10_4: ## %else2
-; SSE2-NEXT:    pextrw $2, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:  LBB10_3: ## %cond.load1
+; SSE2-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,0],xmm0[0,0]
+; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[2,0],xmm0[2,3]
+; SSE2-NEXT:    movaps %xmm2, %xmm0
+; SSE2-NEXT:    testb $4, %al
 ; SSE2-NEXT:    je LBB10_6
-; SSE2-NEXT:  ## %bb.5: ## %cond.load4
-; SSE2-NEXT:    movss {{.*#+}} xmm3 = mem[0],zero,zero,zero
-; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,0],xmm0[3,0]
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm3[0,2]
-; SSE2-NEXT:  LBB10_6: ## %else5
-; SSE2-NEXT:    pextrw $3, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:  LBB10_5: ## %cond.load4
+; SSE2-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,0],xmm0[3,0]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm2[0,2]
+; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je LBB10_8
-; SSE2-NEXT:  ## %bb.7: ## %cond.load7
-; SSE2-NEXT:    movss {{.*#+}} xmm3 = mem[0],zero,zero,zero
-; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,0],xmm0[2,0]
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm3[2,0]
-; SSE2-NEXT:  LBB10_8: ## %else8
-; SSE2-NEXT:    pextrw $4, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:  LBB10_7: ## %cond.load7
+; SSE2-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,0],xmm0[2,0]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,0]
+; SSE2-NEXT:    testb $16, %al
 ; SSE2-NEXT:    je LBB10_10
-; SSE2-NEXT:  ## %bb.9: ## %cond.load10
-; SSE2-NEXT:    movss {{.*#+}} xmm3 = mem[0],zero,zero,zero
-; SSE2-NEXT:    movss {{.*#+}} xmm1 = xmm3[0],xmm1[1,2,3]
-; SSE2-NEXT:  LBB10_10: ## %else11
-; SSE2-NEXT:    pextrw $5, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:  LBB10_9: ## %cond.load10
+; SSE2-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE2-NEXT:    movss {{.*#+}} xmm1 = xmm2[0],xmm1[1,2,3]
+; SSE2-NEXT:    testb $32, %al
 ; SSE2-NEXT:    je LBB10_12
-; SSE2-NEXT:  ## %bb.11: ## %cond.load13
-; SSE2-NEXT:    movss {{.*#+}} xmm3 = mem[0],zero,zero,zero
-; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,0],xmm1[0,0]
-; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[2,0],xmm1[2,3]
-; SSE2-NEXT:    movaps %xmm3, %xmm1
-; SSE2-NEXT:  LBB10_12: ## %else14
-; SSE2-NEXT:    pextrw $6, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:  LBB10_11: ## %cond.load13
+; SSE2-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,0],xmm1[0,0]
+; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[2,0],xmm1[2,3]
+; SSE2-NEXT:    movaps %xmm2, %xmm1
+; SSE2-NEXT:    testb $64, %al
 ; SSE2-NEXT:    je LBB10_14
-; SSE2-NEXT:  ## %bb.13: ## %cond.load16
-; SSE2-NEXT:    movss {{.*#+}} xmm3 = mem[0],zero,zero,zero
-; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,0],xmm1[3,0]
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,1],xmm3[0,2]
-; SSE2-NEXT:  LBB10_14: ## %else17
-; SSE2-NEXT:    pextrw $7, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:  LBB10_13: ## %cond.load16
+; SSE2-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,0],xmm1[3,0]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0,2]
+; SSE2-NEXT:    testb $-128, %al
 ; SSE2-NEXT:    je LBB10_16
-; SSE2-NEXT:  ## %bb.15: ## %cond.load19
+; SSE2-NEXT:  LBB10_15: ## %cond.load19
 ; SSE2-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
 ; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,0],xmm1[2,0]
 ; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,0]
-; SSE2-NEXT:  LBB10_16: ## %else20
 ; SSE2-NEXT:    retq
 ;
 ; SSE42-LABEL: load_v8f32_v8i1_zero:
 ; SSE42:       ## %bb.0:
-; SSE42-NEXT:    movdqa %xmm0, %xmm2
-; SSE42-NEXT:    pextrb $0, %xmm0, %eax
+; SSE42-NEXT:    psllw $15, %xmm0
+; SSE42-NEXT:    packsswb %xmm0, %xmm0
+; SSE42-NEXT:    pmovmskb %xmm0, %eax
 ; SSE42-NEXT:    pxor %xmm0, %xmm0
 ; SSE42-NEXT:    testb $1, %al
 ; SSE42-NEXT:    xorps %xmm1, %xmm1
-; SSE42-NEXT:    je LBB10_2
-; SSE42-NEXT:  ## %bb.1: ## %cond.load
+; SSE42-NEXT:    jne LBB10_1
+; SSE42-NEXT:  ## %bb.2: ## %else
+; SSE42-NEXT:    testb $2, %al
+; SSE42-NEXT:    jne LBB10_3
+; SSE42-NEXT:  LBB10_4: ## %else2
+; SSE42-NEXT:    testb $4, %al
+; SSE42-NEXT:    jne LBB10_5
+; SSE42-NEXT:  LBB10_6: ## %else5
+; SSE42-NEXT:    testb $8, %al
+; SSE42-NEXT:    jne LBB10_7
+; SSE42-NEXT:  LBB10_8: ## %else8
+; SSE42-NEXT:    testb $16, %al
+; SSE42-NEXT:    jne LBB10_9
+; SSE42-NEXT:  LBB10_10: ## %else11
+; SSE42-NEXT:    testb $32, %al
+; SSE42-NEXT:    jne LBB10_11
+; SSE42-NEXT:  LBB10_12: ## %else14
+; SSE42-NEXT:    testb $64, %al
+; SSE42-NEXT:    jne LBB10_13
+; SSE42-NEXT:  LBB10_14: ## %else17
+; SSE42-NEXT:    testb $-128, %al
+; SSE42-NEXT:    jne LBB10_15
+; SSE42-NEXT:  LBB10_16: ## %else20
+; SSE42-NEXT:    retq
+; SSE42-NEXT:  LBB10_1: ## %cond.load
 ; SSE42-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; SSE42-NEXT:    xorps %xmm1, %xmm1
-; SSE42-NEXT:  LBB10_2: ## %else
-; SSE42-NEXT:    pextrb $2, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $2, %al
 ; SSE42-NEXT:    je LBB10_4
-; SSE42-NEXT:  ## %bb.3: ## %cond.load1
+; SSE42-NEXT:  LBB10_3: ## %cond.load1
 ; SSE42-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[2,3]
-; SSE42-NEXT:  LBB10_4: ## %else2
-; SSE42-NEXT:    pextrb $4, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $4, %al
 ; SSE42-NEXT:    je LBB10_6
-; SSE42-NEXT:  ## %bb.5: ## %cond.load4
+; SSE42-NEXT:  LBB10_5: ## %cond.load4
 ; SSE42-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0,1],mem[0],xmm0[3]
-; SSE42-NEXT:  LBB10_6: ## %else5
-; SSE42-NEXT:    pextrb $6, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $8, %al
 ; SSE42-NEXT:    je LBB10_8
-; SSE42-NEXT:  ## %bb.7: ## %cond.load7
+; SSE42-NEXT:  LBB10_7: ## %cond.load7
 ; SSE42-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0,1,2],mem[0]
-; SSE42-NEXT:  LBB10_8: ## %else8
-; SSE42-NEXT:    pextrb $8, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $16, %al
 ; SSE42-NEXT:    je LBB10_10
-; SSE42-NEXT:  ## %bb.9: ## %cond.load10
-; SSE42-NEXT:    movss {{.*#+}} xmm3 = mem[0],zero,zero,zero
-; SSE42-NEXT:    blendps {{.*#+}} xmm1 = xmm3[0],xmm1[1,2,3]
-; SSE42-NEXT:  LBB10_10: ## %else11
-; SSE42-NEXT:    pextrb $10, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:  LBB10_9: ## %cond.load10
+; SSE42-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE42-NEXT:    blendps {{.*#+}} xmm1 = xmm2[0],xmm1[1,2,3]
+; SSE42-NEXT:    testb $32, %al
 ; SSE42-NEXT:    je LBB10_12
-; SSE42-NEXT:  ## %bb.11: ## %cond.load13
+; SSE42-NEXT:  LBB10_11: ## %cond.load13
 ; SSE42-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[2,3]
-; SSE42-NEXT:  LBB10_12: ## %else14
-; SSE42-NEXT:    pextrb $12, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $64, %al
 ; SSE42-NEXT:    je LBB10_14
-; SSE42-NEXT:  ## %bb.13: ## %cond.load16
+; SSE42-NEXT:  LBB10_13: ## %cond.load16
 ; SSE42-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0,1],mem[0],xmm1[3]
-; SSE42-NEXT:  LBB10_14: ## %else17
-; SSE42-NEXT:    pextrb $14, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $-128, %al
 ; SSE42-NEXT:    je LBB10_16
-; SSE42-NEXT:  ## %bb.15: ## %cond.load19
+; SSE42-NEXT:  LBB10_15: ## %cond.load19
 ; SSE42-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0,1,2],mem[0]
-; SSE42-NEXT:  LBB10_16: ## %else20
 ; SSE42-NEXT:    retq
 ;
 ; AVX1-LABEL: load_v8f32_v8i1_zero:
@@ -1285,156 +1210,162 @@ define <8 x float> @load_v8f32_v8i32(<8 x i32> %trigger, <8 x float>* %addr, <8
 ; SSE2-LABEL: load_v8f32_v8i32:
 ; SSE2:       ## %bb.0:
 ; SSE2-NEXT:    pxor %xmm4, %xmm4
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm4
-; SSE2-NEXT:    movdqa %xmm4, %xmm5
-; SSE2-NEXT:    packssdw %xmm0, %xmm5
-; SSE2-NEXT:    movd %xmm5, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB11_2
-; SSE2-NEXT:  ## %bb.1: ## %cond.load
-; SSE2-NEXT:    movss {{.*#+}} xmm5 = mem[0],zero,zero,zero
-; SSE2-NEXT:    movss {{.*#+}} xmm2 = xmm5[0],xmm2[1,2,3]
-; SSE2-NEXT:  LBB11_2: ## %else
-; SSE2-NEXT:    psrlq $16, %xmm4
-; SSE2-NEXT:    movd %xmm4, %eax
-; SSE2-NEXT:    shrl $16, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB11_4
-; SSE2-NEXT:  ## %bb.3: ## %cond.load1
-; SSE2-NEXT:    movss {{.*#+}} xmm4 = mem[0],zero,zero,zero
-; SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[0,0],xmm2[0,0]
-; SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[2,0],xmm2[2,3]
-; SSE2-NEXT:    movaps %xmm4, %xmm2
-; SSE2-NEXT:  LBB11_4: ## %else2
-; SSE2-NEXT:    xorps %xmm4, %xmm4
+; SSE2-NEXT:    pcmpeqd %xmm4, %xmm1
 ; SSE2-NEXT:    pcmpeqd %xmm4, %xmm0
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB11_6
-; SSE2-NEXT:  ## %bb.5: ## %cond.load4
-; SSE2-NEXT:    movss {{.*#+}} xmm4 = mem[0],zero,zero,zero
-; SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[0,0],xmm2[3,0]
-; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,1],xmm4[0,2]
+; SSE2-NEXT:    packssdw %xmm1, %xmm0
+; SSE2-NEXT:    packsswb %xmm0, %xmm0
+; SSE2-NEXT:    pmovmskb %xmm0, %eax
+; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    jne LBB11_1
+; SSE2-NEXT:  ## %bb.2: ## %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne LBB11_3
+; SSE2-NEXT:  LBB11_4: ## %else2
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne LBB11_5
 ; SSE2-NEXT:  LBB11_6: ## %else5
-; SSE2-NEXT:    pextrw $6, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne LBB11_7
+; SSE2-NEXT:  LBB11_8: ## %else8
+; SSE2-NEXT:    testb $16, %al
+; SSE2-NEXT:    jne LBB11_9
+; SSE2-NEXT:  LBB11_10: ## %else11
+; SSE2-NEXT:    testb $32, %al
+; SSE2-NEXT:    jne LBB11_11
+; SSE2-NEXT:  LBB11_12: ## %else14
+; SSE2-NEXT:    testb $64, %al
+; SSE2-NEXT:    jne LBB11_13
+; SSE2-NEXT:  LBB11_14: ## %else17
+; SSE2-NEXT:    testb $-128, %al
+; SSE2-NEXT:    je LBB11_16
+; SSE2-NEXT:  LBB11_15: ## %cond.load19
+; SSE2-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,0],xmm3[2,0]
+; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,1],xmm0[2,0]
+; SSE2-NEXT:  LBB11_16: ## %else20
+; SSE2-NEXT:    movaps %xmm2, %xmm0
+; SSE2-NEXT:    movaps %xmm3, %xmm1
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  LBB11_1: ## %cond.load
+; SSE2-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT:    movss {{.*#+}} xmm2 = xmm0[0],xmm2[1,2,3]
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    je LBB11_4
+; SSE2-NEXT:  LBB11_3: ## %cond.load1
+; SSE2-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,0],xmm2[0,0]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[2,0],xmm2[2,3]
+; SSE2-NEXT:    movaps %xmm0, %xmm2
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    je LBB11_6
+; SSE2-NEXT:  LBB11_5: ## %cond.load4
+; SSE2-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,0],xmm2[3,0]
+; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,1],xmm0[0,2]
+; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je LBB11_8
-; SSE2-NEXT:  ## %bb.7: ## %cond.load7
+; SSE2-NEXT:  LBB11_7: ## %cond.load7
 ; SSE2-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,0],xmm2[2,0]
 ; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,1],xmm0[2,0]
-; SSE2-NEXT:  LBB11_8: ## %else8
-; SSE2-NEXT:    xorps %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
-; SSE2-NEXT:    pextrw $0, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $16, %al
 ; SSE2-NEXT:    je LBB11_10
-; SSE2-NEXT:  ## %bb.9: ## %cond.load10
-; SSE2-NEXT:    movss {{.*#+}} xmm4 = mem[0],zero,zero,zero
-; SSE2-NEXT:    movss {{.*#+}} xmm3 = xmm4[0],xmm3[1,2,3]
-; SSE2-NEXT:  LBB11_10: ## %else11
-; SSE2-NEXT:    pextrw $2, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:  LBB11_9: ## %cond.load10
+; SSE2-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT:    movss {{.*#+}} xmm3 = xmm0[0],xmm3[1,2,3]
+; SSE2-NEXT:    testb $32, %al
 ; SSE2-NEXT:    je LBB11_12
-; SSE2-NEXT:  ## %bb.11: ## %cond.load13
+; SSE2-NEXT:  LBB11_11: ## %cond.load13
 ; SSE2-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,0],xmm3[0,0]
 ; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[2,0],xmm3[2,3]
 ; SSE2-NEXT:    movaps %xmm0, %xmm3
-; SSE2-NEXT:  LBB11_12: ## %else14
-; SSE2-NEXT:    xorps %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE2-NEXT:    pextrw $4, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $64, %al
 ; SSE2-NEXT:    je LBB11_14
-; SSE2-NEXT:  ## %bb.13: ## %cond.load16
+; SSE2-NEXT:  LBB11_13: ## %cond.load16
 ; SSE2-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,0],xmm3[3,0]
 ; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,1],xmm0[0,2]
-; SSE2-NEXT:  LBB11_14: ## %else17
-; SSE2-NEXT:    pextrw $6, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB11_16
-; SSE2-NEXT:  ## %bb.15: ## %cond.load19
-; SSE2-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,0],xmm3[2,0]
-; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,1],xmm0[2,0]
-; SSE2-NEXT:  LBB11_16: ## %else20
-; SSE2-NEXT:    movaps %xmm2, %xmm0
-; SSE2-NEXT:    movaps %xmm3, %xmm1
-; SSE2-NEXT:    retq
+; SSE2-NEXT:    testb $-128, %al
+; SSE2-NEXT:    jne LBB11_15
+; SSE2-NEXT:    jmp LBB11_16
 ;
 ; SSE42-LABEL: load_v8f32_v8i32:
 ; SSE42:       ## %bb.0:
 ; SSE42-NEXT:    pxor %xmm4, %xmm4
-; SSE42-NEXT:    pcmpeqd %xmm0, %xmm4
-; SSE42-NEXT:    pextrb $0, %xmm4, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB11_2
-; SSE42-NEXT:  ## %bb.1: ## %cond.load
-; SSE42-NEXT:    movss {{.*#+}} xmm5 = mem[0],zero,zero,zero
-; SSE42-NEXT:    blendps {{.*#+}} xmm2 = xmm5[0],xmm2[1,2,3]
-; SSE42-NEXT:  LBB11_2: ## %else
-; SSE42-NEXT:    pextrb $4, %xmm4, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB11_4
-; SSE42-NEXT:  ## %bb.3: ## %cond.load1
-; SSE42-NEXT:    insertps {{.*#+}} xmm2 = xmm2[0],mem[0],xmm2[2,3]
-; SSE42-NEXT:  LBB11_4: ## %else2
-; SSE42-NEXT:    pxor %xmm4, %xmm4
+; SSE42-NEXT:    pcmpeqd %xmm4, %xmm1
 ; SSE42-NEXT:    pcmpeqd %xmm4, %xmm0
-; SSE42-NEXT:    pextrb $8, %xmm0, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB11_6
-; SSE42-NEXT:  ## %bb.5: ## %cond.load4
-; SSE42-NEXT:    insertps {{.*#+}} xmm2 = xmm2[0,1],mem[0],xmm2[3]
+; SSE42-NEXT:    packssdw %xmm1, %xmm0
+; SSE42-NEXT:    packsswb %xmm0, %xmm0
+; SSE42-NEXT:    pmovmskb %xmm0, %eax
+; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    jne LBB11_1
+; SSE42-NEXT:  ## %bb.2: ## %else
+; SSE42-NEXT:    testb $2, %al
+; SSE42-NEXT:    jne LBB11_3
+; SSE42-NEXT:  LBB11_4: ## %else2
+; SSE42-NEXT:    testb $4, %al
+; SSE42-NEXT:    jne LBB11_5
 ; SSE42-NEXT:  LBB11_6: ## %else5
-; SSE42-NEXT:    pextrb $12, %xmm0, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB11_8
-; SSE42-NEXT:  ## %bb.7: ## %cond.load7
-; SSE42-NEXT:    insertps {{.*#+}} xmm2 = xmm2[0,1,2],mem[0]
+; SSE42-NEXT:    testb $8, %al
+; SSE42-NEXT:    jne LBB11_7
 ; SSE42-NEXT:  LBB11_8: ## %else8
-; SSE42-NEXT:    pxor %xmm0, %xmm0
-; SSE42-NEXT:    pcmpeqd %xmm1, %xmm0
-; SSE42-NEXT:    pextrb $0, %xmm0, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB11_10
-; SSE42-NEXT:  ## %bb.9: ## %cond.load10
-; SSE42-NEXT:    movd {{.*#+}} xmm4 = mem[0],zero,zero,zero
-; SSE42-NEXT:    pblendw {{.*#+}} xmm3 = xmm4[0,1],xmm3[2,3,4,5,6,7]
+; SSE42-NEXT:    testb $16, %al
+; SSE42-NEXT:    jne LBB11_9
 ; SSE42-NEXT:  LBB11_10: ## %else11
-; SSE42-NEXT:    pextrb $4, %xmm0, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB11_12
-; SSE42-NEXT:  ## %bb.11: ## %cond.load13
-; SSE42-NEXT:    insertps {{.*#+}} xmm3 = xmm3[0],mem[0],xmm3[2,3]
+; SSE42-NEXT:    testb $32, %al
+; SSE42-NEXT:    jne LBB11_11
 ; SSE42-NEXT:  LBB11_12: ## %else14
-; SSE42-NEXT:    pxor %xmm0, %xmm0
-; SSE42-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE42-NEXT:    pextrb $8, %xmm1, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB11_14
-; SSE42-NEXT:  ## %bb.13: ## %cond.load16
-; SSE42-NEXT:    insertps {{.*#+}} xmm3 = xmm3[0,1],mem[0],xmm3[3]
+; SSE42-NEXT:    testb $64, %al
+; SSE42-NEXT:    jne LBB11_13
 ; SSE42-NEXT:  LBB11_14: ## %else17
-; SSE42-NEXT:    pextrb $12, %xmm1, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $-128, %al
 ; SSE42-NEXT:    je LBB11_16
-; SSE42-NEXT:  ## %bb.15: ## %cond.load19
+; SSE42-NEXT:  LBB11_15: ## %cond.load19
 ; SSE42-NEXT:    insertps {{.*#+}} xmm3 = xmm3[0,1,2],mem[0]
 ; SSE42-NEXT:  LBB11_16: ## %else20
 ; SSE42-NEXT:    movaps %xmm2, %xmm0
 ; SSE42-NEXT:    movaps %xmm3, %xmm1
 ; SSE42-NEXT:    retq
-;
-; AVX1-LABEL: load_v8f32_v8i32:
-; AVX1:       ## %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; SSE42-NEXT:  LBB11_1: ## %cond.load
+; SSE42-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE42-NEXT:    pblendw {{.*#+}} xmm2 = xmm0[0,1],xmm2[2,3,4,5,6,7]
+; SSE42-NEXT:    testb $2, %al
+; SSE42-NEXT:    je LBB11_4
+; SSE42-NEXT:  LBB11_3: ## %cond.load1
+; SSE42-NEXT:    insertps {{.*#+}} xmm2 = xmm2[0],mem[0],xmm2[2,3]
+; SSE42-NEXT:    testb $4, %al
+; SSE42-NEXT:    je LBB11_6
+; SSE42-NEXT:  LBB11_5: ## %cond.load4
+; SSE42-NEXT:    insertps {{.*#+}} xmm2 = xmm2[0,1],mem[0],xmm2[3]
+; SSE42-NEXT:    testb $8, %al
+; SSE42-NEXT:    je LBB11_8
+; SSE42-NEXT:  LBB11_7: ## %cond.load7
+; SSE42-NEXT:    insertps {{.*#+}} xmm2 = xmm2[0,1,2],mem[0]
+; SSE42-NEXT:    testb $16, %al
+; SSE42-NEXT:    je LBB11_10
+; SSE42-NEXT:  LBB11_9: ## %cond.load10
+; SSE42-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE42-NEXT:    pblendw {{.*#+}} xmm3 = xmm0[0,1],xmm3[2,3,4,5,6,7]
+; SSE42-NEXT:    testb $32, %al
+; SSE42-NEXT:    je LBB11_12
+; SSE42-NEXT:  LBB11_11: ## %cond.load13
+; SSE42-NEXT:    insertps {{.*#+}} xmm3 = xmm3[0],mem[0],xmm3[2,3]
+; SSE42-NEXT:    testb $64, %al
+; SSE42-NEXT:    je LBB11_14
+; SSE42-NEXT:  LBB11_13: ## %cond.load16
+; SSE42-NEXT:    insertps {{.*#+}} xmm3 = xmm3[0,1],mem[0],xmm3[3]
+; SSE42-NEXT:    testb $-128, %al
+; SSE42-NEXT:    jne LBB11_15
+; SSE42-NEXT:    jmp LBB11_16
+;
+; AVX1-LABEL: load_v8f32_v8i32:
+; AVX1:       ## %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; AVX1-NEXT:    vmaskmovps (%rdi), %ymm0, %ymm2
 ; AVX1-NEXT:    vblendvps %ymm0, %ymm2, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
@@ -1507,19 +1438,22 @@ define <2 x i64> @load_v2i64_v2i64(<2 x i64> %trigger, <2 x i64>* %addr, <2 x i6
 ; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,0,3,2]
 ; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    movmskpd %xmm0, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB13_2
-; SSE2-NEXT:  ## %bb.1: ## %cond.load
+; SSE2-NEXT:    jne LBB13_1
+; SSE2-NEXT:  ## %bb.2: ## %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne LBB13_3
+; SSE2-NEXT:  LBB13_4: ## %else2
+; SSE2-NEXT:    movaps %xmm1, %xmm0
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  LBB13_1: ## %cond.load
 ; SSE2-NEXT:    movlps {{.*#+}} xmm1 = mem[0,1],xmm1[2,3]
-; SSE2-NEXT:  LBB13_2: ## %else
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $2, %al
 ; SSE2-NEXT:    je LBB13_4
-; SSE2-NEXT:  ## %bb.3: ## %cond.load1
+; SSE2-NEXT:  LBB13_3: ## %cond.load1
 ; SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; SSE2-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0]
-; SSE2-NEXT:  LBB13_4: ## %else2
 ; SSE2-NEXT:    movaps %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
@@ -1527,18 +1461,21 @@ define <2 x i64> @load_v2i64_v2i64(<2 x i64> %trigger, <2 x i64>* %addr, <2 x i6
 ; SSE42:       ## %bb.0:
 ; SSE42-NEXT:    pxor %xmm2, %xmm2
 ; SSE42-NEXT:    pcmpeqq %xmm0, %xmm2
-; SSE42-NEXT:    pextrb $0, %xmm2, %eax
+; SSE42-NEXT:    movmskpd %xmm2, %eax
 ; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB13_2
-; SSE42-NEXT:  ## %bb.1: ## %cond.load
+; SSE42-NEXT:    jne LBB13_1
+; SSE42-NEXT:  ## %bb.2: ## %else
+; SSE42-NEXT:    testb $2, %al
+; SSE42-NEXT:    jne LBB13_3
+; SSE42-NEXT:  LBB13_4: ## %else2
+; SSE42-NEXT:    movdqa %xmm1, %xmm0
+; SSE42-NEXT:    retq
+; SSE42-NEXT:  LBB13_1: ## %cond.load
 ; SSE42-NEXT:    pinsrq $0, (%rdi), %xmm1
-; SSE42-NEXT:  LBB13_2: ## %else
-; SSE42-NEXT:    pextrb $8, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $2, %al
 ; SSE42-NEXT:    je LBB13_4
-; SSE42-NEXT:  ## %bb.3: ## %cond.load1
+; SSE42-NEXT:  LBB13_3: ## %cond.load1
 ; SSE42-NEXT:    pinsrq $1, 8(%rdi), %xmm1
-; SSE42-NEXT:  LBB13_4: ## %else2
 ; SSE42-NEXT:    movdqa %xmm1, %xmm0
 ; SSE42-NEXT:    retq
 ;
@@ -1584,76 +1521,84 @@ define <4 x i64> @load_v4i64_v4i64(<4 x i64> %trigger, <4 x i64>* %addr, <4 x i6
 ; SSE2-LABEL: load_v4i64_v4i64:
 ; SSE2:       ## %bb.0:
 ; SSE2-NEXT:    pxor %xmm4, %xmm4
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[1,0,3,2]
-; SSE2-NEXT:    pand %xmm4, %xmm0
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB14_2
-; SSE2-NEXT:  ## %bb.1: ## %cond.load
-; SSE2-NEXT:    movlps {{.*#+}} xmm2 = mem[0,1],xmm2[2,3]
-; SSE2-NEXT:  LBB14_2: ## %else
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
+; SSE2-NEXT:    pcmpeqd %xmm4, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[1,0,3,2]
+; SSE2-NEXT:    pand %xmm1, %xmm5
+; SSE2-NEXT:    pcmpeqd %xmm4, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,0,3,2]
+; SSE2-NEXT:    pand %xmm0, %xmm1
+; SSE2-NEXT:    packssdw %xmm5, %xmm1
+; SSE2-NEXT:    movmskps %xmm1, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB14_4
-; SSE2-NEXT:  ## %bb.3: ## %cond.load1
-; SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; SSE2-NEXT:    movlhps {{.*#+}} xmm2 = xmm2[0],xmm0[0]
+; SSE2-NEXT:    jne LBB14_1
+; SSE2-NEXT:  ## %bb.2: ## %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne LBB14_3
 ; SSE2-NEXT:  LBB14_4: ## %else2
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,0,3,2]
-; SSE2-NEXT:    pand %xmm1, %xmm0
-; SSE2-NEXT:    pextrw $0, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB14_6
-; SSE2-NEXT:  ## %bb.5: ## %cond.load4
-; SSE2-NEXT:    movlps {{.*#+}} xmm3 = mem[0,1],xmm3[2,3]
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne LBB14_5
 ; SSE2-NEXT:  LBB14_6: ## %else5
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je LBB14_8
-; SSE2-NEXT:  ## %bb.7: ## %cond.load7
+; SSE2-NEXT:  LBB14_7: ## %cond.load7
 ; SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; SSE2-NEXT:    movlhps {{.*#+}} xmm3 = xmm3[0],xmm0[0]
 ; SSE2-NEXT:  LBB14_8: ## %else8
 ; SSE2-NEXT:    movaps %xmm2, %xmm0
 ; SSE2-NEXT:    movaps %xmm3, %xmm1
 ; SSE2-NEXT:    retq
+; SSE2-NEXT:  LBB14_1: ## %cond.load
+; SSE2-NEXT:    movlps {{.*#+}} xmm2 = mem[0,1],xmm2[2,3]
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    je LBB14_4
+; SSE2-NEXT:  LBB14_3: ## %cond.load1
+; SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE2-NEXT:    movlhps {{.*#+}} xmm2 = xmm2[0],xmm0[0]
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    je LBB14_6
+; SSE2-NEXT:  LBB14_5: ## %cond.load4
+; SSE2-NEXT:    movlps {{.*#+}} xmm3 = mem[0,1],xmm3[2,3]
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne LBB14_7
+; SSE2-NEXT:    jmp LBB14_8
 ;
 ; SSE42-LABEL: load_v4i64_v4i64:
 ; SSE42:       ## %bb.0:
 ; SSE42-NEXT:    pxor %xmm4, %xmm4
-; SSE42-NEXT:    pcmpeqq %xmm0, %xmm4
-; SSE42-NEXT:    pextrb $0, %xmm4, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB14_2
-; SSE42-NEXT:  ## %bb.1: ## %cond.load
-; SSE42-NEXT:    pinsrq $0, (%rdi), %xmm2
-; SSE42-NEXT:  LBB14_2: ## %else
-; SSE42-NEXT:    pextrb $8, %xmm4, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB14_4
-; SSE42-NEXT:  ## %bb.3: ## %cond.load1
-; SSE42-NEXT:    pinsrq $1, 8(%rdi), %xmm2
+; SSE42-NEXT:    pcmpeqq %xmm4, %xmm1
+; SSE42-NEXT:    pcmpeqq %xmm4, %xmm0
+; SSE42-NEXT:    packssdw %xmm1, %xmm0
+; SSE42-NEXT:    movmskps %xmm0, %eax
+; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    jne LBB14_1
+; SSE42-NEXT:  ## %bb.2: ## %else
+; SSE42-NEXT:    testb $2, %al
+; SSE42-NEXT:    jne LBB14_3
 ; SSE42-NEXT:  LBB14_4: ## %else2
-; SSE42-NEXT:    pxor %xmm0, %xmm0
-; SSE42-NEXT:    pcmpeqq %xmm0, %xmm1
-; SSE42-NEXT:    pextrb $0, %xmm1, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB14_6
-; SSE42-NEXT:  ## %bb.5: ## %cond.load4
-; SSE42-NEXT:    pinsrq $0, 16(%rdi), %xmm3
+; SSE42-NEXT:    testb $4, %al
+; SSE42-NEXT:    jne LBB14_5
 ; SSE42-NEXT:  LBB14_6: ## %else5
-; SSE42-NEXT:    pextrb $8, %xmm1, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $8, %al
 ; SSE42-NEXT:    je LBB14_8
-; SSE42-NEXT:  ## %bb.7: ## %cond.load7
+; SSE42-NEXT:  LBB14_7: ## %cond.load7
 ; SSE42-NEXT:    pinsrq $1, 24(%rdi), %xmm3
 ; SSE42-NEXT:  LBB14_8: ## %else8
 ; SSE42-NEXT:    movdqa %xmm2, %xmm0
 ; SSE42-NEXT:    movdqa %xmm3, %xmm1
 ; SSE42-NEXT:    retq
+; SSE42-NEXT:  LBB14_1: ## %cond.load
+; SSE42-NEXT:    pinsrq $0, (%rdi), %xmm2
+; SSE42-NEXT:    testb $2, %al
+; SSE42-NEXT:    je LBB14_4
+; SSE42-NEXT:  LBB14_3: ## %cond.load1
+; SSE42-NEXT:    pinsrq $1, 8(%rdi), %xmm2
+; SSE42-NEXT:    testb $4, %al
+; SSE42-NEXT:    je LBB14_6
+; SSE42-NEXT:  LBB14_5: ## %cond.load4
+; SSE42-NEXT:    pinsrq $0, 16(%rdi), %xmm3
+; SSE42-NEXT:    testb $8, %al
+; SSE42-NEXT:    jne LBB14_7
+; SSE42-NEXT:    jmp LBB14_8
 ;
 ; AVX1-LABEL: load_v4i64_v4i64:
 ; AVX1:       ## %bb.0:
@@ -1700,61 +1645,32 @@ define <8 x i64> @load_v8i64_v8i16(<8 x i16> %trigger, <8 x i64>* %addr, <8 x i6
 ; SSE2:       ## %bb.0:
 ; SSE2-NEXT:    pxor %xmm5, %xmm5
 ; SSE2-NEXT:    pcmpeqw %xmm0, %xmm5
-; SSE2-NEXT:    movd %xmm5, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB15_2
-; SSE2-NEXT:  ## %bb.1: ## %cond.load
-; SSE2-NEXT:    movlps {{.*#+}} xmm1 = mem[0,1],xmm1[2,3]
-; SSE2-NEXT:  LBB15_2: ## %else
-; SSE2-NEXT:    shrl $16, %eax
+; SSE2-NEXT:    packsswb %xmm0, %xmm5
+; SSE2-NEXT:    pmovmskb %xmm5, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB15_4
-; SSE2-NEXT:  ## %bb.3: ## %cond.load1
-; SSE2-NEXT:    movsd {{.*#+}} xmm5 = mem[0],zero
-; SSE2-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm5[0]
+; SSE2-NEXT:    jne LBB15_1
+; SSE2-NEXT:  ## %bb.2: ## %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne LBB15_3
 ; SSE2-NEXT:  LBB15_4: ## %else2
-; SSE2-NEXT:    pxor %xmm5, %xmm5
-; SSE2-NEXT:    pcmpeqw %xmm0, %xmm5
-; SSE2-NEXT:    pextrw $2, %xmm5, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB15_6
-; SSE2-NEXT:  ## %bb.5: ## %cond.load4
-; SSE2-NEXT:    movlps {{.*#+}} xmm2 = mem[0,1],xmm2[2,3]
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne LBB15_5
 ; SSE2-NEXT:  LBB15_6: ## %else5
-; SSE2-NEXT:    pextrw $3, %xmm5, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB15_8
-; SSE2-NEXT:  ## %bb.7: ## %cond.load7
-; SSE2-NEXT:    movsd {{.*#+}} xmm5 = mem[0],zero
-; SSE2-NEXT:    movlhps {{.*#+}} xmm2 = xmm2[0],xmm5[0]
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne LBB15_7
 ; SSE2-NEXT:  LBB15_8: ## %else8
-; SSE2-NEXT:    pxor %xmm5, %xmm5
-; SSE2-NEXT:    pcmpeqw %xmm0, %xmm5
-; SSE2-NEXT:    pextrw $4, %xmm5, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB15_10
-; SSE2-NEXT:  ## %bb.9: ## %cond.load10
-; SSE2-NEXT:    movlps {{.*#+}} xmm3 = mem[0,1],xmm3[2,3]
+; SSE2-NEXT:    testb $16, %al
+; SSE2-NEXT:    jne LBB15_9
 ; SSE2-NEXT:  LBB15_10: ## %else11
-; SSE2-NEXT:    pextrw $5, %xmm5, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB15_12
-; SSE2-NEXT:  ## %bb.11: ## %cond.load13
-; SSE2-NEXT:    movsd {{.*#+}} xmm5 = mem[0],zero
-; SSE2-NEXT:    movlhps {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; SSE2-NEXT:    testb $32, %al
+; SSE2-NEXT:    jne LBB15_11
 ; SSE2-NEXT:  LBB15_12: ## %else14
-; SSE2-NEXT:    pxor %xmm5, %xmm5
-; SSE2-NEXT:    pcmpeqw %xmm5, %xmm0
-; SSE2-NEXT:    pextrw $6, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB15_14
-; SSE2-NEXT:  ## %bb.13: ## %cond.load16
-; SSE2-NEXT:    movlps {{.*#+}} xmm4 = mem[0,1],xmm4[2,3]
+; SSE2-NEXT:    testb $64, %al
+; SSE2-NEXT:    jne LBB15_13
 ; SSE2-NEXT:  LBB15_14: ## %else17
-; SSE2-NEXT:    pextrw $7, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $-128, %al
 ; SSE2-NEXT:    je LBB15_16
-; SSE2-NEXT:  ## %bb.15: ## %cond.load19
+; SSE2-NEXT:  LBB15_15: ## %cond.load19
 ; SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; SSE2-NEXT:    movlhps {{.*#+}} xmm4 = xmm4[0],xmm0[0]
 ; SSE2-NEXT:  LBB15_16: ## %else20
@@ -1763,63 +1679,69 @@ define <8 x i64> @load_v8i64_v8i16(<8 x i16> %trigger, <8 x i64>* %addr, <8 x i6
 ; SSE2-NEXT:    movaps %xmm3, %xmm2
 ; SSE2-NEXT:    movaps %xmm4, %xmm3
 ; SSE2-NEXT:    retq
+; SSE2-NEXT:  LBB15_1: ## %cond.load
+; SSE2-NEXT:    movlps {{.*#+}} xmm1 = mem[0,1],xmm1[2,3]
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    je LBB15_4
+; SSE2-NEXT:  LBB15_3: ## %cond.load1
+; SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE2-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    je LBB15_6
+; SSE2-NEXT:  LBB15_5: ## %cond.load4
+; SSE2-NEXT:    movlps {{.*#+}} xmm2 = mem[0,1],xmm2[2,3]
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    je LBB15_8
+; SSE2-NEXT:  LBB15_7: ## %cond.load7
+; SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE2-NEXT:    movlhps {{.*#+}} xmm2 = xmm2[0],xmm0[0]
+; SSE2-NEXT:    testb $16, %al
+; SSE2-NEXT:    je LBB15_10
+; SSE2-NEXT:  LBB15_9: ## %cond.load10
+; SSE2-NEXT:    movlps {{.*#+}} xmm3 = mem[0,1],xmm3[2,3]
+; SSE2-NEXT:    testb $32, %al
+; SSE2-NEXT:    je LBB15_12
+; SSE2-NEXT:  LBB15_11: ## %cond.load13
+; SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE2-NEXT:    movlhps {{.*#+}} xmm3 = xmm3[0],xmm0[0]
+; SSE2-NEXT:    testb $64, %al
+; SSE2-NEXT:    je LBB15_14
+; SSE2-NEXT:  LBB15_13: ## %cond.load16
+; SSE2-NEXT:    movlps {{.*#+}} xmm4 = mem[0,1],xmm4[2,3]
+; SSE2-NEXT:    testb $-128, %al
+; SSE2-NEXT:    jne LBB15_15
+; SSE2-NEXT:    jmp LBB15_16
 ;
 ; SSE42-LABEL: load_v8i64_v8i16:
 ; SSE42:       ## %bb.0:
 ; SSE42-NEXT:    pxor %xmm5, %xmm5
 ; SSE42-NEXT:    pcmpeqw %xmm0, %xmm5
-; SSE42-NEXT:    pextrb $0, %xmm5, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB15_2
-; SSE42-NEXT:  ## %bb.1: ## %cond.load
-; SSE42-NEXT:    pinsrq $0, (%rdi), %xmm1
-; SSE42-NEXT:  LBB15_2: ## %else
-; SSE42-NEXT:    pextrb $2, %xmm5, %eax
+; SSE42-NEXT:    packsswb %xmm0, %xmm5
+; SSE42-NEXT:    pmovmskb %xmm5, %eax
 ; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB15_4
-; SSE42-NEXT:  ## %bb.3: ## %cond.load1
-; SSE42-NEXT:    pinsrq $1, 8(%rdi), %xmm1
+; SSE42-NEXT:    jne LBB15_1
+; SSE42-NEXT:  ## %bb.2: ## %else
+; SSE42-NEXT:    testb $2, %al
+; SSE42-NEXT:    jne LBB15_3
 ; SSE42-NEXT:  LBB15_4: ## %else2
-; SSE42-NEXT:    pxor %xmm5, %xmm5
-; SSE42-NEXT:    pcmpeqw %xmm0, %xmm5
-; SSE42-NEXT:    pextrb $4, %xmm5, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB15_6
-; SSE42-NEXT:  ## %bb.5: ## %cond.load4
-; SSE42-NEXT:    pinsrq $0, 16(%rdi), %xmm2
+; SSE42-NEXT:    testb $4, %al
+; SSE42-NEXT:    jne LBB15_5
 ; SSE42-NEXT:  LBB15_6: ## %else5
-; SSE42-NEXT:    pextrb $6, %xmm5, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB15_8
-; SSE42-NEXT:  ## %bb.7: ## %cond.load7
-; SSE42-NEXT:    pinsrq $1, 24(%rdi), %xmm2
+; SSE42-NEXT:    testb $8, %al
+; SSE42-NEXT:    jne LBB15_7
 ; SSE42-NEXT:  LBB15_8: ## %else8
-; SSE42-NEXT:    pxor %xmm5, %xmm5
-; SSE42-NEXT:    pcmpeqw %xmm0, %xmm5
-; SSE42-NEXT:    pextrb $8, %xmm5, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB15_10
-; SSE42-NEXT:  ## %bb.9: ## %cond.load10
-; SSE42-NEXT:    pinsrq $0, 32(%rdi), %xmm3
+; SSE42-NEXT:    testb $16, %al
+; SSE42-NEXT:    jne LBB15_9
 ; SSE42-NEXT:  LBB15_10: ## %else11
-; SSE42-NEXT:    pextrb $10, %xmm5, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB15_12
-; SSE42-NEXT:  ## %bb.11: ## %cond.load13
-; SSE42-NEXT:    pinsrq $1, 40(%rdi), %xmm3
+; SSE42-NEXT:    testb $32, %al
+; SSE42-NEXT:    jne LBB15_11
 ; SSE42-NEXT:  LBB15_12: ## %else14
-; SSE42-NEXT:    pxor %xmm5, %xmm5
-; SSE42-NEXT:    pcmpeqw %xmm5, %xmm0
-; SSE42-NEXT:    pextrb $12, %xmm0, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB15_14
-; SSE42-NEXT:  ## %bb.13: ## %cond.load16
-; SSE42-NEXT:    pinsrq $0, 48(%rdi), %xmm4
+; SSE42-NEXT:    testb $64, %al
+; SSE42-NEXT:    jne LBB15_13
 ; SSE42-NEXT:  LBB15_14: ## %else17
-; SSE42-NEXT:    pextrb $14, %xmm0, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $-128, %al
 ; SSE42-NEXT:    je LBB15_16
-; SSE42-NEXT:  ## %bb.15: ## %cond.load19
+; SSE42-NEXT:  LBB15_15: ## %cond.load19
 ; SSE42-NEXT:    pinsrq $1, 56(%rdi), %xmm4
 ; SSE42-NEXT:  LBB15_16: ## %else20
 ; SSE42-NEXT:    movdqa %xmm1, %xmm0
@@ -1827,41 +1749,70 @@ define <8 x i64> @load_v8i64_v8i16(<8 x i16> %trigger, <8 x i64>* %addr, <8 x i6
 ; SSE42-NEXT:    movdqa %xmm3, %xmm2
 ; SSE42-NEXT:    movdqa %xmm4, %xmm3
 ; SSE42-NEXT:    retq
+; SSE42-NEXT:  LBB15_1: ## %cond.load
+; SSE42-NEXT:    pinsrq $0, (%rdi), %xmm1
+; SSE42-NEXT:    testb $2, %al
+; SSE42-NEXT:    je LBB15_4
+; SSE42-NEXT:  LBB15_3: ## %cond.load1
+; SSE42-NEXT:    pinsrq $1, 8(%rdi), %xmm1
+; SSE42-NEXT:    testb $4, %al
+; SSE42-NEXT:    je LBB15_6
+; SSE42-NEXT:  LBB15_5: ## %cond.load4
+; SSE42-NEXT:    pinsrq $0, 16(%rdi), %xmm2
+; SSE42-NEXT:    testb $8, %al
+; SSE42-NEXT:    je LBB15_8
+; SSE42-NEXT:  LBB15_7: ## %cond.load7
+; SSE42-NEXT:    pinsrq $1, 24(%rdi), %xmm2
+; SSE42-NEXT:    testb $16, %al
+; SSE42-NEXT:    je LBB15_10
+; SSE42-NEXT:  LBB15_9: ## %cond.load10
+; SSE42-NEXT:    pinsrq $0, 32(%rdi), %xmm3
+; SSE42-NEXT:    testb $32, %al
+; SSE42-NEXT:    je LBB15_12
+; SSE42-NEXT:  LBB15_11: ## %cond.load13
+; SSE42-NEXT:    pinsrq $1, 40(%rdi), %xmm3
+; SSE42-NEXT:    testb $64, %al
+; SSE42-NEXT:    je LBB15_14
+; SSE42-NEXT:  LBB15_13: ## %cond.load16
+; SSE42-NEXT:    pinsrq $0, 48(%rdi), %xmm4
+; SSE42-NEXT:    testb $-128, %al
+; SSE42-NEXT:    jne LBB15_15
+; SSE42-NEXT:    jmp LBB15_16
 ;
 ; AVX1-LABEL: load_v8i64_v8i16:
 ; AVX1:       ## %bb.0:
-; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7]
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm4, %xmm4
-; AVX1-NEXT:    vpmovsxdq %xmm4, %xmm5
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[2,3,0,1]
-; AVX1-NEXT:    vpmovsxdq %xmm4, %xmm4
-; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm5, %ymm4
-; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm0, %xmm0
-; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm3
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm3, %ymm0
-; AVX1-NEXT:    vmaskmovpd (%rdi), %ymm0, %ymm3
-; AVX1-NEXT:    vblendvpd %ymm0, %ymm3, %ymm1, %ymm0
-; AVX1-NEXT:    vmaskmovpd 32(%rdi), %ymm4, %ymm1
-; AVX1-NEXT:    vblendvpd %ymm4, %ymm1, %ymm2, %ymm1
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; AVX1-NEXT:    vpcmpeqw %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpmovsxwd %xmm3, %xmm3
+; AVX1-NEXT:    vpmovsxdq %xmm3, %xmm5
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,3,0,1]
+; AVX1-NEXT:    vpmovsxdq %xmm3, %xmm3
+; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm5, %ymm3
+; AVX1-NEXT:    vpcmpeqw %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vpmovsxwq %xmm0, %xmm4
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpmovsxwq %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm4, %ymm0
+; AVX1-NEXT:    vmaskmovpd (%rdi), %ymm0, %ymm4
+; AVX1-NEXT:    vblendvpd %ymm0, %ymm4, %ymm1, %ymm0
+; AVX1-NEXT:    vmaskmovpd 32(%rdi), %ymm3, %ymm1
+; AVX1-NEXT:    vblendvpd %ymm3, %ymm1, %ymm2, %ymm1
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: load_v8i64_v8i16:
 ; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7]
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm4, %xmm4
-; AVX2-NEXT:    vpmovsxdq %xmm4, %ymm4
-; AVX2-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm0, %xmm0
-; AVX2-NEXT:    vpmovsxdq %xmm0, %ymm0
-; AVX2-NEXT:    vpmaskmovq (%rdi), %ymm0, %ymm3
-; AVX2-NEXT:    vblendvpd %ymm0, %ymm3, %ymm1, %ymm0
-; AVX2-NEXT:    vpmaskmovq 32(%rdi), %ymm4, %ymm1
-; AVX2-NEXT:    vblendvpd %ymm4, %ymm1, %ymm2, %ymm1
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; AVX2-NEXT:    vpcmpeqw %xmm4, %xmm3, %xmm3
+; AVX2-NEXT:    vpmovsxwd %xmm3, %xmm3
+; AVX2-NEXT:    vpmovsxdq %xmm3, %ymm3
+; AVX2-NEXT:    vpcmpeqw %xmm4, %xmm0, %xmm0
+; AVX2-NEXT:    vpmovsxwq %xmm0, %ymm0
+; AVX2-NEXT:    vpmaskmovq (%rdi), %ymm0, %ymm4
+; AVX2-NEXT:    vblendvpd %ymm0, %ymm4, %ymm1, %ymm0
+; AVX2-NEXT:    vpmaskmovq 32(%rdi), %ymm3, %ymm1
+; AVX2-NEXT:    vblendvpd %ymm3, %ymm1, %ymm2, %ymm1
 ; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: load_v8i64_v8i16:
@@ -1896,139 +1847,127 @@ define <8 x i64> @load_v8i64_v8i64(<8 x i64> %trigger, <8 x i64>* %addr, <8 x i6
 ; SSE2-LABEL: load_v8i64_v8i64:
 ; SSE2:       ## %bb.0:
 ; SSE2-NEXT:    movdqa %xmm7, %xmm8
+; SSE2-NEXT:    movaps %xmm6, %xmm9
 ; SSE2-NEXT:    pxor %xmm7, %xmm7
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[1,0,3,2]
-; SSE2-NEXT:    pand %xmm7, %xmm0
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB16_2
-; SSE2-NEXT:  ## %bb.1: ## %cond.load
-; SSE2-NEXT:    movlps {{.*#+}} xmm4 = mem[0,1],xmm4[2,3]
-; SSE2-NEXT:  LBB16_2: ## %else
-; SSE2-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    shrl $16, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB16_4
-; SSE2-NEXT:  ## %bb.3: ## %cond.load1
-; SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; SSE2-NEXT:    movlhps {{.*#+}} xmm4 = xmm4[0],xmm0[0]
+; SSE2-NEXT:    pcmpeqd %xmm7, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[1,0,3,2]
+; SSE2-NEXT:    pand %xmm3, %xmm6
+; SSE2-NEXT:    pcmpeqd %xmm7, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,0,3,2]
+; SSE2-NEXT:    pand %xmm2, %xmm3
+; SSE2-NEXT:    packssdw %xmm6, %xmm3
+; SSE2-NEXT:    pcmpeqd %xmm7, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,0,3,2]
+; SSE2-NEXT:    pand %xmm1, %xmm2
+; SSE2-NEXT:    pcmpeqd %xmm7, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,0,3,2]
+; SSE2-NEXT:    pand %xmm0, %xmm1
+; SSE2-NEXT:    packssdw %xmm2, %xmm1
+; SSE2-NEXT:    packssdw %xmm3, %xmm1
+; SSE2-NEXT:    packsswb %xmm0, %xmm1
+; SSE2-NEXT:    pmovmskb %xmm1, %eax
+; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    jne LBB16_1
+; SSE2-NEXT:  ## %bb.2: ## %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne LBB16_3
 ; SSE2-NEXT:  LBB16_4: ## %else2
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,0,3,2]
-; SSE2-NEXT:    pand %xmm1, %xmm0
-; SSE2-NEXT:    pextrw $0, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB16_6
-; SSE2-NEXT:  ## %bb.5: ## %cond.load4
-; SSE2-NEXT:    movlps {{.*#+}} xmm5 = mem[0,1],xmm5[2,3]
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne LBB16_5
 ; SSE2-NEXT:  LBB16_6: ## %else5
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB16_8
-; SSE2-NEXT:  ## %bb.7: ## %cond.load7
-; SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; SSE2-NEXT:    movlhps {{.*#+}} xmm5 = xmm5[0],xmm0[0]
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne LBB16_7
 ; SSE2-NEXT:  LBB16_8: ## %else8
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,0,3,2]
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    pextrw $0, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB16_10
-; SSE2-NEXT:  ## %bb.9: ## %cond.load10
-; SSE2-NEXT:    movlps {{.*#+}} xmm6 = mem[0,1],xmm6[2,3]
+; SSE2-NEXT:    testb $16, %al
+; SSE2-NEXT:    jne LBB16_9
 ; SSE2-NEXT:  LBB16_10: ## %else11
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB16_12
-; SSE2-NEXT:  ## %bb.11: ## %cond.load13
-; SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; SSE2-NEXT:    movlhps {{.*#+}} xmm6 = xmm6[0],xmm0[0]
+; SSE2-NEXT:    testb $32, %al
+; SSE2-NEXT:    jne LBB16_11
 ; SSE2-NEXT:  LBB16_12: ## %else14
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,0,3,2]
-; SSE2-NEXT:    pand %xmm3, %xmm0
-; SSE2-NEXT:    pextrw $0, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB16_14
-; SSE2-NEXT:  ## %bb.13: ## %cond.load16
-; SSE2-NEXT:    movlps {{.*#+}} xmm8 = mem[0,1],xmm8[2,3]
+; SSE2-NEXT:    testb $64, %al
+; SSE2-NEXT:    jne LBB16_13
 ; SSE2-NEXT:  LBB16_14: ## %else17
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $-128, %al
 ; SSE2-NEXT:    je LBB16_16
-; SSE2-NEXT:  ## %bb.15: ## %cond.load19
+; SSE2-NEXT:  LBB16_15: ## %cond.load19
 ; SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
 ; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm8 = xmm8[0],xmm0[0]
 ; SSE2-NEXT:  LBB16_16: ## %else20
 ; SSE2-NEXT:    movaps %xmm4, %xmm0
 ; SSE2-NEXT:    movaps %xmm5, %xmm1
-; SSE2-NEXT:    movaps %xmm6, %xmm2
+; SSE2-NEXT:    movaps %xmm9, %xmm2
 ; SSE2-NEXT:    movdqa %xmm8, %xmm3
 ; SSE2-NEXT:    retq
+; SSE2-NEXT:  LBB16_1: ## %cond.load
+; SSE2-NEXT:    movlps {{.*#+}} xmm4 = mem[0,1],xmm4[2,3]
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    je LBB16_4
+; SSE2-NEXT:  LBB16_3: ## %cond.load1
+; SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE2-NEXT:    movlhps {{.*#+}} xmm4 = xmm4[0],xmm0[0]
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    je LBB16_6
+; SSE2-NEXT:  LBB16_5: ## %cond.load4
+; SSE2-NEXT:    movlps {{.*#+}} xmm5 = mem[0,1],xmm5[2,3]
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    je LBB16_8
+; SSE2-NEXT:  LBB16_7: ## %cond.load7
+; SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE2-NEXT:    movlhps {{.*#+}} xmm5 = xmm5[0],xmm0[0]
+; SSE2-NEXT:    testb $16, %al
+; SSE2-NEXT:    je LBB16_10
+; SSE2-NEXT:  LBB16_9: ## %cond.load10
+; SSE2-NEXT:    movlps {{.*#+}} xmm9 = mem[0,1],xmm9[2,3]
+; SSE2-NEXT:    testb $32, %al
+; SSE2-NEXT:    je LBB16_12
+; SSE2-NEXT:  LBB16_11: ## %cond.load13
+; SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE2-NEXT:    movlhps {{.*#+}} xmm9 = xmm9[0],xmm0[0]
+; SSE2-NEXT:    testb $64, %al
+; SSE2-NEXT:    je LBB16_14
+; SSE2-NEXT:  LBB16_13: ## %cond.load16
+; SSE2-NEXT:    movlps {{.*#+}} xmm8 = mem[0,1],xmm8[2,3]
+; SSE2-NEXT:    testb $-128, %al
+; SSE2-NEXT:    jne LBB16_15
+; SSE2-NEXT:    jmp LBB16_16
 ;
 ; SSE42-LABEL: load_v8i64_v8i64:
 ; SSE42:       ## %bb.0:
 ; SSE42-NEXT:    movdqa %xmm7, %xmm8
 ; SSE42-NEXT:    pxor %xmm7, %xmm7
-; SSE42-NEXT:    pcmpeqq %xmm0, %xmm7
-; SSE42-NEXT:    pextrb $0, %xmm7, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB16_2
-; SSE42-NEXT:  ## %bb.1: ## %cond.load
-; SSE42-NEXT:    pinsrq $0, (%rdi), %xmm4
-; SSE42-NEXT:  LBB16_2: ## %else
-; SSE42-NEXT:    pextrb $8, %xmm7, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB16_4
-; SSE42-NEXT:  ## %bb.3: ## %cond.load1
-; SSE42-NEXT:    pinsrq $1, 8(%rdi), %xmm4
+; SSE42-NEXT:    pcmpeqq %xmm7, %xmm3
+; SSE42-NEXT:    pcmpeqq %xmm7, %xmm2
+; SSE42-NEXT:    packssdw %xmm3, %xmm2
+; SSE42-NEXT:    pcmpeqq %xmm7, %xmm1
+; SSE42-NEXT:    pcmpeqq %xmm7, %xmm0
+; SSE42-NEXT:    packssdw %xmm1, %xmm0
+; SSE42-NEXT:    packssdw %xmm2, %xmm0
+; SSE42-NEXT:    packsswb %xmm0, %xmm0
+; SSE42-NEXT:    pmovmskb %xmm0, %eax
+; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    jne LBB16_1
+; SSE42-NEXT:  ## %bb.2: ## %else
+; SSE42-NEXT:    testb $2, %al
+; SSE42-NEXT:    jne LBB16_3
 ; SSE42-NEXT:  LBB16_4: ## %else2
-; SSE42-NEXT:    pxor %xmm0, %xmm0
-; SSE42-NEXT:    pcmpeqq %xmm0, %xmm1
-; SSE42-NEXT:    pextrb $0, %xmm1, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB16_6
-; SSE42-NEXT:  ## %bb.5: ## %cond.load4
-; SSE42-NEXT:    pinsrq $0, 16(%rdi), %xmm5
+; SSE42-NEXT:    testb $4, %al
+; SSE42-NEXT:    jne LBB16_5
 ; SSE42-NEXT:  LBB16_6: ## %else5
-; SSE42-NEXT:    pextrb $8, %xmm1, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB16_8
-; SSE42-NEXT:  ## %bb.7: ## %cond.load7
-; SSE42-NEXT:    pinsrq $1, 24(%rdi), %xmm5
+; SSE42-NEXT:    testb $8, %al
+; SSE42-NEXT:    jne LBB16_7
 ; SSE42-NEXT:  LBB16_8: ## %else8
-; SSE42-NEXT:    pxor %xmm0, %xmm0
-; SSE42-NEXT:    pcmpeqq %xmm0, %xmm2
-; SSE42-NEXT:    pextrb $0, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB16_10
-; SSE42-NEXT:  ## %bb.9: ## %cond.load10
-; SSE42-NEXT:    pinsrq $0, 32(%rdi), %xmm6
+; SSE42-NEXT:    testb $16, %al
+; SSE42-NEXT:    jne LBB16_9
 ; SSE42-NEXT:  LBB16_10: ## %else11
-; SSE42-NEXT:    pextrb $8, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB16_12
-; SSE42-NEXT:  ## %bb.11: ## %cond.load13
-; SSE42-NEXT:    pinsrq $1, 40(%rdi), %xmm6
+; SSE42-NEXT:    testb $32, %al
+; SSE42-NEXT:    jne LBB16_11
 ; SSE42-NEXT:  LBB16_12: ## %else14
-; SSE42-NEXT:    pxor %xmm0, %xmm0
-; SSE42-NEXT:    pcmpeqq %xmm0, %xmm3
-; SSE42-NEXT:    pextrb $0, %xmm3, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB16_14
-; SSE42-NEXT:  ## %bb.13: ## %cond.load16
-; SSE42-NEXT:    pinsrq $0, 48(%rdi), %xmm8
+; SSE42-NEXT:    testb $64, %al
+; SSE42-NEXT:    jne LBB16_13
 ; SSE42-NEXT:  LBB16_14: ## %else17
-; SSE42-NEXT:    pextrb $8, %xmm3, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $-128, %al
 ; SSE42-NEXT:    je LBB16_16
-; SSE42-NEXT:  ## %bb.15: ## %cond.load19
+; SSE42-NEXT:  LBB16_15: ## %cond.load19
 ; SSE42-NEXT:    pinsrq $1, 56(%rdi), %xmm8
 ; SSE42-NEXT:  LBB16_16: ## %else20
 ; SSE42-NEXT:    movdqa %xmm4, %xmm0
@@ -2036,6 +1975,35 @@ define <8 x i64> @load_v8i64_v8i64(<8 x i64> %trigger, <8 x i64>* %addr, <8 x i6
 ; SSE42-NEXT:    movdqa %xmm6, %xmm2
 ; SSE42-NEXT:    movdqa %xmm8, %xmm3
 ; SSE42-NEXT:    retq
+; SSE42-NEXT:  LBB16_1: ## %cond.load
+; SSE42-NEXT:    pinsrq $0, (%rdi), %xmm4
+; SSE42-NEXT:    testb $2, %al
+; SSE42-NEXT:    je LBB16_4
+; SSE42-NEXT:  LBB16_3: ## %cond.load1
+; SSE42-NEXT:    pinsrq $1, 8(%rdi), %xmm4
+; SSE42-NEXT:    testb $4, %al
+; SSE42-NEXT:    je LBB16_6
+; SSE42-NEXT:  LBB16_5: ## %cond.load4
+; SSE42-NEXT:    pinsrq $0, 16(%rdi), %xmm5
+; SSE42-NEXT:    testb $8, %al
+; SSE42-NEXT:    je LBB16_8
+; SSE42-NEXT:  LBB16_7: ## %cond.load7
+; SSE42-NEXT:    pinsrq $1, 24(%rdi), %xmm5
+; SSE42-NEXT:    testb $16, %al
+; SSE42-NEXT:    je LBB16_10
+; SSE42-NEXT:  LBB16_9: ## %cond.load10
+; SSE42-NEXT:    pinsrq $0, 32(%rdi), %xmm6
+; SSE42-NEXT:    testb $32, %al
+; SSE42-NEXT:    je LBB16_12
+; SSE42-NEXT:  LBB16_11: ## %cond.load13
+; SSE42-NEXT:    pinsrq $1, 40(%rdi), %xmm6
+; SSE42-NEXT:    testb $64, %al
+; SSE42-NEXT:    je LBB16_14
+; SSE42-NEXT:  LBB16_13: ## %cond.load16
+; SSE42-NEXT:    pinsrq $0, 48(%rdi), %xmm8
+; SSE42-NEXT:    testb $-128, %al
+; SSE42-NEXT:    jne LBB16_15
+; SSE42-NEXT:    jmp LBB16_16
 ;
 ; AVX1-LABEL: load_v8i64_v8i64:
 ; AVX1:       ## %bb.0:
@@ -2082,98 +2050,99 @@ define <8 x i64> @load_v8i64_v8i64(<8 x i64> %trigger, <8 x i64>* %addr, <8 x i6
 define <2 x i32> @load_v2i32_v2i32(<2 x i32> %trigger, <2 x i32>* %addr, <2 x i32> %dst) {
 ; SSE2-LABEL: load_v2i32_v2i32:
 ; SSE2:       ## %bb.0:
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
 ; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,0,3,2]
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB17_2
-; SSE2-NEXT:  ## %bb.1: ## %cond.load
-; SSE2-NEXT:    movl (%rdi), %eax
-; SSE2-NEXT:    movq %rax, %xmm2
-; SSE2-NEXT:    movsd {{.*#+}} xmm1 = xmm2[0],xmm1[1]
-; SSE2-NEXT:  LBB17_2: ## %else
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,0,1,1]
+; SSE2-NEXT:    movmskpd %xmm0, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB17_4
-; SSE2-NEXT:  ## %bb.3: ## %cond.load1
-; SSE2-NEXT:    movl 4(%rdi), %eax
-; SSE2-NEXT:    movq %rax, %xmm0
-; SSE2-NEXT:    unpcklpd {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSE2-NEXT:    jne LBB17_1
+; SSE2-NEXT:  ## %bb.2: ## %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne LBB17_3
 ; SSE2-NEXT:  LBB17_4: ## %else2
-; SSE2-NEXT:    movapd %xmm1, %xmm0
+; SSE2-NEXT:    movaps %xmm1, %xmm0
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  LBB17_1: ## %cond.load
+; SSE2-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT:    movss {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3]
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    je LBB17_4
+; SSE2-NEXT:  LBB17_3: ## %cond.load1
+; SSE2-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,0],xmm1[0,0]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[2,0],xmm1[2,3]
+; SSE2-NEXT:    movaps %xmm0, %xmm1
+; SSE2-NEXT:    movaps %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE42-LABEL: load_v2i32_v2i32:
 ; SSE42:       ## %bb.0:
 ; SSE42-NEXT:    pxor %xmm2, %xmm2
-; SSE42-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
-; SSE42-NEXT:    pcmpeqq %xmm2, %xmm0
-; SSE42-NEXT:    pextrb $0, %xmm0, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB17_2
-; SSE42-NEXT:  ## %bb.1: ## %cond.load
-; SSE42-NEXT:    movl (%rdi), %eax
-; SSE42-NEXT:    pinsrq $0, %rax, %xmm1
-; SSE42-NEXT:  LBB17_2: ## %else
-; SSE42-NEXT:    pextrb $8, %xmm0, %eax
+; SSE42-NEXT:    pcmpeqd %xmm0, %xmm2
+; SSE42-NEXT:    pmovsxdq %xmm2, %xmm0
+; SSE42-NEXT:    movmskpd %xmm0, %eax
 ; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB17_4
-; SSE42-NEXT:  ## %bb.3: ## %cond.load1
-; SSE42-NEXT:    movl 4(%rdi), %eax
-; SSE42-NEXT:    pinsrq $1, %rax, %xmm1
+; SSE42-NEXT:    jne LBB17_1
+; SSE42-NEXT:  ## %bb.2: ## %else
+; SSE42-NEXT:    testb $2, %al
+; SSE42-NEXT:    jne LBB17_3
 ; SSE42-NEXT:  LBB17_4: ## %else2
 ; SSE42-NEXT:    movdqa %xmm1, %xmm0
 ; SSE42-NEXT:    retq
+; SSE42-NEXT:  LBB17_1: ## %cond.load
+; SSE42-NEXT:    pinsrd $0, (%rdi), %xmm1
+; SSE42-NEXT:    testb $2, %al
+; SSE42-NEXT:    je LBB17_4
+; SSE42-NEXT:  LBB17_3: ## %cond.load1
+; SSE42-NEXT:    pinsrd $1, 4(%rdi), %xmm1
+; SSE42-NEXT:    movdqa %xmm1, %xmm0
+; SSE42-NEXT:    retq
 ;
 ; AVX1-LABEL: load_v2i32_v2i32:
 ; AVX1:       ## %bb.0:
 ; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
-; AVX1-NEXT:    vpcmpeqq %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,2],zero,zero
+; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; AVX1-NEXT:    vmaskmovps (%rdi), %xmm0, %xmm2
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[0,2,2,3]
 ; AVX1-NEXT:    vblendvps %xmm0, %xmm2, %xmm1, %xmm0
-; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: load_v2i32_v2i32:
 ; AVX2:       ## %bb.0:
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
-; AVX2-NEXT:    vpcmpeqq %xmm2, %xmm0, %xmm0
-; AVX2-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,2],zero,zero
+; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; AVX2-NEXT:    vpmaskmovd (%rdi), %xmm0, %xmm2
-; AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[0,2,2,3]
 ; AVX2-NEXT:    vblendvps %xmm0, %xmm2, %xmm1, %xmm0
-; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
 ; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: load_v2i32_v2i32:
 ; AVX512F:       ## %bb.0:
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
-; AVX512F-NEXT:    vptestnmq %zmm0, %zmm0, %k0
-; AVX512F-NEXT:    vpshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
+; AVX512F-NEXT:    ## kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512F-NEXT:    ## kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512F-NEXT:    vptestnmd %zmm0, %zmm0, %k0
 ; AVX512F-NEXT:    kshiftlw $14, %k0, %k0
 ; AVX512F-NEXT:    kshiftrw $14, %k0, %k1
-; AVX512F-NEXT:    vmovdqu32 (%rdi), %zmm0 {%k1}
-; AVX512F-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
+; AVX512F-NEXT:    vpblendmd (%rdi), %zmm1, %zmm0 {%k1}
+; AVX512F-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $zmm0
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
-; AVX512VL-LABEL: load_v2i32_v2i32:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VL-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
-; AVX512VL-NEXT:    vptestnmq %xmm0, %xmm0, %k1
-; AVX512VL-NEXT:    vpshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
-; AVX512VL-NEXT:    vmovdqu32 (%rdi), %xmm0 {%k1}
-; AVX512VL-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
-; AVX512VL-NEXT:    retq
+; AVX512VLDQ-LABEL: load_v2i32_v2i32:
+; AVX512VLDQ:       ## %bb.0:
+; AVX512VLDQ-NEXT:    vptestnmd %xmm0, %xmm0, %k0
+; AVX512VLDQ-NEXT:    kshiftlb $6, %k0, %k0
+; AVX512VLDQ-NEXT:    kshiftrb $6, %k0, %k1
+; AVX512VLDQ-NEXT:    vpblendmd (%rdi), %xmm1, %xmm0 {%k1}
+; AVX512VLDQ-NEXT:    retq
+;
+; AVX512VLBW-LABEL: load_v2i32_v2i32:
+; AVX512VLBW:       ## %bb.0:
+; AVX512VLBW-NEXT:    vptestnmd %xmm0, %xmm0, %k0
+; AVX512VLBW-NEXT:    kshiftlw $14, %k0, %k0
+; AVX512VLBW-NEXT:    kshiftrw $14, %k0, %k1
+; AVX512VLBW-NEXT:    vpblendmd (%rdi), %xmm1, %xmm0 {%k1}
+; AVX512VLBW-NEXT:    retq
   %mask = icmp eq <2 x i32> %trigger, zeroinitializer
   %res = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* %addr, i32 4, <2 x i1> %mask, <2 x i32> %dst)
   ret <2 x i32> %res
@@ -2184,40 +2153,43 @@ define <4 x i32> @load_v4i32_v4i32(<4 x i32> %trigger, <4 x i32>* %addr, <4 x i3
 ; SSE2:       ## %bb.0:
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
 ; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    movd %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB18_2
-; SSE2-NEXT:  ## %bb.1: ## %cond.load
-; SSE2-NEXT:    movss {{.*#+}} xmm3 = mem[0],zero,zero,zero
-; SSE2-NEXT:    movss {{.*#+}} xmm1 = xmm3[0],xmm1[1,2,3]
-; SSE2-NEXT:  LBB18_2: ## %else
-; SSE2-NEXT:    pextrw $2, %xmm2, %eax
+; SSE2-NEXT:    movmskps %xmm2, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB18_4
-; SSE2-NEXT:  ## %bb.3: ## %cond.load1
-; SSE2-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,0],xmm1[0,0]
-; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[2,0],xmm1[2,3]
-; SSE2-NEXT:    movaps %xmm2, %xmm1
+; SSE2-NEXT:    jne LBB18_1
+; SSE2-NEXT:  ## %bb.2: ## %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne LBB18_3
 ; SSE2-NEXT:  LBB18_4: ## %else2
-; SSE2-NEXT:    xorps %xmm2, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm0
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB18_6
-; SSE2-NEXT:  ## %bb.5: ## %cond.load4
-; SSE2-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,0],xmm1[3,0]
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0,2]
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne LBB18_5
 ; SSE2-NEXT:  LBB18_6: ## %else5
-; SSE2-NEXT:    pextrw $6, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne LBB18_7
+; SSE2-NEXT:  LBB18_8: ## %else8
+; SSE2-NEXT:    movaps %xmm1, %xmm0
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  LBB18_1: ## %cond.load
+; SSE2-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT:    movss {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3]
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    je LBB18_4
+; SSE2-NEXT:  LBB18_3: ## %cond.load1
+; SSE2-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,0],xmm1[0,0]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[2,0],xmm1[2,3]
+; SSE2-NEXT:    movaps %xmm0, %xmm1
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    je LBB18_6
+; SSE2-NEXT:  LBB18_5: ## %cond.load4
+; SSE2-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,0],xmm1[3,0]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,1],xmm0[0,2]
+; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je LBB18_8
-; SSE2-NEXT:  ## %bb.7: ## %cond.load7
+; SSE2-NEXT:  LBB18_7: ## %cond.load7
 ; SSE2-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,0],xmm1[2,0]
 ; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,1],xmm0[2,0]
-; SSE2-NEXT:  LBB18_8: ## %else8
 ; SSE2-NEXT:    movaps %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
@@ -2225,32 +2197,35 @@ define <4 x i32> @load_v4i32_v4i32(<4 x i32> %trigger, <4 x i32>* %addr, <4 x i3
 ; SSE42:       ## %bb.0:
 ; SSE42-NEXT:    pxor %xmm2, %xmm2
 ; SSE42-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE42-NEXT:    pextrb $0, %xmm2, %eax
+; SSE42-NEXT:    movmskps %xmm2, %eax
 ; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB18_2
-; SSE42-NEXT:  ## %bb.1: ## %cond.load
+; SSE42-NEXT:    jne LBB18_1
+; SSE42-NEXT:  ## %bb.2: ## %else
+; SSE42-NEXT:    testb $2, %al
+; SSE42-NEXT:    jne LBB18_3
+; SSE42-NEXT:  LBB18_4: ## %else2
+; SSE42-NEXT:    testb $4, %al
+; SSE42-NEXT:    jne LBB18_5
+; SSE42-NEXT:  LBB18_6: ## %else5
+; SSE42-NEXT:    testb $8, %al
+; SSE42-NEXT:    jne LBB18_7
+; SSE42-NEXT:  LBB18_8: ## %else8
+; SSE42-NEXT:    movdqa %xmm1, %xmm0
+; SSE42-NEXT:    retq
+; SSE42-NEXT:  LBB18_1: ## %cond.load
 ; SSE42-NEXT:    pinsrd $0, (%rdi), %xmm1
-; SSE42-NEXT:  LBB18_2: ## %else
-; SSE42-NEXT:    pextrb $4, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $2, %al
 ; SSE42-NEXT:    je LBB18_4
-; SSE42-NEXT:  ## %bb.3: ## %cond.load1
+; SSE42-NEXT:  LBB18_3: ## %cond.load1
 ; SSE42-NEXT:    pinsrd $1, 4(%rdi), %xmm1
-; SSE42-NEXT:  LBB18_4: ## %else2
-; SSE42-NEXT:    pxor %xmm2, %xmm2
-; SSE42-NEXT:    pcmpeqd %xmm2, %xmm0
-; SSE42-NEXT:    pextrb $8, %xmm0, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $4, %al
 ; SSE42-NEXT:    je LBB18_6
-; SSE42-NEXT:  ## %bb.5: ## %cond.load4
+; SSE42-NEXT:  LBB18_5: ## %cond.load4
 ; SSE42-NEXT:    pinsrd $2, 8(%rdi), %xmm1
-; SSE42-NEXT:  LBB18_6: ## %else5
-; SSE42-NEXT:    pextrb $12, %xmm0, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $8, %al
 ; SSE42-NEXT:    je LBB18_8
-; SSE42-NEXT:  ## %bb.7: ## %cond.load7
+; SSE42-NEXT:  LBB18_7: ## %cond.load7
 ; SSE42-NEXT:    pinsrd $3, 12(%rdi), %xmm1
-; SSE42-NEXT:  LBB18_8: ## %else8
 ; SSE42-NEXT:    movdqa %xmm1, %xmm0
 ; SSE42-NEXT:    retq
 ;
@@ -2295,66 +2270,33 @@ define <4 x i32> @load_v4i32_v4i32(<4 x i32> %trigger, <4 x i32>* %addr, <4 x i3
 define <8 x i32> @load_v8i32_v8i1(<8 x i1> %mask, <8 x i32>* %addr, <8 x i32> %dst) {
 ; SSE2-LABEL: load_v8i32_v8i1:
 ; SSE2:       ## %bb.0:
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB19_2
-; SSE2-NEXT:  ## %bb.1: ## %cond.load
-; SSE2-NEXT:    movss {{.*#+}} xmm3 = mem[0],zero,zero,zero
-; SSE2-NEXT:    movss {{.*#+}} xmm1 = xmm3[0],xmm1[1,2,3]
-; SSE2-NEXT:  LBB19_2: ## %else
-; SSE2-NEXT:    shrl $16, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB19_4
-; SSE2-NEXT:  ## %bb.3: ## %cond.load1
-; SSE2-NEXT:    movss {{.*#+}} xmm3 = mem[0],zero,zero,zero
-; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,0],xmm1[0,0]
-; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[2,0],xmm1[2,3]
-; SSE2-NEXT:    movaps %xmm3, %xmm1
+; SSE2-NEXT:    psllw $15, %xmm0
+; SSE2-NEXT:    packsswb %xmm0, %xmm0
+; SSE2-NEXT:    pmovmskb %xmm0, %eax
+; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    jne LBB19_1
+; SSE2-NEXT:  ## %bb.2: ## %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne LBB19_3
 ; SSE2-NEXT:  LBB19_4: ## %else2
-; SSE2-NEXT:    pextrw $2, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB19_6
-; SSE2-NEXT:  ## %bb.5: ## %cond.load4
-; SSE2-NEXT:    movss {{.*#+}} xmm3 = mem[0],zero,zero,zero
-; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,0],xmm1[3,0]
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,1],xmm3[0,2]
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne LBB19_5
 ; SSE2-NEXT:  LBB19_6: ## %else5
-; SSE2-NEXT:    pextrw $3, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB19_8
-; SSE2-NEXT:  ## %bb.7: ## %cond.load7
-; SSE2-NEXT:    movss {{.*#+}} xmm3 = mem[0],zero,zero,zero
-; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,0],xmm1[2,0]
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,0]
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne LBB19_7
 ; SSE2-NEXT:  LBB19_8: ## %else8
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB19_10
-; SSE2-NEXT:  ## %bb.9: ## %cond.load10
-; SSE2-NEXT:    movss {{.*#+}} xmm3 = mem[0],zero,zero,zero
-; SSE2-NEXT:    movss {{.*#+}} xmm2 = xmm3[0],xmm2[1,2,3]
+; SSE2-NEXT:    testb $16, %al
+; SSE2-NEXT:    jne LBB19_9
 ; SSE2-NEXT:  LBB19_10: ## %else11
-; SSE2-NEXT:    pextrw $5, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB19_12
-; SSE2-NEXT:  ## %bb.11: ## %cond.load13
-; SSE2-NEXT:    movss {{.*#+}} xmm3 = mem[0],zero,zero,zero
-; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,0],xmm2[0,0]
-; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[2,0],xmm2[2,3]
-; SSE2-NEXT:    movaps %xmm3, %xmm2
+; SSE2-NEXT:    testb $32, %al
+; SSE2-NEXT:    jne LBB19_11
 ; SSE2-NEXT:  LBB19_12: ## %else14
-; SSE2-NEXT:    pextrw $6, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB19_14
-; SSE2-NEXT:  ## %bb.13: ## %cond.load16
-; SSE2-NEXT:    movss {{.*#+}} xmm3 = mem[0],zero,zero,zero
-; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,0],xmm2[3,0]
-; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,1],xmm3[0,2]
+; SSE2-NEXT:    testb $64, %al
+; SSE2-NEXT:    jne LBB19_13
 ; SSE2-NEXT:  LBB19_14: ## %else17
-; SSE2-NEXT:    pextrw $7, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $-128, %al
 ; SSE2-NEXT:    je LBB19_16
-; SSE2-NEXT:  ## %bb.15: ## %cond.load19
+; SSE2-NEXT:  LBB19_15: ## %cond.load19
 ; SSE2-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,0],xmm2[2,0]
 ; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,1],xmm0[2,0]
@@ -2362,60 +2304,113 @@ define <8 x i32> @load_v8i32_v8i1(<8 x i1> %mask, <8 x i32>* %addr, <8 x i32> %d
 ; SSE2-NEXT:    movaps %xmm1, %xmm0
 ; SSE2-NEXT:    movaps %xmm2, %xmm1
 ; SSE2-NEXT:    retq
+; SSE2-NEXT:  LBB19_1: ## %cond.load
+; SSE2-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT:    movss {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3]
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    je LBB19_4
+; SSE2-NEXT:  LBB19_3: ## %cond.load1
+; SSE2-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,0],xmm1[0,0]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[2,0],xmm1[2,3]
+; SSE2-NEXT:    movaps %xmm0, %xmm1
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    je LBB19_6
+; SSE2-NEXT:  LBB19_5: ## %cond.load4
+; SSE2-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,0],xmm1[3,0]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,1],xmm0[0,2]
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    je LBB19_8
+; SSE2-NEXT:  LBB19_7: ## %cond.load7
+; SSE2-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,0],xmm1[2,0]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,1],xmm0[2,0]
+; SSE2-NEXT:    testb $16, %al
+; SSE2-NEXT:    je LBB19_10
+; SSE2-NEXT:  LBB19_9: ## %cond.load10
+; SSE2-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT:    movss {{.*#+}} xmm2 = xmm0[0],xmm2[1,2,3]
+; SSE2-NEXT:    testb $32, %al
+; SSE2-NEXT:    je LBB19_12
+; SSE2-NEXT:  LBB19_11: ## %cond.load13
+; SSE2-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,0],xmm2[0,0]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[2,0],xmm2[2,3]
+; SSE2-NEXT:    movaps %xmm0, %xmm2
+; SSE2-NEXT:    testb $64, %al
+; SSE2-NEXT:    je LBB19_14
+; SSE2-NEXT:  LBB19_13: ## %cond.load16
+; SSE2-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,0],xmm2[3,0]
+; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,1],xmm0[0,2]
+; SSE2-NEXT:    testb $-128, %al
+; SSE2-NEXT:    jne LBB19_15
+; SSE2-NEXT:    jmp LBB19_16
 ;
 ; SSE42-LABEL: load_v8i32_v8i1:
 ; SSE42:       ## %bb.0:
-; SSE42-NEXT:    pextrb $0, %xmm0, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB19_2
-; SSE42-NEXT:  ## %bb.1: ## %cond.load
+; SSE42-NEXT:    psllw $15, %xmm0
+; SSE42-NEXT:    packsswb %xmm0, %xmm0
+; SSE42-NEXT:    pmovmskb %xmm0, %eax
+; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    jne LBB19_1
+; SSE42-NEXT:  ## %bb.2: ## %else
+; SSE42-NEXT:    testb $2, %al
+; SSE42-NEXT:    jne LBB19_3
+; SSE42-NEXT:  LBB19_4: ## %else2
+; SSE42-NEXT:    testb $4, %al
+; SSE42-NEXT:    jne LBB19_5
+; SSE42-NEXT:  LBB19_6: ## %else5
+; SSE42-NEXT:    testb $8, %al
+; SSE42-NEXT:    jne LBB19_7
+; SSE42-NEXT:  LBB19_8: ## %else8
+; SSE42-NEXT:    testb $16, %al
+; SSE42-NEXT:    jne LBB19_9
+; SSE42-NEXT:  LBB19_10: ## %else11
+; SSE42-NEXT:    testb $32, %al
+; SSE42-NEXT:    jne LBB19_11
+; SSE42-NEXT:  LBB19_12: ## %else14
+; SSE42-NEXT:    testb $64, %al
+; SSE42-NEXT:    jne LBB19_13
+; SSE42-NEXT:  LBB19_14: ## %else17
+; SSE42-NEXT:    testb $-128, %al
+; SSE42-NEXT:    je LBB19_16
+; SSE42-NEXT:  LBB19_15: ## %cond.load19
+; SSE42-NEXT:    pinsrd $3, 28(%rdi), %xmm2
+; SSE42-NEXT:  LBB19_16: ## %else20
+; SSE42-NEXT:    movdqa %xmm1, %xmm0
+; SSE42-NEXT:    movdqa %xmm2, %xmm1
+; SSE42-NEXT:    retq
+; SSE42-NEXT:  LBB19_1: ## %cond.load
 ; SSE42-NEXT:    pinsrd $0, (%rdi), %xmm1
-; SSE42-NEXT:  LBB19_2: ## %else
-; SSE42-NEXT:    pextrb $2, %xmm0, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $2, %al
 ; SSE42-NEXT:    je LBB19_4
-; SSE42-NEXT:  ## %bb.3: ## %cond.load1
+; SSE42-NEXT:  LBB19_3: ## %cond.load1
 ; SSE42-NEXT:    pinsrd $1, 4(%rdi), %xmm1
-; SSE42-NEXT:  LBB19_4: ## %else2
-; SSE42-NEXT:    pextrb $4, %xmm0, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $4, %al
 ; SSE42-NEXT:    je LBB19_6
-; SSE42-NEXT:  ## %bb.5: ## %cond.load4
+; SSE42-NEXT:  LBB19_5: ## %cond.load4
 ; SSE42-NEXT:    pinsrd $2, 8(%rdi), %xmm1
-; SSE42-NEXT:  LBB19_6: ## %else5
-; SSE42-NEXT:    pextrb $6, %xmm0, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $8, %al
 ; SSE42-NEXT:    je LBB19_8
-; SSE42-NEXT:  ## %bb.7: ## %cond.load7
+; SSE42-NEXT:  LBB19_7: ## %cond.load7
 ; SSE42-NEXT:    pinsrd $3, 12(%rdi), %xmm1
-; SSE42-NEXT:  LBB19_8: ## %else8
-; SSE42-NEXT:    pextrb $8, %xmm0, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $16, %al
 ; SSE42-NEXT:    je LBB19_10
-; SSE42-NEXT:  ## %bb.9: ## %cond.load10
+; SSE42-NEXT:  LBB19_9: ## %cond.load10
 ; SSE42-NEXT:    pinsrd $0, 16(%rdi), %xmm2
-; SSE42-NEXT:  LBB19_10: ## %else11
-; SSE42-NEXT:    pextrb $10, %xmm0, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $32, %al
 ; SSE42-NEXT:    je LBB19_12
-; SSE42-NEXT:  ## %bb.11: ## %cond.load13
+; SSE42-NEXT:  LBB19_11: ## %cond.load13
 ; SSE42-NEXT:    pinsrd $1, 20(%rdi), %xmm2
-; SSE42-NEXT:  LBB19_12: ## %else14
-; SSE42-NEXT:    pextrb $12, %xmm0, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $64, %al
 ; SSE42-NEXT:    je LBB19_14
-; SSE42-NEXT:  ## %bb.13: ## %cond.load16
+; SSE42-NEXT:  LBB19_13: ## %cond.load16
 ; SSE42-NEXT:    pinsrd $2, 24(%rdi), %xmm2
-; SSE42-NEXT:  LBB19_14: ## %else17
-; SSE42-NEXT:    pextrb $14, %xmm0, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB19_16
-; SSE42-NEXT:  ## %bb.15: ## %cond.load19
-; SSE42-NEXT:    pinsrd $3, 28(%rdi), %xmm2
-; SSE42-NEXT:  LBB19_16: ## %else20
-; SSE42-NEXT:    movdqa %xmm1, %xmm0
-; SSE42-NEXT:    movdqa %xmm2, %xmm1
-; SSE42-NEXT:    retq
+; SSE42-NEXT:    testb $-128, %al
+; SSE42-NEXT:    jne LBB19_15
+; SSE42-NEXT:    jmp LBB19_16
 ;
 ; AVX1-LABEL: load_v8i32_v8i1:
 ; AVX1:       ## %bb.0:
@@ -2470,129 +2465,147 @@ define <8 x i32> @load_v8i32_v8i1(<8 x i1> %mask, <8 x i32>* %addr, <8 x i32> %d
 define <8 x i32> @load_v8i32_v8i1_zero(<8 x i1> %mask, <8 x i32>* %addr) {
 ; SSE2-LABEL: load_v8i32_v8i1_zero:
 ; SSE2:       ## %bb.0:
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    psllw $15, %xmm0
+; SSE2-NEXT:    packsswb %xmm0, %xmm0
+; SSE2-NEXT:    pmovmskb %xmm0, %eax
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
 ; SSE2-NEXT:    testb $1, %al
 ; SSE2-NEXT:    xorps %xmm1, %xmm1
-; SSE2-NEXT:    je LBB20_2
-; SSE2-NEXT:  ## %bb.1: ## %cond.load
+; SSE2-NEXT:    jne LBB20_1
+; SSE2-NEXT:  ## %bb.2: ## %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne LBB20_3
+; SSE2-NEXT:  LBB20_4: ## %else2
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne LBB20_5
+; SSE2-NEXT:  LBB20_6: ## %else5
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne LBB20_7
+; SSE2-NEXT:  LBB20_8: ## %else8
+; SSE2-NEXT:    testb $16, %al
+; SSE2-NEXT:    jne LBB20_9
+; SSE2-NEXT:  LBB20_10: ## %else11
+; SSE2-NEXT:    testb $32, %al
+; SSE2-NEXT:    jne LBB20_11
+; SSE2-NEXT:  LBB20_12: ## %else14
+; SSE2-NEXT:    testb $64, %al
+; SSE2-NEXT:    jne LBB20_13
+; SSE2-NEXT:  LBB20_14: ## %else17
+; SSE2-NEXT:    testb $-128, %al
+; SSE2-NEXT:    jne LBB20_15
+; SSE2-NEXT:  LBB20_16: ## %else20
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  LBB20_1: ## %cond.load
 ; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; SSE2-NEXT:    xorps %xmm1, %xmm1
-; SSE2-NEXT:  LBB20_2: ## %else
-; SSE2-NEXT:    shrl $16, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $2, %al
 ; SSE2-NEXT:    je LBB20_4
-; SSE2-NEXT:  ## %bb.3: ## %cond.load1
-; SSE2-NEXT:    movss {{.*#+}} xmm3 = mem[0],zero,zero,zero
-; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,0],xmm0[0,0]
-; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[2,0],xmm0[2,3]
-; SSE2-NEXT:    movaps %xmm3, %xmm0
-; SSE2-NEXT:  LBB20_4: ## %else2
-; SSE2-NEXT:    pextrw $2, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:  LBB20_3: ## %cond.load1
+; SSE2-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,0],xmm0[0,0]
+; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[2,0],xmm0[2,3]
+; SSE2-NEXT:    movaps %xmm2, %xmm0
+; SSE2-NEXT:    testb $4, %al
 ; SSE2-NEXT:    je LBB20_6
-; SSE2-NEXT:  ## %bb.5: ## %cond.load4
-; SSE2-NEXT:    movss {{.*#+}} xmm3 = mem[0],zero,zero,zero
-; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,0],xmm0[3,0]
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm3[0,2]
-; SSE2-NEXT:  LBB20_6: ## %else5
-; SSE2-NEXT:    pextrw $3, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:  LBB20_5: ## %cond.load4
+; SSE2-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,0],xmm0[3,0]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm2[0,2]
+; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je LBB20_8
-; SSE2-NEXT:  ## %bb.7: ## %cond.load7
-; SSE2-NEXT:    movss {{.*#+}} xmm3 = mem[0],zero,zero,zero
-; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,0],xmm0[2,0]
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm3[2,0]
-; SSE2-NEXT:  LBB20_8: ## %else8
-; SSE2-NEXT:    pextrw $4, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:  LBB20_7: ## %cond.load7
+; SSE2-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,0],xmm0[2,0]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,0]
+; SSE2-NEXT:    testb $16, %al
 ; SSE2-NEXT:    je LBB20_10
-; SSE2-NEXT:  ## %bb.9: ## %cond.load10
-; SSE2-NEXT:    movss {{.*#+}} xmm3 = mem[0],zero,zero,zero
-; SSE2-NEXT:    movss {{.*#+}} xmm1 = xmm3[0],xmm1[1,2,3]
-; SSE2-NEXT:  LBB20_10: ## %else11
-; SSE2-NEXT:    pextrw $5, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:  LBB20_9: ## %cond.load10
+; SSE2-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE2-NEXT:    movss {{.*#+}} xmm1 = xmm2[0],xmm1[1,2,3]
+; SSE2-NEXT:    testb $32, %al
 ; SSE2-NEXT:    je LBB20_12
-; SSE2-NEXT:  ## %bb.11: ## %cond.load13
-; SSE2-NEXT:    movss {{.*#+}} xmm3 = mem[0],zero,zero,zero
-; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,0],xmm1[0,0]
-; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[2,0],xmm1[2,3]
-; SSE2-NEXT:    movaps %xmm3, %xmm1
-; SSE2-NEXT:  LBB20_12: ## %else14
-; SSE2-NEXT:    pextrw $6, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:  LBB20_11: ## %cond.load13
+; SSE2-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,0],xmm1[0,0]
+; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[2,0],xmm1[2,3]
+; SSE2-NEXT:    movaps %xmm2, %xmm1
+; SSE2-NEXT:    testb $64, %al
 ; SSE2-NEXT:    je LBB20_14
-; SSE2-NEXT:  ## %bb.13: ## %cond.load16
-; SSE2-NEXT:    movss {{.*#+}} xmm3 = mem[0],zero,zero,zero
-; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,0],xmm1[3,0]
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,1],xmm3[0,2]
-; SSE2-NEXT:  LBB20_14: ## %else17
-; SSE2-NEXT:    pextrw $7, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:  LBB20_13: ## %cond.load16
+; SSE2-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,0],xmm1[3,0]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0,2]
+; SSE2-NEXT:    testb $-128, %al
 ; SSE2-NEXT:    je LBB20_16
-; SSE2-NEXT:  ## %bb.15: ## %cond.load19
+; SSE2-NEXT:  LBB20_15: ## %cond.load19
 ; SSE2-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
 ; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,0],xmm1[2,0]
 ; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,0]
-; SSE2-NEXT:  LBB20_16: ## %else20
 ; SSE2-NEXT:    retq
 ;
 ; SSE42-LABEL: load_v8i32_v8i1_zero:
 ; SSE42:       ## %bb.0:
-; SSE42-NEXT:    movdqa %xmm0, %xmm2
-; SSE42-NEXT:    pextrb $0, %xmm0, %eax
+; SSE42-NEXT:    psllw $15, %xmm0
+; SSE42-NEXT:    packsswb %xmm0, %xmm0
+; SSE42-NEXT:    pmovmskb %xmm0, %eax
 ; SSE42-NEXT:    pxor %xmm0, %xmm0
 ; SSE42-NEXT:    testb $1, %al
 ; SSE42-NEXT:    pxor %xmm1, %xmm1
-; SSE42-NEXT:    je LBB20_2
-; SSE42-NEXT:  ## %bb.1: ## %cond.load
+; SSE42-NEXT:    jne LBB20_1
+; SSE42-NEXT:  ## %bb.2: ## %else
+; SSE42-NEXT:    testb $2, %al
+; SSE42-NEXT:    jne LBB20_3
+; SSE42-NEXT:  LBB20_4: ## %else2
+; SSE42-NEXT:    testb $4, %al
+; SSE42-NEXT:    jne LBB20_5
+; SSE42-NEXT:  LBB20_6: ## %else5
+; SSE42-NEXT:    testb $8, %al
+; SSE42-NEXT:    jne LBB20_7
+; SSE42-NEXT:  LBB20_8: ## %else8
+; SSE42-NEXT:    testb $16, %al
+; SSE42-NEXT:    jne LBB20_9
+; SSE42-NEXT:  LBB20_10: ## %else11
+; SSE42-NEXT:    testb $32, %al
+; SSE42-NEXT:    jne LBB20_11
+; SSE42-NEXT:  LBB20_12: ## %else14
+; SSE42-NEXT:    testb $64, %al
+; SSE42-NEXT:    jne LBB20_13
+; SSE42-NEXT:  LBB20_14: ## %else17
+; SSE42-NEXT:    testb $-128, %al
+; SSE42-NEXT:    jne LBB20_15
+; SSE42-NEXT:  LBB20_16: ## %else20
+; SSE42-NEXT:    retq
+; SSE42-NEXT:  LBB20_1: ## %cond.load
 ; SSE42-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; SSE42-NEXT:    pxor %xmm1, %xmm1
-; SSE42-NEXT:  LBB20_2: ## %else
-; SSE42-NEXT:    pextrb $2, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $2, %al
 ; SSE42-NEXT:    je LBB20_4
-; SSE42-NEXT:  ## %bb.3: ## %cond.load1
+; SSE42-NEXT:  LBB20_3: ## %cond.load1
 ; SSE42-NEXT:    pinsrd $1, 4(%rdi), %xmm0
-; SSE42-NEXT:  LBB20_4: ## %else2
-; SSE42-NEXT:    pextrb $4, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $4, %al
 ; SSE42-NEXT:    je LBB20_6
-; SSE42-NEXT:  ## %bb.5: ## %cond.load4
+; SSE42-NEXT:  LBB20_5: ## %cond.load4
 ; SSE42-NEXT:    pinsrd $2, 8(%rdi), %xmm0
-; SSE42-NEXT:  LBB20_6: ## %else5
-; SSE42-NEXT:    pextrb $6, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $8, %al
 ; SSE42-NEXT:    je LBB20_8
-; SSE42-NEXT:  ## %bb.7: ## %cond.load7
+; SSE42-NEXT:  LBB20_7: ## %cond.load7
 ; SSE42-NEXT:    pinsrd $3, 12(%rdi), %xmm0
-; SSE42-NEXT:  LBB20_8: ## %else8
-; SSE42-NEXT:    pextrb $8, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $16, %al
 ; SSE42-NEXT:    je LBB20_10
-; SSE42-NEXT:  ## %bb.9: ## %cond.load10
+; SSE42-NEXT:  LBB20_9: ## %cond.load10
 ; SSE42-NEXT:    pinsrd $0, 16(%rdi), %xmm1
-; SSE42-NEXT:  LBB20_10: ## %else11
-; SSE42-NEXT:    pextrb $10, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $32, %al
 ; SSE42-NEXT:    je LBB20_12
-; SSE42-NEXT:  ## %bb.11: ## %cond.load13
+; SSE42-NEXT:  LBB20_11: ## %cond.load13
 ; SSE42-NEXT:    pinsrd $1, 20(%rdi), %xmm1
-; SSE42-NEXT:  LBB20_12: ## %else14
-; SSE42-NEXT:    pextrb $12, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $64, %al
 ; SSE42-NEXT:    je LBB20_14
-; SSE42-NEXT:  ## %bb.13: ## %cond.load16
+; SSE42-NEXT:  LBB20_13: ## %cond.load16
 ; SSE42-NEXT:    pinsrd $2, 24(%rdi), %xmm1
-; SSE42-NEXT:  LBB20_14: ## %else17
-; SSE42-NEXT:    pextrb $14, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $-128, %al
 ; SSE42-NEXT:    je LBB20_16
-; SSE42-NEXT:  ## %bb.15: ## %cond.load19
+; SSE42-NEXT:  LBB20_15: ## %cond.load19
 ; SSE42-NEXT:    pinsrd $3, 28(%rdi), %xmm1
-; SSE42-NEXT:  LBB20_16: ## %else20
 ; SSE42-NEXT:    retq
 ;
 ; AVX1-LABEL: load_v8i32_v8i1_zero:
@@ -2647,262 +2660,196 @@ define <8 x i32> @load_v8i32_v8i1_zero(<8 x i1> %mask, <8 x i32>* %addr) {
 ;
 
 define <8 x i16> @load_v8i16_v8i16(<8 x i16> %trigger, <8 x i16>* %addr, <8 x i16> %dst) {
-; SSE2-LABEL: load_v8i16_v8i16:
-; SSE2:       ## %bb.0:
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpgtw %xmm0, %xmm2
-; SSE2-NEXT:    movd %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB21_2
-; SSE2-NEXT:  ## %bb.1: ## %cond.load
-; SSE2-NEXT:    pinsrw $0, (%rdi), %xmm1
-; SSE2-NEXT:  LBB21_2: ## %else
-; SSE2-NEXT:    shrl $16, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB21_4
-; SSE2-NEXT:  ## %bb.3: ## %cond.load1
-; SSE2-NEXT:    pinsrw $1, 2(%rdi), %xmm1
-; SSE2-NEXT:  LBB21_4: ## %else2
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpgtw %xmm0, %xmm2
-; SSE2-NEXT:    pextrw $2, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB21_6
-; SSE2-NEXT:  ## %bb.5: ## %cond.load4
-; SSE2-NEXT:    pinsrw $2, 4(%rdi), %xmm1
-; SSE2-NEXT:  LBB21_6: ## %else5
-; SSE2-NEXT:    pextrw $3, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB21_8
-; SSE2-NEXT:  ## %bb.7: ## %cond.load7
-; SSE2-NEXT:    pinsrw $3, 6(%rdi), %xmm1
-; SSE2-NEXT:  LBB21_8: ## %else8
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpgtw %xmm0, %xmm2
-; SSE2-NEXT:    pextrw $4, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB21_10
-; SSE2-NEXT:  ## %bb.9: ## %cond.load10
-; SSE2-NEXT:    pinsrw $4, 8(%rdi), %xmm1
-; SSE2-NEXT:  LBB21_10: ## %else11
-; SSE2-NEXT:    pextrw $5, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB21_12
-; SSE2-NEXT:  ## %bb.11: ## %cond.load13
-; SSE2-NEXT:    pinsrw $5, 10(%rdi), %xmm1
-; SSE2-NEXT:  LBB21_12: ## %else14
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpgtw %xmm0, %xmm2
-; SSE2-NEXT:    pextrw $6, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB21_14
-; SSE2-NEXT:  ## %bb.13: ## %cond.load16
-; SSE2-NEXT:    pinsrw $6, 12(%rdi), %xmm1
-; SSE2-NEXT:  LBB21_14: ## %else17
-; SSE2-NEXT:    pextrw $7, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB21_16
-; SSE2-NEXT:  ## %bb.15: ## %cond.load19
-; SSE2-NEXT:    pinsrw $7, 14(%rdi), %xmm1
-; SSE2-NEXT:  LBB21_16: ## %else20
-; SSE2-NEXT:    movdqa %xmm1, %xmm0
-; SSE2-NEXT:    retq
-;
-; SSE42-LABEL: load_v8i16_v8i16:
-; SSE42:       ## %bb.0:
-; SSE42-NEXT:    pxor %xmm2, %xmm2
-; SSE42-NEXT:    pcmpgtw %xmm0, %xmm2
-; SSE42-NEXT:    pextrb $0, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB21_2
-; SSE42-NEXT:  ## %bb.1: ## %cond.load
-; SSE42-NEXT:    pinsrw $0, (%rdi), %xmm1
-; SSE42-NEXT:  LBB21_2: ## %else
-; SSE42-NEXT:    pextrb $2, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB21_4
-; SSE42-NEXT:  ## %bb.3: ## %cond.load1
-; SSE42-NEXT:    pinsrw $1, 2(%rdi), %xmm1
-; SSE42-NEXT:  LBB21_4: ## %else2
-; SSE42-NEXT:    pxor %xmm2, %xmm2
-; SSE42-NEXT:    pcmpgtw %xmm0, %xmm2
-; SSE42-NEXT:    pextrb $4, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB21_6
-; SSE42-NEXT:  ## %bb.5: ## %cond.load4
-; SSE42-NEXT:    pinsrw $2, 4(%rdi), %xmm1
-; SSE42-NEXT:  LBB21_6: ## %else5
-; SSE42-NEXT:    pextrb $6, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB21_8
-; SSE42-NEXT:  ## %bb.7: ## %cond.load7
-; SSE42-NEXT:    pinsrw $3, 6(%rdi), %xmm1
-; SSE42-NEXT:  LBB21_8: ## %else8
-; SSE42-NEXT:    pxor %xmm2, %xmm2
-; SSE42-NEXT:    pcmpgtw %xmm0, %xmm2
-; SSE42-NEXT:    pextrb $8, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB21_10
-; SSE42-NEXT:  ## %bb.9: ## %cond.load10
-; SSE42-NEXT:    pinsrw $4, 8(%rdi), %xmm1
-; SSE42-NEXT:  LBB21_10: ## %else11
-; SSE42-NEXT:    pextrb $10, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB21_12
-; SSE42-NEXT:  ## %bb.11: ## %cond.load13
-; SSE42-NEXT:    pinsrw $5, 10(%rdi), %xmm1
-; SSE42-NEXT:  LBB21_12: ## %else14
-; SSE42-NEXT:    pxor %xmm2, %xmm2
-; SSE42-NEXT:    pcmpgtw %xmm0, %xmm2
-; SSE42-NEXT:    pextrb $12, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB21_14
-; SSE42-NEXT:  ## %bb.13: ## %cond.load16
-; SSE42-NEXT:    pinsrw $6, 12(%rdi), %xmm1
-; SSE42-NEXT:  LBB21_14: ## %else17
-; SSE42-NEXT:    pextrb $14, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB21_16
-; SSE42-NEXT:  ## %bb.15: ## %cond.load19
-; SSE42-NEXT:    pinsrw $7, 14(%rdi), %xmm1
-; SSE42-NEXT:  LBB21_16: ## %else20
-; SSE42-NEXT:    movdqa %xmm1, %xmm0
-; SSE42-NEXT:    retq
+; SSE-LABEL: load_v8i16_v8i16:
+; SSE:       ## %bb.0:
+; SSE-NEXT:    packsswb %xmm0, %xmm0
+; SSE-NEXT:    pmovmskb %xmm0, %eax
+; SSE-NEXT:    testb $1, %al
+; SSE-NEXT:    jne LBB21_1
+; SSE-NEXT:  ## %bb.2: ## %else
+; SSE-NEXT:    testb $2, %al
+; SSE-NEXT:    jne LBB21_3
+; SSE-NEXT:  LBB21_4: ## %else2
+; SSE-NEXT:    testb $4, %al
+; SSE-NEXT:    jne LBB21_5
+; SSE-NEXT:  LBB21_6: ## %else5
+; SSE-NEXT:    testb $8, %al
+; SSE-NEXT:    jne LBB21_7
+; SSE-NEXT:  LBB21_8: ## %else8
+; SSE-NEXT:    testb $16, %al
+; SSE-NEXT:    jne LBB21_9
+; SSE-NEXT:  LBB21_10: ## %else11
+; SSE-NEXT:    testb $32, %al
+; SSE-NEXT:    jne LBB21_11
+; SSE-NEXT:  LBB21_12: ## %else14
+; SSE-NEXT:    testb $64, %al
+; SSE-NEXT:    jne LBB21_13
+; SSE-NEXT:  LBB21_14: ## %else17
+; SSE-NEXT:    testb $-128, %al
+; SSE-NEXT:    jne LBB21_15
+; SSE-NEXT:  LBB21_16: ## %else20
+; SSE-NEXT:    movdqa %xmm1, %xmm0
+; SSE-NEXT:    retq
+; SSE-NEXT:  LBB21_1: ## %cond.load
+; SSE-NEXT:    pinsrw $0, (%rdi), %xmm1
+; SSE-NEXT:    testb $2, %al
+; SSE-NEXT:    je LBB21_4
+; SSE-NEXT:  LBB21_3: ## %cond.load1
+; SSE-NEXT:    pinsrw $1, 2(%rdi), %xmm1
+; SSE-NEXT:    testb $4, %al
+; SSE-NEXT:    je LBB21_6
+; SSE-NEXT:  LBB21_5: ## %cond.load4
+; SSE-NEXT:    pinsrw $2, 4(%rdi), %xmm1
+; SSE-NEXT:    testb $8, %al
+; SSE-NEXT:    je LBB21_8
+; SSE-NEXT:  LBB21_7: ## %cond.load7
+; SSE-NEXT:    pinsrw $3, 6(%rdi), %xmm1
+; SSE-NEXT:    testb $16, %al
+; SSE-NEXT:    je LBB21_10
+; SSE-NEXT:  LBB21_9: ## %cond.load10
+; SSE-NEXT:    pinsrw $4, 8(%rdi), %xmm1
+; SSE-NEXT:    testb $32, %al
+; SSE-NEXT:    je LBB21_12
+; SSE-NEXT:  LBB21_11: ## %cond.load13
+; SSE-NEXT:    pinsrw $5, 10(%rdi), %xmm1
+; SSE-NEXT:    testb $64, %al
+; SSE-NEXT:    je LBB21_14
+; SSE-NEXT:  LBB21_13: ## %cond.load16
+; SSE-NEXT:    pinsrw $6, 12(%rdi), %xmm1
+; SSE-NEXT:    testb $-128, %al
+; SSE-NEXT:    je LBB21_16
+; SSE-NEXT:  LBB21_15: ## %cond.load19
+; SSE-NEXT:    pinsrw $7, 14(%rdi), %xmm1
+; SSE-NEXT:    movdqa %xmm1, %xmm0
+; SSE-NEXT:    retq
 ;
 ; AVX1OR2-LABEL: load_v8i16_v8i16:
 ; AVX1OR2:       ## %bb.0:
-; AVX1OR2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1OR2-NEXT:    vpcmpgtw %xmm0, %xmm2, %xmm2
-; AVX1OR2-NEXT:    vpextrb $0, %xmm2, %eax
+; AVX1OR2-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
+; AVX1OR2-NEXT:    vpmovmskb %xmm0, %eax
 ; AVX1OR2-NEXT:    testb $1, %al
-; AVX1OR2-NEXT:    je LBB21_2
-; AVX1OR2-NEXT:  ## %bb.1: ## %cond.load
+; AVX1OR2-NEXT:    jne LBB21_1
+; AVX1OR2-NEXT:  ## %bb.2: ## %else
+; AVX1OR2-NEXT:    testb $2, %al
+; AVX1OR2-NEXT:    jne LBB21_3
+; AVX1OR2-NEXT:  LBB21_4: ## %else2
+; AVX1OR2-NEXT:    testb $4, %al
+; AVX1OR2-NEXT:    jne LBB21_5
+; AVX1OR2-NEXT:  LBB21_6: ## %else5
+; AVX1OR2-NEXT:    testb $8, %al
+; AVX1OR2-NEXT:    jne LBB21_7
+; AVX1OR2-NEXT:  LBB21_8: ## %else8
+; AVX1OR2-NEXT:    testb $16, %al
+; AVX1OR2-NEXT:    jne LBB21_9
+; AVX1OR2-NEXT:  LBB21_10: ## %else11
+; AVX1OR2-NEXT:    testb $32, %al
+; AVX1OR2-NEXT:    jne LBB21_11
+; AVX1OR2-NEXT:  LBB21_12: ## %else14
+; AVX1OR2-NEXT:    testb $64, %al
+; AVX1OR2-NEXT:    jne LBB21_13
+; AVX1OR2-NEXT:  LBB21_14: ## %else17
+; AVX1OR2-NEXT:    testb $-128, %al
+; AVX1OR2-NEXT:    jne LBB21_15
+; AVX1OR2-NEXT:  LBB21_16: ## %else20
+; AVX1OR2-NEXT:    vmovdqa %xmm1, %xmm0
+; AVX1OR2-NEXT:    retq
+; AVX1OR2-NEXT:  LBB21_1: ## %cond.load
 ; AVX1OR2-NEXT:    vpinsrw $0, (%rdi), %xmm1, %xmm1
-; AVX1OR2-NEXT:  LBB21_2: ## %else
-; AVX1OR2-NEXT:    vpextrb $2, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $2, %al
 ; AVX1OR2-NEXT:    je LBB21_4
-; AVX1OR2-NEXT:  ## %bb.3: ## %cond.load1
+; AVX1OR2-NEXT:  LBB21_3: ## %cond.load1
 ; AVX1OR2-NEXT:    vpinsrw $1, 2(%rdi), %xmm1, %xmm1
-; AVX1OR2-NEXT:  LBB21_4: ## %else2
-; AVX1OR2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1OR2-NEXT:    vpcmpgtw %xmm0, %xmm2, %xmm2
-; AVX1OR2-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $4, %al
 ; AVX1OR2-NEXT:    je LBB21_6
-; AVX1OR2-NEXT:  ## %bb.5: ## %cond.load4
+; AVX1OR2-NEXT:  LBB21_5: ## %cond.load4
 ; AVX1OR2-NEXT:    vpinsrw $2, 4(%rdi), %xmm1, %xmm1
-; AVX1OR2-NEXT:  LBB21_6: ## %else5
-; AVX1OR2-NEXT:    vpextrb $6, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $8, %al
 ; AVX1OR2-NEXT:    je LBB21_8
-; AVX1OR2-NEXT:  ## %bb.7: ## %cond.load7
+; AVX1OR2-NEXT:  LBB21_7: ## %cond.load7
 ; AVX1OR2-NEXT:    vpinsrw $3, 6(%rdi), %xmm1, %xmm1
-; AVX1OR2-NEXT:  LBB21_8: ## %else8
-; AVX1OR2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1OR2-NEXT:    vpcmpgtw %xmm0, %xmm2, %xmm2
-; AVX1OR2-NEXT:    vpextrb $8, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $16, %al
 ; AVX1OR2-NEXT:    je LBB21_10
-; AVX1OR2-NEXT:  ## %bb.9: ## %cond.load10
+; AVX1OR2-NEXT:  LBB21_9: ## %cond.load10
 ; AVX1OR2-NEXT:    vpinsrw $4, 8(%rdi), %xmm1, %xmm1
-; AVX1OR2-NEXT:  LBB21_10: ## %else11
-; AVX1OR2-NEXT:    vpextrb $10, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $32, %al
 ; AVX1OR2-NEXT:    je LBB21_12
-; AVX1OR2-NEXT:  ## %bb.11: ## %cond.load13
+; AVX1OR2-NEXT:  LBB21_11: ## %cond.load13
 ; AVX1OR2-NEXT:    vpinsrw $5, 10(%rdi), %xmm1, %xmm1
-; AVX1OR2-NEXT:  LBB21_12: ## %else14
-; AVX1OR2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1OR2-NEXT:    vpcmpgtw %xmm0, %xmm2, %xmm0
-; AVX1OR2-NEXT:    vpextrb $12, %xmm0, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $64, %al
 ; AVX1OR2-NEXT:    je LBB21_14
-; AVX1OR2-NEXT:  ## %bb.13: ## %cond.load16
+; AVX1OR2-NEXT:  LBB21_13: ## %cond.load16
 ; AVX1OR2-NEXT:    vpinsrw $6, 12(%rdi), %xmm1, %xmm1
-; AVX1OR2-NEXT:  LBB21_14: ## %else17
-; AVX1OR2-NEXT:    vpextrb $14, %xmm0, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $-128, %al
 ; AVX1OR2-NEXT:    je LBB21_16
-; AVX1OR2-NEXT:  ## %bb.15: ## %cond.load19
+; AVX1OR2-NEXT:  LBB21_15: ## %cond.load19
 ; AVX1OR2-NEXT:    vpinsrw $7, 14(%rdi), %xmm1, %xmm1
-; AVX1OR2-NEXT:  LBB21_16: ## %else20
 ; AVX1OR2-NEXT:    vmovdqa %xmm1, %xmm0
 ; AVX1OR2-NEXT:    retq
 ;
 ; AVX512F-LABEL: load_v8i16_v8i16:
 ; AVX512F:       ## %bb.0:
 ; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpgtw %xmm0, %xmm2, %xmm2
-; AVX512F-NEXT:    vpmovsxwq %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmq %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je LBB21_2
-; AVX512F-NEXT:  ## %bb.1: ## %cond.load
-; AVX512F-NEXT:    vpinsrw $0, (%rdi), %xmm1, %xmm1
-; AVX512F-NEXT:  LBB21_2: ## %else
-; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
+; AVX512F-NEXT:    vpcmpgtw %xmm0, %xmm2, %xmm0
+; AVX512F-NEXT:    vpmovsxwq %xmm0, %zmm0
+; AVX512F-NEXT:    vptestmq %zmm0, %zmm0, %k0
 ; AVX512F-NEXT:    kmovw %k0, %eax
 ; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    jne LBB21_1
+; AVX512F-NEXT:  ## %bb.2: ## %else
+; AVX512F-NEXT:    testb $2, %al
+; AVX512F-NEXT:    jne LBB21_3
+; AVX512F-NEXT:  LBB21_4: ## %else2
+; AVX512F-NEXT:    testb $4, %al
+; AVX512F-NEXT:    jne LBB21_5
+; AVX512F-NEXT:  LBB21_6: ## %else5
+; AVX512F-NEXT:    testb $8, %al
+; AVX512F-NEXT:    jne LBB21_7
+; AVX512F-NEXT:  LBB21_8: ## %else8
+; AVX512F-NEXT:    testb $16, %al
+; AVX512F-NEXT:    jne LBB21_9
+; AVX512F-NEXT:  LBB21_10: ## %else11
+; AVX512F-NEXT:    testb $32, %al
+; AVX512F-NEXT:    jne LBB21_11
+; AVX512F-NEXT:  LBB21_12: ## %else14
+; AVX512F-NEXT:    testb $64, %al
+; AVX512F-NEXT:    jne LBB21_13
+; AVX512F-NEXT:  LBB21_14: ## %else17
+; AVX512F-NEXT:    testb $-128, %al
+; AVX512F-NEXT:    jne LBB21_15
+; AVX512F-NEXT:  LBB21_16: ## %else20
+; AVX512F-NEXT:    vmovdqa %xmm1, %xmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+; AVX512F-NEXT:  LBB21_1: ## %cond.load
+; AVX512F-NEXT:    vpinsrw $0, (%rdi), %xmm1, %xmm1
+; AVX512F-NEXT:    testb $2, %al
 ; AVX512F-NEXT:    je LBB21_4
-; AVX512F-NEXT:  ## %bb.3: ## %cond.load1
+; AVX512F-NEXT:  LBB21_3: ## %cond.load1
 ; AVX512F-NEXT:    vpinsrw $1, 2(%rdi), %xmm1, %xmm1
-; AVX512F-NEXT:  LBB21_4: ## %else2
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpgtw %xmm0, %xmm2, %xmm2
-; AVX512F-NEXT:    vpmovsxwq %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmq %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $2, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $4, %al
 ; AVX512F-NEXT:    je LBB21_6
-; AVX512F-NEXT:  ## %bb.5: ## %cond.load4
+; AVX512F-NEXT:  LBB21_5: ## %cond.load4
 ; AVX512F-NEXT:    vpinsrw $2, 4(%rdi), %xmm1, %xmm1
-; AVX512F-NEXT:  LBB21_6: ## %else5
-; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $8, %al
 ; AVX512F-NEXT:    je LBB21_8
-; AVX512F-NEXT:  ## %bb.7: ## %cond.load7
+; AVX512F-NEXT:  LBB21_7: ## %cond.load7
 ; AVX512F-NEXT:    vpinsrw $3, 6(%rdi), %xmm1, %xmm1
-; AVX512F-NEXT:  LBB21_8: ## %else8
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpgtw %xmm0, %xmm2, %xmm2
-; AVX512F-NEXT:    vpmovsxwq %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmq %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $4, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $16, %al
 ; AVX512F-NEXT:    je LBB21_10
-; AVX512F-NEXT:  ## %bb.9: ## %cond.load10
+; AVX512F-NEXT:  LBB21_9: ## %cond.load10
 ; AVX512F-NEXT:    vpinsrw $4, 8(%rdi), %xmm1, %xmm1
-; AVX512F-NEXT:  LBB21_10: ## %else11
-; AVX512F-NEXT:    kshiftrw $5, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $32, %al
 ; AVX512F-NEXT:    je LBB21_12
-; AVX512F-NEXT:  ## %bb.11: ## %cond.load13
+; AVX512F-NEXT:  LBB21_11: ## %cond.load13
 ; AVX512F-NEXT:    vpinsrw $5, 10(%rdi), %xmm1, %xmm1
-; AVX512F-NEXT:  LBB21_12: ## %else14
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpgtw %xmm0, %xmm2, %xmm0
-; AVX512F-NEXT:    vpmovsxwq %xmm0, %zmm0
-; AVX512F-NEXT:    vptestmq %zmm0, %zmm0, %k0
-; AVX512F-NEXT:    kshiftrw $6, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $64, %al
 ; AVX512F-NEXT:    je LBB21_14
-; AVX512F-NEXT:  ## %bb.13: ## %cond.load16
+; AVX512F-NEXT:  LBB21_13: ## %cond.load16
 ; AVX512F-NEXT:    vpinsrw $6, 12(%rdi), %xmm1, %xmm1
-; AVX512F-NEXT:  LBB21_14: ## %else17
-; AVX512F-NEXT:    kshiftrw $7, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $-128, %al
 ; AVX512F-NEXT:    je LBB21_16
-; AVX512F-NEXT:  ## %bb.15: ## %cond.load19
+; AVX512F-NEXT:  LBB21_15: ## %cond.load19
 ; AVX512F-NEXT:    vpinsrw $7, 14(%rdi), %xmm1, %xmm1
-; AVX512F-NEXT:  LBB21_16: ## %else20
 ; AVX512F-NEXT:    vmovdqa %xmm1, %xmm0
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
@@ -2910,76 +2857,67 @@ define <8 x i16> @load_v8i16_v8i16(<8 x i16> %trigger, <8 x i16>* %addr, <8 x i1
 ; AVX512VLDQ-LABEL: load_v8i16_v8i16:
 ; AVX512VLDQ:       ## %bb.0:
 ; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpgtw %xmm0, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpmovsxwd %xmm2, %ymm2
-; AVX512VLDQ-NEXT:    vpmovd2m %ymm2, %k0
+; AVX512VLDQ-NEXT:    vpcmpgtw %xmm0, %xmm2, %xmm0
+; AVX512VLDQ-NEXT:    vpmovsxwd %xmm0, %ymm0
+; AVX512VLDQ-NEXT:    vpmovd2m %ymm0, %k0
 ; AVX512VLDQ-NEXT:    kmovw %k0, %eax
 ; AVX512VLDQ-NEXT:    testb $1, %al
-; AVX512VLDQ-NEXT:    je LBB21_2
-; AVX512VLDQ-NEXT:  ## %bb.1: ## %cond.load
+; AVX512VLDQ-NEXT:    jne LBB21_1
+; AVX512VLDQ-NEXT:  ## %bb.2: ## %else
+; AVX512VLDQ-NEXT:    testb $2, %al
+; AVX512VLDQ-NEXT:    jne LBB21_3
+; AVX512VLDQ-NEXT:  LBB21_4: ## %else2
+; AVX512VLDQ-NEXT:    testb $4, %al
+; AVX512VLDQ-NEXT:    jne LBB21_5
+; AVX512VLDQ-NEXT:  LBB21_6: ## %else5
+; AVX512VLDQ-NEXT:    testb $8, %al
+; AVX512VLDQ-NEXT:    jne LBB21_7
+; AVX512VLDQ-NEXT:  LBB21_8: ## %else8
+; AVX512VLDQ-NEXT:    testb $16, %al
+; AVX512VLDQ-NEXT:    jne LBB21_9
+; AVX512VLDQ-NEXT:  LBB21_10: ## %else11
+; AVX512VLDQ-NEXT:    testb $32, %al
+; AVX512VLDQ-NEXT:    jne LBB21_11
+; AVX512VLDQ-NEXT:  LBB21_12: ## %else14
+; AVX512VLDQ-NEXT:    testb $64, %al
+; AVX512VLDQ-NEXT:    jne LBB21_13
+; AVX512VLDQ-NEXT:  LBB21_14: ## %else17
+; AVX512VLDQ-NEXT:    testb $-128, %al
+; AVX512VLDQ-NEXT:    jne LBB21_15
+; AVX512VLDQ-NEXT:  LBB21_16: ## %else20
+; AVX512VLDQ-NEXT:    vmovdqa %xmm1, %xmm0
+; AVX512VLDQ-NEXT:    vzeroupper
+; AVX512VLDQ-NEXT:    retq
+; AVX512VLDQ-NEXT:  LBB21_1: ## %cond.load
 ; AVX512VLDQ-NEXT:    vpinsrw $0, (%rdi), %xmm1, %xmm1
-; AVX512VLDQ-NEXT:  LBB21_2: ## %else
-; AVX512VLDQ-NEXT:    kshiftrb $1, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $2, %al
 ; AVX512VLDQ-NEXT:    je LBB21_4
-; AVX512VLDQ-NEXT:  ## %bb.3: ## %cond.load1
+; AVX512VLDQ-NEXT:  LBB21_3: ## %cond.load1
 ; AVX512VLDQ-NEXT:    vpinsrw $1, 2(%rdi), %xmm1, %xmm1
-; AVX512VLDQ-NEXT:  LBB21_4: ## %else2
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpgtw %xmm0, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpmovsxwd %xmm2, %ymm2
-; AVX512VLDQ-NEXT:    vpmovd2m %ymm2, %k0
-; AVX512VLDQ-NEXT:    kshiftrb $2, %k0, %k1
-; AVX512VLDQ-NEXT:    kmovw %k1, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $4, %al
 ; AVX512VLDQ-NEXT:    je LBB21_6
-; AVX512VLDQ-NEXT:  ## %bb.5: ## %cond.load4
+; AVX512VLDQ-NEXT:  LBB21_5: ## %cond.load4
 ; AVX512VLDQ-NEXT:    vpinsrw $2, 4(%rdi), %xmm1, %xmm1
-; AVX512VLDQ-NEXT:  LBB21_6: ## %else5
-; AVX512VLDQ-NEXT:    kshiftrb $3, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $8, %al
 ; AVX512VLDQ-NEXT:    je LBB21_8
-; AVX512VLDQ-NEXT:  ## %bb.7: ## %cond.load7
+; AVX512VLDQ-NEXT:  LBB21_7: ## %cond.load7
 ; AVX512VLDQ-NEXT:    vpinsrw $3, 6(%rdi), %xmm1, %xmm1
-; AVX512VLDQ-NEXT:  LBB21_8: ## %else8
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpgtw %xmm0, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpmovsxwd %xmm2, %ymm2
-; AVX512VLDQ-NEXT:    vpmovd2m %ymm2, %k0
-; AVX512VLDQ-NEXT:    kshiftrb $4, %k0, %k1
-; AVX512VLDQ-NEXT:    kmovw %k1, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $16, %al
 ; AVX512VLDQ-NEXT:    je LBB21_10
-; AVX512VLDQ-NEXT:  ## %bb.9: ## %cond.load10
+; AVX512VLDQ-NEXT:  LBB21_9: ## %cond.load10
 ; AVX512VLDQ-NEXT:    vpinsrw $4, 8(%rdi), %xmm1, %xmm1
-; AVX512VLDQ-NEXT:  LBB21_10: ## %else11
-; AVX512VLDQ-NEXT:    kshiftrb $5, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $32, %al
 ; AVX512VLDQ-NEXT:    je LBB21_12
-; AVX512VLDQ-NEXT:  ## %bb.11: ## %cond.load13
+; AVX512VLDQ-NEXT:  LBB21_11: ## %cond.load13
 ; AVX512VLDQ-NEXT:    vpinsrw $5, 10(%rdi), %xmm1, %xmm1
-; AVX512VLDQ-NEXT:  LBB21_12: ## %else14
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpgtw %xmm0, %xmm2, %xmm0
-; AVX512VLDQ-NEXT:    vpmovsxwd %xmm0, %ymm0
-; AVX512VLDQ-NEXT:    vpmovd2m %ymm0, %k0
-; AVX512VLDQ-NEXT:    kshiftrb $6, %k0, %k1
-; AVX512VLDQ-NEXT:    kmovw %k1, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $64, %al
 ; AVX512VLDQ-NEXT:    je LBB21_14
-; AVX512VLDQ-NEXT:  ## %bb.13: ## %cond.load16
+; AVX512VLDQ-NEXT:  LBB21_13: ## %cond.load16
 ; AVX512VLDQ-NEXT:    vpinsrw $6, 12(%rdi), %xmm1, %xmm1
-; AVX512VLDQ-NEXT:  LBB21_14: ## %else17
-; AVX512VLDQ-NEXT:    kshiftrb $7, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $-128, %al
 ; AVX512VLDQ-NEXT:    je LBB21_16
-; AVX512VLDQ-NEXT:  ## %bb.15: ## %cond.load19
+; AVX512VLDQ-NEXT:  LBB21_15: ## %cond.load19
 ; AVX512VLDQ-NEXT:    vpinsrw $7, 14(%rdi), %xmm1, %xmm1
-; AVX512VLDQ-NEXT:  LBB21_16: ## %else20
 ; AVX512VLDQ-NEXT:    vmovdqa %xmm1, %xmm0
 ; AVX512VLDQ-NEXT:    vzeroupper
 ; AVX512VLDQ-NEXT:    retq
@@ -2995,875 +2933,704 @@ define <8 x i16> @load_v8i16_v8i16(<8 x i16> %trigger, <8 x i16>* %addr, <8 x i1
 }
 
 define <16 x i16> @load_v16i16_v16i16(<16 x i16> %trigger, <16 x i16>* %addr, <16 x i16> %dst) {
-; SSE2-LABEL: load_v16i16_v16i16:
-; SSE2:       ## %bb.0:
-; SSE2-NEXT:    pxor %xmm4, %xmm4
-; SSE2-NEXT:    pcmpgtw %xmm0, %xmm4
-; SSE2-NEXT:    movd %xmm4, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB22_2
-; SSE2-NEXT:  ## %bb.1: ## %cond.load
-; SSE2-NEXT:    pinsrw $0, (%rdi), %xmm2
-; SSE2-NEXT:  LBB22_2: ## %else
-; SSE2-NEXT:    shrl $16, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB22_4
-; SSE2-NEXT:  ## %bb.3: ## %cond.load1
-; SSE2-NEXT:    pinsrw $1, 2(%rdi), %xmm2
-; SSE2-NEXT:  LBB22_4: ## %else2
-; SSE2-NEXT:    pxor %xmm4, %xmm4
-; SSE2-NEXT:    pcmpgtw %xmm0, %xmm4
-; SSE2-NEXT:    pextrw $2, %xmm4, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB22_6
-; SSE2-NEXT:  ## %bb.5: ## %cond.load4
-; SSE2-NEXT:    pinsrw $2, 4(%rdi), %xmm2
-; SSE2-NEXT:  LBB22_6: ## %else5
-; SSE2-NEXT:    pextrw $3, %xmm4, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB22_8
-; SSE2-NEXT:  ## %bb.7: ## %cond.load7
-; SSE2-NEXT:    pinsrw $3, 6(%rdi), %xmm2
-; SSE2-NEXT:  LBB22_8: ## %else8
-; SSE2-NEXT:    pxor %xmm4, %xmm4
-; SSE2-NEXT:    pcmpgtw %xmm0, %xmm4
-; SSE2-NEXT:    pextrw $4, %xmm4, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB22_10
-; SSE2-NEXT:  ## %bb.9: ## %cond.load10
-; SSE2-NEXT:    pinsrw $4, 8(%rdi), %xmm2
-; SSE2-NEXT:  LBB22_10: ## %else11
-; SSE2-NEXT:    pextrw $5, %xmm4, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB22_12
-; SSE2-NEXT:  ## %bb.11: ## %cond.load13
-; SSE2-NEXT:    pinsrw $5, 10(%rdi), %xmm2
-; SSE2-NEXT:  LBB22_12: ## %else14
-; SSE2-NEXT:    pxor %xmm4, %xmm4
-; SSE2-NEXT:    pcmpgtw %xmm0, %xmm4
-; SSE2-NEXT:    pextrw $6, %xmm4, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB22_14
-; SSE2-NEXT:  ## %bb.13: ## %cond.load16
-; SSE2-NEXT:    pinsrw $6, 12(%rdi), %xmm2
-; SSE2-NEXT:  LBB22_14: ## %else17
-; SSE2-NEXT:    pextrw $7, %xmm4, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB22_16
-; SSE2-NEXT:  ## %bb.15: ## %cond.load19
-; SSE2-NEXT:    pinsrw $7, 14(%rdi), %xmm2
-; SSE2-NEXT:  LBB22_16: ## %else20
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpgtw %xmm1, %xmm0
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB22_18
-; SSE2-NEXT:  ## %bb.17: ## %cond.load22
-; SSE2-NEXT:    pinsrw $0, 16(%rdi), %xmm3
-; SSE2-NEXT:  LBB22_18: ## %else23
-; SSE2-NEXT:    shrl $16, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB22_20
-; SSE2-NEXT:  ## %bb.19: ## %cond.load25
-; SSE2-NEXT:    pinsrw $1, 18(%rdi), %xmm3
-; SSE2-NEXT:  LBB22_20: ## %else26
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpgtw %xmm1, %xmm0
-; SSE2-NEXT:    pextrw $2, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB22_22
-; SSE2-NEXT:  ## %bb.21: ## %cond.load28
-; SSE2-NEXT:    pinsrw $2, 20(%rdi), %xmm3
-; SSE2-NEXT:  LBB22_22: ## %else29
-; SSE2-NEXT:    pextrw $3, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB22_24
-; SSE2-NEXT:  ## %bb.23: ## %cond.load31
-; SSE2-NEXT:    pinsrw $3, 22(%rdi), %xmm3
-; SSE2-NEXT:  LBB22_24: ## %else32
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpgtw %xmm1, %xmm0
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB22_26
-; SSE2-NEXT:  ## %bb.25: ## %cond.load34
-; SSE2-NEXT:    pinsrw $4, 24(%rdi), %xmm3
-; SSE2-NEXT:  LBB22_26: ## %else35
-; SSE2-NEXT:    pextrw $5, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB22_28
-; SSE2-NEXT:  ## %bb.27: ## %cond.load37
-; SSE2-NEXT:    pinsrw $5, 26(%rdi), %xmm3
-; SSE2-NEXT:  LBB22_28: ## %else38
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpgtw %xmm1, %xmm0
-; SSE2-NEXT:    pextrw $6, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB22_30
-; SSE2-NEXT:  ## %bb.29: ## %cond.load40
-; SSE2-NEXT:    pinsrw $6, 28(%rdi), %xmm3
-; SSE2-NEXT:  LBB22_30: ## %else41
-; SSE2-NEXT:    pextrw $7, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB22_32
-; SSE2-NEXT:  ## %bb.31: ## %cond.load43
-; SSE2-NEXT:    pinsrw $7, 30(%rdi), %xmm3
-; SSE2-NEXT:  LBB22_32: ## %else44
-; SSE2-NEXT:    movdqa %xmm2, %xmm0
-; SSE2-NEXT:    movdqa %xmm3, %xmm1
-; SSE2-NEXT:    retq
-;
-; SSE42-LABEL: load_v16i16_v16i16:
-; SSE42:       ## %bb.0:
-; SSE42-NEXT:    pxor %xmm4, %xmm4
-; SSE42-NEXT:    pcmpgtw %xmm0, %xmm4
-; SSE42-NEXT:    pextrb $0, %xmm4, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB22_2
-; SSE42-NEXT:  ## %bb.1: ## %cond.load
-; SSE42-NEXT:    pinsrw $0, (%rdi), %xmm2
-; SSE42-NEXT:  LBB22_2: ## %else
-; SSE42-NEXT:    pextrb $2, %xmm4, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB22_4
-; SSE42-NEXT:  ## %bb.3: ## %cond.load1
-; SSE42-NEXT:    pinsrw $1, 2(%rdi), %xmm2
-; SSE42-NEXT:  LBB22_4: ## %else2
-; SSE42-NEXT:    pxor %xmm4, %xmm4
-; SSE42-NEXT:    pcmpgtw %xmm0, %xmm4
-; SSE42-NEXT:    pextrb $4, %xmm4, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB22_6
-; SSE42-NEXT:  ## %bb.5: ## %cond.load4
-; SSE42-NEXT:    pinsrw $2, 4(%rdi), %xmm2
-; SSE42-NEXT:  LBB22_6: ## %else5
-; SSE42-NEXT:    pextrb $6, %xmm4, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB22_8
-; SSE42-NEXT:  ## %bb.7: ## %cond.load7
-; SSE42-NEXT:    pinsrw $3, 6(%rdi), %xmm2
-; SSE42-NEXT:  LBB22_8: ## %else8
-; SSE42-NEXT:    pxor %xmm4, %xmm4
-; SSE42-NEXT:    pcmpgtw %xmm0, %xmm4
-; SSE42-NEXT:    pextrb $8, %xmm4, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB22_10
-; SSE42-NEXT:  ## %bb.9: ## %cond.load10
-; SSE42-NEXT:    pinsrw $4, 8(%rdi), %xmm2
-; SSE42-NEXT:  LBB22_10: ## %else11
-; SSE42-NEXT:    pextrb $10, %xmm4, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB22_12
-; SSE42-NEXT:  ## %bb.11: ## %cond.load13
-; SSE42-NEXT:    pinsrw $5, 10(%rdi), %xmm2
-; SSE42-NEXT:  LBB22_12: ## %else14
-; SSE42-NEXT:    pxor %xmm4, %xmm4
-; SSE42-NEXT:    pcmpgtw %xmm0, %xmm4
-; SSE42-NEXT:    pextrb $12, %xmm4, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB22_14
-; SSE42-NEXT:  ## %bb.13: ## %cond.load16
-; SSE42-NEXT:    pinsrw $6, 12(%rdi), %xmm2
-; SSE42-NEXT:  LBB22_14: ## %else17
-; SSE42-NEXT:    pextrb $14, %xmm4, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB22_16
-; SSE42-NEXT:  ## %bb.15: ## %cond.load19
-; SSE42-NEXT:    pinsrw $7, 14(%rdi), %xmm2
-; SSE42-NEXT:  LBB22_16: ## %else20
-; SSE42-NEXT:    pxor %xmm0, %xmm0
-; SSE42-NEXT:    pcmpgtw %xmm1, %xmm0
-; SSE42-NEXT:    pextrb $0, %xmm0, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB22_18
-; SSE42-NEXT:  ## %bb.17: ## %cond.load22
-; SSE42-NEXT:    pinsrw $0, 16(%rdi), %xmm3
-; SSE42-NEXT:  LBB22_18: ## %else23
-; SSE42-NEXT:    pextrb $2, %xmm0, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB22_20
-; SSE42-NEXT:  ## %bb.19: ## %cond.load25
-; SSE42-NEXT:    pinsrw $1, 18(%rdi), %xmm3
-; SSE42-NEXT:  LBB22_20: ## %else26
-; SSE42-NEXT:    pxor %xmm0, %xmm0
-; SSE42-NEXT:    pcmpgtw %xmm1, %xmm0
-; SSE42-NEXT:    pextrb $4, %xmm0, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB22_22
-; SSE42-NEXT:  ## %bb.21: ## %cond.load28
-; SSE42-NEXT:    pinsrw $2, 20(%rdi), %xmm3
-; SSE42-NEXT:  LBB22_22: ## %else29
-; SSE42-NEXT:    pextrb $6, %xmm0, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB22_24
-; SSE42-NEXT:  ## %bb.23: ## %cond.load31
-; SSE42-NEXT:    pinsrw $3, 22(%rdi), %xmm3
-; SSE42-NEXT:  LBB22_24: ## %else32
-; SSE42-NEXT:    pxor %xmm0, %xmm0
-; SSE42-NEXT:    pcmpgtw %xmm1, %xmm0
-; SSE42-NEXT:    pextrb $8, %xmm0, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB22_26
-; SSE42-NEXT:  ## %bb.25: ## %cond.load34
-; SSE42-NEXT:    pinsrw $4, 24(%rdi), %xmm3
-; SSE42-NEXT:  LBB22_26: ## %else35
-; SSE42-NEXT:    pextrb $10, %xmm0, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB22_28
-; SSE42-NEXT:  ## %bb.27: ## %cond.load37
-; SSE42-NEXT:    pinsrw $5, 26(%rdi), %xmm3
-; SSE42-NEXT:  LBB22_28: ## %else38
-; SSE42-NEXT:    pxor %xmm0, %xmm0
-; SSE42-NEXT:    pcmpgtw %xmm1, %xmm0
-; SSE42-NEXT:    pextrb $12, %xmm0, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB22_30
-; SSE42-NEXT:  ## %bb.29: ## %cond.load40
-; SSE42-NEXT:    pinsrw $6, 28(%rdi), %xmm3
-; SSE42-NEXT:  LBB22_30: ## %else41
-; SSE42-NEXT:    pextrb $14, %xmm0, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB22_32
-; SSE42-NEXT:  ## %bb.31: ## %cond.load43
-; SSE42-NEXT:    pinsrw $7, 30(%rdi), %xmm3
-; SSE42-NEXT:  LBB22_32: ## %else44
-; SSE42-NEXT:    movdqa %xmm2, %xmm0
-; SSE42-NEXT:    movdqa %xmm3, %xmm1
-; SSE42-NEXT:    retq
+; SSE-LABEL: load_v16i16_v16i16:
+; SSE:       ## %bb.0:
+; SSE-NEXT:    packsswb %xmm1, %xmm0
+; SSE-NEXT:    pmovmskb %xmm0, %eax
+; SSE-NEXT:    testb $1, %al
+; SSE-NEXT:    jne LBB22_1
+; SSE-NEXT:  ## %bb.2: ## %else
+; SSE-NEXT:    testb $2, %al
+; SSE-NEXT:    jne LBB22_3
+; SSE-NEXT:  LBB22_4: ## %else2
+; SSE-NEXT:    testb $4, %al
+; SSE-NEXT:    jne LBB22_5
+; SSE-NEXT:  LBB22_6: ## %else5
+; SSE-NEXT:    testb $8, %al
+; SSE-NEXT:    jne LBB22_7
+; SSE-NEXT:  LBB22_8: ## %else8
+; SSE-NEXT:    testb $16, %al
+; SSE-NEXT:    jne LBB22_9
+; SSE-NEXT:  LBB22_10: ## %else11
+; SSE-NEXT:    testb $32, %al
+; SSE-NEXT:    jne LBB22_11
+; SSE-NEXT:  LBB22_12: ## %else14
+; SSE-NEXT:    testb $64, %al
+; SSE-NEXT:    jne LBB22_13
+; SSE-NEXT:  LBB22_14: ## %else17
+; SSE-NEXT:    testb $-128, %al
+; SSE-NEXT:    jne LBB22_15
+; SSE-NEXT:  LBB22_16: ## %else20
+; SSE-NEXT:    testl $256, %eax ## imm = 0x100
+; SSE-NEXT:    jne LBB22_17
+; SSE-NEXT:  LBB22_18: ## %else23
+; SSE-NEXT:    testl $512, %eax ## imm = 0x200
+; SSE-NEXT:    jne LBB22_19
+; SSE-NEXT:  LBB22_20: ## %else26
+; SSE-NEXT:    testl $1024, %eax ## imm = 0x400
+; SSE-NEXT:    jne LBB22_21
+; SSE-NEXT:  LBB22_22: ## %else29
+; SSE-NEXT:    testl $2048, %eax ## imm = 0x800
+; SSE-NEXT:    jne LBB22_23
+; SSE-NEXT:  LBB22_24: ## %else32
+; SSE-NEXT:    testl $4096, %eax ## imm = 0x1000
+; SSE-NEXT:    jne LBB22_25
+; SSE-NEXT:  LBB22_26: ## %else35
+; SSE-NEXT:    testl $8192, %eax ## imm = 0x2000
+; SSE-NEXT:    jne LBB22_27
+; SSE-NEXT:  LBB22_28: ## %else38
+; SSE-NEXT:    testl $16384, %eax ## imm = 0x4000
+; SSE-NEXT:    jne LBB22_29
+; SSE-NEXT:  LBB22_30: ## %else41
+; SSE-NEXT:    testl $32768, %eax ## imm = 0x8000
+; SSE-NEXT:    je LBB22_32
+; SSE-NEXT:  LBB22_31: ## %cond.load43
+; SSE-NEXT:    pinsrw $7, 30(%rdi), %xmm3
+; SSE-NEXT:  LBB22_32: ## %else44
+; SSE-NEXT:    movdqa %xmm2, %xmm0
+; SSE-NEXT:    movdqa %xmm3, %xmm1
+; SSE-NEXT:    retq
+; SSE-NEXT:  LBB22_1: ## %cond.load
+; SSE-NEXT:    pinsrw $0, (%rdi), %xmm2
+; SSE-NEXT:    testb $2, %al
+; SSE-NEXT:    je LBB22_4
+; SSE-NEXT:  LBB22_3: ## %cond.load1
+; SSE-NEXT:    pinsrw $1, 2(%rdi), %xmm2
+; SSE-NEXT:    testb $4, %al
+; SSE-NEXT:    je LBB22_6
+; SSE-NEXT:  LBB22_5: ## %cond.load4
+; SSE-NEXT:    pinsrw $2, 4(%rdi), %xmm2
+; SSE-NEXT:    testb $8, %al
+; SSE-NEXT:    je LBB22_8
+; SSE-NEXT:  LBB22_7: ## %cond.load7
+; SSE-NEXT:    pinsrw $3, 6(%rdi), %xmm2
+; SSE-NEXT:    testb $16, %al
+; SSE-NEXT:    je LBB22_10
+; SSE-NEXT:  LBB22_9: ## %cond.load10
+; SSE-NEXT:    pinsrw $4, 8(%rdi), %xmm2
+; SSE-NEXT:    testb $32, %al
+; SSE-NEXT:    je LBB22_12
+; SSE-NEXT:  LBB22_11: ## %cond.load13
+; SSE-NEXT:    pinsrw $5, 10(%rdi), %xmm2
+; SSE-NEXT:    testb $64, %al
+; SSE-NEXT:    je LBB22_14
+; SSE-NEXT:  LBB22_13: ## %cond.load16
+; SSE-NEXT:    pinsrw $6, 12(%rdi), %xmm2
+; SSE-NEXT:    testb $-128, %al
+; SSE-NEXT:    je LBB22_16
+; SSE-NEXT:  LBB22_15: ## %cond.load19
+; SSE-NEXT:    pinsrw $7, 14(%rdi), %xmm2
+; SSE-NEXT:    testl $256, %eax ## imm = 0x100
+; SSE-NEXT:    je LBB22_18
+; SSE-NEXT:  LBB22_17: ## %cond.load22
+; SSE-NEXT:    pinsrw $0, 16(%rdi), %xmm3
+; SSE-NEXT:    testl $512, %eax ## imm = 0x200
+; SSE-NEXT:    je LBB22_20
+; SSE-NEXT:  LBB22_19: ## %cond.load25
+; SSE-NEXT:    pinsrw $1, 18(%rdi), %xmm3
+; SSE-NEXT:    testl $1024, %eax ## imm = 0x400
+; SSE-NEXT:    je LBB22_22
+; SSE-NEXT:  LBB22_21: ## %cond.load28
+; SSE-NEXT:    pinsrw $2, 20(%rdi), %xmm3
+; SSE-NEXT:    testl $2048, %eax ## imm = 0x800
+; SSE-NEXT:    je LBB22_24
+; SSE-NEXT:  LBB22_23: ## %cond.load31
+; SSE-NEXT:    pinsrw $3, 22(%rdi), %xmm3
+; SSE-NEXT:    testl $4096, %eax ## imm = 0x1000
+; SSE-NEXT:    je LBB22_26
+; SSE-NEXT:  LBB22_25: ## %cond.load34
+; SSE-NEXT:    pinsrw $4, 24(%rdi), %xmm3
+; SSE-NEXT:    testl $8192, %eax ## imm = 0x2000
+; SSE-NEXT:    je LBB22_28
+; SSE-NEXT:  LBB22_27: ## %cond.load37
+; SSE-NEXT:    pinsrw $5, 26(%rdi), %xmm3
+; SSE-NEXT:    testl $16384, %eax ## imm = 0x4000
+; SSE-NEXT:    je LBB22_30
+; SSE-NEXT:  LBB22_29: ## %cond.load40
+; SSE-NEXT:    pinsrw $6, 28(%rdi), %xmm3
+; SSE-NEXT:    testl $32768, %eax ## imm = 0x8000
+; SSE-NEXT:    jne LBB22_31
+; SSE-NEXT:    jmp LBB22_32
 ;
 ; AVX1-LABEL: load_v16i16_v16i16:
 ; AVX1:       ## %bb.0:
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpgtw %xmm0, %xmm2, %xmm3
-; AVX1-NEXT:    vpextrb $0, %xmm3, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je LBB22_2
-; AVX1-NEXT:  ## %bb.1: ## %cond.load
-; AVX1-NEXT:    vpinsrw $0, (%rdi), %xmm1, %xmm3
-; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
-; AVX1-NEXT:  LBB22_2: ## %else
-; AVX1-NEXT:    vpcmpgtw %xmm0, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $2, %xmm2, %eax
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT:    vpacksswb %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpmovmskb %xmm0, %eax
 ; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je LBB22_4
-; AVX1-NEXT:  ## %bb.3: ## %cond.load1
-; AVX1-NEXT:    vpinsrw $1, 2(%rdi), %xmm1, %xmm2
-; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX1-NEXT:    jne LBB22_1
+; AVX1-NEXT:  ## %bb.2: ## %else
+; AVX1-NEXT:    testb $2, %al
+; AVX1-NEXT:    jne LBB22_3
 ; AVX1-NEXT:  LBB22_4: ## %else2
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpgtw %xmm0, %xmm2, %xmm3
-; AVX1-NEXT:    vpextrb $4, %xmm3, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je LBB22_6
-; AVX1-NEXT:  ## %bb.5: ## %cond.load4
-; AVX1-NEXT:    vpinsrw $2, 4(%rdi), %xmm1, %xmm3
-; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
+; AVX1-NEXT:    testb $4, %al
+; AVX1-NEXT:    jne LBB22_5
 ; AVX1-NEXT:  LBB22_6: ## %else5
-; AVX1-NEXT:    vpcmpgtw %xmm0, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $6, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je LBB22_8
-; AVX1-NEXT:  ## %bb.7: ## %cond.load7
-; AVX1-NEXT:    vpinsrw $3, 6(%rdi), %xmm1, %xmm2
-; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX1-NEXT:    testb $8, %al
+; AVX1-NEXT:    jne LBB22_7
 ; AVX1-NEXT:  LBB22_8: ## %else8
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpgtw %xmm0, %xmm2, %xmm3
-; AVX1-NEXT:    vpextrb $8, %xmm3, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je LBB22_10
-; AVX1-NEXT:  ## %bb.9: ## %cond.load10
-; AVX1-NEXT:    vpinsrw $4, 8(%rdi), %xmm1, %xmm3
-; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
+; AVX1-NEXT:    testb $16, %al
+; AVX1-NEXT:    jne LBB22_9
 ; AVX1-NEXT:  LBB22_10: ## %else11
-; AVX1-NEXT:    vpcmpgtw %xmm0, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $10, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je LBB22_12
-; AVX1-NEXT:  ## %bb.11: ## %cond.load13
-; AVX1-NEXT:    vpinsrw $5, 10(%rdi), %xmm1, %xmm2
-; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX1-NEXT:    testb $32, %al
+; AVX1-NEXT:    jne LBB22_11
 ; AVX1-NEXT:  LBB22_12: ## %else14
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpgtw %xmm0, %xmm2, %xmm3
-; AVX1-NEXT:    vpextrb $12, %xmm3, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je LBB22_14
-; AVX1-NEXT:  ## %bb.13: ## %cond.load16
-; AVX1-NEXT:    vpinsrw $6, 12(%rdi), %xmm1, %xmm3
-; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
+; AVX1-NEXT:    testb $64, %al
+; AVX1-NEXT:    jne LBB22_13
 ; AVX1-NEXT:  LBB22_14: ## %else17
-; AVX1-NEXT:    vpcmpgtw %xmm0, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $14, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je LBB22_16
-; AVX1-NEXT:  ## %bb.15: ## %cond.load19
-; AVX1-NEXT:    vpinsrw $7, 14(%rdi), %xmm1, %xmm2
-; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX1-NEXT:    testb $-128, %al
+; AVX1-NEXT:    jne LBB22_15
 ; AVX1-NEXT:  LBB22_16: ## %else20
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpgtw %xmm0, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $0, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je LBB22_18
-; AVX1-NEXT:  ## %bb.17: ## %cond.load22
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
-; AVX1-NEXT:    vpinsrw $0, 16(%rdi), %xmm3, %xmm3
-; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
+; AVX1-NEXT:    testl $256, %eax ## imm = 0x100
+; AVX1-NEXT:    jne LBB22_17
 ; AVX1-NEXT:  LBB22_18: ## %else23
-; AVX1-NEXT:    vpextrb $2, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je LBB22_20
-; AVX1-NEXT:  ## %bb.19: ## %cond.load25
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT:    vpinsrw $1, 18(%rdi), %xmm2, %xmm2
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; AVX1-NEXT:    testl $512, %eax ## imm = 0x200
+; AVX1-NEXT:    jne LBB22_19
 ; AVX1-NEXT:  LBB22_20: ## %else26
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpgtw %xmm0, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je LBB22_22
-; AVX1-NEXT:  ## %bb.21: ## %cond.load28
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
-; AVX1-NEXT:    vpinsrw $2, 20(%rdi), %xmm3, %xmm3
-; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
+; AVX1-NEXT:    testl $1024, %eax ## imm = 0x400
+; AVX1-NEXT:    jne LBB22_21
 ; AVX1-NEXT:  LBB22_22: ## %else29
-; AVX1-NEXT:    vpextrb $6, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je LBB22_24
-; AVX1-NEXT:  ## %bb.23: ## %cond.load31
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT:    vpinsrw $3, 22(%rdi), %xmm2, %xmm2
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; AVX1-NEXT:    testl $2048, %eax ## imm = 0x800
+; AVX1-NEXT:    jne LBB22_23
 ; AVX1-NEXT:  LBB22_24: ## %else32
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpgtw %xmm0, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $8, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je LBB22_26
-; AVX1-NEXT:  ## %bb.25: ## %cond.load34
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
-; AVX1-NEXT:    vpinsrw $4, 24(%rdi), %xmm3, %xmm3
-; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
+; AVX1-NEXT:    testl $4096, %eax ## imm = 0x1000
+; AVX1-NEXT:    jne LBB22_25
 ; AVX1-NEXT:  LBB22_26: ## %else35
-; AVX1-NEXT:    vpextrb $10, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je LBB22_28
-; AVX1-NEXT:  ## %bb.27: ## %cond.load37
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT:    vpinsrw $5, 26(%rdi), %xmm2, %xmm2
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; AVX1-NEXT:    testl $8192, %eax ## imm = 0x2000
+; AVX1-NEXT:    jne LBB22_27
 ; AVX1-NEXT:  LBB22_28: ## %else38
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpgtw %xmm0, %xmm2, %xmm0
-; AVX1-NEXT:    vpextrb $12, %xmm0, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je LBB22_30
-; AVX1-NEXT:  ## %bb.29: ## %cond.load40
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT:    vpinsrw $6, 28(%rdi), %xmm2, %xmm2
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; AVX1-NEXT:    testl $16384, %eax ## imm = 0x4000
+; AVX1-NEXT:    jne LBB22_29
 ; AVX1-NEXT:  LBB22_30: ## %else41
-; AVX1-NEXT:    vpextrb $14, %xmm0, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $32768, %eax ## imm = 0x8000
+; AVX1-NEXT:    jne LBB22_31
+; AVX1-NEXT:  LBB22_32: ## %else44
+; AVX1-NEXT:    vmovaps %ymm1, %ymm0
+; AVX1-NEXT:    retq
+; AVX1-NEXT:  LBB22_1: ## %cond.load
+; AVX1-NEXT:    vpinsrw $0, (%rdi), %xmm1, %xmm0
+; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX1-NEXT:    testb $2, %al
+; AVX1-NEXT:    je LBB22_4
+; AVX1-NEXT:  LBB22_3: ## %cond.load1
+; AVX1-NEXT:    vpinsrw $1, 2(%rdi), %xmm1, %xmm0
+; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX1-NEXT:    testb $4, %al
+; AVX1-NEXT:    je LBB22_6
+; AVX1-NEXT:  LBB22_5: ## %cond.load4
+; AVX1-NEXT:    vpinsrw $2, 4(%rdi), %xmm1, %xmm0
+; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX1-NEXT:    testb $8, %al
+; AVX1-NEXT:    je LBB22_8
+; AVX1-NEXT:  LBB22_7: ## %cond.load7
+; AVX1-NEXT:    vpinsrw $3, 6(%rdi), %xmm1, %xmm0
+; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX1-NEXT:    testb $16, %al
+; AVX1-NEXT:    je LBB22_10
+; AVX1-NEXT:  LBB22_9: ## %cond.load10
+; AVX1-NEXT:    vpinsrw $4, 8(%rdi), %xmm1, %xmm0
+; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX1-NEXT:    testb $32, %al
+; AVX1-NEXT:    je LBB22_12
+; AVX1-NEXT:  LBB22_11: ## %cond.load13
+; AVX1-NEXT:    vpinsrw $5, 10(%rdi), %xmm1, %xmm0
+; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX1-NEXT:    testb $64, %al
+; AVX1-NEXT:    je LBB22_14
+; AVX1-NEXT:  LBB22_13: ## %cond.load16
+; AVX1-NEXT:    vpinsrw $6, 12(%rdi), %xmm1, %xmm0
+; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX1-NEXT:    testb $-128, %al
+; AVX1-NEXT:    je LBB22_16
+; AVX1-NEXT:  LBB22_15: ## %cond.load19
+; AVX1-NEXT:    vpinsrw $7, 14(%rdi), %xmm1, %xmm0
+; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX1-NEXT:    testl $256, %eax ## imm = 0x100
+; AVX1-NEXT:    je LBB22_18
+; AVX1-NEXT:  LBB22_17: ## %cond.load22
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm0
+; AVX1-NEXT:    vpinsrw $0, 16(%rdi), %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm1
+; AVX1-NEXT:    testl $512, %eax ## imm = 0x200
+; AVX1-NEXT:    je LBB22_20
+; AVX1-NEXT:  LBB22_19: ## %cond.load25
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm0
+; AVX1-NEXT:    vpinsrw $1, 18(%rdi), %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm1
+; AVX1-NEXT:    testl $1024, %eax ## imm = 0x400
+; AVX1-NEXT:    je LBB22_22
+; AVX1-NEXT:  LBB22_21: ## %cond.load28
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm0
+; AVX1-NEXT:    vpinsrw $2, 20(%rdi), %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm1
+; AVX1-NEXT:    testl $2048, %eax ## imm = 0x800
+; AVX1-NEXT:    je LBB22_24
+; AVX1-NEXT:  LBB22_23: ## %cond.load31
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm0
+; AVX1-NEXT:    vpinsrw $3, 22(%rdi), %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm1
+; AVX1-NEXT:    testl $4096, %eax ## imm = 0x1000
+; AVX1-NEXT:    je LBB22_26
+; AVX1-NEXT:  LBB22_25: ## %cond.load34
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm0
+; AVX1-NEXT:    vpinsrw $4, 24(%rdi), %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm1
+; AVX1-NEXT:    testl $8192, %eax ## imm = 0x2000
+; AVX1-NEXT:    je LBB22_28
+; AVX1-NEXT:  LBB22_27: ## %cond.load37
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm0
+; AVX1-NEXT:    vpinsrw $5, 26(%rdi), %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm1
+; AVX1-NEXT:    testl $16384, %eax ## imm = 0x4000
+; AVX1-NEXT:    je LBB22_30
+; AVX1-NEXT:  LBB22_29: ## %cond.load40
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm0
+; AVX1-NEXT:    vpinsrw $6, 28(%rdi), %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm1
+; AVX1-NEXT:    testl $32768, %eax ## imm = 0x8000
 ; AVX1-NEXT:    je LBB22_32
-; AVX1-NEXT:  ## %bb.31: ## %cond.load43
+; AVX1-NEXT:  LBB22_31: ## %cond.load43
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm0
 ; AVX1-NEXT:    vpinsrw $7, 30(%rdi), %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm1
-; AVX1-NEXT:  LBB22_32: ## %else44
 ; AVX1-NEXT:    vmovaps %ymm1, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: load_v16i16_v16i16:
 ; AVX2:       ## %bb.0:
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtw %xmm0, %xmm2, %xmm3
-; AVX2-NEXT:    vpextrb $0, %xmm3, %eax
+; AVX2-NEXT:    vpcmpgtw %ymm0, %ymm2, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm2
+; AVX2-NEXT:    vpacksswb %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpmovmskb %xmm0, %eax
 ; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je LBB22_2
-; AVX2-NEXT:  ## %bb.1: ## %cond.load
-; AVX2-NEXT:    vpinsrw $0, (%rdi), %xmm1, %xmm3
-; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
-; AVX2-NEXT:  LBB22_2: ## %else
-; AVX2-NEXT:    vpcmpgtw %xmm0, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrb $2, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je LBB22_4
-; AVX2-NEXT:  ## %bb.3: ## %cond.load1
-; AVX2-NEXT:    vpinsrw $1, 2(%rdi), %xmm1, %xmm2
-; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-NEXT:    jne LBB22_1
+; AVX2-NEXT:  ## %bb.2: ## %else
+; AVX2-NEXT:    testb $2, %al
+; AVX2-NEXT:    jne LBB22_3
 ; AVX2-NEXT:  LBB22_4: ## %else2
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtw %xmm0, %xmm2, %xmm3
-; AVX2-NEXT:    vpextrb $4, %xmm3, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je LBB22_6
-; AVX2-NEXT:  ## %bb.5: ## %cond.load4
-; AVX2-NEXT:    vpinsrw $2, 4(%rdi), %xmm1, %xmm3
-; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-NEXT:    testb $4, %al
+; AVX2-NEXT:    jne LBB22_5
 ; AVX2-NEXT:  LBB22_6: ## %else5
-; AVX2-NEXT:    vpcmpgtw %xmm0, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrb $6, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je LBB22_8
-; AVX2-NEXT:  ## %bb.7: ## %cond.load7
-; AVX2-NEXT:    vpinsrw $3, 6(%rdi), %xmm1, %xmm2
-; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-NEXT:    testb $8, %al
+; AVX2-NEXT:    jne LBB22_7
 ; AVX2-NEXT:  LBB22_8: ## %else8
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtw %xmm0, %xmm2, %xmm3
-; AVX2-NEXT:    vpextrb $8, %xmm3, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je LBB22_10
-; AVX2-NEXT:  ## %bb.9: ## %cond.load10
-; AVX2-NEXT:    vpinsrw $4, 8(%rdi), %xmm1, %xmm3
-; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-NEXT:    testb $16, %al
+; AVX2-NEXT:    jne LBB22_9
 ; AVX2-NEXT:  LBB22_10: ## %else11
-; AVX2-NEXT:    vpcmpgtw %xmm0, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrb $10, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je LBB22_12
-; AVX2-NEXT:  ## %bb.11: ## %cond.load13
-; AVX2-NEXT:    vpinsrw $5, 10(%rdi), %xmm1, %xmm2
-; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-NEXT:    testb $32, %al
+; AVX2-NEXT:    jne LBB22_11
 ; AVX2-NEXT:  LBB22_12: ## %else14
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtw %xmm0, %xmm2, %xmm3
-; AVX2-NEXT:    vpextrb $12, %xmm3, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je LBB22_14
-; AVX2-NEXT:  ## %bb.13: ## %cond.load16
-; AVX2-NEXT:    vpinsrw $6, 12(%rdi), %xmm1, %xmm3
-; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-NEXT:    testb $64, %al
+; AVX2-NEXT:    jne LBB22_13
 ; AVX2-NEXT:  LBB22_14: ## %else17
-; AVX2-NEXT:    vpcmpgtw %xmm0, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrb $14, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je LBB22_16
-; AVX2-NEXT:  ## %bb.15: ## %cond.load19
-; AVX2-NEXT:    vpinsrw $7, 14(%rdi), %xmm1, %xmm2
-; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-NEXT:    testb $-128, %al
+; AVX2-NEXT:    jne LBB22_15
 ; AVX2-NEXT:  LBB22_16: ## %else20
-; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtw %xmm0, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrb $0, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je LBB22_18
-; AVX2-NEXT:  ## %bb.17: ## %cond.load22
-; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm3
-; AVX2-NEXT:    vpinsrw $0, 16(%rdi), %xmm3, %xmm3
-; AVX2-NEXT:    vinserti128 $1, %xmm3, %ymm1, %ymm1
+; AVX2-NEXT:    testl $256, %eax ## imm = 0x100
+; AVX2-NEXT:    jne LBB22_17
 ; AVX2-NEXT:  LBB22_18: ## %else23
-; AVX2-NEXT:    vpextrb $2, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je LBB22_20
-; AVX2-NEXT:  ## %bb.19: ## %cond.load25
-; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX2-NEXT:    vpinsrw $1, 18(%rdi), %xmm2, %xmm2
-; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX2-NEXT:    testl $512, %eax ## imm = 0x200
+; AVX2-NEXT:    jne LBB22_19
 ; AVX2-NEXT:  LBB22_20: ## %else26
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtw %xmm0, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je LBB22_22
-; AVX2-NEXT:  ## %bb.21: ## %cond.load28
-; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm3
-; AVX2-NEXT:    vpinsrw $2, 20(%rdi), %xmm3, %xmm3
-; AVX2-NEXT:    vinserti128 $1, %xmm3, %ymm1, %ymm1
+; AVX2-NEXT:    testl $1024, %eax ## imm = 0x400
+; AVX2-NEXT:    jne LBB22_21
 ; AVX2-NEXT:  LBB22_22: ## %else29
-; AVX2-NEXT:    vpextrb $6, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je LBB22_24
-; AVX2-NEXT:  ## %bb.23: ## %cond.load31
-; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX2-NEXT:    vpinsrw $3, 22(%rdi), %xmm2, %xmm2
-; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX2-NEXT:    testl $2048, %eax ## imm = 0x800
+; AVX2-NEXT:    jne LBB22_23
 ; AVX2-NEXT:  LBB22_24: ## %else32
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtw %xmm0, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrb $8, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je LBB22_26
-; AVX2-NEXT:  ## %bb.25: ## %cond.load34
-; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm3
-; AVX2-NEXT:    vpinsrw $4, 24(%rdi), %xmm3, %xmm3
-; AVX2-NEXT:    vinserti128 $1, %xmm3, %ymm1, %ymm1
+; AVX2-NEXT:    testl $4096, %eax ## imm = 0x1000
+; AVX2-NEXT:    jne LBB22_25
 ; AVX2-NEXT:  LBB22_26: ## %else35
-; AVX2-NEXT:    vpextrb $10, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je LBB22_28
-; AVX2-NEXT:  ## %bb.27: ## %cond.load37
-; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX2-NEXT:    vpinsrw $5, 26(%rdi), %xmm2, %xmm2
-; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX2-NEXT:    testl $8192, %eax ## imm = 0x2000
+; AVX2-NEXT:    jne LBB22_27
 ; AVX2-NEXT:  LBB22_28: ## %else38
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtw %xmm0, %xmm2, %xmm0
-; AVX2-NEXT:    vpextrb $12, %xmm0, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je LBB22_30
-; AVX2-NEXT:  ## %bb.29: ## %cond.load40
-; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX2-NEXT:    vpinsrw $6, 28(%rdi), %xmm2, %xmm2
-; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX2-NEXT:    testl $16384, %eax ## imm = 0x4000
+; AVX2-NEXT:    jne LBB22_29
 ; AVX2-NEXT:  LBB22_30: ## %else41
-; AVX2-NEXT:    vpextrb $14, %xmm0, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je LBB22_32
-; AVX2-NEXT:  ## %bb.31: ## %cond.load43
-; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm0
-; AVX2-NEXT:    vpinsrw $7, 30(%rdi), %xmm0, %xmm0
-; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
+; AVX2-NEXT:    testl $32768, %eax ## imm = 0x8000
+; AVX2-NEXT:    jne LBB22_31
 ; AVX2-NEXT:  LBB22_32: ## %else44
 ; AVX2-NEXT:    vmovdqa %ymm1, %ymm0
 ; AVX2-NEXT:    retq
-;
-; AVX512F-LABEL: load_v16i16_v16i16:
-; AVX512F:       ## %bb.0:
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpgtw %ymm0, %ymm2, %ymm2
-; AVX512F-NEXT:    vpmovsxwd %ymm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je LBB22_2
-; AVX512F-NEXT:  ## %bb.1: ## %cond.load
-; AVX512F-NEXT:    vpinsrw $0, (%rdi), %xmm1, %xmm2
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; AVX512F-NEXT:  LBB22_2: ## %else
-; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je LBB22_4
-; AVX512F-NEXT:  ## %bb.3: ## %cond.load1
-; AVX512F-NEXT:    vpinsrw $1, 2(%rdi), %xmm1, %xmm2
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; AVX512F-NEXT:  LBB22_4: ## %else2
+; AVX2-NEXT:  LBB22_1: ## %cond.load
+; AVX2-NEXT:    vpinsrw $0, (%rdi), %xmm1, %xmm0
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-NEXT:    testb $2, %al
+; AVX2-NEXT:    je LBB22_4
+; AVX2-NEXT:  LBB22_3: ## %cond.load1
+; AVX2-NEXT:    vpinsrw $1, 2(%rdi), %xmm1, %xmm0
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-NEXT:    testb $4, %al
+; AVX2-NEXT:    je LBB22_6
+; AVX2-NEXT:  LBB22_5: ## %cond.load4
+; AVX2-NEXT:    vpinsrw $2, 4(%rdi), %xmm1, %xmm0
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-NEXT:    testb $8, %al
+; AVX2-NEXT:    je LBB22_8
+; AVX2-NEXT:  LBB22_7: ## %cond.load7
+; AVX2-NEXT:    vpinsrw $3, 6(%rdi), %xmm1, %xmm0
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-NEXT:    testb $16, %al
+; AVX2-NEXT:    je LBB22_10
+; AVX2-NEXT:  LBB22_9: ## %cond.load10
+; AVX2-NEXT:    vpinsrw $4, 8(%rdi), %xmm1, %xmm0
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-NEXT:    testb $32, %al
+; AVX2-NEXT:    je LBB22_12
+; AVX2-NEXT:  LBB22_11: ## %cond.load13
+; AVX2-NEXT:    vpinsrw $5, 10(%rdi), %xmm1, %xmm0
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-NEXT:    testb $64, %al
+; AVX2-NEXT:    je LBB22_14
+; AVX2-NEXT:  LBB22_13: ## %cond.load16
+; AVX2-NEXT:    vpinsrw $6, 12(%rdi), %xmm1, %xmm0
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-NEXT:    testb $-128, %al
+; AVX2-NEXT:    je LBB22_16
+; AVX2-NEXT:  LBB22_15: ## %cond.load19
+; AVX2-NEXT:    vpinsrw $7, 14(%rdi), %xmm1, %xmm0
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-NEXT:    testl $256, %eax ## imm = 0x100
+; AVX2-NEXT:    je LBB22_18
+; AVX2-NEXT:  LBB22_17: ## %cond.load22
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX2-NEXT:    vpinsrw $0, 16(%rdi), %xmm0, %xmm0
+; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
+; AVX2-NEXT:    testl $512, %eax ## imm = 0x200
+; AVX2-NEXT:    je LBB22_20
+; AVX2-NEXT:  LBB22_19: ## %cond.load25
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX2-NEXT:    vpinsrw $1, 18(%rdi), %xmm0, %xmm0
+; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
+; AVX2-NEXT:    testl $1024, %eax ## imm = 0x400
+; AVX2-NEXT:    je LBB22_22
+; AVX2-NEXT:  LBB22_21: ## %cond.load28
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX2-NEXT:    vpinsrw $2, 20(%rdi), %xmm0, %xmm0
+; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
+; AVX2-NEXT:    testl $2048, %eax ## imm = 0x800
+; AVX2-NEXT:    je LBB22_24
+; AVX2-NEXT:  LBB22_23: ## %cond.load31
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX2-NEXT:    vpinsrw $3, 22(%rdi), %xmm0, %xmm0
+; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
+; AVX2-NEXT:    testl $4096, %eax ## imm = 0x1000
+; AVX2-NEXT:    je LBB22_26
+; AVX2-NEXT:  LBB22_25: ## %cond.load34
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX2-NEXT:    vpinsrw $4, 24(%rdi), %xmm0, %xmm0
+; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
+; AVX2-NEXT:    testl $8192, %eax ## imm = 0x2000
+; AVX2-NEXT:    je LBB22_28
+; AVX2-NEXT:  LBB22_27: ## %cond.load37
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX2-NEXT:    vpinsrw $5, 26(%rdi), %xmm0, %xmm0
+; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
+; AVX2-NEXT:    testl $16384, %eax ## imm = 0x4000
+; AVX2-NEXT:    je LBB22_30
+; AVX2-NEXT:  LBB22_29: ## %cond.load40
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX2-NEXT:    vpinsrw $6, 28(%rdi), %xmm0, %xmm0
+; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
+; AVX2-NEXT:    testl $32768, %eax ## imm = 0x8000
+; AVX2-NEXT:    je LBB22_32
+; AVX2-NEXT:  LBB22_31: ## %cond.load43
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX2-NEXT:    vpinsrw $7, 30(%rdi), %xmm0, %xmm0
+; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
+; AVX2-NEXT:    vmovdqa %ymm1, %ymm0
+; AVX2-NEXT:    retq
+;
+; AVX512F-LABEL: load_v16i16_v16i16:
+; AVX512F:       ## %bb.0:
 ; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpgtw %ymm0, %ymm2, %ymm2
-; AVX512F-NEXT:    vpmovsxwd %ymm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $2, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je LBB22_6
-; AVX512F-NEXT:  ## %bb.5: ## %cond.load4
-; AVX512F-NEXT:    vpinsrw $2, 4(%rdi), %xmm1, %xmm2
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; AVX512F-NEXT:  LBB22_6: ## %else5
-; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
+; AVX512F-NEXT:    vpcmpgtw %ymm0, %ymm2, %ymm0
+; AVX512F-NEXT:    vpmovsxwd %ymm0, %zmm0
+; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k0
 ; AVX512F-NEXT:    kmovw %k0, %eax
 ; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je LBB22_8
-; AVX512F-NEXT:  ## %bb.7: ## %cond.load7
-; AVX512F-NEXT:    vpinsrw $3, 6(%rdi), %xmm1, %xmm2
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    jne LBB22_1
+; AVX512F-NEXT:  ## %bb.2: ## %else
+; AVX512F-NEXT:    testb $2, %al
+; AVX512F-NEXT:    jne LBB22_3
+; AVX512F-NEXT:  LBB22_4: ## %else2
+; AVX512F-NEXT:    testb $4, %al
+; AVX512F-NEXT:    jne LBB22_5
+; AVX512F-NEXT:  LBB22_6: ## %else5
+; AVX512F-NEXT:    testb $8, %al
+; AVX512F-NEXT:    jne LBB22_7
 ; AVX512F-NEXT:  LBB22_8: ## %else8
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpgtw %ymm0, %ymm2, %ymm2
-; AVX512F-NEXT:    vpmovsxwd %ymm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $4, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je LBB22_10
-; AVX512F-NEXT:  ## %bb.9: ## %cond.load10
-; AVX512F-NEXT:    vpinsrw $4, 8(%rdi), %xmm1, %xmm2
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    testb $16, %al
+; AVX512F-NEXT:    jne LBB22_9
 ; AVX512F-NEXT:  LBB22_10: ## %else11
-; AVX512F-NEXT:    kshiftrw $5, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je LBB22_12
-; AVX512F-NEXT:  ## %bb.11: ## %cond.load13
-; AVX512F-NEXT:    vpinsrw $5, 10(%rdi), %xmm1, %xmm2
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    testb $32, %al
+; AVX512F-NEXT:    jne LBB22_11
 ; AVX512F-NEXT:  LBB22_12: ## %else14
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpgtw %ymm0, %ymm2, %ymm2
-; AVX512F-NEXT:    vpmovsxwd %ymm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $6, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je LBB22_14
-; AVX512F-NEXT:  ## %bb.13: ## %cond.load16
-; AVX512F-NEXT:    vpinsrw $6, 12(%rdi), %xmm1, %xmm2
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    testb $64, %al
+; AVX512F-NEXT:    jne LBB22_13
 ; AVX512F-NEXT:  LBB22_14: ## %else17
-; AVX512F-NEXT:    kshiftrw $7, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je LBB22_16
-; AVX512F-NEXT:  ## %bb.15: ## %cond.load19
-; AVX512F-NEXT:    vpinsrw $7, 14(%rdi), %xmm1, %xmm2
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    testb $-128, %al
+; AVX512F-NEXT:    jne LBB22_15
 ; AVX512F-NEXT:  LBB22_16: ## %else20
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpgtw %ymm0, %ymm2, %ymm2
-; AVX512F-NEXT:    vpmovsxwd %ymm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $8, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je LBB22_18
-; AVX512F-NEXT:  ## %bb.17: ## %cond.load22
-; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX512F-NEXT:    vpinsrw $0, 16(%rdi), %xmm2, %xmm2
-; AVX512F-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512F-NEXT:    testl $256, %eax ## imm = 0x100
+; AVX512F-NEXT:    jne LBB22_17
 ; AVX512F-NEXT:  LBB22_18: ## %else23
-; AVX512F-NEXT:    kshiftrw $9, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je LBB22_20
-; AVX512F-NEXT:  ## %bb.19: ## %cond.load25
-; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX512F-NEXT:    vpinsrw $1, 18(%rdi), %xmm2, %xmm2
-; AVX512F-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512F-NEXT:    testl $512, %eax ## imm = 0x200
+; AVX512F-NEXT:    jne LBB22_19
 ; AVX512F-NEXT:  LBB22_20: ## %else26
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpgtw %ymm0, %ymm2, %ymm2
-; AVX512F-NEXT:    vpmovsxwd %ymm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $10, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je LBB22_22
-; AVX512F-NEXT:  ## %bb.21: ## %cond.load28
-; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX512F-NEXT:    vpinsrw $2, 20(%rdi), %xmm2, %xmm2
-; AVX512F-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512F-NEXT:    testl $1024, %eax ## imm = 0x400
+; AVX512F-NEXT:    jne LBB22_21
 ; AVX512F-NEXT:  LBB22_22: ## %else29
-; AVX512F-NEXT:    kshiftrw $11, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je LBB22_24
-; AVX512F-NEXT:  ## %bb.23: ## %cond.load31
-; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX512F-NEXT:    vpinsrw $3, 22(%rdi), %xmm2, %xmm2
-; AVX512F-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512F-NEXT:    testl $2048, %eax ## imm = 0x800
+; AVX512F-NEXT:    jne LBB22_23
 ; AVX512F-NEXT:  LBB22_24: ## %else32
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpgtw %ymm0, %ymm2, %ymm2
-; AVX512F-NEXT:    vpmovsxwd %ymm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $12, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je LBB22_26
-; AVX512F-NEXT:  ## %bb.25: ## %cond.load34
-; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX512F-NEXT:    vpinsrw $4, 24(%rdi), %xmm2, %xmm2
-; AVX512F-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512F-NEXT:    testl $4096, %eax ## imm = 0x1000
+; AVX512F-NEXT:    jne LBB22_25
 ; AVX512F-NEXT:  LBB22_26: ## %else35
-; AVX512F-NEXT:    kshiftrw $13, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je LBB22_28
-; AVX512F-NEXT:  ## %bb.27: ## %cond.load37
-; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX512F-NEXT:    vpinsrw $5, 26(%rdi), %xmm2, %xmm2
-; AVX512F-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512F-NEXT:    testl $8192, %eax ## imm = 0x2000
+; AVX512F-NEXT:    jne LBB22_27
 ; AVX512F-NEXT:  LBB22_28: ## %else38
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpgtw %ymm0, %ymm2, %ymm0
-; AVX512F-NEXT:    vpmovsxwd %ymm0, %zmm0
-; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k0
-; AVX512F-NEXT:    kshiftrw $14, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $16384, %eax ## imm = 0x4000
+; AVX512F-NEXT:    jne LBB22_29
+; AVX512F-NEXT:  LBB22_30: ## %else41
+; AVX512F-NEXT:    testl $32768, %eax ## imm = 0x8000
+; AVX512F-NEXT:    jne LBB22_31
+; AVX512F-NEXT:  LBB22_32: ## %else44
+; AVX512F-NEXT:    vmovdqa %ymm1, %ymm0
+; AVX512F-NEXT:    retq
+; AVX512F-NEXT:  LBB22_1: ## %cond.load
+; AVX512F-NEXT:    vpinsrw $0, (%rdi), %xmm1, %xmm0
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    testb $2, %al
+; AVX512F-NEXT:    je LBB22_4
+; AVX512F-NEXT:  LBB22_3: ## %cond.load1
+; AVX512F-NEXT:    vpinsrw $1, 2(%rdi), %xmm1, %xmm0
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    testb $4, %al
+; AVX512F-NEXT:    je LBB22_6
+; AVX512F-NEXT:  LBB22_5: ## %cond.load4
+; AVX512F-NEXT:    vpinsrw $2, 4(%rdi), %xmm1, %xmm0
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    testb $8, %al
+; AVX512F-NEXT:    je LBB22_8
+; AVX512F-NEXT:  LBB22_7: ## %cond.load7
+; AVX512F-NEXT:    vpinsrw $3, 6(%rdi), %xmm1, %xmm0
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    testb $16, %al
+; AVX512F-NEXT:    je LBB22_10
+; AVX512F-NEXT:  LBB22_9: ## %cond.load10
+; AVX512F-NEXT:    vpinsrw $4, 8(%rdi), %xmm1, %xmm0
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    testb $32, %al
+; AVX512F-NEXT:    je LBB22_12
+; AVX512F-NEXT:  LBB22_11: ## %cond.load13
+; AVX512F-NEXT:    vpinsrw $5, 10(%rdi), %xmm1, %xmm0
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    testb $64, %al
+; AVX512F-NEXT:    je LBB22_14
+; AVX512F-NEXT:  LBB22_13: ## %cond.load16
+; AVX512F-NEXT:    vpinsrw $6, 12(%rdi), %xmm1, %xmm0
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    testb $-128, %al
+; AVX512F-NEXT:    je LBB22_16
+; AVX512F-NEXT:  LBB22_15: ## %cond.load19
+; AVX512F-NEXT:    vpinsrw $7, 14(%rdi), %xmm1, %xmm0
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    testl $256, %eax ## imm = 0x100
+; AVX512F-NEXT:    je LBB22_18
+; AVX512F-NEXT:  LBB22_17: ## %cond.load22
+; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX512F-NEXT:    vpinsrw $0, 16(%rdi), %xmm0, %xmm0
+; AVX512F-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
+; AVX512F-NEXT:    testl $512, %eax ## imm = 0x200
+; AVX512F-NEXT:    je LBB22_20
+; AVX512F-NEXT:  LBB22_19: ## %cond.load25
+; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX512F-NEXT:    vpinsrw $1, 18(%rdi), %xmm0, %xmm0
+; AVX512F-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
+; AVX512F-NEXT:    testl $1024, %eax ## imm = 0x400
+; AVX512F-NEXT:    je LBB22_22
+; AVX512F-NEXT:  LBB22_21: ## %cond.load28
+; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX512F-NEXT:    vpinsrw $2, 20(%rdi), %xmm0, %xmm0
+; AVX512F-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
+; AVX512F-NEXT:    testl $2048, %eax ## imm = 0x800
+; AVX512F-NEXT:    je LBB22_24
+; AVX512F-NEXT:  LBB22_23: ## %cond.load31
+; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX512F-NEXT:    vpinsrw $3, 22(%rdi), %xmm0, %xmm0
+; AVX512F-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
+; AVX512F-NEXT:    testl $4096, %eax ## imm = 0x1000
+; AVX512F-NEXT:    je LBB22_26
+; AVX512F-NEXT:  LBB22_25: ## %cond.load34
+; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX512F-NEXT:    vpinsrw $4, 24(%rdi), %xmm0, %xmm0
+; AVX512F-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
+; AVX512F-NEXT:    testl $8192, %eax ## imm = 0x2000
+; AVX512F-NEXT:    je LBB22_28
+; AVX512F-NEXT:  LBB22_27: ## %cond.load37
+; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX512F-NEXT:    vpinsrw $5, 26(%rdi), %xmm0, %xmm0
+; AVX512F-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
+; AVX512F-NEXT:    testl $16384, %eax ## imm = 0x4000
 ; AVX512F-NEXT:    je LBB22_30
-; AVX512F-NEXT:  ## %bb.29: ## %cond.load40
+; AVX512F-NEXT:  LBB22_29: ## %cond.load40
 ; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm0
 ; AVX512F-NEXT:    vpinsrw $6, 28(%rdi), %xmm0, %xmm0
 ; AVX512F-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
-; AVX512F-NEXT:  LBB22_30: ## %else41
-; AVX512F-NEXT:    kshiftrw $15, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $32768, %eax ## imm = 0x8000
 ; AVX512F-NEXT:    je LBB22_32
-; AVX512F-NEXT:  ## %bb.31: ## %cond.load43
+; AVX512F-NEXT:  LBB22_31: ## %cond.load43
 ; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm0
 ; AVX512F-NEXT:    vpinsrw $7, 30(%rdi), %xmm0, %xmm0
 ; AVX512F-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
-; AVX512F-NEXT:  LBB22_32: ## %else44
 ; AVX512F-NEXT:    vmovdqa %ymm1, %ymm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VLDQ-LABEL: load_v16i16_v16i16:
 ; AVX512VLDQ:       ## %bb.0:
 ; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpgtw %ymm0, %ymm2, %ymm2
-; AVX512VLDQ-NEXT:    vpmovsxwd %ymm2, %zmm2
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm2, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
-; AVX512VLDQ-NEXT:    je LBB22_2
-; AVX512VLDQ-NEXT:  ## %bb.1: ## %cond.load
-; AVX512VLDQ-NEXT:    vpinsrw $0, (%rdi), %xmm1, %xmm2
-; AVX512VLDQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; AVX512VLDQ-NEXT:  LBB22_2: ## %else
-; AVX512VLDQ-NEXT:    kshiftrw $1, %k0, %k0
+; AVX512VLDQ-NEXT:    vpcmpgtw %ymm0, %ymm2, %ymm0
+; AVX512VLDQ-NEXT:    vpmovsxwd %ymm0, %zmm0
+; AVX512VLDQ-NEXT:    vpmovd2m %zmm0, %k0
 ; AVX512VLDQ-NEXT:    kmovw %k0, %eax
 ; AVX512VLDQ-NEXT:    testb $1, %al
-; AVX512VLDQ-NEXT:    je LBB22_4
-; AVX512VLDQ-NEXT:  ## %bb.3: ## %cond.load1
-; AVX512VLDQ-NEXT:    vpinsrw $1, 2(%rdi), %xmm1, %xmm2
-; AVX512VLDQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512VLDQ-NEXT:    jne LBB22_1
+; AVX512VLDQ-NEXT:  ## %bb.2: ## %else
+; AVX512VLDQ-NEXT:    testb $2, %al
+; AVX512VLDQ-NEXT:    jne LBB22_3
 ; AVX512VLDQ-NEXT:  LBB22_4: ## %else2
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpgtw %ymm0, %ymm2, %ymm2
-; AVX512VLDQ-NEXT:    vpmovsxwd %ymm2, %zmm2
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm2, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $2, %k0, %k1
-; AVX512VLDQ-NEXT:    kmovw %k1, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
-; AVX512VLDQ-NEXT:    je LBB22_6
-; AVX512VLDQ-NEXT:  ## %bb.5: ## %cond.load4
-; AVX512VLDQ-NEXT:    vpinsrw $2, 4(%rdi), %xmm1, %xmm2
-; AVX512VLDQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512VLDQ-NEXT:    testb $4, %al
+; AVX512VLDQ-NEXT:    jne LBB22_5
 ; AVX512VLDQ-NEXT:  LBB22_6: ## %else5
-; AVX512VLDQ-NEXT:    kshiftrw $3, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
-; AVX512VLDQ-NEXT:    je LBB22_8
-; AVX512VLDQ-NEXT:  ## %bb.7: ## %cond.load7
-; AVX512VLDQ-NEXT:    vpinsrw $3, 6(%rdi), %xmm1, %xmm2
-; AVX512VLDQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512VLDQ-NEXT:    testb $8, %al
+; AVX512VLDQ-NEXT:    jne LBB22_7
 ; AVX512VLDQ-NEXT:  LBB22_8: ## %else8
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpgtw %ymm0, %ymm2, %ymm2
-; AVX512VLDQ-NEXT:    vpmovsxwd %ymm2, %zmm2
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm2, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $4, %k0, %k1
-; AVX512VLDQ-NEXT:    kmovw %k1, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
-; AVX512VLDQ-NEXT:    je LBB22_10
-; AVX512VLDQ-NEXT:  ## %bb.9: ## %cond.load10
-; AVX512VLDQ-NEXT:    vpinsrw $4, 8(%rdi), %xmm1, %xmm2
-; AVX512VLDQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512VLDQ-NEXT:    testb $16, %al
+; AVX512VLDQ-NEXT:    jne LBB22_9
 ; AVX512VLDQ-NEXT:  LBB22_10: ## %else11
-; AVX512VLDQ-NEXT:    kshiftrw $5, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
-; AVX512VLDQ-NEXT:    je LBB22_12
-; AVX512VLDQ-NEXT:  ## %bb.11: ## %cond.load13
-; AVX512VLDQ-NEXT:    vpinsrw $5, 10(%rdi), %xmm1, %xmm2
-; AVX512VLDQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512VLDQ-NEXT:    testb $32, %al
+; AVX512VLDQ-NEXT:    jne LBB22_11
 ; AVX512VLDQ-NEXT:  LBB22_12: ## %else14
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpgtw %ymm0, %ymm2, %ymm2
-; AVX512VLDQ-NEXT:    vpmovsxwd %ymm2, %zmm2
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm2, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $6, %k0, %k1
-; AVX512VLDQ-NEXT:    kmovw %k1, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
-; AVX512VLDQ-NEXT:    je LBB22_14
-; AVX512VLDQ-NEXT:  ## %bb.13: ## %cond.load16
-; AVX512VLDQ-NEXT:    vpinsrw $6, 12(%rdi), %xmm1, %xmm2
-; AVX512VLDQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512VLDQ-NEXT:    testb $64, %al
+; AVX512VLDQ-NEXT:    jne LBB22_13
 ; AVX512VLDQ-NEXT:  LBB22_14: ## %else17
-; AVX512VLDQ-NEXT:    kshiftrw $7, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
-; AVX512VLDQ-NEXT:    je LBB22_16
-; AVX512VLDQ-NEXT:  ## %bb.15: ## %cond.load19
-; AVX512VLDQ-NEXT:    vpinsrw $7, 14(%rdi), %xmm1, %xmm2
-; AVX512VLDQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512VLDQ-NEXT:    testb $-128, %al
+; AVX512VLDQ-NEXT:    jne LBB22_15
 ; AVX512VLDQ-NEXT:  LBB22_16: ## %else20
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpgtw %ymm0, %ymm2, %ymm2
-; AVX512VLDQ-NEXT:    vpmovsxwd %ymm2, %zmm2
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm2, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $8, %k0, %k1
-; AVX512VLDQ-NEXT:    kmovw %k1, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
-; AVX512VLDQ-NEXT:    je LBB22_18
-; AVX512VLDQ-NEXT:  ## %bb.17: ## %cond.load22
-; AVX512VLDQ-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX512VLDQ-NEXT:    vpinsrw $0, 16(%rdi), %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512VLDQ-NEXT:    testl $256, %eax ## imm = 0x100
+; AVX512VLDQ-NEXT:    jne LBB22_17
 ; AVX512VLDQ-NEXT:  LBB22_18: ## %else23
-; AVX512VLDQ-NEXT:    kshiftrw $9, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
-; AVX512VLDQ-NEXT:    je LBB22_20
-; AVX512VLDQ-NEXT:  ## %bb.19: ## %cond.load25
-; AVX512VLDQ-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX512VLDQ-NEXT:    vpinsrw $1, 18(%rdi), %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512VLDQ-NEXT:    testl $512, %eax ## imm = 0x200
+; AVX512VLDQ-NEXT:    jne LBB22_19
 ; AVX512VLDQ-NEXT:  LBB22_20: ## %else26
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpgtw %ymm0, %ymm2, %ymm2
-; AVX512VLDQ-NEXT:    vpmovsxwd %ymm2, %zmm2
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm2, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $10, %k0, %k1
-; AVX512VLDQ-NEXT:    kmovw %k1, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
-; AVX512VLDQ-NEXT:    je LBB22_22
-; AVX512VLDQ-NEXT:  ## %bb.21: ## %cond.load28
-; AVX512VLDQ-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX512VLDQ-NEXT:    vpinsrw $2, 20(%rdi), %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512VLDQ-NEXT:    testl $1024, %eax ## imm = 0x400
+; AVX512VLDQ-NEXT:    jne LBB22_21
 ; AVX512VLDQ-NEXT:  LBB22_22: ## %else29
-; AVX512VLDQ-NEXT:    kshiftrw $11, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
-; AVX512VLDQ-NEXT:    je LBB22_24
-; AVX512VLDQ-NEXT:  ## %bb.23: ## %cond.load31
-; AVX512VLDQ-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX512VLDQ-NEXT:    vpinsrw $3, 22(%rdi), %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512VLDQ-NEXT:    testl $2048, %eax ## imm = 0x800
+; AVX512VLDQ-NEXT:    jne LBB22_23
 ; AVX512VLDQ-NEXT:  LBB22_24: ## %else32
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpgtw %ymm0, %ymm2, %ymm2
-; AVX512VLDQ-NEXT:    vpmovsxwd %ymm2, %zmm2
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm2, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $12, %k0, %k1
-; AVX512VLDQ-NEXT:    kmovw %k1, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
-; AVX512VLDQ-NEXT:    je LBB22_26
-; AVX512VLDQ-NEXT:  ## %bb.25: ## %cond.load34
-; AVX512VLDQ-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX512VLDQ-NEXT:    vpinsrw $4, 24(%rdi), %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512VLDQ-NEXT:    testl $4096, %eax ## imm = 0x1000
+; AVX512VLDQ-NEXT:    jne LBB22_25
 ; AVX512VLDQ-NEXT:  LBB22_26: ## %else35
-; AVX512VLDQ-NEXT:    kshiftrw $13, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
-; AVX512VLDQ-NEXT:    je LBB22_28
-; AVX512VLDQ-NEXT:  ## %bb.27: ## %cond.load37
-; AVX512VLDQ-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX512VLDQ-NEXT:    vpinsrw $5, 26(%rdi), %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512VLDQ-NEXT:    testl $8192, %eax ## imm = 0x2000
+; AVX512VLDQ-NEXT:    jne LBB22_27
 ; AVX512VLDQ-NEXT:  LBB22_28: ## %else38
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpgtw %ymm0, %ymm2, %ymm0
-; AVX512VLDQ-NEXT:    vpmovsxwd %ymm0, %zmm0
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm0, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $14, %k0, %k1
-; AVX512VLDQ-NEXT:    kmovw %k1, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testl $16384, %eax ## imm = 0x4000
+; AVX512VLDQ-NEXT:    jne LBB22_29
+; AVX512VLDQ-NEXT:  LBB22_30: ## %else41
+; AVX512VLDQ-NEXT:    testl $32768, %eax ## imm = 0x8000
+; AVX512VLDQ-NEXT:    jne LBB22_31
+; AVX512VLDQ-NEXT:  LBB22_32: ## %else44
+; AVX512VLDQ-NEXT:    vmovdqa %ymm1, %ymm0
+; AVX512VLDQ-NEXT:    retq
+; AVX512VLDQ-NEXT:  LBB22_1: ## %cond.load
+; AVX512VLDQ-NEXT:    vpinsrw $0, (%rdi), %xmm1, %xmm0
+; AVX512VLDQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512VLDQ-NEXT:    testb $2, %al
+; AVX512VLDQ-NEXT:    je LBB22_4
+; AVX512VLDQ-NEXT:  LBB22_3: ## %cond.load1
+; AVX512VLDQ-NEXT:    vpinsrw $1, 2(%rdi), %xmm1, %xmm0
+; AVX512VLDQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512VLDQ-NEXT:    testb $4, %al
+; AVX512VLDQ-NEXT:    je LBB22_6
+; AVX512VLDQ-NEXT:  LBB22_5: ## %cond.load4
+; AVX512VLDQ-NEXT:    vpinsrw $2, 4(%rdi), %xmm1, %xmm0
+; AVX512VLDQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512VLDQ-NEXT:    testb $8, %al
+; AVX512VLDQ-NEXT:    je LBB22_8
+; AVX512VLDQ-NEXT:  LBB22_7: ## %cond.load7
+; AVX512VLDQ-NEXT:    vpinsrw $3, 6(%rdi), %xmm1, %xmm0
+; AVX512VLDQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512VLDQ-NEXT:    testb $16, %al
+; AVX512VLDQ-NEXT:    je LBB22_10
+; AVX512VLDQ-NEXT:  LBB22_9: ## %cond.load10
+; AVX512VLDQ-NEXT:    vpinsrw $4, 8(%rdi), %xmm1, %xmm0
+; AVX512VLDQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512VLDQ-NEXT:    testb $32, %al
+; AVX512VLDQ-NEXT:    je LBB22_12
+; AVX512VLDQ-NEXT:  LBB22_11: ## %cond.load13
+; AVX512VLDQ-NEXT:    vpinsrw $5, 10(%rdi), %xmm1, %xmm0
+; AVX512VLDQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512VLDQ-NEXT:    testb $64, %al
+; AVX512VLDQ-NEXT:    je LBB22_14
+; AVX512VLDQ-NEXT:  LBB22_13: ## %cond.load16
+; AVX512VLDQ-NEXT:    vpinsrw $6, 12(%rdi), %xmm1, %xmm0
+; AVX512VLDQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512VLDQ-NEXT:    testb $-128, %al
+; AVX512VLDQ-NEXT:    je LBB22_16
+; AVX512VLDQ-NEXT:  LBB22_15: ## %cond.load19
+; AVX512VLDQ-NEXT:    vpinsrw $7, 14(%rdi), %xmm1, %xmm0
+; AVX512VLDQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512VLDQ-NEXT:    testl $256, %eax ## imm = 0x100
+; AVX512VLDQ-NEXT:    je LBB22_18
+; AVX512VLDQ-NEXT:  LBB22_17: ## %cond.load22
+; AVX512VLDQ-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX512VLDQ-NEXT:    vpinsrw $0, 16(%rdi), %xmm0, %xmm0
+; AVX512VLDQ-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
+; AVX512VLDQ-NEXT:    testl $512, %eax ## imm = 0x200
+; AVX512VLDQ-NEXT:    je LBB22_20
+; AVX512VLDQ-NEXT:  LBB22_19: ## %cond.load25
+; AVX512VLDQ-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX512VLDQ-NEXT:    vpinsrw $1, 18(%rdi), %xmm0, %xmm0
+; AVX512VLDQ-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
+; AVX512VLDQ-NEXT:    testl $1024, %eax ## imm = 0x400
+; AVX512VLDQ-NEXT:    je LBB22_22
+; AVX512VLDQ-NEXT:  LBB22_21: ## %cond.load28
+; AVX512VLDQ-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX512VLDQ-NEXT:    vpinsrw $2, 20(%rdi), %xmm0, %xmm0
+; AVX512VLDQ-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
+; AVX512VLDQ-NEXT:    testl $2048, %eax ## imm = 0x800
+; AVX512VLDQ-NEXT:    je LBB22_24
+; AVX512VLDQ-NEXT:  LBB22_23: ## %cond.load31
+; AVX512VLDQ-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX512VLDQ-NEXT:    vpinsrw $3, 22(%rdi), %xmm0, %xmm0
+; AVX512VLDQ-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
+; AVX512VLDQ-NEXT:    testl $4096, %eax ## imm = 0x1000
+; AVX512VLDQ-NEXT:    je LBB22_26
+; AVX512VLDQ-NEXT:  LBB22_25: ## %cond.load34
+; AVX512VLDQ-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX512VLDQ-NEXT:    vpinsrw $4, 24(%rdi), %xmm0, %xmm0
+; AVX512VLDQ-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
+; AVX512VLDQ-NEXT:    testl $8192, %eax ## imm = 0x2000
+; AVX512VLDQ-NEXT:    je LBB22_28
+; AVX512VLDQ-NEXT:  LBB22_27: ## %cond.load37
+; AVX512VLDQ-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX512VLDQ-NEXT:    vpinsrw $5, 26(%rdi), %xmm0, %xmm0
+; AVX512VLDQ-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
+; AVX512VLDQ-NEXT:    testl $16384, %eax ## imm = 0x4000
 ; AVX512VLDQ-NEXT:    je LBB22_30
-; AVX512VLDQ-NEXT:  ## %bb.29: ## %cond.load40
+; AVX512VLDQ-NEXT:  LBB22_29: ## %cond.load40
 ; AVX512VLDQ-NEXT:    vextracti128 $1, %ymm1, %xmm0
 ; AVX512VLDQ-NEXT:    vpinsrw $6, 28(%rdi), %xmm0, %xmm0
 ; AVX512VLDQ-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
-; AVX512VLDQ-NEXT:  LBB22_30: ## %else41
-; AVX512VLDQ-NEXT:    kshiftrw $15, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testl $32768, %eax ## imm = 0x8000
 ; AVX512VLDQ-NEXT:    je LBB22_32
-; AVX512VLDQ-NEXT:  ## %bb.31: ## %cond.load43
+; AVX512VLDQ-NEXT:  LBB22_31: ## %cond.load43
 ; AVX512VLDQ-NEXT:    vextracti128 $1, %ymm1, %xmm0
 ; AVX512VLDQ-NEXT:    vpinsrw $7, 30(%rdi), %xmm0, %xmm0
 ; AVX512VLDQ-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
-; AVX512VLDQ-NEXT:  LBB22_32: ## %else44
 ; AVX512VLDQ-NEXT:    vmovdqa %ymm1, %ymm0
 ; AVX512VLDQ-NEXT:    retq
 ;
@@ -3884,746 +3651,685 @@ define <16 x i16> @load_v16i16_v16i16(<16 x i16> %trigger, <16 x i16>* %addr, <1
 define <16 x i8> @load_v16i8_v16i8(<16 x i8> %trigger, <16 x i8>* %addr, <16 x i8> %dst) {
 ; SSE2-LABEL: load_v16i8_v16i8:
 ; SSE2:       ## %bb.0:
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpgtb %xmm0, %xmm2
-; SSE2-NEXT:    movd %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB23_2
-; SSE2-NEXT:  ## %bb.1: ## %cond.load
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    movzbl (%rdi), %ecx
-; SSE2-NEXT:    movd %ecx, %xmm3
-; SSE2-NEXT:    pandn %xmm3, %xmm2
-; SSE2-NEXT:    por %xmm2, %xmm1
-; SSE2-NEXT:  LBB23_2: ## %else
-; SSE2-NEXT:    shrl $8, %eax
+; SSE2-NEXT:    pmovmskb %xmm0, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB23_4
-; SSE2-NEXT:  ## %bb.3: ## %cond.load1
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    movzbl 1(%rdi), %eax
-; SSE2-NEXT:    movd %eax, %xmm3
-; SSE2-NEXT:    psllw $8, %xmm3
-; SSE2-NEXT:    pandn %xmm3, %xmm2
-; SSE2-NEXT:    por %xmm2, %xmm1
+; SSE2-NEXT:    jne LBB23_1
+; SSE2-NEXT:  ## %bb.2: ## %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne LBB23_3
 ; SSE2-NEXT:  LBB23_4: ## %else2
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpgtb %xmm0, %xmm2
-; SSE2-NEXT:    movd %xmm2, %eax
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    shrl $16, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    je LBB23_6
-; SSE2-NEXT:  ## %bb.5: ## %cond.load4
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    movzbl 2(%rdi), %ecx
-; SSE2-NEXT:    movd %ecx, %xmm3
-; SSE2-NEXT:    pslld $16, %xmm3
-; SSE2-NEXT:    pandn %xmm3, %xmm2
-; SSE2-NEXT:    por %xmm2, %xmm1
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne LBB23_5
 ; SSE2-NEXT:  LBB23_6: ## %else5
-; SSE2-NEXT:    shrl $24, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB23_8
-; SSE2-NEXT:  ## %bb.7: ## %cond.load7
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    movzbl 3(%rdi), %eax
-; SSE2-NEXT:    movd %eax, %xmm3
-; SSE2-NEXT:    pslld $24, %xmm3
-; SSE2-NEXT:    pandn %xmm3, %xmm2
-; SSE2-NEXT:    por %xmm2, %xmm1
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne LBB23_7
 ; SSE2-NEXT:  LBB23_8: ## %else8
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpgtb %xmm0, %xmm2
-; SSE2-NEXT:    pextrw $2, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB23_10
-; SSE2-NEXT:  ## %bb.9: ## %cond.load10
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    movzbl 4(%rdi), %ecx
-; SSE2-NEXT:    movd %ecx, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,0,1,1]
-; SSE2-NEXT:    pandn %xmm3, %xmm2
-; SSE2-NEXT:    por %xmm2, %xmm1
+; SSE2-NEXT:    testb $16, %al
+; SSE2-NEXT:    jne LBB23_9
 ; SSE2-NEXT:  LBB23_10: ## %else11
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB23_12
-; SSE2-NEXT:  ## %bb.11: ## %cond.load13
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    movzbl 5(%rdi), %eax
-; SSE2-NEXT:    movd %eax, %xmm3
-; SSE2-NEXT:    psllq $40, %xmm3
-; SSE2-NEXT:    pandn %xmm3, %xmm2
-; SSE2-NEXT:    por %xmm2, %xmm1
+; SSE2-NEXT:    testb $32, %al
+; SSE2-NEXT:    jne LBB23_11
 ; SSE2-NEXT:  LBB23_12: ## %else14
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpgtb %xmm0, %xmm2
-; SSE2-NEXT:    pextrw $3, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB23_14
-; SSE2-NEXT:  ## %bb.13: ## %cond.load16
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    movzbl 6(%rdi), %ecx
-; SSE2-NEXT:    movd %ecx, %xmm3
-; SSE2-NEXT:    psllq $48, %xmm3
-; SSE2-NEXT:    pandn %xmm3, %xmm2
-; SSE2-NEXT:    por %xmm2, %xmm1
+; SSE2-NEXT:    testb $64, %al
+; SSE2-NEXT:    jne LBB23_13
 ; SSE2-NEXT:  LBB23_14: ## %else17
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB23_16
-; SSE2-NEXT:  ## %bb.15: ## %cond.load19
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    movzbl 7(%rdi), %eax
-; SSE2-NEXT:    movd %eax, %xmm3
-; SSE2-NEXT:    psllq $56, %xmm3
-; SSE2-NEXT:    pandn %xmm3, %xmm2
-; SSE2-NEXT:    por %xmm2, %xmm1
+; SSE2-NEXT:    testb $-128, %al
+; SSE2-NEXT:    jne LBB23_15
 ; SSE2-NEXT:  LBB23_16: ## %else20
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpgtb %xmm0, %xmm2
-; SSE2-NEXT:    pextrw $4, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB23_18
-; SSE2-NEXT:  ## %bb.17: ## %cond.load22
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    movzbl 8(%rdi), %ecx
-; SSE2-NEXT:    movd %ecx, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,1,0,1]
-; SSE2-NEXT:    pandn %xmm3, %xmm2
-; SSE2-NEXT:    por %xmm2, %xmm1
+; SSE2-NEXT:    testl $256, %eax ## imm = 0x100
+; SSE2-NEXT:    jne LBB23_17
 ; SSE2-NEXT:  LBB23_18: ## %else23
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB23_20
-; SSE2-NEXT:  ## %bb.19: ## %cond.load25
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    movzbl 9(%rdi), %eax
-; SSE2-NEXT:    movd %eax, %xmm3
-; SSE2-NEXT:    pslldq {{.*#+}} xmm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm3[0,1,2,3,4,5,6]
-; SSE2-NEXT:    pandn %xmm3, %xmm2
-; SSE2-NEXT:    por %xmm2, %xmm1
+; SSE2-NEXT:    testl $512, %eax ## imm = 0x200
+; SSE2-NEXT:    jne LBB23_19
 ; SSE2-NEXT:  LBB23_20: ## %else26
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpgtb %xmm0, %xmm2
-; SSE2-NEXT:    pextrw $5, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB23_22
-; SSE2-NEXT:  ## %bb.21: ## %cond.load28
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    movzbl 10(%rdi), %ecx
-; SSE2-NEXT:    movd %ecx, %xmm3
-; SSE2-NEXT:    pslldq {{.*#+}} xmm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm3[0,1,2,3,4,5]
-; SSE2-NEXT:    pandn %xmm3, %xmm2
-; SSE2-NEXT:    por %xmm2, %xmm1
+; SSE2-NEXT:    testl $1024, %eax ## imm = 0x400
+; SSE2-NEXT:    jne LBB23_21
 ; SSE2-NEXT:  LBB23_22: ## %else29
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB23_24
-; SSE2-NEXT:  ## %bb.23: ## %cond.load31
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    movzbl 11(%rdi), %eax
-; SSE2-NEXT:    movd %eax, %xmm3
-; SSE2-NEXT:    pslldq {{.*#+}} xmm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm3[0,1,2,3,4]
-; SSE2-NEXT:    pandn %xmm3, %xmm2
-; SSE2-NEXT:    por %xmm2, %xmm1
+; SSE2-NEXT:    testl $2048, %eax ## imm = 0x800
+; SSE2-NEXT:    jne LBB23_23
 ; SSE2-NEXT:  LBB23_24: ## %else32
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpgtb %xmm0, %xmm2
-; SSE2-NEXT:    pextrw $6, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB23_26
-; SSE2-NEXT:  ## %bb.25: ## %cond.load34
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    movzbl 12(%rdi), %ecx
-; SSE2-NEXT:    movd %ecx, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,1,2,0]
-; SSE2-NEXT:    pandn %xmm3, %xmm2
-; SSE2-NEXT:    por %xmm2, %xmm1
+; SSE2-NEXT:    testl $4096, %eax ## imm = 0x1000
+; SSE2-NEXT:    jne LBB23_25
 ; SSE2-NEXT:  LBB23_26: ## %else35
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB23_28
-; SSE2-NEXT:  ## %bb.27: ## %cond.load37
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    movzbl 13(%rdi), %eax
-; SSE2-NEXT:    movd %eax, %xmm3
-; SSE2-NEXT:    pslldq {{.*#+}} xmm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm3[0,1,2]
-; SSE2-NEXT:    pandn %xmm3, %xmm2
-; SSE2-NEXT:    por %xmm2, %xmm1
+; SSE2-NEXT:    testl $8192, %eax ## imm = 0x2000
+; SSE2-NEXT:    jne LBB23_27
 ; SSE2-NEXT:  LBB23_28: ## %else38
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpgtb %xmm0, %xmm2
-; SSE2-NEXT:    pextrw $7, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB23_30
-; SSE2-NEXT:  ## %bb.29: ## %cond.load40
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255]
-; SSE2-NEXT:    pand %xmm0, %xmm1
-; SSE2-NEXT:    movzbl 14(%rdi), %ecx
-; SSE2-NEXT:    movd %ecx, %xmm2
-; SSE2-NEXT:    pslldq {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm2[0,1]
-; SSE2-NEXT:    pandn %xmm2, %xmm0
-; SSE2-NEXT:    por %xmm0, %xmm1
+; SSE2-NEXT:    testl $16384, %eax ## imm = 0x4000
+; SSE2-NEXT:    jne LBB23_29
 ; SSE2-NEXT:  LBB23_30: ## %else41
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB23_32
-; SSE2-NEXT:  ## %bb.31: ## %cond.load43
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
-; SSE2-NEXT:    movzbl 15(%rdi), %eax
-; SSE2-NEXT:    movd %eax, %xmm0
-; SSE2-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0]
-; SSE2-NEXT:    por %xmm0, %xmm1
+; SSE2-NEXT:    testl $32768, %eax ## imm = 0x8000
+; SSE2-NEXT:    jne LBB23_31
 ; SSE2-NEXT:  LBB23_32: ## %else44
 ; SSE2-NEXT:    movdqa %xmm1, %xmm0
 ; SSE2-NEXT:    retq
-;
-; SSE42-LABEL: load_v16i8_v16i8:
-; SSE42:       ## %bb.0:
-; SSE42-NEXT:    pxor %xmm2, %xmm2
-; SSE42-NEXT:    pcmpgtb %xmm0, %xmm2
-; SSE42-NEXT:    pextrb $0, %xmm2, %eax
+; SSE2-NEXT:  LBB23_1: ## %cond.load
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; SSE2-NEXT:    pand %xmm0, %xmm1
+; SSE2-NEXT:    movzbl (%rdi), %ecx
+; SSE2-NEXT:    movd %ecx, %xmm2
+; SSE2-NEXT:    pandn %xmm2, %xmm0
+; SSE2-NEXT:    por %xmm0, %xmm1
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    je LBB23_4
+; SSE2-NEXT:  LBB23_3: ## %cond.load1
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; SSE2-NEXT:    pand %xmm0, %xmm1
+; SSE2-NEXT:    movzbl 1(%rdi), %ecx
+; SSE2-NEXT:    movd %ecx, %xmm2
+; SSE2-NEXT:    psllw $8, %xmm2
+; SSE2-NEXT:    pandn %xmm2, %xmm0
+; SSE2-NEXT:    por %xmm0, %xmm1
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    je LBB23_6
+; SSE2-NEXT:  LBB23_5: ## %cond.load4
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; SSE2-NEXT:    pand %xmm0, %xmm1
+; SSE2-NEXT:    movzbl 2(%rdi), %ecx
+; SSE2-NEXT:    movd %ecx, %xmm2
+; SSE2-NEXT:    pslld $16, %xmm2
+; SSE2-NEXT:    pandn %xmm2, %xmm0
+; SSE2-NEXT:    por %xmm0, %xmm1
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    je LBB23_8
+; SSE2-NEXT:  LBB23_7: ## %cond.load7
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255]
+; SSE2-NEXT:    pand %xmm0, %xmm1
+; SSE2-NEXT:    movzbl 3(%rdi), %ecx
+; SSE2-NEXT:    movd %ecx, %xmm2
+; SSE2-NEXT:    pslld $24, %xmm2
+; SSE2-NEXT:    pandn %xmm2, %xmm0
+; SSE2-NEXT:    por %xmm0, %xmm1
+; SSE2-NEXT:    testb $16, %al
+; SSE2-NEXT:    je LBB23_10
+; SSE2-NEXT:  LBB23_9: ## %cond.load10
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255]
+; SSE2-NEXT:    pand %xmm0, %xmm1
+; SSE2-NEXT:    movzbl 4(%rdi), %ecx
+; SSE2-NEXT:    movd %ecx, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,0,1,1]
+; SSE2-NEXT:    pandn %xmm2, %xmm0
+; SSE2-NEXT:    por %xmm0, %xmm1
+; SSE2-NEXT:    testb $32, %al
+; SSE2-NEXT:    je LBB23_12
+; SSE2-NEXT:  LBB23_11: ## %cond.load13
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255]
+; SSE2-NEXT:    pand %xmm0, %xmm1
+; SSE2-NEXT:    movzbl 5(%rdi), %ecx
+; SSE2-NEXT:    movd %ecx, %xmm2
+; SSE2-NEXT:    psllq $40, %xmm2
+; SSE2-NEXT:    pandn %xmm2, %xmm0
+; SSE2-NEXT:    por %xmm0, %xmm1
+; SSE2-NEXT:    testb $64, %al
+; SSE2-NEXT:    je LBB23_14
+; SSE2-NEXT:  LBB23_13: ## %cond.load16
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255]
+; SSE2-NEXT:    pand %xmm0, %xmm1
+; SSE2-NEXT:    movzbl 6(%rdi), %ecx
+; SSE2-NEXT:    movd %ecx, %xmm2
+; SSE2-NEXT:    psllq $48, %xmm2
+; SSE2-NEXT:    pandn %xmm2, %xmm0
+; SSE2-NEXT:    por %xmm0, %xmm1
+; SSE2-NEXT:    testb $-128, %al
+; SSE2-NEXT:    je LBB23_16
+; SSE2-NEXT:  LBB23_15: ## %cond.load19
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255]
+; SSE2-NEXT:    pand %xmm0, %xmm1
+; SSE2-NEXT:    movzbl 7(%rdi), %ecx
+; SSE2-NEXT:    movd %ecx, %xmm2
+; SSE2-NEXT:    psllq $56, %xmm2
+; SSE2-NEXT:    pandn %xmm2, %xmm0
+; SSE2-NEXT:    por %xmm0, %xmm1
+; SSE2-NEXT:    testl $256, %eax ## imm = 0x100
+; SSE2-NEXT:    je LBB23_18
+; SSE2-NEXT:  LBB23_17: ## %cond.load22
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255]
+; SSE2-NEXT:    pand %xmm0, %xmm1
+; SSE2-NEXT:    movzbl 8(%rdi), %ecx
+; SSE2-NEXT:    movd %ecx, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,1,0,1]
+; SSE2-NEXT:    pandn %xmm2, %xmm0
+; SSE2-NEXT:    por %xmm0, %xmm1
+; SSE2-NEXT:    testl $512, %eax ## imm = 0x200
+; SSE2-NEXT:    je LBB23_20
+; SSE2-NEXT:  LBB23_19: ## %cond.load25
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255]
+; SSE2-NEXT:    pand %xmm0, %xmm1
+; SSE2-NEXT:    movzbl 9(%rdi), %ecx
+; SSE2-NEXT:    movd %ecx, %xmm2
+; SSE2-NEXT:    pslldq {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm2[0,1,2,3,4,5,6]
+; SSE2-NEXT:    pandn %xmm2, %xmm0
+; SSE2-NEXT:    por %xmm0, %xmm1
+; SSE2-NEXT:    testl $1024, %eax ## imm = 0x400
+; SSE2-NEXT:    je LBB23_22
+; SSE2-NEXT:  LBB23_21: ## %cond.load28
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255]
+; SSE2-NEXT:    pand %xmm0, %xmm1
+; SSE2-NEXT:    movzbl 10(%rdi), %ecx
+; SSE2-NEXT:    movd %ecx, %xmm2
+; SSE2-NEXT:    pslldq {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm2[0,1,2,3,4,5]
+; SSE2-NEXT:    pandn %xmm2, %xmm0
+; SSE2-NEXT:    por %xmm0, %xmm1
+; SSE2-NEXT:    testl $2048, %eax ## imm = 0x800
+; SSE2-NEXT:    je LBB23_24
+; SSE2-NEXT:  LBB23_23: ## %cond.load31
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255]
+; SSE2-NEXT:    pand %xmm0, %xmm1
+; SSE2-NEXT:    movzbl 11(%rdi), %ecx
+; SSE2-NEXT:    movd %ecx, %xmm2
+; SSE2-NEXT:    pslldq {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm2[0,1,2,3,4]
+; SSE2-NEXT:    pandn %xmm2, %xmm0
+; SSE2-NEXT:    por %xmm0, %xmm1
+; SSE2-NEXT:    testl $4096, %eax ## imm = 0x1000
+; SSE2-NEXT:    je LBB23_26
+; SSE2-NEXT:  LBB23_25: ## %cond.load34
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255]
+; SSE2-NEXT:    pand %xmm0, %xmm1
+; SSE2-NEXT:    movzbl 12(%rdi), %ecx
+; SSE2-NEXT:    movd %ecx, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,1,2,0]
+; SSE2-NEXT:    pandn %xmm2, %xmm0
+; SSE2-NEXT:    por %xmm0, %xmm1
+; SSE2-NEXT:    testl $8192, %eax ## imm = 0x2000
+; SSE2-NEXT:    je LBB23_28
+; SSE2-NEXT:  LBB23_27: ## %cond.load37
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255]
+; SSE2-NEXT:    pand %xmm0, %xmm1
+; SSE2-NEXT:    movzbl 13(%rdi), %ecx
+; SSE2-NEXT:    movd %ecx, %xmm2
+; SSE2-NEXT:    pslldq {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm2[0,1,2]
+; SSE2-NEXT:    pandn %xmm2, %xmm0
+; SSE2-NEXT:    por %xmm0, %xmm1
+; SSE2-NEXT:    testl $16384, %eax ## imm = 0x4000
+; SSE2-NEXT:    je LBB23_30
+; SSE2-NEXT:  LBB23_29: ## %cond.load40
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255]
+; SSE2-NEXT:    pand %xmm0, %xmm1
+; SSE2-NEXT:    movzbl 14(%rdi), %ecx
+; SSE2-NEXT:    movd %ecx, %xmm2
+; SSE2-NEXT:    pslldq {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm2[0,1]
+; SSE2-NEXT:    pandn %xmm2, %xmm0
+; SSE2-NEXT:    por %xmm0, %xmm1
+; SSE2-NEXT:    testl $32768, %eax ## imm = 0x8000
+; SSE2-NEXT:    je LBB23_32
+; SSE2-NEXT:  LBB23_31: ## %cond.load43
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE2-NEXT:    movzbl 15(%rdi), %eax
+; SSE2-NEXT:    movd %eax, %xmm0
+; SSE2-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0]
+; SSE2-NEXT:    por %xmm0, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE42-LABEL: load_v16i8_v16i8:
+; SSE42:       ## %bb.0:
+; SSE42-NEXT:    pmovmskb %xmm0, %eax
 ; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB23_2
-; SSE42-NEXT:  ## %bb.1: ## %cond.load
+; SSE42-NEXT:    jne LBB23_1
+; SSE42-NEXT:  ## %bb.2: ## %else
+; SSE42-NEXT:    testb $2, %al
+; SSE42-NEXT:    jne LBB23_3
+; SSE42-NEXT:  LBB23_4: ## %else2
+; SSE42-NEXT:    testb $4, %al
+; SSE42-NEXT:    jne LBB23_5
+; SSE42-NEXT:  LBB23_6: ## %else5
+; SSE42-NEXT:    testb $8, %al
+; SSE42-NEXT:    jne LBB23_7
+; SSE42-NEXT:  LBB23_8: ## %else8
+; SSE42-NEXT:    testb $16, %al
+; SSE42-NEXT:    jne LBB23_9
+; SSE42-NEXT:  LBB23_10: ## %else11
+; SSE42-NEXT:    testb $32, %al
+; SSE42-NEXT:    jne LBB23_11
+; SSE42-NEXT:  LBB23_12: ## %else14
+; SSE42-NEXT:    testb $64, %al
+; SSE42-NEXT:    jne LBB23_13
+; SSE42-NEXT:  LBB23_14: ## %else17
+; SSE42-NEXT:    testb $-128, %al
+; SSE42-NEXT:    jne LBB23_15
+; SSE42-NEXT:  LBB23_16: ## %else20
+; SSE42-NEXT:    testl $256, %eax ## imm = 0x100
+; SSE42-NEXT:    jne LBB23_17
+; SSE42-NEXT:  LBB23_18: ## %else23
+; SSE42-NEXT:    testl $512, %eax ## imm = 0x200
+; SSE42-NEXT:    jne LBB23_19
+; SSE42-NEXT:  LBB23_20: ## %else26
+; SSE42-NEXT:    testl $1024, %eax ## imm = 0x400
+; SSE42-NEXT:    jne LBB23_21
+; SSE42-NEXT:  LBB23_22: ## %else29
+; SSE42-NEXT:    testl $2048, %eax ## imm = 0x800
+; SSE42-NEXT:    jne LBB23_23
+; SSE42-NEXT:  LBB23_24: ## %else32
+; SSE42-NEXT:    testl $4096, %eax ## imm = 0x1000
+; SSE42-NEXT:    jne LBB23_25
+; SSE42-NEXT:  LBB23_26: ## %else35
+; SSE42-NEXT:    testl $8192, %eax ## imm = 0x2000
+; SSE42-NEXT:    jne LBB23_27
+; SSE42-NEXT:  LBB23_28: ## %else38
+; SSE42-NEXT:    testl $16384, %eax ## imm = 0x4000
+; SSE42-NEXT:    jne LBB23_29
+; SSE42-NEXT:  LBB23_30: ## %else41
+; SSE42-NEXT:    testl $32768, %eax ## imm = 0x8000
+; SSE42-NEXT:    jne LBB23_31
+; SSE42-NEXT:  LBB23_32: ## %else44
+; SSE42-NEXT:    movdqa %xmm1, %xmm0
+; SSE42-NEXT:    retq
+; SSE42-NEXT:  LBB23_1: ## %cond.load
 ; SSE42-NEXT:    pinsrb $0, (%rdi), %xmm1
-; SSE42-NEXT:  LBB23_2: ## %else
-; SSE42-NEXT:    pextrb $1, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $2, %al
 ; SSE42-NEXT:    je LBB23_4
-; SSE42-NEXT:  ## %bb.3: ## %cond.load1
+; SSE42-NEXT:  LBB23_3: ## %cond.load1
 ; SSE42-NEXT:    pinsrb $1, 1(%rdi), %xmm1
-; SSE42-NEXT:  LBB23_4: ## %else2
-; SSE42-NEXT:    pxor %xmm2, %xmm2
-; SSE42-NEXT:    pcmpgtb %xmm0, %xmm2
-; SSE42-NEXT:    pextrb $2, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $4, %al
 ; SSE42-NEXT:    je LBB23_6
-; SSE42-NEXT:  ## %bb.5: ## %cond.load4
+; SSE42-NEXT:  LBB23_5: ## %cond.load4
 ; SSE42-NEXT:    pinsrb $2, 2(%rdi), %xmm1
-; SSE42-NEXT:  LBB23_6: ## %else5
-; SSE42-NEXT:    pextrb $3, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $8, %al
 ; SSE42-NEXT:    je LBB23_8
-; SSE42-NEXT:  ## %bb.7: ## %cond.load7
+; SSE42-NEXT:  LBB23_7: ## %cond.load7
 ; SSE42-NEXT:    pinsrb $3, 3(%rdi), %xmm1
-; SSE42-NEXT:  LBB23_8: ## %else8
-; SSE42-NEXT:    pxor %xmm2, %xmm2
-; SSE42-NEXT:    pcmpgtb %xmm0, %xmm2
-; SSE42-NEXT:    pextrb $4, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $16, %al
 ; SSE42-NEXT:    je LBB23_10
-; SSE42-NEXT:  ## %bb.9: ## %cond.load10
+; SSE42-NEXT:  LBB23_9: ## %cond.load10
 ; SSE42-NEXT:    pinsrb $4, 4(%rdi), %xmm1
-; SSE42-NEXT:  LBB23_10: ## %else11
-; SSE42-NEXT:    pextrb $5, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $32, %al
 ; SSE42-NEXT:    je LBB23_12
-; SSE42-NEXT:  ## %bb.11: ## %cond.load13
+; SSE42-NEXT:  LBB23_11: ## %cond.load13
 ; SSE42-NEXT:    pinsrb $5, 5(%rdi), %xmm1
-; SSE42-NEXT:  LBB23_12: ## %else14
-; SSE42-NEXT:    pxor %xmm2, %xmm2
-; SSE42-NEXT:    pcmpgtb %xmm0, %xmm2
-; SSE42-NEXT:    pextrb $6, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $64, %al
 ; SSE42-NEXT:    je LBB23_14
-; SSE42-NEXT:  ## %bb.13: ## %cond.load16
+; SSE42-NEXT:  LBB23_13: ## %cond.load16
 ; SSE42-NEXT:    pinsrb $6, 6(%rdi), %xmm1
-; SSE42-NEXT:  LBB23_14: ## %else17
-; SSE42-NEXT:    pextrb $7, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $-128, %al
 ; SSE42-NEXT:    je LBB23_16
-; SSE42-NEXT:  ## %bb.15: ## %cond.load19
+; SSE42-NEXT:  LBB23_15: ## %cond.load19
 ; SSE42-NEXT:    pinsrb $7, 7(%rdi), %xmm1
-; SSE42-NEXT:  LBB23_16: ## %else20
-; SSE42-NEXT:    pxor %xmm2, %xmm2
-; SSE42-NEXT:    pcmpgtb %xmm0, %xmm2
-; SSE42-NEXT:    pextrb $8, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $256, %eax ## imm = 0x100
 ; SSE42-NEXT:    je LBB23_18
-; SSE42-NEXT:  ## %bb.17: ## %cond.load22
+; SSE42-NEXT:  LBB23_17: ## %cond.load22
 ; SSE42-NEXT:    pinsrb $8, 8(%rdi), %xmm1
-; SSE42-NEXT:  LBB23_18: ## %else23
-; SSE42-NEXT:    pextrb $9, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $512, %eax ## imm = 0x200
 ; SSE42-NEXT:    je LBB23_20
-; SSE42-NEXT:  ## %bb.19: ## %cond.load25
+; SSE42-NEXT:  LBB23_19: ## %cond.load25
 ; SSE42-NEXT:    pinsrb $9, 9(%rdi), %xmm1
-; SSE42-NEXT:  LBB23_20: ## %else26
-; SSE42-NEXT:    pxor %xmm2, %xmm2
-; SSE42-NEXT:    pcmpgtb %xmm0, %xmm2
-; SSE42-NEXT:    pextrb $10, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $1024, %eax ## imm = 0x400
 ; SSE42-NEXT:    je LBB23_22
-; SSE42-NEXT:  ## %bb.21: ## %cond.load28
+; SSE42-NEXT:  LBB23_21: ## %cond.load28
 ; SSE42-NEXT:    pinsrb $10, 10(%rdi), %xmm1
-; SSE42-NEXT:  LBB23_22: ## %else29
-; SSE42-NEXT:    pextrb $11, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $2048, %eax ## imm = 0x800
 ; SSE42-NEXT:    je LBB23_24
-; SSE42-NEXT:  ## %bb.23: ## %cond.load31
+; SSE42-NEXT:  LBB23_23: ## %cond.load31
 ; SSE42-NEXT:    pinsrb $11, 11(%rdi), %xmm1
-; SSE42-NEXT:  LBB23_24: ## %else32
-; SSE42-NEXT:    pxor %xmm2, %xmm2
-; SSE42-NEXT:    pcmpgtb %xmm0, %xmm2
-; SSE42-NEXT:    pextrb $12, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $4096, %eax ## imm = 0x1000
 ; SSE42-NEXT:    je LBB23_26
-; SSE42-NEXT:  ## %bb.25: ## %cond.load34
+; SSE42-NEXT:  LBB23_25: ## %cond.load34
 ; SSE42-NEXT:    pinsrb $12, 12(%rdi), %xmm1
-; SSE42-NEXT:  LBB23_26: ## %else35
-; SSE42-NEXT:    pextrb $13, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $8192, %eax ## imm = 0x2000
 ; SSE42-NEXT:    je LBB23_28
-; SSE42-NEXT:  ## %bb.27: ## %cond.load37
+; SSE42-NEXT:  LBB23_27: ## %cond.load37
 ; SSE42-NEXT:    pinsrb $13, 13(%rdi), %xmm1
-; SSE42-NEXT:  LBB23_28: ## %else38
-; SSE42-NEXT:    pxor %xmm2, %xmm2
-; SSE42-NEXT:    pcmpgtb %xmm0, %xmm2
-; SSE42-NEXT:    pextrb $14, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $16384, %eax ## imm = 0x4000
 ; SSE42-NEXT:    je LBB23_30
-; SSE42-NEXT:  ## %bb.29: ## %cond.load40
+; SSE42-NEXT:  LBB23_29: ## %cond.load40
 ; SSE42-NEXT:    pinsrb $14, 14(%rdi), %xmm1
-; SSE42-NEXT:  LBB23_30: ## %else41
-; SSE42-NEXT:    pextrb $15, %xmm2, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $32768, %eax ## imm = 0x8000
 ; SSE42-NEXT:    je LBB23_32
-; SSE42-NEXT:  ## %bb.31: ## %cond.load43
+; SSE42-NEXT:  LBB23_31: ## %cond.load43
 ; SSE42-NEXT:    pinsrb $15, 15(%rdi), %xmm1
-; SSE42-NEXT:  LBB23_32: ## %else44
 ; SSE42-NEXT:    movdqa %xmm1, %xmm0
 ; SSE42-NEXT:    retq
 ;
 ; AVX1OR2-LABEL: load_v16i8_v16i8:
 ; AVX1OR2:       ## %bb.0:
-; AVX1OR2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1OR2-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX1OR2-NEXT:    vpextrb $0, %xmm2, %eax
+; AVX1OR2-NEXT:    vpmovmskb %xmm0, %eax
 ; AVX1OR2-NEXT:    testb $1, %al
-; AVX1OR2-NEXT:    je LBB23_2
-; AVX1OR2-NEXT:  ## %bb.1: ## %cond.load
+; AVX1OR2-NEXT:    jne LBB23_1
+; AVX1OR2-NEXT:  ## %bb.2: ## %else
+; AVX1OR2-NEXT:    testb $2, %al
+; AVX1OR2-NEXT:    jne LBB23_3
+; AVX1OR2-NEXT:  LBB23_4: ## %else2
+; AVX1OR2-NEXT:    testb $4, %al
+; AVX1OR2-NEXT:    jne LBB23_5
+; AVX1OR2-NEXT:  LBB23_6: ## %else5
+; AVX1OR2-NEXT:    testb $8, %al
+; AVX1OR2-NEXT:    jne LBB23_7
+; AVX1OR2-NEXT:  LBB23_8: ## %else8
+; AVX1OR2-NEXT:    testb $16, %al
+; AVX1OR2-NEXT:    jne LBB23_9
+; AVX1OR2-NEXT:  LBB23_10: ## %else11
+; AVX1OR2-NEXT:    testb $32, %al
+; AVX1OR2-NEXT:    jne LBB23_11
+; AVX1OR2-NEXT:  LBB23_12: ## %else14
+; AVX1OR2-NEXT:    testb $64, %al
+; AVX1OR2-NEXT:    jne LBB23_13
+; AVX1OR2-NEXT:  LBB23_14: ## %else17
+; AVX1OR2-NEXT:    testb $-128, %al
+; AVX1OR2-NEXT:    jne LBB23_15
+; AVX1OR2-NEXT:  LBB23_16: ## %else20
+; AVX1OR2-NEXT:    testl $256, %eax ## imm = 0x100
+; AVX1OR2-NEXT:    jne LBB23_17
+; AVX1OR2-NEXT:  LBB23_18: ## %else23
+; AVX1OR2-NEXT:    testl $512, %eax ## imm = 0x200
+; AVX1OR2-NEXT:    jne LBB23_19
+; AVX1OR2-NEXT:  LBB23_20: ## %else26
+; AVX1OR2-NEXT:    testl $1024, %eax ## imm = 0x400
+; AVX1OR2-NEXT:    jne LBB23_21
+; AVX1OR2-NEXT:  LBB23_22: ## %else29
+; AVX1OR2-NEXT:    testl $2048, %eax ## imm = 0x800
+; AVX1OR2-NEXT:    jne LBB23_23
+; AVX1OR2-NEXT:  LBB23_24: ## %else32
+; AVX1OR2-NEXT:    testl $4096, %eax ## imm = 0x1000
+; AVX1OR2-NEXT:    jne LBB23_25
+; AVX1OR2-NEXT:  LBB23_26: ## %else35
+; AVX1OR2-NEXT:    testl $8192, %eax ## imm = 0x2000
+; AVX1OR2-NEXT:    jne LBB23_27
+; AVX1OR2-NEXT:  LBB23_28: ## %else38
+; AVX1OR2-NEXT:    testl $16384, %eax ## imm = 0x4000
+; AVX1OR2-NEXT:    jne LBB23_29
+; AVX1OR2-NEXT:  LBB23_30: ## %else41
+; AVX1OR2-NEXT:    testl $32768, %eax ## imm = 0x8000
+; AVX1OR2-NEXT:    jne LBB23_31
+; AVX1OR2-NEXT:  LBB23_32: ## %else44
+; AVX1OR2-NEXT:    vmovdqa %xmm1, %xmm0
+; AVX1OR2-NEXT:    retq
+; AVX1OR2-NEXT:  LBB23_1: ## %cond.load
 ; AVX1OR2-NEXT:    vpinsrb $0, (%rdi), %xmm1, %xmm1
-; AVX1OR2-NEXT:  LBB23_2: ## %else
-; AVX1OR2-NEXT:    vpextrb $1, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $2, %al
 ; AVX1OR2-NEXT:    je LBB23_4
-; AVX1OR2-NEXT:  ## %bb.3: ## %cond.load1
+; AVX1OR2-NEXT:  LBB23_3: ## %cond.load1
 ; AVX1OR2-NEXT:    vpinsrb $1, 1(%rdi), %xmm1, %xmm1
-; AVX1OR2-NEXT:  LBB23_4: ## %else2
-; AVX1OR2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1OR2-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX1OR2-NEXT:    vpextrb $2, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $4, %al
 ; AVX1OR2-NEXT:    je LBB23_6
-; AVX1OR2-NEXT:  ## %bb.5: ## %cond.load4
+; AVX1OR2-NEXT:  LBB23_5: ## %cond.load4
 ; AVX1OR2-NEXT:    vpinsrb $2, 2(%rdi), %xmm1, %xmm1
-; AVX1OR2-NEXT:  LBB23_6: ## %else5
-; AVX1OR2-NEXT:    vpextrb $3, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $8, %al
 ; AVX1OR2-NEXT:    je LBB23_8
-; AVX1OR2-NEXT:  ## %bb.7: ## %cond.load7
+; AVX1OR2-NEXT:  LBB23_7: ## %cond.load7
 ; AVX1OR2-NEXT:    vpinsrb $3, 3(%rdi), %xmm1, %xmm1
-; AVX1OR2-NEXT:  LBB23_8: ## %else8
-; AVX1OR2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1OR2-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX1OR2-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $16, %al
 ; AVX1OR2-NEXT:    je LBB23_10
-; AVX1OR2-NEXT:  ## %bb.9: ## %cond.load10
+; AVX1OR2-NEXT:  LBB23_9: ## %cond.load10
 ; AVX1OR2-NEXT:    vpinsrb $4, 4(%rdi), %xmm1, %xmm1
-; AVX1OR2-NEXT:  LBB23_10: ## %else11
-; AVX1OR2-NEXT:    vpextrb $5, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $32, %al
 ; AVX1OR2-NEXT:    je LBB23_12
-; AVX1OR2-NEXT:  ## %bb.11: ## %cond.load13
+; AVX1OR2-NEXT:  LBB23_11: ## %cond.load13
 ; AVX1OR2-NEXT:    vpinsrb $5, 5(%rdi), %xmm1, %xmm1
-; AVX1OR2-NEXT:  LBB23_12: ## %else14
-; AVX1OR2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1OR2-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX1OR2-NEXT:    vpextrb $6, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $64, %al
 ; AVX1OR2-NEXT:    je LBB23_14
-; AVX1OR2-NEXT:  ## %bb.13: ## %cond.load16
+; AVX1OR2-NEXT:  LBB23_13: ## %cond.load16
 ; AVX1OR2-NEXT:    vpinsrb $6, 6(%rdi), %xmm1, %xmm1
-; AVX1OR2-NEXT:  LBB23_14: ## %else17
-; AVX1OR2-NEXT:    vpextrb $7, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $-128, %al
 ; AVX1OR2-NEXT:    je LBB23_16
-; AVX1OR2-NEXT:  ## %bb.15: ## %cond.load19
+; AVX1OR2-NEXT:  LBB23_15: ## %cond.load19
 ; AVX1OR2-NEXT:    vpinsrb $7, 7(%rdi), %xmm1, %xmm1
-; AVX1OR2-NEXT:  LBB23_16: ## %else20
-; AVX1OR2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1OR2-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX1OR2-NEXT:    vpextrb $8, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testl $256, %eax ## imm = 0x100
 ; AVX1OR2-NEXT:    je LBB23_18
-; AVX1OR2-NEXT:  ## %bb.17: ## %cond.load22
+; AVX1OR2-NEXT:  LBB23_17: ## %cond.load22
 ; AVX1OR2-NEXT:    vpinsrb $8, 8(%rdi), %xmm1, %xmm1
-; AVX1OR2-NEXT:  LBB23_18: ## %else23
-; AVX1OR2-NEXT:    vpextrb $9, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testl $512, %eax ## imm = 0x200
 ; AVX1OR2-NEXT:    je LBB23_20
-; AVX1OR2-NEXT:  ## %bb.19: ## %cond.load25
+; AVX1OR2-NEXT:  LBB23_19: ## %cond.load25
 ; AVX1OR2-NEXT:    vpinsrb $9, 9(%rdi), %xmm1, %xmm1
-; AVX1OR2-NEXT:  LBB23_20: ## %else26
-; AVX1OR2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1OR2-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX1OR2-NEXT:    vpextrb $10, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testl $1024, %eax ## imm = 0x400
 ; AVX1OR2-NEXT:    je LBB23_22
-; AVX1OR2-NEXT:  ## %bb.21: ## %cond.load28
+; AVX1OR2-NEXT:  LBB23_21: ## %cond.load28
 ; AVX1OR2-NEXT:    vpinsrb $10, 10(%rdi), %xmm1, %xmm1
-; AVX1OR2-NEXT:  LBB23_22: ## %else29
-; AVX1OR2-NEXT:    vpextrb $11, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testl $2048, %eax ## imm = 0x800
 ; AVX1OR2-NEXT:    je LBB23_24
-; AVX1OR2-NEXT:  ## %bb.23: ## %cond.load31
+; AVX1OR2-NEXT:  LBB23_23: ## %cond.load31
 ; AVX1OR2-NEXT:    vpinsrb $11, 11(%rdi), %xmm1, %xmm1
-; AVX1OR2-NEXT:  LBB23_24: ## %else32
-; AVX1OR2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1OR2-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX1OR2-NEXT:    vpextrb $12, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testl $4096, %eax ## imm = 0x1000
 ; AVX1OR2-NEXT:    je LBB23_26
-; AVX1OR2-NEXT:  ## %bb.25: ## %cond.load34
+; AVX1OR2-NEXT:  LBB23_25: ## %cond.load34
 ; AVX1OR2-NEXT:    vpinsrb $12, 12(%rdi), %xmm1, %xmm1
-; AVX1OR2-NEXT:  LBB23_26: ## %else35
-; AVX1OR2-NEXT:    vpextrb $13, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testl $8192, %eax ## imm = 0x2000
 ; AVX1OR2-NEXT:    je LBB23_28
-; AVX1OR2-NEXT:  ## %bb.27: ## %cond.load37
+; AVX1OR2-NEXT:  LBB23_27: ## %cond.load37
 ; AVX1OR2-NEXT:    vpinsrb $13, 13(%rdi), %xmm1, %xmm1
-; AVX1OR2-NEXT:  LBB23_28: ## %else38
-; AVX1OR2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1OR2-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm0
-; AVX1OR2-NEXT:    vpextrb $14, %xmm0, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testl $16384, %eax ## imm = 0x4000
 ; AVX1OR2-NEXT:    je LBB23_30
-; AVX1OR2-NEXT:  ## %bb.29: ## %cond.load40
+; AVX1OR2-NEXT:  LBB23_29: ## %cond.load40
 ; AVX1OR2-NEXT:    vpinsrb $14, 14(%rdi), %xmm1, %xmm1
-; AVX1OR2-NEXT:  LBB23_30: ## %else41
-; AVX1OR2-NEXT:    vpextrb $15, %xmm0, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testl $32768, %eax ## imm = 0x8000
 ; AVX1OR2-NEXT:    je LBB23_32
-; AVX1OR2-NEXT:  ## %bb.31: ## %cond.load43
+; AVX1OR2-NEXT:  LBB23_31: ## %cond.load43
 ; AVX1OR2-NEXT:    vpinsrb $15, 15(%rdi), %xmm1, %xmm1
-; AVX1OR2-NEXT:  LBB23_32: ## %else44
 ; AVX1OR2-NEXT:    vmovdqa %xmm1, %xmm0
 ; AVX1OR2-NEXT:    retq
 ;
 ; AVX512F-LABEL: load_v16i8_v16i8:
 ; AVX512F:       ## %bb.0:
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    vpmovmskb %xmm0, %eax
 ; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je LBB23_2
-; AVX512F-NEXT:  ## %bb.1: ## %cond.load
+; AVX512F-NEXT:    jne LBB23_1
+; AVX512F-NEXT:  ## %bb.2: ## %else
+; AVX512F-NEXT:    testb $2, %al
+; AVX512F-NEXT:    jne LBB23_3
+; AVX512F-NEXT:  LBB23_4: ## %else2
+; AVX512F-NEXT:    testb $4, %al
+; AVX512F-NEXT:    jne LBB23_5
+; AVX512F-NEXT:  LBB23_6: ## %else5
+; AVX512F-NEXT:    testb $8, %al
+; AVX512F-NEXT:    jne LBB23_7
+; AVX512F-NEXT:  LBB23_8: ## %else8
+; AVX512F-NEXT:    testb $16, %al
+; AVX512F-NEXT:    jne LBB23_9
+; AVX512F-NEXT:  LBB23_10: ## %else11
+; AVX512F-NEXT:    testb $32, %al
+; AVX512F-NEXT:    jne LBB23_11
+; AVX512F-NEXT:  LBB23_12: ## %else14
+; AVX512F-NEXT:    testb $64, %al
+; AVX512F-NEXT:    jne LBB23_13
+; AVX512F-NEXT:  LBB23_14: ## %else17
+; AVX512F-NEXT:    testb $-128, %al
+; AVX512F-NEXT:    jne LBB23_15
+; AVX512F-NEXT:  LBB23_16: ## %else20
+; AVX512F-NEXT:    testl $256, %eax ## imm = 0x100
+; AVX512F-NEXT:    jne LBB23_17
+; AVX512F-NEXT:  LBB23_18: ## %else23
+; AVX512F-NEXT:    testl $512, %eax ## imm = 0x200
+; AVX512F-NEXT:    jne LBB23_19
+; AVX512F-NEXT:  LBB23_20: ## %else26
+; AVX512F-NEXT:    testl $1024, %eax ## imm = 0x400
+; AVX512F-NEXT:    jne LBB23_21
+; AVX512F-NEXT:  LBB23_22: ## %else29
+; AVX512F-NEXT:    testl $2048, %eax ## imm = 0x800
+; AVX512F-NEXT:    jne LBB23_23
+; AVX512F-NEXT:  LBB23_24: ## %else32
+; AVX512F-NEXT:    testl $4096, %eax ## imm = 0x1000
+; AVX512F-NEXT:    jne LBB23_25
+; AVX512F-NEXT:  LBB23_26: ## %else35
+; AVX512F-NEXT:    testl $8192, %eax ## imm = 0x2000
+; AVX512F-NEXT:    jne LBB23_27
+; AVX512F-NEXT:  LBB23_28: ## %else38
+; AVX512F-NEXT:    testl $16384, %eax ## imm = 0x4000
+; AVX512F-NEXT:    jne LBB23_29
+; AVX512F-NEXT:  LBB23_30: ## %else41
+; AVX512F-NEXT:    testl $32768, %eax ## imm = 0x8000
+; AVX512F-NEXT:    jne LBB23_31
+; AVX512F-NEXT:  LBB23_32: ## %else44
+; AVX512F-NEXT:    vmovdqa %xmm1, %xmm0
+; AVX512F-NEXT:    retq
+; AVX512F-NEXT:  LBB23_1: ## %cond.load
 ; AVX512F-NEXT:    vpinsrb $0, (%rdi), %xmm1, %xmm1
-; AVX512F-NEXT:  LBB23_2: ## %else
-; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $2, %al
 ; AVX512F-NEXT:    je LBB23_4
-; AVX512F-NEXT:  ## %bb.3: ## %cond.load1
+; AVX512F-NEXT:  LBB23_3: ## %cond.load1
 ; AVX512F-NEXT:    vpinsrb $1, 1(%rdi), %xmm1, %xmm1
-; AVX512F-NEXT:  LBB23_4: ## %else2
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $2, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $4, %al
 ; AVX512F-NEXT:    je LBB23_6
-; AVX512F-NEXT:  ## %bb.5: ## %cond.load4
+; AVX512F-NEXT:  LBB23_5: ## %cond.load4
 ; AVX512F-NEXT:    vpinsrb $2, 2(%rdi), %xmm1, %xmm1
-; AVX512F-NEXT:  LBB23_6: ## %else5
-; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $8, %al
 ; AVX512F-NEXT:    je LBB23_8
-; AVX512F-NEXT:  ## %bb.7: ## %cond.load7
+; AVX512F-NEXT:  LBB23_7: ## %cond.load7
 ; AVX512F-NEXT:    vpinsrb $3, 3(%rdi), %xmm1, %xmm1
-; AVX512F-NEXT:  LBB23_8: ## %else8
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $4, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $16, %al
 ; AVX512F-NEXT:    je LBB23_10
-; AVX512F-NEXT:  ## %bb.9: ## %cond.load10
+; AVX512F-NEXT:  LBB23_9: ## %cond.load10
 ; AVX512F-NEXT:    vpinsrb $4, 4(%rdi), %xmm1, %xmm1
-; AVX512F-NEXT:  LBB23_10: ## %else11
-; AVX512F-NEXT:    kshiftrw $5, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $32, %al
 ; AVX512F-NEXT:    je LBB23_12
-; AVX512F-NEXT:  ## %bb.11: ## %cond.load13
+; AVX512F-NEXT:  LBB23_11: ## %cond.load13
 ; AVX512F-NEXT:    vpinsrb $5, 5(%rdi), %xmm1, %xmm1
-; AVX512F-NEXT:  LBB23_12: ## %else14
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $6, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $64, %al
 ; AVX512F-NEXT:    je LBB23_14
-; AVX512F-NEXT:  ## %bb.13: ## %cond.load16
+; AVX512F-NEXT:  LBB23_13: ## %cond.load16
 ; AVX512F-NEXT:    vpinsrb $6, 6(%rdi), %xmm1, %xmm1
-; AVX512F-NEXT:  LBB23_14: ## %else17
-; AVX512F-NEXT:    kshiftrw $7, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $-128, %al
 ; AVX512F-NEXT:    je LBB23_16
-; AVX512F-NEXT:  ## %bb.15: ## %cond.load19
+; AVX512F-NEXT:  LBB23_15: ## %cond.load19
 ; AVX512F-NEXT:    vpinsrb $7, 7(%rdi), %xmm1, %xmm1
-; AVX512F-NEXT:  LBB23_16: ## %else20
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $8, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $256, %eax ## imm = 0x100
 ; AVX512F-NEXT:    je LBB23_18
-; AVX512F-NEXT:  ## %bb.17: ## %cond.load22
+; AVX512F-NEXT:  LBB23_17: ## %cond.load22
 ; AVX512F-NEXT:    vpinsrb $8, 8(%rdi), %xmm1, %xmm1
-; AVX512F-NEXT:  LBB23_18: ## %else23
-; AVX512F-NEXT:    kshiftrw $9, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $512, %eax ## imm = 0x200
 ; AVX512F-NEXT:    je LBB23_20
-; AVX512F-NEXT:  ## %bb.19: ## %cond.load25
+; AVX512F-NEXT:  LBB23_19: ## %cond.load25
 ; AVX512F-NEXT:    vpinsrb $9, 9(%rdi), %xmm1, %xmm1
-; AVX512F-NEXT:  LBB23_20: ## %else26
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $10, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $1024, %eax ## imm = 0x400
 ; AVX512F-NEXT:    je LBB23_22
-; AVX512F-NEXT:  ## %bb.21: ## %cond.load28
+; AVX512F-NEXT:  LBB23_21: ## %cond.load28
 ; AVX512F-NEXT:    vpinsrb $10, 10(%rdi), %xmm1, %xmm1
-; AVX512F-NEXT:  LBB23_22: ## %else29
-; AVX512F-NEXT:    kshiftrw $11, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $2048, %eax ## imm = 0x800
 ; AVX512F-NEXT:    je LBB23_24
-; AVX512F-NEXT:  ## %bb.23: ## %cond.load31
+; AVX512F-NEXT:  LBB23_23: ## %cond.load31
 ; AVX512F-NEXT:    vpinsrb $11, 11(%rdi), %xmm1, %xmm1
-; AVX512F-NEXT:  LBB23_24: ## %else32
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $12, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $4096, %eax ## imm = 0x1000
 ; AVX512F-NEXT:    je LBB23_26
-; AVX512F-NEXT:  ## %bb.25: ## %cond.load34
+; AVX512F-NEXT:  LBB23_25: ## %cond.load34
 ; AVX512F-NEXT:    vpinsrb $12, 12(%rdi), %xmm1, %xmm1
-; AVX512F-NEXT:  LBB23_26: ## %else35
-; AVX512F-NEXT:    kshiftrw $13, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $8192, %eax ## imm = 0x2000
 ; AVX512F-NEXT:    je LBB23_28
-; AVX512F-NEXT:  ## %bb.27: ## %cond.load37
+; AVX512F-NEXT:  LBB23_27: ## %cond.load37
 ; AVX512F-NEXT:    vpinsrb $13, 13(%rdi), %xmm1, %xmm1
-; AVX512F-NEXT:  LBB23_28: ## %else38
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm0
-; AVX512F-NEXT:    vpmovsxbd %xmm0, %zmm0
-; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k0
-; AVX512F-NEXT:    kshiftrw $14, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $16384, %eax ## imm = 0x4000
 ; AVX512F-NEXT:    je LBB23_30
-; AVX512F-NEXT:  ## %bb.29: ## %cond.load40
+; AVX512F-NEXT:  LBB23_29: ## %cond.load40
 ; AVX512F-NEXT:    vpinsrb $14, 14(%rdi), %xmm1, %xmm1
-; AVX512F-NEXT:  LBB23_30: ## %else41
-; AVX512F-NEXT:    kshiftrw $15, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $32768, %eax ## imm = 0x8000
 ; AVX512F-NEXT:    je LBB23_32
-; AVX512F-NEXT:  ## %bb.31: ## %cond.load43
+; AVX512F-NEXT:  LBB23_31: ## %cond.load43
 ; AVX512F-NEXT:    vpinsrb $15, 15(%rdi), %xmm1, %xmm1
-; AVX512F-NEXT:  LBB23_32: ## %else44
 ; AVX512F-NEXT:    vmovdqa %xmm1, %xmm0
-; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VLDQ-LABEL: load_v16i8_v16i8:
 ; AVX512VLDQ:       ## %bb.0:
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm2, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
+; AVX512VLDQ-NEXT:    vpmovmskb %xmm0, %eax
 ; AVX512VLDQ-NEXT:    testb $1, %al
-; AVX512VLDQ-NEXT:    je LBB23_2
-; AVX512VLDQ-NEXT:  ## %bb.1: ## %cond.load
+; AVX512VLDQ-NEXT:    jne LBB23_1
+; AVX512VLDQ-NEXT:  ## %bb.2: ## %else
+; AVX512VLDQ-NEXT:    testb $2, %al
+; AVX512VLDQ-NEXT:    jne LBB23_3
+; AVX512VLDQ-NEXT:  LBB23_4: ## %else2
+; AVX512VLDQ-NEXT:    testb $4, %al
+; AVX512VLDQ-NEXT:    jne LBB23_5
+; AVX512VLDQ-NEXT:  LBB23_6: ## %else5
+; AVX512VLDQ-NEXT:    testb $8, %al
+; AVX512VLDQ-NEXT:    jne LBB23_7
+; AVX512VLDQ-NEXT:  LBB23_8: ## %else8
+; AVX512VLDQ-NEXT:    testb $16, %al
+; AVX512VLDQ-NEXT:    jne LBB23_9
+; AVX512VLDQ-NEXT:  LBB23_10: ## %else11
+; AVX512VLDQ-NEXT:    testb $32, %al
+; AVX512VLDQ-NEXT:    jne LBB23_11
+; AVX512VLDQ-NEXT:  LBB23_12: ## %else14
+; AVX512VLDQ-NEXT:    testb $64, %al
+; AVX512VLDQ-NEXT:    jne LBB23_13
+; AVX512VLDQ-NEXT:  LBB23_14: ## %else17
+; AVX512VLDQ-NEXT:    testb $-128, %al
+; AVX512VLDQ-NEXT:    jne LBB23_15
+; AVX512VLDQ-NEXT:  LBB23_16: ## %else20
+; AVX512VLDQ-NEXT:    testl $256, %eax ## imm = 0x100
+; AVX512VLDQ-NEXT:    jne LBB23_17
+; AVX512VLDQ-NEXT:  LBB23_18: ## %else23
+; AVX512VLDQ-NEXT:    testl $512, %eax ## imm = 0x200
+; AVX512VLDQ-NEXT:    jne LBB23_19
+; AVX512VLDQ-NEXT:  LBB23_20: ## %else26
+; AVX512VLDQ-NEXT:    testl $1024, %eax ## imm = 0x400
+; AVX512VLDQ-NEXT:    jne LBB23_21
+; AVX512VLDQ-NEXT:  LBB23_22: ## %else29
+; AVX512VLDQ-NEXT:    testl $2048, %eax ## imm = 0x800
+; AVX512VLDQ-NEXT:    jne LBB23_23
+; AVX512VLDQ-NEXT:  LBB23_24: ## %else32
+; AVX512VLDQ-NEXT:    testl $4096, %eax ## imm = 0x1000
+; AVX512VLDQ-NEXT:    jne LBB23_25
+; AVX512VLDQ-NEXT:  LBB23_26: ## %else35
+; AVX512VLDQ-NEXT:    testl $8192, %eax ## imm = 0x2000
+; AVX512VLDQ-NEXT:    jne LBB23_27
+; AVX512VLDQ-NEXT:  LBB23_28: ## %else38
+; AVX512VLDQ-NEXT:    testl $16384, %eax ## imm = 0x4000
+; AVX512VLDQ-NEXT:    jne LBB23_29
+; AVX512VLDQ-NEXT:  LBB23_30: ## %else41
+; AVX512VLDQ-NEXT:    testl $32768, %eax ## imm = 0x8000
+; AVX512VLDQ-NEXT:    jne LBB23_31
+; AVX512VLDQ-NEXT:  LBB23_32: ## %else44
+; AVX512VLDQ-NEXT:    vmovdqa %xmm1, %xmm0
+; AVX512VLDQ-NEXT:    retq
+; AVX512VLDQ-NEXT:  LBB23_1: ## %cond.load
 ; AVX512VLDQ-NEXT:    vpinsrb $0, (%rdi), %xmm1, %xmm1
-; AVX512VLDQ-NEXT:  LBB23_2: ## %else
-; AVX512VLDQ-NEXT:    kshiftrw $1, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $2, %al
 ; AVX512VLDQ-NEXT:    je LBB23_4
-; AVX512VLDQ-NEXT:  ## %bb.3: ## %cond.load1
+; AVX512VLDQ-NEXT:  LBB23_3: ## %cond.load1
 ; AVX512VLDQ-NEXT:    vpinsrb $1, 1(%rdi), %xmm1, %xmm1
-; AVX512VLDQ-NEXT:  LBB23_4: ## %else2
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm2, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $2, %k0, %k1
-; AVX512VLDQ-NEXT:    kmovw %k1, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $4, %al
 ; AVX512VLDQ-NEXT:    je LBB23_6
-; AVX512VLDQ-NEXT:  ## %bb.5: ## %cond.load4
+; AVX512VLDQ-NEXT:  LBB23_5: ## %cond.load4
 ; AVX512VLDQ-NEXT:    vpinsrb $2, 2(%rdi), %xmm1, %xmm1
-; AVX512VLDQ-NEXT:  LBB23_6: ## %else5
-; AVX512VLDQ-NEXT:    kshiftrw $3, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $8, %al
 ; AVX512VLDQ-NEXT:    je LBB23_8
-; AVX512VLDQ-NEXT:  ## %bb.7: ## %cond.load7
+; AVX512VLDQ-NEXT:  LBB23_7: ## %cond.load7
 ; AVX512VLDQ-NEXT:    vpinsrb $3, 3(%rdi), %xmm1, %xmm1
-; AVX512VLDQ-NEXT:  LBB23_8: ## %else8
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm2, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $4, %k0, %k1
-; AVX512VLDQ-NEXT:    kmovw %k1, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $16, %al
 ; AVX512VLDQ-NEXT:    je LBB23_10
-; AVX512VLDQ-NEXT:  ## %bb.9: ## %cond.load10
+; AVX512VLDQ-NEXT:  LBB23_9: ## %cond.load10
 ; AVX512VLDQ-NEXT:    vpinsrb $4, 4(%rdi), %xmm1, %xmm1
-; AVX512VLDQ-NEXT:  LBB23_10: ## %else11
-; AVX512VLDQ-NEXT:    kshiftrw $5, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $32, %al
 ; AVX512VLDQ-NEXT:    je LBB23_12
-; AVX512VLDQ-NEXT:  ## %bb.11: ## %cond.load13
+; AVX512VLDQ-NEXT:  LBB23_11: ## %cond.load13
 ; AVX512VLDQ-NEXT:    vpinsrb $5, 5(%rdi), %xmm1, %xmm1
-; AVX512VLDQ-NEXT:  LBB23_12: ## %else14
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm2, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $6, %k0, %k1
-; AVX512VLDQ-NEXT:    kmovw %k1, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $64, %al
 ; AVX512VLDQ-NEXT:    je LBB23_14
-; AVX512VLDQ-NEXT:  ## %bb.13: ## %cond.load16
+; AVX512VLDQ-NEXT:  LBB23_13: ## %cond.load16
 ; AVX512VLDQ-NEXT:    vpinsrb $6, 6(%rdi), %xmm1, %xmm1
-; AVX512VLDQ-NEXT:  LBB23_14: ## %else17
-; AVX512VLDQ-NEXT:    kshiftrw $7, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $-128, %al
 ; AVX512VLDQ-NEXT:    je LBB23_16
-; AVX512VLDQ-NEXT:  ## %bb.15: ## %cond.load19
+; AVX512VLDQ-NEXT:  LBB23_15: ## %cond.load19
 ; AVX512VLDQ-NEXT:    vpinsrb $7, 7(%rdi), %xmm1, %xmm1
-; AVX512VLDQ-NEXT:  LBB23_16: ## %else20
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm2, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $8, %k0, %k1
-; AVX512VLDQ-NEXT:    kmovw %k1, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testl $256, %eax ## imm = 0x100
 ; AVX512VLDQ-NEXT:    je LBB23_18
-; AVX512VLDQ-NEXT:  ## %bb.17: ## %cond.load22
+; AVX512VLDQ-NEXT:  LBB23_17: ## %cond.load22
 ; AVX512VLDQ-NEXT:    vpinsrb $8, 8(%rdi), %xmm1, %xmm1
-; AVX512VLDQ-NEXT:  LBB23_18: ## %else23
-; AVX512VLDQ-NEXT:    kshiftrw $9, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testl $512, %eax ## imm = 0x200
 ; AVX512VLDQ-NEXT:    je LBB23_20
-; AVX512VLDQ-NEXT:  ## %bb.19: ## %cond.load25
+; AVX512VLDQ-NEXT:  LBB23_19: ## %cond.load25
 ; AVX512VLDQ-NEXT:    vpinsrb $9, 9(%rdi), %xmm1, %xmm1
-; AVX512VLDQ-NEXT:  LBB23_20: ## %else26
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm2, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $10, %k0, %k1
-; AVX512VLDQ-NEXT:    kmovw %k1, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testl $1024, %eax ## imm = 0x400
 ; AVX512VLDQ-NEXT:    je LBB23_22
-; AVX512VLDQ-NEXT:  ## %bb.21: ## %cond.load28
+; AVX512VLDQ-NEXT:  LBB23_21: ## %cond.load28
 ; AVX512VLDQ-NEXT:    vpinsrb $10, 10(%rdi), %xmm1, %xmm1
-; AVX512VLDQ-NEXT:  LBB23_22: ## %else29
-; AVX512VLDQ-NEXT:    kshiftrw $11, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testl $2048, %eax ## imm = 0x800
 ; AVX512VLDQ-NEXT:    je LBB23_24
-; AVX512VLDQ-NEXT:  ## %bb.23: ## %cond.load31
+; AVX512VLDQ-NEXT:  LBB23_23: ## %cond.load31
 ; AVX512VLDQ-NEXT:    vpinsrb $11, 11(%rdi), %xmm1, %xmm1
-; AVX512VLDQ-NEXT:  LBB23_24: ## %else32
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm2, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $12, %k0, %k1
-; AVX512VLDQ-NEXT:    kmovw %k1, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testl $4096, %eax ## imm = 0x1000
 ; AVX512VLDQ-NEXT:    je LBB23_26
-; AVX512VLDQ-NEXT:  ## %bb.25: ## %cond.load34
+; AVX512VLDQ-NEXT:  LBB23_25: ## %cond.load34
 ; AVX512VLDQ-NEXT:    vpinsrb $12, 12(%rdi), %xmm1, %xmm1
-; AVX512VLDQ-NEXT:  LBB23_26: ## %else35
-; AVX512VLDQ-NEXT:    kshiftrw $13, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testl $8192, %eax ## imm = 0x2000
 ; AVX512VLDQ-NEXT:    je LBB23_28
-; AVX512VLDQ-NEXT:  ## %bb.27: ## %cond.load37
+; AVX512VLDQ-NEXT:  LBB23_27: ## %cond.load37
 ; AVX512VLDQ-NEXT:    vpinsrb $13, 13(%rdi), %xmm1, %xmm1
-; AVX512VLDQ-NEXT:  LBB23_28: ## %else38
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm0
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm0, %zmm0
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm0, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $14, %k0, %k1
-; AVX512VLDQ-NEXT:    kmovw %k1, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testl $16384, %eax ## imm = 0x4000
 ; AVX512VLDQ-NEXT:    je LBB23_30
-; AVX512VLDQ-NEXT:  ## %bb.29: ## %cond.load40
+; AVX512VLDQ-NEXT:  LBB23_29: ## %cond.load40
 ; AVX512VLDQ-NEXT:    vpinsrb $14, 14(%rdi), %xmm1, %xmm1
-; AVX512VLDQ-NEXT:  LBB23_30: ## %else41
-; AVX512VLDQ-NEXT:    kshiftrw $15, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testl $32768, %eax ## imm = 0x8000
 ; AVX512VLDQ-NEXT:    je LBB23_32
-; AVX512VLDQ-NEXT:  ## %bb.31: ## %cond.load43
+; AVX512VLDQ-NEXT:  LBB23_31: ## %cond.load43
 ; AVX512VLDQ-NEXT:    vpinsrb $15, 15(%rdi), %xmm1, %xmm1
-; AVX512VLDQ-NEXT:  LBB23_32: ## %else44
 ; AVX512VLDQ-NEXT:    vmovdqa %xmm1, %xmm0
-; AVX512VLDQ-NEXT:    vzeroupper
 ; AVX512VLDQ-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: load_v16i8_v16i8:
@@ -4639,414 +4345,106 @@ define <16 x i8> @load_v16i8_v16i8(<16 x i8> %trigger, <16 x i8>* %addr, <16 x i
 define <32 x i8> @load_v32i8_v32i8(<32 x i8> %trigger, <32 x i8>* %addr, <32 x i8> %dst) {
 ; SSE2-LABEL: load_v32i8_v32i8:
 ; SSE2:       ## %bb.0:
-; SSE2-NEXT:    pxor %xmm4, %xmm4
-; SSE2-NEXT:    pcmpgtb %xmm0, %xmm4
-; SSE2-NEXT:    movd %xmm4, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB24_2
-; SSE2-NEXT:  ## %bb.1: ## %cond.load
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; SSE2-NEXT:    pand %xmm4, %xmm2
-; SSE2-NEXT:    movzbl (%rdi), %ecx
-; SSE2-NEXT:    movd %ecx, %xmm5
-; SSE2-NEXT:    pandn %xmm5, %xmm4
-; SSE2-NEXT:    por %xmm4, %xmm2
-; SSE2-NEXT:  LBB24_2: ## %else
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB24_4
-; SSE2-NEXT:  ## %bb.3: ## %cond.load1
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; SSE2-NEXT:    pand %xmm4, %xmm2
-; SSE2-NEXT:    movzbl 1(%rdi), %eax
-; SSE2-NEXT:    movd %eax, %xmm5
-; SSE2-NEXT:    psllw $8, %xmm5
-; SSE2-NEXT:    pandn %xmm5, %xmm4
-; SSE2-NEXT:    por %xmm4, %xmm2
+; SSE2-NEXT:    pmovmskb %xmm0, %ecx
+; SSE2-NEXT:    pmovmskb %xmm1, %eax
+; SSE2-NEXT:    shll $16, %eax
+; SSE2-NEXT:    orl %ecx, %eax
+; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    jne LBB24_1
+; SSE2-NEXT:  ## %bb.2: ## %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne LBB24_3
 ; SSE2-NEXT:  LBB24_4: ## %else2
-; SSE2-NEXT:    pxor %xmm4, %xmm4
-; SSE2-NEXT:    pcmpgtb %xmm0, %xmm4
-; SSE2-NEXT:    movd %xmm4, %eax
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    shrl $16, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    je LBB24_6
-; SSE2-NEXT:  ## %bb.5: ## %cond.load4
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; SSE2-NEXT:    pand %xmm4, %xmm2
-; SSE2-NEXT:    movzbl 2(%rdi), %ecx
-; SSE2-NEXT:    movd %ecx, %xmm5
-; SSE2-NEXT:    pslld $16, %xmm5
-; SSE2-NEXT:    pandn %xmm5, %xmm4
-; SSE2-NEXT:    por %xmm4, %xmm2
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne LBB24_5
 ; SSE2-NEXT:  LBB24_6: ## %else5
-; SSE2-NEXT:    shrl $24, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB24_8
-; SSE2-NEXT:  ## %bb.7: ## %cond.load7
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255]
-; SSE2-NEXT:    pand %xmm4, %xmm2
-; SSE2-NEXT:    movzbl 3(%rdi), %eax
-; SSE2-NEXT:    movd %eax, %xmm5
-; SSE2-NEXT:    pslld $24, %xmm5
-; SSE2-NEXT:    pandn %xmm5, %xmm4
-; SSE2-NEXT:    por %xmm4, %xmm2
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne LBB24_7
 ; SSE2-NEXT:  LBB24_8: ## %else8
-; SSE2-NEXT:    pxor %xmm4, %xmm4
-; SSE2-NEXT:    pcmpgtb %xmm0, %xmm4
-; SSE2-NEXT:    pextrw $2, %xmm4, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB24_10
-; SSE2-NEXT:  ## %bb.9: ## %cond.load10
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255]
-; SSE2-NEXT:    pand %xmm4, %xmm2
-; SSE2-NEXT:    movzbl 4(%rdi), %ecx
-; SSE2-NEXT:    movd %ecx, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[0,0,1,1]
-; SSE2-NEXT:    pandn %xmm5, %xmm4
-; SSE2-NEXT:    por %xmm4, %xmm2
+; SSE2-NEXT:    testb $16, %al
+; SSE2-NEXT:    jne LBB24_9
 ; SSE2-NEXT:  LBB24_10: ## %else11
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB24_12
-; SSE2-NEXT:  ## %bb.11: ## %cond.load13
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255]
-; SSE2-NEXT:    pand %xmm4, %xmm2
-; SSE2-NEXT:    movzbl 5(%rdi), %eax
-; SSE2-NEXT:    movd %eax, %xmm5
-; SSE2-NEXT:    psllq $40, %xmm5
-; SSE2-NEXT:    pandn %xmm5, %xmm4
-; SSE2-NEXT:    por %xmm4, %xmm2
+; SSE2-NEXT:    testb $32, %al
+; SSE2-NEXT:    jne LBB24_11
 ; SSE2-NEXT:  LBB24_12: ## %else14
-; SSE2-NEXT:    pxor %xmm4, %xmm4
-; SSE2-NEXT:    pcmpgtb %xmm0, %xmm4
-; SSE2-NEXT:    pextrw $3, %xmm4, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB24_14
-; SSE2-NEXT:  ## %bb.13: ## %cond.load16
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255]
-; SSE2-NEXT:    pand %xmm4, %xmm2
-; SSE2-NEXT:    movzbl 6(%rdi), %ecx
-; SSE2-NEXT:    movd %ecx, %xmm5
-; SSE2-NEXT:    psllq $48, %xmm5
-; SSE2-NEXT:    pandn %xmm5, %xmm4
-; SSE2-NEXT:    por %xmm4, %xmm2
+; SSE2-NEXT:    testb $64, %al
+; SSE2-NEXT:    jne LBB24_13
 ; SSE2-NEXT:  LBB24_14: ## %else17
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB24_16
-; SSE2-NEXT:  ## %bb.15: ## %cond.load19
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255]
-; SSE2-NEXT:    pand %xmm4, %xmm2
-; SSE2-NEXT:    movzbl 7(%rdi), %eax
-; SSE2-NEXT:    movd %eax, %xmm5
-; SSE2-NEXT:    psllq $56, %xmm5
-; SSE2-NEXT:    pandn %xmm5, %xmm4
-; SSE2-NEXT:    por %xmm4, %xmm2
+; SSE2-NEXT:    testb $-128, %al
+; SSE2-NEXT:    jne LBB24_15
 ; SSE2-NEXT:  LBB24_16: ## %else20
-; SSE2-NEXT:    pxor %xmm4, %xmm4
-; SSE2-NEXT:    pcmpgtb %xmm0, %xmm4
-; SSE2-NEXT:    pextrw $4, %xmm4, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB24_18
-; SSE2-NEXT:  ## %bb.17: ## %cond.load22
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255]
-; SSE2-NEXT:    pand %xmm4, %xmm2
-; SSE2-NEXT:    movzbl 8(%rdi), %ecx
-; SSE2-NEXT:    movd %ecx, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[0,1,0,1]
-; SSE2-NEXT:    pandn %xmm5, %xmm4
-; SSE2-NEXT:    por %xmm4, %xmm2
+; SSE2-NEXT:    testl $256, %eax ## imm = 0x100
+; SSE2-NEXT:    jne LBB24_17
 ; SSE2-NEXT:  LBB24_18: ## %else23
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB24_20
-; SSE2-NEXT:  ## %bb.19: ## %cond.load25
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255]
-; SSE2-NEXT:    pand %xmm4, %xmm2
-; SSE2-NEXT:    movzbl 9(%rdi), %eax
-; SSE2-NEXT:    movd %eax, %xmm5
-; SSE2-NEXT:    pslldq {{.*#+}} xmm5 = zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm5[0,1,2,3,4,5,6]
-; SSE2-NEXT:    pandn %xmm5, %xmm4
-; SSE2-NEXT:    por %xmm4, %xmm2
+; SSE2-NEXT:    testl $512, %eax ## imm = 0x200
+; SSE2-NEXT:    jne LBB24_19
 ; SSE2-NEXT:  LBB24_20: ## %else26
-; SSE2-NEXT:    pxor %xmm4, %xmm4
-; SSE2-NEXT:    pcmpgtb %xmm0, %xmm4
-; SSE2-NEXT:    pextrw $5, %xmm4, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB24_22
-; SSE2-NEXT:  ## %bb.21: ## %cond.load28
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255]
-; SSE2-NEXT:    pand %xmm4, %xmm2
-; SSE2-NEXT:    movzbl 10(%rdi), %ecx
-; SSE2-NEXT:    movd %ecx, %xmm5
-; SSE2-NEXT:    pslldq {{.*#+}} xmm5 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm5[0,1,2,3,4,5]
-; SSE2-NEXT:    pandn %xmm5, %xmm4
-; SSE2-NEXT:    por %xmm4, %xmm2
+; SSE2-NEXT:    testl $1024, %eax ## imm = 0x400
+; SSE2-NEXT:    jne LBB24_21
 ; SSE2-NEXT:  LBB24_22: ## %else29
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB24_24
-; SSE2-NEXT:  ## %bb.23: ## %cond.load31
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255]
-; SSE2-NEXT:    pand %xmm4, %xmm2
-; SSE2-NEXT:    movzbl 11(%rdi), %eax
-; SSE2-NEXT:    movd %eax, %xmm5
-; SSE2-NEXT:    pslldq {{.*#+}} xmm5 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm5[0,1,2,3,4]
-; SSE2-NEXT:    pandn %xmm5, %xmm4
-; SSE2-NEXT:    por %xmm4, %xmm2
+; SSE2-NEXT:    testl $2048, %eax ## imm = 0x800
+; SSE2-NEXT:    jne LBB24_23
 ; SSE2-NEXT:  LBB24_24: ## %else32
-; SSE2-NEXT:    pxor %xmm4, %xmm4
-; SSE2-NEXT:    pcmpgtb %xmm0, %xmm4
-; SSE2-NEXT:    pextrw $6, %xmm4, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB24_26
-; SSE2-NEXT:  ## %bb.25: ## %cond.load34
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255]
-; SSE2-NEXT:    pand %xmm4, %xmm2
-; SSE2-NEXT:    movzbl 12(%rdi), %ecx
-; SSE2-NEXT:    movd %ecx, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[0,1,2,0]
-; SSE2-NEXT:    pandn %xmm5, %xmm4
-; SSE2-NEXT:    por %xmm4, %xmm2
+; SSE2-NEXT:    testl $4096, %eax ## imm = 0x1000
+; SSE2-NEXT:    jne LBB24_25
 ; SSE2-NEXT:  LBB24_26: ## %else35
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB24_28
-; SSE2-NEXT:  ## %bb.27: ## %cond.load37
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255]
-; SSE2-NEXT:    pand %xmm4, %xmm2
-; SSE2-NEXT:    movzbl 13(%rdi), %eax
-; SSE2-NEXT:    movd %eax, %xmm5
-; SSE2-NEXT:    pslldq {{.*#+}} xmm5 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm5[0,1,2]
-; SSE2-NEXT:    pandn %xmm5, %xmm4
-; SSE2-NEXT:    por %xmm4, %xmm2
+; SSE2-NEXT:    testl $8192, %eax ## imm = 0x2000
+; SSE2-NEXT:    jne LBB24_27
 ; SSE2-NEXT:  LBB24_28: ## %else38
-; SSE2-NEXT:    pxor %xmm4, %xmm4
-; SSE2-NEXT:    pcmpgtb %xmm0, %xmm4
-; SSE2-NEXT:    pextrw $7, %xmm4, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB24_30
-; SSE2-NEXT:  ## %bb.29: ## %cond.load40
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255]
-; SSE2-NEXT:    pand %xmm0, %xmm2
-; SSE2-NEXT:    movzbl 14(%rdi), %ecx
-; SSE2-NEXT:    movd %ecx, %xmm4
-; SSE2-NEXT:    pslldq {{.*#+}} xmm4 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm4[0,1]
-; SSE2-NEXT:    pandn %xmm4, %xmm0
-; SSE2-NEXT:    por %xmm0, %xmm2
+; SSE2-NEXT:    testl $16384, %eax ## imm = 0x4000
+; SSE2-NEXT:    jne LBB24_29
 ; SSE2-NEXT:  LBB24_30: ## %else41
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB24_32
-; SSE2-NEXT:  ## %bb.31: ## %cond.load43
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm2
-; SSE2-NEXT:    movzbl 15(%rdi), %eax
-; SSE2-NEXT:    movd %eax, %xmm0
-; SSE2-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0]
-; SSE2-NEXT:    por %xmm0, %xmm2
+; SSE2-NEXT:    testl $32768, %eax ## imm = 0x8000
+; SSE2-NEXT:    jne LBB24_31
 ; SSE2-NEXT:  LBB24_32: ## %else44
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpgtb %xmm1, %xmm0
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB24_34
-; SSE2-NEXT:  ## %bb.33: ## %cond.load46
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; SSE2-NEXT:    pand %xmm0, %xmm3
-; SSE2-NEXT:    movzbl 16(%rdi), %ecx
-; SSE2-NEXT:    movd %ecx, %xmm4
-; SSE2-NEXT:    pandn %xmm4, %xmm0
-; SSE2-NEXT:    por %xmm0, %xmm3
+; SSE2-NEXT:    testl $65536, %eax ## imm = 0x10000
+; SSE2-NEXT:    jne LBB24_33
 ; SSE2-NEXT:  LBB24_34: ## %else47
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB24_36
-; SSE2-NEXT:  ## %bb.35: ## %cond.load49
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; SSE2-NEXT:    pand %xmm0, %xmm3
-; SSE2-NEXT:    movzbl 17(%rdi), %eax
-; SSE2-NEXT:    movd %eax, %xmm4
-; SSE2-NEXT:    psllw $8, %xmm4
-; SSE2-NEXT:    pandn %xmm4, %xmm0
-; SSE2-NEXT:    por %xmm0, %xmm3
+; SSE2-NEXT:    testl $131072, %eax ## imm = 0x20000
+; SSE2-NEXT:    jne LBB24_35
 ; SSE2-NEXT:  LBB24_36: ## %else50
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpgtb %xmm1, %xmm0
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    shrl $16, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    je LBB24_38
-; SSE2-NEXT:  ## %bb.37: ## %cond.load52
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; SSE2-NEXT:    pand %xmm0, %xmm3
-; SSE2-NEXT:    movzbl 18(%rdi), %ecx
-; SSE2-NEXT:    movd %ecx, %xmm4
-; SSE2-NEXT:    pslld $16, %xmm4
-; SSE2-NEXT:    pandn %xmm4, %xmm0
-; SSE2-NEXT:    por %xmm0, %xmm3
+; SSE2-NEXT:    testl $262144, %eax ## imm = 0x40000
+; SSE2-NEXT:    jne LBB24_37
 ; SSE2-NEXT:  LBB24_38: ## %else53
-; SSE2-NEXT:    shrl $24, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB24_40
-; SSE2-NEXT:  ## %bb.39: ## %cond.load55
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255]
-; SSE2-NEXT:    pand %xmm0, %xmm3
-; SSE2-NEXT:    movzbl 19(%rdi), %eax
-; SSE2-NEXT:    movd %eax, %xmm4
-; SSE2-NEXT:    pslld $24, %xmm4
-; SSE2-NEXT:    pandn %xmm4, %xmm0
-; SSE2-NEXT:    por %xmm0, %xmm3
+; SSE2-NEXT:    testl $524288, %eax ## imm = 0x80000
+; SSE2-NEXT:    jne LBB24_39
 ; SSE2-NEXT:  LBB24_40: ## %else56
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpgtb %xmm1, %xmm0
-; SSE2-NEXT:    pextrw $2, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB24_42
-; SSE2-NEXT:  ## %bb.41: ## %cond.load58
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255]
-; SSE2-NEXT:    pand %xmm0, %xmm3
-; SSE2-NEXT:    movzbl 20(%rdi), %ecx
-; SSE2-NEXT:    movd %ecx, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[0,0,1,1]
-; SSE2-NEXT:    pandn %xmm4, %xmm0
-; SSE2-NEXT:    por %xmm0, %xmm3
+; SSE2-NEXT:    testl $1048576, %eax ## imm = 0x100000
+; SSE2-NEXT:    jne LBB24_41
 ; SSE2-NEXT:  LBB24_42: ## %else59
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB24_44
-; SSE2-NEXT:  ## %bb.43: ## %cond.load61
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255]
-; SSE2-NEXT:    pand %xmm0, %xmm3
-; SSE2-NEXT:    movzbl 21(%rdi), %eax
-; SSE2-NEXT:    movd %eax, %xmm4
-; SSE2-NEXT:    psllq $40, %xmm4
-; SSE2-NEXT:    pandn %xmm4, %xmm0
-; SSE2-NEXT:    por %xmm0, %xmm3
+; SSE2-NEXT:    testl $2097152, %eax ## imm = 0x200000
+; SSE2-NEXT:    jne LBB24_43
 ; SSE2-NEXT:  LBB24_44: ## %else62
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpgtb %xmm1, %xmm0
-; SSE2-NEXT:    pextrw $3, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB24_46
-; SSE2-NEXT:  ## %bb.45: ## %cond.load64
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255]
-; SSE2-NEXT:    pand %xmm0, %xmm3
-; SSE2-NEXT:    movzbl 22(%rdi), %ecx
-; SSE2-NEXT:    movd %ecx, %xmm4
-; SSE2-NEXT:    psllq $48, %xmm4
-; SSE2-NEXT:    pandn %xmm4, %xmm0
-; SSE2-NEXT:    por %xmm0, %xmm3
+; SSE2-NEXT:    testl $4194304, %eax ## imm = 0x400000
+; SSE2-NEXT:    jne LBB24_45
 ; SSE2-NEXT:  LBB24_46: ## %else65
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB24_48
-; SSE2-NEXT:  ## %bb.47: ## %cond.load67
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255]
-; SSE2-NEXT:    pand %xmm0, %xmm3
-; SSE2-NEXT:    movzbl 23(%rdi), %eax
-; SSE2-NEXT:    movd %eax, %xmm4
-; SSE2-NEXT:    psllq $56, %xmm4
-; SSE2-NEXT:    pandn %xmm4, %xmm0
-; SSE2-NEXT:    por %xmm0, %xmm3
+; SSE2-NEXT:    testl $8388608, %eax ## imm = 0x800000
+; SSE2-NEXT:    jne LBB24_47
 ; SSE2-NEXT:  LBB24_48: ## %else68
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpgtb %xmm1, %xmm0
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB24_50
-; SSE2-NEXT:  ## %bb.49: ## %cond.load70
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255]
-; SSE2-NEXT:    pand %xmm0, %xmm3
-; SSE2-NEXT:    movzbl 24(%rdi), %ecx
-; SSE2-NEXT:    movd %ecx, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[0,1,0,1]
-; SSE2-NEXT:    pandn %xmm4, %xmm0
-; SSE2-NEXT:    por %xmm0, %xmm3
+; SSE2-NEXT:    testl $16777216, %eax ## imm = 0x1000000
+; SSE2-NEXT:    jne LBB24_49
 ; SSE2-NEXT:  LBB24_50: ## %else71
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB24_52
-; SSE2-NEXT:  ## %bb.51: ## %cond.load73
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255]
-; SSE2-NEXT:    pand %xmm0, %xmm3
-; SSE2-NEXT:    movzbl 25(%rdi), %eax
-; SSE2-NEXT:    movd %eax, %xmm4
-; SSE2-NEXT:    pslldq {{.*#+}} xmm4 = zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm4[0,1,2,3,4,5,6]
-; SSE2-NEXT:    pandn %xmm4, %xmm0
-; SSE2-NEXT:    por %xmm0, %xmm3
+; SSE2-NEXT:    testl $33554432, %eax ## imm = 0x2000000
+; SSE2-NEXT:    jne LBB24_51
 ; SSE2-NEXT:  LBB24_52: ## %else74
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpgtb %xmm1, %xmm0
-; SSE2-NEXT:    pextrw $5, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB24_54
-; SSE2-NEXT:  ## %bb.53: ## %cond.load76
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255]
-; SSE2-NEXT:    pand %xmm0, %xmm3
-; SSE2-NEXT:    movzbl 26(%rdi), %ecx
-; SSE2-NEXT:    movd %ecx, %xmm4
-; SSE2-NEXT:    pslldq {{.*#+}} xmm4 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm4[0,1,2,3,4,5]
-; SSE2-NEXT:    pandn %xmm4, %xmm0
-; SSE2-NEXT:    por %xmm0, %xmm3
+; SSE2-NEXT:    testl $67108864, %eax ## imm = 0x4000000
+; SSE2-NEXT:    jne LBB24_53
 ; SSE2-NEXT:  LBB24_54: ## %else77
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB24_56
-; SSE2-NEXT:  ## %bb.55: ## %cond.load79
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255]
-; SSE2-NEXT:    pand %xmm0, %xmm3
-; SSE2-NEXT:    movzbl 27(%rdi), %eax
-; SSE2-NEXT:    movd %eax, %xmm4
-; SSE2-NEXT:    pslldq {{.*#+}} xmm4 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm4[0,1,2,3,4]
-; SSE2-NEXT:    pandn %xmm4, %xmm0
-; SSE2-NEXT:    por %xmm0, %xmm3
+; SSE2-NEXT:    testl $134217728, %eax ## imm = 0x8000000
+; SSE2-NEXT:    jne LBB24_55
 ; SSE2-NEXT:  LBB24_56: ## %else80
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpgtb %xmm1, %xmm0
-; SSE2-NEXT:    pextrw $6, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB24_58
-; SSE2-NEXT:  ## %bb.57: ## %cond.load82
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255]
-; SSE2-NEXT:    pand %xmm0, %xmm3
-; SSE2-NEXT:    movzbl 28(%rdi), %ecx
-; SSE2-NEXT:    movd %ecx, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[0,1,2,0]
-; SSE2-NEXT:    pandn %xmm4, %xmm0
-; SSE2-NEXT:    por %xmm0, %xmm3
+; SSE2-NEXT:    testl $268435456, %eax ## imm = 0x10000000
+; SSE2-NEXT:    jne LBB24_57
 ; SSE2-NEXT:  LBB24_58: ## %else83
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB24_60
-; SSE2-NEXT:  ## %bb.59: ## %cond.load85
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255]
-; SSE2-NEXT:    pand %xmm0, %xmm3
-; SSE2-NEXT:    movzbl 29(%rdi), %eax
-; SSE2-NEXT:    movd %eax, %xmm4
-; SSE2-NEXT:    pslldq {{.*#+}} xmm4 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm4[0,1,2]
-; SSE2-NEXT:    pandn %xmm4, %xmm0
-; SSE2-NEXT:    por %xmm0, %xmm3
+; SSE2-NEXT:    testl $536870912, %eax ## imm = 0x20000000
+; SSE2-NEXT:    jne LBB24_59
 ; SSE2-NEXT:  LBB24_60: ## %else86
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpgtb %xmm1, %xmm0
-; SSE2-NEXT:    pextrw $7, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB24_62
-; SSE2-NEXT:  ## %bb.61: ## %cond.load88
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255]
-; SSE2-NEXT:    pand %xmm0, %xmm3
-; SSE2-NEXT:    movzbl 30(%rdi), %ecx
-; SSE2-NEXT:    movd %ecx, %xmm1
-; SSE2-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1]
-; SSE2-NEXT:    pandn %xmm1, %xmm0
-; SSE2-NEXT:    por %xmm0, %xmm3
+; SSE2-NEXT:    testl $1073741824, %eax ## imm = 0x40000000
+; SSE2-NEXT:    jne LBB24_61
 ; SSE2-NEXT:  LBB24_62: ## %else89
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $-2147483648, %eax ## imm = 0x80000000
 ; SSE2-NEXT:    je LBB24_64
-; SSE2-NEXT:  ## %bb.63: ## %cond.load91
+; SSE2-NEXT:  LBB24_63: ## %cond.load91
 ; SSE2-NEXT:    pand {{.*}}(%rip), %xmm3
 ; SSE2-NEXT:    movzbl 31(%rdi), %eax
 ; SSE2-NEXT:    movd %eax, %xmm0
@@ -5056,1534 +4454,1660 @@ define <32 x i8> @load_v32i8_v32i8(<32 x i8> %trigger, <32 x i8>* %addr, <32 x i
 ; SSE2-NEXT:    movdqa %xmm2, %xmm0
 ; SSE2-NEXT:    movdqa %xmm3, %xmm1
 ; SSE2-NEXT:    retq
-;
-; SSE42-LABEL: load_v32i8_v32i8:
-; SSE42:       ## %bb.0:
-; SSE42-NEXT:    pxor %xmm4, %xmm4
-; SSE42-NEXT:    pcmpgtb %xmm0, %xmm4
-; SSE42-NEXT:    pextrb $0, %xmm4, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB24_2
-; SSE42-NEXT:  ## %bb.1: ## %cond.load
-; SSE42-NEXT:    pinsrb $0, (%rdi), %xmm2
-; SSE42-NEXT:  LBB24_2: ## %else
-; SSE42-NEXT:    pextrb $1, %xmm4, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB24_4
-; SSE42-NEXT:  ## %bb.3: ## %cond.load1
-; SSE42-NEXT:    pinsrb $1, 1(%rdi), %xmm2
-; SSE42-NEXT:  LBB24_4: ## %else2
-; SSE42-NEXT:    pxor %xmm4, %xmm4
-; SSE42-NEXT:    pcmpgtb %xmm0, %xmm4
-; SSE42-NEXT:    pextrb $2, %xmm4, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE2-NEXT:  LBB24_1: ## %cond.load
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; SSE2-NEXT:    pand %xmm0, %xmm2
+; SSE2-NEXT:    movzbl (%rdi), %ecx
+; SSE2-NEXT:    movd %ecx, %xmm1
+; SSE2-NEXT:    pandn %xmm1, %xmm0
+; SSE2-NEXT:    por %xmm0, %xmm2
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    je LBB24_4
+; SSE2-NEXT:  LBB24_3: ## %cond.load1
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; SSE2-NEXT:    pand %xmm0, %xmm2
+; SSE2-NEXT:    movzbl 1(%rdi), %ecx
+; SSE2-NEXT:    movd %ecx, %xmm1
+; SSE2-NEXT:    psllw $8, %xmm1
+; SSE2-NEXT:    pandn %xmm1, %xmm0
+; SSE2-NEXT:    por %xmm0, %xmm2
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    je LBB24_6
+; SSE2-NEXT:  LBB24_5: ## %cond.load4
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; SSE2-NEXT:    pand %xmm0, %xmm2
+; SSE2-NEXT:    movzbl 2(%rdi), %ecx
+; SSE2-NEXT:    movd %ecx, %xmm1
+; SSE2-NEXT:    pslld $16, %xmm1
+; SSE2-NEXT:    pandn %xmm1, %xmm0
+; SSE2-NEXT:    por %xmm0, %xmm2
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    je LBB24_8
+; SSE2-NEXT:  LBB24_7: ## %cond.load7
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255]
+; SSE2-NEXT:    pand %xmm0, %xmm2
+; SSE2-NEXT:    movzbl 3(%rdi), %ecx
+; SSE2-NEXT:    movd %ecx, %xmm1
+; SSE2-NEXT:    pslld $24, %xmm1
+; SSE2-NEXT:    pandn %xmm1, %xmm0
+; SSE2-NEXT:    por %xmm0, %xmm2
+; SSE2-NEXT:    testb $16, %al
+; SSE2-NEXT:    je LBB24_10
+; SSE2-NEXT:  LBB24_9: ## %cond.load10
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255]
+; SSE2-NEXT:    pand %xmm0, %xmm2
+; SSE2-NEXT:    movzbl 4(%rdi), %ecx
+; SSE2-NEXT:    movd %ecx, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,1,1]
+; SSE2-NEXT:    pandn %xmm1, %xmm0
+; SSE2-NEXT:    por %xmm0, %xmm2
+; SSE2-NEXT:    testb $32, %al
+; SSE2-NEXT:    je LBB24_12
+; SSE2-NEXT:  LBB24_11: ## %cond.load13
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255]
+; SSE2-NEXT:    pand %xmm0, %xmm2
+; SSE2-NEXT:    movzbl 5(%rdi), %ecx
+; SSE2-NEXT:    movd %ecx, %xmm1
+; SSE2-NEXT:    psllq $40, %xmm1
+; SSE2-NEXT:    pandn %xmm1, %xmm0
+; SSE2-NEXT:    por %xmm0, %xmm2
+; SSE2-NEXT:    testb $64, %al
+; SSE2-NEXT:    je LBB24_14
+; SSE2-NEXT:  LBB24_13: ## %cond.load16
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255]
+; SSE2-NEXT:    pand %xmm0, %xmm2
+; SSE2-NEXT:    movzbl 6(%rdi), %ecx
+; SSE2-NEXT:    movd %ecx, %xmm1
+; SSE2-NEXT:    psllq $48, %xmm1
+; SSE2-NEXT:    pandn %xmm1, %xmm0
+; SSE2-NEXT:    por %xmm0, %xmm2
+; SSE2-NEXT:    testb $-128, %al
+; SSE2-NEXT:    je LBB24_16
+; SSE2-NEXT:  LBB24_15: ## %cond.load19
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255]
+; SSE2-NEXT:    pand %xmm0, %xmm2
+; SSE2-NEXT:    movzbl 7(%rdi), %ecx
+; SSE2-NEXT:    movd %ecx, %xmm1
+; SSE2-NEXT:    psllq $56, %xmm1
+; SSE2-NEXT:    pandn %xmm1, %xmm0
+; SSE2-NEXT:    por %xmm0, %xmm2
+; SSE2-NEXT:    testl $256, %eax ## imm = 0x100
+; SSE2-NEXT:    je LBB24_18
+; SSE2-NEXT:  LBB24_17: ## %cond.load22
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255]
+; SSE2-NEXT:    pand %xmm0, %xmm2
+; SSE2-NEXT:    movzbl 8(%rdi), %ecx
+; SSE2-NEXT:    movd %ecx, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
+; SSE2-NEXT:    pandn %xmm1, %xmm0
+; SSE2-NEXT:    por %xmm0, %xmm2
+; SSE2-NEXT:    testl $512, %eax ## imm = 0x200
+; SSE2-NEXT:    je LBB24_20
+; SSE2-NEXT:  LBB24_19: ## %cond.load25
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255]
+; SSE2-NEXT:    pand %xmm0, %xmm2
+; SSE2-NEXT:    movzbl 9(%rdi), %ecx
+; SSE2-NEXT:    movd %ecx, %xmm1
+; SSE2-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1,2,3,4,5,6]
+; SSE2-NEXT:    pandn %xmm1, %xmm0
+; SSE2-NEXT:    por %xmm0, %xmm2
+; SSE2-NEXT:    testl $1024, %eax ## imm = 0x400
+; SSE2-NEXT:    je LBB24_22
+; SSE2-NEXT:  LBB24_21: ## %cond.load28
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255]
+; SSE2-NEXT:    pand %xmm0, %xmm2
+; SSE2-NEXT:    movzbl 10(%rdi), %ecx
+; SSE2-NEXT:    movd %ecx, %xmm1
+; SSE2-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1,2,3,4,5]
+; SSE2-NEXT:    pandn %xmm1, %xmm0
+; SSE2-NEXT:    por %xmm0, %xmm2
+; SSE2-NEXT:    testl $2048, %eax ## imm = 0x800
+; SSE2-NEXT:    je LBB24_24
+; SSE2-NEXT:  LBB24_23: ## %cond.load31
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255]
+; SSE2-NEXT:    pand %xmm0, %xmm2
+; SSE2-NEXT:    movzbl 11(%rdi), %ecx
+; SSE2-NEXT:    movd %ecx, %xmm1
+; SSE2-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1,2,3,4]
+; SSE2-NEXT:    pandn %xmm1, %xmm0
+; SSE2-NEXT:    por %xmm0, %xmm2
+; SSE2-NEXT:    testl $4096, %eax ## imm = 0x1000
+; SSE2-NEXT:    je LBB24_26
+; SSE2-NEXT:  LBB24_25: ## %cond.load34
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255]
+; SSE2-NEXT:    pand %xmm0, %xmm2
+; SSE2-NEXT:    movzbl 12(%rdi), %ecx
+; SSE2-NEXT:    movd %ecx, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,1,2,0]
+; SSE2-NEXT:    pandn %xmm1, %xmm0
+; SSE2-NEXT:    por %xmm0, %xmm2
+; SSE2-NEXT:    testl $8192, %eax ## imm = 0x2000
+; SSE2-NEXT:    je LBB24_28
+; SSE2-NEXT:  LBB24_27: ## %cond.load37
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255]
+; SSE2-NEXT:    pand %xmm0, %xmm2
+; SSE2-NEXT:    movzbl 13(%rdi), %ecx
+; SSE2-NEXT:    movd %ecx, %xmm1
+; SSE2-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1,2]
+; SSE2-NEXT:    pandn %xmm1, %xmm0
+; SSE2-NEXT:    por %xmm0, %xmm2
+; SSE2-NEXT:    testl $16384, %eax ## imm = 0x4000
+; SSE2-NEXT:    je LBB24_30
+; SSE2-NEXT:  LBB24_29: ## %cond.load40
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255]
+; SSE2-NEXT:    pand %xmm0, %xmm2
+; SSE2-NEXT:    movzbl 14(%rdi), %ecx
+; SSE2-NEXT:    movd %ecx, %xmm1
+; SSE2-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1]
+; SSE2-NEXT:    pandn %xmm1, %xmm0
+; SSE2-NEXT:    por %xmm0, %xmm2
+; SSE2-NEXT:    testl $32768, %eax ## imm = 0x8000
+; SSE2-NEXT:    je LBB24_32
+; SSE2-NEXT:  LBB24_31: ## %cond.load43
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm2
+; SSE2-NEXT:    movzbl 15(%rdi), %ecx
+; SSE2-NEXT:    movd %ecx, %xmm0
+; SSE2-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0]
+; SSE2-NEXT:    por %xmm0, %xmm2
+; SSE2-NEXT:    testl $65536, %eax ## imm = 0x10000
+; SSE2-NEXT:    je LBB24_34
+; SSE2-NEXT:  LBB24_33: ## %cond.load46
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; SSE2-NEXT:    pand %xmm0, %xmm3
+; SSE2-NEXT:    movzbl 16(%rdi), %ecx
+; SSE2-NEXT:    movd %ecx, %xmm1
+; SSE2-NEXT:    pandn %xmm1, %xmm0
+; SSE2-NEXT:    por %xmm0, %xmm3
+; SSE2-NEXT:    testl $131072, %eax ## imm = 0x20000
+; SSE2-NEXT:    je LBB24_36
+; SSE2-NEXT:  LBB24_35: ## %cond.load49
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; SSE2-NEXT:    pand %xmm0, %xmm3
+; SSE2-NEXT:    movzbl 17(%rdi), %ecx
+; SSE2-NEXT:    movd %ecx, %xmm1
+; SSE2-NEXT:    psllw $8, %xmm1
+; SSE2-NEXT:    pandn %xmm1, %xmm0
+; SSE2-NEXT:    por %xmm0, %xmm3
+; SSE2-NEXT:    testl $262144, %eax ## imm = 0x40000
+; SSE2-NEXT:    je LBB24_38
+; SSE2-NEXT:  LBB24_37: ## %cond.load52
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; SSE2-NEXT:    pand %xmm0, %xmm3
+; SSE2-NEXT:    movzbl 18(%rdi), %ecx
+; SSE2-NEXT:    movd %ecx, %xmm1
+; SSE2-NEXT:    pslld $16, %xmm1
+; SSE2-NEXT:    pandn %xmm1, %xmm0
+; SSE2-NEXT:    por %xmm0, %xmm3
+; SSE2-NEXT:    testl $524288, %eax ## imm = 0x80000
+; SSE2-NEXT:    je LBB24_40
+; SSE2-NEXT:  LBB24_39: ## %cond.load55
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255]
+; SSE2-NEXT:    pand %xmm0, %xmm3
+; SSE2-NEXT:    movzbl 19(%rdi), %ecx
+; SSE2-NEXT:    movd %ecx, %xmm1
+; SSE2-NEXT:    pslld $24, %xmm1
+; SSE2-NEXT:    pandn %xmm1, %xmm0
+; SSE2-NEXT:    por %xmm0, %xmm3
+; SSE2-NEXT:    testl $1048576, %eax ## imm = 0x100000
+; SSE2-NEXT:    je LBB24_42
+; SSE2-NEXT:  LBB24_41: ## %cond.load58
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255]
+; SSE2-NEXT:    pand %xmm0, %xmm3
+; SSE2-NEXT:    movzbl 20(%rdi), %ecx
+; SSE2-NEXT:    movd %ecx, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,1,1]
+; SSE2-NEXT:    pandn %xmm1, %xmm0
+; SSE2-NEXT:    por %xmm0, %xmm3
+; SSE2-NEXT:    testl $2097152, %eax ## imm = 0x200000
+; SSE2-NEXT:    je LBB24_44
+; SSE2-NEXT:  LBB24_43: ## %cond.load61
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255]
+; SSE2-NEXT:    pand %xmm0, %xmm3
+; SSE2-NEXT:    movzbl 21(%rdi), %ecx
+; SSE2-NEXT:    movd %ecx, %xmm1
+; SSE2-NEXT:    psllq $40, %xmm1
+; SSE2-NEXT:    pandn %xmm1, %xmm0
+; SSE2-NEXT:    por %xmm0, %xmm3
+; SSE2-NEXT:    testl $4194304, %eax ## imm = 0x400000
+; SSE2-NEXT:    je LBB24_46
+; SSE2-NEXT:  LBB24_45: ## %cond.load64
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255]
+; SSE2-NEXT:    pand %xmm0, %xmm3
+; SSE2-NEXT:    movzbl 22(%rdi), %ecx
+; SSE2-NEXT:    movd %ecx, %xmm1
+; SSE2-NEXT:    psllq $48, %xmm1
+; SSE2-NEXT:    pandn %xmm1, %xmm0
+; SSE2-NEXT:    por %xmm0, %xmm3
+; SSE2-NEXT:    testl $8388608, %eax ## imm = 0x800000
+; SSE2-NEXT:    je LBB24_48
+; SSE2-NEXT:  LBB24_47: ## %cond.load67
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255]
+; SSE2-NEXT:    pand %xmm0, %xmm3
+; SSE2-NEXT:    movzbl 23(%rdi), %ecx
+; SSE2-NEXT:    movd %ecx, %xmm1
+; SSE2-NEXT:    psllq $56, %xmm1
+; SSE2-NEXT:    pandn %xmm1, %xmm0
+; SSE2-NEXT:    por %xmm0, %xmm3
+; SSE2-NEXT:    testl $16777216, %eax ## imm = 0x1000000
+; SSE2-NEXT:    je LBB24_50
+; SSE2-NEXT:  LBB24_49: ## %cond.load70
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255]
+; SSE2-NEXT:    pand %xmm0, %xmm3
+; SSE2-NEXT:    movzbl 24(%rdi), %ecx
+; SSE2-NEXT:    movd %ecx, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
+; SSE2-NEXT:    pandn %xmm1, %xmm0
+; SSE2-NEXT:    por %xmm0, %xmm3
+; SSE2-NEXT:    testl $33554432, %eax ## imm = 0x2000000
+; SSE2-NEXT:    je LBB24_52
+; SSE2-NEXT:  LBB24_51: ## %cond.load73
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255]
+; SSE2-NEXT:    pand %xmm0, %xmm3
+; SSE2-NEXT:    movzbl 25(%rdi), %ecx
+; SSE2-NEXT:    movd %ecx, %xmm1
+; SSE2-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1,2,3,4,5,6]
+; SSE2-NEXT:    pandn %xmm1, %xmm0
+; SSE2-NEXT:    por %xmm0, %xmm3
+; SSE2-NEXT:    testl $67108864, %eax ## imm = 0x4000000
+; SSE2-NEXT:    je LBB24_54
+; SSE2-NEXT:  LBB24_53: ## %cond.load76
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255]
+; SSE2-NEXT:    pand %xmm0, %xmm3
+; SSE2-NEXT:    movzbl 26(%rdi), %ecx
+; SSE2-NEXT:    movd %ecx, %xmm1
+; SSE2-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1,2,3,4,5]
+; SSE2-NEXT:    pandn %xmm1, %xmm0
+; SSE2-NEXT:    por %xmm0, %xmm3
+; SSE2-NEXT:    testl $134217728, %eax ## imm = 0x8000000
+; SSE2-NEXT:    je LBB24_56
+; SSE2-NEXT:  LBB24_55: ## %cond.load79
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255]
+; SSE2-NEXT:    pand %xmm0, %xmm3
+; SSE2-NEXT:    movzbl 27(%rdi), %ecx
+; SSE2-NEXT:    movd %ecx, %xmm1
+; SSE2-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1,2,3,4]
+; SSE2-NEXT:    pandn %xmm1, %xmm0
+; SSE2-NEXT:    por %xmm0, %xmm3
+; SSE2-NEXT:    testl $268435456, %eax ## imm = 0x10000000
+; SSE2-NEXT:    je LBB24_58
+; SSE2-NEXT:  LBB24_57: ## %cond.load82
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255]
+; SSE2-NEXT:    pand %xmm0, %xmm3
+; SSE2-NEXT:    movzbl 28(%rdi), %ecx
+; SSE2-NEXT:    movd %ecx, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,1,2,0]
+; SSE2-NEXT:    pandn %xmm1, %xmm0
+; SSE2-NEXT:    por %xmm0, %xmm3
+; SSE2-NEXT:    testl $536870912, %eax ## imm = 0x20000000
+; SSE2-NEXT:    je LBB24_60
+; SSE2-NEXT:  LBB24_59: ## %cond.load85
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255]
+; SSE2-NEXT:    pand %xmm0, %xmm3
+; SSE2-NEXT:    movzbl 29(%rdi), %ecx
+; SSE2-NEXT:    movd %ecx, %xmm1
+; SSE2-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1,2]
+; SSE2-NEXT:    pandn %xmm1, %xmm0
+; SSE2-NEXT:    por %xmm0, %xmm3
+; SSE2-NEXT:    testl $1073741824, %eax ## imm = 0x40000000
+; SSE2-NEXT:    je LBB24_62
+; SSE2-NEXT:  LBB24_61: ## %cond.load88
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255]
+; SSE2-NEXT:    pand %xmm0, %xmm3
+; SSE2-NEXT:    movzbl 30(%rdi), %ecx
+; SSE2-NEXT:    movd %ecx, %xmm1
+; SSE2-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1]
+; SSE2-NEXT:    pandn %xmm1, %xmm0
+; SSE2-NEXT:    por %xmm0, %xmm3
+; SSE2-NEXT:    testl $-2147483648, %eax ## imm = 0x80000000
+; SSE2-NEXT:    jne LBB24_63
+; SSE2-NEXT:    jmp LBB24_64
+;
+; SSE42-LABEL: load_v32i8_v32i8:
+; SSE42:       ## %bb.0:
+; SSE42-NEXT:    pmovmskb %xmm0, %ecx
+; SSE42-NEXT:    pmovmskb %xmm1, %eax
+; SSE42-NEXT:    shll $16, %eax
+; SSE42-NEXT:    orl %ecx, %eax
+; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    jne LBB24_1
+; SSE42-NEXT:  ## %bb.2: ## %else
+; SSE42-NEXT:    testb $2, %al
+; SSE42-NEXT:    jne LBB24_3
+; SSE42-NEXT:  LBB24_4: ## %else2
+; SSE42-NEXT:    testb $4, %al
+; SSE42-NEXT:    jne LBB24_5
+; SSE42-NEXT:  LBB24_6: ## %else5
+; SSE42-NEXT:    testb $8, %al
+; SSE42-NEXT:    jne LBB24_7
+; SSE42-NEXT:  LBB24_8: ## %else8
+; SSE42-NEXT:    testb $16, %al
+; SSE42-NEXT:    jne LBB24_9
+; SSE42-NEXT:  LBB24_10: ## %else11
+; SSE42-NEXT:    testb $32, %al
+; SSE42-NEXT:    jne LBB24_11
+; SSE42-NEXT:  LBB24_12: ## %else14
+; SSE42-NEXT:    testb $64, %al
+; SSE42-NEXT:    jne LBB24_13
+; SSE42-NEXT:  LBB24_14: ## %else17
+; SSE42-NEXT:    testb $-128, %al
+; SSE42-NEXT:    jne LBB24_15
+; SSE42-NEXT:  LBB24_16: ## %else20
+; SSE42-NEXT:    testl $256, %eax ## imm = 0x100
+; SSE42-NEXT:    jne LBB24_17
+; SSE42-NEXT:  LBB24_18: ## %else23
+; SSE42-NEXT:    testl $512, %eax ## imm = 0x200
+; SSE42-NEXT:    jne LBB24_19
+; SSE42-NEXT:  LBB24_20: ## %else26
+; SSE42-NEXT:    testl $1024, %eax ## imm = 0x400
+; SSE42-NEXT:    jne LBB24_21
+; SSE42-NEXT:  LBB24_22: ## %else29
+; SSE42-NEXT:    testl $2048, %eax ## imm = 0x800
+; SSE42-NEXT:    jne LBB24_23
+; SSE42-NEXT:  LBB24_24: ## %else32
+; SSE42-NEXT:    testl $4096, %eax ## imm = 0x1000
+; SSE42-NEXT:    jne LBB24_25
+; SSE42-NEXT:  LBB24_26: ## %else35
+; SSE42-NEXT:    testl $8192, %eax ## imm = 0x2000
+; SSE42-NEXT:    jne LBB24_27
+; SSE42-NEXT:  LBB24_28: ## %else38
+; SSE42-NEXT:    testl $16384, %eax ## imm = 0x4000
+; SSE42-NEXT:    jne LBB24_29
+; SSE42-NEXT:  LBB24_30: ## %else41
+; SSE42-NEXT:    testl $32768, %eax ## imm = 0x8000
+; SSE42-NEXT:    jne LBB24_31
+; SSE42-NEXT:  LBB24_32: ## %else44
+; SSE42-NEXT:    testl $65536, %eax ## imm = 0x10000
+; SSE42-NEXT:    jne LBB24_33
+; SSE42-NEXT:  LBB24_34: ## %else47
+; SSE42-NEXT:    testl $131072, %eax ## imm = 0x20000
+; SSE42-NEXT:    jne LBB24_35
+; SSE42-NEXT:  LBB24_36: ## %else50
+; SSE42-NEXT:    testl $262144, %eax ## imm = 0x40000
+; SSE42-NEXT:    jne LBB24_37
+; SSE42-NEXT:  LBB24_38: ## %else53
+; SSE42-NEXT:    testl $524288, %eax ## imm = 0x80000
+; SSE42-NEXT:    jne LBB24_39
+; SSE42-NEXT:  LBB24_40: ## %else56
+; SSE42-NEXT:    testl $1048576, %eax ## imm = 0x100000
+; SSE42-NEXT:    jne LBB24_41
+; SSE42-NEXT:  LBB24_42: ## %else59
+; SSE42-NEXT:    testl $2097152, %eax ## imm = 0x200000
+; SSE42-NEXT:    jne LBB24_43
+; SSE42-NEXT:  LBB24_44: ## %else62
+; SSE42-NEXT:    testl $4194304, %eax ## imm = 0x400000
+; SSE42-NEXT:    jne LBB24_45
+; SSE42-NEXT:  LBB24_46: ## %else65
+; SSE42-NEXT:    testl $8388608, %eax ## imm = 0x800000
+; SSE42-NEXT:    jne LBB24_47
+; SSE42-NEXT:  LBB24_48: ## %else68
+; SSE42-NEXT:    testl $16777216, %eax ## imm = 0x1000000
+; SSE42-NEXT:    jne LBB24_49
+; SSE42-NEXT:  LBB24_50: ## %else71
+; SSE42-NEXT:    testl $33554432, %eax ## imm = 0x2000000
+; SSE42-NEXT:    jne LBB24_51
+; SSE42-NEXT:  LBB24_52: ## %else74
+; SSE42-NEXT:    testl $67108864, %eax ## imm = 0x4000000
+; SSE42-NEXT:    jne LBB24_53
+; SSE42-NEXT:  LBB24_54: ## %else77
+; SSE42-NEXT:    testl $134217728, %eax ## imm = 0x8000000
+; SSE42-NEXT:    jne LBB24_55
+; SSE42-NEXT:  LBB24_56: ## %else80
+; SSE42-NEXT:    testl $268435456, %eax ## imm = 0x10000000
+; SSE42-NEXT:    jne LBB24_57
+; SSE42-NEXT:  LBB24_58: ## %else83
+; SSE42-NEXT:    testl $536870912, %eax ## imm = 0x20000000
+; SSE42-NEXT:    jne LBB24_59
+; SSE42-NEXT:  LBB24_60: ## %else86
+; SSE42-NEXT:    testl $1073741824, %eax ## imm = 0x40000000
+; SSE42-NEXT:    jne LBB24_61
+; SSE42-NEXT:  LBB24_62: ## %else89
+; SSE42-NEXT:    testl $-2147483648, %eax ## imm = 0x80000000
+; SSE42-NEXT:    je LBB24_64
+; SSE42-NEXT:  LBB24_63: ## %cond.load91
+; SSE42-NEXT:    pinsrb $15, 31(%rdi), %xmm3
+; SSE42-NEXT:  LBB24_64: ## %else92
+; SSE42-NEXT:    movdqa %xmm2, %xmm0
+; SSE42-NEXT:    movdqa %xmm3, %xmm1
+; SSE42-NEXT:    retq
+; SSE42-NEXT:  LBB24_1: ## %cond.load
+; SSE42-NEXT:    pinsrb $0, (%rdi), %xmm2
+; SSE42-NEXT:    testb $2, %al
+; SSE42-NEXT:    je LBB24_4
+; SSE42-NEXT:  LBB24_3: ## %cond.load1
+; SSE42-NEXT:    pinsrb $1, 1(%rdi), %xmm2
+; SSE42-NEXT:    testb $4, %al
 ; SSE42-NEXT:    je LBB24_6
-; SSE42-NEXT:  ## %bb.5: ## %cond.load4
+; SSE42-NEXT:  LBB24_5: ## %cond.load4
 ; SSE42-NEXT:    pinsrb $2, 2(%rdi), %xmm2
-; SSE42-NEXT:  LBB24_6: ## %else5
-; SSE42-NEXT:    pextrb $3, %xmm4, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $8, %al
 ; SSE42-NEXT:    je LBB24_8
-; SSE42-NEXT:  ## %bb.7: ## %cond.load7
+; SSE42-NEXT:  LBB24_7: ## %cond.load7
 ; SSE42-NEXT:    pinsrb $3, 3(%rdi), %xmm2
-; SSE42-NEXT:  LBB24_8: ## %else8
-; SSE42-NEXT:    pxor %xmm4, %xmm4
-; SSE42-NEXT:    pcmpgtb %xmm0, %xmm4
-; SSE42-NEXT:    pextrb $4, %xmm4, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $16, %al
 ; SSE42-NEXT:    je LBB24_10
-; SSE42-NEXT:  ## %bb.9: ## %cond.load10
+; SSE42-NEXT:  LBB24_9: ## %cond.load10
 ; SSE42-NEXT:    pinsrb $4, 4(%rdi), %xmm2
-; SSE42-NEXT:  LBB24_10: ## %else11
-; SSE42-NEXT:    pextrb $5, %xmm4, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $32, %al
 ; SSE42-NEXT:    je LBB24_12
-; SSE42-NEXT:  ## %bb.11: ## %cond.load13
+; SSE42-NEXT:  LBB24_11: ## %cond.load13
 ; SSE42-NEXT:    pinsrb $5, 5(%rdi), %xmm2
-; SSE42-NEXT:  LBB24_12: ## %else14
-; SSE42-NEXT:    pxor %xmm4, %xmm4
-; SSE42-NEXT:    pcmpgtb %xmm0, %xmm4
-; SSE42-NEXT:    pextrb $6, %xmm4, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $64, %al
 ; SSE42-NEXT:    je LBB24_14
-; SSE42-NEXT:  ## %bb.13: ## %cond.load16
+; SSE42-NEXT:  LBB24_13: ## %cond.load16
 ; SSE42-NEXT:    pinsrb $6, 6(%rdi), %xmm2
-; SSE42-NEXT:  LBB24_14: ## %else17
-; SSE42-NEXT:    pextrb $7, %xmm4, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testb $-128, %al
 ; SSE42-NEXT:    je LBB24_16
-; SSE42-NEXT:  ## %bb.15: ## %cond.load19
+; SSE42-NEXT:  LBB24_15: ## %cond.load19
 ; SSE42-NEXT:    pinsrb $7, 7(%rdi), %xmm2
-; SSE42-NEXT:  LBB24_16: ## %else20
-; SSE42-NEXT:    pxor %xmm4, %xmm4
-; SSE42-NEXT:    pcmpgtb %xmm0, %xmm4
-; SSE42-NEXT:    pextrb $8, %xmm4, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $256, %eax ## imm = 0x100
 ; SSE42-NEXT:    je LBB24_18
-; SSE42-NEXT:  ## %bb.17: ## %cond.load22
+; SSE42-NEXT:  LBB24_17: ## %cond.load22
 ; SSE42-NEXT:    pinsrb $8, 8(%rdi), %xmm2
-; SSE42-NEXT:  LBB24_18: ## %else23
-; SSE42-NEXT:    pextrb $9, %xmm4, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $512, %eax ## imm = 0x200
 ; SSE42-NEXT:    je LBB24_20
-; SSE42-NEXT:  ## %bb.19: ## %cond.load25
+; SSE42-NEXT:  LBB24_19: ## %cond.load25
 ; SSE42-NEXT:    pinsrb $9, 9(%rdi), %xmm2
-; SSE42-NEXT:  LBB24_20: ## %else26
-; SSE42-NEXT:    pxor %xmm4, %xmm4
-; SSE42-NEXT:    pcmpgtb %xmm0, %xmm4
-; SSE42-NEXT:    pextrb $10, %xmm4, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $1024, %eax ## imm = 0x400
 ; SSE42-NEXT:    je LBB24_22
-; SSE42-NEXT:  ## %bb.21: ## %cond.load28
+; SSE42-NEXT:  LBB24_21: ## %cond.load28
 ; SSE42-NEXT:    pinsrb $10, 10(%rdi), %xmm2
-; SSE42-NEXT:  LBB24_22: ## %else29
-; SSE42-NEXT:    pextrb $11, %xmm4, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $2048, %eax ## imm = 0x800
 ; SSE42-NEXT:    je LBB24_24
-; SSE42-NEXT:  ## %bb.23: ## %cond.load31
+; SSE42-NEXT:  LBB24_23: ## %cond.load31
 ; SSE42-NEXT:    pinsrb $11, 11(%rdi), %xmm2
-; SSE42-NEXT:  LBB24_24: ## %else32
-; SSE42-NEXT:    pxor %xmm4, %xmm4
-; SSE42-NEXT:    pcmpgtb %xmm0, %xmm4
-; SSE42-NEXT:    pextrb $12, %xmm4, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $4096, %eax ## imm = 0x1000
 ; SSE42-NEXT:    je LBB24_26
-; SSE42-NEXT:  ## %bb.25: ## %cond.load34
+; SSE42-NEXT:  LBB24_25: ## %cond.load34
 ; SSE42-NEXT:    pinsrb $12, 12(%rdi), %xmm2
-; SSE42-NEXT:  LBB24_26: ## %else35
-; SSE42-NEXT:    pextrb $13, %xmm4, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $8192, %eax ## imm = 0x2000
 ; SSE42-NEXT:    je LBB24_28
-; SSE42-NEXT:  ## %bb.27: ## %cond.load37
+; SSE42-NEXT:  LBB24_27: ## %cond.load37
 ; SSE42-NEXT:    pinsrb $13, 13(%rdi), %xmm2
-; SSE42-NEXT:  LBB24_28: ## %else38
-; SSE42-NEXT:    pxor %xmm4, %xmm4
-; SSE42-NEXT:    pcmpgtb %xmm0, %xmm4
-; SSE42-NEXT:    pextrb $14, %xmm4, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $16384, %eax ## imm = 0x4000
 ; SSE42-NEXT:    je LBB24_30
-; SSE42-NEXT:  ## %bb.29: ## %cond.load40
+; SSE42-NEXT:  LBB24_29: ## %cond.load40
 ; SSE42-NEXT:    pinsrb $14, 14(%rdi), %xmm2
-; SSE42-NEXT:  LBB24_30: ## %else41
-; SSE42-NEXT:    pextrb $15, %xmm4, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $32768, %eax ## imm = 0x8000
 ; SSE42-NEXT:    je LBB24_32
-; SSE42-NEXT:  ## %bb.31: ## %cond.load43
+; SSE42-NEXT:  LBB24_31: ## %cond.load43
 ; SSE42-NEXT:    pinsrb $15, 15(%rdi), %xmm2
-; SSE42-NEXT:  LBB24_32: ## %else44
-; SSE42-NEXT:    pxor %xmm0, %xmm0
-; SSE42-NEXT:    pcmpgtb %xmm1, %xmm0
-; SSE42-NEXT:    pextrb $0, %xmm0, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $65536, %eax ## imm = 0x10000
 ; SSE42-NEXT:    je LBB24_34
-; SSE42-NEXT:  ## %bb.33: ## %cond.load46
+; SSE42-NEXT:  LBB24_33: ## %cond.load46
 ; SSE42-NEXT:    pinsrb $0, 16(%rdi), %xmm3
-; SSE42-NEXT:  LBB24_34: ## %else47
-; SSE42-NEXT:    pextrb $1, %xmm0, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $131072, %eax ## imm = 0x20000
 ; SSE42-NEXT:    je LBB24_36
-; SSE42-NEXT:  ## %bb.35: ## %cond.load49
+; SSE42-NEXT:  LBB24_35: ## %cond.load49
 ; SSE42-NEXT:    pinsrb $1, 17(%rdi), %xmm3
-; SSE42-NEXT:  LBB24_36: ## %else50
-; SSE42-NEXT:    pxor %xmm0, %xmm0
-; SSE42-NEXT:    pcmpgtb %xmm1, %xmm0
-; SSE42-NEXT:    pextrb $2, %xmm0, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $262144, %eax ## imm = 0x40000
 ; SSE42-NEXT:    je LBB24_38
-; SSE42-NEXT:  ## %bb.37: ## %cond.load52
+; SSE42-NEXT:  LBB24_37: ## %cond.load52
 ; SSE42-NEXT:    pinsrb $2, 18(%rdi), %xmm3
-; SSE42-NEXT:  LBB24_38: ## %else53
-; SSE42-NEXT:    pextrb $3, %xmm0, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $524288, %eax ## imm = 0x80000
 ; SSE42-NEXT:    je LBB24_40
-; SSE42-NEXT:  ## %bb.39: ## %cond.load55
+; SSE42-NEXT:  LBB24_39: ## %cond.load55
 ; SSE42-NEXT:    pinsrb $3, 19(%rdi), %xmm3
-; SSE42-NEXT:  LBB24_40: ## %else56
-; SSE42-NEXT:    pxor %xmm0, %xmm0
-; SSE42-NEXT:    pcmpgtb %xmm1, %xmm0
-; SSE42-NEXT:    pextrb $4, %xmm0, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $1048576, %eax ## imm = 0x100000
 ; SSE42-NEXT:    je LBB24_42
-; SSE42-NEXT:  ## %bb.41: ## %cond.load58
+; SSE42-NEXT:  LBB24_41: ## %cond.load58
 ; SSE42-NEXT:    pinsrb $4, 20(%rdi), %xmm3
-; SSE42-NEXT:  LBB24_42: ## %else59
-; SSE42-NEXT:    pextrb $5, %xmm0, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $2097152, %eax ## imm = 0x200000
 ; SSE42-NEXT:    je LBB24_44
-; SSE42-NEXT:  ## %bb.43: ## %cond.load61
+; SSE42-NEXT:  LBB24_43: ## %cond.load61
 ; SSE42-NEXT:    pinsrb $5, 21(%rdi), %xmm3
-; SSE42-NEXT:  LBB24_44: ## %else62
-; SSE42-NEXT:    pxor %xmm0, %xmm0
-; SSE42-NEXT:    pcmpgtb %xmm1, %xmm0
-; SSE42-NEXT:    pextrb $6, %xmm0, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $4194304, %eax ## imm = 0x400000
 ; SSE42-NEXT:    je LBB24_46
-; SSE42-NEXT:  ## %bb.45: ## %cond.load64
+; SSE42-NEXT:  LBB24_45: ## %cond.load64
 ; SSE42-NEXT:    pinsrb $6, 22(%rdi), %xmm3
-; SSE42-NEXT:  LBB24_46: ## %else65
-; SSE42-NEXT:    pextrb $7, %xmm0, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $8388608, %eax ## imm = 0x800000
 ; SSE42-NEXT:    je LBB24_48
-; SSE42-NEXT:  ## %bb.47: ## %cond.load67
+; SSE42-NEXT:  LBB24_47: ## %cond.load67
 ; SSE42-NEXT:    pinsrb $7, 23(%rdi), %xmm3
-; SSE42-NEXT:  LBB24_48: ## %else68
-; SSE42-NEXT:    pxor %xmm0, %xmm0
-; SSE42-NEXT:    pcmpgtb %xmm1, %xmm0
-; SSE42-NEXT:    pextrb $8, %xmm0, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $16777216, %eax ## imm = 0x1000000
 ; SSE42-NEXT:    je LBB24_50
-; SSE42-NEXT:  ## %bb.49: ## %cond.load70
+; SSE42-NEXT:  LBB24_49: ## %cond.load70
 ; SSE42-NEXT:    pinsrb $8, 24(%rdi), %xmm3
-; SSE42-NEXT:  LBB24_50: ## %else71
-; SSE42-NEXT:    pextrb $9, %xmm0, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $33554432, %eax ## imm = 0x2000000
 ; SSE42-NEXT:    je LBB24_52
-; SSE42-NEXT:  ## %bb.51: ## %cond.load73
+; SSE42-NEXT:  LBB24_51: ## %cond.load73
 ; SSE42-NEXT:    pinsrb $9, 25(%rdi), %xmm3
-; SSE42-NEXT:  LBB24_52: ## %else74
-; SSE42-NEXT:    pxor %xmm0, %xmm0
-; SSE42-NEXT:    pcmpgtb %xmm1, %xmm0
-; SSE42-NEXT:    pextrb $10, %xmm0, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $67108864, %eax ## imm = 0x4000000
 ; SSE42-NEXT:    je LBB24_54
-; SSE42-NEXT:  ## %bb.53: ## %cond.load76
+; SSE42-NEXT:  LBB24_53: ## %cond.load76
 ; SSE42-NEXT:    pinsrb $10, 26(%rdi), %xmm3
-; SSE42-NEXT:  LBB24_54: ## %else77
-; SSE42-NEXT:    pextrb $11, %xmm0, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $134217728, %eax ## imm = 0x8000000
 ; SSE42-NEXT:    je LBB24_56
-; SSE42-NEXT:  ## %bb.55: ## %cond.load79
+; SSE42-NEXT:  LBB24_55: ## %cond.load79
 ; SSE42-NEXT:    pinsrb $11, 27(%rdi), %xmm3
-; SSE42-NEXT:  LBB24_56: ## %else80
-; SSE42-NEXT:    pxor %xmm0, %xmm0
-; SSE42-NEXT:    pcmpgtb %xmm1, %xmm0
-; SSE42-NEXT:    pextrb $12, %xmm0, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $268435456, %eax ## imm = 0x10000000
 ; SSE42-NEXT:    je LBB24_58
-; SSE42-NEXT:  ## %bb.57: ## %cond.load82
+; SSE42-NEXT:  LBB24_57: ## %cond.load82
 ; SSE42-NEXT:    pinsrb $12, 28(%rdi), %xmm3
-; SSE42-NEXT:  LBB24_58: ## %else83
-; SSE42-NEXT:    pextrb $13, %xmm0, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $536870912, %eax ## imm = 0x20000000
 ; SSE42-NEXT:    je LBB24_60
-; SSE42-NEXT:  ## %bb.59: ## %cond.load85
+; SSE42-NEXT:  LBB24_59: ## %cond.load85
 ; SSE42-NEXT:    pinsrb $13, 29(%rdi), %xmm3
-; SSE42-NEXT:  LBB24_60: ## %else86
-; SSE42-NEXT:    pxor %xmm0, %xmm0
-; SSE42-NEXT:    pcmpgtb %xmm1, %xmm0
-; SSE42-NEXT:    pextrb $14, %xmm0, %eax
-; SSE42-NEXT:    testb $1, %al
+; SSE42-NEXT:    testl $1073741824, %eax ## imm = 0x40000000
 ; SSE42-NEXT:    je LBB24_62
-; SSE42-NEXT:  ## %bb.61: ## %cond.load88
+; SSE42-NEXT:  LBB24_61: ## %cond.load88
 ; SSE42-NEXT:    pinsrb $14, 30(%rdi), %xmm3
-; SSE42-NEXT:  LBB24_62: ## %else89
-; SSE42-NEXT:    pextrb $15, %xmm0, %eax
-; SSE42-NEXT:    testb $1, %al
-; SSE42-NEXT:    je LBB24_64
-; SSE42-NEXT:  ## %bb.63: ## %cond.load91
-; SSE42-NEXT:    pinsrb $15, 31(%rdi), %xmm3
-; SSE42-NEXT:  LBB24_64: ## %else92
-; SSE42-NEXT:    movdqa %xmm2, %xmm0
-; SSE42-NEXT:    movdqa %xmm3, %xmm1
-; SSE42-NEXT:    retq
+; SSE42-NEXT:    testl $-2147483648, %eax ## imm = 0x80000000
+; SSE42-NEXT:    jne LBB24_63
+; SSE42-NEXT:    jmp LBB24_64
 ;
 ; AVX1-LABEL: load_v32i8_v32i8:
 ; AVX1:       ## %bb.0:
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm3
-; AVX1-NEXT:    vpextrb $0, %xmm3, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je LBB24_2
-; AVX1-NEXT:  ## %bb.1: ## %cond.load
-; AVX1-NEXT:    vpinsrb $0, (%rdi), %xmm1, %xmm3
-; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
-; AVX1-NEXT:  LBB24_2: ## %else
-; AVX1-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $1, %xmm2, %eax
+; AVX1-NEXT:    vpmovmskb %xmm0, %ecx
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vpmovmskb %xmm0, %eax
+; AVX1-NEXT:    shll $16, %eax
+; AVX1-NEXT:    orl %ecx, %eax
 ; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je LBB24_4
-; AVX1-NEXT:  ## %bb.3: ## %cond.load1
-; AVX1-NEXT:    vpinsrb $1, 1(%rdi), %xmm1, %xmm2
-; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX1-NEXT:    jne LBB24_1
+; AVX1-NEXT:  ## %bb.2: ## %else
+; AVX1-NEXT:    testb $2, %al
+; AVX1-NEXT:    jne LBB24_3
 ; AVX1-NEXT:  LBB24_4: ## %else2
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm3
-; AVX1-NEXT:    vpextrb $2, %xmm3, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je LBB24_6
-; AVX1-NEXT:  ## %bb.5: ## %cond.load4
-; AVX1-NEXT:    vpinsrb $2, 2(%rdi), %xmm1, %xmm3
-; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
+; AVX1-NEXT:    testb $4, %al
+; AVX1-NEXT:    jne LBB24_5
 ; AVX1-NEXT:  LBB24_6: ## %else5
-; AVX1-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $3, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je LBB24_8
-; AVX1-NEXT:  ## %bb.7: ## %cond.load7
-; AVX1-NEXT:    vpinsrb $3, 3(%rdi), %xmm1, %xmm2
-; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX1-NEXT:    testb $8, %al
+; AVX1-NEXT:    jne LBB24_7
 ; AVX1-NEXT:  LBB24_8: ## %else8
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm3
-; AVX1-NEXT:    vpextrb $4, %xmm3, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je LBB24_10
-; AVX1-NEXT:  ## %bb.9: ## %cond.load10
-; AVX1-NEXT:    vpinsrb $4, 4(%rdi), %xmm1, %xmm3
-; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
+; AVX1-NEXT:    testb $16, %al
+; AVX1-NEXT:    jne LBB24_9
 ; AVX1-NEXT:  LBB24_10: ## %else11
-; AVX1-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $5, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je LBB24_12
-; AVX1-NEXT:  ## %bb.11: ## %cond.load13
-; AVX1-NEXT:    vpinsrb $5, 5(%rdi), %xmm1, %xmm2
-; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX1-NEXT:    testb $32, %al
+; AVX1-NEXT:    jne LBB24_11
 ; AVX1-NEXT:  LBB24_12: ## %else14
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm3
-; AVX1-NEXT:    vpextrb $6, %xmm3, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je LBB24_14
-; AVX1-NEXT:  ## %bb.13: ## %cond.load16
-; AVX1-NEXT:    vpinsrb $6, 6(%rdi), %xmm1, %xmm3
-; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
+; AVX1-NEXT:    testb $64, %al
+; AVX1-NEXT:    jne LBB24_13
 ; AVX1-NEXT:  LBB24_14: ## %else17
-; AVX1-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $7, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je LBB24_16
-; AVX1-NEXT:  ## %bb.15: ## %cond.load19
-; AVX1-NEXT:    vpinsrb $7, 7(%rdi), %xmm1, %xmm2
-; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX1-NEXT:    testb $-128, %al
+; AVX1-NEXT:    jne LBB24_15
 ; AVX1-NEXT:  LBB24_16: ## %else20
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm3
-; AVX1-NEXT:    vpextrb $8, %xmm3, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je LBB24_18
-; AVX1-NEXT:  ## %bb.17: ## %cond.load22
-; AVX1-NEXT:    vpinsrb $8, 8(%rdi), %xmm1, %xmm3
-; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
+; AVX1-NEXT:    testl $256, %eax ## imm = 0x100
+; AVX1-NEXT:    jne LBB24_17
 ; AVX1-NEXT:  LBB24_18: ## %else23
-; AVX1-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $9, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je LBB24_20
-; AVX1-NEXT:  ## %bb.19: ## %cond.load25
-; AVX1-NEXT:    vpinsrb $9, 9(%rdi), %xmm1, %xmm2
-; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX1-NEXT:    testl $512, %eax ## imm = 0x200
+; AVX1-NEXT:    jne LBB24_19
 ; AVX1-NEXT:  LBB24_20: ## %else26
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm3
-; AVX1-NEXT:    vpextrb $10, %xmm3, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je LBB24_22
-; AVX1-NEXT:  ## %bb.21: ## %cond.load28
-; AVX1-NEXT:    vpinsrb $10, 10(%rdi), %xmm1, %xmm3
-; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
+; AVX1-NEXT:    testl $1024, %eax ## imm = 0x400
+; AVX1-NEXT:    jne LBB24_21
 ; AVX1-NEXT:  LBB24_22: ## %else29
-; AVX1-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $11, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je LBB24_24
-; AVX1-NEXT:  ## %bb.23: ## %cond.load31
-; AVX1-NEXT:    vpinsrb $11, 11(%rdi), %xmm1, %xmm2
-; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX1-NEXT:    testl $2048, %eax ## imm = 0x800
+; AVX1-NEXT:    jne LBB24_23
 ; AVX1-NEXT:  LBB24_24: ## %else32
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm3
-; AVX1-NEXT:    vpextrb $12, %xmm3, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je LBB24_26
-; AVX1-NEXT:  ## %bb.25: ## %cond.load34
-; AVX1-NEXT:    vpinsrb $12, 12(%rdi), %xmm1, %xmm3
-; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
+; AVX1-NEXT:    testl $4096, %eax ## imm = 0x1000
+; AVX1-NEXT:    jne LBB24_25
 ; AVX1-NEXT:  LBB24_26: ## %else35
-; AVX1-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $13, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je LBB24_28
-; AVX1-NEXT:  ## %bb.27: ## %cond.load37
-; AVX1-NEXT:    vpinsrb $13, 13(%rdi), %xmm1, %xmm2
-; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX1-NEXT:    testl $8192, %eax ## imm = 0x2000
+; AVX1-NEXT:    jne LBB24_27
 ; AVX1-NEXT:  LBB24_28: ## %else38
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm3
-; AVX1-NEXT:    vpextrb $14, %xmm3, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je LBB24_30
-; AVX1-NEXT:  ## %bb.29: ## %cond.load40
-; AVX1-NEXT:    vpinsrb $14, 14(%rdi), %xmm1, %xmm3
-; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
+; AVX1-NEXT:    testl $16384, %eax ## imm = 0x4000
+; AVX1-NEXT:    jne LBB24_29
 ; AVX1-NEXT:  LBB24_30: ## %else41
-; AVX1-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $15, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je LBB24_32
-; AVX1-NEXT:  ## %bb.31: ## %cond.load43
-; AVX1-NEXT:    vpinsrb $15, 15(%rdi), %xmm1, %xmm2
-; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX1-NEXT:    testl $32768, %eax ## imm = 0x8000
+; AVX1-NEXT:    jne LBB24_31
 ; AVX1-NEXT:  LBB24_32: ## %else44
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $0, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je LBB24_34
-; AVX1-NEXT:  ## %bb.33: ## %cond.load46
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
-; AVX1-NEXT:    vpinsrb $0, 16(%rdi), %xmm3, %xmm3
-; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
+; AVX1-NEXT:    testl $65536, %eax ## imm = 0x10000
+; AVX1-NEXT:    jne LBB24_33
 ; AVX1-NEXT:  LBB24_34: ## %else47
-; AVX1-NEXT:    vpextrb $1, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je LBB24_36
-; AVX1-NEXT:  ## %bb.35: ## %cond.load49
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT:    vpinsrb $1, 17(%rdi), %xmm2, %xmm2
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; AVX1-NEXT:    testl $131072, %eax ## imm = 0x20000
+; AVX1-NEXT:    jne LBB24_35
 ; AVX1-NEXT:  LBB24_36: ## %else50
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $2, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je LBB24_38
-; AVX1-NEXT:  ## %bb.37: ## %cond.load52
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
-; AVX1-NEXT:    vpinsrb $2, 18(%rdi), %xmm3, %xmm3
-; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
+; AVX1-NEXT:    testl $262144, %eax ## imm = 0x40000
+; AVX1-NEXT:    jne LBB24_37
 ; AVX1-NEXT:  LBB24_38: ## %else53
-; AVX1-NEXT:    vpextrb $3, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je LBB24_40
-; AVX1-NEXT:  ## %bb.39: ## %cond.load55
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT:    vpinsrb $3, 19(%rdi), %xmm2, %xmm2
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; AVX1-NEXT:    testl $524288, %eax ## imm = 0x80000
+; AVX1-NEXT:    jne LBB24_39
 ; AVX1-NEXT:  LBB24_40: ## %else56
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je LBB24_42
-; AVX1-NEXT:  ## %bb.41: ## %cond.load58
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
-; AVX1-NEXT:    vpinsrb $4, 20(%rdi), %xmm3, %xmm3
-; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
+; AVX1-NEXT:    testl $1048576, %eax ## imm = 0x100000
+; AVX1-NEXT:    jne LBB24_41
 ; AVX1-NEXT:  LBB24_42: ## %else59
-; AVX1-NEXT:    vpextrb $5, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je LBB24_44
-; AVX1-NEXT:  ## %bb.43: ## %cond.load61
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT:    vpinsrb $5, 21(%rdi), %xmm2, %xmm2
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; AVX1-NEXT:    testl $2097152, %eax ## imm = 0x200000
+; AVX1-NEXT:    jne LBB24_43
 ; AVX1-NEXT:  LBB24_44: ## %else62
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $6, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je LBB24_46
-; AVX1-NEXT:  ## %bb.45: ## %cond.load64
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
-; AVX1-NEXT:    vpinsrb $6, 22(%rdi), %xmm3, %xmm3
-; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
+; AVX1-NEXT:    testl $4194304, %eax ## imm = 0x400000
+; AVX1-NEXT:    jne LBB24_45
 ; AVX1-NEXT:  LBB24_46: ## %else65
-; AVX1-NEXT:    vpextrb $7, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $8388608, %eax ## imm = 0x800000
+; AVX1-NEXT:    jne LBB24_47
+; AVX1-NEXT:  LBB24_48: ## %else68
+; AVX1-NEXT:    testl $16777216, %eax ## imm = 0x1000000
+; AVX1-NEXT:    jne LBB24_49
+; AVX1-NEXT:  LBB24_50: ## %else71
+; AVX1-NEXT:    testl $33554432, %eax ## imm = 0x2000000
+; AVX1-NEXT:    jne LBB24_51
+; AVX1-NEXT:  LBB24_52: ## %else74
+; AVX1-NEXT:    testl $67108864, %eax ## imm = 0x4000000
+; AVX1-NEXT:    jne LBB24_53
+; AVX1-NEXT:  LBB24_54: ## %else77
+; AVX1-NEXT:    testl $134217728, %eax ## imm = 0x8000000
+; AVX1-NEXT:    jne LBB24_55
+; AVX1-NEXT:  LBB24_56: ## %else80
+; AVX1-NEXT:    testl $268435456, %eax ## imm = 0x10000000
+; AVX1-NEXT:    jne LBB24_57
+; AVX1-NEXT:  LBB24_58: ## %else83
+; AVX1-NEXT:    testl $536870912, %eax ## imm = 0x20000000
+; AVX1-NEXT:    jne LBB24_59
+; AVX1-NEXT:  LBB24_60: ## %else86
+; AVX1-NEXT:    testl $1073741824, %eax ## imm = 0x40000000
+; AVX1-NEXT:    jne LBB24_61
+; AVX1-NEXT:  LBB24_62: ## %else89
+; AVX1-NEXT:    testl $-2147483648, %eax ## imm = 0x80000000
+; AVX1-NEXT:    jne LBB24_63
+; AVX1-NEXT:  LBB24_64: ## %else92
+; AVX1-NEXT:    vmovaps %ymm1, %ymm0
+; AVX1-NEXT:    retq
+; AVX1-NEXT:  LBB24_1: ## %cond.load
+; AVX1-NEXT:    vpinsrb $0, (%rdi), %xmm1, %xmm0
+; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX1-NEXT:    testb $2, %al
+; AVX1-NEXT:    je LBB24_4
+; AVX1-NEXT:  LBB24_3: ## %cond.load1
+; AVX1-NEXT:    vpinsrb $1, 1(%rdi), %xmm1, %xmm0
+; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX1-NEXT:    testb $4, %al
+; AVX1-NEXT:    je LBB24_6
+; AVX1-NEXT:  LBB24_5: ## %cond.load4
+; AVX1-NEXT:    vpinsrb $2, 2(%rdi), %xmm1, %xmm0
+; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX1-NEXT:    testb $8, %al
+; AVX1-NEXT:    je LBB24_8
+; AVX1-NEXT:  LBB24_7: ## %cond.load7
+; AVX1-NEXT:    vpinsrb $3, 3(%rdi), %xmm1, %xmm0
+; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX1-NEXT:    testb $16, %al
+; AVX1-NEXT:    je LBB24_10
+; AVX1-NEXT:  LBB24_9: ## %cond.load10
+; AVX1-NEXT:    vpinsrb $4, 4(%rdi), %xmm1, %xmm0
+; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX1-NEXT:    testb $32, %al
+; AVX1-NEXT:    je LBB24_12
+; AVX1-NEXT:  LBB24_11: ## %cond.load13
+; AVX1-NEXT:    vpinsrb $5, 5(%rdi), %xmm1, %xmm0
+; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX1-NEXT:    testb $64, %al
+; AVX1-NEXT:    je LBB24_14
+; AVX1-NEXT:  LBB24_13: ## %cond.load16
+; AVX1-NEXT:    vpinsrb $6, 6(%rdi), %xmm1, %xmm0
+; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX1-NEXT:    testb $-128, %al
+; AVX1-NEXT:    je LBB24_16
+; AVX1-NEXT:  LBB24_15: ## %cond.load19
+; AVX1-NEXT:    vpinsrb $7, 7(%rdi), %xmm1, %xmm0
+; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX1-NEXT:    testl $256, %eax ## imm = 0x100
+; AVX1-NEXT:    je LBB24_18
+; AVX1-NEXT:  LBB24_17: ## %cond.load22
+; AVX1-NEXT:    vpinsrb $8, 8(%rdi), %xmm1, %xmm0
+; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX1-NEXT:    testl $512, %eax ## imm = 0x200
+; AVX1-NEXT:    je LBB24_20
+; AVX1-NEXT:  LBB24_19: ## %cond.load25
+; AVX1-NEXT:    vpinsrb $9, 9(%rdi), %xmm1, %xmm0
+; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX1-NEXT:    testl $1024, %eax ## imm = 0x400
+; AVX1-NEXT:    je LBB24_22
+; AVX1-NEXT:  LBB24_21: ## %cond.load28
+; AVX1-NEXT:    vpinsrb $10, 10(%rdi), %xmm1, %xmm0
+; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX1-NEXT:    testl $2048, %eax ## imm = 0x800
+; AVX1-NEXT:    je LBB24_24
+; AVX1-NEXT:  LBB24_23: ## %cond.load31
+; AVX1-NEXT:    vpinsrb $11, 11(%rdi), %xmm1, %xmm0
+; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX1-NEXT:    testl $4096, %eax ## imm = 0x1000
+; AVX1-NEXT:    je LBB24_26
+; AVX1-NEXT:  LBB24_25: ## %cond.load34
+; AVX1-NEXT:    vpinsrb $12, 12(%rdi), %xmm1, %xmm0
+; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX1-NEXT:    testl $8192, %eax ## imm = 0x2000
+; AVX1-NEXT:    je LBB24_28
+; AVX1-NEXT:  LBB24_27: ## %cond.load37
+; AVX1-NEXT:    vpinsrb $13, 13(%rdi), %xmm1, %xmm0
+; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX1-NEXT:    testl $16384, %eax ## imm = 0x4000
+; AVX1-NEXT:    je LBB24_30
+; AVX1-NEXT:  LBB24_29: ## %cond.load40
+; AVX1-NEXT:    vpinsrb $14, 14(%rdi), %xmm1, %xmm0
+; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX1-NEXT:    testl $32768, %eax ## imm = 0x8000
+; AVX1-NEXT:    je LBB24_32
+; AVX1-NEXT:  LBB24_31: ## %cond.load43
+; AVX1-NEXT:    vpinsrb $15, 15(%rdi), %xmm1, %xmm0
+; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX1-NEXT:    testl $65536, %eax ## imm = 0x10000
+; AVX1-NEXT:    je LBB24_34
+; AVX1-NEXT:  LBB24_33: ## %cond.load46
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm0
+; AVX1-NEXT:    vpinsrb $0, 16(%rdi), %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm1
+; AVX1-NEXT:    testl $131072, %eax ## imm = 0x20000
+; AVX1-NEXT:    je LBB24_36
+; AVX1-NEXT:  LBB24_35: ## %cond.load49
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm0
+; AVX1-NEXT:    vpinsrb $1, 17(%rdi), %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm1
+; AVX1-NEXT:    testl $262144, %eax ## imm = 0x40000
+; AVX1-NEXT:    je LBB24_38
+; AVX1-NEXT:  LBB24_37: ## %cond.load52
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm0
+; AVX1-NEXT:    vpinsrb $2, 18(%rdi), %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm1
+; AVX1-NEXT:    testl $524288, %eax ## imm = 0x80000
+; AVX1-NEXT:    je LBB24_40
+; AVX1-NEXT:  LBB24_39: ## %cond.load55
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm0
+; AVX1-NEXT:    vpinsrb $3, 19(%rdi), %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm1
+; AVX1-NEXT:    testl $1048576, %eax ## imm = 0x100000
+; AVX1-NEXT:    je LBB24_42
+; AVX1-NEXT:  LBB24_41: ## %cond.load58
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm0
+; AVX1-NEXT:    vpinsrb $4, 20(%rdi), %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm1
+; AVX1-NEXT:    testl $2097152, %eax ## imm = 0x200000
+; AVX1-NEXT:    je LBB24_44
+; AVX1-NEXT:  LBB24_43: ## %cond.load61
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm0
+; AVX1-NEXT:    vpinsrb $5, 21(%rdi), %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm1
+; AVX1-NEXT:    testl $4194304, %eax ## imm = 0x400000
+; AVX1-NEXT:    je LBB24_46
+; AVX1-NEXT:  LBB24_45: ## %cond.load64
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm0
+; AVX1-NEXT:    vpinsrb $6, 22(%rdi), %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm1
+; AVX1-NEXT:    testl $8388608, %eax ## imm = 0x800000
 ; AVX1-NEXT:    je LBB24_48
-; AVX1-NEXT:  ## %bb.47: ## %cond.load67
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT:    vpinsrb $7, 23(%rdi), %xmm2, %xmm2
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
-; AVX1-NEXT:  LBB24_48: ## %else68
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $8, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:  LBB24_47: ## %cond.load67
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm0
+; AVX1-NEXT:    vpinsrb $7, 23(%rdi), %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm1
+; AVX1-NEXT:    testl $16777216, %eax ## imm = 0x1000000
 ; AVX1-NEXT:    je LBB24_50
-; AVX1-NEXT:  ## %bb.49: ## %cond.load70
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
-; AVX1-NEXT:    vpinsrb $8, 24(%rdi), %xmm3, %xmm3
-; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
-; AVX1-NEXT:  LBB24_50: ## %else71
-; AVX1-NEXT:    vpextrb $9, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:  LBB24_49: ## %cond.load70
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm0
+; AVX1-NEXT:    vpinsrb $8, 24(%rdi), %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm1
+; AVX1-NEXT:    testl $33554432, %eax ## imm = 0x2000000
 ; AVX1-NEXT:    je LBB24_52
-; AVX1-NEXT:  ## %bb.51: ## %cond.load73
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT:    vpinsrb $9, 25(%rdi), %xmm2, %xmm2
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
-; AVX1-NEXT:  LBB24_52: ## %else74
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $10, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:  LBB24_51: ## %cond.load73
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm0
+; AVX1-NEXT:    vpinsrb $9, 25(%rdi), %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm1
+; AVX1-NEXT:    testl $67108864, %eax ## imm = 0x4000000
 ; AVX1-NEXT:    je LBB24_54
-; AVX1-NEXT:  ## %bb.53: ## %cond.load76
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
-; AVX1-NEXT:    vpinsrb $10, 26(%rdi), %xmm3, %xmm3
-; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
-; AVX1-NEXT:  LBB24_54: ## %else77
-; AVX1-NEXT:    vpextrb $11, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:  LBB24_53: ## %cond.load76
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm0
+; AVX1-NEXT:    vpinsrb $10, 26(%rdi), %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm1
+; AVX1-NEXT:    testl $134217728, %eax ## imm = 0x8000000
 ; AVX1-NEXT:    je LBB24_56
-; AVX1-NEXT:  ## %bb.55: ## %cond.load79
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT:    vpinsrb $11, 27(%rdi), %xmm2, %xmm2
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
-; AVX1-NEXT:  LBB24_56: ## %else80
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $12, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:  LBB24_55: ## %cond.load79
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm0
+; AVX1-NEXT:    vpinsrb $11, 27(%rdi), %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm1
+; AVX1-NEXT:    testl $268435456, %eax ## imm = 0x10000000
 ; AVX1-NEXT:    je LBB24_58
-; AVX1-NEXT:  ## %bb.57: ## %cond.load82
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
-; AVX1-NEXT:    vpinsrb $12, 28(%rdi), %xmm3, %xmm3
-; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
-; AVX1-NEXT:  LBB24_58: ## %else83
-; AVX1-NEXT:    vpextrb $13, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:  LBB24_57: ## %cond.load82
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm0
+; AVX1-NEXT:    vpinsrb $12, 28(%rdi), %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm1
+; AVX1-NEXT:    testl $536870912, %eax ## imm = 0x20000000
 ; AVX1-NEXT:    je LBB24_60
-; AVX1-NEXT:  ## %bb.59: ## %cond.load85
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT:    vpinsrb $13, 29(%rdi), %xmm2, %xmm2
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
-; AVX1-NEXT:  LBB24_60: ## %else86
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm0
-; AVX1-NEXT:    vpextrb $14, %xmm0, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:  LBB24_59: ## %cond.load85
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm0
+; AVX1-NEXT:    vpinsrb $13, 29(%rdi), %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm1
+; AVX1-NEXT:    testl $1073741824, %eax ## imm = 0x40000000
 ; AVX1-NEXT:    je LBB24_62
-; AVX1-NEXT:  ## %bb.61: ## %cond.load88
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT:    vpinsrb $14, 30(%rdi), %xmm2, %xmm2
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
-; AVX1-NEXT:  LBB24_62: ## %else89
-; AVX1-NEXT:    vpextrb $15, %xmm0, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:  LBB24_61: ## %cond.load88
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm0
+; AVX1-NEXT:    vpinsrb $14, 30(%rdi), %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm1
+; AVX1-NEXT:    testl $-2147483648, %eax ## imm = 0x80000000
 ; AVX1-NEXT:    je LBB24_64
-; AVX1-NEXT:  ## %bb.63: ## %cond.load91
+; AVX1-NEXT:  LBB24_63: ## %cond.load91
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm0
 ; AVX1-NEXT:    vpinsrb $15, 31(%rdi), %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm1
-; AVX1-NEXT:  LBB24_64: ## %else92
 ; AVX1-NEXT:    vmovaps %ymm1, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: load_v32i8_v32i8:
 ; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm3
-; AVX2-NEXT:    vpextrb $0, %xmm3, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je LBB24_2
-; AVX2-NEXT:  ## %bb.1: ## %cond.load
-; AVX2-NEXT:    vpinsrb $0, (%rdi), %xmm1, %xmm3
-; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
-; AVX2-NEXT:  LBB24_2: ## %else
-; AVX2-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrb $1, %xmm2, %eax
+; AVX2-NEXT:    vpmovmskb %ymm0, %eax
 ; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je LBB24_4
-; AVX2-NEXT:  ## %bb.3: ## %cond.load1
-; AVX2-NEXT:    vpinsrb $1, 1(%rdi), %xmm1, %xmm2
-; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-NEXT:    jne LBB24_1
+; AVX2-NEXT:  ## %bb.2: ## %else
+; AVX2-NEXT:    testb $2, %al
+; AVX2-NEXT:    jne LBB24_3
 ; AVX2-NEXT:  LBB24_4: ## %else2
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm3
-; AVX2-NEXT:    vpextrb $2, %xmm3, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je LBB24_6
-; AVX2-NEXT:  ## %bb.5: ## %cond.load4
-; AVX2-NEXT:    vpinsrb $2, 2(%rdi), %xmm1, %xmm3
-; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-NEXT:    testb $4, %al
+; AVX2-NEXT:    jne LBB24_5
 ; AVX2-NEXT:  LBB24_6: ## %else5
-; AVX2-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrb $3, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je LBB24_8
-; AVX2-NEXT:  ## %bb.7: ## %cond.load7
-; AVX2-NEXT:    vpinsrb $3, 3(%rdi), %xmm1, %xmm2
-; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-NEXT:    testb $8, %al
+; AVX2-NEXT:    jne LBB24_7
 ; AVX2-NEXT:  LBB24_8: ## %else8
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm3
-; AVX2-NEXT:    vpextrb $4, %xmm3, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je LBB24_10
-; AVX2-NEXT:  ## %bb.9: ## %cond.load10
-; AVX2-NEXT:    vpinsrb $4, 4(%rdi), %xmm1, %xmm3
-; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-NEXT:    testb $16, %al
+; AVX2-NEXT:    jne LBB24_9
 ; AVX2-NEXT:  LBB24_10: ## %else11
-; AVX2-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrb $5, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je LBB24_12
-; AVX2-NEXT:  ## %bb.11: ## %cond.load13
-; AVX2-NEXT:    vpinsrb $5, 5(%rdi), %xmm1, %xmm2
-; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-NEXT:    testb $32, %al
+; AVX2-NEXT:    jne LBB24_11
 ; AVX2-NEXT:  LBB24_12: ## %else14
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm3
-; AVX2-NEXT:    vpextrb $6, %xmm3, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je LBB24_14
-; AVX2-NEXT:  ## %bb.13: ## %cond.load16
-; AVX2-NEXT:    vpinsrb $6, 6(%rdi), %xmm1, %xmm3
-; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-NEXT:    testb $64, %al
+; AVX2-NEXT:    jne LBB24_13
 ; AVX2-NEXT:  LBB24_14: ## %else17
-; AVX2-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrb $7, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je LBB24_16
-; AVX2-NEXT:  ## %bb.15: ## %cond.load19
-; AVX2-NEXT:    vpinsrb $7, 7(%rdi), %xmm1, %xmm2
-; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-NEXT:    testb $-128, %al
+; AVX2-NEXT:    jne LBB24_15
 ; AVX2-NEXT:  LBB24_16: ## %else20
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm3
-; AVX2-NEXT:    vpextrb $8, %xmm3, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je LBB24_18
-; AVX2-NEXT:  ## %bb.17: ## %cond.load22
-; AVX2-NEXT:    vpinsrb $8, 8(%rdi), %xmm1, %xmm3
-; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-NEXT:    testl $256, %eax ## imm = 0x100
+; AVX2-NEXT:    jne LBB24_17
 ; AVX2-NEXT:  LBB24_18: ## %else23
-; AVX2-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrb $9, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je LBB24_20
-; AVX2-NEXT:  ## %bb.19: ## %cond.load25
-; AVX2-NEXT:    vpinsrb $9, 9(%rdi), %xmm1, %xmm2
-; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-NEXT:    testl $512, %eax ## imm = 0x200
+; AVX2-NEXT:    jne LBB24_19
 ; AVX2-NEXT:  LBB24_20: ## %else26
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm3
-; AVX2-NEXT:    vpextrb $10, %xmm3, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je LBB24_22
-; AVX2-NEXT:  ## %bb.21: ## %cond.load28
-; AVX2-NEXT:    vpinsrb $10, 10(%rdi), %xmm1, %xmm3
-; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-NEXT:    testl $1024, %eax ## imm = 0x400
+; AVX2-NEXT:    jne LBB24_21
 ; AVX2-NEXT:  LBB24_22: ## %else29
-; AVX2-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrb $11, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je LBB24_24
-; AVX2-NEXT:  ## %bb.23: ## %cond.load31
-; AVX2-NEXT:    vpinsrb $11, 11(%rdi), %xmm1, %xmm2
-; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-NEXT:    testl $2048, %eax ## imm = 0x800
+; AVX2-NEXT:    jne LBB24_23
 ; AVX2-NEXT:  LBB24_24: ## %else32
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm3
-; AVX2-NEXT:    vpextrb $12, %xmm3, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je LBB24_26
-; AVX2-NEXT:  ## %bb.25: ## %cond.load34
-; AVX2-NEXT:    vpinsrb $12, 12(%rdi), %xmm1, %xmm3
-; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-NEXT:    testl $4096, %eax ## imm = 0x1000
+; AVX2-NEXT:    jne LBB24_25
 ; AVX2-NEXT:  LBB24_26: ## %else35
-; AVX2-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrb $13, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je LBB24_28
-; AVX2-NEXT:  ## %bb.27: ## %cond.load37
-; AVX2-NEXT:    vpinsrb $13, 13(%rdi), %xmm1, %xmm2
-; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-NEXT:    testl $8192, %eax ## imm = 0x2000
+; AVX2-NEXT:    jne LBB24_27
 ; AVX2-NEXT:  LBB24_28: ## %else38
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm3
-; AVX2-NEXT:    vpextrb $14, %xmm3, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je LBB24_30
-; AVX2-NEXT:  ## %bb.29: ## %cond.load40
-; AVX2-NEXT:    vpinsrb $14, 14(%rdi), %xmm1, %xmm3
-; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-NEXT:    testl $16384, %eax ## imm = 0x4000
+; AVX2-NEXT:    jne LBB24_29
 ; AVX2-NEXT:  LBB24_30: ## %else41
-; AVX2-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrb $15, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je LBB24_32
-; AVX2-NEXT:  ## %bb.31: ## %cond.load43
-; AVX2-NEXT:    vpinsrb $15, 15(%rdi), %xmm1, %xmm2
-; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-NEXT:    testl $32768, %eax ## imm = 0x8000
+; AVX2-NEXT:    jne LBB24_31
 ; AVX2-NEXT:  LBB24_32: ## %else44
-; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrb $0, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je LBB24_34
-; AVX2-NEXT:  ## %bb.33: ## %cond.load46
-; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm3
-; AVX2-NEXT:    vpinsrb $0, 16(%rdi), %xmm3, %xmm3
-; AVX2-NEXT:    vinserti128 $1, %xmm3, %ymm1, %ymm1
+; AVX2-NEXT:    testl $65536, %eax ## imm = 0x10000
+; AVX2-NEXT:    jne LBB24_33
 ; AVX2-NEXT:  LBB24_34: ## %else47
-; AVX2-NEXT:    vpextrb $1, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je LBB24_36
-; AVX2-NEXT:  ## %bb.35: ## %cond.load49
-; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX2-NEXT:    vpinsrb $1, 17(%rdi), %xmm2, %xmm2
-; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX2-NEXT:    testl $131072, %eax ## imm = 0x20000
+; AVX2-NEXT:    jne LBB24_35
 ; AVX2-NEXT:  LBB24_36: ## %else50
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrb $2, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je LBB24_38
-; AVX2-NEXT:  ## %bb.37: ## %cond.load52
-; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm3
-; AVX2-NEXT:    vpinsrb $2, 18(%rdi), %xmm3, %xmm3
-; AVX2-NEXT:    vinserti128 $1, %xmm3, %ymm1, %ymm1
+; AVX2-NEXT:    testl $262144, %eax ## imm = 0x40000
+; AVX2-NEXT:    jne LBB24_37
 ; AVX2-NEXT:  LBB24_38: ## %else53
-; AVX2-NEXT:    vpextrb $3, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je LBB24_40
-; AVX2-NEXT:  ## %bb.39: ## %cond.load55
-; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX2-NEXT:    vpinsrb $3, 19(%rdi), %xmm2, %xmm2
-; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX2-NEXT:    testl $524288, %eax ## imm = 0x80000
+; AVX2-NEXT:    jne LBB24_39
 ; AVX2-NEXT:  LBB24_40: ## %else56
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je LBB24_42
-; AVX2-NEXT:  ## %bb.41: ## %cond.load58
-; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm3
-; AVX2-NEXT:    vpinsrb $4, 20(%rdi), %xmm3, %xmm3
-; AVX2-NEXT:    vinserti128 $1, %xmm3, %ymm1, %ymm1
+; AVX2-NEXT:    testl $1048576, %eax ## imm = 0x100000
+; AVX2-NEXT:    jne LBB24_41
 ; AVX2-NEXT:  LBB24_42: ## %else59
-; AVX2-NEXT:    vpextrb $5, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je LBB24_44
-; AVX2-NEXT:  ## %bb.43: ## %cond.load61
-; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX2-NEXT:    vpinsrb $5, 21(%rdi), %xmm2, %xmm2
-; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX2-NEXT:    testl $2097152, %eax ## imm = 0x200000
+; AVX2-NEXT:    jne LBB24_43
 ; AVX2-NEXT:  LBB24_44: ## %else62
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrb $6, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je LBB24_46
-; AVX2-NEXT:  ## %bb.45: ## %cond.load64
-; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm3
-; AVX2-NEXT:    vpinsrb $6, 22(%rdi), %xmm3, %xmm3
-; AVX2-NEXT:    vinserti128 $1, %xmm3, %ymm1, %ymm1
+; AVX2-NEXT:    testl $4194304, %eax ## imm = 0x400000
+; AVX2-NEXT:    jne LBB24_45
 ; AVX2-NEXT:  LBB24_46: ## %else65
-; AVX2-NEXT:    vpextrb $7, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je LBB24_48
-; AVX2-NEXT:  ## %bb.47: ## %cond.load67
-; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX2-NEXT:    vpinsrb $7, 23(%rdi), %xmm2, %xmm2
-; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX2-NEXT:    testl $8388608, %eax ## imm = 0x800000
+; AVX2-NEXT:    jne LBB24_47
 ; AVX2-NEXT:  LBB24_48: ## %else68
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrb $8, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je LBB24_50
-; AVX2-NEXT:  ## %bb.49: ## %cond.load70
-; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm3
-; AVX2-NEXT:    vpinsrb $8, 24(%rdi), %xmm3, %xmm3
-; AVX2-NEXT:    vinserti128 $1, %xmm3, %ymm1, %ymm1
+; AVX2-NEXT:    testl $16777216, %eax ## imm = 0x1000000
+; AVX2-NEXT:    jne LBB24_49
 ; AVX2-NEXT:  LBB24_50: ## %else71
-; AVX2-NEXT:    vpextrb $9, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je LBB24_52
-; AVX2-NEXT:  ## %bb.51: ## %cond.load73
-; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX2-NEXT:    vpinsrb $9, 25(%rdi), %xmm2, %xmm2
-; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX2-NEXT:    testl $33554432, %eax ## imm = 0x2000000
+; AVX2-NEXT:    jne LBB24_51
 ; AVX2-NEXT:  LBB24_52: ## %else74
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrb $10, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je LBB24_54
-; AVX2-NEXT:  ## %bb.53: ## %cond.load76
-; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm3
-; AVX2-NEXT:    vpinsrb $10, 26(%rdi), %xmm3, %xmm3
-; AVX2-NEXT:    vinserti128 $1, %xmm3, %ymm1, %ymm1
+; AVX2-NEXT:    testl $67108864, %eax ## imm = 0x4000000
+; AVX2-NEXT:    jne LBB24_53
 ; AVX2-NEXT:  LBB24_54: ## %else77
-; AVX2-NEXT:    vpextrb $11, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je LBB24_56
-; AVX2-NEXT:  ## %bb.55: ## %cond.load79
-; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX2-NEXT:    vpinsrb $11, 27(%rdi), %xmm2, %xmm2
-; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX2-NEXT:    testl $134217728, %eax ## imm = 0x8000000
+; AVX2-NEXT:    jne LBB24_55
 ; AVX2-NEXT:  LBB24_56: ## %else80
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrb $12, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je LBB24_58
-; AVX2-NEXT:  ## %bb.57: ## %cond.load82
-; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm3
-; AVX2-NEXT:    vpinsrb $12, 28(%rdi), %xmm3, %xmm3
-; AVX2-NEXT:    vinserti128 $1, %xmm3, %ymm1, %ymm1
+; AVX2-NEXT:    testl $268435456, %eax ## imm = 0x10000000
+; AVX2-NEXT:    jne LBB24_57
 ; AVX2-NEXT:  LBB24_58: ## %else83
-; AVX2-NEXT:    vpextrb $13, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je LBB24_60
-; AVX2-NEXT:  ## %bb.59: ## %cond.load85
-; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX2-NEXT:    vpinsrb $13, 29(%rdi), %xmm2, %xmm2
-; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX2-NEXT:    testl $536870912, %eax ## imm = 0x20000000
+; AVX2-NEXT:    jne LBB24_59
 ; AVX2-NEXT:  LBB24_60: ## %else86
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm0
-; AVX2-NEXT:    vpextrb $14, %xmm0, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je LBB24_62
-; AVX2-NEXT:  ## %bb.61: ## %cond.load88
-; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX2-NEXT:    vpinsrb $14, 30(%rdi), %xmm2, %xmm2
-; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX2-NEXT:    testl $1073741824, %eax ## imm = 0x40000000
+; AVX2-NEXT:    jne LBB24_61
 ; AVX2-NEXT:  LBB24_62: ## %else89
-; AVX2-NEXT:    vpextrb $15, %xmm0, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $-2147483648, %eax ## imm = 0x80000000
+; AVX2-NEXT:    jne LBB24_63
+; AVX2-NEXT:  LBB24_64: ## %else92
+; AVX2-NEXT:    vmovdqa %ymm1, %ymm0
+; AVX2-NEXT:    retq
+; AVX2-NEXT:  LBB24_1: ## %cond.load
+; AVX2-NEXT:    vpinsrb $0, (%rdi), %xmm1, %xmm0
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-NEXT:    testb $2, %al
+; AVX2-NEXT:    je LBB24_4
+; AVX2-NEXT:  LBB24_3: ## %cond.load1
+; AVX2-NEXT:    vpinsrb $1, 1(%rdi), %xmm1, %xmm0
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-NEXT:    testb $4, %al
+; AVX2-NEXT:    je LBB24_6
+; AVX2-NEXT:  LBB24_5: ## %cond.load4
+; AVX2-NEXT:    vpinsrb $2, 2(%rdi), %xmm1, %xmm0
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-NEXT:    testb $8, %al
+; AVX2-NEXT:    je LBB24_8
+; AVX2-NEXT:  LBB24_7: ## %cond.load7
+; AVX2-NEXT:    vpinsrb $3, 3(%rdi), %xmm1, %xmm0
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-NEXT:    testb $16, %al
+; AVX2-NEXT:    je LBB24_10
+; AVX2-NEXT:  LBB24_9: ## %cond.load10
+; AVX2-NEXT:    vpinsrb $4, 4(%rdi), %xmm1, %xmm0
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-NEXT:    testb $32, %al
+; AVX2-NEXT:    je LBB24_12
+; AVX2-NEXT:  LBB24_11: ## %cond.load13
+; AVX2-NEXT:    vpinsrb $5, 5(%rdi), %xmm1, %xmm0
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-NEXT:    testb $64, %al
+; AVX2-NEXT:    je LBB24_14
+; AVX2-NEXT:  LBB24_13: ## %cond.load16
+; AVX2-NEXT:    vpinsrb $6, 6(%rdi), %xmm1, %xmm0
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-NEXT:    testb $-128, %al
+; AVX2-NEXT:    je LBB24_16
+; AVX2-NEXT:  LBB24_15: ## %cond.load19
+; AVX2-NEXT:    vpinsrb $7, 7(%rdi), %xmm1, %xmm0
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-NEXT:    testl $256, %eax ## imm = 0x100
+; AVX2-NEXT:    je LBB24_18
+; AVX2-NEXT:  LBB24_17: ## %cond.load22
+; AVX2-NEXT:    vpinsrb $8, 8(%rdi), %xmm1, %xmm0
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-NEXT:    testl $512, %eax ## imm = 0x200
+; AVX2-NEXT:    je LBB24_20
+; AVX2-NEXT:  LBB24_19: ## %cond.load25
+; AVX2-NEXT:    vpinsrb $9, 9(%rdi), %xmm1, %xmm0
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-NEXT:    testl $1024, %eax ## imm = 0x400
+; AVX2-NEXT:    je LBB24_22
+; AVX2-NEXT:  LBB24_21: ## %cond.load28
+; AVX2-NEXT:    vpinsrb $10, 10(%rdi), %xmm1, %xmm0
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-NEXT:    testl $2048, %eax ## imm = 0x800
+; AVX2-NEXT:    je LBB24_24
+; AVX2-NEXT:  LBB24_23: ## %cond.load31
+; AVX2-NEXT:    vpinsrb $11, 11(%rdi), %xmm1, %xmm0
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-NEXT:    testl $4096, %eax ## imm = 0x1000
+; AVX2-NEXT:    je LBB24_26
+; AVX2-NEXT:  LBB24_25: ## %cond.load34
+; AVX2-NEXT:    vpinsrb $12, 12(%rdi), %xmm1, %xmm0
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-NEXT:    testl $8192, %eax ## imm = 0x2000
+; AVX2-NEXT:    je LBB24_28
+; AVX2-NEXT:  LBB24_27: ## %cond.load37
+; AVX2-NEXT:    vpinsrb $13, 13(%rdi), %xmm1, %xmm0
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-NEXT:    testl $16384, %eax ## imm = 0x4000
+; AVX2-NEXT:    je LBB24_30
+; AVX2-NEXT:  LBB24_29: ## %cond.load40
+; AVX2-NEXT:    vpinsrb $14, 14(%rdi), %xmm1, %xmm0
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-NEXT:    testl $32768, %eax ## imm = 0x8000
+; AVX2-NEXT:    je LBB24_32
+; AVX2-NEXT:  LBB24_31: ## %cond.load43
+; AVX2-NEXT:    vpinsrb $15, 15(%rdi), %xmm1, %xmm0
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-NEXT:    testl $65536, %eax ## imm = 0x10000
+; AVX2-NEXT:    je LBB24_34
+; AVX2-NEXT:  LBB24_33: ## %cond.load46
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX2-NEXT:    vpinsrb $0, 16(%rdi), %xmm0, %xmm0
+; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
+; AVX2-NEXT:    testl $131072, %eax ## imm = 0x20000
+; AVX2-NEXT:    je LBB24_36
+; AVX2-NEXT:  LBB24_35: ## %cond.load49
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX2-NEXT:    vpinsrb $1, 17(%rdi), %xmm0, %xmm0
+; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
+; AVX2-NEXT:    testl $262144, %eax ## imm = 0x40000
+; AVX2-NEXT:    je LBB24_38
+; AVX2-NEXT:  LBB24_37: ## %cond.load52
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX2-NEXT:    vpinsrb $2, 18(%rdi), %xmm0, %xmm0
+; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
+; AVX2-NEXT:    testl $524288, %eax ## imm = 0x80000
+; AVX2-NEXT:    je LBB24_40
+; AVX2-NEXT:  LBB24_39: ## %cond.load55
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX2-NEXT:    vpinsrb $3, 19(%rdi), %xmm0, %xmm0
+; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
+; AVX2-NEXT:    testl $1048576, %eax ## imm = 0x100000
+; AVX2-NEXT:    je LBB24_42
+; AVX2-NEXT:  LBB24_41: ## %cond.load58
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX2-NEXT:    vpinsrb $4, 20(%rdi), %xmm0, %xmm0
+; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
+; AVX2-NEXT:    testl $2097152, %eax ## imm = 0x200000
+; AVX2-NEXT:    je LBB24_44
+; AVX2-NEXT:  LBB24_43: ## %cond.load61
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX2-NEXT:    vpinsrb $5, 21(%rdi), %xmm0, %xmm0
+; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
+; AVX2-NEXT:    testl $4194304, %eax ## imm = 0x400000
+; AVX2-NEXT:    je LBB24_46
+; AVX2-NEXT:  LBB24_45: ## %cond.load64
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX2-NEXT:    vpinsrb $6, 22(%rdi), %xmm0, %xmm0
+; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
+; AVX2-NEXT:    testl $8388608, %eax ## imm = 0x800000
+; AVX2-NEXT:    je LBB24_48
+; AVX2-NEXT:  LBB24_47: ## %cond.load67
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX2-NEXT:    vpinsrb $7, 23(%rdi), %xmm0, %xmm0
+; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
+; AVX2-NEXT:    testl $16777216, %eax ## imm = 0x1000000
+; AVX2-NEXT:    je LBB24_50
+; AVX2-NEXT:  LBB24_49: ## %cond.load70
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX2-NEXT:    vpinsrb $8, 24(%rdi), %xmm0, %xmm0
+; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
+; AVX2-NEXT:    testl $33554432, %eax ## imm = 0x2000000
+; AVX2-NEXT:    je LBB24_52
+; AVX2-NEXT:  LBB24_51: ## %cond.load73
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX2-NEXT:    vpinsrb $9, 25(%rdi), %xmm0, %xmm0
+; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
+; AVX2-NEXT:    testl $67108864, %eax ## imm = 0x4000000
+; AVX2-NEXT:    je LBB24_54
+; AVX2-NEXT:  LBB24_53: ## %cond.load76
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX2-NEXT:    vpinsrb $10, 26(%rdi), %xmm0, %xmm0
+; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
+; AVX2-NEXT:    testl $134217728, %eax ## imm = 0x8000000
+; AVX2-NEXT:    je LBB24_56
+; AVX2-NEXT:  LBB24_55: ## %cond.load79
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX2-NEXT:    vpinsrb $11, 27(%rdi), %xmm0, %xmm0
+; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
+; AVX2-NEXT:    testl $268435456, %eax ## imm = 0x10000000
+; AVX2-NEXT:    je LBB24_58
+; AVX2-NEXT:  LBB24_57: ## %cond.load82
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX2-NEXT:    vpinsrb $12, 28(%rdi), %xmm0, %xmm0
+; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
+; AVX2-NEXT:    testl $536870912, %eax ## imm = 0x20000000
+; AVX2-NEXT:    je LBB24_60
+; AVX2-NEXT:  LBB24_59: ## %cond.load85
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX2-NEXT:    vpinsrb $13, 29(%rdi), %xmm0, %xmm0
+; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
+; AVX2-NEXT:    testl $1073741824, %eax ## imm = 0x40000000
+; AVX2-NEXT:    je LBB24_62
+; AVX2-NEXT:  LBB24_61: ## %cond.load88
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX2-NEXT:    vpinsrb $14, 30(%rdi), %xmm0, %xmm0
+; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
+; AVX2-NEXT:    testl $-2147483648, %eax ## imm = 0x80000000
 ; AVX2-NEXT:    je LBB24_64
-; AVX2-NEXT:  ## %bb.63: ## %cond.load91
+; AVX2-NEXT:  LBB24_63: ## %cond.load91
 ; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm0
 ; AVX2-NEXT:    vpinsrb $15, 31(%rdi), %xmm0, %xmm0
 ; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
-; AVX2-NEXT:  LBB24_64: ## %else92
 ; AVX2-NEXT:    vmovdqa %ymm1, %ymm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: load_v32i8_v32i8:
 ; AVX512F:       ## %bb.0:
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je LBB24_2
-; AVX512F-NEXT:  ## %bb.1: ## %cond.load
-; AVX512F-NEXT:    vpinsrb $0, (%rdi), %xmm1, %xmm3
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
-; AVX512F-NEXT:  LBB24_2: ## %else
-; AVX512F-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    vpmovmskb %ymm0, %eax
 ; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je LBB24_4
-; AVX512F-NEXT:  ## %bb.3: ## %cond.load1
-; AVX512F-NEXT:    vpinsrb $1, 1(%rdi), %xmm1, %xmm2
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    jne LBB24_1
+; AVX512F-NEXT:  ## %bb.2: ## %else
+; AVX512F-NEXT:    testb $2, %al
+; AVX512F-NEXT:    jne LBB24_3
 ; AVX512F-NEXT:  LBB24_4: ## %else2
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je LBB24_6
-; AVX512F-NEXT:  ## %bb.5: ## %cond.load4
-; AVX512F-NEXT:    vpinsrb $2, 2(%rdi), %xmm1, %xmm3
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    testb $4, %al
+; AVX512F-NEXT:    jne LBB24_5
 ; AVX512F-NEXT:  LBB24_6: ## %else5
-; AVX512F-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je LBB24_8
-; AVX512F-NEXT:  ## %bb.7: ## %cond.load7
-; AVX512F-NEXT:    vpinsrb $3, 3(%rdi), %xmm1, %xmm2
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    testb $8, %al
+; AVX512F-NEXT:    jne LBB24_7
 ; AVX512F-NEXT:  LBB24_8: ## %else8
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $4, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je LBB24_10
-; AVX512F-NEXT:  ## %bb.9: ## %cond.load10
-; AVX512F-NEXT:    vpinsrb $4, 4(%rdi), %xmm1, %xmm3
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    testb $16, %al
+; AVX512F-NEXT:    jne LBB24_9
 ; AVX512F-NEXT:  LBB24_10: ## %else11
-; AVX512F-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $5, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je LBB24_12
-; AVX512F-NEXT:  ## %bb.11: ## %cond.load13
-; AVX512F-NEXT:    vpinsrb $5, 5(%rdi), %xmm1, %xmm2
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    testb $32, %al
+; AVX512F-NEXT:    jne LBB24_11
 ; AVX512F-NEXT:  LBB24_12: ## %else14
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $6, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je LBB24_14
-; AVX512F-NEXT:  ## %bb.13: ## %cond.load16
-; AVX512F-NEXT:    vpinsrb $6, 6(%rdi), %xmm1, %xmm3
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    testb $64, %al
+; AVX512F-NEXT:    jne LBB24_13
 ; AVX512F-NEXT:  LBB24_14: ## %else17
-; AVX512F-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $7, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je LBB24_16
-; AVX512F-NEXT:  ## %bb.15: ## %cond.load19
-; AVX512F-NEXT:    vpinsrb $7, 7(%rdi), %xmm1, %xmm2
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    testb $-128, %al
+; AVX512F-NEXT:    jne LBB24_15
 ; AVX512F-NEXT:  LBB24_16: ## %else20
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $8, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je LBB24_18
-; AVX512F-NEXT:  ## %bb.17: ## %cond.load22
-; AVX512F-NEXT:    vpinsrb $8, 8(%rdi), %xmm1, %xmm3
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    testl $256, %eax ## imm = 0x100
+; AVX512F-NEXT:    jne LBB24_17
 ; AVX512F-NEXT:  LBB24_18: ## %else23
-; AVX512F-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $9, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je LBB24_20
-; AVX512F-NEXT:  ## %bb.19: ## %cond.load25
-; AVX512F-NEXT:    vpinsrb $9, 9(%rdi), %xmm1, %xmm2
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    testl $512, %eax ## imm = 0x200
+; AVX512F-NEXT:    jne LBB24_19
 ; AVX512F-NEXT:  LBB24_20: ## %else26
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $10, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je LBB24_22
-; AVX512F-NEXT:  ## %bb.21: ## %cond.load28
-; AVX512F-NEXT:    vpinsrb $10, 10(%rdi), %xmm1, %xmm3
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    testl $1024, %eax ## imm = 0x400
+; AVX512F-NEXT:    jne LBB24_21
 ; AVX512F-NEXT:  LBB24_22: ## %else29
-; AVX512F-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $11, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je LBB24_24
-; AVX512F-NEXT:  ## %bb.23: ## %cond.load31
-; AVX512F-NEXT:    vpinsrb $11, 11(%rdi), %xmm1, %xmm2
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    testl $2048, %eax ## imm = 0x800
+; AVX512F-NEXT:    jne LBB24_23
 ; AVX512F-NEXT:  LBB24_24: ## %else32
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $12, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je LBB24_26
-; AVX512F-NEXT:  ## %bb.25: ## %cond.load34
-; AVX512F-NEXT:    vpinsrb $12, 12(%rdi), %xmm1, %xmm3
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    testl $4096, %eax ## imm = 0x1000
+; AVX512F-NEXT:    jne LBB24_25
 ; AVX512F-NEXT:  LBB24_26: ## %else35
-; AVX512F-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $13, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $8192, %eax ## imm = 0x2000
+; AVX512F-NEXT:    jne LBB24_27
+; AVX512F-NEXT:  LBB24_28: ## %else38
+; AVX512F-NEXT:    testl $16384, %eax ## imm = 0x4000
+; AVX512F-NEXT:    jne LBB24_29
+; AVX512F-NEXT:  LBB24_30: ## %else41
+; AVX512F-NEXT:    testl $32768, %eax ## imm = 0x8000
+; AVX512F-NEXT:    jne LBB24_31
+; AVX512F-NEXT:  LBB24_32: ## %else44
+; AVX512F-NEXT:    testl $65536, %eax ## imm = 0x10000
+; AVX512F-NEXT:    jne LBB24_33
+; AVX512F-NEXT:  LBB24_34: ## %else47
+; AVX512F-NEXT:    testl $131072, %eax ## imm = 0x20000
+; AVX512F-NEXT:    jne LBB24_35
+; AVX512F-NEXT:  LBB24_36: ## %else50
+; AVX512F-NEXT:    testl $262144, %eax ## imm = 0x40000
+; AVX512F-NEXT:    jne LBB24_37
+; AVX512F-NEXT:  LBB24_38: ## %else53
+; AVX512F-NEXT:    testl $524288, %eax ## imm = 0x80000
+; AVX512F-NEXT:    jne LBB24_39
+; AVX512F-NEXT:  LBB24_40: ## %else56
+; AVX512F-NEXT:    testl $1048576, %eax ## imm = 0x100000
+; AVX512F-NEXT:    jne LBB24_41
+; AVX512F-NEXT:  LBB24_42: ## %else59
+; AVX512F-NEXT:    testl $2097152, %eax ## imm = 0x200000
+; AVX512F-NEXT:    jne LBB24_43
+; AVX512F-NEXT:  LBB24_44: ## %else62
+; AVX512F-NEXT:    testl $4194304, %eax ## imm = 0x400000
+; AVX512F-NEXT:    jne LBB24_45
+; AVX512F-NEXT:  LBB24_46: ## %else65
+; AVX512F-NEXT:    testl $8388608, %eax ## imm = 0x800000
+; AVX512F-NEXT:    jne LBB24_47
+; AVX512F-NEXT:  LBB24_48: ## %else68
+; AVX512F-NEXT:    testl $16777216, %eax ## imm = 0x1000000
+; AVX512F-NEXT:    jne LBB24_49
+; AVX512F-NEXT:  LBB24_50: ## %else71
+; AVX512F-NEXT:    testl $33554432, %eax ## imm = 0x2000000
+; AVX512F-NEXT:    jne LBB24_51
+; AVX512F-NEXT:  LBB24_52: ## %else74
+; AVX512F-NEXT:    testl $67108864, %eax ## imm = 0x4000000
+; AVX512F-NEXT:    jne LBB24_53
+; AVX512F-NEXT:  LBB24_54: ## %else77
+; AVX512F-NEXT:    testl $134217728, %eax ## imm = 0x8000000
+; AVX512F-NEXT:    jne LBB24_55
+; AVX512F-NEXT:  LBB24_56: ## %else80
+; AVX512F-NEXT:    testl $268435456, %eax ## imm = 0x10000000
+; AVX512F-NEXT:    jne LBB24_57
+; AVX512F-NEXT:  LBB24_58: ## %else83
+; AVX512F-NEXT:    testl $536870912, %eax ## imm = 0x20000000
+; AVX512F-NEXT:    jne LBB24_59
+; AVX512F-NEXT:  LBB24_60: ## %else86
+; AVX512F-NEXT:    testl $1073741824, %eax ## imm = 0x40000000
+; AVX512F-NEXT:    jne LBB24_61
+; AVX512F-NEXT:  LBB24_62: ## %else89
+; AVX512F-NEXT:    testl $-2147483648, %eax ## imm = 0x80000000
+; AVX512F-NEXT:    jne LBB24_63
+; AVX512F-NEXT:  LBB24_64: ## %else92
+; AVX512F-NEXT:    vmovdqa %ymm1, %ymm0
+; AVX512F-NEXT:    retq
+; AVX512F-NEXT:  LBB24_1: ## %cond.load
+; AVX512F-NEXT:    vpinsrb $0, (%rdi), %xmm1, %xmm0
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    testb $2, %al
+; AVX512F-NEXT:    je LBB24_4
+; AVX512F-NEXT:  LBB24_3: ## %cond.load1
+; AVX512F-NEXT:    vpinsrb $1, 1(%rdi), %xmm1, %xmm0
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    testb $4, %al
+; AVX512F-NEXT:    je LBB24_6
+; AVX512F-NEXT:  LBB24_5: ## %cond.load4
+; AVX512F-NEXT:    vpinsrb $2, 2(%rdi), %xmm1, %xmm0
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    testb $8, %al
+; AVX512F-NEXT:    je LBB24_8
+; AVX512F-NEXT:  LBB24_7: ## %cond.load7
+; AVX512F-NEXT:    vpinsrb $3, 3(%rdi), %xmm1, %xmm0
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    testb $16, %al
+; AVX512F-NEXT:    je LBB24_10
+; AVX512F-NEXT:  LBB24_9: ## %cond.load10
+; AVX512F-NEXT:    vpinsrb $4, 4(%rdi), %xmm1, %xmm0
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    testb $32, %al
+; AVX512F-NEXT:    je LBB24_12
+; AVX512F-NEXT:  LBB24_11: ## %cond.load13
+; AVX512F-NEXT:    vpinsrb $5, 5(%rdi), %xmm1, %xmm0
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    testb $64, %al
+; AVX512F-NEXT:    je LBB24_14
+; AVX512F-NEXT:  LBB24_13: ## %cond.load16
+; AVX512F-NEXT:    vpinsrb $6, 6(%rdi), %xmm1, %xmm0
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    testb $-128, %al
+; AVX512F-NEXT:    je LBB24_16
+; AVX512F-NEXT:  LBB24_15: ## %cond.load19
+; AVX512F-NEXT:    vpinsrb $7, 7(%rdi), %xmm1, %xmm0
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    testl $256, %eax ## imm = 0x100
+; AVX512F-NEXT:    je LBB24_18
+; AVX512F-NEXT:  LBB24_17: ## %cond.load22
+; AVX512F-NEXT:    vpinsrb $8, 8(%rdi), %xmm1, %xmm0
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    testl $512, %eax ## imm = 0x200
+; AVX512F-NEXT:    je LBB24_20
+; AVX512F-NEXT:  LBB24_19: ## %cond.load25
+; AVX512F-NEXT:    vpinsrb $9, 9(%rdi), %xmm1, %xmm0
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    testl $1024, %eax ## imm = 0x400
+; AVX512F-NEXT:    je LBB24_22
+; AVX512F-NEXT:  LBB24_21: ## %cond.load28
+; AVX512F-NEXT:    vpinsrb $10, 10(%rdi), %xmm1, %xmm0
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    testl $2048, %eax ## imm = 0x800
+; AVX512F-NEXT:    je LBB24_24
+; AVX512F-NEXT:  LBB24_23: ## %cond.load31
+; AVX512F-NEXT:    vpinsrb $11, 11(%rdi), %xmm1, %xmm0
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    testl $4096, %eax ## imm = 0x1000
+; AVX512F-NEXT:    je LBB24_26
+; AVX512F-NEXT:  LBB24_25: ## %cond.load34
+; AVX512F-NEXT:    vpinsrb $12, 12(%rdi), %xmm1, %xmm0
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    testl $8192, %eax ## imm = 0x2000
 ; AVX512F-NEXT:    je LBB24_28
-; AVX512F-NEXT:  ## %bb.27: ## %cond.load37
-; AVX512F-NEXT:    vpinsrb $13, 13(%rdi), %xmm1, %xmm2
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; AVX512F-NEXT:  LBB24_28: ## %else38
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $14, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  LBB24_27: ## %cond.load37
+; AVX512F-NEXT:    vpinsrb $13, 13(%rdi), %xmm1, %xmm0
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    testl $16384, %eax ## imm = 0x4000
 ; AVX512F-NEXT:    je LBB24_30
-; AVX512F-NEXT:  ## %bb.29: ## %cond.load40
-; AVX512F-NEXT:    vpinsrb $14, 14(%rdi), %xmm1, %xmm3
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
-; AVX512F-NEXT:  LBB24_30: ## %else41
-; AVX512F-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $15, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  LBB24_29: ## %cond.load40
+; AVX512F-NEXT:    vpinsrb $14, 14(%rdi), %xmm1, %xmm0
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    testl $32768, %eax ## imm = 0x8000
 ; AVX512F-NEXT:    je LBB24_32
-; AVX512F-NEXT:  ## %bb.31: ## %cond.load43
-; AVX512F-NEXT:    vpinsrb $15, 15(%rdi), %xmm1, %xmm2
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; AVX512F-NEXT:  LBB24_32: ## %else44
-; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  LBB24_31: ## %cond.load43
+; AVX512F-NEXT:    vpinsrb $15, 15(%rdi), %xmm1, %xmm0
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    testl $65536, %eax ## imm = 0x10000
 ; AVX512F-NEXT:    je LBB24_34
-; AVX512F-NEXT:  ## %bb.33: ## %cond.load46
-; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX512F-NEXT:    vpinsrb $0, 16(%rdi), %xmm2, %xmm2
-; AVX512F-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
-; AVX512F-NEXT:  LBB24_34: ## %else47
-; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  LBB24_33: ## %cond.load46
+; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX512F-NEXT:    vpinsrb $0, 16(%rdi), %xmm0, %xmm0
+; AVX512F-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
+; AVX512F-NEXT:    testl $131072, %eax ## imm = 0x20000
 ; AVX512F-NEXT:    je LBB24_36
-; AVX512F-NEXT:  ## %bb.35: ## %cond.load49
-; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX512F-NEXT:    vpinsrb $1, 17(%rdi), %xmm2, %xmm2
-; AVX512F-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
-; AVX512F-NEXT:  LBB24_36: ## %else50
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $2, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  LBB24_35: ## %cond.load49
+; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX512F-NEXT:    vpinsrb $1, 17(%rdi), %xmm0, %xmm0
+; AVX512F-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
+; AVX512F-NEXT:    testl $262144, %eax ## imm = 0x40000
 ; AVX512F-NEXT:    je LBB24_38
-; AVX512F-NEXT:  ## %bb.37: ## %cond.load52
-; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX512F-NEXT:    vpinsrb $2, 18(%rdi), %xmm2, %xmm2
-; AVX512F-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
-; AVX512F-NEXT:  LBB24_38: ## %else53
-; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  LBB24_37: ## %cond.load52
+; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX512F-NEXT:    vpinsrb $2, 18(%rdi), %xmm0, %xmm0
+; AVX512F-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
+; AVX512F-NEXT:    testl $524288, %eax ## imm = 0x80000
 ; AVX512F-NEXT:    je LBB24_40
-; AVX512F-NEXT:  ## %bb.39: ## %cond.load55
-; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX512F-NEXT:    vpinsrb $3, 19(%rdi), %xmm2, %xmm2
-; AVX512F-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
-; AVX512F-NEXT:  LBB24_40: ## %else56
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $4, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  LBB24_39: ## %cond.load55
+; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX512F-NEXT:    vpinsrb $3, 19(%rdi), %xmm0, %xmm0
+; AVX512F-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
+; AVX512F-NEXT:    testl $1048576, %eax ## imm = 0x100000
 ; AVX512F-NEXT:    je LBB24_42
-; AVX512F-NEXT:  ## %bb.41: ## %cond.load58
-; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX512F-NEXT:    vpinsrb $4, 20(%rdi), %xmm2, %xmm2
-; AVX512F-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
-; AVX512F-NEXT:  LBB24_42: ## %else59
-; AVX512F-NEXT:    kshiftrw $5, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  LBB24_41: ## %cond.load58
+; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX512F-NEXT:    vpinsrb $4, 20(%rdi), %xmm0, %xmm0
+; AVX512F-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
+; AVX512F-NEXT:    testl $2097152, %eax ## imm = 0x200000
 ; AVX512F-NEXT:    je LBB24_44
-; AVX512F-NEXT:  ## %bb.43: ## %cond.load61
-; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX512F-NEXT:    vpinsrb $5, 21(%rdi), %xmm2, %xmm2
-; AVX512F-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
-; AVX512F-NEXT:  LBB24_44: ## %else62
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $6, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  LBB24_43: ## %cond.load61
+; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX512F-NEXT:    vpinsrb $5, 21(%rdi), %xmm0, %xmm0
+; AVX512F-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
+; AVX512F-NEXT:    testl $4194304, %eax ## imm = 0x400000
 ; AVX512F-NEXT:    je LBB24_46
-; AVX512F-NEXT:  ## %bb.45: ## %cond.load64
-; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX512F-NEXT:    vpinsrb $6, 22(%rdi), %xmm2, %xmm2
-; AVX512F-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
-; AVX512F-NEXT:  LBB24_46: ## %else65
-; AVX512F-NEXT:    kshiftrw $7, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  LBB24_45: ## %cond.load64
+; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX512F-NEXT:    vpinsrb $6, 22(%rdi), %xmm0, %xmm0
+; AVX512F-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
+; AVX512F-NEXT:    testl $8388608, %eax ## imm = 0x800000
 ; AVX512F-NEXT:    je LBB24_48
-; AVX512F-NEXT:  ## %bb.47: ## %cond.load67
-; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX512F-NEXT:    vpinsrb $7, 23(%rdi), %xmm2, %xmm2
-; AVX512F-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
-; AVX512F-NEXT:  LBB24_48: ## %else68
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $8, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  LBB24_47: ## %cond.load67
+; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX512F-NEXT:    vpinsrb $7, 23(%rdi), %xmm0, %xmm0
+; AVX512F-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
+; AVX512F-NEXT:    testl $16777216, %eax ## imm = 0x1000000
 ; AVX512F-NEXT:    je LBB24_50
-; AVX512F-NEXT:  ## %bb.49: ## %cond.load70
-; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX512F-NEXT:    vpinsrb $8, 24(%rdi), %xmm2, %xmm2
-; AVX512F-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
-; AVX512F-NEXT:  LBB24_50: ## %else71
-; AVX512F-NEXT:    kshiftrw $9, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  LBB24_49: ## %cond.load70
+; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX512F-NEXT:    vpinsrb $8, 24(%rdi), %xmm0, %xmm0
+; AVX512F-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
+; AVX512F-NEXT:    testl $33554432, %eax ## imm = 0x2000000
 ; AVX512F-NEXT:    je LBB24_52
-; AVX512F-NEXT:  ## %bb.51: ## %cond.load73
-; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX512F-NEXT:    vpinsrb $9, 25(%rdi), %xmm2, %xmm2
-; AVX512F-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
-; AVX512F-NEXT:  LBB24_52: ## %else74
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $10, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  LBB24_51: ## %cond.load73
+; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX512F-NEXT:    vpinsrb $9, 25(%rdi), %xmm0, %xmm0
+; AVX512F-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
+; AVX512F-NEXT:    testl $67108864, %eax ## imm = 0x4000000
 ; AVX512F-NEXT:    je LBB24_54
-; AVX512F-NEXT:  ## %bb.53: ## %cond.load76
-; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX512F-NEXT:    vpinsrb $10, 26(%rdi), %xmm2, %xmm2
-; AVX512F-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
-; AVX512F-NEXT:  LBB24_54: ## %else77
-; AVX512F-NEXT:    kshiftrw $11, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  LBB24_53: ## %cond.load76
+; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX512F-NEXT:    vpinsrb $10, 26(%rdi), %xmm0, %xmm0
+; AVX512F-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
+; AVX512F-NEXT:    testl $134217728, %eax ## imm = 0x8000000
 ; AVX512F-NEXT:    je LBB24_56
-; AVX512F-NEXT:  ## %bb.55: ## %cond.load79
-; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX512F-NEXT:    vpinsrb $11, 27(%rdi), %xmm2, %xmm2
-; AVX512F-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
-; AVX512F-NEXT:  LBB24_56: ## %else80
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $12, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  LBB24_55: ## %cond.load79
+; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX512F-NEXT:    vpinsrb $11, 27(%rdi), %xmm0, %xmm0
+; AVX512F-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
+; AVX512F-NEXT:    testl $268435456, %eax ## imm = 0x10000000
 ; AVX512F-NEXT:    je LBB24_58
-; AVX512F-NEXT:  ## %bb.57: ## %cond.load82
-; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX512F-NEXT:    vpinsrb $12, 28(%rdi), %xmm2, %xmm2
-; AVX512F-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
-; AVX512F-NEXT:  LBB24_58: ## %else83
-; AVX512F-NEXT:    kshiftrw $13, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  LBB24_57: ## %cond.load82
+; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX512F-NEXT:    vpinsrb $12, 28(%rdi), %xmm0, %xmm0
+; AVX512F-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
+; AVX512F-NEXT:    testl $536870912, %eax ## imm = 0x20000000
 ; AVX512F-NEXT:    je LBB24_60
-; AVX512F-NEXT:  ## %bb.59: ## %cond.load85
-; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX512F-NEXT:    vpinsrb $13, 29(%rdi), %xmm2, %xmm2
-; AVX512F-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
-; AVX512F-NEXT:  LBB24_60: ## %else86
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm0
-; AVX512F-NEXT:    vpmovsxbd %xmm0, %zmm0
-; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k0
-; AVX512F-NEXT:    kshiftrw $14, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  LBB24_59: ## %cond.load85
+; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX512F-NEXT:    vpinsrb $13, 29(%rdi), %xmm0, %xmm0
+; AVX512F-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
+; AVX512F-NEXT:    testl $1073741824, %eax ## imm = 0x40000000
 ; AVX512F-NEXT:    je LBB24_62
-; AVX512F-NEXT:  ## %bb.61: ## %cond.load88
+; AVX512F-NEXT:  LBB24_61: ## %cond.load88
 ; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm0
 ; AVX512F-NEXT:    vpinsrb $14, 30(%rdi), %xmm0, %xmm0
 ; AVX512F-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
-; AVX512F-NEXT:  LBB24_62: ## %else89
-; AVX512F-NEXT:    kshiftrw $15, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $-2147483648, %eax ## imm = 0x80000000
 ; AVX512F-NEXT:    je LBB24_64
-; AVX512F-NEXT:  ## %bb.63: ## %cond.load91
+; AVX512F-NEXT:  LBB24_63: ## %cond.load91
 ; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm0
 ; AVX512F-NEXT:    vpinsrb $15, 31(%rdi), %xmm0, %xmm0
 ; AVX512F-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
-; AVX512F-NEXT:  LBB24_64: ## %else92
 ; AVX512F-NEXT:    vmovdqa %ymm1, %ymm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VLDQ-LABEL: load_v32i8_v32i8:
 ; AVX512VLDQ:       ## %bb.0:
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm3
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm3, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
-; AVX512VLDQ-NEXT:    je LBB24_2
-; AVX512VLDQ-NEXT:  ## %bb.1: ## %cond.load
-; AVX512VLDQ-NEXT:    vpinsrb $0, (%rdi), %xmm1, %xmm3
-; AVX512VLDQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
-; AVX512VLDQ-NEXT:  LBB24_2: ## %else
-; AVX512VLDQ-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm2, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $1, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
+; AVX512VLDQ-NEXT:    vpmovmskb %ymm0, %eax
 ; AVX512VLDQ-NEXT:    testb $1, %al
-; AVX512VLDQ-NEXT:    je LBB24_4
-; AVX512VLDQ-NEXT:  ## %bb.3: ## %cond.load1
-; AVX512VLDQ-NEXT:    vpinsrb $1, 1(%rdi), %xmm1, %xmm2
-; AVX512VLDQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512VLDQ-NEXT:    jne LBB24_1
+; AVX512VLDQ-NEXT:  ## %bb.2: ## %else
+; AVX512VLDQ-NEXT:    testb $2, %al
+; AVX512VLDQ-NEXT:    jne LBB24_3
 ; AVX512VLDQ-NEXT:  LBB24_4: ## %else2
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm3
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm3, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $2, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
-; AVX512VLDQ-NEXT:    je LBB24_6
-; AVX512VLDQ-NEXT:  ## %bb.5: ## %cond.load4
-; AVX512VLDQ-NEXT:    vpinsrb $2, 2(%rdi), %xmm1, %xmm3
-; AVX512VLDQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
+; AVX512VLDQ-NEXT:    testb $4, %al
+; AVX512VLDQ-NEXT:    jne LBB24_5
 ; AVX512VLDQ-NEXT:  LBB24_6: ## %else5
-; AVX512VLDQ-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm2, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $3, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
-; AVX512VLDQ-NEXT:    je LBB24_8
-; AVX512VLDQ-NEXT:  ## %bb.7: ## %cond.load7
-; AVX512VLDQ-NEXT:    vpinsrb $3, 3(%rdi), %xmm1, %xmm2
-; AVX512VLDQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512VLDQ-NEXT:    testb $8, %al
+; AVX512VLDQ-NEXT:    jne LBB24_7
 ; AVX512VLDQ-NEXT:  LBB24_8: ## %else8
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm3
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm3, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $4, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
-; AVX512VLDQ-NEXT:    je LBB24_10
-; AVX512VLDQ-NEXT:  ## %bb.9: ## %cond.load10
-; AVX512VLDQ-NEXT:    vpinsrb $4, 4(%rdi), %xmm1, %xmm3
-; AVX512VLDQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
+; AVX512VLDQ-NEXT:    testb $16, %al
+; AVX512VLDQ-NEXT:    jne LBB24_9
 ; AVX512VLDQ-NEXT:  LBB24_10: ## %else11
-; AVX512VLDQ-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm2, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $5, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
-; AVX512VLDQ-NEXT:    je LBB24_12
-; AVX512VLDQ-NEXT:  ## %bb.11: ## %cond.load13
-; AVX512VLDQ-NEXT:    vpinsrb $5, 5(%rdi), %xmm1, %xmm2
-; AVX512VLDQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512VLDQ-NEXT:    testb $32, %al
+; AVX512VLDQ-NEXT:    jne LBB24_11
 ; AVX512VLDQ-NEXT:  LBB24_12: ## %else14
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm3
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm3, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $6, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
-; AVX512VLDQ-NEXT:    je LBB24_14
-; AVX512VLDQ-NEXT:  ## %bb.13: ## %cond.load16
-; AVX512VLDQ-NEXT:    vpinsrb $6, 6(%rdi), %xmm1, %xmm3
-; AVX512VLDQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
+; AVX512VLDQ-NEXT:    testb $64, %al
+; AVX512VLDQ-NEXT:    jne LBB24_13
 ; AVX512VLDQ-NEXT:  LBB24_14: ## %else17
-; AVX512VLDQ-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm2, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $7, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
-; AVX512VLDQ-NEXT:    je LBB24_16
-; AVX512VLDQ-NEXT:  ## %bb.15: ## %cond.load19
-; AVX512VLDQ-NEXT:    vpinsrb $7, 7(%rdi), %xmm1, %xmm2
-; AVX512VLDQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512VLDQ-NEXT:    testb $-128, %al
+; AVX512VLDQ-NEXT:    jne LBB24_15
 ; AVX512VLDQ-NEXT:  LBB24_16: ## %else20
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm3
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm3, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $8, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
-; AVX512VLDQ-NEXT:    je LBB24_18
-; AVX512VLDQ-NEXT:  ## %bb.17: ## %cond.load22
-; AVX512VLDQ-NEXT:    vpinsrb $8, 8(%rdi), %xmm1, %xmm3
-; AVX512VLDQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
+; AVX512VLDQ-NEXT:    testl $256, %eax ## imm = 0x100
+; AVX512VLDQ-NEXT:    jne LBB24_17
 ; AVX512VLDQ-NEXT:  LBB24_18: ## %else23
-; AVX512VLDQ-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm2, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $9, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
-; AVX512VLDQ-NEXT:    je LBB24_20
-; AVX512VLDQ-NEXT:  ## %bb.19: ## %cond.load25
-; AVX512VLDQ-NEXT:    vpinsrb $9, 9(%rdi), %xmm1, %xmm2
-; AVX512VLDQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512VLDQ-NEXT:    testl $512, %eax ## imm = 0x200
+; AVX512VLDQ-NEXT:    jne LBB24_19
 ; AVX512VLDQ-NEXT:  LBB24_20: ## %else26
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm3
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm3, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $10, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
-; AVX512VLDQ-NEXT:    je LBB24_22
-; AVX512VLDQ-NEXT:  ## %bb.21: ## %cond.load28
-; AVX512VLDQ-NEXT:    vpinsrb $10, 10(%rdi), %xmm1, %xmm3
-; AVX512VLDQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
+; AVX512VLDQ-NEXT:    testl $1024, %eax ## imm = 0x400
+; AVX512VLDQ-NEXT:    jne LBB24_21
 ; AVX512VLDQ-NEXT:  LBB24_22: ## %else29
-; AVX512VLDQ-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm2, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $11, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
-; AVX512VLDQ-NEXT:    je LBB24_24
-; AVX512VLDQ-NEXT:  ## %bb.23: ## %cond.load31
-; AVX512VLDQ-NEXT:    vpinsrb $11, 11(%rdi), %xmm1, %xmm2
-; AVX512VLDQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512VLDQ-NEXT:    testl $2048, %eax ## imm = 0x800
+; AVX512VLDQ-NEXT:    jne LBB24_23
 ; AVX512VLDQ-NEXT:  LBB24_24: ## %else32
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm3
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm3, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $12, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
-; AVX512VLDQ-NEXT:    je LBB24_26
-; AVX512VLDQ-NEXT:  ## %bb.25: ## %cond.load34
-; AVX512VLDQ-NEXT:    vpinsrb $12, 12(%rdi), %xmm1, %xmm3
-; AVX512VLDQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
+; AVX512VLDQ-NEXT:    testl $4096, %eax ## imm = 0x1000
+; AVX512VLDQ-NEXT:    jne LBB24_25
 ; AVX512VLDQ-NEXT:  LBB24_26: ## %else35
-; AVX512VLDQ-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm2, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $13, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
-; AVX512VLDQ-NEXT:    je LBB24_28
-; AVX512VLDQ-NEXT:  ## %bb.27: ## %cond.load37
-; AVX512VLDQ-NEXT:    vpinsrb $13, 13(%rdi), %xmm1, %xmm2
-; AVX512VLDQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512VLDQ-NEXT:    testl $8192, %eax ## imm = 0x2000
+; AVX512VLDQ-NEXT:    jne LBB24_27
 ; AVX512VLDQ-NEXT:  LBB24_28: ## %else38
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm3
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm3, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $14, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
-; AVX512VLDQ-NEXT:    je LBB24_30
-; AVX512VLDQ-NEXT:  ## %bb.29: ## %cond.load40
-; AVX512VLDQ-NEXT:    vpinsrb $14, 14(%rdi), %xmm1, %xmm3
-; AVX512VLDQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
+; AVX512VLDQ-NEXT:    testl $16384, %eax ## imm = 0x4000
+; AVX512VLDQ-NEXT:    jne LBB24_29
 ; AVX512VLDQ-NEXT:  LBB24_30: ## %else41
-; AVX512VLDQ-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm2, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $15, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
-; AVX512VLDQ-NEXT:    je LBB24_32
-; AVX512VLDQ-NEXT:  ## %bb.31: ## %cond.load43
-; AVX512VLDQ-NEXT:    vpinsrb $15, 15(%rdi), %xmm1, %xmm2
-; AVX512VLDQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512VLDQ-NEXT:    testl $32768, %eax ## imm = 0x8000
+; AVX512VLDQ-NEXT:    jne LBB24_31
 ; AVX512VLDQ-NEXT:  LBB24_32: ## %else44
-; AVX512VLDQ-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm2, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
-; AVX512VLDQ-NEXT:    je LBB24_34
-; AVX512VLDQ-NEXT:  ## %bb.33: ## %cond.load46
-; AVX512VLDQ-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX512VLDQ-NEXT:    vpinsrb $0, 16(%rdi), %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512VLDQ-NEXT:    testl $65536, %eax ## imm = 0x10000
+; AVX512VLDQ-NEXT:    jne LBB24_33
 ; AVX512VLDQ-NEXT:  LBB24_34: ## %else47
-; AVX512VLDQ-NEXT:    kshiftrw $1, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
-; AVX512VLDQ-NEXT:    je LBB24_36
-; AVX512VLDQ-NEXT:  ## %bb.35: ## %cond.load49
-; AVX512VLDQ-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX512VLDQ-NEXT:    vpinsrb $1, 17(%rdi), %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512VLDQ-NEXT:    testl $131072, %eax ## imm = 0x20000
+; AVX512VLDQ-NEXT:    jne LBB24_35
 ; AVX512VLDQ-NEXT:  LBB24_36: ## %else50
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm2, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $2, %k0, %k1
-; AVX512VLDQ-NEXT:    kmovw %k1, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
-; AVX512VLDQ-NEXT:    je LBB24_38
-; AVX512VLDQ-NEXT:  ## %bb.37: ## %cond.load52
-; AVX512VLDQ-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX512VLDQ-NEXT:    vpinsrb $2, 18(%rdi), %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512VLDQ-NEXT:    testl $262144, %eax ## imm = 0x40000
+; AVX512VLDQ-NEXT:    jne LBB24_37
 ; AVX512VLDQ-NEXT:  LBB24_38: ## %else53
-; AVX512VLDQ-NEXT:    kshiftrw $3, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
-; AVX512VLDQ-NEXT:    je LBB24_40
-; AVX512VLDQ-NEXT:  ## %bb.39: ## %cond.load55
-; AVX512VLDQ-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX512VLDQ-NEXT:    vpinsrb $3, 19(%rdi), %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512VLDQ-NEXT:    testl $524288, %eax ## imm = 0x80000
+; AVX512VLDQ-NEXT:    jne LBB24_39
 ; AVX512VLDQ-NEXT:  LBB24_40: ## %else56
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm2, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $4, %k0, %k1
-; AVX512VLDQ-NEXT:    kmovw %k1, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
-; AVX512VLDQ-NEXT:    je LBB24_42
-; AVX512VLDQ-NEXT:  ## %bb.41: ## %cond.load58
-; AVX512VLDQ-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX512VLDQ-NEXT:    vpinsrb $4, 20(%rdi), %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512VLDQ-NEXT:    testl $1048576, %eax ## imm = 0x100000
+; AVX512VLDQ-NEXT:    jne LBB24_41
 ; AVX512VLDQ-NEXT:  LBB24_42: ## %else59
-; AVX512VLDQ-NEXT:    kshiftrw $5, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
-; AVX512VLDQ-NEXT:    je LBB24_44
-; AVX512VLDQ-NEXT:  ## %bb.43: ## %cond.load61
-; AVX512VLDQ-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX512VLDQ-NEXT:    vpinsrb $5, 21(%rdi), %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512VLDQ-NEXT:    testl $2097152, %eax ## imm = 0x200000
+; AVX512VLDQ-NEXT:    jne LBB24_43
 ; AVX512VLDQ-NEXT:  LBB24_44: ## %else62
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm2, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $6, %k0, %k1
-; AVX512VLDQ-NEXT:    kmovw %k1, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
-; AVX512VLDQ-NEXT:    je LBB24_46
-; AVX512VLDQ-NEXT:  ## %bb.45: ## %cond.load64
-; AVX512VLDQ-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX512VLDQ-NEXT:    vpinsrb $6, 22(%rdi), %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512VLDQ-NEXT:    testl $4194304, %eax ## imm = 0x400000
+; AVX512VLDQ-NEXT:    jne LBB24_45
 ; AVX512VLDQ-NEXT:  LBB24_46: ## %else65
-; AVX512VLDQ-NEXT:    kshiftrw $7, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
-; AVX512VLDQ-NEXT:    je LBB24_48
-; AVX512VLDQ-NEXT:  ## %bb.47: ## %cond.load67
-; AVX512VLDQ-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX512VLDQ-NEXT:    vpinsrb $7, 23(%rdi), %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512VLDQ-NEXT:    testl $8388608, %eax ## imm = 0x800000
+; AVX512VLDQ-NEXT:    jne LBB24_47
 ; AVX512VLDQ-NEXT:  LBB24_48: ## %else68
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm2, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $8, %k0, %k1
-; AVX512VLDQ-NEXT:    kmovw %k1, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
-; AVX512VLDQ-NEXT:    je LBB24_50
-; AVX512VLDQ-NEXT:  ## %bb.49: ## %cond.load70
-; AVX512VLDQ-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX512VLDQ-NEXT:    vpinsrb $8, 24(%rdi), %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512VLDQ-NEXT:    testl $16777216, %eax ## imm = 0x1000000
+; AVX512VLDQ-NEXT:    jne LBB24_49
 ; AVX512VLDQ-NEXT:  LBB24_50: ## %else71
-; AVX512VLDQ-NEXT:    kshiftrw $9, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
-; AVX512VLDQ-NEXT:    je LBB24_52
-; AVX512VLDQ-NEXT:  ## %bb.51: ## %cond.load73
-; AVX512VLDQ-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX512VLDQ-NEXT:    vpinsrb $9, 25(%rdi), %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512VLDQ-NEXT:    testl $33554432, %eax ## imm = 0x2000000
+; AVX512VLDQ-NEXT:    jne LBB24_51
 ; AVX512VLDQ-NEXT:  LBB24_52: ## %else74
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm2, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $10, %k0, %k1
-; AVX512VLDQ-NEXT:    kmovw %k1, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
-; AVX512VLDQ-NEXT:    je LBB24_54
-; AVX512VLDQ-NEXT:  ## %bb.53: ## %cond.load76
-; AVX512VLDQ-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX512VLDQ-NEXT:    vpinsrb $10, 26(%rdi), %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512VLDQ-NEXT:    testl $67108864, %eax ## imm = 0x4000000
+; AVX512VLDQ-NEXT:    jne LBB24_53
 ; AVX512VLDQ-NEXT:  LBB24_54: ## %else77
-; AVX512VLDQ-NEXT:    kshiftrw $11, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
-; AVX512VLDQ-NEXT:    je LBB24_56
-; AVX512VLDQ-NEXT:  ## %bb.55: ## %cond.load79
-; AVX512VLDQ-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX512VLDQ-NEXT:    vpinsrb $11, 27(%rdi), %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512VLDQ-NEXT:    testl $134217728, %eax ## imm = 0x8000000
+; AVX512VLDQ-NEXT:    jne LBB24_55
 ; AVX512VLDQ-NEXT:  LBB24_56: ## %else80
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm2, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $12, %k0, %k1
-; AVX512VLDQ-NEXT:    kmovw %k1, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
-; AVX512VLDQ-NEXT:    je LBB24_58
-; AVX512VLDQ-NEXT:  ## %bb.57: ## %cond.load82
-; AVX512VLDQ-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX512VLDQ-NEXT:    vpinsrb $12, 28(%rdi), %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512VLDQ-NEXT:    testl $268435456, %eax ## imm = 0x10000000
+; AVX512VLDQ-NEXT:    jne LBB24_57
 ; AVX512VLDQ-NEXT:  LBB24_58: ## %else83
-; AVX512VLDQ-NEXT:    kshiftrw $13, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
-; AVX512VLDQ-NEXT:    je LBB24_60
-; AVX512VLDQ-NEXT:  ## %bb.59: ## %cond.load85
-; AVX512VLDQ-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX512VLDQ-NEXT:    vpinsrb $13, 29(%rdi), %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512VLDQ-NEXT:    testl $536870912, %eax ## imm = 0x20000000
+; AVX512VLDQ-NEXT:    jne LBB24_59
 ; AVX512VLDQ-NEXT:  LBB24_60: ## %else86
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm0
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm0, %zmm0
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm0, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $14, %k0, %k1
-; AVX512VLDQ-NEXT:    kmovw %k1, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testl $1073741824, %eax ## imm = 0x40000000
+; AVX512VLDQ-NEXT:    jne LBB24_61
+; AVX512VLDQ-NEXT:  LBB24_62: ## %else89
+; AVX512VLDQ-NEXT:    testl $-2147483648, %eax ## imm = 0x80000000
+; AVX512VLDQ-NEXT:    jne LBB24_63
+; AVX512VLDQ-NEXT:  LBB24_64: ## %else92
+; AVX512VLDQ-NEXT:    vmovdqa %ymm1, %ymm0
+; AVX512VLDQ-NEXT:    retq
+; AVX512VLDQ-NEXT:  LBB24_1: ## %cond.load
+; AVX512VLDQ-NEXT:    vpinsrb $0, (%rdi), %xmm1, %xmm0
+; AVX512VLDQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512VLDQ-NEXT:    testb $2, %al
+; AVX512VLDQ-NEXT:    je LBB24_4
+; AVX512VLDQ-NEXT:  LBB24_3: ## %cond.load1
+; AVX512VLDQ-NEXT:    vpinsrb $1, 1(%rdi), %xmm1, %xmm0
+; AVX512VLDQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512VLDQ-NEXT:    testb $4, %al
+; AVX512VLDQ-NEXT:    je LBB24_6
+; AVX512VLDQ-NEXT:  LBB24_5: ## %cond.load4
+; AVX512VLDQ-NEXT:    vpinsrb $2, 2(%rdi), %xmm1, %xmm0
+; AVX512VLDQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512VLDQ-NEXT:    testb $8, %al
+; AVX512VLDQ-NEXT:    je LBB24_8
+; AVX512VLDQ-NEXT:  LBB24_7: ## %cond.load7
+; AVX512VLDQ-NEXT:    vpinsrb $3, 3(%rdi), %xmm1, %xmm0
+; AVX512VLDQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512VLDQ-NEXT:    testb $16, %al
+; AVX512VLDQ-NEXT:    je LBB24_10
+; AVX512VLDQ-NEXT:  LBB24_9: ## %cond.load10
+; AVX512VLDQ-NEXT:    vpinsrb $4, 4(%rdi), %xmm1, %xmm0
+; AVX512VLDQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512VLDQ-NEXT:    testb $32, %al
+; AVX512VLDQ-NEXT:    je LBB24_12
+; AVX512VLDQ-NEXT:  LBB24_11: ## %cond.load13
+; AVX512VLDQ-NEXT:    vpinsrb $5, 5(%rdi), %xmm1, %xmm0
+; AVX512VLDQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512VLDQ-NEXT:    testb $64, %al
+; AVX512VLDQ-NEXT:    je LBB24_14
+; AVX512VLDQ-NEXT:  LBB24_13: ## %cond.load16
+; AVX512VLDQ-NEXT:    vpinsrb $6, 6(%rdi), %xmm1, %xmm0
+; AVX512VLDQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512VLDQ-NEXT:    testb $-128, %al
+; AVX512VLDQ-NEXT:    je LBB24_16
+; AVX512VLDQ-NEXT:  LBB24_15: ## %cond.load19
+; AVX512VLDQ-NEXT:    vpinsrb $7, 7(%rdi), %xmm1, %xmm0
+; AVX512VLDQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512VLDQ-NEXT:    testl $256, %eax ## imm = 0x100
+; AVX512VLDQ-NEXT:    je LBB24_18
+; AVX512VLDQ-NEXT:  LBB24_17: ## %cond.load22
+; AVX512VLDQ-NEXT:    vpinsrb $8, 8(%rdi), %xmm1, %xmm0
+; AVX512VLDQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512VLDQ-NEXT:    testl $512, %eax ## imm = 0x200
+; AVX512VLDQ-NEXT:    je LBB24_20
+; AVX512VLDQ-NEXT:  LBB24_19: ## %cond.load25
+; AVX512VLDQ-NEXT:    vpinsrb $9, 9(%rdi), %xmm1, %xmm0
+; AVX512VLDQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512VLDQ-NEXT:    testl $1024, %eax ## imm = 0x400
+; AVX512VLDQ-NEXT:    je LBB24_22
+; AVX512VLDQ-NEXT:  LBB24_21: ## %cond.load28
+; AVX512VLDQ-NEXT:    vpinsrb $10, 10(%rdi), %xmm1, %xmm0
+; AVX512VLDQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512VLDQ-NEXT:    testl $2048, %eax ## imm = 0x800
+; AVX512VLDQ-NEXT:    je LBB24_24
+; AVX512VLDQ-NEXT:  LBB24_23: ## %cond.load31
+; AVX512VLDQ-NEXT:    vpinsrb $11, 11(%rdi), %xmm1, %xmm0
+; AVX512VLDQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512VLDQ-NEXT:    testl $4096, %eax ## imm = 0x1000
+; AVX512VLDQ-NEXT:    je LBB24_26
+; AVX512VLDQ-NEXT:  LBB24_25: ## %cond.load34
+; AVX512VLDQ-NEXT:    vpinsrb $12, 12(%rdi), %xmm1, %xmm0
+; AVX512VLDQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512VLDQ-NEXT:    testl $8192, %eax ## imm = 0x2000
+; AVX512VLDQ-NEXT:    je LBB24_28
+; AVX512VLDQ-NEXT:  LBB24_27: ## %cond.load37
+; AVX512VLDQ-NEXT:    vpinsrb $13, 13(%rdi), %xmm1, %xmm0
+; AVX512VLDQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512VLDQ-NEXT:    testl $16384, %eax ## imm = 0x4000
+; AVX512VLDQ-NEXT:    je LBB24_30
+; AVX512VLDQ-NEXT:  LBB24_29: ## %cond.load40
+; AVX512VLDQ-NEXT:    vpinsrb $14, 14(%rdi), %xmm1, %xmm0
+; AVX512VLDQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512VLDQ-NEXT:    testl $32768, %eax ## imm = 0x8000
+; AVX512VLDQ-NEXT:    je LBB24_32
+; AVX512VLDQ-NEXT:  LBB24_31: ## %cond.load43
+; AVX512VLDQ-NEXT:    vpinsrb $15, 15(%rdi), %xmm1, %xmm0
+; AVX512VLDQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512VLDQ-NEXT:    testl $65536, %eax ## imm = 0x10000
+; AVX512VLDQ-NEXT:    je LBB24_34
+; AVX512VLDQ-NEXT:  LBB24_33: ## %cond.load46
+; AVX512VLDQ-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX512VLDQ-NEXT:    vpinsrb $0, 16(%rdi), %xmm0, %xmm0
+; AVX512VLDQ-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
+; AVX512VLDQ-NEXT:    testl $131072, %eax ## imm = 0x20000
+; AVX512VLDQ-NEXT:    je LBB24_36
+; AVX512VLDQ-NEXT:  LBB24_35: ## %cond.load49
+; AVX512VLDQ-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX512VLDQ-NEXT:    vpinsrb $1, 17(%rdi), %xmm0, %xmm0
+; AVX512VLDQ-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
+; AVX512VLDQ-NEXT:    testl $262144, %eax ## imm = 0x40000
+; AVX512VLDQ-NEXT:    je LBB24_38
+; AVX512VLDQ-NEXT:  LBB24_37: ## %cond.load52
+; AVX512VLDQ-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX512VLDQ-NEXT:    vpinsrb $2, 18(%rdi), %xmm0, %xmm0
+; AVX512VLDQ-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
+; AVX512VLDQ-NEXT:    testl $524288, %eax ## imm = 0x80000
+; AVX512VLDQ-NEXT:    je LBB24_40
+; AVX512VLDQ-NEXT:  LBB24_39: ## %cond.load55
+; AVX512VLDQ-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX512VLDQ-NEXT:    vpinsrb $3, 19(%rdi), %xmm0, %xmm0
+; AVX512VLDQ-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
+; AVX512VLDQ-NEXT:    testl $1048576, %eax ## imm = 0x100000
+; AVX512VLDQ-NEXT:    je LBB24_42
+; AVX512VLDQ-NEXT:  LBB24_41: ## %cond.load58
+; AVX512VLDQ-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX512VLDQ-NEXT:    vpinsrb $4, 20(%rdi), %xmm0, %xmm0
+; AVX512VLDQ-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
+; AVX512VLDQ-NEXT:    testl $2097152, %eax ## imm = 0x200000
+; AVX512VLDQ-NEXT:    je LBB24_44
+; AVX512VLDQ-NEXT:  LBB24_43: ## %cond.load61
+; AVX512VLDQ-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX512VLDQ-NEXT:    vpinsrb $5, 21(%rdi), %xmm0, %xmm0
+; AVX512VLDQ-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
+; AVX512VLDQ-NEXT:    testl $4194304, %eax ## imm = 0x400000
+; AVX512VLDQ-NEXT:    je LBB24_46
+; AVX512VLDQ-NEXT:  LBB24_45: ## %cond.load64
+; AVX512VLDQ-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX512VLDQ-NEXT:    vpinsrb $6, 22(%rdi), %xmm0, %xmm0
+; AVX512VLDQ-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
+; AVX512VLDQ-NEXT:    testl $8388608, %eax ## imm = 0x800000
+; AVX512VLDQ-NEXT:    je LBB24_48
+; AVX512VLDQ-NEXT:  LBB24_47: ## %cond.load67
+; AVX512VLDQ-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX512VLDQ-NEXT:    vpinsrb $7, 23(%rdi), %xmm0, %xmm0
+; AVX512VLDQ-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
+; AVX512VLDQ-NEXT:    testl $16777216, %eax ## imm = 0x1000000
+; AVX512VLDQ-NEXT:    je LBB24_50
+; AVX512VLDQ-NEXT:  LBB24_49: ## %cond.load70
+; AVX512VLDQ-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX512VLDQ-NEXT:    vpinsrb $8, 24(%rdi), %xmm0, %xmm0
+; AVX512VLDQ-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
+; AVX512VLDQ-NEXT:    testl $33554432, %eax ## imm = 0x2000000
+; AVX512VLDQ-NEXT:    je LBB24_52
+; AVX512VLDQ-NEXT:  LBB24_51: ## %cond.load73
+; AVX512VLDQ-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX512VLDQ-NEXT:    vpinsrb $9, 25(%rdi), %xmm0, %xmm0
+; AVX512VLDQ-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
+; AVX512VLDQ-NEXT:    testl $67108864, %eax ## imm = 0x4000000
+; AVX512VLDQ-NEXT:    je LBB24_54
+; AVX512VLDQ-NEXT:  LBB24_53: ## %cond.load76
+; AVX512VLDQ-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX512VLDQ-NEXT:    vpinsrb $10, 26(%rdi), %xmm0, %xmm0
+; AVX512VLDQ-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
+; AVX512VLDQ-NEXT:    testl $134217728, %eax ## imm = 0x8000000
+; AVX512VLDQ-NEXT:    je LBB24_56
+; AVX512VLDQ-NEXT:  LBB24_55: ## %cond.load79
+; AVX512VLDQ-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX512VLDQ-NEXT:    vpinsrb $11, 27(%rdi), %xmm0, %xmm0
+; AVX512VLDQ-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
+; AVX512VLDQ-NEXT:    testl $268435456, %eax ## imm = 0x10000000
+; AVX512VLDQ-NEXT:    je LBB24_58
+; AVX512VLDQ-NEXT:  LBB24_57: ## %cond.load82
+; AVX512VLDQ-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX512VLDQ-NEXT:    vpinsrb $12, 28(%rdi), %xmm0, %xmm0
+; AVX512VLDQ-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
+; AVX512VLDQ-NEXT:    testl $536870912, %eax ## imm = 0x20000000
+; AVX512VLDQ-NEXT:    je LBB24_60
+; AVX512VLDQ-NEXT:  LBB24_59: ## %cond.load85
+; AVX512VLDQ-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX512VLDQ-NEXT:    vpinsrb $13, 29(%rdi), %xmm0, %xmm0
+; AVX512VLDQ-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
+; AVX512VLDQ-NEXT:    testl $1073741824, %eax ## imm = 0x40000000
 ; AVX512VLDQ-NEXT:    je LBB24_62
-; AVX512VLDQ-NEXT:  ## %bb.61: ## %cond.load88
+; AVX512VLDQ-NEXT:  LBB24_61: ## %cond.load88
 ; AVX512VLDQ-NEXT:    vextracti128 $1, %ymm1, %xmm0
 ; AVX512VLDQ-NEXT:    vpinsrb $14, 30(%rdi), %xmm0, %xmm0
 ; AVX512VLDQ-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
-; AVX512VLDQ-NEXT:  LBB24_62: ## %else89
-; AVX512VLDQ-NEXT:    kshiftrw $15, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testl $-2147483648, %eax ## imm = 0x80000000
 ; AVX512VLDQ-NEXT:    je LBB24_64
-; AVX512VLDQ-NEXT:  ## %bb.63: ## %cond.load91
+; AVX512VLDQ-NEXT:  LBB24_63: ## %cond.load91
 ; AVX512VLDQ-NEXT:    vextracti128 $1, %ymm1, %xmm0
 ; AVX512VLDQ-NEXT:    vpinsrb $15, 31(%rdi), %xmm0, %xmm0
 ; AVX512VLDQ-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm1
-; AVX512VLDQ-NEXT:  LBB24_64: ## %else92
 ; AVX512VLDQ-NEXT:    vmovdqa %ymm1, %ymm0
 ; AVX512VLDQ-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/masked_store.ll b/llvm/test/CodeGen/X86/masked_store.ll
index 0e715e898..eb1fba14a 100644
--- a/llvm/test/CodeGen/X86/masked_store.ll
+++ b/llvm/test/CodeGen/X86/masked_store.ll
@@ -35,48 +35,23 @@ define void @store_v1f64_v1i64(<1 x i64> %trigger, <1 x double>* %addr, <1 x dou
 }
 
 define void @store_v2f64_v2i64(<2 x i64> %trigger, <2 x double>* %addr, <2 x double> %val) {
-; SSE2-LABEL: store_v2f64_v2i64:
-; SSE2:       ## %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
-; SSE2-NEXT:    pxor %xmm2, %xmm0
-; SSE2-NEXT:    movdqa %xmm2, %xmm3
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pand %xmm3, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE2-NEXT:    por %xmm2, %xmm0
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB1_2
-; SSE2-NEXT:  ## %bb.1: ## %cond.store
-; SSE2-NEXT:    movlps %xmm1, (%rdi)
-; SSE2-NEXT:  LBB1_2: ## %else
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB1_4
-; SSE2-NEXT:  ## %bb.3: ## %cond.store1
-; SSE2-NEXT:    movhps %xmm1, 8(%rdi)
-; SSE2-NEXT:  LBB1_4: ## %else2
-; SSE2-NEXT:    retq
-;
-; SSE4-LABEL: store_v2f64_v2i64:
-; SSE4:       ## %bb.0:
-; SSE4-NEXT:    pxor %xmm2, %xmm2
-; SSE4-NEXT:    pcmpgtq %xmm0, %xmm2
-; SSE4-NEXT:    pextrb $0, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je LBB1_2
-; SSE4-NEXT:  ## %bb.1: ## %cond.store
-; SSE4-NEXT:    movlps %xmm1, (%rdi)
-; SSE4-NEXT:  LBB1_2: ## %else
-; SSE4-NEXT:    pextrb $8, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je LBB1_4
-; SSE4-NEXT:  ## %bb.3: ## %cond.store1
-; SSE4-NEXT:    movhps %xmm1, 8(%rdi)
-; SSE4-NEXT:  LBB1_4: ## %else2
-; SSE4-NEXT:    retq
+; SSE-LABEL: store_v2f64_v2i64:
+; SSE:       ## %bb.0:
+; SSE-NEXT:    movmskpd %xmm0, %eax
+; SSE-NEXT:    testb $1, %al
+; SSE-NEXT:    jne LBB1_1
+; SSE-NEXT:  ## %bb.2: ## %else
+; SSE-NEXT:    testb $2, %al
+; SSE-NEXT:    jne LBB1_3
+; SSE-NEXT:  LBB1_4: ## %else2
+; SSE-NEXT:    retq
+; SSE-NEXT:  LBB1_1: ## %cond.store
+; SSE-NEXT:    movlps %xmm1, (%rdi)
+; SSE-NEXT:    testb $2, %al
+; SSE-NEXT:    je LBB1_4
+; SSE-NEXT:  LBB1_3: ## %cond.store1
+; SSE-NEXT:    movhps %xmm1, 8(%rdi)
+; SSE-NEXT:    retq
 ;
 ; AVX1OR2-LABEL: store_v2f64_v2i64:
 ; AVX1OR2:       ## %bb.0:
@@ -113,82 +88,38 @@ define void @store_v2f64_v2i64(<2 x i64> %trigger, <2 x double>* %addr, <2 x dou
 }
 
 define void @store_v4f64_v4i64(<4 x i64> %trigger, <4 x double>* %addr, <4 x double> %val) {
-; SSE2-LABEL: store_v4f64_v4i64:
-; SSE2:       ## %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [2147483648,2147483648]
-; SSE2-NEXT:    pxor %xmm4, %xmm0
-; SSE2-NEXT:    movdqa %xmm4, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm5
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pand %xmm5, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[1,1,3,3]
-; SSE2-NEXT:    por %xmm6, %xmm0
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB2_2
-; SSE2-NEXT:  ## %bb.1: ## %cond.store
-; SSE2-NEXT:    movlps %xmm2, (%rdi)
-; SSE2-NEXT:  LBB2_2: ## %else
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB2_4
-; SSE2-NEXT:  ## %bb.3: ## %cond.store1
-; SSE2-NEXT:    movhps %xmm2, 8(%rdi)
-; SSE2-NEXT:  LBB2_4: ## %else2
-; SSE2-NEXT:    pxor %xmm4, %xmm1
-; SSE2-NEXT:    movdqa %xmm4, %xmm0
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pand %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm0
-; SSE2-NEXT:    pextrw $0, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB2_6
-; SSE2-NEXT:  ## %bb.5: ## %cond.store3
-; SSE2-NEXT:    movlps %xmm3, 16(%rdi)
-; SSE2-NEXT:  LBB2_6: ## %else4
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB2_8
-; SSE2-NEXT:  ## %bb.7: ## %cond.store5
-; SSE2-NEXT:    movhps %xmm3, 24(%rdi)
-; SSE2-NEXT:  LBB2_8: ## %else6
-; SSE2-NEXT:    retq
-;
-; SSE4-LABEL: store_v4f64_v4i64:
-; SSE4:       ## %bb.0:
-; SSE4-NEXT:    pxor %xmm4, %xmm4
-; SSE4-NEXT:    pcmpgtq %xmm0, %xmm4
-; SSE4-NEXT:    pextrb $0, %xmm4, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je LBB2_2
-; SSE4-NEXT:  ## %bb.1: ## %cond.store
-; SSE4-NEXT:    movlps %xmm2, (%rdi)
-; SSE4-NEXT:  LBB2_2: ## %else
-; SSE4-NEXT:    pextrb $8, %xmm4, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je LBB2_4
-; SSE4-NEXT:  ## %bb.3: ## %cond.store1
-; SSE4-NEXT:    movhps %xmm2, 8(%rdi)
-; SSE4-NEXT:  LBB2_4: ## %else2
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpgtq %xmm1, %xmm0
-; SSE4-NEXT:    pextrb $0, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je LBB2_6
-; SSE4-NEXT:  ## %bb.5: ## %cond.store3
-; SSE4-NEXT:    movlps %xmm3, 16(%rdi)
-; SSE4-NEXT:  LBB2_6: ## %else4
-; SSE4-NEXT:    pextrb $8, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je LBB2_8
-; SSE4-NEXT:  ## %bb.7: ## %cond.store5
-; SSE4-NEXT:    movhps %xmm3, 24(%rdi)
-; SSE4-NEXT:  LBB2_8: ## %else6
-; SSE4-NEXT:    retq
+; SSE-LABEL: store_v4f64_v4i64:
+; SSE:       ## %bb.0:
+; SSE-NEXT:    packssdw %xmm1, %xmm0
+; SSE-NEXT:    movmskps %xmm0, %eax
+; SSE-NEXT:    testb $1, %al
+; SSE-NEXT:    jne LBB2_1
+; SSE-NEXT:  ## %bb.2: ## %else
+; SSE-NEXT:    testb $2, %al
+; SSE-NEXT:    jne LBB2_3
+; SSE-NEXT:  LBB2_4: ## %else2
+; SSE-NEXT:    testb $4, %al
+; SSE-NEXT:    jne LBB2_5
+; SSE-NEXT:  LBB2_6: ## %else4
+; SSE-NEXT:    testb $8, %al
+; SSE-NEXT:    jne LBB2_7
+; SSE-NEXT:  LBB2_8: ## %else6
+; SSE-NEXT:    retq
+; SSE-NEXT:  LBB2_1: ## %cond.store
+; SSE-NEXT:    movlps %xmm2, (%rdi)
+; SSE-NEXT:    testb $2, %al
+; SSE-NEXT:    je LBB2_4
+; SSE-NEXT:  LBB2_3: ## %cond.store1
+; SSE-NEXT:    movhps %xmm2, 8(%rdi)
+; SSE-NEXT:    testb $4, %al
+; SSE-NEXT:    je LBB2_6
+; SSE-NEXT:  LBB2_5: ## %cond.store3
+; SSE-NEXT:    movlps %xmm3, 16(%rdi)
+; SSE-NEXT:    testb $8, %al
+; SSE-NEXT:    je LBB2_8
+; SSE-NEXT:  LBB2_7: ## %cond.store5
+; SSE-NEXT:    movhps %xmm3, 24(%rdi)
+; SSE-NEXT:    retq
 ;
 ; AVX1OR2-LABEL: store_v4f64_v4i64:
 ; AVX1OR2:       ## %bb.0:
@@ -234,82 +165,81 @@ define void @store_v4f64_v4i64(<4 x i64> %trigger, <4 x double>* %addr, <4 x dou
 define void @store_v2f32_v2i32(<2 x i32> %trigger, <2 x float>* %addr, <2 x float> %val) {
 ; SSE2-LABEL: store_v2f32_v2i32:
 ; SSE2:       ## %bb.0:
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
 ; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,0,3,2]
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,0,1,1]
+; SSE2-NEXT:    movmskpd %xmm0, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB3_2
-; SSE2-NEXT:  ## %bb.1: ## %cond.store
+; SSE2-NEXT:    jne LBB3_1
+; SSE2-NEXT:  ## %bb.2: ## %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne LBB3_3
+; SSE2-NEXT:  LBB3_4: ## %else2
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  LBB3_1: ## %cond.store
 ; SSE2-NEXT:    movss %xmm1, (%rdi)
-; SSE2-NEXT:  LBB3_2: ## %else
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $2, %al
 ; SSE2-NEXT:    je LBB3_4
-; SSE2-NEXT:  ## %bb.3: ## %cond.store1
+; SSE2-NEXT:  LBB3_3: ## %cond.store1
 ; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,2,3]
 ; SSE2-NEXT:    movss %xmm1, 4(%rdi)
-; SSE2-NEXT:  LBB3_4: ## %else2
 ; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: store_v2f32_v2i32:
 ; SSE4:       ## %bb.0:
 ; SSE4-NEXT:    pxor %xmm2, %xmm2
-; SSE4-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
-; SSE4-NEXT:    pcmpeqq %xmm2, %xmm0
-; SSE4-NEXT:    pextrb $0, %xmm0, %eax
+; SSE4-NEXT:    pcmpeqd %xmm0, %xmm2
+; SSE4-NEXT:    pmovsxdq %xmm2, %xmm0
+; SSE4-NEXT:    movmskpd %xmm0, %eax
 ; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je LBB3_2
-; SSE4-NEXT:  ## %bb.1: ## %cond.store
+; SSE4-NEXT:    jne LBB3_1
+; SSE4-NEXT:  ## %bb.2: ## %else
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    jne LBB3_3
+; SSE4-NEXT:  LBB3_4: ## %else2
+; SSE4-NEXT:    retq
+; SSE4-NEXT:  LBB3_1: ## %cond.store
 ; SSE4-NEXT:    movss %xmm1, (%rdi)
-; SSE4-NEXT:  LBB3_2: ## %else
-; SSE4-NEXT:    pextrb $8, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $2, %al
 ; SSE4-NEXT:    je LBB3_4
-; SSE4-NEXT:  ## %bb.3: ## %cond.store1
+; SSE4-NEXT:  LBB3_3: ## %cond.store1
 ; SSE4-NEXT:    extractps $1, %xmm1, 4(%rdi)
-; SSE4-NEXT:  LBB3_4: ## %else2
 ; SSE4-NEXT:    retq
 ;
-; AVX1-LABEL: store_v2f32_v2i32:
-; AVX1:       ## %bb.0:
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
-; AVX1-NEXT:    vpcmpeqq %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,2],zero,zero
-; AVX1-NEXT:    vmaskmovps %xmm1, %xmm0, (%rdi)
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: store_v2f32_v2i32:
-; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
-; AVX2-NEXT:    vpcmpeqq %xmm2, %xmm0, %xmm0
-; AVX2-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,2],zero,zero
-; AVX2-NEXT:    vmaskmovps %xmm1, %xmm0, (%rdi)
-; AVX2-NEXT:    retq
+; AVX1OR2-LABEL: store_v2f32_v2i32:
+; AVX1OR2:       ## %bb.0:
+; AVX1OR2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX1OR2-NEXT:    vpcmpeqd %xmm2, %xmm0, %xmm0
+; AVX1OR2-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
+; AVX1OR2-NEXT:    vmaskmovps %xmm1, %xmm0, (%rdi)
+; AVX1OR2-NEXT:    retq
 ;
 ; AVX512F-LABEL: store_v2f32_v2i32:
 ; AVX512F:       ## %bb.0:
 ; AVX512F-NEXT:    ## kill: def $xmm1 killed $xmm1 def $zmm1
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
-; AVX512F-NEXT:    vptestnmq %zmm0, %zmm0, %k0
+; AVX512F-NEXT:    ## kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512F-NEXT:    vptestnmd %zmm0, %zmm0, %k0
 ; AVX512F-NEXT:    kshiftlw $14, %k0, %k0
 ; AVX512F-NEXT:    kshiftrw $14, %k0, %k1
 ; AVX512F-NEXT:    vmovups %zmm1, (%rdi) {%k1}
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
-; AVX512VL-LABEL: store_v2f32_v2i32:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VL-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
-; AVX512VL-NEXT:    vptestnmq %xmm0, %xmm0, %k1
-; AVX512VL-NEXT:    vmovups %xmm1, (%rdi) {%k1}
-; AVX512VL-NEXT:    retq
+; AVX512VLDQ-LABEL: store_v2f32_v2i32:
+; AVX512VLDQ:       ## %bb.0:
+; AVX512VLDQ-NEXT:    vptestnmd %xmm0, %xmm0, %k0
+; AVX512VLDQ-NEXT:    kshiftlb $6, %k0, %k0
+; AVX512VLDQ-NEXT:    kshiftrb $6, %k0, %k1
+; AVX512VLDQ-NEXT:    vmovups %xmm1, (%rdi) {%k1}
+; AVX512VLDQ-NEXT:    retq
+;
+; AVX512VLBW-LABEL: store_v2f32_v2i32:
+; AVX512VLBW:       ## %bb.0:
+; AVX512VLBW-NEXT:    vptestnmd %xmm0, %xmm0, %k0
+; AVX512VLBW-NEXT:    kshiftlw $14, %k0, %k0
+; AVX512VLBW-NEXT:    kshiftrw $14, %k0, %k1
+; AVX512VLBW-NEXT:    vmovups %xmm1, (%rdi) {%k1}
+; AVX512VLBW-NEXT:    retq
   %mask = icmp eq <2 x i32> %trigger, zeroinitializer
   call void @llvm.masked.store.v2f32.p0v2f32(<2 x float> %val, <2 x float>* %addr, i32 4, <2 x i1> %mask)
   ret void
@@ -318,71 +248,71 @@ define void @store_v2f32_v2i32(<2 x i32> %trigger, <2 x float>* %addr, <2 x floa
 define void @store_v4f32_v4i32(<4 x float> %x, <4 x float>* %ptr, <4 x float> %y, <4 x i32> %mask) {
 ; SSE2-LABEL: store_v4f32_v4i32:
 ; SSE2:       ## %bb.0:
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm1
-; SSE2-NEXT:    movd %xmm1, %eax
+; SSE2-NEXT:    movmskps %xmm2, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB4_2
-; SSE2-NEXT:  ## %bb.1: ## %cond.store
+; SSE2-NEXT:    jne LBB4_1
+; SSE2-NEXT:  ## %bb.2: ## %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne LBB4_3
+; SSE2-NEXT:  LBB4_4: ## %else2
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne LBB4_5
+; SSE2-NEXT:  LBB4_6: ## %else4
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne LBB4_7
+; SSE2-NEXT:  LBB4_8: ## %else6
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  LBB4_1: ## %cond.store
 ; SSE2-NEXT:    movss %xmm0, (%rdi)
-; SSE2-NEXT:  LBB4_2: ## %else
-; SSE2-NEXT:    pextrw $2, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $2, %al
 ; SSE2-NEXT:    je LBB4_4
-; SSE2-NEXT:  ## %bb.3: ## %cond.store1
+; SSE2-NEXT:  LBB4_3: ## %cond.store1
 ; SSE2-NEXT:    movaps %xmm0, %xmm1
 ; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[2,3]
 ; SSE2-NEXT:    movss %xmm1, 4(%rdi)
-; SSE2-NEXT:  LBB4_4: ## %else2
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm1
-; SSE2-NEXT:    pextrw $4, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $4, %al
 ; SSE2-NEXT:    je LBB4_6
-; SSE2-NEXT:  ## %bb.5: ## %cond.store3
-; SSE2-NEXT:    movaps %xmm0, %xmm2
-; SSE2-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm0[1]
-; SSE2-NEXT:    movss %xmm2, 8(%rdi)
-; SSE2-NEXT:  LBB4_6: ## %else4
-; SSE2-NEXT:    pextrw $6, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:  LBB4_5: ## %cond.store3
+; SSE2-NEXT:    movaps %xmm0, %xmm1
+; SSE2-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
+; SSE2-NEXT:    movss %xmm1, 8(%rdi)
+; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je LBB4_8
-; SSE2-NEXT:  ## %bb.7: ## %cond.store5
+; SSE2-NEXT:  LBB4_7: ## %cond.store5
 ; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
 ; SSE2-NEXT:    movss %xmm0, 12(%rdi)
-; SSE2-NEXT:  LBB4_8: ## %else6
 ; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: store_v4f32_v4i32:
 ; SSE4:       ## %bb.0:
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpgtd %xmm2, %xmm1
-; SSE4-NEXT:    pextrb $0, %xmm1, %eax
+; SSE4-NEXT:    movmskps %xmm2, %eax
 ; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je LBB4_2
-; SSE4-NEXT:  ## %bb.1: ## %cond.store
+; SSE4-NEXT:    jne LBB4_1
+; SSE4-NEXT:  ## %bb.2: ## %else
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    jne LBB4_3
+; SSE4-NEXT:  LBB4_4: ## %else2
+; SSE4-NEXT:    testb $4, %al
+; SSE4-NEXT:    jne LBB4_5
+; SSE4-NEXT:  LBB4_6: ## %else4
+; SSE4-NEXT:    testb $8, %al
+; SSE4-NEXT:    jne LBB4_7
+; SSE4-NEXT:  LBB4_8: ## %else6
+; SSE4-NEXT:    retq
+; SSE4-NEXT:  LBB4_1: ## %cond.store
 ; SSE4-NEXT:    movss %xmm0, (%rdi)
-; SSE4-NEXT:  LBB4_2: ## %else
-; SSE4-NEXT:    pextrb $4, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $2, %al
 ; SSE4-NEXT:    je LBB4_4
-; SSE4-NEXT:  ## %bb.3: ## %cond.store1
+; SSE4-NEXT:  LBB4_3: ## %cond.store1
 ; SSE4-NEXT:    extractps $1, %xmm0, 4(%rdi)
-; SSE4-NEXT:  LBB4_4: ## %else2
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpgtd %xmm2, %xmm1
-; SSE4-NEXT:    pextrb $8, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $4, %al
 ; SSE4-NEXT:    je LBB4_6
-; SSE4-NEXT:  ## %bb.5: ## %cond.store3
+; SSE4-NEXT:  LBB4_5: ## %cond.store3
 ; SSE4-NEXT:    extractps $2, %xmm0, 8(%rdi)
-; SSE4-NEXT:  LBB4_6: ## %else4
-; SSE4-NEXT:    pextrb $12, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $8, %al
 ; SSE4-NEXT:    je LBB4_8
-; SSE4-NEXT:  ## %bb.7: ## %cond.store5
+; SSE4-NEXT:  LBB4_7: ## %cond.store5
 ; SSE4-NEXT:    extractps $3, %xmm0, 12(%rdi)
-; SSE4-NEXT:  LBB4_8: ## %else6
 ; SSE4-NEXT:    retq
 ;
 ; AVX1OR2-LABEL: store_v4f32_v4i32:
@@ -422,136 +352,136 @@ define void @store_v4f32_v4i32(<4 x float> %x, <4 x float>* %ptr, <4 x float> %y
 define void @store_v8f32_v8i32(<8 x float> %x, <8 x float>* %ptr, <8 x float> %y, <8 x i32> %mask) {
 ; SSE2-LABEL: store_v8f32_v8i32:
 ; SSE2:       ## %bb.0:
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpgtd %xmm4, %xmm2
-; SSE2-NEXT:    movdqa %xmm2, %xmm3
-; SSE2-NEXT:    packssdw %xmm0, %xmm3
-; SSE2-NEXT:    movd %xmm3, %eax
+; SSE2-NEXT:    packssdw %xmm5, %xmm4
+; SSE2-NEXT:    packsswb %xmm0, %xmm4
+; SSE2-NEXT:    pmovmskb %xmm4, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB5_2
-; SSE2-NEXT:  ## %bb.1: ## %cond.store
+; SSE2-NEXT:    jne LBB5_1
+; SSE2-NEXT:  ## %bb.2: ## %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne LBB5_3
+; SSE2-NEXT:  LBB5_4: ## %else2
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne LBB5_5
+; SSE2-NEXT:  LBB5_6: ## %else4
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne LBB5_7
+; SSE2-NEXT:  LBB5_8: ## %else6
+; SSE2-NEXT:    testb $16, %al
+; SSE2-NEXT:    jne LBB5_9
+; SSE2-NEXT:  LBB5_10: ## %else8
+; SSE2-NEXT:    testb $32, %al
+; SSE2-NEXT:    jne LBB5_11
+; SSE2-NEXT:  LBB5_12: ## %else10
+; SSE2-NEXT:    testb $64, %al
+; SSE2-NEXT:    jne LBB5_13
+; SSE2-NEXT:  LBB5_14: ## %else12
+; SSE2-NEXT:    testb $-128, %al
+; SSE2-NEXT:    jne LBB5_15
+; SSE2-NEXT:  LBB5_16: ## %else14
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  LBB5_1: ## %cond.store
 ; SSE2-NEXT:    movd %xmm0, (%rdi)
-; SSE2-NEXT:  LBB5_2: ## %else
-; SSE2-NEXT:    psrlq $16, %xmm2
-; SSE2-NEXT:    movd %xmm2, %eax
-; SSE2-NEXT:    shrl $16, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $2, %al
 ; SSE2-NEXT:    je LBB5_4
-; SSE2-NEXT:  ## %bb.3: ## %cond.store1
+; SSE2-NEXT:  LBB5_3: ## %cond.store1
 ; SSE2-NEXT:    movdqa %xmm0, %xmm2
 ; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1],xmm0[2,3]
 ; SSE2-NEXT:    movss %xmm2, 4(%rdi)
-; SSE2-NEXT:  LBB5_4: ## %else2
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpgtd %xmm4, %xmm2
-; SSE2-NEXT:    pextrw $4, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $4, %al
 ; SSE2-NEXT:    je LBB5_6
-; SSE2-NEXT:  ## %bb.5: ## %cond.store3
-; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    punpckhqdq {{.*#+}} xmm3 = xmm3[1],xmm0[1]
-; SSE2-NEXT:    movd %xmm3, 8(%rdi)
-; SSE2-NEXT:  LBB5_6: ## %else4
-; SSE2-NEXT:    pextrw $6, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:  LBB5_5: ## %cond.store3
+; SSE2-NEXT:    movaps %xmm0, %xmm2
+; SSE2-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm0[1]
+; SSE2-NEXT:    movss %xmm2, 8(%rdi)
+; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je LBB5_8
-; SSE2-NEXT:  ## %bb.7: ## %cond.store5
+; SSE2-NEXT:  LBB5_7: ## %cond.store5
 ; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
 ; SSE2-NEXT:    movss %xmm0, 12(%rdi)
-; SSE2-NEXT:  LBB5_8: ## %else6
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpgtd %xmm5, %xmm0
-; SSE2-NEXT:    pextrw $0, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $16, %al
 ; SSE2-NEXT:    je LBB5_10
-; SSE2-NEXT:  ## %bb.9: ## %cond.store7
+; SSE2-NEXT:  LBB5_9: ## %cond.store7
 ; SSE2-NEXT:    movss %xmm1, 16(%rdi)
-; SSE2-NEXT:  LBB5_10: ## %else8
-; SSE2-NEXT:    pextrw $2, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $32, %al
 ; SSE2-NEXT:    je LBB5_12
-; SSE2-NEXT:  ## %bb.11: ## %cond.store9
+; SSE2-NEXT:  LBB5_11: ## %cond.store9
 ; SSE2-NEXT:    movaps %xmm1, %xmm0
 ; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm1[2,3]
 ; SSE2-NEXT:    movss %xmm0, 20(%rdi)
-; SSE2-NEXT:  LBB5_12: ## %else10
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpgtd %xmm5, %xmm0
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $64, %al
 ; SSE2-NEXT:    je LBB5_14
-; SSE2-NEXT:  ## %bb.13: ## %cond.store11
-; SSE2-NEXT:    movaps %xmm1, %xmm2
-; SSE2-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm1[1]
-; SSE2-NEXT:    movss %xmm2, 24(%rdi)
-; SSE2-NEXT:  LBB5_14: ## %else12
-; SSE2-NEXT:    pextrw $6, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:  LBB5_13: ## %cond.store11
+; SSE2-NEXT:    movaps %xmm1, %xmm0
+; SSE2-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1]
+; SSE2-NEXT:    movss %xmm0, 24(%rdi)
+; SSE2-NEXT:    testb $-128, %al
 ; SSE2-NEXT:    je LBB5_16
-; SSE2-NEXT:  ## %bb.15: ## %cond.store13
+; SSE2-NEXT:  LBB5_15: ## %cond.store13
 ; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,1,2,3]
 ; SSE2-NEXT:    movss %xmm1, 28(%rdi)
-; SSE2-NEXT:  LBB5_16: ## %else14
 ; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: store_v8f32_v8i32:
 ; SSE4:       ## %bb.0:
-; SSE4-NEXT:    pxor %xmm2, %xmm2
-; SSE4-NEXT:    pcmpgtd %xmm4, %xmm2
-; SSE4-NEXT:    pextrb $0, %xmm2, %eax
+; SSE4-NEXT:    packssdw %xmm5, %xmm4
+; SSE4-NEXT:    packsswb %xmm0, %xmm4
+; SSE4-NEXT:    pmovmskb %xmm4, %eax
 ; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je LBB5_2
-; SSE4-NEXT:  ## %bb.1: ## %cond.store
+; SSE4-NEXT:    jne LBB5_1
+; SSE4-NEXT:  ## %bb.2: ## %else
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    jne LBB5_3
+; SSE4-NEXT:  LBB5_4: ## %else2
+; SSE4-NEXT:    testb $4, %al
+; SSE4-NEXT:    jne LBB5_5
+; SSE4-NEXT:  LBB5_6: ## %else4
+; SSE4-NEXT:    testb $8, %al
+; SSE4-NEXT:    jne LBB5_7
+; SSE4-NEXT:  LBB5_8: ## %else6
+; SSE4-NEXT:    testb $16, %al
+; SSE4-NEXT:    jne LBB5_9
+; SSE4-NEXT:  LBB5_10: ## %else8
+; SSE4-NEXT:    testb $32, %al
+; SSE4-NEXT:    jne LBB5_11
+; SSE4-NEXT:  LBB5_12: ## %else10
+; SSE4-NEXT:    testb $64, %al
+; SSE4-NEXT:    jne LBB5_13
+; SSE4-NEXT:  LBB5_14: ## %else12
+; SSE4-NEXT:    testb $-128, %al
+; SSE4-NEXT:    jne LBB5_15
+; SSE4-NEXT:  LBB5_16: ## %else14
+; SSE4-NEXT:    retq
+; SSE4-NEXT:  LBB5_1: ## %cond.store
 ; SSE4-NEXT:    movd %xmm0, (%rdi)
-; SSE4-NEXT:  LBB5_2: ## %else
-; SSE4-NEXT:    pextrb $4, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $2, %al
 ; SSE4-NEXT:    je LBB5_4
-; SSE4-NEXT:  ## %bb.3: ## %cond.store1
+; SSE4-NEXT:  LBB5_3: ## %cond.store1
 ; SSE4-NEXT:    pextrd $1, %xmm0, 4(%rdi)
-; SSE4-NEXT:  LBB5_4: ## %else2
-; SSE4-NEXT:    pxor %xmm2, %xmm2
-; SSE4-NEXT:    pcmpgtd %xmm4, %xmm2
-; SSE4-NEXT:    pextrb $8, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $4, %al
 ; SSE4-NEXT:    je LBB5_6
-; SSE4-NEXT:  ## %bb.5: ## %cond.store3
+; SSE4-NEXT:  LBB5_5: ## %cond.store3
 ; SSE4-NEXT:    pextrd $2, %xmm0, 8(%rdi)
-; SSE4-NEXT:  LBB5_6: ## %else4
-; SSE4-NEXT:    pextrb $12, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $8, %al
 ; SSE4-NEXT:    je LBB5_8
-; SSE4-NEXT:  ## %bb.7: ## %cond.store5
+; SSE4-NEXT:  LBB5_7: ## %cond.store5
 ; SSE4-NEXT:    pextrd $3, %xmm0, 12(%rdi)
-; SSE4-NEXT:  LBB5_8: ## %else6
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpgtd %xmm5, %xmm0
-; SSE4-NEXT:    pextrb $0, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $16, %al
 ; SSE4-NEXT:    je LBB5_10
-; SSE4-NEXT:  ## %bb.9: ## %cond.store7
+; SSE4-NEXT:  LBB5_9: ## %cond.store7
 ; SSE4-NEXT:    movss %xmm1, 16(%rdi)
-; SSE4-NEXT:  LBB5_10: ## %else8
-; SSE4-NEXT:    pextrb $4, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $32, %al
 ; SSE4-NEXT:    je LBB5_12
-; SSE4-NEXT:  ## %bb.11: ## %cond.store9
+; SSE4-NEXT:  LBB5_11: ## %cond.store9
 ; SSE4-NEXT:    extractps $1, %xmm1, 20(%rdi)
-; SSE4-NEXT:  LBB5_12: ## %else10
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpgtd %xmm5, %xmm0
-; SSE4-NEXT:    pextrb $8, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $64, %al
 ; SSE4-NEXT:    je LBB5_14
-; SSE4-NEXT:  ## %bb.13: ## %cond.store11
+; SSE4-NEXT:  LBB5_13: ## %cond.store11
 ; SSE4-NEXT:    extractps $2, %xmm1, 24(%rdi)
-; SSE4-NEXT:  LBB5_14: ## %else12
-; SSE4-NEXT:    pextrb $12, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $-128, %al
 ; SSE4-NEXT:    je LBB5_16
-; SSE4-NEXT:  ## %bb.15: ## %cond.store13
+; SSE4-NEXT:  LBB5_15: ## %cond.store13
 ; SSE4-NEXT:    extractps $3, %xmm1, 28(%rdi)
-; SSE4-NEXT:  LBB5_16: ## %else14
 ; SSE4-NEXT:    retq
 ;
 ; AVX1OR2-LABEL: store_v8f32_v8i32:
@@ -595,261 +525,263 @@ define void @store_v16f32_v16i32(<16 x float> %x, <16 x float>* %ptr, <16 x floa
 ; SSE2-LABEL: store_v16f32_v16i32:
 ; SSE2:       ## %bb.0:
 ; SSE2-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm4
-; SSE2-NEXT:    pxor %xmm5, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm4, %xmm5
-; SSE2-NEXT:    movd %xmm5, %eax
+; SSE2-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm5
+; SSE2-NEXT:    packssdw {{[0-9]+}}(%rsp), %xmm5
+; SSE2-NEXT:    packssdw {{[0-9]+}}(%rsp), %xmm4
+; SSE2-NEXT:    packsswb %xmm5, %xmm4
+; SSE2-NEXT:    pmovmskb %xmm4, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB6_2
-; SSE2-NEXT:  ## %bb.1: ## %cond.store
+; SSE2-NEXT:    jne LBB6_1
+; SSE2-NEXT:  ## %bb.2: ## %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne LBB6_3
+; SSE2-NEXT:  LBB6_4: ## %else2
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne LBB6_5
+; SSE2-NEXT:  LBB6_6: ## %else4
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne LBB6_7
+; SSE2-NEXT:  LBB6_8: ## %else6
+; SSE2-NEXT:    testb $16, %al
+; SSE2-NEXT:    jne LBB6_9
+; SSE2-NEXT:  LBB6_10: ## %else8
+; SSE2-NEXT:    testb $32, %al
+; SSE2-NEXT:    jne LBB6_11
+; SSE2-NEXT:  LBB6_12: ## %else10
+; SSE2-NEXT:    testb $64, %al
+; SSE2-NEXT:    jne LBB6_13
+; SSE2-NEXT:  LBB6_14: ## %else12
+; SSE2-NEXT:    testb $-128, %al
+; SSE2-NEXT:    jne LBB6_15
+; SSE2-NEXT:  LBB6_16: ## %else14
+; SSE2-NEXT:    testl $256, %eax ## imm = 0x100
+; SSE2-NEXT:    jne LBB6_17
+; SSE2-NEXT:  LBB6_18: ## %else16
+; SSE2-NEXT:    testl $512, %eax ## imm = 0x200
+; SSE2-NEXT:    jne LBB6_19
+; SSE2-NEXT:  LBB6_20: ## %else18
+; SSE2-NEXT:    testl $1024, %eax ## imm = 0x400
+; SSE2-NEXT:    jne LBB6_21
+; SSE2-NEXT:  LBB6_22: ## %else20
+; SSE2-NEXT:    testl $2048, %eax ## imm = 0x800
+; SSE2-NEXT:    jne LBB6_23
+; SSE2-NEXT:  LBB6_24: ## %else22
+; SSE2-NEXT:    testl $4096, %eax ## imm = 0x1000
+; SSE2-NEXT:    jne LBB6_25
+; SSE2-NEXT:  LBB6_26: ## %else24
+; SSE2-NEXT:    testl $8192, %eax ## imm = 0x2000
+; SSE2-NEXT:    jne LBB6_27
+; SSE2-NEXT:  LBB6_28: ## %else26
+; SSE2-NEXT:    testl $16384, %eax ## imm = 0x4000
+; SSE2-NEXT:    jne LBB6_29
+; SSE2-NEXT:  LBB6_30: ## %else28
+; SSE2-NEXT:    testl $32768, %eax ## imm = 0x8000
+; SSE2-NEXT:    jne LBB6_31
+; SSE2-NEXT:  LBB6_32: ## %else30
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  LBB6_1: ## %cond.store
 ; SSE2-NEXT:    movss %xmm0, (%rdi)
-; SSE2-NEXT:  LBB6_2: ## %else
-; SSE2-NEXT:    pextrw $2, %xmm5, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $2, %al
 ; SSE2-NEXT:    je LBB6_4
-; SSE2-NEXT:  ## %bb.3: ## %cond.store1
-; SSE2-NEXT:    movaps %xmm0, %xmm5
-; SSE2-NEXT:    shufps {{.*#+}} xmm5 = xmm5[1,1],xmm0[2,3]
-; SSE2-NEXT:    movss %xmm5, 4(%rdi)
-; SSE2-NEXT:  LBB6_4: ## %else2
-; SSE2-NEXT:    pxor %xmm5, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm4, %xmm5
-; SSE2-NEXT:    pextrw $4, %xmm5, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:  LBB6_3: ## %cond.store1
+; SSE2-NEXT:    movaps %xmm0, %xmm4
+; SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[1,1],xmm0[2,3]
+; SSE2-NEXT:    movss %xmm4, 4(%rdi)
+; SSE2-NEXT:    testb $4, %al
 ; SSE2-NEXT:    je LBB6_6
-; SSE2-NEXT:  ## %bb.5: ## %cond.store3
+; SSE2-NEXT:  LBB6_5: ## %cond.store3
 ; SSE2-NEXT:    movaps %xmm0, %xmm4
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm0[1]
 ; SSE2-NEXT:    movss %xmm4, 8(%rdi)
-; SSE2-NEXT:  LBB6_6: ## %else4
-; SSE2-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm4
-; SSE2-NEXT:    pextrw $6, %xmm5, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je LBB6_8
-; SSE2-NEXT:  ## %bb.7: ## %cond.store5
+; SSE2-NEXT:  LBB6_7: ## %cond.store5
 ; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
 ; SSE2-NEXT:    movss %xmm0, 12(%rdi)
-; SSE2-NEXT:  LBB6_8: ## %else6
-; SSE2-NEXT:    xorps %xmm0, %xmm0
-; SSE2-NEXT:    pcmpgtd %xmm4, %xmm0
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $16, %al
 ; SSE2-NEXT:    je LBB6_10
-; SSE2-NEXT:  ## %bb.9: ## %cond.store7
+; SSE2-NEXT:  LBB6_9: ## %cond.store7
 ; SSE2-NEXT:    movss %xmm1, 16(%rdi)
-; SSE2-NEXT:  LBB6_10: ## %else8
-; SSE2-NEXT:    pextrw $2, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $32, %al
 ; SSE2-NEXT:    je LBB6_12
-; SSE2-NEXT:  ## %bb.11: ## %cond.store9
+; SSE2-NEXT:  LBB6_11: ## %cond.store9
 ; SSE2-NEXT:    movaps %xmm1, %xmm0
 ; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm1[2,3]
 ; SSE2-NEXT:    movss %xmm0, 20(%rdi)
-; SSE2-NEXT:  LBB6_12: ## %else10
-; SSE2-NEXT:    pxor %xmm5, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm4, %xmm5
-; SSE2-NEXT:    pextrw $4, %xmm5, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $64, %al
 ; SSE2-NEXT:    je LBB6_14
-; SSE2-NEXT:  ## %bb.13: ## %cond.store11
+; SSE2-NEXT:  LBB6_13: ## %cond.store11
 ; SSE2-NEXT:    movaps %xmm1, %xmm0
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1]
 ; SSE2-NEXT:    movss %xmm0, 24(%rdi)
-; SSE2-NEXT:  LBB6_14: ## %else12
-; SSE2-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm0
-; SSE2-NEXT:    pextrw $6, %xmm5, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $-128, %al
 ; SSE2-NEXT:    je LBB6_16
-; SSE2-NEXT:  ## %bb.15: ## %cond.store13
+; SSE2-NEXT:  LBB6_15: ## %cond.store13
 ; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,1,2,3]
 ; SSE2-NEXT:    movss %xmm1, 28(%rdi)
-; SSE2-NEXT:  LBB6_16: ## %else14
-; SSE2-NEXT:    xorps %xmm1, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
-; SSE2-NEXT:    movd %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $256, %eax ## imm = 0x100
 ; SSE2-NEXT:    je LBB6_18
-; SSE2-NEXT:  ## %bb.17: ## %cond.store15
+; SSE2-NEXT:  LBB6_17: ## %cond.store15
 ; SSE2-NEXT:    movss %xmm2, 32(%rdi)
-; SSE2-NEXT:  LBB6_18: ## %else16
-; SSE2-NEXT:    pextrw $2, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $512, %eax ## imm = 0x200
 ; SSE2-NEXT:    je LBB6_20
-; SSE2-NEXT:  ## %bb.19: ## %cond.store17
-; SSE2-NEXT:    movaps %xmm2, %xmm1
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm2[2,3]
-; SSE2-NEXT:    movss %xmm1, 36(%rdi)
-; SSE2-NEXT:  LBB6_20: ## %else18
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
-; SSE2-NEXT:    pextrw $4, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:  LBB6_19: ## %cond.store17
+; SSE2-NEXT:    movaps %xmm2, %xmm0
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm2[2,3]
+; SSE2-NEXT:    movss %xmm0, 36(%rdi)
+; SSE2-NEXT:    testl $1024, %eax ## imm = 0x400
 ; SSE2-NEXT:    je LBB6_22
-; SSE2-NEXT:  ## %bb.21: ## %cond.store19
+; SSE2-NEXT:  LBB6_21: ## %cond.store19
 ; SSE2-NEXT:    movaps %xmm2, %xmm0
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1],xmm2[1]
 ; SSE2-NEXT:    movss %xmm0, 40(%rdi)
-; SSE2-NEXT:  LBB6_22: ## %else20
-; SSE2-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm0
-; SSE2-NEXT:    pextrw $6, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $2048, %eax ## imm = 0x800
 ; SSE2-NEXT:    je LBB6_24
-; SSE2-NEXT:  ## %bb.23: ## %cond.store21
+; SSE2-NEXT:  LBB6_23: ## %cond.store21
 ; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[3,1,2,3]
 ; SSE2-NEXT:    movss %xmm2, 44(%rdi)
-; SSE2-NEXT:  LBB6_24: ## %else22
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
-; SSE2-NEXT:    movd %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $4096, %eax ## imm = 0x1000
 ; SSE2-NEXT:    je LBB6_26
-; SSE2-NEXT:  ## %bb.25: ## %cond.store23
+; SSE2-NEXT:  LBB6_25: ## %cond.store23
 ; SSE2-NEXT:    movss %xmm3, 48(%rdi)
-; SSE2-NEXT:  LBB6_26: ## %else24
-; SSE2-NEXT:    pextrw $2, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $8192, %eax ## imm = 0x2000
 ; SSE2-NEXT:    je LBB6_28
-; SSE2-NEXT:  ## %bb.27: ## %cond.store25
-; SSE2-NEXT:    movaps %xmm3, %xmm1
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm3[2,3]
-; SSE2-NEXT:    movss %xmm1, 52(%rdi)
-; SSE2-NEXT:  LBB6_28: ## %else26
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
-; SSE2-NEXT:    pextrw $4, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:  LBB6_27: ## %cond.store25
+; SSE2-NEXT:    movaps %xmm3, %xmm0
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm3[2,3]
+; SSE2-NEXT:    movss %xmm0, 52(%rdi)
+; SSE2-NEXT:    testl $16384, %eax ## imm = 0x4000
 ; SSE2-NEXT:    je LBB6_30
-; SSE2-NEXT:  ## %bb.29: ## %cond.store27
+; SSE2-NEXT:  LBB6_29: ## %cond.store27
 ; SSE2-NEXT:    movaps %xmm3, %xmm0
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1],xmm3[1]
 ; SSE2-NEXT:    movss %xmm0, 56(%rdi)
-; SSE2-NEXT:  LBB6_30: ## %else28
-; SSE2-NEXT:    pextrw $6, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $32768, %eax ## imm = 0x8000
 ; SSE2-NEXT:    je LBB6_32
-; SSE2-NEXT:  ## %bb.31: ## %cond.store29
+; SSE2-NEXT:  LBB6_31: ## %cond.store29
 ; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,1,2,3]
 ; SSE2-NEXT:    movss %xmm3, 60(%rdi)
-; SSE2-NEXT:  LBB6_32: ## %else30
 ; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: store_v16f32_v16i32:
 ; SSE4:       ## %bb.0:
 ; SSE4-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm4
-; SSE4-NEXT:    pxor %xmm5, %xmm5
-; SSE4-NEXT:    pcmpgtd %xmm4, %xmm5
-; SSE4-NEXT:    pextrb $0, %xmm5, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je LBB6_2
-; SSE4-NEXT:  ## %bb.1: ## %cond.store
-; SSE4-NEXT:    movd %xmm0, (%rdi)
-; SSE4-NEXT:  LBB6_2: ## %else
-; SSE4-NEXT:    pextrb $4, %xmm5, %eax
+; SSE4-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm5
+; SSE4-NEXT:    packssdw {{[0-9]+}}(%rsp), %xmm5
+; SSE4-NEXT:    packssdw {{[0-9]+}}(%rsp), %xmm4
+; SSE4-NEXT:    packsswb %xmm5, %xmm4
+; SSE4-NEXT:    pmovmskb %xmm4, %eax
 ; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je LBB6_4
-; SSE4-NEXT:  ## %bb.3: ## %cond.store1
-; SSE4-NEXT:    pextrd $1, %xmm0, 4(%rdi)
+; SSE4-NEXT:    jne LBB6_1
+; SSE4-NEXT:  ## %bb.2: ## %else
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    jne LBB6_3
 ; SSE4-NEXT:  LBB6_4: ## %else2
-; SSE4-NEXT:    pxor %xmm5, %xmm5
-; SSE4-NEXT:    pcmpgtd %xmm4, %xmm5
-; SSE4-NEXT:    pextrb $8, %xmm5, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je LBB6_6
-; SSE4-NEXT:  ## %bb.5: ## %cond.store3
-; SSE4-NEXT:    pextrd $2, %xmm0, 8(%rdi)
+; SSE4-NEXT:    testb $4, %al
+; SSE4-NEXT:    jne LBB6_5
 ; SSE4-NEXT:  LBB6_6: ## %else4
-; SSE4-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm4
-; SSE4-NEXT:    pextrb $12, %xmm5, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je LBB6_8
-; SSE4-NEXT:  ## %bb.7: ## %cond.store5
-; SSE4-NEXT:    pextrd $3, %xmm0, 12(%rdi)
+; SSE4-NEXT:    testb $8, %al
+; SSE4-NEXT:    jne LBB6_7
 ; SSE4-NEXT:  LBB6_8: ## %else6
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpgtd %xmm4, %xmm0
-; SSE4-NEXT:    pextrb $0, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je LBB6_10
-; SSE4-NEXT:  ## %bb.9: ## %cond.store7
-; SSE4-NEXT:    movd %xmm1, 16(%rdi)
+; SSE4-NEXT:    testb $16, %al
+; SSE4-NEXT:    jne LBB6_9
 ; SSE4-NEXT:  LBB6_10: ## %else8
-; SSE4-NEXT:    pextrb $4, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je LBB6_12
-; SSE4-NEXT:  ## %bb.11: ## %cond.store9
-; SSE4-NEXT:    pextrd $1, %xmm1, 20(%rdi)
+; SSE4-NEXT:    testb $32, %al
+; SSE4-NEXT:    jne LBB6_11
 ; SSE4-NEXT:  LBB6_12: ## %else10
-; SSE4-NEXT:    pxor %xmm5, %xmm5
-; SSE4-NEXT:    pcmpgtd %xmm4, %xmm5
-; SSE4-NEXT:    pextrb $8, %xmm5, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je LBB6_14
-; SSE4-NEXT:  ## %bb.13: ## %cond.store11
-; SSE4-NEXT:    pextrd $2, %xmm1, 24(%rdi)
+; SSE4-NEXT:    testb $64, %al
+; SSE4-NEXT:    jne LBB6_13
 ; SSE4-NEXT:  LBB6_14: ## %else12
-; SSE4-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm0
-; SSE4-NEXT:    pextrb $12, %xmm5, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je LBB6_16
-; SSE4-NEXT:  ## %bb.15: ## %cond.store13
-; SSE4-NEXT:    pextrd $3, %xmm1, 28(%rdi)
+; SSE4-NEXT:    testb $-128, %al
+; SSE4-NEXT:    jne LBB6_15
 ; SSE4-NEXT:  LBB6_16: ## %else14
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpgtd %xmm0, %xmm1
-; SSE4-NEXT:    pextrb $0, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je LBB6_18
-; SSE4-NEXT:  ## %bb.17: ## %cond.store15
-; SSE4-NEXT:    movss %xmm2, 32(%rdi)
+; SSE4-NEXT:    testl $256, %eax ## imm = 0x100
+; SSE4-NEXT:    jne LBB6_17
 ; SSE4-NEXT:  LBB6_18: ## %else16
-; SSE4-NEXT:    pextrb $4, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je LBB6_20
-; SSE4-NEXT:  ## %bb.19: ## %cond.store17
-; SSE4-NEXT:    extractps $1, %xmm2, 36(%rdi)
+; SSE4-NEXT:    testl $512, %eax ## imm = 0x200
+; SSE4-NEXT:    jne LBB6_19
 ; SSE4-NEXT:  LBB6_20: ## %else18
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpgtd %xmm0, %xmm1
-; SSE4-NEXT:    pextrb $8, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je LBB6_22
-; SSE4-NEXT:  ## %bb.21: ## %cond.store19
-; SSE4-NEXT:    extractps $2, %xmm2, 40(%rdi)
+; SSE4-NEXT:    testl $1024, %eax ## imm = 0x400
+; SSE4-NEXT:    jne LBB6_21
 ; SSE4-NEXT:  LBB6_22: ## %else20
-; SSE4-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm0
-; SSE4-NEXT:    pextrb $12, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je LBB6_24
-; SSE4-NEXT:  ## %bb.23: ## %cond.store21
-; SSE4-NEXT:    extractps $3, %xmm2, 44(%rdi)
+; SSE4-NEXT:    testl $2048, %eax ## imm = 0x800
+; SSE4-NEXT:    jne LBB6_23
 ; SSE4-NEXT:  LBB6_24: ## %else22
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpgtd %xmm0, %xmm1
-; SSE4-NEXT:    pextrb $0, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je LBB6_26
-; SSE4-NEXT:  ## %bb.25: ## %cond.store23
-; SSE4-NEXT:    movss %xmm3, 48(%rdi)
+; SSE4-NEXT:    testl $4096, %eax ## imm = 0x1000
+; SSE4-NEXT:    jne LBB6_25
 ; SSE4-NEXT:  LBB6_26: ## %else24
-; SSE4-NEXT:    pextrb $4, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je LBB6_28
-; SSE4-NEXT:  ## %bb.27: ## %cond.store25
-; SSE4-NEXT:    extractps $1, %xmm3, 52(%rdi)
+; SSE4-NEXT:    testl $8192, %eax ## imm = 0x2000
+; SSE4-NEXT:    jne LBB6_27
 ; SSE4-NEXT:  LBB6_28: ## %else26
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpgtd %xmm0, %xmm1
-; SSE4-NEXT:    pextrb $8, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je LBB6_30
-; SSE4-NEXT:  ## %bb.29: ## %cond.store27
-; SSE4-NEXT:    extractps $2, %xmm3, 56(%rdi)
+; SSE4-NEXT:    testl $16384, %eax ## imm = 0x4000
+; SSE4-NEXT:    jne LBB6_29
 ; SSE4-NEXT:  LBB6_30: ## %else28
-; SSE4-NEXT:    pextrb $12, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $32768, %eax ## imm = 0x8000
+; SSE4-NEXT:    jne LBB6_31
+; SSE4-NEXT:  LBB6_32: ## %else30
+; SSE4-NEXT:    retq
+; SSE4-NEXT:  LBB6_1: ## %cond.store
+; SSE4-NEXT:    movss %xmm0, (%rdi)
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    je LBB6_4
+; SSE4-NEXT:  LBB6_3: ## %cond.store1
+; SSE4-NEXT:    extractps $1, %xmm0, 4(%rdi)
+; SSE4-NEXT:    testb $4, %al
+; SSE4-NEXT:    je LBB6_6
+; SSE4-NEXT:  LBB6_5: ## %cond.store3
+; SSE4-NEXT:    extractps $2, %xmm0, 8(%rdi)
+; SSE4-NEXT:    testb $8, %al
+; SSE4-NEXT:    je LBB6_8
+; SSE4-NEXT:  LBB6_7: ## %cond.store5
+; SSE4-NEXT:    extractps $3, %xmm0, 12(%rdi)
+; SSE4-NEXT:    testb $16, %al
+; SSE4-NEXT:    je LBB6_10
+; SSE4-NEXT:  LBB6_9: ## %cond.store7
+; SSE4-NEXT:    movss %xmm1, 16(%rdi)
+; SSE4-NEXT:    testb $32, %al
+; SSE4-NEXT:    je LBB6_12
+; SSE4-NEXT:  LBB6_11: ## %cond.store9
+; SSE4-NEXT:    extractps $1, %xmm1, 20(%rdi)
+; SSE4-NEXT:    testb $64, %al
+; SSE4-NEXT:    je LBB6_14
+; SSE4-NEXT:  LBB6_13: ## %cond.store11
+; SSE4-NEXT:    extractps $2, %xmm1, 24(%rdi)
+; SSE4-NEXT:    testb $-128, %al
+; SSE4-NEXT:    je LBB6_16
+; SSE4-NEXT:  LBB6_15: ## %cond.store13
+; SSE4-NEXT:    extractps $3, %xmm1, 28(%rdi)
+; SSE4-NEXT:    testl $256, %eax ## imm = 0x100
+; SSE4-NEXT:    je LBB6_18
+; SSE4-NEXT:  LBB6_17: ## %cond.store15
+; SSE4-NEXT:    movss %xmm2, 32(%rdi)
+; SSE4-NEXT:    testl $512, %eax ## imm = 0x200
+; SSE4-NEXT:    je LBB6_20
+; SSE4-NEXT:  LBB6_19: ## %cond.store17
+; SSE4-NEXT:    extractps $1, %xmm2, 36(%rdi)
+; SSE4-NEXT:    testl $1024, %eax ## imm = 0x400
+; SSE4-NEXT:    je LBB6_22
+; SSE4-NEXT:  LBB6_21: ## %cond.store19
+; SSE4-NEXT:    extractps $2, %xmm2, 40(%rdi)
+; SSE4-NEXT:    testl $2048, %eax ## imm = 0x800
+; SSE4-NEXT:    je LBB6_24
+; SSE4-NEXT:  LBB6_23: ## %cond.store21
+; SSE4-NEXT:    extractps $3, %xmm2, 44(%rdi)
+; SSE4-NEXT:    testl $4096, %eax ## imm = 0x1000
+; SSE4-NEXT:    je LBB6_26
+; SSE4-NEXT:  LBB6_25: ## %cond.store23
+; SSE4-NEXT:    movss %xmm3, 48(%rdi)
+; SSE4-NEXT:    testl $8192, %eax ## imm = 0x2000
+; SSE4-NEXT:    je LBB6_28
+; SSE4-NEXT:  LBB6_27: ## %cond.store25
+; SSE4-NEXT:    extractps $1, %xmm3, 52(%rdi)
+; SSE4-NEXT:    testl $16384, %eax ## imm = 0x4000
+; SSE4-NEXT:    je LBB6_30
+; SSE4-NEXT:  LBB6_29: ## %cond.store27
+; SSE4-NEXT:    extractps $2, %xmm3, 56(%rdi)
+; SSE4-NEXT:    testl $32768, %eax ## imm = 0x8000
 ; SSE4-NEXT:    je LBB6_32
-; SSE4-NEXT:  ## %bb.31: ## %cond.store29
+; SSE4-NEXT:  LBB6_31: ## %cond.store29
 ; SSE4-NEXT:    extractps $3, %xmm3, 60(%rdi)
-; SSE4-NEXT:  LBB6_32: ## %else30
 ; SSE4-NEXT:    retq
 ;
 ; AVX1OR2-LABEL: store_v16f32_v16i32:
@@ -893,46 +825,39 @@ define void @store_v16f32_v16i32(<16 x float> %x, <16 x float>* %ptr, <16 x floa
 define void @store_v2i64_v2i64(<2 x i64> %trigger, <2 x i64>* %addr, <2 x i64> %val) {
 ; SSE2-LABEL: store_v2i64_v2i64:
 ; SSE2:       ## %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
-; SSE2-NEXT:    pxor %xmm2, %xmm0
-; SSE2-NEXT:    movdqa %xmm2, %xmm3
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pand %xmm3, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE2-NEXT:    por %xmm2, %xmm0
-; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    movmskpd %xmm0, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB7_2
-; SSE2-NEXT:  ## %bb.1: ## %cond.store
+; SSE2-NEXT:    jne LBB7_1
+; SSE2-NEXT:  ## %bb.2: ## %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne LBB7_3
+; SSE2-NEXT:  LBB7_4: ## %else2
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  LBB7_1: ## %cond.store
 ; SSE2-NEXT:    movq %xmm1, (%rdi)
-; SSE2-NEXT:  LBB7_2: ## %else
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $2, %al
 ; SSE2-NEXT:    je LBB7_4
-; SSE2-NEXT:  ## %bb.3: ## %cond.store1
+; SSE2-NEXT:  LBB7_3: ## %cond.store1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
 ; SSE2-NEXT:    movq %xmm0, 8(%rdi)
-; SSE2-NEXT:  LBB7_4: ## %else2
 ; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: store_v2i64_v2i64:
 ; SSE4:       ## %bb.0:
-; SSE4-NEXT:    pxor %xmm2, %xmm2
-; SSE4-NEXT:    pcmpgtq %xmm0, %xmm2
-; SSE4-NEXT:    pextrb $0, %xmm2, %eax
+; SSE4-NEXT:    movmskpd %xmm0, %eax
 ; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je LBB7_2
-; SSE4-NEXT:  ## %bb.1: ## %cond.store
+; SSE4-NEXT:    jne LBB7_1
+; SSE4-NEXT:  ## %bb.2: ## %else
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    jne LBB7_3
+; SSE4-NEXT:  LBB7_4: ## %else2
+; SSE4-NEXT:    retq
+; SSE4-NEXT:  LBB7_1: ## %cond.store
 ; SSE4-NEXT:    movq %xmm1, (%rdi)
-; SSE4-NEXT:  LBB7_2: ## %else
-; SSE4-NEXT:    pextrb $8, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $2, %al
 ; SSE4-NEXT:    je LBB7_4
-; SSE4-NEXT:  ## %bb.3: ## %cond.store1
+; SSE4-NEXT:  LBB7_3: ## %cond.store1
 ; SSE4-NEXT:    pextrq $1, %xmm1, 8(%rdi)
-; SSE4-NEXT:  LBB7_4: ## %else2
 ; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: store_v2i64_v2i64:
@@ -977,81 +902,70 @@ define void @store_v2i64_v2i64(<2 x i64> %trigger, <2 x i64>* %addr, <2 x i64> %
 define void @store_v4i64_v4i64(<4 x i64> %trigger, <4 x i64>* %addr, <4 x i64> %val) {
 ; SSE2-LABEL: store_v4i64_v4i64:
 ; SSE2:       ## %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [2147483648,2147483648]
-; SSE2-NEXT:    pxor %xmm4, %xmm0
-; SSE2-NEXT:    movdqa %xmm4, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm5
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pand %xmm5, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[1,1,3,3]
-; SSE2-NEXT:    por %xmm6, %xmm0
-; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    packssdw %xmm1, %xmm0
+; SSE2-NEXT:    movmskps %xmm0, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB8_2
-; SSE2-NEXT:  ## %bb.1: ## %cond.store
+; SSE2-NEXT:    jne LBB8_1
+; SSE2-NEXT:  ## %bb.2: ## %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne LBB8_3
+; SSE2-NEXT:  LBB8_4: ## %else2
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne LBB8_5
+; SSE2-NEXT:  LBB8_6: ## %else4
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne LBB8_7
+; SSE2-NEXT:  LBB8_8: ## %else6
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  LBB8_1: ## %cond.store
 ; SSE2-NEXT:    movq %xmm2, (%rdi)
-; SSE2-NEXT:  LBB8_2: ## %else
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $2, %al
 ; SSE2-NEXT:    je LBB8_4
-; SSE2-NEXT:  ## %bb.3: ## %cond.store1
+; SSE2-NEXT:  LBB8_3: ## %cond.store1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
 ; SSE2-NEXT:    movq %xmm0, 8(%rdi)
-; SSE2-NEXT:  LBB8_4: ## %else2
-; SSE2-NEXT:    pxor %xmm4, %xmm1
-; SSE2-NEXT:    movdqa %xmm4, %xmm0
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pand %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm0
-; SSE2-NEXT:    pextrw $0, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $4, %al
 ; SSE2-NEXT:    je LBB8_6
-; SSE2-NEXT:  ## %bb.5: ## %cond.store3
+; SSE2-NEXT:  LBB8_5: ## %cond.store3
 ; SSE2-NEXT:    movq %xmm3, 16(%rdi)
-; SSE2-NEXT:  LBB8_6: ## %else4
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je LBB8_8
-; SSE2-NEXT:  ## %bb.7: ## %cond.store5
+; SSE2-NEXT:  LBB8_7: ## %cond.store5
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[2,3,0,1]
 ; SSE2-NEXT:    movq %xmm0, 24(%rdi)
-; SSE2-NEXT:  LBB8_8: ## %else6
 ; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: store_v4i64_v4i64:
 ; SSE4:       ## %bb.0:
-; SSE4-NEXT:    pxor %xmm4, %xmm4
-; SSE4-NEXT:    pcmpgtq %xmm0, %xmm4
-; SSE4-NEXT:    pextrb $0, %xmm4, %eax
+; SSE4-NEXT:    packssdw %xmm1, %xmm0
+; SSE4-NEXT:    movmskps %xmm0, %eax
 ; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je LBB8_2
-; SSE4-NEXT:  ## %bb.1: ## %cond.store
+; SSE4-NEXT:    jne LBB8_1
+; SSE4-NEXT:  ## %bb.2: ## %else
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    jne LBB8_3
+; SSE4-NEXT:  LBB8_4: ## %else2
+; SSE4-NEXT:    testb $4, %al
+; SSE4-NEXT:    jne LBB8_5
+; SSE4-NEXT:  LBB8_6: ## %else4
+; SSE4-NEXT:    testb $8, %al
+; SSE4-NEXT:    jne LBB8_7
+; SSE4-NEXT:  LBB8_8: ## %else6
+; SSE4-NEXT:    retq
+; SSE4-NEXT:  LBB8_1: ## %cond.store
 ; SSE4-NEXT:    movq %xmm2, (%rdi)
-; SSE4-NEXT:  LBB8_2: ## %else
-; SSE4-NEXT:    pextrb $8, %xmm4, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $2, %al
 ; SSE4-NEXT:    je LBB8_4
-; SSE4-NEXT:  ## %bb.3: ## %cond.store1
+; SSE4-NEXT:  LBB8_3: ## %cond.store1
 ; SSE4-NEXT:    pextrq $1, %xmm2, 8(%rdi)
-; SSE4-NEXT:  LBB8_4: ## %else2
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpgtq %xmm1, %xmm0
-; SSE4-NEXT:    pextrb $0, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $4, %al
 ; SSE4-NEXT:    je LBB8_6
-; SSE4-NEXT:  ## %bb.5: ## %cond.store3
+; SSE4-NEXT:  LBB8_5: ## %cond.store3
 ; SSE4-NEXT:    movq %xmm3, 16(%rdi)
-; SSE4-NEXT:  LBB8_6: ## %else4
-; SSE4-NEXT:    pextrb $8, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $8, %al
 ; SSE4-NEXT:    je LBB8_8
-; SSE4-NEXT:  ## %bb.7: ## %cond.store5
+; SSE4-NEXT:  LBB8_7: ## %cond.store5
 ; SSE4-NEXT:    pextrq $1, %xmm3, 24(%rdi)
-; SSE4-NEXT:  LBB8_8: ## %else6
 ; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: store_v4i64_v4i64:
@@ -1127,84 +1041,89 @@ define void @store_v1i32_v1i32(<1 x i32> %trigger, <1 x i32>* %addr, <1 x i32> %
 define void @store_v2i32_v2i32(<2 x i32> %trigger, <2 x i32>* %addr, <2 x i32> %val) {
 ; SSE2-LABEL: store_v2i32_v2i32:
 ; SSE2:       ## %bb.0:
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
 ; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,0,3,2]
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,0,1,1]
+; SSE2-NEXT:    movmskpd %xmm0, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB10_2
-; SSE2-NEXT:  ## %bb.1: ## %cond.store
+; SSE2-NEXT:    jne LBB10_1
+; SSE2-NEXT:  ## %bb.2: ## %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne LBB10_3
+; SSE2-NEXT:  LBB10_4: ## %else2
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  LBB10_1: ## %cond.store
 ; SSE2-NEXT:    movd %xmm1, (%rdi)
-; SSE2-NEXT:  LBB10_2: ## %else
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $2, %al
 ; SSE2-NEXT:    je LBB10_4
-; SSE2-NEXT:  ## %bb.3: ## %cond.store1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE2-NEXT:  LBB10_3: ## %cond.store1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE2-NEXT:    movd %xmm0, 4(%rdi)
-; SSE2-NEXT:  LBB10_4: ## %else2
 ; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: store_v2i32_v2i32:
 ; SSE4:       ## %bb.0:
 ; SSE4-NEXT:    pxor %xmm2, %xmm2
-; SSE4-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
-; SSE4-NEXT:    pcmpeqq %xmm2, %xmm0
-; SSE4-NEXT:    pextrb $0, %xmm0, %eax
+; SSE4-NEXT:    pcmpeqd %xmm0, %xmm2
+; SSE4-NEXT:    pmovsxdq %xmm2, %xmm0
+; SSE4-NEXT:    movmskpd %xmm0, %eax
 ; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je LBB10_2
-; SSE4-NEXT:  ## %bb.1: ## %cond.store
+; SSE4-NEXT:    jne LBB10_1
+; SSE4-NEXT:  ## %bb.2: ## %else
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    jne LBB10_3
+; SSE4-NEXT:  LBB10_4: ## %else2
+; SSE4-NEXT:    retq
+; SSE4-NEXT:  LBB10_1: ## %cond.store
 ; SSE4-NEXT:    movss %xmm1, (%rdi)
-; SSE4-NEXT:  LBB10_2: ## %else
-; SSE4-NEXT:    pextrb $8, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $2, %al
 ; SSE4-NEXT:    je LBB10_4
-; SSE4-NEXT:  ## %bb.3: ## %cond.store1
-; SSE4-NEXT:    extractps $2, %xmm1, 4(%rdi)
-; SSE4-NEXT:  LBB10_4: ## %else2
+; SSE4-NEXT:  LBB10_3: ## %cond.store1
+; SSE4-NEXT:    extractps $1, %xmm1, 4(%rdi)
 ; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: store_v2i32_v2i32:
 ; AVX1:       ## %bb.0:
 ; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
-; AVX1-NEXT:    vpcmpeqq %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,2],zero,zero
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; AVX1-NEXT:    vmaskmovps %xmm1, %xmm0, (%rdi)
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: store_v2i32_v2i32:
 ; AVX2:       ## %bb.0:
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
-; AVX2-NEXT:    vpcmpeqq %xmm2, %xmm0, %xmm0
-; AVX2-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,2],zero,zero
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; AVX2-NEXT:    vpmaskmovd %xmm1, %xmm0, (%rdi)
 ; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: store_v2i32_v2i32:
 ; AVX512F:       ## %bb.0:
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
-; AVX512F-NEXT:    vptestnmq %zmm0, %zmm0, %k0
-; AVX512F-NEXT:    vpshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
+; AVX512F-NEXT:    ## kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512F-NEXT:    ## kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512F-NEXT:    vptestnmd %zmm0, %zmm0, %k0
 ; AVX512F-NEXT:    kshiftlw $14, %k0, %k0
 ; AVX512F-NEXT:    kshiftrw $14, %k0, %k1
-; AVX512F-NEXT:    vmovdqu32 %zmm0, (%rdi) {%k1}
+; AVX512F-NEXT:    vmovdqu32 %zmm1, (%rdi) {%k1}
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
-; AVX512VL-LABEL: store_v2i32_v2i32:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VL-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
-; AVX512VL-NEXT:    vptestnmq %xmm0, %xmm0, %k1
-; AVX512VL-NEXT:    vpmovqd %xmm1, (%rdi) {%k1}
-; AVX512VL-NEXT:    retq
+; AVX512VLDQ-LABEL: store_v2i32_v2i32:
+; AVX512VLDQ:       ## %bb.0:
+; AVX512VLDQ-NEXT:    vptestnmd %xmm0, %xmm0, %k0
+; AVX512VLDQ-NEXT:    kshiftlb $6, %k0, %k0
+; AVX512VLDQ-NEXT:    kshiftrb $6, %k0, %k1
+; AVX512VLDQ-NEXT:    vmovdqu32 %xmm1, (%rdi) {%k1}
+; AVX512VLDQ-NEXT:    retq
+;
+; AVX512VLBW-LABEL: store_v2i32_v2i32:
+; AVX512VLBW:       ## %bb.0:
+; AVX512VLBW-NEXT:    vptestnmd %xmm0, %xmm0, %k0
+; AVX512VLBW-NEXT:    kshiftlw $14, %k0, %k0
+; AVX512VLBW-NEXT:    kshiftrw $14, %k0, %k1
+; AVX512VLBW-NEXT:    vmovdqu32 %xmm1, (%rdi) {%k1}
+; AVX512VLBW-NEXT:    retq
   %mask = icmp eq <2 x i32> %trigger, zeroinitializer
   call void @llvm.masked.store.v2i32.p0v2i32(<2 x i32> %val, <2 x i32>* %addr, i32 4, <2 x i1> %mask)
   ret void
@@ -1215,67 +1134,71 @@ define void @store_v4i32_v4i32(<4 x i32> %trigger, <4 x i32>* %addr, <4 x i32> %
 ; SSE2:       ## %bb.0:
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
 ; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    movd %xmm2, %eax
+; SSE2-NEXT:    movmskps %xmm2, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB11_2
-; SSE2-NEXT:  ## %bb.1: ## %cond.store
+; SSE2-NEXT:    jne LBB11_1
+; SSE2-NEXT:  ## %bb.2: ## %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne LBB11_3
+; SSE2-NEXT:  LBB11_4: ## %else2
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne LBB11_5
+; SSE2-NEXT:  LBB11_6: ## %else4
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne LBB11_7
+; SSE2-NEXT:  LBB11_8: ## %else6
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  LBB11_1: ## %cond.store
 ; SSE2-NEXT:    movd %xmm1, (%rdi)
-; SSE2-NEXT:  LBB11_2: ## %else
-; SSE2-NEXT:    pextrw $2, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $2, %al
 ; SSE2-NEXT:    je LBB11_4
-; SSE2-NEXT:  ## %bb.3: ## %cond.store1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,2,3]
-; SSE2-NEXT:    movd %xmm2, 4(%rdi)
-; SSE2-NEXT:  LBB11_4: ## %else2
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm0
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:  LBB11_3: ## %cond.store1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE2-NEXT:    movd %xmm0, 4(%rdi)
+; SSE2-NEXT:    testb $4, %al
 ; SSE2-NEXT:    je LBB11_6
-; SSE2-NEXT:  ## %bb.5: ## %cond.store3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
-; SSE2-NEXT:    movd %xmm2, 8(%rdi)
-; SSE2-NEXT:  LBB11_6: ## %else4
-; SSE2-NEXT:    pextrw $6, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:  LBB11_5: ## %cond.store3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE2-NEXT:    movd %xmm0, 8(%rdi)
+; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je LBB11_8
-; SSE2-NEXT:  ## %bb.7: ## %cond.store5
+; SSE2-NEXT:  LBB11_7: ## %cond.store5
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[3,1,2,3]
 ; SSE2-NEXT:    movd %xmm0, 12(%rdi)
-; SSE2-NEXT:  LBB11_8: ## %else6
 ; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: store_v4i32_v4i32:
 ; SSE4:       ## %bb.0:
 ; SSE4-NEXT:    pxor %xmm2, %xmm2
 ; SSE4-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE4-NEXT:    pextrb $0, %xmm2, %eax
+; SSE4-NEXT:    movmskps %xmm2, %eax
 ; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je LBB11_2
-; SSE4-NEXT:  ## %bb.1: ## %cond.store
+; SSE4-NEXT:    jne LBB11_1
+; SSE4-NEXT:  ## %bb.2: ## %else
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    jne LBB11_3
+; SSE4-NEXT:  LBB11_4: ## %else2
+; SSE4-NEXT:    testb $4, %al
+; SSE4-NEXT:    jne LBB11_5
+; SSE4-NEXT:  LBB11_6: ## %else4
+; SSE4-NEXT:    testb $8, %al
+; SSE4-NEXT:    jne LBB11_7
+; SSE4-NEXT:  LBB11_8: ## %else6
+; SSE4-NEXT:    retq
+; SSE4-NEXT:  LBB11_1: ## %cond.store
 ; SSE4-NEXT:    movss %xmm1, (%rdi)
-; SSE4-NEXT:  LBB11_2: ## %else
-; SSE4-NEXT:    pextrb $4, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $2, %al
 ; SSE4-NEXT:    je LBB11_4
-; SSE4-NEXT:  ## %bb.3: ## %cond.store1
+; SSE4-NEXT:  LBB11_3: ## %cond.store1
 ; SSE4-NEXT:    extractps $1, %xmm1, 4(%rdi)
-; SSE4-NEXT:  LBB11_4: ## %else2
-; SSE4-NEXT:    pxor %xmm2, %xmm2
-; SSE4-NEXT:    pcmpeqd %xmm2, %xmm0
-; SSE4-NEXT:    pextrb $8, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $4, %al
 ; SSE4-NEXT:    je LBB11_6
-; SSE4-NEXT:  ## %bb.5: ## %cond.store3
+; SSE4-NEXT:  LBB11_5: ## %cond.store3
 ; SSE4-NEXT:    extractps $2, %xmm1, 8(%rdi)
-; SSE4-NEXT:  LBB11_6: ## %else4
-; SSE4-NEXT:    pextrb $12, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $8, %al
 ; SSE4-NEXT:    je LBB11_8
-; SSE4-NEXT:  ## %bb.7: ## %cond.store5
+; SSE4-NEXT:  LBB11_7: ## %cond.store5
 ; SSE4-NEXT:    extractps $3, %xmm1, 12(%rdi)
-; SSE4-NEXT:  LBB11_8: ## %else6
 ; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: store_v4i32_v4i32:
@@ -1317,131 +1240,137 @@ define void @store_v8i32_v8i32(<8 x i32> %trigger, <8 x i32>* %addr, <8 x i32> %
 ; SSE2-LABEL: store_v8i32_v8i32:
 ; SSE2:       ## %bb.0:
 ; SSE2-NEXT:    pxor %xmm4, %xmm4
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm4
-; SSE2-NEXT:    movdqa %xmm4, %xmm5
-; SSE2-NEXT:    packssdw %xmm0, %xmm5
-; SSE2-NEXT:    movd %xmm5, %eax
+; SSE2-NEXT:    pcmpeqd %xmm4, %xmm1
+; SSE2-NEXT:    pcmpeqd %xmm4, %xmm0
+; SSE2-NEXT:    packssdw %xmm1, %xmm0
+; SSE2-NEXT:    packsswb %xmm0, %xmm0
+; SSE2-NEXT:    pmovmskb %xmm0, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB12_2
-; SSE2-NEXT:  ## %bb.1: ## %cond.store
+; SSE2-NEXT:    jne LBB12_1
+; SSE2-NEXT:  ## %bb.2: ## %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne LBB12_3
+; SSE2-NEXT:  LBB12_4: ## %else2
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne LBB12_5
+; SSE2-NEXT:  LBB12_6: ## %else4
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne LBB12_7
+; SSE2-NEXT:  LBB12_8: ## %else6
+; SSE2-NEXT:    testb $16, %al
+; SSE2-NEXT:    jne LBB12_9
+; SSE2-NEXT:  LBB12_10: ## %else8
+; SSE2-NEXT:    testb $32, %al
+; SSE2-NEXT:    jne LBB12_11
+; SSE2-NEXT:  LBB12_12: ## %else10
+; SSE2-NEXT:    testb $64, %al
+; SSE2-NEXT:    jne LBB12_13
+; SSE2-NEXT:  LBB12_14: ## %else12
+; SSE2-NEXT:    testb $-128, %al
+; SSE2-NEXT:    jne LBB12_15
+; SSE2-NEXT:  LBB12_16: ## %else14
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  LBB12_1: ## %cond.store
 ; SSE2-NEXT:    movd %xmm2, (%rdi)
-; SSE2-NEXT:  LBB12_2: ## %else
-; SSE2-NEXT:    psrlq $16, %xmm4
-; SSE2-NEXT:    movd %xmm4, %eax
-; SSE2-NEXT:    shrl $16, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $2, %al
 ; SSE2-NEXT:    je LBB12_4
-; SSE2-NEXT:  ## %bb.3: ## %cond.store1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,1,2,3]
-; SSE2-NEXT:    movd %xmm4, 4(%rdi)
-; SSE2-NEXT:  LBB12_4: ## %else2
-; SSE2-NEXT:    pxor %xmm4, %xmm4
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm0
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:  LBB12_3: ## %cond.store1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
+; SSE2-NEXT:    movd %xmm0, 4(%rdi)
+; SSE2-NEXT:    testb $4, %al
 ; SSE2-NEXT:    je LBB12_6
-; SSE2-NEXT:  ## %bb.5: ## %cond.store3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[2,3,0,1]
-; SSE2-NEXT:    movd %xmm4, 8(%rdi)
-; SSE2-NEXT:  LBB12_6: ## %else4
-; SSE2-NEXT:    pextrw $6, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:  LBB12_5: ## %cond.store3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE2-NEXT:    movd %xmm0, 8(%rdi)
+; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je LBB12_8
-; SSE2-NEXT:  ## %bb.7: ## %cond.store5
+; SSE2-NEXT:  LBB12_7: ## %cond.store5
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[3,1,2,3]
 ; SSE2-NEXT:    movd %xmm0, 12(%rdi)
-; SSE2-NEXT:  LBB12_8: ## %else6
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
-; SSE2-NEXT:    pextrw $0, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $16, %al
 ; SSE2-NEXT:    je LBB12_10
-; SSE2-NEXT:  ## %bb.9: ## %cond.store7
+; SSE2-NEXT:  LBB12_9: ## %cond.store7
 ; SSE2-NEXT:    movd %xmm3, 16(%rdi)
-; SSE2-NEXT:  LBB12_10: ## %else8
-; SSE2-NEXT:    pextrw $2, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $32, %al
 ; SSE2-NEXT:    je LBB12_12
-; SSE2-NEXT:  ## %bb.11: ## %cond.store9
+; SSE2-NEXT:  LBB12_11: ## %cond.store9
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,2,3]
 ; SSE2-NEXT:    movd %xmm0, 20(%rdi)
-; SSE2-NEXT:  LBB12_12: ## %else10
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE2-NEXT:    pextrw $4, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $64, %al
 ; SSE2-NEXT:    je LBB12_14
-; SSE2-NEXT:  ## %bb.13: ## %cond.store11
+; SSE2-NEXT:  LBB12_13: ## %cond.store11
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[2,3,0,1]
 ; SSE2-NEXT:    movd %xmm0, 24(%rdi)
-; SSE2-NEXT:  LBB12_14: ## %else12
-; SSE2-NEXT:    pextrw $6, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $-128, %al
 ; SSE2-NEXT:    je LBB12_16
-; SSE2-NEXT:  ## %bb.15: ## %cond.store13
+; SSE2-NEXT:  LBB12_15: ## %cond.store13
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[3,1,2,3]
 ; SSE2-NEXT:    movd %xmm0, 28(%rdi)
-; SSE2-NEXT:  LBB12_16: ## %else14
 ; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: store_v8i32_v8i32:
 ; SSE4:       ## %bb.0:
 ; SSE4-NEXT:    pxor %xmm4, %xmm4
-; SSE4-NEXT:    pcmpeqd %xmm0, %xmm4
-; SSE4-NEXT:    pextrb $0, %xmm4, %eax
+; SSE4-NEXT:    pcmpeqd %xmm4, %xmm1
+; SSE4-NEXT:    pcmpeqd %xmm4, %xmm0
+; SSE4-NEXT:    packssdw %xmm1, %xmm0
+; SSE4-NEXT:    packsswb %xmm0, %xmm0
+; SSE4-NEXT:    pmovmskb %xmm0, %eax
 ; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je LBB12_2
-; SSE4-NEXT:  ## %bb.1: ## %cond.store
+; SSE4-NEXT:    jne LBB12_1
+; SSE4-NEXT:  ## %bb.2: ## %else
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    jne LBB12_3
+; SSE4-NEXT:  LBB12_4: ## %else2
+; SSE4-NEXT:    testb $4, %al
+; SSE4-NEXT:    jne LBB12_5
+; SSE4-NEXT:  LBB12_6: ## %else4
+; SSE4-NEXT:    testb $8, %al
+; SSE4-NEXT:    jne LBB12_7
+; SSE4-NEXT:  LBB12_8: ## %else6
+; SSE4-NEXT:    testb $16, %al
+; SSE4-NEXT:    jne LBB12_9
+; SSE4-NEXT:  LBB12_10: ## %else8
+; SSE4-NEXT:    testb $32, %al
+; SSE4-NEXT:    jne LBB12_11
+; SSE4-NEXT:  LBB12_12: ## %else10
+; SSE4-NEXT:    testb $64, %al
+; SSE4-NEXT:    jne LBB12_13
+; SSE4-NEXT:  LBB12_14: ## %else12
+; SSE4-NEXT:    testb $-128, %al
+; SSE4-NEXT:    jne LBB12_15
+; SSE4-NEXT:  LBB12_16: ## %else14
+; SSE4-NEXT:    retq
+; SSE4-NEXT:  LBB12_1: ## %cond.store
 ; SSE4-NEXT:    movss %xmm2, (%rdi)
-; SSE4-NEXT:  LBB12_2: ## %else
-; SSE4-NEXT:    pextrb $4, %xmm4, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $2, %al
 ; SSE4-NEXT:    je LBB12_4
-; SSE4-NEXT:  ## %bb.3: ## %cond.store1
+; SSE4-NEXT:  LBB12_3: ## %cond.store1
 ; SSE4-NEXT:    extractps $1, %xmm2, 4(%rdi)
-; SSE4-NEXT:  LBB12_4: ## %else2
-; SSE4-NEXT:    pxor %xmm4, %xmm4
-; SSE4-NEXT:    pcmpeqd %xmm4, %xmm0
-; SSE4-NEXT:    pextrb $8, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $4, %al
 ; SSE4-NEXT:    je LBB12_6
-; SSE4-NEXT:  ## %bb.5: ## %cond.store3
+; SSE4-NEXT:  LBB12_5: ## %cond.store3
 ; SSE4-NEXT:    extractps $2, %xmm2, 8(%rdi)
-; SSE4-NEXT:  LBB12_6: ## %else4
-; SSE4-NEXT:    pextrb $12, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $8, %al
 ; SSE4-NEXT:    je LBB12_8
-; SSE4-NEXT:  ## %bb.7: ## %cond.store5
+; SSE4-NEXT:  LBB12_7: ## %cond.store5
 ; SSE4-NEXT:    extractps $3, %xmm2, 12(%rdi)
-; SSE4-NEXT:  LBB12_8: ## %else6
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm0
-; SSE4-NEXT:    pextrb $0, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $16, %al
 ; SSE4-NEXT:    je LBB12_10
-; SSE4-NEXT:  ## %bb.9: ## %cond.store7
+; SSE4-NEXT:  LBB12_9: ## %cond.store7
 ; SSE4-NEXT:    movss %xmm3, 16(%rdi)
-; SSE4-NEXT:  LBB12_10: ## %else8
-; SSE4-NEXT:    pextrb $4, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $32, %al
 ; SSE4-NEXT:    je LBB12_12
-; SSE4-NEXT:  ## %bb.11: ## %cond.store9
+; SSE4-NEXT:  LBB12_11: ## %cond.store9
 ; SSE4-NEXT:    extractps $1, %xmm3, 20(%rdi)
-; SSE4-NEXT:  LBB12_12: ## %else10
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE4-NEXT:    pextrb $8, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $64, %al
 ; SSE4-NEXT:    je LBB12_14
-; SSE4-NEXT:  ## %bb.13: ## %cond.store11
+; SSE4-NEXT:  LBB12_13: ## %cond.store11
 ; SSE4-NEXT:    extractps $2, %xmm3, 24(%rdi)
-; SSE4-NEXT:  LBB12_14: ## %else12
-; SSE4-NEXT:    pextrb $12, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $-128, %al
 ; SSE4-NEXT:    je LBB12_16
-; SSE4-NEXT:  ## %bb.15: ## %cond.store13
+; SSE4-NEXT:  LBB12_15: ## %cond.store13
 ; SSE4-NEXT:    extractps $3, %xmm3, 28(%rdi)
-; SSE4-NEXT:  LBB12_16: ## %else14
 ; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: store_v8i32_v8i32:
@@ -1494,339 +1423,328 @@ define void @store_v8i16_v8i16(<8 x i16> %trigger, <8 x i16>* %addr, <8 x i16> %
 ; SSE2:       ## %bb.0:
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
 ; SSE2-NEXT:    pcmpeqw %xmm0, %xmm2
-; SSE2-NEXT:    movd %xmm2, %eax
+; SSE2-NEXT:    packsswb %xmm0, %xmm2
+; SSE2-NEXT:    pmovmskb %xmm2, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB13_2
-; SSE2-NEXT:  ## %bb.1: ## %cond.store
+; SSE2-NEXT:    jne LBB13_1
+; SSE2-NEXT:  ## %bb.2: ## %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne LBB13_3
+; SSE2-NEXT:  LBB13_4: ## %else2
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne LBB13_5
+; SSE2-NEXT:  LBB13_6: ## %else4
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne LBB13_7
+; SSE2-NEXT:  LBB13_8: ## %else6
+; SSE2-NEXT:    testb $16, %al
+; SSE2-NEXT:    jne LBB13_9
+; SSE2-NEXT:  LBB13_10: ## %else8
+; SSE2-NEXT:    testb $32, %al
+; SSE2-NEXT:    jne LBB13_11
+; SSE2-NEXT:  LBB13_12: ## %else10
+; SSE2-NEXT:    testb $64, %al
+; SSE2-NEXT:    jne LBB13_13
+; SSE2-NEXT:  LBB13_14: ## %else12
+; SSE2-NEXT:    testb $-128, %al
+; SSE2-NEXT:    jne LBB13_15
+; SSE2-NEXT:  LBB13_16: ## %else14
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  LBB13_1: ## %cond.store
 ; SSE2-NEXT:    movd %xmm1, %ecx
 ; SSE2-NEXT:    movw %cx, (%rdi)
-; SSE2-NEXT:  LBB13_2: ## %else
-; SSE2-NEXT:    shrl $16, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $2, %al
 ; SSE2-NEXT:    je LBB13_4
-; SSE2-NEXT:  ## %bb.3: ## %cond.store1
-; SSE2-NEXT:    pextrw $1, %xmm1, %eax
-; SSE2-NEXT:    movw %ax, 2(%rdi)
-; SSE2-NEXT:  LBB13_4: ## %else2
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpeqw %xmm0, %xmm2
-; SSE2-NEXT:    pextrw $2, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:  LBB13_3: ## %cond.store1
+; SSE2-NEXT:    pextrw $1, %xmm1, %ecx
+; SSE2-NEXT:    movw %cx, 2(%rdi)
+; SSE2-NEXT:    testb $4, %al
 ; SSE2-NEXT:    je LBB13_6
-; SSE2-NEXT:  ## %bb.5: ## %cond.store3
-; SSE2-NEXT:    pextrw $2, %xmm1, %eax
-; SSE2-NEXT:    movw %ax, 4(%rdi)
-; SSE2-NEXT:  LBB13_6: ## %else4
-; SSE2-NEXT:    pextrw $3, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:  LBB13_5: ## %cond.store3
+; SSE2-NEXT:    pextrw $2, %xmm1, %ecx
+; SSE2-NEXT:    movw %cx, 4(%rdi)
+; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je LBB13_8
-; SSE2-NEXT:  ## %bb.7: ## %cond.store5
-; SSE2-NEXT:    pextrw $3, %xmm1, %eax
-; SSE2-NEXT:    movw %ax, 6(%rdi)
-; SSE2-NEXT:  LBB13_8: ## %else6
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpeqw %xmm0, %xmm2
-; SSE2-NEXT:    pextrw $4, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:  LBB13_7: ## %cond.store5
+; SSE2-NEXT:    pextrw $3, %xmm1, %ecx
+; SSE2-NEXT:    movw %cx, 6(%rdi)
+; SSE2-NEXT:    testb $16, %al
 ; SSE2-NEXT:    je LBB13_10
-; SSE2-NEXT:  ## %bb.9: ## %cond.store7
-; SSE2-NEXT:    pextrw $4, %xmm1, %eax
-; SSE2-NEXT:    movw %ax, 8(%rdi)
-; SSE2-NEXT:  LBB13_10: ## %else8
-; SSE2-NEXT:    pextrw $5, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:  LBB13_9: ## %cond.store7
+; SSE2-NEXT:    pextrw $4, %xmm1, %ecx
+; SSE2-NEXT:    movw %cx, 8(%rdi)
+; SSE2-NEXT:    testb $32, %al
 ; SSE2-NEXT:    je LBB13_12
-; SSE2-NEXT:  ## %bb.11: ## %cond.store9
-; SSE2-NEXT:    pextrw $5, %xmm1, %eax
-; SSE2-NEXT:    movw %ax, 10(%rdi)
-; SSE2-NEXT:  LBB13_12: ## %else10
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpeqw %xmm2, %xmm0
-; SSE2-NEXT:    pextrw $6, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:  LBB13_11: ## %cond.store9
+; SSE2-NEXT:    pextrw $5, %xmm1, %ecx
+; SSE2-NEXT:    movw %cx, 10(%rdi)
+; SSE2-NEXT:    testb $64, %al
 ; SSE2-NEXT:    je LBB13_14
-; SSE2-NEXT:  ## %bb.13: ## %cond.store11
-; SSE2-NEXT:    pextrw $6, %xmm1, %eax
-; SSE2-NEXT:    movw %ax, 12(%rdi)
-; SSE2-NEXT:  LBB13_14: ## %else12
-; SSE2-NEXT:    pextrw $7, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:  LBB13_13: ## %cond.store11
+; SSE2-NEXT:    pextrw $6, %xmm1, %ecx
+; SSE2-NEXT:    movw %cx, 12(%rdi)
+; SSE2-NEXT:    testb $-128, %al
 ; SSE2-NEXT:    je LBB13_16
-; SSE2-NEXT:  ## %bb.15: ## %cond.store13
+; SSE2-NEXT:  LBB13_15: ## %cond.store13
 ; SSE2-NEXT:    pextrw $7, %xmm1, %eax
 ; SSE2-NEXT:    movw %ax, 14(%rdi)
-; SSE2-NEXT:  LBB13_16: ## %else14
 ; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: store_v8i16_v8i16:
 ; SSE4:       ## %bb.0:
 ; SSE4-NEXT:    pxor %xmm2, %xmm2
 ; SSE4-NEXT:    pcmpeqw %xmm0, %xmm2
-; SSE4-NEXT:    pextrb $0, %xmm2, %eax
+; SSE4-NEXT:    packsswb %xmm0, %xmm2
+; SSE4-NEXT:    pmovmskb %xmm2, %eax
 ; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je LBB13_2
-; SSE4-NEXT:  ## %bb.1: ## %cond.store
+; SSE4-NEXT:    jne LBB13_1
+; SSE4-NEXT:  ## %bb.2: ## %else
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    jne LBB13_3
+; SSE4-NEXT:  LBB13_4: ## %else2
+; SSE4-NEXT:    testb $4, %al
+; SSE4-NEXT:    jne LBB13_5
+; SSE4-NEXT:  LBB13_6: ## %else4
+; SSE4-NEXT:    testb $8, %al
+; SSE4-NEXT:    jne LBB13_7
+; SSE4-NEXT:  LBB13_8: ## %else6
+; SSE4-NEXT:    testb $16, %al
+; SSE4-NEXT:    jne LBB13_9
+; SSE4-NEXT:  LBB13_10: ## %else8
+; SSE4-NEXT:    testb $32, %al
+; SSE4-NEXT:    jne LBB13_11
+; SSE4-NEXT:  LBB13_12: ## %else10
+; SSE4-NEXT:    testb $64, %al
+; SSE4-NEXT:    jne LBB13_13
+; SSE4-NEXT:  LBB13_14: ## %else12
+; SSE4-NEXT:    testb $-128, %al
+; SSE4-NEXT:    jne LBB13_15
+; SSE4-NEXT:  LBB13_16: ## %else14
+; SSE4-NEXT:    retq
+; SSE4-NEXT:  LBB13_1: ## %cond.store
 ; SSE4-NEXT:    pextrw $0, %xmm1, (%rdi)
-; SSE4-NEXT:  LBB13_2: ## %else
-; SSE4-NEXT:    pextrb $2, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $2, %al
 ; SSE4-NEXT:    je LBB13_4
-; SSE4-NEXT:  ## %bb.3: ## %cond.store1
+; SSE4-NEXT:  LBB13_3: ## %cond.store1
 ; SSE4-NEXT:    pextrw $1, %xmm1, 2(%rdi)
-; SSE4-NEXT:  LBB13_4: ## %else2
-; SSE4-NEXT:    pxor %xmm2, %xmm2
-; SSE4-NEXT:    pcmpeqw %xmm0, %xmm2
-; SSE4-NEXT:    pextrb $4, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $4, %al
 ; SSE4-NEXT:    je LBB13_6
-; SSE4-NEXT:  ## %bb.5: ## %cond.store3
+; SSE4-NEXT:  LBB13_5: ## %cond.store3
 ; SSE4-NEXT:    pextrw $2, %xmm1, 4(%rdi)
-; SSE4-NEXT:  LBB13_6: ## %else4
-; SSE4-NEXT:    pextrb $6, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $8, %al
 ; SSE4-NEXT:    je LBB13_8
-; SSE4-NEXT:  ## %bb.7: ## %cond.store5
+; SSE4-NEXT:  LBB13_7: ## %cond.store5
 ; SSE4-NEXT:    pextrw $3, %xmm1, 6(%rdi)
-; SSE4-NEXT:  LBB13_8: ## %else6
-; SSE4-NEXT:    pxor %xmm2, %xmm2
-; SSE4-NEXT:    pcmpeqw %xmm0, %xmm2
-; SSE4-NEXT:    pextrb $8, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $16, %al
 ; SSE4-NEXT:    je LBB13_10
-; SSE4-NEXT:  ## %bb.9: ## %cond.store7
+; SSE4-NEXT:  LBB13_9: ## %cond.store7
 ; SSE4-NEXT:    pextrw $4, %xmm1, 8(%rdi)
-; SSE4-NEXT:  LBB13_10: ## %else8
-; SSE4-NEXT:    pextrb $10, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $32, %al
 ; SSE4-NEXT:    je LBB13_12
-; SSE4-NEXT:  ## %bb.11: ## %cond.store9
+; SSE4-NEXT:  LBB13_11: ## %cond.store9
 ; SSE4-NEXT:    pextrw $5, %xmm1, 10(%rdi)
-; SSE4-NEXT:  LBB13_12: ## %else10
-; SSE4-NEXT:    pxor %xmm2, %xmm2
-; SSE4-NEXT:    pcmpeqw %xmm2, %xmm0
-; SSE4-NEXT:    pextrb $12, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $64, %al
 ; SSE4-NEXT:    je LBB13_14
-; SSE4-NEXT:  ## %bb.13: ## %cond.store11
+; SSE4-NEXT:  LBB13_13: ## %cond.store11
 ; SSE4-NEXT:    pextrw $6, %xmm1, 12(%rdi)
-; SSE4-NEXT:  LBB13_14: ## %else12
-; SSE4-NEXT:    pextrb $14, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $-128, %al
 ; SSE4-NEXT:    je LBB13_16
-; SSE4-NEXT:  ## %bb.15: ## %cond.store13
+; SSE4-NEXT:  LBB13_15: ## %cond.store13
 ; SSE4-NEXT:    pextrw $7, %xmm1, 14(%rdi)
-; SSE4-NEXT:  LBB13_16: ## %else14
 ; SSE4-NEXT:    retq
 ;
 ; AVX1OR2-LABEL: store_v8i16_v8i16:
 ; AVX1OR2:       ## %bb.0:
 ; AVX1OR2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1OR2-NEXT:    vpcmpeqw %xmm2, %xmm0, %xmm2
-; AVX1OR2-NEXT:    vpextrb $0, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
-; AVX1OR2-NEXT:    je LBB13_2
-; AVX1OR2-NEXT:  ## %bb.1: ## %cond.store
-; AVX1OR2-NEXT:    vpextrw $0, %xmm1, (%rdi)
-; AVX1OR2-NEXT:  LBB13_2: ## %else
-; AVX1OR2-NEXT:    vpextrb $2, %xmm2, %eax
+; AVX1OR2-NEXT:    vpcmpeqw %xmm2, %xmm0, %xmm0
+; AVX1OR2-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
+; AVX1OR2-NEXT:    vpmovmskb %xmm0, %eax
 ; AVX1OR2-NEXT:    testb $1, %al
-; AVX1OR2-NEXT:    je LBB13_4
-; AVX1OR2-NEXT:  ## %bb.3: ## %cond.store1
-; AVX1OR2-NEXT:    vpextrw $1, %xmm1, 2(%rdi)
+; AVX1OR2-NEXT:    jne LBB13_1
+; AVX1OR2-NEXT:  ## %bb.2: ## %else
+; AVX1OR2-NEXT:    testb $2, %al
+; AVX1OR2-NEXT:    jne LBB13_3
 ; AVX1OR2-NEXT:  LBB13_4: ## %else2
-; AVX1OR2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1OR2-NEXT:    vpcmpeqw %xmm2, %xmm0, %xmm2
-; AVX1OR2-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
-; AVX1OR2-NEXT:    je LBB13_6
-; AVX1OR2-NEXT:  ## %bb.5: ## %cond.store3
-; AVX1OR2-NEXT:    vpextrw $2, %xmm1, 4(%rdi)
+; AVX1OR2-NEXT:    testb $4, %al
+; AVX1OR2-NEXT:    jne LBB13_5
 ; AVX1OR2-NEXT:  LBB13_6: ## %else4
-; AVX1OR2-NEXT:    vpextrb $6, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
-; AVX1OR2-NEXT:    je LBB13_8
-; AVX1OR2-NEXT:  ## %bb.7: ## %cond.store5
-; AVX1OR2-NEXT:    vpextrw $3, %xmm1, 6(%rdi)
+; AVX1OR2-NEXT:    testb $8, %al
+; AVX1OR2-NEXT:    jne LBB13_7
 ; AVX1OR2-NEXT:  LBB13_8: ## %else6
-; AVX1OR2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1OR2-NEXT:    vpcmpeqw %xmm2, %xmm0, %xmm2
-; AVX1OR2-NEXT:    vpextrb $8, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $16, %al
+; AVX1OR2-NEXT:    jne LBB13_9
+; AVX1OR2-NEXT:  LBB13_10: ## %else8
+; AVX1OR2-NEXT:    testb $32, %al
+; AVX1OR2-NEXT:    jne LBB13_11
+; AVX1OR2-NEXT:  LBB13_12: ## %else10
+; AVX1OR2-NEXT:    testb $64, %al
+; AVX1OR2-NEXT:    jne LBB13_13
+; AVX1OR2-NEXT:  LBB13_14: ## %else12
+; AVX1OR2-NEXT:    testb $-128, %al
+; AVX1OR2-NEXT:    jne LBB13_15
+; AVX1OR2-NEXT:  LBB13_16: ## %else14
+; AVX1OR2-NEXT:    retq
+; AVX1OR2-NEXT:  LBB13_1: ## %cond.store
+; AVX1OR2-NEXT:    vpextrw $0, %xmm1, (%rdi)
+; AVX1OR2-NEXT:    testb $2, %al
+; AVX1OR2-NEXT:    je LBB13_4
+; AVX1OR2-NEXT:  LBB13_3: ## %cond.store1
+; AVX1OR2-NEXT:    vpextrw $1, %xmm1, 2(%rdi)
+; AVX1OR2-NEXT:    testb $4, %al
+; AVX1OR2-NEXT:    je LBB13_6
+; AVX1OR2-NEXT:  LBB13_5: ## %cond.store3
+; AVX1OR2-NEXT:    vpextrw $2, %xmm1, 4(%rdi)
+; AVX1OR2-NEXT:    testb $8, %al
+; AVX1OR2-NEXT:    je LBB13_8
+; AVX1OR2-NEXT:  LBB13_7: ## %cond.store5
+; AVX1OR2-NEXT:    vpextrw $3, %xmm1, 6(%rdi)
+; AVX1OR2-NEXT:    testb $16, %al
 ; AVX1OR2-NEXT:    je LBB13_10
-; AVX1OR2-NEXT:  ## %bb.9: ## %cond.store7
+; AVX1OR2-NEXT:  LBB13_9: ## %cond.store7
 ; AVX1OR2-NEXT:    vpextrw $4, %xmm1, 8(%rdi)
-; AVX1OR2-NEXT:  LBB13_10: ## %else8
-; AVX1OR2-NEXT:    vpextrb $10, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $32, %al
 ; AVX1OR2-NEXT:    je LBB13_12
-; AVX1OR2-NEXT:  ## %bb.11: ## %cond.store9
+; AVX1OR2-NEXT:  LBB13_11: ## %cond.store9
 ; AVX1OR2-NEXT:    vpextrw $5, %xmm1, 10(%rdi)
-; AVX1OR2-NEXT:  LBB13_12: ## %else10
-; AVX1OR2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1OR2-NEXT:    vpcmpeqw %xmm2, %xmm0, %xmm0
-; AVX1OR2-NEXT:    vpextrb $12, %xmm0, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $64, %al
 ; AVX1OR2-NEXT:    je LBB13_14
-; AVX1OR2-NEXT:  ## %bb.13: ## %cond.store11
+; AVX1OR2-NEXT:  LBB13_13: ## %cond.store11
 ; AVX1OR2-NEXT:    vpextrw $6, %xmm1, 12(%rdi)
-; AVX1OR2-NEXT:  LBB13_14: ## %else12
-; AVX1OR2-NEXT:    vpextrb $14, %xmm0, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $-128, %al
 ; AVX1OR2-NEXT:    je LBB13_16
-; AVX1OR2-NEXT:  ## %bb.15: ## %cond.store13
+; AVX1OR2-NEXT:  LBB13_15: ## %cond.store13
 ; AVX1OR2-NEXT:    vpextrw $7, %xmm1, 14(%rdi)
-; AVX1OR2-NEXT:  LBB13_16: ## %else14
 ; AVX1OR2-NEXT:    retq
 ;
 ; AVX512F-LABEL: store_v8i16_v8i16:
 ; AVX512F:       ## %bb.0:
 ; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqw %xmm2, %xmm0, %xmm2
-; AVX512F-NEXT:    vpmovsxwq %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmq %zmm2, %zmm2, %k0
+; AVX512F-NEXT:    vpcmpeqw %xmm2, %xmm0, %xmm0
+; AVX512F-NEXT:    vpmovsxwq %xmm0, %zmm0
+; AVX512F-NEXT:    vptestmq %zmm0, %zmm0, %k0
 ; AVX512F-NEXT:    kmovw %k0, %eax
 ; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je LBB13_2
-; AVX512F-NEXT:  ## %bb.1: ## %cond.store
+; AVX512F-NEXT:    jne LBB13_1
+; AVX512F-NEXT:  ## %bb.2: ## %else
+; AVX512F-NEXT:    testb $2, %al
+; AVX512F-NEXT:    jne LBB13_3
+; AVX512F-NEXT:  LBB13_4: ## %else2
+; AVX512F-NEXT:    testb $4, %al
+; AVX512F-NEXT:    jne LBB13_5
+; AVX512F-NEXT:  LBB13_6: ## %else4
+; AVX512F-NEXT:    testb $8, %al
+; AVX512F-NEXT:    jne LBB13_7
+; AVX512F-NEXT:  LBB13_8: ## %else6
+; AVX512F-NEXT:    testb $16, %al
+; AVX512F-NEXT:    jne LBB13_9
+; AVX512F-NEXT:  LBB13_10: ## %else8
+; AVX512F-NEXT:    testb $32, %al
+; AVX512F-NEXT:    jne LBB13_11
+; AVX512F-NEXT:  LBB13_12: ## %else10
+; AVX512F-NEXT:    testb $64, %al
+; AVX512F-NEXT:    jne LBB13_13
+; AVX512F-NEXT:  LBB13_14: ## %else12
+; AVX512F-NEXT:    testb $-128, %al
+; AVX512F-NEXT:    jne LBB13_15
+; AVX512F-NEXT:  LBB13_16: ## %else14
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+; AVX512F-NEXT:  LBB13_1: ## %cond.store
 ; AVX512F-NEXT:    vpextrw $0, %xmm1, (%rdi)
-; AVX512F-NEXT:  LBB13_2: ## %else
-; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $2, %al
 ; AVX512F-NEXT:    je LBB13_4
-; AVX512F-NEXT:  ## %bb.3: ## %cond.store1
+; AVX512F-NEXT:  LBB13_3: ## %cond.store1
 ; AVX512F-NEXT:    vpextrw $1, %xmm1, 2(%rdi)
-; AVX512F-NEXT:  LBB13_4: ## %else2
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqw %xmm2, %xmm0, %xmm2
-; AVX512F-NEXT:    vpmovsxwq %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmq %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $2, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $4, %al
 ; AVX512F-NEXT:    je LBB13_6
-; AVX512F-NEXT:  ## %bb.5: ## %cond.store3
+; AVX512F-NEXT:  LBB13_5: ## %cond.store3
 ; AVX512F-NEXT:    vpextrw $2, %xmm1, 4(%rdi)
-; AVX512F-NEXT:  LBB13_6: ## %else4
-; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $8, %al
 ; AVX512F-NEXT:    je LBB13_8
-; AVX512F-NEXT:  ## %bb.7: ## %cond.store5
+; AVX512F-NEXT:  LBB13_7: ## %cond.store5
 ; AVX512F-NEXT:    vpextrw $3, %xmm1, 6(%rdi)
-; AVX512F-NEXT:  LBB13_8: ## %else6
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqw %xmm2, %xmm0, %xmm2
-; AVX512F-NEXT:    vpmovsxwq %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmq %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $4, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $16, %al
 ; AVX512F-NEXT:    je LBB13_10
-; AVX512F-NEXT:  ## %bb.9: ## %cond.store7
+; AVX512F-NEXT:  LBB13_9: ## %cond.store7
 ; AVX512F-NEXT:    vpextrw $4, %xmm1, 8(%rdi)
-; AVX512F-NEXT:  LBB13_10: ## %else8
-; AVX512F-NEXT:    kshiftrw $5, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $32, %al
 ; AVX512F-NEXT:    je LBB13_12
-; AVX512F-NEXT:  ## %bb.11: ## %cond.store9
+; AVX512F-NEXT:  LBB13_11: ## %cond.store9
 ; AVX512F-NEXT:    vpextrw $5, %xmm1, 10(%rdi)
-; AVX512F-NEXT:  LBB13_12: ## %else10
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqw %xmm2, %xmm0, %xmm0
-; AVX512F-NEXT:    vpmovsxwq %xmm0, %zmm0
-; AVX512F-NEXT:    vptestmq %zmm0, %zmm0, %k0
-; AVX512F-NEXT:    kshiftrw $6, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $64, %al
 ; AVX512F-NEXT:    je LBB13_14
-; AVX512F-NEXT:  ## %bb.13: ## %cond.store11
+; AVX512F-NEXT:  LBB13_13: ## %cond.store11
 ; AVX512F-NEXT:    vpextrw $6, %xmm1, 12(%rdi)
-; AVX512F-NEXT:  LBB13_14: ## %else12
-; AVX512F-NEXT:    kshiftrw $7, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $-128, %al
 ; AVX512F-NEXT:    je LBB13_16
-; AVX512F-NEXT:  ## %bb.15: ## %cond.store13
+; AVX512F-NEXT:  LBB13_15: ## %cond.store13
 ; AVX512F-NEXT:    vpextrw $7, %xmm1, 14(%rdi)
-; AVX512F-NEXT:  LBB13_16: ## %else14
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VLDQ-LABEL: store_v8i16_v8i16:
 ; AVX512VLDQ:       ## %bb.0:
 ; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpeqw %xmm2, %xmm0, %xmm2
-; AVX512VLDQ-NEXT:    vpmovsxwd %xmm2, %ymm2
-; AVX512VLDQ-NEXT:    vpmovd2m %ymm2, %k0
+; AVX512VLDQ-NEXT:    vpcmpeqw %xmm2, %xmm0, %xmm0
+; AVX512VLDQ-NEXT:    vpmovsxwd %xmm0, %ymm0
+; AVX512VLDQ-NEXT:    vpmovd2m %ymm0, %k0
 ; AVX512VLDQ-NEXT:    kmovw %k0, %eax
 ; AVX512VLDQ-NEXT:    testb $1, %al
-; AVX512VLDQ-NEXT:    je LBB13_2
-; AVX512VLDQ-NEXT:  ## %bb.1: ## %cond.store
+; AVX512VLDQ-NEXT:    jne LBB13_1
+; AVX512VLDQ-NEXT:  ## %bb.2: ## %else
+; AVX512VLDQ-NEXT:    testb $2, %al
+; AVX512VLDQ-NEXT:    jne LBB13_3
+; AVX512VLDQ-NEXT:  LBB13_4: ## %else2
+; AVX512VLDQ-NEXT:    testb $4, %al
+; AVX512VLDQ-NEXT:    jne LBB13_5
+; AVX512VLDQ-NEXT:  LBB13_6: ## %else4
+; AVX512VLDQ-NEXT:    testb $8, %al
+; AVX512VLDQ-NEXT:    jne LBB13_7
+; AVX512VLDQ-NEXT:  LBB13_8: ## %else6
+; AVX512VLDQ-NEXT:    testb $16, %al
+; AVX512VLDQ-NEXT:    jne LBB13_9
+; AVX512VLDQ-NEXT:  LBB13_10: ## %else8
+; AVX512VLDQ-NEXT:    testb $32, %al
+; AVX512VLDQ-NEXT:    jne LBB13_11
+; AVX512VLDQ-NEXT:  LBB13_12: ## %else10
+; AVX512VLDQ-NEXT:    testb $64, %al
+; AVX512VLDQ-NEXT:    jne LBB13_13
+; AVX512VLDQ-NEXT:  LBB13_14: ## %else12
+; AVX512VLDQ-NEXT:    testb $-128, %al
+; AVX512VLDQ-NEXT:    jne LBB13_15
+; AVX512VLDQ-NEXT:  LBB13_16: ## %else14
+; AVX512VLDQ-NEXT:    vzeroupper
+; AVX512VLDQ-NEXT:    retq
+; AVX512VLDQ-NEXT:  LBB13_1: ## %cond.store
 ; AVX512VLDQ-NEXT:    vpextrw $0, %xmm1, (%rdi)
-; AVX512VLDQ-NEXT:  LBB13_2: ## %else
-; AVX512VLDQ-NEXT:    kshiftrb $1, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $2, %al
 ; AVX512VLDQ-NEXT:    je LBB13_4
-; AVX512VLDQ-NEXT:  ## %bb.3: ## %cond.store1
+; AVX512VLDQ-NEXT:  LBB13_3: ## %cond.store1
 ; AVX512VLDQ-NEXT:    vpextrw $1, %xmm1, 2(%rdi)
-; AVX512VLDQ-NEXT:  LBB13_4: ## %else2
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpeqw %xmm2, %xmm0, %xmm2
-; AVX512VLDQ-NEXT:    vpmovsxwd %xmm2, %ymm2
-; AVX512VLDQ-NEXT:    vpmovd2m %ymm2, %k0
-; AVX512VLDQ-NEXT:    kshiftrb $2, %k0, %k1
-; AVX512VLDQ-NEXT:    kmovw %k1, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $4, %al
 ; AVX512VLDQ-NEXT:    je LBB13_6
-; AVX512VLDQ-NEXT:  ## %bb.5: ## %cond.store3
+; AVX512VLDQ-NEXT:  LBB13_5: ## %cond.store3
 ; AVX512VLDQ-NEXT:    vpextrw $2, %xmm1, 4(%rdi)
-; AVX512VLDQ-NEXT:  LBB13_6: ## %else4
-; AVX512VLDQ-NEXT:    kshiftrb $3, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $8, %al
 ; AVX512VLDQ-NEXT:    je LBB13_8
-; AVX512VLDQ-NEXT:  ## %bb.7: ## %cond.store5
+; AVX512VLDQ-NEXT:  LBB13_7: ## %cond.store5
 ; AVX512VLDQ-NEXT:    vpextrw $3, %xmm1, 6(%rdi)
-; AVX512VLDQ-NEXT:  LBB13_8: ## %else6
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpeqw %xmm2, %xmm0, %xmm2
-; AVX512VLDQ-NEXT:    vpmovsxwd %xmm2, %ymm2
-; AVX512VLDQ-NEXT:    vpmovd2m %ymm2, %k0
-; AVX512VLDQ-NEXT:    kshiftrb $4, %k0, %k1
-; AVX512VLDQ-NEXT:    kmovw %k1, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $16, %al
 ; AVX512VLDQ-NEXT:    je LBB13_10
-; AVX512VLDQ-NEXT:  ## %bb.9: ## %cond.store7
+; AVX512VLDQ-NEXT:  LBB13_9: ## %cond.store7
 ; AVX512VLDQ-NEXT:    vpextrw $4, %xmm1, 8(%rdi)
-; AVX512VLDQ-NEXT:  LBB13_10: ## %else8
-; AVX512VLDQ-NEXT:    kshiftrb $5, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $32, %al
 ; AVX512VLDQ-NEXT:    je LBB13_12
-; AVX512VLDQ-NEXT:  ## %bb.11: ## %cond.store9
+; AVX512VLDQ-NEXT:  LBB13_11: ## %cond.store9
 ; AVX512VLDQ-NEXT:    vpextrw $5, %xmm1, 10(%rdi)
-; AVX512VLDQ-NEXT:  LBB13_12: ## %else10
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpeqw %xmm2, %xmm0, %xmm0
-; AVX512VLDQ-NEXT:    vpmovsxwd %xmm0, %ymm0
-; AVX512VLDQ-NEXT:    vpmovd2m %ymm0, %k0
-; AVX512VLDQ-NEXT:    kshiftrb $6, %k0, %k1
-; AVX512VLDQ-NEXT:    kmovw %k1, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $64, %al
 ; AVX512VLDQ-NEXT:    je LBB13_14
-; AVX512VLDQ-NEXT:  ## %bb.13: ## %cond.store11
+; AVX512VLDQ-NEXT:  LBB13_13: ## %cond.store11
 ; AVX512VLDQ-NEXT:    vpextrw $6, %xmm1, 12(%rdi)
-; AVX512VLDQ-NEXT:  LBB13_14: ## %else12
-; AVX512VLDQ-NEXT:    kshiftrb $7, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $-128, %al
 ; AVX512VLDQ-NEXT:    je LBB13_16
-; AVX512VLDQ-NEXT:  ## %bb.15: ## %cond.store13
+; AVX512VLDQ-NEXT:  LBB13_15: ## %cond.store13
 ; AVX512VLDQ-NEXT:    vpextrw $7, %xmm1, 14(%rdi)
-; AVX512VLDQ-NEXT:  LBB13_16: ## %else14
 ; AVX512VLDQ-NEXT:    vzeroupper
 ; AVX512VLDQ-NEXT:    retq
 ;
@@ -1844,792 +1762,745 @@ define void @store_v16i16_v16i16(<16 x i16> %trigger, <16 x i16>* %addr, <16 x i
 ; SSE2-LABEL: store_v16i16_v16i16:
 ; SSE2:       ## %bb.0:
 ; SSE2-NEXT:    pxor %xmm4, %xmm4
-; SSE2-NEXT:    pcmpeqw %xmm0, %xmm4
-; SSE2-NEXT:    movd %xmm4, %eax
+; SSE2-NEXT:    pcmpeqw %xmm4, %xmm1
+; SSE2-NEXT:    pcmpeqw %xmm4, %xmm0
+; SSE2-NEXT:    packsswb %xmm1, %xmm0
+; SSE2-NEXT:    pmovmskb %xmm0, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB14_2
-; SSE2-NEXT:  ## %bb.1: ## %cond.store
+; SSE2-NEXT:    jne LBB14_1
+; SSE2-NEXT:  ## %bb.2: ## %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne LBB14_3
+; SSE2-NEXT:  LBB14_4: ## %else2
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne LBB14_5
+; SSE2-NEXT:  LBB14_6: ## %else4
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne LBB14_7
+; SSE2-NEXT:  LBB14_8: ## %else6
+; SSE2-NEXT:    testb $16, %al
+; SSE2-NEXT:    jne LBB14_9
+; SSE2-NEXT:  LBB14_10: ## %else8
+; SSE2-NEXT:    testb $32, %al
+; SSE2-NEXT:    jne LBB14_11
+; SSE2-NEXT:  LBB14_12: ## %else10
+; SSE2-NEXT:    testb $64, %al
+; SSE2-NEXT:    jne LBB14_13
+; SSE2-NEXT:  LBB14_14: ## %else12
+; SSE2-NEXT:    testb $-128, %al
+; SSE2-NEXT:    jne LBB14_15
+; SSE2-NEXT:  LBB14_16: ## %else14
+; SSE2-NEXT:    testl $256, %eax ## imm = 0x100
+; SSE2-NEXT:    jne LBB14_17
+; SSE2-NEXT:  LBB14_18: ## %else16
+; SSE2-NEXT:    testl $512, %eax ## imm = 0x200
+; SSE2-NEXT:    jne LBB14_19
+; SSE2-NEXT:  LBB14_20: ## %else18
+; SSE2-NEXT:    testl $1024, %eax ## imm = 0x400
+; SSE2-NEXT:    jne LBB14_21
+; SSE2-NEXT:  LBB14_22: ## %else20
+; SSE2-NEXT:    testl $2048, %eax ## imm = 0x800
+; SSE2-NEXT:    jne LBB14_23
+; SSE2-NEXT:  LBB14_24: ## %else22
+; SSE2-NEXT:    testl $4096, %eax ## imm = 0x1000
+; SSE2-NEXT:    jne LBB14_25
+; SSE2-NEXT:  LBB14_26: ## %else24
+; SSE2-NEXT:    testl $8192, %eax ## imm = 0x2000
+; SSE2-NEXT:    jne LBB14_27
+; SSE2-NEXT:  LBB14_28: ## %else26
+; SSE2-NEXT:    testl $16384, %eax ## imm = 0x4000
+; SSE2-NEXT:    jne LBB14_29
+; SSE2-NEXT:  LBB14_30: ## %else28
+; SSE2-NEXT:    testl $32768, %eax ## imm = 0x8000
+; SSE2-NEXT:    jne LBB14_31
+; SSE2-NEXT:  LBB14_32: ## %else30
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  LBB14_1: ## %cond.store
 ; SSE2-NEXT:    movd %xmm2, %ecx
 ; SSE2-NEXT:    movw %cx, (%rdi)
-; SSE2-NEXT:  LBB14_2: ## %else
-; SSE2-NEXT:    shrl $16, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $2, %al
 ; SSE2-NEXT:    je LBB14_4
-; SSE2-NEXT:  ## %bb.3: ## %cond.store1
-; SSE2-NEXT:    pextrw $1, %xmm2, %eax
-; SSE2-NEXT:    movw %ax, 2(%rdi)
-; SSE2-NEXT:  LBB14_4: ## %else2
-; SSE2-NEXT:    pxor %xmm4, %xmm4
-; SSE2-NEXT:    pcmpeqw %xmm0, %xmm4
-; SSE2-NEXT:    pextrw $2, %xmm4, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:  LBB14_3: ## %cond.store1
+; SSE2-NEXT:    pextrw $1, %xmm2, %ecx
+; SSE2-NEXT:    movw %cx, 2(%rdi)
+; SSE2-NEXT:    testb $4, %al
 ; SSE2-NEXT:    je LBB14_6
-; SSE2-NEXT:  ## %bb.5: ## %cond.store3
-; SSE2-NEXT:    pextrw $2, %xmm2, %eax
-; SSE2-NEXT:    movw %ax, 4(%rdi)
-; SSE2-NEXT:  LBB14_6: ## %else4
-; SSE2-NEXT:    pextrw $3, %xmm4, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:  LBB14_5: ## %cond.store3
+; SSE2-NEXT:    pextrw $2, %xmm2, %ecx
+; SSE2-NEXT:    movw %cx, 4(%rdi)
+; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je LBB14_8
-; SSE2-NEXT:  ## %bb.7: ## %cond.store5
-; SSE2-NEXT:    pextrw $3, %xmm2, %eax
-; SSE2-NEXT:    movw %ax, 6(%rdi)
-; SSE2-NEXT:  LBB14_8: ## %else6
-; SSE2-NEXT:    pxor %xmm4, %xmm4
-; SSE2-NEXT:    pcmpeqw %xmm0, %xmm4
-; SSE2-NEXT:    pextrw $4, %xmm4, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:  LBB14_7: ## %cond.store5
+; SSE2-NEXT:    pextrw $3, %xmm2, %ecx
+; SSE2-NEXT:    movw %cx, 6(%rdi)
+; SSE2-NEXT:    testb $16, %al
 ; SSE2-NEXT:    je LBB14_10
-; SSE2-NEXT:  ## %bb.9: ## %cond.store7
-; SSE2-NEXT:    pextrw $4, %xmm2, %eax
-; SSE2-NEXT:    movw %ax, 8(%rdi)
-; SSE2-NEXT:  LBB14_10: ## %else8
-; SSE2-NEXT:    pextrw $5, %xmm4, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:  LBB14_9: ## %cond.store7
+; SSE2-NEXT:    pextrw $4, %xmm2, %ecx
+; SSE2-NEXT:    movw %cx, 8(%rdi)
+; SSE2-NEXT:    testb $32, %al
 ; SSE2-NEXT:    je LBB14_12
-; SSE2-NEXT:  ## %bb.11: ## %cond.store9
-; SSE2-NEXT:    pextrw $5, %xmm2, %eax
-; SSE2-NEXT:    movw %ax, 10(%rdi)
-; SSE2-NEXT:  LBB14_12: ## %else10
-; SSE2-NEXT:    pxor %xmm4, %xmm4
-; SSE2-NEXT:    pcmpeqw %xmm4, %xmm0
-; SSE2-NEXT:    pextrw $6, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:  LBB14_11: ## %cond.store9
+; SSE2-NEXT:    pextrw $5, %xmm2, %ecx
+; SSE2-NEXT:    movw %cx, 10(%rdi)
+; SSE2-NEXT:    testb $64, %al
 ; SSE2-NEXT:    je LBB14_14
-; SSE2-NEXT:  ## %bb.13: ## %cond.store11
-; SSE2-NEXT:    pextrw $6, %xmm2, %eax
-; SSE2-NEXT:    movw %ax, 12(%rdi)
-; SSE2-NEXT:  LBB14_14: ## %else12
-; SSE2-NEXT:    pextrw $7, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:  LBB14_13: ## %cond.store11
+; SSE2-NEXT:    pextrw $6, %xmm2, %ecx
+; SSE2-NEXT:    movw %cx, 12(%rdi)
+; SSE2-NEXT:    testb $-128, %al
 ; SSE2-NEXT:    je LBB14_16
-; SSE2-NEXT:  ## %bb.15: ## %cond.store13
-; SSE2-NEXT:    pextrw $7, %xmm2, %eax
-; SSE2-NEXT:    movw %ax, 14(%rdi)
-; SSE2-NEXT:  LBB14_16: ## %else14
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqw %xmm1, %xmm0
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:  LBB14_15: ## %cond.store13
+; SSE2-NEXT:    pextrw $7, %xmm2, %ecx
+; SSE2-NEXT:    movw %cx, 14(%rdi)
+; SSE2-NEXT:    testl $256, %eax ## imm = 0x100
 ; SSE2-NEXT:    je LBB14_18
-; SSE2-NEXT:  ## %bb.17: ## %cond.store15
+; SSE2-NEXT:  LBB14_17: ## %cond.store15
 ; SSE2-NEXT:    movd %xmm3, %ecx
 ; SSE2-NEXT:    movw %cx, 16(%rdi)
-; SSE2-NEXT:  LBB14_18: ## %else16
-; SSE2-NEXT:    shrl $16, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $512, %eax ## imm = 0x200
 ; SSE2-NEXT:    je LBB14_20
-; SSE2-NEXT:  ## %bb.19: ## %cond.store17
-; SSE2-NEXT:    pextrw $1, %xmm3, %eax
-; SSE2-NEXT:    movw %ax, 18(%rdi)
-; SSE2-NEXT:  LBB14_20: ## %else18
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqw %xmm1, %xmm0
-; SSE2-NEXT:    pextrw $2, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:  LBB14_19: ## %cond.store17
+; SSE2-NEXT:    pextrw $1, %xmm3, %ecx
+; SSE2-NEXT:    movw %cx, 18(%rdi)
+; SSE2-NEXT:    testl $1024, %eax ## imm = 0x400
 ; SSE2-NEXT:    je LBB14_22
-; SSE2-NEXT:  ## %bb.21: ## %cond.store19
-; SSE2-NEXT:    pextrw $2, %xmm3, %eax
-; SSE2-NEXT:    movw %ax, 20(%rdi)
-; SSE2-NEXT:  LBB14_22: ## %else20
-; SSE2-NEXT:    pextrw $3, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:  LBB14_21: ## %cond.store19
+; SSE2-NEXT:    pextrw $2, %xmm3, %ecx
+; SSE2-NEXT:    movw %cx, 20(%rdi)
+; SSE2-NEXT:    testl $2048, %eax ## imm = 0x800
 ; SSE2-NEXT:    je LBB14_24
-; SSE2-NEXT:  ## %bb.23: ## %cond.store21
-; SSE2-NEXT:    pextrw $3, %xmm3, %eax
-; SSE2-NEXT:    movw %ax, 22(%rdi)
-; SSE2-NEXT:  LBB14_24: ## %else22
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqw %xmm1, %xmm0
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:  LBB14_23: ## %cond.store21
+; SSE2-NEXT:    pextrw $3, %xmm3, %ecx
+; SSE2-NEXT:    movw %cx, 22(%rdi)
+; SSE2-NEXT:    testl $4096, %eax ## imm = 0x1000
 ; SSE2-NEXT:    je LBB14_26
-; SSE2-NEXT:  ## %bb.25: ## %cond.store23
-; SSE2-NEXT:    pextrw $4, %xmm3, %eax
-; SSE2-NEXT:    movw %ax, 24(%rdi)
-; SSE2-NEXT:  LBB14_26: ## %else24
-; SSE2-NEXT:    pextrw $5, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:  LBB14_25: ## %cond.store23
+; SSE2-NEXT:    pextrw $4, %xmm3, %ecx
+; SSE2-NEXT:    movw %cx, 24(%rdi)
+; SSE2-NEXT:    testl $8192, %eax ## imm = 0x2000
 ; SSE2-NEXT:    je LBB14_28
-; SSE2-NEXT:  ## %bb.27: ## %cond.store25
-; SSE2-NEXT:    pextrw $5, %xmm3, %eax
-; SSE2-NEXT:    movw %ax, 26(%rdi)
-; SSE2-NEXT:  LBB14_28: ## %else26
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqw %xmm0, %xmm1
-; SSE2-NEXT:    pextrw $6, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:  LBB14_27: ## %cond.store25
+; SSE2-NEXT:    pextrw $5, %xmm3, %ecx
+; SSE2-NEXT:    movw %cx, 26(%rdi)
+; SSE2-NEXT:    testl $16384, %eax ## imm = 0x4000
 ; SSE2-NEXT:    je LBB14_30
-; SSE2-NEXT:  ## %bb.29: ## %cond.store27
-; SSE2-NEXT:    pextrw $6, %xmm3, %eax
-; SSE2-NEXT:    movw %ax, 28(%rdi)
-; SSE2-NEXT:  LBB14_30: ## %else28
-; SSE2-NEXT:    pextrw $7, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:  LBB14_29: ## %cond.store27
+; SSE2-NEXT:    pextrw $6, %xmm3, %ecx
+; SSE2-NEXT:    movw %cx, 28(%rdi)
+; SSE2-NEXT:    testl $32768, %eax ## imm = 0x8000
 ; SSE2-NEXT:    je LBB14_32
-; SSE2-NEXT:  ## %bb.31: ## %cond.store29
+; SSE2-NEXT:  LBB14_31: ## %cond.store29
 ; SSE2-NEXT:    pextrw $7, %xmm3, %eax
 ; SSE2-NEXT:    movw %ax, 30(%rdi)
-; SSE2-NEXT:  LBB14_32: ## %else30
 ; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: store_v16i16_v16i16:
 ; SSE4:       ## %bb.0:
 ; SSE4-NEXT:    pxor %xmm4, %xmm4
-; SSE4-NEXT:    pcmpeqw %xmm0, %xmm4
-; SSE4-NEXT:    pextrb $0, %xmm4, %eax
+; SSE4-NEXT:    pcmpeqw %xmm4, %xmm1
+; SSE4-NEXT:    pcmpeqw %xmm4, %xmm0
+; SSE4-NEXT:    packsswb %xmm1, %xmm0
+; SSE4-NEXT:    pmovmskb %xmm0, %eax
 ; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je LBB14_2
-; SSE4-NEXT:  ## %bb.1: ## %cond.store
+; SSE4-NEXT:    jne LBB14_1
+; SSE4-NEXT:  ## %bb.2: ## %else
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    jne LBB14_3
+; SSE4-NEXT:  LBB14_4: ## %else2
+; SSE4-NEXT:    testb $4, %al
+; SSE4-NEXT:    jne LBB14_5
+; SSE4-NEXT:  LBB14_6: ## %else4
+; SSE4-NEXT:    testb $8, %al
+; SSE4-NEXT:    jne LBB14_7
+; SSE4-NEXT:  LBB14_8: ## %else6
+; SSE4-NEXT:    testb $16, %al
+; SSE4-NEXT:    jne LBB14_9
+; SSE4-NEXT:  LBB14_10: ## %else8
+; SSE4-NEXT:    testb $32, %al
+; SSE4-NEXT:    jne LBB14_11
+; SSE4-NEXT:  LBB14_12: ## %else10
+; SSE4-NEXT:    testb $64, %al
+; SSE4-NEXT:    jne LBB14_13
+; SSE4-NEXT:  LBB14_14: ## %else12
+; SSE4-NEXT:    testb $-128, %al
+; SSE4-NEXT:    jne LBB14_15
+; SSE4-NEXT:  LBB14_16: ## %else14
+; SSE4-NEXT:    testl $256, %eax ## imm = 0x100
+; SSE4-NEXT:    jne LBB14_17
+; SSE4-NEXT:  LBB14_18: ## %else16
+; SSE4-NEXT:    testl $512, %eax ## imm = 0x200
+; SSE4-NEXT:    jne LBB14_19
+; SSE4-NEXT:  LBB14_20: ## %else18
+; SSE4-NEXT:    testl $1024, %eax ## imm = 0x400
+; SSE4-NEXT:    jne LBB14_21
+; SSE4-NEXT:  LBB14_22: ## %else20
+; SSE4-NEXT:    testl $2048, %eax ## imm = 0x800
+; SSE4-NEXT:    jne LBB14_23
+; SSE4-NEXT:  LBB14_24: ## %else22
+; SSE4-NEXT:    testl $4096, %eax ## imm = 0x1000
+; SSE4-NEXT:    jne LBB14_25
+; SSE4-NEXT:  LBB14_26: ## %else24
+; SSE4-NEXT:    testl $8192, %eax ## imm = 0x2000
+; SSE4-NEXT:    jne LBB14_27
+; SSE4-NEXT:  LBB14_28: ## %else26
+; SSE4-NEXT:    testl $16384, %eax ## imm = 0x4000
+; SSE4-NEXT:    jne LBB14_29
+; SSE4-NEXT:  LBB14_30: ## %else28
+; SSE4-NEXT:    testl $32768, %eax ## imm = 0x8000
+; SSE4-NEXT:    jne LBB14_31
+; SSE4-NEXT:  LBB14_32: ## %else30
+; SSE4-NEXT:    retq
+; SSE4-NEXT:  LBB14_1: ## %cond.store
 ; SSE4-NEXT:    pextrw $0, %xmm2, (%rdi)
-; SSE4-NEXT:  LBB14_2: ## %else
-; SSE4-NEXT:    pextrb $2, %xmm4, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $2, %al
 ; SSE4-NEXT:    je LBB14_4
-; SSE4-NEXT:  ## %bb.3: ## %cond.store1
+; SSE4-NEXT:  LBB14_3: ## %cond.store1
 ; SSE4-NEXT:    pextrw $1, %xmm2, 2(%rdi)
-; SSE4-NEXT:  LBB14_4: ## %else2
-; SSE4-NEXT:    pxor %xmm4, %xmm4
-; SSE4-NEXT:    pcmpeqw %xmm0, %xmm4
-; SSE4-NEXT:    pextrb $4, %xmm4, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $4, %al
 ; SSE4-NEXT:    je LBB14_6
-; SSE4-NEXT:  ## %bb.5: ## %cond.store3
+; SSE4-NEXT:  LBB14_5: ## %cond.store3
 ; SSE4-NEXT:    pextrw $2, %xmm2, 4(%rdi)
-; SSE4-NEXT:  LBB14_6: ## %else4
-; SSE4-NEXT:    pextrb $6, %xmm4, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $8, %al
 ; SSE4-NEXT:    je LBB14_8
-; SSE4-NEXT:  ## %bb.7: ## %cond.store5
+; SSE4-NEXT:  LBB14_7: ## %cond.store5
 ; SSE4-NEXT:    pextrw $3, %xmm2, 6(%rdi)
-; SSE4-NEXT:  LBB14_8: ## %else6
-; SSE4-NEXT:    pxor %xmm4, %xmm4
-; SSE4-NEXT:    pcmpeqw %xmm0, %xmm4
-; SSE4-NEXT:    pextrb $8, %xmm4, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $16, %al
 ; SSE4-NEXT:    je LBB14_10
-; SSE4-NEXT:  ## %bb.9: ## %cond.store7
+; SSE4-NEXT:  LBB14_9: ## %cond.store7
 ; SSE4-NEXT:    pextrw $4, %xmm2, 8(%rdi)
-; SSE4-NEXT:  LBB14_10: ## %else8
-; SSE4-NEXT:    pextrb $10, %xmm4, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $32, %al
 ; SSE4-NEXT:    je LBB14_12
-; SSE4-NEXT:  ## %bb.11: ## %cond.store9
+; SSE4-NEXT:  LBB14_11: ## %cond.store9
 ; SSE4-NEXT:    pextrw $5, %xmm2, 10(%rdi)
-; SSE4-NEXT:  LBB14_12: ## %else10
-; SSE4-NEXT:    pxor %xmm4, %xmm4
-; SSE4-NEXT:    pcmpeqw %xmm4, %xmm0
-; SSE4-NEXT:    pextrb $12, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $64, %al
 ; SSE4-NEXT:    je LBB14_14
-; SSE4-NEXT:  ## %bb.13: ## %cond.store11
+; SSE4-NEXT:  LBB14_13: ## %cond.store11
 ; SSE4-NEXT:    pextrw $6, %xmm2, 12(%rdi)
-; SSE4-NEXT:  LBB14_14: ## %else12
-; SSE4-NEXT:    pextrb $14, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $-128, %al
 ; SSE4-NEXT:    je LBB14_16
-; SSE4-NEXT:  ## %bb.15: ## %cond.store13
+; SSE4-NEXT:  LBB14_15: ## %cond.store13
 ; SSE4-NEXT:    pextrw $7, %xmm2, 14(%rdi)
-; SSE4-NEXT:  LBB14_16: ## %else14
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqw %xmm1, %xmm0
-; SSE4-NEXT:    pextrb $0, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $256, %eax ## imm = 0x100
 ; SSE4-NEXT:    je LBB14_18
-; SSE4-NEXT:  ## %bb.17: ## %cond.store15
+; SSE4-NEXT:  LBB14_17: ## %cond.store15
 ; SSE4-NEXT:    pextrw $0, %xmm3, 16(%rdi)
-; SSE4-NEXT:  LBB14_18: ## %else16
-; SSE4-NEXT:    pextrb $2, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $512, %eax ## imm = 0x200
 ; SSE4-NEXT:    je LBB14_20
-; SSE4-NEXT:  ## %bb.19: ## %cond.store17
+; SSE4-NEXT:  LBB14_19: ## %cond.store17
 ; SSE4-NEXT:    pextrw $1, %xmm3, 18(%rdi)
-; SSE4-NEXT:  LBB14_20: ## %else18
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqw %xmm1, %xmm0
-; SSE4-NEXT:    pextrb $4, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $1024, %eax ## imm = 0x400
 ; SSE4-NEXT:    je LBB14_22
-; SSE4-NEXT:  ## %bb.21: ## %cond.store19
+; SSE4-NEXT:  LBB14_21: ## %cond.store19
 ; SSE4-NEXT:    pextrw $2, %xmm3, 20(%rdi)
-; SSE4-NEXT:  LBB14_22: ## %else20
-; SSE4-NEXT:    pextrb $6, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $2048, %eax ## imm = 0x800
 ; SSE4-NEXT:    je LBB14_24
-; SSE4-NEXT:  ## %bb.23: ## %cond.store21
+; SSE4-NEXT:  LBB14_23: ## %cond.store21
 ; SSE4-NEXT:    pextrw $3, %xmm3, 22(%rdi)
-; SSE4-NEXT:  LBB14_24: ## %else22
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqw %xmm1, %xmm0
-; SSE4-NEXT:    pextrb $8, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $4096, %eax ## imm = 0x1000
 ; SSE4-NEXT:    je LBB14_26
-; SSE4-NEXT:  ## %bb.25: ## %cond.store23
+; SSE4-NEXT:  LBB14_25: ## %cond.store23
 ; SSE4-NEXT:    pextrw $4, %xmm3, 24(%rdi)
-; SSE4-NEXT:  LBB14_26: ## %else24
-; SSE4-NEXT:    pextrb $10, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $8192, %eax ## imm = 0x2000
 ; SSE4-NEXT:    je LBB14_28
-; SSE4-NEXT:  ## %bb.27: ## %cond.store25
+; SSE4-NEXT:  LBB14_27: ## %cond.store25
 ; SSE4-NEXT:    pextrw $5, %xmm3, 26(%rdi)
-; SSE4-NEXT:  LBB14_28: ## %else26
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqw %xmm0, %xmm1
-; SSE4-NEXT:    pextrb $12, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $16384, %eax ## imm = 0x4000
 ; SSE4-NEXT:    je LBB14_30
-; SSE4-NEXT:  ## %bb.29: ## %cond.store27
+; SSE4-NEXT:  LBB14_29: ## %cond.store27
 ; SSE4-NEXT:    pextrw $6, %xmm3, 28(%rdi)
-; SSE4-NEXT:  LBB14_30: ## %else28
-; SSE4-NEXT:    pextrb $14, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $32768, %eax ## imm = 0x8000
 ; SSE4-NEXT:    je LBB14_32
-; SSE4-NEXT:  ## %bb.31: ## %cond.store29
+; SSE4-NEXT:  LBB14_31: ## %cond.store29
 ; SSE4-NEXT:    pextrw $7, %xmm3, 30(%rdi)
-; SSE4-NEXT:  LBB14_32: ## %else30
 ; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: store_v16i16_v16i16:
 ; AVX1:       ## %bb.0:
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqw %xmm2, %xmm0, %xmm3
-; AVX1-NEXT:    vpextrb $0, %xmm3, %eax
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpeqw %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpcmpeqw %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpacksswb %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpmovmskb %xmm0, %eax
 ; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je LBB14_2
-; AVX1-NEXT:  ## %bb.1: ## %cond.store
+; AVX1-NEXT:    jne LBB14_1
+; AVX1-NEXT:  ## %bb.2: ## %else
+; AVX1-NEXT:    testb $2, %al
+; AVX1-NEXT:    jne LBB14_3
+; AVX1-NEXT:  LBB14_4: ## %else2
+; AVX1-NEXT:    testb $4, %al
+; AVX1-NEXT:    jne LBB14_5
+; AVX1-NEXT:  LBB14_6: ## %else4
+; AVX1-NEXT:    testb $8, %al
+; AVX1-NEXT:    jne LBB14_7
+; AVX1-NEXT:  LBB14_8: ## %else6
+; AVX1-NEXT:    testb $16, %al
+; AVX1-NEXT:    jne LBB14_9
+; AVX1-NEXT:  LBB14_10: ## %else8
+; AVX1-NEXT:    testb $32, %al
+; AVX1-NEXT:    jne LBB14_11
+; AVX1-NEXT:  LBB14_12: ## %else10
+; AVX1-NEXT:    testb $64, %al
+; AVX1-NEXT:    jne LBB14_13
+; AVX1-NEXT:  LBB14_14: ## %else12
+; AVX1-NEXT:    testb $-128, %al
+; AVX1-NEXT:    je LBB14_16
+; AVX1-NEXT:  LBB14_15: ## %cond.store13
+; AVX1-NEXT:    vpextrw $7, %xmm1, 14(%rdi)
+; AVX1-NEXT:  LBB14_16: ## %else14
+; AVX1-NEXT:    testl $256, %eax ## imm = 0x100
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm0
+; AVX1-NEXT:    jne LBB14_17
+; AVX1-NEXT:  ## %bb.18: ## %else16
+; AVX1-NEXT:    testl $512, %eax ## imm = 0x200
+; AVX1-NEXT:    jne LBB14_19
+; AVX1-NEXT:  LBB14_20: ## %else18
+; AVX1-NEXT:    testl $1024, %eax ## imm = 0x400
+; AVX1-NEXT:    jne LBB14_21
+; AVX1-NEXT:  LBB14_22: ## %else20
+; AVX1-NEXT:    testl $2048, %eax ## imm = 0x800
+; AVX1-NEXT:    jne LBB14_23
+; AVX1-NEXT:  LBB14_24: ## %else22
+; AVX1-NEXT:    testl $4096, %eax ## imm = 0x1000
+; AVX1-NEXT:    jne LBB14_25
+; AVX1-NEXT:  LBB14_26: ## %else24
+; AVX1-NEXT:    testl $8192, %eax ## imm = 0x2000
+; AVX1-NEXT:    jne LBB14_27
+; AVX1-NEXT:  LBB14_28: ## %else26
+; AVX1-NEXT:    testl $16384, %eax ## imm = 0x4000
+; AVX1-NEXT:    jne LBB14_29
+; AVX1-NEXT:  LBB14_30: ## %else28
+; AVX1-NEXT:    testl $32768, %eax ## imm = 0x8000
+; AVX1-NEXT:    jne LBB14_31
+; AVX1-NEXT:  LBB14_32: ## %else30
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+; AVX1-NEXT:  LBB14_1: ## %cond.store
 ; AVX1-NEXT:    vpextrw $0, %xmm1, (%rdi)
-; AVX1-NEXT:  LBB14_2: ## %else
-; AVX1-NEXT:    vpcmpeqw %xmm2, %xmm0, %xmm2
-; AVX1-NEXT:    vpextrb $2, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $2, %al
 ; AVX1-NEXT:    je LBB14_4
-; AVX1-NEXT:  ## %bb.3: ## %cond.store1
+; AVX1-NEXT:  LBB14_3: ## %cond.store1
 ; AVX1-NEXT:    vpextrw $1, %xmm1, 2(%rdi)
-; AVX1-NEXT:  LBB14_4: ## %else2
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqw %xmm2, %xmm0, %xmm3
-; AVX1-NEXT:    vpextrb $4, %xmm3, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $4, %al
 ; AVX1-NEXT:    je LBB14_6
-; AVX1-NEXT:  ## %bb.5: ## %cond.store3
+; AVX1-NEXT:  LBB14_5: ## %cond.store3
 ; AVX1-NEXT:    vpextrw $2, %xmm1, 4(%rdi)
-; AVX1-NEXT:  LBB14_6: ## %else4
-; AVX1-NEXT:    vpcmpeqw %xmm2, %xmm0, %xmm2
-; AVX1-NEXT:    vpextrb $6, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $8, %al
 ; AVX1-NEXT:    je LBB14_8
-; AVX1-NEXT:  ## %bb.7: ## %cond.store5
+; AVX1-NEXT:  LBB14_7: ## %cond.store5
 ; AVX1-NEXT:    vpextrw $3, %xmm1, 6(%rdi)
-; AVX1-NEXT:  LBB14_8: ## %else6
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqw %xmm2, %xmm0, %xmm3
-; AVX1-NEXT:    vpextrb $8, %xmm3, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $16, %al
 ; AVX1-NEXT:    je LBB14_10
-; AVX1-NEXT:  ## %bb.9: ## %cond.store7
+; AVX1-NEXT:  LBB14_9: ## %cond.store7
 ; AVX1-NEXT:    vpextrw $4, %xmm1, 8(%rdi)
-; AVX1-NEXT:  LBB14_10: ## %else8
-; AVX1-NEXT:    vpcmpeqw %xmm2, %xmm0, %xmm2
-; AVX1-NEXT:    vpextrb $10, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $32, %al
 ; AVX1-NEXT:    je LBB14_12
-; AVX1-NEXT:  ## %bb.11: ## %cond.store9
+; AVX1-NEXT:  LBB14_11: ## %cond.store9
 ; AVX1-NEXT:    vpextrw $5, %xmm1, 10(%rdi)
-; AVX1-NEXT:  LBB14_12: ## %else10
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqw %xmm2, %xmm0, %xmm3
-; AVX1-NEXT:    vpextrb $12, %xmm3, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $64, %al
 ; AVX1-NEXT:    je LBB14_14
-; AVX1-NEXT:  ## %bb.13: ## %cond.store11
+; AVX1-NEXT:  LBB14_13: ## %cond.store11
 ; AVX1-NEXT:    vpextrw $6, %xmm1, 12(%rdi)
-; AVX1-NEXT:  LBB14_14: ## %else12
-; AVX1-NEXT:    vpcmpeqw %xmm2, %xmm0, %xmm2
-; AVX1-NEXT:    vpextrb $14, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je LBB14_16
-; AVX1-NEXT:  ## %bb.15: ## %cond.store13
-; AVX1-NEXT:    vpextrw $7, %xmm1, 14(%rdi)
-; AVX1-NEXT:  LBB14_16: ## %else14
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX1-NEXT:    vpxor %xmm0, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpeqw %xmm0, %xmm2, %xmm3
-; AVX1-NEXT:    vpextrb $0, %xmm3, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm0
-; AVX1-NEXT:    je LBB14_18
-; AVX1-NEXT:  ## %bb.17: ## %cond.store15
+; AVX1-NEXT:    testb $-128, %al
+; AVX1-NEXT:    jne LBB14_15
+; AVX1-NEXT:    jmp LBB14_16
+; AVX1-NEXT:  LBB14_17: ## %cond.store15
 ; AVX1-NEXT:    vpextrw $0, %xmm0, 16(%rdi)
-; AVX1-NEXT:  LBB14_18: ## %else16
-; AVX1-NEXT:    vpextrb $2, %xmm3, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $512, %eax ## imm = 0x200
 ; AVX1-NEXT:    je LBB14_20
-; AVX1-NEXT:  ## %bb.19: ## %cond.store17
+; AVX1-NEXT:  LBB14_19: ## %cond.store17
 ; AVX1-NEXT:    vpextrw $1, %xmm0, 18(%rdi)
-; AVX1-NEXT:  LBB14_20: ## %else18
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqw %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vpextrb $4, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $1024, %eax ## imm = 0x400
 ; AVX1-NEXT:    je LBB14_22
-; AVX1-NEXT:  ## %bb.21: ## %cond.store19
+; AVX1-NEXT:  LBB14_21: ## %cond.store19
 ; AVX1-NEXT:    vpextrw $2, %xmm0, 20(%rdi)
-; AVX1-NEXT:  LBB14_22: ## %else20
-; AVX1-NEXT:    vpextrb $6, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $2048, %eax ## imm = 0x800
 ; AVX1-NEXT:    je LBB14_24
-; AVX1-NEXT:  ## %bb.23: ## %cond.store21
+; AVX1-NEXT:  LBB14_23: ## %cond.store21
 ; AVX1-NEXT:    vpextrw $3, %xmm0, 22(%rdi)
-; AVX1-NEXT:  LBB14_24: ## %else22
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqw %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $4096, %eax ## imm = 0x1000
 ; AVX1-NEXT:    je LBB14_26
-; AVX1-NEXT:  ## %bb.25: ## %cond.store23
+; AVX1-NEXT:  LBB14_25: ## %cond.store23
 ; AVX1-NEXT:    vpextrw $4, %xmm0, 24(%rdi)
-; AVX1-NEXT:  LBB14_26: ## %else24
-; AVX1-NEXT:    vpextrb $10, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $8192, %eax ## imm = 0x2000
 ; AVX1-NEXT:    je LBB14_28
-; AVX1-NEXT:  ## %bb.27: ## %cond.store25
+; AVX1-NEXT:  LBB14_27: ## %cond.store25
 ; AVX1-NEXT:    vpextrw $5, %xmm0, 26(%rdi)
-; AVX1-NEXT:  LBB14_28: ## %else26
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqw %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $16384, %eax ## imm = 0x4000
 ; AVX1-NEXT:    je LBB14_30
-; AVX1-NEXT:  ## %bb.29: ## %cond.store27
+; AVX1-NEXT:  LBB14_29: ## %cond.store27
 ; AVX1-NEXT:    vpextrw $6, %xmm0, 28(%rdi)
-; AVX1-NEXT:  LBB14_30: ## %else28
-; AVX1-NEXT:    vpextrb $14, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $32768, %eax ## imm = 0x8000
 ; AVX1-NEXT:    je LBB14_32
-; AVX1-NEXT:  ## %bb.31: ## %cond.store29
+; AVX1-NEXT:  LBB14_31: ## %cond.store29
 ; AVX1-NEXT:    vpextrw $7, %xmm0, 30(%rdi)
-; AVX1-NEXT:  LBB14_32: ## %else30
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: store_v16i16_v16i16:
 ; AVX2:       ## %bb.0:
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqw %xmm2, %xmm0, %xmm3
-; AVX2-NEXT:    vpextrb $0, %xmm3, %eax
+; AVX2-NEXT:    vpcmpeqw %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm2
+; AVX2-NEXT:    vpacksswb %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpmovmskb %xmm0, %eax
 ; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je LBB14_2
-; AVX2-NEXT:  ## %bb.1: ## %cond.store
+; AVX2-NEXT:    jne LBB14_1
+; AVX2-NEXT:  ## %bb.2: ## %else
+; AVX2-NEXT:    testb $2, %al
+; AVX2-NEXT:    jne LBB14_3
+; AVX2-NEXT:  LBB14_4: ## %else2
+; AVX2-NEXT:    testb $4, %al
+; AVX2-NEXT:    jne LBB14_5
+; AVX2-NEXT:  LBB14_6: ## %else4
+; AVX2-NEXT:    testb $8, %al
+; AVX2-NEXT:    jne LBB14_7
+; AVX2-NEXT:  LBB14_8: ## %else6
+; AVX2-NEXT:    testb $16, %al
+; AVX2-NEXT:    jne LBB14_9
+; AVX2-NEXT:  LBB14_10: ## %else8
+; AVX2-NEXT:    testb $32, %al
+; AVX2-NEXT:    jne LBB14_11
+; AVX2-NEXT:  LBB14_12: ## %else10
+; AVX2-NEXT:    testb $64, %al
+; AVX2-NEXT:    jne LBB14_13
+; AVX2-NEXT:  LBB14_14: ## %else12
+; AVX2-NEXT:    testb $-128, %al
+; AVX2-NEXT:    je LBB14_16
+; AVX2-NEXT:  LBB14_15: ## %cond.store13
+; AVX2-NEXT:    vpextrw $7, %xmm1, 14(%rdi)
+; AVX2-NEXT:  LBB14_16: ## %else14
+; AVX2-NEXT:    testl $256, %eax ## imm = 0x100
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX2-NEXT:    jne LBB14_17
+; AVX2-NEXT:  ## %bb.18: ## %else16
+; AVX2-NEXT:    testl $512, %eax ## imm = 0x200
+; AVX2-NEXT:    jne LBB14_19
+; AVX2-NEXT:  LBB14_20: ## %else18
+; AVX2-NEXT:    testl $1024, %eax ## imm = 0x400
+; AVX2-NEXT:    jne LBB14_21
+; AVX2-NEXT:  LBB14_22: ## %else20
+; AVX2-NEXT:    testl $2048, %eax ## imm = 0x800
+; AVX2-NEXT:    jne LBB14_23
+; AVX2-NEXT:  LBB14_24: ## %else22
+; AVX2-NEXT:    testl $4096, %eax ## imm = 0x1000
+; AVX2-NEXT:    jne LBB14_25
+; AVX2-NEXT:  LBB14_26: ## %else24
+; AVX2-NEXT:    testl $8192, %eax ## imm = 0x2000
+; AVX2-NEXT:    jne LBB14_27
+; AVX2-NEXT:  LBB14_28: ## %else26
+; AVX2-NEXT:    testl $16384, %eax ## imm = 0x4000
+; AVX2-NEXT:    jne LBB14_29
+; AVX2-NEXT:  LBB14_30: ## %else28
+; AVX2-NEXT:    testl $32768, %eax ## imm = 0x8000
+; AVX2-NEXT:    jne LBB14_31
+; AVX2-NEXT:  LBB14_32: ## %else30
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+; AVX2-NEXT:  LBB14_1: ## %cond.store
 ; AVX2-NEXT:    vpextrw $0, %xmm1, (%rdi)
-; AVX2-NEXT:  LBB14_2: ## %else
-; AVX2-NEXT:    vpcmpeqw %xmm2, %xmm0, %xmm2
-; AVX2-NEXT:    vpextrb $2, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $2, %al
 ; AVX2-NEXT:    je LBB14_4
-; AVX2-NEXT:  ## %bb.3: ## %cond.store1
+; AVX2-NEXT:  LBB14_3: ## %cond.store1
 ; AVX2-NEXT:    vpextrw $1, %xmm1, 2(%rdi)
-; AVX2-NEXT:  LBB14_4: ## %else2
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqw %xmm2, %xmm0, %xmm3
-; AVX2-NEXT:    vpextrb $4, %xmm3, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $4, %al
 ; AVX2-NEXT:    je LBB14_6
-; AVX2-NEXT:  ## %bb.5: ## %cond.store3
+; AVX2-NEXT:  LBB14_5: ## %cond.store3
 ; AVX2-NEXT:    vpextrw $2, %xmm1, 4(%rdi)
-; AVX2-NEXT:  LBB14_6: ## %else4
-; AVX2-NEXT:    vpcmpeqw %xmm2, %xmm0, %xmm2
-; AVX2-NEXT:    vpextrb $6, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $8, %al
 ; AVX2-NEXT:    je LBB14_8
-; AVX2-NEXT:  ## %bb.7: ## %cond.store5
+; AVX2-NEXT:  LBB14_7: ## %cond.store5
 ; AVX2-NEXT:    vpextrw $3, %xmm1, 6(%rdi)
-; AVX2-NEXT:  LBB14_8: ## %else6
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqw %xmm2, %xmm0, %xmm3
-; AVX2-NEXT:    vpextrb $8, %xmm3, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $16, %al
 ; AVX2-NEXT:    je LBB14_10
-; AVX2-NEXT:  ## %bb.9: ## %cond.store7
+; AVX2-NEXT:  LBB14_9: ## %cond.store7
 ; AVX2-NEXT:    vpextrw $4, %xmm1, 8(%rdi)
-; AVX2-NEXT:  LBB14_10: ## %else8
-; AVX2-NEXT:    vpcmpeqw %xmm2, %xmm0, %xmm2
-; AVX2-NEXT:    vpextrb $10, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $32, %al
 ; AVX2-NEXT:    je LBB14_12
-; AVX2-NEXT:  ## %bb.11: ## %cond.store9
+; AVX2-NEXT:  LBB14_11: ## %cond.store9
 ; AVX2-NEXT:    vpextrw $5, %xmm1, 10(%rdi)
-; AVX2-NEXT:  LBB14_12: ## %else10
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqw %xmm2, %xmm0, %xmm3
-; AVX2-NEXT:    vpextrb $12, %xmm3, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $64, %al
 ; AVX2-NEXT:    je LBB14_14
-; AVX2-NEXT:  ## %bb.13: ## %cond.store11
+; AVX2-NEXT:  LBB14_13: ## %cond.store11
 ; AVX2-NEXT:    vpextrw $6, %xmm1, 12(%rdi)
-; AVX2-NEXT:  LBB14_14: ## %else12
-; AVX2-NEXT:    vpcmpeqw %xmm2, %xmm0, %xmm2
-; AVX2-NEXT:    vpextrb $14, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je LBB14_16
-; AVX2-NEXT:  ## %bb.15: ## %cond.store13
-; AVX2-NEXT:    vpextrw $7, %xmm1, 14(%rdi)
-; AVX2-NEXT:  LBB14_16: ## %else14
-; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm2
-; AVX2-NEXT:    vpxor %xmm0, %xmm0, %xmm0
-; AVX2-NEXT:    vpcmpeqw %xmm0, %xmm2, %xmm3
-; AVX2-NEXT:    vpextrb $0, %xmm3, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm0
-; AVX2-NEXT:    je LBB14_18
-; AVX2-NEXT:  ## %bb.17: ## %cond.store15
+; AVX2-NEXT:    testb $-128, %al
+; AVX2-NEXT:    jne LBB14_15
+; AVX2-NEXT:    jmp LBB14_16
+; AVX2-NEXT:  LBB14_17: ## %cond.store15
 ; AVX2-NEXT:    vpextrw $0, %xmm0, 16(%rdi)
-; AVX2-NEXT:  LBB14_18: ## %else16
-; AVX2-NEXT:    vpextrb $2, %xmm3, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $512, %eax ## imm = 0x200
 ; AVX2-NEXT:    je LBB14_20
-; AVX2-NEXT:  ## %bb.19: ## %cond.store17
+; AVX2-NEXT:  LBB14_19: ## %cond.store17
 ; AVX2-NEXT:    vpextrw $1, %xmm0, 18(%rdi)
-; AVX2-NEXT:  LBB14_20: ## %else18
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqw %xmm1, %xmm2, %xmm1
-; AVX2-NEXT:    vpextrb $4, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $1024, %eax ## imm = 0x400
 ; AVX2-NEXT:    je LBB14_22
-; AVX2-NEXT:  ## %bb.21: ## %cond.store19
+; AVX2-NEXT:  LBB14_21: ## %cond.store19
 ; AVX2-NEXT:    vpextrw $2, %xmm0, 20(%rdi)
-; AVX2-NEXT:  LBB14_22: ## %else20
-; AVX2-NEXT:    vpextrb $6, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $2048, %eax ## imm = 0x800
 ; AVX2-NEXT:    je LBB14_24
-; AVX2-NEXT:  ## %bb.23: ## %cond.store21
+; AVX2-NEXT:  LBB14_23: ## %cond.store21
 ; AVX2-NEXT:    vpextrw $3, %xmm0, 22(%rdi)
-; AVX2-NEXT:  LBB14_24: ## %else22
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqw %xmm1, %xmm2, %xmm1
-; AVX2-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $4096, %eax ## imm = 0x1000
 ; AVX2-NEXT:    je LBB14_26
-; AVX2-NEXT:  ## %bb.25: ## %cond.store23
+; AVX2-NEXT:  LBB14_25: ## %cond.store23
 ; AVX2-NEXT:    vpextrw $4, %xmm0, 24(%rdi)
-; AVX2-NEXT:  LBB14_26: ## %else24
-; AVX2-NEXT:    vpextrb $10, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $8192, %eax ## imm = 0x2000
 ; AVX2-NEXT:    je LBB14_28
-; AVX2-NEXT:  ## %bb.27: ## %cond.store25
+; AVX2-NEXT:  LBB14_27: ## %cond.store25
 ; AVX2-NEXT:    vpextrw $5, %xmm0, 26(%rdi)
-; AVX2-NEXT:  LBB14_28: ## %else26
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqw %xmm1, %xmm2, %xmm1
-; AVX2-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $16384, %eax ## imm = 0x4000
 ; AVX2-NEXT:    je LBB14_30
-; AVX2-NEXT:  ## %bb.29: ## %cond.store27
+; AVX2-NEXT:  LBB14_29: ## %cond.store27
 ; AVX2-NEXT:    vpextrw $6, %xmm0, 28(%rdi)
-; AVX2-NEXT:  LBB14_30: ## %else28
-; AVX2-NEXT:    vpextrb $14, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $32768, %eax ## imm = 0x8000
 ; AVX2-NEXT:    je LBB14_32
-; AVX2-NEXT:  ## %bb.31: ## %cond.store29
+; AVX2-NEXT:  LBB14_31: ## %cond.store29
 ; AVX2-NEXT:    vpextrw $7, %xmm0, 30(%rdi)
-; AVX2-NEXT:  LBB14_32: ## %else30
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: store_v16i16_v16i16:
 ; AVX512F:       ## %bb.0:
 ; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqw %ymm2, %ymm0, %ymm2
-; AVX512F-NEXT:    vpmovsxwd %ymm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
+; AVX512F-NEXT:    vpcmpeqw %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vpmovsxwd %ymm0, %zmm0
+; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k0
 ; AVX512F-NEXT:    kmovw %k0, %eax
 ; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je LBB14_2
-; AVX512F-NEXT:  ## %bb.1: ## %cond.store
+; AVX512F-NEXT:    jne LBB14_1
+; AVX512F-NEXT:  ## %bb.2: ## %else
+; AVX512F-NEXT:    testb $2, %al
+; AVX512F-NEXT:    jne LBB14_3
+; AVX512F-NEXT:  LBB14_4: ## %else2
+; AVX512F-NEXT:    testb $4, %al
+; AVX512F-NEXT:    jne LBB14_5
+; AVX512F-NEXT:  LBB14_6: ## %else4
+; AVX512F-NEXT:    testb $8, %al
+; AVX512F-NEXT:    jne LBB14_7
+; AVX512F-NEXT:  LBB14_8: ## %else6
+; AVX512F-NEXT:    testb $16, %al
+; AVX512F-NEXT:    jne LBB14_9
+; AVX512F-NEXT:  LBB14_10: ## %else8
+; AVX512F-NEXT:    testb $32, %al
+; AVX512F-NEXT:    jne LBB14_11
+; AVX512F-NEXT:  LBB14_12: ## %else10
+; AVX512F-NEXT:    testb $64, %al
+; AVX512F-NEXT:    jne LBB14_13
+; AVX512F-NEXT:  LBB14_14: ## %else12
+; AVX512F-NEXT:    testb $-128, %al
+; AVX512F-NEXT:    je LBB14_16
+; AVX512F-NEXT:  LBB14_15: ## %cond.store13
+; AVX512F-NEXT:    vpextrw $7, %xmm1, 14(%rdi)
+; AVX512F-NEXT:  LBB14_16: ## %else14
+; AVX512F-NEXT:    testl $256, %eax ## imm = 0x100
+; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX512F-NEXT:    jne LBB14_17
+; AVX512F-NEXT:  ## %bb.18: ## %else16
+; AVX512F-NEXT:    testl $512, %eax ## imm = 0x200
+; AVX512F-NEXT:    jne LBB14_19
+; AVX512F-NEXT:  LBB14_20: ## %else18
+; AVX512F-NEXT:    testl $1024, %eax ## imm = 0x400
+; AVX512F-NEXT:    jne LBB14_21
+; AVX512F-NEXT:  LBB14_22: ## %else20
+; AVX512F-NEXT:    testl $2048, %eax ## imm = 0x800
+; AVX512F-NEXT:    jne LBB14_23
+; AVX512F-NEXT:  LBB14_24: ## %else22
+; AVX512F-NEXT:    testl $4096, %eax ## imm = 0x1000
+; AVX512F-NEXT:    jne LBB14_25
+; AVX512F-NEXT:  LBB14_26: ## %else24
+; AVX512F-NEXT:    testl $8192, %eax ## imm = 0x2000
+; AVX512F-NEXT:    jne LBB14_27
+; AVX512F-NEXT:  LBB14_28: ## %else26
+; AVX512F-NEXT:    testl $16384, %eax ## imm = 0x4000
+; AVX512F-NEXT:    jne LBB14_29
+; AVX512F-NEXT:  LBB14_30: ## %else28
+; AVX512F-NEXT:    testl $32768, %eax ## imm = 0x8000
+; AVX512F-NEXT:    jne LBB14_31
+; AVX512F-NEXT:  LBB14_32: ## %else30
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+; AVX512F-NEXT:  LBB14_1: ## %cond.store
 ; AVX512F-NEXT:    vpextrw $0, %xmm1, (%rdi)
-; AVX512F-NEXT:  LBB14_2: ## %else
-; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $2, %al
 ; AVX512F-NEXT:    je LBB14_4
-; AVX512F-NEXT:  ## %bb.3: ## %cond.store1
+; AVX512F-NEXT:  LBB14_3: ## %cond.store1
 ; AVX512F-NEXT:    vpextrw $1, %xmm1, 2(%rdi)
-; AVX512F-NEXT:  LBB14_4: ## %else2
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqw %ymm2, %ymm0, %ymm2
-; AVX512F-NEXT:    vpmovsxwd %ymm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $2, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $4, %al
 ; AVX512F-NEXT:    je LBB14_6
-; AVX512F-NEXT:  ## %bb.5: ## %cond.store3
+; AVX512F-NEXT:  LBB14_5: ## %cond.store3
 ; AVX512F-NEXT:    vpextrw $2, %xmm1, 4(%rdi)
-; AVX512F-NEXT:  LBB14_6: ## %else4
-; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $8, %al
 ; AVX512F-NEXT:    je LBB14_8
-; AVX512F-NEXT:  ## %bb.7: ## %cond.store5
+; AVX512F-NEXT:  LBB14_7: ## %cond.store5
 ; AVX512F-NEXT:    vpextrw $3, %xmm1, 6(%rdi)
-; AVX512F-NEXT:  LBB14_8: ## %else6
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqw %ymm2, %ymm0, %ymm2
-; AVX512F-NEXT:    vpmovsxwd %ymm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $4, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $16, %al
 ; AVX512F-NEXT:    je LBB14_10
-; AVX512F-NEXT:  ## %bb.9: ## %cond.store7
+; AVX512F-NEXT:  LBB14_9: ## %cond.store7
 ; AVX512F-NEXT:    vpextrw $4, %xmm1, 8(%rdi)
-; AVX512F-NEXT:  LBB14_10: ## %else8
-; AVX512F-NEXT:    kshiftrw $5, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $32, %al
 ; AVX512F-NEXT:    je LBB14_12
-; AVX512F-NEXT:  ## %bb.11: ## %cond.store9
+; AVX512F-NEXT:  LBB14_11: ## %cond.store9
 ; AVX512F-NEXT:    vpextrw $5, %xmm1, 10(%rdi)
-; AVX512F-NEXT:  LBB14_12: ## %else10
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqw %ymm2, %ymm0, %ymm2
-; AVX512F-NEXT:    vpmovsxwd %ymm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $6, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $64, %al
 ; AVX512F-NEXT:    je LBB14_14
-; AVX512F-NEXT:  ## %bb.13: ## %cond.store11
+; AVX512F-NEXT:  LBB14_13: ## %cond.store11
 ; AVX512F-NEXT:    vpextrw $6, %xmm1, 12(%rdi)
-; AVX512F-NEXT:  LBB14_14: ## %else12
-; AVX512F-NEXT:    kshiftrw $7, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je LBB14_16
-; AVX512F-NEXT:  ## %bb.15: ## %cond.store13
-; AVX512F-NEXT:    vpextrw $7, %xmm1, 14(%rdi)
-; AVX512F-NEXT:  LBB14_16: ## %else14
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqw %ymm2, %ymm0, %ymm2
-; AVX512F-NEXT:    vpmovsxwd %ymm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $8, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm1
-; AVX512F-NEXT:    je LBB14_18
-; AVX512F-NEXT:  ## %bb.17: ## %cond.store15
-; AVX512F-NEXT:    vpextrw $0, %xmm1, 16(%rdi)
-; AVX512F-NEXT:  LBB14_18: ## %else16
-; AVX512F-NEXT:    kshiftrw $9, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $-128, %al
+; AVX512F-NEXT:    jne LBB14_15
+; AVX512F-NEXT:    jmp LBB14_16
+; AVX512F-NEXT:  LBB14_17: ## %cond.store15
+; AVX512F-NEXT:    vpextrw $0, %xmm0, 16(%rdi)
+; AVX512F-NEXT:    testl $512, %eax ## imm = 0x200
 ; AVX512F-NEXT:    je LBB14_20
-; AVX512F-NEXT:  ## %bb.19: ## %cond.store17
-; AVX512F-NEXT:    vpextrw $1, %xmm1, 18(%rdi)
-; AVX512F-NEXT:  LBB14_20: ## %else18
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqw %ymm2, %ymm0, %ymm2
-; AVX512F-NEXT:    vpmovsxwd %ymm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $10, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  LBB14_19: ## %cond.store17
+; AVX512F-NEXT:    vpextrw $1, %xmm0, 18(%rdi)
+; AVX512F-NEXT:    testl $1024, %eax ## imm = 0x400
 ; AVX512F-NEXT:    je LBB14_22
-; AVX512F-NEXT:  ## %bb.21: ## %cond.store19
-; AVX512F-NEXT:    vpextrw $2, %xmm1, 20(%rdi)
-; AVX512F-NEXT:  LBB14_22: ## %else20
-; AVX512F-NEXT:    kshiftrw $11, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  LBB14_21: ## %cond.store19
+; AVX512F-NEXT:    vpextrw $2, %xmm0, 20(%rdi)
+; AVX512F-NEXT:    testl $2048, %eax ## imm = 0x800
 ; AVX512F-NEXT:    je LBB14_24
-; AVX512F-NEXT:  ## %bb.23: ## %cond.store21
-; AVX512F-NEXT:    vpextrw $3, %xmm1, 22(%rdi)
-; AVX512F-NEXT:  LBB14_24: ## %else22
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqw %ymm2, %ymm0, %ymm2
-; AVX512F-NEXT:    vpmovsxwd %ymm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $12, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  LBB14_23: ## %cond.store21
+; AVX512F-NEXT:    vpextrw $3, %xmm0, 22(%rdi)
+; AVX512F-NEXT:    testl $4096, %eax ## imm = 0x1000
 ; AVX512F-NEXT:    je LBB14_26
-; AVX512F-NEXT:  ## %bb.25: ## %cond.store23
-; AVX512F-NEXT:    vpextrw $4, %xmm1, 24(%rdi)
-; AVX512F-NEXT:  LBB14_26: ## %else24
-; AVX512F-NEXT:    kshiftrw $13, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  LBB14_25: ## %cond.store23
+; AVX512F-NEXT:    vpextrw $4, %xmm0, 24(%rdi)
+; AVX512F-NEXT:    testl $8192, %eax ## imm = 0x2000
 ; AVX512F-NEXT:    je LBB14_28
-; AVX512F-NEXT:  ## %bb.27: ## %cond.store25
-; AVX512F-NEXT:    vpextrw $5, %xmm1, 26(%rdi)
-; AVX512F-NEXT:  LBB14_28: ## %else26
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqw %ymm2, %ymm0, %ymm0
-; AVX512F-NEXT:    vpmovsxwd %ymm0, %zmm0
-; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k0
-; AVX512F-NEXT:    kshiftrw $14, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  LBB14_27: ## %cond.store25
+; AVX512F-NEXT:    vpextrw $5, %xmm0, 26(%rdi)
+; AVX512F-NEXT:    testl $16384, %eax ## imm = 0x4000
 ; AVX512F-NEXT:    je LBB14_30
-; AVX512F-NEXT:  ## %bb.29: ## %cond.store27
-; AVX512F-NEXT:    vpextrw $6, %xmm1, 28(%rdi)
-; AVX512F-NEXT:  LBB14_30: ## %else28
-; AVX512F-NEXT:    kshiftrw $15, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  LBB14_29: ## %cond.store27
+; AVX512F-NEXT:    vpextrw $6, %xmm0, 28(%rdi)
+; AVX512F-NEXT:    testl $32768, %eax ## imm = 0x8000
 ; AVX512F-NEXT:    je LBB14_32
-; AVX512F-NEXT:  ## %bb.31: ## %cond.store29
-; AVX512F-NEXT:    vpextrw $7, %xmm1, 30(%rdi)
-; AVX512F-NEXT:  LBB14_32: ## %else30
+; AVX512F-NEXT:  LBB14_31: ## %cond.store29
+; AVX512F-NEXT:    vpextrw $7, %xmm0, 30(%rdi)
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VLDQ-LABEL: store_v16i16_v16i16:
 ; AVX512VLDQ:       ## %bb.0:
 ; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpeqw %ymm2, %ymm0, %ymm2
-; AVX512VLDQ-NEXT:    vpmovsxwd %ymm2, %zmm2
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm2, %k0
+; AVX512VLDQ-NEXT:    vpcmpeqw %ymm2, %ymm0, %ymm0
+; AVX512VLDQ-NEXT:    vpmovsxwd %ymm0, %zmm0
+; AVX512VLDQ-NEXT:    vpmovd2m %zmm0, %k0
 ; AVX512VLDQ-NEXT:    kmovw %k0, %eax
 ; AVX512VLDQ-NEXT:    testb $1, %al
-; AVX512VLDQ-NEXT:    je LBB14_2
-; AVX512VLDQ-NEXT:  ## %bb.1: ## %cond.store
+; AVX512VLDQ-NEXT:    jne LBB14_1
+; AVX512VLDQ-NEXT:  ## %bb.2: ## %else
+; AVX512VLDQ-NEXT:    testb $2, %al
+; AVX512VLDQ-NEXT:    jne LBB14_3
+; AVX512VLDQ-NEXT:  LBB14_4: ## %else2
+; AVX512VLDQ-NEXT:    testb $4, %al
+; AVX512VLDQ-NEXT:    jne LBB14_5
+; AVX512VLDQ-NEXT:  LBB14_6: ## %else4
+; AVX512VLDQ-NEXT:    testb $8, %al
+; AVX512VLDQ-NEXT:    jne LBB14_7
+; AVX512VLDQ-NEXT:  LBB14_8: ## %else6
+; AVX512VLDQ-NEXT:    testb $16, %al
+; AVX512VLDQ-NEXT:    jne LBB14_9
+; AVX512VLDQ-NEXT:  LBB14_10: ## %else8
+; AVX512VLDQ-NEXT:    testb $32, %al
+; AVX512VLDQ-NEXT:    jne LBB14_11
+; AVX512VLDQ-NEXT:  LBB14_12: ## %else10
+; AVX512VLDQ-NEXT:    testb $64, %al
+; AVX512VLDQ-NEXT:    jne LBB14_13
+; AVX512VLDQ-NEXT:  LBB14_14: ## %else12
+; AVX512VLDQ-NEXT:    testb $-128, %al
+; AVX512VLDQ-NEXT:    je LBB14_16
+; AVX512VLDQ-NEXT:  LBB14_15: ## %cond.store13
+; AVX512VLDQ-NEXT:    vpextrw $7, %xmm1, 14(%rdi)
+; AVX512VLDQ-NEXT:  LBB14_16: ## %else14
+; AVX512VLDQ-NEXT:    testl $256, %eax ## imm = 0x100
+; AVX512VLDQ-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX512VLDQ-NEXT:    jne LBB14_17
+; AVX512VLDQ-NEXT:  ## %bb.18: ## %else16
+; AVX512VLDQ-NEXT:    testl $512, %eax ## imm = 0x200
+; AVX512VLDQ-NEXT:    jne LBB14_19
+; AVX512VLDQ-NEXT:  LBB14_20: ## %else18
+; AVX512VLDQ-NEXT:    testl $1024, %eax ## imm = 0x400
+; AVX512VLDQ-NEXT:    jne LBB14_21
+; AVX512VLDQ-NEXT:  LBB14_22: ## %else20
+; AVX512VLDQ-NEXT:    testl $2048, %eax ## imm = 0x800
+; AVX512VLDQ-NEXT:    jne LBB14_23
+; AVX512VLDQ-NEXT:  LBB14_24: ## %else22
+; AVX512VLDQ-NEXT:    testl $4096, %eax ## imm = 0x1000
+; AVX512VLDQ-NEXT:    jne LBB14_25
+; AVX512VLDQ-NEXT:  LBB14_26: ## %else24
+; AVX512VLDQ-NEXT:    testl $8192, %eax ## imm = 0x2000
+; AVX512VLDQ-NEXT:    jne LBB14_27
+; AVX512VLDQ-NEXT:  LBB14_28: ## %else26
+; AVX512VLDQ-NEXT:    testl $16384, %eax ## imm = 0x4000
+; AVX512VLDQ-NEXT:    jne LBB14_29
+; AVX512VLDQ-NEXT:  LBB14_30: ## %else28
+; AVX512VLDQ-NEXT:    testl $32768, %eax ## imm = 0x8000
+; AVX512VLDQ-NEXT:    jne LBB14_31
+; AVX512VLDQ-NEXT:  LBB14_32: ## %else30
+; AVX512VLDQ-NEXT:    vzeroupper
+; AVX512VLDQ-NEXT:    retq
+; AVX512VLDQ-NEXT:  LBB14_1: ## %cond.store
 ; AVX512VLDQ-NEXT:    vpextrw $0, %xmm1, (%rdi)
-; AVX512VLDQ-NEXT:  LBB14_2: ## %else
-; AVX512VLDQ-NEXT:    kshiftrw $1, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $2, %al
 ; AVX512VLDQ-NEXT:    je LBB14_4
-; AVX512VLDQ-NEXT:  ## %bb.3: ## %cond.store1
+; AVX512VLDQ-NEXT:  LBB14_3: ## %cond.store1
 ; AVX512VLDQ-NEXT:    vpextrw $1, %xmm1, 2(%rdi)
-; AVX512VLDQ-NEXT:  LBB14_4: ## %else2
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpeqw %ymm2, %ymm0, %ymm2
-; AVX512VLDQ-NEXT:    vpmovsxwd %ymm2, %zmm2
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm2, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $2, %k0, %k1
-; AVX512VLDQ-NEXT:    kmovw %k1, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $4, %al
 ; AVX512VLDQ-NEXT:    je LBB14_6
-; AVX512VLDQ-NEXT:  ## %bb.5: ## %cond.store3
+; AVX512VLDQ-NEXT:  LBB14_5: ## %cond.store3
 ; AVX512VLDQ-NEXT:    vpextrw $2, %xmm1, 4(%rdi)
-; AVX512VLDQ-NEXT:  LBB14_6: ## %else4
-; AVX512VLDQ-NEXT:    kshiftrw $3, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $8, %al
 ; AVX512VLDQ-NEXT:    je LBB14_8
-; AVX512VLDQ-NEXT:  ## %bb.7: ## %cond.store5
+; AVX512VLDQ-NEXT:  LBB14_7: ## %cond.store5
 ; AVX512VLDQ-NEXT:    vpextrw $3, %xmm1, 6(%rdi)
-; AVX512VLDQ-NEXT:  LBB14_8: ## %else6
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpeqw %ymm2, %ymm0, %ymm2
-; AVX512VLDQ-NEXT:    vpmovsxwd %ymm2, %zmm2
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm2, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $4, %k0, %k1
-; AVX512VLDQ-NEXT:    kmovw %k1, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $16, %al
 ; AVX512VLDQ-NEXT:    je LBB14_10
-; AVX512VLDQ-NEXT:  ## %bb.9: ## %cond.store7
+; AVX512VLDQ-NEXT:  LBB14_9: ## %cond.store7
 ; AVX512VLDQ-NEXT:    vpextrw $4, %xmm1, 8(%rdi)
-; AVX512VLDQ-NEXT:  LBB14_10: ## %else8
-; AVX512VLDQ-NEXT:    kshiftrw $5, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $32, %al
 ; AVX512VLDQ-NEXT:    je LBB14_12
-; AVX512VLDQ-NEXT:  ## %bb.11: ## %cond.store9
+; AVX512VLDQ-NEXT:  LBB14_11: ## %cond.store9
 ; AVX512VLDQ-NEXT:    vpextrw $5, %xmm1, 10(%rdi)
-; AVX512VLDQ-NEXT:  LBB14_12: ## %else10
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpeqw %ymm2, %ymm0, %ymm2
-; AVX512VLDQ-NEXT:    vpmovsxwd %ymm2, %zmm2
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm2, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $6, %k0, %k1
-; AVX512VLDQ-NEXT:    kmovw %k1, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $64, %al
 ; AVX512VLDQ-NEXT:    je LBB14_14
-; AVX512VLDQ-NEXT:  ## %bb.13: ## %cond.store11
+; AVX512VLDQ-NEXT:  LBB14_13: ## %cond.store11
 ; AVX512VLDQ-NEXT:    vpextrw $6, %xmm1, 12(%rdi)
-; AVX512VLDQ-NEXT:  LBB14_14: ## %else12
-; AVX512VLDQ-NEXT:    kshiftrw $7, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
-; AVX512VLDQ-NEXT:    je LBB14_16
-; AVX512VLDQ-NEXT:  ## %bb.15: ## %cond.store13
-; AVX512VLDQ-NEXT:    vpextrw $7, %xmm1, 14(%rdi)
-; AVX512VLDQ-NEXT:  LBB14_16: ## %else14
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpeqw %ymm2, %ymm0, %ymm2
-; AVX512VLDQ-NEXT:    vpmovsxwd %ymm2, %zmm2
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm2, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $8, %k0, %k1
-; AVX512VLDQ-NEXT:    kmovw %k1, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
-; AVX512VLDQ-NEXT:    vextracti128 $1, %ymm1, %xmm1
-; AVX512VLDQ-NEXT:    je LBB14_18
-; AVX512VLDQ-NEXT:  ## %bb.17: ## %cond.store15
-; AVX512VLDQ-NEXT:    vpextrw $0, %xmm1, 16(%rdi)
-; AVX512VLDQ-NEXT:  LBB14_18: ## %else16
-; AVX512VLDQ-NEXT:    kshiftrw $9, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $-128, %al
+; AVX512VLDQ-NEXT:    jne LBB14_15
+; AVX512VLDQ-NEXT:    jmp LBB14_16
+; AVX512VLDQ-NEXT:  LBB14_17: ## %cond.store15
+; AVX512VLDQ-NEXT:    vpextrw $0, %xmm0, 16(%rdi)
+; AVX512VLDQ-NEXT:    testl $512, %eax ## imm = 0x200
 ; AVX512VLDQ-NEXT:    je LBB14_20
-; AVX512VLDQ-NEXT:  ## %bb.19: ## %cond.store17
-; AVX512VLDQ-NEXT:    vpextrw $1, %xmm1, 18(%rdi)
-; AVX512VLDQ-NEXT:  LBB14_20: ## %else18
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpeqw %ymm2, %ymm0, %ymm2
-; AVX512VLDQ-NEXT:    vpmovsxwd %ymm2, %zmm2
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm2, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $10, %k0, %k1
-; AVX512VLDQ-NEXT:    kmovw %k1, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:  LBB14_19: ## %cond.store17
+; AVX512VLDQ-NEXT:    vpextrw $1, %xmm0, 18(%rdi)
+; AVX512VLDQ-NEXT:    testl $1024, %eax ## imm = 0x400
 ; AVX512VLDQ-NEXT:    je LBB14_22
-; AVX512VLDQ-NEXT:  ## %bb.21: ## %cond.store19
-; AVX512VLDQ-NEXT:    vpextrw $2, %xmm1, 20(%rdi)
-; AVX512VLDQ-NEXT:  LBB14_22: ## %else20
-; AVX512VLDQ-NEXT:    kshiftrw $11, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:  LBB14_21: ## %cond.store19
+; AVX512VLDQ-NEXT:    vpextrw $2, %xmm0, 20(%rdi)
+; AVX512VLDQ-NEXT:    testl $2048, %eax ## imm = 0x800
 ; AVX512VLDQ-NEXT:    je LBB14_24
-; AVX512VLDQ-NEXT:  ## %bb.23: ## %cond.store21
-; AVX512VLDQ-NEXT:    vpextrw $3, %xmm1, 22(%rdi)
-; AVX512VLDQ-NEXT:  LBB14_24: ## %else22
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpeqw %ymm2, %ymm0, %ymm2
-; AVX512VLDQ-NEXT:    vpmovsxwd %ymm2, %zmm2
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm2, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $12, %k0, %k1
-; AVX512VLDQ-NEXT:    kmovw %k1, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:  LBB14_23: ## %cond.store21
+; AVX512VLDQ-NEXT:    vpextrw $3, %xmm0, 22(%rdi)
+; AVX512VLDQ-NEXT:    testl $4096, %eax ## imm = 0x1000
 ; AVX512VLDQ-NEXT:    je LBB14_26
-; AVX512VLDQ-NEXT:  ## %bb.25: ## %cond.store23
-; AVX512VLDQ-NEXT:    vpextrw $4, %xmm1, 24(%rdi)
-; AVX512VLDQ-NEXT:  LBB14_26: ## %else24
-; AVX512VLDQ-NEXT:    kshiftrw $13, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:  LBB14_25: ## %cond.store23
+; AVX512VLDQ-NEXT:    vpextrw $4, %xmm0, 24(%rdi)
+; AVX512VLDQ-NEXT:    testl $8192, %eax ## imm = 0x2000
 ; AVX512VLDQ-NEXT:    je LBB14_28
-; AVX512VLDQ-NEXT:  ## %bb.27: ## %cond.store25
-; AVX512VLDQ-NEXT:    vpextrw $5, %xmm1, 26(%rdi)
-; AVX512VLDQ-NEXT:  LBB14_28: ## %else26
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpeqw %ymm2, %ymm0, %ymm0
-; AVX512VLDQ-NEXT:    vpmovsxwd %ymm0, %zmm0
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm0, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $14, %k0, %k1
-; AVX512VLDQ-NEXT:    kmovw %k1, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:  LBB14_27: ## %cond.store25
+; AVX512VLDQ-NEXT:    vpextrw $5, %xmm0, 26(%rdi)
+; AVX512VLDQ-NEXT:    testl $16384, %eax ## imm = 0x4000
 ; AVX512VLDQ-NEXT:    je LBB14_30
-; AVX512VLDQ-NEXT:  ## %bb.29: ## %cond.store27
-; AVX512VLDQ-NEXT:    vpextrw $6, %xmm1, 28(%rdi)
-; AVX512VLDQ-NEXT:  LBB14_30: ## %else28
-; AVX512VLDQ-NEXT:    kshiftrw $15, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:  LBB14_29: ## %cond.store27
+; AVX512VLDQ-NEXT:    vpextrw $6, %xmm0, 28(%rdi)
+; AVX512VLDQ-NEXT:    testl $32768, %eax ## imm = 0x8000
 ; AVX512VLDQ-NEXT:    je LBB14_32
-; AVX512VLDQ-NEXT:  ## %bb.31: ## %cond.store29
-; AVX512VLDQ-NEXT:    vpextrw $7, %xmm1, 30(%rdi)
-; AVX512VLDQ-NEXT:  LBB14_32: ## %else30
+; AVX512VLDQ-NEXT:  LBB14_31: ## %cond.store29
+; AVX512VLDQ-NEXT:    vpextrw $7, %xmm0, 30(%rdi)
 ; AVX512VLDQ-NEXT:    vzeroupper
 ; AVX512VLDQ-NEXT:    retq
 ;
@@ -2653,656 +2524,579 @@ define void @store_v16i8_v16i8(<16 x i8> %trigger, <16 x i8>* %addr, <16 x i8> %
 ; SSE2:       ## %bb.0:
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
 ; SSE2-NEXT:    pcmpeqb %xmm0, %xmm2
-; SSE2-NEXT:    movd %xmm2, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    movd %xmm1, %eax
-; SSE2-NEXT:    je LBB15_2
-; SSE2-NEXT:  ## %bb.1: ## %cond.store
-; SSE2-NEXT:    movb %al, (%rdi)
-; SSE2-NEXT:  LBB15_2: ## %else
-; SSE2-NEXT:    shrl $8, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    je LBB15_4
-; SSE2-NEXT:  ## %bb.3: ## %cond.store1
-; SSE2-NEXT:    movb %ah, 1(%rdi)
+; SSE2-NEXT:    pmovmskb %xmm2, %eax
+; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    movd %xmm1, %ecx
+; SSE2-NEXT:    jne LBB15_1
+; SSE2-NEXT:  ## %bb.2: ## %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne LBB15_3
 ; SSE2-NEXT:  LBB15_4: ## %else2
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpeqb %xmm0, %xmm2
-; SSE2-NEXT:    movd %xmm2, %ecx
-; SSE2-NEXT:    movl %ecx, %edx
-; SSE2-NEXT:    shrl $16, %edx
-; SSE2-NEXT:    testb $1, %dl
-; SSE2-NEXT:    je LBB15_6
-; SSE2-NEXT:  ## %bb.5: ## %cond.store3
-; SSE2-NEXT:    movl %eax, %edx
-; SSE2-NEXT:    shrl $16, %edx
-; SSE2-NEXT:    movb %dl, 2(%rdi)
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne LBB15_5
 ; SSE2-NEXT:  LBB15_6: ## %else4
-; SSE2-NEXT:    shrl $24, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je LBB15_8
-; SSE2-NEXT:  ## %bb.7: ## %cond.store5
-; SSE2-NEXT:    shrl $24, %eax
-; SSE2-NEXT:    movb %al, 3(%rdi)
+; SSE2-NEXT:  LBB15_7: ## %cond.store5
+; SSE2-NEXT:    shrl $24, %ecx
+; SSE2-NEXT:    movb %cl, 3(%rdi)
 ; SSE2-NEXT:  LBB15_8: ## %else6
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpeqb %xmm0, %xmm2
-; SSE2-NEXT:    pextrw $2, %xmm2, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    pextrw $2, %xmm1, %eax
+; SSE2-NEXT:    testb $16, %al
+; SSE2-NEXT:    pextrw $2, %xmm1, %ecx
 ; SSE2-NEXT:    je LBB15_10
 ; SSE2-NEXT:  ## %bb.9: ## %cond.store7
-; SSE2-NEXT:    movb %al, 4(%rdi)
+; SSE2-NEXT:    movb %cl, 4(%rdi)
 ; SSE2-NEXT:  LBB15_10: ## %else8
-; SSE2-NEXT:    shrl $8, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testb $32, %al
 ; SSE2-NEXT:    je LBB15_12
 ; SSE2-NEXT:  ## %bb.11: ## %cond.store9
-; SSE2-NEXT:    movb %ah, 5(%rdi)
+; SSE2-NEXT:    movb %ch, 5(%rdi)
 ; SSE2-NEXT:  LBB15_12: ## %else10
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpeqb %xmm0, %xmm2
-; SSE2-NEXT:    pextrw $3, %xmm2, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    pextrw $3, %xmm1, %eax
+; SSE2-NEXT:    testb $64, %al
+; SSE2-NEXT:    pextrw $3, %xmm1, %ecx
 ; SSE2-NEXT:    je LBB15_14
 ; SSE2-NEXT:  ## %bb.13: ## %cond.store11
-; SSE2-NEXT:    movb %al, 6(%rdi)
+; SSE2-NEXT:    movb %cl, 6(%rdi)
 ; SSE2-NEXT:  LBB15_14: ## %else12
-; SSE2-NEXT:    shrl $8, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testb $-128, %al
 ; SSE2-NEXT:    je LBB15_16
 ; SSE2-NEXT:  ## %bb.15: ## %cond.store13
-; SSE2-NEXT:    movb %ah, 7(%rdi)
+; SSE2-NEXT:    movb %ch, 7(%rdi)
 ; SSE2-NEXT:  LBB15_16: ## %else14
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpeqb %xmm0, %xmm2
-; SSE2-NEXT:    pextrw $4, %xmm2, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    pextrw $4, %xmm1, %eax
+; SSE2-NEXT:    testl $256, %eax ## imm = 0x100
+; SSE2-NEXT:    pextrw $4, %xmm1, %ecx
 ; SSE2-NEXT:    je LBB15_18
 ; SSE2-NEXT:  ## %bb.17: ## %cond.store15
-; SSE2-NEXT:    movb %al, 8(%rdi)
+; SSE2-NEXT:    movb %cl, 8(%rdi)
 ; SSE2-NEXT:  LBB15_18: ## %else16
-; SSE2-NEXT:    shrl $8, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $512, %eax ## imm = 0x200
 ; SSE2-NEXT:    je LBB15_20
 ; SSE2-NEXT:  ## %bb.19: ## %cond.store17
-; SSE2-NEXT:    movb %ah, 9(%rdi)
+; SSE2-NEXT:    movb %ch, 9(%rdi)
 ; SSE2-NEXT:  LBB15_20: ## %else18
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpeqb %xmm0, %xmm2
-; SSE2-NEXT:    pextrw $5, %xmm2, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    pextrw $5, %xmm1, %eax
+; SSE2-NEXT:    testl $1024, %eax ## imm = 0x400
+; SSE2-NEXT:    pextrw $5, %xmm1, %ecx
 ; SSE2-NEXT:    je LBB15_22
 ; SSE2-NEXT:  ## %bb.21: ## %cond.store19
-; SSE2-NEXT:    movb %al, 10(%rdi)
+; SSE2-NEXT:    movb %cl, 10(%rdi)
 ; SSE2-NEXT:  LBB15_22: ## %else20
-; SSE2-NEXT:    shrl $8, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $2048, %eax ## imm = 0x800
 ; SSE2-NEXT:    je LBB15_24
 ; SSE2-NEXT:  ## %bb.23: ## %cond.store21
-; SSE2-NEXT:    movb %ah, 11(%rdi)
+; SSE2-NEXT:    movb %ch, 11(%rdi)
 ; SSE2-NEXT:  LBB15_24: ## %else22
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpeqb %xmm0, %xmm2
-; SSE2-NEXT:    pextrw $6, %xmm2, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    pextrw $6, %xmm1, %eax
+; SSE2-NEXT:    testl $4096, %eax ## imm = 0x1000
+; SSE2-NEXT:    pextrw $6, %xmm1, %ecx
 ; SSE2-NEXT:    je LBB15_26
 ; SSE2-NEXT:  ## %bb.25: ## %cond.store23
-; SSE2-NEXT:    movb %al, 12(%rdi)
+; SSE2-NEXT:    movb %cl, 12(%rdi)
 ; SSE2-NEXT:  LBB15_26: ## %else24
-; SSE2-NEXT:    shrl $8, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $8192, %eax ## imm = 0x2000
 ; SSE2-NEXT:    je LBB15_28
 ; SSE2-NEXT:  ## %bb.27: ## %cond.store25
-; SSE2-NEXT:    movb %ah, 13(%rdi)
+; SSE2-NEXT:    movb %ch, 13(%rdi)
 ; SSE2-NEXT:  LBB15_28: ## %else26
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpeqb %xmm2, %xmm0
-; SSE2-NEXT:    pextrw $7, %xmm0, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    pextrw $7, %xmm1, %eax
-; SSE2-NEXT:    je LBB15_30
-; SSE2-NEXT:  ## %bb.29: ## %cond.store27
-; SSE2-NEXT:    movb %al, 14(%rdi)
-; SSE2-NEXT:  LBB15_30: ## %else28
-; SSE2-NEXT:    shrl $8, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    je LBB15_32
-; SSE2-NEXT:  ## %bb.31: ## %cond.store29
-; SSE2-NEXT:    movb %ah, 15(%rdi)
+; SSE2-NEXT:    testl $16384, %eax ## imm = 0x4000
+; SSE2-NEXT:    pextrw $7, %xmm1, %ecx
+; SSE2-NEXT:    jne LBB15_29
+; SSE2-NEXT:  ## %bb.30: ## %else28
+; SSE2-NEXT:    testl $32768, %eax ## imm = 0x8000
+; SSE2-NEXT:    jne LBB15_31
 ; SSE2-NEXT:  LBB15_32: ## %else30
 ; SSE2-NEXT:    retq
+; SSE2-NEXT:  LBB15_1: ## %cond.store
+; SSE2-NEXT:    movb %cl, (%rdi)
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    je LBB15_4
+; SSE2-NEXT:  LBB15_3: ## %cond.store1
+; SSE2-NEXT:    movb %ch, 1(%rdi)
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    je LBB15_6
+; SSE2-NEXT:  LBB15_5: ## %cond.store3
+; SSE2-NEXT:    movl %ecx, %edx
+; SSE2-NEXT:    shrl $16, %edx
+; SSE2-NEXT:    movb %dl, 2(%rdi)
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne LBB15_7
+; SSE2-NEXT:    jmp LBB15_8
+; SSE2-NEXT:  LBB15_29: ## %cond.store27
+; SSE2-NEXT:    movb %cl, 14(%rdi)
+; SSE2-NEXT:    testl $32768, %eax ## imm = 0x8000
+; SSE2-NEXT:    je LBB15_32
+; SSE2-NEXT:  LBB15_31: ## %cond.store29
+; SSE2-NEXT:    movb %ch, 15(%rdi)
+; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: store_v16i8_v16i8:
 ; SSE4:       ## %bb.0:
 ; SSE4-NEXT:    pxor %xmm2, %xmm2
 ; SSE4-NEXT:    pcmpeqb %xmm0, %xmm2
-; SSE4-NEXT:    pextrb $0, %xmm2, %eax
+; SSE4-NEXT:    pmovmskb %xmm2, %eax
 ; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je LBB15_2
-; SSE4-NEXT:  ## %bb.1: ## %cond.store
+; SSE4-NEXT:    jne LBB15_1
+; SSE4-NEXT:  ## %bb.2: ## %else
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    jne LBB15_3
+; SSE4-NEXT:  LBB15_4: ## %else2
+; SSE4-NEXT:    testb $4, %al
+; SSE4-NEXT:    jne LBB15_5
+; SSE4-NEXT:  LBB15_6: ## %else4
+; SSE4-NEXT:    testb $8, %al
+; SSE4-NEXT:    jne LBB15_7
+; SSE4-NEXT:  LBB15_8: ## %else6
+; SSE4-NEXT:    testb $16, %al
+; SSE4-NEXT:    jne LBB15_9
+; SSE4-NEXT:  LBB15_10: ## %else8
+; SSE4-NEXT:    testb $32, %al
+; SSE4-NEXT:    jne LBB15_11
+; SSE4-NEXT:  LBB15_12: ## %else10
+; SSE4-NEXT:    testb $64, %al
+; SSE4-NEXT:    jne LBB15_13
+; SSE4-NEXT:  LBB15_14: ## %else12
+; SSE4-NEXT:    testb $-128, %al
+; SSE4-NEXT:    jne LBB15_15
+; SSE4-NEXT:  LBB15_16: ## %else14
+; SSE4-NEXT:    testl $256, %eax ## imm = 0x100
+; SSE4-NEXT:    jne LBB15_17
+; SSE4-NEXT:  LBB15_18: ## %else16
+; SSE4-NEXT:    testl $512, %eax ## imm = 0x200
+; SSE4-NEXT:    jne LBB15_19
+; SSE4-NEXT:  LBB15_20: ## %else18
+; SSE4-NEXT:    testl $1024, %eax ## imm = 0x400
+; SSE4-NEXT:    jne LBB15_21
+; SSE4-NEXT:  LBB15_22: ## %else20
+; SSE4-NEXT:    testl $2048, %eax ## imm = 0x800
+; SSE4-NEXT:    jne LBB15_23
+; SSE4-NEXT:  LBB15_24: ## %else22
+; SSE4-NEXT:    testl $4096, %eax ## imm = 0x1000
+; SSE4-NEXT:    jne LBB15_25
+; SSE4-NEXT:  LBB15_26: ## %else24
+; SSE4-NEXT:    testl $8192, %eax ## imm = 0x2000
+; SSE4-NEXT:    jne LBB15_27
+; SSE4-NEXT:  LBB15_28: ## %else26
+; SSE4-NEXT:    testl $16384, %eax ## imm = 0x4000
+; SSE4-NEXT:    jne LBB15_29
+; SSE4-NEXT:  LBB15_30: ## %else28
+; SSE4-NEXT:    testl $32768, %eax ## imm = 0x8000
+; SSE4-NEXT:    jne LBB15_31
+; SSE4-NEXT:  LBB15_32: ## %else30
+; SSE4-NEXT:    retq
+; SSE4-NEXT:  LBB15_1: ## %cond.store
 ; SSE4-NEXT:    pextrb $0, %xmm1, (%rdi)
-; SSE4-NEXT:  LBB15_2: ## %else
-; SSE4-NEXT:    pextrb $1, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $2, %al
 ; SSE4-NEXT:    je LBB15_4
-; SSE4-NEXT:  ## %bb.3: ## %cond.store1
+; SSE4-NEXT:  LBB15_3: ## %cond.store1
 ; SSE4-NEXT:    pextrb $1, %xmm1, 1(%rdi)
-; SSE4-NEXT:  LBB15_4: ## %else2
-; SSE4-NEXT:    pxor %xmm2, %xmm2
-; SSE4-NEXT:    pcmpeqb %xmm0, %xmm2
-; SSE4-NEXT:    pextrb $2, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $4, %al
 ; SSE4-NEXT:    je LBB15_6
-; SSE4-NEXT:  ## %bb.5: ## %cond.store3
+; SSE4-NEXT:  LBB15_5: ## %cond.store3
 ; SSE4-NEXT:    pextrb $2, %xmm1, 2(%rdi)
-; SSE4-NEXT:  LBB15_6: ## %else4
-; SSE4-NEXT:    pextrb $3, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $8, %al
 ; SSE4-NEXT:    je LBB15_8
-; SSE4-NEXT:  ## %bb.7: ## %cond.store5
+; SSE4-NEXT:  LBB15_7: ## %cond.store5
 ; SSE4-NEXT:    pextrb $3, %xmm1, 3(%rdi)
-; SSE4-NEXT:  LBB15_8: ## %else6
-; SSE4-NEXT:    pxor %xmm2, %xmm2
-; SSE4-NEXT:    pcmpeqb %xmm0, %xmm2
-; SSE4-NEXT:    pextrb $4, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $16, %al
 ; SSE4-NEXT:    je LBB15_10
-; SSE4-NEXT:  ## %bb.9: ## %cond.store7
+; SSE4-NEXT:  LBB15_9: ## %cond.store7
 ; SSE4-NEXT:    pextrb $4, %xmm1, 4(%rdi)
-; SSE4-NEXT:  LBB15_10: ## %else8
-; SSE4-NEXT:    pextrb $5, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $32, %al
 ; SSE4-NEXT:    je LBB15_12
-; SSE4-NEXT:  ## %bb.11: ## %cond.store9
+; SSE4-NEXT:  LBB15_11: ## %cond.store9
 ; SSE4-NEXT:    pextrb $5, %xmm1, 5(%rdi)
-; SSE4-NEXT:  LBB15_12: ## %else10
-; SSE4-NEXT:    pxor %xmm2, %xmm2
-; SSE4-NEXT:    pcmpeqb %xmm0, %xmm2
-; SSE4-NEXT:    pextrb $6, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $64, %al
 ; SSE4-NEXT:    je LBB15_14
-; SSE4-NEXT:  ## %bb.13: ## %cond.store11
+; SSE4-NEXT:  LBB15_13: ## %cond.store11
 ; SSE4-NEXT:    pextrb $6, %xmm1, 6(%rdi)
-; SSE4-NEXT:  LBB15_14: ## %else12
-; SSE4-NEXT:    pextrb $7, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $-128, %al
 ; SSE4-NEXT:    je LBB15_16
-; SSE4-NEXT:  ## %bb.15: ## %cond.store13
+; SSE4-NEXT:  LBB15_15: ## %cond.store13
 ; SSE4-NEXT:    pextrb $7, %xmm1, 7(%rdi)
-; SSE4-NEXT:  LBB15_16: ## %else14
-; SSE4-NEXT:    pxor %xmm2, %xmm2
-; SSE4-NEXT:    pcmpeqb %xmm0, %xmm2
-; SSE4-NEXT:    pextrb $8, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $256, %eax ## imm = 0x100
 ; SSE4-NEXT:    je LBB15_18
-; SSE4-NEXT:  ## %bb.17: ## %cond.store15
+; SSE4-NEXT:  LBB15_17: ## %cond.store15
 ; SSE4-NEXT:    pextrb $8, %xmm1, 8(%rdi)
-; SSE4-NEXT:  LBB15_18: ## %else16
-; SSE4-NEXT:    pextrb $9, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $512, %eax ## imm = 0x200
 ; SSE4-NEXT:    je LBB15_20
-; SSE4-NEXT:  ## %bb.19: ## %cond.store17
+; SSE4-NEXT:  LBB15_19: ## %cond.store17
 ; SSE4-NEXT:    pextrb $9, %xmm1, 9(%rdi)
-; SSE4-NEXT:  LBB15_20: ## %else18
-; SSE4-NEXT:    pxor %xmm2, %xmm2
-; SSE4-NEXT:    pcmpeqb %xmm0, %xmm2
-; SSE4-NEXT:    pextrb $10, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $1024, %eax ## imm = 0x400
 ; SSE4-NEXT:    je LBB15_22
-; SSE4-NEXT:  ## %bb.21: ## %cond.store19
+; SSE4-NEXT:  LBB15_21: ## %cond.store19
 ; SSE4-NEXT:    pextrb $10, %xmm1, 10(%rdi)
-; SSE4-NEXT:  LBB15_22: ## %else20
-; SSE4-NEXT:    pextrb $11, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $2048, %eax ## imm = 0x800
 ; SSE4-NEXT:    je LBB15_24
-; SSE4-NEXT:  ## %bb.23: ## %cond.store21
+; SSE4-NEXT:  LBB15_23: ## %cond.store21
 ; SSE4-NEXT:    pextrb $11, %xmm1, 11(%rdi)
-; SSE4-NEXT:  LBB15_24: ## %else22
-; SSE4-NEXT:    pxor %xmm2, %xmm2
-; SSE4-NEXT:    pcmpeqb %xmm0, %xmm2
-; SSE4-NEXT:    pextrb $12, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $4096, %eax ## imm = 0x1000
 ; SSE4-NEXT:    je LBB15_26
-; SSE4-NEXT:  ## %bb.25: ## %cond.store23
+; SSE4-NEXT:  LBB15_25: ## %cond.store23
 ; SSE4-NEXT:    pextrb $12, %xmm1, 12(%rdi)
-; SSE4-NEXT:  LBB15_26: ## %else24
-; SSE4-NEXT:    pextrb $13, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $8192, %eax ## imm = 0x2000
 ; SSE4-NEXT:    je LBB15_28
-; SSE4-NEXT:  ## %bb.27: ## %cond.store25
+; SSE4-NEXT:  LBB15_27: ## %cond.store25
 ; SSE4-NEXT:    pextrb $13, %xmm1, 13(%rdi)
-; SSE4-NEXT:  LBB15_28: ## %else26
-; SSE4-NEXT:    pxor %xmm2, %xmm2
-; SSE4-NEXT:    pcmpeqb %xmm2, %xmm0
-; SSE4-NEXT:    pextrb $14, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $16384, %eax ## imm = 0x4000
 ; SSE4-NEXT:    je LBB15_30
-; SSE4-NEXT:  ## %bb.29: ## %cond.store27
+; SSE4-NEXT:  LBB15_29: ## %cond.store27
 ; SSE4-NEXT:    pextrb $14, %xmm1, 14(%rdi)
-; SSE4-NEXT:  LBB15_30: ## %else28
-; SSE4-NEXT:    pextrb $15, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $32768, %eax ## imm = 0x8000
 ; SSE4-NEXT:    je LBB15_32
-; SSE4-NEXT:  ## %bb.31: ## %cond.store29
+; SSE4-NEXT:  LBB15_31: ## %cond.store29
 ; SSE4-NEXT:    pextrb $15, %xmm1, 15(%rdi)
-; SSE4-NEXT:  LBB15_32: ## %else30
 ; SSE4-NEXT:    retq
 ;
 ; AVX1OR2-LABEL: store_v16i8_v16i8:
 ; AVX1OR2:       ## %bb.0:
 ; AVX1OR2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1OR2-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm2
-; AVX1OR2-NEXT:    vpextrb $0, %xmm2, %eax
+; AVX1OR2-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm0
+; AVX1OR2-NEXT:    vpmovmskb %xmm0, %eax
 ; AVX1OR2-NEXT:    testb $1, %al
-; AVX1OR2-NEXT:    je LBB15_2
-; AVX1OR2-NEXT:  ## %bb.1: ## %cond.store
+; AVX1OR2-NEXT:    jne LBB15_1
+; AVX1OR2-NEXT:  ## %bb.2: ## %else
+; AVX1OR2-NEXT:    testb $2, %al
+; AVX1OR2-NEXT:    jne LBB15_3
+; AVX1OR2-NEXT:  LBB15_4: ## %else2
+; AVX1OR2-NEXT:    testb $4, %al
+; AVX1OR2-NEXT:    jne LBB15_5
+; AVX1OR2-NEXT:  LBB15_6: ## %else4
+; AVX1OR2-NEXT:    testb $8, %al
+; AVX1OR2-NEXT:    jne LBB15_7
+; AVX1OR2-NEXT:  LBB15_8: ## %else6
+; AVX1OR2-NEXT:    testb $16, %al
+; AVX1OR2-NEXT:    jne LBB15_9
+; AVX1OR2-NEXT:  LBB15_10: ## %else8
+; AVX1OR2-NEXT:    testb $32, %al
+; AVX1OR2-NEXT:    jne LBB15_11
+; AVX1OR2-NEXT:  LBB15_12: ## %else10
+; AVX1OR2-NEXT:    testb $64, %al
+; AVX1OR2-NEXT:    jne LBB15_13
+; AVX1OR2-NEXT:  LBB15_14: ## %else12
+; AVX1OR2-NEXT:    testb $-128, %al
+; AVX1OR2-NEXT:    jne LBB15_15
+; AVX1OR2-NEXT:  LBB15_16: ## %else14
+; AVX1OR2-NEXT:    testl $256, %eax ## imm = 0x100
+; AVX1OR2-NEXT:    jne LBB15_17
+; AVX1OR2-NEXT:  LBB15_18: ## %else16
+; AVX1OR2-NEXT:    testl $512, %eax ## imm = 0x200
+; AVX1OR2-NEXT:    jne LBB15_19
+; AVX1OR2-NEXT:  LBB15_20: ## %else18
+; AVX1OR2-NEXT:    testl $1024, %eax ## imm = 0x400
+; AVX1OR2-NEXT:    jne LBB15_21
+; AVX1OR2-NEXT:  LBB15_22: ## %else20
+; AVX1OR2-NEXT:    testl $2048, %eax ## imm = 0x800
+; AVX1OR2-NEXT:    jne LBB15_23
+; AVX1OR2-NEXT:  LBB15_24: ## %else22
+; AVX1OR2-NEXT:    testl $4096, %eax ## imm = 0x1000
+; AVX1OR2-NEXT:    jne LBB15_25
+; AVX1OR2-NEXT:  LBB15_26: ## %else24
+; AVX1OR2-NEXT:    testl $8192, %eax ## imm = 0x2000
+; AVX1OR2-NEXT:    jne LBB15_27
+; AVX1OR2-NEXT:  LBB15_28: ## %else26
+; AVX1OR2-NEXT:    testl $16384, %eax ## imm = 0x4000
+; AVX1OR2-NEXT:    jne LBB15_29
+; AVX1OR2-NEXT:  LBB15_30: ## %else28
+; AVX1OR2-NEXT:    testl $32768, %eax ## imm = 0x8000
+; AVX1OR2-NEXT:    jne LBB15_31
+; AVX1OR2-NEXT:  LBB15_32: ## %else30
+; AVX1OR2-NEXT:    retq
+; AVX1OR2-NEXT:  LBB15_1: ## %cond.store
 ; AVX1OR2-NEXT:    vpextrb $0, %xmm1, (%rdi)
-; AVX1OR2-NEXT:  LBB15_2: ## %else
-; AVX1OR2-NEXT:    vpextrb $1, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $2, %al
 ; AVX1OR2-NEXT:    je LBB15_4
-; AVX1OR2-NEXT:  ## %bb.3: ## %cond.store1
+; AVX1OR2-NEXT:  LBB15_3: ## %cond.store1
 ; AVX1OR2-NEXT:    vpextrb $1, %xmm1, 1(%rdi)
-; AVX1OR2-NEXT:  LBB15_4: ## %else2
-; AVX1OR2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1OR2-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm2
-; AVX1OR2-NEXT:    vpextrb $2, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $4, %al
 ; AVX1OR2-NEXT:    je LBB15_6
-; AVX1OR2-NEXT:  ## %bb.5: ## %cond.store3
+; AVX1OR2-NEXT:  LBB15_5: ## %cond.store3
 ; AVX1OR2-NEXT:    vpextrb $2, %xmm1, 2(%rdi)
-; AVX1OR2-NEXT:  LBB15_6: ## %else4
-; AVX1OR2-NEXT:    vpextrb $3, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $8, %al
 ; AVX1OR2-NEXT:    je LBB15_8
-; AVX1OR2-NEXT:  ## %bb.7: ## %cond.store5
+; AVX1OR2-NEXT:  LBB15_7: ## %cond.store5
 ; AVX1OR2-NEXT:    vpextrb $3, %xmm1, 3(%rdi)
-; AVX1OR2-NEXT:  LBB15_8: ## %else6
-; AVX1OR2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1OR2-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm2
-; AVX1OR2-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $16, %al
 ; AVX1OR2-NEXT:    je LBB15_10
-; AVX1OR2-NEXT:  ## %bb.9: ## %cond.store7
+; AVX1OR2-NEXT:  LBB15_9: ## %cond.store7
 ; AVX1OR2-NEXT:    vpextrb $4, %xmm1, 4(%rdi)
-; AVX1OR2-NEXT:  LBB15_10: ## %else8
-; AVX1OR2-NEXT:    vpextrb $5, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $32, %al
 ; AVX1OR2-NEXT:    je LBB15_12
-; AVX1OR2-NEXT:  ## %bb.11: ## %cond.store9
+; AVX1OR2-NEXT:  LBB15_11: ## %cond.store9
 ; AVX1OR2-NEXT:    vpextrb $5, %xmm1, 5(%rdi)
-; AVX1OR2-NEXT:  LBB15_12: ## %else10
-; AVX1OR2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1OR2-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm2
-; AVX1OR2-NEXT:    vpextrb $6, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $64, %al
 ; AVX1OR2-NEXT:    je LBB15_14
-; AVX1OR2-NEXT:  ## %bb.13: ## %cond.store11
+; AVX1OR2-NEXT:  LBB15_13: ## %cond.store11
 ; AVX1OR2-NEXT:    vpextrb $6, %xmm1, 6(%rdi)
-; AVX1OR2-NEXT:  LBB15_14: ## %else12
-; AVX1OR2-NEXT:    vpextrb $7, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testb $-128, %al
 ; AVX1OR2-NEXT:    je LBB15_16
-; AVX1OR2-NEXT:  ## %bb.15: ## %cond.store13
+; AVX1OR2-NEXT:  LBB15_15: ## %cond.store13
 ; AVX1OR2-NEXT:    vpextrb $7, %xmm1, 7(%rdi)
-; AVX1OR2-NEXT:  LBB15_16: ## %else14
-; AVX1OR2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1OR2-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm2
-; AVX1OR2-NEXT:    vpextrb $8, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testl $256, %eax ## imm = 0x100
 ; AVX1OR2-NEXT:    je LBB15_18
-; AVX1OR2-NEXT:  ## %bb.17: ## %cond.store15
+; AVX1OR2-NEXT:  LBB15_17: ## %cond.store15
 ; AVX1OR2-NEXT:    vpextrb $8, %xmm1, 8(%rdi)
-; AVX1OR2-NEXT:  LBB15_18: ## %else16
-; AVX1OR2-NEXT:    vpextrb $9, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testl $512, %eax ## imm = 0x200
 ; AVX1OR2-NEXT:    je LBB15_20
-; AVX1OR2-NEXT:  ## %bb.19: ## %cond.store17
+; AVX1OR2-NEXT:  LBB15_19: ## %cond.store17
 ; AVX1OR2-NEXT:    vpextrb $9, %xmm1, 9(%rdi)
-; AVX1OR2-NEXT:  LBB15_20: ## %else18
-; AVX1OR2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1OR2-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm2
-; AVX1OR2-NEXT:    vpextrb $10, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testl $1024, %eax ## imm = 0x400
 ; AVX1OR2-NEXT:    je LBB15_22
-; AVX1OR2-NEXT:  ## %bb.21: ## %cond.store19
+; AVX1OR2-NEXT:  LBB15_21: ## %cond.store19
 ; AVX1OR2-NEXT:    vpextrb $10, %xmm1, 10(%rdi)
-; AVX1OR2-NEXT:  LBB15_22: ## %else20
-; AVX1OR2-NEXT:    vpextrb $11, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testl $2048, %eax ## imm = 0x800
 ; AVX1OR2-NEXT:    je LBB15_24
-; AVX1OR2-NEXT:  ## %bb.23: ## %cond.store21
+; AVX1OR2-NEXT:  LBB15_23: ## %cond.store21
 ; AVX1OR2-NEXT:    vpextrb $11, %xmm1, 11(%rdi)
-; AVX1OR2-NEXT:  LBB15_24: ## %else22
-; AVX1OR2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1OR2-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm2
-; AVX1OR2-NEXT:    vpextrb $12, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testl $4096, %eax ## imm = 0x1000
 ; AVX1OR2-NEXT:    je LBB15_26
-; AVX1OR2-NEXT:  ## %bb.25: ## %cond.store23
+; AVX1OR2-NEXT:  LBB15_25: ## %cond.store23
 ; AVX1OR2-NEXT:    vpextrb $12, %xmm1, 12(%rdi)
-; AVX1OR2-NEXT:  LBB15_26: ## %else24
-; AVX1OR2-NEXT:    vpextrb $13, %xmm2, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testl $8192, %eax ## imm = 0x2000
 ; AVX1OR2-NEXT:    je LBB15_28
-; AVX1OR2-NEXT:  ## %bb.27: ## %cond.store25
+; AVX1OR2-NEXT:  LBB15_27: ## %cond.store25
 ; AVX1OR2-NEXT:    vpextrb $13, %xmm1, 13(%rdi)
-; AVX1OR2-NEXT:  LBB15_28: ## %else26
-; AVX1OR2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1OR2-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm0
-; AVX1OR2-NEXT:    vpextrb $14, %xmm0, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testl $16384, %eax ## imm = 0x4000
 ; AVX1OR2-NEXT:    je LBB15_30
-; AVX1OR2-NEXT:  ## %bb.29: ## %cond.store27
+; AVX1OR2-NEXT:  LBB15_29: ## %cond.store27
 ; AVX1OR2-NEXT:    vpextrb $14, %xmm1, 14(%rdi)
-; AVX1OR2-NEXT:  LBB15_30: ## %else28
-; AVX1OR2-NEXT:    vpextrb $15, %xmm0, %eax
-; AVX1OR2-NEXT:    testb $1, %al
+; AVX1OR2-NEXT:    testl $32768, %eax ## imm = 0x8000
 ; AVX1OR2-NEXT:    je LBB15_32
-; AVX1OR2-NEXT:  ## %bb.31: ## %cond.store29
+; AVX1OR2-NEXT:  LBB15_31: ## %cond.store29
 ; AVX1OR2-NEXT:    vpextrb $15, %xmm1, 15(%rdi)
-; AVX1OR2-NEXT:  LBB15_32: ## %else30
 ; AVX1OR2-NEXT:    retq
 ;
 ; AVX512F-LABEL: store_v16i8_v16i8:
 ; AVX512F:       ## %bb.0:
 ; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm0
+; AVX512F-NEXT:    vpmovmskb %xmm0, %eax
 ; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je LBB15_2
-; AVX512F-NEXT:  ## %bb.1: ## %cond.store
+; AVX512F-NEXT:    jne LBB15_1
+; AVX512F-NEXT:  ## %bb.2: ## %else
+; AVX512F-NEXT:    testb $2, %al
+; AVX512F-NEXT:    jne LBB15_3
+; AVX512F-NEXT:  LBB15_4: ## %else2
+; AVX512F-NEXT:    testb $4, %al
+; AVX512F-NEXT:    jne LBB15_5
+; AVX512F-NEXT:  LBB15_6: ## %else4
+; AVX512F-NEXT:    testb $8, %al
+; AVX512F-NEXT:    jne LBB15_7
+; AVX512F-NEXT:  LBB15_8: ## %else6
+; AVX512F-NEXT:    testb $16, %al
+; AVX512F-NEXT:    jne LBB15_9
+; AVX512F-NEXT:  LBB15_10: ## %else8
+; AVX512F-NEXT:    testb $32, %al
+; AVX512F-NEXT:    jne LBB15_11
+; AVX512F-NEXT:  LBB15_12: ## %else10
+; AVX512F-NEXT:    testb $64, %al
+; AVX512F-NEXT:    jne LBB15_13
+; AVX512F-NEXT:  LBB15_14: ## %else12
+; AVX512F-NEXT:    testb $-128, %al
+; AVX512F-NEXT:    jne LBB15_15
+; AVX512F-NEXT:  LBB15_16: ## %else14
+; AVX512F-NEXT:    testl $256, %eax ## imm = 0x100
+; AVX512F-NEXT:    jne LBB15_17
+; AVX512F-NEXT:  LBB15_18: ## %else16
+; AVX512F-NEXT:    testl $512, %eax ## imm = 0x200
+; AVX512F-NEXT:    jne LBB15_19
+; AVX512F-NEXT:  LBB15_20: ## %else18
+; AVX512F-NEXT:    testl $1024, %eax ## imm = 0x400
+; AVX512F-NEXT:    jne LBB15_21
+; AVX512F-NEXT:  LBB15_22: ## %else20
+; AVX512F-NEXT:    testl $2048, %eax ## imm = 0x800
+; AVX512F-NEXT:    jne LBB15_23
+; AVX512F-NEXT:  LBB15_24: ## %else22
+; AVX512F-NEXT:    testl $4096, %eax ## imm = 0x1000
+; AVX512F-NEXT:    jne LBB15_25
+; AVX512F-NEXT:  LBB15_26: ## %else24
+; AVX512F-NEXT:    testl $8192, %eax ## imm = 0x2000
+; AVX512F-NEXT:    jne LBB15_27
+; AVX512F-NEXT:  LBB15_28: ## %else26
+; AVX512F-NEXT:    testl $16384, %eax ## imm = 0x4000
+; AVX512F-NEXT:    jne LBB15_29
+; AVX512F-NEXT:  LBB15_30: ## %else28
+; AVX512F-NEXT:    testl $32768, %eax ## imm = 0x8000
+; AVX512F-NEXT:    jne LBB15_31
+; AVX512F-NEXT:  LBB15_32: ## %else30
+; AVX512F-NEXT:    retq
+; AVX512F-NEXT:  LBB15_1: ## %cond.store
 ; AVX512F-NEXT:    vpextrb $0, %xmm1, (%rdi)
-; AVX512F-NEXT:  LBB15_2: ## %else
-; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $2, %al
 ; AVX512F-NEXT:    je LBB15_4
-; AVX512F-NEXT:  ## %bb.3: ## %cond.store1
+; AVX512F-NEXT:  LBB15_3: ## %cond.store1
 ; AVX512F-NEXT:    vpextrb $1, %xmm1, 1(%rdi)
-; AVX512F-NEXT:  LBB15_4: ## %else2
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $2, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $4, %al
 ; AVX512F-NEXT:    je LBB15_6
-; AVX512F-NEXT:  ## %bb.5: ## %cond.store3
+; AVX512F-NEXT:  LBB15_5: ## %cond.store3
 ; AVX512F-NEXT:    vpextrb $2, %xmm1, 2(%rdi)
-; AVX512F-NEXT:  LBB15_6: ## %else4
-; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $8, %al
 ; AVX512F-NEXT:    je LBB15_8
-; AVX512F-NEXT:  ## %bb.7: ## %cond.store5
+; AVX512F-NEXT:  LBB15_7: ## %cond.store5
 ; AVX512F-NEXT:    vpextrb $3, %xmm1, 3(%rdi)
-; AVX512F-NEXT:  LBB15_8: ## %else6
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $4, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $16, %al
 ; AVX512F-NEXT:    je LBB15_10
-; AVX512F-NEXT:  ## %bb.9: ## %cond.store7
+; AVX512F-NEXT:  LBB15_9: ## %cond.store7
 ; AVX512F-NEXT:    vpextrb $4, %xmm1, 4(%rdi)
-; AVX512F-NEXT:  LBB15_10: ## %else8
-; AVX512F-NEXT:    kshiftrw $5, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $32, %al
 ; AVX512F-NEXT:    je LBB15_12
-; AVX512F-NEXT:  ## %bb.11: ## %cond.store9
+; AVX512F-NEXT:  LBB15_11: ## %cond.store9
 ; AVX512F-NEXT:    vpextrb $5, %xmm1, 5(%rdi)
-; AVX512F-NEXT:  LBB15_12: ## %else10
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $6, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $64, %al
 ; AVX512F-NEXT:    je LBB15_14
-; AVX512F-NEXT:  ## %bb.13: ## %cond.store11
+; AVX512F-NEXT:  LBB15_13: ## %cond.store11
 ; AVX512F-NEXT:    vpextrb $6, %xmm1, 6(%rdi)
-; AVX512F-NEXT:  LBB15_14: ## %else12
-; AVX512F-NEXT:    kshiftrw $7, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $-128, %al
 ; AVX512F-NEXT:    je LBB15_16
-; AVX512F-NEXT:  ## %bb.15: ## %cond.store13
+; AVX512F-NEXT:  LBB15_15: ## %cond.store13
 ; AVX512F-NEXT:    vpextrb $7, %xmm1, 7(%rdi)
-; AVX512F-NEXT:  LBB15_16: ## %else14
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $8, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $256, %eax ## imm = 0x100
 ; AVX512F-NEXT:    je LBB15_18
-; AVX512F-NEXT:  ## %bb.17: ## %cond.store15
+; AVX512F-NEXT:  LBB15_17: ## %cond.store15
 ; AVX512F-NEXT:    vpextrb $8, %xmm1, 8(%rdi)
-; AVX512F-NEXT:  LBB15_18: ## %else16
-; AVX512F-NEXT:    kshiftrw $9, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $512, %eax ## imm = 0x200
 ; AVX512F-NEXT:    je LBB15_20
-; AVX512F-NEXT:  ## %bb.19: ## %cond.store17
+; AVX512F-NEXT:  LBB15_19: ## %cond.store17
 ; AVX512F-NEXT:    vpextrb $9, %xmm1, 9(%rdi)
-; AVX512F-NEXT:  LBB15_20: ## %else18
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $10, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $1024, %eax ## imm = 0x400
 ; AVX512F-NEXT:    je LBB15_22
-; AVX512F-NEXT:  ## %bb.21: ## %cond.store19
+; AVX512F-NEXT:  LBB15_21: ## %cond.store19
 ; AVX512F-NEXT:    vpextrb $10, %xmm1, 10(%rdi)
-; AVX512F-NEXT:  LBB15_22: ## %else20
-; AVX512F-NEXT:    kshiftrw $11, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $2048, %eax ## imm = 0x800
 ; AVX512F-NEXT:    je LBB15_24
-; AVX512F-NEXT:  ## %bb.23: ## %cond.store21
+; AVX512F-NEXT:  LBB15_23: ## %cond.store21
 ; AVX512F-NEXT:    vpextrb $11, %xmm1, 11(%rdi)
-; AVX512F-NEXT:  LBB15_24: ## %else22
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $12, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $4096, %eax ## imm = 0x1000
 ; AVX512F-NEXT:    je LBB15_26
-; AVX512F-NEXT:  ## %bb.25: ## %cond.store23
+; AVX512F-NEXT:  LBB15_25: ## %cond.store23
 ; AVX512F-NEXT:    vpextrb $12, %xmm1, 12(%rdi)
-; AVX512F-NEXT:  LBB15_26: ## %else24
-; AVX512F-NEXT:    kshiftrw $13, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $8192, %eax ## imm = 0x2000
 ; AVX512F-NEXT:    je LBB15_28
-; AVX512F-NEXT:  ## %bb.27: ## %cond.store25
+; AVX512F-NEXT:  LBB15_27: ## %cond.store25
 ; AVX512F-NEXT:    vpextrb $13, %xmm1, 13(%rdi)
-; AVX512F-NEXT:  LBB15_28: ## %else26
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm0
-; AVX512F-NEXT:    vpmovsxbd %xmm0, %zmm0
-; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k0
-; AVX512F-NEXT:    kshiftrw $14, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $16384, %eax ## imm = 0x4000
 ; AVX512F-NEXT:    je LBB15_30
-; AVX512F-NEXT:  ## %bb.29: ## %cond.store27
+; AVX512F-NEXT:  LBB15_29: ## %cond.store27
 ; AVX512F-NEXT:    vpextrb $14, %xmm1, 14(%rdi)
-; AVX512F-NEXT:  LBB15_30: ## %else28
-; AVX512F-NEXT:    kshiftrw $15, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $32768, %eax ## imm = 0x8000
 ; AVX512F-NEXT:    je LBB15_32
-; AVX512F-NEXT:  ## %bb.31: ## %cond.store29
+; AVX512F-NEXT:  LBB15_31: ## %cond.store29
 ; AVX512F-NEXT:    vpextrb $15, %xmm1, 15(%rdi)
-; AVX512F-NEXT:  LBB15_32: ## %else30
-; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VLDQ-LABEL: store_v16i8_v16i8:
 ; AVX512VLDQ:       ## %bb.0:
 ; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm2
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm2, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
+; AVX512VLDQ-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm0
+; AVX512VLDQ-NEXT:    vpmovmskb %xmm0, %eax
 ; AVX512VLDQ-NEXT:    testb $1, %al
-; AVX512VLDQ-NEXT:    je LBB15_2
-; AVX512VLDQ-NEXT:  ## %bb.1: ## %cond.store
+; AVX512VLDQ-NEXT:    jne LBB15_1
+; AVX512VLDQ-NEXT:  ## %bb.2: ## %else
+; AVX512VLDQ-NEXT:    testb $2, %al
+; AVX512VLDQ-NEXT:    jne LBB15_3
+; AVX512VLDQ-NEXT:  LBB15_4: ## %else2
+; AVX512VLDQ-NEXT:    testb $4, %al
+; AVX512VLDQ-NEXT:    jne LBB15_5
+; AVX512VLDQ-NEXT:  LBB15_6: ## %else4
+; AVX512VLDQ-NEXT:    testb $8, %al
+; AVX512VLDQ-NEXT:    jne LBB15_7
+; AVX512VLDQ-NEXT:  LBB15_8: ## %else6
+; AVX512VLDQ-NEXT:    testb $16, %al
+; AVX512VLDQ-NEXT:    jne LBB15_9
+; AVX512VLDQ-NEXT:  LBB15_10: ## %else8
+; AVX512VLDQ-NEXT:    testb $32, %al
+; AVX512VLDQ-NEXT:    jne LBB15_11
+; AVX512VLDQ-NEXT:  LBB15_12: ## %else10
+; AVX512VLDQ-NEXT:    testb $64, %al
+; AVX512VLDQ-NEXT:    jne LBB15_13
+; AVX512VLDQ-NEXT:  LBB15_14: ## %else12
+; AVX512VLDQ-NEXT:    testb $-128, %al
+; AVX512VLDQ-NEXT:    jne LBB15_15
+; AVX512VLDQ-NEXT:  LBB15_16: ## %else14
+; AVX512VLDQ-NEXT:    testl $256, %eax ## imm = 0x100
+; AVX512VLDQ-NEXT:    jne LBB15_17
+; AVX512VLDQ-NEXT:  LBB15_18: ## %else16
+; AVX512VLDQ-NEXT:    testl $512, %eax ## imm = 0x200
+; AVX512VLDQ-NEXT:    jne LBB15_19
+; AVX512VLDQ-NEXT:  LBB15_20: ## %else18
+; AVX512VLDQ-NEXT:    testl $1024, %eax ## imm = 0x400
+; AVX512VLDQ-NEXT:    jne LBB15_21
+; AVX512VLDQ-NEXT:  LBB15_22: ## %else20
+; AVX512VLDQ-NEXT:    testl $2048, %eax ## imm = 0x800
+; AVX512VLDQ-NEXT:    jne LBB15_23
+; AVX512VLDQ-NEXT:  LBB15_24: ## %else22
+; AVX512VLDQ-NEXT:    testl $4096, %eax ## imm = 0x1000
+; AVX512VLDQ-NEXT:    jne LBB15_25
+; AVX512VLDQ-NEXT:  LBB15_26: ## %else24
+; AVX512VLDQ-NEXT:    testl $8192, %eax ## imm = 0x2000
+; AVX512VLDQ-NEXT:    jne LBB15_27
+; AVX512VLDQ-NEXT:  LBB15_28: ## %else26
+; AVX512VLDQ-NEXT:    testl $16384, %eax ## imm = 0x4000
+; AVX512VLDQ-NEXT:    jne LBB15_29
+; AVX512VLDQ-NEXT:  LBB15_30: ## %else28
+; AVX512VLDQ-NEXT:    testl $32768, %eax ## imm = 0x8000
+; AVX512VLDQ-NEXT:    jne LBB15_31
+; AVX512VLDQ-NEXT:  LBB15_32: ## %else30
+; AVX512VLDQ-NEXT:    retq
+; AVX512VLDQ-NEXT:  LBB15_1: ## %cond.store
 ; AVX512VLDQ-NEXT:    vpextrb $0, %xmm1, (%rdi)
-; AVX512VLDQ-NEXT:  LBB15_2: ## %else
-; AVX512VLDQ-NEXT:    kshiftrw $1, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $2, %al
 ; AVX512VLDQ-NEXT:    je LBB15_4
-; AVX512VLDQ-NEXT:  ## %bb.3: ## %cond.store1
+; AVX512VLDQ-NEXT:  LBB15_3: ## %cond.store1
 ; AVX512VLDQ-NEXT:    vpextrb $1, %xmm1, 1(%rdi)
-; AVX512VLDQ-NEXT:  LBB15_4: ## %else2
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm2
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm2, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $2, %k0, %k1
-; AVX512VLDQ-NEXT:    kmovw %k1, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $4, %al
 ; AVX512VLDQ-NEXT:    je LBB15_6
-; AVX512VLDQ-NEXT:  ## %bb.5: ## %cond.store3
+; AVX512VLDQ-NEXT:  LBB15_5: ## %cond.store3
 ; AVX512VLDQ-NEXT:    vpextrb $2, %xmm1, 2(%rdi)
-; AVX512VLDQ-NEXT:  LBB15_6: ## %else4
-; AVX512VLDQ-NEXT:    kshiftrw $3, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $8, %al
 ; AVX512VLDQ-NEXT:    je LBB15_8
-; AVX512VLDQ-NEXT:  ## %bb.7: ## %cond.store5
+; AVX512VLDQ-NEXT:  LBB15_7: ## %cond.store5
 ; AVX512VLDQ-NEXT:    vpextrb $3, %xmm1, 3(%rdi)
-; AVX512VLDQ-NEXT:  LBB15_8: ## %else6
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm2
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm2, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $4, %k0, %k1
-; AVX512VLDQ-NEXT:    kmovw %k1, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $16, %al
 ; AVX512VLDQ-NEXT:    je LBB15_10
-; AVX512VLDQ-NEXT:  ## %bb.9: ## %cond.store7
+; AVX512VLDQ-NEXT:  LBB15_9: ## %cond.store7
 ; AVX512VLDQ-NEXT:    vpextrb $4, %xmm1, 4(%rdi)
-; AVX512VLDQ-NEXT:  LBB15_10: ## %else8
-; AVX512VLDQ-NEXT:    kshiftrw $5, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $32, %al
 ; AVX512VLDQ-NEXT:    je LBB15_12
-; AVX512VLDQ-NEXT:  ## %bb.11: ## %cond.store9
+; AVX512VLDQ-NEXT:  LBB15_11: ## %cond.store9
 ; AVX512VLDQ-NEXT:    vpextrb $5, %xmm1, 5(%rdi)
-; AVX512VLDQ-NEXT:  LBB15_12: ## %else10
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm2
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm2, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $6, %k0, %k1
-; AVX512VLDQ-NEXT:    kmovw %k1, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $64, %al
 ; AVX512VLDQ-NEXT:    je LBB15_14
-; AVX512VLDQ-NEXT:  ## %bb.13: ## %cond.store11
+; AVX512VLDQ-NEXT:  LBB15_13: ## %cond.store11
 ; AVX512VLDQ-NEXT:    vpextrb $6, %xmm1, 6(%rdi)
-; AVX512VLDQ-NEXT:  LBB15_14: ## %else12
-; AVX512VLDQ-NEXT:    kshiftrw $7, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $-128, %al
 ; AVX512VLDQ-NEXT:    je LBB15_16
-; AVX512VLDQ-NEXT:  ## %bb.15: ## %cond.store13
+; AVX512VLDQ-NEXT:  LBB15_15: ## %cond.store13
 ; AVX512VLDQ-NEXT:    vpextrb $7, %xmm1, 7(%rdi)
-; AVX512VLDQ-NEXT:  LBB15_16: ## %else14
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm2
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm2, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $8, %k0, %k1
-; AVX512VLDQ-NEXT:    kmovw %k1, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testl $256, %eax ## imm = 0x100
 ; AVX512VLDQ-NEXT:    je LBB15_18
-; AVX512VLDQ-NEXT:  ## %bb.17: ## %cond.store15
+; AVX512VLDQ-NEXT:  LBB15_17: ## %cond.store15
 ; AVX512VLDQ-NEXT:    vpextrb $8, %xmm1, 8(%rdi)
-; AVX512VLDQ-NEXT:  LBB15_18: ## %else16
-; AVX512VLDQ-NEXT:    kshiftrw $9, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testl $512, %eax ## imm = 0x200
 ; AVX512VLDQ-NEXT:    je LBB15_20
-; AVX512VLDQ-NEXT:  ## %bb.19: ## %cond.store17
+; AVX512VLDQ-NEXT:  LBB15_19: ## %cond.store17
 ; AVX512VLDQ-NEXT:    vpextrb $9, %xmm1, 9(%rdi)
-; AVX512VLDQ-NEXT:  LBB15_20: ## %else18
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm2
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm2, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $10, %k0, %k1
-; AVX512VLDQ-NEXT:    kmovw %k1, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testl $1024, %eax ## imm = 0x400
 ; AVX512VLDQ-NEXT:    je LBB15_22
-; AVX512VLDQ-NEXT:  ## %bb.21: ## %cond.store19
+; AVX512VLDQ-NEXT:  LBB15_21: ## %cond.store19
 ; AVX512VLDQ-NEXT:    vpextrb $10, %xmm1, 10(%rdi)
-; AVX512VLDQ-NEXT:  LBB15_22: ## %else20
-; AVX512VLDQ-NEXT:    kshiftrw $11, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testl $2048, %eax ## imm = 0x800
 ; AVX512VLDQ-NEXT:    je LBB15_24
-; AVX512VLDQ-NEXT:  ## %bb.23: ## %cond.store21
+; AVX512VLDQ-NEXT:  LBB15_23: ## %cond.store21
 ; AVX512VLDQ-NEXT:    vpextrb $11, %xmm1, 11(%rdi)
-; AVX512VLDQ-NEXT:  LBB15_24: ## %else22
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm2
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm2, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $12, %k0, %k1
-; AVX512VLDQ-NEXT:    kmovw %k1, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testl $4096, %eax ## imm = 0x1000
 ; AVX512VLDQ-NEXT:    je LBB15_26
-; AVX512VLDQ-NEXT:  ## %bb.25: ## %cond.store23
+; AVX512VLDQ-NEXT:  LBB15_25: ## %cond.store23
 ; AVX512VLDQ-NEXT:    vpextrb $12, %xmm1, 12(%rdi)
-; AVX512VLDQ-NEXT:  LBB15_26: ## %else24
-; AVX512VLDQ-NEXT:    kshiftrw $13, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testl $8192, %eax ## imm = 0x2000
 ; AVX512VLDQ-NEXT:    je LBB15_28
-; AVX512VLDQ-NEXT:  ## %bb.27: ## %cond.store25
+; AVX512VLDQ-NEXT:  LBB15_27: ## %cond.store25
 ; AVX512VLDQ-NEXT:    vpextrb $13, %xmm1, 13(%rdi)
-; AVX512VLDQ-NEXT:  LBB15_28: ## %else26
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm0
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm0, %zmm0
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm0, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $14, %k0, %k1
-; AVX512VLDQ-NEXT:    kmovw %k1, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testl $16384, %eax ## imm = 0x4000
 ; AVX512VLDQ-NEXT:    je LBB15_30
-; AVX512VLDQ-NEXT:  ## %bb.29: ## %cond.store27
+; AVX512VLDQ-NEXT:  LBB15_29: ## %cond.store27
 ; AVX512VLDQ-NEXT:    vpextrb $14, %xmm1, 14(%rdi)
-; AVX512VLDQ-NEXT:  LBB15_30: ## %else28
-; AVX512VLDQ-NEXT:    kshiftrw $15, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testl $32768, %eax ## imm = 0x8000
 ; AVX512VLDQ-NEXT:    je LBB15_32
-; AVX512VLDQ-NEXT:  ## %bb.31: ## %cond.store29
+; AVX512VLDQ-NEXT:  LBB15_31: ## %cond.store29
 ; AVX512VLDQ-NEXT:    vpextrb $15, %xmm1, 15(%rdi)
-; AVX512VLDQ-NEXT:  LBB15_32: ## %else30
-; AVX512VLDQ-NEXT:    vzeroupper
 ; AVX512VLDQ-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: store_v16i8_v16i8:
@@ -3319,1592 +3113,1375 @@ define void @store_v32i8_v32i8(<32 x i8> %trigger, <32 x i8>* %addr, <32 x i8> %
 ; SSE2-LABEL: store_v32i8_v32i8:
 ; SSE2:       ## %bb.0:
 ; SSE2-NEXT:    pxor %xmm4, %xmm4
-; SSE2-NEXT:    pcmpeqb %xmm0, %xmm4
-; SSE2-NEXT:    movd %xmm4, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    movd %xmm2, %eax
-; SSE2-NEXT:    je LBB16_2
-; SSE2-NEXT:  ## %bb.1: ## %cond.store
-; SSE2-NEXT:    movb %al, (%rdi)
-; SSE2-NEXT:  LBB16_2: ## %else
-; SSE2-NEXT:    shrl $8, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    je LBB16_4
-; SSE2-NEXT:  ## %bb.3: ## %cond.store1
-; SSE2-NEXT:    movb %ah, 1(%rdi)
+; SSE2-NEXT:    pcmpeqb %xmm4, %xmm0
+; SSE2-NEXT:    pmovmskb %xmm0, %ecx
+; SSE2-NEXT:    pcmpeqb %xmm4, %xmm1
+; SSE2-NEXT:    pmovmskb %xmm1, %eax
+; SSE2-NEXT:    shll $16, %eax
+; SSE2-NEXT:    orl %ecx, %eax
+; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    movd %xmm2, %ecx
+; SSE2-NEXT:    jne LBB16_1
+; SSE2-NEXT:  ## %bb.2: ## %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne LBB16_3
 ; SSE2-NEXT:  LBB16_4: ## %else2
-; SSE2-NEXT:    pxor %xmm4, %xmm4
-; SSE2-NEXT:    pcmpeqb %xmm0, %xmm4
-; SSE2-NEXT:    movd %xmm4, %ecx
-; SSE2-NEXT:    movl %ecx, %edx
-; SSE2-NEXT:    shrl $16, %edx
-; SSE2-NEXT:    testb $1, %dl
-; SSE2-NEXT:    je LBB16_6
-; SSE2-NEXT:  ## %bb.5: ## %cond.store3
-; SSE2-NEXT:    movl %eax, %edx
-; SSE2-NEXT:    shrl $16, %edx
-; SSE2-NEXT:    movb %dl, 2(%rdi)
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne LBB16_5
 ; SSE2-NEXT:  LBB16_6: ## %else4
-; SSE2-NEXT:    shrl $24, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je LBB16_8
-; SSE2-NEXT:  ## %bb.7: ## %cond.store5
-; SSE2-NEXT:    shrl $24, %eax
-; SSE2-NEXT:    movb %al, 3(%rdi)
+; SSE2-NEXT:  LBB16_7: ## %cond.store5
+; SSE2-NEXT:    shrl $24, %ecx
+; SSE2-NEXT:    movb %cl, 3(%rdi)
 ; SSE2-NEXT:  LBB16_8: ## %else6
-; SSE2-NEXT:    pxor %xmm4, %xmm4
-; SSE2-NEXT:    pcmpeqb %xmm0, %xmm4
-; SSE2-NEXT:    pextrw $2, %xmm4, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    pextrw $2, %xmm2, %eax
+; SSE2-NEXT:    testb $16, %al
+; SSE2-NEXT:    pextrw $2, %xmm2, %ecx
 ; SSE2-NEXT:    je LBB16_10
 ; SSE2-NEXT:  ## %bb.9: ## %cond.store7
-; SSE2-NEXT:    movb %al, 4(%rdi)
+; SSE2-NEXT:    movb %cl, 4(%rdi)
 ; SSE2-NEXT:  LBB16_10: ## %else8
-; SSE2-NEXT:    shrl $8, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testb $32, %al
 ; SSE2-NEXT:    je LBB16_12
 ; SSE2-NEXT:  ## %bb.11: ## %cond.store9
-; SSE2-NEXT:    movb %ah, 5(%rdi)
+; SSE2-NEXT:    movb %ch, 5(%rdi)
 ; SSE2-NEXT:  LBB16_12: ## %else10
-; SSE2-NEXT:    pxor %xmm4, %xmm4
-; SSE2-NEXT:    pcmpeqb %xmm0, %xmm4
-; SSE2-NEXT:    pextrw $3, %xmm4, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    pextrw $3, %xmm2, %eax
+; SSE2-NEXT:    testb $64, %al
+; SSE2-NEXT:    pextrw $3, %xmm2, %ecx
 ; SSE2-NEXT:    je LBB16_14
 ; SSE2-NEXT:  ## %bb.13: ## %cond.store11
-; SSE2-NEXT:    movb %al, 6(%rdi)
+; SSE2-NEXT:    movb %cl, 6(%rdi)
 ; SSE2-NEXT:  LBB16_14: ## %else12
-; SSE2-NEXT:    shrl $8, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testb $-128, %al
 ; SSE2-NEXT:    je LBB16_16
 ; SSE2-NEXT:  ## %bb.15: ## %cond.store13
-; SSE2-NEXT:    movb %ah, 7(%rdi)
+; SSE2-NEXT:    movb %ch, 7(%rdi)
 ; SSE2-NEXT:  LBB16_16: ## %else14
-; SSE2-NEXT:    pxor %xmm4, %xmm4
-; SSE2-NEXT:    pcmpeqb %xmm0, %xmm4
-; SSE2-NEXT:    pextrw $4, %xmm4, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    pextrw $4, %xmm2, %eax
+; SSE2-NEXT:    testl $256, %eax ## imm = 0x100
+; SSE2-NEXT:    pextrw $4, %xmm2, %ecx
 ; SSE2-NEXT:    je LBB16_18
 ; SSE2-NEXT:  ## %bb.17: ## %cond.store15
-; SSE2-NEXT:    movb %al, 8(%rdi)
+; SSE2-NEXT:    movb %cl, 8(%rdi)
 ; SSE2-NEXT:  LBB16_18: ## %else16
-; SSE2-NEXT:    shrl $8, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $512, %eax ## imm = 0x200
 ; SSE2-NEXT:    je LBB16_20
 ; SSE2-NEXT:  ## %bb.19: ## %cond.store17
-; SSE2-NEXT:    movb %ah, 9(%rdi)
+; SSE2-NEXT:    movb %ch, 9(%rdi)
 ; SSE2-NEXT:  LBB16_20: ## %else18
-; SSE2-NEXT:    pxor %xmm4, %xmm4
-; SSE2-NEXT:    pcmpeqb %xmm0, %xmm4
-; SSE2-NEXT:    pextrw $5, %xmm4, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    pextrw $5, %xmm2, %eax
+; SSE2-NEXT:    testl $1024, %eax ## imm = 0x400
+; SSE2-NEXT:    pextrw $5, %xmm2, %ecx
 ; SSE2-NEXT:    je LBB16_22
 ; SSE2-NEXT:  ## %bb.21: ## %cond.store19
-; SSE2-NEXT:    movb %al, 10(%rdi)
+; SSE2-NEXT:    movb %cl, 10(%rdi)
 ; SSE2-NEXT:  LBB16_22: ## %else20
-; SSE2-NEXT:    shrl $8, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $2048, %eax ## imm = 0x800
 ; SSE2-NEXT:    je LBB16_24
 ; SSE2-NEXT:  ## %bb.23: ## %cond.store21
-; SSE2-NEXT:    movb %ah, 11(%rdi)
+; SSE2-NEXT:    movb %ch, 11(%rdi)
 ; SSE2-NEXT:  LBB16_24: ## %else22
-; SSE2-NEXT:    pxor %xmm4, %xmm4
-; SSE2-NEXT:    pcmpeqb %xmm0, %xmm4
-; SSE2-NEXT:    pextrw $6, %xmm4, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    pextrw $6, %xmm2, %eax
+; SSE2-NEXT:    testl $4096, %eax ## imm = 0x1000
+; SSE2-NEXT:    pextrw $6, %xmm2, %ecx
 ; SSE2-NEXT:    je LBB16_26
 ; SSE2-NEXT:  ## %bb.25: ## %cond.store23
-; SSE2-NEXT:    movb %al, 12(%rdi)
+; SSE2-NEXT:    movb %cl, 12(%rdi)
 ; SSE2-NEXT:  LBB16_26: ## %else24
-; SSE2-NEXT:    shrl $8, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $8192, %eax ## imm = 0x2000
 ; SSE2-NEXT:    je LBB16_28
 ; SSE2-NEXT:  ## %bb.27: ## %cond.store25
-; SSE2-NEXT:    movb %ah, 13(%rdi)
+; SSE2-NEXT:    movb %ch, 13(%rdi)
 ; SSE2-NEXT:  LBB16_28: ## %else26
-; SSE2-NEXT:    pxor %xmm4, %xmm4
-; SSE2-NEXT:    pcmpeqb %xmm4, %xmm0
-; SSE2-NEXT:    pextrw $7, %xmm0, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    pextrw $7, %xmm2, %eax
+; SSE2-NEXT:    testl $16384, %eax ## imm = 0x4000
+; SSE2-NEXT:    pextrw $7, %xmm2, %ecx
 ; SSE2-NEXT:    je LBB16_30
 ; SSE2-NEXT:  ## %bb.29: ## %cond.store27
-; SSE2-NEXT:    movb %al, 14(%rdi)
+; SSE2-NEXT:    movb %cl, 14(%rdi)
 ; SSE2-NEXT:  LBB16_30: ## %else28
-; SSE2-NEXT:    shrl $8, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $32768, %eax ## imm = 0x8000
 ; SSE2-NEXT:    je LBB16_32
 ; SSE2-NEXT:  ## %bb.31: ## %cond.store29
-; SSE2-NEXT:    movb %ah, 15(%rdi)
+; SSE2-NEXT:    movb %ch, 15(%rdi)
 ; SSE2-NEXT:  LBB16_32: ## %else30
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqb %xmm1, %xmm0
-; SSE2-NEXT:    movd %xmm0, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    movd %xmm3, %eax
-; SSE2-NEXT:    je LBB16_34
-; SSE2-NEXT:  ## %bb.33: ## %cond.store31
-; SSE2-NEXT:    movb %al, 16(%rdi)
-; SSE2-NEXT:  LBB16_34: ## %else32
-; SSE2-NEXT:    shrl $8, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    je LBB16_36
-; SSE2-NEXT:  ## %bb.35: ## %cond.store33
-; SSE2-NEXT:    movb %ah, 17(%rdi)
+; SSE2-NEXT:    testl $65536, %eax ## imm = 0x10000
+; SSE2-NEXT:    movd %xmm3, %ecx
+; SSE2-NEXT:    jne LBB16_33
+; SSE2-NEXT:  ## %bb.34: ## %else32
+; SSE2-NEXT:    testl $131072, %eax ## imm = 0x20000
+; SSE2-NEXT:    jne LBB16_35
 ; SSE2-NEXT:  LBB16_36: ## %else34
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqb %xmm1, %xmm0
-; SSE2-NEXT:    movd %xmm0, %ecx
-; SSE2-NEXT:    movl %ecx, %edx
-; SSE2-NEXT:    shrl $16, %edx
-; SSE2-NEXT:    testb $1, %dl
-; SSE2-NEXT:    je LBB16_38
-; SSE2-NEXT:  ## %bb.37: ## %cond.store35
-; SSE2-NEXT:    movl %eax, %edx
-; SSE2-NEXT:    shrl $16, %edx
-; SSE2-NEXT:    movb %dl, 18(%rdi)
+; SSE2-NEXT:    testl $262144, %eax ## imm = 0x40000
+; SSE2-NEXT:    jne LBB16_37
 ; SSE2-NEXT:  LBB16_38: ## %else36
-; SSE2-NEXT:    shrl $24, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $524288, %eax ## imm = 0x80000
 ; SSE2-NEXT:    je LBB16_40
-; SSE2-NEXT:  ## %bb.39: ## %cond.store37
-; SSE2-NEXT:    shrl $24, %eax
-; SSE2-NEXT:    movb %al, 19(%rdi)
+; SSE2-NEXT:  LBB16_39: ## %cond.store37
+; SSE2-NEXT:    shrl $24, %ecx
+; SSE2-NEXT:    movb %cl, 19(%rdi)
 ; SSE2-NEXT:  LBB16_40: ## %else38
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqb %xmm1, %xmm0
-; SSE2-NEXT:    pextrw $2, %xmm0, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    pextrw $2, %xmm3, %eax
+; SSE2-NEXT:    testl $1048576, %eax ## imm = 0x100000
+; SSE2-NEXT:    pextrw $2, %xmm3, %ecx
 ; SSE2-NEXT:    je LBB16_42
 ; SSE2-NEXT:  ## %bb.41: ## %cond.store39
-; SSE2-NEXT:    movb %al, 20(%rdi)
+; SSE2-NEXT:    movb %cl, 20(%rdi)
 ; SSE2-NEXT:  LBB16_42: ## %else40
-; SSE2-NEXT:    shrl $8, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $2097152, %eax ## imm = 0x200000
 ; SSE2-NEXT:    je LBB16_44
 ; SSE2-NEXT:  ## %bb.43: ## %cond.store41
-; SSE2-NEXT:    movb %ah, 21(%rdi)
+; SSE2-NEXT:    movb %ch, 21(%rdi)
 ; SSE2-NEXT:  LBB16_44: ## %else42
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqb %xmm1, %xmm0
-; SSE2-NEXT:    pextrw $3, %xmm0, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    pextrw $3, %xmm3, %eax
+; SSE2-NEXT:    testl $4194304, %eax ## imm = 0x400000
+; SSE2-NEXT:    pextrw $3, %xmm3, %ecx
 ; SSE2-NEXT:    je LBB16_46
 ; SSE2-NEXT:  ## %bb.45: ## %cond.store43
-; SSE2-NEXT:    movb %al, 22(%rdi)
+; SSE2-NEXT:    movb %cl, 22(%rdi)
 ; SSE2-NEXT:  LBB16_46: ## %else44
-; SSE2-NEXT:    shrl $8, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $8388608, %eax ## imm = 0x800000
 ; SSE2-NEXT:    je LBB16_48
 ; SSE2-NEXT:  ## %bb.47: ## %cond.store45
-; SSE2-NEXT:    movb %ah, 23(%rdi)
+; SSE2-NEXT:    movb %ch, 23(%rdi)
 ; SSE2-NEXT:  LBB16_48: ## %else46
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqb %xmm1, %xmm0
-; SSE2-NEXT:    pextrw $4, %xmm0, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    pextrw $4, %xmm3, %eax
+; SSE2-NEXT:    testl $16777216, %eax ## imm = 0x1000000
+; SSE2-NEXT:    pextrw $4, %xmm3, %ecx
 ; SSE2-NEXT:    je LBB16_50
 ; SSE2-NEXT:  ## %bb.49: ## %cond.store47
-; SSE2-NEXT:    movb %al, 24(%rdi)
+; SSE2-NEXT:    movb %cl, 24(%rdi)
 ; SSE2-NEXT:  LBB16_50: ## %else48
-; SSE2-NEXT:    shrl $8, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $33554432, %eax ## imm = 0x2000000
 ; SSE2-NEXT:    je LBB16_52
 ; SSE2-NEXT:  ## %bb.51: ## %cond.store49
-; SSE2-NEXT:    movb %ah, 25(%rdi)
+; SSE2-NEXT:    movb %ch, 25(%rdi)
 ; SSE2-NEXT:  LBB16_52: ## %else50
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqb %xmm1, %xmm0
-; SSE2-NEXT:    pextrw $5, %xmm0, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    pextrw $5, %xmm3, %eax
+; SSE2-NEXT:    testl $67108864, %eax ## imm = 0x4000000
+; SSE2-NEXT:    pextrw $5, %xmm3, %ecx
 ; SSE2-NEXT:    je LBB16_54
 ; SSE2-NEXT:  ## %bb.53: ## %cond.store51
-; SSE2-NEXT:    movb %al, 26(%rdi)
+; SSE2-NEXT:    movb %cl, 26(%rdi)
 ; SSE2-NEXT:  LBB16_54: ## %else52
-; SSE2-NEXT:    shrl $8, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $134217728, %eax ## imm = 0x8000000
 ; SSE2-NEXT:    je LBB16_56
 ; SSE2-NEXT:  ## %bb.55: ## %cond.store53
-; SSE2-NEXT:    movb %ah, 27(%rdi)
+; SSE2-NEXT:    movb %ch, 27(%rdi)
 ; SSE2-NEXT:  LBB16_56: ## %else54
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqb %xmm1, %xmm0
-; SSE2-NEXT:    pextrw $6, %xmm0, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    pextrw $6, %xmm3, %eax
+; SSE2-NEXT:    testl $268435456, %eax ## imm = 0x10000000
+; SSE2-NEXT:    pextrw $6, %xmm3, %ecx
 ; SSE2-NEXT:    je LBB16_58
 ; SSE2-NEXT:  ## %bb.57: ## %cond.store55
-; SSE2-NEXT:    movb %al, 28(%rdi)
+; SSE2-NEXT:    movb %cl, 28(%rdi)
 ; SSE2-NEXT:  LBB16_58: ## %else56
-; SSE2-NEXT:    shrl $8, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $536870912, %eax ## imm = 0x20000000
 ; SSE2-NEXT:    je LBB16_60
 ; SSE2-NEXT:  ## %bb.59: ## %cond.store57
-; SSE2-NEXT:    movb %ah, 29(%rdi)
+; SSE2-NEXT:    movb %ch, 29(%rdi)
 ; SSE2-NEXT:  LBB16_60: ## %else58
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqb %xmm0, %xmm1
-; SSE2-NEXT:    pextrw $7, %xmm1, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    pextrw $7, %xmm3, %eax
-; SSE2-NEXT:    je LBB16_62
-; SSE2-NEXT:  ## %bb.61: ## %cond.store59
-; SSE2-NEXT:    movb %al, 30(%rdi)
-; SSE2-NEXT:  LBB16_62: ## %else60
-; SSE2-NEXT:    shrl $8, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    je LBB16_64
-; SSE2-NEXT:  ## %bb.63: ## %cond.store61
-; SSE2-NEXT:    movb %ah, 31(%rdi)
+; SSE2-NEXT:    testl $1073741824, %eax ## imm = 0x40000000
+; SSE2-NEXT:    pextrw $7, %xmm3, %ecx
+; SSE2-NEXT:    jne LBB16_61
+; SSE2-NEXT:  ## %bb.62: ## %else60
+; SSE2-NEXT:    testl $-2147483648, %eax ## imm = 0x80000000
+; SSE2-NEXT:    jne LBB16_63
 ; SSE2-NEXT:  LBB16_64: ## %else62
 ; SSE2-NEXT:    retq
+; SSE2-NEXT:  LBB16_1: ## %cond.store
+; SSE2-NEXT:    movb %cl, (%rdi)
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    je LBB16_4
+; SSE2-NEXT:  LBB16_3: ## %cond.store1
+; SSE2-NEXT:    movb %ch, 1(%rdi)
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    je LBB16_6
+; SSE2-NEXT:  LBB16_5: ## %cond.store3
+; SSE2-NEXT:    movl %ecx, %edx
+; SSE2-NEXT:    shrl $16, %edx
+; SSE2-NEXT:    movb %dl, 2(%rdi)
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne LBB16_7
+; SSE2-NEXT:    jmp LBB16_8
+; SSE2-NEXT:  LBB16_33: ## %cond.store31
+; SSE2-NEXT:    movb %cl, 16(%rdi)
+; SSE2-NEXT:    testl $131072, %eax ## imm = 0x20000
+; SSE2-NEXT:    je LBB16_36
+; SSE2-NEXT:  LBB16_35: ## %cond.store33
+; SSE2-NEXT:    movb %ch, 17(%rdi)
+; SSE2-NEXT:    testl $262144, %eax ## imm = 0x40000
+; SSE2-NEXT:    je LBB16_38
+; SSE2-NEXT:  LBB16_37: ## %cond.store35
+; SSE2-NEXT:    movl %ecx, %edx
+; SSE2-NEXT:    shrl $16, %edx
+; SSE2-NEXT:    movb %dl, 18(%rdi)
+; SSE2-NEXT:    testl $524288, %eax ## imm = 0x80000
+; SSE2-NEXT:    jne LBB16_39
+; SSE2-NEXT:    jmp LBB16_40
+; SSE2-NEXT:  LBB16_61: ## %cond.store59
+; SSE2-NEXT:    movb %cl, 30(%rdi)
+; SSE2-NEXT:    testl $-2147483648, %eax ## imm = 0x80000000
+; SSE2-NEXT:    je LBB16_64
+; SSE2-NEXT:  LBB16_63: ## %cond.store61
+; SSE2-NEXT:    movb %ch, 31(%rdi)
+; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: store_v32i8_v32i8:
 ; SSE4:       ## %bb.0:
 ; SSE4-NEXT:    pxor %xmm4, %xmm4
-; SSE4-NEXT:    pcmpeqb %xmm0, %xmm4
-; SSE4-NEXT:    pextrb $0, %xmm4, %eax
+; SSE4-NEXT:    pcmpeqb %xmm4, %xmm0
+; SSE4-NEXT:    pmovmskb %xmm0, %ecx
+; SSE4-NEXT:    pcmpeqb %xmm4, %xmm1
+; SSE4-NEXT:    pmovmskb %xmm1, %eax
+; SSE4-NEXT:    shll $16, %eax
+; SSE4-NEXT:    orl %ecx, %eax
 ; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je LBB16_2
-; SSE4-NEXT:  ## %bb.1: ## %cond.store
+; SSE4-NEXT:    jne LBB16_1
+; SSE4-NEXT:  ## %bb.2: ## %else
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    jne LBB16_3
+; SSE4-NEXT:  LBB16_4: ## %else2
+; SSE4-NEXT:    testb $4, %al
+; SSE4-NEXT:    jne LBB16_5
+; SSE4-NEXT:  LBB16_6: ## %else4
+; SSE4-NEXT:    testb $8, %al
+; SSE4-NEXT:    jne LBB16_7
+; SSE4-NEXT:  LBB16_8: ## %else6
+; SSE4-NEXT:    testb $16, %al
+; SSE4-NEXT:    jne LBB16_9
+; SSE4-NEXT:  LBB16_10: ## %else8
+; SSE4-NEXT:    testb $32, %al
+; SSE4-NEXT:    jne LBB16_11
+; SSE4-NEXT:  LBB16_12: ## %else10
+; SSE4-NEXT:    testb $64, %al
+; SSE4-NEXT:    jne LBB16_13
+; SSE4-NEXT:  LBB16_14: ## %else12
+; SSE4-NEXT:    testb $-128, %al
+; SSE4-NEXT:    jne LBB16_15
+; SSE4-NEXT:  LBB16_16: ## %else14
+; SSE4-NEXT:    testl $256, %eax ## imm = 0x100
+; SSE4-NEXT:    jne LBB16_17
+; SSE4-NEXT:  LBB16_18: ## %else16
+; SSE4-NEXT:    testl $512, %eax ## imm = 0x200
+; SSE4-NEXT:    jne LBB16_19
+; SSE4-NEXT:  LBB16_20: ## %else18
+; SSE4-NEXT:    testl $1024, %eax ## imm = 0x400
+; SSE4-NEXT:    jne LBB16_21
+; SSE4-NEXT:  LBB16_22: ## %else20
+; SSE4-NEXT:    testl $2048, %eax ## imm = 0x800
+; SSE4-NEXT:    jne LBB16_23
+; SSE4-NEXT:  LBB16_24: ## %else22
+; SSE4-NEXT:    testl $4096, %eax ## imm = 0x1000
+; SSE4-NEXT:    jne LBB16_25
+; SSE4-NEXT:  LBB16_26: ## %else24
+; SSE4-NEXT:    testl $8192, %eax ## imm = 0x2000
+; SSE4-NEXT:    jne LBB16_27
+; SSE4-NEXT:  LBB16_28: ## %else26
+; SSE4-NEXT:    testl $16384, %eax ## imm = 0x4000
+; SSE4-NEXT:    jne LBB16_29
+; SSE4-NEXT:  LBB16_30: ## %else28
+; SSE4-NEXT:    testl $32768, %eax ## imm = 0x8000
+; SSE4-NEXT:    jne LBB16_31
+; SSE4-NEXT:  LBB16_32: ## %else30
+; SSE4-NEXT:    testl $65536, %eax ## imm = 0x10000
+; SSE4-NEXT:    jne LBB16_33
+; SSE4-NEXT:  LBB16_34: ## %else32
+; SSE4-NEXT:    testl $131072, %eax ## imm = 0x20000
+; SSE4-NEXT:    jne LBB16_35
+; SSE4-NEXT:  LBB16_36: ## %else34
+; SSE4-NEXT:    testl $262144, %eax ## imm = 0x40000
+; SSE4-NEXT:    jne LBB16_37
+; SSE4-NEXT:  LBB16_38: ## %else36
+; SSE4-NEXT:    testl $524288, %eax ## imm = 0x80000
+; SSE4-NEXT:    jne LBB16_39
+; SSE4-NEXT:  LBB16_40: ## %else38
+; SSE4-NEXT:    testl $1048576, %eax ## imm = 0x100000
+; SSE4-NEXT:    jne LBB16_41
+; SSE4-NEXT:  LBB16_42: ## %else40
+; SSE4-NEXT:    testl $2097152, %eax ## imm = 0x200000
+; SSE4-NEXT:    jne LBB16_43
+; SSE4-NEXT:  LBB16_44: ## %else42
+; SSE4-NEXT:    testl $4194304, %eax ## imm = 0x400000
+; SSE4-NEXT:    jne LBB16_45
+; SSE4-NEXT:  LBB16_46: ## %else44
+; SSE4-NEXT:    testl $8388608, %eax ## imm = 0x800000
+; SSE4-NEXT:    jne LBB16_47
+; SSE4-NEXT:  LBB16_48: ## %else46
+; SSE4-NEXT:    testl $16777216, %eax ## imm = 0x1000000
+; SSE4-NEXT:    jne LBB16_49
+; SSE4-NEXT:  LBB16_50: ## %else48
+; SSE4-NEXT:    testl $33554432, %eax ## imm = 0x2000000
+; SSE4-NEXT:    jne LBB16_51
+; SSE4-NEXT:  LBB16_52: ## %else50
+; SSE4-NEXT:    testl $67108864, %eax ## imm = 0x4000000
+; SSE4-NEXT:    jne LBB16_53
+; SSE4-NEXT:  LBB16_54: ## %else52
+; SSE4-NEXT:    testl $134217728, %eax ## imm = 0x8000000
+; SSE4-NEXT:    jne LBB16_55
+; SSE4-NEXT:  LBB16_56: ## %else54
+; SSE4-NEXT:    testl $268435456, %eax ## imm = 0x10000000
+; SSE4-NEXT:    jne LBB16_57
+; SSE4-NEXT:  LBB16_58: ## %else56
+; SSE4-NEXT:    testl $536870912, %eax ## imm = 0x20000000
+; SSE4-NEXT:    jne LBB16_59
+; SSE4-NEXT:  LBB16_60: ## %else58
+; SSE4-NEXT:    testl $1073741824, %eax ## imm = 0x40000000
+; SSE4-NEXT:    jne LBB16_61
+; SSE4-NEXT:  LBB16_62: ## %else60
+; SSE4-NEXT:    testl $-2147483648, %eax ## imm = 0x80000000
+; SSE4-NEXT:    jne LBB16_63
+; SSE4-NEXT:  LBB16_64: ## %else62
+; SSE4-NEXT:    retq
+; SSE4-NEXT:  LBB16_1: ## %cond.store
 ; SSE4-NEXT:    pextrb $0, %xmm2, (%rdi)
-; SSE4-NEXT:  LBB16_2: ## %else
-; SSE4-NEXT:    pextrb $1, %xmm4, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $2, %al
 ; SSE4-NEXT:    je LBB16_4
-; SSE4-NEXT:  ## %bb.3: ## %cond.store1
+; SSE4-NEXT:  LBB16_3: ## %cond.store1
 ; SSE4-NEXT:    pextrb $1, %xmm2, 1(%rdi)
-; SSE4-NEXT:  LBB16_4: ## %else2
-; SSE4-NEXT:    pxor %xmm4, %xmm4
-; SSE4-NEXT:    pcmpeqb %xmm0, %xmm4
-; SSE4-NEXT:    pextrb $2, %xmm4, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $4, %al
 ; SSE4-NEXT:    je LBB16_6
-; SSE4-NEXT:  ## %bb.5: ## %cond.store3
+; SSE4-NEXT:  LBB16_5: ## %cond.store3
 ; SSE4-NEXT:    pextrb $2, %xmm2, 2(%rdi)
-; SSE4-NEXT:  LBB16_6: ## %else4
-; SSE4-NEXT:    pextrb $3, %xmm4, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $8, %al
 ; SSE4-NEXT:    je LBB16_8
-; SSE4-NEXT:  ## %bb.7: ## %cond.store5
+; SSE4-NEXT:  LBB16_7: ## %cond.store5
 ; SSE4-NEXT:    pextrb $3, %xmm2, 3(%rdi)
-; SSE4-NEXT:  LBB16_8: ## %else6
-; SSE4-NEXT:    pxor %xmm4, %xmm4
-; SSE4-NEXT:    pcmpeqb %xmm0, %xmm4
-; SSE4-NEXT:    pextrb $4, %xmm4, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $16, %al
 ; SSE4-NEXT:    je LBB16_10
-; SSE4-NEXT:  ## %bb.9: ## %cond.store7
+; SSE4-NEXT:  LBB16_9: ## %cond.store7
 ; SSE4-NEXT:    pextrb $4, %xmm2, 4(%rdi)
-; SSE4-NEXT:  LBB16_10: ## %else8
-; SSE4-NEXT:    pextrb $5, %xmm4, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $32, %al
 ; SSE4-NEXT:    je LBB16_12
-; SSE4-NEXT:  ## %bb.11: ## %cond.store9
+; SSE4-NEXT:  LBB16_11: ## %cond.store9
 ; SSE4-NEXT:    pextrb $5, %xmm2, 5(%rdi)
-; SSE4-NEXT:  LBB16_12: ## %else10
-; SSE4-NEXT:    pxor %xmm4, %xmm4
-; SSE4-NEXT:    pcmpeqb %xmm0, %xmm4
-; SSE4-NEXT:    pextrb $6, %xmm4, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $64, %al
 ; SSE4-NEXT:    je LBB16_14
-; SSE4-NEXT:  ## %bb.13: ## %cond.store11
+; SSE4-NEXT:  LBB16_13: ## %cond.store11
 ; SSE4-NEXT:    pextrb $6, %xmm2, 6(%rdi)
-; SSE4-NEXT:  LBB16_14: ## %else12
-; SSE4-NEXT:    pextrb $7, %xmm4, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $-128, %al
 ; SSE4-NEXT:    je LBB16_16
-; SSE4-NEXT:  ## %bb.15: ## %cond.store13
+; SSE4-NEXT:  LBB16_15: ## %cond.store13
 ; SSE4-NEXT:    pextrb $7, %xmm2, 7(%rdi)
-; SSE4-NEXT:  LBB16_16: ## %else14
-; SSE4-NEXT:    pxor %xmm4, %xmm4
-; SSE4-NEXT:    pcmpeqb %xmm0, %xmm4
-; SSE4-NEXT:    pextrb $8, %xmm4, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $256, %eax ## imm = 0x100
 ; SSE4-NEXT:    je LBB16_18
-; SSE4-NEXT:  ## %bb.17: ## %cond.store15
+; SSE4-NEXT:  LBB16_17: ## %cond.store15
 ; SSE4-NEXT:    pextrb $8, %xmm2, 8(%rdi)
-; SSE4-NEXT:  LBB16_18: ## %else16
-; SSE4-NEXT:    pextrb $9, %xmm4, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $512, %eax ## imm = 0x200
 ; SSE4-NEXT:    je LBB16_20
-; SSE4-NEXT:  ## %bb.19: ## %cond.store17
+; SSE4-NEXT:  LBB16_19: ## %cond.store17
 ; SSE4-NEXT:    pextrb $9, %xmm2, 9(%rdi)
-; SSE4-NEXT:  LBB16_20: ## %else18
-; SSE4-NEXT:    pxor %xmm4, %xmm4
-; SSE4-NEXT:    pcmpeqb %xmm0, %xmm4
-; SSE4-NEXT:    pextrb $10, %xmm4, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $1024, %eax ## imm = 0x400
 ; SSE4-NEXT:    je LBB16_22
-; SSE4-NEXT:  ## %bb.21: ## %cond.store19
+; SSE4-NEXT:  LBB16_21: ## %cond.store19
 ; SSE4-NEXT:    pextrb $10, %xmm2, 10(%rdi)
-; SSE4-NEXT:  LBB16_22: ## %else20
-; SSE4-NEXT:    pextrb $11, %xmm4, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $2048, %eax ## imm = 0x800
 ; SSE4-NEXT:    je LBB16_24
-; SSE4-NEXT:  ## %bb.23: ## %cond.store21
+; SSE4-NEXT:  LBB16_23: ## %cond.store21
 ; SSE4-NEXT:    pextrb $11, %xmm2, 11(%rdi)
-; SSE4-NEXT:  LBB16_24: ## %else22
-; SSE4-NEXT:    pxor %xmm4, %xmm4
-; SSE4-NEXT:    pcmpeqb %xmm0, %xmm4
-; SSE4-NEXT:    pextrb $12, %xmm4, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $4096, %eax ## imm = 0x1000
 ; SSE4-NEXT:    je LBB16_26
-; SSE4-NEXT:  ## %bb.25: ## %cond.store23
+; SSE4-NEXT:  LBB16_25: ## %cond.store23
 ; SSE4-NEXT:    pextrb $12, %xmm2, 12(%rdi)
-; SSE4-NEXT:  LBB16_26: ## %else24
-; SSE4-NEXT:    pextrb $13, %xmm4, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $8192, %eax ## imm = 0x2000
 ; SSE4-NEXT:    je LBB16_28
-; SSE4-NEXT:  ## %bb.27: ## %cond.store25
+; SSE4-NEXT:  LBB16_27: ## %cond.store25
 ; SSE4-NEXT:    pextrb $13, %xmm2, 13(%rdi)
-; SSE4-NEXT:  LBB16_28: ## %else26
-; SSE4-NEXT:    pxor %xmm4, %xmm4
-; SSE4-NEXT:    pcmpeqb %xmm4, %xmm0
-; SSE4-NEXT:    pextrb $14, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $16384, %eax ## imm = 0x4000
 ; SSE4-NEXT:    je LBB16_30
-; SSE4-NEXT:  ## %bb.29: ## %cond.store27
+; SSE4-NEXT:  LBB16_29: ## %cond.store27
 ; SSE4-NEXT:    pextrb $14, %xmm2, 14(%rdi)
-; SSE4-NEXT:  LBB16_30: ## %else28
-; SSE4-NEXT:    pextrb $15, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $32768, %eax ## imm = 0x8000
 ; SSE4-NEXT:    je LBB16_32
-; SSE4-NEXT:  ## %bb.31: ## %cond.store29
+; SSE4-NEXT:  LBB16_31: ## %cond.store29
 ; SSE4-NEXT:    pextrb $15, %xmm2, 15(%rdi)
-; SSE4-NEXT:  LBB16_32: ## %else30
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqb %xmm1, %xmm0
-; SSE4-NEXT:    pextrb $0, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $65536, %eax ## imm = 0x10000
 ; SSE4-NEXT:    je LBB16_34
-; SSE4-NEXT:  ## %bb.33: ## %cond.store31
+; SSE4-NEXT:  LBB16_33: ## %cond.store31
 ; SSE4-NEXT:    pextrb $0, %xmm3, 16(%rdi)
-; SSE4-NEXT:  LBB16_34: ## %else32
-; SSE4-NEXT:    pextrb $1, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $131072, %eax ## imm = 0x20000
 ; SSE4-NEXT:    je LBB16_36
-; SSE4-NEXT:  ## %bb.35: ## %cond.store33
+; SSE4-NEXT:  LBB16_35: ## %cond.store33
 ; SSE4-NEXT:    pextrb $1, %xmm3, 17(%rdi)
-; SSE4-NEXT:  LBB16_36: ## %else34
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqb %xmm1, %xmm0
-; SSE4-NEXT:    pextrb $2, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $262144, %eax ## imm = 0x40000
 ; SSE4-NEXT:    je LBB16_38
-; SSE4-NEXT:  ## %bb.37: ## %cond.store35
+; SSE4-NEXT:  LBB16_37: ## %cond.store35
 ; SSE4-NEXT:    pextrb $2, %xmm3, 18(%rdi)
-; SSE4-NEXT:  LBB16_38: ## %else36
-; SSE4-NEXT:    pextrb $3, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $524288, %eax ## imm = 0x80000
 ; SSE4-NEXT:    je LBB16_40
-; SSE4-NEXT:  ## %bb.39: ## %cond.store37
+; SSE4-NEXT:  LBB16_39: ## %cond.store37
 ; SSE4-NEXT:    pextrb $3, %xmm3, 19(%rdi)
-; SSE4-NEXT:  LBB16_40: ## %else38
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqb %xmm1, %xmm0
-; SSE4-NEXT:    pextrb $4, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $1048576, %eax ## imm = 0x100000
 ; SSE4-NEXT:    je LBB16_42
-; SSE4-NEXT:  ## %bb.41: ## %cond.store39
+; SSE4-NEXT:  LBB16_41: ## %cond.store39
 ; SSE4-NEXT:    pextrb $4, %xmm3, 20(%rdi)
-; SSE4-NEXT:  LBB16_42: ## %else40
-; SSE4-NEXT:    pextrb $5, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $2097152, %eax ## imm = 0x200000
 ; SSE4-NEXT:    je LBB16_44
-; SSE4-NEXT:  ## %bb.43: ## %cond.store41
+; SSE4-NEXT:  LBB16_43: ## %cond.store41
 ; SSE4-NEXT:    pextrb $5, %xmm3, 21(%rdi)
-; SSE4-NEXT:  LBB16_44: ## %else42
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqb %xmm1, %xmm0
-; SSE4-NEXT:    pextrb $6, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $4194304, %eax ## imm = 0x400000
 ; SSE4-NEXT:    je LBB16_46
-; SSE4-NEXT:  ## %bb.45: ## %cond.store43
+; SSE4-NEXT:  LBB16_45: ## %cond.store43
 ; SSE4-NEXT:    pextrb $6, %xmm3, 22(%rdi)
-; SSE4-NEXT:  LBB16_46: ## %else44
-; SSE4-NEXT:    pextrb $7, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $8388608, %eax ## imm = 0x800000
 ; SSE4-NEXT:    je LBB16_48
-; SSE4-NEXT:  ## %bb.47: ## %cond.store45
+; SSE4-NEXT:  LBB16_47: ## %cond.store45
 ; SSE4-NEXT:    pextrb $7, %xmm3, 23(%rdi)
-; SSE4-NEXT:  LBB16_48: ## %else46
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqb %xmm1, %xmm0
-; SSE4-NEXT:    pextrb $8, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $16777216, %eax ## imm = 0x1000000
 ; SSE4-NEXT:    je LBB16_50
-; SSE4-NEXT:  ## %bb.49: ## %cond.store47
+; SSE4-NEXT:  LBB16_49: ## %cond.store47
 ; SSE4-NEXT:    pextrb $8, %xmm3, 24(%rdi)
-; SSE4-NEXT:  LBB16_50: ## %else48
-; SSE4-NEXT:    pextrb $9, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $33554432, %eax ## imm = 0x2000000
 ; SSE4-NEXT:    je LBB16_52
-; SSE4-NEXT:  ## %bb.51: ## %cond.store49
+; SSE4-NEXT:  LBB16_51: ## %cond.store49
 ; SSE4-NEXT:    pextrb $9, %xmm3, 25(%rdi)
-; SSE4-NEXT:  LBB16_52: ## %else50
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqb %xmm1, %xmm0
-; SSE4-NEXT:    pextrb $10, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $67108864, %eax ## imm = 0x4000000
 ; SSE4-NEXT:    je LBB16_54
-; SSE4-NEXT:  ## %bb.53: ## %cond.store51
+; SSE4-NEXT:  LBB16_53: ## %cond.store51
 ; SSE4-NEXT:    pextrb $10, %xmm3, 26(%rdi)
-; SSE4-NEXT:  LBB16_54: ## %else52
-; SSE4-NEXT:    pextrb $11, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $134217728, %eax ## imm = 0x8000000
 ; SSE4-NEXT:    je LBB16_56
-; SSE4-NEXT:  ## %bb.55: ## %cond.store53
+; SSE4-NEXT:  LBB16_55: ## %cond.store53
 ; SSE4-NEXT:    pextrb $11, %xmm3, 27(%rdi)
-; SSE4-NEXT:  LBB16_56: ## %else54
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqb %xmm1, %xmm0
-; SSE4-NEXT:    pextrb $12, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $268435456, %eax ## imm = 0x10000000
 ; SSE4-NEXT:    je LBB16_58
-; SSE4-NEXT:  ## %bb.57: ## %cond.store55
+; SSE4-NEXT:  LBB16_57: ## %cond.store55
 ; SSE4-NEXT:    pextrb $12, %xmm3, 28(%rdi)
-; SSE4-NEXT:  LBB16_58: ## %else56
-; SSE4-NEXT:    pextrb $13, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $536870912, %eax ## imm = 0x20000000
 ; SSE4-NEXT:    je LBB16_60
-; SSE4-NEXT:  ## %bb.59: ## %cond.store57
+; SSE4-NEXT:  LBB16_59: ## %cond.store57
 ; SSE4-NEXT:    pextrb $13, %xmm3, 29(%rdi)
-; SSE4-NEXT:  LBB16_60: ## %else58
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqb %xmm0, %xmm1
-; SSE4-NEXT:    pextrb $14, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $1073741824, %eax ## imm = 0x40000000
 ; SSE4-NEXT:    je LBB16_62
-; SSE4-NEXT:  ## %bb.61: ## %cond.store59
+; SSE4-NEXT:  LBB16_61: ## %cond.store59
 ; SSE4-NEXT:    pextrb $14, %xmm3, 30(%rdi)
-; SSE4-NEXT:  LBB16_62: ## %else60
-; SSE4-NEXT:    pextrb $15, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $-2147483648, %eax ## imm = 0x80000000
 ; SSE4-NEXT:    je LBB16_64
-; SSE4-NEXT:  ## %bb.63: ## %cond.store61
+; SSE4-NEXT:  LBB16_63: ## %cond.store61
 ; SSE4-NEXT:    pextrb $15, %xmm3, 31(%rdi)
-; SSE4-NEXT:  LBB16_64: ## %else62
 ; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: store_v32i8_v32i8:
 ; AVX1:       ## %bb.0:
 ; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm3
-; AVX1-NEXT:    vpextrb $0, %xmm3, %eax
+; AVX1-NEXT:    vpmovmskb %xmm3, %ecx
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpmovmskb %xmm0, %eax
+; AVX1-NEXT:    shll $16, %eax
+; AVX1-NEXT:    orl %ecx, %eax
 ; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je LBB16_2
-; AVX1-NEXT:  ## %bb.1: ## %cond.store
+; AVX1-NEXT:    jne LBB16_1
+; AVX1-NEXT:  ## %bb.2: ## %else
+; AVX1-NEXT:    testb $2, %al
+; AVX1-NEXT:    jne LBB16_3
+; AVX1-NEXT:  LBB16_4: ## %else2
+; AVX1-NEXT:    testb $4, %al
+; AVX1-NEXT:    jne LBB16_5
+; AVX1-NEXT:  LBB16_6: ## %else4
+; AVX1-NEXT:    testb $8, %al
+; AVX1-NEXT:    jne LBB16_7
+; AVX1-NEXT:  LBB16_8: ## %else6
+; AVX1-NEXT:    testb $16, %al
+; AVX1-NEXT:    jne LBB16_9
+; AVX1-NEXT:  LBB16_10: ## %else8
+; AVX1-NEXT:    testb $32, %al
+; AVX1-NEXT:    jne LBB16_11
+; AVX1-NEXT:  LBB16_12: ## %else10
+; AVX1-NEXT:    testb $64, %al
+; AVX1-NEXT:    jne LBB16_13
+; AVX1-NEXT:  LBB16_14: ## %else12
+; AVX1-NEXT:    testb $-128, %al
+; AVX1-NEXT:    jne LBB16_15
+; AVX1-NEXT:  LBB16_16: ## %else14
+; AVX1-NEXT:    testl $256, %eax ## imm = 0x100
+; AVX1-NEXT:    jne LBB16_17
+; AVX1-NEXT:  LBB16_18: ## %else16
+; AVX1-NEXT:    testl $512, %eax ## imm = 0x200
+; AVX1-NEXT:    jne LBB16_19
+; AVX1-NEXT:  LBB16_20: ## %else18
+; AVX1-NEXT:    testl $1024, %eax ## imm = 0x400
+; AVX1-NEXT:    jne LBB16_21
+; AVX1-NEXT:  LBB16_22: ## %else20
+; AVX1-NEXT:    testl $2048, %eax ## imm = 0x800
+; AVX1-NEXT:    jne LBB16_23
+; AVX1-NEXT:  LBB16_24: ## %else22
+; AVX1-NEXT:    testl $4096, %eax ## imm = 0x1000
+; AVX1-NEXT:    jne LBB16_25
+; AVX1-NEXT:  LBB16_26: ## %else24
+; AVX1-NEXT:    testl $8192, %eax ## imm = 0x2000
+; AVX1-NEXT:    jne LBB16_27
+; AVX1-NEXT:  LBB16_28: ## %else26
+; AVX1-NEXT:    testl $16384, %eax ## imm = 0x4000
+; AVX1-NEXT:    jne LBB16_29
+; AVX1-NEXT:  LBB16_30: ## %else28
+; AVX1-NEXT:    testl $32768, %eax ## imm = 0x8000
+; AVX1-NEXT:    je LBB16_32
+; AVX1-NEXT:  LBB16_31: ## %cond.store29
+; AVX1-NEXT:    vpextrb $15, %xmm1, 15(%rdi)
+; AVX1-NEXT:  LBB16_32: ## %else30
+; AVX1-NEXT:    testl $65536, %eax ## imm = 0x10000
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm0
+; AVX1-NEXT:    jne LBB16_33
+; AVX1-NEXT:  ## %bb.34: ## %else32
+; AVX1-NEXT:    testl $131072, %eax ## imm = 0x20000
+; AVX1-NEXT:    jne LBB16_35
+; AVX1-NEXT:  LBB16_36: ## %else34
+; AVX1-NEXT:    testl $262144, %eax ## imm = 0x40000
+; AVX1-NEXT:    jne LBB16_37
+; AVX1-NEXT:  LBB16_38: ## %else36
+; AVX1-NEXT:    testl $524288, %eax ## imm = 0x80000
+; AVX1-NEXT:    jne LBB16_39
+; AVX1-NEXT:  LBB16_40: ## %else38
+; AVX1-NEXT:    testl $1048576, %eax ## imm = 0x100000
+; AVX1-NEXT:    jne LBB16_41
+; AVX1-NEXT:  LBB16_42: ## %else40
+; AVX1-NEXT:    testl $2097152, %eax ## imm = 0x200000
+; AVX1-NEXT:    jne LBB16_43
+; AVX1-NEXT:  LBB16_44: ## %else42
+; AVX1-NEXT:    testl $4194304, %eax ## imm = 0x400000
+; AVX1-NEXT:    jne LBB16_45
+; AVX1-NEXT:  LBB16_46: ## %else44
+; AVX1-NEXT:    testl $8388608, %eax ## imm = 0x800000
+; AVX1-NEXT:    jne LBB16_47
+; AVX1-NEXT:  LBB16_48: ## %else46
+; AVX1-NEXT:    testl $16777216, %eax ## imm = 0x1000000
+; AVX1-NEXT:    jne LBB16_49
+; AVX1-NEXT:  LBB16_50: ## %else48
+; AVX1-NEXT:    testl $33554432, %eax ## imm = 0x2000000
+; AVX1-NEXT:    jne LBB16_51
+; AVX1-NEXT:  LBB16_52: ## %else50
+; AVX1-NEXT:    testl $67108864, %eax ## imm = 0x4000000
+; AVX1-NEXT:    jne LBB16_53
+; AVX1-NEXT:  LBB16_54: ## %else52
+; AVX1-NEXT:    testl $134217728, %eax ## imm = 0x8000000
+; AVX1-NEXT:    jne LBB16_55
+; AVX1-NEXT:  LBB16_56: ## %else54
+; AVX1-NEXT:    testl $268435456, %eax ## imm = 0x10000000
+; AVX1-NEXT:    jne LBB16_57
+; AVX1-NEXT:  LBB16_58: ## %else56
+; AVX1-NEXT:    testl $536870912, %eax ## imm = 0x20000000
+; AVX1-NEXT:    jne LBB16_59
+; AVX1-NEXT:  LBB16_60: ## %else58
+; AVX1-NEXT:    testl $1073741824, %eax ## imm = 0x40000000
+; AVX1-NEXT:    jne LBB16_61
+; AVX1-NEXT:  LBB16_62: ## %else60
+; AVX1-NEXT:    testl $-2147483648, %eax ## imm = 0x80000000
+; AVX1-NEXT:    jne LBB16_63
+; AVX1-NEXT:  LBB16_64: ## %else62
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+; AVX1-NEXT:  LBB16_1: ## %cond.store
 ; AVX1-NEXT:    vpextrb $0, %xmm1, (%rdi)
-; AVX1-NEXT:  LBB16_2: ## %else
-; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm2
-; AVX1-NEXT:    vpextrb $1, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $2, %al
 ; AVX1-NEXT:    je LBB16_4
-; AVX1-NEXT:  ## %bb.3: ## %cond.store1
+; AVX1-NEXT:  LBB16_3: ## %cond.store1
 ; AVX1-NEXT:    vpextrb $1, %xmm1, 1(%rdi)
-; AVX1-NEXT:  LBB16_4: ## %else2
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm3
-; AVX1-NEXT:    vpextrb $2, %xmm3, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $4, %al
 ; AVX1-NEXT:    je LBB16_6
-; AVX1-NEXT:  ## %bb.5: ## %cond.store3
+; AVX1-NEXT:  LBB16_5: ## %cond.store3
 ; AVX1-NEXT:    vpextrb $2, %xmm1, 2(%rdi)
-; AVX1-NEXT:  LBB16_6: ## %else4
-; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm2
-; AVX1-NEXT:    vpextrb $3, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $8, %al
 ; AVX1-NEXT:    je LBB16_8
-; AVX1-NEXT:  ## %bb.7: ## %cond.store5
+; AVX1-NEXT:  LBB16_7: ## %cond.store5
 ; AVX1-NEXT:    vpextrb $3, %xmm1, 3(%rdi)
-; AVX1-NEXT:  LBB16_8: ## %else6
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm3
-; AVX1-NEXT:    vpextrb $4, %xmm3, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $16, %al
 ; AVX1-NEXT:    je LBB16_10
-; AVX1-NEXT:  ## %bb.9: ## %cond.store7
+; AVX1-NEXT:  LBB16_9: ## %cond.store7
 ; AVX1-NEXT:    vpextrb $4, %xmm1, 4(%rdi)
-; AVX1-NEXT:  LBB16_10: ## %else8
-; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm2
-; AVX1-NEXT:    vpextrb $5, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $32, %al
 ; AVX1-NEXT:    je LBB16_12
-; AVX1-NEXT:  ## %bb.11: ## %cond.store9
+; AVX1-NEXT:  LBB16_11: ## %cond.store9
 ; AVX1-NEXT:    vpextrb $5, %xmm1, 5(%rdi)
-; AVX1-NEXT:  LBB16_12: ## %else10
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm3
-; AVX1-NEXT:    vpextrb $6, %xmm3, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $64, %al
 ; AVX1-NEXT:    je LBB16_14
-; AVX1-NEXT:  ## %bb.13: ## %cond.store11
+; AVX1-NEXT:  LBB16_13: ## %cond.store11
 ; AVX1-NEXT:    vpextrb $6, %xmm1, 6(%rdi)
-; AVX1-NEXT:  LBB16_14: ## %else12
-; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm2
-; AVX1-NEXT:    vpextrb $7, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $-128, %al
 ; AVX1-NEXT:    je LBB16_16
-; AVX1-NEXT:  ## %bb.15: ## %cond.store13
+; AVX1-NEXT:  LBB16_15: ## %cond.store13
 ; AVX1-NEXT:    vpextrb $7, %xmm1, 7(%rdi)
-; AVX1-NEXT:  LBB16_16: ## %else14
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm3
-; AVX1-NEXT:    vpextrb $8, %xmm3, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $256, %eax ## imm = 0x100
 ; AVX1-NEXT:    je LBB16_18
-; AVX1-NEXT:  ## %bb.17: ## %cond.store15
+; AVX1-NEXT:  LBB16_17: ## %cond.store15
 ; AVX1-NEXT:    vpextrb $8, %xmm1, 8(%rdi)
-; AVX1-NEXT:  LBB16_18: ## %else16
-; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm2
-; AVX1-NEXT:    vpextrb $9, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $512, %eax ## imm = 0x200
 ; AVX1-NEXT:    je LBB16_20
-; AVX1-NEXT:  ## %bb.19: ## %cond.store17
+; AVX1-NEXT:  LBB16_19: ## %cond.store17
 ; AVX1-NEXT:    vpextrb $9, %xmm1, 9(%rdi)
-; AVX1-NEXT:  LBB16_20: ## %else18
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm3
-; AVX1-NEXT:    vpextrb $10, %xmm3, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $1024, %eax ## imm = 0x400
 ; AVX1-NEXT:    je LBB16_22
-; AVX1-NEXT:  ## %bb.21: ## %cond.store19
+; AVX1-NEXT:  LBB16_21: ## %cond.store19
 ; AVX1-NEXT:    vpextrb $10, %xmm1, 10(%rdi)
-; AVX1-NEXT:  LBB16_22: ## %else20
-; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm2
-; AVX1-NEXT:    vpextrb $11, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $2048, %eax ## imm = 0x800
 ; AVX1-NEXT:    je LBB16_24
-; AVX1-NEXT:  ## %bb.23: ## %cond.store21
+; AVX1-NEXT:  LBB16_23: ## %cond.store21
 ; AVX1-NEXT:    vpextrb $11, %xmm1, 11(%rdi)
-; AVX1-NEXT:  LBB16_24: ## %else22
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm3
-; AVX1-NEXT:    vpextrb $12, %xmm3, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $4096, %eax ## imm = 0x1000
 ; AVX1-NEXT:    je LBB16_26
-; AVX1-NEXT:  ## %bb.25: ## %cond.store23
+; AVX1-NEXT:  LBB16_25: ## %cond.store23
 ; AVX1-NEXT:    vpextrb $12, %xmm1, 12(%rdi)
-; AVX1-NEXT:  LBB16_26: ## %else24
-; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm2
-; AVX1-NEXT:    vpextrb $13, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $8192, %eax ## imm = 0x2000
 ; AVX1-NEXT:    je LBB16_28
-; AVX1-NEXT:  ## %bb.27: ## %cond.store25
+; AVX1-NEXT:  LBB16_27: ## %cond.store25
 ; AVX1-NEXT:    vpextrb $13, %xmm1, 13(%rdi)
-; AVX1-NEXT:  LBB16_28: ## %else26
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm3
-; AVX1-NEXT:    vpextrb $14, %xmm3, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $16384, %eax ## imm = 0x4000
 ; AVX1-NEXT:    je LBB16_30
-; AVX1-NEXT:  ## %bb.29: ## %cond.store27
+; AVX1-NEXT:  LBB16_29: ## %cond.store27
 ; AVX1-NEXT:    vpextrb $14, %xmm1, 14(%rdi)
-; AVX1-NEXT:  LBB16_30: ## %else28
-; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm2
-; AVX1-NEXT:    vpextrb $15, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je LBB16_32
-; AVX1-NEXT:  ## %bb.31: ## %cond.store29
-; AVX1-NEXT:    vpextrb $15, %xmm1, 15(%rdi)
-; AVX1-NEXT:  LBB16_32: ## %else30
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX1-NEXT:    vpxor %xmm0, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpeqb %xmm0, %xmm2, %xmm3
-; AVX1-NEXT:    vpextrb $0, %xmm3, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm0
-; AVX1-NEXT:    je LBB16_34
-; AVX1-NEXT:  ## %bb.33: ## %cond.store31
+; AVX1-NEXT:    testl $32768, %eax ## imm = 0x8000
+; AVX1-NEXT:    jne LBB16_31
+; AVX1-NEXT:    jmp LBB16_32
+; AVX1-NEXT:  LBB16_33: ## %cond.store31
 ; AVX1-NEXT:    vpextrb $0, %xmm0, 16(%rdi)
-; AVX1-NEXT:  LBB16_34: ## %else32
-; AVX1-NEXT:    vpextrb $1, %xmm3, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $131072, %eax ## imm = 0x20000
 ; AVX1-NEXT:    je LBB16_36
-; AVX1-NEXT:  ## %bb.35: ## %cond.store33
+; AVX1-NEXT:  LBB16_35: ## %cond.store33
 ; AVX1-NEXT:    vpextrb $1, %xmm0, 17(%rdi)
-; AVX1-NEXT:  LBB16_36: ## %else34
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vpextrb $2, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $262144, %eax ## imm = 0x40000
 ; AVX1-NEXT:    je LBB16_38
-; AVX1-NEXT:  ## %bb.37: ## %cond.store35
+; AVX1-NEXT:  LBB16_37: ## %cond.store35
 ; AVX1-NEXT:    vpextrb $2, %xmm0, 18(%rdi)
-; AVX1-NEXT:  LBB16_38: ## %else36
-; AVX1-NEXT:    vpextrb $3, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $524288, %eax ## imm = 0x80000
 ; AVX1-NEXT:    je LBB16_40
-; AVX1-NEXT:  ## %bb.39: ## %cond.store37
+; AVX1-NEXT:  LBB16_39: ## %cond.store37
 ; AVX1-NEXT:    vpextrb $3, %xmm0, 19(%rdi)
-; AVX1-NEXT:  LBB16_40: ## %else38
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vpextrb $4, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $1048576, %eax ## imm = 0x100000
 ; AVX1-NEXT:    je LBB16_42
-; AVX1-NEXT:  ## %bb.41: ## %cond.store39
+; AVX1-NEXT:  LBB16_41: ## %cond.store39
 ; AVX1-NEXT:    vpextrb $4, %xmm0, 20(%rdi)
-; AVX1-NEXT:  LBB16_42: ## %else40
-; AVX1-NEXT:    vpextrb $5, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $2097152, %eax ## imm = 0x200000
 ; AVX1-NEXT:    je LBB16_44
-; AVX1-NEXT:  ## %bb.43: ## %cond.store41
+; AVX1-NEXT:  LBB16_43: ## %cond.store41
 ; AVX1-NEXT:    vpextrb $5, %xmm0, 21(%rdi)
-; AVX1-NEXT:  LBB16_44: ## %else42
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vpextrb $6, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $4194304, %eax ## imm = 0x400000
 ; AVX1-NEXT:    je LBB16_46
-; AVX1-NEXT:  ## %bb.45: ## %cond.store43
+; AVX1-NEXT:  LBB16_45: ## %cond.store43
 ; AVX1-NEXT:    vpextrb $6, %xmm0, 22(%rdi)
-; AVX1-NEXT:  LBB16_46: ## %else44
-; AVX1-NEXT:    vpextrb $7, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $8388608, %eax ## imm = 0x800000
 ; AVX1-NEXT:    je LBB16_48
-; AVX1-NEXT:  ## %bb.47: ## %cond.store45
+; AVX1-NEXT:  LBB16_47: ## %cond.store45
 ; AVX1-NEXT:    vpextrb $7, %xmm0, 23(%rdi)
-; AVX1-NEXT:  LBB16_48: ## %else46
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $16777216, %eax ## imm = 0x1000000
 ; AVX1-NEXT:    je LBB16_50
-; AVX1-NEXT:  ## %bb.49: ## %cond.store47
+; AVX1-NEXT:  LBB16_49: ## %cond.store47
 ; AVX1-NEXT:    vpextrb $8, %xmm0, 24(%rdi)
-; AVX1-NEXT:  LBB16_50: ## %else48
-; AVX1-NEXT:    vpextrb $9, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $33554432, %eax ## imm = 0x2000000
 ; AVX1-NEXT:    je LBB16_52
-; AVX1-NEXT:  ## %bb.51: ## %cond.store49
+; AVX1-NEXT:  LBB16_51: ## %cond.store49
 ; AVX1-NEXT:    vpextrb $9, %xmm0, 25(%rdi)
-; AVX1-NEXT:  LBB16_52: ## %else50
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vpextrb $10, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $67108864, %eax ## imm = 0x4000000
 ; AVX1-NEXT:    je LBB16_54
-; AVX1-NEXT:  ## %bb.53: ## %cond.store51
+; AVX1-NEXT:  LBB16_53: ## %cond.store51
 ; AVX1-NEXT:    vpextrb $10, %xmm0, 26(%rdi)
-; AVX1-NEXT:  LBB16_54: ## %else52
-; AVX1-NEXT:    vpextrb $11, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $134217728, %eax ## imm = 0x8000000
 ; AVX1-NEXT:    je LBB16_56
-; AVX1-NEXT:  ## %bb.55: ## %cond.store53
+; AVX1-NEXT:  LBB16_55: ## %cond.store53
 ; AVX1-NEXT:    vpextrb $11, %xmm0, 27(%rdi)
-; AVX1-NEXT:  LBB16_56: ## %else54
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $268435456, %eax ## imm = 0x10000000
 ; AVX1-NEXT:    je LBB16_58
-; AVX1-NEXT:  ## %bb.57: ## %cond.store55
+; AVX1-NEXT:  LBB16_57: ## %cond.store55
 ; AVX1-NEXT:    vpextrb $12, %xmm0, 28(%rdi)
-; AVX1-NEXT:  LBB16_58: ## %else56
-; AVX1-NEXT:    vpextrb $13, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $536870912, %eax ## imm = 0x20000000
 ; AVX1-NEXT:    je LBB16_60
-; AVX1-NEXT:  ## %bb.59: ## %cond.store57
+; AVX1-NEXT:  LBB16_59: ## %cond.store57
 ; AVX1-NEXT:    vpextrb $13, %xmm0, 29(%rdi)
-; AVX1-NEXT:  LBB16_60: ## %else58
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vpextrb $14, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $1073741824, %eax ## imm = 0x40000000
 ; AVX1-NEXT:    je LBB16_62
-; AVX1-NEXT:  ## %bb.61: ## %cond.store59
+; AVX1-NEXT:  LBB16_61: ## %cond.store59
 ; AVX1-NEXT:    vpextrb $14, %xmm0, 30(%rdi)
-; AVX1-NEXT:  LBB16_62: ## %else60
-; AVX1-NEXT:    vpextrb $15, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $-2147483648, %eax ## imm = 0x80000000
 ; AVX1-NEXT:    je LBB16_64
-; AVX1-NEXT:  ## %bb.63: ## %cond.store61
+; AVX1-NEXT:  LBB16_63: ## %cond.store61
 ; AVX1-NEXT:    vpextrb $15, %xmm0, 31(%rdi)
-; AVX1-NEXT:  LBB16_64: ## %else62
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: store_v32i8_v32i8:
 ; AVX2:       ## %bb.0:
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm3
-; AVX2-NEXT:    vpextrb $0, %xmm3, %eax
+; AVX2-NEXT:    vpcmpeqb %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpmovmskb %ymm0, %eax
 ; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je LBB16_2
-; AVX2-NEXT:  ## %bb.1: ## %cond.store
+; AVX2-NEXT:    jne LBB16_1
+; AVX2-NEXT:  ## %bb.2: ## %else
+; AVX2-NEXT:    testb $2, %al
+; AVX2-NEXT:    jne LBB16_3
+; AVX2-NEXT:  LBB16_4: ## %else2
+; AVX2-NEXT:    testb $4, %al
+; AVX2-NEXT:    jne LBB16_5
+; AVX2-NEXT:  LBB16_6: ## %else4
+; AVX2-NEXT:    testb $8, %al
+; AVX2-NEXT:    jne LBB16_7
+; AVX2-NEXT:  LBB16_8: ## %else6
+; AVX2-NEXT:    testb $16, %al
+; AVX2-NEXT:    jne LBB16_9
+; AVX2-NEXT:  LBB16_10: ## %else8
+; AVX2-NEXT:    testb $32, %al
+; AVX2-NEXT:    jne LBB16_11
+; AVX2-NEXT:  LBB16_12: ## %else10
+; AVX2-NEXT:    testb $64, %al
+; AVX2-NEXT:    jne LBB16_13
+; AVX2-NEXT:  LBB16_14: ## %else12
+; AVX2-NEXT:    testb $-128, %al
+; AVX2-NEXT:    jne LBB16_15
+; AVX2-NEXT:  LBB16_16: ## %else14
+; AVX2-NEXT:    testl $256, %eax ## imm = 0x100
+; AVX2-NEXT:    jne LBB16_17
+; AVX2-NEXT:  LBB16_18: ## %else16
+; AVX2-NEXT:    testl $512, %eax ## imm = 0x200
+; AVX2-NEXT:    jne LBB16_19
+; AVX2-NEXT:  LBB16_20: ## %else18
+; AVX2-NEXT:    testl $1024, %eax ## imm = 0x400
+; AVX2-NEXT:    jne LBB16_21
+; AVX2-NEXT:  LBB16_22: ## %else20
+; AVX2-NEXT:    testl $2048, %eax ## imm = 0x800
+; AVX2-NEXT:    jne LBB16_23
+; AVX2-NEXT:  LBB16_24: ## %else22
+; AVX2-NEXT:    testl $4096, %eax ## imm = 0x1000
+; AVX2-NEXT:    jne LBB16_25
+; AVX2-NEXT:  LBB16_26: ## %else24
+; AVX2-NEXT:    testl $8192, %eax ## imm = 0x2000
+; AVX2-NEXT:    jne LBB16_27
+; AVX2-NEXT:  LBB16_28: ## %else26
+; AVX2-NEXT:    testl $16384, %eax ## imm = 0x4000
+; AVX2-NEXT:    jne LBB16_29
+; AVX2-NEXT:  LBB16_30: ## %else28
+; AVX2-NEXT:    testl $32768, %eax ## imm = 0x8000
+; AVX2-NEXT:    je LBB16_32
+; AVX2-NEXT:  LBB16_31: ## %cond.store29
+; AVX2-NEXT:    vpextrb $15, %xmm1, 15(%rdi)
+; AVX2-NEXT:  LBB16_32: ## %else30
+; AVX2-NEXT:    testl $65536, %eax ## imm = 0x10000
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX2-NEXT:    jne LBB16_33
+; AVX2-NEXT:  ## %bb.34: ## %else32
+; AVX2-NEXT:    testl $131072, %eax ## imm = 0x20000
+; AVX2-NEXT:    jne LBB16_35
+; AVX2-NEXT:  LBB16_36: ## %else34
+; AVX2-NEXT:    testl $262144, %eax ## imm = 0x40000
+; AVX2-NEXT:    jne LBB16_37
+; AVX2-NEXT:  LBB16_38: ## %else36
+; AVX2-NEXT:    testl $524288, %eax ## imm = 0x80000
+; AVX2-NEXT:    jne LBB16_39
+; AVX2-NEXT:  LBB16_40: ## %else38
+; AVX2-NEXT:    testl $1048576, %eax ## imm = 0x100000
+; AVX2-NEXT:    jne LBB16_41
+; AVX2-NEXT:  LBB16_42: ## %else40
+; AVX2-NEXT:    testl $2097152, %eax ## imm = 0x200000
+; AVX2-NEXT:    jne LBB16_43
+; AVX2-NEXT:  LBB16_44: ## %else42
+; AVX2-NEXT:    testl $4194304, %eax ## imm = 0x400000
+; AVX2-NEXT:    jne LBB16_45
+; AVX2-NEXT:  LBB16_46: ## %else44
+; AVX2-NEXT:    testl $8388608, %eax ## imm = 0x800000
+; AVX2-NEXT:    jne LBB16_47
+; AVX2-NEXT:  LBB16_48: ## %else46
+; AVX2-NEXT:    testl $16777216, %eax ## imm = 0x1000000
+; AVX2-NEXT:    jne LBB16_49
+; AVX2-NEXT:  LBB16_50: ## %else48
+; AVX2-NEXT:    testl $33554432, %eax ## imm = 0x2000000
+; AVX2-NEXT:    jne LBB16_51
+; AVX2-NEXT:  LBB16_52: ## %else50
+; AVX2-NEXT:    testl $67108864, %eax ## imm = 0x4000000
+; AVX2-NEXT:    jne LBB16_53
+; AVX2-NEXT:  LBB16_54: ## %else52
+; AVX2-NEXT:    testl $134217728, %eax ## imm = 0x8000000
+; AVX2-NEXT:    jne LBB16_55
+; AVX2-NEXT:  LBB16_56: ## %else54
+; AVX2-NEXT:    testl $268435456, %eax ## imm = 0x10000000
+; AVX2-NEXT:    jne LBB16_57
+; AVX2-NEXT:  LBB16_58: ## %else56
+; AVX2-NEXT:    testl $536870912, %eax ## imm = 0x20000000
+; AVX2-NEXT:    jne LBB16_59
+; AVX2-NEXT:  LBB16_60: ## %else58
+; AVX2-NEXT:    testl $1073741824, %eax ## imm = 0x40000000
+; AVX2-NEXT:    jne LBB16_61
+; AVX2-NEXT:  LBB16_62: ## %else60
+; AVX2-NEXT:    testl $-2147483648, %eax ## imm = 0x80000000
+; AVX2-NEXT:    jne LBB16_63
+; AVX2-NEXT:  LBB16_64: ## %else62
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+; AVX2-NEXT:  LBB16_1: ## %cond.store
 ; AVX2-NEXT:    vpextrb $0, %xmm1, (%rdi)
-; AVX2-NEXT:  LBB16_2: ## %else
-; AVX2-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm2
-; AVX2-NEXT:    vpextrb $1, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $2, %al
 ; AVX2-NEXT:    je LBB16_4
-; AVX2-NEXT:  ## %bb.3: ## %cond.store1
+; AVX2-NEXT:  LBB16_3: ## %cond.store1
 ; AVX2-NEXT:    vpextrb $1, %xmm1, 1(%rdi)
-; AVX2-NEXT:  LBB16_4: ## %else2
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm3
-; AVX2-NEXT:    vpextrb $2, %xmm3, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $4, %al
 ; AVX2-NEXT:    je LBB16_6
-; AVX2-NEXT:  ## %bb.5: ## %cond.store3
+; AVX2-NEXT:  LBB16_5: ## %cond.store3
 ; AVX2-NEXT:    vpextrb $2, %xmm1, 2(%rdi)
-; AVX2-NEXT:  LBB16_6: ## %else4
-; AVX2-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm2
-; AVX2-NEXT:    vpextrb $3, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $8, %al
 ; AVX2-NEXT:    je LBB16_8
-; AVX2-NEXT:  ## %bb.7: ## %cond.store5
+; AVX2-NEXT:  LBB16_7: ## %cond.store5
 ; AVX2-NEXT:    vpextrb $3, %xmm1, 3(%rdi)
-; AVX2-NEXT:  LBB16_8: ## %else6
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm3
-; AVX2-NEXT:    vpextrb $4, %xmm3, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $16, %al
 ; AVX2-NEXT:    je LBB16_10
-; AVX2-NEXT:  ## %bb.9: ## %cond.store7
+; AVX2-NEXT:  LBB16_9: ## %cond.store7
 ; AVX2-NEXT:    vpextrb $4, %xmm1, 4(%rdi)
-; AVX2-NEXT:  LBB16_10: ## %else8
-; AVX2-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm2
-; AVX2-NEXT:    vpextrb $5, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $32, %al
 ; AVX2-NEXT:    je LBB16_12
-; AVX2-NEXT:  ## %bb.11: ## %cond.store9
+; AVX2-NEXT:  LBB16_11: ## %cond.store9
 ; AVX2-NEXT:    vpextrb $5, %xmm1, 5(%rdi)
-; AVX2-NEXT:  LBB16_12: ## %else10
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm3
-; AVX2-NEXT:    vpextrb $6, %xmm3, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $64, %al
 ; AVX2-NEXT:    je LBB16_14
-; AVX2-NEXT:  ## %bb.13: ## %cond.store11
+; AVX2-NEXT:  LBB16_13: ## %cond.store11
 ; AVX2-NEXT:    vpextrb $6, %xmm1, 6(%rdi)
-; AVX2-NEXT:  LBB16_14: ## %else12
-; AVX2-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm2
-; AVX2-NEXT:    vpextrb $7, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $-128, %al
 ; AVX2-NEXT:    je LBB16_16
-; AVX2-NEXT:  ## %bb.15: ## %cond.store13
+; AVX2-NEXT:  LBB16_15: ## %cond.store13
 ; AVX2-NEXT:    vpextrb $7, %xmm1, 7(%rdi)
-; AVX2-NEXT:  LBB16_16: ## %else14
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm3
-; AVX2-NEXT:    vpextrb $8, %xmm3, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $256, %eax ## imm = 0x100
 ; AVX2-NEXT:    je LBB16_18
-; AVX2-NEXT:  ## %bb.17: ## %cond.store15
+; AVX2-NEXT:  LBB16_17: ## %cond.store15
 ; AVX2-NEXT:    vpextrb $8, %xmm1, 8(%rdi)
-; AVX2-NEXT:  LBB16_18: ## %else16
-; AVX2-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm2
-; AVX2-NEXT:    vpextrb $9, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $512, %eax ## imm = 0x200
 ; AVX2-NEXT:    je LBB16_20
-; AVX2-NEXT:  ## %bb.19: ## %cond.store17
+; AVX2-NEXT:  LBB16_19: ## %cond.store17
 ; AVX2-NEXT:    vpextrb $9, %xmm1, 9(%rdi)
-; AVX2-NEXT:  LBB16_20: ## %else18
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm3
-; AVX2-NEXT:    vpextrb $10, %xmm3, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $1024, %eax ## imm = 0x400
 ; AVX2-NEXT:    je LBB16_22
-; AVX2-NEXT:  ## %bb.21: ## %cond.store19
+; AVX2-NEXT:  LBB16_21: ## %cond.store19
 ; AVX2-NEXT:    vpextrb $10, %xmm1, 10(%rdi)
-; AVX2-NEXT:  LBB16_22: ## %else20
-; AVX2-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm2
-; AVX2-NEXT:    vpextrb $11, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $2048, %eax ## imm = 0x800
 ; AVX2-NEXT:    je LBB16_24
-; AVX2-NEXT:  ## %bb.23: ## %cond.store21
+; AVX2-NEXT:  LBB16_23: ## %cond.store21
 ; AVX2-NEXT:    vpextrb $11, %xmm1, 11(%rdi)
-; AVX2-NEXT:  LBB16_24: ## %else22
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm3
-; AVX2-NEXT:    vpextrb $12, %xmm3, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $4096, %eax ## imm = 0x1000
 ; AVX2-NEXT:    je LBB16_26
-; AVX2-NEXT:  ## %bb.25: ## %cond.store23
+; AVX2-NEXT:  LBB16_25: ## %cond.store23
 ; AVX2-NEXT:    vpextrb $12, %xmm1, 12(%rdi)
-; AVX2-NEXT:  LBB16_26: ## %else24
-; AVX2-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm2
-; AVX2-NEXT:    vpextrb $13, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $8192, %eax ## imm = 0x2000
 ; AVX2-NEXT:    je LBB16_28
-; AVX2-NEXT:  ## %bb.27: ## %cond.store25
+; AVX2-NEXT:  LBB16_27: ## %cond.store25
 ; AVX2-NEXT:    vpextrb $13, %xmm1, 13(%rdi)
-; AVX2-NEXT:  LBB16_28: ## %else26
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm3
-; AVX2-NEXT:    vpextrb $14, %xmm3, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $16384, %eax ## imm = 0x4000
 ; AVX2-NEXT:    je LBB16_30
-; AVX2-NEXT:  ## %bb.29: ## %cond.store27
+; AVX2-NEXT:  LBB16_29: ## %cond.store27
 ; AVX2-NEXT:    vpextrb $14, %xmm1, 14(%rdi)
-; AVX2-NEXT:  LBB16_30: ## %else28
-; AVX2-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm2
-; AVX2-NEXT:    vpextrb $15, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je LBB16_32
-; AVX2-NEXT:  ## %bb.31: ## %cond.store29
-; AVX2-NEXT:    vpextrb $15, %xmm1, 15(%rdi)
-; AVX2-NEXT:  LBB16_32: ## %else30
-; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm2
-; AVX2-NEXT:    vpxor %xmm0, %xmm0, %xmm0
-; AVX2-NEXT:    vpcmpeqb %xmm0, %xmm2, %xmm3
-; AVX2-NEXT:    vpextrb $0, %xmm3, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm0
-; AVX2-NEXT:    je LBB16_34
-; AVX2-NEXT:  ## %bb.33: ## %cond.store31
+; AVX2-NEXT:    testl $32768, %eax ## imm = 0x8000
+; AVX2-NEXT:    jne LBB16_31
+; AVX2-NEXT:    jmp LBB16_32
+; AVX2-NEXT:  LBB16_33: ## %cond.store31
 ; AVX2-NEXT:    vpextrb $0, %xmm0, 16(%rdi)
-; AVX2-NEXT:  LBB16_34: ## %else32
-; AVX2-NEXT:    vpextrb $1, %xmm3, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $131072, %eax ## imm = 0x20000
 ; AVX2-NEXT:    je LBB16_36
-; AVX2-NEXT:  ## %bb.35: ## %cond.store33
+; AVX2-NEXT:  LBB16_35: ## %cond.store33
 ; AVX2-NEXT:    vpextrb $1, %xmm0, 17(%rdi)
-; AVX2-NEXT:  LBB16_36: ## %else34
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX2-NEXT:    vpextrb $2, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $262144, %eax ## imm = 0x40000
 ; AVX2-NEXT:    je LBB16_38
-; AVX2-NEXT:  ## %bb.37: ## %cond.store35
+; AVX2-NEXT:  LBB16_37: ## %cond.store35
 ; AVX2-NEXT:    vpextrb $2, %xmm0, 18(%rdi)
-; AVX2-NEXT:  LBB16_38: ## %else36
-; AVX2-NEXT:    vpextrb $3, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $524288, %eax ## imm = 0x80000
 ; AVX2-NEXT:    je LBB16_40
-; AVX2-NEXT:  ## %bb.39: ## %cond.store37
+; AVX2-NEXT:  LBB16_39: ## %cond.store37
 ; AVX2-NEXT:    vpextrb $3, %xmm0, 19(%rdi)
-; AVX2-NEXT:  LBB16_40: ## %else38
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX2-NEXT:    vpextrb $4, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $1048576, %eax ## imm = 0x100000
 ; AVX2-NEXT:    je LBB16_42
-; AVX2-NEXT:  ## %bb.41: ## %cond.store39
+; AVX2-NEXT:  LBB16_41: ## %cond.store39
 ; AVX2-NEXT:    vpextrb $4, %xmm0, 20(%rdi)
-; AVX2-NEXT:  LBB16_42: ## %else40
-; AVX2-NEXT:    vpextrb $5, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $2097152, %eax ## imm = 0x200000
 ; AVX2-NEXT:    je LBB16_44
-; AVX2-NEXT:  ## %bb.43: ## %cond.store41
+; AVX2-NEXT:  LBB16_43: ## %cond.store41
 ; AVX2-NEXT:    vpextrb $5, %xmm0, 21(%rdi)
-; AVX2-NEXT:  LBB16_44: ## %else42
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX2-NEXT:    vpextrb $6, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $4194304, %eax ## imm = 0x400000
 ; AVX2-NEXT:    je LBB16_46
-; AVX2-NEXT:  ## %bb.45: ## %cond.store43
+; AVX2-NEXT:  LBB16_45: ## %cond.store43
 ; AVX2-NEXT:    vpextrb $6, %xmm0, 22(%rdi)
-; AVX2-NEXT:  LBB16_46: ## %else44
-; AVX2-NEXT:    vpextrb $7, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $8388608, %eax ## imm = 0x800000
 ; AVX2-NEXT:    je LBB16_48
-; AVX2-NEXT:  ## %bb.47: ## %cond.store45
+; AVX2-NEXT:  LBB16_47: ## %cond.store45
 ; AVX2-NEXT:    vpextrb $7, %xmm0, 23(%rdi)
-; AVX2-NEXT:  LBB16_48: ## %else46
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX2-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $16777216, %eax ## imm = 0x1000000
 ; AVX2-NEXT:    je LBB16_50
-; AVX2-NEXT:  ## %bb.49: ## %cond.store47
+; AVX2-NEXT:  LBB16_49: ## %cond.store47
 ; AVX2-NEXT:    vpextrb $8, %xmm0, 24(%rdi)
-; AVX2-NEXT:  LBB16_50: ## %else48
-; AVX2-NEXT:    vpextrb $9, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $33554432, %eax ## imm = 0x2000000
 ; AVX2-NEXT:    je LBB16_52
-; AVX2-NEXT:  ## %bb.51: ## %cond.store49
+; AVX2-NEXT:  LBB16_51: ## %cond.store49
 ; AVX2-NEXT:    vpextrb $9, %xmm0, 25(%rdi)
-; AVX2-NEXT:  LBB16_52: ## %else50
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX2-NEXT:    vpextrb $10, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $67108864, %eax ## imm = 0x4000000
 ; AVX2-NEXT:    je LBB16_54
-; AVX2-NEXT:  ## %bb.53: ## %cond.store51
+; AVX2-NEXT:  LBB16_53: ## %cond.store51
 ; AVX2-NEXT:    vpextrb $10, %xmm0, 26(%rdi)
-; AVX2-NEXT:  LBB16_54: ## %else52
-; AVX2-NEXT:    vpextrb $11, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $134217728, %eax ## imm = 0x8000000
 ; AVX2-NEXT:    je LBB16_56
-; AVX2-NEXT:  ## %bb.55: ## %cond.store53
+; AVX2-NEXT:  LBB16_55: ## %cond.store53
 ; AVX2-NEXT:    vpextrb $11, %xmm0, 27(%rdi)
-; AVX2-NEXT:  LBB16_56: ## %else54
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX2-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $268435456, %eax ## imm = 0x10000000
 ; AVX2-NEXT:    je LBB16_58
-; AVX2-NEXT:  ## %bb.57: ## %cond.store55
+; AVX2-NEXT:  LBB16_57: ## %cond.store55
 ; AVX2-NEXT:    vpextrb $12, %xmm0, 28(%rdi)
-; AVX2-NEXT:  LBB16_58: ## %else56
-; AVX2-NEXT:    vpextrb $13, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $536870912, %eax ## imm = 0x20000000
 ; AVX2-NEXT:    je LBB16_60
-; AVX2-NEXT:  ## %bb.59: ## %cond.store57
+; AVX2-NEXT:  LBB16_59: ## %cond.store57
 ; AVX2-NEXT:    vpextrb $13, %xmm0, 29(%rdi)
-; AVX2-NEXT:  LBB16_60: ## %else58
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX2-NEXT:    vpextrb $14, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $1073741824, %eax ## imm = 0x40000000
 ; AVX2-NEXT:    je LBB16_62
-; AVX2-NEXT:  ## %bb.61: ## %cond.store59
+; AVX2-NEXT:  LBB16_61: ## %cond.store59
 ; AVX2-NEXT:    vpextrb $14, %xmm0, 30(%rdi)
-; AVX2-NEXT:  LBB16_62: ## %else60
-; AVX2-NEXT:    vpextrb $15, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $-2147483648, %eax ## imm = 0x80000000
 ; AVX2-NEXT:    je LBB16_64
-; AVX2-NEXT:  ## %bb.63: ## %cond.store61
+; AVX2-NEXT:  LBB16_63: ## %cond.store61
 ; AVX2-NEXT:    vpextrb $15, %xmm0, 31(%rdi)
-; AVX2-NEXT:  LBB16_64: ## %else62
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: store_v32i8_v32i8:
 ; AVX512F:       ## %bb.0:
 ; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    vpcmpeqb %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vpmovmskb %ymm0, %eax
 ; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je LBB16_2
-; AVX512F-NEXT:  ## %bb.1: ## %cond.store
+; AVX512F-NEXT:    jne LBB16_1
+; AVX512F-NEXT:  ## %bb.2: ## %else
+; AVX512F-NEXT:    testb $2, %al
+; AVX512F-NEXT:    jne LBB16_3
+; AVX512F-NEXT:  LBB16_4: ## %else2
+; AVX512F-NEXT:    testb $4, %al
+; AVX512F-NEXT:    jne LBB16_5
+; AVX512F-NEXT:  LBB16_6: ## %else4
+; AVX512F-NEXT:    testb $8, %al
+; AVX512F-NEXT:    jne LBB16_7
+; AVX512F-NEXT:  LBB16_8: ## %else6
+; AVX512F-NEXT:    testb $16, %al
+; AVX512F-NEXT:    jne LBB16_9
+; AVX512F-NEXT:  LBB16_10: ## %else8
+; AVX512F-NEXT:    testb $32, %al
+; AVX512F-NEXT:    jne LBB16_11
+; AVX512F-NEXT:  LBB16_12: ## %else10
+; AVX512F-NEXT:    testb $64, %al
+; AVX512F-NEXT:    jne LBB16_13
+; AVX512F-NEXT:  LBB16_14: ## %else12
+; AVX512F-NEXT:    testb $-128, %al
+; AVX512F-NEXT:    jne LBB16_15
+; AVX512F-NEXT:  LBB16_16: ## %else14
+; AVX512F-NEXT:    testl $256, %eax ## imm = 0x100
+; AVX512F-NEXT:    jne LBB16_17
+; AVX512F-NEXT:  LBB16_18: ## %else16
+; AVX512F-NEXT:    testl $512, %eax ## imm = 0x200
+; AVX512F-NEXT:    jne LBB16_19
+; AVX512F-NEXT:  LBB16_20: ## %else18
+; AVX512F-NEXT:    testl $1024, %eax ## imm = 0x400
+; AVX512F-NEXT:    jne LBB16_21
+; AVX512F-NEXT:  LBB16_22: ## %else20
+; AVX512F-NEXT:    testl $2048, %eax ## imm = 0x800
+; AVX512F-NEXT:    jne LBB16_23
+; AVX512F-NEXT:  LBB16_24: ## %else22
+; AVX512F-NEXT:    testl $4096, %eax ## imm = 0x1000
+; AVX512F-NEXT:    jne LBB16_25
+; AVX512F-NEXT:  LBB16_26: ## %else24
+; AVX512F-NEXT:    testl $8192, %eax ## imm = 0x2000
+; AVX512F-NEXT:    jne LBB16_27
+; AVX512F-NEXT:  LBB16_28: ## %else26
+; AVX512F-NEXT:    testl $16384, %eax ## imm = 0x4000
+; AVX512F-NEXT:    jne LBB16_29
+; AVX512F-NEXT:  LBB16_30: ## %else28
+; AVX512F-NEXT:    testl $32768, %eax ## imm = 0x8000
+; AVX512F-NEXT:    je LBB16_32
+; AVX512F-NEXT:  LBB16_31: ## %cond.store29
+; AVX512F-NEXT:    vpextrb $15, %xmm1, 15(%rdi)
+; AVX512F-NEXT:  LBB16_32: ## %else30
+; AVX512F-NEXT:    testl $65536, %eax ## imm = 0x10000
+; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX512F-NEXT:    jne LBB16_33
+; AVX512F-NEXT:  ## %bb.34: ## %else32
+; AVX512F-NEXT:    testl $131072, %eax ## imm = 0x20000
+; AVX512F-NEXT:    jne LBB16_35
+; AVX512F-NEXT:  LBB16_36: ## %else34
+; AVX512F-NEXT:    testl $262144, %eax ## imm = 0x40000
+; AVX512F-NEXT:    jne LBB16_37
+; AVX512F-NEXT:  LBB16_38: ## %else36
+; AVX512F-NEXT:    testl $524288, %eax ## imm = 0x80000
+; AVX512F-NEXT:    jne LBB16_39
+; AVX512F-NEXT:  LBB16_40: ## %else38
+; AVX512F-NEXT:    testl $1048576, %eax ## imm = 0x100000
+; AVX512F-NEXT:    jne LBB16_41
+; AVX512F-NEXT:  LBB16_42: ## %else40
+; AVX512F-NEXT:    testl $2097152, %eax ## imm = 0x200000
+; AVX512F-NEXT:    jne LBB16_43
+; AVX512F-NEXT:  LBB16_44: ## %else42
+; AVX512F-NEXT:    testl $4194304, %eax ## imm = 0x400000
+; AVX512F-NEXT:    jne LBB16_45
+; AVX512F-NEXT:  LBB16_46: ## %else44
+; AVX512F-NEXT:    testl $8388608, %eax ## imm = 0x800000
+; AVX512F-NEXT:    jne LBB16_47
+; AVX512F-NEXT:  LBB16_48: ## %else46
+; AVX512F-NEXT:    testl $16777216, %eax ## imm = 0x1000000
+; AVX512F-NEXT:    jne LBB16_49
+; AVX512F-NEXT:  LBB16_50: ## %else48
+; AVX512F-NEXT:    testl $33554432, %eax ## imm = 0x2000000
+; AVX512F-NEXT:    jne LBB16_51
+; AVX512F-NEXT:  LBB16_52: ## %else50
+; AVX512F-NEXT:    testl $67108864, %eax ## imm = 0x4000000
+; AVX512F-NEXT:    jne LBB16_53
+; AVX512F-NEXT:  LBB16_54: ## %else52
+; AVX512F-NEXT:    testl $134217728, %eax ## imm = 0x8000000
+; AVX512F-NEXT:    jne LBB16_55
+; AVX512F-NEXT:  LBB16_56: ## %else54
+; AVX512F-NEXT:    testl $268435456, %eax ## imm = 0x10000000
+; AVX512F-NEXT:    jne LBB16_57
+; AVX512F-NEXT:  LBB16_58: ## %else56
+; AVX512F-NEXT:    testl $536870912, %eax ## imm = 0x20000000
+; AVX512F-NEXT:    jne LBB16_59
+; AVX512F-NEXT:  LBB16_60: ## %else58
+; AVX512F-NEXT:    testl $1073741824, %eax ## imm = 0x40000000
+; AVX512F-NEXT:    jne LBB16_61
+; AVX512F-NEXT:  LBB16_62: ## %else60
+; AVX512F-NEXT:    testl $-2147483648, %eax ## imm = 0x80000000
+; AVX512F-NEXT:    jne LBB16_63
+; AVX512F-NEXT:  LBB16_64: ## %else62
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+; AVX512F-NEXT:  LBB16_1: ## %cond.store
 ; AVX512F-NEXT:    vpextrb $0, %xmm1, (%rdi)
-; AVX512F-NEXT:  LBB16_2: ## %else
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $2, %al
 ; AVX512F-NEXT:    je LBB16_4
-; AVX512F-NEXT:  ## %bb.3: ## %cond.store1
+; AVX512F-NEXT:  LBB16_3: ## %cond.store1
 ; AVX512F-NEXT:    vpextrb $1, %xmm1, 1(%rdi)
-; AVX512F-NEXT:  LBB16_4: ## %else2
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $4, %al
 ; AVX512F-NEXT:    je LBB16_6
-; AVX512F-NEXT:  ## %bb.5: ## %cond.store3
+; AVX512F-NEXT:  LBB16_5: ## %cond.store3
 ; AVX512F-NEXT:    vpextrb $2, %xmm1, 2(%rdi)
-; AVX512F-NEXT:  LBB16_6: ## %else4
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $8, %al
 ; AVX512F-NEXT:    je LBB16_8
-; AVX512F-NEXT:  ## %bb.7: ## %cond.store5
+; AVX512F-NEXT:  LBB16_7: ## %cond.store5
 ; AVX512F-NEXT:    vpextrb $3, %xmm1, 3(%rdi)
-; AVX512F-NEXT:  LBB16_8: ## %else6
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $4, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $16, %al
 ; AVX512F-NEXT:    je LBB16_10
-; AVX512F-NEXT:  ## %bb.9: ## %cond.store7
+; AVX512F-NEXT:  LBB16_9: ## %cond.store7
 ; AVX512F-NEXT:    vpextrb $4, %xmm1, 4(%rdi)
-; AVX512F-NEXT:  LBB16_10: ## %else8
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $5, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $32, %al
 ; AVX512F-NEXT:    je LBB16_12
-; AVX512F-NEXT:  ## %bb.11: ## %cond.store9
+; AVX512F-NEXT:  LBB16_11: ## %cond.store9
 ; AVX512F-NEXT:    vpextrb $5, %xmm1, 5(%rdi)
-; AVX512F-NEXT:  LBB16_12: ## %else10
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $6, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $64, %al
 ; AVX512F-NEXT:    je LBB16_14
-; AVX512F-NEXT:  ## %bb.13: ## %cond.store11
+; AVX512F-NEXT:  LBB16_13: ## %cond.store11
 ; AVX512F-NEXT:    vpextrb $6, %xmm1, 6(%rdi)
-; AVX512F-NEXT:  LBB16_14: ## %else12
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $7, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $-128, %al
 ; AVX512F-NEXT:    je LBB16_16
-; AVX512F-NEXT:  ## %bb.15: ## %cond.store13
+; AVX512F-NEXT:  LBB16_15: ## %cond.store13
 ; AVX512F-NEXT:    vpextrb $7, %xmm1, 7(%rdi)
-; AVX512F-NEXT:  LBB16_16: ## %else14
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $8, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $256, %eax ## imm = 0x100
 ; AVX512F-NEXT:    je LBB16_18
-; AVX512F-NEXT:  ## %bb.17: ## %cond.store15
+; AVX512F-NEXT:  LBB16_17: ## %cond.store15
 ; AVX512F-NEXT:    vpextrb $8, %xmm1, 8(%rdi)
-; AVX512F-NEXT:  LBB16_18: ## %else16
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $9, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $512, %eax ## imm = 0x200
 ; AVX512F-NEXT:    je LBB16_20
-; AVX512F-NEXT:  ## %bb.19: ## %cond.store17
+; AVX512F-NEXT:  LBB16_19: ## %cond.store17
 ; AVX512F-NEXT:    vpextrb $9, %xmm1, 9(%rdi)
-; AVX512F-NEXT:  LBB16_20: ## %else18
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $10, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $1024, %eax ## imm = 0x400
 ; AVX512F-NEXT:    je LBB16_22
-; AVX512F-NEXT:  ## %bb.21: ## %cond.store19
+; AVX512F-NEXT:  LBB16_21: ## %cond.store19
 ; AVX512F-NEXT:    vpextrb $10, %xmm1, 10(%rdi)
-; AVX512F-NEXT:  LBB16_22: ## %else20
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $11, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $2048, %eax ## imm = 0x800
 ; AVX512F-NEXT:    je LBB16_24
-; AVX512F-NEXT:  ## %bb.23: ## %cond.store21
+; AVX512F-NEXT:  LBB16_23: ## %cond.store21
 ; AVX512F-NEXT:    vpextrb $11, %xmm1, 11(%rdi)
-; AVX512F-NEXT:  LBB16_24: ## %else22
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $12, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $4096, %eax ## imm = 0x1000
 ; AVX512F-NEXT:    je LBB16_26
-; AVX512F-NEXT:  ## %bb.25: ## %cond.store23
+; AVX512F-NEXT:  LBB16_25: ## %cond.store23
 ; AVX512F-NEXT:    vpextrb $12, %xmm1, 12(%rdi)
-; AVX512F-NEXT:  LBB16_26: ## %else24
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $13, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $8192, %eax ## imm = 0x2000
 ; AVX512F-NEXT:    je LBB16_28
-; AVX512F-NEXT:  ## %bb.27: ## %cond.store25
-; AVX512F-NEXT:    vpextrb $13, %xmm1, 13(%rdi)
-; AVX512F-NEXT:  LBB16_28: ## %else26
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $14, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je LBB16_30
-; AVX512F-NEXT:  ## %bb.29: ## %cond.store27
-; AVX512F-NEXT:    vpextrb $14, %xmm1, 14(%rdi)
-; AVX512F-NEXT:  LBB16_30: ## %else28
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $15, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je LBB16_32
-; AVX512F-NEXT:  ## %bb.31: ## %cond.store29
-; AVX512F-NEXT:    vpextrb $15, %xmm1, 15(%rdi)
-; AVX512F-NEXT:  LBB16_32: ## %else30
-; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm2
-; AVX512F-NEXT:    vpxor %xmm0, %xmm0, %xmm0
-; AVX512F-NEXT:    vpcmpeqb %xmm0, %xmm2, %xmm0
-; AVX512F-NEXT:    vpmovsxbd %xmm0, %zmm0
-; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm0
-; AVX512F-NEXT:    je LBB16_34
-; AVX512F-NEXT:  ## %bb.33: ## %cond.store31
+; AVX512F-NEXT:  LBB16_27: ## %cond.store25
+; AVX512F-NEXT:    vpextrb $13, %xmm1, 13(%rdi)
+; AVX512F-NEXT:    testl $16384, %eax ## imm = 0x4000
+; AVX512F-NEXT:    je LBB16_30
+; AVX512F-NEXT:  LBB16_29: ## %cond.store27
+; AVX512F-NEXT:    vpextrb $14, %xmm1, 14(%rdi)
+; AVX512F-NEXT:    testl $32768, %eax ## imm = 0x8000
+; AVX512F-NEXT:    jne LBB16_31
+; AVX512F-NEXT:    jmp LBB16_32
+; AVX512F-NEXT:  LBB16_33: ## %cond.store31
 ; AVX512F-NEXT:    vpextrb $0, %xmm0, 16(%rdi)
-; AVX512F-NEXT:  LBB16_34: ## %else32
-; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $131072, %eax ## imm = 0x20000
 ; AVX512F-NEXT:    je LBB16_36
-; AVX512F-NEXT:  ## %bb.35: ## %cond.store33
+; AVX512F-NEXT:  LBB16_35: ## %cond.store33
 ; AVX512F-NEXT:    vpextrb $1, %xmm0, 17(%rdi)
-; AVX512F-NEXT:  LBB16_36: ## %else34
-; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $2, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $262144, %eax ## imm = 0x40000
 ; AVX512F-NEXT:    je LBB16_38
-; AVX512F-NEXT:  ## %bb.37: ## %cond.store35
+; AVX512F-NEXT:  LBB16_37: ## %cond.store35
 ; AVX512F-NEXT:    vpextrb $2, %xmm0, 18(%rdi)
-; AVX512F-NEXT:  LBB16_38: ## %else36
-; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $524288, %eax ## imm = 0x80000
 ; AVX512F-NEXT:    je LBB16_40
-; AVX512F-NEXT:  ## %bb.39: ## %cond.store37
+; AVX512F-NEXT:  LBB16_39: ## %cond.store37
 ; AVX512F-NEXT:    vpextrb $3, %xmm0, 19(%rdi)
-; AVX512F-NEXT:  LBB16_40: ## %else38
-; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $4, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $1048576, %eax ## imm = 0x100000
 ; AVX512F-NEXT:    je LBB16_42
-; AVX512F-NEXT:  ## %bb.41: ## %cond.store39
+; AVX512F-NEXT:  LBB16_41: ## %cond.store39
 ; AVX512F-NEXT:    vpextrb $4, %xmm0, 20(%rdi)
-; AVX512F-NEXT:  LBB16_42: ## %else40
-; AVX512F-NEXT:    kshiftrw $5, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $2097152, %eax ## imm = 0x200000
 ; AVX512F-NEXT:    je LBB16_44
-; AVX512F-NEXT:  ## %bb.43: ## %cond.store41
+; AVX512F-NEXT:  LBB16_43: ## %cond.store41
 ; AVX512F-NEXT:    vpextrb $5, %xmm0, 21(%rdi)
-; AVX512F-NEXT:  LBB16_44: ## %else42
-; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $6, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $4194304, %eax ## imm = 0x400000
 ; AVX512F-NEXT:    je LBB16_46
-; AVX512F-NEXT:  ## %bb.45: ## %cond.store43
+; AVX512F-NEXT:  LBB16_45: ## %cond.store43
 ; AVX512F-NEXT:    vpextrb $6, %xmm0, 22(%rdi)
-; AVX512F-NEXT:  LBB16_46: ## %else44
-; AVX512F-NEXT:    kshiftrw $7, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $8388608, %eax ## imm = 0x800000
 ; AVX512F-NEXT:    je LBB16_48
-; AVX512F-NEXT:  ## %bb.47: ## %cond.store45
+; AVX512F-NEXT:  LBB16_47: ## %cond.store45
 ; AVX512F-NEXT:    vpextrb $7, %xmm0, 23(%rdi)
-; AVX512F-NEXT:  LBB16_48: ## %else46
-; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $8, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $16777216, %eax ## imm = 0x1000000
 ; AVX512F-NEXT:    je LBB16_50
-; AVX512F-NEXT:  ## %bb.49: ## %cond.store47
+; AVX512F-NEXT:  LBB16_49: ## %cond.store47
 ; AVX512F-NEXT:    vpextrb $8, %xmm0, 24(%rdi)
-; AVX512F-NEXT:  LBB16_50: ## %else48
-; AVX512F-NEXT:    kshiftrw $9, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $33554432, %eax ## imm = 0x2000000
 ; AVX512F-NEXT:    je LBB16_52
-; AVX512F-NEXT:  ## %bb.51: ## %cond.store49
+; AVX512F-NEXT:  LBB16_51: ## %cond.store49
 ; AVX512F-NEXT:    vpextrb $9, %xmm0, 25(%rdi)
-; AVX512F-NEXT:  LBB16_52: ## %else50
-; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $10, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $67108864, %eax ## imm = 0x4000000
 ; AVX512F-NEXT:    je LBB16_54
-; AVX512F-NEXT:  ## %bb.53: ## %cond.store51
+; AVX512F-NEXT:  LBB16_53: ## %cond.store51
 ; AVX512F-NEXT:    vpextrb $10, %xmm0, 26(%rdi)
-; AVX512F-NEXT:  LBB16_54: ## %else52
-; AVX512F-NEXT:    kshiftrw $11, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $134217728, %eax ## imm = 0x8000000
 ; AVX512F-NEXT:    je LBB16_56
-; AVX512F-NEXT:  ## %bb.55: ## %cond.store53
+; AVX512F-NEXT:  LBB16_55: ## %cond.store53
 ; AVX512F-NEXT:    vpextrb $11, %xmm0, 27(%rdi)
-; AVX512F-NEXT:  LBB16_56: ## %else54
-; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $12, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $268435456, %eax ## imm = 0x10000000
 ; AVX512F-NEXT:    je LBB16_58
-; AVX512F-NEXT:  ## %bb.57: ## %cond.store55
+; AVX512F-NEXT:  LBB16_57: ## %cond.store55
 ; AVX512F-NEXT:    vpextrb $12, %xmm0, 28(%rdi)
-; AVX512F-NEXT:  LBB16_58: ## %else56
-; AVX512F-NEXT:    kshiftrw $13, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $536870912, %eax ## imm = 0x20000000
 ; AVX512F-NEXT:    je LBB16_60
-; AVX512F-NEXT:  ## %bb.59: ## %cond.store57
+; AVX512F-NEXT:  LBB16_59: ## %cond.store57
 ; AVX512F-NEXT:    vpextrb $13, %xmm0, 29(%rdi)
-; AVX512F-NEXT:  LBB16_60: ## %else58
-; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $14, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $1073741824, %eax ## imm = 0x40000000
 ; AVX512F-NEXT:    je LBB16_62
-; AVX512F-NEXT:  ## %bb.61: ## %cond.store59
+; AVX512F-NEXT:  LBB16_61: ## %cond.store59
 ; AVX512F-NEXT:    vpextrb $14, %xmm0, 30(%rdi)
-; AVX512F-NEXT:  LBB16_62: ## %else60
-; AVX512F-NEXT:    kshiftrw $15, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $-2147483648, %eax ## imm = 0x80000000
 ; AVX512F-NEXT:    je LBB16_64
-; AVX512F-NEXT:  ## %bb.63: ## %cond.store61
+; AVX512F-NEXT:  LBB16_63: ## %cond.store61
 ; AVX512F-NEXT:    vpextrb $15, %xmm0, 31(%rdi)
-; AVX512F-NEXT:  LBB16_64: ## %else62
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VLDQ-LABEL: store_v32i8_v32i8:
 ; AVX512VLDQ:       ## %bb.0:
 ; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm3
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm3, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
+; AVX512VLDQ-NEXT:    vpcmpeqb %ymm2, %ymm0, %ymm0
+; AVX512VLDQ-NEXT:    vpmovmskb %ymm0, %eax
 ; AVX512VLDQ-NEXT:    testb $1, %al
-; AVX512VLDQ-NEXT:    je LBB16_2
-; AVX512VLDQ-NEXT:  ## %bb.1: ## %cond.store
+; AVX512VLDQ-NEXT:    jne LBB16_1
+; AVX512VLDQ-NEXT:  ## %bb.2: ## %else
+; AVX512VLDQ-NEXT:    testb $2, %al
+; AVX512VLDQ-NEXT:    jne LBB16_3
+; AVX512VLDQ-NEXT:  LBB16_4: ## %else2
+; AVX512VLDQ-NEXT:    testb $4, %al
+; AVX512VLDQ-NEXT:    jne LBB16_5
+; AVX512VLDQ-NEXT:  LBB16_6: ## %else4
+; AVX512VLDQ-NEXT:    testb $8, %al
+; AVX512VLDQ-NEXT:    jne LBB16_7
+; AVX512VLDQ-NEXT:  LBB16_8: ## %else6
+; AVX512VLDQ-NEXT:    testb $16, %al
+; AVX512VLDQ-NEXT:    jne LBB16_9
+; AVX512VLDQ-NEXT:  LBB16_10: ## %else8
+; AVX512VLDQ-NEXT:    testb $32, %al
+; AVX512VLDQ-NEXT:    jne LBB16_11
+; AVX512VLDQ-NEXT:  LBB16_12: ## %else10
+; AVX512VLDQ-NEXT:    testb $64, %al
+; AVX512VLDQ-NEXT:    jne LBB16_13
+; AVX512VLDQ-NEXT:  LBB16_14: ## %else12
+; AVX512VLDQ-NEXT:    testb $-128, %al
+; AVX512VLDQ-NEXT:    jne LBB16_15
+; AVX512VLDQ-NEXT:  LBB16_16: ## %else14
+; AVX512VLDQ-NEXT:    testl $256, %eax ## imm = 0x100
+; AVX512VLDQ-NEXT:    jne LBB16_17
+; AVX512VLDQ-NEXT:  LBB16_18: ## %else16
+; AVX512VLDQ-NEXT:    testl $512, %eax ## imm = 0x200
+; AVX512VLDQ-NEXT:    jne LBB16_19
+; AVX512VLDQ-NEXT:  LBB16_20: ## %else18
+; AVX512VLDQ-NEXT:    testl $1024, %eax ## imm = 0x400
+; AVX512VLDQ-NEXT:    jne LBB16_21
+; AVX512VLDQ-NEXT:  LBB16_22: ## %else20
+; AVX512VLDQ-NEXT:    testl $2048, %eax ## imm = 0x800
+; AVX512VLDQ-NEXT:    jne LBB16_23
+; AVX512VLDQ-NEXT:  LBB16_24: ## %else22
+; AVX512VLDQ-NEXT:    testl $4096, %eax ## imm = 0x1000
+; AVX512VLDQ-NEXT:    jne LBB16_25
+; AVX512VLDQ-NEXT:  LBB16_26: ## %else24
+; AVX512VLDQ-NEXT:    testl $8192, %eax ## imm = 0x2000
+; AVX512VLDQ-NEXT:    jne LBB16_27
+; AVX512VLDQ-NEXT:  LBB16_28: ## %else26
+; AVX512VLDQ-NEXT:    testl $16384, %eax ## imm = 0x4000
+; AVX512VLDQ-NEXT:    jne LBB16_29
+; AVX512VLDQ-NEXT:  LBB16_30: ## %else28
+; AVX512VLDQ-NEXT:    testl $32768, %eax ## imm = 0x8000
+; AVX512VLDQ-NEXT:    je LBB16_32
+; AVX512VLDQ-NEXT:  LBB16_31: ## %cond.store29
+; AVX512VLDQ-NEXT:    vpextrb $15, %xmm1, 15(%rdi)
+; AVX512VLDQ-NEXT:  LBB16_32: ## %else30
+; AVX512VLDQ-NEXT:    testl $65536, %eax ## imm = 0x10000
+; AVX512VLDQ-NEXT:    vextracti128 $1, %ymm1, %xmm0
+; AVX512VLDQ-NEXT:    jne LBB16_33
+; AVX512VLDQ-NEXT:  ## %bb.34: ## %else32
+; AVX512VLDQ-NEXT:    testl $131072, %eax ## imm = 0x20000
+; AVX512VLDQ-NEXT:    jne LBB16_35
+; AVX512VLDQ-NEXT:  LBB16_36: ## %else34
+; AVX512VLDQ-NEXT:    testl $262144, %eax ## imm = 0x40000
+; AVX512VLDQ-NEXT:    jne LBB16_37
+; AVX512VLDQ-NEXT:  LBB16_38: ## %else36
+; AVX512VLDQ-NEXT:    testl $524288, %eax ## imm = 0x80000
+; AVX512VLDQ-NEXT:    jne LBB16_39
+; AVX512VLDQ-NEXT:  LBB16_40: ## %else38
+; AVX512VLDQ-NEXT:    testl $1048576, %eax ## imm = 0x100000
+; AVX512VLDQ-NEXT:    jne LBB16_41
+; AVX512VLDQ-NEXT:  LBB16_42: ## %else40
+; AVX512VLDQ-NEXT:    testl $2097152, %eax ## imm = 0x200000
+; AVX512VLDQ-NEXT:    jne LBB16_43
+; AVX512VLDQ-NEXT:  LBB16_44: ## %else42
+; AVX512VLDQ-NEXT:    testl $4194304, %eax ## imm = 0x400000
+; AVX512VLDQ-NEXT:    jne LBB16_45
+; AVX512VLDQ-NEXT:  LBB16_46: ## %else44
+; AVX512VLDQ-NEXT:    testl $8388608, %eax ## imm = 0x800000
+; AVX512VLDQ-NEXT:    jne LBB16_47
+; AVX512VLDQ-NEXT:  LBB16_48: ## %else46
+; AVX512VLDQ-NEXT:    testl $16777216, %eax ## imm = 0x1000000
+; AVX512VLDQ-NEXT:    jne LBB16_49
+; AVX512VLDQ-NEXT:  LBB16_50: ## %else48
+; AVX512VLDQ-NEXT:    testl $33554432, %eax ## imm = 0x2000000
+; AVX512VLDQ-NEXT:    jne LBB16_51
+; AVX512VLDQ-NEXT:  LBB16_52: ## %else50
+; AVX512VLDQ-NEXT:    testl $67108864, %eax ## imm = 0x4000000
+; AVX512VLDQ-NEXT:    jne LBB16_53
+; AVX512VLDQ-NEXT:  LBB16_54: ## %else52
+; AVX512VLDQ-NEXT:    testl $134217728, %eax ## imm = 0x8000000
+; AVX512VLDQ-NEXT:    jne LBB16_55
+; AVX512VLDQ-NEXT:  LBB16_56: ## %else54
+; AVX512VLDQ-NEXT:    testl $268435456, %eax ## imm = 0x10000000
+; AVX512VLDQ-NEXT:    jne LBB16_57
+; AVX512VLDQ-NEXT:  LBB16_58: ## %else56
+; AVX512VLDQ-NEXT:    testl $536870912, %eax ## imm = 0x20000000
+; AVX512VLDQ-NEXT:    jne LBB16_59
+; AVX512VLDQ-NEXT:  LBB16_60: ## %else58
+; AVX512VLDQ-NEXT:    testl $1073741824, %eax ## imm = 0x40000000
+; AVX512VLDQ-NEXT:    jne LBB16_61
+; AVX512VLDQ-NEXT:  LBB16_62: ## %else60
+; AVX512VLDQ-NEXT:    testl $-2147483648, %eax ## imm = 0x80000000
+; AVX512VLDQ-NEXT:    jne LBB16_63
+; AVX512VLDQ-NEXT:  LBB16_64: ## %else62
+; AVX512VLDQ-NEXT:    vzeroupper
+; AVX512VLDQ-NEXT:    retq
+; AVX512VLDQ-NEXT:  LBB16_1: ## %cond.store
 ; AVX512VLDQ-NEXT:    vpextrb $0, %xmm1, (%rdi)
-; AVX512VLDQ-NEXT:  LBB16_2: ## %else
-; AVX512VLDQ-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm2
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm2, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $1, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $2, %al
 ; AVX512VLDQ-NEXT:    je LBB16_4
-; AVX512VLDQ-NEXT:  ## %bb.3: ## %cond.store1
+; AVX512VLDQ-NEXT:  LBB16_3: ## %cond.store1
 ; AVX512VLDQ-NEXT:    vpextrb $1, %xmm1, 1(%rdi)
-; AVX512VLDQ-NEXT:  LBB16_4: ## %else2
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm3
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm3, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $2, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $4, %al
 ; AVX512VLDQ-NEXT:    je LBB16_6
-; AVX512VLDQ-NEXT:  ## %bb.5: ## %cond.store3
+; AVX512VLDQ-NEXT:  LBB16_5: ## %cond.store3
 ; AVX512VLDQ-NEXT:    vpextrb $2, %xmm1, 2(%rdi)
-; AVX512VLDQ-NEXT:  LBB16_6: ## %else4
-; AVX512VLDQ-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm2
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm2, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $3, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $8, %al
 ; AVX512VLDQ-NEXT:    je LBB16_8
-; AVX512VLDQ-NEXT:  ## %bb.7: ## %cond.store5
+; AVX512VLDQ-NEXT:  LBB16_7: ## %cond.store5
 ; AVX512VLDQ-NEXT:    vpextrb $3, %xmm1, 3(%rdi)
-; AVX512VLDQ-NEXT:  LBB16_8: ## %else6
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm3
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm3, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $4, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $16, %al
 ; AVX512VLDQ-NEXT:    je LBB16_10
-; AVX512VLDQ-NEXT:  ## %bb.9: ## %cond.store7
+; AVX512VLDQ-NEXT:  LBB16_9: ## %cond.store7
 ; AVX512VLDQ-NEXT:    vpextrb $4, %xmm1, 4(%rdi)
-; AVX512VLDQ-NEXT:  LBB16_10: ## %else8
-; AVX512VLDQ-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm2
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm2, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $5, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $32, %al
 ; AVX512VLDQ-NEXT:    je LBB16_12
-; AVX512VLDQ-NEXT:  ## %bb.11: ## %cond.store9
+; AVX512VLDQ-NEXT:  LBB16_11: ## %cond.store9
 ; AVX512VLDQ-NEXT:    vpextrb $5, %xmm1, 5(%rdi)
-; AVX512VLDQ-NEXT:  LBB16_12: ## %else10
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm3
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm3, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $6, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $64, %al
 ; AVX512VLDQ-NEXT:    je LBB16_14
-; AVX512VLDQ-NEXT:  ## %bb.13: ## %cond.store11
+; AVX512VLDQ-NEXT:  LBB16_13: ## %cond.store11
 ; AVX512VLDQ-NEXT:    vpextrb $6, %xmm1, 6(%rdi)
-; AVX512VLDQ-NEXT:  LBB16_14: ## %else12
-; AVX512VLDQ-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm2
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm2, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $7, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testb $-128, %al
 ; AVX512VLDQ-NEXT:    je LBB16_16
-; AVX512VLDQ-NEXT:  ## %bb.15: ## %cond.store13
+; AVX512VLDQ-NEXT:  LBB16_15: ## %cond.store13
 ; AVX512VLDQ-NEXT:    vpextrb $7, %xmm1, 7(%rdi)
-; AVX512VLDQ-NEXT:  LBB16_16: ## %else14
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm3
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm3, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $8, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testl $256, %eax ## imm = 0x100
 ; AVX512VLDQ-NEXT:    je LBB16_18
-; AVX512VLDQ-NEXT:  ## %bb.17: ## %cond.store15
+; AVX512VLDQ-NEXT:  LBB16_17: ## %cond.store15
 ; AVX512VLDQ-NEXT:    vpextrb $8, %xmm1, 8(%rdi)
-; AVX512VLDQ-NEXT:  LBB16_18: ## %else16
-; AVX512VLDQ-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm2
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm2, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $9, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testl $512, %eax ## imm = 0x200
 ; AVX512VLDQ-NEXT:    je LBB16_20
-; AVX512VLDQ-NEXT:  ## %bb.19: ## %cond.store17
+; AVX512VLDQ-NEXT:  LBB16_19: ## %cond.store17
 ; AVX512VLDQ-NEXT:    vpextrb $9, %xmm1, 9(%rdi)
-; AVX512VLDQ-NEXT:  LBB16_20: ## %else18
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm3
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm3, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $10, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testl $1024, %eax ## imm = 0x400
 ; AVX512VLDQ-NEXT:    je LBB16_22
-; AVX512VLDQ-NEXT:  ## %bb.21: ## %cond.store19
+; AVX512VLDQ-NEXT:  LBB16_21: ## %cond.store19
 ; AVX512VLDQ-NEXT:    vpextrb $10, %xmm1, 10(%rdi)
-; AVX512VLDQ-NEXT:  LBB16_22: ## %else20
-; AVX512VLDQ-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm2
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm2, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $11, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testl $2048, %eax ## imm = 0x800
 ; AVX512VLDQ-NEXT:    je LBB16_24
-; AVX512VLDQ-NEXT:  ## %bb.23: ## %cond.store21
+; AVX512VLDQ-NEXT:  LBB16_23: ## %cond.store21
 ; AVX512VLDQ-NEXT:    vpextrb $11, %xmm1, 11(%rdi)
-; AVX512VLDQ-NEXT:  LBB16_24: ## %else22
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm3
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm3, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $12, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testl $4096, %eax ## imm = 0x1000
 ; AVX512VLDQ-NEXT:    je LBB16_26
-; AVX512VLDQ-NEXT:  ## %bb.25: ## %cond.store23
+; AVX512VLDQ-NEXT:  LBB16_25: ## %cond.store23
 ; AVX512VLDQ-NEXT:    vpextrb $12, %xmm1, 12(%rdi)
-; AVX512VLDQ-NEXT:  LBB16_26: ## %else24
-; AVX512VLDQ-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm2
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm2, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $13, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testl $8192, %eax ## imm = 0x2000
 ; AVX512VLDQ-NEXT:    je LBB16_28
-; AVX512VLDQ-NEXT:  ## %bb.27: ## %cond.store25
+; AVX512VLDQ-NEXT:  LBB16_27: ## %cond.store25
 ; AVX512VLDQ-NEXT:    vpextrb $13, %xmm1, 13(%rdi)
-; AVX512VLDQ-NEXT:  LBB16_28: ## %else26
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm3
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm3, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $14, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testl $16384, %eax ## imm = 0x4000
 ; AVX512VLDQ-NEXT:    je LBB16_30
-; AVX512VLDQ-NEXT:  ## %bb.29: ## %cond.store27
+; AVX512VLDQ-NEXT:  LBB16_29: ## %cond.store27
 ; AVX512VLDQ-NEXT:    vpextrb $14, %xmm1, 14(%rdi)
-; AVX512VLDQ-NEXT:  LBB16_30: ## %else28
-; AVX512VLDQ-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm2
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm2, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $15, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
-; AVX512VLDQ-NEXT:    je LBB16_32
-; AVX512VLDQ-NEXT:  ## %bb.31: ## %cond.store29
-; AVX512VLDQ-NEXT:    vpextrb $15, %xmm1, 15(%rdi)
-; AVX512VLDQ-NEXT:  LBB16_32: ## %else30
-; AVX512VLDQ-NEXT:    vextracti128 $1, %ymm0, %xmm2
-; AVX512VLDQ-NEXT:    vpxor %xmm0, %xmm0, %xmm0
-; AVX512VLDQ-NEXT:    vpcmpeqb %xmm0, %xmm2, %xmm0
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm0, %zmm0
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
-; AVX512VLDQ-NEXT:    vextracti128 $1, %ymm1, %xmm0
-; AVX512VLDQ-NEXT:    je LBB16_34
-; AVX512VLDQ-NEXT:  ## %bb.33: ## %cond.store31
+; AVX512VLDQ-NEXT:    testl $32768, %eax ## imm = 0x8000
+; AVX512VLDQ-NEXT:    jne LBB16_31
+; AVX512VLDQ-NEXT:    jmp LBB16_32
+; AVX512VLDQ-NEXT:  LBB16_33: ## %cond.store31
 ; AVX512VLDQ-NEXT:    vpextrb $0, %xmm0, 16(%rdi)
-; AVX512VLDQ-NEXT:  LBB16_34: ## %else32
-; AVX512VLDQ-NEXT:    kshiftrw $1, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testl $131072, %eax ## imm = 0x20000
 ; AVX512VLDQ-NEXT:    je LBB16_36
-; AVX512VLDQ-NEXT:  ## %bb.35: ## %cond.store33
+; AVX512VLDQ-NEXT:  LBB16_35: ## %cond.store33
 ; AVX512VLDQ-NEXT:    vpextrb $1, %xmm0, 17(%rdi)
-; AVX512VLDQ-NEXT:  LBB16_36: ## %else34
-; AVX512VLDQ-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512VLDQ-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm1, %zmm1
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm1, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $2, %k0, %k1
-; AVX512VLDQ-NEXT:    kmovw %k1, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testl $262144, %eax ## imm = 0x40000
 ; AVX512VLDQ-NEXT:    je LBB16_38
-; AVX512VLDQ-NEXT:  ## %bb.37: ## %cond.store35
+; AVX512VLDQ-NEXT:  LBB16_37: ## %cond.store35
 ; AVX512VLDQ-NEXT:    vpextrb $2, %xmm0, 18(%rdi)
-; AVX512VLDQ-NEXT:  LBB16_38: ## %else36
-; AVX512VLDQ-NEXT:    kshiftrw $3, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testl $524288, %eax ## imm = 0x80000
 ; AVX512VLDQ-NEXT:    je LBB16_40
-; AVX512VLDQ-NEXT:  ## %bb.39: ## %cond.store37
+; AVX512VLDQ-NEXT:  LBB16_39: ## %cond.store37
 ; AVX512VLDQ-NEXT:    vpextrb $3, %xmm0, 19(%rdi)
-; AVX512VLDQ-NEXT:  LBB16_40: ## %else38
-; AVX512VLDQ-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512VLDQ-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm1, %zmm1
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm1, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $4, %k0, %k1
-; AVX512VLDQ-NEXT:    kmovw %k1, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testl $1048576, %eax ## imm = 0x100000
 ; AVX512VLDQ-NEXT:    je LBB16_42
-; AVX512VLDQ-NEXT:  ## %bb.41: ## %cond.store39
+; AVX512VLDQ-NEXT:  LBB16_41: ## %cond.store39
 ; AVX512VLDQ-NEXT:    vpextrb $4, %xmm0, 20(%rdi)
-; AVX512VLDQ-NEXT:  LBB16_42: ## %else40
-; AVX512VLDQ-NEXT:    kshiftrw $5, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testl $2097152, %eax ## imm = 0x200000
 ; AVX512VLDQ-NEXT:    je LBB16_44
-; AVX512VLDQ-NEXT:  ## %bb.43: ## %cond.store41
+; AVX512VLDQ-NEXT:  LBB16_43: ## %cond.store41
 ; AVX512VLDQ-NEXT:    vpextrb $5, %xmm0, 21(%rdi)
-; AVX512VLDQ-NEXT:  LBB16_44: ## %else42
-; AVX512VLDQ-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512VLDQ-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm1, %zmm1
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm1, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $6, %k0, %k1
-; AVX512VLDQ-NEXT:    kmovw %k1, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testl $4194304, %eax ## imm = 0x400000
 ; AVX512VLDQ-NEXT:    je LBB16_46
-; AVX512VLDQ-NEXT:  ## %bb.45: ## %cond.store43
+; AVX512VLDQ-NEXT:  LBB16_45: ## %cond.store43
 ; AVX512VLDQ-NEXT:    vpextrb $6, %xmm0, 22(%rdi)
-; AVX512VLDQ-NEXT:  LBB16_46: ## %else44
-; AVX512VLDQ-NEXT:    kshiftrw $7, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testl $8388608, %eax ## imm = 0x800000
 ; AVX512VLDQ-NEXT:    je LBB16_48
-; AVX512VLDQ-NEXT:  ## %bb.47: ## %cond.store45
+; AVX512VLDQ-NEXT:  LBB16_47: ## %cond.store45
 ; AVX512VLDQ-NEXT:    vpextrb $7, %xmm0, 23(%rdi)
-; AVX512VLDQ-NEXT:  LBB16_48: ## %else46
-; AVX512VLDQ-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512VLDQ-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm1, %zmm1
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm1, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $8, %k0, %k1
-; AVX512VLDQ-NEXT:    kmovw %k1, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testl $16777216, %eax ## imm = 0x1000000
 ; AVX512VLDQ-NEXT:    je LBB16_50
-; AVX512VLDQ-NEXT:  ## %bb.49: ## %cond.store47
+; AVX512VLDQ-NEXT:  LBB16_49: ## %cond.store47
 ; AVX512VLDQ-NEXT:    vpextrb $8, %xmm0, 24(%rdi)
-; AVX512VLDQ-NEXT:  LBB16_50: ## %else48
-; AVX512VLDQ-NEXT:    kshiftrw $9, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testl $33554432, %eax ## imm = 0x2000000
 ; AVX512VLDQ-NEXT:    je LBB16_52
-; AVX512VLDQ-NEXT:  ## %bb.51: ## %cond.store49
+; AVX512VLDQ-NEXT:  LBB16_51: ## %cond.store49
 ; AVX512VLDQ-NEXT:    vpextrb $9, %xmm0, 25(%rdi)
-; AVX512VLDQ-NEXT:  LBB16_52: ## %else50
-; AVX512VLDQ-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512VLDQ-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm1, %zmm1
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm1, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $10, %k0, %k1
-; AVX512VLDQ-NEXT:    kmovw %k1, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testl $67108864, %eax ## imm = 0x4000000
 ; AVX512VLDQ-NEXT:    je LBB16_54
-; AVX512VLDQ-NEXT:  ## %bb.53: ## %cond.store51
+; AVX512VLDQ-NEXT:  LBB16_53: ## %cond.store51
 ; AVX512VLDQ-NEXT:    vpextrb $10, %xmm0, 26(%rdi)
-; AVX512VLDQ-NEXT:  LBB16_54: ## %else52
-; AVX512VLDQ-NEXT:    kshiftrw $11, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testl $134217728, %eax ## imm = 0x8000000
 ; AVX512VLDQ-NEXT:    je LBB16_56
-; AVX512VLDQ-NEXT:  ## %bb.55: ## %cond.store53
+; AVX512VLDQ-NEXT:  LBB16_55: ## %cond.store53
 ; AVX512VLDQ-NEXT:    vpextrb $11, %xmm0, 27(%rdi)
-; AVX512VLDQ-NEXT:  LBB16_56: ## %else54
-; AVX512VLDQ-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512VLDQ-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm1, %zmm1
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm1, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $12, %k0, %k1
-; AVX512VLDQ-NEXT:    kmovw %k1, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testl $268435456, %eax ## imm = 0x10000000
 ; AVX512VLDQ-NEXT:    je LBB16_58
-; AVX512VLDQ-NEXT:  ## %bb.57: ## %cond.store55
+; AVX512VLDQ-NEXT:  LBB16_57: ## %cond.store55
 ; AVX512VLDQ-NEXT:    vpextrb $12, %xmm0, 28(%rdi)
-; AVX512VLDQ-NEXT:  LBB16_58: ## %else56
-; AVX512VLDQ-NEXT:    kshiftrw $13, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testl $536870912, %eax ## imm = 0x20000000
 ; AVX512VLDQ-NEXT:    je LBB16_60
-; AVX512VLDQ-NEXT:  ## %bb.59: ## %cond.store57
+; AVX512VLDQ-NEXT:  LBB16_59: ## %cond.store57
 ; AVX512VLDQ-NEXT:    vpextrb $13, %xmm0, 29(%rdi)
-; AVX512VLDQ-NEXT:  LBB16_60: ## %else58
-; AVX512VLDQ-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512VLDQ-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX512VLDQ-NEXT:    vpmovsxbd %xmm1, %zmm1
-; AVX512VLDQ-NEXT:    vpmovd2m %zmm1, %k0
-; AVX512VLDQ-NEXT:    kshiftrw $14, %k0, %k1
-; AVX512VLDQ-NEXT:    kmovw %k1, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testl $1073741824, %eax ## imm = 0x40000000
 ; AVX512VLDQ-NEXT:    je LBB16_62
-; AVX512VLDQ-NEXT:  ## %bb.61: ## %cond.store59
+; AVX512VLDQ-NEXT:  LBB16_61: ## %cond.store59
 ; AVX512VLDQ-NEXT:    vpextrb $14, %xmm0, 30(%rdi)
-; AVX512VLDQ-NEXT:  LBB16_62: ## %else60
-; AVX512VLDQ-NEXT:    kshiftrw $15, %k0, %k0
-; AVX512VLDQ-NEXT:    kmovw %k0, %eax
-; AVX512VLDQ-NEXT:    testb $1, %al
+; AVX512VLDQ-NEXT:    testl $-2147483648, %eax ## imm = 0x80000000
 ; AVX512VLDQ-NEXT:    je LBB16_64
-; AVX512VLDQ-NEXT:  ## %bb.63: ## %cond.store61
+; AVX512VLDQ-NEXT:  LBB16_63: ## %cond.store61
 ; AVX512VLDQ-NEXT:    vpextrb $15, %xmm0, 31(%rdi)
-; AVX512VLDQ-NEXT:  LBB16_64: ## %else62
 ; AVX512VLDQ-NEXT:    vzeroupper
 ; AVX512VLDQ-NEXT:    retq
 ;
@@ -5060,61 +4637,38 @@ define void @one_mask_bit_set5(<8 x double>* %addr, <8 x double> %val) {
 ; SimplifyDemandedBits eliminates an ashr here.
 
 define void @masked_store_bool_mask_demand_trunc_sext(<4 x double> %x, <4 x double>* %p, <4 x i32> %masksrc) {
-; SSE2-LABEL: masked_store_bool_mask_demand_trunc_sext:
-; SSE2:       ## %bb.0:
-; SSE2-NEXT:    movd %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB23_2
-; SSE2-NEXT:  ## %bb.1: ## %cond.store
-; SSE2-NEXT:    movlps %xmm0, (%rdi)
-; SSE2-NEXT:  LBB23_2: ## %else
-; SSE2-NEXT:    pextrw $2, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB23_4
-; SSE2-NEXT:  ## %bb.3: ## %cond.store1
-; SSE2-NEXT:    movhps %xmm0, 8(%rdi)
-; SSE2-NEXT:  LBB23_4: ## %else2
-; SSE2-NEXT:    pextrw $4, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB23_6
-; SSE2-NEXT:  ## %bb.5: ## %cond.store3
-; SSE2-NEXT:    movlps %xmm1, 16(%rdi)
-; SSE2-NEXT:  LBB23_6: ## %else4
-; SSE2-NEXT:    pextrw $6, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB23_8
-; SSE2-NEXT:  ## %bb.7: ## %cond.store5
-; SSE2-NEXT:    movhps %xmm1, 24(%rdi)
-; SSE2-NEXT:  LBB23_8: ## %else6
-; SSE2-NEXT:    retq
-;
-; SSE4-LABEL: masked_store_bool_mask_demand_trunc_sext:
-; SSE4:       ## %bb.0:
-; SSE4-NEXT:    pextrb $0, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je LBB23_2
-; SSE4-NEXT:  ## %bb.1: ## %cond.store
-; SSE4-NEXT:    movlps %xmm0, (%rdi)
-; SSE4-NEXT:  LBB23_2: ## %else
-; SSE4-NEXT:    pextrb $4, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je LBB23_4
-; SSE4-NEXT:  ## %bb.3: ## %cond.store1
-; SSE4-NEXT:    movhps %xmm0, 8(%rdi)
-; SSE4-NEXT:  LBB23_4: ## %else2
-; SSE4-NEXT:    pextrb $8, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je LBB23_6
-; SSE4-NEXT:  ## %bb.5: ## %cond.store3
-; SSE4-NEXT:    movlps %xmm1, 16(%rdi)
-; SSE4-NEXT:  LBB23_6: ## %else4
-; SSE4-NEXT:    pextrb $12, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je LBB23_8
-; SSE4-NEXT:  ## %bb.7: ## %cond.store5
-; SSE4-NEXT:    movhps %xmm1, 24(%rdi)
-; SSE4-NEXT:  LBB23_8: ## %else6
-; SSE4-NEXT:    retq
+; SSE-LABEL: masked_store_bool_mask_demand_trunc_sext:
+; SSE:       ## %bb.0:
+; SSE-NEXT:    pslld $31, %xmm2
+; SSE-NEXT:    movmskps %xmm2, %eax
+; SSE-NEXT:    testb $1, %al
+; SSE-NEXT:    jne LBB23_1
+; SSE-NEXT:  ## %bb.2: ## %else
+; SSE-NEXT:    testb $2, %al
+; SSE-NEXT:    jne LBB23_3
+; SSE-NEXT:  LBB23_4: ## %else2
+; SSE-NEXT:    testb $4, %al
+; SSE-NEXT:    jne LBB23_5
+; SSE-NEXT:  LBB23_6: ## %else4
+; SSE-NEXT:    testb $8, %al
+; SSE-NEXT:    jne LBB23_7
+; SSE-NEXT:  LBB23_8: ## %else6
+; SSE-NEXT:    retq
+; SSE-NEXT:  LBB23_1: ## %cond.store
+; SSE-NEXT:    movlps %xmm0, (%rdi)
+; SSE-NEXT:    testb $2, %al
+; SSE-NEXT:    je LBB23_4
+; SSE-NEXT:  LBB23_3: ## %cond.store1
+; SSE-NEXT:    movhps %xmm0, 8(%rdi)
+; SSE-NEXT:    testb $4, %al
+; SSE-NEXT:    je LBB23_6
+; SSE-NEXT:  LBB23_5: ## %cond.store3
+; SSE-NEXT:    movlps %xmm1, 16(%rdi)
+; SSE-NEXT:    testb $8, %al
+; SSE-NEXT:    je LBB23_8
+; SSE-NEXT:  LBB23_7: ## %cond.store5
+; SSE-NEXT:    movhps %xmm1, 24(%rdi)
+; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: masked_store_bool_mask_demand_trunc_sext:
 ; AVX1:       ## %bb.0:
@@ -5172,85 +4726,71 @@ define void @masked_store_bool_mask_demand_trunc_sext(<4 x double> %x, <4 x doub
 define void @one_mask_bit_set1_variable(<4 x float>* %addr, <4 x float> %val, <4 x i32> %mask) {
 ; SSE2-LABEL: one_mask_bit_set1_variable:
 ; SSE2:       ## %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm2
-; SSE2-NEXT:    movd %xmm2, %eax
+; SSE2-NEXT:    movmskps %xmm1, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je LBB24_2
-; SSE2-NEXT:  ## %bb.1: ## %cond.store
+; SSE2-NEXT:    jne LBB24_1
+; SSE2-NEXT:  ## %bb.2: ## %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne LBB24_3
+; SSE2-NEXT:  LBB24_4: ## %else2
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne LBB24_5
+; SSE2-NEXT:  LBB24_6: ## %else4
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne LBB24_7
+; SSE2-NEXT:  LBB24_8: ## %else6
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  LBB24_1: ## %cond.store
 ; SSE2-NEXT:    movss %xmm0, (%rdi)
-; SSE2-NEXT:  LBB24_2: ## %else
-; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm3
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT:    pxor %xmm3, %xmm2
-; SSE2-NEXT:    pextrw $2, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $2, %al
 ; SSE2-NEXT:    je LBB24_4
-; SSE2-NEXT:  ## %bb.3: ## %cond.store1
-; SSE2-NEXT:    movaps %xmm0, %xmm3
-; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm0[2,3]
-; SSE2-NEXT:    movss %xmm3, 4(%rdi)
-; SSE2-NEXT:  LBB24_4: ## %else2
-; SSE2-NEXT:    pextrw $4, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:  LBB24_3: ## %cond.store1
+; SSE2-NEXT:    movaps %xmm0, %xmm1
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[2,3]
+; SSE2-NEXT:    movss %xmm1, 4(%rdi)
+; SSE2-NEXT:    testb $4, %al
 ; SSE2-NEXT:    je LBB24_6
-; SSE2-NEXT:  ## %bb.5: ## %cond.store3
-; SSE2-NEXT:    movaps %xmm0, %xmm2
-; SSE2-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm0[1]
-; SSE2-NEXT:    movss %xmm2, 8(%rdi)
-; SSE2-NEXT:  LBB24_6: ## %else4
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT:    pxor %xmm1, %xmm2
-; SSE2-NEXT:    pextrw $6, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:  LBB24_5: ## %cond.store3
+; SSE2-NEXT:    movaps %xmm0, %xmm1
+; SSE2-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
+; SSE2-NEXT:    movss %xmm1, 8(%rdi)
+; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je LBB24_8
-; SSE2-NEXT:  ## %bb.7: ## %cond.store5
+; SSE2-NEXT:  LBB24_7: ## %cond.store5
 ; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
 ; SSE2-NEXT:    movss %xmm0, 12(%rdi)
-; SSE2-NEXT:  LBB24_8: ## %else6
 ; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: one_mask_bit_set1_variable:
 ; SSE4:       ## %bb.0:
-; SSE4-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648]
-; SSE4-NEXT:    pand %xmm2, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm2
-; SSE4-NEXT:    pextrb $0, %xmm2, %eax
+; SSE4-NEXT:    movmskps %xmm1, %eax
 ; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je LBB24_2
-; SSE4-NEXT:  ## %bb.1: ## %cond.store
+; SSE4-NEXT:    jne LBB24_1
+; SSE4-NEXT:  ## %bb.2: ## %else
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    jne LBB24_3
+; SSE4-NEXT:  LBB24_4: ## %else2
+; SSE4-NEXT:    testb $4, %al
+; SSE4-NEXT:    jne LBB24_5
+; SSE4-NEXT:  LBB24_6: ## %else4
+; SSE4-NEXT:    testb $8, %al
+; SSE4-NEXT:    jne LBB24_7
+; SSE4-NEXT:  LBB24_8: ## %else6
+; SSE4-NEXT:    retq
+; SSE4-NEXT:  LBB24_1: ## %cond.store
 ; SSE4-NEXT:    movss %xmm0, (%rdi)
-; SSE4-NEXT:  LBB24_2: ## %else
-; SSE4-NEXT:    pxor %xmm3, %xmm3
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm3
-; SSE4-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE4-NEXT:    pxor %xmm3, %xmm2
-; SSE4-NEXT:    pextrb $4, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $2, %al
 ; SSE4-NEXT:    je LBB24_4
-; SSE4-NEXT:  ## %bb.3: ## %cond.store1
+; SSE4-NEXT:  LBB24_3: ## %cond.store1
 ; SSE4-NEXT:    extractps $1, %xmm0, 4(%rdi)
-; SSE4-NEXT:  LBB24_4: ## %else2
-; SSE4-NEXT:    pextrb $8, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $4, %al
 ; SSE4-NEXT:    je LBB24_6
-; SSE4-NEXT:  ## %bb.5: ## %cond.store3
+; SSE4-NEXT:  LBB24_5: ## %cond.store3
 ; SSE4-NEXT:    extractps $2, %xmm0, 8(%rdi)
-; SSE4-NEXT:  LBB24_6: ## %else4
-; SSE4-NEXT:    pxor %xmm2, %xmm2
-; SSE4-NEXT:    pcmpeqd %xmm2, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE4-NEXT:    pxor %xmm1, %xmm2
-; SSE4-NEXT:    pextrb $12, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $8, %al
 ; SSE4-NEXT:    je LBB24_8
-; SSE4-NEXT:  ## %bb.7: ## %cond.store5
+; SSE4-NEXT:  LBB24_7: ## %cond.store5
 ; SSE4-NEXT:    extractps $3, %xmm0, 12(%rdi)
-; SSE4-NEXT:  LBB24_8: ## %else6
 ; SSE4-NEXT:    retq
 ;
 ; AVX1OR2-LABEL: one_mask_bit_set1_variable:
@@ -5286,24 +4826,31 @@ define void @one_mask_bit_set1_variable(<4 x float>* %addr, <4 x float> %val, <4
 define void @widen_masked_store(<3 x i32> %v, <3 x i32>* %p, <3 x i1> %mask) {
 ; SSE2-LABEL: widen_masked_store:
 ; SSE2:       ## %bb.0:
-; SSE2-NEXT:    testb $1, %sil
+; SSE2-NEXT:    andb $1, %sil
+; SSE2-NEXT:    andb $1, %dl
+; SSE2-NEXT:    addb %dl, %dl
+; SSE2-NEXT:    orb %sil, %dl
+; SSE2-NEXT:    andb $1, %cl
+; SSE2-NEXT:    shlb $2, %cl
+; SSE2-NEXT:    orb %dl, %cl
+; SSE2-NEXT:    testb $1, %cl
 ; SSE2-NEXT:    jne LBB25_1
 ; SSE2-NEXT:  ## %bb.2: ## %else
-; SSE2-NEXT:    testb $1, %dl
+; SSE2-NEXT:    testb $2, %cl
 ; SSE2-NEXT:    jne LBB25_3
 ; SSE2-NEXT:  LBB25_4: ## %else2
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testb $4, %cl
 ; SSE2-NEXT:    jne LBB25_5
 ; SSE2-NEXT:  LBB25_6: ## %else4
 ; SSE2-NEXT:    retq
 ; SSE2-NEXT:  LBB25_1: ## %cond.store
 ; SSE2-NEXT:    movd %xmm0, (%rdi)
-; SSE2-NEXT:    testb $1, %dl
+; SSE2-NEXT:    testb $2, %cl
 ; SSE2-NEXT:    je LBB25_4
 ; SSE2-NEXT:  LBB25_3: ## %cond.store1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE2-NEXT:    movd %xmm1, 4(%rdi)
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testb $4, %cl
 ; SSE2-NEXT:    je LBB25_6
 ; SSE2-NEXT:  LBB25_5: ## %cond.store3
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
@@ -5312,23 +4859,30 @@ define void @widen_masked_store(<3 x i32> %v, <3 x i32>* %p, <3 x i1> %mask) {
 ;
 ; SSE4-LABEL: widen_masked_store:
 ; SSE4:       ## %bb.0:
-; SSE4-NEXT:    testb $1, %sil
+; SSE4-NEXT:    andb $1, %sil
+; SSE4-NEXT:    andb $1, %dl
+; SSE4-NEXT:    addb %dl, %dl
+; SSE4-NEXT:    orb %sil, %dl
+; SSE4-NEXT:    andb $1, %cl
+; SSE4-NEXT:    shlb $2, %cl
+; SSE4-NEXT:    orb %dl, %cl
+; SSE4-NEXT:    testb $1, %cl
 ; SSE4-NEXT:    jne LBB25_1
 ; SSE4-NEXT:  ## %bb.2: ## %else
-; SSE4-NEXT:    testb $1, %dl
+; SSE4-NEXT:    testb $2, %cl
 ; SSE4-NEXT:    jne LBB25_3
 ; SSE4-NEXT:  LBB25_4: ## %else2
-; SSE4-NEXT:    testb $1, %cl
+; SSE4-NEXT:    testb $4, %cl
 ; SSE4-NEXT:    jne LBB25_5
 ; SSE4-NEXT:  LBB25_6: ## %else4
 ; SSE4-NEXT:    retq
 ; SSE4-NEXT:  LBB25_1: ## %cond.store
 ; SSE4-NEXT:    movss %xmm0, (%rdi)
-; SSE4-NEXT:    testb $1, %dl
+; SSE4-NEXT:    testb $2, %cl
 ; SSE4-NEXT:    je LBB25_4
 ; SSE4-NEXT:  LBB25_3: ## %cond.store1
 ; SSE4-NEXT:    extractps $1, %xmm0, 4(%rdi)
-; SSE4-NEXT:    testb $1, %cl
+; SSE4-NEXT:    testb $4, %cl
 ; SSE4-NEXT:    je LBB25_6
 ; SSE4-NEXT:  LBB25_5: ## %cond.store3
 ; SSE4-NEXT:    extractps $2, %xmm0, 8(%rdi)
@@ -5359,12 +4913,22 @@ define void @widen_masked_store(<3 x i32> %v, <3 x i32>* %p, <3 x i1> %mask) {
 ; AVX512F-LABEL: widen_masked_store:
 ; AVX512F:       ## %bb.0:
 ; AVX512F-NEXT:    ## kill: def $xmm0 killed $xmm0 def $zmm0
-; AVX512F-NEXT:    vpslld $31, %xmm1, %xmm1
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k1
-; AVX512F-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k1} {z}
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0,1,2],xmm2[3]
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    movw $-3, %ax
+; AVX512F-NEXT:    kmovw %eax, %k0
+; AVX512F-NEXT:    andl $1, %esi
+; AVX512F-NEXT:    kmovw %esi, %k1
+; AVX512F-NEXT:    kandw %k0, %k1, %k0
+; AVX512F-NEXT:    kmovw %edx, %k1
+; AVX512F-NEXT:    kshiftlw $15, %k1, %k1
+; AVX512F-NEXT:    kshiftrw $14, %k1, %k1
+; AVX512F-NEXT:    korw %k1, %k0, %k0
+; AVX512F-NEXT:    movw $-5, %ax
+; AVX512F-NEXT:    kmovw %eax, %k1
+; AVX512F-NEXT:    kandw %k1, %k0, %k0
+; AVX512F-NEXT:    kmovw %ecx, %k1
+; AVX512F-NEXT:    kshiftlw $15, %k1, %k1
+; AVX512F-NEXT:    kshiftrw $13, %k1, %k1
+; AVX512F-NEXT:    korw %k1, %k0, %k0
 ; AVX512F-NEXT:    kshiftlw $12, %k0, %k0
 ; AVX512F-NEXT:    kshiftrw $12, %k0, %k1
 ; AVX512F-NEXT:    vmovdqu32 %zmm0, (%rdi) {%k1}
@@ -5373,24 +4937,44 @@ define void @widen_masked_store(<3 x i32> %v, <3 x i32>* %p, <3 x i1> %mask) {
 ;
 ; AVX512VLDQ-LABEL: widen_masked_store:
 ; AVX512VLDQ:       ## %bb.0:
-; AVX512VLDQ-NEXT:    vpslld $31, %xmm1, %xmm1
-; AVX512VLDQ-NEXT:    vpmovd2m %xmm1, %k0
-; AVX512VLDQ-NEXT:    vpmovm2d %k0, %xmm1
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0,1,2],xmm2[3]
-; AVX512VLDQ-NEXT:    vpmovd2m %xmm1, %k1
+; AVX512VLDQ-NEXT:    movb $-3, %al
+; AVX512VLDQ-NEXT:    kmovw %eax, %k0
+; AVX512VLDQ-NEXT:    kmovw %esi, %k1
+; AVX512VLDQ-NEXT:    kshiftlb $7, %k1, %k1
+; AVX512VLDQ-NEXT:    kshiftrb $7, %k1, %k1
+; AVX512VLDQ-NEXT:    kandw %k0, %k1, %k0
+; AVX512VLDQ-NEXT:    kmovw %edx, %k1
+; AVX512VLDQ-NEXT:    kshiftlb $7, %k1, %k1
+; AVX512VLDQ-NEXT:    kshiftrb $6, %k1, %k1
+; AVX512VLDQ-NEXT:    korw %k1, %k0, %k0
+; AVX512VLDQ-NEXT:    movb $-5, %al
+; AVX512VLDQ-NEXT:    kmovw %eax, %k1
+; AVX512VLDQ-NEXT:    kandw %k1, %k0, %k0
+; AVX512VLDQ-NEXT:    kmovw %ecx, %k1
+; AVX512VLDQ-NEXT:    kshiftlb $7, %k1, %k1
+; AVX512VLDQ-NEXT:    kshiftrb $5, %k1, %k1
+; AVX512VLDQ-NEXT:    korw %k1, %k0, %k1
 ; AVX512VLDQ-NEXT:    vmovdqa32 %xmm0, (%rdi) {%k1}
 ; AVX512VLDQ-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: widen_masked_store:
 ; AVX512VLBW:       ## %bb.0:
-; AVX512VLBW-NEXT:    vpslld $31, %xmm1, %xmm1
-; AVX512VLBW-NEXT:    vptestmd %xmm1, %xmm1, %k1
-; AVX512VLBW-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
-; AVX512VLBW-NEXT:    vmovdqa32 %xmm1, %xmm1 {%k1} {z}
-; AVX512VLBW-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLBW-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0,1,2],xmm2[3]
-; AVX512VLBW-NEXT:    vptestmd %xmm1, %xmm1, %k1
+; AVX512VLBW-NEXT:    movw $-3, %ax
+; AVX512VLBW-NEXT:    kmovd %eax, %k0
+; AVX512VLBW-NEXT:    andl $1, %esi
+; AVX512VLBW-NEXT:    kmovw %esi, %k1
+; AVX512VLBW-NEXT:    kandw %k0, %k1, %k0
+; AVX512VLBW-NEXT:    kmovd %edx, %k1
+; AVX512VLBW-NEXT:    kshiftlw $15, %k1, %k1
+; AVX512VLBW-NEXT:    kshiftrw $14, %k1, %k1
+; AVX512VLBW-NEXT:    korw %k1, %k0, %k0
+; AVX512VLBW-NEXT:    movw $-5, %ax
+; AVX512VLBW-NEXT:    kmovd %eax, %k1
+; AVX512VLBW-NEXT:    kandw %k1, %k0, %k0
+; AVX512VLBW-NEXT:    kmovd %ecx, %k1
+; AVX512VLBW-NEXT:    kshiftlw $15, %k1, %k1
+; AVX512VLBW-NEXT:    kshiftrw $13, %k1, %k1
+; AVX512VLBW-NEXT:    korw %k1, %k0, %k1
 ; AVX512VLBW-NEXT:    vmovdqa32 %xmm0, (%rdi) {%k1}
 ; AVX512VLBW-NEXT:    retq
   call void @llvm.masked.store.v3i32.p0v3i32(<3 x i32> %v, <3 x i32>* %p, i32 16, <3 x i1> %mask)
diff --git a/llvm/test/CodeGen/X86/masked_store_trunc.ll b/llvm/test/CodeGen/X86/masked_store_trunc.ll
index 8e73194bf..5b2bf3b17 100644
--- a/llvm/test/CodeGen/X86/masked_store_trunc.ll
+++ b/llvm/test/CodeGen/X86/masked_store_trunc.ll
@@ -11,152 +11,146 @@
 define void @truncstore_v8i64_v8i32(<8 x i64> %x, <8 x i32>* %p, <8 x i32> %mask) {
 ; SSE2-LABEL: truncstore_v8i64_v8i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pxor %xmm7, %xmm7
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm7
-; SSE2-NEXT:    pcmpeqd %xmm6, %xmm6
-; SSE2-NEXT:    pxor %xmm7, %xmm6
-; SSE2-NEXT:    movdqa %xmm6, %xmm7
-; SSE2-NEXT:    packssdw %xmm0, %xmm7
+; SSE2-NEXT:    pxor %xmm6, %xmm6
 ; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
-; SSE2-NEXT:    movd %xmm7, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB0_2
-; SSE2-NEXT:  # %bb.1: # %cond.store
-; SSE2-NEXT:    movss %xmm0, (%rdi)
-; SSE2-NEXT:  .LBB0_2: # %else
-; SSE2-NEXT:    psrlq $16, %xmm6
-; SSE2-NEXT:    movd %xmm6, %eax
-; SSE2-NEXT:    shrl $16, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB0_4
-; SSE2-NEXT:  # %bb.3: # %cond.store1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; SSE2-NEXT:    movd %xmm1, 4(%rdi)
-; SSE2-NEXT:  .LBB0_4: # %else2
-; SSE2-NEXT:    xorps %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm4
+; SSE2-NEXT:    pcmpeqd %xmm6, %xmm5
 ; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pxor %xmm4, %xmm1
-; SSE2-NEXT:    pextrw $4, %xmm1, %eax
+; SSE2-NEXT:    pxor %xmm1, %xmm5
+; SSE2-NEXT:    pcmpeqd %xmm6, %xmm4
+; SSE2-NEXT:    pxor %xmm1, %xmm4
+; SSE2-NEXT:    packssdw %xmm5, %xmm4
+; SSE2-NEXT:    packsswb %xmm0, %xmm4
+; SSE2-NEXT:    pmovmskb %xmm4, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB0_6
-; SSE2-NEXT:  # %bb.5: # %cond.store3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[2,3,0,1]
-; SSE2-NEXT:    movd %xmm4, 8(%rdi)
+; SSE2-NEXT:    jne .LBB0_1
+; SSE2-NEXT:  # %bb.2: # %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne .LBB0_3
+; SSE2-NEXT:  .LBB0_4: # %else2
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne .LBB0_5
 ; SSE2-NEXT:  .LBB0_6: # %else4
-; SSE2-NEXT:    pextrw $6, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je .LBB0_8
-; SSE2-NEXT:  # %bb.7: # %cond.store5
+; SSE2-NEXT:  .LBB0_7: # %cond.store5
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
 ; SSE2-NEXT:    movd %xmm0, 12(%rdi)
 ; SSE2-NEXT:  .LBB0_8: # %else6
 ; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,2],xmm3[0,2]
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE2-NEXT:    pxor %xmm1, %xmm0
-; SSE2-NEXT:    pextrw $0, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB0_10
-; SSE2-NEXT:  # %bb.9: # %cond.store7
+; SSE2-NEXT:    testb $16, %al
+; SSE2-NEXT:    jne .LBB0_9
+; SSE2-NEXT:  # %bb.10: # %else8
+; SSE2-NEXT:    testb $32, %al
+; SSE2-NEXT:    jne .LBB0_11
+; SSE2-NEXT:  .LBB0_12: # %else10
+; SSE2-NEXT:    testb $64, %al
+; SSE2-NEXT:    jne .LBB0_13
+; SSE2-NEXT:  .LBB0_14: # %else12
+; SSE2-NEXT:    testb $-128, %al
+; SSE2-NEXT:    jne .LBB0_15
+; SSE2-NEXT:  .LBB0_16: # %else14
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  .LBB0_1: # %cond.store
+; SSE2-NEXT:    movd %xmm0, (%rdi)
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    je .LBB0_4
+; SSE2-NEXT:  .LBB0_3: # %cond.store1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE2-NEXT:    movd %xmm1, 4(%rdi)
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    je .LBB0_6
+; SSE2-NEXT:  .LBB0_5: # %cond.store3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    movd %xmm1, 8(%rdi)
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne .LBB0_7
+; SSE2-NEXT:    jmp .LBB0_8
+; SSE2-NEXT:  .LBB0_9: # %cond.store7
 ; SSE2-NEXT:    movss %xmm2, 16(%rdi)
-; SSE2-NEXT:  .LBB0_10: # %else8
-; SSE2-NEXT:    pextrw $2, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $32, %al
 ; SSE2-NEXT:    je .LBB0_12
-; SSE2-NEXT:  # %bb.11: # %cond.store9
+; SSE2-NEXT:  .LBB0_11: # %cond.store9
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
 ; SSE2-NEXT:    movd %xmm0, 20(%rdi)
-; SSE2-NEXT:  .LBB0_12: # %else10
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm5
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE2-NEXT:    pxor %xmm5, %xmm0
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $64, %al
 ; SSE2-NEXT:    je .LBB0_14
-; SSE2-NEXT:  # %bb.13: # %cond.store11
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,3,0,1]
-; SSE2-NEXT:    movd %xmm1, 24(%rdi)
-; SSE2-NEXT:  .LBB0_14: # %else12
-; SSE2-NEXT:    pextrw $6, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:  .LBB0_13: # %cond.store11
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE2-NEXT:    movd %xmm0, 24(%rdi)
+; SSE2-NEXT:    testb $-128, %al
 ; SSE2-NEXT:    je .LBB0_16
-; SSE2-NEXT:  # %bb.15: # %cond.store13
+; SSE2-NEXT:  .LBB0_15: # %cond.store13
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[3,1,2,3]
 ; SSE2-NEXT:    movd %xmm0, 28(%rdi)
-; SSE2-NEXT:  .LBB0_16: # %else14
 ; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: truncstore_v8i64_v8i32:
 ; SSE4:       # %bb.0:
-; SSE4-NEXT:    pxor %xmm7, %xmm7
-; SSE4-NEXT:    pcmpeqd %xmm4, %xmm7
-; SSE4-NEXT:    pcmpeqd %xmm6, %xmm6
-; SSE4-NEXT:    pxor %xmm7, %xmm6
+; SSE4-NEXT:    pxor %xmm6, %xmm6
 ; SSE4-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
-; SSE4-NEXT:    pextrb $0, %xmm6, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB0_2
-; SSE4-NEXT:  # %bb.1: # %cond.store
-; SSE4-NEXT:    movss %xmm0, (%rdi)
-; SSE4-NEXT:  .LBB0_2: # %else
-; SSE4-NEXT:    pextrb $4, %xmm6, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB0_4
-; SSE4-NEXT:  # %bb.3: # %cond.store1
-; SSE4-NEXT:    extractps $1, %xmm0, 4(%rdi)
-; SSE4-NEXT:  .LBB0_4: # %else2
-; SSE4-NEXT:    xorps %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm4
+; SSE4-NEXT:    pcmpeqd %xmm6, %xmm5
 ; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE4-NEXT:    pxor %xmm4, %xmm1
-; SSE4-NEXT:    pextrb $8, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB0_6
-; SSE4-NEXT:  # %bb.5: # %cond.store3
-; SSE4-NEXT:    extractps $2, %xmm0, 8(%rdi)
+; SSE4-NEXT:    pxor %xmm1, %xmm5
+; SSE4-NEXT:    pcmpeqd %xmm6, %xmm4
+; SSE4-NEXT:    pxor %xmm1, %xmm4
+; SSE4-NEXT:    packssdw %xmm5, %xmm4
+; SSE4-NEXT:    packsswb %xmm0, %xmm4
+; SSE4-NEXT:    pmovmskb %xmm4, %eax
+; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    jne .LBB0_1
+; SSE4-NEXT:  # %bb.2: # %else
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    jne .LBB0_3
+; SSE4-NEXT:  .LBB0_4: # %else2
+; SSE4-NEXT:    testb $4, %al
+; SSE4-NEXT:    jne .LBB0_5
 ; SSE4-NEXT:  .LBB0_6: # %else4
-; SSE4-NEXT:    pextrb $12, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $8, %al
 ; SSE4-NEXT:    je .LBB0_8
-; SSE4-NEXT:  # %bb.7: # %cond.store5
-; SSE4-NEXT:    extractps $3, %xmm0, 12(%rdi)
+; SSE4-NEXT:  .LBB0_7: # %cond.store5
+; SSE4-NEXT:    pextrd $3, %xmm0, 12(%rdi)
 ; SSE4-NEXT:  .LBB0_8: # %else6
 ; SSE4-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,2],xmm3[0,2]
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm5, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE4-NEXT:    pxor %xmm1, %xmm0
-; SSE4-NEXT:    pextrb $0, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB0_10
-; SSE4-NEXT:  # %bb.9: # %cond.store7
+; SSE4-NEXT:    testb $16, %al
+; SSE4-NEXT:    jne .LBB0_9
+; SSE4-NEXT:  # %bb.10: # %else8
+; SSE4-NEXT:    testb $32, %al
+; SSE4-NEXT:    jne .LBB0_11
+; SSE4-NEXT:  .LBB0_12: # %else10
+; SSE4-NEXT:    testb $64, %al
+; SSE4-NEXT:    jne .LBB0_13
+; SSE4-NEXT:  .LBB0_14: # %else12
+; SSE4-NEXT:    testb $-128, %al
+; SSE4-NEXT:    jne .LBB0_15
+; SSE4-NEXT:  .LBB0_16: # %else14
+; SSE4-NEXT:    retq
+; SSE4-NEXT:  .LBB0_1: # %cond.store
+; SSE4-NEXT:    movd %xmm0, (%rdi)
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    je .LBB0_4
+; SSE4-NEXT:  .LBB0_3: # %cond.store1
+; SSE4-NEXT:    pextrd $1, %xmm0, 4(%rdi)
+; SSE4-NEXT:    testb $4, %al
+; SSE4-NEXT:    je .LBB0_6
+; SSE4-NEXT:  .LBB0_5: # %cond.store3
+; SSE4-NEXT:    pextrd $2, %xmm0, 8(%rdi)
+; SSE4-NEXT:    testb $8, %al
+; SSE4-NEXT:    jne .LBB0_7
+; SSE4-NEXT:    jmp .LBB0_8
+; SSE4-NEXT:  .LBB0_9: # %cond.store7
 ; SSE4-NEXT:    movss %xmm2, 16(%rdi)
-; SSE4-NEXT:  .LBB0_10: # %else8
-; SSE4-NEXT:    pextrb $4, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $32, %al
 ; SSE4-NEXT:    je .LBB0_12
-; SSE4-NEXT:  # %bb.11: # %cond.store9
+; SSE4-NEXT:  .LBB0_11: # %cond.store9
 ; SSE4-NEXT:    extractps $1, %xmm2, 20(%rdi)
-; SSE4-NEXT:  .LBB0_12: # %else10
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqd %xmm0, %xmm5
-; SSE4-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE4-NEXT:    pxor %xmm5, %xmm0
-; SSE4-NEXT:    pextrb $8, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $64, %al
 ; SSE4-NEXT:    je .LBB0_14
-; SSE4-NEXT:  # %bb.13: # %cond.store11
+; SSE4-NEXT:  .LBB0_13: # %cond.store11
 ; SSE4-NEXT:    extractps $2, %xmm2, 24(%rdi)
-; SSE4-NEXT:  .LBB0_14: # %else12
-; SSE4-NEXT:    pextrb $12, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $-128, %al
 ; SSE4-NEXT:    je .LBB0_16
-; SSE4-NEXT:  # %bb.15: # %cond.store13
+; SSE4-NEXT:  .LBB0_15: # %cond.store13
 ; SSE4-NEXT:    extractps $3, %xmm2, 28(%rdi)
-; SSE4-NEXT:  .LBB0_16: # %else14
 ; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: truncstore_v8i64_v8i32:
@@ -224,331 +218,311 @@ define void @truncstore_v8i64_v8i32(<8 x i64> %x, <8 x i32>* %p, <8 x i32> %mask
 define void @truncstore_v8i64_v8i16(<8 x i64> %x, <8 x i16>* %p, <8 x i32> %mask) {
 ; SSE2-LABEL: truncstore_v8i64_v8i16:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pxor %xmm7, %xmm7
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm7
-; SSE2-NEXT:    pcmpeqd %xmm6, %xmm6
-; SSE2-NEXT:    pxor %xmm7, %xmm6
-; SSE2-NEXT:    movdqa %xmm6, %xmm7
-; SSE2-NEXT:    packssdw %xmm0, %xmm7
+; SSE2-NEXT:    pxor %xmm6, %xmm6
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm8 = xmm1[0,2,2,3,4,5,6,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm0[0,2,2,3,4,5,6,7]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm8[0],xmm1[1],xmm8[1]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm7 = xmm0[0,2,2,3,4,5,6,7]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm7 = xmm7[0],xmm1[0],xmm7[1],xmm1[1]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,2,2,3]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm0[0,1,0,2,4,5,6,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm0[0,1,0,2,4,5,6,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,1,0,2,4,5,6,7]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
-; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
-; SSE2-NEXT:    movd %xmm7, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB1_2
-; SSE2-NEXT:  # %bb.1: # %cond.store
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    movw %ax, (%rdi)
-; SSE2-NEXT:  .LBB1_2: # %else
-; SSE2-NEXT:    psrlq $16, %xmm6
-; SSE2-NEXT:    movd %xmm6, %eax
-; SSE2-NEXT:    shrl $16, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB1_4
-; SSE2-NEXT:  # %bb.3: # %cond.store1
-; SSE2-NEXT:    pextrw $1, %xmm0, %eax
-; SSE2-NEXT:    movw %ax, 2(%rdi)
-; SSE2-NEXT:  .LBB1_4: # %else2
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm4
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm7[0],xmm0[1]
+; SSE2-NEXT:    pcmpeqd %xmm6, %xmm5
 ; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pxor %xmm4, %xmm1
-; SSE2-NEXT:    pextrw $4, %xmm1, %eax
+; SSE2-NEXT:    pxor %xmm1, %xmm5
+; SSE2-NEXT:    pcmpeqd %xmm6, %xmm4
+; SSE2-NEXT:    pxor %xmm1, %xmm4
+; SSE2-NEXT:    packssdw %xmm5, %xmm4
+; SSE2-NEXT:    packsswb %xmm0, %xmm4
+; SSE2-NEXT:    pmovmskb %xmm4, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB1_6
-; SSE2-NEXT:  # %bb.5: # %cond.store3
-; SSE2-NEXT:    pextrw $2, %xmm0, %eax
-; SSE2-NEXT:    movw %ax, 4(%rdi)
+; SSE2-NEXT:    jne .LBB1_1
+; SSE2-NEXT:  # %bb.2: # %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne .LBB1_3
+; SSE2-NEXT:  .LBB1_4: # %else2
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne .LBB1_5
 ; SSE2-NEXT:  .LBB1_6: # %else4
-; SSE2-NEXT:    pextrw $6, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB1_8
-; SSE2-NEXT:  # %bb.7: # %cond.store5
-; SSE2-NEXT:    pextrw $3, %xmm0, %eax
-; SSE2-NEXT:    movw %ax, 6(%rdi)
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne .LBB1_7
 ; SSE2-NEXT:  .LBB1_8: # %else6
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pxor %xmm2, %xmm1
-; SSE2-NEXT:    pextrw $0, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB1_10
-; SSE2-NEXT:  # %bb.9: # %cond.store7
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    movw %ax, 8(%rdi)
+; SSE2-NEXT:    testb $16, %al
+; SSE2-NEXT:    jne .LBB1_9
 ; SSE2-NEXT:  .LBB1_10: # %else8
-; SSE2-NEXT:    pextrw $2, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB1_12
-; SSE2-NEXT:  # %bb.11: # %cond.store9
-; SSE2-NEXT:    pextrw $5, %xmm0, %eax
-; SSE2-NEXT:    movw %ax, 10(%rdi)
+; SSE2-NEXT:    testb $32, %al
+; SSE2-NEXT:    jne .LBB1_11
 ; SSE2-NEXT:  .LBB1_12: # %else10
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm5
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pxor %xmm5, %xmm1
-; SSE2-NEXT:    pextrw $4, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB1_14
-; SSE2-NEXT:  # %bb.13: # %cond.store11
-; SSE2-NEXT:    pextrw $6, %xmm0, %eax
-; SSE2-NEXT:    movw %ax, 12(%rdi)
+; SSE2-NEXT:    testb $64, %al
+; SSE2-NEXT:    jne .LBB1_13
 ; SSE2-NEXT:  .LBB1_14: # %else12
-; SSE2-NEXT:    pextrw $6, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $-128, %al
+; SSE2-NEXT:    jne .LBB1_15
+; SSE2-NEXT:  .LBB1_16: # %else14
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  .LBB1_1: # %cond.store
+; SSE2-NEXT:    movd %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, (%rdi)
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    je .LBB1_4
+; SSE2-NEXT:  .LBB1_3: # %cond.store1
+; SSE2-NEXT:    pextrw $1, %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, 2(%rdi)
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    je .LBB1_6
+; SSE2-NEXT:  .LBB1_5: # %cond.store3
+; SSE2-NEXT:    pextrw $2, %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, 4(%rdi)
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    je .LBB1_8
+; SSE2-NEXT:  .LBB1_7: # %cond.store5
+; SSE2-NEXT:    pextrw $3, %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, 6(%rdi)
+; SSE2-NEXT:    testb $16, %al
+; SSE2-NEXT:    je .LBB1_10
+; SSE2-NEXT:  .LBB1_9: # %cond.store7
+; SSE2-NEXT:    pextrw $4, %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, 8(%rdi)
+; SSE2-NEXT:    testb $32, %al
+; SSE2-NEXT:    je .LBB1_12
+; SSE2-NEXT:  .LBB1_11: # %cond.store9
+; SSE2-NEXT:    pextrw $5, %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, 10(%rdi)
+; SSE2-NEXT:    testb $64, %al
+; SSE2-NEXT:    je .LBB1_14
+; SSE2-NEXT:  .LBB1_13: # %cond.store11
+; SSE2-NEXT:    pextrw $6, %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, 12(%rdi)
+; SSE2-NEXT:    testb $-128, %al
 ; SSE2-NEXT:    je .LBB1_16
-; SSE2-NEXT:  # %bb.15: # %cond.store13
+; SSE2-NEXT:  .LBB1_15: # %cond.store13
 ; SSE2-NEXT:    pextrw $7, %xmm0, %eax
 ; SSE2-NEXT:    movw %ax, 14(%rdi)
-; SSE2-NEXT:  .LBB1_16: # %else14
 ; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: truncstore_v8i64_v8i16:
 ; SSE4:       # %bb.0:
-; SSE4-NEXT:    pxor %xmm8, %xmm8
-; SSE4-NEXT:    movdqa %xmm4, %xmm7
-; SSE4-NEXT:    pcmpeqd %xmm8, %xmm7
-; SSE4-NEXT:    pcmpeqd %xmm6, %xmm6
-; SSE4-NEXT:    pxor %xmm7, %xmm6
-; SSE4-NEXT:    pblendw {{.*#+}} xmm3 = xmm3[0],xmm8[1,2,3],xmm3[4],xmm8[5,6,7]
-; SSE4-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0],xmm8[1,2,3],xmm2[4],xmm8[5,6,7]
+; SSE4-NEXT:    pxor %xmm6, %xmm6
+; SSE4-NEXT:    pblendw {{.*#+}} xmm3 = xmm3[0],xmm6[1,2,3],xmm3[4],xmm6[5,6,7]
+; SSE4-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0],xmm6[1,2,3],xmm2[4],xmm6[5,6,7]
 ; SSE4-NEXT:    packusdw %xmm3, %xmm2
-; SSE4-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0],xmm8[1,2,3],xmm1[4],xmm8[5,6,7]
-; SSE4-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0],xmm8[1,2,3],xmm0[4],xmm8[5,6,7]
+; SSE4-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0],xmm6[1,2,3],xmm1[4],xmm6[5,6,7]
+; SSE4-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0],xmm6[1,2,3],xmm0[4],xmm6[5,6,7]
 ; SSE4-NEXT:    packusdw %xmm1, %xmm0
 ; SSE4-NEXT:    packusdw %xmm2, %xmm0
-; SSE4-NEXT:    pextrb $0, %xmm6, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB1_2
-; SSE4-NEXT:  # %bb.1: # %cond.store
+; SSE4-NEXT:    pcmpeqd %xmm6, %xmm5
+; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
+; SSE4-NEXT:    pxor %xmm1, %xmm5
+; SSE4-NEXT:    pcmpeqd %xmm6, %xmm4
+; SSE4-NEXT:    pxor %xmm1, %xmm4
+; SSE4-NEXT:    packssdw %xmm5, %xmm4
+; SSE4-NEXT:    packsswb %xmm0, %xmm4
+; SSE4-NEXT:    pmovmskb %xmm4, %eax
+; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    jne .LBB1_1
+; SSE4-NEXT:  # %bb.2: # %else
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    jne .LBB1_3
+; SSE4-NEXT:  .LBB1_4: # %else2
+; SSE4-NEXT:    testb $4, %al
+; SSE4-NEXT:    jne .LBB1_5
+; SSE4-NEXT:  .LBB1_6: # %else4
+; SSE4-NEXT:    testb $8, %al
+; SSE4-NEXT:    jne .LBB1_7
+; SSE4-NEXT:  .LBB1_8: # %else6
+; SSE4-NEXT:    testb $16, %al
+; SSE4-NEXT:    jne .LBB1_9
+; SSE4-NEXT:  .LBB1_10: # %else8
+; SSE4-NEXT:    testb $32, %al
+; SSE4-NEXT:    jne .LBB1_11
+; SSE4-NEXT:  .LBB1_12: # %else10
+; SSE4-NEXT:    testb $64, %al
+; SSE4-NEXT:    jne .LBB1_13
+; SSE4-NEXT:  .LBB1_14: # %else12
+; SSE4-NEXT:    testb $-128, %al
+; SSE4-NEXT:    jne .LBB1_15
+; SSE4-NEXT:  .LBB1_16: # %else14
+; SSE4-NEXT:    retq
+; SSE4-NEXT:  .LBB1_1: # %cond.store
 ; SSE4-NEXT:    pextrw $0, %xmm0, (%rdi)
-; SSE4-NEXT:  .LBB1_2: # %else
-; SSE4-NEXT:    pextrb $4, %xmm6, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $2, %al
 ; SSE4-NEXT:    je .LBB1_4
-; SSE4-NEXT:  # %bb.3: # %cond.store1
+; SSE4-NEXT:  .LBB1_3: # %cond.store1
 ; SSE4-NEXT:    pextrw $1, %xmm0, 2(%rdi)
-; SSE4-NEXT:  .LBB1_4: # %else2
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm4
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE4-NEXT:    pxor %xmm4, %xmm1
-; SSE4-NEXT:    pextrb $8, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $4, %al
 ; SSE4-NEXT:    je .LBB1_6
-; SSE4-NEXT:  # %bb.5: # %cond.store3
+; SSE4-NEXT:  .LBB1_5: # %cond.store3
 ; SSE4-NEXT:    pextrw $2, %xmm0, 4(%rdi)
-; SSE4-NEXT:  .LBB1_6: # %else4
-; SSE4-NEXT:    pextrb $12, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $8, %al
 ; SSE4-NEXT:    je .LBB1_8
-; SSE4-NEXT:  # %bb.7: # %cond.store5
+; SSE4-NEXT:  .LBB1_7: # %cond.store5
 ; SSE4-NEXT:    pextrw $3, %xmm0, 6(%rdi)
-; SSE4-NEXT:  .LBB1_8: # %else6
-; SSE4-NEXT:    pxor %xmm2, %xmm2
-; SSE4-NEXT:    pcmpeqd %xmm5, %xmm2
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE4-NEXT:    pxor %xmm2, %xmm1
-; SSE4-NEXT:    pextrb $0, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $16, %al
 ; SSE4-NEXT:    je .LBB1_10
-; SSE4-NEXT:  # %bb.9: # %cond.store7
+; SSE4-NEXT:  .LBB1_9: # %cond.store7
 ; SSE4-NEXT:    pextrw $4, %xmm0, 8(%rdi)
-; SSE4-NEXT:  .LBB1_10: # %else8
-; SSE4-NEXT:    pextrb $4, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $32, %al
 ; SSE4-NEXT:    je .LBB1_12
-; SSE4-NEXT:  # %bb.11: # %cond.store9
+; SSE4-NEXT:  .LBB1_11: # %cond.store9
 ; SSE4-NEXT:    pextrw $5, %xmm0, 10(%rdi)
-; SSE4-NEXT:  .LBB1_12: # %else10
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm5
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE4-NEXT:    pxor %xmm5, %xmm1
-; SSE4-NEXT:    pextrb $8, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $64, %al
 ; SSE4-NEXT:    je .LBB1_14
-; SSE4-NEXT:  # %bb.13: # %cond.store11
+; SSE4-NEXT:  .LBB1_13: # %cond.store11
 ; SSE4-NEXT:    pextrw $6, %xmm0, 12(%rdi)
-; SSE4-NEXT:  .LBB1_14: # %else12
-; SSE4-NEXT:    pextrb $12, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $-128, %al
 ; SSE4-NEXT:    je .LBB1_16
-; SSE4-NEXT:  # %bb.15: # %cond.store13
+; SSE4-NEXT:  .LBB1_15: # %cond.store13
 ; SSE4-NEXT:    pextrw $7, %xmm0, 14(%rdi)
-; SSE4-NEXT:  .LBB1_16: # %else14
 ; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: truncstore_v8i64_v8i16:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm5
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX1-NEXT:    vmovaps {{.*#+}} ymm6 = [65535,65535,65535,65535]
-; AVX1-NEXT:    vandps %ymm6, %ymm1, %ymm1
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm7
-; AVX1-NEXT:    vpackusdw %xmm7, %xmm1, %xmm1
-; AVX1-NEXT:    vandps %ymm6, %ymm0, %ymm0
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm6
-; AVX1-NEXT:    vpackusdw %xmm6, %xmm0, %xmm0
+; AVX1-NEXT:    vmovaps {{.*#+}} ymm3 = [65535,65535,65535,65535]
+; AVX1-NEXT:    vandps %ymm3, %ymm1, %ymm1
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm4
+; AVX1-NEXT:    vpackusdw %xmm4, %xmm1, %xmm1
+; AVX1-NEXT:    vandps %ymm3, %ymm0, %ymm0
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; AVX1-NEXT:    vpackusdw %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpextrb $0, %xmm5, %eax
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm1
+; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
+; AVX1-NEXT:    vmovmskps %ymm1, %eax
+; AVX1-NEXT:    notl %eax
 ; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je .LBB1_2
-; AVX1-NEXT:  # %bb.1: # %cond.store
+; AVX1-NEXT:    jne .LBB1_1
+; AVX1-NEXT:  # %bb.2: # %else
+; AVX1-NEXT:    testb $2, %al
+; AVX1-NEXT:    jne .LBB1_3
+; AVX1-NEXT:  .LBB1_4: # %else2
+; AVX1-NEXT:    testb $4, %al
+; AVX1-NEXT:    jne .LBB1_5
+; AVX1-NEXT:  .LBB1_6: # %else4
+; AVX1-NEXT:    testb $8, %al
+; AVX1-NEXT:    jne .LBB1_7
+; AVX1-NEXT:  .LBB1_8: # %else6
+; AVX1-NEXT:    testb $16, %al
+; AVX1-NEXT:    jne .LBB1_9
+; AVX1-NEXT:  .LBB1_10: # %else8
+; AVX1-NEXT:    testb $32, %al
+; AVX1-NEXT:    jne .LBB1_11
+; AVX1-NEXT:  .LBB1_12: # %else10
+; AVX1-NEXT:    testb $64, %al
+; AVX1-NEXT:    jne .LBB1_13
+; AVX1-NEXT:  .LBB1_14: # %else12
+; AVX1-NEXT:    testb $-128, %al
+; AVX1-NEXT:    jne .LBB1_15
+; AVX1-NEXT:  .LBB1_16: # %else14
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+; AVX1-NEXT:  .LBB1_1: # %cond.store
 ; AVX1-NEXT:    vpextrw $0, %xmm0, (%rdi)
-; AVX1-NEXT:  .LBB1_2: # %else
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm4, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $4, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $2, %al
 ; AVX1-NEXT:    je .LBB1_4
-; AVX1-NEXT:  # %bb.3: # %cond.store1
+; AVX1-NEXT:  .LBB1_3: # %cond.store1
 ; AVX1-NEXT:    vpextrw $1, %xmm0, 2(%rdi)
-; AVX1-NEXT:  .LBB1_4: # %else2
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm2, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm4, %xmm4
-; AVX1-NEXT:    vpextrb $8, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $4, %al
 ; AVX1-NEXT:    je .LBB1_6
-; AVX1-NEXT:  # %bb.5: # %cond.store3
+; AVX1-NEXT:  .LBB1_5: # %cond.store3
 ; AVX1-NEXT:    vpextrw $2, %xmm0, 4(%rdi)
-; AVX1-NEXT:  .LBB1_6: # %else4
-; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $8, %al
 ; AVX1-NEXT:    je .LBB1_8
-; AVX1-NEXT:  # %bb.7: # %cond.store5
+; AVX1-NEXT:  .LBB1_7: # %cond.store5
 ; AVX1-NEXT:    vpextrw $3, %xmm0, 6(%rdi)
-; AVX1-NEXT:  .LBB1_8: # %else6
-; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $0, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $16, %al
 ; AVX1-NEXT:    je .LBB1_10
-; AVX1-NEXT:  # %bb.9: # %cond.store7
+; AVX1-NEXT:  .LBB1_9: # %cond.store7
 ; AVX1-NEXT:    vpextrw $4, %xmm0, 8(%rdi)
-; AVX1-NEXT:  .LBB1_10: # %else8
-; AVX1-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $32, %al
 ; AVX1-NEXT:    je .LBB1_12
-; AVX1-NEXT:  # %bb.11: # %cond.store9
+; AVX1-NEXT:  .LBB1_11: # %cond.store9
 ; AVX1-NEXT:    vpextrw $5, %xmm0, 10(%rdi)
-; AVX1-NEXT:  .LBB1_12: # %else10
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $64, %al
 ; AVX1-NEXT:    je .LBB1_14
-; AVX1-NEXT:  # %bb.13: # %cond.store11
+; AVX1-NEXT:  .LBB1_13: # %cond.store11
 ; AVX1-NEXT:    vpextrw $6, %xmm0, 12(%rdi)
-; AVX1-NEXT:  .LBB1_14: # %else12
-; AVX1-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $-128, %al
 ; AVX1-NEXT:    je .LBB1_16
-; AVX1-NEXT:  # %bb.15: # %cond.store13
+; AVX1-NEXT:  .LBB1_15: # %cond.store13
 ; AVX1-NEXT:    vpextrw $7, %xmm0, 14(%rdi)
-; AVX1-NEXT:  .LBB1_16: # %else14
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: truncstore_v8i64_v8i16:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm5
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX2-NEXT:    vextractf128 $1, %ymm1, %xmm6
-; AVX2-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[0,2],xmm6[0,2]
-; AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm6
-; AVX2-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm6[0,2]
+; AVX2-NEXT:    vextractf128 $1, %ymm1, %xmm4
+; AVX2-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[0,2],xmm4[0,2]
+; AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm4
+; AVX2-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm4[0,2]
 ; AVX2-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-NEXT:    vpextrb $0, %xmm5, %eax
+; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm2, %ymm1
+; AVX2-NEXT:    vmovmskps %ymm1, %eax
+; AVX2-NEXT:    notl %eax
 ; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je .LBB1_2
-; AVX2-NEXT:  # %bb.1: # %cond.store
+; AVX2-NEXT:    jne .LBB1_1
+; AVX2-NEXT:  # %bb.2: # %else
+; AVX2-NEXT:    testb $2, %al
+; AVX2-NEXT:    jne .LBB1_3
+; AVX2-NEXT:  .LBB1_4: # %else2
+; AVX2-NEXT:    testb $4, %al
+; AVX2-NEXT:    jne .LBB1_5
+; AVX2-NEXT:  .LBB1_6: # %else4
+; AVX2-NEXT:    testb $8, %al
+; AVX2-NEXT:    jne .LBB1_7
+; AVX2-NEXT:  .LBB1_8: # %else6
+; AVX2-NEXT:    testb $16, %al
+; AVX2-NEXT:    jne .LBB1_9
+; AVX2-NEXT:  .LBB1_10: # %else8
+; AVX2-NEXT:    testb $32, %al
+; AVX2-NEXT:    jne .LBB1_11
+; AVX2-NEXT:  .LBB1_12: # %else10
+; AVX2-NEXT:    testb $64, %al
+; AVX2-NEXT:    jne .LBB1_13
+; AVX2-NEXT:  .LBB1_14: # %else12
+; AVX2-NEXT:    testb $-128, %al
+; AVX2-NEXT:    jne .LBB1_15
+; AVX2-NEXT:  .LBB1_16: # %else14
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+; AVX2-NEXT:  .LBB1_1: # %cond.store
 ; AVX2-NEXT:    vpextrw $0, %xmm0, (%rdi)
-; AVX2-NEXT:  .LBB1_2: # %else
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm1
-; AVX2-NEXT:    vpxor %xmm4, %xmm1, %xmm1
-; AVX2-NEXT:    vpextrb $4, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $2, %al
 ; AVX2-NEXT:    je .LBB1_4
-; AVX2-NEXT:  # %bb.3: # %cond.store1
+; AVX2-NEXT:  .LBB1_3: # %cond.store1
 ; AVX2-NEXT:    vpextrw $1, %xmm0, 2(%rdi)
-; AVX2-NEXT:  .LBB1_4: # %else2
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm2, %xmm4
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    vpxor %xmm3, %xmm4, %xmm4
-; AVX2-NEXT:    vpextrb $8, %xmm4, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $4, %al
 ; AVX2-NEXT:    je .LBB1_6
-; AVX2-NEXT:  # %bb.5: # %cond.store3
+; AVX2-NEXT:  .LBB1_5: # %cond.store3
 ; AVX2-NEXT:    vpextrw $2, %xmm0, 4(%rdi)
-; AVX2-NEXT:  .LBB1_6: # %else4
-; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm2, %xmm1
-; AVX2-NEXT:    vpxor %xmm3, %xmm1, %xmm1
-; AVX2-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $8, %al
 ; AVX2-NEXT:    je .LBB1_8
-; AVX2-NEXT:  # %bb.7: # %cond.store5
+; AVX2-NEXT:  .LBB1_7: # %cond.store5
 ; AVX2-NEXT:    vpextrw $3, %xmm0, 6(%rdi)
-; AVX2-NEXT:  .LBB1_8: # %else6
-; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm1
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrb $0, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $16, %al
 ; AVX2-NEXT:    je .LBB1_10
-; AVX2-NEXT:  # %bb.9: # %cond.store7
+; AVX2-NEXT:  .LBB1_9: # %cond.store7
 ; AVX2-NEXT:    vpextrw $4, %xmm0, 8(%rdi)
-; AVX2-NEXT:  .LBB1_10: # %else8
-; AVX2-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $32, %al
 ; AVX2-NEXT:    je .LBB1_12
-; AVX2-NEXT:  # %bb.11: # %cond.store9
+; AVX2-NEXT:  .LBB1_11: # %cond.store9
 ; AVX2-NEXT:    vpextrw $5, %xmm0, 10(%rdi)
-; AVX2-NEXT:  .LBB1_12: # %else10
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $64, %al
 ; AVX2-NEXT:    je .LBB1_14
-; AVX2-NEXT:  # %bb.13: # %cond.store11
+; AVX2-NEXT:  .LBB1_13: # %cond.store11
 ; AVX2-NEXT:    vpextrw $6, %xmm0, 12(%rdi)
-; AVX2-NEXT:  .LBB1_14: # %else12
-; AVX2-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $-128, %al
 ; AVX2-NEXT:    je .LBB1_16
-; AVX2-NEXT:  # %bb.15: # %cond.store13
+; AVX2-NEXT:  .LBB1_15: # %cond.store13
 ; AVX2-NEXT:    vpextrw $7, %xmm0, 14(%rdi)
-; AVX2-NEXT:  .LBB1_16: # %else14
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
@@ -559,66 +533,61 @@ define void @truncstore_v8i64_v8i16(<8 x i64> %x, <8 x i16>* %p, <8 x i32> %mask
 ; AVX512F-NEXT:    vpmovqw %zmm0, %xmm0
 ; AVX512F-NEXT:    kmovw %k0, %eax
 ; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB1_2
-; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    jne .LBB1_1
+; AVX512F-NEXT:  # %bb.2: # %else
+; AVX512F-NEXT:    testb $2, %al
+; AVX512F-NEXT:    jne .LBB1_3
+; AVX512F-NEXT:  .LBB1_4: # %else2
+; AVX512F-NEXT:    testb $4, %al
+; AVX512F-NEXT:    jne .LBB1_5
+; AVX512F-NEXT:  .LBB1_6: # %else4
+; AVX512F-NEXT:    testb $8, %al
+; AVX512F-NEXT:    jne .LBB1_7
+; AVX512F-NEXT:  .LBB1_8: # %else6
+; AVX512F-NEXT:    testb $16, %al
+; AVX512F-NEXT:    jne .LBB1_9
+; AVX512F-NEXT:  .LBB1_10: # %else8
+; AVX512F-NEXT:    testb $32, %al
+; AVX512F-NEXT:    jne .LBB1_11
+; AVX512F-NEXT:  .LBB1_12: # %else10
+; AVX512F-NEXT:    testb $64, %al
+; AVX512F-NEXT:    jne .LBB1_13
+; AVX512F-NEXT:  .LBB1_14: # %else12
+; AVX512F-NEXT:    testb $-128, %al
+; AVX512F-NEXT:    jne .LBB1_15
+; AVX512F-NEXT:  .LBB1_16: # %else14
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+; AVX512F-NEXT:  .LBB1_1: # %cond.store
 ; AVX512F-NEXT:    vpextrw $0, %xmm0, (%rdi)
-; AVX512F-NEXT:  .LBB1_2: # %else
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $2, %al
 ; AVX512F-NEXT:    je .LBB1_4
-; AVX512F-NEXT:  # %bb.3: # %cond.store1
+; AVX512F-NEXT:  .LBB1_3: # %cond.store1
 ; AVX512F-NEXT:    vpextrw $1, %xmm0, 2(%rdi)
-; AVX512F-NEXT:  .LBB1_4: # %else2
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $4, %al
 ; AVX512F-NEXT:    je .LBB1_6
-; AVX512F-NEXT:  # %bb.5: # %cond.store3
+; AVX512F-NEXT:  .LBB1_5: # %cond.store3
 ; AVX512F-NEXT:    vpextrw $2, %xmm0, 4(%rdi)
-; AVX512F-NEXT:  .LBB1_6: # %else4
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $8, %al
 ; AVX512F-NEXT:    je .LBB1_8
-; AVX512F-NEXT:  # %bb.7: # %cond.store5
+; AVX512F-NEXT:  .LBB1_7: # %cond.store5
 ; AVX512F-NEXT:    vpextrw $3, %xmm0, 6(%rdi)
-; AVX512F-NEXT:  .LBB1_8: # %else6
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $4, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $16, %al
 ; AVX512F-NEXT:    je .LBB1_10
-; AVX512F-NEXT:  # %bb.9: # %cond.store7
+; AVX512F-NEXT:  .LBB1_9: # %cond.store7
 ; AVX512F-NEXT:    vpextrw $4, %xmm0, 8(%rdi)
-; AVX512F-NEXT:  .LBB1_10: # %else8
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $5, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $32, %al
 ; AVX512F-NEXT:    je .LBB1_12
-; AVX512F-NEXT:  # %bb.11: # %cond.store9
+; AVX512F-NEXT:  .LBB1_11: # %cond.store9
 ; AVX512F-NEXT:    vpextrw $5, %xmm0, 10(%rdi)
-; AVX512F-NEXT:  .LBB1_12: # %else10
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $6, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $64, %al
 ; AVX512F-NEXT:    je .LBB1_14
-; AVX512F-NEXT:  # %bb.13: # %cond.store11
+; AVX512F-NEXT:  .LBB1_13: # %cond.store11
 ; AVX512F-NEXT:    vpextrw $6, %xmm0, 12(%rdi)
-; AVX512F-NEXT:  .LBB1_14: # %else12
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $7, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $-128, %al
 ; AVX512F-NEXT:    je .LBB1_16
-; AVX512F-NEXT:  # %bb.15: # %cond.store13
+; AVX512F-NEXT:  .LBB1_15: # %cond.store13
 ; AVX512F-NEXT:    vpextrw $7, %xmm0, 14(%rdi)
-; AVX512F-NEXT:  .LBB1_16: # %else14
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
@@ -645,331 +614,309 @@ define void @truncstore_v8i64_v8i16(<8 x i64> %x, <8 x i16>* %p, <8 x i32> %mask
 define void @truncstore_v8i64_v8i8(<8 x i64> %x, <8 x i8>* %p, <8 x i32> %mask) {
 ; SSE2-LABEL: truncstore_v8i64_v8i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pxor %xmm7, %xmm7
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm7
-; SSE2-NEXT:    pcmpeqd %xmm6, %xmm6
-; SSE2-NEXT:    pxor %xmm7, %xmm6
-; SSE2-NEXT:    movdqa %xmm6, %xmm7
-; SSE2-NEXT:    packssdw %xmm0, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm8 = xmm1[0,2,2,3,4,5,6,7]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm0[0,2,2,3,4,5,6,7]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm8[0],xmm1[1],xmm8[1]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,2,2,3]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm0[0,1,0,2,4,5,6,7]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,1,0,2,4,5,6,7]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
-; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
-; SSE2-NEXT:    movd %xmm7, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    je .LBB2_2
-; SSE2-NEXT:  # %bb.1: # %cond.store
-; SSE2-NEXT:    movb %al, (%rdi)
-; SSE2-NEXT:  .LBB2_2: # %else
-; SSE2-NEXT:    psrlq $16, %xmm6
-; SSE2-NEXT:    movd %xmm6, %ecx
-; SSE2-NEXT:    shrl $16, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    je .LBB2_4
-; SSE2-NEXT:  # %bb.3: # %cond.store1
-; SSE2-NEXT:    shrl $16, %eax
-; SSE2-NEXT:    movb %al, 1(%rdi)
-; SSE2-NEXT:  .LBB2_4: # %else2
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm4
+; SSE2-NEXT:    pxor %xmm6, %xmm6
+; SSE2-NEXT:    movdqa {{.*#+}} xmm7 = [255,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0]
+; SSE2-NEXT:    pand %xmm7, %xmm3
+; SSE2-NEXT:    pand %xmm7, %xmm2
+; SSE2-NEXT:    packuswb %xmm3, %xmm2
+; SSE2-NEXT:    pand %xmm7, %xmm1
+; SSE2-NEXT:    pand %xmm7, %xmm0
+; SSE2-NEXT:    packuswb %xmm1, %xmm0
+; SSE2-NEXT:    packuswb %xmm2, %xmm0
+; SSE2-NEXT:    packuswb %xmm0, %xmm0
+; SSE2-NEXT:    pcmpeqd %xmm6, %xmm5
 ; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pxor %xmm4, %xmm1
-; SSE2-NEXT:    pextrw $4, %xmm1, %eax
+; SSE2-NEXT:    pxor %xmm1, %xmm5
+; SSE2-NEXT:    pcmpeqd %xmm6, %xmm4
+; SSE2-NEXT:    pxor %xmm1, %xmm4
+; SSE2-NEXT:    packssdw %xmm5, %xmm4
+; SSE2-NEXT:    packsswb %xmm0, %xmm4
+; SSE2-NEXT:    pmovmskb %xmm4, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB2_6
-; SSE2-NEXT:  # %bb.5: # %cond.store3
-; SSE2-NEXT:    pextrw $2, %xmm0, %eax
-; SSE2-NEXT:    movb %al, 2(%rdi)
+; SSE2-NEXT:    movd %xmm0, %ecx
+; SSE2-NEXT:    jne .LBB2_1
+; SSE2-NEXT:  # %bb.2: # %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne .LBB2_3
+; SSE2-NEXT:  .LBB2_4: # %else2
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne .LBB2_5
 ; SSE2-NEXT:  .LBB2_6: # %else4
-; SSE2-NEXT:    pextrw $6, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je .LBB2_8
-; SSE2-NEXT:  # %bb.7: # %cond.store5
-; SSE2-NEXT:    pextrw $3, %xmm0, %eax
-; SSE2-NEXT:    movb %al, 3(%rdi)
+; SSE2-NEXT:  .LBB2_7: # %cond.store5
+; SSE2-NEXT:    shrl $24, %ecx
+; SSE2-NEXT:    movb %cl, 3(%rdi)
 ; SSE2-NEXT:  .LBB2_8: # %else6
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pxor %xmm2, %xmm1
-; SSE2-NEXT:    pextrw $0, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $16, %al
+; SSE2-NEXT:    pextrw $2, %xmm0, %ecx
 ; SSE2-NEXT:    je .LBB2_10
 ; SSE2-NEXT:  # %bb.9: # %cond.store7
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    movb %al, 4(%rdi)
+; SSE2-NEXT:    movb %cl, 4(%rdi)
 ; SSE2-NEXT:  .LBB2_10: # %else8
-; SSE2-NEXT:    pextrw $2, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $32, %al
 ; SSE2-NEXT:    je .LBB2_12
 ; SSE2-NEXT:  # %bb.11: # %cond.store9
-; SSE2-NEXT:    pextrw $5, %xmm0, %eax
-; SSE2-NEXT:    movb %al, 5(%rdi)
+; SSE2-NEXT:    movb %ch, 5(%rdi)
 ; SSE2-NEXT:  .LBB2_12: # %else10
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm5
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pxor %xmm5, %xmm1
-; SSE2-NEXT:    pextrw $4, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB2_14
-; SSE2-NEXT:  # %bb.13: # %cond.store11
-; SSE2-NEXT:    pextrw $6, %xmm0, %eax
-; SSE2-NEXT:    movb %al, 6(%rdi)
-; SSE2-NEXT:  .LBB2_14: # %else12
-; SSE2-NEXT:    pextrw $6, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB2_16
-; SSE2-NEXT:  # %bb.15: # %cond.store13
-; SSE2-NEXT:    pextrw $7, %xmm0, %eax
-; SSE2-NEXT:    movb %al, 7(%rdi)
+; SSE2-NEXT:    testb $64, %al
+; SSE2-NEXT:    pextrw $3, %xmm0, %ecx
+; SSE2-NEXT:    jne .LBB2_13
+; SSE2-NEXT:  # %bb.14: # %else12
+; SSE2-NEXT:    testb $-128, %al
+; SSE2-NEXT:    jne .LBB2_15
 ; SSE2-NEXT:  .LBB2_16: # %else14
 ; SSE2-NEXT:    retq
+; SSE2-NEXT:  .LBB2_1: # %cond.store
+; SSE2-NEXT:    movb %cl, (%rdi)
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    je .LBB2_4
+; SSE2-NEXT:  .LBB2_3: # %cond.store1
+; SSE2-NEXT:    movb %ch, 1(%rdi)
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    je .LBB2_6
+; SSE2-NEXT:  .LBB2_5: # %cond.store3
+; SSE2-NEXT:    movl %ecx, %edx
+; SSE2-NEXT:    shrl $16, %edx
+; SSE2-NEXT:    movb %dl, 2(%rdi)
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne .LBB2_7
+; SSE2-NEXT:    jmp .LBB2_8
+; SSE2-NEXT:  .LBB2_13: # %cond.store11
+; SSE2-NEXT:    movb %cl, 6(%rdi)
+; SSE2-NEXT:    testb $-128, %al
+; SSE2-NEXT:    je .LBB2_16
+; SSE2-NEXT:  .LBB2_15: # %cond.store13
+; SSE2-NEXT:    movb %ch, 7(%rdi)
+; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: truncstore_v8i64_v8i8:
 ; SSE4:       # %bb.0:
-; SSE4-NEXT:    pxor %xmm8, %xmm8
-; SSE4-NEXT:    movdqa %xmm4, %xmm7
-; SSE4-NEXT:    pcmpeqd %xmm8, %xmm7
-; SSE4-NEXT:    pcmpeqd %xmm6, %xmm6
-; SSE4-NEXT:    pxor %xmm7, %xmm6
-; SSE4-NEXT:    pblendw {{.*#+}} xmm3 = xmm3[0],xmm8[1,2,3],xmm3[4],xmm8[5,6,7]
-; SSE4-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0],xmm8[1,2,3],xmm2[4],xmm8[5,6,7]
+; SSE4-NEXT:    pxor %xmm6, %xmm6
+; SSE4-NEXT:    movdqa {{.*#+}} xmm7 = [255,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0]
+; SSE4-NEXT:    pand %xmm7, %xmm3
+; SSE4-NEXT:    pand %xmm7, %xmm2
 ; SSE4-NEXT:    packusdw %xmm3, %xmm2
-; SSE4-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0],xmm8[1,2,3],xmm1[4],xmm8[5,6,7]
-; SSE4-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0],xmm8[1,2,3],xmm0[4],xmm8[5,6,7]
+; SSE4-NEXT:    pand %xmm7, %xmm1
+; SSE4-NEXT:    pand %xmm7, %xmm0
 ; SSE4-NEXT:    packusdw %xmm1, %xmm0
 ; SSE4-NEXT:    packusdw %xmm2, %xmm0
-; SSE4-NEXT:    pextrb $0, %xmm6, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB2_2
-; SSE4-NEXT:  # %bb.1: # %cond.store
+; SSE4-NEXT:    packuswb %xmm0, %xmm0
+; SSE4-NEXT:    pcmpeqd %xmm6, %xmm5
+; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
+; SSE4-NEXT:    pxor %xmm1, %xmm5
+; SSE4-NEXT:    pcmpeqd %xmm6, %xmm4
+; SSE4-NEXT:    pxor %xmm1, %xmm4
+; SSE4-NEXT:    packssdw %xmm5, %xmm4
+; SSE4-NEXT:    packsswb %xmm0, %xmm4
+; SSE4-NEXT:    pmovmskb %xmm4, %eax
+; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    jne .LBB2_1
+; SSE4-NEXT:  # %bb.2: # %else
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    jne .LBB2_3
+; SSE4-NEXT:  .LBB2_4: # %else2
+; SSE4-NEXT:    testb $4, %al
+; SSE4-NEXT:    jne .LBB2_5
+; SSE4-NEXT:  .LBB2_6: # %else4
+; SSE4-NEXT:    testb $8, %al
+; SSE4-NEXT:    jne .LBB2_7
+; SSE4-NEXT:  .LBB2_8: # %else6
+; SSE4-NEXT:    testb $16, %al
+; SSE4-NEXT:    jne .LBB2_9
+; SSE4-NEXT:  .LBB2_10: # %else8
+; SSE4-NEXT:    testb $32, %al
+; SSE4-NEXT:    jne .LBB2_11
+; SSE4-NEXT:  .LBB2_12: # %else10
+; SSE4-NEXT:    testb $64, %al
+; SSE4-NEXT:    jne .LBB2_13
+; SSE4-NEXT:  .LBB2_14: # %else12
+; SSE4-NEXT:    testb $-128, %al
+; SSE4-NEXT:    jne .LBB2_15
+; SSE4-NEXT:  .LBB2_16: # %else14
+; SSE4-NEXT:    retq
+; SSE4-NEXT:  .LBB2_1: # %cond.store
 ; SSE4-NEXT:    pextrb $0, %xmm0, (%rdi)
-; SSE4-NEXT:  .LBB2_2: # %else
-; SSE4-NEXT:    pextrb $4, %xmm6, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $2, %al
 ; SSE4-NEXT:    je .LBB2_4
-; SSE4-NEXT:  # %bb.3: # %cond.store1
-; SSE4-NEXT:    pextrb $2, %xmm0, 1(%rdi)
-; SSE4-NEXT:  .LBB2_4: # %else2
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm4
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE4-NEXT:    pxor %xmm4, %xmm1
-; SSE4-NEXT:    pextrb $8, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:  .LBB2_3: # %cond.store1
+; SSE4-NEXT:    pextrb $1, %xmm0, 1(%rdi)
+; SSE4-NEXT:    testb $4, %al
 ; SSE4-NEXT:    je .LBB2_6
-; SSE4-NEXT:  # %bb.5: # %cond.store3
-; SSE4-NEXT:    pextrb $4, %xmm0, 2(%rdi)
-; SSE4-NEXT:  .LBB2_6: # %else4
-; SSE4-NEXT:    pextrb $12, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:  .LBB2_5: # %cond.store3
+; SSE4-NEXT:    pextrb $2, %xmm0, 2(%rdi)
+; SSE4-NEXT:    testb $8, %al
 ; SSE4-NEXT:    je .LBB2_8
-; SSE4-NEXT:  # %bb.7: # %cond.store5
-; SSE4-NEXT:    pextrb $6, %xmm0, 3(%rdi)
-; SSE4-NEXT:  .LBB2_8: # %else6
-; SSE4-NEXT:    pxor %xmm2, %xmm2
-; SSE4-NEXT:    pcmpeqd %xmm5, %xmm2
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE4-NEXT:    pxor %xmm2, %xmm1
-; SSE4-NEXT:    pextrb $0, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:  .LBB2_7: # %cond.store5
+; SSE4-NEXT:    pextrb $3, %xmm0, 3(%rdi)
+; SSE4-NEXT:    testb $16, %al
 ; SSE4-NEXT:    je .LBB2_10
-; SSE4-NEXT:  # %bb.9: # %cond.store7
-; SSE4-NEXT:    pextrb $8, %xmm0, 4(%rdi)
-; SSE4-NEXT:  .LBB2_10: # %else8
-; SSE4-NEXT:    pextrb $4, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:  .LBB2_9: # %cond.store7
+; SSE4-NEXT:    pextrb $4, %xmm0, 4(%rdi)
+; SSE4-NEXT:    testb $32, %al
 ; SSE4-NEXT:    je .LBB2_12
-; SSE4-NEXT:  # %bb.11: # %cond.store9
-; SSE4-NEXT:    pextrb $10, %xmm0, 5(%rdi)
-; SSE4-NEXT:  .LBB2_12: # %else10
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm5
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE4-NEXT:    pxor %xmm5, %xmm1
-; SSE4-NEXT:    pextrb $8, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:  .LBB2_11: # %cond.store9
+; SSE4-NEXT:    pextrb $5, %xmm0, 5(%rdi)
+; SSE4-NEXT:    testb $64, %al
 ; SSE4-NEXT:    je .LBB2_14
-; SSE4-NEXT:  # %bb.13: # %cond.store11
-; SSE4-NEXT:    pextrb $12, %xmm0, 6(%rdi)
-; SSE4-NEXT:  .LBB2_14: # %else12
-; SSE4-NEXT:    pextrb $12, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:  .LBB2_13: # %cond.store11
+; SSE4-NEXT:    pextrb $6, %xmm0, 6(%rdi)
+; SSE4-NEXT:    testb $-128, %al
 ; SSE4-NEXT:    je .LBB2_16
-; SSE4-NEXT:  # %bb.15: # %cond.store13
-; SSE4-NEXT:    pextrb $14, %xmm0, 7(%rdi)
-; SSE4-NEXT:  .LBB2_16: # %else14
+; SSE4-NEXT:  .LBB2_15: # %cond.store13
+; SSE4-NEXT:    pextrb $7, %xmm0, 7(%rdi)
 ; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: truncstore_v8i64_v8i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm5
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX1-NEXT:    vmovaps {{.*#+}} ymm6 = [65535,65535,65535,65535]
-; AVX1-NEXT:    vandps %ymm6, %ymm1, %ymm1
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm7
-; AVX1-NEXT:    vpackusdw %xmm7, %xmm1, %xmm1
-; AVX1-NEXT:    vandps %ymm6, %ymm0, %ymm0
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm6
-; AVX1-NEXT:    vpackusdw %xmm6, %xmm0, %xmm0
+; AVX1-NEXT:    vmovaps {{.*#+}} ymm3 = [255,255,255,255]
+; AVX1-NEXT:    vandps %ymm3, %ymm1, %ymm1
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm4
+; AVX1-NEXT:    vpackusdw %xmm4, %xmm1, %xmm1
+; AVX1-NEXT:    vandps %ymm3, %ymm0, %ymm0
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; AVX1-NEXT:    vpackusdw %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpextrb $0, %xmm5, %eax
+; AVX1-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm1
+; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
+; AVX1-NEXT:    vmovmskps %ymm1, %eax
+; AVX1-NEXT:    notl %eax
 ; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je .LBB2_2
-; AVX1-NEXT:  # %bb.1: # %cond.store
+; AVX1-NEXT:    jne .LBB2_1
+; AVX1-NEXT:  # %bb.2: # %else
+; AVX1-NEXT:    testb $2, %al
+; AVX1-NEXT:    jne .LBB2_3
+; AVX1-NEXT:  .LBB2_4: # %else2
+; AVX1-NEXT:    testb $4, %al
+; AVX1-NEXT:    jne .LBB2_5
+; AVX1-NEXT:  .LBB2_6: # %else4
+; AVX1-NEXT:    testb $8, %al
+; AVX1-NEXT:    jne .LBB2_7
+; AVX1-NEXT:  .LBB2_8: # %else6
+; AVX1-NEXT:    testb $16, %al
+; AVX1-NEXT:    jne .LBB2_9
+; AVX1-NEXT:  .LBB2_10: # %else8
+; AVX1-NEXT:    testb $32, %al
+; AVX1-NEXT:    jne .LBB2_11
+; AVX1-NEXT:  .LBB2_12: # %else10
+; AVX1-NEXT:    testb $64, %al
+; AVX1-NEXT:    jne .LBB2_13
+; AVX1-NEXT:  .LBB2_14: # %else12
+; AVX1-NEXT:    testb $-128, %al
+; AVX1-NEXT:    jne .LBB2_15
+; AVX1-NEXT:  .LBB2_16: # %else14
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+; AVX1-NEXT:  .LBB2_1: # %cond.store
 ; AVX1-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX1-NEXT:  .LBB2_2: # %else
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm4, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $4, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $2, %al
 ; AVX1-NEXT:    je .LBB2_4
-; AVX1-NEXT:  # %bb.3: # %cond.store1
-; AVX1-NEXT:    vpextrb $2, %xmm0, 1(%rdi)
-; AVX1-NEXT:  .LBB2_4: # %else2
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm2, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm4, %xmm4
-; AVX1-NEXT:    vpextrb $8, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:  .LBB2_3: # %cond.store1
+; AVX1-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
+; AVX1-NEXT:    testb $4, %al
 ; AVX1-NEXT:    je .LBB2_6
-; AVX1-NEXT:  # %bb.5: # %cond.store3
-; AVX1-NEXT:    vpextrb $4, %xmm0, 2(%rdi)
-; AVX1-NEXT:  .LBB2_6: # %else4
-; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:  .LBB2_5: # %cond.store3
+; AVX1-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
+; AVX1-NEXT:    testb $8, %al
 ; AVX1-NEXT:    je .LBB2_8
-; AVX1-NEXT:  # %bb.7: # %cond.store5
-; AVX1-NEXT:    vpextrb $6, %xmm0, 3(%rdi)
-; AVX1-NEXT:  .LBB2_8: # %else6
-; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $0, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:  .LBB2_7: # %cond.store5
+; AVX1-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
+; AVX1-NEXT:    testb $16, %al
 ; AVX1-NEXT:    je .LBB2_10
-; AVX1-NEXT:  # %bb.9: # %cond.store7
-; AVX1-NEXT:    vpextrb $8, %xmm0, 4(%rdi)
-; AVX1-NEXT:  .LBB2_10: # %else8
-; AVX1-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:  .LBB2_9: # %cond.store7
+; AVX1-NEXT:    vpextrb $4, %xmm0, 4(%rdi)
+; AVX1-NEXT:    testb $32, %al
 ; AVX1-NEXT:    je .LBB2_12
-; AVX1-NEXT:  # %bb.11: # %cond.store9
-; AVX1-NEXT:    vpextrb $10, %xmm0, 5(%rdi)
-; AVX1-NEXT:  .LBB2_12: # %else10
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:  .LBB2_11: # %cond.store9
+; AVX1-NEXT:    vpextrb $5, %xmm0, 5(%rdi)
+; AVX1-NEXT:    testb $64, %al
 ; AVX1-NEXT:    je .LBB2_14
-; AVX1-NEXT:  # %bb.13: # %cond.store11
-; AVX1-NEXT:    vpextrb $12, %xmm0, 6(%rdi)
-; AVX1-NEXT:  .LBB2_14: # %else12
-; AVX1-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:  .LBB2_13: # %cond.store11
+; AVX1-NEXT:    vpextrb $6, %xmm0, 6(%rdi)
+; AVX1-NEXT:    testb $-128, %al
 ; AVX1-NEXT:    je .LBB2_16
-; AVX1-NEXT:  # %bb.15: # %cond.store13
-; AVX1-NEXT:    vpextrb $14, %xmm0, 7(%rdi)
-; AVX1-NEXT:  .LBB2_16: # %else14
+; AVX1-NEXT:  .LBB2_15: # %cond.store13
+; AVX1-NEXT:    vpextrb $7, %xmm0, 7(%rdi)
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: truncstore_v8i64_v8i8:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm5
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX2-NEXT:    vextractf128 $1, %ymm1, %xmm6
-; AVX2-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[0,2],xmm6[0,2]
-; AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm6
-; AVX2-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm6[0,2]
-; AVX2-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-NEXT:    vpextrb $0, %xmm5, %eax
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm4
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm5 = <u,u,0,8,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX2-NEXT:    vpshufb %xmm5, %xmm4, %xmm4
+; AVX2-NEXT:    vpshufb %xmm5, %xmm1, %xmm1
+; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3]
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm4
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm5 = <0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX2-NEXT:    vpshufb %xmm5, %xmm4, %xmm4
+; AVX2-NEXT:    vpshufb %xmm5, %xmm0, %xmm0
+; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
+; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3]
+; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm2, %ymm1
+; AVX2-NEXT:    vmovmskps %ymm1, %eax
+; AVX2-NEXT:    notl %eax
 ; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je .LBB2_2
-; AVX2-NEXT:  # %bb.1: # %cond.store
+; AVX2-NEXT:    jne .LBB2_1
+; AVX2-NEXT:  # %bb.2: # %else
+; AVX2-NEXT:    testb $2, %al
+; AVX2-NEXT:    jne .LBB2_3
+; AVX2-NEXT:  .LBB2_4: # %else2
+; AVX2-NEXT:    testb $4, %al
+; AVX2-NEXT:    jne .LBB2_5
+; AVX2-NEXT:  .LBB2_6: # %else4
+; AVX2-NEXT:    testb $8, %al
+; AVX2-NEXT:    jne .LBB2_7
+; AVX2-NEXT:  .LBB2_8: # %else6
+; AVX2-NEXT:    testb $16, %al
+; AVX2-NEXT:    jne .LBB2_9
+; AVX2-NEXT:  .LBB2_10: # %else8
+; AVX2-NEXT:    testb $32, %al
+; AVX2-NEXT:    jne .LBB2_11
+; AVX2-NEXT:  .LBB2_12: # %else10
+; AVX2-NEXT:    testb $64, %al
+; AVX2-NEXT:    jne .LBB2_13
+; AVX2-NEXT:  .LBB2_14: # %else12
+; AVX2-NEXT:    testb $-128, %al
+; AVX2-NEXT:    jne .LBB2_15
+; AVX2-NEXT:  .LBB2_16: # %else14
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+; AVX2-NEXT:  .LBB2_1: # %cond.store
 ; AVX2-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX2-NEXT:  .LBB2_2: # %else
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm1
-; AVX2-NEXT:    vpxor %xmm4, %xmm1, %xmm1
-; AVX2-NEXT:    vpextrb $4, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $2, %al
 ; AVX2-NEXT:    je .LBB2_4
-; AVX2-NEXT:  # %bb.3: # %cond.store1
-; AVX2-NEXT:    vpextrb $2, %xmm0, 1(%rdi)
-; AVX2-NEXT:  .LBB2_4: # %else2
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm2, %xmm4
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    vpxor %xmm3, %xmm4, %xmm4
-; AVX2-NEXT:    vpextrb $8, %xmm4, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:  .LBB2_3: # %cond.store1
+; AVX2-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
+; AVX2-NEXT:    testb $4, %al
 ; AVX2-NEXT:    je .LBB2_6
-; AVX2-NEXT:  # %bb.5: # %cond.store3
-; AVX2-NEXT:    vpextrb $4, %xmm0, 2(%rdi)
-; AVX2-NEXT:  .LBB2_6: # %else4
-; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm2, %xmm1
-; AVX2-NEXT:    vpxor %xmm3, %xmm1, %xmm1
-; AVX2-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:  .LBB2_5: # %cond.store3
+; AVX2-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
+; AVX2-NEXT:    testb $8, %al
 ; AVX2-NEXT:    je .LBB2_8
-; AVX2-NEXT:  # %bb.7: # %cond.store5
-; AVX2-NEXT:    vpextrb $6, %xmm0, 3(%rdi)
-; AVX2-NEXT:  .LBB2_8: # %else6
-; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm1
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrb $0, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:  .LBB2_7: # %cond.store5
+; AVX2-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
+; AVX2-NEXT:    testb $16, %al
 ; AVX2-NEXT:    je .LBB2_10
-; AVX2-NEXT:  # %bb.9: # %cond.store7
-; AVX2-NEXT:    vpextrb $8, %xmm0, 4(%rdi)
-; AVX2-NEXT:  .LBB2_10: # %else8
-; AVX2-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:  .LBB2_9: # %cond.store7
+; AVX2-NEXT:    vpextrb $4, %xmm0, 4(%rdi)
+; AVX2-NEXT:    testb $32, %al
 ; AVX2-NEXT:    je .LBB2_12
-; AVX2-NEXT:  # %bb.11: # %cond.store9
-; AVX2-NEXT:    vpextrb $10, %xmm0, 5(%rdi)
-; AVX2-NEXT:  .LBB2_12: # %else10
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:  .LBB2_11: # %cond.store9
+; AVX2-NEXT:    vpextrb $5, %xmm0, 5(%rdi)
+; AVX2-NEXT:    testb $64, %al
 ; AVX2-NEXT:    je .LBB2_14
-; AVX2-NEXT:  # %bb.13: # %cond.store11
-; AVX2-NEXT:    vpextrb $12, %xmm0, 6(%rdi)
-; AVX2-NEXT:  .LBB2_14: # %else12
-; AVX2-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:  .LBB2_13: # %cond.store11
+; AVX2-NEXT:    vpextrb $6, %xmm0, 6(%rdi)
+; AVX2-NEXT:    testb $-128, %al
 ; AVX2-NEXT:    je .LBB2_16
-; AVX2-NEXT:  # %bb.15: # %cond.store13
-; AVX2-NEXT:    vpextrb $14, %xmm0, 7(%rdi)
-; AVX2-NEXT:  .LBB2_16: # %else14
+; AVX2-NEXT:  .LBB2_15: # %cond.store13
+; AVX2-NEXT:    vpextrb $7, %xmm0, 7(%rdi)
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
@@ -977,69 +924,64 @@ define void @truncstore_v8i64_v8i8(<8 x i64> %x, <8 x i8>* %p, <8 x i32> %mask)
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
 ; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    vpmovqw %zmm0, %xmm0
+; AVX512F-NEXT:    vpmovqb %zmm0, %xmm0
 ; AVX512F-NEXT:    kmovw %k0, %eax
 ; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB2_2
-; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    jne .LBB2_1
+; AVX512F-NEXT:  # %bb.2: # %else
+; AVX512F-NEXT:    testb $2, %al
+; AVX512F-NEXT:    jne .LBB2_3
+; AVX512F-NEXT:  .LBB2_4: # %else2
+; AVX512F-NEXT:    testb $4, %al
+; AVX512F-NEXT:    jne .LBB2_5
+; AVX512F-NEXT:  .LBB2_6: # %else4
+; AVX512F-NEXT:    testb $8, %al
+; AVX512F-NEXT:    jne .LBB2_7
+; AVX512F-NEXT:  .LBB2_8: # %else6
+; AVX512F-NEXT:    testb $16, %al
+; AVX512F-NEXT:    jne .LBB2_9
+; AVX512F-NEXT:  .LBB2_10: # %else8
+; AVX512F-NEXT:    testb $32, %al
+; AVX512F-NEXT:    jne .LBB2_11
+; AVX512F-NEXT:  .LBB2_12: # %else10
+; AVX512F-NEXT:    testb $64, %al
+; AVX512F-NEXT:    jne .LBB2_13
+; AVX512F-NEXT:  .LBB2_14: # %else12
+; AVX512F-NEXT:    testb $-128, %al
+; AVX512F-NEXT:    jne .LBB2_15
+; AVX512F-NEXT:  .LBB2_16: # %else14
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+; AVX512F-NEXT:  .LBB2_1: # %cond.store
 ; AVX512F-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX512F-NEXT:  .LBB2_2: # %else
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $2, %al
 ; AVX512F-NEXT:    je .LBB2_4
-; AVX512F-NEXT:  # %bb.3: # %cond.store1
-; AVX512F-NEXT:    vpextrb $2, %xmm0, 1(%rdi)
-; AVX512F-NEXT:  .LBB2_4: # %else2
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  .LBB2_3: # %cond.store1
+; AVX512F-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
+; AVX512F-NEXT:    testb $4, %al
 ; AVX512F-NEXT:    je .LBB2_6
-; AVX512F-NEXT:  # %bb.5: # %cond.store3
-; AVX512F-NEXT:    vpextrb $4, %xmm0, 2(%rdi)
-; AVX512F-NEXT:  .LBB2_6: # %else4
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  .LBB2_5: # %cond.store3
+; AVX512F-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
+; AVX512F-NEXT:    testb $8, %al
 ; AVX512F-NEXT:    je .LBB2_8
-; AVX512F-NEXT:  # %bb.7: # %cond.store5
-; AVX512F-NEXT:    vpextrb $6, %xmm0, 3(%rdi)
-; AVX512F-NEXT:  .LBB2_8: # %else6
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $4, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  .LBB2_7: # %cond.store5
+; AVX512F-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
+; AVX512F-NEXT:    testb $16, %al
 ; AVX512F-NEXT:    je .LBB2_10
-; AVX512F-NEXT:  # %bb.9: # %cond.store7
-; AVX512F-NEXT:    vpextrb $8, %xmm0, 4(%rdi)
-; AVX512F-NEXT:  .LBB2_10: # %else8
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $5, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  .LBB2_9: # %cond.store7
+; AVX512F-NEXT:    vpextrb $4, %xmm0, 4(%rdi)
+; AVX512F-NEXT:    testb $32, %al
 ; AVX512F-NEXT:    je .LBB2_12
-; AVX512F-NEXT:  # %bb.11: # %cond.store9
-; AVX512F-NEXT:    vpextrb $10, %xmm0, 5(%rdi)
-; AVX512F-NEXT:  .LBB2_12: # %else10
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $6, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  .LBB2_11: # %cond.store9
+; AVX512F-NEXT:    vpextrb $5, %xmm0, 5(%rdi)
+; AVX512F-NEXT:    testb $64, %al
 ; AVX512F-NEXT:    je .LBB2_14
-; AVX512F-NEXT:  # %bb.13: # %cond.store11
-; AVX512F-NEXT:    vpextrb $12, %xmm0, 6(%rdi)
-; AVX512F-NEXT:  .LBB2_14: # %else12
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $7, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  .LBB2_13: # %cond.store11
+; AVX512F-NEXT:    vpextrb $6, %xmm0, 6(%rdi)
+; AVX512F-NEXT:    testb $-128, %al
 ; AVX512F-NEXT:    je .LBB2_16
-; AVX512F-NEXT:  # %bb.15: # %cond.store13
-; AVX512F-NEXT:    vpextrb $14, %xmm0, 7(%rdi)
-; AVX512F-NEXT:  .LBB2_16: # %else14
+; AVX512F-NEXT:  .LBB2_15: # %cond.store13
+; AVX512F-NEXT:    vpextrb $7, %xmm0, 7(%rdi)
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
@@ -1067,80 +1009,76 @@ define void @truncstore_v4i64_v4i32(<4 x i64> %x, <4 x i32>* %p, <4 x i32> %mask
 ; SSE2-LABEL: truncstore_v4i64_v4i32:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm3
 ; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
-; SSE2-NEXT:    movd %xmm3, %eax
-; SSE2-NEXT:    notl %eax
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm3
+; SSE2-NEXT:    movmskps %xmm3, %eax
+; SSE2-NEXT:    xorl $15, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB3_2
-; SSE2-NEXT:  # %bb.1: # %cond.store
+; SSE2-NEXT:    jne .LBB3_1
+; SSE2-NEXT:  # %bb.2: # %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne .LBB3_3
+; SSE2-NEXT:  .LBB3_4: # %else2
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne .LBB3_5
+; SSE2-NEXT:  .LBB3_6: # %else4
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne .LBB3_7
+; SSE2-NEXT:  .LBB3_8: # %else6
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  .LBB3_1: # %cond.store
 ; SSE2-NEXT:    movss %xmm0, (%rdi)
-; SSE2-NEXT:  .LBB3_2: # %else
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pxor %xmm1, %xmm3
-; SSE2-NEXT:    pextrw $2, %xmm3, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $2, %al
 ; SSE2-NEXT:    je .LBB3_4
-; SSE2-NEXT:  # %bb.3: # %cond.store1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,2,3]
-; SSE2-NEXT:    movd %xmm3, 4(%rdi)
-; SSE2-NEXT:  .LBB3_4: # %else2
-; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm2
-; SSE2-NEXT:    pxor %xmm2, %xmm1
-; SSE2-NEXT:    pextrw $4, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:  .LBB3_3: # %cond.store1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE2-NEXT:    movd %xmm1, 4(%rdi)
+; SSE2-NEXT:    testb $4, %al
 ; SSE2-NEXT:    je .LBB3_6
-; SSE2-NEXT:  # %bb.5: # %cond.store3
+; SSE2-NEXT:  .LBB3_5: # %cond.store3
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; SSE2-NEXT:    movd %xmm1, 8(%rdi)
-; SSE2-NEXT:  .LBB3_6: # %else4
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pxor %xmm1, %xmm2
-; SSE2-NEXT:    pextrw $6, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je .LBB3_8
-; SSE2-NEXT:  # %bb.7: # %cond.store5
+; SSE2-NEXT:  .LBB3_7: # %cond.store5
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
 ; SSE2-NEXT:    movd %xmm0, 12(%rdi)
-; SSE2-NEXT:  .LBB3_8: # %else6
 ; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: truncstore_v4i64_v4i32:
 ; SSE4:       # %bb.0:
-; SSE4-NEXT:    pxor %xmm4, %xmm4
-; SSE4-NEXT:    pcmpeqd %xmm2, %xmm4
-; SSE4-NEXT:    pcmpeqd %xmm3, %xmm3
-; SSE4-NEXT:    pxor %xmm4, %xmm3
+; SSE4-NEXT:    pxor %xmm3, %xmm3
 ; SSE4-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
-; SSE4-NEXT:    pextrb $0, %xmm3, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB3_2
-; SSE4-NEXT:  # %bb.1: # %cond.store
+; SSE4-NEXT:    pcmpeqd %xmm2, %xmm3
+; SSE4-NEXT:    movmskps %xmm3, %eax
+; SSE4-NEXT:    xorl $15, %eax
+; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    jne .LBB3_1
+; SSE4-NEXT:  # %bb.2: # %else
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    jne .LBB3_3
+; SSE4-NEXT:  .LBB3_4: # %else2
+; SSE4-NEXT:    testb $4, %al
+; SSE4-NEXT:    jne .LBB3_5
+; SSE4-NEXT:  .LBB3_6: # %else4
+; SSE4-NEXT:    testb $8, %al
+; SSE4-NEXT:    jne .LBB3_7
+; SSE4-NEXT:  .LBB3_8: # %else6
+; SSE4-NEXT:    retq
+; SSE4-NEXT:  .LBB3_1: # %cond.store
 ; SSE4-NEXT:    movss %xmm0, (%rdi)
-; SSE4-NEXT:  .LBB3_2: # %else
-; SSE4-NEXT:    pextrb $4, %xmm3, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $2, %al
 ; SSE4-NEXT:    je .LBB3_4
-; SSE4-NEXT:  # %bb.3: # %cond.store1
+; SSE4-NEXT:  .LBB3_3: # %cond.store1
 ; SSE4-NEXT:    extractps $1, %xmm0, 4(%rdi)
-; SSE4-NEXT:  .LBB3_4: # %else2
-; SSE4-NEXT:    xorps %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm2
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE4-NEXT:    pxor %xmm2, %xmm1
-; SSE4-NEXT:    pextrb $8, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $4, %al
 ; SSE4-NEXT:    je .LBB3_6
-; SSE4-NEXT:  # %bb.5: # %cond.store3
+; SSE4-NEXT:  .LBB3_5: # %cond.store3
 ; SSE4-NEXT:    extractps $2, %xmm0, 8(%rdi)
-; SSE4-NEXT:  .LBB3_6: # %else4
-; SSE4-NEXT:    pextrb $12, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $8, %al
 ; SSE4-NEXT:    je .LBB3_8
-; SSE4-NEXT:  # %bb.7: # %cond.store5
+; SSE4-NEXT:  .LBB3_7: # %cond.store5
 ; SSE4-NEXT:    extractps $3, %xmm0, 12(%rdi)
-; SSE4-NEXT:  .LBB3_8: # %else6
 ; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: truncstore_v4i64_v4i32:
@@ -1207,158 +1145,208 @@ define void @truncstore_v4i64_v4i16(<4 x i64> %x, <4 x i16>* %p, <4 x i32> %mask
 ; SSE2-LABEL: truncstore_v4i64_v4i16:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; SSE2-NEXT:    pcmpeqd %xmm2, %xmm3
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
-; SSE2-NEXT:    movd %xmm3, %eax
-; SSE2-NEXT:    notl %eax
+; SSE2-NEXT:    movmskps %xmm3, %eax
+; SSE2-NEXT:    xorl $15, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB4_2
-; SSE2-NEXT:  # %bb.1: # %cond.store
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    movw %ax, (%rdi)
-; SSE2-NEXT:  .LBB4_2: # %else
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pxor %xmm1, %xmm3
-; SSE2-NEXT:    pextrw $2, %xmm3, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB4_4
-; SSE2-NEXT:  # %bb.3: # %cond.store1
-; SSE2-NEXT:    pextrw $2, %xmm0, %eax
-; SSE2-NEXT:    movw %ax, 2(%rdi)
+; SSE2-NEXT:    jne .LBB4_1
+; SSE2-NEXT:  # %bb.2: # %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne .LBB4_3
 ; SSE2-NEXT:  .LBB4_4: # %else2
-; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm2
-; SSE2-NEXT:    pxor %xmm2, %xmm1
-; SSE2-NEXT:    pextrw $4, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB4_6
-; SSE2-NEXT:  # %bb.5: # %cond.store3
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    movw %ax, 4(%rdi)
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne .LBB4_5
 ; SSE2-NEXT:  .LBB4_6: # %else4
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pxor %xmm1, %xmm2
-; SSE2-NEXT:    pextrw $6, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB4_8
-; SSE2-NEXT:  # %bb.7: # %cond.store5
-; SSE2-NEXT:    pextrw $6, %xmm0, %eax
-; SSE2-NEXT:    movw %ax, 6(%rdi)
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne .LBB4_7
 ; SSE2-NEXT:  .LBB4_8: # %else6
 ; SSE2-NEXT:    retq
-;
+; SSE2-NEXT:  .LBB4_1: # %cond.store
+; SSE2-NEXT:    movd %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, (%rdi)
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    je .LBB4_4
+; SSE2-NEXT:  .LBB4_3: # %cond.store1
+; SSE2-NEXT:    pextrw $1, %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, 2(%rdi)
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    je .LBB4_6
+; SSE2-NEXT:  .LBB4_5: # %cond.store3
+; SSE2-NEXT:    pextrw $2, %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, 4(%rdi)
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    je .LBB4_8
+; SSE2-NEXT:  .LBB4_7: # %cond.store5
+; SSE2-NEXT:    pextrw $3, %xmm0, %eax
+; SSE2-NEXT:    movw %ax, 6(%rdi)
+; SSE2-NEXT:    retq
+;
 ; SSE4-LABEL: truncstore_v4i64_v4i16:
 ; SSE4:       # %bb.0:
-; SSE4-NEXT:    pxor %xmm4, %xmm4
-; SSE4-NEXT:    pcmpeqd %xmm2, %xmm4
-; SSE4-NEXT:    pcmpeqd %xmm3, %xmm3
-; SSE4-NEXT:    pxor %xmm4, %xmm3
-; SSE4-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
-; SSE4-NEXT:    pextrb $0, %xmm3, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB4_2
-; SSE4-NEXT:  # %bb.1: # %cond.store
+; SSE4-NEXT:    pxor %xmm3, %xmm3
+; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; SSE4-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
+; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSE4-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; SSE4-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE4-NEXT:    pcmpeqd %xmm2, %xmm3
+; SSE4-NEXT:    movmskps %xmm3, %eax
+; SSE4-NEXT:    xorl $15, %eax
+; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    jne .LBB4_1
+; SSE4-NEXT:  # %bb.2: # %else
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    jne .LBB4_3
+; SSE4-NEXT:  .LBB4_4: # %else2
+; SSE4-NEXT:    testb $4, %al
+; SSE4-NEXT:    jne .LBB4_5
+; SSE4-NEXT:  .LBB4_6: # %else4
+; SSE4-NEXT:    testb $8, %al
+; SSE4-NEXT:    jne .LBB4_7
+; SSE4-NEXT:  .LBB4_8: # %else6
+; SSE4-NEXT:    retq
+; SSE4-NEXT:  .LBB4_1: # %cond.store
 ; SSE4-NEXT:    pextrw $0, %xmm0, (%rdi)
-; SSE4-NEXT:  .LBB4_2: # %else
-; SSE4-NEXT:    pextrb $4, %xmm3, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $2, %al
 ; SSE4-NEXT:    je .LBB4_4
-; SSE4-NEXT:  # %bb.3: # %cond.store1
-; SSE4-NEXT:    pextrw $2, %xmm0, 2(%rdi)
-; SSE4-NEXT:  .LBB4_4: # %else2
-; SSE4-NEXT:    xorps %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm2
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE4-NEXT:    pxor %xmm2, %xmm1
-; SSE4-NEXT:    pextrb $8, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:  .LBB4_3: # %cond.store1
+; SSE4-NEXT:    pextrw $1, %xmm0, 2(%rdi)
+; SSE4-NEXT:    testb $4, %al
 ; SSE4-NEXT:    je .LBB4_6
-; SSE4-NEXT:  # %bb.5: # %cond.store3
-; SSE4-NEXT:    pextrw $4, %xmm0, 4(%rdi)
-; SSE4-NEXT:  .LBB4_6: # %else4
-; SSE4-NEXT:    pextrb $12, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:  .LBB4_5: # %cond.store3
+; SSE4-NEXT:    pextrw $2, %xmm0, 4(%rdi)
+; SSE4-NEXT:    testb $8, %al
 ; SSE4-NEXT:    je .LBB4_8
-; SSE4-NEXT:  # %bb.7: # %cond.store5
-; SSE4-NEXT:    pextrw $6, %xmm0, 6(%rdi)
-; SSE4-NEXT:  .LBB4_8: # %else6
+; SSE4-NEXT:  .LBB4_7: # %cond.store5
+; SSE4-NEXT:    pextrw $3, %xmm0, 6(%rdi)
 ; SSE4-NEXT:    retq
 ;
-; AVX-LABEL: truncstore_v4i64_v4i16:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm3
-; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm3[0,2]
-; AVX-NEXT:    vpextrb $0, %xmm2, %eax
-; AVX-NEXT:    testb $1, %al
-; AVX-NEXT:    je .LBB4_2
-; AVX-NEXT:  # %bb.1: # %cond.store
-; AVX-NEXT:    vpextrw $0, %xmm0, (%rdi)
-; AVX-NEXT:  .LBB4_2: # %else
-; AVX-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX-NEXT:    testb $1, %al
-; AVX-NEXT:    je .LBB4_4
-; AVX-NEXT:  # %bb.3: # %cond.store1
-; AVX-NEXT:    vpextrw $2, %xmm0, 2(%rdi)
-; AVX-NEXT:  .LBB4_4: # %else2
-; AVX-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX-NEXT:    testb $1, %al
-; AVX-NEXT:    je .LBB4_6
-; AVX-NEXT:  # %bb.5: # %cond.store3
-; AVX-NEXT:    vpextrw $4, %xmm0, 4(%rdi)
-; AVX-NEXT:  .LBB4_6: # %else4
-; AVX-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX-NEXT:    testb $1, %al
-; AVX-NEXT:    je .LBB4_8
-; AVX-NEXT:  # %bb.7: # %cond.store5
-; AVX-NEXT:    vpextrw $6, %xmm0, 6(%rdi)
-; AVX-NEXT:  .LBB4_8: # %else6
-; AVX-NEXT:    vzeroupper
-; AVX-NEXT:    retq
+; AVX1-LABEL: truncstore_v4i64_v4i16:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm3[0,2,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vmovmskps %xmm1, %eax
+; AVX1-NEXT:    xorl $15, %eax
+; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    jne .LBB4_1
+; AVX1-NEXT:  # %bb.2: # %else
+; AVX1-NEXT:    testb $2, %al
+; AVX1-NEXT:    jne .LBB4_3
+; AVX1-NEXT:  .LBB4_4: # %else2
+; AVX1-NEXT:    testb $4, %al
+; AVX1-NEXT:    jne .LBB4_5
+; AVX1-NEXT:  .LBB4_6: # %else4
+; AVX1-NEXT:    testb $8, %al
+; AVX1-NEXT:    jne .LBB4_7
+; AVX1-NEXT:  .LBB4_8: # %else6
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+; AVX1-NEXT:  .LBB4_1: # %cond.store
+; AVX1-NEXT:    vpextrw $0, %xmm0, (%rdi)
+; AVX1-NEXT:    testb $2, %al
+; AVX1-NEXT:    je .LBB4_4
+; AVX1-NEXT:  .LBB4_3: # %cond.store1
+; AVX1-NEXT:    vpextrw $1, %xmm0, 2(%rdi)
+; AVX1-NEXT:    testb $4, %al
+; AVX1-NEXT:    je .LBB4_6
+; AVX1-NEXT:  .LBB4_5: # %cond.store3
+; AVX1-NEXT:    vpextrw $2, %xmm0, 4(%rdi)
+; AVX1-NEXT:    testb $8, %al
+; AVX1-NEXT:    je .LBB4_8
+; AVX1-NEXT:  .LBB4_7: # %cond.store5
+; AVX1-NEXT:    vpextrw $3, %xmm0, 6(%rdi)
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: truncstore_v4i64_v4i16:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
+; AVX2-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm3[0,2,2,3,4,5,6,7]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX2-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vmovmskps %xmm1, %eax
+; AVX2-NEXT:    xorl $15, %eax
+; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    jne .LBB4_1
+; AVX2-NEXT:  # %bb.2: # %else
+; AVX2-NEXT:    testb $2, %al
+; AVX2-NEXT:    jne .LBB4_3
+; AVX2-NEXT:  .LBB4_4: # %else2
+; AVX2-NEXT:    testb $4, %al
+; AVX2-NEXT:    jne .LBB4_5
+; AVX2-NEXT:  .LBB4_6: # %else4
+; AVX2-NEXT:    testb $8, %al
+; AVX2-NEXT:    jne .LBB4_7
+; AVX2-NEXT:  .LBB4_8: # %else6
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+; AVX2-NEXT:  .LBB4_1: # %cond.store
+; AVX2-NEXT:    vpextrw $0, %xmm0, (%rdi)
+; AVX2-NEXT:    testb $2, %al
+; AVX2-NEXT:    je .LBB4_4
+; AVX2-NEXT:  .LBB4_3: # %cond.store1
+; AVX2-NEXT:    vpextrw $1, %xmm0, 2(%rdi)
+; AVX2-NEXT:    testb $4, %al
+; AVX2-NEXT:    je .LBB4_6
+; AVX2-NEXT:  .LBB4_5: # %cond.store3
+; AVX2-NEXT:    vpextrw $2, %xmm0, 4(%rdi)
+; AVX2-NEXT:    testb $8, %al
+; AVX2-NEXT:    je .LBB4_8
+; AVX2-NEXT:  .LBB4_7: # %cond.store5
+; AVX2-NEXT:    vpextrw $3, %xmm0, 6(%rdi)
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: truncstore_v4i64_v4i16:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
 ; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
 ; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    vpmovqd %zmm0, %ymm0
+; AVX512F-NEXT:    vpmovqw %zmm0, %xmm0
 ; AVX512F-NEXT:    kmovw %k0, %eax
 ; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB4_2
-; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    jne .LBB4_1
+; AVX512F-NEXT:  # %bb.2: # %else
+; AVX512F-NEXT:    testb $2, %al
+; AVX512F-NEXT:    jne .LBB4_3
+; AVX512F-NEXT:  .LBB4_4: # %else2
+; AVX512F-NEXT:    testb $4, %al
+; AVX512F-NEXT:    jne .LBB4_5
+; AVX512F-NEXT:  .LBB4_6: # %else4
+; AVX512F-NEXT:    testb $8, %al
+; AVX512F-NEXT:    jne .LBB4_7
+; AVX512F-NEXT:  .LBB4_8: # %else6
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+; AVX512F-NEXT:  .LBB4_1: # %cond.store
 ; AVX512F-NEXT:    vpextrw $0, %xmm0, (%rdi)
-; AVX512F-NEXT:  .LBB4_2: # %else
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $2, %al
 ; AVX512F-NEXT:    je .LBB4_4
-; AVX512F-NEXT:  # %bb.3: # %cond.store1
-; AVX512F-NEXT:    vpextrw $2, %xmm0, 2(%rdi)
-; AVX512F-NEXT:  .LBB4_4: # %else2
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  .LBB4_3: # %cond.store1
+; AVX512F-NEXT:    vpextrw $1, %xmm0, 2(%rdi)
+; AVX512F-NEXT:    testb $4, %al
 ; AVX512F-NEXT:    je .LBB4_6
-; AVX512F-NEXT:  # %bb.5: # %cond.store3
-; AVX512F-NEXT:    vpextrw $4, %xmm0, 4(%rdi)
-; AVX512F-NEXT:  .LBB4_6: # %else4
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  .LBB4_5: # %cond.store3
+; AVX512F-NEXT:    vpextrw $2, %xmm0, 4(%rdi)
+; AVX512F-NEXT:    testb $8, %al
 ; AVX512F-NEXT:    je .LBB4_8
-; AVX512F-NEXT:  # %bb.7: # %cond.store5
-; AVX512F-NEXT:    vpextrw $6, %xmm0, 6(%rdi)
-; AVX512F-NEXT:  .LBB4_8: # %else6
+; AVX512F-NEXT:  .LBB4_7: # %cond.store5
+; AVX512F-NEXT:    vpextrw $3, %xmm0, 6(%rdi)
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
@@ -1367,10 +1355,9 @@ define void @truncstore_v4i64_v4i16(<4 x i64> %x, <4 x i16>* %p, <4 x i32> %mask
 ; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
 ; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
 ; AVX512BW-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512BW-NEXT:    vpmovqd %zmm0, %ymm0
-; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
 ; AVX512BW-NEXT:    kshiftld $28, %k0, %k0
 ; AVX512BW-NEXT:    kshiftrd $28, %k0, %k1
+; AVX512BW-NEXT:    vpmovqw %zmm0, %xmm0
 ; AVX512BW-NEXT:    vmovdqu16 %zmm0, (%rdi) {%k1}
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -1391,158 +1378,206 @@ define void @truncstore_v4i64_v4i8(<4 x i64> %x, <4 x i8>* %p, <4 x i32> %mask)
 ; SSE2-LABEL: truncstore_v4i64_v4i8:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [255,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0]
+; SSE2-NEXT:    pand %xmm4, %xmm1
+; SSE2-NEXT:    pand %xmm4, %xmm0
+; SSE2-NEXT:    packuswb %xmm1, %xmm0
+; SSE2-NEXT:    packuswb %xmm0, %xmm0
+; SSE2-NEXT:    packuswb %xmm0, %xmm0
 ; SSE2-NEXT:    pcmpeqd %xmm2, %xmm3
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
-; SSE2-NEXT:    movd %xmm3, %eax
-; SSE2-NEXT:    notl %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB5_2
-; SSE2-NEXT:  # %bb.1: # %cond.store
+; SSE2-NEXT:    movmskps %xmm3, %ecx
+; SSE2-NEXT:    xorl $15, %ecx
+; SSE2-NEXT:    testb $1, %cl
 ; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    jne .LBB5_1
+; SSE2-NEXT:  # %bb.2: # %else
+; SSE2-NEXT:    testb $2, %cl
+; SSE2-NEXT:    jne .LBB5_3
+; SSE2-NEXT:  .LBB5_4: # %else2
+; SSE2-NEXT:    testb $4, %cl
+; SSE2-NEXT:    jne .LBB5_5
+; SSE2-NEXT:  .LBB5_6: # %else4
+; SSE2-NEXT:    testb $8, %cl
+; SSE2-NEXT:    jne .LBB5_7
+; SSE2-NEXT:  .LBB5_8: # %else6
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  .LBB5_1: # %cond.store
 ; SSE2-NEXT:    movb %al, (%rdi)
-; SSE2-NEXT:  .LBB5_2: # %else
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pxor %xmm1, %xmm3
-; SSE2-NEXT:    pextrw $2, %xmm3, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $2, %cl
 ; SSE2-NEXT:    je .LBB5_4
-; SSE2-NEXT:  # %bb.3: # %cond.store1
-; SSE2-NEXT:    pextrw $2, %xmm0, %eax
-; SSE2-NEXT:    movb %al, 1(%rdi)
-; SSE2-NEXT:  .LBB5_4: # %else2
-; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm2
-; SSE2-NEXT:    pxor %xmm2, %xmm1
-; SSE2-NEXT:    pextrw $4, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:  .LBB5_3: # %cond.store1
+; SSE2-NEXT:    movb %ah, 1(%rdi)
+; SSE2-NEXT:    testb $4, %cl
 ; SSE2-NEXT:    je .LBB5_6
-; SSE2-NEXT:  # %bb.5: # %cond.store3
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    movb %al, 2(%rdi)
-; SSE2-NEXT:  .LBB5_6: # %else4
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pxor %xmm1, %xmm2
-; SSE2-NEXT:    pextrw $6, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:  .LBB5_5: # %cond.store3
+; SSE2-NEXT:    movl %eax, %edx
+; SSE2-NEXT:    shrl $16, %edx
+; SSE2-NEXT:    movb %dl, 2(%rdi)
+; SSE2-NEXT:    testb $8, %cl
 ; SSE2-NEXT:    je .LBB5_8
-; SSE2-NEXT:  # %bb.7: # %cond.store5
-; SSE2-NEXT:    pextrw $6, %xmm0, %eax
+; SSE2-NEXT:  .LBB5_7: # %cond.store5
+; SSE2-NEXT:    shrl $24, %eax
 ; SSE2-NEXT:    movb %al, 3(%rdi)
-; SSE2-NEXT:  .LBB5_8: # %else6
 ; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: truncstore_v4i64_v4i8:
 ; SSE4:       # %bb.0:
-; SSE4-NEXT:    pxor %xmm4, %xmm4
-; SSE4-NEXT:    pcmpeqd %xmm2, %xmm4
-; SSE4-NEXT:    pcmpeqd %xmm3, %xmm3
-; SSE4-NEXT:    pxor %xmm4, %xmm3
-; SSE4-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
-; SSE4-NEXT:    pextrb $0, %xmm3, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB5_2
-; SSE4-NEXT:  # %bb.1: # %cond.store
+; SSE4-NEXT:    pxor %xmm3, %xmm3
+; SSE4-NEXT:    movdqa {{.*#+}} xmm4 = <0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; SSE4-NEXT:    pshufb %xmm4, %xmm1
+; SSE4-NEXT:    pshufb %xmm4, %xmm0
+; SSE4-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSE4-NEXT:    pcmpeqd %xmm2, %xmm3
+; SSE4-NEXT:    movmskps %xmm3, %eax
+; SSE4-NEXT:    xorl $15, %eax
+; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    jne .LBB5_1
+; SSE4-NEXT:  # %bb.2: # %else
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    jne .LBB5_3
+; SSE4-NEXT:  .LBB5_4: # %else2
+; SSE4-NEXT:    testb $4, %al
+; SSE4-NEXT:    jne .LBB5_5
+; SSE4-NEXT:  .LBB5_6: # %else4
+; SSE4-NEXT:    testb $8, %al
+; SSE4-NEXT:    jne .LBB5_7
+; SSE4-NEXT:  .LBB5_8: # %else6
+; SSE4-NEXT:    retq
+; SSE4-NEXT:  .LBB5_1: # %cond.store
 ; SSE4-NEXT:    pextrb $0, %xmm0, (%rdi)
-; SSE4-NEXT:  .LBB5_2: # %else
-; SSE4-NEXT:    pextrb $4, %xmm3, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $2, %al
 ; SSE4-NEXT:    je .LBB5_4
-; SSE4-NEXT:  # %bb.3: # %cond.store1
-; SSE4-NEXT:    pextrb $4, %xmm0, 1(%rdi)
-; SSE4-NEXT:  .LBB5_4: # %else2
-; SSE4-NEXT:    xorps %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm2
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE4-NEXT:    pxor %xmm2, %xmm1
-; SSE4-NEXT:    pextrb $8, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:  .LBB5_3: # %cond.store1
+; SSE4-NEXT:    pextrb $1, %xmm0, 1(%rdi)
+; SSE4-NEXT:    testb $4, %al
 ; SSE4-NEXT:    je .LBB5_6
-; SSE4-NEXT:  # %bb.5: # %cond.store3
-; SSE4-NEXT:    pextrb $8, %xmm0, 2(%rdi)
-; SSE4-NEXT:  .LBB5_6: # %else4
-; SSE4-NEXT:    pextrb $12, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:  .LBB5_5: # %cond.store3
+; SSE4-NEXT:    pextrb $2, %xmm0, 2(%rdi)
+; SSE4-NEXT:    testb $8, %al
 ; SSE4-NEXT:    je .LBB5_8
-; SSE4-NEXT:  # %bb.7: # %cond.store5
-; SSE4-NEXT:    pextrb $12, %xmm0, 3(%rdi)
-; SSE4-NEXT:  .LBB5_8: # %else6
+; SSE4-NEXT:  .LBB5_7: # %cond.store5
+; SSE4-NEXT:    pextrb $3, %xmm0, 3(%rdi)
 ; SSE4-NEXT:    retq
 ;
-; AVX-LABEL: truncstore_v4i64_v4i8:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm3
-; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm3[0,2]
-; AVX-NEXT:    vpextrb $0, %xmm2, %eax
-; AVX-NEXT:    testb $1, %al
-; AVX-NEXT:    je .LBB5_2
-; AVX-NEXT:  # %bb.1: # %cond.store
-; AVX-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX-NEXT:  .LBB5_2: # %else
-; AVX-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX-NEXT:    testb $1, %al
-; AVX-NEXT:    je .LBB5_4
-; AVX-NEXT:  # %bb.3: # %cond.store1
-; AVX-NEXT:    vpextrb $4, %xmm0, 1(%rdi)
-; AVX-NEXT:  .LBB5_4: # %else2
-; AVX-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX-NEXT:    testb $1, %al
-; AVX-NEXT:    je .LBB5_6
-; AVX-NEXT:  # %bb.5: # %cond.store3
-; AVX-NEXT:    vpextrb $8, %xmm0, 2(%rdi)
-; AVX-NEXT:  .LBB5_6: # %else4
-; AVX-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX-NEXT:    testb $1, %al
-; AVX-NEXT:    je .LBB5_8
-; AVX-NEXT:  # %bb.7: # %cond.store5
-; AVX-NEXT:    vpextrb $12, %xmm0, 3(%rdi)
-; AVX-NEXT:  .LBB5_8: # %else6
-; AVX-NEXT:    vzeroupper
-; AVX-NEXT:    retq
+; AVX1-LABEL: truncstore_v4i64_v4i8:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = <0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX1-NEXT:    vpshufb %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpshufb %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vmovmskps %xmm1, %eax
+; AVX1-NEXT:    xorl $15, %eax
+; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    jne .LBB5_1
+; AVX1-NEXT:  # %bb.2: # %else
+; AVX1-NEXT:    testb $2, %al
+; AVX1-NEXT:    jne .LBB5_3
+; AVX1-NEXT:  .LBB5_4: # %else2
+; AVX1-NEXT:    testb $4, %al
+; AVX1-NEXT:    jne .LBB5_5
+; AVX1-NEXT:  .LBB5_6: # %else4
+; AVX1-NEXT:    testb $8, %al
+; AVX1-NEXT:    jne .LBB5_7
+; AVX1-NEXT:  .LBB5_8: # %else6
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+; AVX1-NEXT:  .LBB5_1: # %cond.store
+; AVX1-NEXT:    vpextrb $0, %xmm0, (%rdi)
+; AVX1-NEXT:    testb $2, %al
+; AVX1-NEXT:    je .LBB5_4
+; AVX1-NEXT:  .LBB5_3: # %cond.store1
+; AVX1-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
+; AVX1-NEXT:    testb $4, %al
+; AVX1-NEXT:    je .LBB5_6
+; AVX1-NEXT:  .LBB5_5: # %cond.store3
+; AVX1-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
+; AVX1-NEXT:    testb $8, %al
+; AVX1-NEXT:    je .LBB5_8
+; AVX1-NEXT:  .LBB5_7: # %cond.store5
+; AVX1-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: truncstore_v4i64_v4i8:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm4 = <0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX2-NEXT:    vpshufb %xmm4, %xmm3, %xmm3
+; AVX2-NEXT:    vpshufb %xmm4, %xmm0, %xmm0
+; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vmovmskps %xmm1, %eax
+; AVX2-NEXT:    xorl $15, %eax
+; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    jne .LBB5_1
+; AVX2-NEXT:  # %bb.2: # %else
+; AVX2-NEXT:    testb $2, %al
+; AVX2-NEXT:    jne .LBB5_3
+; AVX2-NEXT:  .LBB5_4: # %else2
+; AVX2-NEXT:    testb $4, %al
+; AVX2-NEXT:    jne .LBB5_5
+; AVX2-NEXT:  .LBB5_6: # %else4
+; AVX2-NEXT:    testb $8, %al
+; AVX2-NEXT:    jne .LBB5_7
+; AVX2-NEXT:  .LBB5_8: # %else6
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+; AVX2-NEXT:  .LBB5_1: # %cond.store
+; AVX2-NEXT:    vpextrb $0, %xmm0, (%rdi)
+; AVX2-NEXT:    testb $2, %al
+; AVX2-NEXT:    je .LBB5_4
+; AVX2-NEXT:  .LBB5_3: # %cond.store1
+; AVX2-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
+; AVX2-NEXT:    testb $4, %al
+; AVX2-NEXT:    je .LBB5_6
+; AVX2-NEXT:  .LBB5_5: # %cond.store3
+; AVX2-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
+; AVX2-NEXT:    testb $8, %al
+; AVX2-NEXT:    je .LBB5_8
+; AVX2-NEXT:  .LBB5_7: # %cond.store5
+; AVX2-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: truncstore_v4i64_v4i8:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
 ; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
 ; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    vpmovqd %zmm0, %ymm0
+; AVX512F-NEXT:    vpmovqb %zmm0, %xmm0
 ; AVX512F-NEXT:    kmovw %k0, %eax
 ; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB5_2
-; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    jne .LBB5_1
+; AVX512F-NEXT:  # %bb.2: # %else
+; AVX512F-NEXT:    testb $2, %al
+; AVX512F-NEXT:    jne .LBB5_3
+; AVX512F-NEXT:  .LBB5_4: # %else2
+; AVX512F-NEXT:    testb $4, %al
+; AVX512F-NEXT:    jne .LBB5_5
+; AVX512F-NEXT:  .LBB5_6: # %else4
+; AVX512F-NEXT:    testb $8, %al
+; AVX512F-NEXT:    jne .LBB5_7
+; AVX512F-NEXT:  .LBB5_8: # %else6
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+; AVX512F-NEXT:  .LBB5_1: # %cond.store
 ; AVX512F-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX512F-NEXT:  .LBB5_2: # %else
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $2, %al
 ; AVX512F-NEXT:    je .LBB5_4
-; AVX512F-NEXT:  # %bb.3: # %cond.store1
-; AVX512F-NEXT:    vpextrb $4, %xmm0, 1(%rdi)
-; AVX512F-NEXT:  .LBB5_4: # %else2
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  .LBB5_3: # %cond.store1
+; AVX512F-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
+; AVX512F-NEXT:    testb $4, %al
 ; AVX512F-NEXT:    je .LBB5_6
-; AVX512F-NEXT:  # %bb.5: # %cond.store3
-; AVX512F-NEXT:    vpextrb $8, %xmm0, 2(%rdi)
-; AVX512F-NEXT:  .LBB5_6: # %else4
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  .LBB5_5: # %cond.store3
+; AVX512F-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
+; AVX512F-NEXT:    testb $8, %al
 ; AVX512F-NEXT:    je .LBB5_8
-; AVX512F-NEXT:  # %bb.7: # %cond.store5
-; AVX512F-NEXT:    vpextrb $12, %xmm0, 3(%rdi)
-; AVX512F-NEXT:  .LBB5_8: # %else6
+; AVX512F-NEXT:  .LBB5_7: # %cond.store5
+; AVX512F-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
@@ -1551,10 +1586,9 @@ define void @truncstore_v4i64_v4i8(<4 x i64> %x, <4 x i8>* %p, <4 x i32> %mask)
 ; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
 ; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
 ; AVX512BW-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512BW-NEXT:    vpmovqd %zmm0, %ymm0
-; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
 ; AVX512BW-NEXT:    kshiftlq $60, %k0, %k0
 ; AVX512BW-NEXT:    kshiftrq $60, %k0, %k1
+; AVX512BW-NEXT:    vpmovqb %zmm0, %xmm0
 ; AVX512BW-NEXT:    vmovdqu8 %zmm0, (%rdi) {%k1}
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -1575,46 +1609,49 @@ define void @truncstore_v2i64_v2i32(<2 x i64> %x, <2 x i32>* %p, <2 x i64> %mask
 ; SSE2-LABEL: truncstore_v2i64_v2i32:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; SSE2-NEXT:    pcmpeqd %xmm1, %xmm2
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,0,3,2]
 ; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    movd %xmm1, %eax
-; SSE2-NEXT:    notl %eax
+; SSE2-NEXT:    movmskpd %xmm1, %eax
+; SSE2-NEXT:    xorl $3, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB6_2
-; SSE2-NEXT:  # %bb.1: # %cond.store
+; SSE2-NEXT:    jne .LBB6_1
+; SSE2-NEXT:  # %bb.2: # %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne .LBB6_3
+; SSE2-NEXT:  .LBB6_4: # %else2
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  .LBB6_1: # %cond.store
 ; SSE2-NEXT:    movd %xmm0, (%rdi)
-; SSE2-NEXT:  .LBB6_2: # %else
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT:    pxor %xmm2, %xmm1
-; SSE2-NEXT:    pextrw $4, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $2, %al
 ; SSE2-NEXT:    je .LBB6_4
-; SSE2-NEXT:  # %bb.3: # %cond.store1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE2-NEXT:  .LBB6_3: # %cond.store1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
 ; SSE2-NEXT:    movd %xmm0, 4(%rdi)
-; SSE2-NEXT:  .LBB6_4: # %else2
 ; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: truncstore_v2i64_v2i32:
 ; SSE4:       # %bb.0:
 ; SSE4-NEXT:    pxor %xmm2, %xmm2
+; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; SSE4-NEXT:    pcmpeqq %xmm1, %xmm2
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE4-NEXT:    pxor %xmm2, %xmm1
-; SSE4-NEXT:    pextrb $0, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB6_2
-; SSE4-NEXT:  # %bb.1: # %cond.store
-; SSE4-NEXT:    movss %xmm0, (%rdi)
-; SSE4-NEXT:  .LBB6_2: # %else
-; SSE4-NEXT:    pextrb $8, %xmm1, %eax
+; SSE4-NEXT:    movmskpd %xmm2, %eax
+; SSE4-NEXT:    xorl $3, %eax
 ; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB6_4
-; SSE4-NEXT:  # %bb.3: # %cond.store1
-; SSE4-NEXT:    extractps $2, %xmm0, 4(%rdi)
+; SSE4-NEXT:    jne .LBB6_1
+; SSE4-NEXT:  # %bb.2: # %else
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    jne .LBB6_3
 ; SSE4-NEXT:  .LBB6_4: # %else2
 ; SSE4-NEXT:    retq
+; SSE4-NEXT:  .LBB6_1: # %cond.store
+; SSE4-NEXT:    movd %xmm0, (%rdi)
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    je .LBB6_4
+; SSE4-NEXT:  .LBB6_3: # %cond.store1
+; SSE4-NEXT:    pextrd $1, %xmm0, 4(%rdi)
+; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: truncstore_v2i64_v2i32:
 ; AVX1:       # %bb.0:
@@ -1642,9 +1679,9 @@ define void @truncstore_v2i64_v2i32(<2 x i64> %x, <2 x i32>* %p, <2 x i64> %mask
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
 ; AVX512F-NEXT:    vptestmq %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; AVX512F-NEXT:    kshiftlw $14, %k0, %k0
 ; AVX512F-NEXT:    kshiftrw $14, %k0, %k1
+; AVX512F-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; AVX512F-NEXT:    vmovdqu32 %zmm0, (%rdi) {%k1}
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
@@ -1659,9 +1696,9 @@ define void @truncstore_v2i64_v2i32(<2 x i64> %x, <2 x i32>* %p, <2 x i64> %mask
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
 ; AVX512BW-NEXT:    vptestmq %zmm1, %zmm1, %k0
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; AVX512BW-NEXT:    kshiftlw $14, %k0, %k0
 ; AVX512BW-NEXT:    kshiftrw $14, %k0, %k1
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; AVX512BW-NEXT:    vmovdqu32 %zmm0, (%rdi) {%k1}
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -1675,86 +1712,97 @@ define void @truncstore_v2i64_v2i16(<2 x i64> %x, <2 x i16>* %p, <2 x i64> %mask
 ; SSE2-LABEL: truncstore_v2i64_v2i16:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
 ; SSE2-NEXT:    pcmpeqd %xmm1, %xmm2
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,0,3,2]
 ; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    movd %xmm1, %eax
-; SSE2-NEXT:    notl %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB7_2
-; SSE2-NEXT:  # %bb.1: # %cond.store
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    movw %ax, (%rdi)
-; SSE2-NEXT:  .LBB7_2: # %else
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT:    pxor %xmm2, %xmm1
-; SSE2-NEXT:    pextrw $4, %xmm1, %eax
+; SSE2-NEXT:    movmskpd %xmm1, %eax
+; SSE2-NEXT:    xorl $3, %eax
 ; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    jne .LBB7_1
+; SSE2-NEXT:  # %bb.2: # %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne .LBB7_3
+; SSE2-NEXT:  .LBB7_4: # %else2
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  .LBB7_1: # %cond.store
+; SSE2-NEXT:    movd %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, (%rdi)
+; SSE2-NEXT:    testb $2, %al
 ; SSE2-NEXT:    je .LBB7_4
-; SSE2-NEXT:  # %bb.3: # %cond.store1
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
+; SSE2-NEXT:  .LBB7_3: # %cond.store1
+; SSE2-NEXT:    pextrw $1, %xmm0, %eax
 ; SSE2-NEXT:    movw %ax, 2(%rdi)
-; SSE2-NEXT:  .LBB7_4: # %else2
 ; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: truncstore_v2i64_v2i16:
 ; SSE4:       # %bb.0:
 ; SSE4-NEXT:    pxor %xmm2, %xmm2
+; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSE4-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
 ; SSE4-NEXT:    pcmpeqq %xmm1, %xmm2
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE4-NEXT:    pxor %xmm2, %xmm1
-; SSE4-NEXT:    pextrb $0, %xmm1, %eax
+; SSE4-NEXT:    movmskpd %xmm2, %eax
+; SSE4-NEXT:    xorl $3, %eax
 ; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB7_2
-; SSE4-NEXT:  # %bb.1: # %cond.store
+; SSE4-NEXT:    jne .LBB7_1
+; SSE4-NEXT:  # %bb.2: # %else
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    jne .LBB7_3
+; SSE4-NEXT:  .LBB7_4: # %else2
+; SSE4-NEXT:    retq
+; SSE4-NEXT:  .LBB7_1: # %cond.store
 ; SSE4-NEXT:    pextrw $0, %xmm0, (%rdi)
-; SSE4-NEXT:  .LBB7_2: # %else
-; SSE4-NEXT:    pextrb $8, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $2, %al
 ; SSE4-NEXT:    je .LBB7_4
-; SSE4-NEXT:  # %bb.3: # %cond.store1
-; SSE4-NEXT:    pextrw $4, %xmm0, 2(%rdi)
-; SSE4-NEXT:  .LBB7_4: # %else2
+; SSE4-NEXT:  .LBB7_3: # %cond.store1
+; SSE4-NEXT:    pextrw $1, %xmm0, 2(%rdi)
 ; SSE4-NEXT:    retq
 ;
 ; AVX-LABEL: truncstore_v2i64_v2i16:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
 ; AVX-NEXT:    vpcmpeqq %xmm2, %xmm1, %xmm1
-; AVX-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX-NEXT:    vpextrb $0, %xmm1, %eax
+; AVX-NEXT:    vmovmskpd %xmm1, %eax
+; AVX-NEXT:    xorl $3, %eax
 ; AVX-NEXT:    testb $1, %al
-; AVX-NEXT:    je .LBB7_2
-; AVX-NEXT:  # %bb.1: # %cond.store
+; AVX-NEXT:    jne .LBB7_1
+; AVX-NEXT:  # %bb.2: # %else
+; AVX-NEXT:    testb $2, %al
+; AVX-NEXT:    jne .LBB7_3
+; AVX-NEXT:  .LBB7_4: # %else2
+; AVX-NEXT:    retq
+; AVX-NEXT:  .LBB7_1: # %cond.store
 ; AVX-NEXT:    vpextrw $0, %xmm0, (%rdi)
-; AVX-NEXT:  .LBB7_2: # %else
-; AVX-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX-NEXT:    testb $1, %al
+; AVX-NEXT:    testb $2, %al
 ; AVX-NEXT:    je .LBB7_4
-; AVX-NEXT:  # %bb.3: # %cond.store1
-; AVX-NEXT:    vpextrw $4, %xmm0, 2(%rdi)
-; AVX-NEXT:  .LBB7_4: # %else2
+; AVX-NEXT:  .LBB7_3: # %cond.store1
+; AVX-NEXT:    vpextrw $1, %xmm0, 2(%rdi)
 ; AVX-NEXT:    retq
 ;
 ; AVX512F-LABEL: truncstore_v2i64_v2i16:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
 ; AVX512F-NEXT:    vptestmq %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX512F-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
 ; AVX512F-NEXT:    kmovw %k0, %eax
 ; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB7_2
-; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    jne .LBB7_1
+; AVX512F-NEXT:  # %bb.2: # %else
+; AVX512F-NEXT:    testb $2, %al
+; AVX512F-NEXT:    jne .LBB7_3
+; AVX512F-NEXT:  .LBB7_4: # %else2
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+; AVX512F-NEXT:  .LBB7_1: # %cond.store
 ; AVX512F-NEXT:    vpextrw $0, %xmm0, (%rdi)
-; AVX512F-NEXT:  .LBB7_2: # %else
-; AVX512F-NEXT:    vptestmq %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $2, %al
 ; AVX512F-NEXT:    je .LBB7_4
-; AVX512F-NEXT:  # %bb.3: # %cond.store1
-; AVX512F-NEXT:    vpextrw $4, %xmm0, 2(%rdi)
-; AVX512F-NEXT:  .LBB7_4: # %else2
+; AVX512F-NEXT:  .LBB7_3: # %cond.store1
+; AVX512F-NEXT:    vpextrw $1, %xmm0, 2(%rdi)
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
@@ -1762,10 +1810,10 @@ define void @truncstore_v2i64_v2i16(<2 x i64> %x, <2 x i16>* %p, <2 x i64> %mask
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
 ; AVX512BW-NEXT:    vptestmq %zmm1, %zmm1, %k0
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; AVX512BW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
 ; AVX512BW-NEXT:    kshiftld $30, %k0, %k0
 ; AVX512BW-NEXT:    kshiftrd $30, %k0, %k1
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX512BW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
 ; AVX512BW-NEXT:    vmovdqu16 %zmm0, (%rdi) {%k1}
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -1785,86 +1833,95 @@ define void @truncstore_v2i64_v2i8(<2 x i64> %x, <2 x i8>* %p, <2 x i64> %mask)
 ; SSE2-LABEL: truncstore_v2i64_v2i8:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    packuswb %xmm0, %xmm0
+; SSE2-NEXT:    packuswb %xmm0, %xmm0
+; SSE2-NEXT:    packuswb %xmm0, %xmm0
 ; SSE2-NEXT:    pcmpeqd %xmm1, %xmm2
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,0,3,2]
 ; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    movd %xmm1, %eax
-; SSE2-NEXT:    notl %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB8_2
-; SSE2-NEXT:  # %bb.1: # %cond.store
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    movb %al, (%rdi)
-; SSE2-NEXT:  .LBB8_2: # %else
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT:    pxor %xmm2, %xmm1
-; SSE2-NEXT:    pextrw $4, %xmm1, %eax
+; SSE2-NEXT:    movmskpd %xmm1, %eax
+; SSE2-NEXT:    xorl $3, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB8_4
-; SSE2-NEXT:  # %bb.3: # %cond.store1
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    movb %al, 1(%rdi)
+; SSE2-NEXT:    movd %xmm0, %ecx
+; SSE2-NEXT:    jne .LBB8_1
+; SSE2-NEXT:  # %bb.2: # %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne .LBB8_3
 ; SSE2-NEXT:  .LBB8_4: # %else2
 ; SSE2-NEXT:    retq
+; SSE2-NEXT:  .LBB8_1: # %cond.store
+; SSE2-NEXT:    movb %cl, (%rdi)
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    je .LBB8_4
+; SSE2-NEXT:  .LBB8_3: # %cond.store1
+; SSE2-NEXT:    movb %ch, 1(%rdi)
+; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: truncstore_v2i64_v2i8:
 ; SSE4:       # %bb.0:
 ; SSE4-NEXT:    pxor %xmm2, %xmm2
+; SSE4-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
 ; SSE4-NEXT:    pcmpeqq %xmm1, %xmm2
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE4-NEXT:    pxor %xmm2, %xmm1
-; SSE4-NEXT:    pextrb $0, %xmm1, %eax
+; SSE4-NEXT:    movmskpd %xmm2, %eax
+; SSE4-NEXT:    xorl $3, %eax
 ; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB8_2
-; SSE4-NEXT:  # %bb.1: # %cond.store
+; SSE4-NEXT:    jne .LBB8_1
+; SSE4-NEXT:  # %bb.2: # %else
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    jne .LBB8_3
+; SSE4-NEXT:  .LBB8_4: # %else2
+; SSE4-NEXT:    retq
+; SSE4-NEXT:  .LBB8_1: # %cond.store
 ; SSE4-NEXT:    pextrb $0, %xmm0, (%rdi)
-; SSE4-NEXT:  .LBB8_2: # %else
-; SSE4-NEXT:    pextrb $8, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $2, %al
 ; SSE4-NEXT:    je .LBB8_4
-; SSE4-NEXT:  # %bb.3: # %cond.store1
-; SSE4-NEXT:    pextrb $8, %xmm0, 1(%rdi)
-; SSE4-NEXT:  .LBB8_4: # %else2
+; SSE4-NEXT:  .LBB8_3: # %cond.store1
+; SSE4-NEXT:    pextrb $1, %xmm0, 1(%rdi)
 ; SSE4-NEXT:    retq
 ;
 ; AVX-LABEL: truncstore_v2i64_v2i8:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
 ; AVX-NEXT:    vpcmpeqq %xmm2, %xmm1, %xmm1
-; AVX-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX-NEXT:    vpextrb $0, %xmm1, %eax
+; AVX-NEXT:    vmovmskpd %xmm1, %eax
+; AVX-NEXT:    xorl $3, %eax
 ; AVX-NEXT:    testb $1, %al
-; AVX-NEXT:    je .LBB8_2
-; AVX-NEXT:  # %bb.1: # %cond.store
+; AVX-NEXT:    jne .LBB8_1
+; AVX-NEXT:  # %bb.2: # %else
+; AVX-NEXT:    testb $2, %al
+; AVX-NEXT:    jne .LBB8_3
+; AVX-NEXT:  .LBB8_4: # %else2
+; AVX-NEXT:    retq
+; AVX-NEXT:  .LBB8_1: # %cond.store
 ; AVX-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX-NEXT:  .LBB8_2: # %else
-; AVX-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX-NEXT:    testb $1, %al
+; AVX-NEXT:    testb $2, %al
 ; AVX-NEXT:    je .LBB8_4
-; AVX-NEXT:  # %bb.3: # %cond.store1
-; AVX-NEXT:    vpextrb $8, %xmm0, 1(%rdi)
-; AVX-NEXT:  .LBB8_4: # %else2
+; AVX-NEXT:  .LBB8_3: # %cond.store1
+; AVX-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
 ; AVX-NEXT:    retq
 ;
 ; AVX512F-LABEL: truncstore_v2i64_v2i8:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
 ; AVX512F-NEXT:    vptestmq %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
 ; AVX512F-NEXT:    kmovw %k0, %eax
 ; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB8_2
-; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    jne .LBB8_1
+; AVX512F-NEXT:  # %bb.2: # %else
+; AVX512F-NEXT:    testb $2, %al
+; AVX512F-NEXT:    jne .LBB8_3
+; AVX512F-NEXT:  .LBB8_4: # %else2
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+; AVX512F-NEXT:  .LBB8_1: # %cond.store
 ; AVX512F-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX512F-NEXT:  .LBB8_2: # %else
-; AVX512F-NEXT:    vptestmq %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $2, %al
 ; AVX512F-NEXT:    je .LBB8_4
-; AVX512F-NEXT:  # %bb.3: # %cond.store1
-; AVX512F-NEXT:    vpextrb $8, %xmm0, 1(%rdi)
-; AVX512F-NEXT:  .LBB8_4: # %else2
+; AVX512F-NEXT:  .LBB8_3: # %cond.store1
+; AVX512F-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
@@ -1872,9 +1929,9 @@ define void @truncstore_v2i64_v2i8(<2 x i64> %x, <2 x i8>* %p, <2 x i64> %mask)
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
 ; AVX512BW-NEXT:    vptestmq %zmm1, %zmm1, %k0
-; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
 ; AVX512BW-NEXT:    kshiftlq $62, %k0, %k0
 ; AVX512BW-NEXT:    kshiftrq $62, %k0, %k1
+; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
 ; AVX512BW-NEXT:    vmovdqu8 %zmm0, (%rdi) {%k1}
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -1894,671 +1951,557 @@ define void @truncstore_v16i32_v16i16(<16 x i32> %x, <16 x i16>* %p, <16 x i32>
 ; SSE2-LABEL: truncstore_v16i32_v16i16:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pxor %xmm8, %xmm8
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm8
 ; SSE2-NEXT:    pslld $16, %xmm1
 ; SSE2-NEXT:    psrad $16, %xmm1
 ; SSE2-NEXT:    pslld $16, %xmm0
 ; SSE2-NEXT:    psrad $16, %xmm0
 ; SSE2-NEXT:    packssdw %xmm1, %xmm0
-; SSE2-NEXT:    movd %xmm8, %eax
-; SSE2-NEXT:    notl %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB9_2
-; SSE2-NEXT:  # %bb.1: # %cond.store
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    movw %ax, (%rdi)
-; SSE2-NEXT:  .LBB9_2: # %else
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm9
-; SSE2-NEXT:    pxor %xmm9, %xmm8
-; SSE2-NEXT:    pextrw $2, %xmm8, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB9_4
-; SSE2-NEXT:  # %bb.3: # %cond.store1
-; SSE2-NEXT:    pextrw $1, %xmm0, %eax
-; SSE2-NEXT:    movw %ax, 2(%rdi)
-; SSE2-NEXT:  .LBB9_4: # %else2
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm4
-; SSE2-NEXT:    pxor %xmm4, %xmm9
-; SSE2-NEXT:    pextrw $4, %xmm9, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB9_6
-; SSE2-NEXT:  # %bb.5: # %cond.store3
-; SSE2-NEXT:    pextrw $2, %xmm0, %eax
-; SSE2-NEXT:    movw %ax, 4(%rdi)
-; SSE2-NEXT:  .LBB9_6: # %else4
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm7
 ; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
+; SSE2-NEXT:    pxor %xmm1, %xmm7
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm6
+; SSE2-NEXT:    pxor %xmm1, %xmm6
+; SSE2-NEXT:    packssdw %xmm7, %xmm6
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm5
+; SSE2-NEXT:    pxor %xmm1, %xmm5
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm4
 ; SSE2-NEXT:    pxor %xmm1, %xmm4
-; SSE2-NEXT:    pextrw $6, %xmm4, %eax
+; SSE2-NEXT:    packssdw %xmm5, %xmm4
+; SSE2-NEXT:    packsswb %xmm6, %xmm4
+; SSE2-NEXT:    pmovmskb %xmm4, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB9_8
-; SSE2-NEXT:  # %bb.7: # %cond.store5
-; SSE2-NEXT:    pextrw $3, %xmm0, %eax
-; SSE2-NEXT:    movw %ax, 6(%rdi)
+; SSE2-NEXT:    jne .LBB9_1
+; SSE2-NEXT:  # %bb.2: # %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne .LBB9_3
+; SSE2-NEXT:  .LBB9_4: # %else2
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne .LBB9_5
+; SSE2-NEXT:  .LBB9_6: # %else4
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne .LBB9_7
 ; SSE2-NEXT:  .LBB9_8: # %else6
-; SSE2-NEXT:    pxor %xmm4, %xmm4
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm4
-; SSE2-NEXT:    movd %xmm4, %eax
-; SSE2-NEXT:    notl %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB9_10
-; SSE2-NEXT:  # %bb.9: # %cond.store7
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    movw %ax, 8(%rdi)
+; SSE2-NEXT:    testb $16, %al
+; SSE2-NEXT:    jne .LBB9_9
 ; SSE2-NEXT:  .LBB9_10: # %else8
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pxor %xmm1, %xmm4
-; SSE2-NEXT:    pextrw $2, %xmm4, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $32, %al
 ; SSE2-NEXT:    je .LBB9_12
-; SSE2-NEXT:  # %bb.11: # %cond.store9
-; SSE2-NEXT:    pextrw $5, %xmm0, %eax
-; SSE2-NEXT:    movw %ax, 10(%rdi)
+; SSE2-NEXT:  .LBB9_11: # %cond.store9
+; SSE2-NEXT:    pextrw $5, %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, 10(%rdi)
 ; SSE2-NEXT:  .LBB9_12: # %else10
 ; SSE2-NEXT:    pslld $16, %xmm3
 ; SSE2-NEXT:    pslld $16, %xmm2
-; SSE2-NEXT:    pxor %xmm4, %xmm4
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm5
-; SSE2-NEXT:    pxor %xmm5, %xmm1
-; SSE2-NEXT:    pextrw $4, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $64, %al
 ; SSE2-NEXT:    je .LBB9_14
 ; SSE2-NEXT:  # %bb.13: # %cond.store11
-; SSE2-NEXT:    pextrw $6, %xmm0, %eax
-; SSE2-NEXT:    movw %ax, 12(%rdi)
+; SSE2-NEXT:    pextrw $6, %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, 12(%rdi)
 ; SSE2-NEXT:  .LBB9_14: # %else12
 ; SSE2-NEXT:    psrad $16, %xmm3
 ; SSE2-NEXT:    psrad $16, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pxor %xmm1, %xmm5
-; SSE2-NEXT:    pextrw $6, %xmm5, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $-128, %al
 ; SSE2-NEXT:    je .LBB9_16
 ; SSE2-NEXT:  # %bb.15: # %cond.store13
-; SSE2-NEXT:    pextrw $7, %xmm0, %eax
-; SSE2-NEXT:    movw %ax, 14(%rdi)
+; SSE2-NEXT:    pextrw $7, %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, 14(%rdi)
 ; SSE2-NEXT:  .LBB9_16: # %else14
 ; SSE2-NEXT:    packssdw %xmm3, %xmm2
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqd %xmm6, %xmm0
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    notl %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB9_18
-; SSE2-NEXT:  # %bb.17: # %cond.store15
-; SSE2-NEXT:    movd %xmm2, %eax
-; SSE2-NEXT:    movw %ax, 16(%rdi)
-; SSE2-NEXT:  .LBB9_18: # %else16
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pxor %xmm1, %xmm0
-; SSE2-NEXT:    pextrw $2, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB9_20
-; SSE2-NEXT:  # %bb.19: # %cond.store17
-; SSE2-NEXT:    pextrw $1, %xmm2, %eax
-; SSE2-NEXT:    movw %ax, 18(%rdi)
+; SSE2-NEXT:    testl $256, %eax # imm = 0x100
+; SSE2-NEXT:    jne .LBB9_17
+; SSE2-NEXT:  # %bb.18: # %else16
+; SSE2-NEXT:    testl $512, %eax # imm = 0x200
+; SSE2-NEXT:    jne .LBB9_19
 ; SSE2-NEXT:  .LBB9_20: # %else18
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm6
-; SSE2-NEXT:    pxor %xmm6, %xmm1
-; SSE2-NEXT:    pextrw $4, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB9_22
-; SSE2-NEXT:  # %bb.21: # %cond.store19
-; SSE2-NEXT:    pextrw $2, %xmm2, %eax
-; SSE2-NEXT:    movw %ax, 20(%rdi)
+; SSE2-NEXT:    testl $1024, %eax # imm = 0x400
+; SSE2-NEXT:    jne .LBB9_21
 ; SSE2-NEXT:  .LBB9_22: # %else20
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE2-NEXT:    pxor %xmm0, %xmm6
-; SSE2-NEXT:    pextrw $6, %xmm6, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB9_24
-; SSE2-NEXT:  # %bb.23: # %cond.store21
-; SSE2-NEXT:    pextrw $3, %xmm2, %eax
-; SSE2-NEXT:    movw %ax, 22(%rdi)
+; SSE2-NEXT:    testl $2048, %eax # imm = 0x800
+; SSE2-NEXT:    jne .LBB9_23
 ; SSE2-NEXT:  .LBB9_24: # %else22
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqd %xmm7, %xmm0
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    notl %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB9_26
-; SSE2-NEXT:  # %bb.25: # %cond.store23
-; SSE2-NEXT:    pextrw $4, %xmm2, %eax
-; SSE2-NEXT:    movw %ax, 24(%rdi)
+; SSE2-NEXT:    testl $4096, %eax # imm = 0x1000
+; SSE2-NEXT:    jne .LBB9_25
 ; SSE2-NEXT:  .LBB9_26: # %else24
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pxor %xmm1, %xmm0
-; SSE2-NEXT:    pextrw $2, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB9_28
-; SSE2-NEXT:  # %bb.27: # %cond.store25
-; SSE2-NEXT:    pextrw $5, %xmm2, %eax
-; SSE2-NEXT:    movw %ax, 26(%rdi)
+; SSE2-NEXT:    testl $8192, %eax # imm = 0x2000
+; SSE2-NEXT:    jne .LBB9_27
 ; SSE2-NEXT:  .LBB9_28: # %else26
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm7
-; SSE2-NEXT:    pxor %xmm7, %xmm1
-; SSE2-NEXT:    pextrw $4, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB9_30
-; SSE2-NEXT:  # %bb.29: # %cond.store27
-; SSE2-NEXT:    pextrw $6, %xmm2, %eax
-; SSE2-NEXT:    movw %ax, 28(%rdi)
+; SSE2-NEXT:    testl $16384, %eax # imm = 0x4000
+; SSE2-NEXT:    jne .LBB9_29
 ; SSE2-NEXT:  .LBB9_30: # %else28
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE2-NEXT:    pxor %xmm0, %xmm7
-; SSE2-NEXT:    pextrw $6, %xmm7, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $32768, %eax # imm = 0x8000
+; SSE2-NEXT:    jne .LBB9_31
+; SSE2-NEXT:  .LBB9_32: # %else30
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  .LBB9_1: # %cond.store
+; SSE2-NEXT:    movd %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, (%rdi)
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    je .LBB9_4
+; SSE2-NEXT:  .LBB9_3: # %cond.store1
+; SSE2-NEXT:    pextrw $1, %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, 2(%rdi)
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    je .LBB9_6
+; SSE2-NEXT:  .LBB9_5: # %cond.store3
+; SSE2-NEXT:    pextrw $2, %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, 4(%rdi)
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    je .LBB9_8
+; SSE2-NEXT:  .LBB9_7: # %cond.store5
+; SSE2-NEXT:    pextrw $3, %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, 6(%rdi)
+; SSE2-NEXT:    testb $16, %al
+; SSE2-NEXT:    je .LBB9_10
+; SSE2-NEXT:  .LBB9_9: # %cond.store7
+; SSE2-NEXT:    pextrw $4, %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, 8(%rdi)
+; SSE2-NEXT:    testb $32, %al
+; SSE2-NEXT:    jne .LBB9_11
+; SSE2-NEXT:    jmp .LBB9_12
+; SSE2-NEXT:  .LBB9_17: # %cond.store15
+; SSE2-NEXT:    movd %xmm2, %ecx
+; SSE2-NEXT:    movw %cx, 16(%rdi)
+; SSE2-NEXT:    testl $512, %eax # imm = 0x200
+; SSE2-NEXT:    je .LBB9_20
+; SSE2-NEXT:  .LBB9_19: # %cond.store17
+; SSE2-NEXT:    pextrw $1, %xmm2, %ecx
+; SSE2-NEXT:    movw %cx, 18(%rdi)
+; SSE2-NEXT:    testl $1024, %eax # imm = 0x400
+; SSE2-NEXT:    je .LBB9_22
+; SSE2-NEXT:  .LBB9_21: # %cond.store19
+; SSE2-NEXT:    pextrw $2, %xmm2, %ecx
+; SSE2-NEXT:    movw %cx, 20(%rdi)
+; SSE2-NEXT:    testl $2048, %eax # imm = 0x800
+; SSE2-NEXT:    je .LBB9_24
+; SSE2-NEXT:  .LBB9_23: # %cond.store21
+; SSE2-NEXT:    pextrw $3, %xmm2, %ecx
+; SSE2-NEXT:    movw %cx, 22(%rdi)
+; SSE2-NEXT:    testl $4096, %eax # imm = 0x1000
+; SSE2-NEXT:    je .LBB9_26
+; SSE2-NEXT:  .LBB9_25: # %cond.store23
+; SSE2-NEXT:    pextrw $4, %xmm2, %ecx
+; SSE2-NEXT:    movw %cx, 24(%rdi)
+; SSE2-NEXT:    testl $8192, %eax # imm = 0x2000
+; SSE2-NEXT:    je .LBB9_28
+; SSE2-NEXT:  .LBB9_27: # %cond.store25
+; SSE2-NEXT:    pextrw $5, %xmm2, %ecx
+; SSE2-NEXT:    movw %cx, 26(%rdi)
+; SSE2-NEXT:    testl $16384, %eax # imm = 0x4000
+; SSE2-NEXT:    je .LBB9_30
+; SSE2-NEXT:  .LBB9_29: # %cond.store27
+; SSE2-NEXT:    pextrw $6, %xmm2, %ecx
+; SSE2-NEXT:    movw %cx, 28(%rdi)
+; SSE2-NEXT:    testl $32768, %eax # imm = 0x8000
 ; SSE2-NEXT:    je .LBB9_32
-; SSE2-NEXT:  # %bb.31: # %cond.store29
+; SSE2-NEXT:  .LBB9_31: # %cond.store29
 ; SSE2-NEXT:    pextrw $7, %xmm2, %eax
 ; SSE2-NEXT:    movw %ax, 30(%rdi)
-; SSE2-NEXT:  .LBB9_32: # %else30
 ; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: truncstore_v16i32_v16i16:
 ; SSE4:       # %bb.0:
 ; SSE4-NEXT:    pxor %xmm8, %xmm8
-; SSE4-NEXT:    movdqa %xmm4, %xmm10
-; SSE4-NEXT:    pcmpeqd %xmm8, %xmm10
-; SSE4-NEXT:    pcmpeqd %xmm9, %xmm9
-; SSE4-NEXT:    pxor %xmm10, %xmm9
 ; SSE4-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0],xmm8[1],xmm1[2],xmm8[3],xmm1[4],xmm8[5],xmm1[6],xmm8[7]
 ; SSE4-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0],xmm8[1],xmm0[2],xmm8[3],xmm0[4],xmm8[5],xmm0[6],xmm8[7]
 ; SSE4-NEXT:    packusdw %xmm1, %xmm0
-; SSE4-NEXT:    pextrb $0, %xmm9, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB9_2
-; SSE4-NEXT:  # %bb.1: # %cond.store
-; SSE4-NEXT:    pextrw $0, %xmm0, (%rdi)
-; SSE4-NEXT:  .LBB9_2: # %else
-; SSE4-NEXT:    pextrb $4, %xmm9, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB9_4
-; SSE4-NEXT:  # %bb.3: # %cond.store1
-; SSE4-NEXT:    pextrw $1, %xmm0, 2(%rdi)
-; SSE4-NEXT:  .LBB9_4: # %else2
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm4
+; SSE4-NEXT:    pcmpeqd %xmm8, %xmm7
 ; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE4-NEXT:    pxor %xmm4, %xmm1
-; SSE4-NEXT:    pextrb $8, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB9_6
-; SSE4-NEXT:  # %bb.5: # %cond.store3
-; SSE4-NEXT:    pextrw $2, %xmm0, 4(%rdi)
+; SSE4-NEXT:    pxor %xmm1, %xmm7
+; SSE4-NEXT:    pcmpeqd %xmm8, %xmm6
+; SSE4-NEXT:    pxor %xmm1, %xmm6
+; SSE4-NEXT:    packssdw %xmm7, %xmm6
+; SSE4-NEXT:    pcmpeqd %xmm8, %xmm5
+; SSE4-NEXT:    pxor %xmm1, %xmm5
+; SSE4-NEXT:    pcmpeqd %xmm8, %xmm4
+; SSE4-NEXT:    pxor %xmm1, %xmm4
+; SSE4-NEXT:    packssdw %xmm5, %xmm4
+; SSE4-NEXT:    packsswb %xmm6, %xmm4
+; SSE4-NEXT:    pmovmskb %xmm4, %eax
+; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    jne .LBB9_1
+; SSE4-NEXT:  # %bb.2: # %else
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    jne .LBB9_3
+; SSE4-NEXT:  .LBB9_4: # %else2
+; SSE4-NEXT:    testb $4, %al
+; SSE4-NEXT:    jne .LBB9_5
 ; SSE4-NEXT:  .LBB9_6: # %else4
-; SSE4-NEXT:    pextrb $12, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB9_8
-; SSE4-NEXT:  # %bb.7: # %cond.store5
-; SSE4-NEXT:    pextrw $3, %xmm0, 6(%rdi)
+; SSE4-NEXT:    testb $8, %al
+; SSE4-NEXT:    jne .LBB9_7
 ; SSE4-NEXT:  .LBB9_8: # %else6
-; SSE4-NEXT:    pxor %xmm4, %xmm4
-; SSE4-NEXT:    pcmpeqd %xmm5, %xmm4
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE4-NEXT:    pxor %xmm4, %xmm1
-; SSE4-NEXT:    pextrb $0, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB9_10
-; SSE4-NEXT:  # %bb.9: # %cond.store7
-; SSE4-NEXT:    pextrw $4, %xmm0, 8(%rdi)
+; SSE4-NEXT:    testb $16, %al
+; SSE4-NEXT:    jne .LBB9_9
 ; SSE4-NEXT:  .LBB9_10: # %else8
-; SSE4-NEXT:    pextrb $4, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB9_12
-; SSE4-NEXT:  # %bb.11: # %cond.store9
-; SSE4-NEXT:    pextrw $5, %xmm0, 10(%rdi)
+; SSE4-NEXT:    testb $32, %al
+; SSE4-NEXT:    jne .LBB9_11
 ; SSE4-NEXT:  .LBB9_12: # %else10
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm5
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE4-NEXT:    pxor %xmm5, %xmm1
-; SSE4-NEXT:    pextrb $8, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $64, %al
 ; SSE4-NEXT:    je .LBB9_14
-; SSE4-NEXT:  # %bb.13: # %cond.store11
+; SSE4-NEXT:  .LBB9_13: # %cond.store11
 ; SSE4-NEXT:    pextrw $6, %xmm0, 12(%rdi)
 ; SSE4-NEXT:  .LBB9_14: # %else12
 ; SSE4-NEXT:    pblendw {{.*#+}} xmm3 = xmm3[0],xmm8[1],xmm3[2],xmm8[3],xmm3[4],xmm8[5],xmm3[6],xmm8[7]
 ; SSE4-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0],xmm8[1],xmm2[2],xmm8[3],xmm2[4],xmm8[5],xmm2[6],xmm8[7]
-; SSE4-NEXT:    pextrb $12, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $-128, %al
 ; SSE4-NEXT:    je .LBB9_16
 ; SSE4-NEXT:  # %bb.15: # %cond.store13
 ; SSE4-NEXT:    pextrw $7, %xmm0, 14(%rdi)
 ; SSE4-NEXT:  .LBB9_16: # %else14
 ; SSE4-NEXT:    packusdw %xmm3, %xmm2
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm6, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE4-NEXT:    pxor %xmm1, %xmm0
-; SSE4-NEXT:    pextrb $0, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB9_18
-; SSE4-NEXT:  # %bb.17: # %cond.store15
+; SSE4-NEXT:    testl $256, %eax # imm = 0x100
+; SSE4-NEXT:    jne .LBB9_17
+; SSE4-NEXT:  # %bb.18: # %else16
+; SSE4-NEXT:    testl $512, %eax # imm = 0x200
+; SSE4-NEXT:    jne .LBB9_19
+; SSE4-NEXT:  .LBB9_20: # %else18
+; SSE4-NEXT:    testl $1024, %eax # imm = 0x400
+; SSE4-NEXT:    jne .LBB9_21
+; SSE4-NEXT:  .LBB9_22: # %else20
+; SSE4-NEXT:    testl $2048, %eax # imm = 0x800
+; SSE4-NEXT:    jne .LBB9_23
+; SSE4-NEXT:  .LBB9_24: # %else22
+; SSE4-NEXT:    testl $4096, %eax # imm = 0x1000
+; SSE4-NEXT:    jne .LBB9_25
+; SSE4-NEXT:  .LBB9_26: # %else24
+; SSE4-NEXT:    testl $8192, %eax # imm = 0x2000
+; SSE4-NEXT:    jne .LBB9_27
+; SSE4-NEXT:  .LBB9_28: # %else26
+; SSE4-NEXT:    testl $16384, %eax # imm = 0x4000
+; SSE4-NEXT:    jne .LBB9_29
+; SSE4-NEXT:  .LBB9_30: # %else28
+; SSE4-NEXT:    testl $32768, %eax # imm = 0x8000
+; SSE4-NEXT:    jne .LBB9_31
+; SSE4-NEXT:  .LBB9_32: # %else30
+; SSE4-NEXT:    retq
+; SSE4-NEXT:  .LBB9_1: # %cond.store
+; SSE4-NEXT:    pextrw $0, %xmm0, (%rdi)
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    je .LBB9_4
+; SSE4-NEXT:  .LBB9_3: # %cond.store1
+; SSE4-NEXT:    pextrw $1, %xmm0, 2(%rdi)
+; SSE4-NEXT:    testb $4, %al
+; SSE4-NEXT:    je .LBB9_6
+; SSE4-NEXT:  .LBB9_5: # %cond.store3
+; SSE4-NEXT:    pextrw $2, %xmm0, 4(%rdi)
+; SSE4-NEXT:    testb $8, %al
+; SSE4-NEXT:    je .LBB9_8
+; SSE4-NEXT:  .LBB9_7: # %cond.store5
+; SSE4-NEXT:    pextrw $3, %xmm0, 6(%rdi)
+; SSE4-NEXT:    testb $16, %al
+; SSE4-NEXT:    je .LBB9_10
+; SSE4-NEXT:  .LBB9_9: # %cond.store7
+; SSE4-NEXT:    pextrw $4, %xmm0, 8(%rdi)
+; SSE4-NEXT:    testb $32, %al
+; SSE4-NEXT:    je .LBB9_12
+; SSE4-NEXT:  .LBB9_11: # %cond.store9
+; SSE4-NEXT:    pextrw $5, %xmm0, 10(%rdi)
+; SSE4-NEXT:    testb $64, %al
+; SSE4-NEXT:    jne .LBB9_13
+; SSE4-NEXT:    jmp .LBB9_14
+; SSE4-NEXT:  .LBB9_17: # %cond.store15
 ; SSE4-NEXT:    pextrw $0, %xmm2, 16(%rdi)
-; SSE4-NEXT:  .LBB9_18: # %else16
-; SSE4-NEXT:    pextrb $4, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $512, %eax # imm = 0x200
 ; SSE4-NEXT:    je .LBB9_20
-; SSE4-NEXT:  # %bb.19: # %cond.store17
+; SSE4-NEXT:  .LBB9_19: # %cond.store17
 ; SSE4-NEXT:    pextrw $1, %xmm2, 18(%rdi)
-; SSE4-NEXT:  .LBB9_20: # %else18
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqd %xmm0, %xmm6
-; SSE4-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE4-NEXT:    pxor %xmm6, %xmm0
-; SSE4-NEXT:    pextrb $8, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $1024, %eax # imm = 0x400
 ; SSE4-NEXT:    je .LBB9_22
-; SSE4-NEXT:  # %bb.21: # %cond.store19
+; SSE4-NEXT:  .LBB9_21: # %cond.store19
 ; SSE4-NEXT:    pextrw $2, %xmm2, 20(%rdi)
-; SSE4-NEXT:  .LBB9_22: # %else20
-; SSE4-NEXT:    pextrb $12, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $2048, %eax # imm = 0x800
 ; SSE4-NEXT:    je .LBB9_24
-; SSE4-NEXT:  # %bb.23: # %cond.store21
+; SSE4-NEXT:  .LBB9_23: # %cond.store21
 ; SSE4-NEXT:    pextrw $3, %xmm2, 22(%rdi)
-; SSE4-NEXT:  .LBB9_24: # %else22
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm7, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE4-NEXT:    pxor %xmm1, %xmm0
-; SSE4-NEXT:    pextrb $0, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $4096, %eax # imm = 0x1000
 ; SSE4-NEXT:    je .LBB9_26
-; SSE4-NEXT:  # %bb.25: # %cond.store23
+; SSE4-NEXT:  .LBB9_25: # %cond.store23
 ; SSE4-NEXT:    pextrw $4, %xmm2, 24(%rdi)
-; SSE4-NEXT:  .LBB9_26: # %else24
-; SSE4-NEXT:    pextrb $4, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $8192, %eax # imm = 0x2000
 ; SSE4-NEXT:    je .LBB9_28
-; SSE4-NEXT:  # %bb.27: # %cond.store25
+; SSE4-NEXT:  .LBB9_27: # %cond.store25
 ; SSE4-NEXT:    pextrw $5, %xmm2, 26(%rdi)
-; SSE4-NEXT:  .LBB9_28: # %else26
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqd %xmm0, %xmm7
-; SSE4-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE4-NEXT:    pxor %xmm7, %xmm0
-; SSE4-NEXT:    pextrb $8, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $16384, %eax # imm = 0x4000
 ; SSE4-NEXT:    je .LBB9_30
-; SSE4-NEXT:  # %bb.29: # %cond.store27
+; SSE4-NEXT:  .LBB9_29: # %cond.store27
 ; SSE4-NEXT:    pextrw $6, %xmm2, 28(%rdi)
-; SSE4-NEXT:  .LBB9_30: # %else28
-; SSE4-NEXT:    pextrb $12, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $32768, %eax # imm = 0x8000
 ; SSE4-NEXT:    je .LBB9_32
-; SSE4-NEXT:  # %bb.31: # %cond.store29
+; SSE4-NEXT:  .LBB9_31: # %cond.store29
 ; SSE4-NEXT:    pextrw $7, %xmm2, 30(%rdi)
-; SSE4-NEXT:  .LBB9_32: # %else30
 ; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: truncstore_v16i32_v16i16:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpxor %xmm8, %xmm8, %xmm8
-; AVX1-NEXT:    vpcmpeqd %xmm8, %xmm2, %xmm6
-; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
-; AVX1-NEXT:    vpxor %xmm5, %xmm6, %xmm6
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm6, %xmm6
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm6, %xmm6
-; AVX1-NEXT:    vmovaps {{.*#+}} ymm7 = [65535,65535,65535,65535,65535,65535,65535,65535]
-; AVX1-NEXT:    vandps %ymm7, %ymm1, %ymm1
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm4
-; AVX1-NEXT:    vpackusdw %xmm4, %xmm1, %xmm1
-; AVX1-NEXT:    vandps %ymm7, %ymm0, %ymm0
+; AVX1-NEXT:    vmovaps {{.*#+}} ymm4 = [65535,65535,65535,65535,65535,65535,65535,65535]
+; AVX1-NEXT:    vandps %ymm4, %ymm1, %ymm1
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm5
+; AVX1-NEXT:    vpackusdw %xmm5, %xmm1, %xmm1
+; AVX1-NEXT:    vandps %ymm4, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
 ; AVX1-NEXT:    vpackusdw %xmm4, %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX1-NEXT:    vpextrb $0, %xmm6, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je .LBB9_2
-; AVX1-NEXT:  # %bb.1: # %cond.store
-; AVX1-NEXT:    vpextrw $0, %xmm0, (%rdi)
-; AVX1-NEXT:  .LBB9_2: # %else
-; AVX1-NEXT:    vpcmpeqd %xmm8, %xmm2, %xmm1
+; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm1
+; AVX1-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm1, %xmm1
+; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
 ; AVX1-NEXT:    vpxor %xmm5, %xmm1, %xmm1
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm1, %xmm1
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $1, %xmm1, %eax
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpxor %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vpackssdw %xmm1, %xmm3, %xmm1
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm3
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpxor %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm2, %xmm2
+; AVX1-NEXT:    vpxor %xmm5, %xmm2, %xmm2
+; AVX1-NEXT:    vpackssdw %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpacksswb %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vpmovmskb %xmm1, %eax
 ; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    jne .LBB9_1
+; AVX1-NEXT:  # %bb.2: # %else
+; AVX1-NEXT:    testb $2, %al
+; AVX1-NEXT:    jne .LBB9_3
+; AVX1-NEXT:  .LBB9_4: # %else2
+; AVX1-NEXT:    testb $4, %al
+; AVX1-NEXT:    jne .LBB9_5
+; AVX1-NEXT:  .LBB9_6: # %else4
+; AVX1-NEXT:    testb $8, %al
+; AVX1-NEXT:    jne .LBB9_7
+; AVX1-NEXT:  .LBB9_8: # %else6
+; AVX1-NEXT:    testb $16, %al
+; AVX1-NEXT:    jne .LBB9_9
+; AVX1-NEXT:  .LBB9_10: # %else8
+; AVX1-NEXT:    testb $32, %al
+; AVX1-NEXT:    jne .LBB9_11
+; AVX1-NEXT:  .LBB9_12: # %else10
+; AVX1-NEXT:    testb $64, %al
+; AVX1-NEXT:    jne .LBB9_13
+; AVX1-NEXT:  .LBB9_14: # %else12
+; AVX1-NEXT:    testb $-128, %al
+; AVX1-NEXT:    je .LBB9_16
+; AVX1-NEXT:  .LBB9_15: # %cond.store13
+; AVX1-NEXT:    vpextrw $7, %xmm0, 14(%rdi)
+; AVX1-NEXT:  .LBB9_16: # %else14
+; AVX1-NEXT:    testl $256, %eax # imm = 0x100
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    jne .LBB9_17
+; AVX1-NEXT:  # %bb.18: # %else16
+; AVX1-NEXT:    testl $512, %eax # imm = 0x200
+; AVX1-NEXT:    jne .LBB9_19
+; AVX1-NEXT:  .LBB9_20: # %else18
+; AVX1-NEXT:    testl $1024, %eax # imm = 0x400
+; AVX1-NEXT:    jne .LBB9_21
+; AVX1-NEXT:  .LBB9_22: # %else20
+; AVX1-NEXT:    testl $2048, %eax # imm = 0x800
+; AVX1-NEXT:    jne .LBB9_23
+; AVX1-NEXT:  .LBB9_24: # %else22
+; AVX1-NEXT:    testl $4096, %eax # imm = 0x1000
+; AVX1-NEXT:    jne .LBB9_25
+; AVX1-NEXT:  .LBB9_26: # %else24
+; AVX1-NEXT:    testl $8192, %eax # imm = 0x2000
+; AVX1-NEXT:    jne .LBB9_27
+; AVX1-NEXT:  .LBB9_28: # %else26
+; AVX1-NEXT:    testl $16384, %eax # imm = 0x4000
+; AVX1-NEXT:    jne .LBB9_29
+; AVX1-NEXT:  .LBB9_30: # %else28
+; AVX1-NEXT:    testl $32768, %eax # imm = 0x8000
+; AVX1-NEXT:    jne .LBB9_31
+; AVX1-NEXT:  .LBB9_32: # %else30
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+; AVX1-NEXT:  .LBB9_1: # %cond.store
+; AVX1-NEXT:    vpextrw $0, %xmm0, (%rdi)
+; AVX1-NEXT:    testb $2, %al
 ; AVX1-NEXT:    je .LBB9_4
-; AVX1-NEXT:  # %bb.3: # %cond.store1
+; AVX1-NEXT:  .LBB9_3: # %cond.store1
 ; AVX1-NEXT:    vpextrw $1, %xmm0, 2(%rdi)
-; AVX1-NEXT:  .LBB9_4: # %else2
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm2, %xmm5
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm5, %xmm5
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm5, %xmm5
-; AVX1-NEXT:    vpextrb $2, %xmm5, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $4, %al
 ; AVX1-NEXT:    je .LBB9_6
-; AVX1-NEXT:  # %bb.5: # %cond.store3
+; AVX1-NEXT:  .LBB9_5: # %cond.store3
 ; AVX1-NEXT:    vpextrw $2, %xmm0, 4(%rdi)
-; AVX1-NEXT:  .LBB9_6: # %else4
-; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm4, %xmm1, %xmm1
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm1, %xmm1
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $3, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $8, %al
 ; AVX1-NEXT:    je .LBB9_8
-; AVX1-NEXT:  # %bb.7: # %cond.store5
+; AVX1-NEXT:  .LBB9_7: # %cond.store5
 ; AVX1-NEXT:    vpextrw $3, %xmm0, 6(%rdi)
-; AVX1-NEXT:  .LBB9_8: # %else6
-; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm2
-; AVX1-NEXT:    vpackssdw %xmm2, %xmm0, %xmm4
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm4, %xmm4
-; AVX1-NEXT:    vpextrb $4, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $16, %al
 ; AVX1-NEXT:    je .LBB9_10
-; AVX1-NEXT:  # %bb.9: # %cond.store7
+; AVX1-NEXT:  .LBB9_9: # %cond.store7
 ; AVX1-NEXT:    vpextrw $4, %xmm0, 8(%rdi)
-; AVX1-NEXT:  .LBB9_10: # %else8
-; AVX1-NEXT:    vpackssdw %xmm2, %xmm0, %xmm2
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $5, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $32, %al
 ; AVX1-NEXT:    je .LBB9_12
-; AVX1-NEXT:  # %bb.11: # %cond.store9
+; AVX1-NEXT:  .LBB9_11: # %cond.store9
 ; AVX1-NEXT:    vpextrw $5, %xmm0, 10(%rdi)
-; AVX1-NEXT:  .LBB9_12: # %else10
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpackssdw %xmm1, %xmm0, %xmm2
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $6, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $64, %al
 ; AVX1-NEXT:    je .LBB9_14
-; AVX1-NEXT:  # %bb.13: # %cond.store11
+; AVX1-NEXT:  .LBB9_13: # %cond.store11
 ; AVX1-NEXT:    vpextrw $6, %xmm0, 12(%rdi)
-; AVX1-NEXT:  .LBB9_14: # %else12
-; AVX1-NEXT:    vpackssdw %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $7, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je .LBB9_16
-; AVX1-NEXT:  # %bb.15: # %cond.store13
-; AVX1-NEXT:    vpextrw $7, %xmm0, 14(%rdi)
-; AVX1-NEXT:  .LBB9_16: # %else14
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm3, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpxor %xmm2, %xmm4, %xmm4
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm4, %xmm4
-; AVX1-NEXT:    vpacksswb %xmm4, %xmm0, %xmm4
-; AVX1-NEXT:    vpextrb $8, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT:    je .LBB9_18
-; AVX1-NEXT:  # %bb.17: # %cond.store15
+; AVX1-NEXT:    testb $-128, %al
+; AVX1-NEXT:    jne .LBB9_15
+; AVX1-NEXT:    jmp .LBB9_16
+; AVX1-NEXT:  .LBB9_17: # %cond.store15
 ; AVX1-NEXT:    vpextrw $0, %xmm0, 16(%rdi)
-; AVX1-NEXT:  .LBB9_18: # %else16
-; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm3, %xmm1
-; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm1, %xmm1
-; AVX1-NEXT:    vpacksswb %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpextrb $9, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $512, %eax # imm = 0x200
 ; AVX1-NEXT:    je .LBB9_20
-; AVX1-NEXT:  # %bb.19: # %cond.store17
+; AVX1-NEXT:  .LBB9_19: # %cond.store17
 ; AVX1-NEXT:    vpextrw $1, %xmm0, 18(%rdi)
-; AVX1-NEXT:  .LBB9_20: # %else18
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm3, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpxor %xmm2, %xmm4, %xmm4
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm4, %xmm4
-; AVX1-NEXT:    vpacksswb %xmm4, %xmm0, %xmm4
-; AVX1-NEXT:    vpextrb $10, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $1024, %eax # imm = 0x400
 ; AVX1-NEXT:    je .LBB9_22
-; AVX1-NEXT:  # %bb.21: # %cond.store19
+; AVX1-NEXT:  .LBB9_21: # %cond.store19
 ; AVX1-NEXT:    vpextrw $2, %xmm0, 20(%rdi)
-; AVX1-NEXT:  .LBB9_22: # %else20
-; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm3, %xmm1
-; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm1, %xmm1
-; AVX1-NEXT:    vpacksswb %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpextrb $11, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $2048, %eax # imm = 0x800
 ; AVX1-NEXT:    je .LBB9_24
-; AVX1-NEXT:  # %bb.23: # %cond.store21
+; AVX1-NEXT:  .LBB9_23: # %cond.store21
 ; AVX1-NEXT:    vpextrw $3, %xmm0, 22(%rdi)
-; AVX1-NEXT:  .LBB9_24: # %else22
-; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm1
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpackssdw %xmm2, %xmm0, %xmm3
-; AVX1-NEXT:    vpacksswb %xmm3, %xmm0, %xmm3
-; AVX1-NEXT:    vpextrb $12, %xmm3, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $4096, %eax # imm = 0x1000
 ; AVX1-NEXT:    je .LBB9_26
-; AVX1-NEXT:  # %bb.25: # %cond.store23
+; AVX1-NEXT:  .LBB9_25: # %cond.store23
 ; AVX1-NEXT:    vpextrw $4, %xmm0, 24(%rdi)
-; AVX1-NEXT:  .LBB9_26: # %else24
-; AVX1-NEXT:    vpackssdw %xmm2, %xmm0, %xmm2
-; AVX1-NEXT:    vpacksswb %xmm2, %xmm0, %xmm2
-; AVX1-NEXT:    vpextrb $13, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $8192, %eax # imm = 0x2000
 ; AVX1-NEXT:    je .LBB9_28
-; AVX1-NEXT:  # %bb.27: # %cond.store25
+; AVX1-NEXT:  .LBB9_27: # %cond.store25
 ; AVX1-NEXT:    vpextrw $5, %xmm0, 26(%rdi)
-; AVX1-NEXT:  .LBB9_28: # %else26
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpackssdw %xmm1, %xmm0, %xmm2
-; AVX1-NEXT:    vpacksswb %xmm2, %xmm0, %xmm2
-; AVX1-NEXT:    vpextrb $14, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $16384, %eax # imm = 0x4000
 ; AVX1-NEXT:    je .LBB9_30
-; AVX1-NEXT:  # %bb.29: # %cond.store27
+; AVX1-NEXT:  .LBB9_29: # %cond.store27
 ; AVX1-NEXT:    vpextrw $6, %xmm0, 28(%rdi)
-; AVX1-NEXT:  .LBB9_30: # %else28
-; AVX1-NEXT:    vpackssdw %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpacksswb %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpextrb $15, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $32768, %eax # imm = 0x8000
 ; AVX1-NEXT:    je .LBB9_32
-; AVX1-NEXT:  # %bb.31: # %cond.store29
+; AVX1-NEXT:  .LBB9_31: # %cond.store29
 ; AVX1-NEXT:    vpextrw $7, %xmm0, 30(%rdi)
-; AVX1-NEXT:  .LBB9_32: # %else30
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: truncstore_v16i32_v16i16:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm2, %xmm6
-; AVX2-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
-; AVX2-NEXT:    vpxor %xmm5, %xmm6, %xmm6
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm6, %xmm6
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm6, %xmm6
-; AVX2-NEXT:    vmovdqa {{.*#+}} ymm7 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; AVX2-NEXT:    vpshufb %ymm7, %ymm0, %ymm0
+; AVX2-NEXT:    vmovdqa {{.*#+}} ymm5 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
+; AVX2-NEXT:    vpshufb %ymm5, %ymm0, %ymm0
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-NEXT:    vpshufb %ymm7, %ymm1, %ymm1
+; AVX2-NEXT:    vpshufb %ymm5, %ymm1, %ymm1
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3]
 ; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpextrb $0, %xmm6, %eax
+; AVX2-NEXT:    vpcmpeqd %ymm4, %ymm3, %ymm1
+; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm3, %ymm3
+; AVX2-NEXT:    vpxor %ymm3, %ymm1, %ymm1
+; AVX2-NEXT:    vpcmpeqd %ymm4, %ymm2, %ymm2
+; AVX2-NEXT:    vpxor %ymm3, %ymm2, %ymm2
+; AVX2-NEXT:    vpackssdw %ymm1, %ymm2, %ymm1
+; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
+; AVX2-NEXT:    vpacksswb %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpmovmskb %xmm1, %eax
 ; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je .LBB9_2
-; AVX2-NEXT:  # %bb.1: # %cond.store
+; AVX2-NEXT:    jne .LBB9_1
+; AVX2-NEXT:  # %bb.2: # %else
+; AVX2-NEXT:    testb $2, %al
+; AVX2-NEXT:    jne .LBB9_3
+; AVX2-NEXT:  .LBB9_4: # %else2
+; AVX2-NEXT:    testb $4, %al
+; AVX2-NEXT:    jne .LBB9_5
+; AVX2-NEXT:  .LBB9_6: # %else4
+; AVX2-NEXT:    testb $8, %al
+; AVX2-NEXT:    jne .LBB9_7
+; AVX2-NEXT:  .LBB9_8: # %else6
+; AVX2-NEXT:    testb $16, %al
+; AVX2-NEXT:    jne .LBB9_9
+; AVX2-NEXT:  .LBB9_10: # %else8
+; AVX2-NEXT:    testb $32, %al
+; AVX2-NEXT:    jne .LBB9_11
+; AVX2-NEXT:  .LBB9_12: # %else10
+; AVX2-NEXT:    testb $64, %al
+; AVX2-NEXT:    jne .LBB9_13
+; AVX2-NEXT:  .LBB9_14: # %else12
+; AVX2-NEXT:    testb $-128, %al
+; AVX2-NEXT:    je .LBB9_16
+; AVX2-NEXT:  .LBB9_15: # %cond.store13
+; AVX2-NEXT:    vpextrw $7, %xmm0, 14(%rdi)
+; AVX2-NEXT:  .LBB9_16: # %else14
+; AVX2-NEXT:    testl $256, %eax # imm = 0x100
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
+; AVX2-NEXT:    jne .LBB9_17
+; AVX2-NEXT:  # %bb.18: # %else16
+; AVX2-NEXT:    testl $512, %eax # imm = 0x200
+; AVX2-NEXT:    jne .LBB9_19
+; AVX2-NEXT:  .LBB9_20: # %else18
+; AVX2-NEXT:    testl $1024, %eax # imm = 0x400
+; AVX2-NEXT:    jne .LBB9_21
+; AVX2-NEXT:  .LBB9_22: # %else20
+; AVX2-NEXT:    testl $2048, %eax # imm = 0x800
+; AVX2-NEXT:    jne .LBB9_23
+; AVX2-NEXT:  .LBB9_24: # %else22
+; AVX2-NEXT:    testl $4096, %eax # imm = 0x1000
+; AVX2-NEXT:    jne .LBB9_25
+; AVX2-NEXT:  .LBB9_26: # %else24
+; AVX2-NEXT:    testl $8192, %eax # imm = 0x2000
+; AVX2-NEXT:    jne .LBB9_27
+; AVX2-NEXT:  .LBB9_28: # %else26
+; AVX2-NEXT:    testl $16384, %eax # imm = 0x4000
+; AVX2-NEXT:    jne .LBB9_29
+; AVX2-NEXT:  .LBB9_30: # %else28
+; AVX2-NEXT:    testl $32768, %eax # imm = 0x8000
+; AVX2-NEXT:    jne .LBB9_31
+; AVX2-NEXT:  .LBB9_32: # %else30
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+; AVX2-NEXT:  .LBB9_1: # %cond.store
 ; AVX2-NEXT:    vpextrw $0, %xmm0, (%rdi)
-; AVX2-NEXT:  .LBB9_2: # %else
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm2, %xmm1
-; AVX2-NEXT:    vpxor %xmm5, %xmm1, %xmm1
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm1, %xmm1
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm1, %xmm1
-; AVX2-NEXT:    vpextrb $1, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $2, %al
 ; AVX2-NEXT:    je .LBB9_4
-; AVX2-NEXT:  # %bb.3: # %cond.store1
+; AVX2-NEXT:  .LBB9_3: # %cond.store1
 ; AVX2-NEXT:    vpextrw $1, %xmm0, 2(%rdi)
-; AVX2-NEXT:  .LBB9_4: # %else2
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm2, %xmm5
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm5, %xmm5
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm5, %xmm5
-; AVX2-NEXT:    vpextrb $2, %xmm5, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $4, %al
 ; AVX2-NEXT:    je .LBB9_6
-; AVX2-NEXT:  # %bb.5: # %cond.store3
+; AVX2-NEXT:  .LBB9_5: # %cond.store3
 ; AVX2-NEXT:    vpextrw $2, %xmm0, 4(%rdi)
-; AVX2-NEXT:  .LBB9_6: # %else4
-; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm2, %xmm1
-; AVX2-NEXT:    vpxor %xmm4, %xmm1, %xmm1
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm1, %xmm1
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm1, %xmm1
-; AVX2-NEXT:    vpextrb $3, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $8, %al
 ; AVX2-NEXT:    je .LBB9_8
-; AVX2-NEXT:  # %bb.7: # %cond.store5
+; AVX2-NEXT:  .LBB9_7: # %cond.store5
 ; AVX2-NEXT:    vpextrw $3, %xmm0, 6(%rdi)
-; AVX2-NEXT:  .LBB9_8: # %else6
-; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm1
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpxor %xmm4, %xmm2, %xmm2
-; AVX2-NEXT:    vpackssdw %xmm2, %xmm0, %xmm4
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm4, %xmm4
-; AVX2-NEXT:    vpextrb $4, %xmm4, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $16, %al
 ; AVX2-NEXT:    je .LBB9_10
-; AVX2-NEXT:  # %bb.9: # %cond.store7
+; AVX2-NEXT:  .LBB9_9: # %cond.store7
 ; AVX2-NEXT:    vpextrw $4, %xmm0, 8(%rdi)
-; AVX2-NEXT:  .LBB9_10: # %else8
-; AVX2-NEXT:    vpackssdw %xmm2, %xmm0, %xmm2
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrb $5, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $32, %al
 ; AVX2-NEXT:    je .LBB9_12
-; AVX2-NEXT:  # %bb.11: # %cond.store9
+; AVX2-NEXT:  .LBB9_11: # %cond.store9
 ; AVX2-NEXT:    vpextrw $5, %xmm0, 10(%rdi)
-; AVX2-NEXT:  .LBB9_12: # %else10
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpackssdw %xmm1, %xmm0, %xmm2
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrb $6, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $64, %al
 ; AVX2-NEXT:    je .LBB9_14
-; AVX2-NEXT:  # %bb.13: # %cond.store11
+; AVX2-NEXT:  .LBB9_13: # %cond.store11
 ; AVX2-NEXT:    vpextrw $6, %xmm0, 12(%rdi)
-; AVX2-NEXT:  .LBB9_14: # %else12
-; AVX2-NEXT:    vpackssdw %xmm1, %xmm0, %xmm1
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm1, %xmm1
-; AVX2-NEXT:    vpextrb $7, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je .LBB9_16
-; AVX2-NEXT:  # %bb.15: # %cond.store13
-; AVX2-NEXT:    vpextrw $7, %xmm0, 14(%rdi)
-; AVX2-NEXT:  .LBB9_16: # %else14
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm3, %xmm4
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpxor %xmm2, %xmm4, %xmm4
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm4, %xmm4
-; AVX2-NEXT:    vpacksswb %xmm4, %xmm0, %xmm4
-; AVX2-NEXT:    vpextrb $8, %xmm4, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; AVX2-NEXT:    je .LBB9_18
-; AVX2-NEXT:  # %bb.17: # %cond.store15
+; AVX2-NEXT:    testb $-128, %al
+; AVX2-NEXT:    jne .LBB9_15
+; AVX2-NEXT:    jmp .LBB9_16
+; AVX2-NEXT:  .LBB9_17: # %cond.store15
 ; AVX2-NEXT:    vpextrw $0, %xmm0, 16(%rdi)
-; AVX2-NEXT:  .LBB9_18: # %else16
-; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm3, %xmm1
-; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm1, %xmm1
-; AVX2-NEXT:    vpacksswb %xmm1, %xmm0, %xmm1
-; AVX2-NEXT:    vpextrb $9, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $512, %eax # imm = 0x200
 ; AVX2-NEXT:    je .LBB9_20
-; AVX2-NEXT:  # %bb.19: # %cond.store17
+; AVX2-NEXT:  .LBB9_19: # %cond.store17
 ; AVX2-NEXT:    vpextrw $1, %xmm0, 18(%rdi)
-; AVX2-NEXT:  .LBB9_20: # %else18
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm3, %xmm4
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpxor %xmm2, %xmm4, %xmm4
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm4, %xmm4
-; AVX2-NEXT:    vpacksswb %xmm4, %xmm0, %xmm4
-; AVX2-NEXT:    vpextrb $10, %xmm4, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $1024, %eax # imm = 0x400
 ; AVX2-NEXT:    je .LBB9_22
-; AVX2-NEXT:  # %bb.21: # %cond.store19
+; AVX2-NEXT:  .LBB9_21: # %cond.store19
 ; AVX2-NEXT:    vpextrw $2, %xmm0, 20(%rdi)
-; AVX2-NEXT:  .LBB9_22: # %else20
-; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm3, %xmm1
-; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm1, %xmm1
-; AVX2-NEXT:    vpacksswb %xmm1, %xmm0, %xmm1
-; AVX2-NEXT:    vpextrb $11, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $2048, %eax # imm = 0x800
 ; AVX2-NEXT:    je .LBB9_24
-; AVX2-NEXT:  # %bb.23: # %cond.store21
+; AVX2-NEXT:  .LBB9_23: # %cond.store21
 ; AVX2-NEXT:    vpextrw $3, %xmm0, 22(%rdi)
-; AVX2-NEXT:  .LBB9_24: # %else22
-; AVX2-NEXT:    vextracti128 $1, %ymm3, %xmm1
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX2-NEXT:    vpackssdw %xmm2, %xmm0, %xmm3
-; AVX2-NEXT:    vpacksswb %xmm3, %xmm0, %xmm3
-; AVX2-NEXT:    vpextrb $12, %xmm3, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $4096, %eax # imm = 0x1000
 ; AVX2-NEXT:    je .LBB9_26
-; AVX2-NEXT:  # %bb.25: # %cond.store23
+; AVX2-NEXT:  .LBB9_25: # %cond.store23
 ; AVX2-NEXT:    vpextrw $4, %xmm0, 24(%rdi)
-; AVX2-NEXT:  .LBB9_26: # %else24
-; AVX2-NEXT:    vpackssdw %xmm2, %xmm0, %xmm2
-; AVX2-NEXT:    vpacksswb %xmm2, %xmm0, %xmm2
-; AVX2-NEXT:    vpextrb $13, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $8192, %eax # imm = 0x2000
 ; AVX2-NEXT:    je .LBB9_28
-; AVX2-NEXT:  # %bb.27: # %cond.store25
+; AVX2-NEXT:  .LBB9_27: # %cond.store25
 ; AVX2-NEXT:    vpextrw $5, %xmm0, 26(%rdi)
-; AVX2-NEXT:  .LBB9_28: # %else26
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpackssdw %xmm1, %xmm0, %xmm2
-; AVX2-NEXT:    vpacksswb %xmm2, %xmm0, %xmm2
-; AVX2-NEXT:    vpextrb $14, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $16384, %eax # imm = 0x4000
 ; AVX2-NEXT:    je .LBB9_30
-; AVX2-NEXT:  # %bb.29: # %cond.store27
+; AVX2-NEXT:  .LBB9_29: # %cond.store27
 ; AVX2-NEXT:    vpextrw $6, %xmm0, 28(%rdi)
-; AVX2-NEXT:  .LBB9_30: # %else28
-; AVX2-NEXT:    vpackssdw %xmm1, %xmm0, %xmm1
-; AVX2-NEXT:    vpacksswb %xmm1, %xmm0, %xmm1
-; AVX2-NEXT:    vpextrb $15, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $32768, %eax # imm = 0x8000
 ; AVX2-NEXT:    je .LBB9_32
-; AVX2-NEXT:  # %bb.31: # %cond.store29
+; AVX2-NEXT:  .LBB9_31: # %cond.store29
 ; AVX2-NEXT:    vpextrw $7, %xmm0, 30(%rdi)
-; AVX2-NEXT:  .LBB9_32: # %else30
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
@@ -2568,116 +2511,117 @@ define void @truncstore_v16i32_v16i16(<16 x i32> %x, <16 x i16>* %p, <16 x i32>
 ; AVX512F-NEXT:    vpmovdw %zmm0, %ymm0
 ; AVX512F-NEXT:    kmovw %k0, %eax
 ; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB9_2
-; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    jne .LBB9_1
+; AVX512F-NEXT:  # %bb.2: # %else
+; AVX512F-NEXT:    testb $2, %al
+; AVX512F-NEXT:    jne .LBB9_3
+; AVX512F-NEXT:  .LBB9_4: # %else2
+; AVX512F-NEXT:    testb $4, %al
+; AVX512F-NEXT:    jne .LBB9_5
+; AVX512F-NEXT:  .LBB9_6: # %else4
+; AVX512F-NEXT:    testb $8, %al
+; AVX512F-NEXT:    jne .LBB9_7
+; AVX512F-NEXT:  .LBB9_8: # %else6
+; AVX512F-NEXT:    testb $16, %al
+; AVX512F-NEXT:    jne .LBB9_9
+; AVX512F-NEXT:  .LBB9_10: # %else8
+; AVX512F-NEXT:    testb $32, %al
+; AVX512F-NEXT:    jne .LBB9_11
+; AVX512F-NEXT:  .LBB9_12: # %else10
+; AVX512F-NEXT:    testb $64, %al
+; AVX512F-NEXT:    jne .LBB9_13
+; AVX512F-NEXT:  .LBB9_14: # %else12
+; AVX512F-NEXT:    testb $-128, %al
+; AVX512F-NEXT:    je .LBB9_16
+; AVX512F-NEXT:  .LBB9_15: # %cond.store13
+; AVX512F-NEXT:    vpextrw $7, %xmm0, 14(%rdi)
+; AVX512F-NEXT:  .LBB9_16: # %else14
+; AVX512F-NEXT:    testl $256, %eax # imm = 0x100
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm0
+; AVX512F-NEXT:    jne .LBB9_17
+; AVX512F-NEXT:  # %bb.18: # %else16
+; AVX512F-NEXT:    testl $512, %eax # imm = 0x200
+; AVX512F-NEXT:    jne .LBB9_19
+; AVX512F-NEXT:  .LBB9_20: # %else18
+; AVX512F-NEXT:    testl $1024, %eax # imm = 0x400
+; AVX512F-NEXT:    jne .LBB9_21
+; AVX512F-NEXT:  .LBB9_22: # %else20
+; AVX512F-NEXT:    testl $2048, %eax # imm = 0x800
+; AVX512F-NEXT:    jne .LBB9_23
+; AVX512F-NEXT:  .LBB9_24: # %else22
+; AVX512F-NEXT:    testl $4096, %eax # imm = 0x1000
+; AVX512F-NEXT:    jne .LBB9_25
+; AVX512F-NEXT:  .LBB9_26: # %else24
+; AVX512F-NEXT:    testl $8192, %eax # imm = 0x2000
+; AVX512F-NEXT:    jne .LBB9_27
+; AVX512F-NEXT:  .LBB9_28: # %else26
+; AVX512F-NEXT:    testl $16384, %eax # imm = 0x4000
+; AVX512F-NEXT:    jne .LBB9_29
+; AVX512F-NEXT:  .LBB9_30: # %else28
+; AVX512F-NEXT:    testl $32768, %eax # imm = 0x8000
+; AVX512F-NEXT:    jne .LBB9_31
+; AVX512F-NEXT:  .LBB9_32: # %else30
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+; AVX512F-NEXT:  .LBB9_1: # %cond.store
 ; AVX512F-NEXT:    vpextrw $0, %xmm0, (%rdi)
-; AVX512F-NEXT:  .LBB9_2: # %else
-; AVX512F-NEXT:    kshiftrw $1, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $2, %al
 ; AVX512F-NEXT:    je .LBB9_4
-; AVX512F-NEXT:  # %bb.3: # %cond.store1
+; AVX512F-NEXT:  .LBB9_3: # %cond.store1
 ; AVX512F-NEXT:    vpextrw $1, %xmm0, 2(%rdi)
-; AVX512F-NEXT:  .LBB9_4: # %else2
-; AVX512F-NEXT:    kshiftrw $2, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $4, %al
 ; AVX512F-NEXT:    je .LBB9_6
-; AVX512F-NEXT:  # %bb.5: # %cond.store3
+; AVX512F-NEXT:  .LBB9_5: # %cond.store3
 ; AVX512F-NEXT:    vpextrw $2, %xmm0, 4(%rdi)
-; AVX512F-NEXT:  .LBB9_6: # %else4
-; AVX512F-NEXT:    kshiftrw $3, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $8, %al
 ; AVX512F-NEXT:    je .LBB9_8
-; AVX512F-NEXT:  # %bb.7: # %cond.store5
+; AVX512F-NEXT:  .LBB9_7: # %cond.store5
 ; AVX512F-NEXT:    vpextrw $3, %xmm0, 6(%rdi)
-; AVX512F-NEXT:  .LBB9_8: # %else6
-; AVX512F-NEXT:    kshiftrw $4, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $16, %al
 ; AVX512F-NEXT:    je .LBB9_10
-; AVX512F-NEXT:  # %bb.9: # %cond.store7
+; AVX512F-NEXT:  .LBB9_9: # %cond.store7
 ; AVX512F-NEXT:    vpextrw $4, %xmm0, 8(%rdi)
-; AVX512F-NEXT:  .LBB9_10: # %else8
-; AVX512F-NEXT:    kshiftrw $5, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $32, %al
 ; AVX512F-NEXT:    je .LBB9_12
-; AVX512F-NEXT:  # %bb.11: # %cond.store9
+; AVX512F-NEXT:  .LBB9_11: # %cond.store9
 ; AVX512F-NEXT:    vpextrw $5, %xmm0, 10(%rdi)
-; AVX512F-NEXT:  .LBB9_12: # %else10
-; AVX512F-NEXT:    kshiftrw $6, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $64, %al
 ; AVX512F-NEXT:    je .LBB9_14
-; AVX512F-NEXT:  # %bb.13: # %cond.store11
+; AVX512F-NEXT:  .LBB9_13: # %cond.store11
 ; AVX512F-NEXT:    vpextrw $6, %xmm0, 12(%rdi)
-; AVX512F-NEXT:  .LBB9_14: # %else12
-; AVX512F-NEXT:    kshiftrw $7, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB9_16
-; AVX512F-NEXT:  # %bb.15: # %cond.store13
-; AVX512F-NEXT:    vpextrw $7, %xmm0, 14(%rdi)
-; AVX512F-NEXT:  .LBB9_16: # %else14
-; AVX512F-NEXT:    kshiftrw $8, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; AVX512F-NEXT:    je .LBB9_18
-; AVX512F-NEXT:  # %bb.17: # %cond.store15
+; AVX512F-NEXT:    testb $-128, %al
+; AVX512F-NEXT:    jne .LBB9_15
+; AVX512F-NEXT:    jmp .LBB9_16
+; AVX512F-NEXT:  .LBB9_17: # %cond.store15
 ; AVX512F-NEXT:    vpextrw $0, %xmm0, 16(%rdi)
-; AVX512F-NEXT:  .LBB9_18: # %else16
-; AVX512F-NEXT:    kshiftrw $9, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $512, %eax # imm = 0x200
 ; AVX512F-NEXT:    je .LBB9_20
-; AVX512F-NEXT:  # %bb.19: # %cond.store17
+; AVX512F-NEXT:  .LBB9_19: # %cond.store17
 ; AVX512F-NEXT:    vpextrw $1, %xmm0, 18(%rdi)
-; AVX512F-NEXT:  .LBB9_20: # %else18
-; AVX512F-NEXT:    kshiftrw $10, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $1024, %eax # imm = 0x400
 ; AVX512F-NEXT:    je .LBB9_22
-; AVX512F-NEXT:  # %bb.21: # %cond.store19
+; AVX512F-NEXT:  .LBB9_21: # %cond.store19
 ; AVX512F-NEXT:    vpextrw $2, %xmm0, 20(%rdi)
-; AVX512F-NEXT:  .LBB9_22: # %else20
-; AVX512F-NEXT:    kshiftrw $11, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $2048, %eax # imm = 0x800
 ; AVX512F-NEXT:    je .LBB9_24
-; AVX512F-NEXT:  # %bb.23: # %cond.store21
+; AVX512F-NEXT:  .LBB9_23: # %cond.store21
 ; AVX512F-NEXT:    vpextrw $3, %xmm0, 22(%rdi)
-; AVX512F-NEXT:  .LBB9_24: # %else22
-; AVX512F-NEXT:    kshiftrw $12, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $4096, %eax # imm = 0x1000
 ; AVX512F-NEXT:    je .LBB9_26
-; AVX512F-NEXT:  # %bb.25: # %cond.store23
+; AVX512F-NEXT:  .LBB9_25: # %cond.store23
 ; AVX512F-NEXT:    vpextrw $4, %xmm0, 24(%rdi)
-; AVX512F-NEXT:  .LBB9_26: # %else24
-; AVX512F-NEXT:    kshiftrw $13, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $8192, %eax # imm = 0x2000
 ; AVX512F-NEXT:    je .LBB9_28
-; AVX512F-NEXT:  # %bb.27: # %cond.store25
+; AVX512F-NEXT:  .LBB9_27: # %cond.store25
 ; AVX512F-NEXT:    vpextrw $5, %xmm0, 26(%rdi)
-; AVX512F-NEXT:  .LBB9_28: # %else26
-; AVX512F-NEXT:    kshiftrw $14, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $16384, %eax # imm = 0x4000
 ; AVX512F-NEXT:    je .LBB9_30
-; AVX512F-NEXT:  # %bb.29: # %cond.store27
+; AVX512F-NEXT:  .LBB9_29: # %cond.store27
 ; AVX512F-NEXT:    vpextrw $6, %xmm0, 28(%rdi)
-; AVX512F-NEXT:  .LBB9_30: # %else28
-; AVX512F-NEXT:    kshiftrw $15, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $32768, %eax # imm = 0x8000
 ; AVX512F-NEXT:    je .LBB9_32
-; AVX512F-NEXT:  # %bb.31: # %cond.store29
+; AVX512F-NEXT:  .LBB9_31: # %cond.store29
 ; AVX512F-NEXT:    vpextrw $7, %xmm0, 30(%rdi)
-; AVX512F-NEXT:  .LBB9_32: # %else30
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
@@ -2704,7 +2648,6 @@ define void @truncstore_v16i32_v16i8(<16 x i32> %x, <16 x i8>* %p, <16 x i32> %m
 ; SSE2-LABEL: truncstore_v16i32_v16i8:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pxor %xmm8, %xmm8
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm8
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0]
 ; SSE2-NEXT:    pand %xmm9, %xmm3
 ; SSE2-NEXT:    pand %xmm9, %xmm2
@@ -2713,158 +2656,124 @@ define void @truncstore_v16i32_v16i8(<16 x i32> %x, <16 x i8>* %p, <16 x i32> %m
 ; SSE2-NEXT:    pand %xmm9, %xmm0
 ; SSE2-NEXT:    packuswb %xmm1, %xmm0
 ; SSE2-NEXT:    packuswb %xmm2, %xmm0
-; SSE2-NEXT:    movd %xmm8, %eax
-; SSE2-NEXT:    notl %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    je .LBB10_2
-; SSE2-NEXT:  # %bb.1: # %cond.store
-; SSE2-NEXT:    movb %al, (%rdi)
-; SSE2-NEXT:  .LBB10_2: # %else
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm7
 ; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pxor %xmm1, %xmm8
-; SSE2-NEXT:    pextrw $2, %xmm8, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    je .LBB10_4
-; SSE2-NEXT:  # %bb.3: # %cond.store1
-; SSE2-NEXT:    movb %ah, 1(%rdi)
+; SSE2-NEXT:    pxor %xmm1, %xmm7
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm6
+; SSE2-NEXT:    pxor %xmm1, %xmm6
+; SSE2-NEXT:    packssdw %xmm7, %xmm6
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm5
+; SSE2-NEXT:    pxor %xmm1, %xmm5
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm4
+; SSE2-NEXT:    pxor %xmm1, %xmm4
+; SSE2-NEXT:    packssdw %xmm5, %xmm4
+; SSE2-NEXT:    packsswb %xmm6, %xmm4
+; SSE2-NEXT:    pmovmskb %xmm4, %eax
+; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    movd %xmm0, %ecx
+; SSE2-NEXT:    jne .LBB10_1
+; SSE2-NEXT:  # %bb.2: # %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne .LBB10_3
 ; SSE2-NEXT:  .LBB10_4: # %else2
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm4
-; SSE2-NEXT:    pxor %xmm4, %xmm1
-; SSE2-NEXT:    pextrw $4, %xmm1, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    je .LBB10_6
-; SSE2-NEXT:  # %bb.5: # %cond.store3
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    shrl $16, %ecx
-; SSE2-NEXT:    movb %cl, 2(%rdi)
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne .LBB10_5
 ; SSE2-NEXT:  .LBB10_6: # %else4
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pxor %xmm1, %xmm4
-; SSE2-NEXT:    pextrw $6, %xmm4, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je .LBB10_8
-; SSE2-NEXT:  # %bb.7: # %cond.store5
-; SSE2-NEXT:    shrl $24, %eax
-; SSE2-NEXT:    movb %al, 3(%rdi)
+; SSE2-NEXT:  .LBB10_7: # %cond.store5
+; SSE2-NEXT:    shrl $24, %ecx
+; SSE2-NEXT:    movb %cl, 3(%rdi)
 ; SSE2-NEXT:  .LBB10_8: # %else6
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm1
-; SSE2-NEXT:    movd %xmm1, %eax
-; SSE2-NEXT:    notl %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    pextrw $2, %xmm0, %eax
+; SSE2-NEXT:    testb $16, %al
+; SSE2-NEXT:    pextrw $2, %xmm0, %ecx
 ; SSE2-NEXT:    je .LBB10_10
 ; SSE2-NEXT:  # %bb.9: # %cond.store7
-; SSE2-NEXT:    movb %al, 4(%rdi)
+; SSE2-NEXT:    movb %cl, 4(%rdi)
 ; SSE2-NEXT:  .LBB10_10: # %else8
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT:    pxor %xmm2, %xmm1
-; SSE2-NEXT:    pextrw $2, %xmm1, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testb $32, %al
 ; SSE2-NEXT:    je .LBB10_12
 ; SSE2-NEXT:  # %bb.11: # %cond.store9
-; SSE2-NEXT:    movb %ah, 5(%rdi)
+; SSE2-NEXT:    movb %ch, 5(%rdi)
 ; SSE2-NEXT:  .LBB10_12: # %else10
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm5
-; SSE2-NEXT:    pxor %xmm5, %xmm2
-; SSE2-NEXT:    pextrw $4, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    pextrw $3, %xmm0, %eax
+; SSE2-NEXT:    testb $64, %al
+; SSE2-NEXT:    pextrw $3, %xmm0, %ecx
 ; SSE2-NEXT:    je .LBB10_14
 ; SSE2-NEXT:  # %bb.13: # %cond.store11
-; SSE2-NEXT:    movb %al, 6(%rdi)
+; SSE2-NEXT:    movb %cl, 6(%rdi)
 ; SSE2-NEXT:  .LBB10_14: # %else12
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pxor %xmm1, %xmm5
-; SSE2-NEXT:    pextrw $6, %xmm5, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testb $-128, %al
 ; SSE2-NEXT:    je .LBB10_16
 ; SSE2-NEXT:  # %bb.15: # %cond.store13
-; SSE2-NEXT:    movb %ah, 7(%rdi)
+; SSE2-NEXT:    movb %ch, 7(%rdi)
 ; SSE2-NEXT:  .LBB10_16: # %else14
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm6, %xmm1
-; SSE2-NEXT:    movd %xmm1, %eax
-; SSE2-NEXT:    notl %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
+; SSE2-NEXT:    testl $256, %eax # imm = 0x100
+; SSE2-NEXT:    pextrw $4, %xmm0, %ecx
 ; SSE2-NEXT:    je .LBB10_18
 ; SSE2-NEXT:  # %bb.17: # %cond.store15
-; SSE2-NEXT:    movb %al, 8(%rdi)
+; SSE2-NEXT:    movb %cl, 8(%rdi)
 ; SSE2-NEXT:  .LBB10_18: # %else16
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT:    pxor %xmm2, %xmm1
-; SSE2-NEXT:    pextrw $2, %xmm1, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $512, %eax # imm = 0x200
 ; SSE2-NEXT:    je .LBB10_20
 ; SSE2-NEXT:  # %bb.19: # %cond.store17
-; SSE2-NEXT:    movb %ah, 9(%rdi)
+; SSE2-NEXT:    movb %ch, 9(%rdi)
 ; SSE2-NEXT:  .LBB10_20: # %else18
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm6
-; SSE2-NEXT:    pxor %xmm6, %xmm2
-; SSE2-NEXT:    pextrw $4, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    pextrw $5, %xmm0, %eax
+; SSE2-NEXT:    testl $1024, %eax # imm = 0x400
+; SSE2-NEXT:    pextrw $5, %xmm0, %ecx
 ; SSE2-NEXT:    je .LBB10_22
 ; SSE2-NEXT:  # %bb.21: # %cond.store19
-; SSE2-NEXT:    movb %al, 10(%rdi)
+; SSE2-NEXT:    movb %cl, 10(%rdi)
 ; SSE2-NEXT:  .LBB10_22: # %else20
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pxor %xmm1, %xmm6
-; SSE2-NEXT:    pextrw $6, %xmm6, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $2048, %eax # imm = 0x800
 ; SSE2-NEXT:    je .LBB10_24
 ; SSE2-NEXT:  # %bb.23: # %cond.store21
-; SSE2-NEXT:    movb %ah, 11(%rdi)
+; SSE2-NEXT:    movb %ch, 11(%rdi)
 ; SSE2-NEXT:  .LBB10_24: # %else22
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm7, %xmm1
-; SSE2-NEXT:    movd %xmm1, %eax
-; SSE2-NEXT:    notl %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    pextrw $6, %xmm0, %eax
+; SSE2-NEXT:    testl $4096, %eax # imm = 0x1000
+; SSE2-NEXT:    pextrw $6, %xmm0, %ecx
 ; SSE2-NEXT:    je .LBB10_26
 ; SSE2-NEXT:  # %bb.25: # %cond.store23
-; SSE2-NEXT:    movb %al, 12(%rdi)
+; SSE2-NEXT:    movb %cl, 12(%rdi)
 ; SSE2-NEXT:  .LBB10_26: # %else24
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT:    pxor %xmm2, %xmm1
-; SSE2-NEXT:    pextrw $2, %xmm1, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $8192, %eax # imm = 0x2000
 ; SSE2-NEXT:    je .LBB10_28
 ; SSE2-NEXT:  # %bb.27: # %cond.store25
-; SSE2-NEXT:    movb %ah, 13(%rdi)
+; SSE2-NEXT:    movb %ch, 13(%rdi)
 ; SSE2-NEXT:  .LBB10_28: # %else26
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm7
-; SSE2-NEXT:    pxor %xmm7, %xmm2
-; SSE2-NEXT:    pextrw $4, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    pextrw $7, %xmm0, %eax
-; SSE2-NEXT:    je .LBB10_30
-; SSE2-NEXT:  # %bb.29: # %cond.store27
-; SSE2-NEXT:    movb %al, 14(%rdi)
-; SSE2-NEXT:  .LBB10_30: # %else28
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE2-NEXT:    pxor %xmm0, %xmm7
-; SSE2-NEXT:    pextrw $6, %xmm7, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    je .LBB10_32
-; SSE2-NEXT:  # %bb.31: # %cond.store29
-; SSE2-NEXT:    movb %ah, 15(%rdi)
+; SSE2-NEXT:    testl $16384, %eax # imm = 0x4000
+; SSE2-NEXT:    pextrw $7, %xmm0, %ecx
+; SSE2-NEXT:    jne .LBB10_29
+; SSE2-NEXT:  # %bb.30: # %else28
+; SSE2-NEXT:    testl $32768, %eax # imm = 0x8000
+; SSE2-NEXT:    jne .LBB10_31
 ; SSE2-NEXT:  .LBB10_32: # %else30
 ; SSE2-NEXT:    retq
+; SSE2-NEXT:  .LBB10_1: # %cond.store
+; SSE2-NEXT:    movb %cl, (%rdi)
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    je .LBB10_4
+; SSE2-NEXT:  .LBB10_3: # %cond.store1
+; SSE2-NEXT:    movb %ch, 1(%rdi)
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    je .LBB10_6
+; SSE2-NEXT:  .LBB10_5: # %cond.store3
+; SSE2-NEXT:    movl %ecx, %edx
+; SSE2-NEXT:    shrl $16, %edx
+; SSE2-NEXT:    movb %dl, 2(%rdi)
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne .LBB10_7
+; SSE2-NEXT:    jmp .LBB10_8
+; SSE2-NEXT:  .LBB10_29: # %cond.store27
+; SSE2-NEXT:    movb %cl, 14(%rdi)
+; SSE2-NEXT:    testl $32768, %eax # imm = 0x8000
+; SSE2-NEXT:    je .LBB10_32
+; SSE2-NEXT:  .LBB10_31: # %cond.store29
+; SSE2-NEXT:    movb %ch, 15(%rdi)
+; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: truncstore_v16i32_v16i8:
 ; SSE4:       # %bb.0:
-; SSE4-NEXT:    pxor %xmm9, %xmm9
-; SSE4-NEXT:    pcmpeqd %xmm4, %xmm9
-; SSE4-NEXT:    pcmpeqd %xmm8, %xmm8
-; SSE4-NEXT:    pxor %xmm9, %xmm8
+; SSE4-NEXT:    pxor %xmm8, %xmm8
 ; SSE4-NEXT:    movdqa {{.*#+}} xmm9 = [255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0]
 ; SSE4-NEXT:    pand %xmm9, %xmm3
 ; SSE4-NEXT:    pand %xmm9, %xmm2
@@ -2873,496 +2782,407 @@ define void @truncstore_v16i32_v16i8(<16 x i32> %x, <16 x i8>* %p, <16 x i32> %m
 ; SSE4-NEXT:    pand %xmm9, %xmm0
 ; SSE4-NEXT:    packusdw %xmm1, %xmm0
 ; SSE4-NEXT:    packuswb %xmm2, %xmm0
-; SSE4-NEXT:    pextrb $0, %xmm8, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB10_2
-; SSE4-NEXT:  # %bb.1: # %cond.store
+; SSE4-NEXT:    pcmpeqd %xmm8, %xmm7
+; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
+; SSE4-NEXT:    pxor %xmm1, %xmm7
+; SSE4-NEXT:    pcmpeqd %xmm8, %xmm6
+; SSE4-NEXT:    pxor %xmm1, %xmm6
+; SSE4-NEXT:    packssdw %xmm7, %xmm6
+; SSE4-NEXT:    pcmpeqd %xmm8, %xmm5
+; SSE4-NEXT:    pxor %xmm1, %xmm5
+; SSE4-NEXT:    pcmpeqd %xmm8, %xmm4
+; SSE4-NEXT:    pxor %xmm1, %xmm4
+; SSE4-NEXT:    packssdw %xmm5, %xmm4
+; SSE4-NEXT:    packsswb %xmm6, %xmm4
+; SSE4-NEXT:    pmovmskb %xmm4, %eax
+; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    jne .LBB10_1
+; SSE4-NEXT:  # %bb.2: # %else
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    jne .LBB10_3
+; SSE4-NEXT:  .LBB10_4: # %else2
+; SSE4-NEXT:    testb $4, %al
+; SSE4-NEXT:    jne .LBB10_5
+; SSE4-NEXT:  .LBB10_6: # %else4
+; SSE4-NEXT:    testb $8, %al
+; SSE4-NEXT:    jne .LBB10_7
+; SSE4-NEXT:  .LBB10_8: # %else6
+; SSE4-NEXT:    testb $16, %al
+; SSE4-NEXT:    jne .LBB10_9
+; SSE4-NEXT:  .LBB10_10: # %else8
+; SSE4-NEXT:    testb $32, %al
+; SSE4-NEXT:    jne .LBB10_11
+; SSE4-NEXT:  .LBB10_12: # %else10
+; SSE4-NEXT:    testb $64, %al
+; SSE4-NEXT:    jne .LBB10_13
+; SSE4-NEXT:  .LBB10_14: # %else12
+; SSE4-NEXT:    testb $-128, %al
+; SSE4-NEXT:    jne .LBB10_15
+; SSE4-NEXT:  .LBB10_16: # %else14
+; SSE4-NEXT:    testl $256, %eax # imm = 0x100
+; SSE4-NEXT:    jne .LBB10_17
+; SSE4-NEXT:  .LBB10_18: # %else16
+; SSE4-NEXT:    testl $512, %eax # imm = 0x200
+; SSE4-NEXT:    jne .LBB10_19
+; SSE4-NEXT:  .LBB10_20: # %else18
+; SSE4-NEXT:    testl $1024, %eax # imm = 0x400
+; SSE4-NEXT:    jne .LBB10_21
+; SSE4-NEXT:  .LBB10_22: # %else20
+; SSE4-NEXT:    testl $2048, %eax # imm = 0x800
+; SSE4-NEXT:    jne .LBB10_23
+; SSE4-NEXT:  .LBB10_24: # %else22
+; SSE4-NEXT:    testl $4096, %eax # imm = 0x1000
+; SSE4-NEXT:    jne .LBB10_25
+; SSE4-NEXT:  .LBB10_26: # %else24
+; SSE4-NEXT:    testl $8192, %eax # imm = 0x2000
+; SSE4-NEXT:    jne .LBB10_27
+; SSE4-NEXT:  .LBB10_28: # %else26
+; SSE4-NEXT:    testl $16384, %eax # imm = 0x4000
+; SSE4-NEXT:    jne .LBB10_29
+; SSE4-NEXT:  .LBB10_30: # %else28
+; SSE4-NEXT:    testl $32768, %eax # imm = 0x8000
+; SSE4-NEXT:    jne .LBB10_31
+; SSE4-NEXT:  .LBB10_32: # %else30
+; SSE4-NEXT:    retq
+; SSE4-NEXT:  .LBB10_1: # %cond.store
 ; SSE4-NEXT:    pextrb $0, %xmm0, (%rdi)
-; SSE4-NEXT:  .LBB10_2: # %else
-; SSE4-NEXT:    pextrb $4, %xmm8, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $2, %al
 ; SSE4-NEXT:    je .LBB10_4
-; SSE4-NEXT:  # %bb.3: # %cond.store1
+; SSE4-NEXT:  .LBB10_3: # %cond.store1
 ; SSE4-NEXT:    pextrb $1, %xmm0, 1(%rdi)
-; SSE4-NEXT:  .LBB10_4: # %else2
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm4
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE4-NEXT:    pxor %xmm4, %xmm1
-; SSE4-NEXT:    pextrb $8, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $4, %al
 ; SSE4-NEXT:    je .LBB10_6
-; SSE4-NEXT:  # %bb.5: # %cond.store3
+; SSE4-NEXT:  .LBB10_5: # %cond.store3
 ; SSE4-NEXT:    pextrb $2, %xmm0, 2(%rdi)
-; SSE4-NEXT:  .LBB10_6: # %else4
-; SSE4-NEXT:    pextrb $12, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $8, %al
 ; SSE4-NEXT:    je .LBB10_8
-; SSE4-NEXT:  # %bb.7: # %cond.store5
+; SSE4-NEXT:  .LBB10_7: # %cond.store5
 ; SSE4-NEXT:    pextrb $3, %xmm0, 3(%rdi)
-; SSE4-NEXT:  .LBB10_8: # %else6
-; SSE4-NEXT:    pxor %xmm2, %xmm2
-; SSE4-NEXT:    pcmpeqd %xmm5, %xmm2
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE4-NEXT:    pxor %xmm2, %xmm1
-; SSE4-NEXT:    pextrb $0, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $16, %al
 ; SSE4-NEXT:    je .LBB10_10
-; SSE4-NEXT:  # %bb.9: # %cond.store7
+; SSE4-NEXT:  .LBB10_9: # %cond.store7
 ; SSE4-NEXT:    pextrb $4, %xmm0, 4(%rdi)
-; SSE4-NEXT:  .LBB10_10: # %else8
-; SSE4-NEXT:    pextrb $4, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $32, %al
 ; SSE4-NEXT:    je .LBB10_12
-; SSE4-NEXT:  # %bb.11: # %cond.store9
+; SSE4-NEXT:  .LBB10_11: # %cond.store9
 ; SSE4-NEXT:    pextrb $5, %xmm0, 5(%rdi)
-; SSE4-NEXT:  .LBB10_12: # %else10
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm5
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE4-NEXT:    pxor %xmm5, %xmm1
-; SSE4-NEXT:    pextrb $8, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $64, %al
 ; SSE4-NEXT:    je .LBB10_14
-; SSE4-NEXT:  # %bb.13: # %cond.store11
+; SSE4-NEXT:  .LBB10_13: # %cond.store11
 ; SSE4-NEXT:    pextrb $6, %xmm0, 6(%rdi)
-; SSE4-NEXT:  .LBB10_14: # %else12
-; SSE4-NEXT:    pextrb $12, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $-128, %al
 ; SSE4-NEXT:    je .LBB10_16
-; SSE4-NEXT:  # %bb.15: # %cond.store13
+; SSE4-NEXT:  .LBB10_15: # %cond.store13
 ; SSE4-NEXT:    pextrb $7, %xmm0, 7(%rdi)
-; SSE4-NEXT:  .LBB10_16: # %else14
-; SSE4-NEXT:    pxor %xmm2, %xmm2
-; SSE4-NEXT:    pcmpeqd %xmm6, %xmm2
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE4-NEXT:    pxor %xmm2, %xmm1
-; SSE4-NEXT:    pextrb $0, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $256, %eax # imm = 0x100
 ; SSE4-NEXT:    je .LBB10_18
-; SSE4-NEXT:  # %bb.17: # %cond.store15
+; SSE4-NEXT:  .LBB10_17: # %cond.store15
 ; SSE4-NEXT:    pextrb $8, %xmm0, 8(%rdi)
-; SSE4-NEXT:  .LBB10_18: # %else16
-; SSE4-NEXT:    pextrb $4, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $512, %eax # imm = 0x200
 ; SSE4-NEXT:    je .LBB10_20
-; SSE4-NEXT:  # %bb.19: # %cond.store17
+; SSE4-NEXT:  .LBB10_19: # %cond.store17
 ; SSE4-NEXT:    pextrb $9, %xmm0, 9(%rdi)
-; SSE4-NEXT:  .LBB10_20: # %else18
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm6
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE4-NEXT:    pxor %xmm6, %xmm1
-; SSE4-NEXT:    pextrb $8, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $1024, %eax # imm = 0x400
 ; SSE4-NEXT:    je .LBB10_22
-; SSE4-NEXT:  # %bb.21: # %cond.store19
+; SSE4-NEXT:  .LBB10_21: # %cond.store19
 ; SSE4-NEXT:    pextrb $10, %xmm0, 10(%rdi)
-; SSE4-NEXT:  .LBB10_22: # %else20
-; SSE4-NEXT:    pextrb $12, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $2048, %eax # imm = 0x800
 ; SSE4-NEXT:    je .LBB10_24
-; SSE4-NEXT:  # %bb.23: # %cond.store21
+; SSE4-NEXT:  .LBB10_23: # %cond.store21
 ; SSE4-NEXT:    pextrb $11, %xmm0, 11(%rdi)
-; SSE4-NEXT:  .LBB10_24: # %else22
-; SSE4-NEXT:    pxor %xmm2, %xmm2
-; SSE4-NEXT:    pcmpeqd %xmm7, %xmm2
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE4-NEXT:    pxor %xmm2, %xmm1
-; SSE4-NEXT:    pextrb $0, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $4096, %eax # imm = 0x1000
 ; SSE4-NEXT:    je .LBB10_26
-; SSE4-NEXT:  # %bb.25: # %cond.store23
+; SSE4-NEXT:  .LBB10_25: # %cond.store23
 ; SSE4-NEXT:    pextrb $12, %xmm0, 12(%rdi)
-; SSE4-NEXT:  .LBB10_26: # %else24
-; SSE4-NEXT:    pextrb $4, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $8192, %eax # imm = 0x2000
 ; SSE4-NEXT:    je .LBB10_28
-; SSE4-NEXT:  # %bb.27: # %cond.store25
+; SSE4-NEXT:  .LBB10_27: # %cond.store25
 ; SSE4-NEXT:    pextrb $13, %xmm0, 13(%rdi)
-; SSE4-NEXT:  .LBB10_28: # %else26
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm7
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE4-NEXT:    pxor %xmm7, %xmm1
-; SSE4-NEXT:    pextrb $8, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $16384, %eax # imm = 0x4000
 ; SSE4-NEXT:    je .LBB10_30
-; SSE4-NEXT:  # %bb.29: # %cond.store27
+; SSE4-NEXT:  .LBB10_29: # %cond.store27
 ; SSE4-NEXT:    pextrb $14, %xmm0, 14(%rdi)
-; SSE4-NEXT:  .LBB10_30: # %else28
-; SSE4-NEXT:    pextrb $12, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $32768, %eax # imm = 0x8000
 ; SSE4-NEXT:    je .LBB10_32
-; SSE4-NEXT:  # %bb.31: # %cond.store29
+; SSE4-NEXT:  .LBB10_31: # %cond.store29
 ; SSE4-NEXT:    pextrb $15, %xmm0, 15(%rdi)
-; SSE4-NEXT:  .LBB10_32: # %else30
 ; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: truncstore_v16i32_v16i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpxor %xmm8, %xmm8, %xmm8
-; AVX1-NEXT:    vpcmpeqd %xmm8, %xmm2, %xmm6
-; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
-; AVX1-NEXT:    vpxor %xmm5, %xmm6, %xmm6
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm6, %xmm6
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm6, %xmm6
-; AVX1-NEXT:    vmovaps {{.*#+}} ymm7 = [255,255,255,255,255,255,255,255]
-; AVX1-NEXT:    vandps %ymm7, %ymm1, %ymm1
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm4
-; AVX1-NEXT:    vpackusdw %xmm4, %xmm1, %xmm1
-; AVX1-NEXT:    vandps %ymm7, %ymm0, %ymm0
+; AVX1-NEXT:    vmovaps {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255]
+; AVX1-NEXT:    vandps %ymm4, %ymm1, %ymm1
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm5
+; AVX1-NEXT:    vpackusdw %xmm5, %xmm1, %xmm1
+; AVX1-NEXT:    vandps %ymm4, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
 ; AVX1-NEXT:    vpackusdw %xmm4, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpextrb $0, %xmm6, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je .LBB10_2
-; AVX1-NEXT:  # %bb.1: # %cond.store
-; AVX1-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX1-NEXT:  .LBB10_2: # %else
-; AVX1-NEXT:    vpcmpeqd %xmm8, %xmm2, %xmm1
+; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm1
+; AVX1-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm1, %xmm1
+; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
 ; AVX1-NEXT:    vpxor %xmm5, %xmm1, %xmm1
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm1, %xmm1
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $1, %xmm1, %eax
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpxor %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vpackssdw %xmm1, %xmm3, %xmm1
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm3
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpxor %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm2, %xmm2
+; AVX1-NEXT:    vpxor %xmm5, %xmm2, %xmm2
+; AVX1-NEXT:    vpackssdw %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpacksswb %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vpmovmskb %xmm1, %eax
 ; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    jne .LBB10_1
+; AVX1-NEXT:  # %bb.2: # %else
+; AVX1-NEXT:    testb $2, %al
+; AVX1-NEXT:    jne .LBB10_3
+; AVX1-NEXT:  .LBB10_4: # %else2
+; AVX1-NEXT:    testb $4, %al
+; AVX1-NEXT:    jne .LBB10_5
+; AVX1-NEXT:  .LBB10_6: # %else4
+; AVX1-NEXT:    testb $8, %al
+; AVX1-NEXT:    jne .LBB10_7
+; AVX1-NEXT:  .LBB10_8: # %else6
+; AVX1-NEXT:    testb $16, %al
+; AVX1-NEXT:    jne .LBB10_9
+; AVX1-NEXT:  .LBB10_10: # %else8
+; AVX1-NEXT:    testb $32, %al
+; AVX1-NEXT:    jne .LBB10_11
+; AVX1-NEXT:  .LBB10_12: # %else10
+; AVX1-NEXT:    testb $64, %al
+; AVX1-NEXT:    jne .LBB10_13
+; AVX1-NEXT:  .LBB10_14: # %else12
+; AVX1-NEXT:    testb $-128, %al
+; AVX1-NEXT:    jne .LBB10_15
+; AVX1-NEXT:  .LBB10_16: # %else14
+; AVX1-NEXT:    testl $256, %eax # imm = 0x100
+; AVX1-NEXT:    jne .LBB10_17
+; AVX1-NEXT:  .LBB10_18: # %else16
+; AVX1-NEXT:    testl $512, %eax # imm = 0x200
+; AVX1-NEXT:    jne .LBB10_19
+; AVX1-NEXT:  .LBB10_20: # %else18
+; AVX1-NEXT:    testl $1024, %eax # imm = 0x400
+; AVX1-NEXT:    jne .LBB10_21
+; AVX1-NEXT:  .LBB10_22: # %else20
+; AVX1-NEXT:    testl $2048, %eax # imm = 0x800
+; AVX1-NEXT:    jne .LBB10_23
+; AVX1-NEXT:  .LBB10_24: # %else22
+; AVX1-NEXT:    testl $4096, %eax # imm = 0x1000
+; AVX1-NEXT:    jne .LBB10_25
+; AVX1-NEXT:  .LBB10_26: # %else24
+; AVX1-NEXT:    testl $8192, %eax # imm = 0x2000
+; AVX1-NEXT:    jne .LBB10_27
+; AVX1-NEXT:  .LBB10_28: # %else26
+; AVX1-NEXT:    testl $16384, %eax # imm = 0x4000
+; AVX1-NEXT:    jne .LBB10_29
+; AVX1-NEXT:  .LBB10_30: # %else28
+; AVX1-NEXT:    testl $32768, %eax # imm = 0x8000
+; AVX1-NEXT:    jne .LBB10_31
+; AVX1-NEXT:  .LBB10_32: # %else30
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+; AVX1-NEXT:  .LBB10_1: # %cond.store
+; AVX1-NEXT:    vpextrb $0, %xmm0, (%rdi)
+; AVX1-NEXT:    testb $2, %al
 ; AVX1-NEXT:    je .LBB10_4
-; AVX1-NEXT:  # %bb.3: # %cond.store1
+; AVX1-NEXT:  .LBB10_3: # %cond.store1
 ; AVX1-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
-; AVX1-NEXT:  .LBB10_4: # %else2
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm2, %xmm5
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm5, %xmm5
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm5, %xmm5
-; AVX1-NEXT:    vpextrb $2, %xmm5, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $4, %al
 ; AVX1-NEXT:    je .LBB10_6
-; AVX1-NEXT:  # %bb.5: # %cond.store3
+; AVX1-NEXT:  .LBB10_5: # %cond.store3
 ; AVX1-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
-; AVX1-NEXT:  .LBB10_6: # %else4
-; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm4, %xmm1, %xmm1
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm1, %xmm1
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $3, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $8, %al
 ; AVX1-NEXT:    je .LBB10_8
-; AVX1-NEXT:  # %bb.7: # %cond.store5
+; AVX1-NEXT:  .LBB10_7: # %cond.store5
 ; AVX1-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
-; AVX1-NEXT:  .LBB10_8: # %else6
-; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm2
-; AVX1-NEXT:    vpackssdw %xmm2, %xmm0, %xmm4
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm4, %xmm4
-; AVX1-NEXT:    vpextrb $4, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $16, %al
 ; AVX1-NEXT:    je .LBB10_10
-; AVX1-NEXT:  # %bb.9: # %cond.store7
+; AVX1-NEXT:  .LBB10_9: # %cond.store7
 ; AVX1-NEXT:    vpextrb $4, %xmm0, 4(%rdi)
-; AVX1-NEXT:  .LBB10_10: # %else8
-; AVX1-NEXT:    vpackssdw %xmm2, %xmm0, %xmm2
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $5, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $32, %al
 ; AVX1-NEXT:    je .LBB10_12
-; AVX1-NEXT:  # %bb.11: # %cond.store9
+; AVX1-NEXT:  .LBB10_11: # %cond.store9
 ; AVX1-NEXT:    vpextrb $5, %xmm0, 5(%rdi)
-; AVX1-NEXT:  .LBB10_12: # %else10
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpackssdw %xmm1, %xmm0, %xmm2
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $6, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $64, %al
 ; AVX1-NEXT:    je .LBB10_14
-; AVX1-NEXT:  # %bb.13: # %cond.store11
+; AVX1-NEXT:  .LBB10_13: # %cond.store11
 ; AVX1-NEXT:    vpextrb $6, %xmm0, 6(%rdi)
-; AVX1-NEXT:  .LBB10_14: # %else12
-; AVX1-NEXT:    vpackssdw %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $7, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $-128, %al
 ; AVX1-NEXT:    je .LBB10_16
-; AVX1-NEXT:  # %bb.15: # %cond.store13
+; AVX1-NEXT:  .LBB10_15: # %cond.store13
 ; AVX1-NEXT:    vpextrb $7, %xmm0, 7(%rdi)
-; AVX1-NEXT:  .LBB10_16: # %else14
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm3, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpxor %xmm2, %xmm4, %xmm4
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm4, %xmm4
-; AVX1-NEXT:    vpacksswb %xmm4, %xmm0, %xmm4
-; AVX1-NEXT:    vpextrb $8, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $256, %eax # imm = 0x100
 ; AVX1-NEXT:    je .LBB10_18
-; AVX1-NEXT:  # %bb.17: # %cond.store15
+; AVX1-NEXT:  .LBB10_17: # %cond.store15
 ; AVX1-NEXT:    vpextrb $8, %xmm0, 8(%rdi)
-; AVX1-NEXT:  .LBB10_18: # %else16
-; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm3, %xmm1
-; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm1, %xmm1
-; AVX1-NEXT:    vpacksswb %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpextrb $9, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $512, %eax # imm = 0x200
 ; AVX1-NEXT:    je .LBB10_20
-; AVX1-NEXT:  # %bb.19: # %cond.store17
+; AVX1-NEXT:  .LBB10_19: # %cond.store17
 ; AVX1-NEXT:    vpextrb $9, %xmm0, 9(%rdi)
-; AVX1-NEXT:  .LBB10_20: # %else18
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm3, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpxor %xmm2, %xmm4, %xmm4
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm4, %xmm4
-; AVX1-NEXT:    vpacksswb %xmm4, %xmm0, %xmm4
-; AVX1-NEXT:    vpextrb $10, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $1024, %eax # imm = 0x400
 ; AVX1-NEXT:    je .LBB10_22
-; AVX1-NEXT:  # %bb.21: # %cond.store19
+; AVX1-NEXT:  .LBB10_21: # %cond.store19
 ; AVX1-NEXT:    vpextrb $10, %xmm0, 10(%rdi)
-; AVX1-NEXT:  .LBB10_22: # %else20
-; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm3, %xmm1
-; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm1, %xmm1
-; AVX1-NEXT:    vpacksswb %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpextrb $11, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $2048, %eax # imm = 0x800
 ; AVX1-NEXT:    je .LBB10_24
-; AVX1-NEXT:  # %bb.23: # %cond.store21
+; AVX1-NEXT:  .LBB10_23: # %cond.store21
 ; AVX1-NEXT:    vpextrb $11, %xmm0, 11(%rdi)
-; AVX1-NEXT:  .LBB10_24: # %else22
-; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm1
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpackssdw %xmm2, %xmm0, %xmm3
-; AVX1-NEXT:    vpacksswb %xmm3, %xmm0, %xmm3
-; AVX1-NEXT:    vpextrb $12, %xmm3, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $4096, %eax # imm = 0x1000
 ; AVX1-NEXT:    je .LBB10_26
-; AVX1-NEXT:  # %bb.25: # %cond.store23
+; AVX1-NEXT:  .LBB10_25: # %cond.store23
 ; AVX1-NEXT:    vpextrb $12, %xmm0, 12(%rdi)
-; AVX1-NEXT:  .LBB10_26: # %else24
-; AVX1-NEXT:    vpackssdw %xmm2, %xmm0, %xmm2
-; AVX1-NEXT:    vpacksswb %xmm2, %xmm0, %xmm2
-; AVX1-NEXT:    vpextrb $13, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $8192, %eax # imm = 0x2000
 ; AVX1-NEXT:    je .LBB10_28
-; AVX1-NEXT:  # %bb.27: # %cond.store25
+; AVX1-NEXT:  .LBB10_27: # %cond.store25
 ; AVX1-NEXT:    vpextrb $13, %xmm0, 13(%rdi)
-; AVX1-NEXT:  .LBB10_28: # %else26
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpackssdw %xmm1, %xmm0, %xmm2
-; AVX1-NEXT:    vpacksswb %xmm2, %xmm0, %xmm2
-; AVX1-NEXT:    vpextrb $14, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $16384, %eax # imm = 0x4000
 ; AVX1-NEXT:    je .LBB10_30
-; AVX1-NEXT:  # %bb.29: # %cond.store27
+; AVX1-NEXT:  .LBB10_29: # %cond.store27
 ; AVX1-NEXT:    vpextrb $14, %xmm0, 14(%rdi)
-; AVX1-NEXT:  .LBB10_30: # %else28
-; AVX1-NEXT:    vpackssdw %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpacksswb %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpextrb $15, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $32768, %eax # imm = 0x8000
 ; AVX1-NEXT:    je .LBB10_32
-; AVX1-NEXT:  # %bb.31: # %cond.store29
+; AVX1-NEXT:  .LBB10_31: # %cond.store29
 ; AVX1-NEXT:    vpextrb $15, %xmm0, 15(%rdi)
-; AVX1-NEXT:  .LBB10_32: # %else30
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: truncstore_v16i32_v16i8:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpxor %xmm8, %xmm8, %xmm8
-; AVX2-NEXT:    vpcmpeqd %xmm8, %xmm2, %xmm6
-; AVX2-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
-; AVX2-NEXT:    vpxor %xmm5, %xmm6, %xmm6
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm6, %xmm6
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm6, %xmm6
-; AVX2-NEXT:    vmovdqa {{.*#+}} ymm7 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; AVX2-NEXT:    vpshufb %ymm7, %ymm1, %ymm1
+; AVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; AVX2-NEXT:    vmovdqa {{.*#+}} ymm5 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
+; AVX2-NEXT:    vpshufb %ymm5, %ymm1, %ymm1
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3]
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
-; AVX2-NEXT:    vpand %xmm4, %xmm1, %xmm1
-; AVX2-NEXT:    vpshufb %ymm7, %ymm0, %ymm0
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm6 = [255,255,255,255,255,255,255,255]
+; AVX2-NEXT:    vpand %xmm6, %xmm1, %xmm1
+; AVX2-NEXT:    vpshufb %ymm5, %ymm0, %ymm0
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-NEXT:    vpand %xmm4, %xmm0, %xmm0
+; AVX2-NEXT:    vpand %xmm6, %xmm0, %xmm0
 ; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpextrb $0, %xmm6, %eax
+; AVX2-NEXT:    vpcmpeqd %ymm4, %ymm3, %ymm1
+; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm3, %ymm3
+; AVX2-NEXT:    vpxor %ymm3, %ymm1, %ymm1
+; AVX2-NEXT:    vpcmpeqd %ymm4, %ymm2, %ymm2
+; AVX2-NEXT:    vpxor %ymm3, %ymm2, %ymm2
+; AVX2-NEXT:    vpackssdw %ymm1, %ymm2, %ymm1
+; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
+; AVX2-NEXT:    vpacksswb %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpmovmskb %xmm1, %eax
 ; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je .LBB10_2
-; AVX2-NEXT:  # %bb.1: # %cond.store
+; AVX2-NEXT:    jne .LBB10_1
+; AVX2-NEXT:  # %bb.2: # %else
+; AVX2-NEXT:    testb $2, %al
+; AVX2-NEXT:    jne .LBB10_3
+; AVX2-NEXT:  .LBB10_4: # %else2
+; AVX2-NEXT:    testb $4, %al
+; AVX2-NEXT:    jne .LBB10_5
+; AVX2-NEXT:  .LBB10_6: # %else4
+; AVX2-NEXT:    testb $8, %al
+; AVX2-NEXT:    jne .LBB10_7
+; AVX2-NEXT:  .LBB10_8: # %else6
+; AVX2-NEXT:    testb $16, %al
+; AVX2-NEXT:    jne .LBB10_9
+; AVX2-NEXT:  .LBB10_10: # %else8
+; AVX2-NEXT:    testb $32, %al
+; AVX2-NEXT:    jne .LBB10_11
+; AVX2-NEXT:  .LBB10_12: # %else10
+; AVX2-NEXT:    testb $64, %al
+; AVX2-NEXT:    jne .LBB10_13
+; AVX2-NEXT:  .LBB10_14: # %else12
+; AVX2-NEXT:    testb $-128, %al
+; AVX2-NEXT:    jne .LBB10_15
+; AVX2-NEXT:  .LBB10_16: # %else14
+; AVX2-NEXT:    testl $256, %eax # imm = 0x100
+; AVX2-NEXT:    jne .LBB10_17
+; AVX2-NEXT:  .LBB10_18: # %else16
+; AVX2-NEXT:    testl $512, %eax # imm = 0x200
+; AVX2-NEXT:    jne .LBB10_19
+; AVX2-NEXT:  .LBB10_20: # %else18
+; AVX2-NEXT:    testl $1024, %eax # imm = 0x400
+; AVX2-NEXT:    jne .LBB10_21
+; AVX2-NEXT:  .LBB10_22: # %else20
+; AVX2-NEXT:    testl $2048, %eax # imm = 0x800
+; AVX2-NEXT:    jne .LBB10_23
+; AVX2-NEXT:  .LBB10_24: # %else22
+; AVX2-NEXT:    testl $4096, %eax # imm = 0x1000
+; AVX2-NEXT:    jne .LBB10_25
+; AVX2-NEXT:  .LBB10_26: # %else24
+; AVX2-NEXT:    testl $8192, %eax # imm = 0x2000
+; AVX2-NEXT:    jne .LBB10_27
+; AVX2-NEXT:  .LBB10_28: # %else26
+; AVX2-NEXT:    testl $16384, %eax # imm = 0x4000
+; AVX2-NEXT:    jne .LBB10_29
+; AVX2-NEXT:  .LBB10_30: # %else28
+; AVX2-NEXT:    testl $32768, %eax # imm = 0x8000
+; AVX2-NEXT:    jne .LBB10_31
+; AVX2-NEXT:  .LBB10_32: # %else30
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+; AVX2-NEXT:  .LBB10_1: # %cond.store
 ; AVX2-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX2-NEXT:  .LBB10_2: # %else
-; AVX2-NEXT:    vpcmpeqd %xmm8, %xmm2, %xmm1
-; AVX2-NEXT:    vpxor %xmm5, %xmm1, %xmm1
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm1, %xmm1
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm1, %xmm1
-; AVX2-NEXT:    vpextrb $1, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $2, %al
 ; AVX2-NEXT:    je .LBB10_4
-; AVX2-NEXT:  # %bb.3: # %cond.store1
+; AVX2-NEXT:  .LBB10_3: # %cond.store1
 ; AVX2-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
-; AVX2-NEXT:  .LBB10_4: # %else2
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm2, %xmm5
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm5, %xmm5
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm5, %xmm5
-; AVX2-NEXT:    vpextrb $2, %xmm5, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $4, %al
 ; AVX2-NEXT:    je .LBB10_6
-; AVX2-NEXT:  # %bb.5: # %cond.store3
+; AVX2-NEXT:  .LBB10_5: # %cond.store3
 ; AVX2-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
-; AVX2-NEXT:  .LBB10_6: # %else4
-; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm2, %xmm1
-; AVX2-NEXT:    vpxor %xmm4, %xmm1, %xmm1
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm1, %xmm1
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm1, %xmm1
-; AVX2-NEXT:    vpextrb $3, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $8, %al
 ; AVX2-NEXT:    je .LBB10_8
-; AVX2-NEXT:  # %bb.7: # %cond.store5
+; AVX2-NEXT:  .LBB10_7: # %cond.store5
 ; AVX2-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
-; AVX2-NEXT:  .LBB10_8: # %else6
-; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm1
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpxor %xmm4, %xmm2, %xmm2
-; AVX2-NEXT:    vpackssdw %xmm2, %xmm0, %xmm4
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm4, %xmm4
-; AVX2-NEXT:    vpextrb $4, %xmm4, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $16, %al
 ; AVX2-NEXT:    je .LBB10_10
-; AVX2-NEXT:  # %bb.9: # %cond.store7
+; AVX2-NEXT:  .LBB10_9: # %cond.store7
 ; AVX2-NEXT:    vpextrb $4, %xmm0, 4(%rdi)
-; AVX2-NEXT:  .LBB10_10: # %else8
-; AVX2-NEXT:    vpackssdw %xmm2, %xmm0, %xmm2
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrb $5, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $32, %al
 ; AVX2-NEXT:    je .LBB10_12
-; AVX2-NEXT:  # %bb.11: # %cond.store9
+; AVX2-NEXT:  .LBB10_11: # %cond.store9
 ; AVX2-NEXT:    vpextrb $5, %xmm0, 5(%rdi)
-; AVX2-NEXT:  .LBB10_12: # %else10
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpackssdw %xmm1, %xmm0, %xmm2
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrb $6, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $64, %al
 ; AVX2-NEXT:    je .LBB10_14
-; AVX2-NEXT:  # %bb.13: # %cond.store11
+; AVX2-NEXT:  .LBB10_13: # %cond.store11
 ; AVX2-NEXT:    vpextrb $6, %xmm0, 6(%rdi)
-; AVX2-NEXT:  .LBB10_14: # %else12
-; AVX2-NEXT:    vpackssdw %xmm1, %xmm0, %xmm1
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm1, %xmm1
-; AVX2-NEXT:    vpextrb $7, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $-128, %al
 ; AVX2-NEXT:    je .LBB10_16
-; AVX2-NEXT:  # %bb.15: # %cond.store13
+; AVX2-NEXT:  .LBB10_15: # %cond.store13
 ; AVX2-NEXT:    vpextrb $7, %xmm0, 7(%rdi)
-; AVX2-NEXT:  .LBB10_16: # %else14
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm3, %xmm4
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpxor %xmm2, %xmm4, %xmm4
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm4, %xmm4
-; AVX2-NEXT:    vpacksswb %xmm4, %xmm0, %xmm4
-; AVX2-NEXT:    vpextrb $8, %xmm4, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $256, %eax # imm = 0x100
 ; AVX2-NEXT:    je .LBB10_18
-; AVX2-NEXT:  # %bb.17: # %cond.store15
+; AVX2-NEXT:  .LBB10_17: # %cond.store15
 ; AVX2-NEXT:    vpextrb $8, %xmm0, 8(%rdi)
-; AVX2-NEXT:  .LBB10_18: # %else16
-; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm3, %xmm1
-; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm1, %xmm1
-; AVX2-NEXT:    vpacksswb %xmm1, %xmm0, %xmm1
-; AVX2-NEXT:    vpextrb $9, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $512, %eax # imm = 0x200
 ; AVX2-NEXT:    je .LBB10_20
-; AVX2-NEXT:  # %bb.19: # %cond.store17
+; AVX2-NEXT:  .LBB10_19: # %cond.store17
 ; AVX2-NEXT:    vpextrb $9, %xmm0, 9(%rdi)
-; AVX2-NEXT:  .LBB10_20: # %else18
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm3, %xmm4
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpxor %xmm2, %xmm4, %xmm4
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm4, %xmm4
-; AVX2-NEXT:    vpacksswb %xmm4, %xmm0, %xmm4
-; AVX2-NEXT:    vpextrb $10, %xmm4, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $1024, %eax # imm = 0x400
 ; AVX2-NEXT:    je .LBB10_22
-; AVX2-NEXT:  # %bb.21: # %cond.store19
+; AVX2-NEXT:  .LBB10_21: # %cond.store19
 ; AVX2-NEXT:    vpextrb $10, %xmm0, 10(%rdi)
-; AVX2-NEXT:  .LBB10_22: # %else20
-; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm3, %xmm1
-; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm1, %xmm1
-; AVX2-NEXT:    vpacksswb %xmm1, %xmm0, %xmm1
-; AVX2-NEXT:    vpextrb $11, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $2048, %eax # imm = 0x800
 ; AVX2-NEXT:    je .LBB10_24
-; AVX2-NEXT:  # %bb.23: # %cond.store21
+; AVX2-NEXT:  .LBB10_23: # %cond.store21
 ; AVX2-NEXT:    vpextrb $11, %xmm0, 11(%rdi)
-; AVX2-NEXT:  .LBB10_24: # %else22
-; AVX2-NEXT:    vextracti128 $1, %ymm3, %xmm1
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX2-NEXT:    vpackssdw %xmm2, %xmm0, %xmm3
-; AVX2-NEXT:    vpacksswb %xmm3, %xmm0, %xmm3
-; AVX2-NEXT:    vpextrb $12, %xmm3, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $4096, %eax # imm = 0x1000
 ; AVX2-NEXT:    je .LBB10_26
-; AVX2-NEXT:  # %bb.25: # %cond.store23
+; AVX2-NEXT:  .LBB10_25: # %cond.store23
 ; AVX2-NEXT:    vpextrb $12, %xmm0, 12(%rdi)
-; AVX2-NEXT:  .LBB10_26: # %else24
-; AVX2-NEXT:    vpackssdw %xmm2, %xmm0, %xmm2
-; AVX2-NEXT:    vpacksswb %xmm2, %xmm0, %xmm2
-; AVX2-NEXT:    vpextrb $13, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $8192, %eax # imm = 0x2000
 ; AVX2-NEXT:    je .LBB10_28
-; AVX2-NEXT:  # %bb.27: # %cond.store25
+; AVX2-NEXT:  .LBB10_27: # %cond.store25
 ; AVX2-NEXT:    vpextrb $13, %xmm0, 13(%rdi)
-; AVX2-NEXT:  .LBB10_28: # %else26
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpackssdw %xmm1, %xmm0, %xmm2
-; AVX2-NEXT:    vpacksswb %xmm2, %xmm0, %xmm2
-; AVX2-NEXT:    vpextrb $14, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $16384, %eax # imm = 0x4000
 ; AVX2-NEXT:    je .LBB10_30
-; AVX2-NEXT:  # %bb.29: # %cond.store27
+; AVX2-NEXT:  .LBB10_29: # %cond.store27
 ; AVX2-NEXT:    vpextrb $14, %xmm0, 14(%rdi)
-; AVX2-NEXT:  .LBB10_30: # %else28
-; AVX2-NEXT:    vpackssdw %xmm1, %xmm0, %xmm1
-; AVX2-NEXT:    vpacksswb %xmm1, %xmm0, %xmm1
-; AVX2-NEXT:    vpextrb $15, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $32768, %eax # imm = 0x8000
 ; AVX2-NEXT:    je .LBB10_32
-; AVX2-NEXT:  # %bb.31: # %cond.store29
+; AVX2-NEXT:  .LBB10_31: # %cond.store29
 ; AVX2-NEXT:    vpextrb $15, %xmm0, 15(%rdi)
-; AVX2-NEXT:  .LBB10_32: # %else30
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
@@ -3372,115 +3192,117 @@ define void @truncstore_v16i32_v16i8(<16 x i32> %x, <16 x i8>* %p, <16 x i32> %m
 ; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
 ; AVX512F-NEXT:    kmovw %k0, %eax
 ; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB10_2
-; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    jne .LBB10_1
+; AVX512F-NEXT:  # %bb.2: # %else
+; AVX512F-NEXT:    testb $2, %al
+; AVX512F-NEXT:    jne .LBB10_3
+; AVX512F-NEXT:  .LBB10_4: # %else2
+; AVX512F-NEXT:    testb $4, %al
+; AVX512F-NEXT:    jne .LBB10_5
+; AVX512F-NEXT:  .LBB10_6: # %else4
+; AVX512F-NEXT:    testb $8, %al
+; AVX512F-NEXT:    jne .LBB10_7
+; AVX512F-NEXT:  .LBB10_8: # %else6
+; AVX512F-NEXT:    testb $16, %al
+; AVX512F-NEXT:    jne .LBB10_9
+; AVX512F-NEXT:  .LBB10_10: # %else8
+; AVX512F-NEXT:    testb $32, %al
+; AVX512F-NEXT:    jne .LBB10_11
+; AVX512F-NEXT:  .LBB10_12: # %else10
+; AVX512F-NEXT:    testb $64, %al
+; AVX512F-NEXT:    jne .LBB10_13
+; AVX512F-NEXT:  .LBB10_14: # %else12
+; AVX512F-NEXT:    testb $-128, %al
+; AVX512F-NEXT:    jne .LBB10_15
+; AVX512F-NEXT:  .LBB10_16: # %else14
+; AVX512F-NEXT:    testl $256, %eax # imm = 0x100
+; AVX512F-NEXT:    jne .LBB10_17
+; AVX512F-NEXT:  .LBB10_18: # %else16
+; AVX512F-NEXT:    testl $512, %eax # imm = 0x200
+; AVX512F-NEXT:    jne .LBB10_19
+; AVX512F-NEXT:  .LBB10_20: # %else18
+; AVX512F-NEXT:    testl $1024, %eax # imm = 0x400
+; AVX512F-NEXT:    jne .LBB10_21
+; AVX512F-NEXT:  .LBB10_22: # %else20
+; AVX512F-NEXT:    testl $2048, %eax # imm = 0x800
+; AVX512F-NEXT:    jne .LBB10_23
+; AVX512F-NEXT:  .LBB10_24: # %else22
+; AVX512F-NEXT:    testl $4096, %eax # imm = 0x1000
+; AVX512F-NEXT:    jne .LBB10_25
+; AVX512F-NEXT:  .LBB10_26: # %else24
+; AVX512F-NEXT:    testl $8192, %eax # imm = 0x2000
+; AVX512F-NEXT:    jne .LBB10_27
+; AVX512F-NEXT:  .LBB10_28: # %else26
+; AVX512F-NEXT:    testl $16384, %eax # imm = 0x4000
+; AVX512F-NEXT:    jne .LBB10_29
+; AVX512F-NEXT:  .LBB10_30: # %else28
+; AVX512F-NEXT:    testl $32768, %eax # imm = 0x8000
+; AVX512F-NEXT:    jne .LBB10_31
+; AVX512F-NEXT:  .LBB10_32: # %else30
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+; AVX512F-NEXT:  .LBB10_1: # %cond.store
 ; AVX512F-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX512F-NEXT:  .LBB10_2: # %else
-; AVX512F-NEXT:    kshiftrw $1, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $2, %al
 ; AVX512F-NEXT:    je .LBB10_4
-; AVX512F-NEXT:  # %bb.3: # %cond.store1
+; AVX512F-NEXT:  .LBB10_3: # %cond.store1
 ; AVX512F-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
-; AVX512F-NEXT:  .LBB10_4: # %else2
-; AVX512F-NEXT:    kshiftrw $2, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $4, %al
 ; AVX512F-NEXT:    je .LBB10_6
-; AVX512F-NEXT:  # %bb.5: # %cond.store3
+; AVX512F-NEXT:  .LBB10_5: # %cond.store3
 ; AVX512F-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
-; AVX512F-NEXT:  .LBB10_6: # %else4
-; AVX512F-NEXT:    kshiftrw $3, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $8, %al
 ; AVX512F-NEXT:    je .LBB10_8
-; AVX512F-NEXT:  # %bb.7: # %cond.store5
+; AVX512F-NEXT:  .LBB10_7: # %cond.store5
 ; AVX512F-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
-; AVX512F-NEXT:  .LBB10_8: # %else6
-; AVX512F-NEXT:    kshiftrw $4, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $16, %al
 ; AVX512F-NEXT:    je .LBB10_10
-; AVX512F-NEXT:  # %bb.9: # %cond.store7
+; AVX512F-NEXT:  .LBB10_9: # %cond.store7
 ; AVX512F-NEXT:    vpextrb $4, %xmm0, 4(%rdi)
-; AVX512F-NEXT:  .LBB10_10: # %else8
-; AVX512F-NEXT:    kshiftrw $5, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $32, %al
 ; AVX512F-NEXT:    je .LBB10_12
-; AVX512F-NEXT:  # %bb.11: # %cond.store9
+; AVX512F-NEXT:  .LBB10_11: # %cond.store9
 ; AVX512F-NEXT:    vpextrb $5, %xmm0, 5(%rdi)
-; AVX512F-NEXT:  .LBB10_12: # %else10
-; AVX512F-NEXT:    kshiftrw $6, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $64, %al
 ; AVX512F-NEXT:    je .LBB10_14
-; AVX512F-NEXT:  # %bb.13: # %cond.store11
+; AVX512F-NEXT:  .LBB10_13: # %cond.store11
 ; AVX512F-NEXT:    vpextrb $6, %xmm0, 6(%rdi)
-; AVX512F-NEXT:  .LBB10_14: # %else12
-; AVX512F-NEXT:    kshiftrw $7, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $-128, %al
 ; AVX512F-NEXT:    je .LBB10_16
-; AVX512F-NEXT:  # %bb.15: # %cond.store13
+; AVX512F-NEXT:  .LBB10_15: # %cond.store13
 ; AVX512F-NEXT:    vpextrb $7, %xmm0, 7(%rdi)
-; AVX512F-NEXT:  .LBB10_16: # %else14
-; AVX512F-NEXT:    kshiftrw $8, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $256, %eax # imm = 0x100
 ; AVX512F-NEXT:    je .LBB10_18
-; AVX512F-NEXT:  # %bb.17: # %cond.store15
+; AVX512F-NEXT:  .LBB10_17: # %cond.store15
 ; AVX512F-NEXT:    vpextrb $8, %xmm0, 8(%rdi)
-; AVX512F-NEXT:  .LBB10_18: # %else16
-; AVX512F-NEXT:    kshiftrw $9, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $512, %eax # imm = 0x200
 ; AVX512F-NEXT:    je .LBB10_20
-; AVX512F-NEXT:  # %bb.19: # %cond.store17
+; AVX512F-NEXT:  .LBB10_19: # %cond.store17
 ; AVX512F-NEXT:    vpextrb $9, %xmm0, 9(%rdi)
-; AVX512F-NEXT:  .LBB10_20: # %else18
-; AVX512F-NEXT:    kshiftrw $10, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $1024, %eax # imm = 0x400
 ; AVX512F-NEXT:    je .LBB10_22
-; AVX512F-NEXT:  # %bb.21: # %cond.store19
+; AVX512F-NEXT:  .LBB10_21: # %cond.store19
 ; AVX512F-NEXT:    vpextrb $10, %xmm0, 10(%rdi)
-; AVX512F-NEXT:  .LBB10_22: # %else20
-; AVX512F-NEXT:    kshiftrw $11, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $2048, %eax # imm = 0x800
 ; AVX512F-NEXT:    je .LBB10_24
-; AVX512F-NEXT:  # %bb.23: # %cond.store21
+; AVX512F-NEXT:  .LBB10_23: # %cond.store21
 ; AVX512F-NEXT:    vpextrb $11, %xmm0, 11(%rdi)
-; AVX512F-NEXT:  .LBB10_24: # %else22
-; AVX512F-NEXT:    kshiftrw $12, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $4096, %eax # imm = 0x1000
 ; AVX512F-NEXT:    je .LBB10_26
-; AVX512F-NEXT:  # %bb.25: # %cond.store23
+; AVX512F-NEXT:  .LBB10_25: # %cond.store23
 ; AVX512F-NEXT:    vpextrb $12, %xmm0, 12(%rdi)
-; AVX512F-NEXT:  .LBB10_26: # %else24
-; AVX512F-NEXT:    kshiftrw $13, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $8192, %eax # imm = 0x2000
 ; AVX512F-NEXT:    je .LBB10_28
-; AVX512F-NEXT:  # %bb.27: # %cond.store25
+; AVX512F-NEXT:  .LBB10_27: # %cond.store25
 ; AVX512F-NEXT:    vpextrb $13, %xmm0, 13(%rdi)
-; AVX512F-NEXT:  .LBB10_28: # %else26
-; AVX512F-NEXT:    kshiftrw $14, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $16384, %eax # imm = 0x4000
 ; AVX512F-NEXT:    je .LBB10_30
-; AVX512F-NEXT:  # %bb.29: # %cond.store27
+; AVX512F-NEXT:  .LBB10_29: # %cond.store27
 ; AVX512F-NEXT:    vpextrb $14, %xmm0, 14(%rdi)
-; AVX512F-NEXT:  .LBB10_30: # %else28
-; AVX512F-NEXT:    kshiftrw $15, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $32768, %eax # imm = 0x8000
 ; AVX512F-NEXT:    je .LBB10_32
-; AVX512F-NEXT:  # %bb.31: # %cond.store29
+; AVX512F-NEXT:  .LBB10_31: # %cond.store29
 ; AVX512F-NEXT:    vpextrb $15, %xmm0, 15(%rdi)
-; AVX512F-NEXT:  .LBB10_32: # %else30
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
@@ -3506,313 +3328,294 @@ define void @truncstore_v16i32_v16i8(<16 x i32> %x, <16 x i8>* %p, <16 x i32> %m
 define void @truncstore_v8i32_v8i16(<8 x i32> %x, <8 x i16>* %p, <8 x i32> %mask) {
 ; SSE2-LABEL: truncstore_v8i32_v8i16:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pxor %xmm5, %xmm5
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm5
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm4
-; SSE2-NEXT:    pxor %xmm5, %xmm4
-; SSE2-NEXT:    movdqa %xmm4, %xmm5
-; SSE2-NEXT:    packssdw %xmm0, %xmm5
+; SSE2-NEXT:    pxor %xmm4, %xmm4
 ; SSE2-NEXT:    pslld $16, %xmm1
 ; SSE2-NEXT:    psrad $16, %xmm1
 ; SSE2-NEXT:    pslld $16, %xmm0
 ; SSE2-NEXT:    psrad $16, %xmm0
 ; SSE2-NEXT:    packssdw %xmm1, %xmm0
-; SSE2-NEXT:    movd %xmm5, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB11_2
-; SSE2-NEXT:  # %bb.1: # %cond.store
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    movw %ax, (%rdi)
-; SSE2-NEXT:  .LBB11_2: # %else
-; SSE2-NEXT:    psrlq $16, %xmm4
-; SSE2-NEXT:    movd %xmm4, %eax
-; SSE2-NEXT:    shrl $16, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB11_4
-; SSE2-NEXT:  # %bb.3: # %cond.store1
-; SSE2-NEXT:    pextrw $1, %xmm0, %eax
-; SSE2-NEXT:    movw %ax, 2(%rdi)
-; SSE2-NEXT:  .LBB11_4: # %else2
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm2
+; SSE2-NEXT:    pcmpeqd %xmm4, %xmm3
 ; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pxor %xmm2, %xmm1
-; SSE2-NEXT:    pextrw $4, %xmm1, %eax
+; SSE2-NEXT:    pxor %xmm1, %xmm3
+; SSE2-NEXT:    pcmpeqd %xmm4, %xmm2
+; SSE2-NEXT:    pxor %xmm1, %xmm2
+; SSE2-NEXT:    packssdw %xmm3, %xmm2
+; SSE2-NEXT:    packsswb %xmm0, %xmm2
+; SSE2-NEXT:    pmovmskb %xmm2, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB11_6
-; SSE2-NEXT:  # %bb.5: # %cond.store3
-; SSE2-NEXT:    pextrw $2, %xmm0, %eax
-; SSE2-NEXT:    movw %ax, 4(%rdi)
+; SSE2-NEXT:    jne .LBB11_1
+; SSE2-NEXT:  # %bb.2: # %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne .LBB11_3
+; SSE2-NEXT:  .LBB11_4: # %else2
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne .LBB11_5
 ; SSE2-NEXT:  .LBB11_6: # %else4
-; SSE2-NEXT:    pextrw $6, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB11_8
-; SSE2-NEXT:  # %bb.7: # %cond.store5
-; SSE2-NEXT:    pextrw $3, %xmm0, %eax
-; SSE2-NEXT:    movw %ax, 6(%rdi)
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne .LBB11_7
 ; SSE2-NEXT:  .LBB11_8: # %else6
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pxor %xmm2, %xmm1
-; SSE2-NEXT:    pextrw $0, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB11_10
-; SSE2-NEXT:  # %bb.9: # %cond.store7
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    movw %ax, 8(%rdi)
+; SSE2-NEXT:    testb $16, %al
+; SSE2-NEXT:    jne .LBB11_9
 ; SSE2-NEXT:  .LBB11_10: # %else8
-; SSE2-NEXT:    pextrw $2, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB11_12
-; SSE2-NEXT:  # %bb.11: # %cond.store9
-; SSE2-NEXT:    pextrw $5, %xmm0, %eax
-; SSE2-NEXT:    movw %ax, 10(%rdi)
+; SSE2-NEXT:    testb $32, %al
+; SSE2-NEXT:    jne .LBB11_11
 ; SSE2-NEXT:  .LBB11_12: # %else10
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm3
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pxor %xmm3, %xmm1
-; SSE2-NEXT:    pextrw $4, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB11_14
-; SSE2-NEXT:  # %bb.13: # %cond.store11
-; SSE2-NEXT:    pextrw $6, %xmm0, %eax
-; SSE2-NEXT:    movw %ax, 12(%rdi)
+; SSE2-NEXT:    testb $64, %al
+; SSE2-NEXT:    jne .LBB11_13
 ; SSE2-NEXT:  .LBB11_14: # %else12
-; SSE2-NEXT:    pextrw $6, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB11_16
-; SSE2-NEXT:  # %bb.15: # %cond.store13
-; SSE2-NEXT:    pextrw $7, %xmm0, %eax
-; SSE2-NEXT:    movw %ax, 14(%rdi)
+; SSE2-NEXT:    testb $-128, %al
+; SSE2-NEXT:    jne .LBB11_15
 ; SSE2-NEXT:  .LBB11_16: # %else14
 ; SSE2-NEXT:    retq
-;
-; SSE4-LABEL: truncstore_v8i32_v8i16:
-; SSE4:       # %bb.0:
-; SSE4-NEXT:    pxor %xmm5, %xmm5
-; SSE4-NEXT:    pcmpeqd %xmm2, %xmm5
-; SSE4-NEXT:    pcmpeqd %xmm4, %xmm4
-; SSE4-NEXT:    pxor %xmm5, %xmm4
-; SSE4-NEXT:    movdqa {{.*#+}} xmm5 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
-; SSE4-NEXT:    pshufb %xmm5, %xmm1
-; SSE4-NEXT:    pshufb %xmm5, %xmm0
-; SSE4-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSE4-NEXT:    pextrb $0, %xmm4, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB11_2
-; SSE4-NEXT:  # %bb.1: # %cond.store
-; SSE4-NEXT:    pextrw $0, %xmm0, (%rdi)
-; SSE4-NEXT:  .LBB11_2: # %else
-; SSE4-NEXT:    pextrb $4, %xmm4, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE2-NEXT:  .LBB11_1: # %cond.store
+; SSE2-NEXT:    movd %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, (%rdi)
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    je .LBB11_4
+; SSE2-NEXT:  .LBB11_3: # %cond.store1
+; SSE2-NEXT:    pextrw $1, %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, 2(%rdi)
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    je .LBB11_6
+; SSE2-NEXT:  .LBB11_5: # %cond.store3
+; SSE2-NEXT:    pextrw $2, %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, 4(%rdi)
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    je .LBB11_8
+; SSE2-NEXT:  .LBB11_7: # %cond.store5
+; SSE2-NEXT:    pextrw $3, %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, 6(%rdi)
+; SSE2-NEXT:    testb $16, %al
+; SSE2-NEXT:    je .LBB11_10
+; SSE2-NEXT:  .LBB11_9: # %cond.store7
+; SSE2-NEXT:    pextrw $4, %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, 8(%rdi)
+; SSE2-NEXT:    testb $32, %al
+; SSE2-NEXT:    je .LBB11_12
+; SSE2-NEXT:  .LBB11_11: # %cond.store9
+; SSE2-NEXT:    pextrw $5, %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, 10(%rdi)
+; SSE2-NEXT:    testb $64, %al
+; SSE2-NEXT:    je .LBB11_14
+; SSE2-NEXT:  .LBB11_13: # %cond.store11
+; SSE2-NEXT:    pextrw $6, %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, 12(%rdi)
+; SSE2-NEXT:    testb $-128, %al
+; SSE2-NEXT:    je .LBB11_16
+; SSE2-NEXT:  .LBB11_15: # %cond.store13
+; SSE2-NEXT:    pextrw $7, %xmm0, %eax
+; SSE2-NEXT:    movw %ax, 14(%rdi)
+; SSE2-NEXT:    retq
+;
+; SSE4-LABEL: truncstore_v8i32_v8i16:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pxor %xmm4, %xmm4
+; SSE4-NEXT:    movdqa {{.*#+}} xmm5 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
+; SSE4-NEXT:    pshufb %xmm5, %xmm1
+; SSE4-NEXT:    pshufb %xmm5, %xmm0
+; SSE4-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE4-NEXT:    pcmpeqd %xmm4, %xmm3
+; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
+; SSE4-NEXT:    pxor %xmm1, %xmm3
+; SSE4-NEXT:    pcmpeqd %xmm4, %xmm2
+; SSE4-NEXT:    pxor %xmm1, %xmm2
+; SSE4-NEXT:    packssdw %xmm3, %xmm2
+; SSE4-NEXT:    packsswb %xmm0, %xmm2
+; SSE4-NEXT:    pmovmskb %xmm2, %eax
+; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    jne .LBB11_1
+; SSE4-NEXT:  # %bb.2: # %else
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    jne .LBB11_3
+; SSE4-NEXT:  .LBB11_4: # %else2
+; SSE4-NEXT:    testb $4, %al
+; SSE4-NEXT:    jne .LBB11_5
+; SSE4-NEXT:  .LBB11_6: # %else4
+; SSE4-NEXT:    testb $8, %al
+; SSE4-NEXT:    jne .LBB11_7
+; SSE4-NEXT:  .LBB11_8: # %else6
+; SSE4-NEXT:    testb $16, %al
+; SSE4-NEXT:    jne .LBB11_9
+; SSE4-NEXT:  .LBB11_10: # %else8
+; SSE4-NEXT:    testb $32, %al
+; SSE4-NEXT:    jne .LBB11_11
+; SSE4-NEXT:  .LBB11_12: # %else10
+; SSE4-NEXT:    testb $64, %al
+; SSE4-NEXT:    jne .LBB11_13
+; SSE4-NEXT:  .LBB11_14: # %else12
+; SSE4-NEXT:    testb $-128, %al
+; SSE4-NEXT:    jne .LBB11_15
+; SSE4-NEXT:  .LBB11_16: # %else14
+; SSE4-NEXT:    retq
+; SSE4-NEXT:  .LBB11_1: # %cond.store
+; SSE4-NEXT:    pextrw $0, %xmm0, (%rdi)
+; SSE4-NEXT:    testb $2, %al
 ; SSE4-NEXT:    je .LBB11_4
-; SSE4-NEXT:  # %bb.3: # %cond.store1
+; SSE4-NEXT:  .LBB11_3: # %cond.store1
 ; SSE4-NEXT:    pextrw $1, %xmm0, 2(%rdi)
-; SSE4-NEXT:  .LBB11_4: # %else2
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm2
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE4-NEXT:    pxor %xmm2, %xmm1
-; SSE4-NEXT:    pextrb $8, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $4, %al
 ; SSE4-NEXT:    je .LBB11_6
-; SSE4-NEXT:  # %bb.5: # %cond.store3
+; SSE4-NEXT:  .LBB11_5: # %cond.store3
 ; SSE4-NEXT:    pextrw $2, %xmm0, 4(%rdi)
-; SSE4-NEXT:  .LBB11_6: # %else4
-; SSE4-NEXT:    pextrb $12, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $8, %al
 ; SSE4-NEXT:    je .LBB11_8
-; SSE4-NEXT:  # %bb.7: # %cond.store5
+; SSE4-NEXT:  .LBB11_7: # %cond.store5
 ; SSE4-NEXT:    pextrw $3, %xmm0, 6(%rdi)
-; SSE4-NEXT:  .LBB11_8: # %else6
-; SSE4-NEXT:    pxor %xmm2, %xmm2
-; SSE4-NEXT:    pcmpeqd %xmm3, %xmm2
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE4-NEXT:    pxor %xmm2, %xmm1
-; SSE4-NEXT:    pextrb $0, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $16, %al
 ; SSE4-NEXT:    je .LBB11_10
-; SSE4-NEXT:  # %bb.9: # %cond.store7
+; SSE4-NEXT:  .LBB11_9: # %cond.store7
 ; SSE4-NEXT:    pextrw $4, %xmm0, 8(%rdi)
-; SSE4-NEXT:  .LBB11_10: # %else8
-; SSE4-NEXT:    pextrb $4, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $32, %al
 ; SSE4-NEXT:    je .LBB11_12
-; SSE4-NEXT:  # %bb.11: # %cond.store9
+; SSE4-NEXT:  .LBB11_11: # %cond.store9
 ; SSE4-NEXT:    pextrw $5, %xmm0, 10(%rdi)
-; SSE4-NEXT:  .LBB11_12: # %else10
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm3
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE4-NEXT:    pxor %xmm3, %xmm1
-; SSE4-NEXT:    pextrb $8, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $64, %al
 ; SSE4-NEXT:    je .LBB11_14
-; SSE4-NEXT:  # %bb.13: # %cond.store11
+; SSE4-NEXT:  .LBB11_13: # %cond.store11
 ; SSE4-NEXT:    pextrw $6, %xmm0, 12(%rdi)
-; SSE4-NEXT:  .LBB11_14: # %else12
-; SSE4-NEXT:    pextrb $12, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $-128, %al
 ; SSE4-NEXT:    je .LBB11_16
-; SSE4-NEXT:  # %bb.15: # %cond.store13
+; SSE4-NEXT:  .LBB11_15: # %cond.store13
 ; SSE4-NEXT:    pextrw $7, %xmm0, 14(%rdi)
-; SSE4-NEXT:  .LBB11_16: # %else14
 ; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: truncstore_v8i32_v8i16:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm4, %xmm4
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm5
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
-; AVX1-NEXT:    vpshufb %xmm6, %xmm5, %xmm5
-; AVX1-NEXT:    vpshufb %xmm6, %xmm0, %xmm0
-; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm5[0]
-; AVX1-NEXT:    vpextrb $0, %xmm4, %eax
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
+; AVX1-NEXT:    vpshufb %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; AVX1-NEXT:    vmovmskps %ymm1, %eax
+; AVX1-NEXT:    notl %eax
 ; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je .LBB11_2
-; AVX1-NEXT:  # %bb.1: # %cond.store
+; AVX1-NEXT:    jne .LBB11_1
+; AVX1-NEXT:  # %bb.2: # %else
+; AVX1-NEXT:    testb $2, %al
+; AVX1-NEXT:    jne .LBB11_3
+; AVX1-NEXT:  .LBB11_4: # %else2
+; AVX1-NEXT:    testb $4, %al
+; AVX1-NEXT:    jne .LBB11_5
+; AVX1-NEXT:  .LBB11_6: # %else4
+; AVX1-NEXT:    testb $8, %al
+; AVX1-NEXT:    jne .LBB11_7
+; AVX1-NEXT:  .LBB11_8: # %else6
+; AVX1-NEXT:    testb $16, %al
+; AVX1-NEXT:    jne .LBB11_9
+; AVX1-NEXT:  .LBB11_10: # %else8
+; AVX1-NEXT:    testb $32, %al
+; AVX1-NEXT:    jne .LBB11_11
+; AVX1-NEXT:  .LBB11_12: # %else10
+; AVX1-NEXT:    testb $64, %al
+; AVX1-NEXT:    jne .LBB11_13
+; AVX1-NEXT:  .LBB11_14: # %else12
+; AVX1-NEXT:    testb $-128, %al
+; AVX1-NEXT:    jne .LBB11_15
+; AVX1-NEXT:  .LBB11_16: # %else14
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+; AVX1-NEXT:  .LBB11_1: # %cond.store
 ; AVX1-NEXT:    vpextrw $0, %xmm0, (%rdi)
-; AVX1-NEXT:  .LBB11_2: # %else
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $2, %al
 ; AVX1-NEXT:    je .LBB11_4
-; AVX1-NEXT:  # %bb.3: # %cond.store1
+; AVX1-NEXT:  .LBB11_3: # %cond.store1
 ; AVX1-NEXT:    vpextrw $1, %xmm0, 2(%rdi)
-; AVX1-NEXT:  .LBB11_4: # %else2
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm4, %xmm4
-; AVX1-NEXT:    vpextrb $8, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $4, %al
 ; AVX1-NEXT:    je .LBB11_6
-; AVX1-NEXT:  # %bb.5: # %cond.store3
+; AVX1-NEXT:  .LBB11_5: # %cond.store3
 ; AVX1-NEXT:    vpextrw $2, %xmm0, 4(%rdi)
-; AVX1-NEXT:  .LBB11_6: # %else4
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $12, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $8, %al
 ; AVX1-NEXT:    je .LBB11_8
-; AVX1-NEXT:  # %bb.7: # %cond.store5
+; AVX1-NEXT:  .LBB11_7: # %cond.store5
 ; AVX1-NEXT:    vpextrw $3, %xmm0, 6(%rdi)
-; AVX1-NEXT:  .LBB11_8: # %else6
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm1
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $0, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $16, %al
 ; AVX1-NEXT:    je .LBB11_10
-; AVX1-NEXT:  # %bb.9: # %cond.store7
+; AVX1-NEXT:  .LBB11_9: # %cond.store7
 ; AVX1-NEXT:    vpextrw $4, %xmm0, 8(%rdi)
-; AVX1-NEXT:  .LBB11_10: # %else8
-; AVX1-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $32, %al
 ; AVX1-NEXT:    je .LBB11_12
-; AVX1-NEXT:  # %bb.11: # %cond.store9
+; AVX1-NEXT:  .LBB11_11: # %cond.store9
 ; AVX1-NEXT:    vpextrw $5, %xmm0, 10(%rdi)
-; AVX1-NEXT:  .LBB11_12: # %else10
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $64, %al
 ; AVX1-NEXT:    je .LBB11_14
-; AVX1-NEXT:  # %bb.13: # %cond.store11
+; AVX1-NEXT:  .LBB11_13: # %cond.store11
 ; AVX1-NEXT:    vpextrw $6, %xmm0, 12(%rdi)
-; AVX1-NEXT:  .LBB11_14: # %else12
-; AVX1-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $-128, %al
 ; AVX1-NEXT:    je .LBB11_16
-; AVX1-NEXT:  # %bb.15: # %cond.store13
+; AVX1-NEXT:  .LBB11_15: # %cond.store13
 ; AVX1-NEXT:    vpextrw $7, %xmm0, 14(%rdi)
-; AVX1-NEXT:  .LBB11_16: # %else14
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: truncstore_v8i32_v8i16:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm4
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    vpxor %xmm3, %xmm4, %xmm4
 ; AVX2-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-NEXT:    vpextrb $0, %xmm4, %eax
+; AVX2-NEXT:    vpcmpeqd %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    vmovmskps %ymm1, %eax
+; AVX2-NEXT:    notl %eax
 ; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je .LBB11_2
-; AVX2-NEXT:  # %bb.1: # %cond.store
+; AVX2-NEXT:    jne .LBB11_1
+; AVX2-NEXT:  # %bb.2: # %else
+; AVX2-NEXT:    testb $2, %al
+; AVX2-NEXT:    jne .LBB11_3
+; AVX2-NEXT:  .LBB11_4: # %else2
+; AVX2-NEXT:    testb $4, %al
+; AVX2-NEXT:    jne .LBB11_5
+; AVX2-NEXT:  .LBB11_6: # %else4
+; AVX2-NEXT:    testb $8, %al
+; AVX2-NEXT:    jne .LBB11_7
+; AVX2-NEXT:  .LBB11_8: # %else6
+; AVX2-NEXT:    testb $16, %al
+; AVX2-NEXT:    jne .LBB11_9
+; AVX2-NEXT:  .LBB11_10: # %else8
+; AVX2-NEXT:    testb $32, %al
+; AVX2-NEXT:    jne .LBB11_11
+; AVX2-NEXT:  .LBB11_12: # %else10
+; AVX2-NEXT:    testb $64, %al
+; AVX2-NEXT:    jne .LBB11_13
+; AVX2-NEXT:  .LBB11_14: # %else12
+; AVX2-NEXT:    testb $-128, %al
+; AVX2-NEXT:    jne .LBB11_15
+; AVX2-NEXT:  .LBB11_16: # %else14
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+; AVX2-NEXT:  .LBB11_1: # %cond.store
 ; AVX2-NEXT:    vpextrw $0, %xmm0, (%rdi)
-; AVX2-NEXT:  .LBB11_2: # %else
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $2, %al
 ; AVX2-NEXT:    je .LBB11_4
-; AVX2-NEXT:  # %bb.3: # %cond.store1
+; AVX2-NEXT:  .LBB11_3: # %cond.store1
 ; AVX2-NEXT:    vpextrw $1, %xmm0, 2(%rdi)
-; AVX2-NEXT:  .LBB11_4: # %else2
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm4
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    vpxor %xmm3, %xmm4, %xmm4
-; AVX2-NEXT:    vpextrb $8, %xmm4, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $4, %al
 ; AVX2-NEXT:    je .LBB11_6
-; AVX2-NEXT:  # %bb.5: # %cond.store3
+; AVX2-NEXT:  .LBB11_5: # %cond.store3
 ; AVX2-NEXT:    vpextrw $2, %xmm0, 4(%rdi)
-; AVX2-NEXT:  .LBB11_6: # %else4
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrb $12, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $8, %al
 ; AVX2-NEXT:    je .LBB11_8
-; AVX2-NEXT:  # %bb.7: # %cond.store5
+; AVX2-NEXT:  .LBB11_7: # %cond.store5
 ; AVX2-NEXT:    vpextrw $3, %xmm0, 6(%rdi)
-; AVX2-NEXT:  .LBB11_8: # %else6
-; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm1
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrb $0, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $16, %al
 ; AVX2-NEXT:    je .LBB11_10
-; AVX2-NEXT:  # %bb.9: # %cond.store7
+; AVX2-NEXT:  .LBB11_9: # %cond.store7
 ; AVX2-NEXT:    vpextrw $4, %xmm0, 8(%rdi)
-; AVX2-NEXT:  .LBB11_10: # %else8
-; AVX2-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $32, %al
 ; AVX2-NEXT:    je .LBB11_12
-; AVX2-NEXT:  # %bb.11: # %cond.store9
+; AVX2-NEXT:  .LBB11_11: # %cond.store9
 ; AVX2-NEXT:    vpextrw $5, %xmm0, 10(%rdi)
-; AVX2-NEXT:  .LBB11_12: # %else10
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $64, %al
 ; AVX2-NEXT:    je .LBB11_14
-; AVX2-NEXT:  # %bb.13: # %cond.store11
+; AVX2-NEXT:  .LBB11_13: # %cond.store11
 ; AVX2-NEXT:    vpextrw $6, %xmm0, 12(%rdi)
-; AVX2-NEXT:  .LBB11_14: # %else12
-; AVX2-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $-128, %al
 ; AVX2-NEXT:    je .LBB11_16
-; AVX2-NEXT:  # %bb.15: # %cond.store13
+; AVX2-NEXT:  .LBB11_15: # %cond.store13
 ; AVX2-NEXT:    vpextrw $7, %xmm0, 14(%rdi)
-; AVX2-NEXT:  .LBB11_16: # %else14
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
@@ -3824,66 +3627,61 @@ define void @truncstore_v8i32_v8i16(<8 x i32> %x, <8 x i16>* %p, <8 x i32> %mask
 ; AVX512F-NEXT:    vpmovdw %zmm0, %ymm0
 ; AVX512F-NEXT:    kmovw %k0, %eax
 ; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB11_2
-; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    jne .LBB11_1
+; AVX512F-NEXT:  # %bb.2: # %else
+; AVX512F-NEXT:    testb $2, %al
+; AVX512F-NEXT:    jne .LBB11_3
+; AVX512F-NEXT:  .LBB11_4: # %else2
+; AVX512F-NEXT:    testb $4, %al
+; AVX512F-NEXT:    jne .LBB11_5
+; AVX512F-NEXT:  .LBB11_6: # %else4
+; AVX512F-NEXT:    testb $8, %al
+; AVX512F-NEXT:    jne .LBB11_7
+; AVX512F-NEXT:  .LBB11_8: # %else6
+; AVX512F-NEXT:    testb $16, %al
+; AVX512F-NEXT:    jne .LBB11_9
+; AVX512F-NEXT:  .LBB11_10: # %else8
+; AVX512F-NEXT:    testb $32, %al
+; AVX512F-NEXT:    jne .LBB11_11
+; AVX512F-NEXT:  .LBB11_12: # %else10
+; AVX512F-NEXT:    testb $64, %al
+; AVX512F-NEXT:    jne .LBB11_13
+; AVX512F-NEXT:  .LBB11_14: # %else12
+; AVX512F-NEXT:    testb $-128, %al
+; AVX512F-NEXT:    jne .LBB11_15
+; AVX512F-NEXT:  .LBB11_16: # %else14
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+; AVX512F-NEXT:  .LBB11_1: # %cond.store
 ; AVX512F-NEXT:    vpextrw $0, %xmm0, (%rdi)
-; AVX512F-NEXT:  .LBB11_2: # %else
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $2, %al
 ; AVX512F-NEXT:    je .LBB11_4
-; AVX512F-NEXT:  # %bb.3: # %cond.store1
+; AVX512F-NEXT:  .LBB11_3: # %cond.store1
 ; AVX512F-NEXT:    vpextrw $1, %xmm0, 2(%rdi)
-; AVX512F-NEXT:  .LBB11_4: # %else2
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $4, %al
 ; AVX512F-NEXT:    je .LBB11_6
-; AVX512F-NEXT:  # %bb.5: # %cond.store3
+; AVX512F-NEXT:  .LBB11_5: # %cond.store3
 ; AVX512F-NEXT:    vpextrw $2, %xmm0, 4(%rdi)
-; AVX512F-NEXT:  .LBB11_6: # %else4
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $8, %al
 ; AVX512F-NEXT:    je .LBB11_8
-; AVX512F-NEXT:  # %bb.7: # %cond.store5
+; AVX512F-NEXT:  .LBB11_7: # %cond.store5
 ; AVX512F-NEXT:    vpextrw $3, %xmm0, 6(%rdi)
-; AVX512F-NEXT:  .LBB11_8: # %else6
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $4, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $16, %al
 ; AVX512F-NEXT:    je .LBB11_10
-; AVX512F-NEXT:  # %bb.9: # %cond.store7
+; AVX512F-NEXT:  .LBB11_9: # %cond.store7
 ; AVX512F-NEXT:    vpextrw $4, %xmm0, 8(%rdi)
-; AVX512F-NEXT:  .LBB11_10: # %else8
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $5, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $32, %al
 ; AVX512F-NEXT:    je .LBB11_12
-; AVX512F-NEXT:  # %bb.11: # %cond.store9
+; AVX512F-NEXT:  .LBB11_11: # %cond.store9
 ; AVX512F-NEXT:    vpextrw $5, %xmm0, 10(%rdi)
-; AVX512F-NEXT:  .LBB11_12: # %else10
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $6, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $64, %al
 ; AVX512F-NEXT:    je .LBB11_14
-; AVX512F-NEXT:  # %bb.13: # %cond.store11
+; AVX512F-NEXT:  .LBB11_13: # %cond.store11
 ; AVX512F-NEXT:    vpextrw $6, %xmm0, 12(%rdi)
-; AVX512F-NEXT:  .LBB11_14: # %else12
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $7, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $-128, %al
 ; AVX512F-NEXT:    je .LBB11_16
-; AVX512F-NEXT:  # %bb.15: # %cond.store13
+; AVX512F-NEXT:  .LBB11_15: # %cond.store13
 ; AVX512F-NEXT:    vpextrw $7, %xmm0, 14(%rdi)
-; AVX512F-NEXT:  .LBB11_16: # %else14
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
@@ -3914,313 +3712,290 @@ define void @truncstore_v8i32_v8i16(<8 x i32> %x, <8 x i16>* %p, <8 x i32> %mask
 define void @truncstore_v8i32_v8i8(<8 x i32> %x, <8 x i8>* %p, <8 x i32> %mask) {
 ; SSE2-LABEL: truncstore_v8i32_v8i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pxor %xmm5, %xmm5
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm5
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm4
-; SSE2-NEXT:    pxor %xmm5, %xmm4
-; SSE2-NEXT:    movdqa %xmm4, %xmm5
-; SSE2-NEXT:    packssdw %xmm0, %xmm5
-; SSE2-NEXT:    pslld $16, %xmm1
-; SSE2-NEXT:    psrad $16, %xmm1
-; SSE2-NEXT:    pslld $16, %xmm0
-; SSE2-NEXT:    psrad $16, %xmm0
-; SSE2-NEXT:    packssdw %xmm1, %xmm0
-; SSE2-NEXT:    movd %xmm5, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    je .LBB12_2
-; SSE2-NEXT:  # %bb.1: # %cond.store
-; SSE2-NEXT:    movb %al, (%rdi)
-; SSE2-NEXT:  .LBB12_2: # %else
-; SSE2-NEXT:    psrlq $16, %xmm4
-; SSE2-NEXT:    movd %xmm4, %ecx
-; SSE2-NEXT:    shrl $16, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    je .LBB12_4
-; SSE2-NEXT:  # %bb.3: # %cond.store1
-; SSE2-NEXT:    shrl $16, %eax
-; SSE2-NEXT:    movb %al, 1(%rdi)
-; SSE2-NEXT:  .LBB12_4: # %else2
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm2
+; SSE2-NEXT:    pxor %xmm4, %xmm4
+; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0]
+; SSE2-NEXT:    pand %xmm5, %xmm1
+; SSE2-NEXT:    pand %xmm5, %xmm0
+; SSE2-NEXT:    packuswb %xmm1, %xmm0
+; SSE2-NEXT:    packuswb %xmm0, %xmm0
+; SSE2-NEXT:    pcmpeqd %xmm4, %xmm3
 ; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pxor %xmm2, %xmm1
-; SSE2-NEXT:    pextrw $4, %xmm1, %eax
+; SSE2-NEXT:    pxor %xmm1, %xmm3
+; SSE2-NEXT:    pcmpeqd %xmm4, %xmm2
+; SSE2-NEXT:    pxor %xmm1, %xmm2
+; SSE2-NEXT:    packssdw %xmm3, %xmm2
+; SSE2-NEXT:    packsswb %xmm0, %xmm2
+; SSE2-NEXT:    pmovmskb %xmm2, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB12_6
-; SSE2-NEXT:  # %bb.5: # %cond.store3
-; SSE2-NEXT:    pextrw $2, %xmm0, %eax
-; SSE2-NEXT:    movb %al, 2(%rdi)
+; SSE2-NEXT:    movd %xmm0, %ecx
+; SSE2-NEXT:    jne .LBB12_1
+; SSE2-NEXT:  # %bb.2: # %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne .LBB12_3
+; SSE2-NEXT:  .LBB12_4: # %else2
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne .LBB12_5
 ; SSE2-NEXT:  .LBB12_6: # %else4
-; SSE2-NEXT:    pextrw $6, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je .LBB12_8
-; SSE2-NEXT:  # %bb.7: # %cond.store5
-; SSE2-NEXT:    pextrw $3, %xmm0, %eax
-; SSE2-NEXT:    movb %al, 3(%rdi)
+; SSE2-NEXT:  .LBB12_7: # %cond.store5
+; SSE2-NEXT:    shrl $24, %ecx
+; SSE2-NEXT:    movb %cl, 3(%rdi)
 ; SSE2-NEXT:  .LBB12_8: # %else6
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pxor %xmm2, %xmm1
-; SSE2-NEXT:    pextrw $0, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $16, %al
+; SSE2-NEXT:    pextrw $2, %xmm0, %ecx
 ; SSE2-NEXT:    je .LBB12_10
 ; SSE2-NEXT:  # %bb.9: # %cond.store7
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    movb %al, 4(%rdi)
+; SSE2-NEXT:    movb %cl, 4(%rdi)
 ; SSE2-NEXT:  .LBB12_10: # %else8
-; SSE2-NEXT:    pextrw $2, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $32, %al
 ; SSE2-NEXT:    je .LBB12_12
 ; SSE2-NEXT:  # %bb.11: # %cond.store9
-; SSE2-NEXT:    pextrw $5, %xmm0, %eax
-; SSE2-NEXT:    movb %al, 5(%rdi)
+; SSE2-NEXT:    movb %ch, 5(%rdi)
 ; SSE2-NEXT:  .LBB12_12: # %else10
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm3
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pxor %xmm3, %xmm1
-; SSE2-NEXT:    pextrw $4, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB12_14
-; SSE2-NEXT:  # %bb.13: # %cond.store11
-; SSE2-NEXT:    pextrw $6, %xmm0, %eax
-; SSE2-NEXT:    movb %al, 6(%rdi)
-; SSE2-NEXT:  .LBB12_14: # %else12
-; SSE2-NEXT:    pextrw $6, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB12_16
-; SSE2-NEXT:  # %bb.15: # %cond.store13
-; SSE2-NEXT:    pextrw $7, %xmm0, %eax
-; SSE2-NEXT:    movb %al, 7(%rdi)
+; SSE2-NEXT:    testb $64, %al
+; SSE2-NEXT:    pextrw $3, %xmm0, %ecx
+; SSE2-NEXT:    jne .LBB12_13
+; SSE2-NEXT:  # %bb.14: # %else12
+; SSE2-NEXT:    testb $-128, %al
+; SSE2-NEXT:    jne .LBB12_15
 ; SSE2-NEXT:  .LBB12_16: # %else14
 ; SSE2-NEXT:    retq
+; SSE2-NEXT:  .LBB12_1: # %cond.store
+; SSE2-NEXT:    movb %cl, (%rdi)
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    je .LBB12_4
+; SSE2-NEXT:  .LBB12_3: # %cond.store1
+; SSE2-NEXT:    movb %ch, 1(%rdi)
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    je .LBB12_6
+; SSE2-NEXT:  .LBB12_5: # %cond.store3
+; SSE2-NEXT:    movl %ecx, %edx
+; SSE2-NEXT:    shrl $16, %edx
+; SSE2-NEXT:    movb %dl, 2(%rdi)
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne .LBB12_7
+; SSE2-NEXT:    jmp .LBB12_8
+; SSE2-NEXT:  .LBB12_13: # %cond.store11
+; SSE2-NEXT:    movb %cl, 6(%rdi)
+; SSE2-NEXT:    testb $-128, %al
+; SSE2-NEXT:    je .LBB12_16
+; SSE2-NEXT:  .LBB12_15: # %cond.store13
+; SSE2-NEXT:    movb %ch, 7(%rdi)
+; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: truncstore_v8i32_v8i8:
 ; SSE4:       # %bb.0:
-; SSE4-NEXT:    pxor %xmm5, %xmm5
-; SSE4-NEXT:    pcmpeqd %xmm2, %xmm5
-; SSE4-NEXT:    pcmpeqd %xmm4, %xmm4
-; SSE4-NEXT:    pxor %xmm5, %xmm4
-; SSE4-NEXT:    movdqa {{.*#+}} xmm5 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
+; SSE4-NEXT:    pxor %xmm4, %xmm4
+; SSE4-NEXT:    movdqa {{.*#+}} xmm5 = <0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u>
 ; SSE4-NEXT:    pshufb %xmm5, %xmm1
 ; SSE4-NEXT:    pshufb %xmm5, %xmm0
-; SSE4-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSE4-NEXT:    pextrb $0, %xmm4, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB12_2
-; SSE4-NEXT:  # %bb.1: # %cond.store
+; SSE4-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE4-NEXT:    pcmpeqd %xmm4, %xmm3
+; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
+; SSE4-NEXT:    pxor %xmm1, %xmm3
+; SSE4-NEXT:    pcmpeqd %xmm4, %xmm2
+; SSE4-NEXT:    pxor %xmm1, %xmm2
+; SSE4-NEXT:    packssdw %xmm3, %xmm2
+; SSE4-NEXT:    packsswb %xmm0, %xmm2
+; SSE4-NEXT:    pmovmskb %xmm2, %eax
+; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    jne .LBB12_1
+; SSE4-NEXT:  # %bb.2: # %else
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    jne .LBB12_3
+; SSE4-NEXT:  .LBB12_4: # %else2
+; SSE4-NEXT:    testb $4, %al
+; SSE4-NEXT:    jne .LBB12_5
+; SSE4-NEXT:  .LBB12_6: # %else4
+; SSE4-NEXT:    testb $8, %al
+; SSE4-NEXT:    jne .LBB12_7
+; SSE4-NEXT:  .LBB12_8: # %else6
+; SSE4-NEXT:    testb $16, %al
+; SSE4-NEXT:    jne .LBB12_9
+; SSE4-NEXT:  .LBB12_10: # %else8
+; SSE4-NEXT:    testb $32, %al
+; SSE4-NEXT:    jne .LBB12_11
+; SSE4-NEXT:  .LBB12_12: # %else10
+; SSE4-NEXT:    testb $64, %al
+; SSE4-NEXT:    jne .LBB12_13
+; SSE4-NEXT:  .LBB12_14: # %else12
+; SSE4-NEXT:    testb $-128, %al
+; SSE4-NEXT:    jne .LBB12_15
+; SSE4-NEXT:  .LBB12_16: # %else14
+; SSE4-NEXT:    retq
+; SSE4-NEXT:  .LBB12_1: # %cond.store
 ; SSE4-NEXT:    pextrb $0, %xmm0, (%rdi)
-; SSE4-NEXT:  .LBB12_2: # %else
-; SSE4-NEXT:    pextrb $4, %xmm4, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $2, %al
 ; SSE4-NEXT:    je .LBB12_4
-; SSE4-NEXT:  # %bb.3: # %cond.store1
-; SSE4-NEXT:    pextrb $2, %xmm0, 1(%rdi)
-; SSE4-NEXT:  .LBB12_4: # %else2
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm2
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE4-NEXT:    pxor %xmm2, %xmm1
-; SSE4-NEXT:    pextrb $8, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:  .LBB12_3: # %cond.store1
+; SSE4-NEXT:    pextrb $1, %xmm0, 1(%rdi)
+; SSE4-NEXT:    testb $4, %al
 ; SSE4-NEXT:    je .LBB12_6
-; SSE4-NEXT:  # %bb.5: # %cond.store3
-; SSE4-NEXT:    pextrb $4, %xmm0, 2(%rdi)
-; SSE4-NEXT:  .LBB12_6: # %else4
-; SSE4-NEXT:    pextrb $12, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:  .LBB12_5: # %cond.store3
+; SSE4-NEXT:    pextrb $2, %xmm0, 2(%rdi)
+; SSE4-NEXT:    testb $8, %al
 ; SSE4-NEXT:    je .LBB12_8
-; SSE4-NEXT:  # %bb.7: # %cond.store5
-; SSE4-NEXT:    pextrb $6, %xmm0, 3(%rdi)
-; SSE4-NEXT:  .LBB12_8: # %else6
-; SSE4-NEXT:    pxor %xmm2, %xmm2
-; SSE4-NEXT:    pcmpeqd %xmm3, %xmm2
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE4-NEXT:    pxor %xmm2, %xmm1
-; SSE4-NEXT:    pextrb $0, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:  .LBB12_7: # %cond.store5
+; SSE4-NEXT:    pextrb $3, %xmm0, 3(%rdi)
+; SSE4-NEXT:    testb $16, %al
 ; SSE4-NEXT:    je .LBB12_10
-; SSE4-NEXT:  # %bb.9: # %cond.store7
-; SSE4-NEXT:    pextrb $8, %xmm0, 4(%rdi)
-; SSE4-NEXT:  .LBB12_10: # %else8
-; SSE4-NEXT:    pextrb $4, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:  .LBB12_9: # %cond.store7
+; SSE4-NEXT:    pextrb $4, %xmm0, 4(%rdi)
+; SSE4-NEXT:    testb $32, %al
 ; SSE4-NEXT:    je .LBB12_12
-; SSE4-NEXT:  # %bb.11: # %cond.store9
-; SSE4-NEXT:    pextrb $10, %xmm0, 5(%rdi)
-; SSE4-NEXT:  .LBB12_12: # %else10
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm3
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE4-NEXT:    pxor %xmm3, %xmm1
-; SSE4-NEXT:    pextrb $8, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:  .LBB12_11: # %cond.store9
+; SSE4-NEXT:    pextrb $5, %xmm0, 5(%rdi)
+; SSE4-NEXT:    testb $64, %al
 ; SSE4-NEXT:    je .LBB12_14
-; SSE4-NEXT:  # %bb.13: # %cond.store11
-; SSE4-NEXT:    pextrb $12, %xmm0, 6(%rdi)
-; SSE4-NEXT:  .LBB12_14: # %else12
-; SSE4-NEXT:    pextrb $12, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:  .LBB12_13: # %cond.store11
+; SSE4-NEXT:    pextrb $6, %xmm0, 6(%rdi)
+; SSE4-NEXT:    testb $-128, %al
 ; SSE4-NEXT:    je .LBB12_16
-; SSE4-NEXT:  # %bb.15: # %cond.store13
-; SSE4-NEXT:    pextrb $14, %xmm0, 7(%rdi)
-; SSE4-NEXT:  .LBB12_16: # %else14
+; SSE4-NEXT:  .LBB12_15: # %cond.store13
+; SSE4-NEXT:    pextrb $7, %xmm0, 7(%rdi)
 ; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: truncstore_v8i32_v8i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm4, %xmm4
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm5
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
-; AVX1-NEXT:    vpshufb %xmm6, %xmm5, %xmm5
-; AVX1-NEXT:    vpshufb %xmm6, %xmm0, %xmm0
-; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm5[0]
-; AVX1-NEXT:    vpextrb $0, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je .LBB12_2
-; AVX1-NEXT:  # %bb.1: # %cond.store
-; AVX1-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX1-NEXT:  .LBB12_2: # %else
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $4, %xmm2, %eax
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = <0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX1-NEXT:    vpshufb %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; AVX1-NEXT:    vmovmskps %ymm1, %eax
+; AVX1-NEXT:    notl %eax
 ; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je .LBB12_4
-; AVX1-NEXT:  # %bb.3: # %cond.store1
-; AVX1-NEXT:    vpextrb $2, %xmm0, 1(%rdi)
+; AVX1-NEXT:    jne .LBB12_1
+; AVX1-NEXT:  # %bb.2: # %else
+; AVX1-NEXT:    testb $2, %al
+; AVX1-NEXT:    jne .LBB12_3
 ; AVX1-NEXT:  .LBB12_4: # %else2
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm4, %xmm4
-; AVX1-NEXT:    vpextrb $8, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je .LBB12_6
-; AVX1-NEXT:  # %bb.5: # %cond.store3
-; AVX1-NEXT:    vpextrb $4, %xmm0, 2(%rdi)
+; AVX1-NEXT:    testb $4, %al
+; AVX1-NEXT:    jne .LBB12_5
 ; AVX1-NEXT:  .LBB12_6: # %else4
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $12, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je .LBB12_8
-; AVX1-NEXT:  # %bb.7: # %cond.store5
-; AVX1-NEXT:    vpextrb $6, %xmm0, 3(%rdi)
+; AVX1-NEXT:    testb $8, %al
+; AVX1-NEXT:    jne .LBB12_7
 ; AVX1-NEXT:  .LBB12_8: # %else6
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm1
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $0, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je .LBB12_10
-; AVX1-NEXT:  # %bb.9: # %cond.store7
-; AVX1-NEXT:    vpextrb $8, %xmm0, 4(%rdi)
+; AVX1-NEXT:    testb $16, %al
+; AVX1-NEXT:    jne .LBB12_9
 ; AVX1-NEXT:  .LBB12_10: # %else8
-; AVX1-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je .LBB12_12
-; AVX1-NEXT:  # %bb.11: # %cond.store9
-; AVX1-NEXT:    vpextrb $10, %xmm0, 5(%rdi)
+; AVX1-NEXT:    testb $32, %al
+; AVX1-NEXT:    jne .LBB12_11
 ; AVX1-NEXT:  .LBB12_12: # %else10
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je .LBB12_14
-; AVX1-NEXT:  # %bb.13: # %cond.store11
-; AVX1-NEXT:    vpextrb $12, %xmm0, 6(%rdi)
+; AVX1-NEXT:    testb $64, %al
+; AVX1-NEXT:    jne .LBB12_13
 ; AVX1-NEXT:  .LBB12_14: # %else12
-; AVX1-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je .LBB12_16
-; AVX1-NEXT:  # %bb.15: # %cond.store13
-; AVX1-NEXT:    vpextrb $14, %xmm0, 7(%rdi)
+; AVX1-NEXT:    testb $-128, %al
+; AVX1-NEXT:    jne .LBB12_15
 ; AVX1-NEXT:  .LBB12_16: # %else14
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
+; AVX1-NEXT:  .LBB12_1: # %cond.store
+; AVX1-NEXT:    vpextrb $0, %xmm0, (%rdi)
+; AVX1-NEXT:    testb $2, %al
+; AVX1-NEXT:    je .LBB12_4
+; AVX1-NEXT:  .LBB12_3: # %cond.store1
+; AVX1-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
+; AVX1-NEXT:    testb $4, %al
+; AVX1-NEXT:    je .LBB12_6
+; AVX1-NEXT:  .LBB12_5: # %cond.store3
+; AVX1-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
+; AVX1-NEXT:    testb $8, %al
+; AVX1-NEXT:    je .LBB12_8
+; AVX1-NEXT:  .LBB12_7: # %cond.store5
+; AVX1-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
+; AVX1-NEXT:    testb $16, %al
+; AVX1-NEXT:    je .LBB12_10
+; AVX1-NEXT:  .LBB12_9: # %cond.store7
+; AVX1-NEXT:    vpextrb $4, %xmm0, 4(%rdi)
+; AVX1-NEXT:    testb $32, %al
+; AVX1-NEXT:    je .LBB12_12
+; AVX1-NEXT:  .LBB12_11: # %cond.store9
+; AVX1-NEXT:    vpextrb $5, %xmm0, 5(%rdi)
+; AVX1-NEXT:    testb $64, %al
+; AVX1-NEXT:    je .LBB12_14
+; AVX1-NEXT:  .LBB12_13: # %cond.store11
+; AVX1-NEXT:    vpextrb $6, %xmm0, 6(%rdi)
+; AVX1-NEXT:    testb $-128, %al
+; AVX1-NEXT:    je .LBB12_16
+; AVX1-NEXT:  .LBB12_15: # %cond.store13
+; AVX1-NEXT:    vpextrb $7, %xmm0, 7(%rdi)
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: truncstore_v8i32_v8i8:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm4
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    vpxor %xmm3, %xmm4, %xmm4
-; AVX2-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-NEXT:    vpextrb $0, %xmm4, %eax
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm4 = <0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX2-NEXT:    vpshufb %xmm4, %xmm3, %xmm3
+; AVX2-NEXT:    vpshufb %xmm4, %xmm0, %xmm0
+; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; AVX2-NEXT:    vpcmpeqd %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    vmovmskps %ymm1, %eax
+; AVX2-NEXT:    notl %eax
 ; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je .LBB12_2
-; AVX2-NEXT:  # %bb.1: # %cond.store
+; AVX2-NEXT:    jne .LBB12_1
+; AVX2-NEXT:  # %bb.2: # %else
+; AVX2-NEXT:    testb $2, %al
+; AVX2-NEXT:    jne .LBB12_3
+; AVX2-NEXT:  .LBB12_4: # %else2
+; AVX2-NEXT:    testb $4, %al
+; AVX2-NEXT:    jne .LBB12_5
+; AVX2-NEXT:  .LBB12_6: # %else4
+; AVX2-NEXT:    testb $8, %al
+; AVX2-NEXT:    jne .LBB12_7
+; AVX2-NEXT:  .LBB12_8: # %else6
+; AVX2-NEXT:    testb $16, %al
+; AVX2-NEXT:    jne .LBB12_9
+; AVX2-NEXT:  .LBB12_10: # %else8
+; AVX2-NEXT:    testb $32, %al
+; AVX2-NEXT:    jne .LBB12_11
+; AVX2-NEXT:  .LBB12_12: # %else10
+; AVX2-NEXT:    testb $64, %al
+; AVX2-NEXT:    jne .LBB12_13
+; AVX2-NEXT:  .LBB12_14: # %else12
+; AVX2-NEXT:    testb $-128, %al
+; AVX2-NEXT:    jne .LBB12_15
+; AVX2-NEXT:  .LBB12_16: # %else14
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+; AVX2-NEXT:  .LBB12_1: # %cond.store
 ; AVX2-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX2-NEXT:  .LBB12_2: # %else
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $2, %al
 ; AVX2-NEXT:    je .LBB12_4
-; AVX2-NEXT:  # %bb.3: # %cond.store1
-; AVX2-NEXT:    vpextrb $2, %xmm0, 1(%rdi)
-; AVX2-NEXT:  .LBB12_4: # %else2
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm4
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    vpxor %xmm3, %xmm4, %xmm4
-; AVX2-NEXT:    vpextrb $8, %xmm4, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:  .LBB12_3: # %cond.store1
+; AVX2-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
+; AVX2-NEXT:    testb $4, %al
 ; AVX2-NEXT:    je .LBB12_6
-; AVX2-NEXT:  # %bb.5: # %cond.store3
-; AVX2-NEXT:    vpextrb $4, %xmm0, 2(%rdi)
-; AVX2-NEXT:  .LBB12_6: # %else4
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrb $12, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:  .LBB12_5: # %cond.store3
+; AVX2-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
+; AVX2-NEXT:    testb $8, %al
 ; AVX2-NEXT:    je .LBB12_8
-; AVX2-NEXT:  # %bb.7: # %cond.store5
-; AVX2-NEXT:    vpextrb $6, %xmm0, 3(%rdi)
-; AVX2-NEXT:  .LBB12_8: # %else6
-; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm1
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrb $0, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:  .LBB12_7: # %cond.store5
+; AVX2-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
+; AVX2-NEXT:    testb $16, %al
 ; AVX2-NEXT:    je .LBB12_10
-; AVX2-NEXT:  # %bb.9: # %cond.store7
-; AVX2-NEXT:    vpextrb $8, %xmm0, 4(%rdi)
-; AVX2-NEXT:  .LBB12_10: # %else8
-; AVX2-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:  .LBB12_9: # %cond.store7
+; AVX2-NEXT:    vpextrb $4, %xmm0, 4(%rdi)
+; AVX2-NEXT:    testb $32, %al
 ; AVX2-NEXT:    je .LBB12_12
-; AVX2-NEXT:  # %bb.11: # %cond.store9
-; AVX2-NEXT:    vpextrb $10, %xmm0, 5(%rdi)
-; AVX2-NEXT:  .LBB12_12: # %else10
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:  .LBB12_11: # %cond.store9
+; AVX2-NEXT:    vpextrb $5, %xmm0, 5(%rdi)
+; AVX2-NEXT:    testb $64, %al
 ; AVX2-NEXT:    je .LBB12_14
-; AVX2-NEXT:  # %bb.13: # %cond.store11
-; AVX2-NEXT:    vpextrb $12, %xmm0, 6(%rdi)
-; AVX2-NEXT:  .LBB12_14: # %else12
-; AVX2-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:  .LBB12_13: # %cond.store11
+; AVX2-NEXT:    vpextrb $6, %xmm0, 6(%rdi)
+; AVX2-NEXT:    testb $-128, %al
 ; AVX2-NEXT:    je .LBB12_16
-; AVX2-NEXT:  # %bb.15: # %cond.store13
-; AVX2-NEXT:    vpextrb $14, %xmm0, 7(%rdi)
-; AVX2-NEXT:  .LBB12_16: # %else14
+; AVX2-NEXT:  .LBB12_15: # %cond.store13
+; AVX2-NEXT:    vpextrb $7, %xmm0, 7(%rdi)
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
@@ -4229,69 +4004,64 @@ define void @truncstore_v8i32_v8i8(<8 x i32> %x, <8 x i8>* %p, <8 x i32> %mask)
 ; AVX512F-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
 ; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
 ; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
 ; AVX512F-NEXT:    kmovw %k0, %eax
 ; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB12_2
-; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    jne .LBB12_1
+; AVX512F-NEXT:  # %bb.2: # %else
+; AVX512F-NEXT:    testb $2, %al
+; AVX512F-NEXT:    jne .LBB12_3
+; AVX512F-NEXT:  .LBB12_4: # %else2
+; AVX512F-NEXT:    testb $4, %al
+; AVX512F-NEXT:    jne .LBB12_5
+; AVX512F-NEXT:  .LBB12_6: # %else4
+; AVX512F-NEXT:    testb $8, %al
+; AVX512F-NEXT:    jne .LBB12_7
+; AVX512F-NEXT:  .LBB12_8: # %else6
+; AVX512F-NEXT:    testb $16, %al
+; AVX512F-NEXT:    jne .LBB12_9
+; AVX512F-NEXT:  .LBB12_10: # %else8
+; AVX512F-NEXT:    testb $32, %al
+; AVX512F-NEXT:    jne .LBB12_11
+; AVX512F-NEXT:  .LBB12_12: # %else10
+; AVX512F-NEXT:    testb $64, %al
+; AVX512F-NEXT:    jne .LBB12_13
+; AVX512F-NEXT:  .LBB12_14: # %else12
+; AVX512F-NEXT:    testb $-128, %al
+; AVX512F-NEXT:    jne .LBB12_15
+; AVX512F-NEXT:  .LBB12_16: # %else14
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+; AVX512F-NEXT:  .LBB12_1: # %cond.store
 ; AVX512F-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX512F-NEXT:  .LBB12_2: # %else
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $2, %al
 ; AVX512F-NEXT:    je .LBB12_4
-; AVX512F-NEXT:  # %bb.3: # %cond.store1
-; AVX512F-NEXT:    vpextrb $2, %xmm0, 1(%rdi)
-; AVX512F-NEXT:  .LBB12_4: # %else2
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  .LBB12_3: # %cond.store1
+; AVX512F-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
+; AVX512F-NEXT:    testb $4, %al
 ; AVX512F-NEXT:    je .LBB12_6
-; AVX512F-NEXT:  # %bb.5: # %cond.store3
-; AVX512F-NEXT:    vpextrb $4, %xmm0, 2(%rdi)
-; AVX512F-NEXT:  .LBB12_6: # %else4
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  .LBB12_5: # %cond.store3
+; AVX512F-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
+; AVX512F-NEXT:    testb $8, %al
 ; AVX512F-NEXT:    je .LBB12_8
-; AVX512F-NEXT:  # %bb.7: # %cond.store5
-; AVX512F-NEXT:    vpextrb $6, %xmm0, 3(%rdi)
-; AVX512F-NEXT:  .LBB12_8: # %else6
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $4, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  .LBB12_7: # %cond.store5
+; AVX512F-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
+; AVX512F-NEXT:    testb $16, %al
 ; AVX512F-NEXT:    je .LBB12_10
-; AVX512F-NEXT:  # %bb.9: # %cond.store7
-; AVX512F-NEXT:    vpextrb $8, %xmm0, 4(%rdi)
-; AVX512F-NEXT:  .LBB12_10: # %else8
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $5, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  .LBB12_9: # %cond.store7
+; AVX512F-NEXT:    vpextrb $4, %xmm0, 4(%rdi)
+; AVX512F-NEXT:    testb $32, %al
 ; AVX512F-NEXT:    je .LBB12_12
-; AVX512F-NEXT:  # %bb.11: # %cond.store9
-; AVX512F-NEXT:    vpextrb $10, %xmm0, 5(%rdi)
-; AVX512F-NEXT:  .LBB12_12: # %else10
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $6, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  .LBB12_11: # %cond.store9
+; AVX512F-NEXT:    vpextrb $5, %xmm0, 5(%rdi)
+; AVX512F-NEXT:    testb $64, %al
 ; AVX512F-NEXT:    je .LBB12_14
-; AVX512F-NEXT:  # %bb.13: # %cond.store11
-; AVX512F-NEXT:    vpextrb $12, %xmm0, 6(%rdi)
-; AVX512F-NEXT:  .LBB12_14: # %else12
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $7, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  .LBB12_13: # %cond.store11
+; AVX512F-NEXT:    vpextrb $6, %xmm0, 6(%rdi)
+; AVX512F-NEXT:    testb $-128, %al
 ; AVX512F-NEXT:    je .LBB12_16
-; AVX512F-NEXT:  # %bb.15: # %cond.store13
-; AVX512F-NEXT:    vpextrb $14, %xmm0, 7(%rdi)
-; AVX512F-NEXT:  .LBB12_16: # %else14
+; AVX512F-NEXT:  .LBB12_15: # %cond.store13
+; AVX512F-NEXT:    vpextrb $7, %xmm0, 7(%rdi)
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
@@ -4300,10 +4070,9 @@ define void @truncstore_v8i32_v8i8(<8 x i32> %x, <8 x i8>* %p, <8 x i32> %mask)
 ; AVX512BW-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
 ; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
 ; AVX512BW-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512BW-NEXT:    vpmovdw %zmm0, %ymm0
-; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
 ; AVX512BW-NEXT:    kshiftlq $56, %k0, %k0
 ; AVX512BW-NEXT:    kshiftrq $56, %k0, %k1
+; AVX512BW-NEXT:    vpmovdb %zmm0, %xmm0
 ; AVX512BW-NEXT:    vmovdqu8 %zmm0, (%rdi) {%k1}
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -4324,151 +4093,151 @@ define void @truncstore_v4i32_v4i16(<4 x i32> %x, <4 x i16>* %p, <4 x i32> %mask
 ; SSE2-LABEL: truncstore_v4i32_v4i16:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,6,6,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; SSE2-NEXT:    pcmpeqd %xmm1, %xmm2
-; SSE2-NEXT:    movd %xmm2, %eax
-; SSE2-NEXT:    notl %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB13_2
-; SSE2-NEXT:  # %bb.1: # %cond.store
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    movw %ax, (%rdi)
-; SSE2-NEXT:  .LBB13_2: # %else
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm3
-; SSE2-NEXT:    pxor %xmm3, %xmm2
-; SSE2-NEXT:    pextrw $2, %xmm2, %eax
+; SSE2-NEXT:    movmskps %xmm2, %eax
+; SSE2-NEXT:    xorl $15, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB13_4
-; SSE2-NEXT:  # %bb.3: # %cond.store1
-; SSE2-NEXT:    pextrw $2, %xmm0, %eax
-; SSE2-NEXT:    movw %ax, 2(%rdi)
+; SSE2-NEXT:    jne .LBB13_1
+; SSE2-NEXT:  # %bb.2: # %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne .LBB13_3
 ; SSE2-NEXT:  .LBB13_4: # %else2
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm1
-; SSE2-NEXT:    pxor %xmm1, %xmm3
-; SSE2-NEXT:    pextrw $4, %xmm3, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB13_6
-; SSE2-NEXT:  # %bb.5: # %cond.store3
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    movw %ax, 4(%rdi)
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne .LBB13_5
 ; SSE2-NEXT:  .LBB13_6: # %else4
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT:    pxor %xmm2, %xmm1
-; SSE2-NEXT:    pextrw $6, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne .LBB13_7
+; SSE2-NEXT:  .LBB13_8: # %else6
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  .LBB13_1: # %cond.store
+; SSE2-NEXT:    movd %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, (%rdi)
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    je .LBB13_4
+; SSE2-NEXT:  .LBB13_3: # %cond.store1
+; SSE2-NEXT:    pextrw $1, %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, 2(%rdi)
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    je .LBB13_6
+; SSE2-NEXT:  .LBB13_5: # %cond.store3
+; SSE2-NEXT:    pextrw $2, %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, 4(%rdi)
+; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je .LBB13_8
-; SSE2-NEXT:  # %bb.7: # %cond.store5
-; SSE2-NEXT:    pextrw $6, %xmm0, %eax
+; SSE2-NEXT:  .LBB13_7: # %cond.store5
+; SSE2-NEXT:    pextrw $3, %xmm0, %eax
 ; SSE2-NEXT:    movw %ax, 6(%rdi)
-; SSE2-NEXT:  .LBB13_8: # %else6
 ; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: truncstore_v4i32_v4i16:
 ; SSE4:       # %bb.0:
-; SSE4-NEXT:    pxor %xmm3, %xmm3
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm3
-; SSE4-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE4-NEXT:    pxor %xmm3, %xmm2
-; SSE4-NEXT:    pextrb $0, %xmm2, %eax
+; SSE4-NEXT:    pxor %xmm2, %xmm2
+; SSE4-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
+; SSE4-NEXT:    pcmpeqd %xmm1, %xmm2
+; SSE4-NEXT:    movmskps %xmm2, %eax
+; SSE4-NEXT:    xorl $15, %eax
 ; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB13_2
-; SSE4-NEXT:  # %bb.1: # %cond.store
+; SSE4-NEXT:    jne .LBB13_1
+; SSE4-NEXT:  # %bb.2: # %else
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    jne .LBB13_3
+; SSE4-NEXT:  .LBB13_4: # %else2
+; SSE4-NEXT:    testb $4, %al
+; SSE4-NEXT:    jne .LBB13_5
+; SSE4-NEXT:  .LBB13_6: # %else4
+; SSE4-NEXT:    testb $8, %al
+; SSE4-NEXT:    jne .LBB13_7
+; SSE4-NEXT:  .LBB13_8: # %else6
+; SSE4-NEXT:    retq
+; SSE4-NEXT:  .LBB13_1: # %cond.store
 ; SSE4-NEXT:    pextrw $0, %xmm0, (%rdi)
-; SSE4-NEXT:  .LBB13_2: # %else
-; SSE4-NEXT:    pextrb $4, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $2, %al
 ; SSE4-NEXT:    je .LBB13_4
-; SSE4-NEXT:  # %bb.3: # %cond.store1
-; SSE4-NEXT:    pextrw $2, %xmm0, 2(%rdi)
-; SSE4-NEXT:  .LBB13_4: # %else2
-; SSE4-NEXT:    pxor %xmm2, %xmm2
-; SSE4-NEXT:    pcmpeqd %xmm2, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE4-NEXT:    pxor %xmm1, %xmm2
-; SSE4-NEXT:    pextrb $8, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:  .LBB13_3: # %cond.store1
+; SSE4-NEXT:    pextrw $1, %xmm0, 2(%rdi)
+; SSE4-NEXT:    testb $4, %al
 ; SSE4-NEXT:    je .LBB13_6
-; SSE4-NEXT:  # %bb.5: # %cond.store3
-; SSE4-NEXT:    pextrw $4, %xmm0, 4(%rdi)
-; SSE4-NEXT:  .LBB13_6: # %else4
-; SSE4-NEXT:    pextrb $12, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:  .LBB13_5: # %cond.store3
+; SSE4-NEXT:    pextrw $2, %xmm0, 4(%rdi)
+; SSE4-NEXT:    testb $8, %al
 ; SSE4-NEXT:    je .LBB13_8
-; SSE4-NEXT:  # %bb.7: # %cond.store5
-; SSE4-NEXT:    pextrw $6, %xmm0, 6(%rdi)
-; SSE4-NEXT:  .LBB13_8: # %else6
+; SSE4-NEXT:  .LBB13_7: # %cond.store5
+; SSE4-NEXT:    pextrw $3, %xmm0, 6(%rdi)
 ; SSE4-NEXT:    retq
 ;
 ; AVX-LABEL: truncstore_v4i32_v4i16:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX-NEXT:    vpextrb $0, %xmm2, %eax
+; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
+; AVX-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
+; AVX-NEXT:    vmovmskps %xmm1, %eax
+; AVX-NEXT:    xorl $15, %eax
 ; AVX-NEXT:    testb $1, %al
-; AVX-NEXT:    je .LBB13_2
-; AVX-NEXT:  # %bb.1: # %cond.store
+; AVX-NEXT:    jne .LBB13_1
+; AVX-NEXT:  # %bb.2: # %else
+; AVX-NEXT:    testb $2, %al
+; AVX-NEXT:    jne .LBB13_3
+; AVX-NEXT:  .LBB13_4: # %else2
+; AVX-NEXT:    testb $4, %al
+; AVX-NEXT:    jne .LBB13_5
+; AVX-NEXT:  .LBB13_6: # %else4
+; AVX-NEXT:    testb $8, %al
+; AVX-NEXT:    jne .LBB13_7
+; AVX-NEXT:  .LBB13_8: # %else6
+; AVX-NEXT:    retq
+; AVX-NEXT:  .LBB13_1: # %cond.store
 ; AVX-NEXT:    vpextrw $0, %xmm0, (%rdi)
-; AVX-NEXT:  .LBB13_2: # %else
-; AVX-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX-NEXT:    testb $1, %al
+; AVX-NEXT:    testb $2, %al
 ; AVX-NEXT:    je .LBB13_4
-; AVX-NEXT:  # %bb.3: # %cond.store1
-; AVX-NEXT:    vpextrw $2, %xmm0, 2(%rdi)
-; AVX-NEXT:  .LBB13_4: # %else2
-; AVX-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX-NEXT:    testb $1, %al
+; AVX-NEXT:  .LBB13_3: # %cond.store1
+; AVX-NEXT:    vpextrw $1, %xmm0, 2(%rdi)
+; AVX-NEXT:    testb $4, %al
 ; AVX-NEXT:    je .LBB13_6
-; AVX-NEXT:  # %bb.5: # %cond.store3
-; AVX-NEXT:    vpextrw $4, %xmm0, 4(%rdi)
-; AVX-NEXT:  .LBB13_6: # %else4
-; AVX-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX-NEXT:    testb $1, %al
+; AVX-NEXT:  .LBB13_5: # %cond.store3
+; AVX-NEXT:    vpextrw $2, %xmm0, 4(%rdi)
+; AVX-NEXT:    testb $8, %al
 ; AVX-NEXT:    je .LBB13_8
-; AVX-NEXT:  # %bb.7: # %cond.store5
-; AVX-NEXT:    vpextrw $6, %xmm0, 6(%rdi)
-; AVX-NEXT:  .LBB13_8: # %else6
+; AVX-NEXT:  .LBB13_7: # %cond.store5
+; AVX-NEXT:    vpextrw $3, %xmm0, 6(%rdi)
 ; AVX-NEXT:    retq
 ;
 ; AVX512F-LABEL: truncstore_v4i32_v4i16:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
 ; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
 ; AVX512F-NEXT:    kmovw %k0, %eax
 ; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB13_2
-; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    jne .LBB13_1
+; AVX512F-NEXT:  # %bb.2: # %else
+; AVX512F-NEXT:    testb $2, %al
+; AVX512F-NEXT:    jne .LBB13_3
+; AVX512F-NEXT:  .LBB13_4: # %else2
+; AVX512F-NEXT:    testb $4, %al
+; AVX512F-NEXT:    jne .LBB13_5
+; AVX512F-NEXT:  .LBB13_6: # %else4
+; AVX512F-NEXT:    testb $8, %al
+; AVX512F-NEXT:    jne .LBB13_7
+; AVX512F-NEXT:  .LBB13_8: # %else6
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+; AVX512F-NEXT:  .LBB13_1: # %cond.store
 ; AVX512F-NEXT:    vpextrw $0, %xmm0, (%rdi)
-; AVX512F-NEXT:  .LBB13_2: # %else
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $2, %al
 ; AVX512F-NEXT:    je .LBB13_4
-; AVX512F-NEXT:  # %bb.3: # %cond.store1
-; AVX512F-NEXT:    vpextrw $2, %xmm0, 2(%rdi)
-; AVX512F-NEXT:  .LBB13_4: # %else2
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  .LBB13_3: # %cond.store1
+; AVX512F-NEXT:    vpextrw $1, %xmm0, 2(%rdi)
+; AVX512F-NEXT:    testb $4, %al
 ; AVX512F-NEXT:    je .LBB13_6
-; AVX512F-NEXT:  # %bb.5: # %cond.store3
-; AVX512F-NEXT:    vpextrw $4, %xmm0, 4(%rdi)
-; AVX512F-NEXT:  .LBB13_6: # %else4
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  .LBB13_5: # %cond.store3
+; AVX512F-NEXT:    vpextrw $2, %xmm0, 4(%rdi)
+; AVX512F-NEXT:    testb $8, %al
 ; AVX512F-NEXT:    je .LBB13_8
-; AVX512F-NEXT:  # %bb.7: # %cond.store5
-; AVX512F-NEXT:    vpextrw $6, %xmm0, 6(%rdi)
-; AVX512F-NEXT:  .LBB13_8: # %else6
+; AVX512F-NEXT:  .LBB13_7: # %cond.store5
+; AVX512F-NEXT:    vpextrw $3, %xmm0, 6(%rdi)
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
@@ -4476,9 +4245,9 @@ define void @truncstore_v4i32_v4i16(<4 x i32> %x, <4 x i16>* %p, <4 x i32> %mask
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
 ; AVX512BW-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
 ; AVX512BW-NEXT:    kshiftld $28, %k0, %k0
 ; AVX512BW-NEXT:    kshiftrd $28, %k0, %k1
+; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
 ; AVX512BW-NEXT:    vmovdqu16 %zmm0, (%rdi) {%k1}
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -4498,151 +4267,151 @@ define void @truncstore_v4i32_v4i8(<4 x i32> %x, <4 x i8>* %p, <4 x i32> %mask)
 ; SSE2-LABEL: truncstore_v4i32_v4i8:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    packuswb %xmm0, %xmm0
+; SSE2-NEXT:    packuswb %xmm0, %xmm0
 ; SSE2-NEXT:    pcmpeqd %xmm1, %xmm2
-; SSE2-NEXT:    movd %xmm2, %eax
-; SSE2-NEXT:    notl %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB14_2
-; SSE2-NEXT:  # %bb.1: # %cond.store
+; SSE2-NEXT:    movmskps %xmm2, %ecx
+; SSE2-NEXT:    xorl $15, %ecx
+; SSE2-NEXT:    testb $1, %cl
 ; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    jne .LBB14_1
+; SSE2-NEXT:  # %bb.2: # %else
+; SSE2-NEXT:    testb $2, %cl
+; SSE2-NEXT:    jne .LBB14_3
+; SSE2-NEXT:  .LBB14_4: # %else2
+; SSE2-NEXT:    testb $4, %cl
+; SSE2-NEXT:    jne .LBB14_5
+; SSE2-NEXT:  .LBB14_6: # %else4
+; SSE2-NEXT:    testb $8, %cl
+; SSE2-NEXT:    jne .LBB14_7
+; SSE2-NEXT:  .LBB14_8: # %else6
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  .LBB14_1: # %cond.store
 ; SSE2-NEXT:    movb %al, (%rdi)
-; SSE2-NEXT:  .LBB14_2: # %else
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm3
-; SSE2-NEXT:    pxor %xmm3, %xmm2
-; SSE2-NEXT:    pextrw $2, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $2, %cl
 ; SSE2-NEXT:    je .LBB14_4
-; SSE2-NEXT:  # %bb.3: # %cond.store1
-; SSE2-NEXT:    pextrw $2, %xmm0, %eax
-; SSE2-NEXT:    movb %al, 1(%rdi)
-; SSE2-NEXT:  .LBB14_4: # %else2
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm1
-; SSE2-NEXT:    pxor %xmm1, %xmm3
-; SSE2-NEXT:    pextrw $4, %xmm3, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:  .LBB14_3: # %cond.store1
+; SSE2-NEXT:    movb %ah, 1(%rdi)
+; SSE2-NEXT:    testb $4, %cl
 ; SSE2-NEXT:    je .LBB14_6
-; SSE2-NEXT:  # %bb.5: # %cond.store3
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    movb %al, 2(%rdi)
-; SSE2-NEXT:  .LBB14_6: # %else4
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT:    pxor %xmm2, %xmm1
-; SSE2-NEXT:    pextrw $6, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:  .LBB14_5: # %cond.store3
+; SSE2-NEXT:    movl %eax, %edx
+; SSE2-NEXT:    shrl $16, %edx
+; SSE2-NEXT:    movb %dl, 2(%rdi)
+; SSE2-NEXT:    testb $8, %cl
 ; SSE2-NEXT:    je .LBB14_8
-; SSE2-NEXT:  # %bb.7: # %cond.store5
-; SSE2-NEXT:    pextrw $6, %xmm0, %eax
+; SSE2-NEXT:  .LBB14_7: # %cond.store5
+; SSE2-NEXT:    shrl $24, %eax
 ; SSE2-NEXT:    movb %al, 3(%rdi)
-; SSE2-NEXT:  .LBB14_8: # %else6
 ; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: truncstore_v4i32_v4i8:
 ; SSE4:       # %bb.0:
-; SSE4-NEXT:    pxor %xmm3, %xmm3
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm3
-; SSE4-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE4-NEXT:    pxor %xmm3, %xmm2
-; SSE4-NEXT:    pextrb $0, %xmm2, %eax
+; SSE4-NEXT:    pxor %xmm2, %xmm2
+; SSE4-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; SSE4-NEXT:    pcmpeqd %xmm1, %xmm2
+; SSE4-NEXT:    movmskps %xmm2, %eax
+; SSE4-NEXT:    xorl $15, %eax
 ; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB14_2
-; SSE4-NEXT:  # %bb.1: # %cond.store
+; SSE4-NEXT:    jne .LBB14_1
+; SSE4-NEXT:  # %bb.2: # %else
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    jne .LBB14_3
+; SSE4-NEXT:  .LBB14_4: # %else2
+; SSE4-NEXT:    testb $4, %al
+; SSE4-NEXT:    jne .LBB14_5
+; SSE4-NEXT:  .LBB14_6: # %else4
+; SSE4-NEXT:    testb $8, %al
+; SSE4-NEXT:    jne .LBB14_7
+; SSE4-NEXT:  .LBB14_8: # %else6
+; SSE4-NEXT:    retq
+; SSE4-NEXT:  .LBB14_1: # %cond.store
 ; SSE4-NEXT:    pextrb $0, %xmm0, (%rdi)
-; SSE4-NEXT:  .LBB14_2: # %else
-; SSE4-NEXT:    pextrb $4, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $2, %al
 ; SSE4-NEXT:    je .LBB14_4
-; SSE4-NEXT:  # %bb.3: # %cond.store1
-; SSE4-NEXT:    pextrb $4, %xmm0, 1(%rdi)
-; SSE4-NEXT:  .LBB14_4: # %else2
-; SSE4-NEXT:    pxor %xmm2, %xmm2
-; SSE4-NEXT:    pcmpeqd %xmm2, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE4-NEXT:    pxor %xmm1, %xmm2
-; SSE4-NEXT:    pextrb $8, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:  .LBB14_3: # %cond.store1
+; SSE4-NEXT:    pextrb $1, %xmm0, 1(%rdi)
+; SSE4-NEXT:    testb $4, %al
 ; SSE4-NEXT:    je .LBB14_6
-; SSE4-NEXT:  # %bb.5: # %cond.store3
-; SSE4-NEXT:    pextrb $8, %xmm0, 2(%rdi)
-; SSE4-NEXT:  .LBB14_6: # %else4
-; SSE4-NEXT:    pextrb $12, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:  .LBB14_5: # %cond.store3
+; SSE4-NEXT:    pextrb $2, %xmm0, 2(%rdi)
+; SSE4-NEXT:    testb $8, %al
 ; SSE4-NEXT:    je .LBB14_8
-; SSE4-NEXT:  # %bb.7: # %cond.store5
-; SSE4-NEXT:    pextrb $12, %xmm0, 3(%rdi)
-; SSE4-NEXT:  .LBB14_8: # %else6
+; SSE4-NEXT:  .LBB14_7: # %cond.store5
+; SSE4-NEXT:    pextrb $3, %xmm0, 3(%rdi)
 ; SSE4-NEXT:    retq
 ;
 ; AVX-LABEL: truncstore_v4i32_v4i8:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX-NEXT:    vpextrb $0, %xmm2, %eax
+; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
+; AVX-NEXT:    vmovmskps %xmm1, %eax
+; AVX-NEXT:    xorl $15, %eax
 ; AVX-NEXT:    testb $1, %al
-; AVX-NEXT:    je .LBB14_2
-; AVX-NEXT:  # %bb.1: # %cond.store
+; AVX-NEXT:    jne .LBB14_1
+; AVX-NEXT:  # %bb.2: # %else
+; AVX-NEXT:    testb $2, %al
+; AVX-NEXT:    jne .LBB14_3
+; AVX-NEXT:  .LBB14_4: # %else2
+; AVX-NEXT:    testb $4, %al
+; AVX-NEXT:    jne .LBB14_5
+; AVX-NEXT:  .LBB14_6: # %else4
+; AVX-NEXT:    testb $8, %al
+; AVX-NEXT:    jne .LBB14_7
+; AVX-NEXT:  .LBB14_8: # %else6
+; AVX-NEXT:    retq
+; AVX-NEXT:  .LBB14_1: # %cond.store
 ; AVX-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX-NEXT:  .LBB14_2: # %else
-; AVX-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX-NEXT:    testb $1, %al
+; AVX-NEXT:    testb $2, %al
 ; AVX-NEXT:    je .LBB14_4
-; AVX-NEXT:  # %bb.3: # %cond.store1
-; AVX-NEXT:    vpextrb $4, %xmm0, 1(%rdi)
-; AVX-NEXT:  .LBB14_4: # %else2
-; AVX-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX-NEXT:    testb $1, %al
+; AVX-NEXT:  .LBB14_3: # %cond.store1
+; AVX-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
+; AVX-NEXT:    testb $4, %al
 ; AVX-NEXT:    je .LBB14_6
-; AVX-NEXT:  # %bb.5: # %cond.store3
-; AVX-NEXT:    vpextrb $8, %xmm0, 2(%rdi)
-; AVX-NEXT:  .LBB14_6: # %else4
-; AVX-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX-NEXT:    testb $1, %al
+; AVX-NEXT:  .LBB14_5: # %cond.store3
+; AVX-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
+; AVX-NEXT:    testb $8, %al
 ; AVX-NEXT:    je .LBB14_8
-; AVX-NEXT:  # %bb.7: # %cond.store5
-; AVX-NEXT:    vpextrb $12, %xmm0, 3(%rdi)
-; AVX-NEXT:  .LBB14_8: # %else6
+; AVX-NEXT:  .LBB14_7: # %cond.store5
+; AVX-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
 ; AVX-NEXT:    retq
 ;
 ; AVX512F-LABEL: truncstore_v4i32_v4i8:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
 ; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
 ; AVX512F-NEXT:    kmovw %k0, %eax
 ; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB14_2
-; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    jne .LBB14_1
+; AVX512F-NEXT:  # %bb.2: # %else
+; AVX512F-NEXT:    testb $2, %al
+; AVX512F-NEXT:    jne .LBB14_3
+; AVX512F-NEXT:  .LBB14_4: # %else2
+; AVX512F-NEXT:    testb $4, %al
+; AVX512F-NEXT:    jne .LBB14_5
+; AVX512F-NEXT:  .LBB14_6: # %else4
+; AVX512F-NEXT:    testb $8, %al
+; AVX512F-NEXT:    jne .LBB14_7
+; AVX512F-NEXT:  .LBB14_8: # %else6
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+; AVX512F-NEXT:  .LBB14_1: # %cond.store
 ; AVX512F-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX512F-NEXT:  .LBB14_2: # %else
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $2, %al
 ; AVX512F-NEXT:    je .LBB14_4
-; AVX512F-NEXT:  # %bb.3: # %cond.store1
-; AVX512F-NEXT:    vpextrb $4, %xmm0, 1(%rdi)
-; AVX512F-NEXT:  .LBB14_4: # %else2
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  .LBB14_3: # %cond.store1
+; AVX512F-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
+; AVX512F-NEXT:    testb $4, %al
 ; AVX512F-NEXT:    je .LBB14_6
-; AVX512F-NEXT:  # %bb.5: # %cond.store3
-; AVX512F-NEXT:    vpextrb $8, %xmm0, 2(%rdi)
-; AVX512F-NEXT:  .LBB14_6: # %else4
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  .LBB14_5: # %cond.store3
+; AVX512F-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
+; AVX512F-NEXT:    testb $8, %al
 ; AVX512F-NEXT:    je .LBB14_8
-; AVX512F-NEXT:  # %bb.7: # %cond.store5
-; AVX512F-NEXT:    vpextrb $12, %xmm0, 3(%rdi)
-; AVX512F-NEXT:  .LBB14_8: # %else6
+; AVX512F-NEXT:  .LBB14_7: # %cond.store5
+; AVX512F-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
@@ -4650,9 +4419,9 @@ define void @truncstore_v4i32_v4i8(<4 x i32> %x, <4 x i8>* %p, <4 x i32> %mask)
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
 ; AVX512BW-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
 ; AVX512BW-NEXT:    kshiftlq $60, %k0, %k0
 ; AVX512BW-NEXT:    kshiftrq $60, %k0, %k1
+; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
 ; AVX512BW-NEXT:    vmovdqu8 %zmm0, (%rdi) {%k1}
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -4672,1515 +4441,1183 @@ define void @truncstore_v32i16_v32i8(<32 x i16> %x, <32 x i8>* %p, <32 x i8> %ma
 ; SSE2-LABEL: truncstore_v32i16_v32i8:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pxor %xmm7, %xmm7
-; SSE2-NEXT:    pcmpeqb %xmm4, %xmm7
-; SSE2-NEXT:    movdqa {{.*#+}} xmm6 = [255,255,255,255,255,255,255,255]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm6 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
 ; SSE2-NEXT:    pand %xmm6, %xmm1
 ; SSE2-NEXT:    pand %xmm6, %xmm0
 ; SSE2-NEXT:    packuswb %xmm1, %xmm0
-; SSE2-NEXT:    movd %xmm7, %ecx
-; SSE2-NEXT:    movl %ecx, %eax
-; SSE2-NEXT:    notb %al
+; SSE2-NEXT:    pcmpeqb %xmm7, %xmm4
+; SSE2-NEXT:    pmovmskb %xmm4, %ecx
+; SSE2-NEXT:    xorl $65535, %ecx # imm = 0xFFFF
+; SSE2-NEXT:    pcmpeqb %xmm7, %xmm5
+; SSE2-NEXT:    pmovmskb %xmm5, %eax
+; SSE2-NEXT:    notl %eax
+; SSE2-NEXT:    shll $16, %eax
+; SSE2-NEXT:    orl %ecx, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    je .LBB15_2
-; SSE2-NEXT:  # %bb.1: # %cond.store
-; SSE2-NEXT:    movb %al, (%rdi)
-; SSE2-NEXT:  .LBB15_2: # %else
-; SSE2-NEXT:    shrl $8, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    je .LBB15_4
-; SSE2-NEXT:  # %bb.3: # %cond.store1
-; SSE2-NEXT:    movb %ah, 1(%rdi)
+; SSE2-NEXT:    movd %xmm0, %ecx
+; SSE2-NEXT:    jne .LBB15_1
+; SSE2-NEXT:  # %bb.2: # %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne .LBB15_3
 ; SSE2-NEXT:  .LBB15_4: # %else2
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqb %xmm4, %xmm1
-; SSE2-NEXT:    movd %xmm1, %ecx
-; SSE2-NEXT:    movl %ecx, %edx
-; SSE2-NEXT:    shrl $16, %edx
-; SSE2-NEXT:    notb %dl
-; SSE2-NEXT:    testb $1, %dl
-; SSE2-NEXT:    je .LBB15_6
-; SSE2-NEXT:  # %bb.5: # %cond.store3
-; SSE2-NEXT:    movl %eax, %edx
-; SSE2-NEXT:    shrl $16, %edx
-; SSE2-NEXT:    movb %dl, 2(%rdi)
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne .LBB15_5
 ; SSE2-NEXT:  .LBB15_6: # %else4
-; SSE2-NEXT:    shrl $24, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je .LBB15_8
-; SSE2-NEXT:  # %bb.7: # %cond.store5
-; SSE2-NEXT:    shrl $24, %eax
-; SSE2-NEXT:    movb %al, 3(%rdi)
+; SSE2-NEXT:  .LBB15_7: # %cond.store5
+; SSE2-NEXT:    shrl $24, %ecx
+; SSE2-NEXT:    movb %cl, 3(%rdi)
 ; SSE2-NEXT:  .LBB15_8: # %else6
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqb %xmm4, %xmm1
-; SSE2-NEXT:    pextrw $2, %xmm1, %eax
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testb $16, %al
 ; SSE2-NEXT:    pextrw $2, %xmm0, %ecx
 ; SSE2-NEXT:    je .LBB15_10
 ; SSE2-NEXT:  # %bb.9: # %cond.store7
 ; SSE2-NEXT:    movb %cl, 4(%rdi)
 ; SSE2-NEXT:  .LBB15_10: # %else8
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    notb %al
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $32, %al
 ; SSE2-NEXT:    je .LBB15_12
 ; SSE2-NEXT:  # %bb.11: # %cond.store9
 ; SSE2-NEXT:    movb %ch, 5(%rdi)
 ; SSE2-NEXT:  .LBB15_12: # %else10
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqb %xmm4, %xmm1
-; SSE2-NEXT:    pextrw $3, %xmm1, %eax
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testb $64, %al
 ; SSE2-NEXT:    pextrw $3, %xmm0, %ecx
 ; SSE2-NEXT:    je .LBB15_14
 ; SSE2-NEXT:  # %bb.13: # %cond.store11
 ; SSE2-NEXT:    movb %cl, 6(%rdi)
 ; SSE2-NEXT:  .LBB15_14: # %else12
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    notb %al
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $-128, %al
 ; SSE2-NEXT:    je .LBB15_16
 ; SSE2-NEXT:  # %bb.15: # %cond.store13
 ; SSE2-NEXT:    movb %ch, 7(%rdi)
 ; SSE2-NEXT:  .LBB15_16: # %else14
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqb %xmm4, %xmm1
-; SSE2-NEXT:    pextrw $4, %xmm1, %eax
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $256, %eax # imm = 0x100
 ; SSE2-NEXT:    pextrw $4, %xmm0, %ecx
 ; SSE2-NEXT:    je .LBB15_18
 ; SSE2-NEXT:  # %bb.17: # %cond.store15
 ; SSE2-NEXT:    movb %cl, 8(%rdi)
 ; SSE2-NEXT:  .LBB15_18: # %else16
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    notb %al
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $512, %eax # imm = 0x200
 ; SSE2-NEXT:    je .LBB15_20
 ; SSE2-NEXT:  # %bb.19: # %cond.store17
 ; SSE2-NEXT:    movb %ch, 9(%rdi)
 ; SSE2-NEXT:  .LBB15_20: # %else18
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqb %xmm4, %xmm1
-; SSE2-NEXT:    pextrw $5, %xmm1, %eax
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $1024, %eax # imm = 0x400
 ; SSE2-NEXT:    pextrw $5, %xmm0, %ecx
 ; SSE2-NEXT:    je .LBB15_22
 ; SSE2-NEXT:  # %bb.21: # %cond.store19
 ; SSE2-NEXT:    movb %cl, 10(%rdi)
 ; SSE2-NEXT:  .LBB15_22: # %else20
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    notb %al
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $2048, %eax # imm = 0x800
 ; SSE2-NEXT:    je .LBB15_24
 ; SSE2-NEXT:  # %bb.23: # %cond.store21
 ; SSE2-NEXT:    movb %ch, 11(%rdi)
 ; SSE2-NEXT:  .LBB15_24: # %else22
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqb %xmm4, %xmm1
-; SSE2-NEXT:    pextrw $6, %xmm1, %eax
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $4096, %eax # imm = 0x1000
 ; SSE2-NEXT:    pextrw $6, %xmm0, %ecx
 ; SSE2-NEXT:    je .LBB15_26
 ; SSE2-NEXT:  # %bb.25: # %cond.store23
 ; SSE2-NEXT:    movb %cl, 12(%rdi)
 ; SSE2-NEXT:  .LBB15_26: # %else24
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    notb %al
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $8192, %eax # imm = 0x2000
 ; SSE2-NEXT:    je .LBB15_28
 ; SSE2-NEXT:  # %bb.27: # %cond.store25
 ; SSE2-NEXT:    movb %ch, 13(%rdi)
 ; SSE2-NEXT:  .LBB15_28: # %else26
 ; SSE2-NEXT:    pand %xmm6, %xmm3
 ; SSE2-NEXT:    pand %xmm6, %xmm2
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqb %xmm1, %xmm4
-; SSE2-NEXT:    pextrw $7, %xmm4, %eax
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $16384, %eax # imm = 0x4000
 ; SSE2-NEXT:    pextrw $7, %xmm0, %ecx
 ; SSE2-NEXT:    je .LBB15_30
 ; SSE2-NEXT:  # %bb.29: # %cond.store27
 ; SSE2-NEXT:    movb %cl, 14(%rdi)
 ; SSE2-NEXT:  .LBB15_30: # %else28
 ; SSE2-NEXT:    packuswb %xmm3, %xmm2
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    notb %al
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $32768, %eax # imm = 0x8000
 ; SSE2-NEXT:    je .LBB15_32
 ; SSE2-NEXT:  # %bb.31: # %cond.store29
 ; SSE2-NEXT:    movb %ch, 15(%rdi)
 ; SSE2-NEXT:  .LBB15_32: # %else30
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqb %xmm5, %xmm0
-; SSE2-NEXT:    movd %xmm0, %ecx
-; SSE2-NEXT:    movl %ecx, %eax
-; SSE2-NEXT:    notb %al
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    movd %xmm2, %eax
-; SSE2-NEXT:    je .LBB15_34
-; SSE2-NEXT:  # %bb.33: # %cond.store31
-; SSE2-NEXT:    movb %al, 16(%rdi)
-; SSE2-NEXT:  .LBB15_34: # %else32
-; SSE2-NEXT:    shrl $8, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    je .LBB15_36
-; SSE2-NEXT:  # %bb.35: # %cond.store33
-; SSE2-NEXT:    movb %ah, 17(%rdi)
+; SSE2-NEXT:    testl $65536, %eax # imm = 0x10000
+; SSE2-NEXT:    movd %xmm2, %ecx
+; SSE2-NEXT:    jne .LBB15_33
+; SSE2-NEXT:  # %bb.34: # %else32
+; SSE2-NEXT:    testl $131072, %eax # imm = 0x20000
+; SSE2-NEXT:    jne .LBB15_35
 ; SSE2-NEXT:  .LBB15_36: # %else34
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqb %xmm5, %xmm0
-; SSE2-NEXT:    movd %xmm0, %ecx
-; SSE2-NEXT:    movl %ecx, %edx
-; SSE2-NEXT:    shrl $16, %edx
-; SSE2-NEXT:    notb %dl
-; SSE2-NEXT:    testb $1, %dl
-; SSE2-NEXT:    je .LBB15_38
-; SSE2-NEXT:  # %bb.37: # %cond.store35
-; SSE2-NEXT:    movl %eax, %edx
-; SSE2-NEXT:    shrl $16, %edx
-; SSE2-NEXT:    movb %dl, 18(%rdi)
+; SSE2-NEXT:    testl $262144, %eax # imm = 0x40000
+; SSE2-NEXT:    jne .LBB15_37
 ; SSE2-NEXT:  .LBB15_38: # %else36
-; SSE2-NEXT:    shrl $24, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $524288, %eax # imm = 0x80000
 ; SSE2-NEXT:    je .LBB15_40
-; SSE2-NEXT:  # %bb.39: # %cond.store37
-; SSE2-NEXT:    shrl $24, %eax
-; SSE2-NEXT:    movb %al, 19(%rdi)
+; SSE2-NEXT:  .LBB15_39: # %cond.store37
+; SSE2-NEXT:    shrl $24, %ecx
+; SSE2-NEXT:    movb %cl, 19(%rdi)
 ; SSE2-NEXT:  .LBB15_40: # %else38
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqb %xmm5, %xmm0
-; SSE2-NEXT:    pextrw $2, %xmm0, %eax
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $1048576, %eax # imm = 0x100000
 ; SSE2-NEXT:    pextrw $2, %xmm2, %ecx
 ; SSE2-NEXT:    je .LBB15_42
 ; SSE2-NEXT:  # %bb.41: # %cond.store39
 ; SSE2-NEXT:    movb %cl, 20(%rdi)
 ; SSE2-NEXT:  .LBB15_42: # %else40
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    notb %al
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $2097152, %eax # imm = 0x200000
 ; SSE2-NEXT:    je .LBB15_44
 ; SSE2-NEXT:  # %bb.43: # %cond.store41
 ; SSE2-NEXT:    movb %ch, 21(%rdi)
 ; SSE2-NEXT:  .LBB15_44: # %else42
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqb %xmm5, %xmm0
-; SSE2-NEXT:    pextrw $3, %xmm0, %eax
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $4194304, %eax # imm = 0x400000
 ; SSE2-NEXT:    pextrw $3, %xmm2, %ecx
 ; SSE2-NEXT:    je .LBB15_46
 ; SSE2-NEXT:  # %bb.45: # %cond.store43
 ; SSE2-NEXT:    movb %cl, 22(%rdi)
 ; SSE2-NEXT:  .LBB15_46: # %else44
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    notb %al
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $8388608, %eax # imm = 0x800000
 ; SSE2-NEXT:    je .LBB15_48
 ; SSE2-NEXT:  # %bb.47: # %cond.store45
 ; SSE2-NEXT:    movb %ch, 23(%rdi)
 ; SSE2-NEXT:  .LBB15_48: # %else46
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqb %xmm5, %xmm0
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $16777216, %eax # imm = 0x1000000
 ; SSE2-NEXT:    pextrw $4, %xmm2, %ecx
 ; SSE2-NEXT:    je .LBB15_50
 ; SSE2-NEXT:  # %bb.49: # %cond.store47
 ; SSE2-NEXT:    movb %cl, 24(%rdi)
 ; SSE2-NEXT:  .LBB15_50: # %else48
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    notb %al
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $33554432, %eax # imm = 0x2000000
 ; SSE2-NEXT:    je .LBB15_52
 ; SSE2-NEXT:  # %bb.51: # %cond.store49
 ; SSE2-NEXT:    movb %ch, 25(%rdi)
 ; SSE2-NEXT:  .LBB15_52: # %else50
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqb %xmm5, %xmm0
-; SSE2-NEXT:    pextrw $5, %xmm0, %eax
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $67108864, %eax # imm = 0x4000000
 ; SSE2-NEXT:    pextrw $5, %xmm2, %ecx
 ; SSE2-NEXT:    je .LBB15_54
 ; SSE2-NEXT:  # %bb.53: # %cond.store51
 ; SSE2-NEXT:    movb %cl, 26(%rdi)
 ; SSE2-NEXT:  .LBB15_54: # %else52
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    notb %al
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $134217728, %eax # imm = 0x8000000
 ; SSE2-NEXT:    je .LBB15_56
 ; SSE2-NEXT:  # %bb.55: # %cond.store53
 ; SSE2-NEXT:    movb %ch, 27(%rdi)
 ; SSE2-NEXT:  .LBB15_56: # %else54
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqb %xmm5, %xmm0
-; SSE2-NEXT:    pextrw $6, %xmm0, %eax
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $268435456, %eax # imm = 0x10000000
 ; SSE2-NEXT:    pextrw $6, %xmm2, %ecx
 ; SSE2-NEXT:    je .LBB15_58
 ; SSE2-NEXT:  # %bb.57: # %cond.store55
 ; SSE2-NEXT:    movb %cl, 28(%rdi)
 ; SSE2-NEXT:  .LBB15_58: # %else56
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    notb %al
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $536870912, %eax # imm = 0x20000000
 ; SSE2-NEXT:    je .LBB15_60
 ; SSE2-NEXT:  # %bb.59: # %cond.store57
 ; SSE2-NEXT:    movb %ch, 29(%rdi)
 ; SSE2-NEXT:  .LBB15_60: # %else58
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqb %xmm0, %xmm5
-; SSE2-NEXT:    pextrw $7, %xmm5, %eax
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $1073741824, %eax # imm = 0x40000000
 ; SSE2-NEXT:    pextrw $7, %xmm2, %ecx
-; SSE2-NEXT:    je .LBB15_62
-; SSE2-NEXT:  # %bb.61: # %cond.store59
+; SSE2-NEXT:    jne .LBB15_61
+; SSE2-NEXT:  # %bb.62: # %else60
+; SSE2-NEXT:    testl $-2147483648, %eax # imm = 0x80000000
+; SSE2-NEXT:    jne .LBB15_63
+; SSE2-NEXT:  .LBB15_64: # %else62
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  .LBB15_1: # %cond.store
+; SSE2-NEXT:    movb %cl, (%rdi)
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    je .LBB15_4
+; SSE2-NEXT:  .LBB15_3: # %cond.store1
+; SSE2-NEXT:    movb %ch, 1(%rdi)
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    je .LBB15_6
+; SSE2-NEXT:  .LBB15_5: # %cond.store3
+; SSE2-NEXT:    movl %ecx, %edx
+; SSE2-NEXT:    shrl $16, %edx
+; SSE2-NEXT:    movb %dl, 2(%rdi)
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne .LBB15_7
+; SSE2-NEXT:    jmp .LBB15_8
+; SSE2-NEXT:  .LBB15_33: # %cond.store31
+; SSE2-NEXT:    movb %cl, 16(%rdi)
+; SSE2-NEXT:    testl $131072, %eax # imm = 0x20000
+; SSE2-NEXT:    je .LBB15_36
+; SSE2-NEXT:  .LBB15_35: # %cond.store33
+; SSE2-NEXT:    movb %ch, 17(%rdi)
+; SSE2-NEXT:    testl $262144, %eax # imm = 0x40000
+; SSE2-NEXT:    je .LBB15_38
+; SSE2-NEXT:  .LBB15_37: # %cond.store35
+; SSE2-NEXT:    movl %ecx, %edx
+; SSE2-NEXT:    shrl $16, %edx
+; SSE2-NEXT:    movb %dl, 18(%rdi)
+; SSE2-NEXT:    testl $524288, %eax # imm = 0x80000
+; SSE2-NEXT:    jne .LBB15_39
+; SSE2-NEXT:    jmp .LBB15_40
+; SSE2-NEXT:  .LBB15_61: # %cond.store59
 ; SSE2-NEXT:    movb %cl, 30(%rdi)
-; SSE2-NEXT:  .LBB15_62: # %else60
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    notb %al
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $-2147483648, %eax # imm = 0x80000000
 ; SSE2-NEXT:    je .LBB15_64
-; SSE2-NEXT:  # %bb.63: # %cond.store61
+; SSE2-NEXT:  .LBB15_63: # %cond.store61
 ; SSE2-NEXT:    movb %ch, 31(%rdi)
-; SSE2-NEXT:  .LBB15_64: # %else62
 ; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: truncstore_v32i16_v32i8:
 ; SSE4:       # %bb.0:
 ; SSE4-NEXT:    pxor %xmm7, %xmm7
-; SSE4-NEXT:    pcmpeqb %xmm4, %xmm7
 ; SSE4-NEXT:    movdqa {{.*#+}} xmm6 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
 ; SSE4-NEXT:    pshufb %xmm6, %xmm1
 ; SSE4-NEXT:    pshufb %xmm6, %xmm0
 ; SSE4-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSE4-NEXT:    pextrb $0, %xmm7, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB15_2
-; SSE4-NEXT:  # %bb.1: # %cond.store
+; SSE4-NEXT:    pcmpeqb %xmm7, %xmm4
+; SSE4-NEXT:    pmovmskb %xmm4, %ecx
+; SSE4-NEXT:    xorl $65535, %ecx # imm = 0xFFFF
+; SSE4-NEXT:    pcmpeqb %xmm7, %xmm5
+; SSE4-NEXT:    pmovmskb %xmm5, %eax
+; SSE4-NEXT:    notl %eax
+; SSE4-NEXT:    shll $16, %eax
+; SSE4-NEXT:    orl %ecx, %eax
+; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    jne .LBB15_1
+; SSE4-NEXT:  # %bb.2: # %else
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    jne .LBB15_3
+; SSE4-NEXT:  .LBB15_4: # %else2
+; SSE4-NEXT:    testb $4, %al
+; SSE4-NEXT:    jne .LBB15_5
+; SSE4-NEXT:  .LBB15_6: # %else4
+; SSE4-NEXT:    testb $8, %al
+; SSE4-NEXT:    jne .LBB15_7
+; SSE4-NEXT:  .LBB15_8: # %else6
+; SSE4-NEXT:    testb $16, %al
+; SSE4-NEXT:    jne .LBB15_9
+; SSE4-NEXT:  .LBB15_10: # %else8
+; SSE4-NEXT:    testb $32, %al
+; SSE4-NEXT:    jne .LBB15_11
+; SSE4-NEXT:  .LBB15_12: # %else10
+; SSE4-NEXT:    testb $64, %al
+; SSE4-NEXT:    jne .LBB15_13
+; SSE4-NEXT:  .LBB15_14: # %else12
+; SSE4-NEXT:    testb $-128, %al
+; SSE4-NEXT:    jne .LBB15_15
+; SSE4-NEXT:  .LBB15_16: # %else14
+; SSE4-NEXT:    testl $256, %eax # imm = 0x100
+; SSE4-NEXT:    jne .LBB15_17
+; SSE4-NEXT:  .LBB15_18: # %else16
+; SSE4-NEXT:    testl $512, %eax # imm = 0x200
+; SSE4-NEXT:    jne .LBB15_19
+; SSE4-NEXT:  .LBB15_20: # %else18
+; SSE4-NEXT:    testl $1024, %eax # imm = 0x400
+; SSE4-NEXT:    jne .LBB15_21
+; SSE4-NEXT:  .LBB15_22: # %else20
+; SSE4-NEXT:    testl $2048, %eax # imm = 0x800
+; SSE4-NEXT:    jne .LBB15_23
+; SSE4-NEXT:  .LBB15_24: # %else22
+; SSE4-NEXT:    testl $4096, %eax # imm = 0x1000
+; SSE4-NEXT:    jne .LBB15_25
+; SSE4-NEXT:  .LBB15_26: # %else24
+; SSE4-NEXT:    testl $8192, %eax # imm = 0x2000
+; SSE4-NEXT:    jne .LBB15_27
+; SSE4-NEXT:  .LBB15_28: # %else26
+; SSE4-NEXT:    testl $16384, %eax # imm = 0x4000
+; SSE4-NEXT:    je .LBB15_30
+; SSE4-NEXT:  .LBB15_29: # %cond.store27
+; SSE4-NEXT:    pextrb $14, %xmm0, 14(%rdi)
+; SSE4-NEXT:  .LBB15_30: # %else28
+; SSE4-NEXT:    pshufb %xmm6, %xmm3
+; SSE4-NEXT:    pshufb %xmm6, %xmm2
+; SSE4-NEXT:    testl $32768, %eax # imm = 0x8000
+; SSE4-NEXT:    je .LBB15_32
+; SSE4-NEXT:  # %bb.31: # %cond.store29
+; SSE4-NEXT:    pextrb $15, %xmm0, 15(%rdi)
+; SSE4-NEXT:  .LBB15_32: # %else30
+; SSE4-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; SSE4-NEXT:    testl $65536, %eax # imm = 0x10000
+; SSE4-NEXT:    jne .LBB15_33
+; SSE4-NEXT:  # %bb.34: # %else32
+; SSE4-NEXT:    testl $131072, %eax # imm = 0x20000
+; SSE4-NEXT:    jne .LBB15_35
+; SSE4-NEXT:  .LBB15_36: # %else34
+; SSE4-NEXT:    testl $262144, %eax # imm = 0x40000
+; SSE4-NEXT:    jne .LBB15_37
+; SSE4-NEXT:  .LBB15_38: # %else36
+; SSE4-NEXT:    testl $524288, %eax # imm = 0x80000
+; SSE4-NEXT:    jne .LBB15_39
+; SSE4-NEXT:  .LBB15_40: # %else38
+; SSE4-NEXT:    testl $1048576, %eax # imm = 0x100000
+; SSE4-NEXT:    jne .LBB15_41
+; SSE4-NEXT:  .LBB15_42: # %else40
+; SSE4-NEXT:    testl $2097152, %eax # imm = 0x200000
+; SSE4-NEXT:    jne .LBB15_43
+; SSE4-NEXT:  .LBB15_44: # %else42
+; SSE4-NEXT:    testl $4194304, %eax # imm = 0x400000
+; SSE4-NEXT:    jne .LBB15_45
+; SSE4-NEXT:  .LBB15_46: # %else44
+; SSE4-NEXT:    testl $8388608, %eax # imm = 0x800000
+; SSE4-NEXT:    jne .LBB15_47
+; SSE4-NEXT:  .LBB15_48: # %else46
+; SSE4-NEXT:    testl $16777216, %eax # imm = 0x1000000
+; SSE4-NEXT:    jne .LBB15_49
+; SSE4-NEXT:  .LBB15_50: # %else48
+; SSE4-NEXT:    testl $33554432, %eax # imm = 0x2000000
+; SSE4-NEXT:    jne .LBB15_51
+; SSE4-NEXT:  .LBB15_52: # %else50
+; SSE4-NEXT:    testl $67108864, %eax # imm = 0x4000000
+; SSE4-NEXT:    jne .LBB15_53
+; SSE4-NEXT:  .LBB15_54: # %else52
+; SSE4-NEXT:    testl $134217728, %eax # imm = 0x8000000
+; SSE4-NEXT:    jne .LBB15_55
+; SSE4-NEXT:  .LBB15_56: # %else54
+; SSE4-NEXT:    testl $268435456, %eax # imm = 0x10000000
+; SSE4-NEXT:    jne .LBB15_57
+; SSE4-NEXT:  .LBB15_58: # %else56
+; SSE4-NEXT:    testl $536870912, %eax # imm = 0x20000000
+; SSE4-NEXT:    jne .LBB15_59
+; SSE4-NEXT:  .LBB15_60: # %else58
+; SSE4-NEXT:    testl $1073741824, %eax # imm = 0x40000000
+; SSE4-NEXT:    jne .LBB15_61
+; SSE4-NEXT:  .LBB15_62: # %else60
+; SSE4-NEXT:    testl $-2147483648, %eax # imm = 0x80000000
+; SSE4-NEXT:    jne .LBB15_63
+; SSE4-NEXT:  .LBB15_64: # %else62
+; SSE4-NEXT:    retq
+; SSE4-NEXT:  .LBB15_1: # %cond.store
 ; SSE4-NEXT:    pextrb $0, %xmm0, (%rdi)
-; SSE4-NEXT:  .LBB15_2: # %else
-; SSE4-NEXT:    pextrb $1, %xmm7, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $2, %al
 ; SSE4-NEXT:    je .LBB15_4
-; SSE4-NEXT:  # %bb.3: # %cond.store1
+; SSE4-NEXT:  .LBB15_3: # %cond.store1
 ; SSE4-NEXT:    pextrb $1, %xmm0, 1(%rdi)
-; SSE4-NEXT:  .LBB15_4: # %else2
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqb %xmm4, %xmm1
-; SSE4-NEXT:    pextrb $2, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $4, %al
 ; SSE4-NEXT:    je .LBB15_6
-; SSE4-NEXT:  # %bb.5: # %cond.store3
+; SSE4-NEXT:  .LBB15_5: # %cond.store3
 ; SSE4-NEXT:    pextrb $2, %xmm0, 2(%rdi)
-; SSE4-NEXT:  .LBB15_6: # %else4
-; SSE4-NEXT:    pextrb $3, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $8, %al
 ; SSE4-NEXT:    je .LBB15_8
-; SSE4-NEXT:  # %bb.7: # %cond.store5
+; SSE4-NEXT:  .LBB15_7: # %cond.store5
 ; SSE4-NEXT:    pextrb $3, %xmm0, 3(%rdi)
-; SSE4-NEXT:  .LBB15_8: # %else6
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqb %xmm4, %xmm1
-; SSE4-NEXT:    pextrb $4, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $16, %al
 ; SSE4-NEXT:    je .LBB15_10
-; SSE4-NEXT:  # %bb.9: # %cond.store7
+; SSE4-NEXT:  .LBB15_9: # %cond.store7
 ; SSE4-NEXT:    pextrb $4, %xmm0, 4(%rdi)
-; SSE4-NEXT:  .LBB15_10: # %else8
-; SSE4-NEXT:    pextrb $5, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $32, %al
 ; SSE4-NEXT:    je .LBB15_12
-; SSE4-NEXT:  # %bb.11: # %cond.store9
+; SSE4-NEXT:  .LBB15_11: # %cond.store9
 ; SSE4-NEXT:    pextrb $5, %xmm0, 5(%rdi)
-; SSE4-NEXT:  .LBB15_12: # %else10
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqb %xmm4, %xmm1
-; SSE4-NEXT:    pextrb $6, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $64, %al
 ; SSE4-NEXT:    je .LBB15_14
-; SSE4-NEXT:  # %bb.13: # %cond.store11
+; SSE4-NEXT:  .LBB15_13: # %cond.store11
 ; SSE4-NEXT:    pextrb $6, %xmm0, 6(%rdi)
-; SSE4-NEXT:  .LBB15_14: # %else12
-; SSE4-NEXT:    pextrb $7, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $-128, %al
 ; SSE4-NEXT:    je .LBB15_16
-; SSE4-NEXT:  # %bb.15: # %cond.store13
+; SSE4-NEXT:  .LBB15_15: # %cond.store13
 ; SSE4-NEXT:    pextrb $7, %xmm0, 7(%rdi)
-; SSE4-NEXT:  .LBB15_16: # %else14
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqb %xmm4, %xmm1
-; SSE4-NEXT:    pextrb $8, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $256, %eax # imm = 0x100
 ; SSE4-NEXT:    je .LBB15_18
-; SSE4-NEXT:  # %bb.17: # %cond.store15
+; SSE4-NEXT:  .LBB15_17: # %cond.store15
 ; SSE4-NEXT:    pextrb $8, %xmm0, 8(%rdi)
-; SSE4-NEXT:  .LBB15_18: # %else16
-; SSE4-NEXT:    pextrb $9, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $512, %eax # imm = 0x200
 ; SSE4-NEXT:    je .LBB15_20
-; SSE4-NEXT:  # %bb.19: # %cond.store17
+; SSE4-NEXT:  .LBB15_19: # %cond.store17
 ; SSE4-NEXT:    pextrb $9, %xmm0, 9(%rdi)
-; SSE4-NEXT:  .LBB15_20: # %else18
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqb %xmm4, %xmm1
-; SSE4-NEXT:    pextrb $10, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $1024, %eax # imm = 0x400
 ; SSE4-NEXT:    je .LBB15_22
-; SSE4-NEXT:  # %bb.21: # %cond.store19
+; SSE4-NEXT:  .LBB15_21: # %cond.store19
 ; SSE4-NEXT:    pextrb $10, %xmm0, 10(%rdi)
-; SSE4-NEXT:  .LBB15_22: # %else20
-; SSE4-NEXT:    pextrb $11, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $2048, %eax # imm = 0x800
 ; SSE4-NEXT:    je .LBB15_24
-; SSE4-NEXT:  # %bb.23: # %cond.store21
+; SSE4-NEXT:  .LBB15_23: # %cond.store21
 ; SSE4-NEXT:    pextrb $11, %xmm0, 11(%rdi)
-; SSE4-NEXT:  .LBB15_24: # %else22
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqb %xmm4, %xmm1
-; SSE4-NEXT:    pextrb $12, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $4096, %eax # imm = 0x1000
 ; SSE4-NEXT:    je .LBB15_26
-; SSE4-NEXT:  # %bb.25: # %cond.store23
+; SSE4-NEXT:  .LBB15_25: # %cond.store23
 ; SSE4-NEXT:    pextrb $12, %xmm0, 12(%rdi)
-; SSE4-NEXT:  .LBB15_26: # %else24
-; SSE4-NEXT:    pextrb $13, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $8192, %eax # imm = 0x2000
 ; SSE4-NEXT:    je .LBB15_28
-; SSE4-NEXT:  # %bb.27: # %cond.store25
+; SSE4-NEXT:  .LBB15_27: # %cond.store25
 ; SSE4-NEXT:    pextrb $13, %xmm0, 13(%rdi)
-; SSE4-NEXT:  .LBB15_28: # %else26
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqb %xmm1, %xmm4
-; SSE4-NEXT:    pextrb $14, %xmm4, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB15_30
-; SSE4-NEXT:  # %bb.29: # %cond.store27
-; SSE4-NEXT:    pextrb $14, %xmm0, 14(%rdi)
-; SSE4-NEXT:  .LBB15_30: # %else28
-; SSE4-NEXT:    pshufb %xmm6, %xmm3
-; SSE4-NEXT:    pshufb %xmm6, %xmm2
-; SSE4-NEXT:    pextrb $15, %xmm4, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB15_32
-; SSE4-NEXT:  # %bb.31: # %cond.store29
-; SSE4-NEXT:    pextrb $15, %xmm0, 15(%rdi)
-; SSE4-NEXT:  .LBB15_32: # %else30
-; SSE4-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqb %xmm5, %xmm0
-; SSE4-NEXT:    pextrb $0, %xmm0, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB15_34
-; SSE4-NEXT:  # %bb.33: # %cond.store31
+; SSE4-NEXT:    testl $16384, %eax # imm = 0x4000
+; SSE4-NEXT:    jne .LBB15_29
+; SSE4-NEXT:    jmp .LBB15_30
+; SSE4-NEXT:  .LBB15_33: # %cond.store31
 ; SSE4-NEXT:    pextrb $0, %xmm2, 16(%rdi)
-; SSE4-NEXT:  .LBB15_34: # %else32
-; SSE4-NEXT:    pextrb $1, %xmm0, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $131072, %eax # imm = 0x20000
 ; SSE4-NEXT:    je .LBB15_36
-; SSE4-NEXT:  # %bb.35: # %cond.store33
+; SSE4-NEXT:  .LBB15_35: # %cond.store33
 ; SSE4-NEXT:    pextrb $1, %xmm2, 17(%rdi)
-; SSE4-NEXT:  .LBB15_36: # %else34
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqb %xmm5, %xmm0
-; SSE4-NEXT:    pextrb $2, %xmm0, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $262144, %eax # imm = 0x40000
 ; SSE4-NEXT:    je .LBB15_38
-; SSE4-NEXT:  # %bb.37: # %cond.store35
+; SSE4-NEXT:  .LBB15_37: # %cond.store35
 ; SSE4-NEXT:    pextrb $2, %xmm2, 18(%rdi)
-; SSE4-NEXT:  .LBB15_38: # %else36
-; SSE4-NEXT:    pextrb $3, %xmm0, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $524288, %eax # imm = 0x80000
 ; SSE4-NEXT:    je .LBB15_40
-; SSE4-NEXT:  # %bb.39: # %cond.store37
+; SSE4-NEXT:  .LBB15_39: # %cond.store37
 ; SSE4-NEXT:    pextrb $3, %xmm2, 19(%rdi)
-; SSE4-NEXT:  .LBB15_40: # %else38
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqb %xmm5, %xmm0
-; SSE4-NEXT:    pextrb $4, %xmm0, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $1048576, %eax # imm = 0x100000
 ; SSE4-NEXT:    je .LBB15_42
-; SSE4-NEXT:  # %bb.41: # %cond.store39
+; SSE4-NEXT:  .LBB15_41: # %cond.store39
 ; SSE4-NEXT:    pextrb $4, %xmm2, 20(%rdi)
-; SSE4-NEXT:  .LBB15_42: # %else40
-; SSE4-NEXT:    pextrb $5, %xmm0, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $2097152, %eax # imm = 0x200000
 ; SSE4-NEXT:    je .LBB15_44
-; SSE4-NEXT:  # %bb.43: # %cond.store41
+; SSE4-NEXT:  .LBB15_43: # %cond.store41
 ; SSE4-NEXT:    pextrb $5, %xmm2, 21(%rdi)
-; SSE4-NEXT:  .LBB15_44: # %else42
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqb %xmm5, %xmm0
-; SSE4-NEXT:    pextrb $6, %xmm0, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $4194304, %eax # imm = 0x400000
 ; SSE4-NEXT:    je .LBB15_46
-; SSE4-NEXT:  # %bb.45: # %cond.store43
+; SSE4-NEXT:  .LBB15_45: # %cond.store43
 ; SSE4-NEXT:    pextrb $6, %xmm2, 22(%rdi)
-; SSE4-NEXT:  .LBB15_46: # %else44
-; SSE4-NEXT:    pextrb $7, %xmm0, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $8388608, %eax # imm = 0x800000
 ; SSE4-NEXT:    je .LBB15_48
-; SSE4-NEXT:  # %bb.47: # %cond.store45
+; SSE4-NEXT:  .LBB15_47: # %cond.store45
 ; SSE4-NEXT:    pextrb $7, %xmm2, 23(%rdi)
-; SSE4-NEXT:  .LBB15_48: # %else46
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqb %xmm5, %xmm0
-; SSE4-NEXT:    pextrb $8, %xmm0, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $16777216, %eax # imm = 0x1000000
 ; SSE4-NEXT:    je .LBB15_50
-; SSE4-NEXT:  # %bb.49: # %cond.store47
+; SSE4-NEXT:  .LBB15_49: # %cond.store47
 ; SSE4-NEXT:    pextrb $8, %xmm2, 24(%rdi)
-; SSE4-NEXT:  .LBB15_50: # %else48
-; SSE4-NEXT:    pextrb $9, %xmm0, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $33554432, %eax # imm = 0x2000000
 ; SSE4-NEXT:    je .LBB15_52
-; SSE4-NEXT:  # %bb.51: # %cond.store49
+; SSE4-NEXT:  .LBB15_51: # %cond.store49
 ; SSE4-NEXT:    pextrb $9, %xmm2, 25(%rdi)
-; SSE4-NEXT:  .LBB15_52: # %else50
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqb %xmm5, %xmm0
-; SSE4-NEXT:    pextrb $10, %xmm0, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $67108864, %eax # imm = 0x4000000
 ; SSE4-NEXT:    je .LBB15_54
-; SSE4-NEXT:  # %bb.53: # %cond.store51
+; SSE4-NEXT:  .LBB15_53: # %cond.store51
 ; SSE4-NEXT:    pextrb $10, %xmm2, 26(%rdi)
-; SSE4-NEXT:  .LBB15_54: # %else52
-; SSE4-NEXT:    pextrb $11, %xmm0, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $134217728, %eax # imm = 0x8000000
 ; SSE4-NEXT:    je .LBB15_56
-; SSE4-NEXT:  # %bb.55: # %cond.store53
+; SSE4-NEXT:  .LBB15_55: # %cond.store53
 ; SSE4-NEXT:    pextrb $11, %xmm2, 27(%rdi)
-; SSE4-NEXT:  .LBB15_56: # %else54
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqb %xmm5, %xmm0
-; SSE4-NEXT:    pextrb $12, %xmm0, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $268435456, %eax # imm = 0x10000000
 ; SSE4-NEXT:    je .LBB15_58
-; SSE4-NEXT:  # %bb.57: # %cond.store55
+; SSE4-NEXT:  .LBB15_57: # %cond.store55
 ; SSE4-NEXT:    pextrb $12, %xmm2, 28(%rdi)
-; SSE4-NEXT:  .LBB15_58: # %else56
-; SSE4-NEXT:    pextrb $13, %xmm0, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $536870912, %eax # imm = 0x20000000
 ; SSE4-NEXT:    je .LBB15_60
-; SSE4-NEXT:  # %bb.59: # %cond.store57
+; SSE4-NEXT:  .LBB15_59: # %cond.store57
 ; SSE4-NEXT:    pextrb $13, %xmm2, 29(%rdi)
-; SSE4-NEXT:  .LBB15_60: # %else58
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqb %xmm0, %xmm5
-; SSE4-NEXT:    pextrb $14, %xmm5, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $1073741824, %eax # imm = 0x40000000
 ; SSE4-NEXT:    je .LBB15_62
-; SSE4-NEXT:  # %bb.61: # %cond.store59
+; SSE4-NEXT:  .LBB15_61: # %cond.store59
 ; SSE4-NEXT:    pextrb $14, %xmm2, 30(%rdi)
-; SSE4-NEXT:  .LBB15_62: # %else60
-; SSE4-NEXT:    pextrb $15, %xmm5, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $-2147483648, %eax # imm = 0x80000000
 ; SSE4-NEXT:    je .LBB15_64
-; SSE4-NEXT:  # %bb.63: # %cond.store61
+; SSE4-NEXT:  .LBB15_63: # %cond.store61
 ; SSE4-NEXT:    pextrb $15, %xmm2, 31(%rdi)
-; SSE4-NEXT:  .LBB15_64: # %else62
 ; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: truncstore_v32i16_v32i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpeqb %xmm3, %xmm2, %xmm5
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX1-NEXT:    vmovaps {{.*#+}} ymm6 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX1-NEXT:    vandps %ymm6, %ymm1, %ymm1
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm7
-; AVX1-NEXT:    vpackuswb %xmm7, %xmm1, %xmm1
-; AVX1-NEXT:    vandps %ymm6, %ymm0, %ymm0
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm6
-; AVX1-NEXT:    vpackuswb %xmm6, %xmm0, %xmm0
+; AVX1-NEXT:    vmovaps {{.*#+}} ymm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; AVX1-NEXT:    vandps %ymm3, %ymm1, %ymm1
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm4
+; AVX1-NEXT:    vpackuswb %xmm4, %xmm1, %xmm1
+; AVX1-NEXT:    vandps %ymm3, %ymm0, %ymm0
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; AVX1-NEXT:    vpackuswb %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX1-NEXT:    vpextrb $0, %xmm5, %eax
+; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm3
+; AVX1-NEXT:    vpmovmskb %xmm3, %ecx
+; AVX1-NEXT:    xorl $65535, %ecx # imm = 0xFFFF
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm2
+; AVX1-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vpmovmskb %xmm1, %eax
+; AVX1-NEXT:    notl %eax
+; AVX1-NEXT:    shll $16, %eax
+; AVX1-NEXT:    orl %ecx, %eax
 ; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je .LBB15_2
-; AVX1-NEXT:  # %bb.1: # %cond.store
+; AVX1-NEXT:    jne .LBB15_1
+; AVX1-NEXT:  # %bb.2: # %else
+; AVX1-NEXT:    testb $2, %al
+; AVX1-NEXT:    jne .LBB15_3
+; AVX1-NEXT:  .LBB15_4: # %else2
+; AVX1-NEXT:    testb $4, %al
+; AVX1-NEXT:    jne .LBB15_5
+; AVX1-NEXT:  .LBB15_6: # %else4
+; AVX1-NEXT:    testb $8, %al
+; AVX1-NEXT:    jne .LBB15_7
+; AVX1-NEXT:  .LBB15_8: # %else6
+; AVX1-NEXT:    testb $16, %al
+; AVX1-NEXT:    jne .LBB15_9
+; AVX1-NEXT:  .LBB15_10: # %else8
+; AVX1-NEXT:    testb $32, %al
+; AVX1-NEXT:    jne .LBB15_11
+; AVX1-NEXT:  .LBB15_12: # %else10
+; AVX1-NEXT:    testb $64, %al
+; AVX1-NEXT:    jne .LBB15_13
+; AVX1-NEXT:  .LBB15_14: # %else12
+; AVX1-NEXT:    testb $-128, %al
+; AVX1-NEXT:    jne .LBB15_15
+; AVX1-NEXT:  .LBB15_16: # %else14
+; AVX1-NEXT:    testl $256, %eax # imm = 0x100
+; AVX1-NEXT:    jne .LBB15_17
+; AVX1-NEXT:  .LBB15_18: # %else16
+; AVX1-NEXT:    testl $512, %eax # imm = 0x200
+; AVX1-NEXT:    jne .LBB15_19
+; AVX1-NEXT:  .LBB15_20: # %else18
+; AVX1-NEXT:    testl $1024, %eax # imm = 0x400
+; AVX1-NEXT:    jne .LBB15_21
+; AVX1-NEXT:  .LBB15_22: # %else20
+; AVX1-NEXT:    testl $2048, %eax # imm = 0x800
+; AVX1-NEXT:    jne .LBB15_23
+; AVX1-NEXT:  .LBB15_24: # %else22
+; AVX1-NEXT:    testl $4096, %eax # imm = 0x1000
+; AVX1-NEXT:    jne .LBB15_25
+; AVX1-NEXT:  .LBB15_26: # %else24
+; AVX1-NEXT:    testl $8192, %eax # imm = 0x2000
+; AVX1-NEXT:    jne .LBB15_27
+; AVX1-NEXT:  .LBB15_28: # %else26
+; AVX1-NEXT:    testl $16384, %eax # imm = 0x4000
+; AVX1-NEXT:    jne .LBB15_29
+; AVX1-NEXT:  .LBB15_30: # %else28
+; AVX1-NEXT:    testl $32768, %eax # imm = 0x8000
+; AVX1-NEXT:    je .LBB15_32
+; AVX1-NEXT:  .LBB15_31: # %cond.store29
+; AVX1-NEXT:    vpextrb $15, %xmm0, 15(%rdi)
+; AVX1-NEXT:  .LBB15_32: # %else30
+; AVX1-NEXT:    testl $65536, %eax # imm = 0x10000
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    jne .LBB15_33
+; AVX1-NEXT:  # %bb.34: # %else32
+; AVX1-NEXT:    testl $131072, %eax # imm = 0x20000
+; AVX1-NEXT:    jne .LBB15_35
+; AVX1-NEXT:  .LBB15_36: # %else34
+; AVX1-NEXT:    testl $262144, %eax # imm = 0x40000
+; AVX1-NEXT:    jne .LBB15_37
+; AVX1-NEXT:  .LBB15_38: # %else36
+; AVX1-NEXT:    testl $524288, %eax # imm = 0x80000
+; AVX1-NEXT:    jne .LBB15_39
+; AVX1-NEXT:  .LBB15_40: # %else38
+; AVX1-NEXT:    testl $1048576, %eax # imm = 0x100000
+; AVX1-NEXT:    jne .LBB15_41
+; AVX1-NEXT:  .LBB15_42: # %else40
+; AVX1-NEXT:    testl $2097152, %eax # imm = 0x200000
+; AVX1-NEXT:    jne .LBB15_43
+; AVX1-NEXT:  .LBB15_44: # %else42
+; AVX1-NEXT:    testl $4194304, %eax # imm = 0x400000
+; AVX1-NEXT:    jne .LBB15_45
+; AVX1-NEXT:  .LBB15_46: # %else44
+; AVX1-NEXT:    testl $8388608, %eax # imm = 0x800000
+; AVX1-NEXT:    jne .LBB15_47
+; AVX1-NEXT:  .LBB15_48: # %else46
+; AVX1-NEXT:    testl $16777216, %eax # imm = 0x1000000
+; AVX1-NEXT:    jne .LBB15_49
+; AVX1-NEXT:  .LBB15_50: # %else48
+; AVX1-NEXT:    testl $33554432, %eax # imm = 0x2000000
+; AVX1-NEXT:    jne .LBB15_51
+; AVX1-NEXT:  .LBB15_52: # %else50
+; AVX1-NEXT:    testl $67108864, %eax # imm = 0x4000000
+; AVX1-NEXT:    jne .LBB15_53
+; AVX1-NEXT:  .LBB15_54: # %else52
+; AVX1-NEXT:    testl $134217728, %eax # imm = 0x8000000
+; AVX1-NEXT:    jne .LBB15_55
+; AVX1-NEXT:  .LBB15_56: # %else54
+; AVX1-NEXT:    testl $268435456, %eax # imm = 0x10000000
+; AVX1-NEXT:    jne .LBB15_57
+; AVX1-NEXT:  .LBB15_58: # %else56
+; AVX1-NEXT:    testl $536870912, %eax # imm = 0x20000000
+; AVX1-NEXT:    jne .LBB15_59
+; AVX1-NEXT:  .LBB15_60: # %else58
+; AVX1-NEXT:    testl $1073741824, %eax # imm = 0x40000000
+; AVX1-NEXT:    jne .LBB15_61
+; AVX1-NEXT:  .LBB15_62: # %else60
+; AVX1-NEXT:    testl $-2147483648, %eax # imm = 0x80000000
+; AVX1-NEXT:    jne .LBB15_63
+; AVX1-NEXT:  .LBB15_64: # %else62
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+; AVX1-NEXT:  .LBB15_1: # %cond.store
 ; AVX1-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX1-NEXT:  .LBB15_2: # %else
-; AVX1-NEXT:    vpcmpeqb %xmm3, %xmm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm4, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $1, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $2, %al
 ; AVX1-NEXT:    je .LBB15_4
-; AVX1-NEXT:  # %bb.3: # %cond.store1
+; AVX1-NEXT:  .LBB15_3: # %cond.store1
 ; AVX1-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
-; AVX1-NEXT:  .LBB15_4: # %else2
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm4, %xmm4
-; AVX1-NEXT:    vpextrb $2, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $4, %al
 ; AVX1-NEXT:    je .LBB15_6
-; AVX1-NEXT:  # %bb.5: # %cond.store3
+; AVX1-NEXT:  .LBB15_5: # %cond.store3
 ; AVX1-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
-; AVX1-NEXT:  .LBB15_6: # %else4
-; AVX1-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $3, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $8, %al
 ; AVX1-NEXT:    je .LBB15_8
-; AVX1-NEXT:  # %bb.7: # %cond.store5
+; AVX1-NEXT:  .LBB15_7: # %cond.store5
 ; AVX1-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
-; AVX1-NEXT:  .LBB15_8: # %else6
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm4, %xmm4
-; AVX1-NEXT:    vpextrb $4, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $16, %al
 ; AVX1-NEXT:    je .LBB15_10
-; AVX1-NEXT:  # %bb.9: # %cond.store7
+; AVX1-NEXT:  .LBB15_9: # %cond.store7
 ; AVX1-NEXT:    vpextrb $4, %xmm0, 4(%rdi)
-; AVX1-NEXT:  .LBB15_10: # %else8
-; AVX1-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $5, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $32, %al
 ; AVX1-NEXT:    je .LBB15_12
-; AVX1-NEXT:  # %bb.11: # %cond.store9
+; AVX1-NEXT:  .LBB15_11: # %cond.store9
 ; AVX1-NEXT:    vpextrb $5, %xmm0, 5(%rdi)
-; AVX1-NEXT:  .LBB15_12: # %else10
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm4, %xmm4
-; AVX1-NEXT:    vpextrb $6, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $64, %al
 ; AVX1-NEXT:    je .LBB15_14
-; AVX1-NEXT:  # %bb.13: # %cond.store11
+; AVX1-NEXT:  .LBB15_13: # %cond.store11
 ; AVX1-NEXT:    vpextrb $6, %xmm0, 6(%rdi)
-; AVX1-NEXT:  .LBB15_14: # %else12
-; AVX1-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $7, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $-128, %al
 ; AVX1-NEXT:    je .LBB15_16
-; AVX1-NEXT:  # %bb.15: # %cond.store13
+; AVX1-NEXT:  .LBB15_15: # %cond.store13
 ; AVX1-NEXT:    vpextrb $7, %xmm0, 7(%rdi)
-; AVX1-NEXT:  .LBB15_16: # %else14
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm4, %xmm4
-; AVX1-NEXT:    vpextrb $8, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $256, %eax # imm = 0x100
 ; AVX1-NEXT:    je .LBB15_18
-; AVX1-NEXT:  # %bb.17: # %cond.store15
+; AVX1-NEXT:  .LBB15_17: # %cond.store15
 ; AVX1-NEXT:    vpextrb $8, %xmm0, 8(%rdi)
-; AVX1-NEXT:  .LBB15_18: # %else16
-; AVX1-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $9, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $512, %eax # imm = 0x200
 ; AVX1-NEXT:    je .LBB15_20
-; AVX1-NEXT:  # %bb.19: # %cond.store17
+; AVX1-NEXT:  .LBB15_19: # %cond.store17
 ; AVX1-NEXT:    vpextrb $9, %xmm0, 9(%rdi)
-; AVX1-NEXT:  .LBB15_20: # %else18
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm4, %xmm4
-; AVX1-NEXT:    vpextrb $10, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $1024, %eax # imm = 0x400
 ; AVX1-NEXT:    je .LBB15_22
-; AVX1-NEXT:  # %bb.21: # %cond.store19
+; AVX1-NEXT:  .LBB15_21: # %cond.store19
 ; AVX1-NEXT:    vpextrb $10, %xmm0, 10(%rdi)
-; AVX1-NEXT:  .LBB15_22: # %else20
-; AVX1-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $11, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $2048, %eax # imm = 0x800
 ; AVX1-NEXT:    je .LBB15_24
-; AVX1-NEXT:  # %bb.23: # %cond.store21
+; AVX1-NEXT:  .LBB15_23: # %cond.store21
 ; AVX1-NEXT:    vpextrb $11, %xmm0, 11(%rdi)
-; AVX1-NEXT:  .LBB15_24: # %else22
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm4, %xmm4
-; AVX1-NEXT:    vpextrb $12, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $4096, %eax # imm = 0x1000
 ; AVX1-NEXT:    je .LBB15_26
-; AVX1-NEXT:  # %bb.25: # %cond.store23
+; AVX1-NEXT:  .LBB15_25: # %cond.store23
 ; AVX1-NEXT:    vpextrb $12, %xmm0, 12(%rdi)
-; AVX1-NEXT:  .LBB15_26: # %else24
-; AVX1-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $13, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $8192, %eax # imm = 0x2000
 ; AVX1-NEXT:    je .LBB15_28
-; AVX1-NEXT:  # %bb.27: # %cond.store25
+; AVX1-NEXT:  .LBB15_27: # %cond.store25
 ; AVX1-NEXT:    vpextrb $13, %xmm0, 13(%rdi)
-; AVX1-NEXT:  .LBB15_28: # %else26
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm4, %xmm4
-; AVX1-NEXT:    vpextrb $14, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $16384, %eax # imm = 0x4000
 ; AVX1-NEXT:    je .LBB15_30
-; AVX1-NEXT:  # %bb.29: # %cond.store27
+; AVX1-NEXT:  .LBB15_29: # %cond.store27
 ; AVX1-NEXT:    vpextrb $14, %xmm0, 14(%rdi)
-; AVX1-NEXT:  .LBB15_30: # %else28
-; AVX1-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $15, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je .LBB15_32
-; AVX1-NEXT:  # %bb.31: # %cond.store29
-; AVX1-NEXT:    vpextrb $15, %xmm0, 15(%rdi)
-; AVX1-NEXT:  .LBB15_32: # %else30
-; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $0, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT:    je .LBB15_34
-; AVX1-NEXT:  # %bb.33: # %cond.store31
+; AVX1-NEXT:    testl $32768, %eax # imm = 0x8000
+; AVX1-NEXT:    jne .LBB15_31
+; AVX1-NEXT:    jmp .LBB15_32
+; AVX1-NEXT:  .LBB15_33: # %cond.store31
 ; AVX1-NEXT:    vpextrb $0, %xmm0, 16(%rdi)
-; AVX1-NEXT:  .LBB15_34: # %else32
-; AVX1-NEXT:    vpextrb $1, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $131072, %eax # imm = 0x20000
 ; AVX1-NEXT:    je .LBB15_36
-; AVX1-NEXT:  # %bb.35: # %cond.store33
+; AVX1-NEXT:  .LBB15_35: # %cond.store33
 ; AVX1-NEXT:    vpextrb $1, %xmm0, 17(%rdi)
-; AVX1-NEXT:  .LBB15_36: # %else34
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $2, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $262144, %eax # imm = 0x40000
 ; AVX1-NEXT:    je .LBB15_38
-; AVX1-NEXT:  # %bb.37: # %cond.store35
+; AVX1-NEXT:  .LBB15_37: # %cond.store35
 ; AVX1-NEXT:    vpextrb $2, %xmm0, 18(%rdi)
-; AVX1-NEXT:  .LBB15_38: # %else36
-; AVX1-NEXT:    vpextrb $3, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $524288, %eax # imm = 0x80000
 ; AVX1-NEXT:    je .LBB15_40
-; AVX1-NEXT:  # %bb.39: # %cond.store37
+; AVX1-NEXT:  .LBB15_39: # %cond.store37
 ; AVX1-NEXT:    vpextrb $3, %xmm0, 19(%rdi)
-; AVX1-NEXT:  .LBB15_40: # %else38
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $1048576, %eax # imm = 0x100000
 ; AVX1-NEXT:    je .LBB15_42
-; AVX1-NEXT:  # %bb.41: # %cond.store39
+; AVX1-NEXT:  .LBB15_41: # %cond.store39
 ; AVX1-NEXT:    vpextrb $4, %xmm0, 20(%rdi)
-; AVX1-NEXT:  .LBB15_42: # %else40
-; AVX1-NEXT:    vpextrb $5, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $2097152, %eax # imm = 0x200000
 ; AVX1-NEXT:    je .LBB15_44
-; AVX1-NEXT:  # %bb.43: # %cond.store41
+; AVX1-NEXT:  .LBB15_43: # %cond.store41
 ; AVX1-NEXT:    vpextrb $5, %xmm0, 21(%rdi)
-; AVX1-NEXT:  .LBB15_44: # %else42
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $6, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $4194304, %eax # imm = 0x400000
 ; AVX1-NEXT:    je .LBB15_46
-; AVX1-NEXT:  # %bb.45: # %cond.store43
+; AVX1-NEXT:  .LBB15_45: # %cond.store43
 ; AVX1-NEXT:    vpextrb $6, %xmm0, 22(%rdi)
-; AVX1-NEXT:  .LBB15_46: # %else44
-; AVX1-NEXT:    vpextrb $7, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $8388608, %eax # imm = 0x800000
 ; AVX1-NEXT:    je .LBB15_48
-; AVX1-NEXT:  # %bb.47: # %cond.store45
+; AVX1-NEXT:  .LBB15_47: # %cond.store45
 ; AVX1-NEXT:    vpextrb $7, %xmm0, 23(%rdi)
-; AVX1-NEXT:  .LBB15_48: # %else46
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $8, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $16777216, %eax # imm = 0x1000000
 ; AVX1-NEXT:    je .LBB15_50
-; AVX1-NEXT:  # %bb.49: # %cond.store47
+; AVX1-NEXT:  .LBB15_49: # %cond.store47
 ; AVX1-NEXT:    vpextrb $8, %xmm0, 24(%rdi)
-; AVX1-NEXT:  .LBB15_50: # %else48
-; AVX1-NEXT:    vpextrb $9, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $33554432, %eax # imm = 0x2000000
 ; AVX1-NEXT:    je .LBB15_52
-; AVX1-NEXT:  # %bb.51: # %cond.store49
+; AVX1-NEXT:  .LBB15_51: # %cond.store49
 ; AVX1-NEXT:    vpextrb $9, %xmm0, 25(%rdi)
-; AVX1-NEXT:  .LBB15_52: # %else50
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $10, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $67108864, %eax # imm = 0x4000000
 ; AVX1-NEXT:    je .LBB15_54
-; AVX1-NEXT:  # %bb.53: # %cond.store51
+; AVX1-NEXT:  .LBB15_53: # %cond.store51
 ; AVX1-NEXT:    vpextrb $10, %xmm0, 26(%rdi)
-; AVX1-NEXT:  .LBB15_54: # %else52
-; AVX1-NEXT:    vpextrb $11, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $134217728, %eax # imm = 0x8000000
 ; AVX1-NEXT:    je .LBB15_56
-; AVX1-NEXT:  # %bb.55: # %cond.store53
+; AVX1-NEXT:  .LBB15_55: # %cond.store53
 ; AVX1-NEXT:    vpextrb $11, %xmm0, 27(%rdi)
-; AVX1-NEXT:  .LBB15_56: # %else54
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $12, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $268435456, %eax # imm = 0x10000000
 ; AVX1-NEXT:    je .LBB15_58
-; AVX1-NEXT:  # %bb.57: # %cond.store55
+; AVX1-NEXT:  .LBB15_57: # %cond.store55
 ; AVX1-NEXT:    vpextrb $12, %xmm0, 28(%rdi)
-; AVX1-NEXT:  .LBB15_58: # %else56
-; AVX1-NEXT:    vpextrb $13, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $536870912, %eax # imm = 0x20000000
 ; AVX1-NEXT:    je .LBB15_60
-; AVX1-NEXT:  # %bb.59: # %cond.store57
+; AVX1-NEXT:  .LBB15_59: # %cond.store57
 ; AVX1-NEXT:    vpextrb $13, %xmm0, 29(%rdi)
-; AVX1-NEXT:  .LBB15_60: # %else58
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $14, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $1073741824, %eax # imm = 0x40000000
 ; AVX1-NEXT:    je .LBB15_62
-; AVX1-NEXT:  # %bb.61: # %cond.store59
+; AVX1-NEXT:  .LBB15_61: # %cond.store59
 ; AVX1-NEXT:    vpextrb $14, %xmm0, 30(%rdi)
-; AVX1-NEXT:  .LBB15_62: # %else60
-; AVX1-NEXT:    vpextrb $15, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $-2147483648, %eax # imm = 0x80000000
 ; AVX1-NEXT:    je .LBB15_64
-; AVX1-NEXT:  # %bb.63: # %cond.store61
+; AVX1-NEXT:  .LBB15_63: # %cond.store61
 ; AVX1-NEXT:    vpextrb $15, %xmm0, 31(%rdi)
-; AVX1-NEXT:  .LBB15_64: # %else62
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: truncstore_v32i16_v32i8:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vmovdqa {{.*#+}} ymm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX2-NEXT:    vpand %ymm3, %ymm0, %ymm0
-; AVX2-NEXT:    vpand %ymm3, %ymm1, %ymm1
-; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm3
+; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX2-NEXT:    vmovdqa {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; AVX2-NEXT:    vpand %ymm4, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %ymm4, %ymm1, %ymm1
+; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm4
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
 ; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
-; AVX2-NEXT:    vpackuswb %ymm0, %ymm3, %ymm0
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm3
-; AVX2-NEXT:    vpextrb $0, %xmm3, %eax
-; AVX2-NEXT:    notb %al
+; AVX2-NEXT:    vpackuswb %ymm0, %ymm4, %ymm0
+; AVX2-NEXT:    vpcmpeqb %ymm3, %ymm2, %ymm1
+; AVX2-NEXT:    vpmovmskb %ymm1, %eax
+; AVX2-NEXT:    notl %eax
 ; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je .LBB15_2
-; AVX2-NEXT:  # %bb.1: # %cond.store
+; AVX2-NEXT:    jne .LBB15_1
+; AVX2-NEXT:  # %bb.2: # %else
+; AVX2-NEXT:    testb $2, %al
+; AVX2-NEXT:    jne .LBB15_3
+; AVX2-NEXT:  .LBB15_4: # %else2
+; AVX2-NEXT:    testb $4, %al
+; AVX2-NEXT:    jne .LBB15_5
+; AVX2-NEXT:  .LBB15_6: # %else4
+; AVX2-NEXT:    testb $8, %al
+; AVX2-NEXT:    jne .LBB15_7
+; AVX2-NEXT:  .LBB15_8: # %else6
+; AVX2-NEXT:    testb $16, %al
+; AVX2-NEXT:    jne .LBB15_9
+; AVX2-NEXT:  .LBB15_10: # %else8
+; AVX2-NEXT:    testb $32, %al
+; AVX2-NEXT:    jne .LBB15_11
+; AVX2-NEXT:  .LBB15_12: # %else10
+; AVX2-NEXT:    testb $64, %al
+; AVX2-NEXT:    jne .LBB15_13
+; AVX2-NEXT:  .LBB15_14: # %else12
+; AVX2-NEXT:    testb $-128, %al
+; AVX2-NEXT:    jne .LBB15_15
+; AVX2-NEXT:  .LBB15_16: # %else14
+; AVX2-NEXT:    testl $256, %eax # imm = 0x100
+; AVX2-NEXT:    jne .LBB15_17
+; AVX2-NEXT:  .LBB15_18: # %else16
+; AVX2-NEXT:    testl $512, %eax # imm = 0x200
+; AVX2-NEXT:    jne .LBB15_19
+; AVX2-NEXT:  .LBB15_20: # %else18
+; AVX2-NEXT:    testl $1024, %eax # imm = 0x400
+; AVX2-NEXT:    jne .LBB15_21
+; AVX2-NEXT:  .LBB15_22: # %else20
+; AVX2-NEXT:    testl $2048, %eax # imm = 0x800
+; AVX2-NEXT:    jne .LBB15_23
+; AVX2-NEXT:  .LBB15_24: # %else22
+; AVX2-NEXT:    testl $4096, %eax # imm = 0x1000
+; AVX2-NEXT:    jne .LBB15_25
+; AVX2-NEXT:  .LBB15_26: # %else24
+; AVX2-NEXT:    testl $8192, %eax # imm = 0x2000
+; AVX2-NEXT:    jne .LBB15_27
+; AVX2-NEXT:  .LBB15_28: # %else26
+; AVX2-NEXT:    testl $16384, %eax # imm = 0x4000
+; AVX2-NEXT:    jne .LBB15_29
+; AVX2-NEXT:  .LBB15_30: # %else28
+; AVX2-NEXT:    testl $32768, %eax # imm = 0x8000
+; AVX2-NEXT:    je .LBB15_32
+; AVX2-NEXT:  .LBB15_31: # %cond.store29
+; AVX2-NEXT:    vpextrb $15, %xmm0, 15(%rdi)
+; AVX2-NEXT:  .LBB15_32: # %else30
+; AVX2-NEXT:    testl $65536, %eax # imm = 0x10000
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
+; AVX2-NEXT:    jne .LBB15_33
+; AVX2-NEXT:  # %bb.34: # %else32
+; AVX2-NEXT:    testl $131072, %eax # imm = 0x20000
+; AVX2-NEXT:    jne .LBB15_35
+; AVX2-NEXT:  .LBB15_36: # %else34
+; AVX2-NEXT:    testl $262144, %eax # imm = 0x40000
+; AVX2-NEXT:    jne .LBB15_37
+; AVX2-NEXT:  .LBB15_38: # %else36
+; AVX2-NEXT:    testl $524288, %eax # imm = 0x80000
+; AVX2-NEXT:    jne .LBB15_39
+; AVX2-NEXT:  .LBB15_40: # %else38
+; AVX2-NEXT:    testl $1048576, %eax # imm = 0x100000
+; AVX2-NEXT:    jne .LBB15_41
+; AVX2-NEXT:  .LBB15_42: # %else40
+; AVX2-NEXT:    testl $2097152, %eax # imm = 0x200000
+; AVX2-NEXT:    jne .LBB15_43
+; AVX2-NEXT:  .LBB15_44: # %else42
+; AVX2-NEXT:    testl $4194304, %eax # imm = 0x400000
+; AVX2-NEXT:    jne .LBB15_45
+; AVX2-NEXT:  .LBB15_46: # %else44
+; AVX2-NEXT:    testl $8388608, %eax # imm = 0x800000
+; AVX2-NEXT:    jne .LBB15_47
+; AVX2-NEXT:  .LBB15_48: # %else46
+; AVX2-NEXT:    testl $16777216, %eax # imm = 0x1000000
+; AVX2-NEXT:    jne .LBB15_49
+; AVX2-NEXT:  .LBB15_50: # %else48
+; AVX2-NEXT:    testl $33554432, %eax # imm = 0x2000000
+; AVX2-NEXT:    jne .LBB15_51
+; AVX2-NEXT:  .LBB15_52: # %else50
+; AVX2-NEXT:    testl $67108864, %eax # imm = 0x4000000
+; AVX2-NEXT:    jne .LBB15_53
+; AVX2-NEXT:  .LBB15_54: # %else52
+; AVX2-NEXT:    testl $134217728, %eax # imm = 0x8000000
+; AVX2-NEXT:    jne .LBB15_55
+; AVX2-NEXT:  .LBB15_56: # %else54
+; AVX2-NEXT:    testl $268435456, %eax # imm = 0x10000000
+; AVX2-NEXT:    jne .LBB15_57
+; AVX2-NEXT:  .LBB15_58: # %else56
+; AVX2-NEXT:    testl $536870912, %eax # imm = 0x20000000
+; AVX2-NEXT:    jne .LBB15_59
+; AVX2-NEXT:  .LBB15_60: # %else58
+; AVX2-NEXT:    testl $1073741824, %eax # imm = 0x40000000
+; AVX2-NEXT:    jne .LBB15_61
+; AVX2-NEXT:  .LBB15_62: # %else60
+; AVX2-NEXT:    testl $-2147483648, %eax # imm = 0x80000000
+; AVX2-NEXT:    jne .LBB15_63
+; AVX2-NEXT:  .LBB15_64: # %else62
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+; AVX2-NEXT:  .LBB15_1: # %cond.store
 ; AVX2-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX2-NEXT:  .LBB15_2: # %else
-; AVX2-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX2-NEXT:    vpextrb $1, %xmm1, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $2, %al
 ; AVX2-NEXT:    je .LBB15_4
-; AVX2-NEXT:  # %bb.3: # %cond.store1
+; AVX2-NEXT:  .LBB15_3: # %cond.store1
 ; AVX2-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
-; AVX2-NEXT:  .LBB15_4: # %else2
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm3
-; AVX2-NEXT:    vpextrb $2, %xmm3, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $4, %al
 ; AVX2-NEXT:    je .LBB15_6
-; AVX2-NEXT:  # %bb.5: # %cond.store3
+; AVX2-NEXT:  .LBB15_5: # %cond.store3
 ; AVX2-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
-; AVX2-NEXT:  .LBB15_6: # %else4
-; AVX2-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX2-NEXT:    vpextrb $3, %xmm1, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $8, %al
 ; AVX2-NEXT:    je .LBB15_8
-; AVX2-NEXT:  # %bb.7: # %cond.store5
+; AVX2-NEXT:  .LBB15_7: # %cond.store5
 ; AVX2-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
-; AVX2-NEXT:  .LBB15_8: # %else6
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm3
-; AVX2-NEXT:    vpextrb $4, %xmm3, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $16, %al
 ; AVX2-NEXT:    je .LBB15_10
-; AVX2-NEXT:  # %bb.9: # %cond.store7
+; AVX2-NEXT:  .LBB15_9: # %cond.store7
 ; AVX2-NEXT:    vpextrb $4, %xmm0, 4(%rdi)
-; AVX2-NEXT:  .LBB15_10: # %else8
-; AVX2-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX2-NEXT:    vpextrb $5, %xmm1, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $32, %al
 ; AVX2-NEXT:    je .LBB15_12
-; AVX2-NEXT:  # %bb.11: # %cond.store9
+; AVX2-NEXT:  .LBB15_11: # %cond.store9
 ; AVX2-NEXT:    vpextrb $5, %xmm0, 5(%rdi)
-; AVX2-NEXT:  .LBB15_12: # %else10
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm3
-; AVX2-NEXT:    vpextrb $6, %xmm3, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $64, %al
 ; AVX2-NEXT:    je .LBB15_14
-; AVX2-NEXT:  # %bb.13: # %cond.store11
+; AVX2-NEXT:  .LBB15_13: # %cond.store11
 ; AVX2-NEXT:    vpextrb $6, %xmm0, 6(%rdi)
-; AVX2-NEXT:  .LBB15_14: # %else12
-; AVX2-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX2-NEXT:    vpextrb $7, %xmm1, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $-128, %al
 ; AVX2-NEXT:    je .LBB15_16
-; AVX2-NEXT:  # %bb.15: # %cond.store13
+; AVX2-NEXT:  .LBB15_15: # %cond.store13
 ; AVX2-NEXT:    vpextrb $7, %xmm0, 7(%rdi)
-; AVX2-NEXT:  .LBB15_16: # %else14
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm3
-; AVX2-NEXT:    vpextrb $8, %xmm3, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $256, %eax # imm = 0x100
 ; AVX2-NEXT:    je .LBB15_18
-; AVX2-NEXT:  # %bb.17: # %cond.store15
+; AVX2-NEXT:  .LBB15_17: # %cond.store15
 ; AVX2-NEXT:    vpextrb $8, %xmm0, 8(%rdi)
-; AVX2-NEXT:  .LBB15_18: # %else16
-; AVX2-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX2-NEXT:    vpextrb $9, %xmm1, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $512, %eax # imm = 0x200
 ; AVX2-NEXT:    je .LBB15_20
-; AVX2-NEXT:  # %bb.19: # %cond.store17
+; AVX2-NEXT:  .LBB15_19: # %cond.store17
 ; AVX2-NEXT:    vpextrb $9, %xmm0, 9(%rdi)
-; AVX2-NEXT:  .LBB15_20: # %else18
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm3
-; AVX2-NEXT:    vpextrb $10, %xmm3, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $1024, %eax # imm = 0x400
 ; AVX2-NEXT:    je .LBB15_22
-; AVX2-NEXT:  # %bb.21: # %cond.store19
+; AVX2-NEXT:  .LBB15_21: # %cond.store19
 ; AVX2-NEXT:    vpextrb $10, %xmm0, 10(%rdi)
-; AVX2-NEXT:  .LBB15_22: # %else20
-; AVX2-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX2-NEXT:    vpextrb $11, %xmm1, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $2048, %eax # imm = 0x800
 ; AVX2-NEXT:    je .LBB15_24
-; AVX2-NEXT:  # %bb.23: # %cond.store21
+; AVX2-NEXT:  .LBB15_23: # %cond.store21
 ; AVX2-NEXT:    vpextrb $11, %xmm0, 11(%rdi)
-; AVX2-NEXT:  .LBB15_24: # %else22
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm3
-; AVX2-NEXT:    vpextrb $12, %xmm3, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $4096, %eax # imm = 0x1000
 ; AVX2-NEXT:    je .LBB15_26
-; AVX2-NEXT:  # %bb.25: # %cond.store23
+; AVX2-NEXT:  .LBB15_25: # %cond.store23
 ; AVX2-NEXT:    vpextrb $12, %xmm0, 12(%rdi)
-; AVX2-NEXT:  .LBB15_26: # %else24
-; AVX2-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX2-NEXT:    vpextrb $13, %xmm1, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $8192, %eax # imm = 0x2000
 ; AVX2-NEXT:    je .LBB15_28
-; AVX2-NEXT:  # %bb.27: # %cond.store25
+; AVX2-NEXT:  .LBB15_27: # %cond.store25
 ; AVX2-NEXT:    vpextrb $13, %xmm0, 13(%rdi)
-; AVX2-NEXT:  .LBB15_28: # %else26
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm3
-; AVX2-NEXT:    vpextrb $14, %xmm3, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $16384, %eax # imm = 0x4000
 ; AVX2-NEXT:    je .LBB15_30
-; AVX2-NEXT:  # %bb.29: # %cond.store27
+; AVX2-NEXT:  .LBB15_29: # %cond.store27
 ; AVX2-NEXT:    vpextrb $14, %xmm0, 14(%rdi)
-; AVX2-NEXT:  .LBB15_30: # %else28
-; AVX2-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX2-NEXT:    vpextrb $15, %xmm1, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je .LBB15_32
-; AVX2-NEXT:  # %bb.31: # %cond.store29
-; AVX2-NEXT:    vpextrb $15, %xmm0, 15(%rdi)
-; AVX2-NEXT:  .LBB15_32: # %else30
-; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm1
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpextrb $0, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; AVX2-NEXT:    je .LBB15_34
-; AVX2-NEXT:  # %bb.33: # %cond.store31
+; AVX2-NEXT:    testl $32768, %eax # imm = 0x8000
+; AVX2-NEXT:    jne .LBB15_31
+; AVX2-NEXT:    jmp .LBB15_32
+; AVX2-NEXT:  .LBB15_33: # %cond.store31
 ; AVX2-NEXT:    vpextrb $0, %xmm0, 16(%rdi)
-; AVX2-NEXT:  .LBB15_34: # %else32
-; AVX2-NEXT:    vpextrb $1, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $131072, %eax # imm = 0x20000
 ; AVX2-NEXT:    je .LBB15_36
-; AVX2-NEXT:  # %bb.35: # %cond.store33
+; AVX2-NEXT:  .LBB15_35: # %cond.store33
 ; AVX2-NEXT:    vpextrb $1, %xmm0, 17(%rdi)
-; AVX2-NEXT:  .LBB15_36: # %else34
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpextrb $2, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $262144, %eax # imm = 0x40000
 ; AVX2-NEXT:    je .LBB15_38
-; AVX2-NEXT:  # %bb.37: # %cond.store35
+; AVX2-NEXT:  .LBB15_37: # %cond.store35
 ; AVX2-NEXT:    vpextrb $2, %xmm0, 18(%rdi)
-; AVX2-NEXT:  .LBB15_38: # %else36
-; AVX2-NEXT:    vpextrb $3, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $524288, %eax # imm = 0x80000
 ; AVX2-NEXT:    je .LBB15_40
-; AVX2-NEXT:  # %bb.39: # %cond.store37
+; AVX2-NEXT:  .LBB15_39: # %cond.store37
 ; AVX2-NEXT:    vpextrb $3, %xmm0, 19(%rdi)
-; AVX2-NEXT:  .LBB15_40: # %else38
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $1048576, %eax # imm = 0x100000
 ; AVX2-NEXT:    je .LBB15_42
-; AVX2-NEXT:  # %bb.41: # %cond.store39
+; AVX2-NEXT:  .LBB15_41: # %cond.store39
 ; AVX2-NEXT:    vpextrb $4, %xmm0, 20(%rdi)
-; AVX2-NEXT:  .LBB15_42: # %else40
-; AVX2-NEXT:    vpextrb $5, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $2097152, %eax # imm = 0x200000
 ; AVX2-NEXT:    je .LBB15_44
-; AVX2-NEXT:  # %bb.43: # %cond.store41
+; AVX2-NEXT:  .LBB15_43: # %cond.store41
 ; AVX2-NEXT:    vpextrb $5, %xmm0, 21(%rdi)
-; AVX2-NEXT:  .LBB15_44: # %else42
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpextrb $6, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $4194304, %eax # imm = 0x400000
 ; AVX2-NEXT:    je .LBB15_46
-; AVX2-NEXT:  # %bb.45: # %cond.store43
+; AVX2-NEXT:  .LBB15_45: # %cond.store43
 ; AVX2-NEXT:    vpextrb $6, %xmm0, 22(%rdi)
-; AVX2-NEXT:  .LBB15_46: # %else44
-; AVX2-NEXT:    vpextrb $7, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $8388608, %eax # imm = 0x800000
 ; AVX2-NEXT:    je .LBB15_48
-; AVX2-NEXT:  # %bb.47: # %cond.store45
+; AVX2-NEXT:  .LBB15_47: # %cond.store45
 ; AVX2-NEXT:    vpextrb $7, %xmm0, 23(%rdi)
-; AVX2-NEXT:  .LBB15_48: # %else46
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpextrb $8, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $16777216, %eax # imm = 0x1000000
 ; AVX2-NEXT:    je .LBB15_50
-; AVX2-NEXT:  # %bb.49: # %cond.store47
+; AVX2-NEXT:  .LBB15_49: # %cond.store47
 ; AVX2-NEXT:    vpextrb $8, %xmm0, 24(%rdi)
-; AVX2-NEXT:  .LBB15_50: # %else48
-; AVX2-NEXT:    vpextrb $9, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $33554432, %eax # imm = 0x2000000
 ; AVX2-NEXT:    je .LBB15_52
-; AVX2-NEXT:  # %bb.51: # %cond.store49
+; AVX2-NEXT:  .LBB15_51: # %cond.store49
 ; AVX2-NEXT:    vpextrb $9, %xmm0, 25(%rdi)
-; AVX2-NEXT:  .LBB15_52: # %else50
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpextrb $10, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $67108864, %eax # imm = 0x4000000
 ; AVX2-NEXT:    je .LBB15_54
-; AVX2-NEXT:  # %bb.53: # %cond.store51
+; AVX2-NEXT:  .LBB15_53: # %cond.store51
 ; AVX2-NEXT:    vpextrb $10, %xmm0, 26(%rdi)
-; AVX2-NEXT:  .LBB15_54: # %else52
-; AVX2-NEXT:    vpextrb $11, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $134217728, %eax # imm = 0x8000000
 ; AVX2-NEXT:    je .LBB15_56
-; AVX2-NEXT:  # %bb.55: # %cond.store53
+; AVX2-NEXT:  .LBB15_55: # %cond.store53
 ; AVX2-NEXT:    vpextrb $11, %xmm0, 27(%rdi)
-; AVX2-NEXT:  .LBB15_56: # %else54
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpextrb $12, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $268435456, %eax # imm = 0x10000000
 ; AVX2-NEXT:    je .LBB15_58
-; AVX2-NEXT:  # %bb.57: # %cond.store55
+; AVX2-NEXT:  .LBB15_57: # %cond.store55
 ; AVX2-NEXT:    vpextrb $12, %xmm0, 28(%rdi)
-; AVX2-NEXT:  .LBB15_58: # %else56
-; AVX2-NEXT:    vpextrb $13, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $536870912, %eax # imm = 0x20000000
 ; AVX2-NEXT:    je .LBB15_60
-; AVX2-NEXT:  # %bb.59: # %cond.store57
+; AVX2-NEXT:  .LBB15_59: # %cond.store57
 ; AVX2-NEXT:    vpextrb $13, %xmm0, 29(%rdi)
-; AVX2-NEXT:  .LBB15_60: # %else58
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpextrb $14, %xmm1, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $1073741824, %eax # imm = 0x40000000
 ; AVX2-NEXT:    je .LBB15_62
-; AVX2-NEXT:  # %bb.61: # %cond.store59
+; AVX2-NEXT:  .LBB15_61: # %cond.store59
 ; AVX2-NEXT:    vpextrb $14, %xmm0, 30(%rdi)
-; AVX2-NEXT:  .LBB15_62: # %else60
-; AVX2-NEXT:    vpextrb $15, %xmm1, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $-2147483648, %eax # imm = 0x80000000
 ; AVX2-NEXT:    je .LBB15_64
-; AVX2-NEXT:  # %bb.63: # %cond.store61
+; AVX2-NEXT:  .LBB15_63: # %cond.store61
 ; AVX2-NEXT:    vpextrb $15, %xmm0, 31(%rdi)
-; AVX2-NEXT:  .LBB15_64: # %else62
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: truncstore_v32i16_v32i8:
 ; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
 ; AVX512F-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX512F-NEXT:    vpcmpeqb %xmm3, %xmm2, %xmm4
-; AVX512F-NEXT:    vpternlogq $15, %zmm4, %zmm4, %zmm4
-; AVX512F-NEXT:    vpmovsxbd %xmm4, %zmm4
-; AVX512F-NEXT:    vptestmd %zmm4, %zmm4, %k0
+; AVX512F-NEXT:    vpcmpeqb %ymm3, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
 ; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
-; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
-; AVX512F-NEXT:    vpmovdb %zmm1, %xmm1
-; AVX512F-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB15_2
-; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
+; AVX512F-NEXT:    vpmovdb %zmm2, %xmm2
+; AVX512F-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vpmovmskb %ymm1, %eax
+; AVX512F-NEXT:    notl %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    jne .LBB15_1
+; AVX512F-NEXT:  # %bb.2: # %else
+; AVX512F-NEXT:    testb $2, %al
+; AVX512F-NEXT:    jne .LBB15_3
+; AVX512F-NEXT:  .LBB15_4: # %else2
+; AVX512F-NEXT:    testb $4, %al
+; AVX512F-NEXT:    jne .LBB15_5
+; AVX512F-NEXT:  .LBB15_6: # %else4
+; AVX512F-NEXT:    testb $8, %al
+; AVX512F-NEXT:    jne .LBB15_7
+; AVX512F-NEXT:  .LBB15_8: # %else6
+; AVX512F-NEXT:    testb $16, %al
+; AVX512F-NEXT:    jne .LBB15_9
+; AVX512F-NEXT:  .LBB15_10: # %else8
+; AVX512F-NEXT:    testb $32, %al
+; AVX512F-NEXT:    jne .LBB15_11
+; AVX512F-NEXT:  .LBB15_12: # %else10
+; AVX512F-NEXT:    testb $64, %al
+; AVX512F-NEXT:    jne .LBB15_13
+; AVX512F-NEXT:  .LBB15_14: # %else12
+; AVX512F-NEXT:    testb $-128, %al
+; AVX512F-NEXT:    jne .LBB15_15
+; AVX512F-NEXT:  .LBB15_16: # %else14
+; AVX512F-NEXT:    testl $256, %eax # imm = 0x100
+; AVX512F-NEXT:    jne .LBB15_17
+; AVX512F-NEXT:  .LBB15_18: # %else16
+; AVX512F-NEXT:    testl $512, %eax # imm = 0x200
+; AVX512F-NEXT:    jne .LBB15_19
+; AVX512F-NEXT:  .LBB15_20: # %else18
+; AVX512F-NEXT:    testl $1024, %eax # imm = 0x400
+; AVX512F-NEXT:    jne .LBB15_21
+; AVX512F-NEXT:  .LBB15_22: # %else20
+; AVX512F-NEXT:    testl $2048, %eax # imm = 0x800
+; AVX512F-NEXT:    jne .LBB15_23
+; AVX512F-NEXT:  .LBB15_24: # %else22
+; AVX512F-NEXT:    testl $4096, %eax # imm = 0x1000
+; AVX512F-NEXT:    jne .LBB15_25
+; AVX512F-NEXT:  .LBB15_26: # %else24
+; AVX512F-NEXT:    testl $8192, %eax # imm = 0x2000
+; AVX512F-NEXT:    jne .LBB15_27
+; AVX512F-NEXT:  .LBB15_28: # %else26
+; AVX512F-NEXT:    testl $16384, %eax # imm = 0x4000
+; AVX512F-NEXT:    jne .LBB15_29
+; AVX512F-NEXT:  .LBB15_30: # %else28
+; AVX512F-NEXT:    testl $32768, %eax # imm = 0x8000
+; AVX512F-NEXT:    je .LBB15_32
+; AVX512F-NEXT:  .LBB15_31: # %cond.store29
+; AVX512F-NEXT:    vpextrb $15, %xmm0, 15(%rdi)
+; AVX512F-NEXT:  .LBB15_32: # %else30
+; AVX512F-NEXT:    testl $65536, %eax # imm = 0x10000
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm0
+; AVX512F-NEXT:    jne .LBB15_33
+; AVX512F-NEXT:  # %bb.34: # %else32
+; AVX512F-NEXT:    testl $131072, %eax # imm = 0x20000
+; AVX512F-NEXT:    jne .LBB15_35
+; AVX512F-NEXT:  .LBB15_36: # %else34
+; AVX512F-NEXT:    testl $262144, %eax # imm = 0x40000
+; AVX512F-NEXT:    jne .LBB15_37
+; AVX512F-NEXT:  .LBB15_38: # %else36
+; AVX512F-NEXT:    testl $524288, %eax # imm = 0x80000
+; AVX512F-NEXT:    jne .LBB15_39
+; AVX512F-NEXT:  .LBB15_40: # %else38
+; AVX512F-NEXT:    testl $1048576, %eax # imm = 0x100000
+; AVX512F-NEXT:    jne .LBB15_41
+; AVX512F-NEXT:  .LBB15_42: # %else40
+; AVX512F-NEXT:    testl $2097152, %eax # imm = 0x200000
+; AVX512F-NEXT:    jne .LBB15_43
+; AVX512F-NEXT:  .LBB15_44: # %else42
+; AVX512F-NEXT:    testl $4194304, %eax # imm = 0x400000
+; AVX512F-NEXT:    jne .LBB15_45
+; AVX512F-NEXT:  .LBB15_46: # %else44
+; AVX512F-NEXT:    testl $8388608, %eax # imm = 0x800000
+; AVX512F-NEXT:    jne .LBB15_47
+; AVX512F-NEXT:  .LBB15_48: # %else46
+; AVX512F-NEXT:    testl $16777216, %eax # imm = 0x1000000
+; AVX512F-NEXT:    jne .LBB15_49
+; AVX512F-NEXT:  .LBB15_50: # %else48
+; AVX512F-NEXT:    testl $33554432, %eax # imm = 0x2000000
+; AVX512F-NEXT:    jne .LBB15_51
+; AVX512F-NEXT:  .LBB15_52: # %else50
+; AVX512F-NEXT:    testl $67108864, %eax # imm = 0x4000000
+; AVX512F-NEXT:    jne .LBB15_53
+; AVX512F-NEXT:  .LBB15_54: # %else52
+; AVX512F-NEXT:    testl $134217728, %eax # imm = 0x8000000
+; AVX512F-NEXT:    jne .LBB15_55
+; AVX512F-NEXT:  .LBB15_56: # %else54
+; AVX512F-NEXT:    testl $268435456, %eax # imm = 0x10000000
+; AVX512F-NEXT:    jne .LBB15_57
+; AVX512F-NEXT:  .LBB15_58: # %else56
+; AVX512F-NEXT:    testl $536870912, %eax # imm = 0x20000000
+; AVX512F-NEXT:    jne .LBB15_59
+; AVX512F-NEXT:  .LBB15_60: # %else58
+; AVX512F-NEXT:    testl $1073741824, %eax # imm = 0x40000000
+; AVX512F-NEXT:    jne .LBB15_61
+; AVX512F-NEXT:  .LBB15_62: # %else60
+; AVX512F-NEXT:    testl $-2147483648, %eax # imm = 0x80000000
+; AVX512F-NEXT:    jne .LBB15_63
+; AVX512F-NEXT:  .LBB15_64: # %else62
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+; AVX512F-NEXT:  .LBB15_1: # %cond.store
 ; AVX512F-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX512F-NEXT:  .LBB15_2: # %else
-; AVX512F-NEXT:    vpcmpeqb %xmm3, %xmm2, %xmm1
-; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
-; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $2, %al
 ; AVX512F-NEXT:    je .LBB15_4
-; AVX512F-NEXT:  # %bb.3: # %cond.store1
+; AVX512F-NEXT:  .LBB15_3: # %cond.store1
 ; AVX512F-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
-; AVX512F-NEXT:  .LBB15_4: # %else2
-; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm3, %zmm3, %zmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $4, %al
 ; AVX512F-NEXT:    je .LBB15_6
-; AVX512F-NEXT:  # %bb.5: # %cond.store3
+; AVX512F-NEXT:  .LBB15_5: # %cond.store3
 ; AVX512F-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
-; AVX512F-NEXT:  .LBB15_6: # %else4
-; AVX512F-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
-; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $8, %al
 ; AVX512F-NEXT:    je .LBB15_8
-; AVX512F-NEXT:  # %bb.7: # %cond.store5
+; AVX512F-NEXT:  .LBB15_7: # %cond.store5
 ; AVX512F-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
-; AVX512F-NEXT:  .LBB15_8: # %else6
-; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm3, %zmm3, %zmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $4, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $16, %al
 ; AVX512F-NEXT:    je .LBB15_10
-; AVX512F-NEXT:  # %bb.9: # %cond.store7
+; AVX512F-NEXT:  .LBB15_9: # %cond.store7
 ; AVX512F-NEXT:    vpextrb $4, %xmm0, 4(%rdi)
-; AVX512F-NEXT:  .LBB15_10: # %else8
-; AVX512F-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
-; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $5, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $32, %al
 ; AVX512F-NEXT:    je .LBB15_12
-; AVX512F-NEXT:  # %bb.11: # %cond.store9
+; AVX512F-NEXT:  .LBB15_11: # %cond.store9
 ; AVX512F-NEXT:    vpextrb $5, %xmm0, 5(%rdi)
-; AVX512F-NEXT:  .LBB15_12: # %else10
-; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm3, %zmm3, %zmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $6, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $64, %al
 ; AVX512F-NEXT:    je .LBB15_14
-; AVX512F-NEXT:  # %bb.13: # %cond.store11
+; AVX512F-NEXT:  .LBB15_13: # %cond.store11
 ; AVX512F-NEXT:    vpextrb $6, %xmm0, 6(%rdi)
-; AVX512F-NEXT:  .LBB15_14: # %else12
-; AVX512F-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
-; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $7, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $-128, %al
 ; AVX512F-NEXT:    je .LBB15_16
-; AVX512F-NEXT:  # %bb.15: # %cond.store13
+; AVX512F-NEXT:  .LBB15_15: # %cond.store13
 ; AVX512F-NEXT:    vpextrb $7, %xmm0, 7(%rdi)
-; AVX512F-NEXT:  .LBB15_16: # %else14
-; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm3, %zmm3, %zmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $8, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $256, %eax # imm = 0x100
 ; AVX512F-NEXT:    je .LBB15_18
-; AVX512F-NEXT:  # %bb.17: # %cond.store15
+; AVX512F-NEXT:  .LBB15_17: # %cond.store15
 ; AVX512F-NEXT:    vpextrb $8, %xmm0, 8(%rdi)
-; AVX512F-NEXT:  .LBB15_18: # %else16
-; AVX512F-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
-; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $9, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $512, %eax # imm = 0x200
 ; AVX512F-NEXT:    je .LBB15_20
-; AVX512F-NEXT:  # %bb.19: # %cond.store17
+; AVX512F-NEXT:  .LBB15_19: # %cond.store17
 ; AVX512F-NEXT:    vpextrb $9, %xmm0, 9(%rdi)
-; AVX512F-NEXT:  .LBB15_20: # %else18
-; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm3, %zmm3, %zmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $10, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $1024, %eax # imm = 0x400
 ; AVX512F-NEXT:    je .LBB15_22
-; AVX512F-NEXT:  # %bb.21: # %cond.store19
+; AVX512F-NEXT:  .LBB15_21: # %cond.store19
 ; AVX512F-NEXT:    vpextrb $10, %xmm0, 10(%rdi)
-; AVX512F-NEXT:  .LBB15_22: # %else20
-; AVX512F-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
-; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $11, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB15_24
-; AVX512F-NEXT:  # %bb.23: # %cond.store21
-; AVX512F-NEXT:    vpextrb $11, %xmm0, 11(%rdi)
-; AVX512F-NEXT:  .LBB15_24: # %else22
-; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm3, %zmm3, %zmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $12, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB15_26
-; AVX512F-NEXT:  # %bb.25: # %cond.store23
-; AVX512F-NEXT:    vpextrb $12, %xmm0, 12(%rdi)
-; AVX512F-NEXT:  .LBB15_26: # %else24
-; AVX512F-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
-; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $13, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $2048, %eax # imm = 0x800
+; AVX512F-NEXT:    je .LBB15_24
+; AVX512F-NEXT:  .LBB15_23: # %cond.store21
+; AVX512F-NEXT:    vpextrb $11, %xmm0, 11(%rdi)
+; AVX512F-NEXT:    testl $4096, %eax # imm = 0x1000
+; AVX512F-NEXT:    je .LBB15_26
+; AVX512F-NEXT:  .LBB15_25: # %cond.store23
+; AVX512F-NEXT:    vpextrb $12, %xmm0, 12(%rdi)
+; AVX512F-NEXT:    testl $8192, %eax # imm = 0x2000
 ; AVX512F-NEXT:    je .LBB15_28
-; AVX512F-NEXT:  # %bb.27: # %cond.store25
+; AVX512F-NEXT:  .LBB15_27: # %cond.store25
 ; AVX512F-NEXT:    vpextrb $13, %xmm0, 13(%rdi)
-; AVX512F-NEXT:  .LBB15_28: # %else26
-; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm3, %zmm3, %zmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $14, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $16384, %eax # imm = 0x4000
 ; AVX512F-NEXT:    je .LBB15_30
-; AVX512F-NEXT:  # %bb.29: # %cond.store27
+; AVX512F-NEXT:  .LBB15_29: # %cond.store27
 ; AVX512F-NEXT:    vpextrb $14, %xmm0, 14(%rdi)
-; AVX512F-NEXT:  .LBB15_30: # %else28
-; AVX512F-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
-; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $15, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB15_32
-; AVX512F-NEXT:  # %bb.31: # %cond.store29
-; AVX512F-NEXT:    vpextrb $15, %xmm0, 15(%rdi)
-; AVX512F-NEXT:  .LBB15_32: # %else30
-; AVX512F-NEXT:    vextracti128 $1, %ymm2, %xmm1
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; AVX512F-NEXT:    je .LBB15_34
-; AVX512F-NEXT:  # %bb.33: # %cond.store31
+; AVX512F-NEXT:    testl $32768, %eax # imm = 0x8000
+; AVX512F-NEXT:    jne .LBB15_31
+; AVX512F-NEXT:    jmp .LBB15_32
+; AVX512F-NEXT:  .LBB15_33: # %cond.store31
 ; AVX512F-NEXT:    vpextrb $0, %xmm0, 16(%rdi)
-; AVX512F-NEXT:  .LBB15_34: # %else32
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $131072, %eax # imm = 0x20000
 ; AVX512F-NEXT:    je .LBB15_36
-; AVX512F-NEXT:  # %bb.35: # %cond.store33
+; AVX512F-NEXT:  .LBB15_35: # %cond.store33
 ; AVX512F-NEXT:    vpextrb $1, %xmm0, 17(%rdi)
-; AVX512F-NEXT:  .LBB15_36: # %else34
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $262144, %eax # imm = 0x40000
 ; AVX512F-NEXT:    je .LBB15_38
-; AVX512F-NEXT:  # %bb.37: # %cond.store35
+; AVX512F-NEXT:  .LBB15_37: # %cond.store35
 ; AVX512F-NEXT:    vpextrb $2, %xmm0, 18(%rdi)
-; AVX512F-NEXT:  .LBB15_38: # %else36
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $524288, %eax # imm = 0x80000
 ; AVX512F-NEXT:    je .LBB15_40
-; AVX512F-NEXT:  # %bb.39: # %cond.store37
+; AVX512F-NEXT:  .LBB15_39: # %cond.store37
 ; AVX512F-NEXT:    vpextrb $3, %xmm0, 19(%rdi)
-; AVX512F-NEXT:  .LBB15_40: # %else38
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $4, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $1048576, %eax # imm = 0x100000
 ; AVX512F-NEXT:    je .LBB15_42
-; AVX512F-NEXT:  # %bb.41: # %cond.store39
+; AVX512F-NEXT:  .LBB15_41: # %cond.store39
 ; AVX512F-NEXT:    vpextrb $4, %xmm0, 20(%rdi)
-; AVX512F-NEXT:  .LBB15_42: # %else40
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $5, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $2097152, %eax # imm = 0x200000
 ; AVX512F-NEXT:    je .LBB15_44
-; AVX512F-NEXT:  # %bb.43: # %cond.store41
+; AVX512F-NEXT:  .LBB15_43: # %cond.store41
 ; AVX512F-NEXT:    vpextrb $5, %xmm0, 21(%rdi)
-; AVX512F-NEXT:  .LBB15_44: # %else42
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $6, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $4194304, %eax # imm = 0x400000
 ; AVX512F-NEXT:    je .LBB15_46
-; AVX512F-NEXT:  # %bb.45: # %cond.store43
+; AVX512F-NEXT:  .LBB15_45: # %cond.store43
 ; AVX512F-NEXT:    vpextrb $6, %xmm0, 22(%rdi)
-; AVX512F-NEXT:  .LBB15_46: # %else44
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $7, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $8388608, %eax # imm = 0x800000
 ; AVX512F-NEXT:    je .LBB15_48
-; AVX512F-NEXT:  # %bb.47: # %cond.store45
+; AVX512F-NEXT:  .LBB15_47: # %cond.store45
 ; AVX512F-NEXT:    vpextrb $7, %xmm0, 23(%rdi)
-; AVX512F-NEXT:  .LBB15_48: # %else46
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $8, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $16777216, %eax # imm = 0x1000000
 ; AVX512F-NEXT:    je .LBB15_50
-; AVX512F-NEXT:  # %bb.49: # %cond.store47
+; AVX512F-NEXT:  .LBB15_49: # %cond.store47
 ; AVX512F-NEXT:    vpextrb $8, %xmm0, 24(%rdi)
-; AVX512F-NEXT:  .LBB15_50: # %else48
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $9, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $33554432, %eax # imm = 0x2000000
 ; AVX512F-NEXT:    je .LBB15_52
-; AVX512F-NEXT:  # %bb.51: # %cond.store49
+; AVX512F-NEXT:  .LBB15_51: # %cond.store49
 ; AVX512F-NEXT:    vpextrb $9, %xmm0, 25(%rdi)
-; AVX512F-NEXT:  .LBB15_52: # %else50
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $10, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $67108864, %eax # imm = 0x4000000
 ; AVX512F-NEXT:    je .LBB15_54
-; AVX512F-NEXT:  # %bb.53: # %cond.store51
+; AVX512F-NEXT:  .LBB15_53: # %cond.store51
 ; AVX512F-NEXT:    vpextrb $10, %xmm0, 26(%rdi)
-; AVX512F-NEXT:  .LBB15_54: # %else52
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $11, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $134217728, %eax # imm = 0x8000000
 ; AVX512F-NEXT:    je .LBB15_56
-; AVX512F-NEXT:  # %bb.55: # %cond.store53
+; AVX512F-NEXT:  .LBB15_55: # %cond.store53
 ; AVX512F-NEXT:    vpextrb $11, %xmm0, 27(%rdi)
-; AVX512F-NEXT:  .LBB15_56: # %else54
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $12, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $268435456, %eax # imm = 0x10000000
 ; AVX512F-NEXT:    je .LBB15_58
-; AVX512F-NEXT:  # %bb.57: # %cond.store55
+; AVX512F-NEXT:  .LBB15_57: # %cond.store55
 ; AVX512F-NEXT:    vpextrb $12, %xmm0, 28(%rdi)
-; AVX512F-NEXT:  .LBB15_58: # %else56
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $13, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $536870912, %eax # imm = 0x20000000
 ; AVX512F-NEXT:    je .LBB15_60
-; AVX512F-NEXT:  # %bb.59: # %cond.store57
+; AVX512F-NEXT:  .LBB15_59: # %cond.store57
 ; AVX512F-NEXT:    vpextrb $13, %xmm0, 29(%rdi)
-; AVX512F-NEXT:  .LBB15_60: # %else58
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm1
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm2
-; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $14, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $1073741824, %eax # imm = 0x40000000
 ; AVX512F-NEXT:    je .LBB15_62
-; AVX512F-NEXT:  # %bb.61: # %cond.store59
+; AVX512F-NEXT:  .LBB15_61: # %cond.store59
 ; AVX512F-NEXT:    vpextrb $14, %xmm0, 30(%rdi)
-; AVX512F-NEXT:  .LBB15_62: # %else60
-; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
-; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $15, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $-2147483648, %eax # imm = 0x80000000
 ; AVX512F-NEXT:    je .LBB15_64
-; AVX512F-NEXT:  # %bb.63: # %cond.store61
+; AVX512F-NEXT:  .LBB15_63: # %cond.store61
 ; AVX512F-NEXT:    vpextrb $15, %xmm0, 31(%rdi)
-; AVX512F-NEXT:  .LBB15_64: # %else62
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
@@ -6208,753 +5645,606 @@ define void @truncstore_v16i16_v16i8(<16 x i16> %x, <16 x i8>* %p, <16 x i8> %ma
 ; SSE2-LABEL: truncstore_v16i16_v16i8:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    pcmpeqb %xmm2, %xmm3
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
 ; SSE2-NEXT:    pand %xmm4, %xmm1
 ; SSE2-NEXT:    pand %xmm4, %xmm0
 ; SSE2-NEXT:    packuswb %xmm1, %xmm0
-; SSE2-NEXT:    movd %xmm3, %ecx
-; SSE2-NEXT:    movl %ecx, %eax
-; SSE2-NEXT:    notb %al
+; SSE2-NEXT:    pcmpeqb %xmm2, %xmm3
+; SSE2-NEXT:    pmovmskb %xmm3, %eax
+; SSE2-NEXT:    xorl $65535, %eax # imm = 0xFFFF
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    je .LBB16_2
-; SSE2-NEXT:  # %bb.1: # %cond.store
-; SSE2-NEXT:    movb %al, (%rdi)
-; SSE2-NEXT:  .LBB16_2: # %else
-; SSE2-NEXT:    shrl $8, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    je .LBB16_4
-; SSE2-NEXT:  # %bb.3: # %cond.store1
-; SSE2-NEXT:    movb %ah, 1(%rdi)
+; SSE2-NEXT:    movd %xmm0, %ecx
+; SSE2-NEXT:    jne .LBB16_1
+; SSE2-NEXT:  # %bb.2: # %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne .LBB16_3
 ; SSE2-NEXT:  .LBB16_4: # %else2
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqb %xmm2, %xmm1
-; SSE2-NEXT:    movd %xmm1, %ecx
-; SSE2-NEXT:    movl %ecx, %edx
-; SSE2-NEXT:    shrl $16, %edx
-; SSE2-NEXT:    notb %dl
-; SSE2-NEXT:    testb $1, %dl
-; SSE2-NEXT:    je .LBB16_6
-; SSE2-NEXT:  # %bb.5: # %cond.store3
-; SSE2-NEXT:    movl %eax, %edx
-; SSE2-NEXT:    shrl $16, %edx
-; SSE2-NEXT:    movb %dl, 2(%rdi)
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne .LBB16_5
 ; SSE2-NEXT:  .LBB16_6: # %else4
-; SSE2-NEXT:    shrl $24, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je .LBB16_8
-; SSE2-NEXT:  # %bb.7: # %cond.store5
-; SSE2-NEXT:    shrl $24, %eax
-; SSE2-NEXT:    movb %al, 3(%rdi)
+; SSE2-NEXT:  .LBB16_7: # %cond.store5
+; SSE2-NEXT:    shrl $24, %ecx
+; SSE2-NEXT:    movb %cl, 3(%rdi)
 ; SSE2-NEXT:  .LBB16_8: # %else6
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqb %xmm2, %xmm1
-; SSE2-NEXT:    pextrw $2, %xmm1, %eax
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testb $16, %al
 ; SSE2-NEXT:    pextrw $2, %xmm0, %ecx
 ; SSE2-NEXT:    je .LBB16_10
 ; SSE2-NEXT:  # %bb.9: # %cond.store7
 ; SSE2-NEXT:    movb %cl, 4(%rdi)
 ; SSE2-NEXT:  .LBB16_10: # %else8
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    notb %al
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $32, %al
 ; SSE2-NEXT:    je .LBB16_12
 ; SSE2-NEXT:  # %bb.11: # %cond.store9
 ; SSE2-NEXT:    movb %ch, 5(%rdi)
 ; SSE2-NEXT:  .LBB16_12: # %else10
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqb %xmm2, %xmm1
-; SSE2-NEXT:    pextrw $3, %xmm1, %eax
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testb $64, %al
 ; SSE2-NEXT:    pextrw $3, %xmm0, %ecx
 ; SSE2-NEXT:    je .LBB16_14
 ; SSE2-NEXT:  # %bb.13: # %cond.store11
 ; SSE2-NEXT:    movb %cl, 6(%rdi)
 ; SSE2-NEXT:  .LBB16_14: # %else12
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    notb %al
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $-128, %al
 ; SSE2-NEXT:    je .LBB16_16
 ; SSE2-NEXT:  # %bb.15: # %cond.store13
 ; SSE2-NEXT:    movb %ch, 7(%rdi)
 ; SSE2-NEXT:  .LBB16_16: # %else14
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqb %xmm2, %xmm1
-; SSE2-NEXT:    pextrw $4, %xmm1, %eax
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $256, %eax # imm = 0x100
 ; SSE2-NEXT:    pextrw $4, %xmm0, %ecx
 ; SSE2-NEXT:    je .LBB16_18
 ; SSE2-NEXT:  # %bb.17: # %cond.store15
 ; SSE2-NEXT:    movb %cl, 8(%rdi)
 ; SSE2-NEXT:  .LBB16_18: # %else16
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    notb %al
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $512, %eax # imm = 0x200
 ; SSE2-NEXT:    je .LBB16_20
 ; SSE2-NEXT:  # %bb.19: # %cond.store17
 ; SSE2-NEXT:    movb %ch, 9(%rdi)
 ; SSE2-NEXT:  .LBB16_20: # %else18
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqb %xmm2, %xmm1
-; SSE2-NEXT:    pextrw $5, %xmm1, %eax
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $1024, %eax # imm = 0x400
 ; SSE2-NEXT:    pextrw $5, %xmm0, %ecx
 ; SSE2-NEXT:    je .LBB16_22
 ; SSE2-NEXT:  # %bb.21: # %cond.store19
 ; SSE2-NEXT:    movb %cl, 10(%rdi)
 ; SSE2-NEXT:  .LBB16_22: # %else20
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    notb %al
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $2048, %eax # imm = 0x800
 ; SSE2-NEXT:    je .LBB16_24
 ; SSE2-NEXT:  # %bb.23: # %cond.store21
 ; SSE2-NEXT:    movb %ch, 11(%rdi)
 ; SSE2-NEXT:  .LBB16_24: # %else22
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqb %xmm2, %xmm1
-; SSE2-NEXT:    pextrw $6, %xmm1, %eax
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $4096, %eax # imm = 0x1000
 ; SSE2-NEXT:    pextrw $6, %xmm0, %ecx
 ; SSE2-NEXT:    je .LBB16_26
 ; SSE2-NEXT:  # %bb.25: # %cond.store23
 ; SSE2-NEXT:    movb %cl, 12(%rdi)
 ; SSE2-NEXT:  .LBB16_26: # %else24
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    notb %al
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $8192, %eax # imm = 0x2000
 ; SSE2-NEXT:    je .LBB16_28
 ; SSE2-NEXT:  # %bb.27: # %cond.store25
 ; SSE2-NEXT:    movb %ch, 13(%rdi)
 ; SSE2-NEXT:  .LBB16_28: # %else26
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqb %xmm1, %xmm2
-; SSE2-NEXT:    pextrw $7, %xmm2, %eax
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $16384, %eax # imm = 0x4000
 ; SSE2-NEXT:    pextrw $7, %xmm0, %ecx
-; SSE2-NEXT:    je .LBB16_30
-; SSE2-NEXT:  # %bb.29: # %cond.store27
+; SSE2-NEXT:    jne .LBB16_29
+; SSE2-NEXT:  # %bb.30: # %else28
+; SSE2-NEXT:    testl $32768, %eax # imm = 0x8000
+; SSE2-NEXT:    jne .LBB16_31
+; SSE2-NEXT:  .LBB16_32: # %else30
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  .LBB16_1: # %cond.store
+; SSE2-NEXT:    movb %cl, (%rdi)
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    je .LBB16_4
+; SSE2-NEXT:  .LBB16_3: # %cond.store1
+; SSE2-NEXT:    movb %ch, 1(%rdi)
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    je .LBB16_6
+; SSE2-NEXT:  .LBB16_5: # %cond.store3
+; SSE2-NEXT:    movl %ecx, %edx
+; SSE2-NEXT:    shrl $16, %edx
+; SSE2-NEXT:    movb %dl, 2(%rdi)
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne .LBB16_7
+; SSE2-NEXT:    jmp .LBB16_8
+; SSE2-NEXT:  .LBB16_29: # %cond.store27
 ; SSE2-NEXT:    movb %cl, 14(%rdi)
-; SSE2-NEXT:  .LBB16_30: # %else28
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    notb %al
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $32768, %eax # imm = 0x8000
 ; SSE2-NEXT:    je .LBB16_32
-; SSE2-NEXT:  # %bb.31: # %cond.store29
+; SSE2-NEXT:  .LBB16_31: # %cond.store29
 ; SSE2-NEXT:    movb %ch, 15(%rdi)
-; SSE2-NEXT:  .LBB16_32: # %else30
 ; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: truncstore_v16i16_v16i8:
 ; SSE4:       # %bb.0:
 ; SSE4-NEXT:    pxor %xmm3, %xmm3
-; SSE4-NEXT:    pcmpeqb %xmm2, %xmm3
 ; SSE4-NEXT:    movdqa {{.*#+}} xmm4 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
 ; SSE4-NEXT:    pshufb %xmm4, %xmm1
 ; SSE4-NEXT:    pshufb %xmm4, %xmm0
 ; SSE4-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSE4-NEXT:    pextrb $0, %xmm3, %eax
-; SSE4-NEXT:    notb %al
+; SSE4-NEXT:    pcmpeqb %xmm2, %xmm3
+; SSE4-NEXT:    pmovmskb %xmm3, %eax
+; SSE4-NEXT:    xorl $65535, %eax # imm = 0xFFFF
 ; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB16_2
-; SSE4-NEXT:  # %bb.1: # %cond.store
+; SSE4-NEXT:    jne .LBB16_1
+; SSE4-NEXT:  # %bb.2: # %else
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    jne .LBB16_3
+; SSE4-NEXT:  .LBB16_4: # %else2
+; SSE4-NEXT:    testb $4, %al
+; SSE4-NEXT:    jne .LBB16_5
+; SSE4-NEXT:  .LBB16_6: # %else4
+; SSE4-NEXT:    testb $8, %al
+; SSE4-NEXT:    jne .LBB16_7
+; SSE4-NEXT:  .LBB16_8: # %else6
+; SSE4-NEXT:    testb $16, %al
+; SSE4-NEXT:    jne .LBB16_9
+; SSE4-NEXT:  .LBB16_10: # %else8
+; SSE4-NEXT:    testb $32, %al
+; SSE4-NEXT:    jne .LBB16_11
+; SSE4-NEXT:  .LBB16_12: # %else10
+; SSE4-NEXT:    testb $64, %al
+; SSE4-NEXT:    jne .LBB16_13
+; SSE4-NEXT:  .LBB16_14: # %else12
+; SSE4-NEXT:    testb $-128, %al
+; SSE4-NEXT:    jne .LBB16_15
+; SSE4-NEXT:  .LBB16_16: # %else14
+; SSE4-NEXT:    testl $256, %eax # imm = 0x100
+; SSE4-NEXT:    jne .LBB16_17
+; SSE4-NEXT:  .LBB16_18: # %else16
+; SSE4-NEXT:    testl $512, %eax # imm = 0x200
+; SSE4-NEXT:    jne .LBB16_19
+; SSE4-NEXT:  .LBB16_20: # %else18
+; SSE4-NEXT:    testl $1024, %eax # imm = 0x400
+; SSE4-NEXT:    jne .LBB16_21
+; SSE4-NEXT:  .LBB16_22: # %else20
+; SSE4-NEXT:    testl $2048, %eax # imm = 0x800
+; SSE4-NEXT:    jne .LBB16_23
+; SSE4-NEXT:  .LBB16_24: # %else22
+; SSE4-NEXT:    testl $4096, %eax # imm = 0x1000
+; SSE4-NEXT:    jne .LBB16_25
+; SSE4-NEXT:  .LBB16_26: # %else24
+; SSE4-NEXT:    testl $8192, %eax # imm = 0x2000
+; SSE4-NEXT:    jne .LBB16_27
+; SSE4-NEXT:  .LBB16_28: # %else26
+; SSE4-NEXT:    testl $16384, %eax # imm = 0x4000
+; SSE4-NEXT:    jne .LBB16_29
+; SSE4-NEXT:  .LBB16_30: # %else28
+; SSE4-NEXT:    testl $32768, %eax # imm = 0x8000
+; SSE4-NEXT:    jne .LBB16_31
+; SSE4-NEXT:  .LBB16_32: # %else30
+; SSE4-NEXT:    retq
+; SSE4-NEXT:  .LBB16_1: # %cond.store
 ; SSE4-NEXT:    pextrb $0, %xmm0, (%rdi)
-; SSE4-NEXT:  .LBB16_2: # %else
-; SSE4-NEXT:    pextrb $1, %xmm3, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $2, %al
 ; SSE4-NEXT:    je .LBB16_4
-; SSE4-NEXT:  # %bb.3: # %cond.store1
+; SSE4-NEXT:  .LBB16_3: # %cond.store1
 ; SSE4-NEXT:    pextrb $1, %xmm0, 1(%rdi)
-; SSE4-NEXT:  .LBB16_4: # %else2
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqb %xmm2, %xmm1
-; SSE4-NEXT:    pextrb $2, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $4, %al
 ; SSE4-NEXT:    je .LBB16_6
-; SSE4-NEXT:  # %bb.5: # %cond.store3
+; SSE4-NEXT:  .LBB16_5: # %cond.store3
 ; SSE4-NEXT:    pextrb $2, %xmm0, 2(%rdi)
-; SSE4-NEXT:  .LBB16_6: # %else4
-; SSE4-NEXT:    pextrb $3, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $8, %al
 ; SSE4-NEXT:    je .LBB16_8
-; SSE4-NEXT:  # %bb.7: # %cond.store5
+; SSE4-NEXT:  .LBB16_7: # %cond.store5
 ; SSE4-NEXT:    pextrb $3, %xmm0, 3(%rdi)
-; SSE4-NEXT:  .LBB16_8: # %else6
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqb %xmm2, %xmm1
-; SSE4-NEXT:    pextrb $4, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $16, %al
 ; SSE4-NEXT:    je .LBB16_10
-; SSE4-NEXT:  # %bb.9: # %cond.store7
+; SSE4-NEXT:  .LBB16_9: # %cond.store7
 ; SSE4-NEXT:    pextrb $4, %xmm0, 4(%rdi)
-; SSE4-NEXT:  .LBB16_10: # %else8
-; SSE4-NEXT:    pextrb $5, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $32, %al
 ; SSE4-NEXT:    je .LBB16_12
-; SSE4-NEXT:  # %bb.11: # %cond.store9
+; SSE4-NEXT:  .LBB16_11: # %cond.store9
 ; SSE4-NEXT:    pextrb $5, %xmm0, 5(%rdi)
-; SSE4-NEXT:  .LBB16_12: # %else10
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqb %xmm2, %xmm1
-; SSE4-NEXT:    pextrb $6, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $64, %al
 ; SSE4-NEXT:    je .LBB16_14
-; SSE4-NEXT:  # %bb.13: # %cond.store11
+; SSE4-NEXT:  .LBB16_13: # %cond.store11
 ; SSE4-NEXT:    pextrb $6, %xmm0, 6(%rdi)
-; SSE4-NEXT:  .LBB16_14: # %else12
-; SSE4-NEXT:    pextrb $7, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $-128, %al
 ; SSE4-NEXT:    je .LBB16_16
-; SSE4-NEXT:  # %bb.15: # %cond.store13
+; SSE4-NEXT:  .LBB16_15: # %cond.store13
 ; SSE4-NEXT:    pextrb $7, %xmm0, 7(%rdi)
-; SSE4-NEXT:  .LBB16_16: # %else14
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqb %xmm2, %xmm1
-; SSE4-NEXT:    pextrb $8, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $256, %eax # imm = 0x100
 ; SSE4-NEXT:    je .LBB16_18
-; SSE4-NEXT:  # %bb.17: # %cond.store15
+; SSE4-NEXT:  .LBB16_17: # %cond.store15
 ; SSE4-NEXT:    pextrb $8, %xmm0, 8(%rdi)
-; SSE4-NEXT:  .LBB16_18: # %else16
-; SSE4-NEXT:    pextrb $9, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $512, %eax # imm = 0x200
 ; SSE4-NEXT:    je .LBB16_20
-; SSE4-NEXT:  # %bb.19: # %cond.store17
+; SSE4-NEXT:  .LBB16_19: # %cond.store17
 ; SSE4-NEXT:    pextrb $9, %xmm0, 9(%rdi)
-; SSE4-NEXT:  .LBB16_20: # %else18
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqb %xmm2, %xmm1
-; SSE4-NEXT:    pextrb $10, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $1024, %eax # imm = 0x400
 ; SSE4-NEXT:    je .LBB16_22
-; SSE4-NEXT:  # %bb.21: # %cond.store19
+; SSE4-NEXT:  .LBB16_21: # %cond.store19
 ; SSE4-NEXT:    pextrb $10, %xmm0, 10(%rdi)
-; SSE4-NEXT:  .LBB16_22: # %else20
-; SSE4-NEXT:    pextrb $11, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $2048, %eax # imm = 0x800
 ; SSE4-NEXT:    je .LBB16_24
-; SSE4-NEXT:  # %bb.23: # %cond.store21
+; SSE4-NEXT:  .LBB16_23: # %cond.store21
 ; SSE4-NEXT:    pextrb $11, %xmm0, 11(%rdi)
-; SSE4-NEXT:  .LBB16_24: # %else22
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqb %xmm2, %xmm1
-; SSE4-NEXT:    pextrb $12, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $4096, %eax # imm = 0x1000
 ; SSE4-NEXT:    je .LBB16_26
-; SSE4-NEXT:  # %bb.25: # %cond.store23
+; SSE4-NEXT:  .LBB16_25: # %cond.store23
 ; SSE4-NEXT:    pextrb $12, %xmm0, 12(%rdi)
-; SSE4-NEXT:  .LBB16_26: # %else24
-; SSE4-NEXT:    pextrb $13, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $8192, %eax # imm = 0x2000
 ; SSE4-NEXT:    je .LBB16_28
-; SSE4-NEXT:  # %bb.27: # %cond.store25
+; SSE4-NEXT:  .LBB16_27: # %cond.store25
 ; SSE4-NEXT:    pextrb $13, %xmm0, 13(%rdi)
-; SSE4-NEXT:  .LBB16_28: # %else26
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqb %xmm1, %xmm2
-; SSE4-NEXT:    pextrb $14, %xmm2, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $16384, %eax # imm = 0x4000
 ; SSE4-NEXT:    je .LBB16_30
-; SSE4-NEXT:  # %bb.29: # %cond.store27
+; SSE4-NEXT:  .LBB16_29: # %cond.store27
 ; SSE4-NEXT:    pextrb $14, %xmm0, 14(%rdi)
-; SSE4-NEXT:  .LBB16_30: # %else28
-; SSE4-NEXT:    pextrb $15, %xmm2, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $32768, %eax # imm = 0x8000
 ; SSE4-NEXT:    je .LBB16_32
-; SSE4-NEXT:  # %bb.31: # %cond.store29
+; SSE4-NEXT:  .LBB16_31: # %cond.store29
 ; SSE4-NEXT:    pextrb $15, %xmm0, 15(%rdi)
-; SSE4-NEXT:  .LBB16_32: # %else30
 ; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: truncstore_v16i16_v16i8:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
 ; AVX1-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
 ; AVX1-NEXT:    vpackuswb %xmm3, %xmm0, %xmm0
-; AVX1-NEXT:    vpextrb $0, %xmm2, %eax
-; AVX1-NEXT:    notb %al
+; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpmovmskb %xmm1, %eax
+; AVX1-NEXT:    xorl $65535, %eax # imm = 0xFFFF
 ; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je .LBB16_2
-; AVX1-NEXT:  # %bb.1: # %cond.store
+; AVX1-NEXT:    jne .LBB16_1
+; AVX1-NEXT:  # %bb.2: # %else
+; AVX1-NEXT:    testb $2, %al
+; AVX1-NEXT:    jne .LBB16_3
+; AVX1-NEXT:  .LBB16_4: # %else2
+; AVX1-NEXT:    testb $4, %al
+; AVX1-NEXT:    jne .LBB16_5
+; AVX1-NEXT:  .LBB16_6: # %else4
+; AVX1-NEXT:    testb $8, %al
+; AVX1-NEXT:    jne .LBB16_7
+; AVX1-NEXT:  .LBB16_8: # %else6
+; AVX1-NEXT:    testb $16, %al
+; AVX1-NEXT:    jne .LBB16_9
+; AVX1-NEXT:  .LBB16_10: # %else8
+; AVX1-NEXT:    testb $32, %al
+; AVX1-NEXT:    jne .LBB16_11
+; AVX1-NEXT:  .LBB16_12: # %else10
+; AVX1-NEXT:    testb $64, %al
+; AVX1-NEXT:    jne .LBB16_13
+; AVX1-NEXT:  .LBB16_14: # %else12
+; AVX1-NEXT:    testb $-128, %al
+; AVX1-NEXT:    jne .LBB16_15
+; AVX1-NEXT:  .LBB16_16: # %else14
+; AVX1-NEXT:    testl $256, %eax # imm = 0x100
+; AVX1-NEXT:    jne .LBB16_17
+; AVX1-NEXT:  .LBB16_18: # %else16
+; AVX1-NEXT:    testl $512, %eax # imm = 0x200
+; AVX1-NEXT:    jne .LBB16_19
+; AVX1-NEXT:  .LBB16_20: # %else18
+; AVX1-NEXT:    testl $1024, %eax # imm = 0x400
+; AVX1-NEXT:    jne .LBB16_21
+; AVX1-NEXT:  .LBB16_22: # %else20
+; AVX1-NEXT:    testl $2048, %eax # imm = 0x800
+; AVX1-NEXT:    jne .LBB16_23
+; AVX1-NEXT:  .LBB16_24: # %else22
+; AVX1-NEXT:    testl $4096, %eax # imm = 0x1000
+; AVX1-NEXT:    jne .LBB16_25
+; AVX1-NEXT:  .LBB16_26: # %else24
+; AVX1-NEXT:    testl $8192, %eax # imm = 0x2000
+; AVX1-NEXT:    jne .LBB16_27
+; AVX1-NEXT:  .LBB16_28: # %else26
+; AVX1-NEXT:    testl $16384, %eax # imm = 0x4000
+; AVX1-NEXT:    jne .LBB16_29
+; AVX1-NEXT:  .LBB16_30: # %else28
+; AVX1-NEXT:    testl $32768, %eax # imm = 0x8000
+; AVX1-NEXT:    jne .LBB16_31
+; AVX1-NEXT:  .LBB16_32: # %else30
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+; AVX1-NEXT:  .LBB16_1: # %cond.store
 ; AVX1-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX1-NEXT:  .LBB16_2: # %else
-; AVX1-NEXT:    vpextrb $1, %xmm2, %eax
-; AVX1-NEXT:    notb %al
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $2, %al
 ; AVX1-NEXT:    je .LBB16_4
-; AVX1-NEXT:  # %bb.3: # %cond.store1
+; AVX1-NEXT:  .LBB16_3: # %cond.store1
 ; AVX1-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
-; AVX1-NEXT:  .LBB16_4: # %else2
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpextrb $2, %xmm2, %eax
-; AVX1-NEXT:    notb %al
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $4, %al
 ; AVX1-NEXT:    je .LBB16_6
-; AVX1-NEXT:  # %bb.5: # %cond.store3
+; AVX1-NEXT:  .LBB16_5: # %cond.store3
 ; AVX1-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
-; AVX1-NEXT:  .LBB16_6: # %else4
-; AVX1-NEXT:    vpextrb $3, %xmm2, %eax
-; AVX1-NEXT:    notb %al
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $8, %al
 ; AVX1-NEXT:    je .LBB16_8
-; AVX1-NEXT:  # %bb.7: # %cond.store5
+; AVX1-NEXT:  .LBB16_7: # %cond.store5
 ; AVX1-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
-; AVX1-NEXT:  .LBB16_8: # %else6
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX1-NEXT:    notb %al
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $16, %al
 ; AVX1-NEXT:    je .LBB16_10
-; AVX1-NEXT:  # %bb.9: # %cond.store7
+; AVX1-NEXT:  .LBB16_9: # %cond.store7
 ; AVX1-NEXT:    vpextrb $4, %xmm0, 4(%rdi)
-; AVX1-NEXT:  .LBB16_10: # %else8
-; AVX1-NEXT:    vpextrb $5, %xmm2, %eax
-; AVX1-NEXT:    notb %al
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $32, %al
 ; AVX1-NEXT:    je .LBB16_12
-; AVX1-NEXT:  # %bb.11: # %cond.store9
+; AVX1-NEXT:  .LBB16_11: # %cond.store9
 ; AVX1-NEXT:    vpextrb $5, %xmm0, 5(%rdi)
-; AVX1-NEXT:  .LBB16_12: # %else10
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpextrb $6, %xmm2, %eax
-; AVX1-NEXT:    notb %al
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $64, %al
 ; AVX1-NEXT:    je .LBB16_14
-; AVX1-NEXT:  # %bb.13: # %cond.store11
+; AVX1-NEXT:  .LBB16_13: # %cond.store11
 ; AVX1-NEXT:    vpextrb $6, %xmm0, 6(%rdi)
-; AVX1-NEXT:  .LBB16_14: # %else12
-; AVX1-NEXT:    vpextrb $7, %xmm2, %eax
-; AVX1-NEXT:    notb %al
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $-128, %al
 ; AVX1-NEXT:    je .LBB16_16
-; AVX1-NEXT:  # %bb.15: # %cond.store13
+; AVX1-NEXT:  .LBB16_15: # %cond.store13
 ; AVX1-NEXT:    vpextrb $7, %xmm0, 7(%rdi)
-; AVX1-NEXT:  .LBB16_16: # %else14
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpextrb $8, %xmm2, %eax
-; AVX1-NEXT:    notb %al
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $256, %eax # imm = 0x100
 ; AVX1-NEXT:    je .LBB16_18
-; AVX1-NEXT:  # %bb.17: # %cond.store15
+; AVX1-NEXT:  .LBB16_17: # %cond.store15
 ; AVX1-NEXT:    vpextrb $8, %xmm0, 8(%rdi)
-; AVX1-NEXT:  .LBB16_18: # %else16
-; AVX1-NEXT:    vpextrb $9, %xmm2, %eax
-; AVX1-NEXT:    notb %al
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $512, %eax # imm = 0x200
 ; AVX1-NEXT:    je .LBB16_20
-; AVX1-NEXT:  # %bb.19: # %cond.store17
+; AVX1-NEXT:  .LBB16_19: # %cond.store17
 ; AVX1-NEXT:    vpextrb $9, %xmm0, 9(%rdi)
-; AVX1-NEXT:  .LBB16_20: # %else18
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpextrb $10, %xmm2, %eax
-; AVX1-NEXT:    notb %al
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $1024, %eax # imm = 0x400
 ; AVX1-NEXT:    je .LBB16_22
-; AVX1-NEXT:  # %bb.21: # %cond.store19
+; AVX1-NEXT:  .LBB16_21: # %cond.store19
 ; AVX1-NEXT:    vpextrb $10, %xmm0, 10(%rdi)
-; AVX1-NEXT:  .LBB16_22: # %else20
-; AVX1-NEXT:    vpextrb $11, %xmm2, %eax
-; AVX1-NEXT:    notb %al
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $2048, %eax # imm = 0x800
 ; AVX1-NEXT:    je .LBB16_24
-; AVX1-NEXT:  # %bb.23: # %cond.store21
+; AVX1-NEXT:  .LBB16_23: # %cond.store21
 ; AVX1-NEXT:    vpextrb $11, %xmm0, 11(%rdi)
-; AVX1-NEXT:  .LBB16_24: # %else22
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpextrb $12, %xmm2, %eax
-; AVX1-NEXT:    notb %al
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $4096, %eax # imm = 0x1000
 ; AVX1-NEXT:    je .LBB16_26
-; AVX1-NEXT:  # %bb.25: # %cond.store23
+; AVX1-NEXT:  .LBB16_25: # %cond.store23
 ; AVX1-NEXT:    vpextrb $12, %xmm0, 12(%rdi)
-; AVX1-NEXT:  .LBB16_26: # %else24
-; AVX1-NEXT:    vpextrb $13, %xmm2, %eax
-; AVX1-NEXT:    notb %al
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $8192, %eax # imm = 0x2000
 ; AVX1-NEXT:    je .LBB16_28
-; AVX1-NEXT:  # %bb.27: # %cond.store25
+; AVX1-NEXT:  .LBB16_27: # %cond.store25
 ; AVX1-NEXT:    vpextrb $13, %xmm0, 13(%rdi)
-; AVX1-NEXT:  .LBB16_28: # %else26
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $14, %xmm1, %eax
-; AVX1-NEXT:    notb %al
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $16384, %eax # imm = 0x4000
 ; AVX1-NEXT:    je .LBB16_30
-; AVX1-NEXT:  # %bb.29: # %cond.store27
+; AVX1-NEXT:  .LBB16_29: # %cond.store27
 ; AVX1-NEXT:    vpextrb $14, %xmm0, 14(%rdi)
-; AVX1-NEXT:  .LBB16_30: # %else28
-; AVX1-NEXT:    vpextrb $15, %xmm1, %eax
-; AVX1-NEXT:    notb %al
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $32768, %eax # imm = 0x8000
 ; AVX1-NEXT:    je .LBB16_32
-; AVX1-NEXT:  # %bb.31: # %cond.store29
+; AVX1-NEXT:  .LBB16_31: # %cond.store29
 ; AVX1-NEXT:    vpextrb $15, %xmm0, 15(%rdi)
-; AVX1-NEXT:  .LBB16_32: # %else30
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: truncstore_v16i16_v16i8:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
 ; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm3
 ; AVX2-NEXT:    vpackuswb %xmm3, %xmm0, %xmm0
-; AVX2-NEXT:    vpextrb $0, %xmm2, %eax
-; AVX2-NEXT:    notb %al
+; AVX2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpmovmskb %xmm1, %eax
+; AVX2-NEXT:    xorl $65535, %eax # imm = 0xFFFF
 ; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je .LBB16_2
-; AVX2-NEXT:  # %bb.1: # %cond.store
+; AVX2-NEXT:    jne .LBB16_1
+; AVX2-NEXT:  # %bb.2: # %else
+; AVX2-NEXT:    testb $2, %al
+; AVX2-NEXT:    jne .LBB16_3
+; AVX2-NEXT:  .LBB16_4: # %else2
+; AVX2-NEXT:    testb $4, %al
+; AVX2-NEXT:    jne .LBB16_5
+; AVX2-NEXT:  .LBB16_6: # %else4
+; AVX2-NEXT:    testb $8, %al
+; AVX2-NEXT:    jne .LBB16_7
+; AVX2-NEXT:  .LBB16_8: # %else6
+; AVX2-NEXT:    testb $16, %al
+; AVX2-NEXT:    jne .LBB16_9
+; AVX2-NEXT:  .LBB16_10: # %else8
+; AVX2-NEXT:    testb $32, %al
+; AVX2-NEXT:    jne .LBB16_11
+; AVX2-NEXT:  .LBB16_12: # %else10
+; AVX2-NEXT:    testb $64, %al
+; AVX2-NEXT:    jne .LBB16_13
+; AVX2-NEXT:  .LBB16_14: # %else12
+; AVX2-NEXT:    testb $-128, %al
+; AVX2-NEXT:    jne .LBB16_15
+; AVX2-NEXT:  .LBB16_16: # %else14
+; AVX2-NEXT:    testl $256, %eax # imm = 0x100
+; AVX2-NEXT:    jne .LBB16_17
+; AVX2-NEXT:  .LBB16_18: # %else16
+; AVX2-NEXT:    testl $512, %eax # imm = 0x200
+; AVX2-NEXT:    jne .LBB16_19
+; AVX2-NEXT:  .LBB16_20: # %else18
+; AVX2-NEXT:    testl $1024, %eax # imm = 0x400
+; AVX2-NEXT:    jne .LBB16_21
+; AVX2-NEXT:  .LBB16_22: # %else20
+; AVX2-NEXT:    testl $2048, %eax # imm = 0x800
+; AVX2-NEXT:    jne .LBB16_23
+; AVX2-NEXT:  .LBB16_24: # %else22
+; AVX2-NEXT:    testl $4096, %eax # imm = 0x1000
+; AVX2-NEXT:    jne .LBB16_25
+; AVX2-NEXT:  .LBB16_26: # %else24
+; AVX2-NEXT:    testl $8192, %eax # imm = 0x2000
+; AVX2-NEXT:    jne .LBB16_27
+; AVX2-NEXT:  .LBB16_28: # %else26
+; AVX2-NEXT:    testl $16384, %eax # imm = 0x4000
+; AVX2-NEXT:    jne .LBB16_29
+; AVX2-NEXT:  .LBB16_30: # %else28
+; AVX2-NEXT:    testl $32768, %eax # imm = 0x8000
+; AVX2-NEXT:    jne .LBB16_31
+; AVX2-NEXT:  .LBB16_32: # %else30
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+; AVX2-NEXT:  .LBB16_1: # %cond.store
 ; AVX2-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX2-NEXT:  .LBB16_2: # %else
-; AVX2-NEXT:    vpextrb $1, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $2, %al
 ; AVX2-NEXT:    je .LBB16_4
-; AVX2-NEXT:  # %bb.3: # %cond.store1
+; AVX2-NEXT:  .LBB16_3: # %cond.store1
 ; AVX2-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
-; AVX2-NEXT:  .LBB16_4: # %else2
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpextrb $2, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $4, %al
 ; AVX2-NEXT:    je .LBB16_6
-; AVX2-NEXT:  # %bb.5: # %cond.store3
+; AVX2-NEXT:  .LBB16_5: # %cond.store3
 ; AVX2-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
-; AVX2-NEXT:  .LBB16_6: # %else4
-; AVX2-NEXT:    vpextrb $3, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $8, %al
 ; AVX2-NEXT:    je .LBB16_8
-; AVX2-NEXT:  # %bb.7: # %cond.store5
+; AVX2-NEXT:  .LBB16_7: # %cond.store5
 ; AVX2-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
-; AVX2-NEXT:  .LBB16_8: # %else6
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $16, %al
 ; AVX2-NEXT:    je .LBB16_10
-; AVX2-NEXT:  # %bb.9: # %cond.store7
+; AVX2-NEXT:  .LBB16_9: # %cond.store7
 ; AVX2-NEXT:    vpextrb $4, %xmm0, 4(%rdi)
-; AVX2-NEXT:  .LBB16_10: # %else8
-; AVX2-NEXT:    vpextrb $5, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $32, %al
 ; AVX2-NEXT:    je .LBB16_12
-; AVX2-NEXT:  # %bb.11: # %cond.store9
+; AVX2-NEXT:  .LBB16_11: # %cond.store9
 ; AVX2-NEXT:    vpextrb $5, %xmm0, 5(%rdi)
-; AVX2-NEXT:  .LBB16_12: # %else10
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpextrb $6, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $64, %al
 ; AVX2-NEXT:    je .LBB16_14
-; AVX2-NEXT:  # %bb.13: # %cond.store11
+; AVX2-NEXT:  .LBB16_13: # %cond.store11
 ; AVX2-NEXT:    vpextrb $6, %xmm0, 6(%rdi)
-; AVX2-NEXT:  .LBB16_14: # %else12
-; AVX2-NEXT:    vpextrb $7, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $-128, %al
 ; AVX2-NEXT:    je .LBB16_16
-; AVX2-NEXT:  # %bb.15: # %cond.store13
+; AVX2-NEXT:  .LBB16_15: # %cond.store13
 ; AVX2-NEXT:    vpextrb $7, %xmm0, 7(%rdi)
-; AVX2-NEXT:  .LBB16_16: # %else14
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpextrb $8, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $256, %eax # imm = 0x100
 ; AVX2-NEXT:    je .LBB16_18
-; AVX2-NEXT:  # %bb.17: # %cond.store15
+; AVX2-NEXT:  .LBB16_17: # %cond.store15
 ; AVX2-NEXT:    vpextrb $8, %xmm0, 8(%rdi)
-; AVX2-NEXT:  .LBB16_18: # %else16
-; AVX2-NEXT:    vpextrb $9, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $512, %eax # imm = 0x200
 ; AVX2-NEXT:    je .LBB16_20
-; AVX2-NEXT:  # %bb.19: # %cond.store17
+; AVX2-NEXT:  .LBB16_19: # %cond.store17
 ; AVX2-NEXT:    vpextrb $9, %xmm0, 9(%rdi)
-; AVX2-NEXT:  .LBB16_20: # %else18
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpextrb $10, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $1024, %eax # imm = 0x400
 ; AVX2-NEXT:    je .LBB16_22
-; AVX2-NEXT:  # %bb.21: # %cond.store19
+; AVX2-NEXT:  .LBB16_21: # %cond.store19
 ; AVX2-NEXT:    vpextrb $10, %xmm0, 10(%rdi)
-; AVX2-NEXT:  .LBB16_22: # %else20
-; AVX2-NEXT:    vpextrb $11, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $2048, %eax # imm = 0x800
 ; AVX2-NEXT:    je .LBB16_24
-; AVX2-NEXT:  # %bb.23: # %cond.store21
+; AVX2-NEXT:  .LBB16_23: # %cond.store21
 ; AVX2-NEXT:    vpextrb $11, %xmm0, 11(%rdi)
-; AVX2-NEXT:  .LBB16_24: # %else22
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpextrb $12, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $4096, %eax # imm = 0x1000
 ; AVX2-NEXT:    je .LBB16_26
-; AVX2-NEXT:  # %bb.25: # %cond.store23
+; AVX2-NEXT:  .LBB16_25: # %cond.store23
 ; AVX2-NEXT:    vpextrb $12, %xmm0, 12(%rdi)
-; AVX2-NEXT:  .LBB16_26: # %else24
-; AVX2-NEXT:    vpextrb $13, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $8192, %eax # imm = 0x2000
 ; AVX2-NEXT:    je .LBB16_28
-; AVX2-NEXT:  # %bb.27: # %cond.store25
+; AVX2-NEXT:  .LBB16_27: # %cond.store25
 ; AVX2-NEXT:    vpextrb $13, %xmm0, 13(%rdi)
-; AVX2-NEXT:  .LBB16_28: # %else26
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpextrb $14, %xmm1, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $16384, %eax # imm = 0x4000
 ; AVX2-NEXT:    je .LBB16_30
-; AVX2-NEXT:  # %bb.29: # %cond.store27
+; AVX2-NEXT:  .LBB16_29: # %cond.store27
 ; AVX2-NEXT:    vpextrb $14, %xmm0, 14(%rdi)
-; AVX2-NEXT:  .LBB16_30: # %else28
-; AVX2-NEXT:    vpextrb $15, %xmm1, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $32768, %eax # imm = 0x8000
 ; AVX2-NEXT:    je .LBB16_32
-; AVX2-NEXT:  # %bb.31: # %cond.store29
+; AVX2-NEXT:  .LBB16_31: # %cond.store29
 ; AVX2-NEXT:    vpextrb $15, %xmm0, 15(%rdi)
-; AVX2-NEXT:  .LBB16_32: # %else30
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: truncstore_v16i16_v16i8:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm1
 ; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
 ; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
-; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    vpmovmskb %xmm1, %eax
+; AVX512F-NEXT:    xorl $65535, %eax # imm = 0xFFFF
 ; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB16_2
-; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    jne .LBB16_1
+; AVX512F-NEXT:  # %bb.2: # %else
+; AVX512F-NEXT:    testb $2, %al
+; AVX512F-NEXT:    jne .LBB16_3
+; AVX512F-NEXT:  .LBB16_4: # %else2
+; AVX512F-NEXT:    testb $4, %al
+; AVX512F-NEXT:    jne .LBB16_5
+; AVX512F-NEXT:  .LBB16_6: # %else4
+; AVX512F-NEXT:    testb $8, %al
+; AVX512F-NEXT:    jne .LBB16_7
+; AVX512F-NEXT:  .LBB16_8: # %else6
+; AVX512F-NEXT:    testb $16, %al
+; AVX512F-NEXT:    jne .LBB16_9
+; AVX512F-NEXT:  .LBB16_10: # %else8
+; AVX512F-NEXT:    testb $32, %al
+; AVX512F-NEXT:    jne .LBB16_11
+; AVX512F-NEXT:  .LBB16_12: # %else10
+; AVX512F-NEXT:    testb $64, %al
+; AVX512F-NEXT:    jne .LBB16_13
+; AVX512F-NEXT:  .LBB16_14: # %else12
+; AVX512F-NEXT:    testb $-128, %al
+; AVX512F-NEXT:    jne .LBB16_15
+; AVX512F-NEXT:  .LBB16_16: # %else14
+; AVX512F-NEXT:    testl $256, %eax # imm = 0x100
+; AVX512F-NEXT:    jne .LBB16_17
+; AVX512F-NEXT:  .LBB16_18: # %else16
+; AVX512F-NEXT:    testl $512, %eax # imm = 0x200
+; AVX512F-NEXT:    jne .LBB16_19
+; AVX512F-NEXT:  .LBB16_20: # %else18
+; AVX512F-NEXT:    testl $1024, %eax # imm = 0x400
+; AVX512F-NEXT:    jne .LBB16_21
+; AVX512F-NEXT:  .LBB16_22: # %else20
+; AVX512F-NEXT:    testl $2048, %eax # imm = 0x800
+; AVX512F-NEXT:    jne .LBB16_23
+; AVX512F-NEXT:  .LBB16_24: # %else22
+; AVX512F-NEXT:    testl $4096, %eax # imm = 0x1000
+; AVX512F-NEXT:    jne .LBB16_25
+; AVX512F-NEXT:  .LBB16_26: # %else24
+; AVX512F-NEXT:    testl $8192, %eax # imm = 0x2000
+; AVX512F-NEXT:    jne .LBB16_27
+; AVX512F-NEXT:  .LBB16_28: # %else26
+; AVX512F-NEXT:    testl $16384, %eax # imm = 0x4000
+; AVX512F-NEXT:    jne .LBB16_29
+; AVX512F-NEXT:  .LBB16_30: # %else28
+; AVX512F-NEXT:    testl $32768, %eax # imm = 0x8000
+; AVX512F-NEXT:    jne .LBB16_31
+; AVX512F-NEXT:  .LBB16_32: # %else30
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+; AVX512F-NEXT:  .LBB16_1: # %cond.store
 ; AVX512F-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX512F-NEXT:  .LBB16_2: # %else
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $2, %al
 ; AVX512F-NEXT:    je .LBB16_4
-; AVX512F-NEXT:  # %bb.3: # %cond.store1
+; AVX512F-NEXT:  .LBB16_3: # %cond.store1
 ; AVX512F-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
-; AVX512F-NEXT:  .LBB16_4: # %else2
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $4, %al
 ; AVX512F-NEXT:    je .LBB16_6
-; AVX512F-NEXT:  # %bb.5: # %cond.store3
+; AVX512F-NEXT:  .LBB16_5: # %cond.store3
 ; AVX512F-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
-; AVX512F-NEXT:  .LBB16_6: # %else4
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $8, %al
 ; AVX512F-NEXT:    je .LBB16_8
-; AVX512F-NEXT:  # %bb.7: # %cond.store5
+; AVX512F-NEXT:  .LBB16_7: # %cond.store5
 ; AVX512F-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
-; AVX512F-NEXT:  .LBB16_8: # %else6
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $4, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $16, %al
 ; AVX512F-NEXT:    je .LBB16_10
-; AVX512F-NEXT:  # %bb.9: # %cond.store7
+; AVX512F-NEXT:  .LBB16_9: # %cond.store7
 ; AVX512F-NEXT:    vpextrb $4, %xmm0, 4(%rdi)
-; AVX512F-NEXT:  .LBB16_10: # %else8
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $5, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $32, %al
 ; AVX512F-NEXT:    je .LBB16_12
-; AVX512F-NEXT:  # %bb.11: # %cond.store9
+; AVX512F-NEXT:  .LBB16_11: # %cond.store9
 ; AVX512F-NEXT:    vpextrb $5, %xmm0, 5(%rdi)
-; AVX512F-NEXT:  .LBB16_12: # %else10
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $6, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $64, %al
 ; AVX512F-NEXT:    je .LBB16_14
-; AVX512F-NEXT:  # %bb.13: # %cond.store11
+; AVX512F-NEXT:  .LBB16_13: # %cond.store11
 ; AVX512F-NEXT:    vpextrb $6, %xmm0, 6(%rdi)
-; AVX512F-NEXT:  .LBB16_14: # %else12
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $7, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $-128, %al
 ; AVX512F-NEXT:    je .LBB16_16
-; AVX512F-NEXT:  # %bb.15: # %cond.store13
+; AVX512F-NEXT:  .LBB16_15: # %cond.store13
 ; AVX512F-NEXT:    vpextrb $7, %xmm0, 7(%rdi)
-; AVX512F-NEXT:  .LBB16_16: # %else14
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $8, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $256, %eax # imm = 0x100
 ; AVX512F-NEXT:    je .LBB16_18
-; AVX512F-NEXT:  # %bb.17: # %cond.store15
+; AVX512F-NEXT:  .LBB16_17: # %cond.store15
 ; AVX512F-NEXT:    vpextrb $8, %xmm0, 8(%rdi)
-; AVX512F-NEXT:  .LBB16_18: # %else16
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $9, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $512, %eax # imm = 0x200
 ; AVX512F-NEXT:    je .LBB16_20
-; AVX512F-NEXT:  # %bb.19: # %cond.store17
+; AVX512F-NEXT:  .LBB16_19: # %cond.store17
 ; AVX512F-NEXT:    vpextrb $9, %xmm0, 9(%rdi)
-; AVX512F-NEXT:  .LBB16_20: # %else18
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $10, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $1024, %eax # imm = 0x400
 ; AVX512F-NEXT:    je .LBB16_22
-; AVX512F-NEXT:  # %bb.21: # %cond.store19
+; AVX512F-NEXT:  .LBB16_21: # %cond.store19
 ; AVX512F-NEXT:    vpextrb $10, %xmm0, 10(%rdi)
-; AVX512F-NEXT:  .LBB16_22: # %else20
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $11, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $2048, %eax # imm = 0x800
 ; AVX512F-NEXT:    je .LBB16_24
-; AVX512F-NEXT:  # %bb.23: # %cond.store21
+; AVX512F-NEXT:  .LBB16_23: # %cond.store21
 ; AVX512F-NEXT:    vpextrb $11, %xmm0, 11(%rdi)
-; AVX512F-NEXT:  .LBB16_24: # %else22
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $12, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $4096, %eax # imm = 0x1000
 ; AVX512F-NEXT:    je .LBB16_26
-; AVX512F-NEXT:  # %bb.25: # %cond.store23
+; AVX512F-NEXT:  .LBB16_25: # %cond.store23
 ; AVX512F-NEXT:    vpextrb $12, %xmm0, 12(%rdi)
-; AVX512F-NEXT:  .LBB16_26: # %else24
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $13, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $8192, %eax # imm = 0x2000
 ; AVX512F-NEXT:    je .LBB16_28
-; AVX512F-NEXT:  # %bb.27: # %cond.store25
+; AVX512F-NEXT:  .LBB16_27: # %cond.store25
 ; AVX512F-NEXT:    vpextrb $13, %xmm0, 13(%rdi)
-; AVX512F-NEXT:  .LBB16_28: # %else26
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm1
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm2
-; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $14, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $16384, %eax # imm = 0x4000
 ; AVX512F-NEXT:    je .LBB16_30
-; AVX512F-NEXT:  # %bb.29: # %cond.store27
+; AVX512F-NEXT:  .LBB16_29: # %cond.store27
 ; AVX512F-NEXT:    vpextrb $14, %xmm0, 14(%rdi)
-; AVX512F-NEXT:  .LBB16_30: # %else28
-; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
-; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $15, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $32768, %eax # imm = 0x8000
 ; AVX512F-NEXT:    je .LBB16_32
-; AVX512F-NEXT:  # %bb.31: # %cond.store29
+; AVX512F-NEXT:  .LBB16_31: # %cond.store29
 ; AVX512F-NEXT:    vpextrb $15, %xmm0, 15(%rdi)
-; AVX512F-NEXT:  .LBB16_32: # %else30
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
@@ -6985,308 +6275,268 @@ define void @truncstore_v8i16_v8i8(<8 x i16> %x, <8 x i8>* %p, <8 x i16> %mask)
 ; SSE2-LABEL: truncstore_v8i16_v8i8:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    packuswb %xmm0, %xmm0
 ; SSE2-NEXT:    pcmpeqw %xmm1, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm3
-; SSE2-NEXT:    pxor %xmm2, %xmm3
-; SSE2-NEXT:    movd %xmm3, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    je .LBB17_2
-; SSE2-NEXT:  # %bb.1: # %cond.store
-; SSE2-NEXT:    movb %al, (%rdi)
-; SSE2-NEXT:  .LBB17_2: # %else
-; SSE2-NEXT:    shrl $16, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    je .LBB17_4
-; SSE2-NEXT:  # %bb.3: # %cond.store1
-; SSE2-NEXT:    shrl $16, %eax
-; SSE2-NEXT:    movb %al, 1(%rdi)
-; SSE2-NEXT:  .LBB17_4: # %else2
-; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    pcmpeqw %xmm1, %xmm3
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT:    pxor %xmm3, %xmm2
-; SSE2-NEXT:    pextrw $2, %xmm2, %eax
+; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
+; SSE2-NEXT:    pxor %xmm2, %xmm1
+; SSE2-NEXT:    packsswb %xmm0, %xmm1
+; SSE2-NEXT:    pmovmskb %xmm1, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB17_6
-; SSE2-NEXT:  # %bb.5: # %cond.store3
-; SSE2-NEXT:    pextrw $2, %xmm0, %eax
-; SSE2-NEXT:    movb %al, 2(%rdi)
+; SSE2-NEXT:    movd %xmm0, %ecx
+; SSE2-NEXT:    jne .LBB17_1
+; SSE2-NEXT:  # %bb.2: # %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne .LBB17_3
+; SSE2-NEXT:  .LBB17_4: # %else2
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne .LBB17_5
 ; SSE2-NEXT:  .LBB17_6: # %else4
-; SSE2-NEXT:    pextrw $3, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je .LBB17_8
-; SSE2-NEXT:  # %bb.7: # %cond.store5
-; SSE2-NEXT:    pextrw $3, %xmm0, %eax
-; SSE2-NEXT:    movb %al, 3(%rdi)
+; SSE2-NEXT:  .LBB17_7: # %cond.store5
+; SSE2-NEXT:    shrl $24, %ecx
+; SSE2-NEXT:    movb %cl, 3(%rdi)
 ; SSE2-NEXT:  .LBB17_8: # %else6
-; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    pcmpeqw %xmm1, %xmm3
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT:    pxor %xmm3, %xmm2
-; SSE2-NEXT:    pextrw $4, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $16, %al
+; SSE2-NEXT:    pextrw $2, %xmm0, %ecx
 ; SSE2-NEXT:    je .LBB17_10
 ; SSE2-NEXT:  # %bb.9: # %cond.store7
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    movb %al, 4(%rdi)
+; SSE2-NEXT:    movb %cl, 4(%rdi)
 ; SSE2-NEXT:  .LBB17_10: # %else8
-; SSE2-NEXT:    pextrw $5, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $32, %al
 ; SSE2-NEXT:    je .LBB17_12
 ; SSE2-NEXT:  # %bb.11: # %cond.store9
-; SSE2-NEXT:    pextrw $5, %xmm0, %eax
-; SSE2-NEXT:    movb %al, 5(%rdi)
+; SSE2-NEXT:    movb %ch, 5(%rdi)
 ; SSE2-NEXT:  .LBB17_12: # %else10
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpeqw %xmm2, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT:    pxor %xmm1, %xmm2
-; SSE2-NEXT:    pextrw $6, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB17_14
-; SSE2-NEXT:  # %bb.13: # %cond.store11
-; SSE2-NEXT:    pextrw $6, %xmm0, %eax
-; SSE2-NEXT:    movb %al, 6(%rdi)
-; SSE2-NEXT:  .LBB17_14: # %else12
-; SSE2-NEXT:    pextrw $7, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB17_16
-; SSE2-NEXT:  # %bb.15: # %cond.store13
-; SSE2-NEXT:    pextrw $7, %xmm0, %eax
-; SSE2-NEXT:    movb %al, 7(%rdi)
+; SSE2-NEXT:    testb $64, %al
+; SSE2-NEXT:    pextrw $3, %xmm0, %ecx
+; SSE2-NEXT:    jne .LBB17_13
+; SSE2-NEXT:  # %bb.14: # %else12
+; SSE2-NEXT:    testb $-128, %al
+; SSE2-NEXT:    jne .LBB17_15
 ; SSE2-NEXT:  .LBB17_16: # %else14
 ; SSE2-NEXT:    retq
+; SSE2-NEXT:  .LBB17_1: # %cond.store
+; SSE2-NEXT:    movb %cl, (%rdi)
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    je .LBB17_4
+; SSE2-NEXT:  .LBB17_3: # %cond.store1
+; SSE2-NEXT:    movb %ch, 1(%rdi)
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    je .LBB17_6
+; SSE2-NEXT:  .LBB17_5: # %cond.store3
+; SSE2-NEXT:    movl %ecx, %edx
+; SSE2-NEXT:    shrl $16, %edx
+; SSE2-NEXT:    movb %dl, 2(%rdi)
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne .LBB17_7
+; SSE2-NEXT:    jmp .LBB17_8
+; SSE2-NEXT:  .LBB17_13: # %cond.store11
+; SSE2-NEXT:    movb %cl, 6(%rdi)
+; SSE2-NEXT:    testb $-128, %al
+; SSE2-NEXT:    je .LBB17_16
+; SSE2-NEXT:  .LBB17_15: # %cond.store13
+; SSE2-NEXT:    movb %ch, 7(%rdi)
+; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: truncstore_v8i16_v8i8:
 ; SSE4:       # %bb.0:
-; SSE4-NEXT:    pxor %xmm3, %xmm3
-; SSE4-NEXT:    pcmpeqw %xmm1, %xmm3
-; SSE4-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE4-NEXT:    pxor %xmm3, %xmm2
-; SSE4-NEXT:    pextrb $0, %xmm2, %eax
+; SSE4-NEXT:    pxor %xmm2, %xmm2
+; SSE4-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
+; SSE4-NEXT:    pcmpeqw %xmm1, %xmm2
+; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
+; SSE4-NEXT:    pxor %xmm2, %xmm1
+; SSE4-NEXT:    packsswb %xmm0, %xmm1
+; SSE4-NEXT:    pmovmskb %xmm1, %eax
 ; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB17_2
-; SSE4-NEXT:  # %bb.1: # %cond.store
+; SSE4-NEXT:    jne .LBB17_1
+; SSE4-NEXT:  # %bb.2: # %else
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    jne .LBB17_3
+; SSE4-NEXT:  .LBB17_4: # %else2
+; SSE4-NEXT:    testb $4, %al
+; SSE4-NEXT:    jne .LBB17_5
+; SSE4-NEXT:  .LBB17_6: # %else4
+; SSE4-NEXT:    testb $8, %al
+; SSE4-NEXT:    jne .LBB17_7
+; SSE4-NEXT:  .LBB17_8: # %else6
+; SSE4-NEXT:    testb $16, %al
+; SSE4-NEXT:    jne .LBB17_9
+; SSE4-NEXT:  .LBB17_10: # %else8
+; SSE4-NEXT:    testb $32, %al
+; SSE4-NEXT:    jne .LBB17_11
+; SSE4-NEXT:  .LBB17_12: # %else10
+; SSE4-NEXT:    testb $64, %al
+; SSE4-NEXT:    jne .LBB17_13
+; SSE4-NEXT:  .LBB17_14: # %else12
+; SSE4-NEXT:    testb $-128, %al
+; SSE4-NEXT:    jne .LBB17_15
+; SSE4-NEXT:  .LBB17_16: # %else14
+; SSE4-NEXT:    retq
+; SSE4-NEXT:  .LBB17_1: # %cond.store
 ; SSE4-NEXT:    pextrb $0, %xmm0, (%rdi)
-; SSE4-NEXT:  .LBB17_2: # %else
-; SSE4-NEXT:    pextrb $2, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $2, %al
 ; SSE4-NEXT:    je .LBB17_4
-; SSE4-NEXT:  # %bb.3: # %cond.store1
-; SSE4-NEXT:    pextrb $2, %xmm0, 1(%rdi)
-; SSE4-NEXT:  .LBB17_4: # %else2
-; SSE4-NEXT:    pxor %xmm3, %xmm3
-; SSE4-NEXT:    pcmpeqw %xmm1, %xmm3
-; SSE4-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE4-NEXT:    pxor %xmm3, %xmm2
-; SSE4-NEXT:    pextrb $4, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:  .LBB17_3: # %cond.store1
+; SSE4-NEXT:    pextrb $1, %xmm0, 1(%rdi)
+; SSE4-NEXT:    testb $4, %al
 ; SSE4-NEXT:    je .LBB17_6
-; SSE4-NEXT:  # %bb.5: # %cond.store3
-; SSE4-NEXT:    pextrb $4, %xmm0, 2(%rdi)
-; SSE4-NEXT:  .LBB17_6: # %else4
-; SSE4-NEXT:    pextrb $6, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:  .LBB17_5: # %cond.store3
+; SSE4-NEXT:    pextrb $2, %xmm0, 2(%rdi)
+; SSE4-NEXT:    testb $8, %al
 ; SSE4-NEXT:    je .LBB17_8
-; SSE4-NEXT:  # %bb.7: # %cond.store5
-; SSE4-NEXT:    pextrb $6, %xmm0, 3(%rdi)
-; SSE4-NEXT:  .LBB17_8: # %else6
-; SSE4-NEXT:    pxor %xmm3, %xmm3
-; SSE4-NEXT:    pcmpeqw %xmm1, %xmm3
-; SSE4-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE4-NEXT:    pxor %xmm3, %xmm2
-; SSE4-NEXT:    pextrb $8, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:  .LBB17_7: # %cond.store5
+; SSE4-NEXT:    pextrb $3, %xmm0, 3(%rdi)
+; SSE4-NEXT:    testb $16, %al
 ; SSE4-NEXT:    je .LBB17_10
-; SSE4-NEXT:  # %bb.9: # %cond.store7
-; SSE4-NEXT:    pextrb $8, %xmm0, 4(%rdi)
-; SSE4-NEXT:  .LBB17_10: # %else8
-; SSE4-NEXT:    pextrb $10, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:  .LBB17_9: # %cond.store7
+; SSE4-NEXT:    pextrb $4, %xmm0, 4(%rdi)
+; SSE4-NEXT:    testb $32, %al
 ; SSE4-NEXT:    je .LBB17_12
-; SSE4-NEXT:  # %bb.11: # %cond.store9
-; SSE4-NEXT:    pextrb $10, %xmm0, 5(%rdi)
-; SSE4-NEXT:  .LBB17_12: # %else10
-; SSE4-NEXT:    pxor %xmm2, %xmm2
-; SSE4-NEXT:    pcmpeqw %xmm2, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE4-NEXT:    pxor %xmm1, %xmm2
-; SSE4-NEXT:    pextrb $12, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:  .LBB17_11: # %cond.store9
+; SSE4-NEXT:    pextrb $5, %xmm0, 5(%rdi)
+; SSE4-NEXT:    testb $64, %al
 ; SSE4-NEXT:    je .LBB17_14
-; SSE4-NEXT:  # %bb.13: # %cond.store11
-; SSE4-NEXT:    pextrb $12, %xmm0, 6(%rdi)
-; SSE4-NEXT:  .LBB17_14: # %else12
-; SSE4-NEXT:    pextrb $14, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:  .LBB17_13: # %cond.store11
+; SSE4-NEXT:    pextrb $6, %xmm0, 6(%rdi)
+; SSE4-NEXT:    testb $-128, %al
 ; SSE4-NEXT:    je .LBB17_16
-; SSE4-NEXT:  # %bb.15: # %cond.store13
-; SSE4-NEXT:    pextrb $14, %xmm0, 7(%rdi)
-; SSE4-NEXT:  .LBB17_16: # %else14
+; SSE4-NEXT:  .LBB17_15: # %cond.store13
+; SSE4-NEXT:    pextrb $7, %xmm0, 7(%rdi)
 ; SSE4-NEXT:    retq
 ;
 ; AVX-LABEL: truncstore_v8i16_v8i8:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm2
-; AVX-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX-NEXT:    vpextrb $0, %xmm2, %eax
+; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
+; AVX-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm1
+; AVX-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX-NEXT:    vpxor %xmm2, %xmm1, %xmm1
+; AVX-NEXT:    vpacksswb %xmm0, %xmm1, %xmm1
+; AVX-NEXT:    vpmovmskb %xmm1, %eax
 ; AVX-NEXT:    testb $1, %al
-; AVX-NEXT:    je .LBB17_2
-; AVX-NEXT:  # %bb.1: # %cond.store
+; AVX-NEXT:    jne .LBB17_1
+; AVX-NEXT:  # %bb.2: # %else
+; AVX-NEXT:    testb $2, %al
+; AVX-NEXT:    jne .LBB17_3
+; AVX-NEXT:  .LBB17_4: # %else2
+; AVX-NEXT:    testb $4, %al
+; AVX-NEXT:    jne .LBB17_5
+; AVX-NEXT:  .LBB17_6: # %else4
+; AVX-NEXT:    testb $8, %al
+; AVX-NEXT:    jne .LBB17_7
+; AVX-NEXT:  .LBB17_8: # %else6
+; AVX-NEXT:    testb $16, %al
+; AVX-NEXT:    jne .LBB17_9
+; AVX-NEXT:  .LBB17_10: # %else8
+; AVX-NEXT:    testb $32, %al
+; AVX-NEXT:    jne .LBB17_11
+; AVX-NEXT:  .LBB17_12: # %else10
+; AVX-NEXT:    testb $64, %al
+; AVX-NEXT:    jne .LBB17_13
+; AVX-NEXT:  .LBB17_14: # %else12
+; AVX-NEXT:    testb $-128, %al
+; AVX-NEXT:    jne .LBB17_15
+; AVX-NEXT:  .LBB17_16: # %else14
+; AVX-NEXT:    retq
+; AVX-NEXT:  .LBB17_1: # %cond.store
 ; AVX-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX-NEXT:  .LBB17_2: # %else
-; AVX-NEXT:    vpextrb $2, %xmm2, %eax
-; AVX-NEXT:    testb $1, %al
+; AVX-NEXT:    testb $2, %al
 ; AVX-NEXT:    je .LBB17_4
-; AVX-NEXT:  # %bb.3: # %cond.store1
-; AVX-NEXT:    vpextrb $2, %xmm0, 1(%rdi)
-; AVX-NEXT:  .LBB17_4: # %else2
-; AVX-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm2
-; AVX-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX-NEXT:    testb $1, %al
+; AVX-NEXT:  .LBB17_3: # %cond.store1
+; AVX-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
+; AVX-NEXT:    testb $4, %al
 ; AVX-NEXT:    je .LBB17_6
-; AVX-NEXT:  # %bb.5: # %cond.store3
-; AVX-NEXT:    vpextrb $4, %xmm0, 2(%rdi)
-; AVX-NEXT:  .LBB17_6: # %else4
-; AVX-NEXT:    vpextrb $6, %xmm2, %eax
-; AVX-NEXT:    testb $1, %al
+; AVX-NEXT:  .LBB17_5: # %cond.store3
+; AVX-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
+; AVX-NEXT:    testb $8, %al
 ; AVX-NEXT:    je .LBB17_8
-; AVX-NEXT:  # %bb.7: # %cond.store5
-; AVX-NEXT:    vpextrb $6, %xmm0, 3(%rdi)
-; AVX-NEXT:  .LBB17_8: # %else6
-; AVX-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm2
-; AVX-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX-NEXT:    vpextrb $8, %xmm2, %eax
-; AVX-NEXT:    testb $1, %al
+; AVX-NEXT:  .LBB17_7: # %cond.store5
+; AVX-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
+; AVX-NEXT:    testb $16, %al
 ; AVX-NEXT:    je .LBB17_10
-; AVX-NEXT:  # %bb.9: # %cond.store7
-; AVX-NEXT:    vpextrb $8, %xmm0, 4(%rdi)
-; AVX-NEXT:  .LBB17_10: # %else8
-; AVX-NEXT:    vpextrb $10, %xmm2, %eax
-; AVX-NEXT:    testb $1, %al
+; AVX-NEXT:  .LBB17_9: # %cond.store7
+; AVX-NEXT:    vpextrb $4, %xmm0, 4(%rdi)
+; AVX-NEXT:    testb $32, %al
 ; AVX-NEXT:    je .LBB17_12
-; AVX-NEXT:  # %bb.11: # %cond.store9
-; AVX-NEXT:    vpextrb $10, %xmm0, 5(%rdi)
-; AVX-NEXT:  .LBB17_12: # %else10
-; AVX-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm1
-; AVX-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX-NEXT:    testb $1, %al
+; AVX-NEXT:  .LBB17_11: # %cond.store9
+; AVX-NEXT:    vpextrb $5, %xmm0, 5(%rdi)
+; AVX-NEXT:    testb $64, %al
 ; AVX-NEXT:    je .LBB17_14
-; AVX-NEXT:  # %bb.13: # %cond.store11
-; AVX-NEXT:    vpextrb $12, %xmm0, 6(%rdi)
-; AVX-NEXT:  .LBB17_14: # %else12
-; AVX-NEXT:    vpextrb $14, %xmm1, %eax
-; AVX-NEXT:    testb $1, %al
+; AVX-NEXT:  .LBB17_13: # %cond.store11
+; AVX-NEXT:    vpextrb $6, %xmm0, 6(%rdi)
+; AVX-NEXT:    testb $-128, %al
 ; AVX-NEXT:    je .LBB17_16
-; AVX-NEXT:  # %bb.15: # %cond.store13
-; AVX-NEXT:    vpextrb $14, %xmm0, 7(%rdi)
-; AVX-NEXT:  .LBB17_16: # %else14
+; AVX-NEXT:  .LBB17_15: # %cond.store13
+; AVX-NEXT:    vpextrb $7, %xmm0, 7(%rdi)
 ; AVX-NEXT:    retq
 ;
 ; AVX512F-LABEL: truncstore_v8i16_v8i8:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
-; AVX512F-NEXT:    vpmovsxwq %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmq %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm1
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
+; AVX512F-NEXT:    vpmovsxwq %xmm1, %zmm1
+; AVX512F-NEXT:    vptestmq %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
 ; AVX512F-NEXT:    kmovw %k0, %eax
 ; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB17_2
-; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    jne .LBB17_1
+; AVX512F-NEXT:  # %bb.2: # %else
+; AVX512F-NEXT:    testb $2, %al
+; AVX512F-NEXT:    jne .LBB17_3
+; AVX512F-NEXT:  .LBB17_4: # %else2
+; AVX512F-NEXT:    testb $4, %al
+; AVX512F-NEXT:    jne .LBB17_5
+; AVX512F-NEXT:  .LBB17_6: # %else4
+; AVX512F-NEXT:    testb $8, %al
+; AVX512F-NEXT:    jne .LBB17_7
+; AVX512F-NEXT:  .LBB17_8: # %else6
+; AVX512F-NEXT:    testb $16, %al
+; AVX512F-NEXT:    jne .LBB17_9
+; AVX512F-NEXT:  .LBB17_10: # %else8
+; AVX512F-NEXT:    testb $32, %al
+; AVX512F-NEXT:    jne .LBB17_11
+; AVX512F-NEXT:  .LBB17_12: # %else10
+; AVX512F-NEXT:    testb $64, %al
+; AVX512F-NEXT:    jne .LBB17_13
+; AVX512F-NEXT:  .LBB17_14: # %else12
+; AVX512F-NEXT:    testb $-128, %al
+; AVX512F-NEXT:    jne .LBB17_15
+; AVX512F-NEXT:  .LBB17_16: # %else14
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+; AVX512F-NEXT:  .LBB17_1: # %cond.store
 ; AVX512F-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX512F-NEXT:  .LBB17_2: # %else
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
-; AVX512F-NEXT:    vpmovsxwq %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmq %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $2, %al
 ; AVX512F-NEXT:    je .LBB17_4
-; AVX512F-NEXT:  # %bb.3: # %cond.store1
-; AVX512F-NEXT:    vpextrb $2, %xmm0, 1(%rdi)
-; AVX512F-NEXT:  .LBB17_4: # %else2
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
-; AVX512F-NEXT:    vpmovsxwq %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmq %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  .LBB17_3: # %cond.store1
+; AVX512F-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
+; AVX512F-NEXT:    testb $4, %al
 ; AVX512F-NEXT:    je .LBB17_6
-; AVX512F-NEXT:  # %bb.5: # %cond.store3
-; AVX512F-NEXT:    vpextrb $4, %xmm0, 2(%rdi)
-; AVX512F-NEXT:  .LBB17_6: # %else4
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
-; AVX512F-NEXT:    vpmovsxwq %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmq %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  .LBB17_5: # %cond.store3
+; AVX512F-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
+; AVX512F-NEXT:    testb $8, %al
 ; AVX512F-NEXT:    je .LBB17_8
-; AVX512F-NEXT:  # %bb.7: # %cond.store5
-; AVX512F-NEXT:    vpextrb $6, %xmm0, 3(%rdi)
-; AVX512F-NEXT:  .LBB17_8: # %else6
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
-; AVX512F-NEXT:    vpmovsxwq %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmq %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $4, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  .LBB17_7: # %cond.store5
+; AVX512F-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
+; AVX512F-NEXT:    testb $16, %al
 ; AVX512F-NEXT:    je .LBB17_10
-; AVX512F-NEXT:  # %bb.9: # %cond.store7
-; AVX512F-NEXT:    vpextrb $8, %xmm0, 4(%rdi)
-; AVX512F-NEXT:  .LBB17_10: # %else8
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
-; AVX512F-NEXT:    vpmovsxwq %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmq %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $5, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  .LBB17_9: # %cond.store7
+; AVX512F-NEXT:    vpextrb $4, %xmm0, 4(%rdi)
+; AVX512F-NEXT:    testb $32, %al
 ; AVX512F-NEXT:    je .LBB17_12
-; AVX512F-NEXT:  # %bb.11: # %cond.store9
-; AVX512F-NEXT:    vpextrb $10, %xmm0, 5(%rdi)
-; AVX512F-NEXT:  .LBB17_12: # %else10
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm1
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm2
-; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm2
-; AVX512F-NEXT:    vpmovsxwq %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmq %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $6, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  .LBB17_11: # %cond.store9
+; AVX512F-NEXT:    vpextrb $5, %xmm0, 5(%rdi)
+; AVX512F-NEXT:    testb $64, %al
 ; AVX512F-NEXT:    je .LBB17_14
-; AVX512F-NEXT:  # %bb.13: # %cond.store11
-; AVX512F-NEXT:    vpextrb $12, %xmm0, 6(%rdi)
-; AVX512F-NEXT:  .LBB17_14: # %else12
-; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
-; AVX512F-NEXT:    vpmovsxwq %xmm1, %zmm1
-; AVX512F-NEXT:    vptestmq %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $7, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  .LBB17_13: # %cond.store11
+; AVX512F-NEXT:    vpextrb $6, %xmm0, 6(%rdi)
+; AVX512F-NEXT:    testb $-128, %al
 ; AVX512F-NEXT:    je .LBB17_16
-; AVX512F-NEXT:  # %bb.15: # %cond.store13
-; AVX512F-NEXT:    vpextrb $14, %xmm0, 7(%rdi)
-; AVX512F-NEXT:  .LBB17_16: # %else14
+; AVX512F-NEXT:  .LBB17_15: # %cond.store13
+; AVX512F-NEXT:    vpextrb $7, %xmm0, 7(%rdi)
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
@@ -7294,9 +6544,9 @@ define void @truncstore_v8i16_v8i8(<8 x i16> %x, <8 x i8>* %p, <8 x i16> %mask)
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
 ; AVX512BW-NEXT:    vptestmw %zmm1, %zmm1, %k0
-; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
 ; AVX512BW-NEXT:    kshiftlq $56, %k0, %k0
 ; AVX512BW-NEXT:    kshiftrq $56, %k0, %k1
+; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
 ; AVX512BW-NEXT:    vmovdqu8 %zmm0, (%rdi) {%k1}
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/masked_store_trunc_ssat.ll b/llvm/test/CodeGen/X86/masked_store_trunc_ssat.ll
index a010d066a..b79fb5c35 100644
--- a/llvm/test/CodeGen/X86/masked_store_trunc_ssat.ll
+++ b/llvm/test/CodeGen/X86/masked_store_trunc_ssat.ll
@@ -11,140 +11,134 @@
 define void @truncstore_v8i64_v8i32(<8 x i64> %x, <8 x i32>* %p, <8 x i32> %mask) {
 ; SSE2-LABEL: truncstore_v8i64_v8i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pxor %xmm6, %xmm6
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm6
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm9
-; SSE2-NEXT:    pxor %xmm6, %xmm9
-; SSE2-NEXT:    movdqa %xmm9, %xmm8
-; SSE2-NEXT:    packssdw %xmm0, %xmm8
-; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [2147483647,2147483647]
-; SSE2-NEXT:    movdqa {{.*#+}} xmm12 = [2147483648,2147483648]
+; SSE2-NEXT:    pxor %xmm8, %xmm8
+; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [2147483647,2147483647]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm11 = [2147483648,2147483648]
 ; SSE2-NEXT:    movdqa %xmm2, %xmm6
-; SSE2-NEXT:    pxor %xmm12, %xmm6
-; SSE2-NEXT:    movdqa {{.*#+}} xmm11 = [4294967295,4294967295]
-; SSE2-NEXT:    movdqa %xmm11, %xmm7
+; SSE2-NEXT:    pxor %xmm11, %xmm6
+; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [4294967295,4294967295]
+; SSE2-NEXT:    movdqa %xmm10, %xmm7
 ; SSE2-NEXT:    pcmpgtd %xmm6, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm7[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm11, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm7[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm6
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT:    pand %xmm13, %xmm6
+; SSE2-NEXT:    pand %xmm12, %xmm6
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm7[1,1,3,3]
 ; SSE2-NEXT:    por %xmm6, %xmm13
 ; SSE2-NEXT:    pand %xmm13, %xmm2
-; SSE2-NEXT:    pandn %xmm10, %xmm13
+; SSE2-NEXT:    pandn %xmm9, %xmm13
 ; SSE2-NEXT:    por %xmm2, %xmm13
 ; SSE2-NEXT:    movdqa %xmm3, %xmm2
-; SSE2-NEXT:    pxor %xmm12, %xmm2
-; SSE2-NEXT:    movdqa %xmm11, %xmm6
+; SSE2-NEXT:    pxor %xmm11, %xmm2
+; SSE2-NEXT:    movdqa %xmm10, %xmm6
 ; SSE2-NEXT:    pcmpgtd %xmm2, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm14 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm11, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm6[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm2
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm2[1,1,3,3]
-; SSE2-NEXT:    pand %xmm14, %xmm7
+; SSE2-NEXT:    pand %xmm12, %xmm7
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[1,1,3,3]
 ; SSE2-NEXT:    por %xmm7, %xmm2
 ; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pandn %xmm10, %xmm2
+; SSE2-NEXT:    pandn %xmm9, %xmm2
 ; SSE2-NEXT:    por %xmm3, %xmm2
 ; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    pxor %xmm12, %xmm3
-; SSE2-NEXT:    movdqa %xmm11, %xmm6
+; SSE2-NEXT:    pxor %xmm11, %xmm3
+; SSE2-NEXT:    movdqa %xmm10, %xmm6
 ; SSE2-NEXT:    pcmpgtd %xmm3, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm14 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm11, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm6[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm3
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm3[1,1,3,3]
-; SSE2-NEXT:    pand %xmm14, %xmm7
+; SSE2-NEXT:    pand %xmm12, %xmm7
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm6[1,1,3,3]
 ; SSE2-NEXT:    por %xmm7, %xmm3
 ; SSE2-NEXT:    pand %xmm3, %xmm0
-; SSE2-NEXT:    pandn %xmm10, %xmm3
+; SSE2-NEXT:    pandn %xmm9, %xmm3
 ; SSE2-NEXT:    por %xmm0, %xmm3
 ; SSE2-NEXT:    movdqa %xmm1, %xmm0
-; SSE2-NEXT:    pxor %xmm12, %xmm0
-; SSE2-NEXT:    movdqa %xmm11, %xmm6
+; SSE2-NEXT:    pxor %xmm11, %xmm0
+; SSE2-NEXT:    movdqa %xmm10, %xmm6
 ; SSE2-NEXT:    pcmpgtd %xmm0, %xmm6
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm11, %xmm0
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm7, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
 ; SSE2-NEXT:    por %xmm0, %xmm6
 ; SSE2-NEXT:    pand %xmm6, %xmm1
-; SSE2-NEXT:    pandn %xmm10, %xmm6
+; SSE2-NEXT:    pandn %xmm9, %xmm6
 ; SSE2-NEXT:    por %xmm1, %xmm6
-; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [18446744071562067968,18446744071562067968]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [18446744071562067968,18446744071562067968]
 ; SSE2-NEXT:    movdqa %xmm6, %xmm1
-; SSE2-NEXT:    pxor %xmm12, %xmm1
-; SSE2-NEXT:    movdqa {{.*#+}} xmm14 = [18446744069414584320,18446744069414584320]
+; SSE2-NEXT:    pxor %xmm11, %xmm1
+; SSE2-NEXT:    movdqa {{.*#+}} xmm12 = [18446744069414584320,18446744069414584320]
 ; SSE2-NEXT:    movdqa %xmm1, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm14, %xmm7
+; SSE2-NEXT:    pcmpgtd %xmm12, %xmm7
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm14, %xmm1
+; SSE2-NEXT:    pcmpeqd %xmm12, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm0, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[1,1,3,3]
 ; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    pand %xmm0, %xmm6
-; SSE2-NEXT:    pandn %xmm10, %xmm0
+; SSE2-NEXT:    pandn %xmm9, %xmm0
 ; SSE2-NEXT:    por %xmm6, %xmm0
 ; SSE2-NEXT:    movdqa %xmm3, %xmm1
-; SSE2-NEXT:    pxor %xmm12, %xmm1
+; SSE2-NEXT:    pxor %xmm11, %xmm1
 ; SSE2-NEXT:    movdqa %xmm1, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm14, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm14, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm12, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm6[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm12, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pand %xmm11, %xmm7
+; SSE2-NEXT:    pand %xmm10, %xmm7
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm6[1,1,3,3]
 ; SSE2-NEXT:    por %xmm7, %xmm1
 ; SSE2-NEXT:    pand %xmm1, %xmm3
-; SSE2-NEXT:    pandn %xmm10, %xmm1
+; SSE2-NEXT:    pandn %xmm9, %xmm1
 ; SSE2-NEXT:    por %xmm3, %xmm1
 ; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,2],xmm0[0,2]
 ; SSE2-NEXT:    movdqa %xmm2, %xmm0
-; SSE2-NEXT:    pxor %xmm12, %xmm0
+; SSE2-NEXT:    pxor %xmm11, %xmm0
 ; SSE2-NEXT:    movdqa %xmm0, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm14, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm12, %xmm6
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm14, %xmm0
+; SSE2-NEXT:    pcmpeqd %xmm12, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm7, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm6[1,1,3,3]
-; SSE2-NEXT:    pxor %xmm13, %xmm12
-; SSE2-NEXT:    movdqa %xmm12, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm14, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm6[1,1,3,3]
+; SSE2-NEXT:    pxor %xmm13, %xmm11
+; SSE2-NEXT:    movdqa %xmm11, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm12, %xmm6
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm14, %xmm12
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm12[1,1,3,3]
+; SSE2-NEXT:    pcmpeqd %xmm12, %xmm11
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm11[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm7, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[1,1,3,3]
-; SSE2-NEXT:    movd %xmm8, %eax
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm5
+; SSE2-NEXT:    pcmpeqd %xmm7, %xmm7
+; SSE2-NEXT:    pxor %xmm7, %xmm5
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm4
+; SSE2-NEXT:    pxor %xmm7, %xmm4
+; SSE2-NEXT:    packssdw %xmm5, %xmm4
+; SSE2-NEXT:    packsswb %xmm0, %xmm4
+; SSE2-NEXT:    pmovmskb %xmm4, %eax
 ; SSE2-NEXT:    testb $1, %al
 ; SSE2-NEXT:    je .LBB0_2
 ; SSE2-NEXT:  # %bb.1: # %cond.store
 ; SSE2-NEXT:    movss %xmm1, (%rdi)
 ; SSE2-NEXT:  .LBB0_2: # %else
-; SSE2-NEXT:    por %xmm11, %xmm3
-; SSE2-NEXT:    por %xmm7, %xmm0
-; SSE2-NEXT:    psrlq $16, %xmm9
-; SSE2-NEXT:    movd %xmm9, %eax
-; SSE2-NEXT:    shrl $16, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    por %xmm10, %xmm3
+; SSE2-NEXT:    por %xmm6, %xmm0
+; SSE2-NEXT:    testb $2, %al
 ; SSE2-NEXT:    je .LBB0_4
 ; SSE2-NEXT:  # %bb.3: # %cond.store1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[1,1,2,3]
-; SSE2-NEXT:    movd %xmm6, 4(%rdi)
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,2,3]
+; SSE2-NEXT:    movd %xmm4, 4(%rdi)
 ; SSE2-NEXT:  .LBB0_4: # %else2
 ; SSE2-NEXT:    pand %xmm3, %xmm2
-; SSE2-NEXT:    pandn %xmm10, %xmm3
+; SSE2-NEXT:    pandn %xmm9, %xmm3
 ; SSE2-NEXT:    pand %xmm0, %xmm13
-; SSE2-NEXT:    pandn %xmm10, %xmm0
-; SSE2-NEXT:    pxor %xmm6, %xmm6
-; SSE2-NEXT:    pcmpeqd %xmm6, %xmm4
-; SSE2-NEXT:    pcmpeqd %xmm7, %xmm7
-; SSE2-NEXT:    pxor %xmm4, %xmm7
-; SSE2-NEXT:    pextrw $4, %xmm7, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    pandn %xmm9, %xmm0
+; SSE2-NEXT:    testb $4, %al
 ; SSE2-NEXT:    je .LBB0_6
 ; SSE2-NEXT:  # %bb.5: # %cond.store3
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[2,3,0,1]
@@ -152,58 +146,49 @@ define void @truncstore_v8i64_v8i32(<8 x i64> %x, <8 x i32>* %p, <8 x i32> %mask
 ; SSE2-NEXT:  .LBB0_6: # %else4
 ; SSE2-NEXT:    por %xmm3, %xmm2
 ; SSE2-NEXT:    por %xmm0, %xmm13
-; SSE2-NEXT:    pextrw $6, %xmm7, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je .LBB0_8
 ; SSE2-NEXT:  # %bb.7: # %cond.store5
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[3,1,2,3]
 ; SSE2-NEXT:    movd %xmm0, 12(%rdi)
 ; SSE2-NEXT:  .LBB0_8: # %else6
 ; SSE2-NEXT:    shufps {{.*#+}} xmm13 = xmm13[0,2],xmm2[0,2]
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE2-NEXT:    pxor %xmm1, %xmm0
-; SSE2-NEXT:    pextrw $0, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB0_10
-; SSE2-NEXT:  # %bb.9: # %cond.store7
+; SSE2-NEXT:    testb $16, %al
+; SSE2-NEXT:    jne .LBB0_9
+; SSE2-NEXT:  # %bb.10: # %else8
+; SSE2-NEXT:    testb $32, %al
+; SSE2-NEXT:    jne .LBB0_11
+; SSE2-NEXT:  .LBB0_12: # %else10
+; SSE2-NEXT:    testb $64, %al
+; SSE2-NEXT:    jne .LBB0_13
+; SSE2-NEXT:  .LBB0_14: # %else12
+; SSE2-NEXT:    testb $-128, %al
+; SSE2-NEXT:    jne .LBB0_15
+; SSE2-NEXT:  .LBB0_16: # %else14
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  .LBB0_9: # %cond.store7
 ; SSE2-NEXT:    movss %xmm13, 16(%rdi)
-; SSE2-NEXT:  .LBB0_10: # %else8
-; SSE2-NEXT:    pextrw $2, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $32, %al
 ; SSE2-NEXT:    je .LBB0_12
-; SSE2-NEXT:  # %bb.11: # %cond.store9
+; SSE2-NEXT:  .LBB0_11: # %cond.store9
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm13[1,1,2,3]
 ; SSE2-NEXT:    movd %xmm0, 20(%rdi)
-; SSE2-NEXT:  .LBB0_12: # %else10
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm5
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE2-NEXT:    pxor %xmm5, %xmm0
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $64, %al
 ; SSE2-NEXT:    je .LBB0_14
-; SSE2-NEXT:  # %bb.13: # %cond.store11
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm13[2,3,0,1]
-; SSE2-NEXT:    movd %xmm1, 24(%rdi)
-; SSE2-NEXT:  .LBB0_14: # %else12
-; SSE2-NEXT:    pextrw $6, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:  .LBB0_13: # %cond.store11
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm13[2,3,0,1]
+; SSE2-NEXT:    movd %xmm0, 24(%rdi)
+; SSE2-NEXT:    testb $-128, %al
 ; SSE2-NEXT:    je .LBB0_16
-; SSE2-NEXT:  # %bb.15: # %cond.store13
+; SSE2-NEXT:  .LBB0_15: # %cond.store13
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm13[3,1,2,3]
 ; SSE2-NEXT:    movd %xmm0, 28(%rdi)
-; SSE2-NEXT:  .LBB0_16: # %else14
 ; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: truncstore_v8i64_v8i32:
 ; SSE4:       # %bb.0:
 ; SSE4-NEXT:    movdqa %xmm0, %xmm9
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqd %xmm4, %xmm0
-; SSE4-NEXT:    pcmpeqd %xmm8, %xmm8
-; SSE4-NEXT:    pxor %xmm0, %xmm8
+; SSE4-NEXT:    pxor %xmm8, %xmm8
 ; SSE4-NEXT:    movdqa {{.*#+}} xmm7 = [2147483647,2147483647]
 ; SSE4-NEXT:    movdqa %xmm7, %xmm0
 ; SSE4-NEXT:    pcmpgtq %xmm2, %xmm0
@@ -220,85 +205,87 @@ define void @truncstore_v8i64_v8i32(<8 x i64> %x, <8 x i32>* %p, <8 x i32> %mask
 ; SSE4-NEXT:    movdqa %xmm7, %xmm0
 ; SSE4-NEXT:    pcmpgtq %xmm1, %xmm0
 ; SSE4-NEXT:    blendvpd %xmm0, %xmm1, %xmm7
-; SSE4-NEXT:    movdqa {{.*#+}} xmm2 = [18446744071562067968,18446744071562067968]
+; SSE4-NEXT:    movdqa {{.*#+}} xmm1 = [18446744071562067968,18446744071562067968]
 ; SSE4-NEXT:    movapd %xmm7, %xmm0
-; SSE4-NEXT:    pcmpgtq %xmm2, %xmm0
-; SSE4-NEXT:    movdqa %xmm2, %xmm1
-; SSE4-NEXT:    blendvpd %xmm0, %xmm7, %xmm1
+; SSE4-NEXT:    pcmpgtq %xmm1, %xmm0
+; SSE4-NEXT:    movdqa %xmm1, %xmm2
+; SSE4-NEXT:    blendvpd %xmm0, %xmm7, %xmm2
 ; SSE4-NEXT:    movapd %xmm3, %xmm0
-; SSE4-NEXT:    pcmpgtq %xmm2, %xmm0
-; SSE4-NEXT:    movdqa %xmm2, %xmm7
+; SSE4-NEXT:    pcmpgtq %xmm1, %xmm0
+; SSE4-NEXT:    movdqa %xmm1, %xmm7
 ; SSE4-NEXT:    blendvpd %xmm0, %xmm3, %xmm7
-; SSE4-NEXT:    shufps {{.*#+}} xmm7 = xmm7[0,2],xmm1[0,2]
+; SSE4-NEXT:    shufps {{.*#+}} xmm7 = xmm7[0,2],xmm2[0,2]
 ; SSE4-NEXT:    movapd %xmm6, %xmm0
-; SSE4-NEXT:    pcmpgtq %xmm2, %xmm0
-; SSE4-NEXT:    movdqa %xmm2, %xmm3
+; SSE4-NEXT:    pcmpgtq %xmm1, %xmm0
+; SSE4-NEXT:    movdqa %xmm1, %xmm3
 ; SSE4-NEXT:    blendvpd %xmm0, %xmm6, %xmm3
 ; SSE4-NEXT:    movapd %xmm10, %xmm0
-; SSE4-NEXT:    pcmpgtq %xmm2, %xmm0
-; SSE4-NEXT:    blendvpd %xmm0, %xmm10, %xmm2
-; SSE4-NEXT:    pextrb $0, %xmm8, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB0_2
-; SSE4-NEXT:  # %bb.1: # %cond.store
-; SSE4-NEXT:    movss %xmm7, (%rdi)
-; SSE4-NEXT:  .LBB0_2: # %else
-; SSE4-NEXT:    pextrb $4, %xmm8, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB0_4
-; SSE4-NEXT:  # %bb.3: # %cond.store1
-; SSE4-NEXT:    extractps $1, %xmm7, 4(%rdi)
-; SSE4-NEXT:  .LBB0_4: # %else2
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE4-NEXT:    pcmpgtq %xmm1, %xmm0
+; SSE4-NEXT:    blendvpd %xmm0, %xmm10, %xmm1
+; SSE4-NEXT:    pcmpeqd %xmm8, %xmm5
 ; SSE4-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE4-NEXT:    pxor %xmm4, %xmm0
-; SSE4-NEXT:    pextrb $8, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB0_6
-; SSE4-NEXT:  # %bb.5: # %cond.store3
-; SSE4-NEXT:    extractps $2, %xmm7, 8(%rdi)
+; SSE4-NEXT:    pxor %xmm0, %xmm5
+; SSE4-NEXT:    pcmpeqd %xmm8, %xmm4
+; SSE4-NEXT:    pxor %xmm0, %xmm4
+; SSE4-NEXT:    packssdw %xmm5, %xmm4
+; SSE4-NEXT:    packsswb %xmm0, %xmm4
+; SSE4-NEXT:    pmovmskb %xmm4, %eax
+; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    jne .LBB0_1
+; SSE4-NEXT:  # %bb.2: # %else
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    jne .LBB0_3
+; SSE4-NEXT:  .LBB0_4: # %else2
+; SSE4-NEXT:    testb $4, %al
+; SSE4-NEXT:    jne .LBB0_5
 ; SSE4-NEXT:  .LBB0_6: # %else4
-; SSE4-NEXT:    pextrb $12, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $8, %al
 ; SSE4-NEXT:    je .LBB0_8
-; SSE4-NEXT:  # %bb.7: # %cond.store5
+; SSE4-NEXT:  .LBB0_7: # %cond.store5
 ; SSE4-NEXT:    extractps $3, %xmm7, 12(%rdi)
 ; SSE4-NEXT:  .LBB0_8: # %else6
-; SSE4-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,2],xmm3[0,2]
-; SSE4-NEXT:    xorps %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm5, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE4-NEXT:    pxor %xmm1, %xmm0
-; SSE4-NEXT:    pextrb $0, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB0_10
-; SSE4-NEXT:  # %bb.9: # %cond.store7
-; SSE4-NEXT:    movss %xmm2, 16(%rdi)
-; SSE4-NEXT:  .LBB0_10: # %else8
-; SSE4-NEXT:    pextrb $4, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB0_12
-; SSE4-NEXT:  # %bb.11: # %cond.store9
-; SSE4-NEXT:    extractps $1, %xmm2, 20(%rdi)
+; SSE4-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,2],xmm3[0,2]
+; SSE4-NEXT:    testb $16, %al
+; SSE4-NEXT:    jne .LBB0_9
+; SSE4-NEXT:  # %bb.10: # %else8
+; SSE4-NEXT:    testb $32, %al
+; SSE4-NEXT:    jne .LBB0_11
 ; SSE4-NEXT:  .LBB0_12: # %else10
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqd %xmm0, %xmm5
-; SSE4-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE4-NEXT:    pxor %xmm5, %xmm0
-; SSE4-NEXT:    pextrb $8, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB0_14
-; SSE4-NEXT:  # %bb.13: # %cond.store11
-; SSE4-NEXT:    extractps $2, %xmm2, 24(%rdi)
+; SSE4-NEXT:    testb $64, %al
+; SSE4-NEXT:    jne .LBB0_13
 ; SSE4-NEXT:  .LBB0_14: # %else12
-; SSE4-NEXT:    pextrb $12, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB0_16
-; SSE4-NEXT:  # %bb.15: # %cond.store13
-; SSE4-NEXT:    extractps $3, %xmm2, 28(%rdi)
+; SSE4-NEXT:    testb $-128, %al
+; SSE4-NEXT:    jne .LBB0_15
 ; SSE4-NEXT:  .LBB0_16: # %else14
 ; SSE4-NEXT:    retq
+; SSE4-NEXT:  .LBB0_1: # %cond.store
+; SSE4-NEXT:    movss %xmm7, (%rdi)
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    je .LBB0_4
+; SSE4-NEXT:  .LBB0_3: # %cond.store1
+; SSE4-NEXT:    extractps $1, %xmm7, 4(%rdi)
+; SSE4-NEXT:    testb $4, %al
+; SSE4-NEXT:    je .LBB0_6
+; SSE4-NEXT:  .LBB0_5: # %cond.store3
+; SSE4-NEXT:    extractps $2, %xmm7, 8(%rdi)
+; SSE4-NEXT:    testb $8, %al
+; SSE4-NEXT:    jne .LBB0_7
+; SSE4-NEXT:    jmp .LBB0_8
+; SSE4-NEXT:  .LBB0_9: # %cond.store7
+; SSE4-NEXT:    movss %xmm1, 16(%rdi)
+; SSE4-NEXT:    testb $32, %al
+; SSE4-NEXT:    je .LBB0_12
+; SSE4-NEXT:  .LBB0_11: # %cond.store9
+; SSE4-NEXT:    extractps $1, %xmm1, 20(%rdi)
+; SSE4-NEXT:    testb $64, %al
+; SSE4-NEXT:    je .LBB0_14
+; SSE4-NEXT:  .LBB0_13: # %cond.store11
+; SSE4-NEXT:    extractps $2, %xmm1, 24(%rdi)
+; SSE4-NEXT:    testb $-128, %al
+; SSE4-NEXT:    je .LBB0_16
+; SSE4-NEXT:  .LBB0_15: # %cond.store13
+; SSE4-NEXT:    extractps $3, %xmm1, 28(%rdi)
+; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: truncstore_v8i64_v8i32:
 ; AVX1:       # %bb.0:
@@ -403,202 +390,195 @@ define void @truncstore_v8i64_v8i32(<8 x i64> %x, <8 x i32>* %p, <8 x i32> %mask
 define void @truncstore_v8i64_v8i16(<8 x i64> %x, <8 x i16>* %p, <8 x i32> %mask) {
 ; SSE2-LABEL: truncstore_v8i64_v8i16:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pxor %xmm6, %xmm6
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm6
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm9
-; SSE2-NEXT:    pxor %xmm6, %xmm9
-; SSE2-NEXT:    movdqa %xmm9, %xmm8
-; SSE2-NEXT:    packssdw %xmm0, %xmm8
-; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [32767,32767]
-; SSE2-NEXT:    movdqa {{.*#+}} xmm12 = [2147483648,2147483648]
+; SSE2-NEXT:    pxor %xmm8, %xmm8
+; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [32767,32767]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm11 = [2147483648,2147483648]
 ; SSE2-NEXT:    movdqa %xmm2, %xmm6
-; SSE2-NEXT:    pxor %xmm12, %xmm6
-; SSE2-NEXT:    movdqa {{.*#+}} xmm11 = [2147516415,2147516415]
-; SSE2-NEXT:    movdqa %xmm11, %xmm7
+; SSE2-NEXT:    pxor %xmm11, %xmm6
+; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [2147516415,2147516415]
+; SSE2-NEXT:    movdqa %xmm10, %xmm7
 ; SSE2-NEXT:    pcmpgtd %xmm6, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm7[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm11, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm7[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm6
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT:    pand %xmm13, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm14 = xmm7[1,1,3,3]
-; SSE2-NEXT:    por %xmm6, %xmm14
-; SSE2-NEXT:    pand %xmm14, %xmm2
-; SSE2-NEXT:    pandn %xmm10, %xmm14
-; SSE2-NEXT:    por %xmm2, %xmm14
+; SSE2-NEXT:    pand %xmm12, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm7[1,1,3,3]
+; SSE2-NEXT:    por %xmm6, %xmm13
+; SSE2-NEXT:    pand %xmm13, %xmm2
+; SSE2-NEXT:    pandn %xmm9, %xmm13
+; SSE2-NEXT:    por %xmm2, %xmm13
 ; SSE2-NEXT:    movdqa %xmm3, %xmm2
-; SSE2-NEXT:    pxor %xmm12, %xmm2
-; SSE2-NEXT:    movdqa %xmm11, %xmm6
+; SSE2-NEXT:    pxor %xmm11, %xmm2
+; SSE2-NEXT:    movdqa %xmm10, %xmm6
 ; SSE2-NEXT:    pcmpgtd %xmm2, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm11, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm6[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm2
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm2[1,1,3,3]
-; SSE2-NEXT:    pand %xmm13, %xmm7
+; SSE2-NEXT:    pand %xmm12, %xmm7
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[1,1,3,3]
 ; SSE2-NEXT:    por %xmm7, %xmm2
 ; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pandn %xmm10, %xmm2
+; SSE2-NEXT:    pandn %xmm9, %xmm2
 ; SSE2-NEXT:    por %xmm3, %xmm2
 ; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    pxor %xmm12, %xmm3
-; SSE2-NEXT:    movdqa %xmm11, %xmm6
+; SSE2-NEXT:    pxor %xmm11, %xmm3
+; SSE2-NEXT:    movdqa %xmm10, %xmm6
 ; SSE2-NEXT:    pcmpgtd %xmm3, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm11, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm6[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm3
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm3[1,1,3,3]
-; SSE2-NEXT:    pand %xmm13, %xmm7
+; SSE2-NEXT:    pand %xmm12, %xmm7
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm6[1,1,3,3]
 ; SSE2-NEXT:    por %xmm7, %xmm3
 ; SSE2-NEXT:    pand %xmm3, %xmm0
-; SSE2-NEXT:    pandn %xmm10, %xmm3
+; SSE2-NEXT:    pandn %xmm9, %xmm3
 ; SSE2-NEXT:    por %xmm0, %xmm3
 ; SSE2-NEXT:    movdqa %xmm1, %xmm0
-; SSE2-NEXT:    pxor %xmm12, %xmm0
-; SSE2-NEXT:    movdqa %xmm11, %xmm6
+; SSE2-NEXT:    pxor %xmm11, %xmm0
+; SSE2-NEXT:    movdqa %xmm10, %xmm6
 ; SSE2-NEXT:    pcmpgtd %xmm0, %xmm6
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm11, %xmm0
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm7, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
 ; SSE2-NEXT:    por %xmm0, %xmm6
 ; SSE2-NEXT:    pand %xmm6, %xmm1
-; SSE2-NEXT:    pandn %xmm10, %xmm6
+; SSE2-NEXT:    pandn %xmm9, %xmm6
 ; SSE2-NEXT:    por %xmm1, %xmm6
-; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [18446744073709518848,18446744073709518848]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [18446744073709518848,18446744073709518848]
 ; SSE2-NEXT:    movdqa %xmm6, %xmm0
-; SSE2-NEXT:    pxor %xmm12, %xmm0
-; SSE2-NEXT:    movdqa {{.*#+}} xmm11 = [18446744071562035200,18446744071562035200]
+; SSE2-NEXT:    pxor %xmm11, %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [18446744071562035200,18446744071562035200]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm11, %xmm7
+; SSE2-NEXT:    pcmpgtd %xmm10, %xmm7
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm7[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm11, %xmm0
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm1, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm7[1,1,3,3]
 ; SSE2-NEXT:    por %xmm0, %xmm1
 ; SSE2-NEXT:    pand %xmm1, %xmm6
-; SSE2-NEXT:    pandn %xmm10, %xmm1
+; SSE2-NEXT:    pandn %xmm9, %xmm1
 ; SSE2-NEXT:    por %xmm6, %xmm1
 ; SSE2-NEXT:    movdqa %xmm3, %xmm0
-; SSE2-NEXT:    pxor %xmm12, %xmm0
+; SSE2-NEXT:    pxor %xmm11, %xmm0
 ; SSE2-NEXT:    movdqa %xmm0, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm11, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm11, %xmm0
+; SSE2-NEXT:    pcmpgtd %xmm10, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm6[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pand %xmm13, %xmm7
+; SSE2-NEXT:    pand %xmm12, %xmm7
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3]
 ; SSE2-NEXT:    por %xmm7, %xmm0
 ; SSE2-NEXT:    pand %xmm0, %xmm3
-; SSE2-NEXT:    pandn %xmm10, %xmm0
+; SSE2-NEXT:    pandn %xmm9, %xmm0
 ; SSE2-NEXT:    por %xmm3, %xmm0
 ; SSE2-NEXT:    packssdw %xmm1, %xmm0
 ; SSE2-NEXT:    movdqa %xmm2, %xmm1
-; SSE2-NEXT:    pxor %xmm12, %xmm1
+; SSE2-NEXT:    pxor %xmm11, %xmm1
 ; SSE2-NEXT:    movdqa %xmm1, %xmm3
-; SSE2-NEXT:    pcmpgtd %xmm11, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm10, %xmm3
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm11, %xmm1
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm6, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
 ; SSE2-NEXT:    por %xmm1, %xmm3
 ; SSE2-NEXT:    pand %xmm3, %xmm2
-; SSE2-NEXT:    pandn %xmm10, %xmm3
+; SSE2-NEXT:    pandn %xmm9, %xmm3
 ; SSE2-NEXT:    por %xmm2, %xmm3
-; SSE2-NEXT:    pxor %xmm14, %xmm12
-; SSE2-NEXT:    movdqa %xmm12, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm11, %xmm1
+; SSE2-NEXT:    pxor %xmm13, %xmm11
+; SSE2-NEXT:    movdqa %xmm11, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm10, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm11, %xmm12
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm12[1,1,3,3]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm11
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm11[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm2, %xmm6
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
 ; SSE2-NEXT:    por %xmm6, %xmm1
-; SSE2-NEXT:    pand %xmm1, %xmm14
-; SSE2-NEXT:    pandn %xmm10, %xmm1
-; SSE2-NEXT:    por %xmm14, %xmm1
+; SSE2-NEXT:    pand %xmm1, %xmm13
+; SSE2-NEXT:    pandn %xmm9, %xmm1
+; SSE2-NEXT:    por %xmm13, %xmm1
 ; SSE2-NEXT:    packssdw %xmm3, %xmm1
 ; SSE2-NEXT:    packssdw %xmm1, %xmm0
-; SSE2-NEXT:    movd %xmm8, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB1_2
-; SSE2-NEXT:  # %bb.1: # %cond.store
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    movw %ax, (%rdi)
-; SSE2-NEXT:  .LBB1_2: # %else
-; SSE2-NEXT:    psrlq $16, %xmm9
-; SSE2-NEXT:    movd %xmm9, %eax
-; SSE2-NEXT:    shrl $16, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB1_4
-; SSE2-NEXT:  # %bb.3: # %cond.store1
-; SSE2-NEXT:    pextrw $1, %xmm0, %eax
-; SSE2-NEXT:    movw %ax, 2(%rdi)
-; SSE2-NEXT:  .LBB1_4: # %else2
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm4
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm5
 ; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pxor %xmm4, %xmm1
-; SSE2-NEXT:    pextrw $4, %xmm1, %eax
+; SSE2-NEXT:    pxor %xmm1, %xmm5
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm4
+; SSE2-NEXT:    pxor %xmm1, %xmm4
+; SSE2-NEXT:    packssdw %xmm5, %xmm4
+; SSE2-NEXT:    packsswb %xmm0, %xmm4
+; SSE2-NEXT:    pmovmskb %xmm4, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB1_6
-; SSE2-NEXT:  # %bb.5: # %cond.store3
-; SSE2-NEXT:    pextrw $2, %xmm0, %eax
-; SSE2-NEXT:    movw %ax, 4(%rdi)
+; SSE2-NEXT:    jne .LBB1_1
+; SSE2-NEXT:  # %bb.2: # %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne .LBB1_3
+; SSE2-NEXT:  .LBB1_4: # %else2
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne .LBB1_5
 ; SSE2-NEXT:  .LBB1_6: # %else4
-; SSE2-NEXT:    pextrw $6, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB1_8
-; SSE2-NEXT:  # %bb.7: # %cond.store5
-; SSE2-NEXT:    pextrw $3, %xmm0, %eax
-; SSE2-NEXT:    movw %ax, 6(%rdi)
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne .LBB1_7
 ; SSE2-NEXT:  .LBB1_8: # %else6
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pxor %xmm2, %xmm1
-; SSE2-NEXT:    pextrw $0, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB1_10
-; SSE2-NEXT:  # %bb.9: # %cond.store7
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    movw %ax, 8(%rdi)
+; SSE2-NEXT:    testb $16, %al
+; SSE2-NEXT:    jne .LBB1_9
 ; SSE2-NEXT:  .LBB1_10: # %else8
-; SSE2-NEXT:    pextrw $2, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB1_12
-; SSE2-NEXT:  # %bb.11: # %cond.store9
-; SSE2-NEXT:    pextrw $5, %xmm0, %eax
-; SSE2-NEXT:    movw %ax, 10(%rdi)
+; SSE2-NEXT:    testb $32, %al
+; SSE2-NEXT:    jne .LBB1_11
 ; SSE2-NEXT:  .LBB1_12: # %else10
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm5
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pxor %xmm5, %xmm1
-; SSE2-NEXT:    pextrw $4, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB1_14
-; SSE2-NEXT:  # %bb.13: # %cond.store11
-; SSE2-NEXT:    pextrw $6, %xmm0, %eax
-; SSE2-NEXT:    movw %ax, 12(%rdi)
+; SSE2-NEXT:    testb $64, %al
+; SSE2-NEXT:    jne .LBB1_13
 ; SSE2-NEXT:  .LBB1_14: # %else12
-; SSE2-NEXT:    pextrw $6, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $-128, %al
+; SSE2-NEXT:    jne .LBB1_15
+; SSE2-NEXT:  .LBB1_16: # %else14
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  .LBB1_1: # %cond.store
+; SSE2-NEXT:    movd %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, (%rdi)
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    je .LBB1_4
+; SSE2-NEXT:  .LBB1_3: # %cond.store1
+; SSE2-NEXT:    pextrw $1, %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, 2(%rdi)
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    je .LBB1_6
+; SSE2-NEXT:  .LBB1_5: # %cond.store3
+; SSE2-NEXT:    pextrw $2, %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, 4(%rdi)
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    je .LBB1_8
+; SSE2-NEXT:  .LBB1_7: # %cond.store5
+; SSE2-NEXT:    pextrw $3, %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, 6(%rdi)
+; SSE2-NEXT:    testb $16, %al
+; SSE2-NEXT:    je .LBB1_10
+; SSE2-NEXT:  .LBB1_9: # %cond.store7
+; SSE2-NEXT:    pextrw $4, %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, 8(%rdi)
+; SSE2-NEXT:    testb $32, %al
+; SSE2-NEXT:    je .LBB1_12
+; SSE2-NEXT:  .LBB1_11: # %cond.store9
+; SSE2-NEXT:    pextrw $5, %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, 10(%rdi)
+; SSE2-NEXT:    testb $64, %al
+; SSE2-NEXT:    je .LBB1_14
+; SSE2-NEXT:  .LBB1_13: # %cond.store11
+; SSE2-NEXT:    pextrw $6, %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, 12(%rdi)
+; SSE2-NEXT:    testb $-128, %al
 ; SSE2-NEXT:    je .LBB1_16
-; SSE2-NEXT:  # %bb.15: # %cond.store13
+; SSE2-NEXT:  .LBB1_15: # %cond.store13
 ; SSE2-NEXT:    pextrw $7, %xmm0, %eax
 ; SSE2-NEXT:    movw %ax, 14(%rdi)
-; SSE2-NEXT:  .LBB1_16: # %else14
 ; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: truncstore_v8i64_v8i16:
 ; SSE4:       # %bb.0:
 ; SSE4-NEXT:    movdqa %xmm0, %xmm9
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqd %xmm4, %xmm0
-; SSE4-NEXT:    pcmpeqd %xmm8, %xmm8
-; SSE4-NEXT:    pxor %xmm0, %xmm8
+; SSE4-NEXT:    pxor %xmm8, %xmm8
 ; SSE4-NEXT:    movdqa {{.*#+}} xmm7 = [32767,32767]
 ; SSE4-NEXT:    movdqa %xmm7, %xmm0
 ; SSE4-NEXT:    pcmpgtq %xmm2, %xmm0
@@ -634,250 +614,238 @@ define void @truncstore_v8i64_v8i16(<8 x i64> %x, <8 x i16>* %p, <8 x i32> %mask
 ; SSE4-NEXT:    blendvpd %xmm0, %xmm10, %xmm1
 ; SSE4-NEXT:    packssdw %xmm3, %xmm1
 ; SSE4-NEXT:    packssdw %xmm1, %xmm7
-; SSE4-NEXT:    pextrb $0, %xmm8, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB1_2
-; SSE4-NEXT:  # %bb.1: # %cond.store
+; SSE4-NEXT:    pcmpeqd %xmm8, %xmm5
+; SSE4-NEXT:    pcmpeqd %xmm0, %xmm0
+; SSE4-NEXT:    pxor %xmm0, %xmm5
+; SSE4-NEXT:    pcmpeqd %xmm8, %xmm4
+; SSE4-NEXT:    pxor %xmm0, %xmm4
+; SSE4-NEXT:    packssdw %xmm5, %xmm4
+; SSE4-NEXT:    packsswb %xmm0, %xmm4
+; SSE4-NEXT:    pmovmskb %xmm4, %eax
+; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    jne .LBB1_1
+; SSE4-NEXT:  # %bb.2: # %else
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    jne .LBB1_3
+; SSE4-NEXT:  .LBB1_4: # %else2
+; SSE4-NEXT:    testb $4, %al
+; SSE4-NEXT:    jne .LBB1_5
+; SSE4-NEXT:  .LBB1_6: # %else4
+; SSE4-NEXT:    testb $8, %al
+; SSE4-NEXT:    jne .LBB1_7
+; SSE4-NEXT:  .LBB1_8: # %else6
+; SSE4-NEXT:    testb $16, %al
+; SSE4-NEXT:    jne .LBB1_9
+; SSE4-NEXT:  .LBB1_10: # %else8
+; SSE4-NEXT:    testb $32, %al
+; SSE4-NEXT:    jne .LBB1_11
+; SSE4-NEXT:  .LBB1_12: # %else10
+; SSE4-NEXT:    testb $64, %al
+; SSE4-NEXT:    jne .LBB1_13
+; SSE4-NEXT:  .LBB1_14: # %else12
+; SSE4-NEXT:    testb $-128, %al
+; SSE4-NEXT:    jne .LBB1_15
+; SSE4-NEXT:  .LBB1_16: # %else14
+; SSE4-NEXT:    retq
+; SSE4-NEXT:  .LBB1_1: # %cond.store
 ; SSE4-NEXT:    pextrw $0, %xmm7, (%rdi)
-; SSE4-NEXT:  .LBB1_2: # %else
-; SSE4-NEXT:    pextrb $4, %xmm8, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $2, %al
 ; SSE4-NEXT:    je .LBB1_4
-; SSE4-NEXT:  # %bb.3: # %cond.store1
+; SSE4-NEXT:  .LBB1_3: # %cond.store1
 ; SSE4-NEXT:    pextrw $1, %xmm7, 2(%rdi)
-; SSE4-NEXT:  .LBB1_4: # %else2
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqd %xmm0, %xmm4
-; SSE4-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE4-NEXT:    pxor %xmm4, %xmm0
-; SSE4-NEXT:    pextrb $8, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $4, %al
 ; SSE4-NEXT:    je .LBB1_6
-; SSE4-NEXT:  # %bb.5: # %cond.store3
+; SSE4-NEXT:  .LBB1_5: # %cond.store3
 ; SSE4-NEXT:    pextrw $2, %xmm7, 4(%rdi)
-; SSE4-NEXT:  .LBB1_6: # %else4
-; SSE4-NEXT:    pextrb $12, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $8, %al
 ; SSE4-NEXT:    je .LBB1_8
-; SSE4-NEXT:  # %bb.7: # %cond.store5
+; SSE4-NEXT:  .LBB1_7: # %cond.store5
 ; SSE4-NEXT:    pextrw $3, %xmm7, 6(%rdi)
-; SSE4-NEXT:  .LBB1_8: # %else6
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm5, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE4-NEXT:    pxor %xmm1, %xmm0
-; SSE4-NEXT:    pextrb $0, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $16, %al
 ; SSE4-NEXT:    je .LBB1_10
-; SSE4-NEXT:  # %bb.9: # %cond.store7
+; SSE4-NEXT:  .LBB1_9: # %cond.store7
 ; SSE4-NEXT:    pextrw $4, %xmm7, 8(%rdi)
-; SSE4-NEXT:  .LBB1_10: # %else8
-; SSE4-NEXT:    pextrb $4, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $32, %al
 ; SSE4-NEXT:    je .LBB1_12
-; SSE4-NEXT:  # %bb.11: # %cond.store9
+; SSE4-NEXT:  .LBB1_11: # %cond.store9
 ; SSE4-NEXT:    pextrw $5, %xmm7, 10(%rdi)
-; SSE4-NEXT:  .LBB1_12: # %else10
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqd %xmm0, %xmm5
-; SSE4-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE4-NEXT:    pxor %xmm5, %xmm0
-; SSE4-NEXT:    pextrb $8, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $64, %al
 ; SSE4-NEXT:    je .LBB1_14
-; SSE4-NEXT:  # %bb.13: # %cond.store11
+; SSE4-NEXT:  .LBB1_13: # %cond.store11
 ; SSE4-NEXT:    pextrw $6, %xmm7, 12(%rdi)
-; SSE4-NEXT:  .LBB1_14: # %else12
-; SSE4-NEXT:    pextrb $12, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $-128, %al
 ; SSE4-NEXT:    je .LBB1_16
-; SSE4-NEXT:  # %bb.15: # %cond.store13
+; SSE4-NEXT:  .LBB1_15: # %cond.store13
 ; SSE4-NEXT:    pextrw $7, %xmm7, 14(%rdi)
-; SSE4-NEXT:  .LBB1_16: # %else14
 ; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: truncstore_v8i64_v8i16:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpxor %xmm8, %xmm8, %xmm8
-; AVX1-NEXT:    vpcmpeqd %xmm8, %xmm2, %xmm5
-; AVX1-NEXT:    vpcmpeqd %xmm9, %xmm9, %xmm9
-; AVX1-NEXT:    vpxor %xmm9, %xmm5, %xmm10
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm6
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm7 = [32767,32767]
-; AVX1-NEXT:    vpcmpgtq %xmm6, %xmm7, %xmm11
-; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm7, %xmm12
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm5
-; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm7, %xmm3
-; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm7, %xmm4
-; AVX1-NEXT:    vblendvpd %xmm4, %xmm0, %xmm7, %xmm0
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [18446744073709518848,18446744073709518848]
-; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm0, %xmm13
-; AVX1-NEXT:    vblendvpd %xmm3, %xmm5, %xmm7, %xmm3
-; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm3, %xmm14
-; AVX1-NEXT:    vblendvpd %xmm12, %xmm1, %xmm7, %xmm1
-; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm1, %xmm5
-; AVX1-NEXT:    vblendvpd %xmm11, %xmm6, %xmm7, %xmm6
-; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm6, %xmm7
-; AVX1-NEXT:    vblendvpd %xmm7, %xmm6, %xmm4, %xmm6
-; AVX1-NEXT:    vblendvpd %xmm5, %xmm1, %xmm4, %xmm1
-; AVX1-NEXT:    vpackssdw %xmm6, %xmm1, %xmm1
-; AVX1-NEXT:    vblendvpd %xmm14, %xmm3, %xmm4, %xmm3
-; AVX1-NEXT:    vblendvpd %xmm13, %xmm0, %xmm4, %xmm0
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [32767,32767]
+; AVX1-NEXT:    vpcmpgtq %xmm3, %xmm4, %xmm8
+; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm4, %xmm9
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm7
+; AVX1-NEXT:    vpcmpgtq %xmm7, %xmm4, %xmm5
+; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm4, %xmm6
+; AVX1-NEXT:    vblendvpd %xmm6, %xmm0, %xmm4, %xmm0
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [18446744073709518848,18446744073709518848]
+; AVX1-NEXT:    vpcmpgtq %xmm6, %xmm0, %xmm10
+; AVX1-NEXT:    vblendvpd %xmm5, %xmm7, %xmm4, %xmm5
+; AVX1-NEXT:    vpcmpgtq %xmm6, %xmm5, %xmm11
+; AVX1-NEXT:    vblendvpd %xmm9, %xmm1, %xmm4, %xmm1
+; AVX1-NEXT:    vpcmpgtq %xmm6, %xmm1, %xmm7
+; AVX1-NEXT:    vblendvpd %xmm8, %xmm3, %xmm4, %xmm3
+; AVX1-NEXT:    vpcmpgtq %xmm6, %xmm3, %xmm4
+; AVX1-NEXT:    vblendvpd %xmm4, %xmm3, %xmm6, %xmm3
+; AVX1-NEXT:    vblendvpd %xmm7, %xmm1, %xmm6, %xmm1
+; AVX1-NEXT:    vpackssdw %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vblendvpd %xmm11, %xmm5, %xmm6, %xmm3
+; AVX1-NEXT:    vblendvpd %xmm10, %xmm0, %xmm6, %xmm0
 ; AVX1-NEXT:    vpackssdw %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpextrb $0, %xmm10, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je .LBB1_2
-; AVX1-NEXT:  # %bb.1: # %cond.store
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm1
+; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
+; AVX1-NEXT:    vmovmskps %ymm1, %eax
+; AVX1-NEXT:    notl %eax
+; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    jne .LBB1_1
+; AVX1-NEXT:  # %bb.2: # %else
+; AVX1-NEXT:    testb $2, %al
+; AVX1-NEXT:    jne .LBB1_3
+; AVX1-NEXT:  .LBB1_4: # %else2
+; AVX1-NEXT:    testb $4, %al
+; AVX1-NEXT:    jne .LBB1_5
+; AVX1-NEXT:  .LBB1_6: # %else4
+; AVX1-NEXT:    testb $8, %al
+; AVX1-NEXT:    jne .LBB1_7
+; AVX1-NEXT:  .LBB1_8: # %else6
+; AVX1-NEXT:    testb $16, %al
+; AVX1-NEXT:    jne .LBB1_9
+; AVX1-NEXT:  .LBB1_10: # %else8
+; AVX1-NEXT:    testb $32, %al
+; AVX1-NEXT:    jne .LBB1_11
+; AVX1-NEXT:  .LBB1_12: # %else10
+; AVX1-NEXT:    testb $64, %al
+; AVX1-NEXT:    jne .LBB1_13
+; AVX1-NEXT:  .LBB1_14: # %else12
+; AVX1-NEXT:    testb $-128, %al
+; AVX1-NEXT:    jne .LBB1_15
+; AVX1-NEXT:  .LBB1_16: # %else14
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+; AVX1-NEXT:  .LBB1_1: # %cond.store
 ; AVX1-NEXT:    vpextrw $0, %xmm0, (%rdi)
-; AVX1-NEXT:  .LBB1_2: # %else
-; AVX1-NEXT:    vpcmpeqd %xmm8, %xmm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm9, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $4, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $2, %al
 ; AVX1-NEXT:    je .LBB1_4
-; AVX1-NEXT:  # %bb.3: # %cond.store1
+; AVX1-NEXT:  .LBB1_3: # %cond.store1
 ; AVX1-NEXT:    vpextrw $1, %xmm0, 2(%rdi)
-; AVX1-NEXT:  .LBB1_4: # %else2
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm2, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm4, %xmm4
-; AVX1-NEXT:    vpextrb $8, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $4, %al
 ; AVX1-NEXT:    je .LBB1_6
-; AVX1-NEXT:  # %bb.5: # %cond.store3
+; AVX1-NEXT:  .LBB1_5: # %cond.store3
 ; AVX1-NEXT:    vpextrw $2, %xmm0, 4(%rdi)
-; AVX1-NEXT:  .LBB1_6: # %else4
-; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $8, %al
 ; AVX1-NEXT:    je .LBB1_8
-; AVX1-NEXT:  # %bb.7: # %cond.store5
+; AVX1-NEXT:  .LBB1_7: # %cond.store5
 ; AVX1-NEXT:    vpextrw $3, %xmm0, 6(%rdi)
-; AVX1-NEXT:  .LBB1_8: # %else6
-; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $0, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $16, %al
 ; AVX1-NEXT:    je .LBB1_10
-; AVX1-NEXT:  # %bb.9: # %cond.store7
+; AVX1-NEXT:  .LBB1_9: # %cond.store7
 ; AVX1-NEXT:    vpextrw $4, %xmm0, 8(%rdi)
-; AVX1-NEXT:  .LBB1_10: # %else8
-; AVX1-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $32, %al
 ; AVX1-NEXT:    je .LBB1_12
-; AVX1-NEXT:  # %bb.11: # %cond.store9
+; AVX1-NEXT:  .LBB1_11: # %cond.store9
 ; AVX1-NEXT:    vpextrw $5, %xmm0, 10(%rdi)
-; AVX1-NEXT:  .LBB1_12: # %else10
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $64, %al
 ; AVX1-NEXT:    je .LBB1_14
-; AVX1-NEXT:  # %bb.13: # %cond.store11
+; AVX1-NEXT:  .LBB1_13: # %cond.store11
 ; AVX1-NEXT:    vpextrw $6, %xmm0, 12(%rdi)
-; AVX1-NEXT:  .LBB1_14: # %else12
-; AVX1-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $-128, %al
 ; AVX1-NEXT:    je .LBB1_16
-; AVX1-NEXT:  # %bb.15: # %cond.store13
+; AVX1-NEXT:  .LBB1_15: # %cond.store13
 ; AVX1-NEXT:    vpextrw $7, %xmm0, 14(%rdi)
-; AVX1-NEXT:  .LBB1_16: # %else14
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: truncstore_v8i64_v8i16:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm5
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm6 = [32767,32767,32767,32767]
-; AVX2-NEXT:    vpcmpgtq %ymm0, %ymm6, %ymm7
-; AVX2-NEXT:    vblendvpd %ymm7, %ymm0, %ymm6, %ymm0
-; AVX2-NEXT:    vpcmpgtq %ymm1, %ymm6, %ymm7
-; AVX2-NEXT:    vblendvpd %ymm7, %ymm1, %ymm6, %ymm1
-; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm6 = [18446744073709518848,18446744073709518848,18446744073709518848,18446744073709518848]
-; AVX2-NEXT:    vpcmpgtq %ymm6, %ymm1, %ymm7
-; AVX2-NEXT:    vblendvpd %ymm7, %ymm1, %ymm6, %ymm1
-; AVX2-NEXT:    vpcmpgtq %ymm6, %ymm0, %ymm7
-; AVX2-NEXT:    vblendvpd %ymm7, %ymm0, %ymm6, %ymm0
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [32767,32767,32767,32767]
+; AVX2-NEXT:    vpcmpgtq %ymm0, %ymm4, %ymm5
+; AVX2-NEXT:    vblendvpd %ymm5, %ymm0, %ymm4, %ymm0
+; AVX2-NEXT:    vpcmpgtq %ymm1, %ymm4, %ymm5
+; AVX2-NEXT:    vblendvpd %ymm5, %ymm1, %ymm4, %ymm1
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [18446744073709518848,18446744073709518848,18446744073709518848,18446744073709518848]
+; AVX2-NEXT:    vpcmpgtq %ymm4, %ymm1, %ymm5
+; AVX2-NEXT:    vblendvpd %ymm5, %ymm1, %ymm4, %ymm1
+; AVX2-NEXT:    vpcmpgtq %ymm4, %ymm0, %ymm5
+; AVX2-NEXT:    vblendvpd %ymm5, %ymm0, %ymm4, %ymm0
 ; AVX2-NEXT:    vpackssdw %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpextrb $0, %xmm5, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je .LBB1_2
-; AVX2-NEXT:  # %bb.1: # %cond.store
+; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm2, %ymm1
+; AVX2-NEXT:    vmovmskps %ymm1, %eax
+; AVX2-NEXT:    notl %eax
+; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    jne .LBB1_1
+; AVX2-NEXT:  # %bb.2: # %else
+; AVX2-NEXT:    testb $2, %al
+; AVX2-NEXT:    jne .LBB1_3
+; AVX2-NEXT:  .LBB1_4: # %else2
+; AVX2-NEXT:    testb $4, %al
+; AVX2-NEXT:    jne .LBB1_5
+; AVX2-NEXT:  .LBB1_6: # %else4
+; AVX2-NEXT:    testb $8, %al
+; AVX2-NEXT:    jne .LBB1_7
+; AVX2-NEXT:  .LBB1_8: # %else6
+; AVX2-NEXT:    testb $16, %al
+; AVX2-NEXT:    jne .LBB1_9
+; AVX2-NEXT:  .LBB1_10: # %else8
+; AVX2-NEXT:    testb $32, %al
+; AVX2-NEXT:    jne .LBB1_11
+; AVX2-NEXT:  .LBB1_12: # %else10
+; AVX2-NEXT:    testb $64, %al
+; AVX2-NEXT:    jne .LBB1_13
+; AVX2-NEXT:  .LBB1_14: # %else12
+; AVX2-NEXT:    testb $-128, %al
+; AVX2-NEXT:    jne .LBB1_15
+; AVX2-NEXT:  .LBB1_16: # %else14
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+; AVX2-NEXT:  .LBB1_1: # %cond.store
 ; AVX2-NEXT:    vpextrw $0, %xmm0, (%rdi)
-; AVX2-NEXT:  .LBB1_2: # %else
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm1
-; AVX2-NEXT:    vpxor %xmm4, %xmm1, %xmm1
-; AVX2-NEXT:    vpextrb $4, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $2, %al
 ; AVX2-NEXT:    je .LBB1_4
-; AVX2-NEXT:  # %bb.3: # %cond.store1
+; AVX2-NEXT:  .LBB1_3: # %cond.store1
 ; AVX2-NEXT:    vpextrw $1, %xmm0, 2(%rdi)
-; AVX2-NEXT:  .LBB1_4: # %else2
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm2, %xmm4
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    vpxor %xmm3, %xmm4, %xmm4
-; AVX2-NEXT:    vpextrb $8, %xmm4, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $4, %al
 ; AVX2-NEXT:    je .LBB1_6
-; AVX2-NEXT:  # %bb.5: # %cond.store3
+; AVX2-NEXT:  .LBB1_5: # %cond.store3
 ; AVX2-NEXT:    vpextrw $2, %xmm0, 4(%rdi)
-; AVX2-NEXT:  .LBB1_6: # %else4
-; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm2, %xmm1
-; AVX2-NEXT:    vpxor %xmm3, %xmm1, %xmm1
-; AVX2-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $8, %al
 ; AVX2-NEXT:    je .LBB1_8
-; AVX2-NEXT:  # %bb.7: # %cond.store5
+; AVX2-NEXT:  .LBB1_7: # %cond.store5
 ; AVX2-NEXT:    vpextrw $3, %xmm0, 6(%rdi)
-; AVX2-NEXT:  .LBB1_8: # %else6
-; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm1
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrb $0, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $16, %al
 ; AVX2-NEXT:    je .LBB1_10
-; AVX2-NEXT:  # %bb.9: # %cond.store7
+; AVX2-NEXT:  .LBB1_9: # %cond.store7
 ; AVX2-NEXT:    vpextrw $4, %xmm0, 8(%rdi)
-; AVX2-NEXT:  .LBB1_10: # %else8
-; AVX2-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $32, %al
 ; AVX2-NEXT:    je .LBB1_12
-; AVX2-NEXT:  # %bb.11: # %cond.store9
+; AVX2-NEXT:  .LBB1_11: # %cond.store9
 ; AVX2-NEXT:    vpextrw $5, %xmm0, 10(%rdi)
-; AVX2-NEXT:  .LBB1_12: # %else10
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $64, %al
 ; AVX2-NEXT:    je .LBB1_14
-; AVX2-NEXT:  # %bb.13: # %cond.store11
+; AVX2-NEXT:  .LBB1_13: # %cond.store11
 ; AVX2-NEXT:    vpextrw $6, %xmm0, 12(%rdi)
-; AVX2-NEXT:  .LBB1_14: # %else12
-; AVX2-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $-128, %al
 ; AVX2-NEXT:    je .LBB1_16
-; AVX2-NEXT:  # %bb.15: # %cond.store13
+; AVX2-NEXT:  .LBB1_15: # %cond.store13
 ; AVX2-NEXT:    vpextrw $7, %xmm0, 14(%rdi)
-; AVX2-NEXT:  .LBB1_16: # %else14
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
@@ -888,66 +856,61 @@ define void @truncstore_v8i64_v8i16(<8 x i64> %x, <8 x i16>* %p, <8 x i32> %mask
 ; AVX512F-NEXT:    vpmovsqw %zmm0, %xmm0
 ; AVX512F-NEXT:    kmovw %k0, %eax
 ; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB1_2
-; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    jne .LBB1_1
+; AVX512F-NEXT:  # %bb.2: # %else
+; AVX512F-NEXT:    testb $2, %al
+; AVX512F-NEXT:    jne .LBB1_3
+; AVX512F-NEXT:  .LBB1_4: # %else2
+; AVX512F-NEXT:    testb $4, %al
+; AVX512F-NEXT:    jne .LBB1_5
+; AVX512F-NEXT:  .LBB1_6: # %else4
+; AVX512F-NEXT:    testb $8, %al
+; AVX512F-NEXT:    jne .LBB1_7
+; AVX512F-NEXT:  .LBB1_8: # %else6
+; AVX512F-NEXT:    testb $16, %al
+; AVX512F-NEXT:    jne .LBB1_9
+; AVX512F-NEXT:  .LBB1_10: # %else8
+; AVX512F-NEXT:    testb $32, %al
+; AVX512F-NEXT:    jne .LBB1_11
+; AVX512F-NEXT:  .LBB1_12: # %else10
+; AVX512F-NEXT:    testb $64, %al
+; AVX512F-NEXT:    jne .LBB1_13
+; AVX512F-NEXT:  .LBB1_14: # %else12
+; AVX512F-NEXT:    testb $-128, %al
+; AVX512F-NEXT:    jne .LBB1_15
+; AVX512F-NEXT:  .LBB1_16: # %else14
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+; AVX512F-NEXT:  .LBB1_1: # %cond.store
 ; AVX512F-NEXT:    vpextrw $0, %xmm0, (%rdi)
-; AVX512F-NEXT:  .LBB1_2: # %else
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $2, %al
 ; AVX512F-NEXT:    je .LBB1_4
-; AVX512F-NEXT:  # %bb.3: # %cond.store1
+; AVX512F-NEXT:  .LBB1_3: # %cond.store1
 ; AVX512F-NEXT:    vpextrw $1, %xmm0, 2(%rdi)
-; AVX512F-NEXT:  .LBB1_4: # %else2
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $4, %al
 ; AVX512F-NEXT:    je .LBB1_6
-; AVX512F-NEXT:  # %bb.5: # %cond.store3
+; AVX512F-NEXT:  .LBB1_5: # %cond.store3
 ; AVX512F-NEXT:    vpextrw $2, %xmm0, 4(%rdi)
-; AVX512F-NEXT:  .LBB1_6: # %else4
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $8, %al
 ; AVX512F-NEXT:    je .LBB1_8
-; AVX512F-NEXT:  # %bb.7: # %cond.store5
+; AVX512F-NEXT:  .LBB1_7: # %cond.store5
 ; AVX512F-NEXT:    vpextrw $3, %xmm0, 6(%rdi)
-; AVX512F-NEXT:  .LBB1_8: # %else6
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $4, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $16, %al
 ; AVX512F-NEXT:    je .LBB1_10
-; AVX512F-NEXT:  # %bb.9: # %cond.store7
+; AVX512F-NEXT:  .LBB1_9: # %cond.store7
 ; AVX512F-NEXT:    vpextrw $4, %xmm0, 8(%rdi)
-; AVX512F-NEXT:  .LBB1_10: # %else8
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $5, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $32, %al
 ; AVX512F-NEXT:    je .LBB1_12
-; AVX512F-NEXT:  # %bb.11: # %cond.store9
+; AVX512F-NEXT:  .LBB1_11: # %cond.store9
 ; AVX512F-NEXT:    vpextrw $5, %xmm0, 10(%rdi)
-; AVX512F-NEXT:  .LBB1_12: # %else10
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $6, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $64, %al
 ; AVX512F-NEXT:    je .LBB1_14
-; AVX512F-NEXT:  # %bb.13: # %cond.store11
+; AVX512F-NEXT:  .LBB1_13: # %cond.store11
 ; AVX512F-NEXT:    vpextrw $6, %xmm0, 12(%rdi)
-; AVX512F-NEXT:  .LBB1_14: # %else12
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $7, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $-128, %al
 ; AVX512F-NEXT:    je .LBB1_16
-; AVX512F-NEXT:  # %bb.15: # %cond.store13
+; AVX512F-NEXT:  .LBB1_15: # %cond.store13
 ; AVX512F-NEXT:    vpextrw $7, %xmm0, 14(%rdi)
-; AVX512F-NEXT:  .LBB1_16: # %else14
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
@@ -982,553 +945,551 @@ define void @truncstore_v8i64_v8i16(<8 x i64> %x, <8 x i16>* %p, <8 x i32> %mask
 define void @truncstore_v8i64_v8i8(<8 x i64> %x, <8 x i8>* %p, <8 x i32> %mask) {
 ; SSE2-LABEL: truncstore_v8i64_v8i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pxor %xmm6, %xmm6
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm6
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm9
-; SSE2-NEXT:    pxor %xmm6, %xmm9
-; SSE2-NEXT:    movdqa %xmm9, %xmm8
-; SSE2-NEXT:    packssdw %xmm0, %xmm8
-; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [127,127]
-; SSE2-NEXT:    movdqa {{.*#+}} xmm12 = [2147483648,2147483648]
-; SSE2-NEXT:    movdqa %xmm2, %xmm6
-; SSE2-NEXT:    pxor %xmm12, %xmm6
-; SSE2-NEXT:    movdqa {{.*#+}} xmm11 = [2147483775,2147483775]
-; SSE2-NEXT:    movdqa %xmm11, %xmm7
+; SSE2-NEXT:    pxor %xmm8, %xmm8
+; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [127,127]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm11 = [2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm3, %xmm6
+; SSE2-NEXT:    pxor %xmm11, %xmm6
+; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [2147483775,2147483775]
+; SSE2-NEXT:    movdqa %xmm10, %xmm7
 ; SSE2-NEXT:    pcmpgtd %xmm6, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm7[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm11, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm7[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm6
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT:    pand %xmm13, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm14 = xmm7[1,1,3,3]
-; SSE2-NEXT:    por %xmm6, %xmm14
-; SSE2-NEXT:    pand %xmm14, %xmm2
-; SSE2-NEXT:    pandn %xmm10, %xmm14
-; SSE2-NEXT:    por %xmm2, %xmm14
-; SSE2-NEXT:    movdqa %xmm3, %xmm2
-; SSE2-NEXT:    pxor %xmm12, %xmm2
-; SSE2-NEXT:    movdqa %xmm11, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm11, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm2[1,1,3,3]
-; SSE2-NEXT:    pand %xmm13, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[1,1,3,3]
-; SSE2-NEXT:    por %xmm7, %xmm2
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pandn %xmm10, %xmm2
-; SSE2-NEXT:    por %xmm3, %xmm2
-; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    pxor %xmm12, %xmm3
-; SSE2-NEXT:    movdqa %xmm11, %xmm6
+; SSE2-NEXT:    pand %xmm12, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm7[1,1,3,3]
+; SSE2-NEXT:    por %xmm6, %xmm13
+; SSE2-NEXT:    pand %xmm13, %xmm3
+; SSE2-NEXT:    pandn %xmm9, %xmm13
+; SSE2-NEXT:    por %xmm3, %xmm13
+; SSE2-NEXT:    movdqa %xmm2, %xmm3
+; SSE2-NEXT:    pxor %xmm11, %xmm3
+; SSE2-NEXT:    movdqa %xmm10, %xmm6
 ; SSE2-NEXT:    pcmpgtd %xmm3, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm11, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm6[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm3
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm3[1,1,3,3]
-; SSE2-NEXT:    pand %xmm13, %xmm7
+; SSE2-NEXT:    pand %xmm12, %xmm7
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm6[1,1,3,3]
 ; SSE2-NEXT:    por %xmm7, %xmm3
-; SSE2-NEXT:    pand %xmm3, %xmm0
-; SSE2-NEXT:    pandn %xmm10, %xmm3
-; SSE2-NEXT:    por %xmm0, %xmm3
-; SSE2-NEXT:    movdqa %xmm1, %xmm0
-; SSE2-NEXT:    pxor %xmm12, %xmm0
-; SSE2-NEXT:    movdqa %xmm11, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm6
+; SSE2-NEXT:    pand %xmm3, %xmm2
+; SSE2-NEXT:    pandn %xmm9, %xmm3
+; SSE2-NEXT:    por %xmm2, %xmm3
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    pxor %xmm11, %xmm2
+; SSE2-NEXT:    movdqa %xmm10, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm6[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm2[1,1,3,3]
+; SSE2-NEXT:    pand %xmm12, %xmm7
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[1,1,3,3]
+; SSE2-NEXT:    por %xmm7, %xmm2
+; SSE2-NEXT:    pand %xmm2, %xmm1
+; SSE2-NEXT:    pandn %xmm9, %xmm2
+; SSE2-NEXT:    por %xmm1, %xmm2
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    pxor %xmm11, %xmm1
+; SSE2-NEXT:    movdqa %xmm10, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm6
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm11, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pand %xmm7, %xmm0
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pand %xmm7, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT:    por %xmm0, %xmm6
-; SSE2-NEXT:    pand %xmm6, %xmm1
-; SSE2-NEXT:    pandn %xmm10, %xmm6
 ; SSE2-NEXT:    por %xmm1, %xmm6
-; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [18446744073709551488,18446744073709551488]
+; SSE2-NEXT:    pand %xmm6, %xmm0
+; SSE2-NEXT:    pandn %xmm9, %xmm6
+; SSE2-NEXT:    por %xmm0, %xmm6
+; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [18446744073709551488,18446744073709551488]
 ; SSE2-NEXT:    movdqa %xmm6, %xmm0
-; SSE2-NEXT:    pxor %xmm12, %xmm0
-; SSE2-NEXT:    movdqa {{.*#+}} xmm11 = [18446744071562067840,18446744071562067840]
-; SSE2-NEXT:    movdqa %xmm0, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm11, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm7[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm11, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pand %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm7[1,1,3,3]
-; SSE2-NEXT:    por %xmm0, %xmm1
-; SSE2-NEXT:    pand %xmm1, %xmm6
-; SSE2-NEXT:    pandn %xmm10, %xmm1
-; SSE2-NEXT:    por %xmm6, %xmm1
-; SSE2-NEXT:    movdqa %xmm3, %xmm0
-; SSE2-NEXT:    pxor %xmm12, %xmm0
-; SSE2-NEXT:    movdqa %xmm0, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm11, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm11, %xmm0
+; SSE2-NEXT:    pxor %xmm11, %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [18446744071562067840,18446744071562067840]
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm10, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm1[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pand %xmm13, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3]
+; SSE2-NEXT:    pand %xmm12, %xmm7
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
 ; SSE2-NEXT:    por %xmm7, %xmm0
-; SSE2-NEXT:    pand %xmm0, %xmm3
-; SSE2-NEXT:    pandn %xmm10, %xmm0
-; SSE2-NEXT:    por %xmm3, %xmm0
-; SSE2-NEXT:    packssdw %xmm1, %xmm0
+; SSE2-NEXT:    pand %xmm0, %xmm6
+; SSE2-NEXT:    pandn %xmm9, %xmm0
+; SSE2-NEXT:    por %xmm6, %xmm0
 ; SSE2-NEXT:    movdqa %xmm2, %xmm1
-; SSE2-NEXT:    pxor %xmm12, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm3
-; SSE2-NEXT:    pcmpgtd %xmm11, %xmm3
+; SSE2-NEXT:    pxor %xmm11, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm10, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm6[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pand %xmm12, %xmm7
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm6[1,1,3,3]
+; SSE2-NEXT:    por %xmm7, %xmm1
+; SSE2-NEXT:    pand %xmm1, %xmm2
+; SSE2-NEXT:    pandn %xmm9, %xmm1
+; SSE2-NEXT:    por %xmm2, %xmm1
+; SSE2-NEXT:    movdqa %xmm3, %xmm2
+; SSE2-NEXT:    pxor %xmm11, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm10, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm6[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm2[1,1,3,3]
+; SSE2-NEXT:    pand %xmm12, %xmm7
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[1,1,3,3]
+; SSE2-NEXT:    por %xmm7, %xmm2
+; SSE2-NEXT:    pand %xmm2, %xmm3
+; SSE2-NEXT:    pandn %xmm9, %xmm2
+; SSE2-NEXT:    por %xmm3, %xmm2
+; SSE2-NEXT:    pxor %xmm13, %xmm11
+; SSE2-NEXT:    movdqa %xmm11, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm10, %xmm3
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm11, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm1
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm11
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm11[1,1,3,3]
+; SSE2-NEXT:    pand %xmm6, %xmm7
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm3
-; SSE2-NEXT:    pand %xmm3, %xmm2
-; SSE2-NEXT:    pandn %xmm10, %xmm3
-; SSE2-NEXT:    por %xmm2, %xmm3
-; SSE2-NEXT:    pxor %xmm14, %xmm12
-; SSE2-NEXT:    movdqa %xmm12, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm11, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm11, %xmm12
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm12[1,1,3,3]
-; SSE2-NEXT:    pand %xmm2, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm6, %xmm1
-; SSE2-NEXT:    pand %xmm1, %xmm14
-; SSE2-NEXT:    pandn %xmm10, %xmm1
-; SSE2-NEXT:    por %xmm14, %xmm1
-; SSE2-NEXT:    packssdw %xmm3, %xmm1
-; SSE2-NEXT:    packssdw %xmm1, %xmm0
-; SSE2-NEXT:    movd %xmm8, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    je .LBB2_2
-; SSE2-NEXT:  # %bb.1: # %cond.store
-; SSE2-NEXT:    movb %al, (%rdi)
-; SSE2-NEXT:  .LBB2_2: # %else
-; SSE2-NEXT:    psrlq $16, %xmm9
-; SSE2-NEXT:    movd %xmm9, %ecx
-; SSE2-NEXT:    shrl $16, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    je .LBB2_4
-; SSE2-NEXT:  # %bb.3: # %cond.store1
-; SSE2-NEXT:    shrl $16, %eax
-; SSE2-NEXT:    movb %al, 1(%rdi)
-; SSE2-NEXT:  .LBB2_4: # %else2
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm4
+; SSE2-NEXT:    por %xmm7, %xmm3
+; SSE2-NEXT:    pand %xmm3, %xmm13
+; SSE2-NEXT:    pandn %xmm9, %xmm3
+; SSE2-NEXT:    por %xmm13, %xmm3
+; SSE2-NEXT:    movdqa {{.*#+}} xmm6 = [255,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0]
+; SSE2-NEXT:    pand %xmm6, %xmm3
+; SSE2-NEXT:    pand %xmm6, %xmm2
+; SSE2-NEXT:    packuswb %xmm3, %xmm2
+; SSE2-NEXT:    pand %xmm6, %xmm1
+; SSE2-NEXT:    pand %xmm6, %xmm0
+; SSE2-NEXT:    packuswb %xmm1, %xmm0
+; SSE2-NEXT:    packuswb %xmm2, %xmm0
+; SSE2-NEXT:    packuswb %xmm0, %xmm0
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm5
 ; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pxor %xmm4, %xmm1
-; SSE2-NEXT:    pextrw $4, %xmm1, %eax
+; SSE2-NEXT:    pxor %xmm1, %xmm5
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm4
+; SSE2-NEXT:    pxor %xmm1, %xmm4
+; SSE2-NEXT:    packssdw %xmm5, %xmm4
+; SSE2-NEXT:    packsswb %xmm0, %xmm4
+; SSE2-NEXT:    pmovmskb %xmm4, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB2_6
-; SSE2-NEXT:  # %bb.5: # %cond.store3
-; SSE2-NEXT:    pextrw $2, %xmm0, %eax
-; SSE2-NEXT:    movb %al, 2(%rdi)
+; SSE2-NEXT:    movd %xmm0, %ecx
+; SSE2-NEXT:    jne .LBB2_1
+; SSE2-NEXT:  # %bb.2: # %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne .LBB2_3
+; SSE2-NEXT:  .LBB2_4: # %else2
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne .LBB2_5
 ; SSE2-NEXT:  .LBB2_6: # %else4
-; SSE2-NEXT:    pextrw $6, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je .LBB2_8
-; SSE2-NEXT:  # %bb.7: # %cond.store5
-; SSE2-NEXT:    pextrw $3, %xmm0, %eax
-; SSE2-NEXT:    movb %al, 3(%rdi)
+; SSE2-NEXT:  .LBB2_7: # %cond.store5
+; SSE2-NEXT:    shrl $24, %ecx
+; SSE2-NEXT:    movb %cl, 3(%rdi)
 ; SSE2-NEXT:  .LBB2_8: # %else6
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pxor %xmm2, %xmm1
-; SSE2-NEXT:    pextrw $0, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $16, %al
+; SSE2-NEXT:    pextrw $2, %xmm0, %ecx
 ; SSE2-NEXT:    je .LBB2_10
 ; SSE2-NEXT:  # %bb.9: # %cond.store7
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    movb %al, 4(%rdi)
+; SSE2-NEXT:    movb %cl, 4(%rdi)
 ; SSE2-NEXT:  .LBB2_10: # %else8
-; SSE2-NEXT:    pextrw $2, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $32, %al
 ; SSE2-NEXT:    je .LBB2_12
 ; SSE2-NEXT:  # %bb.11: # %cond.store9
-; SSE2-NEXT:    pextrw $5, %xmm0, %eax
-; SSE2-NEXT:    movb %al, 5(%rdi)
+; SSE2-NEXT:    movb %ch, 5(%rdi)
 ; SSE2-NEXT:  .LBB2_12: # %else10
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm5
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pxor %xmm5, %xmm1
-; SSE2-NEXT:    pextrw $4, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB2_14
-; SSE2-NEXT:  # %bb.13: # %cond.store11
-; SSE2-NEXT:    pextrw $6, %xmm0, %eax
-; SSE2-NEXT:    movb %al, 6(%rdi)
-; SSE2-NEXT:  .LBB2_14: # %else12
-; SSE2-NEXT:    pextrw $6, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB2_16
-; SSE2-NEXT:  # %bb.15: # %cond.store13
-; SSE2-NEXT:    pextrw $7, %xmm0, %eax
-; SSE2-NEXT:    movb %al, 7(%rdi)
+; SSE2-NEXT:    testb $64, %al
+; SSE2-NEXT:    pextrw $3, %xmm0, %ecx
+; SSE2-NEXT:    jne .LBB2_13
+; SSE2-NEXT:  # %bb.14: # %else12
+; SSE2-NEXT:    testb $-128, %al
+; SSE2-NEXT:    jne .LBB2_15
 ; SSE2-NEXT:  .LBB2_16: # %else14
 ; SSE2-NEXT:    retq
+; SSE2-NEXT:  .LBB2_1: # %cond.store
+; SSE2-NEXT:    movb %cl, (%rdi)
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    je .LBB2_4
+; SSE2-NEXT:  .LBB2_3: # %cond.store1
+; SSE2-NEXT:    movb %ch, 1(%rdi)
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    je .LBB2_6
+; SSE2-NEXT:  .LBB2_5: # %cond.store3
+; SSE2-NEXT:    movl %ecx, %edx
+; SSE2-NEXT:    shrl $16, %edx
+; SSE2-NEXT:    movb %dl, 2(%rdi)
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne .LBB2_7
+; SSE2-NEXT:    jmp .LBB2_8
+; SSE2-NEXT:  .LBB2_13: # %cond.store11
+; SSE2-NEXT:    movb %cl, 6(%rdi)
+; SSE2-NEXT:    testb $-128, %al
+; SSE2-NEXT:    je .LBB2_16
+; SSE2-NEXT:  .LBB2_15: # %cond.store13
+; SSE2-NEXT:    movb %ch, 7(%rdi)
+; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: truncstore_v8i64_v8i8:
 ; SSE4:       # %bb.0:
 ; SSE4-NEXT:    movdqa %xmm0, %xmm9
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqd %xmm4, %xmm0
-; SSE4-NEXT:    pcmpeqd %xmm8, %xmm8
-; SSE4-NEXT:    pxor %xmm0, %xmm8
+; SSE4-NEXT:    pxor %xmm8, %xmm8
 ; SSE4-NEXT:    movdqa {{.*#+}} xmm7 = [127,127]
 ; SSE4-NEXT:    movdqa %xmm7, %xmm0
-; SSE4-NEXT:    pcmpgtq %xmm2, %xmm0
-; SSE4-NEXT:    movdqa %xmm7, %xmm10
-; SSE4-NEXT:    blendvpd %xmm0, %xmm2, %xmm10
-; SSE4-NEXT:    movdqa %xmm7, %xmm0
 ; SSE4-NEXT:    pcmpgtq %xmm3, %xmm0
-; SSE4-NEXT:    movdqa %xmm7, %xmm2
-; SSE4-NEXT:    blendvpd %xmm0, %xmm3, %xmm2
+; SSE4-NEXT:    movdqa %xmm7, %xmm10
+; SSE4-NEXT:    blendvpd %xmm0, %xmm3, %xmm10
 ; SSE4-NEXT:    movdqa %xmm7, %xmm0
-; SSE4-NEXT:    pcmpgtq %xmm9, %xmm0
+; SSE4-NEXT:    pcmpgtq %xmm2, %xmm0
 ; SSE4-NEXT:    movdqa %xmm7, %xmm3
-; SSE4-NEXT:    blendvpd %xmm0, %xmm9, %xmm3
+; SSE4-NEXT:    blendvpd %xmm0, %xmm2, %xmm3
 ; SSE4-NEXT:    movdqa %xmm7, %xmm0
 ; SSE4-NEXT:    pcmpgtq %xmm1, %xmm0
-; SSE4-NEXT:    blendvpd %xmm0, %xmm1, %xmm7
+; SSE4-NEXT:    movdqa %xmm7, %xmm2
+; SSE4-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
+; SSE4-NEXT:    movdqa %xmm7, %xmm0
+; SSE4-NEXT:    pcmpgtq %xmm9, %xmm0
+; SSE4-NEXT:    blendvpd %xmm0, %xmm9, %xmm7
 ; SSE4-NEXT:    movdqa {{.*#+}} xmm1 = [18446744073709551488,18446744073709551488]
 ; SSE4-NEXT:    movapd %xmm7, %xmm0
 ; SSE4-NEXT:    pcmpgtq %xmm1, %xmm0
 ; SSE4-NEXT:    movdqa %xmm1, %xmm6
 ; SSE4-NEXT:    blendvpd %xmm0, %xmm7, %xmm6
-; SSE4-NEXT:    movapd %xmm3, %xmm0
+; SSE4-NEXT:    movapd %xmm2, %xmm0
 ; SSE4-NEXT:    pcmpgtq %xmm1, %xmm0
 ; SSE4-NEXT:    movdqa %xmm1, %xmm7
-; SSE4-NEXT:    blendvpd %xmm0, %xmm3, %xmm7
-; SSE4-NEXT:    packssdw %xmm6, %xmm7
-; SSE4-NEXT:    movapd %xmm2, %xmm0
+; SSE4-NEXT:    blendvpd %xmm0, %xmm2, %xmm7
+; SSE4-NEXT:    movapd %xmm3, %xmm0
 ; SSE4-NEXT:    pcmpgtq %xmm1, %xmm0
-; SSE4-NEXT:    movdqa %xmm1, %xmm3
-; SSE4-NEXT:    blendvpd %xmm0, %xmm2, %xmm3
+; SSE4-NEXT:    movdqa %xmm1, %xmm2
+; SSE4-NEXT:    blendvpd %xmm0, %xmm3, %xmm2
 ; SSE4-NEXT:    movapd %xmm10, %xmm0
 ; SSE4-NEXT:    pcmpgtq %xmm1, %xmm0
 ; SSE4-NEXT:    blendvpd %xmm0, %xmm10, %xmm1
-; SSE4-NEXT:    packssdw %xmm3, %xmm1
-; SSE4-NEXT:    packssdw %xmm1, %xmm7
-; SSE4-NEXT:    pextrb $0, %xmm8, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB2_2
-; SSE4-NEXT:  # %bb.1: # %cond.store
-; SSE4-NEXT:    pextrb $0, %xmm7, (%rdi)
-; SSE4-NEXT:  .LBB2_2: # %else
-; SSE4-NEXT:    pextrb $4, %xmm8, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB2_4
-; SSE4-NEXT:  # %bb.3: # %cond.store1
-; SSE4-NEXT:    pextrb $2, %xmm7, 1(%rdi)
-; SSE4-NEXT:  .LBB2_4: # %else2
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE4-NEXT:    movapd {{.*#+}} xmm0 = [255,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0]
+; SSE4-NEXT:    andpd %xmm0, %xmm1
+; SSE4-NEXT:    andpd %xmm0, %xmm2
+; SSE4-NEXT:    packusdw %xmm1, %xmm2
+; SSE4-NEXT:    andpd %xmm0, %xmm7
+; SSE4-NEXT:    andpd %xmm0, %xmm6
+; SSE4-NEXT:    packusdw %xmm7, %xmm6
+; SSE4-NEXT:    packusdw %xmm2, %xmm6
+; SSE4-NEXT:    packuswb %xmm6, %xmm6
+; SSE4-NEXT:    pcmpeqd %xmm8, %xmm5
 ; SSE4-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE4-NEXT:    pxor %xmm4, %xmm0
-; SSE4-NEXT:    pextrb $8, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB2_6
-; SSE4-NEXT:  # %bb.5: # %cond.store3
-; SSE4-NEXT:    pextrb $4, %xmm7, 2(%rdi)
+; SSE4-NEXT:    pxor %xmm0, %xmm5
+; SSE4-NEXT:    pcmpeqd %xmm8, %xmm4
+; SSE4-NEXT:    pxor %xmm0, %xmm4
+; SSE4-NEXT:    packssdw %xmm5, %xmm4
+; SSE4-NEXT:    packsswb %xmm0, %xmm4
+; SSE4-NEXT:    pmovmskb %xmm4, %eax
+; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    jne .LBB2_1
+; SSE4-NEXT:  # %bb.2: # %else
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    jne .LBB2_3
+; SSE4-NEXT:  .LBB2_4: # %else2
+; SSE4-NEXT:    testb $4, %al
+; SSE4-NEXT:    jne .LBB2_5
 ; SSE4-NEXT:  .LBB2_6: # %else4
-; SSE4-NEXT:    pextrb $12, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB2_8
-; SSE4-NEXT:  # %bb.7: # %cond.store5
-; SSE4-NEXT:    pextrb $6, %xmm7, 3(%rdi)
+; SSE4-NEXT:    testb $8, %al
+; SSE4-NEXT:    jne .LBB2_7
 ; SSE4-NEXT:  .LBB2_8: # %else6
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm5, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE4-NEXT:    pxor %xmm1, %xmm0
-; SSE4-NEXT:    pextrb $0, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB2_10
-; SSE4-NEXT:  # %bb.9: # %cond.store7
-; SSE4-NEXT:    pextrb $8, %xmm7, 4(%rdi)
+; SSE4-NEXT:    testb $16, %al
+; SSE4-NEXT:    jne .LBB2_9
 ; SSE4-NEXT:  .LBB2_10: # %else8
-; SSE4-NEXT:    pextrb $4, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB2_12
-; SSE4-NEXT:  # %bb.11: # %cond.store9
-; SSE4-NEXT:    pextrb $10, %xmm7, 5(%rdi)
+; SSE4-NEXT:    testb $32, %al
+; SSE4-NEXT:    jne .LBB2_11
 ; SSE4-NEXT:  .LBB2_12: # %else10
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqd %xmm0, %xmm5
-; SSE4-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE4-NEXT:    pxor %xmm5, %xmm0
-; SSE4-NEXT:    pextrb $8, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB2_14
-; SSE4-NEXT:  # %bb.13: # %cond.store11
-; SSE4-NEXT:    pextrb $12, %xmm7, 6(%rdi)
+; SSE4-NEXT:    testb $64, %al
+; SSE4-NEXT:    jne .LBB2_13
 ; SSE4-NEXT:  .LBB2_14: # %else12
-; SSE4-NEXT:    pextrb $12, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB2_16
-; SSE4-NEXT:  # %bb.15: # %cond.store13
-; SSE4-NEXT:    pextrb $14, %xmm7, 7(%rdi)
+; SSE4-NEXT:    testb $-128, %al
+; SSE4-NEXT:    jne .LBB2_15
 ; SSE4-NEXT:  .LBB2_16: # %else14
 ; SSE4-NEXT:    retq
+; SSE4-NEXT:  .LBB2_1: # %cond.store
+; SSE4-NEXT:    pextrb $0, %xmm6, (%rdi)
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    je .LBB2_4
+; SSE4-NEXT:  .LBB2_3: # %cond.store1
+; SSE4-NEXT:    pextrb $1, %xmm6, 1(%rdi)
+; SSE4-NEXT:    testb $4, %al
+; SSE4-NEXT:    je .LBB2_6
+; SSE4-NEXT:  .LBB2_5: # %cond.store3
+; SSE4-NEXT:    pextrb $2, %xmm6, 2(%rdi)
+; SSE4-NEXT:    testb $8, %al
+; SSE4-NEXT:    je .LBB2_8
+; SSE4-NEXT:  .LBB2_7: # %cond.store5
+; SSE4-NEXT:    pextrb $3, %xmm6, 3(%rdi)
+; SSE4-NEXT:    testb $16, %al
+; SSE4-NEXT:    je .LBB2_10
+; SSE4-NEXT:  .LBB2_9: # %cond.store7
+; SSE4-NEXT:    pextrb $4, %xmm6, 4(%rdi)
+; SSE4-NEXT:    testb $32, %al
+; SSE4-NEXT:    je .LBB2_12
+; SSE4-NEXT:  .LBB2_11: # %cond.store9
+; SSE4-NEXT:    pextrb $5, %xmm6, 5(%rdi)
+; SSE4-NEXT:    testb $64, %al
+; SSE4-NEXT:    je .LBB2_14
+; SSE4-NEXT:  .LBB2_13: # %cond.store11
+; SSE4-NEXT:    pextrb $6, %xmm6, 6(%rdi)
+; SSE4-NEXT:    testb $-128, %al
+; SSE4-NEXT:    je .LBB2_16
+; SSE4-NEXT:  .LBB2_15: # %cond.store13
+; SSE4-NEXT:    pextrb $7, %xmm6, 7(%rdi)
+; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: truncstore_v8i64_v8i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpxor %xmm8, %xmm8, %xmm8
-; AVX1-NEXT:    vpcmpeqd %xmm8, %xmm2, %xmm5
-; AVX1-NEXT:    vpcmpeqd %xmm9, %xmm9, %xmm9
-; AVX1-NEXT:    vpxor %xmm9, %xmm5, %xmm10
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm6
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm7 = [127,127]
-; AVX1-NEXT:    vpcmpgtq %xmm6, %xmm7, %xmm11
-; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm7, %xmm12
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm5
-; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm7, %xmm3
-; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm7, %xmm4
-; AVX1-NEXT:    vblendvpd %xmm4, %xmm0, %xmm7, %xmm0
+; AVX1-NEXT:    vmovapd {{.*#+}} ymm9 = [127,127,127,127]
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm10
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm5 = [127,127]
+; AVX1-NEXT:    vpcmpgtq %xmm10, %xmm5, %xmm6
+; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm5, %xmm7
+; AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm7, %ymm8
+; AVX1-NEXT:    vblendvpd %ymm8, %ymm1, %ymm9, %ymm8
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; AVX1-NEXT:    vpcmpgtq %xmm3, %xmm5, %xmm4
+; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm5, %xmm11
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm11, %ymm12
+; AVX1-NEXT:    vblendvpd %ymm12, %ymm0, %ymm9, %ymm9
+; AVX1-NEXT:    vmovapd {{.*#+}} ymm12 = [18446744073709551488,18446744073709551488,18446744073709551488,18446744073709551488]
+; AVX1-NEXT:    vblendvpd %xmm4, %xmm3, %xmm5, %xmm3
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [18446744073709551488,18446744073709551488]
-; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm0, %xmm13
-; AVX1-NEXT:    vblendvpd %xmm3, %xmm5, %xmm7, %xmm3
-; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm3, %xmm14
-; AVX1-NEXT:    vblendvpd %xmm12, %xmm1, %xmm7, %xmm1
-; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm1, %xmm5
-; AVX1-NEXT:    vblendvpd %xmm11, %xmm6, %xmm7, %xmm6
-; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm6, %xmm7
-; AVX1-NEXT:    vblendvpd %xmm7, %xmm6, %xmm4, %xmm6
-; AVX1-NEXT:    vblendvpd %xmm5, %xmm1, %xmm4, %xmm1
-; AVX1-NEXT:    vpackssdw %xmm6, %xmm1, %xmm1
-; AVX1-NEXT:    vblendvpd %xmm14, %xmm3, %xmm4, %xmm3
-; AVX1-NEXT:    vblendvpd %xmm13, %xmm0, %xmm4, %xmm0
-; AVX1-NEXT:    vpackssdw %xmm3, %xmm0, %xmm0
-; AVX1-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpextrb $0, %xmm10, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je .LBB2_2
-; AVX1-NEXT:  # %bb.1: # %cond.store
+; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vblendvpd %xmm11, %xmm0, %xmm5, %xmm0
+; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
+; AVX1-NEXT:    vblendvpd %ymm0, %ymm9, %ymm12, %ymm0
+; AVX1-NEXT:    vblendvpd %xmm6, %xmm10, %xmm5, %xmm3
+; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vblendvpd %xmm7, %xmm1, %xmm5, %xmm1
+; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm1, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
+; AVX1-NEXT:    vblendvpd %ymm1, %ymm8, %ymm12, %ymm1
+; AVX1-NEXT:    vmovapd {{.*#+}} ymm3 = [255,255,255,255]
+; AVX1-NEXT:    vandpd %ymm3, %ymm1, %ymm1
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm4
+; AVX1-NEXT:    vpackusdw %xmm4, %xmm1, %xmm1
+; AVX1-NEXT:    vandpd %ymm3, %ymm0, %ymm0
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; AVX1-NEXT:    vpackusdw %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm1
+; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
+; AVX1-NEXT:    vmovmskps %ymm1, %eax
+; AVX1-NEXT:    notl %eax
+; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    jne .LBB2_1
+; AVX1-NEXT:  # %bb.2: # %else
+; AVX1-NEXT:    testb $2, %al
+; AVX1-NEXT:    jne .LBB2_3
+; AVX1-NEXT:  .LBB2_4: # %else2
+; AVX1-NEXT:    testb $4, %al
+; AVX1-NEXT:    jne .LBB2_5
+; AVX1-NEXT:  .LBB2_6: # %else4
+; AVX1-NEXT:    testb $8, %al
+; AVX1-NEXT:    jne .LBB2_7
+; AVX1-NEXT:  .LBB2_8: # %else6
+; AVX1-NEXT:    testb $16, %al
+; AVX1-NEXT:    jne .LBB2_9
+; AVX1-NEXT:  .LBB2_10: # %else8
+; AVX1-NEXT:    testb $32, %al
+; AVX1-NEXT:    jne .LBB2_11
+; AVX1-NEXT:  .LBB2_12: # %else10
+; AVX1-NEXT:    testb $64, %al
+; AVX1-NEXT:    jne .LBB2_13
+; AVX1-NEXT:  .LBB2_14: # %else12
+; AVX1-NEXT:    testb $-128, %al
+; AVX1-NEXT:    jne .LBB2_15
+; AVX1-NEXT:  .LBB2_16: # %else14
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+; AVX1-NEXT:  .LBB2_1: # %cond.store
 ; AVX1-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX1-NEXT:  .LBB2_2: # %else
-; AVX1-NEXT:    vpcmpeqd %xmm8, %xmm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm9, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $4, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $2, %al
 ; AVX1-NEXT:    je .LBB2_4
-; AVX1-NEXT:  # %bb.3: # %cond.store1
-; AVX1-NEXT:    vpextrb $2, %xmm0, 1(%rdi)
-; AVX1-NEXT:  .LBB2_4: # %else2
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm2, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm4, %xmm4
-; AVX1-NEXT:    vpextrb $8, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:  .LBB2_3: # %cond.store1
+; AVX1-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
+; AVX1-NEXT:    testb $4, %al
 ; AVX1-NEXT:    je .LBB2_6
-; AVX1-NEXT:  # %bb.5: # %cond.store3
-; AVX1-NEXT:    vpextrb $4, %xmm0, 2(%rdi)
-; AVX1-NEXT:  .LBB2_6: # %else4
-; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:  .LBB2_5: # %cond.store3
+; AVX1-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
+; AVX1-NEXT:    testb $8, %al
 ; AVX1-NEXT:    je .LBB2_8
-; AVX1-NEXT:  # %bb.7: # %cond.store5
-; AVX1-NEXT:    vpextrb $6, %xmm0, 3(%rdi)
-; AVX1-NEXT:  .LBB2_8: # %else6
-; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $0, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:  .LBB2_7: # %cond.store5
+; AVX1-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
+; AVX1-NEXT:    testb $16, %al
 ; AVX1-NEXT:    je .LBB2_10
-; AVX1-NEXT:  # %bb.9: # %cond.store7
-; AVX1-NEXT:    vpextrb $8, %xmm0, 4(%rdi)
-; AVX1-NEXT:  .LBB2_10: # %else8
-; AVX1-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:  .LBB2_9: # %cond.store7
+; AVX1-NEXT:    vpextrb $4, %xmm0, 4(%rdi)
+; AVX1-NEXT:    testb $32, %al
 ; AVX1-NEXT:    je .LBB2_12
-; AVX1-NEXT:  # %bb.11: # %cond.store9
-; AVX1-NEXT:    vpextrb $10, %xmm0, 5(%rdi)
-; AVX1-NEXT:  .LBB2_12: # %else10
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:  .LBB2_11: # %cond.store9
+; AVX1-NEXT:    vpextrb $5, %xmm0, 5(%rdi)
+; AVX1-NEXT:    testb $64, %al
 ; AVX1-NEXT:    je .LBB2_14
-; AVX1-NEXT:  # %bb.13: # %cond.store11
-; AVX1-NEXT:    vpextrb $12, %xmm0, 6(%rdi)
-; AVX1-NEXT:  .LBB2_14: # %else12
-; AVX1-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:  .LBB2_13: # %cond.store11
+; AVX1-NEXT:    vpextrb $6, %xmm0, 6(%rdi)
+; AVX1-NEXT:    testb $-128, %al
 ; AVX1-NEXT:    je .LBB2_16
-; AVX1-NEXT:  # %bb.15: # %cond.store13
-; AVX1-NEXT:    vpextrb $14, %xmm0, 7(%rdi)
-; AVX1-NEXT:  .LBB2_16: # %else14
+; AVX1-NEXT:  .LBB2_15: # %cond.store13
+; AVX1-NEXT:    vpextrb $7, %xmm0, 7(%rdi)
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: truncstore_v8i64_v8i8:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm5
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm6 = [127,127,127,127]
-; AVX2-NEXT:    vpcmpgtq %ymm0, %ymm6, %ymm7
-; AVX2-NEXT:    vblendvpd %ymm7, %ymm0, %ymm6, %ymm0
-; AVX2-NEXT:    vpcmpgtq %ymm1, %ymm6, %ymm7
-; AVX2-NEXT:    vblendvpd %ymm7, %ymm1, %ymm6, %ymm1
-; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm6 = [18446744073709551488,18446744073709551488,18446744073709551488,18446744073709551488]
-; AVX2-NEXT:    vpcmpgtq %ymm6, %ymm1, %ymm7
-; AVX2-NEXT:    vblendvpd %ymm7, %ymm1, %ymm6, %ymm1
-; AVX2-NEXT:    vpcmpgtq %ymm6, %ymm0, %ymm7
-; AVX2-NEXT:    vblendvpd %ymm7, %ymm0, %ymm6, %ymm0
-; AVX2-NEXT:    vpackssdw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpextrb $0, %xmm5, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je .LBB2_2
-; AVX2-NEXT:  # %bb.1: # %cond.store
-; AVX2-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX2-NEXT:  .LBB2_2: # %else
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm1
-; AVX2-NEXT:    vpxor %xmm4, %xmm1, %xmm1
-; AVX2-NEXT:    vpextrb $4, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je .LBB2_4
-; AVX2-NEXT:  # %bb.3: # %cond.store1
-; AVX2-NEXT:    vpextrb $2, %xmm0, 1(%rdi)
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [127,127,127,127]
+; AVX2-NEXT:    vpcmpgtq %ymm1, %ymm4, %ymm5
+; AVX2-NEXT:    vblendvpd %ymm5, %ymm1, %ymm4, %ymm1
+; AVX2-NEXT:    vpcmpgtq %ymm0, %ymm4, %ymm5
+; AVX2-NEXT:    vblendvpd %ymm5, %ymm0, %ymm4, %ymm0
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [18446744073709551488,18446744073709551488,18446744073709551488,18446744073709551488]
+; AVX2-NEXT:    vpcmpgtq %ymm4, %ymm0, %ymm5
+; AVX2-NEXT:    vblendvpd %ymm5, %ymm0, %ymm4, %ymm0
+; AVX2-NEXT:    vpcmpgtq %ymm4, %ymm1, %ymm5
+; AVX2-NEXT:    vblendvpd %ymm5, %ymm1, %ymm4, %ymm1
+; AVX2-NEXT:    vextractf128 $1, %ymm1, %xmm4
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm5 = <u,u,0,8,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX2-NEXT:    vpshufb %xmm5, %xmm4, %xmm4
+; AVX2-NEXT:    vpshufb %xmm5, %xmm1, %xmm1
+; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3]
+; AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm4
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm5 = <0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX2-NEXT:    vpshufb %xmm5, %xmm4, %xmm4
+; AVX2-NEXT:    vpshufb %xmm5, %xmm0, %xmm0
+; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
+; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3]
+; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm2, %ymm1
+; AVX2-NEXT:    vmovmskps %ymm1, %eax
+; AVX2-NEXT:    notl %eax
+; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    jne .LBB2_1
+; AVX2-NEXT:  # %bb.2: # %else
+; AVX2-NEXT:    testb $2, %al
+; AVX2-NEXT:    jne .LBB2_3
 ; AVX2-NEXT:  .LBB2_4: # %else2
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm2, %xmm4
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    vpxor %xmm3, %xmm4, %xmm4
-; AVX2-NEXT:    vpextrb $8, %xmm4, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je .LBB2_6
-; AVX2-NEXT:  # %bb.5: # %cond.store3
-; AVX2-NEXT:    vpextrb $4, %xmm0, 2(%rdi)
+; AVX2-NEXT:    testb $4, %al
+; AVX2-NEXT:    jne .LBB2_5
 ; AVX2-NEXT:  .LBB2_6: # %else4
-; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm2, %xmm1
-; AVX2-NEXT:    vpxor %xmm3, %xmm1, %xmm1
-; AVX2-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je .LBB2_8
-; AVX2-NEXT:  # %bb.7: # %cond.store5
-; AVX2-NEXT:    vpextrb $6, %xmm0, 3(%rdi)
+; AVX2-NEXT:    testb $8, %al
+; AVX2-NEXT:    jne .LBB2_7
 ; AVX2-NEXT:  .LBB2_8: # %else6
-; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm1
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrb $0, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je .LBB2_10
-; AVX2-NEXT:  # %bb.9: # %cond.store7
-; AVX2-NEXT:    vpextrb $8, %xmm0, 4(%rdi)
+; AVX2-NEXT:    testb $16, %al
+; AVX2-NEXT:    jne .LBB2_9
 ; AVX2-NEXT:  .LBB2_10: # %else8
-; AVX2-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je .LBB2_12
-; AVX2-NEXT:  # %bb.11: # %cond.store9
-; AVX2-NEXT:    vpextrb $10, %xmm0, 5(%rdi)
+; AVX2-NEXT:    testb $32, %al
+; AVX2-NEXT:    jne .LBB2_11
 ; AVX2-NEXT:  .LBB2_12: # %else10
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je .LBB2_14
-; AVX2-NEXT:  # %bb.13: # %cond.store11
-; AVX2-NEXT:    vpextrb $12, %xmm0, 6(%rdi)
+; AVX2-NEXT:    testb $64, %al
+; AVX2-NEXT:    jne .LBB2_13
 ; AVX2-NEXT:  .LBB2_14: # %else12
-; AVX2-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je .LBB2_16
-; AVX2-NEXT:  # %bb.15: # %cond.store13
-; AVX2-NEXT:    vpextrb $14, %xmm0, 7(%rdi)
+; AVX2-NEXT:    testb $-128, %al
+; AVX2-NEXT:    jne .LBB2_15
 ; AVX2-NEXT:  .LBB2_16: # %else14
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
+; AVX2-NEXT:  .LBB2_1: # %cond.store
+; AVX2-NEXT:    vpextrb $0, %xmm0, (%rdi)
+; AVX2-NEXT:    testb $2, %al
+; AVX2-NEXT:    je .LBB2_4
+; AVX2-NEXT:  .LBB2_3: # %cond.store1
+; AVX2-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
+; AVX2-NEXT:    testb $4, %al
+; AVX2-NEXT:    je .LBB2_6
+; AVX2-NEXT:  .LBB2_5: # %cond.store3
+; AVX2-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
+; AVX2-NEXT:    testb $8, %al
+; AVX2-NEXT:    je .LBB2_8
+; AVX2-NEXT:  .LBB2_7: # %cond.store5
+; AVX2-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
+; AVX2-NEXT:    testb $16, %al
+; AVX2-NEXT:    je .LBB2_10
+; AVX2-NEXT:  .LBB2_9: # %cond.store7
+; AVX2-NEXT:    vpextrb $4, %xmm0, 4(%rdi)
+; AVX2-NEXT:    testb $32, %al
+; AVX2-NEXT:    je .LBB2_12
+; AVX2-NEXT:  .LBB2_11: # %cond.store9
+; AVX2-NEXT:    vpextrb $5, %xmm0, 5(%rdi)
+; AVX2-NEXT:    testb $64, %al
+; AVX2-NEXT:    je .LBB2_14
+; AVX2-NEXT:  .LBB2_13: # %cond.store11
+; AVX2-NEXT:    vpextrb $6, %xmm0, 6(%rdi)
+; AVX2-NEXT:    testb $-128, %al
+; AVX2-NEXT:    je .LBB2_16
+; AVX2-NEXT:  .LBB2_15: # %cond.store13
+; AVX2-NEXT:    vpextrb $7, %xmm0, 7(%rdi)
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: truncstore_v8i64_v8i8:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
 ; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    vpminsq {{.*}}(%rip){1to8}, %zmm0, %zmm0
-; AVX512F-NEXT:    vpmaxsq {{.*}}(%rip){1to8}, %zmm0, %zmm0
-; AVX512F-NEXT:    vpmovqw %zmm0, %xmm0
+; AVX512F-NEXT:    vpmovsqb %zmm0, %xmm0
 ; AVX512F-NEXT:    kmovw %k0, %eax
 ; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB2_2
-; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    jne .LBB2_1
+; AVX512F-NEXT:  # %bb.2: # %else
+; AVX512F-NEXT:    testb $2, %al
+; AVX512F-NEXT:    jne .LBB2_3
+; AVX512F-NEXT:  .LBB2_4: # %else2
+; AVX512F-NEXT:    testb $4, %al
+; AVX512F-NEXT:    jne .LBB2_5
+; AVX512F-NEXT:  .LBB2_6: # %else4
+; AVX512F-NEXT:    testb $8, %al
+; AVX512F-NEXT:    jne .LBB2_7
+; AVX512F-NEXT:  .LBB2_8: # %else6
+; AVX512F-NEXT:    testb $16, %al
+; AVX512F-NEXT:    jne .LBB2_9
+; AVX512F-NEXT:  .LBB2_10: # %else8
+; AVX512F-NEXT:    testb $32, %al
+; AVX512F-NEXT:    jne .LBB2_11
+; AVX512F-NEXT:  .LBB2_12: # %else10
+; AVX512F-NEXT:    testb $64, %al
+; AVX512F-NEXT:    jne .LBB2_13
+; AVX512F-NEXT:  .LBB2_14: # %else12
+; AVX512F-NEXT:    testb $-128, %al
+; AVX512F-NEXT:    jne .LBB2_15
+; AVX512F-NEXT:  .LBB2_16: # %else14
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+; AVX512F-NEXT:  .LBB2_1: # %cond.store
 ; AVX512F-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX512F-NEXT:  .LBB2_2: # %else
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $2, %al
 ; AVX512F-NEXT:    je .LBB2_4
-; AVX512F-NEXT:  # %bb.3: # %cond.store1
-; AVX512F-NEXT:    vpextrb $2, %xmm0, 1(%rdi)
-; AVX512F-NEXT:  .LBB2_4: # %else2
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  .LBB2_3: # %cond.store1
+; AVX512F-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
+; AVX512F-NEXT:    testb $4, %al
 ; AVX512F-NEXT:    je .LBB2_6
-; AVX512F-NEXT:  # %bb.5: # %cond.store3
-; AVX512F-NEXT:    vpextrb $4, %xmm0, 2(%rdi)
-; AVX512F-NEXT:  .LBB2_6: # %else4
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  .LBB2_5: # %cond.store3
+; AVX512F-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
+; AVX512F-NEXT:    testb $8, %al
 ; AVX512F-NEXT:    je .LBB2_8
-; AVX512F-NEXT:  # %bb.7: # %cond.store5
-; AVX512F-NEXT:    vpextrb $6, %xmm0, 3(%rdi)
-; AVX512F-NEXT:  .LBB2_8: # %else6
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $4, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  .LBB2_7: # %cond.store5
+; AVX512F-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
+; AVX512F-NEXT:    testb $16, %al
 ; AVX512F-NEXT:    je .LBB2_10
-; AVX512F-NEXT:  # %bb.9: # %cond.store7
-; AVX512F-NEXT:    vpextrb $8, %xmm0, 4(%rdi)
-; AVX512F-NEXT:  .LBB2_10: # %else8
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $5, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  .LBB2_9: # %cond.store7
+; AVX512F-NEXT:    vpextrb $4, %xmm0, 4(%rdi)
+; AVX512F-NEXT:    testb $32, %al
 ; AVX512F-NEXT:    je .LBB2_12
-; AVX512F-NEXT:  # %bb.11: # %cond.store9
-; AVX512F-NEXT:    vpextrb $10, %xmm0, 5(%rdi)
-; AVX512F-NEXT:  .LBB2_12: # %else10
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $6, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  .LBB2_11: # %cond.store9
+; AVX512F-NEXT:    vpextrb $5, %xmm0, 5(%rdi)
+; AVX512F-NEXT:    testb $64, %al
 ; AVX512F-NEXT:    je .LBB2_14
-; AVX512F-NEXT:  # %bb.13: # %cond.store11
-; AVX512F-NEXT:    vpextrb $12, %xmm0, 6(%rdi)
-; AVX512F-NEXT:  .LBB2_14: # %else12
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $7, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  .LBB2_13: # %cond.store11
+; AVX512F-NEXT:    vpextrb $6, %xmm0, 6(%rdi)
+; AVX512F-NEXT:    testb $-128, %al
 ; AVX512F-NEXT:    je .LBB2_16
-; AVX512F-NEXT:  # %bb.15: # %cond.store13
-; AVX512F-NEXT:    vpextrb $14, %xmm0, 7(%rdi)
-; AVX512F-NEXT:  .LBB2_16: # %else14
+; AVX512F-NEXT:  .LBB2_15: # %cond.store13
+; AVX512F-NEXT:    vpextrb $7, %xmm0, 7(%rdi)
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
@@ -1563,30 +1524,29 @@ define void @truncstore_v8i64_v8i8(<8 x i64> %x, <8 x i8>* %p, <8 x i32> %mask)
 define void @truncstore_v4i64_v4i32(<4 x i64> %x, <4 x i32>* %p, <4 x i32> %mask) {
 ; SSE2-LABEL: truncstore_v4i64_v4i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pxor %xmm10, %xmm10
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm10
+; SSE2-NEXT:    pxor %xmm9, %xmm9
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [2147483647,2147483647]
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [2147483648,2147483648]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm5
 ; SSE2-NEXT:    pxor %xmm4, %xmm5
-; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [4294967295,4294967295]
-; SSE2-NEXT:    movdqa %xmm9, %xmm7
+; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [4294967295,4294967295]
+; SSE2-NEXT:    movdqa %xmm10, %xmm7
 ; SSE2-NEXT:    pcmpgtd %xmm5, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm7[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm5[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm7[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[1,1,3,3]
+; SSE2-NEXT:    pand %xmm3, %xmm6
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm7[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm5
+; SSE2-NEXT:    por %xmm6, %xmm5
 ; SSE2-NEXT:    pand %xmm5, %xmm0
 ; SSE2-NEXT:    pandn %xmm8, %xmm5
 ; SSE2-NEXT:    por %xmm0, %xmm5
 ; SSE2-NEXT:    movdqa %xmm1, %xmm0
 ; SSE2-NEXT:    pxor %xmm4, %xmm0
-; SSE2-NEXT:    movdqa %xmm9, %xmm3
+; SSE2-NEXT:    movdqa %xmm10, %xmm3
 ; SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm0
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm6, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
@@ -1622,50 +1582,45 @@ define void @truncstore_v4i64_v4i32(<4 x i64> %x, <4 x i32>* %p, <4 x i32> %mask
 ; SSE2-NEXT:    pandn %xmm8, %xmm0
 ; SSE2-NEXT:    por %xmm5, %xmm0
 ; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
-; SSE2-NEXT:    movd %xmm10, %eax
-; SSE2-NEXT:    notl %eax
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm9
+; SSE2-NEXT:    movmskps %xmm9, %eax
+; SSE2-NEXT:    xorl $15, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB3_2
-; SSE2-NEXT:  # %bb.1: # %cond.store
+; SSE2-NEXT:    jne .LBB3_1
+; SSE2-NEXT:  # %bb.2: # %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne .LBB3_3
+; SSE2-NEXT:  .LBB3_4: # %else2
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne .LBB3_5
+; SSE2-NEXT:  .LBB3_6: # %else4
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne .LBB3_7
+; SSE2-NEXT:  .LBB3_8: # %else6
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  .LBB3_1: # %cond.store
 ; SSE2-NEXT:    movss %xmm0, (%rdi)
-; SSE2-NEXT:  .LBB3_2: # %else
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pxor %xmm1, %xmm10
-; SSE2-NEXT:    pextrw $2, %xmm10, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $2, %al
 ; SSE2-NEXT:    je .LBB3_4
-; SSE2-NEXT:  # %bb.3: # %cond.store1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,2,3]
-; SSE2-NEXT:    movd %xmm3, 4(%rdi)
-; SSE2-NEXT:  .LBB3_4: # %else2
-; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm2
-; SSE2-NEXT:    pxor %xmm2, %xmm1
-; SSE2-NEXT:    pextrw $4, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:  .LBB3_3: # %cond.store1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE2-NEXT:    movd %xmm1, 4(%rdi)
+; SSE2-NEXT:    testb $4, %al
 ; SSE2-NEXT:    je .LBB3_6
-; SSE2-NEXT:  # %bb.5: # %cond.store3
+; SSE2-NEXT:  .LBB3_5: # %cond.store3
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; SSE2-NEXT:    movd %xmm1, 8(%rdi)
-; SSE2-NEXT:  .LBB3_6: # %else4
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pxor %xmm1, %xmm2
-; SSE2-NEXT:    pextrw $6, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je .LBB3_8
-; SSE2-NEXT:  # %bb.7: # %cond.store5
+; SSE2-NEXT:  .LBB3_7: # %cond.store5
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
 ; SSE2-NEXT:    movd %xmm0, 12(%rdi)
-; SSE2-NEXT:  .LBB3_8: # %else6
 ; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: truncstore_v4i64_v4i32:
 ; SSE4:       # %bb.0:
 ; SSE4-NEXT:    movdqa %xmm0, %xmm3
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqd %xmm2, %xmm0
-; SSE4-NEXT:    pcmpeqd %xmm4, %xmm4
-; SSE4-NEXT:    pxor %xmm0, %xmm4
+; SSE4-NEXT:    pxor %xmm4, %xmm4
 ; SSE4-NEXT:    movdqa {{.*#+}} xmm5 = [2147483647,2147483647]
 ; SSE4-NEXT:    movdqa %xmm5, %xmm0
 ; SSE4-NEXT:    pcmpgtq %xmm3, %xmm0
@@ -1683,34 +1638,36 @@ define void @truncstore_v4i64_v4i32(<4 x i64> %x, <4 x i32>* %p, <4 x i32> %mask
 ; SSE4-NEXT:    pcmpgtq %xmm1, %xmm0
 ; SSE4-NEXT:    blendvpd %xmm0, %xmm6, %xmm1
 ; SSE4-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,2],xmm3[0,2]
-; SSE4-NEXT:    pextrb $0, %xmm4, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB3_2
-; SSE4-NEXT:  # %bb.1: # %cond.store
+; SSE4-NEXT:    pcmpeqd %xmm2, %xmm4
+; SSE4-NEXT:    movmskps %xmm4, %eax
+; SSE4-NEXT:    xorl $15, %eax
+; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    jne .LBB3_1
+; SSE4-NEXT:  # %bb.2: # %else
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    jne .LBB3_3
+; SSE4-NEXT:  .LBB3_4: # %else2
+; SSE4-NEXT:    testb $4, %al
+; SSE4-NEXT:    jne .LBB3_5
+; SSE4-NEXT:  .LBB3_6: # %else4
+; SSE4-NEXT:    testb $8, %al
+; SSE4-NEXT:    jne .LBB3_7
+; SSE4-NEXT:  .LBB3_8: # %else6
+; SSE4-NEXT:    retq
+; SSE4-NEXT:  .LBB3_1: # %cond.store
 ; SSE4-NEXT:    movss %xmm1, (%rdi)
-; SSE4-NEXT:  .LBB3_2: # %else
-; SSE4-NEXT:    pextrb $4, %xmm4, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $2, %al
 ; SSE4-NEXT:    je .LBB3_4
-; SSE4-NEXT:  # %bb.3: # %cond.store1
+; SSE4-NEXT:  .LBB3_3: # %cond.store1
 ; SSE4-NEXT:    extractps $1, %xmm1, 4(%rdi)
-; SSE4-NEXT:  .LBB3_4: # %else2
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE4-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE4-NEXT:    pxor %xmm2, %xmm0
-; SSE4-NEXT:    pextrb $8, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $4, %al
 ; SSE4-NEXT:    je .LBB3_6
-; SSE4-NEXT:  # %bb.5: # %cond.store3
+; SSE4-NEXT:  .LBB3_5: # %cond.store3
 ; SSE4-NEXT:    extractps $2, %xmm1, 8(%rdi)
-; SSE4-NEXT:  .LBB3_6: # %else4
-; SSE4-NEXT:    pextrb $12, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $8, %al
 ; SSE4-NEXT:    je .LBB3_8
-; SSE4-NEXT:  # %bb.7: # %cond.store5
+; SSE4-NEXT:  .LBB3_7: # %cond.store5
 ; SSE4-NEXT:    extractps $3, %xmm1, 12(%rdi)
-; SSE4-NEXT:  .LBB3_8: # %else6
 ; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: truncstore_v4i64_v4i32:
@@ -1760,11 +1717,7 @@ define void @truncstore_v4i64_v4i32(<4 x i64> %x, <4 x i32>* %p, <4 x i32> %mask
 ; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
 ; AVX512F-NEXT:    kshiftlw $12, %k0, %k0
 ; AVX512F-NEXT:    kshiftrw $12, %k0, %k1
-; AVX512F-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [2147483647,2147483647,2147483647,2147483647]
-; AVX512F-NEXT:    vpminsq %zmm1, %zmm0, %zmm0
-; AVX512F-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [18446744071562067968,18446744071562067968,18446744071562067968,18446744071562067968]
-; AVX512F-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm0
-; AVX512F-NEXT:    vpmovqd %zmm0, %ymm0
+; AVX512F-NEXT:    vpmovsqd %zmm0, %ymm0
 ; AVX512F-NEXT:    vmovdqu32 %zmm0, (%rdi) {%k1}
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
@@ -1785,11 +1738,7 @@ define void @truncstore_v4i64_v4i32(<4 x i64> %x, <4 x i32>* %p, <4 x i32> %mask
 ; AVX512BW-NEXT:    vptestmd %zmm1, %zmm1, %k0
 ; AVX512BW-NEXT:    kshiftlw $12, %k0, %k0
 ; AVX512BW-NEXT:    kshiftrw $12, %k0, %k1
-; AVX512BW-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [2147483647,2147483647,2147483647,2147483647]
-; AVX512BW-NEXT:    vpminsq %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [18446744071562067968,18446744071562067968,18446744071562067968,18446744071562067968]
-; AVX512BW-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpmovqd %zmm0, %ymm0
+; AVX512BW-NEXT:    vpmovsqd %zmm0, %ymm0
 ; AVX512BW-NEXT:    vmovdqu32 %zmm0, (%rdi) {%k1}
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -1806,118 +1755,116 @@ define void @truncstore_v4i64_v4i32(<4 x i64> %x, <4 x i32>* %p, <4 x i32> %mask
 define void @truncstore_v4i64_v4i16(<4 x i64> %x, <4 x i16>* %p, <4 x i32> %mask) {
 ; SSE2-LABEL: truncstore_v4i64_v4i16:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pxor %xmm10, %xmm10
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm10
+; SSE2-NEXT:    pxor %xmm9, %xmm9
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [32767,32767]
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [2147483648,2147483648]
-; SSE2-NEXT:    movdqa %xmm0, %xmm5
+; SSE2-NEXT:    movdqa %xmm1, %xmm5
 ; SSE2-NEXT:    pxor %xmm4, %xmm5
-; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [2147516415,2147516415]
-; SSE2-NEXT:    movdqa %xmm9, %xmm7
+; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [2147516415,2147516415]
+; SSE2-NEXT:    movdqa %xmm10, %xmm7
 ; SSE2-NEXT:    pcmpgtd %xmm5, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm7[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm5[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm7[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[1,1,3,3]
+; SSE2-NEXT:    pand %xmm3, %xmm6
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm7[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm5
-; SSE2-NEXT:    pand %xmm5, %xmm0
+; SSE2-NEXT:    por %xmm6, %xmm5
+; SSE2-NEXT:    pand %xmm5, %xmm1
 ; SSE2-NEXT:    pandn %xmm8, %xmm5
-; SSE2-NEXT:    por %xmm0, %xmm5
-; SSE2-NEXT:    movdqa %xmm1, %xmm0
-; SSE2-NEXT:    pxor %xmm4, %xmm0
-; SSE2-NEXT:    movdqa %xmm9, %xmm3
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSE2-NEXT:    por %xmm1, %xmm5
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    pxor %xmm4, %xmm1
+; SSE2-NEXT:    movdqa %xmm10, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm0
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pand %xmm6, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE2-NEXT:    por %xmm0, %xmm3
-; SSE2-NEXT:    pand %xmm3, %xmm1
-; SSE2-NEXT:    pandn %xmm8, %xmm3
 ; SSE2-NEXT:    por %xmm1, %xmm3
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    pandn %xmm8, %xmm3
+; SSE2-NEXT:    por %xmm0, %xmm3
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [18446744073709518848,18446744073709518848]
-; SSE2-NEXT:    movdqa %xmm3, %xmm0
-; SSE2-NEXT:    pxor %xmm4, %xmm0
+; SSE2-NEXT:    movdqa %xmm3, %xmm1
+; SSE2-NEXT:    pxor %xmm4, %xmm1
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm6 = [18446744071562035200,18446744071562035200]
-; SSE2-NEXT:    movdqa %xmm0, %xmm7
+; SSE2-NEXT:    movdqa %xmm1, %xmm7
 ; SSE2-NEXT:    pcmpgtd %xmm6, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm7[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm6, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pand %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm7[1,1,3,3]
-; SSE2-NEXT:    por %xmm0, %xmm1
-; SSE2-NEXT:    pand %xmm1, %xmm3
-; SSE2-NEXT:    pandn %xmm8, %xmm1
-; SSE2-NEXT:    por %xmm3, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm6, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pand %xmm0, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[1,1,3,3]
+; SSE2-NEXT:    por %xmm1, %xmm0
+; SSE2-NEXT:    pand %xmm0, %xmm3
+; SSE2-NEXT:    pandn %xmm8, %xmm0
+; SSE2-NEXT:    por %xmm3, %xmm0
 ; SSE2-NEXT:    pxor %xmm5, %xmm4
-; SSE2-NEXT:    movdqa %xmm4, %xmm0
-; SSE2-NEXT:    pcmpgtd %xmm6, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[0,0,2,2]
+; SSE2-NEXT:    movdqa %xmm4, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm6, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
 ; SSE2-NEXT:    pcmpeqd %xmm6, %xmm4
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm3, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm4, %xmm0
-; SSE2-NEXT:    pand %xmm0, %xmm5
-; SSE2-NEXT:    pandn %xmm8, %xmm0
-; SSE2-NEXT:    por %xmm5, %xmm0
-; SSE2-NEXT:    packssdw %xmm1, %xmm0
-; SSE2-NEXT:    movd %xmm10, %eax
-; SSE2-NEXT:    notl %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB4_2
-; SSE2-NEXT:  # %bb.1: # %cond.store
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    movw %ax, (%rdi)
-; SSE2-NEXT:  .LBB4_2: # %else
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pxor %xmm1, %xmm10
-; SSE2-NEXT:    pextrw $2, %xmm10, %eax
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    por %xmm4, %xmm1
+; SSE2-NEXT:    pand %xmm1, %xmm5
+; SSE2-NEXT:    pandn %xmm8, %xmm1
+; SSE2-NEXT:    por %xmm5, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm9
+; SSE2-NEXT:    movmskps %xmm9, %eax
+; SSE2-NEXT:    xorl $15, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB4_4
-; SSE2-NEXT:  # %bb.3: # %cond.store1
-; SSE2-NEXT:    pextrw $2, %xmm0, %eax
-; SSE2-NEXT:    movw %ax, 2(%rdi)
+; SSE2-NEXT:    jne .LBB4_1
+; SSE2-NEXT:  # %bb.2: # %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne .LBB4_3
 ; SSE2-NEXT:  .LBB4_4: # %else2
-; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm2
-; SSE2-NEXT:    pxor %xmm2, %xmm1
-; SSE2-NEXT:    pextrw $4, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB4_6
-; SSE2-NEXT:  # %bb.5: # %cond.store3
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    movw %ax, 4(%rdi)
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne .LBB4_5
 ; SSE2-NEXT:  .LBB4_6: # %else4
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pxor %xmm1, %xmm2
-; SSE2-NEXT:    pextrw $6, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne .LBB4_7
+; SSE2-NEXT:  .LBB4_8: # %else6
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  .LBB4_1: # %cond.store
+; SSE2-NEXT:    movd %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, (%rdi)
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    je .LBB4_4
+; SSE2-NEXT:  .LBB4_3: # %cond.store1
+; SSE2-NEXT:    pextrw $1, %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, 2(%rdi)
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    je .LBB4_6
+; SSE2-NEXT:  .LBB4_5: # %cond.store3
+; SSE2-NEXT:    pextrw $2, %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, 4(%rdi)
+; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je .LBB4_8
-; SSE2-NEXT:  # %bb.7: # %cond.store5
-; SSE2-NEXT:    pextrw $6, %xmm0, %eax
+; SSE2-NEXT:  .LBB4_7: # %cond.store5
+; SSE2-NEXT:    pextrw $3, %xmm0, %eax
 ; SSE2-NEXT:    movw %ax, 6(%rdi)
-; SSE2-NEXT:  .LBB4_8: # %else6
 ; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: truncstore_v4i64_v4i16:
 ; SSE4:       # %bb.0:
 ; SSE4-NEXT:    movdqa %xmm0, %xmm3
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqd %xmm2, %xmm0
-; SSE4-NEXT:    pcmpeqd %xmm4, %xmm4
-; SSE4-NEXT:    pxor %xmm0, %xmm4
+; SSE4-NEXT:    pxor %xmm4, %xmm4
 ; SSE4-NEXT:    movdqa {{.*#+}} xmm5 = [32767,32767]
 ; SSE4-NEXT:    movdqa %xmm5, %xmm0
-; SSE4-NEXT:    pcmpgtq %xmm3, %xmm0
+; SSE4-NEXT:    pcmpgtq %xmm1, %xmm0
 ; SSE4-NEXT:    movdqa %xmm5, %xmm6
-; SSE4-NEXT:    blendvpd %xmm0, %xmm3, %xmm6
+; SSE4-NEXT:    blendvpd %xmm0, %xmm1, %xmm6
 ; SSE4-NEXT:    movdqa %xmm5, %xmm0
-; SSE4-NEXT:    pcmpgtq %xmm1, %xmm0
-; SSE4-NEXT:    blendvpd %xmm0, %xmm1, %xmm5
+; SSE4-NEXT:    pcmpgtq %xmm3, %xmm0
+; SSE4-NEXT:    blendvpd %xmm0, %xmm3, %xmm5
 ; SSE4-NEXT:    movdqa {{.*#+}} xmm1 = [18446744073709518848,18446744073709518848]
 ; SSE4-NEXT:    movapd %xmm5, %xmm0
 ; SSE4-NEXT:    pcmpgtq %xmm1, %xmm0
@@ -1926,43 +1873,46 @@ define void @truncstore_v4i64_v4i16(<4 x i64> %x, <4 x i16>* %p, <4 x i32> %mask
 ; SSE4-NEXT:    movapd %xmm6, %xmm0
 ; SSE4-NEXT:    pcmpgtq %xmm1, %xmm0
 ; SSE4-NEXT:    blendvpd %xmm0, %xmm6, %xmm1
-; SSE4-NEXT:    packssdw %xmm3, %xmm1
-; SSE4-NEXT:    pextrb $0, %xmm4, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB4_2
-; SSE4-NEXT:  # %bb.1: # %cond.store
-; SSE4-NEXT:    pextrw $0, %xmm1, (%rdi)
-; SSE4-NEXT:  .LBB4_2: # %else
-; SSE4-NEXT:    pextrb $4, %xmm4, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB4_4
-; SSE4-NEXT:  # %bb.3: # %cond.store1
-; SSE4-NEXT:    pextrw $2, %xmm1, 2(%rdi)
+; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
+; SSE4-NEXT:    pshuflw {{.*#+}} xmm1 = xmm0[0,2,2,3,4,5,6,7]
+; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,2,2,3]
+; SSE4-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; SSE4-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE4-NEXT:    pcmpeqd %xmm2, %xmm4
+; SSE4-NEXT:    movmskps %xmm4, %eax
+; SSE4-NEXT:    xorl $15, %eax
+; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    jne .LBB4_1
+; SSE4-NEXT:  # %bb.2: # %else
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    jne .LBB4_3
 ; SSE4-NEXT:  .LBB4_4: # %else2
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE4-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE4-NEXT:    pxor %xmm2, %xmm0
-; SSE4-NEXT:    pextrb $8, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB4_6
-; SSE4-NEXT:  # %bb.5: # %cond.store3
-; SSE4-NEXT:    pextrw $4, %xmm1, 4(%rdi)
+; SSE4-NEXT:    testb $4, %al
+; SSE4-NEXT:    jne .LBB4_5
 ; SSE4-NEXT:  .LBB4_6: # %else4
-; SSE4-NEXT:    pextrb $12, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB4_8
-; SSE4-NEXT:  # %bb.7: # %cond.store5
-; SSE4-NEXT:    pextrw $6, %xmm1, 6(%rdi)
+; SSE4-NEXT:    testb $8, %al
+; SSE4-NEXT:    jne .LBB4_7
 ; SSE4-NEXT:  .LBB4_8: # %else6
 ; SSE4-NEXT:    retq
+; SSE4-NEXT:  .LBB4_1: # %cond.store
+; SSE4-NEXT:    pextrw $0, %xmm0, (%rdi)
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    je .LBB4_4
+; SSE4-NEXT:  .LBB4_3: # %cond.store1
+; SSE4-NEXT:    pextrw $1, %xmm0, 2(%rdi)
+; SSE4-NEXT:    testb $4, %al
+; SSE4-NEXT:    je .LBB4_6
+; SSE4-NEXT:  .LBB4_5: # %cond.store3
+; SSE4-NEXT:    pextrw $2, %xmm0, 4(%rdi)
+; SSE4-NEXT:    testb $8, %al
+; SSE4-NEXT:    je .LBB4_8
+; SSE4-NEXT:  .LBB4_7: # %cond.store5
+; SSE4-NEXT:    pextrw $3, %xmm0, 6(%rdi)
+; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: truncstore_v4i64_v4i16:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [32767,32767]
 ; AVX1-NEXT:    vpcmpgtq %xmm3, %xmm4, %xmm5
@@ -1973,45 +1923,49 @@ define void @truncstore_v4i64_v4i16(<4 x i64> %x, <4 x i16>* %p, <4 x i32> %mask
 ; AVX1-NEXT:    vblendvpd %xmm5, %xmm3, %xmm4, %xmm3
 ; AVX1-NEXT:    vpcmpgtq %xmm6, %xmm3, %xmm4
 ; AVX1-NEXT:    vblendvpd %xmm4, %xmm3, %xmm6, %xmm3
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm3 = xmm3[0,2,2,3]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm3[0,2,2,3,4,5,6,7]
 ; AVX1-NEXT:    vblendvpd %xmm7, %xmm0, %xmm6, %xmm0
-; AVX1-NEXT:    vpackssdw %xmm3, %xmm0, %xmm0
-; AVX1-NEXT:    vpextrb $0, %xmm2, %eax
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vmovmskps %xmm1, %eax
+; AVX1-NEXT:    xorl $15, %eax
 ; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je .LBB4_2
-; AVX1-NEXT:  # %bb.1: # %cond.store
+; AVX1-NEXT:    jne .LBB4_1
+; AVX1-NEXT:  # %bb.2: # %else
+; AVX1-NEXT:    testb $2, %al
+; AVX1-NEXT:    jne .LBB4_3
+; AVX1-NEXT:  .LBB4_4: # %else2
+; AVX1-NEXT:    testb $4, %al
+; AVX1-NEXT:    jne .LBB4_5
+; AVX1-NEXT:  .LBB4_6: # %else4
+; AVX1-NEXT:    testb $8, %al
+; AVX1-NEXT:    jne .LBB4_7
+; AVX1-NEXT:  .LBB4_8: # %else6
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+; AVX1-NEXT:  .LBB4_1: # %cond.store
 ; AVX1-NEXT:    vpextrw $0, %xmm0, (%rdi)
-; AVX1-NEXT:  .LBB4_2: # %else
-; AVX1-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $2, %al
 ; AVX1-NEXT:    je .LBB4_4
-; AVX1-NEXT:  # %bb.3: # %cond.store1
-; AVX1-NEXT:    vpextrw $2, %xmm0, 2(%rdi)
-; AVX1-NEXT:  .LBB4_4: # %else2
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:  .LBB4_3: # %cond.store1
+; AVX1-NEXT:    vpextrw $1, %xmm0, 2(%rdi)
+; AVX1-NEXT:    testb $4, %al
 ; AVX1-NEXT:    je .LBB4_6
-; AVX1-NEXT:  # %bb.5: # %cond.store3
-; AVX1-NEXT:    vpextrw $4, %xmm0, 4(%rdi)
-; AVX1-NEXT:  .LBB4_6: # %else4
-; AVX1-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:  .LBB4_5: # %cond.store3
+; AVX1-NEXT:    vpextrw $2, %xmm0, 4(%rdi)
+; AVX1-NEXT:    testb $8, %al
 ; AVX1-NEXT:    je .LBB4_8
-; AVX1-NEXT:  # %bb.7: # %cond.store5
-; AVX1-NEXT:    vpextrw $6, %xmm0, 6(%rdi)
-; AVX1-NEXT:  .LBB4_8: # %else6
+; AVX1-NEXT:  .LBB4_7: # %cond.store5
+; AVX1-NEXT:    vpextrw $3, %xmm0, 6(%rdi)
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: truncstore_v4i64_v4i16:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    vpxor %xmm3, %xmm2, %xmm2
 ; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [32767,32767,32767,32767]
 ; AVX2-NEXT:    vpcmpgtq %ymm0, %ymm3, %ymm4
 ; AVX2-NEXT:    vblendvpd %ymm4, %ymm0, %ymm3, %ymm0
@@ -2019,35 +1973,42 @@ define void @truncstore_v4i64_v4i16(<4 x i64> %x, <4 x i16>* %p, <4 x i32> %mask
 ; AVX2-NEXT:    vpcmpgtq %ymm3, %ymm0, %ymm4
 ; AVX2-NEXT:    vblendvpd %ymm4, %ymm0, %ymm3, %ymm0
 ; AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm3
-; AVX2-NEXT:    vpackssdw %xmm3, %xmm0, %xmm0
-; AVX2-NEXT:    vpextrb $0, %xmm2, %eax
+; AVX2-NEXT:    vpermilps {{.*#+}} xmm3 = xmm3[0,2,2,3]
+; AVX2-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm3[0,2,2,3,4,5,6,7]
+; AVX2-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX2-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vmovmskps %xmm1, %eax
+; AVX2-NEXT:    xorl $15, %eax
 ; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je .LBB4_2
-; AVX2-NEXT:  # %bb.1: # %cond.store
+; AVX2-NEXT:    jne .LBB4_1
+; AVX2-NEXT:  # %bb.2: # %else
+; AVX2-NEXT:    testb $2, %al
+; AVX2-NEXT:    jne .LBB4_3
+; AVX2-NEXT:  .LBB4_4: # %else2
+; AVX2-NEXT:    testb $4, %al
+; AVX2-NEXT:    jne .LBB4_5
+; AVX2-NEXT:  .LBB4_6: # %else4
+; AVX2-NEXT:    testb $8, %al
+; AVX2-NEXT:    jne .LBB4_7
+; AVX2-NEXT:  .LBB4_8: # %else6
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+; AVX2-NEXT:  .LBB4_1: # %cond.store
 ; AVX2-NEXT:    vpextrw $0, %xmm0, (%rdi)
-; AVX2-NEXT:  .LBB4_2: # %else
-; AVX2-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $2, %al
 ; AVX2-NEXT:    je .LBB4_4
-; AVX2-NEXT:  # %bb.3: # %cond.store1
-; AVX2-NEXT:    vpextrw $2, %xmm0, 2(%rdi)
-; AVX2-NEXT:  .LBB4_4: # %else2
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:  .LBB4_3: # %cond.store1
+; AVX2-NEXT:    vpextrw $1, %xmm0, 2(%rdi)
+; AVX2-NEXT:    testb $4, %al
 ; AVX2-NEXT:    je .LBB4_6
-; AVX2-NEXT:  # %bb.5: # %cond.store3
-; AVX2-NEXT:    vpextrw $4, %xmm0, 4(%rdi)
-; AVX2-NEXT:  .LBB4_6: # %else4
-; AVX2-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:  .LBB4_5: # %cond.store3
+; AVX2-NEXT:    vpextrw $2, %xmm0, 4(%rdi)
+; AVX2-NEXT:    testb $8, %al
 ; AVX2-NEXT:    je .LBB4_8
-; AVX2-NEXT:  # %bb.7: # %cond.store5
-; AVX2-NEXT:    vpextrw $6, %xmm0, 6(%rdi)
-; AVX2-NEXT:  .LBB4_8: # %else6
+; AVX2-NEXT:  .LBB4_7: # %cond.store5
+; AVX2-NEXT:    vpextrw $3, %xmm0, 6(%rdi)
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
@@ -2056,57 +2017,47 @@ define void @truncstore_v4i64_v4i16(<4 x i64> %x, <4 x i16>* %p, <4 x i32> %mask
 ; AVX512F-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
 ; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
 ; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [32767,32767,32767,32767]
-; AVX512F-NEXT:    vpminsq %zmm2, %zmm0, %zmm0
-; AVX512F-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [18446744073709518848,18446744073709518848,18446744073709518848,18446744073709518848]
-; AVX512F-NEXT:    vpmaxsq %zmm2, %zmm0, %zmm0
-; AVX512F-NEXT:    vpmovqd %zmm0, %ymm0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB4_2
-; AVX512F-NEXT:  # %bb.1: # %cond.store
-; AVX512F-NEXT:    vpextrw $0, %xmm0, (%rdi)
-; AVX512F-NEXT:  .LBB4_2: # %else
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
+; AVX512F-NEXT:    vpmovsqw %zmm0, %xmm0
 ; AVX512F-NEXT:    kmovw %k0, %eax
 ; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB4_4
-; AVX512F-NEXT:  # %bb.3: # %cond.store1
-; AVX512F-NEXT:    vpextrw $2, %xmm0, 2(%rdi)
+; AVX512F-NEXT:    jne .LBB4_1
+; AVX512F-NEXT:  # %bb.2: # %else
+; AVX512F-NEXT:    testb $2, %al
+; AVX512F-NEXT:    jne .LBB4_3
 ; AVX512F-NEXT:  .LBB4_4: # %else2
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB4_6
-; AVX512F-NEXT:  # %bb.5: # %cond.store3
-; AVX512F-NEXT:    vpextrw $4, %xmm0, 4(%rdi)
+; AVX512F-NEXT:    testb $4, %al
+; AVX512F-NEXT:    jne .LBB4_5
 ; AVX512F-NEXT:  .LBB4_6: # %else4
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB4_8
-; AVX512F-NEXT:  # %bb.7: # %cond.store5
-; AVX512F-NEXT:    vpextrw $6, %xmm0, 6(%rdi)
+; AVX512F-NEXT:    testb $8, %al
+; AVX512F-NEXT:    jne .LBB4_7
 ; AVX512F-NEXT:  .LBB4_8: # %else6
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
+; AVX512F-NEXT:  .LBB4_1: # %cond.store
+; AVX512F-NEXT:    vpextrw $0, %xmm0, (%rdi)
+; AVX512F-NEXT:    testb $2, %al
+; AVX512F-NEXT:    je .LBB4_4
+; AVX512F-NEXT:  .LBB4_3: # %cond.store1
+; AVX512F-NEXT:    vpextrw $1, %xmm0, 2(%rdi)
+; AVX512F-NEXT:    testb $4, %al
+; AVX512F-NEXT:    je .LBB4_6
+; AVX512F-NEXT:  .LBB4_5: # %cond.store3
+; AVX512F-NEXT:    vpextrw $2, %xmm0, 4(%rdi)
+; AVX512F-NEXT:    testb $8, %al
+; AVX512F-NEXT:    je .LBB4_8
+; AVX512F-NEXT:  .LBB4_7: # %cond.store5
+; AVX512F-NEXT:    vpextrw $3, %xmm0, 6(%rdi)
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: truncstore_v4i64_v4i16:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
 ; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
 ; AVX512BW-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512BW-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [32767,32767,32767,32767]
-; AVX512BW-NEXT:    vpminsq %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [18446744073709518848,18446744073709518848,18446744073709518848,18446744073709518848]
-; AVX512BW-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpmovqd %zmm0, %ymm0
-; AVX512BW-NEXT:    vpackssdw %xmm0, %xmm0, %xmm0
 ; AVX512BW-NEXT:    kshiftld $28, %k0, %k0
 ; AVX512BW-NEXT:    kshiftrd $28, %k0, %k1
+; AVX512BW-NEXT:    vpmovsqw %zmm0, %xmm0
 ; AVX512BW-NEXT:    vmovdqu16 %zmm0, (%rdi) {%k1}
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -2132,163 +2083,164 @@ define void @truncstore_v4i64_v4i16(<4 x i64> %x, <4 x i16>* %p, <4 x i32> %mask
 define void @truncstore_v4i64_v4i8(<4 x i64> %x, <4 x i8>* %p, <4 x i32> %mask) {
 ; SSE2-LABEL: truncstore_v4i64_v4i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pxor %xmm10, %xmm10
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm10
+; SSE2-NEXT:    pxor %xmm9, %xmm9
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [127,127]
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [2147483648,2147483648]
-; SSE2-NEXT:    movdqa %xmm0, %xmm5
+; SSE2-NEXT:    movdqa %xmm1, %xmm5
 ; SSE2-NEXT:    pxor %xmm4, %xmm5
-; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [2147483775,2147483775]
-; SSE2-NEXT:    movdqa %xmm9, %xmm7
+; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [2147483775,2147483775]
+; SSE2-NEXT:    movdqa %xmm10, %xmm7
 ; SSE2-NEXT:    pcmpgtd %xmm5, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm7[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm5[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm7[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[1,1,3,3]
+; SSE2-NEXT:    pand %xmm3, %xmm6
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm7[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm5
-; SSE2-NEXT:    pand %xmm5, %xmm0
+; SSE2-NEXT:    por %xmm6, %xmm5
+; SSE2-NEXT:    pand %xmm5, %xmm1
 ; SSE2-NEXT:    pandn %xmm8, %xmm5
-; SSE2-NEXT:    por %xmm0, %xmm5
-; SSE2-NEXT:    movdqa %xmm1, %xmm0
-; SSE2-NEXT:    pxor %xmm4, %xmm0
-; SSE2-NEXT:    movdqa %xmm9, %xmm3
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSE2-NEXT:    por %xmm1, %xmm5
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    pxor %xmm4, %xmm1
+; SSE2-NEXT:    movdqa %xmm10, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm0
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pand %xmm6, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE2-NEXT:    por %xmm0, %xmm3
-; SSE2-NEXT:    pand %xmm3, %xmm1
-; SSE2-NEXT:    pandn %xmm8, %xmm3
 ; SSE2-NEXT:    por %xmm1, %xmm3
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    pandn %xmm8, %xmm3
+; SSE2-NEXT:    por %xmm0, %xmm3
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [18446744073709551488,18446744073709551488]
 ; SSE2-NEXT:    movdqa %xmm3, %xmm0
 ; SSE2-NEXT:    pxor %xmm4, %xmm0
-; SSE2-NEXT:    movdqa {{.*#+}} xmm6 = [18446744071562067840,18446744071562067840]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [18446744071562067840,18446744071562067840]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm6, %xmm7
+; SSE2-NEXT:    pcmpgtd %xmm10, %xmm7
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm7[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm6, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pand %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm7[1,1,3,3]
-; SSE2-NEXT:    por %xmm0, %xmm1
-; SSE2-NEXT:    pand %xmm1, %xmm3
-; SSE2-NEXT:    pandn %xmm8, %xmm1
-; SSE2-NEXT:    por %xmm3, %xmm1
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pand %xmm1, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[1,1,3,3]
+; SSE2-NEXT:    por %xmm6, %xmm0
+; SSE2-NEXT:    pand %xmm0, %xmm3
+; SSE2-NEXT:    pandn %xmm8, %xmm0
+; SSE2-NEXT:    por %xmm3, %xmm0
 ; SSE2-NEXT:    pxor %xmm5, %xmm4
-; SSE2-NEXT:    movdqa %xmm4, %xmm0
-; SSE2-NEXT:    pcmpgtd %xmm6, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm6, %xmm4
+; SSE2-NEXT:    movdqa %xmm4, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm10, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm4
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm3, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm4, %xmm0
-; SSE2-NEXT:    pand %xmm0, %xmm5
-; SSE2-NEXT:    pandn %xmm8, %xmm0
-; SSE2-NEXT:    por %xmm5, %xmm0
-; SSE2-NEXT:    packssdw %xmm1, %xmm0
-; SSE2-NEXT:    movd %xmm10, %eax
-; SSE2-NEXT:    notl %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB5_2
-; SSE2-NEXT:  # %bb.1: # %cond.store
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    por %xmm4, %xmm1
+; SSE2-NEXT:    pand %xmm1, %xmm5
+; SSE2-NEXT:    pandn %xmm8, %xmm1
+; SSE2-NEXT:    por %xmm5, %xmm1
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [255,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0]
+; SSE2-NEXT:    pand %xmm3, %xmm1
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    packuswb %xmm1, %xmm0
+; SSE2-NEXT:    packuswb %xmm0, %xmm0
+; SSE2-NEXT:    packuswb %xmm0, %xmm0
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm9
+; SSE2-NEXT:    movmskps %xmm9, %ecx
+; SSE2-NEXT:    xorl $15, %ecx
+; SSE2-NEXT:    testb $1, %cl
 ; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    jne .LBB5_1
+; SSE2-NEXT:  # %bb.2: # %else
+; SSE2-NEXT:    testb $2, %cl
+; SSE2-NEXT:    jne .LBB5_3
+; SSE2-NEXT:  .LBB5_4: # %else2
+; SSE2-NEXT:    testb $4, %cl
+; SSE2-NEXT:    jne .LBB5_5
+; SSE2-NEXT:  .LBB5_6: # %else4
+; SSE2-NEXT:    testb $8, %cl
+; SSE2-NEXT:    jne .LBB5_7
+; SSE2-NEXT:  .LBB5_8: # %else6
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  .LBB5_1: # %cond.store
 ; SSE2-NEXT:    movb %al, (%rdi)
-; SSE2-NEXT:  .LBB5_2: # %else
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pxor %xmm1, %xmm10
-; SSE2-NEXT:    pextrw $2, %xmm10, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $2, %cl
 ; SSE2-NEXT:    je .LBB5_4
-; SSE2-NEXT:  # %bb.3: # %cond.store1
-; SSE2-NEXT:    pextrw $2, %xmm0, %eax
-; SSE2-NEXT:    movb %al, 1(%rdi)
-; SSE2-NEXT:  .LBB5_4: # %else2
-; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm2
-; SSE2-NEXT:    pxor %xmm2, %xmm1
-; SSE2-NEXT:    pextrw $4, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:  .LBB5_3: # %cond.store1
+; SSE2-NEXT:    movb %ah, 1(%rdi)
+; SSE2-NEXT:    testb $4, %cl
 ; SSE2-NEXT:    je .LBB5_6
-; SSE2-NEXT:  # %bb.5: # %cond.store3
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    movb %al, 2(%rdi)
-; SSE2-NEXT:  .LBB5_6: # %else4
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pxor %xmm1, %xmm2
-; SSE2-NEXT:    pextrw $6, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:  .LBB5_5: # %cond.store3
+; SSE2-NEXT:    movl %eax, %edx
+; SSE2-NEXT:    shrl $16, %edx
+; SSE2-NEXT:    movb %dl, 2(%rdi)
+; SSE2-NEXT:    testb $8, %cl
 ; SSE2-NEXT:    je .LBB5_8
-; SSE2-NEXT:  # %bb.7: # %cond.store5
-; SSE2-NEXT:    pextrw $6, %xmm0, %eax
+; SSE2-NEXT:  .LBB5_7: # %cond.store5
+; SSE2-NEXT:    shrl $24, %eax
 ; SSE2-NEXT:    movb %al, 3(%rdi)
-; SSE2-NEXT:  .LBB5_8: # %else6
 ; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: truncstore_v4i64_v4i8:
 ; SSE4:       # %bb.0:
 ; SSE4-NEXT:    movdqa %xmm0, %xmm3
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqd %xmm2, %xmm0
-; SSE4-NEXT:    pcmpeqd %xmm4, %xmm4
-; SSE4-NEXT:    pxor %xmm0, %xmm4
+; SSE4-NEXT:    pxor %xmm4, %xmm4
 ; SSE4-NEXT:    movdqa {{.*#+}} xmm5 = [127,127]
 ; SSE4-NEXT:    movdqa %xmm5, %xmm0
-; SSE4-NEXT:    pcmpgtq %xmm3, %xmm0
+; SSE4-NEXT:    pcmpgtq %xmm1, %xmm0
 ; SSE4-NEXT:    movdqa %xmm5, %xmm6
-; SSE4-NEXT:    blendvpd %xmm0, %xmm3, %xmm6
+; SSE4-NEXT:    blendvpd %xmm0, %xmm1, %xmm6
 ; SSE4-NEXT:    movdqa %xmm5, %xmm0
-; SSE4-NEXT:    pcmpgtq %xmm1, %xmm0
-; SSE4-NEXT:    blendvpd %xmm0, %xmm1, %xmm5
-; SSE4-NEXT:    movdqa {{.*#+}} xmm1 = [18446744073709551488,18446744073709551488]
+; SSE4-NEXT:    pcmpgtq %xmm3, %xmm0
+; SSE4-NEXT:    blendvpd %xmm0, %xmm3, %xmm5
+; SSE4-NEXT:    movdqa {{.*#+}} xmm3 = [18446744073709551488,18446744073709551488]
 ; SSE4-NEXT:    movapd %xmm5, %xmm0
-; SSE4-NEXT:    pcmpgtq %xmm1, %xmm0
-; SSE4-NEXT:    movdqa %xmm1, %xmm3
-; SSE4-NEXT:    blendvpd %xmm0, %xmm5, %xmm3
+; SSE4-NEXT:    pcmpgtq %xmm3, %xmm0
+; SSE4-NEXT:    movdqa %xmm3, %xmm1
+; SSE4-NEXT:    blendvpd %xmm0, %xmm5, %xmm1
 ; SSE4-NEXT:    movapd %xmm6, %xmm0
-; SSE4-NEXT:    pcmpgtq %xmm1, %xmm0
-; SSE4-NEXT:    blendvpd %xmm0, %xmm6, %xmm1
-; SSE4-NEXT:    packssdw %xmm3, %xmm1
-; SSE4-NEXT:    pextrb $0, %xmm4, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB5_2
-; SSE4-NEXT:  # %bb.1: # %cond.store
+; SSE4-NEXT:    pcmpgtq %xmm3, %xmm0
+; SSE4-NEXT:    blendvpd %xmm0, %xmm6, %xmm3
+; SSE4-NEXT:    movdqa {{.*#+}} xmm0 = <0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; SSE4-NEXT:    pshufb %xmm0, %xmm3
+; SSE4-NEXT:    pshufb %xmm0, %xmm1
+; SSE4-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; SSE4-NEXT:    pcmpeqd %xmm2, %xmm4
+; SSE4-NEXT:    movmskps %xmm4, %eax
+; SSE4-NEXT:    xorl $15, %eax
+; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    jne .LBB5_1
+; SSE4-NEXT:  # %bb.2: # %else
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    jne .LBB5_3
+; SSE4-NEXT:  .LBB5_4: # %else2
+; SSE4-NEXT:    testb $4, %al
+; SSE4-NEXT:    jne .LBB5_5
+; SSE4-NEXT:  .LBB5_6: # %else4
+; SSE4-NEXT:    testb $8, %al
+; SSE4-NEXT:    jne .LBB5_7
+; SSE4-NEXT:  .LBB5_8: # %else6
+; SSE4-NEXT:    retq
+; SSE4-NEXT:  .LBB5_1: # %cond.store
 ; SSE4-NEXT:    pextrb $0, %xmm1, (%rdi)
-; SSE4-NEXT:  .LBB5_2: # %else
-; SSE4-NEXT:    pextrb $4, %xmm4, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $2, %al
 ; SSE4-NEXT:    je .LBB5_4
-; SSE4-NEXT:  # %bb.3: # %cond.store1
-; SSE4-NEXT:    pextrb $4, %xmm1, 1(%rdi)
-; SSE4-NEXT:  .LBB5_4: # %else2
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE4-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE4-NEXT:    pxor %xmm2, %xmm0
-; SSE4-NEXT:    pextrb $8, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:  .LBB5_3: # %cond.store1
+; SSE4-NEXT:    pextrb $1, %xmm1, 1(%rdi)
+; SSE4-NEXT:    testb $4, %al
 ; SSE4-NEXT:    je .LBB5_6
-; SSE4-NEXT:  # %bb.5: # %cond.store3
-; SSE4-NEXT:    pextrb $8, %xmm1, 2(%rdi)
-; SSE4-NEXT:  .LBB5_6: # %else4
-; SSE4-NEXT:    pextrb $12, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:  .LBB5_5: # %cond.store3
+; SSE4-NEXT:    pextrb $2, %xmm1, 2(%rdi)
+; SSE4-NEXT:    testb $8, %al
 ; SSE4-NEXT:    je .LBB5_8
-; SSE4-NEXT:  # %bb.7: # %cond.store5
-; SSE4-NEXT:    pextrb $12, %xmm1, 3(%rdi)
-; SSE4-NEXT:  .LBB5_8: # %else6
+; SSE4-NEXT:  .LBB5_7: # %cond.store5
+; SSE4-NEXT:    pextrb $3, %xmm1, 3(%rdi)
 ; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: truncstore_v4i64_v4i8:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [127,127]
 ; AVX1-NEXT:    vpcmpgtq %xmm3, %xmm4, %xmm5
@@ -2299,45 +2251,48 @@ define void @truncstore_v4i64_v4i8(<4 x i64> %x, <4 x i8>* %p, <4 x i32> %mask)
 ; AVX1-NEXT:    vblendvpd %xmm5, %xmm3, %xmm4, %xmm3
 ; AVX1-NEXT:    vpcmpgtq %xmm6, %xmm3, %xmm4
 ; AVX1-NEXT:    vblendvpd %xmm4, %xmm3, %xmm6, %xmm3
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = <0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX1-NEXT:    vpshufb %xmm4, %xmm3, %xmm3
 ; AVX1-NEXT:    vblendvpd %xmm7, %xmm0, %xmm6, %xmm0
-; AVX1-NEXT:    vpackssdw %xmm3, %xmm0, %xmm0
-; AVX1-NEXT:    vpextrb $0, %xmm2, %eax
+; AVX1-NEXT:    vpshufb %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vmovmskps %xmm1, %eax
+; AVX1-NEXT:    xorl $15, %eax
 ; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je .LBB5_2
-; AVX1-NEXT:  # %bb.1: # %cond.store
+; AVX1-NEXT:    jne .LBB5_1
+; AVX1-NEXT:  # %bb.2: # %else
+; AVX1-NEXT:    testb $2, %al
+; AVX1-NEXT:    jne .LBB5_3
+; AVX1-NEXT:  .LBB5_4: # %else2
+; AVX1-NEXT:    testb $4, %al
+; AVX1-NEXT:    jne .LBB5_5
+; AVX1-NEXT:  .LBB5_6: # %else4
+; AVX1-NEXT:    testb $8, %al
+; AVX1-NEXT:    jne .LBB5_7
+; AVX1-NEXT:  .LBB5_8: # %else6
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+; AVX1-NEXT:  .LBB5_1: # %cond.store
 ; AVX1-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX1-NEXT:  .LBB5_2: # %else
-; AVX1-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $2, %al
 ; AVX1-NEXT:    je .LBB5_4
-; AVX1-NEXT:  # %bb.3: # %cond.store1
-; AVX1-NEXT:    vpextrb $4, %xmm0, 1(%rdi)
-; AVX1-NEXT:  .LBB5_4: # %else2
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:  .LBB5_3: # %cond.store1
+; AVX1-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
+; AVX1-NEXT:    testb $4, %al
 ; AVX1-NEXT:    je .LBB5_6
-; AVX1-NEXT:  # %bb.5: # %cond.store3
-; AVX1-NEXT:    vpextrb $8, %xmm0, 2(%rdi)
-; AVX1-NEXT:  .LBB5_6: # %else4
-; AVX1-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:  .LBB5_5: # %cond.store3
+; AVX1-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
+; AVX1-NEXT:    testb $8, %al
 ; AVX1-NEXT:    je .LBB5_8
-; AVX1-NEXT:  # %bb.7: # %cond.store5
-; AVX1-NEXT:    vpextrb $12, %xmm0, 3(%rdi)
-; AVX1-NEXT:  .LBB5_8: # %else6
+; AVX1-NEXT:  .LBB5_7: # %cond.store5
+; AVX1-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: truncstore_v4i64_v4i8:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    vpxor %xmm3, %xmm2, %xmm2
 ; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [127,127,127,127]
 ; AVX2-NEXT:    vpcmpgtq %ymm0, %ymm3, %ymm4
 ; AVX2-NEXT:    vblendvpd %ymm4, %ymm0, %ymm3, %ymm0
@@ -2345,35 +2300,41 @@ define void @truncstore_v4i64_v4i8(<4 x i64> %x, <4 x i8>* %p, <4 x i32> %mask)
 ; AVX2-NEXT:    vpcmpgtq %ymm3, %ymm0, %ymm4
 ; AVX2-NEXT:    vblendvpd %ymm4, %ymm0, %ymm3, %ymm0
 ; AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm3
-; AVX2-NEXT:    vpackssdw %xmm3, %xmm0, %xmm0
-; AVX2-NEXT:    vpextrb $0, %xmm2, %eax
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm4 = <0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX2-NEXT:    vpshufb %xmm4, %xmm3, %xmm3
+; AVX2-NEXT:    vpshufb %xmm4, %xmm0, %xmm0
+; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vmovmskps %xmm1, %eax
+; AVX2-NEXT:    xorl $15, %eax
 ; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je .LBB5_2
-; AVX2-NEXT:  # %bb.1: # %cond.store
+; AVX2-NEXT:    jne .LBB5_1
+; AVX2-NEXT:  # %bb.2: # %else
+; AVX2-NEXT:    testb $2, %al
+; AVX2-NEXT:    jne .LBB5_3
+; AVX2-NEXT:  .LBB5_4: # %else2
+; AVX2-NEXT:    testb $4, %al
+; AVX2-NEXT:    jne .LBB5_5
+; AVX2-NEXT:  .LBB5_6: # %else4
+; AVX2-NEXT:    testb $8, %al
+; AVX2-NEXT:    jne .LBB5_7
+; AVX2-NEXT:  .LBB5_8: # %else6
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+; AVX2-NEXT:  .LBB5_1: # %cond.store
 ; AVX2-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX2-NEXT:  .LBB5_2: # %else
-; AVX2-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $2, %al
 ; AVX2-NEXT:    je .LBB5_4
-; AVX2-NEXT:  # %bb.3: # %cond.store1
-; AVX2-NEXT:    vpextrb $4, %xmm0, 1(%rdi)
-; AVX2-NEXT:  .LBB5_4: # %else2
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:  .LBB5_3: # %cond.store1
+; AVX2-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
+; AVX2-NEXT:    testb $4, %al
 ; AVX2-NEXT:    je .LBB5_6
-; AVX2-NEXT:  # %bb.5: # %cond.store3
-; AVX2-NEXT:    vpextrb $8, %xmm0, 2(%rdi)
-; AVX2-NEXT:  .LBB5_6: # %else4
-; AVX2-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:  .LBB5_5: # %cond.store3
+; AVX2-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
+; AVX2-NEXT:    testb $8, %al
 ; AVX2-NEXT:    je .LBB5_8
-; AVX2-NEXT:  # %bb.7: # %cond.store5
-; AVX2-NEXT:    vpextrb $12, %xmm0, 3(%rdi)
-; AVX2-NEXT:  .LBB5_8: # %else6
+; AVX2-NEXT:  .LBB5_7: # %cond.store5
+; AVX2-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
@@ -2382,41 +2343,36 @@ define void @truncstore_v4i64_v4i8(<4 x i64> %x, <4 x i8>* %p, <4 x i32> %mask)
 ; AVX512F-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
 ; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
 ; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [127,127,127,127]
-; AVX512F-NEXT:    vpminsq %zmm2, %zmm0, %zmm0
-; AVX512F-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [18446744073709551488,18446744073709551488,18446744073709551488,18446744073709551488]
-; AVX512F-NEXT:    vpmaxsq %zmm2, %zmm0, %zmm0
-; AVX512F-NEXT:    vpmovqd %zmm0, %ymm0
+; AVX512F-NEXT:    vpmovsqb %zmm0, %xmm0
 ; AVX512F-NEXT:    kmovw %k0, %eax
 ; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB5_2
-; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    jne .LBB5_1
+; AVX512F-NEXT:  # %bb.2: # %else
+; AVX512F-NEXT:    testb $2, %al
+; AVX512F-NEXT:    jne .LBB5_3
+; AVX512F-NEXT:  .LBB5_4: # %else2
+; AVX512F-NEXT:    testb $4, %al
+; AVX512F-NEXT:    jne .LBB5_5
+; AVX512F-NEXT:  .LBB5_6: # %else4
+; AVX512F-NEXT:    testb $8, %al
+; AVX512F-NEXT:    jne .LBB5_7
+; AVX512F-NEXT:  .LBB5_8: # %else6
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+; AVX512F-NEXT:  .LBB5_1: # %cond.store
 ; AVX512F-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX512F-NEXT:  .LBB5_2: # %else
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $2, %al
 ; AVX512F-NEXT:    je .LBB5_4
-; AVX512F-NEXT:  # %bb.3: # %cond.store1
-; AVX512F-NEXT:    vpextrb $4, %xmm0, 1(%rdi)
-; AVX512F-NEXT:  .LBB5_4: # %else2
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  .LBB5_3: # %cond.store1
+; AVX512F-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
+; AVX512F-NEXT:    testb $4, %al
 ; AVX512F-NEXT:    je .LBB5_6
-; AVX512F-NEXT:  # %bb.5: # %cond.store3
-; AVX512F-NEXT:    vpextrb $8, %xmm0, 2(%rdi)
-; AVX512F-NEXT:  .LBB5_6: # %else4
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  .LBB5_5: # %cond.store3
+; AVX512F-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
+; AVX512F-NEXT:    testb $8, %al
 ; AVX512F-NEXT:    je .LBB5_8
-; AVX512F-NEXT:  # %bb.7: # %cond.store5
-; AVX512F-NEXT:    vpextrb $12, %xmm0, 3(%rdi)
-; AVX512F-NEXT:  .LBB5_8: # %else6
+; AVX512F-NEXT:  .LBB5_7: # %cond.store5
+; AVX512F-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
@@ -2425,14 +2381,9 @@ define void @truncstore_v4i64_v4i8(<4 x i64> %x, <4 x i8>* %p, <4 x i32> %mask)
 ; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
 ; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
 ; AVX512BW-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512BW-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [127,127,127,127]
-; AVX512BW-NEXT:    vpminsq %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [18446744073709551488,18446744073709551488,18446744073709551488,18446744073709551488]
-; AVX512BW-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpmovqd %zmm0, %ymm0
-; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
 ; AVX512BW-NEXT:    kshiftlq $60, %k0, %k0
 ; AVX512BW-NEXT:    kshiftrq $60, %k0, %k1
+; AVX512BW-NEXT:    vpmovsqb %zmm0, %xmm0
 ; AVX512BW-NEXT:    vmovdqu8 %zmm0, (%rdi) {%k1}
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -2459,83 +2410,86 @@ define void @truncstore_v2i64_v2i32(<2 x i64> %x, <2 x i32>* %p, <2 x i64> %mask
 ; SSE2-LABEL: truncstore_v2i64_v2i32:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm0, %xmm4
+; SSE2-NEXT:    pxor %xmm3, %xmm4
+; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [4294967295,4294967295]
+; SSE2-NEXT:    movdqa %xmm5, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    pand %xmm7, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm6[1,1,3,3]
+; SSE2-NEXT:    por %xmm4, %xmm5
+; SSE2-NEXT:    pand %xmm5, %xmm0
+; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm5
+; SSE2-NEXT:    por %xmm0, %xmm5
+; SSE2-NEXT:    pxor %xmm5, %xmm3
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [18446744069414584320,18446744069414584320]
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm0, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
+; SSE2-NEXT:    pand %xmm6, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm0, %xmm3
+; SSE2-NEXT:    pand %xmm3, %xmm5
+; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm3
+; SSE2-NEXT:    por %xmm5, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,2,2,3]
 ; SSE2-NEXT:    pcmpeqd %xmm1, %xmm2
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,0,3,2]
 ; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
-; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    pxor %xmm2, %xmm3
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [4294967295,4294967295]
-; SSE2-NEXT:    movdqa %xmm4, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm5[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm4
-; SSE2-NEXT:    pand %xmm4, %xmm0
-; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm4
-; SSE2-NEXT:    por %xmm0, %xmm4
-; SSE2-NEXT:    pxor %xmm4, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [18446744069414584320,18446744069414584320]
-; SSE2-NEXT:    movdqa %xmm2, %xmm3
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT:    pand %xmm5, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE2-NEXT:    por %xmm2, %xmm0
-; SSE2-NEXT:    pand %xmm0, %xmm4
-; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm0
-; SSE2-NEXT:    por %xmm4, %xmm0
-; SSE2-NEXT:    movd %xmm1, %eax
-; SSE2-NEXT:    notl %eax
+; SSE2-NEXT:    movmskpd %xmm1, %eax
+; SSE2-NEXT:    xorl $3, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB6_2
-; SSE2-NEXT:  # %bb.1: # %cond.store
+; SSE2-NEXT:    jne .LBB6_1
+; SSE2-NEXT:  # %bb.2: # %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne .LBB6_3
+; SSE2-NEXT:  .LBB6_4: # %else2
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  .LBB6_1: # %cond.store
 ; SSE2-NEXT:    movd %xmm0, (%rdi)
-; SSE2-NEXT:  .LBB6_2: # %else
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT:    pxor %xmm2, %xmm1
-; SSE2-NEXT:    pextrw $4, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $2, %al
 ; SSE2-NEXT:    je .LBB6_4
-; SSE2-NEXT:  # %bb.3: # %cond.store1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE2-NEXT:  .LBB6_3: # %cond.store1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
 ; SSE2-NEXT:    movd %xmm0, 4(%rdi)
-; SSE2-NEXT:  .LBB6_4: # %else2
 ; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: truncstore_v2i64_v2i32:
 ; SSE4:       # %bb.0:
 ; SSE4-NEXT:    movdqa %xmm0, %xmm2
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqq %xmm1, %xmm0
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE4-NEXT:    pxor %xmm0, %xmm1
-; SSE4-NEXT:    movdqa {{.*#+}} xmm3 = [2147483647,2147483647]
-; SSE4-NEXT:    movdqa %xmm3, %xmm0
+; SSE4-NEXT:    pxor %xmm3, %xmm3
+; SSE4-NEXT:    movdqa {{.*#+}} xmm4 = [2147483647,2147483647]
+; SSE4-NEXT:    movdqa %xmm4, %xmm0
 ; SSE4-NEXT:    pcmpgtq %xmm2, %xmm0
-; SSE4-NEXT:    blendvpd %xmm0, %xmm2, %xmm3
+; SSE4-NEXT:    blendvpd %xmm0, %xmm2, %xmm4
 ; SSE4-NEXT:    movdqa {{.*#+}} xmm2 = [18446744071562067968,18446744071562067968]
-; SSE4-NEXT:    movapd %xmm3, %xmm0
+; SSE4-NEXT:    movapd %xmm4, %xmm0
 ; SSE4-NEXT:    pcmpgtq %xmm2, %xmm0
-; SSE4-NEXT:    blendvpd %xmm0, %xmm3, %xmm2
-; SSE4-NEXT:    pextrb $0, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB6_2
-; SSE4-NEXT:  # %bb.1: # %cond.store
-; SSE4-NEXT:    movss %xmm2, (%rdi)
-; SSE4-NEXT:  .LBB6_2: # %else
-; SSE4-NEXT:    pextrb $8, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB6_4
-; SSE4-NEXT:  # %bb.3: # %cond.store1
-; SSE4-NEXT:    extractps $2, %xmm2, 4(%rdi)
+; SSE4-NEXT:    blendvpd %xmm0, %xmm4, %xmm2
+; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
+; SSE4-NEXT:    pcmpeqq %xmm1, %xmm3
+; SSE4-NEXT:    movmskpd %xmm3, %eax
+; SSE4-NEXT:    xorl $3, %eax
+; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    jne .LBB6_1
+; SSE4-NEXT:  # %bb.2: # %else
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    jne .LBB6_3
 ; SSE4-NEXT:  .LBB6_4: # %else2
 ; SSE4-NEXT:    retq
+; SSE4-NEXT:  .LBB6_1: # %cond.store
+; SSE4-NEXT:    movd %xmm0, (%rdi)
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    je .LBB6_4
+; SSE4-NEXT:  .LBB6_3: # %cond.store1
+; SSE4-NEXT:    pextrd $1, %xmm0, 4(%rdi)
+; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: truncstore_v2i64_v2i32:
 ; AVX1:       # %bb.0:
@@ -2543,6 +2497,7 @@ define void @truncstore_v2i64_v2i32(<2 x i64> %x, <2 x i32>* %p, <2 x i64> %mask
 ; AVX1-NEXT:    vpcmpeqq %xmm2, %xmm1, %xmm1
 ; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
 ; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,2],zero,zero
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [2147483647,2147483647]
 ; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm3
 ; AVX1-NEXT:    vblendvpd %xmm3, %xmm0, %xmm2, %xmm0
@@ -2550,7 +2505,6 @@ define void @truncstore_v2i64_v2i32(<2 x i64> %x, <2 x i32>* %p, <2 x i64> %mask
 ; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm0, %xmm3
 ; AVX1-NEXT:    vblendvpd %xmm3, %xmm0, %xmm2, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; AVX1-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,2],zero,zero
 ; AVX1-NEXT:    vmaskmovps %xmm0, %xmm1, (%rdi)
 ; AVX1-NEXT:    retq
 ;
@@ -2560,6 +2514,7 @@ define void @truncstore_v2i64_v2i32(<2 x i64> %x, <2 x i32>* %p, <2 x i64> %mask
 ; AVX2-NEXT:    vpcmpeqq %xmm2, %xmm1, %xmm1
 ; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
 ; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,2],zero,zero
 ; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [2147483647,2147483647]
 ; AVX2-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm3
 ; AVX2-NEXT:    vblendvpd %xmm3, %xmm0, %xmm2, %xmm0
@@ -2567,7 +2522,6 @@ define void @truncstore_v2i64_v2i32(<2 x i64> %x, <2 x i32>* %p, <2 x i64> %mask
 ; AVX2-NEXT:    vpcmpgtq %xmm2, %xmm0, %xmm3
 ; AVX2-NEXT:    vblendvpd %xmm3, %xmm0, %xmm2, %xmm0
 ; AVX2-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; AVX2-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,2],zero,zero
 ; AVX2-NEXT:    vpmaskmovd %xmm0, %xmm1, (%rdi)
 ; AVX2-NEXT:    retq
 ;
@@ -2576,13 +2530,9 @@ define void @truncstore_v2i64_v2i32(<2 x i64> %x, <2 x i32>* %p, <2 x i64> %mask
 ; AVX512F-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
 ; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; AVX512F-NEXT:    vptestmq %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm1 = [2147483647,2147483647]
-; AVX512F-NEXT:    vpminsq %zmm1, %zmm0, %zmm0
-; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm1 = [18446744071562067968,18446744071562067968]
-; AVX512F-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm0
-; AVX512F-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; AVX512F-NEXT:    kshiftlw $14, %k0, %k0
 ; AVX512F-NEXT:    kshiftrw $14, %k0, %k1
+; AVX512F-NEXT:    vpmovsqd %zmm0, %ymm0
 ; AVX512F-NEXT:    vmovdqu32 %zmm0, (%rdi) {%k1}
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
@@ -2600,13 +2550,9 @@ define void @truncstore_v2i64_v2i32(<2 x i64> %x, <2 x i32>* %p, <2 x i64> %mask
 ; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
 ; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; AVX512BW-NEXT:    vptestmq %zmm1, %zmm1, %k0
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm1 = [2147483647,2147483647]
-; AVX512BW-NEXT:    vpminsq %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm1 = [18446744071562067968,18446744071562067968]
-; AVX512BW-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; AVX512BW-NEXT:    kshiftlw $14, %k0, %k0
 ; AVX512BW-NEXT:    kshiftrw $14, %k0, %k1
+; AVX512BW-NEXT:    vpmovsqd %zmm0, %ymm0
 ; AVX512BW-NEXT:    vmovdqu32 %zmm0, (%rdi) {%k1}
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -2624,109 +2570,117 @@ define void @truncstore_v2i64_v2i16(<2 x i64> %x, <2 x i16>* %p, <2 x i64> %mask
 ; SSE2-LABEL: truncstore_v2i64_v2i16:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm0, %xmm4
+; SSE2-NEXT:    pxor %xmm3, %xmm4
+; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [2147516415,2147516415]
+; SSE2-NEXT:    movdqa %xmm5, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    pand %xmm7, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm6[1,1,3,3]
+; SSE2-NEXT:    por %xmm4, %xmm5
+; SSE2-NEXT:    pand %xmm5, %xmm0
+; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm5
+; SSE2-NEXT:    por %xmm0, %xmm5
+; SSE2-NEXT:    pxor %xmm5, %xmm3
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [18446744071562035200,18446744071562035200]
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm0, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
+; SSE2-NEXT:    pand %xmm6, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm0, %xmm3
+; SSE2-NEXT:    pand %xmm3, %xmm5
+; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm3
+; SSE2-NEXT:    por %xmm5, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,2,2,3]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
 ; SSE2-NEXT:    pcmpeqd %xmm1, %xmm2
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,0,3,2]
 ; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
-; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    pxor %xmm2, %xmm3
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [2147516415,2147516415]
-; SSE2-NEXT:    movdqa %xmm4, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm5[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm4
-; SSE2-NEXT:    pand %xmm4, %xmm0
-; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm4
-; SSE2-NEXT:    por %xmm0, %xmm4
-; SSE2-NEXT:    pxor %xmm4, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [18446744071562035200,18446744071562035200]
-; SSE2-NEXT:    movdqa %xmm2, %xmm3
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT:    pand %xmm5, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE2-NEXT:    por %xmm2, %xmm0
-; SSE2-NEXT:    pand %xmm0, %xmm4
-; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm0
-; SSE2-NEXT:    por %xmm4, %xmm0
-; SSE2-NEXT:    movd %xmm1, %eax
-; SSE2-NEXT:    notl %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB7_2
-; SSE2-NEXT:  # %bb.1: # %cond.store
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    movw %ax, (%rdi)
-; SSE2-NEXT:  .LBB7_2: # %else
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT:    pxor %xmm2, %xmm1
-; SSE2-NEXT:    pextrw $4, %xmm1, %eax
+; SSE2-NEXT:    movmskpd %xmm1, %eax
+; SSE2-NEXT:    xorl $3, %eax
 ; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    jne .LBB7_1
+; SSE2-NEXT:  # %bb.2: # %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne .LBB7_3
+; SSE2-NEXT:  .LBB7_4: # %else2
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  .LBB7_1: # %cond.store
+; SSE2-NEXT:    movd %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, (%rdi)
+; SSE2-NEXT:    testb $2, %al
 ; SSE2-NEXT:    je .LBB7_4
-; SSE2-NEXT:  # %bb.3: # %cond.store1
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
+; SSE2-NEXT:  .LBB7_3: # %cond.store1
+; SSE2-NEXT:    pextrw $1, %xmm0, %eax
 ; SSE2-NEXT:    movw %ax, 2(%rdi)
-; SSE2-NEXT:  .LBB7_4: # %else2
 ; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: truncstore_v2i64_v2i16:
 ; SSE4:       # %bb.0:
 ; SSE4-NEXT:    movdqa %xmm0, %xmm2
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqq %xmm1, %xmm0
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE4-NEXT:    pxor %xmm0, %xmm1
-; SSE4-NEXT:    movdqa {{.*#+}} xmm3 = [32767,32767]
-; SSE4-NEXT:    movdqa %xmm3, %xmm0
+; SSE4-NEXT:    pxor %xmm3, %xmm3
+; SSE4-NEXT:    movdqa {{.*#+}} xmm4 = [32767,32767]
+; SSE4-NEXT:    movdqa %xmm4, %xmm0
 ; SSE4-NEXT:    pcmpgtq %xmm2, %xmm0
-; SSE4-NEXT:    blendvpd %xmm0, %xmm2, %xmm3
+; SSE4-NEXT:    blendvpd %xmm0, %xmm2, %xmm4
 ; SSE4-NEXT:    movdqa {{.*#+}} xmm2 = [18446744073709518848,18446744073709518848]
-; SSE4-NEXT:    movapd %xmm3, %xmm0
+; SSE4-NEXT:    movapd %xmm4, %xmm0
 ; SSE4-NEXT:    pcmpgtq %xmm2, %xmm0
-; SSE4-NEXT:    blendvpd %xmm0, %xmm3, %xmm2
-; SSE4-NEXT:    pextrb $0, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB7_2
-; SSE4-NEXT:  # %bb.1: # %cond.store
-; SSE4-NEXT:    pextrw $0, %xmm2, (%rdi)
-; SSE4-NEXT:  .LBB7_2: # %else
-; SSE4-NEXT:    pextrb $8, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB7_4
-; SSE4-NEXT:  # %bb.3: # %cond.store1
-; SSE4-NEXT:    pextrw $4, %xmm2, 2(%rdi)
+; SSE4-NEXT:    blendvpd %xmm0, %xmm4, %xmm2
+; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
+; SSE4-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; SSE4-NEXT:    pcmpeqq %xmm1, %xmm3
+; SSE4-NEXT:    movmskpd %xmm3, %eax
+; SSE4-NEXT:    xorl $3, %eax
+; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    jne .LBB7_1
+; SSE4-NEXT:  # %bb.2: # %else
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    jne .LBB7_3
 ; SSE4-NEXT:  .LBB7_4: # %else2
 ; SSE4-NEXT:    retq
+; SSE4-NEXT:  .LBB7_1: # %cond.store
+; SSE4-NEXT:    pextrw $0, %xmm0, (%rdi)
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    je .LBB7_4
+; SSE4-NEXT:  .LBB7_3: # %cond.store1
+; SSE4-NEXT:    pextrw $1, %xmm0, 2(%rdi)
+; SSE4-NEXT:    retq
 ;
 ; AVX-LABEL: truncstore_v2i64_v2i16:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm3 = [32767,32767]
+; AVX-NEXT:    vpcmpgtq %xmm0, %xmm3, %xmm4
+; AVX-NEXT:    vblendvpd %xmm4, %xmm0, %xmm3, %xmm0
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm3 = [18446744073709518848,18446744073709518848]
+; AVX-NEXT:    vpcmpgtq %xmm3, %xmm0, %xmm4
+; AVX-NEXT:    vblendvpd %xmm4, %xmm0, %xmm3, %xmm0
+; AVX-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
 ; AVX-NEXT:    vpcmpeqq %xmm2, %xmm1, %xmm1
-; AVX-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX-NEXT:    vmovdqa {{.*#+}} xmm2 = [32767,32767]
-; AVX-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm3
-; AVX-NEXT:    vblendvpd %xmm3, %xmm0, %xmm2, %xmm0
-; AVX-NEXT:    vmovdqa {{.*#+}} xmm2 = [18446744073709518848,18446744073709518848]
-; AVX-NEXT:    vpcmpgtq %xmm2, %xmm0, %xmm3
-; AVX-NEXT:    vblendvpd %xmm3, %xmm0, %xmm2, %xmm0
-; AVX-NEXT:    vpextrb $0, %xmm1, %eax
+; AVX-NEXT:    vmovmskpd %xmm1, %eax
+; AVX-NEXT:    xorl $3, %eax
 ; AVX-NEXT:    testb $1, %al
-; AVX-NEXT:    je .LBB7_2
-; AVX-NEXT:  # %bb.1: # %cond.store
+; AVX-NEXT:    jne .LBB7_1
+; AVX-NEXT:  # %bb.2: # %else
+; AVX-NEXT:    testb $2, %al
+; AVX-NEXT:    jne .LBB7_3
+; AVX-NEXT:  .LBB7_4: # %else2
+; AVX-NEXT:    retq
+; AVX-NEXT:  .LBB7_1: # %cond.store
 ; AVX-NEXT:    vpextrw $0, %xmm0, (%rdi)
-; AVX-NEXT:  .LBB7_2: # %else
-; AVX-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX-NEXT:    testb $1, %al
+; AVX-NEXT:    testb $2, %al
 ; AVX-NEXT:    je .LBB7_4
-; AVX-NEXT:  # %bb.3: # %cond.store1
-; AVX-NEXT:    vpextrw $4, %xmm0, 2(%rdi)
-; AVX-NEXT:  .LBB7_4: # %else2
+; AVX-NEXT:  .LBB7_3: # %cond.store1
+; AVX-NEXT:    vpextrw $1, %xmm0, 2(%rdi)
 ; AVX-NEXT:    retq
 ;
 ; AVX512F-LABEL: truncstore_v2i64_v2i16:
@@ -2734,24 +2688,22 @@ define void @truncstore_v2i64_v2i16(<2 x i64> %x, <2 x i16>* %p, <2 x i64> %mask
 ; AVX512F-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
 ; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; AVX512F-NEXT:    vptestmq %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm2 = [32767,32767]
-; AVX512F-NEXT:    vpminsq %zmm2, %zmm0, %zmm0
-; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm2 = [18446744073709518848,18446744073709518848]
-; AVX512F-NEXT:    vpmaxsq %zmm2, %zmm0, %zmm0
+; AVX512F-NEXT:    vpmovsqw %zmm0, %xmm0
 ; AVX512F-NEXT:    kmovw %k0, %eax
 ; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB7_2
-; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    jne .LBB7_1
+; AVX512F-NEXT:  # %bb.2: # %else
+; AVX512F-NEXT:    testb $2, %al
+; AVX512F-NEXT:    jne .LBB7_3
+; AVX512F-NEXT:  .LBB7_4: # %else2
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+; AVX512F-NEXT:  .LBB7_1: # %cond.store
 ; AVX512F-NEXT:    vpextrw $0, %xmm0, (%rdi)
-; AVX512F-NEXT:  .LBB7_2: # %else
-; AVX512F-NEXT:    vptestmq %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $2, %al
 ; AVX512F-NEXT:    je .LBB7_4
-; AVX512F-NEXT:  # %bb.3: # %cond.store1
-; AVX512F-NEXT:    vpextrw $4, %xmm0, 2(%rdi)
-; AVX512F-NEXT:  .LBB7_4: # %else2
+; AVX512F-NEXT:  .LBB7_3: # %cond.store1
+; AVX512F-NEXT:    vpextrw $1, %xmm0, 2(%rdi)
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
@@ -2760,14 +2712,9 @@ define void @truncstore_v2i64_v2i16(<2 x i64> %x, <2 x i16>* %p, <2 x i64> %mask
 ; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
 ; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; AVX512BW-NEXT:    vptestmq %zmm1, %zmm1, %k0
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm1 = [32767,32767]
-; AVX512BW-NEXT:    vpminsq %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm1 = [18446744073709518848,18446744073709518848]
-; AVX512BW-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; AVX512BW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
 ; AVX512BW-NEXT:    kshiftld $30, %k0, %k0
 ; AVX512BW-NEXT:    kshiftrd $30, %k0, %k1
+; AVX512BW-NEXT:    vpmovsqw %zmm0, %xmm0
 ; AVX512BW-NEXT:    vmovdqu16 %zmm0, (%rdi) {%k1}
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -2793,109 +2740,116 @@ define void @truncstore_v2i64_v2i8(<2 x i64> %x, <2 x i8>* %p, <2 x i64> %mask)
 ; SSE2-LABEL: truncstore_v2i64_v2i8:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,0,3,2]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
-; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    pxor %xmm2, %xmm3
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [2147483775,2147483775]
-; SSE2-NEXT:    movdqa %xmm4, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm5[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm4
-; SSE2-NEXT:    pand %xmm4, %xmm0
-; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm4
-; SSE2-NEXT:    por %xmm0, %xmm4
-; SSE2-NEXT:    pxor %xmm4, %xmm2
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm0, %xmm4
+; SSE2-NEXT:    pxor %xmm3, %xmm4
+; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [2147483775,2147483775]
+; SSE2-NEXT:    movdqa %xmm5, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    pand %xmm7, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm6[1,1,3,3]
+; SSE2-NEXT:    por %xmm4, %xmm5
+; SSE2-NEXT:    pand %xmm5, %xmm0
+; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm5
+; SSE2-NEXT:    por %xmm0, %xmm5
+; SSE2-NEXT:    pxor %xmm5, %xmm3
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [18446744071562067840,18446744071562067840]
-; SSE2-NEXT:    movdqa %xmm2, %xmm3
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT:    pand %xmm5, %xmm2
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm0, %xmm3
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE2-NEXT:    por %xmm2, %xmm0
-; SSE2-NEXT:    pand %xmm0, %xmm4
-; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm0
-; SSE2-NEXT:    por %xmm4, %xmm0
-; SSE2-NEXT:    movd %xmm1, %eax
-; SSE2-NEXT:    notl %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB8_2
-; SSE2-NEXT:  # %bb.1: # %cond.store
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    movb %al, (%rdi)
-; SSE2-NEXT:  .LBB8_2: # %else
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT:    pxor %xmm2, %xmm1
-; SSE2-NEXT:    pextrw $4, %xmm1, %eax
+; SSE2-NEXT:    pand %xmm6, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm0, %xmm3
+; SSE2-NEXT:    pand %xmm3, %xmm5
+; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm3
+; SSE2-NEXT:    por %xmm5, %xmm3
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm3
+; SSE2-NEXT:    packuswb %xmm3, %xmm3
+; SSE2-NEXT:    packuswb %xmm3, %xmm3
+; SSE2-NEXT:    packuswb %xmm3, %xmm3
+; SSE2-NEXT:    pcmpeqd %xmm1, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,0,3,2]
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    movmskpd %xmm0, %eax
+; SSE2-NEXT:    xorl $3, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB8_4
-; SSE2-NEXT:  # %bb.3: # %cond.store1
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    movb %al, 1(%rdi)
+; SSE2-NEXT:    movd %xmm3, %ecx
+; SSE2-NEXT:    jne .LBB8_1
+; SSE2-NEXT:  # %bb.2: # %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne .LBB8_3
 ; SSE2-NEXT:  .LBB8_4: # %else2
 ; SSE2-NEXT:    retq
+; SSE2-NEXT:  .LBB8_1: # %cond.store
+; SSE2-NEXT:    movb %cl, (%rdi)
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    je .LBB8_4
+; SSE2-NEXT:  .LBB8_3: # %cond.store1
+; SSE2-NEXT:    movb %ch, 1(%rdi)
+; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: truncstore_v2i64_v2i8:
 ; SSE4:       # %bb.0:
 ; SSE4-NEXT:    movdqa %xmm0, %xmm2
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqq %xmm1, %xmm0
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE4-NEXT:    pxor %xmm0, %xmm1
-; SSE4-NEXT:    movdqa {{.*#+}} xmm3 = [127,127]
-; SSE4-NEXT:    movdqa %xmm3, %xmm0
+; SSE4-NEXT:    pxor %xmm3, %xmm3
+; SSE4-NEXT:    movdqa {{.*#+}} xmm4 = [127,127]
+; SSE4-NEXT:    movdqa %xmm4, %xmm0
 ; SSE4-NEXT:    pcmpgtq %xmm2, %xmm0
-; SSE4-NEXT:    blendvpd %xmm0, %xmm2, %xmm3
+; SSE4-NEXT:    blendvpd %xmm0, %xmm2, %xmm4
 ; SSE4-NEXT:    movdqa {{.*#+}} xmm2 = [18446744073709551488,18446744073709551488]
-; SSE4-NEXT:    movapd %xmm3, %xmm0
+; SSE4-NEXT:    movapd %xmm4, %xmm0
 ; SSE4-NEXT:    pcmpgtq %xmm2, %xmm0
-; SSE4-NEXT:    blendvpd %xmm0, %xmm3, %xmm2
-; SSE4-NEXT:    pextrb $0, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB8_2
-; SSE4-NEXT:  # %bb.1: # %cond.store
+; SSE4-NEXT:    blendvpd %xmm0, %xmm4, %xmm2
+; SSE4-NEXT:    pshufb {{.*#+}} xmm2 = xmm2[0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; SSE4-NEXT:    pcmpeqq %xmm1, %xmm3
+; SSE4-NEXT:    movmskpd %xmm3, %eax
+; SSE4-NEXT:    xorl $3, %eax
+; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    jne .LBB8_1
+; SSE4-NEXT:  # %bb.2: # %else
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    jne .LBB8_3
+; SSE4-NEXT:  .LBB8_4: # %else2
+; SSE4-NEXT:    retq
+; SSE4-NEXT:  .LBB8_1: # %cond.store
 ; SSE4-NEXT:    pextrb $0, %xmm2, (%rdi)
-; SSE4-NEXT:  .LBB8_2: # %else
-; SSE4-NEXT:    pextrb $8, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $2, %al
 ; SSE4-NEXT:    je .LBB8_4
-; SSE4-NEXT:  # %bb.3: # %cond.store1
-; SSE4-NEXT:    pextrb $8, %xmm2, 1(%rdi)
-; SSE4-NEXT:  .LBB8_4: # %else2
+; SSE4-NEXT:  .LBB8_3: # %cond.store1
+; SSE4-NEXT:    pextrb $1, %xmm2, 1(%rdi)
 ; SSE4-NEXT:    retq
 ;
 ; AVX-LABEL: truncstore_v2i64_v2i8:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm3 = [127,127]
+; AVX-NEXT:    vpcmpgtq %xmm0, %xmm3, %xmm4
+; AVX-NEXT:    vblendvpd %xmm4, %xmm0, %xmm3, %xmm0
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm3 = [18446744073709551488,18446744073709551488]
+; AVX-NEXT:    vpcmpgtq %xmm3, %xmm0, %xmm4
+; AVX-NEXT:    vblendvpd %xmm4, %xmm0, %xmm3, %xmm0
+; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
 ; AVX-NEXT:    vpcmpeqq %xmm2, %xmm1, %xmm1
-; AVX-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX-NEXT:    vmovdqa {{.*#+}} xmm2 = [127,127]
-; AVX-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm3
-; AVX-NEXT:    vblendvpd %xmm3, %xmm0, %xmm2, %xmm0
-; AVX-NEXT:    vmovdqa {{.*#+}} xmm2 = [18446744073709551488,18446744073709551488]
-; AVX-NEXT:    vpcmpgtq %xmm2, %xmm0, %xmm3
-; AVX-NEXT:    vblendvpd %xmm3, %xmm0, %xmm2, %xmm0
-; AVX-NEXT:    vpextrb $0, %xmm1, %eax
+; AVX-NEXT:    vmovmskpd %xmm1, %eax
+; AVX-NEXT:    xorl $3, %eax
 ; AVX-NEXT:    testb $1, %al
-; AVX-NEXT:    je .LBB8_2
-; AVX-NEXT:  # %bb.1: # %cond.store
+; AVX-NEXT:    jne .LBB8_1
+; AVX-NEXT:  # %bb.2: # %else
+; AVX-NEXT:    testb $2, %al
+; AVX-NEXT:    jne .LBB8_3
+; AVX-NEXT:  .LBB8_4: # %else2
+; AVX-NEXT:    retq
+; AVX-NEXT:  .LBB8_1: # %cond.store
 ; AVX-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX-NEXT:  .LBB8_2: # %else
-; AVX-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX-NEXT:    testb $1, %al
+; AVX-NEXT:    testb $2, %al
 ; AVX-NEXT:    je .LBB8_4
-; AVX-NEXT:  # %bb.3: # %cond.store1
-; AVX-NEXT:    vpextrb $8, %xmm0, 1(%rdi)
-; AVX-NEXT:  .LBB8_4: # %else2
+; AVX-NEXT:  .LBB8_3: # %cond.store1
+; AVX-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
 ; AVX-NEXT:    retq
 ;
 ; AVX512F-LABEL: truncstore_v2i64_v2i8:
@@ -2903,24 +2857,22 @@ define void @truncstore_v2i64_v2i8(<2 x i64> %x, <2 x i8>* %p, <2 x i64> %mask)
 ; AVX512F-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
 ; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; AVX512F-NEXT:    vptestmq %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm2 = [127,127]
-; AVX512F-NEXT:    vpminsq %zmm2, %zmm0, %zmm0
-; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm2 = [18446744073709551488,18446744073709551488]
-; AVX512F-NEXT:    vpmaxsq %zmm2, %zmm0, %zmm0
+; AVX512F-NEXT:    vpmovsqb %zmm0, %xmm0
 ; AVX512F-NEXT:    kmovw %k0, %eax
 ; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB8_2
-; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    jne .LBB8_1
+; AVX512F-NEXT:  # %bb.2: # %else
+; AVX512F-NEXT:    testb $2, %al
+; AVX512F-NEXT:    jne .LBB8_3
+; AVX512F-NEXT:  .LBB8_4: # %else2
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+; AVX512F-NEXT:  .LBB8_1: # %cond.store
 ; AVX512F-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX512F-NEXT:  .LBB8_2: # %else
-; AVX512F-NEXT:    vptestmq %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $2, %al
 ; AVX512F-NEXT:    je .LBB8_4
-; AVX512F-NEXT:  # %bb.3: # %cond.store1
-; AVX512F-NEXT:    vpextrb $8, %xmm0, 1(%rdi)
-; AVX512F-NEXT:  .LBB8_4: # %else2
+; AVX512F-NEXT:  .LBB8_3: # %cond.store1
+; AVX512F-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
@@ -2929,13 +2881,9 @@ define void @truncstore_v2i64_v2i8(<2 x i64> %x, <2 x i8>* %p, <2 x i64> %mask)
 ; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
 ; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; AVX512BW-NEXT:    vptestmq %zmm1, %zmm1, %k0
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm1 = [127,127]
-; AVX512BW-NEXT:    vpminsq %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm1 = [18446744073709551488,18446744073709551488]
-; AVX512BW-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
 ; AVX512BW-NEXT:    kshiftlq $62, %k0, %k0
 ; AVX512BW-NEXT:    kshiftrq $62, %k0, %k1
+; AVX512BW-NEXT:    vpmovsqb %zmm0, %xmm0
 ; AVX512BW-NEXT:    vmovdqu8 %zmm0, (%rdi) {%k1}
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -2961,651 +2909,544 @@ define void @truncstore_v16i32_v16i16(<16 x i32> %x, <16 x i16>* %p, <16 x i32>
 ; SSE2-LABEL: truncstore_v16i32_v16i16:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pxor %xmm8, %xmm8
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm8
 ; SSE2-NEXT:    packssdw %xmm1, %xmm0
-; SSE2-NEXT:    movd %xmm8, %eax
-; SSE2-NEXT:    notl %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB9_2
-; SSE2-NEXT:  # %bb.1: # %cond.store
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    movw %ax, (%rdi)
-; SSE2-NEXT:  .LBB9_2: # %else
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm9
-; SSE2-NEXT:    pxor %xmm9, %xmm8
-; SSE2-NEXT:    pextrw $2, %xmm8, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB9_4
-; SSE2-NEXT:  # %bb.3: # %cond.store1
-; SSE2-NEXT:    pextrw $1, %xmm0, %eax
-; SSE2-NEXT:    movw %ax, 2(%rdi)
-; SSE2-NEXT:  .LBB9_4: # %else2
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm4
-; SSE2-NEXT:    pxor %xmm4, %xmm9
-; SSE2-NEXT:    pextrw $4, %xmm9, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB9_6
-; SSE2-NEXT:  # %bb.5: # %cond.store3
-; SSE2-NEXT:    pextrw $2, %xmm0, %eax
-; SSE2-NEXT:    movw %ax, 4(%rdi)
-; SSE2-NEXT:  .LBB9_6: # %else4
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm7
 ; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
+; SSE2-NEXT:    pxor %xmm1, %xmm7
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm6
+; SSE2-NEXT:    pxor %xmm1, %xmm6
+; SSE2-NEXT:    packssdw %xmm7, %xmm6
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm5
+; SSE2-NEXT:    pxor %xmm1, %xmm5
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm4
 ; SSE2-NEXT:    pxor %xmm1, %xmm4
-; SSE2-NEXT:    pextrw $6, %xmm4, %eax
+; SSE2-NEXT:    packssdw %xmm5, %xmm4
+; SSE2-NEXT:    packsswb %xmm6, %xmm4
+; SSE2-NEXT:    pmovmskb %xmm4, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB9_8
-; SSE2-NEXT:  # %bb.7: # %cond.store5
-; SSE2-NEXT:    pextrw $3, %xmm0, %eax
-; SSE2-NEXT:    movw %ax, 6(%rdi)
+; SSE2-NEXT:    jne .LBB9_1
+; SSE2-NEXT:  # %bb.2: # %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne .LBB9_3
+; SSE2-NEXT:  .LBB9_4: # %else2
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne .LBB9_5
+; SSE2-NEXT:  .LBB9_6: # %else4
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne .LBB9_7
 ; SSE2-NEXT:  .LBB9_8: # %else6
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm1
-; SSE2-NEXT:    movd %xmm1, %eax
-; SSE2-NEXT:    notl %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB9_10
-; SSE2-NEXT:  # %bb.9: # %cond.store7
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    movw %ax, 8(%rdi)
+; SSE2-NEXT:    testb $16, %al
+; SSE2-NEXT:    jne .LBB9_9
 ; SSE2-NEXT:  .LBB9_10: # %else8
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm4
-; SSE2-NEXT:    pxor %xmm4, %xmm1
-; SSE2-NEXT:    pextrw $2, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB9_12
-; SSE2-NEXT:  # %bb.11: # %cond.store9
-; SSE2-NEXT:    pextrw $5, %xmm0, %eax
-; SSE2-NEXT:    movw %ax, 10(%rdi)
+; SSE2-NEXT:    testb $32, %al
+; SSE2-NEXT:    jne .LBB9_11
 ; SSE2-NEXT:  .LBB9_12: # %else10
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm5
-; SSE2-NEXT:    pxor %xmm5, %xmm4
-; SSE2-NEXT:    pextrw $4, %xmm4, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB9_14
-; SSE2-NEXT:  # %bb.13: # %cond.store11
-; SSE2-NEXT:    pextrw $6, %xmm0, %eax
-; SSE2-NEXT:    movw %ax, 12(%rdi)
+; SSE2-NEXT:    testb $64, %al
+; SSE2-NEXT:    jne .LBB9_13
 ; SSE2-NEXT:  .LBB9_14: # %else12
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pxor %xmm1, %xmm5
-; SSE2-NEXT:    pextrw $6, %xmm5, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $-128, %al
 ; SSE2-NEXT:    je .LBB9_16
-; SSE2-NEXT:  # %bb.15: # %cond.store13
-; SSE2-NEXT:    pextrw $7, %xmm0, %eax
-; SSE2-NEXT:    movw %ax, 14(%rdi)
+; SSE2-NEXT:  .LBB9_15: # %cond.store13
+; SSE2-NEXT:    pextrw $7, %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, 14(%rdi)
 ; SSE2-NEXT:  .LBB9_16: # %else14
 ; SSE2-NEXT:    packssdw %xmm3, %xmm2
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqd %xmm6, %xmm0
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    notl %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB9_18
-; SSE2-NEXT:  # %bb.17: # %cond.store15
-; SSE2-NEXT:    movd %xmm2, %eax
-; SSE2-NEXT:    movw %ax, 16(%rdi)
-; SSE2-NEXT:  .LBB9_18: # %else16
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pxor %xmm1, %xmm0
-; SSE2-NEXT:    pextrw $2, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB9_20
-; SSE2-NEXT:  # %bb.19: # %cond.store17
-; SSE2-NEXT:    pextrw $1, %xmm2, %eax
-; SSE2-NEXT:    movw %ax, 18(%rdi)
+; SSE2-NEXT:    testl $256, %eax # imm = 0x100
+; SSE2-NEXT:    jne .LBB9_17
+; SSE2-NEXT:  # %bb.18: # %else16
+; SSE2-NEXT:    testl $512, %eax # imm = 0x200
+; SSE2-NEXT:    jne .LBB9_19
 ; SSE2-NEXT:  .LBB9_20: # %else18
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm6
-; SSE2-NEXT:    pxor %xmm6, %xmm1
-; SSE2-NEXT:    pextrw $4, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB9_22
-; SSE2-NEXT:  # %bb.21: # %cond.store19
-; SSE2-NEXT:    pextrw $2, %xmm2, %eax
-; SSE2-NEXT:    movw %ax, 20(%rdi)
+; SSE2-NEXT:    testl $1024, %eax # imm = 0x400
+; SSE2-NEXT:    jne .LBB9_21
 ; SSE2-NEXT:  .LBB9_22: # %else20
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE2-NEXT:    pxor %xmm0, %xmm6
-; SSE2-NEXT:    pextrw $6, %xmm6, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB9_24
-; SSE2-NEXT:  # %bb.23: # %cond.store21
-; SSE2-NEXT:    pextrw $3, %xmm2, %eax
-; SSE2-NEXT:    movw %ax, 22(%rdi)
+; SSE2-NEXT:    testl $2048, %eax # imm = 0x800
+; SSE2-NEXT:    jne .LBB9_23
 ; SSE2-NEXT:  .LBB9_24: # %else22
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqd %xmm7, %xmm0
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    notl %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB9_26
-; SSE2-NEXT:  # %bb.25: # %cond.store23
-; SSE2-NEXT:    pextrw $4, %xmm2, %eax
-; SSE2-NEXT:    movw %ax, 24(%rdi)
+; SSE2-NEXT:    testl $4096, %eax # imm = 0x1000
+; SSE2-NEXT:    jne .LBB9_25
 ; SSE2-NEXT:  .LBB9_26: # %else24
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pxor %xmm1, %xmm0
-; SSE2-NEXT:    pextrw $2, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB9_28
-; SSE2-NEXT:  # %bb.27: # %cond.store25
-; SSE2-NEXT:    pextrw $5, %xmm2, %eax
-; SSE2-NEXT:    movw %ax, 26(%rdi)
+; SSE2-NEXT:    testl $8192, %eax # imm = 0x2000
+; SSE2-NEXT:    jne .LBB9_27
 ; SSE2-NEXT:  .LBB9_28: # %else26
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm7
-; SSE2-NEXT:    pxor %xmm7, %xmm1
-; SSE2-NEXT:    pextrw $4, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB9_30
-; SSE2-NEXT:  # %bb.29: # %cond.store27
-; SSE2-NEXT:    pextrw $6, %xmm2, %eax
-; SSE2-NEXT:    movw %ax, 28(%rdi)
+; SSE2-NEXT:    testl $16384, %eax # imm = 0x4000
+; SSE2-NEXT:    jne .LBB9_29
 ; SSE2-NEXT:  .LBB9_30: # %else28
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE2-NEXT:    pxor %xmm0, %xmm7
-; SSE2-NEXT:    pextrw $6, %xmm7, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $32768, %eax # imm = 0x8000
+; SSE2-NEXT:    jne .LBB9_31
+; SSE2-NEXT:  .LBB9_32: # %else30
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  .LBB9_1: # %cond.store
+; SSE2-NEXT:    movd %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, (%rdi)
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    je .LBB9_4
+; SSE2-NEXT:  .LBB9_3: # %cond.store1
+; SSE2-NEXT:    pextrw $1, %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, 2(%rdi)
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    je .LBB9_6
+; SSE2-NEXT:  .LBB9_5: # %cond.store3
+; SSE2-NEXT:    pextrw $2, %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, 4(%rdi)
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    je .LBB9_8
+; SSE2-NEXT:  .LBB9_7: # %cond.store5
+; SSE2-NEXT:    pextrw $3, %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, 6(%rdi)
+; SSE2-NEXT:    testb $16, %al
+; SSE2-NEXT:    je .LBB9_10
+; SSE2-NEXT:  .LBB9_9: # %cond.store7
+; SSE2-NEXT:    pextrw $4, %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, 8(%rdi)
+; SSE2-NEXT:    testb $32, %al
+; SSE2-NEXT:    je .LBB9_12
+; SSE2-NEXT:  .LBB9_11: # %cond.store9
+; SSE2-NEXT:    pextrw $5, %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, 10(%rdi)
+; SSE2-NEXT:    testb $64, %al
+; SSE2-NEXT:    je .LBB9_14
+; SSE2-NEXT:  .LBB9_13: # %cond.store11
+; SSE2-NEXT:    pextrw $6, %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, 12(%rdi)
+; SSE2-NEXT:    testb $-128, %al
+; SSE2-NEXT:    jne .LBB9_15
+; SSE2-NEXT:    jmp .LBB9_16
+; SSE2-NEXT:  .LBB9_17: # %cond.store15
+; SSE2-NEXT:    movd %xmm2, %ecx
+; SSE2-NEXT:    movw %cx, 16(%rdi)
+; SSE2-NEXT:    testl $512, %eax # imm = 0x200
+; SSE2-NEXT:    je .LBB9_20
+; SSE2-NEXT:  .LBB9_19: # %cond.store17
+; SSE2-NEXT:    pextrw $1, %xmm2, %ecx
+; SSE2-NEXT:    movw %cx, 18(%rdi)
+; SSE2-NEXT:    testl $1024, %eax # imm = 0x400
+; SSE2-NEXT:    je .LBB9_22
+; SSE2-NEXT:  .LBB9_21: # %cond.store19
+; SSE2-NEXT:    pextrw $2, %xmm2, %ecx
+; SSE2-NEXT:    movw %cx, 20(%rdi)
+; SSE2-NEXT:    testl $2048, %eax # imm = 0x800
+; SSE2-NEXT:    je .LBB9_24
+; SSE2-NEXT:  .LBB9_23: # %cond.store21
+; SSE2-NEXT:    pextrw $3, %xmm2, %ecx
+; SSE2-NEXT:    movw %cx, 22(%rdi)
+; SSE2-NEXT:    testl $4096, %eax # imm = 0x1000
+; SSE2-NEXT:    je .LBB9_26
+; SSE2-NEXT:  .LBB9_25: # %cond.store23
+; SSE2-NEXT:    pextrw $4, %xmm2, %ecx
+; SSE2-NEXT:    movw %cx, 24(%rdi)
+; SSE2-NEXT:    testl $8192, %eax # imm = 0x2000
+; SSE2-NEXT:    je .LBB9_28
+; SSE2-NEXT:  .LBB9_27: # %cond.store25
+; SSE2-NEXT:    pextrw $5, %xmm2, %ecx
+; SSE2-NEXT:    movw %cx, 26(%rdi)
+; SSE2-NEXT:    testl $16384, %eax # imm = 0x4000
+; SSE2-NEXT:    je .LBB9_30
+; SSE2-NEXT:  .LBB9_29: # %cond.store27
+; SSE2-NEXT:    pextrw $6, %xmm2, %ecx
+; SSE2-NEXT:    movw %cx, 28(%rdi)
+; SSE2-NEXT:    testl $32768, %eax # imm = 0x8000
 ; SSE2-NEXT:    je .LBB9_32
-; SSE2-NEXT:  # %bb.31: # %cond.store29
+; SSE2-NEXT:  .LBB9_31: # %cond.store29
 ; SSE2-NEXT:    pextrw $7, %xmm2, %eax
 ; SSE2-NEXT:    movw %ax, 30(%rdi)
-; SSE2-NEXT:  .LBB9_32: # %else30
 ; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: truncstore_v16i32_v16i16:
 ; SSE4:       # %bb.0:
-; SSE4-NEXT:    pxor %xmm9, %xmm9
-; SSE4-NEXT:    pcmpeqd %xmm4, %xmm9
-; SSE4-NEXT:    pcmpeqd %xmm8, %xmm8
-; SSE4-NEXT:    pxor %xmm9, %xmm8
+; SSE4-NEXT:    pxor %xmm8, %xmm8
 ; SSE4-NEXT:    packssdw %xmm1, %xmm0
-; SSE4-NEXT:    pextrb $0, %xmm8, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB9_2
-; SSE4-NEXT:  # %bb.1: # %cond.store
+; SSE4-NEXT:    pcmpeqd %xmm8, %xmm7
+; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
+; SSE4-NEXT:    pxor %xmm1, %xmm7
+; SSE4-NEXT:    pcmpeqd %xmm8, %xmm6
+; SSE4-NEXT:    pxor %xmm1, %xmm6
+; SSE4-NEXT:    packssdw %xmm7, %xmm6
+; SSE4-NEXT:    pcmpeqd %xmm8, %xmm5
+; SSE4-NEXT:    pxor %xmm1, %xmm5
+; SSE4-NEXT:    pcmpeqd %xmm8, %xmm4
+; SSE4-NEXT:    pxor %xmm1, %xmm4
+; SSE4-NEXT:    packssdw %xmm5, %xmm4
+; SSE4-NEXT:    packsswb %xmm6, %xmm4
+; SSE4-NEXT:    pmovmskb %xmm4, %eax
+; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    jne .LBB9_1
+; SSE4-NEXT:  # %bb.2: # %else
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    jne .LBB9_3
+; SSE4-NEXT:  .LBB9_4: # %else2
+; SSE4-NEXT:    testb $4, %al
+; SSE4-NEXT:    jne .LBB9_5
+; SSE4-NEXT:  .LBB9_6: # %else4
+; SSE4-NEXT:    testb $8, %al
+; SSE4-NEXT:    jne .LBB9_7
+; SSE4-NEXT:  .LBB9_8: # %else6
+; SSE4-NEXT:    testb $16, %al
+; SSE4-NEXT:    jne .LBB9_9
+; SSE4-NEXT:  .LBB9_10: # %else8
+; SSE4-NEXT:    testb $32, %al
+; SSE4-NEXT:    jne .LBB9_11
+; SSE4-NEXT:  .LBB9_12: # %else10
+; SSE4-NEXT:    testb $64, %al
+; SSE4-NEXT:    jne .LBB9_13
+; SSE4-NEXT:  .LBB9_14: # %else12
+; SSE4-NEXT:    testb $-128, %al
+; SSE4-NEXT:    je .LBB9_16
+; SSE4-NEXT:  .LBB9_15: # %cond.store13
+; SSE4-NEXT:    pextrw $7, %xmm0, 14(%rdi)
+; SSE4-NEXT:  .LBB9_16: # %else14
+; SSE4-NEXT:    packssdw %xmm3, %xmm2
+; SSE4-NEXT:    testl $256, %eax # imm = 0x100
+; SSE4-NEXT:    jne .LBB9_17
+; SSE4-NEXT:  # %bb.18: # %else16
+; SSE4-NEXT:    testl $512, %eax # imm = 0x200
+; SSE4-NEXT:    jne .LBB9_19
+; SSE4-NEXT:  .LBB9_20: # %else18
+; SSE4-NEXT:    testl $1024, %eax # imm = 0x400
+; SSE4-NEXT:    jne .LBB9_21
+; SSE4-NEXT:  .LBB9_22: # %else20
+; SSE4-NEXT:    testl $2048, %eax # imm = 0x800
+; SSE4-NEXT:    jne .LBB9_23
+; SSE4-NEXT:  .LBB9_24: # %else22
+; SSE4-NEXT:    testl $4096, %eax # imm = 0x1000
+; SSE4-NEXT:    jne .LBB9_25
+; SSE4-NEXT:  .LBB9_26: # %else24
+; SSE4-NEXT:    testl $8192, %eax # imm = 0x2000
+; SSE4-NEXT:    jne .LBB9_27
+; SSE4-NEXT:  .LBB9_28: # %else26
+; SSE4-NEXT:    testl $16384, %eax # imm = 0x4000
+; SSE4-NEXT:    jne .LBB9_29
+; SSE4-NEXT:  .LBB9_30: # %else28
+; SSE4-NEXT:    testl $32768, %eax # imm = 0x8000
+; SSE4-NEXT:    jne .LBB9_31
+; SSE4-NEXT:  .LBB9_32: # %else30
+; SSE4-NEXT:    retq
+; SSE4-NEXT:  .LBB9_1: # %cond.store
 ; SSE4-NEXT:    pextrw $0, %xmm0, (%rdi)
-; SSE4-NEXT:  .LBB9_2: # %else
-; SSE4-NEXT:    pextrb $4, %xmm8, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $2, %al
 ; SSE4-NEXT:    je .LBB9_4
-; SSE4-NEXT:  # %bb.3: # %cond.store1
+; SSE4-NEXT:  .LBB9_3: # %cond.store1
 ; SSE4-NEXT:    pextrw $1, %xmm0, 2(%rdi)
-; SSE4-NEXT:  .LBB9_4: # %else2
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm4
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE4-NEXT:    pxor %xmm4, %xmm1
-; SSE4-NEXT:    pextrb $8, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $4, %al
 ; SSE4-NEXT:    je .LBB9_6
-; SSE4-NEXT:  # %bb.5: # %cond.store3
+; SSE4-NEXT:  .LBB9_5: # %cond.store3
 ; SSE4-NEXT:    pextrw $2, %xmm0, 4(%rdi)
-; SSE4-NEXT:  .LBB9_6: # %else4
-; SSE4-NEXT:    pextrb $12, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $8, %al
 ; SSE4-NEXT:    je .LBB9_8
-; SSE4-NEXT:  # %bb.7: # %cond.store5
+; SSE4-NEXT:  .LBB9_7: # %cond.store5
 ; SSE4-NEXT:    pextrw $3, %xmm0, 6(%rdi)
-; SSE4-NEXT:  .LBB9_8: # %else6
-; SSE4-NEXT:    pxor %xmm4, %xmm4
-; SSE4-NEXT:    pcmpeqd %xmm5, %xmm4
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE4-NEXT:    pxor %xmm4, %xmm1
-; SSE4-NEXT:    pextrb $0, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $16, %al
 ; SSE4-NEXT:    je .LBB9_10
-; SSE4-NEXT:  # %bb.9: # %cond.store7
+; SSE4-NEXT:  .LBB9_9: # %cond.store7
 ; SSE4-NEXT:    pextrw $4, %xmm0, 8(%rdi)
-; SSE4-NEXT:  .LBB9_10: # %else8
-; SSE4-NEXT:    pextrb $4, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $32, %al
 ; SSE4-NEXT:    je .LBB9_12
-; SSE4-NEXT:  # %bb.11: # %cond.store9
+; SSE4-NEXT:  .LBB9_11: # %cond.store9
 ; SSE4-NEXT:    pextrw $5, %xmm0, 10(%rdi)
-; SSE4-NEXT:  .LBB9_12: # %else10
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm5
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE4-NEXT:    pxor %xmm5, %xmm1
-; SSE4-NEXT:    pextrb $8, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $64, %al
 ; SSE4-NEXT:    je .LBB9_14
-; SSE4-NEXT:  # %bb.13: # %cond.store11
+; SSE4-NEXT:  .LBB9_13: # %cond.store11
 ; SSE4-NEXT:    pextrw $6, %xmm0, 12(%rdi)
-; SSE4-NEXT:  .LBB9_14: # %else12
-; SSE4-NEXT:    pextrb $12, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB9_16
-; SSE4-NEXT:  # %bb.15: # %cond.store13
-; SSE4-NEXT:    pextrw $7, %xmm0, 14(%rdi)
-; SSE4-NEXT:  .LBB9_16: # %else14
-; SSE4-NEXT:    packssdw %xmm3, %xmm2
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm6, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE4-NEXT:    pxor %xmm1, %xmm0
-; SSE4-NEXT:    pextrb $0, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB9_18
-; SSE4-NEXT:  # %bb.17: # %cond.store15
+; SSE4-NEXT:    testb $-128, %al
+; SSE4-NEXT:    jne .LBB9_15
+; SSE4-NEXT:    jmp .LBB9_16
+; SSE4-NEXT:  .LBB9_17: # %cond.store15
 ; SSE4-NEXT:    pextrw $0, %xmm2, 16(%rdi)
-; SSE4-NEXT:  .LBB9_18: # %else16
-; SSE4-NEXT:    pextrb $4, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $512, %eax # imm = 0x200
 ; SSE4-NEXT:    je .LBB9_20
-; SSE4-NEXT:  # %bb.19: # %cond.store17
+; SSE4-NEXT:  .LBB9_19: # %cond.store17
 ; SSE4-NEXT:    pextrw $1, %xmm2, 18(%rdi)
-; SSE4-NEXT:  .LBB9_20: # %else18
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqd %xmm0, %xmm6
-; SSE4-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE4-NEXT:    pxor %xmm6, %xmm0
-; SSE4-NEXT:    pextrb $8, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $1024, %eax # imm = 0x400
 ; SSE4-NEXT:    je .LBB9_22
-; SSE4-NEXT:  # %bb.21: # %cond.store19
+; SSE4-NEXT:  .LBB9_21: # %cond.store19
 ; SSE4-NEXT:    pextrw $2, %xmm2, 20(%rdi)
-; SSE4-NEXT:  .LBB9_22: # %else20
-; SSE4-NEXT:    pextrb $12, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $2048, %eax # imm = 0x800
 ; SSE4-NEXT:    je .LBB9_24
-; SSE4-NEXT:  # %bb.23: # %cond.store21
+; SSE4-NEXT:  .LBB9_23: # %cond.store21
 ; SSE4-NEXT:    pextrw $3, %xmm2, 22(%rdi)
-; SSE4-NEXT:  .LBB9_24: # %else22
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm7, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE4-NEXT:    pxor %xmm1, %xmm0
-; SSE4-NEXT:    pextrb $0, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $4096, %eax # imm = 0x1000
 ; SSE4-NEXT:    je .LBB9_26
-; SSE4-NEXT:  # %bb.25: # %cond.store23
+; SSE4-NEXT:  .LBB9_25: # %cond.store23
 ; SSE4-NEXT:    pextrw $4, %xmm2, 24(%rdi)
-; SSE4-NEXT:  .LBB9_26: # %else24
-; SSE4-NEXT:    pextrb $4, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $8192, %eax # imm = 0x2000
 ; SSE4-NEXT:    je .LBB9_28
-; SSE4-NEXT:  # %bb.27: # %cond.store25
+; SSE4-NEXT:  .LBB9_27: # %cond.store25
 ; SSE4-NEXT:    pextrw $5, %xmm2, 26(%rdi)
-; SSE4-NEXT:  .LBB9_28: # %else26
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqd %xmm0, %xmm7
-; SSE4-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE4-NEXT:    pxor %xmm7, %xmm0
-; SSE4-NEXT:    pextrb $8, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $16384, %eax # imm = 0x4000
 ; SSE4-NEXT:    je .LBB9_30
-; SSE4-NEXT:  # %bb.29: # %cond.store27
+; SSE4-NEXT:  .LBB9_29: # %cond.store27
 ; SSE4-NEXT:    pextrw $6, %xmm2, 28(%rdi)
-; SSE4-NEXT:  .LBB9_30: # %else28
-; SSE4-NEXT:    pextrb $12, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $32768, %eax # imm = 0x8000
 ; SSE4-NEXT:    je .LBB9_32
-; SSE4-NEXT:  # %bb.31: # %cond.store29
+; SSE4-NEXT:  .LBB9_31: # %cond.store29
 ; SSE4-NEXT:    pextrw $7, %xmm2, 30(%rdi)
-; SSE4-NEXT:  .LBB9_32: # %else30
 ; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: truncstore_v16i32_v16i16:
 ; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm4
+; AVX1-NEXT:    vpackssdw %xmm4, %xmm1, %xmm1
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
+; AVX1-NEXT:    vpackssdw %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm1
 ; AVX1-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm2, %xmm6
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm1, %xmm1
 ; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
-; AVX1-NEXT:    vpxor %xmm5, %xmm6, %xmm6
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm6, %xmm6
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm6, %xmm6
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm7
-; AVX1-NEXT:    vpackssdw %xmm7, %xmm1, %xmm1
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm7
-; AVX1-NEXT:    vpackssdw %xmm7, %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX1-NEXT:    vpextrb $0, %xmm6, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je .LBB9_2
-; AVX1-NEXT:  # %bb.1: # %cond.store
-; AVX1-NEXT:    vpextrw $0, %xmm0, (%rdi)
-; AVX1-NEXT:  .LBB9_2: # %else
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm2, %xmm1
 ; AVX1-NEXT:    vpxor %xmm5, %xmm1, %xmm1
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm1, %xmm1
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $1, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je .LBB9_4
-; AVX1-NEXT:  # %bb.3: # %cond.store1
-; AVX1-NEXT:    vpextrw $1, %xmm0, 2(%rdi)
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpxor %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vpackssdw %xmm1, %xmm3, %xmm1
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm3
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpxor %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm2, %xmm2
+; AVX1-NEXT:    vpxor %xmm5, %xmm2, %xmm2
+; AVX1-NEXT:    vpackssdw %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpacksswb %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vpmovmskb %xmm1, %eax
+; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    jne .LBB9_1
+; AVX1-NEXT:  # %bb.2: # %else
+; AVX1-NEXT:    testb $2, %al
+; AVX1-NEXT:    jne .LBB9_3
 ; AVX1-NEXT:  .LBB9_4: # %else2
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm2, %xmm5
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm5, %xmm5
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm5, %xmm5
-; AVX1-NEXT:    vpextrb $2, %xmm5, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $4, %al
+; AVX1-NEXT:    jne .LBB9_5
+; AVX1-NEXT:  .LBB9_6: # %else4
+; AVX1-NEXT:    testb $8, %al
+; AVX1-NEXT:    jne .LBB9_7
+; AVX1-NEXT:  .LBB9_8: # %else6
+; AVX1-NEXT:    testb $16, %al
+; AVX1-NEXT:    jne .LBB9_9
+; AVX1-NEXT:  .LBB9_10: # %else8
+; AVX1-NEXT:    testb $32, %al
+; AVX1-NEXT:    jne .LBB9_11
+; AVX1-NEXT:  .LBB9_12: # %else10
+; AVX1-NEXT:    testb $64, %al
+; AVX1-NEXT:    jne .LBB9_13
+; AVX1-NEXT:  .LBB9_14: # %else12
+; AVX1-NEXT:    testb $-128, %al
+; AVX1-NEXT:    je .LBB9_16
+; AVX1-NEXT:  .LBB9_15: # %cond.store13
+; AVX1-NEXT:    vpextrw $7, %xmm0, 14(%rdi)
+; AVX1-NEXT:  .LBB9_16: # %else14
+; AVX1-NEXT:    testl $256, %eax # imm = 0x100
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    jne .LBB9_17
+; AVX1-NEXT:  # %bb.18: # %else16
+; AVX1-NEXT:    testl $512, %eax # imm = 0x200
+; AVX1-NEXT:    jne .LBB9_19
+; AVX1-NEXT:  .LBB9_20: # %else18
+; AVX1-NEXT:    testl $1024, %eax # imm = 0x400
+; AVX1-NEXT:    jne .LBB9_21
+; AVX1-NEXT:  .LBB9_22: # %else20
+; AVX1-NEXT:    testl $2048, %eax # imm = 0x800
+; AVX1-NEXT:    jne .LBB9_23
+; AVX1-NEXT:  .LBB9_24: # %else22
+; AVX1-NEXT:    testl $4096, %eax # imm = 0x1000
+; AVX1-NEXT:    jne .LBB9_25
+; AVX1-NEXT:  .LBB9_26: # %else24
+; AVX1-NEXT:    testl $8192, %eax # imm = 0x2000
+; AVX1-NEXT:    jne .LBB9_27
+; AVX1-NEXT:  .LBB9_28: # %else26
+; AVX1-NEXT:    testl $16384, %eax # imm = 0x4000
+; AVX1-NEXT:    jne .LBB9_29
+; AVX1-NEXT:  .LBB9_30: # %else28
+; AVX1-NEXT:    testl $32768, %eax # imm = 0x8000
+; AVX1-NEXT:    jne .LBB9_31
+; AVX1-NEXT:  .LBB9_32: # %else30
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+; AVX1-NEXT:  .LBB9_1: # %cond.store
+; AVX1-NEXT:    vpextrw $0, %xmm0, (%rdi)
+; AVX1-NEXT:    testb $2, %al
+; AVX1-NEXT:    je .LBB9_4
+; AVX1-NEXT:  .LBB9_3: # %cond.store1
+; AVX1-NEXT:    vpextrw $1, %xmm0, 2(%rdi)
+; AVX1-NEXT:    testb $4, %al
 ; AVX1-NEXT:    je .LBB9_6
-; AVX1-NEXT:  # %bb.5: # %cond.store3
+; AVX1-NEXT:  .LBB9_5: # %cond.store3
 ; AVX1-NEXT:    vpextrw $2, %xmm0, 4(%rdi)
-; AVX1-NEXT:  .LBB9_6: # %else4
-; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm4, %xmm1, %xmm1
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm1, %xmm1
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $3, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $8, %al
 ; AVX1-NEXT:    je .LBB9_8
-; AVX1-NEXT:  # %bb.7: # %cond.store5
+; AVX1-NEXT:  .LBB9_7: # %cond.store5
 ; AVX1-NEXT:    vpextrw $3, %xmm0, 6(%rdi)
-; AVX1-NEXT:  .LBB9_8: # %else6
-; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm2
-; AVX1-NEXT:    vpackssdw %xmm2, %xmm0, %xmm4
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm4, %xmm4
-; AVX1-NEXT:    vpextrb $4, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $16, %al
 ; AVX1-NEXT:    je .LBB9_10
-; AVX1-NEXT:  # %bb.9: # %cond.store7
+; AVX1-NEXT:  .LBB9_9: # %cond.store7
 ; AVX1-NEXT:    vpextrw $4, %xmm0, 8(%rdi)
-; AVX1-NEXT:  .LBB9_10: # %else8
-; AVX1-NEXT:    vpackssdw %xmm2, %xmm0, %xmm2
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $5, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $32, %al
 ; AVX1-NEXT:    je .LBB9_12
-; AVX1-NEXT:  # %bb.11: # %cond.store9
+; AVX1-NEXT:  .LBB9_11: # %cond.store9
 ; AVX1-NEXT:    vpextrw $5, %xmm0, 10(%rdi)
-; AVX1-NEXT:  .LBB9_12: # %else10
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpackssdw %xmm1, %xmm0, %xmm2
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $6, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $64, %al
 ; AVX1-NEXT:    je .LBB9_14
-; AVX1-NEXT:  # %bb.13: # %cond.store11
+; AVX1-NEXT:  .LBB9_13: # %cond.store11
 ; AVX1-NEXT:    vpextrw $6, %xmm0, 12(%rdi)
-; AVX1-NEXT:  .LBB9_14: # %else12
-; AVX1-NEXT:    vpackssdw %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $7, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je .LBB9_16
-; AVX1-NEXT:  # %bb.15: # %cond.store13
-; AVX1-NEXT:    vpextrw $7, %xmm0, 14(%rdi)
-; AVX1-NEXT:  .LBB9_16: # %else14
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm3, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpxor %xmm2, %xmm4, %xmm4
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm4, %xmm4
-; AVX1-NEXT:    vpacksswb %xmm4, %xmm0, %xmm4
-; AVX1-NEXT:    vpextrb $8, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT:    je .LBB9_18
-; AVX1-NEXT:  # %bb.17: # %cond.store15
+; AVX1-NEXT:    testb $-128, %al
+; AVX1-NEXT:    jne .LBB9_15
+; AVX1-NEXT:    jmp .LBB9_16
+; AVX1-NEXT:  .LBB9_17: # %cond.store15
 ; AVX1-NEXT:    vpextrw $0, %xmm0, 16(%rdi)
-; AVX1-NEXT:  .LBB9_18: # %else16
-; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm3, %xmm1
-; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm1, %xmm1
-; AVX1-NEXT:    vpacksswb %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpextrb $9, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $512, %eax # imm = 0x200
 ; AVX1-NEXT:    je .LBB9_20
-; AVX1-NEXT:  # %bb.19: # %cond.store17
+; AVX1-NEXT:  .LBB9_19: # %cond.store17
 ; AVX1-NEXT:    vpextrw $1, %xmm0, 18(%rdi)
-; AVX1-NEXT:  .LBB9_20: # %else18
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm3, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpxor %xmm2, %xmm4, %xmm4
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm4, %xmm4
-; AVX1-NEXT:    vpacksswb %xmm4, %xmm0, %xmm4
-; AVX1-NEXT:    vpextrb $10, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $1024, %eax # imm = 0x400
 ; AVX1-NEXT:    je .LBB9_22
-; AVX1-NEXT:  # %bb.21: # %cond.store19
+; AVX1-NEXT:  .LBB9_21: # %cond.store19
 ; AVX1-NEXT:    vpextrw $2, %xmm0, 20(%rdi)
-; AVX1-NEXT:  .LBB9_22: # %else20
-; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm3, %xmm1
-; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm1, %xmm1
-; AVX1-NEXT:    vpacksswb %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpextrb $11, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $2048, %eax # imm = 0x800
 ; AVX1-NEXT:    je .LBB9_24
-; AVX1-NEXT:  # %bb.23: # %cond.store21
+; AVX1-NEXT:  .LBB9_23: # %cond.store21
 ; AVX1-NEXT:    vpextrw $3, %xmm0, 22(%rdi)
-; AVX1-NEXT:  .LBB9_24: # %else22
-; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm1
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpackssdw %xmm2, %xmm0, %xmm3
-; AVX1-NEXT:    vpacksswb %xmm3, %xmm0, %xmm3
-; AVX1-NEXT:    vpextrb $12, %xmm3, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $4096, %eax # imm = 0x1000
 ; AVX1-NEXT:    je .LBB9_26
-; AVX1-NEXT:  # %bb.25: # %cond.store23
+; AVX1-NEXT:  .LBB9_25: # %cond.store23
 ; AVX1-NEXT:    vpextrw $4, %xmm0, 24(%rdi)
-; AVX1-NEXT:  .LBB9_26: # %else24
-; AVX1-NEXT:    vpackssdw %xmm2, %xmm0, %xmm2
-; AVX1-NEXT:    vpacksswb %xmm2, %xmm0, %xmm2
-; AVX1-NEXT:    vpextrb $13, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $8192, %eax # imm = 0x2000
 ; AVX1-NEXT:    je .LBB9_28
-; AVX1-NEXT:  # %bb.27: # %cond.store25
+; AVX1-NEXT:  .LBB9_27: # %cond.store25
 ; AVX1-NEXT:    vpextrw $5, %xmm0, 26(%rdi)
-; AVX1-NEXT:  .LBB9_28: # %else26
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpackssdw %xmm1, %xmm0, %xmm2
-; AVX1-NEXT:    vpacksswb %xmm2, %xmm0, %xmm2
-; AVX1-NEXT:    vpextrb $14, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $16384, %eax # imm = 0x4000
 ; AVX1-NEXT:    je .LBB9_30
-; AVX1-NEXT:  # %bb.29: # %cond.store27
+; AVX1-NEXT:  .LBB9_29: # %cond.store27
 ; AVX1-NEXT:    vpextrw $6, %xmm0, 28(%rdi)
-; AVX1-NEXT:  .LBB9_30: # %else28
-; AVX1-NEXT:    vpackssdw %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpacksswb %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpextrb $15, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $32768, %eax # imm = 0x8000
 ; AVX1-NEXT:    je .LBB9_32
-; AVX1-NEXT:  # %bb.31: # %cond.store29
+; AVX1-NEXT:  .LBB9_31: # %cond.store29
 ; AVX1-NEXT:    vpextrw $7, %xmm0, 30(%rdi)
-; AVX1-NEXT:  .LBB9_32: # %else30
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: truncstore_v16i32_v16i16:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm2, %xmm6
-; AVX2-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
-; AVX2-NEXT:    vpxor %xmm5, %xmm6, %xmm6
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm6, %xmm6
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm6, %xmm6
 ; AVX2-NEXT:    vpackssdw %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX2-NEXT:    vpextrb $0, %xmm6, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je .LBB9_2
-; AVX2-NEXT:  # %bb.1: # %cond.store
+; AVX2-NEXT:    vpcmpeqd %ymm4, %ymm3, %ymm1
+; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm3, %ymm3
+; AVX2-NEXT:    vpxor %ymm3, %ymm1, %ymm1
+; AVX2-NEXT:    vpcmpeqd %ymm4, %ymm2, %ymm2
+; AVX2-NEXT:    vpxor %ymm3, %ymm2, %ymm2
+; AVX2-NEXT:    vpackssdw %ymm1, %ymm2, %ymm1
+; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
+; AVX2-NEXT:    vpacksswb %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpmovmskb %xmm1, %eax
+; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    jne .LBB9_1
+; AVX2-NEXT:  # %bb.2: # %else
+; AVX2-NEXT:    testb $2, %al
+; AVX2-NEXT:    jne .LBB9_3
+; AVX2-NEXT:  .LBB9_4: # %else2
+; AVX2-NEXT:    testb $4, %al
+; AVX2-NEXT:    jne .LBB9_5
+; AVX2-NEXT:  .LBB9_6: # %else4
+; AVX2-NEXT:    testb $8, %al
+; AVX2-NEXT:    jne .LBB9_7
+; AVX2-NEXT:  .LBB9_8: # %else6
+; AVX2-NEXT:    testb $16, %al
+; AVX2-NEXT:    jne .LBB9_9
+; AVX2-NEXT:  .LBB9_10: # %else8
+; AVX2-NEXT:    testb $32, %al
+; AVX2-NEXT:    jne .LBB9_11
+; AVX2-NEXT:  .LBB9_12: # %else10
+; AVX2-NEXT:    testb $64, %al
+; AVX2-NEXT:    jne .LBB9_13
+; AVX2-NEXT:  .LBB9_14: # %else12
+; AVX2-NEXT:    testb $-128, %al
+; AVX2-NEXT:    je .LBB9_16
+; AVX2-NEXT:  .LBB9_15: # %cond.store13
+; AVX2-NEXT:    vpextrw $7, %xmm0, 14(%rdi)
+; AVX2-NEXT:  .LBB9_16: # %else14
+; AVX2-NEXT:    testl $256, %eax # imm = 0x100
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
+; AVX2-NEXT:    jne .LBB9_17
+; AVX2-NEXT:  # %bb.18: # %else16
+; AVX2-NEXT:    testl $512, %eax # imm = 0x200
+; AVX2-NEXT:    jne .LBB9_19
+; AVX2-NEXT:  .LBB9_20: # %else18
+; AVX2-NEXT:    testl $1024, %eax # imm = 0x400
+; AVX2-NEXT:    jne .LBB9_21
+; AVX2-NEXT:  .LBB9_22: # %else20
+; AVX2-NEXT:    testl $2048, %eax # imm = 0x800
+; AVX2-NEXT:    jne .LBB9_23
+; AVX2-NEXT:  .LBB9_24: # %else22
+; AVX2-NEXT:    testl $4096, %eax # imm = 0x1000
+; AVX2-NEXT:    jne .LBB9_25
+; AVX2-NEXT:  .LBB9_26: # %else24
+; AVX2-NEXT:    testl $8192, %eax # imm = 0x2000
+; AVX2-NEXT:    jne .LBB9_27
+; AVX2-NEXT:  .LBB9_28: # %else26
+; AVX2-NEXT:    testl $16384, %eax # imm = 0x4000
+; AVX2-NEXT:    jne .LBB9_29
+; AVX2-NEXT:  .LBB9_30: # %else28
+; AVX2-NEXT:    testl $32768, %eax # imm = 0x8000
+; AVX2-NEXT:    jne .LBB9_31
+; AVX2-NEXT:  .LBB9_32: # %else30
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+; AVX2-NEXT:  .LBB9_1: # %cond.store
 ; AVX2-NEXT:    vpextrw $0, %xmm0, (%rdi)
-; AVX2-NEXT:  .LBB9_2: # %else
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm2, %xmm1
-; AVX2-NEXT:    vpxor %xmm5, %xmm1, %xmm1
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm1, %xmm1
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm1, %xmm1
-; AVX2-NEXT:    vpextrb $1, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $2, %al
 ; AVX2-NEXT:    je .LBB9_4
-; AVX2-NEXT:  # %bb.3: # %cond.store1
+; AVX2-NEXT:  .LBB9_3: # %cond.store1
 ; AVX2-NEXT:    vpextrw $1, %xmm0, 2(%rdi)
-; AVX2-NEXT:  .LBB9_4: # %else2
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm2, %xmm5
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm5, %xmm5
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm5, %xmm5
-; AVX2-NEXT:    vpextrb $2, %xmm5, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $4, %al
 ; AVX2-NEXT:    je .LBB9_6
-; AVX2-NEXT:  # %bb.5: # %cond.store3
+; AVX2-NEXT:  .LBB9_5: # %cond.store3
 ; AVX2-NEXT:    vpextrw $2, %xmm0, 4(%rdi)
-; AVX2-NEXT:  .LBB9_6: # %else4
-; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm2, %xmm1
-; AVX2-NEXT:    vpxor %xmm4, %xmm1, %xmm1
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm1, %xmm1
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm1, %xmm1
-; AVX2-NEXT:    vpextrb $3, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $8, %al
 ; AVX2-NEXT:    je .LBB9_8
-; AVX2-NEXT:  # %bb.7: # %cond.store5
+; AVX2-NEXT:  .LBB9_7: # %cond.store5
 ; AVX2-NEXT:    vpextrw $3, %xmm0, 6(%rdi)
-; AVX2-NEXT:  .LBB9_8: # %else6
-; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm1
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpxor %xmm4, %xmm2, %xmm2
-; AVX2-NEXT:    vpackssdw %xmm2, %xmm0, %xmm4
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm4, %xmm4
-; AVX2-NEXT:    vpextrb $4, %xmm4, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $16, %al
 ; AVX2-NEXT:    je .LBB9_10
-; AVX2-NEXT:  # %bb.9: # %cond.store7
+; AVX2-NEXT:  .LBB9_9: # %cond.store7
 ; AVX2-NEXT:    vpextrw $4, %xmm0, 8(%rdi)
-; AVX2-NEXT:  .LBB9_10: # %else8
-; AVX2-NEXT:    vpackssdw %xmm2, %xmm0, %xmm2
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrb $5, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $32, %al
 ; AVX2-NEXT:    je .LBB9_12
-; AVX2-NEXT:  # %bb.11: # %cond.store9
+; AVX2-NEXT:  .LBB9_11: # %cond.store9
 ; AVX2-NEXT:    vpextrw $5, %xmm0, 10(%rdi)
-; AVX2-NEXT:  .LBB9_12: # %else10
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpackssdw %xmm1, %xmm0, %xmm2
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrb $6, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $64, %al
 ; AVX2-NEXT:    je .LBB9_14
-; AVX2-NEXT:  # %bb.13: # %cond.store11
+; AVX2-NEXT:  .LBB9_13: # %cond.store11
 ; AVX2-NEXT:    vpextrw $6, %xmm0, 12(%rdi)
-; AVX2-NEXT:  .LBB9_14: # %else12
-; AVX2-NEXT:    vpackssdw %xmm1, %xmm0, %xmm1
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm1, %xmm1
-; AVX2-NEXT:    vpextrb $7, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je .LBB9_16
-; AVX2-NEXT:  # %bb.15: # %cond.store13
-; AVX2-NEXT:    vpextrw $7, %xmm0, 14(%rdi)
-; AVX2-NEXT:  .LBB9_16: # %else14
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm3, %xmm4
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpxor %xmm2, %xmm4, %xmm4
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm4, %xmm4
-; AVX2-NEXT:    vpacksswb %xmm4, %xmm0, %xmm4
-; AVX2-NEXT:    vpextrb $8, %xmm4, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; AVX2-NEXT:    je .LBB9_18
-; AVX2-NEXT:  # %bb.17: # %cond.store15
+; AVX2-NEXT:    testb $-128, %al
+; AVX2-NEXT:    jne .LBB9_15
+; AVX2-NEXT:    jmp .LBB9_16
+; AVX2-NEXT:  .LBB9_17: # %cond.store15
 ; AVX2-NEXT:    vpextrw $0, %xmm0, 16(%rdi)
-; AVX2-NEXT:  .LBB9_18: # %else16
-; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm3, %xmm1
-; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm1, %xmm1
-; AVX2-NEXT:    vpacksswb %xmm1, %xmm0, %xmm1
-; AVX2-NEXT:    vpextrb $9, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $512, %eax # imm = 0x200
 ; AVX2-NEXT:    je .LBB9_20
-; AVX2-NEXT:  # %bb.19: # %cond.store17
+; AVX2-NEXT:  .LBB9_19: # %cond.store17
 ; AVX2-NEXT:    vpextrw $1, %xmm0, 18(%rdi)
-; AVX2-NEXT:  .LBB9_20: # %else18
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm3, %xmm4
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpxor %xmm2, %xmm4, %xmm4
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm4, %xmm4
-; AVX2-NEXT:    vpacksswb %xmm4, %xmm0, %xmm4
-; AVX2-NEXT:    vpextrb $10, %xmm4, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $1024, %eax # imm = 0x400
 ; AVX2-NEXT:    je .LBB9_22
-; AVX2-NEXT:  # %bb.21: # %cond.store19
+; AVX2-NEXT:  .LBB9_21: # %cond.store19
 ; AVX2-NEXT:    vpextrw $2, %xmm0, 20(%rdi)
-; AVX2-NEXT:  .LBB9_22: # %else20
-; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm3, %xmm1
-; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm1, %xmm1
-; AVX2-NEXT:    vpacksswb %xmm1, %xmm0, %xmm1
-; AVX2-NEXT:    vpextrb $11, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $2048, %eax # imm = 0x800
 ; AVX2-NEXT:    je .LBB9_24
-; AVX2-NEXT:  # %bb.23: # %cond.store21
+; AVX2-NEXT:  .LBB9_23: # %cond.store21
 ; AVX2-NEXT:    vpextrw $3, %xmm0, 22(%rdi)
-; AVX2-NEXT:  .LBB9_24: # %else22
-; AVX2-NEXT:    vextracti128 $1, %ymm3, %xmm1
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX2-NEXT:    vpackssdw %xmm2, %xmm0, %xmm3
-; AVX2-NEXT:    vpacksswb %xmm3, %xmm0, %xmm3
-; AVX2-NEXT:    vpextrb $12, %xmm3, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $4096, %eax # imm = 0x1000
 ; AVX2-NEXT:    je .LBB9_26
-; AVX2-NEXT:  # %bb.25: # %cond.store23
+; AVX2-NEXT:  .LBB9_25: # %cond.store23
 ; AVX2-NEXT:    vpextrw $4, %xmm0, 24(%rdi)
-; AVX2-NEXT:  .LBB9_26: # %else24
-; AVX2-NEXT:    vpackssdw %xmm2, %xmm0, %xmm2
-; AVX2-NEXT:    vpacksswb %xmm2, %xmm0, %xmm2
-; AVX2-NEXT:    vpextrb $13, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $8192, %eax # imm = 0x2000
 ; AVX2-NEXT:    je .LBB9_28
-; AVX2-NEXT:  # %bb.27: # %cond.store25
+; AVX2-NEXT:  .LBB9_27: # %cond.store25
 ; AVX2-NEXT:    vpextrw $5, %xmm0, 26(%rdi)
-; AVX2-NEXT:  .LBB9_28: # %else26
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpackssdw %xmm1, %xmm0, %xmm2
-; AVX2-NEXT:    vpacksswb %xmm2, %xmm0, %xmm2
-; AVX2-NEXT:    vpextrb $14, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $16384, %eax # imm = 0x4000
 ; AVX2-NEXT:    je .LBB9_30
-; AVX2-NEXT:  # %bb.29: # %cond.store27
+; AVX2-NEXT:  .LBB9_29: # %cond.store27
 ; AVX2-NEXT:    vpextrw $6, %xmm0, 28(%rdi)
-; AVX2-NEXT:  .LBB9_30: # %else28
-; AVX2-NEXT:    vpackssdw %xmm1, %xmm0, %xmm1
-; AVX2-NEXT:    vpacksswb %xmm1, %xmm0, %xmm1
-; AVX2-NEXT:    vpextrb $15, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $32768, %eax # imm = 0x8000
 ; AVX2-NEXT:    je .LBB9_32
-; AVX2-NEXT:  # %bb.31: # %cond.store29
+; AVX2-NEXT:  .LBB9_31: # %cond.store29
 ; AVX2-NEXT:    vpextrw $7, %xmm0, 30(%rdi)
-; AVX2-NEXT:  .LBB9_32: # %else30
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
@@ -3615,116 +3456,117 @@ define void @truncstore_v16i32_v16i16(<16 x i32> %x, <16 x i16>* %p, <16 x i32>
 ; AVX512F-NEXT:    vpmovsdw %zmm0, %ymm0
 ; AVX512F-NEXT:    kmovw %k0, %eax
 ; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB9_2
-; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    jne .LBB9_1
+; AVX512F-NEXT:  # %bb.2: # %else
+; AVX512F-NEXT:    testb $2, %al
+; AVX512F-NEXT:    jne .LBB9_3
+; AVX512F-NEXT:  .LBB9_4: # %else2
+; AVX512F-NEXT:    testb $4, %al
+; AVX512F-NEXT:    jne .LBB9_5
+; AVX512F-NEXT:  .LBB9_6: # %else4
+; AVX512F-NEXT:    testb $8, %al
+; AVX512F-NEXT:    jne .LBB9_7
+; AVX512F-NEXT:  .LBB9_8: # %else6
+; AVX512F-NEXT:    testb $16, %al
+; AVX512F-NEXT:    jne .LBB9_9
+; AVX512F-NEXT:  .LBB9_10: # %else8
+; AVX512F-NEXT:    testb $32, %al
+; AVX512F-NEXT:    jne .LBB9_11
+; AVX512F-NEXT:  .LBB9_12: # %else10
+; AVX512F-NEXT:    testb $64, %al
+; AVX512F-NEXT:    jne .LBB9_13
+; AVX512F-NEXT:  .LBB9_14: # %else12
+; AVX512F-NEXT:    testb $-128, %al
+; AVX512F-NEXT:    je .LBB9_16
+; AVX512F-NEXT:  .LBB9_15: # %cond.store13
+; AVX512F-NEXT:    vpextrw $7, %xmm0, 14(%rdi)
+; AVX512F-NEXT:  .LBB9_16: # %else14
+; AVX512F-NEXT:    testl $256, %eax # imm = 0x100
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm0
+; AVX512F-NEXT:    jne .LBB9_17
+; AVX512F-NEXT:  # %bb.18: # %else16
+; AVX512F-NEXT:    testl $512, %eax # imm = 0x200
+; AVX512F-NEXT:    jne .LBB9_19
+; AVX512F-NEXT:  .LBB9_20: # %else18
+; AVX512F-NEXT:    testl $1024, %eax # imm = 0x400
+; AVX512F-NEXT:    jne .LBB9_21
+; AVX512F-NEXT:  .LBB9_22: # %else20
+; AVX512F-NEXT:    testl $2048, %eax # imm = 0x800
+; AVX512F-NEXT:    jne .LBB9_23
+; AVX512F-NEXT:  .LBB9_24: # %else22
+; AVX512F-NEXT:    testl $4096, %eax # imm = 0x1000
+; AVX512F-NEXT:    jne .LBB9_25
+; AVX512F-NEXT:  .LBB9_26: # %else24
+; AVX512F-NEXT:    testl $8192, %eax # imm = 0x2000
+; AVX512F-NEXT:    jne .LBB9_27
+; AVX512F-NEXT:  .LBB9_28: # %else26
+; AVX512F-NEXT:    testl $16384, %eax # imm = 0x4000
+; AVX512F-NEXT:    jne .LBB9_29
+; AVX512F-NEXT:  .LBB9_30: # %else28
+; AVX512F-NEXT:    testl $32768, %eax # imm = 0x8000
+; AVX512F-NEXT:    jne .LBB9_31
+; AVX512F-NEXT:  .LBB9_32: # %else30
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+; AVX512F-NEXT:  .LBB9_1: # %cond.store
 ; AVX512F-NEXT:    vpextrw $0, %xmm0, (%rdi)
-; AVX512F-NEXT:  .LBB9_2: # %else
-; AVX512F-NEXT:    kshiftrw $1, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $2, %al
 ; AVX512F-NEXT:    je .LBB9_4
-; AVX512F-NEXT:  # %bb.3: # %cond.store1
+; AVX512F-NEXT:  .LBB9_3: # %cond.store1
 ; AVX512F-NEXT:    vpextrw $1, %xmm0, 2(%rdi)
-; AVX512F-NEXT:  .LBB9_4: # %else2
-; AVX512F-NEXT:    kshiftrw $2, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $4, %al
 ; AVX512F-NEXT:    je .LBB9_6
-; AVX512F-NEXT:  # %bb.5: # %cond.store3
+; AVX512F-NEXT:  .LBB9_5: # %cond.store3
 ; AVX512F-NEXT:    vpextrw $2, %xmm0, 4(%rdi)
-; AVX512F-NEXT:  .LBB9_6: # %else4
-; AVX512F-NEXT:    kshiftrw $3, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $8, %al
 ; AVX512F-NEXT:    je .LBB9_8
-; AVX512F-NEXT:  # %bb.7: # %cond.store5
+; AVX512F-NEXT:  .LBB9_7: # %cond.store5
 ; AVX512F-NEXT:    vpextrw $3, %xmm0, 6(%rdi)
-; AVX512F-NEXT:  .LBB9_8: # %else6
-; AVX512F-NEXT:    kshiftrw $4, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $16, %al
 ; AVX512F-NEXT:    je .LBB9_10
-; AVX512F-NEXT:  # %bb.9: # %cond.store7
+; AVX512F-NEXT:  .LBB9_9: # %cond.store7
 ; AVX512F-NEXT:    vpextrw $4, %xmm0, 8(%rdi)
-; AVX512F-NEXT:  .LBB9_10: # %else8
-; AVX512F-NEXT:    kshiftrw $5, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $32, %al
 ; AVX512F-NEXT:    je .LBB9_12
-; AVX512F-NEXT:  # %bb.11: # %cond.store9
+; AVX512F-NEXT:  .LBB9_11: # %cond.store9
 ; AVX512F-NEXT:    vpextrw $5, %xmm0, 10(%rdi)
-; AVX512F-NEXT:  .LBB9_12: # %else10
-; AVX512F-NEXT:    kshiftrw $6, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $64, %al
 ; AVX512F-NEXT:    je .LBB9_14
-; AVX512F-NEXT:  # %bb.13: # %cond.store11
+; AVX512F-NEXT:  .LBB9_13: # %cond.store11
 ; AVX512F-NEXT:    vpextrw $6, %xmm0, 12(%rdi)
-; AVX512F-NEXT:  .LBB9_14: # %else12
-; AVX512F-NEXT:    kshiftrw $7, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB9_16
-; AVX512F-NEXT:  # %bb.15: # %cond.store13
-; AVX512F-NEXT:    vpextrw $7, %xmm0, 14(%rdi)
-; AVX512F-NEXT:  .LBB9_16: # %else14
-; AVX512F-NEXT:    kshiftrw $8, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; AVX512F-NEXT:    je .LBB9_18
-; AVX512F-NEXT:  # %bb.17: # %cond.store15
+; AVX512F-NEXT:    testb $-128, %al
+; AVX512F-NEXT:    jne .LBB9_15
+; AVX512F-NEXT:    jmp .LBB9_16
+; AVX512F-NEXT:  .LBB9_17: # %cond.store15
 ; AVX512F-NEXT:    vpextrw $0, %xmm0, 16(%rdi)
-; AVX512F-NEXT:  .LBB9_18: # %else16
-; AVX512F-NEXT:    kshiftrw $9, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $512, %eax # imm = 0x200
 ; AVX512F-NEXT:    je .LBB9_20
-; AVX512F-NEXT:  # %bb.19: # %cond.store17
+; AVX512F-NEXT:  .LBB9_19: # %cond.store17
 ; AVX512F-NEXT:    vpextrw $1, %xmm0, 18(%rdi)
-; AVX512F-NEXT:  .LBB9_20: # %else18
-; AVX512F-NEXT:    kshiftrw $10, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $1024, %eax # imm = 0x400
 ; AVX512F-NEXT:    je .LBB9_22
-; AVX512F-NEXT:  # %bb.21: # %cond.store19
+; AVX512F-NEXT:  .LBB9_21: # %cond.store19
 ; AVX512F-NEXT:    vpextrw $2, %xmm0, 20(%rdi)
-; AVX512F-NEXT:  .LBB9_22: # %else20
-; AVX512F-NEXT:    kshiftrw $11, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $2048, %eax # imm = 0x800
 ; AVX512F-NEXT:    je .LBB9_24
-; AVX512F-NEXT:  # %bb.23: # %cond.store21
+; AVX512F-NEXT:  .LBB9_23: # %cond.store21
 ; AVX512F-NEXT:    vpextrw $3, %xmm0, 22(%rdi)
-; AVX512F-NEXT:  .LBB9_24: # %else22
-; AVX512F-NEXT:    kshiftrw $12, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $4096, %eax # imm = 0x1000
 ; AVX512F-NEXT:    je .LBB9_26
-; AVX512F-NEXT:  # %bb.25: # %cond.store23
+; AVX512F-NEXT:  .LBB9_25: # %cond.store23
 ; AVX512F-NEXT:    vpextrw $4, %xmm0, 24(%rdi)
-; AVX512F-NEXT:  .LBB9_26: # %else24
-; AVX512F-NEXT:    kshiftrw $13, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $8192, %eax # imm = 0x2000
 ; AVX512F-NEXT:    je .LBB9_28
-; AVX512F-NEXT:  # %bb.27: # %cond.store25
+; AVX512F-NEXT:  .LBB9_27: # %cond.store25
 ; AVX512F-NEXT:    vpextrw $5, %xmm0, 26(%rdi)
-; AVX512F-NEXT:  .LBB9_28: # %else26
-; AVX512F-NEXT:    kshiftrw $14, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $16384, %eax # imm = 0x4000
 ; AVX512F-NEXT:    je .LBB9_30
-; AVX512F-NEXT:  # %bb.29: # %cond.store27
+; AVX512F-NEXT:  .LBB9_29: # %cond.store27
 ; AVX512F-NEXT:    vpextrw $6, %xmm0, 28(%rdi)
-; AVX512F-NEXT:  .LBB9_30: # %else28
-; AVX512F-NEXT:    kshiftrw $15, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $32768, %eax # imm = 0x8000
 ; AVX512F-NEXT:    je .LBB9_32
-; AVX512F-NEXT:  # %bb.31: # %cond.store29
+; AVX512F-NEXT:  .LBB9_31: # %cond.store29
 ; AVX512F-NEXT:    vpextrw $7, %xmm0, 30(%rdi)
-; AVX512F-NEXT:  .LBB9_32: # %else30
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
@@ -3759,647 +3601,523 @@ define void @truncstore_v16i32_v16i8(<16 x i32> %x, <16 x i8>* %p, <16 x i32> %m
 ; SSE2-LABEL: truncstore_v16i32_v16i8:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pxor %xmm8, %xmm8
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm8
 ; SSE2-NEXT:    packssdw %xmm3, %xmm2
 ; SSE2-NEXT:    packssdw %xmm1, %xmm0
 ; SSE2-NEXT:    packsswb %xmm2, %xmm0
-; SSE2-NEXT:    movd %xmm8, %eax
-; SSE2-NEXT:    notl %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    je .LBB10_2
-; SSE2-NEXT:  # %bb.1: # %cond.store
-; SSE2-NEXT:    movb %al, (%rdi)
-; SSE2-NEXT:  .LBB10_2: # %else
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm7
 ; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pxor %xmm1, %xmm8
-; SSE2-NEXT:    pextrw $2, %xmm8, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    je .LBB10_4
-; SSE2-NEXT:  # %bb.3: # %cond.store1
-; SSE2-NEXT:    movb %ah, 1(%rdi)
+; SSE2-NEXT:    pxor %xmm1, %xmm7
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm6
+; SSE2-NEXT:    pxor %xmm1, %xmm6
+; SSE2-NEXT:    packssdw %xmm7, %xmm6
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm5
+; SSE2-NEXT:    pxor %xmm1, %xmm5
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm4
+; SSE2-NEXT:    pxor %xmm1, %xmm4
+; SSE2-NEXT:    packssdw %xmm5, %xmm4
+; SSE2-NEXT:    packsswb %xmm6, %xmm4
+; SSE2-NEXT:    pmovmskb %xmm4, %eax
+; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    movd %xmm0, %ecx
+; SSE2-NEXT:    jne .LBB10_1
+; SSE2-NEXT:  # %bb.2: # %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne .LBB10_3
 ; SSE2-NEXT:  .LBB10_4: # %else2
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm4
-; SSE2-NEXT:    pxor %xmm4, %xmm1
-; SSE2-NEXT:    pextrw $4, %xmm1, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    je .LBB10_6
-; SSE2-NEXT:  # %bb.5: # %cond.store3
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    shrl $16, %ecx
-; SSE2-NEXT:    movb %cl, 2(%rdi)
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne .LBB10_5
 ; SSE2-NEXT:  .LBB10_6: # %else4
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pxor %xmm1, %xmm4
-; SSE2-NEXT:    pextrw $6, %xmm4, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je .LBB10_8
-; SSE2-NEXT:  # %bb.7: # %cond.store5
-; SSE2-NEXT:    shrl $24, %eax
-; SSE2-NEXT:    movb %al, 3(%rdi)
+; SSE2-NEXT:  .LBB10_7: # %cond.store5
+; SSE2-NEXT:    shrl $24, %ecx
+; SSE2-NEXT:    movb %cl, 3(%rdi)
 ; SSE2-NEXT:  .LBB10_8: # %else6
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm1
-; SSE2-NEXT:    movd %xmm1, %eax
-; SSE2-NEXT:    notl %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    pextrw $2, %xmm0, %eax
+; SSE2-NEXT:    testb $16, %al
+; SSE2-NEXT:    pextrw $2, %xmm0, %ecx
 ; SSE2-NEXT:    je .LBB10_10
 ; SSE2-NEXT:  # %bb.9: # %cond.store7
-; SSE2-NEXT:    movb %al, 4(%rdi)
+; SSE2-NEXT:    movb %cl, 4(%rdi)
 ; SSE2-NEXT:  .LBB10_10: # %else8
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT:    pxor %xmm2, %xmm1
-; SSE2-NEXT:    pextrw $2, %xmm1, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testb $32, %al
 ; SSE2-NEXT:    je .LBB10_12
 ; SSE2-NEXT:  # %bb.11: # %cond.store9
-; SSE2-NEXT:    movb %ah, 5(%rdi)
+; SSE2-NEXT:    movb %ch, 5(%rdi)
 ; SSE2-NEXT:  .LBB10_12: # %else10
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm5
-; SSE2-NEXT:    pxor %xmm5, %xmm2
-; SSE2-NEXT:    pextrw $4, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    pextrw $3, %xmm0, %eax
+; SSE2-NEXT:    testb $64, %al
+; SSE2-NEXT:    pextrw $3, %xmm0, %ecx
 ; SSE2-NEXT:    je .LBB10_14
 ; SSE2-NEXT:  # %bb.13: # %cond.store11
-; SSE2-NEXT:    movb %al, 6(%rdi)
+; SSE2-NEXT:    movb %cl, 6(%rdi)
 ; SSE2-NEXT:  .LBB10_14: # %else12
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pxor %xmm1, %xmm5
-; SSE2-NEXT:    pextrw $6, %xmm5, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testb $-128, %al
 ; SSE2-NEXT:    je .LBB10_16
 ; SSE2-NEXT:  # %bb.15: # %cond.store13
-; SSE2-NEXT:    movb %ah, 7(%rdi)
+; SSE2-NEXT:    movb %ch, 7(%rdi)
 ; SSE2-NEXT:  .LBB10_16: # %else14
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm6, %xmm1
-; SSE2-NEXT:    movd %xmm1, %eax
-; SSE2-NEXT:    notl %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
+; SSE2-NEXT:    testl $256, %eax # imm = 0x100
+; SSE2-NEXT:    pextrw $4, %xmm0, %ecx
 ; SSE2-NEXT:    je .LBB10_18
 ; SSE2-NEXT:  # %bb.17: # %cond.store15
-; SSE2-NEXT:    movb %al, 8(%rdi)
+; SSE2-NEXT:    movb %cl, 8(%rdi)
 ; SSE2-NEXT:  .LBB10_18: # %else16
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT:    pxor %xmm2, %xmm1
-; SSE2-NEXT:    pextrw $2, %xmm1, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $512, %eax # imm = 0x200
 ; SSE2-NEXT:    je .LBB10_20
 ; SSE2-NEXT:  # %bb.19: # %cond.store17
-; SSE2-NEXT:    movb %ah, 9(%rdi)
+; SSE2-NEXT:    movb %ch, 9(%rdi)
 ; SSE2-NEXT:  .LBB10_20: # %else18
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm6
-; SSE2-NEXT:    pxor %xmm6, %xmm2
-; SSE2-NEXT:    pextrw $4, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    pextrw $5, %xmm0, %eax
+; SSE2-NEXT:    testl $1024, %eax # imm = 0x400
+; SSE2-NEXT:    pextrw $5, %xmm0, %ecx
 ; SSE2-NEXT:    je .LBB10_22
 ; SSE2-NEXT:  # %bb.21: # %cond.store19
-; SSE2-NEXT:    movb %al, 10(%rdi)
+; SSE2-NEXT:    movb %cl, 10(%rdi)
 ; SSE2-NEXT:  .LBB10_22: # %else20
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pxor %xmm1, %xmm6
-; SSE2-NEXT:    pextrw $6, %xmm6, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $2048, %eax # imm = 0x800
 ; SSE2-NEXT:    je .LBB10_24
 ; SSE2-NEXT:  # %bb.23: # %cond.store21
-; SSE2-NEXT:    movb %ah, 11(%rdi)
+; SSE2-NEXT:    movb %ch, 11(%rdi)
 ; SSE2-NEXT:  .LBB10_24: # %else22
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm7, %xmm1
-; SSE2-NEXT:    movd %xmm1, %eax
-; SSE2-NEXT:    notl %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    pextrw $6, %xmm0, %eax
+; SSE2-NEXT:    testl $4096, %eax # imm = 0x1000
+; SSE2-NEXT:    pextrw $6, %xmm0, %ecx
 ; SSE2-NEXT:    je .LBB10_26
 ; SSE2-NEXT:  # %bb.25: # %cond.store23
-; SSE2-NEXT:    movb %al, 12(%rdi)
+; SSE2-NEXT:    movb %cl, 12(%rdi)
 ; SSE2-NEXT:  .LBB10_26: # %else24
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT:    pxor %xmm2, %xmm1
-; SSE2-NEXT:    pextrw $2, %xmm1, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $8192, %eax # imm = 0x2000
 ; SSE2-NEXT:    je .LBB10_28
 ; SSE2-NEXT:  # %bb.27: # %cond.store25
-; SSE2-NEXT:    movb %ah, 13(%rdi)
+; SSE2-NEXT:    movb %ch, 13(%rdi)
 ; SSE2-NEXT:  .LBB10_28: # %else26
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm7
-; SSE2-NEXT:    pxor %xmm7, %xmm2
-; SSE2-NEXT:    pextrw $4, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    pextrw $7, %xmm0, %eax
-; SSE2-NEXT:    je .LBB10_30
-; SSE2-NEXT:  # %bb.29: # %cond.store27
-; SSE2-NEXT:    movb %al, 14(%rdi)
-; SSE2-NEXT:  .LBB10_30: # %else28
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE2-NEXT:    pxor %xmm0, %xmm7
-; SSE2-NEXT:    pextrw $6, %xmm7, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    je .LBB10_32
-; SSE2-NEXT:  # %bb.31: # %cond.store29
-; SSE2-NEXT:    movb %ah, 15(%rdi)
+; SSE2-NEXT:    testl $16384, %eax # imm = 0x4000
+; SSE2-NEXT:    pextrw $7, %xmm0, %ecx
+; SSE2-NEXT:    jne .LBB10_29
+; SSE2-NEXT:  # %bb.30: # %else28
+; SSE2-NEXT:    testl $32768, %eax # imm = 0x8000
+; SSE2-NEXT:    jne .LBB10_31
 ; SSE2-NEXT:  .LBB10_32: # %else30
 ; SSE2-NEXT:    retq
+; SSE2-NEXT:  .LBB10_1: # %cond.store
+; SSE2-NEXT:    movb %cl, (%rdi)
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    je .LBB10_4
+; SSE2-NEXT:  .LBB10_3: # %cond.store1
+; SSE2-NEXT:    movb %ch, 1(%rdi)
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    je .LBB10_6
+; SSE2-NEXT:  .LBB10_5: # %cond.store3
+; SSE2-NEXT:    movl %ecx, %edx
+; SSE2-NEXT:    shrl $16, %edx
+; SSE2-NEXT:    movb %dl, 2(%rdi)
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne .LBB10_7
+; SSE2-NEXT:    jmp .LBB10_8
+; SSE2-NEXT:  .LBB10_29: # %cond.store27
+; SSE2-NEXT:    movb %cl, 14(%rdi)
+; SSE2-NEXT:    testl $32768, %eax # imm = 0x8000
+; SSE2-NEXT:    je .LBB10_32
+; SSE2-NEXT:  .LBB10_31: # %cond.store29
+; SSE2-NEXT:    movb %ch, 15(%rdi)
+; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: truncstore_v16i32_v16i8:
 ; SSE4:       # %bb.0:
-; SSE4-NEXT:    pxor %xmm9, %xmm9
-; SSE4-NEXT:    pcmpeqd %xmm4, %xmm9
-; SSE4-NEXT:    pcmpeqd %xmm8, %xmm8
-; SSE4-NEXT:    pxor %xmm9, %xmm8
+; SSE4-NEXT:    pxor %xmm8, %xmm8
 ; SSE4-NEXT:    packssdw %xmm3, %xmm2
 ; SSE4-NEXT:    packssdw %xmm1, %xmm0
 ; SSE4-NEXT:    packsswb %xmm2, %xmm0
-; SSE4-NEXT:    pextrb $0, %xmm8, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB10_2
-; SSE4-NEXT:  # %bb.1: # %cond.store
+; SSE4-NEXT:    pcmpeqd %xmm8, %xmm7
+; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
+; SSE4-NEXT:    pxor %xmm1, %xmm7
+; SSE4-NEXT:    pcmpeqd %xmm8, %xmm6
+; SSE4-NEXT:    pxor %xmm1, %xmm6
+; SSE4-NEXT:    packssdw %xmm7, %xmm6
+; SSE4-NEXT:    pcmpeqd %xmm8, %xmm5
+; SSE4-NEXT:    pxor %xmm1, %xmm5
+; SSE4-NEXT:    pcmpeqd %xmm8, %xmm4
+; SSE4-NEXT:    pxor %xmm1, %xmm4
+; SSE4-NEXT:    packssdw %xmm5, %xmm4
+; SSE4-NEXT:    packsswb %xmm6, %xmm4
+; SSE4-NEXT:    pmovmskb %xmm4, %eax
+; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    jne .LBB10_1
+; SSE4-NEXT:  # %bb.2: # %else
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    jne .LBB10_3
+; SSE4-NEXT:  .LBB10_4: # %else2
+; SSE4-NEXT:    testb $4, %al
+; SSE4-NEXT:    jne .LBB10_5
+; SSE4-NEXT:  .LBB10_6: # %else4
+; SSE4-NEXT:    testb $8, %al
+; SSE4-NEXT:    jne .LBB10_7
+; SSE4-NEXT:  .LBB10_8: # %else6
+; SSE4-NEXT:    testb $16, %al
+; SSE4-NEXT:    jne .LBB10_9
+; SSE4-NEXT:  .LBB10_10: # %else8
+; SSE4-NEXT:    testb $32, %al
+; SSE4-NEXT:    jne .LBB10_11
+; SSE4-NEXT:  .LBB10_12: # %else10
+; SSE4-NEXT:    testb $64, %al
+; SSE4-NEXT:    jne .LBB10_13
+; SSE4-NEXT:  .LBB10_14: # %else12
+; SSE4-NEXT:    testb $-128, %al
+; SSE4-NEXT:    jne .LBB10_15
+; SSE4-NEXT:  .LBB10_16: # %else14
+; SSE4-NEXT:    testl $256, %eax # imm = 0x100
+; SSE4-NEXT:    jne .LBB10_17
+; SSE4-NEXT:  .LBB10_18: # %else16
+; SSE4-NEXT:    testl $512, %eax # imm = 0x200
+; SSE4-NEXT:    jne .LBB10_19
+; SSE4-NEXT:  .LBB10_20: # %else18
+; SSE4-NEXT:    testl $1024, %eax # imm = 0x400
+; SSE4-NEXT:    jne .LBB10_21
+; SSE4-NEXT:  .LBB10_22: # %else20
+; SSE4-NEXT:    testl $2048, %eax # imm = 0x800
+; SSE4-NEXT:    jne .LBB10_23
+; SSE4-NEXT:  .LBB10_24: # %else22
+; SSE4-NEXT:    testl $4096, %eax # imm = 0x1000
+; SSE4-NEXT:    jne .LBB10_25
+; SSE4-NEXT:  .LBB10_26: # %else24
+; SSE4-NEXT:    testl $8192, %eax # imm = 0x2000
+; SSE4-NEXT:    jne .LBB10_27
+; SSE4-NEXT:  .LBB10_28: # %else26
+; SSE4-NEXT:    testl $16384, %eax # imm = 0x4000
+; SSE4-NEXT:    jne .LBB10_29
+; SSE4-NEXT:  .LBB10_30: # %else28
+; SSE4-NEXT:    testl $32768, %eax # imm = 0x8000
+; SSE4-NEXT:    jne .LBB10_31
+; SSE4-NEXT:  .LBB10_32: # %else30
+; SSE4-NEXT:    retq
+; SSE4-NEXT:  .LBB10_1: # %cond.store
 ; SSE4-NEXT:    pextrb $0, %xmm0, (%rdi)
-; SSE4-NEXT:  .LBB10_2: # %else
-; SSE4-NEXT:    pextrb $4, %xmm8, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $2, %al
 ; SSE4-NEXT:    je .LBB10_4
-; SSE4-NEXT:  # %bb.3: # %cond.store1
+; SSE4-NEXT:  .LBB10_3: # %cond.store1
 ; SSE4-NEXT:    pextrb $1, %xmm0, 1(%rdi)
-; SSE4-NEXT:  .LBB10_4: # %else2
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm4
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE4-NEXT:    pxor %xmm4, %xmm1
-; SSE4-NEXT:    pextrb $8, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $4, %al
 ; SSE4-NEXT:    je .LBB10_6
-; SSE4-NEXT:  # %bb.5: # %cond.store3
+; SSE4-NEXT:  .LBB10_5: # %cond.store3
 ; SSE4-NEXT:    pextrb $2, %xmm0, 2(%rdi)
-; SSE4-NEXT:  .LBB10_6: # %else4
-; SSE4-NEXT:    pextrb $12, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $8, %al
 ; SSE4-NEXT:    je .LBB10_8
-; SSE4-NEXT:  # %bb.7: # %cond.store5
+; SSE4-NEXT:  .LBB10_7: # %cond.store5
 ; SSE4-NEXT:    pextrb $3, %xmm0, 3(%rdi)
-; SSE4-NEXT:  .LBB10_8: # %else6
-; SSE4-NEXT:    pxor %xmm2, %xmm2
-; SSE4-NEXT:    pcmpeqd %xmm5, %xmm2
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE4-NEXT:    pxor %xmm2, %xmm1
-; SSE4-NEXT:    pextrb $0, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $16, %al
 ; SSE4-NEXT:    je .LBB10_10
-; SSE4-NEXT:  # %bb.9: # %cond.store7
+; SSE4-NEXT:  .LBB10_9: # %cond.store7
 ; SSE4-NEXT:    pextrb $4, %xmm0, 4(%rdi)
-; SSE4-NEXT:  .LBB10_10: # %else8
-; SSE4-NEXT:    pextrb $4, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $32, %al
 ; SSE4-NEXT:    je .LBB10_12
-; SSE4-NEXT:  # %bb.11: # %cond.store9
+; SSE4-NEXT:  .LBB10_11: # %cond.store9
 ; SSE4-NEXT:    pextrb $5, %xmm0, 5(%rdi)
-; SSE4-NEXT:  .LBB10_12: # %else10
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm5
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE4-NEXT:    pxor %xmm5, %xmm1
-; SSE4-NEXT:    pextrb $8, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $64, %al
 ; SSE4-NEXT:    je .LBB10_14
-; SSE4-NEXT:  # %bb.13: # %cond.store11
+; SSE4-NEXT:  .LBB10_13: # %cond.store11
 ; SSE4-NEXT:    pextrb $6, %xmm0, 6(%rdi)
-; SSE4-NEXT:  .LBB10_14: # %else12
-; SSE4-NEXT:    pextrb $12, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $-128, %al
 ; SSE4-NEXT:    je .LBB10_16
-; SSE4-NEXT:  # %bb.15: # %cond.store13
+; SSE4-NEXT:  .LBB10_15: # %cond.store13
 ; SSE4-NEXT:    pextrb $7, %xmm0, 7(%rdi)
-; SSE4-NEXT:  .LBB10_16: # %else14
-; SSE4-NEXT:    pxor %xmm2, %xmm2
-; SSE4-NEXT:    pcmpeqd %xmm6, %xmm2
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE4-NEXT:    pxor %xmm2, %xmm1
-; SSE4-NEXT:    pextrb $0, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $256, %eax # imm = 0x100
 ; SSE4-NEXT:    je .LBB10_18
-; SSE4-NEXT:  # %bb.17: # %cond.store15
+; SSE4-NEXT:  .LBB10_17: # %cond.store15
 ; SSE4-NEXT:    pextrb $8, %xmm0, 8(%rdi)
-; SSE4-NEXT:  .LBB10_18: # %else16
-; SSE4-NEXT:    pextrb $4, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $512, %eax # imm = 0x200
 ; SSE4-NEXT:    je .LBB10_20
-; SSE4-NEXT:  # %bb.19: # %cond.store17
+; SSE4-NEXT:  .LBB10_19: # %cond.store17
 ; SSE4-NEXT:    pextrb $9, %xmm0, 9(%rdi)
-; SSE4-NEXT:  .LBB10_20: # %else18
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm6
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE4-NEXT:    pxor %xmm6, %xmm1
-; SSE4-NEXT:    pextrb $8, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $1024, %eax # imm = 0x400
 ; SSE4-NEXT:    je .LBB10_22
-; SSE4-NEXT:  # %bb.21: # %cond.store19
+; SSE4-NEXT:  .LBB10_21: # %cond.store19
 ; SSE4-NEXT:    pextrb $10, %xmm0, 10(%rdi)
-; SSE4-NEXT:  .LBB10_22: # %else20
-; SSE4-NEXT:    pextrb $12, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $2048, %eax # imm = 0x800
 ; SSE4-NEXT:    je .LBB10_24
-; SSE4-NEXT:  # %bb.23: # %cond.store21
+; SSE4-NEXT:  .LBB10_23: # %cond.store21
 ; SSE4-NEXT:    pextrb $11, %xmm0, 11(%rdi)
-; SSE4-NEXT:  .LBB10_24: # %else22
-; SSE4-NEXT:    pxor %xmm2, %xmm2
-; SSE4-NEXT:    pcmpeqd %xmm7, %xmm2
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE4-NEXT:    pxor %xmm2, %xmm1
-; SSE4-NEXT:    pextrb $0, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $4096, %eax # imm = 0x1000
 ; SSE4-NEXT:    je .LBB10_26
-; SSE4-NEXT:  # %bb.25: # %cond.store23
+; SSE4-NEXT:  .LBB10_25: # %cond.store23
 ; SSE4-NEXT:    pextrb $12, %xmm0, 12(%rdi)
-; SSE4-NEXT:  .LBB10_26: # %else24
-; SSE4-NEXT:    pextrb $4, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $8192, %eax # imm = 0x2000
 ; SSE4-NEXT:    je .LBB10_28
-; SSE4-NEXT:  # %bb.27: # %cond.store25
+; SSE4-NEXT:  .LBB10_27: # %cond.store25
 ; SSE4-NEXT:    pextrb $13, %xmm0, 13(%rdi)
-; SSE4-NEXT:  .LBB10_28: # %else26
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm7
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE4-NEXT:    pxor %xmm7, %xmm1
-; SSE4-NEXT:    pextrb $8, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $16384, %eax # imm = 0x4000
 ; SSE4-NEXT:    je .LBB10_30
-; SSE4-NEXT:  # %bb.29: # %cond.store27
+; SSE4-NEXT:  .LBB10_29: # %cond.store27
 ; SSE4-NEXT:    pextrb $14, %xmm0, 14(%rdi)
-; SSE4-NEXT:  .LBB10_30: # %else28
-; SSE4-NEXT:    pextrb $12, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $32768, %eax # imm = 0x8000
 ; SSE4-NEXT:    je .LBB10_32
-; SSE4-NEXT:  # %bb.31: # %cond.store29
+; SSE4-NEXT:  .LBB10_31: # %cond.store29
 ; SSE4-NEXT:    pextrb $15, %xmm0, 15(%rdi)
-; SSE4-NEXT:  .LBB10_32: # %else30
 ; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: truncstore_v16i32_v16i8:
 ; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm4
+; AVX1-NEXT:    vpackssdw %xmm4, %xmm1, %xmm1
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
+; AVX1-NEXT:    vpackssdw %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm1
 ; AVX1-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm2, %xmm6
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm1, %xmm1
 ; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
-; AVX1-NEXT:    vpxor %xmm5, %xmm6, %xmm6
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm6, %xmm6
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm6, %xmm6
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm7
-; AVX1-NEXT:    vpackssdw %xmm7, %xmm1, %xmm1
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm7
-; AVX1-NEXT:    vpackssdw %xmm7, %xmm0, %xmm0
-; AVX1-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpextrb $0, %xmm6, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je .LBB10_2
-; AVX1-NEXT:  # %bb.1: # %cond.store
-; AVX1-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX1-NEXT:  .LBB10_2: # %else
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm2, %xmm1
 ; AVX1-NEXT:    vpxor %xmm5, %xmm1, %xmm1
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm1, %xmm1
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $1, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpxor %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vpackssdw %xmm1, %xmm3, %xmm1
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm3
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpxor %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm2, %xmm2
+; AVX1-NEXT:    vpxor %xmm5, %xmm2, %xmm2
+; AVX1-NEXT:    vpackssdw %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpacksswb %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vpmovmskb %xmm1, %eax
+; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    jne .LBB10_1
+; AVX1-NEXT:  # %bb.2: # %else
+; AVX1-NEXT:    testb $2, %al
+; AVX1-NEXT:    jne .LBB10_3
+; AVX1-NEXT:  .LBB10_4: # %else2
+; AVX1-NEXT:    testb $4, %al
+; AVX1-NEXT:    jne .LBB10_5
+; AVX1-NEXT:  .LBB10_6: # %else4
+; AVX1-NEXT:    testb $8, %al
+; AVX1-NEXT:    jne .LBB10_7
+; AVX1-NEXT:  .LBB10_8: # %else6
+; AVX1-NEXT:    testb $16, %al
+; AVX1-NEXT:    jne .LBB10_9
+; AVX1-NEXT:  .LBB10_10: # %else8
+; AVX1-NEXT:    testb $32, %al
+; AVX1-NEXT:    jne .LBB10_11
+; AVX1-NEXT:  .LBB10_12: # %else10
+; AVX1-NEXT:    testb $64, %al
+; AVX1-NEXT:    jne .LBB10_13
+; AVX1-NEXT:  .LBB10_14: # %else12
+; AVX1-NEXT:    testb $-128, %al
+; AVX1-NEXT:    jne .LBB10_15
+; AVX1-NEXT:  .LBB10_16: # %else14
+; AVX1-NEXT:    testl $256, %eax # imm = 0x100
+; AVX1-NEXT:    jne .LBB10_17
+; AVX1-NEXT:  .LBB10_18: # %else16
+; AVX1-NEXT:    testl $512, %eax # imm = 0x200
+; AVX1-NEXT:    jne .LBB10_19
+; AVX1-NEXT:  .LBB10_20: # %else18
+; AVX1-NEXT:    testl $1024, %eax # imm = 0x400
+; AVX1-NEXT:    jne .LBB10_21
+; AVX1-NEXT:  .LBB10_22: # %else20
+; AVX1-NEXT:    testl $2048, %eax # imm = 0x800
+; AVX1-NEXT:    jne .LBB10_23
+; AVX1-NEXT:  .LBB10_24: # %else22
+; AVX1-NEXT:    testl $4096, %eax # imm = 0x1000
+; AVX1-NEXT:    jne .LBB10_25
+; AVX1-NEXT:  .LBB10_26: # %else24
+; AVX1-NEXT:    testl $8192, %eax # imm = 0x2000
+; AVX1-NEXT:    jne .LBB10_27
+; AVX1-NEXT:  .LBB10_28: # %else26
+; AVX1-NEXT:    testl $16384, %eax # imm = 0x4000
+; AVX1-NEXT:    jne .LBB10_29
+; AVX1-NEXT:  .LBB10_30: # %else28
+; AVX1-NEXT:    testl $32768, %eax # imm = 0x8000
+; AVX1-NEXT:    jne .LBB10_31
+; AVX1-NEXT:  .LBB10_32: # %else30
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+; AVX1-NEXT:  .LBB10_1: # %cond.store
+; AVX1-NEXT:    vpextrb $0, %xmm0, (%rdi)
+; AVX1-NEXT:    testb $2, %al
 ; AVX1-NEXT:    je .LBB10_4
-; AVX1-NEXT:  # %bb.3: # %cond.store1
+; AVX1-NEXT:  .LBB10_3: # %cond.store1
 ; AVX1-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
-; AVX1-NEXT:  .LBB10_4: # %else2
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm2, %xmm5
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm5, %xmm5
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm5, %xmm5
-; AVX1-NEXT:    vpextrb $2, %xmm5, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $4, %al
 ; AVX1-NEXT:    je .LBB10_6
-; AVX1-NEXT:  # %bb.5: # %cond.store3
+; AVX1-NEXT:  .LBB10_5: # %cond.store3
 ; AVX1-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
-; AVX1-NEXT:  .LBB10_6: # %else4
-; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm4, %xmm1, %xmm1
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm1, %xmm1
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $3, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $8, %al
 ; AVX1-NEXT:    je .LBB10_8
-; AVX1-NEXT:  # %bb.7: # %cond.store5
+; AVX1-NEXT:  .LBB10_7: # %cond.store5
 ; AVX1-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
-; AVX1-NEXT:  .LBB10_8: # %else6
-; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm2
-; AVX1-NEXT:    vpackssdw %xmm2, %xmm0, %xmm4
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm4, %xmm4
-; AVX1-NEXT:    vpextrb $4, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $16, %al
 ; AVX1-NEXT:    je .LBB10_10
-; AVX1-NEXT:  # %bb.9: # %cond.store7
+; AVX1-NEXT:  .LBB10_9: # %cond.store7
 ; AVX1-NEXT:    vpextrb $4, %xmm0, 4(%rdi)
-; AVX1-NEXT:  .LBB10_10: # %else8
-; AVX1-NEXT:    vpackssdw %xmm2, %xmm0, %xmm2
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $5, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $32, %al
 ; AVX1-NEXT:    je .LBB10_12
-; AVX1-NEXT:  # %bb.11: # %cond.store9
+; AVX1-NEXT:  .LBB10_11: # %cond.store9
 ; AVX1-NEXT:    vpextrb $5, %xmm0, 5(%rdi)
-; AVX1-NEXT:  .LBB10_12: # %else10
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpackssdw %xmm1, %xmm0, %xmm2
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $6, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $64, %al
 ; AVX1-NEXT:    je .LBB10_14
-; AVX1-NEXT:  # %bb.13: # %cond.store11
+; AVX1-NEXT:  .LBB10_13: # %cond.store11
 ; AVX1-NEXT:    vpextrb $6, %xmm0, 6(%rdi)
-; AVX1-NEXT:  .LBB10_14: # %else12
-; AVX1-NEXT:    vpackssdw %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $7, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $-128, %al
 ; AVX1-NEXT:    je .LBB10_16
-; AVX1-NEXT:  # %bb.15: # %cond.store13
+; AVX1-NEXT:  .LBB10_15: # %cond.store13
 ; AVX1-NEXT:    vpextrb $7, %xmm0, 7(%rdi)
-; AVX1-NEXT:  .LBB10_16: # %else14
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm3, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpxor %xmm2, %xmm4, %xmm4
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm4, %xmm4
-; AVX1-NEXT:    vpacksswb %xmm4, %xmm0, %xmm4
-; AVX1-NEXT:    vpextrb $8, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $256, %eax # imm = 0x100
 ; AVX1-NEXT:    je .LBB10_18
-; AVX1-NEXT:  # %bb.17: # %cond.store15
+; AVX1-NEXT:  .LBB10_17: # %cond.store15
 ; AVX1-NEXT:    vpextrb $8, %xmm0, 8(%rdi)
-; AVX1-NEXT:  .LBB10_18: # %else16
-; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm3, %xmm1
-; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm1, %xmm1
-; AVX1-NEXT:    vpacksswb %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpextrb $9, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $512, %eax # imm = 0x200
 ; AVX1-NEXT:    je .LBB10_20
-; AVX1-NEXT:  # %bb.19: # %cond.store17
+; AVX1-NEXT:  .LBB10_19: # %cond.store17
 ; AVX1-NEXT:    vpextrb $9, %xmm0, 9(%rdi)
-; AVX1-NEXT:  .LBB10_20: # %else18
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm3, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpxor %xmm2, %xmm4, %xmm4
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm4, %xmm4
-; AVX1-NEXT:    vpacksswb %xmm4, %xmm0, %xmm4
-; AVX1-NEXT:    vpextrb $10, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $1024, %eax # imm = 0x400
 ; AVX1-NEXT:    je .LBB10_22
-; AVX1-NEXT:  # %bb.21: # %cond.store19
+; AVX1-NEXT:  .LBB10_21: # %cond.store19
 ; AVX1-NEXT:    vpextrb $10, %xmm0, 10(%rdi)
-; AVX1-NEXT:  .LBB10_22: # %else20
-; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm3, %xmm1
-; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm1, %xmm1
-; AVX1-NEXT:    vpacksswb %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpextrb $11, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $2048, %eax # imm = 0x800
 ; AVX1-NEXT:    je .LBB10_24
-; AVX1-NEXT:  # %bb.23: # %cond.store21
+; AVX1-NEXT:  .LBB10_23: # %cond.store21
 ; AVX1-NEXT:    vpextrb $11, %xmm0, 11(%rdi)
-; AVX1-NEXT:  .LBB10_24: # %else22
-; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm1
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpackssdw %xmm2, %xmm0, %xmm3
-; AVX1-NEXT:    vpacksswb %xmm3, %xmm0, %xmm3
-; AVX1-NEXT:    vpextrb $12, %xmm3, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $4096, %eax # imm = 0x1000
 ; AVX1-NEXT:    je .LBB10_26
-; AVX1-NEXT:  # %bb.25: # %cond.store23
+; AVX1-NEXT:  .LBB10_25: # %cond.store23
 ; AVX1-NEXT:    vpextrb $12, %xmm0, 12(%rdi)
-; AVX1-NEXT:  .LBB10_26: # %else24
-; AVX1-NEXT:    vpackssdw %xmm2, %xmm0, %xmm2
-; AVX1-NEXT:    vpacksswb %xmm2, %xmm0, %xmm2
-; AVX1-NEXT:    vpextrb $13, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $8192, %eax # imm = 0x2000
 ; AVX1-NEXT:    je .LBB10_28
-; AVX1-NEXT:  # %bb.27: # %cond.store25
+; AVX1-NEXT:  .LBB10_27: # %cond.store25
 ; AVX1-NEXT:    vpextrb $13, %xmm0, 13(%rdi)
-; AVX1-NEXT:  .LBB10_28: # %else26
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpackssdw %xmm1, %xmm0, %xmm2
-; AVX1-NEXT:    vpacksswb %xmm2, %xmm0, %xmm2
-; AVX1-NEXT:    vpextrb $14, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $16384, %eax # imm = 0x4000
 ; AVX1-NEXT:    je .LBB10_30
-; AVX1-NEXT:  # %bb.29: # %cond.store27
+; AVX1-NEXT:  .LBB10_29: # %cond.store27
 ; AVX1-NEXT:    vpextrb $14, %xmm0, 14(%rdi)
-; AVX1-NEXT:  .LBB10_30: # %else28
-; AVX1-NEXT:    vpackssdw %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpacksswb %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpextrb $15, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $32768, %eax # imm = 0x8000
 ; AVX1-NEXT:    je .LBB10_32
-; AVX1-NEXT:  # %bb.31: # %cond.store29
+; AVX1-NEXT:  .LBB10_31: # %cond.store29
 ; AVX1-NEXT:    vpextrb $15, %xmm0, 15(%rdi)
-; AVX1-NEXT:  .LBB10_32: # %else30
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: truncstore_v16i32_v16i8:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm2, %xmm6
-; AVX2-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
-; AVX2-NEXT:    vpxor %xmm5, %xmm6, %xmm6
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm6, %xmm6
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm6, %xmm6
 ; AVX2-NEXT:    vpackssdw %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpextrb $0, %xmm6, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je .LBB10_2
-; AVX2-NEXT:  # %bb.1: # %cond.store
+; AVX2-NEXT:    vpcmpeqd %ymm4, %ymm3, %ymm1
+; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm3, %ymm3
+; AVX2-NEXT:    vpxor %ymm3, %ymm1, %ymm1
+; AVX2-NEXT:    vpcmpeqd %ymm4, %ymm2, %ymm2
+; AVX2-NEXT:    vpxor %ymm3, %ymm2, %ymm2
+; AVX2-NEXT:    vpackssdw %ymm1, %ymm2, %ymm1
+; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
+; AVX2-NEXT:    vpacksswb %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpmovmskb %xmm1, %eax
+; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    jne .LBB10_1
+; AVX2-NEXT:  # %bb.2: # %else
+; AVX2-NEXT:    testb $2, %al
+; AVX2-NEXT:    jne .LBB10_3
+; AVX2-NEXT:  .LBB10_4: # %else2
+; AVX2-NEXT:    testb $4, %al
+; AVX2-NEXT:    jne .LBB10_5
+; AVX2-NEXT:  .LBB10_6: # %else4
+; AVX2-NEXT:    testb $8, %al
+; AVX2-NEXT:    jne .LBB10_7
+; AVX2-NEXT:  .LBB10_8: # %else6
+; AVX2-NEXT:    testb $16, %al
+; AVX2-NEXT:    jne .LBB10_9
+; AVX2-NEXT:  .LBB10_10: # %else8
+; AVX2-NEXT:    testb $32, %al
+; AVX2-NEXT:    jne .LBB10_11
+; AVX2-NEXT:  .LBB10_12: # %else10
+; AVX2-NEXT:    testb $64, %al
+; AVX2-NEXT:    jne .LBB10_13
+; AVX2-NEXT:  .LBB10_14: # %else12
+; AVX2-NEXT:    testb $-128, %al
+; AVX2-NEXT:    jne .LBB10_15
+; AVX2-NEXT:  .LBB10_16: # %else14
+; AVX2-NEXT:    testl $256, %eax # imm = 0x100
+; AVX2-NEXT:    jne .LBB10_17
+; AVX2-NEXT:  .LBB10_18: # %else16
+; AVX2-NEXT:    testl $512, %eax # imm = 0x200
+; AVX2-NEXT:    jne .LBB10_19
+; AVX2-NEXT:  .LBB10_20: # %else18
+; AVX2-NEXT:    testl $1024, %eax # imm = 0x400
+; AVX2-NEXT:    jne .LBB10_21
+; AVX2-NEXT:  .LBB10_22: # %else20
+; AVX2-NEXT:    testl $2048, %eax # imm = 0x800
+; AVX2-NEXT:    jne .LBB10_23
+; AVX2-NEXT:  .LBB10_24: # %else22
+; AVX2-NEXT:    testl $4096, %eax # imm = 0x1000
+; AVX2-NEXT:    jne .LBB10_25
+; AVX2-NEXT:  .LBB10_26: # %else24
+; AVX2-NEXT:    testl $8192, %eax # imm = 0x2000
+; AVX2-NEXT:    jne .LBB10_27
+; AVX2-NEXT:  .LBB10_28: # %else26
+; AVX2-NEXT:    testl $16384, %eax # imm = 0x4000
+; AVX2-NEXT:    jne .LBB10_29
+; AVX2-NEXT:  .LBB10_30: # %else28
+; AVX2-NEXT:    testl $32768, %eax # imm = 0x8000
+; AVX2-NEXT:    jne .LBB10_31
+; AVX2-NEXT:  .LBB10_32: # %else30
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+; AVX2-NEXT:  .LBB10_1: # %cond.store
 ; AVX2-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX2-NEXT:  .LBB10_2: # %else
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm2, %xmm1
-; AVX2-NEXT:    vpxor %xmm5, %xmm1, %xmm1
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm1, %xmm1
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm1, %xmm1
-; AVX2-NEXT:    vpextrb $1, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $2, %al
 ; AVX2-NEXT:    je .LBB10_4
-; AVX2-NEXT:  # %bb.3: # %cond.store1
+; AVX2-NEXT:  .LBB10_3: # %cond.store1
 ; AVX2-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
-; AVX2-NEXT:  .LBB10_4: # %else2
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm2, %xmm5
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm5, %xmm5
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm5, %xmm5
-; AVX2-NEXT:    vpextrb $2, %xmm5, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $4, %al
 ; AVX2-NEXT:    je .LBB10_6
-; AVX2-NEXT:  # %bb.5: # %cond.store3
+; AVX2-NEXT:  .LBB10_5: # %cond.store3
 ; AVX2-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
-; AVX2-NEXT:  .LBB10_6: # %else4
-; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm2, %xmm1
-; AVX2-NEXT:    vpxor %xmm4, %xmm1, %xmm1
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm1, %xmm1
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm1, %xmm1
-; AVX2-NEXT:    vpextrb $3, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $8, %al
 ; AVX2-NEXT:    je .LBB10_8
-; AVX2-NEXT:  # %bb.7: # %cond.store5
+; AVX2-NEXT:  .LBB10_7: # %cond.store5
 ; AVX2-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
-; AVX2-NEXT:  .LBB10_8: # %else6
-; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm1
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpxor %xmm4, %xmm2, %xmm2
-; AVX2-NEXT:    vpackssdw %xmm2, %xmm0, %xmm4
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm4, %xmm4
-; AVX2-NEXT:    vpextrb $4, %xmm4, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $16, %al
 ; AVX2-NEXT:    je .LBB10_10
-; AVX2-NEXT:  # %bb.9: # %cond.store7
+; AVX2-NEXT:  .LBB10_9: # %cond.store7
 ; AVX2-NEXT:    vpextrb $4, %xmm0, 4(%rdi)
-; AVX2-NEXT:  .LBB10_10: # %else8
-; AVX2-NEXT:    vpackssdw %xmm2, %xmm0, %xmm2
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrb $5, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $32, %al
 ; AVX2-NEXT:    je .LBB10_12
-; AVX2-NEXT:  # %bb.11: # %cond.store9
+; AVX2-NEXT:  .LBB10_11: # %cond.store9
 ; AVX2-NEXT:    vpextrb $5, %xmm0, 5(%rdi)
-; AVX2-NEXT:  .LBB10_12: # %else10
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpackssdw %xmm1, %xmm0, %xmm2
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrb $6, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $64, %al
 ; AVX2-NEXT:    je .LBB10_14
-; AVX2-NEXT:  # %bb.13: # %cond.store11
+; AVX2-NEXT:  .LBB10_13: # %cond.store11
 ; AVX2-NEXT:    vpextrb $6, %xmm0, 6(%rdi)
-; AVX2-NEXT:  .LBB10_14: # %else12
-; AVX2-NEXT:    vpackssdw %xmm1, %xmm0, %xmm1
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm1, %xmm1
-; AVX2-NEXT:    vpextrb $7, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $-128, %al
 ; AVX2-NEXT:    je .LBB10_16
-; AVX2-NEXT:  # %bb.15: # %cond.store13
+; AVX2-NEXT:  .LBB10_15: # %cond.store13
 ; AVX2-NEXT:    vpextrb $7, %xmm0, 7(%rdi)
-; AVX2-NEXT:  .LBB10_16: # %else14
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm3, %xmm4
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpxor %xmm2, %xmm4, %xmm4
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm4, %xmm4
-; AVX2-NEXT:    vpacksswb %xmm4, %xmm0, %xmm4
-; AVX2-NEXT:    vpextrb $8, %xmm4, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $256, %eax # imm = 0x100
 ; AVX2-NEXT:    je .LBB10_18
-; AVX2-NEXT:  # %bb.17: # %cond.store15
+; AVX2-NEXT:  .LBB10_17: # %cond.store15
 ; AVX2-NEXT:    vpextrb $8, %xmm0, 8(%rdi)
-; AVX2-NEXT:  .LBB10_18: # %else16
-; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm3, %xmm1
-; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm1, %xmm1
-; AVX2-NEXT:    vpacksswb %xmm1, %xmm0, %xmm1
-; AVX2-NEXT:    vpextrb $9, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $512, %eax # imm = 0x200
 ; AVX2-NEXT:    je .LBB10_20
-; AVX2-NEXT:  # %bb.19: # %cond.store17
+; AVX2-NEXT:  .LBB10_19: # %cond.store17
 ; AVX2-NEXT:    vpextrb $9, %xmm0, 9(%rdi)
-; AVX2-NEXT:  .LBB10_20: # %else18
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm3, %xmm4
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpxor %xmm2, %xmm4, %xmm4
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm4, %xmm4
-; AVX2-NEXT:    vpacksswb %xmm4, %xmm0, %xmm4
-; AVX2-NEXT:    vpextrb $10, %xmm4, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $1024, %eax # imm = 0x400
 ; AVX2-NEXT:    je .LBB10_22
-; AVX2-NEXT:  # %bb.21: # %cond.store19
+; AVX2-NEXT:  .LBB10_21: # %cond.store19
 ; AVX2-NEXT:    vpextrb $10, %xmm0, 10(%rdi)
-; AVX2-NEXT:  .LBB10_22: # %else20
-; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm3, %xmm1
-; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm1, %xmm1
-; AVX2-NEXT:    vpacksswb %xmm1, %xmm0, %xmm1
-; AVX2-NEXT:    vpextrb $11, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $2048, %eax # imm = 0x800
 ; AVX2-NEXT:    je .LBB10_24
-; AVX2-NEXT:  # %bb.23: # %cond.store21
+; AVX2-NEXT:  .LBB10_23: # %cond.store21
 ; AVX2-NEXT:    vpextrb $11, %xmm0, 11(%rdi)
-; AVX2-NEXT:  .LBB10_24: # %else22
-; AVX2-NEXT:    vextracti128 $1, %ymm3, %xmm1
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX2-NEXT:    vpackssdw %xmm2, %xmm0, %xmm3
-; AVX2-NEXT:    vpacksswb %xmm3, %xmm0, %xmm3
-; AVX2-NEXT:    vpextrb $12, %xmm3, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $4096, %eax # imm = 0x1000
 ; AVX2-NEXT:    je .LBB10_26
-; AVX2-NEXT:  # %bb.25: # %cond.store23
+; AVX2-NEXT:  .LBB10_25: # %cond.store23
 ; AVX2-NEXT:    vpextrb $12, %xmm0, 12(%rdi)
-; AVX2-NEXT:  .LBB10_26: # %else24
-; AVX2-NEXT:    vpackssdw %xmm2, %xmm0, %xmm2
-; AVX2-NEXT:    vpacksswb %xmm2, %xmm0, %xmm2
-; AVX2-NEXT:    vpextrb $13, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $8192, %eax # imm = 0x2000
 ; AVX2-NEXT:    je .LBB10_28
-; AVX2-NEXT:  # %bb.27: # %cond.store25
+; AVX2-NEXT:  .LBB10_27: # %cond.store25
 ; AVX2-NEXT:    vpextrb $13, %xmm0, 13(%rdi)
-; AVX2-NEXT:  .LBB10_28: # %else26
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpackssdw %xmm1, %xmm0, %xmm2
-; AVX2-NEXT:    vpacksswb %xmm2, %xmm0, %xmm2
-; AVX2-NEXT:    vpextrb $14, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $16384, %eax # imm = 0x4000
 ; AVX2-NEXT:    je .LBB10_30
-; AVX2-NEXT:  # %bb.29: # %cond.store27
+; AVX2-NEXT:  .LBB10_29: # %cond.store27
 ; AVX2-NEXT:    vpextrb $14, %xmm0, 14(%rdi)
-; AVX2-NEXT:  .LBB10_30: # %else28
-; AVX2-NEXT:    vpackssdw %xmm1, %xmm0, %xmm1
-; AVX2-NEXT:    vpacksswb %xmm1, %xmm0, %xmm1
-; AVX2-NEXT:    vpextrb $15, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $32768, %eax # imm = 0x8000
 ; AVX2-NEXT:    je .LBB10_32
-; AVX2-NEXT:  # %bb.31: # %cond.store29
+; AVX2-NEXT:  .LBB10_31: # %cond.store29
 ; AVX2-NEXT:    vpextrb $15, %xmm0, 15(%rdi)
-; AVX2-NEXT:  .LBB10_32: # %else30
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
@@ -4409,115 +4127,117 @@ define void @truncstore_v16i32_v16i8(<16 x i32> %x, <16 x i8>* %p, <16 x i32> %m
 ; AVX512F-NEXT:    vpmovsdb %zmm0, %xmm0
 ; AVX512F-NEXT:    kmovw %k0, %eax
 ; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB10_2
-; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    jne .LBB10_1
+; AVX512F-NEXT:  # %bb.2: # %else
+; AVX512F-NEXT:    testb $2, %al
+; AVX512F-NEXT:    jne .LBB10_3
+; AVX512F-NEXT:  .LBB10_4: # %else2
+; AVX512F-NEXT:    testb $4, %al
+; AVX512F-NEXT:    jne .LBB10_5
+; AVX512F-NEXT:  .LBB10_6: # %else4
+; AVX512F-NEXT:    testb $8, %al
+; AVX512F-NEXT:    jne .LBB10_7
+; AVX512F-NEXT:  .LBB10_8: # %else6
+; AVX512F-NEXT:    testb $16, %al
+; AVX512F-NEXT:    jne .LBB10_9
+; AVX512F-NEXT:  .LBB10_10: # %else8
+; AVX512F-NEXT:    testb $32, %al
+; AVX512F-NEXT:    jne .LBB10_11
+; AVX512F-NEXT:  .LBB10_12: # %else10
+; AVX512F-NEXT:    testb $64, %al
+; AVX512F-NEXT:    jne .LBB10_13
+; AVX512F-NEXT:  .LBB10_14: # %else12
+; AVX512F-NEXT:    testb $-128, %al
+; AVX512F-NEXT:    jne .LBB10_15
+; AVX512F-NEXT:  .LBB10_16: # %else14
+; AVX512F-NEXT:    testl $256, %eax # imm = 0x100
+; AVX512F-NEXT:    jne .LBB10_17
+; AVX512F-NEXT:  .LBB10_18: # %else16
+; AVX512F-NEXT:    testl $512, %eax # imm = 0x200
+; AVX512F-NEXT:    jne .LBB10_19
+; AVX512F-NEXT:  .LBB10_20: # %else18
+; AVX512F-NEXT:    testl $1024, %eax # imm = 0x400
+; AVX512F-NEXT:    jne .LBB10_21
+; AVX512F-NEXT:  .LBB10_22: # %else20
+; AVX512F-NEXT:    testl $2048, %eax # imm = 0x800
+; AVX512F-NEXT:    jne .LBB10_23
+; AVX512F-NEXT:  .LBB10_24: # %else22
+; AVX512F-NEXT:    testl $4096, %eax # imm = 0x1000
+; AVX512F-NEXT:    jne .LBB10_25
+; AVX512F-NEXT:  .LBB10_26: # %else24
+; AVX512F-NEXT:    testl $8192, %eax # imm = 0x2000
+; AVX512F-NEXT:    jne .LBB10_27
+; AVX512F-NEXT:  .LBB10_28: # %else26
+; AVX512F-NEXT:    testl $16384, %eax # imm = 0x4000
+; AVX512F-NEXT:    jne .LBB10_29
+; AVX512F-NEXT:  .LBB10_30: # %else28
+; AVX512F-NEXT:    testl $32768, %eax # imm = 0x8000
+; AVX512F-NEXT:    jne .LBB10_31
+; AVX512F-NEXT:  .LBB10_32: # %else30
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+; AVX512F-NEXT:  .LBB10_1: # %cond.store
 ; AVX512F-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX512F-NEXT:  .LBB10_2: # %else
-; AVX512F-NEXT:    kshiftrw $1, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $2, %al
 ; AVX512F-NEXT:    je .LBB10_4
-; AVX512F-NEXT:  # %bb.3: # %cond.store1
+; AVX512F-NEXT:  .LBB10_3: # %cond.store1
 ; AVX512F-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
-; AVX512F-NEXT:  .LBB10_4: # %else2
-; AVX512F-NEXT:    kshiftrw $2, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $4, %al
 ; AVX512F-NEXT:    je .LBB10_6
-; AVX512F-NEXT:  # %bb.5: # %cond.store3
+; AVX512F-NEXT:  .LBB10_5: # %cond.store3
 ; AVX512F-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
-; AVX512F-NEXT:  .LBB10_6: # %else4
-; AVX512F-NEXT:    kshiftrw $3, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $8, %al
 ; AVX512F-NEXT:    je .LBB10_8
-; AVX512F-NEXT:  # %bb.7: # %cond.store5
+; AVX512F-NEXT:  .LBB10_7: # %cond.store5
 ; AVX512F-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
-; AVX512F-NEXT:  .LBB10_8: # %else6
-; AVX512F-NEXT:    kshiftrw $4, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $16, %al
 ; AVX512F-NEXT:    je .LBB10_10
-; AVX512F-NEXT:  # %bb.9: # %cond.store7
+; AVX512F-NEXT:  .LBB10_9: # %cond.store7
 ; AVX512F-NEXT:    vpextrb $4, %xmm0, 4(%rdi)
-; AVX512F-NEXT:  .LBB10_10: # %else8
-; AVX512F-NEXT:    kshiftrw $5, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $32, %al
 ; AVX512F-NEXT:    je .LBB10_12
-; AVX512F-NEXT:  # %bb.11: # %cond.store9
+; AVX512F-NEXT:  .LBB10_11: # %cond.store9
 ; AVX512F-NEXT:    vpextrb $5, %xmm0, 5(%rdi)
-; AVX512F-NEXT:  .LBB10_12: # %else10
-; AVX512F-NEXT:    kshiftrw $6, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $64, %al
 ; AVX512F-NEXT:    je .LBB10_14
-; AVX512F-NEXT:  # %bb.13: # %cond.store11
+; AVX512F-NEXT:  .LBB10_13: # %cond.store11
 ; AVX512F-NEXT:    vpextrb $6, %xmm0, 6(%rdi)
-; AVX512F-NEXT:  .LBB10_14: # %else12
-; AVX512F-NEXT:    kshiftrw $7, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $-128, %al
 ; AVX512F-NEXT:    je .LBB10_16
-; AVX512F-NEXT:  # %bb.15: # %cond.store13
+; AVX512F-NEXT:  .LBB10_15: # %cond.store13
 ; AVX512F-NEXT:    vpextrb $7, %xmm0, 7(%rdi)
-; AVX512F-NEXT:  .LBB10_16: # %else14
-; AVX512F-NEXT:    kshiftrw $8, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $256, %eax # imm = 0x100
 ; AVX512F-NEXT:    je .LBB10_18
-; AVX512F-NEXT:  # %bb.17: # %cond.store15
+; AVX512F-NEXT:  .LBB10_17: # %cond.store15
 ; AVX512F-NEXT:    vpextrb $8, %xmm0, 8(%rdi)
-; AVX512F-NEXT:  .LBB10_18: # %else16
-; AVX512F-NEXT:    kshiftrw $9, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $512, %eax # imm = 0x200
 ; AVX512F-NEXT:    je .LBB10_20
-; AVX512F-NEXT:  # %bb.19: # %cond.store17
+; AVX512F-NEXT:  .LBB10_19: # %cond.store17
 ; AVX512F-NEXT:    vpextrb $9, %xmm0, 9(%rdi)
-; AVX512F-NEXT:  .LBB10_20: # %else18
-; AVX512F-NEXT:    kshiftrw $10, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $1024, %eax # imm = 0x400
 ; AVX512F-NEXT:    je .LBB10_22
-; AVX512F-NEXT:  # %bb.21: # %cond.store19
+; AVX512F-NEXT:  .LBB10_21: # %cond.store19
 ; AVX512F-NEXT:    vpextrb $10, %xmm0, 10(%rdi)
-; AVX512F-NEXT:  .LBB10_22: # %else20
-; AVX512F-NEXT:    kshiftrw $11, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $2048, %eax # imm = 0x800
 ; AVX512F-NEXT:    je .LBB10_24
-; AVX512F-NEXT:  # %bb.23: # %cond.store21
+; AVX512F-NEXT:  .LBB10_23: # %cond.store21
 ; AVX512F-NEXT:    vpextrb $11, %xmm0, 11(%rdi)
-; AVX512F-NEXT:  .LBB10_24: # %else22
-; AVX512F-NEXT:    kshiftrw $12, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $4096, %eax # imm = 0x1000
 ; AVX512F-NEXT:    je .LBB10_26
-; AVX512F-NEXT:  # %bb.25: # %cond.store23
+; AVX512F-NEXT:  .LBB10_25: # %cond.store23
 ; AVX512F-NEXT:    vpextrb $12, %xmm0, 12(%rdi)
-; AVX512F-NEXT:  .LBB10_26: # %else24
-; AVX512F-NEXT:    kshiftrw $13, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $8192, %eax # imm = 0x2000
 ; AVX512F-NEXT:    je .LBB10_28
-; AVX512F-NEXT:  # %bb.27: # %cond.store25
+; AVX512F-NEXT:  .LBB10_27: # %cond.store25
 ; AVX512F-NEXT:    vpextrb $13, %xmm0, 13(%rdi)
-; AVX512F-NEXT:  .LBB10_28: # %else26
-; AVX512F-NEXT:    kshiftrw $14, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $16384, %eax # imm = 0x4000
 ; AVX512F-NEXT:    je .LBB10_30
-; AVX512F-NEXT:  # %bb.29: # %cond.store27
+; AVX512F-NEXT:  .LBB10_29: # %cond.store27
 ; AVX512F-NEXT:    vpextrb $14, %xmm0, 14(%rdi)
-; AVX512F-NEXT:  .LBB10_30: # %else28
-; AVX512F-NEXT:    kshiftrw $15, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $32768, %eax # imm = 0x8000
 ; AVX512F-NEXT:    je .LBB10_32
-; AVX512F-NEXT:  # %bb.31: # %cond.store29
+; AVX512F-NEXT:  .LBB10_31: # %cond.store29
 ; AVX512F-NEXT:    vpextrb $15, %xmm0, 15(%rdi)
-; AVX512F-NEXT:  .LBB10_32: # %else30
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
@@ -4551,303 +4271,284 @@ define void @truncstore_v16i32_v16i8(<16 x i32> %x, <16 x i8>* %p, <16 x i32> %m
 define void @truncstore_v8i32_v8i16(<8 x i32> %x, <8 x i16>* %p, <8 x i32> %mask) {
 ; SSE2-LABEL: truncstore_v8i32_v8i16:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pxor %xmm5, %xmm5
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm5
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm4
-; SSE2-NEXT:    pxor %xmm5, %xmm4
-; SSE2-NEXT:    movdqa %xmm4, %xmm5
-; SSE2-NEXT:    packssdw %xmm0, %xmm5
+; SSE2-NEXT:    pxor %xmm4, %xmm4
 ; SSE2-NEXT:    packssdw %xmm1, %xmm0
-; SSE2-NEXT:    movd %xmm5, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB11_2
-; SSE2-NEXT:  # %bb.1: # %cond.store
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    movw %ax, (%rdi)
-; SSE2-NEXT:  .LBB11_2: # %else
-; SSE2-NEXT:    psrlq $16, %xmm4
-; SSE2-NEXT:    movd %xmm4, %eax
-; SSE2-NEXT:    shrl $16, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB11_4
-; SSE2-NEXT:  # %bb.3: # %cond.store1
-; SSE2-NEXT:    pextrw $1, %xmm0, %eax
-; SSE2-NEXT:    movw %ax, 2(%rdi)
-; SSE2-NEXT:  .LBB11_4: # %else2
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm2
+; SSE2-NEXT:    pcmpeqd %xmm4, %xmm3
 ; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pxor %xmm2, %xmm1
-; SSE2-NEXT:    pextrw $4, %xmm1, %eax
+; SSE2-NEXT:    pxor %xmm1, %xmm3
+; SSE2-NEXT:    pcmpeqd %xmm4, %xmm2
+; SSE2-NEXT:    pxor %xmm1, %xmm2
+; SSE2-NEXT:    packssdw %xmm3, %xmm2
+; SSE2-NEXT:    packsswb %xmm0, %xmm2
+; SSE2-NEXT:    pmovmskb %xmm2, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB11_6
-; SSE2-NEXT:  # %bb.5: # %cond.store3
-; SSE2-NEXT:    pextrw $2, %xmm0, %eax
-; SSE2-NEXT:    movw %ax, 4(%rdi)
+; SSE2-NEXT:    jne .LBB11_1
+; SSE2-NEXT:  # %bb.2: # %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne .LBB11_3
+; SSE2-NEXT:  .LBB11_4: # %else2
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne .LBB11_5
 ; SSE2-NEXT:  .LBB11_6: # %else4
-; SSE2-NEXT:    pextrw $6, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB11_8
-; SSE2-NEXT:  # %bb.7: # %cond.store5
-; SSE2-NEXT:    pextrw $3, %xmm0, %eax
-; SSE2-NEXT:    movw %ax, 6(%rdi)
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne .LBB11_7
 ; SSE2-NEXT:  .LBB11_8: # %else6
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pxor %xmm2, %xmm1
-; SSE2-NEXT:    pextrw $0, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB11_10
-; SSE2-NEXT:  # %bb.9: # %cond.store7
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    movw %ax, 8(%rdi)
+; SSE2-NEXT:    testb $16, %al
+; SSE2-NEXT:    jne .LBB11_9
 ; SSE2-NEXT:  .LBB11_10: # %else8
-; SSE2-NEXT:    pextrw $2, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB11_12
-; SSE2-NEXT:  # %bb.11: # %cond.store9
-; SSE2-NEXT:    pextrw $5, %xmm0, %eax
-; SSE2-NEXT:    movw %ax, 10(%rdi)
+; SSE2-NEXT:    testb $32, %al
+; SSE2-NEXT:    jne .LBB11_11
 ; SSE2-NEXT:  .LBB11_12: # %else10
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm3
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pxor %xmm3, %xmm1
-; SSE2-NEXT:    pextrw $4, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB11_14
-; SSE2-NEXT:  # %bb.13: # %cond.store11
-; SSE2-NEXT:    pextrw $6, %xmm0, %eax
-; SSE2-NEXT:    movw %ax, 12(%rdi)
+; SSE2-NEXT:    testb $64, %al
+; SSE2-NEXT:    jne .LBB11_13
 ; SSE2-NEXT:  .LBB11_14: # %else12
-; SSE2-NEXT:    pextrw $6, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $-128, %al
+; SSE2-NEXT:    jne .LBB11_15
+; SSE2-NEXT:  .LBB11_16: # %else14
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  .LBB11_1: # %cond.store
+; SSE2-NEXT:    movd %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, (%rdi)
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    je .LBB11_4
+; SSE2-NEXT:  .LBB11_3: # %cond.store1
+; SSE2-NEXT:    pextrw $1, %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, 2(%rdi)
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    je .LBB11_6
+; SSE2-NEXT:  .LBB11_5: # %cond.store3
+; SSE2-NEXT:    pextrw $2, %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, 4(%rdi)
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    je .LBB11_8
+; SSE2-NEXT:  .LBB11_7: # %cond.store5
+; SSE2-NEXT:    pextrw $3, %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, 6(%rdi)
+; SSE2-NEXT:    testb $16, %al
+; SSE2-NEXT:    je .LBB11_10
+; SSE2-NEXT:  .LBB11_9: # %cond.store7
+; SSE2-NEXT:    pextrw $4, %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, 8(%rdi)
+; SSE2-NEXT:    testb $32, %al
+; SSE2-NEXT:    je .LBB11_12
+; SSE2-NEXT:  .LBB11_11: # %cond.store9
+; SSE2-NEXT:    pextrw $5, %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, 10(%rdi)
+; SSE2-NEXT:    testb $64, %al
+; SSE2-NEXT:    je .LBB11_14
+; SSE2-NEXT:  .LBB11_13: # %cond.store11
+; SSE2-NEXT:    pextrw $6, %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, 12(%rdi)
+; SSE2-NEXT:    testb $-128, %al
 ; SSE2-NEXT:    je .LBB11_16
-; SSE2-NEXT:  # %bb.15: # %cond.store13
+; SSE2-NEXT:  .LBB11_15: # %cond.store13
 ; SSE2-NEXT:    pextrw $7, %xmm0, %eax
 ; SSE2-NEXT:    movw %ax, 14(%rdi)
-; SSE2-NEXT:  .LBB11_16: # %else14
 ; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: truncstore_v8i32_v8i16:
 ; SSE4:       # %bb.0:
-; SSE4-NEXT:    pxor %xmm5, %xmm5
-; SSE4-NEXT:    pcmpeqd %xmm2, %xmm5
-; SSE4-NEXT:    pcmpeqd %xmm4, %xmm4
-; SSE4-NEXT:    pxor %xmm5, %xmm4
+; SSE4-NEXT:    pxor %xmm4, %xmm4
 ; SSE4-NEXT:    packssdw %xmm1, %xmm0
-; SSE4-NEXT:    pextrb $0, %xmm4, %eax
+; SSE4-NEXT:    pcmpeqd %xmm4, %xmm3
+; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
+; SSE4-NEXT:    pxor %xmm1, %xmm3
+; SSE4-NEXT:    pcmpeqd %xmm4, %xmm2
+; SSE4-NEXT:    pxor %xmm1, %xmm2
+; SSE4-NEXT:    packssdw %xmm3, %xmm2
+; SSE4-NEXT:    packsswb %xmm0, %xmm2
+; SSE4-NEXT:    pmovmskb %xmm2, %eax
 ; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB11_2
-; SSE4-NEXT:  # %bb.1: # %cond.store
+; SSE4-NEXT:    jne .LBB11_1
+; SSE4-NEXT:  # %bb.2: # %else
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    jne .LBB11_3
+; SSE4-NEXT:  .LBB11_4: # %else2
+; SSE4-NEXT:    testb $4, %al
+; SSE4-NEXT:    jne .LBB11_5
+; SSE4-NEXT:  .LBB11_6: # %else4
+; SSE4-NEXT:    testb $8, %al
+; SSE4-NEXT:    jne .LBB11_7
+; SSE4-NEXT:  .LBB11_8: # %else6
+; SSE4-NEXT:    testb $16, %al
+; SSE4-NEXT:    jne .LBB11_9
+; SSE4-NEXT:  .LBB11_10: # %else8
+; SSE4-NEXT:    testb $32, %al
+; SSE4-NEXT:    jne .LBB11_11
+; SSE4-NEXT:  .LBB11_12: # %else10
+; SSE4-NEXT:    testb $64, %al
+; SSE4-NEXT:    jne .LBB11_13
+; SSE4-NEXT:  .LBB11_14: # %else12
+; SSE4-NEXT:    testb $-128, %al
+; SSE4-NEXT:    jne .LBB11_15
+; SSE4-NEXT:  .LBB11_16: # %else14
+; SSE4-NEXT:    retq
+; SSE4-NEXT:  .LBB11_1: # %cond.store
 ; SSE4-NEXT:    pextrw $0, %xmm0, (%rdi)
-; SSE4-NEXT:  .LBB11_2: # %else
-; SSE4-NEXT:    pextrb $4, %xmm4, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $2, %al
 ; SSE4-NEXT:    je .LBB11_4
-; SSE4-NEXT:  # %bb.3: # %cond.store1
+; SSE4-NEXT:  .LBB11_3: # %cond.store1
 ; SSE4-NEXT:    pextrw $1, %xmm0, 2(%rdi)
-; SSE4-NEXT:  .LBB11_4: # %else2
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm2
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE4-NEXT:    pxor %xmm2, %xmm1
-; SSE4-NEXT:    pextrb $8, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $4, %al
 ; SSE4-NEXT:    je .LBB11_6
-; SSE4-NEXT:  # %bb.5: # %cond.store3
+; SSE4-NEXT:  .LBB11_5: # %cond.store3
 ; SSE4-NEXT:    pextrw $2, %xmm0, 4(%rdi)
-; SSE4-NEXT:  .LBB11_6: # %else4
-; SSE4-NEXT:    pextrb $12, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $8, %al
 ; SSE4-NEXT:    je .LBB11_8
-; SSE4-NEXT:  # %bb.7: # %cond.store5
+; SSE4-NEXT:  .LBB11_7: # %cond.store5
 ; SSE4-NEXT:    pextrw $3, %xmm0, 6(%rdi)
-; SSE4-NEXT:  .LBB11_8: # %else6
-; SSE4-NEXT:    pxor %xmm2, %xmm2
-; SSE4-NEXT:    pcmpeqd %xmm3, %xmm2
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE4-NEXT:    pxor %xmm2, %xmm1
-; SSE4-NEXT:    pextrb $0, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $16, %al
 ; SSE4-NEXT:    je .LBB11_10
-; SSE4-NEXT:  # %bb.9: # %cond.store7
+; SSE4-NEXT:  .LBB11_9: # %cond.store7
 ; SSE4-NEXT:    pextrw $4, %xmm0, 8(%rdi)
-; SSE4-NEXT:  .LBB11_10: # %else8
-; SSE4-NEXT:    pextrb $4, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $32, %al
 ; SSE4-NEXT:    je .LBB11_12
-; SSE4-NEXT:  # %bb.11: # %cond.store9
+; SSE4-NEXT:  .LBB11_11: # %cond.store9
 ; SSE4-NEXT:    pextrw $5, %xmm0, 10(%rdi)
-; SSE4-NEXT:  .LBB11_12: # %else10
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm3
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE4-NEXT:    pxor %xmm3, %xmm1
-; SSE4-NEXT:    pextrb $8, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $64, %al
 ; SSE4-NEXT:    je .LBB11_14
-; SSE4-NEXT:  # %bb.13: # %cond.store11
+; SSE4-NEXT:  .LBB11_13: # %cond.store11
 ; SSE4-NEXT:    pextrw $6, %xmm0, 12(%rdi)
-; SSE4-NEXT:  .LBB11_14: # %else12
-; SSE4-NEXT:    pextrb $12, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $-128, %al
 ; SSE4-NEXT:    je .LBB11_16
-; SSE4-NEXT:  # %bb.15: # %cond.store13
+; SSE4-NEXT:  .LBB11_15: # %cond.store13
 ; SSE4-NEXT:    pextrw $7, %xmm0, 14(%rdi)
-; SSE4-NEXT:  .LBB11_16: # %else14
 ; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: truncstore_v8i32_v8i16:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm4, %xmm4
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm5
-; AVX1-NEXT:    vpackssdw %xmm5, %xmm0, %xmm0
-; AVX1-NEXT:    vpextrb $0, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je .LBB11_2
-; AVX1-NEXT:  # %bb.1: # %cond.store
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT:    vpackssdw %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; AVX1-NEXT:    vmovmskps %ymm1, %eax
+; AVX1-NEXT:    notl %eax
+; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    jne .LBB11_1
+; AVX1-NEXT:  # %bb.2: # %else
+; AVX1-NEXT:    testb $2, %al
+; AVX1-NEXT:    jne .LBB11_3
+; AVX1-NEXT:  .LBB11_4: # %else2
+; AVX1-NEXT:    testb $4, %al
+; AVX1-NEXT:    jne .LBB11_5
+; AVX1-NEXT:  .LBB11_6: # %else4
+; AVX1-NEXT:    testb $8, %al
+; AVX1-NEXT:    jne .LBB11_7
+; AVX1-NEXT:  .LBB11_8: # %else6
+; AVX1-NEXT:    testb $16, %al
+; AVX1-NEXT:    jne .LBB11_9
+; AVX1-NEXT:  .LBB11_10: # %else8
+; AVX1-NEXT:    testb $32, %al
+; AVX1-NEXT:    jne .LBB11_11
+; AVX1-NEXT:  .LBB11_12: # %else10
+; AVX1-NEXT:    testb $64, %al
+; AVX1-NEXT:    jne .LBB11_13
+; AVX1-NEXT:  .LBB11_14: # %else12
+; AVX1-NEXT:    testb $-128, %al
+; AVX1-NEXT:    jne .LBB11_15
+; AVX1-NEXT:  .LBB11_16: # %else14
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+; AVX1-NEXT:  .LBB11_1: # %cond.store
 ; AVX1-NEXT:    vpextrw $0, %xmm0, (%rdi)
-; AVX1-NEXT:  .LBB11_2: # %else
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $2, %al
 ; AVX1-NEXT:    je .LBB11_4
-; AVX1-NEXT:  # %bb.3: # %cond.store1
+; AVX1-NEXT:  .LBB11_3: # %cond.store1
 ; AVX1-NEXT:    vpextrw $1, %xmm0, 2(%rdi)
-; AVX1-NEXT:  .LBB11_4: # %else2
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm4, %xmm4
-; AVX1-NEXT:    vpextrb $8, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $4, %al
 ; AVX1-NEXT:    je .LBB11_6
-; AVX1-NEXT:  # %bb.5: # %cond.store3
+; AVX1-NEXT:  .LBB11_5: # %cond.store3
 ; AVX1-NEXT:    vpextrw $2, %xmm0, 4(%rdi)
-; AVX1-NEXT:  .LBB11_6: # %else4
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $12, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $8, %al
 ; AVX1-NEXT:    je .LBB11_8
-; AVX1-NEXT:  # %bb.7: # %cond.store5
+; AVX1-NEXT:  .LBB11_7: # %cond.store5
 ; AVX1-NEXT:    vpextrw $3, %xmm0, 6(%rdi)
-; AVX1-NEXT:  .LBB11_8: # %else6
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm1
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $0, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $16, %al
 ; AVX1-NEXT:    je .LBB11_10
-; AVX1-NEXT:  # %bb.9: # %cond.store7
+; AVX1-NEXT:  .LBB11_9: # %cond.store7
 ; AVX1-NEXT:    vpextrw $4, %xmm0, 8(%rdi)
-; AVX1-NEXT:  .LBB11_10: # %else8
-; AVX1-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $32, %al
 ; AVX1-NEXT:    je .LBB11_12
-; AVX1-NEXT:  # %bb.11: # %cond.store9
+; AVX1-NEXT:  .LBB11_11: # %cond.store9
 ; AVX1-NEXT:    vpextrw $5, %xmm0, 10(%rdi)
-; AVX1-NEXT:  .LBB11_12: # %else10
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $64, %al
 ; AVX1-NEXT:    je .LBB11_14
-; AVX1-NEXT:  # %bb.13: # %cond.store11
+; AVX1-NEXT:  .LBB11_13: # %cond.store11
 ; AVX1-NEXT:    vpextrw $6, %xmm0, 12(%rdi)
-; AVX1-NEXT:  .LBB11_14: # %else12
-; AVX1-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $-128, %al
 ; AVX1-NEXT:    je .LBB11_16
-; AVX1-NEXT:  # %bb.15: # %cond.store13
+; AVX1-NEXT:  .LBB11_15: # %cond.store13
 ; AVX1-NEXT:    vpextrw $7, %xmm0, 14(%rdi)
-; AVX1-NEXT:  .LBB11_16: # %else14
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: truncstore_v8i32_v8i16:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm4
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    vpxor %xmm3, %xmm4, %xmm4
-; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm5
-; AVX2-NEXT:    vpackssdw %xmm5, %xmm0, %xmm0
-; AVX2-NEXT:    vpextrb $0, %xmm4, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je .LBB11_2
-; AVX2-NEXT:  # %bb.1: # %cond.store
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; AVX2-NEXT:    vpackssdw %xmm3, %xmm0, %xmm0
+; AVX2-NEXT:    vpcmpeqd %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    vmovmskps %ymm1, %eax
+; AVX2-NEXT:    notl %eax
+; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    jne .LBB11_1
+; AVX2-NEXT:  # %bb.2: # %else
+; AVX2-NEXT:    testb $2, %al
+; AVX2-NEXT:    jne .LBB11_3
+; AVX2-NEXT:  .LBB11_4: # %else2
+; AVX2-NEXT:    testb $4, %al
+; AVX2-NEXT:    jne .LBB11_5
+; AVX2-NEXT:  .LBB11_6: # %else4
+; AVX2-NEXT:    testb $8, %al
+; AVX2-NEXT:    jne .LBB11_7
+; AVX2-NEXT:  .LBB11_8: # %else6
+; AVX2-NEXT:    testb $16, %al
+; AVX2-NEXT:    jne .LBB11_9
+; AVX2-NEXT:  .LBB11_10: # %else8
+; AVX2-NEXT:    testb $32, %al
+; AVX2-NEXT:    jne .LBB11_11
+; AVX2-NEXT:  .LBB11_12: # %else10
+; AVX2-NEXT:    testb $64, %al
+; AVX2-NEXT:    jne .LBB11_13
+; AVX2-NEXT:  .LBB11_14: # %else12
+; AVX2-NEXT:    testb $-128, %al
+; AVX2-NEXT:    jne .LBB11_15
+; AVX2-NEXT:  .LBB11_16: # %else14
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+; AVX2-NEXT:  .LBB11_1: # %cond.store
 ; AVX2-NEXT:    vpextrw $0, %xmm0, (%rdi)
-; AVX2-NEXT:  .LBB11_2: # %else
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $2, %al
 ; AVX2-NEXT:    je .LBB11_4
-; AVX2-NEXT:  # %bb.3: # %cond.store1
+; AVX2-NEXT:  .LBB11_3: # %cond.store1
 ; AVX2-NEXT:    vpextrw $1, %xmm0, 2(%rdi)
-; AVX2-NEXT:  .LBB11_4: # %else2
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm4
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    vpxor %xmm3, %xmm4, %xmm4
-; AVX2-NEXT:    vpextrb $8, %xmm4, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $4, %al
 ; AVX2-NEXT:    je .LBB11_6
-; AVX2-NEXT:  # %bb.5: # %cond.store3
+; AVX2-NEXT:  .LBB11_5: # %cond.store3
 ; AVX2-NEXT:    vpextrw $2, %xmm0, 4(%rdi)
-; AVX2-NEXT:  .LBB11_6: # %else4
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrb $12, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $8, %al
 ; AVX2-NEXT:    je .LBB11_8
-; AVX2-NEXT:  # %bb.7: # %cond.store5
+; AVX2-NEXT:  .LBB11_7: # %cond.store5
 ; AVX2-NEXT:    vpextrw $3, %xmm0, 6(%rdi)
-; AVX2-NEXT:  .LBB11_8: # %else6
-; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm1
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrb $0, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $16, %al
 ; AVX2-NEXT:    je .LBB11_10
-; AVX2-NEXT:  # %bb.9: # %cond.store7
+; AVX2-NEXT:  .LBB11_9: # %cond.store7
 ; AVX2-NEXT:    vpextrw $4, %xmm0, 8(%rdi)
-; AVX2-NEXT:  .LBB11_10: # %else8
-; AVX2-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $32, %al
 ; AVX2-NEXT:    je .LBB11_12
-; AVX2-NEXT:  # %bb.11: # %cond.store9
+; AVX2-NEXT:  .LBB11_11: # %cond.store9
 ; AVX2-NEXT:    vpextrw $5, %xmm0, 10(%rdi)
-; AVX2-NEXT:  .LBB11_12: # %else10
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $64, %al
 ; AVX2-NEXT:    je .LBB11_14
-; AVX2-NEXT:  # %bb.13: # %cond.store11
+; AVX2-NEXT:  .LBB11_13: # %cond.store11
 ; AVX2-NEXT:    vpextrw $6, %xmm0, 12(%rdi)
-; AVX2-NEXT:  .LBB11_14: # %else12
-; AVX2-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $-128, %al
 ; AVX2-NEXT:    je .LBB11_16
-; AVX2-NEXT:  # %bb.15: # %cond.store13
+; AVX2-NEXT:  .LBB11_15: # %cond.store13
 ; AVX2-NEXT:    vpextrw $7, %xmm0, 14(%rdi)
-; AVX2-NEXT:  .LBB11_16: # %else14
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
@@ -4855,73 +4556,65 @@ define void @truncstore_v8i32_v8i16(<8 x i32> %x, <8 x i16>* %p, <8 x i32> %mask
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
 ; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    vpbroadcastd {{.*#+}} ymm2 = [32767,32767,32767,32767,32767,32767,32767,32767]
-; AVX512F-NEXT:    vpminsd %ymm2, %ymm0, %ymm0
-; AVX512F-NEXT:    vpbroadcastd {{.*#+}} ymm2 = [4294934528,4294934528,4294934528,4294934528,4294934528,4294934528,4294934528,4294934528]
-; AVX512F-NEXT:    vpmaxsd %ymm2, %ymm0, %ymm0
-; AVX512F-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
 ; AVX512F-NEXT:    kmovw %k0, %eax
 ; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB11_2
-; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    jne .LBB11_1
+; AVX512F-NEXT:  # %bb.2: # %else
+; AVX512F-NEXT:    testb $2, %al
+; AVX512F-NEXT:    jne .LBB11_3
+; AVX512F-NEXT:  .LBB11_4: # %else2
+; AVX512F-NEXT:    testb $4, %al
+; AVX512F-NEXT:    jne .LBB11_5
+; AVX512F-NEXT:  .LBB11_6: # %else4
+; AVX512F-NEXT:    testb $8, %al
+; AVX512F-NEXT:    jne .LBB11_7
+; AVX512F-NEXT:  .LBB11_8: # %else6
+; AVX512F-NEXT:    testb $16, %al
+; AVX512F-NEXT:    jne .LBB11_9
+; AVX512F-NEXT:  .LBB11_10: # %else8
+; AVX512F-NEXT:    testb $32, %al
+; AVX512F-NEXT:    jne .LBB11_11
+; AVX512F-NEXT:  .LBB11_12: # %else10
+; AVX512F-NEXT:    testb $64, %al
+; AVX512F-NEXT:    jne .LBB11_13
+; AVX512F-NEXT:  .LBB11_14: # %else12
+; AVX512F-NEXT:    testb $-128, %al
+; AVX512F-NEXT:    jne .LBB11_15
+; AVX512F-NEXT:  .LBB11_16: # %else14
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+; AVX512F-NEXT:  .LBB11_1: # %cond.store
 ; AVX512F-NEXT:    vpextrw $0, %xmm0, (%rdi)
-; AVX512F-NEXT:  .LBB11_2: # %else
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $2, %al
 ; AVX512F-NEXT:    je .LBB11_4
-; AVX512F-NEXT:  # %bb.3: # %cond.store1
+; AVX512F-NEXT:  .LBB11_3: # %cond.store1
 ; AVX512F-NEXT:    vpextrw $1, %xmm0, 2(%rdi)
-; AVX512F-NEXT:  .LBB11_4: # %else2
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $4, %al
 ; AVX512F-NEXT:    je .LBB11_6
-; AVX512F-NEXT:  # %bb.5: # %cond.store3
+; AVX512F-NEXT:  .LBB11_5: # %cond.store3
 ; AVX512F-NEXT:    vpextrw $2, %xmm0, 4(%rdi)
-; AVX512F-NEXT:  .LBB11_6: # %else4
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $8, %al
 ; AVX512F-NEXT:    je .LBB11_8
-; AVX512F-NEXT:  # %bb.7: # %cond.store5
+; AVX512F-NEXT:  .LBB11_7: # %cond.store5
 ; AVX512F-NEXT:    vpextrw $3, %xmm0, 6(%rdi)
-; AVX512F-NEXT:  .LBB11_8: # %else6
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $4, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $16, %al
 ; AVX512F-NEXT:    je .LBB11_10
-; AVX512F-NEXT:  # %bb.9: # %cond.store7
+; AVX512F-NEXT:  .LBB11_9: # %cond.store7
 ; AVX512F-NEXT:    vpextrw $4, %xmm0, 8(%rdi)
-; AVX512F-NEXT:  .LBB11_10: # %else8
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $5, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $32, %al
 ; AVX512F-NEXT:    je .LBB11_12
-; AVX512F-NEXT:  # %bb.11: # %cond.store9
+; AVX512F-NEXT:  .LBB11_11: # %cond.store9
 ; AVX512F-NEXT:    vpextrw $5, %xmm0, 10(%rdi)
-; AVX512F-NEXT:  .LBB11_12: # %else10
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $6, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $64, %al
 ; AVX512F-NEXT:    je .LBB11_14
-; AVX512F-NEXT:  # %bb.13: # %cond.store11
+; AVX512F-NEXT:  .LBB11_13: # %cond.store11
 ; AVX512F-NEXT:    vpextrw $6, %xmm0, 12(%rdi)
-; AVX512F-NEXT:  .LBB11_14: # %else12
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $7, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $-128, %al
 ; AVX512F-NEXT:    je .LBB11_16
-; AVX512F-NEXT:  # %bb.15: # %cond.store13
+; AVX512F-NEXT:  .LBB11_15: # %cond.store13
 ; AVX512F-NEXT:    vpextrw $7, %xmm0, 14(%rdi)
-; AVX512F-NEXT:  .LBB11_16: # %else14
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
@@ -4931,11 +4624,8 @@ define void @truncstore_v8i32_v8i16(<8 x i32> %x, <8 x i16>* %p, <8 x i32> %mask
 ; AVX512BW-NEXT:    vptestmd %zmm1, %zmm1, %k0
 ; AVX512BW-NEXT:    kshiftld $24, %k0, %k0
 ; AVX512BW-NEXT:    kshiftrd $24, %k0, %k1
-; AVX512BW-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [32767,32767,32767,32767,32767,32767,32767,32767]
-; AVX512BW-NEXT:    vpminsd %ymm1, %ymm0, %ymm0
-; AVX512BW-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [4294934528,4294934528,4294934528,4294934528,4294934528,4294934528,4294934528,4294934528]
-; AVX512BW-NEXT:    vpmaxsd %ymm1, %ymm0, %ymm0
-; AVX512BW-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512BW-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vmovdqu16 %zmm0, (%rdi) {%k1}
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -4961,341 +4651,281 @@ define void @truncstore_v8i32_v8i16(<8 x i32> %x, <8 x i16>* %p, <8 x i32> %mask
 define void @truncstore_v8i32_v8i8(<8 x i32> %x, <8 x i8>* %p, <8 x i32> %mask) {
 ; SSE2-LABEL: truncstore_v8i32_v8i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pxor %xmm5, %xmm5
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm5
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm4
-; SSE2-NEXT:    pxor %xmm5, %xmm4
-; SSE2-NEXT:    movdqa %xmm4, %xmm5
-; SSE2-NEXT:    packssdw %xmm0, %xmm5
-; SSE2-NEXT:    movdqa {{.*#+}} xmm6 = [127,127,127,127]
-; SSE2-NEXT:    movdqa %xmm6, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm7
-; SSE2-NEXT:    pand %xmm7, %xmm0
-; SSE2-NEXT:    pandn %xmm6, %xmm7
-; SSE2-NEXT:    por %xmm0, %xmm7
-; SSE2-NEXT:    movdqa %xmm6, %xmm0
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pand %xmm0, %xmm1
-; SSE2-NEXT:    pandn %xmm6, %xmm0
-; SSE2-NEXT:    por %xmm1, %xmm0
-; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [4294967168,4294967168,4294967168,4294967168]
-; SSE2-NEXT:    movdqa %xmm0, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm6
-; SSE2-NEXT:    pand %xmm6, %xmm0
-; SSE2-NEXT:    pandn %xmm1, %xmm6
-; SSE2-NEXT:    por %xmm0, %xmm6
-; SSE2-NEXT:    movdqa %xmm7, %xmm0
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pand %xmm0, %xmm7
-; SSE2-NEXT:    pandn %xmm1, %xmm0
-; SSE2-NEXT:    por %xmm7, %xmm0
-; SSE2-NEXT:    packssdw %xmm6, %xmm0
-; SSE2-NEXT:    movd %xmm5, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    je .LBB12_2
-; SSE2-NEXT:  # %bb.1: # %cond.store
-; SSE2-NEXT:    movb %al, (%rdi)
-; SSE2-NEXT:  .LBB12_2: # %else
-; SSE2-NEXT:    psrlq $16, %xmm4
-; SSE2-NEXT:    movd %xmm4, %ecx
-; SSE2-NEXT:    shrl $16, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    je .LBB12_4
-; SSE2-NEXT:  # %bb.3: # %cond.store1
-; SSE2-NEXT:    shrl $16, %eax
-; SSE2-NEXT:    movb %al, 1(%rdi)
-; SSE2-NEXT:  .LBB12_4: # %else2
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm2
+; SSE2-NEXT:    pxor %xmm4, %xmm4
+; SSE2-NEXT:    packssdw %xmm1, %xmm0
+; SSE2-NEXT:    packsswb %xmm0, %xmm0
+; SSE2-NEXT:    pcmpeqd %xmm4, %xmm3
 ; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pxor %xmm2, %xmm1
-; SSE2-NEXT:    pextrw $4, %xmm1, %eax
+; SSE2-NEXT:    pxor %xmm1, %xmm3
+; SSE2-NEXT:    pcmpeqd %xmm4, %xmm2
+; SSE2-NEXT:    pxor %xmm1, %xmm2
+; SSE2-NEXT:    packssdw %xmm3, %xmm2
+; SSE2-NEXT:    packsswb %xmm0, %xmm2
+; SSE2-NEXT:    pmovmskb %xmm2, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB12_6
-; SSE2-NEXT:  # %bb.5: # %cond.store3
-; SSE2-NEXT:    pextrw $2, %xmm0, %eax
-; SSE2-NEXT:    movb %al, 2(%rdi)
+; SSE2-NEXT:    movd %xmm0, %ecx
+; SSE2-NEXT:    jne .LBB12_1
+; SSE2-NEXT:  # %bb.2: # %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne .LBB12_3
+; SSE2-NEXT:  .LBB12_4: # %else2
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne .LBB12_5
 ; SSE2-NEXT:  .LBB12_6: # %else4
-; SSE2-NEXT:    pextrw $6, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je .LBB12_8
-; SSE2-NEXT:  # %bb.7: # %cond.store5
-; SSE2-NEXT:    pextrw $3, %xmm0, %eax
-; SSE2-NEXT:    movb %al, 3(%rdi)
+; SSE2-NEXT:  .LBB12_7: # %cond.store5
+; SSE2-NEXT:    shrl $24, %ecx
+; SSE2-NEXT:    movb %cl, 3(%rdi)
 ; SSE2-NEXT:  .LBB12_8: # %else6
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pxor %xmm2, %xmm1
-; SSE2-NEXT:    pextrw $0, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $16, %al
+; SSE2-NEXT:    pextrw $2, %xmm0, %ecx
 ; SSE2-NEXT:    je .LBB12_10
 ; SSE2-NEXT:  # %bb.9: # %cond.store7
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    movb %al, 4(%rdi)
+; SSE2-NEXT:    movb %cl, 4(%rdi)
 ; SSE2-NEXT:  .LBB12_10: # %else8
-; SSE2-NEXT:    pextrw $2, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $32, %al
 ; SSE2-NEXT:    je .LBB12_12
 ; SSE2-NEXT:  # %bb.11: # %cond.store9
-; SSE2-NEXT:    pextrw $5, %xmm0, %eax
-; SSE2-NEXT:    movb %al, 5(%rdi)
+; SSE2-NEXT:    movb %ch, 5(%rdi)
 ; SSE2-NEXT:  .LBB12_12: # %else10
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm3
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pxor %xmm3, %xmm1
-; SSE2-NEXT:    pextrw $4, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB12_14
-; SSE2-NEXT:  # %bb.13: # %cond.store11
-; SSE2-NEXT:    pextrw $6, %xmm0, %eax
-; SSE2-NEXT:    movb %al, 6(%rdi)
-; SSE2-NEXT:  .LBB12_14: # %else12
-; SSE2-NEXT:    pextrw $6, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB12_16
-; SSE2-NEXT:  # %bb.15: # %cond.store13
-; SSE2-NEXT:    pextrw $7, %xmm0, %eax
-; SSE2-NEXT:    movb %al, 7(%rdi)
+; SSE2-NEXT:    testb $64, %al
+; SSE2-NEXT:    pextrw $3, %xmm0, %ecx
+; SSE2-NEXT:    jne .LBB12_13
+; SSE2-NEXT:  # %bb.14: # %else12
+; SSE2-NEXT:    testb $-128, %al
+; SSE2-NEXT:    jne .LBB12_15
 ; SSE2-NEXT:  .LBB12_16: # %else14
 ; SSE2-NEXT:    retq
-;
-; SSE4-LABEL: truncstore_v8i32_v8i8:
-; SSE4:       # %bb.0:
-; SSE4-NEXT:    pxor %xmm5, %xmm5
-; SSE4-NEXT:    pcmpeqd %xmm2, %xmm5
-; SSE4-NEXT:    pcmpeqd %xmm4, %xmm4
-; SSE4-NEXT:    pxor %xmm5, %xmm4
-; SSE4-NEXT:    movdqa {{.*#+}} xmm5 = [127,127,127,127]
-; SSE4-NEXT:    pminsd %xmm5, %xmm0
-; SSE4-NEXT:    pminsd %xmm5, %xmm1
-; SSE4-NEXT:    movdqa {{.*#+}} xmm5 = [4294967168,4294967168,4294967168,4294967168]
-; SSE4-NEXT:    pmaxsd %xmm5, %xmm1
-; SSE4-NEXT:    pmaxsd %xmm5, %xmm0
-; SSE4-NEXT:    packssdw %xmm1, %xmm0
-; SSE4-NEXT:    pextrb $0, %xmm4, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB12_2
-; SSE4-NEXT:  # %bb.1: # %cond.store
-; SSE4-NEXT:    pextrb $0, %xmm0, (%rdi)
-; SSE4-NEXT:  .LBB12_2: # %else
-; SSE4-NEXT:    pextrb $4, %xmm4, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB12_4
-; SSE4-NEXT:  # %bb.3: # %cond.store1
-; SSE4-NEXT:    pextrb $2, %xmm0, 1(%rdi)
-; SSE4-NEXT:  .LBB12_4: # %else2
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm2
+; SSE2-NEXT:  .LBB12_1: # %cond.store
+; SSE2-NEXT:    movb %cl, (%rdi)
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    je .LBB12_4
+; SSE2-NEXT:  .LBB12_3: # %cond.store1
+; SSE2-NEXT:    movb %ch, 1(%rdi)
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    je .LBB12_6
+; SSE2-NEXT:  .LBB12_5: # %cond.store3
+; SSE2-NEXT:    movl %ecx, %edx
+; SSE2-NEXT:    shrl $16, %edx
+; SSE2-NEXT:    movb %dl, 2(%rdi)
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne .LBB12_7
+; SSE2-NEXT:    jmp .LBB12_8
+; SSE2-NEXT:  .LBB12_13: # %cond.store11
+; SSE2-NEXT:    movb %cl, 6(%rdi)
+; SSE2-NEXT:    testb $-128, %al
+; SSE2-NEXT:    je .LBB12_16
+; SSE2-NEXT:  .LBB12_15: # %cond.store13
+; SSE2-NEXT:    movb %ch, 7(%rdi)
+; SSE2-NEXT:    retq
+;
+; SSE4-LABEL: truncstore_v8i32_v8i8:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pxor %xmm4, %xmm4
+; SSE4-NEXT:    packssdw %xmm1, %xmm0
+; SSE4-NEXT:    packsswb %xmm0, %xmm0
+; SSE4-NEXT:    pcmpeqd %xmm4, %xmm3
 ; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE4-NEXT:    pxor %xmm2, %xmm1
-; SSE4-NEXT:    pextrb $8, %xmm1, %eax
+; SSE4-NEXT:    pxor %xmm1, %xmm3
+; SSE4-NEXT:    pcmpeqd %xmm4, %xmm2
+; SSE4-NEXT:    pxor %xmm1, %xmm2
+; SSE4-NEXT:    packssdw %xmm3, %xmm2
+; SSE4-NEXT:    packsswb %xmm0, %xmm2
+; SSE4-NEXT:    pmovmskb %xmm2, %eax
 ; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB12_6
-; SSE4-NEXT:  # %bb.5: # %cond.store3
-; SSE4-NEXT:    pextrb $4, %xmm0, 2(%rdi)
+; SSE4-NEXT:    jne .LBB12_1
+; SSE4-NEXT:  # %bb.2: # %else
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    jne .LBB12_3
+; SSE4-NEXT:  .LBB12_4: # %else2
+; SSE4-NEXT:    testb $4, %al
+; SSE4-NEXT:    jne .LBB12_5
 ; SSE4-NEXT:  .LBB12_6: # %else4
-; SSE4-NEXT:    pextrb $12, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB12_8
-; SSE4-NEXT:  # %bb.7: # %cond.store5
-; SSE4-NEXT:    pextrb $6, %xmm0, 3(%rdi)
+; SSE4-NEXT:    testb $8, %al
+; SSE4-NEXT:    jne .LBB12_7
 ; SSE4-NEXT:  .LBB12_8: # %else6
-; SSE4-NEXT:    pxor %xmm2, %xmm2
-; SSE4-NEXT:    pcmpeqd %xmm3, %xmm2
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE4-NEXT:    pxor %xmm2, %xmm1
-; SSE4-NEXT:    pextrb $0, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB12_10
-; SSE4-NEXT:  # %bb.9: # %cond.store7
-; SSE4-NEXT:    pextrb $8, %xmm0, 4(%rdi)
+; SSE4-NEXT:    testb $16, %al
+; SSE4-NEXT:    jne .LBB12_9
 ; SSE4-NEXT:  .LBB12_10: # %else8
-; SSE4-NEXT:    pextrb $4, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB12_12
-; SSE4-NEXT:  # %bb.11: # %cond.store9
-; SSE4-NEXT:    pextrb $10, %xmm0, 5(%rdi)
+; SSE4-NEXT:    testb $32, %al
+; SSE4-NEXT:    jne .LBB12_11
 ; SSE4-NEXT:  .LBB12_12: # %else10
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm3
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE4-NEXT:    pxor %xmm3, %xmm1
-; SSE4-NEXT:    pextrb $8, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB12_14
-; SSE4-NEXT:  # %bb.13: # %cond.store11
-; SSE4-NEXT:    pextrb $12, %xmm0, 6(%rdi)
+; SSE4-NEXT:    testb $64, %al
+; SSE4-NEXT:    jne .LBB12_13
 ; SSE4-NEXT:  .LBB12_14: # %else12
-; SSE4-NEXT:    pextrb $12, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB12_16
-; SSE4-NEXT:  # %bb.15: # %cond.store13
-; SSE4-NEXT:    pextrb $14, %xmm0, 7(%rdi)
+; SSE4-NEXT:    testb $-128, %al
+; SSE4-NEXT:    jne .LBB12_15
 ; SSE4-NEXT:  .LBB12_16: # %else14
 ; SSE4-NEXT:    retq
+; SSE4-NEXT:  .LBB12_1: # %cond.store
+; SSE4-NEXT:    pextrb $0, %xmm0, (%rdi)
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    je .LBB12_4
+; SSE4-NEXT:  .LBB12_3: # %cond.store1
+; SSE4-NEXT:    pextrb $1, %xmm0, 1(%rdi)
+; SSE4-NEXT:    testb $4, %al
+; SSE4-NEXT:    je .LBB12_6
+; SSE4-NEXT:  .LBB12_5: # %cond.store3
+; SSE4-NEXT:    pextrb $2, %xmm0, 2(%rdi)
+; SSE4-NEXT:    testb $8, %al
+; SSE4-NEXT:    je .LBB12_8
+; SSE4-NEXT:  .LBB12_7: # %cond.store5
+; SSE4-NEXT:    pextrb $3, %xmm0, 3(%rdi)
+; SSE4-NEXT:    testb $16, %al
+; SSE4-NEXT:    je .LBB12_10
+; SSE4-NEXT:  .LBB12_9: # %cond.store7
+; SSE4-NEXT:    pextrb $4, %xmm0, 4(%rdi)
+; SSE4-NEXT:    testb $32, %al
+; SSE4-NEXT:    je .LBB12_12
+; SSE4-NEXT:  .LBB12_11: # %cond.store9
+; SSE4-NEXT:    pextrb $5, %xmm0, 5(%rdi)
+; SSE4-NEXT:    testb $64, %al
+; SSE4-NEXT:    je .LBB12_14
+; SSE4-NEXT:  .LBB12_13: # %cond.store11
+; SSE4-NEXT:    pextrb $6, %xmm0, 6(%rdi)
+; SSE4-NEXT:    testb $-128, %al
+; SSE4-NEXT:    je .LBB12_16
+; SSE4-NEXT:  .LBB12_15: # %cond.store13
+; SSE4-NEXT:    pextrb $7, %xmm0, 7(%rdi)
+; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: truncstore_v8i32_v8i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm4, %xmm4
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm5 = [127,127,127,127]
-; AVX1-NEXT:    vpminsd %xmm5, %xmm0, %xmm6
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT:    vpminsd %xmm5, %xmm0, %xmm0
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm5 = [4294967168,4294967168,4294967168,4294967168]
-; AVX1-NEXT:    vpmaxsd %xmm5, %xmm0, %xmm0
-; AVX1-NEXT:    vpmaxsd %xmm5, %xmm6, %xmm5
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm5, %xmm0
-; AVX1-NEXT:    vpextrb $0, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je .LBB12_2
-; AVX1-NEXT:  # %bb.1: # %cond.store
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT:    vpackssdw %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; AVX1-NEXT:    vmovmskps %ymm1, %eax
+; AVX1-NEXT:    notl %eax
+; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    jne .LBB12_1
+; AVX1-NEXT:  # %bb.2: # %else
+; AVX1-NEXT:    testb $2, %al
+; AVX1-NEXT:    jne .LBB12_3
+; AVX1-NEXT:  .LBB12_4: # %else2
+; AVX1-NEXT:    testb $4, %al
+; AVX1-NEXT:    jne .LBB12_5
+; AVX1-NEXT:  .LBB12_6: # %else4
+; AVX1-NEXT:    testb $8, %al
+; AVX1-NEXT:    jne .LBB12_7
+; AVX1-NEXT:  .LBB12_8: # %else6
+; AVX1-NEXT:    testb $16, %al
+; AVX1-NEXT:    jne .LBB12_9
+; AVX1-NEXT:  .LBB12_10: # %else8
+; AVX1-NEXT:    testb $32, %al
+; AVX1-NEXT:    jne .LBB12_11
+; AVX1-NEXT:  .LBB12_12: # %else10
+; AVX1-NEXT:    testb $64, %al
+; AVX1-NEXT:    jne .LBB12_13
+; AVX1-NEXT:  .LBB12_14: # %else12
+; AVX1-NEXT:    testb $-128, %al
+; AVX1-NEXT:    jne .LBB12_15
+; AVX1-NEXT:  .LBB12_16: # %else14
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+; AVX1-NEXT:  .LBB12_1: # %cond.store
 ; AVX1-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX1-NEXT:  .LBB12_2: # %else
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $2, %al
 ; AVX1-NEXT:    je .LBB12_4
-; AVX1-NEXT:  # %bb.3: # %cond.store1
-; AVX1-NEXT:    vpextrb $2, %xmm0, 1(%rdi)
-; AVX1-NEXT:  .LBB12_4: # %else2
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm4, %xmm4
-; AVX1-NEXT:    vpextrb $8, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:  .LBB12_3: # %cond.store1
+; AVX1-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
+; AVX1-NEXT:    testb $4, %al
 ; AVX1-NEXT:    je .LBB12_6
-; AVX1-NEXT:  # %bb.5: # %cond.store3
-; AVX1-NEXT:    vpextrb $4, %xmm0, 2(%rdi)
-; AVX1-NEXT:  .LBB12_6: # %else4
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $12, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:  .LBB12_5: # %cond.store3
+; AVX1-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
+; AVX1-NEXT:    testb $8, %al
 ; AVX1-NEXT:    je .LBB12_8
-; AVX1-NEXT:  # %bb.7: # %cond.store5
-; AVX1-NEXT:    vpextrb $6, %xmm0, 3(%rdi)
-; AVX1-NEXT:  .LBB12_8: # %else6
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm1
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $0, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:  .LBB12_7: # %cond.store5
+; AVX1-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
+; AVX1-NEXT:    testb $16, %al
 ; AVX1-NEXT:    je .LBB12_10
-; AVX1-NEXT:  # %bb.9: # %cond.store7
-; AVX1-NEXT:    vpextrb $8, %xmm0, 4(%rdi)
-; AVX1-NEXT:  .LBB12_10: # %else8
-; AVX1-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:  .LBB12_9: # %cond.store7
+; AVX1-NEXT:    vpextrb $4, %xmm0, 4(%rdi)
+; AVX1-NEXT:    testb $32, %al
 ; AVX1-NEXT:    je .LBB12_12
-; AVX1-NEXT:  # %bb.11: # %cond.store9
-; AVX1-NEXT:    vpextrb $10, %xmm0, 5(%rdi)
-; AVX1-NEXT:  .LBB12_12: # %else10
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:  .LBB12_11: # %cond.store9
+; AVX1-NEXT:    vpextrb $5, %xmm0, 5(%rdi)
+; AVX1-NEXT:    testb $64, %al
 ; AVX1-NEXT:    je .LBB12_14
-; AVX1-NEXT:  # %bb.13: # %cond.store11
-; AVX1-NEXT:    vpextrb $12, %xmm0, 6(%rdi)
-; AVX1-NEXT:  .LBB12_14: # %else12
-; AVX1-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:  .LBB12_13: # %cond.store11
+; AVX1-NEXT:    vpextrb $6, %xmm0, 6(%rdi)
+; AVX1-NEXT:    testb $-128, %al
 ; AVX1-NEXT:    je .LBB12_16
-; AVX1-NEXT:  # %bb.15: # %cond.store13
-; AVX1-NEXT:    vpextrb $14, %xmm0, 7(%rdi)
-; AVX1-NEXT:  .LBB12_16: # %else14
+; AVX1-NEXT:  .LBB12_15: # %cond.store13
+; AVX1-NEXT:    vpextrb $7, %xmm0, 7(%rdi)
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: truncstore_v8i32_v8i8:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm4
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    vpxor %xmm3, %xmm4, %xmm4
-; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm5 = [127,127,127,127,127,127,127,127]
-; AVX2-NEXT:    vpminsd %ymm5, %ymm0, %ymm0
-; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm5 = [4294967168,4294967168,4294967168,4294967168,4294967168,4294967168,4294967168,4294967168]
-; AVX2-NEXT:    vpmaxsd %ymm5, %ymm0, %ymm0
-; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm5
-; AVX2-NEXT:    vpackssdw %xmm5, %xmm0, %xmm0
-; AVX2-NEXT:    vpextrb $0, %xmm4, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je .LBB12_2
-; AVX2-NEXT:  # %bb.1: # %cond.store
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; AVX2-NEXT:    vpackssdw %xmm3, %xmm0, %xmm0
+; AVX2-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
+; AVX2-NEXT:    vpcmpeqd %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    vmovmskps %ymm1, %eax
+; AVX2-NEXT:    notl %eax
+; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    jne .LBB12_1
+; AVX2-NEXT:  # %bb.2: # %else
+; AVX2-NEXT:    testb $2, %al
+; AVX2-NEXT:    jne .LBB12_3
+; AVX2-NEXT:  .LBB12_4: # %else2
+; AVX2-NEXT:    testb $4, %al
+; AVX2-NEXT:    jne .LBB12_5
+; AVX2-NEXT:  .LBB12_6: # %else4
+; AVX2-NEXT:    testb $8, %al
+; AVX2-NEXT:    jne .LBB12_7
+; AVX2-NEXT:  .LBB12_8: # %else6
+; AVX2-NEXT:    testb $16, %al
+; AVX2-NEXT:    jne .LBB12_9
+; AVX2-NEXT:  .LBB12_10: # %else8
+; AVX2-NEXT:    testb $32, %al
+; AVX2-NEXT:    jne .LBB12_11
+; AVX2-NEXT:  .LBB12_12: # %else10
+; AVX2-NEXT:    testb $64, %al
+; AVX2-NEXT:    jne .LBB12_13
+; AVX2-NEXT:  .LBB12_14: # %else12
+; AVX2-NEXT:    testb $-128, %al
+; AVX2-NEXT:    jne .LBB12_15
+; AVX2-NEXT:  .LBB12_16: # %else14
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+; AVX2-NEXT:  .LBB12_1: # %cond.store
 ; AVX2-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX2-NEXT:  .LBB12_2: # %else
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $2, %al
 ; AVX2-NEXT:    je .LBB12_4
-; AVX2-NEXT:  # %bb.3: # %cond.store1
-; AVX2-NEXT:    vpextrb $2, %xmm0, 1(%rdi)
-; AVX2-NEXT:  .LBB12_4: # %else2
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm4
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    vpxor %xmm3, %xmm4, %xmm4
-; AVX2-NEXT:    vpextrb $8, %xmm4, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:  .LBB12_3: # %cond.store1
+; AVX2-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
+; AVX2-NEXT:    testb $4, %al
 ; AVX2-NEXT:    je .LBB12_6
-; AVX2-NEXT:  # %bb.5: # %cond.store3
-; AVX2-NEXT:    vpextrb $4, %xmm0, 2(%rdi)
-; AVX2-NEXT:  .LBB12_6: # %else4
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrb $12, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:  .LBB12_5: # %cond.store3
+; AVX2-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
+; AVX2-NEXT:    testb $8, %al
 ; AVX2-NEXT:    je .LBB12_8
-; AVX2-NEXT:  # %bb.7: # %cond.store5
-; AVX2-NEXT:    vpextrb $6, %xmm0, 3(%rdi)
-; AVX2-NEXT:  .LBB12_8: # %else6
-; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm1
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrb $0, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:  .LBB12_7: # %cond.store5
+; AVX2-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
+; AVX2-NEXT:    testb $16, %al
 ; AVX2-NEXT:    je .LBB12_10
-; AVX2-NEXT:  # %bb.9: # %cond.store7
-; AVX2-NEXT:    vpextrb $8, %xmm0, 4(%rdi)
-; AVX2-NEXT:  .LBB12_10: # %else8
-; AVX2-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:  .LBB12_9: # %cond.store7
+; AVX2-NEXT:    vpextrb $4, %xmm0, 4(%rdi)
+; AVX2-NEXT:    testb $32, %al
 ; AVX2-NEXT:    je .LBB12_12
-; AVX2-NEXT:  # %bb.11: # %cond.store9
-; AVX2-NEXT:    vpextrb $10, %xmm0, 5(%rdi)
-; AVX2-NEXT:  .LBB12_12: # %else10
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:  .LBB12_11: # %cond.store9
+; AVX2-NEXT:    vpextrb $5, %xmm0, 5(%rdi)
+; AVX2-NEXT:    testb $64, %al
 ; AVX2-NEXT:    je .LBB12_14
-; AVX2-NEXT:  # %bb.13: # %cond.store11
-; AVX2-NEXT:    vpextrb $12, %xmm0, 6(%rdi)
-; AVX2-NEXT:  .LBB12_14: # %else12
-; AVX2-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:  .LBB12_13: # %cond.store11
+; AVX2-NEXT:    vpextrb $6, %xmm0, 6(%rdi)
+; AVX2-NEXT:    testb $-128, %al
 ; AVX2-NEXT:    je .LBB12_16
-; AVX2-NEXT:  # %bb.15: # %cond.store13
-; AVX2-NEXT:    vpextrb $14, %xmm0, 7(%rdi)
-; AVX2-NEXT:  .LBB12_16: # %else14
+; AVX2-NEXT:  .LBB12_15: # %cond.store13
+; AVX2-NEXT:    vpextrb $7, %xmm0, 7(%rdi)
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
@@ -5303,73 +4933,66 @@ define void @truncstore_v8i32_v8i8(<8 x i32> %x, <8 x i8>* %p, <8 x i32> %mask)
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
 ; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    vpbroadcastd {{.*#+}} ymm2 = [127,127,127,127,127,127,127,127]
-; AVX512F-NEXT:    vpminsd %ymm2, %ymm0, %ymm0
-; AVX512F-NEXT:    vpbroadcastd {{.*#+}} ymm2 = [4294967168,4294967168,4294967168,4294967168,4294967168,4294967168,4294967168,4294967168]
-; AVX512F-NEXT:    vpmaxsd %ymm2, %ymm0, %ymm0
-; AVX512F-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
 ; AVX512F-NEXT:    kmovw %k0, %eax
 ; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB12_2
-; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    jne .LBB12_1
+; AVX512F-NEXT:  # %bb.2: # %else
+; AVX512F-NEXT:    testb $2, %al
+; AVX512F-NEXT:    jne .LBB12_3
+; AVX512F-NEXT:  .LBB12_4: # %else2
+; AVX512F-NEXT:    testb $4, %al
+; AVX512F-NEXT:    jne .LBB12_5
+; AVX512F-NEXT:  .LBB12_6: # %else4
+; AVX512F-NEXT:    testb $8, %al
+; AVX512F-NEXT:    jne .LBB12_7
+; AVX512F-NEXT:  .LBB12_8: # %else6
+; AVX512F-NEXT:    testb $16, %al
+; AVX512F-NEXT:    jne .LBB12_9
+; AVX512F-NEXT:  .LBB12_10: # %else8
+; AVX512F-NEXT:    testb $32, %al
+; AVX512F-NEXT:    jne .LBB12_11
+; AVX512F-NEXT:  .LBB12_12: # %else10
+; AVX512F-NEXT:    testb $64, %al
+; AVX512F-NEXT:    jne .LBB12_13
+; AVX512F-NEXT:  .LBB12_14: # %else12
+; AVX512F-NEXT:    testb $-128, %al
+; AVX512F-NEXT:    jne .LBB12_15
+; AVX512F-NEXT:  .LBB12_16: # %else14
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+; AVX512F-NEXT:  .LBB12_1: # %cond.store
 ; AVX512F-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX512F-NEXT:  .LBB12_2: # %else
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $2, %al
 ; AVX512F-NEXT:    je .LBB12_4
-; AVX512F-NEXT:  # %bb.3: # %cond.store1
-; AVX512F-NEXT:    vpextrb $2, %xmm0, 1(%rdi)
-; AVX512F-NEXT:  .LBB12_4: # %else2
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  .LBB12_3: # %cond.store1
+; AVX512F-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
+; AVX512F-NEXT:    testb $4, %al
 ; AVX512F-NEXT:    je .LBB12_6
-; AVX512F-NEXT:  # %bb.5: # %cond.store3
-; AVX512F-NEXT:    vpextrb $4, %xmm0, 2(%rdi)
-; AVX512F-NEXT:  .LBB12_6: # %else4
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  .LBB12_5: # %cond.store3
+; AVX512F-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
+; AVX512F-NEXT:    testb $8, %al
 ; AVX512F-NEXT:    je .LBB12_8
-; AVX512F-NEXT:  # %bb.7: # %cond.store5
-; AVX512F-NEXT:    vpextrb $6, %xmm0, 3(%rdi)
-; AVX512F-NEXT:  .LBB12_8: # %else6
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $4, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  .LBB12_7: # %cond.store5
+; AVX512F-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
+; AVX512F-NEXT:    testb $16, %al
 ; AVX512F-NEXT:    je .LBB12_10
-; AVX512F-NEXT:  # %bb.9: # %cond.store7
-; AVX512F-NEXT:    vpextrb $8, %xmm0, 4(%rdi)
-; AVX512F-NEXT:  .LBB12_10: # %else8
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $5, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  .LBB12_9: # %cond.store7
+; AVX512F-NEXT:    vpextrb $4, %xmm0, 4(%rdi)
+; AVX512F-NEXT:    testb $32, %al
 ; AVX512F-NEXT:    je .LBB12_12
-; AVX512F-NEXT:  # %bb.11: # %cond.store9
-; AVX512F-NEXT:    vpextrb $10, %xmm0, 5(%rdi)
-; AVX512F-NEXT:  .LBB12_12: # %else10
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $6, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  .LBB12_11: # %cond.store9
+; AVX512F-NEXT:    vpextrb $5, %xmm0, 5(%rdi)
+; AVX512F-NEXT:    testb $64, %al
 ; AVX512F-NEXT:    je .LBB12_14
-; AVX512F-NEXT:  # %bb.13: # %cond.store11
-; AVX512F-NEXT:    vpextrb $12, %xmm0, 6(%rdi)
-; AVX512F-NEXT:  .LBB12_14: # %else12
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $7, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  .LBB12_13: # %cond.store11
+; AVX512F-NEXT:    vpextrb $6, %xmm0, 6(%rdi)
+; AVX512F-NEXT:    testb $-128, %al
 ; AVX512F-NEXT:    je .LBB12_16
-; AVX512F-NEXT:  # %bb.15: # %cond.store13
-; AVX512F-NEXT:    vpextrb $14, %xmm0, 7(%rdi)
-; AVX512F-NEXT:  .LBB12_16: # %else14
+; AVX512F-NEXT:  .LBB12_15: # %cond.store13
+; AVX512F-NEXT:    vpextrb $7, %xmm0, 7(%rdi)
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
@@ -5377,14 +5000,11 @@ define void @truncstore_v8i32_v8i8(<8 x i32> %x, <8 x i8>* %p, <8 x i32> %mask)
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
 ; AVX512BW-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512BW-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [127,127,127,127,127,127,127,127]
-; AVX512BW-NEXT:    vpminsd %ymm1, %ymm0, %ymm0
-; AVX512BW-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [4294967168,4294967168,4294967168,4294967168,4294967168,4294967168,4294967168,4294967168]
-; AVX512BW-NEXT:    vpmaxsd %ymm1, %ymm0, %ymm0
-; AVX512BW-NEXT:    vpmovdw %zmm0, %ymm0
-; AVX512BW-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
 ; AVX512BW-NEXT:    kshiftlq $56, %k0, %k0
 ; AVX512BW-NEXT:    kshiftrq $56, %k0, %k1
+; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512BW-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vmovdqu8 %zmm0, (%rdi) {%k1}
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -5411,211 +5031,149 @@ define void @truncstore_v4i32_v4i16(<4 x i32> %x, <4 x i16>* %p, <4 x i32> %mask
 ; SSE2-LABEL: truncstore_v4i32_v4i16:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    packssdw %xmm0, %xmm0
 ; SSE2-NEXT:    pcmpeqd %xmm1, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [32767,32767,32767,32767]
-; SSE2-NEXT:    movdqa %xmm3, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE2-NEXT:    pand %xmm4, %xmm0
-; SSE2-NEXT:    pandn %xmm3, %xmm4
-; SSE2-NEXT:    por %xmm0, %xmm4
-; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [4294934528,4294934528,4294934528,4294934528]
-; SSE2-NEXT:    movdqa %xmm4, %xmm0
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE2-NEXT:    pand %xmm0, %xmm4
-; SSE2-NEXT:    pandn %xmm3, %xmm0
-; SSE2-NEXT:    por %xmm4, %xmm0
-; SSE2-NEXT:    movd %xmm2, %eax
-; SSE2-NEXT:    notl %eax
+; SSE2-NEXT:    movmskps %xmm2, %eax
+; SSE2-NEXT:    xorl $15, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB13_2
-; SSE2-NEXT:  # %bb.1: # %cond.store
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    movw %ax, (%rdi)
-; SSE2-NEXT:  .LBB13_2: # %else
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm3
-; SSE2-NEXT:    pxor %xmm3, %xmm2
-; SSE2-NEXT:    pextrw $2, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB13_4
-; SSE2-NEXT:  # %bb.3: # %cond.store1
-; SSE2-NEXT:    pextrw $2, %xmm0, %eax
-; SSE2-NEXT:    movw %ax, 2(%rdi)
+; SSE2-NEXT:    jne .LBB13_1
+; SSE2-NEXT:  # %bb.2: # %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne .LBB13_3
 ; SSE2-NEXT:  .LBB13_4: # %else2
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm1
-; SSE2-NEXT:    pxor %xmm1, %xmm3
-; SSE2-NEXT:    pextrw $4, %xmm3, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB13_6
-; SSE2-NEXT:  # %bb.5: # %cond.store3
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    movw %ax, 4(%rdi)
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne .LBB13_5
 ; SSE2-NEXT:  .LBB13_6: # %else4
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT:    pxor %xmm2, %xmm1
-; SSE2-NEXT:    pextrw $6, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne .LBB13_7
+; SSE2-NEXT:  .LBB13_8: # %else6
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  .LBB13_1: # %cond.store
+; SSE2-NEXT:    movd %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, (%rdi)
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    je .LBB13_4
+; SSE2-NEXT:  .LBB13_3: # %cond.store1
+; SSE2-NEXT:    pextrw $1, %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, 2(%rdi)
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    je .LBB13_6
+; SSE2-NEXT:  .LBB13_5: # %cond.store3
+; SSE2-NEXT:    pextrw $2, %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, 4(%rdi)
+; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je .LBB13_8
-; SSE2-NEXT:  # %bb.7: # %cond.store5
-; SSE2-NEXT:    pextrw $6, %xmm0, %eax
+; SSE2-NEXT:  .LBB13_7: # %cond.store5
+; SSE2-NEXT:    pextrw $3, %xmm0, %eax
 ; SSE2-NEXT:    movw %ax, 6(%rdi)
-; SSE2-NEXT:  .LBB13_8: # %else6
 ; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: truncstore_v4i32_v4i16:
 ; SSE4:       # %bb.0:
-; SSE4-NEXT:    pxor %xmm3, %xmm3
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm3
-; SSE4-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE4-NEXT:    pxor %xmm3, %xmm2
-; SSE4-NEXT:    pminsd {{.*}}(%rip), %xmm0
-; SSE4-NEXT:    pmaxsd {{.*}}(%rip), %xmm0
-; SSE4-NEXT:    pextrb $0, %xmm2, %eax
+; SSE4-NEXT:    pxor %xmm2, %xmm2
+; SSE4-NEXT:    packssdw %xmm0, %xmm0
+; SSE4-NEXT:    pcmpeqd %xmm1, %xmm2
+; SSE4-NEXT:    movmskps %xmm2, %eax
+; SSE4-NEXT:    xorl $15, %eax
 ; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB13_2
-; SSE4-NEXT:  # %bb.1: # %cond.store
+; SSE4-NEXT:    jne .LBB13_1
+; SSE4-NEXT:  # %bb.2: # %else
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    jne .LBB13_3
+; SSE4-NEXT:  .LBB13_4: # %else2
+; SSE4-NEXT:    testb $4, %al
+; SSE4-NEXT:    jne .LBB13_5
+; SSE4-NEXT:  .LBB13_6: # %else4
+; SSE4-NEXT:    testb $8, %al
+; SSE4-NEXT:    jne .LBB13_7
+; SSE4-NEXT:  .LBB13_8: # %else6
+; SSE4-NEXT:    retq
+; SSE4-NEXT:  .LBB13_1: # %cond.store
 ; SSE4-NEXT:    pextrw $0, %xmm0, (%rdi)
-; SSE4-NEXT:  .LBB13_2: # %else
-; SSE4-NEXT:    pextrb $4, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $2, %al
 ; SSE4-NEXT:    je .LBB13_4
-; SSE4-NEXT:  # %bb.3: # %cond.store1
-; SSE4-NEXT:    pextrw $2, %xmm0, 2(%rdi)
-; SSE4-NEXT:  .LBB13_4: # %else2
-; SSE4-NEXT:    pxor %xmm2, %xmm2
-; SSE4-NEXT:    pcmpeqd %xmm2, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE4-NEXT:    pxor %xmm1, %xmm2
-; SSE4-NEXT:    pextrb $8, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:  .LBB13_3: # %cond.store1
+; SSE4-NEXT:    pextrw $1, %xmm0, 2(%rdi)
+; SSE4-NEXT:    testb $4, %al
 ; SSE4-NEXT:    je .LBB13_6
-; SSE4-NEXT:  # %bb.5: # %cond.store3
-; SSE4-NEXT:    pextrw $4, %xmm0, 4(%rdi)
-; SSE4-NEXT:  .LBB13_6: # %else4
-; SSE4-NEXT:    pextrb $12, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:  .LBB13_5: # %cond.store3
+; SSE4-NEXT:    pextrw $2, %xmm0, 4(%rdi)
+; SSE4-NEXT:    testb $8, %al
 ; SSE4-NEXT:    je .LBB13_8
-; SSE4-NEXT:  # %bb.7: # %cond.store5
-; SSE4-NEXT:    pextrw $6, %xmm0, 6(%rdi)
-; SSE4-NEXT:  .LBB13_8: # %else6
+; SSE4-NEXT:  .LBB13_7: # %cond.store5
+; SSE4-NEXT:    pextrw $3, %xmm0, 6(%rdi)
 ; SSE4-NEXT:    retq
 ;
-; AVX1-LABEL: truncstore_v4i32_v4i16:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpminsd {{.*}}(%rip), %xmm0, %xmm0
-; AVX1-NEXT:    vpmaxsd {{.*}}(%rip), %xmm0, %xmm0
-; AVX1-NEXT:    vpextrb $0, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je .LBB13_2
-; AVX1-NEXT:  # %bb.1: # %cond.store
-; AVX1-NEXT:    vpextrw $0, %xmm0, (%rdi)
-; AVX1-NEXT:  .LBB13_2: # %else
-; AVX1-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je .LBB13_4
-; AVX1-NEXT:  # %bb.3: # %cond.store1
-; AVX1-NEXT:    vpextrw $2, %xmm0, 2(%rdi)
-; AVX1-NEXT:  .LBB13_4: # %else2
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je .LBB13_6
-; AVX1-NEXT:  # %bb.5: # %cond.store3
-; AVX1-NEXT:    vpextrw $4, %xmm0, 4(%rdi)
-; AVX1-NEXT:  .LBB13_6: # %else4
-; AVX1-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je .LBB13_8
-; AVX1-NEXT:  # %bb.7: # %cond.store5
-; AVX1-NEXT:    vpextrw $6, %xmm0, 6(%rdi)
-; AVX1-NEXT:  .LBB13_8: # %else6
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: truncstore_v4i32_v4i16:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm3 = [32767,32767,32767,32767]
-; AVX2-NEXT:    vpminsd %xmm3, %xmm0, %xmm0
-; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm3 = [4294934528,4294934528,4294934528,4294934528]
-; AVX2-NEXT:    vpmaxsd %xmm3, %xmm0, %xmm0
-; AVX2-NEXT:    vpextrb $0, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je .LBB13_2
-; AVX2-NEXT:  # %bb.1: # %cond.store
-; AVX2-NEXT:    vpextrw $0, %xmm0, (%rdi)
-; AVX2-NEXT:  .LBB13_2: # %else
-; AVX2-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je .LBB13_4
-; AVX2-NEXT:  # %bb.3: # %cond.store1
-; AVX2-NEXT:    vpextrw $2, %xmm0, 2(%rdi)
-; AVX2-NEXT:  .LBB13_4: # %else2
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je .LBB13_6
-; AVX2-NEXT:  # %bb.5: # %cond.store3
-; AVX2-NEXT:    vpextrw $4, %xmm0, 4(%rdi)
-; AVX2-NEXT:  .LBB13_6: # %else4
-; AVX2-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je .LBB13_8
-; AVX2-NEXT:  # %bb.7: # %cond.store5
-; AVX2-NEXT:    vpextrw $6, %xmm0, 6(%rdi)
-; AVX2-NEXT:  .LBB13_8: # %else6
-; AVX2-NEXT:    retq
+; AVX-LABEL: truncstore_v4i32_v4i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX-NEXT:    vpackssdw %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
+; AVX-NEXT:    vmovmskps %xmm1, %eax
+; AVX-NEXT:    xorl $15, %eax
+; AVX-NEXT:    testb $1, %al
+; AVX-NEXT:    jne .LBB13_1
+; AVX-NEXT:  # %bb.2: # %else
+; AVX-NEXT:    testb $2, %al
+; AVX-NEXT:    jne .LBB13_3
+; AVX-NEXT:  .LBB13_4: # %else2
+; AVX-NEXT:    testb $4, %al
+; AVX-NEXT:    jne .LBB13_5
+; AVX-NEXT:  .LBB13_6: # %else4
+; AVX-NEXT:    testb $8, %al
+; AVX-NEXT:    jne .LBB13_7
+; AVX-NEXT:  .LBB13_8: # %else6
+; AVX-NEXT:    retq
+; AVX-NEXT:  .LBB13_1: # %cond.store
+; AVX-NEXT:    vpextrw $0, %xmm0, (%rdi)
+; AVX-NEXT:    testb $2, %al
+; AVX-NEXT:    je .LBB13_4
+; AVX-NEXT:  .LBB13_3: # %cond.store1
+; AVX-NEXT:    vpextrw $1, %xmm0, 2(%rdi)
+; AVX-NEXT:    testb $4, %al
+; AVX-NEXT:    je .LBB13_6
+; AVX-NEXT:  .LBB13_5: # %cond.store3
+; AVX-NEXT:    vpextrw $2, %xmm0, 4(%rdi)
+; AVX-NEXT:    testb $8, %al
+; AVX-NEXT:    je .LBB13_8
+; AVX-NEXT:  .LBB13_7: # %cond.store5
+; AVX-NEXT:    vpextrw $3, %xmm0, 6(%rdi)
+; AVX-NEXT:    retq
 ;
 ; AVX512F-LABEL: truncstore_v4i32_v4i16:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
 ; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [32767,32767,32767,32767]
-; AVX512F-NEXT:    vpminsd %xmm2, %xmm0, %xmm0
-; AVX512F-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [4294934528,4294934528,4294934528,4294934528]
-; AVX512F-NEXT:    vpmaxsd %xmm2, %xmm0, %xmm0
+; AVX512F-NEXT:    vpackssdw %xmm0, %xmm0, %xmm0
 ; AVX512F-NEXT:    kmovw %k0, %eax
 ; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB13_2
-; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    jne .LBB13_1
+; AVX512F-NEXT:  # %bb.2: # %else
+; AVX512F-NEXT:    testb $2, %al
+; AVX512F-NEXT:    jne .LBB13_3
+; AVX512F-NEXT:  .LBB13_4: # %else2
+; AVX512F-NEXT:    testb $4, %al
+; AVX512F-NEXT:    jne .LBB13_5
+; AVX512F-NEXT:  .LBB13_6: # %else4
+; AVX512F-NEXT:    testb $8, %al
+; AVX512F-NEXT:    jne .LBB13_7
+; AVX512F-NEXT:  .LBB13_8: # %else6
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+; AVX512F-NEXT:  .LBB13_1: # %cond.store
 ; AVX512F-NEXT:    vpextrw $0, %xmm0, (%rdi)
-; AVX512F-NEXT:  .LBB13_2: # %else
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $2, %al
 ; AVX512F-NEXT:    je .LBB13_4
-; AVX512F-NEXT:  # %bb.3: # %cond.store1
-; AVX512F-NEXT:    vpextrw $2, %xmm0, 2(%rdi)
-; AVX512F-NEXT:  .LBB13_4: # %else2
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  .LBB13_3: # %cond.store1
+; AVX512F-NEXT:    vpextrw $1, %xmm0, 2(%rdi)
+; AVX512F-NEXT:    testb $4, %al
 ; AVX512F-NEXT:    je .LBB13_6
-; AVX512F-NEXT:  # %bb.5: # %cond.store3
-; AVX512F-NEXT:    vpextrw $4, %xmm0, 4(%rdi)
-; AVX512F-NEXT:  .LBB13_6: # %else4
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  .LBB13_5: # %cond.store3
+; AVX512F-NEXT:    vpextrw $2, %xmm0, 4(%rdi)
+; AVX512F-NEXT:    testb $8, %al
 ; AVX512F-NEXT:    je .LBB13_8
-; AVX512F-NEXT:  # %bb.7: # %cond.store5
-; AVX512F-NEXT:    vpextrw $6, %xmm0, 6(%rdi)
-; AVX512F-NEXT:  .LBB13_8: # %else6
+; AVX512F-NEXT:  .LBB13_7: # %cond.store5
+; AVX512F-NEXT:    vpextrw $3, %xmm0, 6(%rdi)
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
@@ -5623,13 +5181,9 @@ define void @truncstore_v4i32_v4i16(<4 x i32> %x, <4 x i16>* %p, <4 x i32> %mask
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
 ; AVX512BW-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512BW-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [32767,32767,32767,32767]
-; AVX512BW-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [4294934528,4294934528,4294934528,4294934528]
-; AVX512BW-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpackssdw %xmm0, %xmm0, %xmm0
 ; AVX512BW-NEXT:    kshiftld $28, %k0, %k0
 ; AVX512BW-NEXT:    kshiftrd $28, %k0, %k1
+; AVX512BW-NEXT:    vpackssdw %xmm0, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vmovdqu16 %zmm0, (%rdi) {%k1}
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -5655,225 +5209,219 @@ define void @truncstore_v4i32_v4i8(<4 x i32> %x, <4 x i8>* %p, <4 x i32> %mask)
 ; SSE2-LABEL: truncstore_v4i32_v4i8:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm2
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [127,127,127,127]
 ; SSE2-NEXT:    movdqa %xmm3, %xmm4
 ; SSE2-NEXT:    pcmpgtd %xmm0, %xmm4
 ; SSE2-NEXT:    pand %xmm4, %xmm0
 ; SSE2-NEXT:    pandn %xmm3, %xmm4
 ; SSE2-NEXT:    por %xmm0, %xmm4
-; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [4294967168,4294967168,4294967168,4294967168]
-; SSE2-NEXT:    movdqa %xmm4, %xmm0
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE2-NEXT:    pand %xmm0, %xmm4
-; SSE2-NEXT:    pandn %xmm3, %xmm0
-; SSE2-NEXT:    por %xmm4, %xmm0
-; SSE2-NEXT:    movd %xmm2, %eax
-; SSE2-NEXT:    notl %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB14_2
-; SSE2-NEXT:  # %bb.1: # %cond.store
-; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [4294967168,4294967168,4294967168,4294967168]
+; SSE2-NEXT:    movdqa %xmm4, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSE2-NEXT:    pand %xmm3, %xmm4
+; SSE2-NEXT:    pandn %xmm0, %xmm3
+; SSE2-NEXT:    por %xmm4, %xmm3
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm3
+; SSE2-NEXT:    packuswb %xmm3, %xmm3
+; SSE2-NEXT:    packuswb %xmm3, %xmm3
+; SSE2-NEXT:    pcmpeqd %xmm1, %xmm2
+; SSE2-NEXT:    movmskps %xmm2, %ecx
+; SSE2-NEXT:    xorl $15, %ecx
+; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    movd %xmm3, %eax
+; SSE2-NEXT:    jne .LBB14_1
+; SSE2-NEXT:  # %bb.2: # %else
+; SSE2-NEXT:    testb $2, %cl
+; SSE2-NEXT:    jne .LBB14_3
+; SSE2-NEXT:  .LBB14_4: # %else2
+; SSE2-NEXT:    testb $4, %cl
+; SSE2-NEXT:    jne .LBB14_5
+; SSE2-NEXT:  .LBB14_6: # %else4
+; SSE2-NEXT:    testb $8, %cl
+; SSE2-NEXT:    jne .LBB14_7
+; SSE2-NEXT:  .LBB14_8: # %else6
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  .LBB14_1: # %cond.store
 ; SSE2-NEXT:    movb %al, (%rdi)
-; SSE2-NEXT:  .LBB14_2: # %else
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm3
-; SSE2-NEXT:    pxor %xmm3, %xmm2
-; SSE2-NEXT:    pextrw $2, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $2, %cl
 ; SSE2-NEXT:    je .LBB14_4
-; SSE2-NEXT:  # %bb.3: # %cond.store1
-; SSE2-NEXT:    pextrw $2, %xmm0, %eax
-; SSE2-NEXT:    movb %al, 1(%rdi)
-; SSE2-NEXT:  .LBB14_4: # %else2
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm1
-; SSE2-NEXT:    pxor %xmm1, %xmm3
-; SSE2-NEXT:    pextrw $4, %xmm3, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:  .LBB14_3: # %cond.store1
+; SSE2-NEXT:    movb %ah, 1(%rdi)
+; SSE2-NEXT:    testb $4, %cl
 ; SSE2-NEXT:    je .LBB14_6
-; SSE2-NEXT:  # %bb.5: # %cond.store3
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    movb %al, 2(%rdi)
-; SSE2-NEXT:  .LBB14_6: # %else4
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT:    pxor %xmm2, %xmm1
-; SSE2-NEXT:    pextrw $6, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:  .LBB14_5: # %cond.store3
+; SSE2-NEXT:    movl %eax, %edx
+; SSE2-NEXT:    shrl $16, %edx
+; SSE2-NEXT:    movb %dl, 2(%rdi)
+; SSE2-NEXT:    testb $8, %cl
 ; SSE2-NEXT:    je .LBB14_8
-; SSE2-NEXT:  # %bb.7: # %cond.store5
-; SSE2-NEXT:    pextrw $6, %xmm0, %eax
+; SSE2-NEXT:  .LBB14_7: # %cond.store5
+; SSE2-NEXT:    shrl $24, %eax
 ; SSE2-NEXT:    movb %al, 3(%rdi)
-; SSE2-NEXT:  .LBB14_8: # %else6
 ; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: truncstore_v4i32_v4i8:
 ; SSE4:       # %bb.0:
-; SSE4-NEXT:    pxor %xmm3, %xmm3
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm3
-; SSE4-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE4-NEXT:    pxor %xmm3, %xmm2
+; SSE4-NEXT:    pxor %xmm2, %xmm2
 ; SSE4-NEXT:    pminsd {{.*}}(%rip), %xmm0
 ; SSE4-NEXT:    pmaxsd {{.*}}(%rip), %xmm0
-; SSE4-NEXT:    pextrb $0, %xmm2, %eax
+; SSE4-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; SSE4-NEXT:    pcmpeqd %xmm1, %xmm2
+; SSE4-NEXT:    movmskps %xmm2, %eax
+; SSE4-NEXT:    xorl $15, %eax
 ; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB14_2
-; SSE4-NEXT:  # %bb.1: # %cond.store
+; SSE4-NEXT:    jne .LBB14_1
+; SSE4-NEXT:  # %bb.2: # %else
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    jne .LBB14_3
+; SSE4-NEXT:  .LBB14_4: # %else2
+; SSE4-NEXT:    testb $4, %al
+; SSE4-NEXT:    jne .LBB14_5
+; SSE4-NEXT:  .LBB14_6: # %else4
+; SSE4-NEXT:    testb $8, %al
+; SSE4-NEXT:    jne .LBB14_7
+; SSE4-NEXT:  .LBB14_8: # %else6
+; SSE4-NEXT:    retq
+; SSE4-NEXT:  .LBB14_1: # %cond.store
 ; SSE4-NEXT:    pextrb $0, %xmm0, (%rdi)
-; SSE4-NEXT:  .LBB14_2: # %else
-; SSE4-NEXT:    pextrb $4, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $2, %al
 ; SSE4-NEXT:    je .LBB14_4
-; SSE4-NEXT:  # %bb.3: # %cond.store1
-; SSE4-NEXT:    pextrb $4, %xmm0, 1(%rdi)
-; SSE4-NEXT:  .LBB14_4: # %else2
-; SSE4-NEXT:    pxor %xmm2, %xmm2
-; SSE4-NEXT:    pcmpeqd %xmm2, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE4-NEXT:    pxor %xmm1, %xmm2
-; SSE4-NEXT:    pextrb $8, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:  .LBB14_3: # %cond.store1
+; SSE4-NEXT:    pextrb $1, %xmm0, 1(%rdi)
+; SSE4-NEXT:    testb $4, %al
 ; SSE4-NEXT:    je .LBB14_6
-; SSE4-NEXT:  # %bb.5: # %cond.store3
-; SSE4-NEXT:    pextrb $8, %xmm0, 2(%rdi)
-; SSE4-NEXT:  .LBB14_6: # %else4
-; SSE4-NEXT:    pextrb $12, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:  .LBB14_5: # %cond.store3
+; SSE4-NEXT:    pextrb $2, %xmm0, 2(%rdi)
+; SSE4-NEXT:    testb $8, %al
 ; SSE4-NEXT:    je .LBB14_8
-; SSE4-NEXT:  # %bb.7: # %cond.store5
-; SSE4-NEXT:    pextrb $12, %xmm0, 3(%rdi)
-; SSE4-NEXT:  .LBB14_8: # %else6
+; SSE4-NEXT:  .LBB14_7: # %cond.store5
+; SSE4-NEXT:    pextrb $3, %xmm0, 3(%rdi)
 ; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: truncstore_v4i32_v4i8:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vpminsd {{.*}}(%rip), %xmm0, %xmm0
 ; AVX1-NEXT:    vpmaxsd {{.*}}(%rip), %xmm0, %xmm0
-; AVX1-NEXT:    vpextrb $0, %xmm2, %eax
+; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vmovmskps %xmm1, %eax
+; AVX1-NEXT:    xorl $15, %eax
 ; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je .LBB14_2
-; AVX1-NEXT:  # %bb.1: # %cond.store
+; AVX1-NEXT:    jne .LBB14_1
+; AVX1-NEXT:  # %bb.2: # %else
+; AVX1-NEXT:    testb $2, %al
+; AVX1-NEXT:    jne .LBB14_3
+; AVX1-NEXT:  .LBB14_4: # %else2
+; AVX1-NEXT:    testb $4, %al
+; AVX1-NEXT:    jne .LBB14_5
+; AVX1-NEXT:  .LBB14_6: # %else4
+; AVX1-NEXT:    testb $8, %al
+; AVX1-NEXT:    jne .LBB14_7
+; AVX1-NEXT:  .LBB14_8: # %else6
+; AVX1-NEXT:    retq
+; AVX1-NEXT:  .LBB14_1: # %cond.store
 ; AVX1-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX1-NEXT:  .LBB14_2: # %else
-; AVX1-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $2, %al
 ; AVX1-NEXT:    je .LBB14_4
-; AVX1-NEXT:  # %bb.3: # %cond.store1
-; AVX1-NEXT:    vpextrb $4, %xmm0, 1(%rdi)
-; AVX1-NEXT:  .LBB14_4: # %else2
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:  .LBB14_3: # %cond.store1
+; AVX1-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
+; AVX1-NEXT:    testb $4, %al
 ; AVX1-NEXT:    je .LBB14_6
-; AVX1-NEXT:  # %bb.5: # %cond.store3
-; AVX1-NEXT:    vpextrb $8, %xmm0, 2(%rdi)
-; AVX1-NEXT:  .LBB14_6: # %else4
-; AVX1-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:  .LBB14_5: # %cond.store3
+; AVX1-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
+; AVX1-NEXT:    testb $8, %al
 ; AVX1-NEXT:    je .LBB14_8
-; AVX1-NEXT:  # %bb.7: # %cond.store5
-; AVX1-NEXT:    vpextrb $12, %xmm0, 3(%rdi)
-; AVX1-NEXT:  .LBB14_8: # %else6
+; AVX1-NEXT:  .LBB14_7: # %cond.store5
+; AVX1-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: truncstore_v4i32_v4i8:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    vpxor %xmm3, %xmm2, %xmm2
 ; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm3 = [127,127,127,127]
 ; AVX2-NEXT:    vpminsd %xmm3, %xmm0, %xmm0
 ; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm3 = [4294967168,4294967168,4294967168,4294967168]
 ; AVX2-NEXT:    vpmaxsd %xmm3, %xmm0, %xmm0
-; AVX2-NEXT:    vpextrb $0, %xmm2, %eax
+; AVX2-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vmovmskps %xmm1, %eax
+; AVX2-NEXT:    xorl $15, %eax
 ; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je .LBB14_2
-; AVX2-NEXT:  # %bb.1: # %cond.store
+; AVX2-NEXT:    jne .LBB14_1
+; AVX2-NEXT:  # %bb.2: # %else
+; AVX2-NEXT:    testb $2, %al
+; AVX2-NEXT:    jne .LBB14_3
+; AVX2-NEXT:  .LBB14_4: # %else2
+; AVX2-NEXT:    testb $4, %al
+; AVX2-NEXT:    jne .LBB14_5
+; AVX2-NEXT:  .LBB14_6: # %else4
+; AVX2-NEXT:    testb $8, %al
+; AVX2-NEXT:    jne .LBB14_7
+; AVX2-NEXT:  .LBB14_8: # %else6
+; AVX2-NEXT:    retq
+; AVX2-NEXT:  .LBB14_1: # %cond.store
 ; AVX2-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX2-NEXT:  .LBB14_2: # %else
-; AVX2-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $2, %al
 ; AVX2-NEXT:    je .LBB14_4
-; AVX2-NEXT:  # %bb.3: # %cond.store1
-; AVX2-NEXT:    vpextrb $4, %xmm0, 1(%rdi)
-; AVX2-NEXT:  .LBB14_4: # %else2
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:  .LBB14_3: # %cond.store1
+; AVX2-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
+; AVX2-NEXT:    testb $4, %al
 ; AVX2-NEXT:    je .LBB14_6
-; AVX2-NEXT:  # %bb.5: # %cond.store3
-; AVX2-NEXT:    vpextrb $8, %xmm0, 2(%rdi)
-; AVX2-NEXT:  .LBB14_6: # %else4
-; AVX2-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:  .LBB14_5: # %cond.store3
+; AVX2-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
+; AVX2-NEXT:    testb $8, %al
 ; AVX2-NEXT:    je .LBB14_8
-; AVX2-NEXT:  # %bb.7: # %cond.store5
-; AVX2-NEXT:    vpextrb $12, %xmm0, 3(%rdi)
-; AVX2-NEXT:  .LBB14_8: # %else6
+; AVX2-NEXT:  .LBB14_7: # %cond.store5
+; AVX2-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
 ; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: truncstore_v4i32_v4i8:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [127,127,127,127]
-; AVX512F-NEXT:    vpminsd %xmm2, %xmm0, %xmm0
-; AVX512F-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [4294967168,4294967168,4294967168,4294967168]
-; AVX512F-NEXT:    vpmaxsd %xmm2, %xmm0, %xmm0
+; AVX512F-NEXT:    vpmovsdb %zmm0, %xmm0
 ; AVX512F-NEXT:    kmovw %k0, %eax
 ; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB14_2
-; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    jne .LBB14_1
+; AVX512F-NEXT:  # %bb.2: # %else
+; AVX512F-NEXT:    testb $2, %al
+; AVX512F-NEXT:    jne .LBB14_3
+; AVX512F-NEXT:  .LBB14_4: # %else2
+; AVX512F-NEXT:    testb $4, %al
+; AVX512F-NEXT:    jne .LBB14_5
+; AVX512F-NEXT:  .LBB14_6: # %else4
+; AVX512F-NEXT:    testb $8, %al
+; AVX512F-NEXT:    jne .LBB14_7
+; AVX512F-NEXT:  .LBB14_8: # %else6
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+; AVX512F-NEXT:  .LBB14_1: # %cond.store
 ; AVX512F-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX512F-NEXT:  .LBB14_2: # %else
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $2, %al
 ; AVX512F-NEXT:    je .LBB14_4
-; AVX512F-NEXT:  # %bb.3: # %cond.store1
-; AVX512F-NEXT:    vpextrb $4, %xmm0, 1(%rdi)
-; AVX512F-NEXT:  .LBB14_4: # %else2
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  .LBB14_3: # %cond.store1
+; AVX512F-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
+; AVX512F-NEXT:    testb $4, %al
 ; AVX512F-NEXT:    je .LBB14_6
-; AVX512F-NEXT:  # %bb.5: # %cond.store3
-; AVX512F-NEXT:    vpextrb $8, %xmm0, 2(%rdi)
-; AVX512F-NEXT:  .LBB14_6: # %else4
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  .LBB14_5: # %cond.store3
+; AVX512F-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
+; AVX512F-NEXT:    testb $8, %al
 ; AVX512F-NEXT:    je .LBB14_8
-; AVX512F-NEXT:  # %bb.7: # %cond.store5
-; AVX512F-NEXT:    vpextrb $12, %xmm0, 3(%rdi)
-; AVX512F-NEXT:  .LBB14_8: # %else6
+; AVX512F-NEXT:  .LBB14_7: # %cond.store5
+; AVX512F-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: truncstore_v4i32_v4i8:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; AVX512BW-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512BW-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [127,127,127,127]
-; AVX512BW-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [4294967168,4294967168,4294967168,4294967168]
-; AVX512BW-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
 ; AVX512BW-NEXT:    kshiftlq $60, %k0, %k0
 ; AVX512BW-NEXT:    kshiftrq $60, %k0, %k1
+; AVX512BW-NEXT:    vpmovsdb %zmm0, %xmm0
 ; AVX512BW-NEXT:    vmovdqu8 %zmm0, (%rdi) {%k1}
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -5899,1503 +5447,1164 @@ define void @truncstore_v32i16_v32i8(<32 x i16> %x, <32 x i8>* %p, <32 x i8> %ma
 ; SSE2-LABEL: truncstore_v32i16_v32i8:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pxor %xmm6, %xmm6
-; SSE2-NEXT:    pcmpeqb %xmm4, %xmm6
 ; SSE2-NEXT:    packsswb %xmm1, %xmm0
-; SSE2-NEXT:    movd %xmm6, %ecx
-; SSE2-NEXT:    movl %ecx, %eax
-; SSE2-NEXT:    notb %al
+; SSE2-NEXT:    pcmpeqb %xmm6, %xmm4
+; SSE2-NEXT:    pmovmskb %xmm4, %ecx
+; SSE2-NEXT:    xorl $65535, %ecx # imm = 0xFFFF
+; SSE2-NEXT:    pcmpeqb %xmm6, %xmm5
+; SSE2-NEXT:    pmovmskb %xmm5, %eax
+; SSE2-NEXT:    notl %eax
+; SSE2-NEXT:    shll $16, %eax
+; SSE2-NEXT:    orl %ecx, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    je .LBB15_2
-; SSE2-NEXT:  # %bb.1: # %cond.store
-; SSE2-NEXT:    movb %al, (%rdi)
-; SSE2-NEXT:  .LBB15_2: # %else
-; SSE2-NEXT:    shrl $8, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    je .LBB15_4
-; SSE2-NEXT:  # %bb.3: # %cond.store1
-; SSE2-NEXT:    movb %ah, 1(%rdi)
+; SSE2-NEXT:    movd %xmm0, %ecx
+; SSE2-NEXT:    jne .LBB15_1
+; SSE2-NEXT:  # %bb.2: # %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne .LBB15_3
 ; SSE2-NEXT:  .LBB15_4: # %else2
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqb %xmm4, %xmm1
-; SSE2-NEXT:    movd %xmm1, %ecx
-; SSE2-NEXT:    movl %ecx, %edx
-; SSE2-NEXT:    shrl $16, %edx
-; SSE2-NEXT:    notb %dl
-; SSE2-NEXT:    testb $1, %dl
-; SSE2-NEXT:    je .LBB15_6
-; SSE2-NEXT:  # %bb.5: # %cond.store3
-; SSE2-NEXT:    movl %eax, %edx
-; SSE2-NEXT:    shrl $16, %edx
-; SSE2-NEXT:    movb %dl, 2(%rdi)
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne .LBB15_5
 ; SSE2-NEXT:  .LBB15_6: # %else4
-; SSE2-NEXT:    shrl $24, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je .LBB15_8
-; SSE2-NEXT:  # %bb.7: # %cond.store5
-; SSE2-NEXT:    shrl $24, %eax
-; SSE2-NEXT:    movb %al, 3(%rdi)
+; SSE2-NEXT:  .LBB15_7: # %cond.store5
+; SSE2-NEXT:    shrl $24, %ecx
+; SSE2-NEXT:    movb %cl, 3(%rdi)
 ; SSE2-NEXT:  .LBB15_8: # %else6
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqb %xmm4, %xmm1
-; SSE2-NEXT:    pextrw $2, %xmm1, %eax
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testb $16, %al
 ; SSE2-NEXT:    pextrw $2, %xmm0, %ecx
 ; SSE2-NEXT:    je .LBB15_10
 ; SSE2-NEXT:  # %bb.9: # %cond.store7
 ; SSE2-NEXT:    movb %cl, 4(%rdi)
 ; SSE2-NEXT:  .LBB15_10: # %else8
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    notb %al
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $32, %al
 ; SSE2-NEXT:    je .LBB15_12
 ; SSE2-NEXT:  # %bb.11: # %cond.store9
 ; SSE2-NEXT:    movb %ch, 5(%rdi)
 ; SSE2-NEXT:  .LBB15_12: # %else10
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqb %xmm4, %xmm1
-; SSE2-NEXT:    pextrw $3, %xmm1, %eax
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testb $64, %al
 ; SSE2-NEXT:    pextrw $3, %xmm0, %ecx
 ; SSE2-NEXT:    je .LBB15_14
 ; SSE2-NEXT:  # %bb.13: # %cond.store11
 ; SSE2-NEXT:    movb %cl, 6(%rdi)
 ; SSE2-NEXT:  .LBB15_14: # %else12
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    notb %al
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $-128, %al
 ; SSE2-NEXT:    je .LBB15_16
 ; SSE2-NEXT:  # %bb.15: # %cond.store13
 ; SSE2-NEXT:    movb %ch, 7(%rdi)
 ; SSE2-NEXT:  .LBB15_16: # %else14
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqb %xmm4, %xmm1
-; SSE2-NEXT:    pextrw $4, %xmm1, %eax
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $256, %eax # imm = 0x100
 ; SSE2-NEXT:    pextrw $4, %xmm0, %ecx
 ; SSE2-NEXT:    je .LBB15_18
 ; SSE2-NEXT:  # %bb.17: # %cond.store15
 ; SSE2-NEXT:    movb %cl, 8(%rdi)
 ; SSE2-NEXT:  .LBB15_18: # %else16
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    notb %al
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $512, %eax # imm = 0x200
 ; SSE2-NEXT:    je .LBB15_20
 ; SSE2-NEXT:  # %bb.19: # %cond.store17
 ; SSE2-NEXT:    movb %ch, 9(%rdi)
 ; SSE2-NEXT:  .LBB15_20: # %else18
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqb %xmm4, %xmm1
-; SSE2-NEXT:    pextrw $5, %xmm1, %eax
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $1024, %eax # imm = 0x400
 ; SSE2-NEXT:    pextrw $5, %xmm0, %ecx
 ; SSE2-NEXT:    je .LBB15_22
 ; SSE2-NEXT:  # %bb.21: # %cond.store19
 ; SSE2-NEXT:    movb %cl, 10(%rdi)
 ; SSE2-NEXT:  .LBB15_22: # %else20
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    notb %al
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $2048, %eax # imm = 0x800
 ; SSE2-NEXT:    je .LBB15_24
 ; SSE2-NEXT:  # %bb.23: # %cond.store21
 ; SSE2-NEXT:    movb %ch, 11(%rdi)
 ; SSE2-NEXT:  .LBB15_24: # %else22
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqb %xmm4, %xmm1
-; SSE2-NEXT:    pextrw $6, %xmm1, %eax
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $4096, %eax # imm = 0x1000
 ; SSE2-NEXT:    pextrw $6, %xmm0, %ecx
 ; SSE2-NEXT:    je .LBB15_26
 ; SSE2-NEXT:  # %bb.25: # %cond.store23
 ; SSE2-NEXT:    movb %cl, 12(%rdi)
 ; SSE2-NEXT:  .LBB15_26: # %else24
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    notb %al
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $8192, %eax # imm = 0x2000
 ; SSE2-NEXT:    je .LBB15_28
 ; SSE2-NEXT:  # %bb.27: # %cond.store25
 ; SSE2-NEXT:    movb %ch, 13(%rdi)
 ; SSE2-NEXT:  .LBB15_28: # %else26
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqb %xmm1, %xmm4
-; SSE2-NEXT:    pextrw $7, %xmm4, %eax
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $16384, %eax # imm = 0x4000
 ; SSE2-NEXT:    pextrw $7, %xmm0, %ecx
 ; SSE2-NEXT:    je .LBB15_30
 ; SSE2-NEXT:  # %bb.29: # %cond.store27
 ; SSE2-NEXT:    movb %cl, 14(%rdi)
 ; SSE2-NEXT:  .LBB15_30: # %else28
 ; SSE2-NEXT:    packsswb %xmm3, %xmm2
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    notb %al
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $32768, %eax # imm = 0x8000
 ; SSE2-NEXT:    je .LBB15_32
 ; SSE2-NEXT:  # %bb.31: # %cond.store29
 ; SSE2-NEXT:    movb %ch, 15(%rdi)
 ; SSE2-NEXT:  .LBB15_32: # %else30
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqb %xmm5, %xmm0
-; SSE2-NEXT:    movd %xmm0, %ecx
-; SSE2-NEXT:    movl %ecx, %eax
-; SSE2-NEXT:    notb %al
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    movd %xmm2, %eax
-; SSE2-NEXT:    je .LBB15_34
-; SSE2-NEXT:  # %bb.33: # %cond.store31
-; SSE2-NEXT:    movb %al, 16(%rdi)
-; SSE2-NEXT:  .LBB15_34: # %else32
-; SSE2-NEXT:    shrl $8, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    je .LBB15_36
-; SSE2-NEXT:  # %bb.35: # %cond.store33
-; SSE2-NEXT:    movb %ah, 17(%rdi)
+; SSE2-NEXT:    testl $65536, %eax # imm = 0x10000
+; SSE2-NEXT:    movd %xmm2, %ecx
+; SSE2-NEXT:    jne .LBB15_33
+; SSE2-NEXT:  # %bb.34: # %else32
+; SSE2-NEXT:    testl $131072, %eax # imm = 0x20000
+; SSE2-NEXT:    jne .LBB15_35
 ; SSE2-NEXT:  .LBB15_36: # %else34
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqb %xmm5, %xmm0
-; SSE2-NEXT:    movd %xmm0, %ecx
-; SSE2-NEXT:    movl %ecx, %edx
-; SSE2-NEXT:    shrl $16, %edx
-; SSE2-NEXT:    notb %dl
-; SSE2-NEXT:    testb $1, %dl
-; SSE2-NEXT:    je .LBB15_38
-; SSE2-NEXT:  # %bb.37: # %cond.store35
-; SSE2-NEXT:    movl %eax, %edx
-; SSE2-NEXT:    shrl $16, %edx
-; SSE2-NEXT:    movb %dl, 18(%rdi)
+; SSE2-NEXT:    testl $262144, %eax # imm = 0x40000
+; SSE2-NEXT:    jne .LBB15_37
 ; SSE2-NEXT:  .LBB15_38: # %else36
-; SSE2-NEXT:    shrl $24, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $524288, %eax # imm = 0x80000
 ; SSE2-NEXT:    je .LBB15_40
-; SSE2-NEXT:  # %bb.39: # %cond.store37
-; SSE2-NEXT:    shrl $24, %eax
-; SSE2-NEXT:    movb %al, 19(%rdi)
+; SSE2-NEXT:  .LBB15_39: # %cond.store37
+; SSE2-NEXT:    shrl $24, %ecx
+; SSE2-NEXT:    movb %cl, 19(%rdi)
 ; SSE2-NEXT:  .LBB15_40: # %else38
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqb %xmm5, %xmm0
-; SSE2-NEXT:    pextrw $2, %xmm0, %eax
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $1048576, %eax # imm = 0x100000
 ; SSE2-NEXT:    pextrw $2, %xmm2, %ecx
 ; SSE2-NEXT:    je .LBB15_42
 ; SSE2-NEXT:  # %bb.41: # %cond.store39
 ; SSE2-NEXT:    movb %cl, 20(%rdi)
 ; SSE2-NEXT:  .LBB15_42: # %else40
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    notb %al
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $2097152, %eax # imm = 0x200000
 ; SSE2-NEXT:    je .LBB15_44
 ; SSE2-NEXT:  # %bb.43: # %cond.store41
 ; SSE2-NEXT:    movb %ch, 21(%rdi)
 ; SSE2-NEXT:  .LBB15_44: # %else42
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqb %xmm5, %xmm0
-; SSE2-NEXT:    pextrw $3, %xmm0, %eax
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $4194304, %eax # imm = 0x400000
 ; SSE2-NEXT:    pextrw $3, %xmm2, %ecx
 ; SSE2-NEXT:    je .LBB15_46
 ; SSE2-NEXT:  # %bb.45: # %cond.store43
 ; SSE2-NEXT:    movb %cl, 22(%rdi)
 ; SSE2-NEXT:  .LBB15_46: # %else44
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    notb %al
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $8388608, %eax # imm = 0x800000
 ; SSE2-NEXT:    je .LBB15_48
 ; SSE2-NEXT:  # %bb.47: # %cond.store45
 ; SSE2-NEXT:    movb %ch, 23(%rdi)
 ; SSE2-NEXT:  .LBB15_48: # %else46
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqb %xmm5, %xmm0
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $16777216, %eax # imm = 0x1000000
 ; SSE2-NEXT:    pextrw $4, %xmm2, %ecx
 ; SSE2-NEXT:    je .LBB15_50
 ; SSE2-NEXT:  # %bb.49: # %cond.store47
 ; SSE2-NEXT:    movb %cl, 24(%rdi)
 ; SSE2-NEXT:  .LBB15_50: # %else48
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    notb %al
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $33554432, %eax # imm = 0x2000000
 ; SSE2-NEXT:    je .LBB15_52
 ; SSE2-NEXT:  # %bb.51: # %cond.store49
 ; SSE2-NEXT:    movb %ch, 25(%rdi)
 ; SSE2-NEXT:  .LBB15_52: # %else50
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqb %xmm5, %xmm0
-; SSE2-NEXT:    pextrw $5, %xmm0, %eax
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $67108864, %eax # imm = 0x4000000
 ; SSE2-NEXT:    pextrw $5, %xmm2, %ecx
 ; SSE2-NEXT:    je .LBB15_54
 ; SSE2-NEXT:  # %bb.53: # %cond.store51
 ; SSE2-NEXT:    movb %cl, 26(%rdi)
 ; SSE2-NEXT:  .LBB15_54: # %else52
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    notb %al
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $134217728, %eax # imm = 0x8000000
 ; SSE2-NEXT:    je .LBB15_56
 ; SSE2-NEXT:  # %bb.55: # %cond.store53
 ; SSE2-NEXT:    movb %ch, 27(%rdi)
 ; SSE2-NEXT:  .LBB15_56: # %else54
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqb %xmm5, %xmm0
-; SSE2-NEXT:    pextrw $6, %xmm0, %eax
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $268435456, %eax # imm = 0x10000000
 ; SSE2-NEXT:    pextrw $6, %xmm2, %ecx
 ; SSE2-NEXT:    je .LBB15_58
 ; SSE2-NEXT:  # %bb.57: # %cond.store55
 ; SSE2-NEXT:    movb %cl, 28(%rdi)
 ; SSE2-NEXT:  .LBB15_58: # %else56
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    notb %al
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $536870912, %eax # imm = 0x20000000
 ; SSE2-NEXT:    je .LBB15_60
 ; SSE2-NEXT:  # %bb.59: # %cond.store57
 ; SSE2-NEXT:    movb %ch, 29(%rdi)
 ; SSE2-NEXT:  .LBB15_60: # %else58
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqb %xmm0, %xmm5
-; SSE2-NEXT:    pextrw $7, %xmm5, %eax
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $1073741824, %eax # imm = 0x40000000
 ; SSE2-NEXT:    pextrw $7, %xmm2, %ecx
-; SSE2-NEXT:    je .LBB15_62
-; SSE2-NEXT:  # %bb.61: # %cond.store59
+; SSE2-NEXT:    jne .LBB15_61
+; SSE2-NEXT:  # %bb.62: # %else60
+; SSE2-NEXT:    testl $-2147483648, %eax # imm = 0x80000000
+; SSE2-NEXT:    jne .LBB15_63
+; SSE2-NEXT:  .LBB15_64: # %else62
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  .LBB15_1: # %cond.store
+; SSE2-NEXT:    movb %cl, (%rdi)
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    je .LBB15_4
+; SSE2-NEXT:  .LBB15_3: # %cond.store1
+; SSE2-NEXT:    movb %ch, 1(%rdi)
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    je .LBB15_6
+; SSE2-NEXT:  .LBB15_5: # %cond.store3
+; SSE2-NEXT:    movl %ecx, %edx
+; SSE2-NEXT:    shrl $16, %edx
+; SSE2-NEXT:    movb %dl, 2(%rdi)
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne .LBB15_7
+; SSE2-NEXT:    jmp .LBB15_8
+; SSE2-NEXT:  .LBB15_33: # %cond.store31
+; SSE2-NEXT:    movb %cl, 16(%rdi)
+; SSE2-NEXT:    testl $131072, %eax # imm = 0x20000
+; SSE2-NEXT:    je .LBB15_36
+; SSE2-NEXT:  .LBB15_35: # %cond.store33
+; SSE2-NEXT:    movb %ch, 17(%rdi)
+; SSE2-NEXT:    testl $262144, %eax # imm = 0x40000
+; SSE2-NEXT:    je .LBB15_38
+; SSE2-NEXT:  .LBB15_37: # %cond.store35
+; SSE2-NEXT:    movl %ecx, %edx
+; SSE2-NEXT:    shrl $16, %edx
+; SSE2-NEXT:    movb %dl, 18(%rdi)
+; SSE2-NEXT:    testl $524288, %eax # imm = 0x80000
+; SSE2-NEXT:    jne .LBB15_39
+; SSE2-NEXT:    jmp .LBB15_40
+; SSE2-NEXT:  .LBB15_61: # %cond.store59
 ; SSE2-NEXT:    movb %cl, 30(%rdi)
-; SSE2-NEXT:  .LBB15_62: # %else60
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    notb %al
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $-2147483648, %eax # imm = 0x80000000
 ; SSE2-NEXT:    je .LBB15_64
-; SSE2-NEXT:  # %bb.63: # %cond.store61
+; SSE2-NEXT:  .LBB15_63: # %cond.store61
 ; SSE2-NEXT:    movb %ch, 31(%rdi)
-; SSE2-NEXT:  .LBB15_64: # %else62
 ; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: truncstore_v32i16_v32i8:
 ; SSE4:       # %bb.0:
 ; SSE4-NEXT:    pxor %xmm6, %xmm6
-; SSE4-NEXT:    pcmpeqb %xmm4, %xmm6
 ; SSE4-NEXT:    packsswb %xmm1, %xmm0
-; SSE4-NEXT:    pextrb $0, %xmm6, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB15_2
-; SSE4-NEXT:  # %bb.1: # %cond.store
+; SSE4-NEXT:    pcmpeqb %xmm6, %xmm4
+; SSE4-NEXT:    pmovmskb %xmm4, %ecx
+; SSE4-NEXT:    xorl $65535, %ecx # imm = 0xFFFF
+; SSE4-NEXT:    pcmpeqb %xmm6, %xmm5
+; SSE4-NEXT:    pmovmskb %xmm5, %eax
+; SSE4-NEXT:    notl %eax
+; SSE4-NEXT:    shll $16, %eax
+; SSE4-NEXT:    orl %ecx, %eax
+; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    jne .LBB15_1
+; SSE4-NEXT:  # %bb.2: # %else
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    jne .LBB15_3
+; SSE4-NEXT:  .LBB15_4: # %else2
+; SSE4-NEXT:    testb $4, %al
+; SSE4-NEXT:    jne .LBB15_5
+; SSE4-NEXT:  .LBB15_6: # %else4
+; SSE4-NEXT:    testb $8, %al
+; SSE4-NEXT:    jne .LBB15_7
+; SSE4-NEXT:  .LBB15_8: # %else6
+; SSE4-NEXT:    testb $16, %al
+; SSE4-NEXT:    jne .LBB15_9
+; SSE4-NEXT:  .LBB15_10: # %else8
+; SSE4-NEXT:    testb $32, %al
+; SSE4-NEXT:    jne .LBB15_11
+; SSE4-NEXT:  .LBB15_12: # %else10
+; SSE4-NEXT:    testb $64, %al
+; SSE4-NEXT:    jne .LBB15_13
+; SSE4-NEXT:  .LBB15_14: # %else12
+; SSE4-NEXT:    testb $-128, %al
+; SSE4-NEXT:    jne .LBB15_15
+; SSE4-NEXT:  .LBB15_16: # %else14
+; SSE4-NEXT:    testl $256, %eax # imm = 0x100
+; SSE4-NEXT:    jne .LBB15_17
+; SSE4-NEXT:  .LBB15_18: # %else16
+; SSE4-NEXT:    testl $512, %eax # imm = 0x200
+; SSE4-NEXT:    jne .LBB15_19
+; SSE4-NEXT:  .LBB15_20: # %else18
+; SSE4-NEXT:    testl $1024, %eax # imm = 0x400
+; SSE4-NEXT:    jne .LBB15_21
+; SSE4-NEXT:  .LBB15_22: # %else20
+; SSE4-NEXT:    testl $2048, %eax # imm = 0x800
+; SSE4-NEXT:    jne .LBB15_23
+; SSE4-NEXT:  .LBB15_24: # %else22
+; SSE4-NEXT:    testl $4096, %eax # imm = 0x1000
+; SSE4-NEXT:    jne .LBB15_25
+; SSE4-NEXT:  .LBB15_26: # %else24
+; SSE4-NEXT:    testl $8192, %eax # imm = 0x2000
+; SSE4-NEXT:    jne .LBB15_27
+; SSE4-NEXT:  .LBB15_28: # %else26
+; SSE4-NEXT:    testl $16384, %eax # imm = 0x4000
+; SSE4-NEXT:    jne .LBB15_29
+; SSE4-NEXT:  .LBB15_30: # %else28
+; SSE4-NEXT:    testl $32768, %eax # imm = 0x8000
+; SSE4-NEXT:    je .LBB15_32
+; SSE4-NEXT:  .LBB15_31: # %cond.store29
+; SSE4-NEXT:    pextrb $15, %xmm0, 15(%rdi)
+; SSE4-NEXT:  .LBB15_32: # %else30
+; SSE4-NEXT:    packsswb %xmm3, %xmm2
+; SSE4-NEXT:    testl $65536, %eax # imm = 0x10000
+; SSE4-NEXT:    jne .LBB15_33
+; SSE4-NEXT:  # %bb.34: # %else32
+; SSE4-NEXT:    testl $131072, %eax # imm = 0x20000
+; SSE4-NEXT:    jne .LBB15_35
+; SSE4-NEXT:  .LBB15_36: # %else34
+; SSE4-NEXT:    testl $262144, %eax # imm = 0x40000
+; SSE4-NEXT:    jne .LBB15_37
+; SSE4-NEXT:  .LBB15_38: # %else36
+; SSE4-NEXT:    testl $524288, %eax # imm = 0x80000
+; SSE4-NEXT:    jne .LBB15_39
+; SSE4-NEXT:  .LBB15_40: # %else38
+; SSE4-NEXT:    testl $1048576, %eax # imm = 0x100000
+; SSE4-NEXT:    jne .LBB15_41
+; SSE4-NEXT:  .LBB15_42: # %else40
+; SSE4-NEXT:    testl $2097152, %eax # imm = 0x200000
+; SSE4-NEXT:    jne .LBB15_43
+; SSE4-NEXT:  .LBB15_44: # %else42
+; SSE4-NEXT:    testl $4194304, %eax # imm = 0x400000
+; SSE4-NEXT:    jne .LBB15_45
+; SSE4-NEXT:  .LBB15_46: # %else44
+; SSE4-NEXT:    testl $8388608, %eax # imm = 0x800000
+; SSE4-NEXT:    jne .LBB15_47
+; SSE4-NEXT:  .LBB15_48: # %else46
+; SSE4-NEXT:    testl $16777216, %eax # imm = 0x1000000
+; SSE4-NEXT:    jne .LBB15_49
+; SSE4-NEXT:  .LBB15_50: # %else48
+; SSE4-NEXT:    testl $33554432, %eax # imm = 0x2000000
+; SSE4-NEXT:    jne .LBB15_51
+; SSE4-NEXT:  .LBB15_52: # %else50
+; SSE4-NEXT:    testl $67108864, %eax # imm = 0x4000000
+; SSE4-NEXT:    jne .LBB15_53
+; SSE4-NEXT:  .LBB15_54: # %else52
+; SSE4-NEXT:    testl $134217728, %eax # imm = 0x8000000
+; SSE4-NEXT:    jne .LBB15_55
+; SSE4-NEXT:  .LBB15_56: # %else54
+; SSE4-NEXT:    testl $268435456, %eax # imm = 0x10000000
+; SSE4-NEXT:    jne .LBB15_57
+; SSE4-NEXT:  .LBB15_58: # %else56
+; SSE4-NEXT:    testl $536870912, %eax # imm = 0x20000000
+; SSE4-NEXT:    jne .LBB15_59
+; SSE4-NEXT:  .LBB15_60: # %else58
+; SSE4-NEXT:    testl $1073741824, %eax # imm = 0x40000000
+; SSE4-NEXT:    jne .LBB15_61
+; SSE4-NEXT:  .LBB15_62: # %else60
+; SSE4-NEXT:    testl $-2147483648, %eax # imm = 0x80000000
+; SSE4-NEXT:    jne .LBB15_63
+; SSE4-NEXT:  .LBB15_64: # %else62
+; SSE4-NEXT:    retq
+; SSE4-NEXT:  .LBB15_1: # %cond.store
 ; SSE4-NEXT:    pextrb $0, %xmm0, (%rdi)
-; SSE4-NEXT:  .LBB15_2: # %else
-; SSE4-NEXT:    pextrb $1, %xmm6, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $2, %al
 ; SSE4-NEXT:    je .LBB15_4
-; SSE4-NEXT:  # %bb.3: # %cond.store1
+; SSE4-NEXT:  .LBB15_3: # %cond.store1
 ; SSE4-NEXT:    pextrb $1, %xmm0, 1(%rdi)
-; SSE4-NEXT:  .LBB15_4: # %else2
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqb %xmm4, %xmm1
-; SSE4-NEXT:    pextrb $2, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $4, %al
 ; SSE4-NEXT:    je .LBB15_6
-; SSE4-NEXT:  # %bb.5: # %cond.store3
+; SSE4-NEXT:  .LBB15_5: # %cond.store3
 ; SSE4-NEXT:    pextrb $2, %xmm0, 2(%rdi)
-; SSE4-NEXT:  .LBB15_6: # %else4
-; SSE4-NEXT:    pextrb $3, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $8, %al
 ; SSE4-NEXT:    je .LBB15_8
-; SSE4-NEXT:  # %bb.7: # %cond.store5
+; SSE4-NEXT:  .LBB15_7: # %cond.store5
 ; SSE4-NEXT:    pextrb $3, %xmm0, 3(%rdi)
-; SSE4-NEXT:  .LBB15_8: # %else6
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqb %xmm4, %xmm1
-; SSE4-NEXT:    pextrb $4, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $16, %al
 ; SSE4-NEXT:    je .LBB15_10
-; SSE4-NEXT:  # %bb.9: # %cond.store7
+; SSE4-NEXT:  .LBB15_9: # %cond.store7
 ; SSE4-NEXT:    pextrb $4, %xmm0, 4(%rdi)
-; SSE4-NEXT:  .LBB15_10: # %else8
-; SSE4-NEXT:    pextrb $5, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $32, %al
 ; SSE4-NEXT:    je .LBB15_12
-; SSE4-NEXT:  # %bb.11: # %cond.store9
+; SSE4-NEXT:  .LBB15_11: # %cond.store9
 ; SSE4-NEXT:    pextrb $5, %xmm0, 5(%rdi)
-; SSE4-NEXT:  .LBB15_12: # %else10
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqb %xmm4, %xmm1
-; SSE4-NEXT:    pextrb $6, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $64, %al
 ; SSE4-NEXT:    je .LBB15_14
-; SSE4-NEXT:  # %bb.13: # %cond.store11
+; SSE4-NEXT:  .LBB15_13: # %cond.store11
 ; SSE4-NEXT:    pextrb $6, %xmm0, 6(%rdi)
-; SSE4-NEXT:  .LBB15_14: # %else12
-; SSE4-NEXT:    pextrb $7, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $-128, %al
 ; SSE4-NEXT:    je .LBB15_16
-; SSE4-NEXT:  # %bb.15: # %cond.store13
+; SSE4-NEXT:  .LBB15_15: # %cond.store13
 ; SSE4-NEXT:    pextrb $7, %xmm0, 7(%rdi)
-; SSE4-NEXT:  .LBB15_16: # %else14
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqb %xmm4, %xmm1
-; SSE4-NEXT:    pextrb $8, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $256, %eax # imm = 0x100
 ; SSE4-NEXT:    je .LBB15_18
-; SSE4-NEXT:  # %bb.17: # %cond.store15
+; SSE4-NEXT:  .LBB15_17: # %cond.store15
 ; SSE4-NEXT:    pextrb $8, %xmm0, 8(%rdi)
-; SSE4-NEXT:  .LBB15_18: # %else16
-; SSE4-NEXT:    pextrb $9, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $512, %eax # imm = 0x200
 ; SSE4-NEXT:    je .LBB15_20
-; SSE4-NEXT:  # %bb.19: # %cond.store17
+; SSE4-NEXT:  .LBB15_19: # %cond.store17
 ; SSE4-NEXT:    pextrb $9, %xmm0, 9(%rdi)
-; SSE4-NEXT:  .LBB15_20: # %else18
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqb %xmm4, %xmm1
-; SSE4-NEXT:    pextrb $10, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $1024, %eax # imm = 0x400
 ; SSE4-NEXT:    je .LBB15_22
-; SSE4-NEXT:  # %bb.21: # %cond.store19
+; SSE4-NEXT:  .LBB15_21: # %cond.store19
 ; SSE4-NEXT:    pextrb $10, %xmm0, 10(%rdi)
-; SSE4-NEXT:  .LBB15_22: # %else20
-; SSE4-NEXT:    pextrb $11, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $2048, %eax # imm = 0x800
 ; SSE4-NEXT:    je .LBB15_24
-; SSE4-NEXT:  # %bb.23: # %cond.store21
+; SSE4-NEXT:  .LBB15_23: # %cond.store21
 ; SSE4-NEXT:    pextrb $11, %xmm0, 11(%rdi)
-; SSE4-NEXT:  .LBB15_24: # %else22
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqb %xmm4, %xmm1
-; SSE4-NEXT:    pextrb $12, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $4096, %eax # imm = 0x1000
 ; SSE4-NEXT:    je .LBB15_26
-; SSE4-NEXT:  # %bb.25: # %cond.store23
+; SSE4-NEXT:  .LBB15_25: # %cond.store23
 ; SSE4-NEXT:    pextrb $12, %xmm0, 12(%rdi)
-; SSE4-NEXT:  .LBB15_26: # %else24
-; SSE4-NEXT:    pextrb $13, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $8192, %eax # imm = 0x2000
 ; SSE4-NEXT:    je .LBB15_28
-; SSE4-NEXT:  # %bb.27: # %cond.store25
+; SSE4-NEXT:  .LBB15_27: # %cond.store25
 ; SSE4-NEXT:    pextrb $13, %xmm0, 13(%rdi)
-; SSE4-NEXT:  .LBB15_28: # %else26
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqb %xmm1, %xmm4
-; SSE4-NEXT:    pextrb $14, %xmm4, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $16384, %eax # imm = 0x4000
 ; SSE4-NEXT:    je .LBB15_30
-; SSE4-NEXT:  # %bb.29: # %cond.store27
+; SSE4-NEXT:  .LBB15_29: # %cond.store27
 ; SSE4-NEXT:    pextrb $14, %xmm0, 14(%rdi)
-; SSE4-NEXT:  .LBB15_30: # %else28
-; SSE4-NEXT:    pextrb $15, %xmm4, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB15_32
-; SSE4-NEXT:  # %bb.31: # %cond.store29
-; SSE4-NEXT:    pextrb $15, %xmm0, 15(%rdi)
-; SSE4-NEXT:  .LBB15_32: # %else30
-; SSE4-NEXT:    packsswb %xmm3, %xmm2
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqb %xmm5, %xmm0
-; SSE4-NEXT:    pextrb $0, %xmm0, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB15_34
-; SSE4-NEXT:  # %bb.33: # %cond.store31
+; SSE4-NEXT:    testl $32768, %eax # imm = 0x8000
+; SSE4-NEXT:    jne .LBB15_31
+; SSE4-NEXT:    jmp .LBB15_32
+; SSE4-NEXT:  .LBB15_33: # %cond.store31
 ; SSE4-NEXT:    pextrb $0, %xmm2, 16(%rdi)
-; SSE4-NEXT:  .LBB15_34: # %else32
-; SSE4-NEXT:    pextrb $1, %xmm0, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $131072, %eax # imm = 0x20000
 ; SSE4-NEXT:    je .LBB15_36
-; SSE4-NEXT:  # %bb.35: # %cond.store33
+; SSE4-NEXT:  .LBB15_35: # %cond.store33
 ; SSE4-NEXT:    pextrb $1, %xmm2, 17(%rdi)
-; SSE4-NEXT:  .LBB15_36: # %else34
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqb %xmm5, %xmm0
-; SSE4-NEXT:    pextrb $2, %xmm0, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $262144, %eax # imm = 0x40000
 ; SSE4-NEXT:    je .LBB15_38
-; SSE4-NEXT:  # %bb.37: # %cond.store35
+; SSE4-NEXT:  .LBB15_37: # %cond.store35
 ; SSE4-NEXT:    pextrb $2, %xmm2, 18(%rdi)
-; SSE4-NEXT:  .LBB15_38: # %else36
-; SSE4-NEXT:    pextrb $3, %xmm0, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $524288, %eax # imm = 0x80000
 ; SSE4-NEXT:    je .LBB15_40
-; SSE4-NEXT:  # %bb.39: # %cond.store37
+; SSE4-NEXT:  .LBB15_39: # %cond.store37
 ; SSE4-NEXT:    pextrb $3, %xmm2, 19(%rdi)
-; SSE4-NEXT:  .LBB15_40: # %else38
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqb %xmm5, %xmm0
-; SSE4-NEXT:    pextrb $4, %xmm0, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $1048576, %eax # imm = 0x100000
 ; SSE4-NEXT:    je .LBB15_42
-; SSE4-NEXT:  # %bb.41: # %cond.store39
+; SSE4-NEXT:  .LBB15_41: # %cond.store39
 ; SSE4-NEXT:    pextrb $4, %xmm2, 20(%rdi)
-; SSE4-NEXT:  .LBB15_42: # %else40
-; SSE4-NEXT:    pextrb $5, %xmm0, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $2097152, %eax # imm = 0x200000
 ; SSE4-NEXT:    je .LBB15_44
-; SSE4-NEXT:  # %bb.43: # %cond.store41
+; SSE4-NEXT:  .LBB15_43: # %cond.store41
 ; SSE4-NEXT:    pextrb $5, %xmm2, 21(%rdi)
-; SSE4-NEXT:  .LBB15_44: # %else42
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqb %xmm5, %xmm0
-; SSE4-NEXT:    pextrb $6, %xmm0, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $4194304, %eax # imm = 0x400000
 ; SSE4-NEXT:    je .LBB15_46
-; SSE4-NEXT:  # %bb.45: # %cond.store43
+; SSE4-NEXT:  .LBB15_45: # %cond.store43
 ; SSE4-NEXT:    pextrb $6, %xmm2, 22(%rdi)
-; SSE4-NEXT:  .LBB15_46: # %else44
-; SSE4-NEXT:    pextrb $7, %xmm0, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $8388608, %eax # imm = 0x800000
 ; SSE4-NEXT:    je .LBB15_48
-; SSE4-NEXT:  # %bb.47: # %cond.store45
+; SSE4-NEXT:  .LBB15_47: # %cond.store45
 ; SSE4-NEXT:    pextrb $7, %xmm2, 23(%rdi)
-; SSE4-NEXT:  .LBB15_48: # %else46
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqb %xmm5, %xmm0
-; SSE4-NEXT:    pextrb $8, %xmm0, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $16777216, %eax # imm = 0x1000000
 ; SSE4-NEXT:    je .LBB15_50
-; SSE4-NEXT:  # %bb.49: # %cond.store47
+; SSE4-NEXT:  .LBB15_49: # %cond.store47
 ; SSE4-NEXT:    pextrb $8, %xmm2, 24(%rdi)
-; SSE4-NEXT:  .LBB15_50: # %else48
-; SSE4-NEXT:    pextrb $9, %xmm0, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $33554432, %eax # imm = 0x2000000
 ; SSE4-NEXT:    je .LBB15_52
-; SSE4-NEXT:  # %bb.51: # %cond.store49
+; SSE4-NEXT:  .LBB15_51: # %cond.store49
 ; SSE4-NEXT:    pextrb $9, %xmm2, 25(%rdi)
-; SSE4-NEXT:  .LBB15_52: # %else50
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqb %xmm5, %xmm0
-; SSE4-NEXT:    pextrb $10, %xmm0, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $67108864, %eax # imm = 0x4000000
 ; SSE4-NEXT:    je .LBB15_54
-; SSE4-NEXT:  # %bb.53: # %cond.store51
+; SSE4-NEXT:  .LBB15_53: # %cond.store51
 ; SSE4-NEXT:    pextrb $10, %xmm2, 26(%rdi)
-; SSE4-NEXT:  .LBB15_54: # %else52
-; SSE4-NEXT:    pextrb $11, %xmm0, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $134217728, %eax # imm = 0x8000000
 ; SSE4-NEXT:    je .LBB15_56
-; SSE4-NEXT:  # %bb.55: # %cond.store53
+; SSE4-NEXT:  .LBB15_55: # %cond.store53
 ; SSE4-NEXT:    pextrb $11, %xmm2, 27(%rdi)
-; SSE4-NEXT:  .LBB15_56: # %else54
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqb %xmm5, %xmm0
-; SSE4-NEXT:    pextrb $12, %xmm0, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $268435456, %eax # imm = 0x10000000
 ; SSE4-NEXT:    je .LBB15_58
-; SSE4-NEXT:  # %bb.57: # %cond.store55
+; SSE4-NEXT:  .LBB15_57: # %cond.store55
 ; SSE4-NEXT:    pextrb $12, %xmm2, 28(%rdi)
-; SSE4-NEXT:  .LBB15_58: # %else56
-; SSE4-NEXT:    pextrb $13, %xmm0, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $536870912, %eax # imm = 0x20000000
 ; SSE4-NEXT:    je .LBB15_60
-; SSE4-NEXT:  # %bb.59: # %cond.store57
+; SSE4-NEXT:  .LBB15_59: # %cond.store57
 ; SSE4-NEXT:    pextrb $13, %xmm2, 29(%rdi)
-; SSE4-NEXT:  .LBB15_60: # %else58
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqb %xmm0, %xmm5
-; SSE4-NEXT:    pextrb $14, %xmm5, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $1073741824, %eax # imm = 0x40000000
 ; SSE4-NEXT:    je .LBB15_62
-; SSE4-NEXT:  # %bb.61: # %cond.store59
+; SSE4-NEXT:  .LBB15_61: # %cond.store59
 ; SSE4-NEXT:    pextrb $14, %xmm2, 30(%rdi)
-; SSE4-NEXT:  .LBB15_62: # %else60
-; SSE4-NEXT:    pextrb $15, %xmm5, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $-2147483648, %eax # imm = 0x80000000
 ; SSE4-NEXT:    je .LBB15_64
-; SSE4-NEXT:  # %bb.63: # %cond.store61
+; SSE4-NEXT:  .LBB15_63: # %cond.store61
 ; SSE4-NEXT:    pextrb $15, %xmm2, 31(%rdi)
-; SSE4-NEXT:  .LBB15_64: # %else62
 ; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: truncstore_v32i16_v32i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpeqb %xmm3, %xmm2, %xmm5
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm6
-; AVX1-NEXT:    vpacksswb %xmm6, %xmm1, %xmm1
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm6
-; AVX1-NEXT:    vpacksswb %xmm6, %xmm0, %xmm0
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
+; AVX1-NEXT:    vpacksswb %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; AVX1-NEXT:    vpacksswb %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX1-NEXT:    vpextrb $0, %xmm5, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je .LBB15_2
-; AVX1-NEXT:  # %bb.1: # %cond.store
+; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm3
+; AVX1-NEXT:    vpmovmskb %xmm3, %ecx
+; AVX1-NEXT:    xorl $65535, %ecx # imm = 0xFFFF
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm2
+; AVX1-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vpmovmskb %xmm1, %eax
+; AVX1-NEXT:    notl %eax
+; AVX1-NEXT:    shll $16, %eax
+; AVX1-NEXT:    orl %ecx, %eax
+; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    jne .LBB15_1
+; AVX1-NEXT:  # %bb.2: # %else
+; AVX1-NEXT:    testb $2, %al
+; AVX1-NEXT:    jne .LBB15_3
+; AVX1-NEXT:  .LBB15_4: # %else2
+; AVX1-NEXT:    testb $4, %al
+; AVX1-NEXT:    jne .LBB15_5
+; AVX1-NEXT:  .LBB15_6: # %else4
+; AVX1-NEXT:    testb $8, %al
+; AVX1-NEXT:    jne .LBB15_7
+; AVX1-NEXT:  .LBB15_8: # %else6
+; AVX1-NEXT:    testb $16, %al
+; AVX1-NEXT:    jne .LBB15_9
+; AVX1-NEXT:  .LBB15_10: # %else8
+; AVX1-NEXT:    testb $32, %al
+; AVX1-NEXT:    jne .LBB15_11
+; AVX1-NEXT:  .LBB15_12: # %else10
+; AVX1-NEXT:    testb $64, %al
+; AVX1-NEXT:    jne .LBB15_13
+; AVX1-NEXT:  .LBB15_14: # %else12
+; AVX1-NEXT:    testb $-128, %al
+; AVX1-NEXT:    jne .LBB15_15
+; AVX1-NEXT:  .LBB15_16: # %else14
+; AVX1-NEXT:    testl $256, %eax # imm = 0x100
+; AVX1-NEXT:    jne .LBB15_17
+; AVX1-NEXT:  .LBB15_18: # %else16
+; AVX1-NEXT:    testl $512, %eax # imm = 0x200
+; AVX1-NEXT:    jne .LBB15_19
+; AVX1-NEXT:  .LBB15_20: # %else18
+; AVX1-NEXT:    testl $1024, %eax # imm = 0x400
+; AVX1-NEXT:    jne .LBB15_21
+; AVX1-NEXT:  .LBB15_22: # %else20
+; AVX1-NEXT:    testl $2048, %eax # imm = 0x800
+; AVX1-NEXT:    jne .LBB15_23
+; AVX1-NEXT:  .LBB15_24: # %else22
+; AVX1-NEXT:    testl $4096, %eax # imm = 0x1000
+; AVX1-NEXT:    jne .LBB15_25
+; AVX1-NEXT:  .LBB15_26: # %else24
+; AVX1-NEXT:    testl $8192, %eax # imm = 0x2000
+; AVX1-NEXT:    jne .LBB15_27
+; AVX1-NEXT:  .LBB15_28: # %else26
+; AVX1-NEXT:    testl $16384, %eax # imm = 0x4000
+; AVX1-NEXT:    jne .LBB15_29
+; AVX1-NEXT:  .LBB15_30: # %else28
+; AVX1-NEXT:    testl $32768, %eax # imm = 0x8000
+; AVX1-NEXT:    je .LBB15_32
+; AVX1-NEXT:  .LBB15_31: # %cond.store29
+; AVX1-NEXT:    vpextrb $15, %xmm0, 15(%rdi)
+; AVX1-NEXT:  .LBB15_32: # %else30
+; AVX1-NEXT:    testl $65536, %eax # imm = 0x10000
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    jne .LBB15_33
+; AVX1-NEXT:  # %bb.34: # %else32
+; AVX1-NEXT:    testl $131072, %eax # imm = 0x20000
+; AVX1-NEXT:    jne .LBB15_35
+; AVX1-NEXT:  .LBB15_36: # %else34
+; AVX1-NEXT:    testl $262144, %eax # imm = 0x40000
+; AVX1-NEXT:    jne .LBB15_37
+; AVX1-NEXT:  .LBB15_38: # %else36
+; AVX1-NEXT:    testl $524288, %eax # imm = 0x80000
+; AVX1-NEXT:    jne .LBB15_39
+; AVX1-NEXT:  .LBB15_40: # %else38
+; AVX1-NEXT:    testl $1048576, %eax # imm = 0x100000
+; AVX1-NEXT:    jne .LBB15_41
+; AVX1-NEXT:  .LBB15_42: # %else40
+; AVX1-NEXT:    testl $2097152, %eax # imm = 0x200000
+; AVX1-NEXT:    jne .LBB15_43
+; AVX1-NEXT:  .LBB15_44: # %else42
+; AVX1-NEXT:    testl $4194304, %eax # imm = 0x400000
+; AVX1-NEXT:    jne .LBB15_45
+; AVX1-NEXT:  .LBB15_46: # %else44
+; AVX1-NEXT:    testl $8388608, %eax # imm = 0x800000
+; AVX1-NEXT:    jne .LBB15_47
+; AVX1-NEXT:  .LBB15_48: # %else46
+; AVX1-NEXT:    testl $16777216, %eax # imm = 0x1000000
+; AVX1-NEXT:    jne .LBB15_49
+; AVX1-NEXT:  .LBB15_50: # %else48
+; AVX1-NEXT:    testl $33554432, %eax # imm = 0x2000000
+; AVX1-NEXT:    jne .LBB15_51
+; AVX1-NEXT:  .LBB15_52: # %else50
+; AVX1-NEXT:    testl $67108864, %eax # imm = 0x4000000
+; AVX1-NEXT:    jne .LBB15_53
+; AVX1-NEXT:  .LBB15_54: # %else52
+; AVX1-NEXT:    testl $134217728, %eax # imm = 0x8000000
+; AVX1-NEXT:    jne .LBB15_55
+; AVX1-NEXT:  .LBB15_56: # %else54
+; AVX1-NEXT:    testl $268435456, %eax # imm = 0x10000000
+; AVX1-NEXT:    jne .LBB15_57
+; AVX1-NEXT:  .LBB15_58: # %else56
+; AVX1-NEXT:    testl $536870912, %eax # imm = 0x20000000
+; AVX1-NEXT:    jne .LBB15_59
+; AVX1-NEXT:  .LBB15_60: # %else58
+; AVX1-NEXT:    testl $1073741824, %eax # imm = 0x40000000
+; AVX1-NEXT:    jne .LBB15_61
+; AVX1-NEXT:  .LBB15_62: # %else60
+; AVX1-NEXT:    testl $-2147483648, %eax # imm = 0x80000000
+; AVX1-NEXT:    jne .LBB15_63
+; AVX1-NEXT:  .LBB15_64: # %else62
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+; AVX1-NEXT:  .LBB15_1: # %cond.store
 ; AVX1-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX1-NEXT:  .LBB15_2: # %else
-; AVX1-NEXT:    vpcmpeqb %xmm3, %xmm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm4, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $1, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $2, %al
 ; AVX1-NEXT:    je .LBB15_4
-; AVX1-NEXT:  # %bb.3: # %cond.store1
+; AVX1-NEXT:  .LBB15_3: # %cond.store1
 ; AVX1-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
-; AVX1-NEXT:  .LBB15_4: # %else2
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm4, %xmm4
-; AVX1-NEXT:    vpextrb $2, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $4, %al
 ; AVX1-NEXT:    je .LBB15_6
-; AVX1-NEXT:  # %bb.5: # %cond.store3
+; AVX1-NEXT:  .LBB15_5: # %cond.store3
 ; AVX1-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
-; AVX1-NEXT:  .LBB15_6: # %else4
-; AVX1-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $3, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $8, %al
 ; AVX1-NEXT:    je .LBB15_8
-; AVX1-NEXT:  # %bb.7: # %cond.store5
+; AVX1-NEXT:  .LBB15_7: # %cond.store5
 ; AVX1-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
-; AVX1-NEXT:  .LBB15_8: # %else6
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm4, %xmm4
-; AVX1-NEXT:    vpextrb $4, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $16, %al
 ; AVX1-NEXT:    je .LBB15_10
-; AVX1-NEXT:  # %bb.9: # %cond.store7
+; AVX1-NEXT:  .LBB15_9: # %cond.store7
 ; AVX1-NEXT:    vpextrb $4, %xmm0, 4(%rdi)
-; AVX1-NEXT:  .LBB15_10: # %else8
-; AVX1-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $5, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $32, %al
 ; AVX1-NEXT:    je .LBB15_12
-; AVX1-NEXT:  # %bb.11: # %cond.store9
+; AVX1-NEXT:  .LBB15_11: # %cond.store9
 ; AVX1-NEXT:    vpextrb $5, %xmm0, 5(%rdi)
-; AVX1-NEXT:  .LBB15_12: # %else10
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm4, %xmm4
-; AVX1-NEXT:    vpextrb $6, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $64, %al
 ; AVX1-NEXT:    je .LBB15_14
-; AVX1-NEXT:  # %bb.13: # %cond.store11
+; AVX1-NEXT:  .LBB15_13: # %cond.store11
 ; AVX1-NEXT:    vpextrb $6, %xmm0, 6(%rdi)
-; AVX1-NEXT:  .LBB15_14: # %else12
-; AVX1-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $7, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $-128, %al
 ; AVX1-NEXT:    je .LBB15_16
-; AVX1-NEXT:  # %bb.15: # %cond.store13
+; AVX1-NEXT:  .LBB15_15: # %cond.store13
 ; AVX1-NEXT:    vpextrb $7, %xmm0, 7(%rdi)
-; AVX1-NEXT:  .LBB15_16: # %else14
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm4, %xmm4
-; AVX1-NEXT:    vpextrb $8, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $256, %eax # imm = 0x100
 ; AVX1-NEXT:    je .LBB15_18
-; AVX1-NEXT:  # %bb.17: # %cond.store15
+; AVX1-NEXT:  .LBB15_17: # %cond.store15
 ; AVX1-NEXT:    vpextrb $8, %xmm0, 8(%rdi)
-; AVX1-NEXT:  .LBB15_18: # %else16
-; AVX1-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $9, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $512, %eax # imm = 0x200
 ; AVX1-NEXT:    je .LBB15_20
-; AVX1-NEXT:  # %bb.19: # %cond.store17
+; AVX1-NEXT:  .LBB15_19: # %cond.store17
 ; AVX1-NEXT:    vpextrb $9, %xmm0, 9(%rdi)
-; AVX1-NEXT:  .LBB15_20: # %else18
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm4, %xmm4
-; AVX1-NEXT:    vpextrb $10, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $1024, %eax # imm = 0x400
 ; AVX1-NEXT:    je .LBB15_22
-; AVX1-NEXT:  # %bb.21: # %cond.store19
+; AVX1-NEXT:  .LBB15_21: # %cond.store19
 ; AVX1-NEXT:    vpextrb $10, %xmm0, 10(%rdi)
-; AVX1-NEXT:  .LBB15_22: # %else20
-; AVX1-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $11, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $2048, %eax # imm = 0x800
 ; AVX1-NEXT:    je .LBB15_24
-; AVX1-NEXT:  # %bb.23: # %cond.store21
+; AVX1-NEXT:  .LBB15_23: # %cond.store21
 ; AVX1-NEXT:    vpextrb $11, %xmm0, 11(%rdi)
-; AVX1-NEXT:  .LBB15_24: # %else22
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm4, %xmm4
-; AVX1-NEXT:    vpextrb $12, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $4096, %eax # imm = 0x1000
 ; AVX1-NEXT:    je .LBB15_26
-; AVX1-NEXT:  # %bb.25: # %cond.store23
+; AVX1-NEXT:  .LBB15_25: # %cond.store23
 ; AVX1-NEXT:    vpextrb $12, %xmm0, 12(%rdi)
-; AVX1-NEXT:  .LBB15_26: # %else24
-; AVX1-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $13, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $8192, %eax # imm = 0x2000
 ; AVX1-NEXT:    je .LBB15_28
-; AVX1-NEXT:  # %bb.27: # %cond.store25
+; AVX1-NEXT:  .LBB15_27: # %cond.store25
 ; AVX1-NEXT:    vpextrb $13, %xmm0, 13(%rdi)
-; AVX1-NEXT:  .LBB15_28: # %else26
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm4, %xmm4
-; AVX1-NEXT:    vpextrb $14, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $16384, %eax # imm = 0x4000
 ; AVX1-NEXT:    je .LBB15_30
-; AVX1-NEXT:  # %bb.29: # %cond.store27
+; AVX1-NEXT:  .LBB15_29: # %cond.store27
 ; AVX1-NEXT:    vpextrb $14, %xmm0, 14(%rdi)
-; AVX1-NEXT:  .LBB15_30: # %else28
-; AVX1-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $15, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je .LBB15_32
-; AVX1-NEXT:  # %bb.31: # %cond.store29
-; AVX1-NEXT:    vpextrb $15, %xmm0, 15(%rdi)
-; AVX1-NEXT:  .LBB15_32: # %else30
-; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $0, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT:    je .LBB15_34
-; AVX1-NEXT:  # %bb.33: # %cond.store31
+; AVX1-NEXT:    testl $32768, %eax # imm = 0x8000
+; AVX1-NEXT:    jne .LBB15_31
+; AVX1-NEXT:    jmp .LBB15_32
+; AVX1-NEXT:  .LBB15_33: # %cond.store31
 ; AVX1-NEXT:    vpextrb $0, %xmm0, 16(%rdi)
-; AVX1-NEXT:  .LBB15_34: # %else32
-; AVX1-NEXT:    vpextrb $1, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $131072, %eax # imm = 0x20000
 ; AVX1-NEXT:    je .LBB15_36
-; AVX1-NEXT:  # %bb.35: # %cond.store33
+; AVX1-NEXT:  .LBB15_35: # %cond.store33
 ; AVX1-NEXT:    vpextrb $1, %xmm0, 17(%rdi)
-; AVX1-NEXT:  .LBB15_36: # %else34
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $2, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $262144, %eax # imm = 0x40000
 ; AVX1-NEXT:    je .LBB15_38
-; AVX1-NEXT:  # %bb.37: # %cond.store35
+; AVX1-NEXT:  .LBB15_37: # %cond.store35
 ; AVX1-NEXT:    vpextrb $2, %xmm0, 18(%rdi)
-; AVX1-NEXT:  .LBB15_38: # %else36
-; AVX1-NEXT:    vpextrb $3, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $524288, %eax # imm = 0x80000
 ; AVX1-NEXT:    je .LBB15_40
-; AVX1-NEXT:  # %bb.39: # %cond.store37
+; AVX1-NEXT:  .LBB15_39: # %cond.store37
 ; AVX1-NEXT:    vpextrb $3, %xmm0, 19(%rdi)
-; AVX1-NEXT:  .LBB15_40: # %else38
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $1048576, %eax # imm = 0x100000
 ; AVX1-NEXT:    je .LBB15_42
-; AVX1-NEXT:  # %bb.41: # %cond.store39
-; AVX1-NEXT:    vpextrb $4, %xmm0, 20(%rdi)
-; AVX1-NEXT:  .LBB15_42: # %else40
-; AVX1-NEXT:    vpextrb $5, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:  .LBB15_41: # %cond.store39
+; AVX1-NEXT:    vpextrb $4, %xmm0, 20(%rdi)
+; AVX1-NEXT:    testl $2097152, %eax # imm = 0x200000
 ; AVX1-NEXT:    je .LBB15_44
-; AVX1-NEXT:  # %bb.43: # %cond.store41
+; AVX1-NEXT:  .LBB15_43: # %cond.store41
 ; AVX1-NEXT:    vpextrb $5, %xmm0, 21(%rdi)
-; AVX1-NEXT:  .LBB15_44: # %else42
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $6, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $4194304, %eax # imm = 0x400000
 ; AVX1-NEXT:    je .LBB15_46
-; AVX1-NEXT:  # %bb.45: # %cond.store43
+; AVX1-NEXT:  .LBB15_45: # %cond.store43
 ; AVX1-NEXT:    vpextrb $6, %xmm0, 22(%rdi)
-; AVX1-NEXT:  .LBB15_46: # %else44
-; AVX1-NEXT:    vpextrb $7, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $8388608, %eax # imm = 0x800000
 ; AVX1-NEXT:    je .LBB15_48
-; AVX1-NEXT:  # %bb.47: # %cond.store45
+; AVX1-NEXT:  .LBB15_47: # %cond.store45
 ; AVX1-NEXT:    vpextrb $7, %xmm0, 23(%rdi)
-; AVX1-NEXT:  .LBB15_48: # %else46
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $8, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $16777216, %eax # imm = 0x1000000
 ; AVX1-NEXT:    je .LBB15_50
-; AVX1-NEXT:  # %bb.49: # %cond.store47
+; AVX1-NEXT:  .LBB15_49: # %cond.store47
 ; AVX1-NEXT:    vpextrb $8, %xmm0, 24(%rdi)
-; AVX1-NEXT:  .LBB15_50: # %else48
-; AVX1-NEXT:    vpextrb $9, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $33554432, %eax # imm = 0x2000000
 ; AVX1-NEXT:    je .LBB15_52
-; AVX1-NEXT:  # %bb.51: # %cond.store49
+; AVX1-NEXT:  .LBB15_51: # %cond.store49
 ; AVX1-NEXT:    vpextrb $9, %xmm0, 25(%rdi)
-; AVX1-NEXT:  .LBB15_52: # %else50
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $10, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $67108864, %eax # imm = 0x4000000
 ; AVX1-NEXT:    je .LBB15_54
-; AVX1-NEXT:  # %bb.53: # %cond.store51
+; AVX1-NEXT:  .LBB15_53: # %cond.store51
 ; AVX1-NEXT:    vpextrb $10, %xmm0, 26(%rdi)
-; AVX1-NEXT:  .LBB15_54: # %else52
-; AVX1-NEXT:    vpextrb $11, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $134217728, %eax # imm = 0x8000000
 ; AVX1-NEXT:    je .LBB15_56
-; AVX1-NEXT:  # %bb.55: # %cond.store53
+; AVX1-NEXT:  .LBB15_55: # %cond.store53
 ; AVX1-NEXT:    vpextrb $11, %xmm0, 27(%rdi)
-; AVX1-NEXT:  .LBB15_56: # %else54
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $12, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $268435456, %eax # imm = 0x10000000
 ; AVX1-NEXT:    je .LBB15_58
-; AVX1-NEXT:  # %bb.57: # %cond.store55
+; AVX1-NEXT:  .LBB15_57: # %cond.store55
 ; AVX1-NEXT:    vpextrb $12, %xmm0, 28(%rdi)
-; AVX1-NEXT:  .LBB15_58: # %else56
-; AVX1-NEXT:    vpextrb $13, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $536870912, %eax # imm = 0x20000000
 ; AVX1-NEXT:    je .LBB15_60
-; AVX1-NEXT:  # %bb.59: # %cond.store57
+; AVX1-NEXT:  .LBB15_59: # %cond.store57
 ; AVX1-NEXT:    vpextrb $13, %xmm0, 29(%rdi)
-; AVX1-NEXT:  .LBB15_60: # %else58
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $14, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $1073741824, %eax # imm = 0x40000000
 ; AVX1-NEXT:    je .LBB15_62
-; AVX1-NEXT:  # %bb.61: # %cond.store59
+; AVX1-NEXT:  .LBB15_61: # %cond.store59
 ; AVX1-NEXT:    vpextrb $14, %xmm0, 30(%rdi)
-; AVX1-NEXT:  .LBB15_62: # %else60
-; AVX1-NEXT:    vpextrb $15, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $-2147483648, %eax # imm = 0x80000000
 ; AVX1-NEXT:    je .LBB15_64
-; AVX1-NEXT:  # %bb.63: # %cond.store61
+; AVX1-NEXT:  .LBB15_63: # %cond.store61
 ; AVX1-NEXT:    vpextrb $15, %xmm0, 31(%rdi)
-; AVX1-NEXT:  .LBB15_64: # %else62
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: truncstore_v32i16_v32i8:
 ; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
 ; AVX2-NEXT:    vpacksswb %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm3
-; AVX2-NEXT:    vpextrb $0, %xmm3, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je .LBB15_2
-; AVX2-NEXT:  # %bb.1: # %cond.store
+; AVX2-NEXT:    vpcmpeqb %ymm3, %ymm2, %ymm1
+; AVX2-NEXT:    vpmovmskb %ymm1, %eax
+; AVX2-NEXT:    notl %eax
+; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    jne .LBB15_1
+; AVX2-NEXT:  # %bb.2: # %else
+; AVX2-NEXT:    testb $2, %al
+; AVX2-NEXT:    jne .LBB15_3
+; AVX2-NEXT:  .LBB15_4: # %else2
+; AVX2-NEXT:    testb $4, %al
+; AVX2-NEXT:    jne .LBB15_5
+; AVX2-NEXT:  .LBB15_6: # %else4
+; AVX2-NEXT:    testb $8, %al
+; AVX2-NEXT:    jne .LBB15_7
+; AVX2-NEXT:  .LBB15_8: # %else6
+; AVX2-NEXT:    testb $16, %al
+; AVX2-NEXT:    jne .LBB15_9
+; AVX2-NEXT:  .LBB15_10: # %else8
+; AVX2-NEXT:    testb $32, %al
+; AVX2-NEXT:    jne .LBB15_11
+; AVX2-NEXT:  .LBB15_12: # %else10
+; AVX2-NEXT:    testb $64, %al
+; AVX2-NEXT:    jne .LBB15_13
+; AVX2-NEXT:  .LBB15_14: # %else12
+; AVX2-NEXT:    testb $-128, %al
+; AVX2-NEXT:    jne .LBB15_15
+; AVX2-NEXT:  .LBB15_16: # %else14
+; AVX2-NEXT:    testl $256, %eax # imm = 0x100
+; AVX2-NEXT:    jne .LBB15_17
+; AVX2-NEXT:  .LBB15_18: # %else16
+; AVX2-NEXT:    testl $512, %eax # imm = 0x200
+; AVX2-NEXT:    jne .LBB15_19
+; AVX2-NEXT:  .LBB15_20: # %else18
+; AVX2-NEXT:    testl $1024, %eax # imm = 0x400
+; AVX2-NEXT:    jne .LBB15_21
+; AVX2-NEXT:  .LBB15_22: # %else20
+; AVX2-NEXT:    testl $2048, %eax # imm = 0x800
+; AVX2-NEXT:    jne .LBB15_23
+; AVX2-NEXT:  .LBB15_24: # %else22
+; AVX2-NEXT:    testl $4096, %eax # imm = 0x1000
+; AVX2-NEXT:    jne .LBB15_25
+; AVX2-NEXT:  .LBB15_26: # %else24
+; AVX2-NEXT:    testl $8192, %eax # imm = 0x2000
+; AVX2-NEXT:    jne .LBB15_27
+; AVX2-NEXT:  .LBB15_28: # %else26
+; AVX2-NEXT:    testl $16384, %eax # imm = 0x4000
+; AVX2-NEXT:    jne .LBB15_29
+; AVX2-NEXT:  .LBB15_30: # %else28
+; AVX2-NEXT:    testl $32768, %eax # imm = 0x8000
+; AVX2-NEXT:    je .LBB15_32
+; AVX2-NEXT:  .LBB15_31: # %cond.store29
+; AVX2-NEXT:    vpextrb $15, %xmm0, 15(%rdi)
+; AVX2-NEXT:  .LBB15_32: # %else30
+; AVX2-NEXT:    testl $65536, %eax # imm = 0x10000
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
+; AVX2-NEXT:    jne .LBB15_33
+; AVX2-NEXT:  # %bb.34: # %else32
+; AVX2-NEXT:    testl $131072, %eax # imm = 0x20000
+; AVX2-NEXT:    jne .LBB15_35
+; AVX2-NEXT:  .LBB15_36: # %else34
+; AVX2-NEXT:    testl $262144, %eax # imm = 0x40000
+; AVX2-NEXT:    jne .LBB15_37
+; AVX2-NEXT:  .LBB15_38: # %else36
+; AVX2-NEXT:    testl $524288, %eax # imm = 0x80000
+; AVX2-NEXT:    jne .LBB15_39
+; AVX2-NEXT:  .LBB15_40: # %else38
+; AVX2-NEXT:    testl $1048576, %eax # imm = 0x100000
+; AVX2-NEXT:    jne .LBB15_41
+; AVX2-NEXT:  .LBB15_42: # %else40
+; AVX2-NEXT:    testl $2097152, %eax # imm = 0x200000
+; AVX2-NEXT:    jne .LBB15_43
+; AVX2-NEXT:  .LBB15_44: # %else42
+; AVX2-NEXT:    testl $4194304, %eax # imm = 0x400000
+; AVX2-NEXT:    jne .LBB15_45
+; AVX2-NEXT:  .LBB15_46: # %else44
+; AVX2-NEXT:    testl $8388608, %eax # imm = 0x800000
+; AVX2-NEXT:    jne .LBB15_47
+; AVX2-NEXT:  .LBB15_48: # %else46
+; AVX2-NEXT:    testl $16777216, %eax # imm = 0x1000000
+; AVX2-NEXT:    jne .LBB15_49
+; AVX2-NEXT:  .LBB15_50: # %else48
+; AVX2-NEXT:    testl $33554432, %eax # imm = 0x2000000
+; AVX2-NEXT:    jne .LBB15_51
+; AVX2-NEXT:  .LBB15_52: # %else50
+; AVX2-NEXT:    testl $67108864, %eax # imm = 0x4000000
+; AVX2-NEXT:    jne .LBB15_53
+; AVX2-NEXT:  .LBB15_54: # %else52
+; AVX2-NEXT:    testl $134217728, %eax # imm = 0x8000000
+; AVX2-NEXT:    jne .LBB15_55
+; AVX2-NEXT:  .LBB15_56: # %else54
+; AVX2-NEXT:    testl $268435456, %eax # imm = 0x10000000
+; AVX2-NEXT:    jne .LBB15_57
+; AVX2-NEXT:  .LBB15_58: # %else56
+; AVX2-NEXT:    testl $536870912, %eax # imm = 0x20000000
+; AVX2-NEXT:    jne .LBB15_59
+; AVX2-NEXT:  .LBB15_60: # %else58
+; AVX2-NEXT:    testl $1073741824, %eax # imm = 0x40000000
+; AVX2-NEXT:    jne .LBB15_61
+; AVX2-NEXT:  .LBB15_62: # %else60
+; AVX2-NEXT:    testl $-2147483648, %eax # imm = 0x80000000
+; AVX2-NEXT:    jne .LBB15_63
+; AVX2-NEXT:  .LBB15_64: # %else62
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+; AVX2-NEXT:  .LBB15_1: # %cond.store
 ; AVX2-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX2-NEXT:  .LBB15_2: # %else
-; AVX2-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX2-NEXT:    vpextrb $1, %xmm1, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $2, %al
 ; AVX2-NEXT:    je .LBB15_4
-; AVX2-NEXT:  # %bb.3: # %cond.store1
+; AVX2-NEXT:  .LBB15_3: # %cond.store1
 ; AVX2-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
-; AVX2-NEXT:  .LBB15_4: # %else2
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm3
-; AVX2-NEXT:    vpextrb $2, %xmm3, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $4, %al
 ; AVX2-NEXT:    je .LBB15_6
-; AVX2-NEXT:  # %bb.5: # %cond.store3
+; AVX2-NEXT:  .LBB15_5: # %cond.store3
 ; AVX2-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
-; AVX2-NEXT:  .LBB15_6: # %else4
-; AVX2-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX2-NEXT:    vpextrb $3, %xmm1, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $8, %al
 ; AVX2-NEXT:    je .LBB15_8
-; AVX2-NEXT:  # %bb.7: # %cond.store5
+; AVX2-NEXT:  .LBB15_7: # %cond.store5
 ; AVX2-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
-; AVX2-NEXT:  .LBB15_8: # %else6
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm3
-; AVX2-NEXT:    vpextrb $4, %xmm3, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $16, %al
 ; AVX2-NEXT:    je .LBB15_10
-; AVX2-NEXT:  # %bb.9: # %cond.store7
+; AVX2-NEXT:  .LBB15_9: # %cond.store7
 ; AVX2-NEXT:    vpextrb $4, %xmm0, 4(%rdi)
-; AVX2-NEXT:  .LBB15_10: # %else8
-; AVX2-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX2-NEXT:    vpextrb $5, %xmm1, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $32, %al
 ; AVX2-NEXT:    je .LBB15_12
-; AVX2-NEXT:  # %bb.11: # %cond.store9
+; AVX2-NEXT:  .LBB15_11: # %cond.store9
 ; AVX2-NEXT:    vpextrb $5, %xmm0, 5(%rdi)
-; AVX2-NEXT:  .LBB15_12: # %else10
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm3
-; AVX2-NEXT:    vpextrb $6, %xmm3, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $64, %al
 ; AVX2-NEXT:    je .LBB15_14
-; AVX2-NEXT:  # %bb.13: # %cond.store11
+; AVX2-NEXT:  .LBB15_13: # %cond.store11
 ; AVX2-NEXT:    vpextrb $6, %xmm0, 6(%rdi)
-; AVX2-NEXT:  .LBB15_14: # %else12
-; AVX2-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX2-NEXT:    vpextrb $7, %xmm1, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $-128, %al
 ; AVX2-NEXT:    je .LBB15_16
-; AVX2-NEXT:  # %bb.15: # %cond.store13
+; AVX2-NEXT:  .LBB15_15: # %cond.store13
 ; AVX2-NEXT:    vpextrb $7, %xmm0, 7(%rdi)
-; AVX2-NEXT:  .LBB15_16: # %else14
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm3
-; AVX2-NEXT:    vpextrb $8, %xmm3, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $256, %eax # imm = 0x100
 ; AVX2-NEXT:    je .LBB15_18
-; AVX2-NEXT:  # %bb.17: # %cond.store15
+; AVX2-NEXT:  .LBB15_17: # %cond.store15
 ; AVX2-NEXT:    vpextrb $8, %xmm0, 8(%rdi)
-; AVX2-NEXT:  .LBB15_18: # %else16
-; AVX2-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX2-NEXT:    vpextrb $9, %xmm1, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $512, %eax # imm = 0x200
 ; AVX2-NEXT:    je .LBB15_20
-; AVX2-NEXT:  # %bb.19: # %cond.store17
+; AVX2-NEXT:  .LBB15_19: # %cond.store17
 ; AVX2-NEXT:    vpextrb $9, %xmm0, 9(%rdi)
-; AVX2-NEXT:  .LBB15_20: # %else18
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm3
-; AVX2-NEXT:    vpextrb $10, %xmm3, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $1024, %eax # imm = 0x400
 ; AVX2-NEXT:    je .LBB15_22
-; AVX2-NEXT:  # %bb.21: # %cond.store19
+; AVX2-NEXT:  .LBB15_21: # %cond.store19
 ; AVX2-NEXT:    vpextrb $10, %xmm0, 10(%rdi)
-; AVX2-NEXT:  .LBB15_22: # %else20
-; AVX2-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX2-NEXT:    vpextrb $11, %xmm1, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $2048, %eax # imm = 0x800
 ; AVX2-NEXT:    je .LBB15_24
-; AVX2-NEXT:  # %bb.23: # %cond.store21
+; AVX2-NEXT:  .LBB15_23: # %cond.store21
 ; AVX2-NEXT:    vpextrb $11, %xmm0, 11(%rdi)
-; AVX2-NEXT:  .LBB15_24: # %else22
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm3
-; AVX2-NEXT:    vpextrb $12, %xmm3, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $4096, %eax # imm = 0x1000
 ; AVX2-NEXT:    je .LBB15_26
-; AVX2-NEXT:  # %bb.25: # %cond.store23
+; AVX2-NEXT:  .LBB15_25: # %cond.store23
 ; AVX2-NEXT:    vpextrb $12, %xmm0, 12(%rdi)
-; AVX2-NEXT:  .LBB15_26: # %else24
-; AVX2-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX2-NEXT:    vpextrb $13, %xmm1, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $8192, %eax # imm = 0x2000
 ; AVX2-NEXT:    je .LBB15_28
-; AVX2-NEXT:  # %bb.27: # %cond.store25
+; AVX2-NEXT:  .LBB15_27: # %cond.store25
 ; AVX2-NEXT:    vpextrb $13, %xmm0, 13(%rdi)
-; AVX2-NEXT:  .LBB15_28: # %else26
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm3
-; AVX2-NEXT:    vpextrb $14, %xmm3, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $16384, %eax # imm = 0x4000
 ; AVX2-NEXT:    je .LBB15_30
-; AVX2-NEXT:  # %bb.29: # %cond.store27
+; AVX2-NEXT:  .LBB15_29: # %cond.store27
 ; AVX2-NEXT:    vpextrb $14, %xmm0, 14(%rdi)
-; AVX2-NEXT:  .LBB15_30: # %else28
-; AVX2-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX2-NEXT:    vpextrb $15, %xmm1, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je .LBB15_32
-; AVX2-NEXT:  # %bb.31: # %cond.store29
-; AVX2-NEXT:    vpextrb $15, %xmm0, 15(%rdi)
-; AVX2-NEXT:  .LBB15_32: # %else30
-; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm1
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpextrb $0, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; AVX2-NEXT:    je .LBB15_34
-; AVX2-NEXT:  # %bb.33: # %cond.store31
+; AVX2-NEXT:    testl $32768, %eax # imm = 0x8000
+; AVX2-NEXT:    jne .LBB15_31
+; AVX2-NEXT:    jmp .LBB15_32
+; AVX2-NEXT:  .LBB15_33: # %cond.store31
 ; AVX2-NEXT:    vpextrb $0, %xmm0, 16(%rdi)
-; AVX2-NEXT:  .LBB15_34: # %else32
-; AVX2-NEXT:    vpextrb $1, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $131072, %eax # imm = 0x20000
 ; AVX2-NEXT:    je .LBB15_36
-; AVX2-NEXT:  # %bb.35: # %cond.store33
+; AVX2-NEXT:  .LBB15_35: # %cond.store33
 ; AVX2-NEXT:    vpextrb $1, %xmm0, 17(%rdi)
-; AVX2-NEXT:  .LBB15_36: # %else34
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpextrb $2, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $262144, %eax # imm = 0x40000
 ; AVX2-NEXT:    je .LBB15_38
-; AVX2-NEXT:  # %bb.37: # %cond.store35
+; AVX2-NEXT:  .LBB15_37: # %cond.store35
 ; AVX2-NEXT:    vpextrb $2, %xmm0, 18(%rdi)
-; AVX2-NEXT:  .LBB15_38: # %else36
-; AVX2-NEXT:    vpextrb $3, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $524288, %eax # imm = 0x80000
 ; AVX2-NEXT:    je .LBB15_40
-; AVX2-NEXT:  # %bb.39: # %cond.store37
+; AVX2-NEXT:  .LBB15_39: # %cond.store37
 ; AVX2-NEXT:    vpextrb $3, %xmm0, 19(%rdi)
-; AVX2-NEXT:  .LBB15_40: # %else38
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $1048576, %eax # imm = 0x100000
 ; AVX2-NEXT:    je .LBB15_42
-; AVX2-NEXT:  # %bb.41: # %cond.store39
+; AVX2-NEXT:  .LBB15_41: # %cond.store39
 ; AVX2-NEXT:    vpextrb $4, %xmm0, 20(%rdi)
-; AVX2-NEXT:  .LBB15_42: # %else40
-; AVX2-NEXT:    vpextrb $5, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $2097152, %eax # imm = 0x200000
 ; AVX2-NEXT:    je .LBB15_44
-; AVX2-NEXT:  # %bb.43: # %cond.store41
+; AVX2-NEXT:  .LBB15_43: # %cond.store41
 ; AVX2-NEXT:    vpextrb $5, %xmm0, 21(%rdi)
-; AVX2-NEXT:  .LBB15_44: # %else42
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpextrb $6, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $4194304, %eax # imm = 0x400000
 ; AVX2-NEXT:    je .LBB15_46
-; AVX2-NEXT:  # %bb.45: # %cond.store43
+; AVX2-NEXT:  .LBB15_45: # %cond.store43
 ; AVX2-NEXT:    vpextrb $6, %xmm0, 22(%rdi)
-; AVX2-NEXT:  .LBB15_46: # %else44
-; AVX2-NEXT:    vpextrb $7, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $8388608, %eax # imm = 0x800000
 ; AVX2-NEXT:    je .LBB15_48
-; AVX2-NEXT:  # %bb.47: # %cond.store45
+; AVX2-NEXT:  .LBB15_47: # %cond.store45
 ; AVX2-NEXT:    vpextrb $7, %xmm0, 23(%rdi)
-; AVX2-NEXT:  .LBB15_48: # %else46
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpextrb $8, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $16777216, %eax # imm = 0x1000000
 ; AVX2-NEXT:    je .LBB15_50
-; AVX2-NEXT:  # %bb.49: # %cond.store47
+; AVX2-NEXT:  .LBB15_49: # %cond.store47
 ; AVX2-NEXT:    vpextrb $8, %xmm0, 24(%rdi)
-; AVX2-NEXT:  .LBB15_50: # %else48
-; AVX2-NEXT:    vpextrb $9, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $33554432, %eax # imm = 0x2000000
 ; AVX2-NEXT:    je .LBB15_52
-; AVX2-NEXT:  # %bb.51: # %cond.store49
+; AVX2-NEXT:  .LBB15_51: # %cond.store49
 ; AVX2-NEXT:    vpextrb $9, %xmm0, 25(%rdi)
-; AVX2-NEXT:  .LBB15_52: # %else50
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpextrb $10, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $67108864, %eax # imm = 0x4000000
 ; AVX2-NEXT:    je .LBB15_54
-; AVX2-NEXT:  # %bb.53: # %cond.store51
+; AVX2-NEXT:  .LBB15_53: # %cond.store51
 ; AVX2-NEXT:    vpextrb $10, %xmm0, 26(%rdi)
-; AVX2-NEXT:  .LBB15_54: # %else52
-; AVX2-NEXT:    vpextrb $11, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $134217728, %eax # imm = 0x8000000
 ; AVX2-NEXT:    je .LBB15_56
-; AVX2-NEXT:  # %bb.55: # %cond.store53
+; AVX2-NEXT:  .LBB15_55: # %cond.store53
 ; AVX2-NEXT:    vpextrb $11, %xmm0, 27(%rdi)
-; AVX2-NEXT:  .LBB15_56: # %else54
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpextrb $12, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $268435456, %eax # imm = 0x10000000
 ; AVX2-NEXT:    je .LBB15_58
-; AVX2-NEXT:  # %bb.57: # %cond.store55
+; AVX2-NEXT:  .LBB15_57: # %cond.store55
 ; AVX2-NEXT:    vpextrb $12, %xmm0, 28(%rdi)
-; AVX2-NEXT:  .LBB15_58: # %else56
-; AVX2-NEXT:    vpextrb $13, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $536870912, %eax # imm = 0x20000000
 ; AVX2-NEXT:    je .LBB15_60
-; AVX2-NEXT:  # %bb.59: # %cond.store57
+; AVX2-NEXT:  .LBB15_59: # %cond.store57
 ; AVX2-NEXT:    vpextrb $13, %xmm0, 29(%rdi)
-; AVX2-NEXT:  .LBB15_60: # %else58
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpextrb $14, %xmm1, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $1073741824, %eax # imm = 0x40000000
 ; AVX2-NEXT:    je .LBB15_62
-; AVX2-NEXT:  # %bb.61: # %cond.store59
+; AVX2-NEXT:  .LBB15_61: # %cond.store59
 ; AVX2-NEXT:    vpextrb $14, %xmm0, 30(%rdi)
-; AVX2-NEXT:  .LBB15_62: # %else60
-; AVX2-NEXT:    vpextrb $15, %xmm1, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $-2147483648, %eax # imm = 0x80000000
 ; AVX2-NEXT:    je .LBB15_64
-; AVX2-NEXT:  # %bb.63: # %cond.store61
+; AVX2-NEXT:  .LBB15_63: # %cond.store61
 ; AVX2-NEXT:    vpextrb $15, %xmm0, 31(%rdi)
-; AVX2-NEXT:  .LBB15_64: # %else62
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: truncstore_v32i16_v32i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX512F-NEXT:    vpcmpeqb %xmm3, %xmm2, %xmm4
-; AVX512F-NEXT:    vpternlogq $15, %zmm4, %zmm4, %zmm4
-; AVX512F-NEXT:    vpmovsxbd %xmm4, %zmm4
-; AVX512F-NEXT:    vptestmd %zmm4, %zmm4, %k0
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm4 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; AVX512F-NEXT:    vpminsw %ymm4, %ymm0, %ymm0
-; AVX512F-NEXT:    vpminsw %ymm4, %ymm1, %ymm1
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm4 = [65408,65408,65408,65408,65408,65408,65408,65408,65408,65408,65408,65408,65408,65408,65408,65408]
-; AVX512F-NEXT:    vpmaxsw %ymm4, %ymm1, %ymm1
-; AVX512F-NEXT:    vpmaxsw %ymm4, %ymm0, %ymm0
-; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
-; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
-; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
-; AVX512F-NEXT:    vpmovdb %zmm1, %xmm1
-; AVX512F-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB15_2
-; AVX512F-NEXT:  # %bb.1: # %cond.store
-; AVX512F-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX512F-NEXT:  .LBB15_2: # %else
-; AVX512F-NEXT:    vpcmpeqb %xmm3, %xmm2, %xmm1
-; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
-; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB15_4
-; AVX512F-NEXT:  # %bb.3: # %cond.store1
-; AVX512F-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
+; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512F-NEXT:    vpcmpeqb %ymm2, %ymm1, %ymm1
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512F-NEXT:    vpacksswb %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX512F-NEXT:    vpmovmskb %ymm1, %eax
+; AVX512F-NEXT:    notl %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    jne .LBB15_1
+; AVX512F-NEXT:  # %bb.2: # %else
+; AVX512F-NEXT:    testb $2, %al
+; AVX512F-NEXT:    jne .LBB15_3
 ; AVX512F-NEXT:  .LBB15_4: # %else2
-; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm3, %zmm3, %zmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB15_6
-; AVX512F-NEXT:  # %bb.5: # %cond.store3
-; AVX512F-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
+; AVX512F-NEXT:    testb $4, %al
+; AVX512F-NEXT:    jne .LBB15_5
 ; AVX512F-NEXT:  .LBB15_6: # %else4
-; AVX512F-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
-; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB15_8
-; AVX512F-NEXT:  # %bb.7: # %cond.store5
-; AVX512F-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
+; AVX512F-NEXT:    testb $8, %al
+; AVX512F-NEXT:    jne .LBB15_7
 ; AVX512F-NEXT:  .LBB15_8: # %else6
-; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm3, %zmm3, %zmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $4, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB15_10
-; AVX512F-NEXT:  # %bb.9: # %cond.store7
-; AVX512F-NEXT:    vpextrb $4, %xmm0, 4(%rdi)
+; AVX512F-NEXT:    testb $16, %al
+; AVX512F-NEXT:    jne .LBB15_9
 ; AVX512F-NEXT:  .LBB15_10: # %else8
-; AVX512F-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
-; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $5, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB15_12
-; AVX512F-NEXT:  # %bb.11: # %cond.store9
-; AVX512F-NEXT:    vpextrb $5, %xmm0, 5(%rdi)
+; AVX512F-NEXT:    testb $32, %al
+; AVX512F-NEXT:    jne .LBB15_11
 ; AVX512F-NEXT:  .LBB15_12: # %else10
-; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm3, %zmm3, %zmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $6, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB15_14
-; AVX512F-NEXT:  # %bb.13: # %cond.store11
-; AVX512F-NEXT:    vpextrb $6, %xmm0, 6(%rdi)
+; AVX512F-NEXT:    testb $64, %al
+; AVX512F-NEXT:    jne .LBB15_13
 ; AVX512F-NEXT:  .LBB15_14: # %else12
-; AVX512F-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
-; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $7, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB15_16
-; AVX512F-NEXT:  # %bb.15: # %cond.store13
-; AVX512F-NEXT:    vpextrb $7, %xmm0, 7(%rdi)
+; AVX512F-NEXT:    testb $-128, %al
+; AVX512F-NEXT:    jne .LBB15_15
 ; AVX512F-NEXT:  .LBB15_16: # %else14
-; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm3, %zmm3, %zmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $8, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB15_18
-; AVX512F-NEXT:  # %bb.17: # %cond.store15
-; AVX512F-NEXT:    vpextrb $8, %xmm0, 8(%rdi)
+; AVX512F-NEXT:    testl $256, %eax # imm = 0x100
+; AVX512F-NEXT:    jne .LBB15_17
 ; AVX512F-NEXT:  .LBB15_18: # %else16
-; AVX512F-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
-; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $9, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB15_20
-; AVX512F-NEXT:  # %bb.19: # %cond.store17
-; AVX512F-NEXT:    vpextrb $9, %xmm0, 9(%rdi)
+; AVX512F-NEXT:    testl $512, %eax # imm = 0x200
+; AVX512F-NEXT:    jne .LBB15_19
 ; AVX512F-NEXT:  .LBB15_20: # %else18
-; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm3, %zmm3, %zmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $10, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB15_22
-; AVX512F-NEXT:  # %bb.21: # %cond.store19
-; AVX512F-NEXT:    vpextrb $10, %xmm0, 10(%rdi)
+; AVX512F-NEXT:    testl $1024, %eax # imm = 0x400
+; AVX512F-NEXT:    jne .LBB15_21
 ; AVX512F-NEXT:  .LBB15_22: # %else20
-; AVX512F-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
-; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $11, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB15_24
-; AVX512F-NEXT:  # %bb.23: # %cond.store21
-; AVX512F-NEXT:    vpextrb $11, %xmm0, 11(%rdi)
+; AVX512F-NEXT:    testl $2048, %eax # imm = 0x800
+; AVX512F-NEXT:    jne .LBB15_23
 ; AVX512F-NEXT:  .LBB15_24: # %else22
-; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm3, %zmm3, %zmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $12, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB15_26
-; AVX512F-NEXT:  # %bb.25: # %cond.store23
-; AVX512F-NEXT:    vpextrb $12, %xmm0, 12(%rdi)
+; AVX512F-NEXT:    testl $4096, %eax # imm = 0x1000
+; AVX512F-NEXT:    jne .LBB15_25
 ; AVX512F-NEXT:  .LBB15_26: # %else24
-; AVX512F-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
-; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $13, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB15_28
-; AVX512F-NEXT:  # %bb.27: # %cond.store25
-; AVX512F-NEXT:    vpextrb $13, %xmm0, 13(%rdi)
+; AVX512F-NEXT:    testl $8192, %eax # imm = 0x2000
+; AVX512F-NEXT:    jne .LBB15_27
 ; AVX512F-NEXT:  .LBB15_28: # %else26
-; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm3, %zmm3, %zmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $14, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB15_30
-; AVX512F-NEXT:  # %bb.29: # %cond.store27
-; AVX512F-NEXT:    vpextrb $14, %xmm0, 14(%rdi)
+; AVX512F-NEXT:    testl $16384, %eax # imm = 0x4000
+; AVX512F-NEXT:    jne .LBB15_29
 ; AVX512F-NEXT:  .LBB15_30: # %else28
-; AVX512F-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
-; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $15, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $32768, %eax # imm = 0x8000
 ; AVX512F-NEXT:    je .LBB15_32
-; AVX512F-NEXT:  # %bb.31: # %cond.store29
+; AVX512F-NEXT:  .LBB15_31: # %cond.store29
 ; AVX512F-NEXT:    vpextrb $15, %xmm0, 15(%rdi)
 ; AVX512F-NEXT:  .LBB15_32: # %else30
-; AVX512F-NEXT:    vextracti128 $1, %ymm2, %xmm1
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $65536, %eax # imm = 0x10000
 ; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; AVX512F-NEXT:    je .LBB15_34
-; AVX512F-NEXT:  # %bb.33: # %cond.store31
+; AVX512F-NEXT:    jne .LBB15_33
+; AVX512F-NEXT:  # %bb.34: # %else32
+; AVX512F-NEXT:    testl $131072, %eax # imm = 0x20000
+; AVX512F-NEXT:    jne .LBB15_35
+; AVX512F-NEXT:  .LBB15_36: # %else34
+; AVX512F-NEXT:    testl $262144, %eax # imm = 0x40000
+; AVX512F-NEXT:    jne .LBB15_37
+; AVX512F-NEXT:  .LBB15_38: # %else36
+; AVX512F-NEXT:    testl $524288, %eax # imm = 0x80000
+; AVX512F-NEXT:    jne .LBB15_39
+; AVX512F-NEXT:  .LBB15_40: # %else38
+; AVX512F-NEXT:    testl $1048576, %eax # imm = 0x100000
+; AVX512F-NEXT:    jne .LBB15_41
+; AVX512F-NEXT:  .LBB15_42: # %else40
+; AVX512F-NEXT:    testl $2097152, %eax # imm = 0x200000
+; AVX512F-NEXT:    jne .LBB15_43
+; AVX512F-NEXT:  .LBB15_44: # %else42
+; AVX512F-NEXT:    testl $4194304, %eax # imm = 0x400000
+; AVX512F-NEXT:    jne .LBB15_45
+; AVX512F-NEXT:  .LBB15_46: # %else44
+; AVX512F-NEXT:    testl $8388608, %eax # imm = 0x800000
+; AVX512F-NEXT:    jne .LBB15_47
+; AVX512F-NEXT:  .LBB15_48: # %else46
+; AVX512F-NEXT:    testl $16777216, %eax # imm = 0x1000000
+; AVX512F-NEXT:    jne .LBB15_49
+; AVX512F-NEXT:  .LBB15_50: # %else48
+; AVX512F-NEXT:    testl $33554432, %eax # imm = 0x2000000
+; AVX512F-NEXT:    jne .LBB15_51
+; AVX512F-NEXT:  .LBB15_52: # %else50
+; AVX512F-NEXT:    testl $67108864, %eax # imm = 0x4000000
+; AVX512F-NEXT:    jne .LBB15_53
+; AVX512F-NEXT:  .LBB15_54: # %else52
+; AVX512F-NEXT:    testl $134217728, %eax # imm = 0x8000000
+; AVX512F-NEXT:    jne .LBB15_55
+; AVX512F-NEXT:  .LBB15_56: # %else54
+; AVX512F-NEXT:    testl $268435456, %eax # imm = 0x10000000
+; AVX512F-NEXT:    jne .LBB15_57
+; AVX512F-NEXT:  .LBB15_58: # %else56
+; AVX512F-NEXT:    testl $536870912, %eax # imm = 0x20000000
+; AVX512F-NEXT:    jne .LBB15_59
+; AVX512F-NEXT:  .LBB15_60: # %else58
+; AVX512F-NEXT:    testl $1073741824, %eax # imm = 0x40000000
+; AVX512F-NEXT:    jne .LBB15_61
+; AVX512F-NEXT:  .LBB15_62: # %else60
+; AVX512F-NEXT:    testl $-2147483648, %eax # imm = 0x80000000
+; AVX512F-NEXT:    jne .LBB15_63
+; AVX512F-NEXT:  .LBB15_64: # %else62
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+; AVX512F-NEXT:  .LBB15_1: # %cond.store
+; AVX512F-NEXT:    vpextrb $0, %xmm0, (%rdi)
+; AVX512F-NEXT:    testb $2, %al
+; AVX512F-NEXT:    je .LBB15_4
+; AVX512F-NEXT:  .LBB15_3: # %cond.store1
+; AVX512F-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
+; AVX512F-NEXT:    testb $4, %al
+; AVX512F-NEXT:    je .LBB15_6
+; AVX512F-NEXT:  .LBB15_5: # %cond.store3
+; AVX512F-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
+; AVX512F-NEXT:    testb $8, %al
+; AVX512F-NEXT:    je .LBB15_8
+; AVX512F-NEXT:  .LBB15_7: # %cond.store5
+; AVX512F-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
+; AVX512F-NEXT:    testb $16, %al
+; AVX512F-NEXT:    je .LBB15_10
+; AVX512F-NEXT:  .LBB15_9: # %cond.store7
+; AVX512F-NEXT:    vpextrb $4, %xmm0, 4(%rdi)
+; AVX512F-NEXT:    testb $32, %al
+; AVX512F-NEXT:    je .LBB15_12
+; AVX512F-NEXT:  .LBB15_11: # %cond.store9
+; AVX512F-NEXT:    vpextrb $5, %xmm0, 5(%rdi)
+; AVX512F-NEXT:    testb $64, %al
+; AVX512F-NEXT:    je .LBB15_14
+; AVX512F-NEXT:  .LBB15_13: # %cond.store11
+; AVX512F-NEXT:    vpextrb $6, %xmm0, 6(%rdi)
+; AVX512F-NEXT:    testb $-128, %al
+; AVX512F-NEXT:    je .LBB15_16
+; AVX512F-NEXT:  .LBB15_15: # %cond.store13
+; AVX512F-NEXT:    vpextrb $7, %xmm0, 7(%rdi)
+; AVX512F-NEXT:    testl $256, %eax # imm = 0x100
+; AVX512F-NEXT:    je .LBB15_18
+; AVX512F-NEXT:  .LBB15_17: # %cond.store15
+; AVX512F-NEXT:    vpextrb $8, %xmm0, 8(%rdi)
+; AVX512F-NEXT:    testl $512, %eax # imm = 0x200
+; AVX512F-NEXT:    je .LBB15_20
+; AVX512F-NEXT:  .LBB15_19: # %cond.store17
+; AVX512F-NEXT:    vpextrb $9, %xmm0, 9(%rdi)
+; AVX512F-NEXT:    testl $1024, %eax # imm = 0x400
+; AVX512F-NEXT:    je .LBB15_22
+; AVX512F-NEXT:  .LBB15_21: # %cond.store19
+; AVX512F-NEXT:    vpextrb $10, %xmm0, 10(%rdi)
+; AVX512F-NEXT:    testl $2048, %eax # imm = 0x800
+; AVX512F-NEXT:    je .LBB15_24
+; AVX512F-NEXT:  .LBB15_23: # %cond.store21
+; AVX512F-NEXT:    vpextrb $11, %xmm0, 11(%rdi)
+; AVX512F-NEXT:    testl $4096, %eax # imm = 0x1000
+; AVX512F-NEXT:    je .LBB15_26
+; AVX512F-NEXT:  .LBB15_25: # %cond.store23
+; AVX512F-NEXT:    vpextrb $12, %xmm0, 12(%rdi)
+; AVX512F-NEXT:    testl $8192, %eax # imm = 0x2000
+; AVX512F-NEXT:    je .LBB15_28
+; AVX512F-NEXT:  .LBB15_27: # %cond.store25
+; AVX512F-NEXT:    vpextrb $13, %xmm0, 13(%rdi)
+; AVX512F-NEXT:    testl $16384, %eax # imm = 0x4000
+; AVX512F-NEXT:    je .LBB15_30
+; AVX512F-NEXT:  .LBB15_29: # %cond.store27
+; AVX512F-NEXT:    vpextrb $14, %xmm0, 14(%rdi)
+; AVX512F-NEXT:    testl $32768, %eax # imm = 0x8000
+; AVX512F-NEXT:    jne .LBB15_31
+; AVX512F-NEXT:    jmp .LBB15_32
+; AVX512F-NEXT:  .LBB15_33: # %cond.store31
 ; AVX512F-NEXT:    vpextrb $0, %xmm0, 16(%rdi)
-; AVX512F-NEXT:  .LBB15_34: # %else32
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $131072, %eax # imm = 0x20000
 ; AVX512F-NEXT:    je .LBB15_36
-; AVX512F-NEXT:  # %bb.35: # %cond.store33
+; AVX512F-NEXT:  .LBB15_35: # %cond.store33
 ; AVX512F-NEXT:    vpextrb $1, %xmm0, 17(%rdi)
-; AVX512F-NEXT:  .LBB15_36: # %else34
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $262144, %eax # imm = 0x40000
 ; AVX512F-NEXT:    je .LBB15_38
-; AVX512F-NEXT:  # %bb.37: # %cond.store35
+; AVX512F-NEXT:  .LBB15_37: # %cond.store35
 ; AVX512F-NEXT:    vpextrb $2, %xmm0, 18(%rdi)
-; AVX512F-NEXT:  .LBB15_38: # %else36
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $524288, %eax # imm = 0x80000
 ; AVX512F-NEXT:    je .LBB15_40
-; AVX512F-NEXT:  # %bb.39: # %cond.store37
+; AVX512F-NEXT:  .LBB15_39: # %cond.store37
 ; AVX512F-NEXT:    vpextrb $3, %xmm0, 19(%rdi)
-; AVX512F-NEXT:  .LBB15_40: # %else38
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $4, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $1048576, %eax # imm = 0x100000
 ; AVX512F-NEXT:    je .LBB15_42
-; AVX512F-NEXT:  # %bb.41: # %cond.store39
+; AVX512F-NEXT:  .LBB15_41: # %cond.store39
 ; AVX512F-NEXT:    vpextrb $4, %xmm0, 20(%rdi)
-; AVX512F-NEXT:  .LBB15_42: # %else40
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $5, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $2097152, %eax # imm = 0x200000
 ; AVX512F-NEXT:    je .LBB15_44
-; AVX512F-NEXT:  # %bb.43: # %cond.store41
+; AVX512F-NEXT:  .LBB15_43: # %cond.store41
 ; AVX512F-NEXT:    vpextrb $5, %xmm0, 21(%rdi)
-; AVX512F-NEXT:  .LBB15_44: # %else42
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $6, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $4194304, %eax # imm = 0x400000
 ; AVX512F-NEXT:    je .LBB15_46
-; AVX512F-NEXT:  # %bb.45: # %cond.store43
+; AVX512F-NEXT:  .LBB15_45: # %cond.store43
 ; AVX512F-NEXT:    vpextrb $6, %xmm0, 22(%rdi)
-; AVX512F-NEXT:  .LBB15_46: # %else44
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $7, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $8388608, %eax # imm = 0x800000
 ; AVX512F-NEXT:    je .LBB15_48
-; AVX512F-NEXT:  # %bb.47: # %cond.store45
+; AVX512F-NEXT:  .LBB15_47: # %cond.store45
 ; AVX512F-NEXT:    vpextrb $7, %xmm0, 23(%rdi)
-; AVX512F-NEXT:  .LBB15_48: # %else46
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $8, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $16777216, %eax # imm = 0x1000000
 ; AVX512F-NEXT:    je .LBB15_50
-; AVX512F-NEXT:  # %bb.49: # %cond.store47
+; AVX512F-NEXT:  .LBB15_49: # %cond.store47
 ; AVX512F-NEXT:    vpextrb $8, %xmm0, 24(%rdi)
-; AVX512F-NEXT:  .LBB15_50: # %else48
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $9, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $33554432, %eax # imm = 0x2000000
 ; AVX512F-NEXT:    je .LBB15_52
-; AVX512F-NEXT:  # %bb.51: # %cond.store49
+; AVX512F-NEXT:  .LBB15_51: # %cond.store49
 ; AVX512F-NEXT:    vpextrb $9, %xmm0, 25(%rdi)
-; AVX512F-NEXT:  .LBB15_52: # %else50
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $10, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $67108864, %eax # imm = 0x4000000
 ; AVX512F-NEXT:    je .LBB15_54
-; AVX512F-NEXT:  # %bb.53: # %cond.store51
+; AVX512F-NEXT:  .LBB15_53: # %cond.store51
 ; AVX512F-NEXT:    vpextrb $10, %xmm0, 26(%rdi)
-; AVX512F-NEXT:  .LBB15_54: # %else52
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $11, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $134217728, %eax # imm = 0x8000000
 ; AVX512F-NEXT:    je .LBB15_56
-; AVX512F-NEXT:  # %bb.55: # %cond.store53
+; AVX512F-NEXT:  .LBB15_55: # %cond.store53
 ; AVX512F-NEXT:    vpextrb $11, %xmm0, 27(%rdi)
-; AVX512F-NEXT:  .LBB15_56: # %else54
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $12, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $268435456, %eax # imm = 0x10000000
 ; AVX512F-NEXT:    je .LBB15_58
-; AVX512F-NEXT:  # %bb.57: # %cond.store55
+; AVX512F-NEXT:  .LBB15_57: # %cond.store55
 ; AVX512F-NEXT:    vpextrb $12, %xmm0, 28(%rdi)
-; AVX512F-NEXT:  .LBB15_58: # %else56
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $13, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $536870912, %eax # imm = 0x20000000
 ; AVX512F-NEXT:    je .LBB15_60
-; AVX512F-NEXT:  # %bb.59: # %cond.store57
+; AVX512F-NEXT:  .LBB15_59: # %cond.store57
 ; AVX512F-NEXT:    vpextrb $13, %xmm0, 29(%rdi)
-; AVX512F-NEXT:  .LBB15_60: # %else58
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm1
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm2
-; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $14, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $1073741824, %eax # imm = 0x40000000
 ; AVX512F-NEXT:    je .LBB15_62
-; AVX512F-NEXT:  # %bb.61: # %cond.store59
+; AVX512F-NEXT:  .LBB15_61: # %cond.store59
 ; AVX512F-NEXT:    vpextrb $14, %xmm0, 30(%rdi)
-; AVX512F-NEXT:  .LBB15_62: # %else60
-; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
-; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $15, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $-2147483648, %eax # imm = 0x80000000
 ; AVX512F-NEXT:    je .LBB15_64
-; AVX512F-NEXT:  # %bb.63: # %cond.store61
+; AVX512F-NEXT:  .LBB15_63: # %cond.store61
 ; AVX512F-NEXT:    vpextrb $15, %xmm0, 31(%rdi)
-; AVX512F-NEXT:  .LBB15_64: # %else62
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
@@ -7431,747 +6640,598 @@ define void @truncstore_v16i16_v16i8(<16 x i16> %x, <16 x i8>* %p, <16 x i8> %ma
 ; SSE2-LABEL: truncstore_v16i16_v16i8:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    pcmpeqb %xmm2, %xmm3
 ; SSE2-NEXT:    packsswb %xmm1, %xmm0
-; SSE2-NEXT:    movd %xmm3, %ecx
-; SSE2-NEXT:    movl %ecx, %eax
-; SSE2-NEXT:    notb %al
+; SSE2-NEXT:    pcmpeqb %xmm2, %xmm3
+; SSE2-NEXT:    pmovmskb %xmm3, %eax
+; SSE2-NEXT:    xorl $65535, %eax # imm = 0xFFFF
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    je .LBB16_2
-; SSE2-NEXT:  # %bb.1: # %cond.store
-; SSE2-NEXT:    movb %al, (%rdi)
-; SSE2-NEXT:  .LBB16_2: # %else
-; SSE2-NEXT:    shrl $8, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    je .LBB16_4
-; SSE2-NEXT:  # %bb.3: # %cond.store1
-; SSE2-NEXT:    movb %ah, 1(%rdi)
+; SSE2-NEXT:    movd %xmm0, %ecx
+; SSE2-NEXT:    jne .LBB16_1
+; SSE2-NEXT:  # %bb.2: # %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne .LBB16_3
 ; SSE2-NEXT:  .LBB16_4: # %else2
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqb %xmm2, %xmm1
-; SSE2-NEXT:    movd %xmm1, %ecx
-; SSE2-NEXT:    movl %ecx, %edx
-; SSE2-NEXT:    shrl $16, %edx
-; SSE2-NEXT:    notb %dl
-; SSE2-NEXT:    testb $1, %dl
-; SSE2-NEXT:    je .LBB16_6
-; SSE2-NEXT:  # %bb.5: # %cond.store3
-; SSE2-NEXT:    movl %eax, %edx
-; SSE2-NEXT:    shrl $16, %edx
-; SSE2-NEXT:    movb %dl, 2(%rdi)
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne .LBB16_5
 ; SSE2-NEXT:  .LBB16_6: # %else4
-; SSE2-NEXT:    shrl $24, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je .LBB16_8
-; SSE2-NEXT:  # %bb.7: # %cond.store5
-; SSE2-NEXT:    shrl $24, %eax
-; SSE2-NEXT:    movb %al, 3(%rdi)
+; SSE2-NEXT:  .LBB16_7: # %cond.store5
+; SSE2-NEXT:    shrl $24, %ecx
+; SSE2-NEXT:    movb %cl, 3(%rdi)
 ; SSE2-NEXT:  .LBB16_8: # %else6
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqb %xmm2, %xmm1
-; SSE2-NEXT:    pextrw $2, %xmm1, %eax
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testb $16, %al
 ; SSE2-NEXT:    pextrw $2, %xmm0, %ecx
 ; SSE2-NEXT:    je .LBB16_10
 ; SSE2-NEXT:  # %bb.9: # %cond.store7
 ; SSE2-NEXT:    movb %cl, 4(%rdi)
 ; SSE2-NEXT:  .LBB16_10: # %else8
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    notb %al
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $32, %al
 ; SSE2-NEXT:    je .LBB16_12
 ; SSE2-NEXT:  # %bb.11: # %cond.store9
 ; SSE2-NEXT:    movb %ch, 5(%rdi)
 ; SSE2-NEXT:  .LBB16_12: # %else10
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqb %xmm2, %xmm1
-; SSE2-NEXT:    pextrw $3, %xmm1, %eax
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testb $64, %al
 ; SSE2-NEXT:    pextrw $3, %xmm0, %ecx
 ; SSE2-NEXT:    je .LBB16_14
 ; SSE2-NEXT:  # %bb.13: # %cond.store11
 ; SSE2-NEXT:    movb %cl, 6(%rdi)
 ; SSE2-NEXT:  .LBB16_14: # %else12
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    notb %al
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $-128, %al
 ; SSE2-NEXT:    je .LBB16_16
 ; SSE2-NEXT:  # %bb.15: # %cond.store13
 ; SSE2-NEXT:    movb %ch, 7(%rdi)
 ; SSE2-NEXT:  .LBB16_16: # %else14
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqb %xmm2, %xmm1
-; SSE2-NEXT:    pextrw $4, %xmm1, %eax
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $256, %eax # imm = 0x100
 ; SSE2-NEXT:    pextrw $4, %xmm0, %ecx
 ; SSE2-NEXT:    je .LBB16_18
 ; SSE2-NEXT:  # %bb.17: # %cond.store15
 ; SSE2-NEXT:    movb %cl, 8(%rdi)
 ; SSE2-NEXT:  .LBB16_18: # %else16
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    notb %al
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $512, %eax # imm = 0x200
 ; SSE2-NEXT:    je .LBB16_20
 ; SSE2-NEXT:  # %bb.19: # %cond.store17
 ; SSE2-NEXT:    movb %ch, 9(%rdi)
 ; SSE2-NEXT:  .LBB16_20: # %else18
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqb %xmm2, %xmm1
-; SSE2-NEXT:    pextrw $5, %xmm1, %eax
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $1024, %eax # imm = 0x400
 ; SSE2-NEXT:    pextrw $5, %xmm0, %ecx
 ; SSE2-NEXT:    je .LBB16_22
 ; SSE2-NEXT:  # %bb.21: # %cond.store19
 ; SSE2-NEXT:    movb %cl, 10(%rdi)
 ; SSE2-NEXT:  .LBB16_22: # %else20
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    notb %al
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $2048, %eax # imm = 0x800
 ; SSE2-NEXT:    je .LBB16_24
 ; SSE2-NEXT:  # %bb.23: # %cond.store21
 ; SSE2-NEXT:    movb %ch, 11(%rdi)
 ; SSE2-NEXT:  .LBB16_24: # %else22
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqb %xmm2, %xmm1
-; SSE2-NEXT:    pextrw $6, %xmm1, %eax
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $4096, %eax # imm = 0x1000
 ; SSE2-NEXT:    pextrw $6, %xmm0, %ecx
 ; SSE2-NEXT:    je .LBB16_26
 ; SSE2-NEXT:  # %bb.25: # %cond.store23
 ; SSE2-NEXT:    movb %cl, 12(%rdi)
 ; SSE2-NEXT:  .LBB16_26: # %else24
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    notb %al
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $8192, %eax # imm = 0x2000
 ; SSE2-NEXT:    je .LBB16_28
 ; SSE2-NEXT:  # %bb.27: # %cond.store25
 ; SSE2-NEXT:    movb %ch, 13(%rdi)
 ; SSE2-NEXT:  .LBB16_28: # %else26
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqb %xmm1, %xmm2
-; SSE2-NEXT:    pextrw $7, %xmm2, %eax
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $16384, %eax # imm = 0x4000
 ; SSE2-NEXT:    pextrw $7, %xmm0, %ecx
-; SSE2-NEXT:    je .LBB16_30
-; SSE2-NEXT:  # %bb.29: # %cond.store27
+; SSE2-NEXT:    jne .LBB16_29
+; SSE2-NEXT:  # %bb.30: # %else28
+; SSE2-NEXT:    testl $32768, %eax # imm = 0x8000
+; SSE2-NEXT:    jne .LBB16_31
+; SSE2-NEXT:  .LBB16_32: # %else30
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  .LBB16_1: # %cond.store
+; SSE2-NEXT:    movb %cl, (%rdi)
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    je .LBB16_4
+; SSE2-NEXT:  .LBB16_3: # %cond.store1
+; SSE2-NEXT:    movb %ch, 1(%rdi)
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    je .LBB16_6
+; SSE2-NEXT:  .LBB16_5: # %cond.store3
+; SSE2-NEXT:    movl %ecx, %edx
+; SSE2-NEXT:    shrl $16, %edx
+; SSE2-NEXT:    movb %dl, 2(%rdi)
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne .LBB16_7
+; SSE2-NEXT:    jmp .LBB16_8
+; SSE2-NEXT:  .LBB16_29: # %cond.store27
 ; SSE2-NEXT:    movb %cl, 14(%rdi)
-; SSE2-NEXT:  .LBB16_30: # %else28
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    notb %al
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $32768, %eax # imm = 0x8000
 ; SSE2-NEXT:    je .LBB16_32
-; SSE2-NEXT:  # %bb.31: # %cond.store29
+; SSE2-NEXT:  .LBB16_31: # %cond.store29
 ; SSE2-NEXT:    movb %ch, 15(%rdi)
-; SSE2-NEXT:  .LBB16_32: # %else30
 ; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: truncstore_v16i16_v16i8:
 ; SSE4:       # %bb.0:
 ; SSE4-NEXT:    pxor %xmm3, %xmm3
-; SSE4-NEXT:    pcmpeqb %xmm2, %xmm3
 ; SSE4-NEXT:    packsswb %xmm1, %xmm0
-; SSE4-NEXT:    pextrb $0, %xmm3, %eax
-; SSE4-NEXT:    notb %al
+; SSE4-NEXT:    pcmpeqb %xmm2, %xmm3
+; SSE4-NEXT:    pmovmskb %xmm3, %eax
+; SSE4-NEXT:    xorl $65535, %eax # imm = 0xFFFF
 ; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB16_2
-; SSE4-NEXT:  # %bb.1: # %cond.store
+; SSE4-NEXT:    jne .LBB16_1
+; SSE4-NEXT:  # %bb.2: # %else
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    jne .LBB16_3
+; SSE4-NEXT:  .LBB16_4: # %else2
+; SSE4-NEXT:    testb $4, %al
+; SSE4-NEXT:    jne .LBB16_5
+; SSE4-NEXT:  .LBB16_6: # %else4
+; SSE4-NEXT:    testb $8, %al
+; SSE4-NEXT:    jne .LBB16_7
+; SSE4-NEXT:  .LBB16_8: # %else6
+; SSE4-NEXT:    testb $16, %al
+; SSE4-NEXT:    jne .LBB16_9
+; SSE4-NEXT:  .LBB16_10: # %else8
+; SSE4-NEXT:    testb $32, %al
+; SSE4-NEXT:    jne .LBB16_11
+; SSE4-NEXT:  .LBB16_12: # %else10
+; SSE4-NEXT:    testb $64, %al
+; SSE4-NEXT:    jne .LBB16_13
+; SSE4-NEXT:  .LBB16_14: # %else12
+; SSE4-NEXT:    testb $-128, %al
+; SSE4-NEXT:    jne .LBB16_15
+; SSE4-NEXT:  .LBB16_16: # %else14
+; SSE4-NEXT:    testl $256, %eax # imm = 0x100
+; SSE4-NEXT:    jne .LBB16_17
+; SSE4-NEXT:  .LBB16_18: # %else16
+; SSE4-NEXT:    testl $512, %eax # imm = 0x200
+; SSE4-NEXT:    jne .LBB16_19
+; SSE4-NEXT:  .LBB16_20: # %else18
+; SSE4-NEXT:    testl $1024, %eax # imm = 0x400
+; SSE4-NEXT:    jne .LBB16_21
+; SSE4-NEXT:  .LBB16_22: # %else20
+; SSE4-NEXT:    testl $2048, %eax # imm = 0x800
+; SSE4-NEXT:    jne .LBB16_23
+; SSE4-NEXT:  .LBB16_24: # %else22
+; SSE4-NEXT:    testl $4096, %eax # imm = 0x1000
+; SSE4-NEXT:    jne .LBB16_25
+; SSE4-NEXT:  .LBB16_26: # %else24
+; SSE4-NEXT:    testl $8192, %eax # imm = 0x2000
+; SSE4-NEXT:    jne .LBB16_27
+; SSE4-NEXT:  .LBB16_28: # %else26
+; SSE4-NEXT:    testl $16384, %eax # imm = 0x4000
+; SSE4-NEXT:    jne .LBB16_29
+; SSE4-NEXT:  .LBB16_30: # %else28
+; SSE4-NEXT:    testl $32768, %eax # imm = 0x8000
+; SSE4-NEXT:    jne .LBB16_31
+; SSE4-NEXT:  .LBB16_32: # %else30
+; SSE4-NEXT:    retq
+; SSE4-NEXT:  .LBB16_1: # %cond.store
 ; SSE4-NEXT:    pextrb $0, %xmm0, (%rdi)
-; SSE4-NEXT:  .LBB16_2: # %else
-; SSE4-NEXT:    pextrb $1, %xmm3, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $2, %al
 ; SSE4-NEXT:    je .LBB16_4
-; SSE4-NEXT:  # %bb.3: # %cond.store1
+; SSE4-NEXT:  .LBB16_3: # %cond.store1
 ; SSE4-NEXT:    pextrb $1, %xmm0, 1(%rdi)
-; SSE4-NEXT:  .LBB16_4: # %else2
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqb %xmm2, %xmm1
-; SSE4-NEXT:    pextrb $2, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $4, %al
 ; SSE4-NEXT:    je .LBB16_6
-; SSE4-NEXT:  # %bb.5: # %cond.store3
+; SSE4-NEXT:  .LBB16_5: # %cond.store3
 ; SSE4-NEXT:    pextrb $2, %xmm0, 2(%rdi)
-; SSE4-NEXT:  .LBB16_6: # %else4
-; SSE4-NEXT:    pextrb $3, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $8, %al
 ; SSE4-NEXT:    je .LBB16_8
-; SSE4-NEXT:  # %bb.7: # %cond.store5
+; SSE4-NEXT:  .LBB16_7: # %cond.store5
 ; SSE4-NEXT:    pextrb $3, %xmm0, 3(%rdi)
-; SSE4-NEXT:  .LBB16_8: # %else6
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqb %xmm2, %xmm1
-; SSE4-NEXT:    pextrb $4, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $16, %al
 ; SSE4-NEXT:    je .LBB16_10
-; SSE4-NEXT:  # %bb.9: # %cond.store7
+; SSE4-NEXT:  .LBB16_9: # %cond.store7
 ; SSE4-NEXT:    pextrb $4, %xmm0, 4(%rdi)
-; SSE4-NEXT:  .LBB16_10: # %else8
-; SSE4-NEXT:    pextrb $5, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $32, %al
 ; SSE4-NEXT:    je .LBB16_12
-; SSE4-NEXT:  # %bb.11: # %cond.store9
+; SSE4-NEXT:  .LBB16_11: # %cond.store9
 ; SSE4-NEXT:    pextrb $5, %xmm0, 5(%rdi)
-; SSE4-NEXT:  .LBB16_12: # %else10
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqb %xmm2, %xmm1
-; SSE4-NEXT:    pextrb $6, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $64, %al
 ; SSE4-NEXT:    je .LBB16_14
-; SSE4-NEXT:  # %bb.13: # %cond.store11
+; SSE4-NEXT:  .LBB16_13: # %cond.store11
 ; SSE4-NEXT:    pextrb $6, %xmm0, 6(%rdi)
-; SSE4-NEXT:  .LBB16_14: # %else12
-; SSE4-NEXT:    pextrb $7, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $-128, %al
 ; SSE4-NEXT:    je .LBB16_16
-; SSE4-NEXT:  # %bb.15: # %cond.store13
+; SSE4-NEXT:  .LBB16_15: # %cond.store13
 ; SSE4-NEXT:    pextrb $7, %xmm0, 7(%rdi)
-; SSE4-NEXT:  .LBB16_16: # %else14
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqb %xmm2, %xmm1
-; SSE4-NEXT:    pextrb $8, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $256, %eax # imm = 0x100
 ; SSE4-NEXT:    je .LBB16_18
-; SSE4-NEXT:  # %bb.17: # %cond.store15
+; SSE4-NEXT:  .LBB16_17: # %cond.store15
 ; SSE4-NEXT:    pextrb $8, %xmm0, 8(%rdi)
-; SSE4-NEXT:  .LBB16_18: # %else16
-; SSE4-NEXT:    pextrb $9, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $512, %eax # imm = 0x200
 ; SSE4-NEXT:    je .LBB16_20
-; SSE4-NEXT:  # %bb.19: # %cond.store17
+; SSE4-NEXT:  .LBB16_19: # %cond.store17
 ; SSE4-NEXT:    pextrb $9, %xmm0, 9(%rdi)
-; SSE4-NEXT:  .LBB16_20: # %else18
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqb %xmm2, %xmm1
-; SSE4-NEXT:    pextrb $10, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $1024, %eax # imm = 0x400
 ; SSE4-NEXT:    je .LBB16_22
-; SSE4-NEXT:  # %bb.21: # %cond.store19
+; SSE4-NEXT:  .LBB16_21: # %cond.store19
 ; SSE4-NEXT:    pextrb $10, %xmm0, 10(%rdi)
-; SSE4-NEXT:  .LBB16_22: # %else20
-; SSE4-NEXT:    pextrb $11, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $2048, %eax # imm = 0x800
 ; SSE4-NEXT:    je .LBB16_24
-; SSE4-NEXT:  # %bb.23: # %cond.store21
+; SSE4-NEXT:  .LBB16_23: # %cond.store21
 ; SSE4-NEXT:    pextrb $11, %xmm0, 11(%rdi)
-; SSE4-NEXT:  .LBB16_24: # %else22
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqb %xmm2, %xmm1
-; SSE4-NEXT:    pextrb $12, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $4096, %eax # imm = 0x1000
 ; SSE4-NEXT:    je .LBB16_26
-; SSE4-NEXT:  # %bb.25: # %cond.store23
+; SSE4-NEXT:  .LBB16_25: # %cond.store23
 ; SSE4-NEXT:    pextrb $12, %xmm0, 12(%rdi)
-; SSE4-NEXT:  .LBB16_26: # %else24
-; SSE4-NEXT:    pextrb $13, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $8192, %eax # imm = 0x2000
 ; SSE4-NEXT:    je .LBB16_28
-; SSE4-NEXT:  # %bb.27: # %cond.store25
+; SSE4-NEXT:  .LBB16_27: # %cond.store25
 ; SSE4-NEXT:    pextrb $13, %xmm0, 13(%rdi)
-; SSE4-NEXT:  .LBB16_28: # %else26
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqb %xmm1, %xmm2
-; SSE4-NEXT:    pextrb $14, %xmm2, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $16384, %eax # imm = 0x4000
 ; SSE4-NEXT:    je .LBB16_30
-; SSE4-NEXT:  # %bb.29: # %cond.store27
+; SSE4-NEXT:  .LBB16_29: # %cond.store27
 ; SSE4-NEXT:    pextrb $14, %xmm0, 14(%rdi)
-; SSE4-NEXT:  .LBB16_30: # %else28
-; SSE4-NEXT:    pextrb $15, %xmm2, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $32768, %eax # imm = 0x8000
 ; SSE4-NEXT:    je .LBB16_32
-; SSE4-NEXT:  # %bb.31: # %cond.store29
+; SSE4-NEXT:  .LBB16_31: # %cond.store29
 ; SSE4-NEXT:    pextrb $15, %xmm0, 15(%rdi)
-; SSE4-NEXT:  .LBB16_32: # %else30
 ; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: truncstore_v16i16_v16i8:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
 ; AVX1-NEXT:    vpacksswb %xmm3, %xmm0, %xmm0
-; AVX1-NEXT:    vpextrb $0, %xmm2, %eax
-; AVX1-NEXT:    notb %al
+; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpmovmskb %xmm1, %eax
+; AVX1-NEXT:    xorl $65535, %eax # imm = 0xFFFF
 ; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je .LBB16_2
-; AVX1-NEXT:  # %bb.1: # %cond.store
+; AVX1-NEXT:    jne .LBB16_1
+; AVX1-NEXT:  # %bb.2: # %else
+; AVX1-NEXT:    testb $2, %al
+; AVX1-NEXT:    jne .LBB16_3
+; AVX1-NEXT:  .LBB16_4: # %else2
+; AVX1-NEXT:    testb $4, %al
+; AVX1-NEXT:    jne .LBB16_5
+; AVX1-NEXT:  .LBB16_6: # %else4
+; AVX1-NEXT:    testb $8, %al
+; AVX1-NEXT:    jne .LBB16_7
+; AVX1-NEXT:  .LBB16_8: # %else6
+; AVX1-NEXT:    testb $16, %al
+; AVX1-NEXT:    jne .LBB16_9
+; AVX1-NEXT:  .LBB16_10: # %else8
+; AVX1-NEXT:    testb $32, %al
+; AVX1-NEXT:    jne .LBB16_11
+; AVX1-NEXT:  .LBB16_12: # %else10
+; AVX1-NEXT:    testb $64, %al
+; AVX1-NEXT:    jne .LBB16_13
+; AVX1-NEXT:  .LBB16_14: # %else12
+; AVX1-NEXT:    testb $-128, %al
+; AVX1-NEXT:    jne .LBB16_15
+; AVX1-NEXT:  .LBB16_16: # %else14
+; AVX1-NEXT:    testl $256, %eax # imm = 0x100
+; AVX1-NEXT:    jne .LBB16_17
+; AVX1-NEXT:  .LBB16_18: # %else16
+; AVX1-NEXT:    testl $512, %eax # imm = 0x200
+; AVX1-NEXT:    jne .LBB16_19
+; AVX1-NEXT:  .LBB16_20: # %else18
+; AVX1-NEXT:    testl $1024, %eax # imm = 0x400
+; AVX1-NEXT:    jne .LBB16_21
+; AVX1-NEXT:  .LBB16_22: # %else20
+; AVX1-NEXT:    testl $2048, %eax # imm = 0x800
+; AVX1-NEXT:    jne .LBB16_23
+; AVX1-NEXT:  .LBB16_24: # %else22
+; AVX1-NEXT:    testl $4096, %eax # imm = 0x1000
+; AVX1-NEXT:    jne .LBB16_25
+; AVX1-NEXT:  .LBB16_26: # %else24
+; AVX1-NEXT:    testl $8192, %eax # imm = 0x2000
+; AVX1-NEXT:    jne .LBB16_27
+; AVX1-NEXT:  .LBB16_28: # %else26
+; AVX1-NEXT:    testl $16384, %eax # imm = 0x4000
+; AVX1-NEXT:    jne .LBB16_29
+; AVX1-NEXT:  .LBB16_30: # %else28
+; AVX1-NEXT:    testl $32768, %eax # imm = 0x8000
+; AVX1-NEXT:    jne .LBB16_31
+; AVX1-NEXT:  .LBB16_32: # %else30
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+; AVX1-NEXT:  .LBB16_1: # %cond.store
 ; AVX1-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX1-NEXT:  .LBB16_2: # %else
-; AVX1-NEXT:    vpextrb $1, %xmm2, %eax
-; AVX1-NEXT:    notb %al
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $2, %al
 ; AVX1-NEXT:    je .LBB16_4
-; AVX1-NEXT:  # %bb.3: # %cond.store1
+; AVX1-NEXT:  .LBB16_3: # %cond.store1
 ; AVX1-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
-; AVX1-NEXT:  .LBB16_4: # %else2
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpextrb $2, %xmm2, %eax
-; AVX1-NEXT:    notb %al
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $4, %al
 ; AVX1-NEXT:    je .LBB16_6
-; AVX1-NEXT:  # %bb.5: # %cond.store3
+; AVX1-NEXT:  .LBB16_5: # %cond.store3
 ; AVX1-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
-; AVX1-NEXT:  .LBB16_6: # %else4
-; AVX1-NEXT:    vpextrb $3, %xmm2, %eax
-; AVX1-NEXT:    notb %al
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $8, %al
 ; AVX1-NEXT:    je .LBB16_8
-; AVX1-NEXT:  # %bb.7: # %cond.store5
+; AVX1-NEXT:  .LBB16_7: # %cond.store5
 ; AVX1-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
-; AVX1-NEXT:  .LBB16_8: # %else6
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX1-NEXT:    notb %al
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $16, %al
 ; AVX1-NEXT:    je .LBB16_10
-; AVX1-NEXT:  # %bb.9: # %cond.store7
+; AVX1-NEXT:  .LBB16_9: # %cond.store7
 ; AVX1-NEXT:    vpextrb $4, %xmm0, 4(%rdi)
-; AVX1-NEXT:  .LBB16_10: # %else8
-; AVX1-NEXT:    vpextrb $5, %xmm2, %eax
-; AVX1-NEXT:    notb %al
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $32, %al
 ; AVX1-NEXT:    je .LBB16_12
-; AVX1-NEXT:  # %bb.11: # %cond.store9
+; AVX1-NEXT:  .LBB16_11: # %cond.store9
 ; AVX1-NEXT:    vpextrb $5, %xmm0, 5(%rdi)
-; AVX1-NEXT:  .LBB16_12: # %else10
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpextrb $6, %xmm2, %eax
-; AVX1-NEXT:    notb %al
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $64, %al
 ; AVX1-NEXT:    je .LBB16_14
-; AVX1-NEXT:  # %bb.13: # %cond.store11
+; AVX1-NEXT:  .LBB16_13: # %cond.store11
 ; AVX1-NEXT:    vpextrb $6, %xmm0, 6(%rdi)
-; AVX1-NEXT:  .LBB16_14: # %else12
-; AVX1-NEXT:    vpextrb $7, %xmm2, %eax
-; AVX1-NEXT:    notb %al
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $-128, %al
 ; AVX1-NEXT:    je .LBB16_16
-; AVX1-NEXT:  # %bb.15: # %cond.store13
+; AVX1-NEXT:  .LBB16_15: # %cond.store13
 ; AVX1-NEXT:    vpextrb $7, %xmm0, 7(%rdi)
-; AVX1-NEXT:  .LBB16_16: # %else14
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpextrb $8, %xmm2, %eax
-; AVX1-NEXT:    notb %al
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $256, %eax # imm = 0x100
 ; AVX1-NEXT:    je .LBB16_18
-; AVX1-NEXT:  # %bb.17: # %cond.store15
+; AVX1-NEXT:  .LBB16_17: # %cond.store15
 ; AVX1-NEXT:    vpextrb $8, %xmm0, 8(%rdi)
-; AVX1-NEXT:  .LBB16_18: # %else16
-; AVX1-NEXT:    vpextrb $9, %xmm2, %eax
-; AVX1-NEXT:    notb %al
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $512, %eax # imm = 0x200
 ; AVX1-NEXT:    je .LBB16_20
-; AVX1-NEXT:  # %bb.19: # %cond.store17
+; AVX1-NEXT:  .LBB16_19: # %cond.store17
 ; AVX1-NEXT:    vpextrb $9, %xmm0, 9(%rdi)
-; AVX1-NEXT:  .LBB16_20: # %else18
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpextrb $10, %xmm2, %eax
-; AVX1-NEXT:    notb %al
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $1024, %eax # imm = 0x400
 ; AVX1-NEXT:    je .LBB16_22
-; AVX1-NEXT:  # %bb.21: # %cond.store19
+; AVX1-NEXT:  .LBB16_21: # %cond.store19
 ; AVX1-NEXT:    vpextrb $10, %xmm0, 10(%rdi)
-; AVX1-NEXT:  .LBB16_22: # %else20
-; AVX1-NEXT:    vpextrb $11, %xmm2, %eax
-; AVX1-NEXT:    notb %al
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $2048, %eax # imm = 0x800
 ; AVX1-NEXT:    je .LBB16_24
-; AVX1-NEXT:  # %bb.23: # %cond.store21
+; AVX1-NEXT:  .LBB16_23: # %cond.store21
 ; AVX1-NEXT:    vpextrb $11, %xmm0, 11(%rdi)
-; AVX1-NEXT:  .LBB16_24: # %else22
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpextrb $12, %xmm2, %eax
-; AVX1-NEXT:    notb %al
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $4096, %eax # imm = 0x1000
 ; AVX1-NEXT:    je .LBB16_26
-; AVX1-NEXT:  # %bb.25: # %cond.store23
+; AVX1-NEXT:  .LBB16_25: # %cond.store23
 ; AVX1-NEXT:    vpextrb $12, %xmm0, 12(%rdi)
-; AVX1-NEXT:  .LBB16_26: # %else24
-; AVX1-NEXT:    vpextrb $13, %xmm2, %eax
-; AVX1-NEXT:    notb %al
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $8192, %eax # imm = 0x2000
 ; AVX1-NEXT:    je .LBB16_28
-; AVX1-NEXT:  # %bb.27: # %cond.store25
+; AVX1-NEXT:  .LBB16_27: # %cond.store25
 ; AVX1-NEXT:    vpextrb $13, %xmm0, 13(%rdi)
-; AVX1-NEXT:  .LBB16_28: # %else26
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $14, %xmm1, %eax
-; AVX1-NEXT:    notb %al
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $16384, %eax # imm = 0x4000
 ; AVX1-NEXT:    je .LBB16_30
-; AVX1-NEXT:  # %bb.29: # %cond.store27
+; AVX1-NEXT:  .LBB16_29: # %cond.store27
 ; AVX1-NEXT:    vpextrb $14, %xmm0, 14(%rdi)
-; AVX1-NEXT:  .LBB16_30: # %else28
-; AVX1-NEXT:    vpextrb $15, %xmm1, %eax
-; AVX1-NEXT:    notb %al
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $32768, %eax # imm = 0x8000
 ; AVX1-NEXT:    je .LBB16_32
-; AVX1-NEXT:  # %bb.31: # %cond.store29
+; AVX1-NEXT:  .LBB16_31: # %cond.store29
 ; AVX1-NEXT:    vpextrb $15, %xmm0, 15(%rdi)
-; AVX1-NEXT:  .LBB16_32: # %else30
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: truncstore_v16i16_v16i8:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm3
 ; AVX2-NEXT:    vpacksswb %xmm3, %xmm0, %xmm0
-; AVX2-NEXT:    vpextrb $0, %xmm2, %eax
-; AVX2-NEXT:    notb %al
+; AVX2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpmovmskb %xmm1, %eax
+; AVX2-NEXT:    xorl $65535, %eax # imm = 0xFFFF
 ; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je .LBB16_2
-; AVX2-NEXT:  # %bb.1: # %cond.store
+; AVX2-NEXT:    jne .LBB16_1
+; AVX2-NEXT:  # %bb.2: # %else
+; AVX2-NEXT:    testb $2, %al
+; AVX2-NEXT:    jne .LBB16_3
+; AVX2-NEXT:  .LBB16_4: # %else2
+; AVX2-NEXT:    testb $4, %al
+; AVX2-NEXT:    jne .LBB16_5
+; AVX2-NEXT:  .LBB16_6: # %else4
+; AVX2-NEXT:    testb $8, %al
+; AVX2-NEXT:    jne .LBB16_7
+; AVX2-NEXT:  .LBB16_8: # %else6
+; AVX2-NEXT:    testb $16, %al
+; AVX2-NEXT:    jne .LBB16_9
+; AVX2-NEXT:  .LBB16_10: # %else8
+; AVX2-NEXT:    testb $32, %al
+; AVX2-NEXT:    jne .LBB16_11
+; AVX2-NEXT:  .LBB16_12: # %else10
+; AVX2-NEXT:    testb $64, %al
+; AVX2-NEXT:    jne .LBB16_13
+; AVX2-NEXT:  .LBB16_14: # %else12
+; AVX2-NEXT:    testb $-128, %al
+; AVX2-NEXT:    jne .LBB16_15
+; AVX2-NEXT:  .LBB16_16: # %else14
+; AVX2-NEXT:    testl $256, %eax # imm = 0x100
+; AVX2-NEXT:    jne .LBB16_17
+; AVX2-NEXT:  .LBB16_18: # %else16
+; AVX2-NEXT:    testl $512, %eax # imm = 0x200
+; AVX2-NEXT:    jne .LBB16_19
+; AVX2-NEXT:  .LBB16_20: # %else18
+; AVX2-NEXT:    testl $1024, %eax # imm = 0x400
+; AVX2-NEXT:    jne .LBB16_21
+; AVX2-NEXT:  .LBB16_22: # %else20
+; AVX2-NEXT:    testl $2048, %eax # imm = 0x800
+; AVX2-NEXT:    jne .LBB16_23
+; AVX2-NEXT:  .LBB16_24: # %else22
+; AVX2-NEXT:    testl $4096, %eax # imm = 0x1000
+; AVX2-NEXT:    jne .LBB16_25
+; AVX2-NEXT:  .LBB16_26: # %else24
+; AVX2-NEXT:    testl $8192, %eax # imm = 0x2000
+; AVX2-NEXT:    jne .LBB16_27
+; AVX2-NEXT:  .LBB16_28: # %else26
+; AVX2-NEXT:    testl $16384, %eax # imm = 0x4000
+; AVX2-NEXT:    jne .LBB16_29
+; AVX2-NEXT:  .LBB16_30: # %else28
+; AVX2-NEXT:    testl $32768, %eax # imm = 0x8000
+; AVX2-NEXT:    jne .LBB16_31
+; AVX2-NEXT:  .LBB16_32: # %else30
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+; AVX2-NEXT:  .LBB16_1: # %cond.store
 ; AVX2-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX2-NEXT:  .LBB16_2: # %else
-; AVX2-NEXT:    vpextrb $1, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $2, %al
 ; AVX2-NEXT:    je .LBB16_4
-; AVX2-NEXT:  # %bb.3: # %cond.store1
+; AVX2-NEXT:  .LBB16_3: # %cond.store1
 ; AVX2-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
-; AVX2-NEXT:  .LBB16_4: # %else2
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpextrb $2, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $4, %al
 ; AVX2-NEXT:    je .LBB16_6
-; AVX2-NEXT:  # %bb.5: # %cond.store3
+; AVX2-NEXT:  .LBB16_5: # %cond.store3
 ; AVX2-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
-; AVX2-NEXT:  .LBB16_6: # %else4
-; AVX2-NEXT:    vpextrb $3, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $8, %al
 ; AVX2-NEXT:    je .LBB16_8
-; AVX2-NEXT:  # %bb.7: # %cond.store5
+; AVX2-NEXT:  .LBB16_7: # %cond.store5
 ; AVX2-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
-; AVX2-NEXT:  .LBB16_8: # %else6
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $16, %al
 ; AVX2-NEXT:    je .LBB16_10
-; AVX2-NEXT:  # %bb.9: # %cond.store7
+; AVX2-NEXT:  .LBB16_9: # %cond.store7
 ; AVX2-NEXT:    vpextrb $4, %xmm0, 4(%rdi)
-; AVX2-NEXT:  .LBB16_10: # %else8
-; AVX2-NEXT:    vpextrb $5, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $32, %al
 ; AVX2-NEXT:    je .LBB16_12
-; AVX2-NEXT:  # %bb.11: # %cond.store9
+; AVX2-NEXT:  .LBB16_11: # %cond.store9
 ; AVX2-NEXT:    vpextrb $5, %xmm0, 5(%rdi)
-; AVX2-NEXT:  .LBB16_12: # %else10
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpextrb $6, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $64, %al
 ; AVX2-NEXT:    je .LBB16_14
-; AVX2-NEXT:  # %bb.13: # %cond.store11
+; AVX2-NEXT:  .LBB16_13: # %cond.store11
 ; AVX2-NEXT:    vpextrb $6, %xmm0, 6(%rdi)
-; AVX2-NEXT:  .LBB16_14: # %else12
-; AVX2-NEXT:    vpextrb $7, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $-128, %al
 ; AVX2-NEXT:    je .LBB16_16
-; AVX2-NEXT:  # %bb.15: # %cond.store13
+; AVX2-NEXT:  .LBB16_15: # %cond.store13
 ; AVX2-NEXT:    vpextrb $7, %xmm0, 7(%rdi)
-; AVX2-NEXT:  .LBB16_16: # %else14
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpextrb $8, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $256, %eax # imm = 0x100
 ; AVX2-NEXT:    je .LBB16_18
-; AVX2-NEXT:  # %bb.17: # %cond.store15
+; AVX2-NEXT:  .LBB16_17: # %cond.store15
 ; AVX2-NEXT:    vpextrb $8, %xmm0, 8(%rdi)
-; AVX2-NEXT:  .LBB16_18: # %else16
-; AVX2-NEXT:    vpextrb $9, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $512, %eax # imm = 0x200
 ; AVX2-NEXT:    je .LBB16_20
-; AVX2-NEXT:  # %bb.19: # %cond.store17
+; AVX2-NEXT:  .LBB16_19: # %cond.store17
 ; AVX2-NEXT:    vpextrb $9, %xmm0, 9(%rdi)
-; AVX2-NEXT:  .LBB16_20: # %else18
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpextrb $10, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $1024, %eax # imm = 0x400
 ; AVX2-NEXT:    je .LBB16_22
-; AVX2-NEXT:  # %bb.21: # %cond.store19
+; AVX2-NEXT:  .LBB16_21: # %cond.store19
 ; AVX2-NEXT:    vpextrb $10, %xmm0, 10(%rdi)
-; AVX2-NEXT:  .LBB16_22: # %else20
-; AVX2-NEXT:    vpextrb $11, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $2048, %eax # imm = 0x800
 ; AVX2-NEXT:    je .LBB16_24
-; AVX2-NEXT:  # %bb.23: # %cond.store21
+; AVX2-NEXT:  .LBB16_23: # %cond.store21
 ; AVX2-NEXT:    vpextrb $11, %xmm0, 11(%rdi)
-; AVX2-NEXT:  .LBB16_24: # %else22
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpextrb $12, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $4096, %eax # imm = 0x1000
 ; AVX2-NEXT:    je .LBB16_26
-; AVX2-NEXT:  # %bb.25: # %cond.store23
+; AVX2-NEXT:  .LBB16_25: # %cond.store23
 ; AVX2-NEXT:    vpextrb $12, %xmm0, 12(%rdi)
-; AVX2-NEXT:  .LBB16_26: # %else24
-; AVX2-NEXT:    vpextrb $13, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $8192, %eax # imm = 0x2000
 ; AVX2-NEXT:    je .LBB16_28
-; AVX2-NEXT:  # %bb.27: # %cond.store25
+; AVX2-NEXT:  .LBB16_27: # %cond.store25
 ; AVX2-NEXT:    vpextrb $13, %xmm0, 13(%rdi)
-; AVX2-NEXT:  .LBB16_28: # %else26
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpextrb $14, %xmm1, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $16384, %eax # imm = 0x4000
 ; AVX2-NEXT:    je .LBB16_30
-; AVX2-NEXT:  # %bb.29: # %cond.store27
+; AVX2-NEXT:  .LBB16_29: # %cond.store27
 ; AVX2-NEXT:    vpextrb $14, %xmm0, 14(%rdi)
-; AVX2-NEXT:  .LBB16_30: # %else28
-; AVX2-NEXT:    vpextrb $15, %xmm1, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $32768, %eax # imm = 0x8000
 ; AVX2-NEXT:    je .LBB16_32
-; AVX2-NEXT:  # %bb.31: # %cond.store29
+; AVX2-NEXT:  .LBB16_31: # %cond.store29
 ; AVX2-NEXT:    vpextrb $15, %xmm0, 15(%rdi)
-; AVX2-NEXT:  .LBB16_32: # %else30
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: truncstore_v16i16_v16i8:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    vpminsw {{.*}}(%rip), %ymm0, %ymm0
-; AVX512F-NEXT:    vpmaxsw {{.*}}(%rip), %ymm0, %ymm0
-; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
-; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB16_2
-; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm1
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm2
+; AVX512F-NEXT:    vpacksswb %xmm2, %xmm0, %xmm0
+; AVX512F-NEXT:    vpmovmskb %xmm1, %eax
+; AVX512F-NEXT:    xorl $65535, %eax # imm = 0xFFFF
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    jne .LBB16_1
+; AVX512F-NEXT:  # %bb.2: # %else
+; AVX512F-NEXT:    testb $2, %al
+; AVX512F-NEXT:    jne .LBB16_3
+; AVX512F-NEXT:  .LBB16_4: # %else2
+; AVX512F-NEXT:    testb $4, %al
+; AVX512F-NEXT:    jne .LBB16_5
+; AVX512F-NEXT:  .LBB16_6: # %else4
+; AVX512F-NEXT:    testb $8, %al
+; AVX512F-NEXT:    jne .LBB16_7
+; AVX512F-NEXT:  .LBB16_8: # %else6
+; AVX512F-NEXT:    testb $16, %al
+; AVX512F-NEXT:    jne .LBB16_9
+; AVX512F-NEXT:  .LBB16_10: # %else8
+; AVX512F-NEXT:    testb $32, %al
+; AVX512F-NEXT:    jne .LBB16_11
+; AVX512F-NEXT:  .LBB16_12: # %else10
+; AVX512F-NEXT:    testb $64, %al
+; AVX512F-NEXT:    jne .LBB16_13
+; AVX512F-NEXT:  .LBB16_14: # %else12
+; AVX512F-NEXT:    testb $-128, %al
+; AVX512F-NEXT:    jne .LBB16_15
+; AVX512F-NEXT:  .LBB16_16: # %else14
+; AVX512F-NEXT:    testl $256, %eax # imm = 0x100
+; AVX512F-NEXT:    jne .LBB16_17
+; AVX512F-NEXT:  .LBB16_18: # %else16
+; AVX512F-NEXT:    testl $512, %eax # imm = 0x200
+; AVX512F-NEXT:    jne .LBB16_19
+; AVX512F-NEXT:  .LBB16_20: # %else18
+; AVX512F-NEXT:    testl $1024, %eax # imm = 0x400
+; AVX512F-NEXT:    jne .LBB16_21
+; AVX512F-NEXT:  .LBB16_22: # %else20
+; AVX512F-NEXT:    testl $2048, %eax # imm = 0x800
+; AVX512F-NEXT:    jne .LBB16_23
+; AVX512F-NEXT:  .LBB16_24: # %else22
+; AVX512F-NEXT:    testl $4096, %eax # imm = 0x1000
+; AVX512F-NEXT:    jne .LBB16_25
+; AVX512F-NEXT:  .LBB16_26: # %else24
+; AVX512F-NEXT:    testl $8192, %eax # imm = 0x2000
+; AVX512F-NEXT:    jne .LBB16_27
+; AVX512F-NEXT:  .LBB16_28: # %else26
+; AVX512F-NEXT:    testl $16384, %eax # imm = 0x4000
+; AVX512F-NEXT:    jne .LBB16_29
+; AVX512F-NEXT:  .LBB16_30: # %else28
+; AVX512F-NEXT:    testl $32768, %eax # imm = 0x8000
+; AVX512F-NEXT:    jne .LBB16_31
+; AVX512F-NEXT:  .LBB16_32: # %else30
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+; AVX512F-NEXT:  .LBB16_1: # %cond.store
 ; AVX512F-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX512F-NEXT:  .LBB16_2: # %else
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $2, %al
 ; AVX512F-NEXT:    je .LBB16_4
-; AVX512F-NEXT:  # %bb.3: # %cond.store1
+; AVX512F-NEXT:  .LBB16_3: # %cond.store1
 ; AVX512F-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
-; AVX512F-NEXT:  .LBB16_4: # %else2
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $4, %al
 ; AVX512F-NEXT:    je .LBB16_6
-; AVX512F-NEXT:  # %bb.5: # %cond.store3
+; AVX512F-NEXT:  .LBB16_5: # %cond.store3
 ; AVX512F-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
-; AVX512F-NEXT:  .LBB16_6: # %else4
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $8, %al
 ; AVX512F-NEXT:    je .LBB16_8
-; AVX512F-NEXT:  # %bb.7: # %cond.store5
+; AVX512F-NEXT:  .LBB16_7: # %cond.store5
 ; AVX512F-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
-; AVX512F-NEXT:  .LBB16_8: # %else6
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $4, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $16, %al
 ; AVX512F-NEXT:    je .LBB16_10
-; AVX512F-NEXT:  # %bb.9: # %cond.store7
+; AVX512F-NEXT:  .LBB16_9: # %cond.store7
 ; AVX512F-NEXT:    vpextrb $4, %xmm0, 4(%rdi)
-; AVX512F-NEXT:  .LBB16_10: # %else8
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $5, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $32, %al
 ; AVX512F-NEXT:    je .LBB16_12
-; AVX512F-NEXT:  # %bb.11: # %cond.store9
+; AVX512F-NEXT:  .LBB16_11: # %cond.store9
 ; AVX512F-NEXT:    vpextrb $5, %xmm0, 5(%rdi)
-; AVX512F-NEXT:  .LBB16_12: # %else10
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $6, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $64, %al
 ; AVX512F-NEXT:    je .LBB16_14
-; AVX512F-NEXT:  # %bb.13: # %cond.store11
+; AVX512F-NEXT:  .LBB16_13: # %cond.store11
 ; AVX512F-NEXT:    vpextrb $6, %xmm0, 6(%rdi)
-; AVX512F-NEXT:  .LBB16_14: # %else12
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $7, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $-128, %al
 ; AVX512F-NEXT:    je .LBB16_16
-; AVX512F-NEXT:  # %bb.15: # %cond.store13
+; AVX512F-NEXT:  .LBB16_15: # %cond.store13
 ; AVX512F-NEXT:    vpextrb $7, %xmm0, 7(%rdi)
-; AVX512F-NEXT:  .LBB16_16: # %else14
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $8, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $256, %eax # imm = 0x100
 ; AVX512F-NEXT:    je .LBB16_18
-; AVX512F-NEXT:  # %bb.17: # %cond.store15
+; AVX512F-NEXT:  .LBB16_17: # %cond.store15
 ; AVX512F-NEXT:    vpextrb $8, %xmm0, 8(%rdi)
-; AVX512F-NEXT:  .LBB16_18: # %else16
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $9, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $512, %eax # imm = 0x200
 ; AVX512F-NEXT:    je .LBB16_20
-; AVX512F-NEXT:  # %bb.19: # %cond.store17
+; AVX512F-NEXT:  .LBB16_19: # %cond.store17
 ; AVX512F-NEXT:    vpextrb $9, %xmm0, 9(%rdi)
-; AVX512F-NEXT:  .LBB16_20: # %else18
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $10, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $1024, %eax # imm = 0x400
 ; AVX512F-NEXT:    je .LBB16_22
-; AVX512F-NEXT:  # %bb.21: # %cond.store19
+; AVX512F-NEXT:  .LBB16_21: # %cond.store19
 ; AVX512F-NEXT:    vpextrb $10, %xmm0, 10(%rdi)
-; AVX512F-NEXT:  .LBB16_22: # %else20
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $11, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $2048, %eax # imm = 0x800
 ; AVX512F-NEXT:    je .LBB16_24
-; AVX512F-NEXT:  # %bb.23: # %cond.store21
+; AVX512F-NEXT:  .LBB16_23: # %cond.store21
 ; AVX512F-NEXT:    vpextrb $11, %xmm0, 11(%rdi)
-; AVX512F-NEXT:  .LBB16_24: # %else22
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $12, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $4096, %eax # imm = 0x1000
 ; AVX512F-NEXT:    je .LBB16_26
-; AVX512F-NEXT:  # %bb.25: # %cond.store23
+; AVX512F-NEXT:  .LBB16_25: # %cond.store23
 ; AVX512F-NEXT:    vpextrb $12, %xmm0, 12(%rdi)
-; AVX512F-NEXT:  .LBB16_26: # %else24
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $13, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $8192, %eax # imm = 0x2000
 ; AVX512F-NEXT:    je .LBB16_28
-; AVX512F-NEXT:  # %bb.27: # %cond.store25
+; AVX512F-NEXT:  .LBB16_27: # %cond.store25
 ; AVX512F-NEXT:    vpextrb $13, %xmm0, 13(%rdi)
-; AVX512F-NEXT:  .LBB16_28: # %else26
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm1
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm2
-; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $14, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $16384, %eax # imm = 0x4000
 ; AVX512F-NEXT:    je .LBB16_30
-; AVX512F-NEXT:  # %bb.29: # %cond.store27
+; AVX512F-NEXT:  .LBB16_29: # %cond.store27
 ; AVX512F-NEXT:    vpextrb $14, %xmm0, 14(%rdi)
-; AVX512F-NEXT:  .LBB16_30: # %else28
-; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
-; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $15, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $32768, %eax # imm = 0x8000
 ; AVX512F-NEXT:    je .LBB16_32
-; AVX512F-NEXT:  # %bb.31: # %cond.store29
+; AVX512F-NEXT:  .LBB16_31: # %cond.store29
 ; AVX512F-NEXT:    vpextrb $15, %xmm0, 15(%rdi)
-; AVX512F-NEXT:  .LBB16_32: # %else30
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
@@ -8180,9 +7240,8 @@ define void @truncstore_v16i16_v16i8(<16 x i16> %x, <16 x i8>* %p, <16 x i8> %ma
 ; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
 ; AVX512BW-NEXT:    vptestmb %zmm1, %zmm1, %k0
 ; AVX512BW-NEXT:    kmovw %k0, %k1
-; AVX512BW-NEXT:    vpminsw {{.*}}(%rip), %ymm0, %ymm0
-; AVX512BW-NEXT:    vpmaxsw {{.*}}(%rip), %ymm0, %ymm0
-; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512BW-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vmovdqu8 %zmm0, (%rdi) {%k1}
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -8209,316 +7268,267 @@ define void @truncstore_v8i16_v8i8(<8 x i16> %x, <8 x i8>* %p, <8 x i16> %mask)
 ; SSE2-LABEL: truncstore_v8i16_v8i8:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    packsswb %xmm0, %xmm0
 ; SSE2-NEXT:    pcmpeqw %xmm1, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm3
-; SSE2-NEXT:    pxor %xmm2, %xmm3
-; SSE2-NEXT:    pminsw {{.*}}(%rip), %xmm0
-; SSE2-NEXT:    pmaxsw {{.*}}(%rip), %xmm0
-; SSE2-NEXT:    movd %xmm3, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    je .LBB17_2
-; SSE2-NEXT:  # %bb.1: # %cond.store
-; SSE2-NEXT:    movb %al, (%rdi)
-; SSE2-NEXT:  .LBB17_2: # %else
-; SSE2-NEXT:    shrl $16, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    je .LBB17_4
-; SSE2-NEXT:  # %bb.3: # %cond.store1
-; SSE2-NEXT:    shrl $16, %eax
-; SSE2-NEXT:    movb %al, 1(%rdi)
-; SSE2-NEXT:  .LBB17_4: # %else2
-; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    pcmpeqw %xmm1, %xmm3
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT:    pxor %xmm3, %xmm2
-; SSE2-NEXT:    pextrw $2, %xmm2, %eax
+; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
+; SSE2-NEXT:    pxor %xmm2, %xmm1
+; SSE2-NEXT:    packsswb %xmm0, %xmm1
+; SSE2-NEXT:    pmovmskb %xmm1, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB17_6
-; SSE2-NEXT:  # %bb.5: # %cond.store3
-; SSE2-NEXT:    pextrw $2, %xmm0, %eax
-; SSE2-NEXT:    movb %al, 2(%rdi)
+; SSE2-NEXT:    movd %xmm0, %ecx
+; SSE2-NEXT:    jne .LBB17_1
+; SSE2-NEXT:  # %bb.2: # %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne .LBB17_3
+; SSE2-NEXT:  .LBB17_4: # %else2
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne .LBB17_5
 ; SSE2-NEXT:  .LBB17_6: # %else4
-; SSE2-NEXT:    pextrw $3, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je .LBB17_8
-; SSE2-NEXT:  # %bb.7: # %cond.store5
-; SSE2-NEXT:    pextrw $3, %xmm0, %eax
-; SSE2-NEXT:    movb %al, 3(%rdi)
+; SSE2-NEXT:  .LBB17_7: # %cond.store5
+; SSE2-NEXT:    shrl $24, %ecx
+; SSE2-NEXT:    movb %cl, 3(%rdi)
 ; SSE2-NEXT:  .LBB17_8: # %else6
-; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    pcmpeqw %xmm1, %xmm3
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT:    pxor %xmm3, %xmm2
-; SSE2-NEXT:    pextrw $4, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $16, %al
+; SSE2-NEXT:    pextrw $2, %xmm0, %ecx
 ; SSE2-NEXT:    je .LBB17_10
 ; SSE2-NEXT:  # %bb.9: # %cond.store7
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    movb %al, 4(%rdi)
+; SSE2-NEXT:    movb %cl, 4(%rdi)
 ; SSE2-NEXT:  .LBB17_10: # %else8
-; SSE2-NEXT:    pextrw $5, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $32, %al
 ; SSE2-NEXT:    je .LBB17_12
 ; SSE2-NEXT:  # %bb.11: # %cond.store9
-; SSE2-NEXT:    pextrw $5, %xmm0, %eax
-; SSE2-NEXT:    movb %al, 5(%rdi)
+; SSE2-NEXT:    movb %ch, 5(%rdi)
 ; SSE2-NEXT:  .LBB17_12: # %else10
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpeqw %xmm2, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT:    pxor %xmm1, %xmm2
-; SSE2-NEXT:    pextrw $6, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB17_14
-; SSE2-NEXT:  # %bb.13: # %cond.store11
-; SSE2-NEXT:    pextrw $6, %xmm0, %eax
-; SSE2-NEXT:    movb %al, 6(%rdi)
-; SSE2-NEXT:  .LBB17_14: # %else12
-; SSE2-NEXT:    pextrw $7, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB17_16
-; SSE2-NEXT:  # %bb.15: # %cond.store13
-; SSE2-NEXT:    pextrw $7, %xmm0, %eax
-; SSE2-NEXT:    movb %al, 7(%rdi)
+; SSE2-NEXT:    testb $64, %al
+; SSE2-NEXT:    pextrw $3, %xmm0, %ecx
+; SSE2-NEXT:    jne .LBB17_13
+; SSE2-NEXT:  # %bb.14: # %else12
+; SSE2-NEXT:    testb $-128, %al
+; SSE2-NEXT:    jne .LBB17_15
 ; SSE2-NEXT:  .LBB17_16: # %else14
 ; SSE2-NEXT:    retq
+; SSE2-NEXT:  .LBB17_1: # %cond.store
+; SSE2-NEXT:    movb %cl, (%rdi)
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    je .LBB17_4
+; SSE2-NEXT:  .LBB17_3: # %cond.store1
+; SSE2-NEXT:    movb %ch, 1(%rdi)
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    je .LBB17_6
+; SSE2-NEXT:  .LBB17_5: # %cond.store3
+; SSE2-NEXT:    movl %ecx, %edx
+; SSE2-NEXT:    shrl $16, %edx
+; SSE2-NEXT:    movb %dl, 2(%rdi)
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne .LBB17_7
+; SSE2-NEXT:    jmp .LBB17_8
+; SSE2-NEXT:  .LBB17_13: # %cond.store11
+; SSE2-NEXT:    movb %cl, 6(%rdi)
+; SSE2-NEXT:    testb $-128, %al
+; SSE2-NEXT:    je .LBB17_16
+; SSE2-NEXT:  .LBB17_15: # %cond.store13
+; SSE2-NEXT:    movb %ch, 7(%rdi)
+; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: truncstore_v8i16_v8i8:
 ; SSE4:       # %bb.0:
-; SSE4-NEXT:    pxor %xmm3, %xmm3
-; SSE4-NEXT:    pcmpeqw %xmm1, %xmm3
-; SSE4-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE4-NEXT:    pxor %xmm3, %xmm2
-; SSE4-NEXT:    pminsw {{.*}}(%rip), %xmm0
-; SSE4-NEXT:    pmaxsw {{.*}}(%rip), %xmm0
-; SSE4-NEXT:    pextrb $0, %xmm2, %eax
+; SSE4-NEXT:    pxor %xmm2, %xmm2
+; SSE4-NEXT:    packsswb %xmm0, %xmm0
+; SSE4-NEXT:    pcmpeqw %xmm1, %xmm2
+; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
+; SSE4-NEXT:    pxor %xmm2, %xmm1
+; SSE4-NEXT:    packsswb %xmm0, %xmm1
+; SSE4-NEXT:    pmovmskb %xmm1, %eax
 ; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB17_2
-; SSE4-NEXT:  # %bb.1: # %cond.store
+; SSE4-NEXT:    jne .LBB17_1
+; SSE4-NEXT:  # %bb.2: # %else
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    jne .LBB17_3
+; SSE4-NEXT:  .LBB17_4: # %else2
+; SSE4-NEXT:    testb $4, %al
+; SSE4-NEXT:    jne .LBB17_5
+; SSE4-NEXT:  .LBB17_6: # %else4
+; SSE4-NEXT:    testb $8, %al
+; SSE4-NEXT:    jne .LBB17_7
+; SSE4-NEXT:  .LBB17_8: # %else6
+; SSE4-NEXT:    testb $16, %al
+; SSE4-NEXT:    jne .LBB17_9
+; SSE4-NEXT:  .LBB17_10: # %else8
+; SSE4-NEXT:    testb $32, %al
+; SSE4-NEXT:    jne .LBB17_11
+; SSE4-NEXT:  .LBB17_12: # %else10
+; SSE4-NEXT:    testb $64, %al
+; SSE4-NEXT:    jne .LBB17_13
+; SSE4-NEXT:  .LBB17_14: # %else12
+; SSE4-NEXT:    testb $-128, %al
+; SSE4-NEXT:    jne .LBB17_15
+; SSE4-NEXT:  .LBB17_16: # %else14
+; SSE4-NEXT:    retq
+; SSE4-NEXT:  .LBB17_1: # %cond.store
 ; SSE4-NEXT:    pextrb $0, %xmm0, (%rdi)
-; SSE4-NEXT:  .LBB17_2: # %else
-; SSE4-NEXT:    pextrb $2, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $2, %al
 ; SSE4-NEXT:    je .LBB17_4
-; SSE4-NEXT:  # %bb.3: # %cond.store1
-; SSE4-NEXT:    pextrb $2, %xmm0, 1(%rdi)
-; SSE4-NEXT:  .LBB17_4: # %else2
-; SSE4-NEXT:    pxor %xmm3, %xmm3
-; SSE4-NEXT:    pcmpeqw %xmm1, %xmm3
-; SSE4-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE4-NEXT:    pxor %xmm3, %xmm2
-; SSE4-NEXT:    pextrb $4, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:  .LBB17_3: # %cond.store1
+; SSE4-NEXT:    pextrb $1, %xmm0, 1(%rdi)
+; SSE4-NEXT:    testb $4, %al
 ; SSE4-NEXT:    je .LBB17_6
-; SSE4-NEXT:  # %bb.5: # %cond.store3
-; SSE4-NEXT:    pextrb $4, %xmm0, 2(%rdi)
-; SSE4-NEXT:  .LBB17_6: # %else4
-; SSE4-NEXT:    pextrb $6, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:  .LBB17_5: # %cond.store3
+; SSE4-NEXT:    pextrb $2, %xmm0, 2(%rdi)
+; SSE4-NEXT:    testb $8, %al
 ; SSE4-NEXT:    je .LBB17_8
-; SSE4-NEXT:  # %bb.7: # %cond.store5
-; SSE4-NEXT:    pextrb $6, %xmm0, 3(%rdi)
-; SSE4-NEXT:  .LBB17_8: # %else6
-; SSE4-NEXT:    pxor %xmm3, %xmm3
-; SSE4-NEXT:    pcmpeqw %xmm1, %xmm3
-; SSE4-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE4-NEXT:    pxor %xmm3, %xmm2
-; SSE4-NEXT:    pextrb $8, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:  .LBB17_7: # %cond.store5
+; SSE4-NEXT:    pextrb $3, %xmm0, 3(%rdi)
+; SSE4-NEXT:    testb $16, %al
 ; SSE4-NEXT:    je .LBB17_10
-; SSE4-NEXT:  # %bb.9: # %cond.store7
-; SSE4-NEXT:    pextrb $8, %xmm0, 4(%rdi)
-; SSE4-NEXT:  .LBB17_10: # %else8
-; SSE4-NEXT:    pextrb $10, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:  .LBB17_9: # %cond.store7
+; SSE4-NEXT:    pextrb $4, %xmm0, 4(%rdi)
+; SSE4-NEXT:    testb $32, %al
 ; SSE4-NEXT:    je .LBB17_12
-; SSE4-NEXT:  # %bb.11: # %cond.store9
-; SSE4-NEXT:    pextrb $10, %xmm0, 5(%rdi)
-; SSE4-NEXT:  .LBB17_12: # %else10
-; SSE4-NEXT:    pxor %xmm2, %xmm2
-; SSE4-NEXT:    pcmpeqw %xmm2, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE4-NEXT:    pxor %xmm1, %xmm2
-; SSE4-NEXT:    pextrb $12, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:  .LBB17_11: # %cond.store9
+; SSE4-NEXT:    pextrb $5, %xmm0, 5(%rdi)
+; SSE4-NEXT:    testb $64, %al
 ; SSE4-NEXT:    je .LBB17_14
-; SSE4-NEXT:  # %bb.13: # %cond.store11
-; SSE4-NEXT:    pextrb $12, %xmm0, 6(%rdi)
-; SSE4-NEXT:  .LBB17_14: # %else12
-; SSE4-NEXT:    pextrb $14, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:  .LBB17_13: # %cond.store11
+; SSE4-NEXT:    pextrb $6, %xmm0, 6(%rdi)
+; SSE4-NEXT:    testb $-128, %al
 ; SSE4-NEXT:    je .LBB17_16
-; SSE4-NEXT:  # %bb.15: # %cond.store13
-; SSE4-NEXT:    pextrb $14, %xmm0, 7(%rdi)
-; SSE4-NEXT:  .LBB17_16: # %else14
+; SSE4-NEXT:  .LBB17_15: # %cond.store13
+; SSE4-NEXT:    pextrb $7, %xmm0, 7(%rdi)
 ; SSE4-NEXT:    retq
 ;
 ; AVX-LABEL: truncstore_v8i16_v8i8:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm2
-; AVX-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX-NEXT:    vpminsw {{.*}}(%rip), %xmm0, %xmm0
-; AVX-NEXT:    vpmaxsw {{.*}}(%rip), %xmm0, %xmm0
-; AVX-NEXT:    vpextrb $0, %xmm2, %eax
+; AVX-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm1
+; AVX-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX-NEXT:    vpxor %xmm2, %xmm1, %xmm1
+; AVX-NEXT:    vpacksswb %xmm0, %xmm1, %xmm1
+; AVX-NEXT:    vpmovmskb %xmm1, %eax
 ; AVX-NEXT:    testb $1, %al
-; AVX-NEXT:    je .LBB17_2
-; AVX-NEXT:  # %bb.1: # %cond.store
+; AVX-NEXT:    jne .LBB17_1
+; AVX-NEXT:  # %bb.2: # %else
+; AVX-NEXT:    testb $2, %al
+; AVX-NEXT:    jne .LBB17_3
+; AVX-NEXT:  .LBB17_4: # %else2
+; AVX-NEXT:    testb $4, %al
+; AVX-NEXT:    jne .LBB17_5
+; AVX-NEXT:  .LBB17_6: # %else4
+; AVX-NEXT:    testb $8, %al
+; AVX-NEXT:    jne .LBB17_7
+; AVX-NEXT:  .LBB17_8: # %else6
+; AVX-NEXT:    testb $16, %al
+; AVX-NEXT:    jne .LBB17_9
+; AVX-NEXT:  .LBB17_10: # %else8
+; AVX-NEXT:    testb $32, %al
+; AVX-NEXT:    jne .LBB17_11
+; AVX-NEXT:  .LBB17_12: # %else10
+; AVX-NEXT:    testb $64, %al
+; AVX-NEXT:    jne .LBB17_13
+; AVX-NEXT:  .LBB17_14: # %else12
+; AVX-NEXT:    testb $-128, %al
+; AVX-NEXT:    jne .LBB17_15
+; AVX-NEXT:  .LBB17_16: # %else14
+; AVX-NEXT:    retq
+; AVX-NEXT:  .LBB17_1: # %cond.store
 ; AVX-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX-NEXT:  .LBB17_2: # %else
-; AVX-NEXT:    vpextrb $2, %xmm2, %eax
-; AVX-NEXT:    testb $1, %al
+; AVX-NEXT:    testb $2, %al
 ; AVX-NEXT:    je .LBB17_4
-; AVX-NEXT:  # %bb.3: # %cond.store1
-; AVX-NEXT:    vpextrb $2, %xmm0, 1(%rdi)
-; AVX-NEXT:  .LBB17_4: # %else2
-; AVX-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm2
-; AVX-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX-NEXT:    testb $1, %al
+; AVX-NEXT:  .LBB17_3: # %cond.store1
+; AVX-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
+; AVX-NEXT:    testb $4, %al
 ; AVX-NEXT:    je .LBB17_6
-; AVX-NEXT:  # %bb.5: # %cond.store3
-; AVX-NEXT:    vpextrb $4, %xmm0, 2(%rdi)
-; AVX-NEXT:  .LBB17_6: # %else4
-; AVX-NEXT:    vpextrb $6, %xmm2, %eax
-; AVX-NEXT:    testb $1, %al
+; AVX-NEXT:  .LBB17_5: # %cond.store3
+; AVX-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
+; AVX-NEXT:    testb $8, %al
 ; AVX-NEXT:    je .LBB17_8
-; AVX-NEXT:  # %bb.7: # %cond.store5
-; AVX-NEXT:    vpextrb $6, %xmm0, 3(%rdi)
-; AVX-NEXT:  .LBB17_8: # %else6
-; AVX-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm2
-; AVX-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX-NEXT:    vpextrb $8, %xmm2, %eax
-; AVX-NEXT:    testb $1, %al
+; AVX-NEXT:  .LBB17_7: # %cond.store5
+; AVX-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
+; AVX-NEXT:    testb $16, %al
 ; AVX-NEXT:    je .LBB17_10
-; AVX-NEXT:  # %bb.9: # %cond.store7
-; AVX-NEXT:    vpextrb $8, %xmm0, 4(%rdi)
-; AVX-NEXT:  .LBB17_10: # %else8
-; AVX-NEXT:    vpextrb $10, %xmm2, %eax
-; AVX-NEXT:    testb $1, %al
+; AVX-NEXT:  .LBB17_9: # %cond.store7
+; AVX-NEXT:    vpextrb $4, %xmm0, 4(%rdi)
+; AVX-NEXT:    testb $32, %al
 ; AVX-NEXT:    je .LBB17_12
-; AVX-NEXT:  # %bb.11: # %cond.store9
-; AVX-NEXT:    vpextrb $10, %xmm0, 5(%rdi)
-; AVX-NEXT:  .LBB17_12: # %else10
-; AVX-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm1
-; AVX-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX-NEXT:    testb $1, %al
+; AVX-NEXT:  .LBB17_11: # %cond.store9
+; AVX-NEXT:    vpextrb $5, %xmm0, 5(%rdi)
+; AVX-NEXT:    testb $64, %al
 ; AVX-NEXT:    je .LBB17_14
-; AVX-NEXT:  # %bb.13: # %cond.store11
-; AVX-NEXT:    vpextrb $12, %xmm0, 6(%rdi)
-; AVX-NEXT:  .LBB17_14: # %else12
-; AVX-NEXT:    vpextrb $14, %xmm1, %eax
-; AVX-NEXT:    testb $1, %al
+; AVX-NEXT:  .LBB17_13: # %cond.store11
+; AVX-NEXT:    vpextrb $6, %xmm0, 6(%rdi)
+; AVX-NEXT:    testb $-128, %al
 ; AVX-NEXT:    je .LBB17_16
-; AVX-NEXT:  # %bb.15: # %cond.store13
-; AVX-NEXT:    vpextrb $14, %xmm0, 7(%rdi)
-; AVX-NEXT:  .LBB17_16: # %else14
+; AVX-NEXT:  .LBB17_15: # %cond.store13
+; AVX-NEXT:    vpextrb $7, %xmm0, 7(%rdi)
 ; AVX-NEXT:    retq
 ;
 ; AVX512F-LABEL: truncstore_v8i16_v8i8:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
-; AVX512F-NEXT:    vpmovsxwq %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmq %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    vpminsw {{.*}}(%rip), %xmm0, %xmm0
-; AVX512F-NEXT:    vpmaxsw {{.*}}(%rip), %xmm0, %xmm0
+; AVX512F-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm1
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
+; AVX512F-NEXT:    vpmovsxwq %xmm1, %zmm1
+; AVX512F-NEXT:    vptestmq %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
 ; AVX512F-NEXT:    kmovw %k0, %eax
 ; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB17_2
-; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    jne .LBB17_1
+; AVX512F-NEXT:  # %bb.2: # %else
+; AVX512F-NEXT:    testb $2, %al
+; AVX512F-NEXT:    jne .LBB17_3
+; AVX512F-NEXT:  .LBB17_4: # %else2
+; AVX512F-NEXT:    testb $4, %al
+; AVX512F-NEXT:    jne .LBB17_5
+; AVX512F-NEXT:  .LBB17_6: # %else4
+; AVX512F-NEXT:    testb $8, %al
+; AVX512F-NEXT:    jne .LBB17_7
+; AVX512F-NEXT:  .LBB17_8: # %else6
+; AVX512F-NEXT:    testb $16, %al
+; AVX512F-NEXT:    jne .LBB17_9
+; AVX512F-NEXT:  .LBB17_10: # %else8
+; AVX512F-NEXT:    testb $32, %al
+; AVX512F-NEXT:    jne .LBB17_11
+; AVX512F-NEXT:  .LBB17_12: # %else10
+; AVX512F-NEXT:    testb $64, %al
+; AVX512F-NEXT:    jne .LBB17_13
+; AVX512F-NEXT:  .LBB17_14: # %else12
+; AVX512F-NEXT:    testb $-128, %al
+; AVX512F-NEXT:    jne .LBB17_15
+; AVX512F-NEXT:  .LBB17_16: # %else14
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+; AVX512F-NEXT:  .LBB17_1: # %cond.store
 ; AVX512F-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX512F-NEXT:  .LBB17_2: # %else
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
-; AVX512F-NEXT:    vpmovsxwq %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmq %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $2, %al
 ; AVX512F-NEXT:    je .LBB17_4
-; AVX512F-NEXT:  # %bb.3: # %cond.store1
-; AVX512F-NEXT:    vpextrb $2, %xmm0, 1(%rdi)
-; AVX512F-NEXT:  .LBB17_4: # %else2
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
-; AVX512F-NEXT:    vpmovsxwq %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmq %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  .LBB17_3: # %cond.store1
+; AVX512F-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
+; AVX512F-NEXT:    testb $4, %al
 ; AVX512F-NEXT:    je .LBB17_6
-; AVX512F-NEXT:  # %bb.5: # %cond.store3
-; AVX512F-NEXT:    vpextrb $4, %xmm0, 2(%rdi)
-; AVX512F-NEXT:  .LBB17_6: # %else4
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
-; AVX512F-NEXT:    vpmovsxwq %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmq %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  .LBB17_5: # %cond.store3
+; AVX512F-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
+; AVX512F-NEXT:    testb $8, %al
 ; AVX512F-NEXT:    je .LBB17_8
-; AVX512F-NEXT:  # %bb.7: # %cond.store5
-; AVX512F-NEXT:    vpextrb $6, %xmm0, 3(%rdi)
-; AVX512F-NEXT:  .LBB17_8: # %else6
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
-; AVX512F-NEXT:    vpmovsxwq %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmq %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $4, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  .LBB17_7: # %cond.store5
+; AVX512F-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
+; AVX512F-NEXT:    testb $16, %al
 ; AVX512F-NEXT:    je .LBB17_10
-; AVX512F-NEXT:  # %bb.9: # %cond.store7
-; AVX512F-NEXT:    vpextrb $8, %xmm0, 4(%rdi)
-; AVX512F-NEXT:  .LBB17_10: # %else8
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
-; AVX512F-NEXT:    vpmovsxwq %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmq %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $5, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  .LBB17_9: # %cond.store7
+; AVX512F-NEXT:    vpextrb $4, %xmm0, 4(%rdi)
+; AVX512F-NEXT:    testb $32, %al
 ; AVX512F-NEXT:    je .LBB17_12
-; AVX512F-NEXT:  # %bb.11: # %cond.store9
-; AVX512F-NEXT:    vpextrb $10, %xmm0, 5(%rdi)
-; AVX512F-NEXT:  .LBB17_12: # %else10
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm1
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm2
-; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm2
-; AVX512F-NEXT:    vpmovsxwq %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmq %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $6, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  .LBB17_11: # %cond.store9
+; AVX512F-NEXT:    vpextrb $5, %xmm0, 5(%rdi)
+; AVX512F-NEXT:    testb $64, %al
 ; AVX512F-NEXT:    je .LBB17_14
-; AVX512F-NEXT:  # %bb.13: # %cond.store11
-; AVX512F-NEXT:    vpextrb $12, %xmm0, 6(%rdi)
-; AVX512F-NEXT:  .LBB17_14: # %else12
-; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
-; AVX512F-NEXT:    vpmovsxwq %xmm1, %zmm1
-; AVX512F-NEXT:    vptestmq %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $7, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  .LBB17_13: # %cond.store11
+; AVX512F-NEXT:    vpextrb $6, %xmm0, 6(%rdi)
+; AVX512F-NEXT:    testb $-128, %al
 ; AVX512F-NEXT:    je .LBB17_16
-; AVX512F-NEXT:  # %bb.15: # %cond.store13
-; AVX512F-NEXT:    vpextrb $14, %xmm0, 7(%rdi)
-; AVX512F-NEXT:  .LBB17_16: # %else14
+; AVX512F-NEXT:  .LBB17_15: # %cond.store13
+; AVX512F-NEXT:    vpextrb $7, %xmm0, 7(%rdi)
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
@@ -8526,11 +7536,9 @@ define void @truncstore_v8i16_v8i8(<8 x i16> %x, <8 x i8>* %p, <8 x i16> %mask)
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
 ; AVX512BW-NEXT:    vptestmw %zmm1, %zmm1, %k0
-; AVX512BW-NEXT:    vpminsw {{.*}}(%rip), %xmm0, %xmm0
-; AVX512BW-NEXT:    vpmaxsw {{.*}}(%rip), %xmm0, %xmm0
-; AVX512BW-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
 ; AVX512BW-NEXT:    kshiftlq $56, %k0, %k0
 ; AVX512BW-NEXT:    kshiftrq $56, %k0, %k1
+; AVX512BW-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vmovdqu8 %zmm0, (%rdi) {%k1}
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/masked_store_trunc_usat.ll b/llvm/test/CodeGen/X86/masked_store_trunc_usat.ll
index 5c0116846..ab22ba3b8 100644
--- a/llvm/test/CodeGen/X86/masked_store_trunc_usat.ll
+++ b/llvm/test/CodeGen/X86/masked_store_trunc_usat.ll
@@ -11,233 +11,220 @@
 define void @truncstore_v8i64_v8i32(<8 x i64> %x, <8 x i32>* %p, <8 x i32> %mask) {
 ; SSE2-LABEL: truncstore_v8i64_v8i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pxor %xmm6, %xmm6
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm6
-; SSE2-NEXT:    pcmpeqd %xmm12, %xmm12
-; SSE2-NEXT:    pxor %xmm6, %xmm12
-; SSE2-NEXT:    movdqa %xmm12, %xmm9
-; SSE2-NEXT:    packssdw %xmm0, %xmm9
-; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [4294967295,4294967295]
-; SSE2-NEXT:    movdqa {{.*#+}} xmm13 = [9223372039002259456,9223372039002259456]
-; SSE2-NEXT:    movdqa %xmm1, %xmm7
-; SSE2-NEXT:    pxor %xmm13, %xmm7
-; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [9223372039002259455,9223372039002259455]
-; SSE2-NEXT:    movdqa %xmm10, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm7, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm10, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT:    pand %xmm11, %xmm7
+; SSE2-NEXT:    pxor %xmm8, %xmm8
+; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [4294967295,4294967295]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm12 = [9223372039002259456,9223372039002259456]
+; SSE2-NEXT:    movdqa %xmm1, %xmm6
+; SSE2-NEXT:    pxor %xmm12, %xmm6
+; SSE2-NEXT:    movdqa {{.*#+}} xmm11 = [9223372039002259455,9223372039002259455]
+; SSE2-NEXT:    movdqa %xmm11, %xmm7
+; SSE2-NEXT:    pcmpgtd %xmm6, %xmm7
+; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm7[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm11, %xmm6
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT:    por %xmm7, %xmm6
-; SSE2-NEXT:    pand %xmm6, %xmm1
-; SSE2-NEXT:    pandn %xmm8, %xmm6
-; SSE2-NEXT:    por %xmm1, %xmm6
+; SSE2-NEXT:    pand %xmm10, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
+; SSE2-NEXT:    por %xmm6, %xmm7
+; SSE2-NEXT:    pand %xmm7, %xmm1
+; SSE2-NEXT:    pandn %xmm9, %xmm7
+; SSE2-NEXT:    por %xmm1, %xmm7
 ; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    pxor %xmm13, %xmm1
-; SSE2-NEXT:    movdqa %xmm10, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm7[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm10, %xmm1
+; SSE2-NEXT:    pxor %xmm12, %xmm1
+; SSE2-NEXT:    movdqa %xmm11, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm6[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm11, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pand %xmm11, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm14 = xmm7[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm14
-; SSE2-NEXT:    pand %xmm14, %xmm0
-; SSE2-NEXT:    pandn %xmm8, %xmm14
-; SSE2-NEXT:    por %xmm0, %xmm14
-; SSE2-NEXT:    shufps {{.*#+}} xmm14 = xmm14[0,2],xmm6[0,2]
+; SSE2-NEXT:    pand %xmm10, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm6[1,1,3,3]
+; SSE2-NEXT:    por %xmm1, %xmm13
+; SSE2-NEXT:    pand %xmm13, %xmm0
+; SSE2-NEXT:    pandn %xmm9, %xmm13
+; SSE2-NEXT:    por %xmm0, %xmm13
+; SSE2-NEXT:    shufps {{.*#+}} xmm13 = xmm13[0,2],xmm7[0,2]
 ; SSE2-NEXT:    movdqa %xmm3, %xmm0
-; SSE2-NEXT:    pxor %xmm13, %xmm0
-; SSE2-NEXT:    movdqa %xmm10, %xmm1
+; SSE2-NEXT:    pxor %xmm12, %xmm0
+; SSE2-NEXT:    movdqa %xmm11, %xmm1
 ; SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm10, %xmm0
+; SSE2-NEXT:    pcmpeqd %xmm11, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm6, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pxor %xmm2, %xmm13
-; SSE2-NEXT:    movdqa %xmm10, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm13, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pxor %xmm2, %xmm12
+; SSE2-NEXT:    movdqa %xmm11, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm12, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm10, %xmm13
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm13[1,1,3,3]
+; SSE2-NEXT:    pcmpeqd %xmm11, %xmm12
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm12[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm7, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm1[1,1,3,3]
-; SSE2-NEXT:    movd %xmm9, %eax
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm5
+; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
+; SSE2-NEXT:    pxor %xmm1, %xmm5
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm4
+; SSE2-NEXT:    pxor %xmm1, %xmm4
+; SSE2-NEXT:    packssdw %xmm5, %xmm4
+; SSE2-NEXT:    packsswb %xmm0, %xmm4
+; SSE2-NEXT:    pmovmskb %xmm4, %eax
 ; SSE2-NEXT:    testb $1, %al
 ; SSE2-NEXT:    je .LBB0_2
 ; SSE2-NEXT:  # %bb.1: # %cond.store
-; SSE2-NEXT:    movss %xmm14, (%rdi)
+; SSE2-NEXT:    movss %xmm13, (%rdi)
 ; SSE2-NEXT:  .LBB0_2: # %else
-; SSE2-NEXT:    por %xmm11, %xmm0
-; SSE2-NEXT:    por %xmm10, %xmm6
-; SSE2-NEXT:    psrlq $16, %xmm12
-; SSE2-NEXT:    movd %xmm12, %eax
-; SSE2-NEXT:    shrl $16, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    por %xmm10, %xmm0
+; SSE2-NEXT:    por %xmm7, %xmm6
+; SSE2-NEXT:    testb $2, %al
 ; SSE2-NEXT:    je .LBB0_4
 ; SSE2-NEXT:  # %bb.3: # %cond.store1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm14[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm13[1,1,2,3]
 ; SSE2-NEXT:    movd %xmm1, 4(%rdi)
 ; SSE2-NEXT:  .LBB0_4: # %else2
 ; SSE2-NEXT:    pand %xmm0, %xmm3
-; SSE2-NEXT:    pandn %xmm8, %xmm0
+; SSE2-NEXT:    pandn %xmm9, %xmm0
 ; SSE2-NEXT:    pand %xmm6, %xmm2
-; SSE2-NEXT:    pandn %xmm8, %xmm6
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm4
-; SSE2-NEXT:    pcmpeqd %xmm7, %xmm7
-; SSE2-NEXT:    pxor %xmm4, %xmm7
-; SSE2-NEXT:    pextrw $4, %xmm7, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    pandn %xmm9, %xmm6
+; SSE2-NEXT:    testb $4, %al
 ; SSE2-NEXT:    je .LBB0_6
 ; SSE2-NEXT:  # %bb.5: # %cond.store3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm14[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm13[2,3,0,1]
 ; SSE2-NEXT:    movd %xmm1, 8(%rdi)
 ; SSE2-NEXT:  .LBB0_6: # %else4
 ; SSE2-NEXT:    por %xmm0, %xmm3
 ; SSE2-NEXT:    por %xmm6, %xmm2
-; SSE2-NEXT:    pextrw $6, %xmm7, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je .LBB0_8
 ; SSE2-NEXT:  # %bb.7: # %cond.store5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm14[3,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm13[3,1,2,3]
 ; SSE2-NEXT:    movd %xmm0, 12(%rdi)
 ; SSE2-NEXT:  .LBB0_8: # %else6
 ; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,2],xmm3[0,2]
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE2-NEXT:    pxor %xmm1, %xmm0
-; SSE2-NEXT:    pextrw $0, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB0_10
-; SSE2-NEXT:  # %bb.9: # %cond.store7
+; SSE2-NEXT:    testb $16, %al
+; SSE2-NEXT:    jne .LBB0_9
+; SSE2-NEXT:  # %bb.10: # %else8
+; SSE2-NEXT:    testb $32, %al
+; SSE2-NEXT:    jne .LBB0_11
+; SSE2-NEXT:  .LBB0_12: # %else10
+; SSE2-NEXT:    testb $64, %al
+; SSE2-NEXT:    jne .LBB0_13
+; SSE2-NEXT:  .LBB0_14: # %else12
+; SSE2-NEXT:    testb $-128, %al
+; SSE2-NEXT:    jne .LBB0_15
+; SSE2-NEXT:  .LBB0_16: # %else14
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  .LBB0_9: # %cond.store7
 ; SSE2-NEXT:    movss %xmm2, 16(%rdi)
-; SSE2-NEXT:  .LBB0_10: # %else8
-; SSE2-NEXT:    pextrw $2, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $32, %al
 ; SSE2-NEXT:    je .LBB0_12
-; SSE2-NEXT:  # %bb.11: # %cond.store9
+; SSE2-NEXT:  .LBB0_11: # %cond.store9
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
 ; SSE2-NEXT:    movd %xmm0, 20(%rdi)
-; SSE2-NEXT:  .LBB0_12: # %else10
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm5
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE2-NEXT:    pxor %xmm5, %xmm0
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $64, %al
 ; SSE2-NEXT:    je .LBB0_14
-; SSE2-NEXT:  # %bb.13: # %cond.store11
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,3,0,1]
-; SSE2-NEXT:    movd %xmm1, 24(%rdi)
-; SSE2-NEXT:  .LBB0_14: # %else12
-; SSE2-NEXT:    pextrw $6, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:  .LBB0_13: # %cond.store11
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE2-NEXT:    movd %xmm0, 24(%rdi)
+; SSE2-NEXT:    testb $-128, %al
 ; SSE2-NEXT:    je .LBB0_16
-; SSE2-NEXT:  # %bb.15: # %cond.store13
+; SSE2-NEXT:  .LBB0_15: # %cond.store13
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[3,1,2,3]
 ; SSE2-NEXT:    movd %xmm0, 28(%rdi)
-; SSE2-NEXT:  .LBB0_16: # %else14
 ; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: truncstore_v8i64_v8i32:
 ; SSE4:       # %bb.0:
 ; SSE4-NEXT:    movdqa %xmm0, %xmm8
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqd %xmm4, %xmm0
-; SSE4-NEXT:    pcmpeqd %xmm10, %xmm10
-; SSE4-NEXT:    pxor %xmm0, %xmm10
-; SSE4-NEXT:    movapd {{.*#+}} xmm9 = [4294967295,4294967295]
+; SSE4-NEXT:    pxor %xmm9, %xmm9
+; SSE4-NEXT:    movapd {{.*#+}} xmm10 = [4294967295,4294967295]
 ; SSE4-NEXT:    movdqa {{.*#+}} xmm11 = [9223372036854775808,9223372036854775808]
 ; SSE4-NEXT:    movdqa %xmm1, %xmm6
 ; SSE4-NEXT:    pxor %xmm11, %xmm6
 ; SSE4-NEXT:    movdqa {{.*#+}} xmm7 = [9223372041149743103,9223372041149743103]
 ; SSE4-NEXT:    movdqa %xmm7, %xmm0
 ; SSE4-NEXT:    pcmpgtq %xmm6, %xmm0
-; SSE4-NEXT:    movapd %xmm9, %xmm6
+; SSE4-NEXT:    movapd %xmm10, %xmm6
 ; SSE4-NEXT:    blendvpd %xmm0, %xmm1, %xmm6
 ; SSE4-NEXT:    movdqa %xmm8, %xmm1
 ; SSE4-NEXT:    pxor %xmm11, %xmm1
 ; SSE4-NEXT:    movdqa %xmm7, %xmm0
 ; SSE4-NEXT:    pcmpgtq %xmm1, %xmm0
-; SSE4-NEXT:    movapd %xmm9, %xmm1
+; SSE4-NEXT:    movapd %xmm10, %xmm1
 ; SSE4-NEXT:    blendvpd %xmm0, %xmm8, %xmm1
 ; SSE4-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,2],xmm6[0,2]
 ; SSE4-NEXT:    movdqa %xmm3, %xmm6
 ; SSE4-NEXT:    pxor %xmm11, %xmm6
 ; SSE4-NEXT:    movdqa %xmm7, %xmm0
 ; SSE4-NEXT:    pcmpgtq %xmm6, %xmm0
-; SSE4-NEXT:    movapd %xmm9, %xmm8
+; SSE4-NEXT:    movapd %xmm10, %xmm8
 ; SSE4-NEXT:    blendvpd %xmm0, %xmm3, %xmm8
 ; SSE4-NEXT:    pxor %xmm2, %xmm11
 ; SSE4-NEXT:    pcmpgtq %xmm11, %xmm7
 ; SSE4-NEXT:    movdqa %xmm7, %xmm0
-; SSE4-NEXT:    blendvpd %xmm0, %xmm2, %xmm9
-; SSE4-NEXT:    pextrb $0, %xmm10, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB0_2
-; SSE4-NEXT:  # %bb.1: # %cond.store
-; SSE4-NEXT:    movss %xmm1, (%rdi)
-; SSE4-NEXT:  .LBB0_2: # %else
-; SSE4-NEXT:    pextrb $4, %xmm10, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB0_4
-; SSE4-NEXT:  # %bb.3: # %cond.store1
-; SSE4-NEXT:    extractps $1, %xmm1, 4(%rdi)
-; SSE4-NEXT:  .LBB0_4: # %else2
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE4-NEXT:    blendvpd %xmm0, %xmm2, %xmm10
+; SSE4-NEXT:    pcmpeqd %xmm9, %xmm5
 ; SSE4-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE4-NEXT:    pxor %xmm4, %xmm0
-; SSE4-NEXT:    pextrb $8, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB0_6
-; SSE4-NEXT:  # %bb.5: # %cond.store3
-; SSE4-NEXT:    extractps $2, %xmm1, 8(%rdi)
+; SSE4-NEXT:    pxor %xmm0, %xmm5
+; SSE4-NEXT:    pcmpeqd %xmm9, %xmm4
+; SSE4-NEXT:    pxor %xmm0, %xmm4
+; SSE4-NEXT:    packssdw %xmm5, %xmm4
+; SSE4-NEXT:    packsswb %xmm0, %xmm4
+; SSE4-NEXT:    pmovmskb %xmm4, %eax
+; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    jne .LBB0_1
+; SSE4-NEXT:  # %bb.2: # %else
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    jne .LBB0_3
+; SSE4-NEXT:  .LBB0_4: # %else2
+; SSE4-NEXT:    testb $4, %al
+; SSE4-NEXT:    jne .LBB0_5
 ; SSE4-NEXT:  .LBB0_6: # %else4
-; SSE4-NEXT:    pextrb $12, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $8, %al
 ; SSE4-NEXT:    je .LBB0_8
-; SSE4-NEXT:  # %bb.7: # %cond.store5
+; SSE4-NEXT:  .LBB0_7: # %cond.store5
 ; SSE4-NEXT:    extractps $3, %xmm1, 12(%rdi)
 ; SSE4-NEXT:  .LBB0_8: # %else6
-; SSE4-NEXT:    shufps {{.*#+}} xmm9 = xmm9[0,2],xmm8[0,2]
-; SSE4-NEXT:    xorps %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm5, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE4-NEXT:    pxor %xmm1, %xmm0
-; SSE4-NEXT:    pextrb $0, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB0_10
-; SSE4-NEXT:  # %bb.9: # %cond.store7
-; SSE4-NEXT:    movss %xmm9, 16(%rdi)
-; SSE4-NEXT:  .LBB0_10: # %else8
-; SSE4-NEXT:    pextrb $4, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB0_12
-; SSE4-NEXT:  # %bb.11: # %cond.store9
-; SSE4-NEXT:    extractps $1, %xmm9, 20(%rdi)
+; SSE4-NEXT:    shufps {{.*#+}} xmm10 = xmm10[0,2],xmm8[0,2]
+; SSE4-NEXT:    testb $16, %al
+; SSE4-NEXT:    jne .LBB0_9
+; SSE4-NEXT:  # %bb.10: # %else8
+; SSE4-NEXT:    testb $32, %al
+; SSE4-NEXT:    jne .LBB0_11
 ; SSE4-NEXT:  .LBB0_12: # %else10
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqd %xmm0, %xmm5
-; SSE4-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE4-NEXT:    pxor %xmm5, %xmm0
-; SSE4-NEXT:    pextrb $8, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB0_14
-; SSE4-NEXT:  # %bb.13: # %cond.store11
-; SSE4-NEXT:    extractps $2, %xmm9, 24(%rdi)
+; SSE4-NEXT:    testb $64, %al
+; SSE4-NEXT:    jne .LBB0_13
 ; SSE4-NEXT:  .LBB0_14: # %else12
-; SSE4-NEXT:    pextrb $12, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB0_16
-; SSE4-NEXT:  # %bb.15: # %cond.store13
-; SSE4-NEXT:    extractps $3, %xmm9, 28(%rdi)
+; SSE4-NEXT:    testb $-128, %al
+; SSE4-NEXT:    jne .LBB0_15
 ; SSE4-NEXT:  .LBB0_16: # %else14
 ; SSE4-NEXT:    retq
+; SSE4-NEXT:  .LBB0_1: # %cond.store
+; SSE4-NEXT:    movss %xmm1, (%rdi)
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    je .LBB0_4
+; SSE4-NEXT:  .LBB0_3: # %cond.store1
+; SSE4-NEXT:    extractps $1, %xmm1, 4(%rdi)
+; SSE4-NEXT:    testb $4, %al
+; SSE4-NEXT:    je .LBB0_6
+; SSE4-NEXT:  .LBB0_5: # %cond.store3
+; SSE4-NEXT:    extractps $2, %xmm1, 8(%rdi)
+; SSE4-NEXT:    testb $8, %al
+; SSE4-NEXT:    jne .LBB0_7
+; SSE4-NEXT:    jmp .LBB0_8
+; SSE4-NEXT:  .LBB0_9: # %cond.store7
+; SSE4-NEXT:    movss %xmm10, 16(%rdi)
+; SSE4-NEXT:    testb $32, %al
+; SSE4-NEXT:    je .LBB0_12
+; SSE4-NEXT:  .LBB0_11: # %cond.store9
+; SSE4-NEXT:    extractps $1, %xmm10, 20(%rdi)
+; SSE4-NEXT:    testb $64, %al
+; SSE4-NEXT:    je .LBB0_14
+; SSE4-NEXT:  .LBB0_13: # %cond.store11
+; SSE4-NEXT:    extractps $2, %xmm10, 24(%rdi)
+; SSE4-NEXT:    testb $-128, %al
+; SSE4-NEXT:    je .LBB0_16
+; SSE4-NEXT:  .LBB0_15: # %cond.store13
+; SSE4-NEXT:    extractps $3, %xmm10, 28(%rdi)
+; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: truncstore_v8i64_v8i32:
 ; AVX1:       # %bb.0:
@@ -333,65 +320,60 @@ define void @truncstore_v8i64_v8i32(<8 x i64> %x, <8 x i32>* %p, <8 x i32> %mask
 define void @truncstore_v8i64_v8i16(<8 x i64> %x, <8 x i16>* %p, <8 x i32> %mask) {
 ; SSE2-LABEL: truncstore_v8i64_v8i16:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pxor %xmm6, %xmm6
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm6
-; SSE2-NEXT:    pcmpeqd %xmm10, %xmm10
-; SSE2-NEXT:    pxor %xmm6, %xmm10
-; SSE2-NEXT:    movdqa %xmm10, %xmm9
-; SSE2-NEXT:    packssdw %xmm0, %xmm9
-; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [65535,65535]
-; SSE2-NEXT:    movdqa {{.*#+}} xmm12 = [9223372039002259456,9223372039002259456]
-; SSE2-NEXT:    movdqa %xmm2, %xmm7
-; SSE2-NEXT:    pxor %xmm12, %xmm7
-; SSE2-NEXT:    movdqa {{.*#+}} xmm11 = [9223372039002324991,9223372039002324991]
-; SSE2-NEXT:    movdqa %xmm11, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm7, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm11, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT:    pand %xmm13, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm14 = xmm6[1,1,3,3]
-; SSE2-NEXT:    por %xmm7, %xmm14
-; SSE2-NEXT:    pand %xmm14, %xmm2
-; SSE2-NEXT:    pandn %xmm8, %xmm14
-; SSE2-NEXT:    por %xmm2, %xmm14
+; SSE2-NEXT:    pxor %xmm8, %xmm8
+; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [65535,65535]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm11 = [9223372039002259456,9223372039002259456]
+; SSE2-NEXT:    movdqa %xmm2, %xmm6
+; SSE2-NEXT:    pxor %xmm11, %xmm6
+; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [9223372039002324991,9223372039002324991]
+; SSE2-NEXT:    movdqa %xmm10, %xmm7
+; SSE2-NEXT:    pcmpgtd %xmm6, %xmm7
+; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm7[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSE2-NEXT:    pand %xmm12, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm7[1,1,3,3]
+; SSE2-NEXT:    por %xmm6, %xmm13
+; SSE2-NEXT:    pand %xmm13, %xmm2
+; SSE2-NEXT:    pandn %xmm9, %xmm13
+; SSE2-NEXT:    por %xmm2, %xmm13
 ; SSE2-NEXT:    movdqa %xmm3, %xmm2
-; SSE2-NEXT:    pxor %xmm12, %xmm2
-; SSE2-NEXT:    movdqa %xmm11, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm7[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm11, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[1,1,3,3]
-; SSE2-NEXT:    pand %xmm13, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm7[1,1,3,3]
-; SSE2-NEXT:    por %xmm6, %xmm2
+; SSE2-NEXT:    pxor %xmm11, %xmm2
+; SSE2-NEXT:    movdqa %xmm10, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm6[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm2[1,1,3,3]
+; SSE2-NEXT:    pand %xmm12, %xmm7
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[1,1,3,3]
+; SSE2-NEXT:    por %xmm7, %xmm2
 ; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pandn %xmm8, %xmm2
+; SSE2-NEXT:    pandn %xmm9, %xmm2
 ; SSE2-NEXT:    por %xmm3, %xmm2
 ; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    pxor %xmm12, %xmm3
-; SSE2-NEXT:    movdqa %xmm11, %xmm6
+; SSE2-NEXT:    pxor %xmm11, %xmm3
+; SSE2-NEXT:    movdqa %xmm10, %xmm6
 ; SSE2-NEXT:    pcmpgtd %xmm3, %xmm6
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm11, %xmm3
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm3
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm7, %xmm3
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
 ; SSE2-NEXT:    por %xmm3, %xmm6
 ; SSE2-NEXT:    pand %xmm6, %xmm0
-; SSE2-NEXT:    pandn %xmm8, %xmm6
+; SSE2-NEXT:    pandn %xmm9, %xmm6
 ; SSE2-NEXT:    por %xmm0, %xmm6
-; SSE2-NEXT:    pxor %xmm1, %xmm12
-; SSE2-NEXT:    movdqa %xmm11, %xmm0
-; SSE2-NEXT:    pcmpgtd %xmm12, %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm11
+; SSE2-NEXT:    movdqa %xmm10, %xmm0
+; SSE2-NEXT:    pcmpgtd %xmm11, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm11, %xmm12
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm12[1,1,3,3]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm11
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm11[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm3, %xmm7
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
 ; SSE2-NEXT:    por %xmm7, %xmm0
 ; SSE2-NEXT:    pand %xmm0, %xmm1
-; SSE2-NEXT:    pandn %xmm8, %xmm0
+; SSE2-NEXT:    pandn %xmm9, %xmm0
 ; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
@@ -400,89 +382,87 @@ define void @truncstore_v8i64_v8i16(<8 x i64> %x, <8 x i16>* %p, <8 x i32> %mask
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm0[0,1,0,2,4,5,6,7]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm14[0,2,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm13[0,2,2,3]
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,1,0,2,4,5,6,7]
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
 ; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
-; SSE2-NEXT:    movd %xmm9, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB1_2
-; SSE2-NEXT:  # %bb.1: # %cond.store
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    movw %ax, (%rdi)
-; SSE2-NEXT:  .LBB1_2: # %else
-; SSE2-NEXT:    psrlq $16, %xmm10
-; SSE2-NEXT:    movd %xmm10, %eax
-; SSE2-NEXT:    shrl $16, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB1_4
-; SSE2-NEXT:  # %bb.3: # %cond.store1
-; SSE2-NEXT:    pextrw $1, %xmm0, %eax
-; SSE2-NEXT:    movw %ax, 2(%rdi)
-; SSE2-NEXT:  .LBB1_4: # %else2
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm4
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm5
 ; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pxor %xmm4, %xmm1
-; SSE2-NEXT:    pextrw $4, %xmm1, %eax
+; SSE2-NEXT:    pxor %xmm1, %xmm5
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm4
+; SSE2-NEXT:    pxor %xmm1, %xmm4
+; SSE2-NEXT:    packssdw %xmm5, %xmm4
+; SSE2-NEXT:    packsswb %xmm0, %xmm4
+; SSE2-NEXT:    pmovmskb %xmm4, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB1_6
-; SSE2-NEXT:  # %bb.5: # %cond.store3
-; SSE2-NEXT:    pextrw $2, %xmm0, %eax
-; SSE2-NEXT:    movw %ax, 4(%rdi)
+; SSE2-NEXT:    jne .LBB1_1
+; SSE2-NEXT:  # %bb.2: # %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne .LBB1_3
+; SSE2-NEXT:  .LBB1_4: # %else2
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne .LBB1_5
 ; SSE2-NEXT:  .LBB1_6: # %else4
-; SSE2-NEXT:    pextrw $6, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB1_8
-; SSE2-NEXT:  # %bb.7: # %cond.store5
-; SSE2-NEXT:    pextrw $3, %xmm0, %eax
-; SSE2-NEXT:    movw %ax, 6(%rdi)
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne .LBB1_7
 ; SSE2-NEXT:  .LBB1_8: # %else6
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pxor %xmm2, %xmm1
-; SSE2-NEXT:    pextrw $0, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB1_10
-; SSE2-NEXT:  # %bb.9: # %cond.store7
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    movw %ax, 8(%rdi)
+; SSE2-NEXT:    testb $16, %al
+; SSE2-NEXT:    jne .LBB1_9
 ; SSE2-NEXT:  .LBB1_10: # %else8
-; SSE2-NEXT:    pextrw $2, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB1_12
-; SSE2-NEXT:  # %bb.11: # %cond.store9
-; SSE2-NEXT:    pextrw $5, %xmm0, %eax
-; SSE2-NEXT:    movw %ax, 10(%rdi)
+; SSE2-NEXT:    testb $32, %al
+; SSE2-NEXT:    jne .LBB1_11
 ; SSE2-NEXT:  .LBB1_12: # %else10
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm5
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pxor %xmm5, %xmm1
-; SSE2-NEXT:    pextrw $4, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB1_14
-; SSE2-NEXT:  # %bb.13: # %cond.store11
-; SSE2-NEXT:    pextrw $6, %xmm0, %eax
-; SSE2-NEXT:    movw %ax, 12(%rdi)
+; SSE2-NEXT:    testb $64, %al
+; SSE2-NEXT:    jne .LBB1_13
 ; SSE2-NEXT:  .LBB1_14: # %else12
-; SSE2-NEXT:    pextrw $6, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $-128, %al
+; SSE2-NEXT:    jne .LBB1_15
+; SSE2-NEXT:  .LBB1_16: # %else14
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  .LBB1_1: # %cond.store
+; SSE2-NEXT:    movd %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, (%rdi)
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    je .LBB1_4
+; SSE2-NEXT:  .LBB1_3: # %cond.store1
+; SSE2-NEXT:    pextrw $1, %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, 2(%rdi)
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    je .LBB1_6
+; SSE2-NEXT:  .LBB1_5: # %cond.store3
+; SSE2-NEXT:    pextrw $2, %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, 4(%rdi)
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    je .LBB1_8
+; SSE2-NEXT:  .LBB1_7: # %cond.store5
+; SSE2-NEXT:    pextrw $3, %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, 6(%rdi)
+; SSE2-NEXT:    testb $16, %al
+; SSE2-NEXT:    je .LBB1_10
+; SSE2-NEXT:  .LBB1_9: # %cond.store7
+; SSE2-NEXT:    pextrw $4, %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, 8(%rdi)
+; SSE2-NEXT:    testb $32, %al
+; SSE2-NEXT:    je .LBB1_12
+; SSE2-NEXT:  .LBB1_11: # %cond.store9
+; SSE2-NEXT:    pextrw $5, %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, 10(%rdi)
+; SSE2-NEXT:    testb $64, %al
+; SSE2-NEXT:    je .LBB1_14
+; SSE2-NEXT:  .LBB1_13: # %cond.store11
+; SSE2-NEXT:    pextrw $6, %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, 12(%rdi)
+; SSE2-NEXT:    testb $-128, %al
 ; SSE2-NEXT:    je .LBB1_16
-; SSE2-NEXT:  # %bb.15: # %cond.store13
+; SSE2-NEXT:  .LBB1_15: # %cond.store13
 ; SSE2-NEXT:    pextrw $7, %xmm0, %eax
 ; SSE2-NEXT:    movw %ax, 14(%rdi)
-; SSE2-NEXT:  .LBB1_16: # %else14
 ; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: truncstore_v8i64_v8i16:
 ; SSE4:       # %bb.0:
-; SSE4-NEXT:    movdqa %xmm0, %xmm8
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqd %xmm4, %xmm0
-; SSE4-NEXT:    pcmpeqd %xmm9, %xmm9
-; SSE4-NEXT:    pxor %xmm0, %xmm9
+; SSE4-NEXT:    movdqa %xmm0, %xmm9
+; SSE4-NEXT:    pxor %xmm8, %xmm8
 ; SSE4-NEXT:    movapd {{.*#+}} xmm6 = [65535,65535]
 ; SSE4-NEXT:    movdqa {{.*#+}} xmm10 = [9223372036854775808,9223372036854775808]
 ; SSE4-NEXT:    movdqa %xmm1, %xmm7
@@ -492,12 +472,12 @@ define void @truncstore_v8i64_v8i16(<8 x i64> %x, <8 x i16>* %p, <8 x i32> %mask
 ; SSE4-NEXT:    pcmpgtq %xmm7, %xmm0
 ; SSE4-NEXT:    movapd %xmm6, %xmm7
 ; SSE4-NEXT:    blendvpd %xmm0, %xmm1, %xmm7
-; SSE4-NEXT:    movdqa %xmm8, %xmm1
+; SSE4-NEXT:    movdqa %xmm9, %xmm1
 ; SSE4-NEXT:    pxor %xmm10, %xmm1
 ; SSE4-NEXT:    movdqa %xmm11, %xmm0
 ; SSE4-NEXT:    pcmpgtq %xmm1, %xmm0
 ; SSE4-NEXT:    movapd %xmm6, %xmm1
-; SSE4-NEXT:    blendvpd %xmm0, %xmm8, %xmm1
+; SSE4-NEXT:    blendvpd %xmm0, %xmm9, %xmm1
 ; SSE4-NEXT:    packusdw %xmm7, %xmm1
 ; SSE4-NEXT:    movdqa %xmm3, %xmm7
 ; SSE4-NEXT:    pxor %xmm10, %xmm7
@@ -511,246 +491,234 @@ define void @truncstore_v8i64_v8i16(<8 x i64> %x, <8 x i16>* %p, <8 x i32> %mask
 ; SSE4-NEXT:    blendvpd %xmm0, %xmm2, %xmm6
 ; SSE4-NEXT:    packusdw %xmm7, %xmm6
 ; SSE4-NEXT:    packusdw %xmm6, %xmm1
-; SSE4-NEXT:    pextrb $0, %xmm9, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB1_2
-; SSE4-NEXT:  # %bb.1: # %cond.store
+; SSE4-NEXT:    pcmpeqd %xmm8, %xmm5
+; SSE4-NEXT:    pcmpeqd %xmm0, %xmm0
+; SSE4-NEXT:    pxor %xmm0, %xmm5
+; SSE4-NEXT:    pcmpeqd %xmm8, %xmm4
+; SSE4-NEXT:    pxor %xmm0, %xmm4
+; SSE4-NEXT:    packssdw %xmm5, %xmm4
+; SSE4-NEXT:    packsswb %xmm0, %xmm4
+; SSE4-NEXT:    pmovmskb %xmm4, %eax
+; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    jne .LBB1_1
+; SSE4-NEXT:  # %bb.2: # %else
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    jne .LBB1_3
+; SSE4-NEXT:  .LBB1_4: # %else2
+; SSE4-NEXT:    testb $4, %al
+; SSE4-NEXT:    jne .LBB1_5
+; SSE4-NEXT:  .LBB1_6: # %else4
+; SSE4-NEXT:    testb $8, %al
+; SSE4-NEXT:    jne .LBB1_7
+; SSE4-NEXT:  .LBB1_8: # %else6
+; SSE4-NEXT:    testb $16, %al
+; SSE4-NEXT:    jne .LBB1_9
+; SSE4-NEXT:  .LBB1_10: # %else8
+; SSE4-NEXT:    testb $32, %al
+; SSE4-NEXT:    jne .LBB1_11
+; SSE4-NEXT:  .LBB1_12: # %else10
+; SSE4-NEXT:    testb $64, %al
+; SSE4-NEXT:    jne .LBB1_13
+; SSE4-NEXT:  .LBB1_14: # %else12
+; SSE4-NEXT:    testb $-128, %al
+; SSE4-NEXT:    jne .LBB1_15
+; SSE4-NEXT:  .LBB1_16: # %else14
+; SSE4-NEXT:    retq
+; SSE4-NEXT:  .LBB1_1: # %cond.store
 ; SSE4-NEXT:    pextrw $0, %xmm1, (%rdi)
-; SSE4-NEXT:  .LBB1_2: # %else
-; SSE4-NEXT:    pextrb $4, %xmm9, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $2, %al
 ; SSE4-NEXT:    je .LBB1_4
-; SSE4-NEXT:  # %bb.3: # %cond.store1
+; SSE4-NEXT:  .LBB1_3: # %cond.store1
 ; SSE4-NEXT:    pextrw $1, %xmm1, 2(%rdi)
-; SSE4-NEXT:  .LBB1_4: # %else2
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqd %xmm0, %xmm4
-; SSE4-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE4-NEXT:    pxor %xmm4, %xmm0
-; SSE4-NEXT:    pextrb $8, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $4, %al
 ; SSE4-NEXT:    je .LBB1_6
-; SSE4-NEXT:  # %bb.5: # %cond.store3
+; SSE4-NEXT:  .LBB1_5: # %cond.store3
 ; SSE4-NEXT:    pextrw $2, %xmm1, 4(%rdi)
-; SSE4-NEXT:  .LBB1_6: # %else4
-; SSE4-NEXT:    pextrb $12, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $8, %al
 ; SSE4-NEXT:    je .LBB1_8
-; SSE4-NEXT:  # %bb.7: # %cond.store5
+; SSE4-NEXT:  .LBB1_7: # %cond.store5
 ; SSE4-NEXT:    pextrw $3, %xmm1, 6(%rdi)
-; SSE4-NEXT:  .LBB1_8: # %else6
-; SSE4-NEXT:    pxor %xmm2, %xmm2
-; SSE4-NEXT:    pcmpeqd %xmm5, %xmm2
-; SSE4-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE4-NEXT:    pxor %xmm2, %xmm0
-; SSE4-NEXT:    pextrb $0, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $16, %al
 ; SSE4-NEXT:    je .LBB1_10
-; SSE4-NEXT:  # %bb.9: # %cond.store7
+; SSE4-NEXT:  .LBB1_9: # %cond.store7
 ; SSE4-NEXT:    pextrw $4, %xmm1, 8(%rdi)
-; SSE4-NEXT:  .LBB1_10: # %else8
-; SSE4-NEXT:    pextrb $4, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $32, %al
 ; SSE4-NEXT:    je .LBB1_12
-; SSE4-NEXT:  # %bb.11: # %cond.store9
+; SSE4-NEXT:  .LBB1_11: # %cond.store9
 ; SSE4-NEXT:    pextrw $5, %xmm1, 10(%rdi)
-; SSE4-NEXT:  .LBB1_12: # %else10
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqd %xmm0, %xmm5
-; SSE4-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE4-NEXT:    pxor %xmm5, %xmm0
-; SSE4-NEXT:    pextrb $8, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $64, %al
 ; SSE4-NEXT:    je .LBB1_14
-; SSE4-NEXT:  # %bb.13: # %cond.store11
+; SSE4-NEXT:  .LBB1_13: # %cond.store11
 ; SSE4-NEXT:    pextrw $6, %xmm1, 12(%rdi)
-; SSE4-NEXT:  .LBB1_14: # %else12
-; SSE4-NEXT:    pextrb $12, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $-128, %al
 ; SSE4-NEXT:    je .LBB1_16
-; SSE4-NEXT:  # %bb.15: # %cond.store13
+; SSE4-NEXT:  .LBB1_15: # %cond.store13
 ; SSE4-NEXT:    pextrw $7, %xmm1, 14(%rdi)
-; SSE4-NEXT:  .LBB1_16: # %else14
 ; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: truncstore_v8i64_v8i16:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpxor %xmm8, %xmm8, %xmm8
-; AVX1-NEXT:    vpcmpeqd %xmm8, %xmm2, %xmm5
-; AVX1-NEXT:    vpcmpeqd %xmm9, %xmm9, %xmm9
-; AVX1-NEXT:    vpxor %xmm9, %xmm5, %xmm10
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [9223372036854775808,9223372036854775808]
-; AVX1-NEXT:    vpxor %xmm6, %xmm0, %xmm7
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [9223372036854841343,9223372036854841343]
-; AVX1-NEXT:    vpcmpgtq %xmm7, %xmm3, %xmm11
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
-; AVX1-NEXT:    vpxor %xmm6, %xmm4, %xmm5
-; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm3, %xmm12
-; AVX1-NEXT:    vpxor %xmm6, %xmm1, %xmm7
-; AVX1-NEXT:    vpcmpgtq %xmm7, %xmm3, %xmm7
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm5
-; AVX1-NEXT:    vpxor %xmm6, %xmm5, %xmm6
-; AVX1-NEXT:    vpcmpgtq %xmm6, %xmm3, %xmm3
-; AVX1-NEXT:    vmovapd {{.*#+}} xmm6 = [65535,65535]
-; AVX1-NEXT:    vblendvpd %xmm3, %xmm5, %xmm6, %xmm3
-; AVX1-NEXT:    vblendvpd %xmm7, %xmm1, %xmm6, %xmm1
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808]
+; AVX1-NEXT:    vpxor %xmm3, %xmm0, %xmm4
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm5 = [9223372036854841343,9223372036854841343]
+; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm5, %xmm8
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm6
+; AVX1-NEXT:    vpxor %xmm3, %xmm6, %xmm7
+; AVX1-NEXT:    vpcmpgtq %xmm7, %xmm5, %xmm9
+; AVX1-NEXT:    vpxor %xmm3, %xmm1, %xmm4
+; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm5, %xmm4
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm7
+; AVX1-NEXT:    vpxor %xmm3, %xmm7, %xmm3
+; AVX1-NEXT:    vpcmpgtq %xmm3, %xmm5, %xmm3
+; AVX1-NEXT:    vmovapd {{.*#+}} xmm5 = [65535,65535]
+; AVX1-NEXT:    vblendvpd %xmm3, %xmm7, %xmm5, %xmm3
+; AVX1-NEXT:    vblendvpd %xmm4, %xmm1, %xmm5, %xmm1
 ; AVX1-NEXT:    vpackusdw %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vblendvpd %xmm12, %xmm4, %xmm6, %xmm3
-; AVX1-NEXT:    vblendvpd %xmm11, %xmm0, %xmm6, %xmm0
+; AVX1-NEXT:    vblendvpd %xmm9, %xmm6, %xmm5, %xmm3
+; AVX1-NEXT:    vblendvpd %xmm8, %xmm0, %xmm5, %xmm0
 ; AVX1-NEXT:    vpackusdw %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpextrb $0, %xmm10, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je .LBB1_2
-; AVX1-NEXT:  # %bb.1: # %cond.store
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm1
+; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
+; AVX1-NEXT:    vmovmskps %ymm1, %eax
+; AVX1-NEXT:    notl %eax
+; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    jne .LBB1_1
+; AVX1-NEXT:  # %bb.2: # %else
+; AVX1-NEXT:    testb $2, %al
+; AVX1-NEXT:    jne .LBB1_3
+; AVX1-NEXT:  .LBB1_4: # %else2
+; AVX1-NEXT:    testb $4, %al
+; AVX1-NEXT:    jne .LBB1_5
+; AVX1-NEXT:  .LBB1_6: # %else4
+; AVX1-NEXT:    testb $8, %al
+; AVX1-NEXT:    jne .LBB1_7
+; AVX1-NEXT:  .LBB1_8: # %else6
+; AVX1-NEXT:    testb $16, %al
+; AVX1-NEXT:    jne .LBB1_9
+; AVX1-NEXT:  .LBB1_10: # %else8
+; AVX1-NEXT:    testb $32, %al
+; AVX1-NEXT:    jne .LBB1_11
+; AVX1-NEXT:  .LBB1_12: # %else10
+; AVX1-NEXT:    testb $64, %al
+; AVX1-NEXT:    jne .LBB1_13
+; AVX1-NEXT:  .LBB1_14: # %else12
+; AVX1-NEXT:    testb $-128, %al
+; AVX1-NEXT:    jne .LBB1_15
+; AVX1-NEXT:  .LBB1_16: # %else14
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+; AVX1-NEXT:  .LBB1_1: # %cond.store
 ; AVX1-NEXT:    vpextrw $0, %xmm0, (%rdi)
-; AVX1-NEXT:  .LBB1_2: # %else
-; AVX1-NEXT:    vpcmpeqd %xmm8, %xmm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm9, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $4, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $2, %al
 ; AVX1-NEXT:    je .LBB1_4
-; AVX1-NEXT:  # %bb.3: # %cond.store1
+; AVX1-NEXT:  .LBB1_3: # %cond.store1
 ; AVX1-NEXT:    vpextrw $1, %xmm0, 2(%rdi)
-; AVX1-NEXT:  .LBB1_4: # %else2
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm2, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm4, %xmm4
-; AVX1-NEXT:    vpextrb $8, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $4, %al
 ; AVX1-NEXT:    je .LBB1_6
-; AVX1-NEXT:  # %bb.5: # %cond.store3
+; AVX1-NEXT:  .LBB1_5: # %cond.store3
 ; AVX1-NEXT:    vpextrw $2, %xmm0, 4(%rdi)
-; AVX1-NEXT:  .LBB1_6: # %else4
-; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $8, %al
 ; AVX1-NEXT:    je .LBB1_8
-; AVX1-NEXT:  # %bb.7: # %cond.store5
+; AVX1-NEXT:  .LBB1_7: # %cond.store5
 ; AVX1-NEXT:    vpextrw $3, %xmm0, 6(%rdi)
-; AVX1-NEXT:  .LBB1_8: # %else6
-; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $0, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $16, %al
 ; AVX1-NEXT:    je .LBB1_10
-; AVX1-NEXT:  # %bb.9: # %cond.store7
+; AVX1-NEXT:  .LBB1_9: # %cond.store7
 ; AVX1-NEXT:    vpextrw $4, %xmm0, 8(%rdi)
-; AVX1-NEXT:  .LBB1_10: # %else8
-; AVX1-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $32, %al
 ; AVX1-NEXT:    je .LBB1_12
-; AVX1-NEXT:  # %bb.11: # %cond.store9
+; AVX1-NEXT:  .LBB1_11: # %cond.store9
 ; AVX1-NEXT:    vpextrw $5, %xmm0, 10(%rdi)
-; AVX1-NEXT:  .LBB1_12: # %else10
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $64, %al
 ; AVX1-NEXT:    je .LBB1_14
-; AVX1-NEXT:  # %bb.13: # %cond.store11
+; AVX1-NEXT:  .LBB1_13: # %cond.store11
 ; AVX1-NEXT:    vpextrw $6, %xmm0, 12(%rdi)
-; AVX1-NEXT:  .LBB1_14: # %else12
-; AVX1-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $-128, %al
 ; AVX1-NEXT:    je .LBB1_16
-; AVX1-NEXT:  # %bb.15: # %cond.store13
+; AVX1-NEXT:  .LBB1_15: # %cond.store13
 ; AVX1-NEXT:    vpextrw $7, %xmm0, 14(%rdi)
-; AVX1-NEXT:  .LBB1_16: # %else14
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: truncstore_v8i64_v8i16:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm5
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm6 = [65535,65535,65535,65535]
-; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm7 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
-; AVX2-NEXT:    vpxor %ymm7, %ymm1, %ymm8
-; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm9 = [9223372036854841343,9223372036854841343,9223372036854841343,9223372036854841343]
-; AVX2-NEXT:    vpcmpgtq %ymm8, %ymm9, %ymm8
-; AVX2-NEXT:    vblendvpd %ymm8, %ymm1, %ymm6, %ymm1
-; AVX2-NEXT:    vpxor %ymm7, %ymm0, %ymm7
-; AVX2-NEXT:    vpcmpgtq %ymm7, %ymm9, %ymm7
-; AVX2-NEXT:    vblendvpd %ymm7, %ymm0, %ymm6, %ymm0
+; AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm4 = [65535,65535,65535,65535]
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm5 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
+; AVX2-NEXT:    vpxor %ymm5, %ymm1, %ymm6
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm7 = [9223372036854841343,9223372036854841343,9223372036854841343,9223372036854841343]
+; AVX2-NEXT:    vpcmpgtq %ymm6, %ymm7, %ymm6
+; AVX2-NEXT:    vblendvpd %ymm6, %ymm1, %ymm4, %ymm1
+; AVX2-NEXT:    vpxor %ymm5, %ymm0, %ymm5
+; AVX2-NEXT:    vpcmpgtq %ymm5, %ymm7, %ymm5
+; AVX2-NEXT:    vblendvpd %ymm5, %ymm0, %ymm4, %ymm0
 ; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpextrb $0, %xmm5, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je .LBB1_2
-; AVX2-NEXT:  # %bb.1: # %cond.store
+; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm2, %ymm1
+; AVX2-NEXT:    vmovmskps %ymm1, %eax
+; AVX2-NEXT:    notl %eax
+; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    jne .LBB1_1
+; AVX2-NEXT:  # %bb.2: # %else
+; AVX2-NEXT:    testb $2, %al
+; AVX2-NEXT:    jne .LBB1_3
+; AVX2-NEXT:  .LBB1_4: # %else2
+; AVX2-NEXT:    testb $4, %al
+; AVX2-NEXT:    jne .LBB1_5
+; AVX2-NEXT:  .LBB1_6: # %else4
+; AVX2-NEXT:    testb $8, %al
+; AVX2-NEXT:    jne .LBB1_7
+; AVX2-NEXT:  .LBB1_8: # %else6
+; AVX2-NEXT:    testb $16, %al
+; AVX2-NEXT:    jne .LBB1_9
+; AVX2-NEXT:  .LBB1_10: # %else8
+; AVX2-NEXT:    testb $32, %al
+; AVX2-NEXT:    jne .LBB1_11
+; AVX2-NEXT:  .LBB1_12: # %else10
+; AVX2-NEXT:    testb $64, %al
+; AVX2-NEXT:    jne .LBB1_13
+; AVX2-NEXT:  .LBB1_14: # %else12
+; AVX2-NEXT:    testb $-128, %al
+; AVX2-NEXT:    jne .LBB1_15
+; AVX2-NEXT:  .LBB1_16: # %else14
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+; AVX2-NEXT:  .LBB1_1: # %cond.store
 ; AVX2-NEXT:    vpextrw $0, %xmm0, (%rdi)
-; AVX2-NEXT:  .LBB1_2: # %else
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm1
-; AVX2-NEXT:    vpxor %xmm4, %xmm1, %xmm1
-; AVX2-NEXT:    vpextrb $4, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $2, %al
 ; AVX2-NEXT:    je .LBB1_4
-; AVX2-NEXT:  # %bb.3: # %cond.store1
+; AVX2-NEXT:  .LBB1_3: # %cond.store1
 ; AVX2-NEXT:    vpextrw $1, %xmm0, 2(%rdi)
-; AVX2-NEXT:  .LBB1_4: # %else2
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm2, %xmm4
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    vpxor %xmm3, %xmm4, %xmm4
-; AVX2-NEXT:    vpextrb $8, %xmm4, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $4, %al
 ; AVX2-NEXT:    je .LBB1_6
-; AVX2-NEXT:  # %bb.5: # %cond.store3
+; AVX2-NEXT:  .LBB1_5: # %cond.store3
 ; AVX2-NEXT:    vpextrw $2, %xmm0, 4(%rdi)
-; AVX2-NEXT:  .LBB1_6: # %else4
-; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm2, %xmm1
-; AVX2-NEXT:    vpxor %xmm3, %xmm1, %xmm1
-; AVX2-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $8, %al
 ; AVX2-NEXT:    je .LBB1_8
-; AVX2-NEXT:  # %bb.7: # %cond.store5
+; AVX2-NEXT:  .LBB1_7: # %cond.store5
 ; AVX2-NEXT:    vpextrw $3, %xmm0, 6(%rdi)
-; AVX2-NEXT:  .LBB1_8: # %else6
-; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm1
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrb $0, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $16, %al
 ; AVX2-NEXT:    je .LBB1_10
-; AVX2-NEXT:  # %bb.9: # %cond.store7
+; AVX2-NEXT:  .LBB1_9: # %cond.store7
 ; AVX2-NEXT:    vpextrw $4, %xmm0, 8(%rdi)
-; AVX2-NEXT:  .LBB1_10: # %else8
-; AVX2-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $32, %al
 ; AVX2-NEXT:    je .LBB1_12
-; AVX2-NEXT:  # %bb.11: # %cond.store9
+; AVX2-NEXT:  .LBB1_11: # %cond.store9
 ; AVX2-NEXT:    vpextrw $5, %xmm0, 10(%rdi)
-; AVX2-NEXT:  .LBB1_12: # %else10
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $64, %al
 ; AVX2-NEXT:    je .LBB1_14
-; AVX2-NEXT:  # %bb.13: # %cond.store11
+; AVX2-NEXT:  .LBB1_13: # %cond.store11
 ; AVX2-NEXT:    vpextrw $6, %xmm0, 12(%rdi)
-; AVX2-NEXT:  .LBB1_14: # %else12
-; AVX2-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $-128, %al
 ; AVX2-NEXT:    je .LBB1_16
-; AVX2-NEXT:  # %bb.15: # %cond.store13
+; AVX2-NEXT:  .LBB1_15: # %cond.store13
 ; AVX2-NEXT:    vpextrw $7, %xmm0, 14(%rdi)
-; AVX2-NEXT:  .LBB1_16: # %else14
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
@@ -761,66 +729,61 @@ define void @truncstore_v8i64_v8i16(<8 x i64> %x, <8 x i16>* %p, <8 x i32> %mask
 ; AVX512F-NEXT:    vpmovusqw %zmm0, %xmm0
 ; AVX512F-NEXT:    kmovw %k0, %eax
 ; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB1_2
-; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    jne .LBB1_1
+; AVX512F-NEXT:  # %bb.2: # %else
+; AVX512F-NEXT:    testb $2, %al
+; AVX512F-NEXT:    jne .LBB1_3
+; AVX512F-NEXT:  .LBB1_4: # %else2
+; AVX512F-NEXT:    testb $4, %al
+; AVX512F-NEXT:    jne .LBB1_5
+; AVX512F-NEXT:  .LBB1_6: # %else4
+; AVX512F-NEXT:    testb $8, %al
+; AVX512F-NEXT:    jne .LBB1_7
+; AVX512F-NEXT:  .LBB1_8: # %else6
+; AVX512F-NEXT:    testb $16, %al
+; AVX512F-NEXT:    jne .LBB1_9
+; AVX512F-NEXT:  .LBB1_10: # %else8
+; AVX512F-NEXT:    testb $32, %al
+; AVX512F-NEXT:    jne .LBB1_11
+; AVX512F-NEXT:  .LBB1_12: # %else10
+; AVX512F-NEXT:    testb $64, %al
+; AVX512F-NEXT:    jne .LBB1_13
+; AVX512F-NEXT:  .LBB1_14: # %else12
+; AVX512F-NEXT:    testb $-128, %al
+; AVX512F-NEXT:    jne .LBB1_15
+; AVX512F-NEXT:  .LBB1_16: # %else14
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+; AVX512F-NEXT:  .LBB1_1: # %cond.store
 ; AVX512F-NEXT:    vpextrw $0, %xmm0, (%rdi)
-; AVX512F-NEXT:  .LBB1_2: # %else
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $2, %al
 ; AVX512F-NEXT:    je .LBB1_4
-; AVX512F-NEXT:  # %bb.3: # %cond.store1
+; AVX512F-NEXT:  .LBB1_3: # %cond.store1
 ; AVX512F-NEXT:    vpextrw $1, %xmm0, 2(%rdi)
-; AVX512F-NEXT:  .LBB1_4: # %else2
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $4, %al
 ; AVX512F-NEXT:    je .LBB1_6
-; AVX512F-NEXT:  # %bb.5: # %cond.store3
+; AVX512F-NEXT:  .LBB1_5: # %cond.store3
 ; AVX512F-NEXT:    vpextrw $2, %xmm0, 4(%rdi)
-; AVX512F-NEXT:  .LBB1_6: # %else4
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $8, %al
 ; AVX512F-NEXT:    je .LBB1_8
-; AVX512F-NEXT:  # %bb.7: # %cond.store5
+; AVX512F-NEXT:  .LBB1_7: # %cond.store5
 ; AVX512F-NEXT:    vpextrw $3, %xmm0, 6(%rdi)
-; AVX512F-NEXT:  .LBB1_8: # %else6
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $4, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $16, %al
 ; AVX512F-NEXT:    je .LBB1_10
-; AVX512F-NEXT:  # %bb.9: # %cond.store7
+; AVX512F-NEXT:  .LBB1_9: # %cond.store7
 ; AVX512F-NEXT:    vpextrw $4, %xmm0, 8(%rdi)
-; AVX512F-NEXT:  .LBB1_10: # %else8
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $5, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $32, %al
 ; AVX512F-NEXT:    je .LBB1_12
-; AVX512F-NEXT:  # %bb.11: # %cond.store9
+; AVX512F-NEXT:  .LBB1_11: # %cond.store9
 ; AVX512F-NEXT:    vpextrw $5, %xmm0, 10(%rdi)
-; AVX512F-NEXT:  .LBB1_12: # %else10
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $6, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $64, %al
 ; AVX512F-NEXT:    je .LBB1_14
-; AVX512F-NEXT:  # %bb.13: # %cond.store11
+; AVX512F-NEXT:  .LBB1_13: # %cond.store11
 ; AVX512F-NEXT:    vpextrw $6, %xmm0, 12(%rdi)
-; AVX512F-NEXT:  .LBB1_14: # %else12
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $7, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $-128, %al
 ; AVX512F-NEXT:    je .LBB1_16
-; AVX512F-NEXT:  # %bb.15: # %cond.store13
+; AVX512F-NEXT:  .LBB1_15: # %cond.store13
 ; AVX512F-NEXT:    vpextrw $7, %xmm0, 14(%rdi)
-; AVX512F-NEXT:  .LBB1_16: # %else14
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
@@ -851,148 +814,135 @@ define void @truncstore_v8i64_v8i16(<8 x i64> %x, <8 x i16>* %p, <8 x i32> %mask
 define void @truncstore_v8i64_v8i8(<8 x i64> %x, <8 x i8>* %p, <8 x i32> %mask) {
 ; SSE2-LABEL: truncstore_v8i64_v8i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pxor %xmm7, %xmm7
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm7
-; SSE2-NEXT:    pcmpeqd %xmm11, %xmm11
-; SSE2-NEXT:    pxor %xmm7, %xmm11
-; SSE2-NEXT:    movdqa %xmm11, %xmm9
-; SSE2-NEXT:    packssdw %xmm0, %xmm9
-; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [255,255]
-; SSE2-NEXT:    movdqa {{.*#+}} xmm12 = [9223372039002259456,9223372039002259456]
-; SSE2-NEXT:    movdqa %xmm1, %xmm6
-; SSE2-NEXT:    pxor %xmm12, %xmm6
+; SSE2-NEXT:    pxor %xmm8, %xmm8
+; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [255,255]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm11 = [9223372039002259456,9223372039002259456]
+; SSE2-NEXT:    movdqa %xmm1, %xmm7
+; SSE2-NEXT:    pxor %xmm11, %xmm7
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [9223372039002259711,9223372039002259711]
-; SSE2-NEXT:    movdqa %xmm10, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm6, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm7[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm10, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT:    pand %xmm13, %xmm6
+; SSE2-NEXT:    movdqa %xmm10, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm7, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm6[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm7
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT:    por %xmm6, %xmm7
-; SSE2-NEXT:    pand %xmm7, %xmm1
-; SSE2-NEXT:    pandn %xmm8, %xmm7
-; SSE2-NEXT:    por %xmm1, %xmm7
+; SSE2-NEXT:    pand %xmm12, %xmm7
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSE2-NEXT:    por %xmm7, %xmm6
+; SSE2-NEXT:    pand %xmm6, %xmm1
+; SSE2-NEXT:    pandn %xmm9, %xmm6
+; SSE2-NEXT:    por %xmm1, %xmm6
 ; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    pxor %xmm12, %xmm1
-; SSE2-NEXT:    movdqa %xmm10, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm6[0,0,2,2]
+; SSE2-NEXT:    pxor %xmm11, %xmm1
+; SSE2-NEXT:    movdqa %xmm10, %xmm7
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm7
+; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm7[0,0,2,2]
 ; SSE2-NEXT:    pcmpeqd %xmm10, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pand %xmm13, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm6
-; SSE2-NEXT:    pand %xmm6, %xmm0
-; SSE2-NEXT:    pandn %xmm8, %xmm6
-; SSE2-NEXT:    por %xmm0, %xmm6
-; SSE2-NEXT:    packuswb %xmm7, %xmm6
+; SSE2-NEXT:    pand %xmm12, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
+; SSE2-NEXT:    por %xmm1, %xmm7
+; SSE2-NEXT:    pand %xmm7, %xmm0
+; SSE2-NEXT:    pandn %xmm9, %xmm7
+; SSE2-NEXT:    por %xmm0, %xmm7
+; SSE2-NEXT:    packuswb %xmm6, %xmm7
 ; SSE2-NEXT:    movdqa %xmm3, %xmm0
-; SSE2-NEXT:    pxor %xmm12, %xmm0
+; SSE2-NEXT:    pxor %xmm11, %xmm0
 ; SSE2-NEXT:    movdqa %xmm10, %xmm1
 ; SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm1[0,0,2,2]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[0,0,2,2]
 ; SSE2-NEXT:    pcmpeqd %xmm10, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pand %xmm7, %xmm0
+; SSE2-NEXT:    pand %xmm6, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
 ; SSE2-NEXT:    por %xmm0, %xmm1
 ; SSE2-NEXT:    pand %xmm1, %xmm3
-; SSE2-NEXT:    pandn %xmm8, %xmm1
+; SSE2-NEXT:    pandn %xmm9, %xmm1
 ; SSE2-NEXT:    por %xmm3, %xmm1
-; SSE2-NEXT:    pxor %xmm2, %xmm12
+; SSE2-NEXT:    pxor %xmm2, %xmm11
 ; SSE2-NEXT:    movdqa %xmm10, %xmm0
-; SSE2-NEXT:    pcmpgtd %xmm12, %xmm0
+; SSE2-NEXT:    pcmpgtd %xmm11, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm10, %xmm12
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm12[1,1,3,3]
-; SSE2-NEXT:    pand %xmm3, %xmm7
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm11
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm11[1,1,3,3]
+; SSE2-NEXT:    pand %xmm3, %xmm6
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm7, %xmm0
+; SSE2-NEXT:    por %xmm6, %xmm0
 ; SSE2-NEXT:    pand %xmm0, %xmm2
-; SSE2-NEXT:    pandn %xmm8, %xmm0
+; SSE2-NEXT:    pandn %xmm9, %xmm0
 ; SSE2-NEXT:    por %xmm2, %xmm0
 ; SSE2-NEXT:    packuswb %xmm1, %xmm0
-; SSE2-NEXT:    packuswb %xmm0, %xmm6
-; SSE2-NEXT:    movd %xmm9, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    movd %xmm6, %eax
-; SSE2-NEXT:    je .LBB2_2
-; SSE2-NEXT:  # %bb.1: # %cond.store
-; SSE2-NEXT:    movb %al, (%rdi)
-; SSE2-NEXT:  .LBB2_2: # %else
-; SSE2-NEXT:    psrlq $16, %xmm11
-; SSE2-NEXT:    movd %xmm11, %ecx
-; SSE2-NEXT:    shrl $16, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    je .LBB2_4
-; SSE2-NEXT:  # %bb.3: # %cond.store1
-; SSE2-NEXT:    shrl $16, %eax
-; SSE2-NEXT:    movb %al, 1(%rdi)
-; SSE2-NEXT:  .LBB2_4: # %else2
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE2-NEXT:    packuswb %xmm0, %xmm7
+; SSE2-NEXT:    packuswb %xmm7, %xmm7
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm5
 ; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE2-NEXT:    pxor %xmm4, %xmm0
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
+; SSE2-NEXT:    pxor %xmm0, %xmm5
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm4
+; SSE2-NEXT:    pxor %xmm0, %xmm4
+; SSE2-NEXT:    packssdw %xmm5, %xmm4
+; SSE2-NEXT:    packsswb %xmm0, %xmm4
+; SSE2-NEXT:    pmovmskb %xmm4, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB2_6
-; SSE2-NEXT:  # %bb.5: # %cond.store3
-; SSE2-NEXT:    pextrw $2, %xmm6, %eax
-; SSE2-NEXT:    movb %al, 2(%rdi)
+; SSE2-NEXT:    movd %xmm7, %ecx
+; SSE2-NEXT:    jne .LBB2_1
+; SSE2-NEXT:  # %bb.2: # %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne .LBB2_3
+; SSE2-NEXT:  .LBB2_4: # %else2
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne .LBB2_5
 ; SSE2-NEXT:  .LBB2_6: # %else4
-; SSE2-NEXT:    pextrw $6, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je .LBB2_8
-; SSE2-NEXT:  # %bb.7: # %cond.store5
-; SSE2-NEXT:    pextrw $3, %xmm6, %eax
-; SSE2-NEXT:    movb %al, 3(%rdi)
+; SSE2-NEXT:  .LBB2_7: # %cond.store5
+; SSE2-NEXT:    shrl $24, %ecx
+; SSE2-NEXT:    movb %cl, 3(%rdi)
 ; SSE2-NEXT:  .LBB2_8: # %else6
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE2-NEXT:    pxor %xmm1, %xmm0
-; SSE2-NEXT:    pextrw $0, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $16, %al
+; SSE2-NEXT:    pextrw $2, %xmm7, %ecx
 ; SSE2-NEXT:    je .LBB2_10
 ; SSE2-NEXT:  # %bb.9: # %cond.store7
-; SSE2-NEXT:    pextrw $4, %xmm6, %eax
-; SSE2-NEXT:    movb %al, 4(%rdi)
+; SSE2-NEXT:    movb %cl, 4(%rdi)
 ; SSE2-NEXT:  .LBB2_10: # %else8
-; SSE2-NEXT:    pextrw $2, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $32, %al
 ; SSE2-NEXT:    je .LBB2_12
 ; SSE2-NEXT:  # %bb.11: # %cond.store9
-; SSE2-NEXT:    pextrw $5, %xmm6, %eax
-; SSE2-NEXT:    movb %al, 5(%rdi)
+; SSE2-NEXT:    movb %ch, 5(%rdi)
 ; SSE2-NEXT:  .LBB2_12: # %else10
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm5
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE2-NEXT:    pxor %xmm5, %xmm0
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB2_14
-; SSE2-NEXT:  # %bb.13: # %cond.store11
-; SSE2-NEXT:    pextrw $6, %xmm6, %eax
-; SSE2-NEXT:    movb %al, 6(%rdi)
-; SSE2-NEXT:  .LBB2_14: # %else12
-; SSE2-NEXT:    pextrw $6, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB2_16
-; SSE2-NEXT:  # %bb.15: # %cond.store13
-; SSE2-NEXT:    pextrw $7, %xmm6, %eax
-; SSE2-NEXT:    movb %al, 7(%rdi)
+; SSE2-NEXT:    testb $64, %al
+; SSE2-NEXT:    pextrw $3, %xmm7, %ecx
+; SSE2-NEXT:    jne .LBB2_13
+; SSE2-NEXT:  # %bb.14: # %else12
+; SSE2-NEXT:    testb $-128, %al
+; SSE2-NEXT:    jne .LBB2_15
 ; SSE2-NEXT:  .LBB2_16: # %else14
 ; SSE2-NEXT:    retq
+; SSE2-NEXT:  .LBB2_1: # %cond.store
+; SSE2-NEXT:    movb %cl, (%rdi)
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    je .LBB2_4
+; SSE2-NEXT:  .LBB2_3: # %cond.store1
+; SSE2-NEXT:    movb %ch, 1(%rdi)
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    je .LBB2_6
+; SSE2-NEXT:  .LBB2_5: # %cond.store3
+; SSE2-NEXT:    movl %ecx, %edx
+; SSE2-NEXT:    shrl $16, %edx
+; SSE2-NEXT:    movb %dl, 2(%rdi)
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne .LBB2_7
+; SSE2-NEXT:    jmp .LBB2_8
+; SSE2-NEXT:  .LBB2_13: # %cond.store11
+; SSE2-NEXT:    movb %cl, 6(%rdi)
+; SSE2-NEXT:    testb $-128, %al
+; SSE2-NEXT:    je .LBB2_16
+; SSE2-NEXT:  .LBB2_15: # %cond.store13
+; SSE2-NEXT:    movb %ch, 7(%rdi)
+; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: truncstore_v8i64_v8i8:
 ; SSE4:       # %bb.0:
-; SSE4-NEXT:    movdqa %xmm0, %xmm8
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqd %xmm4, %xmm0
-; SSE4-NEXT:    pcmpeqd %xmm9, %xmm9
-; SSE4-NEXT:    pxor %xmm0, %xmm9
+; SSE4-NEXT:    movdqa %xmm0, %xmm9
+; SSE4-NEXT:    pxor %xmm8, %xmm8
 ; SSE4-NEXT:    movapd {{.*#+}} xmm6 = [255,255]
 ; SSE4-NEXT:    movdqa {{.*#+}} xmm10 = [9223372036854775808,9223372036854775808]
 ; SSE4-NEXT:    movdqa %xmm1, %xmm7
@@ -1002,12 +952,12 @@ define void @truncstore_v8i64_v8i8(<8 x i64> %x, <8 x i8>* %p, <8 x i32> %mask)
 ; SSE4-NEXT:    pcmpgtq %xmm7, %xmm0
 ; SSE4-NEXT:    movapd %xmm6, %xmm7
 ; SSE4-NEXT:    blendvpd %xmm0, %xmm1, %xmm7
-; SSE4-NEXT:    movdqa %xmm8, %xmm1
+; SSE4-NEXT:    movdqa %xmm9, %xmm1
 ; SSE4-NEXT:    pxor %xmm10, %xmm1
 ; SSE4-NEXT:    movdqa %xmm11, %xmm0
 ; SSE4-NEXT:    pcmpgtq %xmm1, %xmm0
 ; SSE4-NEXT:    movapd %xmm6, %xmm1
-; SSE4-NEXT:    blendvpd %xmm0, %xmm8, %xmm1
+; SSE4-NEXT:    blendvpd %xmm0, %xmm9, %xmm1
 ; SSE4-NEXT:    packusdw %xmm7, %xmm1
 ; SSE4-NEXT:    movdqa %xmm3, %xmm7
 ; SSE4-NEXT:    pxor %xmm10, %xmm7
@@ -1021,246 +971,243 @@ define void @truncstore_v8i64_v8i8(<8 x i64> %x, <8 x i8>* %p, <8 x i32> %mask)
 ; SSE4-NEXT:    blendvpd %xmm0, %xmm2, %xmm6
 ; SSE4-NEXT:    packusdw %xmm7, %xmm6
 ; SSE4-NEXT:    packusdw %xmm6, %xmm1
-; SSE4-NEXT:    pextrb $0, %xmm9, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB2_2
-; SSE4-NEXT:  # %bb.1: # %cond.store
+; SSE4-NEXT:    packuswb %xmm1, %xmm1
+; SSE4-NEXT:    pcmpeqd %xmm8, %xmm5
+; SSE4-NEXT:    pcmpeqd %xmm0, %xmm0
+; SSE4-NEXT:    pxor %xmm0, %xmm5
+; SSE4-NEXT:    pcmpeqd %xmm8, %xmm4
+; SSE4-NEXT:    pxor %xmm0, %xmm4
+; SSE4-NEXT:    packssdw %xmm5, %xmm4
+; SSE4-NEXT:    packsswb %xmm0, %xmm4
+; SSE4-NEXT:    pmovmskb %xmm4, %eax
+; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    jne .LBB2_1
+; SSE4-NEXT:  # %bb.2: # %else
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    jne .LBB2_3
+; SSE4-NEXT:  .LBB2_4: # %else2
+; SSE4-NEXT:    testb $4, %al
+; SSE4-NEXT:    jne .LBB2_5
+; SSE4-NEXT:  .LBB2_6: # %else4
+; SSE4-NEXT:    testb $8, %al
+; SSE4-NEXT:    jne .LBB2_7
+; SSE4-NEXT:  .LBB2_8: # %else6
+; SSE4-NEXT:    testb $16, %al
+; SSE4-NEXT:    jne .LBB2_9
+; SSE4-NEXT:  .LBB2_10: # %else8
+; SSE4-NEXT:    testb $32, %al
+; SSE4-NEXT:    jne .LBB2_11
+; SSE4-NEXT:  .LBB2_12: # %else10
+; SSE4-NEXT:    testb $64, %al
+; SSE4-NEXT:    jne .LBB2_13
+; SSE4-NEXT:  .LBB2_14: # %else12
+; SSE4-NEXT:    testb $-128, %al
+; SSE4-NEXT:    jne .LBB2_15
+; SSE4-NEXT:  .LBB2_16: # %else14
+; SSE4-NEXT:    retq
+; SSE4-NEXT:  .LBB2_1: # %cond.store
 ; SSE4-NEXT:    pextrb $0, %xmm1, (%rdi)
-; SSE4-NEXT:  .LBB2_2: # %else
-; SSE4-NEXT:    pextrb $4, %xmm9, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $2, %al
 ; SSE4-NEXT:    je .LBB2_4
-; SSE4-NEXT:  # %bb.3: # %cond.store1
-; SSE4-NEXT:    pextrb $2, %xmm1, 1(%rdi)
-; SSE4-NEXT:  .LBB2_4: # %else2
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqd %xmm0, %xmm4
-; SSE4-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE4-NEXT:    pxor %xmm4, %xmm0
-; SSE4-NEXT:    pextrb $8, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:  .LBB2_3: # %cond.store1
+; SSE4-NEXT:    pextrb $1, %xmm1, 1(%rdi)
+; SSE4-NEXT:    testb $4, %al
 ; SSE4-NEXT:    je .LBB2_6
-; SSE4-NEXT:  # %bb.5: # %cond.store3
-; SSE4-NEXT:    pextrb $4, %xmm1, 2(%rdi)
-; SSE4-NEXT:  .LBB2_6: # %else4
-; SSE4-NEXT:    pextrb $12, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:  .LBB2_5: # %cond.store3
+; SSE4-NEXT:    pextrb $2, %xmm1, 2(%rdi)
+; SSE4-NEXT:    testb $8, %al
 ; SSE4-NEXT:    je .LBB2_8
-; SSE4-NEXT:  # %bb.7: # %cond.store5
-; SSE4-NEXT:    pextrb $6, %xmm1, 3(%rdi)
-; SSE4-NEXT:  .LBB2_8: # %else6
-; SSE4-NEXT:    pxor %xmm2, %xmm2
-; SSE4-NEXT:    pcmpeqd %xmm5, %xmm2
-; SSE4-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE4-NEXT:    pxor %xmm2, %xmm0
-; SSE4-NEXT:    pextrb $0, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:  .LBB2_7: # %cond.store5
+; SSE4-NEXT:    pextrb $3, %xmm1, 3(%rdi)
+; SSE4-NEXT:    testb $16, %al
 ; SSE4-NEXT:    je .LBB2_10
-; SSE4-NEXT:  # %bb.9: # %cond.store7
-; SSE4-NEXT:    pextrb $8, %xmm1, 4(%rdi)
-; SSE4-NEXT:  .LBB2_10: # %else8
-; SSE4-NEXT:    pextrb $4, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:  .LBB2_9: # %cond.store7
+; SSE4-NEXT:    pextrb $4, %xmm1, 4(%rdi)
+; SSE4-NEXT:    testb $32, %al
 ; SSE4-NEXT:    je .LBB2_12
-; SSE4-NEXT:  # %bb.11: # %cond.store9
-; SSE4-NEXT:    pextrb $10, %xmm1, 5(%rdi)
-; SSE4-NEXT:  .LBB2_12: # %else10
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqd %xmm0, %xmm5
-; SSE4-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE4-NEXT:    pxor %xmm5, %xmm0
-; SSE4-NEXT:    pextrb $8, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:  .LBB2_11: # %cond.store9
+; SSE4-NEXT:    pextrb $5, %xmm1, 5(%rdi)
+; SSE4-NEXT:    testb $64, %al
 ; SSE4-NEXT:    je .LBB2_14
-; SSE4-NEXT:  # %bb.13: # %cond.store11
-; SSE4-NEXT:    pextrb $12, %xmm1, 6(%rdi)
-; SSE4-NEXT:  .LBB2_14: # %else12
-; SSE4-NEXT:    pextrb $12, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:  .LBB2_13: # %cond.store11
+; SSE4-NEXT:    pextrb $6, %xmm1, 6(%rdi)
+; SSE4-NEXT:    testb $-128, %al
 ; SSE4-NEXT:    je .LBB2_16
-; SSE4-NEXT:  # %bb.15: # %cond.store13
-; SSE4-NEXT:    pextrb $14, %xmm1, 7(%rdi)
-; SSE4-NEXT:  .LBB2_16: # %else14
+; SSE4-NEXT:  .LBB2_15: # %cond.store13
+; SSE4-NEXT:    pextrb $7, %xmm1, 7(%rdi)
 ; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: truncstore_v8i64_v8i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpxor %xmm8, %xmm8, %xmm8
-; AVX1-NEXT:    vpcmpeqd %xmm8, %xmm2, %xmm5
-; AVX1-NEXT:    vpcmpeqd %xmm9, %xmm9, %xmm9
-; AVX1-NEXT:    vpxor %xmm9, %xmm5, %xmm10
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [9223372036854775808,9223372036854775808]
-; AVX1-NEXT:    vpxor %xmm6, %xmm0, %xmm7
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [9223372036854776063,9223372036854776063]
-; AVX1-NEXT:    vpcmpgtq %xmm7, %xmm3, %xmm11
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
-; AVX1-NEXT:    vpxor %xmm6, %xmm4, %xmm5
-; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm3, %xmm12
-; AVX1-NEXT:    vpxor %xmm6, %xmm1, %xmm7
-; AVX1-NEXT:    vpcmpgtq %xmm7, %xmm3, %xmm7
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm5
-; AVX1-NEXT:    vpxor %xmm6, %xmm5, %xmm6
-; AVX1-NEXT:    vpcmpgtq %xmm6, %xmm3, %xmm3
-; AVX1-NEXT:    vmovapd {{.*#+}} xmm6 = [255,255]
-; AVX1-NEXT:    vblendvpd %xmm3, %xmm5, %xmm6, %xmm3
-; AVX1-NEXT:    vblendvpd %xmm7, %xmm1, %xmm6, %xmm1
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808]
+; AVX1-NEXT:    vpxor %xmm3, %xmm0, %xmm4
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm5 = [9223372036854776063,9223372036854776063]
+; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm5, %xmm8
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm6
+; AVX1-NEXT:    vpxor %xmm3, %xmm6, %xmm7
+; AVX1-NEXT:    vpcmpgtq %xmm7, %xmm5, %xmm9
+; AVX1-NEXT:    vpxor %xmm3, %xmm1, %xmm4
+; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm5, %xmm4
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm7
+; AVX1-NEXT:    vpxor %xmm3, %xmm7, %xmm3
+; AVX1-NEXT:    vpcmpgtq %xmm3, %xmm5, %xmm3
+; AVX1-NEXT:    vmovapd {{.*#+}} xmm5 = [255,255]
+; AVX1-NEXT:    vblendvpd %xmm3, %xmm7, %xmm5, %xmm3
+; AVX1-NEXT:    vblendvpd %xmm4, %xmm1, %xmm5, %xmm1
 ; AVX1-NEXT:    vpackusdw %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vblendvpd %xmm12, %xmm4, %xmm6, %xmm3
-; AVX1-NEXT:    vblendvpd %xmm11, %xmm0, %xmm6, %xmm0
+; AVX1-NEXT:    vblendvpd %xmm9, %xmm6, %xmm5, %xmm3
+; AVX1-NEXT:    vblendvpd %xmm8, %xmm0, %xmm5, %xmm0
 ; AVX1-NEXT:    vpackusdw %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpextrb $0, %xmm10, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je .LBB2_2
-; AVX1-NEXT:  # %bb.1: # %cond.store
+; AVX1-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm1
+; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
+; AVX1-NEXT:    vmovmskps %ymm1, %eax
+; AVX1-NEXT:    notl %eax
+; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    jne .LBB2_1
+; AVX1-NEXT:  # %bb.2: # %else
+; AVX1-NEXT:    testb $2, %al
+; AVX1-NEXT:    jne .LBB2_3
+; AVX1-NEXT:  .LBB2_4: # %else2
+; AVX1-NEXT:    testb $4, %al
+; AVX1-NEXT:    jne .LBB2_5
+; AVX1-NEXT:  .LBB2_6: # %else4
+; AVX1-NEXT:    testb $8, %al
+; AVX1-NEXT:    jne .LBB2_7
+; AVX1-NEXT:  .LBB2_8: # %else6
+; AVX1-NEXT:    testb $16, %al
+; AVX1-NEXT:    jne .LBB2_9
+; AVX1-NEXT:  .LBB2_10: # %else8
+; AVX1-NEXT:    testb $32, %al
+; AVX1-NEXT:    jne .LBB2_11
+; AVX1-NEXT:  .LBB2_12: # %else10
+; AVX1-NEXT:    testb $64, %al
+; AVX1-NEXT:    jne .LBB2_13
+; AVX1-NEXT:  .LBB2_14: # %else12
+; AVX1-NEXT:    testb $-128, %al
+; AVX1-NEXT:    jne .LBB2_15
+; AVX1-NEXT:  .LBB2_16: # %else14
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+; AVX1-NEXT:  .LBB2_1: # %cond.store
 ; AVX1-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX1-NEXT:  .LBB2_2: # %else
-; AVX1-NEXT:    vpcmpeqd %xmm8, %xmm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm9, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $4, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $2, %al
 ; AVX1-NEXT:    je .LBB2_4
-; AVX1-NEXT:  # %bb.3: # %cond.store1
-; AVX1-NEXT:    vpextrb $2, %xmm0, 1(%rdi)
-; AVX1-NEXT:  .LBB2_4: # %else2
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm2, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm4, %xmm4
-; AVX1-NEXT:    vpextrb $8, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:  .LBB2_3: # %cond.store1
+; AVX1-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
+; AVX1-NEXT:    testb $4, %al
 ; AVX1-NEXT:    je .LBB2_6
-; AVX1-NEXT:  # %bb.5: # %cond.store3
-; AVX1-NEXT:    vpextrb $4, %xmm0, 2(%rdi)
-; AVX1-NEXT:  .LBB2_6: # %else4
-; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:  .LBB2_5: # %cond.store3
+; AVX1-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
+; AVX1-NEXT:    testb $8, %al
 ; AVX1-NEXT:    je .LBB2_8
-; AVX1-NEXT:  # %bb.7: # %cond.store5
-; AVX1-NEXT:    vpextrb $6, %xmm0, 3(%rdi)
-; AVX1-NEXT:  .LBB2_8: # %else6
-; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $0, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:  .LBB2_7: # %cond.store5
+; AVX1-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
+; AVX1-NEXT:    testb $16, %al
 ; AVX1-NEXT:    je .LBB2_10
-; AVX1-NEXT:  # %bb.9: # %cond.store7
-; AVX1-NEXT:    vpextrb $8, %xmm0, 4(%rdi)
-; AVX1-NEXT:  .LBB2_10: # %else8
-; AVX1-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:  .LBB2_9: # %cond.store7
+; AVX1-NEXT:    vpextrb $4, %xmm0, 4(%rdi)
+; AVX1-NEXT:    testb $32, %al
 ; AVX1-NEXT:    je .LBB2_12
-; AVX1-NEXT:  # %bb.11: # %cond.store9
-; AVX1-NEXT:    vpextrb $10, %xmm0, 5(%rdi)
-; AVX1-NEXT:  .LBB2_12: # %else10
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:  .LBB2_11: # %cond.store9
+; AVX1-NEXT:    vpextrb $5, %xmm0, 5(%rdi)
+; AVX1-NEXT:    testb $64, %al
 ; AVX1-NEXT:    je .LBB2_14
-; AVX1-NEXT:  # %bb.13: # %cond.store11
-; AVX1-NEXT:    vpextrb $12, %xmm0, 6(%rdi)
-; AVX1-NEXT:  .LBB2_14: # %else12
-; AVX1-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:  .LBB2_13: # %cond.store11
+; AVX1-NEXT:    vpextrb $6, %xmm0, 6(%rdi)
+; AVX1-NEXT:    testb $-128, %al
 ; AVX1-NEXT:    je .LBB2_16
-; AVX1-NEXT:  # %bb.15: # %cond.store13
-; AVX1-NEXT:    vpextrb $14, %xmm0, 7(%rdi)
-; AVX1-NEXT:  .LBB2_16: # %else14
+; AVX1-NEXT:  .LBB2_15: # %cond.store13
+; AVX1-NEXT:    vpextrb $7, %xmm0, 7(%rdi)
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: truncstore_v8i64_v8i8:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm5
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm6 = [255,255,255,255]
-; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm7 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
-; AVX2-NEXT:    vpxor %ymm7, %ymm1, %ymm8
-; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm9 = [9223372036854776063,9223372036854776063,9223372036854776063,9223372036854776063]
-; AVX2-NEXT:    vpcmpgtq %ymm8, %ymm9, %ymm8
-; AVX2-NEXT:    vblendvpd %ymm8, %ymm1, %ymm6, %ymm1
-; AVX2-NEXT:    vpxor %ymm7, %ymm0, %ymm7
-; AVX2-NEXT:    vpcmpgtq %ymm7, %ymm9, %ymm7
-; AVX2-NEXT:    vblendvpd %ymm7, %ymm0, %ymm6, %ymm0
-; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpextrb $0, %xmm5, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je .LBB2_2
-; AVX2-NEXT:  # %bb.1: # %cond.store
+; AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm4 = [255,255,255,255]
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm5 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
+; AVX2-NEXT:    vpxor %ymm5, %ymm0, %ymm6
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm7 = [9223372036854776063,9223372036854776063,9223372036854776063,9223372036854776063]
+; AVX2-NEXT:    vpcmpgtq %ymm6, %ymm7, %ymm6
+; AVX2-NEXT:    vblendvpd %ymm6, %ymm0, %ymm4, %ymm0
+; AVX2-NEXT:    vpxor %ymm5, %ymm1, %ymm5
+; AVX2-NEXT:    vpcmpgtq %ymm5, %ymm7, %ymm5
+; AVX2-NEXT:    vblendvpd %ymm5, %ymm1, %ymm4, %ymm1
+; AVX2-NEXT:    vextractf128 $1, %ymm1, %xmm4
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm5 = <u,u,0,8,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX2-NEXT:    vpshufb %xmm5, %xmm4, %xmm4
+; AVX2-NEXT:    vpshufb %xmm5, %xmm1, %xmm1
+; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3]
+; AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm4
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm5 = <0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX2-NEXT:    vpshufb %xmm5, %xmm4, %xmm4
+; AVX2-NEXT:    vpshufb %xmm5, %xmm0, %xmm0
+; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
+; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3]
+; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm2, %ymm1
+; AVX2-NEXT:    vmovmskps %ymm1, %eax
+; AVX2-NEXT:    notl %eax
+; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    jne .LBB2_1
+; AVX2-NEXT:  # %bb.2: # %else
+; AVX2-NEXT:    testb $2, %al
+; AVX2-NEXT:    jne .LBB2_3
+; AVX2-NEXT:  .LBB2_4: # %else2
+; AVX2-NEXT:    testb $4, %al
+; AVX2-NEXT:    jne .LBB2_5
+; AVX2-NEXT:  .LBB2_6: # %else4
+; AVX2-NEXT:    testb $8, %al
+; AVX2-NEXT:    jne .LBB2_7
+; AVX2-NEXT:  .LBB2_8: # %else6
+; AVX2-NEXT:    testb $16, %al
+; AVX2-NEXT:    jne .LBB2_9
+; AVX2-NEXT:  .LBB2_10: # %else8
+; AVX2-NEXT:    testb $32, %al
+; AVX2-NEXT:    jne .LBB2_11
+; AVX2-NEXT:  .LBB2_12: # %else10
+; AVX2-NEXT:    testb $64, %al
+; AVX2-NEXT:    jne .LBB2_13
+; AVX2-NEXT:  .LBB2_14: # %else12
+; AVX2-NEXT:    testb $-128, %al
+; AVX2-NEXT:    jne .LBB2_15
+; AVX2-NEXT:  .LBB2_16: # %else14
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+; AVX2-NEXT:  .LBB2_1: # %cond.store
 ; AVX2-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX2-NEXT:  .LBB2_2: # %else
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm1
-; AVX2-NEXT:    vpxor %xmm4, %xmm1, %xmm1
-; AVX2-NEXT:    vpextrb $4, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $2, %al
 ; AVX2-NEXT:    je .LBB2_4
-; AVX2-NEXT:  # %bb.3: # %cond.store1
-; AVX2-NEXT:    vpextrb $2, %xmm0, 1(%rdi)
-; AVX2-NEXT:  .LBB2_4: # %else2
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm2, %xmm4
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    vpxor %xmm3, %xmm4, %xmm4
-; AVX2-NEXT:    vpextrb $8, %xmm4, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:  .LBB2_3: # %cond.store1
+; AVX2-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
+; AVX2-NEXT:    testb $4, %al
 ; AVX2-NEXT:    je .LBB2_6
-; AVX2-NEXT:  # %bb.5: # %cond.store3
-; AVX2-NEXT:    vpextrb $4, %xmm0, 2(%rdi)
-; AVX2-NEXT:  .LBB2_6: # %else4
-; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm2, %xmm1
-; AVX2-NEXT:    vpxor %xmm3, %xmm1, %xmm1
-; AVX2-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:  .LBB2_5: # %cond.store3
+; AVX2-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
+; AVX2-NEXT:    testb $8, %al
 ; AVX2-NEXT:    je .LBB2_8
-; AVX2-NEXT:  # %bb.7: # %cond.store5
-; AVX2-NEXT:    vpextrb $6, %xmm0, 3(%rdi)
-; AVX2-NEXT:  .LBB2_8: # %else6
-; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm1
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrb $0, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:  .LBB2_7: # %cond.store5
+; AVX2-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
+; AVX2-NEXT:    testb $16, %al
 ; AVX2-NEXT:    je .LBB2_10
-; AVX2-NEXT:  # %bb.9: # %cond.store7
-; AVX2-NEXT:    vpextrb $8, %xmm0, 4(%rdi)
-; AVX2-NEXT:  .LBB2_10: # %else8
-; AVX2-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:  .LBB2_9: # %cond.store7
+; AVX2-NEXT:    vpextrb $4, %xmm0, 4(%rdi)
+; AVX2-NEXT:    testb $32, %al
 ; AVX2-NEXT:    je .LBB2_12
-; AVX2-NEXT:  # %bb.11: # %cond.store9
-; AVX2-NEXT:    vpextrb $10, %xmm0, 5(%rdi)
-; AVX2-NEXT:  .LBB2_12: # %else10
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:  .LBB2_11: # %cond.store9
+; AVX2-NEXT:    vpextrb $5, %xmm0, 5(%rdi)
+; AVX2-NEXT:    testb $64, %al
 ; AVX2-NEXT:    je .LBB2_14
-; AVX2-NEXT:  # %bb.13: # %cond.store11
-; AVX2-NEXT:    vpextrb $12, %xmm0, 6(%rdi)
-; AVX2-NEXT:  .LBB2_14: # %else12
-; AVX2-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:  .LBB2_13: # %cond.store11
+; AVX2-NEXT:    vpextrb $6, %xmm0, 6(%rdi)
+; AVX2-NEXT:    testb $-128, %al
 ; AVX2-NEXT:    je .LBB2_16
-; AVX2-NEXT:  # %bb.15: # %cond.store13
-; AVX2-NEXT:    vpextrb $14, %xmm0, 7(%rdi)
-; AVX2-NEXT:  .LBB2_16: # %else14
+; AVX2-NEXT:  .LBB2_15: # %cond.store13
+; AVX2-NEXT:    vpextrb $7, %xmm0, 7(%rdi)
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
@@ -1268,72 +1215,66 @@ define void @truncstore_v8i64_v8i8(<8 x i64> %x, <8 x i8>* %p, <8 x i32> %mask)
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
 ; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    vpminuq {{.*}}(%rip){1to8}, %zmm0, %zmm0
-; AVX512F-NEXT:    vpmovqw %zmm0, %xmm0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB2_2
-; AVX512F-NEXT:  # %bb.1: # %cond.store
-; AVX512F-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX512F-NEXT:  .LBB2_2: # %else
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
+; AVX512F-NEXT:    vpmovusqb %zmm0, %xmm0
 ; AVX512F-NEXT:    kmovw %k0, %eax
 ; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB2_4
-; AVX512F-NEXT:  # %bb.3: # %cond.store1
-; AVX512F-NEXT:    vpextrb $2, %xmm0, 1(%rdi)
+; AVX512F-NEXT:    jne .LBB2_1
+; AVX512F-NEXT:  # %bb.2: # %else
+; AVX512F-NEXT:    testb $2, %al
+; AVX512F-NEXT:    jne .LBB2_3
 ; AVX512F-NEXT:  .LBB2_4: # %else2
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB2_6
-; AVX512F-NEXT:  # %bb.5: # %cond.store3
-; AVX512F-NEXT:    vpextrb $4, %xmm0, 2(%rdi)
+; AVX512F-NEXT:    testb $4, %al
+; AVX512F-NEXT:    jne .LBB2_5
 ; AVX512F-NEXT:  .LBB2_6: # %else4
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB2_8
-; AVX512F-NEXT:  # %bb.7: # %cond.store5
-; AVX512F-NEXT:    vpextrb $6, %xmm0, 3(%rdi)
+; AVX512F-NEXT:    testb $8, %al
+; AVX512F-NEXT:    jne .LBB2_7
 ; AVX512F-NEXT:  .LBB2_8: # %else6
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $4, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB2_10
-; AVX512F-NEXT:  # %bb.9: # %cond.store7
-; AVX512F-NEXT:    vpextrb $8, %xmm0, 4(%rdi)
+; AVX512F-NEXT:    testb $16, %al
+; AVX512F-NEXT:    jne .LBB2_9
 ; AVX512F-NEXT:  .LBB2_10: # %else8
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $5, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB2_12
-; AVX512F-NEXT:  # %bb.11: # %cond.store9
-; AVX512F-NEXT:    vpextrb $10, %xmm0, 5(%rdi)
+; AVX512F-NEXT:    testb $32, %al
+; AVX512F-NEXT:    jne .LBB2_11
 ; AVX512F-NEXT:  .LBB2_12: # %else10
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $6, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB2_14
-; AVX512F-NEXT:  # %bb.13: # %cond.store11
-; AVX512F-NEXT:    vpextrb $12, %xmm0, 6(%rdi)
+; AVX512F-NEXT:    testb $64, %al
+; AVX512F-NEXT:    jne .LBB2_13
 ; AVX512F-NEXT:  .LBB2_14: # %else12
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $7, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB2_16
-; AVX512F-NEXT:  # %bb.15: # %cond.store13
-; AVX512F-NEXT:    vpextrb $14, %xmm0, 7(%rdi)
+; AVX512F-NEXT:    testb $-128, %al
+; AVX512F-NEXT:    jne .LBB2_15
 ; AVX512F-NEXT:  .LBB2_16: # %else14
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
+; AVX512F-NEXT:  .LBB2_1: # %cond.store
+; AVX512F-NEXT:    vpextrb $0, %xmm0, (%rdi)
+; AVX512F-NEXT:    testb $2, %al
+; AVX512F-NEXT:    je .LBB2_4
+; AVX512F-NEXT:  .LBB2_3: # %cond.store1
+; AVX512F-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
+; AVX512F-NEXT:    testb $4, %al
+; AVX512F-NEXT:    je .LBB2_6
+; AVX512F-NEXT:  .LBB2_5: # %cond.store3
+; AVX512F-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
+; AVX512F-NEXT:    testb $8, %al
+; AVX512F-NEXT:    je .LBB2_8
+; AVX512F-NEXT:  .LBB2_7: # %cond.store5
+; AVX512F-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
+; AVX512F-NEXT:    testb $16, %al
+; AVX512F-NEXT:    je .LBB2_10
+; AVX512F-NEXT:  .LBB2_9: # %cond.store7
+; AVX512F-NEXT:    vpextrb $4, %xmm0, 4(%rdi)
+; AVX512F-NEXT:    testb $32, %al
+; AVX512F-NEXT:    je .LBB2_12
+; AVX512F-NEXT:  .LBB2_11: # %cond.store9
+; AVX512F-NEXT:    vpextrb $5, %xmm0, 5(%rdi)
+; AVX512F-NEXT:    testb $64, %al
+; AVX512F-NEXT:    je .LBB2_14
+; AVX512F-NEXT:  .LBB2_13: # %cond.store11
+; AVX512F-NEXT:    vpextrb $6, %xmm0, 6(%rdi)
+; AVX512F-NEXT:    testb $-128, %al
+; AVX512F-NEXT:    je .LBB2_16
+; AVX512F-NEXT:  .LBB2_15: # %cond.store13
+; AVX512F-NEXT:    vpextrb $7, %xmm0, 7(%rdi)
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: truncstore_v8i64_v8i8:
 ; AVX512BW:       # %bb.0:
@@ -1363,7 +1304,6 @@ define void @truncstore_v4i64_v4i32(<4 x i64> %x, <4 x i32>* %p, <4 x i32> %mask
 ; SSE2-LABEL: truncstore_v4i64_v4i32:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm3
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [4294967295,4294967295]
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [9223372039002259456,9223372039002259456]
 ; SSE2-NEXT:    movdqa %xmm1, %xmm6
@@ -1393,50 +1333,45 @@ define void @truncstore_v4i64_v4i32(<4 x i64> %x, <4 x i32>* %p, <4 x i32> %mask
 ; SSE2-NEXT:    pandn %xmm8, %xmm1
 ; SSE2-NEXT:    por %xmm0, %xmm1
 ; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,2],xmm4[0,2]
-; SSE2-NEXT:    movd %xmm3, %eax
-; SSE2-NEXT:    notl %eax
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm3
+; SSE2-NEXT:    movmskps %xmm3, %eax
+; SSE2-NEXT:    xorl $15, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB3_2
-; SSE2-NEXT:  # %bb.1: # %cond.store
+; SSE2-NEXT:    jne .LBB3_1
+; SSE2-NEXT:  # %bb.2: # %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne .LBB3_3
+; SSE2-NEXT:  .LBB3_4: # %else2
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne .LBB3_5
+; SSE2-NEXT:  .LBB3_6: # %else4
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne .LBB3_7
+; SSE2-NEXT:  .LBB3_8: # %else6
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  .LBB3_1: # %cond.store
 ; SSE2-NEXT:    movss %xmm1, (%rdi)
-; SSE2-NEXT:  .LBB3_2: # %else
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE2-NEXT:    pxor %xmm0, %xmm3
-; SSE2-NEXT:    pextrw $2, %xmm3, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $2, %al
 ; SSE2-NEXT:    je .LBB3_4
-; SSE2-NEXT:  # %bb.3: # %cond.store1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,1,2,3]
-; SSE2-NEXT:    movd %xmm3, 4(%rdi)
-; SSE2-NEXT:  .LBB3_4: # %else2
-; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm2
-; SSE2-NEXT:    pxor %xmm2, %xmm0
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:  .LBB3_3: # %cond.store1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE2-NEXT:    movd %xmm0, 4(%rdi)
+; SSE2-NEXT:    testb $4, %al
 ; SSE2-NEXT:    je .LBB3_6
-; SSE2-NEXT:  # %bb.5: # %cond.store3
+; SSE2-NEXT:  .LBB3_5: # %cond.store3
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
 ; SSE2-NEXT:    movd %xmm0, 8(%rdi)
-; SSE2-NEXT:  .LBB3_6: # %else4
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE2-NEXT:    pxor %xmm0, %xmm2
-; SSE2-NEXT:    pextrw $6, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je .LBB3_8
-; SSE2-NEXT:  # %bb.7: # %cond.store5
+; SSE2-NEXT:  .LBB3_7: # %cond.store5
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[3,1,2,3]
 ; SSE2-NEXT:    movd %xmm0, 12(%rdi)
-; SSE2-NEXT:  .LBB3_8: # %else6
 ; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: truncstore_v4i64_v4i32:
 ; SSE4:       # %bb.0:
 ; SSE4-NEXT:    movdqa %xmm0, %xmm8
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqd %xmm2, %xmm0
-; SSE4-NEXT:    pcmpeqd %xmm6, %xmm6
-; SSE4-NEXT:    pxor %xmm0, %xmm6
+; SSE4-NEXT:    pxor %xmm6, %xmm6
 ; SSE4-NEXT:    movapd {{.*#+}} xmm5 = [4294967295,4294967295]
 ; SSE4-NEXT:    movdqa {{.*#+}} xmm7 = [9223372036854775808,9223372036854775808]
 ; SSE4-NEXT:    movdqa %xmm1, %xmm3
@@ -1451,34 +1386,36 @@ define void @truncstore_v4i64_v4i32(<4 x i64> %x, <4 x i32>* %p, <4 x i32> %mask
 ; SSE4-NEXT:    movdqa %xmm4, %xmm0
 ; SSE4-NEXT:    blendvpd %xmm0, %xmm8, %xmm5
 ; SSE4-NEXT:    shufps {{.*#+}} xmm5 = xmm5[0,2],xmm3[0,2]
-; SSE4-NEXT:    pextrb $0, %xmm6, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB3_2
-; SSE4-NEXT:  # %bb.1: # %cond.store
+; SSE4-NEXT:    pcmpeqd %xmm2, %xmm6
+; SSE4-NEXT:    movmskps %xmm6, %eax
+; SSE4-NEXT:    xorl $15, %eax
+; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    jne .LBB3_1
+; SSE4-NEXT:  # %bb.2: # %else
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    jne .LBB3_3
+; SSE4-NEXT:  .LBB3_4: # %else2
+; SSE4-NEXT:    testb $4, %al
+; SSE4-NEXT:    jne .LBB3_5
+; SSE4-NEXT:  .LBB3_6: # %else4
+; SSE4-NEXT:    testb $8, %al
+; SSE4-NEXT:    jne .LBB3_7
+; SSE4-NEXT:  .LBB3_8: # %else6
+; SSE4-NEXT:    retq
+; SSE4-NEXT:  .LBB3_1: # %cond.store
 ; SSE4-NEXT:    movss %xmm5, (%rdi)
-; SSE4-NEXT:  .LBB3_2: # %else
-; SSE4-NEXT:    pextrb $4, %xmm6, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $2, %al
 ; SSE4-NEXT:    je .LBB3_4
-; SSE4-NEXT:  # %bb.3: # %cond.store1
+; SSE4-NEXT:  .LBB3_3: # %cond.store1
 ; SSE4-NEXT:    extractps $1, %xmm5, 4(%rdi)
-; SSE4-NEXT:  .LBB3_4: # %else2
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE4-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE4-NEXT:    pxor %xmm2, %xmm0
-; SSE4-NEXT:    pextrb $8, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $4, %al
 ; SSE4-NEXT:    je .LBB3_6
-; SSE4-NEXT:  # %bb.5: # %cond.store3
+; SSE4-NEXT:  .LBB3_5: # %cond.store3
 ; SSE4-NEXT:    extractps $2, %xmm5, 8(%rdi)
-; SSE4-NEXT:  .LBB3_6: # %else4
-; SSE4-NEXT:    pextrb $12, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $8, %al
 ; SSE4-NEXT:    je .LBB3_8
-; SSE4-NEXT:  # %bb.7: # %cond.store5
+; SSE4-NEXT:  .LBB3_7: # %cond.store5
 ; SSE4-NEXT:    extractps $3, %xmm5, 12(%rdi)
-; SSE4-NEXT:  .LBB3_8: # %else6
 ; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: truncstore_v4i64_v4i32:
@@ -1527,9 +1464,7 @@ define void @truncstore_v4i64_v4i32(<4 x i64> %x, <4 x i32>* %p, <4 x i32> %mask
 ; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
 ; AVX512F-NEXT:    kshiftlw $12, %k0, %k0
 ; AVX512F-NEXT:    kshiftrw $12, %k0, %k1
-; AVX512F-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [4294967295,4294967295,4294967295,4294967295]
-; AVX512F-NEXT:    vpminuq %zmm1, %zmm0, %zmm0
-; AVX512F-NEXT:    vpmovqd %zmm0, %ymm0
+; AVX512F-NEXT:    vpmovusqd %zmm0, %ymm0
 ; AVX512F-NEXT:    vmovdqu32 %zmm0, (%rdi) {%k1}
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
@@ -1549,9 +1484,7 @@ define void @truncstore_v4i64_v4i32(<4 x i64> %x, <4 x i32>* %p, <4 x i32> %mask
 ; AVX512BW-NEXT:    vptestmd %zmm1, %zmm1, %k0
 ; AVX512BW-NEXT:    kshiftlw $12, %k0, %k0
 ; AVX512BW-NEXT:    kshiftrw $12, %k0, %k1
-; AVX512BW-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [4294967295,4294967295,4294967295,4294967295]
-; AVX512BW-NEXT:    vpminuq %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpmovqd %zmm0, %ymm0
+; AVX512BW-NEXT:    vpmovusqd %zmm0, %ymm0
 ; AVX512BW-NEXT:    vmovdqu32 %zmm0, (%rdi) {%k1}
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -1567,10 +1500,9 @@ define void @truncstore_v4i64_v4i16(<4 x i64> %x, <4 x i16>* %p, <4 x i32> %mask
 ; SSE2-LABEL: truncstore_v4i64_v4i16:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm3
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [65535,65535]
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [9223372039002259456,9223372039002259456]
-; SSE2-NEXT:    movdqa %xmm1, %xmm6
+; SSE2-NEXT:    movdqa %xmm0, %xmm6
 ; SSE2-NEXT:    pxor %xmm5, %xmm6
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [9223372039002324991,9223372039002324991]
 ; SSE2-NEXT:    movdqa %xmm9, %xmm7
@@ -1581,117 +1513,119 @@ define void @truncstore_v4i64_v4i16(<4 x i64> %x, <4 x i16>* %p, <4 x i32> %mask
 ; SSE2-NEXT:    pand %xmm4, %xmm6
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm7[1,1,3,3]
 ; SSE2-NEXT:    por %xmm6, %xmm4
-; SSE2-NEXT:    pand %xmm4, %xmm1
+; SSE2-NEXT:    pand %xmm4, %xmm0
 ; SSE2-NEXT:    pandn %xmm8, %xmm4
-; SSE2-NEXT:    por %xmm1, %xmm4
-; SSE2-NEXT:    pxor %xmm0, %xmm5
-; SSE2-NEXT:    movdqa %xmm9, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm5, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[0,0,2,2]
+; SSE2-NEXT:    por %xmm0, %xmm4
+; SSE2-NEXT:    pxor %xmm1, %xmm5
+; SSE2-NEXT:    movdqa %xmm9, %xmm0
+; SSE2-NEXT:    pcmpgtd %xmm5, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm0[0,0,2,2]
 ; SSE2-NEXT:    pcmpeqd %xmm9, %xmm5
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm6, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm5, %xmm1
-; SSE2-NEXT:    pand %xmm1, %xmm0
-; SSE2-NEXT:    pandn %xmm8, %xmm1
-; SSE2-NEXT:    por %xmm0, %xmm1
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,2],xmm4[0,2]
-; SSE2-NEXT:    movd %xmm3, %eax
-; SSE2-NEXT:    notl %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB4_2
-; SSE2-NEXT:  # %bb.1: # %cond.store
-; SSE2-NEXT:    movd %xmm1, %eax
-; SSE2-NEXT:    movw %ax, (%rdi)
-; SSE2-NEXT:  .LBB4_2: # %else
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE2-NEXT:    pxor %xmm0, %xmm3
-; SSE2-NEXT:    pextrw $2, %xmm3, %eax
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    por %xmm5, %xmm0
+; SSE2-NEXT:    pand %xmm0, %xmm1
+; SSE2-NEXT:    pandn %xmm8, %xmm0
+; SSE2-NEXT:    por %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm0[0,2,2,3,4,5,6,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,2,2,3]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm3
+; SSE2-NEXT:    movmskps %xmm3, %eax
+; SSE2-NEXT:    xorl $15, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB4_4
-; SSE2-NEXT:  # %bb.3: # %cond.store1
-; SSE2-NEXT:    pextrw $2, %xmm1, %eax
-; SSE2-NEXT:    movw %ax, 2(%rdi)
+; SSE2-NEXT:    jne .LBB4_1
+; SSE2-NEXT:  # %bb.2: # %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne .LBB4_3
 ; SSE2-NEXT:  .LBB4_4: # %else2
-; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm2
-; SSE2-NEXT:    pxor %xmm2, %xmm0
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB4_6
-; SSE2-NEXT:  # %bb.5: # %cond.store3
-; SSE2-NEXT:    pextrw $4, %xmm1, %eax
-; SSE2-NEXT:    movw %ax, 4(%rdi)
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne .LBB4_5
 ; SSE2-NEXT:  .LBB4_6: # %else4
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE2-NEXT:    pxor %xmm0, %xmm2
-; SSE2-NEXT:    pextrw $6, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne .LBB4_7
+; SSE2-NEXT:  .LBB4_8: # %else6
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  .LBB4_1: # %cond.store
+; SSE2-NEXT:    movd %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, (%rdi)
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    je .LBB4_4
+; SSE2-NEXT:  .LBB4_3: # %cond.store1
+; SSE2-NEXT:    pextrw $1, %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, 2(%rdi)
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    je .LBB4_6
+; SSE2-NEXT:  .LBB4_5: # %cond.store3
+; SSE2-NEXT:    pextrw $2, %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, 4(%rdi)
+; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je .LBB4_8
-; SSE2-NEXT:  # %bb.7: # %cond.store5
-; SSE2-NEXT:    pextrw $6, %xmm1, %eax
+; SSE2-NEXT:  .LBB4_7: # %cond.store5
+; SSE2-NEXT:    pextrw $3, %xmm0, %eax
 ; SSE2-NEXT:    movw %ax, 6(%rdi)
-; SSE2-NEXT:  .LBB4_8: # %else6
 ; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: truncstore_v4i64_v4i16:
 ; SSE4:       # %bb.0:
-; SSE4-NEXT:    movdqa %xmm0, %xmm8
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqd %xmm2, %xmm0
-; SSE4-NEXT:    pcmpeqd %xmm6, %xmm6
-; SSE4-NEXT:    pxor %xmm0, %xmm6
-; SSE4-NEXT:    movapd {{.*#+}} xmm5 = [65535,65535]
+; SSE4-NEXT:    movdqa %xmm0, %xmm5
+; SSE4-NEXT:    pxor %xmm8, %xmm8
+; SSE4-NEXT:    movapd {{.*#+}} xmm6 = [65535,65535]
 ; SSE4-NEXT:    movdqa {{.*#+}} xmm7 = [9223372036854775808,9223372036854775808]
-; SSE4-NEXT:    movdqa %xmm1, %xmm3
+; SSE4-NEXT:    movdqa %xmm0, %xmm3
 ; SSE4-NEXT:    pxor %xmm7, %xmm3
 ; SSE4-NEXT:    movdqa {{.*#+}} xmm4 = [9223372036854841343,9223372036854841343]
 ; SSE4-NEXT:    movdqa %xmm4, %xmm0
 ; SSE4-NEXT:    pcmpgtq %xmm3, %xmm0
-; SSE4-NEXT:    movapd %xmm5, %xmm3
-; SSE4-NEXT:    blendvpd %xmm0, %xmm1, %xmm3
-; SSE4-NEXT:    pxor %xmm8, %xmm7
+; SSE4-NEXT:    movapd %xmm6, %xmm3
+; SSE4-NEXT:    blendvpd %xmm0, %xmm5, %xmm3
+; SSE4-NEXT:    pxor %xmm1, %xmm7
 ; SSE4-NEXT:    pcmpgtq %xmm7, %xmm4
 ; SSE4-NEXT:    movdqa %xmm4, %xmm0
-; SSE4-NEXT:    blendvpd %xmm0, %xmm8, %xmm5
-; SSE4-NEXT:    packusdw %xmm3, %xmm5
-; SSE4-NEXT:    pextrb $0, %xmm6, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB4_2
-; SSE4-NEXT:  # %bb.1: # %cond.store
-; SSE4-NEXT:    pextrw $0, %xmm5, (%rdi)
-; SSE4-NEXT:  .LBB4_2: # %else
-; SSE4-NEXT:    pextrb $4, %xmm6, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB4_4
-; SSE4-NEXT:  # %bb.3: # %cond.store1
-; SSE4-NEXT:    pextrw $2, %xmm5, 2(%rdi)
+; SSE4-NEXT:    blendvpd %xmm0, %xmm1, %xmm6
+; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,2,2,3]
+; SSE4-NEXT:    pshuflw {{.*#+}} xmm1 = xmm0[0,2,2,3,4,5,6,7]
+; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,2,2,3]
+; SSE4-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; SSE4-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE4-NEXT:    pcmpeqd %xmm2, %xmm8
+; SSE4-NEXT:    movmskps %xmm8, %eax
+; SSE4-NEXT:    xorl $15, %eax
+; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    jne .LBB4_1
+; SSE4-NEXT:  # %bb.2: # %else
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    jne .LBB4_3
 ; SSE4-NEXT:  .LBB4_4: # %else2
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE4-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE4-NEXT:    pxor %xmm2, %xmm0
-; SSE4-NEXT:    pextrb $8, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB4_6
-; SSE4-NEXT:  # %bb.5: # %cond.store3
-; SSE4-NEXT:    pextrw $4, %xmm5, 4(%rdi)
+; SSE4-NEXT:    testb $4, %al
+; SSE4-NEXT:    jne .LBB4_5
 ; SSE4-NEXT:  .LBB4_6: # %else4
-; SSE4-NEXT:    pextrb $12, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB4_8
-; SSE4-NEXT:  # %bb.7: # %cond.store5
-; SSE4-NEXT:    pextrw $6, %xmm5, 6(%rdi)
+; SSE4-NEXT:    testb $8, %al
+; SSE4-NEXT:    jne .LBB4_7
 ; SSE4-NEXT:  .LBB4_8: # %else6
 ; SSE4-NEXT:    retq
+; SSE4-NEXT:  .LBB4_1: # %cond.store
+; SSE4-NEXT:    pextrw $0, %xmm0, (%rdi)
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    je .LBB4_4
+; SSE4-NEXT:  .LBB4_3: # %cond.store1
+; SSE4-NEXT:    pextrw $1, %xmm0, 2(%rdi)
+; SSE4-NEXT:    testb $4, %al
+; SSE4-NEXT:    je .LBB4_6
+; SSE4-NEXT:  .LBB4_5: # %cond.store3
+; SSE4-NEXT:    pextrw $2, %xmm0, 4(%rdi)
+; SSE4-NEXT:    testb $8, %al
+; SSE4-NEXT:    je .LBB4_8
+; SSE4-NEXT:  .LBB4_7: # %cond.store5
+; SSE4-NEXT:    pextrw $3, %xmm0, 6(%rdi)
+; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: truncstore_v4i64_v4i16:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808]
 ; AVX1-NEXT:    vpxor %xmm3, %xmm0, %xmm4
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm5 = [9223372036854841343,9223372036854841343]
@@ -1701,45 +1635,49 @@ define void @truncstore_v4i64_v4i16(<4 x i64> %x, <4 x i16>* %p, <4 x i32> %mask
 ; AVX1-NEXT:    vpcmpgtq %xmm3, %xmm5, %xmm3
 ; AVX1-NEXT:    vmovapd {{.*#+}} xmm5 = [65535,65535]
 ; AVX1-NEXT:    vblendvpd %xmm3, %xmm6, %xmm5, %xmm3
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm3 = xmm3[0,2,2,3]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm3[0,2,2,3,4,5,6,7]
 ; AVX1-NEXT:    vblendvpd %xmm4, %xmm0, %xmm5, %xmm0
-; AVX1-NEXT:    vpackusdw %xmm3, %xmm0, %xmm0
-; AVX1-NEXT:    vpextrb $0, %xmm2, %eax
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vmovmskps %xmm1, %eax
+; AVX1-NEXT:    xorl $15, %eax
 ; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je .LBB4_2
-; AVX1-NEXT:  # %bb.1: # %cond.store
+; AVX1-NEXT:    jne .LBB4_1
+; AVX1-NEXT:  # %bb.2: # %else
+; AVX1-NEXT:    testb $2, %al
+; AVX1-NEXT:    jne .LBB4_3
+; AVX1-NEXT:  .LBB4_4: # %else2
+; AVX1-NEXT:    testb $4, %al
+; AVX1-NEXT:    jne .LBB4_5
+; AVX1-NEXT:  .LBB4_6: # %else4
+; AVX1-NEXT:    testb $8, %al
+; AVX1-NEXT:    jne .LBB4_7
+; AVX1-NEXT:  .LBB4_8: # %else6
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+; AVX1-NEXT:  .LBB4_1: # %cond.store
 ; AVX1-NEXT:    vpextrw $0, %xmm0, (%rdi)
-; AVX1-NEXT:  .LBB4_2: # %else
-; AVX1-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $2, %al
 ; AVX1-NEXT:    je .LBB4_4
-; AVX1-NEXT:  # %bb.3: # %cond.store1
-; AVX1-NEXT:    vpextrw $2, %xmm0, 2(%rdi)
-; AVX1-NEXT:  .LBB4_4: # %else2
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:  .LBB4_3: # %cond.store1
+; AVX1-NEXT:    vpextrw $1, %xmm0, 2(%rdi)
+; AVX1-NEXT:    testb $4, %al
 ; AVX1-NEXT:    je .LBB4_6
-; AVX1-NEXT:  # %bb.5: # %cond.store3
-; AVX1-NEXT:    vpextrw $4, %xmm0, 4(%rdi)
-; AVX1-NEXT:  .LBB4_6: # %else4
-; AVX1-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:  .LBB4_5: # %cond.store3
+; AVX1-NEXT:    vpextrw $2, %xmm0, 4(%rdi)
+; AVX1-NEXT:    testb $8, %al
 ; AVX1-NEXT:    je .LBB4_8
-; AVX1-NEXT:  # %bb.7: # %cond.store5
-; AVX1-NEXT:    vpextrw $6, %xmm0, 6(%rdi)
-; AVX1-NEXT:  .LBB4_8: # %else6
+; AVX1-NEXT:  .LBB4_7: # %cond.store5
+; AVX1-NEXT:    vpextrw $3, %xmm0, 6(%rdi)
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: truncstore_v4i64_v4i16:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    vpxor %xmm3, %xmm2, %xmm2
 ; AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm3 = [65535,65535,65535,65535]
 ; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
 ; AVX2-NEXT:    vpxor %ymm4, %ymm0, %ymm4
@@ -1747,35 +1685,42 @@ define void @truncstore_v4i64_v4i16(<4 x i64> %x, <4 x i16>* %p, <4 x i32> %mask
 ; AVX2-NEXT:    vpcmpgtq %ymm4, %ymm5, %ymm4
 ; AVX2-NEXT:    vblendvpd %ymm4, %ymm0, %ymm3, %ymm0
 ; AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm3
-; AVX2-NEXT:    vpackusdw %xmm3, %xmm0, %xmm0
-; AVX2-NEXT:    vpextrb $0, %xmm2, %eax
+; AVX2-NEXT:    vpermilps {{.*#+}} xmm3 = xmm3[0,2,2,3]
+; AVX2-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm3[0,2,2,3,4,5,6,7]
+; AVX2-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX2-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vmovmskps %xmm1, %eax
+; AVX2-NEXT:    xorl $15, %eax
 ; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je .LBB4_2
-; AVX2-NEXT:  # %bb.1: # %cond.store
+; AVX2-NEXT:    jne .LBB4_1
+; AVX2-NEXT:  # %bb.2: # %else
+; AVX2-NEXT:    testb $2, %al
+; AVX2-NEXT:    jne .LBB4_3
+; AVX2-NEXT:  .LBB4_4: # %else2
+; AVX2-NEXT:    testb $4, %al
+; AVX2-NEXT:    jne .LBB4_5
+; AVX2-NEXT:  .LBB4_6: # %else4
+; AVX2-NEXT:    testb $8, %al
+; AVX2-NEXT:    jne .LBB4_7
+; AVX2-NEXT:  .LBB4_8: # %else6
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+; AVX2-NEXT:  .LBB4_1: # %cond.store
 ; AVX2-NEXT:    vpextrw $0, %xmm0, (%rdi)
-; AVX2-NEXT:  .LBB4_2: # %else
-; AVX2-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $2, %al
 ; AVX2-NEXT:    je .LBB4_4
-; AVX2-NEXT:  # %bb.3: # %cond.store1
-; AVX2-NEXT:    vpextrw $2, %xmm0, 2(%rdi)
-; AVX2-NEXT:  .LBB4_4: # %else2
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:  .LBB4_3: # %cond.store1
+; AVX2-NEXT:    vpextrw $1, %xmm0, 2(%rdi)
+; AVX2-NEXT:    testb $4, %al
 ; AVX2-NEXT:    je .LBB4_6
-; AVX2-NEXT:  # %bb.5: # %cond.store3
-; AVX2-NEXT:    vpextrw $4, %xmm0, 4(%rdi)
-; AVX2-NEXT:  .LBB4_6: # %else4
-; AVX2-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:  .LBB4_5: # %cond.store3
+; AVX2-NEXT:    vpextrw $2, %xmm0, 4(%rdi)
+; AVX2-NEXT:    testb $8, %al
 ; AVX2-NEXT:    je .LBB4_8
-; AVX2-NEXT:  # %bb.7: # %cond.store5
-; AVX2-NEXT:    vpextrw $6, %xmm0, 6(%rdi)
-; AVX2-NEXT:  .LBB4_8: # %else6
+; AVX2-NEXT:  .LBB4_7: # %cond.store5
+; AVX2-NEXT:    vpextrw $3, %xmm0, 6(%rdi)
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
@@ -1784,39 +1729,36 @@ define void @truncstore_v4i64_v4i16(<4 x i64> %x, <4 x i16>* %p, <4 x i32> %mask
 ; AVX512F-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
 ; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
 ; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [65535,65535,65535,65535]
-; AVX512F-NEXT:    vpminuq %zmm2, %zmm0, %zmm0
-; AVX512F-NEXT:    vpmovqd %zmm0, %ymm0
+; AVX512F-NEXT:    vpmovusqw %zmm0, %xmm0
 ; AVX512F-NEXT:    kmovw %k0, %eax
 ; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB4_2
-; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    jne .LBB4_1
+; AVX512F-NEXT:  # %bb.2: # %else
+; AVX512F-NEXT:    testb $2, %al
+; AVX512F-NEXT:    jne .LBB4_3
+; AVX512F-NEXT:  .LBB4_4: # %else2
+; AVX512F-NEXT:    testb $4, %al
+; AVX512F-NEXT:    jne .LBB4_5
+; AVX512F-NEXT:  .LBB4_6: # %else4
+; AVX512F-NEXT:    testb $8, %al
+; AVX512F-NEXT:    jne .LBB4_7
+; AVX512F-NEXT:  .LBB4_8: # %else6
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+; AVX512F-NEXT:  .LBB4_1: # %cond.store
 ; AVX512F-NEXT:    vpextrw $0, %xmm0, (%rdi)
-; AVX512F-NEXT:  .LBB4_2: # %else
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $2, %al
 ; AVX512F-NEXT:    je .LBB4_4
-; AVX512F-NEXT:  # %bb.3: # %cond.store1
-; AVX512F-NEXT:    vpextrw $2, %xmm0, 2(%rdi)
-; AVX512F-NEXT:  .LBB4_4: # %else2
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  .LBB4_3: # %cond.store1
+; AVX512F-NEXT:    vpextrw $1, %xmm0, 2(%rdi)
+; AVX512F-NEXT:    testb $4, %al
 ; AVX512F-NEXT:    je .LBB4_6
-; AVX512F-NEXT:  # %bb.5: # %cond.store3
-; AVX512F-NEXT:    vpextrw $4, %xmm0, 4(%rdi)
-; AVX512F-NEXT:  .LBB4_6: # %else4
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  .LBB4_5: # %cond.store3
+; AVX512F-NEXT:    vpextrw $2, %xmm0, 4(%rdi)
+; AVX512F-NEXT:    testb $8, %al
 ; AVX512F-NEXT:    je .LBB4_8
-; AVX512F-NEXT:  # %bb.7: # %cond.store5
-; AVX512F-NEXT:    vpextrw $6, %xmm0, 6(%rdi)
-; AVX512F-NEXT:  .LBB4_8: # %else6
+; AVX512F-NEXT:  .LBB4_7: # %cond.store5
+; AVX512F-NEXT:    vpextrw $3, %xmm0, 6(%rdi)
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
@@ -1825,12 +1767,9 @@ define void @truncstore_v4i64_v4i16(<4 x i64> %x, <4 x i16>* %p, <4 x i32> %mask
 ; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
 ; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
 ; AVX512BW-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512BW-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [65535,65535,65535,65535]
-; AVX512BW-NEXT:    vpminuq %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpmovqd %zmm0, %ymm0
-; AVX512BW-NEXT:    vpackusdw %xmm0, %xmm0, %xmm0
 ; AVX512BW-NEXT:    kshiftld $28, %k0, %k0
 ; AVX512BW-NEXT:    kshiftrd $28, %k0, %k1
+; AVX512BW-NEXT:    vpmovusqw %zmm0, %xmm0
 ; AVX512BW-NEXT:    vmovdqu16 %zmm0, (%rdi) {%k1}
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -1853,132 +1792,132 @@ define void @truncstore_v4i64_v4i16(<4 x i64> %x, <4 x i16>* %p, <4 x i32> %mask
 define void @truncstore_v4i64_v4i8(<4 x i64> %x, <4 x i8>* %p, <4 x i32> %mask) {
 ; SSE2-LABEL: truncstore_v4i64_v4i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm3
+; SSE2-NEXT:    pxor %xmm9, %xmm9
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [255,255]
-; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [9223372039002259456,9223372039002259456]
-; SSE2-NEXT:    movdqa %xmm1, %xmm6
-; SSE2-NEXT:    pxor %xmm5, %xmm6
-; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [9223372039002259711,9223372039002259711]
-; SSE2-NEXT:    movdqa %xmm9, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm6, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm7[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT:    pand %xmm4, %xmm6
+; SSE2-NEXT:    movdqa {{.*#+}} xmm6 = [9223372039002259456,9223372039002259456]
+; SSE2-NEXT:    movdqa %xmm0, %xmm4
+; SSE2-NEXT:    pxor %xmm6, %xmm4
+; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [9223372039002259711,9223372039002259711]
+; SSE2-NEXT:    movdqa %xmm10, %xmm7
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm7
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm7[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[1,1,3,3]
+; SSE2-NEXT:    pand %xmm3, %xmm5
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm7[1,1,3,3]
-; SSE2-NEXT:    por %xmm6, %xmm4
-; SSE2-NEXT:    pand %xmm4, %xmm1
+; SSE2-NEXT:    por %xmm5, %xmm4
+; SSE2-NEXT:    pand %xmm4, %xmm0
 ; SSE2-NEXT:    pandn %xmm8, %xmm4
-; SSE2-NEXT:    por %xmm1, %xmm4
-; SSE2-NEXT:    pxor %xmm0, %xmm5
-; SSE2-NEXT:    movdqa %xmm9, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm5, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm5, %xmm1
-; SSE2-NEXT:    pand %xmm1, %xmm0
-; SSE2-NEXT:    pandn %xmm8, %xmm1
-; SSE2-NEXT:    por %xmm0, %xmm1
-; SSE2-NEXT:    packuswb %xmm4, %xmm1
-; SSE2-NEXT:    movd %xmm3, %eax
-; SSE2-NEXT:    notl %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB5_2
-; SSE2-NEXT:  # %bb.1: # %cond.store
-; SSE2-NEXT:    movd %xmm1, %eax
+; SSE2-NEXT:    por %xmm0, %xmm4
+; SSE2-NEXT:    pxor %xmm1, %xmm6
+; SSE2-NEXT:    movdqa %xmm10, %xmm0
+; SSE2-NEXT:    pcmpgtd %xmm6, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm6[1,1,3,3]
+; SSE2-NEXT:    pand %xmm3, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    por %xmm5, %xmm0
+; SSE2-NEXT:    pand %xmm0, %xmm1
+; SSE2-NEXT:    pandn %xmm8, %xmm0
+; SSE2-NEXT:    por %xmm1, %xmm0
+; SSE2-NEXT:    pand %xmm8, %xmm0
+; SSE2-NEXT:    pand %xmm8, %xmm4
+; SSE2-NEXT:    packuswb %xmm0, %xmm4
+; SSE2-NEXT:    packuswb %xmm4, %xmm4
+; SSE2-NEXT:    packuswb %xmm4, %xmm4
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm9
+; SSE2-NEXT:    movmskps %xmm9, %ecx
+; SSE2-NEXT:    xorl $15, %ecx
+; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    movd %xmm4, %eax
+; SSE2-NEXT:    jne .LBB5_1
+; SSE2-NEXT:  # %bb.2: # %else
+; SSE2-NEXT:    testb $2, %cl
+; SSE2-NEXT:    jne .LBB5_3
+; SSE2-NEXT:  .LBB5_4: # %else2
+; SSE2-NEXT:    testb $4, %cl
+; SSE2-NEXT:    jne .LBB5_5
+; SSE2-NEXT:  .LBB5_6: # %else4
+; SSE2-NEXT:    testb $8, %cl
+; SSE2-NEXT:    jne .LBB5_7
+; SSE2-NEXT:  .LBB5_8: # %else6
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  .LBB5_1: # %cond.store
 ; SSE2-NEXT:    movb %al, (%rdi)
-; SSE2-NEXT:  .LBB5_2: # %else
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE2-NEXT:    pxor %xmm0, %xmm3
-; SSE2-NEXT:    pextrw $2, %xmm3, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $2, %cl
 ; SSE2-NEXT:    je .LBB5_4
-; SSE2-NEXT:  # %bb.3: # %cond.store1
-; SSE2-NEXT:    pextrw $2, %xmm1, %eax
-; SSE2-NEXT:    movb %al, 1(%rdi)
-; SSE2-NEXT:  .LBB5_4: # %else2
-; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm2
-; SSE2-NEXT:    pxor %xmm2, %xmm0
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:  .LBB5_3: # %cond.store1
+; SSE2-NEXT:    movb %ah, 1(%rdi)
+; SSE2-NEXT:    testb $4, %cl
 ; SSE2-NEXT:    je .LBB5_6
-; SSE2-NEXT:  # %bb.5: # %cond.store3
-; SSE2-NEXT:    pextrw $4, %xmm1, %eax
-; SSE2-NEXT:    movb %al, 2(%rdi)
-; SSE2-NEXT:  .LBB5_6: # %else4
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE2-NEXT:    pxor %xmm0, %xmm2
-; SSE2-NEXT:    pextrw $6, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:  .LBB5_5: # %cond.store3
+; SSE2-NEXT:    movl %eax, %edx
+; SSE2-NEXT:    shrl $16, %edx
+; SSE2-NEXT:    movb %dl, 2(%rdi)
+; SSE2-NEXT:    testb $8, %cl
 ; SSE2-NEXT:    je .LBB5_8
-; SSE2-NEXT:  # %bb.7: # %cond.store5
-; SSE2-NEXT:    pextrw $6, %xmm1, %eax
+; SSE2-NEXT:  .LBB5_7: # %cond.store5
+; SSE2-NEXT:    shrl $24, %eax
 ; SSE2-NEXT:    movb %al, 3(%rdi)
-; SSE2-NEXT:  .LBB5_8: # %else6
 ; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: truncstore_v4i64_v4i8:
 ; SSE4:       # %bb.0:
-; SSE4-NEXT:    movdqa %xmm0, %xmm8
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqd %xmm2, %xmm0
-; SSE4-NEXT:    pcmpeqd %xmm6, %xmm6
-; SSE4-NEXT:    pxor %xmm0, %xmm6
-; SSE4-NEXT:    movapd {{.*#+}} xmm5 = [255,255]
-; SSE4-NEXT:    movdqa {{.*#+}} xmm7 = [9223372036854775808,9223372036854775808]
-; SSE4-NEXT:    movdqa %xmm1, %xmm3
-; SSE4-NEXT:    pxor %xmm7, %xmm3
+; SSE4-NEXT:    movdqa %xmm0, %xmm3
+; SSE4-NEXT:    pxor %xmm8, %xmm8
+; SSE4-NEXT:    movapd {{.*#+}} xmm7 = [255,255]
+; SSE4-NEXT:    movdqa {{.*#+}} xmm6 = [9223372036854775808,9223372036854775808]
+; SSE4-NEXT:    movdqa %xmm0, %xmm5
+; SSE4-NEXT:    pxor %xmm6, %xmm5
 ; SSE4-NEXT:    movdqa {{.*#+}} xmm4 = [9223372036854776063,9223372036854776063]
 ; SSE4-NEXT:    movdqa %xmm4, %xmm0
-; SSE4-NEXT:    pcmpgtq %xmm3, %xmm0
-; SSE4-NEXT:    movapd %xmm5, %xmm3
-; SSE4-NEXT:    blendvpd %xmm0, %xmm1, %xmm3
-; SSE4-NEXT:    pxor %xmm8, %xmm7
-; SSE4-NEXT:    pcmpgtq %xmm7, %xmm4
+; SSE4-NEXT:    pcmpgtq %xmm5, %xmm0
+; SSE4-NEXT:    movapd %xmm7, %xmm5
+; SSE4-NEXT:    blendvpd %xmm0, %xmm3, %xmm5
+; SSE4-NEXT:    pxor %xmm1, %xmm6
+; SSE4-NEXT:    pcmpgtq %xmm6, %xmm4
 ; SSE4-NEXT:    movdqa %xmm4, %xmm0
-; SSE4-NEXT:    blendvpd %xmm0, %xmm8, %xmm5
-; SSE4-NEXT:    packusdw %xmm3, %xmm5
-; SSE4-NEXT:    pextrb $0, %xmm6, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB5_2
-; SSE4-NEXT:  # %bb.1: # %cond.store
+; SSE4-NEXT:    blendvpd %xmm0, %xmm1, %xmm7
+; SSE4-NEXT:    movdqa {{.*#+}} xmm0 = <0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; SSE4-NEXT:    pshufb %xmm0, %xmm7
+; SSE4-NEXT:    pshufb %xmm0, %xmm5
+; SSE4-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm7[0],xmm5[1],xmm7[1],xmm5[2],xmm7[2],xmm5[3],xmm7[3]
+; SSE4-NEXT:    pcmpeqd %xmm2, %xmm8
+; SSE4-NEXT:    movmskps %xmm8, %eax
+; SSE4-NEXT:    xorl $15, %eax
+; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    jne .LBB5_1
+; SSE4-NEXT:  # %bb.2: # %else
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    jne .LBB5_3
+; SSE4-NEXT:  .LBB5_4: # %else2
+; SSE4-NEXT:    testb $4, %al
+; SSE4-NEXT:    jne .LBB5_5
+; SSE4-NEXT:  .LBB5_6: # %else4
+; SSE4-NEXT:    testb $8, %al
+; SSE4-NEXT:    jne .LBB5_7
+; SSE4-NEXT:  .LBB5_8: # %else6
+; SSE4-NEXT:    retq
+; SSE4-NEXT:  .LBB5_1: # %cond.store
 ; SSE4-NEXT:    pextrb $0, %xmm5, (%rdi)
-; SSE4-NEXT:  .LBB5_2: # %else
-; SSE4-NEXT:    pextrb $4, %xmm6, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $2, %al
 ; SSE4-NEXT:    je .LBB5_4
-; SSE4-NEXT:  # %bb.3: # %cond.store1
-; SSE4-NEXT:    pextrb $4, %xmm5, 1(%rdi)
-; SSE4-NEXT:  .LBB5_4: # %else2
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE4-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE4-NEXT:    pxor %xmm2, %xmm0
-; SSE4-NEXT:    pextrb $8, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:  .LBB5_3: # %cond.store1
+; SSE4-NEXT:    pextrb $1, %xmm5, 1(%rdi)
+; SSE4-NEXT:    testb $4, %al
 ; SSE4-NEXT:    je .LBB5_6
-; SSE4-NEXT:  # %bb.5: # %cond.store3
-; SSE4-NEXT:    pextrb $8, %xmm5, 2(%rdi)
-; SSE4-NEXT:  .LBB5_6: # %else4
-; SSE4-NEXT:    pextrb $12, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:  .LBB5_5: # %cond.store3
+; SSE4-NEXT:    pextrb $2, %xmm5, 2(%rdi)
+; SSE4-NEXT:    testb $8, %al
 ; SSE4-NEXT:    je .LBB5_8
-; SSE4-NEXT:  # %bb.7: # %cond.store5
-; SSE4-NEXT:    pextrb $12, %xmm5, 3(%rdi)
-; SSE4-NEXT:  .LBB5_8: # %else6
+; SSE4-NEXT:  .LBB5_7: # %cond.store5
+; SSE4-NEXT:    pextrb $3, %xmm5, 3(%rdi)
 ; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: truncstore_v4i64_v4i8:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808]
 ; AVX1-NEXT:    vpxor %xmm3, %xmm0, %xmm4
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm5 = [9223372036854776063,9223372036854776063]
@@ -1988,45 +1927,48 @@ define void @truncstore_v4i64_v4i8(<4 x i64> %x, <4 x i8>* %p, <4 x i32> %mask)
 ; AVX1-NEXT:    vpcmpgtq %xmm3, %xmm5, %xmm3
 ; AVX1-NEXT:    vmovapd {{.*#+}} xmm5 = [255,255]
 ; AVX1-NEXT:    vblendvpd %xmm3, %xmm6, %xmm5, %xmm3
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = <0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX1-NEXT:    vpshufb %xmm6, %xmm3, %xmm3
 ; AVX1-NEXT:    vblendvpd %xmm4, %xmm0, %xmm5, %xmm0
-; AVX1-NEXT:    vpackusdw %xmm3, %xmm0, %xmm0
-; AVX1-NEXT:    vpextrb $0, %xmm2, %eax
+; AVX1-NEXT:    vpshufb %xmm6, %xmm0, %xmm0
+; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vmovmskps %xmm1, %eax
+; AVX1-NEXT:    xorl $15, %eax
 ; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je .LBB5_2
-; AVX1-NEXT:  # %bb.1: # %cond.store
+; AVX1-NEXT:    jne .LBB5_1
+; AVX1-NEXT:  # %bb.2: # %else
+; AVX1-NEXT:    testb $2, %al
+; AVX1-NEXT:    jne .LBB5_3
+; AVX1-NEXT:  .LBB5_4: # %else2
+; AVX1-NEXT:    testb $4, %al
+; AVX1-NEXT:    jne .LBB5_5
+; AVX1-NEXT:  .LBB5_6: # %else4
+; AVX1-NEXT:    testb $8, %al
+; AVX1-NEXT:    jne .LBB5_7
+; AVX1-NEXT:  .LBB5_8: # %else6
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+; AVX1-NEXT:  .LBB5_1: # %cond.store
 ; AVX1-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX1-NEXT:  .LBB5_2: # %else
-; AVX1-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $2, %al
 ; AVX1-NEXT:    je .LBB5_4
-; AVX1-NEXT:  # %bb.3: # %cond.store1
-; AVX1-NEXT:    vpextrb $4, %xmm0, 1(%rdi)
-; AVX1-NEXT:  .LBB5_4: # %else2
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:  .LBB5_3: # %cond.store1
+; AVX1-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
+; AVX1-NEXT:    testb $4, %al
 ; AVX1-NEXT:    je .LBB5_6
-; AVX1-NEXT:  # %bb.5: # %cond.store3
-; AVX1-NEXT:    vpextrb $8, %xmm0, 2(%rdi)
-; AVX1-NEXT:  .LBB5_6: # %else4
-; AVX1-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:  .LBB5_5: # %cond.store3
+; AVX1-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
+; AVX1-NEXT:    testb $8, %al
 ; AVX1-NEXT:    je .LBB5_8
-; AVX1-NEXT:  # %bb.7: # %cond.store5
-; AVX1-NEXT:    vpextrb $12, %xmm0, 3(%rdi)
-; AVX1-NEXT:  .LBB5_8: # %else6
+; AVX1-NEXT:  .LBB5_7: # %cond.store5
+; AVX1-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: truncstore_v4i64_v4i8:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    vpxor %xmm3, %xmm2, %xmm2
 ; AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm3 = [255,255,255,255]
 ; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
 ; AVX2-NEXT:    vpxor %ymm4, %ymm0, %ymm4
@@ -2034,35 +1976,41 @@ define void @truncstore_v4i64_v4i8(<4 x i64> %x, <4 x i8>* %p, <4 x i32> %mask)
 ; AVX2-NEXT:    vpcmpgtq %ymm4, %ymm5, %ymm4
 ; AVX2-NEXT:    vblendvpd %ymm4, %ymm0, %ymm3, %ymm0
 ; AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm3
-; AVX2-NEXT:    vpackusdw %xmm3, %xmm0, %xmm0
-; AVX2-NEXT:    vpextrb $0, %xmm2, %eax
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm4 = <0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX2-NEXT:    vpshufb %xmm4, %xmm3, %xmm3
+; AVX2-NEXT:    vpshufb %xmm4, %xmm0, %xmm0
+; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vmovmskps %xmm1, %eax
+; AVX2-NEXT:    xorl $15, %eax
 ; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je .LBB5_2
-; AVX2-NEXT:  # %bb.1: # %cond.store
+; AVX2-NEXT:    jne .LBB5_1
+; AVX2-NEXT:  # %bb.2: # %else
+; AVX2-NEXT:    testb $2, %al
+; AVX2-NEXT:    jne .LBB5_3
+; AVX2-NEXT:  .LBB5_4: # %else2
+; AVX2-NEXT:    testb $4, %al
+; AVX2-NEXT:    jne .LBB5_5
+; AVX2-NEXT:  .LBB5_6: # %else4
+; AVX2-NEXT:    testb $8, %al
+; AVX2-NEXT:    jne .LBB5_7
+; AVX2-NEXT:  .LBB5_8: # %else6
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+; AVX2-NEXT:  .LBB5_1: # %cond.store
 ; AVX2-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX2-NEXT:  .LBB5_2: # %else
-; AVX2-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $2, %al
 ; AVX2-NEXT:    je .LBB5_4
-; AVX2-NEXT:  # %bb.3: # %cond.store1
-; AVX2-NEXT:    vpextrb $4, %xmm0, 1(%rdi)
-; AVX2-NEXT:  .LBB5_4: # %else2
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:  .LBB5_3: # %cond.store1
+; AVX2-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
+; AVX2-NEXT:    testb $4, %al
 ; AVX2-NEXT:    je .LBB5_6
-; AVX2-NEXT:  # %bb.5: # %cond.store3
-; AVX2-NEXT:    vpextrb $8, %xmm0, 2(%rdi)
-; AVX2-NEXT:  .LBB5_6: # %else4
-; AVX2-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:  .LBB5_5: # %cond.store3
+; AVX2-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
+; AVX2-NEXT:    testb $8, %al
 ; AVX2-NEXT:    je .LBB5_8
-; AVX2-NEXT:  # %bb.7: # %cond.store5
-; AVX2-NEXT:    vpextrb $12, %xmm0, 3(%rdi)
-; AVX2-NEXT:  .LBB5_8: # %else6
+; AVX2-NEXT:  .LBB5_7: # %cond.store5
+; AVX2-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
@@ -2071,39 +2019,36 @@ define void @truncstore_v4i64_v4i8(<4 x i64> %x, <4 x i8>* %p, <4 x i32> %mask)
 ; AVX512F-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
 ; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
 ; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [255,255,255,255]
-; AVX512F-NEXT:    vpminuq %zmm2, %zmm0, %zmm0
-; AVX512F-NEXT:    vpmovqd %zmm0, %ymm0
+; AVX512F-NEXT:    vpmovusqb %zmm0, %xmm0
 ; AVX512F-NEXT:    kmovw %k0, %eax
 ; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB5_2
-; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    jne .LBB5_1
+; AVX512F-NEXT:  # %bb.2: # %else
+; AVX512F-NEXT:    testb $2, %al
+; AVX512F-NEXT:    jne .LBB5_3
+; AVX512F-NEXT:  .LBB5_4: # %else2
+; AVX512F-NEXT:    testb $4, %al
+; AVX512F-NEXT:    jne .LBB5_5
+; AVX512F-NEXT:  .LBB5_6: # %else4
+; AVX512F-NEXT:    testb $8, %al
+; AVX512F-NEXT:    jne .LBB5_7
+; AVX512F-NEXT:  .LBB5_8: # %else6
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+; AVX512F-NEXT:  .LBB5_1: # %cond.store
 ; AVX512F-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX512F-NEXT:  .LBB5_2: # %else
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $2, %al
 ; AVX512F-NEXT:    je .LBB5_4
-; AVX512F-NEXT:  # %bb.3: # %cond.store1
-; AVX512F-NEXT:    vpextrb $4, %xmm0, 1(%rdi)
-; AVX512F-NEXT:  .LBB5_4: # %else2
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  .LBB5_3: # %cond.store1
+; AVX512F-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
+; AVX512F-NEXT:    testb $4, %al
 ; AVX512F-NEXT:    je .LBB5_6
-; AVX512F-NEXT:  # %bb.5: # %cond.store3
-; AVX512F-NEXT:    vpextrb $8, %xmm0, 2(%rdi)
-; AVX512F-NEXT:  .LBB5_6: # %else4
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  .LBB5_5: # %cond.store3
+; AVX512F-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
+; AVX512F-NEXT:    testb $8, %al
 ; AVX512F-NEXT:    je .LBB5_8
-; AVX512F-NEXT:  # %bb.7: # %cond.store5
-; AVX512F-NEXT:    vpextrb $12, %xmm0, 3(%rdi)
-; AVX512F-NEXT:  .LBB5_8: # %else6
+; AVX512F-NEXT:  .LBB5_7: # %cond.store5
+; AVX512F-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
@@ -2112,12 +2057,9 @@ define void @truncstore_v4i64_v4i8(<4 x i64> %x, <4 x i8>* %p, <4 x i32> %mask)
 ; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
 ; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
 ; AVX512BW-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512BW-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [255,255,255,255]
-; AVX512BW-NEXT:    vpminuq %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpmovqd %zmm0, %ymm0
-; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
 ; AVX512BW-NEXT:    kshiftlq $60, %k0, %k0
 ; AVX512BW-NEXT:    kshiftrq $60, %k0, %k1
+; AVX512BW-NEXT:    vpmovusqb %zmm0, %xmm0
 ; AVX512BW-NEXT:    vmovdqu8 %zmm0, (%rdi) {%k1}
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -2141,67 +2083,70 @@ define void @truncstore_v2i64_v2i32(<2 x i64> %x, <2 x i32>* %p, <2 x i64> %mask
 ; SSE2-LABEL: truncstore_v2i64_v2i32:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [9223372039002259456,9223372039002259456]
+; SSE2-NEXT:    pxor %xmm0, %xmm3
+; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [9223372039002259455,9223372039002259455]
+; SSE2-NEXT:    movdqa %xmm4, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm4, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSE2-NEXT:    pand %xmm6, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm5[1,1,3,3]
+; SSE2-NEXT:    por %xmm3, %xmm4
+; SSE2-NEXT:    pand %xmm4, %xmm0
+; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm4
+; SSE2-NEXT:    por %xmm0, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,2,2,3]
 ; SSE2-NEXT:    pcmpeqd %xmm1, %xmm2
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,0,3,2]
 ; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [9223372039002259456,9223372039002259456]
-; SSE2-NEXT:    pxor %xmm0, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [9223372039002259455,9223372039002259455]
-; SSE2-NEXT:    movdqa %xmm3, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
-; SSE2-NEXT:    pand %xmm5, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm2
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm2
-; SSE2-NEXT:    por %xmm0, %xmm2
-; SSE2-NEXT:    movd %xmm1, %eax
-; SSE2-NEXT:    notl %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB6_2
-; SSE2-NEXT:  # %bb.1: # %cond.store
-; SSE2-NEXT:    movd %xmm2, (%rdi)
-; SSE2-NEXT:  .LBB6_2: # %else
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE2-NEXT:    pxor %xmm0, %xmm1
-; SSE2-NEXT:    pextrw $4, %xmm1, %eax
+; SSE2-NEXT:    movmskpd %xmm1, %eax
+; SSE2-NEXT:    xorl $3, %eax
 ; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    jne .LBB6_1
+; SSE2-NEXT:  # %bb.2: # %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne .LBB6_3
+; SSE2-NEXT:  .LBB6_4: # %else2
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  .LBB6_1: # %cond.store
+; SSE2-NEXT:    movd %xmm0, (%rdi)
+; SSE2-NEXT:    testb $2, %al
 ; SSE2-NEXT:    je .LBB6_4
-; SSE2-NEXT:  # %bb.3: # %cond.store1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE2-NEXT:  .LBB6_3: # %cond.store1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
 ; SSE2-NEXT:    movd %xmm0, 4(%rdi)
-; SSE2-NEXT:  .LBB6_4: # %else2
 ; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: truncstore_v2i64_v2i32:
 ; SSE4:       # %bb.0:
 ; SSE4-NEXT:    movdqa %xmm0, %xmm2
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqq %xmm1, %xmm0
-; SSE4-NEXT:    pcmpeqd %xmm3, %xmm3
-; SSE4-NEXT:    pxor %xmm0, %xmm3
-; SSE4-NEXT:    movapd {{.*#+}} xmm1 = [4294967295,4294967295]
-; SSE4-NEXT:    movdqa {{.*#+}} xmm4 = [9223372036854775808,9223372036854775808]
-; SSE4-NEXT:    pxor %xmm2, %xmm4
+; SSE4-NEXT:    pxor %xmm3, %xmm3
+; SSE4-NEXT:    movapd {{.*#+}} xmm4 = [4294967295,4294967295]
+; SSE4-NEXT:    movdqa {{.*#+}} xmm5 = [9223372036854775808,9223372036854775808]
+; SSE4-NEXT:    pxor %xmm0, %xmm5
 ; SSE4-NEXT:    movdqa {{.*#+}} xmm0 = [9223372041149743103,9223372041149743103]
-; SSE4-NEXT:    pcmpgtq %xmm4, %xmm0
-; SSE4-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
-; SSE4-NEXT:    pextrb $0, %xmm3, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB6_2
-; SSE4-NEXT:  # %bb.1: # %cond.store
-; SSE4-NEXT:    movss %xmm1, (%rdi)
-; SSE4-NEXT:  .LBB6_2: # %else
-; SSE4-NEXT:    pextrb $8, %xmm3, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB6_4
-; SSE4-NEXT:  # %bb.3: # %cond.store1
-; SSE4-NEXT:    extractps $2, %xmm1, 4(%rdi)
+; SSE4-NEXT:    pcmpgtq %xmm5, %xmm0
+; SSE4-NEXT:    blendvpd %xmm0, %xmm2, %xmm4
+; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,2,2,3]
+; SSE4-NEXT:    pcmpeqq %xmm1, %xmm3
+; SSE4-NEXT:    movmskpd %xmm3, %eax
+; SSE4-NEXT:    xorl $3, %eax
+; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    jne .LBB6_1
+; SSE4-NEXT:  # %bb.2: # %else
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    jne .LBB6_3
 ; SSE4-NEXT:  .LBB6_4: # %else2
 ; SSE4-NEXT:    retq
+; SSE4-NEXT:  .LBB6_1: # %cond.store
+; SSE4-NEXT:    movd %xmm0, (%rdi)
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    je .LBB6_4
+; SSE4-NEXT:  .LBB6_3: # %cond.store1
+; SSE4-NEXT:    pextrd $1, %xmm0, 4(%rdi)
+; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: truncstore_v2i64_v2i32:
 ; AVX1:       # %bb.0:
@@ -2209,12 +2154,12 @@ define void @truncstore_v2i64_v2i32(<2 x i64> %x, <2 x i32>* %p, <2 x i64> %mask
 ; AVX1-NEXT:    vpcmpeqq %xmm2, %xmm1, %xmm1
 ; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
 ; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,2],zero,zero
 ; AVX1-NEXT:    vmovapd {{.*#+}} xmm2 = [4294967295,4294967295]
 ; AVX1-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm3
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [9223372041149743103,9223372041149743103]
 ; AVX1-NEXT:    vpcmpgtq %xmm3, %xmm4, %xmm3
 ; AVX1-NEXT:    vblendvpd %xmm3, %xmm0, %xmm2, %xmm0
-; AVX1-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,2],zero,zero
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; AVX1-NEXT:    vmaskmovps %xmm0, %xmm1, (%rdi)
 ; AVX1-NEXT:    retq
@@ -2225,12 +2170,12 @@ define void @truncstore_v2i64_v2i32(<2 x i64> %x, <2 x i32>* %p, <2 x i64> %mask
 ; AVX2-NEXT:    vpcmpeqq %xmm2, %xmm1, %xmm1
 ; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
 ; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,2],zero,zero
 ; AVX2-NEXT:    vmovapd {{.*#+}} xmm2 = [4294967295,4294967295]
 ; AVX2-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm3
 ; AVX2-NEXT:    vmovdqa {{.*#+}} xmm4 = [9223372041149743103,9223372041149743103]
 ; AVX2-NEXT:    vpcmpgtq %xmm3, %xmm4, %xmm3
 ; AVX2-NEXT:    vblendvpd %xmm3, %xmm0, %xmm2, %xmm0
-; AVX2-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,2],zero,zero
 ; AVX2-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; AVX2-NEXT:    vpmaskmovd %xmm0, %xmm1, (%rdi)
 ; AVX2-NEXT:    retq
@@ -2240,11 +2185,9 @@ define void @truncstore_v2i64_v2i32(<2 x i64> %x, <2 x i32>* %p, <2 x i64> %mask
 ; AVX512F-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
 ; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; AVX512F-NEXT:    vptestmq %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm1 = [4294967295,4294967295]
-; AVX512F-NEXT:    vpminuq %zmm1, %zmm0, %zmm0
-; AVX512F-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; AVX512F-NEXT:    kshiftlw $14, %k0, %k0
 ; AVX512F-NEXT:    kshiftrw $14, %k0, %k1
+; AVX512F-NEXT:    vpmovusqd %zmm0, %ymm0
 ; AVX512F-NEXT:    vmovdqu32 %zmm0, (%rdi) {%k1}
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
@@ -2261,11 +2204,9 @@ define void @truncstore_v2i64_v2i32(<2 x i64> %x, <2 x i32>* %p, <2 x i64> %mask
 ; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
 ; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; AVX512BW-NEXT:    vptestmq %zmm1, %zmm1, %k0
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm1 = [4294967295,4294967295]
-; AVX512BW-NEXT:    vpminuq %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; AVX512BW-NEXT:    kshiftlw $14, %k0, %k0
 ; AVX512BW-NEXT:    kshiftrw $14, %k0, %k1
+; AVX512BW-NEXT:    vpmovusqd %zmm0, %ymm0
 ; AVX512BW-NEXT:    vmovdqu32 %zmm0, (%rdi) {%k1}
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -2281,92 +2222,100 @@ define void @truncstore_v2i64_v2i16(<2 x i64> %x, <2 x i16>* %p, <2 x i64> %mask
 ; SSE2-LABEL: truncstore_v2i64_v2i16:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [9223372039002259456,9223372039002259456]
+; SSE2-NEXT:    pxor %xmm0, %xmm3
+; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [9223372039002324991,9223372039002324991]
+; SSE2-NEXT:    movdqa %xmm4, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm4, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSE2-NEXT:    pand %xmm6, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm5[1,1,3,3]
+; SSE2-NEXT:    por %xmm3, %xmm4
+; SSE2-NEXT:    pand %xmm4, %xmm0
+; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm4
+; SSE2-NEXT:    por %xmm0, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,2,2,3]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
 ; SSE2-NEXT:    pcmpeqd %xmm1, %xmm2
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,0,3,2]
 ; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [9223372039002259456,9223372039002259456]
-; SSE2-NEXT:    pxor %xmm0, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [9223372039002324991,9223372039002324991]
-; SSE2-NEXT:    movdqa %xmm3, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
-; SSE2-NEXT:    pand %xmm5, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm2
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm2
-; SSE2-NEXT:    por %xmm0, %xmm2
-; SSE2-NEXT:    movd %xmm1, %eax
-; SSE2-NEXT:    notl %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB7_2
-; SSE2-NEXT:  # %bb.1: # %cond.store
-; SSE2-NEXT:    movd %xmm2, %eax
-; SSE2-NEXT:    movw %ax, (%rdi)
-; SSE2-NEXT:  .LBB7_2: # %else
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE2-NEXT:    pxor %xmm0, %xmm1
-; SSE2-NEXT:    pextrw $4, %xmm1, %eax
+; SSE2-NEXT:    movmskpd %xmm1, %eax
+; SSE2-NEXT:    xorl $3, %eax
 ; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    jne .LBB7_1
+; SSE2-NEXT:  # %bb.2: # %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne .LBB7_3
+; SSE2-NEXT:  .LBB7_4: # %else2
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  .LBB7_1: # %cond.store
+; SSE2-NEXT:    movd %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, (%rdi)
+; SSE2-NEXT:    testb $2, %al
 ; SSE2-NEXT:    je .LBB7_4
-; SSE2-NEXT:  # %bb.3: # %cond.store1
-; SSE2-NEXT:    pextrw $4, %xmm2, %eax
+; SSE2-NEXT:  .LBB7_3: # %cond.store1
+; SSE2-NEXT:    pextrw $1, %xmm0, %eax
 ; SSE2-NEXT:    movw %ax, 2(%rdi)
-; SSE2-NEXT:  .LBB7_4: # %else2
 ; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: truncstore_v2i64_v2i16:
 ; SSE4:       # %bb.0:
 ; SSE4-NEXT:    movdqa %xmm0, %xmm2
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqq %xmm1, %xmm0
-; SSE4-NEXT:    pcmpeqd %xmm3, %xmm3
-; SSE4-NEXT:    pxor %xmm0, %xmm3
-; SSE4-NEXT:    movapd {{.*#+}} xmm1 = [65535,65535]
-; SSE4-NEXT:    movdqa {{.*#+}} xmm4 = [9223372036854775808,9223372036854775808]
-; SSE4-NEXT:    pxor %xmm2, %xmm4
+; SSE4-NEXT:    pxor %xmm3, %xmm3
+; SSE4-NEXT:    movapd {{.*#+}} xmm4 = [65535,65535]
+; SSE4-NEXT:    movdqa {{.*#+}} xmm5 = [9223372036854775808,9223372036854775808]
+; SSE4-NEXT:    pxor %xmm0, %xmm5
 ; SSE4-NEXT:    movdqa {{.*#+}} xmm0 = [9223372036854841343,9223372036854841343]
-; SSE4-NEXT:    pcmpgtq %xmm4, %xmm0
-; SSE4-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
-; SSE4-NEXT:    pextrb $0, %xmm3, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB7_2
-; SSE4-NEXT:  # %bb.1: # %cond.store
-; SSE4-NEXT:    pextrw $0, %xmm1, (%rdi)
-; SSE4-NEXT:  .LBB7_2: # %else
-; SSE4-NEXT:    pextrb $8, %xmm3, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB7_4
-; SSE4-NEXT:  # %bb.3: # %cond.store1
-; SSE4-NEXT:    pextrw $4, %xmm1, 2(%rdi)
+; SSE4-NEXT:    pcmpgtq %xmm5, %xmm0
+; SSE4-NEXT:    blendvpd %xmm0, %xmm2, %xmm4
+; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,2,2,3]
+; SSE4-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; SSE4-NEXT:    pcmpeqq %xmm1, %xmm3
+; SSE4-NEXT:    movmskpd %xmm3, %eax
+; SSE4-NEXT:    xorl $3, %eax
+; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    jne .LBB7_1
+; SSE4-NEXT:  # %bb.2: # %else
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    jne .LBB7_3
 ; SSE4-NEXT:  .LBB7_4: # %else2
 ; SSE4-NEXT:    retq
+; SSE4-NEXT:  .LBB7_1: # %cond.store
+; SSE4-NEXT:    pextrw $0, %xmm0, (%rdi)
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    je .LBB7_4
+; SSE4-NEXT:  .LBB7_3: # %cond.store1
+; SSE4-NEXT:    pextrw $1, %xmm0, 2(%rdi)
+; SSE4-NEXT:    retq
 ;
 ; AVX-LABEL: truncstore_v2i64_v2i16:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX-NEXT:    vmovapd {{.*#+}} xmm3 = [65535,65535]
+; AVX-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm4
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm5 = [9223372036854841343,9223372036854841343]
+; AVX-NEXT:    vpcmpgtq %xmm4, %xmm5, %xmm4
+; AVX-NEXT:    vblendvpd %xmm4, %xmm0, %xmm3, %xmm0
+; AVX-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
 ; AVX-NEXT:    vpcmpeqq %xmm2, %xmm1, %xmm1
-; AVX-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX-NEXT:    vmovapd {{.*#+}} xmm2 = [65535,65535]
-; AVX-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm3
-; AVX-NEXT:    vmovdqa {{.*#+}} xmm4 = [9223372036854841343,9223372036854841343]
-; AVX-NEXT:    vpcmpgtq %xmm3, %xmm4, %xmm3
-; AVX-NEXT:    vblendvpd %xmm3, %xmm0, %xmm2, %xmm0
-; AVX-NEXT:    vpextrb $0, %xmm1, %eax
+; AVX-NEXT:    vmovmskpd %xmm1, %eax
+; AVX-NEXT:    xorl $3, %eax
 ; AVX-NEXT:    testb $1, %al
-; AVX-NEXT:    je .LBB7_2
-; AVX-NEXT:  # %bb.1: # %cond.store
+; AVX-NEXT:    jne .LBB7_1
+; AVX-NEXT:  # %bb.2: # %else
+; AVX-NEXT:    testb $2, %al
+; AVX-NEXT:    jne .LBB7_3
+; AVX-NEXT:  .LBB7_4: # %else2
+; AVX-NEXT:    retq
+; AVX-NEXT:  .LBB7_1: # %cond.store
 ; AVX-NEXT:    vpextrw $0, %xmm0, (%rdi)
-; AVX-NEXT:  .LBB7_2: # %else
-; AVX-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX-NEXT:    testb $1, %al
+; AVX-NEXT:    testb $2, %al
 ; AVX-NEXT:    je .LBB7_4
-; AVX-NEXT:  # %bb.3: # %cond.store1
-; AVX-NEXT:    vpextrw $4, %xmm0, 2(%rdi)
-; AVX-NEXT:  .LBB7_4: # %else2
+; AVX-NEXT:  .LBB7_3: # %cond.store1
+; AVX-NEXT:    vpextrw $1, %xmm0, 2(%rdi)
 ; AVX-NEXT:    retq
 ;
 ; AVX512F-LABEL: truncstore_v2i64_v2i16:
@@ -2374,22 +2323,22 @@ define void @truncstore_v2i64_v2i16(<2 x i64> %x, <2 x i16>* %p, <2 x i64> %mask
 ; AVX512F-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
 ; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; AVX512F-NEXT:    vptestmq %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm2 = [65535,65535]
-; AVX512F-NEXT:    vpminuq %zmm2, %zmm0, %zmm0
+; AVX512F-NEXT:    vpmovusqw %zmm0, %xmm0
 ; AVX512F-NEXT:    kmovw %k0, %eax
 ; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB7_2
-; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    jne .LBB7_1
+; AVX512F-NEXT:  # %bb.2: # %else
+; AVX512F-NEXT:    testb $2, %al
+; AVX512F-NEXT:    jne .LBB7_3
+; AVX512F-NEXT:  .LBB7_4: # %else2
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+; AVX512F-NEXT:  .LBB7_1: # %cond.store
 ; AVX512F-NEXT:    vpextrw $0, %xmm0, (%rdi)
-; AVX512F-NEXT:  .LBB7_2: # %else
-; AVX512F-NEXT:    vptestmq %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $2, %al
 ; AVX512F-NEXT:    je .LBB7_4
-; AVX512F-NEXT:  # %bb.3: # %cond.store1
-; AVX512F-NEXT:    vpextrw $4, %xmm0, 2(%rdi)
-; AVX512F-NEXT:  .LBB7_4: # %else2
+; AVX512F-NEXT:  .LBB7_3: # %cond.store1
+; AVX512F-NEXT:    vpextrw $1, %xmm0, 2(%rdi)
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
@@ -2398,12 +2347,9 @@ define void @truncstore_v2i64_v2i16(<2 x i64> %x, <2 x i16>* %p, <2 x i64> %mask
 ; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
 ; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; AVX512BW-NEXT:    vptestmq %zmm1, %zmm1, %k0
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm1 = [65535,65535]
-; AVX512BW-NEXT:    vpminuq %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; AVX512BW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
 ; AVX512BW-NEXT:    kshiftld $30, %k0, %k0
 ; AVX512BW-NEXT:    kshiftrd $30, %k0, %k1
+; AVX512BW-NEXT:    vpmovusqw %zmm0, %xmm0
 ; AVX512BW-NEXT:    vmovdqu16 %zmm0, (%rdi) {%k1}
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -2426,92 +2372,99 @@ define void @truncstore_v2i64_v2i8(<2 x i64> %x, <2 x i8>* %p, <2 x i64> %mask)
 ; SSE2-LABEL: truncstore_v2i64_v2i8:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [9223372039002259456,9223372039002259456]
+; SSE2-NEXT:    pxor %xmm0, %xmm3
+; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [9223372039002259711,9223372039002259711]
+; SSE2-NEXT:    movdqa %xmm4, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm4, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSE2-NEXT:    pand %xmm6, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm5[1,1,3,3]
+; SSE2-NEXT:    por %xmm3, %xmm4
+; SSE2-NEXT:    pand %xmm4, %xmm0
+; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm4
+; SSE2-NEXT:    por %xmm0, %xmm4
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm4
+; SSE2-NEXT:    packuswb %xmm4, %xmm4
+; SSE2-NEXT:    packuswb %xmm4, %xmm4
+; SSE2-NEXT:    packuswb %xmm4, %xmm4
 ; SSE2-NEXT:    pcmpeqd %xmm1, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,0,3,2]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [9223372039002259456,9223372039002259456]
-; SSE2-NEXT:    pxor %xmm0, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [9223372039002259711,9223372039002259711]
-; SSE2-NEXT:    movdqa %xmm3, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
-; SSE2-NEXT:    pand %xmm5, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,0,3,2]
 ; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm2
-; SSE2-NEXT:    por %xmm0, %xmm2
-; SSE2-NEXT:    movd %xmm1, %eax
-; SSE2-NEXT:    notl %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB8_2
-; SSE2-NEXT:  # %bb.1: # %cond.store
-; SSE2-NEXT:    movd %xmm2, %eax
-; SSE2-NEXT:    movb %al, (%rdi)
-; SSE2-NEXT:  .LBB8_2: # %else
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE2-NEXT:    pxor %xmm0, %xmm1
-; SSE2-NEXT:    pextrw $4, %xmm1, %eax
+; SSE2-NEXT:    movmskpd %xmm0, %eax
+; SSE2-NEXT:    xorl $3, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB8_4
-; SSE2-NEXT:  # %bb.3: # %cond.store1
-; SSE2-NEXT:    pextrw $4, %xmm2, %eax
-; SSE2-NEXT:    movb %al, 1(%rdi)
+; SSE2-NEXT:    movd %xmm4, %ecx
+; SSE2-NEXT:    jne .LBB8_1
+; SSE2-NEXT:  # %bb.2: # %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne .LBB8_3
 ; SSE2-NEXT:  .LBB8_4: # %else2
 ; SSE2-NEXT:    retq
+; SSE2-NEXT:  .LBB8_1: # %cond.store
+; SSE2-NEXT:    movb %cl, (%rdi)
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    je .LBB8_4
+; SSE2-NEXT:  .LBB8_3: # %cond.store1
+; SSE2-NEXT:    movb %ch, 1(%rdi)
+; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: truncstore_v2i64_v2i8:
 ; SSE4:       # %bb.0:
 ; SSE4-NEXT:    movdqa %xmm0, %xmm2
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqq %xmm1, %xmm0
-; SSE4-NEXT:    pcmpeqd %xmm3, %xmm3
-; SSE4-NEXT:    pxor %xmm0, %xmm3
-; SSE4-NEXT:    movapd {{.*#+}} xmm1 = [255,255]
-; SSE4-NEXT:    movdqa {{.*#+}} xmm4 = [9223372036854775808,9223372036854775808]
-; SSE4-NEXT:    pxor %xmm2, %xmm4
+; SSE4-NEXT:    pxor %xmm4, %xmm4
+; SSE4-NEXT:    movapd {{.*#+}} xmm3 = [255,255]
+; SSE4-NEXT:    movdqa {{.*#+}} xmm5 = [9223372036854775808,9223372036854775808]
+; SSE4-NEXT:    pxor %xmm0, %xmm5
 ; SSE4-NEXT:    movdqa {{.*#+}} xmm0 = [9223372036854776063,9223372036854776063]
-; SSE4-NEXT:    pcmpgtq %xmm4, %xmm0
-; SSE4-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
-; SSE4-NEXT:    pextrb $0, %xmm3, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB8_2
-; SSE4-NEXT:  # %bb.1: # %cond.store
-; SSE4-NEXT:    pextrb $0, %xmm1, (%rdi)
-; SSE4-NEXT:  .LBB8_2: # %else
-; SSE4-NEXT:    pextrb $8, %xmm3, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB8_4
-; SSE4-NEXT:  # %bb.3: # %cond.store1
-; SSE4-NEXT:    pextrb $8, %xmm1, 1(%rdi)
+; SSE4-NEXT:    pcmpgtq %xmm5, %xmm0
+; SSE4-NEXT:    blendvpd %xmm0, %xmm2, %xmm3
+; SSE4-NEXT:    pshufb {{.*#+}} xmm3 = xmm3[0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; SSE4-NEXT:    pcmpeqq %xmm1, %xmm4
+; SSE4-NEXT:    movmskpd %xmm4, %eax
+; SSE4-NEXT:    xorl $3, %eax
+; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    jne .LBB8_1
+; SSE4-NEXT:  # %bb.2: # %else
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    jne .LBB8_3
 ; SSE4-NEXT:  .LBB8_4: # %else2
 ; SSE4-NEXT:    retq
+; SSE4-NEXT:  .LBB8_1: # %cond.store
+; SSE4-NEXT:    pextrb $0, %xmm3, (%rdi)
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    je .LBB8_4
+; SSE4-NEXT:  .LBB8_3: # %cond.store1
+; SSE4-NEXT:    pextrb $1, %xmm3, 1(%rdi)
+; SSE4-NEXT:    retq
 ;
 ; AVX-LABEL: truncstore_v2i64_v2i8:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX-NEXT:    vmovapd {{.*#+}} xmm3 = [255,255]
+; AVX-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm4
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm5 = [9223372036854776063,9223372036854776063]
+; AVX-NEXT:    vpcmpgtq %xmm4, %xmm5, %xmm4
+; AVX-NEXT:    vblendvpd %xmm4, %xmm0, %xmm3, %xmm0
+; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
 ; AVX-NEXT:    vpcmpeqq %xmm2, %xmm1, %xmm1
-; AVX-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX-NEXT:    vmovapd {{.*#+}} xmm2 = [255,255]
-; AVX-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm3
-; AVX-NEXT:    vmovdqa {{.*#+}} xmm4 = [9223372036854776063,9223372036854776063]
-; AVX-NEXT:    vpcmpgtq %xmm3, %xmm4, %xmm3
-; AVX-NEXT:    vblendvpd %xmm3, %xmm0, %xmm2, %xmm0
-; AVX-NEXT:    vpextrb $0, %xmm1, %eax
+; AVX-NEXT:    vmovmskpd %xmm1, %eax
+; AVX-NEXT:    xorl $3, %eax
 ; AVX-NEXT:    testb $1, %al
-; AVX-NEXT:    je .LBB8_2
-; AVX-NEXT:  # %bb.1: # %cond.store
+; AVX-NEXT:    jne .LBB8_1
+; AVX-NEXT:  # %bb.2: # %else
+; AVX-NEXT:    testb $2, %al
+; AVX-NEXT:    jne .LBB8_3
+; AVX-NEXT:  .LBB8_4: # %else2
+; AVX-NEXT:    retq
+; AVX-NEXT:  .LBB8_1: # %cond.store
 ; AVX-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX-NEXT:  .LBB8_2: # %else
-; AVX-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX-NEXT:    testb $1, %al
+; AVX-NEXT:    testb $2, %al
 ; AVX-NEXT:    je .LBB8_4
-; AVX-NEXT:  # %bb.3: # %cond.store1
-; AVX-NEXT:    vpextrb $8, %xmm0, 1(%rdi)
-; AVX-NEXT:  .LBB8_4: # %else2
+; AVX-NEXT:  .LBB8_3: # %cond.store1
+; AVX-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
 ; AVX-NEXT:    retq
 ;
 ; AVX512F-LABEL: truncstore_v2i64_v2i8:
@@ -2519,22 +2472,22 @@ define void @truncstore_v2i64_v2i8(<2 x i64> %x, <2 x i8>* %p, <2 x i64> %mask)
 ; AVX512F-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
 ; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; AVX512F-NEXT:    vptestmq %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm2 = [255,255]
-; AVX512F-NEXT:    vpminuq %zmm2, %zmm0, %zmm0
+; AVX512F-NEXT:    vpmovusqb %zmm0, %xmm0
 ; AVX512F-NEXT:    kmovw %k0, %eax
 ; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB8_2
-; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    jne .LBB8_1
+; AVX512F-NEXT:  # %bb.2: # %else
+; AVX512F-NEXT:    testb $2, %al
+; AVX512F-NEXT:    jne .LBB8_3
+; AVX512F-NEXT:  .LBB8_4: # %else2
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+; AVX512F-NEXT:  .LBB8_1: # %cond.store
 ; AVX512F-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX512F-NEXT:  .LBB8_2: # %else
-; AVX512F-NEXT:    vptestmq %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $2, %al
 ; AVX512F-NEXT:    je .LBB8_4
-; AVX512F-NEXT:  # %bb.3: # %cond.store1
-; AVX512F-NEXT:    vpextrb $8, %xmm0, 1(%rdi)
-; AVX512F-NEXT:  .LBB8_4: # %else2
+; AVX512F-NEXT:  .LBB8_3: # %cond.store1
+; AVX512F-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
@@ -2543,11 +2496,9 @@ define void @truncstore_v2i64_v2i8(<2 x i64> %x, <2 x i8>* %p, <2 x i64> %mask)
 ; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
 ; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; AVX512BW-NEXT:    vptestmq %zmm1, %zmm1, %k0
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm1 = [255,255]
-; AVX512BW-NEXT:    vpminuq %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
 ; AVX512BW-NEXT:    kshiftlq $62, %k0, %k0
 ; AVX512BW-NEXT:    kshiftrq $62, %k0, %k1
+; AVX512BW-NEXT:    vpmovusqb %zmm0, %xmm0
 ; AVX512BW-NEXT:    vmovdqu8 %zmm0, (%rdi) {%k1}
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -2570,14 +2521,13 @@ define void @truncstore_v16i32_v16i16(<16 x i32> %x, <16 x i16>* %p, <16 x i32>
 ; SSE2-LABEL: truncstore_v16i32_v16i16:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pxor %xmm12, %xmm12
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm12
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm9
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm11 = [2147483648,2147483648,2147483648,2147483648]
-; SSE2-NEXT:    movdqa %xmm0, %xmm13
-; SSE2-NEXT:    pxor %xmm11, %xmm13
+; SSE2-NEXT:    movdqa %xmm0, %xmm9
+; SSE2-NEXT:    pxor %xmm11, %xmm9
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [2147549183,2147549183,2147549183,2147549183]
 ; SSE2-NEXT:    movdqa %xmm10, %xmm8
-; SSE2-NEXT:    pcmpgtd %xmm13, %xmm8
+; SSE2-NEXT:    pcmpgtd %xmm9, %xmm8
+; SSE2-NEXT:    pcmpeqd %xmm9, %xmm9
 ; SSE2-NEXT:    pand %xmm8, %xmm0
 ; SSE2-NEXT:    pxor %xmm9, %xmm8
 ; SSE2-NEXT:    por %xmm0, %xmm8
@@ -2593,676 +2543,554 @@ define void @truncstore_v16i32_v16i16(<16 x i32> %x, <16 x i16>* %p, <16 x i32>
 ; SSE2-NEXT:    pslld $16, %xmm8
 ; SSE2-NEXT:    psrad $16, %xmm8
 ; SSE2-NEXT:    packssdw %xmm0, %xmm8
-; SSE2-NEXT:    movd %xmm12, %eax
-; SSE2-NEXT:    notl %eax
+; SSE2-NEXT:    pcmpeqd %xmm12, %xmm7
+; SSE2-NEXT:    pxor %xmm9, %xmm7
+; SSE2-NEXT:    pcmpeqd %xmm12, %xmm6
+; SSE2-NEXT:    pxor %xmm9, %xmm6
+; SSE2-NEXT:    packssdw %xmm7, %xmm6
+; SSE2-NEXT:    pcmpeqd %xmm12, %xmm5
+; SSE2-NEXT:    pxor %xmm9, %xmm5
+; SSE2-NEXT:    pcmpeqd %xmm12, %xmm4
+; SSE2-NEXT:    pxor %xmm9, %xmm4
+; SSE2-NEXT:    packssdw %xmm5, %xmm4
+; SSE2-NEXT:    packsswb %xmm6, %xmm4
+; SSE2-NEXT:    pmovmskb %xmm4, %eax
 ; SSE2-NEXT:    testb $1, %al
 ; SSE2-NEXT:    je .LBB9_2
 ; SSE2-NEXT:  # %bb.1: # %cond.store
-; SSE2-NEXT:    movd %xmm8, %eax
-; SSE2-NEXT:    movw %ax, (%rdi)
+; SSE2-NEXT:    movd %xmm8, %ecx
+; SSE2-NEXT:    movw %cx, (%rdi)
 ; SSE2-NEXT:  .LBB9_2: # %else
-; SSE2-NEXT:    pxor %xmm9, %xmm12
-; SSE2-NEXT:    pextrw $2, %xmm12, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $2, %al
 ; SSE2-NEXT:    je .LBB9_4
 ; SSE2-NEXT:  # %bb.3: # %cond.store1
-; SSE2-NEXT:    pextrw $1, %xmm8, %eax
-; SSE2-NEXT:    movw %ax, 2(%rdi)
+; SSE2-NEXT:    pextrw $1, %xmm8, %ecx
+; SSE2-NEXT:    movw %cx, 2(%rdi)
 ; SSE2-NEXT:  .LBB9_4: # %else2
-; SSE2-NEXT:    movdqa %xmm2, %xmm12
-; SSE2-NEXT:    pxor %xmm11, %xmm12
+; SSE2-NEXT:    movdqa %xmm2, %xmm1
+; SSE2-NEXT:    pxor %xmm11, %xmm1
 ; SSE2-NEXT:    pxor %xmm3, %xmm11
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm4
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pxor %xmm4, %xmm1
-; SSE2-NEXT:    pextrw $4, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $4, %al
 ; SSE2-NEXT:    je .LBB9_6
 ; SSE2-NEXT:  # %bb.5: # %cond.store3
-; SSE2-NEXT:    pextrw $2, %xmm8, %eax
-; SSE2-NEXT:    movw %ax, 4(%rdi)
+; SSE2-NEXT:    pextrw $2, %xmm8, %ecx
+; SSE2-NEXT:    movw %cx, 4(%rdi)
 ; SSE2-NEXT:  .LBB9_6: # %else4
 ; SSE2-NEXT:    movdqa %xmm10, %xmm0
-; SSE2-NEXT:    pcmpgtd %xmm12, %xmm0
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE2-NEXT:    pcmpgtd %xmm11, %xmm10
-; SSE2-NEXT:    pextrw $6, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je .LBB9_8
 ; SSE2-NEXT:  # %bb.7: # %cond.store5
-; SSE2-NEXT:    pextrw $3, %xmm8, %eax
-; SSE2-NEXT:    movw %ax, 6(%rdi)
+; SSE2-NEXT:    pextrw $3, %xmm8, %ecx
+; SSE2-NEXT:    movw %cx, 6(%rdi)
 ; SSE2-NEXT:  .LBB9_8: # %else6
 ; SSE2-NEXT:    pand %xmm0, %xmm2
 ; SSE2-NEXT:    pxor %xmm9, %xmm0
 ; SSE2-NEXT:    pxor %xmm10, %xmm9
 ; SSE2-NEXT:    pand %xmm10, %xmm3
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm1
-; SSE2-NEXT:    movd %xmm1, %eax
-; SSE2-NEXT:    notl %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $16, %al
 ; SSE2-NEXT:    je .LBB9_10
 ; SSE2-NEXT:  # %bb.9: # %cond.store7
-; SSE2-NEXT:    pextrw $4, %xmm8, %eax
-; SSE2-NEXT:    movw %ax, 8(%rdi)
+; SSE2-NEXT:    pextrw $4, %xmm8, %ecx
+; SSE2-NEXT:    movw %cx, 8(%rdi)
 ; SSE2-NEXT:  .LBB9_10: # %else8
 ; SSE2-NEXT:    por %xmm0, %xmm2
 ; SSE2-NEXT:    por %xmm9, %xmm3
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE2-NEXT:    pxor %xmm0, %xmm1
-; SSE2-NEXT:    pextrw $2, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $32, %al
 ; SSE2-NEXT:    je .LBB9_12
 ; SSE2-NEXT:  # %bb.11: # %cond.store9
-; SSE2-NEXT:    pextrw $5, %xmm8, %eax
-; SSE2-NEXT:    movw %ax, 10(%rdi)
+; SSE2-NEXT:    pextrw $5, %xmm8, %ecx
+; SSE2-NEXT:    movw %cx, 10(%rdi)
 ; SSE2-NEXT:  .LBB9_12: # %else10
 ; SSE2-NEXT:    pslld $16, %xmm3
 ; SSE2-NEXT:    pslld $16, %xmm2
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm5
-; SSE2-NEXT:    pxor %xmm5, %xmm0
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $64, %al
 ; SSE2-NEXT:    je .LBB9_14
 ; SSE2-NEXT:  # %bb.13: # %cond.store11
-; SSE2-NEXT:    pextrw $6, %xmm8, %eax
-; SSE2-NEXT:    movw %ax, 12(%rdi)
+; SSE2-NEXT:    pextrw $6, %xmm8, %ecx
+; SSE2-NEXT:    movw %cx, 12(%rdi)
 ; SSE2-NEXT:  .LBB9_14: # %else12
 ; SSE2-NEXT:    psrad $16, %xmm3
 ; SSE2-NEXT:    psrad $16, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE2-NEXT:    pxor %xmm0, %xmm5
-; SSE2-NEXT:    pextrw $6, %xmm5, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $-128, %al
 ; SSE2-NEXT:    je .LBB9_16
 ; SSE2-NEXT:  # %bb.15: # %cond.store13
-; SSE2-NEXT:    pextrw $7, %xmm8, %eax
-; SSE2-NEXT:    movw %ax, 14(%rdi)
+; SSE2-NEXT:    pextrw $7, %xmm8, %ecx
+; SSE2-NEXT:    movw %cx, 14(%rdi)
 ; SSE2-NEXT:  .LBB9_16: # %else14
 ; SSE2-NEXT:    packssdw %xmm3, %xmm2
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqd %xmm6, %xmm0
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    notl %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB9_18
-; SSE2-NEXT:  # %bb.17: # %cond.store15
-; SSE2-NEXT:    movd %xmm2, %eax
-; SSE2-NEXT:    movw %ax, 16(%rdi)
-; SSE2-NEXT:  .LBB9_18: # %else16
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pxor %xmm1, %xmm0
-; SSE2-NEXT:    pextrw $2, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB9_20
-; SSE2-NEXT:  # %bb.19: # %cond.store17
-; SSE2-NEXT:    pextrw $1, %xmm2, %eax
-; SSE2-NEXT:    movw %ax, 18(%rdi)
+; SSE2-NEXT:    testl $256, %eax # imm = 0x100
+; SSE2-NEXT:    jne .LBB9_17
+; SSE2-NEXT:  # %bb.18: # %else16
+; SSE2-NEXT:    testl $512, %eax # imm = 0x200
+; SSE2-NEXT:    jne .LBB9_19
 ; SSE2-NEXT:  .LBB9_20: # %else18
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm6
-; SSE2-NEXT:    pxor %xmm6, %xmm1
-; SSE2-NEXT:    pextrw $4, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB9_22
-; SSE2-NEXT:  # %bb.21: # %cond.store19
-; SSE2-NEXT:    pextrw $2, %xmm2, %eax
-; SSE2-NEXT:    movw %ax, 20(%rdi)
+; SSE2-NEXT:    testl $1024, %eax # imm = 0x400
+; SSE2-NEXT:    jne .LBB9_21
 ; SSE2-NEXT:  .LBB9_22: # %else20
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE2-NEXT:    pxor %xmm0, %xmm6
-; SSE2-NEXT:    pextrw $6, %xmm6, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB9_24
-; SSE2-NEXT:  # %bb.23: # %cond.store21
-; SSE2-NEXT:    pextrw $3, %xmm2, %eax
-; SSE2-NEXT:    movw %ax, 22(%rdi)
+; SSE2-NEXT:    testl $2048, %eax # imm = 0x800
+; SSE2-NEXT:    jne .LBB9_23
 ; SSE2-NEXT:  .LBB9_24: # %else22
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqd %xmm7, %xmm0
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    notl %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB9_26
-; SSE2-NEXT:  # %bb.25: # %cond.store23
-; SSE2-NEXT:    pextrw $4, %xmm2, %eax
-; SSE2-NEXT:    movw %ax, 24(%rdi)
+; SSE2-NEXT:    testl $4096, %eax # imm = 0x1000
+; SSE2-NEXT:    jne .LBB9_25
 ; SSE2-NEXT:  .LBB9_26: # %else24
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pxor %xmm1, %xmm0
-; SSE2-NEXT:    pextrw $2, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB9_28
-; SSE2-NEXT:  # %bb.27: # %cond.store25
-; SSE2-NEXT:    pextrw $5, %xmm2, %eax
-; SSE2-NEXT:    movw %ax, 26(%rdi)
+; SSE2-NEXT:    testl $8192, %eax # imm = 0x2000
+; SSE2-NEXT:    jne .LBB9_27
 ; SSE2-NEXT:  .LBB9_28: # %else26
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm7
-; SSE2-NEXT:    pxor %xmm7, %xmm1
-; SSE2-NEXT:    pextrw $4, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB9_30
-; SSE2-NEXT:  # %bb.29: # %cond.store27
-; SSE2-NEXT:    pextrw $6, %xmm2, %eax
-; SSE2-NEXT:    movw %ax, 28(%rdi)
+; SSE2-NEXT:    testl $16384, %eax # imm = 0x4000
+; SSE2-NEXT:    jne .LBB9_29
 ; SSE2-NEXT:  .LBB9_30: # %else28
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE2-NEXT:    pxor %xmm0, %xmm7
-; SSE2-NEXT:    pextrw $6, %xmm7, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $32768, %eax # imm = 0x8000
+; SSE2-NEXT:    jne .LBB9_31
+; SSE2-NEXT:  .LBB9_32: # %else30
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  .LBB9_17: # %cond.store15
+; SSE2-NEXT:    movd %xmm2, %ecx
+; SSE2-NEXT:    movw %cx, 16(%rdi)
+; SSE2-NEXT:    testl $512, %eax # imm = 0x200
+; SSE2-NEXT:    je .LBB9_20
+; SSE2-NEXT:  .LBB9_19: # %cond.store17
+; SSE2-NEXT:    pextrw $1, %xmm2, %ecx
+; SSE2-NEXT:    movw %cx, 18(%rdi)
+; SSE2-NEXT:    testl $1024, %eax # imm = 0x400
+; SSE2-NEXT:    je .LBB9_22
+; SSE2-NEXT:  .LBB9_21: # %cond.store19
+; SSE2-NEXT:    pextrw $2, %xmm2, %ecx
+; SSE2-NEXT:    movw %cx, 20(%rdi)
+; SSE2-NEXT:    testl $2048, %eax # imm = 0x800
+; SSE2-NEXT:    je .LBB9_24
+; SSE2-NEXT:  .LBB9_23: # %cond.store21
+; SSE2-NEXT:    pextrw $3, %xmm2, %ecx
+; SSE2-NEXT:    movw %cx, 22(%rdi)
+; SSE2-NEXT:    testl $4096, %eax # imm = 0x1000
+; SSE2-NEXT:    je .LBB9_26
+; SSE2-NEXT:  .LBB9_25: # %cond.store23
+; SSE2-NEXT:    pextrw $4, %xmm2, %ecx
+; SSE2-NEXT:    movw %cx, 24(%rdi)
+; SSE2-NEXT:    testl $8192, %eax # imm = 0x2000
+; SSE2-NEXT:    je .LBB9_28
+; SSE2-NEXT:  .LBB9_27: # %cond.store25
+; SSE2-NEXT:    pextrw $5, %xmm2, %ecx
+; SSE2-NEXT:    movw %cx, 26(%rdi)
+; SSE2-NEXT:    testl $16384, %eax # imm = 0x4000
+; SSE2-NEXT:    je .LBB9_30
+; SSE2-NEXT:  .LBB9_29: # %cond.store27
+; SSE2-NEXT:    pextrw $6, %xmm2, %ecx
+; SSE2-NEXT:    movw %cx, 28(%rdi)
+; SSE2-NEXT:    testl $32768, %eax # imm = 0x8000
 ; SSE2-NEXT:    je .LBB9_32
-; SSE2-NEXT:  # %bb.31: # %cond.store29
+; SSE2-NEXT:  .LBB9_31: # %cond.store29
 ; SSE2-NEXT:    pextrw $7, %xmm2, %eax
 ; SSE2-NEXT:    movw %ax, 30(%rdi)
-; SSE2-NEXT:  .LBB9_32: # %else30
 ; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: truncstore_v16i32_v16i16:
 ; SSE4:       # %bb.0:
-; SSE4-NEXT:    pxor %xmm8, %xmm8
-; SSE4-NEXT:    pcmpeqd %xmm4, %xmm8
-; SSE4-NEXT:    pcmpeqd %xmm9, %xmm9
-; SSE4-NEXT:    pxor %xmm8, %xmm9
+; SSE4-NEXT:    pxor %xmm9, %xmm9
 ; SSE4-NEXT:    movdqa {{.*#+}} xmm8 = [65535,65535,65535,65535]
 ; SSE4-NEXT:    pminud %xmm8, %xmm1
 ; SSE4-NEXT:    pminud %xmm8, %xmm0
 ; SSE4-NEXT:    packusdw %xmm1, %xmm0
-; SSE4-NEXT:    pextrb $0, %xmm9, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB9_2
-; SSE4-NEXT:  # %bb.1: # %cond.store
-; SSE4-NEXT:    pextrw $0, %xmm0, (%rdi)
-; SSE4-NEXT:  .LBB9_2: # %else
-; SSE4-NEXT:    pextrb $4, %xmm9, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB9_4
-; SSE4-NEXT:  # %bb.3: # %cond.store1
-; SSE4-NEXT:    pextrw $1, %xmm0, 2(%rdi)
-; SSE4-NEXT:  .LBB9_4: # %else2
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm4
+; SSE4-NEXT:    pcmpeqd %xmm9, %xmm7
 ; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE4-NEXT:    pxor %xmm4, %xmm1
-; SSE4-NEXT:    pextrb $8, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB9_6
-; SSE4-NEXT:  # %bb.5: # %cond.store3
-; SSE4-NEXT:    pextrw $2, %xmm0, 4(%rdi)
+; SSE4-NEXT:    pxor %xmm1, %xmm7
+; SSE4-NEXT:    pcmpeqd %xmm9, %xmm6
+; SSE4-NEXT:    pxor %xmm1, %xmm6
+; SSE4-NEXT:    packssdw %xmm7, %xmm6
+; SSE4-NEXT:    pcmpeqd %xmm9, %xmm5
+; SSE4-NEXT:    pxor %xmm1, %xmm5
+; SSE4-NEXT:    pcmpeqd %xmm9, %xmm4
+; SSE4-NEXT:    pxor %xmm1, %xmm4
+; SSE4-NEXT:    packssdw %xmm5, %xmm4
+; SSE4-NEXT:    packsswb %xmm6, %xmm4
+; SSE4-NEXT:    pmovmskb %xmm4, %eax
+; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    jne .LBB9_1
+; SSE4-NEXT:  # %bb.2: # %else
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    jne .LBB9_3
+; SSE4-NEXT:  .LBB9_4: # %else2
+; SSE4-NEXT:    testb $4, %al
+; SSE4-NEXT:    jne .LBB9_5
 ; SSE4-NEXT:  .LBB9_6: # %else4
-; SSE4-NEXT:    pextrb $12, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB9_8
-; SSE4-NEXT:  # %bb.7: # %cond.store5
-; SSE4-NEXT:    pextrw $3, %xmm0, 6(%rdi)
+; SSE4-NEXT:    testb $8, %al
+; SSE4-NEXT:    jne .LBB9_7
 ; SSE4-NEXT:  .LBB9_8: # %else6
-; SSE4-NEXT:    pxor %xmm4, %xmm4
-; SSE4-NEXT:    pcmpeqd %xmm5, %xmm4
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE4-NEXT:    pxor %xmm4, %xmm1
-; SSE4-NEXT:    pextrb $0, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB9_10
-; SSE4-NEXT:  # %bb.9: # %cond.store7
-; SSE4-NEXT:    pextrw $4, %xmm0, 8(%rdi)
+; SSE4-NEXT:    testb $16, %al
+; SSE4-NEXT:    jne .LBB9_9
 ; SSE4-NEXT:  .LBB9_10: # %else8
-; SSE4-NEXT:    pextrb $4, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB9_12
-; SSE4-NEXT:  # %bb.11: # %cond.store9
-; SSE4-NEXT:    pextrw $5, %xmm0, 10(%rdi)
+; SSE4-NEXT:    testb $32, %al
+; SSE4-NEXT:    jne .LBB9_11
 ; SSE4-NEXT:  .LBB9_12: # %else10
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm5
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE4-NEXT:    pxor %xmm5, %xmm1
-; SSE4-NEXT:    pextrb $8, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $64, %al
 ; SSE4-NEXT:    je .LBB9_14
-; SSE4-NEXT:  # %bb.13: # %cond.store11
+; SSE4-NEXT:  .LBB9_13: # %cond.store11
 ; SSE4-NEXT:    pextrw $6, %xmm0, 12(%rdi)
 ; SSE4-NEXT:  .LBB9_14: # %else12
 ; SSE4-NEXT:    pminud %xmm8, %xmm3
 ; SSE4-NEXT:    pminud %xmm8, %xmm2
-; SSE4-NEXT:    pextrb $12, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $-128, %al
 ; SSE4-NEXT:    je .LBB9_16
 ; SSE4-NEXT:  # %bb.15: # %cond.store13
 ; SSE4-NEXT:    pextrw $7, %xmm0, 14(%rdi)
 ; SSE4-NEXT:  .LBB9_16: # %else14
 ; SSE4-NEXT:    packusdw %xmm3, %xmm2
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm6, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE4-NEXT:    pxor %xmm1, %xmm0
-; SSE4-NEXT:    pextrb $0, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB9_18
-; SSE4-NEXT:  # %bb.17: # %cond.store15
+; SSE4-NEXT:    testl $256, %eax # imm = 0x100
+; SSE4-NEXT:    jne .LBB9_17
+; SSE4-NEXT:  # %bb.18: # %else16
+; SSE4-NEXT:    testl $512, %eax # imm = 0x200
+; SSE4-NEXT:    jne .LBB9_19
+; SSE4-NEXT:  .LBB9_20: # %else18
+; SSE4-NEXT:    testl $1024, %eax # imm = 0x400
+; SSE4-NEXT:    jne .LBB9_21
+; SSE4-NEXT:  .LBB9_22: # %else20
+; SSE4-NEXT:    testl $2048, %eax # imm = 0x800
+; SSE4-NEXT:    jne .LBB9_23
+; SSE4-NEXT:  .LBB9_24: # %else22
+; SSE4-NEXT:    testl $4096, %eax # imm = 0x1000
+; SSE4-NEXT:    jne .LBB9_25
+; SSE4-NEXT:  .LBB9_26: # %else24
+; SSE4-NEXT:    testl $8192, %eax # imm = 0x2000
+; SSE4-NEXT:    jne .LBB9_27
+; SSE4-NEXT:  .LBB9_28: # %else26
+; SSE4-NEXT:    testl $16384, %eax # imm = 0x4000
+; SSE4-NEXT:    jne .LBB9_29
+; SSE4-NEXT:  .LBB9_30: # %else28
+; SSE4-NEXT:    testl $32768, %eax # imm = 0x8000
+; SSE4-NEXT:    jne .LBB9_31
+; SSE4-NEXT:  .LBB9_32: # %else30
+; SSE4-NEXT:    retq
+; SSE4-NEXT:  .LBB9_1: # %cond.store
+; SSE4-NEXT:    pextrw $0, %xmm0, (%rdi)
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    je .LBB9_4
+; SSE4-NEXT:  .LBB9_3: # %cond.store1
+; SSE4-NEXT:    pextrw $1, %xmm0, 2(%rdi)
+; SSE4-NEXT:    testb $4, %al
+; SSE4-NEXT:    je .LBB9_6
+; SSE4-NEXT:  .LBB9_5: # %cond.store3
+; SSE4-NEXT:    pextrw $2, %xmm0, 4(%rdi)
+; SSE4-NEXT:    testb $8, %al
+; SSE4-NEXT:    je .LBB9_8
+; SSE4-NEXT:  .LBB9_7: # %cond.store5
+; SSE4-NEXT:    pextrw $3, %xmm0, 6(%rdi)
+; SSE4-NEXT:    testb $16, %al
+; SSE4-NEXT:    je .LBB9_10
+; SSE4-NEXT:  .LBB9_9: # %cond.store7
+; SSE4-NEXT:    pextrw $4, %xmm0, 8(%rdi)
+; SSE4-NEXT:    testb $32, %al
+; SSE4-NEXT:    je .LBB9_12
+; SSE4-NEXT:  .LBB9_11: # %cond.store9
+; SSE4-NEXT:    pextrw $5, %xmm0, 10(%rdi)
+; SSE4-NEXT:    testb $64, %al
+; SSE4-NEXT:    jne .LBB9_13
+; SSE4-NEXT:    jmp .LBB9_14
+; SSE4-NEXT:  .LBB9_17: # %cond.store15
 ; SSE4-NEXT:    pextrw $0, %xmm2, 16(%rdi)
-; SSE4-NEXT:  .LBB9_18: # %else16
-; SSE4-NEXT:    pextrb $4, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $512, %eax # imm = 0x200
 ; SSE4-NEXT:    je .LBB9_20
-; SSE4-NEXT:  # %bb.19: # %cond.store17
+; SSE4-NEXT:  .LBB9_19: # %cond.store17
 ; SSE4-NEXT:    pextrw $1, %xmm2, 18(%rdi)
-; SSE4-NEXT:  .LBB9_20: # %else18
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqd %xmm0, %xmm6
-; SSE4-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE4-NEXT:    pxor %xmm6, %xmm0
-; SSE4-NEXT:    pextrb $8, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $1024, %eax # imm = 0x400
 ; SSE4-NEXT:    je .LBB9_22
-; SSE4-NEXT:  # %bb.21: # %cond.store19
+; SSE4-NEXT:  .LBB9_21: # %cond.store19
 ; SSE4-NEXT:    pextrw $2, %xmm2, 20(%rdi)
-; SSE4-NEXT:  .LBB9_22: # %else20
-; SSE4-NEXT:    pextrb $12, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $2048, %eax # imm = 0x800
 ; SSE4-NEXT:    je .LBB9_24
-; SSE4-NEXT:  # %bb.23: # %cond.store21
+; SSE4-NEXT:  .LBB9_23: # %cond.store21
 ; SSE4-NEXT:    pextrw $3, %xmm2, 22(%rdi)
-; SSE4-NEXT:  .LBB9_24: # %else22
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm7, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE4-NEXT:    pxor %xmm1, %xmm0
-; SSE4-NEXT:    pextrb $0, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $4096, %eax # imm = 0x1000
 ; SSE4-NEXT:    je .LBB9_26
-; SSE4-NEXT:  # %bb.25: # %cond.store23
+; SSE4-NEXT:  .LBB9_25: # %cond.store23
 ; SSE4-NEXT:    pextrw $4, %xmm2, 24(%rdi)
-; SSE4-NEXT:  .LBB9_26: # %else24
-; SSE4-NEXT:    pextrb $4, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $8192, %eax # imm = 0x2000
 ; SSE4-NEXT:    je .LBB9_28
-; SSE4-NEXT:  # %bb.27: # %cond.store25
+; SSE4-NEXT:  .LBB9_27: # %cond.store25
 ; SSE4-NEXT:    pextrw $5, %xmm2, 26(%rdi)
-; SSE4-NEXT:  .LBB9_28: # %else26
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqd %xmm0, %xmm7
-; SSE4-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE4-NEXT:    pxor %xmm7, %xmm0
-; SSE4-NEXT:    pextrb $8, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $16384, %eax # imm = 0x4000
 ; SSE4-NEXT:    je .LBB9_30
-; SSE4-NEXT:  # %bb.29: # %cond.store27
+; SSE4-NEXT:  .LBB9_29: # %cond.store27
 ; SSE4-NEXT:    pextrw $6, %xmm2, 28(%rdi)
-; SSE4-NEXT:  .LBB9_30: # %else28
-; SSE4-NEXT:    pextrb $12, %xmm0, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $32768, %eax # imm = 0x8000
 ; SSE4-NEXT:    je .LBB9_32
-; SSE4-NEXT:  # %bb.31: # %cond.store29
+; SSE4-NEXT:  .LBB9_31: # %cond.store29
 ; SSE4-NEXT:    pextrw $7, %xmm2, 30(%rdi)
-; SSE4-NEXT:  .LBB9_32: # %else30
 ; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: truncstore_v16i32_v16i16:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpxor %xmm8, %xmm8, %xmm8
-; AVX1-NEXT:    vpcmpeqd %xmm8, %xmm2, %xmm6
-; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
-; AVX1-NEXT:    vpxor %xmm5, %xmm6, %xmm6
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm6, %xmm6
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm6, %xmm6
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm7
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [65535,65535,65535,65535]
-; AVX1-NEXT:    vpminud %xmm4, %xmm7, %xmm7
-; AVX1-NEXT:    vpminud %xmm4, %xmm0, %xmm0
-; AVX1-NEXT:    vpackusdw %xmm7, %xmm0, %xmm0
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm7
-; AVX1-NEXT:    vpminud %xmm4, %xmm7, %xmm7
-; AVX1-NEXT:    vpminud %xmm4, %xmm1, %xmm1
-; AVX1-NEXT:    vpackusdw %xmm7, %xmm1, %xmm1
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm5 = [65535,65535,65535,65535]
+; AVX1-NEXT:    vpminud %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vpminud %xmm5, %xmm0, %xmm0
+; AVX1-NEXT:    vpackusdw %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm4
+; AVX1-NEXT:    vpminud %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vpminud %xmm5, %xmm1, %xmm1
+; AVX1-NEXT:    vpackusdw %xmm4, %xmm1, %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX1-NEXT:    vpextrb $0, %xmm6, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je .LBB9_2
-; AVX1-NEXT:  # %bb.1: # %cond.store
-; AVX1-NEXT:    vpextrw $0, %xmm0, (%rdi)
-; AVX1-NEXT:  .LBB9_2: # %else
-; AVX1-NEXT:    vpcmpeqd %xmm8, %xmm2, %xmm1
+; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm1
+; AVX1-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm1, %xmm1
+; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
 ; AVX1-NEXT:    vpxor %xmm5, %xmm1, %xmm1
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm1, %xmm1
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $1, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpxor %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vpackssdw %xmm1, %xmm3, %xmm1
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm3
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpxor %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm2, %xmm2
+; AVX1-NEXT:    vpxor %xmm5, %xmm2, %xmm2
+; AVX1-NEXT:    vpackssdw %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpacksswb %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vpmovmskb %xmm1, %eax
+; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    jne .LBB9_1
+; AVX1-NEXT:  # %bb.2: # %else
+; AVX1-NEXT:    testb $2, %al
+; AVX1-NEXT:    jne .LBB9_3
+; AVX1-NEXT:  .LBB9_4: # %else2
+; AVX1-NEXT:    testb $4, %al
+; AVX1-NEXT:    jne .LBB9_5
+; AVX1-NEXT:  .LBB9_6: # %else4
+; AVX1-NEXT:    testb $8, %al
+; AVX1-NEXT:    jne .LBB9_7
+; AVX1-NEXT:  .LBB9_8: # %else6
+; AVX1-NEXT:    testb $16, %al
+; AVX1-NEXT:    jne .LBB9_9
+; AVX1-NEXT:  .LBB9_10: # %else8
+; AVX1-NEXT:    testb $32, %al
+; AVX1-NEXT:    jne .LBB9_11
+; AVX1-NEXT:  .LBB9_12: # %else10
+; AVX1-NEXT:    testb $64, %al
+; AVX1-NEXT:    jne .LBB9_13
+; AVX1-NEXT:  .LBB9_14: # %else12
+; AVX1-NEXT:    testb $-128, %al
+; AVX1-NEXT:    je .LBB9_16
+; AVX1-NEXT:  .LBB9_15: # %cond.store13
+; AVX1-NEXT:    vpextrw $7, %xmm0, 14(%rdi)
+; AVX1-NEXT:  .LBB9_16: # %else14
+; AVX1-NEXT:    testl $256, %eax # imm = 0x100
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    jne .LBB9_17
+; AVX1-NEXT:  # %bb.18: # %else16
+; AVX1-NEXT:    testl $512, %eax # imm = 0x200
+; AVX1-NEXT:    jne .LBB9_19
+; AVX1-NEXT:  .LBB9_20: # %else18
+; AVX1-NEXT:    testl $1024, %eax # imm = 0x400
+; AVX1-NEXT:    jne .LBB9_21
+; AVX1-NEXT:  .LBB9_22: # %else20
+; AVX1-NEXT:    testl $2048, %eax # imm = 0x800
+; AVX1-NEXT:    jne .LBB9_23
+; AVX1-NEXT:  .LBB9_24: # %else22
+; AVX1-NEXT:    testl $4096, %eax # imm = 0x1000
+; AVX1-NEXT:    jne .LBB9_25
+; AVX1-NEXT:  .LBB9_26: # %else24
+; AVX1-NEXT:    testl $8192, %eax # imm = 0x2000
+; AVX1-NEXT:    jne .LBB9_27
+; AVX1-NEXT:  .LBB9_28: # %else26
+; AVX1-NEXT:    testl $16384, %eax # imm = 0x4000
+; AVX1-NEXT:    jne .LBB9_29
+; AVX1-NEXT:  .LBB9_30: # %else28
+; AVX1-NEXT:    testl $32768, %eax # imm = 0x8000
+; AVX1-NEXT:    jne .LBB9_31
+; AVX1-NEXT:  .LBB9_32: # %else30
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+; AVX1-NEXT:  .LBB9_1: # %cond.store
+; AVX1-NEXT:    vpextrw $0, %xmm0, (%rdi)
+; AVX1-NEXT:    testb $2, %al
 ; AVX1-NEXT:    je .LBB9_4
-; AVX1-NEXT:  # %bb.3: # %cond.store1
+; AVX1-NEXT:  .LBB9_3: # %cond.store1
 ; AVX1-NEXT:    vpextrw $1, %xmm0, 2(%rdi)
-; AVX1-NEXT:  .LBB9_4: # %else2
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm2, %xmm5
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm5, %xmm5
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm5, %xmm5
-; AVX1-NEXT:    vpextrb $2, %xmm5, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $4, %al
 ; AVX1-NEXT:    je .LBB9_6
-; AVX1-NEXT:  # %bb.5: # %cond.store3
+; AVX1-NEXT:  .LBB9_5: # %cond.store3
 ; AVX1-NEXT:    vpextrw $2, %xmm0, 4(%rdi)
-; AVX1-NEXT:  .LBB9_6: # %else4
-; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm4, %xmm1, %xmm1
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm1, %xmm1
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $3, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $8, %al
 ; AVX1-NEXT:    je .LBB9_8
-; AVX1-NEXT:  # %bb.7: # %cond.store5
+; AVX1-NEXT:  .LBB9_7: # %cond.store5
 ; AVX1-NEXT:    vpextrw $3, %xmm0, 6(%rdi)
-; AVX1-NEXT:  .LBB9_8: # %else6
-; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm2
-; AVX1-NEXT:    vpackssdw %xmm2, %xmm0, %xmm4
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm4, %xmm4
-; AVX1-NEXT:    vpextrb $4, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $16, %al
 ; AVX1-NEXT:    je .LBB9_10
-; AVX1-NEXT:  # %bb.9: # %cond.store7
+; AVX1-NEXT:  .LBB9_9: # %cond.store7
 ; AVX1-NEXT:    vpextrw $4, %xmm0, 8(%rdi)
-; AVX1-NEXT:  .LBB9_10: # %else8
-; AVX1-NEXT:    vpackssdw %xmm2, %xmm0, %xmm2
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $5, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $32, %al
 ; AVX1-NEXT:    je .LBB9_12
-; AVX1-NEXT:  # %bb.11: # %cond.store9
+; AVX1-NEXT:  .LBB9_11: # %cond.store9
 ; AVX1-NEXT:    vpextrw $5, %xmm0, 10(%rdi)
-; AVX1-NEXT:  .LBB9_12: # %else10
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpackssdw %xmm1, %xmm0, %xmm2
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $6, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $64, %al
 ; AVX1-NEXT:    je .LBB9_14
-; AVX1-NEXT:  # %bb.13: # %cond.store11
+; AVX1-NEXT:  .LBB9_13: # %cond.store11
 ; AVX1-NEXT:    vpextrw $6, %xmm0, 12(%rdi)
-; AVX1-NEXT:  .LBB9_14: # %else12
-; AVX1-NEXT:    vpackssdw %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $7, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je .LBB9_16
-; AVX1-NEXT:  # %bb.15: # %cond.store13
-; AVX1-NEXT:    vpextrw $7, %xmm0, 14(%rdi)
-; AVX1-NEXT:  .LBB9_16: # %else14
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm3, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpxor %xmm2, %xmm4, %xmm4
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm4, %xmm4
-; AVX1-NEXT:    vpacksswb %xmm4, %xmm0, %xmm4
-; AVX1-NEXT:    vpextrb $8, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT:    je .LBB9_18
-; AVX1-NEXT:  # %bb.17: # %cond.store15
+; AVX1-NEXT:    testb $-128, %al
+; AVX1-NEXT:    jne .LBB9_15
+; AVX1-NEXT:    jmp .LBB9_16
+; AVX1-NEXT:  .LBB9_17: # %cond.store15
 ; AVX1-NEXT:    vpextrw $0, %xmm0, 16(%rdi)
-; AVX1-NEXT:  .LBB9_18: # %else16
-; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm3, %xmm1
-; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm1, %xmm1
-; AVX1-NEXT:    vpacksswb %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpextrb $9, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $512, %eax # imm = 0x200
 ; AVX1-NEXT:    je .LBB9_20
-; AVX1-NEXT:  # %bb.19: # %cond.store17
+; AVX1-NEXT:  .LBB9_19: # %cond.store17
 ; AVX1-NEXT:    vpextrw $1, %xmm0, 18(%rdi)
-; AVX1-NEXT:  .LBB9_20: # %else18
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm3, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpxor %xmm2, %xmm4, %xmm4
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm4, %xmm4
-; AVX1-NEXT:    vpacksswb %xmm4, %xmm0, %xmm4
-; AVX1-NEXT:    vpextrb $10, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $1024, %eax # imm = 0x400
 ; AVX1-NEXT:    je .LBB9_22
-; AVX1-NEXT:  # %bb.21: # %cond.store19
+; AVX1-NEXT:  .LBB9_21: # %cond.store19
 ; AVX1-NEXT:    vpextrw $2, %xmm0, 20(%rdi)
-; AVX1-NEXT:  .LBB9_22: # %else20
-; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm3, %xmm1
-; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm1, %xmm1
-; AVX1-NEXT:    vpacksswb %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpextrb $11, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $2048, %eax # imm = 0x800
 ; AVX1-NEXT:    je .LBB9_24
-; AVX1-NEXT:  # %bb.23: # %cond.store21
+; AVX1-NEXT:  .LBB9_23: # %cond.store21
 ; AVX1-NEXT:    vpextrw $3, %xmm0, 22(%rdi)
-; AVX1-NEXT:  .LBB9_24: # %else22
-; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm1
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpackssdw %xmm2, %xmm0, %xmm3
-; AVX1-NEXT:    vpacksswb %xmm3, %xmm0, %xmm3
-; AVX1-NEXT:    vpextrb $12, %xmm3, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $4096, %eax # imm = 0x1000
 ; AVX1-NEXT:    je .LBB9_26
-; AVX1-NEXT:  # %bb.25: # %cond.store23
+; AVX1-NEXT:  .LBB9_25: # %cond.store23
 ; AVX1-NEXT:    vpextrw $4, %xmm0, 24(%rdi)
-; AVX1-NEXT:  .LBB9_26: # %else24
-; AVX1-NEXT:    vpackssdw %xmm2, %xmm0, %xmm2
-; AVX1-NEXT:    vpacksswb %xmm2, %xmm0, %xmm2
-; AVX1-NEXT:    vpextrb $13, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $8192, %eax # imm = 0x2000
 ; AVX1-NEXT:    je .LBB9_28
-; AVX1-NEXT:  # %bb.27: # %cond.store25
+; AVX1-NEXT:  .LBB9_27: # %cond.store25
 ; AVX1-NEXT:    vpextrw $5, %xmm0, 26(%rdi)
-; AVX1-NEXT:  .LBB9_28: # %else26
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpackssdw %xmm1, %xmm0, %xmm2
-; AVX1-NEXT:    vpacksswb %xmm2, %xmm0, %xmm2
-; AVX1-NEXT:    vpextrb $14, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $16384, %eax # imm = 0x4000
 ; AVX1-NEXT:    je .LBB9_30
-; AVX1-NEXT:  # %bb.29: # %cond.store27
+; AVX1-NEXT:  .LBB9_29: # %cond.store27
 ; AVX1-NEXT:    vpextrw $6, %xmm0, 28(%rdi)
-; AVX1-NEXT:  .LBB9_30: # %else28
-; AVX1-NEXT:    vpackssdw %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpacksswb %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpextrb $15, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $32768, %eax # imm = 0x8000
 ; AVX1-NEXT:    je .LBB9_32
-; AVX1-NEXT:  # %bb.31: # %cond.store29
+; AVX1-NEXT:  .LBB9_31: # %cond.store29
 ; AVX1-NEXT:    vpextrw $7, %xmm0, 30(%rdi)
-; AVX1-NEXT:  .LBB9_32: # %else30
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: truncstore_v16i32_v16i16:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm2, %xmm6
-; AVX2-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
-; AVX2-NEXT:    vpxor %xmm5, %xmm6, %xmm6
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm6, %xmm6
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm6, %xmm6
-; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm7 = [65535,65535,65535,65535,65535,65535,65535,65535]
-; AVX2-NEXT:    vpminud %ymm7, %ymm1, %ymm1
-; AVX2-NEXT:    vpminud %ymm7, %ymm0, %ymm0
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm5 = [65535,65535,65535,65535,65535,65535,65535,65535]
+; AVX2-NEXT:    vpminud %ymm5, %ymm1, %ymm1
+; AVX2-NEXT:    vpminud %ymm5, %ymm0, %ymm0
 ; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX2-NEXT:    vpextrb $0, %xmm6, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je .LBB9_2
-; AVX2-NEXT:  # %bb.1: # %cond.store
+; AVX2-NEXT:    vpcmpeqd %ymm4, %ymm3, %ymm1
+; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm3, %ymm3
+; AVX2-NEXT:    vpxor %ymm3, %ymm1, %ymm1
+; AVX2-NEXT:    vpcmpeqd %ymm4, %ymm2, %ymm2
+; AVX2-NEXT:    vpxor %ymm3, %ymm2, %ymm2
+; AVX2-NEXT:    vpackssdw %ymm1, %ymm2, %ymm1
+; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
+; AVX2-NEXT:    vpacksswb %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpmovmskb %xmm1, %eax
+; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    jne .LBB9_1
+; AVX2-NEXT:  # %bb.2: # %else
+; AVX2-NEXT:    testb $2, %al
+; AVX2-NEXT:    jne .LBB9_3
+; AVX2-NEXT:  .LBB9_4: # %else2
+; AVX2-NEXT:    testb $4, %al
+; AVX2-NEXT:    jne .LBB9_5
+; AVX2-NEXT:  .LBB9_6: # %else4
+; AVX2-NEXT:    testb $8, %al
+; AVX2-NEXT:    jne .LBB9_7
+; AVX2-NEXT:  .LBB9_8: # %else6
+; AVX2-NEXT:    testb $16, %al
+; AVX2-NEXT:    jne .LBB9_9
+; AVX2-NEXT:  .LBB9_10: # %else8
+; AVX2-NEXT:    testb $32, %al
+; AVX2-NEXT:    jne .LBB9_11
+; AVX2-NEXT:  .LBB9_12: # %else10
+; AVX2-NEXT:    testb $64, %al
+; AVX2-NEXT:    jne .LBB9_13
+; AVX2-NEXT:  .LBB9_14: # %else12
+; AVX2-NEXT:    testb $-128, %al
+; AVX2-NEXT:    je .LBB9_16
+; AVX2-NEXT:  .LBB9_15: # %cond.store13
+; AVX2-NEXT:    vpextrw $7, %xmm0, 14(%rdi)
+; AVX2-NEXT:  .LBB9_16: # %else14
+; AVX2-NEXT:    testl $256, %eax # imm = 0x100
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
+; AVX2-NEXT:    jne .LBB9_17
+; AVX2-NEXT:  # %bb.18: # %else16
+; AVX2-NEXT:    testl $512, %eax # imm = 0x200
+; AVX2-NEXT:    jne .LBB9_19
+; AVX2-NEXT:  .LBB9_20: # %else18
+; AVX2-NEXT:    testl $1024, %eax # imm = 0x400
+; AVX2-NEXT:    jne .LBB9_21
+; AVX2-NEXT:  .LBB9_22: # %else20
+; AVX2-NEXT:    testl $2048, %eax # imm = 0x800
+; AVX2-NEXT:    jne .LBB9_23
+; AVX2-NEXT:  .LBB9_24: # %else22
+; AVX2-NEXT:    testl $4096, %eax # imm = 0x1000
+; AVX2-NEXT:    jne .LBB9_25
+; AVX2-NEXT:  .LBB9_26: # %else24
+; AVX2-NEXT:    testl $8192, %eax # imm = 0x2000
+; AVX2-NEXT:    jne .LBB9_27
+; AVX2-NEXT:  .LBB9_28: # %else26
+; AVX2-NEXT:    testl $16384, %eax # imm = 0x4000
+; AVX2-NEXT:    jne .LBB9_29
+; AVX2-NEXT:  .LBB9_30: # %else28
+; AVX2-NEXT:    testl $32768, %eax # imm = 0x8000
+; AVX2-NEXT:    jne .LBB9_31
+; AVX2-NEXT:  .LBB9_32: # %else30
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+; AVX2-NEXT:  .LBB9_1: # %cond.store
 ; AVX2-NEXT:    vpextrw $0, %xmm0, (%rdi)
-; AVX2-NEXT:  .LBB9_2: # %else
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm2, %xmm1
-; AVX2-NEXT:    vpxor %xmm5, %xmm1, %xmm1
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm1, %xmm1
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm1, %xmm1
-; AVX2-NEXT:    vpextrb $1, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $2, %al
 ; AVX2-NEXT:    je .LBB9_4
-; AVX2-NEXT:  # %bb.3: # %cond.store1
+; AVX2-NEXT:  .LBB9_3: # %cond.store1
 ; AVX2-NEXT:    vpextrw $1, %xmm0, 2(%rdi)
-; AVX2-NEXT:  .LBB9_4: # %else2
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm2, %xmm5
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm5, %xmm5
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm5, %xmm5
-; AVX2-NEXT:    vpextrb $2, %xmm5, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $4, %al
 ; AVX2-NEXT:    je .LBB9_6
-; AVX2-NEXT:  # %bb.5: # %cond.store3
+; AVX2-NEXT:  .LBB9_5: # %cond.store3
 ; AVX2-NEXT:    vpextrw $2, %xmm0, 4(%rdi)
-; AVX2-NEXT:  .LBB9_6: # %else4
-; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm2, %xmm1
-; AVX2-NEXT:    vpxor %xmm4, %xmm1, %xmm1
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm1, %xmm1
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm1, %xmm1
-; AVX2-NEXT:    vpextrb $3, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $8, %al
 ; AVX2-NEXT:    je .LBB9_8
-; AVX2-NEXT:  # %bb.7: # %cond.store5
+; AVX2-NEXT:  .LBB9_7: # %cond.store5
 ; AVX2-NEXT:    vpextrw $3, %xmm0, 6(%rdi)
-; AVX2-NEXT:  .LBB9_8: # %else6
-; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm1
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpxor %xmm4, %xmm2, %xmm2
-; AVX2-NEXT:    vpackssdw %xmm2, %xmm0, %xmm4
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm4, %xmm4
-; AVX2-NEXT:    vpextrb $4, %xmm4, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $16, %al
 ; AVX2-NEXT:    je .LBB9_10
-; AVX2-NEXT:  # %bb.9: # %cond.store7
+; AVX2-NEXT:  .LBB9_9: # %cond.store7
 ; AVX2-NEXT:    vpextrw $4, %xmm0, 8(%rdi)
-; AVX2-NEXT:  .LBB9_10: # %else8
-; AVX2-NEXT:    vpackssdw %xmm2, %xmm0, %xmm2
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrb $5, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $32, %al
 ; AVX2-NEXT:    je .LBB9_12
-; AVX2-NEXT:  # %bb.11: # %cond.store9
+; AVX2-NEXT:  .LBB9_11: # %cond.store9
 ; AVX2-NEXT:    vpextrw $5, %xmm0, 10(%rdi)
-; AVX2-NEXT:  .LBB9_12: # %else10
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpackssdw %xmm1, %xmm0, %xmm2
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrb $6, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $64, %al
 ; AVX2-NEXT:    je .LBB9_14
-; AVX2-NEXT:  # %bb.13: # %cond.store11
+; AVX2-NEXT:  .LBB9_13: # %cond.store11
 ; AVX2-NEXT:    vpextrw $6, %xmm0, 12(%rdi)
-; AVX2-NEXT:  .LBB9_14: # %else12
-; AVX2-NEXT:    vpackssdw %xmm1, %xmm0, %xmm1
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm1, %xmm1
-; AVX2-NEXT:    vpextrb $7, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je .LBB9_16
-; AVX2-NEXT:  # %bb.15: # %cond.store13
-; AVX2-NEXT:    vpextrw $7, %xmm0, 14(%rdi)
-; AVX2-NEXT:  .LBB9_16: # %else14
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm3, %xmm4
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpxor %xmm2, %xmm4, %xmm4
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm4, %xmm4
-; AVX2-NEXT:    vpacksswb %xmm4, %xmm0, %xmm4
-; AVX2-NEXT:    vpextrb $8, %xmm4, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; AVX2-NEXT:    je .LBB9_18
-; AVX2-NEXT:  # %bb.17: # %cond.store15
+; AVX2-NEXT:    testb $-128, %al
+; AVX2-NEXT:    jne .LBB9_15
+; AVX2-NEXT:    jmp .LBB9_16
+; AVX2-NEXT:  .LBB9_17: # %cond.store15
 ; AVX2-NEXT:    vpextrw $0, %xmm0, 16(%rdi)
-; AVX2-NEXT:  .LBB9_18: # %else16
-; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm3, %xmm1
-; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm1, %xmm1
-; AVX2-NEXT:    vpacksswb %xmm1, %xmm0, %xmm1
-; AVX2-NEXT:    vpextrb $9, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $512, %eax # imm = 0x200
 ; AVX2-NEXT:    je .LBB9_20
-; AVX2-NEXT:  # %bb.19: # %cond.store17
+; AVX2-NEXT:  .LBB9_19: # %cond.store17
 ; AVX2-NEXT:    vpextrw $1, %xmm0, 18(%rdi)
-; AVX2-NEXT:  .LBB9_20: # %else18
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm3, %xmm4
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpxor %xmm2, %xmm4, %xmm4
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm4, %xmm4
-; AVX2-NEXT:    vpacksswb %xmm4, %xmm0, %xmm4
-; AVX2-NEXT:    vpextrb $10, %xmm4, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $1024, %eax # imm = 0x400
 ; AVX2-NEXT:    je .LBB9_22
-; AVX2-NEXT:  # %bb.21: # %cond.store19
+; AVX2-NEXT:  .LBB9_21: # %cond.store19
 ; AVX2-NEXT:    vpextrw $2, %xmm0, 20(%rdi)
-; AVX2-NEXT:  .LBB9_22: # %else20
-; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm3, %xmm1
-; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm1, %xmm1
-; AVX2-NEXT:    vpacksswb %xmm1, %xmm0, %xmm1
-; AVX2-NEXT:    vpextrb $11, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $2048, %eax # imm = 0x800
 ; AVX2-NEXT:    je .LBB9_24
-; AVX2-NEXT:  # %bb.23: # %cond.store21
+; AVX2-NEXT:  .LBB9_23: # %cond.store21
 ; AVX2-NEXT:    vpextrw $3, %xmm0, 22(%rdi)
-; AVX2-NEXT:  .LBB9_24: # %else22
-; AVX2-NEXT:    vextracti128 $1, %ymm3, %xmm1
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX2-NEXT:    vpackssdw %xmm2, %xmm0, %xmm3
-; AVX2-NEXT:    vpacksswb %xmm3, %xmm0, %xmm3
-; AVX2-NEXT:    vpextrb $12, %xmm3, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $4096, %eax # imm = 0x1000
 ; AVX2-NEXT:    je .LBB9_26
-; AVX2-NEXT:  # %bb.25: # %cond.store23
+; AVX2-NEXT:  .LBB9_25: # %cond.store23
 ; AVX2-NEXT:    vpextrw $4, %xmm0, 24(%rdi)
-; AVX2-NEXT:  .LBB9_26: # %else24
-; AVX2-NEXT:    vpackssdw %xmm2, %xmm0, %xmm2
-; AVX2-NEXT:    vpacksswb %xmm2, %xmm0, %xmm2
-; AVX2-NEXT:    vpextrb $13, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $8192, %eax # imm = 0x2000
 ; AVX2-NEXT:    je .LBB9_28
-; AVX2-NEXT:  # %bb.27: # %cond.store25
+; AVX2-NEXT:  .LBB9_27: # %cond.store25
 ; AVX2-NEXT:    vpextrw $5, %xmm0, 26(%rdi)
-; AVX2-NEXT:  .LBB9_28: # %else26
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpackssdw %xmm1, %xmm0, %xmm2
-; AVX2-NEXT:    vpacksswb %xmm2, %xmm0, %xmm2
-; AVX2-NEXT:    vpextrb $14, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $16384, %eax # imm = 0x4000
 ; AVX2-NEXT:    je .LBB9_30
-; AVX2-NEXT:  # %bb.29: # %cond.store27
+; AVX2-NEXT:  .LBB9_29: # %cond.store27
 ; AVX2-NEXT:    vpextrw $6, %xmm0, 28(%rdi)
-; AVX2-NEXT:  .LBB9_30: # %else28
-; AVX2-NEXT:    vpackssdw %xmm1, %xmm0, %xmm1
-; AVX2-NEXT:    vpacksswb %xmm1, %xmm0, %xmm1
-; AVX2-NEXT:    vpextrb $15, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $32768, %eax # imm = 0x8000
 ; AVX2-NEXT:    je .LBB9_32
-; AVX2-NEXT:  # %bb.31: # %cond.store29
+; AVX2-NEXT:  .LBB9_31: # %cond.store29
 ; AVX2-NEXT:    vpextrw $7, %xmm0, 30(%rdi)
-; AVX2-NEXT:  .LBB9_32: # %else30
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
@@ -3272,116 +3100,117 @@ define void @truncstore_v16i32_v16i16(<16 x i32> %x, <16 x i16>* %p, <16 x i32>
 ; AVX512F-NEXT:    vpmovusdw %zmm0, %ymm0
 ; AVX512F-NEXT:    kmovw %k0, %eax
 ; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB9_2
-; AVX512F-NEXT:  # %bb.1: # %cond.store
-; AVX512F-NEXT:    vpextrw $0, %xmm0, (%rdi)
-; AVX512F-NEXT:  .LBB9_2: # %else
-; AVX512F-NEXT:    kshiftrw $1, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB9_4
-; AVX512F-NEXT:  # %bb.3: # %cond.store1
-; AVX512F-NEXT:    vpextrw $1, %xmm0, 2(%rdi)
+; AVX512F-NEXT:    jne .LBB9_1
+; AVX512F-NEXT:  # %bb.2: # %else
+; AVX512F-NEXT:    testb $2, %al
+; AVX512F-NEXT:    jne .LBB9_3
 ; AVX512F-NEXT:  .LBB9_4: # %else2
-; AVX512F-NEXT:    kshiftrw $2, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB9_6
-; AVX512F-NEXT:  # %bb.5: # %cond.store3
-; AVX512F-NEXT:    vpextrw $2, %xmm0, 4(%rdi)
+; AVX512F-NEXT:    testb $4, %al
+; AVX512F-NEXT:    jne .LBB9_5
 ; AVX512F-NEXT:  .LBB9_6: # %else4
-; AVX512F-NEXT:    kshiftrw $3, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB9_8
-; AVX512F-NEXT:  # %bb.7: # %cond.store5
-; AVX512F-NEXT:    vpextrw $3, %xmm0, 6(%rdi)
+; AVX512F-NEXT:    testb $8, %al
+; AVX512F-NEXT:    jne .LBB9_7
 ; AVX512F-NEXT:  .LBB9_8: # %else6
-; AVX512F-NEXT:    kshiftrw $4, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB9_10
-; AVX512F-NEXT:  # %bb.9: # %cond.store7
-; AVX512F-NEXT:    vpextrw $4, %xmm0, 8(%rdi)
+; AVX512F-NEXT:    testb $16, %al
+; AVX512F-NEXT:    jne .LBB9_9
 ; AVX512F-NEXT:  .LBB9_10: # %else8
-; AVX512F-NEXT:    kshiftrw $5, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB9_12
-; AVX512F-NEXT:  # %bb.11: # %cond.store9
-; AVX512F-NEXT:    vpextrw $5, %xmm0, 10(%rdi)
+; AVX512F-NEXT:    testb $32, %al
+; AVX512F-NEXT:    jne .LBB9_11
 ; AVX512F-NEXT:  .LBB9_12: # %else10
-; AVX512F-NEXT:    kshiftrw $6, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB9_14
-; AVX512F-NEXT:  # %bb.13: # %cond.store11
-; AVX512F-NEXT:    vpextrw $6, %xmm0, 12(%rdi)
+; AVX512F-NEXT:    testb $64, %al
+; AVX512F-NEXT:    jne .LBB9_13
 ; AVX512F-NEXT:  .LBB9_14: # %else12
-; AVX512F-NEXT:    kshiftrw $7, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $-128, %al
 ; AVX512F-NEXT:    je .LBB9_16
-; AVX512F-NEXT:  # %bb.15: # %cond.store13
+; AVX512F-NEXT:  .LBB9_15: # %cond.store13
 ; AVX512F-NEXT:    vpextrw $7, %xmm0, 14(%rdi)
 ; AVX512F-NEXT:  .LBB9_16: # %else14
-; AVX512F-NEXT:    kshiftrw $8, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $256, %eax # imm = 0x100
 ; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; AVX512F-NEXT:    je .LBB9_18
-; AVX512F-NEXT:  # %bb.17: # %cond.store15
+; AVX512F-NEXT:    jne .LBB9_17
+; AVX512F-NEXT:  # %bb.18: # %else16
+; AVX512F-NEXT:    testl $512, %eax # imm = 0x200
+; AVX512F-NEXT:    jne .LBB9_19
+; AVX512F-NEXT:  .LBB9_20: # %else18
+; AVX512F-NEXT:    testl $1024, %eax # imm = 0x400
+; AVX512F-NEXT:    jne .LBB9_21
+; AVX512F-NEXT:  .LBB9_22: # %else20
+; AVX512F-NEXT:    testl $2048, %eax # imm = 0x800
+; AVX512F-NEXT:    jne .LBB9_23
+; AVX512F-NEXT:  .LBB9_24: # %else22
+; AVX512F-NEXT:    testl $4096, %eax # imm = 0x1000
+; AVX512F-NEXT:    jne .LBB9_25
+; AVX512F-NEXT:  .LBB9_26: # %else24
+; AVX512F-NEXT:    testl $8192, %eax # imm = 0x2000
+; AVX512F-NEXT:    jne .LBB9_27
+; AVX512F-NEXT:  .LBB9_28: # %else26
+; AVX512F-NEXT:    testl $16384, %eax # imm = 0x4000
+; AVX512F-NEXT:    jne .LBB9_29
+; AVX512F-NEXT:  .LBB9_30: # %else28
+; AVX512F-NEXT:    testl $32768, %eax # imm = 0x8000
+; AVX512F-NEXT:    jne .LBB9_31
+; AVX512F-NEXT:  .LBB9_32: # %else30
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+; AVX512F-NEXT:  .LBB9_1: # %cond.store
+; AVX512F-NEXT:    vpextrw $0, %xmm0, (%rdi)
+; AVX512F-NEXT:    testb $2, %al
+; AVX512F-NEXT:    je .LBB9_4
+; AVX512F-NEXT:  .LBB9_3: # %cond.store1
+; AVX512F-NEXT:    vpextrw $1, %xmm0, 2(%rdi)
+; AVX512F-NEXT:    testb $4, %al
+; AVX512F-NEXT:    je .LBB9_6
+; AVX512F-NEXT:  .LBB9_5: # %cond.store3
+; AVX512F-NEXT:    vpextrw $2, %xmm0, 4(%rdi)
+; AVX512F-NEXT:    testb $8, %al
+; AVX512F-NEXT:    je .LBB9_8
+; AVX512F-NEXT:  .LBB9_7: # %cond.store5
+; AVX512F-NEXT:    vpextrw $3, %xmm0, 6(%rdi)
+; AVX512F-NEXT:    testb $16, %al
+; AVX512F-NEXT:    je .LBB9_10
+; AVX512F-NEXT:  .LBB9_9: # %cond.store7
+; AVX512F-NEXT:    vpextrw $4, %xmm0, 8(%rdi)
+; AVX512F-NEXT:    testb $32, %al
+; AVX512F-NEXT:    je .LBB9_12
+; AVX512F-NEXT:  .LBB9_11: # %cond.store9
+; AVX512F-NEXT:    vpextrw $5, %xmm0, 10(%rdi)
+; AVX512F-NEXT:    testb $64, %al
+; AVX512F-NEXT:    je .LBB9_14
+; AVX512F-NEXT:  .LBB9_13: # %cond.store11
+; AVX512F-NEXT:    vpextrw $6, %xmm0, 12(%rdi)
+; AVX512F-NEXT:    testb $-128, %al
+; AVX512F-NEXT:    jne .LBB9_15
+; AVX512F-NEXT:    jmp .LBB9_16
+; AVX512F-NEXT:  .LBB9_17: # %cond.store15
 ; AVX512F-NEXT:    vpextrw $0, %xmm0, 16(%rdi)
-; AVX512F-NEXT:  .LBB9_18: # %else16
-; AVX512F-NEXT:    kshiftrw $9, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $512, %eax # imm = 0x200
 ; AVX512F-NEXT:    je .LBB9_20
-; AVX512F-NEXT:  # %bb.19: # %cond.store17
+; AVX512F-NEXT:  .LBB9_19: # %cond.store17
 ; AVX512F-NEXT:    vpextrw $1, %xmm0, 18(%rdi)
-; AVX512F-NEXT:  .LBB9_20: # %else18
-; AVX512F-NEXT:    kshiftrw $10, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $1024, %eax # imm = 0x400
 ; AVX512F-NEXT:    je .LBB9_22
-; AVX512F-NEXT:  # %bb.21: # %cond.store19
+; AVX512F-NEXT:  .LBB9_21: # %cond.store19
 ; AVX512F-NEXT:    vpextrw $2, %xmm0, 20(%rdi)
-; AVX512F-NEXT:  .LBB9_22: # %else20
-; AVX512F-NEXT:    kshiftrw $11, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $2048, %eax # imm = 0x800
 ; AVX512F-NEXT:    je .LBB9_24
-; AVX512F-NEXT:  # %bb.23: # %cond.store21
+; AVX512F-NEXT:  .LBB9_23: # %cond.store21
 ; AVX512F-NEXT:    vpextrw $3, %xmm0, 22(%rdi)
-; AVX512F-NEXT:  .LBB9_24: # %else22
-; AVX512F-NEXT:    kshiftrw $12, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $4096, %eax # imm = 0x1000
 ; AVX512F-NEXT:    je .LBB9_26
-; AVX512F-NEXT:  # %bb.25: # %cond.store23
+; AVX512F-NEXT:  .LBB9_25: # %cond.store23
 ; AVX512F-NEXT:    vpextrw $4, %xmm0, 24(%rdi)
-; AVX512F-NEXT:  .LBB9_26: # %else24
-; AVX512F-NEXT:    kshiftrw $13, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $8192, %eax # imm = 0x2000
 ; AVX512F-NEXT:    je .LBB9_28
-; AVX512F-NEXT:  # %bb.27: # %cond.store25
+; AVX512F-NEXT:  .LBB9_27: # %cond.store25
 ; AVX512F-NEXT:    vpextrw $5, %xmm0, 26(%rdi)
-; AVX512F-NEXT:  .LBB9_28: # %else26
-; AVX512F-NEXT:    kshiftrw $14, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $16384, %eax # imm = 0x4000
 ; AVX512F-NEXT:    je .LBB9_30
-; AVX512F-NEXT:  # %bb.29: # %cond.store27
+; AVX512F-NEXT:  .LBB9_29: # %cond.store27
 ; AVX512F-NEXT:    vpextrw $6, %xmm0, 28(%rdi)
-; AVX512F-NEXT:  .LBB9_30: # %else28
-; AVX512F-NEXT:    kshiftrw $15, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $32768, %eax # imm = 0x8000
 ; AVX512F-NEXT:    je .LBB9_32
-; AVX512F-NEXT:  # %bb.31: # %cond.store29
+; AVX512F-NEXT:  .LBB9_31: # %cond.store29
 ; AVX512F-NEXT:    vpextrw $7, %xmm0, 30(%rdi)
-; AVX512F-NEXT:  .LBB9_32: # %else30
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
@@ -3412,7 +3241,6 @@ define void @truncstore_v16i32_v16i8(<16 x i32> %x, <16 x i8>* %p, <16 x i32> %m
 ; SSE2-LABEL: truncstore_v16i32_v16i8:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pxor %xmm8, %xmm8
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm8
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [255,255,255,255]
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm11 = [2147483648,2147483648,2147483648,2147483648]
 ; SSE2-NEXT:    movdqa %xmm1, %xmm12
@@ -3445,158 +3273,124 @@ define void @truncstore_v16i32_v16i8(<16 x i32> %x, <16 x i8>* %p, <16 x i32> %m
 ; SSE2-NEXT:    por %xmm2, %xmm9
 ; SSE2-NEXT:    packuswb %xmm1, %xmm9
 ; SSE2-NEXT:    packuswb %xmm9, %xmm12
-; SSE2-NEXT:    movd %xmm8, %eax
-; SSE2-NEXT:    notl %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    movd %xmm12, %eax
-; SSE2-NEXT:    je .LBB10_2
-; SSE2-NEXT:  # %bb.1: # %cond.store
-; SSE2-NEXT:    movb %al, (%rdi)
-; SSE2-NEXT:  .LBB10_2: # %else
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm7
 ; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE2-NEXT:    pxor %xmm0, %xmm8
-; SSE2-NEXT:    pextrw $2, %xmm8, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    je .LBB10_4
-; SSE2-NEXT:  # %bb.3: # %cond.store1
-; SSE2-NEXT:    movb %ah, 1(%rdi)
+; SSE2-NEXT:    pxor %xmm0, %xmm7
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm6
+; SSE2-NEXT:    pxor %xmm0, %xmm6
+; SSE2-NEXT:    packssdw %xmm7, %xmm6
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm5
+; SSE2-NEXT:    pxor %xmm0, %xmm5
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm4
+; SSE2-NEXT:    pxor %xmm0, %xmm4
+; SSE2-NEXT:    packssdw %xmm5, %xmm4
+; SSE2-NEXT:    packsswb %xmm6, %xmm4
+; SSE2-NEXT:    pmovmskb %xmm4, %eax
+; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    movd %xmm12, %ecx
+; SSE2-NEXT:    jne .LBB10_1
+; SSE2-NEXT:  # %bb.2: # %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne .LBB10_3
 ; SSE2-NEXT:  .LBB10_4: # %else2
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm4
-; SSE2-NEXT:    pxor %xmm4, %xmm0
-; SSE2-NEXT:    pextrw $4, %xmm0, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    je .LBB10_6
-; SSE2-NEXT:  # %bb.5: # %cond.store3
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    shrl $16, %ecx
-; SSE2-NEXT:    movb %cl, 2(%rdi)
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne .LBB10_5
 ; SSE2-NEXT:  .LBB10_6: # %else4
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE2-NEXT:    pxor %xmm0, %xmm4
-; SSE2-NEXT:    pextrw $6, %xmm4, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je .LBB10_8
-; SSE2-NEXT:  # %bb.7: # %cond.store5
-; SSE2-NEXT:    shrl $24, %eax
-; SSE2-NEXT:    movb %al, 3(%rdi)
+; SSE2-NEXT:  .LBB10_7: # %cond.store5
+; SSE2-NEXT:    shrl $24, %ecx
+; SSE2-NEXT:    movb %cl, 3(%rdi)
 ; SSE2-NEXT:  .LBB10_8: # %else6
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm0
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    notl %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    pextrw $2, %xmm12, %eax
+; SSE2-NEXT:    testb $16, %al
+; SSE2-NEXT:    pextrw $2, %xmm12, %ecx
 ; SSE2-NEXT:    je .LBB10_10
 ; SSE2-NEXT:  # %bb.9: # %cond.store7
-; SSE2-NEXT:    movb %al, 4(%rdi)
+; SSE2-NEXT:    movb %cl, 4(%rdi)
 ; SSE2-NEXT:  .LBB10_10: # %else8
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT:    pxor %xmm2, %xmm0
-; SSE2-NEXT:    pextrw $2, %xmm0, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testb $32, %al
 ; SSE2-NEXT:    je .LBB10_12
 ; SSE2-NEXT:  # %bb.11: # %cond.store9
-; SSE2-NEXT:    movb %ah, 5(%rdi)
+; SSE2-NEXT:    movb %ch, 5(%rdi)
 ; SSE2-NEXT:  .LBB10_12: # %else10
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm5
-; SSE2-NEXT:    pxor %xmm5, %xmm2
-; SSE2-NEXT:    pextrw $4, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    pextrw $3, %xmm12, %eax
+; SSE2-NEXT:    testb $64, %al
+; SSE2-NEXT:    pextrw $3, %xmm12, %ecx
 ; SSE2-NEXT:    je .LBB10_14
 ; SSE2-NEXT:  # %bb.13: # %cond.store11
-; SSE2-NEXT:    movb %al, 6(%rdi)
+; SSE2-NEXT:    movb %cl, 6(%rdi)
 ; SSE2-NEXT:  .LBB10_14: # %else12
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE2-NEXT:    pxor %xmm0, %xmm5
-; SSE2-NEXT:    pextrw $6, %xmm5, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testb $-128, %al
 ; SSE2-NEXT:    je .LBB10_16
 ; SSE2-NEXT:  # %bb.15: # %cond.store13
-; SSE2-NEXT:    movb %ah, 7(%rdi)
+; SSE2-NEXT:    movb %ch, 7(%rdi)
 ; SSE2-NEXT:  .LBB10_16: # %else14
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqd %xmm6, %xmm0
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    notl %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    pextrw $4, %xmm12, %eax
+; SSE2-NEXT:    testl $256, %eax # imm = 0x100
+; SSE2-NEXT:    pextrw $4, %xmm12, %ecx
 ; SSE2-NEXT:    je .LBB10_18
 ; SSE2-NEXT:  # %bb.17: # %cond.store15
-; SSE2-NEXT:    movb %al, 8(%rdi)
+; SSE2-NEXT:    movb %cl, 8(%rdi)
 ; SSE2-NEXT:  .LBB10_18: # %else16
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT:    pxor %xmm2, %xmm0
-; SSE2-NEXT:    pextrw $2, %xmm0, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $512, %eax # imm = 0x200
 ; SSE2-NEXT:    je .LBB10_20
 ; SSE2-NEXT:  # %bb.19: # %cond.store17
-; SSE2-NEXT:    movb %ah, 9(%rdi)
+; SSE2-NEXT:    movb %ch, 9(%rdi)
 ; SSE2-NEXT:  .LBB10_20: # %else18
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm6
-; SSE2-NEXT:    pxor %xmm6, %xmm2
-; SSE2-NEXT:    pextrw $4, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    pextrw $5, %xmm12, %eax
+; SSE2-NEXT:    testl $1024, %eax # imm = 0x400
+; SSE2-NEXT:    pextrw $5, %xmm12, %ecx
 ; SSE2-NEXT:    je .LBB10_22
 ; SSE2-NEXT:  # %bb.21: # %cond.store19
-; SSE2-NEXT:    movb %al, 10(%rdi)
+; SSE2-NEXT:    movb %cl, 10(%rdi)
 ; SSE2-NEXT:  .LBB10_22: # %else20
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE2-NEXT:    pxor %xmm0, %xmm6
-; SSE2-NEXT:    pextrw $6, %xmm6, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $2048, %eax # imm = 0x800
 ; SSE2-NEXT:    je .LBB10_24
 ; SSE2-NEXT:  # %bb.23: # %cond.store21
-; SSE2-NEXT:    movb %ah, 11(%rdi)
+; SSE2-NEXT:    movb %ch, 11(%rdi)
 ; SSE2-NEXT:  .LBB10_24: # %else22
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqd %xmm7, %xmm0
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    notl %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    pextrw $6, %xmm12, %eax
+; SSE2-NEXT:    testl $4096, %eax # imm = 0x1000
+; SSE2-NEXT:    pextrw $6, %xmm12, %ecx
 ; SSE2-NEXT:    je .LBB10_26
 ; SSE2-NEXT:  # %bb.25: # %cond.store23
-; SSE2-NEXT:    movb %al, 12(%rdi)
+; SSE2-NEXT:    movb %cl, 12(%rdi)
 ; SSE2-NEXT:  .LBB10_26: # %else24
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT:    pxor %xmm2, %xmm0
-; SSE2-NEXT:    pextrw $2, %xmm0, %ecx
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $8192, %eax # imm = 0x2000
 ; SSE2-NEXT:    je .LBB10_28
 ; SSE2-NEXT:  # %bb.27: # %cond.store25
-; SSE2-NEXT:    movb %ah, 13(%rdi)
+; SSE2-NEXT:    movb %ch, 13(%rdi)
 ; SSE2-NEXT:  .LBB10_28: # %else26
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm7
-; SSE2-NEXT:    pxor %xmm7, %xmm2
-; SSE2-NEXT:    pextrw $4, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    pextrw $7, %xmm12, %eax
-; SSE2-NEXT:    je .LBB10_30
-; SSE2-NEXT:  # %bb.29: # %cond.store27
-; SSE2-NEXT:    movb %al, 14(%rdi)
-; SSE2-NEXT:  .LBB10_30: # %else28
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE2-NEXT:    pxor %xmm0, %xmm7
-; SSE2-NEXT:    pextrw $6, %xmm7, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    je .LBB10_32
-; SSE2-NEXT:  # %bb.31: # %cond.store29
-; SSE2-NEXT:    movb %ah, 15(%rdi)
+; SSE2-NEXT:    testl $16384, %eax # imm = 0x4000
+; SSE2-NEXT:    pextrw $7, %xmm12, %ecx
+; SSE2-NEXT:    jne .LBB10_29
+; SSE2-NEXT:  # %bb.30: # %else28
+; SSE2-NEXT:    testl $32768, %eax # imm = 0x8000
+; SSE2-NEXT:    jne .LBB10_31
 ; SSE2-NEXT:  .LBB10_32: # %else30
 ; SSE2-NEXT:    retq
+; SSE2-NEXT:  .LBB10_1: # %cond.store
+; SSE2-NEXT:    movb %cl, (%rdi)
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    je .LBB10_4
+; SSE2-NEXT:  .LBB10_3: # %cond.store1
+; SSE2-NEXT:    movb %ch, 1(%rdi)
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    je .LBB10_6
+; SSE2-NEXT:  .LBB10_5: # %cond.store3
+; SSE2-NEXT:    movl %ecx, %edx
+; SSE2-NEXT:    shrl $16, %edx
+; SSE2-NEXT:    movb %dl, 2(%rdi)
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne .LBB10_7
+; SSE2-NEXT:    jmp .LBB10_8
+; SSE2-NEXT:  .LBB10_29: # %cond.store27
+; SSE2-NEXT:    movb %cl, 14(%rdi)
+; SSE2-NEXT:    testl $32768, %eax # imm = 0x8000
+; SSE2-NEXT:    je .LBB10_32
+; SSE2-NEXT:  .LBB10_31: # %cond.store29
+; SSE2-NEXT:    movb %ch, 15(%rdi)
+; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: truncstore_v16i32_v16i8:
 ; SSE4:       # %bb.0:
-; SSE4-NEXT:    pxor %xmm9, %xmm9
-; SSE4-NEXT:    pcmpeqd %xmm4, %xmm9
-; SSE4-NEXT:    pcmpeqd %xmm8, %xmm8
-; SSE4-NEXT:    pxor %xmm9, %xmm8
+; SSE4-NEXT:    pxor %xmm8, %xmm8
 ; SSE4-NEXT:    movdqa {{.*#+}} xmm9 = [255,255,255,255]
 ; SSE4-NEXT:    pminud %xmm9, %xmm1
 ; SSE4-NEXT:    pminud %xmm9, %xmm0
@@ -3605,496 +3399,407 @@ define void @truncstore_v16i32_v16i8(<16 x i32> %x, <16 x i8>* %p, <16 x i32> %m
 ; SSE4-NEXT:    pminud %xmm9, %xmm2
 ; SSE4-NEXT:    packusdw %xmm3, %xmm2
 ; SSE4-NEXT:    packuswb %xmm2, %xmm0
-; SSE4-NEXT:    pextrb $0, %xmm8, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB10_2
-; SSE4-NEXT:  # %bb.1: # %cond.store
+; SSE4-NEXT:    pcmpeqd %xmm8, %xmm7
+; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
+; SSE4-NEXT:    pxor %xmm1, %xmm7
+; SSE4-NEXT:    pcmpeqd %xmm8, %xmm6
+; SSE4-NEXT:    pxor %xmm1, %xmm6
+; SSE4-NEXT:    packssdw %xmm7, %xmm6
+; SSE4-NEXT:    pcmpeqd %xmm8, %xmm5
+; SSE4-NEXT:    pxor %xmm1, %xmm5
+; SSE4-NEXT:    pcmpeqd %xmm8, %xmm4
+; SSE4-NEXT:    pxor %xmm1, %xmm4
+; SSE4-NEXT:    packssdw %xmm5, %xmm4
+; SSE4-NEXT:    packsswb %xmm6, %xmm4
+; SSE4-NEXT:    pmovmskb %xmm4, %eax
+; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    jne .LBB10_1
+; SSE4-NEXT:  # %bb.2: # %else
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    jne .LBB10_3
+; SSE4-NEXT:  .LBB10_4: # %else2
+; SSE4-NEXT:    testb $4, %al
+; SSE4-NEXT:    jne .LBB10_5
+; SSE4-NEXT:  .LBB10_6: # %else4
+; SSE4-NEXT:    testb $8, %al
+; SSE4-NEXT:    jne .LBB10_7
+; SSE4-NEXT:  .LBB10_8: # %else6
+; SSE4-NEXT:    testb $16, %al
+; SSE4-NEXT:    jne .LBB10_9
+; SSE4-NEXT:  .LBB10_10: # %else8
+; SSE4-NEXT:    testb $32, %al
+; SSE4-NEXT:    jne .LBB10_11
+; SSE4-NEXT:  .LBB10_12: # %else10
+; SSE4-NEXT:    testb $64, %al
+; SSE4-NEXT:    jne .LBB10_13
+; SSE4-NEXT:  .LBB10_14: # %else12
+; SSE4-NEXT:    testb $-128, %al
+; SSE4-NEXT:    jne .LBB10_15
+; SSE4-NEXT:  .LBB10_16: # %else14
+; SSE4-NEXT:    testl $256, %eax # imm = 0x100
+; SSE4-NEXT:    jne .LBB10_17
+; SSE4-NEXT:  .LBB10_18: # %else16
+; SSE4-NEXT:    testl $512, %eax # imm = 0x200
+; SSE4-NEXT:    jne .LBB10_19
+; SSE4-NEXT:  .LBB10_20: # %else18
+; SSE4-NEXT:    testl $1024, %eax # imm = 0x400
+; SSE4-NEXT:    jne .LBB10_21
+; SSE4-NEXT:  .LBB10_22: # %else20
+; SSE4-NEXT:    testl $2048, %eax # imm = 0x800
+; SSE4-NEXT:    jne .LBB10_23
+; SSE4-NEXT:  .LBB10_24: # %else22
+; SSE4-NEXT:    testl $4096, %eax # imm = 0x1000
+; SSE4-NEXT:    jne .LBB10_25
+; SSE4-NEXT:  .LBB10_26: # %else24
+; SSE4-NEXT:    testl $8192, %eax # imm = 0x2000
+; SSE4-NEXT:    jne .LBB10_27
+; SSE4-NEXT:  .LBB10_28: # %else26
+; SSE4-NEXT:    testl $16384, %eax # imm = 0x4000
+; SSE4-NEXT:    jne .LBB10_29
+; SSE4-NEXT:  .LBB10_30: # %else28
+; SSE4-NEXT:    testl $32768, %eax # imm = 0x8000
+; SSE4-NEXT:    jne .LBB10_31
+; SSE4-NEXT:  .LBB10_32: # %else30
+; SSE4-NEXT:    retq
+; SSE4-NEXT:  .LBB10_1: # %cond.store
 ; SSE4-NEXT:    pextrb $0, %xmm0, (%rdi)
-; SSE4-NEXT:  .LBB10_2: # %else
-; SSE4-NEXT:    pextrb $4, %xmm8, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $2, %al
 ; SSE4-NEXT:    je .LBB10_4
-; SSE4-NEXT:  # %bb.3: # %cond.store1
+; SSE4-NEXT:  .LBB10_3: # %cond.store1
 ; SSE4-NEXT:    pextrb $1, %xmm0, 1(%rdi)
-; SSE4-NEXT:  .LBB10_4: # %else2
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm4
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE4-NEXT:    pxor %xmm4, %xmm1
-; SSE4-NEXT:    pextrb $8, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $4, %al
 ; SSE4-NEXT:    je .LBB10_6
-; SSE4-NEXT:  # %bb.5: # %cond.store3
+; SSE4-NEXT:  .LBB10_5: # %cond.store3
 ; SSE4-NEXT:    pextrb $2, %xmm0, 2(%rdi)
-; SSE4-NEXT:  .LBB10_6: # %else4
-; SSE4-NEXT:    pextrb $12, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $8, %al
 ; SSE4-NEXT:    je .LBB10_8
-; SSE4-NEXT:  # %bb.7: # %cond.store5
+; SSE4-NEXT:  .LBB10_7: # %cond.store5
 ; SSE4-NEXT:    pextrb $3, %xmm0, 3(%rdi)
-; SSE4-NEXT:  .LBB10_8: # %else6
-; SSE4-NEXT:    pxor %xmm2, %xmm2
-; SSE4-NEXT:    pcmpeqd %xmm5, %xmm2
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE4-NEXT:    pxor %xmm2, %xmm1
-; SSE4-NEXT:    pextrb $0, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $16, %al
 ; SSE4-NEXT:    je .LBB10_10
-; SSE4-NEXT:  # %bb.9: # %cond.store7
+; SSE4-NEXT:  .LBB10_9: # %cond.store7
 ; SSE4-NEXT:    pextrb $4, %xmm0, 4(%rdi)
-; SSE4-NEXT:  .LBB10_10: # %else8
-; SSE4-NEXT:    pextrb $4, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $32, %al
 ; SSE4-NEXT:    je .LBB10_12
-; SSE4-NEXT:  # %bb.11: # %cond.store9
+; SSE4-NEXT:  .LBB10_11: # %cond.store9
 ; SSE4-NEXT:    pextrb $5, %xmm0, 5(%rdi)
-; SSE4-NEXT:  .LBB10_12: # %else10
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm5
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE4-NEXT:    pxor %xmm5, %xmm1
-; SSE4-NEXT:    pextrb $8, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $64, %al
 ; SSE4-NEXT:    je .LBB10_14
-; SSE4-NEXT:  # %bb.13: # %cond.store11
+; SSE4-NEXT:  .LBB10_13: # %cond.store11
 ; SSE4-NEXT:    pextrb $6, %xmm0, 6(%rdi)
-; SSE4-NEXT:  .LBB10_14: # %else12
-; SSE4-NEXT:    pextrb $12, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $-128, %al
 ; SSE4-NEXT:    je .LBB10_16
-; SSE4-NEXT:  # %bb.15: # %cond.store13
+; SSE4-NEXT:  .LBB10_15: # %cond.store13
 ; SSE4-NEXT:    pextrb $7, %xmm0, 7(%rdi)
-; SSE4-NEXT:  .LBB10_16: # %else14
-; SSE4-NEXT:    pxor %xmm2, %xmm2
-; SSE4-NEXT:    pcmpeqd %xmm6, %xmm2
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE4-NEXT:    pxor %xmm2, %xmm1
-; SSE4-NEXT:    pextrb $0, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $256, %eax # imm = 0x100
 ; SSE4-NEXT:    je .LBB10_18
-; SSE4-NEXT:  # %bb.17: # %cond.store15
+; SSE4-NEXT:  .LBB10_17: # %cond.store15
 ; SSE4-NEXT:    pextrb $8, %xmm0, 8(%rdi)
-; SSE4-NEXT:  .LBB10_18: # %else16
-; SSE4-NEXT:    pextrb $4, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $512, %eax # imm = 0x200
 ; SSE4-NEXT:    je .LBB10_20
-; SSE4-NEXT:  # %bb.19: # %cond.store17
+; SSE4-NEXT:  .LBB10_19: # %cond.store17
 ; SSE4-NEXT:    pextrb $9, %xmm0, 9(%rdi)
-; SSE4-NEXT:  .LBB10_20: # %else18
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm6
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE4-NEXT:    pxor %xmm6, %xmm1
-; SSE4-NEXT:    pextrb $8, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $1024, %eax # imm = 0x400
 ; SSE4-NEXT:    je .LBB10_22
-; SSE4-NEXT:  # %bb.21: # %cond.store19
+; SSE4-NEXT:  .LBB10_21: # %cond.store19
 ; SSE4-NEXT:    pextrb $10, %xmm0, 10(%rdi)
-; SSE4-NEXT:  .LBB10_22: # %else20
-; SSE4-NEXT:    pextrb $12, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $2048, %eax # imm = 0x800
 ; SSE4-NEXT:    je .LBB10_24
-; SSE4-NEXT:  # %bb.23: # %cond.store21
+; SSE4-NEXT:  .LBB10_23: # %cond.store21
 ; SSE4-NEXT:    pextrb $11, %xmm0, 11(%rdi)
-; SSE4-NEXT:  .LBB10_24: # %else22
-; SSE4-NEXT:    pxor %xmm2, %xmm2
-; SSE4-NEXT:    pcmpeqd %xmm7, %xmm2
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE4-NEXT:    pxor %xmm2, %xmm1
-; SSE4-NEXT:    pextrb $0, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $4096, %eax # imm = 0x1000
 ; SSE4-NEXT:    je .LBB10_26
-; SSE4-NEXT:  # %bb.25: # %cond.store23
+; SSE4-NEXT:  .LBB10_25: # %cond.store23
 ; SSE4-NEXT:    pextrb $12, %xmm0, 12(%rdi)
-; SSE4-NEXT:  .LBB10_26: # %else24
-; SSE4-NEXT:    pextrb $4, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $8192, %eax # imm = 0x2000
 ; SSE4-NEXT:    je .LBB10_28
-; SSE4-NEXT:  # %bb.27: # %cond.store25
+; SSE4-NEXT:  .LBB10_27: # %cond.store25
 ; SSE4-NEXT:    pextrb $13, %xmm0, 13(%rdi)
-; SSE4-NEXT:  .LBB10_28: # %else26
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm7
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE4-NEXT:    pxor %xmm7, %xmm1
-; SSE4-NEXT:    pextrb $8, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $16384, %eax # imm = 0x4000
 ; SSE4-NEXT:    je .LBB10_30
-; SSE4-NEXT:  # %bb.29: # %cond.store27
+; SSE4-NEXT:  .LBB10_29: # %cond.store27
 ; SSE4-NEXT:    pextrb $14, %xmm0, 14(%rdi)
-; SSE4-NEXT:  .LBB10_30: # %else28
-; SSE4-NEXT:    pextrb $12, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $32768, %eax # imm = 0x8000
 ; SSE4-NEXT:    je .LBB10_32
-; SSE4-NEXT:  # %bb.31: # %cond.store29
+; SSE4-NEXT:  .LBB10_31: # %cond.store29
 ; SSE4-NEXT:    pextrb $15, %xmm0, 15(%rdi)
-; SSE4-NEXT:  .LBB10_32: # %else30
 ; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: truncstore_v16i32_v16i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpxor %xmm8, %xmm8, %xmm8
-; AVX1-NEXT:    vpcmpeqd %xmm8, %xmm2, %xmm6
-; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
-; AVX1-NEXT:    vpxor %xmm5, %xmm6, %xmm6
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm6, %xmm6
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm6, %xmm6
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm7
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [255,255,255,255]
-; AVX1-NEXT:    vpminud %xmm4, %xmm7, %xmm7
-; AVX1-NEXT:    vpminud %xmm4, %xmm0, %xmm0
-; AVX1-NEXT:    vpackusdw %xmm7, %xmm0, %xmm0
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm7
-; AVX1-NEXT:    vpminud %xmm4, %xmm7, %xmm7
-; AVX1-NEXT:    vpminud %xmm4, %xmm1, %xmm1
-; AVX1-NEXT:    vpackusdw %xmm7, %xmm1, %xmm1
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm5 = [255,255,255,255]
+; AVX1-NEXT:    vpminud %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vpminud %xmm5, %xmm0, %xmm0
+; AVX1-NEXT:    vpackusdw %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm4
+; AVX1-NEXT:    vpminud %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vpminud %xmm5, %xmm1, %xmm1
+; AVX1-NEXT:    vpackusdw %xmm4, %xmm1, %xmm1
 ; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpextrb $0, %xmm6, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je .LBB10_2
-; AVX1-NEXT:  # %bb.1: # %cond.store
-; AVX1-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX1-NEXT:  .LBB10_2: # %else
-; AVX1-NEXT:    vpcmpeqd %xmm8, %xmm2, %xmm1
+; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm1
+; AVX1-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm1, %xmm1
+; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
 ; AVX1-NEXT:    vpxor %xmm5, %xmm1, %xmm1
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm1, %xmm1
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $1, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpxor %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vpackssdw %xmm1, %xmm3, %xmm1
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm3
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpxor %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm2, %xmm2
+; AVX1-NEXT:    vpxor %xmm5, %xmm2, %xmm2
+; AVX1-NEXT:    vpackssdw %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpacksswb %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vpmovmskb %xmm1, %eax
+; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    jne .LBB10_1
+; AVX1-NEXT:  # %bb.2: # %else
+; AVX1-NEXT:    testb $2, %al
+; AVX1-NEXT:    jne .LBB10_3
+; AVX1-NEXT:  .LBB10_4: # %else2
+; AVX1-NEXT:    testb $4, %al
+; AVX1-NEXT:    jne .LBB10_5
+; AVX1-NEXT:  .LBB10_6: # %else4
+; AVX1-NEXT:    testb $8, %al
+; AVX1-NEXT:    jne .LBB10_7
+; AVX1-NEXT:  .LBB10_8: # %else6
+; AVX1-NEXT:    testb $16, %al
+; AVX1-NEXT:    jne .LBB10_9
+; AVX1-NEXT:  .LBB10_10: # %else8
+; AVX1-NEXT:    testb $32, %al
+; AVX1-NEXT:    jne .LBB10_11
+; AVX1-NEXT:  .LBB10_12: # %else10
+; AVX1-NEXT:    testb $64, %al
+; AVX1-NEXT:    jne .LBB10_13
+; AVX1-NEXT:  .LBB10_14: # %else12
+; AVX1-NEXT:    testb $-128, %al
+; AVX1-NEXT:    jne .LBB10_15
+; AVX1-NEXT:  .LBB10_16: # %else14
+; AVX1-NEXT:    testl $256, %eax # imm = 0x100
+; AVX1-NEXT:    jne .LBB10_17
+; AVX1-NEXT:  .LBB10_18: # %else16
+; AVX1-NEXT:    testl $512, %eax # imm = 0x200
+; AVX1-NEXT:    jne .LBB10_19
+; AVX1-NEXT:  .LBB10_20: # %else18
+; AVX1-NEXT:    testl $1024, %eax # imm = 0x400
+; AVX1-NEXT:    jne .LBB10_21
+; AVX1-NEXT:  .LBB10_22: # %else20
+; AVX1-NEXT:    testl $2048, %eax # imm = 0x800
+; AVX1-NEXT:    jne .LBB10_23
+; AVX1-NEXT:  .LBB10_24: # %else22
+; AVX1-NEXT:    testl $4096, %eax # imm = 0x1000
+; AVX1-NEXT:    jne .LBB10_25
+; AVX1-NEXT:  .LBB10_26: # %else24
+; AVX1-NEXT:    testl $8192, %eax # imm = 0x2000
+; AVX1-NEXT:    jne .LBB10_27
+; AVX1-NEXT:  .LBB10_28: # %else26
+; AVX1-NEXT:    testl $16384, %eax # imm = 0x4000
+; AVX1-NEXT:    jne .LBB10_29
+; AVX1-NEXT:  .LBB10_30: # %else28
+; AVX1-NEXT:    testl $32768, %eax # imm = 0x8000
+; AVX1-NEXT:    jne .LBB10_31
+; AVX1-NEXT:  .LBB10_32: # %else30
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+; AVX1-NEXT:  .LBB10_1: # %cond.store
+; AVX1-NEXT:    vpextrb $0, %xmm0, (%rdi)
+; AVX1-NEXT:    testb $2, %al
 ; AVX1-NEXT:    je .LBB10_4
-; AVX1-NEXT:  # %bb.3: # %cond.store1
+; AVX1-NEXT:  .LBB10_3: # %cond.store1
 ; AVX1-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
-; AVX1-NEXT:  .LBB10_4: # %else2
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm2, %xmm5
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm5, %xmm5
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm5, %xmm5
-; AVX1-NEXT:    vpextrb $2, %xmm5, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $4, %al
 ; AVX1-NEXT:    je .LBB10_6
-; AVX1-NEXT:  # %bb.5: # %cond.store3
+; AVX1-NEXT:  .LBB10_5: # %cond.store3
 ; AVX1-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
-; AVX1-NEXT:  .LBB10_6: # %else4
-; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm4, %xmm1, %xmm1
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm1, %xmm1
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $3, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $8, %al
 ; AVX1-NEXT:    je .LBB10_8
-; AVX1-NEXT:  # %bb.7: # %cond.store5
+; AVX1-NEXT:  .LBB10_7: # %cond.store5
 ; AVX1-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
-; AVX1-NEXT:  .LBB10_8: # %else6
-; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm2
-; AVX1-NEXT:    vpackssdw %xmm2, %xmm0, %xmm4
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm4, %xmm4
-; AVX1-NEXT:    vpextrb $4, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $16, %al
 ; AVX1-NEXT:    je .LBB10_10
-; AVX1-NEXT:  # %bb.9: # %cond.store7
+; AVX1-NEXT:  .LBB10_9: # %cond.store7
 ; AVX1-NEXT:    vpextrb $4, %xmm0, 4(%rdi)
-; AVX1-NEXT:  .LBB10_10: # %else8
-; AVX1-NEXT:    vpackssdw %xmm2, %xmm0, %xmm2
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $5, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $32, %al
 ; AVX1-NEXT:    je .LBB10_12
-; AVX1-NEXT:  # %bb.11: # %cond.store9
+; AVX1-NEXT:  .LBB10_11: # %cond.store9
 ; AVX1-NEXT:    vpextrb $5, %xmm0, 5(%rdi)
-; AVX1-NEXT:  .LBB10_12: # %else10
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpackssdw %xmm1, %xmm0, %xmm2
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $6, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $64, %al
 ; AVX1-NEXT:    je .LBB10_14
-; AVX1-NEXT:  # %bb.13: # %cond.store11
+; AVX1-NEXT:  .LBB10_13: # %cond.store11
 ; AVX1-NEXT:    vpextrb $6, %xmm0, 6(%rdi)
-; AVX1-NEXT:  .LBB10_14: # %else12
-; AVX1-NEXT:    vpackssdw %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $7, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $-128, %al
 ; AVX1-NEXT:    je .LBB10_16
-; AVX1-NEXT:  # %bb.15: # %cond.store13
+; AVX1-NEXT:  .LBB10_15: # %cond.store13
 ; AVX1-NEXT:    vpextrb $7, %xmm0, 7(%rdi)
-; AVX1-NEXT:  .LBB10_16: # %else14
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm3, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpxor %xmm2, %xmm4, %xmm4
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm4, %xmm4
-; AVX1-NEXT:    vpacksswb %xmm4, %xmm0, %xmm4
-; AVX1-NEXT:    vpextrb $8, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $256, %eax # imm = 0x100
 ; AVX1-NEXT:    je .LBB10_18
-; AVX1-NEXT:  # %bb.17: # %cond.store15
+; AVX1-NEXT:  .LBB10_17: # %cond.store15
 ; AVX1-NEXT:    vpextrb $8, %xmm0, 8(%rdi)
-; AVX1-NEXT:  .LBB10_18: # %else16
-; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm3, %xmm1
-; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm1, %xmm1
-; AVX1-NEXT:    vpacksswb %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpextrb $9, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $512, %eax # imm = 0x200
 ; AVX1-NEXT:    je .LBB10_20
-; AVX1-NEXT:  # %bb.19: # %cond.store17
+; AVX1-NEXT:  .LBB10_19: # %cond.store17
 ; AVX1-NEXT:    vpextrb $9, %xmm0, 9(%rdi)
-; AVX1-NEXT:  .LBB10_20: # %else18
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm3, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpxor %xmm2, %xmm4, %xmm4
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm4, %xmm4
-; AVX1-NEXT:    vpacksswb %xmm4, %xmm0, %xmm4
-; AVX1-NEXT:    vpextrb $10, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $1024, %eax # imm = 0x400
 ; AVX1-NEXT:    je .LBB10_22
-; AVX1-NEXT:  # %bb.21: # %cond.store19
+; AVX1-NEXT:  .LBB10_21: # %cond.store19
 ; AVX1-NEXT:    vpextrb $10, %xmm0, 10(%rdi)
-; AVX1-NEXT:  .LBB10_22: # %else20
-; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm3, %xmm1
-; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm1, %xmm1
-; AVX1-NEXT:    vpacksswb %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpextrb $11, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $2048, %eax # imm = 0x800
 ; AVX1-NEXT:    je .LBB10_24
-; AVX1-NEXT:  # %bb.23: # %cond.store21
+; AVX1-NEXT:  .LBB10_23: # %cond.store21
 ; AVX1-NEXT:    vpextrb $11, %xmm0, 11(%rdi)
-; AVX1-NEXT:  .LBB10_24: # %else22
-; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm1
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpackssdw %xmm2, %xmm0, %xmm3
-; AVX1-NEXT:    vpacksswb %xmm3, %xmm0, %xmm3
-; AVX1-NEXT:    vpextrb $12, %xmm3, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $4096, %eax # imm = 0x1000
 ; AVX1-NEXT:    je .LBB10_26
-; AVX1-NEXT:  # %bb.25: # %cond.store23
+; AVX1-NEXT:  .LBB10_25: # %cond.store23
 ; AVX1-NEXT:    vpextrb $12, %xmm0, 12(%rdi)
-; AVX1-NEXT:  .LBB10_26: # %else24
-; AVX1-NEXT:    vpackssdw %xmm2, %xmm0, %xmm2
-; AVX1-NEXT:    vpacksswb %xmm2, %xmm0, %xmm2
-; AVX1-NEXT:    vpextrb $13, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $8192, %eax # imm = 0x2000
 ; AVX1-NEXT:    je .LBB10_28
-; AVX1-NEXT:  # %bb.27: # %cond.store25
+; AVX1-NEXT:  .LBB10_27: # %cond.store25
 ; AVX1-NEXT:    vpextrb $13, %xmm0, 13(%rdi)
-; AVX1-NEXT:  .LBB10_28: # %else26
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpackssdw %xmm1, %xmm0, %xmm2
-; AVX1-NEXT:    vpacksswb %xmm2, %xmm0, %xmm2
-; AVX1-NEXT:    vpextrb $14, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $16384, %eax # imm = 0x4000
 ; AVX1-NEXT:    je .LBB10_30
-; AVX1-NEXT:  # %bb.29: # %cond.store27
+; AVX1-NEXT:  .LBB10_29: # %cond.store27
 ; AVX1-NEXT:    vpextrb $14, %xmm0, 14(%rdi)
-; AVX1-NEXT:  .LBB10_30: # %else28
-; AVX1-NEXT:    vpackssdw %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpacksswb %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpextrb $15, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $32768, %eax # imm = 0x8000
 ; AVX1-NEXT:    je .LBB10_32
-; AVX1-NEXT:  # %bb.31: # %cond.store29
+; AVX1-NEXT:  .LBB10_31: # %cond.store29
 ; AVX1-NEXT:    vpextrb $15, %xmm0, 15(%rdi)
-; AVX1-NEXT:  .LBB10_32: # %else30
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: truncstore_v16i32_v16i8:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm2, %xmm6
-; AVX2-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
-; AVX2-NEXT:    vpxor %xmm5, %xmm6, %xmm6
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm6, %xmm6
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm6, %xmm6
-; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm7 = [255,255,255,255,255,255,255,255]
-; AVX2-NEXT:    vpminud %ymm7, %ymm1, %ymm1
-; AVX2-NEXT:    vpminud %ymm7, %ymm0, %ymm0
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm5 = [255,255,255,255,255,255,255,255]
+; AVX2-NEXT:    vpminud %ymm5, %ymm1, %ymm1
+; AVX2-NEXT:    vpminud %ymm5, %ymm0, %ymm0
 ; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpextrb $0, %xmm6, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je .LBB10_2
-; AVX2-NEXT:  # %bb.1: # %cond.store
+; AVX2-NEXT:    vpcmpeqd %ymm4, %ymm3, %ymm1
+; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm3, %ymm3
+; AVX2-NEXT:    vpxor %ymm3, %ymm1, %ymm1
+; AVX2-NEXT:    vpcmpeqd %ymm4, %ymm2, %ymm2
+; AVX2-NEXT:    vpxor %ymm3, %ymm2, %ymm2
+; AVX2-NEXT:    vpackssdw %ymm1, %ymm2, %ymm1
+; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
+; AVX2-NEXT:    vpacksswb %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpmovmskb %xmm1, %eax
+; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    jne .LBB10_1
+; AVX2-NEXT:  # %bb.2: # %else
+; AVX2-NEXT:    testb $2, %al
+; AVX2-NEXT:    jne .LBB10_3
+; AVX2-NEXT:  .LBB10_4: # %else2
+; AVX2-NEXT:    testb $4, %al
+; AVX2-NEXT:    jne .LBB10_5
+; AVX2-NEXT:  .LBB10_6: # %else4
+; AVX2-NEXT:    testb $8, %al
+; AVX2-NEXT:    jne .LBB10_7
+; AVX2-NEXT:  .LBB10_8: # %else6
+; AVX2-NEXT:    testb $16, %al
+; AVX2-NEXT:    jne .LBB10_9
+; AVX2-NEXT:  .LBB10_10: # %else8
+; AVX2-NEXT:    testb $32, %al
+; AVX2-NEXT:    jne .LBB10_11
+; AVX2-NEXT:  .LBB10_12: # %else10
+; AVX2-NEXT:    testb $64, %al
+; AVX2-NEXT:    jne .LBB10_13
+; AVX2-NEXT:  .LBB10_14: # %else12
+; AVX2-NEXT:    testb $-128, %al
+; AVX2-NEXT:    jne .LBB10_15
+; AVX2-NEXT:  .LBB10_16: # %else14
+; AVX2-NEXT:    testl $256, %eax # imm = 0x100
+; AVX2-NEXT:    jne .LBB10_17
+; AVX2-NEXT:  .LBB10_18: # %else16
+; AVX2-NEXT:    testl $512, %eax # imm = 0x200
+; AVX2-NEXT:    jne .LBB10_19
+; AVX2-NEXT:  .LBB10_20: # %else18
+; AVX2-NEXT:    testl $1024, %eax # imm = 0x400
+; AVX2-NEXT:    jne .LBB10_21
+; AVX2-NEXT:  .LBB10_22: # %else20
+; AVX2-NEXT:    testl $2048, %eax # imm = 0x800
+; AVX2-NEXT:    jne .LBB10_23
+; AVX2-NEXT:  .LBB10_24: # %else22
+; AVX2-NEXT:    testl $4096, %eax # imm = 0x1000
+; AVX2-NEXT:    jne .LBB10_25
+; AVX2-NEXT:  .LBB10_26: # %else24
+; AVX2-NEXT:    testl $8192, %eax # imm = 0x2000
+; AVX2-NEXT:    jne .LBB10_27
+; AVX2-NEXT:  .LBB10_28: # %else26
+; AVX2-NEXT:    testl $16384, %eax # imm = 0x4000
+; AVX2-NEXT:    jne .LBB10_29
+; AVX2-NEXT:  .LBB10_30: # %else28
+; AVX2-NEXT:    testl $32768, %eax # imm = 0x8000
+; AVX2-NEXT:    jne .LBB10_31
+; AVX2-NEXT:  .LBB10_32: # %else30
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+; AVX2-NEXT:  .LBB10_1: # %cond.store
 ; AVX2-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX2-NEXT:  .LBB10_2: # %else
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm2, %xmm1
-; AVX2-NEXT:    vpxor %xmm5, %xmm1, %xmm1
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm1, %xmm1
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm1, %xmm1
-; AVX2-NEXT:    vpextrb $1, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $2, %al
 ; AVX2-NEXT:    je .LBB10_4
-; AVX2-NEXT:  # %bb.3: # %cond.store1
+; AVX2-NEXT:  .LBB10_3: # %cond.store1
 ; AVX2-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
-; AVX2-NEXT:  .LBB10_4: # %else2
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm2, %xmm5
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm5, %xmm5
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm5, %xmm5
-; AVX2-NEXT:    vpextrb $2, %xmm5, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $4, %al
 ; AVX2-NEXT:    je .LBB10_6
-; AVX2-NEXT:  # %bb.5: # %cond.store3
+; AVX2-NEXT:  .LBB10_5: # %cond.store3
 ; AVX2-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
-; AVX2-NEXT:  .LBB10_6: # %else4
-; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm2, %xmm1
-; AVX2-NEXT:    vpxor %xmm4, %xmm1, %xmm1
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm1, %xmm1
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm1, %xmm1
-; AVX2-NEXT:    vpextrb $3, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $8, %al
 ; AVX2-NEXT:    je .LBB10_8
-; AVX2-NEXT:  # %bb.7: # %cond.store5
+; AVX2-NEXT:  .LBB10_7: # %cond.store5
 ; AVX2-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
-; AVX2-NEXT:  .LBB10_8: # %else6
-; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm1
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpxor %xmm4, %xmm2, %xmm2
-; AVX2-NEXT:    vpackssdw %xmm2, %xmm0, %xmm4
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm4, %xmm4
-; AVX2-NEXT:    vpextrb $4, %xmm4, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $16, %al
 ; AVX2-NEXT:    je .LBB10_10
-; AVX2-NEXT:  # %bb.9: # %cond.store7
+; AVX2-NEXT:  .LBB10_9: # %cond.store7
 ; AVX2-NEXT:    vpextrb $4, %xmm0, 4(%rdi)
-; AVX2-NEXT:  .LBB10_10: # %else8
-; AVX2-NEXT:    vpackssdw %xmm2, %xmm0, %xmm2
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrb $5, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $32, %al
 ; AVX2-NEXT:    je .LBB10_12
-; AVX2-NEXT:  # %bb.11: # %cond.store9
+; AVX2-NEXT:  .LBB10_11: # %cond.store9
 ; AVX2-NEXT:    vpextrb $5, %xmm0, 5(%rdi)
-; AVX2-NEXT:  .LBB10_12: # %else10
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpackssdw %xmm1, %xmm0, %xmm2
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrb $6, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $64, %al
 ; AVX2-NEXT:    je .LBB10_14
-; AVX2-NEXT:  # %bb.13: # %cond.store11
+; AVX2-NEXT:  .LBB10_13: # %cond.store11
 ; AVX2-NEXT:    vpextrb $6, %xmm0, 6(%rdi)
-; AVX2-NEXT:  .LBB10_14: # %else12
-; AVX2-NEXT:    vpackssdw %xmm1, %xmm0, %xmm1
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm1, %xmm1
-; AVX2-NEXT:    vpextrb $7, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $-128, %al
 ; AVX2-NEXT:    je .LBB10_16
-; AVX2-NEXT:  # %bb.15: # %cond.store13
+; AVX2-NEXT:  .LBB10_15: # %cond.store13
 ; AVX2-NEXT:    vpextrb $7, %xmm0, 7(%rdi)
-; AVX2-NEXT:  .LBB10_16: # %else14
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm3, %xmm4
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpxor %xmm2, %xmm4, %xmm4
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm4, %xmm4
-; AVX2-NEXT:    vpacksswb %xmm4, %xmm0, %xmm4
-; AVX2-NEXT:    vpextrb $8, %xmm4, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $256, %eax # imm = 0x100
 ; AVX2-NEXT:    je .LBB10_18
-; AVX2-NEXT:  # %bb.17: # %cond.store15
+; AVX2-NEXT:  .LBB10_17: # %cond.store15
 ; AVX2-NEXT:    vpextrb $8, %xmm0, 8(%rdi)
-; AVX2-NEXT:  .LBB10_18: # %else16
-; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm3, %xmm1
-; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm1, %xmm1
-; AVX2-NEXT:    vpacksswb %xmm1, %xmm0, %xmm1
-; AVX2-NEXT:    vpextrb $9, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $512, %eax # imm = 0x200
 ; AVX2-NEXT:    je .LBB10_20
-; AVX2-NEXT:  # %bb.19: # %cond.store17
+; AVX2-NEXT:  .LBB10_19: # %cond.store17
 ; AVX2-NEXT:    vpextrb $9, %xmm0, 9(%rdi)
-; AVX2-NEXT:  .LBB10_20: # %else18
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm3, %xmm4
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpxor %xmm2, %xmm4, %xmm4
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm4, %xmm4
-; AVX2-NEXT:    vpacksswb %xmm4, %xmm0, %xmm4
-; AVX2-NEXT:    vpextrb $10, %xmm4, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $1024, %eax # imm = 0x400
 ; AVX2-NEXT:    je .LBB10_22
-; AVX2-NEXT:  # %bb.21: # %cond.store19
+; AVX2-NEXT:  .LBB10_21: # %cond.store19
 ; AVX2-NEXT:    vpextrb $10, %xmm0, 10(%rdi)
-; AVX2-NEXT:  .LBB10_22: # %else20
-; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm3, %xmm1
-; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpackssdw %xmm0, %xmm1, %xmm1
-; AVX2-NEXT:    vpacksswb %xmm1, %xmm0, %xmm1
-; AVX2-NEXT:    vpextrb $11, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $2048, %eax # imm = 0x800
 ; AVX2-NEXT:    je .LBB10_24
-; AVX2-NEXT:  # %bb.23: # %cond.store21
+; AVX2-NEXT:  .LBB10_23: # %cond.store21
 ; AVX2-NEXT:    vpextrb $11, %xmm0, 11(%rdi)
-; AVX2-NEXT:  .LBB10_24: # %else22
-; AVX2-NEXT:    vextracti128 $1, %ymm3, %xmm1
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX2-NEXT:    vpackssdw %xmm2, %xmm0, %xmm3
-; AVX2-NEXT:    vpacksswb %xmm3, %xmm0, %xmm3
-; AVX2-NEXT:    vpextrb $12, %xmm3, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $4096, %eax # imm = 0x1000
 ; AVX2-NEXT:    je .LBB10_26
-; AVX2-NEXT:  # %bb.25: # %cond.store23
+; AVX2-NEXT:  .LBB10_25: # %cond.store23
 ; AVX2-NEXT:    vpextrb $12, %xmm0, 12(%rdi)
-; AVX2-NEXT:  .LBB10_26: # %else24
-; AVX2-NEXT:    vpackssdw %xmm2, %xmm0, %xmm2
-; AVX2-NEXT:    vpacksswb %xmm2, %xmm0, %xmm2
-; AVX2-NEXT:    vpextrb $13, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $8192, %eax # imm = 0x2000
 ; AVX2-NEXT:    je .LBB10_28
-; AVX2-NEXT:  # %bb.27: # %cond.store25
+; AVX2-NEXT:  .LBB10_27: # %cond.store25
 ; AVX2-NEXT:    vpextrb $13, %xmm0, 13(%rdi)
-; AVX2-NEXT:  .LBB10_28: # %else26
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpackssdw %xmm1, %xmm0, %xmm2
-; AVX2-NEXT:    vpacksswb %xmm2, %xmm0, %xmm2
-; AVX2-NEXT:    vpextrb $14, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $16384, %eax # imm = 0x4000
 ; AVX2-NEXT:    je .LBB10_30
-; AVX2-NEXT:  # %bb.29: # %cond.store27
+; AVX2-NEXT:  .LBB10_29: # %cond.store27
 ; AVX2-NEXT:    vpextrb $14, %xmm0, 14(%rdi)
-; AVX2-NEXT:  .LBB10_30: # %else28
-; AVX2-NEXT:    vpackssdw %xmm1, %xmm0, %xmm1
-; AVX2-NEXT:    vpacksswb %xmm1, %xmm0, %xmm1
-; AVX2-NEXT:    vpextrb $15, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $32768, %eax # imm = 0x8000
 ; AVX2-NEXT:    je .LBB10_32
-; AVX2-NEXT:  # %bb.31: # %cond.store29
+; AVX2-NEXT:  .LBB10_31: # %cond.store29
 ; AVX2-NEXT:    vpextrb $15, %xmm0, 15(%rdi)
-; AVX2-NEXT:  .LBB10_32: # %else30
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
@@ -4104,115 +3809,117 @@ define void @truncstore_v16i32_v16i8(<16 x i32> %x, <16 x i8>* %p, <16 x i32> %m
 ; AVX512F-NEXT:    vpmovusdb %zmm0, %xmm0
 ; AVX512F-NEXT:    kmovw %k0, %eax
 ; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB10_2
-; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    jne .LBB10_1
+; AVX512F-NEXT:  # %bb.2: # %else
+; AVX512F-NEXT:    testb $2, %al
+; AVX512F-NEXT:    jne .LBB10_3
+; AVX512F-NEXT:  .LBB10_4: # %else2
+; AVX512F-NEXT:    testb $4, %al
+; AVX512F-NEXT:    jne .LBB10_5
+; AVX512F-NEXT:  .LBB10_6: # %else4
+; AVX512F-NEXT:    testb $8, %al
+; AVX512F-NEXT:    jne .LBB10_7
+; AVX512F-NEXT:  .LBB10_8: # %else6
+; AVX512F-NEXT:    testb $16, %al
+; AVX512F-NEXT:    jne .LBB10_9
+; AVX512F-NEXT:  .LBB10_10: # %else8
+; AVX512F-NEXT:    testb $32, %al
+; AVX512F-NEXT:    jne .LBB10_11
+; AVX512F-NEXT:  .LBB10_12: # %else10
+; AVX512F-NEXT:    testb $64, %al
+; AVX512F-NEXT:    jne .LBB10_13
+; AVX512F-NEXT:  .LBB10_14: # %else12
+; AVX512F-NEXT:    testb $-128, %al
+; AVX512F-NEXT:    jne .LBB10_15
+; AVX512F-NEXT:  .LBB10_16: # %else14
+; AVX512F-NEXT:    testl $256, %eax # imm = 0x100
+; AVX512F-NEXT:    jne .LBB10_17
+; AVX512F-NEXT:  .LBB10_18: # %else16
+; AVX512F-NEXT:    testl $512, %eax # imm = 0x200
+; AVX512F-NEXT:    jne .LBB10_19
+; AVX512F-NEXT:  .LBB10_20: # %else18
+; AVX512F-NEXT:    testl $1024, %eax # imm = 0x400
+; AVX512F-NEXT:    jne .LBB10_21
+; AVX512F-NEXT:  .LBB10_22: # %else20
+; AVX512F-NEXT:    testl $2048, %eax # imm = 0x800
+; AVX512F-NEXT:    jne .LBB10_23
+; AVX512F-NEXT:  .LBB10_24: # %else22
+; AVX512F-NEXT:    testl $4096, %eax # imm = 0x1000
+; AVX512F-NEXT:    jne .LBB10_25
+; AVX512F-NEXT:  .LBB10_26: # %else24
+; AVX512F-NEXT:    testl $8192, %eax # imm = 0x2000
+; AVX512F-NEXT:    jne .LBB10_27
+; AVX512F-NEXT:  .LBB10_28: # %else26
+; AVX512F-NEXT:    testl $16384, %eax # imm = 0x4000
+; AVX512F-NEXT:    jne .LBB10_29
+; AVX512F-NEXT:  .LBB10_30: # %else28
+; AVX512F-NEXT:    testl $32768, %eax # imm = 0x8000
+; AVX512F-NEXT:    jne .LBB10_31
+; AVX512F-NEXT:  .LBB10_32: # %else30
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+; AVX512F-NEXT:  .LBB10_1: # %cond.store
 ; AVX512F-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX512F-NEXT:  .LBB10_2: # %else
-; AVX512F-NEXT:    kshiftrw $1, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $2, %al
 ; AVX512F-NEXT:    je .LBB10_4
-; AVX512F-NEXT:  # %bb.3: # %cond.store1
+; AVX512F-NEXT:  .LBB10_3: # %cond.store1
 ; AVX512F-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
-; AVX512F-NEXT:  .LBB10_4: # %else2
-; AVX512F-NEXT:    kshiftrw $2, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $4, %al
 ; AVX512F-NEXT:    je .LBB10_6
-; AVX512F-NEXT:  # %bb.5: # %cond.store3
+; AVX512F-NEXT:  .LBB10_5: # %cond.store3
 ; AVX512F-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
-; AVX512F-NEXT:  .LBB10_6: # %else4
-; AVX512F-NEXT:    kshiftrw $3, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $8, %al
 ; AVX512F-NEXT:    je .LBB10_8
-; AVX512F-NEXT:  # %bb.7: # %cond.store5
+; AVX512F-NEXT:  .LBB10_7: # %cond.store5
 ; AVX512F-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
-; AVX512F-NEXT:  .LBB10_8: # %else6
-; AVX512F-NEXT:    kshiftrw $4, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $16, %al
 ; AVX512F-NEXT:    je .LBB10_10
-; AVX512F-NEXT:  # %bb.9: # %cond.store7
+; AVX512F-NEXT:  .LBB10_9: # %cond.store7
 ; AVX512F-NEXT:    vpextrb $4, %xmm0, 4(%rdi)
-; AVX512F-NEXT:  .LBB10_10: # %else8
-; AVX512F-NEXT:    kshiftrw $5, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $32, %al
 ; AVX512F-NEXT:    je .LBB10_12
-; AVX512F-NEXT:  # %bb.11: # %cond.store9
+; AVX512F-NEXT:  .LBB10_11: # %cond.store9
 ; AVX512F-NEXT:    vpextrb $5, %xmm0, 5(%rdi)
-; AVX512F-NEXT:  .LBB10_12: # %else10
-; AVX512F-NEXT:    kshiftrw $6, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $64, %al
 ; AVX512F-NEXT:    je .LBB10_14
-; AVX512F-NEXT:  # %bb.13: # %cond.store11
+; AVX512F-NEXT:  .LBB10_13: # %cond.store11
 ; AVX512F-NEXT:    vpextrb $6, %xmm0, 6(%rdi)
-; AVX512F-NEXT:  .LBB10_14: # %else12
-; AVX512F-NEXT:    kshiftrw $7, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $-128, %al
 ; AVX512F-NEXT:    je .LBB10_16
-; AVX512F-NEXT:  # %bb.15: # %cond.store13
+; AVX512F-NEXT:  .LBB10_15: # %cond.store13
 ; AVX512F-NEXT:    vpextrb $7, %xmm0, 7(%rdi)
-; AVX512F-NEXT:  .LBB10_16: # %else14
-; AVX512F-NEXT:    kshiftrw $8, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $256, %eax # imm = 0x100
 ; AVX512F-NEXT:    je .LBB10_18
-; AVX512F-NEXT:  # %bb.17: # %cond.store15
+; AVX512F-NEXT:  .LBB10_17: # %cond.store15
 ; AVX512F-NEXT:    vpextrb $8, %xmm0, 8(%rdi)
-; AVX512F-NEXT:  .LBB10_18: # %else16
-; AVX512F-NEXT:    kshiftrw $9, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $512, %eax # imm = 0x200
 ; AVX512F-NEXT:    je .LBB10_20
-; AVX512F-NEXT:  # %bb.19: # %cond.store17
+; AVX512F-NEXT:  .LBB10_19: # %cond.store17
 ; AVX512F-NEXT:    vpextrb $9, %xmm0, 9(%rdi)
-; AVX512F-NEXT:  .LBB10_20: # %else18
-; AVX512F-NEXT:    kshiftrw $10, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $1024, %eax # imm = 0x400
 ; AVX512F-NEXT:    je .LBB10_22
-; AVX512F-NEXT:  # %bb.21: # %cond.store19
+; AVX512F-NEXT:  .LBB10_21: # %cond.store19
 ; AVX512F-NEXT:    vpextrb $10, %xmm0, 10(%rdi)
-; AVX512F-NEXT:  .LBB10_22: # %else20
-; AVX512F-NEXT:    kshiftrw $11, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $2048, %eax # imm = 0x800
 ; AVX512F-NEXT:    je .LBB10_24
-; AVX512F-NEXT:  # %bb.23: # %cond.store21
+; AVX512F-NEXT:  .LBB10_23: # %cond.store21
 ; AVX512F-NEXT:    vpextrb $11, %xmm0, 11(%rdi)
-; AVX512F-NEXT:  .LBB10_24: # %else22
-; AVX512F-NEXT:    kshiftrw $12, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $4096, %eax # imm = 0x1000
 ; AVX512F-NEXT:    je .LBB10_26
-; AVX512F-NEXT:  # %bb.25: # %cond.store23
+; AVX512F-NEXT:  .LBB10_25: # %cond.store23
 ; AVX512F-NEXT:    vpextrb $12, %xmm0, 12(%rdi)
-; AVX512F-NEXT:  .LBB10_26: # %else24
-; AVX512F-NEXT:    kshiftrw $13, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $8192, %eax # imm = 0x2000
 ; AVX512F-NEXT:    je .LBB10_28
-; AVX512F-NEXT:  # %bb.27: # %cond.store25
+; AVX512F-NEXT:  .LBB10_27: # %cond.store25
 ; AVX512F-NEXT:    vpextrb $13, %xmm0, 13(%rdi)
-; AVX512F-NEXT:  .LBB10_28: # %else26
-; AVX512F-NEXT:    kshiftrw $14, %k0, %k1
-; AVX512F-NEXT:    kmovw %k1, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $16384, %eax # imm = 0x4000
 ; AVX512F-NEXT:    je .LBB10_30
-; AVX512F-NEXT:  # %bb.29: # %cond.store27
+; AVX512F-NEXT:  .LBB10_29: # %cond.store27
 ; AVX512F-NEXT:    vpextrb $14, %xmm0, 14(%rdi)
-; AVX512F-NEXT:  .LBB10_30: # %else28
-; AVX512F-NEXT:    kshiftrw $15, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $32768, %eax # imm = 0x8000
 ; AVX512F-NEXT:    je .LBB10_32
-; AVX512F-NEXT:  # %bb.31: # %cond.store29
+; AVX512F-NEXT:  .LBB10_31: # %cond.store29
 ; AVX512F-NEXT:    vpextrb $15, %xmm0, 15(%rdi)
-; AVX512F-NEXT:  .LBB10_32: # %else30
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
@@ -4242,413 +3949,387 @@ define void @truncstore_v16i32_v16i8(<16 x i32> %x, <16 x i8>* %p, <16 x i32> %m
 define void @truncstore_v8i32_v8i16(<8 x i32> %x, <8 x i16>* %p, <8 x i32> %mask) {
 ; SSE2-LABEL: truncstore_v8i32_v8i16:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pxor %xmm10, %xmm10
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm10
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm9
-; SSE2-NEXT:    pxor %xmm9, %xmm10
-; SSE2-NEXT:    movdqa %xmm10, %xmm8
-; SSE2-NEXT:    packssdw %xmm0, %xmm8
-; SSE2-NEXT:    movdqa {{.*#+}} xmm6 = [2147483648,2147483648,2147483648,2147483648]
-; SSE2-NEXT:    movdqa %xmm0, %xmm7
-; SSE2-NEXT:    pxor %xmm6, %xmm7
+; SSE2-NEXT:    pxor %xmm8, %xmm8
+; SSE2-NEXT:    movdqa {{.*#+}} xmm7 = [2147483648,2147483648,2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm0, %xmm6
+; SSE2-NEXT:    pxor %xmm7, %xmm6
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [2147549183,2147549183,2147549183,2147549183]
 ; SSE2-NEXT:    movdqa %xmm4, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm7, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm6, %xmm5
+; SSE2-NEXT:    pcmpeqd %xmm6, %xmm6
 ; SSE2-NEXT:    pand %xmm5, %xmm0
-; SSE2-NEXT:    pxor %xmm9, %xmm5
+; SSE2-NEXT:    pxor %xmm6, %xmm5
 ; SSE2-NEXT:    por %xmm0, %xmm5
-; SSE2-NEXT:    pxor %xmm1, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm6, %xmm4
-; SSE2-NEXT:    pxor %xmm4, %xmm9
-; SSE2-NEXT:    pand %xmm1, %xmm4
-; SSE2-NEXT:    por %xmm9, %xmm4
+; SSE2-NEXT:    pxor %xmm1, %xmm7
+; SSE2-NEXT:    pcmpgtd %xmm7, %xmm4
+; SSE2-NEXT:    pand %xmm4, %xmm1
+; SSE2-NEXT:    pxor %xmm6, %xmm4
+; SSE2-NEXT:    por %xmm1, %xmm4
 ; SSE2-NEXT:    pslld $16, %xmm4
 ; SSE2-NEXT:    psrad $16, %xmm4
 ; SSE2-NEXT:    pslld $16, %xmm5
 ; SSE2-NEXT:    psrad $16, %xmm5
 ; SSE2-NEXT:    packssdw %xmm4, %xmm5
-; SSE2-NEXT:    movd %xmm8, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB11_2
-; SSE2-NEXT:  # %bb.1: # %cond.store
-; SSE2-NEXT:    movd %xmm5, %eax
-; SSE2-NEXT:    movw %ax, (%rdi)
-; SSE2-NEXT:  .LBB11_2: # %else
-; SSE2-NEXT:    psrlq $16, %xmm10
-; SSE2-NEXT:    movd %xmm10, %eax
-; SSE2-NEXT:    shrl $16, %eax
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm3
+; SSE2-NEXT:    pxor %xmm6, %xmm3
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm2
+; SSE2-NEXT:    pxor %xmm6, %xmm2
+; SSE2-NEXT:    packssdw %xmm3, %xmm2
+; SSE2-NEXT:    packsswb %xmm0, %xmm2
+; SSE2-NEXT:    pmovmskb %xmm2, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB11_4
-; SSE2-NEXT:  # %bb.3: # %cond.store1
-; SSE2-NEXT:    pextrw $1, %xmm5, %eax
-; SSE2-NEXT:    movw %ax, 2(%rdi)
+; SSE2-NEXT:    jne .LBB11_1
+; SSE2-NEXT:  # %bb.2: # %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne .LBB11_3
 ; SSE2-NEXT:  .LBB11_4: # %else2
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE2-NEXT:    pxor %xmm2, %xmm0
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB11_6
-; SSE2-NEXT:  # %bb.5: # %cond.store3
-; SSE2-NEXT:    pextrw $2, %xmm5, %eax
-; SSE2-NEXT:    movw %ax, 4(%rdi)
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne .LBB11_5
 ; SSE2-NEXT:  .LBB11_6: # %else4
-; SSE2-NEXT:    pextrw $6, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB11_8
-; SSE2-NEXT:  # %bb.7: # %cond.store5
-; SSE2-NEXT:    pextrw $3, %xmm5, %eax
-; SSE2-NEXT:    movw %ax, 6(%rdi)
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne .LBB11_7
 ; SSE2-NEXT:  .LBB11_8: # %else6
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE2-NEXT:    pxor %xmm1, %xmm0
-; SSE2-NEXT:    pextrw $0, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB11_10
-; SSE2-NEXT:  # %bb.9: # %cond.store7
-; SSE2-NEXT:    pextrw $4, %xmm5, %eax
-; SSE2-NEXT:    movw %ax, 8(%rdi)
+; SSE2-NEXT:    testb $16, %al
+; SSE2-NEXT:    jne .LBB11_9
 ; SSE2-NEXT:  .LBB11_10: # %else8
-; SSE2-NEXT:    pextrw $2, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB11_12
-; SSE2-NEXT:  # %bb.11: # %cond.store9
-; SSE2-NEXT:    pextrw $5, %xmm5, %eax
-; SSE2-NEXT:    movw %ax, 10(%rdi)
+; SSE2-NEXT:    testb $32, %al
+; SSE2-NEXT:    jne .LBB11_11
 ; SSE2-NEXT:  .LBB11_12: # %else10
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm3
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE2-NEXT:    pxor %xmm3, %xmm0
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB11_14
-; SSE2-NEXT:  # %bb.13: # %cond.store11
-; SSE2-NEXT:    pextrw $6, %xmm5, %eax
-; SSE2-NEXT:    movw %ax, 12(%rdi)
+; SSE2-NEXT:    testb $64, %al
+; SSE2-NEXT:    jne .LBB11_13
 ; SSE2-NEXT:  .LBB11_14: # %else12
-; SSE2-NEXT:    pextrw $6, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $-128, %al
+; SSE2-NEXT:    jne .LBB11_15
+; SSE2-NEXT:  .LBB11_16: # %else14
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  .LBB11_1: # %cond.store
+; SSE2-NEXT:    movd %xmm5, %ecx
+; SSE2-NEXT:    movw %cx, (%rdi)
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    je .LBB11_4
+; SSE2-NEXT:  .LBB11_3: # %cond.store1
+; SSE2-NEXT:    pextrw $1, %xmm5, %ecx
+; SSE2-NEXT:    movw %cx, 2(%rdi)
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    je .LBB11_6
+; SSE2-NEXT:  .LBB11_5: # %cond.store3
+; SSE2-NEXT:    pextrw $2, %xmm5, %ecx
+; SSE2-NEXT:    movw %cx, 4(%rdi)
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    je .LBB11_8
+; SSE2-NEXT:  .LBB11_7: # %cond.store5
+; SSE2-NEXT:    pextrw $3, %xmm5, %ecx
+; SSE2-NEXT:    movw %cx, 6(%rdi)
+; SSE2-NEXT:    testb $16, %al
+; SSE2-NEXT:    je .LBB11_10
+; SSE2-NEXT:  .LBB11_9: # %cond.store7
+; SSE2-NEXT:    pextrw $4, %xmm5, %ecx
+; SSE2-NEXT:    movw %cx, 8(%rdi)
+; SSE2-NEXT:    testb $32, %al
+; SSE2-NEXT:    je .LBB11_12
+; SSE2-NEXT:  .LBB11_11: # %cond.store9
+; SSE2-NEXT:    pextrw $5, %xmm5, %ecx
+; SSE2-NEXT:    movw %cx, 10(%rdi)
+; SSE2-NEXT:    testb $64, %al
+; SSE2-NEXT:    je .LBB11_14
+; SSE2-NEXT:  .LBB11_13: # %cond.store11
+; SSE2-NEXT:    pextrw $6, %xmm5, %ecx
+; SSE2-NEXT:    movw %cx, 12(%rdi)
+; SSE2-NEXT:    testb $-128, %al
 ; SSE2-NEXT:    je .LBB11_16
-; SSE2-NEXT:  # %bb.15: # %cond.store13
+; SSE2-NEXT:  .LBB11_15: # %cond.store13
 ; SSE2-NEXT:    pextrw $7, %xmm5, %eax
 ; SSE2-NEXT:    movw %ax, 14(%rdi)
-; SSE2-NEXT:  .LBB11_16: # %else14
 ; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: truncstore_v8i32_v8i16:
 ; SSE4:       # %bb.0:
-; SSE4-NEXT:    pxor %xmm5, %xmm5
-; SSE4-NEXT:    pcmpeqd %xmm2, %xmm5
-; SSE4-NEXT:    pcmpeqd %xmm4, %xmm4
-; SSE4-NEXT:    pxor %xmm5, %xmm4
+; SSE4-NEXT:    pxor %xmm4, %xmm4
 ; SSE4-NEXT:    movdqa {{.*#+}} xmm5 = [65535,65535,65535,65535]
 ; SSE4-NEXT:    pminud %xmm5, %xmm1
 ; SSE4-NEXT:    pminud %xmm5, %xmm0
 ; SSE4-NEXT:    packusdw %xmm1, %xmm0
-; SSE4-NEXT:    pextrb $0, %xmm4, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB11_2
-; SSE4-NEXT:  # %bb.1: # %cond.store
+; SSE4-NEXT:    pcmpeqd %xmm4, %xmm3
+; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
+; SSE4-NEXT:    pxor %xmm1, %xmm3
+; SSE4-NEXT:    pcmpeqd %xmm4, %xmm2
+; SSE4-NEXT:    pxor %xmm1, %xmm2
+; SSE4-NEXT:    packssdw %xmm3, %xmm2
+; SSE4-NEXT:    packsswb %xmm0, %xmm2
+; SSE4-NEXT:    pmovmskb %xmm2, %eax
+; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    jne .LBB11_1
+; SSE4-NEXT:  # %bb.2: # %else
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    jne .LBB11_3
+; SSE4-NEXT:  .LBB11_4: # %else2
+; SSE4-NEXT:    testb $4, %al
+; SSE4-NEXT:    jne .LBB11_5
+; SSE4-NEXT:  .LBB11_6: # %else4
+; SSE4-NEXT:    testb $8, %al
+; SSE4-NEXT:    jne .LBB11_7
+; SSE4-NEXT:  .LBB11_8: # %else6
+; SSE4-NEXT:    testb $16, %al
+; SSE4-NEXT:    jne .LBB11_9
+; SSE4-NEXT:  .LBB11_10: # %else8
+; SSE4-NEXT:    testb $32, %al
+; SSE4-NEXT:    jne .LBB11_11
+; SSE4-NEXT:  .LBB11_12: # %else10
+; SSE4-NEXT:    testb $64, %al
+; SSE4-NEXT:    jne .LBB11_13
+; SSE4-NEXT:  .LBB11_14: # %else12
+; SSE4-NEXT:    testb $-128, %al
+; SSE4-NEXT:    jne .LBB11_15
+; SSE4-NEXT:  .LBB11_16: # %else14
+; SSE4-NEXT:    retq
+; SSE4-NEXT:  .LBB11_1: # %cond.store
 ; SSE4-NEXT:    pextrw $0, %xmm0, (%rdi)
-; SSE4-NEXT:  .LBB11_2: # %else
-; SSE4-NEXT:    pextrb $4, %xmm4, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $2, %al
 ; SSE4-NEXT:    je .LBB11_4
-; SSE4-NEXT:  # %bb.3: # %cond.store1
+; SSE4-NEXT:  .LBB11_3: # %cond.store1
 ; SSE4-NEXT:    pextrw $1, %xmm0, 2(%rdi)
-; SSE4-NEXT:  .LBB11_4: # %else2
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm2
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE4-NEXT:    pxor %xmm2, %xmm1
-; SSE4-NEXT:    pextrb $8, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $4, %al
 ; SSE4-NEXT:    je .LBB11_6
-; SSE4-NEXT:  # %bb.5: # %cond.store3
+; SSE4-NEXT:  .LBB11_5: # %cond.store3
 ; SSE4-NEXT:    pextrw $2, %xmm0, 4(%rdi)
-; SSE4-NEXT:  .LBB11_6: # %else4
-; SSE4-NEXT:    pextrb $12, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $8, %al
 ; SSE4-NEXT:    je .LBB11_8
-; SSE4-NEXT:  # %bb.7: # %cond.store5
+; SSE4-NEXT:  .LBB11_7: # %cond.store5
 ; SSE4-NEXT:    pextrw $3, %xmm0, 6(%rdi)
-; SSE4-NEXT:  .LBB11_8: # %else6
-; SSE4-NEXT:    pxor %xmm2, %xmm2
-; SSE4-NEXT:    pcmpeqd %xmm3, %xmm2
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE4-NEXT:    pxor %xmm2, %xmm1
-; SSE4-NEXT:    pextrb $0, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $16, %al
 ; SSE4-NEXT:    je .LBB11_10
-; SSE4-NEXT:  # %bb.9: # %cond.store7
+; SSE4-NEXT:  .LBB11_9: # %cond.store7
 ; SSE4-NEXT:    pextrw $4, %xmm0, 8(%rdi)
-; SSE4-NEXT:  .LBB11_10: # %else8
-; SSE4-NEXT:    pextrb $4, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $32, %al
 ; SSE4-NEXT:    je .LBB11_12
-; SSE4-NEXT:  # %bb.11: # %cond.store9
+; SSE4-NEXT:  .LBB11_11: # %cond.store9
 ; SSE4-NEXT:    pextrw $5, %xmm0, 10(%rdi)
-; SSE4-NEXT:  .LBB11_12: # %else10
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm3
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE4-NEXT:    pxor %xmm3, %xmm1
-; SSE4-NEXT:    pextrb $8, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $64, %al
 ; SSE4-NEXT:    je .LBB11_14
-; SSE4-NEXT:  # %bb.13: # %cond.store11
+; SSE4-NEXT:  .LBB11_13: # %cond.store11
 ; SSE4-NEXT:    pextrw $6, %xmm0, 12(%rdi)
-; SSE4-NEXT:  .LBB11_14: # %else12
-; SSE4-NEXT:    pextrb $12, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $-128, %al
 ; SSE4-NEXT:    je .LBB11_16
-; SSE4-NEXT:  # %bb.15: # %cond.store13
+; SSE4-NEXT:  .LBB11_15: # %cond.store13
 ; SSE4-NEXT:    pextrw $7, %xmm0, 14(%rdi)
-; SSE4-NEXT:  .LBB11_16: # %else14
 ; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: truncstore_v8i32_v8i16:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm4, %xmm4
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm5
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [65535,65535,65535,65535]
-; AVX1-NEXT:    vpminud %xmm6, %xmm5, %xmm5
-; AVX1-NEXT:    vpminud %xmm6, %xmm0, %xmm0
-; AVX1-NEXT:    vpackusdw %xmm5, %xmm0, %xmm0
-; AVX1-NEXT:    vpextrb $0, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je .LBB11_2
-; AVX1-NEXT:  # %bb.1: # %cond.store
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [65535,65535,65535,65535]
+; AVX1-NEXT:    vpminud %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpminud %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpackusdw %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; AVX1-NEXT:    vmovmskps %ymm1, %eax
+; AVX1-NEXT:    notl %eax
+; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    jne .LBB11_1
+; AVX1-NEXT:  # %bb.2: # %else
+; AVX1-NEXT:    testb $2, %al
+; AVX1-NEXT:    jne .LBB11_3
+; AVX1-NEXT:  .LBB11_4: # %else2
+; AVX1-NEXT:    testb $4, %al
+; AVX1-NEXT:    jne .LBB11_5
+; AVX1-NEXT:  .LBB11_6: # %else4
+; AVX1-NEXT:    testb $8, %al
+; AVX1-NEXT:    jne .LBB11_7
+; AVX1-NEXT:  .LBB11_8: # %else6
+; AVX1-NEXT:    testb $16, %al
+; AVX1-NEXT:    jne .LBB11_9
+; AVX1-NEXT:  .LBB11_10: # %else8
+; AVX1-NEXT:    testb $32, %al
+; AVX1-NEXT:    jne .LBB11_11
+; AVX1-NEXT:  .LBB11_12: # %else10
+; AVX1-NEXT:    testb $64, %al
+; AVX1-NEXT:    jne .LBB11_13
+; AVX1-NEXT:  .LBB11_14: # %else12
+; AVX1-NEXT:    testb $-128, %al
+; AVX1-NEXT:    jne .LBB11_15
+; AVX1-NEXT:  .LBB11_16: # %else14
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+; AVX1-NEXT:  .LBB11_1: # %cond.store
 ; AVX1-NEXT:    vpextrw $0, %xmm0, (%rdi)
-; AVX1-NEXT:  .LBB11_2: # %else
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $2, %al
 ; AVX1-NEXT:    je .LBB11_4
-; AVX1-NEXT:  # %bb.3: # %cond.store1
+; AVX1-NEXT:  .LBB11_3: # %cond.store1
 ; AVX1-NEXT:    vpextrw $1, %xmm0, 2(%rdi)
-; AVX1-NEXT:  .LBB11_4: # %else2
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm4, %xmm4
-; AVX1-NEXT:    vpextrb $8, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $4, %al
 ; AVX1-NEXT:    je .LBB11_6
-; AVX1-NEXT:  # %bb.5: # %cond.store3
+; AVX1-NEXT:  .LBB11_5: # %cond.store3
 ; AVX1-NEXT:    vpextrw $2, %xmm0, 4(%rdi)
-; AVX1-NEXT:  .LBB11_6: # %else4
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $12, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $8, %al
 ; AVX1-NEXT:    je .LBB11_8
-; AVX1-NEXT:  # %bb.7: # %cond.store5
+; AVX1-NEXT:  .LBB11_7: # %cond.store5
 ; AVX1-NEXT:    vpextrw $3, %xmm0, 6(%rdi)
-; AVX1-NEXT:  .LBB11_8: # %else6
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm1
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $0, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $16, %al
 ; AVX1-NEXT:    je .LBB11_10
-; AVX1-NEXT:  # %bb.9: # %cond.store7
+; AVX1-NEXT:  .LBB11_9: # %cond.store7
 ; AVX1-NEXT:    vpextrw $4, %xmm0, 8(%rdi)
-; AVX1-NEXT:  .LBB11_10: # %else8
-; AVX1-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $32, %al
 ; AVX1-NEXT:    je .LBB11_12
-; AVX1-NEXT:  # %bb.11: # %cond.store9
+; AVX1-NEXT:  .LBB11_11: # %cond.store9
 ; AVX1-NEXT:    vpextrw $5, %xmm0, 10(%rdi)
-; AVX1-NEXT:  .LBB11_12: # %else10
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $64, %al
 ; AVX1-NEXT:    je .LBB11_14
-; AVX1-NEXT:  # %bb.13: # %cond.store11
+; AVX1-NEXT:  .LBB11_13: # %cond.store11
 ; AVX1-NEXT:    vpextrw $6, %xmm0, 12(%rdi)
-; AVX1-NEXT:  .LBB11_14: # %else12
-; AVX1-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $-128, %al
 ; AVX1-NEXT:    je .LBB11_16
-; AVX1-NEXT:  # %bb.15: # %cond.store13
+; AVX1-NEXT:  .LBB11_15: # %cond.store13
 ; AVX1-NEXT:    vpextrw $7, %xmm0, 14(%rdi)
-; AVX1-NEXT:  .LBB11_16: # %else14
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: truncstore_v8i32_v8i16:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm4
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    vpxor %xmm3, %xmm4, %xmm4
-; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm5 = [65535,65535,65535,65535,65535,65535,65535,65535]
-; AVX2-NEXT:    vpminud %ymm5, %ymm0, %ymm0
-; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm5
-; AVX2-NEXT:    vpackusdw %xmm5, %xmm0, %xmm0
-; AVX2-NEXT:    vpextrb $0, %xmm4, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je .LBB11_2
-; AVX2-NEXT:  # %bb.1: # %cond.store
-; AVX2-NEXT:    vpextrw $0, %xmm0, (%rdi)
-; AVX2-NEXT:  .LBB11_2: # %else
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je .LBB11_4
-; AVX2-NEXT:  # %bb.3: # %cond.store1
-; AVX2-NEXT:    vpextrw $1, %xmm0, 2(%rdi)
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm3 = [65535,65535,65535,65535,65535,65535,65535,65535]
+; AVX2-NEXT:    vpminud %ymm3, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; AVX2-NEXT:    vpackusdw %xmm3, %xmm0, %xmm0
+; AVX2-NEXT:    vpcmpeqd %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    vmovmskps %ymm1, %eax
+; AVX2-NEXT:    notl %eax
+; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    jne .LBB11_1
+; AVX2-NEXT:  # %bb.2: # %else
+; AVX2-NEXT:    testb $2, %al
+; AVX2-NEXT:    jne .LBB11_3
 ; AVX2-NEXT:  .LBB11_4: # %else2
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm4
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    vpxor %xmm3, %xmm4, %xmm4
-; AVX2-NEXT:    vpextrb $8, %xmm4, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je .LBB11_6
-; AVX2-NEXT:  # %bb.5: # %cond.store3
-; AVX2-NEXT:    vpextrw $2, %xmm0, 4(%rdi)
+; AVX2-NEXT:    testb $4, %al
+; AVX2-NEXT:    jne .LBB11_5
 ; AVX2-NEXT:  .LBB11_6: # %else4
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrb $12, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $8, %al
+; AVX2-NEXT:    jne .LBB11_7
+; AVX2-NEXT:  .LBB11_8: # %else6
+; AVX2-NEXT:    testb $16, %al
+; AVX2-NEXT:    jne .LBB11_9
+; AVX2-NEXT:  .LBB11_10: # %else8
+; AVX2-NEXT:    testb $32, %al
+; AVX2-NEXT:    jne .LBB11_11
+; AVX2-NEXT:  .LBB11_12: # %else10
+; AVX2-NEXT:    testb $64, %al
+; AVX2-NEXT:    jne .LBB11_13
+; AVX2-NEXT:  .LBB11_14: # %else12
+; AVX2-NEXT:    testb $-128, %al
+; AVX2-NEXT:    jne .LBB11_15
+; AVX2-NEXT:  .LBB11_16: # %else14
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+; AVX2-NEXT:  .LBB11_1: # %cond.store
+; AVX2-NEXT:    vpextrw $0, %xmm0, (%rdi)
+; AVX2-NEXT:    testb $2, %al
+; AVX2-NEXT:    je .LBB11_4
+; AVX2-NEXT:  .LBB11_3: # %cond.store1
+; AVX2-NEXT:    vpextrw $1, %xmm0, 2(%rdi)
+; AVX2-NEXT:    testb $4, %al
+; AVX2-NEXT:    je .LBB11_6
+; AVX2-NEXT:  .LBB11_5: # %cond.store3
+; AVX2-NEXT:    vpextrw $2, %xmm0, 4(%rdi)
+; AVX2-NEXT:    testb $8, %al
 ; AVX2-NEXT:    je .LBB11_8
-; AVX2-NEXT:  # %bb.7: # %cond.store5
+; AVX2-NEXT:  .LBB11_7: # %cond.store5
 ; AVX2-NEXT:    vpextrw $3, %xmm0, 6(%rdi)
-; AVX2-NEXT:  .LBB11_8: # %else6
-; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm1
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrb $0, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $16, %al
 ; AVX2-NEXT:    je .LBB11_10
-; AVX2-NEXT:  # %bb.9: # %cond.store7
+; AVX2-NEXT:  .LBB11_9: # %cond.store7
 ; AVX2-NEXT:    vpextrw $4, %xmm0, 8(%rdi)
-; AVX2-NEXT:  .LBB11_10: # %else8
-; AVX2-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $32, %al
 ; AVX2-NEXT:    je .LBB11_12
-; AVX2-NEXT:  # %bb.11: # %cond.store9
+; AVX2-NEXT:  .LBB11_11: # %cond.store9
 ; AVX2-NEXT:    vpextrw $5, %xmm0, 10(%rdi)
-; AVX2-NEXT:  .LBB11_12: # %else10
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $64, %al
 ; AVX2-NEXT:    je .LBB11_14
-; AVX2-NEXT:  # %bb.13: # %cond.store11
+; AVX2-NEXT:  .LBB11_13: # %cond.store11
 ; AVX2-NEXT:    vpextrw $6, %xmm0, 12(%rdi)
-; AVX2-NEXT:  .LBB11_14: # %else12
-; AVX2-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $-128, %al
 ; AVX2-NEXT:    je .LBB11_16
-; AVX2-NEXT:  # %bb.15: # %cond.store13
+; AVX2-NEXT:  .LBB11_15: # %cond.store13
 ; AVX2-NEXT:    vpextrw $7, %xmm0, 14(%rdi)
-; AVX2-NEXT:  .LBB11_16: # %else14
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: truncstore_v8i32_v8i16:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
 ; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    vpbroadcastd {{.*#+}} ymm2 = [65535,65535,65535,65535,65535,65535,65535,65535]
-; AVX512F-NEXT:    vpminud %ymm2, %ymm0, %ymm0
-; AVX512F-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512F-NEXT:    vpmovusdw %zmm0, %ymm0
 ; AVX512F-NEXT:    kmovw %k0, %eax
 ; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB11_2
-; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    jne .LBB11_1
+; AVX512F-NEXT:  # %bb.2: # %else
+; AVX512F-NEXT:    testb $2, %al
+; AVX512F-NEXT:    jne .LBB11_3
+; AVX512F-NEXT:  .LBB11_4: # %else2
+; AVX512F-NEXT:    testb $4, %al
+; AVX512F-NEXT:    jne .LBB11_5
+; AVX512F-NEXT:  .LBB11_6: # %else4
+; AVX512F-NEXT:    testb $8, %al
+; AVX512F-NEXT:    jne .LBB11_7
+; AVX512F-NEXT:  .LBB11_8: # %else6
+; AVX512F-NEXT:    testb $16, %al
+; AVX512F-NEXT:    jne .LBB11_9
+; AVX512F-NEXT:  .LBB11_10: # %else8
+; AVX512F-NEXT:    testb $32, %al
+; AVX512F-NEXT:    jne .LBB11_11
+; AVX512F-NEXT:  .LBB11_12: # %else10
+; AVX512F-NEXT:    testb $64, %al
+; AVX512F-NEXT:    jne .LBB11_13
+; AVX512F-NEXT:  .LBB11_14: # %else12
+; AVX512F-NEXT:    testb $-128, %al
+; AVX512F-NEXT:    jne .LBB11_15
+; AVX512F-NEXT:  .LBB11_16: # %else14
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+; AVX512F-NEXT:  .LBB11_1: # %cond.store
 ; AVX512F-NEXT:    vpextrw $0, %xmm0, (%rdi)
-; AVX512F-NEXT:  .LBB11_2: # %else
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $2, %al
 ; AVX512F-NEXT:    je .LBB11_4
-; AVX512F-NEXT:  # %bb.3: # %cond.store1
+; AVX512F-NEXT:  .LBB11_3: # %cond.store1
 ; AVX512F-NEXT:    vpextrw $1, %xmm0, 2(%rdi)
-; AVX512F-NEXT:  .LBB11_4: # %else2
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $4, %al
 ; AVX512F-NEXT:    je .LBB11_6
-; AVX512F-NEXT:  # %bb.5: # %cond.store3
+; AVX512F-NEXT:  .LBB11_5: # %cond.store3
 ; AVX512F-NEXT:    vpextrw $2, %xmm0, 4(%rdi)
-; AVX512F-NEXT:  .LBB11_6: # %else4
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $8, %al
 ; AVX512F-NEXT:    je .LBB11_8
-; AVX512F-NEXT:  # %bb.7: # %cond.store5
+; AVX512F-NEXT:  .LBB11_7: # %cond.store5
 ; AVX512F-NEXT:    vpextrw $3, %xmm0, 6(%rdi)
-; AVX512F-NEXT:  .LBB11_8: # %else6
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $4, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $16, %al
 ; AVX512F-NEXT:    je .LBB11_10
-; AVX512F-NEXT:  # %bb.9: # %cond.store7
+; AVX512F-NEXT:  .LBB11_9: # %cond.store7
 ; AVX512F-NEXT:    vpextrw $4, %xmm0, 8(%rdi)
-; AVX512F-NEXT:  .LBB11_10: # %else8
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $5, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $32, %al
 ; AVX512F-NEXT:    je .LBB11_12
-; AVX512F-NEXT:  # %bb.11: # %cond.store9
+; AVX512F-NEXT:  .LBB11_11: # %cond.store9
 ; AVX512F-NEXT:    vpextrw $5, %xmm0, 10(%rdi)
-; AVX512F-NEXT:  .LBB11_12: # %else10
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $6, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $64, %al
 ; AVX512F-NEXT:    je .LBB11_14
-; AVX512F-NEXT:  # %bb.13: # %cond.store11
+; AVX512F-NEXT:  .LBB11_13: # %cond.store11
 ; AVX512F-NEXT:    vpextrw $6, %xmm0, 12(%rdi)
-; AVX512F-NEXT:  .LBB11_14: # %else12
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $7, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $-128, %al
 ; AVX512F-NEXT:    je .LBB11_16
-; AVX512F-NEXT:  # %bb.15: # %cond.store13
+; AVX512F-NEXT:  .LBB11_15: # %cond.store13
 ; AVX512F-NEXT:    vpextrw $7, %xmm0, 14(%rdi)
-; AVX512F-NEXT:  .LBB11_16: # %else14
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: truncstore_v8i32_v8i16:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
 ; AVX512BW-NEXT:    vptestmd %zmm1, %zmm1, %k0
 ; AVX512BW-NEXT:    kshiftld $24, %k0, %k0
 ; AVX512BW-NEXT:    kshiftrd $24, %k0, %k1
-; AVX512BW-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [65535,65535,65535,65535,65535,65535,65535,65535]
-; AVX512BW-NEXT:    vpminud %ymm1, %ymm0, %ymm0
-; AVX512BW-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512BW-NEXT:    vpmovusdw %zmm0, %ymm0
 ; AVX512BW-NEXT:    vmovdqu16 %zmm0, (%rdi) {%k1}
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -4671,411 +4352,387 @@ define void @truncstore_v8i32_v8i16(<8 x i32> %x, <8 x i16>* %p, <8 x i32> %mask
 define void @truncstore_v8i32_v8i8(<8 x i32> %x, <8 x i8>* %p, <8 x i32> %mask) {
 ; SSE2-LABEL: truncstore_v8i32_v8i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pxor %xmm4, %xmm4
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm4
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm5
-; SSE2-NEXT:    pxor %xmm4, %xmm5
-; SSE2-NEXT:    movdqa %xmm5, %xmm9
-; SSE2-NEXT:    packssdw %xmm0, %xmm9
-; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [255,255,255,255]
+; SSE2-NEXT:    pxor %xmm8, %xmm8
+; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [255,255,255,255]
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm7 = [2147483648,2147483648,2147483648,2147483648]
-; SSE2-NEXT:    movdqa %xmm1, %xmm6
-; SSE2-NEXT:    pxor %xmm7, %xmm6
-; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [2147483903,2147483903,2147483903,2147483903]
-; SSE2-NEXT:    movdqa %xmm10, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm6, %xmm4
+; SSE2-NEXT:    movdqa %xmm1, %xmm5
+; SSE2-NEXT:    pxor %xmm7, %xmm5
+; SSE2-NEXT:    movdqa {{.*#+}} xmm6 = [2147483903,2147483903,2147483903,2147483903]
+; SSE2-NEXT:    movdqa %xmm6, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm5, %xmm4
 ; SSE2-NEXT:    pand %xmm4, %xmm1
-; SSE2-NEXT:    pandn %xmm8, %xmm4
+; SSE2-NEXT:    pandn %xmm9, %xmm4
 ; SSE2-NEXT:    por %xmm1, %xmm4
 ; SSE2-NEXT:    pxor %xmm0, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm7, %xmm10
-; SSE2-NEXT:    pand %xmm10, %xmm0
-; SSE2-NEXT:    pandn %xmm8, %xmm10
-; SSE2-NEXT:    por %xmm0, %xmm10
-; SSE2-NEXT:    packuswb %xmm4, %xmm10
-; SSE2-NEXT:    movd %xmm9, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    movd %xmm10, %eax
-; SSE2-NEXT:    je .LBB12_2
-; SSE2-NEXT:  # %bb.1: # %cond.store
-; SSE2-NEXT:    movb %al, (%rdi)
-; SSE2-NEXT:  .LBB12_2: # %else
-; SSE2-NEXT:    psrlq $16, %xmm5
-; SSE2-NEXT:    movd %xmm5, %ecx
-; SSE2-NEXT:    shrl $16, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    je .LBB12_4
-; SSE2-NEXT:  # %bb.3: # %cond.store1
-; SSE2-NEXT:    shrl $16, %eax
-; SSE2-NEXT:    movb %al, 1(%rdi)
-; SSE2-NEXT:  .LBB12_4: # %else2
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm7, %xmm6
+; SSE2-NEXT:    pand %xmm6, %xmm0
+; SSE2-NEXT:    pandn %xmm9, %xmm6
+; SSE2-NEXT:    por %xmm0, %xmm6
+; SSE2-NEXT:    packuswb %xmm4, %xmm6
+; SSE2-NEXT:    packuswb %xmm6, %xmm6
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm3
 ; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE2-NEXT:    pxor %xmm2, %xmm0
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
+; SSE2-NEXT:    pxor %xmm0, %xmm3
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm2
+; SSE2-NEXT:    pxor %xmm0, %xmm2
+; SSE2-NEXT:    packssdw %xmm3, %xmm2
+; SSE2-NEXT:    packsswb %xmm0, %xmm2
+; SSE2-NEXT:    pmovmskb %xmm2, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB12_6
-; SSE2-NEXT:  # %bb.5: # %cond.store3
-; SSE2-NEXT:    pextrw $2, %xmm10, %eax
-; SSE2-NEXT:    movb %al, 2(%rdi)
+; SSE2-NEXT:    movd %xmm6, %ecx
+; SSE2-NEXT:    jne .LBB12_1
+; SSE2-NEXT:  # %bb.2: # %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne .LBB12_3
+; SSE2-NEXT:  .LBB12_4: # %else2
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne .LBB12_5
 ; SSE2-NEXT:  .LBB12_6: # %else4
-; SSE2-NEXT:    pextrw $6, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je .LBB12_8
-; SSE2-NEXT:  # %bb.7: # %cond.store5
-; SSE2-NEXT:    pextrw $3, %xmm10, %eax
-; SSE2-NEXT:    movb %al, 3(%rdi)
+; SSE2-NEXT:  .LBB12_7: # %cond.store5
+; SSE2-NEXT:    shrl $24, %ecx
+; SSE2-NEXT:    movb %cl, 3(%rdi)
 ; SSE2-NEXT:  .LBB12_8: # %else6
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE2-NEXT:    pxor %xmm1, %xmm0
-; SSE2-NEXT:    pextrw $0, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $16, %al
+; SSE2-NEXT:    pextrw $2, %xmm6, %ecx
 ; SSE2-NEXT:    je .LBB12_10
 ; SSE2-NEXT:  # %bb.9: # %cond.store7
-; SSE2-NEXT:    pextrw $4, %xmm10, %eax
-; SSE2-NEXT:    movb %al, 4(%rdi)
+; SSE2-NEXT:    movb %cl, 4(%rdi)
 ; SSE2-NEXT:  .LBB12_10: # %else8
-; SSE2-NEXT:    pextrw $2, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $32, %al
 ; SSE2-NEXT:    je .LBB12_12
 ; SSE2-NEXT:  # %bb.11: # %cond.store9
-; SSE2-NEXT:    pextrw $5, %xmm10, %eax
-; SSE2-NEXT:    movb %al, 5(%rdi)
+; SSE2-NEXT:    movb %ch, 5(%rdi)
 ; SSE2-NEXT:  .LBB12_12: # %else10
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm3
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE2-NEXT:    pxor %xmm3, %xmm0
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB12_14
-; SSE2-NEXT:  # %bb.13: # %cond.store11
-; SSE2-NEXT:    pextrw $6, %xmm10, %eax
-; SSE2-NEXT:    movb %al, 6(%rdi)
-; SSE2-NEXT:  .LBB12_14: # %else12
-; SSE2-NEXT:    pextrw $6, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB12_16
-; SSE2-NEXT:  # %bb.15: # %cond.store13
-; SSE2-NEXT:    pextrw $7, %xmm10, %eax
-; SSE2-NEXT:    movb %al, 7(%rdi)
+; SSE2-NEXT:    testb $64, %al
+; SSE2-NEXT:    pextrw $3, %xmm6, %ecx
+; SSE2-NEXT:    jne .LBB12_13
+; SSE2-NEXT:  # %bb.14: # %else12
+; SSE2-NEXT:    testb $-128, %al
+; SSE2-NEXT:    jne .LBB12_15
 ; SSE2-NEXT:  .LBB12_16: # %else14
 ; SSE2-NEXT:    retq
+; SSE2-NEXT:  .LBB12_1: # %cond.store
+; SSE2-NEXT:    movb %cl, (%rdi)
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    je .LBB12_4
+; SSE2-NEXT:  .LBB12_3: # %cond.store1
+; SSE2-NEXT:    movb %ch, 1(%rdi)
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    je .LBB12_6
+; SSE2-NEXT:  .LBB12_5: # %cond.store3
+; SSE2-NEXT:    movl %ecx, %edx
+; SSE2-NEXT:    shrl $16, %edx
+; SSE2-NEXT:    movb %dl, 2(%rdi)
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne .LBB12_7
+; SSE2-NEXT:    jmp .LBB12_8
+; SSE2-NEXT:  .LBB12_13: # %cond.store11
+; SSE2-NEXT:    movb %cl, 6(%rdi)
+; SSE2-NEXT:    testb $-128, %al
+; SSE2-NEXT:    je .LBB12_16
+; SSE2-NEXT:  .LBB12_15: # %cond.store13
+; SSE2-NEXT:    movb %ch, 7(%rdi)
+; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: truncstore_v8i32_v8i8:
 ; SSE4:       # %bb.0:
-; SSE4-NEXT:    pxor %xmm5, %xmm5
-; SSE4-NEXT:    pcmpeqd %xmm2, %xmm5
-; SSE4-NEXT:    pcmpeqd %xmm4, %xmm4
-; SSE4-NEXT:    pxor %xmm5, %xmm4
+; SSE4-NEXT:    pxor %xmm4, %xmm4
 ; SSE4-NEXT:    movdqa {{.*#+}} xmm5 = [255,255,255,255]
-; SSE4-NEXT:    pminud %xmm5, %xmm1
 ; SSE4-NEXT:    pminud %xmm5, %xmm0
-; SSE4-NEXT:    packusdw %xmm1, %xmm0
-; SSE4-NEXT:    pextrb $0, %xmm4, %eax
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB12_2
-; SSE4-NEXT:  # %bb.1: # %cond.store
+; SSE4-NEXT:    pminud %xmm5, %xmm1
+; SSE4-NEXT:    movdqa {{.*#+}} xmm5 = <0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u>
+; SSE4-NEXT:    pshufb %xmm5, %xmm1
+; SSE4-NEXT:    pshufb %xmm5, %xmm0
+; SSE4-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE4-NEXT:    pcmpeqd %xmm4, %xmm3
+; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
+; SSE4-NEXT:    pxor %xmm1, %xmm3
+; SSE4-NEXT:    pcmpeqd %xmm4, %xmm2
+; SSE4-NEXT:    pxor %xmm1, %xmm2
+; SSE4-NEXT:    packssdw %xmm3, %xmm2
+; SSE4-NEXT:    packsswb %xmm0, %xmm2
+; SSE4-NEXT:    pmovmskb %xmm2, %eax
+; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    jne .LBB12_1
+; SSE4-NEXT:  # %bb.2: # %else
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    jne .LBB12_3
+; SSE4-NEXT:  .LBB12_4: # %else2
+; SSE4-NEXT:    testb $4, %al
+; SSE4-NEXT:    jne .LBB12_5
+; SSE4-NEXT:  .LBB12_6: # %else4
+; SSE4-NEXT:    testb $8, %al
+; SSE4-NEXT:    jne .LBB12_7
+; SSE4-NEXT:  .LBB12_8: # %else6
+; SSE4-NEXT:    testb $16, %al
+; SSE4-NEXT:    jne .LBB12_9
+; SSE4-NEXT:  .LBB12_10: # %else8
+; SSE4-NEXT:    testb $32, %al
+; SSE4-NEXT:    jne .LBB12_11
+; SSE4-NEXT:  .LBB12_12: # %else10
+; SSE4-NEXT:    testb $64, %al
+; SSE4-NEXT:    jne .LBB12_13
+; SSE4-NEXT:  .LBB12_14: # %else12
+; SSE4-NEXT:    testb $-128, %al
+; SSE4-NEXT:    jne .LBB12_15
+; SSE4-NEXT:  .LBB12_16: # %else14
+; SSE4-NEXT:    retq
+; SSE4-NEXT:  .LBB12_1: # %cond.store
 ; SSE4-NEXT:    pextrb $0, %xmm0, (%rdi)
-; SSE4-NEXT:  .LBB12_2: # %else
-; SSE4-NEXT:    pextrb $4, %xmm4, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $2, %al
 ; SSE4-NEXT:    je .LBB12_4
-; SSE4-NEXT:  # %bb.3: # %cond.store1
-; SSE4-NEXT:    pextrb $2, %xmm0, 1(%rdi)
-; SSE4-NEXT:  .LBB12_4: # %else2
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm2
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE4-NEXT:    pxor %xmm2, %xmm1
-; SSE4-NEXT:    pextrb $8, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:  .LBB12_3: # %cond.store1
+; SSE4-NEXT:    pextrb $1, %xmm0, 1(%rdi)
+; SSE4-NEXT:    testb $4, %al
 ; SSE4-NEXT:    je .LBB12_6
-; SSE4-NEXT:  # %bb.5: # %cond.store3
-; SSE4-NEXT:    pextrb $4, %xmm0, 2(%rdi)
-; SSE4-NEXT:  .LBB12_6: # %else4
-; SSE4-NEXT:    pextrb $12, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:  .LBB12_5: # %cond.store3
+; SSE4-NEXT:    pextrb $2, %xmm0, 2(%rdi)
+; SSE4-NEXT:    testb $8, %al
 ; SSE4-NEXT:    je .LBB12_8
-; SSE4-NEXT:  # %bb.7: # %cond.store5
-; SSE4-NEXT:    pextrb $6, %xmm0, 3(%rdi)
-; SSE4-NEXT:  .LBB12_8: # %else6
-; SSE4-NEXT:    pxor %xmm2, %xmm2
-; SSE4-NEXT:    pcmpeqd %xmm3, %xmm2
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE4-NEXT:    pxor %xmm2, %xmm1
-; SSE4-NEXT:    pextrb $0, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:  .LBB12_7: # %cond.store5
+; SSE4-NEXT:    pextrb $3, %xmm0, 3(%rdi)
+; SSE4-NEXT:    testb $16, %al
 ; SSE4-NEXT:    je .LBB12_10
-; SSE4-NEXT:  # %bb.9: # %cond.store7
-; SSE4-NEXT:    pextrb $8, %xmm0, 4(%rdi)
-; SSE4-NEXT:  .LBB12_10: # %else8
-; SSE4-NEXT:    pextrb $4, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:  .LBB12_9: # %cond.store7
+; SSE4-NEXT:    pextrb $4, %xmm0, 4(%rdi)
+; SSE4-NEXT:    testb $32, %al
 ; SSE4-NEXT:    je .LBB12_12
-; SSE4-NEXT:  # %bb.11: # %cond.store9
-; SSE4-NEXT:    pextrb $10, %xmm0, 5(%rdi)
-; SSE4-NEXT:  .LBB12_12: # %else10
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm3
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE4-NEXT:    pxor %xmm3, %xmm1
-; SSE4-NEXT:    pextrb $8, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:  .LBB12_11: # %cond.store9
+; SSE4-NEXT:    pextrb $5, %xmm0, 5(%rdi)
+; SSE4-NEXT:    testb $64, %al
 ; SSE4-NEXT:    je .LBB12_14
-; SSE4-NEXT:  # %bb.13: # %cond.store11
-; SSE4-NEXT:    pextrb $12, %xmm0, 6(%rdi)
-; SSE4-NEXT:  .LBB12_14: # %else12
-; SSE4-NEXT:    pextrb $12, %xmm1, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:  .LBB12_13: # %cond.store11
+; SSE4-NEXT:    pextrb $6, %xmm0, 6(%rdi)
+; SSE4-NEXT:    testb $-128, %al
 ; SSE4-NEXT:    je .LBB12_16
-; SSE4-NEXT:  # %bb.15: # %cond.store13
-; SSE4-NEXT:    pextrb $14, %xmm0, 7(%rdi)
-; SSE4-NEXT:  .LBB12_16: # %else14
+; SSE4-NEXT:  .LBB12_15: # %cond.store13
+; SSE4-NEXT:    pextrb $7, %xmm0, 7(%rdi)
 ; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: truncstore_v8i32_v8i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm4, %xmm4
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm5
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [255,255,255,255]
-; AVX1-NEXT:    vpminud %xmm6, %xmm5, %xmm5
-; AVX1-NEXT:    vpminud %xmm6, %xmm0, %xmm0
-; AVX1-NEXT:    vpackusdw %xmm5, %xmm0, %xmm0
-; AVX1-NEXT:    vpextrb $0, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je .LBB12_2
-; AVX1-NEXT:  # %bb.1: # %cond.store
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [255,255,255,255]
+; AVX1-NEXT:    vpminud %xmm2, %xmm0, %xmm3
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vpminud %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = <0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX1-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufb %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; AVX1-NEXT:    vmovmskps %ymm1, %eax
+; AVX1-NEXT:    notl %eax
+; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    jne .LBB12_1
+; AVX1-NEXT:  # %bb.2: # %else
+; AVX1-NEXT:    testb $2, %al
+; AVX1-NEXT:    jne .LBB12_3
+; AVX1-NEXT:  .LBB12_4: # %else2
+; AVX1-NEXT:    testb $4, %al
+; AVX1-NEXT:    jne .LBB12_5
+; AVX1-NEXT:  .LBB12_6: # %else4
+; AVX1-NEXT:    testb $8, %al
+; AVX1-NEXT:    jne .LBB12_7
+; AVX1-NEXT:  .LBB12_8: # %else6
+; AVX1-NEXT:    testb $16, %al
+; AVX1-NEXT:    jne .LBB12_9
+; AVX1-NEXT:  .LBB12_10: # %else8
+; AVX1-NEXT:    testb $32, %al
+; AVX1-NEXT:    jne .LBB12_11
+; AVX1-NEXT:  .LBB12_12: # %else10
+; AVX1-NEXT:    testb $64, %al
+; AVX1-NEXT:    jne .LBB12_13
+; AVX1-NEXT:  .LBB12_14: # %else12
+; AVX1-NEXT:    testb $-128, %al
+; AVX1-NEXT:    jne .LBB12_15
+; AVX1-NEXT:  .LBB12_16: # %else14
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+; AVX1-NEXT:  .LBB12_1: # %cond.store
 ; AVX1-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX1-NEXT:  .LBB12_2: # %else
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $2, %al
 ; AVX1-NEXT:    je .LBB12_4
-; AVX1-NEXT:  # %bb.3: # %cond.store1
-; AVX1-NEXT:    vpextrb $2, %xmm0, 1(%rdi)
-; AVX1-NEXT:  .LBB12_4: # %else2
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm4, %xmm4
-; AVX1-NEXT:    vpextrb $8, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:  .LBB12_3: # %cond.store1
+; AVX1-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
+; AVX1-NEXT:    testb $4, %al
 ; AVX1-NEXT:    je .LBB12_6
-; AVX1-NEXT:  # %bb.5: # %cond.store3
-; AVX1-NEXT:    vpextrb $4, %xmm0, 2(%rdi)
-; AVX1-NEXT:  .LBB12_6: # %else4
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $12, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:  .LBB12_5: # %cond.store3
+; AVX1-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
+; AVX1-NEXT:    testb $8, %al
 ; AVX1-NEXT:    je .LBB12_8
-; AVX1-NEXT:  # %bb.7: # %cond.store5
-; AVX1-NEXT:    vpextrb $6, %xmm0, 3(%rdi)
-; AVX1-NEXT:  .LBB12_8: # %else6
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm1
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $0, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:  .LBB12_7: # %cond.store5
+; AVX1-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
+; AVX1-NEXT:    testb $16, %al
 ; AVX1-NEXT:    je .LBB12_10
-; AVX1-NEXT:  # %bb.9: # %cond.store7
-; AVX1-NEXT:    vpextrb $8, %xmm0, 4(%rdi)
-; AVX1-NEXT:  .LBB12_10: # %else8
-; AVX1-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:  .LBB12_9: # %cond.store7
+; AVX1-NEXT:    vpextrb $4, %xmm0, 4(%rdi)
+; AVX1-NEXT:    testb $32, %al
 ; AVX1-NEXT:    je .LBB12_12
-; AVX1-NEXT:  # %bb.11: # %cond.store9
-; AVX1-NEXT:    vpextrb $10, %xmm0, 5(%rdi)
-; AVX1-NEXT:  .LBB12_12: # %else10
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:  .LBB12_11: # %cond.store9
+; AVX1-NEXT:    vpextrb $5, %xmm0, 5(%rdi)
+; AVX1-NEXT:    testb $64, %al
 ; AVX1-NEXT:    je .LBB12_14
-; AVX1-NEXT:  # %bb.13: # %cond.store11
-; AVX1-NEXT:    vpextrb $12, %xmm0, 6(%rdi)
-; AVX1-NEXT:  .LBB12_14: # %else12
-; AVX1-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:  .LBB12_13: # %cond.store11
+; AVX1-NEXT:    vpextrb $6, %xmm0, 6(%rdi)
+; AVX1-NEXT:    testb $-128, %al
 ; AVX1-NEXT:    je .LBB12_16
-; AVX1-NEXT:  # %bb.15: # %cond.store13
-; AVX1-NEXT:    vpextrb $14, %xmm0, 7(%rdi)
-; AVX1-NEXT:  .LBB12_16: # %else14
+; AVX1-NEXT:  .LBB12_15: # %cond.store13
+; AVX1-NEXT:    vpextrb $7, %xmm0, 7(%rdi)
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: truncstore_v8i32_v8i8:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm4
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    vpxor %xmm3, %xmm4, %xmm4
-; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm5 = [255,255,255,255,255,255,255,255]
-; AVX2-NEXT:    vpminud %ymm5, %ymm0, %ymm0
-; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm5
-; AVX2-NEXT:    vpackusdw %xmm5, %xmm0, %xmm0
-; AVX2-NEXT:    vpextrb $0, %xmm4, %eax
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je .LBB12_2
-; AVX2-NEXT:  # %bb.1: # %cond.store
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm3 = [255,255,255,255,255,255,255,255]
+; AVX2-NEXT:    vpminud %ymm3, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm4 = <0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX2-NEXT:    vpshufb %xmm4, %xmm3, %xmm3
+; AVX2-NEXT:    vpshufb %xmm4, %xmm0, %xmm0
+; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; AVX2-NEXT:    vpcmpeqd %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    vmovmskps %ymm1, %eax
+; AVX2-NEXT:    notl %eax
+; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    jne .LBB12_1
+; AVX2-NEXT:  # %bb.2: # %else
+; AVX2-NEXT:    testb $2, %al
+; AVX2-NEXT:    jne .LBB12_3
+; AVX2-NEXT:  .LBB12_4: # %else2
+; AVX2-NEXT:    testb $4, %al
+; AVX2-NEXT:    jne .LBB12_5
+; AVX2-NEXT:  .LBB12_6: # %else4
+; AVX2-NEXT:    testb $8, %al
+; AVX2-NEXT:    jne .LBB12_7
+; AVX2-NEXT:  .LBB12_8: # %else6
+; AVX2-NEXT:    testb $16, %al
+; AVX2-NEXT:    jne .LBB12_9
+; AVX2-NEXT:  .LBB12_10: # %else8
+; AVX2-NEXT:    testb $32, %al
+; AVX2-NEXT:    jne .LBB12_11
+; AVX2-NEXT:  .LBB12_12: # %else10
+; AVX2-NEXT:    testb $64, %al
+; AVX2-NEXT:    jne .LBB12_13
+; AVX2-NEXT:  .LBB12_14: # %else12
+; AVX2-NEXT:    testb $-128, %al
+; AVX2-NEXT:    jne .LBB12_15
+; AVX2-NEXT:  .LBB12_16: # %else14
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+; AVX2-NEXT:  .LBB12_1: # %cond.store
 ; AVX2-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX2-NEXT:  .LBB12_2: # %else
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $2, %al
 ; AVX2-NEXT:    je .LBB12_4
-; AVX2-NEXT:  # %bb.3: # %cond.store1
-; AVX2-NEXT:    vpextrb $2, %xmm0, 1(%rdi)
-; AVX2-NEXT:  .LBB12_4: # %else2
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm4
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    vpxor %xmm3, %xmm4, %xmm4
-; AVX2-NEXT:    vpextrb $8, %xmm4, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:  .LBB12_3: # %cond.store1
+; AVX2-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
+; AVX2-NEXT:    testb $4, %al
 ; AVX2-NEXT:    je .LBB12_6
-; AVX2-NEXT:  # %bb.5: # %cond.store3
-; AVX2-NEXT:    vpextrb $4, %xmm0, 2(%rdi)
-; AVX2-NEXT:  .LBB12_6: # %else4
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrb $12, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:  .LBB12_5: # %cond.store3
+; AVX2-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
+; AVX2-NEXT:    testb $8, %al
 ; AVX2-NEXT:    je .LBB12_8
-; AVX2-NEXT:  # %bb.7: # %cond.store5
-; AVX2-NEXT:    vpextrb $6, %xmm0, 3(%rdi)
-; AVX2-NEXT:  .LBB12_8: # %else6
-; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm1
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrb $0, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:  .LBB12_7: # %cond.store5
+; AVX2-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
+; AVX2-NEXT:    testb $16, %al
 ; AVX2-NEXT:    je .LBB12_10
-; AVX2-NEXT:  # %bb.9: # %cond.store7
-; AVX2-NEXT:    vpextrb $8, %xmm0, 4(%rdi)
-; AVX2-NEXT:  .LBB12_10: # %else8
-; AVX2-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:  .LBB12_9: # %cond.store7
+; AVX2-NEXT:    vpextrb $4, %xmm0, 4(%rdi)
+; AVX2-NEXT:    testb $32, %al
 ; AVX2-NEXT:    je .LBB12_12
-; AVX2-NEXT:  # %bb.11: # %cond.store9
-; AVX2-NEXT:    vpextrb $10, %xmm0, 5(%rdi)
-; AVX2-NEXT:  .LBB12_12: # %else10
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:  .LBB12_11: # %cond.store9
+; AVX2-NEXT:    vpextrb $5, %xmm0, 5(%rdi)
+; AVX2-NEXT:    testb $64, %al
 ; AVX2-NEXT:    je .LBB12_14
-; AVX2-NEXT:  # %bb.13: # %cond.store11
-; AVX2-NEXT:    vpextrb $12, %xmm0, 6(%rdi)
-; AVX2-NEXT:  .LBB12_14: # %else12
-; AVX2-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:  .LBB12_13: # %cond.store11
+; AVX2-NEXT:    vpextrb $6, %xmm0, 6(%rdi)
+; AVX2-NEXT:    testb $-128, %al
 ; AVX2-NEXT:    je .LBB12_16
-; AVX2-NEXT:  # %bb.15: # %cond.store13
-; AVX2-NEXT:    vpextrb $14, %xmm0, 7(%rdi)
-; AVX2-NEXT:  .LBB12_16: # %else14
+; AVX2-NEXT:  .LBB12_15: # %cond.store13
+; AVX2-NEXT:    vpextrb $7, %xmm0, 7(%rdi)
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: truncstore_v8i32_v8i8:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
 ; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    vpbroadcastd {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255]
-; AVX512F-NEXT:    vpminud %ymm2, %ymm0, %ymm0
-; AVX512F-NEXT:    vpmovdw %zmm0, %ymm0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB12_2
-; AVX512F-NEXT:  # %bb.1: # %cond.store
-; AVX512F-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX512F-NEXT:  .LBB12_2: # %else
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
+; AVX512F-NEXT:    vpmovusdb %zmm0, %xmm0
 ; AVX512F-NEXT:    kmovw %k0, %eax
 ; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB12_4
-; AVX512F-NEXT:  # %bb.3: # %cond.store1
-; AVX512F-NEXT:    vpextrb $2, %xmm0, 1(%rdi)
+; AVX512F-NEXT:    jne .LBB12_1
+; AVX512F-NEXT:  # %bb.2: # %else
+; AVX512F-NEXT:    testb $2, %al
+; AVX512F-NEXT:    jne .LBB12_3
 ; AVX512F-NEXT:  .LBB12_4: # %else2
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB12_6
-; AVX512F-NEXT:  # %bb.5: # %cond.store3
-; AVX512F-NEXT:    vpextrb $4, %xmm0, 2(%rdi)
+; AVX512F-NEXT:    testb $4, %al
+; AVX512F-NEXT:    jne .LBB12_5
 ; AVX512F-NEXT:  .LBB12_6: # %else4
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB12_8
-; AVX512F-NEXT:  # %bb.7: # %cond.store5
-; AVX512F-NEXT:    vpextrb $6, %xmm0, 3(%rdi)
+; AVX512F-NEXT:    testb $8, %al
+; AVX512F-NEXT:    jne .LBB12_7
 ; AVX512F-NEXT:  .LBB12_8: # %else6
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $4, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB12_10
-; AVX512F-NEXT:  # %bb.9: # %cond.store7
-; AVX512F-NEXT:    vpextrb $8, %xmm0, 4(%rdi)
+; AVX512F-NEXT:    testb $16, %al
+; AVX512F-NEXT:    jne .LBB12_9
 ; AVX512F-NEXT:  .LBB12_10: # %else8
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $5, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB12_12
-; AVX512F-NEXT:  # %bb.11: # %cond.store9
-; AVX512F-NEXT:    vpextrb $10, %xmm0, 5(%rdi)
+; AVX512F-NEXT:    testb $32, %al
+; AVX512F-NEXT:    jne .LBB12_11
 ; AVX512F-NEXT:  .LBB12_12: # %else10
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $6, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB12_14
-; AVX512F-NEXT:  # %bb.13: # %cond.store11
-; AVX512F-NEXT:    vpextrb $12, %xmm0, 6(%rdi)
+; AVX512F-NEXT:    testb $64, %al
+; AVX512F-NEXT:    jne .LBB12_13
 ; AVX512F-NEXT:  .LBB12_14: # %else12
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $7, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB12_16
-; AVX512F-NEXT:  # %bb.15: # %cond.store13
-; AVX512F-NEXT:    vpextrb $14, %xmm0, 7(%rdi)
+; AVX512F-NEXT:    testb $-128, %al
+; AVX512F-NEXT:    jne .LBB12_15
 ; AVX512F-NEXT:  .LBB12_16: # %else14
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
+; AVX512F-NEXT:  .LBB12_1: # %cond.store
+; AVX512F-NEXT:    vpextrb $0, %xmm0, (%rdi)
+; AVX512F-NEXT:    testb $2, %al
+; AVX512F-NEXT:    je .LBB12_4
+; AVX512F-NEXT:  .LBB12_3: # %cond.store1
+; AVX512F-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
+; AVX512F-NEXT:    testb $4, %al
+; AVX512F-NEXT:    je .LBB12_6
+; AVX512F-NEXT:  .LBB12_5: # %cond.store3
+; AVX512F-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
+; AVX512F-NEXT:    testb $8, %al
+; AVX512F-NEXT:    je .LBB12_8
+; AVX512F-NEXT:  .LBB12_7: # %cond.store5
+; AVX512F-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
+; AVX512F-NEXT:    testb $16, %al
+; AVX512F-NEXT:    je .LBB12_10
+; AVX512F-NEXT:  .LBB12_9: # %cond.store7
+; AVX512F-NEXT:    vpextrb $4, %xmm0, 4(%rdi)
+; AVX512F-NEXT:    testb $32, %al
+; AVX512F-NEXT:    je .LBB12_12
+; AVX512F-NEXT:  .LBB12_11: # %cond.store9
+; AVX512F-NEXT:    vpextrb $5, %xmm0, 5(%rdi)
+; AVX512F-NEXT:    testb $64, %al
+; AVX512F-NEXT:    je .LBB12_14
+; AVX512F-NEXT:  .LBB12_13: # %cond.store11
+; AVX512F-NEXT:    vpextrb $6, %xmm0, 6(%rdi)
+; AVX512F-NEXT:    testb $-128, %al
+; AVX512F-NEXT:    je .LBB12_16
+; AVX512F-NEXT:  .LBB12_15: # %cond.store13
+; AVX512F-NEXT:    vpextrb $7, %xmm0, 7(%rdi)
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: truncstore_v8i32_v8i8:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
 ; AVX512BW-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512BW-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [255,255,255,255,255,255,255,255]
-; AVX512BW-NEXT:    vpminud %ymm1, %ymm0, %ymm0
-; AVX512BW-NEXT:    vpmovdw %zmm0, %ymm0
-; AVX512BW-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
 ; AVX512BW-NEXT:    kshiftlq $56, %k0, %k0
 ; AVX512BW-NEXT:    kshiftrq $56, %k0, %k1
+; AVX512BW-NEXT:    vpmovusdb %zmm0, %xmm0
 ; AVX512BW-NEXT:    vmovdqu8 %zmm0, (%rdi) {%k1}
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -5098,213 +4755,211 @@ define void @truncstore_v8i32_v8i8(<8 x i32> %x, <8 x i8>* %p, <8 x i32> %mask)
 define void @truncstore_v4i32_v4i16(<4 x i32> %x, <4 x i16>* %p, <4 x i32> %mask) {
 ; SSE2-LABEL: truncstore_v4i32_v4i16:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm3
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [2147483648,2147483648,2147483648,2147483648]
-; SSE2-NEXT:    pxor %xmm0, %xmm4
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147549183,2147549183,2147549183,2147549183]
-; SSE2-NEXT:    pcmpgtd %xmm4, %xmm2
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm2
-; SSE2-NEXT:    por %xmm0, %xmm2
-; SSE2-NEXT:    movd %xmm3, %eax
-; SSE2-NEXT:    notl %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB13_2
-; SSE2-NEXT:  # %bb.1: # %cond.store
-; SSE2-NEXT:    movd %xmm2, %eax
-; SSE2-NEXT:    movw %ax, (%rdi)
-; SSE2-NEXT:  .LBB13_2: # %else
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [2147483648,2147483648,2147483648,2147483648]
 ; SSE2-NEXT:    pxor %xmm0, %xmm3
-; SSE2-NEXT:    pextrw $2, %xmm3, %eax
+; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [2147549183,2147549183,2147549183,2147549183]
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm4
+; SSE2-NEXT:    pand %xmm4, %xmm0
+; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm4
+; SSE2-NEXT:    por %xmm0, %xmm4
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm4[0,2,2,3,4,5,6,7]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,6,6,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSE2-NEXT:    pcmpeqd %xmm1, %xmm2
+; SSE2-NEXT:    movmskps %xmm2, %eax
+; SSE2-NEXT:    xorl $15, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB13_4
-; SSE2-NEXT:  # %bb.3: # %cond.store1
-; SSE2-NEXT:    pextrw $2, %xmm2, %eax
-; SSE2-NEXT:    movw %ax, 2(%rdi)
+; SSE2-NEXT:    jne .LBB13_1
+; SSE2-NEXT:  # %bb.2: # %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne .LBB13_3
 ; SSE2-NEXT:  .LBB13_4: # %else2
-; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm1
-; SSE2-NEXT:    pxor %xmm1, %xmm0
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB13_6
-; SSE2-NEXT:  # %bb.5: # %cond.store3
-; SSE2-NEXT:    pextrw $4, %xmm2, %eax
-; SSE2-NEXT:    movw %ax, 4(%rdi)
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne .LBB13_5
 ; SSE2-NEXT:  .LBB13_6: # %else4
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE2-NEXT:    pxor %xmm0, %xmm1
-; SSE2-NEXT:    pextrw $6, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne .LBB13_7
+; SSE2-NEXT:  .LBB13_8: # %else6
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  .LBB13_1: # %cond.store
+; SSE2-NEXT:    movd %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, (%rdi)
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    je .LBB13_4
+; SSE2-NEXT:  .LBB13_3: # %cond.store1
+; SSE2-NEXT:    pextrw $1, %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, 2(%rdi)
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    je .LBB13_6
+; SSE2-NEXT:  .LBB13_5: # %cond.store3
+; SSE2-NEXT:    pextrw $2, %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, 4(%rdi)
+; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je .LBB13_8
-; SSE2-NEXT:  # %bb.7: # %cond.store5
-; SSE2-NEXT:    pextrw $6, %xmm2, %eax
+; SSE2-NEXT:  .LBB13_7: # %cond.store5
+; SSE2-NEXT:    pextrw $3, %xmm0, %eax
 ; SSE2-NEXT:    movw %ax, 6(%rdi)
-; SSE2-NEXT:  .LBB13_8: # %else6
 ; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: truncstore_v4i32_v4i16:
 ; SSE4:       # %bb.0:
-; SSE4-NEXT:    pxor %xmm3, %xmm3
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm3
-; SSE4-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE4-NEXT:    pxor %xmm3, %xmm2
+; SSE4-NEXT:    pxor %xmm2, %xmm2
 ; SSE4-NEXT:    pminud {{.*}}(%rip), %xmm0
-; SSE4-NEXT:    pextrb $0, %xmm2, %eax
+; SSE4-NEXT:    packusdw %xmm0, %xmm0
+; SSE4-NEXT:    pcmpeqd %xmm1, %xmm2
+; SSE4-NEXT:    movmskps %xmm2, %eax
+; SSE4-NEXT:    xorl $15, %eax
 ; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB13_2
-; SSE4-NEXT:  # %bb.1: # %cond.store
+; SSE4-NEXT:    jne .LBB13_1
+; SSE4-NEXT:  # %bb.2: # %else
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    jne .LBB13_3
+; SSE4-NEXT:  .LBB13_4: # %else2
+; SSE4-NEXT:    testb $4, %al
+; SSE4-NEXT:    jne .LBB13_5
+; SSE4-NEXT:  .LBB13_6: # %else4
+; SSE4-NEXT:    testb $8, %al
+; SSE4-NEXT:    jne .LBB13_7
+; SSE4-NEXT:  .LBB13_8: # %else6
+; SSE4-NEXT:    retq
+; SSE4-NEXT:  .LBB13_1: # %cond.store
 ; SSE4-NEXT:    pextrw $0, %xmm0, (%rdi)
-; SSE4-NEXT:  .LBB13_2: # %else
-; SSE4-NEXT:    pextrb $4, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $2, %al
 ; SSE4-NEXT:    je .LBB13_4
-; SSE4-NEXT:  # %bb.3: # %cond.store1
-; SSE4-NEXT:    pextrw $2, %xmm0, 2(%rdi)
-; SSE4-NEXT:  .LBB13_4: # %else2
-; SSE4-NEXT:    pxor %xmm2, %xmm2
-; SSE4-NEXT:    pcmpeqd %xmm2, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE4-NEXT:    pxor %xmm1, %xmm2
-; SSE4-NEXT:    pextrb $8, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:  .LBB13_3: # %cond.store1
+; SSE4-NEXT:    pextrw $1, %xmm0, 2(%rdi)
+; SSE4-NEXT:    testb $4, %al
 ; SSE4-NEXT:    je .LBB13_6
-; SSE4-NEXT:  # %bb.5: # %cond.store3
-; SSE4-NEXT:    pextrw $4, %xmm0, 4(%rdi)
-; SSE4-NEXT:  .LBB13_6: # %else4
-; SSE4-NEXT:    pextrb $12, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:  .LBB13_5: # %cond.store3
+; SSE4-NEXT:    pextrw $2, %xmm0, 4(%rdi)
+; SSE4-NEXT:    testb $8, %al
 ; SSE4-NEXT:    je .LBB13_8
-; SSE4-NEXT:  # %bb.7: # %cond.store5
-; SSE4-NEXT:    pextrw $6, %xmm0, 6(%rdi)
-; SSE4-NEXT:  .LBB13_8: # %else6
+; SSE4-NEXT:  .LBB13_7: # %cond.store5
+; SSE4-NEXT:    pextrw $3, %xmm0, 6(%rdi)
 ; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: truncstore_v4i32_v4i16:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm0
-; AVX1-NEXT:    vpextrb $0, %xmm2, %eax
+; AVX1-NEXT:    vpackusdw %xmm0, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vmovmskps %xmm1, %eax
+; AVX1-NEXT:    xorl $15, %eax
 ; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je .LBB13_2
-; AVX1-NEXT:  # %bb.1: # %cond.store
+; AVX1-NEXT:    jne .LBB13_1
+; AVX1-NEXT:  # %bb.2: # %else
+; AVX1-NEXT:    testb $2, %al
+; AVX1-NEXT:    jne .LBB13_3
+; AVX1-NEXT:  .LBB13_4: # %else2
+; AVX1-NEXT:    testb $4, %al
+; AVX1-NEXT:    jne .LBB13_5
+; AVX1-NEXT:  .LBB13_6: # %else4
+; AVX1-NEXT:    testb $8, %al
+; AVX1-NEXT:    jne .LBB13_7
+; AVX1-NEXT:  .LBB13_8: # %else6
+; AVX1-NEXT:    retq
+; AVX1-NEXT:  .LBB13_1: # %cond.store
 ; AVX1-NEXT:    vpextrw $0, %xmm0, (%rdi)
-; AVX1-NEXT:  .LBB13_2: # %else
-; AVX1-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $2, %al
 ; AVX1-NEXT:    je .LBB13_4
-; AVX1-NEXT:  # %bb.3: # %cond.store1
-; AVX1-NEXT:    vpextrw $2, %xmm0, 2(%rdi)
-; AVX1-NEXT:  .LBB13_4: # %else2
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:  .LBB13_3: # %cond.store1
+; AVX1-NEXT:    vpextrw $1, %xmm0, 2(%rdi)
+; AVX1-NEXT:    testb $4, %al
 ; AVX1-NEXT:    je .LBB13_6
-; AVX1-NEXT:  # %bb.5: # %cond.store3
-; AVX1-NEXT:    vpextrw $4, %xmm0, 4(%rdi)
-; AVX1-NEXT:  .LBB13_6: # %else4
-; AVX1-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:  .LBB13_5: # %cond.store3
+; AVX1-NEXT:    vpextrw $2, %xmm0, 4(%rdi)
+; AVX1-NEXT:    testb $8, %al
 ; AVX1-NEXT:    je .LBB13_8
-; AVX1-NEXT:  # %bb.7: # %cond.store5
-; AVX1-NEXT:    vpextrw $6, %xmm0, 6(%rdi)
-; AVX1-NEXT:  .LBB13_8: # %else6
+; AVX1-NEXT:  .LBB13_7: # %cond.store5
+; AVX1-NEXT:    vpextrw $3, %xmm0, 6(%rdi)
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: truncstore_v4i32_v4i16:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    vpxor %xmm3, %xmm2, %xmm2
 ; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm3 = [65535,65535,65535,65535]
 ; AVX2-NEXT:    vpminud %xmm3, %xmm0, %xmm0
-; AVX2-NEXT:    vpextrb $0, %xmm2, %eax
+; AVX2-NEXT:    vpackusdw %xmm0, %xmm0, %xmm0
+; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vmovmskps %xmm1, %eax
+; AVX2-NEXT:    xorl $15, %eax
 ; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je .LBB13_2
-; AVX2-NEXT:  # %bb.1: # %cond.store
+; AVX2-NEXT:    jne .LBB13_1
+; AVX2-NEXT:  # %bb.2: # %else
+; AVX2-NEXT:    testb $2, %al
+; AVX2-NEXT:    jne .LBB13_3
+; AVX2-NEXT:  .LBB13_4: # %else2
+; AVX2-NEXT:    testb $4, %al
+; AVX2-NEXT:    jne .LBB13_5
+; AVX2-NEXT:  .LBB13_6: # %else4
+; AVX2-NEXT:    testb $8, %al
+; AVX2-NEXT:    jne .LBB13_7
+; AVX2-NEXT:  .LBB13_8: # %else6
+; AVX2-NEXT:    retq
+; AVX2-NEXT:  .LBB13_1: # %cond.store
 ; AVX2-NEXT:    vpextrw $0, %xmm0, (%rdi)
-; AVX2-NEXT:  .LBB13_2: # %else
-; AVX2-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $2, %al
 ; AVX2-NEXT:    je .LBB13_4
-; AVX2-NEXT:  # %bb.3: # %cond.store1
-; AVX2-NEXT:    vpextrw $2, %xmm0, 2(%rdi)
-; AVX2-NEXT:  .LBB13_4: # %else2
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:  .LBB13_3: # %cond.store1
+; AVX2-NEXT:    vpextrw $1, %xmm0, 2(%rdi)
+; AVX2-NEXT:    testb $4, %al
 ; AVX2-NEXT:    je .LBB13_6
-; AVX2-NEXT:  # %bb.5: # %cond.store3
-; AVX2-NEXT:    vpextrw $4, %xmm0, 4(%rdi)
-; AVX2-NEXT:  .LBB13_6: # %else4
-; AVX2-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:  .LBB13_5: # %cond.store3
+; AVX2-NEXT:    vpextrw $2, %xmm0, 4(%rdi)
+; AVX2-NEXT:    testb $8, %al
 ; AVX2-NEXT:    je .LBB13_8
-; AVX2-NEXT:  # %bb.7: # %cond.store5
-; AVX2-NEXT:    vpextrw $6, %xmm0, 6(%rdi)
-; AVX2-NEXT:  .LBB13_8: # %else6
+; AVX2-NEXT:  .LBB13_7: # %cond.store5
+; AVX2-NEXT:    vpextrw $3, %xmm0, 6(%rdi)
 ; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: truncstore_v4i32_v4i16:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [65535,65535,65535,65535]
-; AVX512F-NEXT:    vpminud %xmm2, %xmm0, %xmm0
+; AVX512F-NEXT:    vpmovusdw %zmm0, %ymm0
 ; AVX512F-NEXT:    kmovw %k0, %eax
 ; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB13_2
-; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    jne .LBB13_1
+; AVX512F-NEXT:  # %bb.2: # %else
+; AVX512F-NEXT:    testb $2, %al
+; AVX512F-NEXT:    jne .LBB13_3
+; AVX512F-NEXT:  .LBB13_4: # %else2
+; AVX512F-NEXT:    testb $4, %al
+; AVX512F-NEXT:    jne .LBB13_5
+; AVX512F-NEXT:  .LBB13_6: # %else4
+; AVX512F-NEXT:    testb $8, %al
+; AVX512F-NEXT:    jne .LBB13_7
+; AVX512F-NEXT:  .LBB13_8: # %else6
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+; AVX512F-NEXT:  .LBB13_1: # %cond.store
 ; AVX512F-NEXT:    vpextrw $0, %xmm0, (%rdi)
-; AVX512F-NEXT:  .LBB13_2: # %else
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $2, %al
 ; AVX512F-NEXT:    je .LBB13_4
-; AVX512F-NEXT:  # %bb.3: # %cond.store1
-; AVX512F-NEXT:    vpextrw $2, %xmm0, 2(%rdi)
-; AVX512F-NEXT:  .LBB13_4: # %else2
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  .LBB13_3: # %cond.store1
+; AVX512F-NEXT:    vpextrw $1, %xmm0, 2(%rdi)
+; AVX512F-NEXT:    testb $4, %al
 ; AVX512F-NEXT:    je .LBB13_6
-; AVX512F-NEXT:  # %bb.5: # %cond.store3
-; AVX512F-NEXT:    vpextrw $4, %xmm0, 4(%rdi)
-; AVX512F-NEXT:  .LBB13_6: # %else4
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  .LBB13_5: # %cond.store3
+; AVX512F-NEXT:    vpextrw $2, %xmm0, 4(%rdi)
+; AVX512F-NEXT:    testb $8, %al
 ; AVX512F-NEXT:    je .LBB13_8
-; AVX512F-NEXT:  # %bb.7: # %cond.store5
-; AVX512F-NEXT:    vpextrw $6, %xmm0, 6(%rdi)
-; AVX512F-NEXT:  .LBB13_8: # %else6
+; AVX512F-NEXT:  .LBB13_7: # %cond.store5
+; AVX512F-NEXT:    vpextrw $3, %xmm0, 6(%rdi)
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: truncstore_v4i32_v4i16:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; AVX512BW-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512BW-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [65535,65535,65535,65535]
-; AVX512BW-NEXT:    vpminud %xmm1, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpackusdw %xmm0, %xmm0, %xmm0
 ; AVX512BW-NEXT:    kshiftld $28, %k0, %k0
 ; AVX512BW-NEXT:    kshiftrd $28, %k0, %k1
+; AVX512BW-NEXT:    vpmovusdw %zmm0, %ymm0
 ; AVX512BW-NEXT:    vmovdqu16 %zmm0, (%rdi) {%k1}
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -5326,213 +4981,211 @@ define void @truncstore_v4i32_v4i16(<4 x i32> %x, <4 x i16>* %p, <4 x i32> %mask
 define void @truncstore_v4i32_v4i8(<4 x i32> %x, <4 x i8>* %p, <4 x i32> %mask) {
 ; SSE2-LABEL: truncstore_v4i32_v4i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm3
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [2147483648,2147483648,2147483648,2147483648]
-; SSE2-NEXT:    pxor %xmm0, %xmm4
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483903,2147483903,2147483903,2147483903]
-; SSE2-NEXT:    pcmpgtd %xmm4, %xmm2
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm2
-; SSE2-NEXT:    por %xmm0, %xmm2
-; SSE2-NEXT:    movd %xmm3, %eax
-; SSE2-NEXT:    notl %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB14_2
-; SSE2-NEXT:  # %bb.1: # %cond.store
-; SSE2-NEXT:    movd %xmm2, %eax
-; SSE2-NEXT:    movb %al, (%rdi)
-; SSE2-NEXT:  .LBB14_2: # %else
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [2147483648,2147483648,2147483648,2147483648]
 ; SSE2-NEXT:    pxor %xmm0, %xmm3
-; SSE2-NEXT:    pextrw $2, %xmm3, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB14_4
-; SSE2-NEXT:  # %bb.3: # %cond.store1
-; SSE2-NEXT:    pextrw $2, %xmm2, %eax
-; SSE2-NEXT:    movb %al, 1(%rdi)
+; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [2147483903,2147483903,2147483903,2147483903]
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm4
+; SSE2-NEXT:    pand %xmm4, %xmm0
+; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm4
+; SSE2-NEXT:    por %xmm0, %xmm4
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm4
+; SSE2-NEXT:    packuswb %xmm4, %xmm4
+; SSE2-NEXT:    packuswb %xmm4, %xmm4
+; SSE2-NEXT:    pcmpeqd %xmm1, %xmm2
+; SSE2-NEXT:    movmskps %xmm2, %ecx
+; SSE2-NEXT:    xorl $15, %ecx
+; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    movd %xmm4, %eax
+; SSE2-NEXT:    jne .LBB14_1
+; SSE2-NEXT:  # %bb.2: # %else
+; SSE2-NEXT:    testb $2, %cl
+; SSE2-NEXT:    jne .LBB14_3
 ; SSE2-NEXT:  .LBB14_4: # %else2
-; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm1
-; SSE2-NEXT:    pxor %xmm1, %xmm0
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB14_6
-; SSE2-NEXT:  # %bb.5: # %cond.store3
-; SSE2-NEXT:    pextrw $4, %xmm2, %eax
-; SSE2-NEXT:    movb %al, 2(%rdi)
+; SSE2-NEXT:    testb $4, %cl
+; SSE2-NEXT:    jne .LBB14_5
 ; SSE2-NEXT:  .LBB14_6: # %else4
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE2-NEXT:    pxor %xmm0, %xmm1
-; SSE2-NEXT:    pextrw $6, %xmm1, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $8, %cl
+; SSE2-NEXT:    jne .LBB14_7
+; SSE2-NEXT:  .LBB14_8: # %else6
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  .LBB14_1: # %cond.store
+; SSE2-NEXT:    movb %al, (%rdi)
+; SSE2-NEXT:    testb $2, %cl
+; SSE2-NEXT:    je .LBB14_4
+; SSE2-NEXT:  .LBB14_3: # %cond.store1
+; SSE2-NEXT:    movb %ah, 1(%rdi)
+; SSE2-NEXT:    testb $4, %cl
+; SSE2-NEXT:    je .LBB14_6
+; SSE2-NEXT:  .LBB14_5: # %cond.store3
+; SSE2-NEXT:    movl %eax, %edx
+; SSE2-NEXT:    shrl $16, %edx
+; SSE2-NEXT:    movb %dl, 2(%rdi)
+; SSE2-NEXT:    testb $8, %cl
 ; SSE2-NEXT:    je .LBB14_8
-; SSE2-NEXT:  # %bb.7: # %cond.store5
-; SSE2-NEXT:    pextrw $6, %xmm2, %eax
+; SSE2-NEXT:  .LBB14_7: # %cond.store5
+; SSE2-NEXT:    shrl $24, %eax
 ; SSE2-NEXT:    movb %al, 3(%rdi)
-; SSE2-NEXT:  .LBB14_8: # %else6
 ; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: truncstore_v4i32_v4i8:
 ; SSE4:       # %bb.0:
-; SSE4-NEXT:    pxor %xmm3, %xmm3
-; SSE4-NEXT:    pcmpeqd %xmm1, %xmm3
-; SSE4-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE4-NEXT:    pxor %xmm3, %xmm2
+; SSE4-NEXT:    pxor %xmm2, %xmm2
 ; SSE4-NEXT:    pminud {{.*}}(%rip), %xmm0
-; SSE4-NEXT:    pextrb $0, %xmm2, %eax
+; SSE4-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; SSE4-NEXT:    pcmpeqd %xmm1, %xmm2
+; SSE4-NEXT:    movmskps %xmm2, %eax
+; SSE4-NEXT:    xorl $15, %eax
 ; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB14_2
-; SSE4-NEXT:  # %bb.1: # %cond.store
+; SSE4-NEXT:    jne .LBB14_1
+; SSE4-NEXT:  # %bb.2: # %else
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    jne .LBB14_3
+; SSE4-NEXT:  .LBB14_4: # %else2
+; SSE4-NEXT:    testb $4, %al
+; SSE4-NEXT:    jne .LBB14_5
+; SSE4-NEXT:  .LBB14_6: # %else4
+; SSE4-NEXT:    testb $8, %al
+; SSE4-NEXT:    jne .LBB14_7
+; SSE4-NEXT:  .LBB14_8: # %else6
+; SSE4-NEXT:    retq
+; SSE4-NEXT:  .LBB14_1: # %cond.store
 ; SSE4-NEXT:    pextrb $0, %xmm0, (%rdi)
-; SSE4-NEXT:  .LBB14_2: # %else
-; SSE4-NEXT:    pextrb $4, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $2, %al
 ; SSE4-NEXT:    je .LBB14_4
-; SSE4-NEXT:  # %bb.3: # %cond.store1
-; SSE4-NEXT:    pextrb $4, %xmm0, 1(%rdi)
-; SSE4-NEXT:  .LBB14_4: # %else2
-; SSE4-NEXT:    pxor %xmm2, %xmm2
-; SSE4-NEXT:    pcmpeqd %xmm2, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE4-NEXT:    pxor %xmm1, %xmm2
-; SSE4-NEXT:    pextrb $8, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:  .LBB14_3: # %cond.store1
+; SSE4-NEXT:    pextrb $1, %xmm0, 1(%rdi)
+; SSE4-NEXT:    testb $4, %al
 ; SSE4-NEXT:    je .LBB14_6
-; SSE4-NEXT:  # %bb.5: # %cond.store3
-; SSE4-NEXT:    pextrb $8, %xmm0, 2(%rdi)
-; SSE4-NEXT:  .LBB14_6: # %else4
-; SSE4-NEXT:    pextrb $12, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:  .LBB14_5: # %cond.store3
+; SSE4-NEXT:    pextrb $2, %xmm0, 2(%rdi)
+; SSE4-NEXT:    testb $8, %al
 ; SSE4-NEXT:    je .LBB14_8
-; SSE4-NEXT:  # %bb.7: # %cond.store5
-; SSE4-NEXT:    pextrb $12, %xmm0, 3(%rdi)
-; SSE4-NEXT:  .LBB14_8: # %else6
+; SSE4-NEXT:  .LBB14_7: # %cond.store5
+; SSE4-NEXT:    pextrb $3, %xmm0, 3(%rdi)
 ; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: truncstore_v4i32_v4i8:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm0
-; AVX1-NEXT:    vpextrb $0, %xmm2, %eax
+; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vmovmskps %xmm1, %eax
+; AVX1-NEXT:    xorl $15, %eax
 ; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je .LBB14_2
-; AVX1-NEXT:  # %bb.1: # %cond.store
+; AVX1-NEXT:    jne .LBB14_1
+; AVX1-NEXT:  # %bb.2: # %else
+; AVX1-NEXT:    testb $2, %al
+; AVX1-NEXT:    jne .LBB14_3
+; AVX1-NEXT:  .LBB14_4: # %else2
+; AVX1-NEXT:    testb $4, %al
+; AVX1-NEXT:    jne .LBB14_5
+; AVX1-NEXT:  .LBB14_6: # %else4
+; AVX1-NEXT:    testb $8, %al
+; AVX1-NEXT:    jne .LBB14_7
+; AVX1-NEXT:  .LBB14_8: # %else6
+; AVX1-NEXT:    retq
+; AVX1-NEXT:  .LBB14_1: # %cond.store
 ; AVX1-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX1-NEXT:  .LBB14_2: # %else
-; AVX1-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $2, %al
 ; AVX1-NEXT:    je .LBB14_4
-; AVX1-NEXT:  # %bb.3: # %cond.store1
-; AVX1-NEXT:    vpextrb $4, %xmm0, 1(%rdi)
-; AVX1-NEXT:  .LBB14_4: # %else2
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:  .LBB14_3: # %cond.store1
+; AVX1-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
+; AVX1-NEXT:    testb $4, %al
 ; AVX1-NEXT:    je .LBB14_6
-; AVX1-NEXT:  # %bb.5: # %cond.store3
-; AVX1-NEXT:    vpextrb $8, %xmm0, 2(%rdi)
-; AVX1-NEXT:  .LBB14_6: # %else4
-; AVX1-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:  .LBB14_5: # %cond.store3
+; AVX1-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
+; AVX1-NEXT:    testb $8, %al
 ; AVX1-NEXT:    je .LBB14_8
-; AVX1-NEXT:  # %bb.7: # %cond.store5
-; AVX1-NEXT:    vpextrb $12, %xmm0, 3(%rdi)
-; AVX1-NEXT:  .LBB14_8: # %else6
+; AVX1-NEXT:  .LBB14_7: # %cond.store5
+; AVX1-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: truncstore_v4i32_v4i8:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    vpxor %xmm3, %xmm2, %xmm2
 ; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm3 = [255,255,255,255]
 ; AVX2-NEXT:    vpminud %xmm3, %xmm0, %xmm0
-; AVX2-NEXT:    vpextrb $0, %xmm2, %eax
+; AVX2-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vmovmskps %xmm1, %eax
+; AVX2-NEXT:    xorl $15, %eax
 ; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je .LBB14_2
-; AVX2-NEXT:  # %bb.1: # %cond.store
+; AVX2-NEXT:    jne .LBB14_1
+; AVX2-NEXT:  # %bb.2: # %else
+; AVX2-NEXT:    testb $2, %al
+; AVX2-NEXT:    jne .LBB14_3
+; AVX2-NEXT:  .LBB14_4: # %else2
+; AVX2-NEXT:    testb $4, %al
+; AVX2-NEXT:    jne .LBB14_5
+; AVX2-NEXT:  .LBB14_6: # %else4
+; AVX2-NEXT:    testb $8, %al
+; AVX2-NEXT:    jne .LBB14_7
+; AVX2-NEXT:  .LBB14_8: # %else6
+; AVX2-NEXT:    retq
+; AVX2-NEXT:  .LBB14_1: # %cond.store
 ; AVX2-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX2-NEXT:  .LBB14_2: # %else
-; AVX2-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $2, %al
 ; AVX2-NEXT:    je .LBB14_4
-; AVX2-NEXT:  # %bb.3: # %cond.store1
-; AVX2-NEXT:    vpextrb $4, %xmm0, 1(%rdi)
-; AVX2-NEXT:  .LBB14_4: # %else2
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpextrb $8, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:  .LBB14_3: # %cond.store1
+; AVX2-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
+; AVX2-NEXT:    testb $4, %al
 ; AVX2-NEXT:    je .LBB14_6
-; AVX2-NEXT:  # %bb.5: # %cond.store3
-; AVX2-NEXT:    vpextrb $8, %xmm0, 2(%rdi)
-; AVX2-NEXT:  .LBB14_6: # %else4
-; AVX2-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:  .LBB14_5: # %cond.store3
+; AVX2-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
+; AVX2-NEXT:    testb $8, %al
 ; AVX2-NEXT:    je .LBB14_8
-; AVX2-NEXT:  # %bb.7: # %cond.store5
-; AVX2-NEXT:    vpextrb $12, %xmm0, 3(%rdi)
-; AVX2-NEXT:  .LBB14_8: # %else6
+; AVX2-NEXT:  .LBB14_7: # %cond.store5
+; AVX2-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
 ; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: truncstore_v4i32_v4i8:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [255,255,255,255]
-; AVX512F-NEXT:    vpminud %xmm2, %xmm0, %xmm0
+; AVX512F-NEXT:    vpmovusdb %zmm0, %xmm0
 ; AVX512F-NEXT:    kmovw %k0, %eax
 ; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB14_2
-; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    jne .LBB14_1
+; AVX512F-NEXT:  # %bb.2: # %else
+; AVX512F-NEXT:    testb $2, %al
+; AVX512F-NEXT:    jne .LBB14_3
+; AVX512F-NEXT:  .LBB14_4: # %else2
+; AVX512F-NEXT:    testb $4, %al
+; AVX512F-NEXT:    jne .LBB14_5
+; AVX512F-NEXT:  .LBB14_6: # %else4
+; AVX512F-NEXT:    testb $8, %al
+; AVX512F-NEXT:    jne .LBB14_7
+; AVX512F-NEXT:  .LBB14_8: # %else6
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+; AVX512F-NEXT:  .LBB14_1: # %cond.store
 ; AVX512F-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX512F-NEXT:  .LBB14_2: # %else
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $2, %al
 ; AVX512F-NEXT:    je .LBB14_4
-; AVX512F-NEXT:  # %bb.3: # %cond.store1
-; AVX512F-NEXT:    vpextrb $4, %xmm0, 1(%rdi)
-; AVX512F-NEXT:  .LBB14_4: # %else2
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  .LBB14_3: # %cond.store1
+; AVX512F-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
+; AVX512F-NEXT:    testb $4, %al
 ; AVX512F-NEXT:    je .LBB14_6
-; AVX512F-NEXT:  # %bb.5: # %cond.store3
-; AVX512F-NEXT:    vpextrb $8, %xmm0, 2(%rdi)
-; AVX512F-NEXT:  .LBB14_6: # %else4
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  .LBB14_5: # %cond.store3
+; AVX512F-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
+; AVX512F-NEXT:    testb $8, %al
 ; AVX512F-NEXT:    je .LBB14_8
-; AVX512F-NEXT:  # %bb.7: # %cond.store5
-; AVX512F-NEXT:    vpextrb $12, %xmm0, 3(%rdi)
-; AVX512F-NEXT:  .LBB14_8: # %else6
+; AVX512F-NEXT:  .LBB14_7: # %cond.store5
+; AVX512F-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: truncstore_v4i32_v4i8:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; AVX512BW-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512BW-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [255,255,255,255]
-; AVX512BW-NEXT:    vpminud %xmm1, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
 ; AVX512BW-NEXT:    kshiftlq $60, %k0, %k0
 ; AVX512BW-NEXT:    kshiftrq $60, %k0, %k1
+; AVX512BW-NEXT:    vpmovusdb %zmm0, %xmm0
 ; AVX512BW-NEXT:    vmovdqu8 %zmm0, (%rdi) {%k1}
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -5555,7 +5208,6 @@ define void @truncstore_v32i16_v32i8(<32 x i16> %x, <32 x i8>* %p, <32 x i8> %ma
 ; SSE2-LABEL: truncstore_v32i16_v32i8:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pxor %xmm7, %xmm7
-; SSE2-NEXT:    pcmpeqb %xmm4, %xmm7
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm6 = [32768,32768,32768,32768,32768,32768,32768,32768]
 ; SSE2-NEXT:    pxor %xmm6, %xmm1
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [33023,33023,33023,33023,33023,33023,33023,33023]
@@ -5565,123 +5217,77 @@ define void @truncstore_v32i16_v32i8(<32 x i16> %x, <32 x i8>* %p, <32 x i8> %ma
 ; SSE2-NEXT:    pminsw %xmm8, %xmm0
 ; SSE2-NEXT:    pxor %xmm6, %xmm0
 ; SSE2-NEXT:    packuswb %xmm1, %xmm0
-; SSE2-NEXT:    movd %xmm7, %ecx
-; SSE2-NEXT:    movl %ecx, %eax
-; SSE2-NEXT:    notb %al
+; SSE2-NEXT:    pcmpeqb %xmm7, %xmm4
+; SSE2-NEXT:    pmovmskb %xmm4, %ecx
+; SSE2-NEXT:    xorl $65535, %ecx # imm = 0xFFFF
+; SSE2-NEXT:    pcmpeqb %xmm7, %xmm5
+; SSE2-NEXT:    pmovmskb %xmm5, %eax
+; SSE2-NEXT:    notl %eax
+; SSE2-NEXT:    shll $16, %eax
+; SSE2-NEXT:    orl %ecx, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    je .LBB15_2
-; SSE2-NEXT:  # %bb.1: # %cond.store
-; SSE2-NEXT:    movb %al, (%rdi)
-; SSE2-NEXT:  .LBB15_2: # %else
-; SSE2-NEXT:    shrl $8, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    je .LBB15_4
-; SSE2-NEXT:  # %bb.3: # %cond.store1
-; SSE2-NEXT:    movb %ah, 1(%rdi)
+; SSE2-NEXT:    movd %xmm0, %ecx
+; SSE2-NEXT:    jne .LBB15_1
+; SSE2-NEXT:  # %bb.2: # %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne .LBB15_3
 ; SSE2-NEXT:  .LBB15_4: # %else2
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqb %xmm4, %xmm1
-; SSE2-NEXT:    movd %xmm1, %ecx
-; SSE2-NEXT:    movl %ecx, %edx
-; SSE2-NEXT:    shrl $16, %edx
-; SSE2-NEXT:    notb %dl
-; SSE2-NEXT:    testb $1, %dl
-; SSE2-NEXT:    je .LBB15_6
-; SSE2-NEXT:  # %bb.5: # %cond.store3
-; SSE2-NEXT:    movl %eax, %edx
-; SSE2-NEXT:    shrl $16, %edx
-; SSE2-NEXT:    movb %dl, 2(%rdi)
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne .LBB15_5
 ; SSE2-NEXT:  .LBB15_6: # %else4
-; SSE2-NEXT:    shrl $24, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je .LBB15_8
-; SSE2-NEXT:  # %bb.7: # %cond.store5
-; SSE2-NEXT:    shrl $24, %eax
-; SSE2-NEXT:    movb %al, 3(%rdi)
+; SSE2-NEXT:  .LBB15_7: # %cond.store5
+; SSE2-NEXT:    shrl $24, %ecx
+; SSE2-NEXT:    movb %cl, 3(%rdi)
 ; SSE2-NEXT:  .LBB15_8: # %else6
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqb %xmm4, %xmm1
-; SSE2-NEXT:    pextrw $2, %xmm1, %eax
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testb $16, %al
 ; SSE2-NEXT:    pextrw $2, %xmm0, %ecx
 ; SSE2-NEXT:    je .LBB15_10
 ; SSE2-NEXT:  # %bb.9: # %cond.store7
 ; SSE2-NEXT:    movb %cl, 4(%rdi)
 ; SSE2-NEXT:  .LBB15_10: # %else8
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    notb %al
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $32, %al
 ; SSE2-NEXT:    je .LBB15_12
 ; SSE2-NEXT:  # %bb.11: # %cond.store9
 ; SSE2-NEXT:    movb %ch, 5(%rdi)
 ; SSE2-NEXT:  .LBB15_12: # %else10
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqb %xmm4, %xmm1
-; SSE2-NEXT:    pextrw $3, %xmm1, %eax
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testb $64, %al
 ; SSE2-NEXT:    pextrw $3, %xmm0, %ecx
 ; SSE2-NEXT:    je .LBB15_14
 ; SSE2-NEXT:  # %bb.13: # %cond.store11
 ; SSE2-NEXT:    movb %cl, 6(%rdi)
 ; SSE2-NEXT:  .LBB15_14: # %else12
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    notb %al
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $-128, %al
 ; SSE2-NEXT:    je .LBB15_16
 ; SSE2-NEXT:  # %bb.15: # %cond.store13
 ; SSE2-NEXT:    movb %ch, 7(%rdi)
 ; SSE2-NEXT:  .LBB15_16: # %else14
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqb %xmm4, %xmm1
-; SSE2-NEXT:    pextrw $4, %xmm1, %eax
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $256, %eax # imm = 0x100
 ; SSE2-NEXT:    pextrw $4, %xmm0, %ecx
 ; SSE2-NEXT:    je .LBB15_18
 ; SSE2-NEXT:  # %bb.17: # %cond.store15
 ; SSE2-NEXT:    movb %cl, 8(%rdi)
 ; SSE2-NEXT:  .LBB15_18: # %else16
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    notb %al
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $512, %eax # imm = 0x200
 ; SSE2-NEXT:    je .LBB15_20
 ; SSE2-NEXT:  # %bb.19: # %cond.store17
 ; SSE2-NEXT:    movb %ch, 9(%rdi)
 ; SSE2-NEXT:  .LBB15_20: # %else18
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqb %xmm4, %xmm1
-; SSE2-NEXT:    pextrw $5, %xmm1, %eax
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $1024, %eax # imm = 0x400
 ; SSE2-NEXT:    pextrw $5, %xmm0, %ecx
 ; SSE2-NEXT:    je .LBB15_22
 ; SSE2-NEXT:  # %bb.21: # %cond.store19
 ; SSE2-NEXT:    movb %cl, 10(%rdi)
 ; SSE2-NEXT:  .LBB15_22: # %else20
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    notb %al
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $2048, %eax # imm = 0x800
 ; SSE2-NEXT:    je .LBB15_24
 ; SSE2-NEXT:  # %bb.23: # %cond.store21
 ; SSE2-NEXT:    movb %ch, 11(%rdi)
 ; SSE2-NEXT:  .LBB15_24: # %else22
 ; SSE2-NEXT:    pxor %xmm6, %xmm3
 ; SSE2-NEXT:    pxor %xmm6, %xmm2
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqb %xmm4, %xmm1
-; SSE2-NEXT:    pextrw $6, %xmm1, %eax
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $4096, %eax # imm = 0x1000
 ; SSE2-NEXT:    pextrw $6, %xmm0, %ecx
 ; SSE2-NEXT:    je .LBB15_26
 ; SSE2-NEXT:  # %bb.25: # %cond.store23
@@ -5689,1393 +5295,1108 @@ define void @truncstore_v32i16_v32i8(<32 x i16> %x, <32 x i8>* %p, <32 x i8> %ma
 ; SSE2-NEXT:  .LBB15_26: # %else24
 ; SSE2-NEXT:    pminsw %xmm8, %xmm3
 ; SSE2-NEXT:    pminsw %xmm8, %xmm2
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    notb %al
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $8192, %eax # imm = 0x2000
 ; SSE2-NEXT:    je .LBB15_28
 ; SSE2-NEXT:  # %bb.27: # %cond.store25
 ; SSE2-NEXT:    movb %ch, 13(%rdi)
 ; SSE2-NEXT:  .LBB15_28: # %else26
 ; SSE2-NEXT:    pxor %xmm6, %xmm3
 ; SSE2-NEXT:    pxor %xmm6, %xmm2
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqb %xmm1, %xmm4
-; SSE2-NEXT:    pextrw $7, %xmm4, %eax
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $16384, %eax # imm = 0x4000
 ; SSE2-NEXT:    pextrw $7, %xmm0, %ecx
 ; SSE2-NEXT:    je .LBB15_30
 ; SSE2-NEXT:  # %bb.29: # %cond.store27
 ; SSE2-NEXT:    movb %cl, 14(%rdi)
 ; SSE2-NEXT:  .LBB15_30: # %else28
 ; SSE2-NEXT:    packuswb %xmm3, %xmm2
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    notb %al
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $32768, %eax # imm = 0x8000
 ; SSE2-NEXT:    je .LBB15_32
 ; SSE2-NEXT:  # %bb.31: # %cond.store29
 ; SSE2-NEXT:    movb %ch, 15(%rdi)
 ; SSE2-NEXT:  .LBB15_32: # %else30
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqb %xmm5, %xmm0
-; SSE2-NEXT:    movd %xmm0, %ecx
-; SSE2-NEXT:    movl %ecx, %eax
-; SSE2-NEXT:    notb %al
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    movd %xmm2, %eax
-; SSE2-NEXT:    je .LBB15_34
-; SSE2-NEXT:  # %bb.33: # %cond.store31
-; SSE2-NEXT:    movb %al, 16(%rdi)
-; SSE2-NEXT:  .LBB15_34: # %else32
-; SSE2-NEXT:    shrl $8, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    je .LBB15_36
-; SSE2-NEXT:  # %bb.35: # %cond.store33
-; SSE2-NEXT:    movb %ah, 17(%rdi)
+; SSE2-NEXT:    testl $65536, %eax # imm = 0x10000
+; SSE2-NEXT:    movd %xmm2, %ecx
+; SSE2-NEXT:    jne .LBB15_33
+; SSE2-NEXT:  # %bb.34: # %else32
+; SSE2-NEXT:    testl $131072, %eax # imm = 0x20000
+; SSE2-NEXT:    jne .LBB15_35
 ; SSE2-NEXT:  .LBB15_36: # %else34
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqb %xmm5, %xmm0
-; SSE2-NEXT:    movd %xmm0, %ecx
-; SSE2-NEXT:    movl %ecx, %edx
-; SSE2-NEXT:    shrl $16, %edx
-; SSE2-NEXT:    notb %dl
-; SSE2-NEXT:    testb $1, %dl
-; SSE2-NEXT:    je .LBB15_38
-; SSE2-NEXT:  # %bb.37: # %cond.store35
-; SSE2-NEXT:    movl %eax, %edx
-; SSE2-NEXT:    shrl $16, %edx
-; SSE2-NEXT:    movb %dl, 18(%rdi)
+; SSE2-NEXT:    testl $262144, %eax # imm = 0x40000
+; SSE2-NEXT:    jne .LBB15_37
 ; SSE2-NEXT:  .LBB15_38: # %else36
-; SSE2-NEXT:    shrl $24, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $524288, %eax # imm = 0x80000
 ; SSE2-NEXT:    je .LBB15_40
-; SSE2-NEXT:  # %bb.39: # %cond.store37
-; SSE2-NEXT:    shrl $24, %eax
-; SSE2-NEXT:    movb %al, 19(%rdi)
+; SSE2-NEXT:  .LBB15_39: # %cond.store37
+; SSE2-NEXT:    shrl $24, %ecx
+; SSE2-NEXT:    movb %cl, 19(%rdi)
 ; SSE2-NEXT:  .LBB15_40: # %else38
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqb %xmm5, %xmm0
-; SSE2-NEXT:    pextrw $2, %xmm0, %eax
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $1048576, %eax # imm = 0x100000
 ; SSE2-NEXT:    pextrw $2, %xmm2, %ecx
 ; SSE2-NEXT:    je .LBB15_42
 ; SSE2-NEXT:  # %bb.41: # %cond.store39
 ; SSE2-NEXT:    movb %cl, 20(%rdi)
 ; SSE2-NEXT:  .LBB15_42: # %else40
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    notb %al
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $2097152, %eax # imm = 0x200000
 ; SSE2-NEXT:    je .LBB15_44
 ; SSE2-NEXT:  # %bb.43: # %cond.store41
 ; SSE2-NEXT:    movb %ch, 21(%rdi)
 ; SSE2-NEXT:  .LBB15_44: # %else42
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqb %xmm5, %xmm0
-; SSE2-NEXT:    pextrw $3, %xmm0, %eax
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $4194304, %eax # imm = 0x400000
 ; SSE2-NEXT:    pextrw $3, %xmm2, %ecx
 ; SSE2-NEXT:    je .LBB15_46
 ; SSE2-NEXT:  # %bb.45: # %cond.store43
 ; SSE2-NEXT:    movb %cl, 22(%rdi)
 ; SSE2-NEXT:  .LBB15_46: # %else44
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    notb %al
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $8388608, %eax # imm = 0x800000
 ; SSE2-NEXT:    je .LBB15_48
 ; SSE2-NEXT:  # %bb.47: # %cond.store45
 ; SSE2-NEXT:    movb %ch, 23(%rdi)
 ; SSE2-NEXT:  .LBB15_48: # %else46
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqb %xmm5, %xmm0
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $16777216, %eax # imm = 0x1000000
 ; SSE2-NEXT:    pextrw $4, %xmm2, %ecx
 ; SSE2-NEXT:    je .LBB15_50
 ; SSE2-NEXT:  # %bb.49: # %cond.store47
 ; SSE2-NEXT:    movb %cl, 24(%rdi)
 ; SSE2-NEXT:  .LBB15_50: # %else48
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    notb %al
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $33554432, %eax # imm = 0x2000000
 ; SSE2-NEXT:    je .LBB15_52
 ; SSE2-NEXT:  # %bb.51: # %cond.store49
 ; SSE2-NEXT:    movb %ch, 25(%rdi)
 ; SSE2-NEXT:  .LBB15_52: # %else50
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqb %xmm5, %xmm0
-; SSE2-NEXT:    pextrw $5, %xmm0, %eax
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $67108864, %eax # imm = 0x4000000
 ; SSE2-NEXT:    pextrw $5, %xmm2, %ecx
 ; SSE2-NEXT:    je .LBB15_54
 ; SSE2-NEXT:  # %bb.53: # %cond.store51
 ; SSE2-NEXT:    movb %cl, 26(%rdi)
 ; SSE2-NEXT:  .LBB15_54: # %else52
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    notb %al
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $134217728, %eax # imm = 0x8000000
 ; SSE2-NEXT:    je .LBB15_56
 ; SSE2-NEXT:  # %bb.55: # %cond.store53
 ; SSE2-NEXT:    movb %ch, 27(%rdi)
 ; SSE2-NEXT:  .LBB15_56: # %else54
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqb %xmm5, %xmm0
-; SSE2-NEXT:    pextrw $6, %xmm0, %eax
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $268435456, %eax # imm = 0x10000000
 ; SSE2-NEXT:    pextrw $6, %xmm2, %ecx
 ; SSE2-NEXT:    je .LBB15_58
 ; SSE2-NEXT:  # %bb.57: # %cond.store55
 ; SSE2-NEXT:    movb %cl, 28(%rdi)
 ; SSE2-NEXT:  .LBB15_58: # %else56
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    notb %al
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $536870912, %eax # imm = 0x20000000
 ; SSE2-NEXT:    je .LBB15_60
 ; SSE2-NEXT:  # %bb.59: # %cond.store57
 ; SSE2-NEXT:    movb %ch, 29(%rdi)
 ; SSE2-NEXT:  .LBB15_60: # %else58
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpeqb %xmm0, %xmm5
-; SSE2-NEXT:    pextrw $7, %xmm5, %eax
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $1073741824, %eax # imm = 0x40000000
 ; SSE2-NEXT:    pextrw $7, %xmm2, %ecx
-; SSE2-NEXT:    je .LBB15_62
-; SSE2-NEXT:  # %bb.61: # %cond.store59
+; SSE2-NEXT:    jne .LBB15_61
+; SSE2-NEXT:  # %bb.62: # %else60
+; SSE2-NEXT:    testl $-2147483648, %eax # imm = 0x80000000
+; SSE2-NEXT:    jne .LBB15_63
+; SSE2-NEXT:  .LBB15_64: # %else62
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  .LBB15_1: # %cond.store
+; SSE2-NEXT:    movb %cl, (%rdi)
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    je .LBB15_4
+; SSE2-NEXT:  .LBB15_3: # %cond.store1
+; SSE2-NEXT:    movb %ch, 1(%rdi)
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    je .LBB15_6
+; SSE2-NEXT:  .LBB15_5: # %cond.store3
+; SSE2-NEXT:    movl %ecx, %edx
+; SSE2-NEXT:    shrl $16, %edx
+; SSE2-NEXT:    movb %dl, 2(%rdi)
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne .LBB15_7
+; SSE2-NEXT:    jmp .LBB15_8
+; SSE2-NEXT:  .LBB15_33: # %cond.store31
+; SSE2-NEXT:    movb %cl, 16(%rdi)
+; SSE2-NEXT:    testl $131072, %eax # imm = 0x20000
+; SSE2-NEXT:    je .LBB15_36
+; SSE2-NEXT:  .LBB15_35: # %cond.store33
+; SSE2-NEXT:    movb %ch, 17(%rdi)
+; SSE2-NEXT:    testl $262144, %eax # imm = 0x40000
+; SSE2-NEXT:    je .LBB15_38
+; SSE2-NEXT:  .LBB15_37: # %cond.store35
+; SSE2-NEXT:    movl %ecx, %edx
+; SSE2-NEXT:    shrl $16, %edx
+; SSE2-NEXT:    movb %dl, 18(%rdi)
+; SSE2-NEXT:    testl $524288, %eax # imm = 0x80000
+; SSE2-NEXT:    jne .LBB15_39
+; SSE2-NEXT:    jmp .LBB15_40
+; SSE2-NEXT:  .LBB15_61: # %cond.store59
 ; SSE2-NEXT:    movb %cl, 30(%rdi)
-; SSE2-NEXT:  .LBB15_62: # %else60
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    notb %al
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $-2147483648, %eax # imm = 0x80000000
 ; SSE2-NEXT:    je .LBB15_64
-; SSE2-NEXT:  # %bb.63: # %cond.store61
+; SSE2-NEXT:  .LBB15_63: # %cond.store61
 ; SSE2-NEXT:    movb %ch, 31(%rdi)
-; SSE2-NEXT:  .LBB15_64: # %else62
 ; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: truncstore_v32i16_v32i8:
 ; SSE4:       # %bb.0:
 ; SSE4-NEXT:    pxor %xmm7, %xmm7
-; SSE4-NEXT:    pcmpeqb %xmm4, %xmm7
 ; SSE4-NEXT:    movdqa {{.*#+}} xmm6 = [255,255,255,255,255,255,255,255]
 ; SSE4-NEXT:    pminuw %xmm6, %xmm1
 ; SSE4-NEXT:    pminuw %xmm6, %xmm0
 ; SSE4-NEXT:    packuswb %xmm1, %xmm0
-; SSE4-NEXT:    pextrb $0, %xmm7, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB15_2
-; SSE4-NEXT:  # %bb.1: # %cond.store
+; SSE4-NEXT:    pcmpeqb %xmm7, %xmm4
+; SSE4-NEXT:    pmovmskb %xmm4, %ecx
+; SSE4-NEXT:    xorl $65535, %ecx # imm = 0xFFFF
+; SSE4-NEXT:    pcmpeqb %xmm7, %xmm5
+; SSE4-NEXT:    pmovmskb %xmm5, %eax
+; SSE4-NEXT:    notl %eax
+; SSE4-NEXT:    shll $16, %eax
+; SSE4-NEXT:    orl %ecx, %eax
+; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    jne .LBB15_1
+; SSE4-NEXT:  # %bb.2: # %else
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    jne .LBB15_3
+; SSE4-NEXT:  .LBB15_4: # %else2
+; SSE4-NEXT:    testb $4, %al
+; SSE4-NEXT:    jne .LBB15_5
+; SSE4-NEXT:  .LBB15_6: # %else4
+; SSE4-NEXT:    testb $8, %al
+; SSE4-NEXT:    jne .LBB15_7
+; SSE4-NEXT:  .LBB15_8: # %else6
+; SSE4-NEXT:    testb $16, %al
+; SSE4-NEXT:    jne .LBB15_9
+; SSE4-NEXT:  .LBB15_10: # %else8
+; SSE4-NEXT:    testb $32, %al
+; SSE4-NEXT:    jne .LBB15_11
+; SSE4-NEXT:  .LBB15_12: # %else10
+; SSE4-NEXT:    testb $64, %al
+; SSE4-NEXT:    jne .LBB15_13
+; SSE4-NEXT:  .LBB15_14: # %else12
+; SSE4-NEXT:    testb $-128, %al
+; SSE4-NEXT:    jne .LBB15_15
+; SSE4-NEXT:  .LBB15_16: # %else14
+; SSE4-NEXT:    testl $256, %eax # imm = 0x100
+; SSE4-NEXT:    jne .LBB15_17
+; SSE4-NEXT:  .LBB15_18: # %else16
+; SSE4-NEXT:    testl $512, %eax # imm = 0x200
+; SSE4-NEXT:    jne .LBB15_19
+; SSE4-NEXT:  .LBB15_20: # %else18
+; SSE4-NEXT:    testl $1024, %eax # imm = 0x400
+; SSE4-NEXT:    jne .LBB15_21
+; SSE4-NEXT:  .LBB15_22: # %else20
+; SSE4-NEXT:    testl $2048, %eax # imm = 0x800
+; SSE4-NEXT:    jne .LBB15_23
+; SSE4-NEXT:  .LBB15_24: # %else22
+; SSE4-NEXT:    testl $4096, %eax # imm = 0x1000
+; SSE4-NEXT:    jne .LBB15_25
+; SSE4-NEXT:  .LBB15_26: # %else24
+; SSE4-NEXT:    testl $8192, %eax # imm = 0x2000
+; SSE4-NEXT:    jne .LBB15_27
+; SSE4-NEXT:  .LBB15_28: # %else26
+; SSE4-NEXT:    testl $16384, %eax # imm = 0x4000
+; SSE4-NEXT:    je .LBB15_30
+; SSE4-NEXT:  .LBB15_29: # %cond.store27
+; SSE4-NEXT:    pextrb $14, %xmm0, 14(%rdi)
+; SSE4-NEXT:  .LBB15_30: # %else28
+; SSE4-NEXT:    pminuw %xmm6, %xmm3
+; SSE4-NEXT:    pminuw %xmm6, %xmm2
+; SSE4-NEXT:    testl $32768, %eax # imm = 0x8000
+; SSE4-NEXT:    je .LBB15_32
+; SSE4-NEXT:  # %bb.31: # %cond.store29
+; SSE4-NEXT:    pextrb $15, %xmm0, 15(%rdi)
+; SSE4-NEXT:  .LBB15_32: # %else30
+; SSE4-NEXT:    packuswb %xmm3, %xmm2
+; SSE4-NEXT:    testl $65536, %eax # imm = 0x10000
+; SSE4-NEXT:    jne .LBB15_33
+; SSE4-NEXT:  # %bb.34: # %else32
+; SSE4-NEXT:    testl $131072, %eax # imm = 0x20000
+; SSE4-NEXT:    jne .LBB15_35
+; SSE4-NEXT:  .LBB15_36: # %else34
+; SSE4-NEXT:    testl $262144, %eax # imm = 0x40000
+; SSE4-NEXT:    jne .LBB15_37
+; SSE4-NEXT:  .LBB15_38: # %else36
+; SSE4-NEXT:    testl $524288, %eax # imm = 0x80000
+; SSE4-NEXT:    jne .LBB15_39
+; SSE4-NEXT:  .LBB15_40: # %else38
+; SSE4-NEXT:    testl $1048576, %eax # imm = 0x100000
+; SSE4-NEXT:    jne .LBB15_41
+; SSE4-NEXT:  .LBB15_42: # %else40
+; SSE4-NEXT:    testl $2097152, %eax # imm = 0x200000
+; SSE4-NEXT:    jne .LBB15_43
+; SSE4-NEXT:  .LBB15_44: # %else42
+; SSE4-NEXT:    testl $4194304, %eax # imm = 0x400000
+; SSE4-NEXT:    jne .LBB15_45
+; SSE4-NEXT:  .LBB15_46: # %else44
+; SSE4-NEXT:    testl $8388608, %eax # imm = 0x800000
+; SSE4-NEXT:    jne .LBB15_47
+; SSE4-NEXT:  .LBB15_48: # %else46
+; SSE4-NEXT:    testl $16777216, %eax # imm = 0x1000000
+; SSE4-NEXT:    jne .LBB15_49
+; SSE4-NEXT:  .LBB15_50: # %else48
+; SSE4-NEXT:    testl $33554432, %eax # imm = 0x2000000
+; SSE4-NEXT:    jne .LBB15_51
+; SSE4-NEXT:  .LBB15_52: # %else50
+; SSE4-NEXT:    testl $67108864, %eax # imm = 0x4000000
+; SSE4-NEXT:    jne .LBB15_53
+; SSE4-NEXT:  .LBB15_54: # %else52
+; SSE4-NEXT:    testl $134217728, %eax # imm = 0x8000000
+; SSE4-NEXT:    jne .LBB15_55
+; SSE4-NEXT:  .LBB15_56: # %else54
+; SSE4-NEXT:    testl $268435456, %eax # imm = 0x10000000
+; SSE4-NEXT:    jne .LBB15_57
+; SSE4-NEXT:  .LBB15_58: # %else56
+; SSE4-NEXT:    testl $536870912, %eax # imm = 0x20000000
+; SSE4-NEXT:    jne .LBB15_59
+; SSE4-NEXT:  .LBB15_60: # %else58
+; SSE4-NEXT:    testl $1073741824, %eax # imm = 0x40000000
+; SSE4-NEXT:    jne .LBB15_61
+; SSE4-NEXT:  .LBB15_62: # %else60
+; SSE4-NEXT:    testl $-2147483648, %eax # imm = 0x80000000
+; SSE4-NEXT:    jne .LBB15_63
+; SSE4-NEXT:  .LBB15_64: # %else62
+; SSE4-NEXT:    retq
+; SSE4-NEXT:  .LBB15_1: # %cond.store
 ; SSE4-NEXT:    pextrb $0, %xmm0, (%rdi)
-; SSE4-NEXT:  .LBB15_2: # %else
-; SSE4-NEXT:    pextrb $1, %xmm7, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $2, %al
 ; SSE4-NEXT:    je .LBB15_4
-; SSE4-NEXT:  # %bb.3: # %cond.store1
+; SSE4-NEXT:  .LBB15_3: # %cond.store1
 ; SSE4-NEXT:    pextrb $1, %xmm0, 1(%rdi)
-; SSE4-NEXT:  .LBB15_4: # %else2
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqb %xmm4, %xmm1
-; SSE4-NEXT:    pextrb $2, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $4, %al
 ; SSE4-NEXT:    je .LBB15_6
-; SSE4-NEXT:  # %bb.5: # %cond.store3
+; SSE4-NEXT:  .LBB15_5: # %cond.store3
 ; SSE4-NEXT:    pextrb $2, %xmm0, 2(%rdi)
-; SSE4-NEXT:  .LBB15_6: # %else4
-; SSE4-NEXT:    pextrb $3, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $8, %al
 ; SSE4-NEXT:    je .LBB15_8
-; SSE4-NEXT:  # %bb.7: # %cond.store5
+; SSE4-NEXT:  .LBB15_7: # %cond.store5
 ; SSE4-NEXT:    pextrb $3, %xmm0, 3(%rdi)
-; SSE4-NEXT:  .LBB15_8: # %else6
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqb %xmm4, %xmm1
-; SSE4-NEXT:    pextrb $4, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $16, %al
 ; SSE4-NEXT:    je .LBB15_10
-; SSE4-NEXT:  # %bb.9: # %cond.store7
+; SSE4-NEXT:  .LBB15_9: # %cond.store7
 ; SSE4-NEXT:    pextrb $4, %xmm0, 4(%rdi)
-; SSE4-NEXT:  .LBB15_10: # %else8
-; SSE4-NEXT:    pextrb $5, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $32, %al
 ; SSE4-NEXT:    je .LBB15_12
-; SSE4-NEXT:  # %bb.11: # %cond.store9
+; SSE4-NEXT:  .LBB15_11: # %cond.store9
 ; SSE4-NEXT:    pextrb $5, %xmm0, 5(%rdi)
-; SSE4-NEXT:  .LBB15_12: # %else10
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqb %xmm4, %xmm1
-; SSE4-NEXT:    pextrb $6, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $64, %al
 ; SSE4-NEXT:    je .LBB15_14
-; SSE4-NEXT:  # %bb.13: # %cond.store11
+; SSE4-NEXT:  .LBB15_13: # %cond.store11
 ; SSE4-NEXT:    pextrb $6, %xmm0, 6(%rdi)
-; SSE4-NEXT:  .LBB15_14: # %else12
-; SSE4-NEXT:    pextrb $7, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $-128, %al
 ; SSE4-NEXT:    je .LBB15_16
-; SSE4-NEXT:  # %bb.15: # %cond.store13
+; SSE4-NEXT:  .LBB15_15: # %cond.store13
 ; SSE4-NEXT:    pextrb $7, %xmm0, 7(%rdi)
-; SSE4-NEXT:  .LBB15_16: # %else14
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqb %xmm4, %xmm1
-; SSE4-NEXT:    pextrb $8, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $256, %eax # imm = 0x100
 ; SSE4-NEXT:    je .LBB15_18
-; SSE4-NEXT:  # %bb.17: # %cond.store15
+; SSE4-NEXT:  .LBB15_17: # %cond.store15
 ; SSE4-NEXT:    pextrb $8, %xmm0, 8(%rdi)
-; SSE4-NEXT:  .LBB15_18: # %else16
-; SSE4-NEXT:    pextrb $9, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $512, %eax # imm = 0x200
 ; SSE4-NEXT:    je .LBB15_20
-; SSE4-NEXT:  # %bb.19: # %cond.store17
+; SSE4-NEXT:  .LBB15_19: # %cond.store17
 ; SSE4-NEXT:    pextrb $9, %xmm0, 9(%rdi)
-; SSE4-NEXT:  .LBB15_20: # %else18
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqb %xmm4, %xmm1
-; SSE4-NEXT:    pextrb $10, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $1024, %eax # imm = 0x400
 ; SSE4-NEXT:    je .LBB15_22
-; SSE4-NEXT:  # %bb.21: # %cond.store19
+; SSE4-NEXT:  .LBB15_21: # %cond.store19
 ; SSE4-NEXT:    pextrb $10, %xmm0, 10(%rdi)
-; SSE4-NEXT:  .LBB15_22: # %else20
-; SSE4-NEXT:    pextrb $11, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $2048, %eax # imm = 0x800
 ; SSE4-NEXT:    je .LBB15_24
-; SSE4-NEXT:  # %bb.23: # %cond.store21
+; SSE4-NEXT:  .LBB15_23: # %cond.store21
 ; SSE4-NEXT:    pextrb $11, %xmm0, 11(%rdi)
-; SSE4-NEXT:  .LBB15_24: # %else22
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqb %xmm4, %xmm1
-; SSE4-NEXT:    pextrb $12, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $4096, %eax # imm = 0x1000
 ; SSE4-NEXT:    je .LBB15_26
-; SSE4-NEXT:  # %bb.25: # %cond.store23
+; SSE4-NEXT:  .LBB15_25: # %cond.store23
 ; SSE4-NEXT:    pextrb $12, %xmm0, 12(%rdi)
-; SSE4-NEXT:  .LBB15_26: # %else24
-; SSE4-NEXT:    pextrb $13, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $8192, %eax # imm = 0x2000
 ; SSE4-NEXT:    je .LBB15_28
-; SSE4-NEXT:  # %bb.27: # %cond.store25
+; SSE4-NEXT:  .LBB15_27: # %cond.store25
 ; SSE4-NEXT:    pextrb $13, %xmm0, 13(%rdi)
-; SSE4-NEXT:  .LBB15_28: # %else26
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqb %xmm1, %xmm4
-; SSE4-NEXT:    pextrb $14, %xmm4, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB15_30
-; SSE4-NEXT:  # %bb.29: # %cond.store27
-; SSE4-NEXT:    pextrb $14, %xmm0, 14(%rdi)
-; SSE4-NEXT:  .LBB15_30: # %else28
-; SSE4-NEXT:    pminuw %xmm6, %xmm3
-; SSE4-NEXT:    pminuw %xmm6, %xmm2
-; SSE4-NEXT:    pextrb $15, %xmm4, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB15_32
-; SSE4-NEXT:  # %bb.31: # %cond.store29
-; SSE4-NEXT:    pextrb $15, %xmm0, 15(%rdi)
-; SSE4-NEXT:  .LBB15_32: # %else30
-; SSE4-NEXT:    packuswb %xmm3, %xmm2
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqb %xmm5, %xmm0
-; SSE4-NEXT:    pextrb $0, %xmm0, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB15_34
-; SSE4-NEXT:  # %bb.33: # %cond.store31
+; SSE4-NEXT:    testl $16384, %eax # imm = 0x4000
+; SSE4-NEXT:    jne .LBB15_29
+; SSE4-NEXT:    jmp .LBB15_30
+; SSE4-NEXT:  .LBB15_33: # %cond.store31
 ; SSE4-NEXT:    pextrb $0, %xmm2, 16(%rdi)
-; SSE4-NEXT:  .LBB15_34: # %else32
-; SSE4-NEXT:    pextrb $1, %xmm0, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $131072, %eax # imm = 0x20000
 ; SSE4-NEXT:    je .LBB15_36
-; SSE4-NEXT:  # %bb.35: # %cond.store33
+; SSE4-NEXT:  .LBB15_35: # %cond.store33
 ; SSE4-NEXT:    pextrb $1, %xmm2, 17(%rdi)
-; SSE4-NEXT:  .LBB15_36: # %else34
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqb %xmm5, %xmm0
-; SSE4-NEXT:    pextrb $2, %xmm0, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $262144, %eax # imm = 0x40000
 ; SSE4-NEXT:    je .LBB15_38
-; SSE4-NEXT:  # %bb.37: # %cond.store35
+; SSE4-NEXT:  .LBB15_37: # %cond.store35
 ; SSE4-NEXT:    pextrb $2, %xmm2, 18(%rdi)
-; SSE4-NEXT:  .LBB15_38: # %else36
-; SSE4-NEXT:    pextrb $3, %xmm0, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $524288, %eax # imm = 0x80000
 ; SSE4-NEXT:    je .LBB15_40
-; SSE4-NEXT:  # %bb.39: # %cond.store37
+; SSE4-NEXT:  .LBB15_39: # %cond.store37
 ; SSE4-NEXT:    pextrb $3, %xmm2, 19(%rdi)
-; SSE4-NEXT:  .LBB15_40: # %else38
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqb %xmm5, %xmm0
-; SSE4-NEXT:    pextrb $4, %xmm0, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $1048576, %eax # imm = 0x100000
 ; SSE4-NEXT:    je .LBB15_42
-; SSE4-NEXT:  # %bb.41: # %cond.store39
+; SSE4-NEXT:  .LBB15_41: # %cond.store39
 ; SSE4-NEXT:    pextrb $4, %xmm2, 20(%rdi)
-; SSE4-NEXT:  .LBB15_42: # %else40
-; SSE4-NEXT:    pextrb $5, %xmm0, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $2097152, %eax # imm = 0x200000
 ; SSE4-NEXT:    je .LBB15_44
-; SSE4-NEXT:  # %bb.43: # %cond.store41
+; SSE4-NEXT:  .LBB15_43: # %cond.store41
 ; SSE4-NEXT:    pextrb $5, %xmm2, 21(%rdi)
-; SSE4-NEXT:  .LBB15_44: # %else42
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqb %xmm5, %xmm0
-; SSE4-NEXT:    pextrb $6, %xmm0, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $4194304, %eax # imm = 0x400000
 ; SSE4-NEXT:    je .LBB15_46
-; SSE4-NEXT:  # %bb.45: # %cond.store43
+; SSE4-NEXT:  .LBB15_45: # %cond.store43
 ; SSE4-NEXT:    pextrb $6, %xmm2, 22(%rdi)
-; SSE4-NEXT:  .LBB15_46: # %else44
-; SSE4-NEXT:    pextrb $7, %xmm0, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $8388608, %eax # imm = 0x800000
 ; SSE4-NEXT:    je .LBB15_48
-; SSE4-NEXT:  # %bb.47: # %cond.store45
+; SSE4-NEXT:  .LBB15_47: # %cond.store45
 ; SSE4-NEXT:    pextrb $7, %xmm2, 23(%rdi)
-; SSE4-NEXT:  .LBB15_48: # %else46
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqb %xmm5, %xmm0
-; SSE4-NEXT:    pextrb $8, %xmm0, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $16777216, %eax # imm = 0x1000000
 ; SSE4-NEXT:    je .LBB15_50
-; SSE4-NEXT:  # %bb.49: # %cond.store47
+; SSE4-NEXT:  .LBB15_49: # %cond.store47
 ; SSE4-NEXT:    pextrb $8, %xmm2, 24(%rdi)
-; SSE4-NEXT:  .LBB15_50: # %else48
-; SSE4-NEXT:    pextrb $9, %xmm0, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $33554432, %eax # imm = 0x2000000
 ; SSE4-NEXT:    je .LBB15_52
-; SSE4-NEXT:  # %bb.51: # %cond.store49
+; SSE4-NEXT:  .LBB15_51: # %cond.store49
 ; SSE4-NEXT:    pextrb $9, %xmm2, 25(%rdi)
-; SSE4-NEXT:  .LBB15_52: # %else50
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqb %xmm5, %xmm0
-; SSE4-NEXT:    pextrb $10, %xmm0, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $67108864, %eax # imm = 0x4000000
 ; SSE4-NEXT:    je .LBB15_54
-; SSE4-NEXT:  # %bb.53: # %cond.store51
+; SSE4-NEXT:  .LBB15_53: # %cond.store51
 ; SSE4-NEXT:    pextrb $10, %xmm2, 26(%rdi)
-; SSE4-NEXT:  .LBB15_54: # %else52
-; SSE4-NEXT:    pextrb $11, %xmm0, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $134217728, %eax # imm = 0x8000000
 ; SSE4-NEXT:    je .LBB15_56
-; SSE4-NEXT:  # %bb.55: # %cond.store53
+; SSE4-NEXT:  .LBB15_55: # %cond.store53
 ; SSE4-NEXT:    pextrb $11, %xmm2, 27(%rdi)
-; SSE4-NEXT:  .LBB15_56: # %else54
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqb %xmm5, %xmm0
-; SSE4-NEXT:    pextrb $12, %xmm0, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $268435456, %eax # imm = 0x10000000
 ; SSE4-NEXT:    je .LBB15_58
-; SSE4-NEXT:  # %bb.57: # %cond.store55
+; SSE4-NEXT:  .LBB15_57: # %cond.store55
 ; SSE4-NEXT:    pextrb $12, %xmm2, 28(%rdi)
-; SSE4-NEXT:  .LBB15_58: # %else56
-; SSE4-NEXT:    pextrb $13, %xmm0, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $536870912, %eax # imm = 0x20000000
 ; SSE4-NEXT:    je .LBB15_60
-; SSE4-NEXT:  # %bb.59: # %cond.store57
+; SSE4-NEXT:  .LBB15_59: # %cond.store57
 ; SSE4-NEXT:    pextrb $13, %xmm2, 29(%rdi)
-; SSE4-NEXT:  .LBB15_60: # %else58
-; SSE4-NEXT:    pxor %xmm0, %xmm0
-; SSE4-NEXT:    pcmpeqb %xmm0, %xmm5
-; SSE4-NEXT:    pextrb $14, %xmm5, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $1073741824, %eax # imm = 0x40000000
 ; SSE4-NEXT:    je .LBB15_62
-; SSE4-NEXT:  # %bb.61: # %cond.store59
+; SSE4-NEXT:  .LBB15_61: # %cond.store59
 ; SSE4-NEXT:    pextrb $14, %xmm2, 30(%rdi)
-; SSE4-NEXT:  .LBB15_62: # %else60
-; SSE4-NEXT:    pextrb $15, %xmm5, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $-2147483648, %eax # imm = 0x80000000
 ; SSE4-NEXT:    je .LBB15_64
-; SSE4-NEXT:  # %bb.63: # %cond.store61
+; SSE4-NEXT:  .LBB15_63: # %cond.store61
 ; SSE4-NEXT:    pextrb $15, %xmm2, 31(%rdi)
-; SSE4-NEXT:  .LBB15_64: # %else62
 ; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: truncstore_v32i16_v32i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpeqb %xmm3, %xmm2, %xmm5
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm6
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm7 = [255,255,255,255,255,255,255,255]
-; AVX1-NEXT:    vpminuw %xmm7, %xmm6, %xmm6
-; AVX1-NEXT:    vpminuw %xmm7, %xmm0, %xmm0
-; AVX1-NEXT:    vpackuswb %xmm6, %xmm0, %xmm0
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm6
-; AVX1-NEXT:    vpminuw %xmm7, %xmm6, %xmm6
-; AVX1-NEXT:    vpminuw %xmm7, %xmm1, %xmm1
-; AVX1-NEXT:    vpackuswb %xmm6, %xmm1, %xmm1
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
+; AVX1-NEXT:    vpminuw %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpminuw %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vpackuswb %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
+; AVX1-NEXT:    vpminuw %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpminuw %xmm4, %xmm1, %xmm1
+; AVX1-NEXT:    vpackuswb %xmm3, %xmm1, %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX1-NEXT:    vpextrb $0, %xmm5, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je .LBB15_2
-; AVX1-NEXT:  # %bb.1: # %cond.store
+; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm3
+; AVX1-NEXT:    vpmovmskb %xmm3, %ecx
+; AVX1-NEXT:    xorl $65535, %ecx # imm = 0xFFFF
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm2
+; AVX1-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vpmovmskb %xmm1, %eax
+; AVX1-NEXT:    notl %eax
+; AVX1-NEXT:    shll $16, %eax
+; AVX1-NEXT:    orl %ecx, %eax
+; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    jne .LBB15_1
+; AVX1-NEXT:  # %bb.2: # %else
+; AVX1-NEXT:    testb $2, %al
+; AVX1-NEXT:    jne .LBB15_3
+; AVX1-NEXT:  .LBB15_4: # %else2
+; AVX1-NEXT:    testb $4, %al
+; AVX1-NEXT:    jne .LBB15_5
+; AVX1-NEXT:  .LBB15_6: # %else4
+; AVX1-NEXT:    testb $8, %al
+; AVX1-NEXT:    jne .LBB15_7
+; AVX1-NEXT:  .LBB15_8: # %else6
+; AVX1-NEXT:    testb $16, %al
+; AVX1-NEXT:    jne .LBB15_9
+; AVX1-NEXT:  .LBB15_10: # %else8
+; AVX1-NEXT:    testb $32, %al
+; AVX1-NEXT:    jne .LBB15_11
+; AVX1-NEXT:  .LBB15_12: # %else10
+; AVX1-NEXT:    testb $64, %al
+; AVX1-NEXT:    jne .LBB15_13
+; AVX1-NEXT:  .LBB15_14: # %else12
+; AVX1-NEXT:    testb $-128, %al
+; AVX1-NEXT:    jne .LBB15_15
+; AVX1-NEXT:  .LBB15_16: # %else14
+; AVX1-NEXT:    testl $256, %eax # imm = 0x100
+; AVX1-NEXT:    jne .LBB15_17
+; AVX1-NEXT:  .LBB15_18: # %else16
+; AVX1-NEXT:    testl $512, %eax # imm = 0x200
+; AVX1-NEXT:    jne .LBB15_19
+; AVX1-NEXT:  .LBB15_20: # %else18
+; AVX1-NEXT:    testl $1024, %eax # imm = 0x400
+; AVX1-NEXT:    jne .LBB15_21
+; AVX1-NEXT:  .LBB15_22: # %else20
+; AVX1-NEXT:    testl $2048, %eax # imm = 0x800
+; AVX1-NEXT:    jne .LBB15_23
+; AVX1-NEXT:  .LBB15_24: # %else22
+; AVX1-NEXT:    testl $4096, %eax # imm = 0x1000
+; AVX1-NEXT:    jne .LBB15_25
+; AVX1-NEXT:  .LBB15_26: # %else24
+; AVX1-NEXT:    testl $8192, %eax # imm = 0x2000
+; AVX1-NEXT:    jne .LBB15_27
+; AVX1-NEXT:  .LBB15_28: # %else26
+; AVX1-NEXT:    testl $16384, %eax # imm = 0x4000
+; AVX1-NEXT:    jne .LBB15_29
+; AVX1-NEXT:  .LBB15_30: # %else28
+; AVX1-NEXT:    testl $32768, %eax # imm = 0x8000
+; AVX1-NEXT:    je .LBB15_32
+; AVX1-NEXT:  .LBB15_31: # %cond.store29
+; AVX1-NEXT:    vpextrb $15, %xmm0, 15(%rdi)
+; AVX1-NEXT:  .LBB15_32: # %else30
+; AVX1-NEXT:    testl $65536, %eax # imm = 0x10000
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    jne .LBB15_33
+; AVX1-NEXT:  # %bb.34: # %else32
+; AVX1-NEXT:    testl $131072, %eax # imm = 0x20000
+; AVX1-NEXT:    jne .LBB15_35
+; AVX1-NEXT:  .LBB15_36: # %else34
+; AVX1-NEXT:    testl $262144, %eax # imm = 0x40000
+; AVX1-NEXT:    jne .LBB15_37
+; AVX1-NEXT:  .LBB15_38: # %else36
+; AVX1-NEXT:    testl $524288, %eax # imm = 0x80000
+; AVX1-NEXT:    jne .LBB15_39
+; AVX1-NEXT:  .LBB15_40: # %else38
+; AVX1-NEXT:    testl $1048576, %eax # imm = 0x100000
+; AVX1-NEXT:    jne .LBB15_41
+; AVX1-NEXT:  .LBB15_42: # %else40
+; AVX1-NEXT:    testl $2097152, %eax # imm = 0x200000
+; AVX1-NEXT:    jne .LBB15_43
+; AVX1-NEXT:  .LBB15_44: # %else42
+; AVX1-NEXT:    testl $4194304, %eax # imm = 0x400000
+; AVX1-NEXT:    jne .LBB15_45
+; AVX1-NEXT:  .LBB15_46: # %else44
+; AVX1-NEXT:    testl $8388608, %eax # imm = 0x800000
+; AVX1-NEXT:    jne .LBB15_47
+; AVX1-NEXT:  .LBB15_48: # %else46
+; AVX1-NEXT:    testl $16777216, %eax # imm = 0x1000000
+; AVX1-NEXT:    jne .LBB15_49
+; AVX1-NEXT:  .LBB15_50: # %else48
+; AVX1-NEXT:    testl $33554432, %eax # imm = 0x2000000
+; AVX1-NEXT:    jne .LBB15_51
+; AVX1-NEXT:  .LBB15_52: # %else50
+; AVX1-NEXT:    testl $67108864, %eax # imm = 0x4000000
+; AVX1-NEXT:    jne .LBB15_53
+; AVX1-NEXT:  .LBB15_54: # %else52
+; AVX1-NEXT:    testl $134217728, %eax # imm = 0x8000000
+; AVX1-NEXT:    jne .LBB15_55
+; AVX1-NEXT:  .LBB15_56: # %else54
+; AVX1-NEXT:    testl $268435456, %eax # imm = 0x10000000
+; AVX1-NEXT:    jne .LBB15_57
+; AVX1-NEXT:  .LBB15_58: # %else56
+; AVX1-NEXT:    testl $536870912, %eax # imm = 0x20000000
+; AVX1-NEXT:    jne .LBB15_59
+; AVX1-NEXT:  .LBB15_60: # %else58
+; AVX1-NEXT:    testl $1073741824, %eax # imm = 0x40000000
+; AVX1-NEXT:    jne .LBB15_61
+; AVX1-NEXT:  .LBB15_62: # %else60
+; AVX1-NEXT:    testl $-2147483648, %eax # imm = 0x80000000
+; AVX1-NEXT:    jne .LBB15_63
+; AVX1-NEXT:  .LBB15_64: # %else62
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+; AVX1-NEXT:  .LBB15_1: # %cond.store
 ; AVX1-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX1-NEXT:  .LBB15_2: # %else
-; AVX1-NEXT:    vpcmpeqb %xmm3, %xmm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm4, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $1, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $2, %al
 ; AVX1-NEXT:    je .LBB15_4
-; AVX1-NEXT:  # %bb.3: # %cond.store1
+; AVX1-NEXT:  .LBB15_3: # %cond.store1
 ; AVX1-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
-; AVX1-NEXT:  .LBB15_4: # %else2
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm4, %xmm4
-; AVX1-NEXT:    vpextrb $2, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $4, %al
 ; AVX1-NEXT:    je .LBB15_6
-; AVX1-NEXT:  # %bb.5: # %cond.store3
+; AVX1-NEXT:  .LBB15_5: # %cond.store3
 ; AVX1-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
-; AVX1-NEXT:  .LBB15_6: # %else4
-; AVX1-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $3, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $8, %al
 ; AVX1-NEXT:    je .LBB15_8
-; AVX1-NEXT:  # %bb.7: # %cond.store5
+; AVX1-NEXT:  .LBB15_7: # %cond.store5
 ; AVX1-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
-; AVX1-NEXT:  .LBB15_8: # %else6
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm4, %xmm4
-; AVX1-NEXT:    vpextrb $4, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $16, %al
 ; AVX1-NEXT:    je .LBB15_10
-; AVX1-NEXT:  # %bb.9: # %cond.store7
+; AVX1-NEXT:  .LBB15_9: # %cond.store7
 ; AVX1-NEXT:    vpextrb $4, %xmm0, 4(%rdi)
-; AVX1-NEXT:  .LBB15_10: # %else8
-; AVX1-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $5, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $32, %al
 ; AVX1-NEXT:    je .LBB15_12
-; AVX1-NEXT:  # %bb.11: # %cond.store9
+; AVX1-NEXT:  .LBB15_11: # %cond.store9
 ; AVX1-NEXT:    vpextrb $5, %xmm0, 5(%rdi)
-; AVX1-NEXT:  .LBB15_12: # %else10
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm4, %xmm4
-; AVX1-NEXT:    vpextrb $6, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $64, %al
 ; AVX1-NEXT:    je .LBB15_14
-; AVX1-NEXT:  # %bb.13: # %cond.store11
+; AVX1-NEXT:  .LBB15_13: # %cond.store11
 ; AVX1-NEXT:    vpextrb $6, %xmm0, 6(%rdi)
-; AVX1-NEXT:  .LBB15_14: # %else12
-; AVX1-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $7, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $-128, %al
 ; AVX1-NEXT:    je .LBB15_16
-; AVX1-NEXT:  # %bb.15: # %cond.store13
+; AVX1-NEXT:  .LBB15_15: # %cond.store13
 ; AVX1-NEXT:    vpextrb $7, %xmm0, 7(%rdi)
-; AVX1-NEXT:  .LBB15_16: # %else14
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm4, %xmm4
-; AVX1-NEXT:    vpextrb $8, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $256, %eax # imm = 0x100
 ; AVX1-NEXT:    je .LBB15_18
-; AVX1-NEXT:  # %bb.17: # %cond.store15
+; AVX1-NEXT:  .LBB15_17: # %cond.store15
 ; AVX1-NEXT:    vpextrb $8, %xmm0, 8(%rdi)
-; AVX1-NEXT:  .LBB15_18: # %else16
-; AVX1-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $9, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $512, %eax # imm = 0x200
 ; AVX1-NEXT:    je .LBB15_20
-; AVX1-NEXT:  # %bb.19: # %cond.store17
+; AVX1-NEXT:  .LBB15_19: # %cond.store17
 ; AVX1-NEXT:    vpextrb $9, %xmm0, 9(%rdi)
-; AVX1-NEXT:  .LBB15_20: # %else18
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm4, %xmm4
-; AVX1-NEXT:    vpextrb $10, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $1024, %eax # imm = 0x400
 ; AVX1-NEXT:    je .LBB15_22
-; AVX1-NEXT:  # %bb.21: # %cond.store19
+; AVX1-NEXT:  .LBB15_21: # %cond.store19
 ; AVX1-NEXT:    vpextrb $10, %xmm0, 10(%rdi)
-; AVX1-NEXT:  .LBB15_22: # %else20
-; AVX1-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $11, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $2048, %eax # imm = 0x800
 ; AVX1-NEXT:    je .LBB15_24
-; AVX1-NEXT:  # %bb.23: # %cond.store21
+; AVX1-NEXT:  .LBB15_23: # %cond.store21
 ; AVX1-NEXT:    vpextrb $11, %xmm0, 11(%rdi)
-; AVX1-NEXT:  .LBB15_24: # %else22
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm4, %xmm4
-; AVX1-NEXT:    vpextrb $12, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $4096, %eax # imm = 0x1000
 ; AVX1-NEXT:    je .LBB15_26
-; AVX1-NEXT:  # %bb.25: # %cond.store23
+; AVX1-NEXT:  .LBB15_25: # %cond.store23
 ; AVX1-NEXT:    vpextrb $12, %xmm0, 12(%rdi)
-; AVX1-NEXT:  .LBB15_26: # %else24
-; AVX1-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $13, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $8192, %eax # imm = 0x2000
 ; AVX1-NEXT:    je .LBB15_28
-; AVX1-NEXT:  # %bb.27: # %cond.store25
+; AVX1-NEXT:  .LBB15_27: # %cond.store25
 ; AVX1-NEXT:    vpextrb $13, %xmm0, 13(%rdi)
-; AVX1-NEXT:  .LBB15_28: # %else26
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm4, %xmm4
-; AVX1-NEXT:    vpextrb $14, %xmm4, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $16384, %eax # imm = 0x4000
 ; AVX1-NEXT:    je .LBB15_30
-; AVX1-NEXT:  # %bb.29: # %cond.store27
+; AVX1-NEXT:  .LBB15_29: # %cond.store27
 ; AVX1-NEXT:    vpextrb $14, %xmm0, 14(%rdi)
-; AVX1-NEXT:  .LBB15_30: # %else28
-; AVX1-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $15, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je .LBB15_32
-; AVX1-NEXT:  # %bb.31: # %cond.store29
-; AVX1-NEXT:    vpextrb $15, %xmm0, 15(%rdi)
-; AVX1-NEXT:  .LBB15_32: # %else30
-; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $0, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT:    je .LBB15_34
-; AVX1-NEXT:  # %bb.33: # %cond.store31
+; AVX1-NEXT:    testl $32768, %eax # imm = 0x8000
+; AVX1-NEXT:    jne .LBB15_31
+; AVX1-NEXT:    jmp .LBB15_32
+; AVX1-NEXT:  .LBB15_33: # %cond.store31
 ; AVX1-NEXT:    vpextrb $0, %xmm0, 16(%rdi)
-; AVX1-NEXT:  .LBB15_34: # %else32
-; AVX1-NEXT:    vpextrb $1, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $131072, %eax # imm = 0x20000
 ; AVX1-NEXT:    je .LBB15_36
-; AVX1-NEXT:  # %bb.35: # %cond.store33
+; AVX1-NEXT:  .LBB15_35: # %cond.store33
 ; AVX1-NEXT:    vpextrb $1, %xmm0, 17(%rdi)
-; AVX1-NEXT:  .LBB15_36: # %else34
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $2, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $262144, %eax # imm = 0x40000
 ; AVX1-NEXT:    je .LBB15_38
-; AVX1-NEXT:  # %bb.37: # %cond.store35
+; AVX1-NEXT:  .LBB15_37: # %cond.store35
 ; AVX1-NEXT:    vpextrb $2, %xmm0, 18(%rdi)
-; AVX1-NEXT:  .LBB15_38: # %else36
-; AVX1-NEXT:    vpextrb $3, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $524288, %eax # imm = 0x80000
 ; AVX1-NEXT:    je .LBB15_40
-; AVX1-NEXT:  # %bb.39: # %cond.store37
+; AVX1-NEXT:  .LBB15_39: # %cond.store37
 ; AVX1-NEXT:    vpextrb $3, %xmm0, 19(%rdi)
-; AVX1-NEXT:  .LBB15_40: # %else38
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $1048576, %eax # imm = 0x100000
 ; AVX1-NEXT:    je .LBB15_42
-; AVX1-NEXT:  # %bb.41: # %cond.store39
+; AVX1-NEXT:  .LBB15_41: # %cond.store39
 ; AVX1-NEXT:    vpextrb $4, %xmm0, 20(%rdi)
-; AVX1-NEXT:  .LBB15_42: # %else40
-; AVX1-NEXT:    vpextrb $5, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $2097152, %eax # imm = 0x200000
 ; AVX1-NEXT:    je .LBB15_44
-; AVX1-NEXT:  # %bb.43: # %cond.store41
+; AVX1-NEXT:  .LBB15_43: # %cond.store41
 ; AVX1-NEXT:    vpextrb $5, %xmm0, 21(%rdi)
-; AVX1-NEXT:  .LBB15_44: # %else42
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $6, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $4194304, %eax # imm = 0x400000
 ; AVX1-NEXT:    je .LBB15_46
-; AVX1-NEXT:  # %bb.45: # %cond.store43
+; AVX1-NEXT:  .LBB15_45: # %cond.store43
 ; AVX1-NEXT:    vpextrb $6, %xmm0, 22(%rdi)
-; AVX1-NEXT:  .LBB15_46: # %else44
-; AVX1-NEXT:    vpextrb $7, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $8388608, %eax # imm = 0x800000
 ; AVX1-NEXT:    je .LBB15_48
-; AVX1-NEXT:  # %bb.47: # %cond.store45
+; AVX1-NEXT:  .LBB15_47: # %cond.store45
 ; AVX1-NEXT:    vpextrb $7, %xmm0, 23(%rdi)
-; AVX1-NEXT:  .LBB15_48: # %else46
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $8, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $16777216, %eax # imm = 0x1000000
 ; AVX1-NEXT:    je .LBB15_50
-; AVX1-NEXT:  # %bb.49: # %cond.store47
+; AVX1-NEXT:  .LBB15_49: # %cond.store47
 ; AVX1-NEXT:    vpextrb $8, %xmm0, 24(%rdi)
-; AVX1-NEXT:  .LBB15_50: # %else48
-; AVX1-NEXT:    vpextrb $9, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $33554432, %eax # imm = 0x2000000
 ; AVX1-NEXT:    je .LBB15_52
-; AVX1-NEXT:  # %bb.51: # %cond.store49
+; AVX1-NEXT:  .LBB15_51: # %cond.store49
 ; AVX1-NEXT:    vpextrb $9, %xmm0, 25(%rdi)
-; AVX1-NEXT:  .LBB15_52: # %else50
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $10, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $67108864, %eax # imm = 0x4000000
 ; AVX1-NEXT:    je .LBB15_54
-; AVX1-NEXT:  # %bb.53: # %cond.store51
+; AVX1-NEXT:  .LBB15_53: # %cond.store51
 ; AVX1-NEXT:    vpextrb $10, %xmm0, 26(%rdi)
-; AVX1-NEXT:  .LBB15_54: # %else52
-; AVX1-NEXT:    vpextrb $11, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $134217728, %eax # imm = 0x8000000
 ; AVX1-NEXT:    je .LBB15_56
-; AVX1-NEXT:  # %bb.55: # %cond.store53
+; AVX1-NEXT:  .LBB15_55: # %cond.store53
 ; AVX1-NEXT:    vpextrb $11, %xmm0, 27(%rdi)
-; AVX1-NEXT:  .LBB15_56: # %else54
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrb $12, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $268435456, %eax # imm = 0x10000000
 ; AVX1-NEXT:    je .LBB15_58
-; AVX1-NEXT:  # %bb.57: # %cond.store55
+; AVX1-NEXT:  .LBB15_57: # %cond.store55
 ; AVX1-NEXT:    vpextrb $12, %xmm0, 28(%rdi)
-; AVX1-NEXT:  .LBB15_58: # %else56
-; AVX1-NEXT:    vpextrb $13, %xmm2, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $536870912, %eax # imm = 0x20000000
 ; AVX1-NEXT:    je .LBB15_60
-; AVX1-NEXT:  # %bb.59: # %cond.store57
+; AVX1-NEXT:  .LBB15_59: # %cond.store57
 ; AVX1-NEXT:    vpextrb $13, %xmm0, 29(%rdi)
-; AVX1-NEXT:  .LBB15_60: # %else58
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $14, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $1073741824, %eax # imm = 0x40000000
 ; AVX1-NEXT:    je .LBB15_62
-; AVX1-NEXT:  # %bb.61: # %cond.store59
+; AVX1-NEXT:  .LBB15_61: # %cond.store59
 ; AVX1-NEXT:    vpextrb $14, %xmm0, 30(%rdi)
-; AVX1-NEXT:  .LBB15_62: # %else60
-; AVX1-NEXT:    vpextrb $15, %xmm1, %eax
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $-2147483648, %eax # imm = 0x80000000
 ; AVX1-NEXT:    je .LBB15_64
-; AVX1-NEXT:  # %bb.63: # %cond.store61
+; AVX1-NEXT:  .LBB15_63: # %cond.store61
 ; AVX1-NEXT:    vpextrb $15, %xmm0, 31(%rdi)
-; AVX1-NEXT:  .LBB15_64: # %else62
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: truncstore_v32i16_v32i8:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vmovdqa {{.*#+}} ymm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX2-NEXT:    vpminuw %ymm3, %ymm1, %ymm1
-; AVX2-NEXT:    vpminuw %ymm3, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX2-NEXT:    vmovdqa {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; AVX2-NEXT:    vpminuw %ymm4, %ymm1, %ymm1
+; AVX2-NEXT:    vpminuw %ymm4, %ymm0, %ymm0
 ; AVX2-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm3
-; AVX2-NEXT:    vpextrb $0, %xmm3, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je .LBB15_2
-; AVX2-NEXT:  # %bb.1: # %cond.store
+; AVX2-NEXT:    vpcmpeqb %ymm3, %ymm2, %ymm1
+; AVX2-NEXT:    vpmovmskb %ymm1, %eax
+; AVX2-NEXT:    notl %eax
+; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    jne .LBB15_1
+; AVX2-NEXT:  # %bb.2: # %else
+; AVX2-NEXT:    testb $2, %al
+; AVX2-NEXT:    jne .LBB15_3
+; AVX2-NEXT:  .LBB15_4: # %else2
+; AVX2-NEXT:    testb $4, %al
+; AVX2-NEXT:    jne .LBB15_5
+; AVX2-NEXT:  .LBB15_6: # %else4
+; AVX2-NEXT:    testb $8, %al
+; AVX2-NEXT:    jne .LBB15_7
+; AVX2-NEXT:  .LBB15_8: # %else6
+; AVX2-NEXT:    testb $16, %al
+; AVX2-NEXT:    jne .LBB15_9
+; AVX2-NEXT:  .LBB15_10: # %else8
+; AVX2-NEXT:    testb $32, %al
+; AVX2-NEXT:    jne .LBB15_11
+; AVX2-NEXT:  .LBB15_12: # %else10
+; AVX2-NEXT:    testb $64, %al
+; AVX2-NEXT:    jne .LBB15_13
+; AVX2-NEXT:  .LBB15_14: # %else12
+; AVX2-NEXT:    testb $-128, %al
+; AVX2-NEXT:    jne .LBB15_15
+; AVX2-NEXT:  .LBB15_16: # %else14
+; AVX2-NEXT:    testl $256, %eax # imm = 0x100
+; AVX2-NEXT:    jne .LBB15_17
+; AVX2-NEXT:  .LBB15_18: # %else16
+; AVX2-NEXT:    testl $512, %eax # imm = 0x200
+; AVX2-NEXT:    jne .LBB15_19
+; AVX2-NEXT:  .LBB15_20: # %else18
+; AVX2-NEXT:    testl $1024, %eax # imm = 0x400
+; AVX2-NEXT:    jne .LBB15_21
+; AVX2-NEXT:  .LBB15_22: # %else20
+; AVX2-NEXT:    testl $2048, %eax # imm = 0x800
+; AVX2-NEXT:    jne .LBB15_23
+; AVX2-NEXT:  .LBB15_24: # %else22
+; AVX2-NEXT:    testl $4096, %eax # imm = 0x1000
+; AVX2-NEXT:    jne .LBB15_25
+; AVX2-NEXT:  .LBB15_26: # %else24
+; AVX2-NEXT:    testl $8192, %eax # imm = 0x2000
+; AVX2-NEXT:    jne .LBB15_27
+; AVX2-NEXT:  .LBB15_28: # %else26
+; AVX2-NEXT:    testl $16384, %eax # imm = 0x4000
+; AVX2-NEXT:    jne .LBB15_29
+; AVX2-NEXT:  .LBB15_30: # %else28
+; AVX2-NEXT:    testl $32768, %eax # imm = 0x8000
+; AVX2-NEXT:    je .LBB15_32
+; AVX2-NEXT:  .LBB15_31: # %cond.store29
+; AVX2-NEXT:    vpextrb $15, %xmm0, 15(%rdi)
+; AVX2-NEXT:  .LBB15_32: # %else30
+; AVX2-NEXT:    testl $65536, %eax # imm = 0x10000
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
+; AVX2-NEXT:    jne .LBB15_33
+; AVX2-NEXT:  # %bb.34: # %else32
+; AVX2-NEXT:    testl $131072, %eax # imm = 0x20000
+; AVX2-NEXT:    jne .LBB15_35
+; AVX2-NEXT:  .LBB15_36: # %else34
+; AVX2-NEXT:    testl $262144, %eax # imm = 0x40000
+; AVX2-NEXT:    jne .LBB15_37
+; AVX2-NEXT:  .LBB15_38: # %else36
+; AVX2-NEXT:    testl $524288, %eax # imm = 0x80000
+; AVX2-NEXT:    jne .LBB15_39
+; AVX2-NEXT:  .LBB15_40: # %else38
+; AVX2-NEXT:    testl $1048576, %eax # imm = 0x100000
+; AVX2-NEXT:    jne .LBB15_41
+; AVX2-NEXT:  .LBB15_42: # %else40
+; AVX2-NEXT:    testl $2097152, %eax # imm = 0x200000
+; AVX2-NEXT:    jne .LBB15_43
+; AVX2-NEXT:  .LBB15_44: # %else42
+; AVX2-NEXT:    testl $4194304, %eax # imm = 0x400000
+; AVX2-NEXT:    jne .LBB15_45
+; AVX2-NEXT:  .LBB15_46: # %else44
+; AVX2-NEXT:    testl $8388608, %eax # imm = 0x800000
+; AVX2-NEXT:    jne .LBB15_47
+; AVX2-NEXT:  .LBB15_48: # %else46
+; AVX2-NEXT:    testl $16777216, %eax # imm = 0x1000000
+; AVX2-NEXT:    jne .LBB15_49
+; AVX2-NEXT:  .LBB15_50: # %else48
+; AVX2-NEXT:    testl $33554432, %eax # imm = 0x2000000
+; AVX2-NEXT:    jne .LBB15_51
+; AVX2-NEXT:  .LBB15_52: # %else50
+; AVX2-NEXT:    testl $67108864, %eax # imm = 0x4000000
+; AVX2-NEXT:    jne .LBB15_53
+; AVX2-NEXT:  .LBB15_54: # %else52
+; AVX2-NEXT:    testl $134217728, %eax # imm = 0x8000000
+; AVX2-NEXT:    jne .LBB15_55
+; AVX2-NEXT:  .LBB15_56: # %else54
+; AVX2-NEXT:    testl $268435456, %eax # imm = 0x10000000
+; AVX2-NEXT:    jne .LBB15_57
+; AVX2-NEXT:  .LBB15_58: # %else56
+; AVX2-NEXT:    testl $536870912, %eax # imm = 0x20000000
+; AVX2-NEXT:    jne .LBB15_59
+; AVX2-NEXT:  .LBB15_60: # %else58
+; AVX2-NEXT:    testl $1073741824, %eax # imm = 0x40000000
+; AVX2-NEXT:    jne .LBB15_61
+; AVX2-NEXT:  .LBB15_62: # %else60
+; AVX2-NEXT:    testl $-2147483648, %eax # imm = 0x80000000
+; AVX2-NEXT:    jne .LBB15_63
+; AVX2-NEXT:  .LBB15_64: # %else62
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+; AVX2-NEXT:  .LBB15_1: # %cond.store
 ; AVX2-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX2-NEXT:  .LBB15_2: # %else
-; AVX2-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX2-NEXT:    vpextrb $1, %xmm1, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $2, %al
 ; AVX2-NEXT:    je .LBB15_4
-; AVX2-NEXT:  # %bb.3: # %cond.store1
+; AVX2-NEXT:  .LBB15_3: # %cond.store1
 ; AVX2-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
-; AVX2-NEXT:  .LBB15_4: # %else2
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm3
-; AVX2-NEXT:    vpextrb $2, %xmm3, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $4, %al
 ; AVX2-NEXT:    je .LBB15_6
-; AVX2-NEXT:  # %bb.5: # %cond.store3
+; AVX2-NEXT:  .LBB15_5: # %cond.store3
 ; AVX2-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
-; AVX2-NEXT:  .LBB15_6: # %else4
-; AVX2-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX2-NEXT:    vpextrb $3, %xmm1, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $8, %al
 ; AVX2-NEXT:    je .LBB15_8
-; AVX2-NEXT:  # %bb.7: # %cond.store5
+; AVX2-NEXT:  .LBB15_7: # %cond.store5
 ; AVX2-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
-; AVX2-NEXT:  .LBB15_8: # %else6
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm3
-; AVX2-NEXT:    vpextrb $4, %xmm3, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $16, %al
 ; AVX2-NEXT:    je .LBB15_10
-; AVX2-NEXT:  # %bb.9: # %cond.store7
+; AVX2-NEXT:  .LBB15_9: # %cond.store7
 ; AVX2-NEXT:    vpextrb $4, %xmm0, 4(%rdi)
-; AVX2-NEXT:  .LBB15_10: # %else8
-; AVX2-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX2-NEXT:    vpextrb $5, %xmm1, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $32, %al
 ; AVX2-NEXT:    je .LBB15_12
-; AVX2-NEXT:  # %bb.11: # %cond.store9
+; AVX2-NEXT:  .LBB15_11: # %cond.store9
 ; AVX2-NEXT:    vpextrb $5, %xmm0, 5(%rdi)
-; AVX2-NEXT:  .LBB15_12: # %else10
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm3
-; AVX2-NEXT:    vpextrb $6, %xmm3, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $64, %al
 ; AVX2-NEXT:    je .LBB15_14
-; AVX2-NEXT:  # %bb.13: # %cond.store11
+; AVX2-NEXT:  .LBB15_13: # %cond.store11
 ; AVX2-NEXT:    vpextrb $6, %xmm0, 6(%rdi)
-; AVX2-NEXT:  .LBB15_14: # %else12
-; AVX2-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX2-NEXT:    vpextrb $7, %xmm1, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $-128, %al
 ; AVX2-NEXT:    je .LBB15_16
-; AVX2-NEXT:  # %bb.15: # %cond.store13
+; AVX2-NEXT:  .LBB15_15: # %cond.store13
 ; AVX2-NEXT:    vpextrb $7, %xmm0, 7(%rdi)
-; AVX2-NEXT:  .LBB15_16: # %else14
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm3
-; AVX2-NEXT:    vpextrb $8, %xmm3, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $256, %eax # imm = 0x100
 ; AVX2-NEXT:    je .LBB15_18
-; AVX2-NEXT:  # %bb.17: # %cond.store15
+; AVX2-NEXT:  .LBB15_17: # %cond.store15
 ; AVX2-NEXT:    vpextrb $8, %xmm0, 8(%rdi)
-; AVX2-NEXT:  .LBB15_18: # %else16
-; AVX2-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX2-NEXT:    vpextrb $9, %xmm1, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $512, %eax # imm = 0x200
 ; AVX2-NEXT:    je .LBB15_20
-; AVX2-NEXT:  # %bb.19: # %cond.store17
+; AVX2-NEXT:  .LBB15_19: # %cond.store17
 ; AVX2-NEXT:    vpextrb $9, %xmm0, 9(%rdi)
-; AVX2-NEXT:  .LBB15_20: # %else18
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm3
-; AVX2-NEXT:    vpextrb $10, %xmm3, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $1024, %eax # imm = 0x400
 ; AVX2-NEXT:    je .LBB15_22
-; AVX2-NEXT:  # %bb.21: # %cond.store19
+; AVX2-NEXT:  .LBB15_21: # %cond.store19
 ; AVX2-NEXT:    vpextrb $10, %xmm0, 10(%rdi)
-; AVX2-NEXT:  .LBB15_22: # %else20
-; AVX2-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX2-NEXT:    vpextrb $11, %xmm1, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $2048, %eax # imm = 0x800
 ; AVX2-NEXT:    je .LBB15_24
-; AVX2-NEXT:  # %bb.23: # %cond.store21
+; AVX2-NEXT:  .LBB15_23: # %cond.store21
 ; AVX2-NEXT:    vpextrb $11, %xmm0, 11(%rdi)
-; AVX2-NEXT:  .LBB15_24: # %else22
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm3
-; AVX2-NEXT:    vpextrb $12, %xmm3, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $4096, %eax # imm = 0x1000
 ; AVX2-NEXT:    je .LBB15_26
-; AVX2-NEXT:  # %bb.25: # %cond.store23
+; AVX2-NEXT:  .LBB15_25: # %cond.store23
 ; AVX2-NEXT:    vpextrb $12, %xmm0, 12(%rdi)
-; AVX2-NEXT:  .LBB15_26: # %else24
-; AVX2-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX2-NEXT:    vpextrb $13, %xmm1, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $8192, %eax # imm = 0x2000
 ; AVX2-NEXT:    je .LBB15_28
-; AVX2-NEXT:  # %bb.27: # %cond.store25
+; AVX2-NEXT:  .LBB15_27: # %cond.store25
 ; AVX2-NEXT:    vpextrb $13, %xmm0, 13(%rdi)
-; AVX2-NEXT:  .LBB15_28: # %else26
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm3
-; AVX2-NEXT:    vpextrb $14, %xmm3, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $16384, %eax # imm = 0x4000
 ; AVX2-NEXT:    je .LBB15_30
-; AVX2-NEXT:  # %bb.29: # %cond.store27
+; AVX2-NEXT:  .LBB15_29: # %cond.store27
 ; AVX2-NEXT:    vpextrb $14, %xmm0, 14(%rdi)
-; AVX2-NEXT:  .LBB15_30: # %else28
-; AVX2-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX2-NEXT:    vpextrb $15, %xmm1, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je .LBB15_32
-; AVX2-NEXT:  # %bb.31: # %cond.store29
-; AVX2-NEXT:    vpextrb $15, %xmm0, 15(%rdi)
-; AVX2-NEXT:  .LBB15_32: # %else30
-; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm1
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpextrb $0, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; AVX2-NEXT:    je .LBB15_34
-; AVX2-NEXT:  # %bb.33: # %cond.store31
+; AVX2-NEXT:    testl $32768, %eax # imm = 0x8000
+; AVX2-NEXT:    jne .LBB15_31
+; AVX2-NEXT:    jmp .LBB15_32
+; AVX2-NEXT:  .LBB15_33: # %cond.store31
 ; AVX2-NEXT:    vpextrb $0, %xmm0, 16(%rdi)
-; AVX2-NEXT:  .LBB15_34: # %else32
-; AVX2-NEXT:    vpextrb $1, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $131072, %eax # imm = 0x20000
 ; AVX2-NEXT:    je .LBB15_36
-; AVX2-NEXT:  # %bb.35: # %cond.store33
+; AVX2-NEXT:  .LBB15_35: # %cond.store33
 ; AVX2-NEXT:    vpextrb $1, %xmm0, 17(%rdi)
-; AVX2-NEXT:  .LBB15_36: # %else34
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpextrb $2, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $262144, %eax # imm = 0x40000
 ; AVX2-NEXT:    je .LBB15_38
-; AVX2-NEXT:  # %bb.37: # %cond.store35
+; AVX2-NEXT:  .LBB15_37: # %cond.store35
 ; AVX2-NEXT:    vpextrb $2, %xmm0, 18(%rdi)
-; AVX2-NEXT:  .LBB15_38: # %else36
-; AVX2-NEXT:    vpextrb $3, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $524288, %eax # imm = 0x80000
 ; AVX2-NEXT:    je .LBB15_40
-; AVX2-NEXT:  # %bb.39: # %cond.store37
+; AVX2-NEXT:  .LBB15_39: # %cond.store37
 ; AVX2-NEXT:    vpextrb $3, %xmm0, 19(%rdi)
-; AVX2-NEXT:  .LBB15_40: # %else38
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $1048576, %eax # imm = 0x100000
 ; AVX2-NEXT:    je .LBB15_42
-; AVX2-NEXT:  # %bb.41: # %cond.store39
+; AVX2-NEXT:  .LBB15_41: # %cond.store39
 ; AVX2-NEXT:    vpextrb $4, %xmm0, 20(%rdi)
-; AVX2-NEXT:  .LBB15_42: # %else40
-; AVX2-NEXT:    vpextrb $5, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $2097152, %eax # imm = 0x200000
 ; AVX2-NEXT:    je .LBB15_44
-; AVX2-NEXT:  # %bb.43: # %cond.store41
+; AVX2-NEXT:  .LBB15_43: # %cond.store41
 ; AVX2-NEXT:    vpextrb $5, %xmm0, 21(%rdi)
-; AVX2-NEXT:  .LBB15_44: # %else42
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpextrb $6, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $4194304, %eax # imm = 0x400000
 ; AVX2-NEXT:    je .LBB15_46
-; AVX2-NEXT:  # %bb.45: # %cond.store43
+; AVX2-NEXT:  .LBB15_45: # %cond.store43
 ; AVX2-NEXT:    vpextrb $6, %xmm0, 22(%rdi)
-; AVX2-NEXT:  .LBB15_46: # %else44
-; AVX2-NEXT:    vpextrb $7, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $8388608, %eax # imm = 0x800000
 ; AVX2-NEXT:    je .LBB15_48
-; AVX2-NEXT:  # %bb.47: # %cond.store45
+; AVX2-NEXT:  .LBB15_47: # %cond.store45
 ; AVX2-NEXT:    vpextrb $7, %xmm0, 23(%rdi)
-; AVX2-NEXT:  .LBB15_48: # %else46
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpextrb $8, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $16777216, %eax # imm = 0x1000000
 ; AVX2-NEXT:    je .LBB15_50
-; AVX2-NEXT:  # %bb.49: # %cond.store47
+; AVX2-NEXT:  .LBB15_49: # %cond.store47
 ; AVX2-NEXT:    vpextrb $8, %xmm0, 24(%rdi)
-; AVX2-NEXT:  .LBB15_50: # %else48
-; AVX2-NEXT:    vpextrb $9, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $33554432, %eax # imm = 0x2000000
 ; AVX2-NEXT:    je .LBB15_52
-; AVX2-NEXT:  # %bb.51: # %cond.store49
+; AVX2-NEXT:  .LBB15_51: # %cond.store49
 ; AVX2-NEXT:    vpextrb $9, %xmm0, 25(%rdi)
-; AVX2-NEXT:  .LBB15_52: # %else50
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpextrb $10, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $67108864, %eax # imm = 0x4000000
 ; AVX2-NEXT:    je .LBB15_54
-; AVX2-NEXT:  # %bb.53: # %cond.store51
+; AVX2-NEXT:  .LBB15_53: # %cond.store51
 ; AVX2-NEXT:    vpextrb $10, %xmm0, 26(%rdi)
-; AVX2-NEXT:  .LBB15_54: # %else52
-; AVX2-NEXT:    vpextrb $11, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $134217728, %eax # imm = 0x8000000
 ; AVX2-NEXT:    je .LBB15_56
-; AVX2-NEXT:  # %bb.55: # %cond.store53
+; AVX2-NEXT:  .LBB15_55: # %cond.store53
 ; AVX2-NEXT:    vpextrb $11, %xmm0, 27(%rdi)
-; AVX2-NEXT:  .LBB15_56: # %else54
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpextrb $12, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $268435456, %eax # imm = 0x10000000
 ; AVX2-NEXT:    je .LBB15_58
-; AVX2-NEXT:  # %bb.57: # %cond.store55
+; AVX2-NEXT:  .LBB15_57: # %cond.store55
 ; AVX2-NEXT:    vpextrb $12, %xmm0, 28(%rdi)
-; AVX2-NEXT:  .LBB15_58: # %else56
-; AVX2-NEXT:    vpextrb $13, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $536870912, %eax # imm = 0x20000000
 ; AVX2-NEXT:    je .LBB15_60
-; AVX2-NEXT:  # %bb.59: # %cond.store57
+; AVX2-NEXT:  .LBB15_59: # %cond.store57
 ; AVX2-NEXT:    vpextrb $13, %xmm0, 29(%rdi)
-; AVX2-NEXT:  .LBB15_60: # %else58
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpextrb $14, %xmm1, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $1073741824, %eax # imm = 0x40000000
 ; AVX2-NEXT:    je .LBB15_62
-; AVX2-NEXT:  # %bb.61: # %cond.store59
+; AVX2-NEXT:  .LBB15_61: # %cond.store59
 ; AVX2-NEXT:    vpextrb $14, %xmm0, 30(%rdi)
-; AVX2-NEXT:  .LBB15_62: # %else60
-; AVX2-NEXT:    vpextrb $15, %xmm1, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $-2147483648, %eax # imm = 0x80000000
 ; AVX2-NEXT:    je .LBB15_64
-; AVX2-NEXT:  # %bb.63: # %cond.store61
+; AVX2-NEXT:  .LBB15_63: # %cond.store61
 ; AVX2-NEXT:    vpextrb $15, %xmm0, 31(%rdi)
-; AVX2-NEXT:  .LBB15_64: # %else62
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: truncstore_v32i16_v32i8:
 ; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
 ; AVX512F-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX512F-NEXT:    vpcmpeqb %xmm3, %xmm2, %xmm4
-; AVX512F-NEXT:    vpternlogq $15, %zmm4, %zmm4, %zmm4
-; AVX512F-NEXT:    vpmovsxbd %xmm4, %zmm4
-; AVX512F-NEXT:    vptestmd %zmm4, %zmm4, %k0
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512F-NEXT:    vpminuw %ymm4, %ymm1, %ymm1
-; AVX512F-NEXT:    vpminuw %ymm4, %ymm0, %ymm0
+; AVX512F-NEXT:    vpcmpeqb %ymm3, %ymm1, %ymm1
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; AVX512F-NEXT:    vpminuw %ymm3, %ymm2, %ymm2
+; AVX512F-NEXT:    vpminuw %ymm3, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
 ; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
-; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
-; AVX512F-NEXT:    vpmovdb %zmm1, %xmm1
-; AVX512F-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB15_2
-; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
+; AVX512F-NEXT:    vpmovdb %zmm2, %xmm2
+; AVX512F-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vpmovmskb %ymm1, %eax
+; AVX512F-NEXT:    notl %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    jne .LBB15_1
+; AVX512F-NEXT:  # %bb.2: # %else
+; AVX512F-NEXT:    testb $2, %al
+; AVX512F-NEXT:    jne .LBB15_3
+; AVX512F-NEXT:  .LBB15_4: # %else2
+; AVX512F-NEXT:    testb $4, %al
+; AVX512F-NEXT:    jne .LBB15_5
+; AVX512F-NEXT:  .LBB15_6: # %else4
+; AVX512F-NEXT:    testb $8, %al
+; AVX512F-NEXT:    jne .LBB15_7
+; AVX512F-NEXT:  .LBB15_8: # %else6
+; AVX512F-NEXT:    testb $16, %al
+; AVX512F-NEXT:    jne .LBB15_9
+; AVX512F-NEXT:  .LBB15_10: # %else8
+; AVX512F-NEXT:    testb $32, %al
+; AVX512F-NEXT:    jne .LBB15_11
+; AVX512F-NEXT:  .LBB15_12: # %else10
+; AVX512F-NEXT:    testb $64, %al
+; AVX512F-NEXT:    jne .LBB15_13
+; AVX512F-NEXT:  .LBB15_14: # %else12
+; AVX512F-NEXT:    testb $-128, %al
+; AVX512F-NEXT:    jne .LBB15_15
+; AVX512F-NEXT:  .LBB15_16: # %else14
+; AVX512F-NEXT:    testl $256, %eax # imm = 0x100
+; AVX512F-NEXT:    jne .LBB15_17
+; AVX512F-NEXT:  .LBB15_18: # %else16
+; AVX512F-NEXT:    testl $512, %eax # imm = 0x200
+; AVX512F-NEXT:    jne .LBB15_19
+; AVX512F-NEXT:  .LBB15_20: # %else18
+; AVX512F-NEXT:    testl $1024, %eax # imm = 0x400
+; AVX512F-NEXT:    jne .LBB15_21
+; AVX512F-NEXT:  .LBB15_22: # %else20
+; AVX512F-NEXT:    testl $2048, %eax # imm = 0x800
+; AVX512F-NEXT:    jne .LBB15_23
+; AVX512F-NEXT:  .LBB15_24: # %else22
+; AVX512F-NEXT:    testl $4096, %eax # imm = 0x1000
+; AVX512F-NEXT:    jne .LBB15_25
+; AVX512F-NEXT:  .LBB15_26: # %else24
+; AVX512F-NEXT:    testl $8192, %eax # imm = 0x2000
+; AVX512F-NEXT:    jne .LBB15_27
+; AVX512F-NEXT:  .LBB15_28: # %else26
+; AVX512F-NEXT:    testl $16384, %eax # imm = 0x4000
+; AVX512F-NEXT:    jne .LBB15_29
+; AVX512F-NEXT:  .LBB15_30: # %else28
+; AVX512F-NEXT:    testl $32768, %eax # imm = 0x8000
+; AVX512F-NEXT:    je .LBB15_32
+; AVX512F-NEXT:  .LBB15_31: # %cond.store29
+; AVX512F-NEXT:    vpextrb $15, %xmm0, 15(%rdi)
+; AVX512F-NEXT:  .LBB15_32: # %else30
+; AVX512F-NEXT:    testl $65536, %eax # imm = 0x10000
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm0
+; AVX512F-NEXT:    jne .LBB15_33
+; AVX512F-NEXT:  # %bb.34: # %else32
+; AVX512F-NEXT:    testl $131072, %eax # imm = 0x20000
+; AVX512F-NEXT:    jne .LBB15_35
+; AVX512F-NEXT:  .LBB15_36: # %else34
+; AVX512F-NEXT:    testl $262144, %eax # imm = 0x40000
+; AVX512F-NEXT:    jne .LBB15_37
+; AVX512F-NEXT:  .LBB15_38: # %else36
+; AVX512F-NEXT:    testl $524288, %eax # imm = 0x80000
+; AVX512F-NEXT:    jne .LBB15_39
+; AVX512F-NEXT:  .LBB15_40: # %else38
+; AVX512F-NEXT:    testl $1048576, %eax # imm = 0x100000
+; AVX512F-NEXT:    jne .LBB15_41
+; AVX512F-NEXT:  .LBB15_42: # %else40
+; AVX512F-NEXT:    testl $2097152, %eax # imm = 0x200000
+; AVX512F-NEXT:    jne .LBB15_43
+; AVX512F-NEXT:  .LBB15_44: # %else42
+; AVX512F-NEXT:    testl $4194304, %eax # imm = 0x400000
+; AVX512F-NEXT:    jne .LBB15_45
+; AVX512F-NEXT:  .LBB15_46: # %else44
+; AVX512F-NEXT:    testl $8388608, %eax # imm = 0x800000
+; AVX512F-NEXT:    jne .LBB15_47
+; AVX512F-NEXT:  .LBB15_48: # %else46
+; AVX512F-NEXT:    testl $16777216, %eax # imm = 0x1000000
+; AVX512F-NEXT:    jne .LBB15_49
+; AVX512F-NEXT:  .LBB15_50: # %else48
+; AVX512F-NEXT:    testl $33554432, %eax # imm = 0x2000000
+; AVX512F-NEXT:    jne .LBB15_51
+; AVX512F-NEXT:  .LBB15_52: # %else50
+; AVX512F-NEXT:    testl $67108864, %eax # imm = 0x4000000
+; AVX512F-NEXT:    jne .LBB15_53
+; AVX512F-NEXT:  .LBB15_54: # %else52
+; AVX512F-NEXT:    testl $134217728, %eax # imm = 0x8000000
+; AVX512F-NEXT:    jne .LBB15_55
+; AVX512F-NEXT:  .LBB15_56: # %else54
+; AVX512F-NEXT:    testl $268435456, %eax # imm = 0x10000000
+; AVX512F-NEXT:    jne .LBB15_57
+; AVX512F-NEXT:  .LBB15_58: # %else56
+; AVX512F-NEXT:    testl $536870912, %eax # imm = 0x20000000
+; AVX512F-NEXT:    jne .LBB15_59
+; AVX512F-NEXT:  .LBB15_60: # %else58
+; AVX512F-NEXT:    testl $1073741824, %eax # imm = 0x40000000
+; AVX512F-NEXT:    jne .LBB15_61
+; AVX512F-NEXT:  .LBB15_62: # %else60
+; AVX512F-NEXT:    testl $-2147483648, %eax # imm = 0x80000000
+; AVX512F-NEXT:    jne .LBB15_63
+; AVX512F-NEXT:  .LBB15_64: # %else62
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+; AVX512F-NEXT:  .LBB15_1: # %cond.store
 ; AVX512F-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX512F-NEXT:  .LBB15_2: # %else
-; AVX512F-NEXT:    vpcmpeqb %xmm3, %xmm2, %xmm1
-; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
-; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $2, %al
 ; AVX512F-NEXT:    je .LBB15_4
-; AVX512F-NEXT:  # %bb.3: # %cond.store1
+; AVX512F-NEXT:  .LBB15_3: # %cond.store1
 ; AVX512F-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
-; AVX512F-NEXT:  .LBB15_4: # %else2
-; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm3, %zmm3, %zmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $4, %al
 ; AVX512F-NEXT:    je .LBB15_6
-; AVX512F-NEXT:  # %bb.5: # %cond.store3
+; AVX512F-NEXT:  .LBB15_5: # %cond.store3
 ; AVX512F-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
-; AVX512F-NEXT:  .LBB15_6: # %else4
-; AVX512F-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
-; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $8, %al
 ; AVX512F-NEXT:    je .LBB15_8
-; AVX512F-NEXT:  # %bb.7: # %cond.store5
+; AVX512F-NEXT:  .LBB15_7: # %cond.store5
 ; AVX512F-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
-; AVX512F-NEXT:  .LBB15_8: # %else6
-; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm3, %zmm3, %zmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $4, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $16, %al
 ; AVX512F-NEXT:    je .LBB15_10
-; AVX512F-NEXT:  # %bb.9: # %cond.store7
+; AVX512F-NEXT:  .LBB15_9: # %cond.store7
 ; AVX512F-NEXT:    vpextrb $4, %xmm0, 4(%rdi)
-; AVX512F-NEXT:  .LBB15_10: # %else8
-; AVX512F-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
-; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $5, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $32, %al
 ; AVX512F-NEXT:    je .LBB15_12
-; AVX512F-NEXT:  # %bb.11: # %cond.store9
+; AVX512F-NEXT:  .LBB15_11: # %cond.store9
 ; AVX512F-NEXT:    vpextrb $5, %xmm0, 5(%rdi)
-; AVX512F-NEXT:  .LBB15_12: # %else10
-; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm3, %zmm3, %zmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $6, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $64, %al
 ; AVX512F-NEXT:    je .LBB15_14
-; AVX512F-NEXT:  # %bb.13: # %cond.store11
+; AVX512F-NEXT:  .LBB15_13: # %cond.store11
 ; AVX512F-NEXT:    vpextrb $6, %xmm0, 6(%rdi)
-; AVX512F-NEXT:  .LBB15_14: # %else12
-; AVX512F-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
-; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $7, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $-128, %al
 ; AVX512F-NEXT:    je .LBB15_16
-; AVX512F-NEXT:  # %bb.15: # %cond.store13
+; AVX512F-NEXT:  .LBB15_15: # %cond.store13
 ; AVX512F-NEXT:    vpextrb $7, %xmm0, 7(%rdi)
-; AVX512F-NEXT:  .LBB15_16: # %else14
-; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm3, %zmm3, %zmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $8, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $256, %eax # imm = 0x100
 ; AVX512F-NEXT:    je .LBB15_18
-; AVX512F-NEXT:  # %bb.17: # %cond.store15
+; AVX512F-NEXT:  .LBB15_17: # %cond.store15
 ; AVX512F-NEXT:    vpextrb $8, %xmm0, 8(%rdi)
-; AVX512F-NEXT:  .LBB15_18: # %else16
-; AVX512F-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
-; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $9, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $512, %eax # imm = 0x200
 ; AVX512F-NEXT:    je .LBB15_20
-; AVX512F-NEXT:  # %bb.19: # %cond.store17
+; AVX512F-NEXT:  .LBB15_19: # %cond.store17
 ; AVX512F-NEXT:    vpextrb $9, %xmm0, 9(%rdi)
-; AVX512F-NEXT:  .LBB15_20: # %else18
-; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm3, %zmm3, %zmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $10, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB15_22
-; AVX512F-NEXT:  # %bb.21: # %cond.store19
-; AVX512F-NEXT:    vpextrb $10, %xmm0, 10(%rdi)
-; AVX512F-NEXT:  .LBB15_22: # %else20
-; AVX512F-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
-; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $11, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB15_24
-; AVX512F-NEXT:  # %bb.23: # %cond.store21
-; AVX512F-NEXT:    vpextrb $11, %xmm0, 11(%rdi)
-; AVX512F-NEXT:  .LBB15_24: # %else22
-; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm3, %zmm3, %zmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $12, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB15_26
-; AVX512F-NEXT:  # %bb.25: # %cond.store23
-; AVX512F-NEXT:    vpextrb $12, %xmm0, 12(%rdi)
-; AVX512F-NEXT:  .LBB15_26: # %else24
-; AVX512F-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
-; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $13, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB15_28
-; AVX512F-NEXT:  # %bb.27: # %cond.store25
-; AVX512F-NEXT:    vpextrb $13, %xmm0, 13(%rdi)
-; AVX512F-NEXT:  .LBB15_28: # %else26
-; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm3, %zmm3, %zmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $14, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB15_30
-; AVX512F-NEXT:  # %bb.29: # %cond.store27
-; AVX512F-NEXT:    vpextrb $14, %xmm0, 14(%rdi)
-; AVX512F-NEXT:  .LBB15_30: # %else28
-; AVX512F-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
-; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $15, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB15_32
-; AVX512F-NEXT:  # %bb.31: # %cond.store29
-; AVX512F-NEXT:    vpextrb $15, %xmm0, 15(%rdi)
-; AVX512F-NEXT:  .LBB15_32: # %else30
-; AVX512F-NEXT:    vextracti128 $1, %ymm2, %xmm1
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; AVX512F-NEXT:    je .LBB15_34
-; AVX512F-NEXT:  # %bb.33: # %cond.store31
+; AVX512F-NEXT:    testl $1024, %eax # imm = 0x400
+; AVX512F-NEXT:    je .LBB15_22
+; AVX512F-NEXT:  .LBB15_21: # %cond.store19
+; AVX512F-NEXT:    vpextrb $10, %xmm0, 10(%rdi)
+; AVX512F-NEXT:    testl $2048, %eax # imm = 0x800
+; AVX512F-NEXT:    je .LBB15_24
+; AVX512F-NEXT:  .LBB15_23: # %cond.store21
+; AVX512F-NEXT:    vpextrb $11, %xmm0, 11(%rdi)
+; AVX512F-NEXT:    testl $4096, %eax # imm = 0x1000
+; AVX512F-NEXT:    je .LBB15_26
+; AVX512F-NEXT:  .LBB15_25: # %cond.store23
+; AVX512F-NEXT:    vpextrb $12, %xmm0, 12(%rdi)
+; AVX512F-NEXT:    testl $8192, %eax # imm = 0x2000
+; AVX512F-NEXT:    je .LBB15_28
+; AVX512F-NEXT:  .LBB15_27: # %cond.store25
+; AVX512F-NEXT:    vpextrb $13, %xmm0, 13(%rdi)
+; AVX512F-NEXT:    testl $16384, %eax # imm = 0x4000
+; AVX512F-NEXT:    je .LBB15_30
+; AVX512F-NEXT:  .LBB15_29: # %cond.store27
+; AVX512F-NEXT:    vpextrb $14, %xmm0, 14(%rdi)
+; AVX512F-NEXT:    testl $32768, %eax # imm = 0x8000
+; AVX512F-NEXT:    jne .LBB15_31
+; AVX512F-NEXT:    jmp .LBB15_32
+; AVX512F-NEXT:  .LBB15_33: # %cond.store31
 ; AVX512F-NEXT:    vpextrb $0, %xmm0, 16(%rdi)
-; AVX512F-NEXT:  .LBB15_34: # %else32
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $131072, %eax # imm = 0x20000
 ; AVX512F-NEXT:    je .LBB15_36
-; AVX512F-NEXT:  # %bb.35: # %cond.store33
+; AVX512F-NEXT:  .LBB15_35: # %cond.store33
 ; AVX512F-NEXT:    vpextrb $1, %xmm0, 17(%rdi)
-; AVX512F-NEXT:  .LBB15_36: # %else34
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $262144, %eax # imm = 0x40000
 ; AVX512F-NEXT:    je .LBB15_38
-; AVX512F-NEXT:  # %bb.37: # %cond.store35
+; AVX512F-NEXT:  .LBB15_37: # %cond.store35
 ; AVX512F-NEXT:    vpextrb $2, %xmm0, 18(%rdi)
-; AVX512F-NEXT:  .LBB15_38: # %else36
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $524288, %eax # imm = 0x80000
 ; AVX512F-NEXT:    je .LBB15_40
-; AVX512F-NEXT:  # %bb.39: # %cond.store37
+; AVX512F-NEXT:  .LBB15_39: # %cond.store37
 ; AVX512F-NEXT:    vpextrb $3, %xmm0, 19(%rdi)
-; AVX512F-NEXT:  .LBB15_40: # %else38
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $4, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $1048576, %eax # imm = 0x100000
 ; AVX512F-NEXT:    je .LBB15_42
-; AVX512F-NEXT:  # %bb.41: # %cond.store39
+; AVX512F-NEXT:  .LBB15_41: # %cond.store39
 ; AVX512F-NEXT:    vpextrb $4, %xmm0, 20(%rdi)
-; AVX512F-NEXT:  .LBB15_42: # %else40
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $5, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $2097152, %eax # imm = 0x200000
 ; AVX512F-NEXT:    je .LBB15_44
-; AVX512F-NEXT:  # %bb.43: # %cond.store41
+; AVX512F-NEXT:  .LBB15_43: # %cond.store41
 ; AVX512F-NEXT:    vpextrb $5, %xmm0, 21(%rdi)
-; AVX512F-NEXT:  .LBB15_44: # %else42
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $6, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $4194304, %eax # imm = 0x400000
 ; AVX512F-NEXT:    je .LBB15_46
-; AVX512F-NEXT:  # %bb.45: # %cond.store43
+; AVX512F-NEXT:  .LBB15_45: # %cond.store43
 ; AVX512F-NEXT:    vpextrb $6, %xmm0, 22(%rdi)
-; AVX512F-NEXT:  .LBB15_46: # %else44
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $7, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $8388608, %eax # imm = 0x800000
 ; AVX512F-NEXT:    je .LBB15_48
-; AVX512F-NEXT:  # %bb.47: # %cond.store45
+; AVX512F-NEXT:  .LBB15_47: # %cond.store45
 ; AVX512F-NEXT:    vpextrb $7, %xmm0, 23(%rdi)
-; AVX512F-NEXT:  .LBB15_48: # %else46
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $8, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $16777216, %eax # imm = 0x1000000
 ; AVX512F-NEXT:    je .LBB15_50
-; AVX512F-NEXT:  # %bb.49: # %cond.store47
+; AVX512F-NEXT:  .LBB15_49: # %cond.store47
 ; AVX512F-NEXT:    vpextrb $8, %xmm0, 24(%rdi)
-; AVX512F-NEXT:  .LBB15_50: # %else48
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $9, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $33554432, %eax # imm = 0x2000000
 ; AVX512F-NEXT:    je .LBB15_52
-; AVX512F-NEXT:  # %bb.51: # %cond.store49
+; AVX512F-NEXT:  .LBB15_51: # %cond.store49
 ; AVX512F-NEXT:    vpextrb $9, %xmm0, 25(%rdi)
-; AVX512F-NEXT:  .LBB15_52: # %else50
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $10, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $67108864, %eax # imm = 0x4000000
 ; AVX512F-NEXT:    je .LBB15_54
-; AVX512F-NEXT:  # %bb.53: # %cond.store51
+; AVX512F-NEXT:  .LBB15_53: # %cond.store51
 ; AVX512F-NEXT:    vpextrb $10, %xmm0, 26(%rdi)
-; AVX512F-NEXT:  .LBB15_54: # %else52
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $11, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $134217728, %eax # imm = 0x8000000
 ; AVX512F-NEXT:    je .LBB15_56
-; AVX512F-NEXT:  # %bb.55: # %cond.store53
+; AVX512F-NEXT:  .LBB15_55: # %cond.store53
 ; AVX512F-NEXT:    vpextrb $11, %xmm0, 27(%rdi)
-; AVX512F-NEXT:  .LBB15_56: # %else54
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $12, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $268435456, %eax # imm = 0x10000000
 ; AVX512F-NEXT:    je .LBB15_58
-; AVX512F-NEXT:  # %bb.57: # %cond.store55
+; AVX512F-NEXT:  .LBB15_57: # %cond.store55
 ; AVX512F-NEXT:    vpextrb $12, %xmm0, 28(%rdi)
-; AVX512F-NEXT:  .LBB15_58: # %else56
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $13, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $536870912, %eax # imm = 0x20000000
 ; AVX512F-NEXT:    je .LBB15_60
-; AVX512F-NEXT:  # %bb.59: # %cond.store57
+; AVX512F-NEXT:  .LBB15_59: # %cond.store57
 ; AVX512F-NEXT:    vpextrb $13, %xmm0, 29(%rdi)
-; AVX512F-NEXT:  .LBB15_60: # %else58
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm1
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm2
-; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $14, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $1073741824, %eax # imm = 0x40000000
 ; AVX512F-NEXT:    je .LBB15_62
-; AVX512F-NEXT:  # %bb.61: # %cond.store59
+; AVX512F-NEXT:  .LBB15_61: # %cond.store59
 ; AVX512F-NEXT:    vpextrb $14, %xmm0, 30(%rdi)
-; AVX512F-NEXT:  .LBB15_62: # %else60
-; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
-; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $15, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $-2147483648, %eax # imm = 0x80000000
 ; AVX512F-NEXT:    je .LBB15_64
-; AVX512F-NEXT:  # %bb.63: # %cond.store61
+; AVX512F-NEXT:  .LBB15_63: # %cond.store61
 ; AVX512F-NEXT:    vpextrb $15, %xmm0, 31(%rdi)
-; AVX512F-NEXT:  .LBB15_64: # %else62
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
@@ -7107,7 +6428,6 @@ define void @truncstore_v16i16_v16i8(<16 x i16> %x, <16 x i8>* %p, <16 x i8> %ma
 ; SSE2-LABEL: truncstore_v16i16_v16i8:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    pcmpeqb %xmm2, %xmm3
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [32768,32768,32768,32768,32768,32768,32768,32768]
 ; SSE2-NEXT:    pxor %xmm4, %xmm1
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [33023,33023,33023,33023,33023,33023,33023,33023]
@@ -7117,761 +6437,615 @@ define void @truncstore_v16i16_v16i8(<16 x i16> %x, <16 x i8>* %p, <16 x i8> %ma
 ; SSE2-NEXT:    pminsw %xmm5, %xmm0
 ; SSE2-NEXT:    pxor %xmm4, %xmm0
 ; SSE2-NEXT:    packuswb %xmm1, %xmm0
-; SSE2-NEXT:    movd %xmm3, %ecx
-; SSE2-NEXT:    movl %ecx, %eax
-; SSE2-NEXT:    notb %al
+; SSE2-NEXT:    pcmpeqb %xmm2, %xmm3
+; SSE2-NEXT:    pmovmskb %xmm3, %eax
+; SSE2-NEXT:    xorl $65535, %eax # imm = 0xFFFF
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    je .LBB16_2
-; SSE2-NEXT:  # %bb.1: # %cond.store
-; SSE2-NEXT:    movb %al, (%rdi)
-; SSE2-NEXT:  .LBB16_2: # %else
-; SSE2-NEXT:    shrl $8, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    je .LBB16_4
-; SSE2-NEXT:  # %bb.3: # %cond.store1
-; SSE2-NEXT:    movb %ah, 1(%rdi)
+; SSE2-NEXT:    movd %xmm0, %ecx
+; SSE2-NEXT:    jne .LBB16_1
+; SSE2-NEXT:  # %bb.2: # %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne .LBB16_3
 ; SSE2-NEXT:  .LBB16_4: # %else2
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqb %xmm2, %xmm1
-; SSE2-NEXT:    movd %xmm1, %ecx
-; SSE2-NEXT:    movl %ecx, %edx
-; SSE2-NEXT:    shrl $16, %edx
-; SSE2-NEXT:    notb %dl
-; SSE2-NEXT:    testb $1, %dl
-; SSE2-NEXT:    je .LBB16_6
-; SSE2-NEXT:  # %bb.5: # %cond.store3
-; SSE2-NEXT:    movl %eax, %edx
-; SSE2-NEXT:    shrl $16, %edx
-; SSE2-NEXT:    movb %dl, 2(%rdi)
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne .LBB16_5
 ; SSE2-NEXT:  .LBB16_6: # %else4
-; SSE2-NEXT:    shrl $24, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je .LBB16_8
-; SSE2-NEXT:  # %bb.7: # %cond.store5
-; SSE2-NEXT:    shrl $24, %eax
-; SSE2-NEXT:    movb %al, 3(%rdi)
+; SSE2-NEXT:  .LBB16_7: # %cond.store5
+; SSE2-NEXT:    shrl $24, %ecx
+; SSE2-NEXT:    movb %cl, 3(%rdi)
 ; SSE2-NEXT:  .LBB16_8: # %else6
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqb %xmm2, %xmm1
-; SSE2-NEXT:    pextrw $2, %xmm1, %eax
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testb $16, %al
 ; SSE2-NEXT:    pextrw $2, %xmm0, %ecx
 ; SSE2-NEXT:    je .LBB16_10
 ; SSE2-NEXT:  # %bb.9: # %cond.store7
 ; SSE2-NEXT:    movb %cl, 4(%rdi)
 ; SSE2-NEXT:  .LBB16_10: # %else8
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    notb %al
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $32, %al
 ; SSE2-NEXT:    je .LBB16_12
 ; SSE2-NEXT:  # %bb.11: # %cond.store9
 ; SSE2-NEXT:    movb %ch, 5(%rdi)
 ; SSE2-NEXT:  .LBB16_12: # %else10
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqb %xmm2, %xmm1
-; SSE2-NEXT:    pextrw $3, %xmm1, %eax
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testb $64, %al
 ; SSE2-NEXT:    pextrw $3, %xmm0, %ecx
 ; SSE2-NEXT:    je .LBB16_14
 ; SSE2-NEXT:  # %bb.13: # %cond.store11
 ; SSE2-NEXT:    movb %cl, 6(%rdi)
 ; SSE2-NEXT:  .LBB16_14: # %else12
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    notb %al
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $-128, %al
 ; SSE2-NEXT:    je .LBB16_16
 ; SSE2-NEXT:  # %bb.15: # %cond.store13
 ; SSE2-NEXT:    movb %ch, 7(%rdi)
 ; SSE2-NEXT:  .LBB16_16: # %else14
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqb %xmm2, %xmm1
-; SSE2-NEXT:    pextrw $4, %xmm1, %eax
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $256, %eax # imm = 0x100
 ; SSE2-NEXT:    pextrw $4, %xmm0, %ecx
 ; SSE2-NEXT:    je .LBB16_18
 ; SSE2-NEXT:  # %bb.17: # %cond.store15
 ; SSE2-NEXT:    movb %cl, 8(%rdi)
 ; SSE2-NEXT:  .LBB16_18: # %else16
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    notb %al
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $512, %eax # imm = 0x200
 ; SSE2-NEXT:    je .LBB16_20
 ; SSE2-NEXT:  # %bb.19: # %cond.store17
 ; SSE2-NEXT:    movb %ch, 9(%rdi)
 ; SSE2-NEXT:  .LBB16_20: # %else18
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqb %xmm2, %xmm1
-; SSE2-NEXT:    pextrw $5, %xmm1, %eax
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $1024, %eax # imm = 0x400
 ; SSE2-NEXT:    pextrw $5, %xmm0, %ecx
 ; SSE2-NEXT:    je .LBB16_22
 ; SSE2-NEXT:  # %bb.21: # %cond.store19
 ; SSE2-NEXT:    movb %cl, 10(%rdi)
 ; SSE2-NEXT:  .LBB16_22: # %else20
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    notb %al
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $2048, %eax # imm = 0x800
 ; SSE2-NEXT:    je .LBB16_24
 ; SSE2-NEXT:  # %bb.23: # %cond.store21
 ; SSE2-NEXT:    movb %ch, 11(%rdi)
 ; SSE2-NEXT:  .LBB16_24: # %else22
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqb %xmm2, %xmm1
-; SSE2-NEXT:    pextrw $6, %xmm1, %eax
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $4096, %eax # imm = 0x1000
 ; SSE2-NEXT:    pextrw $6, %xmm0, %ecx
 ; SSE2-NEXT:    je .LBB16_26
 ; SSE2-NEXT:  # %bb.25: # %cond.store23
 ; SSE2-NEXT:    movb %cl, 12(%rdi)
 ; SSE2-NEXT:  .LBB16_26: # %else24
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    notb %al
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $8192, %eax # imm = 0x2000
 ; SSE2-NEXT:    je .LBB16_28
 ; SSE2-NEXT:  # %bb.27: # %cond.store25
 ; SSE2-NEXT:    movb %ch, 13(%rdi)
 ; SSE2-NEXT:  .LBB16_28: # %else26
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqb %xmm1, %xmm2
-; SSE2-NEXT:    pextrw $7, %xmm2, %eax
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    notb %cl
-; SSE2-NEXT:    testb $1, %cl
+; SSE2-NEXT:    testl $16384, %eax # imm = 0x4000
 ; SSE2-NEXT:    pextrw $7, %xmm0, %ecx
-; SSE2-NEXT:    je .LBB16_30
-; SSE2-NEXT:  # %bb.29: # %cond.store27
+; SSE2-NEXT:    jne .LBB16_29
+; SSE2-NEXT:  # %bb.30: # %else28
+; SSE2-NEXT:    testl $32768, %eax # imm = 0x8000
+; SSE2-NEXT:    jne .LBB16_31
+; SSE2-NEXT:  .LBB16_32: # %else30
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  .LBB16_1: # %cond.store
+; SSE2-NEXT:    movb %cl, (%rdi)
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    je .LBB16_4
+; SSE2-NEXT:  .LBB16_3: # %cond.store1
+; SSE2-NEXT:    movb %ch, 1(%rdi)
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    je .LBB16_6
+; SSE2-NEXT:  .LBB16_5: # %cond.store3
+; SSE2-NEXT:    movl %ecx, %edx
+; SSE2-NEXT:    shrl $16, %edx
+; SSE2-NEXT:    movb %dl, 2(%rdi)
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne .LBB16_7
+; SSE2-NEXT:    jmp .LBB16_8
+; SSE2-NEXT:  .LBB16_29: # %cond.store27
 ; SSE2-NEXT:    movb %cl, 14(%rdi)
-; SSE2-NEXT:  .LBB16_30: # %else28
-; SSE2-NEXT:    shrl $8, %eax
-; SSE2-NEXT:    notb %al
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testl $32768, %eax # imm = 0x8000
 ; SSE2-NEXT:    je .LBB16_32
-; SSE2-NEXT:  # %bb.31: # %cond.store29
+; SSE2-NEXT:  .LBB16_31: # %cond.store29
 ; SSE2-NEXT:    movb %ch, 15(%rdi)
-; SSE2-NEXT:  .LBB16_32: # %else30
 ; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: truncstore_v16i16_v16i8:
 ; SSE4:       # %bb.0:
 ; SSE4-NEXT:    pxor %xmm3, %xmm3
-; SSE4-NEXT:    pcmpeqb %xmm2, %xmm3
 ; SSE4-NEXT:    movdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
 ; SSE4-NEXT:    pminuw %xmm4, %xmm1
 ; SSE4-NEXT:    pminuw %xmm4, %xmm0
 ; SSE4-NEXT:    packuswb %xmm1, %xmm0
-; SSE4-NEXT:    pextrb $0, %xmm3, %eax
-; SSE4-NEXT:    notb %al
+; SSE4-NEXT:    pcmpeqb %xmm2, %xmm3
+; SSE4-NEXT:    pmovmskb %xmm3, %eax
+; SSE4-NEXT:    xorl $65535, %eax # imm = 0xFFFF
 ; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB16_2
-; SSE4-NEXT:  # %bb.1: # %cond.store
+; SSE4-NEXT:    jne .LBB16_1
+; SSE4-NEXT:  # %bb.2: # %else
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    jne .LBB16_3
+; SSE4-NEXT:  .LBB16_4: # %else2
+; SSE4-NEXT:    testb $4, %al
+; SSE4-NEXT:    jne .LBB16_5
+; SSE4-NEXT:  .LBB16_6: # %else4
+; SSE4-NEXT:    testb $8, %al
+; SSE4-NEXT:    jne .LBB16_7
+; SSE4-NEXT:  .LBB16_8: # %else6
+; SSE4-NEXT:    testb $16, %al
+; SSE4-NEXT:    jne .LBB16_9
+; SSE4-NEXT:  .LBB16_10: # %else8
+; SSE4-NEXT:    testb $32, %al
+; SSE4-NEXT:    jne .LBB16_11
+; SSE4-NEXT:  .LBB16_12: # %else10
+; SSE4-NEXT:    testb $64, %al
+; SSE4-NEXT:    jne .LBB16_13
+; SSE4-NEXT:  .LBB16_14: # %else12
+; SSE4-NEXT:    testb $-128, %al
+; SSE4-NEXT:    jne .LBB16_15
+; SSE4-NEXT:  .LBB16_16: # %else14
+; SSE4-NEXT:    testl $256, %eax # imm = 0x100
+; SSE4-NEXT:    jne .LBB16_17
+; SSE4-NEXT:  .LBB16_18: # %else16
+; SSE4-NEXT:    testl $512, %eax # imm = 0x200
+; SSE4-NEXT:    jne .LBB16_19
+; SSE4-NEXT:  .LBB16_20: # %else18
+; SSE4-NEXT:    testl $1024, %eax # imm = 0x400
+; SSE4-NEXT:    jne .LBB16_21
+; SSE4-NEXT:  .LBB16_22: # %else20
+; SSE4-NEXT:    testl $2048, %eax # imm = 0x800
+; SSE4-NEXT:    jne .LBB16_23
+; SSE4-NEXT:  .LBB16_24: # %else22
+; SSE4-NEXT:    testl $4096, %eax # imm = 0x1000
+; SSE4-NEXT:    jne .LBB16_25
+; SSE4-NEXT:  .LBB16_26: # %else24
+; SSE4-NEXT:    testl $8192, %eax # imm = 0x2000
+; SSE4-NEXT:    jne .LBB16_27
+; SSE4-NEXT:  .LBB16_28: # %else26
+; SSE4-NEXT:    testl $16384, %eax # imm = 0x4000
+; SSE4-NEXT:    jne .LBB16_29
+; SSE4-NEXT:  .LBB16_30: # %else28
+; SSE4-NEXT:    testl $32768, %eax # imm = 0x8000
+; SSE4-NEXT:    jne .LBB16_31
+; SSE4-NEXT:  .LBB16_32: # %else30
+; SSE4-NEXT:    retq
+; SSE4-NEXT:  .LBB16_1: # %cond.store
 ; SSE4-NEXT:    pextrb $0, %xmm0, (%rdi)
-; SSE4-NEXT:  .LBB16_2: # %else
-; SSE4-NEXT:    pextrb $1, %xmm3, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $2, %al
 ; SSE4-NEXT:    je .LBB16_4
-; SSE4-NEXT:  # %bb.3: # %cond.store1
+; SSE4-NEXT:  .LBB16_3: # %cond.store1
 ; SSE4-NEXT:    pextrb $1, %xmm0, 1(%rdi)
-; SSE4-NEXT:  .LBB16_4: # %else2
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqb %xmm2, %xmm1
-; SSE4-NEXT:    pextrb $2, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $4, %al
 ; SSE4-NEXT:    je .LBB16_6
-; SSE4-NEXT:  # %bb.5: # %cond.store3
+; SSE4-NEXT:  .LBB16_5: # %cond.store3
 ; SSE4-NEXT:    pextrb $2, %xmm0, 2(%rdi)
-; SSE4-NEXT:  .LBB16_6: # %else4
-; SSE4-NEXT:    pextrb $3, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $8, %al
 ; SSE4-NEXT:    je .LBB16_8
-; SSE4-NEXT:  # %bb.7: # %cond.store5
+; SSE4-NEXT:  .LBB16_7: # %cond.store5
 ; SSE4-NEXT:    pextrb $3, %xmm0, 3(%rdi)
-; SSE4-NEXT:  .LBB16_8: # %else6
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqb %xmm2, %xmm1
-; SSE4-NEXT:    pextrb $4, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $16, %al
 ; SSE4-NEXT:    je .LBB16_10
-; SSE4-NEXT:  # %bb.9: # %cond.store7
+; SSE4-NEXT:  .LBB16_9: # %cond.store7
 ; SSE4-NEXT:    pextrb $4, %xmm0, 4(%rdi)
-; SSE4-NEXT:  .LBB16_10: # %else8
-; SSE4-NEXT:    pextrb $5, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $32, %al
 ; SSE4-NEXT:    je .LBB16_12
-; SSE4-NEXT:  # %bb.11: # %cond.store9
+; SSE4-NEXT:  .LBB16_11: # %cond.store9
 ; SSE4-NEXT:    pextrb $5, %xmm0, 5(%rdi)
-; SSE4-NEXT:  .LBB16_12: # %else10
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqb %xmm2, %xmm1
-; SSE4-NEXT:    pextrb $6, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $64, %al
 ; SSE4-NEXT:    je .LBB16_14
-; SSE4-NEXT:  # %bb.13: # %cond.store11
+; SSE4-NEXT:  .LBB16_13: # %cond.store11
 ; SSE4-NEXT:    pextrb $6, %xmm0, 6(%rdi)
-; SSE4-NEXT:  .LBB16_14: # %else12
-; SSE4-NEXT:    pextrb $7, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $-128, %al
 ; SSE4-NEXT:    je .LBB16_16
-; SSE4-NEXT:  # %bb.15: # %cond.store13
+; SSE4-NEXT:  .LBB16_15: # %cond.store13
 ; SSE4-NEXT:    pextrb $7, %xmm0, 7(%rdi)
-; SSE4-NEXT:  .LBB16_16: # %else14
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqb %xmm2, %xmm1
-; SSE4-NEXT:    pextrb $8, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $256, %eax # imm = 0x100
 ; SSE4-NEXT:    je .LBB16_18
-; SSE4-NEXT:  # %bb.17: # %cond.store15
+; SSE4-NEXT:  .LBB16_17: # %cond.store15
 ; SSE4-NEXT:    pextrb $8, %xmm0, 8(%rdi)
-; SSE4-NEXT:  .LBB16_18: # %else16
-; SSE4-NEXT:    pextrb $9, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $512, %eax # imm = 0x200
 ; SSE4-NEXT:    je .LBB16_20
-; SSE4-NEXT:  # %bb.19: # %cond.store17
+; SSE4-NEXT:  .LBB16_19: # %cond.store17
 ; SSE4-NEXT:    pextrb $9, %xmm0, 9(%rdi)
-; SSE4-NEXT:  .LBB16_20: # %else18
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqb %xmm2, %xmm1
-; SSE4-NEXT:    pextrb $10, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $1024, %eax # imm = 0x400
 ; SSE4-NEXT:    je .LBB16_22
-; SSE4-NEXT:  # %bb.21: # %cond.store19
+; SSE4-NEXT:  .LBB16_21: # %cond.store19
 ; SSE4-NEXT:    pextrb $10, %xmm0, 10(%rdi)
-; SSE4-NEXT:  .LBB16_22: # %else20
-; SSE4-NEXT:    pextrb $11, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $2048, %eax # imm = 0x800
 ; SSE4-NEXT:    je .LBB16_24
-; SSE4-NEXT:  # %bb.23: # %cond.store21
+; SSE4-NEXT:  .LBB16_23: # %cond.store21
 ; SSE4-NEXT:    pextrb $11, %xmm0, 11(%rdi)
-; SSE4-NEXT:  .LBB16_24: # %else22
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqb %xmm2, %xmm1
-; SSE4-NEXT:    pextrb $12, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $4096, %eax # imm = 0x1000
 ; SSE4-NEXT:    je .LBB16_26
-; SSE4-NEXT:  # %bb.25: # %cond.store23
+; SSE4-NEXT:  .LBB16_25: # %cond.store23
 ; SSE4-NEXT:    pextrb $12, %xmm0, 12(%rdi)
-; SSE4-NEXT:  .LBB16_26: # %else24
-; SSE4-NEXT:    pextrb $13, %xmm1, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $8192, %eax # imm = 0x2000
 ; SSE4-NEXT:    je .LBB16_28
-; SSE4-NEXT:  # %bb.27: # %cond.store25
+; SSE4-NEXT:  .LBB16_27: # %cond.store25
 ; SSE4-NEXT:    pextrb $13, %xmm0, 13(%rdi)
-; SSE4-NEXT:  .LBB16_28: # %else26
-; SSE4-NEXT:    pxor %xmm1, %xmm1
-; SSE4-NEXT:    pcmpeqb %xmm1, %xmm2
-; SSE4-NEXT:    pextrb $14, %xmm2, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $16384, %eax # imm = 0x4000
 ; SSE4-NEXT:    je .LBB16_30
-; SSE4-NEXT:  # %bb.29: # %cond.store27
+; SSE4-NEXT:  .LBB16_29: # %cond.store27
 ; SSE4-NEXT:    pextrb $14, %xmm0, 14(%rdi)
-; SSE4-NEXT:  .LBB16_30: # %else28
-; SSE4-NEXT:    pextrb $15, %xmm2, %eax
-; SSE4-NEXT:    notb %al
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testl $32768, %eax # imm = 0x8000
 ; SSE4-NEXT:    je .LBB16_32
-; SSE4-NEXT:  # %bb.31: # %cond.store29
+; SSE4-NEXT:  .LBB16_31: # %cond.store29
 ; SSE4-NEXT:    pextrb $15, %xmm0, 15(%rdi)
-; SSE4-NEXT:  .LBB16_32: # %else30
 ; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: truncstore_v16i16_v16i8:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
 ; AVX1-NEXT:    vpminuw %xmm4, %xmm3, %xmm3
 ; AVX1-NEXT:    vpminuw %xmm4, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackuswb %xmm3, %xmm0, %xmm0
-; AVX1-NEXT:    vpextrb $0, %xmm2, %eax
-; AVX1-NEXT:    notb %al
+; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpmovmskb %xmm1, %eax
+; AVX1-NEXT:    xorl $65535, %eax # imm = 0xFFFF
 ; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    je .LBB16_2
-; AVX1-NEXT:  # %bb.1: # %cond.store
+; AVX1-NEXT:    jne .LBB16_1
+; AVX1-NEXT:  # %bb.2: # %else
+; AVX1-NEXT:    testb $2, %al
+; AVX1-NEXT:    jne .LBB16_3
+; AVX1-NEXT:  .LBB16_4: # %else2
+; AVX1-NEXT:    testb $4, %al
+; AVX1-NEXT:    jne .LBB16_5
+; AVX1-NEXT:  .LBB16_6: # %else4
+; AVX1-NEXT:    testb $8, %al
+; AVX1-NEXT:    jne .LBB16_7
+; AVX1-NEXT:  .LBB16_8: # %else6
+; AVX1-NEXT:    testb $16, %al
+; AVX1-NEXT:    jne .LBB16_9
+; AVX1-NEXT:  .LBB16_10: # %else8
+; AVX1-NEXT:    testb $32, %al
+; AVX1-NEXT:    jne .LBB16_11
+; AVX1-NEXT:  .LBB16_12: # %else10
+; AVX1-NEXT:    testb $64, %al
+; AVX1-NEXT:    jne .LBB16_13
+; AVX1-NEXT:  .LBB16_14: # %else12
+; AVX1-NEXT:    testb $-128, %al
+; AVX1-NEXT:    jne .LBB16_15
+; AVX1-NEXT:  .LBB16_16: # %else14
+; AVX1-NEXT:    testl $256, %eax # imm = 0x100
+; AVX1-NEXT:    jne .LBB16_17
+; AVX1-NEXT:  .LBB16_18: # %else16
+; AVX1-NEXT:    testl $512, %eax # imm = 0x200
+; AVX1-NEXT:    jne .LBB16_19
+; AVX1-NEXT:  .LBB16_20: # %else18
+; AVX1-NEXT:    testl $1024, %eax # imm = 0x400
+; AVX1-NEXT:    jne .LBB16_21
+; AVX1-NEXT:  .LBB16_22: # %else20
+; AVX1-NEXT:    testl $2048, %eax # imm = 0x800
+; AVX1-NEXT:    jne .LBB16_23
+; AVX1-NEXT:  .LBB16_24: # %else22
+; AVX1-NEXT:    testl $4096, %eax # imm = 0x1000
+; AVX1-NEXT:    jne .LBB16_25
+; AVX1-NEXT:  .LBB16_26: # %else24
+; AVX1-NEXT:    testl $8192, %eax # imm = 0x2000
+; AVX1-NEXT:    jne .LBB16_27
+; AVX1-NEXT:  .LBB16_28: # %else26
+; AVX1-NEXT:    testl $16384, %eax # imm = 0x4000
+; AVX1-NEXT:    jne .LBB16_29
+; AVX1-NEXT:  .LBB16_30: # %else28
+; AVX1-NEXT:    testl $32768, %eax # imm = 0x8000
+; AVX1-NEXT:    jne .LBB16_31
+; AVX1-NEXT:  .LBB16_32: # %else30
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+; AVX1-NEXT:  .LBB16_1: # %cond.store
 ; AVX1-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX1-NEXT:  .LBB16_2: # %else
-; AVX1-NEXT:    vpextrb $1, %xmm2, %eax
-; AVX1-NEXT:    notb %al
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $2, %al
 ; AVX1-NEXT:    je .LBB16_4
-; AVX1-NEXT:  # %bb.3: # %cond.store1
+; AVX1-NEXT:  .LBB16_3: # %cond.store1
 ; AVX1-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
-; AVX1-NEXT:  .LBB16_4: # %else2
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpextrb $2, %xmm2, %eax
-; AVX1-NEXT:    notb %al
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $4, %al
 ; AVX1-NEXT:    je .LBB16_6
-; AVX1-NEXT:  # %bb.5: # %cond.store3
+; AVX1-NEXT:  .LBB16_5: # %cond.store3
 ; AVX1-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
-; AVX1-NEXT:  .LBB16_6: # %else4
-; AVX1-NEXT:    vpextrb $3, %xmm2, %eax
-; AVX1-NEXT:    notb %al
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $8, %al
 ; AVX1-NEXT:    je .LBB16_8
-; AVX1-NEXT:  # %bb.7: # %cond.store5
+; AVX1-NEXT:  .LBB16_7: # %cond.store5
 ; AVX1-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
-; AVX1-NEXT:  .LBB16_8: # %else6
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX1-NEXT:    notb %al
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $16, %al
 ; AVX1-NEXT:    je .LBB16_10
-; AVX1-NEXT:  # %bb.9: # %cond.store7
+; AVX1-NEXT:  .LBB16_9: # %cond.store7
 ; AVX1-NEXT:    vpextrb $4, %xmm0, 4(%rdi)
-; AVX1-NEXT:  .LBB16_10: # %else8
-; AVX1-NEXT:    vpextrb $5, %xmm2, %eax
-; AVX1-NEXT:    notb %al
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $32, %al
 ; AVX1-NEXT:    je .LBB16_12
-; AVX1-NEXT:  # %bb.11: # %cond.store9
+; AVX1-NEXT:  .LBB16_11: # %cond.store9
 ; AVX1-NEXT:    vpextrb $5, %xmm0, 5(%rdi)
-; AVX1-NEXT:  .LBB16_12: # %else10
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpextrb $6, %xmm2, %eax
-; AVX1-NEXT:    notb %al
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $64, %al
 ; AVX1-NEXT:    je .LBB16_14
-; AVX1-NEXT:  # %bb.13: # %cond.store11
+; AVX1-NEXT:  .LBB16_13: # %cond.store11
 ; AVX1-NEXT:    vpextrb $6, %xmm0, 6(%rdi)
-; AVX1-NEXT:  .LBB16_14: # %else12
-; AVX1-NEXT:    vpextrb $7, %xmm2, %eax
-; AVX1-NEXT:    notb %al
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testb $-128, %al
 ; AVX1-NEXT:    je .LBB16_16
-; AVX1-NEXT:  # %bb.15: # %cond.store13
+; AVX1-NEXT:  .LBB16_15: # %cond.store13
 ; AVX1-NEXT:    vpextrb $7, %xmm0, 7(%rdi)
-; AVX1-NEXT:  .LBB16_16: # %else14
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpextrb $8, %xmm2, %eax
-; AVX1-NEXT:    notb %al
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $256, %eax # imm = 0x100
 ; AVX1-NEXT:    je .LBB16_18
-; AVX1-NEXT:  # %bb.17: # %cond.store15
+; AVX1-NEXT:  .LBB16_17: # %cond.store15
 ; AVX1-NEXT:    vpextrb $8, %xmm0, 8(%rdi)
-; AVX1-NEXT:  .LBB16_18: # %else16
-; AVX1-NEXT:    vpextrb $9, %xmm2, %eax
-; AVX1-NEXT:    notb %al
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $512, %eax # imm = 0x200
 ; AVX1-NEXT:    je .LBB16_20
-; AVX1-NEXT:  # %bb.19: # %cond.store17
+; AVX1-NEXT:  .LBB16_19: # %cond.store17
 ; AVX1-NEXT:    vpextrb $9, %xmm0, 9(%rdi)
-; AVX1-NEXT:  .LBB16_20: # %else18
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpextrb $10, %xmm2, %eax
-; AVX1-NEXT:    notb %al
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $1024, %eax # imm = 0x400
 ; AVX1-NEXT:    je .LBB16_22
-; AVX1-NEXT:  # %bb.21: # %cond.store19
+; AVX1-NEXT:  .LBB16_21: # %cond.store19
 ; AVX1-NEXT:    vpextrb $10, %xmm0, 10(%rdi)
-; AVX1-NEXT:  .LBB16_22: # %else20
-; AVX1-NEXT:    vpextrb $11, %xmm2, %eax
-; AVX1-NEXT:    notb %al
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $2048, %eax # imm = 0x800
 ; AVX1-NEXT:    je .LBB16_24
-; AVX1-NEXT:  # %bb.23: # %cond.store21
+; AVX1-NEXT:  .LBB16_23: # %cond.store21
 ; AVX1-NEXT:    vpextrb $11, %xmm0, 11(%rdi)
-; AVX1-NEXT:  .LBB16_24: # %else22
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpextrb $12, %xmm2, %eax
-; AVX1-NEXT:    notb %al
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $4096, %eax # imm = 0x1000
 ; AVX1-NEXT:    je .LBB16_26
-; AVX1-NEXT:  # %bb.25: # %cond.store23
+; AVX1-NEXT:  .LBB16_25: # %cond.store23
 ; AVX1-NEXT:    vpextrb $12, %xmm0, 12(%rdi)
-; AVX1-NEXT:  .LBB16_26: # %else24
-; AVX1-NEXT:    vpextrb $13, %xmm2, %eax
-; AVX1-NEXT:    notb %al
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $8192, %eax # imm = 0x2000
 ; AVX1-NEXT:    je .LBB16_28
-; AVX1-NEXT:  # %bb.27: # %cond.store25
+; AVX1-NEXT:  .LBB16_27: # %cond.store25
 ; AVX1-NEXT:    vpextrb $13, %xmm0, 13(%rdi)
-; AVX1-NEXT:  .LBB16_28: # %else26
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrb $14, %xmm1, %eax
-; AVX1-NEXT:    notb %al
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $16384, %eax # imm = 0x4000
 ; AVX1-NEXT:    je .LBB16_30
-; AVX1-NEXT:  # %bb.29: # %cond.store27
+; AVX1-NEXT:  .LBB16_29: # %cond.store27
 ; AVX1-NEXT:    vpextrb $14, %xmm0, 14(%rdi)
-; AVX1-NEXT:  .LBB16_30: # %else28
-; AVX1-NEXT:    vpextrb $15, %xmm1, %eax
-; AVX1-NEXT:    notb %al
-; AVX1-NEXT:    testb $1, %al
+; AVX1-NEXT:    testl $32768, %eax # imm = 0x8000
 ; AVX1-NEXT:    je .LBB16_32
-; AVX1-NEXT:  # %bb.31: # %cond.store29
+; AVX1-NEXT:  .LBB16_31: # %cond.store29
 ; AVX1-NEXT:    vpextrb $15, %xmm0, 15(%rdi)
-; AVX1-NEXT:  .LBB16_32: # %else30
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: truncstore_v16i16_v16i8:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
 ; AVX2-NEXT:    vpminuw {{.*}}(%rip), %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm3
 ; AVX2-NEXT:    vpackuswb %xmm3, %xmm0, %xmm0
-; AVX2-NEXT:    vpextrb $0, %xmm2, %eax
-; AVX2-NEXT:    notb %al
+; AVX2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpmovmskb %xmm1, %eax
+; AVX2-NEXT:    xorl $65535, %eax # imm = 0xFFFF
 ; AVX2-NEXT:    testb $1, %al
-; AVX2-NEXT:    je .LBB16_2
-; AVX2-NEXT:  # %bb.1: # %cond.store
+; AVX2-NEXT:    jne .LBB16_1
+; AVX2-NEXT:  # %bb.2: # %else
+; AVX2-NEXT:    testb $2, %al
+; AVX2-NEXT:    jne .LBB16_3
+; AVX2-NEXT:  .LBB16_4: # %else2
+; AVX2-NEXT:    testb $4, %al
+; AVX2-NEXT:    jne .LBB16_5
+; AVX2-NEXT:  .LBB16_6: # %else4
+; AVX2-NEXT:    testb $8, %al
+; AVX2-NEXT:    jne .LBB16_7
+; AVX2-NEXT:  .LBB16_8: # %else6
+; AVX2-NEXT:    testb $16, %al
+; AVX2-NEXT:    jne .LBB16_9
+; AVX2-NEXT:  .LBB16_10: # %else8
+; AVX2-NEXT:    testb $32, %al
+; AVX2-NEXT:    jne .LBB16_11
+; AVX2-NEXT:  .LBB16_12: # %else10
+; AVX2-NEXT:    testb $64, %al
+; AVX2-NEXT:    jne .LBB16_13
+; AVX2-NEXT:  .LBB16_14: # %else12
+; AVX2-NEXT:    testb $-128, %al
+; AVX2-NEXT:    jne .LBB16_15
+; AVX2-NEXT:  .LBB16_16: # %else14
+; AVX2-NEXT:    testl $256, %eax # imm = 0x100
+; AVX2-NEXT:    jne .LBB16_17
+; AVX2-NEXT:  .LBB16_18: # %else16
+; AVX2-NEXT:    testl $512, %eax # imm = 0x200
+; AVX2-NEXT:    jne .LBB16_19
+; AVX2-NEXT:  .LBB16_20: # %else18
+; AVX2-NEXT:    testl $1024, %eax # imm = 0x400
+; AVX2-NEXT:    jne .LBB16_21
+; AVX2-NEXT:  .LBB16_22: # %else20
+; AVX2-NEXT:    testl $2048, %eax # imm = 0x800
+; AVX2-NEXT:    jne .LBB16_23
+; AVX2-NEXT:  .LBB16_24: # %else22
+; AVX2-NEXT:    testl $4096, %eax # imm = 0x1000
+; AVX2-NEXT:    jne .LBB16_25
+; AVX2-NEXT:  .LBB16_26: # %else24
+; AVX2-NEXT:    testl $8192, %eax # imm = 0x2000
+; AVX2-NEXT:    jne .LBB16_27
+; AVX2-NEXT:  .LBB16_28: # %else26
+; AVX2-NEXT:    testl $16384, %eax # imm = 0x4000
+; AVX2-NEXT:    jne .LBB16_29
+; AVX2-NEXT:  .LBB16_30: # %else28
+; AVX2-NEXT:    testl $32768, %eax # imm = 0x8000
+; AVX2-NEXT:    jne .LBB16_31
+; AVX2-NEXT:  .LBB16_32: # %else30
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+; AVX2-NEXT:  .LBB16_1: # %cond.store
 ; AVX2-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX2-NEXT:  .LBB16_2: # %else
-; AVX2-NEXT:    vpextrb $1, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $2, %al
 ; AVX2-NEXT:    je .LBB16_4
-; AVX2-NEXT:  # %bb.3: # %cond.store1
+; AVX2-NEXT:  .LBB16_3: # %cond.store1
 ; AVX2-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
-; AVX2-NEXT:  .LBB16_4: # %else2
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpextrb $2, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $4, %al
 ; AVX2-NEXT:    je .LBB16_6
-; AVX2-NEXT:  # %bb.5: # %cond.store3
+; AVX2-NEXT:  .LBB16_5: # %cond.store3
 ; AVX2-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
-; AVX2-NEXT:  .LBB16_6: # %else4
-; AVX2-NEXT:    vpextrb $3, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $8, %al
 ; AVX2-NEXT:    je .LBB16_8
-; AVX2-NEXT:  # %bb.7: # %cond.store5
+; AVX2-NEXT:  .LBB16_7: # %cond.store5
 ; AVX2-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
-; AVX2-NEXT:  .LBB16_8: # %else6
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $16, %al
 ; AVX2-NEXT:    je .LBB16_10
-; AVX2-NEXT:  # %bb.9: # %cond.store7
+; AVX2-NEXT:  .LBB16_9: # %cond.store7
 ; AVX2-NEXT:    vpextrb $4, %xmm0, 4(%rdi)
-; AVX2-NEXT:  .LBB16_10: # %else8
-; AVX2-NEXT:    vpextrb $5, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $32, %al
 ; AVX2-NEXT:    je .LBB16_12
-; AVX2-NEXT:  # %bb.11: # %cond.store9
+; AVX2-NEXT:  .LBB16_11: # %cond.store9
 ; AVX2-NEXT:    vpextrb $5, %xmm0, 5(%rdi)
-; AVX2-NEXT:  .LBB16_12: # %else10
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpextrb $6, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $64, %al
 ; AVX2-NEXT:    je .LBB16_14
-; AVX2-NEXT:  # %bb.13: # %cond.store11
+; AVX2-NEXT:  .LBB16_13: # %cond.store11
 ; AVX2-NEXT:    vpextrb $6, %xmm0, 6(%rdi)
-; AVX2-NEXT:  .LBB16_14: # %else12
-; AVX2-NEXT:    vpextrb $7, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testb $-128, %al
 ; AVX2-NEXT:    je .LBB16_16
-; AVX2-NEXT:  # %bb.15: # %cond.store13
+; AVX2-NEXT:  .LBB16_15: # %cond.store13
 ; AVX2-NEXT:    vpextrb $7, %xmm0, 7(%rdi)
-; AVX2-NEXT:  .LBB16_16: # %else14
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpextrb $8, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $256, %eax # imm = 0x100
 ; AVX2-NEXT:    je .LBB16_18
-; AVX2-NEXT:  # %bb.17: # %cond.store15
-; AVX2-NEXT:    vpextrb $8, %xmm0, 8(%rdi)
-; AVX2-NEXT:  .LBB16_18: # %else16
-; AVX2-NEXT:    vpextrb $9, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:  .LBB16_17: # %cond.store15
+; AVX2-NEXT:    vpextrb $8, %xmm0, 8(%rdi)
+; AVX2-NEXT:    testl $512, %eax # imm = 0x200
 ; AVX2-NEXT:    je .LBB16_20
-; AVX2-NEXT:  # %bb.19: # %cond.store17
+; AVX2-NEXT:  .LBB16_19: # %cond.store17
 ; AVX2-NEXT:    vpextrb $9, %xmm0, 9(%rdi)
-; AVX2-NEXT:  .LBB16_20: # %else18
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpextrb $10, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $1024, %eax # imm = 0x400
 ; AVX2-NEXT:    je .LBB16_22
-; AVX2-NEXT:  # %bb.21: # %cond.store19
+; AVX2-NEXT:  .LBB16_21: # %cond.store19
 ; AVX2-NEXT:    vpextrb $10, %xmm0, 10(%rdi)
-; AVX2-NEXT:  .LBB16_22: # %else20
-; AVX2-NEXT:    vpextrb $11, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $2048, %eax # imm = 0x800
 ; AVX2-NEXT:    je .LBB16_24
-; AVX2-NEXT:  # %bb.23: # %cond.store21
+; AVX2-NEXT:  .LBB16_23: # %cond.store21
 ; AVX2-NEXT:    vpextrb $11, %xmm0, 11(%rdi)
-; AVX2-NEXT:  .LBB16_24: # %else22
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpextrb $12, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $4096, %eax # imm = 0x1000
 ; AVX2-NEXT:    je .LBB16_26
-; AVX2-NEXT:  # %bb.25: # %cond.store23
+; AVX2-NEXT:  .LBB16_25: # %cond.store23
 ; AVX2-NEXT:    vpextrb $12, %xmm0, 12(%rdi)
-; AVX2-NEXT:  .LBB16_26: # %else24
-; AVX2-NEXT:    vpextrb $13, %xmm2, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $8192, %eax # imm = 0x2000
 ; AVX2-NEXT:    je .LBB16_28
-; AVX2-NEXT:  # %bb.27: # %cond.store25
+; AVX2-NEXT:  .LBB16_27: # %cond.store25
 ; AVX2-NEXT:    vpextrb $13, %xmm0, 13(%rdi)
-; AVX2-NEXT:  .LBB16_28: # %else26
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpextrb $14, %xmm1, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $16384, %eax # imm = 0x4000
 ; AVX2-NEXT:    je .LBB16_30
-; AVX2-NEXT:  # %bb.29: # %cond.store27
+; AVX2-NEXT:  .LBB16_29: # %cond.store27
 ; AVX2-NEXT:    vpextrb $14, %xmm0, 14(%rdi)
-; AVX2-NEXT:  .LBB16_30: # %else28
-; AVX2-NEXT:    vpextrb $15, %xmm1, %eax
-; AVX2-NEXT:    notb %al
-; AVX2-NEXT:    testb $1, %al
+; AVX2-NEXT:    testl $32768, %eax # imm = 0x8000
 ; AVX2-NEXT:    je .LBB16_32
-; AVX2-NEXT:  # %bb.31: # %cond.store29
+; AVX2-NEXT:  .LBB16_31: # %cond.store29
 ; AVX2-NEXT:    vpextrb $15, %xmm0, 15(%rdi)
-; AVX2-NEXT:  .LBB16_32: # %else30
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: truncstore_v16i16_v16i8:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm1
 ; AVX512F-NEXT:    vpminuw {{.*}}(%rip), %ymm0, %ymm0
 ; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
 ; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
-; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    vpmovmskb %xmm1, %eax
+; AVX512F-NEXT:    xorl $65535, %eax # imm = 0xFFFF
 ; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB16_2
-; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    jne .LBB16_1
+; AVX512F-NEXT:  # %bb.2: # %else
+; AVX512F-NEXT:    testb $2, %al
+; AVX512F-NEXT:    jne .LBB16_3
+; AVX512F-NEXT:  .LBB16_4: # %else2
+; AVX512F-NEXT:    testb $4, %al
+; AVX512F-NEXT:    jne .LBB16_5
+; AVX512F-NEXT:  .LBB16_6: # %else4
+; AVX512F-NEXT:    testb $8, %al
+; AVX512F-NEXT:    jne .LBB16_7
+; AVX512F-NEXT:  .LBB16_8: # %else6
+; AVX512F-NEXT:    testb $16, %al
+; AVX512F-NEXT:    jne .LBB16_9
+; AVX512F-NEXT:  .LBB16_10: # %else8
+; AVX512F-NEXT:    testb $32, %al
+; AVX512F-NEXT:    jne .LBB16_11
+; AVX512F-NEXT:  .LBB16_12: # %else10
+; AVX512F-NEXT:    testb $64, %al
+; AVX512F-NEXT:    jne .LBB16_13
+; AVX512F-NEXT:  .LBB16_14: # %else12
+; AVX512F-NEXT:    testb $-128, %al
+; AVX512F-NEXT:    jne .LBB16_15
+; AVX512F-NEXT:  .LBB16_16: # %else14
+; AVX512F-NEXT:    testl $256, %eax # imm = 0x100
+; AVX512F-NEXT:    jne .LBB16_17
+; AVX512F-NEXT:  .LBB16_18: # %else16
+; AVX512F-NEXT:    testl $512, %eax # imm = 0x200
+; AVX512F-NEXT:    jne .LBB16_19
+; AVX512F-NEXT:  .LBB16_20: # %else18
+; AVX512F-NEXT:    testl $1024, %eax # imm = 0x400
+; AVX512F-NEXT:    jne .LBB16_21
+; AVX512F-NEXT:  .LBB16_22: # %else20
+; AVX512F-NEXT:    testl $2048, %eax # imm = 0x800
+; AVX512F-NEXT:    jne .LBB16_23
+; AVX512F-NEXT:  .LBB16_24: # %else22
+; AVX512F-NEXT:    testl $4096, %eax # imm = 0x1000
+; AVX512F-NEXT:    jne .LBB16_25
+; AVX512F-NEXT:  .LBB16_26: # %else24
+; AVX512F-NEXT:    testl $8192, %eax # imm = 0x2000
+; AVX512F-NEXT:    jne .LBB16_27
+; AVX512F-NEXT:  .LBB16_28: # %else26
+; AVX512F-NEXT:    testl $16384, %eax # imm = 0x4000
+; AVX512F-NEXT:    jne .LBB16_29
+; AVX512F-NEXT:  .LBB16_30: # %else28
+; AVX512F-NEXT:    testl $32768, %eax # imm = 0x8000
+; AVX512F-NEXT:    jne .LBB16_31
+; AVX512F-NEXT:  .LBB16_32: # %else30
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+; AVX512F-NEXT:  .LBB16_1: # %cond.store
 ; AVX512F-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX512F-NEXT:  .LBB16_2: # %else
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $2, %al
 ; AVX512F-NEXT:    je .LBB16_4
-; AVX512F-NEXT:  # %bb.3: # %cond.store1
+; AVX512F-NEXT:  .LBB16_3: # %cond.store1
 ; AVX512F-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
-; AVX512F-NEXT:  .LBB16_4: # %else2
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $4, %al
 ; AVX512F-NEXT:    je .LBB16_6
-; AVX512F-NEXT:  # %bb.5: # %cond.store3
+; AVX512F-NEXT:  .LBB16_5: # %cond.store3
 ; AVX512F-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
-; AVX512F-NEXT:  .LBB16_6: # %else4
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $8, %al
 ; AVX512F-NEXT:    je .LBB16_8
-; AVX512F-NEXT:  # %bb.7: # %cond.store5
+; AVX512F-NEXT:  .LBB16_7: # %cond.store5
 ; AVX512F-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
-; AVX512F-NEXT:  .LBB16_8: # %else6
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $4, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $16, %al
 ; AVX512F-NEXT:    je .LBB16_10
-; AVX512F-NEXT:  # %bb.9: # %cond.store7
+; AVX512F-NEXT:  .LBB16_9: # %cond.store7
 ; AVX512F-NEXT:    vpextrb $4, %xmm0, 4(%rdi)
-; AVX512F-NEXT:  .LBB16_10: # %else8
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $5, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $32, %al
 ; AVX512F-NEXT:    je .LBB16_12
-; AVX512F-NEXT:  # %bb.11: # %cond.store9
+; AVX512F-NEXT:  .LBB16_11: # %cond.store9
 ; AVX512F-NEXT:    vpextrb $5, %xmm0, 5(%rdi)
-; AVX512F-NEXT:  .LBB16_12: # %else10
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $6, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $64, %al
 ; AVX512F-NEXT:    je .LBB16_14
-; AVX512F-NEXT:  # %bb.13: # %cond.store11
+; AVX512F-NEXT:  .LBB16_13: # %cond.store11
 ; AVX512F-NEXT:    vpextrb $6, %xmm0, 6(%rdi)
-; AVX512F-NEXT:  .LBB16_14: # %else12
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $7, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $-128, %al
 ; AVX512F-NEXT:    je .LBB16_16
-; AVX512F-NEXT:  # %bb.15: # %cond.store13
+; AVX512F-NEXT:  .LBB16_15: # %cond.store13
 ; AVX512F-NEXT:    vpextrb $7, %xmm0, 7(%rdi)
-; AVX512F-NEXT:  .LBB16_16: # %else14
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $8, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $256, %eax # imm = 0x100
 ; AVX512F-NEXT:    je .LBB16_18
-; AVX512F-NEXT:  # %bb.17: # %cond.store15
+; AVX512F-NEXT:  .LBB16_17: # %cond.store15
 ; AVX512F-NEXT:    vpextrb $8, %xmm0, 8(%rdi)
-; AVX512F-NEXT:  .LBB16_18: # %else16
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $9, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $512, %eax # imm = 0x200
 ; AVX512F-NEXT:    je .LBB16_20
-; AVX512F-NEXT:  # %bb.19: # %cond.store17
+; AVX512F-NEXT:  .LBB16_19: # %cond.store17
 ; AVX512F-NEXT:    vpextrb $9, %xmm0, 9(%rdi)
-; AVX512F-NEXT:  .LBB16_20: # %else18
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $10, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $1024, %eax # imm = 0x400
 ; AVX512F-NEXT:    je .LBB16_22
-; AVX512F-NEXT:  # %bb.21: # %cond.store19
+; AVX512F-NEXT:  .LBB16_21: # %cond.store19
 ; AVX512F-NEXT:    vpextrb $10, %xmm0, 10(%rdi)
-; AVX512F-NEXT:  .LBB16_22: # %else20
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $11, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $2048, %eax # imm = 0x800
 ; AVX512F-NEXT:    je .LBB16_24
-; AVX512F-NEXT:  # %bb.23: # %cond.store21
+; AVX512F-NEXT:  .LBB16_23: # %cond.store21
 ; AVX512F-NEXT:    vpextrb $11, %xmm0, 11(%rdi)
-; AVX512F-NEXT:  .LBB16_24: # %else22
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
-; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $12, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $4096, %eax # imm = 0x1000
 ; AVX512F-NEXT:    je .LBB16_26
-; AVX512F-NEXT:  # %bb.25: # %cond.store23
+; AVX512F-NEXT:  .LBB16_25: # %cond.store23
 ; AVX512F-NEXT:    vpextrb $12, %xmm0, 12(%rdi)
-; AVX512F-NEXT:  .LBB16_26: # %else24
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $13, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $8192, %eax # imm = 0x2000
 ; AVX512F-NEXT:    je .LBB16_28
-; AVX512F-NEXT:  # %bb.27: # %cond.store25
+; AVX512F-NEXT:  .LBB16_27: # %cond.store25
 ; AVX512F-NEXT:    vpextrb $13, %xmm0, 13(%rdi)
-; AVX512F-NEXT:  .LBB16_28: # %else26
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm1
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm2
-; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm2
-; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $14, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $16384, %eax # imm = 0x4000
 ; AVX512F-NEXT:    je .LBB16_30
-; AVX512F-NEXT:  # %bb.29: # %cond.store27
+; AVX512F-NEXT:  .LBB16_29: # %cond.store27
 ; AVX512F-NEXT:    vpextrb $14, %xmm0, 14(%rdi)
-; AVX512F-NEXT:  .LBB16_30: # %else28
-; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
-; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $15, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testl $32768, %eax # imm = 0x8000
 ; AVX512F-NEXT:    je .LBB16_32
-; AVX512F-NEXT:  # %bb.31: # %cond.store29
+; AVX512F-NEXT:  .LBB16_31: # %cond.store29
 ; AVX512F-NEXT:    vpextrb $15, %xmm0, 15(%rdi)
-; AVX512F-NEXT:  .LBB16_32: # %else30
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: truncstore_v16i16_v16i8:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
 ; AVX512BW-NEXT:    vptestmb %zmm1, %zmm1, %k0
 ; AVX512BW-NEXT:    kmovw %k0, %k1
-; AVX512BW-NEXT:    vpminuw {{.*}}(%rip), %ymm0, %ymm0
-; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512BW-NEXT:    vpmovuswb %zmm0, %ymm0
 ; AVX512BW-NEXT:    vmovdqu8 %zmm0, (%rdi) {%k1}
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -7895,326 +7069,284 @@ define void @truncstore_v8i16_v8i8(<8 x i16> %x, <8 x i8>* %p, <8 x i16> %mask)
 ; SSE2-LABEL: truncstore_v8i16_v8i8:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpeqw %xmm1, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm3
-; SSE2-NEXT:    pxor %xmm2, %xmm3
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [32768,32768,32768,32768,32768,32768,32768,32768]
-; SSE2-NEXT:    pxor %xmm2, %xmm0
+; SSE2-NEXT:    pxor {{.*}}(%rip), %xmm0
 ; SSE2-NEXT:    pminsw {{.*}}(%rip), %xmm0
-; SSE2-NEXT:    pxor %xmm2, %xmm0
-; SSE2-NEXT:    movd %xmm3, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    je .LBB17_2
-; SSE2-NEXT:  # %bb.1: # %cond.store
-; SSE2-NEXT:    movb %al, (%rdi)
-; SSE2-NEXT:  .LBB17_2: # %else
-; SSE2-NEXT:    shrl $16, %ecx
-; SSE2-NEXT:    testb $1, %cl
-; SSE2-NEXT:    je .LBB17_4
-; SSE2-NEXT:  # %bb.3: # %cond.store1
-; SSE2-NEXT:    shrl $16, %eax
-; SSE2-NEXT:    movb %al, 1(%rdi)
-; SSE2-NEXT:  .LBB17_4: # %else2
-; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    pcmpeqw %xmm1, %xmm3
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT:    pxor %xmm3, %xmm2
-; SSE2-NEXT:    pextrw $2, %xmm2, %eax
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    packuswb %xmm0, %xmm0
+; SSE2-NEXT:    pcmpeqw %xmm1, %xmm2
+; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
+; SSE2-NEXT:    pxor %xmm2, %xmm1
+; SSE2-NEXT:    packsswb %xmm0, %xmm1
+; SSE2-NEXT:    pmovmskb %xmm1, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB17_6
-; SSE2-NEXT:  # %bb.5: # %cond.store3
-; SSE2-NEXT:    pextrw $2, %xmm0, %eax
-; SSE2-NEXT:    movb %al, 2(%rdi)
+; SSE2-NEXT:    movd %xmm0, %ecx
+; SSE2-NEXT:    jne .LBB17_1
+; SSE2-NEXT:  # %bb.2: # %else
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    jne .LBB17_3
+; SSE2-NEXT:  .LBB17_4: # %else2
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    jne .LBB17_5
 ; SSE2-NEXT:  .LBB17_6: # %else4
-; SSE2-NEXT:    pextrw $3, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je .LBB17_8
-; SSE2-NEXT:  # %bb.7: # %cond.store5
-; SSE2-NEXT:    pextrw $3, %xmm0, %eax
-; SSE2-NEXT:    movb %al, 3(%rdi)
+; SSE2-NEXT:  .LBB17_7: # %cond.store5
+; SSE2-NEXT:    shrl $24, %ecx
+; SSE2-NEXT:    movb %cl, 3(%rdi)
 ; SSE2-NEXT:  .LBB17_8: # %else6
-; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    pcmpeqw %xmm1, %xmm3
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT:    pxor %xmm3, %xmm2
-; SSE2-NEXT:    pextrw $4, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $16, %al
+; SSE2-NEXT:    pextrw $2, %xmm0, %ecx
 ; SSE2-NEXT:    je .LBB17_10
 ; SSE2-NEXT:  # %bb.9: # %cond.store7
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    movb %al, 4(%rdi)
+; SSE2-NEXT:    movb %cl, 4(%rdi)
 ; SSE2-NEXT:  .LBB17_10: # %else8
-; SSE2-NEXT:    pextrw $5, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
+; SSE2-NEXT:    testb $32, %al
 ; SSE2-NEXT:    je .LBB17_12
 ; SSE2-NEXT:  # %bb.11: # %cond.store9
-; SSE2-NEXT:    pextrw $5, %xmm0, %eax
-; SSE2-NEXT:    movb %al, 5(%rdi)
+; SSE2-NEXT:    movb %ch, 5(%rdi)
 ; SSE2-NEXT:  .LBB17_12: # %else10
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpeqw %xmm2, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT:    pxor %xmm1, %xmm2
-; SSE2-NEXT:    pextrw $6, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB17_14
-; SSE2-NEXT:  # %bb.13: # %cond.store11
-; SSE2-NEXT:    pextrw $6, %xmm0, %eax
-; SSE2-NEXT:    movb %al, 6(%rdi)
-; SSE2-NEXT:  .LBB17_14: # %else12
-; SSE2-NEXT:    pextrw $7, %xmm2, %eax
-; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB17_16
-; SSE2-NEXT:  # %bb.15: # %cond.store13
-; SSE2-NEXT:    pextrw $7, %xmm0, %eax
-; SSE2-NEXT:    movb %al, 7(%rdi)
+; SSE2-NEXT:    testb $64, %al
+; SSE2-NEXT:    pextrw $3, %xmm0, %ecx
+; SSE2-NEXT:    jne .LBB17_13
+; SSE2-NEXT:  # %bb.14: # %else12
+; SSE2-NEXT:    testb $-128, %al
+; SSE2-NEXT:    jne .LBB17_15
 ; SSE2-NEXT:  .LBB17_16: # %else14
 ; SSE2-NEXT:    retq
+; SSE2-NEXT:  .LBB17_1: # %cond.store
+; SSE2-NEXT:    movb %cl, (%rdi)
+; SSE2-NEXT:    testb $2, %al
+; SSE2-NEXT:    je .LBB17_4
+; SSE2-NEXT:  .LBB17_3: # %cond.store1
+; SSE2-NEXT:    movb %ch, 1(%rdi)
+; SSE2-NEXT:    testb $4, %al
+; SSE2-NEXT:    je .LBB17_6
+; SSE2-NEXT:  .LBB17_5: # %cond.store3
+; SSE2-NEXT:    movl %ecx, %edx
+; SSE2-NEXT:    shrl $16, %edx
+; SSE2-NEXT:    movb %dl, 2(%rdi)
+; SSE2-NEXT:    testb $8, %al
+; SSE2-NEXT:    jne .LBB17_7
+; SSE2-NEXT:    jmp .LBB17_8
+; SSE2-NEXT:  .LBB17_13: # %cond.store11
+; SSE2-NEXT:    movb %cl, 6(%rdi)
+; SSE2-NEXT:    testb $-128, %al
+; SSE2-NEXT:    je .LBB17_16
+; SSE2-NEXT:  .LBB17_15: # %cond.store13
+; SSE2-NEXT:    movb %ch, 7(%rdi)
+; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: truncstore_v8i16_v8i8:
 ; SSE4:       # %bb.0:
-; SSE4-NEXT:    pxor %xmm3, %xmm3
-; SSE4-NEXT:    pcmpeqw %xmm1, %xmm3
-; SSE4-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE4-NEXT:    pxor %xmm3, %xmm2
+; SSE4-NEXT:    pxor %xmm2, %xmm2
 ; SSE4-NEXT:    pminuw {{.*}}(%rip), %xmm0
-; SSE4-NEXT:    pextrb $0, %xmm2, %eax
+; SSE4-NEXT:    packuswb %xmm0, %xmm0
+; SSE4-NEXT:    pcmpeqw %xmm1, %xmm2
+; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
+; SSE4-NEXT:    pxor %xmm2, %xmm1
+; SSE4-NEXT:    packsswb %xmm0, %xmm1
+; SSE4-NEXT:    pmovmskb %xmm1, %eax
 ; SSE4-NEXT:    testb $1, %al
-; SSE4-NEXT:    je .LBB17_2
-; SSE4-NEXT:  # %bb.1: # %cond.store
+; SSE4-NEXT:    jne .LBB17_1
+; SSE4-NEXT:  # %bb.2: # %else
+; SSE4-NEXT:    testb $2, %al
+; SSE4-NEXT:    jne .LBB17_3
+; SSE4-NEXT:  .LBB17_4: # %else2
+; SSE4-NEXT:    testb $4, %al
+; SSE4-NEXT:    jne .LBB17_5
+; SSE4-NEXT:  .LBB17_6: # %else4
+; SSE4-NEXT:    testb $8, %al
+; SSE4-NEXT:    jne .LBB17_7
+; SSE4-NEXT:  .LBB17_8: # %else6
+; SSE4-NEXT:    testb $16, %al
+; SSE4-NEXT:    jne .LBB17_9
+; SSE4-NEXT:  .LBB17_10: # %else8
+; SSE4-NEXT:    testb $32, %al
+; SSE4-NEXT:    jne .LBB17_11
+; SSE4-NEXT:  .LBB17_12: # %else10
+; SSE4-NEXT:    testb $64, %al
+; SSE4-NEXT:    jne .LBB17_13
+; SSE4-NEXT:  .LBB17_14: # %else12
+; SSE4-NEXT:    testb $-128, %al
+; SSE4-NEXT:    jne .LBB17_15
+; SSE4-NEXT:  .LBB17_16: # %else14
+; SSE4-NEXT:    retq
+; SSE4-NEXT:  .LBB17_1: # %cond.store
 ; SSE4-NEXT:    pextrb $0, %xmm0, (%rdi)
-; SSE4-NEXT:  .LBB17_2: # %else
-; SSE4-NEXT:    pextrb $2, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:    testb $2, %al
 ; SSE4-NEXT:    je .LBB17_4
-; SSE4-NEXT:  # %bb.3: # %cond.store1
-; SSE4-NEXT:    pextrb $2, %xmm0, 1(%rdi)
-; SSE4-NEXT:  .LBB17_4: # %else2
-; SSE4-NEXT:    pxor %xmm3, %xmm3
-; SSE4-NEXT:    pcmpeqw %xmm1, %xmm3
-; SSE4-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE4-NEXT:    pxor %xmm3, %xmm2
-; SSE4-NEXT:    pextrb $4, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:  .LBB17_3: # %cond.store1
+; SSE4-NEXT:    pextrb $1, %xmm0, 1(%rdi)
+; SSE4-NEXT:    testb $4, %al
 ; SSE4-NEXT:    je .LBB17_6
-; SSE4-NEXT:  # %bb.5: # %cond.store3
-; SSE4-NEXT:    pextrb $4, %xmm0, 2(%rdi)
-; SSE4-NEXT:  .LBB17_6: # %else4
-; SSE4-NEXT:    pextrb $6, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:  .LBB17_5: # %cond.store3
+; SSE4-NEXT:    pextrb $2, %xmm0, 2(%rdi)
+; SSE4-NEXT:    testb $8, %al
 ; SSE4-NEXT:    je .LBB17_8
-; SSE4-NEXT:  # %bb.7: # %cond.store5
-; SSE4-NEXT:    pextrb $6, %xmm0, 3(%rdi)
-; SSE4-NEXT:  .LBB17_8: # %else6
-; SSE4-NEXT:    pxor %xmm3, %xmm3
-; SSE4-NEXT:    pcmpeqw %xmm1, %xmm3
-; SSE4-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE4-NEXT:    pxor %xmm3, %xmm2
-; SSE4-NEXT:    pextrb $8, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:  .LBB17_7: # %cond.store5
+; SSE4-NEXT:    pextrb $3, %xmm0, 3(%rdi)
+; SSE4-NEXT:    testb $16, %al
 ; SSE4-NEXT:    je .LBB17_10
-; SSE4-NEXT:  # %bb.9: # %cond.store7
-; SSE4-NEXT:    pextrb $8, %xmm0, 4(%rdi)
-; SSE4-NEXT:  .LBB17_10: # %else8
-; SSE4-NEXT:    pextrb $10, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:  .LBB17_9: # %cond.store7
+; SSE4-NEXT:    pextrb $4, %xmm0, 4(%rdi)
+; SSE4-NEXT:    testb $32, %al
 ; SSE4-NEXT:    je .LBB17_12
-; SSE4-NEXT:  # %bb.11: # %cond.store9
-; SSE4-NEXT:    pextrb $10, %xmm0, 5(%rdi)
-; SSE4-NEXT:  .LBB17_12: # %else10
-; SSE4-NEXT:    pxor %xmm2, %xmm2
-; SSE4-NEXT:    pcmpeqw %xmm2, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE4-NEXT:    pxor %xmm1, %xmm2
-; SSE4-NEXT:    pextrb $12, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:  .LBB17_11: # %cond.store9
+; SSE4-NEXT:    pextrb $5, %xmm0, 5(%rdi)
+; SSE4-NEXT:    testb $64, %al
 ; SSE4-NEXT:    je .LBB17_14
-; SSE4-NEXT:  # %bb.13: # %cond.store11
-; SSE4-NEXT:    pextrb $12, %xmm0, 6(%rdi)
-; SSE4-NEXT:  .LBB17_14: # %else12
-; SSE4-NEXT:    pextrb $14, %xmm2, %eax
-; SSE4-NEXT:    testb $1, %al
+; SSE4-NEXT:  .LBB17_13: # %cond.store11
+; SSE4-NEXT:    pextrb $6, %xmm0, 6(%rdi)
+; SSE4-NEXT:    testb $-128, %al
 ; SSE4-NEXT:    je .LBB17_16
-; SSE4-NEXT:  # %bb.15: # %cond.store13
-; SSE4-NEXT:    pextrb $14, %xmm0, 7(%rdi)
-; SSE4-NEXT:  .LBB17_16: # %else14
+; SSE4-NEXT:  .LBB17_15: # %cond.store13
+; SSE4-NEXT:    pextrb $7, %xmm0, 7(%rdi)
 ; SSE4-NEXT:    retq
 ;
 ; AVX-LABEL: truncstore_v8i16_v8i8:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm2
-; AVX-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX-NEXT:    vpxor %xmm3, %xmm2, %xmm2
 ; AVX-NEXT:    vpminuw {{.*}}(%rip), %xmm0, %xmm0
-; AVX-NEXT:    vpextrb $0, %xmm2, %eax
+; AVX-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm1
+; AVX-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX-NEXT:    vpxor %xmm2, %xmm1, %xmm1
+; AVX-NEXT:    vpacksswb %xmm0, %xmm1, %xmm1
+; AVX-NEXT:    vpmovmskb %xmm1, %eax
 ; AVX-NEXT:    testb $1, %al
-; AVX-NEXT:    je .LBB17_2
-; AVX-NEXT:  # %bb.1: # %cond.store
+; AVX-NEXT:    jne .LBB17_1
+; AVX-NEXT:  # %bb.2: # %else
+; AVX-NEXT:    testb $2, %al
+; AVX-NEXT:    jne .LBB17_3
+; AVX-NEXT:  .LBB17_4: # %else2
+; AVX-NEXT:    testb $4, %al
+; AVX-NEXT:    jne .LBB17_5
+; AVX-NEXT:  .LBB17_6: # %else4
+; AVX-NEXT:    testb $8, %al
+; AVX-NEXT:    jne .LBB17_7
+; AVX-NEXT:  .LBB17_8: # %else6
+; AVX-NEXT:    testb $16, %al
+; AVX-NEXT:    jne .LBB17_9
+; AVX-NEXT:  .LBB17_10: # %else8
+; AVX-NEXT:    testb $32, %al
+; AVX-NEXT:    jne .LBB17_11
+; AVX-NEXT:  .LBB17_12: # %else10
+; AVX-NEXT:    testb $64, %al
+; AVX-NEXT:    jne .LBB17_13
+; AVX-NEXT:  .LBB17_14: # %else12
+; AVX-NEXT:    testb $-128, %al
+; AVX-NEXT:    jne .LBB17_15
+; AVX-NEXT:  .LBB17_16: # %else14
+; AVX-NEXT:    retq
+; AVX-NEXT:  .LBB17_1: # %cond.store
 ; AVX-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX-NEXT:  .LBB17_2: # %else
-; AVX-NEXT:    vpextrb $2, %xmm2, %eax
-; AVX-NEXT:    testb $1, %al
+; AVX-NEXT:    testb $2, %al
 ; AVX-NEXT:    je .LBB17_4
-; AVX-NEXT:  # %bb.3: # %cond.store1
-; AVX-NEXT:    vpextrb $2, %xmm0, 1(%rdi)
-; AVX-NEXT:  .LBB17_4: # %else2
-; AVX-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm2
-; AVX-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX-NEXT:    testb $1, %al
+; AVX-NEXT:  .LBB17_3: # %cond.store1
+; AVX-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
+; AVX-NEXT:    testb $4, %al
 ; AVX-NEXT:    je .LBB17_6
-; AVX-NEXT:  # %bb.5: # %cond.store3
-; AVX-NEXT:    vpextrb $4, %xmm0, 2(%rdi)
-; AVX-NEXT:  .LBB17_6: # %else4
-; AVX-NEXT:    vpextrb $6, %xmm2, %eax
-; AVX-NEXT:    testb $1, %al
+; AVX-NEXT:  .LBB17_5: # %cond.store3
+; AVX-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
+; AVX-NEXT:    testb $8, %al
 ; AVX-NEXT:    je .LBB17_8
-; AVX-NEXT:  # %bb.7: # %cond.store5
-; AVX-NEXT:    vpextrb $6, %xmm0, 3(%rdi)
-; AVX-NEXT:  .LBB17_8: # %else6
-; AVX-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm2
-; AVX-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX-NEXT:    vpextrb $8, %xmm2, %eax
-; AVX-NEXT:    testb $1, %al
+; AVX-NEXT:  .LBB17_7: # %cond.store5
+; AVX-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
+; AVX-NEXT:    testb $16, %al
 ; AVX-NEXT:    je .LBB17_10
-; AVX-NEXT:  # %bb.9: # %cond.store7
-; AVX-NEXT:    vpextrb $8, %xmm0, 4(%rdi)
-; AVX-NEXT:  .LBB17_10: # %else8
-; AVX-NEXT:    vpextrb $10, %xmm2, %eax
-; AVX-NEXT:    testb $1, %al
+; AVX-NEXT:  .LBB17_9: # %cond.store7
+; AVX-NEXT:    vpextrb $4, %xmm0, 4(%rdi)
+; AVX-NEXT:    testb $32, %al
 ; AVX-NEXT:    je .LBB17_12
-; AVX-NEXT:  # %bb.11: # %cond.store9
-; AVX-NEXT:    vpextrb $10, %xmm0, 5(%rdi)
-; AVX-NEXT:  .LBB17_12: # %else10
-; AVX-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm1
-; AVX-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX-NEXT:    vpextrb $12, %xmm1, %eax
-; AVX-NEXT:    testb $1, %al
+; AVX-NEXT:  .LBB17_11: # %cond.store9
+; AVX-NEXT:    vpextrb $5, %xmm0, 5(%rdi)
+; AVX-NEXT:    testb $64, %al
 ; AVX-NEXT:    je .LBB17_14
-; AVX-NEXT:  # %bb.13: # %cond.store11
-; AVX-NEXT:    vpextrb $12, %xmm0, 6(%rdi)
-; AVX-NEXT:  .LBB17_14: # %else12
-; AVX-NEXT:    vpextrb $14, %xmm1, %eax
-; AVX-NEXT:    testb $1, %al
+; AVX-NEXT:  .LBB17_13: # %cond.store11
+; AVX-NEXT:    vpextrb $6, %xmm0, 6(%rdi)
+; AVX-NEXT:    testb $-128, %al
 ; AVX-NEXT:    je .LBB17_16
-; AVX-NEXT:  # %bb.15: # %cond.store13
-; AVX-NEXT:    vpextrb $14, %xmm0, 7(%rdi)
-; AVX-NEXT:  .LBB17_16: # %else14
+; AVX-NEXT:  .LBB17_15: # %cond.store13
+; AVX-NEXT:    vpextrb $7, %xmm0, 7(%rdi)
 ; AVX-NEXT:    retq
 ;
 ; AVX512F-LABEL: truncstore_v8i16_v8i8:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
-; AVX512F-NEXT:    vpmovsxwq %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmq %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm1
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
+; AVX512F-NEXT:    vpmovsxwq %xmm1, %zmm1
+; AVX512F-NEXT:    vptestmq %zmm1, %zmm1, %k0
 ; AVX512F-NEXT:    vpminuw {{.*}}(%rip), %xmm0, %xmm0
+; AVX512F-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
 ; AVX512F-NEXT:    kmovw %k0, %eax
 ; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB17_2
-; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    jne .LBB17_1
+; AVX512F-NEXT:  # %bb.2: # %else
+; AVX512F-NEXT:    testb $2, %al
+; AVX512F-NEXT:    jne .LBB17_3
+; AVX512F-NEXT:  .LBB17_4: # %else2
+; AVX512F-NEXT:    testb $4, %al
+; AVX512F-NEXT:    jne .LBB17_5
+; AVX512F-NEXT:  .LBB17_6: # %else4
+; AVX512F-NEXT:    testb $8, %al
+; AVX512F-NEXT:    jne .LBB17_7
+; AVX512F-NEXT:  .LBB17_8: # %else6
+; AVX512F-NEXT:    testb $16, %al
+; AVX512F-NEXT:    jne .LBB17_9
+; AVX512F-NEXT:  .LBB17_10: # %else8
+; AVX512F-NEXT:    testb $32, %al
+; AVX512F-NEXT:    jne .LBB17_11
+; AVX512F-NEXT:  .LBB17_12: # %else10
+; AVX512F-NEXT:    testb $64, %al
+; AVX512F-NEXT:    jne .LBB17_13
+; AVX512F-NEXT:  .LBB17_14: # %else12
+; AVX512F-NEXT:    testb $-128, %al
+; AVX512F-NEXT:    jne .LBB17_15
+; AVX512F-NEXT:  .LBB17_16: # %else14
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+; AVX512F-NEXT:  .LBB17_1: # %cond.store
 ; AVX512F-NEXT:    vpextrb $0, %xmm0, (%rdi)
-; AVX512F-NEXT:  .LBB17_2: # %else
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
-; AVX512F-NEXT:    vpmovsxwq %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmq %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    testb $2, %al
 ; AVX512F-NEXT:    je .LBB17_4
-; AVX512F-NEXT:  # %bb.3: # %cond.store1
-; AVX512F-NEXT:    vpextrb $2, %xmm0, 1(%rdi)
-; AVX512F-NEXT:  .LBB17_4: # %else2
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
-; AVX512F-NEXT:    vpmovsxwq %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmq %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  .LBB17_3: # %cond.store1
+; AVX512F-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
+; AVX512F-NEXT:    testb $4, %al
 ; AVX512F-NEXT:    je .LBB17_6
-; AVX512F-NEXT:  # %bb.5: # %cond.store3
-; AVX512F-NEXT:    vpextrb $4, %xmm0, 2(%rdi)
-; AVX512F-NEXT:  .LBB17_6: # %else4
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
-; AVX512F-NEXT:    vpmovsxwq %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmq %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  .LBB17_5: # %cond.store3
+; AVX512F-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
+; AVX512F-NEXT:    testb $8, %al
 ; AVX512F-NEXT:    je .LBB17_8
-; AVX512F-NEXT:  # %bb.7: # %cond.store5
-; AVX512F-NEXT:    vpextrb $6, %xmm0, 3(%rdi)
-; AVX512F-NEXT:  .LBB17_8: # %else6
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
-; AVX512F-NEXT:    vpmovsxwq %xmm3, %zmm3
-; AVX512F-NEXT:    vptestmq %zmm3, %zmm3, %k0
-; AVX512F-NEXT:    kshiftrw $4, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  .LBB17_7: # %cond.store5
+; AVX512F-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
+; AVX512F-NEXT:    testb $16, %al
 ; AVX512F-NEXT:    je .LBB17_10
-; AVX512F-NEXT:  # %bb.9: # %cond.store7
-; AVX512F-NEXT:    vpextrb $8, %xmm0, 4(%rdi)
-; AVX512F-NEXT:  .LBB17_10: # %else8
-; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
-; AVX512F-NEXT:    vpmovsxwq %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmq %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $5, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  .LBB17_9: # %cond.store7
+; AVX512F-NEXT:    vpextrb $4, %xmm0, 4(%rdi)
+; AVX512F-NEXT:    testb $32, %al
 ; AVX512F-NEXT:    je .LBB17_12
-; AVX512F-NEXT:  # %bb.11: # %cond.store9
-; AVX512F-NEXT:    vpextrb $10, %xmm0, 5(%rdi)
-; AVX512F-NEXT:  .LBB17_12: # %else10
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm1
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm2
-; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm2
-; AVX512F-NEXT:    vpmovsxwq %xmm2, %zmm2
-; AVX512F-NEXT:    vptestmq %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kshiftrw $6, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  .LBB17_11: # %cond.store9
+; AVX512F-NEXT:    vpextrb $5, %xmm0, 5(%rdi)
+; AVX512F-NEXT:    testb $64, %al
 ; AVX512F-NEXT:    je .LBB17_14
-; AVX512F-NEXT:  # %bb.13: # %cond.store11
-; AVX512F-NEXT:    vpextrb $12, %xmm0, 6(%rdi)
-; AVX512F-NEXT:  .LBB17_14: # %else12
-; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
-; AVX512F-NEXT:    vpmovsxwq %xmm1, %zmm1
-; AVX512F-NEXT:    vptestmq %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kshiftrw $7, %k0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:  .LBB17_13: # %cond.store11
+; AVX512F-NEXT:    vpextrb $6, %xmm0, 6(%rdi)
+; AVX512F-NEXT:    testb $-128, %al
 ; AVX512F-NEXT:    je .LBB17_16
-; AVX512F-NEXT:  # %bb.15: # %cond.store13
-; AVX512F-NEXT:    vpextrb $14, %xmm0, 7(%rdi)
-; AVX512F-NEXT:  .LBB17_16: # %else14
+; AVX512F-NEXT:  .LBB17_15: # %cond.store13
+; AVX512F-NEXT:    vpextrb $7, %xmm0, 7(%rdi)
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: truncstore_v8i16_v8i8:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; AVX512BW-NEXT:    vptestmw %zmm1, %zmm1, %k0
-; AVX512BW-NEXT:    vpminuw {{.*}}(%rip), %xmm0, %xmm0
-; AVX512BW-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
 ; AVX512BW-NEXT:    kshiftlq $56, %k0, %k0
 ; AVX512BW-NEXT:    kshiftrq $56, %k0, %k1
+; AVX512BW-NEXT:    vpmovuswb %zmm0, %ymm0
 ; AVX512BW-NEXT:    vmovdqu8 %zmm0, (%rdi) {%k1}
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/maskmovdqu.ll b/llvm/test/CodeGen/X86/maskmovdqu.ll
index 2f13c535e..14d356ffb 100644
--- a/llvm/test/CodeGen/X86/maskmovdqu.ll
+++ b/llvm/test/CodeGen/X86/maskmovdqu.ll
@@ -1,10 +1,38 @@
-; RUN: llc < %s -mtriple=i686--    -mattr=+sse2,-avx | grep -i edi
-; RUN: llc < %s -mtriple=x86_64-- -mattr=+sse2,-avx | grep -i rdi
-; RUN: llc < %s -mtriple=i686--    -mattr=+avx | grep -i edi
-; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx | grep -i rdi
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686--    -mattr=+sse2,-avx | FileCheck %s --check-prefixes=ALL,i686_SSE2
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+sse2,-avx | FileCheck %s --check-prefixes=ALL,x86_64_SSE2
+; RUN: llc < %s -mtriple=i686--    -mattr=+avx | FileCheck %s --check-prefixes=ALL,i686_AVX
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx | FileCheck %s --check-prefixes=ALL,x86_64_AVX
 ; rdar://6573467
 
 define void @test(<16 x i8> %a, <16 x i8> %b, i32 %dummy, i8* %c) nounwind {
+; i686_SSE2-LABEL: test:
+; i686_SSE2:       # %bb.0: # %entry
+; i686_SSE2-NEXT:    pushl %edi
+; i686_SSE2-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; i686_SSE2-NEXT:    maskmovdqu %xmm1, %xmm0
+; i686_SSE2-NEXT:    popl %edi
+; i686_SSE2-NEXT:    retl
+;
+; x86_64_SSE2-LABEL: test:
+; x86_64_SSE2:       # %bb.0: # %entry
+; x86_64_SSE2-NEXT:    movq %rsi, %rdi
+; x86_64_SSE2-NEXT:    maskmovdqu %xmm1, %xmm0
+; x86_64_SSE2-NEXT:    retq
+;
+; i686_AVX-LABEL: test:
+; i686_AVX:       # %bb.0: # %entry
+; i686_AVX-NEXT:    pushl %edi
+; i686_AVX-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; i686_AVX-NEXT:    vmaskmovdqu %xmm1, %xmm0
+; i686_AVX-NEXT:    popl %edi
+; i686_AVX-NEXT:    retl
+;
+; x86_64_AVX-LABEL: test:
+; x86_64_AVX:       # %bb.0: # %entry
+; x86_64_AVX-NEXT:    movq %rsi, %rdi
+; x86_64_AVX-NEXT:    vmaskmovdqu %xmm1, %xmm0
+; x86_64_AVX-NEXT:    retq
 entry:
 	tail call void @llvm.x86.sse2.maskmov.dqu( <16 x i8> %a, <16 x i8> %b, i8* %c )
 	ret void
diff --git a/llvm/test/CodeGen/X86/materialize.ll b/llvm/test/CodeGen/X86/materialize.ll
index 6e1264b4f..429d578fe 100644
--- a/llvm/test/CodeGen/X86/materialize.ll
+++ b/llvm/test/CodeGen/X86/materialize.ll
@@ -30,6 +30,21 @@ entry:
 ; CHECK64-NEXT:  retq
 }
 
+define i32 @one32_pgso() !prof !14 {
+entry:
+  ret i32 1
+
+; CHECK32-LABEL: one32_pgso:
+; CHECK32:       xorl %eax, %eax
+; CHECK32-NEXT:  incl %eax
+; CHECK32-NEXT:  retl
+
+; FIXME: Figure out the best approach in 64-bit mode.
+; CHECK64-LABEL: one32_pgso:
+; CHECK64:       movl $1, %eax
+; CHECK64-NEXT:  retq
+}
+
 define i32 @one32_minsize() minsize {
 entry:
   ret i32 1
@@ -107,6 +122,16 @@ entry:
 ; CHECK32-NEXT:  retl
 }
 
+define i32 @minus_one32_pgso() !prof !14 {
+entry:
+  ret i32 -1
+
+; CHECK32-LABEL: minus_one32_pgso:
+; CHECK32:       xorl %eax, %eax
+; CHECK32-NEXT:  decl %eax
+; CHECK32-NEXT:  retl
+}
+
 define i32 @minus_one32_minsize() minsize {
 entry:
   ret i32 -1
@@ -140,6 +165,28 @@ entry:
 ; CHECK32-NEXT:  retl
 }
 
+define i16 @one16_pgso() !prof !14 {
+entry:
+  ret i16 1
+
+; CHECK32-LABEL: one16_pgso:
+; CHECK32:       xorl %eax, %eax
+; CHECK32-NEXT:  incl %eax
+; CHECK32-NEXT:  # kill
+; CHECK32-NEXT:  retl
+}
+
+define i16 @minus_one16_pgso() !prof !14 {
+entry:
+  ret i16 -1
+
+; CHECK32-LABEL: minus_one16_pgso:
+; CHECK32:       xorl %eax, %eax
+; CHECK32-NEXT:  decl %eax
+; CHECK32-NEXT:  # kill
+; CHECK32-NEXT:  retl
+}
+
 define i32 @minus_five32() minsize {
 entry:
   ret i32 -5
@@ -213,4 +260,72 @@ entry:
 ; CHECK32:       retl
 }
 
+define i32 @rematerialize_minus_one_pgso() !prof !14 {
+entry:
+  ; Materialize -1 (thiscall forces it into %ecx).
+  tail call x86_thiscallcc void @f(i32 -1)
+
+  ; Clobber all registers except %esp, leaving nowhere to store the -1 besides
+  ; spilling it to the stack.
+  tail call void asm sideeffect "", "~{eax},~{ebx},~{ecx},~{edx},~{edi},~{esi},~{ebp},~{dirflag},~{fpsr},~{flags}"()
+
+  ; -1 should be re-materialized here instead of getting spilled above.
+  ret i32 -1
+
+; CHECK32-LABEL: rematerialize_minus_one_pgso
+; CHECK32:       xorl %ecx, %ecx
+; CHECK32-NEXT:  decl %ecx
+; CHECK32:       calll
+; CHECK32:       xorl %eax, %eax
+; CHECK32-NEXT:  decl %eax
+; CHECK32-NOT:   %eax
+; CHECK32:       retl
+}
+
+define i32 @rematerialize_minus_one_eflags_pgso(i32 %x) !prof !14 {
+entry:
+  ; Materialize -1 (thiscall forces it into %ecx).
+  tail call x86_thiscallcc void @f(i32 -1)
+
+  ; Clobber all registers except %esp, leaving nowhere to store the -1 besides
+  ; spilling it to the stack.
+  tail call void asm sideeffect "", "~{eax},~{ebx},~{ecx},~{edx},~{edi},~{esi},~{ebp},~{dirflag},~{fpsr},~{flags}"()
+
+  ; Define eflags.
+  %a = icmp ne i32 %x, 123
+  %b = zext i1 %a to i32
+  ; Cause -1 to be rematerialized right in front of the cmov, which needs eflags.
+  ; It must therefore not use the xor-dec lowering.
+  %c = select i1 %a, i32 %b, i32 -1
+  ret i32 %c
+
+; CHECK32-LABEL: rematerialize_minus_one_eflags_pgso
+; CHECK32:       xorl %ecx, %ecx
+; CHECK32-NEXT:  decl %ecx
+; CHECK32:       calll
+; CHECK32:       cmpl
+; CHECK32:       setne
+; CHECK32-NOT:   xorl
+; CHECK32:       movl $-1
+; CHECK32:       cmov
+; CHECK32:       retl
+}
+
 declare x86_thiscallcc void @f(i32)
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 1, !"ProfileSummary", !1}
+!1 = !{!2, !3, !4, !5, !6, !7, !8, !9}
+!2 = !{!"ProfileFormat", !"InstrProf"}
+!3 = !{!"TotalCount", i64 10000}
+!4 = !{!"MaxCount", i64 10}
+!5 = !{!"MaxInternalCount", i64 1}
+!6 = !{!"MaxFunctionCount", i64 1000}
+!7 = !{!"NumCounts", i64 3}
+!8 = !{!"NumFunctions", i64 3}
+!9 = !{!"DetailedSummary", !10}
+!10 = !{!11, !12, !13}
+!11 = !{i32 10000, i64 100, i32 1}
+!12 = !{i32 999000, i64 100, i32 1}
+!13 = !{i32 999999, i64 1, i32 2}
+!14 = !{!"function_entry_count", i64 0}
diff --git a/llvm/test/CodeGen/X86/memcmp-minsize.ll b/llvm/test/CodeGen/X86/memcmp-minsize.ll
index a1ab4e130..01e6085fb 100644
--- a/llvm/test/CodeGen/X86/memcmp-minsize.ll
+++ b/llvm/test/CodeGen/X86/memcmp-minsize.ll
@@ -2,7 +2,8 @@
 ; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=cmov | FileCheck %s --check-prefix=X86 --check-prefix=X86-NOSSE
 ; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=X86 --check-prefix=X86-SSE2
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s --check-prefix=X64 --check-prefix=X64-SSE2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx2 | FileCheck %s --check-prefix=X64 --check-prefix=X64-AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx | FileCheck %s --check-prefix=X64 --check-prefix=X64-AVX --check-prefix=X64-AVX1
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx2 | FileCheck %s --check-prefix=X64 --check-prefix=X64-AVX --check-prefix=X64-AVX2
 
 ; This tests codegen time inlining/optimization of memcmp
 ; rdar://6480398
@@ -426,14 +427,13 @@ define i1 @length16_eq(i8* %x, i8* %y) nounwind minsize {
 ; X64-SSE2-NEXT:    setne %al
 ; X64-SSE2-NEXT:    retq
 ;
-; X64-AVX2-LABEL: length16_eq:
-; X64-AVX2:       # %bb.0:
-; X64-AVX2-NEXT:    vmovdqu (%rdi), %xmm0
-; X64-AVX2-NEXT:    vpcmpeqb (%rsi), %xmm0, %xmm0
-; X64-AVX2-NEXT:    vpmovmskb %xmm0, %eax
-; X64-AVX2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
-; X64-AVX2-NEXT:    setne %al
-; X64-AVX2-NEXT:    retq
+; X64-AVX-LABEL: length16_eq:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    vmovdqu (%rdi), %xmm0
+; X64-AVX-NEXT:    vpxor (%rsi), %xmm0, %xmm0
+; X64-AVX-NEXT:    vptest %xmm0, %xmm0
+; X64-AVX-NEXT:    setne %al
+; X64-AVX-NEXT:    retq
   %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 16) nounwind
   %cmp = icmp ne i32 %call, 0
   ret i1 %cmp
@@ -471,14 +471,13 @@ define i1 @length16_eq_const(i8* %X) nounwind minsize {
 ; X64-SSE2-NEXT:    sete %al
 ; X64-SSE2-NEXT:    retq
 ;
-; X64-AVX2-LABEL: length16_eq_const:
-; X64-AVX2:       # %bb.0:
-; X64-AVX2-NEXT:    vmovdqu (%rdi), %xmm0
-; X64-AVX2-NEXT:    vpcmpeqb {{.*}}(%rip), %xmm0, %xmm0
-; X64-AVX2-NEXT:    vpmovmskb %xmm0, %eax
-; X64-AVX2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
-; X64-AVX2-NEXT:    sete %al
-; X64-AVX2-NEXT:    retq
+; X64-AVX-LABEL: length16_eq_const:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    vmovdqu (%rdi), %xmm0
+; X64-AVX-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; X64-AVX-NEXT:    vptest %xmm0, %xmm0
+; X64-AVX-NEXT:    sete %al
+; X64-AVX-NEXT:    retq
   %m = tail call i32 @memcmp(i8* %X, i8* getelementptr inbounds ([65 x i8], [65 x i8]* @.str, i32 0, i32 0), i64 16) nounwind
   %c = icmp eq i32 %m, 0
   ret i1 %c
@@ -609,12 +608,20 @@ define i1 @length32_eq(i8* %x, i8* %y) nounwind minsize {
 ; X64-SSE2-NEXT:    popq %rcx
 ; X64-SSE2-NEXT:    retq
 ;
+; X64-AVX1-LABEL: length32_eq:
+; X64-AVX1:       # %bb.0:
+; X64-AVX1-NEXT:    vmovups (%rdi), %ymm0
+; X64-AVX1-NEXT:    vxorps (%rsi), %ymm0, %ymm0
+; X64-AVX1-NEXT:    vptest %ymm0, %ymm
+; X64-AVX1-NEXT:    sete %al
+; X64-AVX1-NEXT:    vzeroupper
+; X64-AVX1-NEXT:    retq
+;
 ; X64-AVX2-LABEL: length32_eq:
 ; X64-AVX2:       # %bb.0:
 ; X64-AVX2-NEXT:    vmovdqu (%rdi), %ymm0
-; X64-AVX2-NEXT:    vpcmpeqb (%rsi), %ymm0, %ymm0
-; X64-AVX2-NEXT:    vpmovmskb %ymm0, %eax
-; X64-AVX2-NEXT:    cmpl $-1, %eax
+; X64-AVX2-NEXT:    vpxor (%rsi), %ymm0, %ymm0
+; X64-AVX2-NEXT:    vptest %ymm0, %ymm
 ; X64-AVX2-NEXT:    sete %al
 ; X64-AVX2-NEXT:    vzeroupper
 ; X64-AVX2-NEXT:    retq
@@ -648,12 +655,20 @@ define i1 @length32_eq_const(i8* %X) nounwind minsize {
 ; X64-SSE2-NEXT:    popq %rcx
 ; X64-SSE2-NEXT:    retq
 ;
+; X64-AVX1-LABEL: length32_eq_const:
+; X64-AVX1:       # %bb.0:
+; X64-AVX1-NEXT:    vmovups (%rdi), %ymm0
+; X64-AVX1-NEXT:    vxorps {{.*}}(%rip), %ymm0, %ymm0
+; X64-AVX1-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX1-NEXT:    setne %al
+; X64-AVX1-NEXT:    vzeroupper
+; X64-AVX1-NEXT:    retq
+;
 ; X64-AVX2-LABEL: length32_eq_const:
 ; X64-AVX2:       # %bb.0:
 ; X64-AVX2-NEXT:    vmovdqu (%rdi), %ymm0
-; X64-AVX2-NEXT:    vpcmpeqb {{.*}}(%rip), %ymm0, %ymm0
-; X64-AVX2-NEXT:    vpmovmskb %ymm0, %eax
-; X64-AVX2-NEXT:    cmpl $-1, %eax
+; X64-AVX2-NEXT:    vpxor {{.*}}(%rip), %ymm0, %ymm0
+; X64-AVX2-NEXT:    vptest %ymm0, %ymm0
 ; X64-AVX2-NEXT:    setne %al
 ; X64-AVX2-NEXT:    vzeroupper
 ; X64-AVX2-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/memcmp-more-load-pairs.ll b/llvm/test/CodeGen/X86/memcmp-more-load-pairs.ll
new file mode 100644
index 000000000..ea08e5316
--- /dev/null
+++ b/llvm/test/CodeGen/X86/memcmp-more-load-pairs.ll
@@ -0,0 +1,6241 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; NOTE: This is a copy of llvm/test/CodeGen/X86/memcmp.ll with more load pairs. Please keep it that way.
+; RUN: llc -max-loads-per-memcmp=4 -memcmp-num-loads-per-block=4 < %s -mtriple=i686-unknown-unknown -mattr=cmov     | FileCheck %s --check-prefixes=X86,X86-NOSSE
+; RUN: llc -max-loads-per-memcmp=4 -memcmp-num-loads-per-block=4 < %s -mtriple=i686-unknown-unknown -mattr=+sse     | FileCheck %s --check-prefixes=X86,X86-SSE,X86-SSE1
+; RUN: llc -max-loads-per-memcmp=4 -memcmp-num-loads-per-block=4 < %s -mtriple=i686-unknown-unknown -mattr=+sse2    | FileCheck %s --check-prefixes=X86,X86-SSE,X86-SSE2
+; RUN: llc -max-loads-per-memcmp=4 -memcmp-num-loads-per-block=4 < %s -mtriple=i686-unknown-unknown -mattr=+sse4.1  | FileCheck %s --check-prefixes=X86,X86-SSE,X86-SSE41
+; RUN: llc -max-loads-per-memcmp=4 -memcmp-num-loads-per-block=4 < %s -mtriple=x86_64-unknown-unknown               | FileCheck %s --check-prefixes=X64,X64-SSE,X64-SSE2
+; RUN: llc -max-loads-per-memcmp=4 -memcmp-num-loads-per-block=4 < %s -mtriple=x86_64-unknown-unknown -mattr=sse4.1 | FileCheck %s --check-prefixes=X64,X64-SSE,X64-SSE41
+; RUN: llc -max-loads-per-memcmp=4 -memcmp-num-loads-per-block=4 < %s -mtriple=x86_64-unknown-unknown -mattr=avx    | FileCheck %s --check-prefixes=X64,X64-AVX,X64-AVX1
+; RUN: llc -max-loads-per-memcmp=4 -memcmp-num-loads-per-block=4 < %s -mtriple=x86_64-unknown-unknown -mattr=avx2   | FileCheck %s --check-prefixes=X64,X64-AVX,X64-AVX2
+; RUN: llc -max-loads-per-memcmp=4 -memcmp-num-loads-per-block=4 < %s -mtriple=x86_64-unknown-unknown -mattr=avx512bw,+prefer-256-bit | FileCheck %s --check-prefixes=X64,X64-AVX,X64-AVX2
+; RUN: llc -max-loads-per-memcmp=4 -memcmp-num-loads-per-block=4 < %s -mtriple=x86_64-unknown-unknown -mattr=avx512bw,-prefer-256-bit | FileCheck %s --check-prefixes=X64,X64-AVX,X64-AVX512,X64-AVX512BW
+; RUN: llc -max-loads-per-memcmp=4 -memcmp-num-loads-per-block=4 < %s -mtriple=x86_64-unknown-unknown -mattr=avx512f,+prefer-256-bit,-prefer-mask-registers | FileCheck %s --check-prefixes=X64,X64-AVX,X64-AVX2
+; RUN: llc -max-loads-per-memcmp=4 -memcmp-num-loads-per-block=4 < %s -mtriple=x86_64-unknown-unknown -mattr=avx512f,-prefer-256-bit,-prefer-mask-registers | FileCheck %s --check-prefixes=X64,X64-AVX,X64-AVX512,X64-AVX512F
+; RUN: llc -max-loads-per-memcmp=4 -memcmp-num-loads-per-block=4 < %s -mtriple=x86_64-unknown-unknown -mattr=avx512f,+prefer-256-bit,+prefer-mask-registers | FileCheck %s --check-prefixes=X64,X64-MIC-AVX,X64-MIC-AVX2
+; RUN: llc -max-loads-per-memcmp=4 -memcmp-num-loads-per-block=4 < %s -mtriple=x86_64-unknown-unknown -mattr=avx512f,-prefer-256-bit,+prefer-mask-registers | FileCheck %s --check-prefixes=X64,X64-MIC-AVX,X64-MIC-AVX512F
+
+; This tests codegen time inlining/optimization of memcmp
+; rdar://6480398
+
+@.str = private constant [513 x i8] c"01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901\00", align 1
+
+declare i32 @memcmp(i8*, i8*, i64)
+
+define i32 @length0(i8* %X, i8* %Y) nounwind {
+; X86-LABEL: length0:
+; X86:       # %bb.0:
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: length0:
+; X64:       # %bb.0:
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    retq
+   %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 0) nounwind
+   ret i32 %m
+ }
+
+define i1 @length0_eq(i8* %X, i8* %Y) nounwind {
+; X86-LABEL: length0_eq:
+; X86:       # %bb.0:
+; X86-NEXT:    movb $1, %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length0_eq:
+; X64:       # %bb.0:
+; X64-NEXT:    movb $1, %al
+; X64-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 0) nounwind
+  %c = icmp eq i32 %m, 0
+  ret i1 %c
+}
+
+define i1 @length0_lt(i8* %X, i8* %Y) nounwind {
+; X86-LABEL: length0_lt:
+; X86:       # %bb.0:
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: length0_lt:
+; X64:       # %bb.0:
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 0) nounwind
+  %c = icmp slt i32 %m, 0
+  ret i1 %c
+}
+
+define i32 @length2(i8* %X, i8* %Y) nounwind {
+; X86-LABEL: length2:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movzwl (%ecx), %ecx
+; X86-NEXT:    movzwl (%eax), %edx
+; X86-NEXT:    rolw $8, %cx
+; X86-NEXT:    rolw $8, %dx
+; X86-NEXT:    movzwl %cx, %eax
+; X86-NEXT:    movzwl %dx, %ecx
+; X86-NEXT:    subl %ecx, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: length2:
+; X64:       # %bb.0:
+; X64-NEXT:    movzwl (%rdi), %eax
+; X64-NEXT:    movzwl (%rsi), %ecx
+; X64-NEXT:    rolw $8, %ax
+; X64-NEXT:    rolw $8, %cx
+; X64-NEXT:    movzwl %ax, %eax
+; X64-NEXT:    movzwl %cx, %ecx
+; X64-NEXT:    subl %ecx, %eax
+; X64-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 2) nounwind
+  ret i32 %m
+}
+
+define i1 @length2_eq(i8* %X, i8* %Y) nounwind {
+; X86-LABEL: length2_eq:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movzwl (%ecx), %ecx
+; X86-NEXT:    cmpw (%eax), %cx
+; X86-NEXT:    sete %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length2_eq:
+; X64:       # %bb.0:
+; X64-NEXT:    movzwl (%rdi), %eax
+; X64-NEXT:    cmpw (%rsi), %ax
+; X64-NEXT:    sete %al
+; X64-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 2) nounwind
+  %c = icmp eq i32 %m, 0
+  ret i1 %c
+}
+
+define i1 @length2_lt(i8* %X, i8* %Y) nounwind {
+; X86-LABEL: length2_lt:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movzwl (%ecx), %ecx
+; X86-NEXT:    movzwl (%eax), %edx
+; X86-NEXT:    rolw $8, %cx
+; X86-NEXT:    rolw $8, %dx
+; X86-NEXT:    movzwl %cx, %eax
+; X86-NEXT:    movzwl %dx, %ecx
+; X86-NEXT:    subl %ecx, %eax
+; X86-NEXT:    shrl $31, %eax
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: length2_lt:
+; X64:       # %bb.0:
+; X64-NEXT:    movzwl (%rdi), %eax
+; X64-NEXT:    movzwl (%rsi), %ecx
+; X64-NEXT:    rolw $8, %ax
+; X64-NEXT:    rolw $8, %cx
+; X64-NEXT:    movzwl %ax, %eax
+; X64-NEXT:    movzwl %cx, %ecx
+; X64-NEXT:    subl %ecx, %eax
+; X64-NEXT:    shrl $31, %eax
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 2) nounwind
+  %c = icmp slt i32 %m, 0
+  ret i1 %c
+}
+
+define i1 @length2_gt(i8* %X, i8* %Y) nounwind {
+; X86-LABEL: length2_gt:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movzwl (%ecx), %ecx
+; X86-NEXT:    movzwl (%eax), %eax
+; X86-NEXT:    rolw $8, %cx
+; X86-NEXT:    rolw $8, %ax
+; X86-NEXT:    movzwl %cx, %ecx
+; X86-NEXT:    movzwl %ax, %eax
+; X86-NEXT:    subl %eax, %ecx
+; X86-NEXT:    testl %ecx, %ecx
+; X86-NEXT:    setg %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length2_gt:
+; X64:       # %bb.0:
+; X64-NEXT:    movzwl (%rdi), %eax
+; X64-NEXT:    movzwl (%rsi), %ecx
+; X64-NEXT:    rolw $8, %ax
+; X64-NEXT:    rolw $8, %cx
+; X64-NEXT:    movzwl %ax, %eax
+; X64-NEXT:    movzwl %cx, %ecx
+; X64-NEXT:    subl %ecx, %eax
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    setg %al
+; X64-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 2) nounwind
+  %c = icmp sgt i32 %m, 0
+  ret i1 %c
+}
+
+define i1 @length2_eq_const(i8* %X) nounwind {
+; X86-LABEL: length2_eq_const:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movzwl (%eax), %eax
+; X86-NEXT:    cmpl $12849, %eax # imm = 0x3231
+; X86-NEXT:    setne %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length2_eq_const:
+; X64:       # %bb.0:
+; X64-NEXT:    movzwl (%rdi), %eax
+; X64-NEXT:    cmpl $12849, %eax # imm = 0x3231
+; X64-NEXT:    setne %al
+; X64-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* getelementptr inbounds ([513 x i8], [513 x i8]* @.str, i32 0, i32 1), i64 2) nounwind
+  %c = icmp ne i32 %m, 0
+  ret i1 %c
+}
+
+define i1 @length2_eq_nobuiltin_attr(i8* %X, i8* %Y) nounwind {
+; X86-LABEL: length2_eq_nobuiltin_attr:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $2
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    sete %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length2_eq_nobuiltin_attr:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $2, %edx
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    sete %al
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 2) nounwind nobuiltin
+  %c = icmp eq i32 %m, 0
+  ret i1 %c
+}
+
+define i32 @length3(i8* %X, i8* %Y) nounwind {
+; X86-LABEL: length3:
+; X86:       # %bb.0: # %loadbb
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movzwl (%eax), %edx
+; X86-NEXT:    movzwl (%ecx), %esi
+; X86-NEXT:    rolw $8, %dx
+; X86-NEXT:    rolw $8, %si
+; X86-NEXT:    cmpw %si, %dx
+; X86-NEXT:    jne .LBB9_1
+; X86-NEXT:  # %bb.2: # %loadbb1
+; X86-NEXT:    movzbl 2(%eax), %eax
+; X86-NEXT:    movzbl 2(%ecx), %ecx
+; X86-NEXT:    subl %ecx, %eax
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
+; X86-NEXT:  .LBB9_1: # %res_block
+; X86-NEXT:    setae %al
+; X86-NEXT:    movzbl %al, %eax
+; X86-NEXT:    leal -1(%eax,%eax), %eax
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
+;
+; X64-LABEL: length3:
+; X64:       # %bb.0: # %loadbb
+; X64-NEXT:    movzwl (%rdi), %eax
+; X64-NEXT:    movzwl (%rsi), %ecx
+; X64-NEXT:    rolw $8, %ax
+; X64-NEXT:    rolw $8, %cx
+; X64-NEXT:    cmpw %cx, %ax
+; X64-NEXT:    jne .LBB9_1
+; X64-NEXT:  # %bb.2: # %loadbb1
+; X64-NEXT:    movzbl 2(%rdi), %eax
+; X64-NEXT:    movzbl 2(%rsi), %ecx
+; X64-NEXT:    subl %ecx, %eax
+; X64-NEXT:    retq
+; X64-NEXT:  .LBB9_1: # %res_block
+; X64-NEXT:    setae %al
+; X64-NEXT:    movzbl %al, %eax
+; X64-NEXT:    leal -1(%rax,%rax), %eax
+; X64-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 3) nounwind
+  ret i32 %m
+}
+
+define i1 @length3_eq(i8* %X, i8* %Y) nounwind {
+; X86-LABEL: length3_eq:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movzwl (%ecx), %edx
+; X86-NEXT:    xorw (%eax), %dx
+; X86-NEXT:    movb 2(%ecx), %cl
+; X86-NEXT:    xorb 2(%eax), %cl
+; X86-NEXT:    movzbl %cl, %eax
+; X86-NEXT:    orw %dx, %ax
+; X86-NEXT:    setne %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length3_eq:
+; X64:       # %bb.0:
+; X64-NEXT:    movzwl (%rdi), %eax
+; X64-NEXT:    xorw (%rsi), %ax
+; X64-NEXT:    movb 2(%rdi), %cl
+; X64-NEXT:    xorb 2(%rsi), %cl
+; X64-NEXT:    movzbl %cl, %ecx
+; X64-NEXT:    orw %ax, %cx
+; X64-NEXT:    setne %al
+; X64-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 3) nounwind
+  %c = icmp ne i32 %m, 0
+  ret i1 %c
+}
+
+define i32 @length4(i8* %X, i8* %Y) nounwind {
+; X86-LABEL: length4:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl (%ecx), %ecx
+; X86-NEXT:    movl (%eax), %edx
+; X86-NEXT:    bswapl %ecx
+; X86-NEXT:    bswapl %edx
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    cmpl %edx, %ecx
+; X86-NEXT:    seta %al
+; X86-NEXT:    sbbl $0, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: length4:
+; X64:       # %bb.0:
+; X64-NEXT:    movl (%rdi), %ecx
+; X64-NEXT:    movl (%rsi), %edx
+; X64-NEXT:    bswapl %ecx
+; X64-NEXT:    bswapl %edx
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpl %edx, %ecx
+; X64-NEXT:    seta %al
+; X64-NEXT:    sbbl $0, %eax
+; X64-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 4) nounwind
+  ret i32 %m
+}
+
+define i1 @length4_eq(i8* %X, i8* %Y) nounwind {
+; X86-LABEL: length4_eq:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl (%ecx), %ecx
+; X86-NEXT:    cmpl (%eax), %ecx
+; X86-NEXT:    setne %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length4_eq:
+; X64:       # %bb.0:
+; X64-NEXT:    movl (%rdi), %eax
+; X64-NEXT:    cmpl (%rsi), %eax
+; X64-NEXT:    setne %al
+; X64-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 4) nounwind
+  %c = icmp ne i32 %m, 0
+  ret i1 %c
+}
+
+define i1 @length4_lt(i8* %X, i8* %Y) nounwind {
+; X86-LABEL: length4_lt:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl (%ecx), %ecx
+; X86-NEXT:    movl (%eax), %edx
+; X86-NEXT:    bswapl %ecx
+; X86-NEXT:    bswapl %edx
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    cmpl %edx, %ecx
+; X86-NEXT:    seta %al
+; X86-NEXT:    sbbl $0, %eax
+; X86-NEXT:    shrl $31, %eax
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: length4_lt:
+; X64:       # %bb.0:
+; X64-NEXT:    movl (%rdi), %ecx
+; X64-NEXT:    movl (%rsi), %edx
+; X64-NEXT:    bswapl %ecx
+; X64-NEXT:    bswapl %edx
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpl %edx, %ecx
+; X64-NEXT:    seta %al
+; X64-NEXT:    sbbl $0, %eax
+; X64-NEXT:    shrl $31, %eax
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 4) nounwind
+  %c = icmp slt i32 %m, 0
+  ret i1 %c
+}
+
+define i1 @length4_gt(i8* %X, i8* %Y) nounwind {
+; X86-LABEL: length4_gt:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl (%ecx), %ecx
+; X86-NEXT:    movl (%eax), %eax
+; X86-NEXT:    bswapl %ecx
+; X86-NEXT:    bswapl %eax
+; X86-NEXT:    xorl %edx, %edx
+; X86-NEXT:    cmpl %eax, %ecx
+; X86-NEXT:    seta %dl
+; X86-NEXT:    sbbl $0, %edx
+; X86-NEXT:    testl %edx, %edx
+; X86-NEXT:    setg %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length4_gt:
+; X64:       # %bb.0:
+; X64-NEXT:    movl (%rdi), %eax
+; X64-NEXT:    movl (%rsi), %ecx
+; X64-NEXT:    bswapl %eax
+; X64-NEXT:    bswapl %ecx
+; X64-NEXT:    xorl %edx, %edx
+; X64-NEXT:    cmpl %ecx, %eax
+; X64-NEXT:    seta %dl
+; X64-NEXT:    sbbl $0, %edx
+; X64-NEXT:    testl %edx, %edx
+; X64-NEXT:    setg %al
+; X64-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 4) nounwind
+  %c = icmp sgt i32 %m, 0
+  ret i1 %c
+}
+
+define i1 @length4_eq_const(i8* %X) nounwind {
+; X86-LABEL: length4_eq_const:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpl $875770417, (%eax) # imm = 0x34333231
+; X86-NEXT:    sete %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length4_eq_const:
+; X64:       # %bb.0:
+; X64-NEXT:    cmpl $875770417, (%rdi) # imm = 0x34333231
+; X64-NEXT:    sete %al
+; X64-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* getelementptr inbounds ([513 x i8], [513 x i8]* @.str, i32 0, i32 1), i64 4) nounwind
+  %c = icmp eq i32 %m, 0
+  ret i1 %c
+}
+
+define i32 @length5(i8* %X, i8* %Y) nounwind {
+; X86-LABEL: length5:
+; X86:       # %bb.0: # %loadbb
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl (%eax), %edx
+; X86-NEXT:    movl (%ecx), %esi
+; X86-NEXT:    bswapl %edx
+; X86-NEXT:    bswapl %esi
+; X86-NEXT:    cmpl %esi, %edx
+; X86-NEXT:    jne .LBB16_1
+; X86-NEXT:  # %bb.2: # %loadbb1
+; X86-NEXT:    movzbl 4(%eax), %eax
+; X86-NEXT:    movzbl 4(%ecx), %ecx
+; X86-NEXT:    subl %ecx, %eax
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
+; X86-NEXT:  .LBB16_1: # %res_block
+; X86-NEXT:    setae %al
+; X86-NEXT:    movzbl %al, %eax
+; X86-NEXT:    leal -1(%eax,%eax), %eax
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
+;
+; X64-LABEL: length5:
+; X64:       # %bb.0: # %loadbb
+; X64-NEXT:    movl (%rdi), %eax
+; X64-NEXT:    movl (%rsi), %ecx
+; X64-NEXT:    bswapl %eax
+; X64-NEXT:    bswapl %ecx
+; X64-NEXT:    cmpl %ecx, %eax
+; X64-NEXT:    jne .LBB16_1
+; X64-NEXT:  # %bb.2: # %loadbb1
+; X64-NEXT:    movzbl 4(%rdi), %eax
+; X64-NEXT:    movzbl 4(%rsi), %ecx
+; X64-NEXT:    subl %ecx, %eax
+; X64-NEXT:    retq
+; X64-NEXT:  .LBB16_1: # %res_block
+; X64-NEXT:    setae %al
+; X64-NEXT:    movzbl %al, %eax
+; X64-NEXT:    leal -1(%rax,%rax), %eax
+; X64-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 5) nounwind
+  ret i32 %m
+}
+
+define i1 @length5_eq(i8* %X, i8* %Y) nounwind {
+; X86-LABEL: length5_eq:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl (%ecx), %edx
+; X86-NEXT:    xorl (%eax), %edx
+; X86-NEXT:    movb 4(%ecx), %cl
+; X86-NEXT:    xorb 4(%eax), %cl
+; X86-NEXT:    movzbl %cl, %eax
+; X86-NEXT:    orl %edx, %eax
+; X86-NEXT:    setne %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length5_eq:
+; X64:       # %bb.0:
+; X64-NEXT:    movl (%rdi), %eax
+; X64-NEXT:    xorl (%rsi), %eax
+; X64-NEXT:    movb 4(%rdi), %cl
+; X64-NEXT:    xorb 4(%rsi), %cl
+; X64-NEXT:    movzbl %cl, %ecx
+; X64-NEXT:    orl %eax, %ecx
+; X64-NEXT:    setne %al
+; X64-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 5) nounwind
+  %c = icmp ne i32 %m, 0
+  ret i1 %c
+}
+
+define i1 @length5_lt(i8* %X, i8* %Y) nounwind {
+; X86-LABEL: length5_lt:
+; X86:       # %bb.0: # %loadbb
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl (%eax), %edx
+; X86-NEXT:    movl (%ecx), %esi
+; X86-NEXT:    bswapl %edx
+; X86-NEXT:    bswapl %esi
+; X86-NEXT:    cmpl %esi, %edx
+; X86-NEXT:    jne .LBB18_1
+; X86-NEXT:  # %bb.2: # %loadbb1
+; X86-NEXT:    movzbl 4(%eax), %eax
+; X86-NEXT:    movzbl 4(%ecx), %ecx
+; X86-NEXT:    subl %ecx, %eax
+; X86-NEXT:    jmp .LBB18_3
+; X86-NEXT:  .LBB18_1: # %res_block
+; X86-NEXT:    setae %al
+; X86-NEXT:    movzbl %al, %eax
+; X86-NEXT:    leal -1(%eax,%eax), %eax
+; X86-NEXT:  .LBB18_3: # %endblock
+; X86-NEXT:    shrl $31, %eax
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
+;
+; X64-LABEL: length5_lt:
+; X64:       # %bb.0: # %loadbb
+; X64-NEXT:    movl (%rdi), %eax
+; X64-NEXT:    movl (%rsi), %ecx
+; X64-NEXT:    bswapl %eax
+; X64-NEXT:    bswapl %ecx
+; X64-NEXT:    cmpl %ecx, %eax
+; X64-NEXT:    jne .LBB18_1
+; X64-NEXT:  # %bb.2: # %loadbb1
+; X64-NEXT:    movzbl 4(%rdi), %eax
+; X64-NEXT:    movzbl 4(%rsi), %ecx
+; X64-NEXT:    subl %ecx, %eax
+; X64-NEXT:    shrl $31, %eax
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    retq
+; X64-NEXT:  .LBB18_1: # %res_block
+; X64-NEXT:    setae %al
+; X64-NEXT:    movzbl %al, %eax
+; X64-NEXT:    leal -1(%rax,%rax), %eax
+; X64-NEXT:    shrl $31, %eax
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 5) nounwind
+  %c = icmp slt i32 %m, 0
+  ret i1 %c
+}
+
+define i1 @length7_eq(i8* %X, i8* %Y) nounwind {
+; X86-LABEL: length7_eq:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl (%ecx), %edx
+; X86-NEXT:    movl 3(%ecx), %ecx
+; X86-NEXT:    xorl (%eax), %edx
+; X86-NEXT:    xorl 3(%eax), %ecx
+; X86-NEXT:    orl %edx, %ecx
+; X86-NEXT:    setne %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length7_eq:
+; X64:       # %bb.0:
+; X64-NEXT:    movl (%rdi), %eax
+; X64-NEXT:    movl 3(%rdi), %ecx
+; X64-NEXT:    xorl (%rsi), %eax
+; X64-NEXT:    xorl 3(%rsi), %ecx
+; X64-NEXT:    orl %eax, %ecx
+; X64-NEXT:    setne %al
+; X64-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 7) nounwind
+  %c = icmp ne i32 %m, 0
+  ret i1 %c
+}
+
+define i32 @length8(i8* %X, i8* %Y) nounwind {
+; X86-LABEL: length8:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl (%esi), %ecx
+; X86-NEXT:    movl (%eax), %edx
+; X86-NEXT:    bswapl %ecx
+; X86-NEXT:    bswapl %edx
+; X86-NEXT:    cmpl %edx, %ecx
+; X86-NEXT:    jne .LBB20_2
+; X86-NEXT:  # %bb.1: # %loadbb1
+; X86-NEXT:    movl 4(%esi), %ecx
+; X86-NEXT:    movl 4(%eax), %edx
+; X86-NEXT:    bswapl %ecx
+; X86-NEXT:    bswapl %edx
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    cmpl %edx, %ecx
+; X86-NEXT:    je .LBB20_3
+; X86-NEXT:  .LBB20_2: # %res_block
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    cmpl %edx, %ecx
+; X86-NEXT:    setae %al
+; X86-NEXT:    leal -1(%eax,%eax), %eax
+; X86-NEXT:  .LBB20_3: # %endblock
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
+;
+; X64-LABEL: length8:
+; X64:       # %bb.0:
+; X64-NEXT:    movq (%rdi), %rcx
+; X64-NEXT:    movq (%rsi), %rdx
+; X64-NEXT:    bswapq %rcx
+; X64-NEXT:    bswapq %rdx
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpq %rdx, %rcx
+; X64-NEXT:    seta %al
+; X64-NEXT:    sbbl $0, %eax
+; X64-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 8) nounwind
+  ret i32 %m
+}
+
+define i1 @length8_eq(i8* %X, i8* %Y) nounwind {
+; X86-LABEL: length8_eq:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl (%ecx), %edx
+; X86-NEXT:    movl 4(%ecx), %ecx
+; X86-NEXT:    xorl (%eax), %edx
+; X86-NEXT:    xorl 4(%eax), %ecx
+; X86-NEXT:    orl %edx, %ecx
+; X86-NEXT:    sete %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length8_eq:
+; X64:       # %bb.0:
+; X64-NEXT:    movq (%rdi), %rax
+; X64-NEXT:    cmpq (%rsi), %rax
+; X64-NEXT:    sete %al
+; X64-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 8) nounwind
+  %c = icmp eq i32 %m, 0
+  ret i1 %c
+}
+
+define i1 @length8_eq_const(i8* %X) nounwind {
+; X86-LABEL: length8_eq_const:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl $858927408, %ecx # imm = 0x33323130
+; X86-NEXT:    xorl (%eax), %ecx
+; X86-NEXT:    movl $926299444, %edx # imm = 0x37363534
+; X86-NEXT:    xorl 4(%eax), %edx
+; X86-NEXT:    orl %ecx, %edx
+; X86-NEXT:    setne %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length8_eq_const:
+; X64:       # %bb.0:
+; X64-NEXT:    movabsq $3978425819141910832, %rax # imm = 0x3736353433323130
+; X64-NEXT:    cmpq %rax, (%rdi)
+; X64-NEXT:    setne %al
+; X64-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* getelementptr inbounds ([513 x i8], [513 x i8]* @.str, i32 0, i32 0), i64 8) nounwind
+  %c = icmp ne i32 %m, 0
+  ret i1 %c
+}
+
+define i1 @length9_eq(i8* %X, i8* %Y) nounwind {
+; X86-LABEL: length9_eq:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl (%ecx), %edx
+; X86-NEXT:    movl 4(%ecx), %esi
+; X86-NEXT:    xorl (%eax), %edx
+; X86-NEXT:    xorl 4(%eax), %esi
+; X86-NEXT:    orl %edx, %esi
+; X86-NEXT:    movb 8(%ecx), %cl
+; X86-NEXT:    xorb 8(%eax), %cl
+; X86-NEXT:    movzbl %cl, %eax
+; X86-NEXT:    orl %esi, %eax
+; X86-NEXT:    sete %al
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
+;
+; X64-LABEL: length9_eq:
+; X64:       # %bb.0:
+; X64-NEXT:    movq (%rdi), %rax
+; X64-NEXT:    xorq (%rsi), %rax
+; X64-NEXT:    movb 8(%rdi), %cl
+; X64-NEXT:    xorb 8(%rsi), %cl
+; X64-NEXT:    movzbl %cl, %ecx
+; X64-NEXT:    orq %rax, %rcx
+; X64-NEXT:    sete %al
+; X64-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 9) nounwind
+  %c = icmp eq i32 %m, 0
+  ret i1 %c
+}
+
+define i1 @length10_eq(i8* %X, i8* %Y) nounwind {
+; X86-LABEL: length10_eq:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl (%ecx), %edx
+; X86-NEXT:    movl 4(%ecx), %esi
+; X86-NEXT:    xorl (%eax), %edx
+; X86-NEXT:    xorl 4(%eax), %esi
+; X86-NEXT:    orl %edx, %esi
+; X86-NEXT:    movzwl 8(%ecx), %ecx
+; X86-NEXT:    xorw 8(%eax), %cx
+; X86-NEXT:    movzwl %cx, %eax
+; X86-NEXT:    orl %esi, %eax
+; X86-NEXT:    sete %al
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
+;
+; X64-LABEL: length10_eq:
+; X64:       # %bb.0:
+; X64-NEXT:    movq (%rdi), %rax
+; X64-NEXT:    xorq (%rsi), %rax
+; X64-NEXT:    movzwl 8(%rdi), %ecx
+; X64-NEXT:    xorw 8(%rsi), %cx
+; X64-NEXT:    movzwl %cx, %ecx
+; X64-NEXT:    orq %rax, %rcx
+; X64-NEXT:    sete %al
+; X64-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 10) nounwind
+  %c = icmp eq i32 %m, 0
+  ret i1 %c
+}
+
+define i1 @length11_eq(i8* %X, i8* %Y) nounwind {
+; X86-LABEL: length11_eq:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl (%ecx), %edx
+; X86-NEXT:    movl 4(%ecx), %esi
+; X86-NEXT:    xorl (%eax), %edx
+; X86-NEXT:    xorl 4(%eax), %esi
+; X86-NEXT:    orl %edx, %esi
+; X86-NEXT:    movl 7(%ecx), %ecx
+; X86-NEXT:    xorl 7(%eax), %ecx
+; X86-NEXT:    orl %esi, %ecx
+; X86-NEXT:    sete %al
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
+;
+; X64-LABEL: length11_eq:
+; X64:       # %bb.0:
+; X64-NEXT:    movq (%rdi), %rax
+; X64-NEXT:    movq 3(%rdi), %rcx
+; X64-NEXT:    xorq (%rsi), %rax
+; X64-NEXT:    xorq 3(%rsi), %rcx
+; X64-NEXT:    orq %rax, %rcx
+; X64-NEXT:    sete %al
+; X64-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 11) nounwind
+  %c = icmp eq i32 %m, 0
+  ret i1 %c
+}
+
+define i1 @length12_eq(i8* %X, i8* %Y) nounwind {
+; X86-LABEL: length12_eq:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl (%ecx), %edx
+; X86-NEXT:    movl 4(%ecx), %esi
+; X86-NEXT:    xorl (%eax), %edx
+; X86-NEXT:    xorl 4(%eax), %esi
+; X86-NEXT:    orl %edx, %esi
+; X86-NEXT:    movl 8(%ecx), %ecx
+; X86-NEXT:    xorl 8(%eax), %ecx
+; X86-NEXT:    orl %esi, %ecx
+; X86-NEXT:    setne %al
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
+;
+; X64-LABEL: length12_eq:
+; X64:       # %bb.0:
+; X64-NEXT:    movq (%rdi), %rax
+; X64-NEXT:    xorq (%rsi), %rax
+; X64-NEXT:    movl 8(%rdi), %ecx
+; X64-NEXT:    xorl 8(%rsi), %ecx
+; X64-NEXT:    orq %rax, %rcx
+; X64-NEXT:    setne %al
+; X64-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 12) nounwind
+  %c = icmp ne i32 %m, 0
+  ret i1 %c
+}
+
+define i32 @length12(i8* %X, i8* %Y) nounwind {
+; X86-LABEL: length12:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl (%esi), %ecx
+; X86-NEXT:    movl (%eax), %edx
+; X86-NEXT:    bswapl %ecx
+; X86-NEXT:    bswapl %edx
+; X86-NEXT:    cmpl %edx, %ecx
+; X86-NEXT:    jne .LBB27_3
+; X86-NEXT:  # %bb.1: # %loadbb1
+; X86-NEXT:    movl 4(%esi), %ecx
+; X86-NEXT:    movl 4(%eax), %edx
+; X86-NEXT:    bswapl %ecx
+; X86-NEXT:    bswapl %edx
+; X86-NEXT:    cmpl %edx, %ecx
+; X86-NEXT:    jne .LBB27_3
+; X86-NEXT:  # %bb.2: # %loadbb2
+; X86-NEXT:    movl 8(%esi), %ecx
+; X86-NEXT:    movl 8(%eax), %edx
+; X86-NEXT:    bswapl %ecx
+; X86-NEXT:    bswapl %edx
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    cmpl %edx, %ecx
+; X86-NEXT:    je .LBB27_4
+; X86-NEXT:  .LBB27_3: # %res_block
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    cmpl %edx, %ecx
+; X86-NEXT:    setae %al
+; X86-NEXT:    leal -1(%eax,%eax), %eax
+; X86-NEXT:  .LBB27_4: # %endblock
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
+;
+; X64-LABEL: length12:
+; X64:       # %bb.0:
+; X64-NEXT:    movq (%rdi), %rcx
+; X64-NEXT:    movq (%rsi), %rdx
+; X64-NEXT:    bswapq %rcx
+; X64-NEXT:    bswapq %rdx
+; X64-NEXT:    cmpq %rdx, %rcx
+; X64-NEXT:    jne .LBB27_2
+; X64-NEXT:  # %bb.1: # %loadbb1
+; X64-NEXT:    movl 8(%rdi), %ecx
+; X64-NEXT:    movl 8(%rsi), %edx
+; X64-NEXT:    bswapl %ecx
+; X64-NEXT:    bswapl %edx
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpq %rdx, %rcx
+; X64-NEXT:    je .LBB27_3
+; X64-NEXT:  .LBB27_2: # %res_block
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpq %rdx, %rcx
+; X64-NEXT:    setae %al
+; X64-NEXT:    leal -1(%rax,%rax), %eax
+; X64-NEXT:  .LBB27_3: # %endblock
+; X64-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 12) nounwind
+  ret i32 %m
+}
+
+define i1 @length13_eq(i8* %X, i8* %Y) nounwind {
+; X86-LABEL: length13_eq:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl (%ecx), %edx
+; X86-NEXT:    movl 4(%ecx), %esi
+; X86-NEXT:    xorl (%eax), %edx
+; X86-NEXT:    xorl 4(%eax), %esi
+; X86-NEXT:    orl %edx, %esi
+; X86-NEXT:    movl 8(%ecx), %edx
+; X86-NEXT:    xorl 8(%eax), %edx
+; X86-NEXT:    movb 12(%ecx), %cl
+; X86-NEXT:    xorb 12(%eax), %cl
+; X86-NEXT:    movzbl %cl, %eax
+; X86-NEXT:    orl %edx, %eax
+; X86-NEXT:    orl %esi, %eax
+; X86-NEXT:    sete %al
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
+;
+; X64-LABEL: length13_eq:
+; X64:       # %bb.0:
+; X64-NEXT:    movq (%rdi), %rax
+; X64-NEXT:    movq 5(%rdi), %rcx
+; X64-NEXT:    xorq (%rsi), %rax
+; X64-NEXT:    xorq 5(%rsi), %rcx
+; X64-NEXT:    orq %rax, %rcx
+; X64-NEXT:    sete %al
+; X64-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 13) nounwind
+  %c = icmp eq i32 %m, 0
+  ret i1 %c
+}
+
+define i1 @length14_eq(i8* %X, i8* %Y) nounwind {
+; X86-LABEL: length14_eq:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl (%ecx), %edx
+; X86-NEXT:    movl 4(%ecx), %esi
+; X86-NEXT:    xorl (%eax), %edx
+; X86-NEXT:    xorl 4(%eax), %esi
+; X86-NEXT:    orl %edx, %esi
+; X86-NEXT:    movl 8(%ecx), %edx
+; X86-NEXT:    xorl 8(%eax), %edx
+; X86-NEXT:    movzwl 12(%ecx), %ecx
+; X86-NEXT:    xorw 12(%eax), %cx
+; X86-NEXT:    movzwl %cx, %eax
+; X86-NEXT:    orl %edx, %eax
+; X86-NEXT:    orl %esi, %eax
+; X86-NEXT:    sete %al
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
+;
+; X64-LABEL: length14_eq:
+; X64:       # %bb.0:
+; X64-NEXT:    movq (%rdi), %rax
+; X64-NEXT:    movq 6(%rdi), %rcx
+; X64-NEXT:    xorq (%rsi), %rax
+; X64-NEXT:    xorq 6(%rsi), %rcx
+; X64-NEXT:    orq %rax, %rcx
+; X64-NEXT:    sete %al
+; X64-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 14) nounwind
+  %c = icmp eq i32 %m, 0
+  ret i1 %c
+}
+
+define i1 @length15_eq(i8* %X, i8* %Y) nounwind {
+; X86-LABEL: length15_eq:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl (%ecx), %edx
+; X86-NEXT:    movl 4(%ecx), %esi
+; X86-NEXT:    xorl (%eax), %edx
+; X86-NEXT:    xorl 4(%eax), %esi
+; X86-NEXT:    orl %edx, %esi
+; X86-NEXT:    movl 8(%ecx), %edx
+; X86-NEXT:    xorl 8(%eax), %edx
+; X86-NEXT:    movl 11(%ecx), %ecx
+; X86-NEXT:    xorl 11(%eax), %ecx
+; X86-NEXT:    orl %edx, %ecx
+; X86-NEXT:    orl %esi, %ecx
+; X86-NEXT:    sete %al
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
+;
+; X64-LABEL: length15_eq:
+; X64:       # %bb.0:
+; X64-NEXT:    movq (%rdi), %rax
+; X64-NEXT:    movq 7(%rdi), %rcx
+; X64-NEXT:    xorq (%rsi), %rax
+; X64-NEXT:    xorq 7(%rsi), %rcx
+; X64-NEXT:    orq %rax, %rcx
+; X64-NEXT:    sete %al
+; X64-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 15) nounwind
+  %c = icmp eq i32 %m, 0
+  ret i1 %c
+}
+
+; PR33329 - https://bugs.llvm.org/show_bug.cgi?id=33329
+
+define i32 @length16(i8* %X, i8* %Y) nounwind {
+; X86-LABEL: length16:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl (%esi), %ecx
+; X86-NEXT:    movl (%eax), %edx
+; X86-NEXT:    bswapl %ecx
+; X86-NEXT:    bswapl %edx
+; X86-NEXT:    cmpl %edx, %ecx
+; X86-NEXT:    jne .LBB31_4
+; X86-NEXT:  # %bb.1: # %loadbb1
+; X86-NEXT:    movl 4(%esi), %ecx
+; X86-NEXT:    movl 4(%eax), %edx
+; X86-NEXT:    bswapl %ecx
+; X86-NEXT:    bswapl %edx
+; X86-NEXT:    cmpl %edx, %ecx
+; X86-NEXT:    jne .LBB31_4
+; X86-NEXT:  # %bb.2: # %loadbb2
+; X86-NEXT:    movl 8(%esi), %ecx
+; X86-NEXT:    movl 8(%eax), %edx
+; X86-NEXT:    bswapl %ecx
+; X86-NEXT:    bswapl %edx
+; X86-NEXT:    cmpl %edx, %ecx
+; X86-NEXT:    jne .LBB31_4
+; X86-NEXT:  # %bb.3: # %loadbb3
+; X86-NEXT:    movl 12(%esi), %ecx
+; X86-NEXT:    movl 12(%eax), %edx
+; X86-NEXT:    bswapl %ecx
+; X86-NEXT:    bswapl %edx
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    cmpl %edx, %ecx
+; X86-NEXT:    je .LBB31_5
+; X86-NEXT:  .LBB31_4: # %res_block
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    cmpl %edx, %ecx
+; X86-NEXT:    setae %al
+; X86-NEXT:    leal -1(%eax,%eax), %eax
+; X86-NEXT:  .LBB31_5: # %endblock
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
+;
+; X64-LABEL: length16:
+; X64:       # %bb.0:
+; X64-NEXT:    movq (%rdi), %rcx
+; X64-NEXT:    movq (%rsi), %rdx
+; X64-NEXT:    bswapq %rcx
+; X64-NEXT:    bswapq %rdx
+; X64-NEXT:    cmpq %rdx, %rcx
+; X64-NEXT:    jne .LBB31_2
+; X64-NEXT:  # %bb.1: # %loadbb1
+; X64-NEXT:    movq 8(%rdi), %rcx
+; X64-NEXT:    movq 8(%rsi), %rdx
+; X64-NEXT:    bswapq %rcx
+; X64-NEXT:    bswapq %rdx
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpq %rdx, %rcx
+; X64-NEXT:    je .LBB31_3
+; X64-NEXT:  .LBB31_2: # %res_block
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpq %rdx, %rcx
+; X64-NEXT:    setae %al
+; X64-NEXT:    leal -1(%rax,%rax), %eax
+; X64-NEXT:  .LBB31_3: # %endblock
+; X64-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 16) nounwind
+  ret i32 %m
+}
+
+define i1 @length16_eq(i8* %x, i8* %y) nounwind {
+; X86-NOSSE-LABEL: length16_eq:
+; X86-NOSSE:       # %bb.0:
+; X86-NOSSE-NEXT:    pushl %esi
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NOSSE-NEXT:    movl (%ecx), %edx
+; X86-NOSSE-NEXT:    movl 4(%ecx), %esi
+; X86-NOSSE-NEXT:    xorl (%eax), %edx
+; X86-NOSSE-NEXT:    xorl 4(%eax), %esi
+; X86-NOSSE-NEXT:    orl %edx, %esi
+; X86-NOSSE-NEXT:    movl 8(%ecx), %edx
+; X86-NOSSE-NEXT:    xorl 8(%eax), %edx
+; X86-NOSSE-NEXT:    movl 12(%ecx), %ecx
+; X86-NOSSE-NEXT:    xorl 12(%eax), %ecx
+; X86-NOSSE-NEXT:    orl %edx, %ecx
+; X86-NOSSE-NEXT:    orl %esi, %ecx
+; X86-NOSSE-NEXT:    setne %al
+; X86-NOSSE-NEXT:    popl %esi
+; X86-NOSSE-NEXT:    retl
+;
+; X86-SSE1-LABEL: length16_eq:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    pushl %esi
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE1-NEXT:    movl (%ecx), %edx
+; X86-SSE1-NEXT:    movl 4(%ecx), %esi
+; X86-SSE1-NEXT:    xorl (%eax), %edx
+; X86-SSE1-NEXT:    xorl 4(%eax), %esi
+; X86-SSE1-NEXT:    orl %edx, %esi
+; X86-SSE1-NEXT:    movl 8(%ecx), %edx
+; X86-SSE1-NEXT:    xorl 8(%eax), %edx
+; X86-SSE1-NEXT:    movl 12(%ecx), %ecx
+; X86-SSE1-NEXT:    xorl 12(%eax), %ecx
+; X86-SSE1-NEXT:    orl %edx, %ecx
+; X86-SSE1-NEXT:    orl %esi, %ecx
+; X86-SSE1-NEXT:    setne %al
+; X86-SSE1-NEXT:    popl %esi
+; X86-SSE1-NEXT:    retl
+;
+; X86-SSE2-LABEL: length16_eq:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE2-NEXT:    movdqu (%ecx), %xmm0
+; X86-SSE2-NEXT:    movdqu (%eax), %xmm1
+; X86-SSE2-NEXT:    pcmpeqb %xmm0, %xmm1
+; X86-SSE2-NEXT:    pmovmskb %xmm1, %eax
+; X86-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; X86-SSE2-NEXT:    setne %al
+; X86-SSE2-NEXT:    retl
+;
+; X86-SSE41-LABEL: length16_eq:
+; X86-SSE41:       # %bb.0:
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE41-NEXT:    movdqu (%ecx), %xmm0
+; X86-SSE41-NEXT:    movdqu (%eax), %xmm1
+; X86-SSE41-NEXT:    pxor %xmm0, %xmm1
+; X86-SSE41-NEXT:    ptest %xmm1, %xmm1
+; X86-SSE41-NEXT:    setne %al
+; X86-SSE41-NEXT:    retl
+;
+; X64-SSE2-LABEL: length16_eq:
+; X64-SSE2:       # %bb.0:
+; X64-SSE2-NEXT:    movdqu (%rdi), %xmm0
+; X64-SSE2-NEXT:    movdqu (%rsi), %xmm1
+; X64-SSE2-NEXT:    pcmpeqb %xmm0, %xmm1
+; X64-SSE2-NEXT:    pmovmskb %xmm1, %eax
+; X64-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; X64-SSE2-NEXT:    setne %al
+; X64-SSE2-NEXT:    retq
+;
+; X64-SSE41-LABEL: length16_eq:
+; X64-SSE41:       # %bb.0:
+; X64-SSE41-NEXT:    movdqu (%rdi), %xmm0
+; X64-SSE41-NEXT:    movdqu (%rsi), %xmm1
+; X64-SSE41-NEXT:    pxor %xmm0, %xmm1
+; X64-SSE41-NEXT:    ptest %xmm1, %xmm1
+; X64-SSE41-NEXT:    setne %al
+; X64-SSE41-NEXT:    retq
+;
+; X64-AVX-LABEL: length16_eq:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    vmovdqu (%rdi), %xmm0
+; X64-AVX-NEXT:    vpxor (%rsi), %xmm0, %xmm0
+; X64-AVX-NEXT:    vptest %xmm0, %xmm0
+; X64-AVX-NEXT:    setne %al
+; X64-AVX-NEXT:    retq
+;
+; X64-MIC-AVX-LABEL: length16_eq:
+; X64-MIC-AVX:       # %bb.0:
+; X64-MIC-AVX-NEXT:    vmovdqu (%rdi), %xmm0
+; X64-MIC-AVX-NEXT:    vmovdqu (%rsi), %xmm1
+; X64-MIC-AVX-NEXT:    vpcmpneqd %zmm1, %zmm0, %k0
+; X64-MIC-AVX-NEXT:    kortestw %k0, %k0
+; X64-MIC-AVX-NEXT:    setne %al
+; X64-MIC-AVX-NEXT:    vzeroupper
+; X64-MIC-AVX-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 16) nounwind
+  %cmp = icmp ne i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length16_lt(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length16_lt:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl (%esi), %ecx
+; X86-NEXT:    movl (%eax), %edx
+; X86-NEXT:    bswapl %ecx
+; X86-NEXT:    bswapl %edx
+; X86-NEXT:    cmpl %edx, %ecx
+; X86-NEXT:    jne .LBB33_4
+; X86-NEXT:  # %bb.1: # %loadbb1
+; X86-NEXT:    movl 4(%esi), %ecx
+; X86-NEXT:    movl 4(%eax), %edx
+; X86-NEXT:    bswapl %ecx
+; X86-NEXT:    bswapl %edx
+; X86-NEXT:    cmpl %edx, %ecx
+; X86-NEXT:    jne .LBB33_4
+; X86-NEXT:  # %bb.2: # %loadbb2
+; X86-NEXT:    movl 8(%esi), %ecx
+; X86-NEXT:    movl 8(%eax), %edx
+; X86-NEXT:    bswapl %ecx
+; X86-NEXT:    bswapl %edx
+; X86-NEXT:    cmpl %edx, %ecx
+; X86-NEXT:    jne .LBB33_4
+; X86-NEXT:  # %bb.3: # %loadbb3
+; X86-NEXT:    movl 12(%esi), %ecx
+; X86-NEXT:    movl 12(%eax), %edx
+; X86-NEXT:    bswapl %ecx
+; X86-NEXT:    bswapl %edx
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    cmpl %edx, %ecx
+; X86-NEXT:    je .LBB33_5
+; X86-NEXT:  .LBB33_4: # %res_block
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    cmpl %edx, %ecx
+; X86-NEXT:    setae %al
+; X86-NEXT:    leal -1(%eax,%eax), %eax
+; X86-NEXT:  .LBB33_5: # %endblock
+; X86-NEXT:    shrl $31, %eax
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
+;
+; X64-LABEL: length16_lt:
+; X64:       # %bb.0:
+; X64-NEXT:    movq (%rdi), %rcx
+; X64-NEXT:    movq (%rsi), %rdx
+; X64-NEXT:    bswapq %rcx
+; X64-NEXT:    bswapq %rdx
+; X64-NEXT:    cmpq %rdx, %rcx
+; X64-NEXT:    jne .LBB33_2
+; X64-NEXT:  # %bb.1: # %loadbb1
+; X64-NEXT:    movq 8(%rdi), %rcx
+; X64-NEXT:    movq 8(%rsi), %rdx
+; X64-NEXT:    bswapq %rcx
+; X64-NEXT:    bswapq %rdx
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpq %rdx, %rcx
+; X64-NEXT:    je .LBB33_3
+; X64-NEXT:  .LBB33_2: # %res_block
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpq %rdx, %rcx
+; X64-NEXT:    setae %al
+; X64-NEXT:    leal -1(%rax,%rax), %eax
+; X64-NEXT:  .LBB33_3: # %endblock
+; X64-NEXT:    shrl $31, %eax
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 16) nounwind
+  %cmp = icmp slt i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length16_gt(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length16_gt:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl (%esi), %eax
+; X86-NEXT:    movl (%edx), %ecx
+; X86-NEXT:    bswapl %eax
+; X86-NEXT:    bswapl %ecx
+; X86-NEXT:    cmpl %ecx, %eax
+; X86-NEXT:    jne .LBB34_4
+; X86-NEXT:  # %bb.1: # %loadbb1
+; X86-NEXT:    movl 4(%esi), %eax
+; X86-NEXT:    movl 4(%edx), %ecx
+; X86-NEXT:    bswapl %eax
+; X86-NEXT:    bswapl %ecx
+; X86-NEXT:    cmpl %ecx, %eax
+; X86-NEXT:    jne .LBB34_4
+; X86-NEXT:  # %bb.2: # %loadbb2
+; X86-NEXT:    movl 8(%esi), %eax
+; X86-NEXT:    movl 8(%edx), %ecx
+; X86-NEXT:    bswapl %eax
+; X86-NEXT:    bswapl %ecx
+; X86-NEXT:    cmpl %ecx, %eax
+; X86-NEXT:    jne .LBB34_4
+; X86-NEXT:  # %bb.3: # %loadbb3
+; X86-NEXT:    movl 12(%esi), %eax
+; X86-NEXT:    movl 12(%edx), %ecx
+; X86-NEXT:    bswapl %eax
+; X86-NEXT:    bswapl %ecx
+; X86-NEXT:    xorl %edx, %edx
+; X86-NEXT:    cmpl %ecx, %eax
+; X86-NEXT:    je .LBB34_5
+; X86-NEXT:  .LBB34_4: # %res_block
+; X86-NEXT:    xorl %edx, %edx
+; X86-NEXT:    cmpl %ecx, %eax
+; X86-NEXT:    setae %dl
+; X86-NEXT:    leal -1(%edx,%edx), %edx
+; X86-NEXT:  .LBB34_5: # %endblock
+; X86-NEXT:    testl %edx, %edx
+; X86-NEXT:    setg %al
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
+;
+; X64-LABEL: length16_gt:
+; X64:       # %bb.0:
+; X64-NEXT:    movq (%rdi), %rax
+; X64-NEXT:    movq (%rsi), %rcx
+; X64-NEXT:    bswapq %rax
+; X64-NEXT:    bswapq %rcx
+; X64-NEXT:    cmpq %rcx, %rax
+; X64-NEXT:    jne .LBB34_2
+; X64-NEXT:  # %bb.1: # %loadbb1
+; X64-NEXT:    movq 8(%rdi), %rax
+; X64-NEXT:    movq 8(%rsi), %rcx
+; X64-NEXT:    bswapq %rax
+; X64-NEXT:    bswapq %rcx
+; X64-NEXT:    xorl %edx, %edx
+; X64-NEXT:    cmpq %rcx, %rax
+; X64-NEXT:    je .LBB34_3
+; X64-NEXT:  .LBB34_2: # %res_block
+; X64-NEXT:    xorl %edx, %edx
+; X64-NEXT:    cmpq %rcx, %rax
+; X64-NEXT:    setae %dl
+; X64-NEXT:    leal -1(%rdx,%rdx), %edx
+; X64-NEXT:  .LBB34_3: # %endblock
+; X64-NEXT:    testl %edx, %edx
+; X64-NEXT:    setg %al
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 16) nounwind
+  %cmp = icmp sgt i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length16_eq_const(i8* %X) nounwind {
+; X86-NOSSE-LABEL: length16_eq_const:
+; X86-NOSSE:       # %bb.0:
+; X86-NOSSE-NEXT:    pushl %esi
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOSSE-NEXT:    movl $858927408, %ecx # imm = 0x33323130
+; X86-NOSSE-NEXT:    xorl (%eax), %ecx
+; X86-NOSSE-NEXT:    movl $926299444, %edx # imm = 0x37363534
+; X86-NOSSE-NEXT:    xorl 4(%eax), %edx
+; X86-NOSSE-NEXT:    orl %ecx, %edx
+; X86-NOSSE-NEXT:    movl $825243960, %ecx # imm = 0x31303938
+; X86-NOSSE-NEXT:    xorl 8(%eax), %ecx
+; X86-NOSSE-NEXT:    movl $892613426, %esi # imm = 0x35343332
+; X86-NOSSE-NEXT:    xorl 12(%eax), %esi
+; X86-NOSSE-NEXT:    orl %ecx, %esi
+; X86-NOSSE-NEXT:    orl %edx, %esi
+; X86-NOSSE-NEXT:    sete %al
+; X86-NOSSE-NEXT:    popl %esi
+; X86-NOSSE-NEXT:    retl
+;
+; X86-SSE1-LABEL: length16_eq_const:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    pushl %esi
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE1-NEXT:    movl $858927408, %ecx # imm = 0x33323130
+; X86-SSE1-NEXT:    xorl (%eax), %ecx
+; X86-SSE1-NEXT:    movl $926299444, %edx # imm = 0x37363534
+; X86-SSE1-NEXT:    xorl 4(%eax), %edx
+; X86-SSE1-NEXT:    orl %ecx, %edx
+; X86-SSE1-NEXT:    movl $825243960, %ecx # imm = 0x31303938
+; X86-SSE1-NEXT:    xorl 8(%eax), %ecx
+; X86-SSE1-NEXT:    movl $892613426, %esi # imm = 0x35343332
+; X86-SSE1-NEXT:    xorl 12(%eax), %esi
+; X86-SSE1-NEXT:    orl %ecx, %esi
+; X86-SSE1-NEXT:    orl %edx, %esi
+; X86-SSE1-NEXT:    sete %al
+; X86-SSE1-NEXT:    popl %esi
+; X86-SSE1-NEXT:    retl
+;
+; X86-SSE2-LABEL: length16_eq_const:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT:    movdqu (%eax), %xmm0
+; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pmovmskb %xmm0, %eax
+; X86-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; X86-SSE2-NEXT:    sete %al
+; X86-SSE2-NEXT:    retl
+;
+; X86-SSE41-LABEL: length16_eq_const:
+; X86-SSE41:       # %bb.0:
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT:    movdqu (%eax), %xmm0
+; X86-SSE41-NEXT:    pxor {{\.LCPI.*}}, %xmm0
+; X86-SSE41-NEXT:    ptest %xmm0, %xmm0
+; X86-SSE41-NEXT:    sete %al
+; X86-SSE41-NEXT:    retl
+;
+; X64-SSE2-LABEL: length16_eq_const:
+; X64-SSE2:       # %bb.0:
+; X64-SSE2-NEXT:    movdqu (%rdi), %xmm0
+; X64-SSE2-NEXT:    pcmpeqb {{.*}}(%rip), %xmm0
+; X64-SSE2-NEXT:    pmovmskb %xmm0, %eax
+; X64-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; X64-SSE2-NEXT:    sete %al
+; X64-SSE2-NEXT:    retq
+;
+; X64-SSE41-LABEL: length16_eq_const:
+; X64-SSE41:       # %bb.0:
+; X64-SSE41-NEXT:    movdqu (%rdi), %xmm0
+; X64-SSE41-NEXT:    pxor {{.*}}(%rip), %xmm0
+; X64-SSE41-NEXT:    ptest %xmm0, %xmm0
+; X64-SSE41-NEXT:    sete %al
+; X64-SSE41-NEXT:    retq
+;
+; X64-AVX-LABEL: length16_eq_const:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    vmovdqu (%rdi), %xmm0
+; X64-AVX-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; X64-AVX-NEXT:    vptest %xmm0, %xmm0
+; X64-AVX-NEXT:    sete %al
+; X64-AVX-NEXT:    retq
+;
+; X64-MIC-AVX-LABEL: length16_eq_const:
+; X64-MIC-AVX:       # %bb.0:
+; X64-MIC-AVX-NEXT:    vmovdqu (%rdi), %xmm0
+; X64-MIC-AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [858927408,926299444,825243960,892613426]
+; X64-MIC-AVX-NEXT:    vpcmpneqd %zmm1, %zmm0, %k0
+; X64-MIC-AVX-NEXT:    kortestw %k0, %k0
+; X64-MIC-AVX-NEXT:    sete %al
+; X64-MIC-AVX-NEXT:    vzeroupper
+; X64-MIC-AVX-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* getelementptr inbounds ([513 x i8], [513 x i8]* @.str, i32 0, i32 0), i64 16) nounwind
+  %c = icmp eq i32 %m, 0
+  ret i1 %c
+}
+
+; PR33914 - https://bugs.llvm.org/show_bug.cgi?id=33914
+
+define i32 @length24(i8* %X, i8* %Y) nounwind {
+; X86-LABEL: length24:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $24
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    retl
+;
+; X64-LABEL: length24:
+; X64:       # %bb.0:
+; X64-NEXT:    movq (%rdi), %rcx
+; X64-NEXT:    movq (%rsi), %rdx
+; X64-NEXT:    bswapq %rcx
+; X64-NEXT:    bswapq %rdx
+; X64-NEXT:    cmpq %rdx, %rcx
+; X64-NEXT:    jne .LBB36_3
+; X64-NEXT:  # %bb.1: # %loadbb1
+; X64-NEXT:    movq 8(%rdi), %rcx
+; X64-NEXT:    movq 8(%rsi), %rdx
+; X64-NEXT:    bswapq %rcx
+; X64-NEXT:    bswapq %rdx
+; X64-NEXT:    cmpq %rdx, %rcx
+; X64-NEXT:    jne .LBB36_3
+; X64-NEXT:  # %bb.2: # %loadbb2
+; X64-NEXT:    movq 16(%rdi), %rcx
+; X64-NEXT:    movq 16(%rsi), %rdx
+; X64-NEXT:    bswapq %rcx
+; X64-NEXT:    bswapq %rdx
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpq %rdx, %rcx
+; X64-NEXT:    je .LBB36_4
+; X64-NEXT:  .LBB36_3: # %res_block
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpq %rdx, %rcx
+; X64-NEXT:    setae %al
+; X64-NEXT:    leal -1(%rax,%rax), %eax
+; X64-NEXT:  .LBB36_4: # %endblock
+; X64-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 24) nounwind
+  ret i32 %m
+}
+
+define i1 @length24_eq(i8* %x, i8* %y) nounwind {
+; X86-NOSSE-LABEL: length24_eq:
+; X86-NOSSE:       # %bb.0:
+; X86-NOSSE-NEXT:    pushl $0
+; X86-NOSSE-NEXT:    pushl $24
+; X86-NOSSE-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    calll memcmp
+; X86-NOSSE-NEXT:    addl $16, %esp
+; X86-NOSSE-NEXT:    testl %eax, %eax
+; X86-NOSSE-NEXT:    sete %al
+; X86-NOSSE-NEXT:    retl
+;
+; X86-SSE1-LABEL: length24_eq:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    pushl $0
+; X86-SSE1-NEXT:    pushl $24
+; X86-SSE1-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    calll memcmp
+; X86-SSE1-NEXT:    addl $16, %esp
+; X86-SSE1-NEXT:    testl %eax, %eax
+; X86-SSE1-NEXT:    sete %al
+; X86-SSE1-NEXT:    retl
+;
+; X86-SSE2-LABEL: length24_eq:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE2-NEXT:    movdqu (%ecx), %xmm0
+; X86-SSE2-NEXT:    movdqu 8(%ecx), %xmm1
+; X86-SSE2-NEXT:    movdqu (%eax), %xmm2
+; X86-SSE2-NEXT:    pcmpeqb %xmm0, %xmm2
+; X86-SSE2-NEXT:    movdqu 8(%eax), %xmm0
+; X86-SSE2-NEXT:    pcmpeqb %xmm1, %xmm0
+; X86-SSE2-NEXT:    pand %xmm2, %xmm0
+; X86-SSE2-NEXT:    pmovmskb %xmm0, %eax
+; X86-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; X86-SSE2-NEXT:    sete %al
+; X86-SSE2-NEXT:    retl
+;
+; X86-SSE41-LABEL: length24_eq:
+; X86-SSE41:       # %bb.0:
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE41-NEXT:    movdqu (%ecx), %xmm0
+; X86-SSE41-NEXT:    movdqu 8(%ecx), %xmm1
+; X86-SSE41-NEXT:    movdqu (%eax), %xmm2
+; X86-SSE41-NEXT:    pxor %xmm0, %xmm2
+; X86-SSE41-NEXT:    movdqu 8(%eax), %xmm0
+; X86-SSE41-NEXT:    pxor %xmm1, %xmm0
+; X86-SSE41-NEXT:    por %xmm2, %xmm0
+; X86-SSE41-NEXT:    ptest %xmm0, %xmm0
+; X86-SSE41-NEXT:    sete %al
+; X86-SSE41-NEXT:    retl
+;
+; X64-SSE2-LABEL: length24_eq:
+; X64-SSE2:       # %bb.0:
+; X64-SSE2-NEXT:    movdqu (%rdi), %xmm0
+; X64-SSE2-NEXT:    movdqu (%rsi), %xmm1
+; X64-SSE2-NEXT:    pcmpeqb %xmm0, %xmm1
+; X64-SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; X64-SSE2-NEXT:    movq {{.*#+}} xmm2 = mem[0],zero
+; X64-SSE2-NEXT:    pcmpeqb %xmm0, %xmm2
+; X64-SSE2-NEXT:    pand %xmm1, %xmm2
+; X64-SSE2-NEXT:    pmovmskb %xmm2, %eax
+; X64-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; X64-SSE2-NEXT:    sete %al
+; X64-SSE2-NEXT:    retq
+;
+; X64-SSE41-LABEL: length24_eq:
+; X64-SSE41:       # %bb.0:
+; X64-SSE41-NEXT:    movdqu (%rdi), %xmm0
+; X64-SSE41-NEXT:    movdqu (%rsi), %xmm1
+; X64-SSE41-NEXT:    pxor %xmm0, %xmm1
+; X64-SSE41-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; X64-SSE41-NEXT:    movq {{.*#+}} xmm2 = mem[0],zero
+; X64-SSE41-NEXT:    pxor %xmm0, %xmm2
+; X64-SSE41-NEXT:    por %xmm1, %xmm2
+; X64-SSE41-NEXT:    ptest %xmm2, %xmm2
+; X64-SSE41-NEXT:    sete %al
+; X64-SSE41-NEXT:    retq
+;
+; X64-AVX-LABEL: length24_eq:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    vmovdqu (%rdi), %xmm0
+; X64-AVX-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
+; X64-AVX-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero
+; X64-AVX-NEXT:    vpxor %xmm2, %xmm1, %xmm1
+; X64-AVX-NEXT:    vpxor (%rsi), %xmm0, %xmm0
+; X64-AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; X64-AVX-NEXT:    vptest %xmm0, %xmm0
+; X64-AVX-NEXT:    sete %al
+; X64-AVX-NEXT:    retq
+;
+; X64-MIC-AVX-LABEL: length24_eq:
+; X64-MIC-AVX:       # %bb.0:
+; X64-MIC-AVX-NEXT:    vmovdqu (%rdi), %xmm0
+; X64-MIC-AVX-NEXT:    vmovdqu (%rsi), %xmm1
+; X64-MIC-AVX-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero
+; X64-MIC-AVX-NEXT:    vmovq {{.*#+}} xmm3 = mem[0],zero
+; X64-MIC-AVX-NEXT:    vpcmpneqd %zmm3, %zmm2, %k0
+; X64-MIC-AVX-NEXT:    vpcmpneqd %zmm1, %zmm0, %k1
+; X64-MIC-AVX-NEXT:    kortestw %k0, %k1
+; X64-MIC-AVX-NEXT:    sete %al
+; X64-MIC-AVX-NEXT:    vzeroupper
+; X64-MIC-AVX-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 24) nounwind
+  %cmp = icmp eq i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length24_lt(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length24_lt:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $24
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    shrl $31, %eax
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: length24_lt:
+; X64:       # %bb.0:
+; X64-NEXT:    movq (%rdi), %rcx
+; X64-NEXT:    movq (%rsi), %rdx
+; X64-NEXT:    bswapq %rcx
+; X64-NEXT:    bswapq %rdx
+; X64-NEXT:    cmpq %rdx, %rcx
+; X64-NEXT:    jne .LBB38_3
+; X64-NEXT:  # %bb.1: # %loadbb1
+; X64-NEXT:    movq 8(%rdi), %rcx
+; X64-NEXT:    movq 8(%rsi), %rdx
+; X64-NEXT:    bswapq %rcx
+; X64-NEXT:    bswapq %rdx
+; X64-NEXT:    cmpq %rdx, %rcx
+; X64-NEXT:    jne .LBB38_3
+; X64-NEXT:  # %bb.2: # %loadbb2
+; X64-NEXT:    movq 16(%rdi), %rcx
+; X64-NEXT:    movq 16(%rsi), %rdx
+; X64-NEXT:    bswapq %rcx
+; X64-NEXT:    bswapq %rdx
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpq %rdx, %rcx
+; X64-NEXT:    je .LBB38_4
+; X64-NEXT:  .LBB38_3: # %res_block
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpq %rdx, %rcx
+; X64-NEXT:    setae %al
+; X64-NEXT:    leal -1(%rax,%rax), %eax
+; X64-NEXT:  .LBB38_4: # %endblock
+; X64-NEXT:    shrl $31, %eax
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 24) nounwind
+  %cmp = icmp slt i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length24_gt(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length24_gt:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $24
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    setg %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length24_gt:
+; X64:       # %bb.0:
+; X64-NEXT:    movq (%rdi), %rax
+; X64-NEXT:    movq (%rsi), %rcx
+; X64-NEXT:    bswapq %rax
+; X64-NEXT:    bswapq %rcx
+; X64-NEXT:    cmpq %rcx, %rax
+; X64-NEXT:    jne .LBB39_3
+; X64-NEXT:  # %bb.1: # %loadbb1
+; X64-NEXT:    movq 8(%rdi), %rax
+; X64-NEXT:    movq 8(%rsi), %rcx
+; X64-NEXT:    bswapq %rax
+; X64-NEXT:    bswapq %rcx
+; X64-NEXT:    cmpq %rcx, %rax
+; X64-NEXT:    jne .LBB39_3
+; X64-NEXT:  # %bb.2: # %loadbb2
+; X64-NEXT:    movq 16(%rdi), %rax
+; X64-NEXT:    movq 16(%rsi), %rcx
+; X64-NEXT:    bswapq %rax
+; X64-NEXT:    bswapq %rcx
+; X64-NEXT:    xorl %edx, %edx
+; X64-NEXT:    cmpq %rcx, %rax
+; X64-NEXT:    je .LBB39_4
+; X64-NEXT:  .LBB39_3: # %res_block
+; X64-NEXT:    xorl %edx, %edx
+; X64-NEXT:    cmpq %rcx, %rax
+; X64-NEXT:    setae %dl
+; X64-NEXT:    leal -1(%rdx,%rdx), %edx
+; X64-NEXT:  .LBB39_4: # %endblock
+; X64-NEXT:    testl %edx, %edx
+; X64-NEXT:    setg %al
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 24) nounwind
+  %cmp = icmp sgt i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length24_eq_const(i8* %X) nounwind {
+; X86-NOSSE-LABEL: length24_eq_const:
+; X86-NOSSE:       # %bb.0:
+; X86-NOSSE-NEXT:    pushl $0
+; X86-NOSSE-NEXT:    pushl $24
+; X86-NOSSE-NEXT:    pushl $.L.str
+; X86-NOSSE-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    calll memcmp
+; X86-NOSSE-NEXT:    addl $16, %esp
+; X86-NOSSE-NEXT:    testl %eax, %eax
+; X86-NOSSE-NEXT:    setne %al
+; X86-NOSSE-NEXT:    retl
+;
+; X86-SSE1-LABEL: length24_eq_const:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    pushl $0
+; X86-SSE1-NEXT:    pushl $24
+; X86-SSE1-NEXT:    pushl $.L.str
+; X86-SSE1-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    calll memcmp
+; X86-SSE1-NEXT:    addl $16, %esp
+; X86-SSE1-NEXT:    testl %eax, %eax
+; X86-SSE1-NEXT:    setne %al
+; X86-SSE1-NEXT:    retl
+;
+; X86-SSE2-LABEL: length24_eq_const:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT:    movdqu (%eax), %xmm0
+; X86-SSE2-NEXT:    movdqu 8(%eax), %xmm1
+; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI.*}}, %xmm1
+; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand %xmm1, %xmm0
+; X86-SSE2-NEXT:    pmovmskb %xmm0, %eax
+; X86-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; X86-SSE2-NEXT:    setne %al
+; X86-SSE2-NEXT:    retl
+;
+; X86-SSE41-LABEL: length24_eq_const:
+; X86-SSE41:       # %bb.0:
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT:    movdqu (%eax), %xmm0
+; X86-SSE41-NEXT:    movdqu 8(%eax), %xmm1
+; X86-SSE41-NEXT:    pxor {{\.LCPI.*}}, %xmm1
+; X86-SSE41-NEXT:    pxor {{\.LCPI.*}}, %xmm0
+; X86-SSE41-NEXT:    por %xmm1, %xmm0
+; X86-SSE41-NEXT:    ptest %xmm0, %xmm0
+; X86-SSE41-NEXT:    setne %al
+; X86-SSE41-NEXT:    retl
+;
+; X64-SSE2-LABEL: length24_eq_const:
+; X64-SSE2:       # %bb.0:
+; X64-SSE2-NEXT:    movdqu (%rdi), %xmm0
+; X64-SSE2-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
+; X64-SSE2-NEXT:    pcmpeqb {{.*}}(%rip), %xmm1
+; X64-SSE2-NEXT:    pcmpeqb {{.*}}(%rip), %xmm0
+; X64-SSE2-NEXT:    pand %xmm1, %xmm0
+; X64-SSE2-NEXT:    pmovmskb %xmm0, %eax
+; X64-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; X64-SSE2-NEXT:    setne %al
+; X64-SSE2-NEXT:    retq
+;
+; X64-SSE41-LABEL: length24_eq_const:
+; X64-SSE41:       # %bb.0:
+; X64-SSE41-NEXT:    movdqu (%rdi), %xmm0
+; X64-SSE41-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
+; X64-SSE41-NEXT:    pxor {{.*}}(%rip), %xmm1
+; X64-SSE41-NEXT:    pxor {{.*}}(%rip), %xmm0
+; X64-SSE41-NEXT:    por %xmm1, %xmm0
+; X64-SSE41-NEXT:    ptest %xmm0, %xmm0
+; X64-SSE41-NEXT:    setne %al
+; X64-SSE41-NEXT:    retq
+;
+; X64-AVX-LABEL: length24_eq_const:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    vmovdqu (%rdi), %xmm0
+; X64-AVX-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
+; X64-AVX-NEXT:    vpxor {{.*}}(%rip), %xmm1, %xmm1
+; X64-AVX-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; X64-AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; X64-AVX-NEXT:    vptest %xmm0, %xmm0
+; X64-AVX-NEXT:    setne %al
+; X64-AVX-NEXT:    retq
+;
+; X64-MIC-AVX-LABEL: length24_eq_const:
+; X64-MIC-AVX:       # %bb.0:
+; X64-MIC-AVX-NEXT:    vmovdqu (%rdi), %xmm0
+; X64-MIC-AVX-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
+; X64-MIC-AVX-NEXT:    vmovdqa {{.*#+}} xmm2 = [959985462,858927408,0,0]
+; X64-MIC-AVX-NEXT:    vpcmpneqd %zmm2, %zmm1, %k0
+; X64-MIC-AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [858927408,926299444,825243960,892613426]
+; X64-MIC-AVX-NEXT:    vpcmpneqd %zmm1, %zmm0, %k1
+; X64-MIC-AVX-NEXT:    kortestw %k0, %k1
+; X64-MIC-AVX-NEXT:    setne %al
+; X64-MIC-AVX-NEXT:    vzeroupper
+; X64-MIC-AVX-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* getelementptr inbounds ([513 x i8], [513 x i8]* @.str, i32 0, i32 0), i64 24) nounwind
+  %c = icmp ne i32 %m, 0
+  ret i1 %c
+}
+
+define i32 @length31(i8* %X, i8* %Y) nounwind {
+; X86-LABEL: length31:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $31
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    retl
+;
+; X64-LABEL: length31:
+; X64:       # %bb.0:
+; X64-NEXT:    movl $31, %edx
+; X64-NEXT:    jmp memcmp # TAILCALL
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 31) nounwind
+  ret i32 %m
+}
+
+define i1 @length31_eq(i8* %x, i8* %y) nounwind {
+; X86-NOSSE-LABEL: length31_eq:
+; X86-NOSSE:       # %bb.0:
+; X86-NOSSE-NEXT:    pushl $0
+; X86-NOSSE-NEXT:    pushl $31
+; X86-NOSSE-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    calll memcmp
+; X86-NOSSE-NEXT:    addl $16, %esp
+; X86-NOSSE-NEXT:    testl %eax, %eax
+; X86-NOSSE-NEXT:    sete %al
+; X86-NOSSE-NEXT:    retl
+;
+; X86-SSE1-LABEL: length31_eq:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    pushl $0
+; X86-SSE1-NEXT:    pushl $31
+; X86-SSE1-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    calll memcmp
+; X86-SSE1-NEXT:    addl $16, %esp
+; X86-SSE1-NEXT:    testl %eax, %eax
+; X86-SSE1-NEXT:    sete %al
+; X86-SSE1-NEXT:    retl
+;
+; X86-SSE2-LABEL: length31_eq:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE2-NEXT:    movdqu (%ecx), %xmm0
+; X86-SSE2-NEXT:    movdqu 15(%ecx), %xmm1
+; X86-SSE2-NEXT:    movdqu (%eax), %xmm2
+; X86-SSE2-NEXT:    pcmpeqb %xmm0, %xmm2
+; X86-SSE2-NEXT:    movdqu 15(%eax), %xmm0
+; X86-SSE2-NEXT:    pcmpeqb %xmm1, %xmm0
+; X86-SSE2-NEXT:    pand %xmm2, %xmm0
+; X86-SSE2-NEXT:    pmovmskb %xmm0, %eax
+; X86-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; X86-SSE2-NEXT:    sete %al
+; X86-SSE2-NEXT:    retl
+;
+; X86-SSE41-LABEL: length31_eq:
+; X86-SSE41:       # %bb.0:
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE41-NEXT:    movdqu (%ecx), %xmm0
+; X86-SSE41-NEXT:    movdqu 15(%ecx), %xmm1
+; X86-SSE41-NEXT:    movdqu (%eax), %xmm2
+; X86-SSE41-NEXT:    pxor %xmm0, %xmm2
+; X86-SSE41-NEXT:    movdqu 15(%eax), %xmm0
+; X86-SSE41-NEXT:    pxor %xmm1, %xmm0
+; X86-SSE41-NEXT:    por %xmm2, %xmm0
+; X86-SSE41-NEXT:    ptest %xmm0, %xmm0
+; X86-SSE41-NEXT:    sete %al
+; X86-SSE41-NEXT:    retl
+;
+; X64-SSE2-LABEL: length31_eq:
+; X64-SSE2:       # %bb.0:
+; X64-SSE2-NEXT:    movdqu (%rdi), %xmm0
+; X64-SSE2-NEXT:    movdqu 15(%rdi), %xmm1
+; X64-SSE2-NEXT:    movdqu (%rsi), %xmm2
+; X64-SSE2-NEXT:    pcmpeqb %xmm0, %xmm2
+; X64-SSE2-NEXT:    movdqu 15(%rsi), %xmm0
+; X64-SSE2-NEXT:    pcmpeqb %xmm1, %xmm0
+; X64-SSE2-NEXT:    pand %xmm2, %xmm0
+; X64-SSE2-NEXT:    pmovmskb %xmm0, %eax
+; X64-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; X64-SSE2-NEXT:    sete %al
+; X64-SSE2-NEXT:    retq
+;
+; X64-SSE41-LABEL: length31_eq:
+; X64-SSE41:       # %bb.0:
+; X64-SSE41-NEXT:    movdqu (%rdi), %xmm0
+; X64-SSE41-NEXT:    movdqu 15(%rdi), %xmm1
+; X64-SSE41-NEXT:    movdqu (%rsi), %xmm2
+; X64-SSE41-NEXT:    pxor %xmm0, %xmm2
+; X64-SSE41-NEXT:    movdqu 15(%rsi), %xmm0
+; X64-SSE41-NEXT:    pxor %xmm1, %xmm0
+; X64-SSE41-NEXT:    por %xmm2, %xmm0
+; X64-SSE41-NEXT:    ptest %xmm0, %xmm0
+; X64-SSE41-NEXT:    sete %al
+; X64-SSE41-NEXT:    retq
+;
+; X64-AVX-LABEL: length31_eq:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    vmovdqu (%rdi), %xmm0
+; X64-AVX-NEXT:    vmovdqu 15(%rdi), %xmm1
+; X64-AVX-NEXT:    vpxor 15(%rsi), %xmm1, %xmm1
+; X64-AVX-NEXT:    vpxor (%rsi), %xmm0, %xmm0
+; X64-AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; X64-AVX-NEXT:    vptest %xmm0, %xmm0
+; X64-AVX-NEXT:    sete %al
+; X64-AVX-NEXT:    retq
+;
+; X64-MIC-AVX-LABEL: length31_eq:
+; X64-MIC-AVX:       # %bb.0:
+; X64-MIC-AVX-NEXT:    vmovdqu (%rdi), %xmm0
+; X64-MIC-AVX-NEXT:    vmovdqu 15(%rdi), %xmm1
+; X64-MIC-AVX-NEXT:    vmovdqu (%rsi), %xmm2
+; X64-MIC-AVX-NEXT:    vmovdqu 15(%rsi), %xmm3
+; X64-MIC-AVX-NEXT:    vpcmpneqd %zmm3, %zmm1, %k0
+; X64-MIC-AVX-NEXT:    vpcmpneqd %zmm2, %zmm0, %k1
+; X64-MIC-AVX-NEXT:    kortestw %k0, %k1
+; X64-MIC-AVX-NEXT:    sete %al
+; X64-MIC-AVX-NEXT:    vzeroupper
+; X64-MIC-AVX-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 31) nounwind
+  %cmp = icmp eq i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length31_lt(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length31_lt:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $31
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    shrl $31, %eax
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: length31_lt:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $31, %edx
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    shrl $31, %eax
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 31) nounwind
+  %cmp = icmp slt i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length31_gt(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length31_gt:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $31
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    setg %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length31_gt:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $31, %edx
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    setg %al
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 31) nounwind
+  %cmp = icmp sgt i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length31_eq_prefer128(i8* %x, i8* %y) nounwind "prefer-vector-width"="128" {
+; X86-NOSSE-LABEL: length31_eq_prefer128:
+; X86-NOSSE:       # %bb.0:
+; X86-NOSSE-NEXT:    pushl $0
+; X86-NOSSE-NEXT:    pushl $31
+; X86-NOSSE-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    calll memcmp
+; X86-NOSSE-NEXT:    addl $16, %esp
+; X86-NOSSE-NEXT:    testl %eax, %eax
+; X86-NOSSE-NEXT:    sete %al
+; X86-NOSSE-NEXT:    retl
+;
+; X86-SSE1-LABEL: length31_eq_prefer128:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    pushl $0
+; X86-SSE1-NEXT:    pushl $31
+; X86-SSE1-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    calll memcmp
+; X86-SSE1-NEXT:    addl $16, %esp
+; X86-SSE1-NEXT:    testl %eax, %eax
+; X86-SSE1-NEXT:    sete %al
+; X86-SSE1-NEXT:    retl
+;
+; X86-SSE2-LABEL: length31_eq_prefer128:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE2-NEXT:    movdqu (%ecx), %xmm0
+; X86-SSE2-NEXT:    movdqu 15(%ecx), %xmm1
+; X86-SSE2-NEXT:    movdqu (%eax), %xmm2
+; X86-SSE2-NEXT:    pcmpeqb %xmm0, %xmm2
+; X86-SSE2-NEXT:    movdqu 15(%eax), %xmm0
+; X86-SSE2-NEXT:    pcmpeqb %xmm1, %xmm0
+; X86-SSE2-NEXT:    pand %xmm2, %xmm0
+; X86-SSE2-NEXT:    pmovmskb %xmm0, %eax
+; X86-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; X86-SSE2-NEXT:    sete %al
+; X86-SSE2-NEXT:    retl
+;
+; X86-SSE41-LABEL: length31_eq_prefer128:
+; X86-SSE41:       # %bb.0:
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE41-NEXT:    movdqu (%ecx), %xmm0
+; X86-SSE41-NEXT:    movdqu 15(%ecx), %xmm1
+; X86-SSE41-NEXT:    movdqu (%eax), %xmm2
+; X86-SSE41-NEXT:    pxor %xmm0, %xmm2
+; X86-SSE41-NEXT:    movdqu 15(%eax), %xmm0
+; X86-SSE41-NEXT:    pxor %xmm1, %xmm0
+; X86-SSE41-NEXT:    por %xmm2, %xmm0
+; X86-SSE41-NEXT:    ptest %xmm0, %xmm0
+; X86-SSE41-NEXT:    sete %al
+; X86-SSE41-NEXT:    retl
+;
+; X64-SSE2-LABEL: length31_eq_prefer128:
+; X64-SSE2:       # %bb.0:
+; X64-SSE2-NEXT:    movdqu (%rdi), %xmm0
+; X64-SSE2-NEXT:    movdqu 15(%rdi), %xmm1
+; X64-SSE2-NEXT:    movdqu (%rsi), %xmm2
+; X64-SSE2-NEXT:    pcmpeqb %xmm0, %xmm2
+; X64-SSE2-NEXT:    movdqu 15(%rsi), %xmm0
+; X64-SSE2-NEXT:    pcmpeqb %xmm1, %xmm0
+; X64-SSE2-NEXT:    pand %xmm2, %xmm0
+; X64-SSE2-NEXT:    pmovmskb %xmm0, %eax
+; X64-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; X64-SSE2-NEXT:    sete %al
+; X64-SSE2-NEXT:    retq
+;
+; X64-SSE41-LABEL: length31_eq_prefer128:
+; X64-SSE41:       # %bb.0:
+; X64-SSE41-NEXT:    movdqu (%rdi), %xmm0
+; X64-SSE41-NEXT:    movdqu 15(%rdi), %xmm1
+; X64-SSE41-NEXT:    movdqu (%rsi), %xmm2
+; X64-SSE41-NEXT:    pxor %xmm0, %xmm2
+; X64-SSE41-NEXT:    movdqu 15(%rsi), %xmm0
+; X64-SSE41-NEXT:    pxor %xmm1, %xmm0
+; X64-SSE41-NEXT:    por %xmm2, %xmm0
+; X64-SSE41-NEXT:    ptest %xmm0, %xmm0
+; X64-SSE41-NEXT:    sete %al
+; X64-SSE41-NEXT:    retq
+;
+; X64-AVX-LABEL: length31_eq_prefer128:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    vmovdqu (%rdi), %xmm0
+; X64-AVX-NEXT:    vmovdqu 15(%rdi), %xmm1
+; X64-AVX-NEXT:    vpxor 15(%rsi), %xmm1, %xmm1
+; X64-AVX-NEXT:    vpxor (%rsi), %xmm0, %xmm0
+; X64-AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; X64-AVX-NEXT:    vptest %xmm0, %xmm0
+; X64-AVX-NEXT:    sete %al
+; X64-AVX-NEXT:    retq
+;
+; X64-MIC-AVX-LABEL: length31_eq_prefer128:
+; X64-MIC-AVX:       # %bb.0:
+; X64-MIC-AVX-NEXT:    vmovdqu (%rdi), %xmm0
+; X64-MIC-AVX-NEXT:    vmovdqu 15(%rdi), %xmm1
+; X64-MIC-AVX-NEXT:    vmovdqu (%rsi), %xmm2
+; X64-MIC-AVX-NEXT:    vmovdqu 15(%rsi), %xmm3
+; X64-MIC-AVX-NEXT:    vpcmpneqd %zmm3, %zmm1, %k0
+; X64-MIC-AVX-NEXT:    vpcmpneqd %zmm2, %zmm0, %k1
+; X64-MIC-AVX-NEXT:    kortestw %k0, %k1
+; X64-MIC-AVX-NEXT:    sete %al
+; X64-MIC-AVX-NEXT:    vzeroupper
+; X64-MIC-AVX-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 31) nounwind
+  %cmp = icmp eq i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length31_eq_const(i8* %X) nounwind {
+; X86-NOSSE-LABEL: length31_eq_const:
+; X86-NOSSE:       # %bb.0:
+; X86-NOSSE-NEXT:    pushl $0
+; X86-NOSSE-NEXT:    pushl $31
+; X86-NOSSE-NEXT:    pushl $.L.str
+; X86-NOSSE-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    calll memcmp
+; X86-NOSSE-NEXT:    addl $16, %esp
+; X86-NOSSE-NEXT:    testl %eax, %eax
+; X86-NOSSE-NEXT:    setne %al
+; X86-NOSSE-NEXT:    retl
+;
+; X86-SSE1-LABEL: length31_eq_const:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    pushl $0
+; X86-SSE1-NEXT:    pushl $31
+; X86-SSE1-NEXT:    pushl $.L.str
+; X86-SSE1-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    calll memcmp
+; X86-SSE1-NEXT:    addl $16, %esp
+; X86-SSE1-NEXT:    testl %eax, %eax
+; X86-SSE1-NEXT:    setne %al
+; X86-SSE1-NEXT:    retl
+;
+; X86-SSE2-LABEL: length31_eq_const:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT:    movdqu (%eax), %xmm0
+; X86-SSE2-NEXT:    movdqu 15(%eax), %xmm1
+; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI.*}}, %xmm1
+; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand %xmm1, %xmm0
+; X86-SSE2-NEXT:    pmovmskb %xmm0, %eax
+; X86-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; X86-SSE2-NEXT:    setne %al
+; X86-SSE2-NEXT:    retl
+;
+; X86-SSE41-LABEL: length31_eq_const:
+; X86-SSE41:       # %bb.0:
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT:    movdqu (%eax), %xmm0
+; X86-SSE41-NEXT:    movdqu 15(%eax), %xmm1
+; X86-SSE41-NEXT:    pxor {{\.LCPI.*}}, %xmm1
+; X86-SSE41-NEXT:    pxor {{\.LCPI.*}}, %xmm0
+; X86-SSE41-NEXT:    por %xmm1, %xmm0
+; X86-SSE41-NEXT:    ptest %xmm0, %xmm0
+; X86-SSE41-NEXT:    setne %al
+; X86-SSE41-NEXT:    retl
+;
+; X64-SSE2-LABEL: length31_eq_const:
+; X64-SSE2:       # %bb.0:
+; X64-SSE2-NEXT:    movdqu (%rdi), %xmm0
+; X64-SSE2-NEXT:    movdqu 15(%rdi), %xmm1
+; X64-SSE2-NEXT:    pcmpeqb {{.*}}(%rip), %xmm1
+; X64-SSE2-NEXT:    pcmpeqb {{.*}}(%rip), %xmm0
+; X64-SSE2-NEXT:    pand %xmm1, %xmm0
+; X64-SSE2-NEXT:    pmovmskb %xmm0, %eax
+; X64-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; X64-SSE2-NEXT:    setne %al
+; X64-SSE2-NEXT:    retq
+;
+; X64-SSE41-LABEL: length31_eq_const:
+; X64-SSE41:       # %bb.0:
+; X64-SSE41-NEXT:    movdqu (%rdi), %xmm0
+; X64-SSE41-NEXT:    movdqu 15(%rdi), %xmm1
+; X64-SSE41-NEXT:    pxor {{.*}}(%rip), %xmm1
+; X64-SSE41-NEXT:    pxor {{.*}}(%rip), %xmm0
+; X64-SSE41-NEXT:    por %xmm1, %xmm0
+; X64-SSE41-NEXT:    ptest %xmm0, %xmm0
+; X64-SSE41-NEXT:    setne %al
+; X64-SSE41-NEXT:    retq
+;
+; X64-AVX-LABEL: length31_eq_const:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    vmovdqu (%rdi), %xmm0
+; X64-AVX-NEXT:    vmovdqu 15(%rdi), %xmm1
+; X64-AVX-NEXT:    vpxor {{.*}}(%rip), %xmm1, %xmm1
+; X64-AVX-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; X64-AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; X64-AVX-NEXT:    vptest %xmm0, %xmm0
+; X64-AVX-NEXT:    setne %al
+; X64-AVX-NEXT:    retq
+;
+; X64-MIC-AVX-LABEL: length31_eq_const:
+; X64-MIC-AVX:       # %bb.0:
+; X64-MIC-AVX-NEXT:    vmovdqu (%rdi), %xmm0
+; X64-MIC-AVX-NEXT:    vmovdqu 15(%rdi), %xmm1
+; X64-MIC-AVX-NEXT:    vmovdqa {{.*#+}} xmm2 = [943142453,842084409,909456435,809056311]
+; X64-MIC-AVX-NEXT:    vpcmpneqd %zmm2, %zmm1, %k0
+; X64-MIC-AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [858927408,926299444,825243960,892613426]
+; X64-MIC-AVX-NEXT:    vpcmpneqd %zmm1, %zmm0, %k1
+; X64-MIC-AVX-NEXT:    kortestw %k0, %k1
+; X64-MIC-AVX-NEXT:    setne %al
+; X64-MIC-AVX-NEXT:    vzeroupper
+; X64-MIC-AVX-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* getelementptr inbounds ([513 x i8], [513 x i8]* @.str, i32 0, i32 0), i64 31) nounwind
+  %c = icmp ne i32 %m, 0
+  ret i1 %c
+}
+
+define i32 @length32(i8* %X, i8* %Y) nounwind {
+; X86-LABEL: length32:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $32
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    retl
+;
+; X64-LABEL: length32:
+; X64:       # %bb.0:
+; X64-NEXT:    movq (%rdi), %rcx
+; X64-NEXT:    movq (%rsi), %rdx
+; X64-NEXT:    bswapq %rcx
+; X64-NEXT:    bswapq %rdx
+; X64-NEXT:    cmpq %rdx, %rcx
+; X64-NEXT:    jne .LBB47_4
+; X64-NEXT:  # %bb.1: # %loadbb1
+; X64-NEXT:    movq 8(%rdi), %rcx
+; X64-NEXT:    movq 8(%rsi), %rdx
+; X64-NEXT:    bswapq %rcx
+; X64-NEXT:    bswapq %rdx
+; X64-NEXT:    cmpq %rdx, %rcx
+; X64-NEXT:    jne .LBB47_4
+; X64-NEXT:  # %bb.2: # %loadbb2
+; X64-NEXT:    movq 16(%rdi), %rcx
+; X64-NEXT:    movq 16(%rsi), %rdx
+; X64-NEXT:    bswapq %rcx
+; X64-NEXT:    bswapq %rdx
+; X64-NEXT:    cmpq %rdx, %rcx
+; X64-NEXT:    jne .LBB47_4
+; X64-NEXT:  # %bb.3: # %loadbb3
+; X64-NEXT:    movq 24(%rdi), %rcx
+; X64-NEXT:    movq 24(%rsi), %rdx
+; X64-NEXT:    bswapq %rcx
+; X64-NEXT:    bswapq %rdx
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpq %rdx, %rcx
+; X64-NEXT:    je .LBB47_5
+; X64-NEXT:  .LBB47_4: # %res_block
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpq %rdx, %rcx
+; X64-NEXT:    setae %al
+; X64-NEXT:    leal -1(%rax,%rax), %eax
+; X64-NEXT:  .LBB47_5: # %endblock
+; X64-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 32) nounwind
+  ret i32 %m
+}
+
+; PR33325 - https://bugs.llvm.org/show_bug.cgi?id=33325
+
+define i1 @length32_eq(i8* %x, i8* %y) nounwind {
+; X86-NOSSE-LABEL: length32_eq:
+; X86-NOSSE:       # %bb.0:
+; X86-NOSSE-NEXT:    pushl $0
+; X86-NOSSE-NEXT:    pushl $32
+; X86-NOSSE-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    calll memcmp
+; X86-NOSSE-NEXT:    addl $16, %esp
+; X86-NOSSE-NEXT:    testl %eax, %eax
+; X86-NOSSE-NEXT:    sete %al
+; X86-NOSSE-NEXT:    retl
+;
+; X86-SSE1-LABEL: length32_eq:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    pushl $0
+; X86-SSE1-NEXT:    pushl $32
+; X86-SSE1-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    calll memcmp
+; X86-SSE1-NEXT:    addl $16, %esp
+; X86-SSE1-NEXT:    testl %eax, %eax
+; X86-SSE1-NEXT:    sete %al
+; X86-SSE1-NEXT:    retl
+;
+; X86-SSE2-LABEL: length32_eq:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE2-NEXT:    movdqu (%ecx), %xmm0
+; X86-SSE2-NEXT:    movdqu 16(%ecx), %xmm1
+; X86-SSE2-NEXT:    movdqu (%eax), %xmm2
+; X86-SSE2-NEXT:    pcmpeqb %xmm0, %xmm2
+; X86-SSE2-NEXT:    movdqu 16(%eax), %xmm0
+; X86-SSE2-NEXT:    pcmpeqb %xmm1, %xmm0
+; X86-SSE2-NEXT:    pand %xmm2, %xmm0
+; X86-SSE2-NEXT:    pmovmskb %xmm0, %eax
+; X86-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; X86-SSE2-NEXT:    sete %al
+; X86-SSE2-NEXT:    retl
+;
+; X86-SSE41-LABEL: length32_eq:
+; X86-SSE41:       # %bb.0:
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE41-NEXT:    movdqu (%ecx), %xmm0
+; X86-SSE41-NEXT:    movdqu 16(%ecx), %xmm1
+; X86-SSE41-NEXT:    movdqu (%eax), %xmm2
+; X86-SSE41-NEXT:    pxor %xmm0, %xmm2
+; X86-SSE41-NEXT:    movdqu 16(%eax), %xmm0
+; X86-SSE41-NEXT:    pxor %xmm1, %xmm0
+; X86-SSE41-NEXT:    por %xmm2, %xmm0
+; X86-SSE41-NEXT:    ptest %xmm0, %xmm0
+; X86-SSE41-NEXT:    sete %al
+; X86-SSE41-NEXT:    retl
+;
+; X64-SSE2-LABEL: length32_eq:
+; X64-SSE2:       # %bb.0:
+; X64-SSE2-NEXT:    movdqu (%rdi), %xmm0
+; X64-SSE2-NEXT:    movdqu 16(%rdi), %xmm1
+; X64-SSE2-NEXT:    movdqu (%rsi), %xmm2
+; X64-SSE2-NEXT:    pcmpeqb %xmm0, %xmm2
+; X64-SSE2-NEXT:    movdqu 16(%rsi), %xmm0
+; X64-SSE2-NEXT:    pcmpeqb %xmm1, %xmm0
+; X64-SSE2-NEXT:    pand %xmm2, %xmm0
+; X64-SSE2-NEXT:    pmovmskb %xmm0, %eax
+; X64-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; X64-SSE2-NEXT:    sete %al
+; X64-SSE2-NEXT:    retq
+;
+; X64-SSE41-LABEL: length32_eq:
+; X64-SSE41:       # %bb.0:
+; X64-SSE41-NEXT:    movdqu (%rdi), %xmm0
+; X64-SSE41-NEXT:    movdqu 16(%rdi), %xmm1
+; X64-SSE41-NEXT:    movdqu (%rsi), %xmm2
+; X64-SSE41-NEXT:    pxor %xmm0, %xmm2
+; X64-SSE41-NEXT:    movdqu 16(%rsi), %xmm0
+; X64-SSE41-NEXT:    pxor %xmm1, %xmm0
+; X64-SSE41-NEXT:    por %xmm2, %xmm0
+; X64-SSE41-NEXT:    ptest %xmm0, %xmm0
+; X64-SSE41-NEXT:    sete %al
+; X64-SSE41-NEXT:    retq
+;
+; X64-AVX1-LABEL: length32_eq:
+; X64-AVX1:       # %bb.0:
+; X64-AVX1-NEXT:    vmovups (%rdi), %ymm0
+; X64-AVX1-NEXT:    vxorps (%rsi), %ymm0, %ymm0
+; X64-AVX1-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX1-NEXT:    sete %al
+; X64-AVX1-NEXT:    vzeroupper
+; X64-AVX1-NEXT:    retq
+;
+; X64-AVX2-LABEL: length32_eq:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    vmovdqu (%rdi), %ymm0
+; X64-AVX2-NEXT:    vpxor (%rsi), %ymm0, %ymm0
+; X64-AVX2-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX2-NEXT:    sete %al
+; X64-AVX2-NEXT:    vzeroupper
+; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512-LABEL: length32_eq:
+; X64-AVX512:       # %bb.0:
+; X64-AVX512-NEXT:    vmovdqu (%rdi), %ymm0
+; X64-AVX512-NEXT:    vpxor (%rsi), %ymm0, %ymm0
+; X64-AVX512-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX512-NEXT:    sete %al
+; X64-AVX512-NEXT:    vzeroupper
+; X64-AVX512-NEXT:    retq
+;
+; X64-MIC-AVX-LABEL: length32_eq:
+; X64-MIC-AVX:       # %bb.0:
+; X64-MIC-AVX-NEXT:    vmovdqu (%rdi), %ymm0
+; X64-MIC-AVX-NEXT:    vmovdqu (%rsi), %ymm1
+; X64-MIC-AVX-NEXT:    vpcmpneqd %zmm1, %zmm0, %k0
+; X64-MIC-AVX-NEXT:    kortestw %k0, %k0
+; X64-MIC-AVX-NEXT:    sete %al
+; X64-MIC-AVX-NEXT:    vzeroupper
+; X64-MIC-AVX-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 32) nounwind
+  %cmp = icmp eq i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length32_lt(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length32_lt:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $32
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    shrl $31, %eax
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: length32_lt:
+; X64:       # %bb.0:
+; X64-NEXT:    movq (%rdi), %rcx
+; X64-NEXT:    movq (%rsi), %rdx
+; X64-NEXT:    bswapq %rcx
+; X64-NEXT:    bswapq %rdx
+; X64-NEXT:    cmpq %rdx, %rcx
+; X64-NEXT:    jne .LBB49_4
+; X64-NEXT:  # %bb.1: # %loadbb1
+; X64-NEXT:    movq 8(%rdi), %rcx
+; X64-NEXT:    movq 8(%rsi), %rdx
+; X64-NEXT:    bswapq %rcx
+; X64-NEXT:    bswapq %rdx
+; X64-NEXT:    cmpq %rdx, %rcx
+; X64-NEXT:    jne .LBB49_4
+; X64-NEXT:  # %bb.2: # %loadbb2
+; X64-NEXT:    movq 16(%rdi), %rcx
+; X64-NEXT:    movq 16(%rsi), %rdx
+; X64-NEXT:    bswapq %rcx
+; X64-NEXT:    bswapq %rdx
+; X64-NEXT:    cmpq %rdx, %rcx
+; X64-NEXT:    jne .LBB49_4
+; X64-NEXT:  # %bb.3: # %loadbb3
+; X64-NEXT:    movq 24(%rdi), %rcx
+; X64-NEXT:    movq 24(%rsi), %rdx
+; X64-NEXT:    bswapq %rcx
+; X64-NEXT:    bswapq %rdx
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpq %rdx, %rcx
+; X64-NEXT:    je .LBB49_5
+; X64-NEXT:  .LBB49_4: # %res_block
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpq %rdx, %rcx
+; X64-NEXT:    setae %al
+; X64-NEXT:    leal -1(%rax,%rax), %eax
+; X64-NEXT:  .LBB49_5: # %endblock
+; X64-NEXT:    shrl $31, %eax
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 32) nounwind
+  %cmp = icmp slt i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length32_gt(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length32_gt:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $32
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    setg %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length32_gt:
+; X64:       # %bb.0:
+; X64-NEXT:    movq (%rdi), %rax
+; X64-NEXT:    movq (%rsi), %rcx
+; X64-NEXT:    bswapq %rax
+; X64-NEXT:    bswapq %rcx
+; X64-NEXT:    cmpq %rcx, %rax
+; X64-NEXT:    jne .LBB50_4
+; X64-NEXT:  # %bb.1: # %loadbb1
+; X64-NEXT:    movq 8(%rdi), %rax
+; X64-NEXT:    movq 8(%rsi), %rcx
+; X64-NEXT:    bswapq %rax
+; X64-NEXT:    bswapq %rcx
+; X64-NEXT:    cmpq %rcx, %rax
+; X64-NEXT:    jne .LBB50_4
+; X64-NEXT:  # %bb.2: # %loadbb2
+; X64-NEXT:    movq 16(%rdi), %rax
+; X64-NEXT:    movq 16(%rsi), %rcx
+; X64-NEXT:    bswapq %rax
+; X64-NEXT:    bswapq %rcx
+; X64-NEXT:    cmpq %rcx, %rax
+; X64-NEXT:    jne .LBB50_4
+; X64-NEXT:  # %bb.3: # %loadbb3
+; X64-NEXT:    movq 24(%rdi), %rax
+; X64-NEXT:    movq 24(%rsi), %rcx
+; X64-NEXT:    bswapq %rax
+; X64-NEXT:    bswapq %rcx
+; X64-NEXT:    xorl %edx, %edx
+; X64-NEXT:    cmpq %rcx, %rax
+; X64-NEXT:    je .LBB50_5
+; X64-NEXT:  .LBB50_4: # %res_block
+; X64-NEXT:    xorl %edx, %edx
+; X64-NEXT:    cmpq %rcx, %rax
+; X64-NEXT:    setae %dl
+; X64-NEXT:    leal -1(%rdx,%rdx), %edx
+; X64-NEXT:  .LBB50_5: # %endblock
+; X64-NEXT:    testl %edx, %edx
+; X64-NEXT:    setg %al
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 32) nounwind
+  %cmp = icmp sgt i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length32_eq_prefer128(i8* %x, i8* %y) nounwind "prefer-vector-width"="128" {
+; X86-NOSSE-LABEL: length32_eq_prefer128:
+; X86-NOSSE:       # %bb.0:
+; X86-NOSSE-NEXT:    pushl $0
+; X86-NOSSE-NEXT:    pushl $32
+; X86-NOSSE-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    calll memcmp
+; X86-NOSSE-NEXT:    addl $16, %esp
+; X86-NOSSE-NEXT:    testl %eax, %eax
+; X86-NOSSE-NEXT:    sete %al
+; X86-NOSSE-NEXT:    retl
+;
+; X86-SSE1-LABEL: length32_eq_prefer128:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    pushl $0
+; X86-SSE1-NEXT:    pushl $32
+; X86-SSE1-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    calll memcmp
+; X86-SSE1-NEXT:    addl $16, %esp
+; X86-SSE1-NEXT:    testl %eax, %eax
+; X86-SSE1-NEXT:    sete %al
+; X86-SSE1-NEXT:    retl
+;
+; X86-SSE2-LABEL: length32_eq_prefer128:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE2-NEXT:    movdqu (%ecx), %xmm0
+; X86-SSE2-NEXT:    movdqu 16(%ecx), %xmm1
+; X86-SSE2-NEXT:    movdqu (%eax), %xmm2
+; X86-SSE2-NEXT:    pcmpeqb %xmm0, %xmm2
+; X86-SSE2-NEXT:    movdqu 16(%eax), %xmm0
+; X86-SSE2-NEXT:    pcmpeqb %xmm1, %xmm0
+; X86-SSE2-NEXT:    pand %xmm2, %xmm0
+; X86-SSE2-NEXT:    pmovmskb %xmm0, %eax
+; X86-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; X86-SSE2-NEXT:    sete %al
+; X86-SSE2-NEXT:    retl
+;
+; X86-SSE41-LABEL: length32_eq_prefer128:
+; X86-SSE41:       # %bb.0:
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE41-NEXT:    movdqu (%ecx), %xmm0
+; X86-SSE41-NEXT:    movdqu 16(%ecx), %xmm1
+; X86-SSE41-NEXT:    movdqu (%eax), %xmm2
+; X86-SSE41-NEXT:    pxor %xmm0, %xmm2
+; X86-SSE41-NEXT:    movdqu 16(%eax), %xmm0
+; X86-SSE41-NEXT:    pxor %xmm1, %xmm0
+; X86-SSE41-NEXT:    por %xmm2, %xmm0
+; X86-SSE41-NEXT:    ptest %xmm0, %xmm0
+; X86-SSE41-NEXT:    sete %al
+; X86-SSE41-NEXT:    retl
+;
+; X64-SSE2-LABEL: length32_eq_prefer128:
+; X64-SSE2:       # %bb.0:
+; X64-SSE2-NEXT:    movdqu (%rdi), %xmm0
+; X64-SSE2-NEXT:    movdqu 16(%rdi), %xmm1
+; X64-SSE2-NEXT:    movdqu (%rsi), %xmm2
+; X64-SSE2-NEXT:    pcmpeqb %xmm0, %xmm2
+; X64-SSE2-NEXT:    movdqu 16(%rsi), %xmm0
+; X64-SSE2-NEXT:    pcmpeqb %xmm1, %xmm0
+; X64-SSE2-NEXT:    pand %xmm2, %xmm0
+; X64-SSE2-NEXT:    pmovmskb %xmm0, %eax
+; X64-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; X64-SSE2-NEXT:    sete %al
+; X64-SSE2-NEXT:    retq
+;
+; X64-SSE41-LABEL: length32_eq_prefer128:
+; X64-SSE41:       # %bb.0:
+; X64-SSE41-NEXT:    movdqu (%rdi), %xmm0
+; X64-SSE41-NEXT:    movdqu 16(%rdi), %xmm1
+; X64-SSE41-NEXT:    movdqu (%rsi), %xmm2
+; X64-SSE41-NEXT:    pxor %xmm0, %xmm2
+; X64-SSE41-NEXT:    movdqu 16(%rsi), %xmm0
+; X64-SSE41-NEXT:    pxor %xmm1, %xmm0
+; X64-SSE41-NEXT:    por %xmm2, %xmm0
+; X64-SSE41-NEXT:    ptest %xmm0, %xmm0
+; X64-SSE41-NEXT:    sete %al
+; X64-SSE41-NEXT:    retq
+;
+; X64-AVX-LABEL: length32_eq_prefer128:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    vmovdqu (%rdi), %xmm0
+; X64-AVX-NEXT:    vmovdqu 16(%rdi), %xmm1
+; X64-AVX-NEXT:    vpxor 16(%rsi), %xmm1, %xmm1
+; X64-AVX-NEXT:    vpxor (%rsi), %xmm0, %xmm0
+; X64-AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; X64-AVX-NEXT:    vptest %xmm0, %xmm0
+; X64-AVX-NEXT:    sete %al
+; X64-AVX-NEXT:    retq
+;
+; X64-MIC-AVX-LABEL: length32_eq_prefer128:
+; X64-MIC-AVX:       # %bb.0:
+; X64-MIC-AVX-NEXT:    vmovdqu (%rdi), %xmm0
+; X64-MIC-AVX-NEXT:    vmovdqu 16(%rdi), %xmm1
+; X64-MIC-AVX-NEXT:    vmovdqu (%rsi), %xmm2
+; X64-MIC-AVX-NEXT:    vmovdqu 16(%rsi), %xmm3
+; X64-MIC-AVX-NEXT:    vpcmpneqd %zmm3, %zmm1, %k0
+; X64-MIC-AVX-NEXT:    vpcmpneqd %zmm2, %zmm0, %k1
+; X64-MIC-AVX-NEXT:    kortestw %k0, %k1
+; X64-MIC-AVX-NEXT:    sete %al
+; X64-MIC-AVX-NEXT:    vzeroupper
+; X64-MIC-AVX-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 32) nounwind
+  %cmp = icmp eq i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length32_eq_const(i8* %X) nounwind {
+; X86-NOSSE-LABEL: length32_eq_const:
+; X86-NOSSE:       # %bb.0:
+; X86-NOSSE-NEXT:    pushl $0
+; X86-NOSSE-NEXT:    pushl $32
+; X86-NOSSE-NEXT:    pushl $.L.str
+; X86-NOSSE-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    calll memcmp
+; X86-NOSSE-NEXT:    addl $16, %esp
+; X86-NOSSE-NEXT:    testl %eax, %eax
+; X86-NOSSE-NEXT:    setne %al
+; X86-NOSSE-NEXT:    retl
+;
+; X86-SSE1-LABEL: length32_eq_const:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    pushl $0
+; X86-SSE1-NEXT:    pushl $32
+; X86-SSE1-NEXT:    pushl $.L.str
+; X86-SSE1-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    calll memcmp
+; X86-SSE1-NEXT:    addl $16, %esp
+; X86-SSE1-NEXT:    testl %eax, %eax
+; X86-SSE1-NEXT:    setne %al
+; X86-SSE1-NEXT:    retl
+;
+; X86-SSE2-LABEL: length32_eq_const:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT:    movdqu (%eax), %xmm0
+; X86-SSE2-NEXT:    movdqu 16(%eax), %xmm1
+; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI.*}}, %xmm1
+; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand %xmm1, %xmm0
+; X86-SSE2-NEXT:    pmovmskb %xmm0, %eax
+; X86-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; X86-SSE2-NEXT:    setne %al
+; X86-SSE2-NEXT:    retl
+;
+; X86-SSE41-LABEL: length32_eq_const:
+; X86-SSE41:       # %bb.0:
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT:    movdqu (%eax), %xmm0
+; X86-SSE41-NEXT:    movdqu 16(%eax), %xmm1
+; X86-SSE41-NEXT:    pxor {{\.LCPI.*}}, %xmm1
+; X86-SSE41-NEXT:    pxor {{\.LCPI.*}}, %xmm0
+; X86-SSE41-NEXT:    por %xmm1, %xmm0
+; X86-SSE41-NEXT:    ptest %xmm0, %xmm0
+; X86-SSE41-NEXT:    setne %al
+; X86-SSE41-NEXT:    retl
+;
+; X64-SSE2-LABEL: length32_eq_const:
+; X64-SSE2:       # %bb.0:
+; X64-SSE2-NEXT:    movdqu (%rdi), %xmm0
+; X64-SSE2-NEXT:    movdqu 16(%rdi), %xmm1
+; X64-SSE2-NEXT:    pcmpeqb {{.*}}(%rip), %xmm1
+; X64-SSE2-NEXT:    pcmpeqb {{.*}}(%rip), %xmm0
+; X64-SSE2-NEXT:    pand %xmm1, %xmm0
+; X64-SSE2-NEXT:    pmovmskb %xmm0, %eax
+; X64-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; X64-SSE2-NEXT:    setne %al
+; X64-SSE2-NEXT:    retq
+;
+; X64-SSE41-LABEL: length32_eq_const:
+; X64-SSE41:       # %bb.0:
+; X64-SSE41-NEXT:    movdqu (%rdi), %xmm0
+; X64-SSE41-NEXT:    movdqu 16(%rdi), %xmm1
+; X64-SSE41-NEXT:    pxor {{.*}}(%rip), %xmm1
+; X64-SSE41-NEXT:    pxor {{.*}}(%rip), %xmm0
+; X64-SSE41-NEXT:    por %xmm1, %xmm0
+; X64-SSE41-NEXT:    ptest %xmm0, %xmm0
+; X64-SSE41-NEXT:    setne %al
+; X64-SSE41-NEXT:    retq
+;
+; X64-AVX1-LABEL: length32_eq_const:
+; X64-AVX1:       # %bb.0:
+; X64-AVX1-NEXT:    vmovups (%rdi), %ymm0
+; X64-AVX1-NEXT:    vxorps {{.*}}(%rip), %ymm0, %ymm0
+; X64-AVX1-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX1-NEXT:    setne %al
+; X64-AVX1-NEXT:    vzeroupper
+; X64-AVX1-NEXT:    retq
+;
+; X64-AVX2-LABEL: length32_eq_const:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    vmovdqu (%rdi), %ymm0
+; X64-AVX2-NEXT:    vpxor {{.*}}(%rip), %ymm0, %ymm0
+; X64-AVX2-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX2-NEXT:    setne %al
+; X64-AVX2-NEXT:    vzeroupper
+; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512-LABEL: length32_eq_const:
+; X64-AVX512:       # %bb.0:
+; X64-AVX512-NEXT:    vmovdqu (%rdi), %ymm0
+; X64-AVX512-NEXT:    vpxor {{.*}}(%rip), %ymm0, %ymm0
+; X64-AVX512-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX512-NEXT:    setne %al
+; X64-AVX512-NEXT:    vzeroupper
+; X64-AVX512-NEXT:    retq
+;
+; X64-MIC-AVX-LABEL: length32_eq_const:
+; X64-MIC-AVX:       # %bb.0:
+; X64-MIC-AVX-NEXT:    vmovdqu (%rdi), %ymm0
+; X64-MIC-AVX-NEXT:    vmovdqa {{.*#+}} ymm1 = [858927408,926299444,825243960,892613426,959985462,858927408,926299444,825243960]
+; X64-MIC-AVX-NEXT:    vpcmpneqd %zmm1, %zmm0, %k0
+; X64-MIC-AVX-NEXT:    kortestw %k0, %k0
+; X64-MIC-AVX-NEXT:    setne %al
+; X64-MIC-AVX-NEXT:    vzeroupper
+; X64-MIC-AVX-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* getelementptr inbounds ([513 x i8], [513 x i8]* @.str, i32 0, i32 0), i64 32) nounwind
+  %c = icmp ne i32 %m, 0
+  ret i1 %c
+}
+
+define i32 @length48(i8* %X, i8* %Y) nounwind {
+; X86-LABEL: length48:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $48
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    retl
+;
+; X64-LABEL: length48:
+; X64:       # %bb.0:
+; X64-NEXT:    movl $48, %edx
+; X64-NEXT:    jmp memcmp # TAILCALL
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 48) nounwind
+  ret i32 %m
+}
+
+define i1 @length48_eq(i8* %x, i8* %y) nounwind {
+; X86-NOSSE-LABEL: length48_eq:
+; X86-NOSSE:       # %bb.0:
+; X86-NOSSE-NEXT:    pushl $0
+; X86-NOSSE-NEXT:    pushl $48
+; X86-NOSSE-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    calll memcmp
+; X86-NOSSE-NEXT:    addl $16, %esp
+; X86-NOSSE-NEXT:    testl %eax, %eax
+; X86-NOSSE-NEXT:    sete %al
+; X86-NOSSE-NEXT:    retl
+;
+; X86-SSE1-LABEL: length48_eq:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    pushl $0
+; X86-SSE1-NEXT:    pushl $48
+; X86-SSE1-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    calll memcmp
+; X86-SSE1-NEXT:    addl $16, %esp
+; X86-SSE1-NEXT:    testl %eax, %eax
+; X86-SSE1-NEXT:    sete %al
+; X86-SSE1-NEXT:    retl
+;
+; X86-SSE2-LABEL: length48_eq:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE2-NEXT:    movdqu (%ecx), %xmm0
+; X86-SSE2-NEXT:    movdqu 16(%ecx), %xmm1
+; X86-SSE2-NEXT:    movdqu (%eax), %xmm2
+; X86-SSE2-NEXT:    pcmpeqb %xmm0, %xmm2
+; X86-SSE2-NEXT:    movdqu 16(%eax), %xmm0
+; X86-SSE2-NEXT:    pcmpeqb %xmm1, %xmm0
+; X86-SSE2-NEXT:    movdqu 32(%ecx), %xmm1
+; X86-SSE2-NEXT:    movdqu 32(%eax), %xmm3
+; X86-SSE2-NEXT:    pcmpeqb %xmm1, %xmm3
+; X86-SSE2-NEXT:    pand %xmm0, %xmm3
+; X86-SSE2-NEXT:    pand %xmm2, %xmm3
+; X86-SSE2-NEXT:    pmovmskb %xmm3, %eax
+; X86-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; X86-SSE2-NEXT:    sete %al
+; X86-SSE2-NEXT:    retl
+;
+; X86-SSE41-LABEL: length48_eq:
+; X86-SSE41:       # %bb.0:
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE41-NEXT:    movdqu (%ecx), %xmm0
+; X86-SSE41-NEXT:    movdqu 16(%ecx), %xmm1
+; X86-SSE41-NEXT:    movdqu (%eax), %xmm2
+; X86-SSE41-NEXT:    pxor %xmm0, %xmm2
+; X86-SSE41-NEXT:    movdqu 16(%eax), %xmm0
+; X86-SSE41-NEXT:    pxor %xmm1, %xmm0
+; X86-SSE41-NEXT:    movdqu 32(%ecx), %xmm1
+; X86-SSE41-NEXT:    movdqu 32(%eax), %xmm3
+; X86-SSE41-NEXT:    pxor %xmm1, %xmm3
+; X86-SSE41-NEXT:    por %xmm0, %xmm3
+; X86-SSE41-NEXT:    por %xmm2, %xmm3
+; X86-SSE41-NEXT:    ptest %xmm3, %xmm3
+; X86-SSE41-NEXT:    sete %al
+; X86-SSE41-NEXT:    retl
+;
+; X64-SSE2-LABEL: length48_eq:
+; X64-SSE2:       # %bb.0:
+; X64-SSE2-NEXT:    movdqu (%rdi), %xmm0
+; X64-SSE2-NEXT:    movdqu 16(%rdi), %xmm1
+; X64-SSE2-NEXT:    movdqu 32(%rdi), %xmm2
+; X64-SSE2-NEXT:    movdqu (%rsi), %xmm3
+; X64-SSE2-NEXT:    pcmpeqb %xmm0, %xmm3
+; X64-SSE2-NEXT:    movdqu 16(%rsi), %xmm0
+; X64-SSE2-NEXT:    pcmpeqb %xmm1, %xmm0
+; X64-SSE2-NEXT:    movdqu 32(%rsi), %xmm1
+; X64-SSE2-NEXT:    pcmpeqb %xmm2, %xmm1
+; X64-SSE2-NEXT:    pand %xmm0, %xmm1
+; X64-SSE2-NEXT:    pand %xmm3, %xmm1
+; X64-SSE2-NEXT:    pmovmskb %xmm1, %eax
+; X64-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; X64-SSE2-NEXT:    sete %al
+; X64-SSE2-NEXT:    retq
+;
+; X64-SSE41-LABEL: length48_eq:
+; X64-SSE41:       # %bb.0:
+; X64-SSE41-NEXT:    movdqu (%rdi), %xmm0
+; X64-SSE41-NEXT:    movdqu 16(%rdi), %xmm1
+; X64-SSE41-NEXT:    movdqu 32(%rdi), %xmm2
+; X64-SSE41-NEXT:    movdqu (%rsi), %xmm3
+; X64-SSE41-NEXT:    pxor %xmm0, %xmm3
+; X64-SSE41-NEXT:    movdqu 16(%rsi), %xmm0
+; X64-SSE41-NEXT:    pxor %xmm1, %xmm0
+; X64-SSE41-NEXT:    movdqu 32(%rsi), %xmm1
+; X64-SSE41-NEXT:    pxor %xmm2, %xmm1
+; X64-SSE41-NEXT:    por %xmm0, %xmm1
+; X64-SSE41-NEXT:    por %xmm3, %xmm1
+; X64-SSE41-NEXT:    ptest %xmm1, %xmm1
+; X64-SSE41-NEXT:    sete %al
+; X64-SSE41-NEXT:    retq
+;
+; X64-AVX1-LABEL: length48_eq:
+; X64-AVX1:       # %bb.0:
+; X64-AVX1-NEXT:    vmovups (%rdi), %ymm0
+; X64-AVX1-NEXT:    vmovups 32(%rdi), %xmm1
+; X64-AVX1-NEXT:    vmovups 32(%rsi), %xmm2
+; X64-AVX1-NEXT:    vxorps (%rsi), %ymm0, %ymm0
+; X64-AVX1-NEXT:    vxorps %ymm2, %ymm1, %ymm1
+; X64-AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; X64-AVX1-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX1-NEXT:    sete %al
+; X64-AVX1-NEXT:    vzeroupper
+; X64-AVX1-NEXT:    retq
+;
+; X64-AVX2-LABEL: length48_eq:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    vmovdqu (%rdi), %ymm0
+; X64-AVX2-NEXT:    vmovdqu 32(%rdi), %xmm1
+; X64-AVX2-NEXT:    vmovdqu 32(%rsi), %xmm2
+; X64-AVX2-NEXT:    vpxor (%rsi), %ymm0, %ymm0
+; X64-AVX2-NEXT:    vpxor %ymm2, %ymm1, %ymm1
+; X64-AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; X64-AVX2-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX2-NEXT:    sete %al
+; X64-AVX2-NEXT:    vzeroupper
+; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512-LABEL: length48_eq:
+; X64-AVX512:       # %bb.0:
+; X64-AVX512-NEXT:    vmovdqu (%rdi), %ymm0
+; X64-AVX512-NEXT:    vmovdqu 32(%rdi), %xmm1
+; X64-AVX512-NEXT:    vmovdqu 32(%rsi), %xmm2
+; X64-AVX512-NEXT:    vpxor (%rsi), %ymm0, %ymm0
+; X64-AVX512-NEXT:    vpxor %ymm2, %ymm1, %ymm1
+; X64-AVX512-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; X64-AVX512-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX512-NEXT:    sete %al
+; X64-AVX512-NEXT:    vzeroupper
+; X64-AVX512-NEXT:    retq
+;
+; X64-MIC-AVX-LABEL: length48_eq:
+; X64-MIC-AVX:       # %bb.0:
+; X64-MIC-AVX-NEXT:    vmovdqu (%rdi), %ymm0
+; X64-MIC-AVX-NEXT:    vmovdqu (%rsi), %ymm1
+; X64-MIC-AVX-NEXT:    vmovdqu 32(%rdi), %xmm2
+; X64-MIC-AVX-NEXT:    vmovdqu 32(%rsi), %xmm3
+; X64-MIC-AVX-NEXT:    vpcmpneqd %zmm3, %zmm2, %k0
+; X64-MIC-AVX-NEXT:    vpcmpneqd %zmm1, %zmm0, %k1
+; X64-MIC-AVX-NEXT:    kortestw %k0, %k1
+; X64-MIC-AVX-NEXT:    sete %al
+; X64-MIC-AVX-NEXT:    vzeroupper
+; X64-MIC-AVX-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 48) nounwind
+  %cmp = icmp eq i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length48_lt(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length48_lt:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $48
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    shrl $31, %eax
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: length48_lt:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $48, %edx
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    shrl $31, %eax
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 48) nounwind
+  %cmp = icmp slt i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length48_gt(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length48_gt:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $48
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    setg %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length48_gt:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $48, %edx
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    setg %al
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 48) nounwind
+  %cmp = icmp sgt i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length48_eq_prefer128(i8* %x, i8* %y) nounwind "prefer-vector-width"="128" {
+; X86-NOSSE-LABEL: length48_eq_prefer128:
+; X86-NOSSE:       # %bb.0:
+; X86-NOSSE-NEXT:    pushl $0
+; X86-NOSSE-NEXT:    pushl $48
+; X86-NOSSE-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    calll memcmp
+; X86-NOSSE-NEXT:    addl $16, %esp
+; X86-NOSSE-NEXT:    testl %eax, %eax
+; X86-NOSSE-NEXT:    sete %al
+; X86-NOSSE-NEXT:    retl
+;
+; X86-SSE1-LABEL: length48_eq_prefer128:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    pushl $0
+; X86-SSE1-NEXT:    pushl $48
+; X86-SSE1-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    calll memcmp
+; X86-SSE1-NEXT:    addl $16, %esp
+; X86-SSE1-NEXT:    testl %eax, %eax
+; X86-SSE1-NEXT:    sete %al
+; X86-SSE1-NEXT:    retl
+;
+; X86-SSE2-LABEL: length48_eq_prefer128:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE2-NEXT:    movdqu (%ecx), %xmm0
+; X86-SSE2-NEXT:    movdqu 16(%ecx), %xmm1
+; X86-SSE2-NEXT:    movdqu (%eax), %xmm2
+; X86-SSE2-NEXT:    pcmpeqb %xmm0, %xmm2
+; X86-SSE2-NEXT:    movdqu 16(%eax), %xmm0
+; X86-SSE2-NEXT:    pcmpeqb %xmm1, %xmm0
+; X86-SSE2-NEXT:    movdqu 32(%ecx), %xmm1
+; X86-SSE2-NEXT:    movdqu 32(%eax), %xmm3
+; X86-SSE2-NEXT:    pcmpeqb %xmm1, %xmm3
+; X86-SSE2-NEXT:    pand %xmm0, %xmm3
+; X86-SSE2-NEXT:    pand %xmm2, %xmm3
+; X86-SSE2-NEXT:    pmovmskb %xmm3, %eax
+; X86-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; X86-SSE2-NEXT:    sete %al
+; X86-SSE2-NEXT:    retl
+;
+; X86-SSE41-LABEL: length48_eq_prefer128:
+; X86-SSE41:       # %bb.0:
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE41-NEXT:    movdqu (%ecx), %xmm0
+; X86-SSE41-NEXT:    movdqu 16(%ecx), %xmm1
+; X86-SSE41-NEXT:    movdqu (%eax), %xmm2
+; X86-SSE41-NEXT:    pxor %xmm0, %xmm2
+; X86-SSE41-NEXT:    movdqu 16(%eax), %xmm0
+; X86-SSE41-NEXT:    pxor %xmm1, %xmm0
+; X86-SSE41-NEXT:    movdqu 32(%ecx), %xmm1
+; X86-SSE41-NEXT:    movdqu 32(%eax), %xmm3
+; X86-SSE41-NEXT:    pxor %xmm1, %xmm3
+; X86-SSE41-NEXT:    por %xmm0, %xmm3
+; X86-SSE41-NEXT:    por %xmm2, %xmm3
+; X86-SSE41-NEXT:    ptest %xmm3, %xmm3
+; X86-SSE41-NEXT:    sete %al
+; X86-SSE41-NEXT:    retl
+;
+; X64-SSE2-LABEL: length48_eq_prefer128:
+; X64-SSE2:       # %bb.0:
+; X64-SSE2-NEXT:    movdqu (%rdi), %xmm0
+; X64-SSE2-NEXT:    movdqu 16(%rdi), %xmm1
+; X64-SSE2-NEXT:    movdqu 32(%rdi), %xmm2
+; X64-SSE2-NEXT:    movdqu (%rsi), %xmm3
+; X64-SSE2-NEXT:    pcmpeqb %xmm0, %xmm3
+; X64-SSE2-NEXT:    movdqu 16(%rsi), %xmm0
+; X64-SSE2-NEXT:    pcmpeqb %xmm1, %xmm0
+; X64-SSE2-NEXT:    movdqu 32(%rsi), %xmm1
+; X64-SSE2-NEXT:    pcmpeqb %xmm2, %xmm1
+; X64-SSE2-NEXT:    pand %xmm0, %xmm1
+; X64-SSE2-NEXT:    pand %xmm3, %xmm1
+; X64-SSE2-NEXT:    pmovmskb %xmm1, %eax
+; X64-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; X64-SSE2-NEXT:    sete %al
+; X64-SSE2-NEXT:    retq
+;
+; X64-SSE41-LABEL: length48_eq_prefer128:
+; X64-SSE41:       # %bb.0:
+; X64-SSE41-NEXT:    movdqu (%rdi), %xmm0
+; X64-SSE41-NEXT:    movdqu 16(%rdi), %xmm1
+; X64-SSE41-NEXT:    movdqu 32(%rdi), %xmm2
+; X64-SSE41-NEXT:    movdqu (%rsi), %xmm3
+; X64-SSE41-NEXT:    pxor %xmm0, %xmm3
+; X64-SSE41-NEXT:    movdqu 16(%rsi), %xmm0
+; X64-SSE41-NEXT:    pxor %xmm1, %xmm0
+; X64-SSE41-NEXT:    movdqu 32(%rsi), %xmm1
+; X64-SSE41-NEXT:    pxor %xmm2, %xmm1
+; X64-SSE41-NEXT:    por %xmm0, %xmm1
+; X64-SSE41-NEXT:    por %xmm3, %xmm1
+; X64-SSE41-NEXT:    ptest %xmm1, %xmm1
+; X64-SSE41-NEXT:    sete %al
+; X64-SSE41-NEXT:    retq
+;
+; X64-AVX-LABEL: length48_eq_prefer128:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    vmovdqu (%rdi), %xmm0
+; X64-AVX-NEXT:    vmovdqu 16(%rdi), %xmm1
+; X64-AVX-NEXT:    vmovdqu 32(%rdi), %xmm2
+; X64-AVX-NEXT:    vpxor 16(%rsi), %xmm1, %xmm1
+; X64-AVX-NEXT:    vpxor (%rsi), %xmm0, %xmm0
+; X64-AVX-NEXT:    vpxor 32(%rsi), %xmm2, %xmm2
+; X64-AVX-NEXT:    vpor %xmm2, %xmm1, %xmm1
+; X64-AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; X64-AVX-NEXT:    vptest %xmm0, %xmm0
+; X64-AVX-NEXT:    sete %al
+; X64-AVX-NEXT:    retq
+;
+; X64-MIC-AVX-LABEL: length48_eq_prefer128:
+; X64-MIC-AVX:       # %bb.0:
+; X64-MIC-AVX-NEXT:    vmovdqu (%rdi), %xmm0
+; X64-MIC-AVX-NEXT:    vmovdqu 16(%rdi), %xmm1
+; X64-MIC-AVX-NEXT:    vmovdqu 32(%rdi), %xmm2
+; X64-MIC-AVX-NEXT:    vmovdqu (%rsi), %xmm3
+; X64-MIC-AVX-NEXT:    vmovdqu 16(%rsi), %xmm4
+; X64-MIC-AVX-NEXT:    vmovdqu 32(%rsi), %xmm5
+; X64-MIC-AVX-NEXT:    vpcmpneqd %zmm4, %zmm1, %k0
+; X64-MIC-AVX-NEXT:    vpcmpneqd %zmm3, %zmm0, %k1
+; X64-MIC-AVX-NEXT:    korw %k0, %k1, %k0
+; X64-MIC-AVX-NEXT:    vpcmpneqd %zmm5, %zmm2, %k1
+; X64-MIC-AVX-NEXT:    kortestw %k1, %k0
+; X64-MIC-AVX-NEXT:    sete %al
+; X64-MIC-AVX-NEXT:    vzeroupper
+; X64-MIC-AVX-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 48) nounwind
+  %cmp = icmp eq i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length48_eq_const(i8* %X) nounwind {
+; X86-NOSSE-LABEL: length48_eq_const:
+; X86-NOSSE:       # %bb.0:
+; X86-NOSSE-NEXT:    pushl $0
+; X86-NOSSE-NEXT:    pushl $48
+; X86-NOSSE-NEXT:    pushl $.L.str
+; X86-NOSSE-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    calll memcmp
+; X86-NOSSE-NEXT:    addl $16, %esp
+; X86-NOSSE-NEXT:    testl %eax, %eax
+; X86-NOSSE-NEXT:    setne %al
+; X86-NOSSE-NEXT:    retl
+;
+; X86-SSE1-LABEL: length48_eq_const:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    pushl $0
+; X86-SSE1-NEXT:    pushl $48
+; X86-SSE1-NEXT:    pushl $.L.str
+; X86-SSE1-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    calll memcmp
+; X86-SSE1-NEXT:    addl $16, %esp
+; X86-SSE1-NEXT:    testl %eax, %eax
+; X86-SSE1-NEXT:    setne %al
+; X86-SSE1-NEXT:    retl
+;
+; X86-SSE2-LABEL: length48_eq_const:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT:    movdqu (%eax), %xmm0
+; X86-SSE2-NEXT:    movdqu 16(%eax), %xmm1
+; X86-SSE2-NEXT:    movdqu 32(%eax), %xmm2
+; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI.*}}, %xmm1
+; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI.*}}, %xmm2
+; X86-SSE2-NEXT:    pand %xmm1, %xmm2
+; X86-SSE2-NEXT:    pand %xmm0, %xmm2
+; X86-SSE2-NEXT:    pmovmskb %xmm2, %eax
+; X86-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; X86-SSE2-NEXT:    setne %al
+; X86-SSE2-NEXT:    retl
+;
+; X86-SSE41-LABEL: length48_eq_const:
+; X86-SSE41:       # %bb.0:
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT:    movdqu (%eax), %xmm0
+; X86-SSE41-NEXT:    movdqu 16(%eax), %xmm1
+; X86-SSE41-NEXT:    movdqu 32(%eax), %xmm2
+; X86-SSE41-NEXT:    pxor {{\.LCPI.*}}, %xmm1
+; X86-SSE41-NEXT:    pxor {{\.LCPI.*}}, %xmm0
+; X86-SSE41-NEXT:    pxor {{\.LCPI.*}}, %xmm2
+; X86-SSE41-NEXT:    por %xmm1, %xmm2
+; X86-SSE41-NEXT:    por %xmm0, %xmm2
+; X86-SSE41-NEXT:    ptest %xmm2, %xmm2
+; X86-SSE41-NEXT:    setne %al
+; X86-SSE41-NEXT:    retl
+;
+; X64-SSE2-LABEL: length48_eq_const:
+; X64-SSE2:       # %bb.0:
+; X64-SSE2-NEXT:    movdqu (%rdi), %xmm0
+; X64-SSE2-NEXT:    movdqu 16(%rdi), %xmm1
+; X64-SSE2-NEXT:    movdqu 32(%rdi), %xmm2
+; X64-SSE2-NEXT:    pcmpeqb {{.*}}(%rip), %xmm1
+; X64-SSE2-NEXT:    pcmpeqb {{.*}}(%rip), %xmm0
+; X64-SSE2-NEXT:    pcmpeqb {{.*}}(%rip), %xmm2
+; X64-SSE2-NEXT:    pand %xmm1, %xmm2
+; X64-SSE2-NEXT:    pand %xmm0, %xmm2
+; X64-SSE2-NEXT:    pmovmskb %xmm2, %eax
+; X64-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; X64-SSE2-NEXT:    setne %al
+; X64-SSE2-NEXT:    retq
+;
+; X64-SSE41-LABEL: length48_eq_const:
+; X64-SSE41:       # %bb.0:
+; X64-SSE41-NEXT:    movdqu (%rdi), %xmm0
+; X64-SSE41-NEXT:    movdqu 16(%rdi), %xmm1
+; X64-SSE41-NEXT:    movdqu 32(%rdi), %xmm2
+; X64-SSE41-NEXT:    pxor {{.*}}(%rip), %xmm1
+; X64-SSE41-NEXT:    pxor {{.*}}(%rip), %xmm0
+; X64-SSE41-NEXT:    pxor {{.*}}(%rip), %xmm2
+; X64-SSE41-NEXT:    por %xmm1, %xmm2
+; X64-SSE41-NEXT:    por %xmm0, %xmm2
+; X64-SSE41-NEXT:    ptest %xmm2, %xmm2
+; X64-SSE41-NEXT:    setne %al
+; X64-SSE41-NEXT:    retq
+;
+; X64-AVX1-LABEL: length48_eq_const:
+; X64-AVX1:       # %bb.0:
+; X64-AVX1-NEXT:    vmovups (%rdi), %ymm0
+; X64-AVX1-NEXT:    vmovups 32(%rdi), %xmm1
+; X64-AVX1-NEXT:    vxorps {{.*}}(%rip), %ymm0, %ymm0
+; X64-AVX1-NEXT:    vxorps {{.*}}(%rip), %ymm1, %ymm1
+; X64-AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; X64-AVX1-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX1-NEXT:    setne %al
+; X64-AVX1-NEXT:    vzeroupper
+; X64-AVX1-NEXT:    retq
+;
+; X64-AVX2-LABEL: length48_eq_const:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    vmovdqu (%rdi), %ymm0
+; X64-AVX2-NEXT:    vmovdqu 32(%rdi), %xmm1
+; X64-AVX2-NEXT:    vpxor {{.*}}(%rip), %ymm0, %ymm0
+; X64-AVX2-NEXT:    vpxor {{.*}}(%rip), %ymm1, %ymm1
+; X64-AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; X64-AVX2-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX2-NEXT:    setne %al
+; X64-AVX2-NEXT:    vzeroupper
+; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512-LABEL: length48_eq_const:
+; X64-AVX512:       # %bb.0:
+; X64-AVX512-NEXT:    vmovdqu (%rdi), %ymm0
+; X64-AVX512-NEXT:    vmovdqu 32(%rdi), %xmm1
+; X64-AVX512-NEXT:    vpxor {{.*}}(%rip), %ymm0, %ymm0
+; X64-AVX512-NEXT:    vpxor {{.*}}(%rip), %ymm1, %ymm1
+; X64-AVX512-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; X64-AVX512-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX512-NEXT:    setne %al
+; X64-AVX512-NEXT:    vzeroupper
+; X64-AVX512-NEXT:    retq
+;
+; X64-MIC-AVX-LABEL: length48_eq_const:
+; X64-MIC-AVX:       # %bb.0:
+; X64-MIC-AVX-NEXT:    vmovdqu (%rdi), %ymm0
+; X64-MIC-AVX-NEXT:    vmovdqu 32(%rdi), %xmm1
+; X64-MIC-AVX-NEXT:    vmovdqa {{.*#+}} ymm2 = [892613426,959985462,858927408,926299444,0,0,0,0]
+; X64-MIC-AVX-NEXT:    vpcmpneqd %zmm2, %zmm1, %k0
+; X64-MIC-AVX-NEXT:    vmovdqa {{.*#+}} ymm1 = [858927408,926299444,825243960,892613426,959985462,858927408,926299444,825243960]
+; X64-MIC-AVX-NEXT:    vpcmpneqd %zmm1, %zmm0, %k1
+; X64-MIC-AVX-NEXT:    kortestw %k0, %k1
+; X64-MIC-AVX-NEXT:    setne %al
+; X64-MIC-AVX-NEXT:    vzeroupper
+; X64-MIC-AVX-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* getelementptr inbounds ([513 x i8], [513 x i8]* @.str, i32 0, i32 0), i64 48) nounwind
+  %c = icmp ne i32 %m, 0
+  ret i1 %c
+}
+
+define i32 @length63(i8* %X, i8* %Y) nounwind {
+; X86-LABEL: length63:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $63
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    retl
+;
+; X64-LABEL: length63:
+; X64:       # %bb.0:
+; X64-NEXT:    movl $63, %edx
+; X64-NEXT:    jmp memcmp # TAILCALL
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 63) nounwind
+  ret i32 %m
+}
+
+define i1 @length63_eq(i8* %x, i8* %y) nounwind {
+; X86-NOSSE-LABEL: length63_eq:
+; X86-NOSSE:       # %bb.0:
+; X86-NOSSE-NEXT:    pushl $0
+; X86-NOSSE-NEXT:    pushl $63
+; X86-NOSSE-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    calll memcmp
+; X86-NOSSE-NEXT:    addl $16, %esp
+; X86-NOSSE-NEXT:    testl %eax, %eax
+; X86-NOSSE-NEXT:    setne %al
+; X86-NOSSE-NEXT:    retl
+;
+; X86-SSE1-LABEL: length63_eq:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    pushl $0
+; X86-SSE1-NEXT:    pushl $63
+; X86-SSE1-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    calll memcmp
+; X86-SSE1-NEXT:    addl $16, %esp
+; X86-SSE1-NEXT:    testl %eax, %eax
+; X86-SSE1-NEXT:    setne %al
+; X86-SSE1-NEXT:    retl
+;
+; X86-SSE2-LABEL: length63_eq:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE2-NEXT:    movdqu (%ecx), %xmm0
+; X86-SSE2-NEXT:    movdqu 16(%ecx), %xmm1
+; X86-SSE2-NEXT:    movdqu (%eax), %xmm2
+; X86-SSE2-NEXT:    pcmpeqb %xmm0, %xmm2
+; X86-SSE2-NEXT:    movdqu 16(%eax), %xmm0
+; X86-SSE2-NEXT:    pcmpeqb %xmm1, %xmm0
+; X86-SSE2-NEXT:    movdqu 32(%ecx), %xmm1
+; X86-SSE2-NEXT:    movdqu 32(%eax), %xmm3
+; X86-SSE2-NEXT:    pcmpeqb %xmm1, %xmm3
+; X86-SSE2-NEXT:    movdqu 47(%ecx), %xmm1
+; X86-SSE2-NEXT:    movdqu 47(%eax), %xmm4
+; X86-SSE2-NEXT:    pcmpeqb %xmm1, %xmm4
+; X86-SSE2-NEXT:    pand %xmm3, %xmm4
+; X86-SSE2-NEXT:    pand %xmm0, %xmm4
+; X86-SSE2-NEXT:    pand %xmm2, %xmm4
+; X86-SSE2-NEXT:    pmovmskb %xmm4, %eax
+; X86-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; X86-SSE2-NEXT:    setne %al
+; X86-SSE2-NEXT:    retl
+;
+; X86-SSE41-LABEL: length63_eq:
+; X86-SSE41:       # %bb.0:
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE41-NEXT:    movdqu (%ecx), %xmm0
+; X86-SSE41-NEXT:    movdqu 16(%ecx), %xmm1
+; X86-SSE41-NEXT:    movdqu (%eax), %xmm2
+; X86-SSE41-NEXT:    pxor %xmm0, %xmm2
+; X86-SSE41-NEXT:    movdqu 16(%eax), %xmm0
+; X86-SSE41-NEXT:    pxor %xmm1, %xmm0
+; X86-SSE41-NEXT:    movdqu 32(%ecx), %xmm1
+; X86-SSE41-NEXT:    movdqu 32(%eax), %xmm3
+; X86-SSE41-NEXT:    pxor %xmm1, %xmm3
+; X86-SSE41-NEXT:    movdqu 47(%ecx), %xmm1
+; X86-SSE41-NEXT:    movdqu 47(%eax), %xmm4
+; X86-SSE41-NEXT:    pxor %xmm1, %xmm4
+; X86-SSE41-NEXT:    por %xmm3, %xmm4
+; X86-SSE41-NEXT:    por %xmm0, %xmm4
+; X86-SSE41-NEXT:    por %xmm2, %xmm4
+; X86-SSE41-NEXT:    ptest %xmm4, %xmm4
+; X86-SSE41-NEXT:    setne %al
+; X86-SSE41-NEXT:    retl
+;
+; X64-SSE2-LABEL: length63_eq:
+; X64-SSE2:       # %bb.0:
+; X64-SSE2-NEXT:    movdqu (%rdi), %xmm0
+; X64-SSE2-NEXT:    movdqu 16(%rdi), %xmm1
+; X64-SSE2-NEXT:    movdqu 32(%rdi), %xmm2
+; X64-SSE2-NEXT:    movdqu 47(%rdi), %xmm3
+; X64-SSE2-NEXT:    movdqu (%rsi), %xmm4
+; X64-SSE2-NEXT:    pcmpeqb %xmm0, %xmm4
+; X64-SSE2-NEXT:    movdqu 16(%rsi), %xmm0
+; X64-SSE2-NEXT:    pcmpeqb %xmm1, %xmm0
+; X64-SSE2-NEXT:    movdqu 32(%rsi), %xmm1
+; X64-SSE2-NEXT:    pcmpeqb %xmm2, %xmm1
+; X64-SSE2-NEXT:    movdqu 47(%rsi), %xmm2
+; X64-SSE2-NEXT:    pcmpeqb %xmm3, %xmm2
+; X64-SSE2-NEXT:    pand %xmm1, %xmm2
+; X64-SSE2-NEXT:    pand %xmm0, %xmm2
+; X64-SSE2-NEXT:    pand %xmm4, %xmm2
+; X64-SSE2-NEXT:    pmovmskb %xmm2, %eax
+; X64-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; X64-SSE2-NEXT:    setne %al
+; X64-SSE2-NEXT:    retq
+;
+; X64-SSE41-LABEL: length63_eq:
+; X64-SSE41:       # %bb.0:
+; X64-SSE41-NEXT:    movdqu (%rdi), %xmm0
+; X64-SSE41-NEXT:    movdqu 16(%rdi), %xmm1
+; X64-SSE41-NEXT:    movdqu 32(%rdi), %xmm2
+; X64-SSE41-NEXT:    movdqu 47(%rdi), %xmm3
+; X64-SSE41-NEXT:    movdqu (%rsi), %xmm4
+; X64-SSE41-NEXT:    pxor %xmm0, %xmm4
+; X64-SSE41-NEXT:    movdqu 16(%rsi), %xmm0
+; X64-SSE41-NEXT:    pxor %xmm1, %xmm0
+; X64-SSE41-NEXT:    movdqu 32(%rsi), %xmm1
+; X64-SSE41-NEXT:    pxor %xmm2, %xmm1
+; X64-SSE41-NEXT:    movdqu 47(%rsi), %xmm2
+; X64-SSE41-NEXT:    pxor %xmm3, %xmm2
+; X64-SSE41-NEXT:    por %xmm1, %xmm2
+; X64-SSE41-NEXT:    por %xmm0, %xmm2
+; X64-SSE41-NEXT:    por %xmm4, %xmm2
+; X64-SSE41-NEXT:    ptest %xmm2, %xmm2
+; X64-SSE41-NEXT:    setne %al
+; X64-SSE41-NEXT:    retq
+;
+; X64-AVX1-LABEL: length63_eq:
+; X64-AVX1:       # %bb.0:
+; X64-AVX1-NEXT:    vmovups (%rdi), %ymm0
+; X64-AVX1-NEXT:    vmovups 31(%rdi), %ymm1
+; X64-AVX1-NEXT:    vxorps 31(%rsi), %ymm1, %ymm1
+; X64-AVX1-NEXT:    vxorps (%rsi), %ymm0, %ymm0
+; X64-AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; X64-AVX1-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX1-NEXT:    setne %al
+; X64-AVX1-NEXT:    vzeroupper
+; X64-AVX1-NEXT:    retq
+;
+; X64-AVX2-LABEL: length63_eq:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    vmovdqu (%rdi), %ymm0
+; X64-AVX2-NEXT:    vmovdqu 31(%rdi), %ymm1
+; X64-AVX2-NEXT:    vpxor 31(%rsi), %ymm1, %ymm1
+; X64-AVX2-NEXT:    vpxor (%rsi), %ymm0, %ymm0
+; X64-AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; X64-AVX2-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX2-NEXT:    setne %al
+; X64-AVX2-NEXT:    vzeroupper
+; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512-LABEL: length63_eq:
+; X64-AVX512:       # %bb.0:
+; X64-AVX512-NEXT:    vmovdqu (%rdi), %ymm0
+; X64-AVX512-NEXT:    vmovdqu 31(%rdi), %ymm1
+; X64-AVX512-NEXT:    vpxor 31(%rsi), %ymm1, %ymm1
+; X64-AVX512-NEXT:    vpxor (%rsi), %ymm0, %ymm0
+; X64-AVX512-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; X64-AVX512-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX512-NEXT:    setne %al
+; X64-AVX512-NEXT:    vzeroupper
+; X64-AVX512-NEXT:    retq
+;
+; X64-MIC-AVX-LABEL: length63_eq:
+; X64-MIC-AVX:       # %bb.0:
+; X64-MIC-AVX-NEXT:    vmovdqu (%rdi), %ymm0
+; X64-MIC-AVX-NEXT:    vmovdqu 31(%rdi), %ymm1
+; X64-MIC-AVX-NEXT:    vmovdqu (%rsi), %ymm2
+; X64-MIC-AVX-NEXT:    vmovdqu 31(%rsi), %ymm3
+; X64-MIC-AVX-NEXT:    vpcmpneqd %zmm3, %zmm1, %k0
+; X64-MIC-AVX-NEXT:    vpcmpneqd %zmm2, %zmm0, %k1
+; X64-MIC-AVX-NEXT:    kortestw %k0, %k1
+; X64-MIC-AVX-NEXT:    setne %al
+; X64-MIC-AVX-NEXT:    vzeroupper
+; X64-MIC-AVX-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 63) nounwind
+  %cmp = icmp ne i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length63_lt(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length63_lt:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $63
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    shrl $31, %eax
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: length63_lt:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $63, %edx
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    shrl $31, %eax
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 63) nounwind
+  %cmp = icmp slt i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length63_gt(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length63_gt:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $63
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    setg %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length63_gt:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $63, %edx
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    setg %al
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 63) nounwind
+  %cmp = icmp sgt i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length63_eq_const(i8* %X) nounwind {
+; X86-NOSSE-LABEL: length63_eq_const:
+; X86-NOSSE:       # %bb.0:
+; X86-NOSSE-NEXT:    pushl $0
+; X86-NOSSE-NEXT:    pushl $63
+; X86-NOSSE-NEXT:    pushl $.L.str
+; X86-NOSSE-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    calll memcmp
+; X86-NOSSE-NEXT:    addl $16, %esp
+; X86-NOSSE-NEXT:    testl %eax, %eax
+; X86-NOSSE-NEXT:    sete %al
+; X86-NOSSE-NEXT:    retl
+;
+; X86-SSE1-LABEL: length63_eq_const:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    pushl $0
+; X86-SSE1-NEXT:    pushl $63
+; X86-SSE1-NEXT:    pushl $.L.str
+; X86-SSE1-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    calll memcmp
+; X86-SSE1-NEXT:    addl $16, %esp
+; X86-SSE1-NEXT:    testl %eax, %eax
+; X86-SSE1-NEXT:    sete %al
+; X86-SSE1-NEXT:    retl
+;
+; X86-SSE2-LABEL: length63_eq_const:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT:    movdqu (%eax), %xmm0
+; X86-SSE2-NEXT:    movdqu 16(%eax), %xmm1
+; X86-SSE2-NEXT:    movdqu 32(%eax), %xmm2
+; X86-SSE2-NEXT:    movdqu 47(%eax), %xmm3
+; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI.*}}, %xmm3
+; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI.*}}, %xmm2
+; X86-SSE2-NEXT:    pand %xmm3, %xmm2
+; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI.*}}, %xmm1
+; X86-SSE2-NEXT:    pand %xmm2, %xmm1
+; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand %xmm1, %xmm0
+; X86-SSE2-NEXT:    pmovmskb %xmm0, %eax
+; X86-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; X86-SSE2-NEXT:    sete %al
+; X86-SSE2-NEXT:    retl
+;
+; X86-SSE41-LABEL: length63_eq_const:
+; X86-SSE41:       # %bb.0:
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT:    movdqu (%eax), %xmm0
+; X86-SSE41-NEXT:    movdqu 16(%eax), %xmm1
+; X86-SSE41-NEXT:    movdqu 32(%eax), %xmm2
+; X86-SSE41-NEXT:    movdqu 47(%eax), %xmm3
+; X86-SSE41-NEXT:    pxor {{\.LCPI.*}}, %xmm3
+; X86-SSE41-NEXT:    pxor {{\.LCPI.*}}, %xmm2
+; X86-SSE41-NEXT:    por %xmm3, %xmm2
+; X86-SSE41-NEXT:    pxor {{\.LCPI.*}}, %xmm1
+; X86-SSE41-NEXT:    por %xmm2, %xmm1
+; X86-SSE41-NEXT:    pxor {{\.LCPI.*}}, %xmm0
+; X86-SSE41-NEXT:    por %xmm1, %xmm0
+; X86-SSE41-NEXT:    ptest %xmm0, %xmm0
+; X86-SSE41-NEXT:    sete %al
+; X86-SSE41-NEXT:    retl
+;
+; X64-SSE2-LABEL: length63_eq_const:
+; X64-SSE2:       # %bb.0:
+; X64-SSE2-NEXT:    movdqu (%rdi), %xmm0
+; X64-SSE2-NEXT:    movdqu 16(%rdi), %xmm1
+; X64-SSE2-NEXT:    movdqu 32(%rdi), %xmm2
+; X64-SSE2-NEXT:    movdqu 47(%rdi), %xmm3
+; X64-SSE2-NEXT:    pcmpeqb {{.*}}(%rip), %xmm3
+; X64-SSE2-NEXT:    pcmpeqb {{.*}}(%rip), %xmm2
+; X64-SSE2-NEXT:    pand %xmm3, %xmm2
+; X64-SSE2-NEXT:    pcmpeqb {{.*}}(%rip), %xmm1
+; X64-SSE2-NEXT:    pand %xmm2, %xmm1
+; X64-SSE2-NEXT:    pcmpeqb {{.*}}(%rip), %xmm0
+; X64-SSE2-NEXT:    pand %xmm1, %xmm0
+; X64-SSE2-NEXT:    pmovmskb %xmm0, %eax
+; X64-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; X64-SSE2-NEXT:    sete %al
+; X64-SSE2-NEXT:    retq
+;
+; X64-SSE41-LABEL: length63_eq_const:
+; X64-SSE41:       # %bb.0:
+; X64-SSE41-NEXT:    movdqu (%rdi), %xmm0
+; X64-SSE41-NEXT:    movdqu 16(%rdi), %xmm1
+; X64-SSE41-NEXT:    movdqu 32(%rdi), %xmm2
+; X64-SSE41-NEXT:    movdqu 47(%rdi), %xmm3
+; X64-SSE41-NEXT:    pxor {{.*}}(%rip), %xmm3
+; X64-SSE41-NEXT:    pxor {{.*}}(%rip), %xmm2
+; X64-SSE41-NEXT:    por %xmm3, %xmm2
+; X64-SSE41-NEXT:    pxor {{.*}}(%rip), %xmm1
+; X64-SSE41-NEXT:    por %xmm2, %xmm1
+; X64-SSE41-NEXT:    pxor {{.*}}(%rip), %xmm0
+; X64-SSE41-NEXT:    por %xmm1, %xmm0
+; X64-SSE41-NEXT:    ptest %xmm0, %xmm0
+; X64-SSE41-NEXT:    sete %al
+; X64-SSE41-NEXT:    retq
+;
+; X64-AVX1-LABEL: length63_eq_const:
+; X64-AVX1:       # %bb.0:
+; X64-AVX1-NEXT:    vmovups (%rdi), %ymm0
+; X64-AVX1-NEXT:    vmovups 31(%rdi), %ymm1
+; X64-AVX1-NEXT:    vxorps {{.*}}(%rip), %ymm1, %ymm1
+; X64-AVX1-NEXT:    vxorps {{.*}}(%rip), %ymm0, %ymm0
+; X64-AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; X64-AVX1-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX1-NEXT:    sete %al
+; X64-AVX1-NEXT:    vzeroupper
+; X64-AVX1-NEXT:    retq
+;
+; X64-AVX2-LABEL: length63_eq_const:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    vmovdqu (%rdi), %ymm0
+; X64-AVX2-NEXT:    vmovdqu 31(%rdi), %ymm1
+; X64-AVX2-NEXT:    vpxor {{.*}}(%rip), %ymm1, %ymm1
+; X64-AVX2-NEXT:    vpxor {{.*}}(%rip), %ymm0, %ymm0
+; X64-AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; X64-AVX2-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX2-NEXT:    sete %al
+; X64-AVX2-NEXT:    vzeroupper
+; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512-LABEL: length63_eq_const:
+; X64-AVX512:       # %bb.0:
+; X64-AVX512-NEXT:    vmovdqu (%rdi), %ymm0
+; X64-AVX512-NEXT:    vmovdqu 31(%rdi), %ymm1
+; X64-AVX512-NEXT:    vpxor {{.*}}(%rip), %ymm1, %ymm1
+; X64-AVX512-NEXT:    vpxor {{.*}}(%rip), %ymm0, %ymm0
+; X64-AVX512-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; X64-AVX512-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX512-NEXT:    sete %al
+; X64-AVX512-NEXT:    vzeroupper
+; X64-AVX512-NEXT:    retq
+;
+; X64-MIC-AVX-LABEL: length63_eq_const:
+; X64-MIC-AVX:       # %bb.0:
+; X64-MIC-AVX-NEXT:    vmovdqu (%rdi), %ymm0
+; X64-MIC-AVX-NEXT:    vmovdqu 31(%rdi), %ymm1
+; X64-MIC-AVX-NEXT:    vmovdqa {{.*#+}} ymm2 = [875770417,943142453,842084409,909456435,809056311,875770417,943142453,842084409]
+; X64-MIC-AVX-NEXT:    vpcmpneqd %zmm2, %zmm1, %k0
+; X64-MIC-AVX-NEXT:    vmovdqa {{.*#+}} ymm1 = [858927408,926299444,825243960,892613426,959985462,858927408,926299444,825243960]
+; X64-MIC-AVX-NEXT:    vpcmpneqd %zmm1, %zmm0, %k1
+; X64-MIC-AVX-NEXT:    kortestw %k0, %k1
+; X64-MIC-AVX-NEXT:    sete %al
+; X64-MIC-AVX-NEXT:    vzeroupper
+; X64-MIC-AVX-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* getelementptr inbounds ([513 x i8], [513 x i8]* @.str, i32 0, i32 0), i64 63) nounwind
+  %c = icmp eq i32 %m, 0
+  ret i1 %c
+}
+
+define i32 @length64(i8* %X, i8* %Y) nounwind {
+; X86-LABEL: length64:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $64
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    retl
+;
+; X64-LABEL: length64:
+; X64:       # %bb.0:
+; X64-NEXT:    movl $64, %edx
+; X64-NEXT:    jmp memcmp # TAILCALL
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 64) nounwind
+  ret i32 %m
+}
+
+define i1 @length64_eq(i8* %x, i8* %y) nounwind {
+; X86-NOSSE-LABEL: length64_eq:
+; X86-NOSSE:       # %bb.0:
+; X86-NOSSE-NEXT:    pushl $0
+; X86-NOSSE-NEXT:    pushl $64
+; X86-NOSSE-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    calll memcmp
+; X86-NOSSE-NEXT:    addl $16, %esp
+; X86-NOSSE-NEXT:    testl %eax, %eax
+; X86-NOSSE-NEXT:    setne %al
+; X86-NOSSE-NEXT:    retl
+;
+; X86-SSE1-LABEL: length64_eq:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    pushl $0
+; X86-SSE1-NEXT:    pushl $64
+; X86-SSE1-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    calll memcmp
+; X86-SSE1-NEXT:    addl $16, %esp
+; X86-SSE1-NEXT:    testl %eax, %eax
+; X86-SSE1-NEXT:    setne %al
+; X86-SSE1-NEXT:    retl
+;
+; X86-SSE2-LABEL: length64_eq:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE2-NEXT:    movdqu (%ecx), %xmm0
+; X86-SSE2-NEXT:    movdqu 16(%ecx), %xmm1
+; X86-SSE2-NEXT:    movdqu (%eax), %xmm2
+; X86-SSE2-NEXT:    pcmpeqb %xmm0, %xmm2
+; X86-SSE2-NEXT:    movdqu 16(%eax), %xmm0
+; X86-SSE2-NEXT:    pcmpeqb %xmm1, %xmm0
+; X86-SSE2-NEXT:    movdqu 32(%ecx), %xmm1
+; X86-SSE2-NEXT:    movdqu 32(%eax), %xmm3
+; X86-SSE2-NEXT:    pcmpeqb %xmm1, %xmm3
+; X86-SSE2-NEXT:    movdqu 48(%ecx), %xmm1
+; X86-SSE2-NEXT:    movdqu 48(%eax), %xmm4
+; X86-SSE2-NEXT:    pcmpeqb %xmm1, %xmm4
+; X86-SSE2-NEXT:    pand %xmm3, %xmm4
+; X86-SSE2-NEXT:    pand %xmm0, %xmm4
+; X86-SSE2-NEXT:    pand %xmm2, %xmm4
+; X86-SSE2-NEXT:    pmovmskb %xmm4, %eax
+; X86-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; X86-SSE2-NEXT:    setne %al
+; X86-SSE2-NEXT:    retl
+;
+; X86-SSE41-LABEL: length64_eq:
+; X86-SSE41:       # %bb.0:
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE41-NEXT:    movdqu (%ecx), %xmm0
+; X86-SSE41-NEXT:    movdqu 16(%ecx), %xmm1
+; X86-SSE41-NEXT:    movdqu (%eax), %xmm2
+; X86-SSE41-NEXT:    pxor %xmm0, %xmm2
+; X86-SSE41-NEXT:    movdqu 16(%eax), %xmm0
+; X86-SSE41-NEXT:    pxor %xmm1, %xmm0
+; X86-SSE41-NEXT:    movdqu 32(%ecx), %xmm1
+; X86-SSE41-NEXT:    movdqu 32(%eax), %xmm3
+; X86-SSE41-NEXT:    pxor %xmm1, %xmm3
+; X86-SSE41-NEXT:    movdqu 48(%ecx), %xmm1
+; X86-SSE41-NEXT:    movdqu 48(%eax), %xmm4
+; X86-SSE41-NEXT:    pxor %xmm1, %xmm4
+; X86-SSE41-NEXT:    por %xmm3, %xmm4
+; X86-SSE41-NEXT:    por %xmm0, %xmm4
+; X86-SSE41-NEXT:    por %xmm2, %xmm4
+; X86-SSE41-NEXT:    ptest %xmm4, %xmm4
+; X86-SSE41-NEXT:    setne %al
+; X86-SSE41-NEXT:    retl
+;
+; X64-SSE2-LABEL: length64_eq:
+; X64-SSE2:       # %bb.0:
+; X64-SSE2-NEXT:    movdqu (%rdi), %xmm0
+; X64-SSE2-NEXT:    movdqu 16(%rdi), %xmm1
+; X64-SSE2-NEXT:    movdqu 32(%rdi), %xmm2
+; X64-SSE2-NEXT:    movdqu 48(%rdi), %xmm3
+; X64-SSE2-NEXT:    movdqu (%rsi), %xmm4
+; X64-SSE2-NEXT:    pcmpeqb %xmm0, %xmm4
+; X64-SSE2-NEXT:    movdqu 16(%rsi), %xmm0
+; X64-SSE2-NEXT:    pcmpeqb %xmm1, %xmm0
+; X64-SSE2-NEXT:    movdqu 32(%rsi), %xmm1
+; X64-SSE2-NEXT:    pcmpeqb %xmm2, %xmm1
+; X64-SSE2-NEXT:    movdqu 48(%rsi), %xmm2
+; X64-SSE2-NEXT:    pcmpeqb %xmm3, %xmm2
+; X64-SSE2-NEXT:    pand %xmm1, %xmm2
+; X64-SSE2-NEXT:    pand %xmm0, %xmm2
+; X64-SSE2-NEXT:    pand %xmm4, %xmm2
+; X64-SSE2-NEXT:    pmovmskb %xmm2, %eax
+; X64-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; X64-SSE2-NEXT:    setne %al
+; X64-SSE2-NEXT:    retq
+;
+; X64-SSE41-LABEL: length64_eq:
+; X64-SSE41:       # %bb.0:
+; X64-SSE41-NEXT:    movdqu (%rdi), %xmm0
+; X64-SSE41-NEXT:    movdqu 16(%rdi), %xmm1
+; X64-SSE41-NEXT:    movdqu 32(%rdi), %xmm2
+; X64-SSE41-NEXT:    movdqu 48(%rdi), %xmm3
+; X64-SSE41-NEXT:    movdqu (%rsi), %xmm4
+; X64-SSE41-NEXT:    pxor %xmm0, %xmm4
+; X64-SSE41-NEXT:    movdqu 16(%rsi), %xmm0
+; X64-SSE41-NEXT:    pxor %xmm1, %xmm0
+; X64-SSE41-NEXT:    movdqu 32(%rsi), %xmm1
+; X64-SSE41-NEXT:    pxor %xmm2, %xmm1
+; X64-SSE41-NEXT:    movdqu 48(%rsi), %xmm2
+; X64-SSE41-NEXT:    pxor %xmm3, %xmm2
+; X64-SSE41-NEXT:    por %xmm1, %xmm2
+; X64-SSE41-NEXT:    por %xmm0, %xmm2
+; X64-SSE41-NEXT:    por %xmm4, %xmm2
+; X64-SSE41-NEXT:    ptest %xmm2, %xmm2
+; X64-SSE41-NEXT:    setne %al
+; X64-SSE41-NEXT:    retq
+;
+; X64-AVX1-LABEL: length64_eq:
+; X64-AVX1:       # %bb.0:
+; X64-AVX1-NEXT:    vmovups (%rdi), %ymm0
+; X64-AVX1-NEXT:    vmovups 32(%rdi), %ymm1
+; X64-AVX1-NEXT:    vxorps 32(%rsi), %ymm1, %ymm1
+; X64-AVX1-NEXT:    vxorps (%rsi), %ymm0, %ymm0
+; X64-AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; X64-AVX1-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX1-NEXT:    setne %al
+; X64-AVX1-NEXT:    vzeroupper
+; X64-AVX1-NEXT:    retq
+;
+; X64-AVX2-LABEL: length64_eq:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    vmovdqu (%rdi), %ymm0
+; X64-AVX2-NEXT:    vmovdqu 32(%rdi), %ymm1
+; X64-AVX2-NEXT:    vpxor 32(%rsi), %ymm1, %ymm1
+; X64-AVX2-NEXT:    vpxor (%rsi), %ymm0, %ymm0
+; X64-AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; X64-AVX2-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX2-NEXT:    setne %al
+; X64-AVX2-NEXT:    vzeroupper
+; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512BW-LABEL: length64_eq:
+; X64-AVX512BW:       # %bb.0:
+; X64-AVX512BW-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-AVX512BW-NEXT:    vpcmpneqb (%rsi), %zmm0, %k0
+; X64-AVX512BW-NEXT:    kortestq %k0, %k0
+; X64-AVX512BW-NEXT:    setne %al
+; X64-AVX512BW-NEXT:    vzeroupper
+; X64-AVX512BW-NEXT:    retq
+;
+; X64-AVX512F-LABEL: length64_eq:
+; X64-AVX512F:       # %bb.0:
+; X64-AVX512F-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-AVX512F-NEXT:    vpcmpneqd (%rsi), %zmm0, %k0
+; X64-AVX512F-NEXT:    kortestw %k0, %k0
+; X64-AVX512F-NEXT:    setne %al
+; X64-AVX512F-NEXT:    vzeroupper
+; X64-AVX512F-NEXT:    retq
+;
+; X64-MIC-AVX2-LABEL: length64_eq:
+; X64-MIC-AVX2:       # %bb.0:
+; X64-MIC-AVX2-NEXT:    vmovdqu (%rdi), %ymm0
+; X64-MIC-AVX2-NEXT:    vmovdqu 32(%rdi), %ymm1
+; X64-MIC-AVX2-NEXT:    vmovdqu (%rsi), %ymm2
+; X64-MIC-AVX2-NEXT:    vmovdqu 32(%rsi), %ymm3
+; X64-MIC-AVX2-NEXT:    vpcmpneqd %zmm3, %zmm1, %k0
+; X64-MIC-AVX2-NEXT:    vpcmpneqd %zmm2, %zmm0, %k1
+; X64-MIC-AVX2-NEXT:    kortestw %k0, %k1
+; X64-MIC-AVX2-NEXT:    setne %al
+; X64-MIC-AVX2-NEXT:    vzeroupper
+; X64-MIC-AVX2-NEXT:    retq
+;
+; X64-MIC-AVX512F-LABEL: length64_eq:
+; X64-MIC-AVX512F:       # %bb.0:
+; X64-MIC-AVX512F-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-MIC-AVX512F-NEXT:    vpcmpneqd (%rsi), %zmm0, %k0
+; X64-MIC-AVX512F-NEXT:    kortestw %k0, %k0
+; X64-MIC-AVX512F-NEXT:    setne %al
+; X64-MIC-AVX512F-NEXT:    vzeroupper
+; X64-MIC-AVX512F-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 64) nounwind
+  %cmp = icmp ne i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length64_lt(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length64_lt:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $64
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    shrl $31, %eax
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: length64_lt:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $64, %edx
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    shrl $31, %eax
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 64) nounwind
+  %cmp = icmp slt i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length64_gt(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length64_gt:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $64
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    setg %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length64_gt:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $64, %edx
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    setg %al
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 64) nounwind
+  %cmp = icmp sgt i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length64_eq_const(i8* %X) nounwind {
+; X86-NOSSE-LABEL: length64_eq_const:
+; X86-NOSSE:       # %bb.0:
+; X86-NOSSE-NEXT:    pushl $0
+; X86-NOSSE-NEXT:    pushl $64
+; X86-NOSSE-NEXT:    pushl $.L.str
+; X86-NOSSE-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    calll memcmp
+; X86-NOSSE-NEXT:    addl $16, %esp
+; X86-NOSSE-NEXT:    testl %eax, %eax
+; X86-NOSSE-NEXT:    sete %al
+; X86-NOSSE-NEXT:    retl
+;
+; X86-SSE1-LABEL: length64_eq_const:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    pushl $0
+; X86-SSE1-NEXT:    pushl $64
+; X86-SSE1-NEXT:    pushl $.L.str
+; X86-SSE1-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    calll memcmp
+; X86-SSE1-NEXT:    addl $16, %esp
+; X86-SSE1-NEXT:    testl %eax, %eax
+; X86-SSE1-NEXT:    sete %al
+; X86-SSE1-NEXT:    retl
+;
+; X86-SSE2-LABEL: length64_eq_const:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT:    movdqu (%eax), %xmm0
+; X86-SSE2-NEXT:    movdqu 16(%eax), %xmm1
+; X86-SSE2-NEXT:    movdqu 32(%eax), %xmm2
+; X86-SSE2-NEXT:    movdqu 48(%eax), %xmm3
+; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI.*}}, %xmm3
+; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI.*}}, %xmm2
+; X86-SSE2-NEXT:    pand %xmm3, %xmm2
+; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI.*}}, %xmm1
+; X86-SSE2-NEXT:    pand %xmm2, %xmm1
+; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand %xmm1, %xmm0
+; X86-SSE2-NEXT:    pmovmskb %xmm0, %eax
+; X86-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; X86-SSE2-NEXT:    sete %al
+; X86-SSE2-NEXT:    retl
+;
+; X86-SSE41-LABEL: length64_eq_const:
+; X86-SSE41:       # %bb.0:
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT:    movdqu (%eax), %xmm0
+; X86-SSE41-NEXT:    movdqu 16(%eax), %xmm1
+; X86-SSE41-NEXT:    movdqu 32(%eax), %xmm2
+; X86-SSE41-NEXT:    movdqu 48(%eax), %xmm3
+; X86-SSE41-NEXT:    pxor {{\.LCPI.*}}, %xmm3
+; X86-SSE41-NEXT:    pxor {{\.LCPI.*}}, %xmm2
+; X86-SSE41-NEXT:    por %xmm3, %xmm2
+; X86-SSE41-NEXT:    pxor {{\.LCPI.*}}, %xmm1
+; X86-SSE41-NEXT:    por %xmm2, %xmm1
+; X86-SSE41-NEXT:    pxor {{\.LCPI.*}}, %xmm0
+; X86-SSE41-NEXT:    por %xmm1, %xmm0
+; X86-SSE41-NEXT:    ptest %xmm0, %xmm0
+; X86-SSE41-NEXT:    sete %al
+; X86-SSE41-NEXT:    retl
+;
+; X64-SSE2-LABEL: length64_eq_const:
+; X64-SSE2:       # %bb.0:
+; X64-SSE2-NEXT:    movdqu (%rdi), %xmm0
+; X64-SSE2-NEXT:    movdqu 16(%rdi), %xmm1
+; X64-SSE2-NEXT:    movdqu 32(%rdi), %xmm2
+; X64-SSE2-NEXT:    movdqu 48(%rdi), %xmm3
+; X64-SSE2-NEXT:    pcmpeqb {{.*}}(%rip), %xmm3
+; X64-SSE2-NEXT:    pcmpeqb {{.*}}(%rip), %xmm2
+; X64-SSE2-NEXT:    pand %xmm3, %xmm2
+; X64-SSE2-NEXT:    pcmpeqb {{.*}}(%rip), %xmm1
+; X64-SSE2-NEXT:    pand %xmm2, %xmm1
+; X64-SSE2-NEXT:    pcmpeqb {{.*}}(%rip), %xmm0
+; X64-SSE2-NEXT:    pand %xmm1, %xmm0
+; X64-SSE2-NEXT:    pmovmskb %xmm0, %eax
+; X64-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; X64-SSE2-NEXT:    sete %al
+; X64-SSE2-NEXT:    retq
+;
+; X64-SSE41-LABEL: length64_eq_const:
+; X64-SSE41:       # %bb.0:
+; X64-SSE41-NEXT:    movdqu (%rdi), %xmm0
+; X64-SSE41-NEXT:    movdqu 16(%rdi), %xmm1
+; X64-SSE41-NEXT:    movdqu 32(%rdi), %xmm2
+; X64-SSE41-NEXT:    movdqu 48(%rdi), %xmm3
+; X64-SSE41-NEXT:    pxor {{.*}}(%rip), %xmm3
+; X64-SSE41-NEXT:    pxor {{.*}}(%rip), %xmm2
+; X64-SSE41-NEXT:    por %xmm3, %xmm2
+; X64-SSE41-NEXT:    pxor {{.*}}(%rip), %xmm1
+; X64-SSE41-NEXT:    por %xmm2, %xmm1
+; X64-SSE41-NEXT:    pxor {{.*}}(%rip), %xmm0
+; X64-SSE41-NEXT:    por %xmm1, %xmm0
+; X64-SSE41-NEXT:    ptest %xmm0, %xmm0
+; X64-SSE41-NEXT:    sete %al
+; X64-SSE41-NEXT:    retq
+;
+; X64-AVX1-LABEL: length64_eq_const:
+; X64-AVX1:       # %bb.0:
+; X64-AVX1-NEXT:    vmovups (%rdi), %ymm0
+; X64-AVX1-NEXT:    vmovups 32(%rdi), %ymm1
+; X64-AVX1-NEXT:    vxorps {{.*}}(%rip), %ymm1, %ymm1
+; X64-AVX1-NEXT:    vxorps {{.*}}(%rip), %ymm0, %ymm0
+; X64-AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; X64-AVX1-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX1-NEXT:    sete %al
+; X64-AVX1-NEXT:    vzeroupper
+; X64-AVX1-NEXT:    retq
+;
+; X64-AVX2-LABEL: length64_eq_const:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    vmovdqu (%rdi), %ymm0
+; X64-AVX2-NEXT:    vmovdqu 32(%rdi), %ymm1
+; X64-AVX2-NEXT:    vpxor {{.*}}(%rip), %ymm1, %ymm1
+; X64-AVX2-NEXT:    vpxor {{.*}}(%rip), %ymm0, %ymm0
+; X64-AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; X64-AVX2-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX2-NEXT:    sete %al
+; X64-AVX2-NEXT:    vzeroupper
+; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512BW-LABEL: length64_eq_const:
+; X64-AVX512BW:       # %bb.0:
+; X64-AVX512BW-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-AVX512BW-NEXT:    vpcmpneqb {{.*}}(%rip), %zmm0, %k0
+; X64-AVX512BW-NEXT:    kortestq %k0, %k0
+; X64-AVX512BW-NEXT:    sete %al
+; X64-AVX512BW-NEXT:    vzeroupper
+; X64-AVX512BW-NEXT:    retq
+;
+; X64-AVX512F-LABEL: length64_eq_const:
+; X64-AVX512F:       # %bb.0:
+; X64-AVX512F-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-AVX512F-NEXT:    vpcmpneqd {{.*}}(%rip), %zmm0, %k0
+; X64-AVX512F-NEXT:    kortestw %k0, %k0
+; X64-AVX512F-NEXT:    sete %al
+; X64-AVX512F-NEXT:    vzeroupper
+; X64-AVX512F-NEXT:    retq
+;
+; X64-MIC-AVX2-LABEL: length64_eq_const:
+; X64-MIC-AVX2:       # %bb.0:
+; X64-MIC-AVX2-NEXT:    vmovdqu (%rdi), %ymm0
+; X64-MIC-AVX2-NEXT:    vmovdqu 32(%rdi), %ymm1
+; X64-MIC-AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [892613426,959985462,858927408,926299444,825243960,892613426,959985462,858927408]
+; X64-MIC-AVX2-NEXT:    vpcmpneqd %zmm2, %zmm1, %k0
+; X64-MIC-AVX2-NEXT:    vmovdqa {{.*#+}} ymm1 = [858927408,926299444,825243960,892613426,959985462,858927408,926299444,825243960]
+; X64-MIC-AVX2-NEXT:    vpcmpneqd %zmm1, %zmm0, %k1
+; X64-MIC-AVX2-NEXT:    kortestw %k0, %k1
+; X64-MIC-AVX2-NEXT:    sete %al
+; X64-MIC-AVX2-NEXT:    vzeroupper
+; X64-MIC-AVX2-NEXT:    retq
+;
+; X64-MIC-AVX512F-LABEL: length64_eq_const:
+; X64-MIC-AVX512F:       # %bb.0:
+; X64-MIC-AVX512F-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-MIC-AVX512F-NEXT:    vpcmpneqd {{.*}}(%rip), %zmm0, %k0
+; X64-MIC-AVX512F-NEXT:    kortestw %k0, %k0
+; X64-MIC-AVX512F-NEXT:    sete %al
+; X64-MIC-AVX512F-NEXT:    vzeroupper
+; X64-MIC-AVX512F-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* getelementptr inbounds ([513 x i8], [513 x i8]* @.str, i32 0, i32 0), i64 64) nounwind
+  %c = icmp eq i32 %m, 0
+  ret i1 %c
+}
+
+define i32 @length96(i8* %X, i8* %Y) nounwind {
+; X86-LABEL: length96:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $96
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    retl
+;
+; X64-LABEL: length96:
+; X64:       # %bb.0:
+; X64-NEXT:    movl $96, %edx
+; X64-NEXT:    jmp memcmp # TAILCALL
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 96) nounwind
+  ret i32 %m
+}
+
+define i1 @length96_eq(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length96_eq:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $96
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    setne %al
+; X86-NEXT:    retl
+;
+; X64-SSE-LABEL: length96_eq:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    pushq %rax
+; X64-SSE-NEXT:    movl $96, %edx
+; X64-SSE-NEXT:    callq memcmp
+; X64-SSE-NEXT:    testl %eax, %eax
+; X64-SSE-NEXT:    setne %al
+; X64-SSE-NEXT:    popq %rcx
+; X64-SSE-NEXT:    retq
+;
+; X64-AVX1-LABEL: length96_eq:
+; X64-AVX1:       # %bb.0:
+; X64-AVX1-NEXT:    vmovups (%rdi), %ymm0
+; X64-AVX1-NEXT:    vmovups 32(%rdi), %ymm1
+; X64-AVX1-NEXT:    vmovups 64(%rdi), %ymm2
+; X64-AVX1-NEXT:    vxorps 32(%rsi), %ymm1, %ymm1
+; X64-AVX1-NEXT:    vxorps (%rsi), %ymm0, %ymm0
+; X64-AVX1-NEXT:    vxorps 64(%rsi), %ymm2, %ymm2
+; X64-AVX1-NEXT:    vorps %ymm2, %ymm1, %ymm1
+; X64-AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; X64-AVX1-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX1-NEXT:    setne %al
+; X64-AVX1-NEXT:    vzeroupper
+; X64-AVX1-NEXT:    retq
+;
+; X64-AVX2-LABEL: length96_eq:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    vmovdqu (%rdi), %ymm0
+; X64-AVX2-NEXT:    vmovdqu 32(%rdi), %ymm1
+; X64-AVX2-NEXT:    vmovdqu 64(%rdi), %ymm2
+; X64-AVX2-NEXT:    vpxor 32(%rsi), %ymm1, %ymm1
+; X64-AVX2-NEXT:    vpxor (%rsi), %ymm0, %ymm0
+; X64-AVX2-NEXT:    vpxor 64(%rsi), %ymm2, %ymm2
+; X64-AVX2-NEXT:    vpor %ymm2, %ymm1, %ymm1
+; X64-AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; X64-AVX2-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX2-NEXT:    setne %al
+; X64-AVX2-NEXT:    vzeroupper
+; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512BW-LABEL: length96_eq:
+; X64-AVX512BW:       # %bb.0:
+; X64-AVX512BW-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-AVX512BW-NEXT:    vmovdqu 64(%rdi), %ymm1
+; X64-AVX512BW-NEXT:    vmovdqu 64(%rsi), %ymm2
+; X64-AVX512BW-NEXT:    vpcmpneqb (%rsi), %zmm0, %k0
+; X64-AVX512BW-NEXT:    vpcmpneqb %zmm2, %zmm1, %k1
+; X64-AVX512BW-NEXT:    kortestq %k1, %k0
+; X64-AVX512BW-NEXT:    setne %al
+; X64-AVX512BW-NEXT:    vzeroupper
+; X64-AVX512BW-NEXT:    retq
+;
+; X64-AVX512F-LABEL: length96_eq:
+; X64-AVX512F:       # %bb.0:
+; X64-AVX512F-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-AVX512F-NEXT:    vmovdqu 64(%rdi), %ymm1
+; X64-AVX512F-NEXT:    vmovdqu 64(%rsi), %ymm2
+; X64-AVX512F-NEXT:    vpcmpneqd (%rsi), %zmm0, %k0
+; X64-AVX512F-NEXT:    vpcmpneqd %zmm2, %zmm1, %k1
+; X64-AVX512F-NEXT:    kortestw %k1, %k0
+; X64-AVX512F-NEXT:    setne %al
+; X64-AVX512F-NEXT:    vzeroupper
+; X64-AVX512F-NEXT:    retq
+;
+; X64-MIC-AVX2-LABEL: length96_eq:
+; X64-MIC-AVX2:       # %bb.0:
+; X64-MIC-AVX2-NEXT:    vmovdqu (%rdi), %ymm0
+; X64-MIC-AVX2-NEXT:    vmovdqu 32(%rdi), %ymm1
+; X64-MIC-AVX2-NEXT:    vmovdqu 64(%rdi), %ymm2
+; X64-MIC-AVX2-NEXT:    vmovdqu (%rsi), %ymm3
+; X64-MIC-AVX2-NEXT:    vmovdqu 32(%rsi), %ymm4
+; X64-MIC-AVX2-NEXT:    vmovdqu 64(%rsi), %ymm5
+; X64-MIC-AVX2-NEXT:    vpcmpneqd %zmm4, %zmm1, %k0
+; X64-MIC-AVX2-NEXT:    vpcmpneqd %zmm3, %zmm0, %k1
+; X64-MIC-AVX2-NEXT:    korw %k0, %k1, %k0
+; X64-MIC-AVX2-NEXT:    vpcmpneqd %zmm5, %zmm2, %k1
+; X64-MIC-AVX2-NEXT:    kortestw %k1, %k0
+; X64-MIC-AVX2-NEXT:    setne %al
+; X64-MIC-AVX2-NEXT:    vzeroupper
+; X64-MIC-AVX2-NEXT:    retq
+;
+; X64-MIC-AVX512F-LABEL: length96_eq:
+; X64-MIC-AVX512F:       # %bb.0:
+; X64-MIC-AVX512F-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-MIC-AVX512F-NEXT:    vmovdqu 64(%rdi), %ymm1
+; X64-MIC-AVX512F-NEXT:    vmovdqu 64(%rsi), %ymm2
+; X64-MIC-AVX512F-NEXT:    vpcmpneqd (%rsi), %zmm0, %k0
+; X64-MIC-AVX512F-NEXT:    vpcmpneqd %zmm2, %zmm1, %k1
+; X64-MIC-AVX512F-NEXT:    kortestw %k1, %k0
+; X64-MIC-AVX512F-NEXT:    setne %al
+; X64-MIC-AVX512F-NEXT:    vzeroupper
+; X64-MIC-AVX512F-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 96) nounwind
+  %cmp = icmp ne i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length96_lt(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length96_lt:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $96
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    shrl $31, %eax
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: length96_lt:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $96, %edx
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    shrl $31, %eax
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 96) nounwind
+  %cmp = icmp slt i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length96_gt(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length96_gt:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $96
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    setg %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length96_gt:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $96, %edx
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    setg %al
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 96) nounwind
+  %cmp = icmp sgt i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length96_eq_const(i8* %X) nounwind {
+; X86-LABEL: length96_eq_const:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $96
+; X86-NEXT:    pushl $.L.str
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    sete %al
+; X86-NEXT:    retl
+;
+; X64-SSE-LABEL: length96_eq_const:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    pushq %rax
+; X64-SSE-NEXT:    movl $.L.str, %esi
+; X64-SSE-NEXT:    movl $96, %edx
+; X64-SSE-NEXT:    callq memcmp
+; X64-SSE-NEXT:    testl %eax, %eax
+; X64-SSE-NEXT:    sete %al
+; X64-SSE-NEXT:    popq %rcx
+; X64-SSE-NEXT:    retq
+;
+; X64-AVX1-LABEL: length96_eq_const:
+; X64-AVX1:       # %bb.0:
+; X64-AVX1-NEXT:    vmovups (%rdi), %ymm0
+; X64-AVX1-NEXT:    vmovups 32(%rdi), %ymm1
+; X64-AVX1-NEXT:    vmovups 64(%rdi), %ymm2
+; X64-AVX1-NEXT:    vxorps {{.*}}(%rip), %ymm1, %ymm1
+; X64-AVX1-NEXT:    vxorps {{.*}}(%rip), %ymm0, %ymm0
+; X64-AVX1-NEXT:    vxorps {{.*}}(%rip), %ymm2, %ymm2
+; X64-AVX1-NEXT:    vorps %ymm2, %ymm1, %ymm1
+; X64-AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; X64-AVX1-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX1-NEXT:    sete %al
+; X64-AVX1-NEXT:    vzeroupper
+; X64-AVX1-NEXT:    retq
+;
+; X64-AVX2-LABEL: length96_eq_const:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    vmovdqu (%rdi), %ymm0
+; X64-AVX2-NEXT:    vmovdqu 32(%rdi), %ymm1
+; X64-AVX2-NEXT:    vmovdqu 64(%rdi), %ymm2
+; X64-AVX2-NEXT:    vpxor {{.*}}(%rip), %ymm1, %ymm1
+; X64-AVX2-NEXT:    vpxor {{.*}}(%rip), %ymm0, %ymm0
+; X64-AVX2-NEXT:    vpxor {{.*}}(%rip), %ymm2, %ymm2
+; X64-AVX2-NEXT:    vpor %ymm2, %ymm1, %ymm1
+; X64-AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; X64-AVX2-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX2-NEXT:    sete %al
+; X64-AVX2-NEXT:    vzeroupper
+; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512BW-LABEL: length96_eq_const:
+; X64-AVX512BW:       # %bb.0:
+; X64-AVX512BW-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-AVX512BW-NEXT:    vmovdqu 64(%rdi), %ymm1
+; X64-AVX512BW-NEXT:    vpcmpneqb {{.*}}(%rip), %zmm0, %k0
+; X64-AVX512BW-NEXT:    vpcmpneqb {{.*}}(%rip), %zmm1, %k1
+; X64-AVX512BW-NEXT:    kortestq %k1, %k0
+; X64-AVX512BW-NEXT:    sete %al
+; X64-AVX512BW-NEXT:    vzeroupper
+; X64-AVX512BW-NEXT:    retq
+;
+; X64-AVX512F-LABEL: length96_eq_const:
+; X64-AVX512F:       # %bb.0:
+; X64-AVX512F-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-AVX512F-NEXT:    vmovdqu 64(%rdi), %ymm1
+; X64-AVX512F-NEXT:    vpcmpneqd {{.*}}(%rip), %zmm0, %k0
+; X64-AVX512F-NEXT:    vpcmpneqd {{.*}}(%rip), %zmm1, %k1
+; X64-AVX512F-NEXT:    kortestw %k1, %k0
+; X64-AVX512F-NEXT:    sete %al
+; X64-AVX512F-NEXT:    vzeroupper
+; X64-AVX512F-NEXT:    retq
+;
+; X64-MIC-AVX2-LABEL: length96_eq_const:
+; X64-MIC-AVX2:       # %bb.0:
+; X64-MIC-AVX2-NEXT:    vmovdqu (%rdi), %ymm0
+; X64-MIC-AVX2-NEXT:    vmovdqu 32(%rdi), %ymm1
+; X64-MIC-AVX2-NEXT:    vmovdqu 64(%rdi), %ymm2
+; X64-MIC-AVX2-NEXT:    vmovdqa {{.*#+}} ymm3 = [892613426,959985462,858927408,926299444,825243960,892613426,959985462,858927408]
+; X64-MIC-AVX2-NEXT:    vpcmpneqd %zmm3, %zmm1, %k0
+; X64-MIC-AVX2-NEXT:    vmovdqa {{.*#+}} ymm1 = [858927408,926299444,825243960,892613426,959985462,858927408,926299444,825243960]
+; X64-MIC-AVX2-NEXT:    vpcmpneqd %zmm1, %zmm0, %k1
+; X64-MIC-AVX2-NEXT:    korw %k0, %k1, %k0
+; X64-MIC-AVX2-NEXT:    vmovdqa {{.*#+}} ymm0 = [926299444,825243960,892613426,959985462,858927408,926299444,825243960,892613426]
+; X64-MIC-AVX2-NEXT:    vpcmpneqd %zmm0, %zmm2, %k1
+; X64-MIC-AVX2-NEXT:    kortestw %k1, %k0
+; X64-MIC-AVX2-NEXT:    sete %al
+; X64-MIC-AVX2-NEXT:    vzeroupper
+; X64-MIC-AVX2-NEXT:    retq
+;
+; X64-MIC-AVX512F-LABEL: length96_eq_const:
+; X64-MIC-AVX512F:       # %bb.0:
+; X64-MIC-AVX512F-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-MIC-AVX512F-NEXT:    vmovdqu 64(%rdi), %ymm1
+; X64-MIC-AVX512F-NEXT:    vpcmpneqd {{.*}}(%rip), %zmm0, %k0
+; X64-MIC-AVX512F-NEXT:    vpcmpneqd {{.*}}(%rip), %zmm1, %k1
+; X64-MIC-AVX512F-NEXT:    kortestw %k1, %k0
+; X64-MIC-AVX512F-NEXT:    sete %al
+; X64-MIC-AVX512F-NEXT:    vzeroupper
+; X64-MIC-AVX512F-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* getelementptr inbounds ([513 x i8], [513 x i8]* @.str, i32 0, i32 0), i64 96) nounwind
+  %c = icmp eq i32 %m, 0
+  ret i1 %c
+}
+
+define i32 @length127(i8* %X, i8* %Y) nounwind {
+; X86-LABEL: length127:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $127
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    retl
+;
+; X64-LABEL: length127:
+; X64:       # %bb.0:
+; X64-NEXT:    movl $127, %edx
+; X64-NEXT:    jmp memcmp # TAILCALL
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 127) nounwind
+  ret i32 %m
+}
+
+define i1 @length127_eq(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length127_eq:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $127
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    setne %al
+; X86-NEXT:    retl
+;
+; X64-SSE-LABEL: length127_eq:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    pushq %rax
+; X64-SSE-NEXT:    movl $127, %edx
+; X64-SSE-NEXT:    callq memcmp
+; X64-SSE-NEXT:    testl %eax, %eax
+; X64-SSE-NEXT:    setne %al
+; X64-SSE-NEXT:    popq %rcx
+; X64-SSE-NEXT:    retq
+;
+; X64-AVX1-LABEL: length127_eq:
+; X64-AVX1:       # %bb.0:
+; X64-AVX1-NEXT:    vmovups (%rdi), %ymm0
+; X64-AVX1-NEXT:    vmovups 32(%rdi), %ymm1
+; X64-AVX1-NEXT:    vmovups 64(%rdi), %ymm2
+; X64-AVX1-NEXT:    vmovups 95(%rdi), %ymm3
+; X64-AVX1-NEXT:    vxorps 95(%rsi), %ymm3, %ymm3
+; X64-AVX1-NEXT:    vxorps 64(%rsi), %ymm2, %ymm2
+; X64-AVX1-NEXT:    vorps %ymm3, %ymm2, %ymm2
+; X64-AVX1-NEXT:    vxorps 32(%rsi), %ymm1, %ymm1
+; X64-AVX1-NEXT:    vorps %ymm2, %ymm1, %ymm1
+; X64-AVX1-NEXT:    vxorps (%rsi), %ymm0, %ymm0
+; X64-AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; X64-AVX1-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX1-NEXT:    setne %al
+; X64-AVX1-NEXT:    vzeroupper
+; X64-AVX1-NEXT:    retq
+;
+; X64-AVX2-LABEL: length127_eq:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    vmovdqu (%rdi), %ymm0
+; X64-AVX2-NEXT:    vmovdqu 32(%rdi), %ymm1
+; X64-AVX2-NEXT:    vmovdqu 64(%rdi), %ymm2
+; X64-AVX2-NEXT:    vmovdqu 95(%rdi), %ymm3
+; X64-AVX2-NEXT:    vpxor 95(%rsi), %ymm3, %ymm3
+; X64-AVX2-NEXT:    vpxor 64(%rsi), %ymm2, %ymm2
+; X64-AVX2-NEXT:    vpor %ymm3, %ymm2, %ymm2
+; X64-AVX2-NEXT:    vpxor 32(%rsi), %ymm1, %ymm1
+; X64-AVX2-NEXT:    vpor %ymm2, %ymm1, %ymm1
+; X64-AVX2-NEXT:    vpxor (%rsi), %ymm0, %ymm0
+; X64-AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; X64-AVX2-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX2-NEXT:    setne %al
+; X64-AVX2-NEXT:    vzeroupper
+; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512BW-LABEL: length127_eq:
+; X64-AVX512BW:       # %bb.0:
+; X64-AVX512BW-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-AVX512BW-NEXT:    vmovdqu64 63(%rdi), %zmm1
+; X64-AVX512BW-NEXT:    vpcmpneqb 63(%rsi), %zmm1, %k0
+; X64-AVX512BW-NEXT:    vpcmpneqb (%rsi), %zmm0, %k1
+; X64-AVX512BW-NEXT:    kortestq %k0, %k1
+; X64-AVX512BW-NEXT:    setne %al
+; X64-AVX512BW-NEXT:    vzeroupper
+; X64-AVX512BW-NEXT:    retq
+;
+; X64-AVX512F-LABEL: length127_eq:
+; X64-AVX512F:       # %bb.0:
+; X64-AVX512F-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-AVX512F-NEXT:    vmovdqu64 63(%rdi), %zmm1
+; X64-AVX512F-NEXT:    vpcmpneqd 63(%rsi), %zmm1, %k0
+; X64-AVX512F-NEXT:    vpcmpneqd (%rsi), %zmm0, %k1
+; X64-AVX512F-NEXT:    kortestw %k0, %k1
+; X64-AVX512F-NEXT:    setne %al
+; X64-AVX512F-NEXT:    vzeroupper
+; X64-AVX512F-NEXT:    retq
+;
+; X64-MIC-AVX2-LABEL: length127_eq:
+; X64-MIC-AVX2:       # %bb.0:
+; X64-MIC-AVX2-NEXT:    vmovdqu (%rdi), %ymm0
+; X64-MIC-AVX2-NEXT:    vmovdqu 32(%rdi), %ymm1
+; X64-MIC-AVX2-NEXT:    vmovdqu 64(%rdi), %ymm2
+; X64-MIC-AVX2-NEXT:    vmovdqu 95(%rdi), %ymm3
+; X64-MIC-AVX2-NEXT:    vmovdqu (%rsi), %ymm4
+; X64-MIC-AVX2-NEXT:    vmovdqu 32(%rsi), %ymm5
+; X64-MIC-AVX2-NEXT:    vmovdqu 64(%rsi), %ymm6
+; X64-MIC-AVX2-NEXT:    vmovdqu 95(%rsi), %ymm7
+; X64-MIC-AVX2-NEXT:    vpcmpneqd %zmm7, %zmm3, %k0
+; X64-MIC-AVX2-NEXT:    vpcmpneqd %zmm6, %zmm2, %k1
+; X64-MIC-AVX2-NEXT:    korw %k0, %k1, %k0
+; X64-MIC-AVX2-NEXT:    vpcmpneqd %zmm5, %zmm1, %k1
+; X64-MIC-AVX2-NEXT:    vpcmpneqd %zmm4, %zmm0, %k2
+; X64-MIC-AVX2-NEXT:    korw %k1, %k2, %k1
+; X64-MIC-AVX2-NEXT:    kortestw %k0, %k1
+; X64-MIC-AVX2-NEXT:    setne %al
+; X64-MIC-AVX2-NEXT:    vzeroupper
+; X64-MIC-AVX2-NEXT:    retq
+;
+; X64-MIC-AVX512F-LABEL: length127_eq:
+; X64-MIC-AVX512F:       # %bb.0:
+; X64-MIC-AVX512F-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-MIC-AVX512F-NEXT:    vmovdqu64 63(%rdi), %zmm1
+; X64-MIC-AVX512F-NEXT:    vpcmpneqd 63(%rsi), %zmm1, %k0
+; X64-MIC-AVX512F-NEXT:    vpcmpneqd (%rsi), %zmm0, %k1
+; X64-MIC-AVX512F-NEXT:    kortestw %k0, %k1
+; X64-MIC-AVX512F-NEXT:    setne %al
+; X64-MIC-AVX512F-NEXT:    vzeroupper
+; X64-MIC-AVX512F-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 127) nounwind
+  %cmp = icmp ne i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length127_lt(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length127_lt:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $127
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    shrl $31, %eax
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: length127_lt:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $127, %edx
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    shrl $31, %eax
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 127) nounwind
+  %cmp = icmp slt i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length127_gt(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length127_gt:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $127
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    setg %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length127_gt:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $127, %edx
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    setg %al
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 127) nounwind
+  %cmp = icmp sgt i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length127_eq_const(i8* %X) nounwind {
+; X86-LABEL: length127_eq_const:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $127
+; X86-NEXT:    pushl $.L.str
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    sete %al
+; X86-NEXT:    retl
+;
+; X64-SSE-LABEL: length127_eq_const:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    pushq %rax
+; X64-SSE-NEXT:    movl $.L.str, %esi
+; X64-SSE-NEXT:    movl $127, %edx
+; X64-SSE-NEXT:    callq memcmp
+; X64-SSE-NEXT:    testl %eax, %eax
+; X64-SSE-NEXT:    sete %al
+; X64-SSE-NEXT:    popq %rcx
+; X64-SSE-NEXT:    retq
+;
+; X64-AVX1-LABEL: length127_eq_const:
+; X64-AVX1:       # %bb.0:
+; X64-AVX1-NEXT:    vmovups (%rdi), %ymm0
+; X64-AVX1-NEXT:    vmovups 32(%rdi), %ymm1
+; X64-AVX1-NEXT:    vmovups 64(%rdi), %ymm2
+; X64-AVX1-NEXT:    vmovups 95(%rdi), %ymm3
+; X64-AVX1-NEXT:    vxorps {{.*}}(%rip), %ymm3, %ymm3
+; X64-AVX1-NEXT:    vxorps {{.*}}(%rip), %ymm2, %ymm2
+; X64-AVX1-NEXT:    vorps %ymm3, %ymm2, %ymm2
+; X64-AVX1-NEXT:    vxorps {{.*}}(%rip), %ymm1, %ymm1
+; X64-AVX1-NEXT:    vorps %ymm2, %ymm1, %ymm1
+; X64-AVX1-NEXT:    vxorps {{.*}}(%rip), %ymm0, %ymm0
+; X64-AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; X64-AVX1-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX1-NEXT:    sete %al
+; X64-AVX1-NEXT:    vzeroupper
+; X64-AVX1-NEXT:    retq
+;
+; X64-AVX2-LABEL: length127_eq_const:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    vmovdqu (%rdi), %ymm0
+; X64-AVX2-NEXT:    vmovdqu 32(%rdi), %ymm1
+; X64-AVX2-NEXT:    vmovdqu 64(%rdi), %ymm2
+; X64-AVX2-NEXT:    vmovdqu 95(%rdi), %ymm3
+; X64-AVX2-NEXT:    vpxor {{.*}}(%rip), %ymm3, %ymm3
+; X64-AVX2-NEXT:    vpxor {{.*}}(%rip), %ymm2, %ymm2
+; X64-AVX2-NEXT:    vpor %ymm3, %ymm2, %ymm2
+; X64-AVX2-NEXT:    vpxor {{.*}}(%rip), %ymm1, %ymm1
+; X64-AVX2-NEXT:    vpor %ymm2, %ymm1, %ymm1
+; X64-AVX2-NEXT:    vpxor {{.*}}(%rip), %ymm0, %ymm0
+; X64-AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; X64-AVX2-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX2-NEXT:    sete %al
+; X64-AVX2-NEXT:    vzeroupper
+; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512BW-LABEL: length127_eq_const:
+; X64-AVX512BW:       # %bb.0:
+; X64-AVX512BW-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-AVX512BW-NEXT:    vmovdqu64 63(%rdi), %zmm1
+; X64-AVX512BW-NEXT:    vpcmpneqb .L.str+{{.*}}(%rip), %zmm1, %k0
+; X64-AVX512BW-NEXT:    vpcmpneqb {{.*}}(%rip), %zmm0, %k1
+; X64-AVX512BW-NEXT:    kortestq %k0, %k1
+; X64-AVX512BW-NEXT:    sete %al
+; X64-AVX512BW-NEXT:    vzeroupper
+; X64-AVX512BW-NEXT:    retq
+;
+; X64-AVX512F-LABEL: length127_eq_const:
+; X64-AVX512F:       # %bb.0:
+; X64-AVX512F-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-AVX512F-NEXT:    vmovdqu64 63(%rdi), %zmm1
+; X64-AVX512F-NEXT:    vpcmpneqd .L.str+{{.*}}(%rip), %zmm1, %k0
+; X64-AVX512F-NEXT:    vpcmpneqd {{.*}}(%rip), %zmm0, %k1
+; X64-AVX512F-NEXT:    kortestw %k0, %k1
+; X64-AVX512F-NEXT:    sete %al
+; X64-AVX512F-NEXT:    vzeroupper
+; X64-AVX512F-NEXT:    retq
+;
+; X64-MIC-AVX2-LABEL: length127_eq_const:
+; X64-MIC-AVX2:       # %bb.0:
+; X64-MIC-AVX2-NEXT:    vmovdqu (%rdi), %ymm0
+; X64-MIC-AVX2-NEXT:    vmovdqu 32(%rdi), %ymm1
+; X64-MIC-AVX2-NEXT:    vmovdqu 64(%rdi), %ymm2
+; X64-MIC-AVX2-NEXT:    vmovdqu 95(%rdi), %ymm3
+; X64-MIC-AVX2-NEXT:    vmovdqa {{.*#+}} ymm4 = [943142453,842084409,909456435,809056311,875770417,943142453,842084409,909456435]
+; X64-MIC-AVX2-NEXT:    vpcmpneqd %zmm4, %zmm3, %k0
+; X64-MIC-AVX2-NEXT:    vmovdqa {{.*#+}} ymm3 = [926299444,825243960,892613426,959985462,858927408,926299444,825243960,892613426]
+; X64-MIC-AVX2-NEXT:    vpcmpneqd %zmm3, %zmm2, %k1
+; X64-MIC-AVX2-NEXT:    korw %k0, %k1, %k0
+; X64-MIC-AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [892613426,959985462,858927408,926299444,825243960,892613426,959985462,858927408]
+; X64-MIC-AVX2-NEXT:    vpcmpneqd %zmm2, %zmm1, %k1
+; X64-MIC-AVX2-NEXT:    vmovdqa {{.*#+}} ymm1 = [858927408,926299444,825243960,892613426,959985462,858927408,926299444,825243960]
+; X64-MIC-AVX2-NEXT:    vpcmpneqd %zmm1, %zmm0, %k2
+; X64-MIC-AVX2-NEXT:    korw %k1, %k2, %k1
+; X64-MIC-AVX2-NEXT:    kortestw %k0, %k1
+; X64-MIC-AVX2-NEXT:    sete %al
+; X64-MIC-AVX2-NEXT:    vzeroupper
+; X64-MIC-AVX2-NEXT:    retq
+;
+; X64-MIC-AVX512F-LABEL: length127_eq_const:
+; X64-MIC-AVX512F:       # %bb.0:
+; X64-MIC-AVX512F-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-MIC-AVX512F-NEXT:    vmovdqu64 63(%rdi), %zmm1
+; X64-MIC-AVX512F-NEXT:    vpcmpneqd .L.str+{{.*}}(%rip), %zmm1, %k0
+; X64-MIC-AVX512F-NEXT:    vpcmpneqd {{.*}}(%rip), %zmm0, %k1
+; X64-MIC-AVX512F-NEXT:    kortestw %k0, %k1
+; X64-MIC-AVX512F-NEXT:    sete %al
+; X64-MIC-AVX512F-NEXT:    vzeroupper
+; X64-MIC-AVX512F-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* getelementptr inbounds ([513 x i8], [513 x i8]* @.str, i32 0, i32 0), i64 127) nounwind
+  %c = icmp eq i32 %m, 0
+  ret i1 %c
+}
+
+define i32 @length128(i8* %X, i8* %Y) nounwind {
+; X86-LABEL: length128:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $128
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    retl
+;
+; X64-LABEL: length128:
+; X64:       # %bb.0:
+; X64-NEXT:    movl $128, %edx
+; X64-NEXT:    jmp memcmp # TAILCALL
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 128) nounwind
+  ret i32 %m
+}
+
+define i1 @length128_eq(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length128_eq:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $128
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    setne %al
+; X86-NEXT:    retl
+;
+; X64-SSE-LABEL: length128_eq:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    pushq %rax
+; X64-SSE-NEXT:    movl $128, %edx
+; X64-SSE-NEXT:    callq memcmp
+; X64-SSE-NEXT:    testl %eax, %eax
+; X64-SSE-NEXT:    setne %al
+; X64-SSE-NEXT:    popq %rcx
+; X64-SSE-NEXT:    retq
+;
+; X64-AVX1-LABEL: length128_eq:
+; X64-AVX1:       # %bb.0:
+; X64-AVX1-NEXT:    vmovups (%rdi), %ymm0
+; X64-AVX1-NEXT:    vmovups 32(%rdi), %ymm1
+; X64-AVX1-NEXT:    vmovups 64(%rdi), %ymm2
+; X64-AVX1-NEXT:    vmovups 96(%rdi), %ymm3
+; X64-AVX1-NEXT:    vxorps 96(%rsi), %ymm3, %ymm3
+; X64-AVX1-NEXT:    vxorps 64(%rsi), %ymm2, %ymm2
+; X64-AVX1-NEXT:    vorps %ymm3, %ymm2, %ymm2
+; X64-AVX1-NEXT:    vxorps 32(%rsi), %ymm1, %ymm1
+; X64-AVX1-NEXT:    vorps %ymm2, %ymm1, %ymm1
+; X64-AVX1-NEXT:    vxorps (%rsi), %ymm0, %ymm0
+; X64-AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; X64-AVX1-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX1-NEXT:    setne %al
+; X64-AVX1-NEXT:    vzeroupper
+; X64-AVX1-NEXT:    retq
+;
+; X64-AVX2-LABEL: length128_eq:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    vmovdqu (%rdi), %ymm0
+; X64-AVX2-NEXT:    vmovdqu 32(%rdi), %ymm1
+; X64-AVX2-NEXT:    vmovdqu 64(%rdi), %ymm2
+; X64-AVX2-NEXT:    vmovdqu 96(%rdi), %ymm3
+; X64-AVX2-NEXT:    vpxor 96(%rsi), %ymm3, %ymm3
+; X64-AVX2-NEXT:    vpxor 64(%rsi), %ymm2, %ymm2
+; X64-AVX2-NEXT:    vpor %ymm3, %ymm2, %ymm2
+; X64-AVX2-NEXT:    vpxor 32(%rsi), %ymm1, %ymm1
+; X64-AVX2-NEXT:    vpor %ymm2, %ymm1, %ymm1
+; X64-AVX2-NEXT:    vpxor (%rsi), %ymm0, %ymm0
+; X64-AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; X64-AVX2-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX2-NEXT:    setne %al
+; X64-AVX2-NEXT:    vzeroupper
+; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512BW-LABEL: length128_eq:
+; X64-AVX512BW:       # %bb.0:
+; X64-AVX512BW-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-AVX512BW-NEXT:    vmovdqu64 64(%rdi), %zmm1
+; X64-AVX512BW-NEXT:    vpcmpneqb 64(%rsi), %zmm1, %k0
+; X64-AVX512BW-NEXT:    vpcmpneqb (%rsi), %zmm0, %k1
+; X64-AVX512BW-NEXT:    kortestq %k0, %k1
+; X64-AVX512BW-NEXT:    setne %al
+; X64-AVX512BW-NEXT:    vzeroupper
+; X64-AVX512BW-NEXT:    retq
+;
+; X64-AVX512F-LABEL: length128_eq:
+; X64-AVX512F:       # %bb.0:
+; X64-AVX512F-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-AVX512F-NEXT:    vmovdqu64 64(%rdi), %zmm1
+; X64-AVX512F-NEXT:    vpcmpneqd 64(%rsi), %zmm1, %k0
+; X64-AVX512F-NEXT:    vpcmpneqd (%rsi), %zmm0, %k1
+; X64-AVX512F-NEXT:    kortestw %k0, %k1
+; X64-AVX512F-NEXT:    setne %al
+; X64-AVX512F-NEXT:    vzeroupper
+; X64-AVX512F-NEXT:    retq
+;
+; X64-MIC-AVX2-LABEL: length128_eq:
+; X64-MIC-AVX2:       # %bb.0:
+; X64-MIC-AVX2-NEXT:    vmovdqu (%rdi), %ymm0
+; X64-MIC-AVX2-NEXT:    vmovdqu 32(%rdi), %ymm1
+; X64-MIC-AVX2-NEXT:    vmovdqu 64(%rdi), %ymm2
+; X64-MIC-AVX2-NEXT:    vmovdqu 96(%rdi), %ymm3
+; X64-MIC-AVX2-NEXT:    vmovdqu (%rsi), %ymm4
+; X64-MIC-AVX2-NEXT:    vmovdqu 32(%rsi), %ymm5
+; X64-MIC-AVX2-NEXT:    vmovdqu 64(%rsi), %ymm6
+; X64-MIC-AVX2-NEXT:    vmovdqu 96(%rsi), %ymm7
+; X64-MIC-AVX2-NEXT:    vpcmpneqd %zmm7, %zmm3, %k0
+; X64-MIC-AVX2-NEXT:    vpcmpneqd %zmm6, %zmm2, %k1
+; X64-MIC-AVX2-NEXT:    korw %k0, %k1, %k0
+; X64-MIC-AVX2-NEXT:    vpcmpneqd %zmm5, %zmm1, %k1
+; X64-MIC-AVX2-NEXT:    vpcmpneqd %zmm4, %zmm0, %k2
+; X64-MIC-AVX2-NEXT:    korw %k1, %k2, %k1
+; X64-MIC-AVX2-NEXT:    kortestw %k0, %k1
+; X64-MIC-AVX2-NEXT:    setne %al
+; X64-MIC-AVX2-NEXT:    vzeroupper
+; X64-MIC-AVX2-NEXT:    retq
+;
+; X64-MIC-AVX512F-LABEL: length128_eq:
+; X64-MIC-AVX512F:       # %bb.0:
+; X64-MIC-AVX512F-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-MIC-AVX512F-NEXT:    vmovdqu64 64(%rdi), %zmm1
+; X64-MIC-AVX512F-NEXT:    vpcmpneqd 64(%rsi), %zmm1, %k0
+; X64-MIC-AVX512F-NEXT:    vpcmpneqd (%rsi), %zmm0, %k1
+; X64-MIC-AVX512F-NEXT:    kortestw %k0, %k1
+; X64-MIC-AVX512F-NEXT:    setne %al
+; X64-MIC-AVX512F-NEXT:    vzeroupper
+; X64-MIC-AVX512F-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 128) nounwind
+  %cmp = icmp ne i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length128_lt(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length128_lt:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $128
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    shrl $31, %eax
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: length128_lt:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $128, %edx
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    shrl $31, %eax
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 128) nounwind
+  %cmp = icmp slt i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length128_gt(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length128_gt:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $128
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    setg %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length128_gt:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $128, %edx
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    setg %al
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 128) nounwind
+  %cmp = icmp sgt i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length128_eq_const(i8* %X) nounwind {
+; X86-LABEL: length128_eq_const:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $128
+; X86-NEXT:    pushl $.L.str
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    sete %al
+; X86-NEXT:    retl
+;
+; X64-SSE-LABEL: length128_eq_const:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    pushq %rax
+; X64-SSE-NEXT:    movl $.L.str, %esi
+; X64-SSE-NEXT:    movl $128, %edx
+; X64-SSE-NEXT:    callq memcmp
+; X64-SSE-NEXT:    testl %eax, %eax
+; X64-SSE-NEXT:    sete %al
+; X64-SSE-NEXT:    popq %rcx
+; X64-SSE-NEXT:    retq
+;
+; X64-AVX1-LABEL: length128_eq_const:
+; X64-AVX1:       # %bb.0:
+; X64-AVX1-NEXT:    vmovups (%rdi), %ymm0
+; X64-AVX1-NEXT:    vmovups 32(%rdi), %ymm1
+; X64-AVX1-NEXT:    vmovups 64(%rdi), %ymm2
+; X64-AVX1-NEXT:    vmovups 96(%rdi), %ymm3
+; X64-AVX1-NEXT:    vxorps {{.*}}(%rip), %ymm3, %ymm3
+; X64-AVX1-NEXT:    vxorps {{.*}}(%rip), %ymm2, %ymm2
+; X64-AVX1-NEXT:    vorps %ymm3, %ymm2, %ymm2
+; X64-AVX1-NEXT:    vxorps {{.*}}(%rip), %ymm1, %ymm1
+; X64-AVX1-NEXT:    vorps %ymm2, %ymm1, %ymm1
+; X64-AVX1-NEXT:    vxorps {{.*}}(%rip), %ymm0, %ymm0
+; X64-AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; X64-AVX1-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX1-NEXT:    sete %al
+; X64-AVX1-NEXT:    vzeroupper
+; X64-AVX1-NEXT:    retq
+;
+; X64-AVX2-LABEL: length128_eq_const:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    vmovdqu (%rdi), %ymm0
+; X64-AVX2-NEXT:    vmovdqu 32(%rdi), %ymm1
+; X64-AVX2-NEXT:    vmovdqu 64(%rdi), %ymm2
+; X64-AVX2-NEXT:    vmovdqu 96(%rdi), %ymm3
+; X64-AVX2-NEXT:    vpxor {{.*}}(%rip), %ymm3, %ymm3
+; X64-AVX2-NEXT:    vpxor {{.*}}(%rip), %ymm2, %ymm2
+; X64-AVX2-NEXT:    vpor %ymm3, %ymm2, %ymm2
+; X64-AVX2-NEXT:    vpxor {{.*}}(%rip), %ymm1, %ymm1
+; X64-AVX2-NEXT:    vpor %ymm2, %ymm1, %ymm1
+; X64-AVX2-NEXT:    vpxor {{.*}}(%rip), %ymm0, %ymm0
+; X64-AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; X64-AVX2-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX2-NEXT:    sete %al
+; X64-AVX2-NEXT:    vzeroupper
+; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512BW-LABEL: length128_eq_const:
+; X64-AVX512BW:       # %bb.0:
+; X64-AVX512BW-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-AVX512BW-NEXT:    vmovdqu64 64(%rdi), %zmm1
+; X64-AVX512BW-NEXT:    vpcmpneqb .L.str+{{.*}}(%rip), %zmm1, %k0
+; X64-AVX512BW-NEXT:    vpcmpneqb {{.*}}(%rip), %zmm0, %k1
+; X64-AVX512BW-NEXT:    kortestq %k0, %k1
+; X64-AVX512BW-NEXT:    sete %al
+; X64-AVX512BW-NEXT:    vzeroupper
+; X64-AVX512BW-NEXT:    retq
+;
+; X64-AVX512F-LABEL: length128_eq_const:
+; X64-AVX512F:       # %bb.0:
+; X64-AVX512F-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-AVX512F-NEXT:    vmovdqu64 64(%rdi), %zmm1
+; X64-AVX512F-NEXT:    vpcmpneqd .L.str+{{.*}}(%rip), %zmm1, %k0
+; X64-AVX512F-NEXT:    vpcmpneqd {{.*}}(%rip), %zmm0, %k1
+; X64-AVX512F-NEXT:    kortestw %k0, %k1
+; X64-AVX512F-NEXT:    sete %al
+; X64-AVX512F-NEXT:    vzeroupper
+; X64-AVX512F-NEXT:    retq
+;
+; X64-MIC-AVX2-LABEL: length128_eq_const:
+; X64-MIC-AVX2:       # %bb.0:
+; X64-MIC-AVX2-NEXT:    vmovdqu (%rdi), %ymm0
+; X64-MIC-AVX2-NEXT:    vmovdqu 32(%rdi), %ymm1
+; X64-MIC-AVX2-NEXT:    vmovdqu 64(%rdi), %ymm2
+; X64-MIC-AVX2-NEXT:    vmovdqu 96(%rdi), %ymm3
+; X64-MIC-AVX2-NEXT:    vmovdqa {{.*#+}} ymm4 = [959985462,858927408,926299444,825243960,892613426,959985462,858927408,926299444]
+; X64-MIC-AVX2-NEXT:    vpcmpneqd %zmm4, %zmm3, %k0
+; X64-MIC-AVX2-NEXT:    vmovdqa {{.*#+}} ymm3 = [926299444,825243960,892613426,959985462,858927408,926299444,825243960,892613426]
+; X64-MIC-AVX2-NEXT:    vpcmpneqd %zmm3, %zmm2, %k1
+; X64-MIC-AVX2-NEXT:    korw %k0, %k1, %k0
+; X64-MIC-AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [892613426,959985462,858927408,926299444,825243960,892613426,959985462,858927408]
+; X64-MIC-AVX2-NEXT:    vpcmpneqd %zmm2, %zmm1, %k1
+; X64-MIC-AVX2-NEXT:    vmovdqa {{.*#+}} ymm1 = [858927408,926299444,825243960,892613426,959985462,858927408,926299444,825243960]
+; X64-MIC-AVX2-NEXT:    vpcmpneqd %zmm1, %zmm0, %k2
+; X64-MIC-AVX2-NEXT:    korw %k1, %k2, %k1
+; X64-MIC-AVX2-NEXT:    kortestw %k0, %k1
+; X64-MIC-AVX2-NEXT:    sete %al
+; X64-MIC-AVX2-NEXT:    vzeroupper
+; X64-MIC-AVX2-NEXT:    retq
+;
+; X64-MIC-AVX512F-LABEL: length128_eq_const:
+; X64-MIC-AVX512F:       # %bb.0:
+; X64-MIC-AVX512F-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-MIC-AVX512F-NEXT:    vmovdqu64 64(%rdi), %zmm1
+; X64-MIC-AVX512F-NEXT:    vpcmpneqd .L.str+{{.*}}(%rip), %zmm1, %k0
+; X64-MIC-AVX512F-NEXT:    vpcmpneqd {{.*}}(%rip), %zmm0, %k1
+; X64-MIC-AVX512F-NEXT:    kortestw %k0, %k1
+; X64-MIC-AVX512F-NEXT:    sete %al
+; X64-MIC-AVX512F-NEXT:    vzeroupper
+; X64-MIC-AVX512F-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* getelementptr inbounds ([513 x i8], [513 x i8]* @.str, i32 0, i32 0), i64 128) nounwind
+  %c = icmp eq i32 %m, 0
+  ret i1 %c
+}
+
+define i32 @length192(i8* %X, i8* %Y) nounwind {
+; X86-LABEL: length192:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $192
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    retl
+;
+; X64-LABEL: length192:
+; X64:       # %bb.0:
+; X64-NEXT:    movl $192, %edx
+; X64-NEXT:    jmp memcmp # TAILCALL
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 192) nounwind
+  ret i32 %m
+}
+
+define i1 @length192_eq(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length192_eq:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $192
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    setne %al
+; X86-NEXT:    retl
+;
+; X64-SSE-LABEL: length192_eq:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    pushq %rax
+; X64-SSE-NEXT:    movl $192, %edx
+; X64-SSE-NEXT:    callq memcmp
+; X64-SSE-NEXT:    testl %eax, %eax
+; X64-SSE-NEXT:    setne %al
+; X64-SSE-NEXT:    popq %rcx
+; X64-SSE-NEXT:    retq
+;
+; X64-AVX1-LABEL: length192_eq:
+; X64-AVX1:       # %bb.0:
+; X64-AVX1-NEXT:    pushq %rax
+; X64-AVX1-NEXT:    movl $192, %edx
+; X64-AVX1-NEXT:    callq memcmp
+; X64-AVX1-NEXT:    testl %eax, %eax
+; X64-AVX1-NEXT:    setne %al
+; X64-AVX1-NEXT:    popq %rcx
+; X64-AVX1-NEXT:    retq
+;
+; X64-AVX2-LABEL: length192_eq:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    pushq %rax
+; X64-AVX2-NEXT:    movl $192, %edx
+; X64-AVX2-NEXT:    callq memcmp
+; X64-AVX2-NEXT:    testl %eax, %eax
+; X64-AVX2-NEXT:    setne %al
+; X64-AVX2-NEXT:    popq %rcx
+; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512BW-LABEL: length192_eq:
+; X64-AVX512BW:       # %bb.0:
+; X64-AVX512BW-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-AVX512BW-NEXT:    vmovdqu64 64(%rdi), %zmm1
+; X64-AVX512BW-NEXT:    vmovdqu64 128(%rdi), %zmm2
+; X64-AVX512BW-NEXT:    vpcmpneqb 64(%rsi), %zmm1, %k0
+; X64-AVX512BW-NEXT:    vpcmpneqb (%rsi), %zmm0, %k1
+; X64-AVX512BW-NEXT:    korq %k0, %k1, %k0
+; X64-AVX512BW-NEXT:    vpcmpneqb 128(%rsi), %zmm2, %k1
+; X64-AVX512BW-NEXT:    kortestq %k1, %k0
+; X64-AVX512BW-NEXT:    setne %al
+; X64-AVX512BW-NEXT:    vzeroupper
+; X64-AVX512BW-NEXT:    retq
+;
+; X64-AVX512F-LABEL: length192_eq:
+; X64-AVX512F:       # %bb.0:
+; X64-AVX512F-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-AVX512F-NEXT:    vmovdqu64 64(%rdi), %zmm1
+; X64-AVX512F-NEXT:    vmovdqu64 128(%rdi), %zmm2
+; X64-AVX512F-NEXT:    vpcmpneqd 64(%rsi), %zmm1, %k0
+; X64-AVX512F-NEXT:    vpcmpneqd (%rsi), %zmm0, %k1
+; X64-AVX512F-NEXT:    korw %k0, %k1, %k0
+; X64-AVX512F-NEXT:    vpcmpneqd 128(%rsi), %zmm2, %k1
+; X64-AVX512F-NEXT:    kortestw %k1, %k0
+; X64-AVX512F-NEXT:    setne %al
+; X64-AVX512F-NEXT:    vzeroupper
+; X64-AVX512F-NEXT:    retq
+;
+; X64-MIC-AVX2-LABEL: length192_eq:
+; X64-MIC-AVX2:       # %bb.0:
+; X64-MIC-AVX2-NEXT:    pushq %rax
+; X64-MIC-AVX2-NEXT:    movl $192, %edx
+; X64-MIC-AVX2-NEXT:    callq memcmp
+; X64-MIC-AVX2-NEXT:    testl %eax, %eax
+; X64-MIC-AVX2-NEXT:    setne %al
+; X64-MIC-AVX2-NEXT:    popq %rcx
+; X64-MIC-AVX2-NEXT:    retq
+;
+; X64-MIC-AVX512F-LABEL: length192_eq:
+; X64-MIC-AVX512F:       # %bb.0:
+; X64-MIC-AVX512F-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-MIC-AVX512F-NEXT:    vmovdqu64 64(%rdi), %zmm1
+; X64-MIC-AVX512F-NEXT:    vmovdqu64 128(%rdi), %zmm2
+; X64-MIC-AVX512F-NEXT:    vpcmpneqd 64(%rsi), %zmm1, %k0
+; X64-MIC-AVX512F-NEXT:    vpcmpneqd (%rsi), %zmm0, %k1
+; X64-MIC-AVX512F-NEXT:    korw %k0, %k1, %k0
+; X64-MIC-AVX512F-NEXT:    vpcmpneqd 128(%rsi), %zmm2, %k1
+; X64-MIC-AVX512F-NEXT:    kortestw %k1, %k0
+; X64-MIC-AVX512F-NEXT:    setne %al
+; X64-MIC-AVX512F-NEXT:    vzeroupper
+; X64-MIC-AVX512F-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 192) nounwind
+  %cmp = icmp ne i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length192_lt(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length192_lt:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $192
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    shrl $31, %eax
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: length192_lt:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $192, %edx
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    shrl $31, %eax
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 192) nounwind
+  %cmp = icmp slt i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length192_gt(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length192_gt:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $192
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    setg %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length192_gt:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $192, %edx
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    setg %al
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 192) nounwind
+  %cmp = icmp sgt i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length192_eq_const(i8* %X) nounwind {
+; X86-LABEL: length192_eq_const:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $192
+; X86-NEXT:    pushl $.L.str
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    sete %al
+; X86-NEXT:    retl
+;
+; X64-SSE-LABEL: length192_eq_const:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    pushq %rax
+; X64-SSE-NEXT:    movl $.L.str, %esi
+; X64-SSE-NEXT:    movl $192, %edx
+; X64-SSE-NEXT:    callq memcmp
+; X64-SSE-NEXT:    testl %eax, %eax
+; X64-SSE-NEXT:    sete %al
+; X64-SSE-NEXT:    popq %rcx
+; X64-SSE-NEXT:    retq
+;
+; X64-AVX1-LABEL: length192_eq_const:
+; X64-AVX1:       # %bb.0:
+; X64-AVX1-NEXT:    pushq %rax
+; X64-AVX1-NEXT:    movl $.L.str, %esi
+; X64-AVX1-NEXT:    movl $192, %edx
+; X64-AVX1-NEXT:    callq memcmp
+; X64-AVX1-NEXT:    testl %eax, %eax
+; X64-AVX1-NEXT:    sete %al
+; X64-AVX1-NEXT:    popq %rcx
+; X64-AVX1-NEXT:    retq
+;
+; X64-AVX2-LABEL: length192_eq_const:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    pushq %rax
+; X64-AVX2-NEXT:    movl $.L.str, %esi
+; X64-AVX2-NEXT:    movl $192, %edx
+; X64-AVX2-NEXT:    callq memcmp
+; X64-AVX2-NEXT:    testl %eax, %eax
+; X64-AVX2-NEXT:    sete %al
+; X64-AVX2-NEXT:    popq %rcx
+; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512BW-LABEL: length192_eq_const:
+; X64-AVX512BW:       # %bb.0:
+; X64-AVX512BW-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-AVX512BW-NEXT:    vmovdqu64 64(%rdi), %zmm1
+; X64-AVX512BW-NEXT:    vmovdqu64 128(%rdi), %zmm2
+; X64-AVX512BW-NEXT:    vpcmpneqb .L.str+{{.*}}(%rip), %zmm1, %k0
+; X64-AVX512BW-NEXT:    vpcmpneqb {{.*}}(%rip), %zmm0, %k1
+; X64-AVX512BW-NEXT:    korq %k0, %k1, %k0
+; X64-AVX512BW-NEXT:    vpcmpneqb .L.str+{{.*}}(%rip), %zmm2, %k1
+; X64-AVX512BW-NEXT:    kortestq %k1, %k0
+; X64-AVX512BW-NEXT:    sete %al
+; X64-AVX512BW-NEXT:    vzeroupper
+; X64-AVX512BW-NEXT:    retq
+;
+; X64-AVX512F-LABEL: length192_eq_const:
+; X64-AVX512F:       # %bb.0:
+; X64-AVX512F-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-AVX512F-NEXT:    vmovdqu64 64(%rdi), %zmm1
+; X64-AVX512F-NEXT:    vmovdqu64 128(%rdi), %zmm2
+; X64-AVX512F-NEXT:    vpcmpneqd .L.str+{{.*}}(%rip), %zmm1, %k0
+; X64-AVX512F-NEXT:    vpcmpneqd {{.*}}(%rip), %zmm0, %k1
+; X64-AVX512F-NEXT:    korw %k0, %k1, %k0
+; X64-AVX512F-NEXT:    vpcmpneqd .L.str+{{.*}}(%rip), %zmm2, %k1
+; X64-AVX512F-NEXT:    kortestw %k1, %k0
+; X64-AVX512F-NEXT:    sete %al
+; X64-AVX512F-NEXT:    vzeroupper
+; X64-AVX512F-NEXT:    retq
+;
+; X64-MIC-AVX2-LABEL: length192_eq_const:
+; X64-MIC-AVX2:       # %bb.0:
+; X64-MIC-AVX2-NEXT:    pushq %rax
+; X64-MIC-AVX2-NEXT:    movl $.L.str, %esi
+; X64-MIC-AVX2-NEXT:    movl $192, %edx
+; X64-MIC-AVX2-NEXT:    callq memcmp
+; X64-MIC-AVX2-NEXT:    testl %eax, %eax
+; X64-MIC-AVX2-NEXT:    sete %al
+; X64-MIC-AVX2-NEXT:    popq %rcx
+; X64-MIC-AVX2-NEXT:    retq
+;
+; X64-MIC-AVX512F-LABEL: length192_eq_const:
+; X64-MIC-AVX512F:       # %bb.0:
+; X64-MIC-AVX512F-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-MIC-AVX512F-NEXT:    vmovdqu64 64(%rdi), %zmm1
+; X64-MIC-AVX512F-NEXT:    vmovdqu64 128(%rdi), %zmm2
+; X64-MIC-AVX512F-NEXT:    vpcmpneqd .L.str+{{.*}}(%rip), %zmm1, %k0
+; X64-MIC-AVX512F-NEXT:    vpcmpneqd {{.*}}(%rip), %zmm0, %k1
+; X64-MIC-AVX512F-NEXT:    korw %k0, %k1, %k0
+; X64-MIC-AVX512F-NEXT:    vpcmpneqd .L.str+{{.*}}(%rip), %zmm2, %k1
+; X64-MIC-AVX512F-NEXT:    kortestw %k1, %k0
+; X64-MIC-AVX512F-NEXT:    sete %al
+; X64-MIC-AVX512F-NEXT:    vzeroupper
+; X64-MIC-AVX512F-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* getelementptr inbounds ([513 x i8], [513 x i8]* @.str, i32 0, i32 0), i64 192) nounwind
+  %c = icmp eq i32 %m, 0
+  ret i1 %c
+}
+
+define i32 @length255(i8* %X, i8* %Y) nounwind {
+; X86-LABEL: length255:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $255
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    retl
+;
+; X64-LABEL: length255:
+; X64:       # %bb.0:
+; X64-NEXT:    movl $255, %edx
+; X64-NEXT:    jmp memcmp # TAILCALL
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 255) nounwind
+  ret i32 %m
+}
+
+define i1 @length255_eq(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length255_eq:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $255
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    setne %al
+; X86-NEXT:    retl
+;
+; X64-SSE-LABEL: length255_eq:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    pushq %rax
+; X64-SSE-NEXT:    movl $255, %edx
+; X64-SSE-NEXT:    callq memcmp
+; X64-SSE-NEXT:    testl %eax, %eax
+; X64-SSE-NEXT:    setne %al
+; X64-SSE-NEXT:    popq %rcx
+; X64-SSE-NEXT:    retq
+;
+; X64-AVX1-LABEL: length255_eq:
+; X64-AVX1:       # %bb.0:
+; X64-AVX1-NEXT:    pushq %rax
+; X64-AVX1-NEXT:    movl $255, %edx
+; X64-AVX1-NEXT:    callq memcmp
+; X64-AVX1-NEXT:    testl %eax, %eax
+; X64-AVX1-NEXT:    setne %al
+; X64-AVX1-NEXT:    popq %rcx
+; X64-AVX1-NEXT:    retq
+;
+; X64-AVX2-LABEL: length255_eq:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    pushq %rax
+; X64-AVX2-NEXT:    movl $255, %edx
+; X64-AVX2-NEXT:    callq memcmp
+; X64-AVX2-NEXT:    testl %eax, %eax
+; X64-AVX2-NEXT:    setne %al
+; X64-AVX2-NEXT:    popq %rcx
+; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512BW-LABEL: length255_eq:
+; X64-AVX512BW:       # %bb.0:
+; X64-AVX512BW-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-AVX512BW-NEXT:    vmovdqu64 64(%rdi), %zmm1
+; X64-AVX512BW-NEXT:    vmovdqu64 128(%rdi), %zmm2
+; X64-AVX512BW-NEXT:    vmovdqu64 191(%rdi), %zmm3
+; X64-AVX512BW-NEXT:    vpcmpneqb 191(%rsi), %zmm3, %k0
+; X64-AVX512BW-NEXT:    vpcmpneqb 128(%rsi), %zmm2, %k1
+; X64-AVX512BW-NEXT:    korq %k0, %k1, %k0
+; X64-AVX512BW-NEXT:    vpcmpneqb 64(%rsi), %zmm1, %k1
+; X64-AVX512BW-NEXT:    vpcmpneqb (%rsi), %zmm0, %k2
+; X64-AVX512BW-NEXT:    korq %k1, %k2, %k1
+; X64-AVX512BW-NEXT:    kortestq %k0, %k1
+; X64-AVX512BW-NEXT:    setne %al
+; X64-AVX512BW-NEXT:    vzeroupper
+; X64-AVX512BW-NEXT:    retq
+;
+; X64-AVX512F-LABEL: length255_eq:
+; X64-AVX512F:       # %bb.0:
+; X64-AVX512F-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-AVX512F-NEXT:    vmovdqu64 64(%rdi), %zmm1
+; X64-AVX512F-NEXT:    vmovdqu64 128(%rdi), %zmm2
+; X64-AVX512F-NEXT:    vmovdqu64 191(%rdi), %zmm3
+; X64-AVX512F-NEXT:    vpcmpneqd 191(%rsi), %zmm3, %k0
+; X64-AVX512F-NEXT:    vpcmpneqd 128(%rsi), %zmm2, %k1
+; X64-AVX512F-NEXT:    korw %k0, %k1, %k0
+; X64-AVX512F-NEXT:    vpcmpneqd 64(%rsi), %zmm1, %k1
+; X64-AVX512F-NEXT:    vpcmpneqd (%rsi), %zmm0, %k2
+; X64-AVX512F-NEXT:    korw %k1, %k2, %k1
+; X64-AVX512F-NEXT:    kortestw %k0, %k1
+; X64-AVX512F-NEXT:    setne %al
+; X64-AVX512F-NEXT:    vzeroupper
+; X64-AVX512F-NEXT:    retq
+;
+; X64-MIC-AVX2-LABEL: length255_eq:
+; X64-MIC-AVX2:       # %bb.0:
+; X64-MIC-AVX2-NEXT:    pushq %rax
+; X64-MIC-AVX2-NEXT:    movl $255, %edx
+; X64-MIC-AVX2-NEXT:    callq memcmp
+; X64-MIC-AVX2-NEXT:    testl %eax, %eax
+; X64-MIC-AVX2-NEXT:    setne %al
+; X64-MIC-AVX2-NEXT:    popq %rcx
+; X64-MIC-AVX2-NEXT:    retq
+;
+; X64-MIC-AVX512F-LABEL: length255_eq:
+; X64-MIC-AVX512F:       # %bb.0:
+; X64-MIC-AVX512F-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-MIC-AVX512F-NEXT:    vmovdqu64 64(%rdi), %zmm1
+; X64-MIC-AVX512F-NEXT:    vmovdqu64 128(%rdi), %zmm2
+; X64-MIC-AVX512F-NEXT:    vmovdqu64 191(%rdi), %zmm3
+; X64-MIC-AVX512F-NEXT:    vpcmpneqd 191(%rsi), %zmm3, %k0
+; X64-MIC-AVX512F-NEXT:    vpcmpneqd 128(%rsi), %zmm2, %k1
+; X64-MIC-AVX512F-NEXT:    korw %k0, %k1, %k0
+; X64-MIC-AVX512F-NEXT:    vpcmpneqd 64(%rsi), %zmm1, %k1
+; X64-MIC-AVX512F-NEXT:    vpcmpneqd (%rsi), %zmm0, %k2
+; X64-MIC-AVX512F-NEXT:    korw %k1, %k2, %k1
+; X64-MIC-AVX512F-NEXT:    kortestw %k0, %k1
+; X64-MIC-AVX512F-NEXT:    setne %al
+; X64-MIC-AVX512F-NEXT:    vzeroupper
+; X64-MIC-AVX512F-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 255) nounwind
+  %cmp = icmp ne i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length255_lt(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length255_lt:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $255
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    shrl $31, %eax
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: length255_lt:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $255, %edx
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    shrl $31, %eax
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 255) nounwind
+  %cmp = icmp slt i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length255_gt(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length255_gt:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $255
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    setg %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length255_gt:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $255, %edx
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    setg %al
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 255) nounwind
+  %cmp = icmp sgt i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length255_eq_const(i8* %X) nounwind {
+; X86-LABEL: length255_eq_const:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $255
+; X86-NEXT:    pushl $.L.str
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    sete %al
+; X86-NEXT:    retl
+;
+; X64-SSE-LABEL: length255_eq_const:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    pushq %rax
+; X64-SSE-NEXT:    movl $.L.str, %esi
+; X64-SSE-NEXT:    movl $255, %edx
+; X64-SSE-NEXT:    callq memcmp
+; X64-SSE-NEXT:    testl %eax, %eax
+; X64-SSE-NEXT:    sete %al
+; X64-SSE-NEXT:    popq %rcx
+; X64-SSE-NEXT:    retq
+;
+; X64-AVX1-LABEL: length255_eq_const:
+; X64-AVX1:       # %bb.0:
+; X64-AVX1-NEXT:    pushq %rax
+; X64-AVX1-NEXT:    movl $.L.str, %esi
+; X64-AVX1-NEXT:    movl $255, %edx
+; X64-AVX1-NEXT:    callq memcmp
+; X64-AVX1-NEXT:    testl %eax, %eax
+; X64-AVX1-NEXT:    sete %al
+; X64-AVX1-NEXT:    popq %rcx
+; X64-AVX1-NEXT:    retq
+;
+; X64-AVX2-LABEL: length255_eq_const:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    pushq %rax
+; X64-AVX2-NEXT:    movl $.L.str, %esi
+; X64-AVX2-NEXT:    movl $255, %edx
+; X64-AVX2-NEXT:    callq memcmp
+; X64-AVX2-NEXT:    testl %eax, %eax
+; X64-AVX2-NEXT:    sete %al
+; X64-AVX2-NEXT:    popq %rcx
+; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512BW-LABEL: length255_eq_const:
+; X64-AVX512BW:       # %bb.0:
+; X64-AVX512BW-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-AVX512BW-NEXT:    vmovdqu64 64(%rdi), %zmm1
+; X64-AVX512BW-NEXT:    vmovdqu64 128(%rdi), %zmm2
+; X64-AVX512BW-NEXT:    vmovdqu64 191(%rdi), %zmm3
+; X64-AVX512BW-NEXT:    vpcmpneqb .L.str+{{.*}}(%rip), %zmm3, %k0
+; X64-AVX512BW-NEXT:    vpcmpneqb .L.str+{{.*}}(%rip), %zmm2, %k1
+; X64-AVX512BW-NEXT:    korq %k0, %k1, %k0
+; X64-AVX512BW-NEXT:    vpcmpneqb .L.str+{{.*}}(%rip), %zmm1, %k1
+; X64-AVX512BW-NEXT:    vpcmpneqb {{.*}}(%rip), %zmm0, %k2
+; X64-AVX512BW-NEXT:    korq %k1, %k2, %k1
+; X64-AVX512BW-NEXT:    kortestq %k0, %k1
+; X64-AVX512BW-NEXT:    sete %al
+; X64-AVX512BW-NEXT:    vzeroupper
+; X64-AVX512BW-NEXT:    retq
+;
+; X64-AVX512F-LABEL: length255_eq_const:
+; X64-AVX512F:       # %bb.0:
+; X64-AVX512F-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-AVX512F-NEXT:    vmovdqu64 64(%rdi), %zmm1
+; X64-AVX512F-NEXT:    vmovdqu64 128(%rdi), %zmm2
+; X64-AVX512F-NEXT:    vmovdqu64 191(%rdi), %zmm3
+; X64-AVX512F-NEXT:    vpcmpneqd .L.str+{{.*}}(%rip), %zmm3, %k0
+; X64-AVX512F-NEXT:    vpcmpneqd .L.str+{{.*}}(%rip), %zmm2, %k1
+; X64-AVX512F-NEXT:    korw %k0, %k1, %k0
+; X64-AVX512F-NEXT:    vpcmpneqd .L.str+{{.*}}(%rip), %zmm1, %k1
+; X64-AVX512F-NEXT:    vpcmpneqd {{.*}}(%rip), %zmm0, %k2
+; X64-AVX512F-NEXT:    korw %k1, %k2, %k1
+; X64-AVX512F-NEXT:    kortestw %k0, %k1
+; X64-AVX512F-NEXT:    sete %al
+; X64-AVX512F-NEXT:    vzeroupper
+; X64-AVX512F-NEXT:    retq
+;
+; X64-MIC-AVX2-LABEL: length255_eq_const:
+; X64-MIC-AVX2:       # %bb.0:
+; X64-MIC-AVX2-NEXT:    pushq %rax
+; X64-MIC-AVX2-NEXT:    movl $.L.str, %esi
+; X64-MIC-AVX2-NEXT:    movl $255, %edx
+; X64-MIC-AVX2-NEXT:    callq memcmp
+; X64-MIC-AVX2-NEXT:    testl %eax, %eax
+; X64-MIC-AVX2-NEXT:    sete %al
+; X64-MIC-AVX2-NEXT:    popq %rcx
+; X64-MIC-AVX2-NEXT:    retq
+;
+; X64-MIC-AVX512F-LABEL: length255_eq_const:
+; X64-MIC-AVX512F:       # %bb.0:
+; X64-MIC-AVX512F-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-MIC-AVX512F-NEXT:    vmovdqu64 64(%rdi), %zmm1
+; X64-MIC-AVX512F-NEXT:    vmovdqu64 128(%rdi), %zmm2
+; X64-MIC-AVX512F-NEXT:    vmovdqu64 191(%rdi), %zmm3
+; X64-MIC-AVX512F-NEXT:    vpcmpneqd .L.str+{{.*}}(%rip), %zmm3, %k0
+; X64-MIC-AVX512F-NEXT:    vpcmpneqd .L.str+{{.*}}(%rip), %zmm2, %k1
+; X64-MIC-AVX512F-NEXT:    korw %k0, %k1, %k0
+; X64-MIC-AVX512F-NEXT:    vpcmpneqd .L.str+{{.*}}(%rip), %zmm1, %k1
+; X64-MIC-AVX512F-NEXT:    vpcmpneqd {{.*}}(%rip), %zmm0, %k2
+; X64-MIC-AVX512F-NEXT:    korw %k1, %k2, %k1
+; X64-MIC-AVX512F-NEXT:    kortestw %k0, %k1
+; X64-MIC-AVX512F-NEXT:    sete %al
+; X64-MIC-AVX512F-NEXT:    vzeroupper
+; X64-MIC-AVX512F-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* getelementptr inbounds ([513 x i8], [513 x i8]* @.str, i32 0, i32 0), i64 255) nounwind
+  %c = icmp eq i32 %m, 0
+  ret i1 %c
+}
+
+define i32 @length256(i8* %X, i8* %Y) nounwind {
+; X86-LABEL: length256:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $256 # imm = 0x100
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    retl
+;
+; X64-LABEL: length256:
+; X64:       # %bb.0:
+; X64-NEXT:    movl $256, %edx # imm = 0x100
+; X64-NEXT:    jmp memcmp # TAILCALL
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 256) nounwind
+  ret i32 %m
+}
+
+define i1 @length256_eq(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length256_eq:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $256 # imm = 0x100
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    setne %al
+; X86-NEXT:    retl
+;
+; X64-SSE-LABEL: length256_eq:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    pushq %rax
+; X64-SSE-NEXT:    movl $256, %edx # imm = 0x100
+; X64-SSE-NEXT:    callq memcmp
+; X64-SSE-NEXT:    testl %eax, %eax
+; X64-SSE-NEXT:    setne %al
+; X64-SSE-NEXT:    popq %rcx
+; X64-SSE-NEXT:    retq
+;
+; X64-AVX1-LABEL: length256_eq:
+; X64-AVX1:       # %bb.0:
+; X64-AVX1-NEXT:    pushq %rax
+; X64-AVX1-NEXT:    movl $256, %edx # imm = 0x100
+; X64-AVX1-NEXT:    callq memcmp
+; X64-AVX1-NEXT:    testl %eax, %eax
+; X64-AVX1-NEXT:    setne %al
+; X64-AVX1-NEXT:    popq %rcx
+; X64-AVX1-NEXT:    retq
+;
+; X64-AVX2-LABEL: length256_eq:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    pushq %rax
+; X64-AVX2-NEXT:    movl $256, %edx # imm = 0x100
+; X64-AVX2-NEXT:    callq memcmp
+; X64-AVX2-NEXT:    testl %eax, %eax
+; X64-AVX2-NEXT:    setne %al
+; X64-AVX2-NEXT:    popq %rcx
+; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512BW-LABEL: length256_eq:
+; X64-AVX512BW:       # %bb.0:
+; X64-AVX512BW-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-AVX512BW-NEXT:    vmovdqu64 64(%rdi), %zmm1
+; X64-AVX512BW-NEXT:    vmovdqu64 128(%rdi), %zmm2
+; X64-AVX512BW-NEXT:    vmovdqu64 192(%rdi), %zmm3
+; X64-AVX512BW-NEXT:    vpcmpneqb 192(%rsi), %zmm3, %k0
+; X64-AVX512BW-NEXT:    vpcmpneqb 128(%rsi), %zmm2, %k1
+; X64-AVX512BW-NEXT:    korq %k0, %k1, %k0
+; X64-AVX512BW-NEXT:    vpcmpneqb 64(%rsi), %zmm1, %k1
+; X64-AVX512BW-NEXT:    vpcmpneqb (%rsi), %zmm0, %k2
+; X64-AVX512BW-NEXT:    korq %k1, %k2, %k1
+; X64-AVX512BW-NEXT:    kortestq %k0, %k1
+; X64-AVX512BW-NEXT:    setne %al
+; X64-AVX512BW-NEXT:    vzeroupper
+; X64-AVX512BW-NEXT:    retq
+;
+; X64-AVX512F-LABEL: length256_eq:
+; X64-AVX512F:       # %bb.0:
+; X64-AVX512F-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-AVX512F-NEXT:    vmovdqu64 64(%rdi), %zmm1
+; X64-AVX512F-NEXT:    vmovdqu64 128(%rdi), %zmm2
+; X64-AVX512F-NEXT:    vmovdqu64 192(%rdi), %zmm3
+; X64-AVX512F-NEXT:    vpcmpneqd 192(%rsi), %zmm3, %k0
+; X64-AVX512F-NEXT:    vpcmpneqd 128(%rsi), %zmm2, %k1
+; X64-AVX512F-NEXT:    korw %k0, %k1, %k0
+; X64-AVX512F-NEXT:    vpcmpneqd 64(%rsi), %zmm1, %k1
+; X64-AVX512F-NEXT:    vpcmpneqd (%rsi), %zmm0, %k2
+; X64-AVX512F-NEXT:    korw %k1, %k2, %k1
+; X64-AVX512F-NEXT:    kortestw %k0, %k1
+; X64-AVX512F-NEXT:    setne %al
+; X64-AVX512F-NEXT:    vzeroupper
+; X64-AVX512F-NEXT:    retq
+;
+; X64-MIC-AVX2-LABEL: length256_eq:
+; X64-MIC-AVX2:       # %bb.0:
+; X64-MIC-AVX2-NEXT:    pushq %rax
+; X64-MIC-AVX2-NEXT:    movl $256, %edx # imm = 0x100
+; X64-MIC-AVX2-NEXT:    callq memcmp
+; X64-MIC-AVX2-NEXT:    testl %eax, %eax
+; X64-MIC-AVX2-NEXT:    setne %al
+; X64-MIC-AVX2-NEXT:    popq %rcx
+; X64-MIC-AVX2-NEXT:    retq
+;
+; X64-MIC-AVX512F-LABEL: length256_eq:
+; X64-MIC-AVX512F:       # %bb.0:
+; X64-MIC-AVX512F-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-MIC-AVX512F-NEXT:    vmovdqu64 64(%rdi), %zmm1
+; X64-MIC-AVX512F-NEXT:    vmovdqu64 128(%rdi), %zmm2
+; X64-MIC-AVX512F-NEXT:    vmovdqu64 192(%rdi), %zmm3
+; X64-MIC-AVX512F-NEXT:    vpcmpneqd 192(%rsi), %zmm3, %k0
+; X64-MIC-AVX512F-NEXT:    vpcmpneqd 128(%rsi), %zmm2, %k1
+; X64-MIC-AVX512F-NEXT:    korw %k0, %k1, %k0
+; X64-MIC-AVX512F-NEXT:    vpcmpneqd 64(%rsi), %zmm1, %k1
+; X64-MIC-AVX512F-NEXT:    vpcmpneqd (%rsi), %zmm0, %k2
+; X64-MIC-AVX512F-NEXT:    korw %k1, %k2, %k1
+; X64-MIC-AVX512F-NEXT:    kortestw %k0, %k1
+; X64-MIC-AVX512F-NEXT:    setne %al
+; X64-MIC-AVX512F-NEXT:    vzeroupper
+; X64-MIC-AVX512F-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 256) nounwind
+  %cmp = icmp ne i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length256_lt(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length256_lt:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $256 # imm = 0x100
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    shrl $31, %eax
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: length256_lt:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $256, %edx # imm = 0x100
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    shrl $31, %eax
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 256) nounwind
+  %cmp = icmp slt i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length256_gt(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length256_gt:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $256 # imm = 0x100
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    setg %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length256_gt:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $256, %edx # imm = 0x100
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    setg %al
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 256) nounwind
+  %cmp = icmp sgt i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length256_eq_const(i8* %X) nounwind {
+; X86-LABEL: length256_eq_const:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $256 # imm = 0x100
+; X86-NEXT:    pushl $.L.str
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    sete %al
+; X86-NEXT:    retl
+;
+; X64-SSE-LABEL: length256_eq_const:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    pushq %rax
+; X64-SSE-NEXT:    movl $.L.str, %esi
+; X64-SSE-NEXT:    movl $256, %edx # imm = 0x100
+; X64-SSE-NEXT:    callq memcmp
+; X64-SSE-NEXT:    testl %eax, %eax
+; X64-SSE-NEXT:    sete %al
+; X64-SSE-NEXT:    popq %rcx
+; X64-SSE-NEXT:    retq
+;
+; X64-AVX1-LABEL: length256_eq_const:
+; X64-AVX1:       # %bb.0:
+; X64-AVX1-NEXT:    pushq %rax
+; X64-AVX1-NEXT:    movl $.L.str, %esi
+; X64-AVX1-NEXT:    movl $256, %edx # imm = 0x100
+; X64-AVX1-NEXT:    callq memcmp
+; X64-AVX1-NEXT:    testl %eax, %eax
+; X64-AVX1-NEXT:    sete %al
+; X64-AVX1-NEXT:    popq %rcx
+; X64-AVX1-NEXT:    retq
+;
+; X64-AVX2-LABEL: length256_eq_const:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    pushq %rax
+; X64-AVX2-NEXT:    movl $.L.str, %esi
+; X64-AVX2-NEXT:    movl $256, %edx # imm = 0x100
+; X64-AVX2-NEXT:    callq memcmp
+; X64-AVX2-NEXT:    testl %eax, %eax
+; X64-AVX2-NEXT:    sete %al
+; X64-AVX2-NEXT:    popq %rcx
+; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512BW-LABEL: length256_eq_const:
+; X64-AVX512BW:       # %bb.0:
+; X64-AVX512BW-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-AVX512BW-NEXT:    vmovdqu64 64(%rdi), %zmm1
+; X64-AVX512BW-NEXT:    vmovdqu64 128(%rdi), %zmm2
+; X64-AVX512BW-NEXT:    vmovdqu64 192(%rdi), %zmm3
+; X64-AVX512BW-NEXT:    vpcmpneqb .L.str+{{.*}}(%rip), %zmm3, %k0
+; X64-AVX512BW-NEXT:    vpcmpneqb .L.str+{{.*}}(%rip), %zmm2, %k1
+; X64-AVX512BW-NEXT:    korq %k0, %k1, %k0
+; X64-AVX512BW-NEXT:    vpcmpneqb .L.str+{{.*}}(%rip), %zmm1, %k1
+; X64-AVX512BW-NEXT:    vpcmpneqb {{.*}}(%rip), %zmm0, %k2
+; X64-AVX512BW-NEXT:    korq %k1, %k2, %k1
+; X64-AVX512BW-NEXT:    kortestq %k0, %k1
+; X64-AVX512BW-NEXT:    sete %al
+; X64-AVX512BW-NEXT:    vzeroupper
+; X64-AVX512BW-NEXT:    retq
+;
+; X64-AVX512F-LABEL: length256_eq_const:
+; X64-AVX512F:       # %bb.0:
+; X64-AVX512F-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-AVX512F-NEXT:    vmovdqu64 64(%rdi), %zmm1
+; X64-AVX512F-NEXT:    vmovdqu64 128(%rdi), %zmm2
+; X64-AVX512F-NEXT:    vmovdqu64 192(%rdi), %zmm3
+; X64-AVX512F-NEXT:    vpcmpneqd .L.str+{{.*}}(%rip), %zmm3, %k0
+; X64-AVX512F-NEXT:    vpcmpneqd .L.str+{{.*}}(%rip), %zmm2, %k1
+; X64-AVX512F-NEXT:    korw %k0, %k1, %k0
+; X64-AVX512F-NEXT:    vpcmpneqd .L.str+{{.*}}(%rip), %zmm1, %k1
+; X64-AVX512F-NEXT:    vpcmpneqd {{.*}}(%rip), %zmm0, %k2
+; X64-AVX512F-NEXT:    korw %k1, %k2, %k1
+; X64-AVX512F-NEXT:    kortestw %k0, %k1
+; X64-AVX512F-NEXT:    sete %al
+; X64-AVX512F-NEXT:    vzeroupper
+; X64-AVX512F-NEXT:    retq
+;
+; X64-MIC-AVX2-LABEL: length256_eq_const:
+; X64-MIC-AVX2:       # %bb.0:
+; X64-MIC-AVX2-NEXT:    pushq %rax
+; X64-MIC-AVX2-NEXT:    movl $.L.str, %esi
+; X64-MIC-AVX2-NEXT:    movl $256, %edx # imm = 0x100
+; X64-MIC-AVX2-NEXT:    callq memcmp
+; X64-MIC-AVX2-NEXT:    testl %eax, %eax
+; X64-MIC-AVX2-NEXT:    sete %al
+; X64-MIC-AVX2-NEXT:    popq %rcx
+; X64-MIC-AVX2-NEXT:    retq
+;
+; X64-MIC-AVX512F-LABEL: length256_eq_const:
+; X64-MIC-AVX512F:       # %bb.0:
+; X64-MIC-AVX512F-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-MIC-AVX512F-NEXT:    vmovdqu64 64(%rdi), %zmm1
+; X64-MIC-AVX512F-NEXT:    vmovdqu64 128(%rdi), %zmm2
+; X64-MIC-AVX512F-NEXT:    vmovdqu64 192(%rdi), %zmm3
+; X64-MIC-AVX512F-NEXT:    vpcmpneqd .L.str+{{.*}}(%rip), %zmm3, %k0
+; X64-MIC-AVX512F-NEXT:    vpcmpneqd .L.str+{{.*}}(%rip), %zmm2, %k1
+; X64-MIC-AVX512F-NEXT:    korw %k0, %k1, %k0
+; X64-MIC-AVX512F-NEXT:    vpcmpneqd .L.str+{{.*}}(%rip), %zmm1, %k1
+; X64-MIC-AVX512F-NEXT:    vpcmpneqd {{.*}}(%rip), %zmm0, %k2
+; X64-MIC-AVX512F-NEXT:    korw %k1, %k2, %k1
+; X64-MIC-AVX512F-NEXT:    kortestw %k0, %k1
+; X64-MIC-AVX512F-NEXT:    sete %al
+; X64-MIC-AVX512F-NEXT:    vzeroupper
+; X64-MIC-AVX512F-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* getelementptr inbounds ([513 x i8], [513 x i8]* @.str, i32 0, i32 0), i64 256) nounwind
+  %c = icmp eq i32 %m, 0
+  ret i1 %c
+}
+
+define i32 @length384(i8* %X, i8* %Y) nounwind {
+; X86-LABEL: length384:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $384 # imm = 0x180
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    retl
+;
+; X64-LABEL: length384:
+; X64:       # %bb.0:
+; X64-NEXT:    movl $384, %edx # imm = 0x180
+; X64-NEXT:    jmp memcmp # TAILCALL
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 384) nounwind
+  ret i32 %m
+}
+
+define i1 @length384_eq(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length384_eq:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $384 # imm = 0x180
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    setne %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length384_eq:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $384, %edx # imm = 0x180
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    setne %al
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 384) nounwind
+  %cmp = icmp ne i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length384_lt(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length384_lt:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $384 # imm = 0x180
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    shrl $31, %eax
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: length384_lt:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $384, %edx # imm = 0x180
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    shrl $31, %eax
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 384) nounwind
+  %cmp = icmp slt i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length384_gt(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length384_gt:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $384 # imm = 0x180
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    setg %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length384_gt:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $384, %edx # imm = 0x180
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    setg %al
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 384) nounwind
+  %cmp = icmp sgt i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length384_eq_const(i8* %X) nounwind {
+; X86-LABEL: length384_eq_const:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $384 # imm = 0x180
+; X86-NEXT:    pushl $.L.str
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    sete %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length384_eq_const:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $.L.str, %esi
+; X64-NEXT:    movl $384, %edx # imm = 0x180
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    sete %al
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* getelementptr inbounds ([513 x i8], [513 x i8]* @.str, i32 0, i32 0), i64 384) nounwind
+  %c = icmp eq i32 %m, 0
+  ret i1 %c
+}
+
+define i32 @length511(i8* %X, i8* %Y) nounwind {
+; X86-LABEL: length511:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $511 # imm = 0x1FF
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    retl
+;
+; X64-LABEL: length511:
+; X64:       # %bb.0:
+; X64-NEXT:    movl $511, %edx # imm = 0x1FF
+; X64-NEXT:    jmp memcmp # TAILCALL
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 511) nounwind
+  ret i32 %m
+}
+
+define i1 @length511_eq(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length511_eq:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $511 # imm = 0x1FF
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    setne %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length511_eq:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $511, %edx # imm = 0x1FF
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    setne %al
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 511) nounwind
+  %cmp = icmp ne i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length511_lt(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length511_lt:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $511 # imm = 0x1FF
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    shrl $31, %eax
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: length511_lt:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $511, %edx # imm = 0x1FF
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    shrl $31, %eax
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 511) nounwind
+  %cmp = icmp slt i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length511_gt(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length511_gt:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $511 # imm = 0x1FF
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    setg %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length511_gt:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $511, %edx # imm = 0x1FF
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    setg %al
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 511) nounwind
+  %cmp = icmp sgt i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length511_eq_const(i8* %X) nounwind {
+; X86-LABEL: length511_eq_const:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $511 # imm = 0x1FF
+; X86-NEXT:    pushl $.L.str
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    sete %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length511_eq_const:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $.L.str, %esi
+; X64-NEXT:    movl $511, %edx # imm = 0x1FF
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    sete %al
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* getelementptr inbounds ([513 x i8], [513 x i8]* @.str, i32 0, i32 0), i64 511) nounwind
+  %c = icmp eq i32 %m, 0
+  ret i1 %c
+}
+
+define i32 @length512(i8* %X, i8* %Y) nounwind {
+; X86-LABEL: length512:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $512 # imm = 0x200
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    retl
+;
+; X64-LABEL: length512:
+; X64:       # %bb.0:
+; X64-NEXT:    movl $512, %edx # imm = 0x200
+; X64-NEXT:    jmp memcmp # TAILCALL
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 512) nounwind
+  ret i32 %m
+}
+
+define i1 @length512_eq(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length512_eq:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $512 # imm = 0x200
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    setne %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length512_eq:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $512, %edx # imm = 0x200
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    setne %al
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 512) nounwind
+  %cmp = icmp ne i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length512_lt(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length512_lt:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $512 # imm = 0x200
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    shrl $31, %eax
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: length512_lt:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $512, %edx # imm = 0x200
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    shrl $31, %eax
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 512) nounwind
+  %cmp = icmp slt i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length512_gt(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length512_gt:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $512 # imm = 0x200
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    setg %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length512_gt:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $512, %edx # imm = 0x200
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    setg %al
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 512) nounwind
+  %cmp = icmp sgt i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length512_eq_const(i8* %X) nounwind {
+; X86-LABEL: length512_eq_const:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $512 # imm = 0x200
+; X86-NEXT:    pushl $.L.str
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    sete %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length512_eq_const:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $.L.str, %esi
+; X64-NEXT:    movl $512, %edx # imm = 0x200
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    sete %al
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* getelementptr inbounds ([513 x i8], [513 x i8]* @.str, i32 0, i32 0), i64 512) nounwind
+  %c = icmp eq i32 %m, 0
+  ret i1 %c
+}
+
+; This checks that we do not do stupid things with huge sizes.
+define i32 @huge_length(i8* %X, i8* %Y) nounwind {
+; X86-LABEL: huge_length:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $2147483647 # imm = 0x7FFFFFFF
+; X86-NEXT:    pushl $-1
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    retl
+;
+; X64-LABEL: huge_length:
+; X64:       # %bb.0:
+; X64-NEXT:    movabsq $9223372036854775807, %rdx # imm = 0x7FFFFFFFFFFFFFFF
+; X64-NEXT:    jmp memcmp # TAILCALL
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 9223372036854775807) nounwind
+  ret i32 %m
+}
+
+define i1 @huge_length_eq(i8* %X, i8* %Y) nounwind {
+; X86-LABEL: huge_length_eq:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $2147483647 # imm = 0x7FFFFFFF
+; X86-NEXT:    pushl $-1
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    sete %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: huge_length_eq:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movabsq $9223372036854775807, %rdx # imm = 0x7FFFFFFFFFFFFFFF
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    sete %al
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 9223372036854775807) nounwind
+  %c = icmp eq i32 %m, 0
+  ret i1 %c
+}
+
+; This checks non-constant sizes.
+define i32 @nonconst_length(i8* %X, i8* %Y, i64 %size) nounwind {
+; X86-LABEL: nonconst_length:
+; X86:       # %bb.0:
+; X86-NEXT:    jmp memcmp # TAILCALL
+;
+; X64-LABEL: nonconst_length:
+; X64:       # %bb.0:
+; X64-NEXT:    jmp memcmp # TAILCALL
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 %size) nounwind
+  ret i32 %m
+}
+
+define i1 @nonconst_length_eq(i8* %X, i8* %Y, i64 %size) nounwind {
+; X86-LABEL: nonconst_length_eq:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    sete %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: nonconst_length_eq:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    sete %al
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 %size) nounwind
+  %c = icmp eq i32 %m, 0
+  ret i1 %c
+}
diff --git a/llvm/test/CodeGen/X86/memcmp-optsize.ll b/llvm/test/CodeGen/X86/memcmp-optsize.ll
index b3bb3ab50..63bbb8f0e 100644
--- a/llvm/test/CodeGen/X86/memcmp-optsize.ll
+++ b/llvm/test/CodeGen/X86/memcmp-optsize.ll
@@ -2,7 +2,8 @@
 ; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=cmov | FileCheck %s --check-prefix=X86 --check-prefix=X86-NOSSE
 ; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=X86 --check-prefix=X86-SSE2
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s --check-prefix=X64 --check-prefix=X64-SSE2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx2 | FileCheck %s --check-prefix=X64 --check-prefix=X64-AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx | FileCheck %s --check-prefix=X64 --check-prefix=X64-AVX --check-prefix=X64-AVX1
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx2 | FileCheck %s --check-prefix=X64 --check-prefix=X64-AVX --check-prefix=X64-AVX2
 
 ; This tests codegen time inlining/optimization of memcmp
 ; rdar://6480398
@@ -560,14 +561,13 @@ define i1 @length16_eq(i8* %x, i8* %y) nounwind optsize {
 ; X64-SSE2-NEXT:    setne %al
 ; X64-SSE2-NEXT:    retq
 ;
-; X64-AVX2-LABEL: length16_eq:
-; X64-AVX2:       # %bb.0:
-; X64-AVX2-NEXT:    vmovdqu (%rdi), %xmm0
-; X64-AVX2-NEXT:    vpcmpeqb (%rsi), %xmm0, %xmm0
-; X64-AVX2-NEXT:    vpmovmskb %xmm0, %eax
-; X64-AVX2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
-; X64-AVX2-NEXT:    setne %al
-; X64-AVX2-NEXT:    retq
+; X64-AVX-LABEL: length16_eq:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    vmovdqu (%rdi), %xmm0
+; X64-AVX-NEXT:    vpxor (%rsi), %xmm0, %xmm0
+; X64-AVX-NEXT:    vptest %xmm0, %xmm0
+; X64-AVX-NEXT:    setne %al
+; X64-AVX-NEXT:    retq
   %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 16) nounwind
   %cmp = icmp ne i32 %call, 0
   ret i1 %cmp
@@ -605,14 +605,13 @@ define i1 @length16_eq_const(i8* %X) nounwind optsize {
 ; X64-SSE2-NEXT:    sete %al
 ; X64-SSE2-NEXT:    retq
 ;
-; X64-AVX2-LABEL: length16_eq_const:
-; X64-AVX2:       # %bb.0:
-; X64-AVX2-NEXT:    vmovdqu (%rdi), %xmm0
-; X64-AVX2-NEXT:    vpcmpeqb {{.*}}(%rip), %xmm0, %xmm0
-; X64-AVX2-NEXT:    vpmovmskb %xmm0, %eax
-; X64-AVX2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
-; X64-AVX2-NEXT:    sete %al
-; X64-AVX2-NEXT:    retq
+; X64-AVX-LABEL: length16_eq_const:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    vmovdqu (%rdi), %xmm0
+; X64-AVX-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; X64-AVX-NEXT:    vptest %xmm0, %xmm0
+; X64-AVX-NEXT:    sete %al
+; X64-AVX-NEXT:    retq
   %m = tail call i32 @memcmp(i8* %X, i8* getelementptr inbounds ([65 x i8], [65 x i8]* @.str, i32 0, i32 0), i64 16) nounwind
   %c = icmp eq i32 %m, 0
   ret i1 %c
@@ -682,18 +681,17 @@ define i1 @length24_eq(i8* %x, i8* %y) nounwind optsize {
 ; X64-SSE2-NEXT:    sete %al
 ; X64-SSE2-NEXT:    retq
 ;
-; X64-AVX2-LABEL: length24_eq:
-; X64-AVX2:       # %bb.0:
-; X64-AVX2-NEXT:    vmovdqu (%rdi), %xmm0
-; X64-AVX2-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
-; X64-AVX2-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero
-; X64-AVX2-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm1
-; X64-AVX2-NEXT:    vpcmpeqb (%rsi), %xmm0, %xmm0
-; X64-AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; X64-AVX2-NEXT:    vpmovmskb %xmm0, %eax
-; X64-AVX2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
-; X64-AVX2-NEXT:    sete %al
-; X64-AVX2-NEXT:    retq
+; X64-AVX-LABEL: length24_eq:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    vmovdqu (%rdi), %xmm0
+; X64-AVX-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
+; X64-AVX-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero
+; X64-AVX-NEXT:    vpxor %xmm2, %xmm1, %xmm1
+; X64-AVX-NEXT:    vpxor (%rsi), %xmm0, %xmm0
+; X64-AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; X64-AVX-NEXT:    vptest %xmm0, %xmm0
+; X64-AVX-NEXT:    sete %al
+; X64-AVX-NEXT:    retq
   %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 24) nounwind
   %cmp = icmp eq i32 %call, 0
   ret i1 %cmp
@@ -737,17 +735,16 @@ define i1 @length24_eq_const(i8* %X) nounwind optsize {
 ; X64-SSE2-NEXT:    setne %al
 ; X64-SSE2-NEXT:    retq
 ;
-; X64-AVX2-LABEL: length24_eq_const:
-; X64-AVX2:       # %bb.0:
-; X64-AVX2-NEXT:    vmovdqu (%rdi), %xmm0
-; X64-AVX2-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
-; X64-AVX2-NEXT:    vpcmpeqb {{.*}}(%rip), %xmm1, %xmm1
-; X64-AVX2-NEXT:    vpcmpeqb {{.*}}(%rip), %xmm0, %xmm0
-; X64-AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; X64-AVX2-NEXT:    vpmovmskb %xmm0, %eax
-; X64-AVX2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
-; X64-AVX2-NEXT:    setne %al
-; X64-AVX2-NEXT:    retq
+; X64-AVX-LABEL: length24_eq_const:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    vmovdqu (%rdi), %xmm0
+; X64-AVX-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
+; X64-AVX-NEXT:    vpxor {{.*}}(%rip), %xmm1, %xmm1
+; X64-AVX-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; X64-AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; X64-AVX-NEXT:    vptest %xmm0, %xmm0
+; X64-AVX-NEXT:    setne %al
+; X64-AVX-NEXT:    retq
   %m = tail call i32 @memcmp(i8* %X, i8* getelementptr inbounds ([65 x i8], [65 x i8]* @.str, i32 0, i32 0), i64 24) nounwind
   %c = icmp ne i32 %m, 0
   ret i1 %c
@@ -817,12 +814,20 @@ define i1 @length32_eq(i8* %x, i8* %y) nounwind optsize {
 ; X64-SSE2-NEXT:    sete %al
 ; X64-SSE2-NEXT:    retq
 ;
+; X64-AVX1-LABEL: length32_eq:
+; X64-AVX1:       # %bb.0:
+; X64-AVX1-NEXT:    vmovups (%rdi), %ymm0
+; X64-AVX1-NEXT:    vxorps (%rsi), %ymm0, %ymm0
+; X64-AVX1-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX1-NEXT:    sete %al
+; X64-AVX1-NEXT:    vzeroupper
+; X64-AVX1-NEXT:    retq
+;
 ; X64-AVX2-LABEL: length32_eq:
 ; X64-AVX2:       # %bb.0:
 ; X64-AVX2-NEXT:    vmovdqu (%rdi), %ymm0
-; X64-AVX2-NEXT:    vpcmpeqb (%rsi), %ymm0, %ymm0
-; X64-AVX2-NEXT:    vpmovmskb %ymm0, %eax
-; X64-AVX2-NEXT:    cmpl $-1, %eax
+; X64-AVX2-NEXT:    vpxor (%rsi), %ymm0, %ymm0
+; X64-AVX2-NEXT:    vptest %ymm0, %ymm0
 ; X64-AVX2-NEXT:    sete %al
 ; X64-AVX2-NEXT:    vzeroupper
 ; X64-AVX2-NEXT:    retq
@@ -869,12 +874,20 @@ define i1 @length32_eq_const(i8* %X) nounwind optsize {
 ; X64-SSE2-NEXT:    setne %al
 ; X64-SSE2-NEXT:    retq
 ;
+; X64-AVX1-LABEL: length32_eq_const:
+; X64-AVX1:       # %bb.0:
+; X64-AVX1-NEXT:    vmovups (%rdi), %ymm0
+; X64-AVX1-NEXT:    vxorps {{.*}}(%rip), %ymm0, %ymm0
+; X64-AVX1-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX1-NEXT:    setne %al
+; X64-AVX1-NEXT:    vzeroupper
+; X64-AVX1-NEXT:    retq
+;
 ; X64-AVX2-LABEL: length32_eq_const:
 ; X64-AVX2:       # %bb.0:
 ; X64-AVX2-NEXT:    vmovdqu (%rdi), %ymm0
-; X64-AVX2-NEXT:    vpcmpeqb {{.*}}(%rip), %ymm0, %ymm0
-; X64-AVX2-NEXT:    vpmovmskb %ymm0, %eax
-; X64-AVX2-NEXT:    cmpl $-1, %eax
+; X64-AVX2-NEXT:    vpxor {{.*}}(%rip), %ymm0, %ymm0
+; X64-AVX2-NEXT:    vptest %ymm0, %ymm0
 ; X64-AVX2-NEXT:    setne %al
 ; X64-AVX2-NEXT:    vzeroupper
 ; X64-AVX2-NEXT:    retq
@@ -925,15 +938,26 @@ define i1 @length64_eq(i8* %x, i8* %y) nounwind optsize {
 ; X64-SSE2-NEXT:    popq %rcx
 ; X64-SSE2-NEXT:    retq
 ;
+; X64-AVX1-LABEL: length64_eq:
+; X64-AVX1:       # %bb.0:
+; X64-AVX1-NEXT:    vmovups (%rdi), %ymm0
+; X64-AVX1-NEXT:    vmovups 32(%rdi), %ymm1
+; X64-AVX1-NEXT:    vxorps 32(%rsi), %ymm1, %ymm1
+; X64-AVX1-NEXT:    vxorps (%rsi), %ymm0, %ymm0
+; X64-AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; X64-AVX1-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX1-NEXT:    setne %al
+; X64-AVX1-NEXT:    vzeroupper
+; X64-AVX1-NEXT:    retq
+;
 ; X64-AVX2-LABEL: length64_eq:
 ; X64-AVX2:       # %bb.0:
 ; X64-AVX2-NEXT:    vmovdqu (%rdi), %ymm0
 ; X64-AVX2-NEXT:    vmovdqu 32(%rdi), %ymm1
-; X64-AVX2-NEXT:    vpcmpeqb 32(%rsi), %ymm1, %ymm1
-; X64-AVX2-NEXT:    vpcmpeqb (%rsi), %ymm0, %ymm0
-; X64-AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
-; X64-AVX2-NEXT:    vpmovmskb %ymm0, %eax
-; X64-AVX2-NEXT:    cmpl $-1, %eax
+; X64-AVX2-NEXT:    vpxor 32(%rsi), %ymm1, %ymm1
+; X64-AVX2-NEXT:    vpxor (%rsi), %ymm0, %ymm0
+; X64-AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; X64-AVX2-NEXT:    vptest %ymm0, %ymm0
 ; X64-AVX2-NEXT:    setne %al
 ; X64-AVX2-NEXT:    vzeroupper
 ; X64-AVX2-NEXT:    retq
@@ -966,15 +990,26 @@ define i1 @length64_eq_const(i8* %X) nounwind optsize {
 ; X64-SSE2-NEXT:    popq %rcx
 ; X64-SSE2-NEXT:    retq
 ;
+; X64-AVX1-LABEL: length64_eq_const:
+; X64-AVX1:       # %bb.0:
+; X64-AVX1-NEXT:    vmovups (%rdi), %ymm0
+; X64-AVX1-NEXT:    vmovups 32(%rdi), %ymm1
+; X64-AVX1-NEXT:    vxorps {{.*}}(%rip), %ymm1, %ymm1
+; X64-AVX1-NEXT:    vxorps {{.*}}(%rip), %ymm0, %ymm0
+; X64-AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; X64-AVX1-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX1-NEXT:    sete %al
+; X64-AVX1-NEXT:    vzeroupper
+; X64-AVX1-NEXT:    retq
+;
 ; X64-AVX2-LABEL: length64_eq_const:
 ; X64-AVX2:       # %bb.0:
 ; X64-AVX2-NEXT:    vmovdqu (%rdi), %ymm0
 ; X64-AVX2-NEXT:    vmovdqu 32(%rdi), %ymm1
-; X64-AVX2-NEXT:    vpcmpeqb {{.*}}(%rip), %ymm1, %ymm1
-; X64-AVX2-NEXT:    vpcmpeqb {{.*}}(%rip), %ymm0, %ymm0
-; X64-AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
-; X64-AVX2-NEXT:    vpmovmskb %ymm0, %eax
-; X64-AVX2-NEXT:    cmpl $-1, %eax
+; X64-AVX2-NEXT:    vpxor {{.*}}(%rip), %ymm1, %ymm1
+; X64-AVX2-NEXT:    vpxor {{.*}}(%rip), %ymm0, %ymm0
+; X64-AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; X64-AVX2-NEXT:    vptest %ymm0, %ymm0
 ; X64-AVX2-NEXT:    sete %al
 ; X64-AVX2-NEXT:    vzeroupper
 ; X64-AVX2-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/memcmp-pgso.ll b/llvm/test/CodeGen/X86/memcmp-pgso.ll
new file mode 100644
index 000000000..c3c3a1db8
--- /dev/null
+++ b/llvm/test/CodeGen/X86/memcmp-pgso.ll
@@ -0,0 +1,1064 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=cmov | FileCheck %s --check-prefix=X86 --check-prefix=X86-NOSSE
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=X86 --check-prefix=X86-SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s --check-prefix=X64 --check-prefix=X64-SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx | FileCheck %s --check-prefix=X64 --check-prefix=X64-AVX --check-prefix=X64-AVX1
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx2 | FileCheck %s --check-prefix=X64 --check-prefix=X64-AVX --check-prefix=X64-AVX2
+
+; This tests codegen time inlining/optimization of memcmp
+; rdar://6480398
+
+@.str = private constant [65 x i8] c"0123456789012345678901234567890123456789012345678901234567890123\00", align 1
+
+declare i32 @memcmp(i8*, i8*, i64)
+declare i32 @bcmp(i8*, i8*, i64)
+
+define i32 @length2(i8* %X, i8* %Y) nounwind !prof !14 {
+; X86-LABEL: length2:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movzwl (%ecx), %ecx
+; X86-NEXT:    movzwl (%eax), %edx
+; X86-NEXT:    rolw $8, %cx
+; X86-NEXT:    rolw $8, %dx
+; X86-NEXT:    movzwl %cx, %eax
+; X86-NEXT:    movzwl %dx, %ecx
+; X86-NEXT:    subl %ecx, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: length2:
+; X64:       # %bb.0:
+; X64-NEXT:    movzwl (%rdi), %eax
+; X64-NEXT:    movzwl (%rsi), %ecx
+; X64-NEXT:    rolw $8, %ax
+; X64-NEXT:    rolw $8, %cx
+; X64-NEXT:    movzwl %ax, %eax
+; X64-NEXT:    movzwl %cx, %ecx
+; X64-NEXT:    subl %ecx, %eax
+; X64-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 2) nounwind
+  ret i32 %m
+}
+
+define i1 @length2_eq(i8* %X, i8* %Y) nounwind !prof !14 {
+; X86-LABEL: length2_eq:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movzwl (%ecx), %ecx
+; X86-NEXT:    cmpw (%eax), %cx
+; X86-NEXT:    sete %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length2_eq:
+; X64:       # %bb.0:
+; X64-NEXT:    movzwl (%rdi), %eax
+; X64-NEXT:    cmpw (%rsi), %ax
+; X64-NEXT:    sete %al
+; X64-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 2) nounwind
+  %c = icmp eq i32 %m, 0
+  ret i1 %c
+}
+
+define i1 @length2_eq_const(i8* %X) nounwind !prof !14 {
+; X86-LABEL: length2_eq_const:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movzwl (%eax), %eax
+; X86-NEXT:    cmpl $12849, %eax # imm = 0x3231
+; X86-NEXT:    setne %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length2_eq_const:
+; X64:       # %bb.0:
+; X64-NEXT:    movzwl (%rdi), %eax
+; X64-NEXT:    cmpl $12849, %eax # imm = 0x3231
+; X64-NEXT:    setne %al
+; X64-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* getelementptr inbounds ([65 x i8], [65 x i8]* @.str, i32 0, i32 1), i64 2) nounwind
+  %c = icmp ne i32 %m, 0
+  ret i1 %c
+}
+
+define i1 @length2_eq_nobuiltin_attr(i8* %X, i8* %Y) nounwind !prof !14 {
+; X86-LABEL: length2_eq_nobuiltin_attr:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $2
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    sete %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length2_eq_nobuiltin_attr:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $2, %edx
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    sete %al
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 2) nounwind nobuiltin
+  %c = icmp eq i32 %m, 0
+  ret i1 %c
+}
+
+define i32 @length3(i8* %X, i8* %Y) nounwind !prof !14 {
+; X86-LABEL: length3:
+; X86:       # %bb.0: # %loadbb
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movzwl (%eax), %edx
+; X86-NEXT:    movzwl (%ecx), %esi
+; X86-NEXT:    rolw $8, %dx
+; X86-NEXT:    rolw $8, %si
+; X86-NEXT:    cmpw %si, %dx
+; X86-NEXT:    jne .LBB4_1
+; X86-NEXT:  # %bb.2: # %loadbb1
+; X86-NEXT:    movzbl 2(%eax), %eax
+; X86-NEXT:    movzbl 2(%ecx), %ecx
+; X86-NEXT:    subl %ecx, %eax
+; X86-NEXT:    jmp .LBB4_3
+; X86-NEXT:  .LBB4_1: # %res_block
+; X86-NEXT:    setae %al
+; X86-NEXT:    movzbl %al, %eax
+; X86-NEXT:    leal -1(%eax,%eax), %eax
+; X86-NEXT:  .LBB4_3: # %endblock
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
+;
+; X64-LABEL: length3:
+; X64:       # %bb.0: # %loadbb
+; X64-NEXT:    movzwl (%rdi), %eax
+; X64-NEXT:    movzwl (%rsi), %ecx
+; X64-NEXT:    rolw $8, %ax
+; X64-NEXT:    rolw $8, %cx
+; X64-NEXT:    cmpw %cx, %ax
+; X64-NEXT:    jne .LBB4_1
+; X64-NEXT:  # %bb.2: # %loadbb1
+; X64-NEXT:    movzbl 2(%rdi), %eax
+; X64-NEXT:    movzbl 2(%rsi), %ecx
+; X64-NEXT:    subl %ecx, %eax
+; X64-NEXT:    retq
+; X64-NEXT:  .LBB4_1: # %res_block
+; X64-NEXT:    setae %al
+; X64-NEXT:    movzbl %al, %eax
+; X64-NEXT:    leal -1(%rax,%rax), %eax
+; X64-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 3) nounwind
+  ret i32 %m
+}
+
+define i1 @length3_eq(i8* %X, i8* %Y) nounwind !prof !14 {
+; X86-LABEL: length3_eq:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movzwl (%ecx), %edx
+; X86-NEXT:    xorw (%eax), %dx
+; X86-NEXT:    movb 2(%ecx), %cl
+; X86-NEXT:    xorb 2(%eax), %cl
+; X86-NEXT:    movzbl %cl, %eax
+; X86-NEXT:    orw %dx, %ax
+; X86-NEXT:    setne %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length3_eq:
+; X64:       # %bb.0:
+; X64-NEXT:    movzwl (%rdi), %eax
+; X64-NEXT:    xorw (%rsi), %ax
+; X64-NEXT:    movb 2(%rdi), %cl
+; X64-NEXT:    xorb 2(%rsi), %cl
+; X64-NEXT:    movzbl %cl, %ecx
+; X64-NEXT:    orw %ax, %cx
+; X64-NEXT:    setne %al
+; X64-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 3) nounwind
+  %c = icmp ne i32 %m, 0
+  ret i1 %c
+}
+
+define i32 @length4(i8* %X, i8* %Y) nounwind !prof !14 {
+; X86-LABEL: length4:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl (%ecx), %ecx
+; X86-NEXT:    movl (%eax), %edx
+; X86-NEXT:    bswapl %ecx
+; X86-NEXT:    bswapl %edx
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    cmpl %edx, %ecx
+; X86-NEXT:    seta %al
+; X86-NEXT:    sbbl $0, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: length4:
+; X64:       # %bb.0:
+; X64-NEXT:    movl (%rdi), %ecx
+; X64-NEXT:    movl (%rsi), %edx
+; X64-NEXT:    bswapl %ecx
+; X64-NEXT:    bswapl %edx
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpl %edx, %ecx
+; X64-NEXT:    seta %al
+; X64-NEXT:    sbbl $0, %eax
+; X64-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 4) nounwind
+  ret i32 %m
+}
+
+define i1 @length4_eq(i8* %X, i8* %Y) nounwind !prof !14 {
+; X86-LABEL: length4_eq:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl (%ecx), %ecx
+; X86-NEXT:    cmpl (%eax), %ecx
+; X86-NEXT:    setne %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length4_eq:
+; X64:       # %bb.0:
+; X64-NEXT:    movl (%rdi), %eax
+; X64-NEXT:    cmpl (%rsi), %eax
+; X64-NEXT:    setne %al
+; X64-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 4) nounwind
+  %c = icmp ne i32 %m, 0
+  ret i1 %c
+}
+
+define i1 @length4_eq_const(i8* %X) nounwind !prof !14 {
+; X86-LABEL: length4_eq_const:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpl $875770417, (%eax) # imm = 0x34333231
+; X86-NEXT:    sete %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length4_eq_const:
+; X64:       # %bb.0:
+; X64-NEXT:    cmpl $875770417, (%rdi) # imm = 0x34333231
+; X64-NEXT:    sete %al
+; X64-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* getelementptr inbounds ([65 x i8], [65 x i8]* @.str, i32 0, i32 1), i64 4) nounwind
+  %c = icmp eq i32 %m, 0
+  ret i1 %c
+}
+
+define i32 @length5(i8* %X, i8* %Y) nounwind !prof !14 {
+; X86-LABEL: length5:
+; X86:       # %bb.0: # %loadbb
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl (%eax), %edx
+; X86-NEXT:    movl (%ecx), %esi
+; X86-NEXT:    bswapl %edx
+; X86-NEXT:    bswapl %esi
+; X86-NEXT:    cmpl %esi, %edx
+; X86-NEXT:    jne .LBB9_1
+; X86-NEXT:  # %bb.2: # %loadbb1
+; X86-NEXT:    movzbl 4(%eax), %eax
+; X86-NEXT:    movzbl 4(%ecx), %ecx
+; X86-NEXT:    subl %ecx, %eax
+; X86-NEXT:    jmp .LBB9_3
+; X86-NEXT:  .LBB9_1: # %res_block
+; X86-NEXT:    setae %al
+; X86-NEXT:    movzbl %al, %eax
+; X86-NEXT:    leal -1(%eax,%eax), %eax
+; X86-NEXT:  .LBB9_3: # %endblock
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
+;
+; X64-LABEL: length5:
+; X64:       # %bb.0: # %loadbb
+; X64-NEXT:    movl (%rdi), %eax
+; X64-NEXT:    movl (%rsi), %ecx
+; X64-NEXT:    bswapl %eax
+; X64-NEXT:    bswapl %ecx
+; X64-NEXT:    cmpl %ecx, %eax
+; X64-NEXT:    jne .LBB9_1
+; X64-NEXT:  # %bb.2: # %loadbb1
+; X64-NEXT:    movzbl 4(%rdi), %eax
+; X64-NEXT:    movzbl 4(%rsi), %ecx
+; X64-NEXT:    subl %ecx, %eax
+; X64-NEXT:    retq
+; X64-NEXT:  .LBB9_1: # %res_block
+; X64-NEXT:    setae %al
+; X64-NEXT:    movzbl %al, %eax
+; X64-NEXT:    leal -1(%rax,%rax), %eax
+; X64-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 5) nounwind
+  ret i32 %m
+}
+
+define i1 @length5_eq(i8* %X, i8* %Y) nounwind !prof !14 {
+; X86-LABEL: length5_eq:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl (%ecx), %edx
+; X86-NEXT:    xorl (%eax), %edx
+; X86-NEXT:    movb 4(%ecx), %cl
+; X86-NEXT:    xorb 4(%eax), %cl
+; X86-NEXT:    movzbl %cl, %eax
+; X86-NEXT:    orl %edx, %eax
+; X86-NEXT:    setne %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length5_eq:
+; X64:       # %bb.0:
+; X64-NEXT:    movl (%rdi), %eax
+; X64-NEXT:    xorl (%rsi), %eax
+; X64-NEXT:    movb 4(%rdi), %cl
+; X64-NEXT:    xorb 4(%rsi), %cl
+; X64-NEXT:    movzbl %cl, %ecx
+; X64-NEXT:    orl %eax, %ecx
+; X64-NEXT:    setne %al
+; X64-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 5) nounwind
+  %c = icmp ne i32 %m, 0
+  ret i1 %c
+}
+
+define i32 @length8(i8* %X, i8* %Y) nounwind !prof !14 {
+; X86-LABEL: length8:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl (%esi), %ecx
+; X86-NEXT:    movl (%eax), %edx
+; X86-NEXT:    bswapl %ecx
+; X86-NEXT:    bswapl %edx
+; X86-NEXT:    cmpl %edx, %ecx
+; X86-NEXT:    jne .LBB11_2
+; X86-NEXT:  # %bb.1: # %loadbb1
+; X86-NEXT:    movl 4(%esi), %ecx
+; X86-NEXT:    movl 4(%eax), %edx
+; X86-NEXT:    bswapl %ecx
+; X86-NEXT:    bswapl %edx
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    cmpl %edx, %ecx
+; X86-NEXT:    je .LBB11_3
+; X86-NEXT:  .LBB11_2: # %res_block
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    cmpl %edx, %ecx
+; X86-NEXT:    setae %al
+; X86-NEXT:    leal -1(%eax,%eax), %eax
+; X86-NEXT:  .LBB11_3: # %endblock
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
+;
+; X64-LABEL: length8:
+; X64:       # %bb.0:
+; X64-NEXT:    movq (%rdi), %rcx
+; X64-NEXT:    movq (%rsi), %rdx
+; X64-NEXT:    bswapq %rcx
+; X64-NEXT:    bswapq %rdx
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpq %rdx, %rcx
+; X64-NEXT:    seta %al
+; X64-NEXT:    sbbl $0, %eax
+; X64-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 8) nounwind
+  ret i32 %m
+}
+
+define i1 @length8_eq(i8* %X, i8* %Y) nounwind !prof !14 {
+; X86-LABEL: length8_eq:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl (%ecx), %edx
+; X86-NEXT:    movl 4(%ecx), %ecx
+; X86-NEXT:    xorl (%eax), %edx
+; X86-NEXT:    xorl 4(%eax), %ecx
+; X86-NEXT:    orl %edx, %ecx
+; X86-NEXT:    sete %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length8_eq:
+; X64:       # %bb.0:
+; X64-NEXT:    movq (%rdi), %rax
+; X64-NEXT:    cmpq (%rsi), %rax
+; X64-NEXT:    sete %al
+; X64-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 8) nounwind
+  %c = icmp eq i32 %m, 0
+  ret i1 %c
+}
+
+define i1 @length8_eq_const(i8* %X) nounwind !prof !14 {
+; X86-LABEL: length8_eq_const:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl $858927408, %ecx # imm = 0x33323130
+; X86-NEXT:    xorl (%eax), %ecx
+; X86-NEXT:    movl $926299444, %edx # imm = 0x37363534
+; X86-NEXT:    xorl 4(%eax), %edx
+; X86-NEXT:    orl %ecx, %edx
+; X86-NEXT:    setne %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length8_eq_const:
+; X64:       # %bb.0:
+; X64-NEXT:    movabsq $3978425819141910832, %rax # imm = 0x3736353433323130
+; X64-NEXT:    cmpq %rax, (%rdi)
+; X64-NEXT:    setne %al
+; X64-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* getelementptr inbounds ([65 x i8], [65 x i8]* @.str, i32 0, i32 0), i64 8) nounwind
+  %c = icmp ne i32 %m, 0
+  ret i1 %c
+}
+
+define i1 @length12_eq(i8* %X, i8* %Y) nounwind !prof !14 {
+; X86-LABEL: length12_eq:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $12
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    setne %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length12_eq:
+; X64:       # %bb.0:
+; X64-NEXT:    movq (%rdi), %rax
+; X64-NEXT:    xorq (%rsi), %rax
+; X64-NEXT:    movl 8(%rdi), %ecx
+; X64-NEXT:    xorl 8(%rsi), %ecx
+; X64-NEXT:    orq %rax, %rcx
+; X64-NEXT:    setne %al
+; X64-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 12) nounwind
+  %c = icmp ne i32 %m, 0
+  ret i1 %c
+}
+
+define i32 @length12(i8* %X, i8* %Y) nounwind !prof !14 {
+; X86-LABEL: length12:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $12
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    retl
+;
+; X64-LABEL: length12:
+; X64:       # %bb.0:
+; X64-NEXT:    movq (%rdi), %rcx
+; X64-NEXT:    movq (%rsi), %rdx
+; X64-NEXT:    bswapq %rcx
+; X64-NEXT:    bswapq %rdx
+; X64-NEXT:    cmpq %rdx, %rcx
+; X64-NEXT:    jne .LBB15_2
+; X64-NEXT:  # %bb.1: # %loadbb1
+; X64-NEXT:    movl 8(%rdi), %ecx
+; X64-NEXT:    movl 8(%rsi), %edx
+; X64-NEXT:    bswapl %ecx
+; X64-NEXT:    bswapl %edx
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpq %rdx, %rcx
+; X64-NEXT:    je .LBB15_3
+; X64-NEXT:  .LBB15_2: # %res_block
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpq %rdx, %rcx
+; X64-NEXT:    setae %al
+; X64-NEXT:    leal -1(%rax,%rax), %eax
+; X64-NEXT:  .LBB15_3: # %endblock
+; X64-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 12) nounwind
+  ret i32 %m
+}
+
+; PR33329 - https://bugs.llvm.org/show_bug.cgi?id=33329
+
+define i32 @length16(i8* %X, i8* %Y) nounwind !prof !14 {
+; X86-LABEL: length16:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $16
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    retl
+;
+; X64-LABEL: length16:
+; X64:       # %bb.0:
+; X64-NEXT:    movq (%rdi), %rcx
+; X64-NEXT:    movq (%rsi), %rdx
+; X64-NEXT:    bswapq %rcx
+; X64-NEXT:    bswapq %rdx
+; X64-NEXT:    cmpq %rdx, %rcx
+; X64-NEXT:    jne .LBB16_2
+; X64-NEXT:  # %bb.1: # %loadbb1
+; X64-NEXT:    movq 8(%rdi), %rcx
+; X64-NEXT:    movq 8(%rsi), %rdx
+; X64-NEXT:    bswapq %rcx
+; X64-NEXT:    bswapq %rdx
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpq %rdx, %rcx
+; X64-NEXT:    je .LBB16_3
+; X64-NEXT:  .LBB16_2: # %res_block
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpq %rdx, %rcx
+; X64-NEXT:    setae %al
+; X64-NEXT:    leal -1(%rax,%rax), %eax
+; X64-NEXT:  .LBB16_3: # %endblock
+; X64-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 16) nounwind
+  ret i32 %m
+}
+
+define i1 @length16_eq(i8* %x, i8* %y) nounwind !prof !14 {
+; X86-NOSSE-LABEL: length16_eq:
+; X86-NOSSE:       # %bb.0:
+; X86-NOSSE-NEXT:    pushl $0
+; X86-NOSSE-NEXT:    pushl $16
+; X86-NOSSE-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    calll memcmp
+; X86-NOSSE-NEXT:    addl $16, %esp
+; X86-NOSSE-NEXT:    testl %eax, %eax
+; X86-NOSSE-NEXT:    setne %al
+; X86-NOSSE-NEXT:    retl
+;
+; X86-SSE2-LABEL: length16_eq:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE2-NEXT:    movdqu (%ecx), %xmm0
+; X86-SSE2-NEXT:    movdqu (%eax), %xmm1
+; X86-SSE2-NEXT:    pcmpeqb %xmm0, %xmm1
+; X86-SSE2-NEXT:    pmovmskb %xmm1, %eax
+; X86-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; X86-SSE2-NEXT:    setne %al
+; X86-SSE2-NEXT:    retl
+;
+; X64-SSE2-LABEL: length16_eq:
+; X64-SSE2:       # %bb.0:
+; X64-SSE2-NEXT:    movdqu (%rdi), %xmm0
+; X64-SSE2-NEXT:    movdqu (%rsi), %xmm1
+; X64-SSE2-NEXT:    pcmpeqb %xmm0, %xmm1
+; X64-SSE2-NEXT:    pmovmskb %xmm1, %eax
+; X64-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; X64-SSE2-NEXT:    setne %al
+; X64-SSE2-NEXT:    retq
+;
+; X64-AVX-LABEL: length16_eq:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    vmovdqu (%rdi), %xmm0
+; X64-AVX-NEXT:    vpxor (%rsi), %xmm0, %xmm0
+; X64-AVX-NEXT:    vptest %xmm0, %xmm0
+; X64-AVX-NEXT:    setne %al
+; X64-AVX-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 16) nounwind
+  %cmp = icmp ne i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length16_eq_const(i8* %X) nounwind !prof !14 {
+; X86-NOSSE-LABEL: length16_eq_const:
+; X86-NOSSE:       # %bb.0:
+; X86-NOSSE-NEXT:    pushl $0
+; X86-NOSSE-NEXT:    pushl $16
+; X86-NOSSE-NEXT:    pushl $.L.str
+; X86-NOSSE-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    calll memcmp
+; X86-NOSSE-NEXT:    addl $16, %esp
+; X86-NOSSE-NEXT:    testl %eax, %eax
+; X86-NOSSE-NEXT:    sete %al
+; X86-NOSSE-NEXT:    retl
+;
+; X86-SSE2-LABEL: length16_eq_const:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT:    movdqu (%eax), %xmm0
+; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pmovmskb %xmm0, %eax
+; X86-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; X86-SSE2-NEXT:    sete %al
+; X86-SSE2-NEXT:    retl
+;
+; X64-SSE2-LABEL: length16_eq_const:
+; X64-SSE2:       # %bb.0:
+; X64-SSE2-NEXT:    movdqu (%rdi), %xmm0
+; X64-SSE2-NEXT:    pcmpeqb {{.*}}(%rip), %xmm0
+; X64-SSE2-NEXT:    pmovmskb %xmm0, %eax
+; X64-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; X64-SSE2-NEXT:    sete %al
+; X64-SSE2-NEXT:    retq
+;
+; X64-AVX-LABEL: length16_eq_const:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    vmovdqu (%rdi), %xmm0
+; X64-AVX-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; X64-AVX-NEXT:    vptest %xmm0, %xmm0
+; X64-AVX-NEXT:    sete %al
+; X64-AVX-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* getelementptr inbounds ([65 x i8], [65 x i8]* @.str, i32 0, i32 0), i64 16) nounwind
+  %c = icmp eq i32 %m, 0
+  ret i1 %c
+}
+
+; PR33914 - https://bugs.llvm.org/show_bug.cgi?id=33914
+
+define i32 @length24(i8* %X, i8* %Y) nounwind !prof !14 {
+; X86-LABEL: length24:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $24
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    retl
+;
+; X64-LABEL: length24:
+; X64:       # %bb.0:
+; X64-NEXT:    movl $24, %edx
+; X64-NEXT:    jmp memcmp # TAILCALL
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 24) nounwind
+  ret i32 %m
+}
+
+define i1 @length24_eq(i8* %x, i8* %y) nounwind !prof !14 {
+; X86-NOSSE-LABEL: length24_eq:
+; X86-NOSSE:       # %bb.0:
+; X86-NOSSE-NEXT:    pushl $0
+; X86-NOSSE-NEXT:    pushl $24
+; X86-NOSSE-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    calll memcmp
+; X86-NOSSE-NEXT:    addl $16, %esp
+; X86-NOSSE-NEXT:    testl %eax, %eax
+; X86-NOSSE-NEXT:    sete %al
+; X86-NOSSE-NEXT:    retl
+;
+; X86-SSE2-LABEL: length24_eq:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE2-NEXT:    movdqu (%ecx), %xmm0
+; X86-SSE2-NEXT:    movdqu 8(%ecx), %xmm1
+; X86-SSE2-NEXT:    movdqu (%eax), %xmm2
+; X86-SSE2-NEXT:    pcmpeqb %xmm0, %xmm2
+; X86-SSE2-NEXT:    movdqu 8(%eax), %xmm0
+; X86-SSE2-NEXT:    pcmpeqb %xmm1, %xmm0
+; X86-SSE2-NEXT:    pand %xmm2, %xmm0
+; X86-SSE2-NEXT:    pmovmskb %xmm0, %eax
+; X86-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; X86-SSE2-NEXT:    sete %al
+; X86-SSE2-NEXT:    retl
+;
+; X64-SSE2-LABEL: length24_eq:
+; X64-SSE2:       # %bb.0:
+; X64-SSE2-NEXT:    movdqu (%rdi), %xmm0
+; X64-SSE2-NEXT:    movdqu (%rsi), %xmm1
+; X64-SSE2-NEXT:    pcmpeqb %xmm0, %xmm1
+; X64-SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; X64-SSE2-NEXT:    movq {{.*#+}} xmm2 = mem[0],zero
+; X64-SSE2-NEXT:    pcmpeqb %xmm0, %xmm2
+; X64-SSE2-NEXT:    pand %xmm1, %xmm2
+; X64-SSE2-NEXT:    pmovmskb %xmm2, %eax
+; X64-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; X64-SSE2-NEXT:    sete %al
+; X64-SSE2-NEXT:    retq
+;
+; X64-AVX-LABEL: length24_eq:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    vmovdqu (%rdi), %xmm0
+; X64-AVX-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
+; X64-AVX-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero
+; X64-AVX-NEXT:    vpxor %xmm2, %xmm1, %xmm1
+; X64-AVX-NEXT:    vpxor (%rsi), %xmm0, %xmm0
+; X64-AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; X64-AVX-NEXT:    vptest %xmm0, %xmm0
+; X64-AVX-NEXT:    sete %al
+; X64-AVX-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 24) nounwind
+  %cmp = icmp eq i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length24_eq_const(i8* %X) nounwind !prof !14 {
+; X86-NOSSE-LABEL: length24_eq_const:
+; X86-NOSSE:       # %bb.0:
+; X86-NOSSE-NEXT:    pushl $0
+; X86-NOSSE-NEXT:    pushl $24
+; X86-NOSSE-NEXT:    pushl $.L.str
+; X86-NOSSE-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    calll memcmp
+; X86-NOSSE-NEXT:    addl $16, %esp
+; X86-NOSSE-NEXT:    testl %eax, %eax
+; X86-NOSSE-NEXT:    setne %al
+; X86-NOSSE-NEXT:    retl
+;
+; X86-SSE2-LABEL: length24_eq_const:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT:    movdqu (%eax), %xmm0
+; X86-SSE2-NEXT:    movdqu 8(%eax), %xmm1
+; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI.*}}, %xmm1
+; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand %xmm1, %xmm0
+; X86-SSE2-NEXT:    pmovmskb %xmm0, %eax
+; X86-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; X86-SSE2-NEXT:    setne %al
+; X86-SSE2-NEXT:    retl
+;
+; X64-SSE2-LABEL: length24_eq_const:
+; X64-SSE2:       # %bb.0:
+; X64-SSE2-NEXT:    movdqu (%rdi), %xmm0
+; X64-SSE2-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
+; X64-SSE2-NEXT:    pcmpeqb {{.*}}(%rip), %xmm1
+; X64-SSE2-NEXT:    pcmpeqb {{.*}}(%rip), %xmm0
+; X64-SSE2-NEXT:    pand %xmm1, %xmm0
+; X64-SSE2-NEXT:    pmovmskb %xmm0, %eax
+; X64-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; X64-SSE2-NEXT:    setne %al
+; X64-SSE2-NEXT:    retq
+;
+; X64-AVX-LABEL: length24_eq_const:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    vmovdqu (%rdi), %xmm0
+; X64-AVX-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
+; X64-AVX-NEXT:    vpxor {{.*}}(%rip), %xmm1, %xmm1
+; X64-AVX-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; X64-AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; X64-AVX-NEXT:    vptest %xmm0, %xmm0
+; X64-AVX-NEXT:    setne %al
+; X64-AVX-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* getelementptr inbounds ([65 x i8], [65 x i8]* @.str, i32 0, i32 0), i64 24) nounwind
+  %c = icmp ne i32 %m, 0
+  ret i1 %c
+}
+
+define i32 @length32(i8* %X, i8* %Y) nounwind !prof !14 {
+; X86-LABEL: length32:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $32
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    retl
+;
+; X64-LABEL: length32:
+; X64:       # %bb.0:
+; X64-NEXT:    movl $32, %edx
+; X64-NEXT:    jmp memcmp # TAILCALL
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 32) nounwind
+  ret i32 %m
+}
+
+; PR33325 - https://bugs.llvm.org/show_bug.cgi?id=33325
+
+define i1 @length32_eq(i8* %x, i8* %y) nounwind !prof !14 {
+; X86-NOSSE-LABEL: length32_eq:
+; X86-NOSSE:       # %bb.0:
+; X86-NOSSE-NEXT:    pushl $0
+; X86-NOSSE-NEXT:    pushl $32
+; X86-NOSSE-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    calll memcmp
+; X86-NOSSE-NEXT:    addl $16, %esp
+; X86-NOSSE-NEXT:    testl %eax, %eax
+; X86-NOSSE-NEXT:    sete %al
+; X86-NOSSE-NEXT:    retl
+;
+; X86-SSE2-LABEL: length32_eq:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE2-NEXT:    movdqu (%ecx), %xmm0
+; X86-SSE2-NEXT:    movdqu 16(%ecx), %xmm1
+; X86-SSE2-NEXT:    movdqu (%eax), %xmm2
+; X86-SSE2-NEXT:    pcmpeqb %xmm0, %xmm2
+; X86-SSE2-NEXT:    movdqu 16(%eax), %xmm0
+; X86-SSE2-NEXT:    pcmpeqb %xmm1, %xmm0
+; X86-SSE2-NEXT:    pand %xmm2, %xmm0
+; X86-SSE2-NEXT:    pmovmskb %xmm0, %eax
+; X86-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; X86-SSE2-NEXT:    sete %al
+; X86-SSE2-NEXT:    retl
+;
+; X64-SSE2-LABEL: length32_eq:
+; X64-SSE2:       # %bb.0:
+; X64-SSE2-NEXT:    movdqu (%rdi), %xmm0
+; X64-SSE2-NEXT:    movdqu 16(%rdi), %xmm1
+; X64-SSE2-NEXT:    movdqu (%rsi), %xmm2
+; X64-SSE2-NEXT:    pcmpeqb %xmm0, %xmm2
+; X64-SSE2-NEXT:    movdqu 16(%rsi), %xmm0
+; X64-SSE2-NEXT:    pcmpeqb %xmm1, %xmm0
+; X64-SSE2-NEXT:    pand %xmm2, %xmm0
+; X64-SSE2-NEXT:    pmovmskb %xmm0, %eax
+; X64-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; X64-SSE2-NEXT:    sete %al
+; X64-SSE2-NEXT:    retq
+;
+; X64-AVX1-LABEL: length32_eq:
+; X64-AVX1:       # %bb.0:
+; X64-AVX1-NEXT:    vmovups (%rdi), %ymm0
+; X64-AVX1-NEXT:    vxorps (%rsi), %ymm0, %ymm0
+; X64-AVX1-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX1-NEXT:    sete %al
+; X64-AVX1-NEXT:    vzeroupper
+; X64-AVX1-NEXT:    retq
+;
+; X64-AVX2-LABEL: length32_eq:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    vmovdqu (%rdi), %ymm0
+; X64-AVX2-NEXT:    vpxor (%rsi), %ymm0, %ymm0
+; X64-AVX2-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX2-NEXT:    sete %al
+; X64-AVX2-NEXT:    vzeroupper
+; X64-AVX2-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 32) nounwind
+  %cmp = icmp eq i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length32_eq_const(i8* %X) nounwind !prof !14 {
+; X86-NOSSE-LABEL: length32_eq_const:
+; X86-NOSSE:       # %bb.0:
+; X86-NOSSE-NEXT:    pushl $0
+; X86-NOSSE-NEXT:    pushl $32
+; X86-NOSSE-NEXT:    pushl $.L.str
+; X86-NOSSE-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    calll memcmp
+; X86-NOSSE-NEXT:    addl $16, %esp
+; X86-NOSSE-NEXT:    testl %eax, %eax
+; X86-NOSSE-NEXT:    setne %al
+; X86-NOSSE-NEXT:    retl
+;
+; X86-SSE2-LABEL: length32_eq_const:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT:    movdqu (%eax), %xmm0
+; X86-SSE2-NEXT:    movdqu 16(%eax), %xmm1
+; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI.*}}, %xmm1
+; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand %xmm1, %xmm0
+; X86-SSE2-NEXT:    pmovmskb %xmm0, %eax
+; X86-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; X86-SSE2-NEXT:    setne %al
+; X86-SSE2-NEXT:    retl
+;
+; X64-SSE2-LABEL: length32_eq_const:
+; X64-SSE2:       # %bb.0:
+; X64-SSE2-NEXT:    movdqu (%rdi), %xmm0
+; X64-SSE2-NEXT:    movdqu 16(%rdi), %xmm1
+; X64-SSE2-NEXT:    pcmpeqb {{.*}}(%rip), %xmm1
+; X64-SSE2-NEXT:    pcmpeqb {{.*}}(%rip), %xmm0
+; X64-SSE2-NEXT:    pand %xmm1, %xmm0
+; X64-SSE2-NEXT:    pmovmskb %xmm0, %eax
+; X64-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; X64-SSE2-NEXT:    setne %al
+; X64-SSE2-NEXT:    retq
+;
+; X64-AVX1-LABEL: length32_eq_const:
+; X64-AVX1:       # %bb.0:
+; X64-AVX1-NEXT:    vmovups (%rdi), %ymm0
+; X64-AVX1-NEXT:    vxorps {{.*}}(%rip), %ymm0, %ymm0
+; X64-AVX1-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX1-NEXT:    setne %al
+; X64-AVX1-NEXT:    vzeroupper
+; X64-AVX1-NEXT:    retq
+;
+; X64-AVX2-LABEL: length32_eq_const:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    vmovdqu (%rdi), %ymm0
+; X64-AVX2-NEXT:    vpxor {{.*}}(%rip), %ymm0, %ymm0
+; X64-AVX2-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX2-NEXT:    setne %al
+; X64-AVX2-NEXT:    vzeroupper
+; X64-AVX2-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* getelementptr inbounds ([65 x i8], [65 x i8]* @.str, i32 0, i32 0), i64 32) nounwind
+  %c = icmp ne i32 %m, 0
+  ret i1 %c
+}
+
+define i32 @length64(i8* %X, i8* %Y) nounwind !prof !14 {
+; X86-LABEL: length64:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $64
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    retl
+;
+; X64-LABEL: length64:
+; X64:       # %bb.0:
+; X64-NEXT:    movl $64, %edx
+; X64-NEXT:    jmp memcmp # TAILCALL
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 64) nounwind
+  ret i32 %m
+}
+
+define i1 @length64_eq(i8* %x, i8* %y) nounwind !prof !14 {
+; X86-LABEL: length64_eq:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $64
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    setne %al
+; X86-NEXT:    retl
+;
+; X64-SSE2-LABEL: length64_eq:
+; X64-SSE2:       # %bb.0:
+; X64-SSE2-NEXT:    pushq %rax
+; X64-SSE2-NEXT:    movl $64, %edx
+; X64-SSE2-NEXT:    callq memcmp
+; X64-SSE2-NEXT:    testl %eax, %eax
+; X64-SSE2-NEXT:    setne %al
+; X64-SSE2-NEXT:    popq %rcx
+; X64-SSE2-NEXT:    retq
+;
+; X64-AVX1-LABEL: length64_eq:
+; X64-AVX1:       # %bb.0:
+; X64-AVX1-NEXT:    vmovups (%rdi), %ymm0
+; X64-AVX1-NEXT:    vmovups 32(%rdi), %ymm1
+; X64-AVX1-NEXT:    vxorps 32(%rsi), %ymm1, %ymm1
+; X64-AVX1-NEXT:    vxorps (%rsi), %ymm0, %ymm0
+; X64-AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; X64-AVX1-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX1-NEXT:    setne %al
+; X64-AVX1-NEXT:    vzeroupper
+; X64-AVX1-NEXT:    retq
+;
+; X64-AVX2-LABEL: length64_eq:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    vmovdqu (%rdi), %ymm0
+; X64-AVX2-NEXT:    vmovdqu 32(%rdi), %ymm1
+; X64-AVX2-NEXT:    vpxor 32(%rsi), %ymm1, %ymm1
+; X64-AVX2-NEXT:    vpxor (%rsi), %ymm0, %ymm0
+; X64-AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; X64-AVX2-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX2-NEXT:    setne %al
+; X64-AVX2-NEXT:    vzeroupper
+; X64-AVX2-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 64) nounwind
+  %cmp = icmp ne i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length64_eq_const(i8* %X) nounwind !prof !14 {
+; X86-LABEL: length64_eq_const:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $64
+; X86-NEXT:    pushl $.L.str
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    sete %al
+; X86-NEXT:    retl
+;
+; X64-SSE2-LABEL: length64_eq_const:
+; X64-SSE2:       # %bb.0:
+; X64-SSE2-NEXT:    pushq %rax
+; X64-SSE2-NEXT:    movl $.L.str, %esi
+; X64-SSE2-NEXT:    movl $64, %edx
+; X64-SSE2-NEXT:    callq memcmp
+; X64-SSE2-NEXT:    testl %eax, %eax
+; X64-SSE2-NEXT:    sete %al
+; X64-SSE2-NEXT:    popq %rcx
+; X64-SSE2-NEXT:    retq
+;
+; X64-AVX1-LABEL: length64_eq_const:
+; X64-AVX1:       # %bb.0:
+; X64-AVX1-NEXT:    vmovups (%rdi), %ymm0
+; X64-AVX1-NEXT:    vmovups 32(%rdi), %ymm1
+; X64-AVX1-NEXT:    vxorps {{.*}}(%rip), %ymm1, %ymm1
+; X64-AVX1-NEXT:    vxorps {{.*}}(%rip), %ymm0, %ymm0
+; X64-AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; X64-AVX1-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX1-NEXT:    sete %al
+; X64-AVX1-NEXT:    vzeroupper
+; X64-AVX1-NEXT:    retq
+;
+; X64-AVX2-LABEL: length64_eq_const:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    vmovdqu (%rdi), %ymm0
+; X64-AVX2-NEXT:    vmovdqu 32(%rdi), %ymm1
+; X64-AVX2-NEXT:    vpxor {{.*}}(%rip), %ymm1, %ymm1
+; X64-AVX2-NEXT:    vpxor {{.*}}(%rip), %ymm0, %ymm0
+; X64-AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; X64-AVX2-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX2-NEXT:    sete %al
+; X64-AVX2-NEXT:    vzeroupper
+; X64-AVX2-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* getelementptr inbounds ([65 x i8], [65 x i8]* @.str, i32 0, i32 0), i64 64) nounwind
+  %c = icmp eq i32 %m, 0
+  ret i1 %c
+}
+
+define i32 @bcmp_length2(i8* %X, i8* %Y) nounwind !prof !14 {
+; X86-LABEL: bcmp_length2:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movzwl (%ecx), %ecx
+; X86-NEXT:    movzwl (%eax), %edx
+; X86-NEXT:    rolw $8, %cx
+; X86-NEXT:    rolw $8, %dx
+; X86-NEXT:    movzwl %cx, %eax
+; X86-NEXT:    movzwl %dx, %ecx
+; X86-NEXT:    subl %ecx, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: bcmp_length2:
+; X64:       # %bb.0:
+; X64-NEXT:    movzwl (%rdi), %eax
+; X64-NEXT:    movzwl (%rsi), %ecx
+; X64-NEXT:    rolw $8, %ax
+; X64-NEXT:    rolw $8, %cx
+; X64-NEXT:    movzwl %ax, %eax
+; X64-NEXT:    movzwl %cx, %ecx
+; X64-NEXT:    subl %ecx, %eax
+; X64-NEXT:    retq
+  %m = tail call i32 @bcmp(i8* %X, i8* %Y, i64 2) nounwind
+  ret i32 %m
+}
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 1, !"ProfileSummary", !1}
+!1 = !{!2, !3, !4, !5, !6, !7, !8, !9}
+!2 = !{!"ProfileFormat", !"InstrProf"}
+!3 = !{!"TotalCount", i64 10000}
+!4 = !{!"MaxCount", i64 10}
+!5 = !{!"MaxInternalCount", i64 1}
+!6 = !{!"MaxFunctionCount", i64 1000}
+!7 = !{!"NumCounts", i64 3}
+!8 = !{!"NumFunctions", i64 3}
+!9 = !{!"DetailedSummary", !10}
+!10 = !{!11, !12, !13}
+!11 = !{i32 10000, i64 100, i32 1}
+!12 = !{i32 999000, i64 100, i32 1}
+!13 = !{i32 999999, i64 1, i32 2}
+!14 = !{!"function_entry_count", i64 0}
diff --git a/llvm/test/CodeGen/X86/memcmp.ll b/llvm/test/CodeGen/X86/memcmp.ll
index 8a7c7847f..793b4e392 100644
--- a/llvm/test/CodeGen/X86/memcmp.ll
+++ b/llvm/test/CodeGen/X86/memcmp.ll
@@ -1,15 +1,23 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=cmov   | FileCheck %s --check-prefix=X86 --check-prefix=X86-NOSSE
-; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse   | FileCheck %s --check-prefix=X86 --check-prefix=SSE --check-prefix=X86-SSE1
-; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse2  | FileCheck %s --check-prefix=X86 --check-prefix=SSE --check-prefix=X86-SSE2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown             | FileCheck %s --check-prefix=X64 --check-prefix=X64-SSE2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx  | FileCheck %s --check-prefix=X64 --check-prefix=X64-AVX --check-prefix=X64-AVX1
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx2 | FileCheck %s --check-prefix=X64 --check-prefix=X64-AVX --check-prefix=X64-AVX2
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=cmov     | FileCheck %s --check-prefixes=X86,X86-NOSSE
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse     | FileCheck %s --check-prefixes=X86,X86-SSE,X86-SSE1
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse2    | FileCheck %s --check-prefixes=X86,X86-SSE,X86-SSE2
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse4.1  | FileCheck %s --check-prefixes=X86,X86-SSE,X86-SSE41
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown               | FileCheck %s --check-prefixes=X64,X64-SSE,X64-SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=sse4.1 | FileCheck %s --check-prefixes=X64,X64-SSE,X64-SSE41
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx    | FileCheck %s --check-prefixes=X64,X64-AVX,X64-AVX1
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx2   | FileCheck %s --check-prefixes=X64,X64-AVX,X64-AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512bw,+prefer-256-bit | FileCheck %s --check-prefixes=X64,X64-AVX,X64-AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512bw,-prefer-256-bit | FileCheck %s --check-prefixes=X64,X64-AVX,X64-AVX512,X64-AVX512BW
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512f,+prefer-256-bit,-prefer-mask-registers | FileCheck %s --check-prefixes=X64,X64-AVX,X64-AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512f,-prefer-256-bit,-prefer-mask-registers | FileCheck %s --check-prefixes=X64,X64-AVX,X64-AVX512,X64-AVX512F
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512f,+prefer-256-bit,+prefer-mask-registers | FileCheck %s --check-prefixes=X64,X64-MIC-AVX,X64-MIC-AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512f,-prefer-256-bit,+prefer-mask-registers | FileCheck %s --check-prefixes=X64,X64-MIC-AVX,X64-MIC-AVX512F
 
 ; This tests codegen time inlining/optimization of memcmp
 ; rdar://6480398
 
-@.str = private constant [65 x i8] c"0123456789012345678901234567890123456789012345678901234567890123\00", align 1
+@.str = private constant [513 x i8] c"01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901\00", align 1
 
 declare i32 @memcmp(i8*, i8*, i64)
 
@@ -187,7 +195,7 @@ define i1 @length2_eq_const(i8* %X) nounwind {
 ; X64-NEXT:    cmpl $12849, %eax # imm = 0x3231
 ; X64-NEXT:    setne %al
 ; X64-NEXT:    retq
-  %m = tail call i32 @memcmp(i8* %X, i8* getelementptr inbounds ([65 x i8], [65 x i8]* @.str, i32 0, i32 1), i64 2) nounwind
+  %m = tail call i32 @memcmp(i8* %X, i8* getelementptr inbounds ([513 x i8], [513 x i8]* @.str, i32 0, i32 1), i64 2) nounwind
   %c = icmp ne i32 %m, 0
   ret i1 %c
 }
@@ -429,7 +437,7 @@ define i1 @length4_eq_const(i8* %X) nounwind {
 ; X64-NEXT:    cmpl $875770417, (%rdi) # imm = 0x34333231
 ; X64-NEXT:    sete %al
 ; X64-NEXT:    retq
-  %m = tail call i32 @memcmp(i8* %X, i8* getelementptr inbounds ([65 x i8], [65 x i8]* @.str, i32 0, i32 1), i64 4) nounwind
+  %m = tail call i32 @memcmp(i8* %X, i8* getelementptr inbounds ([513 x i8], [513 x i8]* @.str, i32 0, i32 1), i64 4) nounwind
   %c = icmp eq i32 %m, 0
   ret i1 %c
 }
@@ -677,7 +685,7 @@ define i1 @length8_eq_const(i8* %X) nounwind {
 ; X64-NEXT:    cmpq %rax, (%rdi)
 ; X64-NEXT:    setne %al
 ; X64-NEXT:    retq
-  %m = tail call i32 @memcmp(i8* %X, i8* getelementptr inbounds ([65 x i8], [65 x i8]* @.str, i32 0, i32 0), i64 8) nounwind
+  %m = tail call i32 @memcmp(i8* %X, i8* getelementptr inbounds ([513 x i8], [513 x i8]* @.str, i32 0, i32 0), i64 8) nounwind
   %c = icmp ne i32 %m, 0
   ret i1 %c
 }
@@ -988,6 +996,17 @@ define i1 @length16_eq(i8* %x, i8* %y) nounwind {
 ; X86-SSE2-NEXT:    setne %al
 ; X86-SSE2-NEXT:    retl
 ;
+; X86-SSE41-LABEL: length16_eq:
+; X86-SSE41:       # %bb.0:
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE41-NEXT:    movdqu (%ecx), %xmm0
+; X86-SSE41-NEXT:    movdqu (%eax), %xmm1
+; X86-SSE41-NEXT:    pxor %xmm0, %xmm1
+; X86-SSE41-NEXT:    ptest %xmm1, %xmm1
+; X86-SSE41-NEXT:    setne %al
+; X86-SSE41-NEXT:    retl
+;
 ; X64-SSE2-LABEL: length16_eq:
 ; X64-SSE2:       # %bb.0:
 ; X64-SSE2-NEXT:    movdqu (%rdi), %xmm0
@@ -998,19 +1017,123 @@ define i1 @length16_eq(i8* %x, i8* %y) nounwind {
 ; X64-SSE2-NEXT:    setne %al
 ; X64-SSE2-NEXT:    retq
 ;
+; X64-SSE41-LABEL: length16_eq:
+; X64-SSE41:       # %bb.0:
+; X64-SSE41-NEXT:    movdqu (%rdi), %xmm0
+; X64-SSE41-NEXT:    movdqu (%rsi), %xmm1
+; X64-SSE41-NEXT:    pxor %xmm0, %xmm1
+; X64-SSE41-NEXT:    ptest %xmm1, %xmm1
+; X64-SSE41-NEXT:    setne %al
+; X64-SSE41-NEXT:    retq
+;
 ; X64-AVX-LABEL: length16_eq:
 ; X64-AVX:       # %bb.0:
 ; X64-AVX-NEXT:    vmovdqu (%rdi), %xmm0
-; X64-AVX-NEXT:    vpcmpeqb (%rsi), %xmm0, %xmm0
-; X64-AVX-NEXT:    vpmovmskb %xmm0, %eax
-; X64-AVX-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; X64-AVX-NEXT:    vpxor (%rsi), %xmm0, %xmm0
+; X64-AVX-NEXT:    vptest %xmm0, %xmm0
 ; X64-AVX-NEXT:    setne %al
 ; X64-AVX-NEXT:    retq
+;
+; X64-MIC-AVX-LABEL: length16_eq:
+; X64-MIC-AVX:       # %bb.0:
+; X64-MIC-AVX-NEXT:    vmovdqu (%rdi), %xmm0
+; X64-MIC-AVX-NEXT:    vmovdqu (%rsi), %xmm1
+; X64-MIC-AVX-NEXT:    vpcmpneqd %zmm1, %zmm0, %k0
+; X64-MIC-AVX-NEXT:    kortestw %k0, %k0
+; X64-MIC-AVX-NEXT:    setne %al
+; X64-MIC-AVX-NEXT:    vzeroupper
+; X64-MIC-AVX-NEXT:    retq
   %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 16) nounwind
   %cmp = icmp ne i32 %call, 0
   ret i1 %cmp
 }
 
+define i1 @length16_lt(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length16_lt:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $16
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    shrl $31, %eax
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: length16_lt:
+; X64:       # %bb.0:
+; X64-NEXT:    movq (%rdi), %rcx
+; X64-NEXT:    movq (%rsi), %rdx
+; X64-NEXT:    bswapq %rcx
+; X64-NEXT:    bswapq %rdx
+; X64-NEXT:    cmpq %rdx, %rcx
+; X64-NEXT:    jne .LBB33_2
+; X64-NEXT:  # %bb.1: # %loadbb1
+; X64-NEXT:    movq 8(%rdi), %rcx
+; X64-NEXT:    movq 8(%rsi), %rdx
+; X64-NEXT:    bswapq %rcx
+; X64-NEXT:    bswapq %rdx
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpq %rdx, %rcx
+; X64-NEXT:    je .LBB33_3
+; X64-NEXT:  .LBB33_2: # %res_block
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpq %rdx, %rcx
+; X64-NEXT:    setae %al
+; X64-NEXT:    leal -1(%rax,%rax), %eax
+; X64-NEXT:  .LBB33_3: # %endblock
+; X64-NEXT:    shrl $31, %eax
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 16) nounwind
+  %cmp = icmp slt i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length16_gt(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length16_gt:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $16
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    setg %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length16_gt:
+; X64:       # %bb.0:
+; X64-NEXT:    movq (%rdi), %rax
+; X64-NEXT:    movq (%rsi), %rcx
+; X64-NEXT:    bswapq %rax
+; X64-NEXT:    bswapq %rcx
+; X64-NEXT:    cmpq %rcx, %rax
+; X64-NEXT:    jne .LBB34_2
+; X64-NEXT:  # %bb.1: # %loadbb1
+; X64-NEXT:    movq 8(%rdi), %rax
+; X64-NEXT:    movq 8(%rsi), %rcx
+; X64-NEXT:    bswapq %rax
+; X64-NEXT:    bswapq %rcx
+; X64-NEXT:    xorl %edx, %edx
+; X64-NEXT:    cmpq %rcx, %rax
+; X64-NEXT:    je .LBB34_3
+; X64-NEXT:  .LBB34_2: # %res_block
+; X64-NEXT:    xorl %edx, %edx
+; X64-NEXT:    cmpq %rcx, %rax
+; X64-NEXT:    setae %dl
+; X64-NEXT:    leal -1(%rdx,%rdx), %edx
+; X64-NEXT:  .LBB34_3: # %endblock
+; X64-NEXT:    testl %edx, %edx
+; X64-NEXT:    setg %al
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 16) nounwind
+  %cmp = icmp sgt i32 %call, 0
+  ret i1 %cmp
+}
+
 define i1 @length16_eq_const(i8* %X) nounwind {
 ; X86-NOSSE-LABEL: length16_eq_const:
 ; X86-NOSSE:       # %bb.0:
@@ -1046,6 +1169,15 @@ define i1 @length16_eq_const(i8* %X) nounwind {
 ; X86-SSE2-NEXT:    sete %al
 ; X86-SSE2-NEXT:    retl
 ;
+; X86-SSE41-LABEL: length16_eq_const:
+; X86-SSE41:       # %bb.0:
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT:    movdqu (%eax), %xmm0
+; X86-SSE41-NEXT:    pxor {{\.LCPI.*}}, %xmm0
+; X86-SSE41-NEXT:    ptest %xmm0, %xmm0
+; X86-SSE41-NEXT:    sete %al
+; X86-SSE41-NEXT:    retl
+;
 ; X64-SSE2-LABEL: length16_eq_const:
 ; X64-SSE2:       # %bb.0:
 ; X64-SSE2-NEXT:    movdqu (%rdi), %xmm0
@@ -1055,15 +1187,32 @@ define i1 @length16_eq_const(i8* %X) nounwind {
 ; X64-SSE2-NEXT:    sete %al
 ; X64-SSE2-NEXT:    retq
 ;
+; X64-SSE41-LABEL: length16_eq_const:
+; X64-SSE41:       # %bb.0:
+; X64-SSE41-NEXT:    movdqu (%rdi), %xmm0
+; X64-SSE41-NEXT:    pxor {{.*}}(%rip), %xmm0
+; X64-SSE41-NEXT:    ptest %xmm0, %xmm0
+; X64-SSE41-NEXT:    sete %al
+; X64-SSE41-NEXT:    retq
+;
 ; X64-AVX-LABEL: length16_eq_const:
 ; X64-AVX:       # %bb.0:
 ; X64-AVX-NEXT:    vmovdqu (%rdi), %xmm0
-; X64-AVX-NEXT:    vpcmpeqb {{.*}}(%rip), %xmm0, %xmm0
-; X64-AVX-NEXT:    vpmovmskb %xmm0, %eax
-; X64-AVX-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; X64-AVX-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; X64-AVX-NEXT:    vptest %xmm0, %xmm0
 ; X64-AVX-NEXT:    sete %al
 ; X64-AVX-NEXT:    retq
-  %m = tail call i32 @memcmp(i8* %X, i8* getelementptr inbounds ([65 x i8], [65 x i8]* @.str, i32 0, i32 0), i64 16) nounwind
+;
+; X64-MIC-AVX-LABEL: length16_eq_const:
+; X64-MIC-AVX:       # %bb.0:
+; X64-MIC-AVX-NEXT:    vmovdqu (%rdi), %xmm0
+; X64-MIC-AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [858927408,926299444,825243960,892613426]
+; X64-MIC-AVX-NEXT:    vpcmpneqd %zmm1, %zmm0, %k0
+; X64-MIC-AVX-NEXT:    kortestw %k0, %k0
+; X64-MIC-AVX-NEXT:    sete %al
+; X64-MIC-AVX-NEXT:    vzeroupper
+; X64-MIC-AVX-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* getelementptr inbounds ([513 x i8], [513 x i8]* @.str, i32 0, i32 0), i64 16) nounwind
   %c = icmp eq i32 %m, 0
   ret i1 %c
 }
@@ -1130,6 +1279,21 @@ define i1 @length24_eq(i8* %x, i8* %y) nounwind {
 ; X86-SSE2-NEXT:    sete %al
 ; X86-SSE2-NEXT:    retl
 ;
+; X86-SSE41-LABEL: length24_eq:
+; X86-SSE41:       # %bb.0:
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE41-NEXT:    movdqu (%ecx), %xmm0
+; X86-SSE41-NEXT:    movdqu 8(%ecx), %xmm1
+; X86-SSE41-NEXT:    movdqu (%eax), %xmm2
+; X86-SSE41-NEXT:    pxor %xmm0, %xmm2
+; X86-SSE41-NEXT:    movdqu 8(%eax), %xmm0
+; X86-SSE41-NEXT:    pxor %xmm1, %xmm0
+; X86-SSE41-NEXT:    por %xmm2, %xmm0
+; X86-SSE41-NEXT:    ptest %xmm0, %xmm0
+; X86-SSE41-NEXT:    sete %al
+; X86-SSE41-NEXT:    retl
+;
 ; X64-SSE2-LABEL: length24_eq:
 ; X64-SSE2:       # %bb.0:
 ; X64-SSE2-NEXT:    movdqu (%rdi), %xmm0
@@ -1144,23 +1308,102 @@ define i1 @length24_eq(i8* %x, i8* %y) nounwind {
 ; X64-SSE2-NEXT:    sete %al
 ; X64-SSE2-NEXT:    retq
 ;
+; X64-SSE41-LABEL: length24_eq:
+; X64-SSE41:       # %bb.0:
+; X64-SSE41-NEXT:    movdqu (%rdi), %xmm0
+; X64-SSE41-NEXT:    movdqu (%rsi), %xmm1
+; X64-SSE41-NEXT:    pxor %xmm0, %xmm1
+; X64-SSE41-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; X64-SSE41-NEXT:    movq {{.*#+}} xmm2 = mem[0],zero
+; X64-SSE41-NEXT:    pxor %xmm0, %xmm2
+; X64-SSE41-NEXT:    por %xmm1, %xmm2
+; X64-SSE41-NEXT:    ptest %xmm2, %xmm2
+; X64-SSE41-NEXT:    sete %al
+; X64-SSE41-NEXT:    retq
+;
 ; X64-AVX-LABEL: length24_eq:
 ; X64-AVX:       # %bb.0:
 ; X64-AVX-NEXT:    vmovdqu (%rdi), %xmm0
 ; X64-AVX-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
 ; X64-AVX-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero
-; X64-AVX-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm1
-; X64-AVX-NEXT:    vpcmpeqb (%rsi), %xmm0, %xmm0
-; X64-AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; X64-AVX-NEXT:    vpmovmskb %xmm0, %eax
-; X64-AVX-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; X64-AVX-NEXT:    vpxor %xmm2, %xmm1, %xmm1
+; X64-AVX-NEXT:    vpxor (%rsi), %xmm0, %xmm0
+; X64-AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; X64-AVX-NEXT:    vptest %xmm0, %xmm0
 ; X64-AVX-NEXT:    sete %al
 ; X64-AVX-NEXT:    retq
+;
+; X64-MIC-AVX-LABEL: length24_eq:
+; X64-MIC-AVX:       # %bb.0:
+; X64-MIC-AVX-NEXT:    vmovdqu (%rdi), %xmm0
+; X64-MIC-AVX-NEXT:    vmovdqu (%rsi), %xmm1
+; X64-MIC-AVX-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero
+; X64-MIC-AVX-NEXT:    vmovq {{.*#+}} xmm3 = mem[0],zero
+; X64-MIC-AVX-NEXT:    vpcmpneqd %zmm3, %zmm2, %k0
+; X64-MIC-AVX-NEXT:    vpcmpneqd %zmm1, %zmm0, %k1
+; X64-MIC-AVX-NEXT:    kortestw %k0, %k1
+; X64-MIC-AVX-NEXT:    sete %al
+; X64-MIC-AVX-NEXT:    vzeroupper
+; X64-MIC-AVX-NEXT:    retq
   %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 24) nounwind
   %cmp = icmp eq i32 %call, 0
   ret i1 %cmp
 }
 
+define i1 @length24_lt(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length24_lt:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $24
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    shrl $31, %eax
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: length24_lt:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $24, %edx
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    shrl $31, %eax
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 24) nounwind
+  %cmp = icmp slt i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length24_gt(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length24_gt:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $24
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    setg %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length24_gt:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $24, %edx
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    setg %al
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 24) nounwind
+  %cmp = icmp sgt i32 %call, 0
+  ret i1 %cmp
+}
+
 define i1 @length24_eq_const(i8* %X) nounwind {
 ; X86-NOSSE-LABEL: length24_eq_const:
 ; X86-NOSSE:       # %bb.0:
@@ -1199,6 +1442,18 @@ define i1 @length24_eq_const(i8* %X) nounwind {
 ; X86-SSE2-NEXT:    setne %al
 ; X86-SSE2-NEXT:    retl
 ;
+; X86-SSE41-LABEL: length24_eq_const:
+; X86-SSE41:       # %bb.0:
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT:    movdqu (%eax), %xmm0
+; X86-SSE41-NEXT:    movdqu 8(%eax), %xmm1
+; X86-SSE41-NEXT:    pxor {{\.LCPI.*}}, %xmm1
+; X86-SSE41-NEXT:    pxor {{\.LCPI.*}}, %xmm0
+; X86-SSE41-NEXT:    por %xmm1, %xmm0
+; X86-SSE41-NEXT:    ptest %xmm0, %xmm0
+; X86-SSE41-NEXT:    setne %al
+; X86-SSE41-NEXT:    retl
+;
 ; X64-SSE2-LABEL: length24_eq_const:
 ; X64-SSE2:       # %bb.0:
 ; X64-SSE2-NEXT:    movdqu (%rdi), %xmm0
@@ -1211,48 +1466,69 @@ define i1 @length24_eq_const(i8* %X) nounwind {
 ; X64-SSE2-NEXT:    setne %al
 ; X64-SSE2-NEXT:    retq
 ;
+; X64-SSE41-LABEL: length24_eq_const:
+; X64-SSE41:       # %bb.0:
+; X64-SSE41-NEXT:    movdqu (%rdi), %xmm0
+; X64-SSE41-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
+; X64-SSE41-NEXT:    pxor {{.*}}(%rip), %xmm1
+; X64-SSE41-NEXT:    pxor {{.*}}(%rip), %xmm0
+; X64-SSE41-NEXT:    por %xmm1, %xmm0
+; X64-SSE41-NEXT:    ptest %xmm0, %xmm0
+; X64-SSE41-NEXT:    setne %al
+; X64-SSE41-NEXT:    retq
+;
 ; X64-AVX-LABEL: length24_eq_const:
 ; X64-AVX:       # %bb.0:
 ; X64-AVX-NEXT:    vmovdqu (%rdi), %xmm0
 ; X64-AVX-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
-; X64-AVX-NEXT:    vpcmpeqb {{.*}}(%rip), %xmm1, %xmm1
-; X64-AVX-NEXT:    vpcmpeqb {{.*}}(%rip), %xmm0, %xmm0
-; X64-AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; X64-AVX-NEXT:    vpmovmskb %xmm0, %eax
-; X64-AVX-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; X64-AVX-NEXT:    vpxor {{.*}}(%rip), %xmm1, %xmm1
+; X64-AVX-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; X64-AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; X64-AVX-NEXT:    vptest %xmm0, %xmm0
 ; X64-AVX-NEXT:    setne %al
 ; X64-AVX-NEXT:    retq
-  %m = tail call i32 @memcmp(i8* %X, i8* getelementptr inbounds ([65 x i8], [65 x i8]* @.str, i32 0, i32 0), i64 24) nounwind
+;
+; X64-MIC-AVX-LABEL: length24_eq_const:
+; X64-MIC-AVX:       # %bb.0:
+; X64-MIC-AVX-NEXT:    vmovdqu (%rdi), %xmm0
+; X64-MIC-AVX-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
+; X64-MIC-AVX-NEXT:    vmovdqa {{.*#+}} xmm2 = [959985462,858927408,0,0]
+; X64-MIC-AVX-NEXT:    vpcmpneqd %zmm2, %zmm1, %k0
+; X64-MIC-AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [858927408,926299444,825243960,892613426]
+; X64-MIC-AVX-NEXT:    vpcmpneqd %zmm1, %zmm0, %k1
+; X64-MIC-AVX-NEXT:    kortestw %k0, %k1
+; X64-MIC-AVX-NEXT:    setne %al
+; X64-MIC-AVX-NEXT:    vzeroupper
+; X64-MIC-AVX-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* getelementptr inbounds ([513 x i8], [513 x i8]* @.str, i32 0, i32 0), i64 24) nounwind
   %c = icmp ne i32 %m, 0
   ret i1 %c
 }
 
-define i32 @length32(i8* %X, i8* %Y) nounwind {
-; X86-LABEL: length32:
+define i32 @length31(i8* %X, i8* %Y) nounwind {
+; X86-LABEL: length31:
 ; X86:       # %bb.0:
 ; X86-NEXT:    pushl $0
-; X86-NEXT:    pushl $32
+; X86-NEXT:    pushl $31
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    calll memcmp
 ; X86-NEXT:    addl $16, %esp
 ; X86-NEXT:    retl
 ;
-; X64-LABEL: length32:
+; X64-LABEL: length31:
 ; X64:       # %bb.0:
-; X64-NEXT:    movl $32, %edx
+; X64-NEXT:    movl $31, %edx
 ; X64-NEXT:    jmp memcmp # TAILCALL
-  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 32) nounwind
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 31) nounwind
   ret i32 %m
 }
 
-; PR33325 - https://bugs.llvm.org/show_bug.cgi?id=33325
-
-define i1 @length32_eq(i8* %x, i8* %y) nounwind {
-; X86-NOSSE-LABEL: length32_eq:
+define i1 @length31_eq(i8* %x, i8* %y) nounwind {
+; X86-NOSSE-LABEL: length31_eq:
 ; X86-NOSSE:       # %bb.0:
 ; X86-NOSSE-NEXT:    pushl $0
-; X86-NOSSE-NEXT:    pushl $32
+; X86-NOSSE-NEXT:    pushl $31
 ; X86-NOSSE-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NOSSE-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NOSSE-NEXT:    calll memcmp
@@ -1261,10 +1537,10 @@ define i1 @length32_eq(i8* %x, i8* %y) nounwind {
 ; X86-NOSSE-NEXT:    sete %al
 ; X86-NOSSE-NEXT:    retl
 ;
-; X86-SSE1-LABEL: length32_eq:
+; X86-SSE1-LABEL: length31_eq:
 ; X86-SSE1:       # %bb.0:
 ; X86-SSE1-NEXT:    pushl $0
-; X86-SSE1-NEXT:    pushl $32
+; X86-SSE1-NEXT:    pushl $31
 ; X86-SSE1-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-SSE1-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-SSE1-NEXT:    calll memcmp
@@ -1273,15 +1549,15 @@ define i1 @length32_eq(i8* %x, i8* %y) nounwind {
 ; X86-SSE1-NEXT:    sete %al
 ; X86-SSE1-NEXT:    retl
 ;
-; X86-SSE2-LABEL: length32_eq:
+; X86-SSE2-LABEL: length31_eq:
 ; X86-SSE2:       # %bb.0:
 ; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-SSE2-NEXT:    movdqu (%ecx), %xmm0
-; X86-SSE2-NEXT:    movdqu 16(%ecx), %xmm1
+; X86-SSE2-NEXT:    movdqu 15(%ecx), %xmm1
 ; X86-SSE2-NEXT:    movdqu (%eax), %xmm2
 ; X86-SSE2-NEXT:    pcmpeqb %xmm0, %xmm2
-; X86-SSE2-NEXT:    movdqu 16(%eax), %xmm0
+; X86-SSE2-NEXT:    movdqu 15(%eax), %xmm0
 ; X86-SSE2-NEXT:    pcmpeqb %xmm1, %xmm0
 ; X86-SSE2-NEXT:    pand %xmm2, %xmm0
 ; X86-SSE2-NEXT:    pmovmskb %xmm0, %eax
@@ -1289,13 +1565,28 @@ define i1 @length32_eq(i8* %x, i8* %y) nounwind {
 ; X86-SSE2-NEXT:    sete %al
 ; X86-SSE2-NEXT:    retl
 ;
-; X64-SSE2-LABEL: length32_eq:
+; X86-SSE41-LABEL: length31_eq:
+; X86-SSE41:       # %bb.0:
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE41-NEXT:    movdqu (%ecx), %xmm0
+; X86-SSE41-NEXT:    movdqu 15(%ecx), %xmm1
+; X86-SSE41-NEXT:    movdqu (%eax), %xmm2
+; X86-SSE41-NEXT:    pxor %xmm0, %xmm2
+; X86-SSE41-NEXT:    movdqu 15(%eax), %xmm0
+; X86-SSE41-NEXT:    pxor %xmm1, %xmm0
+; X86-SSE41-NEXT:    por %xmm2, %xmm0
+; X86-SSE41-NEXT:    ptest %xmm0, %xmm0
+; X86-SSE41-NEXT:    sete %al
+; X86-SSE41-NEXT:    retl
+;
+; X64-SSE2-LABEL: length31_eq:
 ; X64-SSE2:       # %bb.0:
 ; X64-SSE2-NEXT:    movdqu (%rdi), %xmm0
-; X64-SSE2-NEXT:    movdqu 16(%rdi), %xmm1
+; X64-SSE2-NEXT:    movdqu 15(%rdi), %xmm1
 ; X64-SSE2-NEXT:    movdqu (%rsi), %xmm2
 ; X64-SSE2-NEXT:    pcmpeqb %xmm0, %xmm2
-; X64-SSE2-NEXT:    movdqu 16(%rsi), %xmm0
+; X64-SSE2-NEXT:    movdqu 15(%rsi), %xmm0
 ; X64-SSE2-NEXT:    pcmpeqb %xmm1, %xmm0
 ; X64-SSE2-NEXT:    pand %xmm2, %xmm0
 ; X64-SSE2-NEXT:    pmovmskb %xmm0, %eax
@@ -1303,37 +1594,106 @@ define i1 @length32_eq(i8* %x, i8* %y) nounwind {
 ; X64-SSE2-NEXT:    sete %al
 ; X64-SSE2-NEXT:    retq
 ;
-; X64-AVX1-LABEL: length32_eq:
-; X64-AVX1:       # %bb.0:
-; X64-AVX1-NEXT:    vmovdqu (%rdi), %xmm0
-; X64-AVX1-NEXT:    vmovdqu 16(%rdi), %xmm1
-; X64-AVX1-NEXT:    vpcmpeqb 16(%rsi), %xmm1, %xmm1
-; X64-AVX1-NEXT:    vpcmpeqb (%rsi), %xmm0, %xmm0
-; X64-AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; X64-AVX1-NEXT:    vpmovmskb %xmm0, %eax
-; X64-AVX1-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
-; X64-AVX1-NEXT:    sete %al
-; X64-AVX1-NEXT:    retq
+; X64-SSE41-LABEL: length31_eq:
+; X64-SSE41:       # %bb.0:
+; X64-SSE41-NEXT:    movdqu (%rdi), %xmm0
+; X64-SSE41-NEXT:    movdqu 15(%rdi), %xmm1
+; X64-SSE41-NEXT:    movdqu (%rsi), %xmm2
+; X64-SSE41-NEXT:    pxor %xmm0, %xmm2
+; X64-SSE41-NEXT:    movdqu 15(%rsi), %xmm0
+; X64-SSE41-NEXT:    pxor %xmm1, %xmm0
+; X64-SSE41-NEXT:    por %xmm2, %xmm0
+; X64-SSE41-NEXT:    ptest %xmm0, %xmm0
+; X64-SSE41-NEXT:    sete %al
+; X64-SSE41-NEXT:    retq
 ;
-; X64-AVX2-LABEL: length32_eq:
-; X64-AVX2:       # %bb.0:
-; X64-AVX2-NEXT:    vmovdqu (%rdi), %ymm0
-; X64-AVX2-NEXT:    vpcmpeqb (%rsi), %ymm0, %ymm0
-; X64-AVX2-NEXT:    vpmovmskb %ymm0, %eax
-; X64-AVX2-NEXT:    cmpl $-1, %eax
-; X64-AVX2-NEXT:    sete %al
-; X64-AVX2-NEXT:    vzeroupper
-; X64-AVX2-NEXT:    retq
-  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 32) nounwind
+; X64-AVX-LABEL: length31_eq:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    vmovdqu (%rdi), %xmm0
+; X64-AVX-NEXT:    vmovdqu 15(%rdi), %xmm1
+; X64-AVX-NEXT:    vpxor 15(%rsi), %xmm1, %xmm1
+; X64-AVX-NEXT:    vpxor (%rsi), %xmm0, %xmm0
+; X64-AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; X64-AVX-NEXT:    vptest %xmm0, %xmm0
+; X64-AVX-NEXT:    sete %al
+; X64-AVX-NEXT:    retq
+;
+; X64-MIC-AVX-LABEL: length31_eq:
+; X64-MIC-AVX:       # %bb.0:
+; X64-MIC-AVX-NEXT:    vmovdqu (%rdi), %xmm0
+; X64-MIC-AVX-NEXT:    vmovdqu 15(%rdi), %xmm1
+; X64-MIC-AVX-NEXT:    vmovdqu (%rsi), %xmm2
+; X64-MIC-AVX-NEXT:    vmovdqu 15(%rsi), %xmm3
+; X64-MIC-AVX-NEXT:    vpcmpneqd %zmm3, %zmm1, %k0
+; X64-MIC-AVX-NEXT:    vpcmpneqd %zmm2, %zmm0, %k1
+; X64-MIC-AVX-NEXT:    kortestw %k0, %k1
+; X64-MIC-AVX-NEXT:    sete %al
+; X64-MIC-AVX-NEXT:    vzeroupper
+; X64-MIC-AVX-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 31) nounwind
   %cmp = icmp eq i32 %call, 0
   ret i1 %cmp
 }
 
-define i1 @length32_eq_prefer128(i8* %x, i8* %y) nounwind "prefer-vector-width"="128" {
-; X86-NOSSE-LABEL: length32_eq_prefer128:
+define i1 @length31_lt(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length31_lt:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $31
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    shrl $31, %eax
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: length31_lt:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $31, %edx
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    shrl $31, %eax
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 31) nounwind
+  %cmp = icmp slt i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length31_gt(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length31_gt:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $31
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    setg %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length31_gt:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $31, %edx
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    setg %al
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 31) nounwind
+  %cmp = icmp sgt i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length31_eq_prefer128(i8* %x, i8* %y) nounwind "prefer-vector-width"="128" {
+; X86-NOSSE-LABEL: length31_eq_prefer128:
 ; X86-NOSSE:       # %bb.0:
 ; X86-NOSSE-NEXT:    pushl $0
-; X86-NOSSE-NEXT:    pushl $32
+; X86-NOSSE-NEXT:    pushl $31
 ; X86-NOSSE-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NOSSE-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NOSSE-NEXT:    calll memcmp
@@ -1342,10 +1702,10 @@ define i1 @length32_eq_prefer128(i8* %x, i8* %y) nounwind "prefer-vector-width"=
 ; X86-NOSSE-NEXT:    sete %al
 ; X86-NOSSE-NEXT:    retl
 ;
-; X86-SSE1-LABEL: length32_eq_prefer128:
+; X86-SSE1-LABEL: length31_eq_prefer128:
 ; X86-SSE1:       # %bb.0:
 ; X86-SSE1-NEXT:    pushl $0
-; X86-SSE1-NEXT:    pushl $32
+; X86-SSE1-NEXT:    pushl $31
 ; X86-SSE1-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-SSE1-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-SSE1-NEXT:    calll memcmp
@@ -1354,15 +1714,15 @@ define i1 @length32_eq_prefer128(i8* %x, i8* %y) nounwind "prefer-vector-width"=
 ; X86-SSE1-NEXT:    sete %al
 ; X86-SSE1-NEXT:    retl
 ;
-; X86-SSE2-LABEL: length32_eq_prefer128:
+; X86-SSE2-LABEL: length31_eq_prefer128:
 ; X86-SSE2:       # %bb.0:
 ; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-SSE2-NEXT:    movdqu (%ecx), %xmm0
-; X86-SSE2-NEXT:    movdqu 16(%ecx), %xmm1
+; X86-SSE2-NEXT:    movdqu 15(%ecx), %xmm1
 ; X86-SSE2-NEXT:    movdqu (%eax), %xmm2
 ; X86-SSE2-NEXT:    pcmpeqb %xmm0, %xmm2
-; X86-SSE2-NEXT:    movdqu 16(%eax), %xmm0
+; X86-SSE2-NEXT:    movdqu 15(%eax), %xmm0
 ; X86-SSE2-NEXT:    pcmpeqb %xmm1, %xmm0
 ; X86-SSE2-NEXT:    pand %xmm2, %xmm0
 ; X86-SSE2-NEXT:    pmovmskb %xmm0, %eax
@@ -1370,13 +1730,28 @@ define i1 @length32_eq_prefer128(i8* %x, i8* %y) nounwind "prefer-vector-width"=
 ; X86-SSE2-NEXT:    sete %al
 ; X86-SSE2-NEXT:    retl
 ;
-; X64-SSE2-LABEL: length32_eq_prefer128:
+; X86-SSE41-LABEL: length31_eq_prefer128:
+; X86-SSE41:       # %bb.0:
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE41-NEXT:    movdqu (%ecx), %xmm0
+; X86-SSE41-NEXT:    movdqu 15(%ecx), %xmm1
+; X86-SSE41-NEXT:    movdqu (%eax), %xmm2
+; X86-SSE41-NEXT:    pxor %xmm0, %xmm2
+; X86-SSE41-NEXT:    movdqu 15(%eax), %xmm0
+; X86-SSE41-NEXT:    pxor %xmm1, %xmm0
+; X86-SSE41-NEXT:    por %xmm2, %xmm0
+; X86-SSE41-NEXT:    ptest %xmm0, %xmm0
+; X86-SSE41-NEXT:    sete %al
+; X86-SSE41-NEXT:    retl
+;
+; X64-SSE2-LABEL: length31_eq_prefer128:
 ; X64-SSE2:       # %bb.0:
 ; X64-SSE2-NEXT:    movdqu (%rdi), %xmm0
-; X64-SSE2-NEXT:    movdqu 16(%rdi), %xmm1
+; X64-SSE2-NEXT:    movdqu 15(%rdi), %xmm1
 ; X64-SSE2-NEXT:    movdqu (%rsi), %xmm2
 ; X64-SSE2-NEXT:    pcmpeqb %xmm0, %xmm2
-; X64-SSE2-NEXT:    movdqu 16(%rsi), %xmm0
+; X64-SSE2-NEXT:    movdqu 15(%rsi), %xmm0
 ; X64-SSE2-NEXT:    pcmpeqb %xmm1, %xmm0
 ; X64-SSE2-NEXT:    pand %xmm2, %xmm0
 ; X64-SSE2-NEXT:    pmovmskb %xmm0, %eax
@@ -1384,27 +1759,52 @@ define i1 @length32_eq_prefer128(i8* %x, i8* %y) nounwind "prefer-vector-width"=
 ; X64-SSE2-NEXT:    sete %al
 ; X64-SSE2-NEXT:    retq
 ;
-; X64-AVX-LABEL: length32_eq_prefer128:
+; X64-SSE41-LABEL: length31_eq_prefer128:
+; X64-SSE41:       # %bb.0:
+; X64-SSE41-NEXT:    movdqu (%rdi), %xmm0
+; X64-SSE41-NEXT:    movdqu 15(%rdi), %xmm1
+; X64-SSE41-NEXT:    movdqu (%rsi), %xmm2
+; X64-SSE41-NEXT:    pxor %xmm0, %xmm2
+; X64-SSE41-NEXT:    movdqu 15(%rsi), %xmm0
+; X64-SSE41-NEXT:    pxor %xmm1, %xmm0
+; X64-SSE41-NEXT:    por %xmm2, %xmm0
+; X64-SSE41-NEXT:    ptest %xmm0, %xmm0
+; X64-SSE41-NEXT:    sete %al
+; X64-SSE41-NEXT:    retq
+;
+; X64-AVX-LABEL: length31_eq_prefer128:
 ; X64-AVX:       # %bb.0:
 ; X64-AVX-NEXT:    vmovdqu (%rdi), %xmm0
-; X64-AVX-NEXT:    vmovdqu 16(%rdi), %xmm1
-; X64-AVX-NEXT:    vpcmpeqb 16(%rsi), %xmm1, %xmm1
-; X64-AVX-NEXT:    vpcmpeqb (%rsi), %xmm0, %xmm0
-; X64-AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; X64-AVX-NEXT:    vpmovmskb %xmm0, %eax
-; X64-AVX-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; X64-AVX-NEXT:    vmovdqu 15(%rdi), %xmm1
+; X64-AVX-NEXT:    vpxor 15(%rsi), %xmm1, %xmm1
+; X64-AVX-NEXT:    vpxor (%rsi), %xmm0, %xmm0
+; X64-AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; X64-AVX-NEXT:    vptest %xmm0, %xmm0
 ; X64-AVX-NEXT:    sete %al
 ; X64-AVX-NEXT:    retq
-  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 32) nounwind
+;
+; X64-MIC-AVX-LABEL: length31_eq_prefer128:
+; X64-MIC-AVX:       # %bb.0:
+; X64-MIC-AVX-NEXT:    vmovdqu (%rdi), %xmm0
+; X64-MIC-AVX-NEXT:    vmovdqu 15(%rdi), %xmm1
+; X64-MIC-AVX-NEXT:    vmovdqu (%rsi), %xmm2
+; X64-MIC-AVX-NEXT:    vmovdqu 15(%rsi), %xmm3
+; X64-MIC-AVX-NEXT:    vpcmpneqd %zmm3, %zmm1, %k0
+; X64-MIC-AVX-NEXT:    vpcmpneqd %zmm2, %zmm0, %k1
+; X64-MIC-AVX-NEXT:    kortestw %k0, %k1
+; X64-MIC-AVX-NEXT:    sete %al
+; X64-MIC-AVX-NEXT:    vzeroupper
+; X64-MIC-AVX-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 31) nounwind
   %cmp = icmp eq i32 %call, 0
   ret i1 %cmp
 }
 
-define i1 @length32_eq_const(i8* %X) nounwind {
-; X86-NOSSE-LABEL: length32_eq_const:
+define i1 @length31_eq_const(i8* %X) nounwind {
+; X86-NOSSE-LABEL: length31_eq_const:
 ; X86-NOSSE:       # %bb.0:
 ; X86-NOSSE-NEXT:    pushl $0
-; X86-NOSSE-NEXT:    pushl $32
+; X86-NOSSE-NEXT:    pushl $31
 ; X86-NOSSE-NEXT:    pushl $.L.str
 ; X86-NOSSE-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NOSSE-NEXT:    calll memcmp
@@ -1413,10 +1813,10 @@ define i1 @length32_eq_const(i8* %X) nounwind {
 ; X86-NOSSE-NEXT:    setne %al
 ; X86-NOSSE-NEXT:    retl
 ;
-; X86-SSE1-LABEL: length32_eq_const:
+; X86-SSE1-LABEL: length31_eq_const:
 ; X86-SSE1:       # %bb.0:
 ; X86-SSE1-NEXT:    pushl $0
-; X86-SSE1-NEXT:    pushl $32
+; X86-SSE1-NEXT:    pushl $31
 ; X86-SSE1-NEXT:    pushl $.L.str
 ; X86-SSE1-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-SSE1-NEXT:    calll memcmp
@@ -1425,11 +1825,11 @@ define i1 @length32_eq_const(i8* %X) nounwind {
 ; X86-SSE1-NEXT:    setne %al
 ; X86-SSE1-NEXT:    retl
 ;
-; X86-SSE2-LABEL: length32_eq_const:
+; X86-SSE2-LABEL: length31_eq_const:
 ; X86-SSE2:       # %bb.0:
 ; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-SSE2-NEXT:    movdqu (%eax), %xmm0
-; X86-SSE2-NEXT:    movdqu 16(%eax), %xmm1
+; X86-SSE2-NEXT:    movdqu 15(%eax), %xmm1
 ; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI.*}}, %xmm1
 ; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI.*}}, %xmm0
 ; X86-SSE2-NEXT:    pand %xmm1, %xmm0
@@ -1438,10 +1838,22 @@ define i1 @length32_eq_const(i8* %X) nounwind {
 ; X86-SSE2-NEXT:    setne %al
 ; X86-SSE2-NEXT:    retl
 ;
-; X64-SSE2-LABEL: length32_eq_const:
+; X86-SSE41-LABEL: length31_eq_const:
+; X86-SSE41:       # %bb.0:
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT:    movdqu (%eax), %xmm0
+; X86-SSE41-NEXT:    movdqu 15(%eax), %xmm1
+; X86-SSE41-NEXT:    pxor {{\.LCPI.*}}, %xmm1
+; X86-SSE41-NEXT:    pxor {{\.LCPI.*}}, %xmm0
+; X86-SSE41-NEXT:    por %xmm1, %xmm0
+; X86-SSE41-NEXT:    ptest %xmm0, %xmm0
+; X86-SSE41-NEXT:    setne %al
+; X86-SSE41-NEXT:    retl
+;
+; X64-SSE2-LABEL: length31_eq_const:
 ; X64-SSE2:       # %bb.0:
 ; X64-SSE2-NEXT:    movdqu (%rdi), %xmm0
-; X64-SSE2-NEXT:    movdqu 16(%rdi), %xmm1
+; X64-SSE2-NEXT:    movdqu 15(%rdi), %xmm1
 ; X64-SSE2-NEXT:    pcmpeqb {{.*}}(%rip), %xmm1
 ; X64-SSE2-NEXT:    pcmpeqb {{.*}}(%rip), %xmm0
 ; X64-SSE2-NEXT:    pand %xmm1, %xmm0
@@ -1450,106 +1862,2598 @@ define i1 @length32_eq_const(i8* %X) nounwind {
 ; X64-SSE2-NEXT:    setne %al
 ; X64-SSE2-NEXT:    retq
 ;
-; X64-AVX1-LABEL: length32_eq_const:
-; X64-AVX1:       # %bb.0:
-; X64-AVX1-NEXT:    vmovdqu (%rdi), %xmm0
-; X64-AVX1-NEXT:    vmovdqu 16(%rdi), %xmm1
-; X64-AVX1-NEXT:    vpcmpeqb {{.*}}(%rip), %xmm1, %xmm1
-; X64-AVX1-NEXT:    vpcmpeqb {{.*}}(%rip), %xmm0, %xmm0
-; X64-AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; X64-AVX1-NEXT:    vpmovmskb %xmm0, %eax
-; X64-AVX1-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
-; X64-AVX1-NEXT:    setne %al
-; X64-AVX1-NEXT:    retq
+; X64-SSE41-LABEL: length31_eq_const:
+; X64-SSE41:       # %bb.0:
+; X64-SSE41-NEXT:    movdqu (%rdi), %xmm0
+; X64-SSE41-NEXT:    movdqu 15(%rdi), %xmm1
+; X64-SSE41-NEXT:    pxor {{.*}}(%rip), %xmm1
+; X64-SSE41-NEXT:    pxor {{.*}}(%rip), %xmm0
+; X64-SSE41-NEXT:    por %xmm1, %xmm0
+; X64-SSE41-NEXT:    ptest %xmm0, %xmm0
+; X64-SSE41-NEXT:    setne %al
+; X64-SSE41-NEXT:    retq
 ;
-; X64-AVX2-LABEL: length32_eq_const:
-; X64-AVX2:       # %bb.0:
-; X64-AVX2-NEXT:    vmovdqu (%rdi), %ymm0
-; X64-AVX2-NEXT:    vpcmpeqb {{.*}}(%rip), %ymm0, %ymm0
-; X64-AVX2-NEXT:    vpmovmskb %ymm0, %eax
-; X64-AVX2-NEXT:    cmpl $-1, %eax
-; X64-AVX2-NEXT:    setne %al
-; X64-AVX2-NEXT:    vzeroupper
-; X64-AVX2-NEXT:    retq
-  %m = tail call i32 @memcmp(i8* %X, i8* getelementptr inbounds ([65 x i8], [65 x i8]* @.str, i32 0, i32 0), i64 32) nounwind
+; X64-AVX-LABEL: length31_eq_const:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    vmovdqu (%rdi), %xmm0
+; X64-AVX-NEXT:    vmovdqu 15(%rdi), %xmm1
+; X64-AVX-NEXT:    vpxor {{.*}}(%rip), %xmm1, %xmm1
+; X64-AVX-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; X64-AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; X64-AVX-NEXT:    vptest %xmm0, %xmm0
+; X64-AVX-NEXT:    setne %al
+; X64-AVX-NEXT:    retq
+;
+; X64-MIC-AVX-LABEL: length31_eq_const:
+; X64-MIC-AVX:       # %bb.0:
+; X64-MIC-AVX-NEXT:    vmovdqu (%rdi), %xmm0
+; X64-MIC-AVX-NEXT:    vmovdqu 15(%rdi), %xmm1
+; X64-MIC-AVX-NEXT:    vmovdqa {{.*#+}} xmm2 = [943142453,842084409,909456435,809056311]
+; X64-MIC-AVX-NEXT:    vpcmpneqd %zmm2, %zmm1, %k0
+; X64-MIC-AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [858927408,926299444,825243960,892613426]
+; X64-MIC-AVX-NEXT:    vpcmpneqd %zmm1, %zmm0, %k1
+; X64-MIC-AVX-NEXT:    kortestw %k0, %k1
+; X64-MIC-AVX-NEXT:    setne %al
+; X64-MIC-AVX-NEXT:    vzeroupper
+; X64-MIC-AVX-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* getelementptr inbounds ([513 x i8], [513 x i8]* @.str, i32 0, i32 0), i64 31) nounwind
   %c = icmp ne i32 %m, 0
   ret i1 %c
 }
 
-define i32 @length64(i8* %X, i8* %Y) nounwind {
-; X86-LABEL: length64:
+define i32 @length32(i8* %X, i8* %Y) nounwind {
+; X86-LABEL: length32:
 ; X86:       # %bb.0:
 ; X86-NEXT:    pushl $0
-; X86-NEXT:    pushl $64
+; X86-NEXT:    pushl $32
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    calll memcmp
 ; X86-NEXT:    addl $16, %esp
 ; X86-NEXT:    retl
 ;
-; X64-LABEL: length64:
+; X64-LABEL: length32:
 ; X64:       # %bb.0:
-; X64-NEXT:    movl $64, %edx
+; X64-NEXT:    movl $32, %edx
 ; X64-NEXT:    jmp memcmp # TAILCALL
-  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 64) nounwind
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 32) nounwind
   ret i32 %m
 }
 
-define i1 @length64_eq(i8* %x, i8* %y) nounwind {
-; X86-LABEL: length64_eq:
-; X86:       # %bb.0:
-; X86-NEXT:    pushl $0
-; X86-NEXT:    pushl $64
-; X86-NEXT:    pushl {{[0-9]+}}(%esp)
-; X86-NEXT:    pushl {{[0-9]+}}(%esp)
-; X86-NEXT:    calll memcmp
+; PR33325 - https://bugs.llvm.org/show_bug.cgi?id=33325
+
+define i1 @length32_eq(i8* %x, i8* %y) nounwind {
+; X86-NOSSE-LABEL: length32_eq:
+; X86-NOSSE:       # %bb.0:
+; X86-NOSSE-NEXT:    pushl $0
+; X86-NOSSE-NEXT:    pushl $32
+; X86-NOSSE-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    calll memcmp
+; X86-NOSSE-NEXT:    addl $16, %esp
+; X86-NOSSE-NEXT:    testl %eax, %eax
+; X86-NOSSE-NEXT:    sete %al
+; X86-NOSSE-NEXT:    retl
+;
+; X86-SSE1-LABEL: length32_eq:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    pushl $0
+; X86-SSE1-NEXT:    pushl $32
+; X86-SSE1-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    calll memcmp
+; X86-SSE1-NEXT:    addl $16, %esp
+; X86-SSE1-NEXT:    testl %eax, %eax
+; X86-SSE1-NEXT:    sete %al
+; X86-SSE1-NEXT:    retl
+;
+; X86-SSE2-LABEL: length32_eq:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE2-NEXT:    movdqu (%ecx), %xmm0
+; X86-SSE2-NEXT:    movdqu 16(%ecx), %xmm1
+; X86-SSE2-NEXT:    movdqu (%eax), %xmm2
+; X86-SSE2-NEXT:    pcmpeqb %xmm0, %xmm2
+; X86-SSE2-NEXT:    movdqu 16(%eax), %xmm0
+; X86-SSE2-NEXT:    pcmpeqb %xmm1, %xmm0
+; X86-SSE2-NEXT:    pand %xmm2, %xmm0
+; X86-SSE2-NEXT:    pmovmskb %xmm0, %eax
+; X86-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; X86-SSE2-NEXT:    sete %al
+; X86-SSE2-NEXT:    retl
+;
+; X86-SSE41-LABEL: length32_eq:
+; X86-SSE41:       # %bb.0:
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE41-NEXT:    movdqu (%ecx), %xmm0
+; X86-SSE41-NEXT:    movdqu 16(%ecx), %xmm1
+; X86-SSE41-NEXT:    movdqu (%eax), %xmm2
+; X86-SSE41-NEXT:    pxor %xmm0, %xmm2
+; X86-SSE41-NEXT:    movdqu 16(%eax), %xmm0
+; X86-SSE41-NEXT:    pxor %xmm1, %xmm0
+; X86-SSE41-NEXT:    por %xmm2, %xmm0
+; X86-SSE41-NEXT:    ptest %xmm0, %xmm0
+; X86-SSE41-NEXT:    sete %al
+; X86-SSE41-NEXT:    retl
+;
+; X64-SSE2-LABEL: length32_eq:
+; X64-SSE2:       # %bb.0:
+; X64-SSE2-NEXT:    movdqu (%rdi), %xmm0
+; X64-SSE2-NEXT:    movdqu 16(%rdi), %xmm1
+; X64-SSE2-NEXT:    movdqu (%rsi), %xmm2
+; X64-SSE2-NEXT:    pcmpeqb %xmm0, %xmm2
+; X64-SSE2-NEXT:    movdqu 16(%rsi), %xmm0
+; X64-SSE2-NEXT:    pcmpeqb %xmm1, %xmm0
+; X64-SSE2-NEXT:    pand %xmm2, %xmm0
+; X64-SSE2-NEXT:    pmovmskb %xmm0, %eax
+; X64-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; X64-SSE2-NEXT:    sete %al
+; X64-SSE2-NEXT:    retq
+;
+; X64-SSE41-LABEL: length32_eq:
+; X64-SSE41:       # %bb.0:
+; X64-SSE41-NEXT:    movdqu (%rdi), %xmm0
+; X64-SSE41-NEXT:    movdqu 16(%rdi), %xmm1
+; X64-SSE41-NEXT:    movdqu (%rsi), %xmm2
+; X64-SSE41-NEXT:    pxor %xmm0, %xmm2
+; X64-SSE41-NEXT:    movdqu 16(%rsi), %xmm0
+; X64-SSE41-NEXT:    pxor %xmm1, %xmm0
+; X64-SSE41-NEXT:    por %xmm2, %xmm0
+; X64-SSE41-NEXT:    ptest %xmm0, %xmm0
+; X64-SSE41-NEXT:    sete %al
+; X64-SSE41-NEXT:    retq
+;
+; X64-AVX1-LABEL: length32_eq:
+; X64-AVX1:       # %bb.0:
+; X64-AVX1-NEXT:    vmovups (%rdi), %ymm0
+; X64-AVX1-NEXT:    vxorps (%rsi), %ymm0, %ymm0
+; X64-AVX1-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX1-NEXT:    sete %al
+; X64-AVX1-NEXT:    vzeroupper
+; X64-AVX1-NEXT:    retq
+;
+; X64-AVX2-LABEL: length32_eq:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    vmovdqu (%rdi), %ymm0
+; X64-AVX2-NEXT:    vpxor (%rsi), %ymm0, %ymm0
+; X64-AVX2-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX2-NEXT:    sete %al
+; X64-AVX2-NEXT:    vzeroupper
+; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512-LABEL: length32_eq:
+; X64-AVX512:       # %bb.0:
+; X64-AVX512-NEXT:    vmovdqu (%rdi), %ymm0
+; X64-AVX512-NEXT:    vpxor (%rsi), %ymm0, %ymm0
+; X64-AVX512-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX512-NEXT:    sete %al
+; X64-AVX512-NEXT:    vzeroupper
+; X64-AVX512-NEXT:    retq
+;
+; X64-MIC-AVX-LABEL: length32_eq:
+; X64-MIC-AVX:       # %bb.0:
+; X64-MIC-AVX-NEXT:    vmovdqu (%rdi), %ymm0
+; X64-MIC-AVX-NEXT:    vmovdqu (%rsi), %ymm1
+; X64-MIC-AVX-NEXT:    vpcmpneqd %zmm1, %zmm0, %k0
+; X64-MIC-AVX-NEXT:    kortestw %k0, %k0
+; X64-MIC-AVX-NEXT:    sete %al
+; X64-MIC-AVX-NEXT:    vzeroupper
+; X64-MIC-AVX-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 32) nounwind
+  %cmp = icmp eq i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length32_lt(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length32_lt:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $32
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    shrl $31, %eax
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: length32_lt:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $32, %edx
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    shrl $31, %eax
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 32) nounwind
+  %cmp = icmp slt i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length32_gt(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length32_gt:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $32
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    setg %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length32_gt:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $32, %edx
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    setg %al
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 32) nounwind
+  %cmp = icmp sgt i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length32_eq_prefer128(i8* %x, i8* %y) nounwind "prefer-vector-width"="128" {
+; X86-NOSSE-LABEL: length32_eq_prefer128:
+; X86-NOSSE:       # %bb.0:
+; X86-NOSSE-NEXT:    pushl $0
+; X86-NOSSE-NEXT:    pushl $32
+; X86-NOSSE-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    calll memcmp
+; X86-NOSSE-NEXT:    addl $16, %esp
+; X86-NOSSE-NEXT:    testl %eax, %eax
+; X86-NOSSE-NEXT:    sete %al
+; X86-NOSSE-NEXT:    retl
+;
+; X86-SSE1-LABEL: length32_eq_prefer128:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    pushl $0
+; X86-SSE1-NEXT:    pushl $32
+; X86-SSE1-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    calll memcmp
+; X86-SSE1-NEXT:    addl $16, %esp
+; X86-SSE1-NEXT:    testl %eax, %eax
+; X86-SSE1-NEXT:    sete %al
+; X86-SSE1-NEXT:    retl
+;
+; X86-SSE2-LABEL: length32_eq_prefer128:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE2-NEXT:    movdqu (%ecx), %xmm0
+; X86-SSE2-NEXT:    movdqu 16(%ecx), %xmm1
+; X86-SSE2-NEXT:    movdqu (%eax), %xmm2
+; X86-SSE2-NEXT:    pcmpeqb %xmm0, %xmm2
+; X86-SSE2-NEXT:    movdqu 16(%eax), %xmm0
+; X86-SSE2-NEXT:    pcmpeqb %xmm1, %xmm0
+; X86-SSE2-NEXT:    pand %xmm2, %xmm0
+; X86-SSE2-NEXT:    pmovmskb %xmm0, %eax
+; X86-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; X86-SSE2-NEXT:    sete %al
+; X86-SSE2-NEXT:    retl
+;
+; X86-SSE41-LABEL: length32_eq_prefer128:
+; X86-SSE41:       # %bb.0:
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE41-NEXT:    movdqu (%ecx), %xmm0
+; X86-SSE41-NEXT:    movdqu 16(%ecx), %xmm1
+; X86-SSE41-NEXT:    movdqu (%eax), %xmm2
+; X86-SSE41-NEXT:    pxor %xmm0, %xmm2
+; X86-SSE41-NEXT:    movdqu 16(%eax), %xmm0
+; X86-SSE41-NEXT:    pxor %xmm1, %xmm0
+; X86-SSE41-NEXT:    por %xmm2, %xmm0
+; X86-SSE41-NEXT:    ptest %xmm0, %xmm0
+; X86-SSE41-NEXT:    sete %al
+; X86-SSE41-NEXT:    retl
+;
+; X64-SSE2-LABEL: length32_eq_prefer128:
+; X64-SSE2:       # %bb.0:
+; X64-SSE2-NEXT:    movdqu (%rdi), %xmm0
+; X64-SSE2-NEXT:    movdqu 16(%rdi), %xmm1
+; X64-SSE2-NEXT:    movdqu (%rsi), %xmm2
+; X64-SSE2-NEXT:    pcmpeqb %xmm0, %xmm2
+; X64-SSE2-NEXT:    movdqu 16(%rsi), %xmm0
+; X64-SSE2-NEXT:    pcmpeqb %xmm1, %xmm0
+; X64-SSE2-NEXT:    pand %xmm2, %xmm0
+; X64-SSE2-NEXT:    pmovmskb %xmm0, %eax
+; X64-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; X64-SSE2-NEXT:    sete %al
+; X64-SSE2-NEXT:    retq
+;
+; X64-SSE41-LABEL: length32_eq_prefer128:
+; X64-SSE41:       # %bb.0:
+; X64-SSE41-NEXT:    movdqu (%rdi), %xmm0
+; X64-SSE41-NEXT:    movdqu 16(%rdi), %xmm1
+; X64-SSE41-NEXT:    movdqu (%rsi), %xmm2
+; X64-SSE41-NEXT:    pxor %xmm0, %xmm2
+; X64-SSE41-NEXT:    movdqu 16(%rsi), %xmm0
+; X64-SSE41-NEXT:    pxor %xmm1, %xmm0
+; X64-SSE41-NEXT:    por %xmm2, %xmm0
+; X64-SSE41-NEXT:    ptest %xmm0, %xmm0
+; X64-SSE41-NEXT:    sete %al
+; X64-SSE41-NEXT:    retq
+;
+; X64-AVX-LABEL: length32_eq_prefer128:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    vmovdqu (%rdi), %xmm0
+; X64-AVX-NEXT:    vmovdqu 16(%rdi), %xmm1
+; X64-AVX-NEXT:    vpxor 16(%rsi), %xmm1, %xmm1
+; X64-AVX-NEXT:    vpxor (%rsi), %xmm0, %xmm0
+; X64-AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; X64-AVX-NEXT:    vptest %xmm0, %xmm0
+; X64-AVX-NEXT:    sete %al
+; X64-AVX-NEXT:    retq
+;
+; X64-MIC-AVX-LABEL: length32_eq_prefer128:
+; X64-MIC-AVX:       # %bb.0:
+; X64-MIC-AVX-NEXT:    vmovdqu (%rdi), %xmm0
+; X64-MIC-AVX-NEXT:    vmovdqu 16(%rdi), %xmm1
+; X64-MIC-AVX-NEXT:    vmovdqu (%rsi), %xmm2
+; X64-MIC-AVX-NEXT:    vmovdqu 16(%rsi), %xmm3
+; X64-MIC-AVX-NEXT:    vpcmpneqd %zmm3, %zmm1, %k0
+; X64-MIC-AVX-NEXT:    vpcmpneqd %zmm2, %zmm0, %k1
+; X64-MIC-AVX-NEXT:    kortestw %k0, %k1
+; X64-MIC-AVX-NEXT:    sete %al
+; X64-MIC-AVX-NEXT:    vzeroupper
+; X64-MIC-AVX-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 32) nounwind
+  %cmp = icmp eq i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length32_eq_const(i8* %X) nounwind {
+; X86-NOSSE-LABEL: length32_eq_const:
+; X86-NOSSE:       # %bb.0:
+; X86-NOSSE-NEXT:    pushl $0
+; X86-NOSSE-NEXT:    pushl $32
+; X86-NOSSE-NEXT:    pushl $.L.str
+; X86-NOSSE-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    calll memcmp
+; X86-NOSSE-NEXT:    addl $16, %esp
+; X86-NOSSE-NEXT:    testl %eax, %eax
+; X86-NOSSE-NEXT:    setne %al
+; X86-NOSSE-NEXT:    retl
+;
+; X86-SSE1-LABEL: length32_eq_const:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    pushl $0
+; X86-SSE1-NEXT:    pushl $32
+; X86-SSE1-NEXT:    pushl $.L.str
+; X86-SSE1-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    calll memcmp
+; X86-SSE1-NEXT:    addl $16, %esp
+; X86-SSE1-NEXT:    testl %eax, %eax
+; X86-SSE1-NEXT:    setne %al
+; X86-SSE1-NEXT:    retl
+;
+; X86-SSE2-LABEL: length32_eq_const:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT:    movdqu (%eax), %xmm0
+; X86-SSE2-NEXT:    movdqu 16(%eax), %xmm1
+; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI.*}}, %xmm1
+; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand %xmm1, %xmm0
+; X86-SSE2-NEXT:    pmovmskb %xmm0, %eax
+; X86-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; X86-SSE2-NEXT:    setne %al
+; X86-SSE2-NEXT:    retl
+;
+; X86-SSE41-LABEL: length32_eq_const:
+; X86-SSE41:       # %bb.0:
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT:    movdqu (%eax), %xmm0
+; X86-SSE41-NEXT:    movdqu 16(%eax), %xmm1
+; X86-SSE41-NEXT:    pxor {{\.LCPI.*}}, %xmm1
+; X86-SSE41-NEXT:    pxor {{\.LCPI.*}}, %xmm0
+; X86-SSE41-NEXT:    por %xmm1, %xmm0
+; X86-SSE41-NEXT:    ptest %xmm0, %xmm0
+; X86-SSE41-NEXT:    setne %al
+; X86-SSE41-NEXT:    retl
+;
+; X64-SSE2-LABEL: length32_eq_const:
+; X64-SSE2:       # %bb.0:
+; X64-SSE2-NEXT:    movdqu (%rdi), %xmm0
+; X64-SSE2-NEXT:    movdqu 16(%rdi), %xmm1
+; X64-SSE2-NEXT:    pcmpeqb {{.*}}(%rip), %xmm1
+; X64-SSE2-NEXT:    pcmpeqb {{.*}}(%rip), %xmm0
+; X64-SSE2-NEXT:    pand %xmm1, %xmm0
+; X64-SSE2-NEXT:    pmovmskb %xmm0, %eax
+; X64-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; X64-SSE2-NEXT:    setne %al
+; X64-SSE2-NEXT:    retq
+;
+; X64-SSE41-LABEL: length32_eq_const:
+; X64-SSE41:       # %bb.0:
+; X64-SSE41-NEXT:    movdqu (%rdi), %xmm0
+; X64-SSE41-NEXT:    movdqu 16(%rdi), %xmm1
+; X64-SSE41-NEXT:    pxor {{.*}}(%rip), %xmm1
+; X64-SSE41-NEXT:    pxor {{.*}}(%rip), %xmm0
+; X64-SSE41-NEXT:    por %xmm1, %xmm0
+; X64-SSE41-NEXT:    ptest %xmm0, %xmm0
+; X64-SSE41-NEXT:    setne %al
+; X64-SSE41-NEXT:    retq
+;
+; X64-AVX1-LABEL: length32_eq_const:
+; X64-AVX1:       # %bb.0:
+; X64-AVX1-NEXT:    vmovups (%rdi), %ymm0
+; X64-AVX1-NEXT:    vxorps {{.*}}(%rip), %ymm0, %ymm0
+; X64-AVX1-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX1-NEXT:    setne %al
+; X64-AVX1-NEXT:    vzeroupper
+; X64-AVX1-NEXT:    retq
+;
+; X64-AVX2-LABEL: length32_eq_const:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    vmovdqu (%rdi), %ymm0
+; X64-AVX2-NEXT:    vpxor {{.*}}(%rip), %ymm0, %ymm0
+; X64-AVX2-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX2-NEXT:    setne %al
+; X64-AVX2-NEXT:    vzeroupper
+; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512-LABEL: length32_eq_const:
+; X64-AVX512:       # %bb.0:
+; X64-AVX512-NEXT:    vmovdqu (%rdi), %ymm0
+; X64-AVX512-NEXT:    vpxor {{.*}}(%rip), %ymm0, %ymm0
+; X64-AVX512-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX512-NEXT:    setne %al
+; X64-AVX512-NEXT:    vzeroupper
+; X64-AVX512-NEXT:    retq
+;
+; X64-MIC-AVX-LABEL: length32_eq_const:
+; X64-MIC-AVX:       # %bb.0:
+; X64-MIC-AVX-NEXT:    vmovdqu (%rdi), %ymm0
+; X64-MIC-AVX-NEXT:    vmovdqa {{.*#+}} ymm1 = [858927408,926299444,825243960,892613426,959985462,858927408,926299444,825243960]
+; X64-MIC-AVX-NEXT:    vpcmpneqd %zmm1, %zmm0, %k0
+; X64-MIC-AVX-NEXT:    kortestw %k0, %k0
+; X64-MIC-AVX-NEXT:    setne %al
+; X64-MIC-AVX-NEXT:    vzeroupper
+; X64-MIC-AVX-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* getelementptr inbounds ([513 x i8], [513 x i8]* @.str, i32 0, i32 0), i64 32) nounwind
+  %c = icmp ne i32 %m, 0
+  ret i1 %c
+}
+
+define i32 @length48(i8* %X, i8* %Y) nounwind {
+; X86-LABEL: length48:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $48
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    retl
+;
+; X64-LABEL: length48:
+; X64:       # %bb.0:
+; X64-NEXT:    movl $48, %edx
+; X64-NEXT:    jmp memcmp # TAILCALL
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 48) nounwind
+  ret i32 %m
+}
+
+define i1 @length48_eq(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length48_eq:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $48
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    sete %al
+; X86-NEXT:    retl
+;
+; X64-SSE-LABEL: length48_eq:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    pushq %rax
+; X64-SSE-NEXT:    movl $48, %edx
+; X64-SSE-NEXT:    callq memcmp
+; X64-SSE-NEXT:    testl %eax, %eax
+; X64-SSE-NEXT:    sete %al
+; X64-SSE-NEXT:    popq %rcx
+; X64-SSE-NEXT:    retq
+;
+; X64-AVX1-LABEL: length48_eq:
+; X64-AVX1:       # %bb.0:
+; X64-AVX1-NEXT:    vmovups (%rdi), %ymm0
+; X64-AVX1-NEXT:    vmovups 32(%rdi), %xmm1
+; X64-AVX1-NEXT:    vmovups 32(%rsi), %xmm2
+; X64-AVX1-NEXT:    vxorps (%rsi), %ymm0, %ymm0
+; X64-AVX1-NEXT:    vxorps %ymm2, %ymm1, %ymm1
+; X64-AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; X64-AVX1-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX1-NEXT:    sete %al
+; X64-AVX1-NEXT:    vzeroupper
+; X64-AVX1-NEXT:    retq
+;
+; X64-AVX2-LABEL: length48_eq:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    vmovdqu (%rdi), %ymm0
+; X64-AVX2-NEXT:    vmovdqu 32(%rdi), %xmm1
+; X64-AVX2-NEXT:    vmovdqu 32(%rsi), %xmm2
+; X64-AVX2-NEXT:    vpxor (%rsi), %ymm0, %ymm0
+; X64-AVX2-NEXT:    vpxor %ymm2, %ymm1, %ymm1
+; X64-AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; X64-AVX2-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX2-NEXT:    sete %al
+; X64-AVX2-NEXT:    vzeroupper
+; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512-LABEL: length48_eq:
+; X64-AVX512:       # %bb.0:
+; X64-AVX512-NEXT:    vmovdqu (%rdi), %ymm0
+; X64-AVX512-NEXT:    vmovdqu 32(%rdi), %xmm1
+; X64-AVX512-NEXT:    vmovdqu 32(%rsi), %xmm2
+; X64-AVX512-NEXT:    vpxor (%rsi), %ymm0, %ymm0
+; X64-AVX512-NEXT:    vpxor %ymm2, %ymm1, %ymm1
+; X64-AVX512-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; X64-AVX512-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX512-NEXT:    sete %al
+; X64-AVX512-NEXT:    vzeroupper
+; X64-AVX512-NEXT:    retq
+;
+; X64-MIC-AVX-LABEL: length48_eq:
+; X64-MIC-AVX:       # %bb.0:
+; X64-MIC-AVX-NEXT:    vmovdqu (%rdi), %ymm0
+; X64-MIC-AVX-NEXT:    vmovdqu (%rsi), %ymm1
+; X64-MIC-AVX-NEXT:    vmovdqu 32(%rdi), %xmm2
+; X64-MIC-AVX-NEXT:    vmovdqu 32(%rsi), %xmm3
+; X64-MIC-AVX-NEXT:    vpcmpneqd %zmm3, %zmm2, %k0
+; X64-MIC-AVX-NEXT:    vpcmpneqd %zmm1, %zmm0, %k1
+; X64-MIC-AVX-NEXT:    kortestw %k0, %k1
+; X64-MIC-AVX-NEXT:    sete %al
+; X64-MIC-AVX-NEXT:    vzeroupper
+; X64-MIC-AVX-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 48) nounwind
+  %cmp = icmp eq i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length48_lt(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length48_lt:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $48
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    shrl $31, %eax
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: length48_lt:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $48, %edx
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    shrl $31, %eax
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 48) nounwind
+  %cmp = icmp slt i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length48_gt(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length48_gt:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $48
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    setg %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length48_gt:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $48, %edx
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    setg %al
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 48) nounwind
+  %cmp = icmp sgt i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length48_eq_prefer128(i8* %x, i8* %y) nounwind "prefer-vector-width"="128" {
+; X86-LABEL: length48_eq_prefer128:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $48
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    sete %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length48_eq_prefer128:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $48, %edx
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    sete %al
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 48) nounwind
+  %cmp = icmp eq i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length48_eq_const(i8* %X) nounwind {
+; X86-LABEL: length48_eq_const:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $48
+; X86-NEXT:    pushl $.L.str
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    setne %al
+; X86-NEXT:    retl
+;
+; X64-SSE-LABEL: length48_eq_const:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    pushq %rax
+; X64-SSE-NEXT:    movl $.L.str, %esi
+; X64-SSE-NEXT:    movl $48, %edx
+; X64-SSE-NEXT:    callq memcmp
+; X64-SSE-NEXT:    testl %eax, %eax
+; X64-SSE-NEXT:    setne %al
+; X64-SSE-NEXT:    popq %rcx
+; X64-SSE-NEXT:    retq
+;
+; X64-AVX1-LABEL: length48_eq_const:
+; X64-AVX1:       # %bb.0:
+; X64-AVX1-NEXT:    vmovups (%rdi), %ymm0
+; X64-AVX1-NEXT:    vmovups 32(%rdi), %xmm1
+; X64-AVX1-NEXT:    vxorps {{.*}}(%rip), %ymm0, %ymm0
+; X64-AVX1-NEXT:    vxorps {{.*}}(%rip), %ymm1, %ymm1
+; X64-AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; X64-AVX1-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX1-NEXT:    setne %al
+; X64-AVX1-NEXT:    vzeroupper
+; X64-AVX1-NEXT:    retq
+;
+; X64-AVX2-LABEL: length48_eq_const:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    vmovdqu (%rdi), %ymm0
+; X64-AVX2-NEXT:    vmovdqu 32(%rdi), %xmm1
+; X64-AVX2-NEXT:    vpxor {{.*}}(%rip), %ymm0, %ymm0
+; X64-AVX2-NEXT:    vpxor {{.*}}(%rip), %ymm1, %ymm1
+; X64-AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; X64-AVX2-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX2-NEXT:    setne %al
+; X64-AVX2-NEXT:    vzeroupper
+; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512-LABEL: length48_eq_const:
+; X64-AVX512:       # %bb.0:
+; X64-AVX512-NEXT:    vmovdqu (%rdi), %ymm0
+; X64-AVX512-NEXT:    vmovdqu 32(%rdi), %xmm1
+; X64-AVX512-NEXT:    vpxor {{.*}}(%rip), %ymm0, %ymm0
+; X64-AVX512-NEXT:    vpxor {{.*}}(%rip), %ymm1, %ymm1
+; X64-AVX512-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; X64-AVX512-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX512-NEXT:    setne %al
+; X64-AVX512-NEXT:    vzeroupper
+; X64-AVX512-NEXT:    retq
+;
+; X64-MIC-AVX-LABEL: length48_eq_const:
+; X64-MIC-AVX:       # %bb.0:
+; X64-MIC-AVX-NEXT:    vmovdqu (%rdi), %ymm0
+; X64-MIC-AVX-NEXT:    vmovdqu 32(%rdi), %xmm1
+; X64-MIC-AVX-NEXT:    vmovdqa {{.*#+}} ymm2 = [892613426,959985462,858927408,926299444,0,0,0,0]
+; X64-MIC-AVX-NEXT:    vpcmpneqd %zmm2, %zmm1, %k0
+; X64-MIC-AVX-NEXT:    vmovdqa {{.*#+}} ymm1 = [858927408,926299444,825243960,892613426,959985462,858927408,926299444,825243960]
+; X64-MIC-AVX-NEXT:    vpcmpneqd %zmm1, %zmm0, %k1
+; X64-MIC-AVX-NEXT:    kortestw %k0, %k1
+; X64-MIC-AVX-NEXT:    setne %al
+; X64-MIC-AVX-NEXT:    vzeroupper
+; X64-MIC-AVX-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* getelementptr inbounds ([513 x i8], [513 x i8]* @.str, i32 0, i32 0), i64 48) nounwind
+  %c = icmp ne i32 %m, 0
+  ret i1 %c
+}
+
+define i32 @length63(i8* %X, i8* %Y) nounwind {
+; X86-LABEL: length63:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $63
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    retl
+;
+; X64-LABEL: length63:
+; X64:       # %bb.0:
+; X64-NEXT:    movl $63, %edx
+; X64-NEXT:    jmp memcmp # TAILCALL
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 63) nounwind
+  ret i32 %m
+}
+
+define i1 @length63_eq(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length63_eq:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $63
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    setne %al
+; X86-NEXT:    retl
+;
+; X64-SSE-LABEL: length63_eq:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    pushq %rax
+; X64-SSE-NEXT:    movl $63, %edx
+; X64-SSE-NEXT:    callq memcmp
+; X64-SSE-NEXT:    testl %eax, %eax
+; X64-SSE-NEXT:    setne %al
+; X64-SSE-NEXT:    popq %rcx
+; X64-SSE-NEXT:    retq
+;
+; X64-AVX1-LABEL: length63_eq:
+; X64-AVX1:       # %bb.0:
+; X64-AVX1-NEXT:    vmovups (%rdi), %ymm0
+; X64-AVX1-NEXT:    vmovups 31(%rdi), %ymm1
+; X64-AVX1-NEXT:    vxorps 31(%rsi), %ymm1, %ymm1
+; X64-AVX1-NEXT:    vxorps (%rsi), %ymm0, %ymm0
+; X64-AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; X64-AVX1-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX1-NEXT:    setne %al
+; X64-AVX1-NEXT:    vzeroupper
+; X64-AVX1-NEXT:    retq
+;
+; X64-AVX2-LABEL: length63_eq:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    vmovdqu (%rdi), %ymm0
+; X64-AVX2-NEXT:    vmovdqu 31(%rdi), %ymm1
+; X64-AVX2-NEXT:    vpxor 31(%rsi), %ymm1, %ymm1
+; X64-AVX2-NEXT:    vpxor (%rsi), %ymm0, %ymm0
+; X64-AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; X64-AVX2-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX2-NEXT:    setne %al
+; X64-AVX2-NEXT:    vzeroupper
+; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512-LABEL: length63_eq:
+; X64-AVX512:       # %bb.0:
+; X64-AVX512-NEXT:    vmovdqu (%rdi), %ymm0
+; X64-AVX512-NEXT:    vmovdqu 31(%rdi), %ymm1
+; X64-AVX512-NEXT:    vpxor 31(%rsi), %ymm1, %ymm1
+; X64-AVX512-NEXT:    vpxor (%rsi), %ymm0, %ymm0
+; X64-AVX512-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; X64-AVX512-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX512-NEXT:    setne %al
+; X64-AVX512-NEXT:    vzeroupper
+; X64-AVX512-NEXT:    retq
+;
+; X64-MIC-AVX-LABEL: length63_eq:
+; X64-MIC-AVX:       # %bb.0:
+; X64-MIC-AVX-NEXT:    vmovdqu (%rdi), %ymm0
+; X64-MIC-AVX-NEXT:    vmovdqu 31(%rdi), %ymm1
+; X64-MIC-AVX-NEXT:    vmovdqu (%rsi), %ymm2
+; X64-MIC-AVX-NEXT:    vmovdqu 31(%rsi), %ymm3
+; X64-MIC-AVX-NEXT:    vpcmpneqd %zmm3, %zmm1, %k0
+; X64-MIC-AVX-NEXT:    vpcmpneqd %zmm2, %zmm0, %k1
+; X64-MIC-AVX-NEXT:    kortestw %k0, %k1
+; X64-MIC-AVX-NEXT:    setne %al
+; X64-MIC-AVX-NEXT:    vzeroupper
+; X64-MIC-AVX-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 63) nounwind
+  %cmp = icmp ne i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length63_lt(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length63_lt:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $63
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    shrl $31, %eax
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: length63_lt:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $63, %edx
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    shrl $31, %eax
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 63) nounwind
+  %cmp = icmp slt i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length63_gt(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length63_gt:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $63
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    setg %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length63_gt:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $63, %edx
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    setg %al
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 63) nounwind
+  %cmp = icmp sgt i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length63_eq_const(i8* %X) nounwind {
+; X86-LABEL: length63_eq_const:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $63
+; X86-NEXT:    pushl $.L.str
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    sete %al
+; X86-NEXT:    retl
+;
+; X64-SSE-LABEL: length63_eq_const:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    pushq %rax
+; X64-SSE-NEXT:    movl $.L.str, %esi
+; X64-SSE-NEXT:    movl $63, %edx
+; X64-SSE-NEXT:    callq memcmp
+; X64-SSE-NEXT:    testl %eax, %eax
+; X64-SSE-NEXT:    sete %al
+; X64-SSE-NEXT:    popq %rcx
+; X64-SSE-NEXT:    retq
+;
+; X64-AVX1-LABEL: length63_eq_const:
+; X64-AVX1:       # %bb.0:
+; X64-AVX1-NEXT:    vmovups (%rdi), %ymm0
+; X64-AVX1-NEXT:    vmovups 31(%rdi), %ymm1
+; X64-AVX1-NEXT:    vxorps {{.*}}(%rip), %ymm1, %ymm1
+; X64-AVX1-NEXT:    vxorps {{.*}}(%rip), %ymm0, %ymm0
+; X64-AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; X64-AVX1-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX1-NEXT:    sete %al
+; X64-AVX1-NEXT:    vzeroupper
+; X64-AVX1-NEXT:    retq
+;
+; X64-AVX2-LABEL: length63_eq_const:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    vmovdqu (%rdi), %ymm0
+; X64-AVX2-NEXT:    vmovdqu 31(%rdi), %ymm1
+; X64-AVX2-NEXT:    vpxor {{.*}}(%rip), %ymm1, %ymm1
+; X64-AVX2-NEXT:    vpxor {{.*}}(%rip), %ymm0, %ymm0
+; X64-AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; X64-AVX2-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX2-NEXT:    sete %al
+; X64-AVX2-NEXT:    vzeroupper
+; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512-LABEL: length63_eq_const:
+; X64-AVX512:       # %bb.0:
+; X64-AVX512-NEXT:    vmovdqu (%rdi), %ymm0
+; X64-AVX512-NEXT:    vmovdqu 31(%rdi), %ymm1
+; X64-AVX512-NEXT:    vpxor {{.*}}(%rip), %ymm1, %ymm1
+; X64-AVX512-NEXT:    vpxor {{.*}}(%rip), %ymm0, %ymm0
+; X64-AVX512-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; X64-AVX512-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX512-NEXT:    sete %al
+; X64-AVX512-NEXT:    vzeroupper
+; X64-AVX512-NEXT:    retq
+;
+; X64-MIC-AVX-LABEL: length63_eq_const:
+; X64-MIC-AVX:       # %bb.0:
+; X64-MIC-AVX-NEXT:    vmovdqu (%rdi), %ymm0
+; X64-MIC-AVX-NEXT:    vmovdqu 31(%rdi), %ymm1
+; X64-MIC-AVX-NEXT:    vmovdqa {{.*#+}} ymm2 = [875770417,943142453,842084409,909456435,809056311,875770417,943142453,842084409]
+; X64-MIC-AVX-NEXT:    vpcmpneqd %zmm2, %zmm1, %k0
+; X64-MIC-AVX-NEXT:    vmovdqa {{.*#+}} ymm1 = [858927408,926299444,825243960,892613426,959985462,858927408,926299444,825243960]
+; X64-MIC-AVX-NEXT:    vpcmpneqd %zmm1, %zmm0, %k1
+; X64-MIC-AVX-NEXT:    kortestw %k0, %k1
+; X64-MIC-AVX-NEXT:    sete %al
+; X64-MIC-AVX-NEXT:    vzeroupper
+; X64-MIC-AVX-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* getelementptr inbounds ([513 x i8], [513 x i8]* @.str, i32 0, i32 0), i64 63) nounwind
+  %c = icmp eq i32 %m, 0
+  ret i1 %c
+}
+
+define i32 @length64(i8* %X, i8* %Y) nounwind {
+; X86-LABEL: length64:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $64
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    retl
+;
+; X64-LABEL: length64:
+; X64:       # %bb.0:
+; X64-NEXT:    movl $64, %edx
+; X64-NEXT:    jmp memcmp # TAILCALL
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 64) nounwind
+  ret i32 %m
+}
+
+define i1 @length64_eq(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length64_eq:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $64
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    setne %al
+; X86-NEXT:    retl
+;
+; X64-SSE-LABEL: length64_eq:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    pushq %rax
+; X64-SSE-NEXT:    movl $64, %edx
+; X64-SSE-NEXT:    callq memcmp
+; X64-SSE-NEXT:    testl %eax, %eax
+; X64-SSE-NEXT:    setne %al
+; X64-SSE-NEXT:    popq %rcx
+; X64-SSE-NEXT:    retq
+;
+; X64-AVX1-LABEL: length64_eq:
+; X64-AVX1:       # %bb.0:
+; X64-AVX1-NEXT:    vmovups (%rdi), %ymm0
+; X64-AVX1-NEXT:    vmovups 32(%rdi), %ymm1
+; X64-AVX1-NEXT:    vxorps 32(%rsi), %ymm1, %ymm1
+; X64-AVX1-NEXT:    vxorps (%rsi), %ymm0, %ymm0
+; X64-AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; X64-AVX1-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX1-NEXT:    setne %al
+; X64-AVX1-NEXT:    vzeroupper
+; X64-AVX1-NEXT:    retq
+;
+; X64-AVX2-LABEL: length64_eq:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    vmovdqu (%rdi), %ymm0
+; X64-AVX2-NEXT:    vmovdqu 32(%rdi), %ymm1
+; X64-AVX2-NEXT:    vpxor 32(%rsi), %ymm1, %ymm1
+; X64-AVX2-NEXT:    vpxor (%rsi), %ymm0, %ymm0
+; X64-AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; X64-AVX2-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX2-NEXT:    setne %al
+; X64-AVX2-NEXT:    vzeroupper
+; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512BW-LABEL: length64_eq:
+; X64-AVX512BW:       # %bb.0:
+; X64-AVX512BW-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-AVX512BW-NEXT:    vpcmpneqb (%rsi), %zmm0, %k0
+; X64-AVX512BW-NEXT:    kortestq %k0, %k0
+; X64-AVX512BW-NEXT:    setne %al
+; X64-AVX512BW-NEXT:    vzeroupper
+; X64-AVX512BW-NEXT:    retq
+;
+; X64-AVX512F-LABEL: length64_eq:
+; X64-AVX512F:       # %bb.0:
+; X64-AVX512F-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-AVX512F-NEXT:    vpcmpneqd (%rsi), %zmm0, %k0
+; X64-AVX512F-NEXT:    kortestw %k0, %k0
+; X64-AVX512F-NEXT:    setne %al
+; X64-AVX512F-NEXT:    vzeroupper
+; X64-AVX512F-NEXT:    retq
+;
+; X64-MIC-AVX2-LABEL: length64_eq:
+; X64-MIC-AVX2:       # %bb.0:
+; X64-MIC-AVX2-NEXT:    vmovdqu (%rdi), %ymm0
+; X64-MIC-AVX2-NEXT:    vmovdqu 32(%rdi), %ymm1
+; X64-MIC-AVX2-NEXT:    vmovdqu (%rsi), %ymm2
+; X64-MIC-AVX2-NEXT:    vmovdqu 32(%rsi), %ymm3
+; X64-MIC-AVX2-NEXT:    vpcmpneqd %zmm3, %zmm1, %k0
+; X64-MIC-AVX2-NEXT:    vpcmpneqd %zmm2, %zmm0, %k1
+; X64-MIC-AVX2-NEXT:    kortestw %k0, %k1
+; X64-MIC-AVX2-NEXT:    setne %al
+; X64-MIC-AVX2-NEXT:    vzeroupper
+; X64-MIC-AVX2-NEXT:    retq
+;
+; X64-MIC-AVX512F-LABEL: length64_eq:
+; X64-MIC-AVX512F:       # %bb.0:
+; X64-MIC-AVX512F-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-MIC-AVX512F-NEXT:    vpcmpneqd (%rsi), %zmm0, %k0
+; X64-MIC-AVX512F-NEXT:    kortestw %k0, %k0
+; X64-MIC-AVX512F-NEXT:    setne %al
+; X64-MIC-AVX512F-NEXT:    vzeroupper
+; X64-MIC-AVX512F-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 64) nounwind
+  %cmp = icmp ne i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length64_lt(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length64_lt:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $64
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    shrl $31, %eax
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: length64_lt:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $64, %edx
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    shrl $31, %eax
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 64) nounwind
+  %cmp = icmp slt i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length64_gt(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length64_gt:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $64
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    setg %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length64_gt:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $64, %edx
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    setg %al
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 64) nounwind
+  %cmp = icmp sgt i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length64_eq_const(i8* %X) nounwind {
+; X86-LABEL: length64_eq_const:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $64
+; X86-NEXT:    pushl $.L.str
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    sete %al
+; X86-NEXT:    retl
+;
+; X64-SSE-LABEL: length64_eq_const:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    pushq %rax
+; X64-SSE-NEXT:    movl $.L.str, %esi
+; X64-SSE-NEXT:    movl $64, %edx
+; X64-SSE-NEXT:    callq memcmp
+; X64-SSE-NEXT:    testl %eax, %eax
+; X64-SSE-NEXT:    sete %al
+; X64-SSE-NEXT:    popq %rcx
+; X64-SSE-NEXT:    retq
+;
+; X64-AVX1-LABEL: length64_eq_const:
+; X64-AVX1:       # %bb.0:
+; X64-AVX1-NEXT:    vmovups (%rdi), %ymm0
+; X64-AVX1-NEXT:    vmovups 32(%rdi), %ymm1
+; X64-AVX1-NEXT:    vxorps {{.*}}(%rip), %ymm1, %ymm1
+; X64-AVX1-NEXT:    vxorps {{.*}}(%rip), %ymm0, %ymm0
+; X64-AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; X64-AVX1-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX1-NEXT:    sete %al
+; X64-AVX1-NEXT:    vzeroupper
+; X64-AVX1-NEXT:    retq
+;
+; X64-AVX2-LABEL: length64_eq_const:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    vmovdqu (%rdi), %ymm0
+; X64-AVX2-NEXT:    vmovdqu 32(%rdi), %ymm1
+; X64-AVX2-NEXT:    vpxor {{.*}}(%rip), %ymm1, %ymm1
+; X64-AVX2-NEXT:    vpxor {{.*}}(%rip), %ymm0, %ymm0
+; X64-AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; X64-AVX2-NEXT:    vptest %ymm0, %ymm0
+; X64-AVX2-NEXT:    sete %al
+; X64-AVX2-NEXT:    vzeroupper
+; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512BW-LABEL: length64_eq_const:
+; X64-AVX512BW:       # %bb.0:
+; X64-AVX512BW-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-AVX512BW-NEXT:    vpcmpneqb {{.*}}(%rip), %zmm0, %k0
+; X64-AVX512BW-NEXT:    kortestq %k0, %k0
+; X64-AVX512BW-NEXT:    sete %al
+; X64-AVX512BW-NEXT:    vzeroupper
+; X64-AVX512BW-NEXT:    retq
+;
+; X64-AVX512F-LABEL: length64_eq_const:
+; X64-AVX512F:       # %bb.0:
+; X64-AVX512F-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-AVX512F-NEXT:    vpcmpneqd {{.*}}(%rip), %zmm0, %k0
+; X64-AVX512F-NEXT:    kortestw %k0, %k0
+; X64-AVX512F-NEXT:    sete %al
+; X64-AVX512F-NEXT:    vzeroupper
+; X64-AVX512F-NEXT:    retq
+;
+; X64-MIC-AVX2-LABEL: length64_eq_const:
+; X64-MIC-AVX2:       # %bb.0:
+; X64-MIC-AVX2-NEXT:    vmovdqu (%rdi), %ymm0
+; X64-MIC-AVX2-NEXT:    vmovdqu 32(%rdi), %ymm1
+; X64-MIC-AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [892613426,959985462,858927408,926299444,825243960,892613426,959985462,858927408]
+; X64-MIC-AVX2-NEXT:    vpcmpneqd %zmm2, %zmm1, %k0
+; X64-MIC-AVX2-NEXT:    vmovdqa {{.*#+}} ymm1 = [858927408,926299444,825243960,892613426,959985462,858927408,926299444,825243960]
+; X64-MIC-AVX2-NEXT:    vpcmpneqd %zmm1, %zmm0, %k1
+; X64-MIC-AVX2-NEXT:    kortestw %k0, %k1
+; X64-MIC-AVX2-NEXT:    sete %al
+; X64-MIC-AVX2-NEXT:    vzeroupper
+; X64-MIC-AVX2-NEXT:    retq
+;
+; X64-MIC-AVX512F-LABEL: length64_eq_const:
+; X64-MIC-AVX512F:       # %bb.0:
+; X64-MIC-AVX512F-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-MIC-AVX512F-NEXT:    vpcmpneqd {{.*}}(%rip), %zmm0, %k0
+; X64-MIC-AVX512F-NEXT:    kortestw %k0, %k0
+; X64-MIC-AVX512F-NEXT:    sete %al
+; X64-MIC-AVX512F-NEXT:    vzeroupper
+; X64-MIC-AVX512F-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* getelementptr inbounds ([513 x i8], [513 x i8]* @.str, i32 0, i32 0), i64 64) nounwind
+  %c = icmp eq i32 %m, 0
+  ret i1 %c
+}
+
+define i32 @length96(i8* %X, i8* %Y) nounwind {
+; X86-LABEL: length96:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $96
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    retl
+;
+; X64-LABEL: length96:
+; X64:       # %bb.0:
+; X64-NEXT:    movl $96, %edx
+; X64-NEXT:    jmp memcmp # TAILCALL
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 96) nounwind
+  ret i32 %m
+}
+
+define i1 @length96_eq(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length96_eq:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $96
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    setne %al
+; X86-NEXT:    retl
+;
+; X64-SSE-LABEL: length96_eq:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    pushq %rax
+; X64-SSE-NEXT:    movl $96, %edx
+; X64-SSE-NEXT:    callq memcmp
+; X64-SSE-NEXT:    testl %eax, %eax
+; X64-SSE-NEXT:    setne %al
+; X64-SSE-NEXT:    popq %rcx
+; X64-SSE-NEXT:    retq
+;
+; X64-AVX1-LABEL: length96_eq:
+; X64-AVX1:       # %bb.0:
+; X64-AVX1-NEXT:    pushq %rax
+; X64-AVX1-NEXT:    movl $96, %edx
+; X64-AVX1-NEXT:    callq memcmp
+; X64-AVX1-NEXT:    testl %eax, %eax
+; X64-AVX1-NEXT:    setne %al
+; X64-AVX1-NEXT:    popq %rcx
+; X64-AVX1-NEXT:    retq
+;
+; X64-AVX2-LABEL: length96_eq:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    pushq %rax
+; X64-AVX2-NEXT:    movl $96, %edx
+; X64-AVX2-NEXT:    callq memcmp
+; X64-AVX2-NEXT:    testl %eax, %eax
+; X64-AVX2-NEXT:    setne %al
+; X64-AVX2-NEXT:    popq %rcx
+; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512BW-LABEL: length96_eq:
+; X64-AVX512BW:       # %bb.0:
+; X64-AVX512BW-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-AVX512BW-NEXT:    vmovdqu 64(%rdi), %ymm1
+; X64-AVX512BW-NEXT:    vmovdqu 64(%rsi), %ymm2
+; X64-AVX512BW-NEXT:    vpcmpneqb (%rsi), %zmm0, %k0
+; X64-AVX512BW-NEXT:    vpcmpneqb %zmm2, %zmm1, %k1
+; X64-AVX512BW-NEXT:    kortestq %k1, %k0
+; X64-AVX512BW-NEXT:    setne %al
+; X64-AVX512BW-NEXT:    vzeroupper
+; X64-AVX512BW-NEXT:    retq
+;
+; X64-AVX512F-LABEL: length96_eq:
+; X64-AVX512F:       # %bb.0:
+; X64-AVX512F-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-AVX512F-NEXT:    vmovdqu 64(%rdi), %ymm1
+; X64-AVX512F-NEXT:    vmovdqu 64(%rsi), %ymm2
+; X64-AVX512F-NEXT:    vpcmpneqd (%rsi), %zmm0, %k0
+; X64-AVX512F-NEXT:    vpcmpneqd %zmm2, %zmm1, %k1
+; X64-AVX512F-NEXT:    kortestw %k1, %k0
+; X64-AVX512F-NEXT:    setne %al
+; X64-AVX512F-NEXT:    vzeroupper
+; X64-AVX512F-NEXT:    retq
+;
+; X64-MIC-AVX2-LABEL: length96_eq:
+; X64-MIC-AVX2:       # %bb.0:
+; X64-MIC-AVX2-NEXT:    pushq %rax
+; X64-MIC-AVX2-NEXT:    movl $96, %edx
+; X64-MIC-AVX2-NEXT:    callq memcmp
+; X64-MIC-AVX2-NEXT:    testl %eax, %eax
+; X64-MIC-AVX2-NEXT:    setne %al
+; X64-MIC-AVX2-NEXT:    popq %rcx
+; X64-MIC-AVX2-NEXT:    retq
+;
+; X64-MIC-AVX512F-LABEL: length96_eq:
+; X64-MIC-AVX512F:       # %bb.0:
+; X64-MIC-AVX512F-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-MIC-AVX512F-NEXT:    vmovdqu 64(%rdi), %ymm1
+; X64-MIC-AVX512F-NEXT:    vmovdqu 64(%rsi), %ymm2
+; X64-MIC-AVX512F-NEXT:    vpcmpneqd (%rsi), %zmm0, %k0
+; X64-MIC-AVX512F-NEXT:    vpcmpneqd %zmm2, %zmm1, %k1
+; X64-MIC-AVX512F-NEXT:    kortestw %k1, %k0
+; X64-MIC-AVX512F-NEXT:    setne %al
+; X64-MIC-AVX512F-NEXT:    vzeroupper
+; X64-MIC-AVX512F-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 96) nounwind
+  %cmp = icmp ne i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length96_lt(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length96_lt:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $96
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    shrl $31, %eax
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: length96_lt:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $96, %edx
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    shrl $31, %eax
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 96) nounwind
+  %cmp = icmp slt i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length96_gt(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length96_gt:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $96
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    setg %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length96_gt:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $96, %edx
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    setg %al
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 96) nounwind
+  %cmp = icmp sgt i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length96_eq_const(i8* %X) nounwind {
+; X86-LABEL: length96_eq_const:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $96
+; X86-NEXT:    pushl $.L.str
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    sete %al
+; X86-NEXT:    retl
+;
+; X64-SSE-LABEL: length96_eq_const:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    pushq %rax
+; X64-SSE-NEXT:    movl $.L.str, %esi
+; X64-SSE-NEXT:    movl $96, %edx
+; X64-SSE-NEXT:    callq memcmp
+; X64-SSE-NEXT:    testl %eax, %eax
+; X64-SSE-NEXT:    sete %al
+; X64-SSE-NEXT:    popq %rcx
+; X64-SSE-NEXT:    retq
+;
+; X64-AVX1-LABEL: length96_eq_const:
+; X64-AVX1:       # %bb.0:
+; X64-AVX1-NEXT:    pushq %rax
+; X64-AVX1-NEXT:    movl $.L.str, %esi
+; X64-AVX1-NEXT:    movl $96, %edx
+; X64-AVX1-NEXT:    callq memcmp
+; X64-AVX1-NEXT:    testl %eax, %eax
+; X64-AVX1-NEXT:    sete %al
+; X64-AVX1-NEXT:    popq %rcx
+; X64-AVX1-NEXT:    retq
+;
+; X64-AVX2-LABEL: length96_eq_const:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    pushq %rax
+; X64-AVX2-NEXT:    movl $.L.str, %esi
+; X64-AVX2-NEXT:    movl $96, %edx
+; X64-AVX2-NEXT:    callq memcmp
+; X64-AVX2-NEXT:    testl %eax, %eax
+; X64-AVX2-NEXT:    sete %al
+; X64-AVX2-NEXT:    popq %rcx
+; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512BW-LABEL: length96_eq_const:
+; X64-AVX512BW:       # %bb.0:
+; X64-AVX512BW-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-AVX512BW-NEXT:    vmovdqu 64(%rdi), %ymm1
+; X64-AVX512BW-NEXT:    vpcmpneqb {{.*}}(%rip), %zmm0, %k0
+; X64-AVX512BW-NEXT:    vpcmpneqb {{.*}}(%rip), %zmm1, %k1
+; X64-AVX512BW-NEXT:    kortestq %k1, %k0
+; X64-AVX512BW-NEXT:    sete %al
+; X64-AVX512BW-NEXT:    vzeroupper
+; X64-AVX512BW-NEXT:    retq
+;
+; X64-AVX512F-LABEL: length96_eq_const:
+; X64-AVX512F:       # %bb.0:
+; X64-AVX512F-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-AVX512F-NEXT:    vmovdqu 64(%rdi), %ymm1
+; X64-AVX512F-NEXT:    vpcmpneqd {{.*}}(%rip), %zmm0, %k0
+; X64-AVX512F-NEXT:    vpcmpneqd {{.*}}(%rip), %zmm1, %k1
+; X64-AVX512F-NEXT:    kortestw %k1, %k0
+; X64-AVX512F-NEXT:    sete %al
+; X64-AVX512F-NEXT:    vzeroupper
+; X64-AVX512F-NEXT:    retq
+;
+; X64-MIC-AVX2-LABEL: length96_eq_const:
+; X64-MIC-AVX2:       # %bb.0:
+; X64-MIC-AVX2-NEXT:    pushq %rax
+; X64-MIC-AVX2-NEXT:    movl $.L.str, %esi
+; X64-MIC-AVX2-NEXT:    movl $96, %edx
+; X64-MIC-AVX2-NEXT:    callq memcmp
+; X64-MIC-AVX2-NEXT:    testl %eax, %eax
+; X64-MIC-AVX2-NEXT:    sete %al
+; X64-MIC-AVX2-NEXT:    popq %rcx
+; X64-MIC-AVX2-NEXT:    retq
+;
+; X64-MIC-AVX512F-LABEL: length96_eq_const:
+; X64-MIC-AVX512F:       # %bb.0:
+; X64-MIC-AVX512F-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-MIC-AVX512F-NEXT:    vmovdqu 64(%rdi), %ymm1
+; X64-MIC-AVX512F-NEXT:    vpcmpneqd {{.*}}(%rip), %zmm0, %k0
+; X64-MIC-AVX512F-NEXT:    vpcmpneqd {{.*}}(%rip), %zmm1, %k1
+; X64-MIC-AVX512F-NEXT:    kortestw %k1, %k0
+; X64-MIC-AVX512F-NEXT:    sete %al
+; X64-MIC-AVX512F-NEXT:    vzeroupper
+; X64-MIC-AVX512F-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* getelementptr inbounds ([513 x i8], [513 x i8]* @.str, i32 0, i32 0), i64 96) nounwind
+  %c = icmp eq i32 %m, 0
+  ret i1 %c
+}
+
+define i32 @length127(i8* %X, i8* %Y) nounwind {
+; X86-LABEL: length127:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $127
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    retl
+;
+; X64-LABEL: length127:
+; X64:       # %bb.0:
+; X64-NEXT:    movl $127, %edx
+; X64-NEXT:    jmp memcmp # TAILCALL
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 127) nounwind
+  ret i32 %m
+}
+
+define i1 @length127_eq(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length127_eq:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $127
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    setne %al
+; X86-NEXT:    retl
+;
+; X64-SSE-LABEL: length127_eq:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    pushq %rax
+; X64-SSE-NEXT:    movl $127, %edx
+; X64-SSE-NEXT:    callq memcmp
+; X64-SSE-NEXT:    testl %eax, %eax
+; X64-SSE-NEXT:    setne %al
+; X64-SSE-NEXT:    popq %rcx
+; X64-SSE-NEXT:    retq
+;
+; X64-AVX1-LABEL: length127_eq:
+; X64-AVX1:       # %bb.0:
+; X64-AVX1-NEXT:    pushq %rax
+; X64-AVX1-NEXT:    movl $127, %edx
+; X64-AVX1-NEXT:    callq memcmp
+; X64-AVX1-NEXT:    testl %eax, %eax
+; X64-AVX1-NEXT:    setne %al
+; X64-AVX1-NEXT:    popq %rcx
+; X64-AVX1-NEXT:    retq
+;
+; X64-AVX2-LABEL: length127_eq:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    pushq %rax
+; X64-AVX2-NEXT:    movl $127, %edx
+; X64-AVX2-NEXT:    callq memcmp
+; X64-AVX2-NEXT:    testl %eax, %eax
+; X64-AVX2-NEXT:    setne %al
+; X64-AVX2-NEXT:    popq %rcx
+; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512BW-LABEL: length127_eq:
+; X64-AVX512BW:       # %bb.0:
+; X64-AVX512BW-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-AVX512BW-NEXT:    vmovdqu64 63(%rdi), %zmm1
+; X64-AVX512BW-NEXT:    vpcmpneqb 63(%rsi), %zmm1, %k0
+; X64-AVX512BW-NEXT:    vpcmpneqb (%rsi), %zmm0, %k1
+; X64-AVX512BW-NEXT:    kortestq %k0, %k1
+; X64-AVX512BW-NEXT:    setne %al
+; X64-AVX512BW-NEXT:    vzeroupper
+; X64-AVX512BW-NEXT:    retq
+;
+; X64-AVX512F-LABEL: length127_eq:
+; X64-AVX512F:       # %bb.0:
+; X64-AVX512F-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-AVX512F-NEXT:    vmovdqu64 63(%rdi), %zmm1
+; X64-AVX512F-NEXT:    vpcmpneqd 63(%rsi), %zmm1, %k0
+; X64-AVX512F-NEXT:    vpcmpneqd (%rsi), %zmm0, %k1
+; X64-AVX512F-NEXT:    kortestw %k0, %k1
+; X64-AVX512F-NEXT:    setne %al
+; X64-AVX512F-NEXT:    vzeroupper
+; X64-AVX512F-NEXT:    retq
+;
+; X64-MIC-AVX2-LABEL: length127_eq:
+; X64-MIC-AVX2:       # %bb.0:
+; X64-MIC-AVX2-NEXT:    pushq %rax
+; X64-MIC-AVX2-NEXT:    movl $127, %edx
+; X64-MIC-AVX2-NEXT:    callq memcmp
+; X64-MIC-AVX2-NEXT:    testl %eax, %eax
+; X64-MIC-AVX2-NEXT:    setne %al
+; X64-MIC-AVX2-NEXT:    popq %rcx
+; X64-MIC-AVX2-NEXT:    retq
+;
+; X64-MIC-AVX512F-LABEL: length127_eq:
+; X64-MIC-AVX512F:       # %bb.0:
+; X64-MIC-AVX512F-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-MIC-AVX512F-NEXT:    vmovdqu64 63(%rdi), %zmm1
+; X64-MIC-AVX512F-NEXT:    vpcmpneqd 63(%rsi), %zmm1, %k0
+; X64-MIC-AVX512F-NEXT:    vpcmpneqd (%rsi), %zmm0, %k1
+; X64-MIC-AVX512F-NEXT:    kortestw %k0, %k1
+; X64-MIC-AVX512F-NEXT:    setne %al
+; X64-MIC-AVX512F-NEXT:    vzeroupper
+; X64-MIC-AVX512F-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 127) nounwind
+  %cmp = icmp ne i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length127_lt(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length127_lt:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $127
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    shrl $31, %eax
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: length127_lt:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $127, %edx
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    shrl $31, %eax
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 127) nounwind
+  %cmp = icmp slt i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length127_gt(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length127_gt:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $127
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    setg %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length127_gt:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $127, %edx
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    setg %al
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 127) nounwind
+  %cmp = icmp sgt i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length127_eq_const(i8* %X) nounwind {
+; X86-LABEL: length127_eq_const:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $127
+; X86-NEXT:    pushl $.L.str
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    sete %al
+; X86-NEXT:    retl
+;
+; X64-SSE-LABEL: length127_eq_const:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    pushq %rax
+; X64-SSE-NEXT:    movl $.L.str, %esi
+; X64-SSE-NEXT:    movl $127, %edx
+; X64-SSE-NEXT:    callq memcmp
+; X64-SSE-NEXT:    testl %eax, %eax
+; X64-SSE-NEXT:    sete %al
+; X64-SSE-NEXT:    popq %rcx
+; X64-SSE-NEXT:    retq
+;
+; X64-AVX1-LABEL: length127_eq_const:
+; X64-AVX1:       # %bb.0:
+; X64-AVX1-NEXT:    pushq %rax
+; X64-AVX1-NEXT:    movl $.L.str, %esi
+; X64-AVX1-NEXT:    movl $127, %edx
+; X64-AVX1-NEXT:    callq memcmp
+; X64-AVX1-NEXT:    testl %eax, %eax
+; X64-AVX1-NEXT:    sete %al
+; X64-AVX1-NEXT:    popq %rcx
+; X64-AVX1-NEXT:    retq
+;
+; X64-AVX2-LABEL: length127_eq_const:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    pushq %rax
+; X64-AVX2-NEXT:    movl $.L.str, %esi
+; X64-AVX2-NEXT:    movl $127, %edx
+; X64-AVX2-NEXT:    callq memcmp
+; X64-AVX2-NEXT:    testl %eax, %eax
+; X64-AVX2-NEXT:    sete %al
+; X64-AVX2-NEXT:    popq %rcx
+; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512BW-LABEL: length127_eq_const:
+; X64-AVX512BW:       # %bb.0:
+; X64-AVX512BW-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-AVX512BW-NEXT:    vmovdqu64 63(%rdi), %zmm1
+; X64-AVX512BW-NEXT:    vpcmpneqb .L.str+{{.*}}(%rip), %zmm1, %k0
+; X64-AVX512BW-NEXT:    vpcmpneqb {{.*}}(%rip), %zmm0, %k1
+; X64-AVX512BW-NEXT:    kortestq %k0, %k1
+; X64-AVX512BW-NEXT:    sete %al
+; X64-AVX512BW-NEXT:    vzeroupper
+; X64-AVX512BW-NEXT:    retq
+;
+; X64-AVX512F-LABEL: length127_eq_const:
+; X64-AVX512F:       # %bb.0:
+; X64-AVX512F-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-AVX512F-NEXT:    vmovdqu64 63(%rdi), %zmm1
+; X64-AVX512F-NEXT:    vpcmpneqd .L.str+{{.*}}(%rip), %zmm1, %k0
+; X64-AVX512F-NEXT:    vpcmpneqd {{.*}}(%rip), %zmm0, %k1
+; X64-AVX512F-NEXT:    kortestw %k0, %k1
+; X64-AVX512F-NEXT:    sete %al
+; X64-AVX512F-NEXT:    vzeroupper
+; X64-AVX512F-NEXT:    retq
+;
+; X64-MIC-AVX2-LABEL: length127_eq_const:
+; X64-MIC-AVX2:       # %bb.0:
+; X64-MIC-AVX2-NEXT:    pushq %rax
+; X64-MIC-AVX2-NEXT:    movl $.L.str, %esi
+; X64-MIC-AVX2-NEXT:    movl $127, %edx
+; X64-MIC-AVX2-NEXT:    callq memcmp
+; X64-MIC-AVX2-NEXT:    testl %eax, %eax
+; X64-MIC-AVX2-NEXT:    sete %al
+; X64-MIC-AVX2-NEXT:    popq %rcx
+; X64-MIC-AVX2-NEXT:    retq
+;
+; X64-MIC-AVX512F-LABEL: length127_eq_const:
+; X64-MIC-AVX512F:       # %bb.0:
+; X64-MIC-AVX512F-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-MIC-AVX512F-NEXT:    vmovdqu64 63(%rdi), %zmm1
+; X64-MIC-AVX512F-NEXT:    vpcmpneqd .L.str+{{.*}}(%rip), %zmm1, %k0
+; X64-MIC-AVX512F-NEXT:    vpcmpneqd {{.*}}(%rip), %zmm0, %k1
+; X64-MIC-AVX512F-NEXT:    kortestw %k0, %k1
+; X64-MIC-AVX512F-NEXT:    sete %al
+; X64-MIC-AVX512F-NEXT:    vzeroupper
+; X64-MIC-AVX512F-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* getelementptr inbounds ([513 x i8], [513 x i8]* @.str, i32 0, i32 0), i64 127) nounwind
+  %c = icmp eq i32 %m, 0
+  ret i1 %c
+}
+
+define i32 @length128(i8* %X, i8* %Y) nounwind {
+; X86-LABEL: length128:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $128
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    retl
+;
+; X64-LABEL: length128:
+; X64:       # %bb.0:
+; X64-NEXT:    movl $128, %edx
+; X64-NEXT:    jmp memcmp # TAILCALL
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 128) nounwind
+  ret i32 %m
+}
+
+define i1 @length128_eq(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length128_eq:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $128
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    setne %al
+; X86-NEXT:    retl
+;
+; X64-SSE-LABEL: length128_eq:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    pushq %rax
+; X64-SSE-NEXT:    movl $128, %edx
+; X64-SSE-NEXT:    callq memcmp
+; X64-SSE-NEXT:    testl %eax, %eax
+; X64-SSE-NEXT:    setne %al
+; X64-SSE-NEXT:    popq %rcx
+; X64-SSE-NEXT:    retq
+;
+; X64-AVX1-LABEL: length128_eq:
+; X64-AVX1:       # %bb.0:
+; X64-AVX1-NEXT:    pushq %rax
+; X64-AVX1-NEXT:    movl $128, %edx
+; X64-AVX1-NEXT:    callq memcmp
+; X64-AVX1-NEXT:    testl %eax, %eax
+; X64-AVX1-NEXT:    setne %al
+; X64-AVX1-NEXT:    popq %rcx
+; X64-AVX1-NEXT:    retq
+;
+; X64-AVX2-LABEL: length128_eq:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    pushq %rax
+; X64-AVX2-NEXT:    movl $128, %edx
+; X64-AVX2-NEXT:    callq memcmp
+; X64-AVX2-NEXT:    testl %eax, %eax
+; X64-AVX2-NEXT:    setne %al
+; X64-AVX2-NEXT:    popq %rcx
+; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512BW-LABEL: length128_eq:
+; X64-AVX512BW:       # %bb.0:
+; X64-AVX512BW-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-AVX512BW-NEXT:    vmovdqu64 64(%rdi), %zmm1
+; X64-AVX512BW-NEXT:    vpcmpneqb 64(%rsi), %zmm1, %k0
+; X64-AVX512BW-NEXT:    vpcmpneqb (%rsi), %zmm0, %k1
+; X64-AVX512BW-NEXT:    kortestq %k0, %k1
+; X64-AVX512BW-NEXT:    setne %al
+; X64-AVX512BW-NEXT:    vzeroupper
+; X64-AVX512BW-NEXT:    retq
+;
+; X64-AVX512F-LABEL: length128_eq:
+; X64-AVX512F:       # %bb.0:
+; X64-AVX512F-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-AVX512F-NEXT:    vmovdqu64 64(%rdi), %zmm1
+; X64-AVX512F-NEXT:    vpcmpneqd 64(%rsi), %zmm1, %k0
+; X64-AVX512F-NEXT:    vpcmpneqd (%rsi), %zmm0, %k1
+; X64-AVX512F-NEXT:    kortestw %k0, %k1
+; X64-AVX512F-NEXT:    setne %al
+; X64-AVX512F-NEXT:    vzeroupper
+; X64-AVX512F-NEXT:    retq
+;
+; X64-MIC-AVX2-LABEL: length128_eq:
+; X64-MIC-AVX2:       # %bb.0:
+; X64-MIC-AVX2-NEXT:    pushq %rax
+; X64-MIC-AVX2-NEXT:    movl $128, %edx
+; X64-MIC-AVX2-NEXT:    callq memcmp
+; X64-MIC-AVX2-NEXT:    testl %eax, %eax
+; X64-MIC-AVX2-NEXT:    setne %al
+; X64-MIC-AVX2-NEXT:    popq %rcx
+; X64-MIC-AVX2-NEXT:    retq
+;
+; X64-MIC-AVX512F-LABEL: length128_eq:
+; X64-MIC-AVX512F:       # %bb.0:
+; X64-MIC-AVX512F-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-MIC-AVX512F-NEXT:    vmovdqu64 64(%rdi), %zmm1
+; X64-MIC-AVX512F-NEXT:    vpcmpneqd 64(%rsi), %zmm1, %k0
+; X64-MIC-AVX512F-NEXT:    vpcmpneqd (%rsi), %zmm0, %k1
+; X64-MIC-AVX512F-NEXT:    kortestw %k0, %k1
+; X64-MIC-AVX512F-NEXT:    setne %al
+; X64-MIC-AVX512F-NEXT:    vzeroupper
+; X64-MIC-AVX512F-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 128) nounwind
+  %cmp = icmp ne i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length128_lt(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length128_lt:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $128
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    shrl $31, %eax
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: length128_lt:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $128, %edx
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    shrl $31, %eax
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 128) nounwind
+  %cmp = icmp slt i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length128_gt(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length128_gt:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $128
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    setg %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length128_gt:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $128, %edx
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    setg %al
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 128) nounwind
+  %cmp = icmp sgt i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length128_eq_const(i8* %X) nounwind {
+; X86-LABEL: length128_eq_const:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $128
+; X86-NEXT:    pushl $.L.str
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    sete %al
+; X86-NEXT:    retl
+;
+; X64-SSE-LABEL: length128_eq_const:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    pushq %rax
+; X64-SSE-NEXT:    movl $.L.str, %esi
+; X64-SSE-NEXT:    movl $128, %edx
+; X64-SSE-NEXT:    callq memcmp
+; X64-SSE-NEXT:    testl %eax, %eax
+; X64-SSE-NEXT:    sete %al
+; X64-SSE-NEXT:    popq %rcx
+; X64-SSE-NEXT:    retq
+;
+; X64-AVX1-LABEL: length128_eq_const:
+; X64-AVX1:       # %bb.0:
+; X64-AVX1-NEXT:    pushq %rax
+; X64-AVX1-NEXT:    movl $.L.str, %esi
+; X64-AVX1-NEXT:    movl $128, %edx
+; X64-AVX1-NEXT:    callq memcmp
+; X64-AVX1-NEXT:    testl %eax, %eax
+; X64-AVX1-NEXT:    sete %al
+; X64-AVX1-NEXT:    popq %rcx
+; X64-AVX1-NEXT:    retq
+;
+; X64-AVX2-LABEL: length128_eq_const:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    pushq %rax
+; X64-AVX2-NEXT:    movl $.L.str, %esi
+; X64-AVX2-NEXT:    movl $128, %edx
+; X64-AVX2-NEXT:    callq memcmp
+; X64-AVX2-NEXT:    testl %eax, %eax
+; X64-AVX2-NEXT:    sete %al
+; X64-AVX2-NEXT:    popq %rcx
+; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512BW-LABEL: length128_eq_const:
+; X64-AVX512BW:       # %bb.0:
+; X64-AVX512BW-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-AVX512BW-NEXT:    vmovdqu64 64(%rdi), %zmm1
+; X64-AVX512BW-NEXT:    vpcmpneqb .L.str+{{.*}}(%rip), %zmm1, %k0
+; X64-AVX512BW-NEXT:    vpcmpneqb {{.*}}(%rip), %zmm0, %k1
+; X64-AVX512BW-NEXT:    kortestq %k0, %k1
+; X64-AVX512BW-NEXT:    sete %al
+; X64-AVX512BW-NEXT:    vzeroupper
+; X64-AVX512BW-NEXT:    retq
+;
+; X64-AVX512F-LABEL: length128_eq_const:
+; X64-AVX512F:       # %bb.0:
+; X64-AVX512F-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-AVX512F-NEXT:    vmovdqu64 64(%rdi), %zmm1
+; X64-AVX512F-NEXT:    vpcmpneqd .L.str+{{.*}}(%rip), %zmm1, %k0
+; X64-AVX512F-NEXT:    vpcmpneqd {{.*}}(%rip), %zmm0, %k1
+; X64-AVX512F-NEXT:    kortestw %k0, %k1
+; X64-AVX512F-NEXT:    sete %al
+; X64-AVX512F-NEXT:    vzeroupper
+; X64-AVX512F-NEXT:    retq
+;
+; X64-MIC-AVX2-LABEL: length128_eq_const:
+; X64-MIC-AVX2:       # %bb.0:
+; X64-MIC-AVX2-NEXT:    pushq %rax
+; X64-MIC-AVX2-NEXT:    movl $.L.str, %esi
+; X64-MIC-AVX2-NEXT:    movl $128, %edx
+; X64-MIC-AVX2-NEXT:    callq memcmp
+; X64-MIC-AVX2-NEXT:    testl %eax, %eax
+; X64-MIC-AVX2-NEXT:    sete %al
+; X64-MIC-AVX2-NEXT:    popq %rcx
+; X64-MIC-AVX2-NEXT:    retq
+;
+; X64-MIC-AVX512F-LABEL: length128_eq_const:
+; X64-MIC-AVX512F:       # %bb.0:
+; X64-MIC-AVX512F-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-MIC-AVX512F-NEXT:    vmovdqu64 64(%rdi), %zmm1
+; X64-MIC-AVX512F-NEXT:    vpcmpneqd .L.str+{{.*}}(%rip), %zmm1, %k0
+; X64-MIC-AVX512F-NEXT:    vpcmpneqd {{.*}}(%rip), %zmm0, %k1
+; X64-MIC-AVX512F-NEXT:    kortestw %k0, %k1
+; X64-MIC-AVX512F-NEXT:    sete %al
+; X64-MIC-AVX512F-NEXT:    vzeroupper
+; X64-MIC-AVX512F-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* getelementptr inbounds ([513 x i8], [513 x i8]* @.str, i32 0, i32 0), i64 128) nounwind
+  %c = icmp eq i32 %m, 0
+  ret i1 %c
+}
+
+define i32 @length192(i8* %X, i8* %Y) nounwind {
+; X86-LABEL: length192:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $192
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    retl
+;
+; X64-LABEL: length192:
+; X64:       # %bb.0:
+; X64-NEXT:    movl $192, %edx
+; X64-NEXT:    jmp memcmp # TAILCALL
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 192) nounwind
+  ret i32 %m
+}
+
+define i1 @length192_eq(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length192_eq:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $192
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    setne %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length192_eq:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $192, %edx
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    setne %al
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 192) nounwind
+  %cmp = icmp ne i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length192_lt(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length192_lt:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $192
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    shrl $31, %eax
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: length192_lt:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $192, %edx
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    shrl $31, %eax
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 192) nounwind
+  %cmp = icmp slt i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length192_gt(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length192_gt:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $192
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    setg %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length192_gt:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $192, %edx
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    setg %al
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 192) nounwind
+  %cmp = icmp sgt i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length192_eq_const(i8* %X) nounwind {
+; X86-LABEL: length192_eq_const:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $192
+; X86-NEXT:    pushl $.L.str
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    sete %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length192_eq_const:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $.L.str, %esi
+; X64-NEXT:    movl $192, %edx
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    sete %al
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* getelementptr inbounds ([513 x i8], [513 x i8]* @.str, i32 0, i32 0), i64 192) nounwind
+  %c = icmp eq i32 %m, 0
+  ret i1 %c
+}
+
+define i32 @length255(i8* %X, i8* %Y) nounwind {
+; X86-LABEL: length255:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $255
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    retl
+;
+; X64-LABEL: length255:
+; X64:       # %bb.0:
+; X64-NEXT:    movl $255, %edx
+; X64-NEXT:    jmp memcmp # TAILCALL
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 255) nounwind
+  ret i32 %m
+}
+
+define i1 @length255_eq(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length255_eq:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $255
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    setne %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length255_eq:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $255, %edx
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    setne %al
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 255) nounwind
+  %cmp = icmp ne i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length255_lt(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length255_lt:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $255
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    shrl $31, %eax
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: length255_lt:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $255, %edx
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    shrl $31, %eax
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 255) nounwind
+  %cmp = icmp slt i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length255_gt(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length255_gt:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $255
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    setg %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length255_gt:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $255, %edx
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    setg %al
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 255) nounwind
+  %cmp = icmp sgt i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length255_eq_const(i8* %X) nounwind {
+; X86-LABEL: length255_eq_const:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $255
+; X86-NEXT:    pushl $.L.str
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    sete %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length255_eq_const:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $.L.str, %esi
+; X64-NEXT:    movl $255, %edx
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    sete %al
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* getelementptr inbounds ([513 x i8], [513 x i8]* @.str, i32 0, i32 0), i64 255) nounwind
+  %c = icmp eq i32 %m, 0
+  ret i1 %c
+}
+
+define i32 @length256(i8* %X, i8* %Y) nounwind {
+; X86-LABEL: length256:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $256 # imm = 0x100
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    retl
+;
+; X64-LABEL: length256:
+; X64:       # %bb.0:
+; X64-NEXT:    movl $256, %edx # imm = 0x100
+; X64-NEXT:    jmp memcmp # TAILCALL
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 256) nounwind
+  ret i32 %m
+}
+
+define i1 @length256_eq(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length256_eq:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $256 # imm = 0x100
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    setne %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length256_eq:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $256, %edx # imm = 0x100
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    setne %al
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 256) nounwind
+  %cmp = icmp ne i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length256_lt(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length256_lt:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $256 # imm = 0x100
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    shrl $31, %eax
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: length256_lt:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $256, %edx # imm = 0x100
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    shrl $31, %eax
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 256) nounwind
+  %cmp = icmp slt i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length256_gt(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length256_gt:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $256 # imm = 0x100
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    setg %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length256_gt:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $256, %edx # imm = 0x100
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    setg %al
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 256) nounwind
+  %cmp = icmp sgt i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length256_eq_const(i8* %X) nounwind {
+; X86-LABEL: length256_eq_const:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $256 # imm = 0x100
+; X86-NEXT:    pushl $.L.str
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    sete %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length256_eq_const:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $.L.str, %esi
+; X64-NEXT:    movl $256, %edx # imm = 0x100
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    sete %al
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* getelementptr inbounds ([513 x i8], [513 x i8]* @.str, i32 0, i32 0), i64 256) nounwind
+  %c = icmp eq i32 %m, 0
+  ret i1 %c
+}
+
+define i32 @length384(i8* %X, i8* %Y) nounwind {
+; X86-LABEL: length384:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $384 # imm = 0x180
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    retl
+;
+; X64-LABEL: length384:
+; X64:       # %bb.0:
+; X64-NEXT:    movl $384, %edx # imm = 0x180
+; X64-NEXT:    jmp memcmp # TAILCALL
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 384) nounwind
+  ret i32 %m
+}
+
+define i1 @length384_eq(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length384_eq:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $384 # imm = 0x180
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    setne %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length384_eq:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $384, %edx # imm = 0x180
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    setne %al
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 384) nounwind
+  %cmp = icmp ne i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length384_lt(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length384_lt:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $384 # imm = 0x180
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    shrl $31, %eax
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: length384_lt:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $384, %edx # imm = 0x180
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    shrl $31, %eax
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 384) nounwind
+  %cmp = icmp slt i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length384_gt(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length384_gt:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $384 # imm = 0x180
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
 ; X86-NEXT:    addl $16, %esp
 ; X86-NEXT:    testl %eax, %eax
-; X86-NEXT:    setne %al
+; X86-NEXT:    setg %al
 ; X86-NEXT:    retl
 ;
-; X64-SSE2-LABEL: length64_eq:
-; X64-SSE2:       # %bb.0:
-; X64-SSE2-NEXT:    pushq %rax
-; X64-SSE2-NEXT:    movl $64, %edx
-; X64-SSE2-NEXT:    callq memcmp
-; X64-SSE2-NEXT:    testl %eax, %eax
-; X64-SSE2-NEXT:    setne %al
-; X64-SSE2-NEXT:    popq %rcx
-; X64-SSE2-NEXT:    retq
+; X64-LABEL: length384_gt:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $384, %edx # imm = 0x180
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    setg %al
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 384) nounwind
+  %cmp = icmp sgt i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length384_eq_const(i8* %X) nounwind {
+; X86-LABEL: length384_eq_const:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $384 # imm = 0x180
+; X86-NEXT:    pushl $.L.str
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    sete %al
+; X86-NEXT:    retl
 ;
-; X64-AVX1-LABEL: length64_eq:
-; X64-AVX1:       # %bb.0:
-; X64-AVX1-NEXT:    pushq %rax
-; X64-AVX1-NEXT:    movl $64, %edx
-; X64-AVX1-NEXT:    callq memcmp
-; X64-AVX1-NEXT:    testl %eax, %eax
-; X64-AVX1-NEXT:    setne %al
-; X64-AVX1-NEXT:    popq %rcx
-; X64-AVX1-NEXT:    retq
+; X64-LABEL: length384_eq_const:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $.L.str, %esi
+; X64-NEXT:    movl $384, %edx # imm = 0x180
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    sete %al
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* getelementptr inbounds ([513 x i8], [513 x i8]* @.str, i32 0, i32 0), i64 384) nounwind
+  %c = icmp eq i32 %m, 0
+  ret i1 %c
+}
+
+define i32 @length511(i8* %X, i8* %Y) nounwind {
+; X86-LABEL: length511:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $511 # imm = 0x1FF
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    retl
 ;
-; X64-AVX2-LABEL: length64_eq:
-; X64-AVX2:       # %bb.0:
-; X64-AVX2-NEXT:    vmovdqu (%rdi), %ymm0
-; X64-AVX2-NEXT:    vmovdqu 32(%rdi), %ymm1
-; X64-AVX2-NEXT:    vpcmpeqb 32(%rsi), %ymm1, %ymm1
-; X64-AVX2-NEXT:    vpcmpeqb (%rsi), %ymm0, %ymm0
-; X64-AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
-; X64-AVX2-NEXT:    vpmovmskb %ymm0, %eax
-; X64-AVX2-NEXT:    cmpl $-1, %eax
-; X64-AVX2-NEXT:    setne %al
-; X64-AVX2-NEXT:    vzeroupper
-; X64-AVX2-NEXT:    retq
-  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 64) nounwind
+; X64-LABEL: length511:
+; X64:       # %bb.0:
+; X64-NEXT:    movl $511, %edx # imm = 0x1FF
+; X64-NEXT:    jmp memcmp # TAILCALL
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 511) nounwind
+  ret i32 %m
+}
+
+define i1 @length511_eq(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length511_eq:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $511 # imm = 0x1FF
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    setne %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length511_eq:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $511, %edx # imm = 0x1FF
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    setne %al
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 511) nounwind
   %cmp = icmp ne i32 %call, 0
   ret i1 %cmp
 }
 
-define i1 @length64_eq_const(i8* %X) nounwind {
-; X86-LABEL: length64_eq_const:
+define i1 @length511_lt(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length511_lt:
 ; X86:       # %bb.0:
 ; X86-NEXT:    pushl $0
-; X86-NEXT:    pushl $64
+; X86-NEXT:    pushl $511 # imm = 0x1FF
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    shrl $31, %eax
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: length511_lt:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $511, %edx # imm = 0x1FF
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    shrl $31, %eax
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 511) nounwind
+  %cmp = icmp slt i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length511_gt(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length511_gt:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $511 # imm = 0x1FF
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    setg %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length511_gt:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $511, %edx # imm = 0x1FF
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    setg %al
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 511) nounwind
+  %cmp = icmp sgt i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length511_eq_const(i8* %X) nounwind {
+; X86-LABEL: length511_eq_const:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $511 # imm = 0x1FF
 ; X86-NEXT:    pushl $.L.str
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    calll memcmp
@@ -1558,41 +4462,145 @@ define i1 @length64_eq_const(i8* %X) nounwind {
 ; X86-NEXT:    sete %al
 ; X86-NEXT:    retl
 ;
-; X64-SSE2-LABEL: length64_eq_const:
-; X64-SSE2:       # %bb.0:
-; X64-SSE2-NEXT:    pushq %rax
-; X64-SSE2-NEXT:    movl $.L.str, %esi
-; X64-SSE2-NEXT:    movl $64, %edx
-; X64-SSE2-NEXT:    callq memcmp
-; X64-SSE2-NEXT:    testl %eax, %eax
-; X64-SSE2-NEXT:    sete %al
-; X64-SSE2-NEXT:    popq %rcx
-; X64-SSE2-NEXT:    retq
+; X64-LABEL: length511_eq_const:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $.L.str, %esi
+; X64-NEXT:    movl $511, %edx # imm = 0x1FF
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    sete %al
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* getelementptr inbounds ([513 x i8], [513 x i8]* @.str, i32 0, i32 0), i64 511) nounwind
+  %c = icmp eq i32 %m, 0
+  ret i1 %c
+}
+
+define i32 @length512(i8* %X, i8* %Y) nounwind {
+; X86-LABEL: length512:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $512 # imm = 0x200
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    retl
 ;
-; X64-AVX1-LABEL: length64_eq_const:
-; X64-AVX1:       # %bb.0:
-; X64-AVX1-NEXT:    pushq %rax
-; X64-AVX1-NEXT:    movl $.L.str, %esi
-; X64-AVX1-NEXT:    movl $64, %edx
-; X64-AVX1-NEXT:    callq memcmp
-; X64-AVX1-NEXT:    testl %eax, %eax
-; X64-AVX1-NEXT:    sete %al
-; X64-AVX1-NEXT:    popq %rcx
-; X64-AVX1-NEXT:    retq
+; X64-LABEL: length512:
+; X64:       # %bb.0:
+; X64-NEXT:    movl $512, %edx # imm = 0x200
+; X64-NEXT:    jmp memcmp # TAILCALL
+  %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 512) nounwind
+  ret i32 %m
+}
+
+define i1 @length512_eq(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length512_eq:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $512 # imm = 0x200
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    setne %al
+; X86-NEXT:    retl
 ;
-; X64-AVX2-LABEL: length64_eq_const:
-; X64-AVX2:       # %bb.0:
-; X64-AVX2-NEXT:    vmovdqu (%rdi), %ymm0
-; X64-AVX2-NEXT:    vmovdqu 32(%rdi), %ymm1
-; X64-AVX2-NEXT:    vpcmpeqb {{.*}}(%rip), %ymm1, %ymm1
-; X64-AVX2-NEXT:    vpcmpeqb {{.*}}(%rip), %ymm0, %ymm0
-; X64-AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
-; X64-AVX2-NEXT:    vpmovmskb %ymm0, %eax
-; X64-AVX2-NEXT:    cmpl $-1, %eax
-; X64-AVX2-NEXT:    sete %al
-; X64-AVX2-NEXT:    vzeroupper
-; X64-AVX2-NEXT:    retq
-  %m = tail call i32 @memcmp(i8* %X, i8* getelementptr inbounds ([65 x i8], [65 x i8]* @.str, i32 0, i32 0), i64 64) nounwind
+; X64-LABEL: length512_eq:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $512, %edx # imm = 0x200
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    setne %al
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 512) nounwind
+  %cmp = icmp ne i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length512_lt(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length512_lt:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $512 # imm = 0x200
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    shrl $31, %eax
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: length512_lt:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $512, %edx # imm = 0x200
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    shrl $31, %eax
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 512) nounwind
+  %cmp = icmp slt i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length512_gt(i8* %x, i8* %y) nounwind {
+; X86-LABEL: length512_gt:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $512 # imm = 0x200
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    setg %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length512_gt:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $512, %edx # imm = 0x200
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    setg %al
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 512) nounwind
+  %cmp = icmp sgt i32 %call, 0
+  ret i1 %cmp
+}
+
+define i1 @length512_eq_const(i8* %X) nounwind {
+; X86-LABEL: length512_eq_const:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $512 # imm = 0x200
+; X86-NEXT:    pushl $.L.str
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll memcmp
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    sete %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: length512_eq_const:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $.L.str, %esi
+; X64-NEXT:    movl $512, %edx # imm = 0x200
+; X64-NEXT:    callq memcmp
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    sete %al
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+  %m = tail call i32 @memcmp(i8* %X, i8* getelementptr inbounds ([513 x i8], [513 x i8]* @.str, i32 0, i32 0), i64 512) nounwind
   %c = icmp eq i32 %m, 0
   ret i1 %c
 }
@@ -1639,7 +4647,6 @@ define i1 @huge_length_eq(i8* %X, i8* %Y) nounwind {
 ; X64-NEXT:    sete %al
 ; X64-NEXT:    popq %rcx
 ; X64-NEXT:    retq
-
   %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 9223372036854775807) nounwind
   %c = icmp eq i32 %m, 0
   ret i1 %c
diff --git a/llvm/test/CodeGen/X86/memcpy.ll b/llvm/test/CodeGen/X86/memcpy.ll
index 597b776e2..4c0a937a7 100644
--- a/llvm/test/CodeGen/X86/memcpy.ll
+++ b/llvm/test/CodeGen/X86/memcpy.ll
@@ -1,6 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=core2 | FileCheck %s -check-prefix=LINUX
-; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=core2 | FileCheck %s -check-prefix=DARWIN
+; RUN: llc < %s -mtriple=x86_64-apple-darwin      -mcpu=core2     | FileCheck %s -check-prefix=DARWIN
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=core2     | FileCheck %s -check-prefix=LINUX
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=skylake   | FileCheck %s -check-prefix=LINUX-SKL
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=skx       | FileCheck %s -check-prefix=LINUX-SKX
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=knl       | FileCheck %s -check-prefix=LINUX-KNL
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=avx512bw | FileCheck %s -check-prefix=LINUX-AVX512BW
 
 declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i1) nounwind
 declare void @llvm.memcpy.p256i8.p256i8.i64(i8 addrspace(256)* nocapture, i8 addrspace(256)* nocapture, i64, i1) nounwind
@@ -8,13 +12,29 @@ declare void @llvm.memcpy.p256i8.p256i8.i64(i8 addrspace(256)* nocapture, i8 add
 
 ; Variable memcpy's should lower to calls.
 define i8* @test1(i8* %a, i8* %b, i64 %n) nounwind {
+; DARWIN-LABEL: test1:
+; DARWIN:       ## %bb.0: ## %entry
+; DARWIN-NEXT:    jmp _memcpy ## TAILCALL
+;
 ; LINUX-LABEL: test1:
 ; LINUX:       # %bb.0: # %entry
 ; LINUX-NEXT:    jmp memcpy # TAILCALL
 ;
-; DARWIN-LABEL: test1:
-; DARWIN:       ## %bb.0: ## %entry
-; DARWIN-NEXT:    jmp _memcpy ## TAILCALL
+; LINUX-SKL-LABEL: test1:
+; LINUX-SKL:       # %bb.0: # %entry
+; LINUX-SKL-NEXT:    jmp memcpy # TAILCALL
+;
+; LINUX-SKX-LABEL: test1:
+; LINUX-SKX:       # %bb.0: # %entry
+; LINUX-SKX-NEXT:    jmp memcpy # TAILCALL
+;
+; LINUX-KNL-LABEL: test1:
+; LINUX-KNL:       # %bb.0: # %entry
+; LINUX-KNL-NEXT:    jmp memcpy # TAILCALL
+;
+; LINUX-AVX512BW-LABEL: test1:
+; LINUX-AVX512BW:       # %bb.0: # %entry
+; LINUX-AVX512BW-NEXT:    jmp memcpy # TAILCALL
 entry:
 	tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %a, i8* %b, i64 %n, i1 0 )
 	ret i8* %a
@@ -22,13 +42,29 @@ entry:
 
 ; Variable memcpy's should lower to calls.
 define i8* @test2(i64* %a, i64* %b, i64 %n) nounwind {
+; DARWIN-LABEL: test2:
+; DARWIN:       ## %bb.0: ## %entry
+; DARWIN-NEXT:    jmp _memcpy ## TAILCALL
+;
 ; LINUX-LABEL: test2:
 ; LINUX:       # %bb.0: # %entry
 ; LINUX-NEXT:    jmp memcpy # TAILCALL
 ;
-; DARWIN-LABEL: test2:
-; DARWIN:       ## %bb.0: ## %entry
-; DARWIN-NEXT:    jmp _memcpy ## TAILCALL
+; LINUX-SKL-LABEL: test2:
+; LINUX-SKL:       # %bb.0: # %entry
+; LINUX-SKL-NEXT:    jmp memcpy # TAILCALL
+;
+; LINUX-SKX-LABEL: test2:
+; LINUX-SKX:       # %bb.0: # %entry
+; LINUX-SKX-NEXT:    jmp memcpy # TAILCALL
+;
+; LINUX-KNL-LABEL: test2:
+; LINUX-KNL:       # %bb.0: # %entry
+; LINUX-KNL-NEXT:    jmp memcpy # TAILCALL
+;
+; LINUX-AVX512BW-LABEL: test2:
+; LINUX-AVX512BW:       # %bb.0: # %entry
+; LINUX-AVX512BW-NEXT:    jmp memcpy # TAILCALL
 entry:
 	%tmp14 = bitcast i64* %a to i8*
 	%tmp25 = bitcast i64* %b to i8*
@@ -43,12 +79,73 @@ entry:
 ; hurting performance so it should just ignore optsize when expanding memcpy.
 ; rdar://8821501
 define void @test3(i8* nocapture %A, i8* nocapture %B) nounwind optsize noredzone {
+; DARWIN-LABEL: test3:
+; DARWIN:       ## %bb.0: ## %entry
+; DARWIN-NEXT:    movq 56(%rsi), %rax
+; DARWIN-NEXT:    movq %rax, 56(%rdi)
+; DARWIN-NEXT:    movq 48(%rsi), %rax
+; DARWIN-NEXT:    movq %rax, 48(%rdi)
+; DARWIN-NEXT:    movq 40(%rsi), %rax
+; DARWIN-NEXT:    movq %rax, 40(%rdi)
+; DARWIN-NEXT:    movq 32(%rsi), %rax
+; DARWIN-NEXT:    movq %rax, 32(%rdi)
+; DARWIN-NEXT:    movq 24(%rsi), %rax
+; DARWIN-NEXT:    movq %rax, 24(%rdi)
+; DARWIN-NEXT:    movq 16(%rsi), %rax
+; DARWIN-NEXT:    movq %rax, 16(%rdi)
+; DARWIN-NEXT:    movq (%rsi), %rax
+; DARWIN-NEXT:    movq 8(%rsi), %rcx
+; DARWIN-NEXT:    movq %rcx, 8(%rdi)
+; DARWIN-NEXT:    movq %rax, (%rdi)
+; DARWIN-NEXT:    retq
+;
 ; LINUX-LABEL: test3:
 ; LINUX:       # %bb.0: # %entry
 ; LINUX-NEXT:    movl $64, %edx
 ; LINUX-NEXT:    jmp memcpy # TAILCALL
 ;
-; DARWIN-LABEL: test3:
+; LINUX-SKL-LABEL: test3:
+; LINUX-SKL:       # %bb.0: # %entry
+; LINUX-SKL-NEXT:    vmovups (%rsi), %ymm0
+; LINUX-SKL-NEXT:    vmovups 32(%rsi), %ymm1
+; LINUX-SKL-NEXT:    vmovups %ymm1, 32(%rdi)
+; LINUX-SKL-NEXT:    vmovups %ymm0, (%rdi)
+; LINUX-SKL-NEXT:    vzeroupper
+; LINUX-SKL-NEXT:    retq
+;
+; LINUX-SKX-LABEL: test3:
+; LINUX-SKX:       # %bb.0: # %entry
+; LINUX-SKX-NEXT:    vmovups (%rsi), %ymm0
+; LINUX-SKX-NEXT:    vmovups 32(%rsi), %ymm1
+; LINUX-SKX-NEXT:    vmovups %ymm1, 32(%rdi)
+; LINUX-SKX-NEXT:    vmovups %ymm0, (%rdi)
+; LINUX-SKX-NEXT:    vzeroupper
+; LINUX-SKX-NEXT:    retq
+;
+; LINUX-KNL-LABEL: test3:
+; LINUX-KNL:       # %bb.0: # %entry
+; LINUX-KNL-NEXT:    vmovups (%rsi), %zmm0
+; LINUX-KNL-NEXT:    vmovups %zmm0, (%rdi)
+; LINUX-KNL-NEXT:    retq
+;
+; LINUX-AVX512BW-LABEL: test3:
+; LINUX-AVX512BW:       # %bb.0: # %entry
+; LINUX-AVX512BW-NEXT:    vmovups (%rsi), %zmm0
+; LINUX-AVX512BW-NEXT:    vmovups %zmm0, (%rdi)
+; LINUX-AVX512BW-NEXT:    vzeroupper
+; LINUX-AVX512BW-NEXT:    retq
+entry:
+  tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %A, i8* %B, i64 64, i1 false)
+  ret void
+}
+
+define void @test3_pgso(i8* nocapture %A, i8* nocapture %B) nounwind noredzone !prof !14 {
+; LINUX-LABEL: test3_pgso:
+; LINUX:       # %bb.0: # %entry
+; LINUX-NEXT:    movl $64, %edx
+; LINUX-NEXT:    jmp memcpy # TAILCALL
+;
+; DARWIN-LABEL: test3_pgso:
 ; DARWIN:       ## %bb.0: ## %entry
 ; DARWIN-NEXT:    movq 56(%rsi), %rax
 ; DARWIN-NEXT:    movq %rax, 56(%rdi)
@@ -73,39 +170,121 @@ entry:
 }
 
 define void @test3_minsize(i8* nocapture %A, i8* nocapture %B) nounwind minsize noredzone {
+; DARWIN-LABEL: test3_minsize:
+; DARWIN:       ## %bb.0:
+; DARWIN-NEXT:    pushq $64
+; DARWIN-NEXT:    popq %rdx
+; DARWIN-NEXT:    jmp _memcpy ## TAILCALL
+;
 ; LINUX-LABEL: test3_minsize:
 ; LINUX:       # %bb.0:
 ; LINUX-NEXT:    pushq $64
 ; LINUX-NEXT:    popq %rdx
 ; LINUX-NEXT:    jmp memcpy # TAILCALL
 ;
-; DARWIN-LABEL: test3_minsize:
-; DARWIN:       ## %bb.0:
-; DARWIN-NEXT:    pushq $64
-; DARWIN-NEXT:    popq %rdx
-; DARWIN-NEXT:    jmp _memcpy ## TAILCALL
+; LINUX-SKL-LABEL: test3_minsize:
+; LINUX-SKL:       # %bb.0:
+; LINUX-SKL-NEXT:    vmovups (%rsi), %ymm0
+; LINUX-SKL-NEXT:    vmovups 32(%rsi), %ymm1
+; LINUX-SKL-NEXT:    vmovups %ymm1, 32(%rdi)
+; LINUX-SKL-NEXT:    vmovups %ymm0, (%rdi)
+; LINUX-SKL-NEXT:    vzeroupper
+; LINUX-SKL-NEXT:    retq
+;
+; LINUX-SKX-LABEL: test3_minsize:
+; LINUX-SKX:       # %bb.0:
+; LINUX-SKX-NEXT:    vmovups (%rsi), %ymm0
+; LINUX-SKX-NEXT:    vmovups 32(%rsi), %ymm1
+; LINUX-SKX-NEXT:    vmovups %ymm1, 32(%rdi)
+; LINUX-SKX-NEXT:    vmovups %ymm0, (%rdi)
+; LINUX-SKX-NEXT:    vzeroupper
+; LINUX-SKX-NEXT:    retq
+;
+; LINUX-KNL-LABEL: test3_minsize:
+; LINUX-KNL:       # %bb.0:
+; LINUX-KNL-NEXT:    vmovups (%rsi), %zmm0
+; LINUX-KNL-NEXT:    vmovups %zmm0, (%rdi)
+; LINUX-KNL-NEXT:    retq
+;
+; LINUX-AVX512BW-LABEL: test3_minsize:
+; LINUX-AVX512BW:       # %bb.0:
+; LINUX-AVX512BW-NEXT:    vmovups (%rsi), %zmm0
+; LINUX-AVX512BW-NEXT:    vmovups %zmm0, (%rdi)
+; LINUX-AVX512BW-NEXT:    vzeroupper
+; LINUX-AVX512BW-NEXT:    retq
   tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %A, i8* %B, i64 64, i1 false)
   ret void
 }
 
 define void @test3_minsize_optsize(i8* nocapture %A, i8* nocapture %B) nounwind optsize minsize noredzone {
+; DARWIN-LABEL: test3_minsize_optsize:
+; DARWIN:       ## %bb.0:
+; DARWIN-NEXT:    pushq $64
+; DARWIN-NEXT:    popq %rdx
+; DARWIN-NEXT:    jmp _memcpy ## TAILCALL
+;
 ; LINUX-LABEL: test3_minsize_optsize:
 ; LINUX:       # %bb.0:
 ; LINUX-NEXT:    pushq $64
 ; LINUX-NEXT:    popq %rdx
 ; LINUX-NEXT:    jmp memcpy # TAILCALL
 ;
-; DARWIN-LABEL: test3_minsize_optsize:
-; DARWIN:       ## %bb.0:
-; DARWIN-NEXT:    pushq $64
-; DARWIN-NEXT:    popq %rdx
-; DARWIN-NEXT:    jmp _memcpy ## TAILCALL
+; LINUX-SKL-LABEL: test3_minsize_optsize:
+; LINUX-SKL:       # %bb.0:
+; LINUX-SKL-NEXT:    vmovups (%rsi), %ymm0
+; LINUX-SKL-NEXT:    vmovups 32(%rsi), %ymm1
+; LINUX-SKL-NEXT:    vmovups %ymm1, 32(%rdi)
+; LINUX-SKL-NEXT:    vmovups %ymm0, (%rdi)
+; LINUX-SKL-NEXT:    vzeroupper
+; LINUX-SKL-NEXT:    retq
+;
+; LINUX-SKX-LABEL: test3_minsize_optsize:
+; LINUX-SKX:       # %bb.0:
+; LINUX-SKX-NEXT:    vmovups (%rsi), %ymm0
+; LINUX-SKX-NEXT:    vmovups 32(%rsi), %ymm1
+; LINUX-SKX-NEXT:    vmovups %ymm1, 32(%rdi)
+; LINUX-SKX-NEXT:    vmovups %ymm0, (%rdi)
+; LINUX-SKX-NEXT:    vzeroupper
+; LINUX-SKX-NEXT:    retq
+;
+; LINUX-KNL-LABEL: test3_minsize_optsize:
+; LINUX-KNL:       # %bb.0:
+; LINUX-KNL-NEXT:    vmovups (%rsi), %zmm0
+; LINUX-KNL-NEXT:    vmovups %zmm0, (%rdi)
+; LINUX-KNL-NEXT:    retq
+;
+; LINUX-AVX512BW-LABEL: test3_minsize_optsize:
+; LINUX-AVX512BW:       # %bb.0:
+; LINUX-AVX512BW-NEXT:    vmovups (%rsi), %zmm0
+; LINUX-AVX512BW-NEXT:    vmovups %zmm0, (%rdi)
+; LINUX-AVX512BW-NEXT:    vzeroupper
+; LINUX-AVX512BW-NEXT:    retq
   tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %A, i8* %B, i64 64, i1 false)
   ret void
 }
 
 ; Large constant memcpy's should be inlined when not optimizing for size.
 define void @test4(i8* nocapture %A, i8* nocapture %B) nounwind noredzone {
+; DARWIN-LABEL: test4:
+; DARWIN:       ## %bb.0: ## %entry
+; DARWIN-NEXT:    movq 56(%rsi), %rax
+; DARWIN-NEXT:    movq %rax, 56(%rdi)
+; DARWIN-NEXT:    movq 48(%rsi), %rax
+; DARWIN-NEXT:    movq %rax, 48(%rdi)
+; DARWIN-NEXT:    movq 40(%rsi), %rax
+; DARWIN-NEXT:    movq %rax, 40(%rdi)
+; DARWIN-NEXT:    movq 32(%rsi), %rax
+; DARWIN-NEXT:    movq %rax, 32(%rdi)
+; DARWIN-NEXT:    movq 24(%rsi), %rax
+; DARWIN-NEXT:    movq %rax, 24(%rdi)
+; DARWIN-NEXT:    movq 16(%rsi), %rax
+; DARWIN-NEXT:    movq %rax, 16(%rdi)
+; DARWIN-NEXT:    movq (%rsi), %rax
+; DARWIN-NEXT:    movq 8(%rsi), %rcx
+; DARWIN-NEXT:    movq %rcx, 8(%rdi)
+; DARWIN-NEXT:    movq %rax, (%rdi)
+; DARWIN-NEXT:    retq
+;
 ; LINUX-LABEL: test4:
 ; LINUX:       # %bb.0: # %entry
 ; LINUX-NEXT:    movq 56(%rsi), %rax
@@ -126,25 +305,36 @@ define void @test4(i8* nocapture %A, i8* nocapture %B) nounwind noredzone {
 ; LINUX-NEXT:    movq %rax, (%rdi)
 ; LINUX-NEXT:    retq
 ;
-; DARWIN-LABEL: test4:
-; DARWIN:       ## %bb.0: ## %entry
-; DARWIN-NEXT:    movq 56(%rsi), %rax
-; DARWIN-NEXT:    movq %rax, 56(%rdi)
-; DARWIN-NEXT:    movq 48(%rsi), %rax
-; DARWIN-NEXT:    movq %rax, 48(%rdi)
-; DARWIN-NEXT:    movq 40(%rsi), %rax
-; DARWIN-NEXT:    movq %rax, 40(%rdi)
-; DARWIN-NEXT:    movq 32(%rsi), %rax
-; DARWIN-NEXT:    movq %rax, 32(%rdi)
-; DARWIN-NEXT:    movq 24(%rsi), %rax
-; DARWIN-NEXT:    movq %rax, 24(%rdi)
-; DARWIN-NEXT:    movq 16(%rsi), %rax
-; DARWIN-NEXT:    movq %rax, 16(%rdi)
-; DARWIN-NEXT:    movq (%rsi), %rax
-; DARWIN-NEXT:    movq 8(%rsi), %rcx
-; DARWIN-NEXT:    movq %rcx, 8(%rdi)
-; DARWIN-NEXT:    movq %rax, (%rdi)
-; DARWIN-NEXT:    retq
+; LINUX-SKL-LABEL: test4:
+; LINUX-SKL:       # %bb.0: # %entry
+; LINUX-SKL-NEXT:    vmovups (%rsi), %ymm0
+; LINUX-SKL-NEXT:    vmovups 32(%rsi), %ymm1
+; LINUX-SKL-NEXT:    vmovups %ymm1, 32(%rdi)
+; LINUX-SKL-NEXT:    vmovups %ymm0, (%rdi)
+; LINUX-SKL-NEXT:    vzeroupper
+; LINUX-SKL-NEXT:    retq
+;
+; LINUX-SKX-LABEL: test4:
+; LINUX-SKX:       # %bb.0: # %entry
+; LINUX-SKX-NEXT:    vmovups (%rsi), %ymm0
+; LINUX-SKX-NEXT:    vmovups 32(%rsi), %ymm1
+; LINUX-SKX-NEXT:    vmovups %ymm1, 32(%rdi)
+; LINUX-SKX-NEXT:    vmovups %ymm0, (%rdi)
+; LINUX-SKX-NEXT:    vzeroupper
+; LINUX-SKX-NEXT:    retq
+;
+; LINUX-KNL-LABEL: test4:
+; LINUX-KNL:       # %bb.0: # %entry
+; LINUX-KNL-NEXT:    vmovups (%rsi), %zmm0
+; LINUX-KNL-NEXT:    vmovups %zmm0, (%rdi)
+; LINUX-KNL-NEXT:    retq
+;
+; LINUX-AVX512BW-LABEL: test4:
+; LINUX-AVX512BW:       # %bb.0: # %entry
+; LINUX-AVX512BW-NEXT:    vmovups (%rsi), %zmm0
+; LINUX-AVX512BW-NEXT:    vmovups %zmm0, (%rdi)
+; LINUX-AVX512BW-NEXT:    vzeroupper
+; LINUX-AVX512BW-NEXT:    retq
 entry:
   tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %A, i8* %B, i64 64, i1 false)
   ret void
@@ -154,6 +344,14 @@ entry:
 @.str = private unnamed_addr constant [30 x i8] c"\00aaaaaaaaaaaaaaaaaaaaaaaaaaaa\00", align 1
 
 define void @test5(i8* nocapture %C) nounwind uwtable ssp {
+; DARWIN-LABEL: test5:
+; DARWIN:       ## %bb.0: ## %entry
+; DARWIN-NEXT:    movabsq $7016996765293437281, %rax ## imm = 0x6161616161616161
+; DARWIN-NEXT:    movq %rax, 8(%rdi)
+; DARWIN-NEXT:    movabsq $7016996765293437184, %rax ## imm = 0x6161616161616100
+; DARWIN-NEXT:    movq %rax, (%rdi)
+; DARWIN-NEXT:    retq
+;
 ; LINUX-LABEL: test5:
 ; LINUX:       # %bb.0: # %entry
 ; LINUX-NEXT:    movabsq $7016996765293437281, %rax # imm = 0x6161616161616161
@@ -162,13 +360,29 @@ define void @test5(i8* nocapture %C) nounwind uwtable ssp {
 ; LINUX-NEXT:    movq %rax, (%rdi)
 ; LINUX-NEXT:    retq
 ;
-; DARWIN-LABEL: test5:
-; DARWIN:       ## %bb.0: ## %entry
-; DARWIN-NEXT:    movabsq $7016996765293437281, %rax ## imm = 0x6161616161616161
-; DARWIN-NEXT:    movq %rax, 8(%rdi)
-; DARWIN-NEXT:    movabsq $7016996765293437184, %rax ## imm = 0x6161616161616100
-; DARWIN-NEXT:    movq %rax, (%rdi)
-; DARWIN-NEXT:    retq
+; LINUX-SKL-LABEL: test5:
+; LINUX-SKL:       # %bb.0: # %entry
+; LINUX-SKL-NEXT:    vmovups {{.*}}(%rip), %xmm0
+; LINUX-SKL-NEXT:    vmovups %xmm0, (%rdi)
+; LINUX-SKL-NEXT:    retq
+;
+; LINUX-SKX-LABEL: test5:
+; LINUX-SKX:       # %bb.0: # %entry
+; LINUX-SKX-NEXT:    vmovups {{.*}}(%rip), %xmm0
+; LINUX-SKX-NEXT:    vmovups %xmm0, (%rdi)
+; LINUX-SKX-NEXT:    retq
+;
+; LINUX-KNL-LABEL: test5:
+; LINUX-KNL:       # %bb.0: # %entry
+; LINUX-KNL-NEXT:    vmovups {{.*}}(%rip), %xmm0
+; LINUX-KNL-NEXT:    vmovups %xmm0, (%rdi)
+; LINUX-KNL-NEXT:    retq
+;
+; LINUX-AVX512BW-LABEL: test5:
+; LINUX-AVX512BW:       # %bb.0: # %entry
+; LINUX-AVX512BW-NEXT:    vmovups {{.*}}(%rip), %xmm0
+; LINUX-AVX512BW-NEXT:    vmovups %xmm0, (%rdi)
+; LINUX-AVX512BW-NEXT:    retq
 entry:
   tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %C, i8* getelementptr inbounds ([30 x i8], [30 x i8]* @.str, i64 0, i64 0), i64 16, i1 false)
   ret void
@@ -179,17 +393,41 @@ entry:
 @.str2 = private unnamed_addr constant [2 x i8] c"x\00", align 1
 
 define void @test6() nounwind uwtable {
+; DARWIN-LABEL: test6:
+; DARWIN:       ## %bb.0: ## %entry
+; DARWIN-NEXT:    movw $0, 8
+; DARWIN-NEXT:    movq $120, 0
+; DARWIN-NEXT:    retq
+;
 ; LINUX-LABEL: test6:
 ; LINUX:       # %bb.0: # %entry
 ; LINUX-NEXT:    movw $0, 8
 ; LINUX-NEXT:    movq $120, 0
 ; LINUX-NEXT:    retq
 ;
-; DARWIN-LABEL: test6:
-; DARWIN:       ## %bb.0: ## %entry
-; DARWIN-NEXT:    movw $0, 8
-; DARWIN-NEXT:    movq $120, 0
-; DARWIN-NEXT:    retq
+; LINUX-SKL-LABEL: test6:
+; LINUX-SKL:       # %bb.0: # %entry
+; LINUX-SKL-NEXT:    movw $0, 8
+; LINUX-SKL-NEXT:    movq $120, 0
+; LINUX-SKL-NEXT:    retq
+;
+; LINUX-SKX-LABEL: test6:
+; LINUX-SKX:       # %bb.0: # %entry
+; LINUX-SKX-NEXT:    movw $0, 8
+; LINUX-SKX-NEXT:    movq $120, 0
+; LINUX-SKX-NEXT:    retq
+;
+; LINUX-KNL-LABEL: test6:
+; LINUX-KNL:       # %bb.0: # %entry
+; LINUX-KNL-NEXT:    movw $0, 8
+; LINUX-KNL-NEXT:    movq $120, 0
+; LINUX-KNL-NEXT:    retq
+;
+; LINUX-AVX512BW-LABEL: test6:
+; LINUX-AVX512BW:       # %bb.0: # %entry
+; LINUX-AVX512BW-NEXT:    movw $0, 8
+; LINUX-AVX512BW-NEXT:    movq $120, 0
+; LINUX-AVX512BW-NEXT:    retq
 entry:
   tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* null, i8* getelementptr inbounds ([2 x i8], [2 x i8]* @.str2, i64 0, i64 0), i64 10, i1 false)
   ret void
@@ -198,6 +436,16 @@ entry:
 define void @PR15348(i8* %a, i8* %b) {
 ; Ensure that alignment of '0' in an @llvm.memcpy intrinsic results in
 ; unaligned loads and stores.
+; DARWIN-LABEL: PR15348:
+; DARWIN:       ## %bb.0:
+; DARWIN-NEXT:    movb 16(%rsi), %al
+; DARWIN-NEXT:    movb %al, 16(%rdi)
+; DARWIN-NEXT:    movq (%rsi), %rax
+; DARWIN-NEXT:    movq 8(%rsi), %rcx
+; DARWIN-NEXT:    movq %rcx, 8(%rdi)
+; DARWIN-NEXT:    movq %rax, (%rdi)
+; DARWIN-NEXT:    retq
+;
 ; LINUX-LABEL: PR15348:
 ; LINUX:       # %bb.0:
 ; LINUX-NEXT:    movb 16(%rsi), %al
@@ -208,15 +456,37 @@ define void @PR15348(i8* %a, i8* %b) {
 ; LINUX-NEXT:    movq %rax, (%rdi)
 ; LINUX-NEXT:    retq
 ;
-; DARWIN-LABEL: PR15348:
-; DARWIN:       ## %bb.0:
-; DARWIN-NEXT:    movb 16(%rsi), %al
-; DARWIN-NEXT:    movb %al, 16(%rdi)
-; DARWIN-NEXT:    movq (%rsi), %rax
-; DARWIN-NEXT:    movq 8(%rsi), %rcx
-; DARWIN-NEXT:    movq %rcx, 8(%rdi)
-; DARWIN-NEXT:    movq %rax, (%rdi)
-; DARWIN-NEXT:    retq
+; LINUX-SKL-LABEL: PR15348:
+; LINUX-SKL:       # %bb.0:
+; LINUX-SKL-NEXT:    movb 16(%rsi), %al
+; LINUX-SKL-NEXT:    movb %al, 16(%rdi)
+; LINUX-SKL-NEXT:    vmovups (%rsi), %xmm0
+; LINUX-SKL-NEXT:    vmovups %xmm0, (%rdi)
+; LINUX-SKL-NEXT:    retq
+;
+; LINUX-SKX-LABEL: PR15348:
+; LINUX-SKX:       # %bb.0:
+; LINUX-SKX-NEXT:    movb 16(%rsi), %al
+; LINUX-SKX-NEXT:    movb %al, 16(%rdi)
+; LINUX-SKX-NEXT:    vmovups (%rsi), %xmm0
+; LINUX-SKX-NEXT:    vmovups %xmm0, (%rdi)
+; LINUX-SKX-NEXT:    retq
+;
+; LINUX-KNL-LABEL: PR15348:
+; LINUX-KNL:       # %bb.0:
+; LINUX-KNL-NEXT:    movb 16(%rsi), %al
+; LINUX-KNL-NEXT:    movb %al, 16(%rdi)
+; LINUX-KNL-NEXT:    vmovups (%rsi), %xmm0
+; LINUX-KNL-NEXT:    vmovups %xmm0, (%rdi)
+; LINUX-KNL-NEXT:    retq
+;
+; LINUX-AVX512BW-LABEL: PR15348:
+; LINUX-AVX512BW:       # %bb.0:
+; LINUX-AVX512BW-NEXT:    movb 16(%rsi), %al
+; LINUX-AVX512BW-NEXT:    movb %al, 16(%rdi)
+; LINUX-AVX512BW-NEXT:    vmovups (%rsi), %xmm0
+; LINUX-AVX512BW-NEXT:    vmovups %xmm0, (%rdi)
+; LINUX-AVX512BW-NEXT:    retq
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %a, i8* %b, i64 17, i1 false)
   ret void
 }
@@ -224,6 +494,14 @@ define void @PR15348(i8* %a, i8* %b) {
 ; Memcpys from / to address space 256 should be lowered to appropriate loads /
 ; stores if small enough.
 define void @addrspace256(i8 addrspace(256)* %a, i8 addrspace(256)* %b) nounwind {
+; DARWIN-LABEL: addrspace256:
+; DARWIN:       ## %bb.0:
+; DARWIN-NEXT:    movq %gs:(%rsi), %rax
+; DARWIN-NEXT:    movq %gs:8(%rsi), %rcx
+; DARWIN-NEXT:    movq %rcx, %gs:8(%rdi)
+; DARWIN-NEXT:    movq %rax, %gs:(%rdi)
+; DARWIN-NEXT:    retq
+;
 ; LINUX-LABEL: addrspace256:
 ; LINUX:       # %bb.0:
 ; LINUX-NEXT:    movq %gs:(%rsi), %rax
@@ -232,13 +510,46 @@ define void @addrspace256(i8 addrspace(256)* %a, i8 addrspace(256)* %b) nounwind
 ; LINUX-NEXT:    movq %rax, %gs:(%rdi)
 ; LINUX-NEXT:    retq
 ;
-; DARWIN-LABEL: addrspace256:
-; DARWIN:       ## %bb.0:
-; DARWIN-NEXT:    movq %gs:(%rsi), %rax
-; DARWIN-NEXT:    movq %gs:8(%rsi), %rcx
-; DARWIN-NEXT:    movq %rcx, %gs:8(%rdi)
-; DARWIN-NEXT:    movq %rax, %gs:(%rdi)
-; DARWIN-NEXT:    retq
+; LINUX-SKL-LABEL: addrspace256:
+; LINUX-SKL:       # %bb.0:
+; LINUX-SKL-NEXT:    vmovups %gs:(%rsi), %xmm0
+; LINUX-SKL-NEXT:    vmovups %xmm0, %gs:(%rdi)
+; LINUX-SKL-NEXT:    retq
+;
+; LINUX-SKX-LABEL: addrspace256:
+; LINUX-SKX:       # %bb.0:
+; LINUX-SKX-NEXT:    vmovups %gs:(%rsi), %xmm0
+; LINUX-SKX-NEXT:    vmovups %xmm0, %gs:(%rdi)
+; LINUX-SKX-NEXT:    retq
+;
+; LINUX-KNL-LABEL: addrspace256:
+; LINUX-KNL:       # %bb.0:
+; LINUX-KNL-NEXT:    vmovups %gs:(%rsi), %xmm0
+; LINUX-KNL-NEXT:    vmovups %xmm0, %gs:(%rdi)
+; LINUX-KNL-NEXT:    retq
+;
+; LINUX-AVX512BW-LABEL: addrspace256:
+; LINUX-AVX512BW:       # %bb.0:
+; LINUX-AVX512BW-NEXT:    vmovups %gs:(%rsi), %xmm0
+; LINUX-AVX512BW-NEXT:    vmovups %xmm0, %gs:(%rdi)
+; LINUX-AVX512BW-NEXT:    retq
   tail call void @llvm.memcpy.p256i8.p256i8.i64(i8 addrspace(256)* align 8 %a, i8 addrspace(256)* align 8 %b, i64 16, i1 false)
   ret void
 }
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 1, !"ProfileSummary", !1}
+!1 = !{!2, !3, !4, !5, !6, !7, !8, !9}
+!2 = !{!"ProfileFormat", !"InstrProf"}
+!3 = !{!"TotalCount", i64 10000}
+!4 = !{!"MaxCount", i64 10}
+!5 = !{!"MaxInternalCount", i64 1}
+!6 = !{!"MaxFunctionCount", i64 1000}
+!7 = !{!"NumCounts", i64 3}
+!8 = !{!"NumFunctions", i64 3}
+!9 = !{!"DetailedSummary", !10}
+!10 = !{!11, !12, !13}
+!11 = !{i32 10000, i64 100, i32 1}
+!12 = !{i32 999000, i64 100, i32 1}
+!13 = !{i32 999999, i64 1, i32 2}
+!14 = !{!"function_entry_count", i64 0}
diff --git a/llvm/test/CodeGen/X86/memset-3.ll b/llvm/test/CodeGen/X86/memset-3.ll
index 47c7ab99d..1cf377c81 100644
--- a/llvm/test/CodeGen/X86/memset-3.ll
+++ b/llvm/test/CodeGen/X86/memset-3.ll
@@ -1,7 +1,12 @@
-; RUN: llc -mtriple=i386-apple-darwin < %s | not grep memset
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=i386-apple-darwin < %s | FileCheck %s --implicit-check-not memset
 ; PR6767
 
 define void @t() nounwind ssp {
+; CHECK-LABEL: t:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    subl $512, %esp ## imm = 0x200
+; CHECK-NEXT:    ud2
 entry:
   %buf = alloca [512 x i8], align 1
   %ptr = getelementptr inbounds [512 x i8], [512 x i8]* %buf, i32 0, i32 0
diff --git a/llvm/test/CodeGen/X86/memset-nonzero.ll b/llvm/test/CodeGen/X86/memset-nonzero.ll
index 8166d009d..6e3dedbf8 100644
--- a/llvm/test/CodeGen/X86/memset-nonzero.ll
+++ b/llvm/test/CodeGen/X86/memset-nonzero.ll
@@ -1,9 +1,14 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=x86_64-unknown-unknown < %s -mattr=sse | FileCheck %s --check-prefix=SSE
+; RUN: llc -mtriple=x86_64-unknown-unknown < %s -mattr=sse  | FileCheck %s --check-prefix=SSE
 ; RUN: llc -mtriple=x86_64-unknown-unknown < %s -mattr=sse2 | FileCheck %s --check-prefix=SSE
 ; RUN: llc -mtriple=x86_64-unknown-unknown < %s -mattr=sse2,-slow-unaligned-mem-16 | FileCheck %s --check-prefix=SSE2FAST
-; RUN: llc -mtriple=x86_64-unknown-unknown < %s -mattr=avx | FileCheck %s --check-prefix=AVX --check-prefix=AVX1
+; RUN: llc -mtriple=x86_64-unknown-unknown < %s -mattr=avx  | FileCheck %s --check-prefix=AVX --check-prefix=AVX1
 ; RUN: llc -mtriple=x86_64-unknown-unknown < %s -mattr=avx2 | FileCheck %s --check-prefix=AVX --check-prefix=AVX2
+; RUN: llc -mtriple=x86_64-unknown-unknown < %s -mattr=avx512f  -mattr=+prefer-256-bit | FileCheck %s --check-prefix=AVX --check-prefix=AVX2
+; RUN: llc -mtriple=x86_64-unknown-unknown < %s -mattr=avx512bw -mattr=+prefer-256-bit | FileCheck %s --check-prefix=AVX --check-prefix=AVX2
+; RUN: llc -mtriple=x86_64-unknown-unknown < %s -mattr=avx512dq -mattr=+prefer-256-bit | FileCheck %s --check-prefix=AVX --check-prefix=AVX2
+; RUN: llc -mtriple=x86_64-unknown-unknown < %s -mattr=avx512f  -mattr=-prefer-256-bit | FileCheck %s --check-prefix=AVX --check-prefix=AVX512 --check-prefix=AVX512F
+; RUN: llc -mtriple=x86_64-unknown-unknown < %s -mattr=avx512bw -mattr=-prefer-256-bit | FileCheck %s --check-prefix=AVX --check-prefix=AVX512 --check-prefix=AVX512BW
 
 ; https://llvm.org/bugs/show_bug.cgi?id=27100
 
@@ -80,13 +85,36 @@ define void @memset_64_nonzero_bytes(i8* %x) {
 ; SSE2FAST-NEXT:    movups %xmm0, (%rdi)
 ; SSE2FAST-NEXT:    retq
 ;
-; AVX-LABEL: memset_64_nonzero_bytes:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42]
-; AVX-NEXT:    vmovups %ymm0, 32(%rdi)
-; AVX-NEXT:    vmovups %ymm0, (%rdi)
-; AVX-NEXT:    vzeroupper
-; AVX-NEXT:    retq
+; AVX1-LABEL: memset_64_nonzero_bytes:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vmovaps {{.*#+}} ymm0 = [42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42]
+; AVX1-NEXT:    vmovups %ymm0, 32(%rdi)
+; AVX1-NEXT:    vmovups %ymm0, (%rdi)
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: memset_64_nonzero_bytes:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vmovaps {{.*#+}} ymm0 = [42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42]
+; AVX2-NEXT:    vmovups %ymm0, 32(%rdi)
+; AVX2-NEXT:    vmovups %ymm0, (%rdi)
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+;
+; AVX512F-LABEL: memset_64_nonzero_bytes:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vbroadcastss {{.*#+}} zmm0 = [707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378]
+; AVX512F-NEXT:    vmovups %zmm0, (%rdi)
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: memset_64_nonzero_bytes:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vmovaps {{.*#+}} zmm0 = [42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42]
+; AVX512BW-NEXT:    vmovups %zmm0, (%rdi)
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+; AVX512NW-NEXT: retq
   %call = tail call i8* @__memset_chk(i8* %x, i32 42, i64 64, i64 -1)
   ret void
 }
@@ -126,15 +154,41 @@ define void @memset_128_nonzero_bytes(i8* %x) {
 ; SSE2FAST-NEXT:    movups %xmm0, (%rdi)
 ; SSE2FAST-NEXT:    retq
 ;
-; AVX-LABEL: memset_128_nonzero_bytes:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42]
-; AVX-NEXT:    vmovups %ymm0, 96(%rdi)
-; AVX-NEXT:    vmovups %ymm0, 64(%rdi)
-; AVX-NEXT:    vmovups %ymm0, 32(%rdi)
-; AVX-NEXT:    vmovups %ymm0, (%rdi)
-; AVX-NEXT:    vzeroupper
-; AVX-NEXT:    retq
+; AVX1-LABEL: memset_128_nonzero_bytes:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vmovaps {{.*#+}} ymm0 = [42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42]
+; AVX1-NEXT:    vmovups %ymm0, 96(%rdi)
+; AVX1-NEXT:    vmovups %ymm0, 64(%rdi)
+; AVX1-NEXT:    vmovups %ymm0, 32(%rdi)
+; AVX1-NEXT:    vmovups %ymm0, (%rdi)
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: memset_128_nonzero_bytes:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vmovaps {{.*#+}} ymm0 = [42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42]
+; AVX2-NEXT:    vmovups %ymm0, 96(%rdi)
+; AVX2-NEXT:    vmovups %ymm0, 64(%rdi)
+; AVX2-NEXT:    vmovups %ymm0, 32(%rdi)
+; AVX2-NEXT:    vmovups %ymm0, (%rdi)
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+;
+; AVX512F-LABEL: memset_128_nonzero_bytes:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vbroadcastss {{.*#+}} zmm0 = [707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378]
+; AVX512F-NEXT:    vmovups %zmm0, 64(%rdi)
+; AVX512F-NEXT:    vmovups %zmm0, (%rdi)
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: memset_128_nonzero_bytes:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vmovaps {{.*#+}} zmm0 = [42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42]
+; AVX512BW-NEXT:    vmovups %zmm0, 64(%rdi)
+; AVX512BW-NEXT:    vmovups %zmm0, (%rdi)
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
   %call = tail call i8* @__memset_chk(i8* %x, i32 42, i64 128, i64 -1)
   ret void
 }
@@ -172,19 +226,53 @@ define void @memset_256_nonzero_bytes(i8* %x) {
 ; SSE2FAST-NEXT:    movups %xmm0, (%rdi)
 ; SSE2FAST-NEXT:    retq
 ;
-; AVX-LABEL: memset_256_nonzero_bytes:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42]
-; AVX-NEXT:    vmovups %ymm0, 224(%rdi)
-; AVX-NEXT:    vmovups %ymm0, 192(%rdi)
-; AVX-NEXT:    vmovups %ymm0, 160(%rdi)
-; AVX-NEXT:    vmovups %ymm0, 128(%rdi)
-; AVX-NEXT:    vmovups %ymm0, 96(%rdi)
-; AVX-NEXT:    vmovups %ymm0, 64(%rdi)
-; AVX-NEXT:    vmovups %ymm0, 32(%rdi)
-; AVX-NEXT:    vmovups %ymm0, (%rdi)
-; AVX-NEXT:    vzeroupper
-; AVX-NEXT:    retq
+; AVX1-LABEL: memset_256_nonzero_bytes:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vmovaps {{.*#+}} ymm0 = [42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42]
+; AVX1-NEXT:    vmovups %ymm0, 224(%rdi)
+; AVX1-NEXT:    vmovups %ymm0, 192(%rdi)
+; AVX1-NEXT:    vmovups %ymm0, 160(%rdi)
+; AVX1-NEXT:    vmovups %ymm0, 128(%rdi)
+; AVX1-NEXT:    vmovups %ymm0, 96(%rdi)
+; AVX1-NEXT:    vmovups %ymm0, 64(%rdi)
+; AVX1-NEXT:    vmovups %ymm0, 32(%rdi)
+; AVX1-NEXT:    vmovups %ymm0, (%rdi)
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: memset_256_nonzero_bytes:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vmovaps {{.*#+}} ymm0 = [42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42]
+; AVX2-NEXT:    vmovups %ymm0, 224(%rdi)
+; AVX2-NEXT:    vmovups %ymm0, 192(%rdi)
+; AVX2-NEXT:    vmovups %ymm0, 160(%rdi)
+; AVX2-NEXT:    vmovups %ymm0, 128(%rdi)
+; AVX2-NEXT:    vmovups %ymm0, 96(%rdi)
+; AVX2-NEXT:    vmovups %ymm0, 64(%rdi)
+; AVX2-NEXT:    vmovups %ymm0, 32(%rdi)
+; AVX2-NEXT:    vmovups %ymm0, (%rdi)
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+;
+; AVX512F-LABEL: memset_256_nonzero_bytes:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vbroadcastss {{.*#+}} zmm0 = [707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378]
+; AVX512F-NEXT:    vmovups %zmm0, 192(%rdi)
+; AVX512F-NEXT:    vmovups %zmm0, 128(%rdi)
+; AVX512F-NEXT:    vmovups %zmm0, 64(%rdi)
+; AVX512F-NEXT:    vmovups %zmm0, (%rdi)
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: memset_256_nonzero_bytes:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vmovaps {{.*#+}} zmm0 = [42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42]
+; AVX512BW-NEXT:    vmovups %zmm0, 192(%rdi)
+; AVX512BW-NEXT:    vmovups %zmm0, 128(%rdi)
+; AVX512BW-NEXT:    vmovups %zmm0, 64(%rdi)
+; AVX512BW-NEXT:    vmovups %zmm0, (%rdi)
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
   %call = tail call i8* @__memset_chk(i8* %x, i32 42, i64 256, i64 -1)
   ret void
 }
@@ -227,6 +315,13 @@ define void @memset_16_nonconst_bytes(i8* %x, i8 %c) {
 ; AVX2-NEXT:    vpbroadcastb %xmm0, %xmm0
 ; AVX2-NEXT:    vmovdqu %xmm0, (%rdi)
 ; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: memset_16_nonconst_bytes:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovd %esi, %xmm0
+; AVX512-NEXT:    vpbroadcastb %xmm0, %xmm0
+; AVX512-NEXT:    vmovdqu %xmm0, (%rdi)
+; AVX512-NEXT:    retq
   tail call void @llvm.memset.p0i8.i64(i8* %x, i8 %c, i64 16, i1 false)
   ret void
 }
@@ -270,6 +365,14 @@ define void @memset_32_nonconst_bytes(i8* %x, i8 %c) {
 ; AVX2-NEXT:    vmovdqu %ymm0, (%rdi)
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: memset_32_nonconst_bytes:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovd %esi, %xmm0
+; AVX512-NEXT:    vpbroadcastb %xmm0, %ymm0
+; AVX512-NEXT:    vmovdqu %ymm0, (%rdi)
+; AVX512-NEXT:    vzeroupper
+; AVX512-NEXT:    retq
   tail call void @llvm.memset.p0i8.i64(i8* %x, i8 %c, i64 32, i1 false)
   ret void
 }
@@ -322,6 +425,22 @@ define void @memset_64_nonconst_bytes(i8* %x, i8 %c) {
 ; AVX2-NEXT:    vmovdqu %ymm0, (%rdi)
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
+;
+; AVX512F-LABEL: memset_64_nonconst_bytes:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    movzbl %sil, %eax
+; AVX512F-NEXT:    imull $16843009, %eax, %eax # imm = 0x1010101
+; AVX512F-NEXT:    vpbroadcastd %eax, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, (%rdi)
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: memset_64_nonconst_bytes:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpbroadcastb %esi, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, (%rdi)
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
   tail call void @llvm.memset.p0i8.i64(i8* %x, i8 %c, i64 64, i1 false)
   ret void
 }
@@ -390,6 +509,24 @@ define void @memset_128_nonconst_bytes(i8* %x, i8 %c) {
 ; AVX2-NEXT:    vmovdqu %ymm0, (%rdi)
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
+;
+; AVX512F-LABEL: memset_128_nonconst_bytes:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    movzbl %sil, %eax
+; AVX512F-NEXT:    imull $16843009, %eax, %eax # imm = 0x1010101
+; AVX512F-NEXT:    vpbroadcastd %eax, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, 64(%rdi)
+; AVX512F-NEXT:    vmovdqu64 %zmm0, (%rdi)
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: memset_128_nonconst_bytes:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpbroadcastb %esi, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, 64(%rdi)
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, (%rdi)
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
   tail call void @llvm.memset.p0i8.i64(i8* %x, i8 %c, i64 128, i1 false)
   ret void
 }
@@ -455,6 +592,28 @@ define void @memset_256_nonconst_bytes(i8* %x, i8 %c) {
 ; AVX2-NEXT:    vmovdqu %ymm0, (%rdi)
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
+;
+; AVX512F-LABEL: memset_256_nonconst_bytes:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    movzbl %sil, %eax
+; AVX512F-NEXT:    imull $16843009, %eax, %eax # imm = 0x1010101
+; AVX512F-NEXT:    vpbroadcastd %eax, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, 192(%rdi)
+; AVX512F-NEXT:    vmovdqu64 %zmm0, 128(%rdi)
+; AVX512F-NEXT:    vmovdqu64 %zmm0, 64(%rdi)
+; AVX512F-NEXT:    vmovdqu64 %zmm0, (%rdi)
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: memset_256_nonconst_bytes:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpbroadcastb %esi, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, 192(%rdi)
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, 128(%rdi)
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, 64(%rdi)
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, (%rdi)
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
   tail call void @llvm.memset.p0i8.i64(i8* %x, i8 %c, i64 256, i1 false)
   ret void
 }
diff --git a/llvm/test/CodeGen/X86/memset-sse-stack-realignment.ll b/llvm/test/CodeGen/X86/memset-sse-stack-realignment.ll
index 68fa15e33..d1f05d87f 100644
--- a/llvm/test/CodeGen/X86/memset-sse-stack-realignment.ll
+++ b/llvm/test/CodeGen/X86/memset-sse-stack-realignment.ll
@@ -1,75 +1,159 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; Make sure that we realign the stack. Mingw32 uses 4 byte stack alignment, we
 ; need 16 bytes for SSE and 32 bytes for AVX.
 
-; RUN: llc < %s -mtriple=i386-pc-mingw32 -mcpu=pentium2 | FileCheck %s -check-prefix=NOSSE
-; RUN: llc < %s -mtriple=i386-pc-mingw32 -mcpu=pentium3 | FileCheck %s -check-prefix=SSE1
-; RUN: llc < %s -mtriple=i386-pc-mingw32 -mcpu=yonah | FileCheck %s -check-prefix=SSE2
-; RUN: llc < %s -mtriple=i386-pc-mingw32 -mcpu=corei7-avx | FileCheck %s -check-prefix=AVX1
-; RUN: llc < %s -mtriple=i386-pc-mingw32 -mcpu=core-avx2 | FileCheck %s -check-prefix=AVX2
+; RUN: llc < %s -mtriple=i386-pc-mingw32 -mcpu=pentium2 | FileCheck %s --check-prefix=NOSSE
+; RUN: llc < %s -mtriple=i386-pc-mingw32 -mcpu=pentium3 | FileCheck %s --check-prefixes=SSE,SSE1
+; RUN: llc < %s -mtriple=i386-pc-mingw32 -mcpu=yonah | FileCheck %s --check-prefixes=SSE,SSE2
+; RUN: llc < %s -mtriple=i386-pc-mingw32 -mcpu=corei7-avx | FileCheck %s --check-prefixes=AVX,AVX1
+; RUN: llc < %s -mtriple=i386-pc-mingw32 -mcpu=core-avx2 | FileCheck %s --check-prefixes=AVX,AVX2
 
 define void @test1(i32 %t) nounwind {
+; NOSSE-LABEL: test1:
+; NOSSE:       # %bb.0:
+; NOSSE-NEXT:    pushl %ebp
+; NOSSE-NEXT:    movl %esp, %ebp
+; NOSSE-NEXT:    subl $32, %esp
+; NOSSE-NEXT:    movl 8(%ebp), %eax
+; NOSSE-NEXT:    movl $0, -4(%ebp)
+; NOSSE-NEXT:    movl $0, -8(%ebp)
+; NOSSE-NEXT:    movl $0, -12(%ebp)
+; NOSSE-NEXT:    movl $0, -16(%ebp)
+; NOSSE-NEXT:    movl $0, -20(%ebp)
+; NOSSE-NEXT:    movl $0, -24(%ebp)
+; NOSSE-NEXT:    movl $0, -28(%ebp)
+; NOSSE-NEXT:    movl $0, -32(%ebp)
+; NOSSE-NEXT:    addl $3, %eax
+; NOSSE-NEXT:    andl $-4, %eax
+; NOSSE-NEXT:    calll __alloca
+; NOSSE-NEXT:    movl %esp, %eax
+; NOSSE-NEXT:    pushl %eax
+; NOSSE-NEXT:    calll _dummy
+; NOSSE-NEXT:    movl %ebp, %esp
+; NOSSE-NEXT:    popl %ebp
+; NOSSE-NEXT:    retl
+;
+; SSE-LABEL: test1:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pushl %ebp
+; SSE-NEXT:    movl %esp, %ebp
+; SSE-NEXT:    pushl %esi
+; SSE-NEXT:    andl $-16, %esp
+; SSE-NEXT:    subl $48, %esp
+; SSE-NEXT:    movl %esp, %esi
+; SSE-NEXT:    movl 8(%ebp), %eax
+; SSE-NEXT:    xorps %xmm0, %xmm0
+; SSE-NEXT:    movaps %xmm0, 16(%esi)
+; SSE-NEXT:    movaps %xmm0, (%esi)
+; SSE-NEXT:    addl $3, %eax
+; SSE-NEXT:    andl $-4, %eax
+; SSE-NEXT:    calll __alloca
+; SSE-NEXT:    movl %esp, %eax
+; SSE-NEXT:    pushl %eax
+; SSE-NEXT:    calll _dummy
+; SSE-NEXT:    leal -4(%ebp), %esp
+; SSE-NEXT:    popl %esi
+; SSE-NEXT:    popl %ebp
+; SSE-NEXT:    retl
+;
+; AVX-LABEL: test1:
+; AVX:       # %bb.0:
+; AVX-NEXT:    pushl %ebp
+; AVX-NEXT:    movl %esp, %ebp
+; AVX-NEXT:    pushl %esi
+; AVX-NEXT:    andl $-32, %esp
+; AVX-NEXT:    subl $64, %esp
+; AVX-NEXT:    movl %esp, %esi
+; AVX-NEXT:    movl 8(%ebp), %eax
+; AVX-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vmovaps %ymm0, (%esi)
+; AVX-NEXT:    addl $3, %eax
+; AVX-NEXT:    andl $-4, %eax
+; AVX-NEXT:    calll __alloca
+; AVX-NEXT:    movl %esp, %eax
+; AVX-NEXT:    pushl %eax
+; AVX-NEXT:    vzeroupper
+; AVX-NEXT:    calll _dummy
+; AVX-NEXT:    leal -4(%ebp), %esp
+; AVX-NEXT:    popl %esi
+; AVX-NEXT:    popl %ebp
+; AVX-NEXT:    retl
   %tmp1210 = alloca i8, i32 32, align 4
   call void @llvm.memset.p0i8.i64(i8* align 4 %tmp1210, i8 0, i64 32, i1 false)
   %x = alloca i8, i32 %t
   call void @dummy(i8* %x)
   ret void
-
-; NOSSE-LABEL: test1:
-; NOSSE-NOT: and
-; NOSSE: movl $0
-
-; SSE1-LABEL: test1:
-; SSE1: andl $-16
-; SSE1: movl %esp, %esi
-; SSE1: movaps
-
-; SSE2-LABEL: test1:
-; SSE2: andl $-16
-; SSE2: movl %esp, %esi
-; SSE2: movaps
-
-; AVX1-LABEL: test1:
-; AVX1: andl $-32
-; AVX1: movl %esp, %esi
-; AVX1: vmovaps %ymm
-
-; AVX2-LABEL: test1:
-; AVX2: andl $-32
-; AVX2: movl %esp, %esi
-; AVX2: vmovaps %ymm
-
 }
 
 define void @test2(i32 %t) nounwind {
+; NOSSE-LABEL: test2:
+; NOSSE:       # %bb.0:
+; NOSSE-NEXT:    pushl %ebp
+; NOSSE-NEXT:    movl %esp, %ebp
+; NOSSE-NEXT:    subl $16, %esp
+; NOSSE-NEXT:    movl 8(%ebp), %eax
+; NOSSE-NEXT:    movl $0, -4(%ebp)
+; NOSSE-NEXT:    movl $0, -8(%ebp)
+; NOSSE-NEXT:    movl $0, -12(%ebp)
+; NOSSE-NEXT:    movl $0, -16(%ebp)
+; NOSSE-NEXT:    addl $3, %eax
+; NOSSE-NEXT:    andl $-4, %eax
+; NOSSE-NEXT:    calll __alloca
+; NOSSE-NEXT:    movl %esp, %eax
+; NOSSE-NEXT:    pushl %eax
+; NOSSE-NEXT:    calll _dummy
+; NOSSE-NEXT:    movl %ebp, %esp
+; NOSSE-NEXT:    popl %ebp
+; NOSSE-NEXT:    retl
+;
+; SSE-LABEL: test2:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pushl %ebp
+; SSE-NEXT:    movl %esp, %ebp
+; SSE-NEXT:    pushl %esi
+; SSE-NEXT:    andl $-16, %esp
+; SSE-NEXT:    subl $32, %esp
+; SSE-NEXT:    movl %esp, %esi
+; SSE-NEXT:    movl 8(%ebp), %eax
+; SSE-NEXT:    xorps %xmm0, %xmm0
+; SSE-NEXT:    movaps %xmm0, (%esi)
+; SSE-NEXT:    addl $3, %eax
+; SSE-NEXT:    andl $-4, %eax
+; SSE-NEXT:    calll __alloca
+; SSE-NEXT:    movl %esp, %eax
+; SSE-NEXT:    pushl %eax
+; SSE-NEXT:    calll _dummy
+; SSE-NEXT:    leal -4(%ebp), %esp
+; SSE-NEXT:    popl %esi
+; SSE-NEXT:    popl %ebp
+; SSE-NEXT:    retl
+;
+; AVX-LABEL: test2:
+; AVX:       # %bb.0:
+; AVX-NEXT:    pushl %ebp
+; AVX-NEXT:    movl %esp, %ebp
+; AVX-NEXT:    pushl %esi
+; AVX-NEXT:    andl $-16, %esp
+; AVX-NEXT:    subl $32, %esp
+; AVX-NEXT:    movl %esp, %esi
+; AVX-NEXT:    movl 8(%ebp), %eax
+; AVX-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vmovaps %xmm0, (%esi)
+; AVX-NEXT:    addl $3, %eax
+; AVX-NEXT:    andl $-4, %eax
+; AVX-NEXT:    calll __alloca
+; AVX-NEXT:    movl %esp, %eax
+; AVX-NEXT:    pushl %eax
+; AVX-NEXT:    calll _dummy
+; AVX-NEXT:    leal -4(%ebp), %esp
+; AVX-NEXT:    popl %esi
+; AVX-NEXT:    popl %ebp
+; AVX-NEXT:    retl
   %tmp1210 = alloca i8, i32 16, align 4
   call void @llvm.memset.p0i8.i64(i8* align 4 %tmp1210, i8 0, i64 16, i1 false)
   %x = alloca i8, i32 %t
   call void @dummy(i8* %x)
   ret void
-
-; NOSSE-LABEL: test2:
-; NOSSE-NOT: and
-; NOSSE: movl $0
-
-; SSE1-LABEL: test2:
-; SSE1: andl $-16
-; SSE1: movl %esp, %esi
-; SSE1: movaps
-
-; SSE2-LABEL: test2:
-; SSE2: andl $-16
-; SSE2: movl %esp, %esi
-; SSE2: movaps
-
-; AVX1-LABEL: test2:
-; AVX1: andl $-16
-; AVX1: movl %esp, %esi
-; AVX1: vmovaps %xmm
-
-; AVX2-LABEL: test2:
-; AVX2: andl $-16
-; AVX2: movl %esp, %esi
-; AVX2: vmovaps %xmm
 }
 
 declare void @dummy(i8*)
diff --git a/llvm/test/CodeGen/X86/memset-zero.ll b/llvm/test/CodeGen/X86/memset-zero.ll
index fb28742e6..c6465bd46 100644
--- a/llvm/test/CodeGen/X86/memset-zero.ll
+++ b/llvm/test/CodeGen/X86/memset-zero.ll
@@ -4,6 +4,7 @@
 ; RUN: llc < %s -mtriple=x86_64-unknown-linux -mcpu=nehalem | FileCheck %s --check-prefix=NEHALEM
 ; RUN: llc < %s -mtriple=x86_64-unknown-linux -mcpu=sandybridge | FileCheck %s --check-prefix=SANDYBRIDGE
 ; RUN: llc < %s -mtriple=x86_64-unknown-linux -mcpu=skylake | FileCheck %s --check-prefix=SKYLAKE
+; RUN: llc < %s -mtriple=x86_64-unknown-linux -mcpu=knl | FileCheck %s --check-prefix=KNL
 
 
 declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i1) nounwind
@@ -28,6 +29,10 @@ define void @memset_0(i8* %a) nounwind  {
 ; SKYLAKE-LABEL: memset_0:
 ; SKYLAKE:       # %bb.0: # %entry
 ; SKYLAKE-NEXT:    retq
+;
+; KNL-LABEL: memset_0:
+; KNL:       # %bb.0: # %entry
+; KNL-NEXT:    retq
 entry:
 	call void @llvm.memset.p0i8.i64(i8* %a, i8 0, i64 0, i1 false)
 	ret void
@@ -59,6 +64,11 @@ define void @memset_4(i8* %a) nounwind  {
 ; SKYLAKE:       # %bb.0: # %entry
 ; SKYLAKE-NEXT:    movl $0, (%rdi)
 ; SKYLAKE-NEXT:    retq
+;
+; KNL-LABEL: memset_4:
+; KNL:       # %bb.0: # %entry
+; KNL-NEXT:    movl $0, (%rdi)
+; KNL-NEXT:    retq
 entry:
 	call void @llvm.memset.p0i8.i64(i8* %a, i8 0, i64 4, i1 false)
 	ret void
@@ -95,6 +105,12 @@ define void @memset_5(i8* %a) nounwind  {
 ; SKYLAKE-NEXT:    movb $0, 4(%rdi)
 ; SKYLAKE-NEXT:    movl $0, (%rdi)
 ; SKYLAKE-NEXT:    retq
+;
+; KNL-LABEL: memset_5:
+; KNL:       # %bb.0: # %entry
+; KNL-NEXT:    movb $0, 4(%rdi)
+; KNL-NEXT:    movl $0, (%rdi)
+; KNL-NEXT:    retq
 entry:
 	call void @llvm.memset.p0i8.i64(i8* %a, i8 0, i64 5, i1 false)
 	ret void
@@ -131,6 +147,12 @@ define void @memset_7(i8* %a) nounwind  {
 ; SKYLAKE-NEXT:    movl $0, 3(%rdi)
 ; SKYLAKE-NEXT:    movl $0, (%rdi)
 ; SKYLAKE-NEXT:    retq
+;
+; KNL-LABEL: memset_7:
+; KNL:       # %bb.0: # %entry
+; KNL-NEXT:    movl $0, 3(%rdi)
+; KNL-NEXT:    movl $0, (%rdi)
+; KNL-NEXT:    retq
 entry:
 	call void @llvm.memset.p0i8.i64(i8* %a, i8 0, i64 7, i1 false)
 	ret void
@@ -163,6 +185,11 @@ define void @memset_8(i8* %a) nounwind  {
 ; SKYLAKE:       # %bb.0: # %entry
 ; SKYLAKE-NEXT:    movq $0, (%rdi)
 ; SKYLAKE-NEXT:    retq
+;
+; KNL-LABEL: memset_8:
+; KNL:       # %bb.0: # %entry
+; KNL-NEXT:    movq $0, (%rdi)
+; KNL-NEXT:    retq
 entry:
 	call void @llvm.memset.p0i8.i64(i8* %a, i8 0, i64 8, i1 false)
 	ret void
@@ -200,6 +227,12 @@ define void @memset_11(i8* %a) nounwind  {
 ; SKYLAKE-NEXT:    movl $0, 7(%rdi)
 ; SKYLAKE-NEXT:    movq $0, (%rdi)
 ; SKYLAKE-NEXT:    retq
+;
+; KNL-LABEL: memset_11:
+; KNL:       # %bb.0: # %entry
+; KNL-NEXT:    movl $0, 7(%rdi)
+; KNL-NEXT:    movq $0, (%rdi)
+; KNL-NEXT:    retq
 entry:
 	call void @llvm.memset.p0i8.i64(i8* %a, i8 0, i64 11, i1 false)
 	ret void
@@ -238,6 +271,12 @@ define void @memset_13(i8* %a) nounwind  {
 ; SKYLAKE-NEXT:    movq $0, 5(%rdi)
 ; SKYLAKE-NEXT:    movq $0, (%rdi)
 ; SKYLAKE-NEXT:    retq
+;
+; KNL-LABEL: memset_13:
+; KNL:       # %bb.0: # %entry
+; KNL-NEXT:    movq $0, 5(%rdi)
+; KNL-NEXT:    movq $0, (%rdi)
+; KNL-NEXT:    retq
 entry:
 	call void @llvm.memset.p0i8.i64(i8* %a, i8 0, i64 13, i1 false)
 	ret void
@@ -276,6 +315,12 @@ define void @memset_15(i8* %a) nounwind  {
 ; SKYLAKE-NEXT:    movq $0, 7(%rdi)
 ; SKYLAKE-NEXT:    movq $0, (%rdi)
 ; SKYLAKE-NEXT:    retq
+;
+; KNL-LABEL: memset_15:
+; KNL:       # %bb.0: # %entry
+; KNL-NEXT:    movq $0, 7(%rdi)
+; KNL-NEXT:    movq $0, (%rdi)
+; KNL-NEXT:    retq
 entry:
 	call void @llvm.memset.p0i8.i64(i8* %a, i8 0, i64 15, i1 false)
 	ret void
@@ -314,6 +359,12 @@ define void @memset_16(i8* %a) nounwind  {
 ; SKYLAKE-NEXT:    vxorps %xmm0, %xmm0, %xmm0
 ; SKYLAKE-NEXT:    vmovups %xmm0, (%rdi)
 ; SKYLAKE-NEXT:    retq
+;
+; KNL-LABEL: memset_16:
+; KNL:       # %bb.0: # %entry
+; KNL-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; KNL-NEXT:    vmovups %xmm0, (%rdi)
+; KNL-NEXT:    retq
 entry:
 	call void @llvm.memset.p0i8.i64(i8* %a, i8 0, i64 16, i1 false)
 	ret void
@@ -357,6 +408,13 @@ define void @memset_17(i8* %a) nounwind  {
 ; SKYLAKE-NEXT:    vmovups %xmm0, (%rdi)
 ; SKYLAKE-NEXT:    movb $0, 16(%rdi)
 ; SKYLAKE-NEXT:    retq
+;
+; KNL-LABEL: memset_17:
+; KNL:       # %bb.0: # %entry
+; KNL-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; KNL-NEXT:    vmovups %xmm0, (%rdi)
+; KNL-NEXT:    movb $0, 16(%rdi)
+; KNL-NEXT:    retq
 entry:
 	call void @llvm.memset.p0i8.i64(i8* %a, i8 0, i64 17, i1 false)
 	ret void
@@ -400,6 +458,13 @@ define void @memset_19(i8* %a) nounwind  {
 ; SKYLAKE-NEXT:    vmovups %xmm0, (%rdi)
 ; SKYLAKE-NEXT:    movl $0, 15(%rdi)
 ; SKYLAKE-NEXT:    retq
+;
+; KNL-LABEL: memset_19:
+; KNL:       # %bb.0: # %entry
+; KNL-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; KNL-NEXT:    vmovups %xmm0, (%rdi)
+; KNL-NEXT:    movl $0, 15(%rdi)
+; KNL-NEXT:    retq
 entry:
 	call void @llvm.memset.p0i8.i64(i8* %a, i8 0, i64 19, i1 false)
 	ret void
@@ -447,6 +512,13 @@ define void @memset_31(i8* %a) nounwind  {
 ; SKYLAKE-NEXT:    vmovups %xmm0, 15(%rdi)
 ; SKYLAKE-NEXT:    vmovups %xmm0, (%rdi)
 ; SKYLAKE-NEXT:    retq
+;
+; KNL-LABEL: memset_31:
+; KNL:       # %bb.0: # %entry
+; KNL-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; KNL-NEXT:    vmovups %xmm0, 15(%rdi)
+; KNL-NEXT:    vmovups %xmm0, (%rdi)
+; KNL-NEXT:    retq
 entry:
 	call void @llvm.memset.p0i8.i64(i8* %a, i8 0, i64 31, i1 false)
 	ret void
@@ -494,6 +566,12 @@ define void @memset_32(i8* %a) nounwind  {
 ; SKYLAKE-NEXT:    vmovups %ymm0, (%rdi)
 ; SKYLAKE-NEXT:    vzeroupper
 ; SKYLAKE-NEXT:    retq
+;
+; KNL-LABEL: memset_32:
+; KNL:       # %bb.0: # %entry
+; KNL-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; KNL-NEXT:    vmovups %ymm0, (%rdi)
+; KNL-NEXT:    retq
 entry:
 	call void @llvm.memset.p0i8.i64(i8* %a, i8 0, i64 32, i1 false)
 	ret void
@@ -540,6 +618,12 @@ define void @memset_32_align32(i8* %a) nounwind  {
 ; SKYLAKE-NEXT:    vmovaps %ymm0, (%rdi)
 ; SKYLAKE-NEXT:    vzeroupper
 ; SKYLAKE-NEXT:    retq
+;
+; KNL-LABEL: memset_32_align32:
+; KNL:       # %bb.0: # %entry
+; KNL-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; KNL-NEXT:    vmovaps %ymm0, (%rdi)
+; KNL-NEXT:    retq
 entry:
 	call void @llvm.memset.p0i8.i64(i8* align 32 %a, i8 0, i64 32, i1 false)
 	ret void
@@ -593,7 +677,150 @@ define void @memset_35(i8* %a) nounwind  {
 ; SKYLAKE-NEXT:    movl $0, 31(%rdi)
 ; SKYLAKE-NEXT:    vzeroupper
 ; SKYLAKE-NEXT:    retq
+;
+; KNL-LABEL: memset_35:
+; KNL:       # %bb.0: # %entry
+; KNL-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; KNL-NEXT:    vmovups %ymm0, (%rdi)
+; KNL-NEXT:    movl $0, 31(%rdi)
+; KNL-NEXT:    retq
 entry:
 	call void @llvm.memset.p0i8.i64(i8* %a, i8 0, i64 35, i1 false)
 	ret void
 }
+
+define void @memset_64(i8* %a) nounwind  {
+; X86-LABEL: memset_64:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl $0, 60(%eax)
+; X86-NEXT:    movl $0, 56(%eax)
+; X86-NEXT:    movl $0, 52(%eax)
+; X86-NEXT:    movl $0, 48(%eax)
+; X86-NEXT:    movl $0, 44(%eax)
+; X86-NEXT:    movl $0, 40(%eax)
+; X86-NEXT:    movl $0, 36(%eax)
+; X86-NEXT:    movl $0, 32(%eax)
+; X86-NEXT:    movl $0, 28(%eax)
+; X86-NEXT:    movl $0, 24(%eax)
+; X86-NEXT:    movl $0, 20(%eax)
+; X86-NEXT:    movl $0, 16(%eax)
+; X86-NEXT:    movl $0, 12(%eax)
+; X86-NEXT:    movl $0, 8(%eax)
+; X86-NEXT:    movl $0, 4(%eax)
+; X86-NEXT:    movl $0, (%eax)
+; X86-NEXT:    retl
+;
+; CORE2-LABEL: memset_64:
+; CORE2:       # %bb.0: # %entry
+; CORE2-NEXT:    movq $0, 56(%rdi)
+; CORE2-NEXT:    movq $0, 48(%rdi)
+; CORE2-NEXT:    movq $0, 40(%rdi)
+; CORE2-NEXT:    movq $0, 32(%rdi)
+; CORE2-NEXT:    movq $0, 24(%rdi)
+; CORE2-NEXT:    movq $0, 16(%rdi)
+; CORE2-NEXT:    movq $0, 8(%rdi)
+; CORE2-NEXT:    movq $0, (%rdi)
+; CORE2-NEXT:    retq
+;
+; NEHALEM-LABEL: memset_64:
+; NEHALEM:       # %bb.0: # %entry
+; NEHALEM-NEXT:    xorps %xmm0, %xmm0
+; NEHALEM-NEXT:    movups %xmm0, 48(%rdi)
+; NEHALEM-NEXT:    movups %xmm0, 32(%rdi)
+; NEHALEM-NEXT:    movups %xmm0, 16(%rdi)
+; NEHALEM-NEXT:    movups %xmm0, (%rdi)
+; NEHALEM-NEXT:    retq
+;
+; SANDYBRIDGE-LABEL: memset_64:
+; SANDYBRIDGE:       # %bb.0: # %entry
+; SANDYBRIDGE-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; SANDYBRIDGE-NEXT:    vmovups %xmm0, 16(%rdi)
+; SANDYBRIDGE-NEXT:    vmovups %xmm0, (%rdi)
+; SANDYBRIDGE-NEXT:    vmovups %xmm0, 48(%rdi)
+; SANDYBRIDGE-NEXT:    vmovups %xmm0, 32(%rdi)
+; SANDYBRIDGE-NEXT:    retq
+;
+; SKYLAKE-LABEL: memset_64:
+; SKYLAKE:       # %bb.0: # %entry
+; SKYLAKE-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; SKYLAKE-NEXT:    vmovups %ymm0, 32(%rdi)
+; SKYLAKE-NEXT:    vmovups %ymm0, (%rdi)
+; SKYLAKE-NEXT:    vzeroupper
+; SKYLAKE-NEXT:    retq
+;
+; KNL-LABEL: memset_64:
+; KNL:       # %bb.0: # %entry
+; KNL-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; KNL-NEXT:    vmovups %zmm0, (%rdi)
+; KNL-NEXT:    retq
+entry:
+	call void @llvm.memset.p0i8.i64(i8* %a, i8 0, i64 64, i1 false)
+	ret void
+}
+
+define void @memset_64_align64(i8* %a) nounwind  {
+; X86-LABEL: memset_64_align64:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl $0, 60(%eax)
+; X86-NEXT:    movl $0, 56(%eax)
+; X86-NEXT:    movl $0, 52(%eax)
+; X86-NEXT:    movl $0, 48(%eax)
+; X86-NEXT:    movl $0, 44(%eax)
+; X86-NEXT:    movl $0, 40(%eax)
+; X86-NEXT:    movl $0, 36(%eax)
+; X86-NEXT:    movl $0, 32(%eax)
+; X86-NEXT:    movl $0, 28(%eax)
+; X86-NEXT:    movl $0, 24(%eax)
+; X86-NEXT:    movl $0, 20(%eax)
+; X86-NEXT:    movl $0, 16(%eax)
+; X86-NEXT:    movl $0, 12(%eax)
+; X86-NEXT:    movl $0, 8(%eax)
+; X86-NEXT:    movl $0, 4(%eax)
+; X86-NEXT:    movl $0, (%eax)
+; X86-NEXT:    retl
+;
+; CORE2-LABEL: memset_64_align64:
+; CORE2:       # %bb.0: # %entry
+; CORE2-NEXT:    xorps %xmm0, %xmm0
+; CORE2-NEXT:    movaps %xmm0, 48(%rdi)
+; CORE2-NEXT:    movaps %xmm0, 32(%rdi)
+; CORE2-NEXT:    movaps %xmm0, 16(%rdi)
+; CORE2-NEXT:    movaps %xmm0, (%rdi)
+; CORE2-NEXT:    retq
+;
+; NEHALEM-LABEL: memset_64_align64:
+; NEHALEM:       # %bb.0: # %entry
+; NEHALEM-NEXT:    xorps %xmm0, %xmm0
+; NEHALEM-NEXT:    movaps %xmm0, 48(%rdi)
+; NEHALEM-NEXT:    movaps %xmm0, 32(%rdi)
+; NEHALEM-NEXT:    movaps %xmm0, 16(%rdi)
+; NEHALEM-NEXT:    movaps %xmm0, (%rdi)
+; NEHALEM-NEXT:    retq
+;
+; SANDYBRIDGE-LABEL: memset_64_align64:
+; SANDYBRIDGE:       # %bb.0: # %entry
+; SANDYBRIDGE-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; SANDYBRIDGE-NEXT:    vmovaps %ymm0, 32(%rdi)
+; SANDYBRIDGE-NEXT:    vmovaps %ymm0, (%rdi)
+; SANDYBRIDGE-NEXT:    vzeroupper
+; SANDYBRIDGE-NEXT:    retq
+;
+; SKYLAKE-LABEL: memset_64_align64:
+; SKYLAKE:       # %bb.0: # %entry
+; SKYLAKE-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; SKYLAKE-NEXT:    vmovaps %ymm0, 32(%rdi)
+; SKYLAKE-NEXT:    vmovaps %ymm0, (%rdi)
+; SKYLAKE-NEXT:    vzeroupper
+; SKYLAKE-NEXT:    retq
+;
+; KNL-LABEL: memset_64_align64:
+; KNL:       # %bb.0: # %entry
+; KNL-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; KNL-NEXT:    vmovaps %zmm0, (%rdi)
+; KNL-NEXT:    retq
+entry:
+	call void @llvm.memset.p0i8.i64(i8* align 64 %a, i8 0, i64 64, i1 false)
+	ret void
+}
diff --git a/llvm/test/CodeGen/X86/merge-consecutive-loads-256.ll b/llvm/test/CodeGen/X86/merge-consecutive-loads-256.ll
index c03e2267b..252d2e08b 100644
--- a/llvm/test/CodeGen/X86/merge-consecutive-loads-256.ll
+++ b/llvm/test/CodeGen/X86/merge-consecutive-loads-256.ll
@@ -666,3 +666,32 @@ define <16 x i16> @merge_16i16_i16_0uu3zzuuuuuzCuEF_volatile(i16* %ptr) nounwind
   %resF = insertelement <16 x i16> %resE, i16 %valF, i16 15
   ret <16 x i16> %resF
 }
+
+;
+; Volatile tests.
+;
+
+@l = external global <32 x i8>, align 32
+
+define <2 x i8> @PR42846(<2 x i8>* %j, <2 x i8> %k) {
+; AVX-LABEL: PR42846:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovdqa {{.*}}(%rip), %ymm0
+; AVX-NEXT:    vpextrw $0, %xmm0, (%rdi)
+; AVX-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX-NEXT:    vzeroupper
+; AVX-NEXT:    retq
+;
+; X32-AVX-LABEL: PR42846:
+; X32-AVX:       # %bb.0:
+; X32-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-AVX-NEXT:    vmovdqa l, %ymm0
+; X32-AVX-NEXT:    vpextrw $0, %xmm0, (%eax)
+; X32-AVX-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; X32-AVX-NEXT:    vzeroupper
+; X32-AVX-NEXT:    retl
+  %t0 = load volatile <32 x i8>, <32 x i8>* @l, align 32
+  %shuffle = shufflevector <32 x i8> %t0, <32 x i8> undef, <2 x i32> <i32 0, i32 1>
+  store <2 x i8> %shuffle, <2 x i8>* %j, align 2
+  ret <2 x i8> %shuffle
+}
diff --git a/llvm/test/CodeGen/X86/merge-consecutive-loads-512.ll b/llvm/test/CodeGen/X86/merge-consecutive-loads-512.ll
index eb7eff695..940562426 100644
--- a/llvm/test/CodeGen/X86/merge-consecutive-loads-512.ll
+++ b/llvm/test/CodeGen/X86/merge-consecutive-loads-512.ll
@@ -463,7 +463,7 @@ define <32 x i16> @merge_32i16_i16_12u4uuuuuuuuuuuuuuuuuuuuuuuuuuzz(i16* %ptr) n
 ; AVX512F-LABEL: merge_32i16_i16_12u4uuuuuuuuuuuuuuuuuuuuuuuuuuzz:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; AVX512F-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vmovaps %ymm0, %ymm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: merge_32i16_i16_12u4uuuuuuuuuuuuuuuuuuuuuuuuuuzz:
@@ -475,7 +475,7 @@ define <32 x i16> @merge_32i16_i16_12u4uuuuuuuuuuuuuuuuuuuuuuuuuuzz(i16* %ptr) n
 ; X32-AVX512F:       # %bb.0:
 ; X32-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-AVX512F-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; X32-AVX512F-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X32-AVX512F-NEXT:    vmovaps %ymm0, %ymm0
 ; X32-AVX512F-NEXT:    retl
   %ptr0 = getelementptr inbounds i16, i16* %ptr, i64 1
   %ptr1 = getelementptr inbounds i16, i16* %ptr, i64 2
@@ -518,7 +518,7 @@ define <32 x i16> @merge_32i16_i16_23uzuuuuuuuuuuzzzzuuuuuuuuuuuuuu(i16* %ptr) n
 ; AVX512F-LABEL: merge_32i16_i16_23uzuuuuuuuuuuzzzzuuuuuuuuuuuuuu:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX512F-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vmovaps %ymm0, %ymm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: merge_32i16_i16_23uzuuuuuuuuuuzzzzuuuuuuuuuuuuuu:
@@ -530,7 +530,7 @@ define <32 x i16> @merge_32i16_i16_23uzuuuuuuuuuuzzzzuuuuuuuuuuuuuu(i16* %ptr) n
 ; X32-AVX512F:       # %bb.0:
 ; X32-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-AVX512F-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X32-AVX512F-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X32-AVX512F-NEXT:    vmovaps %ymm0, %ymm0
 ; X32-AVX512F-NEXT:    retl
   %ptr0 = getelementptr inbounds i16, i16* %ptr, i64 2
   %ptr1 = getelementptr inbounds i16, i16* %ptr, i64 3
@@ -550,7 +550,7 @@ define <64 x i8> @merge_64i8_i8_12u4uuu8uuuuuuzzzzuuuuuuuuuuuuuuuuuuuuuuuuuuuuuu
 ; AVX512F-LABEL: merge_64i8_i8_12u4uuu8uuuuuuzzzzuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuz:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; AVX512F-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vmovaps %ymm0, %ymm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: merge_64i8_i8_12u4uuu8uuuuuuzzzzuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuz:
@@ -562,7 +562,7 @@ define <64 x i8> @merge_64i8_i8_12u4uuu8uuuuuuzzzzuuuuuuuuuuuuuuuuuuuuuuuuuuuuuu
 ; X32-AVX512F:       # %bb.0:
 ; X32-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-AVX512F-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; X32-AVX512F-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X32-AVX512F-NEXT:    vmovaps %ymm0, %ymm0
 ; X32-AVX512F-NEXT:    retl
   %ptr0 = getelementptr inbounds i8, i8* %ptr, i64 1
   %ptr1 = getelementptr inbounds i8, i8* %ptr, i64 2
@@ -588,7 +588,7 @@ define <64 x i8> @merge_64i8_i8_12u4uuuuuuuuuuzzzzuuuuuuuuuuuuuuuuuuuuuuuuuuuuuu
 ; AVX512F-LABEL: merge_64i8_i8_12u4uuuuuuuuuuzzzzuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuz:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX512F-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vmovaps %ymm0, %ymm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: merge_64i8_i8_12u4uuuuuuuuuuzzzzuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuz:
@@ -600,7 +600,7 @@ define <64 x i8> @merge_64i8_i8_12u4uuuuuuuuuuzzzzuuuuuuuuuuuuuuuuuuuuuuuuuuuuuu
 ; X32-AVX512F:       # %bb.0:
 ; X32-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-AVX512F-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X32-AVX512F-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X32-AVX512F-NEXT:    vmovaps %ymm0, %ymm0
 ; X32-AVX512F-NEXT:    retl
   %ptr0 = getelementptr inbounds i8, i8* %ptr, i64 1
   %ptr1 = getelementptr inbounds i8, i8* %ptr, i64 2
diff --git a/llvm/test/CodeGen/X86/merge-sp-update-lea.ll b/llvm/test/CodeGen/X86/merge-sp-update-lea.ll
index 70209a2ae..497960fab 100644
--- a/llvm/test/CodeGen/X86/merge-sp-update-lea.ll
+++ b/llvm/test/CodeGen/X86/merge-sp-update-lea.ll
@@ -29,4 +29,4 @@ cond.end.3:                                       ; preds = %entry, %cond.false
 ; Function Attrs: nounwind optsize
 declare noalias i8* @realloc(i8* nocapture, i32)
 
-attributes #0 = { nounwind optsize ssp "disable-tail-calls"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "target-features"="+lea-sp" }
+attributes #0 = { nounwind optsize ssp "disable-tail-calls"="false" "frame-pointer"="all" "target-features"="+lea-sp" }
diff --git a/llvm/test/CodeGen/X86/merge-vector-stores-scale-idx-crash.ll b/llvm/test/CodeGen/X86/merge-vector-stores-scale-idx-crash.ll
index 3bef9faee..331936d0d 100644
--- a/llvm/test/CodeGen/X86/merge-vector-stores-scale-idx-crash.ll
+++ b/llvm/test/CodeGen/X86/merge-vector-stores-scale-idx-crash.ll
@@ -45,5 +45,5 @@ if.end:                                           ; preds = %entry
 
 declare <4 x float> @_Z1bv() local_unnamed_addr 
 
-attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="128" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="128" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
diff --git a/llvm/test/CodeGen/X86/midpoint-int-vec-128.ll b/llvm/test/CodeGen/X86/midpoint-int-vec-128.ll
index 47c849264..7f0e19e58 100644
--- a/llvm/test/CodeGen/X86/midpoint-int-vec-128.ll
+++ b/llvm/test/CodeGen/X86/midpoint-int-vec-128.ll
@@ -933,18 +933,18 @@ define <2 x i64> @vec128_i64_signed_reg_reg(<2 x i64> %a1, <2 x i64> %a2) nounwi
 ; SSE41-NEXT:    pxor %xmm2, %xmm0
 ; SSE41-NEXT:    movdqa %xmm0, %xmm3
 ; SSE41-NEXT:    pcmpgtd %xmm5, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[0,0,2,2]
-; SSE41-NEXT:    movdqa %xmm0, %xmm4
-; SSE41-NEXT:    pcmpeqd %xmm5, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[1,1,3,3]
-; SSE41-NEXT:    pand %xmm7, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
+; SSE41-NEXT:    movdqa %xmm0, %xmm6
+; SSE41-NEXT:    pcmpeqd %xmm5, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[1,1,3,3]
+; SSE41-NEXT:    pand %xmm4, %xmm7
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm6, %xmm4
+; SSE41-NEXT:    por %xmm7, %xmm4
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [1,1]
 ; SSE41-NEXT:    por %xmm4, %xmm3
 ; SSE41-NEXT:    pcmpgtd %xmm0, %xmm5
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2]
-; SSE41-NEXT:    pand %xmm7, %xmm0
+; SSE41-NEXT:    pand %xmm6, %xmm0
 ; SSE41-NEXT:    por %xmm5, %xmm0
 ; SSE41-NEXT:    movdqa %xmm1, %xmm5
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm5
@@ -1161,18 +1161,18 @@ define <2 x i64> @vec128_i64_unsigned_reg_reg(<2 x i64> %a1, <2 x i64> %a2) noun
 ; SSE41-NEXT:    pxor %xmm2, %xmm0
 ; SSE41-NEXT:    movdqa %xmm0, %xmm3
 ; SSE41-NEXT:    pcmpgtd %xmm5, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[0,0,2,2]
-; SSE41-NEXT:    movdqa %xmm0, %xmm4
-; SSE41-NEXT:    pcmpeqd %xmm5, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[1,1,3,3]
-; SSE41-NEXT:    pand %xmm7, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
+; SSE41-NEXT:    movdqa %xmm0, %xmm6
+; SSE41-NEXT:    pcmpeqd %xmm5, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[1,1,3,3]
+; SSE41-NEXT:    pand %xmm4, %xmm7
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm6, %xmm4
+; SSE41-NEXT:    por %xmm7, %xmm4
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [1,1]
 ; SSE41-NEXT:    por %xmm4, %xmm3
 ; SSE41-NEXT:    pcmpgtd %xmm0, %xmm5
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2]
-; SSE41-NEXT:    pand %xmm7, %xmm0
+; SSE41-NEXT:    pand %xmm6, %xmm0
 ; SSE41-NEXT:    por %xmm5, %xmm0
 ; SSE41-NEXT:    movdqa %xmm1, %xmm5
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm5
@@ -1400,10 +1400,10 @@ define <2 x i64> @vec128_i64_signed_mem_reg(<2 x i64>* %a1_addr, <2 x i64> %a2)
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
 ; SSE41-NEXT:    movdqa %xmm5, %xmm6
 ; SSE41-NEXT:    pcmpeqd %xmm0, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE41-NEXT:    pand %xmm6, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[1,1,3,3]
+; SSE41-NEXT:    pand %xmm4, %xmm7
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE41-NEXT:    por %xmm4, %xmm2
+; SSE41-NEXT:    por %xmm7, %xmm2
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [1,1]
 ; SSE41-NEXT:    por %xmm2, %xmm4
 ; SSE41-NEXT:    pcmpgtd %xmm5, %xmm0
@@ -1626,21 +1626,22 @@ define <2 x i64> @vec128_i64_signed_reg_mem(<2 x i64> %a1, <2 x i64>* %a2_addr)
 ; SSE41-NEXT:    movdqa %xmm0, %xmm1
 ; SSE41-NEXT:    movdqa (%rdi), %xmm3
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm5 = [2147483648,2147483648]
-; SSE41-NEXT:    pxor %xmm5, %xmm0
+; SSE41-NEXT:    movdqa %xmm0, %xmm6
+; SSE41-NEXT:    pxor %xmm5, %xmm6
 ; SSE41-NEXT:    pxor %xmm3, %xmm5
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    movdqa %xmm6, %xmm2
 ; SSE41-NEXT:    pcmpgtd %xmm5, %xmm2
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
-; SSE41-NEXT:    movdqa %xmm0, %xmm6
-; SSE41-NEXT:    pcmpeqd %xmm5, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE41-NEXT:    pand %xmm6, %xmm4
+; SSE41-NEXT:    movdqa %xmm6, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm5, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[1,1,3,3]
+; SSE41-NEXT:    pand %xmm4, %xmm7
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE41-NEXT:    por %xmm4, %xmm2
+; SSE41-NEXT:    por %xmm7, %xmm2
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [1,1]
 ; SSE41-NEXT:    por %xmm2, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2]
+; SSE41-NEXT:    pcmpgtd %xmm6, %xmm5
+; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
 ; SSE41-NEXT:    pand %xmm6, %xmm0
 ; SSE41-NEXT:    por %xmm5, %xmm0
 ; SSE41-NEXT:    movdqa %xmm3, %xmm5
@@ -1867,10 +1868,10 @@ define <2 x i64> @vec128_i64_signed_mem_mem(<2 x i64>* %a1_addr, <2 x i64>* %a2_
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[0,0,2,2]
 ; SSE41-NEXT:    movdqa %xmm0, %xmm6
 ; SSE41-NEXT:    pcmpeqd %xmm5, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE41-NEXT:    pand %xmm6, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[1,1,3,3]
+; SSE41-NEXT:    pand %xmm4, %xmm7
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE41-NEXT:    por %xmm4, %xmm1
+; SSE41-NEXT:    por %xmm7, %xmm1
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [1,1]
 ; SSE41-NEXT:    por %xmm1, %xmm4
 ; SSE41-NEXT:    pcmpgtd %xmm0, %xmm5
diff --git a/llvm/test/CodeGen/X86/midpoint-int-vec-256.ll b/llvm/test/CodeGen/X86/midpoint-int-vec-256.ll
index 0d28d6145..c4bd5f8cb 100644
--- a/llvm/test/CodeGen/X86/midpoint-int-vec-256.ll
+++ b/llvm/test/CodeGen/X86/midpoint-int-vec-256.ll
@@ -490,7 +490,7 @@ define <4 x i64> @vec256_i64_signed_reg_reg(<4 x i64> %a1, <4 x i64> %a2) nounwi
 ; AVX1-FALLBACK-NEXT:    vpsrlq $1, %xmm2, %xmm2
 ; AVX1-FALLBACK-NEXT:    vpsrlq $1, %xmm1, %xmm1
 ; AVX1-FALLBACK-NEXT:    vmovdqa {{.*#+}} xmm8 = [1,1]
-; AVX1-FALLBACK-NEXT:    vpor %xmm8, %xmm5, %xmm5
+; AVX1-FALLBACK-NEXT:    vpor %xmm5, %xmm8, %xmm5
 ; AVX1-FALLBACK-NEXT:    vpsrlq $32, %xmm5, %xmm7
 ; AVX1-FALLBACK-NEXT:    vpmuludq %xmm7, %xmm1, %xmm7
 ; AVX1-FALLBACK-NEXT:    vpsrlq $32, %xmm1, %xmm6
@@ -498,7 +498,7 @@ define <4 x i64> @vec256_i64_signed_reg_reg(<4 x i64> %a1, <4 x i64> %a2) nounwi
 ; AVX1-FALLBACK-NEXT:    vpaddq %xmm6, %xmm7, %xmm6
 ; AVX1-FALLBACK-NEXT:    vpsllq $32, %xmm6, %xmm6
 ; AVX1-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm1, %xmm1
-; AVX1-FALLBACK-NEXT:    vpor %xmm8, %xmm4, %xmm4
+; AVX1-FALLBACK-NEXT:    vpor %xmm4, %xmm8, %xmm4
 ; AVX1-FALLBACK-NEXT:    vpsrlq $32, %xmm4, %xmm5
 ; AVX1-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm2, %xmm5
 ; AVX1-FALLBACK-NEXT:    vpsrlq $32, %xmm2, %xmm7
@@ -551,7 +551,7 @@ define <4 x i64> @vec256_i64_signed_reg_reg(<4 x i64> %a1, <4 x i64> %a2) nounwi
 ; XOP-FALLBACK-NEXT:    vpsrlq $1, %xmm2, %xmm2
 ; XOP-FALLBACK-NEXT:    vpsrlq $1, %xmm1, %xmm1
 ; XOP-FALLBACK-NEXT:    vmovdqa {{.*#+}} xmm8 = [1,1]
-; XOP-FALLBACK-NEXT:    vpor %xmm8, %xmm5, %xmm5
+; XOP-FALLBACK-NEXT:    vpor %xmm5, %xmm8, %xmm5
 ; XOP-FALLBACK-NEXT:    vpsrlq $32, %xmm5, %xmm7
 ; XOP-FALLBACK-NEXT:    vpmuludq %xmm7, %xmm1, %xmm7
 ; XOP-FALLBACK-NEXT:    vpsrlq $32, %xmm1, %xmm6
@@ -559,7 +559,7 @@ define <4 x i64> @vec256_i64_signed_reg_reg(<4 x i64> %a1, <4 x i64> %a2) nounwi
 ; XOP-FALLBACK-NEXT:    vpaddq %xmm6, %xmm7, %xmm6
 ; XOP-FALLBACK-NEXT:    vpsllq $32, %xmm6, %xmm6
 ; XOP-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm1, %xmm1
-; XOP-FALLBACK-NEXT:    vpor %xmm8, %xmm4, %xmm4
+; XOP-FALLBACK-NEXT:    vpor %xmm4, %xmm8, %xmm4
 ; XOP-FALLBACK-NEXT:    vpsrlq $32, %xmm4, %xmm5
 ; XOP-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm2, %xmm5
 ; XOP-FALLBACK-NEXT:    vpsrlq $32, %xmm2, %xmm7
@@ -591,7 +591,7 @@ define <4 x i64> @vec256_i64_signed_reg_reg(<4 x i64> %a1, <4 x i64> %a2) nounwi
 ; XOPAVX1-NEXT:    vpsrlq $1, %xmm2, %xmm2
 ; XOPAVX1-NEXT:    vpsrlq $1, %xmm1, %xmm1
 ; XOPAVX1-NEXT:    vmovdqa {{.*#+}} xmm8 = [1,1]
-; XOPAVX1-NEXT:    vpor %xmm8, %xmm5, %xmm5
+; XOPAVX1-NEXT:    vpor %xmm5, %xmm8, %xmm5
 ; XOPAVX1-NEXT:    vpsrlq $32, %xmm5, %xmm7
 ; XOPAVX1-NEXT:    vpmuludq %xmm7, %xmm1, %xmm7
 ; XOPAVX1-NEXT:    vpsrlq $32, %xmm1, %xmm6
@@ -599,7 +599,7 @@ define <4 x i64> @vec256_i64_signed_reg_reg(<4 x i64> %a1, <4 x i64> %a2) nounwi
 ; XOPAVX1-NEXT:    vpaddq %xmm6, %xmm7, %xmm6
 ; XOPAVX1-NEXT:    vpsllq $32, %xmm6, %xmm6
 ; XOPAVX1-NEXT:    vpmuludq %xmm5, %xmm1, %xmm1
-; XOPAVX1-NEXT:    vpor %xmm8, %xmm4, %xmm4
+; XOPAVX1-NEXT:    vpor %xmm4, %xmm8, %xmm4
 ; XOPAVX1-NEXT:    vpsrlq $32, %xmm4, %xmm5
 ; XOPAVX1-NEXT:    vpmuludq %xmm5, %xmm2, %xmm5
 ; XOPAVX1-NEXT:    vpsrlq $32, %xmm2, %xmm7
@@ -778,7 +778,7 @@ define <4 x i64> @vec256_i64_unsigned_reg_reg(<4 x i64> %a1, <4 x i64> %a2) noun
 ; XOP-FALLBACK-NEXT:    vpsrlq $1, %xmm2, %xmm2
 ; XOP-FALLBACK-NEXT:    vpsrlq $1, %xmm1, %xmm1
 ; XOP-FALLBACK-NEXT:    vmovdqa {{.*#+}} xmm8 = [1,1]
-; XOP-FALLBACK-NEXT:    vpor %xmm8, %xmm5, %xmm5
+; XOP-FALLBACK-NEXT:    vpor %xmm5, %xmm8, %xmm5
 ; XOP-FALLBACK-NEXT:    vpsrlq $32, %xmm5, %xmm7
 ; XOP-FALLBACK-NEXT:    vpmuludq %xmm7, %xmm1, %xmm7
 ; XOP-FALLBACK-NEXT:    vpsrlq $32, %xmm1, %xmm6
@@ -786,7 +786,7 @@ define <4 x i64> @vec256_i64_unsigned_reg_reg(<4 x i64> %a1, <4 x i64> %a2) noun
 ; XOP-FALLBACK-NEXT:    vpaddq %xmm6, %xmm7, %xmm6
 ; XOP-FALLBACK-NEXT:    vpsllq $32, %xmm6, %xmm6
 ; XOP-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm1, %xmm1
-; XOP-FALLBACK-NEXT:    vpor %xmm8, %xmm4, %xmm4
+; XOP-FALLBACK-NEXT:    vpor %xmm4, %xmm8, %xmm4
 ; XOP-FALLBACK-NEXT:    vpsrlq $32, %xmm4, %xmm5
 ; XOP-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm2, %xmm5
 ; XOP-FALLBACK-NEXT:    vpsrlq $32, %xmm2, %xmm7
@@ -818,7 +818,7 @@ define <4 x i64> @vec256_i64_unsigned_reg_reg(<4 x i64> %a1, <4 x i64> %a2) noun
 ; XOPAVX1-NEXT:    vpsrlq $1, %xmm2, %xmm2
 ; XOPAVX1-NEXT:    vpsrlq $1, %xmm1, %xmm1
 ; XOPAVX1-NEXT:    vmovdqa {{.*#+}} xmm8 = [1,1]
-; XOPAVX1-NEXT:    vpor %xmm8, %xmm5, %xmm5
+; XOPAVX1-NEXT:    vpor %xmm5, %xmm8, %xmm5
 ; XOPAVX1-NEXT:    vpsrlq $32, %xmm5, %xmm7
 ; XOPAVX1-NEXT:    vpmuludq %xmm7, %xmm1, %xmm7
 ; XOPAVX1-NEXT:    vpsrlq $32, %xmm1, %xmm6
@@ -826,7 +826,7 @@ define <4 x i64> @vec256_i64_unsigned_reg_reg(<4 x i64> %a1, <4 x i64> %a2) noun
 ; XOPAVX1-NEXT:    vpaddq %xmm6, %xmm7, %xmm6
 ; XOPAVX1-NEXT:    vpsllq $32, %xmm6, %xmm6
 ; XOPAVX1-NEXT:    vpmuludq %xmm5, %xmm1, %xmm1
-; XOPAVX1-NEXT:    vpor %xmm8, %xmm4, %xmm4
+; XOPAVX1-NEXT:    vpor %xmm4, %xmm8, %xmm4
 ; XOPAVX1-NEXT:    vpsrlq $32, %xmm4, %xmm5
 ; XOPAVX1-NEXT:    vpmuludq %xmm5, %xmm2, %xmm5
 ; XOPAVX1-NEXT:    vpsrlq $32, %xmm2, %xmm7
@@ -939,7 +939,7 @@ define <4 x i64> @vec256_i64_signed_mem_reg(<4 x i64>* %a1_addr, <4 x i64> %a2)
 ; AVX1-FALLBACK-NEXT:    vpsrlq $1, %xmm2, %xmm2
 ; AVX1-FALLBACK-NEXT:    vpsrlq $1, %xmm0, %xmm0
 ; AVX1-FALLBACK-NEXT:    vmovdqa {{.*#+}} xmm8 = [1,1]
-; AVX1-FALLBACK-NEXT:    vpor %xmm8, %xmm5, %xmm5
+; AVX1-FALLBACK-NEXT:    vpor %xmm5, %xmm8, %xmm5
 ; AVX1-FALLBACK-NEXT:    vpsrlq $32, %xmm5, %xmm7
 ; AVX1-FALLBACK-NEXT:    vpmuludq %xmm7, %xmm0, %xmm7
 ; AVX1-FALLBACK-NEXT:    vpsrlq $32, %xmm0, %xmm6
@@ -947,7 +947,7 @@ define <4 x i64> @vec256_i64_signed_mem_reg(<4 x i64>* %a1_addr, <4 x i64> %a2)
 ; AVX1-FALLBACK-NEXT:    vpaddq %xmm6, %xmm7, %xmm6
 ; AVX1-FALLBACK-NEXT:    vpsllq $32, %xmm6, %xmm6
 ; AVX1-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm0, %xmm0
-; AVX1-FALLBACK-NEXT:    vpor %xmm8, %xmm4, %xmm4
+; AVX1-FALLBACK-NEXT:    vpor %xmm4, %xmm8, %xmm4
 ; AVX1-FALLBACK-NEXT:    vpsrlq $32, %xmm4, %xmm5
 ; AVX1-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm2, %xmm5
 ; AVX1-FALLBACK-NEXT:    vpsrlq $32, %xmm2, %xmm7
@@ -1002,7 +1002,7 @@ define <4 x i64> @vec256_i64_signed_mem_reg(<4 x i64>* %a1_addr, <4 x i64> %a2)
 ; XOP-FALLBACK-NEXT:    vpsrlq $1, %xmm2, %xmm2
 ; XOP-FALLBACK-NEXT:    vpsrlq $1, %xmm0, %xmm0
 ; XOP-FALLBACK-NEXT:    vmovdqa {{.*#+}} xmm8 = [1,1]
-; XOP-FALLBACK-NEXT:    vpor %xmm8, %xmm5, %xmm5
+; XOP-FALLBACK-NEXT:    vpor %xmm5, %xmm8, %xmm5
 ; XOP-FALLBACK-NEXT:    vpsrlq $32, %xmm5, %xmm7
 ; XOP-FALLBACK-NEXT:    vpmuludq %xmm7, %xmm0, %xmm7
 ; XOP-FALLBACK-NEXT:    vpsrlq $32, %xmm0, %xmm6
@@ -1010,7 +1010,7 @@ define <4 x i64> @vec256_i64_signed_mem_reg(<4 x i64>* %a1_addr, <4 x i64> %a2)
 ; XOP-FALLBACK-NEXT:    vpaddq %xmm6, %xmm7, %xmm6
 ; XOP-FALLBACK-NEXT:    vpsllq $32, %xmm6, %xmm6
 ; XOP-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm0, %xmm0
-; XOP-FALLBACK-NEXT:    vpor %xmm8, %xmm4, %xmm4
+; XOP-FALLBACK-NEXT:    vpor %xmm4, %xmm8, %xmm4
 ; XOP-FALLBACK-NEXT:    vpsrlq $32, %xmm4, %xmm5
 ; XOP-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm2, %xmm5
 ; XOP-FALLBACK-NEXT:    vpsrlq $32, %xmm2, %xmm7
@@ -1043,7 +1043,7 @@ define <4 x i64> @vec256_i64_signed_mem_reg(<4 x i64>* %a1_addr, <4 x i64> %a2)
 ; XOPAVX1-NEXT:    vpsrlq $1, %xmm2, %xmm2
 ; XOPAVX1-NEXT:    vpsrlq $1, %xmm0, %xmm0
 ; XOPAVX1-NEXT:    vmovdqa {{.*#+}} xmm8 = [1,1]
-; XOPAVX1-NEXT:    vpor %xmm8, %xmm5, %xmm5
+; XOPAVX1-NEXT:    vpor %xmm5, %xmm8, %xmm5
 ; XOPAVX1-NEXT:    vpsrlq $32, %xmm5, %xmm7
 ; XOPAVX1-NEXT:    vpmuludq %xmm7, %xmm0, %xmm7
 ; XOPAVX1-NEXT:    vpsrlq $32, %xmm0, %xmm6
@@ -1051,7 +1051,7 @@ define <4 x i64> @vec256_i64_signed_mem_reg(<4 x i64>* %a1_addr, <4 x i64> %a2)
 ; XOPAVX1-NEXT:    vpaddq %xmm6, %xmm7, %xmm6
 ; XOPAVX1-NEXT:    vpsllq $32, %xmm6, %xmm6
 ; XOPAVX1-NEXT:    vpmuludq %xmm5, %xmm0, %xmm0
-; XOPAVX1-NEXT:    vpor %xmm8, %xmm4, %xmm4
+; XOPAVX1-NEXT:    vpor %xmm4, %xmm8, %xmm4
 ; XOPAVX1-NEXT:    vpsrlq $32, %xmm4, %xmm5
 ; XOPAVX1-NEXT:    vpmuludq %xmm5, %xmm2, %xmm5
 ; XOPAVX1-NEXT:    vpsrlq $32, %xmm2, %xmm7
@@ -1164,7 +1164,7 @@ define <4 x i64> @vec256_i64_signed_reg_mem(<4 x i64> %a1, <4 x i64>* %a2_addr)
 ; AVX1-FALLBACK-NEXT:    vpsrlq $1, %xmm2, %xmm2
 ; AVX1-FALLBACK-NEXT:    vpsrlq $1, %xmm1, %xmm1
 ; AVX1-FALLBACK-NEXT:    vmovdqa {{.*#+}} xmm8 = [1,1]
-; AVX1-FALLBACK-NEXT:    vpor %xmm8, %xmm5, %xmm5
+; AVX1-FALLBACK-NEXT:    vpor %xmm5, %xmm8, %xmm5
 ; AVX1-FALLBACK-NEXT:    vpsrlq $32, %xmm5, %xmm7
 ; AVX1-FALLBACK-NEXT:    vpmuludq %xmm7, %xmm1, %xmm7
 ; AVX1-FALLBACK-NEXT:    vpsrlq $32, %xmm1, %xmm6
@@ -1172,7 +1172,7 @@ define <4 x i64> @vec256_i64_signed_reg_mem(<4 x i64> %a1, <4 x i64>* %a2_addr)
 ; AVX1-FALLBACK-NEXT:    vpaddq %xmm6, %xmm7, %xmm6
 ; AVX1-FALLBACK-NEXT:    vpsllq $32, %xmm6, %xmm6
 ; AVX1-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm1, %xmm1
-; AVX1-FALLBACK-NEXT:    vpor %xmm8, %xmm4, %xmm4
+; AVX1-FALLBACK-NEXT:    vpor %xmm4, %xmm8, %xmm4
 ; AVX1-FALLBACK-NEXT:    vpsrlq $32, %xmm4, %xmm5
 ; AVX1-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm2, %xmm5
 ; AVX1-FALLBACK-NEXT:    vpsrlq $32, %xmm2, %xmm7
@@ -1227,7 +1227,7 @@ define <4 x i64> @vec256_i64_signed_reg_mem(<4 x i64> %a1, <4 x i64>* %a2_addr)
 ; XOP-FALLBACK-NEXT:    vpsrlq $1, %xmm2, %xmm2
 ; XOP-FALLBACK-NEXT:    vpsrlq $1, %xmm1, %xmm1
 ; XOP-FALLBACK-NEXT:    vmovdqa {{.*#+}} xmm8 = [1,1]
-; XOP-FALLBACK-NEXT:    vpor %xmm8, %xmm5, %xmm5
+; XOP-FALLBACK-NEXT:    vpor %xmm5, %xmm8, %xmm5
 ; XOP-FALLBACK-NEXT:    vpsrlq $32, %xmm5, %xmm7
 ; XOP-FALLBACK-NEXT:    vpmuludq %xmm7, %xmm1, %xmm7
 ; XOP-FALLBACK-NEXT:    vpsrlq $32, %xmm1, %xmm6
@@ -1235,7 +1235,7 @@ define <4 x i64> @vec256_i64_signed_reg_mem(<4 x i64> %a1, <4 x i64>* %a2_addr)
 ; XOP-FALLBACK-NEXT:    vpaddq %xmm6, %xmm7, %xmm6
 ; XOP-FALLBACK-NEXT:    vpsllq $32, %xmm6, %xmm6
 ; XOP-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm1, %xmm1
-; XOP-FALLBACK-NEXT:    vpor %xmm8, %xmm4, %xmm4
+; XOP-FALLBACK-NEXT:    vpor %xmm4, %xmm8, %xmm4
 ; XOP-FALLBACK-NEXT:    vpsrlq $32, %xmm4, %xmm5
 ; XOP-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm2, %xmm5
 ; XOP-FALLBACK-NEXT:    vpsrlq $32, %xmm2, %xmm7
@@ -1268,7 +1268,7 @@ define <4 x i64> @vec256_i64_signed_reg_mem(<4 x i64> %a1, <4 x i64>* %a2_addr)
 ; XOPAVX1-NEXT:    vpsrlq $1, %xmm2, %xmm2
 ; XOPAVX1-NEXT:    vpsrlq $1, %xmm1, %xmm1
 ; XOPAVX1-NEXT:    vmovdqa {{.*#+}} xmm8 = [1,1]
-; XOPAVX1-NEXT:    vpor %xmm8, %xmm5, %xmm5
+; XOPAVX1-NEXT:    vpor %xmm5, %xmm8, %xmm5
 ; XOPAVX1-NEXT:    vpsrlq $32, %xmm5, %xmm7
 ; XOPAVX1-NEXT:    vpmuludq %xmm7, %xmm1, %xmm7
 ; XOPAVX1-NEXT:    vpsrlq $32, %xmm1, %xmm6
@@ -1276,7 +1276,7 @@ define <4 x i64> @vec256_i64_signed_reg_mem(<4 x i64> %a1, <4 x i64>* %a2_addr)
 ; XOPAVX1-NEXT:    vpaddq %xmm6, %xmm7, %xmm6
 ; XOPAVX1-NEXT:    vpsllq $32, %xmm6, %xmm6
 ; XOPAVX1-NEXT:    vpmuludq %xmm5, %xmm1, %xmm1
-; XOPAVX1-NEXT:    vpor %xmm8, %xmm4, %xmm4
+; XOPAVX1-NEXT:    vpor %xmm4, %xmm8, %xmm4
 ; XOPAVX1-NEXT:    vpsrlq $32, %xmm4, %xmm5
 ; XOPAVX1-NEXT:    vpmuludq %xmm5, %xmm2, %xmm5
 ; XOPAVX1-NEXT:    vpsrlq $32, %xmm2, %xmm7
@@ -1390,7 +1390,7 @@ define <4 x i64> @vec256_i64_signed_mem_mem(<4 x i64>* %a1_addr, <4 x i64>* %a2_
 ; AVX1-FALLBACK-NEXT:    vpsrlq $1, %xmm2, %xmm2
 ; AVX1-FALLBACK-NEXT:    vpsrlq $1, %xmm1, %xmm1
 ; AVX1-FALLBACK-NEXT:    vmovdqa {{.*#+}} xmm8 = [1,1]
-; AVX1-FALLBACK-NEXT:    vpor %xmm8, %xmm5, %xmm5
+; AVX1-FALLBACK-NEXT:    vpor %xmm5, %xmm8, %xmm5
 ; AVX1-FALLBACK-NEXT:    vpsrlq $32, %xmm5, %xmm7
 ; AVX1-FALLBACK-NEXT:    vpmuludq %xmm7, %xmm1, %xmm7
 ; AVX1-FALLBACK-NEXT:    vpsrlq $32, %xmm1, %xmm6
@@ -1398,7 +1398,7 @@ define <4 x i64> @vec256_i64_signed_mem_mem(<4 x i64>* %a1_addr, <4 x i64>* %a2_
 ; AVX1-FALLBACK-NEXT:    vpaddq %xmm6, %xmm7, %xmm6
 ; AVX1-FALLBACK-NEXT:    vpsllq $32, %xmm6, %xmm6
 ; AVX1-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm1, %xmm1
-; AVX1-FALLBACK-NEXT:    vpor %xmm8, %xmm4, %xmm4
+; AVX1-FALLBACK-NEXT:    vpor %xmm4, %xmm8, %xmm4
 ; AVX1-FALLBACK-NEXT:    vpsrlq $32, %xmm4, %xmm5
 ; AVX1-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm2, %xmm5
 ; AVX1-FALLBACK-NEXT:    vpsrlq $32, %xmm2, %xmm7
@@ -1455,7 +1455,7 @@ define <4 x i64> @vec256_i64_signed_mem_mem(<4 x i64>* %a1_addr, <4 x i64>* %a2_
 ; XOP-FALLBACK-NEXT:    vpsrlq $1, %xmm2, %xmm2
 ; XOP-FALLBACK-NEXT:    vpsrlq $1, %xmm1, %xmm1
 ; XOP-FALLBACK-NEXT:    vmovdqa {{.*#+}} xmm8 = [1,1]
-; XOP-FALLBACK-NEXT:    vpor %xmm8, %xmm5, %xmm5
+; XOP-FALLBACK-NEXT:    vpor %xmm5, %xmm8, %xmm5
 ; XOP-FALLBACK-NEXT:    vpsrlq $32, %xmm5, %xmm7
 ; XOP-FALLBACK-NEXT:    vpmuludq %xmm7, %xmm1, %xmm7
 ; XOP-FALLBACK-NEXT:    vpsrlq $32, %xmm1, %xmm6
@@ -1463,7 +1463,7 @@ define <4 x i64> @vec256_i64_signed_mem_mem(<4 x i64>* %a1_addr, <4 x i64>* %a2_
 ; XOP-FALLBACK-NEXT:    vpaddq %xmm6, %xmm7, %xmm6
 ; XOP-FALLBACK-NEXT:    vpsllq $32, %xmm6, %xmm6
 ; XOP-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm1, %xmm1
-; XOP-FALLBACK-NEXT:    vpor %xmm8, %xmm4, %xmm4
+; XOP-FALLBACK-NEXT:    vpor %xmm4, %xmm8, %xmm4
 ; XOP-FALLBACK-NEXT:    vpsrlq $32, %xmm4, %xmm5
 ; XOP-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm2, %xmm5
 ; XOP-FALLBACK-NEXT:    vpsrlq $32, %xmm2, %xmm7
@@ -1497,7 +1497,7 @@ define <4 x i64> @vec256_i64_signed_mem_mem(<4 x i64>* %a1_addr, <4 x i64>* %a2_
 ; XOPAVX1-NEXT:    vpsrlq $1, %xmm2, %xmm2
 ; XOPAVX1-NEXT:    vpsrlq $1, %xmm1, %xmm1
 ; XOPAVX1-NEXT:    vmovdqa {{.*#+}} xmm8 = [1,1]
-; XOPAVX1-NEXT:    vpor %xmm8, %xmm5, %xmm5
+; XOPAVX1-NEXT:    vpor %xmm5, %xmm8, %xmm5
 ; XOPAVX1-NEXT:    vpsrlq $32, %xmm5, %xmm7
 ; XOPAVX1-NEXT:    vpmuludq %xmm7, %xmm1, %xmm7
 ; XOPAVX1-NEXT:    vpsrlq $32, %xmm1, %xmm6
@@ -1505,7 +1505,7 @@ define <4 x i64> @vec256_i64_signed_mem_mem(<4 x i64>* %a1_addr, <4 x i64>* %a2_
 ; XOPAVX1-NEXT:    vpaddq %xmm6, %xmm7, %xmm6
 ; XOPAVX1-NEXT:    vpsllq $32, %xmm6, %xmm6
 ; XOPAVX1-NEXT:    vpmuludq %xmm5, %xmm1, %xmm1
-; XOPAVX1-NEXT:    vpor %xmm8, %xmm4, %xmm4
+; XOPAVX1-NEXT:    vpor %xmm4, %xmm8, %xmm4
 ; XOPAVX1-NEXT:    vpsrlq $32, %xmm4, %xmm5
 ; XOPAVX1-NEXT:    vpmuludq %xmm5, %xmm2, %xmm5
 ; XOPAVX1-NEXT:    vpsrlq $32, %xmm2, %xmm7
@@ -1760,10 +1760,10 @@ define <16 x i16> @vec256_i16_unsigned_reg_reg(<16 x i16> %a1, <16 x i16> %a2) n
 ; AVX1-FALLBACK-NEXT:    vpminuw %xmm2, %xmm3, %xmm4
 ; AVX1-FALLBACK-NEXT:    vpcmpeqw %xmm4, %xmm3, %xmm5
 ; AVX1-FALLBACK-NEXT:    vpcmpeqd %xmm8, %xmm8, %xmm8
-; AVX1-FALLBACK-NEXT:    vpxor %xmm8, %xmm5, %xmm5
+; AVX1-FALLBACK-NEXT:    vpxor %xmm5, %xmm8, %xmm5
 ; AVX1-FALLBACK-NEXT:    vpminuw %xmm1, %xmm0, %xmm7
 ; AVX1-FALLBACK-NEXT:    vpcmpeqw %xmm7, %xmm0, %xmm6
-; AVX1-FALLBACK-NEXT:    vpxor %xmm8, %xmm6, %xmm6
+; AVX1-FALLBACK-NEXT:    vpxor %xmm6, %xmm8, %xmm6
 ; AVX1-FALLBACK-NEXT:    vpmaxuw %xmm2, %xmm3, %xmm2
 ; AVX1-FALLBACK-NEXT:    vpmaxuw %xmm1, %xmm0, %xmm1
 ; AVX1-FALLBACK-NEXT:    vpsubw %xmm7, %xmm1, %xmm1
diff --git a/llvm/test/CodeGen/X86/midpoint-int-vec-512.ll b/llvm/test/CodeGen/X86/midpoint-int-vec-512.ll
index 0403f46b0..c3743ca82 100644
--- a/llvm/test/CodeGen/X86/midpoint-int-vec-512.ll
+++ b/llvm/test/CodeGen/X86/midpoint-int-vec-512.ll
@@ -315,44 +315,50 @@ define <8 x i64> @vec512_i64_signed_mem_mem(<8 x i64>* %a1_addr, <8 x i64>* %a2_
 define <32 x i16> @vec512_i16_signed_reg_reg(<32 x i16> %a1, <32 x i16> %a2) nounwind {
 ; AVX512F-LABEL: vec512_i16_signed_reg_reg:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpcmpgtw %ymm2, %ymm0, %ymm4
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
+; AVX512F-NEXT:    vpcmpgtw %ymm2, %ymm3, %ymm4
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm5 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
 ; AVX512F-NEXT:    vpor %ymm5, %ymm4, %ymm4
-; AVX512F-NEXT:    vpcmpgtw %ymm3, %ymm1, %ymm6
+; AVX512F-NEXT:    vpcmpgtw %ymm1, %ymm0, %ymm6
 ; AVX512F-NEXT:    vpor %ymm5, %ymm6, %ymm5
-; AVX512F-NEXT:    vpminsw %ymm2, %ymm0, %ymm6
-; AVX512F-NEXT:    vpminsw %ymm3, %ymm1, %ymm7
-; AVX512F-NEXT:    vpmaxsw %ymm2, %ymm0, %ymm2
+; AVX512F-NEXT:    vpminsw %ymm2, %ymm3, %ymm6
+; AVX512F-NEXT:    vpminsw %ymm1, %ymm0, %ymm7
+; AVX512F-NEXT:    vpmaxsw %ymm2, %ymm3, %ymm2
 ; AVX512F-NEXT:    vpsubw %ymm6, %ymm2, %ymm2
-; AVX512F-NEXT:    vpmaxsw %ymm3, %ymm1, %ymm3
-; AVX512F-NEXT:    vpsubw %ymm7, %ymm3, %ymm3
+; AVX512F-NEXT:    vpmaxsw %ymm1, %ymm0, %ymm1
+; AVX512F-NEXT:    vpsubw %ymm7, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpsrlw $1, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpmullw %ymm4, %ymm2, %ymm2
-; AVX512F-NEXT:    vpsrlw $1, %ymm3, %ymm3
-; AVX512F-NEXT:    vpmullw %ymm5, %ymm3, %ymm3
-; AVX512F-NEXT:    vpaddw %ymm0, %ymm2, %ymm0
-; AVX512F-NEXT:    vpaddw %ymm1, %ymm3, %ymm1
+; AVX512F-NEXT:    vpsrlw $1, %ymm1, %ymm1
+; AVX512F-NEXT:    vpmullw %ymm5, %ymm1, %ymm1
+; AVX512F-NEXT:    vpaddw %ymm3, %ymm2, %ymm2
+; AVX512F-NEXT:    vpaddw %ymm0, %ymm1, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-FALLBACK-LABEL: vec512_i16_signed_reg_reg:
 ; AVX512VL-FALLBACK:       # %bb.0:
-; AVX512VL-FALLBACK-NEXT:    vpcmpgtw %ymm2, %ymm0, %ymm4
+; AVX512VL-FALLBACK-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
+; AVX512VL-FALLBACK-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
+; AVX512VL-FALLBACK-NEXT:    vpcmpgtw %ymm2, %ymm3, %ymm4
 ; AVX512VL-FALLBACK-NEXT:    vmovdqa {{.*#+}} ymm5 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
 ; AVX512VL-FALLBACK-NEXT:    vpor %ymm5, %ymm4, %ymm4
-; AVX512VL-FALLBACK-NEXT:    vpcmpgtw %ymm3, %ymm1, %ymm6
+; AVX512VL-FALLBACK-NEXT:    vpcmpgtw %ymm1, %ymm0, %ymm6
 ; AVX512VL-FALLBACK-NEXT:    vpor %ymm5, %ymm6, %ymm5
-; AVX512VL-FALLBACK-NEXT:    vpminsw %ymm2, %ymm0, %ymm6
-; AVX512VL-FALLBACK-NEXT:    vpminsw %ymm3, %ymm1, %ymm7
-; AVX512VL-FALLBACK-NEXT:    vpmaxsw %ymm2, %ymm0, %ymm2
+; AVX512VL-FALLBACK-NEXT:    vpminsw %ymm2, %ymm3, %ymm6
+; AVX512VL-FALLBACK-NEXT:    vpminsw %ymm1, %ymm0, %ymm7
+; AVX512VL-FALLBACK-NEXT:    vpmaxsw %ymm2, %ymm3, %ymm2
 ; AVX512VL-FALLBACK-NEXT:    vpsubw %ymm6, %ymm2, %ymm2
-; AVX512VL-FALLBACK-NEXT:    vpmaxsw %ymm3, %ymm1, %ymm3
-; AVX512VL-FALLBACK-NEXT:    vpsubw %ymm7, %ymm3, %ymm3
+; AVX512VL-FALLBACK-NEXT:    vpmaxsw %ymm1, %ymm0, %ymm1
+; AVX512VL-FALLBACK-NEXT:    vpsubw %ymm7, %ymm1, %ymm1
 ; AVX512VL-FALLBACK-NEXT:    vpsrlw $1, %ymm2, %ymm2
 ; AVX512VL-FALLBACK-NEXT:    vpmullw %ymm4, %ymm2, %ymm2
-; AVX512VL-FALLBACK-NEXT:    vpsrlw $1, %ymm3, %ymm3
-; AVX512VL-FALLBACK-NEXT:    vpmullw %ymm5, %ymm3, %ymm3
-; AVX512VL-FALLBACK-NEXT:    vpaddw %ymm0, %ymm2, %ymm0
-; AVX512VL-FALLBACK-NEXT:    vpaddw %ymm1, %ymm3, %ymm1
+; AVX512VL-FALLBACK-NEXT:    vpsrlw $1, %ymm1, %ymm1
+; AVX512VL-FALLBACK-NEXT:    vpmullw %ymm5, %ymm1, %ymm1
+; AVX512VL-FALLBACK-NEXT:    vpaddw %ymm3, %ymm2, %ymm2
+; AVX512VL-FALLBACK-NEXT:    vpaddw %ymm0, %ymm1, %ymm0
+; AVX512VL-FALLBACK-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512VL-FALLBACK-NEXT:    retq
 ;
 ; AVX512BW-LABEL: vec512_i16_signed_reg_reg:
@@ -382,48 +388,54 @@ define <32 x i16> @vec512_i16_signed_reg_reg(<32 x i16> %a1, <32 x i16> %a2) nou
 define <32 x i16> @vec512_i16_unsigned_reg_reg(<32 x i16> %a1, <32 x i16> %a2) nounwind {
 ; AVX512F-LABEL: vec512_i16_unsigned_reg_reg:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpminuw %ymm2, %ymm0, %ymm4
-; AVX512F-NEXT:    vpcmpeqw %ymm4, %ymm0, %ymm5
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
+; AVX512F-NEXT:    vpminuw %ymm2, %ymm3, %ymm4
+; AVX512F-NEXT:    vpcmpeqw %ymm4, %ymm3, %ymm5
 ; AVX512F-NEXT:    vpternlogq $15, %zmm5, %zmm5, %zmm5
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm6 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
 ; AVX512F-NEXT:    vpor %ymm6, %ymm5, %ymm5
-; AVX512F-NEXT:    vpminuw %ymm3, %ymm1, %ymm7
-; AVX512F-NEXT:    vpcmpeqw %ymm7, %ymm1, %ymm8
+; AVX512F-NEXT:    vpminuw %ymm1, %ymm0, %ymm7
+; AVX512F-NEXT:    vpcmpeqw %ymm7, %ymm0, %ymm8
 ; AVX512F-NEXT:    vpternlogq $15, %zmm8, %zmm8, %zmm8
 ; AVX512F-NEXT:    vpor %ymm6, %ymm8, %ymm6
-; AVX512F-NEXT:    vpmaxuw %ymm2, %ymm0, %ymm2
-; AVX512F-NEXT:    vpmaxuw %ymm3, %ymm1, %ymm3
-; AVX512F-NEXT:    vpsubw %ymm7, %ymm3, %ymm3
+; AVX512F-NEXT:    vpmaxuw %ymm2, %ymm3, %ymm2
+; AVX512F-NEXT:    vpmaxuw %ymm1, %ymm0, %ymm1
+; AVX512F-NEXT:    vpsubw %ymm7, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpsubw %ymm4, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpsrlw $1, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpmullw %ymm5, %ymm2, %ymm2
-; AVX512F-NEXT:    vpsrlw $1, %ymm3, %ymm3
-; AVX512F-NEXT:    vpmullw %ymm6, %ymm3, %ymm3
-; AVX512F-NEXT:    vpaddw %ymm0, %ymm2, %ymm0
-; AVX512F-NEXT:    vpaddw %ymm1, %ymm3, %ymm1
+; AVX512F-NEXT:    vpsrlw $1, %ymm1, %ymm1
+; AVX512F-NEXT:    vpmullw %ymm6, %ymm1, %ymm1
+; AVX512F-NEXT:    vpaddw %ymm3, %ymm2, %ymm2
+; AVX512F-NEXT:    vpaddw %ymm0, %ymm1, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-FALLBACK-LABEL: vec512_i16_unsigned_reg_reg:
 ; AVX512VL-FALLBACK:       # %bb.0:
-; AVX512VL-FALLBACK-NEXT:    vpminuw %ymm2, %ymm0, %ymm4
-; AVX512VL-FALLBACK-NEXT:    vpcmpeqw %ymm4, %ymm0, %ymm5
+; AVX512VL-FALLBACK-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
+; AVX512VL-FALLBACK-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
+; AVX512VL-FALLBACK-NEXT:    vpminuw %ymm2, %ymm3, %ymm4
+; AVX512VL-FALLBACK-NEXT:    vpcmpeqw %ymm4, %ymm3, %ymm5
 ; AVX512VL-FALLBACK-NEXT:    vpternlogq $15, %ymm5, %ymm5, %ymm5
 ; AVX512VL-FALLBACK-NEXT:    vmovdqa {{.*#+}} ymm6 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
 ; AVX512VL-FALLBACK-NEXT:    vpor %ymm6, %ymm5, %ymm5
-; AVX512VL-FALLBACK-NEXT:    vpminuw %ymm3, %ymm1, %ymm7
-; AVX512VL-FALLBACK-NEXT:    vpcmpeqw %ymm7, %ymm1, %ymm8
+; AVX512VL-FALLBACK-NEXT:    vpminuw %ymm1, %ymm0, %ymm7
+; AVX512VL-FALLBACK-NEXT:    vpcmpeqw %ymm7, %ymm0, %ymm8
 ; AVX512VL-FALLBACK-NEXT:    vpternlogq $15, %ymm8, %ymm8, %ymm8
 ; AVX512VL-FALLBACK-NEXT:    vpor %ymm6, %ymm8, %ymm6
-; AVX512VL-FALLBACK-NEXT:    vpmaxuw %ymm2, %ymm0, %ymm2
-; AVX512VL-FALLBACK-NEXT:    vpmaxuw %ymm3, %ymm1, %ymm3
-; AVX512VL-FALLBACK-NEXT:    vpsubw %ymm7, %ymm3, %ymm3
+; AVX512VL-FALLBACK-NEXT:    vpmaxuw %ymm2, %ymm3, %ymm2
+; AVX512VL-FALLBACK-NEXT:    vpmaxuw %ymm1, %ymm0, %ymm1
+; AVX512VL-FALLBACK-NEXT:    vpsubw %ymm7, %ymm1, %ymm1
 ; AVX512VL-FALLBACK-NEXT:    vpsubw %ymm4, %ymm2, %ymm2
 ; AVX512VL-FALLBACK-NEXT:    vpsrlw $1, %ymm2, %ymm2
 ; AVX512VL-FALLBACK-NEXT:    vpmullw %ymm5, %ymm2, %ymm2
-; AVX512VL-FALLBACK-NEXT:    vpsrlw $1, %ymm3, %ymm3
-; AVX512VL-FALLBACK-NEXT:    vpmullw %ymm6, %ymm3, %ymm3
-; AVX512VL-FALLBACK-NEXT:    vpaddw %ymm0, %ymm2, %ymm0
-; AVX512VL-FALLBACK-NEXT:    vpaddw %ymm1, %ymm3, %ymm1
+; AVX512VL-FALLBACK-NEXT:    vpsrlw $1, %ymm1, %ymm1
+; AVX512VL-FALLBACK-NEXT:    vpmullw %ymm6, %ymm1, %ymm1
+; AVX512VL-FALLBACK-NEXT:    vpaddw %ymm3, %ymm2, %ymm2
+; AVX512VL-FALLBACK-NEXT:    vpaddw %ymm0, %ymm1, %ymm0
+; AVX512VL-FALLBACK-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512VL-FALLBACK-NEXT:    retq
 ;
 ; AVX512BW-LABEL: vec512_i16_unsigned_reg_reg:
@@ -455,48 +467,52 @@ define <32 x i16> @vec512_i16_unsigned_reg_reg(<32 x i16> %a1, <32 x i16> %a2) n
 define <32 x i16> @vec512_i16_signed_mem_reg(<32 x i16>* %a1_addr, <32 x i16> %a2) nounwind {
 ; AVX512F-LABEL: vec512_i16_signed_mem_reg:
 ; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512F-NEXT:    vmovdqa (%rdi), %ymm2
 ; AVX512F-NEXT:    vmovdqa 32(%rdi), %ymm3
-; AVX512F-NEXT:    vpcmpgtw %ymm0, %ymm2, %ymm4
+; AVX512F-NEXT:    vpcmpgtw %ymm1, %ymm3, %ymm4
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm5 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
 ; AVX512F-NEXT:    vpor %ymm5, %ymm4, %ymm4
-; AVX512F-NEXT:    vpcmpgtw %ymm1, %ymm3, %ymm6
+; AVX512F-NEXT:    vpcmpgtw %ymm0, %ymm2, %ymm6
 ; AVX512F-NEXT:    vpor %ymm5, %ymm6, %ymm5
-; AVX512F-NEXT:    vpminsw %ymm0, %ymm2, %ymm6
-; AVX512F-NEXT:    vpminsw %ymm1, %ymm3, %ymm7
-; AVX512F-NEXT:    vpmaxsw %ymm0, %ymm2, %ymm0
-; AVX512F-NEXT:    vpsubw %ymm6, %ymm0, %ymm0
+; AVX512F-NEXT:    vpminsw %ymm1, %ymm3, %ymm6
+; AVX512F-NEXT:    vpminsw %ymm0, %ymm2, %ymm7
 ; AVX512F-NEXT:    vpmaxsw %ymm1, %ymm3, %ymm1
-; AVX512F-NEXT:    vpsubw %ymm7, %ymm1, %ymm1
-; AVX512F-NEXT:    vpsrlw $1, %ymm0, %ymm0
-; AVX512F-NEXT:    vpmullw %ymm4, %ymm0, %ymm0
+; AVX512F-NEXT:    vpsubw %ymm6, %ymm1, %ymm1
+; AVX512F-NEXT:    vpmaxsw %ymm0, %ymm2, %ymm0
+; AVX512F-NEXT:    vpsubw %ymm7, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpsrlw $1, %ymm1, %ymm1
-; AVX512F-NEXT:    vpmullw %ymm5, %ymm1, %ymm1
-; AVX512F-NEXT:    vpaddw %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vpmullw %ymm4, %ymm1, %ymm1
+; AVX512F-NEXT:    vpsrlw $1, %ymm0, %ymm0
+; AVX512F-NEXT:    vpmullw %ymm5, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpaddw %ymm3, %ymm1, %ymm1
+; AVX512F-NEXT:    vpaddw %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-FALLBACK-LABEL: vec512_i16_signed_mem_reg:
 ; AVX512VL-FALLBACK:       # %bb.0:
+; AVX512VL-FALLBACK-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512VL-FALLBACK-NEXT:    vmovdqa (%rdi), %ymm2
 ; AVX512VL-FALLBACK-NEXT:    vmovdqa 32(%rdi), %ymm3
-; AVX512VL-FALLBACK-NEXT:    vpcmpgtw %ymm0, %ymm2, %ymm4
+; AVX512VL-FALLBACK-NEXT:    vpcmpgtw %ymm1, %ymm3, %ymm4
 ; AVX512VL-FALLBACK-NEXT:    vmovdqa {{.*#+}} ymm5 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
 ; AVX512VL-FALLBACK-NEXT:    vpor %ymm5, %ymm4, %ymm4
-; AVX512VL-FALLBACK-NEXT:    vpcmpgtw %ymm1, %ymm3, %ymm6
+; AVX512VL-FALLBACK-NEXT:    vpcmpgtw %ymm0, %ymm2, %ymm6
 ; AVX512VL-FALLBACK-NEXT:    vpor %ymm5, %ymm6, %ymm5
-; AVX512VL-FALLBACK-NEXT:    vpminsw %ymm0, %ymm2, %ymm6
-; AVX512VL-FALLBACK-NEXT:    vpminsw %ymm1, %ymm3, %ymm7
-; AVX512VL-FALLBACK-NEXT:    vpmaxsw %ymm0, %ymm2, %ymm0
-; AVX512VL-FALLBACK-NEXT:    vpsubw %ymm6, %ymm0, %ymm0
+; AVX512VL-FALLBACK-NEXT:    vpminsw %ymm1, %ymm3, %ymm6
+; AVX512VL-FALLBACK-NEXT:    vpminsw %ymm0, %ymm2, %ymm7
 ; AVX512VL-FALLBACK-NEXT:    vpmaxsw %ymm1, %ymm3, %ymm1
-; AVX512VL-FALLBACK-NEXT:    vpsubw %ymm7, %ymm1, %ymm1
-; AVX512VL-FALLBACK-NEXT:    vpsrlw $1, %ymm0, %ymm0
-; AVX512VL-FALLBACK-NEXT:    vpmullw %ymm4, %ymm0, %ymm0
+; AVX512VL-FALLBACK-NEXT:    vpsubw %ymm6, %ymm1, %ymm1
+; AVX512VL-FALLBACK-NEXT:    vpmaxsw %ymm0, %ymm2, %ymm0
+; AVX512VL-FALLBACK-NEXT:    vpsubw %ymm7, %ymm0, %ymm0
 ; AVX512VL-FALLBACK-NEXT:    vpsrlw $1, %ymm1, %ymm1
-; AVX512VL-FALLBACK-NEXT:    vpmullw %ymm5, %ymm1, %ymm1
-; AVX512VL-FALLBACK-NEXT:    vpaddw %ymm2, %ymm0, %ymm0
+; AVX512VL-FALLBACK-NEXT:    vpmullw %ymm4, %ymm1, %ymm1
+; AVX512VL-FALLBACK-NEXT:    vpsrlw $1, %ymm0, %ymm0
+; AVX512VL-FALLBACK-NEXT:    vpmullw %ymm5, %ymm0, %ymm0
 ; AVX512VL-FALLBACK-NEXT:    vpaddw %ymm3, %ymm1, %ymm1
+; AVX512VL-FALLBACK-NEXT:    vpaddw %ymm2, %ymm0, %ymm0
+; AVX512VL-FALLBACK-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512VL-FALLBACK-NEXT:    retq
 ;
 ; AVX512BW-LABEL: vec512_i16_signed_mem_reg:
@@ -528,48 +544,52 @@ define <32 x i16> @vec512_i16_signed_mem_reg(<32 x i16>* %a1_addr, <32 x i16> %a
 define <32 x i16> @vec512_i16_signed_reg_mem(<32 x i16> %a1, <32 x i16>* %a2_addr) nounwind {
 ; AVX512F-LABEL: vec512_i16_signed_reg_mem:
 ; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512F-NEXT:    vmovdqa (%rdi), %ymm2
 ; AVX512F-NEXT:    vmovdqa 32(%rdi), %ymm3
-; AVX512F-NEXT:    vpcmpgtw %ymm2, %ymm0, %ymm4
+; AVX512F-NEXT:    vpcmpgtw %ymm3, %ymm1, %ymm4
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm5 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
 ; AVX512F-NEXT:    vpor %ymm5, %ymm4, %ymm4
-; AVX512F-NEXT:    vpcmpgtw %ymm3, %ymm1, %ymm6
+; AVX512F-NEXT:    vpcmpgtw %ymm2, %ymm0, %ymm6
 ; AVX512F-NEXT:    vpor %ymm5, %ymm6, %ymm5
-; AVX512F-NEXT:    vpminsw %ymm2, %ymm0, %ymm6
-; AVX512F-NEXT:    vpminsw %ymm3, %ymm1, %ymm7
-; AVX512F-NEXT:    vpmaxsw %ymm2, %ymm0, %ymm2
-; AVX512F-NEXT:    vpsubw %ymm6, %ymm2, %ymm2
+; AVX512F-NEXT:    vpminsw %ymm3, %ymm1, %ymm6
+; AVX512F-NEXT:    vpminsw %ymm2, %ymm0, %ymm7
 ; AVX512F-NEXT:    vpmaxsw %ymm3, %ymm1, %ymm3
-; AVX512F-NEXT:    vpsubw %ymm7, %ymm3, %ymm3
-; AVX512F-NEXT:    vpsrlw $1, %ymm2, %ymm2
-; AVX512F-NEXT:    vpmullw %ymm4, %ymm2, %ymm2
+; AVX512F-NEXT:    vpsubw %ymm6, %ymm3, %ymm3
+; AVX512F-NEXT:    vpmaxsw %ymm2, %ymm0, %ymm2
+; AVX512F-NEXT:    vpsubw %ymm7, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpsrlw $1, %ymm3, %ymm3
-; AVX512F-NEXT:    vpmullw %ymm5, %ymm3, %ymm3
-; AVX512F-NEXT:    vpaddw %ymm0, %ymm2, %ymm0
+; AVX512F-NEXT:    vpmullw %ymm4, %ymm3, %ymm3
+; AVX512F-NEXT:    vpsrlw $1, %ymm2, %ymm2
+; AVX512F-NEXT:    vpmullw %ymm5, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpaddw %ymm1, %ymm3, %ymm1
+; AVX512F-NEXT:    vpaddw %ymm0, %ymm2, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-FALLBACK-LABEL: vec512_i16_signed_reg_mem:
 ; AVX512VL-FALLBACK:       # %bb.0:
+; AVX512VL-FALLBACK-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512VL-FALLBACK-NEXT:    vmovdqa (%rdi), %ymm2
 ; AVX512VL-FALLBACK-NEXT:    vmovdqa 32(%rdi), %ymm3
-; AVX512VL-FALLBACK-NEXT:    vpcmpgtw %ymm2, %ymm0, %ymm4
+; AVX512VL-FALLBACK-NEXT:    vpcmpgtw %ymm3, %ymm1, %ymm4
 ; AVX512VL-FALLBACK-NEXT:    vmovdqa {{.*#+}} ymm5 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
 ; AVX512VL-FALLBACK-NEXT:    vpor %ymm5, %ymm4, %ymm4
-; AVX512VL-FALLBACK-NEXT:    vpcmpgtw %ymm3, %ymm1, %ymm6
+; AVX512VL-FALLBACK-NEXT:    vpcmpgtw %ymm2, %ymm0, %ymm6
 ; AVX512VL-FALLBACK-NEXT:    vpor %ymm5, %ymm6, %ymm5
-; AVX512VL-FALLBACK-NEXT:    vpminsw %ymm2, %ymm0, %ymm6
-; AVX512VL-FALLBACK-NEXT:    vpminsw %ymm3, %ymm1, %ymm7
-; AVX512VL-FALLBACK-NEXT:    vpmaxsw %ymm2, %ymm0, %ymm2
-; AVX512VL-FALLBACK-NEXT:    vpsubw %ymm6, %ymm2, %ymm2
+; AVX512VL-FALLBACK-NEXT:    vpminsw %ymm3, %ymm1, %ymm6
+; AVX512VL-FALLBACK-NEXT:    vpminsw %ymm2, %ymm0, %ymm7
 ; AVX512VL-FALLBACK-NEXT:    vpmaxsw %ymm3, %ymm1, %ymm3
-; AVX512VL-FALLBACK-NEXT:    vpsubw %ymm7, %ymm3, %ymm3
-; AVX512VL-FALLBACK-NEXT:    vpsrlw $1, %ymm2, %ymm2
-; AVX512VL-FALLBACK-NEXT:    vpmullw %ymm4, %ymm2, %ymm2
+; AVX512VL-FALLBACK-NEXT:    vpsubw %ymm6, %ymm3, %ymm3
+; AVX512VL-FALLBACK-NEXT:    vpmaxsw %ymm2, %ymm0, %ymm2
+; AVX512VL-FALLBACK-NEXT:    vpsubw %ymm7, %ymm2, %ymm2
 ; AVX512VL-FALLBACK-NEXT:    vpsrlw $1, %ymm3, %ymm3
-; AVX512VL-FALLBACK-NEXT:    vpmullw %ymm5, %ymm3, %ymm3
-; AVX512VL-FALLBACK-NEXT:    vpaddw %ymm0, %ymm2, %ymm0
+; AVX512VL-FALLBACK-NEXT:    vpmullw %ymm4, %ymm3, %ymm3
+; AVX512VL-FALLBACK-NEXT:    vpsrlw $1, %ymm2, %ymm2
+; AVX512VL-FALLBACK-NEXT:    vpmullw %ymm5, %ymm2, %ymm2
 ; AVX512VL-FALLBACK-NEXT:    vpaddw %ymm1, %ymm3, %ymm1
+; AVX512VL-FALLBACK-NEXT:    vpaddw %ymm0, %ymm2, %ymm0
+; AVX512VL-FALLBACK-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512VL-FALLBACK-NEXT:    retq
 ;
 ; AVX512BW-LABEL: vec512_i16_signed_reg_mem:
@@ -605,23 +625,24 @@ define <32 x i16> @vec512_i16_signed_mem_mem(<32 x i16>* %a1_addr, <32 x i16>* %
 ; AVX512F-NEXT:    vmovdqa 32(%rdi), %ymm1
 ; AVX512F-NEXT:    vmovdqa (%rsi), %ymm2
 ; AVX512F-NEXT:    vmovdqa 32(%rsi), %ymm3
-; AVX512F-NEXT:    vpcmpgtw %ymm2, %ymm0, %ymm4
+; AVX512F-NEXT:    vpcmpgtw %ymm3, %ymm1, %ymm4
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm5 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
 ; AVX512F-NEXT:    vpor %ymm5, %ymm4, %ymm4
-; AVX512F-NEXT:    vpcmpgtw %ymm3, %ymm1, %ymm6
+; AVX512F-NEXT:    vpcmpgtw %ymm2, %ymm0, %ymm6
 ; AVX512F-NEXT:    vpor %ymm5, %ymm6, %ymm5
-; AVX512F-NEXT:    vpminsw %ymm2, %ymm0, %ymm6
-; AVX512F-NEXT:    vpminsw %ymm3, %ymm1, %ymm7
-; AVX512F-NEXT:    vpmaxsw %ymm2, %ymm0, %ymm2
-; AVX512F-NEXT:    vpsubw %ymm6, %ymm2, %ymm2
+; AVX512F-NEXT:    vpminsw %ymm3, %ymm1, %ymm6
+; AVX512F-NEXT:    vpminsw %ymm2, %ymm0, %ymm7
 ; AVX512F-NEXT:    vpmaxsw %ymm3, %ymm1, %ymm3
-; AVX512F-NEXT:    vpsubw %ymm7, %ymm3, %ymm3
-; AVX512F-NEXT:    vpsrlw $1, %ymm2, %ymm2
-; AVX512F-NEXT:    vpmullw %ymm4, %ymm2, %ymm2
+; AVX512F-NEXT:    vpsubw %ymm6, %ymm3, %ymm3
+; AVX512F-NEXT:    vpmaxsw %ymm2, %ymm0, %ymm2
+; AVX512F-NEXT:    vpsubw %ymm7, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpsrlw $1, %ymm3, %ymm3
-; AVX512F-NEXT:    vpmullw %ymm5, %ymm3, %ymm3
-; AVX512F-NEXT:    vpaddw %ymm0, %ymm2, %ymm0
+; AVX512F-NEXT:    vpmullw %ymm4, %ymm3, %ymm3
+; AVX512F-NEXT:    vpsrlw $1, %ymm2, %ymm2
+; AVX512F-NEXT:    vpmullw %ymm5, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpaddw %ymm1, %ymm3, %ymm1
+; AVX512F-NEXT:    vpaddw %ymm0, %ymm2, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-FALLBACK-LABEL: vec512_i16_signed_mem_mem:
@@ -630,23 +651,24 @@ define <32 x i16> @vec512_i16_signed_mem_mem(<32 x i16>* %a1_addr, <32 x i16>* %
 ; AVX512VL-FALLBACK-NEXT:    vmovdqa 32(%rdi), %ymm1
 ; AVX512VL-FALLBACK-NEXT:    vmovdqa (%rsi), %ymm2
 ; AVX512VL-FALLBACK-NEXT:    vmovdqa 32(%rsi), %ymm3
-; AVX512VL-FALLBACK-NEXT:    vpcmpgtw %ymm2, %ymm0, %ymm4
+; AVX512VL-FALLBACK-NEXT:    vpcmpgtw %ymm3, %ymm1, %ymm4
 ; AVX512VL-FALLBACK-NEXT:    vmovdqa {{.*#+}} ymm5 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
 ; AVX512VL-FALLBACK-NEXT:    vpor %ymm5, %ymm4, %ymm4
-; AVX512VL-FALLBACK-NEXT:    vpcmpgtw %ymm3, %ymm1, %ymm6
+; AVX512VL-FALLBACK-NEXT:    vpcmpgtw %ymm2, %ymm0, %ymm6
 ; AVX512VL-FALLBACK-NEXT:    vpor %ymm5, %ymm6, %ymm5
-; AVX512VL-FALLBACK-NEXT:    vpminsw %ymm2, %ymm0, %ymm6
-; AVX512VL-FALLBACK-NEXT:    vpminsw %ymm3, %ymm1, %ymm7
-; AVX512VL-FALLBACK-NEXT:    vpmaxsw %ymm2, %ymm0, %ymm2
-; AVX512VL-FALLBACK-NEXT:    vpsubw %ymm6, %ymm2, %ymm2
+; AVX512VL-FALLBACK-NEXT:    vpminsw %ymm3, %ymm1, %ymm6
+; AVX512VL-FALLBACK-NEXT:    vpminsw %ymm2, %ymm0, %ymm7
 ; AVX512VL-FALLBACK-NEXT:    vpmaxsw %ymm3, %ymm1, %ymm3
-; AVX512VL-FALLBACK-NEXT:    vpsubw %ymm7, %ymm3, %ymm3
-; AVX512VL-FALLBACK-NEXT:    vpsrlw $1, %ymm2, %ymm2
-; AVX512VL-FALLBACK-NEXT:    vpmullw %ymm4, %ymm2, %ymm2
+; AVX512VL-FALLBACK-NEXT:    vpsubw %ymm6, %ymm3, %ymm3
+; AVX512VL-FALLBACK-NEXT:    vpmaxsw %ymm2, %ymm0, %ymm2
+; AVX512VL-FALLBACK-NEXT:    vpsubw %ymm7, %ymm2, %ymm2
 ; AVX512VL-FALLBACK-NEXT:    vpsrlw $1, %ymm3, %ymm3
-; AVX512VL-FALLBACK-NEXT:    vpmullw %ymm5, %ymm3, %ymm3
-; AVX512VL-FALLBACK-NEXT:    vpaddw %ymm0, %ymm2, %ymm0
+; AVX512VL-FALLBACK-NEXT:    vpmullw %ymm4, %ymm3, %ymm3
+; AVX512VL-FALLBACK-NEXT:    vpsrlw $1, %ymm2, %ymm2
+; AVX512VL-FALLBACK-NEXT:    vpmullw %ymm5, %ymm2, %ymm2
 ; AVX512VL-FALLBACK-NEXT:    vpaddw %ymm1, %ymm3, %ymm1
+; AVX512VL-FALLBACK-NEXT:    vpaddw %ymm0, %ymm2, %ymm0
+; AVX512VL-FALLBACK-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512VL-FALLBACK-NEXT:    retq
 ;
 ; AVX512BW-LABEL: vec512_i16_signed_mem_mem:
@@ -686,84 +708,90 @@ define <32 x i16> @vec512_i16_signed_mem_mem(<32 x i16>* %a1_addr, <32 x i16>* %
 define <64 x i8> @vec512_i8_signed_reg_reg(<64 x i8> %a1, <64 x i8> %a2) nounwind {
 ; AVX512F-LABEL: vec512_i8_signed_reg_reg:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpcmpgtb %ymm2, %ymm0, %ymm4
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512F-NEXT:    vpcmpgtb %ymm3, %ymm2, %ymm4
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm5 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
 ; AVX512F-NEXT:    vpor %ymm5, %ymm4, %ymm4
-; AVX512F-NEXT:    vpcmpgtb %ymm3, %ymm1, %ymm6
+; AVX512F-NEXT:    vpcmpgtb %ymm1, %ymm0, %ymm6
 ; AVX512F-NEXT:    vpor %ymm5, %ymm6, %ymm5
-; AVX512F-NEXT:    vpminsb %ymm2, %ymm0, %ymm6
-; AVX512F-NEXT:    vpminsb %ymm3, %ymm1, %ymm7
-; AVX512F-NEXT:    vpmaxsb %ymm2, %ymm0, %ymm2
-; AVX512F-NEXT:    vpsubb %ymm6, %ymm2, %ymm2
-; AVX512F-NEXT:    vpmaxsb %ymm3, %ymm1, %ymm3
-; AVX512F-NEXT:    vpsubb %ymm7, %ymm3, %ymm3
-; AVX512F-NEXT:    vpsrlw $1, %ymm2, %ymm2
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm6 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; AVX512F-NEXT:    vpand %ymm6, %ymm2, %ymm2
+; AVX512F-NEXT:    vpminsb %ymm3, %ymm2, %ymm6
+; AVX512F-NEXT:    vpminsb %ymm1, %ymm0, %ymm7
+; AVX512F-NEXT:    vpmaxsb %ymm3, %ymm2, %ymm3
+; AVX512F-NEXT:    vpsubb %ymm6, %ymm3, %ymm3
+; AVX512F-NEXT:    vpmaxsb %ymm1, %ymm0, %ymm1
+; AVX512F-NEXT:    vpsubb %ymm7, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpsrlw $1, %ymm3, %ymm3
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm6 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
 ; AVX512F-NEXT:    vpand %ymm6, %ymm3, %ymm3
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm6 = ymm3[8],ymm0[8],ymm3[9],ymm0[9],ymm3[10],ymm0[10],ymm3[11],ymm0[11],ymm3[12],ymm0[12],ymm3[13],ymm0[13],ymm3[14],ymm0[14],ymm3[15],ymm0[15],ymm3[24],ymm0[24],ymm3[25],ymm0[25],ymm3[26],ymm0[26],ymm3[27],ymm0[27],ymm3[28],ymm0[28],ymm3[29],ymm0[29],ymm3[30],ymm0[30],ymm3[31],ymm0[31]
+; AVX512F-NEXT:    vpsrlw $1, %ymm1, %ymm1
+; AVX512F-NEXT:    vpand %ymm6, %ymm1, %ymm1
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm6 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31]
 ; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm7 = ymm5[8],ymm0[8],ymm5[9],ymm0[9],ymm5[10],ymm0[10],ymm5[11],ymm0[11],ymm5[12],ymm0[12],ymm5[13],ymm0[13],ymm5[14],ymm0[14],ymm5[15],ymm0[15],ymm5[24],ymm0[24],ymm5[25],ymm0[25],ymm5[26],ymm0[26],ymm5[27],ymm0[27],ymm5[28],ymm0[28],ymm5[29],ymm0[29],ymm5[30],ymm0[30],ymm5[31],ymm0[31]
 ; AVX512F-NEXT:    vpmullw %ymm7, %ymm6, %ymm6
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm7 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
 ; AVX512F-NEXT:    vpand %ymm7, %ymm6, %ymm6
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23]
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
 ; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm5[0],ymm0[0],ymm5[1],ymm0[1],ymm5[2],ymm0[2],ymm5[3],ymm0[3],ymm5[4],ymm0[4],ymm5[5],ymm0[5],ymm5[6],ymm0[6],ymm5[7],ymm0[7],ymm5[16],ymm0[16],ymm5[17],ymm0[17],ymm5[18],ymm0[18],ymm5[19],ymm0[19],ymm5[20],ymm0[20],ymm5[21],ymm0[21],ymm5[22],ymm0[22],ymm5[23],ymm0[23]
-; AVX512F-NEXT:    vpmullw %ymm5, %ymm3, %ymm3
-; AVX512F-NEXT:    vpand %ymm7, %ymm3, %ymm3
-; AVX512F-NEXT:    vpackuswb %ymm6, %ymm3, %ymm3
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11],ymm2[12],ymm0[12],ymm2[13],ymm0[13],ymm2[14],ymm0[14],ymm2[15],ymm0[15],ymm2[24],ymm0[24],ymm2[25],ymm0[25],ymm2[26],ymm0[26],ymm2[27],ymm0[27],ymm2[28],ymm0[28],ymm2[29],ymm0[29],ymm2[30],ymm0[30],ymm2[31],ymm0[31]
+; AVX512F-NEXT:    vpmullw %ymm5, %ymm1, %ymm1
+; AVX512F-NEXT:    vpand %ymm7, %ymm1, %ymm1
+; AVX512F-NEXT:    vpackuswb %ymm6, %ymm1, %ymm1
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm3[8],ymm0[8],ymm3[9],ymm0[9],ymm3[10],ymm0[10],ymm3[11],ymm0[11],ymm3[12],ymm0[12],ymm3[13],ymm0[13],ymm3[14],ymm0[14],ymm3[15],ymm0[15],ymm3[24],ymm0[24],ymm3[25],ymm0[25],ymm3[26],ymm0[26],ymm3[27],ymm0[27],ymm3[28],ymm0[28],ymm3[29],ymm0[29],ymm3[30],ymm0[30],ymm3[31],ymm0[31]
 ; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm6 = ymm4[8],ymm0[8],ymm4[9],ymm0[9],ymm4[10],ymm0[10],ymm4[11],ymm0[11],ymm4[12],ymm0[12],ymm4[13],ymm0[13],ymm4[14],ymm0[14],ymm4[15],ymm0[15],ymm4[24],ymm0[24],ymm4[25],ymm0[25],ymm4[26],ymm0[26],ymm4[27],ymm0[27],ymm4[28],ymm0[28],ymm4[29],ymm0[29],ymm4[30],ymm0[30],ymm4[31],ymm0[31]
 ; AVX512F-NEXT:    vpmullw %ymm6, %ymm5, %ymm5
 ; AVX512F-NEXT:    vpand %ymm7, %ymm5, %ymm5
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23]
 ; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm4[0],ymm0[0],ymm4[1],ymm0[1],ymm4[2],ymm0[2],ymm4[3],ymm0[3],ymm4[4],ymm0[4],ymm4[5],ymm0[5],ymm4[6],ymm0[6],ymm4[7],ymm0[7],ymm4[16],ymm0[16],ymm4[17],ymm0[17],ymm4[18],ymm0[18],ymm4[19],ymm0[19],ymm4[20],ymm0[20],ymm4[21],ymm0[21],ymm4[22],ymm0[22],ymm4[23],ymm0[23]
-; AVX512F-NEXT:    vpmullw %ymm4, %ymm2, %ymm2
-; AVX512F-NEXT:    vpand %ymm7, %ymm2, %ymm2
-; AVX512F-NEXT:    vpackuswb %ymm5, %ymm2, %ymm2
-; AVX512F-NEXT:    vpaddb %ymm0, %ymm2, %ymm0
-; AVX512F-NEXT:    vpaddb %ymm1, %ymm3, %ymm1
+; AVX512F-NEXT:    vpmullw %ymm4, %ymm3, %ymm3
+; AVX512F-NEXT:    vpand %ymm7, %ymm3, %ymm3
+; AVX512F-NEXT:    vpackuswb %ymm5, %ymm3, %ymm3
+; AVX512F-NEXT:    vpaddb %ymm2, %ymm3, %ymm2
+; AVX512F-NEXT:    vpaddb %ymm0, %ymm1, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-FALLBACK-LABEL: vec512_i8_signed_reg_reg:
 ; AVX512VL-FALLBACK:       # %bb.0:
-; AVX512VL-FALLBACK-NEXT:    vpcmpgtb %ymm2, %ymm0, %ymm4
+; AVX512VL-FALLBACK-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
+; AVX512VL-FALLBACK-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512VL-FALLBACK-NEXT:    vpcmpgtb %ymm3, %ymm2, %ymm4
 ; AVX512VL-FALLBACK-NEXT:    vmovdqa {{.*#+}} ymm5 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
 ; AVX512VL-FALLBACK-NEXT:    vpor %ymm5, %ymm4, %ymm4
-; AVX512VL-FALLBACK-NEXT:    vpcmpgtb %ymm3, %ymm1, %ymm6
+; AVX512VL-FALLBACK-NEXT:    vpcmpgtb %ymm1, %ymm0, %ymm6
 ; AVX512VL-FALLBACK-NEXT:    vpor %ymm5, %ymm6, %ymm5
-; AVX512VL-FALLBACK-NEXT:    vpminsb %ymm2, %ymm0, %ymm6
-; AVX512VL-FALLBACK-NEXT:    vpminsb %ymm3, %ymm1, %ymm7
-; AVX512VL-FALLBACK-NEXT:    vpmaxsb %ymm2, %ymm0, %ymm2
-; AVX512VL-FALLBACK-NEXT:    vpsubb %ymm6, %ymm2, %ymm2
-; AVX512VL-FALLBACK-NEXT:    vpmaxsb %ymm3, %ymm1, %ymm3
-; AVX512VL-FALLBACK-NEXT:    vpsubb %ymm7, %ymm3, %ymm3
-; AVX512VL-FALLBACK-NEXT:    vpsrlw $1, %ymm2, %ymm2
-; AVX512VL-FALLBACK-NEXT:    vmovdqa {{.*#+}} ymm6 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; AVX512VL-FALLBACK-NEXT:    vpand %ymm6, %ymm2, %ymm2
+; AVX512VL-FALLBACK-NEXT:    vpminsb %ymm3, %ymm2, %ymm6
+; AVX512VL-FALLBACK-NEXT:    vpminsb %ymm1, %ymm0, %ymm7
+; AVX512VL-FALLBACK-NEXT:    vpmaxsb %ymm3, %ymm2, %ymm3
+; AVX512VL-FALLBACK-NEXT:    vpsubb %ymm6, %ymm3, %ymm3
+; AVX512VL-FALLBACK-NEXT:    vpmaxsb %ymm1, %ymm0, %ymm1
+; AVX512VL-FALLBACK-NEXT:    vpsubb %ymm7, %ymm1, %ymm1
 ; AVX512VL-FALLBACK-NEXT:    vpsrlw $1, %ymm3, %ymm3
+; AVX512VL-FALLBACK-NEXT:    vmovdqa {{.*#+}} ymm6 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
 ; AVX512VL-FALLBACK-NEXT:    vpand %ymm6, %ymm3, %ymm3
-; AVX512VL-FALLBACK-NEXT:    vpunpckhbw {{.*#+}} ymm6 = ymm3[8],ymm0[8],ymm3[9],ymm0[9],ymm3[10],ymm0[10],ymm3[11],ymm0[11],ymm3[12],ymm0[12],ymm3[13],ymm0[13],ymm3[14],ymm0[14],ymm3[15],ymm0[15],ymm3[24],ymm0[24],ymm3[25],ymm0[25],ymm3[26],ymm0[26],ymm3[27],ymm0[27],ymm3[28],ymm0[28],ymm3[29],ymm0[29],ymm3[30],ymm0[30],ymm3[31],ymm0[31]
+; AVX512VL-FALLBACK-NEXT:    vpsrlw $1, %ymm1, %ymm1
+; AVX512VL-FALLBACK-NEXT:    vpand %ymm6, %ymm1, %ymm1
+; AVX512VL-FALLBACK-NEXT:    vpunpckhbw {{.*#+}} ymm6 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31]
 ; AVX512VL-FALLBACK-NEXT:    vpunpckhbw {{.*#+}} ymm7 = ymm5[8],ymm0[8],ymm5[9],ymm0[9],ymm5[10],ymm0[10],ymm5[11],ymm0[11],ymm5[12],ymm0[12],ymm5[13],ymm0[13],ymm5[14],ymm0[14],ymm5[15],ymm0[15],ymm5[24],ymm0[24],ymm5[25],ymm0[25],ymm5[26],ymm0[26],ymm5[27],ymm0[27],ymm5[28],ymm0[28],ymm5[29],ymm0[29],ymm5[30],ymm0[30],ymm5[31],ymm0[31]
 ; AVX512VL-FALLBACK-NEXT:    vpmullw %ymm7, %ymm6, %ymm6
 ; AVX512VL-FALLBACK-NEXT:    vmovdqa {{.*#+}} ymm7 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
 ; AVX512VL-FALLBACK-NEXT:    vpand %ymm7, %ymm6, %ymm6
-; AVX512VL-FALLBACK-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23]
+; AVX512VL-FALLBACK-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
 ; AVX512VL-FALLBACK-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm5[0],ymm0[0],ymm5[1],ymm0[1],ymm5[2],ymm0[2],ymm5[3],ymm0[3],ymm5[4],ymm0[4],ymm5[5],ymm0[5],ymm5[6],ymm0[6],ymm5[7],ymm0[7],ymm5[16],ymm0[16],ymm5[17],ymm0[17],ymm5[18],ymm0[18],ymm5[19],ymm0[19],ymm5[20],ymm0[20],ymm5[21],ymm0[21],ymm5[22],ymm0[22],ymm5[23],ymm0[23]
-; AVX512VL-FALLBACK-NEXT:    vpmullw %ymm5, %ymm3, %ymm3
-; AVX512VL-FALLBACK-NEXT:    vpand %ymm7, %ymm3, %ymm3
-; AVX512VL-FALLBACK-NEXT:    vpackuswb %ymm6, %ymm3, %ymm3
-; AVX512VL-FALLBACK-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11],ymm2[12],ymm0[12],ymm2[13],ymm0[13],ymm2[14],ymm0[14],ymm2[15],ymm0[15],ymm2[24],ymm0[24],ymm2[25],ymm0[25],ymm2[26],ymm0[26],ymm2[27],ymm0[27],ymm2[28],ymm0[28],ymm2[29],ymm0[29],ymm2[30],ymm0[30],ymm2[31],ymm0[31]
+; AVX512VL-FALLBACK-NEXT:    vpmullw %ymm5, %ymm1, %ymm1
+; AVX512VL-FALLBACK-NEXT:    vpand %ymm7, %ymm1, %ymm1
+; AVX512VL-FALLBACK-NEXT:    vpackuswb %ymm6, %ymm1, %ymm1
+; AVX512VL-FALLBACK-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm3[8],ymm0[8],ymm3[9],ymm0[9],ymm3[10],ymm0[10],ymm3[11],ymm0[11],ymm3[12],ymm0[12],ymm3[13],ymm0[13],ymm3[14],ymm0[14],ymm3[15],ymm0[15],ymm3[24],ymm0[24],ymm3[25],ymm0[25],ymm3[26],ymm0[26],ymm3[27],ymm0[27],ymm3[28],ymm0[28],ymm3[29],ymm0[29],ymm3[30],ymm0[30],ymm3[31],ymm0[31]
 ; AVX512VL-FALLBACK-NEXT:    vpunpckhbw {{.*#+}} ymm6 = ymm4[8],ymm0[8],ymm4[9],ymm0[9],ymm4[10],ymm0[10],ymm4[11],ymm0[11],ymm4[12],ymm0[12],ymm4[13],ymm0[13],ymm4[14],ymm0[14],ymm4[15],ymm0[15],ymm4[24],ymm0[24],ymm4[25],ymm0[25],ymm4[26],ymm0[26],ymm4[27],ymm0[27],ymm4[28],ymm0[28],ymm4[29],ymm0[29],ymm4[30],ymm0[30],ymm4[31],ymm0[31]
 ; AVX512VL-FALLBACK-NEXT:    vpmullw %ymm6, %ymm5, %ymm5
 ; AVX512VL-FALLBACK-NEXT:    vpand %ymm7, %ymm5, %ymm5
-; AVX512VL-FALLBACK-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
+; AVX512VL-FALLBACK-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23]
 ; AVX512VL-FALLBACK-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm4[0],ymm0[0],ymm4[1],ymm0[1],ymm4[2],ymm0[2],ymm4[3],ymm0[3],ymm4[4],ymm0[4],ymm4[5],ymm0[5],ymm4[6],ymm0[6],ymm4[7],ymm0[7],ymm4[16],ymm0[16],ymm4[17],ymm0[17],ymm4[18],ymm0[18],ymm4[19],ymm0[19],ymm4[20],ymm0[20],ymm4[21],ymm0[21],ymm4[22],ymm0[22],ymm4[23],ymm0[23]
-; AVX512VL-FALLBACK-NEXT:    vpmullw %ymm4, %ymm2, %ymm2
-; AVX512VL-FALLBACK-NEXT:    vpand %ymm7, %ymm2, %ymm2
-; AVX512VL-FALLBACK-NEXT:    vpackuswb %ymm5, %ymm2, %ymm2
-; AVX512VL-FALLBACK-NEXT:    vpaddb %ymm0, %ymm2, %ymm0
-; AVX512VL-FALLBACK-NEXT:    vpaddb %ymm1, %ymm3, %ymm1
+; AVX512VL-FALLBACK-NEXT:    vpmullw %ymm4, %ymm3, %ymm3
+; AVX512VL-FALLBACK-NEXT:    vpand %ymm7, %ymm3, %ymm3
+; AVX512VL-FALLBACK-NEXT:    vpackuswb %ymm5, %ymm3, %ymm3
+; AVX512VL-FALLBACK-NEXT:    vpaddb %ymm2, %ymm3, %ymm2
+; AVX512VL-FALLBACK-NEXT:    vpaddb %ymm0, %ymm1, %ymm0
+; AVX512VL-FALLBACK-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512VL-FALLBACK-NEXT:    retq
 ;
 ; AVX512BW-LABEL: vec512_i8_signed_reg_reg:
@@ -803,88 +831,94 @@ define <64 x i8> @vec512_i8_signed_reg_reg(<64 x i8> %a1, <64 x i8> %a2) nounwin
 define <64 x i8> @vec512_i8_unsigned_reg_reg(<64 x i8> %a1, <64 x i8> %a2) nounwind {
 ; AVX512F-LABEL: vec512_i8_unsigned_reg_reg:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpminub %ymm2, %ymm0, %ymm4
-; AVX512F-NEXT:    vpcmpeqb %ymm4, %ymm0, %ymm5
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512F-NEXT:    vpminub %ymm3, %ymm2, %ymm4
+; AVX512F-NEXT:    vpcmpeqb %ymm4, %ymm2, %ymm5
 ; AVX512F-NEXT:    vpternlogq $15, %zmm5, %zmm5, %zmm5
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm6 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
 ; AVX512F-NEXT:    vpor %ymm6, %ymm5, %ymm5
-; AVX512F-NEXT:    vpminub %ymm3, %ymm1, %ymm7
-; AVX512F-NEXT:    vpcmpeqb %ymm7, %ymm1, %ymm8
+; AVX512F-NEXT:    vpminub %ymm1, %ymm0, %ymm7
+; AVX512F-NEXT:    vpcmpeqb %ymm7, %ymm0, %ymm8
 ; AVX512F-NEXT:    vpternlogq $15, %zmm8, %zmm8, %zmm8
 ; AVX512F-NEXT:    vpor %ymm6, %ymm8, %ymm6
-; AVX512F-NEXT:    vpmaxub %ymm2, %ymm0, %ymm2
-; AVX512F-NEXT:    vpmaxub %ymm3, %ymm1, %ymm3
-; AVX512F-NEXT:    vpsubb %ymm7, %ymm3, %ymm3
-; AVX512F-NEXT:    vpsubb %ymm4, %ymm2, %ymm2
-; AVX512F-NEXT:    vpsrlw $1, %ymm2, %ymm2
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm4 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; AVX512F-NEXT:    vpand %ymm4, %ymm2, %ymm2
+; AVX512F-NEXT:    vpmaxub %ymm3, %ymm2, %ymm3
+; AVX512F-NEXT:    vpmaxub %ymm1, %ymm0, %ymm1
+; AVX512F-NEXT:    vpsubb %ymm7, %ymm1, %ymm1
+; AVX512F-NEXT:    vpsubb %ymm4, %ymm3, %ymm3
 ; AVX512F-NEXT:    vpsrlw $1, %ymm3, %ymm3
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm4 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
 ; AVX512F-NEXT:    vpand %ymm4, %ymm3, %ymm3
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm3[8],ymm0[8],ymm3[9],ymm0[9],ymm3[10],ymm0[10],ymm3[11],ymm0[11],ymm3[12],ymm0[12],ymm3[13],ymm0[13],ymm3[14],ymm0[14],ymm3[15],ymm0[15],ymm3[24],ymm0[24],ymm3[25],ymm0[25],ymm3[26],ymm0[26],ymm3[27],ymm0[27],ymm3[28],ymm0[28],ymm3[29],ymm0[29],ymm3[30],ymm0[30],ymm3[31],ymm0[31]
+; AVX512F-NEXT:    vpsrlw $1, %ymm1, %ymm1
+; AVX512F-NEXT:    vpand %ymm4, %ymm1, %ymm1
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31]
 ; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm7 = ymm6[8],ymm0[8],ymm6[9],ymm0[9],ymm6[10],ymm0[10],ymm6[11],ymm0[11],ymm6[12],ymm0[12],ymm6[13],ymm0[13],ymm6[14],ymm0[14],ymm6[15],ymm0[15],ymm6[24],ymm0[24],ymm6[25],ymm0[25],ymm6[26],ymm0[26],ymm6[27],ymm0[27],ymm6[28],ymm0[28],ymm6[29],ymm0[29],ymm6[30],ymm0[30],ymm6[31],ymm0[31]
 ; AVX512F-NEXT:    vpmullw %ymm7, %ymm4, %ymm4
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm7 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
 ; AVX512F-NEXT:    vpand %ymm7, %ymm4, %ymm4
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23]
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
 ; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm6 = ymm6[0],ymm0[0],ymm6[1],ymm0[1],ymm6[2],ymm0[2],ymm6[3],ymm0[3],ymm6[4],ymm0[4],ymm6[5],ymm0[5],ymm6[6],ymm0[6],ymm6[7],ymm0[7],ymm6[16],ymm0[16],ymm6[17],ymm0[17],ymm6[18],ymm0[18],ymm6[19],ymm0[19],ymm6[20],ymm0[20],ymm6[21],ymm0[21],ymm6[22],ymm0[22],ymm6[23],ymm0[23]
-; AVX512F-NEXT:    vpmullw %ymm6, %ymm3, %ymm3
-; AVX512F-NEXT:    vpand %ymm7, %ymm3, %ymm3
-; AVX512F-NEXT:    vpackuswb %ymm4, %ymm3, %ymm3
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11],ymm2[12],ymm0[12],ymm2[13],ymm0[13],ymm2[14],ymm0[14],ymm2[15],ymm0[15],ymm2[24],ymm0[24],ymm2[25],ymm0[25],ymm2[26],ymm0[26],ymm2[27],ymm0[27],ymm2[28],ymm0[28],ymm2[29],ymm0[29],ymm2[30],ymm0[30],ymm2[31],ymm0[31]
+; AVX512F-NEXT:    vpmullw %ymm6, %ymm1, %ymm1
+; AVX512F-NEXT:    vpand %ymm7, %ymm1, %ymm1
+; AVX512F-NEXT:    vpackuswb %ymm4, %ymm1, %ymm1
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm3[8],ymm0[8],ymm3[9],ymm0[9],ymm3[10],ymm0[10],ymm3[11],ymm0[11],ymm3[12],ymm0[12],ymm3[13],ymm0[13],ymm3[14],ymm0[14],ymm3[15],ymm0[15],ymm3[24],ymm0[24],ymm3[25],ymm0[25],ymm3[26],ymm0[26],ymm3[27],ymm0[27],ymm3[28],ymm0[28],ymm3[29],ymm0[29],ymm3[30],ymm0[30],ymm3[31],ymm0[31]
 ; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm6 = ymm5[8],ymm0[8],ymm5[9],ymm0[9],ymm5[10],ymm0[10],ymm5[11],ymm0[11],ymm5[12],ymm0[12],ymm5[13],ymm0[13],ymm5[14],ymm0[14],ymm5[15],ymm0[15],ymm5[24],ymm0[24],ymm5[25],ymm0[25],ymm5[26],ymm0[26],ymm5[27],ymm0[27],ymm5[28],ymm0[28],ymm5[29],ymm0[29],ymm5[30],ymm0[30],ymm5[31],ymm0[31]
 ; AVX512F-NEXT:    vpmullw %ymm6, %ymm4, %ymm4
 ; AVX512F-NEXT:    vpand %ymm7, %ymm4, %ymm4
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23]
 ; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm5[0],ymm0[0],ymm5[1],ymm0[1],ymm5[2],ymm0[2],ymm5[3],ymm0[3],ymm5[4],ymm0[4],ymm5[5],ymm0[5],ymm5[6],ymm0[6],ymm5[7],ymm0[7],ymm5[16],ymm0[16],ymm5[17],ymm0[17],ymm5[18],ymm0[18],ymm5[19],ymm0[19],ymm5[20],ymm0[20],ymm5[21],ymm0[21],ymm5[22],ymm0[22],ymm5[23],ymm0[23]
-; AVX512F-NEXT:    vpmullw %ymm5, %ymm2, %ymm2
-; AVX512F-NEXT:    vpand %ymm7, %ymm2, %ymm2
-; AVX512F-NEXT:    vpackuswb %ymm4, %ymm2, %ymm2
-; AVX512F-NEXT:    vpaddb %ymm0, %ymm2, %ymm0
-; AVX512F-NEXT:    vpaddb %ymm1, %ymm3, %ymm1
+; AVX512F-NEXT:    vpmullw %ymm5, %ymm3, %ymm3
+; AVX512F-NEXT:    vpand %ymm7, %ymm3, %ymm3
+; AVX512F-NEXT:    vpackuswb %ymm4, %ymm3, %ymm3
+; AVX512F-NEXT:    vpaddb %ymm2, %ymm3, %ymm2
+; AVX512F-NEXT:    vpaddb %ymm0, %ymm1, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-FALLBACK-LABEL: vec512_i8_unsigned_reg_reg:
 ; AVX512VL-FALLBACK:       # %bb.0:
-; AVX512VL-FALLBACK-NEXT:    vpminub %ymm2, %ymm0, %ymm4
-; AVX512VL-FALLBACK-NEXT:    vpcmpeqb %ymm4, %ymm0, %ymm5
+; AVX512VL-FALLBACK-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
+; AVX512VL-FALLBACK-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512VL-FALLBACK-NEXT:    vpminub %ymm3, %ymm2, %ymm4
+; AVX512VL-FALLBACK-NEXT:    vpcmpeqb %ymm4, %ymm2, %ymm5
 ; AVX512VL-FALLBACK-NEXT:    vpternlogq $15, %ymm5, %ymm5, %ymm5
 ; AVX512VL-FALLBACK-NEXT:    vmovdqa {{.*#+}} ymm6 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
 ; AVX512VL-FALLBACK-NEXT:    vpor %ymm6, %ymm5, %ymm5
-; AVX512VL-FALLBACK-NEXT:    vpminub %ymm3, %ymm1, %ymm7
-; AVX512VL-FALLBACK-NEXT:    vpcmpeqb %ymm7, %ymm1, %ymm8
+; AVX512VL-FALLBACK-NEXT:    vpminub %ymm1, %ymm0, %ymm7
+; AVX512VL-FALLBACK-NEXT:    vpcmpeqb %ymm7, %ymm0, %ymm8
 ; AVX512VL-FALLBACK-NEXT:    vpternlogq $15, %ymm8, %ymm8, %ymm8
 ; AVX512VL-FALLBACK-NEXT:    vpor %ymm6, %ymm8, %ymm6
-; AVX512VL-FALLBACK-NEXT:    vpmaxub %ymm2, %ymm0, %ymm2
-; AVX512VL-FALLBACK-NEXT:    vpmaxub %ymm3, %ymm1, %ymm3
-; AVX512VL-FALLBACK-NEXT:    vpsubb %ymm7, %ymm3, %ymm3
-; AVX512VL-FALLBACK-NEXT:    vpsubb %ymm4, %ymm2, %ymm2
-; AVX512VL-FALLBACK-NEXT:    vpsrlw $1, %ymm2, %ymm2
-; AVX512VL-FALLBACK-NEXT:    vmovdqa {{.*#+}} ymm4 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; AVX512VL-FALLBACK-NEXT:    vpand %ymm4, %ymm2, %ymm2
+; AVX512VL-FALLBACK-NEXT:    vpmaxub %ymm3, %ymm2, %ymm3
+; AVX512VL-FALLBACK-NEXT:    vpmaxub %ymm1, %ymm0, %ymm1
+; AVX512VL-FALLBACK-NEXT:    vpsubb %ymm7, %ymm1, %ymm1
+; AVX512VL-FALLBACK-NEXT:    vpsubb %ymm4, %ymm3, %ymm3
 ; AVX512VL-FALLBACK-NEXT:    vpsrlw $1, %ymm3, %ymm3
+; AVX512VL-FALLBACK-NEXT:    vmovdqa {{.*#+}} ymm4 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
 ; AVX512VL-FALLBACK-NEXT:    vpand %ymm4, %ymm3, %ymm3
-; AVX512VL-FALLBACK-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm3[8],ymm0[8],ymm3[9],ymm0[9],ymm3[10],ymm0[10],ymm3[11],ymm0[11],ymm3[12],ymm0[12],ymm3[13],ymm0[13],ymm3[14],ymm0[14],ymm3[15],ymm0[15],ymm3[24],ymm0[24],ymm3[25],ymm0[25],ymm3[26],ymm0[26],ymm3[27],ymm0[27],ymm3[28],ymm0[28],ymm3[29],ymm0[29],ymm3[30],ymm0[30],ymm3[31],ymm0[31]
+; AVX512VL-FALLBACK-NEXT:    vpsrlw $1, %ymm1, %ymm1
+; AVX512VL-FALLBACK-NEXT:    vpand %ymm4, %ymm1, %ymm1
+; AVX512VL-FALLBACK-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31]
 ; AVX512VL-FALLBACK-NEXT:    vpunpckhbw {{.*#+}} ymm7 = ymm6[8],ymm0[8],ymm6[9],ymm0[9],ymm6[10],ymm0[10],ymm6[11],ymm0[11],ymm6[12],ymm0[12],ymm6[13],ymm0[13],ymm6[14],ymm0[14],ymm6[15],ymm0[15],ymm6[24],ymm0[24],ymm6[25],ymm0[25],ymm6[26],ymm0[26],ymm6[27],ymm0[27],ymm6[28],ymm0[28],ymm6[29],ymm0[29],ymm6[30],ymm0[30],ymm6[31],ymm0[31]
 ; AVX512VL-FALLBACK-NEXT:    vpmullw %ymm7, %ymm4, %ymm4
 ; AVX512VL-FALLBACK-NEXT:    vmovdqa {{.*#+}} ymm7 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
 ; AVX512VL-FALLBACK-NEXT:    vpand %ymm7, %ymm4, %ymm4
-; AVX512VL-FALLBACK-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23]
+; AVX512VL-FALLBACK-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
 ; AVX512VL-FALLBACK-NEXT:    vpunpcklbw {{.*#+}} ymm6 = ymm6[0],ymm0[0],ymm6[1],ymm0[1],ymm6[2],ymm0[2],ymm6[3],ymm0[3],ymm6[4],ymm0[4],ymm6[5],ymm0[5],ymm6[6],ymm0[6],ymm6[7],ymm0[7],ymm6[16],ymm0[16],ymm6[17],ymm0[17],ymm6[18],ymm0[18],ymm6[19],ymm0[19],ymm6[20],ymm0[20],ymm6[21],ymm0[21],ymm6[22],ymm0[22],ymm6[23],ymm0[23]
-; AVX512VL-FALLBACK-NEXT:    vpmullw %ymm6, %ymm3, %ymm3
-; AVX512VL-FALLBACK-NEXT:    vpand %ymm7, %ymm3, %ymm3
-; AVX512VL-FALLBACK-NEXT:    vpackuswb %ymm4, %ymm3, %ymm3
-; AVX512VL-FALLBACK-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11],ymm2[12],ymm0[12],ymm2[13],ymm0[13],ymm2[14],ymm0[14],ymm2[15],ymm0[15],ymm2[24],ymm0[24],ymm2[25],ymm0[25],ymm2[26],ymm0[26],ymm2[27],ymm0[27],ymm2[28],ymm0[28],ymm2[29],ymm0[29],ymm2[30],ymm0[30],ymm2[31],ymm0[31]
+; AVX512VL-FALLBACK-NEXT:    vpmullw %ymm6, %ymm1, %ymm1
+; AVX512VL-FALLBACK-NEXT:    vpand %ymm7, %ymm1, %ymm1
+; AVX512VL-FALLBACK-NEXT:    vpackuswb %ymm4, %ymm1, %ymm1
+; AVX512VL-FALLBACK-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm3[8],ymm0[8],ymm3[9],ymm0[9],ymm3[10],ymm0[10],ymm3[11],ymm0[11],ymm3[12],ymm0[12],ymm3[13],ymm0[13],ymm3[14],ymm0[14],ymm3[15],ymm0[15],ymm3[24],ymm0[24],ymm3[25],ymm0[25],ymm3[26],ymm0[26],ymm3[27],ymm0[27],ymm3[28],ymm0[28],ymm3[29],ymm0[29],ymm3[30],ymm0[30],ymm3[31],ymm0[31]
 ; AVX512VL-FALLBACK-NEXT:    vpunpckhbw {{.*#+}} ymm6 = ymm5[8],ymm0[8],ymm5[9],ymm0[9],ymm5[10],ymm0[10],ymm5[11],ymm0[11],ymm5[12],ymm0[12],ymm5[13],ymm0[13],ymm5[14],ymm0[14],ymm5[15],ymm0[15],ymm5[24],ymm0[24],ymm5[25],ymm0[25],ymm5[26],ymm0[26],ymm5[27],ymm0[27],ymm5[28],ymm0[28],ymm5[29],ymm0[29],ymm5[30],ymm0[30],ymm5[31],ymm0[31]
 ; AVX512VL-FALLBACK-NEXT:    vpmullw %ymm6, %ymm4, %ymm4
 ; AVX512VL-FALLBACK-NEXT:    vpand %ymm7, %ymm4, %ymm4
-; AVX512VL-FALLBACK-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
+; AVX512VL-FALLBACK-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23]
 ; AVX512VL-FALLBACK-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm5[0],ymm0[0],ymm5[1],ymm0[1],ymm5[2],ymm0[2],ymm5[3],ymm0[3],ymm5[4],ymm0[4],ymm5[5],ymm0[5],ymm5[6],ymm0[6],ymm5[7],ymm0[7],ymm5[16],ymm0[16],ymm5[17],ymm0[17],ymm5[18],ymm0[18],ymm5[19],ymm0[19],ymm5[20],ymm0[20],ymm5[21],ymm0[21],ymm5[22],ymm0[22],ymm5[23],ymm0[23]
-; AVX512VL-FALLBACK-NEXT:    vpmullw %ymm5, %ymm2, %ymm2
-; AVX512VL-FALLBACK-NEXT:    vpand %ymm7, %ymm2, %ymm2
-; AVX512VL-FALLBACK-NEXT:    vpackuswb %ymm4, %ymm2, %ymm2
-; AVX512VL-FALLBACK-NEXT:    vpaddb %ymm0, %ymm2, %ymm0
-; AVX512VL-FALLBACK-NEXT:    vpaddb %ymm1, %ymm3, %ymm1
+; AVX512VL-FALLBACK-NEXT:    vpmullw %ymm5, %ymm3, %ymm3
+; AVX512VL-FALLBACK-NEXT:    vpand %ymm7, %ymm3, %ymm3
+; AVX512VL-FALLBACK-NEXT:    vpackuswb %ymm4, %ymm3, %ymm3
+; AVX512VL-FALLBACK-NEXT:    vpaddb %ymm2, %ymm3, %ymm2
+; AVX512VL-FALLBACK-NEXT:    vpaddb %ymm0, %ymm1, %ymm0
+; AVX512VL-FALLBACK-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512VL-FALLBACK-NEXT:    retq
 ;
 ; AVX512BW-LABEL: vec512_i8_unsigned_reg_reg:
@@ -926,88 +960,92 @@ define <64 x i8> @vec512_i8_unsigned_reg_reg(<64 x i8> %a1, <64 x i8> %a2) nounw
 define <64 x i8> @vec512_i8_signed_mem_reg(<64 x i8>* %a1_addr, <64 x i8> %a2) nounwind {
 ; AVX512F-LABEL: vec512_i8_signed_mem_reg:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa (%rdi), %ymm2
-; AVX512F-NEXT:    vmovdqa 32(%rdi), %ymm3
-; AVX512F-NEXT:    vpcmpgtb %ymm0, %ymm2, %ymm4
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
+; AVX512F-NEXT:    vmovdqa (%rdi), %ymm1
+; AVX512F-NEXT:    vmovdqa 32(%rdi), %ymm2
+; AVX512F-NEXT:    vpcmpgtb %ymm3, %ymm2, %ymm4
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm5 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
 ; AVX512F-NEXT:    vpor %ymm5, %ymm4, %ymm4
-; AVX512F-NEXT:    vpcmpgtb %ymm1, %ymm3, %ymm6
+; AVX512F-NEXT:    vpcmpgtb %ymm0, %ymm1, %ymm6
 ; AVX512F-NEXT:    vpor %ymm5, %ymm6, %ymm5
-; AVX512F-NEXT:    vpminsb %ymm0, %ymm2, %ymm6
-; AVX512F-NEXT:    vpminsb %ymm1, %ymm3, %ymm7
-; AVX512F-NEXT:    vpmaxsb %ymm0, %ymm2, %ymm0
-; AVX512F-NEXT:    vpsubb %ymm6, %ymm0, %ymm0
-; AVX512F-NEXT:    vpmaxsb %ymm1, %ymm3, %ymm1
-; AVX512F-NEXT:    vpsubb %ymm7, %ymm1, %ymm1
-; AVX512F-NEXT:    vpsrlw $1, %ymm0, %ymm0
+; AVX512F-NEXT:    vpminsb %ymm3, %ymm2, %ymm6
+; AVX512F-NEXT:    vpminsb %ymm0, %ymm1, %ymm7
+; AVX512F-NEXT:    vpmaxsb %ymm3, %ymm2, %ymm3
+; AVX512F-NEXT:    vpsubb %ymm6, %ymm3, %ymm3
+; AVX512F-NEXT:    vpmaxsb %ymm0, %ymm1, %ymm0
+; AVX512F-NEXT:    vpsubb %ymm7, %ymm0, %ymm0
+; AVX512F-NEXT:    vpsrlw $1, %ymm3, %ymm3
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm6 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
+; AVX512F-NEXT:    vpand %ymm6, %ymm3, %ymm3
+; AVX512F-NEXT:    vpsrlw $1, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpand %ymm6, %ymm0, %ymm0
-; AVX512F-NEXT:    vpsrlw $1, %ymm1, %ymm1
-; AVX512F-NEXT:    vpand %ymm6, %ymm1, %ymm1
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm6 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31]
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm6 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
 ; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm7 = ymm5[8],ymm0[8],ymm5[9],ymm0[9],ymm5[10],ymm0[10],ymm5[11],ymm0[11],ymm5[12],ymm0[12],ymm5[13],ymm0[13],ymm5[14],ymm0[14],ymm5[15],ymm0[15],ymm5[24],ymm0[24],ymm5[25],ymm0[25],ymm5[26],ymm0[26],ymm5[27],ymm0[27],ymm5[28],ymm0[28],ymm5[29],ymm0[29],ymm5[30],ymm0[30],ymm5[31],ymm0[31]
 ; AVX512F-NEXT:    vpmullw %ymm7, %ymm6, %ymm6
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm7 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
 ; AVX512F-NEXT:    vpand %ymm7, %ymm6, %ymm6
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
 ; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm5[0],ymm0[0],ymm5[1],ymm0[1],ymm5[2],ymm0[2],ymm5[3],ymm0[3],ymm5[4],ymm0[4],ymm5[5],ymm0[5],ymm5[6],ymm0[6],ymm5[7],ymm0[7],ymm5[16],ymm0[16],ymm5[17],ymm0[17],ymm5[18],ymm0[18],ymm5[19],ymm0[19],ymm5[20],ymm0[20],ymm5[21],ymm0[21],ymm5[22],ymm0[22],ymm5[23],ymm0[23]
-; AVX512F-NEXT:    vpmullw %ymm5, %ymm1, %ymm1
-; AVX512F-NEXT:    vpand %ymm7, %ymm1, %ymm1
-; AVX512F-NEXT:    vpackuswb %ymm6, %ymm1, %ymm1
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX512F-NEXT:    vpmullw %ymm5, %ymm0, %ymm0
+; AVX512F-NEXT:    vpand %ymm7, %ymm0, %ymm0
+; AVX512F-NEXT:    vpackuswb %ymm6, %ymm0, %ymm0
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm3[8],ymm0[8],ymm3[9],ymm0[9],ymm3[10],ymm0[10],ymm3[11],ymm0[11],ymm3[12],ymm0[12],ymm3[13],ymm0[13],ymm3[14],ymm0[14],ymm3[15],ymm0[15],ymm3[24],ymm0[24],ymm3[25],ymm0[25],ymm3[26],ymm0[26],ymm3[27],ymm0[27],ymm3[28],ymm0[28],ymm3[29],ymm0[29],ymm3[30],ymm0[30],ymm3[31],ymm0[31]
 ; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm6 = ymm4[8],ymm0[8],ymm4[9],ymm0[9],ymm4[10],ymm0[10],ymm4[11],ymm0[11],ymm4[12],ymm0[12],ymm4[13],ymm0[13],ymm4[14],ymm0[14],ymm4[15],ymm0[15],ymm4[24],ymm0[24],ymm4[25],ymm0[25],ymm4[26],ymm0[26],ymm4[27],ymm0[27],ymm4[28],ymm0[28],ymm4[29],ymm0[29],ymm4[30],ymm0[30],ymm4[31],ymm0[31]
 ; AVX512F-NEXT:    vpmullw %ymm6, %ymm5, %ymm5
 ; AVX512F-NEXT:    vpand %ymm7, %ymm5, %ymm5
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23]
 ; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm4[0],ymm0[0],ymm4[1],ymm0[1],ymm4[2],ymm0[2],ymm4[3],ymm0[3],ymm4[4],ymm0[4],ymm4[5],ymm0[5],ymm4[6],ymm0[6],ymm4[7],ymm0[7],ymm4[16],ymm0[16],ymm4[17],ymm0[17],ymm4[18],ymm0[18],ymm4[19],ymm0[19],ymm4[20],ymm0[20],ymm4[21],ymm0[21],ymm4[22],ymm0[22],ymm4[23],ymm0[23]
-; AVX512F-NEXT:    vpmullw %ymm4, %ymm0, %ymm0
-; AVX512F-NEXT:    vpand %ymm7, %ymm0, %ymm0
-; AVX512F-NEXT:    vpackuswb %ymm5, %ymm0, %ymm0
-; AVX512F-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
-; AVX512F-NEXT:    vpaddb %ymm3, %ymm1, %ymm1
+; AVX512F-NEXT:    vpmullw %ymm4, %ymm3, %ymm3
+; AVX512F-NEXT:    vpand %ymm7, %ymm3, %ymm3
+; AVX512F-NEXT:    vpackuswb %ymm5, %ymm3, %ymm3
+; AVX512F-NEXT:    vpaddb %ymm2, %ymm3, %ymm2
+; AVX512F-NEXT:    vpaddb %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-FALLBACK-LABEL: vec512_i8_signed_mem_reg:
 ; AVX512VL-FALLBACK:       # %bb.0:
-; AVX512VL-FALLBACK-NEXT:    vmovdqa (%rdi), %ymm2
-; AVX512VL-FALLBACK-NEXT:    vmovdqa 32(%rdi), %ymm3
-; AVX512VL-FALLBACK-NEXT:    vpcmpgtb %ymm0, %ymm2, %ymm4
+; AVX512VL-FALLBACK-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
+; AVX512VL-FALLBACK-NEXT:    vmovdqa (%rdi), %ymm1
+; AVX512VL-FALLBACK-NEXT:    vmovdqa 32(%rdi), %ymm2
+; AVX512VL-FALLBACK-NEXT:    vpcmpgtb %ymm3, %ymm2, %ymm4
 ; AVX512VL-FALLBACK-NEXT:    vmovdqa {{.*#+}} ymm5 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
 ; AVX512VL-FALLBACK-NEXT:    vpor %ymm5, %ymm4, %ymm4
-; AVX512VL-FALLBACK-NEXT:    vpcmpgtb %ymm1, %ymm3, %ymm6
+; AVX512VL-FALLBACK-NEXT:    vpcmpgtb %ymm0, %ymm1, %ymm6
 ; AVX512VL-FALLBACK-NEXT:    vpor %ymm5, %ymm6, %ymm5
-; AVX512VL-FALLBACK-NEXT:    vpminsb %ymm0, %ymm2, %ymm6
-; AVX512VL-FALLBACK-NEXT:    vpminsb %ymm1, %ymm3, %ymm7
-; AVX512VL-FALLBACK-NEXT:    vpmaxsb %ymm0, %ymm2, %ymm0
-; AVX512VL-FALLBACK-NEXT:    vpsubb %ymm6, %ymm0, %ymm0
-; AVX512VL-FALLBACK-NEXT:    vpmaxsb %ymm1, %ymm3, %ymm1
-; AVX512VL-FALLBACK-NEXT:    vpsubb %ymm7, %ymm1, %ymm1
-; AVX512VL-FALLBACK-NEXT:    vpsrlw $1, %ymm0, %ymm0
+; AVX512VL-FALLBACK-NEXT:    vpminsb %ymm3, %ymm2, %ymm6
+; AVX512VL-FALLBACK-NEXT:    vpminsb %ymm0, %ymm1, %ymm7
+; AVX512VL-FALLBACK-NEXT:    vpmaxsb %ymm3, %ymm2, %ymm3
+; AVX512VL-FALLBACK-NEXT:    vpsubb %ymm6, %ymm3, %ymm3
+; AVX512VL-FALLBACK-NEXT:    vpmaxsb %ymm0, %ymm1, %ymm0
+; AVX512VL-FALLBACK-NEXT:    vpsubb %ymm7, %ymm0, %ymm0
+; AVX512VL-FALLBACK-NEXT:    vpsrlw $1, %ymm3, %ymm3
 ; AVX512VL-FALLBACK-NEXT:    vmovdqa {{.*#+}} ymm6 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
+; AVX512VL-FALLBACK-NEXT:    vpand %ymm6, %ymm3, %ymm3
+; AVX512VL-FALLBACK-NEXT:    vpsrlw $1, %ymm0, %ymm0
 ; AVX512VL-FALLBACK-NEXT:    vpand %ymm6, %ymm0, %ymm0
-; AVX512VL-FALLBACK-NEXT:    vpsrlw $1, %ymm1, %ymm1
-; AVX512VL-FALLBACK-NEXT:    vpand %ymm6, %ymm1, %ymm1
-; AVX512VL-FALLBACK-NEXT:    vpunpckhbw {{.*#+}} ymm6 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31]
+; AVX512VL-FALLBACK-NEXT:    vpunpckhbw {{.*#+}} ymm6 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
 ; AVX512VL-FALLBACK-NEXT:    vpunpckhbw {{.*#+}} ymm7 = ymm5[8],ymm0[8],ymm5[9],ymm0[9],ymm5[10],ymm0[10],ymm5[11],ymm0[11],ymm5[12],ymm0[12],ymm5[13],ymm0[13],ymm5[14],ymm0[14],ymm5[15],ymm0[15],ymm5[24],ymm0[24],ymm5[25],ymm0[25],ymm5[26],ymm0[26],ymm5[27],ymm0[27],ymm5[28],ymm0[28],ymm5[29],ymm0[29],ymm5[30],ymm0[30],ymm5[31],ymm0[31]
 ; AVX512VL-FALLBACK-NEXT:    vpmullw %ymm7, %ymm6, %ymm6
 ; AVX512VL-FALLBACK-NEXT:    vmovdqa {{.*#+}} ymm7 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
 ; AVX512VL-FALLBACK-NEXT:    vpand %ymm7, %ymm6, %ymm6
-; AVX512VL-FALLBACK-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
+; AVX512VL-FALLBACK-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
 ; AVX512VL-FALLBACK-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm5[0],ymm0[0],ymm5[1],ymm0[1],ymm5[2],ymm0[2],ymm5[3],ymm0[3],ymm5[4],ymm0[4],ymm5[5],ymm0[5],ymm5[6],ymm0[6],ymm5[7],ymm0[7],ymm5[16],ymm0[16],ymm5[17],ymm0[17],ymm5[18],ymm0[18],ymm5[19],ymm0[19],ymm5[20],ymm0[20],ymm5[21],ymm0[21],ymm5[22],ymm0[22],ymm5[23],ymm0[23]
-; AVX512VL-FALLBACK-NEXT:    vpmullw %ymm5, %ymm1, %ymm1
-; AVX512VL-FALLBACK-NEXT:    vpand %ymm7, %ymm1, %ymm1
-; AVX512VL-FALLBACK-NEXT:    vpackuswb %ymm6, %ymm1, %ymm1
-; AVX512VL-FALLBACK-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX512VL-FALLBACK-NEXT:    vpmullw %ymm5, %ymm0, %ymm0
+; AVX512VL-FALLBACK-NEXT:    vpand %ymm7, %ymm0, %ymm0
+; AVX512VL-FALLBACK-NEXT:    vpackuswb %ymm6, %ymm0, %ymm0
+; AVX512VL-FALLBACK-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm3[8],ymm0[8],ymm3[9],ymm0[9],ymm3[10],ymm0[10],ymm3[11],ymm0[11],ymm3[12],ymm0[12],ymm3[13],ymm0[13],ymm3[14],ymm0[14],ymm3[15],ymm0[15],ymm3[24],ymm0[24],ymm3[25],ymm0[25],ymm3[26],ymm0[26],ymm3[27],ymm0[27],ymm3[28],ymm0[28],ymm3[29],ymm0[29],ymm3[30],ymm0[30],ymm3[31],ymm0[31]
 ; AVX512VL-FALLBACK-NEXT:    vpunpckhbw {{.*#+}} ymm6 = ymm4[8],ymm0[8],ymm4[9],ymm0[9],ymm4[10],ymm0[10],ymm4[11],ymm0[11],ymm4[12],ymm0[12],ymm4[13],ymm0[13],ymm4[14],ymm0[14],ymm4[15],ymm0[15],ymm4[24],ymm0[24],ymm4[25],ymm0[25],ymm4[26],ymm0[26],ymm4[27],ymm0[27],ymm4[28],ymm0[28],ymm4[29],ymm0[29],ymm4[30],ymm0[30],ymm4[31],ymm0[31]
 ; AVX512VL-FALLBACK-NEXT:    vpmullw %ymm6, %ymm5, %ymm5
 ; AVX512VL-FALLBACK-NEXT:    vpand %ymm7, %ymm5, %ymm5
-; AVX512VL-FALLBACK-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512VL-FALLBACK-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23]
 ; AVX512VL-FALLBACK-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm4[0],ymm0[0],ymm4[1],ymm0[1],ymm4[2],ymm0[2],ymm4[3],ymm0[3],ymm4[4],ymm0[4],ymm4[5],ymm0[5],ymm4[6],ymm0[6],ymm4[7],ymm0[7],ymm4[16],ymm0[16],ymm4[17],ymm0[17],ymm4[18],ymm0[18],ymm4[19],ymm0[19],ymm4[20],ymm0[20],ymm4[21],ymm0[21],ymm4[22],ymm0[22],ymm4[23],ymm0[23]
-; AVX512VL-FALLBACK-NEXT:    vpmullw %ymm4, %ymm0, %ymm0
-; AVX512VL-FALLBACK-NEXT:    vpand %ymm7, %ymm0, %ymm0
-; AVX512VL-FALLBACK-NEXT:    vpackuswb %ymm5, %ymm0, %ymm0
-; AVX512VL-FALLBACK-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
-; AVX512VL-FALLBACK-NEXT:    vpaddb %ymm3, %ymm1, %ymm1
+; AVX512VL-FALLBACK-NEXT:    vpmullw %ymm4, %ymm3, %ymm3
+; AVX512VL-FALLBACK-NEXT:    vpand %ymm7, %ymm3, %ymm3
+; AVX512VL-FALLBACK-NEXT:    vpackuswb %ymm5, %ymm3, %ymm3
+; AVX512VL-FALLBACK-NEXT:    vpaddb %ymm2, %ymm3, %ymm2
+; AVX512VL-FALLBACK-NEXT:    vpaddb %ymm1, %ymm0, %ymm0
+; AVX512VL-FALLBACK-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512VL-FALLBACK-NEXT:    retq
 ;
 ; AVX512BW-LABEL: vec512_i8_signed_mem_reg:
@@ -1049,88 +1087,92 @@ define <64 x i8> @vec512_i8_signed_mem_reg(<64 x i8>* %a1_addr, <64 x i8> %a2) n
 define <64 x i8> @vec512_i8_signed_reg_mem(<64 x i8> %a1, <64 x i8>* %a2_addr) nounwind {
 ; AVX512F-LABEL: vec512_i8_signed_reg_mem:
 ; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512F-NEXT:    vmovdqa (%rdi), %ymm2
 ; AVX512F-NEXT:    vmovdqa 32(%rdi), %ymm3
-; AVX512F-NEXT:    vpcmpgtb %ymm2, %ymm0, %ymm4
+; AVX512F-NEXT:    vpcmpgtb %ymm3, %ymm1, %ymm4
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm5 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
 ; AVX512F-NEXT:    vpor %ymm5, %ymm4, %ymm4
-; AVX512F-NEXT:    vpcmpgtb %ymm3, %ymm1, %ymm6
+; AVX512F-NEXT:    vpcmpgtb %ymm2, %ymm0, %ymm6
 ; AVX512F-NEXT:    vpor %ymm5, %ymm6, %ymm5
-; AVX512F-NEXT:    vpminsb %ymm2, %ymm0, %ymm6
-; AVX512F-NEXT:    vpminsb %ymm3, %ymm1, %ymm7
-; AVX512F-NEXT:    vpmaxsb %ymm2, %ymm0, %ymm2
-; AVX512F-NEXT:    vpsubb %ymm6, %ymm2, %ymm2
+; AVX512F-NEXT:    vpminsb %ymm3, %ymm1, %ymm6
+; AVX512F-NEXT:    vpminsb %ymm2, %ymm0, %ymm7
 ; AVX512F-NEXT:    vpmaxsb %ymm3, %ymm1, %ymm3
-; AVX512F-NEXT:    vpsubb %ymm7, %ymm3, %ymm3
-; AVX512F-NEXT:    vpsrlw $1, %ymm2, %ymm2
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm6 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; AVX512F-NEXT:    vpand %ymm6, %ymm2, %ymm2
+; AVX512F-NEXT:    vpsubb %ymm6, %ymm3, %ymm3
+; AVX512F-NEXT:    vpmaxsb %ymm2, %ymm0, %ymm2
+; AVX512F-NEXT:    vpsubb %ymm7, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpsrlw $1, %ymm3, %ymm3
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm6 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
 ; AVX512F-NEXT:    vpand %ymm6, %ymm3, %ymm3
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm6 = ymm3[8],ymm0[8],ymm3[9],ymm0[9],ymm3[10],ymm0[10],ymm3[11],ymm0[11],ymm3[12],ymm0[12],ymm3[13],ymm0[13],ymm3[14],ymm0[14],ymm3[15],ymm0[15],ymm3[24],ymm0[24],ymm3[25],ymm0[25],ymm3[26],ymm0[26],ymm3[27],ymm0[27],ymm3[28],ymm0[28],ymm3[29],ymm0[29],ymm3[30],ymm0[30],ymm3[31],ymm0[31]
+; AVX512F-NEXT:    vpsrlw $1, %ymm2, %ymm2
+; AVX512F-NEXT:    vpand %ymm6, %ymm2, %ymm2
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm6 = ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11],ymm2[12],ymm0[12],ymm2[13],ymm0[13],ymm2[14],ymm0[14],ymm2[15],ymm0[15],ymm2[24],ymm0[24],ymm2[25],ymm0[25],ymm2[26],ymm0[26],ymm2[27],ymm0[27],ymm2[28],ymm0[28],ymm2[29],ymm0[29],ymm2[30],ymm0[30],ymm2[31],ymm0[31]
 ; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm7 = ymm5[8],ymm0[8],ymm5[9],ymm0[9],ymm5[10],ymm0[10],ymm5[11],ymm0[11],ymm5[12],ymm0[12],ymm5[13],ymm0[13],ymm5[14],ymm0[14],ymm5[15],ymm0[15],ymm5[24],ymm0[24],ymm5[25],ymm0[25],ymm5[26],ymm0[26],ymm5[27],ymm0[27],ymm5[28],ymm0[28],ymm5[29],ymm0[29],ymm5[30],ymm0[30],ymm5[31],ymm0[31]
 ; AVX512F-NEXT:    vpmullw %ymm7, %ymm6, %ymm6
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm7 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
 ; AVX512F-NEXT:    vpand %ymm7, %ymm6, %ymm6
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23]
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
 ; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm5[0],ymm0[0],ymm5[1],ymm0[1],ymm5[2],ymm0[2],ymm5[3],ymm0[3],ymm5[4],ymm0[4],ymm5[5],ymm0[5],ymm5[6],ymm0[6],ymm5[7],ymm0[7],ymm5[16],ymm0[16],ymm5[17],ymm0[17],ymm5[18],ymm0[18],ymm5[19],ymm0[19],ymm5[20],ymm0[20],ymm5[21],ymm0[21],ymm5[22],ymm0[22],ymm5[23],ymm0[23]
-; AVX512F-NEXT:    vpmullw %ymm5, %ymm3, %ymm3
-; AVX512F-NEXT:    vpand %ymm7, %ymm3, %ymm3
-; AVX512F-NEXT:    vpackuswb %ymm6, %ymm3, %ymm3
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11],ymm2[12],ymm0[12],ymm2[13],ymm0[13],ymm2[14],ymm0[14],ymm2[15],ymm0[15],ymm2[24],ymm0[24],ymm2[25],ymm0[25],ymm2[26],ymm0[26],ymm2[27],ymm0[27],ymm2[28],ymm0[28],ymm2[29],ymm0[29],ymm2[30],ymm0[30],ymm2[31],ymm0[31]
+; AVX512F-NEXT:    vpmullw %ymm5, %ymm2, %ymm2
+; AVX512F-NEXT:    vpand %ymm7, %ymm2, %ymm2
+; AVX512F-NEXT:    vpackuswb %ymm6, %ymm2, %ymm2
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm3[8],ymm0[8],ymm3[9],ymm0[9],ymm3[10],ymm0[10],ymm3[11],ymm0[11],ymm3[12],ymm0[12],ymm3[13],ymm0[13],ymm3[14],ymm0[14],ymm3[15],ymm0[15],ymm3[24],ymm0[24],ymm3[25],ymm0[25],ymm3[26],ymm0[26],ymm3[27],ymm0[27],ymm3[28],ymm0[28],ymm3[29],ymm0[29],ymm3[30],ymm0[30],ymm3[31],ymm0[31]
 ; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm6 = ymm4[8],ymm0[8],ymm4[9],ymm0[9],ymm4[10],ymm0[10],ymm4[11],ymm0[11],ymm4[12],ymm0[12],ymm4[13],ymm0[13],ymm4[14],ymm0[14],ymm4[15],ymm0[15],ymm4[24],ymm0[24],ymm4[25],ymm0[25],ymm4[26],ymm0[26],ymm4[27],ymm0[27],ymm4[28],ymm0[28],ymm4[29],ymm0[29],ymm4[30],ymm0[30],ymm4[31],ymm0[31]
 ; AVX512F-NEXT:    vpmullw %ymm6, %ymm5, %ymm5
 ; AVX512F-NEXT:    vpand %ymm7, %ymm5, %ymm5
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23]
 ; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm4[0],ymm0[0],ymm4[1],ymm0[1],ymm4[2],ymm0[2],ymm4[3],ymm0[3],ymm4[4],ymm0[4],ymm4[5],ymm0[5],ymm4[6],ymm0[6],ymm4[7],ymm0[7],ymm4[16],ymm0[16],ymm4[17],ymm0[17],ymm4[18],ymm0[18],ymm4[19],ymm0[19],ymm4[20],ymm0[20],ymm4[21],ymm0[21],ymm4[22],ymm0[22],ymm4[23],ymm0[23]
-; AVX512F-NEXT:    vpmullw %ymm4, %ymm2, %ymm2
-; AVX512F-NEXT:    vpand %ymm7, %ymm2, %ymm2
-; AVX512F-NEXT:    vpackuswb %ymm5, %ymm2, %ymm2
-; AVX512F-NEXT:    vpaddb %ymm0, %ymm2, %ymm0
+; AVX512F-NEXT:    vpmullw %ymm4, %ymm3, %ymm3
+; AVX512F-NEXT:    vpand %ymm7, %ymm3, %ymm3
+; AVX512F-NEXT:    vpackuswb %ymm5, %ymm3, %ymm3
 ; AVX512F-NEXT:    vpaddb %ymm1, %ymm3, %ymm1
+; AVX512F-NEXT:    vpaddb %ymm0, %ymm2, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-FALLBACK-LABEL: vec512_i8_signed_reg_mem:
 ; AVX512VL-FALLBACK:       # %bb.0:
+; AVX512VL-FALLBACK-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512VL-FALLBACK-NEXT:    vmovdqa (%rdi), %ymm2
 ; AVX512VL-FALLBACK-NEXT:    vmovdqa 32(%rdi), %ymm3
-; AVX512VL-FALLBACK-NEXT:    vpcmpgtb %ymm2, %ymm0, %ymm4
+; AVX512VL-FALLBACK-NEXT:    vpcmpgtb %ymm3, %ymm1, %ymm4
 ; AVX512VL-FALLBACK-NEXT:    vmovdqa {{.*#+}} ymm5 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
 ; AVX512VL-FALLBACK-NEXT:    vpor %ymm5, %ymm4, %ymm4
-; AVX512VL-FALLBACK-NEXT:    vpcmpgtb %ymm3, %ymm1, %ymm6
+; AVX512VL-FALLBACK-NEXT:    vpcmpgtb %ymm2, %ymm0, %ymm6
 ; AVX512VL-FALLBACK-NEXT:    vpor %ymm5, %ymm6, %ymm5
-; AVX512VL-FALLBACK-NEXT:    vpminsb %ymm2, %ymm0, %ymm6
-; AVX512VL-FALLBACK-NEXT:    vpminsb %ymm3, %ymm1, %ymm7
-; AVX512VL-FALLBACK-NEXT:    vpmaxsb %ymm2, %ymm0, %ymm2
-; AVX512VL-FALLBACK-NEXT:    vpsubb %ymm6, %ymm2, %ymm2
+; AVX512VL-FALLBACK-NEXT:    vpminsb %ymm3, %ymm1, %ymm6
+; AVX512VL-FALLBACK-NEXT:    vpminsb %ymm2, %ymm0, %ymm7
 ; AVX512VL-FALLBACK-NEXT:    vpmaxsb %ymm3, %ymm1, %ymm3
-; AVX512VL-FALLBACK-NEXT:    vpsubb %ymm7, %ymm3, %ymm3
-; AVX512VL-FALLBACK-NEXT:    vpsrlw $1, %ymm2, %ymm2
-; AVX512VL-FALLBACK-NEXT:    vmovdqa {{.*#+}} ymm6 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; AVX512VL-FALLBACK-NEXT:    vpand %ymm6, %ymm2, %ymm2
+; AVX512VL-FALLBACK-NEXT:    vpsubb %ymm6, %ymm3, %ymm3
+; AVX512VL-FALLBACK-NEXT:    vpmaxsb %ymm2, %ymm0, %ymm2
+; AVX512VL-FALLBACK-NEXT:    vpsubb %ymm7, %ymm2, %ymm2
 ; AVX512VL-FALLBACK-NEXT:    vpsrlw $1, %ymm3, %ymm3
+; AVX512VL-FALLBACK-NEXT:    vmovdqa {{.*#+}} ymm6 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
 ; AVX512VL-FALLBACK-NEXT:    vpand %ymm6, %ymm3, %ymm3
-; AVX512VL-FALLBACK-NEXT:    vpunpckhbw {{.*#+}} ymm6 = ymm3[8],ymm0[8],ymm3[9],ymm0[9],ymm3[10],ymm0[10],ymm3[11],ymm0[11],ymm3[12],ymm0[12],ymm3[13],ymm0[13],ymm3[14],ymm0[14],ymm3[15],ymm0[15],ymm3[24],ymm0[24],ymm3[25],ymm0[25],ymm3[26],ymm0[26],ymm3[27],ymm0[27],ymm3[28],ymm0[28],ymm3[29],ymm0[29],ymm3[30],ymm0[30],ymm3[31],ymm0[31]
+; AVX512VL-FALLBACK-NEXT:    vpsrlw $1, %ymm2, %ymm2
+; AVX512VL-FALLBACK-NEXT:    vpand %ymm6, %ymm2, %ymm2
+; AVX512VL-FALLBACK-NEXT:    vpunpckhbw {{.*#+}} ymm6 = ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11],ymm2[12],ymm0[12],ymm2[13],ymm0[13],ymm2[14],ymm0[14],ymm2[15],ymm0[15],ymm2[24],ymm0[24],ymm2[25],ymm0[25],ymm2[26],ymm0[26],ymm2[27],ymm0[27],ymm2[28],ymm0[28],ymm2[29],ymm0[29],ymm2[30],ymm0[30],ymm2[31],ymm0[31]
 ; AVX512VL-FALLBACK-NEXT:    vpunpckhbw {{.*#+}} ymm7 = ymm5[8],ymm0[8],ymm5[9],ymm0[9],ymm5[10],ymm0[10],ymm5[11],ymm0[11],ymm5[12],ymm0[12],ymm5[13],ymm0[13],ymm5[14],ymm0[14],ymm5[15],ymm0[15],ymm5[24],ymm0[24],ymm5[25],ymm0[25],ymm5[26],ymm0[26],ymm5[27],ymm0[27],ymm5[28],ymm0[28],ymm5[29],ymm0[29],ymm5[30],ymm0[30],ymm5[31],ymm0[31]
 ; AVX512VL-FALLBACK-NEXT:    vpmullw %ymm7, %ymm6, %ymm6
 ; AVX512VL-FALLBACK-NEXT:    vmovdqa {{.*#+}} ymm7 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
 ; AVX512VL-FALLBACK-NEXT:    vpand %ymm7, %ymm6, %ymm6
-; AVX512VL-FALLBACK-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23]
+; AVX512VL-FALLBACK-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
 ; AVX512VL-FALLBACK-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm5[0],ymm0[0],ymm5[1],ymm0[1],ymm5[2],ymm0[2],ymm5[3],ymm0[3],ymm5[4],ymm0[4],ymm5[5],ymm0[5],ymm5[6],ymm0[6],ymm5[7],ymm0[7],ymm5[16],ymm0[16],ymm5[17],ymm0[17],ymm5[18],ymm0[18],ymm5[19],ymm0[19],ymm5[20],ymm0[20],ymm5[21],ymm0[21],ymm5[22],ymm0[22],ymm5[23],ymm0[23]
-; AVX512VL-FALLBACK-NEXT:    vpmullw %ymm5, %ymm3, %ymm3
-; AVX512VL-FALLBACK-NEXT:    vpand %ymm7, %ymm3, %ymm3
-; AVX512VL-FALLBACK-NEXT:    vpackuswb %ymm6, %ymm3, %ymm3
-; AVX512VL-FALLBACK-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11],ymm2[12],ymm0[12],ymm2[13],ymm0[13],ymm2[14],ymm0[14],ymm2[15],ymm0[15],ymm2[24],ymm0[24],ymm2[25],ymm0[25],ymm2[26],ymm0[26],ymm2[27],ymm0[27],ymm2[28],ymm0[28],ymm2[29],ymm0[29],ymm2[30],ymm0[30],ymm2[31],ymm0[31]
+; AVX512VL-FALLBACK-NEXT:    vpmullw %ymm5, %ymm2, %ymm2
+; AVX512VL-FALLBACK-NEXT:    vpand %ymm7, %ymm2, %ymm2
+; AVX512VL-FALLBACK-NEXT:    vpackuswb %ymm6, %ymm2, %ymm2
+; AVX512VL-FALLBACK-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm3[8],ymm0[8],ymm3[9],ymm0[9],ymm3[10],ymm0[10],ymm3[11],ymm0[11],ymm3[12],ymm0[12],ymm3[13],ymm0[13],ymm3[14],ymm0[14],ymm3[15],ymm0[15],ymm3[24],ymm0[24],ymm3[25],ymm0[25],ymm3[26],ymm0[26],ymm3[27],ymm0[27],ymm3[28],ymm0[28],ymm3[29],ymm0[29],ymm3[30],ymm0[30],ymm3[31],ymm0[31]
 ; AVX512VL-FALLBACK-NEXT:    vpunpckhbw {{.*#+}} ymm6 = ymm4[8],ymm0[8],ymm4[9],ymm0[9],ymm4[10],ymm0[10],ymm4[11],ymm0[11],ymm4[12],ymm0[12],ymm4[13],ymm0[13],ymm4[14],ymm0[14],ymm4[15],ymm0[15],ymm4[24],ymm0[24],ymm4[25],ymm0[25],ymm4[26],ymm0[26],ymm4[27],ymm0[27],ymm4[28],ymm0[28],ymm4[29],ymm0[29],ymm4[30],ymm0[30],ymm4[31],ymm0[31]
 ; AVX512VL-FALLBACK-NEXT:    vpmullw %ymm6, %ymm5, %ymm5
 ; AVX512VL-FALLBACK-NEXT:    vpand %ymm7, %ymm5, %ymm5
-; AVX512VL-FALLBACK-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
+; AVX512VL-FALLBACK-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23]
 ; AVX512VL-FALLBACK-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm4[0],ymm0[0],ymm4[1],ymm0[1],ymm4[2],ymm0[2],ymm4[3],ymm0[3],ymm4[4],ymm0[4],ymm4[5],ymm0[5],ymm4[6],ymm0[6],ymm4[7],ymm0[7],ymm4[16],ymm0[16],ymm4[17],ymm0[17],ymm4[18],ymm0[18],ymm4[19],ymm0[19],ymm4[20],ymm0[20],ymm4[21],ymm0[21],ymm4[22],ymm0[22],ymm4[23],ymm0[23]
-; AVX512VL-FALLBACK-NEXT:    vpmullw %ymm4, %ymm2, %ymm2
-; AVX512VL-FALLBACK-NEXT:    vpand %ymm7, %ymm2, %ymm2
-; AVX512VL-FALLBACK-NEXT:    vpackuswb %ymm5, %ymm2, %ymm2
-; AVX512VL-FALLBACK-NEXT:    vpaddb %ymm0, %ymm2, %ymm0
+; AVX512VL-FALLBACK-NEXT:    vpmullw %ymm4, %ymm3, %ymm3
+; AVX512VL-FALLBACK-NEXT:    vpand %ymm7, %ymm3, %ymm3
+; AVX512VL-FALLBACK-NEXT:    vpackuswb %ymm5, %ymm3, %ymm3
 ; AVX512VL-FALLBACK-NEXT:    vpaddb %ymm1, %ymm3, %ymm1
+; AVX512VL-FALLBACK-NEXT:    vpaddb %ymm0, %ymm2, %ymm0
+; AVX512VL-FALLBACK-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512VL-FALLBACK-NEXT:    retq
 ;
 ; AVX512BW-LABEL: vec512_i8_signed_reg_mem:
@@ -1176,43 +1218,44 @@ define <64 x i8> @vec512_i8_signed_mem_mem(<64 x i8>* %a1_addr, <64 x i8>* %a2_a
 ; AVX512F-NEXT:    vmovdqa 32(%rdi), %ymm1
 ; AVX512F-NEXT:    vmovdqa (%rsi), %ymm2
 ; AVX512F-NEXT:    vmovdqa 32(%rsi), %ymm3
-; AVX512F-NEXT:    vpcmpgtb %ymm2, %ymm0, %ymm4
+; AVX512F-NEXT:    vpcmpgtb %ymm3, %ymm1, %ymm4
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm5 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
 ; AVX512F-NEXT:    vpor %ymm5, %ymm4, %ymm4
-; AVX512F-NEXT:    vpcmpgtb %ymm3, %ymm1, %ymm6
+; AVX512F-NEXT:    vpcmpgtb %ymm2, %ymm0, %ymm6
 ; AVX512F-NEXT:    vpor %ymm5, %ymm6, %ymm5
-; AVX512F-NEXT:    vpminsb %ymm2, %ymm0, %ymm6
-; AVX512F-NEXT:    vpminsb %ymm3, %ymm1, %ymm7
-; AVX512F-NEXT:    vpmaxsb %ymm2, %ymm0, %ymm2
-; AVX512F-NEXT:    vpsubb %ymm6, %ymm2, %ymm2
+; AVX512F-NEXT:    vpminsb %ymm3, %ymm1, %ymm6
+; AVX512F-NEXT:    vpminsb %ymm2, %ymm0, %ymm7
 ; AVX512F-NEXT:    vpmaxsb %ymm3, %ymm1, %ymm3
-; AVX512F-NEXT:    vpsubb %ymm7, %ymm3, %ymm3
-; AVX512F-NEXT:    vpsrlw $1, %ymm2, %ymm2
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm6 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; AVX512F-NEXT:    vpand %ymm6, %ymm2, %ymm2
+; AVX512F-NEXT:    vpsubb %ymm6, %ymm3, %ymm3
+; AVX512F-NEXT:    vpmaxsb %ymm2, %ymm0, %ymm2
+; AVX512F-NEXT:    vpsubb %ymm7, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpsrlw $1, %ymm3, %ymm3
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm6 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
 ; AVX512F-NEXT:    vpand %ymm6, %ymm3, %ymm3
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm6 = ymm3[8],ymm0[8],ymm3[9],ymm0[9],ymm3[10],ymm0[10],ymm3[11],ymm0[11],ymm3[12],ymm0[12],ymm3[13],ymm0[13],ymm3[14],ymm0[14],ymm3[15],ymm0[15],ymm3[24],ymm0[24],ymm3[25],ymm0[25],ymm3[26],ymm0[26],ymm3[27],ymm0[27],ymm3[28],ymm0[28],ymm3[29],ymm0[29],ymm3[30],ymm0[30],ymm3[31],ymm0[31]
+; AVX512F-NEXT:    vpsrlw $1, %ymm2, %ymm2
+; AVX512F-NEXT:    vpand %ymm6, %ymm2, %ymm2
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm6 = ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11],ymm2[12],ymm0[12],ymm2[13],ymm0[13],ymm2[14],ymm0[14],ymm2[15],ymm0[15],ymm2[24],ymm0[24],ymm2[25],ymm0[25],ymm2[26],ymm0[26],ymm2[27],ymm0[27],ymm2[28],ymm0[28],ymm2[29],ymm0[29],ymm2[30],ymm0[30],ymm2[31],ymm0[31]
 ; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm7 = ymm5[8],ymm0[8],ymm5[9],ymm0[9],ymm5[10],ymm0[10],ymm5[11],ymm0[11],ymm5[12],ymm0[12],ymm5[13],ymm0[13],ymm5[14],ymm0[14],ymm5[15],ymm0[15],ymm5[24],ymm0[24],ymm5[25],ymm0[25],ymm5[26],ymm0[26],ymm5[27],ymm0[27],ymm5[28],ymm0[28],ymm5[29],ymm0[29],ymm5[30],ymm0[30],ymm5[31],ymm0[31]
 ; AVX512F-NEXT:    vpmullw %ymm7, %ymm6, %ymm6
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm7 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
 ; AVX512F-NEXT:    vpand %ymm7, %ymm6, %ymm6
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23]
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
 ; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm5[0],ymm0[0],ymm5[1],ymm0[1],ymm5[2],ymm0[2],ymm5[3],ymm0[3],ymm5[4],ymm0[4],ymm5[5],ymm0[5],ymm5[6],ymm0[6],ymm5[7],ymm0[7],ymm5[16],ymm0[16],ymm5[17],ymm0[17],ymm5[18],ymm0[18],ymm5[19],ymm0[19],ymm5[20],ymm0[20],ymm5[21],ymm0[21],ymm5[22],ymm0[22],ymm5[23],ymm0[23]
-; AVX512F-NEXT:    vpmullw %ymm5, %ymm3, %ymm3
-; AVX512F-NEXT:    vpand %ymm7, %ymm3, %ymm3
-; AVX512F-NEXT:    vpackuswb %ymm6, %ymm3, %ymm3
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11],ymm2[12],ymm0[12],ymm2[13],ymm0[13],ymm2[14],ymm0[14],ymm2[15],ymm0[15],ymm2[24],ymm0[24],ymm2[25],ymm0[25],ymm2[26],ymm0[26],ymm2[27],ymm0[27],ymm2[28],ymm0[28],ymm2[29],ymm0[29],ymm2[30],ymm0[30],ymm2[31],ymm0[31]
+; AVX512F-NEXT:    vpmullw %ymm5, %ymm2, %ymm2
+; AVX512F-NEXT:    vpand %ymm7, %ymm2, %ymm2
+; AVX512F-NEXT:    vpackuswb %ymm6, %ymm2, %ymm2
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm3[8],ymm0[8],ymm3[9],ymm0[9],ymm3[10],ymm0[10],ymm3[11],ymm0[11],ymm3[12],ymm0[12],ymm3[13],ymm0[13],ymm3[14],ymm0[14],ymm3[15],ymm0[15],ymm3[24],ymm0[24],ymm3[25],ymm0[25],ymm3[26],ymm0[26],ymm3[27],ymm0[27],ymm3[28],ymm0[28],ymm3[29],ymm0[29],ymm3[30],ymm0[30],ymm3[31],ymm0[31]
 ; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm6 = ymm4[8],ymm0[8],ymm4[9],ymm0[9],ymm4[10],ymm0[10],ymm4[11],ymm0[11],ymm4[12],ymm0[12],ymm4[13],ymm0[13],ymm4[14],ymm0[14],ymm4[15],ymm0[15],ymm4[24],ymm0[24],ymm4[25],ymm0[25],ymm4[26],ymm0[26],ymm4[27],ymm0[27],ymm4[28],ymm0[28],ymm4[29],ymm0[29],ymm4[30],ymm0[30],ymm4[31],ymm0[31]
 ; AVX512F-NEXT:    vpmullw %ymm6, %ymm5, %ymm5
 ; AVX512F-NEXT:    vpand %ymm7, %ymm5, %ymm5
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23]
 ; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm4[0],ymm0[0],ymm4[1],ymm0[1],ymm4[2],ymm0[2],ymm4[3],ymm0[3],ymm4[4],ymm0[4],ymm4[5],ymm0[5],ymm4[6],ymm0[6],ymm4[7],ymm0[7],ymm4[16],ymm0[16],ymm4[17],ymm0[17],ymm4[18],ymm0[18],ymm4[19],ymm0[19],ymm4[20],ymm0[20],ymm4[21],ymm0[21],ymm4[22],ymm0[22],ymm4[23],ymm0[23]
-; AVX512F-NEXT:    vpmullw %ymm4, %ymm2, %ymm2
-; AVX512F-NEXT:    vpand %ymm7, %ymm2, %ymm2
-; AVX512F-NEXT:    vpackuswb %ymm5, %ymm2, %ymm2
-; AVX512F-NEXT:    vpaddb %ymm0, %ymm2, %ymm0
+; AVX512F-NEXT:    vpmullw %ymm4, %ymm3, %ymm3
+; AVX512F-NEXT:    vpand %ymm7, %ymm3, %ymm3
+; AVX512F-NEXT:    vpackuswb %ymm5, %ymm3, %ymm3
 ; AVX512F-NEXT:    vpaddb %ymm1, %ymm3, %ymm1
+; AVX512F-NEXT:    vpaddb %ymm0, %ymm2, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-FALLBACK-LABEL: vec512_i8_signed_mem_mem:
@@ -1221,43 +1264,44 @@ define <64 x i8> @vec512_i8_signed_mem_mem(<64 x i8>* %a1_addr, <64 x i8>* %a2_a
 ; AVX512VL-FALLBACK-NEXT:    vmovdqa 32(%rdi), %ymm1
 ; AVX512VL-FALLBACK-NEXT:    vmovdqa (%rsi), %ymm2
 ; AVX512VL-FALLBACK-NEXT:    vmovdqa 32(%rsi), %ymm3
-; AVX512VL-FALLBACK-NEXT:    vpcmpgtb %ymm2, %ymm0, %ymm4
+; AVX512VL-FALLBACK-NEXT:    vpcmpgtb %ymm3, %ymm1, %ymm4
 ; AVX512VL-FALLBACK-NEXT:    vmovdqa {{.*#+}} ymm5 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
 ; AVX512VL-FALLBACK-NEXT:    vpor %ymm5, %ymm4, %ymm4
-; AVX512VL-FALLBACK-NEXT:    vpcmpgtb %ymm3, %ymm1, %ymm6
+; AVX512VL-FALLBACK-NEXT:    vpcmpgtb %ymm2, %ymm0, %ymm6
 ; AVX512VL-FALLBACK-NEXT:    vpor %ymm5, %ymm6, %ymm5
-; AVX512VL-FALLBACK-NEXT:    vpminsb %ymm2, %ymm0, %ymm6
-; AVX512VL-FALLBACK-NEXT:    vpminsb %ymm3, %ymm1, %ymm7
-; AVX512VL-FALLBACK-NEXT:    vpmaxsb %ymm2, %ymm0, %ymm2
-; AVX512VL-FALLBACK-NEXT:    vpsubb %ymm6, %ymm2, %ymm2
+; AVX512VL-FALLBACK-NEXT:    vpminsb %ymm3, %ymm1, %ymm6
+; AVX512VL-FALLBACK-NEXT:    vpminsb %ymm2, %ymm0, %ymm7
 ; AVX512VL-FALLBACK-NEXT:    vpmaxsb %ymm3, %ymm1, %ymm3
-; AVX512VL-FALLBACK-NEXT:    vpsubb %ymm7, %ymm3, %ymm3
-; AVX512VL-FALLBACK-NEXT:    vpsrlw $1, %ymm2, %ymm2
-; AVX512VL-FALLBACK-NEXT:    vmovdqa {{.*#+}} ymm6 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; AVX512VL-FALLBACK-NEXT:    vpand %ymm6, %ymm2, %ymm2
+; AVX512VL-FALLBACK-NEXT:    vpsubb %ymm6, %ymm3, %ymm3
+; AVX512VL-FALLBACK-NEXT:    vpmaxsb %ymm2, %ymm0, %ymm2
+; AVX512VL-FALLBACK-NEXT:    vpsubb %ymm7, %ymm2, %ymm2
 ; AVX512VL-FALLBACK-NEXT:    vpsrlw $1, %ymm3, %ymm3
+; AVX512VL-FALLBACK-NEXT:    vmovdqa {{.*#+}} ymm6 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
 ; AVX512VL-FALLBACK-NEXT:    vpand %ymm6, %ymm3, %ymm3
-; AVX512VL-FALLBACK-NEXT:    vpunpckhbw {{.*#+}} ymm6 = ymm3[8],ymm0[8],ymm3[9],ymm0[9],ymm3[10],ymm0[10],ymm3[11],ymm0[11],ymm3[12],ymm0[12],ymm3[13],ymm0[13],ymm3[14],ymm0[14],ymm3[15],ymm0[15],ymm3[24],ymm0[24],ymm3[25],ymm0[25],ymm3[26],ymm0[26],ymm3[27],ymm0[27],ymm3[28],ymm0[28],ymm3[29],ymm0[29],ymm3[30],ymm0[30],ymm3[31],ymm0[31]
+; AVX512VL-FALLBACK-NEXT:    vpsrlw $1, %ymm2, %ymm2
+; AVX512VL-FALLBACK-NEXT:    vpand %ymm6, %ymm2, %ymm2
+; AVX512VL-FALLBACK-NEXT:    vpunpckhbw {{.*#+}} ymm6 = ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11],ymm2[12],ymm0[12],ymm2[13],ymm0[13],ymm2[14],ymm0[14],ymm2[15],ymm0[15],ymm2[24],ymm0[24],ymm2[25],ymm0[25],ymm2[26],ymm0[26],ymm2[27],ymm0[27],ymm2[28],ymm0[28],ymm2[29],ymm0[29],ymm2[30],ymm0[30],ymm2[31],ymm0[31]
 ; AVX512VL-FALLBACK-NEXT:    vpunpckhbw {{.*#+}} ymm7 = ymm5[8],ymm0[8],ymm5[9],ymm0[9],ymm5[10],ymm0[10],ymm5[11],ymm0[11],ymm5[12],ymm0[12],ymm5[13],ymm0[13],ymm5[14],ymm0[14],ymm5[15],ymm0[15],ymm5[24],ymm0[24],ymm5[25],ymm0[25],ymm5[26],ymm0[26],ymm5[27],ymm0[27],ymm5[28],ymm0[28],ymm5[29],ymm0[29],ymm5[30],ymm0[30],ymm5[31],ymm0[31]
 ; AVX512VL-FALLBACK-NEXT:    vpmullw %ymm7, %ymm6, %ymm6
 ; AVX512VL-FALLBACK-NEXT:    vmovdqa {{.*#+}} ymm7 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
 ; AVX512VL-FALLBACK-NEXT:    vpand %ymm7, %ymm6, %ymm6
-; AVX512VL-FALLBACK-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23]
+; AVX512VL-FALLBACK-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
 ; AVX512VL-FALLBACK-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm5[0],ymm0[0],ymm5[1],ymm0[1],ymm5[2],ymm0[2],ymm5[3],ymm0[3],ymm5[4],ymm0[4],ymm5[5],ymm0[5],ymm5[6],ymm0[6],ymm5[7],ymm0[7],ymm5[16],ymm0[16],ymm5[17],ymm0[17],ymm5[18],ymm0[18],ymm5[19],ymm0[19],ymm5[20],ymm0[20],ymm5[21],ymm0[21],ymm5[22],ymm0[22],ymm5[23],ymm0[23]
-; AVX512VL-FALLBACK-NEXT:    vpmullw %ymm5, %ymm3, %ymm3
-; AVX512VL-FALLBACK-NEXT:    vpand %ymm7, %ymm3, %ymm3
-; AVX512VL-FALLBACK-NEXT:    vpackuswb %ymm6, %ymm3, %ymm3
-; AVX512VL-FALLBACK-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11],ymm2[12],ymm0[12],ymm2[13],ymm0[13],ymm2[14],ymm0[14],ymm2[15],ymm0[15],ymm2[24],ymm0[24],ymm2[25],ymm0[25],ymm2[26],ymm0[26],ymm2[27],ymm0[27],ymm2[28],ymm0[28],ymm2[29],ymm0[29],ymm2[30],ymm0[30],ymm2[31],ymm0[31]
+; AVX512VL-FALLBACK-NEXT:    vpmullw %ymm5, %ymm2, %ymm2
+; AVX512VL-FALLBACK-NEXT:    vpand %ymm7, %ymm2, %ymm2
+; AVX512VL-FALLBACK-NEXT:    vpackuswb %ymm6, %ymm2, %ymm2
+; AVX512VL-FALLBACK-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm3[8],ymm0[8],ymm3[9],ymm0[9],ymm3[10],ymm0[10],ymm3[11],ymm0[11],ymm3[12],ymm0[12],ymm3[13],ymm0[13],ymm3[14],ymm0[14],ymm3[15],ymm0[15],ymm3[24],ymm0[24],ymm3[25],ymm0[25],ymm3[26],ymm0[26],ymm3[27],ymm0[27],ymm3[28],ymm0[28],ymm3[29],ymm0[29],ymm3[30],ymm0[30],ymm3[31],ymm0[31]
 ; AVX512VL-FALLBACK-NEXT:    vpunpckhbw {{.*#+}} ymm6 = ymm4[8],ymm0[8],ymm4[9],ymm0[9],ymm4[10],ymm0[10],ymm4[11],ymm0[11],ymm4[12],ymm0[12],ymm4[13],ymm0[13],ymm4[14],ymm0[14],ymm4[15],ymm0[15],ymm4[24],ymm0[24],ymm4[25],ymm0[25],ymm4[26],ymm0[26],ymm4[27],ymm0[27],ymm4[28],ymm0[28],ymm4[29],ymm0[29],ymm4[30],ymm0[30],ymm4[31],ymm0[31]
 ; AVX512VL-FALLBACK-NEXT:    vpmullw %ymm6, %ymm5, %ymm5
 ; AVX512VL-FALLBACK-NEXT:    vpand %ymm7, %ymm5, %ymm5
-; AVX512VL-FALLBACK-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
+; AVX512VL-FALLBACK-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23]
 ; AVX512VL-FALLBACK-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm4[0],ymm0[0],ymm4[1],ymm0[1],ymm4[2],ymm0[2],ymm4[3],ymm0[3],ymm4[4],ymm0[4],ymm4[5],ymm0[5],ymm4[6],ymm0[6],ymm4[7],ymm0[7],ymm4[16],ymm0[16],ymm4[17],ymm0[17],ymm4[18],ymm0[18],ymm4[19],ymm0[19],ymm4[20],ymm0[20],ymm4[21],ymm0[21],ymm4[22],ymm0[22],ymm4[23],ymm0[23]
-; AVX512VL-FALLBACK-NEXT:    vpmullw %ymm4, %ymm2, %ymm2
-; AVX512VL-FALLBACK-NEXT:    vpand %ymm7, %ymm2, %ymm2
-; AVX512VL-FALLBACK-NEXT:    vpackuswb %ymm5, %ymm2, %ymm2
-; AVX512VL-FALLBACK-NEXT:    vpaddb %ymm0, %ymm2, %ymm0
+; AVX512VL-FALLBACK-NEXT:    vpmullw %ymm4, %ymm3, %ymm3
+; AVX512VL-FALLBACK-NEXT:    vpand %ymm7, %ymm3, %ymm3
+; AVX512VL-FALLBACK-NEXT:    vpackuswb %ymm5, %ymm3, %ymm3
 ; AVX512VL-FALLBACK-NEXT:    vpaddb %ymm1, %ymm3, %ymm1
+; AVX512VL-FALLBACK-NEXT:    vpaddb %ymm0, %ymm2, %ymm0
+; AVX512VL-FALLBACK-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512VL-FALLBACK-NEXT:    retq
 ;
 ; AVX512BW-LABEL: vec512_i8_signed_mem_mem:
diff --git a/llvm/test/CodeGen/X86/midpoint-int.ll b/llvm/test/CodeGen/X86/midpoint-int.ll
index c032020dd..81744cb10 100644
--- a/llvm/test/CodeGen/X86/midpoint-int.ll
+++ b/llvm/test/CodeGen/X86/midpoint-int.ll
@@ -20,7 +20,7 @@ define i32 @scalar_i32_signed_reg_reg(i32 %a1, i32 %a2) nounwind {
 ; X64-NEXT:    leal -1(%rax,%rax), %eax
 ; X64-NEXT:    movl %edi, %ecx
 ; X64-NEXT:    cmovgl %esi, %ecx
-; X64-NEXT:    cmovgel %edi, %esi
+; X64-NEXT:    cmovgl %edi, %esi
 ; X64-NEXT:    subl %ecx, %esi
 ; X64-NEXT:    shrl %esi
 ; X64-NEXT:    imull %esi, %eax
@@ -29,30 +29,26 @@ define i32 @scalar_i32_signed_reg_reg(i32 %a1, i32 %a2) nounwind {
 ;
 ; X32-LABEL: scalar_i32_signed_reg_reg:
 ; X32:       # %bb.0:
-; X32-NEXT:    pushl %edi
 ; X32-NEXT:    pushl %esi
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    xorl %eax, %eax
-; X32-NEXT:    cmpl %edx, %ecx
-; X32-NEXT:    setle %al
-; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    jg .LBB0_2
-; X32-NEXT:  # %bb.1:
+; X32-NEXT:    xorl %edx, %edx
+; X32-NEXT:    cmpl %eax, %ecx
+; X32-NEXT:    setle %dl
+; X32-NEXT:    leal -1(%edx,%edx), %edx
+; X32-NEXT:    jg .LBB0_1
+; X32-NEXT:  # %bb.2:
 ; X32-NEXT:    movl %ecx, %esi
-; X32-NEXT:  .LBB0_2:
-; X32-NEXT:    leal -1(%eax,%eax), %edi
+; X32-NEXT:    jmp .LBB0_3
+; X32-NEXT:  .LBB0_1:
+; X32-NEXT:    movl %eax, %esi
 ; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    jge .LBB0_4
-; X32-NEXT:  # %bb.3:
-; X32-NEXT:    movl %edx, %eax
-; X32-NEXT:  .LBB0_4:
+; X32-NEXT:  .LBB0_3:
 ; X32-NEXT:    subl %esi, %eax
 ; X32-NEXT:    shrl %eax
-; X32-NEXT:    imull %edi, %eax
+; X32-NEXT:    imull %edx, %eax
 ; X32-NEXT:    addl %ecx, %eax
 ; X32-NEXT:    popl %esi
-; X32-NEXT:    popl %edi
 ; X32-NEXT:    retl
   %t3 = icmp sgt i32 %a1, %a2 ; signed
   %t4 = select i1 %t3, i32 -1, i32 1
@@ -127,7 +123,7 @@ define i32 @scalar_i32_signed_mem_reg(i32* %a1_addr, i32 %a2) nounwind {
 ; X64-NEXT:    leal -1(%rax,%rax), %eax
 ; X64-NEXT:    movl %ecx, %edx
 ; X64-NEXT:    cmovgl %esi, %edx
-; X64-NEXT:    cmovgel %ecx, %esi
+; X64-NEXT:    cmovgl %ecx, %esi
 ; X64-NEXT:    subl %edx, %esi
 ; X64-NEXT:    shrl %esi
 ; X64-NEXT:    imull %esi, %eax
@@ -136,31 +132,27 @@ define i32 @scalar_i32_signed_mem_reg(i32* %a1_addr, i32 %a2) nounwind {
 ;
 ; X32-LABEL: scalar_i32_signed_mem_reg:
 ; X32:       # %bb.0:
-; X32-NEXT:    pushl %edi
 ; X32-NEXT:    pushl %esi
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl (%eax), %ecx
-; X32-NEXT:    xorl %eax, %eax
-; X32-NEXT:    cmpl %edx, %ecx
-; X32-NEXT:    setle %al
-; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    jg .LBB2_2
-; X32-NEXT:  # %bb.1:
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT:    movl (%ecx), %ecx
+; X32-NEXT:    xorl %edx, %edx
+; X32-NEXT:    cmpl %eax, %ecx
+; X32-NEXT:    setle %dl
+; X32-NEXT:    leal -1(%edx,%edx), %edx
+; X32-NEXT:    jg .LBB2_1
+; X32-NEXT:  # %bb.2:
 ; X32-NEXT:    movl %ecx, %esi
-; X32-NEXT:  .LBB2_2:
-; X32-NEXT:    leal -1(%eax,%eax), %edi
+; X32-NEXT:    jmp .LBB2_3
+; X32-NEXT:  .LBB2_1:
+; X32-NEXT:    movl %eax, %esi
 ; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    jge .LBB2_4
-; X32-NEXT:  # %bb.3:
-; X32-NEXT:    movl %edx, %eax
-; X32-NEXT:  .LBB2_4:
+; X32-NEXT:  .LBB2_3:
 ; X32-NEXT:    subl %esi, %eax
 ; X32-NEXT:    shrl %eax
-; X32-NEXT:    imull %edi, %eax
+; X32-NEXT:    imull %edx, %eax
 ; X32-NEXT:    addl %ecx, %eax
 ; X32-NEXT:    popl %esi
-; X32-NEXT:    popl %edi
 ; X32-NEXT:    retl
   %a1 = load i32, i32* %a1_addr
   %t3 = icmp sgt i32 %a1, %a2 ; signed
@@ -184,7 +176,7 @@ define i32 @scalar_i32_signed_reg_mem(i32 %a1, i32* %a2_addr) nounwind {
 ; X64-NEXT:    leal -1(%rcx,%rcx), %ecx
 ; X64-NEXT:    movl %edi, %edx
 ; X64-NEXT:    cmovgl %eax, %edx
-; X64-NEXT:    cmovgel %edi, %eax
+; X64-NEXT:    cmovgl %edi, %eax
 ; X64-NEXT:    subl %edx, %eax
 ; X64-NEXT:    shrl %eax
 ; X64-NEXT:    imull %ecx, %eax
@@ -193,31 +185,27 @@ define i32 @scalar_i32_signed_reg_mem(i32 %a1, i32* %a2_addr) nounwind {
 ;
 ; X32-LABEL: scalar_i32_signed_reg_mem:
 ; X32:       # %bb.0:
-; X32-NEXT:    pushl %edi
 ; X32-NEXT:    pushl %esi
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl (%eax), %edx
-; X32-NEXT:    xorl %eax, %eax
-; X32-NEXT:    cmpl %edx, %ecx
-; X32-NEXT:    setle %al
-; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    jg .LBB3_2
-; X32-NEXT:  # %bb.1:
+; X32-NEXT:    movl (%eax), %eax
+; X32-NEXT:    xorl %edx, %edx
+; X32-NEXT:    cmpl %eax, %ecx
+; X32-NEXT:    setle %dl
+; X32-NEXT:    leal -1(%edx,%edx), %edx
+; X32-NEXT:    jg .LBB3_1
+; X32-NEXT:  # %bb.2:
 ; X32-NEXT:    movl %ecx, %esi
-; X32-NEXT:  .LBB3_2:
-; X32-NEXT:    leal -1(%eax,%eax), %edi
+; X32-NEXT:    jmp .LBB3_3
+; X32-NEXT:  .LBB3_1:
+; X32-NEXT:    movl %eax, %esi
 ; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    jge .LBB3_4
-; X32-NEXT:  # %bb.3:
-; X32-NEXT:    movl %edx, %eax
-; X32-NEXT:  .LBB3_4:
+; X32-NEXT:  .LBB3_3:
 ; X32-NEXT:    subl %esi, %eax
 ; X32-NEXT:    shrl %eax
-; X32-NEXT:    imull %edi, %eax
+; X32-NEXT:    imull %edx, %eax
 ; X32-NEXT:    addl %ecx, %eax
 ; X32-NEXT:    popl %esi
-; X32-NEXT:    popl %edi
 ; X32-NEXT:    retl
   %a2 = load i32, i32* %a2_addr
   %t3 = icmp sgt i32 %a1, %a2 ; signed
@@ -242,7 +230,7 @@ define i32 @scalar_i32_signed_mem_mem(i32* %a1_addr, i32* %a2_addr) nounwind {
 ; X64-NEXT:    leal -1(%rdx,%rdx), %edx
 ; X64-NEXT:    movl %ecx, %esi
 ; X64-NEXT:    cmovgl %eax, %esi
-; X64-NEXT:    cmovgel %ecx, %eax
+; X64-NEXT:    cmovgl %ecx, %eax
 ; X64-NEXT:    subl %esi, %eax
 ; X64-NEXT:    shrl %eax
 ; X64-NEXT:    imull %edx, %eax
@@ -251,32 +239,28 @@ define i32 @scalar_i32_signed_mem_mem(i32* %a1_addr, i32* %a2_addr) nounwind {
 ;
 ; X32-LABEL: scalar_i32_signed_mem_mem:
 ; X32:       # %bb.0:
-; X32-NEXT:    pushl %edi
 ; X32-NEXT:    pushl %esi
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X32-NEXT:    movl (%ecx), %ecx
-; X32-NEXT:    movl (%eax), %edx
-; X32-NEXT:    xorl %eax, %eax
-; X32-NEXT:    cmpl %edx, %ecx
-; X32-NEXT:    setle %al
-; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    jg .LBB4_2
-; X32-NEXT:  # %bb.1:
+; X32-NEXT:    movl (%eax), %eax
+; X32-NEXT:    xorl %edx, %edx
+; X32-NEXT:    cmpl %eax, %ecx
+; X32-NEXT:    setle %dl
+; X32-NEXT:    leal -1(%edx,%edx), %edx
+; X32-NEXT:    jg .LBB4_1
+; X32-NEXT:  # %bb.2:
 ; X32-NEXT:    movl %ecx, %esi
-; X32-NEXT:  .LBB4_2:
-; X32-NEXT:    leal -1(%eax,%eax), %edi
+; X32-NEXT:    jmp .LBB4_3
+; X32-NEXT:  .LBB4_1:
+; X32-NEXT:    movl %eax, %esi
 ; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    jge .LBB4_4
-; X32-NEXT:  # %bb.3:
-; X32-NEXT:    movl %edx, %eax
-; X32-NEXT:  .LBB4_4:
+; X32-NEXT:  .LBB4_3:
 ; X32-NEXT:    subl %esi, %eax
 ; X32-NEXT:    shrl %eax
-; X32-NEXT:    imull %edi, %eax
+; X32-NEXT:    imull %edx, %eax
 ; X32-NEXT:    addl %ecx, %eax
 ; X32-NEXT:    popl %esi
-; X32-NEXT:    popl %edi
 ; X32-NEXT:    retl
   %a1 = load i32, i32* %a1_addr
   %a2 = load i32, i32* %a2_addr
@@ -306,7 +290,7 @@ define i64 @scalar_i64_signed_reg_reg(i64 %a1, i64 %a2) nounwind {
 ; X64-NEXT:    leaq -1(%rax,%rax), %rax
 ; X64-NEXT:    movq %rdi, %rcx
 ; X64-NEXT:    cmovgq %rsi, %rcx
-; X64-NEXT:    cmovgeq %rdi, %rsi
+; X64-NEXT:    cmovgq %rdi, %rsi
 ; X64-NEXT:    subq %rcx, %rsi
 ; X64-NEXT:    shrq %rsi
 ; X64-NEXT:    imulq %rsi, %rax
@@ -319,48 +303,38 @@ define i64 @scalar_i64_signed_reg_reg(i64 %a1, i64 %a2) nounwind {
 ; X32-NEXT:    pushl %ebx
 ; X32-NEXT:    pushl %edi
 ; X32-NEXT:    pushl %esi
-; X32-NEXT:    pushl %eax
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ebp
-; X32-NEXT:    cmpl %esi, %edx
-; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    sbbl %ecx, %eax
-; X32-NEXT:    movl %edx, %eax
-; X32-NEXT:    movl $-1, %edi
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X32-NEXT:    cmpl %ecx, %eax
+; X32-NEXT:    movl %edi, %edx
+; X32-NEXT:    sbbl {{[0-9]+}}(%esp), %edx
 ; X32-NEXT:    movl $-1, %ebx
-; X32-NEXT:    jl .LBB5_2
-; X32-NEXT:  # %bb.1:
-; X32-NEXT:    xorl %ebx, %ebx
-; X32-NEXT:    movl $1, %edi
-; X32-NEXT:    movl %ecx, %ebp
-; X32-NEXT:    movl %esi, %edx
-; X32-NEXT:  .LBB5_2:
-; X32-NEXT:    movl %edi, (%esp) # 4-byte Spill
-; X32-NEXT:    cmpl %eax, %esi
-; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    jl .LBB5_1
+; X32-NEXT:  # %bb.2:
+; X32-NEXT:    xorl %ebp, %ebp
+; X32-NEXT:    movl $1, %ebx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT:    movl %ecx, %esi
+; X32-NEXT:    jmp .LBB5_3
+; X32-NEXT:  .LBB5_1:
+; X32-NEXT:    movl $-1, %ebp
+; X32-NEXT:    movl %edi, %edx
+; X32-NEXT:    movl %eax, %esi
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X32-NEXT:    sbbl %edi, %eax
-; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    jge .LBB5_4
-; X32-NEXT:  # %bb.3:
-; X32-NEXT:    movl %edi, %ecx
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:  .LBB5_4:
-; X32-NEXT:    subl %edx, %eax
-; X32-NEXT:    sbbl %ebp, %ecx
-; X32-NEXT:    shrdl $1, %ecx, %eax
-; X32-NEXT:    imull %eax, %ebx
-; X32-NEXT:    movl (%esp), %esi # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    addl %ebx, %edx
-; X32-NEXT:    shrl %ecx
-; X32-NEXT:    imull %esi, %ecx
-; X32-NEXT:    addl %ecx, %edx
-; X32-NEXT:    addl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:  .LBB5_3:
+; X32-NEXT:    subl %esi, %eax
+; X32-NEXT:    sbbl %edx, %edi
+; X32-NEXT:    shrdl $1, %edi, %eax
+; X32-NEXT:    imull %eax, %ebp
+; X32-NEXT:    mull %ebx
+; X32-NEXT:    addl %ebp, %edx
+; X32-NEXT:    shrl %edi
+; X32-NEXT:    imull %ebx, %edi
+; X32-NEXT:    addl %edi, %edx
+; X32-NEXT:    addl %ecx, %eax
 ; X32-NEXT:    adcl {{[0-9]+}}(%esp), %edx
-; X32-NEXT:    addl $4, %esp
 ; X32-NEXT:    popl %esi
 ; X32-NEXT:    popl %edi
 ; X32-NEXT:    popl %ebx
@@ -459,7 +433,7 @@ define i64 @scalar_i64_signed_mem_reg(i64* %a1_addr, i64 %a2) nounwind {
 ; X64-NEXT:    leaq -1(%rax,%rax), %rax
 ; X64-NEXT:    movq %rcx, %rdx
 ; X64-NEXT:    cmovgq %rsi, %rdx
-; X64-NEXT:    cmovgeq %rcx, %rsi
+; X64-NEXT:    cmovgq %rcx, %rsi
 ; X64-NEXT:    subq %rdx, %rsi
 ; X64-NEXT:    shrq %rsi
 ; X64-NEXT:    imulq %rsi, %rax
@@ -473,48 +447,40 @@ define i64 @scalar_i64_signed_mem_reg(i64* %a1_addr, i64 %a2) nounwind {
 ; X32-NEXT:    pushl %edi
 ; X32-NEXT:    pushl %esi
 ; X32-NEXT:    pushl %eax
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl (%eax), %esi
-; X32-NEXT:    movl 4(%eax), %ebp
-; X32-NEXT:    cmpl %esi, %ecx
-; X32-NEXT:    movl %edx, %eax
-; X32-NEXT:    sbbl %ebp, %eax
-; X32-NEXT:    movl $-1, %eax
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT:    movl (%ecx), %esi
+; X32-NEXT:    movl 4(%ecx), %ecx
+; X32-NEXT:    cmpl %esi, %eax
+; X32-NEXT:    movl %edi, %edx
+; X32-NEXT:    sbbl %ecx, %edx
 ; X32-NEXT:    movl $-1, %ebx
+; X32-NEXT:    jl .LBB7_1
+; X32-NEXT:  # %bb.2:
+; X32-NEXT:    xorl %ebp, %ebp
+; X32-NEXT:    movl $1, %ebx
+; X32-NEXT:    movl %ecx, (%esp) # 4-byte Spill
+; X32-NEXT:    movl %esi, %edx
+; X32-NEXT:    jmp .LBB7_3
+; X32-NEXT:  .LBB7_1:
+; X32-NEXT:    movl $-1, %ebp
+; X32-NEXT:    movl %edi, (%esp) # 4-byte Spill
+; X32-NEXT:    movl %eax, %edx
 ; X32-NEXT:    movl %ecx, %edi
-; X32-NEXT:    jl .LBB7_2
-; X32-NEXT:  # %bb.1:
-; X32-NEXT:    xorl %ebx, %ebx
-; X32-NEXT:    movl $1, %eax
-; X32-NEXT:    movl %ebp, %edx
-; X32-NEXT:    movl %esi, %edi
-; X32-NEXT:  .LBB7_2:
-; X32-NEXT:    movl %eax, (%esp) # 4-byte Spill
-; X32-NEXT:    cmpl %ecx, %esi
-; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    sbbl %ecx, %eax
-; X32-NEXT:    movl %ebp, %ecx
 ; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    jge .LBB7_4
-; X32-NEXT:  # %bb.3:
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:  .LBB7_4:
-; X32-NEXT:    subl %edi, %eax
-; X32-NEXT:    sbbl %edx, %ecx
-; X32-NEXT:    shrdl $1, %ecx, %eax
-; X32-NEXT:    imull %eax, %ebx
-; X32-NEXT:    movl (%esp), %edi # 4-byte Reload
-; X32-NEXT:    mull %edi
-; X32-NEXT:    addl %ebx, %edx
-; X32-NEXT:    shrl %ecx
-; X32-NEXT:    imull %edi, %ecx
-; X32-NEXT:    addl %ecx, %edx
+; X32-NEXT:  .LBB7_3:
+; X32-NEXT:    subl %edx, %eax
+; X32-NEXT:    sbbl (%esp), %edi # 4-byte Folded Reload
+; X32-NEXT:    shrdl $1, %edi, %eax
+; X32-NEXT:    imull %eax, %ebp
+; X32-NEXT:    mull %ebx
+; X32-NEXT:    addl %ebp, %edx
+; X32-NEXT:    shrl %edi
+; X32-NEXT:    imull %ebx, %edi
+; X32-NEXT:    addl %edi, %edx
 ; X32-NEXT:    addl %esi, %eax
-; X32-NEXT:    adcl %ebp, %edx
+; X32-NEXT:    adcl %ecx, %edx
 ; X32-NEXT:    addl $4, %esp
 ; X32-NEXT:    popl %esi
 ; X32-NEXT:    popl %edi
@@ -543,7 +509,7 @@ define i64 @scalar_i64_signed_reg_mem(i64 %a1, i64* %a2_addr) nounwind {
 ; X64-NEXT:    leaq -1(%rcx,%rcx), %rcx
 ; X64-NEXT:    movq %rdi, %rdx
 ; X64-NEXT:    cmovgq %rax, %rdx
-; X64-NEXT:    cmovgeq %rdi, %rax
+; X64-NEXT:    cmovgq %rdi, %rax
 ; X64-NEXT:    subq %rdx, %rax
 ; X64-NEXT:    shrq %rax
 ; X64-NEXT:    imulq %rcx, %rax
@@ -556,49 +522,39 @@ define i64 @scalar_i64_signed_reg_mem(i64 %a1, i64* %a2_addr) nounwind {
 ; X32-NEXT:    pushl %ebx
 ; X32-NEXT:    pushl %edi
 ; X32-NEXT:    pushl %esi
-; X32-NEXT:    subl $8, %esp
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl (%eax), %edx
-; X32-NEXT:    movl 4(%eax), %ebp
-; X32-NEXT:    cmpl %esi, %edx
-; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    sbbl %ecx, %eax
-; X32-NEXT:    movl $-1, %eax
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT:    movl (%edx), %eax
+; X32-NEXT:    movl 4(%edx), %edi
+; X32-NEXT:    cmpl %ecx, %eax
+; X32-NEXT:    movl %edi, %edx
+; X32-NEXT:    sbbl {{[0-9]+}}(%esp), %edx
 ; X32-NEXT:    movl $-1, %ebx
-; X32-NEXT:    movl %ebp, (%esp) # 4-byte Spill
-; X32-NEXT:    movl %edx, %edi
-; X32-NEXT:    jl .LBB8_2
-; X32-NEXT:  # %bb.1:
-; X32-NEXT:    xorl %ebx, %ebx
-; X32-NEXT:    movl $1, %eax
-; X32-NEXT:    movl %ecx, (%esp) # 4-byte Spill
-; X32-NEXT:    movl %esi, %edi
-; X32-NEXT:  .LBB8_2:
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    cmpl %edx, %esi
+; X32-NEXT:    jl .LBB8_1
+; X32-NEXT:  # %bb.2:
+; X32-NEXT:    xorl %ebp, %ebp
+; X32-NEXT:    movl $1, %ebx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT:    movl %ecx, %esi
+; X32-NEXT:    jmp .LBB8_3
+; X32-NEXT:  .LBB8_1:
+; X32-NEXT:    movl $-1, %ebp
+; X32-NEXT:    movl %edi, %edx
+; X32-NEXT:    movl %eax, %esi
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %edi
 ; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    sbbl %ebp, %eax
-; X32-NEXT:    jge .LBB8_4
-; X32-NEXT:  # %bb.3:
-; X32-NEXT:    movl %ebp, %ecx
-; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:  .LBB8_4:
-; X32-NEXT:    subl %edi, %esi
-; X32-NEXT:    sbbl (%esp), %ecx # 4-byte Folded Reload
-; X32-NEXT:    shrdl $1, %ecx, %esi
-; X32-NEXT:    imull %esi, %ebx
-; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    addl %ebx, %edx
-; X32-NEXT:    shrl %ecx
-; X32-NEXT:    imull %esi, %ecx
-; X32-NEXT:    addl %ecx, %edx
-; X32-NEXT:    addl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:  .LBB8_3:
+; X32-NEXT:    subl %esi, %eax
+; X32-NEXT:    sbbl %edx, %edi
+; X32-NEXT:    shrdl $1, %edi, %eax
+; X32-NEXT:    imull %eax, %ebp
+; X32-NEXT:    mull %ebx
+; X32-NEXT:    addl %ebp, %edx
+; X32-NEXT:    shrl %edi
+; X32-NEXT:    imull %ebx, %edi
+; X32-NEXT:    addl %edi, %edx
+; X32-NEXT:    addl %ecx, %eax
 ; X32-NEXT:    adcl {{[0-9]+}}(%esp), %edx
-; X32-NEXT:    addl $8, %esp
 ; X32-NEXT:    popl %esi
 ; X32-NEXT:    popl %edi
 ; X32-NEXT:    popl %ebx
@@ -627,7 +583,7 @@ define i64 @scalar_i64_signed_mem_mem(i64* %a1_addr, i64* %a2_addr) nounwind {
 ; X64-NEXT:    leaq -1(%rdx,%rdx), %rdx
 ; X64-NEXT:    movq %rcx, %rsi
 ; X64-NEXT:    cmovgq %rax, %rsi
-; X64-NEXT:    cmovgeq %rcx, %rax
+; X64-NEXT:    cmovgq %rcx, %rax
 ; X64-NEXT:    subq %rsi, %rax
 ; X64-NEXT:    shrq %rax
 ; X64-NEXT:    imulq %rdx, %rax
@@ -640,52 +596,43 @@ define i64 @scalar_i64_signed_mem_mem(i64* %a1_addr, i64* %a2_addr) nounwind {
 ; X32-NEXT:    pushl %ebx
 ; X32-NEXT:    pushl %edi
 ; X32-NEXT:    pushl %esi
-; X32-NEXT:    subl $12, %esp
+; X32-NEXT:    pushl %eax
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    movl (%ecx), %esi
-; X32-NEXT:    movl 4(%ecx), %edi
-; X32-NEXT:    movl (%eax), %edx
-; X32-NEXT:    movl 4(%eax), %ebp
-; X32-NEXT:    cmpl %esi, %edx
-; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    sbbl %edi, %eax
-; X32-NEXT:    movl $-1, %eax
+; X32-NEXT:    movl (%eax), %esi
+; X32-NEXT:    movl 4(%eax), %ecx
+; X32-NEXT:    movl (%edx), %eax
+; X32-NEXT:    movl 4(%edx), %edi
+; X32-NEXT:    cmpl %esi, %eax
+; X32-NEXT:    movl %edi, %edx
+; X32-NEXT:    sbbl %ecx, %edx
 ; X32-NEXT:    movl $-1, %ebx
-; X32-NEXT:    movl %ebp, %ecx
-; X32-NEXT:    movl %edx, (%esp) # 4-byte Spill
-; X32-NEXT:    jl .LBB9_2
-; X32-NEXT:  # %bb.1:
-; X32-NEXT:    xorl %ebx, %ebx
-; X32-NEXT:    movl $1, %eax
-; X32-NEXT:    movl %edi, %ecx
-; X32-NEXT:    movl %esi, (%esp) # 4-byte Spill
-; X32-NEXT:  .LBB9_2:
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    cmpl %edx, %esi
-; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    sbbl %ebp, %eax
-; X32-NEXT:    movl %edi, %ecx
+; X32-NEXT:    jl .LBB9_1
+; X32-NEXT:  # %bb.2:
+; X32-NEXT:    xorl %ebp, %ebp
+; X32-NEXT:    movl $1, %ebx
+; X32-NEXT:    movl %ecx, (%esp) # 4-byte Spill
+; X32-NEXT:    movl %esi, %edx
+; X32-NEXT:    jmp .LBB9_3
+; X32-NEXT:  .LBB9_1:
+; X32-NEXT:    movl $-1, %ebp
+; X32-NEXT:    movl %edi, (%esp) # 4-byte Spill
+; X32-NEXT:    movl %eax, %edx
+; X32-NEXT:    movl %ecx, %edi
 ; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    jge .LBB9_4
-; X32-NEXT:  # %bb.3:
-; X32-NEXT:    movl %ebp, %ecx
-; X32-NEXT:    movl %edx, %eax
-; X32-NEXT:  .LBB9_4:
-; X32-NEXT:    subl (%esp), %eax # 4-byte Folded Reload
-; X32-NEXT:    sbbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    shrdl $1, %ecx, %eax
-; X32-NEXT:    imull %eax, %ebx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    mull %ebp
-; X32-NEXT:    addl %ebx, %edx
-; X32-NEXT:    shrl %ecx
-; X32-NEXT:    imull %ebp, %ecx
-; X32-NEXT:    addl %ecx, %edx
+; X32-NEXT:  .LBB9_3:
+; X32-NEXT:    subl %edx, %eax
+; X32-NEXT:    sbbl (%esp), %edi # 4-byte Folded Reload
+; X32-NEXT:    shrdl $1, %edi, %eax
+; X32-NEXT:    imull %eax, %ebp
+; X32-NEXT:    mull %ebx
+; X32-NEXT:    addl %ebp, %edx
+; X32-NEXT:    shrl %edi
+; X32-NEXT:    imull %ebx, %edi
+; X32-NEXT:    addl %edi, %edx
 ; X32-NEXT:    addl %esi, %eax
-; X32-NEXT:    adcl %edi, %edx
-; X32-NEXT:    addl $12, %esp
+; X32-NEXT:    adcl %ecx, %edx
+; X32-NEXT:    addl $4, %esp
 ; X32-NEXT:    popl %esi
 ; X32-NEXT:    popl %edi
 ; X32-NEXT:    popl %ebx
@@ -719,7 +666,7 @@ define i16 @scalar_i16_signed_reg_reg(i16 %a1, i16 %a2) nounwind {
 ; X64-NEXT:    leal -1(%rax,%rax), %ecx
 ; X64-NEXT:    movl %edi, %eax
 ; X64-NEXT:    cmovgl %esi, %eax
-; X64-NEXT:    cmovgel %edi, %esi
+; X64-NEXT:    cmovgl %edi, %esi
 ; X64-NEXT:    subl %eax, %esi
 ; X64-NEXT:    movzwl %si, %eax
 ; X64-NEXT:    shrl %eax
@@ -730,32 +677,28 @@ define i16 @scalar_i16_signed_reg_reg(i16 %a1, i16 %a2) nounwind {
 ;
 ; X32-LABEL: scalar_i16_signed_reg_reg:
 ; X32:       # %bb.0:
-; X32-NEXT:    pushl %edi
 ; X32-NEXT:    pushl %esi
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X32-NEXT:    xorl %edx, %edx
 ; X32-NEXT:    cmpw %ax, %cx
 ; X32-NEXT:    setle %dl
-; X32-NEXT:    movl %eax, %esi
-; X32-NEXT:    jg .LBB10_2
-; X32-NEXT:  # %bb.1:
-; X32-NEXT:    movl %ecx, %esi
-; X32-NEXT:  .LBB10_2:
 ; X32-NEXT:    leal -1(%edx,%edx), %edx
-; X32-NEXT:    movl %ecx, %edi
-; X32-NEXT:    jge .LBB10_4
-; X32-NEXT:  # %bb.3:
-; X32-NEXT:    movl %eax, %edi
-; X32-NEXT:  .LBB10_4:
-; X32-NEXT:    subl %esi, %edi
-; X32-NEXT:    movzwl %di, %eax
+; X32-NEXT:    jg .LBB10_1
+; X32-NEXT:  # %bb.2:
+; X32-NEXT:    movl %ecx, %esi
+; X32-NEXT:    jmp .LBB10_3
+; X32-NEXT:  .LBB10_1:
+; X32-NEXT:    movl %eax, %esi
+; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:  .LBB10_3:
+; X32-NEXT:    subl %esi, %eax
+; X32-NEXT:    movzwl %ax, %eax
 ; X32-NEXT:    shrl %eax
 ; X32-NEXT:    imull %edx, %eax
 ; X32-NEXT:    addl %ecx, %eax
 ; X32-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X32-NEXT:    popl %esi
-; X32-NEXT:    popl %edi
 ; X32-NEXT:    retl
   %t3 = icmp sgt i16 %a1, %a2 ; signed
   %t4 = select i1 %t3, i16 -1, i16 1
@@ -834,7 +777,7 @@ define i16 @scalar_i16_signed_mem_reg(i16* %a1_addr, i16 %a2) nounwind {
 ; X64-NEXT:    leal -1(%rax,%rax), %edx
 ; X64-NEXT:    movl %ecx, %eax
 ; X64-NEXT:    cmovgl %esi, %eax
-; X64-NEXT:    cmovgel %ecx, %esi
+; X64-NEXT:    cmovgl %ecx, %esi
 ; X64-NEXT:    subl %eax, %esi
 ; X64-NEXT:    movzwl %si, %eax
 ; X64-NEXT:    shrl %eax
@@ -845,7 +788,6 @@ define i16 @scalar_i16_signed_mem_reg(i16* %a1_addr, i16 %a2) nounwind {
 ;
 ; X32-LABEL: scalar_i16_signed_mem_reg:
 ; X32:       # %bb.0:
-; X32-NEXT:    pushl %edi
 ; X32-NEXT:    pushl %esi
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
@@ -853,25 +795,22 @@ define i16 @scalar_i16_signed_mem_reg(i16* %a1_addr, i16 %a2) nounwind {
 ; X32-NEXT:    xorl %edx, %edx
 ; X32-NEXT:    cmpw %ax, %cx
 ; X32-NEXT:    setle %dl
-; X32-NEXT:    movl %eax, %esi
-; X32-NEXT:    jg .LBB12_2
-; X32-NEXT:  # %bb.1:
-; X32-NEXT:    movl %ecx, %esi
-; X32-NEXT:  .LBB12_2:
 ; X32-NEXT:    leal -1(%edx,%edx), %edx
-; X32-NEXT:    movl %ecx, %edi
-; X32-NEXT:    jge .LBB12_4
-; X32-NEXT:  # %bb.3:
-; X32-NEXT:    movl %eax, %edi
-; X32-NEXT:  .LBB12_4:
-; X32-NEXT:    subl %esi, %edi
-; X32-NEXT:    movzwl %di, %eax
+; X32-NEXT:    jg .LBB12_1
+; X32-NEXT:  # %bb.2:
+; X32-NEXT:    movl %ecx, %esi
+; X32-NEXT:    jmp .LBB12_3
+; X32-NEXT:  .LBB12_1:
+; X32-NEXT:    movl %eax, %esi
+; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:  .LBB12_3:
+; X32-NEXT:    subl %esi, %eax
+; X32-NEXT:    movzwl %ax, %eax
 ; X32-NEXT:    shrl %eax
 ; X32-NEXT:    imull %edx, %eax
 ; X32-NEXT:    addl %ecx, %eax
 ; X32-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X32-NEXT:    popl %esi
-; X32-NEXT:    popl %edi
 ; X32-NEXT:    retl
   %a1 = load i16, i16* %a1_addr
   %t3 = icmp sgt i16 %a1, %a2 ; signed
@@ -895,7 +834,7 @@ define i16 @scalar_i16_signed_reg_mem(i16 %a1, i16* %a2_addr) nounwind {
 ; X64-NEXT:    leal -1(%rcx,%rcx), %ecx
 ; X64-NEXT:    movl %edi, %edx
 ; X64-NEXT:    cmovgl %eax, %edx
-; X64-NEXT:    cmovgel %edi, %eax
+; X64-NEXT:    cmovgl %edi, %eax
 ; X64-NEXT:    subl %edx, %eax
 ; X64-NEXT:    movzwl %ax, %eax
 ; X64-NEXT:    shrl %eax
@@ -906,7 +845,6 @@ define i16 @scalar_i16_signed_reg_mem(i16 %a1, i16* %a2_addr) nounwind {
 ;
 ; X32-LABEL: scalar_i16_signed_reg_mem:
 ; X32:       # %bb.0:
-; X32-NEXT:    pushl %edi
 ; X32-NEXT:    pushl %esi
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
@@ -914,25 +852,22 @@ define i16 @scalar_i16_signed_reg_mem(i16 %a1, i16* %a2_addr) nounwind {
 ; X32-NEXT:    xorl %edx, %edx
 ; X32-NEXT:    cmpw %ax, %cx
 ; X32-NEXT:    setle %dl
-; X32-NEXT:    movl %eax, %esi
-; X32-NEXT:    jg .LBB13_2
-; X32-NEXT:  # %bb.1:
-; X32-NEXT:    movl %ecx, %esi
-; X32-NEXT:  .LBB13_2:
 ; X32-NEXT:    leal -1(%edx,%edx), %edx
-; X32-NEXT:    movl %ecx, %edi
-; X32-NEXT:    jge .LBB13_4
-; X32-NEXT:  # %bb.3:
-; X32-NEXT:    movl %eax, %edi
-; X32-NEXT:  .LBB13_4:
-; X32-NEXT:    subl %esi, %edi
-; X32-NEXT:    movzwl %di, %eax
+; X32-NEXT:    jg .LBB13_1
+; X32-NEXT:  # %bb.2:
+; X32-NEXT:    movl %ecx, %esi
+; X32-NEXT:    jmp .LBB13_3
+; X32-NEXT:  .LBB13_1:
+; X32-NEXT:    movl %eax, %esi
+; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:  .LBB13_3:
+; X32-NEXT:    subl %esi, %eax
+; X32-NEXT:    movzwl %ax, %eax
 ; X32-NEXT:    shrl %eax
 ; X32-NEXT:    imull %edx, %eax
 ; X32-NEXT:    addl %ecx, %eax
 ; X32-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X32-NEXT:    popl %esi
-; X32-NEXT:    popl %edi
 ; X32-NEXT:    retl
   %a2 = load i16, i16* %a2_addr
   %t3 = icmp sgt i16 %a1, %a2 ; signed
@@ -957,7 +892,7 @@ define i16 @scalar_i16_signed_mem_mem(i16* %a1_addr, i16* %a2_addr) nounwind {
 ; X64-NEXT:    leal -1(%rdx,%rdx), %edx
 ; X64-NEXT:    movl %ecx, %esi
 ; X64-NEXT:    cmovgl %eax, %esi
-; X64-NEXT:    cmovgel %ecx, %eax
+; X64-NEXT:    cmovgl %ecx, %eax
 ; X64-NEXT:    subl %esi, %eax
 ; X64-NEXT:    movzwl %ax, %eax
 ; X64-NEXT:    shrl %eax
@@ -968,7 +903,6 @@ define i16 @scalar_i16_signed_mem_mem(i16* %a1_addr, i16* %a2_addr) nounwind {
 ;
 ; X32-LABEL: scalar_i16_signed_mem_mem:
 ; X32:       # %bb.0:
-; X32-NEXT:    pushl %edi
 ; X32-NEXT:    pushl %esi
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
@@ -977,25 +911,22 @@ define i16 @scalar_i16_signed_mem_mem(i16* %a1_addr, i16* %a2_addr) nounwind {
 ; X32-NEXT:    xorl %edx, %edx
 ; X32-NEXT:    cmpw %ax, %cx
 ; X32-NEXT:    setle %dl
-; X32-NEXT:    movl %eax, %esi
-; X32-NEXT:    jg .LBB14_2
-; X32-NEXT:  # %bb.1:
-; X32-NEXT:    movl %ecx, %esi
-; X32-NEXT:  .LBB14_2:
 ; X32-NEXT:    leal -1(%edx,%edx), %edx
-; X32-NEXT:    movl %ecx, %edi
-; X32-NEXT:    jge .LBB14_4
-; X32-NEXT:  # %bb.3:
-; X32-NEXT:    movl %eax, %edi
-; X32-NEXT:  .LBB14_4:
-; X32-NEXT:    subl %esi, %edi
-; X32-NEXT:    movzwl %di, %eax
+; X32-NEXT:    jg .LBB14_1
+; X32-NEXT:  # %bb.2:
+; X32-NEXT:    movl %ecx, %esi
+; X32-NEXT:    jmp .LBB14_3
+; X32-NEXT:  .LBB14_1:
+; X32-NEXT:    movl %eax, %esi
+; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:  .LBB14_3:
+; X32-NEXT:    subl %esi, %eax
+; X32-NEXT:    movzwl %ax, %eax
 ; X32-NEXT:    shrl %eax
 ; X32-NEXT:    imull %edx, %eax
 ; X32-NEXT:    addl %ecx, %eax
 ; X32-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X32-NEXT:    popl %esi
-; X32-NEXT:    popl %edi
 ; X32-NEXT:    retl
   %a1 = load i16, i16* %a1_addr
   %a2 = load i16, i16* %a2_addr
@@ -1024,7 +955,7 @@ define i8 @scalar_i8_signed_reg_reg(i8 %a1, i8 %a2) nounwind {
 ; X64-NEXT:    setle %cl
 ; X64-NEXT:    movl %edi, %edx
 ; X64-NEXT:    cmovgl %esi, %edx
-; X64-NEXT:    cmovgel %edi, %eax
+; X64-NEXT:    cmovgl %edi, %eax
 ; X64-NEXT:    addb %cl, %cl
 ; X64-NEXT:    decb %cl
 ; X64-NEXT:    subb %dl, %al
@@ -1036,21 +967,19 @@ define i8 @scalar_i8_signed_reg_reg(i8 %a1, i8 %a2) nounwind {
 ;
 ; X32-LABEL: scalar_i8_signed_reg_reg:
 ; X32:       # %bb.0:
-; X32-NEXT:    movb {{[0-9]+}}(%esp), %ah
+; X32-NEXT:    movb {{[0-9]+}}(%esp), %al
 ; X32-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X32-NEXT:    cmpb %ah, %cl
+; X32-NEXT:    cmpb %al, %cl
 ; X32-NEXT:    setle %dl
-; X32-NEXT:    movb %ah, %ch
-; X32-NEXT:    jg .LBB15_2
-; X32-NEXT:  # %bb.1:
-; X32-NEXT:    movb %cl, %ch
-; X32-NEXT:  .LBB15_2:
+; X32-NEXT:    jg .LBB15_1
+; X32-NEXT:  # %bb.2:
+; X32-NEXT:    movb %cl, %ah
+; X32-NEXT:    jmp .LBB15_3
+; X32-NEXT:  .LBB15_1:
+; X32-NEXT:    movb %al, %ah
 ; X32-NEXT:    movb %cl, %al
-; X32-NEXT:    jge .LBB15_4
-; X32-NEXT:  # %bb.3:
-; X32-NEXT:    movb %ah, %al
-; X32-NEXT:  .LBB15_4:
-; X32-NEXT:    subb %ch, %al
+; X32-NEXT:  .LBB15_3:
+; X32-NEXT:    subb %ah, %al
 ; X32-NEXT:    addb %dl, %dl
 ; X32-NEXT:    decb %dl
 ; X32-NEXT:    shrb %al
@@ -1129,7 +1058,7 @@ define i8 @scalar_i8_signed_mem_reg(i8* %a1_addr, i8 %a2) nounwind {
 ; X64-NEXT:    movl %ecx, %edi
 ; X64-NEXT:    cmovgl %esi, %edi
 ; X64-NEXT:    movl %ecx, %eax
-; X64-NEXT:    cmovll %esi, %eax
+; X64-NEXT:    cmovlel %esi, %eax
 ; X64-NEXT:    addb %dl, %dl
 ; X64-NEXT:    decb %dl
 ; X64-NEXT:    subb %dil, %al
@@ -1141,22 +1070,20 @@ define i8 @scalar_i8_signed_mem_reg(i8* %a1_addr, i8 %a2) nounwind {
 ;
 ; X32-LABEL: scalar_i8_signed_mem_reg:
 ; X32:       # %bb.0:
-; X32-NEXT:    movb {{[0-9]+}}(%esp), %ah
+; X32-NEXT:    movb {{[0-9]+}}(%esp), %al
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X32-NEXT:    movb (%ecx), %cl
-; X32-NEXT:    cmpb %ah, %cl
+; X32-NEXT:    cmpb %al, %cl
 ; X32-NEXT:    setle %dl
-; X32-NEXT:    movb %ah, %ch
-; X32-NEXT:    jg .LBB17_2
-; X32-NEXT:  # %bb.1:
-; X32-NEXT:    movb %cl, %ch
-; X32-NEXT:  .LBB17_2:
+; X32-NEXT:    jg .LBB17_1
+; X32-NEXT:  # %bb.2:
+; X32-NEXT:    movb %cl, %ah
+; X32-NEXT:    jmp .LBB17_3
+; X32-NEXT:  .LBB17_1:
+; X32-NEXT:    movb %al, %ah
 ; X32-NEXT:    movb %cl, %al
-; X32-NEXT:    jge .LBB17_4
-; X32-NEXT:  # %bb.3:
-; X32-NEXT:    movb %ah, %al
-; X32-NEXT:  .LBB17_4:
-; X32-NEXT:    subb %ch, %al
+; X32-NEXT:  .LBB17_3:
+; X32-NEXT:    subb %ah, %al
 ; X32-NEXT:    addb %dl, %dl
 ; X32-NEXT:    decb %dl
 ; X32-NEXT:    shrb %al
@@ -1183,7 +1110,7 @@ define i8 @scalar_i8_signed_reg_mem(i8 %a1, i8* %a2_addr) nounwind {
 ; X64-NEXT:    setle %cl
 ; X64-NEXT:    movl %edi, %edx
 ; X64-NEXT:    cmovgl %eax, %edx
-; X64-NEXT:    cmovgel %edi, %eax
+; X64-NEXT:    cmovgl %edi, %eax
 ; X64-NEXT:    addb %cl, %cl
 ; X64-NEXT:    decb %cl
 ; X64-NEXT:    subb %dl, %al
@@ -1197,20 +1124,18 @@ define i8 @scalar_i8_signed_reg_mem(i8 %a1, i8* %a2_addr) nounwind {
 ; X32:       # %bb.0:
 ; X32-NEXT:    movb {{[0-9]+}}(%esp), %cl
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movb (%eax), %ah
-; X32-NEXT:    cmpb %ah, %cl
+; X32-NEXT:    movb (%eax), %al
+; X32-NEXT:    cmpb %al, %cl
 ; X32-NEXT:    setle %dl
-; X32-NEXT:    movb %ah, %ch
-; X32-NEXT:    jg .LBB18_2
-; X32-NEXT:  # %bb.1:
-; X32-NEXT:    movb %cl, %ch
-; X32-NEXT:  .LBB18_2:
+; X32-NEXT:    jg .LBB18_1
+; X32-NEXT:  # %bb.2:
+; X32-NEXT:    movb %cl, %ah
+; X32-NEXT:    jmp .LBB18_3
+; X32-NEXT:  .LBB18_1:
+; X32-NEXT:    movb %al, %ah
 ; X32-NEXT:    movb %cl, %al
-; X32-NEXT:    jge .LBB18_4
-; X32-NEXT:  # %bb.3:
-; X32-NEXT:    movb %ah, %al
-; X32-NEXT:  .LBB18_4:
-; X32-NEXT:    subb %ch, %al
+; X32-NEXT:  .LBB18_3:
+; X32-NEXT:    subb %ah, %al
 ; X32-NEXT:    addb %dl, %dl
 ; X32-NEXT:    decb %dl
 ; X32-NEXT:    shrb %al
@@ -1238,7 +1163,7 @@ define i8 @scalar_i8_signed_mem_mem(i8* %a1_addr, i8* %a2_addr) nounwind {
 ; X64-NEXT:    setle %dl
 ; X64-NEXT:    movl %ecx, %esi
 ; X64-NEXT:    cmovgl %eax, %esi
-; X64-NEXT:    cmovgel %ecx, %eax
+; X64-NEXT:    cmovgl %ecx, %eax
 ; X64-NEXT:    addb %dl, %dl
 ; X64-NEXT:    decb %dl
 ; X64-NEXT:    subb %sil, %al
@@ -1253,20 +1178,18 @@ define i8 @scalar_i8_signed_mem_mem(i8* %a1_addr, i8* %a2_addr) nounwind {
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X32-NEXT:    movb (%ecx), %cl
-; X32-NEXT:    movb (%eax), %ah
-; X32-NEXT:    cmpb %ah, %cl
+; X32-NEXT:    movb (%eax), %al
+; X32-NEXT:    cmpb %al, %cl
 ; X32-NEXT:    setle %dl
-; X32-NEXT:    movb %ah, %ch
-; X32-NEXT:    jg .LBB19_2
-; X32-NEXT:  # %bb.1:
-; X32-NEXT:    movb %cl, %ch
-; X32-NEXT:  .LBB19_2:
+; X32-NEXT:    jg .LBB19_1
+; X32-NEXT:  # %bb.2:
+; X32-NEXT:    movb %cl, %ah
+; X32-NEXT:    jmp .LBB19_3
+; X32-NEXT:  .LBB19_1:
+; X32-NEXT:    movb %al, %ah
 ; X32-NEXT:    movb %cl, %al
-; X32-NEXT:    jge .LBB19_4
-; X32-NEXT:  # %bb.3:
-; X32-NEXT:    movb %ah, %al
-; X32-NEXT:  .LBB19_4:
-; X32-NEXT:    subb %ch, %al
+; X32-NEXT:  .LBB19_3:
+; X32-NEXT:    subb %ah, %al
 ; X32-NEXT:    addb %dl, %dl
 ; X32-NEXT:    decb %dl
 ; X32-NEXT:    shrb %al
diff --git a/llvm/test/CodeGen/X86/min-legal-vector-width.ll b/llvm/test/CodeGen/X86/min-legal-vector-width.ll
index 3d39f96ad..543da1e99 100644
--- a/llvm/test/CodeGen/X86/min-legal-vector-width.ll
+++ b/llvm/test/CodeGen/X86/min-legal-vector-width.ll
@@ -1,5 +1,14 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512vl,avx512bw,avx512dq,prefer-256-bit | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=skylake-avx512 -mattr=prefer-256-bit | FileCheck %s --check-prefixes=CHECK,CHECK-AVX512
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=skylake-avx512 -mattr=prefer-256-bit,avx512vbmi | FileCheck %s --check-prefixes=CHECK,CHECK-VBMI
+; Make sure CPUs default to prefer-256-bit. avx512vnni isn't interesting as it just adds an isel peephole for vpmaddwd+vpaddd
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=skylake-avx512 | FileCheck %s --check-prefixes=CHECK,CHECK-AVX512
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=-avx512vnni -mcpu=cascadelake | FileCheck %s --check-prefixes=CHECK,CHECK-AVX512
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=-avx512vnni -mcpu=cooperlake | FileCheck %s --check-prefixes=CHECK,CHECK-AVX512
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=cannonlake | FileCheck %s --check-prefixes=CHECK,CHECK-VBMI
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=-avx512vnni -mcpu=icelake-client | FileCheck %s --check-prefixes=CHECK,CHECK-VBMI
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=-avx512vnni -mcpu=icelake-server | FileCheck %s --check-prefixes=CHECK,CHECK-VBMI
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=-avx512vnni -mcpu=tigerlake | FileCheck %s --check-prefixes=CHECK,CHECK-VBMI
 
 ; This file primarily contains tests for specific places in X86ISelLowering.cpp that needed be made aware of the legalizer not allowing 512-bit vectors due to prefer-256-bit even though AVX512 is enabled.
 
@@ -646,35 +655,61 @@ define <16 x i16> @test_16f32tosb_512(<16 x float>* %ptr, <16 x i16> %passthru)
 }
 
 define void @mul256(<64 x i8>* %a, <64 x i8>* %b, <64 x i8>* %c) "min-legal-vector-width"="256" {
-; CHECK-LABEL: mul256:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmovdqa (%rdi), %ymm0
-; CHECK-NEXT:    vmovdqa 32(%rdi), %ymm1
-; CHECK-NEXT:    vmovdqa (%rsi), %ymm2
-; CHECK-NEXT:    vmovdqa 32(%rsi), %ymm3
-; CHECK-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm3[8],ymm0[8],ymm3[9],ymm0[9],ymm3[10],ymm0[10],ymm3[11],ymm0[11],ymm3[12],ymm0[12],ymm3[13],ymm0[13],ymm3[14],ymm0[14],ymm3[15],ymm0[15],ymm3[24],ymm0[24],ymm3[25],ymm0[25],ymm3[26],ymm0[26],ymm3[27],ymm0[27],ymm3[28],ymm0[28],ymm3[29],ymm0[29],ymm3[30],ymm0[30],ymm3[31],ymm0[31]
-; CHECK-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31]
-; CHECK-NEXT:    vpmullw %ymm4, %ymm5, %ymm4
-; CHECK-NEXT:    vmovdqa {{.*#+}} ymm5 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; CHECK-NEXT:    vpand %ymm5, %ymm4, %ymm4
-; CHECK-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23]
-; CHECK-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
-; CHECK-NEXT:    vpmullw %ymm3, %ymm1, %ymm1
-; CHECK-NEXT:    vpand %ymm5, %ymm1, %ymm1
-; CHECK-NEXT:    vpackuswb %ymm4, %ymm1, %ymm1
-; CHECK-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11],ymm2[12],ymm0[12],ymm2[13],ymm0[13],ymm2[14],ymm0[14],ymm2[15],ymm0[15],ymm2[24],ymm0[24],ymm2[25],ymm0[25],ymm2[26],ymm0[26],ymm2[27],ymm0[27],ymm2[28],ymm0[28],ymm2[29],ymm0[29],ymm2[30],ymm0[30],ymm2[31],ymm0[31]
-; CHECK-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
-; CHECK-NEXT:    vpmullw %ymm3, %ymm4, %ymm3
-; CHECK-NEXT:    vpand %ymm5, %ymm3, %ymm3
-; CHECK-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
-; CHECK-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; CHECK-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
-; CHECK-NEXT:    vpand %ymm5, %ymm0, %ymm0
-; CHECK-NEXT:    vpackuswb %ymm3, %ymm0, %ymm0
-; CHECK-NEXT:    vmovdqa %ymm0, (%rdx)
-; CHECK-NEXT:    vmovdqa %ymm1, 32(%rdx)
-; CHECK-NEXT:    vzeroupper
-; CHECK-NEXT:    retq
+; CHECK-AVX512-LABEL: mul256:
+; CHECK-AVX512:       # %bb.0:
+; CHECK-AVX512-NEXT:    vmovdqa (%rdi), %ymm0
+; CHECK-AVX512-NEXT:    vmovdqa 32(%rdi), %ymm1
+; CHECK-AVX512-NEXT:    vmovdqa (%rsi), %ymm2
+; CHECK-AVX512-NEXT:    vmovdqa 32(%rsi), %ymm3
+; CHECK-AVX512-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm3[8],ymm0[8],ymm3[9],ymm0[9],ymm3[10],ymm0[10],ymm3[11],ymm0[11],ymm3[12],ymm0[12],ymm3[13],ymm0[13],ymm3[14],ymm0[14],ymm3[15],ymm0[15],ymm3[24],ymm0[24],ymm3[25],ymm0[25],ymm3[26],ymm0[26],ymm3[27],ymm0[27],ymm3[28],ymm0[28],ymm3[29],ymm0[29],ymm3[30],ymm0[30],ymm3[31],ymm0[31]
+; CHECK-AVX512-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31]
+; CHECK-AVX512-NEXT:    vpmullw %ymm4, %ymm5, %ymm4
+; CHECK-AVX512-NEXT:    vmovdqa {{.*#+}} ymm5 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; CHECK-AVX512-NEXT:    vpand %ymm5, %ymm4, %ymm4
+; CHECK-AVX512-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23]
+; CHECK-AVX512-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
+; CHECK-AVX512-NEXT:    vpmullw %ymm3, %ymm1, %ymm1
+; CHECK-AVX512-NEXT:    vpand %ymm5, %ymm1, %ymm1
+; CHECK-AVX512-NEXT:    vpackuswb %ymm4, %ymm1, %ymm1
+; CHECK-AVX512-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11],ymm2[12],ymm0[12],ymm2[13],ymm0[13],ymm2[14],ymm0[14],ymm2[15],ymm0[15],ymm2[24],ymm0[24],ymm2[25],ymm0[25],ymm2[26],ymm0[26],ymm2[27],ymm0[27],ymm2[28],ymm0[28],ymm2[29],ymm0[29],ymm2[30],ymm0[30],ymm2[31],ymm0[31]
+; CHECK-AVX512-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; CHECK-AVX512-NEXT:    vpmullw %ymm3, %ymm4, %ymm3
+; CHECK-AVX512-NEXT:    vpand %ymm5, %ymm3, %ymm3
+; CHECK-AVX512-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
+; CHECK-AVX512-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; CHECK-AVX512-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
+; CHECK-AVX512-NEXT:    vpand %ymm5, %ymm0, %ymm0
+; CHECK-AVX512-NEXT:    vpackuswb %ymm3, %ymm0, %ymm0
+; CHECK-AVX512-NEXT:    vmovdqa %ymm0, (%rdx)
+; CHECK-AVX512-NEXT:    vmovdqa %ymm1, 32(%rdx)
+; CHECK-AVX512-NEXT:    vzeroupper
+; CHECK-AVX512-NEXT:    retq
+;
+; CHECK-VBMI-LABEL: mul256:
+; CHECK-VBMI:       # %bb.0:
+; CHECK-VBMI-NEXT:    vmovdqa (%rdi), %ymm0
+; CHECK-VBMI-NEXT:    vmovdqa 32(%rdi), %ymm1
+; CHECK-VBMI-NEXT:    vmovdqa (%rsi), %ymm2
+; CHECK-VBMI-NEXT:    vmovdqa 32(%rsi), %ymm3
+; CHECK-VBMI-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm3[8],ymm0[8],ymm3[9],ymm0[9],ymm3[10],ymm0[10],ymm3[11],ymm0[11],ymm3[12],ymm0[12],ymm3[13],ymm0[13],ymm3[14],ymm0[14],ymm3[15],ymm0[15],ymm3[24],ymm0[24],ymm3[25],ymm0[25],ymm3[26],ymm0[26],ymm3[27],ymm0[27],ymm3[28],ymm0[28],ymm3[29],ymm0[29],ymm3[30],ymm0[30],ymm3[31],ymm0[31]
+; CHECK-VBMI-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31]
+; CHECK-VBMI-NEXT:    vpmullw %ymm4, %ymm5, %ymm4
+; CHECK-VBMI-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23]
+; CHECK-VBMI-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
+; CHECK-VBMI-NEXT:    vpmullw %ymm3, %ymm1, %ymm1
+; CHECK-VBMI-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,2,4,6,8,10,12,14,32,34,36,38,40,42,44,46,16,18,20,22,24,26,28,30,48,50,52,54,56,58,60,62]
+; CHECK-VBMI-NEXT:    vpermt2b %ymm4, %ymm3, %ymm1
+; CHECK-VBMI-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11],ymm2[12],ymm0[12],ymm2[13],ymm0[13],ymm2[14],ymm0[14],ymm2[15],ymm0[15],ymm2[24],ymm0[24],ymm2[25],ymm0[25],ymm2[26],ymm0[26],ymm2[27],ymm0[27],ymm2[28],ymm0[28],ymm2[29],ymm0[29],ymm2[30],ymm0[30],ymm2[31],ymm0[31]
+; CHECK-VBMI-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; CHECK-VBMI-NEXT:    vpmullw %ymm4, %ymm5, %ymm4
+; CHECK-VBMI-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
+; CHECK-VBMI-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; CHECK-VBMI-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
+; CHECK-VBMI-NEXT:    vpermt2b %ymm4, %ymm3, %ymm0
+; CHECK-VBMI-NEXT:    vmovdqa %ymm0, (%rdx)
+; CHECK-VBMI-NEXT:    vmovdqa %ymm1, 32(%rdx)
+; CHECK-VBMI-NEXT:    vzeroupper
+; CHECK-VBMI-NEXT:    retq
   %d = load <64 x i8>, <64 x i8>* %a
   %e = load <64 x i8>, <64 x i8>* %b
   %f = mul <64 x i8> %d, %e
@@ -683,23 +718,39 @@ define void @mul256(<64 x i8>* %a, <64 x i8>* %b, <64 x i8>* %c) "min-legal-vect
 }
 
 define void @mul512(<64 x i8>* %a, <64 x i8>* %b, <64 x i8>* %c) "min-legal-vector-width"="512" {
-; CHECK-LABEL: mul512:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm0
-; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm1
-; CHECK-NEXT:    vpunpckhbw {{.*#+}} zmm2 = zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[12],zmm0[12],zmm1[13],zmm0[13],zmm1[14],zmm0[14],zmm1[15],zmm0[15],zmm1[24],zmm0[24],zmm1[25],zmm0[25],zmm1[26],zmm0[26],zmm1[27],zmm0[27],zmm1[28],zmm0[28],zmm1[29],zmm0[29],zmm1[30],zmm0[30],zmm1[31],zmm0[31],zmm1[40],zmm0[40],zmm1[41],zmm0[41],zmm1[42],zmm0[42],zmm1[43],zmm0[43],zmm1[44],zmm0[44],zmm1[45],zmm0[45],zmm1[46],zmm0[46],zmm1[47],zmm0[47],zmm1[56],zmm0[56],zmm1[57],zmm0[57],zmm1[58],zmm0[58],zmm1[59],zmm0[59],zmm1[60],zmm0[60],zmm1[61],zmm0[61],zmm1[62],zmm0[62],zmm1[63],zmm0[63]
-; CHECK-NEXT:    vpunpckhbw {{.*#+}} zmm3 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
-; CHECK-NEXT:    vpmullw %zmm2, %zmm3, %zmm2
-; CHECK-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; CHECK-NEXT:    vpandq %zmm3, %zmm2, %zmm2
-; CHECK-NEXT:    vpunpcklbw {{.*#+}} zmm1 = zmm1[0],zmm0[0],zmm1[1],zmm0[1],zmm1[2],zmm0[2],zmm1[3],zmm0[3],zmm1[4],zmm0[4],zmm1[5],zmm0[5],zmm1[6],zmm0[6],zmm1[7],zmm0[7],zmm1[16],zmm0[16],zmm1[17],zmm0[17],zmm1[18],zmm0[18],zmm1[19],zmm0[19],zmm1[20],zmm0[20],zmm1[21],zmm0[21],zmm1[22],zmm0[22],zmm1[23],zmm0[23],zmm1[32],zmm0[32],zmm1[33],zmm0[33],zmm1[34],zmm0[34],zmm1[35],zmm0[35],zmm1[36],zmm0[36],zmm1[37],zmm0[37],zmm1[38],zmm0[38],zmm1[39],zmm0[39],zmm1[48],zmm0[48],zmm1[49],zmm0[49],zmm1[50],zmm0[50],zmm1[51],zmm0[51],zmm1[52],zmm0[52],zmm1[53],zmm0[53],zmm1[54],zmm0[54],zmm1[55],zmm0[55]
-; CHECK-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
-; CHECK-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
-; CHECK-NEXT:    vpandq %zmm3, %zmm0, %zmm0
-; CHECK-NEXT:    vpackuswb %zmm2, %zmm0, %zmm0
-; CHECK-NEXT:    vmovdqa64 %zmm0, (%rdx)
-; CHECK-NEXT:    vzeroupper
-; CHECK-NEXT:    retq
+; CHECK-AVX512-LABEL: mul512:
+; CHECK-AVX512:       # %bb.0:
+; CHECK-AVX512-NEXT:    vmovdqa64 (%rdi), %zmm0
+; CHECK-AVX512-NEXT:    vmovdqa64 (%rsi), %zmm1
+; CHECK-AVX512-NEXT:    vpunpckhbw {{.*#+}} zmm2 = zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[12],zmm0[12],zmm1[13],zmm0[13],zmm1[14],zmm0[14],zmm1[15],zmm0[15],zmm1[24],zmm0[24],zmm1[25],zmm0[25],zmm1[26],zmm0[26],zmm1[27],zmm0[27],zmm1[28],zmm0[28],zmm1[29],zmm0[29],zmm1[30],zmm0[30],zmm1[31],zmm0[31],zmm1[40],zmm0[40],zmm1[41],zmm0[41],zmm1[42],zmm0[42],zmm1[43],zmm0[43],zmm1[44],zmm0[44],zmm1[45],zmm0[45],zmm1[46],zmm0[46],zmm1[47],zmm0[47],zmm1[56],zmm0[56],zmm1[57],zmm0[57],zmm1[58],zmm0[58],zmm1[59],zmm0[59],zmm1[60],zmm0[60],zmm1[61],zmm0[61],zmm1[62],zmm0[62],zmm1[63],zmm0[63]
+; CHECK-AVX512-NEXT:    vpunpckhbw {{.*#+}} zmm3 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
+; CHECK-AVX512-NEXT:    vpmullw %zmm2, %zmm3, %zmm2
+; CHECK-AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; CHECK-AVX512-NEXT:    vpandq %zmm3, %zmm2, %zmm2
+; CHECK-AVX512-NEXT:    vpunpcklbw {{.*#+}} zmm1 = zmm1[0],zmm0[0],zmm1[1],zmm0[1],zmm1[2],zmm0[2],zmm1[3],zmm0[3],zmm1[4],zmm0[4],zmm1[5],zmm0[5],zmm1[6],zmm0[6],zmm1[7],zmm0[7],zmm1[16],zmm0[16],zmm1[17],zmm0[17],zmm1[18],zmm0[18],zmm1[19],zmm0[19],zmm1[20],zmm0[20],zmm1[21],zmm0[21],zmm1[22],zmm0[22],zmm1[23],zmm0[23],zmm1[32],zmm0[32],zmm1[33],zmm0[33],zmm1[34],zmm0[34],zmm1[35],zmm0[35],zmm1[36],zmm0[36],zmm1[37],zmm0[37],zmm1[38],zmm0[38],zmm1[39],zmm0[39],zmm1[48],zmm0[48],zmm1[49],zmm0[49],zmm1[50],zmm0[50],zmm1[51],zmm0[51],zmm1[52],zmm0[52],zmm1[53],zmm0[53],zmm1[54],zmm0[54],zmm1[55],zmm0[55]
+; CHECK-AVX512-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
+; CHECK-AVX512-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
+; CHECK-AVX512-NEXT:    vpandq %zmm3, %zmm0, %zmm0
+; CHECK-AVX512-NEXT:    vpackuswb %zmm2, %zmm0, %zmm0
+; CHECK-AVX512-NEXT:    vmovdqa64 %zmm0, (%rdx)
+; CHECK-AVX512-NEXT:    vzeroupper
+; CHECK-AVX512-NEXT:    retq
+;
+; CHECK-VBMI-LABEL: mul512:
+; CHECK-VBMI:       # %bb.0:
+; CHECK-VBMI-NEXT:    vmovdqa64 (%rdi), %zmm0
+; CHECK-VBMI-NEXT:    vmovdqa64 (%rsi), %zmm1
+; CHECK-VBMI-NEXT:    vpunpckhbw {{.*#+}} zmm2 = zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[12],zmm0[12],zmm1[13],zmm0[13],zmm1[14],zmm0[14],zmm1[15],zmm0[15],zmm1[24],zmm0[24],zmm1[25],zmm0[25],zmm1[26],zmm0[26],zmm1[27],zmm0[27],zmm1[28],zmm0[28],zmm1[29],zmm0[29],zmm1[30],zmm0[30],zmm1[31],zmm0[31],zmm1[40],zmm0[40],zmm1[41],zmm0[41],zmm1[42],zmm0[42],zmm1[43],zmm0[43],zmm1[44],zmm0[44],zmm1[45],zmm0[45],zmm1[46],zmm0[46],zmm1[47],zmm0[47],zmm1[56],zmm0[56],zmm1[57],zmm0[57],zmm1[58],zmm0[58],zmm1[59],zmm0[59],zmm1[60],zmm0[60],zmm1[61],zmm0[61],zmm1[62],zmm0[62],zmm1[63],zmm0[63]
+; CHECK-VBMI-NEXT:    vpunpckhbw {{.*#+}} zmm3 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
+; CHECK-VBMI-NEXT:    vpmullw %zmm2, %zmm3, %zmm2
+; CHECK-VBMI-NEXT:    vpunpcklbw {{.*#+}} zmm1 = zmm1[0],zmm0[0],zmm1[1],zmm0[1],zmm1[2],zmm0[2],zmm1[3],zmm0[3],zmm1[4],zmm0[4],zmm1[5],zmm0[5],zmm1[6],zmm0[6],zmm1[7],zmm0[7],zmm1[16],zmm0[16],zmm1[17],zmm0[17],zmm1[18],zmm0[18],zmm1[19],zmm0[19],zmm1[20],zmm0[20],zmm1[21],zmm0[21],zmm1[22],zmm0[22],zmm1[23],zmm0[23],zmm1[32],zmm0[32],zmm1[33],zmm0[33],zmm1[34],zmm0[34],zmm1[35],zmm0[35],zmm1[36],zmm0[36],zmm1[37],zmm0[37],zmm1[38],zmm0[38],zmm1[39],zmm0[39],zmm1[48],zmm0[48],zmm1[49],zmm0[49],zmm1[50],zmm0[50],zmm1[51],zmm0[51],zmm1[52],zmm0[52],zmm1[53],zmm0[53],zmm1[54],zmm0[54],zmm1[55],zmm0[55]
+; CHECK-VBMI-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
+; CHECK-VBMI-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
+; CHECK-VBMI-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,2,4,6,8,10,12,14,64,66,68,70,72,74,76,78,16,18,20,22,24,26,28,30,80,82,84,86,88,90,92,94,32,34,36,38,40,42,44,46,96,98,100,102,104,106,108,110,48,50,52,54,56,58,60,62,112,114,116,118,120,122,124,126]
+; CHECK-VBMI-NEXT:    vpermi2b %zmm2, %zmm0, %zmm1
+; CHECK-VBMI-NEXT:    vmovdqa64 %zmm1, (%rdx)
+; CHECK-VBMI-NEXT:    vzeroupper
+; CHECK-VBMI-NEXT:    retq
   %d = load <64 x i8>, <64 x i8>* %a
   %e = load <64 x i8>, <64 x i8>* %b
   %f = mul <64 x i8> %d, %e
@@ -719,3 +770,833 @@ define <4 x i32> @mload_v4i32(<4 x i32> %trigger, <4 x i32>* %addr, <4 x i32> %d
   ret <4 x i32> %res
 }
 declare <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>*, i32, <4 x i1>, <4 x i32>)
+
+define <16 x i32> @trunc_v16i64_v16i32(<16 x i64>* %x) nounwind "min-legal-vector-width"="256" {
+; CHECK-LABEL: trunc_v16i64_v16i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovdqa (%rdi), %ymm0
+; CHECK-NEXT:    vmovdqa 32(%rdi), %ymm1
+; CHECK-NEXT:    vmovdqa 64(%rdi), %ymm2
+; CHECK-NEXT:    vmovdqa 96(%rdi), %ymm3
+; CHECK-NEXT:    vpmovqd %ymm0, %xmm0
+; CHECK-NEXT:    vpmovqd %ymm1, %xmm1
+; CHECK-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; CHECK-NEXT:    vpmovqd %ymm2, %xmm1
+; CHECK-NEXT:    vpmovqd %ymm3, %xmm2
+; CHECK-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; CHECK-NEXT:    retq
+  %a = load <16 x i64>, <16 x i64>* %x
+  %b = trunc <16 x i64> %a to <16 x i32>
+  ret <16 x i32> %b
+}
+
+define <16 x i8> @trunc_v16i64_v16i8(<16 x i64>* %x) nounwind "min-legal-vector-width"="256" {
+; CHECK-LABEL: trunc_v16i64_v16i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovdqa (%rdi), %ymm0
+; CHECK-NEXT:    vmovdqa 32(%rdi), %ymm1
+; CHECK-NEXT:    vmovdqa 64(%rdi), %ymm2
+; CHECK-NEXT:    vmovdqa 96(%rdi), %ymm3
+; CHECK-NEXT:    vpmovqb %ymm3, %xmm3
+; CHECK-NEXT:    vpmovqb %ymm2, %xmm2
+; CHECK-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; CHECK-NEXT:    vpmovqb %ymm1, %xmm1
+; CHECK-NEXT:    vpmovqb %ymm0, %xmm0
+; CHECK-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; CHECK-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %a = load <16 x i64>, <16 x i64>* %x
+  %b = trunc <16 x i64> %a to <16 x i8>
+  ret <16 x i8> %b
+}
+
+define <16 x i8> @trunc_v16i32_v16i8(<16 x i32>* %x) nounwind "min-legal-vector-width"="256" {
+; CHECK-LABEL: trunc_v16i32_v16i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovdqa (%rdi), %ymm0
+; CHECK-NEXT:    vmovdqa 32(%rdi), %ymm1
+; CHECK-NEXT:    vpmovdb %ymm1, %xmm1
+; CHECK-NEXT:    vpmovdb %ymm0, %xmm0
+; CHECK-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %a = load <16 x i32>, <16 x i32>* %x
+  %b = trunc <16 x i32> %a to <16 x i8>
+  ret <16 x i8> %b
+}
+
+define <8 x i8> @trunc_v8i64_v8i8(<8 x i64>* %x) nounwind "min-legal-vector-width"="256" {
+; CHECK-LABEL: trunc_v8i64_v8i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovdqa (%rdi), %ymm0
+; CHECK-NEXT:    vmovdqa 32(%rdi), %ymm1
+; CHECK-NEXT:    vpmovqb %ymm1, %xmm1
+; CHECK-NEXT:    vpmovqb %ymm0, %xmm0
+; CHECK-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %a = load <8 x i64>, <8 x i64>* %x
+  %b = trunc <8 x i64> %a to <8 x i8>
+  ret <8 x i8> %b
+}
+
+define <8 x i16> @trunc_v8i64_v8i16(<8 x i64>* %x) nounwind "min-legal-vector-width"="256" {
+; CHECK-LABEL: trunc_v8i64_v8i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovdqa (%rdi), %ymm0
+; CHECK-NEXT:    vmovdqa 32(%rdi), %ymm1
+; CHECK-NEXT:    vpmovqw %ymm1, %xmm1
+; CHECK-NEXT:    vpmovqw %ymm0, %xmm0
+; CHECK-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %a = load <8 x i64>, <8 x i64>* %x
+  %b = trunc <8 x i64> %a to <8 x i16>
+  ret <8 x i16> %b
+}
+
+define <8 x i32> @trunc_v8i64_v8i32_zeroes(<8 x i64>* %x) nounwind "min-legal-vector-width"="256" {
+; CHECK-LABEL: trunc_v8i64_v8i32_zeroes:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsrlq $48, 32(%rdi), %ymm1
+; CHECK-NEXT:    vpsrlq $48, (%rdi), %ymm2
+; CHECK-NEXT:    vmovdqa {{.*#+}} ymm0 = [0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30]
+; CHECK-NEXT:    vpermi2w %ymm1, %ymm2, %ymm0
+; CHECK-NEXT:    retq
+  %a = load <8 x i64>, <8 x i64>* %x
+  %b = lshr <8 x i64> %a, <i64 48, i64 48, i64 48, i64 48, i64 48, i64 48, i64 48, i64 48>
+  %c = trunc <8 x i64> %b to <8 x i32>
+  ret <8 x i32> %c
+}
+
+define <16 x i16> @trunc_v16i32_v16i16_zeroes(<16 x i32>* %x) nounwind "min-legal-vector-width"="256" {
+; CHECK-LABEL: trunc_v16i32_v16i16_zeroes:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovdqa (%rdi), %ymm1
+; CHECK-NEXT:    vmovdqa {{.*#+}} ymm0 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31]
+; CHECK-NEXT:    vpermi2w 32(%rdi), %ymm1, %ymm0
+; CHECK-NEXT:    retq
+  %a = load <16 x i32>, <16 x i32>* %x
+  %b = lshr <16 x i32> %a, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
+  %c = trunc <16 x i32> %b to <16 x i16>
+  ret <16 x i16> %c
+}
+
+define <32 x i8> @trunc_v32i16_v32i8_zeroes(<32 x i16>* %x) nounwind "min-legal-vector-width"="256" {
+; CHECK-AVX512-LABEL: trunc_v32i16_v32i8_zeroes:
+; CHECK-AVX512:       # %bb.0:
+; CHECK-AVX512-NEXT:    vpsrlw $8, 32(%rdi), %ymm0
+; CHECK-AVX512-NEXT:    vpsrlw $8, (%rdi), %ymm1
+; CHECK-AVX512-NEXT:    vpackuswb %ymm0, %ymm1, %ymm0
+; CHECK-AVX512-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; CHECK-AVX512-NEXT:    retq
+;
+; CHECK-VBMI-LABEL: trunc_v32i16_v32i8_zeroes:
+; CHECK-VBMI:       # %bb.0:
+; CHECK-VBMI-NEXT:    vmovdqa (%rdi), %ymm1
+; CHECK-VBMI-NEXT:    vmovdqa {{.*#+}} ymm0 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31,33,35,37,39,41,43,45,47,49,51,53,55,57,59,61,63]
+; CHECK-VBMI-NEXT:    vpermi2b 32(%rdi), %ymm1, %ymm0
+; CHECK-VBMI-NEXT:    retq
+  %a = load <32 x i16>, <32 x i16>* %x
+  %b = lshr <32 x i16> %a, <i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8>
+  %c = trunc <32 x i16> %b to <32 x i8>
+  ret <32 x i8> %c
+}
+
+define <8 x i32> @trunc_v8i64_v8i32_sign(<8 x i64>* %x) nounwind "min-legal-vector-width"="256" {
+; CHECK-LABEL: trunc_v8i64_v8i32_sign:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsraq $48, 32(%rdi), %ymm1
+; CHECK-NEXT:    vpsraq $48, (%rdi), %ymm2
+; CHECK-NEXT:    vmovdqa {{.*#+}} ymm0 = [0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30]
+; CHECK-NEXT:    vpermi2w %ymm1, %ymm2, %ymm0
+; CHECK-NEXT:    retq
+  %a = load <8 x i64>, <8 x i64>* %x
+  %b = ashr <8 x i64> %a, <i64 48, i64 48, i64 48, i64 48, i64 48, i64 48, i64 48, i64 48>
+  %c = trunc <8 x i64> %b to <8 x i32>
+  ret <8 x i32> %c
+}
+
+define <16 x i16> @trunc_v16i32_v16i16_sign(<16 x i32>* %x) nounwind "min-legal-vector-width"="256" {
+; CHECK-LABEL: trunc_v16i32_v16i16_sign:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovdqa (%rdi), %ymm1
+; CHECK-NEXT:    vmovdqa {{.*#+}} ymm0 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31]
+; CHECK-NEXT:    vpermi2w 32(%rdi), %ymm1, %ymm0
+; CHECK-NEXT:    retq
+  %a = load <16 x i32>, <16 x i32>* %x
+  %b = ashr <16 x i32> %a, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
+  %c = trunc <16 x i32> %b to <16 x i16>
+  ret <16 x i16> %c
+}
+
+define <32 x i8> @trunc_v32i16_v32i8_sign(<32 x i16>* %x) nounwind "min-legal-vector-width"="256" {
+; CHECK-AVX512-LABEL: trunc_v32i16_v32i8_sign:
+; CHECK-AVX512:       # %bb.0:
+; CHECK-AVX512-NEXT:    vpsraw $8, 32(%rdi), %ymm0
+; CHECK-AVX512-NEXT:    vpsraw $8, (%rdi), %ymm1
+; CHECK-AVX512-NEXT:    vpacksswb %ymm0, %ymm1, %ymm0
+; CHECK-AVX512-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; CHECK-AVX512-NEXT:    retq
+;
+; CHECK-VBMI-LABEL: trunc_v32i16_v32i8_sign:
+; CHECK-VBMI:       # %bb.0:
+; CHECK-VBMI-NEXT:    vmovdqa (%rdi), %ymm1
+; CHECK-VBMI-NEXT:    vmovdqa {{.*#+}} ymm0 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31,33,35,37,39,41,43,45,47,49,51,53,55,57,59,61,63]
+; CHECK-VBMI-NEXT:    vpermi2b 32(%rdi), %ymm1, %ymm0
+; CHECK-VBMI-NEXT:    retq
+  %a = load <32 x i16>, <32 x i16>* %x
+  %b = ashr <32 x i16> %a, <i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8>
+  %c = trunc <32 x i16> %b to <32 x i8>
+  ret <32 x i8> %c
+}
+
+define void @zext_v16i8_v16i64(<16 x i8> %x, <16 x i64>* %y) nounwind "min-legal-vector-width"="256" {
+; CHECK-LABEL: zext_v16i8_v16i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; CHECK-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; CHECK-NEXT:    vpmovzxwq {{.*#+}} ymm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
+; CHECK-NEXT:    vextracti128 $1, %ymm1, %xmm1
+; CHECK-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[2,3,0,1]
+; CHECK-NEXT:    vpmovzxwq {{.*#+}} ymm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
+; CHECK-NEXT:    vpmovzxwq {{.*#+}} ymm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
+; CHECK-NEXT:    vpmovzxbq {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero,xmm0[2],zero,zero,zero,zero,zero,zero,zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero
+; CHECK-NEXT:    vmovdqa %ymm0, (%rdi)
+; CHECK-NEXT:    vmovdqa %ymm1, 64(%rdi)
+; CHECK-NEXT:    vmovdqa %ymm3, 96(%rdi)
+; CHECK-NEXT:    vmovdqa %ymm2, 32(%rdi)
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %a = zext <16 x i8> %x to <16 x i64>
+  store <16 x i64> %a, <16 x i64>* %y
+  ret void
+}
+
+define void @sext_v16i8_v16i64(<16 x i8> %x, <16 x i64>* %y) nounwind "min-legal-vector-width"="256" {
+; CHECK-LABEL: sext_v16i8_v16i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmovsxbw %xmm0, %ymm0
+; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; CHECK-NEXT:    vpmovsxwq %xmm1, %ymm1
+; CHECK-NEXT:    vextracti128 $1, %ymm0, %xmm2
+; CHECK-NEXT:    vpshufd {{.*#+}} xmm3 = xmm2[2,3,0,1]
+; CHECK-NEXT:    vpmovsxwq %xmm3, %ymm3
+; CHECK-NEXT:    vpmovsxwq %xmm0, %ymm0
+; CHECK-NEXT:    vpmovsxwq %xmm2, %ymm2
+; CHECK-NEXT:    vmovdqa %ymm2, 64(%rdi)
+; CHECK-NEXT:    vmovdqa %ymm0, (%rdi)
+; CHECK-NEXT:    vmovdqa %ymm3, 96(%rdi)
+; CHECK-NEXT:    vmovdqa %ymm1, 32(%rdi)
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %a = sext <16 x i8> %x to <16 x i64>
+  store <16 x i64> %a, <16 x i64>* %y
+  ret void
+}
+
+define void @vselect_split_v8i16_setcc(<8 x i16> %s, <8 x i16> %t, <8 x i64>* %p, <8 x i64>* %q, <8 x i64>* %r) "min-legal-vector-width"="256" {
+; CHECK-LABEL: vselect_split_v8i16_setcc:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovdqa (%rsi), %ymm2
+; CHECK-NEXT:    vmovdqa 32(%rsi), %ymm3
+; CHECK-NEXT:    vpcmpeqw %xmm1, %xmm0, %k1
+; CHECK-NEXT:    kshiftrb $4, %k1, %k2
+; CHECK-NEXT:    vmovdqa64 32(%rdi), %ymm3 {%k2}
+; CHECK-NEXT:    vmovdqa64 (%rdi), %ymm2 {%k1}
+; CHECK-NEXT:    vmovdqa %ymm2, (%rdx)
+; CHECK-NEXT:    vmovdqa %ymm3, 32(%rdx)
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %x = load <8 x i64>, <8 x i64>* %p
+  %y = load <8 x i64>, <8 x i64>* %q
+  %a = icmp eq <8 x i16> %s, %t
+  %b = select <8 x i1> %a, <8 x i64> %x, <8 x i64> %y
+  store <8 x i64> %b, <8 x i64>* %r
+  ret void
+}
+
+define void @vselect_split_v8i32_setcc(<8 x i32> %s, <8 x i32> %t, <8 x i64>* %p, <8 x i64>* %q, <8 x i64>* %r) "min-legal-vector-width"="256" {
+; CHECK-LABEL: vselect_split_v8i32_setcc:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovdqa (%rsi), %ymm2
+; CHECK-NEXT:    vmovdqa 32(%rsi), %ymm3
+; CHECK-NEXT:    vpcmpeqd %ymm1, %ymm0, %k1
+; CHECK-NEXT:    kshiftrb $4, %k1, %k2
+; CHECK-NEXT:    vmovdqa64 32(%rdi), %ymm3 {%k2}
+; CHECK-NEXT:    vmovdqa64 (%rdi), %ymm2 {%k1}
+; CHECK-NEXT:    vmovdqa %ymm2, (%rdx)
+; CHECK-NEXT:    vmovdqa %ymm3, 32(%rdx)
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %x = load <8 x i64>, <8 x i64>* %p
+  %y = load <8 x i64>, <8 x i64>* %q
+  %a = icmp eq <8 x i32> %s, %t
+  %b = select <8 x i1> %a, <8 x i64> %x, <8 x i64> %y
+  store <8 x i64> %b, <8 x i64>* %r
+  ret void
+}
+
+define void @vselect_split_v16i8_setcc(<16 x i8> %s, <16 x i8> %t, <16 x i32>* %p, <16 x i32>* %q, <16 x i32>* %r) "min-legal-vector-width"="256" {
+; CHECK-LABEL: vselect_split_v16i8_setcc:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovdqa (%rsi), %ymm2
+; CHECK-NEXT:    vmovdqa 32(%rsi), %ymm3
+; CHECK-NEXT:    vpcmpeqb %xmm1, %xmm0, %k1
+; CHECK-NEXT:    kshiftrw $8, %k1, %k2
+; CHECK-NEXT:    vmovdqa32 32(%rdi), %ymm3 {%k2}
+; CHECK-NEXT:    vmovdqa32 (%rdi), %ymm2 {%k1}
+; CHECK-NEXT:    vmovdqa %ymm2, (%rdx)
+; CHECK-NEXT:    vmovdqa %ymm3, 32(%rdx)
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %x = load <16 x i32>, <16 x i32>* %p
+  %y = load <16 x i32>, <16 x i32>* %q
+  %a = icmp eq <16 x i8> %s, %t
+  %b = select <16 x i1> %a, <16 x i32> %x, <16 x i32> %y
+  store <16 x i32> %b, <16 x i32>* %r
+  ret void
+}
+
+define void @vselect_split_v16i16_setcc(<16 x i16> %s, <16 x i16> %t, <16 x i32>* %p, <16 x i32>* %q, <16 x i32>* %r) "min-legal-vector-width"="256" {
+; CHECK-LABEL: vselect_split_v16i16_setcc:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovdqa (%rsi), %ymm2
+; CHECK-NEXT:    vmovdqa 32(%rsi), %ymm3
+; CHECK-NEXT:    vpcmpeqw %ymm1, %ymm0, %k1
+; CHECK-NEXT:    kshiftrw $8, %k1, %k2
+; CHECK-NEXT:    vmovdqa32 32(%rdi), %ymm3 {%k2}
+; CHECK-NEXT:    vmovdqa32 (%rdi), %ymm2 {%k1}
+; CHECK-NEXT:    vmovdqa %ymm2, (%rdx)
+; CHECK-NEXT:    vmovdqa %ymm3, 32(%rdx)
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %x = load <16 x i32>, <16 x i32>* %p
+  %y = load <16 x i32>, <16 x i32>* %q
+  %a = icmp eq <16 x i16> %s, %t
+  %b = select <16 x i1> %a, <16 x i32> %x, <16 x i32> %y
+  store <16 x i32> %b, <16 x i32>* %r
+  ret void
+}
+
+define <16 x i8> @trunc_packus_v16i32_v16i8(<16 x i32>* %p) "min-legal-vector-width"="256" {
+; CHECK-LABEL: trunc_packus_v16i32_v16i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovdqa (%rdi), %ymm0
+; CHECK-NEXT:    vpackusdw 32(%rdi), %ymm0, %ymm0
+; CHECK-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; CHECK-NEXT:    vpmovuswb %ymm0, %xmm0
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %a = load <16 x i32>, <16 x i32>* %p
+  %b = icmp slt <16 x i32> %a, <i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255>
+  %c = select <16 x i1> %b, <16 x i32> %a, <16 x i32> <i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255>
+  %d = icmp sgt <16 x i32> %c, zeroinitializer
+  %e = select <16 x i1> %d, <16 x i32> %c, <16 x i32> zeroinitializer
+  %f = trunc <16 x i32> %e to <16 x i8>
+  ret <16 x i8> %f
+}
+
+define void @trunc_packus_v16i32_v16i8_store(<16 x i32>* %p, <16 x i8>* %q) "min-legal-vector-width"="256" {
+; CHECK-LABEL: trunc_packus_v16i32_v16i8_store:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovdqa (%rdi), %ymm0
+; CHECK-NEXT:    vpackusdw 32(%rdi), %ymm0, %ymm0
+; CHECK-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; CHECK-NEXT:    vpmovuswb %ymm0, (%rsi)
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %a = load <16 x i32>, <16 x i32>* %p
+  %b = icmp slt <16 x i32> %a, <i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255>
+  %c = select <16 x i1> %b, <16 x i32> %a, <16 x i32> <i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255>
+  %d = icmp sgt <16 x i32> %c, zeroinitializer
+  %e = select <16 x i1> %d, <16 x i32> %c, <16 x i32> zeroinitializer
+  %f = trunc <16 x i32> %e to <16 x i8>
+  store <16 x i8> %f, <16 x i8>* %q
+  ret void
+}
+
+define <64 x i1> @v64i1_argument_return(<64 x i1> %x) "min-legal-vector-width"="256" {
+; CHECK-LABEL: v64i1_argument_return:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    retq
+  ret <64 x i1> %x
+}
+
+define void @v64i1_shuffle(<64 x i8>* %x, <64 x i8>* %y) "min-legal-vector-width"="256" {
+; CHECK-LABEL: v64i1_shuffle:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmovdqa (%rdi), %ymm1
+; CHECK-NEXT:    vmovdqa 32(%rdi), %ymm0
+; CHECK-NEXT:    vptestnmb %ymm1, %ymm1, %k0
+; CHECK-NEXT:    kshiftrd $1, %k0, %k1
+; CHECK-NEXT:    movq $-3, %rax
+; CHECK-NEXT:    kmovq %rax, %k2
+; CHECK-NEXT:    kandq %k2, %k1, %k1
+; CHECK-NEXT:    kshiftlq $63, %k0, %k2
+; CHECK-NEXT:    kshiftrq $62, %k2, %k2
+; CHECK-NEXT:    korq %k2, %k1, %k1
+; CHECK-NEXT:    movq $-5, %rax
+; CHECK-NEXT:    kmovq %rax, %k2
+; CHECK-NEXT:    kandq %k2, %k1, %k1
+; CHECK-NEXT:    kshiftrd $3, %k0, %k2
+; CHECK-NEXT:    kshiftlq $63, %k2, %k2
+; CHECK-NEXT:    kshiftrq $61, %k2, %k2
+; CHECK-NEXT:    korq %k2, %k1, %k1
+; CHECK-NEXT:    movq $-9, %rax
+; CHECK-NEXT:    kmovq %rax, %k2
+; CHECK-NEXT:    kandq %k2, %k1, %k1
+; CHECK-NEXT:    kshiftrd $2, %k0, %k2
+; CHECK-NEXT:    kshiftlq $63, %k2, %k2
+; CHECK-NEXT:    kshiftrq $60, %k2, %k2
+; CHECK-NEXT:    korq %k2, %k1, %k1
+; CHECK-NEXT:    movq $-17, %rax
+; CHECK-NEXT:    kmovq %rax, %k2
+; CHECK-NEXT:    kandq %k2, %k1, %k1
+; CHECK-NEXT:    kshiftrd $5, %k0, %k2
+; CHECK-NEXT:    kshiftlq $63, %k2, %k2
+; CHECK-NEXT:    kshiftrq $59, %k2, %k2
+; CHECK-NEXT:    korq %k2, %k1, %k1
+; CHECK-NEXT:    movq $-33, %rax
+; CHECK-NEXT:    kmovq %rax, %k2
+; CHECK-NEXT:    kandq %k2, %k1, %k1
+; CHECK-NEXT:    kshiftrd $4, %k0, %k2
+; CHECK-NEXT:    kshiftlq $63, %k2, %k2
+; CHECK-NEXT:    kshiftrq $58, %k2, %k2
+; CHECK-NEXT:    korq %k2, %k1, %k1
+; CHECK-NEXT:    movq $-65, %rax
+; CHECK-NEXT:    kmovq %rax, %k2
+; CHECK-NEXT:    kandq %k2, %k1, %k1
+; CHECK-NEXT:    kshiftrd $7, %k0, %k2
+; CHECK-NEXT:    kshiftlq $63, %k2, %k2
+; CHECK-NEXT:    kshiftrq $57, %k2, %k2
+; CHECK-NEXT:    korq %k2, %k1, %k1
+; CHECK-NEXT:    movq $-129, %rax
+; CHECK-NEXT:    kmovq %rax, %k2
+; CHECK-NEXT:    kandq %k2, %k1, %k1
+; CHECK-NEXT:    kshiftrd $6, %k0, %k2
+; CHECK-NEXT:    kshiftlq $63, %k2, %k2
+; CHECK-NEXT:    kshiftrq $56, %k2, %k2
+; CHECK-NEXT:    korq %k2, %k1, %k1
+; CHECK-NEXT:    movq $-257, %rax # imm = 0xFEFF
+; CHECK-NEXT:    kmovq %rax, %k2
+; CHECK-NEXT:    kandq %k2, %k1, %k1
+; CHECK-NEXT:    kshiftrd $9, %k0, %k2
+; CHECK-NEXT:    kshiftlq $63, %k2, %k2
+; CHECK-NEXT:    kshiftrq $55, %k2, %k2
+; CHECK-NEXT:    korq %k2, %k1, %k1
+; CHECK-NEXT:    movq $-513, %rax # imm = 0xFDFF
+; CHECK-NEXT:    kmovq %rax, %k2
+; CHECK-NEXT:    kandq %k2, %k1, %k1
+; CHECK-NEXT:    kshiftrd $8, %k0, %k2
+; CHECK-NEXT:    kshiftlq $63, %k2, %k2
+; CHECK-NEXT:    kshiftrq $54, %k2, %k2
+; CHECK-NEXT:    korq %k2, %k1, %k1
+; CHECK-NEXT:    movq $-1025, %rax # imm = 0xFBFF
+; CHECK-NEXT:    kmovq %rax, %k2
+; CHECK-NEXT:    kandq %k2, %k1, %k1
+; CHECK-NEXT:    kshiftrd $11, %k0, %k2
+; CHECK-NEXT:    kshiftlq $63, %k2, %k2
+; CHECK-NEXT:    kshiftrq $53, %k2, %k2
+; CHECK-NEXT:    korq %k2, %k1, %k1
+; CHECK-NEXT:    movq $-2049, %rax # imm = 0xF7FF
+; CHECK-NEXT:    kmovq %rax, %k2
+; CHECK-NEXT:    kandq %k2, %k1, %k1
+; CHECK-NEXT:    kshiftrd $10, %k0, %k2
+; CHECK-NEXT:    kshiftlq $63, %k2, %k2
+; CHECK-NEXT:    kshiftrq $52, %k2, %k2
+; CHECK-NEXT:    korq %k2, %k1, %k1
+; CHECK-NEXT:    movq $-4097, %rax # imm = 0xEFFF
+; CHECK-NEXT:    kmovq %rax, %k2
+; CHECK-NEXT:    kandq %k2, %k1, %k1
+; CHECK-NEXT:    kshiftrd $13, %k0, %k2
+; CHECK-NEXT:    kshiftlq $63, %k2, %k2
+; CHECK-NEXT:    kshiftrq $51, %k2, %k2
+; CHECK-NEXT:    korq %k2, %k1, %k1
+; CHECK-NEXT:    movq $-8193, %rax # imm = 0xDFFF
+; CHECK-NEXT:    kmovq %rax, %k2
+; CHECK-NEXT:    kandq %k2, %k1, %k1
+; CHECK-NEXT:    kshiftrd $12, %k0, %k2
+; CHECK-NEXT:    kshiftlq $63, %k2, %k2
+; CHECK-NEXT:    kshiftrq $50, %k2, %k2
+; CHECK-NEXT:    korq %k2, %k1, %k1
+; CHECK-NEXT:    movq $-16385, %rax # imm = 0xBFFF
+; CHECK-NEXT:    kmovq %rax, %k2
+; CHECK-NEXT:    kandq %k2, %k1, %k1
+; CHECK-NEXT:    kshiftrd $15, %k0, %k2
+; CHECK-NEXT:    kshiftlq $63, %k2, %k2
+; CHECK-NEXT:    kshiftrq $49, %k2, %k2
+; CHECK-NEXT:    korq %k2, %k1, %k1
+; CHECK-NEXT:    movq $-32769, %rax # imm = 0xFFFF7FFF
+; CHECK-NEXT:    kmovq %rax, %k2
+; CHECK-NEXT:    kandq %k2, %k1, %k1
+; CHECK-NEXT:    kshiftrd $14, %k0, %k2
+; CHECK-NEXT:    kshiftlq $63, %k2, %k2
+; CHECK-NEXT:    kshiftrq $48, %k2, %k2
+; CHECK-NEXT:    korq %k2, %k1, %k1
+; CHECK-NEXT:    movq $-65537, %rax # imm = 0xFFFEFFFF
+; CHECK-NEXT:    kmovq %rax, %k2
+; CHECK-NEXT:    kandq %k2, %k1, %k1
+; CHECK-NEXT:    kshiftrd $17, %k0, %k2
+; CHECK-NEXT:    kshiftlq $63, %k2, %k2
+; CHECK-NEXT:    kshiftrq $47, %k2, %k2
+; CHECK-NEXT:    korq %k2, %k1, %k1
+; CHECK-NEXT:    movq $-131073, %rax # imm = 0xFFFDFFFF
+; CHECK-NEXT:    kmovq %rax, %k2
+; CHECK-NEXT:    kandq %k2, %k1, %k1
+; CHECK-NEXT:    kshiftrd $16, %k0, %k2
+; CHECK-NEXT:    kshiftlq $63, %k2, %k2
+; CHECK-NEXT:    kshiftrq $46, %k2, %k2
+; CHECK-NEXT:    korq %k2, %k1, %k1
+; CHECK-NEXT:    movq $-262145, %rax # imm = 0xFFFBFFFF
+; CHECK-NEXT:    kmovq %rax, %k2
+; CHECK-NEXT:    kandq %k2, %k1, %k1
+; CHECK-NEXT:    kshiftrd $19, %k0, %k2
+; CHECK-NEXT:    kshiftlq $63, %k2, %k2
+; CHECK-NEXT:    kshiftrq $45, %k2, %k2
+; CHECK-NEXT:    korq %k2, %k1, %k1
+; CHECK-NEXT:    movq $-524289, %rax # imm = 0xFFF7FFFF
+; CHECK-NEXT:    kmovq %rax, %k2
+; CHECK-NEXT:    kandq %k2, %k1, %k1
+; CHECK-NEXT:    kshiftrd $18, %k0, %k2
+; CHECK-NEXT:    kshiftlq $63, %k2, %k2
+; CHECK-NEXT:    kshiftrq $44, %k2, %k2
+; CHECK-NEXT:    korq %k2, %k1, %k1
+; CHECK-NEXT:    movq $-1048577, %rax # imm = 0xFFEFFFFF
+; CHECK-NEXT:    kmovq %rax, %k2
+; CHECK-NEXT:    kandq %k2, %k1, %k1
+; CHECK-NEXT:    kshiftrd $21, %k0, %k2
+; CHECK-NEXT:    kshiftlq $63, %k2, %k2
+; CHECK-NEXT:    kshiftrq $43, %k2, %k2
+; CHECK-NEXT:    korq %k2, %k1, %k1
+; CHECK-NEXT:    movq $-2097153, %rax # imm = 0xFFDFFFFF
+; CHECK-NEXT:    kmovq %rax, %k2
+; CHECK-NEXT:    kandq %k2, %k1, %k1
+; CHECK-NEXT:    kshiftrd $20, %k0, %k2
+; CHECK-NEXT:    kshiftlq $63, %k2, %k2
+; CHECK-NEXT:    kshiftrq $42, %k2, %k2
+; CHECK-NEXT:    korq %k2, %k1, %k1
+; CHECK-NEXT:    movq $-4194305, %rax # imm = 0xFFBFFFFF
+; CHECK-NEXT:    kmovq %rax, %k2
+; CHECK-NEXT:    kandq %k2, %k1, %k1
+; CHECK-NEXT:    kshiftrd $23, %k0, %k2
+; CHECK-NEXT:    kshiftlq $63, %k2, %k2
+; CHECK-NEXT:    kshiftrq $41, %k2, %k2
+; CHECK-NEXT:    korq %k2, %k1, %k1
+; CHECK-NEXT:    movq $-8388609, %rax # imm = 0xFF7FFFFF
+; CHECK-NEXT:    kmovq %rax, %k2
+; CHECK-NEXT:    kandq %k2, %k1, %k1
+; CHECK-NEXT:    kshiftrd $22, %k0, %k2
+; CHECK-NEXT:    kshiftlq $63, %k2, %k2
+; CHECK-NEXT:    kshiftrq $40, %k2, %k2
+; CHECK-NEXT:    korq %k2, %k1, %k1
+; CHECK-NEXT:    movq $-16777217, %rax # imm = 0xFEFFFFFF
+; CHECK-NEXT:    kmovq %rax, %k2
+; CHECK-NEXT:    kandq %k2, %k1, %k1
+; CHECK-NEXT:    kshiftrd $25, %k0, %k2
+; CHECK-NEXT:    kshiftlq $63, %k2, %k2
+; CHECK-NEXT:    kshiftrq $39, %k2, %k2
+; CHECK-NEXT:    korq %k2, %k1, %k1
+; CHECK-NEXT:    movq $-33554433, %rax # imm = 0xFDFFFFFF
+; CHECK-NEXT:    kmovq %rax, %k2
+; CHECK-NEXT:    kandq %k2, %k1, %k1
+; CHECK-NEXT:    kshiftrd $24, %k0, %k2
+; CHECK-NEXT:    kshiftlq $63, %k2, %k2
+; CHECK-NEXT:    kshiftrq $38, %k2, %k2
+; CHECK-NEXT:    korq %k2, %k1, %k1
+; CHECK-NEXT:    movq $-67108865, %rax # imm = 0xFBFFFFFF
+; CHECK-NEXT:    kmovq %rax, %k2
+; CHECK-NEXT:    kandq %k2, %k1, %k1
+; CHECK-NEXT:    kshiftrd $27, %k0, %k2
+; CHECK-NEXT:    kshiftlq $63, %k2, %k2
+; CHECK-NEXT:    kshiftrq $37, %k2, %k2
+; CHECK-NEXT:    korq %k2, %k1, %k1
+; CHECK-NEXT:    movq $-134217729, %rax # imm = 0xF7FFFFFF
+; CHECK-NEXT:    kmovq %rax, %k2
+; CHECK-NEXT:    kandq %k2, %k1, %k1
+; CHECK-NEXT:    kshiftrd $26, %k0, %k2
+; CHECK-NEXT:    kshiftlq $63, %k2, %k2
+; CHECK-NEXT:    kshiftrq $36, %k2, %k2
+; CHECK-NEXT:    korq %k2, %k1, %k1
+; CHECK-NEXT:    movq $-268435457, %rax # imm = 0xEFFFFFFF
+; CHECK-NEXT:    kmovq %rax, %k2
+; CHECK-NEXT:    kandq %k2, %k1, %k1
+; CHECK-NEXT:    kshiftrd $29, %k0, %k2
+; CHECK-NEXT:    kshiftlq $63, %k2, %k2
+; CHECK-NEXT:    kshiftrq $35, %k2, %k2
+; CHECK-NEXT:    korq %k2, %k1, %k1
+; CHECK-NEXT:    movq $-536870913, %rax # imm = 0xDFFFFFFF
+; CHECK-NEXT:    kmovq %rax, %k2
+; CHECK-NEXT:    kandq %k2, %k1, %k1
+; CHECK-NEXT:    kshiftrd $28, %k0, %k2
+; CHECK-NEXT:    kshiftlq $63, %k2, %k2
+; CHECK-NEXT:    kshiftrq $34, %k2, %k2
+; CHECK-NEXT:    korq %k2, %k1, %k1
+; CHECK-NEXT:    movq $-1073741825, %rax # imm = 0xBFFFFFFF
+; CHECK-NEXT:    kmovq %rax, %k2
+; CHECK-NEXT:    kandq %k2, %k1, %k1
+; CHECK-NEXT:    kshiftrd $31, %k0, %k2
+; CHECK-NEXT:    kshiftlq $63, %k2, %k2
+; CHECK-NEXT:    kshiftrq $33, %k2, %k2
+; CHECK-NEXT:    korq %k2, %k1, %k1
+; CHECK-NEXT:    movabsq $-2147483649, %rax # imm = 0xFFFFFFFF7FFFFFFF
+; CHECK-NEXT:    kmovq %rax, %k2
+; CHECK-NEXT:    kandq %k2, %k1, %k2
+; CHECK-NEXT:    vptestnmb %ymm0, %ymm0, %k1
+; CHECK-NEXT:    kshiftrd $30, %k0, %k0
+; CHECK-NEXT:    kshiftlq $63, %k0, %k0
+; CHECK-NEXT:    kshiftrq $32, %k0, %k0
+; CHECK-NEXT:    korq %k0, %k2, %k0
+; CHECK-NEXT:    movabsq $-4294967297, %rax # imm = 0xFFFFFFFEFFFFFFFF
+; CHECK-NEXT:    kmovq %rax, %k2
+; CHECK-NEXT:    kandq %k2, %k0, %k0
+; CHECK-NEXT:    kshiftrd $1, %k1, %k2
+; CHECK-NEXT:    kshiftlq $63, %k2, %k2
+; CHECK-NEXT:    kshiftrq $31, %k2, %k2
+; CHECK-NEXT:    korq %k2, %k0, %k0
+; CHECK-NEXT:    movabsq $-8589934593, %rax # imm = 0xFFFFFFFDFFFFFFFF
+; CHECK-NEXT:    kmovq %rax, %k2
+; CHECK-NEXT:    kandq %k2, %k0, %k0
+; CHECK-NEXT:    kshiftlq $63, %k1, %k2
+; CHECK-NEXT:    kshiftrq $30, %k2, %k2
+; CHECK-NEXT:    korq %k2, %k0, %k0
+; CHECK-NEXT:    movabsq $-17179869185, %rax # imm = 0xFFFFFFFBFFFFFFFF
+; CHECK-NEXT:    kmovq %rax, %k2
+; CHECK-NEXT:    kandq %k2, %k0, %k0
+; CHECK-NEXT:    kshiftrd $3, %k1, %k2
+; CHECK-NEXT:    kshiftlq $63, %k2, %k2
+; CHECK-NEXT:    kshiftrq $29, %k2, %k2
+; CHECK-NEXT:    korq %k2, %k0, %k0
+; CHECK-NEXT:    movabsq $-34359738369, %rax # imm = 0xFFFFFFF7FFFFFFFF
+; CHECK-NEXT:    kmovq %rax, %k2
+; CHECK-NEXT:    kandq %k2, %k0, %k0
+; CHECK-NEXT:    kshiftrd $2, %k1, %k2
+; CHECK-NEXT:    kshiftlq $63, %k2, %k2
+; CHECK-NEXT:    kshiftrq $28, %k2, %k2
+; CHECK-NEXT:    korq %k2, %k0, %k0
+; CHECK-NEXT:    movabsq $-68719476737, %rax # imm = 0xFFFFFFEFFFFFFFFF
+; CHECK-NEXT:    kmovq %rax, %k2
+; CHECK-NEXT:    kandq %k2, %k0, %k0
+; CHECK-NEXT:    kshiftrd $5, %k1, %k2
+; CHECK-NEXT:    kshiftlq $63, %k2, %k2
+; CHECK-NEXT:    kshiftrq $27, %k2, %k2
+; CHECK-NEXT:    korq %k2, %k0, %k0
+; CHECK-NEXT:    movabsq $-137438953473, %rax # imm = 0xFFFFFFDFFFFFFFFF
+; CHECK-NEXT:    kmovq %rax, %k2
+; CHECK-NEXT:    kandq %k2, %k0, %k0
+; CHECK-NEXT:    kshiftrd $4, %k1, %k2
+; CHECK-NEXT:    kshiftlq $63, %k2, %k2
+; CHECK-NEXT:    kshiftrq $26, %k2, %k2
+; CHECK-NEXT:    korq %k2, %k0, %k0
+; CHECK-NEXT:    movabsq $-274877906945, %rax # imm = 0xFFFFFFBFFFFFFFFF
+; CHECK-NEXT:    kmovq %rax, %k2
+; CHECK-NEXT:    kandq %k2, %k0, %k0
+; CHECK-NEXT:    kshiftrd $7, %k1, %k2
+; CHECK-NEXT:    kshiftlq $63, %k2, %k2
+; CHECK-NEXT:    kshiftrq $25, %k2, %k2
+; CHECK-NEXT:    korq %k2, %k0, %k0
+; CHECK-NEXT:    movabsq $-549755813889, %rax # imm = 0xFFFFFF7FFFFFFFFF
+; CHECK-NEXT:    kmovq %rax, %k2
+; CHECK-NEXT:    kandq %k2, %k0, %k0
+; CHECK-NEXT:    kshiftrd $6, %k1, %k2
+; CHECK-NEXT:    kshiftlq $63, %k2, %k2
+; CHECK-NEXT:    kshiftrq $24, %k2, %k2
+; CHECK-NEXT:    korq %k2, %k0, %k0
+; CHECK-NEXT:    movabsq $-1099511627777, %rax # imm = 0xFFFFFEFFFFFFFFFF
+; CHECK-NEXT:    kmovq %rax, %k2
+; CHECK-NEXT:    kandq %k2, %k0, %k0
+; CHECK-NEXT:    kshiftrd $9, %k1, %k2
+; CHECK-NEXT:    kshiftlq $63, %k2, %k2
+; CHECK-NEXT:    kshiftrq $23, %k2, %k2
+; CHECK-NEXT:    korq %k2, %k0, %k0
+; CHECK-NEXT:    movabsq $-2199023255553, %rax # imm = 0xFFFFFDFFFFFFFFFF
+; CHECK-NEXT:    kmovq %rax, %k2
+; CHECK-NEXT:    kandq %k2, %k0, %k0
+; CHECK-NEXT:    kshiftrd $8, %k1, %k2
+; CHECK-NEXT:    kshiftlq $63, %k2, %k2
+; CHECK-NEXT:    kshiftrq $22, %k2, %k2
+; CHECK-NEXT:    korq %k2, %k0, %k0
+; CHECK-NEXT:    movabsq $-4398046511105, %rax # imm = 0xFFFFFBFFFFFFFFFF
+; CHECK-NEXT:    kmovq %rax, %k2
+; CHECK-NEXT:    kandq %k2, %k0, %k0
+; CHECK-NEXT:    kshiftrd $11, %k1, %k2
+; CHECK-NEXT:    kshiftlq $63, %k2, %k2
+; CHECK-NEXT:    kshiftrq $21, %k2, %k2
+; CHECK-NEXT:    korq %k2, %k0, %k0
+; CHECK-NEXT:    movabsq $-8796093022209, %rax # imm = 0xFFFFF7FFFFFFFFFF
+; CHECK-NEXT:    kmovq %rax, %k2
+; CHECK-NEXT:    kandq %k2, %k0, %k0
+; CHECK-NEXT:    kshiftrd $10, %k1, %k2
+; CHECK-NEXT:    kshiftlq $63, %k2, %k2
+; CHECK-NEXT:    kshiftrq $20, %k2, %k2
+; CHECK-NEXT:    korq %k2, %k0, %k0
+; CHECK-NEXT:    movabsq $-17592186044417, %rax # imm = 0xFFFFEFFFFFFFFFFF
+; CHECK-NEXT:    kmovq %rax, %k2
+; CHECK-NEXT:    kandq %k2, %k0, %k0
+; CHECK-NEXT:    kshiftrd $13, %k1, %k2
+; CHECK-NEXT:    kshiftlq $63, %k2, %k2
+; CHECK-NEXT:    kshiftrq $19, %k2, %k2
+; CHECK-NEXT:    korq %k2, %k0, %k0
+; CHECK-NEXT:    movabsq $-35184372088833, %rax # imm = 0xFFFFDFFFFFFFFFFF
+; CHECK-NEXT:    kmovq %rax, %k2
+; CHECK-NEXT:    kandq %k2, %k0, %k0
+; CHECK-NEXT:    kshiftrd $12, %k1, %k2
+; CHECK-NEXT:    kshiftlq $63, %k2, %k2
+; CHECK-NEXT:    kshiftrq $18, %k2, %k2
+; CHECK-NEXT:    korq %k2, %k0, %k0
+; CHECK-NEXT:    movabsq $-70368744177665, %rax # imm = 0xFFFFBFFFFFFFFFFF
+; CHECK-NEXT:    kmovq %rax, %k2
+; CHECK-NEXT:    kandq %k2, %k0, %k0
+; CHECK-NEXT:    kshiftrd $15, %k1, %k2
+; CHECK-NEXT:    kshiftlq $63, %k2, %k2
+; CHECK-NEXT:    kshiftrq $17, %k2, %k2
+; CHECK-NEXT:    korq %k2, %k0, %k0
+; CHECK-NEXT:    movabsq $-140737488355329, %rax # imm = 0xFFFF7FFFFFFFFFFF
+; CHECK-NEXT:    kmovq %rax, %k2
+; CHECK-NEXT:    kandq %k2, %k0, %k0
+; CHECK-NEXT:    kshiftrd $14, %k1, %k2
+; CHECK-NEXT:    kshiftlq $63, %k2, %k2
+; CHECK-NEXT:    kshiftrq $16, %k2, %k2
+; CHECK-NEXT:    korq %k2, %k0, %k0
+; CHECK-NEXT:    movabsq $-281474976710657, %rax # imm = 0xFFFEFFFFFFFFFFFF
+; CHECK-NEXT:    kmovq %rax, %k2
+; CHECK-NEXT:    kandq %k2, %k0, %k0
+; CHECK-NEXT:    kshiftrd $17, %k1, %k2
+; CHECK-NEXT:    kshiftlq $63, %k2, %k2
+; CHECK-NEXT:    kshiftrq $15, %k2, %k2
+; CHECK-NEXT:    korq %k2, %k0, %k0
+; CHECK-NEXT:    movabsq $-562949953421313, %rax # imm = 0xFFFDFFFFFFFFFFFF
+; CHECK-NEXT:    kmovq %rax, %k2
+; CHECK-NEXT:    kandq %k2, %k0, %k0
+; CHECK-NEXT:    kshiftrd $16, %k1, %k2
+; CHECK-NEXT:    kshiftlq $63, %k2, %k2
+; CHECK-NEXT:    kshiftrq $14, %k2, %k2
+; CHECK-NEXT:    korq %k2, %k0, %k0
+; CHECK-NEXT:    movabsq $-1125899906842625, %rax # imm = 0xFFFBFFFFFFFFFFFF
+; CHECK-NEXT:    kmovq %rax, %k2
+; CHECK-NEXT:    kandq %k2, %k0, %k0
+; CHECK-NEXT:    kshiftrd $19, %k1, %k2
+; CHECK-NEXT:    kshiftlq $63, %k2, %k2
+; CHECK-NEXT:    kshiftrq $13, %k2, %k2
+; CHECK-NEXT:    korq %k2, %k0, %k0
+; CHECK-NEXT:    movabsq $-2251799813685249, %rax # imm = 0xFFF7FFFFFFFFFFFF
+; CHECK-NEXT:    kmovq %rax, %k2
+; CHECK-NEXT:    kandq %k2, %k0, %k0
+; CHECK-NEXT:    kshiftrd $18, %k1, %k2
+; CHECK-NEXT:    kshiftlq $63, %k2, %k2
+; CHECK-NEXT:    kshiftrq $12, %k2, %k2
+; CHECK-NEXT:    korq %k2, %k0, %k0
+; CHECK-NEXT:    movabsq $-4503599627370497, %rax # imm = 0xFFEFFFFFFFFFFFFF
+; CHECK-NEXT:    kmovq %rax, %k2
+; CHECK-NEXT:    kandq %k2, %k0, %k0
+; CHECK-NEXT:    kshiftrd $21, %k1, %k2
+; CHECK-NEXT:    kshiftlq $63, %k2, %k2
+; CHECK-NEXT:    kshiftrq $11, %k2, %k2
+; CHECK-NEXT:    korq %k2, %k0, %k0
+; CHECK-NEXT:    movabsq $-9007199254740993, %rax # imm = 0xFFDFFFFFFFFFFFFF
+; CHECK-NEXT:    kmovq %rax, %k2
+; CHECK-NEXT:    kandq %k2, %k0, %k0
+; CHECK-NEXT:    kshiftrd $20, %k1, %k2
+; CHECK-NEXT:    kshiftlq $63, %k2, %k2
+; CHECK-NEXT:    kshiftrq $10, %k2, %k2
+; CHECK-NEXT:    korq %k2, %k0, %k0
+; CHECK-NEXT:    movabsq $-18014398509481985, %rax # imm = 0xFFBFFFFFFFFFFFFF
+; CHECK-NEXT:    kmovq %rax, %k2
+; CHECK-NEXT:    kandq %k2, %k0, %k0
+; CHECK-NEXT:    kshiftrd $23, %k1, %k2
+; CHECK-NEXT:    kshiftlq $63, %k2, %k2
+; CHECK-NEXT:    kshiftrq $9, %k2, %k2
+; CHECK-NEXT:    korq %k2, %k0, %k0
+; CHECK-NEXT:    movabsq $-36028797018963969, %rax # imm = 0xFF7FFFFFFFFFFFFF
+; CHECK-NEXT:    kmovq %rax, %k2
+; CHECK-NEXT:    kandq %k2, %k0, %k0
+; CHECK-NEXT:    kshiftrd $22, %k1, %k2
+; CHECK-NEXT:    kshiftlq $63, %k2, %k2
+; CHECK-NEXT:    kshiftrq $8, %k2, %k2
+; CHECK-NEXT:    korq %k2, %k0, %k0
+; CHECK-NEXT:    movabsq $-72057594037927937, %rax # imm = 0xFEFFFFFFFFFFFFFF
+; CHECK-NEXT:    kmovq %rax, %k2
+; CHECK-NEXT:    kandq %k2, %k0, %k0
+; CHECK-NEXT:    kshiftrd $25, %k1, %k2
+; CHECK-NEXT:    kshiftlq $63, %k2, %k2
+; CHECK-NEXT:    kshiftrq $7, %k2, %k2
+; CHECK-NEXT:    korq %k2, %k0, %k0
+; CHECK-NEXT:    movabsq $-144115188075855873, %rax # imm = 0xFDFFFFFFFFFFFFFF
+; CHECK-NEXT:    kmovq %rax, %k2
+; CHECK-NEXT:    kandq %k2, %k0, %k0
+; CHECK-NEXT:    kshiftrd $24, %k1, %k2
+; CHECK-NEXT:    kshiftlq $63, %k2, %k2
+; CHECK-NEXT:    kshiftrq $6, %k2, %k2
+; CHECK-NEXT:    korq %k2, %k0, %k0
+; CHECK-NEXT:    movabsq $-288230376151711745, %rax # imm = 0xFBFFFFFFFFFFFFFF
+; CHECK-NEXT:    kmovq %rax, %k2
+; CHECK-NEXT:    kandq %k2, %k0, %k0
+; CHECK-NEXT:    kshiftrd $27, %k1, %k2
+; CHECK-NEXT:    kshiftlq $63, %k2, %k2
+; CHECK-NEXT:    kshiftrq $5, %k2, %k2
+; CHECK-NEXT:    korq %k2, %k0, %k0
+; CHECK-NEXT:    movabsq $-576460752303423489, %rax # imm = 0xF7FFFFFFFFFFFFFF
+; CHECK-NEXT:    kmovq %rax, %k2
+; CHECK-NEXT:    kandq %k2, %k0, %k0
+; CHECK-NEXT:    kshiftrd $26, %k1, %k2
+; CHECK-NEXT:    kshiftlq $63, %k2, %k2
+; CHECK-NEXT:    kshiftrq $4, %k2, %k2
+; CHECK-NEXT:    korq %k2, %k0, %k0
+; CHECK-NEXT:    movabsq $-1152921504606846977, %rax # imm = 0xEFFFFFFFFFFFFFFF
+; CHECK-NEXT:    kmovq %rax, %k2
+; CHECK-NEXT:    kandq %k2, %k0, %k0
+; CHECK-NEXT:    kshiftrd $29, %k1, %k2
+; CHECK-NEXT:    kshiftlq $63, %k2, %k2
+; CHECK-NEXT:    kshiftrq $3, %k2, %k2
+; CHECK-NEXT:    korq %k2, %k0, %k0
+; CHECK-NEXT:    movabsq $-2305843009213693953, %rax # imm = 0xDFFFFFFFFFFFFFFF
+; CHECK-NEXT:    kmovq %rax, %k2
+; CHECK-NEXT:    kandq %k2, %k0, %k0
+; CHECK-NEXT:    kshiftrd $28, %k1, %k2
+; CHECK-NEXT:    kshiftlq $63, %k2, %k2
+; CHECK-NEXT:    kshiftrq $2, %k2, %k2
+; CHECK-NEXT:    korq %k2, %k0, %k0
+; CHECK-NEXT:    movabsq $-4611686018427387905, %rax # imm = 0xBFFFFFFFFFFFFFFF
+; CHECK-NEXT:    kmovq %rax, %k2
+; CHECK-NEXT:    kandq %k2, %k0, %k0
+; CHECK-NEXT:    kshiftrd $31, %k1, %k2
+; CHECK-NEXT:    kshiftlq $62, %k2, %k2
+; CHECK-NEXT:    korq %k2, %k0, %k0
+; CHECK-NEXT:    kshiftrd $30, %k1, %k1
+; CHECK-NEXT:    kshiftlq $1, %k0, %k0
+; CHECK-NEXT:    kshiftrq $1, %k0, %k0
+; CHECK-NEXT:    kshiftlq $63, %k1, %k1
+; CHECK-NEXT:    korq %k1, %k0, %k1
+; CHECK-NEXT:    vmovdqu8 %ymm1, (%rsi) {%k1}
+; CHECK-NEXT:    kshiftrq $32, %k1, %k1
+; CHECK-NEXT:    vmovdqu8 %ymm0, 32(%rsi) {%k1}
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+entry:
+  %a = load <64 x i8>, <64 x i8>* %x
+  %b = icmp eq <64 x i8> %a, zeroinitializer
+  %shuf = shufflevector <64 x i1> %b, <64 x i1> undef, <64 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14, i32 17, i32 16, i32 19, i32 18, i32 21, i32 20, i32 23, i32 22, i32 25, i32 24, i32 27, i32 26, i32 29, i32 28, i32 31, i32 30, i32 33, i32 32, i32 35, i32 34, i32 37, i32 36, i32 39, i32 38, i32 41, i32 40, i32 43, i32 42, i32 45, i32 44, i32 47, i32 46, i32 49, i32 48, i32 51, i32 50, i32 53, i32 52, i32 55, i32 54, i32 57, i32 56, i32 59, i32 58, i32 61, i32 60, i32 63, i32 62>
+  call void @llvm.masked.store.v64i8.p0v64i8(<64 x i8> %a, <64 x i8>* %y, i32 1, <64 x i1> %shuf)
+  ret void
+}
+declare void @llvm.masked.store.v64i8.p0v64i8(<64 x i8>, <64 x i8>*, i32, <64 x i1>)
+
+@mem64_dst = global i64 0, align 8
+@mem64_src = global i64 0, align 8
+define i32 @v64i1_inline_asm() "min-legal-vector-width"="256" {
+; CHECK-LABEL: v64i1_inline_asm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    kmovq {{.*}}(%rip), %k0
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovq %k0, {{.*}}(%rip)
+; CHECK-NEXT:    movl -{{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    retq
+  %1 = alloca i32, align 4
+  %2 = load i64, i64* @mem64_src, align 8
+  %3 = call i64 asm "", "=k,k,~{dirflag},~{fpsr},~{flags}"(i64 %2)
+  store i64 %3, i64* @mem64_dst, align 8
+  %4 = load i32, i32* %1, align 4
+  ret i32 %4
+}
diff --git a/llvm/test/CodeGen/X86/misched-copy.ll b/llvm/test/CodeGen/X86/misched-copy.ll
index 3a1e0fae6..fa6cd157d 100644
--- a/llvm/test/CodeGen/X86/misched-copy.ll
+++ b/llvm/test/CodeGen/X86/misched-copy.ll
@@ -42,7 +42,7 @@ end:
   ret i64 %add
 }
 
-attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !0 = !{!"float", !1}
 !1 = !{!"omnipotent char", !2}
diff --git a/llvm/test/CodeGen/X86/misched-matmul.ll b/llvm/test/CodeGen/X86/misched-matmul.ll
index 7de9af290..f5547d049 100644
--- a/llvm/test/CodeGen/X86/misched-matmul.ll
+++ b/llvm/test/CodeGen/X86/misched-matmul.ll
@@ -224,4 +224,4 @@ entry:
   ret void
 }
 
-attributes #0 = { noinline nounwind ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noinline nounwind ssp uwtable "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/X86/misched_phys_reg_assign_order.ll b/llvm/test/CodeGen/X86/misched_phys_reg_assign_order.ll
index fd40d7f92..7231ea35b 100644
--- a/llvm/test/CodeGen/X86/misched_phys_reg_assign_order.ll
+++ b/llvm/test/CodeGen/X86/misched_phys_reg_assign_order.ll
@@ -49,4 +49,4 @@ k.end:                                            ; preds = %entry
 
 declare i32 @m()
 
-attributes #0 = { noimplicitfloat "no-frame-pointer-elim-non-leaf" }
+attributes #0 = { noimplicitfloat "frame-pointer"="non-leaf" }
diff --git a/llvm/test/CodeGen/X86/mixed-ptr-sizes.ll b/llvm/test/CodeGen/X86/mixed-ptr-sizes.ll
new file mode 100644
index 000000000..27cd7b98f
--- /dev/null
+++ b/llvm/test/CodeGen/X86/mixed-ptr-sizes.ll
@@ -0,0 +1,128 @@
+; RUN: llc < %s | FileCheck --check-prefixes=CHECK %s
+; RUN: llc -O0 < %s | FileCheck --check-prefixes=CHECK %s
+
+; Source to regenerate:
+; struct Foo {
+;   int * __ptr32 p32;
+;   int * __ptr64 p64;
+;   __attribute__((address_space(9))) int *p_other;
+; };
+; void use_foo(Foo *f);
+; void test_sign_ext(Foo *f, int * __ptr32 __sptr i) {
+;   f->p64 = i;
+;   use_foo(f);
+; }
+; void test_zero_ext(Foo *f, int * __ptr32 __uptr i) {
+;   f->p64 = i;
+;   use_foo(f);
+; }
+; void test_trunc(Foo *f, int * __ptr64 i) {
+;   f->p32 = i;
+;   use_foo(f);
+; }
+; void test_noop1(Foo *f, int * __ptr32 i) {
+;   f->p32 = i;
+;   use_foo(f);
+; }
+; void test_noop2(Foo *f, int * __ptr64 i) {
+;   f->p64 = i;
+;   use_foo(f);
+; }
+; void test_null_arg(Foo *f, int * __ptr32 i) {
+;   test_noop1(f, 0);
+; }
+; void test_unrecognized(Foo *f, __attribute__((address_space(14))) int *i) {
+;   f->p32 = (int * __ptr32)i;
+;   use_foo(f);
+; }
+;
+; $ clang -cc1 -triple x86_64-windows-msvc -fms-extensions -O2 -S t.cpp
+
+target datalayout = "e-m:w-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-windows-msvc"
+
+%struct.Foo = type { i32 addrspace(270)*, i32*, i32 addrspace(9)* }
+declare dso_local void @use_foo(%struct.Foo*)
+
+define dso_local void @test_sign_ext(%struct.Foo* %f, i32 addrspace(270)* %i) {
+; CHECK-LABEL: test_sign_ext
+; CHECK:       movslq %edx, %rax
+entry:
+  %0 = addrspacecast i32 addrspace(270)* %i to i32*
+  %p64 = getelementptr inbounds %struct.Foo, %struct.Foo* %f, i64 0, i32 1
+  store i32* %0, i32** %p64, align 8
+  tail call void @use_foo(%struct.Foo* %f)
+  ret void
+}
+
+define dso_local void @test_zero_ext(%struct.Foo* %f, i32 addrspace(271)* %i) {
+; CHECK-LABEL: test_zero_ext
+; CHECK:       movl %edx, %eax
+entry:
+  %0 = addrspacecast i32 addrspace(271)* %i to i32*
+  %p64 = getelementptr inbounds %struct.Foo, %struct.Foo* %f, i64 0, i32 1
+  store i32* %0, i32** %p64, align 8
+  tail call void @use_foo(%struct.Foo* %f)
+  ret void
+}
+
+define dso_local void @test_trunc(%struct.Foo* %f, i32* %i) {
+; CHECK-LABEL: test_trunc
+; CHECK:       movl %edx, (%rcx)
+entry:
+  %0 = addrspacecast i32* %i to i32 addrspace(270)*
+  %p32 = getelementptr inbounds %struct.Foo, %struct.Foo* %f, i64 0, i32 0
+  store i32 addrspace(270)* %0, i32 addrspace(270)** %p32, align 8
+  tail call void @use_foo(%struct.Foo* %f)
+  ret void
+}
+
+define dso_local void @test_noop1(%struct.Foo* %f, i32 addrspace(270)* %i) {
+; CHECK-LABEL: test_noop1
+; CHECK:       movl %edx, (%rcx)
+entry:
+  %p32 = getelementptr inbounds %struct.Foo, %struct.Foo* %f, i64 0, i32 0
+  store i32 addrspace(270)* %i, i32 addrspace(270)** %p32, align 8
+  tail call void @use_foo(%struct.Foo* %f)
+  ret void
+}
+
+define dso_local void @test_noop2(%struct.Foo* %f, i32* %i) {
+; CHECK-LABEL: test_noop2
+; CHECK:       movq %rdx, 8(%rcx)
+entry:
+  %p64 = getelementptr inbounds %struct.Foo, %struct.Foo* %f, i64 0, i32 1
+  store i32* %i, i32** %p64, align 8
+  tail call void @use_foo(%struct.Foo* %f)
+  ret void
+}
+
+; Test that null can be passed as a 32-bit pointer.
+define dso_local void @test_null_arg(%struct.Foo* %f) {
+entry:
+  call void @test_noop1(%struct.Foo* %f, i32 addrspace(270)* null)
+  ret void
+}
+
+; Test casts between unrecognized address spaces.
+define void @test_unrecognized(%struct.Foo* %f, i32 addrspace(14)* %i) {
+; CHECK-LABEL: test_unrecognized
+; CHECK:       movl %edx, (%rcx)
+entry:
+  %0 = addrspacecast i32 addrspace(14)* %i to i32 addrspace(270)*
+  %p32 = getelementptr inbounds %struct.Foo, %struct.Foo* %f, i64 0, i32 0
+  store i32 addrspace(270)* %0, i32 addrspace(270)** %p32, align 8
+  tail call void @use_foo(%struct.Foo* %f)
+  ret void
+}
+
+define void @test_unrecognized2(%struct.Foo* %f, i32 addrspace(271)* %i) {
+; CHECK-LABEL: test_unrecognized2
+; CHECK:       movl %edx, %eax
+entry:
+  %0 = addrspacecast i32 addrspace(271)* %i to i32 addrspace(9)*
+  %p32 = getelementptr inbounds %struct.Foo, %struct.Foo* %f, i64 0, i32 2
+  store i32 addrspace(9)* %0, i32 addrspace(9)** %p32, align 8
+  tail call void @use_foo(%struct.Foo* %f)
+  ret void
+}
diff --git a/llvm/test/CodeGen/X86/mmx-arg-passing-x86-64.ll b/llvm/test/CodeGen/X86/mmx-arg-passing-x86-64.ll
index b88916053..85fa9bb2a 100644
--- a/llvm/test/CodeGen/X86/mmx-arg-passing-x86-64.ll
+++ b/llvm/test/CodeGen/X86/mmx-arg-passing-x86-64.ll
@@ -23,11 +23,9 @@ define void @t4(x86_mmx %v1, x86_mmx %v2) nounwind  {
 ; X86-64-LABEL: t4:
 ; X86-64:       ## %bb.0:
 ; X86-64-NEXT:    movdq2q %xmm1, %mm0
-; X86-64-NEXT:    movq %mm0, -{{[0-9]+}}(%rsp)
-; X86-64-NEXT:    movdq2q %xmm0, %mm0
-; X86-64-NEXT:    movq %mm0, -{{[0-9]+}}(%rsp)
-; X86-64-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
-; X86-64-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; X86-64-NEXT:    movdq2q %xmm0, %mm1
+; X86-64-NEXT:    movq2dq %mm1, %xmm1
+; X86-64-NEXT:    movq2dq %mm0, %xmm0
 ; X86-64-NEXT:    paddb %xmm1, %xmm0
 ; X86-64-NEXT:    movb $1, %al
 ; X86-64-NEXT:    jmp _pass_v8qi ## TAILCALL
diff --git a/llvm/test/CodeGen/X86/mmx-arith.ll b/llvm/test/CodeGen/X86/mmx-arith.ll
index 5df75efb1..055a5a5ef 100644
--- a/llvm/test/CodeGen/X86/mmx-arith.ll
+++ b/llvm/test/CodeGen/X86/mmx-arith.ll
@@ -8,15 +8,8 @@
 define void @test0(x86_mmx* %A, x86_mmx* %B) {
 ; X32-LABEL: test0:
 ; X32:       # %bb.0: # %entry
-; X32-NEXT:    pushl %ebp
-; X32-NEXT:    .cfi_def_cfa_offset 8
-; X32-NEXT:    .cfi_offset %ebp, -8
-; X32-NEXT:    movl %esp, %ebp
-; X32-NEXT:    .cfi_def_cfa_register %ebp
-; X32-NEXT:    andl $-8, %esp
-; X32-NEXT:    subl $16, %esp
-; X32-NEXT:    movl 12(%ebp), %ecx
-; X32-NEXT:    movl 8(%ebp), %eax
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
 ; X32-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
 ; X32-NEXT:    paddb %xmm0, %xmm1
@@ -25,9 +18,8 @@ define void @test0(x86_mmx* %A, x86_mmx* %B) {
 ; X32-NEXT:    paddsb (%ecx), %mm0
 ; X32-NEXT:    movq %mm0, (%eax)
 ; X32-NEXT:    paddusb (%ecx), %mm0
-; X32-NEXT:    movq %mm0, {{[0-9]+}}(%esp)
-; X32-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
 ; X32-NEXT:    movq %mm0, (%eax)
+; X32-NEXT:    movq2dq %mm0, %xmm0
 ; X32-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
 ; X32-NEXT:    psubb %xmm1, %xmm0
 ; X32-NEXT:    movdq2q %xmm0, %mm0
@@ -35,42 +27,25 @@ define void @test0(x86_mmx* %A, x86_mmx* %B) {
 ; X32-NEXT:    psubsb (%ecx), %mm0
 ; X32-NEXT:    movq %mm0, (%eax)
 ; X32-NEXT:    psubusb (%ecx), %mm0
-; X32-NEXT:    movq %mm0, (%esp)
-; X32-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X32-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; X32-NEXT:    movq %mm0, (%eax)
+; X32-NEXT:    movq2dq %mm0, %xmm0
 ; X32-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
+; X32-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; X32-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
 ; X32-NEXT:    pmullw %xmm0, %xmm1
-; X32-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255]
-; X32-NEXT:    movdqa %xmm1, %xmm2
-; X32-NEXT:    pand %xmm0, %xmm2
-; X32-NEXT:    packuswb %xmm2, %xmm2
-; X32-NEXT:    movq %xmm2, (%eax)
-; X32-NEXT:    movq {{.*#+}} xmm2 = mem[0],zero
-; X32-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
-; X32-NEXT:    pand %xmm1, %xmm2
-; X32-NEXT:    movdqa %xmm2, %xmm1
-; X32-NEXT:    pand %xmm0, %xmm1
+; X32-NEXT:    pand {{\.LCPI.*}}, %xmm1
 ; X32-NEXT:    packuswb %xmm1, %xmm1
 ; X32-NEXT:    movq %xmm1, (%eax)
+; X32-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; X32-NEXT:    pand %xmm1, %xmm0
+; X32-NEXT:    movq %xmm0, (%eax)
 ; X32-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
-; X32-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; X32-NEXT:    por %xmm2, %xmm1
-; X32-NEXT:    movdqa %xmm1, %xmm2
-; X32-NEXT:    pand %xmm0, %xmm2
-; X32-NEXT:    packuswb %xmm2, %xmm2
-; X32-NEXT:    movq %xmm2, (%eax)
-; X32-NEXT:    movq {{.*#+}} xmm2 = mem[0],zero
-; X32-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
-; X32-NEXT:    pxor %xmm1, %xmm2
-; X32-NEXT:    pand %xmm0, %xmm2
-; X32-NEXT:    packuswb %xmm2, %xmm2
-; X32-NEXT:    movq %xmm2, (%eax)
+; X32-NEXT:    por %xmm0, %xmm1
+; X32-NEXT:    movq %xmm1, (%eax)
+; X32-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; X32-NEXT:    pxor %xmm1, %xmm0
+; X32-NEXT:    movq %xmm0, (%eax)
 ; X32-NEXT:    emms
-; X32-NEXT:    movl %ebp, %esp
-; X32-NEXT:    popl %ebp
-; X32-NEXT:    .cfi_def_cfa %esp, 4
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: test0:
@@ -83,9 +58,8 @@ define void @test0(x86_mmx* %A, x86_mmx* %B) {
 ; X64-NEXT:    paddsb (%rsi), %mm0
 ; X64-NEXT:    movq %mm0, (%rdi)
 ; X64-NEXT:    paddusb (%rsi), %mm0
-; X64-NEXT:    movq %mm0, -{{[0-9]+}}(%rsp)
-; X64-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
 ; X64-NEXT:    movq %mm0, (%rdi)
+; X64-NEXT:    movq2dq %mm0, %xmm0
 ; X64-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
 ; X64-NEXT:    psubb %xmm1, %xmm0
 ; X64-NEXT:    movdq2q %xmm0, %mm0
@@ -93,38 +67,24 @@ define void @test0(x86_mmx* %A, x86_mmx* %B) {
 ; X64-NEXT:    psubsb (%rsi), %mm0
 ; X64-NEXT:    movq %mm0, (%rdi)
 ; X64-NEXT:    psubusb (%rsi), %mm0
-; X64-NEXT:    movq %mm0, -{{[0-9]+}}(%rsp)
-; X64-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X64-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; X64-NEXT:    movq %mm0, (%rdi)
+; X64-NEXT:    movq2dq %mm0, %xmm0
 ; X64-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
+; X64-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; X64-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
 ; X64-NEXT:    pmullw %xmm0, %xmm1
-; X64-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255]
-; X64-NEXT:    movdqa %xmm1, %xmm2
-; X64-NEXT:    pand %xmm0, %xmm2
-; X64-NEXT:    packuswb %xmm2, %xmm2
-; X64-NEXT:    movq %xmm2, (%rdi)
-; X64-NEXT:    movq {{.*#+}} xmm2 = mem[0],zero
-; X64-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
-; X64-NEXT:    pand %xmm1, %xmm2
-; X64-NEXT:    movdqa %xmm2, %xmm1
-; X64-NEXT:    pand %xmm0, %xmm1
+; X64-NEXT:    pand {{.*}}(%rip), %xmm1
 ; X64-NEXT:    packuswb %xmm1, %xmm1
 ; X64-NEXT:    movq %xmm1, (%rdi)
+; X64-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; X64-NEXT:    pand %xmm1, %xmm0
+; X64-NEXT:    movq %xmm0, (%rdi)
 ; X64-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
-; X64-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; X64-NEXT:    por %xmm2, %xmm1
-; X64-NEXT:    movdqa %xmm1, %xmm2
-; X64-NEXT:    pand %xmm0, %xmm2
-; X64-NEXT:    packuswb %xmm2, %xmm2
-; X64-NEXT:    movq %xmm2, (%rdi)
-; X64-NEXT:    movq {{.*#+}} xmm2 = mem[0],zero
-; X64-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
-; X64-NEXT:    pxor %xmm1, %xmm2
-; X64-NEXT:    pand %xmm0, %xmm2
-; X64-NEXT:    packuswb %xmm2, %xmm2
-; X64-NEXT:    movq %xmm2, (%rdi)
+; X64-NEXT:    por %xmm0, %xmm1
+; X64-NEXT:    movq %xmm1, (%rdi)
+; X64-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; X64-NEXT:    pxor %xmm1, %xmm0
+; X64-NEXT:    movq %xmm0, (%rdi)
 ; X64-NEXT:    emms
 ; X64-NEXT:    retq
 entry:
@@ -182,66 +142,56 @@ entry:
 define void @test1(x86_mmx* %A, x86_mmx* %B) {
 ; X32-LABEL: test1:
 ; X32:       # %bb.0: # %entry
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; X32-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1,1,3]
-; X32-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
-; X32-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,1,1,3]
-; X32-NEXT:    paddq %xmm0, %xmm1
-; X32-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
-; X32-NEXT:    movq %xmm0, (%eax)
-; X32-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; X32-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1,1,3]
-; X32-NEXT:    pmuludq %xmm1, %xmm0
-; X32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,2,2,3]
-; X32-NEXT:    movq %xmm1, (%eax)
-; X32-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
-; X32-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,1,1,3]
-; X32-NEXT:    andps %xmm0, %xmm1
-; X32-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
-; X32-NEXT:    movq %xmm0, (%eax)
-; X32-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; X32-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1,1,3]
-; X32-NEXT:    orps %xmm1, %xmm0
-; X32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,2,2,3]
-; X32-NEXT:    movq %xmm1, (%eax)
-; X32-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
-; X32-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,1,1,3]
-; X32-NEXT:    xorps %xmm0, %xmm1
-; X32-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
-; X32-NEXT:    movq %xmm0, (%eax)
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; X32-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
+; X32-NEXT:    paddd %xmm0, %xmm1
+; X32-NEXT:    movq %xmm1, (%ecx)
+; X32-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; X32-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; X32-NEXT:    pmuludq %xmm0, %xmm1
+; X32-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; X32-NEXT:    pmuludq %xmm0, %xmm2
+; X32-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
+; X32-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; X32-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; X32-NEXT:    movq %xmm1, (%ecx)
+; X32-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; X32-NEXT:    pand %xmm1, %xmm0
+; X32-NEXT:    movq %xmm0, (%ecx)
+; X32-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
+; X32-NEXT:    por %xmm0, %xmm1
+; X32-NEXT:    movq %xmm1, (%ecx)
+; X32-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; X32-NEXT:    pxor %xmm1, %xmm0
+; X32-NEXT:    movq %xmm0, (%ecx)
 ; X32-NEXT:    emms
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: test1:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,1,3]
 ; X64-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
-; X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,1,1,3]
-; X64-NEXT:    paddq %xmm0, %xmm1
-; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
-; X64-NEXT:    movq %xmm0, (%rdi)
-; X64-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,1,3]
-; X64-NEXT:    pmuludq %xmm1, %xmm0
-; X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,2,2,3]
+; X64-NEXT:    paddd %xmm0, %xmm1
 ; X64-NEXT:    movq %xmm1, (%rdi)
-; X64-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
-; X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,1,1,3]
-; X64-NEXT:    pand %xmm0, %xmm1
-; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
-; X64-NEXT:    movq %xmm0, (%rdi)
 ; X64-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,1,3]
-; X64-NEXT:    por %xmm1, %xmm0
-; X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,2,2,3]
+; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; X64-NEXT:    pmuludq %xmm0, %xmm1
+; X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; X64-NEXT:    pmuludq %xmm2, %xmm0
+; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X64-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
 ; X64-NEXT:    movq %xmm1, (%rdi)
+; X64-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; X64-NEXT:    pand %xmm1, %xmm0
+; X64-NEXT:    movq %xmm0, (%rdi)
 ; X64-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
-; X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,1,1,3]
-; X64-NEXT:    pxor %xmm0, %xmm1
-; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
+; X64-NEXT:    por %xmm0, %xmm1
+; X64-NEXT:    movq %xmm1, (%rdi)
+; X64-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; X64-NEXT:    pxor %xmm1, %xmm0
 ; X64-NEXT:    movq %xmm0, (%rdi)
 ; X64-NEXT:    emms
 ; X64-NEXT:    retq
@@ -289,15 +239,8 @@ entry:
 define void @test2(x86_mmx* %A, x86_mmx* %B) {
 ; X32-LABEL: test2:
 ; X32:       # %bb.0: # %entry
-; X32-NEXT:    pushl %ebp
-; X32-NEXT:    .cfi_def_cfa_offset 8
-; X32-NEXT:    .cfi_offset %ebp, -8
-; X32-NEXT:    movl %esp, %ebp
-; X32-NEXT:    .cfi_def_cfa_register %ebp
-; X32-NEXT:    andl $-8, %esp
-; X32-NEXT:    subl $24, %esp
-; X32-NEXT:    movl 12(%ebp), %ecx
-; X32-NEXT:    movl 8(%ebp), %eax
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
 ; X32-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
 ; X32-NEXT:    paddw %xmm0, %xmm1
@@ -306,9 +249,8 @@ define void @test2(x86_mmx* %A, x86_mmx* %B) {
 ; X32-NEXT:    paddsw (%ecx), %mm0
 ; X32-NEXT:    movq %mm0, (%eax)
 ; X32-NEXT:    paddusw (%ecx), %mm0
-; X32-NEXT:    movq %mm0, {{[0-9]+}}(%esp)
-; X32-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
 ; X32-NEXT:    movq %mm0, (%eax)
+; X32-NEXT:    movq2dq %mm0, %xmm0
 ; X32-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
 ; X32-NEXT:    psubw %xmm1, %xmm0
 ; X32-NEXT:    movdq2q %xmm0, %mm0
@@ -316,9 +258,8 @@ define void @test2(x86_mmx* %A, x86_mmx* %B) {
 ; X32-NEXT:    psubsw (%ecx), %mm0
 ; X32-NEXT:    movq %mm0, (%eax)
 ; X32-NEXT:    psubusw (%ecx), %mm0
-; X32-NEXT:    movq %mm0, {{[0-9]+}}(%esp)
-; X32-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
 ; X32-NEXT:    movq %mm0, (%eax)
+; X32-NEXT:    movq2dq %mm0, %xmm0
 ; X32-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
 ; X32-NEXT:    pmullw %xmm0, %xmm1
 ; X32-NEXT:    movdq2q %xmm1, %mm0
@@ -326,35 +267,18 @@ define void @test2(x86_mmx* %A, x86_mmx* %B) {
 ; X32-NEXT:    pmulhw (%ecx), %mm0
 ; X32-NEXT:    movq %mm0, (%eax)
 ; X32-NEXT:    pmaddwd (%ecx), %mm0
-; X32-NEXT:    movq %mm0, (%esp)
-; X32-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X32-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
 ; X32-NEXT:    movq %mm0, (%eax)
-; X32-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
-; X32-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; X32-NEXT:    pand %xmm0, %xmm1
-; X32-NEXT:    pshuflw {{.*#+}} xmm0 = xmm1[0,2,2,3,4,5,6,7]
-; X32-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,6,6,7]
-; X32-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; X32-NEXT:    movq %xmm0, (%eax)
-; X32-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X32-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
-; X32-NEXT:    por %xmm1, %xmm0
-; X32-NEXT:    pshuflw {{.*#+}} xmm1 = xmm0[0,2,2,3,4,5,6,7]
-; X32-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,6,6,7]
-; X32-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; X32-NEXT:    movq %xmm1, (%eax)
-; X32-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
-; X32-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; X32-NEXT:    pxor %xmm0, %xmm1
-; X32-NEXT:    pshuflw {{.*#+}} xmm0 = xmm1[0,2,2,3,4,5,6,7]
-; X32-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,6,6,7]
-; X32-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; X32-NEXT:    movq %xmm0, (%eax)
+; X32-NEXT:    movq2dq %mm0, %xmm0
+; X32-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
+; X32-NEXT:    andps %xmm0, %xmm1
+; X32-NEXT:    movlps %xmm1, (%eax)
+; X32-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X32-NEXT:    orps %xmm1, %xmm0
+; X32-NEXT:    movlps %xmm0, (%eax)
+; X32-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
+; X32-NEXT:    xorps %xmm0, %xmm1
+; X32-NEXT:    movlps %xmm1, (%eax)
 ; X32-NEXT:    emms
-; X32-NEXT:    movl %ebp, %esp
-; X32-NEXT:    popl %ebp
-; X32-NEXT:    .cfi_def_cfa %esp, 4
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: test2:
@@ -367,9 +291,8 @@ define void @test2(x86_mmx* %A, x86_mmx* %B) {
 ; X64-NEXT:    paddsw (%rsi), %mm0
 ; X64-NEXT:    movq %mm0, (%rdi)
 ; X64-NEXT:    paddusw (%rsi), %mm0
-; X64-NEXT:    movq %mm0, -{{[0-9]+}}(%rsp)
-; X64-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
 ; X64-NEXT:    movq %mm0, (%rdi)
+; X64-NEXT:    movq2dq %mm0, %xmm0
 ; X64-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
 ; X64-NEXT:    psubw %xmm1, %xmm0
 ; X64-NEXT:    movdq2q %xmm0, %mm0
@@ -377,9 +300,8 @@ define void @test2(x86_mmx* %A, x86_mmx* %B) {
 ; X64-NEXT:    psubsw (%rsi), %mm0
 ; X64-NEXT:    movq %mm0, (%rdi)
 ; X64-NEXT:    psubusw (%rsi), %mm0
-; X64-NEXT:    movq %mm0, -{{[0-9]+}}(%rsp)
-; X64-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
 ; X64-NEXT:    movq %mm0, (%rdi)
+; X64-NEXT:    movq2dq %mm0, %xmm0
 ; X64-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
 ; X64-NEXT:    pmullw %xmm0, %xmm1
 ; X64-NEXT:    movdq2q %xmm1, %mm0
@@ -387,31 +309,17 @@ define void @test2(x86_mmx* %A, x86_mmx* %B) {
 ; X64-NEXT:    pmulhw (%rsi), %mm0
 ; X64-NEXT:    movq %mm0, (%rdi)
 ; X64-NEXT:    pmaddwd (%rsi), %mm0
-; X64-NEXT:    movq %mm0, -{{[0-9]+}}(%rsp)
-; X64-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X64-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
 ; X64-NEXT:    movq %mm0, (%rdi)
-; X64-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
-; X64-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; X64-NEXT:    pand %xmm0, %xmm1
-; X64-NEXT:    pshuflw {{.*#+}} xmm0 = xmm1[0,2,2,3,4,5,6,7]
-; X64-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,6,6,7]
-; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; X64-NEXT:    movq %xmm0, (%rdi)
-; X64-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X64-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
-; X64-NEXT:    por %xmm1, %xmm0
-; X64-NEXT:    pshuflw {{.*#+}} xmm1 = xmm0[0,2,2,3,4,5,6,7]
-; X64-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,6,6,7]
-; X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; X64-NEXT:    movq %xmm1, (%rdi)
-; X64-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
-; X64-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; X64-NEXT:    pxor %xmm0, %xmm1
-; X64-NEXT:    pshuflw {{.*#+}} xmm0 = xmm1[0,2,2,3,4,5,6,7]
-; X64-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,6,6,7]
-; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; X64-NEXT:    movq %xmm0, (%rdi)
+; X64-NEXT:    movq2dq %mm0, %xmm0
+; X64-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
+; X64-NEXT:    andps %xmm0, %xmm1
+; X64-NEXT:    movlps %xmm1, (%rdi)
+; X64-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X64-NEXT:    orps %xmm1, %xmm0
+; X64-NEXT:    movlps %xmm0, (%rdi)
+; X64-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
+; X64-NEXT:    xorps %xmm0, %xmm1
+; X64-NEXT:    movlps %xmm1, (%rdi)
 ; X64-NEXT:    emms
 ; X64-NEXT:    retq
 entry:
@@ -479,45 +387,34 @@ define <1 x i64> @test3(<1 x i64>* %a, <1 x i64>* %b, i32 %count) nounwind {
 ; X32-LABEL: test3:
 ; X32:       # %bb.0: # %entry
 ; X32-NEXT:    pushl %ebp
-; X32-NEXT:    movl %esp, %ebp
 ; X32-NEXT:    pushl %ebx
 ; X32-NEXT:    pushl %edi
 ; X32-NEXT:    pushl %esi
-; X32-NEXT:    andl $-8, %esp
-; X32-NEXT:    subl $16, %esp
-; X32-NEXT:    cmpl $0, 16(%ebp)
+; X32-NEXT:    cmpl $0, {{[0-9]+}}(%esp)
 ; X32-NEXT:    je .LBB3_1
 ; X32-NEXT:  # %bb.2: # %bb26.preheader
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %edi
 ; X32-NEXT:    xorl %ebx, %ebx
 ; X32-NEXT:    xorl %eax, %eax
 ; X32-NEXT:    xorl %edx, %edx
 ; X32-NEXT:    .p2align 4, 0x90
 ; X32-NEXT:  .LBB3_3: # %bb26
 ; X32-NEXT:    # =>This Inner Loop Header: Depth=1
-; X32-NEXT:    movl 8(%ebp), %ecx
-; X32-NEXT:    movl %ecx, %esi
-; X32-NEXT:    movl (%ecx,%ebx,8), %ecx
-; X32-NEXT:    movl 4(%esi,%ebx,8), %esi
-; X32-NEXT:    movl 12(%ebp), %edi
-; X32-NEXT:    addl (%edi,%ebx,8), %ecx
-; X32-NEXT:    adcl 4(%edi,%ebx,8), %esi
-; X32-NEXT:    addl %eax, %ecx
-; X32-NEXT:    movl %ecx, (%esp)
-; X32-NEXT:    adcl %edx, %esi
-; X32-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; X32-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X32-NEXT:    movd %xmm0, %eax
-; X32-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,0,1]
-; X32-NEXT:    movd %xmm0, %edx
+; X32-NEXT:    movl (%edi,%ebx,8), %ebp
+; X32-NEXT:    movl 4(%edi,%ebx,8), %ecx
+; X32-NEXT:    addl (%esi,%ebx,8), %ebp
+; X32-NEXT:    adcl 4(%esi,%ebx,8), %ecx
+; X32-NEXT:    addl %ebp, %eax
+; X32-NEXT:    adcl %ecx, %edx
 ; X32-NEXT:    incl %ebx
-; X32-NEXT:    cmpl 16(%ebp), %ebx
+; X32-NEXT:    cmpl {{[0-9]+}}(%esp), %ebx
 ; X32-NEXT:    jb .LBB3_3
 ; X32-NEXT:    jmp .LBB3_4
 ; X32-NEXT:  .LBB3_1:
 ; X32-NEXT:    xorl %eax, %eax
 ; X32-NEXT:    xorl %edx, %edx
 ; X32-NEXT:  .LBB3_4: # %bb31
-; X32-NEXT:    leal -12(%ebp), %esp
 ; X32-NEXT:    popl %esi
 ; X32-NEXT:    popl %edi
 ; X32-NEXT:    popl %ebx
@@ -750,6 +647,40 @@ entry:
   ret void
 }
 
+; Make sure we clamp large shift amounts to 255
+define i64 @pr43922() {
+; X32-LABEL: pr43922:
+; X32:       # %bb.0: # %entry
+; X32-NEXT:    pushl %ebp
+; X32-NEXT:    .cfi_def_cfa_offset 8
+; X32-NEXT:    .cfi_offset %ebp, -8
+; X32-NEXT:    movl %esp, %ebp
+; X32-NEXT:    .cfi_def_cfa_register %ebp
+; X32-NEXT:    andl $-8, %esp
+; X32-NEXT:    subl $8, %esp
+; X32-NEXT:    movq {{\.LCPI.*}}, %mm0 # mm0 = 0x7AAAAAAA7AAAAAAA
+; X32-NEXT:    psrad $255, %mm0
+; X32-NEXT:    movq %mm0, (%esp)
+; X32-NEXT:    movl (%esp), %eax
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT:    movl %ebp, %esp
+; X32-NEXT:    popl %ebp
+; X32-NEXT:    .cfi_def_cfa %esp, 4
+; X32-NEXT:    retl
+;
+; X64-LABEL: pr43922:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    movq {{.*}}(%rip), %mm0 # mm0 = 0x7AAAAAAA7AAAAAAA
+; X64-NEXT:    psrad $255, %mm0
+; X64-NEXT:    movq %mm0, %rax
+; X64-NEXT:    retq
+entry:
+  %0 = tail call x86_mmx @llvm.x86.mmx.psrai.d(x86_mmx bitcast (<2 x i32> <i32 2058005162, i32 2058005162> to x86_mmx), i32 268435456)
+  %1 = bitcast x86_mmx %0 to i64
+  ret i64 %1
+}
+declare x86_mmx @llvm.x86.mmx.psrai.d(x86_mmx, i32)
+
 declare x86_mmx @llvm.x86.mmx.padd.b(x86_mmx, x86_mmx)
 declare x86_mmx @llvm.x86.mmx.padd.w(x86_mmx, x86_mmx)
 declare x86_mmx @llvm.x86.mmx.padd.d(x86_mmx, x86_mmx)
diff --git a/llvm/test/CodeGen/X86/mmx-coalescing.ll b/llvm/test/CodeGen/X86/mmx-coalescing.ll
index 8cd57aa8c..cd343c4c6 100644
--- a/llvm/test/CodeGen/X86/mmx-coalescing.ll
+++ b/llvm/test/CodeGen/X86/mmx-coalescing.ll
@@ -16,17 +16,14 @@ define i32 @test(%SA* %pSA, i16* %A, i32 %B, i32 %C, i32 %D, i8* %E) {
 ; CHECK-NEXT:  # %bb.2: # %if.B
 ; CHECK-NEXT:    pshufw $238, %mm0, %mm0 # mm0 = mm0[2,3,2,3]
 ; CHECK-NEXT:    movq %mm0, %rax
-; CHECK-NEXT:    testl %eax, %eax
-; CHECK-NEXT:    jne .LBB0_4
+; CHECK-NEXT:    jmp .LBB0_3
 ; CHECK-NEXT:  .LBB0_1: # %if.A
-; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movd %edx, %mm1
 ; CHECK-NEXT:    psllq %mm1, %mm0
 ; CHECK-NEXT:    movq %mm0, %rax
 ; CHECK-NEXT:    testq %rax, %rax
 ; CHECK-NEXT:    jne .LBB0_4
-; CHECK-NEXT:  # %bb.3: # %if.C
-; CHECK-NEXT:    # in Loop: Header=BB0_1 Depth=1
+; CHECK-NEXT:  .LBB0_3: # %if.C
 ; CHECK-NEXT:    testl %eax, %eax
 ; CHECK-NEXT:    je .LBB0_1
 ; CHECK-NEXT:  .LBB0_4: # %merge
diff --git a/llvm/test/CodeGen/X86/mmx-cvt.ll b/llvm/test/CodeGen/X86/mmx-cvt.ll
index c238109da..339df3089 100644
--- a/llvm/test/CodeGen/X86/mmx-cvt.ll
+++ b/llvm/test/CodeGen/X86/mmx-cvt.ll
@@ -294,26 +294,20 @@ define <2 x double> @sitofp_v2i32_v2f64(<1 x i64>*) nounwind {
 define <4 x float> @sitofp_v2i32_v2f32(<1 x i64>*) nounwind {
 ; X86-LABEL: sitofp_v2i32_v2f32:
 ; X86:       # %bb.0:
-; X86-NEXT:    pushl %ebp
-; X86-NEXT:    movl %esp, %ebp
-; X86-NEXT:    andl $-8, %esp
-; X86-NEXT:    subl $8, %esp
-; X86-NEXT:    movl 8(%ebp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    movq (%eax), %mm0
 ; X86-NEXT:    paddd %mm0, %mm0
-; X86-NEXT:    movq %mm0, (%esp)
-; X86-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-NEXT:    movq2dq %mm0, %xmm0
+; X86-NEXT:    movq {{.*#+}} xmm0 = xmm0[0],zero
 ; X86-NEXT:    cvtdq2ps %xmm0, %xmm0
-; X86-NEXT:    movl %ebp, %esp
-; X86-NEXT:    popl %ebp
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: sitofp_v2i32_v2f32:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movq (%rdi), %mm0
 ; X64-NEXT:    paddd %mm0, %mm0
-; X64-NEXT:    movq %mm0, -{{[0-9]+}}(%rsp)
-; X64-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X64-NEXT:    movq2dq %mm0, %xmm0
+; X64-NEXT:    movq {{.*#+}} xmm0 = xmm0[0],zero
 ; X64-NEXT:    cvtdq2ps %xmm0, %xmm0
 ; X64-NEXT:    retq
   %2 = bitcast <1 x i64>* %0 to x86_mmx*
@@ -346,8 +340,7 @@ define <4 x float> @cvt_v2i32_v2f32(<1 x i64>*) nounwind {
 ; X64:       # %bb.0:
 ; X64-NEXT:    movq (%rdi), %mm0
 ; X64-NEXT:    paddd %mm0, %mm0
-; X64-NEXT:    movq %mm0, %rax
-; X64-NEXT:    movq %rax, %xmm0
+; X64-NEXT:    movq2dq %mm0, %xmm0
 ; X64-NEXT:    cvtdq2ps %xmm0, %xmm0
 ; X64-NEXT:    retq
   %2 = bitcast <1 x i64>* %0 to x86_mmx*
diff --git a/llvm/test/CodeGen/X86/mmx-fold-load.ll b/llvm/test/CodeGen/X86/mmx-fold-load.ll
index 71b8b40fc..5ad2d50c1 100644
--- a/llvm/test/CodeGen/X86/mmx-fold-load.ll
+++ b/llvm/test/CodeGen/X86/mmx-fold-load.ll
@@ -585,22 +585,22 @@ define void @test_psrlq_by_volatile_shift_amount(x86_mmx* %t) nounwind {
 ; X86-NEXT:    pushl %eax
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    movl $1, (%esp)
-; X86-NEXT:    movd (%esp), %mm0
 ; X86-NEXT:    movl $255, %ecx
-; X86-NEXT:    movd %ecx, %mm1
-; X86-NEXT:    psrlq %mm0, %mm1
-; X86-NEXT:    movq %mm1, (%eax)
+; X86-NEXT:    movd %ecx, %mm0
+; X86-NEXT:    movd (%esp), %mm1
+; X86-NEXT:    psrlq %mm1, %mm0
+; X86-NEXT:    movq %mm0, (%eax)
 ; X86-NEXT:    popl %eax
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: test_psrlq_by_volatile_shift_amount:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    movl $1, -{{[0-9]+}}(%rsp)
-; X64-NEXT:    movd -{{[0-9]+}}(%rsp), %mm0
 ; X64-NEXT:    movl $255, %eax
-; X64-NEXT:    movd %eax, %mm1
-; X64-NEXT:    psrlq %mm0, %mm1
-; X64-NEXT:    movq %mm1, (%rdi)
+; X64-NEXT:    movd %eax, %mm0
+; X64-NEXT:    movd -{{[0-9]+}}(%rsp), %mm1
+; X64-NEXT:    psrlq %mm1, %mm0
+; X64-NEXT:    movq %mm0, (%rdi)
 ; X64-NEXT:    retq
 entry:
   %0 = alloca i32, align 4
diff --git a/llvm/test/CodeGen/X86/mmx-intrinsics.ll b/llvm/test/CodeGen/X86/mmx-intrinsics.ll
index b96558306..48d4ad049 100644
--- a/llvm/test/CodeGen/X86/mmx-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/mmx-intrinsics.ll
@@ -1453,3 +1453,68 @@ define void @test90() {
 }
 
 declare void @llvm.x86.mmx.emms()
+
+define <1 x i64> @test_mm_insert_pi16(<1 x i64> %a.coerce, i32 %d) nounwind {
+; X86-LABEL: test_mm_insert_pi16:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    movl %esp, %ebp
+; X86-NEXT:    andl $-8, %esp
+; X86-NEXT:    subl $16, %esp
+; X86-NEXT:    movl 8(%ebp), %eax
+; X86-NEXT:    movl 12(%ebp), %ecx
+; X86-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT:    movq {{[0-9]+}}(%esp), %mm0
+; X86-NEXT:    pinsrw $2, 16(%ebp), %mm0
+; X86-NEXT:    movq %mm0, (%esp)
+; X86-NEXT:    movl (%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl %ebp, %esp
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl
+;
+; X64-LABEL: test_mm_insert_pi16:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    movq %rdi, %mm0
+; X64-NEXT:    pinsrw $2, %esi, %mm0
+; X64-NEXT:    movq %mm0, %rax
+; X64-NEXT:    retq
+entry:
+  %0 = bitcast <1 x i64> %a.coerce to x86_mmx
+  %1 = tail call x86_mmx @llvm.x86.mmx.pinsr.w(x86_mmx %0, i32 %d, i32 2)
+  %2 = bitcast x86_mmx %1 to <1 x i64>
+  ret <1 x i64> %2
+}
+
+declare x86_mmx @llvm.x86.mmx.pinsr.w(x86_mmx, i32, i32 immarg)
+
+define i32 @test_mm_extract_pi16(<1 x i64> %a.coerce) nounwind {
+; X86-LABEL: test_mm_extract_pi16:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    movl %esp, %ebp
+; X86-NEXT:    andl $-8, %esp
+; X86-NEXT:    subl $8, %esp
+; X86-NEXT:    movl 8(%ebp), %eax
+; X86-NEXT:    movl 12(%ebp), %ecx
+; X86-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NEXT:    movl %eax, (%esp)
+; X86-NEXT:    movq (%esp), %mm0
+; X86-NEXT:    pextrw $2, %mm0, %eax
+; X86-NEXT:    movl %ebp, %esp
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl
+;
+; X64-LABEL: test_mm_extract_pi16:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    movq %rdi, %mm0
+; X64-NEXT:    pextrw $2, %mm0, %eax
+; X64-NEXT:    retq
+entry:
+  %0 = bitcast <1 x i64> %a.coerce to x86_mmx
+  %1 = tail call i32 @llvm.x86.mmx.pextr.w(x86_mmx %0, i32 2)
+  ret i32 %1
+}
+
+declare i32 @llvm.x86.mmx.pextr.w(x86_mmx, i32 immarg)
diff --git a/llvm/test/CodeGen/X86/movfs.ll b/llvm/test/CodeGen/X86/movfs.ll
index e6ac21d26..fe21717d5 100644
--- a/llvm/test/CodeGen/X86/movfs.ll
+++ b/llvm/test/CodeGen/X86/movfs.ll
@@ -1,6 +1,12 @@
-; RUN: llc < %s -mtriple=i686-- | grep fs
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- | FileCheck %s
 
 define i32 @foo() nounwind readonly {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movl %fs:196, %eax
+; CHECK-NEXT:    movl (%eax), %eax
+; CHECK-NEXT:    retl
 entry:
 	%tmp = load i32*, i32* addrspace(257)* getelementptr (i32*, i32* addrspace(257)* inttoptr (i32 72 to i32* addrspace(257)*), i32 31)		; <i32*> [#uses=1]
 	%tmp1 = load i32, i32* %tmp		; <i32> [#uses=1]
diff --git a/llvm/test/CodeGen/X86/movmsk-cmp.ll b/llvm/test/CodeGen/X86/movmsk-cmp.ll
index c144ca618..3758b1a6c 100644
--- a/llvm/test/CodeGen/X86/movmsk-cmp.ll
+++ b/llvm/test/CodeGen/X86/movmsk-cmp.ll
@@ -224,6 +224,7 @@ define i1 @allones_v64i8_sign(<64 x i8> %arg) {
 ;
 ; KNL-LABEL: allones_v64i8_sign:
 ; KNL:       # %bb.0:
+; KNL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; KNL-NEXT:    vpmovmskb %ymm1, %eax
 ; KNL-NEXT:    shlq $32, %rax
 ; KNL-NEXT:    vpmovmskb %ymm0, %ecx
@@ -292,6 +293,7 @@ define i1 @allzeros_v64i8_sign(<64 x i8> %arg) {
 ;
 ; KNL-LABEL: allzeros_v64i8_sign:
 ; KNL:       # %bb.0:
+; KNL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; KNL-NEXT:    vpmovmskb %ymm1, %eax
 ; KNL-NEXT:    shlq $32, %rax
 ; KNL-NEXT:    vpmovmskb %ymm0, %ecx
@@ -545,6 +547,7 @@ define i1 @allones_v32i16_sign(<32 x i16> %arg) {
 ;
 ; KNL-LABEL: allones_v32i16_sign:
 ; KNL:       # %bb.0:
+; KNL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; KNL-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; KNL-NEXT:    vpcmpgtw %ymm0, %ymm2, %ymm0
 ; KNL-NEXT:    vpmovsxwd %ymm0, %zmm0
@@ -612,6 +615,7 @@ define i1 @allzeros_v32i16_sign(<32 x i16> %arg) {
 ;
 ; KNL-LABEL: allzeros_v32i16_sign:
 ; KNL:       # %bb.0:
+; KNL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; KNL-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; KNL-NEXT:    vpcmpgtw %ymm0, %ymm2, %ymm0
 ; KNL-NEXT:    vpmovsxwd %ymm0, %zmm0
@@ -671,7 +675,6 @@ define i1 @allones_v4i32_sign(<4 x i32> %arg) {
 ; SKX:       # %bb.0:
 ; SKX-NEXT:    vpmovd2m %xmm0, %k0
 ; SKX-NEXT:    kmovd %k0, %eax
-; SKX-NEXT:    andb $15, %al
 ; SKX-NEXT:    cmpb $15, %al
 ; SKX-NEXT:    sete %al
 ; SKX-NEXT:    retq
@@ -710,8 +713,7 @@ define i1 @allzeros_v4i32_sign(<4 x i32> %arg) {
 ; SKX-LABEL: allzeros_v4i32_sign:
 ; SKX:       # %bb.0:
 ; SKX-NEXT:    vpmovd2m %xmm0, %k0
-; SKX-NEXT:    kmovd %k0, %eax
-; SKX-NEXT:    testb $15, %al
+; SKX-NEXT:    kortestb %k0, %k0
 ; SKX-NEXT:    sete %al
 ; SKX-NEXT:    retq
   %tmp = icmp slt <4 x i32> %arg, zeroinitializer
@@ -959,7 +961,6 @@ define i1 @allones_v4i64_sign(<4 x i64> %arg) {
 ; SKX:       # %bb.0:
 ; SKX-NEXT:    vpmovq2m %ymm0, %k0
 ; SKX-NEXT:    kmovd %k0, %eax
-; SKX-NEXT:    andb $15, %al
 ; SKX-NEXT:    cmpb $15, %al
 ; SKX-NEXT:    sete %al
 ; SKX-NEXT:    vzeroupper
@@ -1001,8 +1002,7 @@ define i1 @allzeros_v4i64_sign(<4 x i64> %arg) {
 ; SKX-LABEL: allzeros_v4i64_sign:
 ; SKX:       # %bb.0:
 ; SKX-NEXT:    vpmovq2m %ymm0, %k0
-; SKX-NEXT:    kmovd %k0, %eax
-; SKX-NEXT:    testb $15, %al
+; SKX-NEXT:    kortestb %k0, %k0
 ; SKX-NEXT:    sete %al
 ; SKX-NEXT:    vzeroupper
 ; SKX-NEXT:    retq
@@ -1015,48 +1015,11 @@ define i1 @allzeros_v4i64_sign(<4 x i64> %arg) {
 define i1 @allones_v8i64_sign(<8 x i64> %arg) {
 ; SSE2-LABEL: allones_v8i64_sign:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [2147483648,2147483648]
-; SSE2-NEXT:    pxor %xmm4, %xmm3
-; SSE2-NEXT:    movdqa %xmm4, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm5
-; SSE2-NEXT:    pxor %xmm4, %xmm2
-; SSE2-NEXT:    movdqa %xmm4, %xmm3
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm2[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,1,3,3]
-; SSE2-NEXT:    por %xmm7, %xmm2
-; SSE2-NEXT:    packssdw %xmm5, %xmm2
-; SSE2-NEXT:    pxor %xmm4, %xmm1
-; SSE2-NEXT:    movdqa %xmm4, %xmm3
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pand %xmm5, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm3
-; SSE2-NEXT:    pxor %xmm4, %xmm0
-; SSE2-NEXT:    movdqa %xmm4, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pand %xmm5, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm0, %xmm1
-; SSE2-NEXT:    packssdw %xmm3, %xmm1
-; SSE2-NEXT:    packssdw %xmm2, %xmm1
-; SSE2-NEXT:    packsswb %xmm0, %xmm1
-; SSE2-NEXT:    pmovmskb %xmm1, %eax
+; SSE2-NEXT:    packssdw %xmm3, %xmm2
+; SSE2-NEXT:    packssdw %xmm1, %xmm0
+; SSE2-NEXT:    packssdw %xmm2, %xmm0
+; SSE2-NEXT:    packsswb %xmm0, %xmm0
+; SSE2-NEXT:    pmovmskb %xmm0, %eax
 ; SSE2-NEXT:    cmpb $-1, %al
 ; SSE2-NEXT:    sete %al
 ; SSE2-NEXT:    retq
@@ -1113,48 +1076,11 @@ define i1 @allones_v8i64_sign(<8 x i64> %arg) {
 define i1 @allzeros_v8i64_sign(<8 x i64> %arg) {
 ; SSE2-LABEL: allzeros_v8i64_sign:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [2147483648,2147483648]
-; SSE2-NEXT:    pxor %xmm4, %xmm3
-; SSE2-NEXT:    movdqa %xmm4, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm5
-; SSE2-NEXT:    pxor %xmm4, %xmm2
-; SSE2-NEXT:    movdqa %xmm4, %xmm3
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm2[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,1,3,3]
-; SSE2-NEXT:    por %xmm7, %xmm2
-; SSE2-NEXT:    packssdw %xmm5, %xmm2
-; SSE2-NEXT:    pxor %xmm4, %xmm1
-; SSE2-NEXT:    movdqa %xmm4, %xmm3
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pand %xmm5, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm3
-; SSE2-NEXT:    pxor %xmm4, %xmm0
-; SSE2-NEXT:    movdqa %xmm4, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pand %xmm5, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm0, %xmm1
-; SSE2-NEXT:    packssdw %xmm3, %xmm1
-; SSE2-NEXT:    packssdw %xmm2, %xmm1
-; SSE2-NEXT:    packsswb %xmm0, %xmm1
-; SSE2-NEXT:    pmovmskb %xmm1, %eax
+; SSE2-NEXT:    packssdw %xmm3, %xmm2
+; SSE2-NEXT:    packssdw %xmm1, %xmm0
+; SSE2-NEXT:    packssdw %xmm2, %xmm0
+; SSE2-NEXT:    packsswb %xmm0, %xmm0
+; SSE2-NEXT:    pmovmskb %xmm0, %eax
 ; SSE2-NEXT:    testb %al, %al
 ; SSE2-NEXT:    sete %al
 ; SSE2-NEXT:    retq
@@ -1459,6 +1385,7 @@ define i1 @allones_v64i8_and1(<64 x i8> %arg) {
 ;
 ; KNL-LABEL: allones_v64i8_and1:
 ; KNL:       # %bb.0:
+; KNL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; KNL-NEXT:    vpsllw $7, %ymm0, %ymm0
 ; KNL-NEXT:    vpsllw $7, %ymm1, %ymm1
 ; KNL-NEXT:    vpmovmskb %ymm1, %eax
@@ -1540,6 +1467,7 @@ define i1 @allzeros_v64i8_and1(<64 x i8> %arg) {
 ;
 ; KNL-LABEL: allzeros_v64i8_and1:
 ; KNL:       # %bb.0:
+; KNL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; KNL-NEXT:    vpsllw $7, %ymm0, %ymm0
 ; KNL-NEXT:    vpsllw $7, %ymm1, %ymm1
 ; KNL-NEXT:    vpmovmskb %ymm1, %eax
@@ -1762,6 +1690,7 @@ define i1 @allones_v32i16_and1(<32 x i16> %arg) {
 ;
 ; KNL-LABEL: allones_v32i16_and1:
 ; KNL:       # %bb.0:
+; KNL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; KNL-NEXT:    vpsllw $15, %ymm0, %ymm0
 ; KNL-NEXT:    vpsraw $15, %ymm0, %ymm0
 ; KNL-NEXT:    vpmovsxwd %ymm0, %zmm0
@@ -1841,6 +1770,7 @@ define i1 @allzeros_v32i16_and1(<32 x i16> %arg) {
 ;
 ; KNL-LABEL: allzeros_v32i16_and1:
 ; KNL:       # %bb.0:
+; KNL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; KNL-NEXT:    vpsllw $15, %ymm0, %ymm0
 ; KNL-NEXT:    vpsraw $15, %ymm0, %ymm0
 ; KNL-NEXT:    vpmovsxwd %ymm0, %zmm0
@@ -1963,7 +1893,6 @@ define i1 @allones_v4i32_and1(<4 x i32> %arg) {
 ; SKX:       # %bb.0:
 ; SKX-NEXT:    vptestmd {{.*}}(%rip){1to4}, %xmm0, %k0
 ; SKX-NEXT:    kmovd %k0, %eax
-; SKX-NEXT:    andb $15, %al
 ; SKX-NEXT:    cmpb $15, %al
 ; SKX-NEXT:    sete %al
 ; SKX-NEXT:    retq
@@ -2004,8 +1933,7 @@ define i1 @allzeros_v4i32_and1(<4 x i32> %arg) {
 ; SKX-LABEL: allzeros_v4i32_and1:
 ; SKX:       # %bb.0:
 ; SKX-NEXT:    vptestmd {{.*}}(%rip){1to4}, %xmm0, %k0
-; SKX-NEXT:    kmovd %k0, %eax
-; SKX-NEXT:    testb $15, %al
+; SKX-NEXT:    kortestb %k0, %k0
 ; SKX-NEXT:    sete %al
 ; SKX-NEXT:    retq
   %tmp = and <4 x i32> %arg, <i32 1, i32 1, i32 1, i32 1>
@@ -2302,7 +2230,6 @@ define i1 @allones_v2i64_and1(<2 x i64> %arg) {
 ; SKX:       # %bb.0:
 ; SKX-NEXT:    vptestmq {{.*}}(%rip), %xmm0, %k0
 ; SKX-NEXT:    kmovd %k0, %eax
-; SKX-NEXT:    andb $3, %al
 ; SKX-NEXT:    cmpb $3, %al
 ; SKX-NEXT:    sete %al
 ; SKX-NEXT:    retq
@@ -2344,8 +2271,7 @@ define i1 @allzeros_v2i64_and1(<2 x i64> %arg) {
 ; SKX-LABEL: allzeros_v2i64_and1:
 ; SKX:       # %bb.0:
 ; SKX-NEXT:    vptestmq {{.*}}(%rip), %xmm0, %k0
-; SKX-NEXT:    kmovd %k0, %eax
-; SKX-NEXT:    testb $3, %al
+; SKX-NEXT:    kortestb %k0, %k0
 ; SKX-NEXT:    sete %al
 ; SKX-NEXT:    retq
   %tmp = and <2 x i64> %arg, <i64 1, i64 1>
@@ -2402,7 +2328,6 @@ define i1 @allones_v4i64_and1(<4 x i64> %arg) {
 ; SKX:       # %bb.0:
 ; SKX-NEXT:    vptestmq {{.*}}(%rip){1to4}, %ymm0, %k0
 ; SKX-NEXT:    kmovd %k0, %eax
-; SKX-NEXT:    andb $15, %al
 ; SKX-NEXT:    cmpb $15, %al
 ; SKX-NEXT:    sete %al
 ; SKX-NEXT:    vzeroupper
@@ -2459,8 +2384,7 @@ define i1 @allzeros_v4i64_and1(<4 x i64> %arg) {
 ; SKX-LABEL: allzeros_v4i64_and1:
 ; SKX:       # %bb.0:
 ; SKX-NEXT:    vptestmq {{.*}}(%rip){1to4}, %ymm0, %k0
-; SKX-NEXT:    kmovd %k0, %eax
-; SKX-NEXT:    testb $15, %al
+; SKX-NEXT:    kortestb %k0, %k0
 ; SKX-NEXT:    sete %al
 ; SKX-NEXT:    vzeroupper
 ; SKX-NEXT:    retq
@@ -2864,6 +2788,7 @@ define i1 @allones_v64i8_and4(<64 x i8> %arg) {
 ;
 ; KNL-LABEL: allones_v64i8_and4:
 ; KNL:       # %bb.0:
+; KNL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; KNL-NEXT:    vpsllw $5, %ymm0, %ymm0
 ; KNL-NEXT:    vpsllw $5, %ymm1, %ymm1
 ; KNL-NEXT:    vpmovmskb %ymm1, %eax
@@ -2945,6 +2870,7 @@ define i1 @allzeros_v64i8_and4(<64 x i8> %arg) {
 ;
 ; KNL-LABEL: allzeros_v64i8_and4:
 ; KNL:       # %bb.0:
+; KNL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; KNL-NEXT:    vpsllw $5, %ymm0, %ymm0
 ; KNL-NEXT:    vpsllw $5, %ymm1, %ymm1
 ; KNL-NEXT:    vpmovmskb %ymm1, %eax
@@ -3167,6 +3093,7 @@ define i1 @allones_v32i16_and4(<32 x i16> %arg) {
 ;
 ; KNL-LABEL: allones_v32i16_and4:
 ; KNL:       # %bb.0:
+; KNL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; KNL-NEXT:    vpsllw $13, %ymm0, %ymm0
 ; KNL-NEXT:    vpsraw $15, %ymm0, %ymm0
 ; KNL-NEXT:    vpmovsxwd %ymm0, %zmm0
@@ -3246,6 +3173,7 @@ define i1 @allzeros_v32i16_and4(<32 x i16> %arg) {
 ;
 ; KNL-LABEL: allzeros_v32i16_and4:
 ; KNL:       # %bb.0:
+; KNL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; KNL-NEXT:    vpsllw $13, %ymm0, %ymm0
 ; KNL-NEXT:    vpsraw $15, %ymm0, %ymm0
 ; KNL-NEXT:    vpmovsxwd %ymm0, %zmm0
@@ -3368,7 +3296,6 @@ define i1 @allones_v4i32_and4(<4 x i32> %arg) {
 ; SKX:       # %bb.0:
 ; SKX-NEXT:    vptestmd {{.*}}(%rip){1to4}, %xmm0, %k0
 ; SKX-NEXT:    kmovd %k0, %eax
-; SKX-NEXT:    andb $15, %al
 ; SKX-NEXT:    cmpb $15, %al
 ; SKX-NEXT:    sete %al
 ; SKX-NEXT:    retq
@@ -3409,8 +3336,7 @@ define i1 @allzeros_v4i32_and4(<4 x i32> %arg) {
 ; SKX-LABEL: allzeros_v4i32_and4:
 ; SKX:       # %bb.0:
 ; SKX-NEXT:    vptestmd {{.*}}(%rip){1to4}, %xmm0, %k0
-; SKX-NEXT:    kmovd %k0, %eax
-; SKX-NEXT:    testb $15, %al
+; SKX-NEXT:    kortestb %k0, %k0
 ; SKX-NEXT:    sete %al
 ; SKX-NEXT:    retq
   %tmp = and <4 x i32> %arg, <i32 4, i32 4, i32 4, i32 4>
@@ -3707,7 +3633,6 @@ define i1 @allones_v2i64_and4(<2 x i64> %arg) {
 ; SKX:       # %bb.0:
 ; SKX-NEXT:    vptestmq {{.*}}(%rip), %xmm0, %k0
 ; SKX-NEXT:    kmovd %k0, %eax
-; SKX-NEXT:    andb $3, %al
 ; SKX-NEXT:    cmpb $3, %al
 ; SKX-NEXT:    sete %al
 ; SKX-NEXT:    retq
@@ -3749,8 +3674,7 @@ define i1 @allzeros_v2i64_and4(<2 x i64> %arg) {
 ; SKX-LABEL: allzeros_v2i64_and4:
 ; SKX:       # %bb.0:
 ; SKX-NEXT:    vptestmq {{.*}}(%rip), %xmm0, %k0
-; SKX-NEXT:    kmovd %k0, %eax
-; SKX-NEXT:    testb $3, %al
+; SKX-NEXT:    kortestb %k0, %k0
 ; SKX-NEXT:    sete %al
 ; SKX-NEXT:    retq
   %tmp = and <2 x i64> %arg, <i64 4, i64 4>
@@ -3807,7 +3731,6 @@ define i1 @allones_v4i64_and4(<4 x i64> %arg) {
 ; SKX:       # %bb.0:
 ; SKX-NEXT:    vptestmq {{.*}}(%rip){1to4}, %ymm0, %k0
 ; SKX-NEXT:    kmovd %k0, %eax
-; SKX-NEXT:    andb $15, %al
 ; SKX-NEXT:    cmpb $15, %al
 ; SKX-NEXT:    sete %al
 ; SKX-NEXT:    vzeroupper
@@ -3864,8 +3787,7 @@ define i1 @allzeros_v4i64_and4(<4 x i64> %arg) {
 ; SKX-LABEL: allzeros_v4i64_and4:
 ; SKX:       # %bb.0:
 ; SKX-NEXT:    vptestmq {{.*}}(%rip){1to4}, %ymm0, %k0
-; SKX-NEXT:    kmovd %k0, %eax
-; SKX-NEXT:    testb $15, %al
+; SKX-NEXT:    kortestb %k0, %k0
 ; SKX-NEXT:    sete %al
 ; SKX-NEXT:    vzeroupper
 ; SKX-NEXT:    retq
@@ -4445,8 +4367,8 @@ define i1 @movmsk_v4i32(<4 x i32> %x, <4 x i32> %y) {
   ret i1 %u1
 }
 
-define i1 @movmsk_v2i64(<2 x i64> %x, <2 x i64> %y) {
-; SSE2-LABEL: movmsk_v2i64:
+define i1 @movmsk_and_v2i64(<2 x i64> %x, <2 x i64> %y) {
+; SSE2-LABEL: movmsk_and_v2i64:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,0,3,2]
@@ -4457,7 +4379,7 @@ define i1 @movmsk_v2i64(<2 x i64> %x, <2 x i64> %y) {
 ; SSE2-NEXT:    sete %al
 ; SSE2-NEXT:    retq
 ;
-; AVX-LABEL: movmsk_v2i64:
+; AVX-LABEL: movmsk_and_v2i64:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpcmpeqq %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovmskpd %xmm0, %eax
@@ -4466,27 +4388,24 @@ define i1 @movmsk_v2i64(<2 x i64> %x, <2 x i64> %y) {
 ; AVX-NEXT:    sete %al
 ; AVX-NEXT:    retq
 ;
-; KNL-LABEL: movmsk_v2i64:
+; KNL-LABEL: movmsk_and_v2i64:
 ; KNL:       # %bb.0:
 ; KNL-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
 ; KNL-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; KNL-NEXT:    vpcmpneqq %zmm1, %zmm0, %k0
-; KNL-NEXT:    kshiftrw $1, %k0, %k1
-; KNL-NEXT:    kmovw %k1, %ecx
 ; KNL-NEXT:    kmovw %k0, %eax
-; KNL-NEXT:    andb %cl, %al
-; KNL-NEXT:    # kill: def $al killed $al killed $eax
+; KNL-NEXT:    andb $3, %al
+; KNL-NEXT:    cmpb $3, %al
+; KNL-NEXT:    sete %al
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    retq
 ;
-; SKX-LABEL: movmsk_v2i64:
+; SKX-LABEL: movmsk_and_v2i64:
 ; SKX:       # %bb.0:
 ; SKX-NEXT:    vpcmpneqq %xmm1, %xmm0, %k0
-; SKX-NEXT:    kshiftrb $1, %k0, %k1
-; SKX-NEXT:    kmovd %k1, %ecx
 ; SKX-NEXT:    kmovd %k0, %eax
-; SKX-NEXT:    andb %cl, %al
-; SKX-NEXT:    # kill: def $al killed $al killed $eax
+; SKX-NEXT:    cmpb $3, %al
+; SKX-NEXT:    sete %al
 ; SKX-NEXT:    retq
   %cmp = icmp ne <2 x i64> %x, %y
   %e1 = extractelement <2 x i1> %cmp, i32 0
@@ -4495,6 +4414,49 @@ define i1 @movmsk_v2i64(<2 x i64> %x, <2 x i64> %y) {
   ret i1 %u1
 }
 
+define i1 @movmsk_or_v2i64(<2 x i64> %x, <2 x i64> %y) {
+; SSE2-LABEL: movmsk_or_v2i64:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,0,3,2]
+; SSE2-NEXT:    pand %xmm0, %xmm1
+; SSE2-NEXT:    movmskpd %xmm1, %eax
+; SSE2-NEXT:    xorb $3, %al
+; SSE2-NEXT:    setne %al
+; SSE2-NEXT:    retq
+;
+; AVX-LABEL: movmsk_or_v2i64:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpcmpeqq %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vmovmskpd %xmm0, %eax
+; AVX-NEXT:    xorb $3, %al
+; AVX-NEXT:    setne %al
+; AVX-NEXT:    retq
+;
+; KNL-LABEL: movmsk_or_v2i64:
+; KNL:       # %bb.0:
+; KNL-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; KNL-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; KNL-NEXT:    vpcmpneqq %zmm1, %zmm0, %k0
+; KNL-NEXT:    kmovw %k0, %eax
+; KNL-NEXT:    testb $3, %al
+; KNL-NEXT:    setne %al
+; KNL-NEXT:    vzeroupper
+; KNL-NEXT:    retq
+;
+; SKX-LABEL: movmsk_or_v2i64:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpcmpneqq %xmm1, %xmm0, %k0
+; SKX-NEXT:    kortestb %k0, %k0
+; SKX-NEXT:    setne %al
+; SKX-NEXT:    retq
+  %cmp = icmp ne <2 x i64> %x, %y
+  %e1 = extractelement <2 x i1> %cmp, i32 0
+  %e2 = extractelement <2 x i1> %cmp, i32 1
+  %u1 = or i1 %e1, %e2
+  ret i1 %u1
+}
+
 define i1 @movmsk_v4f32(<4 x float> %x, <4 x float> %y) {
 ; SSE2-LABEL: movmsk_v4f32:
 ; SSE2:       # %bb.0:
@@ -4554,8 +4516,8 @@ define i1 @movmsk_v4f32(<4 x float> %x, <4 x float> %y) {
   ret i1 %u2
 }
 
-define i1 @movmsk_v2f64(<2 x double> %x, <2 x double> %y) {
-; SSE2-LABEL: movmsk_v2f64:
+define i1 @movmsk_and_v2f64(<2 x double> %x, <2 x double> %y) {
+; SSE2-LABEL: movmsk_and_v2f64:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    cmplepd %xmm0, %xmm1
 ; SSE2-NEXT:    movmskpd %xmm1, %eax
@@ -4563,7 +4525,7 @@ define i1 @movmsk_v2f64(<2 x double> %x, <2 x double> %y) {
 ; SSE2-NEXT:    sete %al
 ; SSE2-NEXT:    retq
 ;
-; AVX-LABEL: movmsk_v2f64:
+; AVX-LABEL: movmsk_and_v2f64:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vcmplepd %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    vmovmskpd %xmm0, %eax
@@ -4571,27 +4533,24 @@ define i1 @movmsk_v2f64(<2 x double> %x, <2 x double> %y) {
 ; AVX-NEXT:    sete %al
 ; AVX-NEXT:    retq
 ;
-; KNL-LABEL: movmsk_v2f64:
+; KNL-LABEL: movmsk_and_v2f64:
 ; KNL:       # %bb.0:
 ; KNL-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
 ; KNL-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; KNL-NEXT:    vcmplepd %zmm0, %zmm1, %k0
-; KNL-NEXT:    kshiftrw $1, %k0, %k1
-; KNL-NEXT:    kmovw %k1, %ecx
 ; KNL-NEXT:    kmovw %k0, %eax
-; KNL-NEXT:    andb %cl, %al
-; KNL-NEXT:    # kill: def $al killed $al killed $eax
+; KNL-NEXT:    andb $3, %al
+; KNL-NEXT:    cmpb $3, %al
+; KNL-NEXT:    sete %al
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    retq
 ;
-; SKX-LABEL: movmsk_v2f64:
+; SKX-LABEL: movmsk_and_v2f64:
 ; SKX:       # %bb.0:
 ; SKX-NEXT:    vcmplepd %xmm0, %xmm1, %k0
-; SKX-NEXT:    kshiftrb $1, %k0, %k1
-; SKX-NEXT:    kmovd %k1, %ecx
 ; SKX-NEXT:    kmovd %k0, %eax
-; SKX-NEXT:    andb %cl, %al
-; SKX-NEXT:    # kill: def $al killed $al killed $eax
+; SKX-NEXT:    cmpb $3, %al
+; SKX-NEXT:    sete %al
 ; SKX-NEXT:    retq
   %cmp = fcmp oge <2 x double> %x, %y
   %e1 = extractelement <2 x i1> %cmp, i32 0
@@ -4600,6 +4559,47 @@ define i1 @movmsk_v2f64(<2 x double> %x, <2 x double> %y) {
   ret i1 %u1
 }
 
+define i1 @movmsk_or_v2f64(<2 x double> %x, <2 x double> %y) {
+; SSE2-LABEL: movmsk_or_v2f64:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    cmplepd %xmm0, %xmm1
+; SSE2-NEXT:    movmskpd %xmm1, %eax
+; SSE2-NEXT:    testb %al, %al
+; SSE2-NEXT:    setne %al
+; SSE2-NEXT:    retq
+;
+; AVX-LABEL: movmsk_or_v2f64:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcmplepd %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    vmovmskpd %xmm0, %eax
+; AVX-NEXT:    testb %al, %al
+; AVX-NEXT:    setne %al
+; AVX-NEXT:    retq
+;
+; KNL-LABEL: movmsk_or_v2f64:
+; KNL:       # %bb.0:
+; KNL-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; KNL-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; KNL-NEXT:    vcmplepd %zmm0, %zmm1, %k0
+; KNL-NEXT:    kmovw %k0, %eax
+; KNL-NEXT:    testb $3, %al
+; KNL-NEXT:    setne %al
+; KNL-NEXT:    vzeroupper
+; KNL-NEXT:    retq
+;
+; SKX-LABEL: movmsk_or_v2f64:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vcmplepd %xmm0, %xmm1, %k0
+; SKX-NEXT:    kortestb %k0, %k0
+; SKX-NEXT:    setne %al
+; SKX-NEXT:    retq
+  %cmp = fcmp oge <2 x double> %x, %y
+  %e1 = extractelement <2 x i1> %cmp, i32 0
+  %e2 = extractelement <2 x i1> %cmp, i32 1
+  %u1 = or i1 %e1, %e2
+  ret i1 %u1
+}
+
 define i32 @PR39665_c_ray(<2 x double> %x, <2 x double> %y) {
 ; SSE2-LABEL: PR39665_c_ray:
 ; SSE2:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/movpc32-check.ll b/llvm/test/CodeGen/X86/movpc32-check.ll
index 7545b52d6..bd1939906 100644
--- a/llvm/test/CodeGen/X86/movpc32-check.ll
+++ b/llvm/test/CodeGen/X86/movpc32-check.ll
@@ -12,8 +12,8 @@ entry:
 
 declare void @bar(...) #1
 
-attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="i686" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="i686" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="i686" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="i686" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!7, !8, !9}
diff --git a/llvm/test/CodeGen/X86/ms-inline-asm-PR44272.ll b/llvm/test/CodeGen/X86/ms-inline-asm-PR44272.ll
new file mode 100644
index 000000000..ea66ca3ea
--- /dev/null
+++ b/llvm/test/CodeGen/X86/ms-inline-asm-PR44272.ll
@@ -0,0 +1,18 @@
+; RUN: llc < %s -mtriple=i686-- | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-- | FileCheck %s
+
+define void @func() {
+entry:
+  ret void
+}
+
+define void @main() {
+entry:
+  call void asm sideeffect inteldialect "call ${0:P}", "*m,~{dirflag},~{fpsr},~{flags}"(void ()* @func)
+  ret void
+; CHECK-LABEL: main:
+; CHECK: {{## InlineAsm Start|#APP}}
+; CHECK: {{call(l|q) func$}}
+; CHECK: {{## InlineAsm End|#NO_APP}}
+; CHECK: ret{{l|q}}
+}
diff --git a/llvm/test/CodeGen/X86/ms-inline-asm-avx512.ll b/llvm/test/CodeGen/X86/ms-inline-asm-avx512.ll
index be60f5bca..4eb966f9c 100644
--- a/llvm/test/CodeGen/X86/ms-inline-asm-avx512.ll
+++ b/llvm/test/CodeGen/X86/ms-inline-asm-avx512.ll
@@ -20,5 +20,5 @@ entry:
 ; CHECK: movq    %rax, 7(%rsp)
 ; CHECK: retq
 
-attributes #0 = { noinline nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="skylake-avx512" "target-features"="+adx,+aes,+avx,+avx2,+avx512bw,+avx512cd,+avx512dq,+avx512f,+avx512vl,+bmi,+bmi2,+clflushopt,+clwb,+cx16,+f16c,+fma,+fsgsbase,+fxsr,+lzcnt,+mmx,+movbe,+mpx,+pclmul,+pku,+popcnt,+rdrnd,+rdseed,+rtm,+sgx,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87,+xsave,+xsavec,+xsaveopt,+xsaves" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noinline nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="skylake-avx512" "target-features"="+adx,+aes,+avx,+avx2,+avx512bw,+avx512cd,+avx512dq,+avx512f,+avx512vl,+bmi,+bmi2,+clflushopt,+clwb,+cx16,+f16c,+fma,+fsgsbase,+fxsr,+lzcnt,+mmx,+movbe,+pclmul,+pku,+popcnt,+rdrnd,+rdseed,+rtm,+sgx,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87,+xsave,+xsavec,+xsaveopt,+xsaves" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind }
diff --git a/llvm/test/CodeGen/X86/ms-inline-asm.ll b/llvm/test/CodeGen/X86/ms-inline-asm.ll
index 62525b072..828a76e6a 100644
--- a/llvm/test/CodeGen/X86/ms-inline-asm.ll
+++ b/llvm/test/CodeGen/X86/ms-inline-asm.ll
@@ -92,7 +92,7 @@ entry:
 ; CHECK-LABEL: t30:
 ; CHECK: {{## InlineAsm Start|#APP}}
 ; CHECK: .intel_syntax
-; CHECK: lea edi, dword ptr [{{_?}}results]
+; CHECK: lea edi, dword ptr [offset {{_?}}results]
 ; CHECK: .att_syntax
 ; CHECK: {{## InlineAsm End|#NO_APP}}
 ; CHECK: {{## InlineAsm Start|#APP}}
diff --git a/llvm/test/CodeGen/X86/mul-i1024.ll b/llvm/test/CodeGen/X86/mul-i1024.ll
index 56db554ef..96c054343 100644
--- a/llvm/test/CodeGen/X86/mul-i1024.ll
+++ b/llvm/test/CodeGen/X86/mul-i1024.ll
@@ -9,1291 +9,1069 @@ define void @test_1024(i1024* %a, i1024* %b, i1024* %out) nounwind {
 ; X32-NEXT:    pushl %ebx
 ; X32-NEXT:    pushl %edi
 ; X32-NEXT:    pushl %esi
-; X32-NEXT:    subl $1000, %esp # imm = 0x3E8
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT:    subl $400, %esp # imm = 0x190
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl 48(%eax), %ecx
-; X32-NEXT:    movl %eax, %esi
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl 32(%edx), %eax
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT:    movl 60(%ecx), %esi
+; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl 56(%ecx), %edi
+; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl (%eax), %ebp
+; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    mull %ebp
+; X32-NEXT:    movl %edx, %ecx
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    xorl %edi, %edi
-; X32-NEXT:    mull %edi
-; X32-NEXT:    movl %edx, %ebp
+; X32-NEXT:    movl %esi, %eax
+; X32-NEXT:    mull %ebp
+; X32-NEXT:    movl %edx, %esi
 ; X32-NEXT:    movl %eax, %ebx
-; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    addl %ecx, %ebx
+; X32-NEXT:    adcl $0, %esi
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl 4(%eax), %ecx
+; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %ecx, %edi
+; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    addl %ebx, %eax
+; X32-NEXT:    movl %eax, (%esp) # 4-byte Spill
+; X32-NEXT:    adcl %esi, %ecx
+; X32-NEXT:    setb %bl
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X32-NEXT:    mull %edi
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %eax, %ecx
-; X32-NEXT:    addl %ebx, %ecx
-; X32-NEXT:    movl %edx, %eax
-; X32-NEXT:    adcl %ebp, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl 32(%esi), %eax
+; X32-NEXT:    addl %ecx, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    mull %edi
+; X32-NEXT:    movzbl %bl, %eax
+; X32-NEXT:    adcl %eax, %edx
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %eax, %ecx
-; X32-NEXT:    addl %ebx, %ecx
-; X32-NEXT:    movl %edx, %eax
-; X32-NEXT:    adcl %ebp, %eax
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X32-NEXT:    movl 48(%edi), %esi
+; X32-NEXT:    movl %esi, %eax
+; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    mull %ebp
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl 36(%eax), %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    mull %edi
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    addl %ebp, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %edx, %eax
-; X32-NEXT:    adcl $0, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl 36(%esi), %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    mull %edi
-; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    movl 52(%edi), %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %eax, %ebp
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    addl %edi, %ebp
-; X32-NEXT:    adcl $0, %esi
+; X32-NEXT:    mull %ebp
+; X32-NEXT:    movl %edx, %ebx
+; X32-NEXT:    movl %eax, %edi
+; X32-NEXT:    addl %ecx, %edi
+; X32-NEXT:    adcl $0, %ebx
+; X32-NEXT:    movl %esi, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    addl %ebx, %eax
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %edx, %ebp
+; X32-NEXT:    addl %edi, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    leal (%ebx,%eax), %eax
-; X32-NEXT:    leal (%ecx,%ebp), %edx
+; X32-NEXT:    adcl %ebx, %ebp
+; X32-NEXT:    setb %bl
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X32-NEXT:    movl %esi, %eax
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    addl %ebp, %ecx
+; X32-NEXT:    movzbl %bl, %eax
 ; X32-NEXT:    adcl %eax, %edx
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X32-NEXT:    adcl (%esp), %edx # 4-byte Folded Reload
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    addl %ecx, %ebp
-; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edi, %esi
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl (%eax), %eax
+; X32-NEXT:    movl 8(%eax), %ebp
+; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X32-NEXT:    movl %ebx, %eax
+; X32-NEXT:    mull %ebp
+; X32-NEXT:    movl %edx, %edi
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    xorl %ecx, %ecx
-; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %esi, %eax
+; X32-NEXT:    mull %ebp
+; X32-NEXT:    movl %edx, %ebp
 ; X32-NEXT:    movl %eax, %esi
+; X32-NEXT:    addl %edi, %esi
+; X32-NEXT:    adcl $0, %ebp
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl 12(%eax), %edi
+; X32-NEXT:    movl %ebx, %eax
+; X32-NEXT:    mull %edi
 ; X32-NEXT:    movl %edx, %ebx
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ebp
-; X32-NEXT:    movl 16(%ebp), %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %eax, %ecx
-; X32-NEXT:    addl %esi, %ecx
-; X32-NEXT:    adcl %ebx, %edx
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl (%ebp), %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    xorl %ecx, %ecx
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %esi, %ebp
-; X32-NEXT:    addl %esi, %eax
-; X32-NEXT:    movl %edx, %eax
-; X32-NEXT:    adcl %ebx, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X32-NEXT:    addl %esi, %eax
+; X32-NEXT:    movl %eax, %esi
+; X32-NEXT:    adcl %ebp, %ebx
+; X32-NEXT:    setb (%esp) # 1-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %ebx, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    addl %esi, %eax
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edi, %ebp
+; X32-NEXT:    movl %eax, %edi
+; X32-NEXT:    addl %ebx, %edi
+; X32-NEXT:    movzbl (%esp), %eax # 1-byte Folded Reload
+; X32-NEXT:    adcl %eax, %edx
+; X32-NEXT:    addl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
 ; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    adcl %ebx, %eax
+; X32-NEXT:    adcl $0, %edi
+; X32-NEXT:    adcl $0, %edx
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, (%esp) # 4-byte Spill
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, %esi
+; X32-NEXT:    movl %eax, %ebx
+; X32-NEXT:    addl (%esp), %ebx # 4-byte Folded Reload
+; X32-NEXT:    adcl $0, %esi
+; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    movl %ebp, %ecx
+; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    mull %ebp
+; X32-NEXT:    movl %edx, %ebp
+; X32-NEXT:    addl %ebx, %eax
+; X32-NEXT:    movl %eax, %ebx
+; X32-NEXT:    adcl %esi, %ebp
+; X32-NEXT:    setb (%esp) # 1-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X32-NEXT:    movl 4(%esi), %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    xorl %ecx, %ecx
 ; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %eax, %ecx
-; X32-NEXT:    addl %ebx, %ecx
-; X32-NEXT:    movl %edx, %edi
-; X32-NEXT:    adcl $0, %edi
-; X32-NEXT:    addl %ebp, %ecx
-; X32-NEXT:    movl %ebp, %esi
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %ebx, %edi
-; X32-NEXT:    movl %ebx, %ebp
+; X32-NEXT:    addl %ebp, %eax
+; X32-NEXT:    movzbl (%esp), %ecx # 1-byte Folded Reload
+; X32-NEXT:    adcl %ecx, %edx
+; X32-NEXT:    addl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
 ; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    setb %cl
-; X32-NEXT:    addl %eax, %edi
-; X32-NEXT:    movl %edi, (%esp) # 4-byte Spill
-; X32-NEXT:    movzbl %cl, %eax
-; X32-NEXT:    adcl %edx, %eax
-; X32-NEXT:    movl %eax, %ebx
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl 8(%eax), %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    xorl %ecx, %ecx
-; X32-NEXT:    mull %ecx
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
+; X32-NEXT:    adcl %ecx, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, %edx
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    addl %eax, %esi
-; X32-NEXT:    adcl %edx, %ebp
-; X32-NEXT:    addl %edi, %esi
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %ebx, %ebp
-; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ebp
-; X32-NEXT:    movl 52(%ebp), %eax
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT:    movl 40(%esi), %edi
+; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X32-NEXT:    mull %ebx
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    xorl %ecx, %ecx
-; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    movl 44(%esi), %ebp
+; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    mull %ebx
+; X32-NEXT:    movl %edx, %esi
 ; X32-NEXT:    movl %eax, %ebx
+; X32-NEXT:    addl %ecx, %ebx
+; X32-NEXT:    adcl $0, %esi
+; X32-NEXT:    movl %edi, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    addl %edi, %ebx
+; X32-NEXT:    mull %edi
 ; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    adcl $0, %ecx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    addl %esi, %ebx
-; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edi, %ecx
-; X32-NEXT:    setb %bl
-; X32-NEXT:    addl %eax, %ecx
-; X32-NEXT:    movzbl %bl, %ebx
-; X32-NEXT:    adcl %edx, %ebx
-; X32-NEXT:    movl 56(%ebp), %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    xorl %edx, %edx
-; X32-NEXT:    mull %edx
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %esi, %ebp
-; X32-NEXT:    addl %eax, %ebp
-; X32-NEXT:    adcl %edx, %edi
-; X32-NEXT:    addl %ecx, %ebp
-; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %ebx, %edi
-; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X32-NEXT:    addl %ebx, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl %esi, %ecx
+; X32-NEXT:    setb %bl
 ; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    mull %edi
+; X32-NEXT:    addl %ecx, %eax
+; X32-NEXT:    movl %eax, (%esp) # 4-byte Spill
+; X32-NEXT:    movzbl %bl, %eax
+; X32-NEXT:    adcl %eax, %edx
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X32-NEXT:    movl 32(%ebp), %edi
 ; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 1-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl 40(%eax), %eax
+; X32-NEXT:    mull %esi
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    xorl %ecx, %ecx
-; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    movl 36(%ebp), %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    mull %esi
 ; X32-NEXT:    movl %edx, %ebx
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X32-NEXT:    movl %edx, %edi
-; X32-NEXT:    addl %eax, %edi
+; X32-NEXT:    movl %eax, %ebp
+; X32-NEXT:    addl %ecx, %ebp
+; X32-NEXT:    adcl $0, %ebx
+; X32-NEXT:    movl %edi, %esi
+; X32-NEXT:    movl %edi, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl %ebx, %ecx
-; X32-NEXT:    addl %esi, %edi
-; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %ebp, %ecx
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %edx, %edi
+; X32-NEXT:    addl %ebp, %eax
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl %ebx, %edi
+; X32-NEXT:    setb %bl
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    addl %edi, %ecx
+; X32-NEXT:    movzbl %bl, %eax
+; X32-NEXT:    adcl %eax, %edx
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X32-NEXT:    adcl $0, (%esp) # 4-byte Folded Spill
+; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl %esi, %ebx
+; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl %esi, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, %edi
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, %ebp
+; X32-NEXT:    movl %eax, %esi
+; X32-NEXT:    addl %edi, %esi
+; X32-NEXT:    adcl $0, %ebp
+; X32-NEXT:    movl %ebx, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edx, %ebx
+; X32-NEXT:    addl %esi, %eax
+; X32-NEXT:    movl %eax, %esi
+; X32-NEXT:    adcl %ebp, %ebx
 ; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    movl 16(%ecx), %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    xorl %ebx, %ebx
-; X32-NEXT:    mull %ebx
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    mull %edi
 ; X32-NEXT:    movl %eax, %edi
-; X32-NEXT:    movl %edx, %ebp
-; X32-NEXT:    movl 20(%ecx), %eax
+; X32-NEXT:    addl %ebx, %edi
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; X32-NEXT:    adcl %eax, %edx
+; X32-NEXT:    addl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, %edi
+; X32-NEXT:    adcl $0, %edx
+; X32-NEXT:    addl (%esp), %edi # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, %ecx
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    mull %ebx
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, %esi
 ; X32-NEXT:    movl %eax, %ebx
-; X32-NEXT:    addl %ebp, %ebx
+; X32-NEXT:    addl %ecx, %ebx
+; X32-NEXT:    adcl $0, %esi
+; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X32-NEXT:    mull %ebp
 ; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    adcl $0, %ecx
-; X32-NEXT:    addl %edi, %ebx
-; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %ebp, %ecx
+; X32-NEXT:    addl %ebx, %eax
+; X32-NEXT:    movl %eax, (%esp) # 4-byte Spill
+; X32-NEXT:    adcl %esi, %ecx
 ; X32-NEXT:    setb %bl
-; X32-NEXT:    addl %eax, %ecx
-; X32-NEXT:    movzbl %bl, %esi
-; X32-NEXT:    adcl %edx, %esi
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl 24(%eax), %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    xorl %edx, %edx
-; X32-NEXT:    mull %edx
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    mull %ebp
+; X32-NEXT:    addl %ecx, %eax
+; X32-NEXT:    movzbl %bl, %ecx
+; X32-NEXT:    adcl %ecx, %edx
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    addl %edi, %ecx
+; X32-NEXT:    movl (%esp), %esi # 4-byte Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 1-byte Folded Reload
+; X32-NEXT:    adcl %edi, %eax
+; X32-NEXT:    adcl $0, %edx
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X32-NEXT:    movl %esi, (%esp) # 4-byte Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %edi, %ebx
-; X32-NEXT:    addl %eax, %ebx
-; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    adcl %edx, %eax
-; X32-NEXT:    addl %ecx, %ebx
-; X32-NEXT:    adcl %esi, %eax
-; X32-NEXT:    movl %eax, %esi
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl 16(%eax), %edi
 ; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    addl %edi, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %ebp, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X32-NEXT:    movl %esi, %eax
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edx, %ebp
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    addl %edi, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X32-NEXT:    adcl %edx, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %ebx, %eax
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edx, %edi
+; X32-NEXT:    movl %eax, %ebx
+; X32-NEXT:    addl %ebp, %ebx
+; X32-NEXT:    adcl $0, %edi
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl 20(%eax), %edx
+; X32-NEXT:    movl %esi, %eax
+; X32-NEXT:    movl %edx, %esi
+; X32-NEXT:    mull %edx
+; X32-NEXT:    movl %edx, %ebp
+; X32-NEXT:    addl %ebx, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %esi, %eax
+; X32-NEXT:    adcl %edi, %ebp
+; X32-NEXT:    setb %bl
+; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %esi, %edi
 ; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    addl %ebp, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movzbl %bl, %eax
+; X32-NEXT:    adcl %eax, %edx
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    addl %edi, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %ebp, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X32-NEXT:    movl %ebx, %eax
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, %ebp
+; X32-NEXT:    movl %eax, %esi
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X32-NEXT:    adcl $0, %ebp
 ; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    addl %edi, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %edx, %eax
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    addl %esi, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %ebx, %eax
+; X32-NEXT:    adcl %ebp, %ecx
+; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X32-NEXT:    movl %ebx, %esi
+; X32-NEXT:    movl %ebx, %eax
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %eax, %edi
+; X32-NEXT:    addl %ecx, %edi
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; X32-NEXT:    adcl %eax, %edx
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl 24(%eax), %ebx
+; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    mull %ebx
+; X32-NEXT:    movl %edx, %ecx
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl %esi, %eax
+; X32-NEXT:    mull %ebx
+; X32-NEXT:    movl %edx, %esi
+; X32-NEXT:    movl %eax, %ebx
+; X32-NEXT:    addl %ecx, %ebx
+; X32-NEXT:    adcl $0, %esi
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl 28(%eax), %ecx
+; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl %edx, %ebp
+; X32-NEXT:    addl %ebx, %eax
+; X32-NEXT:    movl %eax, %ebx
+; X32-NEXT:    adcl %esi, %ebp
+; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %esi, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X32-NEXT:    movl 20(%edi), %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    xorl %ecx, %ecx
 ; X32-NEXT:    mull %ecx
 ; X32-NEXT:    movl %eax, %esi
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    addl %ebx, %esi
-; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    adcl $0, %ecx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
 ; X32-NEXT:    addl %ebp, %esi
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %ebx, %ecx
-; X32-NEXT:    setb %bl
-; X32-NEXT:    addl %eax, %ecx
-; X32-NEXT:    movzbl %bl, %esi
-; X32-NEXT:    adcl %edx, %esi
-; X32-NEXT:    movl 24(%edi), %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    xorl %edx, %edx
-; X32-NEXT:    mull %edx
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %ebp, %edi
-; X32-NEXT:    addl %eax, %edi
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    adcl %edx, %ebx
-; X32-NEXT:    addl %ecx, %edi
-; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %esi, %ebx
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; X32-NEXT:    adcl %eax, %edx
+; X32-NEXT:    addl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
 ; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, %esi
+; X32-NEXT:    adcl $0, %edx
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
 ; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    movl 4(%ecx), %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    xorl %ecx, %ecx
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %eax, %esi
+; X32-NEXT:    movl %edx, %ebx
+; X32-NEXT:    movl %eax, %ebp
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X32-NEXT:    adcl $0, %ebx
+; X32-NEXT:    movl %edi, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    addl %ecx, %esi
+; X32-NEXT:    mull %ecx
 ; X32-NEXT:    movl %edx, %edi
-; X32-NEXT:    adcl $0, %edi
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    addl %ebp, %esi
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %ecx, %edi
-; X32-NEXT:    setb %cl
-; X32-NEXT:    addl %eax, %edi
-; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movzbl %cl, %eax
-; X32-NEXT:    adcl %edx, %eax
-; X32-NEXT:    movl %eax, %ebx
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl 8(%eax), %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    xorl %ecx, %ecx
+; X32-NEXT:    addl %ebp, %eax
+; X32-NEXT:    movl %eax, %ebp
+; X32-NEXT:    adcl %ebx, %edi
+; X32-NEXT:    setb %bl
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %eax, %ecx
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %ebp, %esi
-; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    addl %ecx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    movl %ebp, %ecx
-; X32-NEXT:    adcl %edx, %ecx
 ; X32-NEXT:    addl %edi, %eax
-; X32-NEXT:    adcl %ebx, %ecx
-; X32-NEXT:    movl %esi, %edx
-; X32-NEXT:    movl %esi, %ebx
+; X32-NEXT:    movzbl %bl, %edi
+; X32-NEXT:    adcl %edi, %edx
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NEXT:    addl %esi, %edi
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
+; X32-NEXT:    adcl %ecx, %eax
+; X32-NEXT:    adcl $0, %edx
+; X32-NEXT:    movl %edx, %esi
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    addl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl (%esp), %ecx # 4-byte Reload
+; X32-NEXT:    adcl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    adcl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    adcl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl %edi, %edx
+; X32-NEXT:    adcl $0, %edx
+; X32-NEXT:    movl %ebp, %ecx
+; X32-NEXT:    adcl $0, %ecx
+; X32-NEXT:    adcl $0, %eax
+; X32-NEXT:    adcl $0, %esi
 ; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    movl %esi, %edx
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl %eax, %edx
-; X32-NEXT:    movl %eax, %edi
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    movl %ecx, %edx
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
 ; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    addl %ecx, %eax
-; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    setb (%esp) # 1-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    adcl %ebp, %eax
+; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, %ecx
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, %esi
+; X32-NEXT:    movl %eax, %ebx
 ; X32-NEXT:    addl %ecx, %ebx
-; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X32-NEXT:    adcl $0, %esi
+; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X32-NEXT:    mull %ebp
+; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    addl %ebx, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl %esi, %ecx
+; X32-NEXT:    setb %bl
 ; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X32-NEXT:    mull %ebp
+; X32-NEXT:    addl %ecx, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %edx, %eax
+; X32-NEXT:    movzbl %bl, %eax
+; X32-NEXT:    adcl %eax, %edx
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    movl %ecx, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    adcl %edi, %eax
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X32-NEXT:    addl %edx, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X32-NEXT:    movl %esi, %eax
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edx, %ebx
+; X32-NEXT:    movl %eax, %ebp
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X32-NEXT:    adcl $0, %ebx
+; X32-NEXT:    movl %ecx, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl %ecx, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    addl %edx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %ecx, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X32-NEXT:    addl %edx, %eax
-; X32-NEXT:    adcl %ebp, %esi
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    addl %edx, %ebx
-; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %edi, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT:    movl %eax, %ebx
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl 40(%eax), %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    xorl %ecx, %ecx
 ; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %edx, %edi
+; X32-NEXT:    addl %ebp, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    adcl %ebx, %edi
+; X32-NEXT:    setb %bl
+; X32-NEXT:    movl %esi, %eax
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    addl %edi, %ecx
+; X32-NEXT:    movzbl %bl, %eax
+; X32-NEXT:    adcl %eax, %edx
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    movl %esi, %edx
-; X32-NEXT:    addl %eax, %edx
+; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    adcl %ecx, %ebp
-; X32-NEXT:    addl %edi, %edx
-; X32-NEXT:    adcl %ebx, %ebp
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    addl %esi, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl %ecx, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %edx, %eax
+; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NEXT:    mull %edi
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %ebp, %eax
-; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    movl %esi, %eax
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edx, %ebx
+; X32-NEXT:    movl %eax, %esi
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X32-NEXT:    adcl $0, %ebx
+; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edx, %ebp
 ; X32-NEXT:    addl %esi, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %ecx, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl %eax, %esi
+; X32-NEXT:    adcl %ebx, %ebp
 ; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %edx, %eax
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edi, %ebx
+; X32-NEXT:    movl %eax, %edi
+; X32-NEXT:    addl %ebp, %edi
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; X32-NEXT:    adcl %eax, %edx
+; X32-NEXT:    addl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, %edi
+; X32-NEXT:    adcl $0, %edx
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %ebp, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X32-NEXT:    movl 48(%esi), %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    xorl %ecx, %ecx
-; X32-NEXT:    mull %ecx
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, %esi
 ; X32-NEXT:    movl %eax, %ebp
-; X32-NEXT:    movl %edx, %edi
-; X32-NEXT:    movl 52(%esi), %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X32-NEXT:    adcl $0, %esi
+; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    movl %ebx, %ecx
+; X32-NEXT:    mull %ebx
+; X32-NEXT:    movl %edx, %ebx
+; X32-NEXT:    addl %ebp, %eax
+; X32-NEXT:    movl %eax, %ebp
+; X32-NEXT:    adcl %esi, %ebx
+; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X32-NEXT:    mull %ecx
+; X32-NEXT:    addl %ebx, %eax
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
+; X32-NEXT:    adcl %ecx, %edx
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    addl %edi, %ecx
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 1-byte Folded Reload
+; X32-NEXT:    adcl %esi, %eax
 ; X32-NEXT:    movl %eax, %esi
-; X32-NEXT:    addl %edi, %esi
-; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    adcl $0, %ecx
-; X32-NEXT:    addl %ebp, %esi
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edi, %ecx
-; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    setb %bl
-; X32-NEXT:    addl %eax, %ecx
-; X32-NEXT:    movzbl %bl, %esi
-; X32-NEXT:    adcl %edx, %esi
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl 56(%eax), %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    xorl %edx, %edx
-; X32-NEXT:    mull %edx
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %ebp, %ebx
-; X32-NEXT:    addl %eax, %ebx
-; X32-NEXT:    adcl %edx, %edi
-; X32-NEXT:    addl %ecx, %ebx
-; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %esi, %edi
-; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X32-NEXT:    movl %edx, %eax
-; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    addl %ebp, %eax
+; X32-NEXT:    adcl $0, %edx
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %edx, %eax
-; X32-NEXT:    addl %ebp, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    addl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %ebx, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %edi, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X32-NEXT:    movl 64(%edi), %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    xorl %ecx, %ecx
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    movl %esi, %ecx
-; X32-NEXT:    movl %eax, %ebx
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    addl %eax, %ecx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movzbl (%esp), %eax # 1-byte Folded Reload
+; X32-NEXT:    adcl %eax, %ecx
+; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, %ebp
+; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, %esi
+; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, %edx
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edx, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT:    movl 24(%esi), %ebp
 ; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    addl %ebx, %eax
+; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    adcl %edx, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl 80(%edi), %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    xorl %edx, %edx
-; X32-NEXT:    mull %edx
-; X32-NEXT:    movl %ebp, %edi
+; X32-NEXT:    mull %ebx
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    addl %eax, %edi
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edx, %ebx
-; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    addl %eax, %esi
-; X32-NEXT:    adcl %edx, %ecx
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    movl 80(%ecx), %eax
+; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    movl 28(%esi), %edi
+; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    mull %ebx
+; X32-NEXT:    movl %edx, %esi
+; X32-NEXT:    movl %eax, %ebx
+; X32-NEXT:    addl %ecx, %ebx
+; X32-NEXT:    adcl $0, %esi
+; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X32-NEXT:    mull %ebp
+; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    addl %ebx, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    xorl %edi, %edi
-; X32-NEXT:    mull %edi
+; X32-NEXT:    adcl %esi, %ecx
+; X32-NEXT:    setb %bl
+; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    mull %ebp
+; X32-NEXT:    addl %ecx, %eax
+; X32-NEXT:    movl %eax, (%esp) # 4-byte Spill
+; X32-NEXT:    movzbl %bl, %eax
+; X32-NEXT:    adcl %eax, %edx
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    addl %esi, %eax
-; X32-NEXT:    movl %edx, %eax
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X32-NEXT:    movl 16(%edi), %esi
+; X32-NEXT:    movl %esi, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    adcl %ebx, %eax
+; X32-NEXT:    mull %ebx
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl 64(%ecx), %eax
+; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    movl 20(%edi), %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    mull %edi
+; X32-NEXT:    mull %ebx
+; X32-NEXT:    movl %edx, %ebx
+; X32-NEXT:    movl %eax, %edi
+; X32-NEXT:    addl %ecx, %edi
+; X32-NEXT:    adcl $0, %ebx
+; X32-NEXT:    movl %esi, %eax
+; X32-NEXT:    movl %ebp, %ecx
+; X32-NEXT:    mull %ebp
+; X32-NEXT:    movl %edx, %ebp
+; X32-NEXT:    addl %edi, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl %ebx, %ebp
+; X32-NEXT:    setb %bl
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    mull %ecx
 ; X32-NEXT:    movl %eax, %ecx
-; X32-NEXT:    addl %esi, %ecx
-; X32-NEXT:    movl %edx, %esi
+; X32-NEXT:    addl %ebp, %ecx
+; X32-NEXT:    movzbl %bl, %eax
+; X32-NEXT:    adcl %eax, %edx
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    adcl %ebx, %ecx
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %eax, %ecx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X32-NEXT:    addl %edx, %ecx
+; X32-NEXT:    adcl $0, (%esp) # 4-byte Folded Spill
+; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl %esi, %ebx
+; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl %ecx, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %edx, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    addl %esi, %eax
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %edx, %eax
-; X32-NEXT:    addl %esi, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    adcl %ebp, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl %ecx, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X32-NEXT:    adcl %edx, %eax
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movb {{[-0-9]+}}(%e{{[sb]}}p), %al # 1-byte Reload
-; X32-NEXT:    addb $255, %al
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, %ebp
+; X32-NEXT:    movl %eax, %esi
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X32-NEXT:    adcl $0, %ebp
 ; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    adcl %ecx, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    adcl %edx, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edx, %ebx
 ; X32-NEXT:    addl %esi, %eax
+; X32-NEXT:    movl %eax, %esi
+; X32-NEXT:    adcl %ebp, %ebx
+; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X32-NEXT:    adcl %edx, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edi, %ebp
+; X32-NEXT:    movl %eax, %edi
+; X32-NEXT:    addl %ebx, %edi
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; X32-NEXT:    adcl %eax, %edx
+; X32-NEXT:    addl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, %edi
+; X32-NEXT:    adcl $0, %edx
+; X32-NEXT:    addl (%esp), %edi # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT:    movl %edx, (%esp) # 4-byte Spill
+; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    addl %esi, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %ebp, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    adcl %ebx, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    adcl %edi, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, %esi
+; X32-NEXT:    movl %eax, %ebx
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X32-NEXT:    adcl $0, %esi
 ; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    addl %esi, %eax
+; X32-NEXT:    movl %ebp, %ecx
+; X32-NEXT:    mull %ebp
+; X32-NEXT:    movl %edx, %ebp
+; X32-NEXT:    addl %ebx, %eax
+; X32-NEXT:    movl %eax, %ebx
+; X32-NEXT:    adcl %esi, %ebp
+; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %edx, %eax
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    addl %ebp, %eax
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
+; X32-NEXT:    adcl %ecx, %edx
+; X32-NEXT:    addl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl (%esp), %ebx # 4-byte Folded Reload
+; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
+; X32-NEXT:    adcl %ecx, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, %edx
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT:    movl 8(%esi), %ecx
 ; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    addl %esi, %eax
+; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NEXT:    mull %edi
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %ebp, %eax
+; X32-NEXT:    movl %edx, (%esp) # 4-byte Spill
+; X32-NEXT:    movl 12(%esi), %ebp
+; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edx, %esi
+; X32-NEXT:    movl %eax, %ebx
+; X32-NEXT:    addl (%esp), %ebx # 4-byte Folded Reload
+; X32-NEXT:    adcl $0, %esi
+; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    addl %ebx, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %ebx, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %edi, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl 68(%eax), %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    xorl %ecx, %ecx
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %eax, %edi
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    addl %ebp, %edi
-; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    adcl $0, %ecx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    addl %ebx, %edi
-; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %ebp, %ecx
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    addl %eax, %ecx
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 1-byte Folded Reload
-; X32-NEXT:    adcl %edx, %esi
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl 72(%eax), %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    xorl %edx, %edx
-; X32-NEXT:    mull %edx
+; X32-NEXT:    adcl %esi, %ecx
+; X32-NEXT:    setb %bl
+; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    mull %edi
+; X32-NEXT:    addl %ecx, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %edx, %edi
+; X32-NEXT:    movzbl %bl, %eax
+; X32-NEXT:    adcl %eax, %edx
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %ebx, %edx
-; X32-NEXT:    addl %eax, %ebx
-; X32-NEXT:    adcl %edi, %ebp
-; X32-NEXT:    addl %ecx, %ebx
-; X32-NEXT:    adcl %esi, %ebp
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    addl %edx, %eax
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X32-NEXT:    movl (%ebp), %edi
+; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X32-NEXT:    mull %esi
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    movl 4(%ebp), %eax
+; X32-NEXT:    movl %eax, (%esp) # 4-byte Spill
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, %ebx
+; X32-NEXT:    movl %eax, %ebp
+; X32-NEXT:    addl %ecx, %ebp
+; X32-NEXT:    adcl $0, %ebx
+; X32-NEXT:    movl %edi, %esi
+; X32-NEXT:    movl %edi, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %eax, %ecx
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl %ebx, %ecx
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl %ebp, %ecx
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    addl %edx, %ecx
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    movl 84(%ecx), %eax
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %edx, %edi
+; X32-NEXT:    addl %ebp, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    xorl %ecx, %ecx
+; X32-NEXT:    adcl %ebx, %edi
+; X32-NEXT:    setb %bl
+; X32-NEXT:    movl (%esp), %ebp # 4-byte Reload
+; X32-NEXT:    movl %ebp, %eax
 ; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %eax, %esi
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    addl %ebx, %esi
-; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    adcl $0, %ecx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    addl %ebp, %esi
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    addl %edi, %ecx
+; X32-NEXT:    movzbl %bl, %eax
+; X32-NEXT:    adcl %eax, %edx
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl %esi, %ebx
 ; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %ebx, %ecx
-; X32-NEXT:    movl %ebx, %esi
-; X32-NEXT:    setb %bl
-; X32-NEXT:    addl %eax, %ecx
-; X32-NEXT:    movzbl %bl, %edi
-; X32-NEXT:    adcl %edx, %edi
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl 88(%eax), %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    xorl %edx, %edx
-; X32-NEXT:    mull %edx
+; X32-NEXT:    movl %esi, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, %edi
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, %ebp
+; X32-NEXT:    movl %eax, %esi
+; X32-NEXT:    addl %edi, %esi
+; X32-NEXT:    adcl $0, %ebp
+; X32-NEXT:    movl %ebx, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edx, %ebx
+; X32-NEXT:    addl %esi, %eax
+; X32-NEXT:    movl %eax, %esi
+; X32-NEXT:    adcl %ebp, %ebx
+; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X32-NEXT:    movl (%esp), %eax # 4-byte Reload
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %eax, %edi
+; X32-NEXT:    addl %ebx, %edi
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; X32-NEXT:    adcl %eax, %edx
+; X32-NEXT:    addl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, %edi
+; X32-NEXT:    adcl $0, %edx
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %ebp, %ebx
-; X32-NEXT:    addl %eax, %ebx
-; X32-NEXT:    adcl %edx, %esi
-; X32-NEXT:    addl %ecx, %ebx
-; X32-NEXT:    adcl %edi, %esi
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    movl %ebp, %ecx
-; X32-NEXT:    addl %ebp, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    adcl %ebp, %eax
+; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, %ecx
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %ebx, %eax
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, %esi
+; X32-NEXT:    movl %eax, %ebx
+; X32-NEXT:    addl %ecx, %ebx
+; X32-NEXT:    adcl $0, %esi
+; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X32-NEXT:    mull %ebp
+; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    addl %ebx, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl %esi, %ecx
+; X32-NEXT:    setb %bl
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %esi, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %esi, %eax
+; X32-NEXT:    mull %ebp
+; X32-NEXT:    addl %ecx, %eax
+; X32-NEXT:    movzbl %bl, %ecx
+; X32-NEXT:    adcl %ecx, %edx
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    addl %edi, %ecx
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    addl %ecx, %esi
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 1-byte Folded Reload
+; X32-NEXT:    adcl %edi, %eax
+; X32-NEXT:    adcl $0, %edx
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
 ; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl 84(%eax), %eax
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X32-NEXT:    movl %esi, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edx, %ebp
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    xorl %ecx, %ecx
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %eax, %edi
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edx, %edi
+; X32-NEXT:    movl %eax, %ebx
+; X32-NEXT:    addl %ebp, %ebx
+; X32-NEXT:    adcl $0, %edi
+; X32-NEXT:    movl %esi, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    addl %esi, %edi
-; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    adcl $0, %ecx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    addl %ebp, %edi
-; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %esi, %ecx
-; X32-NEXT:    setb %bl
-; X32-NEXT:    addl %eax, %ecx
-; X32-NEXT:    movzbl %bl, %edi
-; X32-NEXT:    adcl %edx, %edi
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl 88(%eax), %eax
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, %ebp
+; X32-NEXT:    addl %ebx, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    xorl %edx, %edx
-; X32-NEXT:    mull %edx
+; X32-NEXT:    adcl %edi, %ebp
+; X32-NEXT:    setb %bl
+; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    mull %esi
+; X32-NEXT:    addl %ebp, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movzbl %bl, %eax
+; X32-NEXT:    adcl %eax, %edx
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %ebp, %ebx
-; X32-NEXT:    addl %eax, %ebx
-; X32-NEXT:    adcl %edx, %esi
-; X32-NEXT:    addl %ecx, %ebx
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
 ; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edi, %esi
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %ebp, %ecx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ebp
-; X32-NEXT:    movl 68(%ebp), %eax
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %edx, %edi
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    xorl %ecx, %ecx
+; X32-NEXT:    movl (%esp), %eax # 4-byte Reload
 ; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %edx, %ebp
 ; X32-NEXT:    movl %eax, %esi
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
 ; X32-NEXT:    addl %edi, %esi
+; X32-NEXT:    adcl $0, %ebp
+; X32-NEXT:    movl %ebx, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NEXT:    mull %edi
 ; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    adcl $0, %ecx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    addl %ebx, %esi
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edi, %ecx
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    addl %eax, %ecx
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 1-byte Folded Reload
-; X32-NEXT:    adcl %edx, %esi
-; X32-NEXT:    movl 72(%ebp), %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    xorl %edx, %edx
-; X32-NEXT:    mull %edx
+; X32-NEXT:    addl %esi, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %edx, %edi
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %ebx, %edx
-; X32-NEXT:    movl %ebx, %ebp
-; X32-NEXT:    addl %eax, %ebp
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    adcl %edi, %ebx
-; X32-NEXT:    addl %ecx, %ebp
-; X32-NEXT:    adcl %esi, %ebx
-; X32-NEXT:    movl %edx, %eax
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    movl %eax, %ecx
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %ebp, %ecx
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %ebx, %ecx
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl %ebp, %ecx
+; X32-NEXT:    setb %bl
+; X32-NEXT:    movl (%esp), %esi # 4-byte Reload
+; X32-NEXT:    movl %esi, %eax
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %eax, %edi
+; X32-NEXT:    addl %ecx, %edi
+; X32-NEXT:    movzbl %bl, %eax
+; X32-NEXT:    adcl %eax, %edx
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X32-NEXT:    mull %ebx
 ; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X32-NEXT:    addl %edx, %ecx
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    adcl %esi, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    adcl %edi, %ebp
-; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    adcl %ebp, %ebx
-; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl %esi, %eax
+; X32-NEXT:    mull %ebx
+; X32-NEXT:    movl %edx, %esi
+; X32-NEXT:    movl %eax, %ebx
+; X32-NEXT:    addl %ecx, %ebx
+; X32-NEXT:    adcl $0, %esi
+; X32-NEXT:    movl %ebp, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    addl %edx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    addl %edx, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl 12(%eax), %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    xorl %ecx, %ecx
 ; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %eax, %edi
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    addl %esi, %edi
-; X32-NEXT:    movl %edx, %ebx
-; X32-NEXT:    adcl $0, %ebx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    addl %ecx, %edi
-; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %esi, %ebx
+; X32-NEXT:    movl %edx, %ebp
+; X32-NEXT:    addl %ebx, %eax
+; X32-NEXT:    movl %eax, %ebx
+; X32-NEXT:    adcl %esi, %ebp
 ; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    addl %eax, %ebx
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
-; X32-NEXT:    adcl %edx, %eax
-; X32-NEXT:    movl %ecx, %edx
-; X32-NEXT:    addl %ecx, %ebx
-; X32-NEXT:    adcl %esi, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    movl (%esp), %edi # 4-byte Reload
-; X32-NEXT:    addl %ebp, %edi
-; X32-NEXT:    movl %edi, (%esp) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    addl %edi, %esi
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    movl %ebx, %ecx
-; X32-NEXT:    adcl $0, %ecx
-; X32-NEXT:    movl %eax, %edi
-; X32-NEXT:    adcl $0, %edi
+; X32-NEXT:    movl (%esp), %eax # 4-byte Reload
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %eax, %esi
 ; X32-NEXT:    addl %ebp, %esi
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl (%esp), %esi # 4-byte Reload
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; X32-NEXT:    adcl %eax, %edx
+; X32-NEXT:    addl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    adcl $0, %esi
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
 ; X32-NEXT:    adcl $0, %edx
-; X32-NEXT:    addl %ecx, %esi
-; X32-NEXT:    adcl %edi, %edx
-; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    setb %dl
 ; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    movl %esi, (%esp) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movzbl %dl, %edx
-; X32-NEXT:    adcl %ebx, %edx
-; X32-NEXT:    movl %edx, %ebx
-; X32-NEXT:    adcl $0, %eax
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    addl (%esp), %ebp # 4-byte Folded Reload
-; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %ecx, %edx
+; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    mull %ecx
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %ebx, %esi
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %eax, %edi
-; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl 44(%eax), %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    xorl %ecx, %ecx
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    movl %eax, %ebx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    addl %esi, %ebx
-; X32-NEXT:    adcl $0, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    addl %edi, %ebx
-; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %esi, %edx
+; X32-NEXT:    movl %edx, %ebx
+; X32-NEXT:    movl %eax, %ebp
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X32-NEXT:    adcl $0, %ebx
+; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %edx, %edi
+; X32-NEXT:    addl %ebp, %eax
+; X32-NEXT:    movl %eax, %ebp
+; X32-NEXT:    adcl %ebx, %edi
 ; X32-NEXT:    setb %bl
-; X32-NEXT:    addl %eax, %edx
-; X32-NEXT:    movzbl %bl, %eax
-; X32-NEXT:    adcl %ecx, %eax
-; X32-NEXT:    addl %edi, %edx
-; X32-NEXT:    adcl %esi, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    addl %ebp, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %edi
-; X32-NEXT:    movl %edi, (%esp) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl %ebx, %ecx
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    addl %edi, %eax
+; X32-NEXT:    movzbl %bl, %edi
+; X32-NEXT:    adcl %edi, %edx
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NEXT:    addl %esi, %edi
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
+; X32-NEXT:    adcl %ecx, %eax
+; X32-NEXT:    adcl $0, %edx
 ; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    adcl $0, %esi
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    movl %ecx, %edi
-; X32-NEXT:    adcl $0, %edi
-; X32-NEXT:    addl %ebp, (%esp) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl $0, %eax
-; X32-NEXT:    adcl $0, %ebx
-; X32-NEXT:    addl %esi, %eax
-; X32-NEXT:    adcl %edi, %ebx
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 1-byte Folded Reload
-; X32-NEXT:    adcl %edx, %ebp
-; X32-NEXT:    adcl $0, %ecx
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    addl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %ecx
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %ebx, %edx
+; X32-NEXT:    adcl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    adcl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    adcl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl %edi, %edx
+; X32-NEXT:    adcl $0, %edx
+; X32-NEXT:    movl %ebp, %ecx
+; X32-NEXT:    adcl $0, %ecx
+; X32-NEXT:    adcl $0, %eax
+; X32-NEXT:    adcl $0, %esi
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %ebp, %esi
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    addb $255, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl (%esp), %eax # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
 ; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %edx, %eax
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %esi, %eax
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
 ; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl 12(%eax), %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    xorl %ecx, %ecx
-; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X32-NEXT:    mull %esi
 ; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, %esi
 ; X32-NEXT:    movl %eax, %ebx
+; X32-NEXT:    addl %ecx, %ebx
+; X32-NEXT:    adcl $0, %esi
+; X32-NEXT:    movl %edi, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    addl %edi, %ebx
-; X32-NEXT:    adcl $0, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    addl %esi, %ebx
-; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edi, %edx
-; X32-NEXT:    setb %bl
-; X32-NEXT:    addl %eax, %edx
-; X32-NEXT:    movzbl %bl, %ebp
-; X32-NEXT:    adcl %ecx, %ebp
-; X32-NEXT:    movl %esi, %ecx
-; X32-NEXT:    addl %esi, %edx
-; X32-NEXT:    adcl %edi, %ebp
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edx, %ecx
 ; X32-NEXT:    addl %ebx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %ecx, %edi
-; X32-NEXT:    addl %eax, %edi
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    adcl %esi, %ecx
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    adcl $0, %ecx
-; X32-NEXT:    movl %ebp, %esi
-; X32-NEXT:    adcl $0, %esi
-; X32-NEXT:    addl %ebx, %edi
-; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    adcl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl $0, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    adcl $0, %edi
+; X32-NEXT:    setb %bl
+; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    mull %edi
 ; X32-NEXT:    addl %ecx, %eax
-; X32-NEXT:    adcl %esi, %edi
-; X32-NEXT:    setb %cl
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movzbl %cl, %ecx
-; X32-NEXT:    adcl %edx, %ecx
-; X32-NEXT:    adcl $0, %ebp
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %ebx
-; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %ecx, %esi
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %ebp, %edi
-; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    addb $255, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl 44(%eax), %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    xorl %ecx, %ecx
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %eax, %ebx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    addl %esi, %ebx
-; X32-NEXT:    movl %edx, %ebp
-; X32-NEXT:    adcl $0, %ebp
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    addl %edi, %ebx
-; X32-NEXT:    adcl %esi, %ebp
-; X32-NEXT:    setb %cl
-; X32-NEXT:    addl %eax, %ebp
-; X32-NEXT:    movzbl %cl, %eax
-; X32-NEXT:    adcl %edx, %eax
-; X32-NEXT:    addl %edi, %ebp
-; X32-NEXT:    adcl %esi, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    addl %esi, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %edi
-; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %ebx, %edx
-; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %ebp, %edx
-; X32-NEXT:    adcl $0, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    adcl $0, %edi
-; X32-NEXT:    addl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl $0, %eax
-; X32-NEXT:    adcl $0, %ecx
-; X32-NEXT:    addl %edx, %eax
-; X32-NEXT:    adcl %edi, %ecx
-; X32-NEXT:    setb %dl
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %ebx, %ecx
-; X32-NEXT:    movzbl %dl, %eax
-; X32-NEXT:    adcl %ebp, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    adcl $0, %ebp
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    adcl %eax, %edi
-; X32-NEXT:    adcl %ebp, %esi
-; X32-NEXT:    addb $255, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    addb $255, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    adcl (%esp), %eax # 4-byte Folded Reload
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT:    movzbl %bl, %eax
+; X32-NEXT:    adcl %eax, %edx
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    mull %edi
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %edx, %ebp
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl 60(%eax), %esi
-; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    mull %edi
-; X32-NEXT:    movl %edx, %edi
-; X32-NEXT:    movl %eax, %ebx
-; X32-NEXT:    addl %ebp, %ebx
-; X32-NEXT:    adcl $0, %edi
-; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %ebp
-; X32-NEXT:    addl %ebx, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edi, %ebp
-; X32-NEXT:    setb %bl
-; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    addl %ebp, %eax
-; X32-NEXT:    movzbl %bl, %edi
-; X32-NEXT:    adcl %edi, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %ecx
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edx, %esi
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    mull %ebx
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X32-NEXT:    mull %ebp
 ; X32-NEXT:    movl %edx, %edi
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    mull %ebx
+; X32-NEXT:    mull %ebp
 ; X32-NEXT:    movl %edx, %ebx
 ; X32-NEXT:    movl %eax, %ebp
 ; X32-NEXT:    addl %edi, %ebp
@@ -1308,17 +1086,50 @@ define void @test_1024(i1024* %a, i1024* %b, i1024* %out) nounwind {
 ; X32-NEXT:    setb %bl
 ; X32-NEXT:    movl %esi, %eax
 ; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %eax, %ebp
-; X32-NEXT:    addl %edi, %ebp
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    addl %edi, %ecx
 ; X32-NEXT:    movzbl %bl, %eax
 ; X32-NEXT:    adcl %eax, %edx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
 ; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X32-NEXT:    mull %ebx
+; X32-NEXT:    movl %edx, %edi
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl %esi, %eax
+; X32-NEXT:    mull %ebx
+; X32-NEXT:    movl %edx, %ebx
+; X32-NEXT:    movl %eax, %esi
+; X32-NEXT:    addl %edi, %esi
+; X32-NEXT:    adcl $0, %ebx
+; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edx, %ebp
+; X32-NEXT:    addl %esi, %eax
+; X32-NEXT:    movl %eax, %esi
+; X32-NEXT:    adcl %ebx, %ebp
+; X32-NEXT:    setb %bl
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %eax, %edi
+; X32-NEXT:    addl %ebp, %edi
+; X32-NEXT:    movzbl %bl, %eax
+; X32-NEXT:    adcl %eax, %edx
+; X32-NEXT:    addl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, %edi
+; X32-NEXT:    adcl $0, %edx
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
 ; X32-NEXT:    movl %ebx, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
@@ -1328,687 +1139,807 @@ define void @test_1024(i1024* %a, i1024* %b, i1024* %out) nounwind {
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X32-NEXT:    mull %esi
 ; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    movl %eax, %edi
-; X32-NEXT:    addl %ecx, %edi
+; X32-NEXT:    movl %eax, %ebp
+; X32-NEXT:    addl %ecx, %ebp
 ; X32-NEXT:    adcl $0, %esi
 ; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %ebx
-; X32-NEXT:    addl %edi, %eax
-; X32-NEXT:    movl %eax, %edi
-; X32-NEXT:    adcl %esi, %ebx
-; X32-NEXT:    setb (%esp) # 1-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X32-NEXT:    mull %ebx
+; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    addl %ebp, %eax
+; X32-NEXT:    movl %eax, %ebp
+; X32-NEXT:    adcl %esi, %ecx
+; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    addl %ebx, %eax
-; X32-NEXT:    movzbl (%esp), %ecx # 1-byte Folded Reload
+; X32-NEXT:    mull %ebx
+; X32-NEXT:    addl %ecx, %eax
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
 ; X32-NEXT:    adcl %ecx, %edx
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X32-NEXT:    addl %edi, %ebx
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
+; X32-NEXT:    adcl %ecx, %eax
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, %edx
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %ecx
-; X32-NEXT:    adcl %edx, %esi
-; X32-NEXT:    addl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl $0, %ecx
-; X32-NEXT:    adcl $0, %esi
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; X32-NEXT:    adcl %eax, %ebx
+; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, %ebp
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    mull %edi
-; X32-NEXT:    movl %edx, %ebp
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %edi
-; X32-NEXT:    movl %eax, %edi
-; X32-NEXT:    addl %ebp, %edi
-; X32-NEXT:    adcl $0, %edx
-; X32-NEXT:    movl %edx, (%esp) # 4-byte Spill
-; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    mull %ebp
-; X32-NEXT:    movl %edx, %ebx
-; X32-NEXT:    addl %edi, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl (%esp), %ebx # 4-byte Folded Reload
-; X32-NEXT:    setb (%esp) # 1-byte Folded Spill
+; X32-NEXT:    adcl $0, %ebx
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %ebp
-; X32-NEXT:    addl %ebx, %eax
-; X32-NEXT:    movzbl (%esp), %edi # 1-byte Folded Reload
-; X32-NEXT:    adcl %edi, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %edi
-; X32-NEXT:    adcl %edx, %ebx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X32-NEXT:    addl %ecx, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl %esi, %ecx
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
-; X32-NEXT:    adcl %eax, %edi
-; X32-NEXT:    adcl $0, %ebx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X32-NEXT:    adcl $0, %eax
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
 ; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
 ; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
 ; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    mull %edi
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X32-NEXT:    movl 32(%ebx), %esi
+; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X32-NEXT:    movl %ebx, %eax
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    mull %esi
 ; X32-NEXT:    movl %edx, %esi
+; X32-NEXT:    movl %eax, %edi
+; X32-NEXT:    addl %ecx, %edi
+; X32-NEXT:    adcl $0, %esi
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl 36(%eax), %ecx
+; X32-NEXT:    movl %ebx, %eax
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %ecx, %ebx
+; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    addl %edi, %eax
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl %esi, %ecx
+; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    mull %ebx
+; X32-NEXT:    addl %ecx, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; X32-NEXT:    adcl %eax, %edx
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
 ; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    mull %edi
-; X32-NEXT:    movl %edx, %edi
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, %esi
 ; X32-NEXT:    movl %eax, %ebp
-; X32-NEXT:    addl %esi, %ebp
-; X32-NEXT:    adcl $0, %edi
-; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    addl %ecx, %ebp
+; X32-NEXT:    adcl $0, %esi
+; X32-NEXT:    movl %ebx, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %esi
+; X32-NEXT:    movl %edx, %ebx
 ; X32-NEXT:    addl %ebp, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edi, %esi
+; X32-NEXT:    adcl %esi, %ebx
 ; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl %ebx, %eax
+; X32-NEXT:    movl %edi, %eax
 ; X32-NEXT:    mull %ecx
-; X32-NEXT:    addl %esi, %eax
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
-; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    addl %ebx, %ecx
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; X32-NEXT:    adcl %eax, %edx
 ; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl 40(%eax), %ebx
+; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %ecx
-; X32-NEXT:    movl %ecx, (%esp) # 4-byte Spill
-; X32-NEXT:    adcl %edx, %esi
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    mull %edi
-; X32-NEXT:    movl %edx, %esi
+; X32-NEXT:    movl %esi, %eax
+; X32-NEXT:    mull %ebx
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    mull %edi
-; X32-NEXT:    movl %edx, %edi
-; X32-NEXT:    movl %eax, %ebp
-; X32-NEXT:    addl %esi, %ebp
-; X32-NEXT:    adcl $0, %edi
-; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    mull %ebx
+; X32-NEXT:    movl %edx, %ebp
+; X32-NEXT:    movl %eax, %edi
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT:    adcl $0, %ebp
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl 44(%eax), %ebx
+; X32-NEXT:    movl %esi, %eax
 ; X32-NEXT:    mull %ebx
 ; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    addl %ebp, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edi, %esi
+; X32-NEXT:    addl %edi, %eax
+; X32-NEXT:    movl %eax, %edi
+; X32-NEXT:    adcl %ebp, %esi
 ; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X32-NEXT:    mull %ebx
 ; X32-NEXT:    movl %eax, %ebp
 ; X32-NEXT:    addl %esi, %ebp
 ; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
 ; X32-NEXT:    adcl %eax, %edx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT:    addl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, %ebp
+; X32-NEXT:    adcl $0, %edx
 ; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl $0, (%esp) # 4-byte Folded Spill
-; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    movl %ecx, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X32-NEXT:    mull %esi
 ; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    movl %eax, %ebx
-; X32-NEXT:    addl %ecx, %ebx
+; X32-NEXT:    movl %eax, %edi
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
 ; X32-NEXT:    adcl $0, %esi
-; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %edi
-; X32-NEXT:    addl %ebx, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %esi, %edi
-; X32-NEXT:    setb %bl
+; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    movl %ebx, %ecx
+; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    mull %ebx
+; X32-NEXT:    movl %edx, %ebx
+; X32-NEXT:    addl %edi, %eax
+; X32-NEXT:    movl %eax, %edi
+; X32-NEXT:    adcl %esi, %ebx
+; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X32-NEXT:    mull %ecx
-; X32-NEXT:    addl %edi, %eax
-; X32-NEXT:    movzbl %bl, %ecx
+; X32-NEXT:    addl %ebx, %eax
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
 ; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %ebx
-; X32-NEXT:    adcl %edx, %ecx
 ; X32-NEXT:    addl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl $0, %ebx
-; X32-NEXT:    adcl $0, %ecx
-; X32-NEXT:    addl (%esp), %ebx # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    setb (%esp) # 1-byte Folded Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
+; X32-NEXT:    adcl %ecx, %eax
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, %edx
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
 ; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl %esi, %eax
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edx, %edi
+; X32-NEXT:    movl %eax, %ebx
+; X32-NEXT:    addl %ecx, %ebx
+; X32-NEXT:    adcl $0, %edi
+; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X32-NEXT:    mull %ebp
+; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    addl %ebx, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %eax, %edi
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    adcl $0, %edx
+; X32-NEXT:    adcl %edi, %ecx
+; X32-NEXT:    setb %bl
+; X32-NEXT:    movl %esi, %eax
+; X32-NEXT:    mull %ebp
+; X32-NEXT:    addl %ecx, %eax
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movzbl %bl, %eax
+; X32-NEXT:    adcl %eax, %edx
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    movl %ecx, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %ebp
-; X32-NEXT:    addl %edi, %eax
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    movl (%esp), %edi # 4-byte Reload
+; X32-NEXT:    movl %edi, %eax
 ; X32-NEXT:    mull %esi
-; X32-NEXT:    addl %ebp, %eax
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 1-byte Folded Reload
-; X32-NEXT:    adcl %esi, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X32-NEXT:    movl %edx, %ebx
+; X32-NEXT:    movl %eax, %ebp
 ; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %ebp
-; X32-NEXT:    adcl %edx, %edi
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X32-NEXT:    addl %ebx, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    adcl %ecx, %esi
-; X32-NEXT:    movzbl (%esp), %eax # 1-byte Folded Reload
-; X32-NEXT:    adcl %eax, %ebp
-; X32-NEXT:    adcl $0, %edi
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT:    adcl $0, %ebx
+; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %edx, %esi
+; X32-NEXT:    addl %ebp, %eax
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl %ebx, %esi
+; X32-NEXT:    setb %bl
+; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    movl %edi, %ebp
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    addl %esi, %ecx
+; X32-NEXT:    movzbl %bl, %eax
+; X32-NEXT:    adcl %eax, %edx
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
 ; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
 ; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
 ; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NEXT:    mull %edi
 ; X32-NEXT:    movl %edx, %esi
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edx, %ebp
+; X32-NEXT:    movl %eax, %edi
+; X32-NEXT:    addl %esi, %edi
+; X32-NEXT:    adcl $0, %ebp
+; X32-NEXT:    movl %ebx, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, %ebx
+; X32-NEXT:    addl %edi, %eax
+; X32-NEXT:    movl %eax, %edi
+; X32-NEXT:    adcl %ebp, %ebx
+; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X32-NEXT:    movl (%esp), %eax # 4-byte Reload
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %eax, %esi
+; X32-NEXT:    addl %ebx, %esi
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; X32-NEXT:    adcl %eax, %edx
+; X32-NEXT:    addl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, %esi
+; X32-NEXT:    adcl $0, %edx
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
 ; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    mull %edi
 ; X32-NEXT:    movl %edx, %edi
-; X32-NEXT:    movl %eax, %ecx
-; X32-NEXT:    addl %esi, %ecx
+; X32-NEXT:    movl %eax, %ebx
+; X32-NEXT:    addl %ecx, %ebx
 ; X32-NEXT:    adcl $0, %edi
-; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    addl %ecx, %eax
-; X32-NEXT:    movl %eax, (%esp) # 4-byte Spill
-; X32-NEXT:    adcl %edi, %esi
-; X32-NEXT:    setb %cl
 ; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    addl %esi, %eax
-; X32-NEXT:    movzbl %cl, %ecx
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X32-NEXT:    mull %ebp
+; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    addl %ebx, %eax
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl %edi, %ecx
+; X32-NEXT:    setb %bl
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    mull %ebp
+; X32-NEXT:    addl %ecx, %eax
+; X32-NEXT:    movzbl %bl, %ecx
 ; X32-NEXT:    adcl %ecx, %edx
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X32-NEXT:    addl %esi, %ecx
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %ecx
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 1-byte Folded Reload
+; X32-NEXT:    adcl %ebx, %eax
+; X32-NEXT:    adcl $0, %edx
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
 ; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edx, %esi
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl 48(%eax), %esi
 ; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
 ; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    movl %edx, %ebp
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    movl %edi, %ecx
 ; X32-NEXT:    mull %esi
 ; X32-NEXT:    movl %edx, %esi
 ; X32-NEXT:    movl %eax, %edi
-; X32-NEXT:    addl %ecx, %edi
+; X32-NEXT:    addl %ebp, %edi
 ; X32-NEXT:    adcl $0, %esi
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl 52(%eax), %edx
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl %ebx, %eax
+; X32-NEXT:    mull %edx
+; X32-NEXT:    movl %edx, %ebp
+; X32-NEXT:    addl %edi, %eax
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl %esi, %ebp
+; X32-NEXT:    setb %bl
+; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X32-NEXT:    mull %esi
+; X32-NEXT:    addl %ebp, %eax
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movzbl %bl, %eax
+; X32-NEXT:    adcl %eax, %edx
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    movl %ecx, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
 ; X32-NEXT:    mull %ebx
-; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    addl %edi, %eax
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %esi, %ecx
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    movl (%esp), %edi # 4-byte Reload
+; X32-NEXT:    movl %edi, %eax
 ; X32-NEXT:    mull %ebx
-; X32-NEXT:    movl %eax, %edi
-; X32-NEXT:    addl %ecx, %edi
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; X32-NEXT:    movl %edx, %ebp
+; X32-NEXT:    movl %eax, %ebx
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X32-NEXT:    adcl $0, %ebp
+; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    addl %ebx, %eax
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl %ebp, %ecx
+; X32-NEXT:    setb %bl
+; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %eax, %esi
+; X32-NEXT:    addl %ecx, %esi
+; X32-NEXT:    movzbl %bl, %eax
 ; X32-NEXT:    adcl %eax, %edx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    adcl (%esp), %edx # 4-byte Folded Reload
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
 ; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl 56(%eax), %ecx
+; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X32-NEXT:    movl %ebp, %eax
 ; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %ebx
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %edx, %ebx
+; X32-NEXT:    movl %eax, %edi
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT:    adcl $0, %ebx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl 60(%eax), %ecx
 ; X32-NEXT:    movl %ebp, %eax
 ; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl %edx, %ebp
-; X32-NEXT:    movl %eax, %ecx
-; X32-NEXT:    addl %ebx, %ecx
-; X32-NEXT:    adcl $0, %ebp
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl 28(%eax), %ebx
-; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    movl %ebx, %esi
-; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %edx, %ebx
-; X32-NEXT:    addl %ecx, %eax
-; X32-NEXT:    movl %eax, (%esp) # 4-byte Spill
-; X32-NEXT:    adcl %ebp, %ebx
-; X32-NEXT:    setb %cl
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    addl %ebx, %eax
-; X32-NEXT:    movzbl %cl, %ecx
-; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %ebp
-; X32-NEXT:    adcl %edx, %ecx
-; X32-NEXT:    addl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl (%esp), %eax # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT:    movl %eax, (%esp) # 4-byte Spill
-; X32-NEXT:    adcl $0, %ebp
-; X32-NEXT:    adcl $0, %ecx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X32-NEXT:    addl %edi, %eax
+; X32-NEXT:    movl %eax, %edi
+; X32-NEXT:    adcl %ebx, %ebp
 ; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %esi
+; X32-NEXT:    movl (%esp), %eax # 4-byte Reload
+; X32-NEXT:    mull %ecx
 ; X32-NEXT:    movl %eax, %ebx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X32-NEXT:    addl %ebp, %ebx
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; X32-NEXT:    adcl %eax, %edx
+; X32-NEXT:    addl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, %ebx
 ; X32-NEXT:    adcl $0, %edx
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %edi
-; X32-NEXT:    addl %ebx, %eax
+; X32-NEXT:    movl %esi, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    setb %bl
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    addl %edi, %eax
-; X32-NEXT:    movzbl %bl, %esi
-; X32-NEXT:    adcl %esi, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %edi
-; X32-NEXT:    adcl %edx, %ebx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X32-NEXT:    addl %ebp, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    adcl %ecx, %esi
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
-; X32-NEXT:    adcl %eax, %edi
-; X32-NEXT:    adcl $0, %ebx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    addl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %edx, %edi
+; X32-NEXT:    movl %eax, %ebp
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X32-NEXT:    adcl $0, %edi
+; X32-NEXT:    movl %esi, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %edx, %esi
+; X32-NEXT:    addl %ebp, %eax
+; X32-NEXT:    movl %eax, %ebp
+; X32-NEXT:    adcl %edi, %esi
+; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %eax, (%esp) # 4-byte Folded Spill
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    addl %esi, %eax
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 1-byte Folded Reload
+; X32-NEXT:    adcl %esi, %edx
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NEXT:    addl %ebx, %edi
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
+; X32-NEXT:    adcl %ecx, %eax
+; X32-NEXT:    adcl $0, %edx
+; X32-NEXT:    movl %edx, %esi
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    addl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    adcl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    adcl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    adcl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl %edi, %edx
 ; X32-NEXT:    adcl $0, %edx
+; X32-NEXT:    movl %ebp, %ecx
+; X32-NEXT:    adcl $0, %ecx
+; X32-NEXT:    adcl $0, %eax
 ; X32-NEXT:    adcl $0, %esi
-; X32-NEXT:    adcl $0, %edi
-; X32-NEXT:    adcl $0, %ebx
 ; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
 ; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
 ; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %ebx
-; X32-NEXT:    movl %eax, %esi
-; X32-NEXT:    addl %ecx, %esi
-; X32-NEXT:    adcl $0, %ebx
-; X32-NEXT:    movl %edi, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
 ; X32-NEXT:    mull %edi
 ; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    addl %esi, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %ebx, %ecx
-; X32-NEXT:    setb %bl
-; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    mull %edi
-; X32-NEXT:    addl %ecx, %eax
-; X32-NEXT:    movzbl %bl, %ecx
-; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %ecx
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edx, %esi
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    movl %ebp, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %esi
+; X32-NEXT:    movl %esi, %eax
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edx, %edi
 ; X32-NEXT:    movl %eax, %ebx
 ; X32-NEXT:    addl %ecx, %ebx
-; X32-NEXT:    adcl $0, %esi
+; X32-NEXT:    adcl $0, %edi
 ; X32-NEXT:    movl %ebp, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
 ; X32-NEXT:    mull %ebp
 ; X32-NEXT:    movl %edx, %ecx
 ; X32-NEXT:    addl %ebx, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %esi, %ecx
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    adcl %edi, %ecx
+; X32-NEXT:    setb %bl
+; X32-NEXT:    movl %esi, %eax
 ; X32-NEXT:    mull %ebp
-; X32-NEXT:    movl %eax, %ebx
-; X32-NEXT:    addl %ecx, %ebx
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; X32-NEXT:    addl %ecx, %eax
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movzbl %bl, %eax
 ; X32-NEXT:    adcl %eax, %edx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X32-NEXT:    movl %ebx, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, %edi
+; X32-NEXT:    movl %eax, %ebp
+; X32-NEXT:    addl %ecx, %ebp
+; X32-NEXT:    adcl $0, %edi
+; X32-NEXT:    movl %ebx, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl %edx, %esi
+; X32-NEXT:    addl %ebp, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl %edi, %esi
+; X32-NEXT:    setb %bl
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
 ; X32-NEXT:    movl %edi, %eax
 ; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %esi
 ; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    addl %esi, %ecx
+; X32-NEXT:    movzbl %bl, %eax
+; X32-NEXT:    adcl %eax, %edx
 ; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    adcl $0, %esi
-; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    mull %edi
-; X32-NEXT:    movl %edx, %ebp
-; X32-NEXT:    addl %ecx, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %esi, %ebp
-; X32-NEXT:    setb %cl
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %edi
-; X32-NEXT:    addl %ebp, %eax
-; X32-NEXT:    movzbl %cl, %ecx
-; X32-NEXT:    adcl %ecx, %edx
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %ebp
-; X32-NEXT:    adcl %edx, %ecx
-; X32-NEXT:    addl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl $0, %ebp
-; X32-NEXT:    adcl $0, %ecx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    movl %ebx, %eax
+; X32-NEXT:    movl %ebp, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X32-NEXT:    mull %esi
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, %edi
+; X32-NEXT:    movl %eax, %ebx
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X32-NEXT:    adcl $0, %edi
+; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, %ebp
+; X32-NEXT:    addl %ebx, %eax
+; X32-NEXT:    movl %eax, %ebx
+; X32-NEXT:    adcl %edi, %ebp
+; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X32-NEXT:    mull %esi
 ; X32-NEXT:    movl %eax, %edi
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT:    addl %ebp, %edi
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; X32-NEXT:    adcl %eax, %edx
+; X32-NEXT:    addl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, %edi
 ; X32-NEXT:    adcl $0, %edx
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %ebx, %eax
+; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    movl %ecx, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %ebx
-; X32-NEXT:    addl %edi, %eax
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X32-NEXT:    mull %esi
-; X32-NEXT:    addl %ebx, %eax
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 1-byte Folded Reload
-; X32-NEXT:    adcl %esi, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT:    movl %edx, %ebx
+; X32-NEXT:    movl %eax, %ebp
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X32-NEXT:    adcl $0, %ebx
+; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %edx, %esi
+; X32-NEXT:    addl %ebp, %eax
+; X32-NEXT:    movl %eax, %ebp
+; X32-NEXT:    adcl %ebx, %esi
+; X32-NEXT:    setb %bl
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    addl %esi, %eax
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    movzbl %bl, %eax
+; X32-NEXT:    adcl %eax, %edx
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %edi
-; X32-NEXT:    adcl %edx, %ebx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X32-NEXT:    addl %ebp, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    adcl %ecx, %esi
+; X32-NEXT:    addl %edi, %ebx
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
 ; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; X32-NEXT:    movl %ecx, %edi
 ; X32-NEXT:    adcl %eax, %edi
-; X32-NEXT:    adcl $0, %ebx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    movl %edx, %ebp
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, %edx
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
 ; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
-; X32-NEXT:    adcl %eax, %ebp
+; X32-NEXT:    adcl %eax, %ebx
+; X32-NEXT:    adcl $0, %ebp
+; X32-NEXT:    adcl $0, %edi
+; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NEXT:    adcl $0, %edi
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    addl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl %ecx, %eax
 ; X32-NEXT:    adcl $0, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    adcl $0, %ebx
 ; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, %edx
+; X32-NEXT:    adcl $0, %esi
+; X32-NEXT:    adcl $0, %ebx
+; X32-NEXT:    adcl $0, %ebp
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    adcl $0, %ecx
+; X32-NEXT:    adcl $0, %edi
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
 ; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
 ; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X32-NEXT:    movl %ebx, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, %ecx
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
 ; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    mull %edi
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    mull %esi
 ; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl 28(%eax), %ecx
-; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    mull %edi
-; X32-NEXT:    movl %edx, %edi
-; X32-NEXT:    movl %eax, %ebx
-; X32-NEXT:    addl %esi, %ebx
-; X32-NEXT:    adcl $0, %edi
+; X32-NEXT:    movl %eax, %edi
+; X32-NEXT:    addl %ecx, %edi
+; X32-NEXT:    adcl $0, %esi
+; X32-NEXT:    movl %ebx, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X32-NEXT:    mull %ebx
+; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    addl %edi, %eax
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl %esi, %ecx
+; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
 ; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    mull %ebx
+; X32-NEXT:    addl %ecx, %eax
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; X32-NEXT:    adcl %eax, %edx
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NEXT:    movl %edi, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %ebp
-; X32-NEXT:    addl %ebx, %eax
+; X32-NEXT:    movl %edx, %ebx
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edi, %ebp
-; X32-NEXT:    setb %bl
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X32-NEXT:    movl %ecx, %eax
 ; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, %esi
+; X32-NEXT:    movl %eax, %ebp
+; X32-NEXT:    addl %ebx, %ebp
+; X32-NEXT:    adcl $0, %esi
+; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edx, %ebx
 ; X32-NEXT:    addl %ebp, %eax
-; X32-NEXT:    movzbl %bl, %edi
-; X32-NEXT:    adcl %edi, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %ecx
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edx, %esi
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl %esi, %ebx
+; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X32-NEXT:    movl %ecx, %esi
 ; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    addl %ebx, %ecx
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; X32-NEXT:    adcl %eax, %edx
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    movl %edx, %edi
+; X32-NEXT:    movl %ebx, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X32-NEXT:    movl %esi, %eax
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edx, %ebp
+; X32-NEXT:    movl %eax, %edi
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT:    adcl $0, %ebp
+; X32-NEXT:    movl %ebx, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
 ; X32-NEXT:    mull %ebx
-; X32-NEXT:    movl %edx, %ebx
-; X32-NEXT:    movl %eax, %ebp
-; X32-NEXT:    addl %edi, %ebp
-; X32-NEXT:    adcl $0, %ebx
-; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %edi
-; X32-NEXT:    addl %ebp, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %ebx, %edi
+; X32-NEXT:    movl %edx, %esi
+; X32-NEXT:    addl %edi, %eax
+; X32-NEXT:    movl %eax, %edi
+; X32-NEXT:    adcl %ebp, %esi
 ; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl %esi, %ebx
-; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    mull %ebx
 ; X32-NEXT:    movl %eax, %ebp
-; X32-NEXT:    addl %edi, %ebp
+; X32-NEXT:    addl %esi, %ebp
 ; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
 ; X32-NEXT:    adcl %eax, %edx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT:    addl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, %ebp
+; X32-NEXT:    adcl $0, %edx
 ; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X32-NEXT:    movl %ecx, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X32-NEXT:    mull %esi
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %ebx, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X32-NEXT:    mull %esi
 ; X32-NEXT:    movl %edx, %esi
 ; X32-NEXT:    movl %eax, %edi
 ; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
 ; X32-NEXT:    adcl $0, %esi
 ; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %ebx, %ecx
+; X32-NEXT:    mull %ebx
 ; X32-NEXT:    movl %edx, %ebx
 ; X32-NEXT:    addl %edi, %eax
 ; X32-NEXT:    movl %eax, %edi
@@ -2019,291 +1950,211 @@ define void @test_1024(i1024* %a, i1024* %b, i1024* %out) nounwind {
 ; X32-NEXT:    addl %ebx, %eax
 ; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
 ; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %ecx
-; X32-NEXT:    adcl %edx, %esi
 ; X32-NEXT:    addl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
 ; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl $0, %ecx
-; X32-NEXT:    adcl $0, %esi
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
+; X32-NEXT:    adcl %ecx, %eax
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, %edx
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
 ; X32-NEXT:    movl %ebx, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
 ; X32-NEXT:    mull %edi
-; X32-NEXT:    movl %edx, %ebp
+; X32-NEXT:    movl %edx, %ecx
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X32-NEXT:    movl %esi, %eax
 ; X32-NEXT:    mull %edi
-; X32-NEXT:    movl %eax, %edi
-; X32-NEXT:    addl %ebp, %edi
-; X32-NEXT:    adcl $0, %edx
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl %edx, %edi
+; X32-NEXT:    movl %eax, %ebp
+; X32-NEXT:    addl %ecx, %ebp
+; X32-NEXT:    adcl $0, %edi
 ; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    mull %ebp
-; X32-NEXT:    movl %edx, %ebx
-; X32-NEXT:    addl %edi, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X32-NEXT:    mull %ebx
+; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    addl %ebp, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X32-NEXT:    adcl %edi, %ecx
 ; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %ebp
-; X32-NEXT:    addl %ebx, %eax
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 1-byte Folded Reload
-; X32-NEXT:    adcl %edi, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %edi
-; X32-NEXT:    adcl %edx, %ebx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X32-NEXT:    addl %ecx, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl %esi, %ecx
+; X32-NEXT:    movl %esi, %eax
+; X32-NEXT:    mull %ebx
+; X32-NEXT:    addl %ecx, %eax
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
-; X32-NEXT:    adcl %eax, %edi
-; X32-NEXT:    adcl $0, %ebx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT:    adcl %eax, %edx
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    movl %ebp, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %esi
+; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %edi
-; X32-NEXT:    movl %eax, %ebx
-; X32-NEXT:    addl %esi, %ebx
-; X32-NEXT:    adcl $0, %edi
-; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    addl %ebx, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edi, %esi
-; X32-NEXT:    setb %bl
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    addl %esi, %eax
-; X32-NEXT:    movzbl %bl, %esi
-; X32-NEXT:    adcl %esi, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %ecx
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edx, %esi
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %edi
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, %ebp
 ; X32-NEXT:    movl %eax, %ebx
 ; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    adcl $0, %edi
-; X32-NEXT:    movl %esi, %eax
+; X32-NEXT:    adcl $0, %ebp
+; X32-NEXT:    movl %ecx, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X32-NEXT:    mull %ecx
 ; X32-NEXT:    movl %edx, %esi
 ; X32-NEXT:    addl %ebx, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edi, %esi
+; X32-NEXT:    adcl %ebp, %esi
 ; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    movl %edi, %ebx
 ; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %eax, %ebx
-; X32-NEXT:    addl %esi, %ebx
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    addl %esi, %ecx
 ; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
 ; X32-NEXT:    adcl %eax, %edx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
 ; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
 ; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    mull %esi
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NEXT:    mull %edi
 ; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    movl %eax, %ebp
-; X32-NEXT:    addl %ecx, %ebp
-; X32-NEXT:    adcl $0, %esi
-; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %edi
-; X32-NEXT:    addl %ebp, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %esi, %edi
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    addl %edi, %eax
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
-; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %ebp
-; X32-NEXT:    adcl %edx, %edi
-; X32-NEXT:    addl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl $0, %ebp
-; X32-NEXT:    adcl $0, %edi
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
 ; X32-NEXT:    movl %ebx, %eax
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edx, %ebx
+; X32-NEXT:    movl %eax, %edi
+; X32-NEXT:    addl %esi, %edi
+; X32-NEXT:    adcl $0, %ebx
+; X32-NEXT:    movl %ebp, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl %edx, %ebp
+; X32-NEXT:    addl %edi, %eax
+; X32-NEXT:    movl %eax, %edi
+; X32-NEXT:    adcl %ebx, %ebp
+; X32-NEXT:    setb %bl
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %eax, %ecx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X32-NEXT:    movl %eax, %esi
+; X32-NEXT:    addl %ebp, %esi
+; X32-NEXT:    movzbl %bl, %eax
+; X32-NEXT:    adcl %eax, %edx
+; X32-NEXT:    addl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, %esi
 ; X32-NEXT:    adcl $0, %edx
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
 ; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %ebx
-; X32-NEXT:    addl %ecx, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edx, %ecx
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    setb %cl
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    addl %ebx, %eax
-; X32-NEXT:    movzbl %cl, %esi
-; X32-NEXT:    adcl %esi, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edx, %edi
+; X32-NEXT:    movl %eax, %ebp
+; X32-NEXT:    addl %ecx, %ebp
+; X32-NEXT:    adcl $0, %edi
+; X32-NEXT:    movl %ebx, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %esi
-; X32-NEXT:    adcl %edx, %ebx
+; X32-NEXT:    mull %ebx
+; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    addl %ebp, %eax
+; X32-NEXT:    movl %eax, %ebp
+; X32-NEXT:    adcl %edi, %ecx
+; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    mull %ebx
+; X32-NEXT:    addl %ecx, %eax
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
+; X32-NEXT:    adcl %ecx, %edx
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    addl %ebp, %ecx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X32-NEXT:    adcl %edi, %edx
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
-; X32-NEXT:    adcl %eax, %esi
-; X32-NEXT:    adcl $0, %ebx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT:    addl %esi, %ecx
+; X32-NEXT:    movl %ebp, %esi
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 1-byte Folded Reload
+; X32-NEXT:    adcl %ebx, %eax
+; X32-NEXT:    adcl $0, %edx
 ; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
 ; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
 ; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
 ; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
 ; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
 ; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %edi
-; X32-NEXT:    movl %eax, %ecx
-; X32-NEXT:    addl %esi, %ecx
-; X32-NEXT:    adcl $0, %edi
-; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    mull %ebp
-; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    addl %ecx, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edi, %esi
-; X32-NEXT:    setb %cl
-; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    mull %ebp
-; X32-NEXT:    addl %esi, %eax
-; X32-NEXT:    movzbl %cl, %ecx
-; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %ecx
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edx, %esi
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
 ; X32-NEXT:    movl %ebx, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X32-NEXT:    mull %esi
 ; X32-NEXT:    movl %edx, %ecx
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    movl %edi, %ebp
 ; X32-NEXT:    mull %esi
 ; X32-NEXT:    movl %edx, %esi
 ; X32-NEXT:    movl %eax, %edi
 ; X32-NEXT:    addl %ecx, %edi
 ; X32-NEXT:    adcl $0, %esi
 ; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %edx, %ebx
 ; X32-NEXT:    addl %edi, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %esi, %ecx
+; X32-NEXT:    adcl %esi, %ebx
 ; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
 ; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    movl %ebp, %esi
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    movl %eax, %edi
-; X32-NEXT:    addl %ecx, %edi
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    addl %ebx, %eax
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
 ; X32-NEXT:    adcl %eax, %edx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X32-NEXT:    movl %esi, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %edx, %ebx
+; X32-NEXT:    movl %eax, %ebp
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X32-NEXT:    adcl $0, %ebx
+; X32-NEXT:    movl %esi, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    addl %ebp, %eax
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl %ebx, %ecx
+; X32-NEXT:    setb %bl
+; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %eax, %esi
+; X32-NEXT:    addl %ecx, %esi
+; X32-NEXT:    movzbl %bl, %eax
+; X32-NEXT:    adcl %eax, %edx
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
@@ -2312,257 +2163,220 @@ define void @test_1024(i1024* %a, i1024* %b, i1024* %out) nounwind {
 ; X32-NEXT:    movl %ebx, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %ebp
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %esi, %eax
+; X32-NEXT:    movl %edi, %eax
 ; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    movl %eax, %ecx
-; X32-NEXT:    addl %ebp, %ecx
-; X32-NEXT:    adcl $0, %esi
-; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    mull %ebx
 ; X32-NEXT:    movl %edx, %ebp
-; X32-NEXT:    addl %ecx, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %esi, %ebp
-; X32-NEXT:    setb %cl
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    addl %ebp, %eax
-; X32-NEXT:    movzbl %cl, %ecx
-; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X32-NEXT:    movl %eax, %edi
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT:    adcl $0, %ebp
+; X32-NEXT:    movl %ebx, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %ebx
-; X32-NEXT:    adcl %edx, %ecx
-; X32-NEXT:    addl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl $0, %ebx
-; X32-NEXT:    adcl $0, %ecx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %edx, %ebx
+; X32-NEXT:    addl %edi, %eax
+; X32-NEXT:    movl %eax, %edi
+; X32-NEXT:    adcl %ebp, %ebx
 ; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %esi
+; X32-NEXT:    mull %ecx
 ; X32-NEXT:    movl %eax, %ebp
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X32-NEXT:    addl %ebx, %ebp
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; X32-NEXT:    adcl %eax, %edx
+; X32-NEXT:    addl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, %ebp
 ; X32-NEXT:    adcl $0, %edx
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %edi
-; X32-NEXT:    addl %ebp, %eax
+; X32-NEXT:    movl %esi, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %edx, %edi
+; X32-NEXT:    movl %eax, %ebx
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X32-NEXT:    adcl $0, %edi
+; X32-NEXT:    movl %esi, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %edx, %esi
+; X32-NEXT:    addl %ebx, %eax
+; X32-NEXT:    movl %eax, %ebx
+; X32-NEXT:    adcl %edi, %esi
 ; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    addl %edi, %eax
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    addl %esi, %eax
 ; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 1-byte Folded Reload
 ; X32-NEXT:    adcl %esi, %edx
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %edi
-; X32-NEXT:    adcl %edx, %ebp
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X32-NEXT:    addl %ebx, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    adcl %ecx, %esi
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
-; X32-NEXT:    adcl %eax, %edi
-; X32-NEXT:    adcl $0, %ebp
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    addl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    addl %ebp, %edi
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
+; X32-NEXT:    adcl %ecx, %eax
+; X32-NEXT:    adcl $0, %edx
+; X32-NEXT:    movl %edx, %esi
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    addl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    adcl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    adcl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    adcl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl %edi, %edx
 ; X32-NEXT:    adcl $0, %edx
+; X32-NEXT:    adcl $0, %ebx
+; X32-NEXT:    adcl $0, %eax
 ; X32-NEXT:    adcl $0, %esi
-; X32-NEXT:    adcl $0, %edi
-; X32-NEXT:    adcl $0, %ebp
 ; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
 ; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X32-NEXT:    movl %ebp, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %ebp
-; X32-NEXT:    movl %eax, %esi
-; X32-NEXT:    addl %ecx, %esi
-; X32-NEXT:    adcl $0, %ebp
-; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    mull %ebx
+; X32-NEXT:    mull %edi
 ; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    addl %esi, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %ebp, %ecx
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    movl %ebx, %edi
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    addl %ecx, %eax
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
-; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %ecx
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edx, %esi
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    movl %ebx, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    movl %eax, %ebp
-; X32-NEXT:    addl %ecx, %ebp
-; X32-NEXT:    adcl $0, %esi
-; X32-NEXT:    movl %ebx, %eax
+; X32-NEXT:    movl %esi, %eax
 ; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edx, %edi
+; X32-NEXT:    movl %eax, %ebx
+; X32-NEXT:    addl %ecx, %ebx
+; X32-NEXT:    adcl $0, %edi
+; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X32-NEXT:    mull %ebp
 ; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    addl %ebp, %eax
+; X32-NEXT:    addl %ebx, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %esi, %ecx
+; X32-NEXT:    adcl %edi, %ecx
 ; X32-NEXT:    setb %bl
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    mull %edi
-; X32-NEXT:    movl %eax, %ebp
-; X32-NEXT:    addl %ecx, %ebp
+; X32-NEXT:    mull %ebp
+; X32-NEXT:    addl %ecx, %eax
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movzbl %bl, %eax
 ; X32-NEXT:    adcl %eax, %edx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %ebx
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    movl %eax, %ecx
-; X32-NEXT:    addl %ebx, %ecx
-; X32-NEXT:    adcl $0, %esi
-; X32-NEXT:    movl %edi, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
 ; X32-NEXT:    mull %edi
-; X32-NEXT:    movl %edx, %ebx
-; X32-NEXT:    addl %ecx, %eax
+; X32-NEXT:    movl %edx, %ecx
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %esi, %ebx
-; X32-NEXT:    setb %cl
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X32-NEXT:    movl %ebp, %eax
 ; X32-NEXT:    mull %edi
-; X32-NEXT:    addl %ebx, %eax
-; X32-NEXT:    movzbl %cl, %ecx
-; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X32-NEXT:    movl %edx, %edi
+; X32-NEXT:    movl %eax, %ebx
+; X32-NEXT:    addl %ecx, %ebx
+; X32-NEXT:    adcl $0, %edi
+; X32-NEXT:    movl %esi, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %ebx
-; X32-NEXT:    adcl %edx, %ecx
-; X32-NEXT:    addl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl $0, %ebx
-; X32-NEXT:    adcl $0, %ecx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %edx, %esi
+; X32-NEXT:    addl %ebx, %eax
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl %edi, %esi
+; X32-NEXT:    setb %bl
 ; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    addl %esi, %ecx
+; X32-NEXT:    movzbl %bl, %eax
+; X32-NEXT:    adcl %eax, %edx
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NEXT:    movl %edi, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X32-NEXT:    mull %esi
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    movl %ebp, %eax
 ; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %eax, %edi
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT:    movl %edx, %ebx
+; X32-NEXT:    movl %eax, %ebp
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X32-NEXT:    adcl $0, %ebx
+; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, %edi
+; X32-NEXT:    addl %ebp, %eax
+; X32-NEXT:    movl %eax, %ebp
+; X32-NEXT:    adcl %ebx, %edi
+; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %eax, %ebx
+; X32-NEXT:    addl %edi, %ebx
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; X32-NEXT:    adcl %eax, %edx
+; X32-NEXT:    addl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, %ebx
 ; X32-NEXT:    adcl $0, %edx
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    movl %ecx, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %ebp
-; X32-NEXT:    addl %edi, %eax
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, %edi
+; X32-NEXT:    movl %eax, %ebp
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X32-NEXT:    adcl $0, %edi
+; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %edx, %esi
 ; X32-NEXT:    addl %ebp, %eax
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 1-byte Folded Reload
-; X32-NEXT:    adcl %esi, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl %edi, %esi
+; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    addl %esi, %eax
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
+; X32-NEXT:    adcl %ecx, %edx
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %edi
-; X32-NEXT:    adcl %edx, %ebp
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X32-NEXT:    addl %ebx, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    adcl %ecx, %esi
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
-; X32-NEXT:    adcl %eax, %edi
-; X32-NEXT:    adcl $0, %ebp
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    movl %edx, %ebx
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    addl %ebx, %ebp
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
+; X32-NEXT:    adcl %ecx, %eax
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, %edx
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
@@ -2572,715 +2386,610 @@ define void @test_1024(i1024* %a, i1024* %b, i1024* %out) nounwind {
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
 ; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
-; X32-NEXT:    adcl %eax, %ebx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl $0, %eax
+; X32-NEXT:    adcl %eax, %ebp
+; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, %ebx
+; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
 ; X32-NEXT:    adcl $0, %ebp
-; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X32-NEXT:    adcl $0, %ebx
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X32-NEXT:    addl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; X32-NEXT:    adcl %eax, %ecx
 ; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT:    adcl $0, %edx
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X32-NEXT:    adcl $0, %esi
 ; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT:    adcl $0, %edi
 ; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl (%esp), %eax # 4-byte Folded Reload
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
 ; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    mull %edi
-; X32-NEXT:    movl %edx, %ebp
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, %ebp
+; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, %ebx
+; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl 64(%eax), %edi
+; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X32-NEXT:    movl %esi, %eax
 ; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    mull %edi
 ; X32-NEXT:    movl %edx, %edi
 ; X32-NEXT:    movl %eax, %ebx
-; X32-NEXT:    addl %ebp, %ebx
+; X32-NEXT:    addl %ecx, %ebx
 ; X32-NEXT:    adcl $0, %edi
-; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl 68(%eax), %ecx
+; X32-NEXT:    movl %esi, %eax
+; X32-NEXT:    movl %ecx, %esi
+; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %ebp
+; X32-NEXT:    movl %edx, %ecx
 ; X32-NEXT:    addl %ebx, %eax
-; X32-NEXT:    movl %eax, (%esp) # 4-byte Spill
-; X32-NEXT:    adcl %edi, %ebp
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl %edi, %ecx
 ; X32-NEXT:    setb %bl
-; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    addl %ebp, %eax
-; X32-NEXT:    movzbl %bl, %ecx
-; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    mull %esi
+; X32-NEXT:    addl %ecx, %eax
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movzbl %bl, %eax
+; X32-NEXT:    adcl %eax, %edx
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NEXT:    movl %edi, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %ecx
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edx, %esi
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, %ebp
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    movl %edx, %edi
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    mull %ebx
+; X32-NEXT:    mull %esi
 ; X32-NEXT:    movl %edx, %ebx
-; X32-NEXT:    movl %eax, %ebp
-; X32-NEXT:    addl %edi, %ebp
+; X32-NEXT:    movl %eax, %esi
+; X32-NEXT:    addl %ebp, %esi
 ; X32-NEXT:    adcl $0, %ebx
-; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    movl %edi, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
 ; X32-NEXT:    mull %edi
-; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    addl %ebp, %eax
+; X32-NEXT:    movl %edx, %ebp
+; X32-NEXT:    addl %esi, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %ebx, %ecx
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl %esi, %ebp
-; X32-NEXT:    movl %esi, %eax
+; X32-NEXT:    adcl %ebx, %ebp
+; X32-NEXT:    setb %bl
+; X32-NEXT:    movl %ecx, %esi
+; X32-NEXT:    movl %ecx, %eax
 ; X32-NEXT:    mull %edi
-; X32-NEXT:    movl %eax, %ebx
-; X32-NEXT:    addl %ecx, %ebx
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    addl %ebp, %ecx
+; X32-NEXT:    movzbl %bl, %eax
 ; X32-NEXT:    adcl %eax, %edx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    adcl (%esp), %edx # 4-byte Folded Reload
-; X32-NEXT:    movl %edx, (%esp) # 4-byte Spill
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
 ; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl 72(%eax), %edi
+; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X32-NEXT:    movl %ebx, %eax
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    movl %esi, %eax
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edx, %ebp
+; X32-NEXT:    movl %eax, %edi
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT:    adcl $0, %ebp
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl 76(%eax), %esi
+; X32-NEXT:    movl %ebx, %eax
 ; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    movl %eax, %ebp
-; X32-NEXT:    addl %ecx, %ebp
-; X32-NEXT:    adcl $0, %esi
+; X32-NEXT:    movl %edx, %ebx
+; X32-NEXT:    addl %edi, %eax
+; X32-NEXT:    movl %eax, %edi
+; X32-NEXT:    adcl %ebp, %ebx
+; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %edi
-; X32-NEXT:    addl %ebp, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %esi, %edi
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %esi, %ebp
+; X32-NEXT:    movl %eax, %esi
+; X32-NEXT:    addl %ebx, %esi
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; X32-NEXT:    adcl %eax, %edx
+; X32-NEXT:    addl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, %esi
+; X32-NEXT:    adcl $0, %edx
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    addl %edi, %eax
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
-; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %ebp
-; X32-NEXT:    adcl %edx, %ecx
-; X32-NEXT:    addl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl (%esp), %eax # 4-byte Reload
-; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl $0, %ebp
-; X32-NEXT:    adcl $0, %ecx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    setb (%esp) # 1-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NEXT:    mull %edi
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %eax, %edi
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    adcl $0, %edx
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %ebx
-; X32-NEXT:    addl %edi, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edx, %edi
+; X32-NEXT:    movl %eax, %ebx
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X32-NEXT:    adcl $0, %edi
+; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    movl %ebp, %ecx
+; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    mull %ebp
+; X32-NEXT:    movl %edx, %ebp
+; X32-NEXT:    addl %ebx, %eax
+; X32-NEXT:    movl %eax, %ebx
+; X32-NEXT:    adcl %edi, %ebp
 ; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    addl %ebx, %eax
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 1-byte Folded Reload
-; X32-NEXT:    adcl %esi, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %edi
-; X32-NEXT:    adcl %edx, %ebx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X32-NEXT:    addl %ebp, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    adcl %ecx, %esi
-; X32-NEXT:    movzbl (%esp), %eax # 1-byte Folded Reload
-; X32-NEXT:    adcl %eax, %edi
-; X32-NEXT:    adcl $0, %ebx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    addl %ebp, %eax
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
+; X32-NEXT:    adcl %ecx, %edx
+; X32-NEXT:    addl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
 ; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
+; X32-NEXT:    adcl %ecx, %eax
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, %edx
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
 ; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, (%esp) # 4-byte Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edx, %ecx
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    mull %ecx
+; X32-NEXT:    mull %edi
 ; X32-NEXT:    movl %edx, %edi
 ; X32-NEXT:    movl %eax, %ebx
-; X32-NEXT:    addl (%esp), %ebx # 4-byte Folded Reload
+; X32-NEXT:    addl %ecx, %ebx
 ; X32-NEXT:    adcl $0, %edi
 ; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %ebp
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X32-NEXT:    mull %ebp
+; X32-NEXT:    movl %edx, %ecx
 ; X32-NEXT:    addl %ebx, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edi, %ebp
+; X32-NEXT:    adcl %edi, %ecx
 ; X32-NEXT:    setb %bl
 ; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    addl %ebp, %eax
-; X32-NEXT:    movzbl %bl, %edi
-; X32-NEXT:    adcl %edi, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %ecx
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edx, %esi
-; X32-NEXT:    movl %esi, (%esp) # 4-byte Spill
+; X32-NEXT:    mull %ebp
+; X32-NEXT:    addl %ecx, %eax
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movzbl %bl, %eax
+; X32-NEXT:    adcl %eax, %edx
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %edi
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X32-NEXT:    mull %ebx
+; X32-NEXT:    movl %edx, %ecx
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    mull %ebx
 ; X32-NEXT:    movl %edx, %ebx
 ; X32-NEXT:    movl %eax, %ebp
-; X32-NEXT:    addl %edi, %ebp
+; X32-NEXT:    addl %ecx, %ebp
 ; X32-NEXT:    adcl $0, %ebx
 ; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %edi
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %edx, %esi
 ; X32-NEXT:    addl %ebp, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %ebx, %edi
-; X32-NEXT:    setb %cl
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %eax, %ebp
-; X32-NEXT:    addl %edi, %ebp
-; X32-NEXT:    movzbl %cl, %eax
+; X32-NEXT:    adcl %ebx, %esi
+; X32-NEXT:    setb %bl
+; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    movl %edi, %ebp
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    addl %esi, %ecx
+; X32-NEXT:    movzbl %bl, %eax
 ; X32-NEXT:    adcl %eax, %edx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl $0, (%esp) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
 ; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    mull %esi
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NEXT:    mull %edi
 ; X32-NEXT:    movl %edx, %esi
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edx, %ebp
 ; X32-NEXT:    movl %eax, %edi
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    adcl $0, %esi
-; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    mull %ecx
+; X32-NEXT:    addl %esi, %edi
+; X32-NEXT:    adcl $0, %ebp
+; X32-NEXT:    movl %ebx, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X32-NEXT:    mull %esi
 ; X32-NEXT:    movl %edx, %ebx
 ; X32-NEXT:    addl %edi, %eax
 ; X32-NEXT:    movl %eax, %edi
-; X32-NEXT:    adcl %esi, %ebx
+; X32-NEXT:    adcl %ebp, %ebx
 ; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    addl %ebx, %eax
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
-; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %esi
-; X32-NEXT:    adcl %edx, %ecx
-; X32-NEXT:    addl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %eax, %esi
+; X32-NEXT:    addl %ebx, %esi
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; X32-NEXT:    adcl %eax, %edx
+; X32-NEXT:    addl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
 ; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    adcl $0, %esi
-; X32-NEXT:    adcl $0, %ecx
+; X32-NEXT:    adcl $0, %edx
 ; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    adcl (%esp), %ecx # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
 ; X32-NEXT:    movl %ebp, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
 ; X32-NEXT:    mull %edi
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %eax, (%esp) # 4-byte Spill
+; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edx, %edi
 ; X32-NEXT:    movl %eax, %ebx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    adcl $0, %edx
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    addl %ecx, %ebx
+; X32-NEXT:    adcl $0, %edi
 ; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    mull %edi
-; X32-NEXT:    movl %edx, %ebp
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X32-NEXT:    mull %ebp
+; X32-NEXT:    movl %edx, %ecx
 ; X32-NEXT:    addl %ebx, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X32-NEXT:    adcl %edi, %ecx
 ; X32-NEXT:    setb %bl
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %edi
-; X32-NEXT:    addl %ebp, %eax
-; X32-NEXT:    movzbl %bl, %edi
-; X32-NEXT:    adcl %edi, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %edi
-; X32-NEXT:    adcl %edx, %ebx
-; X32-NEXT:    movl (%esp), %edx # 4-byte Reload
-; X32-NEXT:    addl %esi, %edx
+; X32-NEXT:    mull %ebp
+; X32-NEXT:    addl %ecx, %eax
+; X32-NEXT:    movzbl %bl, %ecx
+; X32-NEXT:    adcl %ecx, %edx
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    addl %esi, %ecx
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    adcl %ecx, %esi
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
-; X32-NEXT:    adcl %eax, %edi
-; X32-NEXT:    adcl $0, %ebx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    movl %edx, (%esp) # 4-byte Spill
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 1-byte Folded Reload
+; X32-NEXT:    adcl %edi, %eax
+; X32-NEXT:    adcl $0, %edx
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
 ; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
 ; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
 ; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %edi
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %ebx
-; X32-NEXT:    movl %eax, %ecx
-; X32-NEXT:    addl %edi, %ecx
-; X32-NEXT:    adcl $0, %ebx
-; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %edi
-; X32-NEXT:    addl %ecx, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %ebx, %edi
-; X32-NEXT:    setb %cl
-; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    mull %esi
-; X32-NEXT:    addl %edi, %eax
-; X32-NEXT:    movzbl %cl, %ecx
-; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %ecx
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edx, %esi
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl 80(%eax), %esi
 ; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
 ; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    mull %edi
+; X32-NEXT:    mull %esi
 ; X32-NEXT:    movl %edx, %ecx
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    mull %edi
-; X32-NEXT:    movl %edx, %edi
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, %esi
 ; X32-NEXT:    movl %eax, %ebx
 ; X32-NEXT:    addl %ecx, %ebx
-; X32-NEXT:    adcl $0, %edi
+; X32-NEXT:    adcl $0, %esi
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl 84(%eax), %ecx
 ; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    mull %ebp
+; X32-NEXT:    movl %ecx, %ebp
+; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    mull %ecx
 ; X32-NEXT:    movl %edx, %ecx
 ; X32-NEXT:    addl %ebx, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edi, %ecx
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    movl %esi, %edi
+; X32-NEXT:    adcl %esi, %ecx
+; X32-NEXT:    setb %bl
+; X32-NEXT:    movl %edi, %eax
 ; X32-NEXT:    mull %ebp
-; X32-NEXT:    movl %eax, %ebx
-; X32-NEXT:    addl %ecx, %ebx
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; X32-NEXT:    addl %ecx, %eax
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movzbl %bl, %eax
 ; X32-NEXT:    adcl %eax, %edx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NEXT:    movl %edi, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %ebp
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, %ecx
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X32-NEXT:    movl %ebx, %eax
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, %esi
+; X32-NEXT:    movl %eax, %ebp
+; X32-NEXT:    addl %ecx, %ebp
+; X32-NEXT:    adcl $0, %esi
 ; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X32-NEXT:    mull %ecx
 ; X32-NEXT:    movl %edx, %edi
-; X32-NEXT:    movl %eax, %ecx
-; X32-NEXT:    addl %ebp, %ecx
-; X32-NEXT:    adcl $0, %edi
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl 60(%eax), %ebp
-; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    mull %ebp
-; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    addl %ecx, %eax
+; X32-NEXT:    addl %ebp, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edi, %esi
-; X32-NEXT:    setb %cl
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %ebp
-; X32-NEXT:    addl %esi, %eax
-; X32-NEXT:    movzbl %cl, %ecx
-; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %ebp
-; X32-NEXT:    adcl %edx, %ecx
-; X32-NEXT:    addl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl $0, %ebp
-; X32-NEXT:    adcl $0, %ecx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X32-NEXT:    adcl %esi, %edi
 ; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
 ; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    mull %esi
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %eax, %esi
+; X32-NEXT:    addl %edi, %esi
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; X32-NEXT:    adcl %eax, %edx
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl 88(%eax), %edi
+; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    mull %edi
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl %ebx, %eax
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edx, %ebp
+; X32-NEXT:    movl %eax, %ebx
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X32-NEXT:    adcl $0, %ebp
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl 92(%eax), %edi
+; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    addl %ebx, %eax
+; X32-NEXT:    movl %eax, %ebx
+; X32-NEXT:    adcl %ebp, %ecx
+; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %eax, %edi
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %eax, %ebp
+; X32-NEXT:    addl %ecx, %ebp
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; X32-NEXT:    adcl %eax, %edx
+; X32-NEXT:    addl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, %ebp
 ; X32-NEXT:    adcl $0, %edx
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %ebx, %eax
+; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    movl %ecx, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %ebx
-; X32-NEXT:    addl %edi, %eax
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, %esi
+; X32-NEXT:    movl %eax, %ebx
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X32-NEXT:    adcl $0, %esi
+; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    movl %edi, %ecx
+; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edx, %edi
 ; X32-NEXT:    addl %ebx, %eax
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 1-byte Folded Reload
-; X32-NEXT:    adcl %esi, %edx
+; X32-NEXT:    movl %eax, %ebx
+; X32-NEXT:    adcl %esi, %edi
+; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    addl %edi, %eax
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
+; X32-NEXT:    adcl %ecx, %edx
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %edi
-; X32-NEXT:    adcl %edx, %ebx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X32-NEXT:    addl %ebp, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    adcl %ecx, %esi
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
-; X32-NEXT:    adcl %eax, %edi
-; X32-NEXT:    adcl $0, %ebx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT:    addl %ebp, %edi
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    movl (%esp), %eax # 4-byte Reload
-; X32-NEXT:    addl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
+; X32-NEXT:    adcl %ecx, %eax
+; X32-NEXT:    adcl $0, %edx
+; X32-NEXT:    movl %edx, %esi
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    addl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    adcl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    adcl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    adcl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl %edi, %edx
 ; X32-NEXT:    adcl $0, %edx
-; X32-NEXT:    adcl $0, %esi
-; X32-NEXT:    adcl $0, %edi
 ; X32-NEXT:    adcl $0, %ebx
+; X32-NEXT:    adcl $0, %eax
+; X32-NEXT:    adcl $0, %esi
 ; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
 ; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
 ; X32-NEXT:    movl %ebx, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X32-NEXT:    mull %esi
 ; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    movl %eax, (%esp) # 4-byte Spill
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
 ; X32-NEXT:    movl %ebp, %eax
 ; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %edi
-; X32-NEXT:    movl %eax, %esi
-; X32-NEXT:    addl %ecx, %esi
-; X32-NEXT:    adcl $0, %edi
-; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X32-NEXT:    movl %edx, %esi
+; X32-NEXT:    movl %eax, %edi
+; X32-NEXT:    addl %ecx, %edi
+; X32-NEXT:    adcl $0, %esi
+; X32-NEXT:    movl %ebx, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
 ; X32-NEXT:    mull %ebx
 ; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    addl %esi, %eax
+; X32-NEXT:    addl %edi, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edi, %ecx
+; X32-NEXT:    adcl %esi, %ecx
 ; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
 ; X32-NEXT:    movl %ebp, %eax
 ; X32-NEXT:    mull %ebx
 ; X32-NEXT:    addl %ecx, %eax
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
-; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %ecx
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edx, %esi
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; X32-NEXT:    adcl %eax, %edx
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
 ; X32-NEXT:    movl %ebx, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X32-NEXT:    mull %esi
 ; X32-NEXT:    movl %edx, %ecx
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NEXT:    movl %edi, %eax
 ; X32-NEXT:    mull %esi
 ; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    movl %eax, %edi
-; X32-NEXT:    addl %ecx, %edi
+; X32-NEXT:    movl %eax, %ebp
+; X32-NEXT:    addl %ecx, %ebp
 ; X32-NEXT:    adcl $0, %esi
 ; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    addl %edi, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %esi, %ecx
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    movl %eax, %edi
-; X32-NEXT:    addl %ecx, %edi
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
-; X32-NEXT:    adcl %eax, %edx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    addl (%esp), %edi # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    movl %edx, (%esp) # 4-byte Spill
-; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    movl %ebp, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X32-NEXT:    mull %ecx
 ; X32-NEXT:    movl %edx, %ebx
+; X32-NEXT:    addl %ebp, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    adcl %esi, %ebx
+; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X32-NEXT:    movl %edi, %eax
 ; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %esi
 ; X32-NEXT:    movl %eax, %ecx
 ; X32-NEXT:    addl %ebx, %ecx
-; X32-NEXT:    adcl $0, %esi
-; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; X32-NEXT:    adcl %eax, %edx
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X32-NEXT:    movl %esi, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
 ; X32-NEXT:    mull %ebx
-; X32-NEXT:    movl %edx, %ebp
-; X32-NEXT:    addl %ecx, %eax
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %esi, %ebp
-; X32-NEXT:    setb %cl
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    movl %edi, %eax
 ; X32-NEXT:    mull %ebx
-; X32-NEXT:    addl %ebp, %eax
-; X32-NEXT:    movzbl %cl, %ecx
-; X32-NEXT:    adcl %ecx, %edx
+; X32-NEXT:    movl %edx, %ebp
+; X32-NEXT:    movl %eax, %edi
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT:    adcl $0, %ebp
+; X32-NEXT:    movl %esi, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %ebx
-; X32-NEXT:    adcl %edx, %ebp
-; X32-NEXT:    addl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl (%esp), %eax # 4-byte Reload
-; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    mull %ebx
+; X32-NEXT:    movl %edx, %esi
+; X32-NEXT:    addl %edi, %eax
+; X32-NEXT:    movl %eax, %edi
+; X32-NEXT:    adcl %ebp, %esi
+; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    mull %ebx
+; X32-NEXT:    movl %ebx, %ebp
+; X32-NEXT:    movl %eax, %ebx
+; X32-NEXT:    addl %esi, %ebx
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; X32-NEXT:    adcl %eax, %edx
+; X32-NEXT:    addl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    adcl $0, %ebx
-; X32-NEXT:    adcl $0, %ebp
+; X32-NEXT:    adcl $0, %edx
 ; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X32-NEXT:    movl %ecx, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, (%esp) # 4-byte Spill
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X32-NEXT:    mull %esi
 ; X32-NEXT:    movl %edx, %esi
 ; X32-NEXT:    movl %eax, %edi
-; X32-NEXT:    addl (%esp), %edi # 4-byte Folded Reload
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
 ; X32-NEXT:    adcl $0, %esi
 ; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %ebp, %ecx
+; X32-NEXT:    mull %ebp
+; X32-NEXT:    movl %edx, %ebp
 ; X32-NEXT:    addl %edi, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %esi, %edx
-; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    setb (%esp) # 1-byte Folded Spill
+; X32-NEXT:    movl %eax, %edi
+; X32-NEXT:    adcl %esi, %ebp
+; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X32-NEXT:    mull %ecx
-; X32-NEXT:    addl %esi, %eax
-; X32-NEXT:    movzbl (%esp), %ecx # 1-byte Folded Reload
+; X32-NEXT:    addl %ebp, %eax
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
 ; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %esi
-; X32-NEXT:    adcl %edx, %edi
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X32-NEXT:    addl %ebx, %ecx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X32-NEXT:    adcl %ebp, %edx
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
-; X32-NEXT:    adcl %eax, %esi
-; X32-NEXT:    adcl $0, %edi
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl %ecx, %ebx
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    movl %edx, %ebp
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    movl %esi, (%esp) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
-; X32-NEXT:    adcl %eax, %ebx
-; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl $0, %ebp
-; X32-NEXT:    adcl $0, (%esp) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    adcl $0, %ebx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    movl (%esp), %eax # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT:    movl %eax, (%esp) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 1-byte Folded Reload
+; X32-NEXT:    adcl %esi, %eax
+; X32-NEXT:    movl %eax, %esi
+; X32-NEXT:    adcl $0, %edx
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X32-NEXT:    addl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
@@ -3289,399 +2998,272 @@ define void @test_1024(i1024* %a, i1024* %b, i1024* %out) nounwind {
 ; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl $0, %ecx
-; X32-NEXT:    adcl $0, %edx
-; X32-NEXT:    adcl $0, %esi
-; X32-NEXT:    adcl $0, %edi
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl $0, %eax
-; X32-NEXT:    adcl $0, %ebp
-; X32-NEXT:    movl (%esp), %ebx # 4-byte Reload
-; X32-NEXT:    adcl $0, %ebx
-; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; X32-NEXT:    adcl %eax, %ecx
 ; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT:    adcl $0, %edi
 ; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X32-NEXT:    adcl $0, %esi
+; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, %edx
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, %ecx
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    movl %ebx, (%esp) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, %esi
+; X32-NEXT:    movl %eax, %ebx
+; X32-NEXT:    addl %ecx, %ebx
+; X32-NEXT:    adcl $0, %esi
+; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    addl %ebx, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    adcl %esi, %ecx
+; X32-NEXT:    setb %bl
+; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    mull %edi
+; X32-NEXT:    addl %ecx, %eax
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movzbl %bl, %eax
+; X32-NEXT:    adcl %eax, %edx
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X32-NEXT:    movl %ebp, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
 ; X32-NEXT:    mull %edi
-; X32-NEXT:    movl %edx, %ebp
+; X32-NEXT:    movl %edx, %ecx
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X32-NEXT:    movl %esi, %eax
 ; X32-NEXT:    mull %edi
-; X32-NEXT:    movl %edx, %edi
-; X32-NEXT:    movl %eax, %ebx
-; X32-NEXT:    addl %ebp, %ebx
-; X32-NEXT:    adcl $0, %edi
-; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    movl %edx, %ebx
+; X32-NEXT:    movl %eax, %edi
+; X32-NEXT:    addl %ecx, %edi
+; X32-NEXT:    adcl $0, %ebx
+; X32-NEXT:    movl %ebp, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X32-NEXT:    mull %ecx
 ; X32-NEXT:    movl %edx, %ebp
-; X32-NEXT:    addl %ebx, %eax
+; X32-NEXT:    addl %edi, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edi, %ebp
+; X32-NEXT:    adcl %ebx, %ebp
 ; X32-NEXT:    setb %bl
 ; X32-NEXT:    movl %esi, %eax
+; X32-NEXT:    movl %esi, %edi
 ; X32-NEXT:    mull %ecx
-; X32-NEXT:    addl %ebp, %eax
-; X32-NEXT:    movzbl %bl, %ecx
-; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    addl %ebp, %ecx
+; X32-NEXT:    movzbl %bl, %eax
+; X32-NEXT:    adcl %eax, %edx
 ; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X32-NEXT:    movl %ebx, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %ecx
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edx, %esi
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    mull %ecx
+; X32-NEXT:    mull %esi
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %ebx
-; X32-NEXT:    movl %eax, %ebp
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    adcl $0, %ebx
 ; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, %ebp
+; X32-NEXT:    movl %eax, %esi
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X32-NEXT:    adcl $0, %ebp
+; X32-NEXT:    movl %ebx, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
 ; X32-NEXT:    mull %edi
-; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    addl %ebp, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %ebx, %ecx
-; X32-NEXT:    setb %bl
-; X32-NEXT:    movl %esi, %eax
+; X32-NEXT:    movl %edx, %ebx
+; X32-NEXT:    addl %esi, %eax
+; X32-NEXT:    movl %eax, %esi
+; X32-NEXT:    adcl %ebp, %ebx
+; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X32-NEXT:    mull %edi
-; X32-NEXT:    movl %eax, %ebp
-; X32-NEXT:    addl %ecx, %ebp
-; X32-NEXT:    movzbl %bl, %eax
-; X32-NEXT:    adcl %eax, %edx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    movl %edx, %ebp
+; X32-NEXT:    movl %eax, %edi
+; X32-NEXT:    addl %ebx, %edi
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; X32-NEXT:    adcl %eax, %ebp
+; X32-NEXT:    addl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, %edi
+; X32-NEXT:    adcl $0, %ebp
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    movl %ecx, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X32-NEXT:    mull %esi
 ; X32-NEXT:    movl %edx, %esi
 ; X32-NEXT:    movl %eax, %ebx
-; X32-NEXT:    addl %ecx, %ebx
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
 ; X32-NEXT:    adcl $0, %esi
-; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %edi
+; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    mull {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; X32-NEXT:    movl %edx, %ecx
 ; X32-NEXT:    addl %ebx, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %esi, %edi
-; X32-NEXT:    setb %bl
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    addl %edi, %eax
-; X32-NEXT:    movzbl %bl, %ecx
-; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %ebx
-; X32-NEXT:    adcl %edx, %ecx
-; X32-NEXT:    addl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl $0, %ebx
-; X32-NEXT:    adcl $0, %ecx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X32-NEXT:    movl %eax, %ebx
+; X32-NEXT:    adcl %esi, %ecx
 ; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    addl %ecx, %eax
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
+; X32-NEXT:    adcl %ecx, %edx
+; X32-NEXT:    addl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl %ebp, %ebx
+; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
+; X32-NEXT:    adcl %ecx, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %eax, %edi
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
 ; X32-NEXT:    adcl $0, %edx
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    movl %esi, %ecx
+; X32-NEXT:    imull %eax, %ecx
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %ebp
-; X32-NEXT:    addl %edi, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X32-NEXT:    addl %ecx, %edx
+; X32-NEXT:    imull {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X32-NEXT:    addl %edx, %esi
+; X32-NEXT:    movl %esi, %ebp
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    addl %ebp, %eax
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 1-byte Folded Reload
-; X32-NEXT:    adcl %esi, %edx
+; X32-NEXT:    movl %eax, %esi
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X32-NEXT:    imull %ebx, %esi
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %edi
-; X32-NEXT:    adcl %edx, %ebp
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X32-NEXT:    addl %ebx, %edx
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    addl %esi, %edx
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    adcl %ecx, %esi
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
-; X32-NEXT:    adcl %eax, %edi
-; X32-NEXT:    adcl $0, %ebp
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    imull %edi, %esi
+; X32-NEXT:    addl %edx, %esi
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X32-NEXT:    movl %ecx, %edi
+; X32-NEXT:    adcl %ebp, %esi
 ; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    movl %ebp, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl %edx, %ebp
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    movl %esi, %eax
+; X32-NEXT:    movl %ebx, %eax
 ; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %edi
+; X32-NEXT:    movl %edx, %esi
 ; X32-NEXT:    movl %eax, %ebx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    adcl $0, %edi
-; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    addl %ebp, %ebx
+; X32-NEXT:    adcl $0, %esi
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X32-NEXT:    mull %ecx
 ; X32-NEXT:    movl %edx, %ebp
 ; X32-NEXT:    addl %ebx, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edi, %ebp
+; X32-NEXT:    adcl %esi, %ebp
 ; X32-NEXT:    setb %bl
-; X32-NEXT:    movl %esi, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X32-NEXT:    mull %ecx
 ; X32-NEXT:    addl %ebp, %eax
 ; X32-NEXT:    movzbl %bl, %ecx
 ; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %ecx
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edx, %esi
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    movl %edx, %edi
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    movl %edx, %ebx
-; X32-NEXT:    movl %eax, %ebp
-; X32-NEXT:    addl %edi, %ebp
-; X32-NEXT:    adcl $0, %ebx
-; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    mull %edi
-; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    addl %ebp, %eax
+; X32-NEXT:    addl %edi, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %ebx, %ecx
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl %esi, %ebp
-; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    mull %edi
-; X32-NEXT:    movl %eax, %ebx
-; X32-NEXT:    addl %ecx, %ebx
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
-; X32-NEXT:    adcl %eax, %edx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X32-NEXT:    imull %ebp, %edi
 ; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    movl %eax, %ebp
-; X32-NEXT:    addl %ecx, %ebp
-; X32-NEXT:    adcl $0, %esi
-; X32-NEXT:    movl %edi, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %edi
-; X32-NEXT:    addl %ebp, %eax
-; X32-NEXT:    movl %eax, %ebp
-; X32-NEXT:    adcl %esi, %edi
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    addl %edi, %edx
+; X32-NEXT:    imull {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X32-NEXT:    addl %edx, %ecx
+; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    addl %edi, %eax
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
-; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %ecx
-; X32-NEXT:    adcl %edx, %esi
-; X32-NEXT:    addl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl $0, %ecx
-; X32-NEXT:    adcl $0, %esi
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    mull %edi
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %edi
-; X32-NEXT:    movl %eax, %ebx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    adcl $0, %edx
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X32-NEXT:    imull %ebx, %ecx
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
 ; X32-NEXT:    mull %edi
-; X32-NEXT:    movl %edx, %ebp
-; X32-NEXT:    addl %ebx, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    setb %bl
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %edi
-; X32-NEXT:    addl %ebp, %eax
-; X32-NEXT:    movzbl %bl, %edi
-; X32-NEXT:    adcl %edi, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %edi
-; X32-NEXT:    adcl %edx, %ebx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X32-NEXT:    movl %eax, %esi
 ; X32-NEXT:    addl %ecx, %edx
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl %esi, %ecx
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
-; X32-NEXT:    adcl %eax, %edi
-; X32-NEXT:    adcl $0, %ebx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    imull %edi, %ecx
+; X32-NEXT:    addl %edx, %ecx
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
 ; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %edi
+; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    mull %ebp
+; X32-NEXT:    movl %edx, %esi
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
 ; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    mull %ecx
+; X32-NEXT:    mull %ebp
 ; X32-NEXT:    movl %edx, %ebp
 ; X32-NEXT:    movl %eax, %ecx
-; X32-NEXT:    addl %edi, %ecx
+; X32-NEXT:    addl %esi, %ecx
 ; X32-NEXT:    adcl $0, %ebp
-; X32-NEXT:    movl %esi, %eax
+; X32-NEXT:    movl %edi, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %edi
-; X32-NEXT:    addl %ecx, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %ebp, %edi
+; X32-NEXT:    movl %edx, %ebx
+; X32-NEXT:    movl %eax, %edi
+; X32-NEXT:    addl %ecx, %edi
+; X32-NEXT:    adcl %ebp, %ebx
 ; X32-NEXT:    setb %cl
-; X32-NEXT:    movl %ebx, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X32-NEXT:    mull %esi
-; X32-NEXT:    addl %edi, %eax
+; X32-NEXT:    addl %ebx, %eax
 ; X32-NEXT:    movzbl %cl, %ecx
 ; X32-NEXT:    adcl %ecx, %edx
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %ecx
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
 ; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edx, %esi
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT:    movl 104(%esi), %ebx
 ; X32-NEXT:    movl %ebx, %eax
+; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
 ; X32-NEXT:    mull %edi
-; X32-NEXT:    movl %edx, %ecx
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    movl 108(%esi), %esi
 ; X32-NEXT:    movl %esi, %eax
+; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    mull %edi
 ; X32-NEXT:    movl %edx, %edi
 ; X32-NEXT:    movl %eax, %ebp
@@ -3697,798 +3279,825 @@ define void @test_1024(i1024* %a, i1024* %b, i1024* %out) nounwind {
 ; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
 ; X32-NEXT:    movl %esi, %eax
 ; X32-NEXT:    mull %ebx
-; X32-NEXT:    movl %eax, %ebp
-; X32-NEXT:    addl %ecx, %ebp
+; X32-NEXT:    addl %ecx, %eax
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
 ; X32-NEXT:    adcl %eax, %edx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    movl %esi, %eax
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT:    movl 96(%esi), %eax
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    movl 100(%esi), %eax
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edx, %ebp
+; X32-NEXT:    movl %eax, %ebx
+; X32-NEXT:    addl %ecx, %ebx
+; X32-NEXT:    adcl $0, %ebp
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NEXT:    movl %edi, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %ebx
+; X32-NEXT:    movl %edx, %esi
+; X32-NEXT:    addl %ebx, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    adcl %ebp, %esi
+; X32-NEXT:    setb %bl
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X32-NEXT:    movl %ebp, %eax
 ; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %edi
 ; X32-NEXT:    movl %eax, %ecx
-; X32-NEXT:    addl %ebx, %ecx
-; X32-NEXT:    adcl $0, %edi
-; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    addl %ecx, %eax
+; X32-NEXT:    addl %esi, %ecx
+; X32-NEXT:    movzbl %bl, %eax
+; X32-NEXT:    adcl %eax, %edx
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    movl %edi, %esi
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edi, %esi
-; X32-NEXT:    setb %cl
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    addl %esi, %eax
-; X32-NEXT:    movzbl %cl, %ecx
-; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %ebx
-; X32-NEXT:    adcl %edx, %ecx
-; X32-NEXT:    addl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edx, %ebx
+; X32-NEXT:    movl %eax, %edi
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
 ; X32-NEXT:    adcl $0, %ebx
-; X32-NEXT:    adcl $0, %ecx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    movl %esi, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, %ebp
+; X32-NEXT:    addl %edi, %eax
+; X32-NEXT:    movl %eax, %edi
+; X32-NEXT:    adcl %ebx, %ebp
+; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %esi, %ebx
+; X32-NEXT:    movl %eax, %esi
+; X32-NEXT:    addl %ebp, %esi
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; X32-NEXT:    adcl %eax, %edx
+; X32-NEXT:    addl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, %esi
+; X32-NEXT:    adcl $0, %edx
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NEXT:    mull %edi
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %esi
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edx, %edi
 ; X32-NEXT:    movl %eax, %ebp
 ; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X32-NEXT:    adcl $0, %edi
+; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    movl %ebx, %ecx
+; X32-NEXT:    mull %ebx
+; X32-NEXT:    movl %edx, %ebx
+; X32-NEXT:    addl %ebp, %eax
+; X32-NEXT:    movl %eax, %ebp
+; X32-NEXT:    adcl %edi, %ebx
+; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    addl %ebx, %eax
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
+; X32-NEXT:    adcl %ecx, %edx
+; X32-NEXT:    addl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
+; X32-NEXT:    adcl %ecx, %eax
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    adcl $0, %edx
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT:    movl 112(%ecx), %edi
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X32-NEXT:    imull %edi, %ebx
 ; X32-NEXT:    movl %edi, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %edi
-; X32-NEXT:    addl %ebp, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    addl %edi, %eax
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 1-byte Folded Reload
-; X32-NEXT:    adcl %esi, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %edi
-; X32-NEXT:    adcl %edx, %ebp
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
 ; X32-NEXT:    addl %ebx, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    adcl %ecx, %esi
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
-; X32-NEXT:    adcl %eax, %edi
-; X32-NEXT:    adcl $0, %ebp
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X32-NEXT:    movl 116(%ecx), %eax
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    imull %eax, %esi
+; X32-NEXT:    addl %edx, %esi
+; X32-NEXT:    movl %esi, %ebx
+; X32-NEXT:    movl 120(%ecx), %eax
+; X32-NEXT:    movl %eax, %esi
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    imull %ecx, %esi
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X32-NEXT:    mull %ebp
+; X32-NEXT:    addl %esi, %edx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT:    movl 124(%esi), %esi
+; X32-NEXT:    imull %ebp, %esi
+; X32-NEXT:    addl %edx, %esi
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl %ebx, %esi
+; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edx, %ebp
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    movl %eax, %ebx
+; X32-NEXT:    addl %ebp, %ebx
+; X32-NEXT:    adcl $0, %ecx
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    addl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edx, %ebp
+; X32-NEXT:    addl %ebx, %eax
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl %ecx, %ebp
+; X32-NEXT:    setb %bl
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    mull %edi
+; X32-NEXT:    addl %ebp, %eax
+; X32-NEXT:    movzbl %bl, %ecx
+; X32-NEXT:    adcl %ecx, %edx
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl %esi, %edx
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    imull %eax, %edi
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X32-NEXT:    mull %ebp
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    addl %edi, %edx
+; X32-NEXT:    imull {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X32-NEXT:    addl %edx, %ebp
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl $0, %edx
-; X32-NEXT:    adcl $0, %esi
+; X32-NEXT:    movl %eax, %edi
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X32-NEXT:    imull %ebx, %edi
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %eax, %esi
+; X32-NEXT:    addl %edi, %edx
+; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    imull %eax, %ecx
+; X32-NEXT:    addl %edx, %ecx
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl %ebp, %ecx
+; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl %eax, %esi
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %edx, %ebp
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl %ebx, %eax
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %edx, %edi
+; X32-NEXT:    movl %eax, %ebx
+; X32-NEXT:    addl %ebp, %ebx
 ; X32-NEXT:    adcl $0, %edi
-; X32-NEXT:    adcl $0, %ebp
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT:    movl %esi, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, %ebp
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    addl %ebx, %ecx
+; X32-NEXT:    adcl %edi, %ebp
+; X32-NEXT:    setb %bl
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    mull %esi
+; X32-NEXT:    addl %ebp, %eax
+; X32-NEXT:    movzbl %bl, %edi
+; X32-NEXT:    adcl %edi, %edx
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X32-NEXT:    movl %esi, %edi
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X32-NEXT:    movl %ecx, %ebx
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT:    movl %edx, %ebp
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
 ; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
 ; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
 ; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT:    movl 88(%esi), %ebp
+; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %ebp
-; X32-NEXT:    movl %eax, %esi
-; X32-NEXT:    addl %ecx, %esi
-; X32-NEXT:    adcl $0, %ebp
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    movl 92(%esi), %ebx
 ; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    mull %ebx
+; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edx, %esi
+; X32-NEXT:    movl %eax, %edi
+; X32-NEXT:    addl %ecx, %edi
+; X32-NEXT:    adcl $0, %esi
+; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    movl (%esp), %ebp # 4-byte Reload
+; X32-NEXT:    mull %ebp
 ; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    addl %esi, %eax
+; X32-NEXT:    addl %edi, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %ebp, %ecx
+; X32-NEXT:    adcl %esi, %ecx
 ; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    addl %ecx, %eax
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
-; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %esi
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edx, %ecx
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
 ; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    mull %ebp
+; X32-NEXT:    movl %ebp, %ebx
+; X32-NEXT:    addl %ecx, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; X32-NEXT:    adcl %eax, %edx
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT:    movl 80(%esi), %edi
 ; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X32-NEXT:    mull %ebp
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    movl 84(%esi), %eax
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    mull %ebp
 ; X32-NEXT:    movl %edx, %esi
 ; X32-NEXT:    movl %eax, %ebp
 ; X32-NEXT:    addl %ecx, %ebp
 ; X32-NEXT:    adcl $0, %esi
-; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    movl %ebx, %ecx
 ; X32-NEXT:    mull %ebx
-; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    movl %edx, %ebx
 ; X32-NEXT:    addl %ebp, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %esi, %ecx
+; X32-NEXT:    adcl %esi, %ebx
 ; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
 ; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    movl %eax, %ebp
-; X32-NEXT:    addl %ecx, %ebp
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    addl %ebx, %ecx
 ; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
 ; X32-NEXT:    adcl %eax, %edx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
 ; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
 ; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X32-NEXT:    mull %esi
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    movl %eax, %ecx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    adcl $0, %esi
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, %ebp
+; X32-NEXT:    movl %eax, %edi
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT:    adcl $0, %ebp
 ; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    mull %edi
-; X32-NEXT:    movl %edx, %ebx
-; X32-NEXT:    addl %ecx, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %esi, %ebx
-; X32-NEXT:    setb %cl
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X32-NEXT:    mull %ebx
+; X32-NEXT:    movl %edx, %esi
+; X32-NEXT:    addl %edi, %eax
+; X32-NEXT:    movl %eax, %edi
+; X32-NEXT:    adcl %ebp, %esi
+; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %edi
-; X32-NEXT:    addl %ebx, %eax
-; X32-NEXT:    movzbl %cl, %ecx
-; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %esi
-; X32-NEXT:    adcl %edx, %ecx
-; X32-NEXT:    addl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl $0, %esi
-; X32-NEXT:    adcl $0, %ecx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    mull %ebx
+; X32-NEXT:    movl %ebx, %ebp
+; X32-NEXT:    movl %eax, %ebx
+; X32-NEXT:    addl %esi, %ebx
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; X32-NEXT:    adcl %eax, %edx
+; X32-NEXT:    addl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, %ebx
+; X32-NEXT:    adcl $0, %edx
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    movl %ecx, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X32-NEXT:    mull %esi
 ; X32-NEXT:    movl %edx, %esi
 ; X32-NEXT:    movl %eax, %edi
-; X32-NEXT:    addl %ecx, %edi
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
 ; X32-NEXT:    adcl $0, %esi
-; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    movl %eax, %ebp
-; X32-NEXT:    addl %edi, %ebp
-; X32-NEXT:    adcl %esi, %ecx
+; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    movl %ebp, %ecx
+; X32-NEXT:    mull %ebp
+; X32-NEXT:    movl %edx, %ebp
+; X32-NEXT:    addl %edi, %eax
+; X32-NEXT:    movl %eax, %edi
+; X32-NEXT:    adcl %esi, %ebp
 ; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    addl %ecx, %eax
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    addl %ebp, %eax
 ; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
 ; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %esi
-; X32-NEXT:    adcl %edx, %ecx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
-; X32-NEXT:    adcl %eax, %esi
-; X32-NEXT:    adcl $0, %ecx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    movl %esi, %ebp
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
-; X32-NEXT:    adcl %eax, %ebx
-; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl $0, %eax
-; X32-NEXT:    adcl $0, %ebp
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    adcl $0, %ebx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X32-NEXT:    addl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
+; X32-NEXT:    adcl %ecx, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    addl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl (%esp), %eax # 4-byte Reload
-; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
-; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
 ; X32-NEXT:    adcl $0, %edx
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl $0, %esi
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl $0, %edi
-; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl $0, %ecx
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl $0, %ebp
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT:    movl 72(%esi), %ebp
+; X32-NEXT:    movl %ebp, %eax
 ; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl $0, %ebx
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    movl 76(%esi), %ebx
+; X32-NEXT:    movl %ebx, %eax
 ; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edx, %esi
+; X32-NEXT:    movl %eax, %edi
+; X32-NEXT:    addl %ecx, %edi
+; X32-NEXT:    adcl $0, %esi
 ; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl (%esp), %ebp # 4-byte Reload
+; X32-NEXT:    mull %ebp
+; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    addl %edi, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl %esi, %ecx
+; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X32-NEXT:    movl %ebx, %eax
+; X32-NEXT:    mull %ebp
+; X32-NEXT:    addl %ecx, %eax
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; X32-NEXT:    adcl %eax, %edx
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X32-NEXT:    movl 64(%ebx), %ecx
+; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    mull %ecx
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl %edx, %edi
-; X32-NEXT:    movl %eax, %ebx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    adcl $0, %edi
-; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %ebp
-; X32-NEXT:    addl %ebx, %eax
+; X32-NEXT:    movl 68(%ebx), %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edi, %ebp
-; X32-NEXT:    setb %bl
-; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    addl %ebp, %eax
-; X32-NEXT:    movzbl %bl, %ecx
-; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %ecx
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edx, %esi
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    mull %ecx
+; X32-NEXT:    mull %esi
 ; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %ebx
 ; X32-NEXT:    movl %eax, %ebp
-; X32-NEXT:    addl %esi, %ebp
-; X32-NEXT:    adcl $0, %ebx
-; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    addl %edi, %ebp
+; X32-NEXT:    adcl $0, %esi
+; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    movl (%esp), %edi # 4-byte Reload
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edx, %ebx
 ; X32-NEXT:    addl %ebp, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %ebx, %ecx
+; X32-NEXT:    adcl %esi, %ebx
 ; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
 ; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %eax, %ebx
-; X32-NEXT:    addl %ecx, %ebx
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    addl %ebx, %ecx
 ; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
 ; X32-NEXT:    adcl %eax, %edx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
 ; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    movl %eax, %ebp
-; X32-NEXT:    addl %ecx, %ebp
-; X32-NEXT:    adcl $0, %esi
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl 76(%eax), %ecx
-; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %ecx, %edi
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    addl %ebp, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %esi, %ecx
-; X32-NEXT:    setb (%esp) # 1-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %edi
-; X32-NEXT:    addl %ecx, %eax
-; X32-NEXT:    movzbl (%esp), %ecx # 1-byte Folded Reload
-; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %ebp
-; X32-NEXT:    adcl %edx, %ecx
-; X32-NEXT:    addl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl $0, %ebp
-; X32-NEXT:    adcl $0, %ecx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
 ; X32-NEXT:    movl %ebx, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X32-NEXT:    mull %esi
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    movl %ebp, %eax
 ; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, %ebp
 ; X32-NEXT:    movl %eax, %edi
 ; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    adcl $0, %edx
-; X32-NEXT:    movl %edx, (%esp) # 4-byte Spill
+; X32-NEXT:    adcl $0, %ebp
 ; X32-NEXT:    movl %ebx, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X32-NEXT:    mull %ebx
+; X32-NEXT:    movl %edx, %esi
+; X32-NEXT:    addl %edi, %eax
+; X32-NEXT:    movl %eax, %edi
+; X32-NEXT:    adcl %ebp, %esi
+; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    mull %ebx
+; X32-NEXT:    movl %eax, %ebx
+; X32-NEXT:    addl %esi, %ebx
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; X32-NEXT:    adcl %eax, %edx
+; X32-NEXT:    addl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, %ebx
+; X32-NEXT:    adcl $0, %edx
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X32-NEXT:    movl %ebp, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %ebx
-; X32-NEXT:    addl %edi, %eax
+; X32-NEXT:    movl %edx, %ecx
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl (%esp), %ebx # 4-byte Folded Reload
-; X32-NEXT:    setb (%esp) # 1-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X32-NEXT:    mull %esi
-; X32-NEXT:    addl %ebx, %eax
-; X32-NEXT:    movzbl (%esp), %esi # 1-byte Folded Reload
-; X32-NEXT:    adcl %esi, %edx
+; X32-NEXT:    movl %edx, %esi
+; X32-NEXT:    movl %eax, %edi
+; X32-NEXT:    addl %ecx, %edi
+; X32-NEXT:    adcl $0, %esi
+; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X32-NEXT:    mull %ebp
+; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    addl %edi, %eax
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl %esi, %ecx
+; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %edi
-; X32-NEXT:    adcl %edx, %ebx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X32-NEXT:    addl %ebp, %edx
+; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    mull %ebp
+; X32-NEXT:    addl %ecx, %eax
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
+; X32-NEXT:    adcl %ecx, %edx
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    addl %ebx, %ecx
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    adcl %ecx, %esi
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
-; X32-NEXT:    adcl %eax, %edi
-; X32-NEXT:    adcl $0, %ebx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 1-byte Folded Reload
+; X32-NEXT:    adcl %ebx, %eax
+; X32-NEXT:    adcl $0, %edx
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
 ; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
 ; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X32-NEXT:    mull %esi
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %edi
+; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    movl %edi, %ecx
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, %esi
 ; X32-NEXT:    movl %eax, %ebx
 ; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    adcl $0, %edi
+; X32-NEXT:    adcl $0, %esi
 ; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NEXT:    mull %edi
 ; X32-NEXT:    movl %edx, %ebp
 ; X32-NEXT:    addl %ebx, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edi, %ebp
+; X32-NEXT:    adcl %esi, %ebp
 ; X32-NEXT:    setb %bl
-; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    mull %edi
 ; X32-NEXT:    addl %ebp, %eax
-; X32-NEXT:    movzbl %bl, %edi
-; X32-NEXT:    adcl %edi, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %ecx
-; X32-NEXT:    movl %ecx, (%esp) # 4-byte Spill
-; X32-NEXT:    adcl %edx, %esi
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    movl %esi, %eax
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movzbl %bl, %eax
+; X32-NEXT:    adcl %eax, %edx
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X32-NEXT:    movl %ebx, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %edi
+; X32-NEXT:    movl %edx, %esi
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %ebx
-; X32-NEXT:    movl %eax, %ebp
-; X32-NEXT:    addl %edi, %ebp
-; X32-NEXT:    adcl $0, %ebx
-; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %edi
-; X32-NEXT:    addl %ebp, %eax
+; X32-NEXT:    movl %edx, %ebp
+; X32-NEXT:    movl %eax, %edi
+; X32-NEXT:    addl %esi, %edi
+; X32-NEXT:    adcl $0, %ebp
+; X32-NEXT:    movl %ebx, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    addl %edi, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %ebx, %edi
+; X32-NEXT:    adcl %ebp, %ecx
 ; X32-NEXT:    setb %bl
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %eax, %ebp
-; X32-NEXT:    addl %edi, %ebp
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %eax, %esi
+; X32-NEXT:    addl %ecx, %esi
 ; X32-NEXT:    movzbl %bl, %eax
 ; X32-NEXT:    adcl %eax, %edx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl $0, (%esp) # 4-byte Folded Spill
 ; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    movl %ebx, %eax
+; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    movl %ecx, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
 ; X32-NEXT:    mull %edi
-; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %esi, %eax
+; X32-NEXT:    movl %ebp, %eax
 ; X32-NEXT:    mull %edi
-; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    movl %eax, %edi
-; X32-NEXT:    addl %ecx, %edi
-; X32-NEXT:    adcl $0, %esi
-; X32-NEXT:    movl %ebx, %eax
+; X32-NEXT:    movl %edx, %edi
+; X32-NEXT:    movl %eax, %ebx
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X32-NEXT:    adcl $0, %edi
+; X32-NEXT:    movl %ecx, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %ebx
-; X32-NEXT:    addl %edi, %eax
+; X32-NEXT:    movl %edx, %ebp
+; X32-NEXT:    addl %ebx, %eax
+; X32-NEXT:    movl %eax, %ebx
+; X32-NEXT:    adcl %edi, %ebp
+; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    mull %ecx
 ; X32-NEXT:    movl %eax, %edi
-; X32-NEXT:    adcl %esi, %ebx
+; X32-NEXT:    addl %ebp, %edi
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; X32-NEXT:    adcl %eax, %edx
+; X32-NEXT:    addl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, %edi
+; X32-NEXT:    adcl $0, %edx
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X32-NEXT:    movl %esi, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X32-NEXT:    mull %ecx
-; X32-NEXT:    addl %ebx, %eax
+; X32-NEXT:    movl %edx, %ebx
+; X32-NEXT:    movl %eax, %ebp
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X32-NEXT:    adcl $0, %ebx
+; X32-NEXT:    movl %esi, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %edx, %esi
+; X32-NEXT:    addl %ebp, %eax
+; X32-NEXT:    movl %eax, %ebp
+; X32-NEXT:    adcl %ebx, %esi
+; X32-NEXT:    setb %bl
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    addl %esi, %eax
+; X32-NEXT:    movzbl %bl, %esi
+; X32-NEXT:    adcl %esi, %edx
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X32-NEXT:    addl %edi, %ebx
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
 ; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
-; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X32-NEXT:    adcl %ecx, %eax
+; X32-NEXT:    adcl $0, %edx
+; X32-NEXT:    movl %edx, %esi
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %esi
-; X32-NEXT:    adcl %edx, %ecx
-; X32-NEXT:    addl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl $0, %esi
+; X32-NEXT:    addl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    adcl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    adcl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    adcl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl %ebx, %edx
+; X32-NEXT:    adcl $0, %edx
+; X32-NEXT:    movl %ebp, %ecx
 ; X32-NEXT:    adcl $0, %ecx
-; X32-NEXT:    addl (%esp), %esi # 4-byte Folded Reload
+; X32-NEXT:    adcl $0, %eax
+; X32-NEXT:    adcl $0, %esi
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
 ; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    mull %edi
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %eax, (%esp) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %edi
-; X32-NEXT:    movl %eax, %ebx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    adcl $0, %edx
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    mull %edi
-; X32-NEXT:    movl %edx, %ebp
-; X32-NEXT:    addl %ebx, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    setb %bl
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %edi
-; X32-NEXT:    addl %ebp, %eax
-; X32-NEXT:    movzbl %bl, %edi
-; X32-NEXT:    adcl %edi, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %edi
-; X32-NEXT:    adcl %edx, %ebx
-; X32-NEXT:    movl (%esp), %edx # 4-byte Reload
-; X32-NEXT:    addl %esi, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    adcl %ecx, %esi
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
-; X32-NEXT:    adcl %eax, %edi
-; X32-NEXT:    adcl $0, %ebx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    movl %edx, (%esp) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %edi
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %ebx
-; X32-NEXT:    movl %eax, %ecx
-; X32-NEXT:    addl %edi, %ecx
-; X32-NEXT:    adcl $0, %ebx
-; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    mull %ebp
-; X32-NEXT:    movl %edx, %edi
-; X32-NEXT:    addl %ecx, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %ebx, %edi
-; X32-NEXT:    setb %cl
-; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    movl %ebp, %esi
-; X32-NEXT:    mull %ebp
-; X32-NEXT:    addl %edi, %eax
-; X32-NEXT:    movzbl %cl, %ecx
-; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %ecx
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edx, %edi
-; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    mull %edi
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, %ecx
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    mull %edi
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    mull %esi
 ; X32-NEXT:    movl %edx, %edi
 ; X32-NEXT:    movl %eax, %ebx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X32-NEXT:    addl %ecx, %ebx
 ; X32-NEXT:    adcl $0, %edi
-; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X32-NEXT:    mull %esi
 ; X32-NEXT:    movl %edx, %ecx
 ; X32-NEXT:    addl %ebx, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    adcl %edi, %ecx
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    setb %bl
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %eax, %ebx
-; X32-NEXT:    addl %ecx, %ebx
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; X32-NEXT:    addl %ecx, %eax
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movzbl %bl, %eax
 ; X32-NEXT:    adcl %eax, %edx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X32-NEXT:    movl %esi, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X32-NEXT:    mull %ebx
+; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    mull %ebx
+; X32-NEXT:    movl %edx, %ebx
+; X32-NEXT:    movl %eax, %ebp
+; X32-NEXT:    addl %ecx, %ebp
+; X32-NEXT:    adcl $0, %ebx
+; X32-NEXT:    movl %esi, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl %edx, %esi
+; X32-NEXT:    addl %ebp, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    adcl %ebx, %esi
+; X32-NEXT:    setb %bl
+; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    movl %edi, %ebp
 ; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %edi
 ; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    addl %esi, %ecx
+; X32-NEXT:    movzbl %bl, %eax
+; X32-NEXT:    adcl %eax, %edx
 ; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    adcl $0, %edi
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl 92(%eax), %ebp
-; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    mull %ebp
-; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X32-NEXT:    movl %ebx, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NEXT:    mull %edi
 ; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    addl %ecx, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edi, %esi
-; X32-NEXT:    setb %cl
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %ebp
-; X32-NEXT:    addl %esi, %eax
-; X32-NEXT:    movzbl %cl, %ecx
-; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %ebp
-; X32-NEXT:    adcl %edx, %ecx
-; X32-NEXT:    addl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edx, %ebp
+; X32-NEXT:    movl %eax, %edi
+; X32-NEXT:    addl %esi, %edi
 ; X32-NEXT:    adcl $0, %ebp
-; X32-NEXT:    adcl $0, %ecx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
 ; X32-NEXT:    movl %ebx, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl %edx, %ebx
+; X32-NEXT:    addl %edi, %eax
+; X32-NEXT:    movl %eax, %edi
+; X32-NEXT:    adcl %ebp, %ebx
+; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %eax, %edi
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT:    movl %eax, %ebp
+; X32-NEXT:    addl %ebx, %ebp
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; X32-NEXT:    adcl %eax, %edx
+; X32-NEXT:    addl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, %ebp
 ; X32-NEXT:    adcl $0, %edx
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %ebx, %eax
+; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    movl %ecx, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %ebx
-; X32-NEXT:    addl %edi, %eax
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, %edi
+; X32-NEXT:    movl %eax, %ebx
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X32-NEXT:    adcl $0, %edi
+; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %edx, %esi
 ; X32-NEXT:    addl %ebx, %eax
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 1-byte Folded Reload
-; X32-NEXT:    adcl %esi, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %edi
-; X32-NEXT:    adcl %edx, %ebx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X32-NEXT:    addl %ebp, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    adcl %ecx, %esi
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl %edi, %esi
+; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %eax, %ebx
+; X32-NEXT:    addl %esi, %ebx
 ; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
-; X32-NEXT:    adcl %eax, %edi
-; X32-NEXT:    adcl $0, %ebx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X32-NEXT:    adcl %eax, %edx
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    addl %ebp, %ecx
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    movl (%esp), %eax # 4-byte Reload
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; X32-NEXT:    adcl %eax, %ebx
+; X32-NEXT:    adcl $0, %edx
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X32-NEXT:    addl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
@@ -4496,445 +4105,254 @@ define void @test_1024(i1024* %a, i1024* %b, i1024* %out) nounwind {
 ; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl $0, %edx
-; X32-NEXT:    adcl $0, %esi
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; X32-NEXT:    adcl %eax, %ecx
+; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    adcl $0, %edi
+; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    adcl $0, %ebx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, %edx
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl 96(%eax), %edi
 ; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    movl %ebp, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    movl %esi, %eax
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edx, %ecx
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    mull %esi
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    mull %edi
 ; X32-NEXT:    movl %edx, %edi
-; X32-NEXT:    movl %eax, %esi
-; X32-NEXT:    addl %ecx, %esi
+; X32-NEXT:    movl %eax, %ebx
+; X32-NEXT:    addl %ecx, %ebx
 ; X32-NEXT:    adcl $0, %edi
-; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    mull %ebp
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl 100(%eax), %ecx
+; X32-NEXT:    movl %esi, %eax
+; X32-NEXT:    movl %ecx, %esi
+; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    mull %ecx
 ; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    addl %esi, %eax
+; X32-NEXT:    addl %ebx, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    adcl %edi, %ecx
-; X32-NEXT:    setb (%esp) # 1-byte Folded Spill
-; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    mull %ebp
+; X32-NEXT:    setb %bl
+; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    mull %esi
 ; X32-NEXT:    addl %ecx, %eax
-; X32-NEXT:    movzbl (%esp), %ecx # 1-byte Folded Reload
-; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %esi
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edx, %ecx
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    movl %ebx, %eax
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movzbl %bl, %eax
+; X32-NEXT:    adcl %eax, %edx
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %esi, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NEXT:    mull %edi
 ; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    movl %eax, (%esp) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    movl %eax, %edi
-; X32-NEXT:    addl %ecx, %edi
-; X32-NEXT:    adcl $0, %esi
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl (%esp), %ebx # 4-byte Reload
 ; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    addl %edi, %eax
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edx, %edi
+; X32-NEXT:    movl %eax, %ebp
+; X32-NEXT:    addl %ecx, %ebp
+; X32-NEXT:    adcl $0, %edi
+; X32-NEXT:    movl %esi, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %edx, %esi
+; X32-NEXT:    addl %ebp, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %esi, %ecx
+; X32-NEXT:    adcl %edi, %esi
 ; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    mull %ebx
+; X32-NEXT:    movl %ebx, %eax
+; X32-NEXT:    mull %ecx
 ; X32-NEXT:    movl %eax, %edi
-; X32-NEXT:    addl %ecx, %edi
+; X32-NEXT:    addl %esi, %edi
 ; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
 ; X32-NEXT:    adcl %eax, %edx
 ; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
 ; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl 104(%eax), %esi
+; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %ebx
+; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    movl %eax, %ecx
-; X32-NEXT:    addl %ebx, %ecx
-; X32-NEXT:    adcl $0, %esi
-; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    mull %ebx
+; X32-NEXT:    movl %ebx, %eax
+; X32-NEXT:    mull %esi
 ; X32-NEXT:    movl %edx, %ebp
-; X32-NEXT:    addl %ecx, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %esi, %ebp
-; X32-NEXT:    setb %cl
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    addl %ebp, %eax
-; X32-NEXT:    movzbl %cl, %ecx
-; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %ecx
-; X32-NEXT:    adcl %edx, %esi
+; X32-NEXT:    movl %eax, %ebx
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X32-NEXT:    adcl $0, %ebp
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl 108(%eax), %esi
+; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    addl %ebx, %eax
+; X32-NEXT:    movl %eax, %ebx
+; X32-NEXT:    adcl %ebp, %ecx
+; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X32-NEXT:    movl (%esp), %eax # 4-byte Reload
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, %ebp
+; X32-NEXT:    movl %eax, %esi
+; X32-NEXT:    addl %ecx, %esi
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; X32-NEXT:    adcl %eax, %ebp
 ; X32-NEXT:    addl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl $0, %ecx
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    adcl $0, %esi
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, %ebp
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
 ; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %ebx
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    movl %eax, %edi
-; X32-NEXT:    addl %ebx, %edi
-; X32-NEXT:    adcl $0, %esi
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edx, %edi
+; X32-NEXT:    movl %eax, %ebx
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X32-NEXT:    adcl $0, %edi
 ; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    mull %ebx
+; X32-NEXT:    mull {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
 ; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    addl %edi, %eax
-; X32-NEXT:    movl %eax, %edi
-; X32-NEXT:    adcl %esi, %ecx
+; X32-NEXT:    addl %ebx, %eax
+; X32-NEXT:    movl %eax, %ebx
+; X32-NEXT:    adcl %edi, %ecx
 ; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    mull %ebx
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NEXT:    mull %edi
 ; X32-NEXT:    addl %ecx, %eax
 ; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
 ; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %esi
-; X32-NEXT:    adcl %edx, %ecx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    movl %edi, %ebp
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
-; X32-NEXT:    adcl %eax, %esi
-; X32-NEXT:    adcl $0, %ecx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    addl %eax, (%esp) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
-; X32-NEXT:    adcl %eax, %edx
+; X32-NEXT:    addl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl %ebp, %ebx
+; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
+; X32-NEXT:    adcl %ecx, %eax
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, %edx
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl $0, %ebp
-; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl $0, %esi
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl $0, %ecx
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    movl %edi, %esi
+; X32-NEXT:    imull %eax, %esi
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    addl %esi, %edx
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X32-NEXT:    imull %ebx, %ecx
+; X32-NEXT:    addl %edx, %ecx
+; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    movl %eax, %esi
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NEXT:    imull %edi, %esi
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X32-NEXT:    mull %ebp
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    addl %esi, %edx
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %edi
-; X32-NEXT:    movl %eax, %ebx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    adcl $0, %edi
 ; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    imull %ebp, %esi
+; X32-NEXT:    addl %edx, %esi
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X32-NEXT:    mull %ecx
 ; X32-NEXT:    movl %edx, %ebp
-; X32-NEXT:    addl %ebx, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edi, %ebp
-; X32-NEXT:    setb %bl
-; X32-NEXT:    movl %esi, %eax
+; X32-NEXT:    movl %edi, %eax
 ; X32-NEXT:    mull %ecx
-; X32-NEXT:    addl %ebp, %eax
-; X32-NEXT:    movzbl %bl, %edi
-; X32-NEXT:    adcl %edi, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %ecx
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edx, %esi
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    movl %edx, %ebx
-; X32-NEXT:    movl %eax, %ebp
-; X32-NEXT:    addl %ecx, %ebp
-; X32-NEXT:    adcl $0, %ebx
-; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %edi
-; X32-NEXT:    addl %ebp, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %ebx, %edi
-; X32-NEXT:    setb %bl
-; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    addl %edi, %eax
-; X32-NEXT:    movzbl %bl, %edi
-; X32-NEXT:    adcl %edi, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %edi
-; X32-NEXT:    adcl %edx, %ecx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    mull %ebp
-; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    mull %ebp
 ; X32-NEXT:    movl %edx, %esi
 ; X32-NEXT:    movl %eax, %edi
-; X32-NEXT:    addl %ecx, %edi
+; X32-NEXT:    addl %ebp, %edi
 ; X32-NEXT:    adcl $0, %esi
-; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X32-NEXT:    mull %ebx
+; X32-NEXT:    movl %ebx, %ebp
 ; X32-NEXT:    movl %edx, %ecx
 ; X32-NEXT:    addl %edi, %eax
-; X32-NEXT:    movl %eax, %edi
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    adcl %esi, %ecx
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    addl %ecx, %eax
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
-; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %esi
-; X32-NEXT:    adcl %edx, %ecx
+; X32-NEXT:    setb %bl
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    addl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl $0, %esi
-; X32-NEXT:    adcl $0, %ecx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    movl %ebp, %edi
 ; X32-NEXT:    mull %ebp
-; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    mull %edi
-; X32-NEXT:    movl %edx, %edi
-; X32-NEXT:    movl %eax, %ebx
-; X32-NEXT:    addl %ecx, %ebx
-; X32-NEXT:    adcl $0, %edi
-; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    addl %ebx, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edi, %ecx
-; X32-NEXT:    setb %bl
-; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %esi, %edi
 ; X32-NEXT:    addl %ecx, %eax
 ; X32-NEXT:    movzbl %bl, %ecx
 ; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %esi
-; X32-NEXT:    adcl %edx, %ecx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    addl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X32-NEXT:    movl 124(%edi), %ecx
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
-; X32-NEXT:    adcl %eax, %esi
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl $0, %ecx
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    imull %ebp, %edi
-; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    mull %ecx
+; X32-NEXT:    imull %eax, %ecx
+; X32-NEXT:    movl 120(%edi), %esi
+; X32-NEXT:    movl %edi, %ebx
+; X32-NEXT:    mull %esi
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    addl %edi, %edx
-; X32-NEXT:    imull {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    addl %edx, %ecx
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    addl %ecx, %edx
+; X32-NEXT:    imull (%esp), %esi # 4-byte Folded Reload
+; X32-NEXT:    addl %edx, %esi
+; X32-NEXT:    movl 112(%edi), %edi
+; X32-NEXT:    movl 116(%ebx), %ebp
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    movl %eax, %esi
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    imull %ebx, %esi
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NEXT:    movl %eax, %ebx
+; X32-NEXT:    imull %ebp, %ebx
+; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    mull %edi
-; X32-NEXT:    addl %esi, %edx
+; X32-NEXT:    addl %ebx, %edx
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X32-NEXT:    imull %edi, %ecx
 ; X32-NEXT:    addl %edx, %ecx
 ; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X32-NEXT:    adcl %esi, %ecx
 ; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    movl %edi, %esi
-; X32-NEXT:    mull %ebp
-; X32-NEXT:    movl %edx, %edi
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    movl %ebx, %ecx
-; X32-NEXT:    mull %ebp
-; X32-NEXT:    movl %edx, %ebp
-; X32-NEXT:    movl %eax, %ebx
-; X32-NEXT:    addl %edi, %ebx
-; X32-NEXT:    adcl $0, %ebp
-; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    mull %edi
-; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    addl %ebx, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %ebp, %esi
-; X32-NEXT:    setb %bl
-; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    mull %edi
-; X32-NEXT:    addl %esi, %eax
-; X32-NEXT:    movzbl %bl, %ecx
-; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    imull %ebp, %esi
-; X32-NEXT:    movl %ebp, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %edx, %esi
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    addl %esi, %edx
-; X32-NEXT:    imull {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    addl %edx, %ecx
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    movl %eax, %ebx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    imull %edi, %ebx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    movl %ebp, %eax
 ; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %eax, %esi
-; X32-NEXT:    addl %ebx, %edx
-; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    imull %eax, %ecx
-; X32-NEXT:    addl %edx, %ecx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %eax, %esi
-; X32-NEXT:    mull %ebp
-; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    mull %ebp
 ; X32-NEXT:    movl %edx, %ebx
 ; X32-NEXT:    movl %eax, %ebp
-; X32-NEXT:    addl %ecx, %ebp
+; X32-NEXT:    addl %esi, %ebp
 ; X32-NEXT:    adcl $0, %ebx
-; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %esi
+; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    movl (%esp), %esi # 4-byte Reload
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, %ecx
 ; X32-NEXT:    movl %eax, %edi
 ; X32-NEXT:    addl %ebp, %edi
-; X32-NEXT:    adcl %ebx, %esi
+; X32-NEXT:    adcl %ebx, %ecx
 ; X32-NEXT:    setb %bl
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    addl %esi, %eax
+; X32-NEXT:    mull %esi
+; X32-NEXT:    addl %ecx, %eax
 ; X32-NEXT:    movzbl %bl, %ecx
 ; X32-NEXT:    adcl %ecx, %edx
 ; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
@@ -4952,3155 +4370,1517 @@ define void @test_1024(i1024* %a, i1024* %b, i1024* %out) nounwind {
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X32-NEXT:    movl 104(%esi), %ebp
-; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    movl %ecx, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
 ; X32-NEXT:    mull %edi
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    movl 108(%esi), %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    mull %edi
 ; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    movl %eax, %ebx
-; X32-NEXT:    addl %ecx, %ebx
-; X32-NEXT:    adcl $0, %esi
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
 ; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
 ; X32-NEXT:    mull %edi
-; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    movl %edx, %edi
+; X32-NEXT:    movl %eax, %ebx
+; X32-NEXT:    addl %esi, %ebx
+; X32-NEXT:    adcl $0, %edi
+; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %edx, %esi
 ; X32-NEXT:    addl %ebx, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %esi, %ecx
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %edi
-; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    movl %eax, %ebx
-; X32-NEXT:    addl %ecx, %ebx
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
-; X32-NEXT:    adcl %eax, %esi
+; X32-NEXT:    adcl %edi, %esi
+; X32-NEXT:    setb %bl
 ; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    xorl %ecx, %ecx
 ; X32-NEXT:    mull %ecx
+; X32-NEXT:    addl %esi, %eax
+; X32-NEXT:    movl %eax, (%esp) # 4-byte Spill
+; X32-NEXT:    movzbl %bl, %eax
+; X32-NEXT:    adcl %eax, %edx
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %edx, %esi
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    addl %ebx, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %esi, %edx
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X32-NEXT:    movl 96(%edi), %ebx
-; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %edx, %ebx
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    addl %esi, %ecx
+; X32-NEXT:    adcl $0, %ebx
+; X32-NEXT:    movl %ebp, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, %ebp
+; X32-NEXT:    addl %ecx, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    movl 100(%edi), %edi
+; X32-NEXT:    adcl %ebx, %ebp
+; X32-NEXT:    setb %cl
 ; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    movl %eax, %ebp
-; X32-NEXT:    addl %ecx, %ebp
-; X32-NEXT:    adcl $0, %esi
-; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    addl %ebp, %eax
+; X32-NEXT:    movl %eax, %ebx
+; X32-NEXT:    addl %ebp, %ebx
+; X32-NEXT:    movzbl %cl, %eax
+; X32-NEXT:    adcl %eax, %edx
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, (%esp) # 4-byte Folded Spill
+; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %esi, %ecx
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
 ; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    movl %edx, %ebx
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, %ebp
 ; X32-NEXT:    movl %eax, %edi
-; X32-NEXT:    addl %ecx, %edi
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
-; X32-NEXT:    adcl %eax, %ebx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    xorl %ecx, %ecx
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %eax, %ebp
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT:    adcl $0, %ebp
+; X32-NEXT:    movl %ecx, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    addl %eax, %ecx
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %edx, %esi
+; X32-NEXT:    addl %edi, %eax
+; X32-NEXT:    movl %eax, %edi
+; X32-NEXT:    adcl %ebp, %esi
+; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %edx, %eax
-; X32-NEXT:    addl %edi, %ecx
-; X32-NEXT:    adcl %ebx, %eax
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    movl %eax, %ebp
+; X32-NEXT:    addl %esi, %ebp
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; X32-NEXT:    adcl %eax, %ecx
+; X32-NEXT:    addl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, %ebp
+; X32-NEXT:    adcl $0, %ecx
+; X32-NEXT:    addl (%esp), %ebp # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X32-NEXT:    setb (%esp) # 1-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X32-NEXT:    movl %esi, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
 ; X32-NEXT:    mull %edi
-; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X32-NEXT:    mull %edi
 ; X32-NEXT:    movl %edx, %edi
 ; X32-NEXT:    movl %eax, %ebx
-; X32-NEXT:    addl %ecx, %ebx
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
 ; X32-NEXT:    adcl $0, %edi
 ; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    mull %ecx
+; X32-NEXT:    mull {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
 ; X32-NEXT:    movl %edx, %esi
 ; X32-NEXT:    addl %ebx, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl %eax, %ebx
 ; X32-NEXT:    adcl %edi, %esi
-; X32-NEXT:    setb %bl
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    addl %esi, %eax
-; X32-NEXT:    movzbl %bl, %ecx
-; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %ebp
-; X32-NEXT:    adcl %edx, %ecx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    addl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl $0, %ebp
-; X32-NEXT:    adcl $0, %ecx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
 ; X32-NEXT:    mull %edi
-; X32-NEXT:    movl %edx, %ebp
+; X32-NEXT:    addl %esi, %eax
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 1-byte Folded Reload
+; X32-NEXT:    adcl %esi, %edx
+; X32-NEXT:    addl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl %ecx, %ebx
+; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movzbl (%esp), %ecx # 1-byte Folded Reload
+; X32-NEXT:    adcl %ecx, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, %edx
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    movl %edi, %ecx
+; X32-NEXT:    imull %eax, %ecx
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %eax, (%esp) # 4-byte Spill
+; X32-NEXT:    addl %ecx, %edx
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X32-NEXT:    imull %ebp, %esi
+; X32-NEXT:    addl %edx, %esi
+; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    movl %eax, %edi
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X32-NEXT:    imull %ebx, %edi
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    addl %edi, %edx
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
 ; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    mull %edi
+; X32-NEXT:    imull %esi, %edi
+; X32-NEXT:    addl %edx, %edi
+; X32-NEXT:    addl (%esp), %ecx # 4-byte Folded Reload
+; X32-NEXT:    movl %ecx, (%esp) # 4-byte Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %edx, %esi
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl %ebx, %eax
+; X32-NEXT:    mull %ecx
 ; X32-NEXT:    movl %edx, %edi
 ; X32-NEXT:    movl %eax, %ebx
-; X32-NEXT:    addl %ebp, %ebx
+; X32-NEXT:    addl %esi, %ebx
 ; X32-NEXT:    adcl $0, %edi
-; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X32-NEXT:    mull %ebp
-; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    movl %ebp, %esi
+; X32-NEXT:    movl %edx, %ebp
 ; X32-NEXT:    addl %ebx, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edi, %ecx
-; X32-NEXT:    setb %bl
-; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    mull %ebp
-; X32-NEXT:    addl %ecx, %eax
-; X32-NEXT:    movzbl %bl, %ecx
+; X32-NEXT:    adcl %edi, %ebp
+; X32-NEXT:    setb %cl
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    mull %esi
+; X32-NEXT:    addl %ebp, %eax
+; X32-NEXT:    movzbl %cl, %ecx
 ; X32-NEXT:    adcl %ecx, %edx
+; X32-NEXT:    addl (%esp), %eax # 4-byte Folded Reload
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %esi
-; X32-NEXT:    adcl %edx, %ecx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    addl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
-; X32-NEXT:    adcl %eax, %esi
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl $0, %ecx
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X32-NEXT:    movl 112(%esi), %edi
-; X32-NEXT:    imull %edi, %ebp
-; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X32-NEXT:    imull %esi, %ebx
+; X32-NEXT:    movl %esi, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    addl %ebp, %edx
-; X32-NEXT:    movl 116(%esi), %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    imull %eax, %ecx
+; X32-NEXT:    movl %eax, (%esp) # 4-byte Spill
+; X32-NEXT:    addl %ebx, %edx
+; X32-NEXT:    imull {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
 ; X32-NEXT:    addl %edx, %ecx
 ; X32-NEXT:    movl %ecx, %ebx
-; X32-NEXT:    movl 120(%esi), %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X32-NEXT:    movl %eax, %ecx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    imull %esi, %ecx
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NEXT:    imull %edi, %ecx
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
 ; X32-NEXT:    mull %ebp
 ; X32-NEXT:    addl %ecx, %edx
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    movl 124(%ecx), %ecx
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X32-NEXT:    imull %ebp, %ecx
 ; X32-NEXT:    addl %edx, %ecx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X32-NEXT:    addl (%esp), %eax # 4-byte Folded Reload
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    adcl %ebx, %ecx
+; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %ebp, %ebx
+; X32-NEXT:    mull %esi
 ; X32-NEXT:    movl %edx, %ebp
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %eax, (%esp) # 4-byte Spill
+; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    mull %esi
 ; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    movl %eax, %ebx
-; X32-NEXT:    addl %ebp, %ebx
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    addl %ebp, %ecx
 ; X32-NEXT:    adcl $0, %esi
+; X32-NEXT:    movl %ebx, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X32-NEXT:    mull %ebp
+; X32-NEXT:    movl %edx, %edi
+; X32-NEXT:    movl %eax, %ebx
+; X32-NEXT:    addl %ecx, %ebx
+; X32-NEXT:    adcl %esi, %edi
+; X32-NEXT:    setb %cl
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    mull %edi
-; X32-NEXT:    movl %edx, %ebp
-; X32-NEXT:    addl %ebx, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %esi, %ebp
-; X32-NEXT:    setb %bl
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %edi
-; X32-NEXT:    addl %ebp, %eax
-; X32-NEXT:    movzbl %bl, %esi
-; X32-NEXT:    adcl %esi, %edx
+; X32-NEXT:    mull %ebp
+; X32-NEXT:    movl %edx, %esi
+; X32-NEXT:    addl %edi, %eax
+; X32-NEXT:    movzbl %cl, %ecx
+; X32-NEXT:    adcl %ecx, %esi
 ; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    imull %eax, %esi
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    addl %esi, %edx
-; X32-NEXT:    imull {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    addl %edx, %ebx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    movl %eax, %esi
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    imull %ebp, %esi
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %eax, %edi
-; X32-NEXT:    addl %esi, %edx
-; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X32-NEXT:    movl (%esp), %ecx # 4-byte Reload
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X32-NEXT:    movl %eax, %ebp
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X32-NEXT:    movl %ecx, %edx
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X32-NEXT:    movl %esi, %edi
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    imull %eax, %ecx
-; X32-NEXT:    addl %edx, %ecx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %ebx, %ecx
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %ebx
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %ebp
-; X32-NEXT:    movl %eax, %esi
-; X32-NEXT:    addl %ebx, %esi
-; X32-NEXT:    adcl $0, %ebp
-; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %ebx
-; X32-NEXT:    movl %eax, %edi
-; X32-NEXT:    addl %esi, %edi
-; X32-NEXT:    adcl %ebp, %ebx
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    addl %ebx, %eax
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 1-byte Folded Reload
-; X32-NEXT:    adcl %esi, %edx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    movl %edi, %ecx
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    movl %esi, %edi
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl %ecx, %ebx
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    movl %edx, %ebp
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X32-NEXT:    movl %edx, (%esp) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl (%esp), %eax # 4-byte Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X32-NEXT:    movl %eax, (%esp) # 4-byte Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
 ; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    addl (%esp), %eax # 4-byte Folded Reload
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
 ; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
 ; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl 92(%eax), %esi
-; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %edi
-; X32-NEXT:    movl %eax, %ebx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    adcl $0, %edi
-; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %ebp
-; X32-NEXT:    addl %ebx, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edi, %ebp
-; X32-NEXT:    setb %bl
-; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    addl %ebp, %eax
-; X32-NEXT:    movzbl %bl, %edi
-; X32-NEXT:    adcl %edi, %edx
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %ecx
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
 ; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edx, %esi
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    movl %edx, %edi
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    movl %edx, %ebx
-; X32-NEXT:    movl %eax, %ebp
-; X32-NEXT:    addl %edi, %ebp
-; X32-NEXT:    adcl $0, %ebx
-; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %edi
-; X32-NEXT:    addl %ebp, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %ebx, %edi
-; X32-NEXT:    setb %bl
-; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %eax, %ebp
-; X32-NEXT:    addl %edi, %ebp
-; X32-NEXT:    movzbl %bl, %eax
-; X32-NEXT:    adcl %eax, %edx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X32-NEXT:    movl (%esp), %edx # 4-byte Reload
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X32-NEXT:    movl %edx, (%esp) # 4-byte Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    movl %eax, %edi
-; X32-NEXT:    addl %ecx, %edi
-; X32-NEXT:    adcl $0, %esi
-; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %ebx
-; X32-NEXT:    addl %edi, %eax
-; X32-NEXT:    movl %eax, %edi
-; X32-NEXT:    adcl %esi, %ebx
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    addl %ebx, %eax
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
-; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %ecx
-; X32-NEXT:    adcl %edx, %esi
-; X32-NEXT:    addl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
 ; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl $0, %ecx
-; X32-NEXT:    adcl $0, %esi
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    mull %edi
-; X32-NEXT:    movl %edx, %ebp
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %edi
-; X32-NEXT:    movl %eax, %edi
-; X32-NEXT:    addl %ebp, %edi
-; X32-NEXT:    adcl $0, %edx
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    mull %ebp
-; X32-NEXT:    movl %edx, %ebx
-; X32-NEXT:    addl %edi, %eax
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %ebp
-; X32-NEXT:    addl %ebx, %eax
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 1-byte Folded Reload
-; X32-NEXT:    adcl %edi, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %edi
-; X32-NEXT:    adcl %edx, %ebx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X32-NEXT:    addl %ecx, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl %esi, %ecx
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
-; X32-NEXT:    adcl %eax, %edi
-; X32-NEXT:    adcl $0, %ebx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
 ; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl (%esp), %edi # 4-byte Reload
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl 76(%eax), %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %edi
-; X32-NEXT:    movl %eax, %ebp
-; X32-NEXT:    addl %esi, %ebp
-; X32-NEXT:    adcl $0, %edi
-; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    addl %ebp, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edi, %esi
-; X32-NEXT:    setb %cl
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    addl %esi, %eax
-; X32-NEXT:    movzbl %cl, %ecx
-; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %ecx
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edx, %esi
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %edi
-; X32-NEXT:    movl %eax, %ebp
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    adcl $0, %edi
-; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    addl %ebp, %eax
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edi, %esi
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %eax, %ebp
-; X32-NEXT:    addl %esi, %ebp
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
-; X32-NEXT:    adcl %eax, %edx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    movl %eax, %ebx
-; X32-NEXT:    addl %ecx, %ebx
-; X32-NEXT:    adcl $0, %esi
-; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %edi
-; X32-NEXT:    addl %ebx, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %esi, %edi
-; X32-NEXT:    setb %bl
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    addl %edi, %eax
-; X32-NEXT:    movzbl %bl, %ecx
-; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %ebx
-; X32-NEXT:    adcl %edx, %ecx
-; X32-NEXT:    addl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl $0, %ebx
-; X32-NEXT:    adcl $0, %ecx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    movl %esi, (%ecx)
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %eax, %edi
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    adcl $0, %edx
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    movl %esi, 4(%ecx)
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %ebp
-; X32-NEXT:    addl %edi, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    addl %ebp, %eax
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 1-byte Folded Reload
-; X32-NEXT:    adcl %esi, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %edi
-; X32-NEXT:    adcl %edx, %ebp
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X32-NEXT:    addl %ebx, %edx
+; X32-NEXT:    movl %esi, 8(%ecx)
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    adcl %ecx, %esi
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
-; X32-NEXT:    adcl %eax, %edi
-; X32-NEXT:    adcl $0, %ebp
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %edi
-; X32-NEXT:    movl %eax, %ecx
-; X32-NEXT:    addl %esi, %ecx
-; X32-NEXT:    adcl $0, %edi
-; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    addl %ecx, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edi, %esi
-; X32-NEXT:    setb %cl
-; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    addl %esi, %eax
-; X32-NEXT:    movzbl %cl, %ecx
-; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X32-NEXT:    movl %esi, 12(%ecx)
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %ecx
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edx, %esi
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    movl %ebx, %eax
+; X32-NEXT:    movl %esi, 16(%ecx)
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    movl %eax, %edi
-; X32-NEXT:    addl %ecx, %edi
-; X32-NEXT:    adcl $0, %esi
-; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    addl %edi, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %esi, %ecx
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    movl %eax, %edi
-; X32-NEXT:    addl %ecx, %edi
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
-; X32-NEXT:    adcl %eax, %edx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    movl %eax, %ecx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    adcl $0, %esi
-; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    mull %ebp
-; X32-NEXT:    movl %edx, %ebx
-; X32-NEXT:    addl %ecx, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %esi, %ebx
-; X32-NEXT:    setb %cl
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %ebp
-; X32-NEXT:    addl %ebx, %eax
-; X32-NEXT:    movzbl %cl, %ecx
-; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %ebp
-; X32-NEXT:    adcl %edx, %ecx
-; X32-NEXT:    addl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl $0, %ebp
-; X32-NEXT:    adcl $0, %ecx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    movl %esi, 20(%ecx)
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %eax, %ebx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    adcl $0, %edx
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    movl %esi, 24(%ecx)
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %edi
-; X32-NEXT:    addl %ebx, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    setb %bl
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    addl %edi, %eax
-; X32-NEXT:    movzbl %bl, %esi
-; X32-NEXT:    adcl %esi, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %edi
-; X32-NEXT:    adcl %edx, %ebx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X32-NEXT:    addl %ebp, %edx
+; X32-NEXT:    movl %esi, 28(%ecx)
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    adcl %ecx, %esi
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
-; X32-NEXT:    adcl %eax, %edi
-; X32-NEXT:    adcl $0, %ebx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    addl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl $0, %edx
-; X32-NEXT:    adcl $0, %esi
-; X32-NEXT:    adcl $0, %edi
-; X32-NEXT:    adcl $0, %ebx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    movl %esi, 32(%ecx)
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %ebx
-; X32-NEXT:    movl %eax, %esi
-; X32-NEXT:    addl %ecx, %esi
-; X32-NEXT:    adcl $0, %ebx
-; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    mull %edi
-; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    addl %esi, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %ebx, %ecx
-; X32-NEXT:    setb %bl
-; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    mull %edi
-; X32-NEXT:    addl %ecx, %eax
-; X32-NEXT:    movzbl %bl, %ecx
-; X32-NEXT:    adcl %ecx, %edx
+; X32-NEXT:    movl %esi, 36(%ecx)
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %esi
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edx, %ecx
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    movl %esi, 40(%ecx)
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    movl %eax, %ebx
-; X32-NEXT:    addl %ecx, %ebx
-; X32-NEXT:    adcl $0, %esi
-; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    mull %ebp
-; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    addl %ebx, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %esi, %ecx
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    mull %ebp
-; X32-NEXT:    movl %eax, %ebx
-; X32-NEXT:    addl %ecx, %ebx
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
-; X32-NEXT:    adcl %eax, %edx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    movl %eax, %ecx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    adcl $0, %esi
-; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    mull %edi
-; X32-NEXT:    movl %edx, %ebp
-; X32-NEXT:    addl %ecx, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %esi, %ebp
-; X32-NEXT:    setb %cl
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %edi
-; X32-NEXT:    addl %ebp, %eax
-; X32-NEXT:    movzbl %cl, %ecx
-; X32-NEXT:    adcl %ecx, %edx
+; X32-NEXT:    movl %esi, 44(%ecx)
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %esi
-; X32-NEXT:    adcl %edx, %ecx
-; X32-NEXT:    addl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl $0, %esi
-; X32-NEXT:    adcl $0, %ecx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    movl %esi, 48(%ecx)
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    movl %eax, %edi
-; X32-NEXT:    addl %ecx, %edi
-; X32-NEXT:    adcl $0, %esi
-; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    mull %ebp
-; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    addl %edi, %eax
-; X32-NEXT:    movl %eax, %edi
-; X32-NEXT:    adcl %esi, %ecx
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    mull %ebp
-; X32-NEXT:    addl %ecx, %eax
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
-; X32-NEXT:    adcl %ecx, %edx
+; X32-NEXT:    movl %esi, 52(%ecx)
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %esi
-; X32-NEXT:    adcl %edx, %ecx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
-; X32-NEXT:    adcl %eax, %esi
-; X32-NEXT:    adcl $0, %ecx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X32-NEXT:    movl %esi, 56(%ecx)
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X32-NEXT:    movl %esi, 60(%ecx)
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    addl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl %eax, 64(%ecx)
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl %eax, 68(%ecx)
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl %eax, 72(%ecx)
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
-; X32-NEXT:    adcl %eax, %edx
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl $0, %edi
-; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl $0, %esi
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl $0, %ecx
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    movl 96(%ecx), %ebx
-; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    movl %edx, %ebx
-; X32-NEXT:    movl %eax, %ebp
-; X32-NEXT:    addl %esi, %ebp
-; X32-NEXT:    adcl $0, %ebx
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl 100(%eax), %esi
-; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %esi, %ecx
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    addl %ebp, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %ebx, %esi
-; X32-NEXT:    setb %bl
-; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    movl %eax, %edi
-; X32-NEXT:    addl %esi, %edi
-; X32-NEXT:    movzbl %bl, %eax
-; X32-NEXT:    adcl %eax, %ecx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    xorl %edx, %edx
-; X32-NEXT:    mull %edx
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    addl %eax, %ebx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    adcl %edx, %esi
-; X32-NEXT:    addl %edi, %ebx
-; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %ecx, %esi
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    mull %ebp
-; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %ebp
-; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    movl %eax, %edi
-; X32-NEXT:    addl %ecx, %edi
-; X32-NEXT:    adcl $0, %esi
-; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    addl %edi, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %esi, %ecx
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    addl %ecx, %eax
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
-; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %ebp
-; X32-NEXT:    adcl %edx, %ecx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl 104(%eax), %ecx
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %ebx
-; X32-NEXT:    movl %eax, %ecx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    adcl $0, %ebx
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl 108(%eax), %esi
-; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %edx, %edi
-; X32-NEXT:    addl %ecx, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %ebx, %edi
-; X32-NEXT:    setb %bl
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    movl %eax, %esi
-; X32-NEXT:    addl %edi, %esi
-; X32-NEXT:    movzbl %bl, %eax
-; X32-NEXT:    adcl %eax, %ecx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    xorl %edx, %edx
-; X32-NEXT:    mull %edx
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    addl %eax, %edi
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %edx, %eax
-; X32-NEXT:    addl %esi, %edi
-; X32-NEXT:    adcl %ecx, %eax
-; X32-NEXT:    addl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl $0, %edi
-; X32-NEXT:    adcl $0, %eax
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    movl %ebx, %esi
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    movl %edx, %ebx
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    movl %eax, %ebp
-; X32-NEXT:    addl %ebx, %ebp
-; X32-NEXT:    adcl $0, %esi
-; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    addl %ebp, %eax
-; X32-NEXT:    movl %eax, %ebp
-; X32-NEXT:    adcl %esi, %ecx
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    addl %ecx, %eax
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
-; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %esi
-; X32-NEXT:    adcl %edx, %ecx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    addl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
-; X32-NEXT:    adcl %eax, %esi
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl $0, %ecx
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    imull %eax, %ebx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    addl %ebx, %edx
-; X32-NEXT:    imull {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    addl %edx, %esi
-; X32-NEXT:    movl %esi, %ebx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    movl %eax, %esi
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    imull %edi, %esi
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %eax, %ebp
-; X32-NEXT:    addl %esi, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    imull %ecx, %esi
-; X32-NEXT:    addl %edx, %esi
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    adcl %ebx, %esi
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %ebx
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    movl %eax, %edi
-; X32-NEXT:    addl %ebx, %edi
-; X32-NEXT:    adcl $0, %esi
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %ebx
-; X32-NEXT:    addl %edi, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %esi, %ebx
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    addl %ebx, %eax
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
-; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    addl %ebp, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X32-NEXT:    movl 124(%edi), %ecx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    imull %eax, %ecx
-; X32-NEXT:    movl 120(%edi), %esi
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    addl %ecx, %edx
-; X32-NEXT:    imull {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    addl %edx, %esi
-; X32-NEXT:    movl 112(%edi), %ebp
-; X32-NEXT:    movl 116(%edi), %ebx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    movl %eax, %edi
-; X32-NEXT:    imull %ebx, %edi
-; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    mull %ebp
-; X32-NEXT:    addl %edi, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    imull %ebp, %ecx
-; X32-NEXT:    addl %edx, %ecx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %esi, %ecx
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %edi
-; X32-NEXT:    movl %eax, %ebx
-; X32-NEXT:    addl %esi, %ebx
-; X32-NEXT:    adcl $0, %edi
-; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    movl %eax, %ebp
-; X32-NEXT:    addl %ebx, %ebp
-; X32-NEXT:    adcl %edi, %ecx
-; X32-NEXT:    setb %bl
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    addl %ecx, %eax
-; X32-NEXT:    movzbl %bl, %ecx
-; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %edi
-; X32-NEXT:    movl %eax, %ebx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    adcl $0, %edi
-; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %ebp
-; X32-NEXT:    addl %ebx, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edi, %ebp
-; X32-NEXT:    setb %bl
-; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    addl %ebp, %eax
-; X32-NEXT:    movzbl %bl, %edi
-; X32-NEXT:    adcl %edi, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %ecx
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edx, %esi
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    movl %edx, %ebx
-; X32-NEXT:    movl %eax, %ebp
-; X32-NEXT:    addl %esi, %ebp
-; X32-NEXT:    adcl $0, %ebx
-; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %edi
-; X32-NEXT:    addl %ebp, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %ebx, %edi
-; X32-NEXT:    setb %bl
-; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    movl %ecx, %ebp
-; X32-NEXT:    mull %esi
-; X32-NEXT:    addl %edi, %eax
-; X32-NEXT:    movzbl %bl, %edi
-; X32-NEXT:    adcl %edi, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %edi
-; X32-NEXT:    adcl %edx, %esi
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    movl %eax, %edi
-; X32-NEXT:    addl %ecx, %edi
-; X32-NEXT:    adcl $0, %esi
-; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    addl %edi, %eax
-; X32-NEXT:    movl %eax, %edi
-; X32-NEXT:    adcl %esi, %ecx
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    movl %ebx, %ebp
-; X32-NEXT:    addl %ecx, %eax
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
-; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %ecx
-; X32-NEXT:    adcl %edx, %esi
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    addl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl $0, %ecx
-; X32-NEXT:    adcl $0, %esi
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %edi
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    movl %eax, %ecx
-; X32-NEXT:    addl %edi, %ecx
-; X32-NEXT:    adcl $0, %esi
-; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    movl %ebp, %edi
-; X32-NEXT:    mull %ebp
-; X32-NEXT:    movl %edx, %ebx
-; X32-NEXT:    movl %eax, %ebp
-; X32-NEXT:    addl %ecx, %ebp
-; X32-NEXT:    adcl %esi, %ebx
-; X32-NEXT:    setb %cl
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %edi
-; X32-NEXT:    addl %ebx, %eax
-; X32-NEXT:    movzbl %cl, %ecx
-; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %esi
-; X32-NEXT:    adcl %edx, %ecx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    addl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
-; X32-NEXT:    adcl %eax, %esi
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl $0, %ecx
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    imull %eax, %edi
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    addl %edi, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    imull %ebp, %esi
-; X32-NEXT:    addl %edx, %esi
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    movl %eax, %edi
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    imull %ebx, %edi
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %eax, %ecx
-; X32-NEXT:    addl %edi, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    imull %esi, %edi
-; X32-NEXT:    addl %edx, %edi
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %edi
-; X32-NEXT:    movl %eax, %ebx
-; X32-NEXT:    addl %esi, %ebx
-; X32-NEXT:    adcl $0, %edi
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %ebp
-; X32-NEXT:    movl %ebp, %esi
-; X32-NEXT:    movl %edx, %ebp
-; X32-NEXT:    addl %ebx, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edi, %ebp
-; X32-NEXT:    setb %cl
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    addl %ebp, %eax
-; X32-NEXT:    movzbl %cl, %ecx
-; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    imull %eax, %edi
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %eax, %ebp
-; X32-NEXT:    addl %edi, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    imull %ebx, %ecx
-; X32-NEXT:    addl %edx, %ecx
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    movl %eax, %ecx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    imull %esi, %ecx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    mull %edi
-; X32-NEXT:    addl %ecx, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    imull %edi, %ecx
-; X32-NEXT:    addl %edx, %ecx
-; X32-NEXT:    addl %ebp, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    movl %edi, %ebp
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %edi
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    movl %eax, %ecx
-; X32-NEXT:    addl %edi, %ecx
-; X32-NEXT:    adcl $0, %esi
-; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    movl %edx, %edi
-; X32-NEXT:    movl %eax, %ebp
-; X32-NEXT:    addl %ecx, %ebp
-; X32-NEXT:    adcl %esi, %edi
-; X32-NEXT:    setb %cl
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    movl %eax, %edx
-; X32-NEXT:    addl %edi, %edx
-; X32-NEXT:    movzbl %cl, %ecx
-; X32-NEXT:    adcl %ecx, %esi
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    movl %ecx, %ebp
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    movl %esi, (%ecx)
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    movl %esi, 4(%ecx)
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    movl %esi, 8(%ecx)
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    movl %esi, 12(%ecx)
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    movl %esi, 16(%ecx)
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    movl %esi, 20(%ecx)
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    movl %esi, 24(%ecx)
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    movl %esi, 28(%ecx)
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    movl %esi, 32(%ecx)
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    movl %esi, 36(%ecx)
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    movl %esi, 40(%ecx)
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    movl %esi, 44(%ecx)
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    movl %esi, 48(%ecx)
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    movl %esi, 52(%ecx)
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    movl %esi, 56(%ecx)
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    movl %esi, 60(%ecx)
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    movl %esi, 64(%ecx)
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    movl %esi, 68(%ecx)
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    movl %esi, 72(%ecx)
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    movl %esi, 76(%ecx)
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    movl %esi, 80(%ecx)
-; X32-NEXT:    movl %ebp, 84(%ecx)
-; X32-NEXT:    movl %edi, 88(%ecx)
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    movl %esi, 92(%ecx)
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    movl %esi, 96(%ecx)
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    movl %esi, 100(%ecx)
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    movl %esi, 104(%ecx)
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    movl %esi, 108(%ecx)
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    movl %esi, 112(%ecx)
-; X32-NEXT:    movl %ebx, 116(%ecx)
-; X32-NEXT:    movl %edx, 120(%ecx)
-; X32-NEXT:    movl %eax, 124(%ecx)
-; X32-NEXT:    addl $1000, %esp # imm = 0x3E8
-; X32-NEXT:    popl %esi
-; X32-NEXT:    popl %edi
-; X32-NEXT:    popl %ebx
-; X32-NEXT:    popl %ebp
-; X32-NEXT:    retl
-;
-; X64-LABEL: test_1024:
-; X64:       # %bb.0:
-; X64-NEXT:    pushq %rbp
-; X64-NEXT:    pushq %r15
-; X64-NEXT:    pushq %r14
-; X64-NEXT:    pushq %r13
-; X64-NEXT:    pushq %r12
-; X64-NEXT:    pushq %rbx
-; X64-NEXT:    subq $352, %rsp # imm = 0x160
-; X64-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    movq 48(%rdi), %r9
-; X64-NEXT:    movq %r9, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    movq 40(%rdi), %rbp
-; X64-NEXT:    movq %rbp, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    movq 32(%rdi), %rax
-; X64-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    movq %rdi, %r10
-; X64-NEXT:    xorl %r8d, %r8d
-; X64-NEXT:    mulq %r8
-; X64-NEXT:    movq %rdx, %rdi
-; X64-NEXT:    movq %rax, %rcx
-; X64-NEXT:    movq %rbp, %rax
-; X64-NEXT:    mulq %r8
-; X64-NEXT:    movq %rax, %rbx
-; X64-NEXT:    addq %rdi, %rbx
-; X64-NEXT:    movq %rdx, %rbp
-; X64-NEXT:    adcq $0, %rbp
-; X64-NEXT:    addq %rcx, %rbx
-; X64-NEXT:    movq %rbx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    movq %rcx, %r11
-; X64-NEXT:    adcq %rdi, %rbp
-; X64-NEXT:    setb %bl
-; X64-NEXT:    movzbl %bl, %ebx
-; X64-NEXT:    addq %rax, %rbp
-; X64-NEXT:    adcq %rdx, %rbx
-; X64-NEXT:    movq %r9, %rax
-; X64-NEXT:    mulq %r8
-; X64-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    addq %rax, %rcx
-; X64-NEXT:    movq %rdi, %r14
-; X64-NEXT:    movq %rdi, %r8
-; X64-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    adcq %rdx, %r14
-; X64-NEXT:    addq %rbp, %rcx
-; X64-NEXT:    movq %rcx, %r12
-; X64-NEXT:    movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    adcq %rbx, %r14
-; X64-NEXT:    movq %r14, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    movq (%rsi), %rax
-; X64-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    xorl %ebp, %ebp
-; X64-NEXT:    mulq %rbp
-; X64-NEXT:    movq %rax, %rdi
-; X64-NEXT:    movq %rdx, %rcx
-; X64-NEXT:    movq 8(%rsi), %rax
-; X64-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    mulq %rbp
-; X64-NEXT:    xorl %r9d, %r9d
-; X64-NEXT:    movq %rax, %r15
-; X64-NEXT:    addq %rcx, %r15
-; X64-NEXT:    movq %rdx, %rbp
-; X64-NEXT:    adcq $0, %rbp
-; X64-NEXT:    addq %rdi, %r15
-; X64-NEXT:    adcq %rcx, %rbp
-; X64-NEXT:    movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    setb %bl
-; X64-NEXT:    addq %rax, %rbp
-; X64-NEXT:    movzbl %bl, %ebx
-; X64-NEXT:    adcq %rdx, %rbx
-; X64-NEXT:    movq 16(%rsi), %rax
-; X64-NEXT:    movq %rsi, %r13
-; X64-NEXT:    movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    mulq %r9
-; X64-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    movq %rdi, %r9
-; X64-NEXT:    addq %rax, %r9
-; X64-NEXT:    movq %rcx, %rax
-; X64-NEXT:    adcq %rdx, %rax
-; X64-NEXT:    addq %rbp, %r9
-; X64-NEXT:    adcq %rbx, %rax
-; X64-NEXT:    movq %rax, %rbp
-; X64-NEXT:    movq %r11, %rax
-; X64-NEXT:    movq %r11, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    addq %rdi, %rax
-; X64-NEXT:    adcq %rcx, %r8
-; X64-NEXT:    movq %r8, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    movq (%r10), %rax
-; X64-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    xorl %r8d, %r8d
-; X64-NEXT:    mulq %r8
-; X64-NEXT:    movq %rdx, %rsi
-; X64-NEXT:    movq %rax, %rbx
-; X64-NEXT:    addq %rdi, %rax
-; X64-NEXT:    movq %rdx, %rax
-; X64-NEXT:    adcq %rcx, %rax
-; X64-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    movq 32(%r13), %rax
-; X64-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    mulq %r8
-; X64-NEXT:    xorl %ecx, %ecx
-; X64-NEXT:    movq %rax, %r13
+; X32-NEXT:    movl %eax, 76(%ecx)
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    movl %eax, 80(%ecx)
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    movl %eax, 84(%ecx)
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    movl %eax, 88(%ecx)
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    movl %eax, 92(%ecx)
+; X32-NEXT:    movl %ebp, 96(%ecx)
+; X32-NEXT:    movl %ebx, 100(%ecx)
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    movl %eax, 104(%ecx)
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    movl %eax, 108(%ecx)
+; X32-NEXT:    movl %edi, 112(%ecx)
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    movl %eax, 116(%ecx)
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    movl %eax, 120(%ecx)
+; X32-NEXT:    movl %edx, 124(%ecx)
+; X32-NEXT:    addl $400, %esp # imm = 0x190
+; X32-NEXT:    popl %esi
+; X32-NEXT:    popl %edi
+; X32-NEXT:    popl %ebx
+; X32-NEXT:    popl %ebp
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_1024:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rbp
+; X64-NEXT:    pushq %r15
+; X64-NEXT:    pushq %r14
+; X64-NEXT:    pushq %r13
+; X64-NEXT:    pushq %r12
+; X64-NEXT:    pushq %rbx
+; X64-NEXT:    subq $240, %rsp
 ; X64-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    movq %rbx, %rax
-; X64-NEXT:    movq %rbx, %r8
-; X64-NEXT:    addq %r13, %rax
-; X64-NEXT:    movq %rsi, %rax
-; X64-NEXT:    adcq %rdx, %rax
-; X64-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    movq %r11, %rax
-; X64-NEXT:    addq %rdi, %rax
-; X64-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    movq %rdi, %r11
 ; X64-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
-; X64-NEXT:    adcq %r15, %rax
-; X64-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    movq %r12, %rax
-; X64-NEXT:    adcq %r9, %rax
-; X64-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    adcq %rbp, %r14
-; X64-NEXT:    movq %r14, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    movq %rbp, %rdi
-; X64-NEXT:    movq 8(%r10), %rax
-; X64-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    movq %r10, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    mulq %rcx
+; X64-NEXT:    movq 40(%rdi), %r15
+; X64-NEXT:    movq 32(%rdi), %r9
+; X64-NEXT:    movq 56(%rdi), %r8
+; X64-NEXT:    movq 48(%rdi), %rbx
+; X64-NEXT:    movq %rbx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    movq (%rsi), %rdi
+; X64-NEXT:    movq 8(%rsi), %r11
+; X64-NEXT:    movq %rsi, %r13
+; X64-NEXT:    movq %rbx, %rax
+; X64-NEXT:    mulq %rdi
+; X64-NEXT:    movq %rdx, %rcx
 ; X64-NEXT:    movq %rax, %r12
-; X64-NEXT:    addq %rsi, %r12
-; X64-NEXT:    movq %rdx, %rbp
-; X64-NEXT:    adcq $0, %rbp
-; X64-NEXT:    addq %rbx, %r12
-; X64-NEXT:    adcq %rsi, %rbp
-; X64-NEXT:    movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    setb %bl
-; X64-NEXT:    addq %rax, %rbp
-; X64-NEXT:    movzbl %bl, %ebx
-; X64-NEXT:    adcq %rdx, %rbx
-; X64-NEXT:    movq 16(%r10), %rax
-; X64-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    mulq %rcx
-; X64-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    movq %r8, %rcx
-; X64-NEXT:    addq %rax, %rcx
-; X64-NEXT:    movq %rsi, %r10
-; X64-NEXT:    adcq %rdx, %r10
-; X64-NEXT:    addq %rbp, %rcx
-; X64-NEXT:    adcq %rbx, %r10
-; X64-NEXT:    movq %r8, %rdx
-; X64-NEXT:    movq %r8, %r14
-; X64-NEXT:    movq %r8, (%rsp) # 8-byte Spill
-; X64-NEXT:    addq %r11, %rdx
-; X64-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    movq %r12, %rsi
-; X64-NEXT:    adcq %r12, %r15
-; X64-NEXT:    movq %r15, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    adcq %rcx, %r9
-; X64-NEXT:    movq %r9, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    movq %rcx, %r8
-; X64-NEXT:    adcq %r10, %rdi
-; X64-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
-; X64-NEXT:    movq 40(%rdi), %rax
-; X64-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    xorl %r9d, %r9d
-; X64-NEXT:    mulq %r9
-; X64-NEXT:    movq %rax, %rcx
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r11 # 8-byte Reload
-; X64-NEXT:    addq %r11, %rcx
-; X64-NEXT:    movq %rdx, %rbp
-; X64-NEXT:    adcq $0, %rbp
-; X64-NEXT:    addq %r13, %rcx
-; X64-NEXT:    adcq %r11, %rbp
-; X64-NEXT:    setb %bl
-; X64-NEXT:    addq %rax, %rbp
-; X64-NEXT:    movzbl %bl, %ebx
-; X64-NEXT:    adcq %rdx, %rbx
-; X64-NEXT:    movq 48(%rdi), %rax
-; X64-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    mulq %r9
-; X64-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    movq %r13, %r12
-; X64-NEXT:    addq %rax, %r12
-; X64-NEXT:    movq %r11, %rdi
-; X64-NEXT:    adcq %rdx, %rdi
-; X64-NEXT:    addq %rbp, %r12
-; X64-NEXT:    adcq %rbx, %rdi
-; X64-NEXT:    movq %r13, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    addq %r13, %r14
-; X64-NEXT:    movq %r14, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    adcq %rcx, %rsi
-; X64-NEXT:    movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    adcq %r12, %r8
-; X64-NEXT:    movq %r8, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    adcq %rdi, %r10
-; X64-NEXT:    movq %r10, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Reload
-; X64-NEXT:    movq %r8, %rax
-; X64-NEXT:    addq %r13, %rax
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
-; X64-NEXT:    adcq %r11, %rax
-; X64-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; X64-NEXT:    movq %r8, %rax
-; X64-NEXT:    movq %r8, %r10
-; X64-NEXT:    addq %r13, %rax
-; X64-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Folded Reload
-; X64-NEXT:    movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %r12 # 8-byte Folded Reload
-; X64-NEXT:    movq %r12, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Folded Reload
-; X64-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
-; X64-NEXT:    movq %rcx, %rax
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r9 # 8-byte Reload
-; X64-NEXT:    mulq %r9
-; X64-NEXT:    movq %rax, %r14
-; X64-NEXT:    movq %rdx, %rsi
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
-; X64-NEXT:    movq 56(%rax), %r11
-; X64-NEXT:    movq %r11, %rax
-; X64-NEXT:    movq %r11, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    mulq %r9
-; X64-NEXT:    movq %rdx, %rbp
-; X64-NEXT:    movq %rax, %rbx
-; X64-NEXT:    addq %rsi, %rbx
-; X64-NEXT:    adcq $0, %rbp
-; X64-NEXT:    movq %rcx, %rax
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
-; X64-NEXT:    mulq %rdi
-; X64-NEXT:    movq %rdx, %rsi
-; X64-NEXT:    movq %rax, %r8
-; X64-NEXT:    addq %rbx, %r8
-; X64-NEXT:    adcq %rbp, %rsi
-; X64-NEXT:    setb %cl
-; X64-NEXT:    movq %r11, %rax
+; X64-NEXT:    movq %r8, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; X64-NEXT:    mulq %rdi
-; X64-NEXT:    movq %rdi, %r13
-; X64-NEXT:    addq %rsi, %rax
-; X64-NEXT:    movzbl %cl, %ecx
-; X64-NEXT:    adcq %rcx, %rdx
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r15 # 8-byte Reload
-; X64-NEXT:    addq {{[-0-9]+}}(%r{{[sb]}}p), %r15 # 8-byte Folded Reload
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r12 # 8-byte Reload
-; X64-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %r12 # 8-byte Folded Reload
-; X64-NEXT:    addq %rax, %r15
-; X64-NEXT:    adcq %rdx, %r12
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
-; X64-NEXT:    movq %rcx, %rax
-; X64-NEXT:    mulq %r9
 ; X64-NEXT:    movq %rdx, %rsi
-; X64-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
-; X64-NEXT:    movq %rdi, %rax
-; X64-NEXT:    mulq %r9
-; X64-NEXT:    movq %rdx, %rbp
-; X64-NEXT:    movq %rax, %rbx
-; X64-NEXT:    addq %rsi, %rbx
-; X64-NEXT:    adcq $0, %rbp
-; X64-NEXT:    movq %rcx, %rax
-; X64-NEXT:    movq %rcx, %r11
-; X64-NEXT:    mulq %r13
-; X64-NEXT:    movq %rdx, %rcx
-; X64-NEXT:    addq %rbx, %rax
-; X64-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    adcq %rbp, %rcx
-; X64-NEXT:    setb %bl
-; X64-NEXT:    movq %rdi, %rax
-; X64-NEXT:    mulq %r13
-; X64-NEXT:    movq %rdx, %r13
-; X64-NEXT:    movq %rax, %rsi
-; X64-NEXT:    addq %rcx, %rsi
-; X64-NEXT:    movzbl %bl, %eax
-; X64-NEXT:    adcq %rax, %r13
-; X64-NEXT:    addq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Folded Reload
-; X64-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %r13 # 8-byte Folded Reload
-; X64-NEXT:    addq %r14, %rsi
-; X64-NEXT:    adcq %r8, %r13
-; X64-NEXT:    adcq $0, %r15
-; X64-NEXT:    adcq $0, %r12
-; X64-NEXT:    movq %r11, %rbx
-; X64-NEXT:    movq %r11, %rax
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rbp # 8-byte Reload
-; X64-NEXT:    mulq %rbp
-; X64-NEXT:    movq %rdx, %rcx
-; X64-NEXT:    movq %rax, %r14
-; X64-NEXT:    movq %rdi, %rax
-; X64-NEXT:    movq %rdi, %r11
-; X64-NEXT:    mulq %rbp
-; X64-NEXT:    movq %rdx, %rdi
 ; X64-NEXT:    movq %rax, %rbp
 ; X64-NEXT:    addq %rcx, %rbp
-; X64-NEXT:    adcq $0, %rdi
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
-; X64-NEXT:    movq 24(%rax), %r9
-; X64-NEXT:    movq %rbx, %rax
-; X64-NEXT:    mulq %r9
-; X64-NEXT:    movq %rdx, %rcx
-; X64-NEXT:    movq %rax, %r8
-; X64-NEXT:    addq %rbp, %r8
-; X64-NEXT:    adcq %rdi, %rcx
-; X64-NEXT:    setb %bl
-; X64-NEXT:    movq %r11, %rax
-; X64-NEXT:    mulq %r9
-; X64-NEXT:    addq %rcx, %rax
-; X64-NEXT:    movzbl %bl, %ecx
-; X64-NEXT:    adcq %rcx, %rdx
-; X64-NEXT:    movq %r10, %rbp
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r11 # 8-byte Reload
-; X64-NEXT:    addq %r11, %rbp
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rbx # 8-byte Reload
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r10 # 8-byte Reload
-; X64-NEXT:    adcq %r10, %rbx
-; X64-NEXT:    addq %rax, %rbp
-; X64-NEXT:    adcq %rdx, %rbx
-; X64-NEXT:    addq %rsi, %r14
-; X64-NEXT:    movq %r14, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    adcq %r13, %r8
-; X64-NEXT:    movq %r8, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    adcq $0, %rbp
-; X64-NEXT:    adcq $0, %rbx
-; X64-NEXT:    addq %r15, %rbp
-; X64-NEXT:    adcq %r12, %rbx
-; X64-NEXT:    setb %r15b
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
-; X64-NEXT:    movq %rcx, %rax
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
-; X64-NEXT:    mulq %rsi
-; X64-NEXT:    movq %rdx, %r14
-; X64-NEXT:    movq %rax, %r13
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r12 # 8-byte Reload
-; X64-NEXT:    movq %r12, %rax
-; X64-NEXT:    mulq %rsi
-; X64-NEXT:    movq %rdx, %rsi
-; X64-NEXT:    movq %rax, %rdi
-; X64-NEXT:    addq %r14, %rdi
 ; X64-NEXT:    adcq $0, %rsi
-; X64-NEXT:    movq %rcx, %rax
-; X64-NEXT:    mulq %r9
+; X64-NEXT:    movq %rbx, %rax
+; X64-NEXT:    mulq %r11
 ; X64-NEXT:    movq %rdx, %rcx
-; X64-NEXT:    movq %rax, %r14
-; X64-NEXT:    addq %rdi, %r14
+; X64-NEXT:    movq %rax, %r10
+; X64-NEXT:    addq %rbp, %r10
 ; X64-NEXT:    adcq %rsi, %rcx
-; X64-NEXT:    setb %sil
-; X64-NEXT:    movq %r12, %rax
-; X64-NEXT:    mulq %r9
-; X64-NEXT:    movq %r9, %r12
-; X64-NEXT:    addq %rcx, %rax
-; X64-NEXT:    movzbl %sil, %ecx
-; X64-NEXT:    adcq %rcx, %rdx
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
-; X64-NEXT:    addq %r11, %rcx
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r9 # 8-byte Reload
-; X64-NEXT:    adcq %r10, %r9
-; X64-NEXT:    addq %rax, %rcx
-; X64-NEXT:    adcq %rdx, %r9
-; X64-NEXT:    addq %rbp, %r13
-; X64-NEXT:    adcq %rbx, %r14
-; X64-NEXT:    movzbl %r15b, %eax
-; X64-NEXT:    adcq %rax, %rcx
-; X64-NEXT:    adcq $0, %r9
-; X64-NEXT:    addq {{[-0-9]+}}(%r{{[sb]}}p), %r13 # 8-byte Folded Reload
-; X64-NEXT:    movq %r13, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %r14 # 8-byte Folded Reload
-; X64-NEXT:    movq %r14, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Folded Reload
-; X64-NEXT:    movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %r9 # 8-byte Folded Reload
-; X64-NEXT:    movq %r9, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
-; X64-NEXT:    movq %rdi, %rax
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r9 # 8-byte Reload
-; X64-NEXT:    mulq %r9
-; X64-NEXT:    movq %rax, %r14
-; X64-NEXT:    movq %rdx, %rbx
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
-; X64-NEXT:    movq 24(%rax), %rcx
-; X64-NEXT:    movq %rcx, %rax
-; X64-NEXT:    movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    mulq %r9
-; X64-NEXT:    movq %rdx, %rsi
-; X64-NEXT:    movq %rax, %rbp
-; X64-NEXT:    addq %rbx, %rbp
-; X64-NEXT:    adcq $0, %rsi
-; X64-NEXT:    movq %rdi, %rax
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r11 # 8-byte Reload
-; X64-NEXT:    mulq %r11
-; X64-NEXT:    movq %rdx, %rbx
-; X64-NEXT:    movq %rax, %r15
-; X64-NEXT:    addq %rbp, %r15
-; X64-NEXT:    adcq %rsi, %rbx
-; X64-NEXT:    setb %sil
-; X64-NEXT:    movq %rcx, %rax
+; X64-NEXT:    setb %al
+; X64-NEXT:    movzbl %al, %esi
+; X64-NEXT:    movq %r8, %rax
 ; X64-NEXT:    mulq %r11
-; X64-NEXT:    addq %rbx, %rax
-; X64-NEXT:    movzbl %sil, %ecx
-; X64-NEXT:    adcq %rcx, %rdx
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Reload
-; X64-NEXT:    addq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Folded Reload
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r10 # 8-byte Reload
-; X64-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %r10 # 8-byte Folded Reload
-; X64-NEXT:    addq %rax, %r8
-; X64-NEXT:    adcq %rdx, %r10
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
-; X64-NEXT:    movq %rcx, %rax
-; X64-NEXT:    mulq %r9
-; X64-NEXT:    movq %rdx, %rdi
+; X64-NEXT:    movq %rdx, %r8
+; X64-NEXT:    movq %rax, %r14
+; X64-NEXT:    addq %rcx, %r14
+; X64-NEXT:    adcq %rsi, %r8
+; X64-NEXT:    movq %r9, %rax
+; X64-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    mulq %rdi
+; X64-NEXT:    movq %rdx, %rcx
 ; X64-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
-; X64-NEXT:    movq %rsi, %rax
-; X64-NEXT:    mulq %r9
+; X64-NEXT:    movq %r15, %rax
+; X64-NEXT:    mulq %rdi
 ; X64-NEXT:    movq %rdx, %rbp
-; X64-NEXT:    movq %rax, %rbx
-; X64-NEXT:    addq %rdi, %rbx
+; X64-NEXT:    movq %rax, %rdi
+; X64-NEXT:    addq %rcx, %rdi
 ; X64-NEXT:    adcq $0, %rbp
-; X64-NEXT:    movq %rcx, %rax
-; X64-NEXT:    movq %rcx, %r9
+; X64-NEXT:    movq %r9, %rax
+; X64-NEXT:    movq %r9, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    movq %r11, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; X64-NEXT:    mulq %r11
-; X64-NEXT:    movq %rdx, %rdi
-; X64-NEXT:    addq %rbx, %rax
+; X64-NEXT:    movq %rdx, %rbx
+; X64-NEXT:    addq %rdi, %rax
 ; X64-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    adcq %rbp, %rdi
-; X64-NEXT:    setb %cl
-; X64-NEXT:    movq %rsi, %rax
-; X64-NEXT:    movq %rsi, %rbp
+; X64-NEXT:    adcq %rbp, %rbx
+; X64-NEXT:    setb %sil
+; X64-NEXT:    movq %r15, %rbp
+; X64-NEXT:    movq %r15, %rax
 ; X64-NEXT:    mulq %r11
-; X64-NEXT:    movq %rdx, %rsi
-; X64-NEXT:    movq %rax, %rbx
-; X64-NEXT:    addq %rdi, %rbx
-; X64-NEXT:    movzbl %cl, %eax
-; X64-NEXT:    adcq %rax, %rsi
-; X64-NEXT:    addq {{[-0-9]+}}(%r{{[sb]}}p), %rbx # 8-byte Folded Reload
-; X64-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Folded Reload
-; X64-NEXT:    addq %r14, %rbx
-; X64-NEXT:    adcq %r15, %rsi
+; X64-NEXT:    movq %rdx, %r15
+; X64-NEXT:    movq %rax, %rcx
+; X64-NEXT:    addq %rbx, %rcx
+; X64-NEXT:    movzbl %sil, %eax
+; X64-NEXT:    adcq %rax, %r15
+; X64-NEXT:    addq %r12, %rcx
+; X64-NEXT:    adcq %r10, %r15
+; X64-NEXT:    adcq $0, %r14
 ; X64-NEXT:    adcq $0, %r8
-; X64-NEXT:    adcq $0, %r10
+; X64-NEXT:    movq %r13, %rdi
+; X64-NEXT:    movq %r13, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    movq 16(%r13), %r10
 ; X64-NEXT:    movq %r9, %rax
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
-; X64-NEXT:    mulq %rdi
-; X64-NEXT:    movq %rdx, %rcx
-; X64-NEXT:    movq %rax, %r11
+; X64-NEXT:    mulq %r10
+; X64-NEXT:    movq %rdx, %r12
+; X64-NEXT:    movq %rax, %r13
 ; X64-NEXT:    movq %rbp, %rax
-; X64-NEXT:    movq %rbp, %r14
+; X64-NEXT:    movq %rbp, %r11
+; X64-NEXT:    movq %rbp, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    mulq %r10
+; X64-NEXT:    movq %rdx, %rbx
+; X64-NEXT:    movq %rax, %rsi
+; X64-NEXT:    addq %r12, %rsi
+; X64-NEXT:    adcq $0, %rbx
+; X64-NEXT:    movq 24(%rdi), %rdi
+; X64-NEXT:    movq %r9, %rax
+; X64-NEXT:    mulq %rdi
+; X64-NEXT:    movq %rdi, %r12
+; X64-NEXT:    movq %rdx, %rbp
+; X64-NEXT:    addq %rsi, %rax
+; X64-NEXT:    movq %rax, %rsi
+; X64-NEXT:    adcq %rbx, %rbp
+; X64-NEXT:    setb %r9b
+; X64-NEXT:    movq %r11, %rax
 ; X64-NEXT:    mulq %rdi
-; X64-NEXT:    movq %rdi, %r15
 ; X64-NEXT:    movq %rdx, %rdi
-; X64-NEXT:    movq %rax, %rbp
-; X64-NEXT:    addq %rcx, %rbp
-; X64-NEXT:    adcq $0, %rdi
-; X64-NEXT:    movq %r9, %rax
-; X64-NEXT:    mulq %r12
-; X64-NEXT:    movq %rdx, %rcx
-; X64-NEXT:    addq %rbp, %rax
-; X64-NEXT:    movq %rax, %r9
-; X64-NEXT:    adcq %rdi, %rcx
-; X64-NEXT:    setb %dil
-; X64-NEXT:    movq %r14, %rax
-; X64-NEXT:    mulq %r12
-; X64-NEXT:    addq %rcx, %rax
-; X64-NEXT:    movzbl %dil, %ecx
-; X64-NEXT:    adcq %rcx, %rdx
-; X64-NEXT:    movq (%rsp), %rdi # 8-byte Reload
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r13 # 8-byte Reload
-; X64-NEXT:    addq %r13, %rdi
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rbp # 8-byte Reload
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r14 # 8-byte Reload
-; X64-NEXT:    adcq %r14, %rbp
-; X64-NEXT:    addq %rax, %rdi
-; X64-NEXT:    adcq %rdx, %rbp
-; X64-NEXT:    addq %rbx, %r11
-; X64-NEXT:    movq %r11, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    adcq %rsi, %r9
-; X64-NEXT:    movq %r9, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    movq %rax, %rbx
+; X64-NEXT:    addq %rbp, %rbx
+; X64-NEXT:    movzbl %r9b, %eax
+; X64-NEXT:    adcq %rax, %rdi
+; X64-NEXT:    addq %rcx, %r13
+; X64-NEXT:    movq %r13, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    adcq %r15, %rsi
+; X64-NEXT:    movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    adcq $0, %rbx
 ; X64-NEXT:    adcq $0, %rdi
-; X64-NEXT:    adcq $0, %rbp
-; X64-NEXT:    addq %r8, %rdi
-; X64-NEXT:    adcq %r10, %rbp
-; X64-NEXT:    setb %r10b
+; X64-NEXT:    addq %r14, %rbx
+; X64-NEXT:    adcq %r8, %rdi
+; X64-NEXT:    setb %r11b
 ; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
 ; X64-NEXT:    movq %rcx, %rax
-; X64-NEXT:    mulq %r15
-; X64-NEXT:    movq %rdx, %r8
-; X64-NEXT:    movq %rax, %r11
+; X64-NEXT:    mulq %r10
+; X64-NEXT:    movq %rdx, %r14
+; X64-NEXT:    movq %rax, %r15
 ; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r9 # 8-byte Reload
 ; X64-NEXT:    movq %r9, %rax
-; X64-NEXT:    mulq %r15
+; X64-NEXT:    mulq %r10
 ; X64-NEXT:    movq %rdx, %rsi
-; X64-NEXT:    movq %rax, %rbx
-; X64-NEXT:    addq %r8, %rbx
+; X64-NEXT:    movq %rax, %rbp
+; X64-NEXT:    addq %r14, %rbp
 ; X64-NEXT:    adcq $0, %rsi
 ; X64-NEXT:    movq %rcx, %rax
 ; X64-NEXT:    movq %r12, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; X64-NEXT:    mulq %r12
 ; X64-NEXT:    movq %rdx, %rcx
-; X64-NEXT:    movq %rax, %r8
-; X64-NEXT:    addq %rbx, %r8
+; X64-NEXT:    addq %rbp, %rax
+; X64-NEXT:    movq %rax, %rbp
 ; X64-NEXT:    adcq %rsi, %rcx
-; X64-NEXT:    setb %bl
+; X64-NEXT:    setb %sil
 ; X64-NEXT:    movq %r9, %rax
 ; X64-NEXT:    mulq %r12
 ; X64-NEXT:    addq %rcx, %rax
-; X64-NEXT:    movzbl %bl, %ecx
+; X64-NEXT:    movzbl %sil, %ecx
 ; X64-NEXT:    adcq %rcx, %rdx
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r9 # 8-byte Reload
-; X64-NEXT:    movq %r9, %r15
-; X64-NEXT:    addq %r13, %r15
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r12 # 8-byte Reload
-; X64-NEXT:    movq %r12, %r13
-; X64-NEXT:    adcq %r14, %r13
-; X64-NEXT:    addq %rax, %r15
-; X64-NEXT:    adcq %rdx, %r13
-; X64-NEXT:    addq %rdi, %r11
-; X64-NEXT:    adcq %rbp, %r8
-; X64-NEXT:    movzbl %r10b, %eax
-; X64-NEXT:    adcq %rax, %r15
-; X64-NEXT:    adcq $0, %r13
-; X64-NEXT:    addq {{[-0-9]+}}(%r{{[sb]}}p), %r11 # 8-byte Folded Reload
-; X64-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Folded Reload
-; X64-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %r15 # 8-byte Folded Reload
-; X64-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %r13 # 8-byte Folded Reload
-; X64-NEXT:    addq {{[-0-9]+}}(%r{{[sb]}}p), %r11 # 8-byte Folded Reload
-; X64-NEXT:    movq %r11, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Folded Reload
-; X64-NEXT:    movq %r8, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %r15 # 8-byte Folded Reload
-; X64-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %r13 # 8-byte Folded Reload
-; X64-NEXT:    adcq $0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
-; X64-NEXT:    adcq $0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
-; X64-NEXT:    adcq $0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
-; X64-NEXT:    adcq $0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
+; X64-NEXT:    addq %rbx, %r15
+; X64-NEXT:    movq %r15, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    adcq %rdi, %rbp
+; X64-NEXT:    movq %rbp, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    movzbl %r11b, %ecx
+; X64-NEXT:    adcq %rcx, %rax
+; X64-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    adcq $0, %rdx
+; X64-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
+; X64-NEXT:    movq 16(%rsi), %rcx
+; X64-NEXT:    movq %rcx, %rax
+; X64-NEXT:    movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Reload
+; X64-NEXT:    mulq %r8
+; X64-NEXT:    movq %rax, %r14
+; X64-NEXT:    movq %rdx, %r9
+; X64-NEXT:    movq 24(%rsi), %rbx
+; X64-NEXT:    movq %rbx, %rax
+; X64-NEXT:    movq %rbx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    mulq %r8
+; X64-NEXT:    movq %rdx, %rdi
+; X64-NEXT:    movq %rax, %rbp
+; X64-NEXT:    addq %r9, %rbp
+; X64-NEXT:    adcq $0, %rdi
+; X64-NEXT:    movq %rcx, %rax
+; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r13 # 8-byte Reload
+; X64-NEXT:    mulq %r13
+; X64-NEXT:    movq %rdx, %rcx
+; X64-NEXT:    movq %rax, %r15
+; X64-NEXT:    addq %rbp, %r15
+; X64-NEXT:    adcq %rdi, %rcx
+; X64-NEXT:    setb %dil
+; X64-NEXT:    movq %rbx, %rax
+; X64-NEXT:    mulq %r13
+; X64-NEXT:    movq %rdx, %r11
+; X64-NEXT:    movq %rax, %r12
+; X64-NEXT:    addq %rcx, %r12
+; X64-NEXT:    movzbl %dil, %eax
+; X64-NEXT:    adcq %rax, %r11
+; X64-NEXT:    movq (%rsi), %rdi
 ; X64-NEXT:    movq %rdi, %rax
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
-; X64-NEXT:    mulq %rcx
+; X64-NEXT:    mulq %r8
+; X64-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    movq %rdx, %rcx
+; X64-NEXT:    movq 8(%rsi), %r9
+; X64-NEXT:    movq %r9, %rax
+; X64-NEXT:    mulq %r8
+; X64-NEXT:    movq %rdx, %rbp
+; X64-NEXT:    movq %rax, %rbx
+; X64-NEXT:    addq %rcx, %rbx
+; X64-NEXT:    adcq $0, %rbp
+; X64-NEXT:    movq %rdi, %rax
+; X64-NEXT:    mulq %r13
 ; X64-NEXT:    movq %rdx, %rsi
-; X64-NEXT:    movq %rax, %r14
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r10 # 8-byte Reload
-; X64-NEXT:    movq %r10, %rax
-; X64-NEXT:    mulq %rcx
-; X64-NEXT:    movq %rcx, %rbp
-; X64-NEXT:    movq %rdx, %rbx
+; X64-NEXT:    addq %rbx, %rax
+; X64-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    adcq %rbp, %rsi
+; X64-NEXT:    setb %bl
+; X64-NEXT:    movq %r9, %rax
+; X64-NEXT:    mulq %r13
+; X64-NEXT:    movq %rdx, %r13
 ; X64-NEXT:    movq %rax, %rcx
 ; X64-NEXT:    addq %rsi, %rcx
-; X64-NEXT:    adcq $0, %rbx
+; X64-NEXT:    movzbl %bl, %eax
+; X64-NEXT:    adcq %rax, %r13
+; X64-NEXT:    addq %r14, %rcx
+; X64-NEXT:    adcq %r15, %r13
+; X64-NEXT:    adcq $0, %r12
+; X64-NEXT:    adcq $0, %r11
+; X64-NEXT:    movq %rdi, %rbp
+; X64-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; X64-NEXT:    movq %rdi, %rax
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r11 # 8-byte Reload
-; X64-NEXT:    mulq %r11
+; X64-NEXT:    mulq %r10
 ; X64-NEXT:    movq %rdx, %rsi
-; X64-NEXT:    movq %rax, %r8
-; X64-NEXT:    addq %rcx, %r8
-; X64-NEXT:    adcq %rbx, %rsi
-; X64-NEXT:    setb %cl
-; X64-NEXT:    movq %r10, %rax
-; X64-NEXT:    mulq %r11
-; X64-NEXT:    addq %rsi, %rax
-; X64-NEXT:    movzbl %cl, %ecx
+; X64-NEXT:    movq %rax, %r14
+; X64-NEXT:    movq %r9, %rax
+; X64-NEXT:    movq %r9, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    mulq %r10
+; X64-NEXT:    movq %rdx, %rbx
+; X64-NEXT:    movq %rax, %rdi
+; X64-NEXT:    addq %rsi, %rdi
+; X64-NEXT:    adcq $0, %rbx
+; X64-NEXT:    movq %rbp, %rax
+; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
+; X64-NEXT:    mulq %rsi
+; X64-NEXT:    movq %rdx, %rbp
+; X64-NEXT:    addq %rdi, %rax
+; X64-NEXT:    movq %rax, %rdi
+; X64-NEXT:    adcq %rbx, %rbp
+; X64-NEXT:    setb %r15b
+; X64-NEXT:    movq %r9, %rax
+; X64-NEXT:    mulq %rsi
+; X64-NEXT:    movq %rdx, %r8
+; X64-NEXT:    movq %rax, %rbx
+; X64-NEXT:    addq %rbp, %rbx
+; X64-NEXT:    movzbl %r15b, %eax
+; X64-NEXT:    adcq %rax, %r8
+; X64-NEXT:    addq %rcx, %r14
+; X64-NEXT:    movq %r14, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    adcq %r13, %rdi
+; X64-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    adcq $0, %rbx
+; X64-NEXT:    adcq $0, %r8
+; X64-NEXT:    addq %r12, %rbx
+; X64-NEXT:    adcq %r11, %r8
+; X64-NEXT:    setb %r14b
+; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
+; X64-NEXT:    movq %rsi, %rax
+; X64-NEXT:    movq %r10, (%rsp) # 8-byte Spill
+; X64-NEXT:    mulq %r10
+; X64-NEXT:    movq %rdx, %rcx
+; X64-NEXT:    movq %rax, %r11
+; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r9 # 8-byte Reload
+; X64-NEXT:    movq %r9, %rax
+; X64-NEXT:    mulq %r10
+; X64-NEXT:    movq %rdx, %rdi
+; X64-NEXT:    movq %rax, %rbp
+; X64-NEXT:    addq %rcx, %rbp
+; X64-NEXT:    adcq $0, %rdi
+; X64-NEXT:    movq %rsi, %rax
+; X64-NEXT:    movq %rsi, %r15
+; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
+; X64-NEXT:    mulq %rsi
+; X64-NEXT:    movq %rdx, %rcx
+; X64-NEXT:    movq %rax, %r10
+; X64-NEXT:    addq %rbp, %r10
+; X64-NEXT:    adcq %rdi, %rcx
+; X64-NEXT:    setb %dil
+; X64-NEXT:    movq %r9, %rax
+; X64-NEXT:    mulq %rsi
+; X64-NEXT:    addq %rcx, %rax
+; X64-NEXT:    movzbl %dil, %ecx
 ; X64-NEXT:    adcq %rcx, %rdx
-; X64-NEXT:    addq {{[-0-9]+}}(%r{{[sb]}}p), %r9 # 8-byte Folded Reload
-; X64-NEXT:    movq %r12, %r10
+; X64-NEXT:    addq %rbx, %r11
+; X64-NEXT:    adcq %r8, %r10
+; X64-NEXT:    movzbl %r14b, %ecx
+; X64-NEXT:    adcq %rcx, %rax
+; X64-NEXT:    adcq $0, %rdx
+; X64-NEXT:    addq {{[-0-9]+}}(%r{{[sb]}}p), %r11 # 8-byte Folded Reload
+; X64-NEXT:    movq %r11, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; X64-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %r10 # 8-byte Folded Reload
-; X64-NEXT:    addq %rax, %r9
-; X64-NEXT:    adcq %rdx, %r10
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
-; X64-NEXT:    movq %rcx, %rax
-; X64-NEXT:    mulq %rbp
-; X64-NEXT:    movq %rdx, %r12
+; X64-NEXT:    movq %r10, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Folded Reload
 ; X64-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
-; X64-NEXT:    movq %rdi, %rax
-; X64-NEXT:    mulq %rbp
-; X64-NEXT:    movq %rdx, %rsi
-; X64-NEXT:    movq %rax, %rbx
-; X64-NEXT:    addq %r12, %rbx
-; X64-NEXT:    adcq $0, %rsi
-; X64-NEXT:    movq %rcx, %rax
-; X64-NEXT:    movq %rcx, %r12
-; X64-NEXT:    mulq %r11
+; X64-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Folded Reload
+; X64-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    adcq $0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
+; X64-NEXT:    adcq $0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
+; X64-NEXT:    adcq $0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
+; X64-NEXT:    adcq $0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
+; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
+; X64-NEXT:    movq 32(%rsi), %rdi
+; X64-NEXT:    movq %r15, %rax
+; X64-NEXT:    mulq %rdi
 ; X64-NEXT:    movq %rdx, %rcx
-; X64-NEXT:    addq %rbx, %rax
 ; X64-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    adcq %rsi, %rcx
+; X64-NEXT:    movq %r9, %rbx
+; X64-NEXT:    movq %r9, %rax
+; X64-NEXT:    mulq %rdi
+; X64-NEXT:    movq %rdi, %r11
+; X64-NEXT:    movq %rdx, %rdi
+; X64-NEXT:    movq %rax, %rbp
+; X64-NEXT:    addq %rcx, %rbp
+; X64-NEXT:    adcq $0, %rdi
+; X64-NEXT:    movq 40(%rsi), %rcx
+; X64-NEXT:    movq %rsi, %r13
+; X64-NEXT:    movq %r15, %rax
+; X64-NEXT:    mulq %rcx
+; X64-NEXT:    movq %rcx, %r8
+; X64-NEXT:    movq %rdx, %rcx
+; X64-NEXT:    movq %rax, %r9
+; X64-NEXT:    addq %rbp, %r9
+; X64-NEXT:    adcq %rdi, %rcx
 ; X64-NEXT:    setb %sil
-; X64-NEXT:    movq %rdi, %rax
-; X64-NEXT:    movq %rdi, %rbp
+; X64-NEXT:    movq %rbx, %rax
+; X64-NEXT:    mulq %r8
+; X64-NEXT:    movq %rdx, %r10
+; X64-NEXT:    movq %rax, %r14
+; X64-NEXT:    addq %rcx, %r14
+; X64-NEXT:    movzbl %sil, %eax
+; X64-NEXT:    adcq %rax, %r10
+; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r12 # 8-byte Reload
+; X64-NEXT:    movq %r12, %rax
+; X64-NEXT:    movq %r11, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    mulq %r11
+; X64-NEXT:    movq %rdx, %rcx
+; X64-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r15 # 8-byte Reload
+; X64-NEXT:    movq %r15, %rax
 ; X64-NEXT:    mulq %r11
 ; X64-NEXT:    movq %rdx, %rdi
-; X64-NEXT:    movq %rax, %rbx
-; X64-NEXT:    addq %rcx, %rbx
+; X64-NEXT:    movq %rax, %rsi
+; X64-NEXT:    addq %rcx, %rsi
+; X64-NEXT:    adcq $0, %rdi
+; X64-NEXT:    movq %r12, %rax
+; X64-NEXT:    movq %r8, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    mulq %r8
+; X64-NEXT:    movq %rdx, %rbp
+; X64-NEXT:    addq %rsi, %rax
+; X64-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    adcq %rdi, %rbp
+; X64-NEXT:    setb %sil
+; X64-NEXT:    movq %r15, %rax
+; X64-NEXT:    mulq %r8
+; X64-NEXT:    movq %rdx, %r8
+; X64-NEXT:    movq %rax, %rcx
+; X64-NEXT:    addq %rbp, %rcx
 ; X64-NEXT:    movzbl %sil, %eax
-; X64-NEXT:    adcq %rax, %rdi
-; X64-NEXT:    addq {{[-0-9]+}}(%r{{[sb]}}p), %rbx # 8-byte Folded Reload
-; X64-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Folded Reload
-; X64-NEXT:    addq %r14, %rbx
-; X64-NEXT:    adcq %r8, %rdi
-; X64-NEXT:    adcq $0, %r9
+; X64-NEXT:    adcq %rax, %r8
+; X64-NEXT:    addq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Folded Reload
+; X64-NEXT:    adcq %r9, %r8
+; X64-NEXT:    adcq $0, %r14
 ; X64-NEXT:    adcq $0, %r10
-; X64-NEXT:    movq %r12, %r11
+; X64-NEXT:    movq 48(%r13), %r11
+; X64-NEXT:    movq %r12, %rdi
 ; X64-NEXT:    movq %r12, %rax
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
-; X64-NEXT:    mulq %rcx
-; X64-NEXT:    movq %rdx, %r14
-; X64-NEXT:    movq %rax, %r12
-; X64-NEXT:    movq %rbp, %rax
-; X64-NEXT:    movq %rbp, %r8
-; X64-NEXT:    mulq %rcx
+; X64-NEXT:    mulq %r11
+; X64-NEXT:    movq %rdx, %r12
+; X64-NEXT:    movq %rax, %r9
+; X64-NEXT:    movq %r15, %rax
+; X64-NEXT:    mulq %r11
 ; X64-NEXT:    movq %rdx, %rbp
-; X64-NEXT:    movq %rax, %rcx
-; X64-NEXT:    addq %r14, %rcx
+; X64-NEXT:    movq %rax, %rbx
+; X64-NEXT:    addq %r12, %rbx
 ; X64-NEXT:    adcq $0, %rbp
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
-; X64-NEXT:    movq 56(%rax), %rsi
-; X64-NEXT:    movq %r11, %rax
+; X64-NEXT:    movq 56(%r13), %rsi
+; X64-NEXT:    movq %rdi, %rax
 ; X64-NEXT:    mulq %rsi
-; X64-NEXT:    movq %rsi, %r11
-; X64-NEXT:    movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    movq %rdx, %rdi
+; X64-NEXT:    movq %rax, %r13
+; X64-NEXT:    addq %rbx, %r13
+; X64-NEXT:    adcq %rbp, %rdi
+; X64-NEXT:    setb %bl
+; X64-NEXT:    movq %r15, %rax
+; X64-NEXT:    mulq %rsi
+; X64-NEXT:    movq %rsi, %r15
 ; X64-NEXT:    movq %rdx, %rsi
-; X64-NEXT:    movq %rax, %r14
-; X64-NEXT:    addq %rcx, %r14
-; X64-NEXT:    adcq %rbp, %rsi
-; X64-NEXT:    setb %cl
-; X64-NEXT:    movq %r8, %rax
-; X64-NEXT:    mulq %r11
-; X64-NEXT:    addq %rsi, %rax
-; X64-NEXT:    movzbl %cl, %ecx
-; X64-NEXT:    adcq %rcx, %rdx
-; X64-NEXT:    movq (%rsp), %rcx # 8-byte Reload
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Reload
-; X64-NEXT:    addq %r8, %rcx
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r11 # 8-byte Reload
-; X64-NEXT:    adcq %r11, %rsi
-; X64-NEXT:    addq %rax, %rcx
-; X64-NEXT:    adcq %rdx, %rsi
-; X64-NEXT:    addq %rbx, %r12
-; X64-NEXT:    movq %r12, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    adcq %rdi, %r14
-; X64-NEXT:    adcq $0, %rcx
+; X64-NEXT:    movq %rax, %r12
+; X64-NEXT:    addq %rdi, %r12
+; X64-NEXT:    movzbl %bl, %eax
+; X64-NEXT:    adcq %rax, %rsi
+; X64-NEXT:    addq %rcx, %r9
+; X64-NEXT:    movq %r9, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    adcq %r8, %r13
+; X64-NEXT:    adcq $0, %r12
 ; X64-NEXT:    adcq $0, %rsi
-; X64-NEXT:    addq %r9, %rcx
+; X64-NEXT:    addq %r14, %r12
 ; X64-NEXT:    adcq %r10, %rsi
 ; X64-NEXT:    setb {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Folded Spill
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rbp # 8-byte Reload
-; X64-NEXT:    movq %rbp, %rax
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
-; X64-NEXT:    mulq %rdi
+; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
+; X64-NEXT:    movq %rcx, %rax
+; X64-NEXT:    movq %r11, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    mulq %r11
 ; X64-NEXT:    movq %rdx, %r9
-; X64-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    movq %rax, %r8
 ; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r10 # 8-byte Reload
 ; X64-NEXT:    movq %r10, %rax
-; X64-NEXT:    mulq %rdi
-; X64-NEXT:    movq %rdx, %r12
-; X64-NEXT:    movq %rax, %rbx
-; X64-NEXT:    addq %r9, %rbx
-; X64-NEXT:    adcq $0, %r12
-; X64-NEXT:    movq %rbp, %rax
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
-; X64-NEXT:    mulq %rdi
-; X64-NEXT:    movq %rdx, %r9
+; X64-NEXT:    mulq %r11
+; X64-NEXT:    movq %rdx, %rdi
 ; X64-NEXT:    movq %rax, %rbp
-; X64-NEXT:    addq %rbx, %rbp
-; X64-NEXT:    adcq %r12, %r9
-; X64-NEXT:    setb %bl
+; X64-NEXT:    addq %r9, %rbp
+; X64-NEXT:    adcq $0, %rdi
+; X64-NEXT:    movq %rcx, %rax
+; X64-NEXT:    movq %r15, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    mulq %r15
+; X64-NEXT:    movq %rdx, %rcx
+; X64-NEXT:    movq %rax, %r9
+; X64-NEXT:    addq %rbp, %r9
+; X64-NEXT:    adcq %rdi, %rcx
+; X64-NEXT:    setb %dil
 ; X64-NEXT:    movq %r10, %rax
-; X64-NEXT:    mulq %rdi
-; X64-NEXT:    addq %r9, %rax
-; X64-NEXT:    movzbl %bl, %edi
-; X64-NEXT:    adcq %rdi, %rdx
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r12 # 8-byte Reload
-; X64-NEXT:    addq %r8, %r12
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r10 # 8-byte Reload
-; X64-NEXT:    adcq %r11, %r10
-; X64-NEXT:    addq %rax, %r12
-; X64-NEXT:    adcq %rdx, %r10
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
-; X64-NEXT:    addq %rcx, %rdx
-; X64-NEXT:    adcq %rsi, %rbp
+; X64-NEXT:    mulq %r15
+; X64-NEXT:    movq %rdx, %r10
+; X64-NEXT:    movq %rax, %r14
+; X64-NEXT:    addq %rcx, %r14
+; X64-NEXT:    movzbl %dil, %eax
+; X64-NEXT:    adcq %rax, %r10
+; X64-NEXT:    addq %r12, %r8
+; X64-NEXT:    adcq %rsi, %r9
 ; X64-NEXT:    movzbl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 1-byte Folded Reload
-; X64-NEXT:    adcq %rax, %r12
+; X64-NEXT:    adcq %rax, %r14
 ; X64-NEXT:    adcq $0, %r10
-; X64-NEXT:    addq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Folded Reload
-; X64-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %rbp # 8-byte Folded Reload
-; X64-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %r12 # 8-byte Folded Reload
-; X64-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %r10 # 8-byte Folded Reload
 ; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
 ; X64-NEXT:    addq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
 ; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
 ; X64-NEXT:    adcq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
-; X64-NEXT:    adcq %r15, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
-; X64-NEXT:    adcq %r13, %r14
-; X64-NEXT:    movq %r14, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    adcq $0, %rdx
-; X64-NEXT:    adcq $0, %rbp
-; X64-NEXT:    adcq $0, %r12
+; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; X64-NEXT:    adcq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
+; X64-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %r13 # 8-byte Folded Reload
+; X64-NEXT:    movq %r13, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    adcq $0, %r8
+; X64-NEXT:    adcq $0, %r9
+; X64-NEXT:    adcq $0, %r14
 ; X64-NEXT:    adcq $0, %r10
-; X64-NEXT:    addq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Folded Reload
-; X64-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %rbp # 8-byte Folded Reload
-; X64-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %r12 # 8-byte Folded Reload
+; X64-NEXT:    addq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Folded Reload
+; X64-NEXT:    movq %r8, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %r9 # 8-byte Folded Reload
+; X64-NEXT:    movq %r9, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %r14 # 8-byte Folded Reload
 ; X64-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %r10 # 8-byte Folded Reload
 ; X64-NEXT:    setb {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Folded Spill
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
-; X64-NEXT:    movq %rcx, %rax
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r14 # 8-byte Reload
-; X64-NEXT:    mulq %r14
-; X64-NEXT:    movq %rdx, %r11
-; X64-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r15 # 8-byte Reload
-; X64-NEXT:    movq %r15, %rax
-; X64-NEXT:    mulq %r14
-; X64-NEXT:    movq %rdx, %rdi
-; X64-NEXT:    movq %rax, %rbx
-; X64-NEXT:    addq %r11, %rbx
-; X64-NEXT:    adcq $0, %rdi
-; X64-NEXT:    movq %rcx, %rax
+; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
+; X64-NEXT:    movq %rdi, %rax
 ; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
 ; X64-NEXT:    mulq %rsi
 ; X64-NEXT:    movq %rdx, %rcx
-; X64-NEXT:    movq %rax, %r9
-; X64-NEXT:    addq %rbx, %r9
-; X64-NEXT:    adcq %rdi, %rcx
-; X64-NEXT:    setb %bl
-; X64-NEXT:    movq %r15, %rax
-; X64-NEXT:    mulq %rsi
-; X64-NEXT:    movq %rsi, %r13
-; X64-NEXT:    addq %rcx, %rax
-; X64-NEXT:    movzbl %bl, %ecx
-; X64-NEXT:    adcq %rcx, %rdx
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Reload
-; X64-NEXT:    addq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Folded Reload
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r15 # 8-byte Reload
-; X64-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %r15 # 8-byte Folded Reload
-; X64-NEXT:    addq %rax, %r8
-; X64-NEXT:    adcq %rdx, %r15
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
-; X64-NEXT:    movq %rcx, %rax
-; X64-NEXT:    mulq %r14
-; X64-NEXT:    movq %rdx, %r11
 ; X64-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
-; X64-NEXT:    movq %rsi, %rax
-; X64-NEXT:    mulq %r14
-; X64-NEXT:    movq %rdx, %rdi
+; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rbp # 8-byte Reload
+; X64-NEXT:    movq %rbp, %rax
+; X64-NEXT:    mulq %rsi
+; X64-NEXT:    movq %rsi, %r8
+; X64-NEXT:    movq %rdx, %rsi
 ; X64-NEXT:    movq %rax, %rbx
-; X64-NEXT:    addq %r11, %rbx
-; X64-NEXT:    adcq $0, %rdi
-; X64-NEXT:    movq %rcx, %rax
-; X64-NEXT:    movq %rcx, %r14
-; X64-NEXT:    mulq %r13
-; X64-NEXT:    movq %rdx, %rcx
-; X64-NEXT:    addq %rbx, %rax
-; X64-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    adcq %rdi, %rcx
+; X64-NEXT:    addq %rcx, %rbx
+; X64-NEXT:    adcq $0, %rsi
+; X64-NEXT:    movq %rdi, %rax
+; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
+; X64-NEXT:    mulq %rdi
+; X64-NEXT:    movq %rdx, %rcx
+; X64-NEXT:    movq %rax, %r12
+; X64-NEXT:    addq %rbx, %r12
+; X64-NEXT:    adcq %rsi, %rcx
 ; X64-NEXT:    setb %bl
-; X64-NEXT:    movq %rsi, %rax
-; X64-NEXT:    mulq %r13
+; X64-NEXT:    movq %rbp, %rax
+; X64-NEXT:    mulq %rdi
+; X64-NEXT:    movq %rdi, %r9
 ; X64-NEXT:    movq %rdx, %r11
 ; X64-NEXT:    movq %rax, %r13
 ; X64-NEXT:    addq %rcx, %r13
 ; X64-NEXT:    movzbl %bl, %eax
 ; X64-NEXT:    adcq %rax, %r11
-; X64-NEXT:    addq {{[-0-9]+}}(%r{{[sb]}}p), %r13 # 8-byte Folded Reload
-; X64-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %r11 # 8-byte Folded Reload
-; X64-NEXT:    addq {{[-0-9]+}}(%r{{[sb]}}p), %r13 # 8-byte Folded Reload
-; X64-NEXT:    adcq %r9, %r11
-; X64-NEXT:    adcq $0, %r8
-; X64-NEXT:    movq %r8, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    adcq $0, %r15
-; X64-NEXT:    movq %r15, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    movq %r14, %rax
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
-; X64-NEXT:    mulq %rcx
+; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rbp # 8-byte Reload
+; X64-NEXT:    movq %rbp, %rax
+; X64-NEXT:    mulq %r8
+; X64-NEXT:    movq %rdx, %rcx
+; X64-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
+; X64-NEXT:    movq %rsi, %rax
+; X64-NEXT:    mulq %r8
 ; X64-NEXT:    movq %rdx, %rbx
-; X64-NEXT:    movq %rax, %r9
+; X64-NEXT:    movq %rax, %rdi
+; X64-NEXT:    addq %rcx, %rdi
+; X64-NEXT:    adcq $0, %rbx
+; X64-NEXT:    movq %rbp, %rax
+; X64-NEXT:    movq %rbp, %r8
+; X64-NEXT:    mulq %r9
+; X64-NEXT:    movq %rdx, %rbp
+; X64-NEXT:    addq %rdi, %rax
+; X64-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    adcq %rbx, %rbp
+; X64-NEXT:    setb %bl
 ; X64-NEXT:    movq %rsi, %rax
-; X64-NEXT:    movq %rsi, %r15
-; X64-NEXT:    mulq %rcx
-; X64-NEXT:    movq %rdx, %rsi
+; X64-NEXT:    mulq %r9
+; X64-NEXT:    movq %rdx, %r15
 ; X64-NEXT:    movq %rax, %rcx
-; X64-NEXT:    addq %rbx, %rcx
-; X64-NEXT:    adcq $0, %rsi
-; X64-NEXT:    movq %r14, %rax
+; X64-NEXT:    addq %rbp, %rcx
+; X64-NEXT:    movzbl %bl, %eax
+; X64-NEXT:    adcq %rax, %r15
+; X64-NEXT:    addq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Folded Reload
+; X64-NEXT:    adcq %r12, %r15
+; X64-NEXT:    adcq $0, %r13
+; X64-NEXT:    adcq $0, %r11
+; X64-NEXT:    movq %r8, %rbx
+; X64-NEXT:    movq %r8, %rax
+; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
+; X64-NEXT:    mulq %rdi
+; X64-NEXT:    movq %rdx, %r8
+; X64-NEXT:    movq %rax, %r12
+; X64-NEXT:    movq %rsi, %rax
+; X64-NEXT:    movq %rsi, %r9
+; X64-NEXT:    mulq %rdi
+; X64-NEXT:    movq %rdx, %rbp
+; X64-NEXT:    movq %rax, %rsi
+; X64-NEXT:    addq %r8, %rsi
+; X64-NEXT:    adcq $0, %rbp
+; X64-NEXT:    movq %rbx, %rax
 ; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
 ; X64-NEXT:    mulq %rdi
 ; X64-NEXT:    movq %rdx, %rbx
 ; X64-NEXT:    movq %rax, %r8
-; X64-NEXT:    addq %rcx, %r8
-; X64-NEXT:    adcq %rsi, %rbx
-; X64-NEXT:    setb %cl
-; X64-NEXT:    movq %r15, %rax
+; X64-NEXT:    addq %rsi, %r8
+; X64-NEXT:    adcq %rbp, %rbx
+; X64-NEXT:    setb %sil
+; X64-NEXT:    movq %r9, %rax
 ; X64-NEXT:    mulq %rdi
-; X64-NEXT:    addq %rbx, %rax
-; X64-NEXT:    movzbl %cl, %ecx
-; X64-NEXT:    adcq %rcx, %rdx
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r14 # 8-byte Reload
-; X64-NEXT:    addq {{[-0-9]+}}(%r{{[sb]}}p), %r14 # 8-byte Folded Reload
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
-; X64-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Folded Reload
-; X64-NEXT:    addq %rax, %r14
-; X64-NEXT:    adcq %rdx, %rcx
-; X64-NEXT:    addq %r13, %r9
-; X64-NEXT:    movq %r9, %r13
-; X64-NEXT:    adcq %r11, %r8
-; X64-NEXT:    movq %r8, %r15
-; X64-NEXT:    adcq $0, %r14
-; X64-NEXT:    adcq $0, %rcx
-; X64-NEXT:    addq {{[-0-9]+}}(%r{{[sb]}}p), %r14 # 8-byte Folded Reload
-; X64-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Folded Reload
-; X64-NEXT:    movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    movq %rdx, %rdi
+; X64-NEXT:    movq %rax, %rbp
+; X64-NEXT:    addq %rbx, %rbp
+; X64-NEXT:    movzbl %sil, %eax
+; X64-NEXT:    adcq %rax, %rdi
+; X64-NEXT:    addq %rcx, %r12
+; X64-NEXT:    adcq %r15, %r8
+; X64-NEXT:    adcq $0, %rbp
+; X64-NEXT:    adcq $0, %rdi
+; X64-NEXT:    addq %r13, %rbp
+; X64-NEXT:    adcq %r11, %rdi
 ; X64-NEXT:    setb {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Folded Spill
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rbx # 8-byte Reload
-; X64-NEXT:    movq %rbx, %rax
+; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
+; X64-NEXT:    movq %rcx, %rax
 ; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
 ; X64-NEXT:    mulq %rsi
-; X64-NEXT:    movq %rdx, %rcx
-; X64-NEXT:    movq %rax, %r11
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Reload
-; X64-NEXT:    movq %r8, %rax
+; X64-NEXT:    movq %rdx, %r15
+; X64-NEXT:    movq %rax, %r13
+; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r9 # 8-byte Reload
+; X64-NEXT:    movq %r9, %rax
 ; X64-NEXT:    mulq %rsi
 ; X64-NEXT:    movq %rdx, %rsi
-; X64-NEXT:    movq %rax, %rdi
-; X64-NEXT:    addq %rcx, %rdi
+; X64-NEXT:    movq %rax, %rbx
+; X64-NEXT:    addq %r15, %rbx
 ; X64-NEXT:    adcq $0, %rsi
-; X64-NEXT:    movq %rbx, %rax
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rbx # 8-byte Reload
-; X64-NEXT:    mulq %rbx
-; X64-NEXT:    movq %rdx, %rcx
-; X64-NEXT:    movq %rax, %r9
-; X64-NEXT:    addq %rdi, %r9
-; X64-NEXT:    adcq %rsi, %rcx
-; X64-NEXT:    setb %sil
-; X64-NEXT:    movq %r8, %rax
-; X64-NEXT:    mulq %rbx
-; X64-NEXT:    addq %rcx, %rax
-; X64-NEXT:    movzbl %sil, %ecx
-; X64-NEXT:    adcq %rcx, %rdx
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
-; X64-NEXT:    addq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Folded Reload
+; X64-NEXT:    movq %rcx, %rax
 ; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
-; X64-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Folded Reload
-; X64-NEXT:    addq %rax, %rsi
-; X64-NEXT:    adcq %rdx, %rcx
-; X64-NEXT:    addq %r14, %r11
-; X64-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %r9 # 8-byte Folded Reload
-; X64-NEXT:    movzbl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 1-byte Folded Reload
-; X64-NEXT:    adcq %rax, %rsi
-; X64-NEXT:    adcq $0, %rcx
-; X64-NEXT:    addq {{[-0-9]+}}(%r{{[sb]}}p), %r11 # 8-byte Folded Reload
-; X64-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %r9 # 8-byte Folded Reload
-; X64-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Folded Reload
-; X64-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Folded Reload
+; X64-NEXT:    mulq %rcx
+; X64-NEXT:    movq %rdx, %r11
+; X64-NEXT:    addq %rbx, %rax
+; X64-NEXT:    movq %rax, %r15
+; X64-NEXT:    adcq %rsi, %r11
+; X64-NEXT:    setb %bl
+; X64-NEXT:    movq %r9, %rax
+; X64-NEXT:    mulq %rcx
+; X64-NEXT:    addq %r11, %rax
+; X64-NEXT:    movzbl %bl, %ecx
+; X64-NEXT:    adcq %rcx, %rdx
+; X64-NEXT:    addq %rbp, %r13
+; X64-NEXT:    adcq %rdi, %r15
+; X64-NEXT:    movzbl {{[-0-9]+}}(%r{{[sb]}}p), %ecx # 1-byte Folded Reload
+; X64-NEXT:    adcq %rcx, %rax
+; X64-NEXT:    movq %rax, %rcx
+; X64-NEXT:    adcq $0, %rdx
 ; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
 ; X64-NEXT:    addq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
-; X64-NEXT:    adcq %rbp, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
-; X64-NEXT:    adcq %r12, %r13
+; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; X64-NEXT:    adcq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
+; X64-NEXT:    adcq %r14, %r12
+; X64-NEXT:    movq %r12, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    adcq %r10, %r8
+; X64-NEXT:    movq %r8, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    movzbl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 1-byte Folded Reload
+; X64-NEXT:    adcq %rax, %r13
 ; X64-NEXT:    movq %r13, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    adcq %r10, %r15
+; X64-NEXT:    adcq $0, %r15
 ; X64-NEXT:    movq %r15, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    movzbl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 1-byte Folded Reload
-; X64-NEXT:    adcq %rax, %r11
-; X64-NEXT:    movq %r11, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    adcq $0, %r9
-; X64-NEXT:    movq %r9, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    adcq $0, %rsi
-; X64-NEXT:    movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; X64-NEXT:    adcq $0, %rcx
 ; X64-NEXT:    movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    adcq $0, %rdx
+; X64-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
+; X64-NEXT:    movq 64(%rsi), %rdi
+; X64-NEXT:    movq (%rsp), %rbx # 8-byte Reload
+; X64-NEXT:    movq %rbx, %rax
+; X64-NEXT:    mulq %rdi
+; X64-NEXT:    movq %rdx, %rcx
+; X64-NEXT:    movq %rax, %r11
 ; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r9 # 8-byte Reload
-; X64-NEXT:    movq 64(%r9), %r11
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
-; X64-NEXT:    movq %rcx, %rax
-; X64-NEXT:    mulq %r11
-; X64-NEXT:    movq %rdx, %rsi
-; X64-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
-; X64-NEXT:    movq %rdi, %rax
-; X64-NEXT:    mulq %r11
-; X64-NEXT:    movq %rdx, %rbp
-; X64-NEXT:    movq %rax, %rbx
-; X64-NEXT:    addq %rsi, %rbx
-; X64-NEXT:    adcq $0, %rbp
-; X64-NEXT:    movq 72(%r9), %rsi
-; X64-NEXT:    movq %rcx, %rax
-; X64-NEXT:    mulq %rsi
-; X64-NEXT:    movq %rsi, %rcx
-; X64-NEXT:    movq %rdx, %rsi
-; X64-NEXT:    movq %rax, %r8
-; X64-NEXT:    addq %rbx, %r8
-; X64-NEXT:    adcq %rbp, %rsi
-; X64-NEXT:    setb %bl
-; X64-NEXT:    movq %rdi, %rax
+; X64-NEXT:    movq %r9, %rax
+; X64-NEXT:    mulq %rdi
+; X64-NEXT:    movq %rdi, %r8
+; X64-NEXT:    movq %rdx, %rdi
+; X64-NEXT:    movq %rax, %rbp
+; X64-NEXT:    addq %rcx, %rbp
+; X64-NEXT:    adcq $0, %rdi
+; X64-NEXT:    movq 72(%rsi), %rcx
+; X64-NEXT:    movq %rsi, %r13
+; X64-NEXT:    movq %rbx, %rax
 ; X64-NEXT:    mulq %rcx
-; X64-NEXT:    movq %rcx, %r13
-; X64-NEXT:    movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    movq %rcx, %rsi
 ; X64-NEXT:    movq %rdx, %rcx
-; X64-NEXT:    movq %rax, %rdi
-; X64-NEXT:    addq %rsi, %rdi
+; X64-NEXT:    movq %rax, %r14
+; X64-NEXT:    addq %rbp, %r14
+; X64-NEXT:    adcq %rdi, %rcx
+; X64-NEXT:    setb %bl
+; X64-NEXT:    movq %r9, %rax
+; X64-NEXT:    mulq %rsi
+; X64-NEXT:    movq %rsi, %r12
+; X64-NEXT:    movq %rdx, %r10
+; X64-NEXT:    movq %rax, %r15
+; X64-NEXT:    addq %rcx, %r15
 ; X64-NEXT:    movzbl %bl, %eax
-; X64-NEXT:    adcq %rax, %rcx
-; X64-NEXT:    movq %r11, %rax
-; X64-NEXT:    xorl %edx, %edx
-; X64-NEXT:    mulq %rdx
-; X64-NEXT:    movq %rax, %rbx
-; X64-NEXT:    movq %rdx, %r14
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r10 # 8-byte Reload
-; X64-NEXT:    addq %rax, %r10
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r15 # 8-byte Reload
-; X64-NEXT:    adcq %rdx, %r15
-; X64-NEXT:    addq %rdi, %r10
-; X64-NEXT:    adcq %rcx, %r15
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r12 # 8-byte Reload
-; X64-NEXT:    movq %r12, %rax
-; X64-NEXT:    movq %r11, %rsi
-; X64-NEXT:    movq %r11, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    mulq %r11
-; X64-NEXT:    movq %rdx, %r11
+; X64-NEXT:    adcq %rax, %r10
+; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r9 # 8-byte Reload
+; X64-NEXT:    movq %r9, %rax
+; X64-NEXT:    movq %r8, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    mulq %r8
+; X64-NEXT:    movq %rdx, %rcx
 ; X64-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rbp # 8-byte Reload
 ; X64-NEXT:    movq %rbp, %rax
+; X64-NEXT:    mulq %r8
+; X64-NEXT:    movq %rdx, %rdi
+; X64-NEXT:    movq %rax, %rbx
+; X64-NEXT:    addq %rcx, %rbx
+; X64-NEXT:    adcq $0, %rdi
+; X64-NEXT:    movq %r9, %rax
+; X64-NEXT:    movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; X64-NEXT:    mulq %rsi
 ; X64-NEXT:    movq %rdx, %rsi
-; X64-NEXT:    movq %rax, %rdi
-; X64-NEXT:    addq %r11, %rdi
-; X64-NEXT:    adcq $0, %rsi
-; X64-NEXT:    movq %r12, %rax
-; X64-NEXT:    mulq %r13
-; X64-NEXT:    movq %rdx, %rcx
-; X64-NEXT:    addq %rdi, %rax
+; X64-NEXT:    addq %rbx, %rax
 ; X64-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    adcq %rsi, %rcx
-; X64-NEXT:    setb %sil
+; X64-NEXT:    adcq %rdi, %rsi
+; X64-NEXT:    setb %cl
 ; X64-NEXT:    movq %rbp, %rax
-; X64-NEXT:    movq %rbp, %r11
-; X64-NEXT:    mulq %r13
-; X64-NEXT:    addq %rcx, %rax
-; X64-NEXT:    movzbl %sil, %ecx
-; X64-NEXT:    adcq %rcx, %rdx
-; X64-NEXT:    addq {{[-0-9]+}}(%r{{[sb]}}p), %rbx # 8-byte Folded Reload
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r13 # 8-byte Reload
-; X64-NEXT:    adcq %r13, %r14
-; X64-NEXT:    addq %rax, %rbx
-; X64-NEXT:    adcq %rdx, %r14
-; X64-NEXT:    addq {{[-0-9]+}}(%r{{[sb]}}p), %rbx # 8-byte Folded Reload
-; X64-NEXT:    adcq %r8, %r14
-; X64-NEXT:    adcq $0, %r10
+; X64-NEXT:    mulq %r12
+; X64-NEXT:    movq %rdx, %r12
+; X64-NEXT:    movq %rax, %rdi
+; X64-NEXT:    addq %rsi, %rdi
+; X64-NEXT:    movzbl %cl, %eax
+; X64-NEXT:    adcq %rax, %r12
+; X64-NEXT:    addq %r11, %rdi
+; X64-NEXT:    adcq %r14, %r12
 ; X64-NEXT:    adcq $0, %r15
-; X64-NEXT:    movq 80(%r9), %rdi
-; X64-NEXT:    movq %r12, %rax
-; X64-NEXT:    mulq %rdi
-; X64-NEXT:    movq %rdx, %r8
-; X64-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    movq %rbp, %rax
-; X64-NEXT:    mulq %rdi
-; X64-NEXT:    movq %rdx, %rsi
-; X64-NEXT:    movq %rax, %rcx
-; X64-NEXT:    addq %r8, %rcx
-; X64-NEXT:    adcq $0, %rsi
-; X64-NEXT:    movq 88(%r9), %r9
-; X64-NEXT:    movq %r12, %rax
-; X64-NEXT:    mulq %r9
-; X64-NEXT:    movq %rdx, %rbp
-; X64-NEXT:    movq %rax, %r8
-; X64-NEXT:    addq %rcx, %r8
-; X64-NEXT:    adcq %rsi, %rbp
-; X64-NEXT:    setb %r12b
-; X64-NEXT:    movq %r11, %rax
-; X64-NEXT:    mulq %r9
+; X64-NEXT:    adcq $0, %r10
+; X64-NEXT:    movq 80(%r13), %r11
+; X64-NEXT:    movq %r9, %rax
+; X64-NEXT:    mulq %r11
 ; X64-NEXT:    movq %rdx, %rcx
-; X64-NEXT:    movq %rax, %rsi
-; X64-NEXT:    addq %rbp, %rsi
-; X64-NEXT:    movzbl %r12b, %eax
-; X64-NEXT:    adcq %rax, %rcx
-; X64-NEXT:    movq %rdi, %rax
-; X64-NEXT:    xorl %edx, %edx
-; X64-NEXT:    mulq %rdx
-; X64-NEXT:    movq %rdx, %r12
-; X64-NEXT:    movq %rax, %r11
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
-; X64-NEXT:    addq %r11, %rax
-; X64-NEXT:    adcq %rdx, %r13
-; X64-NEXT:    addq %rsi, %rax
-; X64-NEXT:    adcq %rcx, %r13
-; X64-NEXT:    addq %rbx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
-; X64-NEXT:    adcq %r14, %r8
-; X64-NEXT:    movq %r8, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    adcq $0, %rax
-; X64-NEXT:    adcq $0, %r13
-; X64-NEXT:    addq %r10, %rax
-; X64-NEXT:    movq %rax, %r8
-; X64-NEXT:    adcq %r15, %r13
-; X64-NEXT:    setb %r14b
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rbp # 8-byte Reload
+; X64-NEXT:    movq %rax, %r14
 ; X64-NEXT:    movq %rbp, %rax
-; X64-NEXT:    mulq %rdi
+; X64-NEXT:    movq %rbp, %rsi
+; X64-NEXT:    mulq %r11
+; X64-NEXT:    movq %rdx, %rbx
+; X64-NEXT:    movq %rax, %rbp
+; X64-NEXT:    addq %rcx, %rbp
+; X64-NEXT:    adcq $0, %rbx
+; X64-NEXT:    movq 88(%r13), %r13
+; X64-NEXT:    movq %r9, %rax
+; X64-NEXT:    mulq %r13
 ; X64-NEXT:    movq %rdx, %rcx
-; X64-NEXT:    movq %rax, %r15
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r10 # 8-byte Reload
-; X64-NEXT:    movq %r10, %rax
-; X64-NEXT:    mulq %rdi
-; X64-NEXT:    movq %rdx, %rsi
-; X64-NEXT:    movq %rax, %rbx
-; X64-NEXT:    addq %rcx, %rbx
+; X64-NEXT:    addq %rbp, %rax
+; X64-NEXT:    movq %rax, %rbp
+; X64-NEXT:    adcq %rbx, %rcx
+; X64-NEXT:    setb %r9b
+; X64-NEXT:    movq %rsi, %rax
+; X64-NEXT:    mulq %r13
+; X64-NEXT:    movq %rdx, %rbx
+; X64-NEXT:    movq %rax, %rsi
+; X64-NEXT:    addq %rcx, %rsi
+; X64-NEXT:    movzbl %r9b, %eax
+; X64-NEXT:    adcq %rax, %rbx
+; X64-NEXT:    addq %rdi, %r14
+; X64-NEXT:    movq %r14, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    adcq %r12, %rbp
+; X64-NEXT:    movq %rbp, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; X64-NEXT:    adcq $0, %rsi
-; X64-NEXT:    movq %rbp, %rax
-; X64-NEXT:    mulq %r9
+; X64-NEXT:    adcq $0, %rbx
+; X64-NEXT:    addq %r15, %rsi
+; X64-NEXT:    adcq %r10, %rbx
+; X64-NEXT:    setb %r9b
+; X64-NEXT:    movq (%rsp), %r14 # 8-byte Reload
+; X64-NEXT:    movq %r14, %rax
+; X64-NEXT:    mulq %r11
 ; X64-NEXT:    movq %rdx, %rcx
-; X64-NEXT:    addq %rbx, %rax
+; X64-NEXT:    movq %rax, %r10
+; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Reload
+; X64-NEXT:    movq %r8, %rax
+; X64-NEXT:    mulq %r11
+; X64-NEXT:    movq %rdx, %rdi
 ; X64-NEXT:    movq %rax, %rbp
-; X64-NEXT:    adcq %rsi, %rcx
-; X64-NEXT:    setb %bl
-; X64-NEXT:    movq %r10, %rax
-; X64-NEXT:    mulq %r9
-; X64-NEXT:    addq %rcx, %rax
-; X64-NEXT:    movzbl %bl, %ecx
-; X64-NEXT:    adcq %rcx, %rdx
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
-; X64-NEXT:    addq %r11, %rsi
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
-; X64-NEXT:    adcq %r12, %rcx
-; X64-NEXT:    addq %rax, %rsi
-; X64-NEXT:    adcq %rdx, %rcx
-; X64-NEXT:    addq %r8, %r15
-; X64-NEXT:    movq %r15, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    adcq %r13, %rbp
+; X64-NEXT:    addq %rcx, %rbp
+; X64-NEXT:    adcq $0, %rdi
+; X64-NEXT:    movq %r14, %rax
+; X64-NEXT:    movq %r14, %r15
+; X64-NEXT:    mulq %r13
+; X64-NEXT:    movq %rdx, %rcx
+; X64-NEXT:    addq %rbp, %rax
+; X64-NEXT:    movq %rax, %rbp
+; X64-NEXT:    adcq %rdi, %rcx
+; X64-NEXT:    setb %dil
+; X64-NEXT:    movq %r8, %rax
+; X64-NEXT:    mulq %r13
+; X64-NEXT:    movq %rax, %r12
+; X64-NEXT:    addq %rcx, %r12
+; X64-NEXT:    movzbl %dil, %eax
+; X64-NEXT:    adcq %rax, %rdx
+; X64-NEXT:    addq %rsi, %r10
+; X64-NEXT:    movq %r10, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    adcq %rbx, %rbp
 ; X64-NEXT:    movq %rbp, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    movzbl %r14b, %eax
-; X64-NEXT:    adcq %rax, %rsi
-; X64-NEXT:    movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    adcq $0, %rcx
-; X64-NEXT:    movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    movzbl %r9b, %eax
+; X64-NEXT:    adcq %rax, %r12
+; X64-NEXT:    adcq $0, %rdx
+; X64-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
-; X64-NEXT:    imulq %rax, %r9
+; X64-NEXT:    imulq %rax, %r13
 ; X64-NEXT:    movq %rax, %r10
-; X64-NEXT:    mulq %rdi
+; X64-NEXT:    mulq %r11
 ; X64-NEXT:    movq %rax, %r8
-; X64-NEXT:    addq %r9, %rdx
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rbp # 8-byte Reload
-; X64-NEXT:    imulq %rbp, %rdi
-; X64-NEXT:    addq %rdx, %rdi
+; X64-NEXT:    addq %r13, %rdx
+; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
+; X64-NEXT:    imulq %rdi, %r11
+; X64-NEXT:    addq %rdx, %r11
 ; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
 ; X64-NEXT:    movq %rax, %rsi
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r11 # 8-byte Reload
-; X64-NEXT:    imulq %r11, %rsi
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
-; X64-NEXT:    mulq %rcx
+; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rbx # 8-byte Reload
+; X64-NEXT:    imulq %rbx, %rsi
+; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rbp # 8-byte Reload
+; X64-NEXT:    mulq %rbp
 ; X64-NEXT:    movq %rax, %r9
 ; X64-NEXT:    addq %rsi, %rdx
 ; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
-; X64-NEXT:    imulq %rcx, %rax
+; X64-NEXT:    imulq %rbp, %rax
 ; X64-NEXT:    addq %rdx, %rax
 ; X64-NEXT:    addq %r8, %r9
-; X64-NEXT:    adcq %rdi, %rax
+; X64-NEXT:    adcq %r11, %rax
 ; X64-NEXT:    movq %rax, %r8
-; X64-NEXT:    movq %rcx, %rax
-; X64-NEXT:    movq %rcx, %rdi
+; X64-NEXT:    movq %rbp, %rax
 ; X64-NEXT:    mulq %r10
 ; X64-NEXT:    movq %rdx, %rcx
 ; X64-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    movq %r11, %rax
+; X64-NEXT:    movq %rbx, %rax
+; X64-NEXT:    movq %rbx, %r11
 ; X64-NEXT:    mulq %r10
 ; X64-NEXT:    movq %rdx, %rsi
 ; X64-NEXT:    movq %rax, %rbx
 ; X64-NEXT:    addq %rcx, %rbx
 ; X64-NEXT:    adcq $0, %rsi
-; X64-NEXT:    movq %rdi, %rax
-; X64-NEXT:    mulq %rbp
-; X64-NEXT:    movq %rdx, %rdi
-; X64-NEXT:    movq %rax, %r15
-; X64-NEXT:    addq %rbx, %r15
-; X64-NEXT:    adcq %rsi, %rdi
+; X64-NEXT:    movq %rbp, %rax
+; X64-NEXT:    mulq %rdi
+; X64-NEXT:    movq %rdx, %rbp
+; X64-NEXT:    movq %rax, %r14
+; X64-NEXT:    addq %rbx, %r14
+; X64-NEXT:    adcq %rsi, %rbp
 ; X64-NEXT:    setb %cl
 ; X64-NEXT:    movq %r11, %rax
-; X64-NEXT:    mulq %rbp
-; X64-NEXT:    movq %rdx, %r12
-; X64-NEXT:    movq %rax, %r13
-; X64-NEXT:    addq %rdi, %r13
+; X64-NEXT:    mulq %rdi
+; X64-NEXT:    movq %rdx, %r13
+; X64-NEXT:    movq %rax, %r11
+; X64-NEXT:    addq %rbp, %r11
 ; X64-NEXT:    movzbl %cl, %eax
-; X64-NEXT:    adcq %rax, %r12
-; X64-NEXT:    addq %r9, %r13
-; X64-NEXT:    adcq %r8, %r12
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rbp # 8-byte Reload
-; X64-NEXT:    movq 120(%rbp), %rcx
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r10 # 8-byte Reload
-; X64-NEXT:    imulq %r10, %rcx
-; X64-NEXT:    movq 112(%rbp), %rsi
-; X64-NEXT:    movq %r10, %rax
+; X64-NEXT:    adcq %rax, %r13
+; X64-NEXT:    addq %r9, %r11
+; X64-NEXT:    adcq %r8, %r13
+; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
+; X64-NEXT:    movq 120(%rdx), %rcx
+; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; X64-NEXT:    imulq %rax, %rcx
+; X64-NEXT:    movq 112(%rdx), %rsi
+; X64-NEXT:    movq %rdx, %rdi
+; X64-NEXT:    movq %rax, %rbx
 ; X64-NEXT:    mulq %rsi
-; X64-NEXT:    movq %rax, %r11
+; X64-NEXT:    movq %rax, %r10
 ; X64-NEXT:    addq %rcx, %rdx
 ; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Reload
 ; X64-NEXT:    imulq %r8, %rsi
 ; X64-NEXT:    addq %rdx, %rsi
-; X64-NEXT:    movq 96(%rbp), %rdi
-; X64-NEXT:    movq 104(%rbp), %rbx
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
-; X64-NEXT:    movq %rax, %rcx
-; X64-NEXT:    imulq %rbx, %rcx
-; X64-NEXT:    mulq %rdi
+; X64-NEXT:    movq 96(%rdi), %rcx
+; X64-NEXT:    movq 104(%rdi), %rbp
+; X64-NEXT:    movq %r15, %rax
+; X64-NEXT:    movq %r15, %rdi
+; X64-NEXT:    imulq %rbp, %rdi
+; X64-NEXT:    mulq %rcx
 ; X64-NEXT:    movq %rax, %r9
-; X64-NEXT:    addq %rcx, %rdx
+; X64-NEXT:    addq %rdi, %rdx
 ; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
-; X64-NEXT:    imulq %rdi, %rax
+; X64-NEXT:    imulq %rcx, %rax
 ; X64-NEXT:    addq %rdx, %rax
-; X64-NEXT:    addq %r11, %r9
+; X64-NEXT:    addq %r10, %r9
 ; X64-NEXT:    adcq %rsi, %rax
-; X64-NEXT:    movq %rax, %r11
-; X64-NEXT:    movq %rdi, %rax
-; X64-NEXT:    mulq %r10
-; X64-NEXT:    movq %rdx, %rcx
-; X64-NEXT:    movq %rax, %r14
-; X64-NEXT:    movq %rbx, %rax
-; X64-NEXT:    mulq %r10
+; X64-NEXT:    movq %rax, %r10
+; X64-NEXT:    movq %rcx, %rax
+; X64-NEXT:    mulq %rbx
 ; X64-NEXT:    movq %rdx, %rsi
-; X64-NEXT:    movq %rax, %rbp
-; X64-NEXT:    addq %rcx, %rbp
-; X64-NEXT:    adcq $0, %rsi
-; X64-NEXT:    movq %rdi, %rax
+; X64-NEXT:    movq %rax, %r15
+; X64-NEXT:    movq %rbp, %rax
+; X64-NEXT:    mulq %rbx
+; X64-NEXT:    movq %rdx, %rbx
+; X64-NEXT:    movq %rax, %rdi
+; X64-NEXT:    addq %rsi, %rdi
+; X64-NEXT:    adcq $0, %rbx
+; X64-NEXT:    movq %rcx, %rax
 ; X64-NEXT:    mulq %r8
 ; X64-NEXT:    movq %rdx, %rcx
-; X64-NEXT:    movq %rax, %rdi
-; X64-NEXT:    addq %rbp, %rdi
-; X64-NEXT:    adcq %rsi, %rcx
-; X64-NEXT:    setb %sil
-; X64-NEXT:    movq %rbx, %rax
+; X64-NEXT:    movq %rax, %rsi
+; X64-NEXT:    addq %rdi, %rsi
+; X64-NEXT:    adcq %rbx, %rcx
+; X64-NEXT:    setb %bl
+; X64-NEXT:    movq %rbp, %rax
 ; X64-NEXT:    mulq %r8
 ; X64-NEXT:    addq %rcx, %rax
-; X64-NEXT:    movzbl %sil, %ecx
+; X64-NEXT:    movzbl %bl, %ecx
 ; X64-NEXT:    adcq %rcx, %rdx
 ; X64-NEXT:    addq %r9, %rax
-; X64-NEXT:    adcq %r11, %rdx
-; X64-NEXT:    addq {{[-0-9]+}}(%r{{[sb]}}p), %r14 # 8-byte Folded Reload
-; X64-NEXT:    adcq %r15, %rdi
-; X64-NEXT:    adcq %r13, %rax
-; X64-NEXT:    adcq %r12, %rdx
-; X64-NEXT:    addq {{[-0-9]+}}(%r{{[sb]}}p), %r14 # 8-byte Folded Reload
-; X64-NEXT:    movq %r14, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Folded Reload
-; X64-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Folded Reload
+; X64-NEXT:    adcq %r10, %rdx
+; X64-NEXT:    addq {{[-0-9]+}}(%r{{[sb]}}p), %r15 # 8-byte Folded Reload
+; X64-NEXT:    adcq %r14, %rsi
+; X64-NEXT:    adcq %r11, %rax
+; X64-NEXT:    adcq %r13, %rdx
+; X64-NEXT:    addq {{[-0-9]+}}(%r{{[sb]}}p), %r15 # 8-byte Folded Reload
+; X64-NEXT:    movq %r15, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Folded Reload
+; X64-NEXT:    movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    adcq %r12, %rax
 ; X64-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; X64-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Folded Reload
 ; X64-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r9 # 8-byte Reload
-; X64-NEXT:    movq 80(%r9), %rsi
-; X64-NEXT:    movq %rsi, %rax
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
-; X64-NEXT:    mulq %rcx
-; X64-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    movq %rdx, %rdi
-; X64-NEXT:    movq 88(%r9), %r8
-; X64-NEXT:    movq %r8, %rax
-; X64-NEXT:    movq %r8, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    mulq %rcx
-; X64-NEXT:    movq %rcx, %r11
-; X64-NEXT:    movq %rdx, %rbp
-; X64-NEXT:    movq %rax, %rbx
-; X64-NEXT:    addq %rdi, %rbx
-; X64-NEXT:    adcq $0, %rbp
-; X64-NEXT:    movq %rsi, %rax
-; X64-NEXT:    movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r15 # 8-byte Reload
-; X64-NEXT:    mulq %r15
+; X64-NEXT:    movq 80(%r9), %rbp
+; X64-NEXT:    movq %rbp, %rax
+; X64-NEXT:    movq %rbp, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r14 # 8-byte Reload
+; X64-NEXT:    mulq %r14
+; X64-NEXT:    movq %rax, %r15
 ; X64-NEXT:    movq %rdx, %rcx
-; X64-NEXT:    movq %rax, %r14
-; X64-NEXT:    addq %rbx, %r14
-; X64-NEXT:    adcq %rbp, %rcx
-; X64-NEXT:    setb %r10b
-; X64-NEXT:    movq %r8, %rax
-; X64-NEXT:    mulq %r15
-; X64-NEXT:    movq %rdx, %rbp
-; X64-NEXT:    movq %rax, %rbx
-; X64-NEXT:    addq %rcx, %rbx
-; X64-NEXT:    movzbl %r10b, %eax
-; X64-NEXT:    adcq %rax, %rbp
-; X64-NEXT:    movq %rsi, %rax
-; X64-NEXT:    xorl %ecx, %ecx
+; X64-NEXT:    movq 88(%r9), %rbx
+; X64-NEXT:    movq %rbx, %rax
+; X64-NEXT:    movq %rbx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    mulq %r14
+; X64-NEXT:    movq %rdx, %rsi
+; X64-NEXT:    movq %rax, %rdi
+; X64-NEXT:    addq %rcx, %rdi
+; X64-NEXT:    adcq $0, %rsi
+; X64-NEXT:    movq %rbp, %rax
+; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
 ; X64-NEXT:    mulq %rcx
-; X64-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    movq %rax, %rsi
-; X64-NEXT:    movq (%rsp), %r12 # 8-byte Reload
-; X64-NEXT:    addq %r12, %rsi
-; X64-NEXT:    movq %rdx, %r10
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Reload
-; X64-NEXT:    adcq %r8, %r10
-; X64-NEXT:    addq %rbx, %rsi
-; X64-NEXT:    adcq %rbp, %r10
-; X64-NEXT:    movq 64(%r9), %r13
-; X64-NEXT:    movq %r13, %rax
-; X64-NEXT:    mulq %r11
-; X64-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    movq %rdx, %rcx
-; X64-NEXT:    movq 72(%r9), %rdi
-; X64-NEXT:    movq %rdi, %rax
-; X64-NEXT:    mulq %r11
 ; X64-NEXT:    movq %rdx, %rbp
-; X64-NEXT:    movq %rax, %rbx
-; X64-NEXT:    addq %rcx, %rbx
-; X64-NEXT:    adcq $0, %rbp
-; X64-NEXT:    movq %r13, %rax
-; X64-NEXT:    mulq %r15
-; X64-NEXT:    movq %rdx, %rcx
-; X64-NEXT:    addq %rbx, %rax
-; X64-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    adcq %rbp, %rcx
-; X64-NEXT:    setb %r11b
-; X64-NEXT:    movq %rdi, %rax
-; X64-NEXT:    mulq %r15
-; X64-NEXT:    movq %rdx, %rbx
-; X64-NEXT:    movq %rax, %rbp
-; X64-NEXT:    addq %rcx, %rbp
-; X64-NEXT:    movzbl %r11b, %eax
-; X64-NEXT:    adcq %rax, %rbx
-; X64-NEXT:    movq %r13, %rax
-; X64-NEXT:    xorl %ecx, %ecx
+; X64-NEXT:    movq %rax, %r10
+; X64-NEXT:    addq %rdi, %r10
+; X64-NEXT:    adcq %rsi, %rbp
+; X64-NEXT:    setb %sil
+; X64-NEXT:    movq %rbx, %rax
 ; X64-NEXT:    mulq %rcx
-; X64-NEXT:    movq %rdx, %r11
-; X64-NEXT:    movq %rax, %r15
-; X64-NEXT:    movq %r12, %rcx
-; X64-NEXT:    addq %rax, %rcx
-; X64-NEXT:    adcq %rdx, %r8
-; X64-NEXT:    addq %rbp, %rcx
-; X64-NEXT:    adcq %rbx, %r8
-; X64-NEXT:    addq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Folded Reload
-; X64-NEXT:    movq %rcx, (%rsp) # 8-byte Spill
-; X64-NEXT:    adcq %r14, %r8
-; X64-NEXT:    movq %r8, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    adcq $0, %rsi
-; X64-NEXT:    adcq $0, %r10
+; X64-NEXT:    movq %rcx, %rbx
+; X64-NEXT:    movq %rdx, %r8
+; X64-NEXT:    movq %rax, %r11
+; X64-NEXT:    addq %rbp, %r11
+; X64-NEXT:    movzbl %sil, %eax
+; X64-NEXT:    adcq %rax, %r8
+; X64-NEXT:    movq 64(%r9), %r13
 ; X64-NEXT:    movq %r13, %rax
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r14 # 8-byte Reload
-; X64-NEXT:    mulq %r14
-; X64-NEXT:    movq %rdx, %rcx
-; X64-NEXT:    movq %rax, %r12
-; X64-NEXT:    movq %rdi, %rax
-; X64-NEXT:    movq %rdi, %r8
-; X64-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    movq %r14, %rdi
 ; X64-NEXT:    mulq %r14
+; X64-NEXT:    movq %rax, (%rsp) # 8-byte Spill
+; X64-NEXT:    movq %rdx, %r14
+; X64-NEXT:    movq 72(%r9), %rax
+; X64-NEXT:    movq %rax, %r9
+; X64-NEXT:    mulq %rdi
 ; X64-NEXT:    movq %rdx, %rdi
 ; X64-NEXT:    movq %rax, %rbp
-; X64-NEXT:    addq %rcx, %rbp
+; X64-NEXT:    addq %r14, %rbp
 ; X64-NEXT:    adcq $0, %rdi
 ; X64-NEXT:    movq %r13, %rax
+; X64-NEXT:    movq %r13, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    mulq %rcx
+; X64-NEXT:    movq %rdx, %rsi
+; X64-NEXT:    addq %rbp, %rax
+; X64-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    adcq %rdi, %rsi
+; X64-NEXT:    setb %cl
+; X64-NEXT:    movq %r9, %rax
+; X64-NEXT:    mulq %rbx
+; X64-NEXT:    movq %rdx, %r14
+; X64-NEXT:    movq %rax, %r12
+; X64-NEXT:    addq %rsi, %r12
+; X64-NEXT:    movzbl %cl, %eax
+; X64-NEXT:    adcq %rax, %r14
+; X64-NEXT:    addq %r15, %r12
+; X64-NEXT:    adcq %r10, %r14
+; X64-NEXT:    adcq $0, %r11
+; X64-NEXT:    adcq $0, %r8
+; X64-NEXT:    movq %r13, %rax
 ; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rbx # 8-byte Reload
 ; X64-NEXT:    mulq %rbx
-; X64-NEXT:    movq %rdx, %rcx
-; X64-NEXT:    addq %rbp, %rax
-; X64-NEXT:    movq %rax, %rbp
-; X64-NEXT:    adcq %rdi, %rcx
-; X64-NEXT:    setb %dil
-; X64-NEXT:    movq %r8, %rax
+; X64-NEXT:    movq %rdx, %rbp
+; X64-NEXT:    movq %rax, %r10
+; X64-NEXT:    movq %r9, %rax
+; X64-NEXT:    movq %r9, %r15
+; X64-NEXT:    movq %r9, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; X64-NEXT:    mulq %rbx
-; X64-NEXT:    addq %rcx, %rax
-; X64-NEXT:    movzbl %dil, %ecx
-; X64-NEXT:    adcq %rcx, %rdx
+; X64-NEXT:    movq %rdx, %rsi
+; X64-NEXT:    movq %rax, %rcx
+; X64-NEXT:    addq %rbp, %rcx
+; X64-NEXT:    adcq $0, %rsi
+; X64-NEXT:    movq %r13, %rax
 ; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r9 # 8-byte Reload
-; X64-NEXT:    addq %r9, %r15
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Reload
-; X64-NEXT:    adcq %r8, %r11
-; X64-NEXT:    addq %rax, %r15
-; X64-NEXT:    adcq %rdx, %r11
-; X64-NEXT:    addq (%rsp), %r12 # 8-byte Folded Reload
-; X64-NEXT:    movq %r12, (%rsp) # 8-byte Spill
-; X64-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %rbp # 8-byte Folded Reload
+; X64-NEXT:    mulq %r9
+; X64-NEXT:    movq %rdx, %rdi
+; X64-NEXT:    addq %rcx, %rax
+; X64-NEXT:    movq %rax, %rbp
+; X64-NEXT:    adcq %rsi, %rdi
+; X64-NEXT:    setb %cl
+; X64-NEXT:    movq %r15, %rax
+; X64-NEXT:    mulq %r9
+; X64-NEXT:    movq %rdx, %r15
+; X64-NEXT:    movq %rax, %r13
+; X64-NEXT:    addq %rdi, %r13
+; X64-NEXT:    movzbl %cl, %eax
+; X64-NEXT:    adcq %rax, %r15
+; X64-NEXT:    addq %r12, %r10
+; X64-NEXT:    movq %r10, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    adcq %r14, %rbp
 ; X64-NEXT:    movq %rbp, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    adcq $0, %r13
 ; X64-NEXT:    adcq $0, %r15
-; X64-NEXT:    adcq $0, %r11
-; X64-NEXT:    addq %rsi, %r15
-; X64-NEXT:    adcq %r10, %r11
-; X64-NEXT:    setb {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Folded Spill
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
-; X64-NEXT:    movq %rcx, %rax
-; X64-NEXT:    movq %r14, %rsi
-; X64-NEXT:    mulq %r14
-; X64-NEXT:    movq %rdx, %r10
-; X64-NEXT:    movq %rax, %rbp
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r14 # 8-byte Reload
-; X64-NEXT:    movq %r14, %rax
-; X64-NEXT:    mulq %rsi
-; X64-NEXT:    movq %rdx, %rdi
-; X64-NEXT:    movq %rax, %rbx
-; X64-NEXT:    addq %r10, %rbx
-; X64-NEXT:    adcq $0, %rdi
-; X64-NEXT:    movq %rcx, %rax
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
-; X64-NEXT:    mulq %rcx
-; X64-NEXT:    movq %rdx, %r10
-; X64-NEXT:    addq %rbx, %rax
+; X64-NEXT:    addq %r11, %r13
+; X64-NEXT:    adcq %r8, %r15
+; X64-NEXT:    setb %r8b
+; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rbp # 8-byte Reload
+; X64-NEXT:    movq %rbp, %rax
+; X64-NEXT:    mulq %rbx
+; X64-NEXT:    movq %rdx, %r11
 ; X64-NEXT:    movq %rax, %r12
-; X64-NEXT:    adcq %rdi, %r10
-; X64-NEXT:    setb %bl
-; X64-NEXT:    movq %r14, %rax
-; X64-NEXT:    mulq %rcx
-; X64-NEXT:    movq %rcx, %rdi
-; X64-NEXT:    addq %r10, %rax
-; X64-NEXT:    movzbl %bl, %ecx
-; X64-NEXT:    adcq %rcx, %rdx
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rbx # 8-byte Reload
-; X64-NEXT:    addq %r9, %rbx
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
-; X64-NEXT:    adcq %r8, %rcx
-; X64-NEXT:    addq %rax, %rbx
-; X64-NEXT:    adcq %rdx, %rcx
-; X64-NEXT:    addq %r15, %rbp
-; X64-NEXT:    movq %rbp, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    adcq %r11, %r12
+; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r10 # 8-byte Reload
+; X64-NEXT:    movq %r10, %rax
+; X64-NEXT:    mulq %rbx
+; X64-NEXT:    movq %rdx, %rsi
+; X64-NEXT:    movq %rax, %rcx
+; X64-NEXT:    addq %r11, %rcx
+; X64-NEXT:    adcq $0, %rsi
+; X64-NEXT:    movq %rbp, %rax
+; X64-NEXT:    mulq %r9
+; X64-NEXT:    movq %rdx, %r14
+; X64-NEXT:    addq %rcx, %rax
+; X64-NEXT:    movq %rax, %rcx
+; X64-NEXT:    adcq %rsi, %r14
+; X64-NEXT:    setb %sil
+; X64-NEXT:    movq %r10, %rax
+; X64-NEXT:    mulq %r9
+; X64-NEXT:    addq %r14, %rax
+; X64-NEXT:    movzbl %sil, %esi
+; X64-NEXT:    adcq %rsi, %rdx
+; X64-NEXT:    addq %r13, %r12
 ; X64-NEXT:    movq %r12, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    movzbl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 1-byte Folded Reload
-; X64-NEXT:    adcq %rax, %rbx
-; X64-NEXT:    movq %rbx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    adcq $0, %rcx
+; X64-NEXT:    adcq %r15, %rcx
 ; X64-NEXT:    movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    movzbl %r8b, %ecx
+; X64-NEXT:    adcq %rcx, %rax
+; X64-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    adcq $0, %rdx
+; X64-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rbp # 8-byte Reload
-; X64-NEXT:    movq 96(%rbp), %rcx
-; X64-NEXT:    imulq %rcx, %rdi
-; X64-NEXT:    movq %rcx, %rax
-; X64-NEXT:    mulq %rsi
-; X64-NEXT:    movq %rax, %r9
-; X64-NEXT:    addq %rdi, %rdx
-; X64-NEXT:    movq 104(%rbp), %r8
-; X64-NEXT:    imulq %r8, %rsi
-; X64-NEXT:    addq %rdx, %rsi
-; X64-NEXT:    movq %rsi, %r11
+; X64-NEXT:    movq 96(%rbp), %rsi
+; X64-NEXT:    imulq %rsi, %r9
+; X64-NEXT:    movq %rsi, %rax
+; X64-NEXT:    mulq %rbx
+; X64-NEXT:    movq %rax, %r14
+; X64-NEXT:    addq %r9, %rdx
+; X64-NEXT:    movq 104(%rbp), %r15
+; X64-NEXT:    imulq %r15, %rbx
+; X64-NEXT:    addq %rdx, %rbx
+; X64-NEXT:    movq %rbx, %r9
 ; X64-NEXT:    movq 112(%rbp), %rax
 ; X64-NEXT:    movq %rbp, %rdi
-; X64-NEXT:    movq %rax, %rsi
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rbp # 8-byte Reload
-; X64-NEXT:    imulq %rbp, %rsi
+; X64-NEXT:    movq %rax, %rcx
 ; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rbx # 8-byte Reload
-; X64-NEXT:    mulq %rbx
-; X64-NEXT:    movq %rax, %r10
-; X64-NEXT:    addq %rsi, %rdx
-; X64-NEXT:    movq 120(%rdi), %rdi
-; X64-NEXT:    imulq %rbx, %rdi
-; X64-NEXT:    addq %rdx, %rdi
-; X64-NEXT:    addq %r9, %r10
-; X64-NEXT:    adcq %r11, %rdi
-; X64-NEXT:    movq %rbx, %rax
-; X64-NEXT:    movq %rbx, %rsi
-; X64-NEXT:    mulq %rcx
-; X64-NEXT:    movq %rdx, %rbx
-; X64-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    imulq %rbx, %rcx
+; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rbp # 8-byte Reload
+; X64-NEXT:    mulq %rbp
+; X64-NEXT:    movq %rax, %r13
+; X64-NEXT:    addq %rcx, %rdx
+; X64-NEXT:    movq 120(%rdi), %r8
+; X64-NEXT:    imulq %rbp, %r8
+; X64-NEXT:    addq %rdx, %r8
+; X64-NEXT:    addq %r14, %r13
+; X64-NEXT:    adcq %r9, %r8
 ; X64-NEXT:    movq %rbp, %rax
-; X64-NEXT:    movq %rbp, %r9
-; X64-NEXT:    mulq %rcx
-; X64-NEXT:    movq %rdx, %rcx
-; X64-NEXT:    movq %rax, %rbp
-; X64-NEXT:    addq %rbx, %rbp
-; X64-NEXT:    adcq $0, %rcx
-; X64-NEXT:    movq %rsi, %rax
-; X64-NEXT:    mulq %r8
+; X64-NEXT:    mulq %rsi
+; X64-NEXT:    movq %rdx, %r14
+; X64-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    movq %rbx, %rax
+; X64-NEXT:    mulq %rsi
 ; X64-NEXT:    movq %rdx, %rsi
-; X64-NEXT:    movq %rax, %r12
-; X64-NEXT:    addq %rbp, %r12
-; X64-NEXT:    adcq %rcx, %rsi
+; X64-NEXT:    movq %rax, %rcx
+; X64-NEXT:    addq %r14, %rcx
+; X64-NEXT:    adcq $0, %rsi
+; X64-NEXT:    movq %rbp, %rax
+; X64-NEXT:    mulq %r15
+; X64-NEXT:    movq %rdx, %rdi
+; X64-NEXT:    addq %rcx, %rax
+; X64-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    adcq %rsi, %rdi
 ; X64-NEXT:    setb %cl
-; X64-NEXT:    movq %r9, %rax
-; X64-NEXT:    mulq %r8
-; X64-NEXT:    movq %rdx, %rbx
-; X64-NEXT:    movq %rax, %rbp
-; X64-NEXT:    addq %rsi, %rbp
+; X64-NEXT:    movq %rbx, %rax
+; X64-NEXT:    mulq %r15
+; X64-NEXT:    movq %rdx, %r12
+; X64-NEXT:    movq %rax, %r15
+; X64-NEXT:    addq %rdi, %r15
 ; X64-NEXT:    movzbl %cl, %eax
-; X64-NEXT:    adcq %rax, %rbx
-; X64-NEXT:    addq %r10, %rbp
-; X64-NEXT:    adcq %rdi, %rbx
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
-; X64-NEXT:    imulq %r13, %rsi
-; X64-NEXT:    movq %r13, %rax
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Reload
-; X64-NEXT:    mulq %r8
-; X64-NEXT:    movq %rax, %r9
-; X64-NEXT:    addq %rsi, %rdx
+; X64-NEXT:    adcq %rax, %r12
+; X64-NEXT:    addq %r13, %r15
+; X64-NEXT:    adcq %r8, %r12
+; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
 ; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r11 # 8-byte Reload
-; X64-NEXT:    imulq %r11, %r8
-; X64-NEXT:    addq %rdx, %r8
+; X64-NEXT:    imulq %r11, %rcx
+; X64-NEXT:    movq %r11, %rax
+; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
+; X64-NEXT:    mulq %rsi
+; X64-NEXT:    movq %rax, %r8
+; X64-NEXT:    addq %rcx, %rdx
+; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r14 # 8-byte Reload
+; X64-NEXT:    imulq %r14, %rsi
+; X64-NEXT:    addq %rdx, %rsi
+; X64-NEXT:    movq %rsi, %r9
 ; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
-; X64-NEXT:    movq %rax, %rcx
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r15 # 8-byte Reload
-; X64-NEXT:    imulq %r15, %rcx
+; X64-NEXT:    movq %rax, %rsi
+; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rbp # 8-byte Reload
+; X64-NEXT:    imulq %rbp, %rsi
 ; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
 ; X64-NEXT:    mulq %rdi
-; X64-NEXT:    movq %rax, %r10
-; X64-NEXT:    addq %rcx, %rdx
-; X64-NEXT:    movq %r14, %rax
-; X64-NEXT:    imulq %rdi, %rax
-; X64-NEXT:    addq %rdx, %rax
-; X64-NEXT:    addq %r9, %r10
-; X64-NEXT:    adcq %r8, %rax
-; X64-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    movq %rax, %r13
+; X64-NEXT:    addq %rsi, %rdx
+; X64-NEXT:    imulq %rdi, %r10
+; X64-NEXT:    addq %rdx, %r10
+; X64-NEXT:    addq %r8, %r13
+; X64-NEXT:    adcq %r9, %r10
+; X64-NEXT:    movq %r10, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; X64-NEXT:    movq %rdi, %rax
-; X64-NEXT:    movq %rdi, %r14
-; X64-NEXT:    mulq %r13
-; X64-NEXT:    movq %rdx, %rdi
+; X64-NEXT:    movq %rdi, %r10
+; X64-NEXT:    mulq %r11
+; X64-NEXT:    movq %rdx, %rsi
 ; X64-NEXT:    movq %rax, %r8
-; X64-NEXT:    movq %r15, %rax
-; X64-NEXT:    mulq %r13
+; X64-NEXT:    movq %rbp, %rax
+; X64-NEXT:    mulq %r11
 ; X64-NEXT:    movq %rdx, %r9
-; X64-NEXT:    movq %rax, %rcx
-; X64-NEXT:    addq %rdi, %rcx
+; X64-NEXT:    movq %rax, %rdi
+; X64-NEXT:    addq %rsi, %rdi
 ; X64-NEXT:    adcq $0, %r9
-; X64-NEXT:    movq %r14, %rax
-; X64-NEXT:    mulq %r11
-; X64-NEXT:    movq %rdx, %rdi
+; X64-NEXT:    movq %r10, %rax
+; X64-NEXT:    mulq %r14
+; X64-NEXT:    movq %rdx, %rcx
 ; X64-NEXT:    movq %rax, %rsi
-; X64-NEXT:    addq %rcx, %rsi
-; X64-NEXT:    adcq %r9, %rdi
-; X64-NEXT:    setb %cl
-; X64-NEXT:    movq %r15, %rax
-; X64-NEXT:    mulq %r11
-; X64-NEXT:    addq %rdi, %rax
-; X64-NEXT:    movzbl %cl, %ecx
+; X64-NEXT:    addq %rdi, %rsi
+; X64-NEXT:    adcq %r9, %rcx
+; X64-NEXT:    setb %dil
+; X64-NEXT:    movq %rbp, %rax
+; X64-NEXT:    mulq %r14
+; X64-NEXT:    addq %rcx, %rax
+; X64-NEXT:    movzbl %dil, %ecx
 ; X64-NEXT:    adcq %rcx, %rdx
-; X64-NEXT:    addq %r10, %rax
+; X64-NEXT:    addq %r13, %rax
 ; X64-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Folded Reload
 ; X64-NEXT:    addq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Folded Reload
-; X64-NEXT:    adcq %r12, %rsi
-; X64-NEXT:    adcq %rbp, %rax
-; X64-NEXT:    adcq %rbx, %rdx
+; X64-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Folded Reload
+; X64-NEXT:    adcq %r15, %rax
+; X64-NEXT:    adcq %r12, %rdx
 ; X64-NEXT:    addq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Folded Reload
 ; X64-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Folded Reload
 ; X64-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Folded Reload
 ; X64-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Folded Reload
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
+; X64-NEXT:    movq (%rsp), %rcx # 8-byte Reload
 ; X64-NEXT:    addq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Folded Reload
 ; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
 ; X64-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Folded Reload
-; X64-NEXT:    movq (%rsp), %rbp # 8-byte Reload
-; X64-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %rbp # 8-byte Folded Reload
 ; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rbx # 8-byte Reload
 ; X64-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %rbx # 8-byte Folded Reload
+; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rbp # 8-byte Reload
+; X64-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %rbp # 8-byte Folded Reload
 ; X64-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Folded Reload
 ; X64-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Folded Reload
 ; X64-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Folded Reload
@@ -8109,8 +5889,8 @@ define void @test_1024(i1024* %a, i1024* %b, i1024* %out) nounwind {
 ; X64-NEXT:    movq %rcx, %r9
 ; X64-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Folded Reload
 ; X64-NEXT:    movq %rdi, %r10
-; X64-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %rbp # 8-byte Folded Reload
 ; X64-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %rbx # 8-byte Folded Reload
+; X64-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %rbp # 8-byte Folded Reload
 ; X64-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Folded Reload
 ; X64-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Folded Reload
 ; X64-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Folded Reload
@@ -8134,13 +5914,13 @@ define void @test_1024(i1024* %a, i1024* %b, i1024* %out) nounwind {
 ; X64-NEXT:    movq %rdi, 56(%rcx)
 ; X64-NEXT:    movq %r9, 64(%rcx)
 ; X64-NEXT:    movq %r10, 72(%rcx)
-; X64-NEXT:    movq %rbp, 80(%rcx)
-; X64-NEXT:    movq %rbx, 88(%rcx)
+; X64-NEXT:    movq %rbx, 80(%rcx)
+; X64-NEXT:    movq %rbp, 88(%rcx)
 ; X64-NEXT:    movq %r8, 96(%rcx)
 ; X64-NEXT:    movq %rsi, 104(%rcx)
 ; X64-NEXT:    movq %rax, 112(%rcx)
 ; X64-NEXT:    movq %rdx, 120(%rcx)
-; X64-NEXT:    addq $352, %rsp # imm = 0x160
+; X64-NEXT:    addq $240, %rsp
 ; X64-NEXT:    popq %rbx
 ; X64-NEXT:    popq %r12
 ; X64-NEXT:    popq %r13
diff --git a/llvm/test/CodeGen/X86/mul-i256.ll b/llvm/test/CodeGen/X86/mul-i256.ll
index df64b9054..7d1a3b70e 100644
--- a/llvm/test/CodeGen/X86/mul-i256.ll
+++ b/llvm/test/CodeGen/X86/mul-i256.ll
@@ -15,264 +15,207 @@ define void @test(i256* %a, i256* %b, i256* %out) #0 {
 ; X32-NEXT:    .cfi_def_cfa_offset 16
 ; X32-NEXT:    pushl %esi
 ; X32-NEXT:    .cfi_def_cfa_offset 20
-; X32-NEXT:    subl $88, %esp
-; X32-NEXT:    .cfi_def_cfa_offset 108
+; X32-NEXT:    subl $72, %esp
+; X32-NEXT:    .cfi_def_cfa_offset 92
 ; X32-NEXT:    .cfi_offset %esi, -20
 ; X32-NEXT:    .cfi_offset %edi, -16
 ; X32-NEXT:    .cfi_offset %ebx, -12
 ; X32-NEXT:    .cfi_offset %ebp, -8
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    movl 12(%ecx), %ebp
-; X32-NEXT:    movl 8(%ecx), %edi
-; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl (%eax), %ebx
+; X32-NEXT:    movl 12(%ecx), %esi
+; X32-NEXT:    movl 8(%ecx), %ebx
 ; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    movl (%eax), %edi
+; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl %ebx, %eax
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edx, %ebp
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    mull %ebx
+; X32-NEXT:    movl %esi, %eax
+; X32-NEXT:    movl %esi, %ecx
+; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    mull %edi
 ; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    movl %eax, %ebx
-; X32-NEXT:    addl %ecx, %ebx
+; X32-NEXT:    movl %eax, %edi
+; X32-NEXT:    addl %ebp, %edi
 ; X32-NEXT:    adcl $0, %esi
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl 4(%eax), %ecx
-; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %ecx, %edi
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    addl %ebx, %eax
+; X32-NEXT:    movl 4(%eax), %ebp
+; X32-NEXT:    movl %ebx, %eax
+; X32-NEXT:    mull %ebp
+; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl %edx, %ebx
+; X32-NEXT:    addl %edi, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %esi, %ecx
+; X32-NEXT:    adcl %esi, %ebx
 ; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    mull %edi
-; X32-NEXT:    movl %edx, %ebx
-; X32-NEXT:    movl %eax, %edi
-; X32-NEXT:    addl %ecx, %edi
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
-; X32-NEXT:    adcl %eax, %ebx
-; X32-NEXT:    xorl %edx, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    mull %edx
-; X32-NEXT:    movl %edx, %ebp
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %eax, %esi
-; X32-NEXT:    movl %eax, (%esp) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    xorl %edx, %edx
-; X32-NEXT:    mull %edx
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    addl %esi, %eax
-; X32-NEXT:    adcl %ebp, %edx
-; X32-NEXT:    addl %edi, %eax
+; X32-NEXT:    mull %ebp
+; X32-NEXT:    addl %ebx, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %ebx, %edx
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; X32-NEXT:    adcl %eax, %edx
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X32-NEXT:    movl (%esi), %ebp
-; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X32-NEXT:    movl (%edi), %eax
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %ecx, %edi
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl %edx, %esi
+; X32-NEXT:    movl 4(%edi), %eax
+; X32-NEXT:    movl %eax, %edi
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %edx, %ebx
+; X32-NEXT:    movl %eax, %ebp
+; X32-NEXT:    addl %esi, %ebp
+; X32-NEXT:    adcl $0, %ebx
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X32-NEXT:    mull %esi
 ; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    movl 4(%esi), %esi
-; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    mull %edi
-; X32-NEXT:    movl %edx, %edi
-; X32-NEXT:    movl %eax, %ebx
-; X32-NEXT:    addl %ecx, %ebx
-; X32-NEXT:    adcl $0, %edi
-; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    mull %ebp
-; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    addl %ebx, %eax
+; X32-NEXT:    addl %ebp, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edi, %ecx
+; X32-NEXT:    adcl %ebx, %ecx
 ; X32-NEXT:    setb %bl
-; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    mull %ebp
-; X32-NEXT:    movl %edx, %edi
+; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    mull %esi
 ; X32-NEXT:    movl %eax, %ebp
 ; X32-NEXT:    addl %ecx, %ebp
 ; X32-NEXT:    movzbl %bl, %eax
-; X32-NEXT:    adcl %eax, %edi
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    xorl %edx, %edx
-; X32-NEXT:    mull %edx
+; X32-NEXT:    adcl %eax, %edx
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl (%esp), %ecx # 4-byte Reload
-; X32-NEXT:    addl %eax, %ecx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %edx, %eax
-; X32-NEXT:    addl %ebp, %ecx
-; X32-NEXT:    adcl %edi, %eax
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl %ecx, (%esp) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
 ; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X32-NEXT:    movl 8(%edi), %ebx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl 8(%eax), %ecx
+; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    movl %esi, %ecx
-; X32-NEXT:    mull %ebx
+; X32-NEXT:    mull %ecx
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    movl %edx, %ebp
-; X32-NEXT:    movl %eax, %esi
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    adcl $0, %ebp
-; X32-NEXT:    movl 12(%edi), %edi
-; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    mull %edi
-; X32-NEXT:    movl %edi, %ecx
-; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %edx, %edi
-; X32-NEXT:    addl %esi, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %ebp, %edi
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %edx, %ebx
+; X32-NEXT:    movl %eax, %edi
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT:    adcl $0, %ebx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl 12(%eax), %ecx
+; X32-NEXT:    movl %esi, %eax
 ; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %ecx, (%esp) # 4-byte Spill
 ; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    movl %eax, %ebp
-; X32-NEXT:    addl %edi, %ebp
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
-; X32-NEXT:    adcl %eax, %esi
-; X32-NEXT:    movl %ebx, %edi
-; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    xorl %ecx, %ecx
+; X32-NEXT:    addl %edi, %eax
+; X32-NEXT:    movl %eax, %edi
+; X32-NEXT:    adcl %ebx, %esi
+; X32-NEXT:    setb %bl
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    addl %esi, %ecx
+; X32-NEXT:    movzbl %bl, %eax
+; X32-NEXT:    adcl %eax, %edx
+; X32-NEXT:    addl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, %ecx
+; X32-NEXT:    adcl $0, %edx
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    addl %eax, %ebx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %edx, %eax
-; X32-NEXT:    addl %ebp, %ebx
-; X32-NEXT:    adcl %esi, %eax
-; X32-NEXT:    movl (%esp), %ecx # 4-byte Reload
-; X32-NEXT:    addl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl $0, %ebx
-; X32-NEXT:    adcl $0, %eax
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    setb (%esp) # 1-byte Folded Spill
+; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
 ; X32-NEXT:    mull %edi
-; X32-NEXT:    movl %edx, %ebx
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X32-NEXT:    mull %edi
-; X32-NEXT:    movl %edx, %edi
+; X32-NEXT:    movl %edx, %ebx
 ; X32-NEXT:    movl %eax, %ebp
-; X32-NEXT:    addl %ebx, %ebp
-; X32-NEXT:    adcl $0, %edi
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X32-NEXT:    adcl $0, %ebx
 ; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    mull %ebx
+; X32-NEXT:    movl (%esp), %edi # 4-byte Reload
+; X32-NEXT:    mull %edi
 ; X32-NEXT:    movl %edx, %esi
 ; X32-NEXT:    addl %ebp, %eax
 ; X32-NEXT:    movl %eax, %ebp
-; X32-NEXT:    adcl %edi, %esi
+; X32-NEXT:    adcl %ebx, %esi
 ; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    mull %ebx
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edi, %ebx
 ; X32-NEXT:    addl %esi, %eax
 ; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 1-byte Folded Reload
 ; X32-NEXT:    adcl %esi, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %esi
-; X32-NEXT:    adcl %edx, %ecx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    addl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    addl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
 ; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movzbl (%esp), %eax # 1-byte Folded Reload
-; X32-NEXT:    adcl %eax, %esi
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl $0, %ecx
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    movl 16(%ecx), %esi
-; X32-NEXT:    imull %esi, %ebx
-; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    mull %edi
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
+; X32-NEXT:    adcl %ecx, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, %edx
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT:    movl 16(%ecx), %edi
+; X32-NEXT:    imull %edi, %ebx
+; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X32-NEXT:    mull %ebp
+; X32-NEXT:    movl %eax, (%esp) # 4-byte Spill
 ; X32-NEXT:    addl %ebx, %edx
 ; X32-NEXT:    movl 20(%ecx), %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    imull %eax, %edi
-; X32-NEXT:    addl %edx, %edi
-; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    imull %eax, %ebp
+; X32-NEXT:    addl %edx, %ebp
 ; X32-NEXT:    movl 24(%ecx), %eax
-; X32-NEXT:    movl %ecx, %ebp
-; X32-NEXT:    movl %eax, %edi
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    imull %ecx, %edi
+; X32-NEXT:    movl %eax, %ecx
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    movl %eax, (%esp) # 4-byte Spill
-; X32-NEXT:    addl %edi, %edx
-; X32-NEXT:    movl 28(%ebp), %ebp
-; X32-NEXT:    imull %ebx, %ebp
-; X32-NEXT:    addl %edx, %ebp
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X32-NEXT:    addl %edx, (%esp) # 4-byte Folded Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    movl %ebx, %eax
+; X32-NEXT:    imull %ebx, %ecx
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %ebx
+; X32-NEXT:    addl %ecx, %edx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT:    movl 28(%ecx), %ecx
+; X32-NEXT:    imull %esi, %ecx
+; X32-NEXT:    addl %edx, %ecx
+; X32-NEXT:    addl (%esp), %eax # 4-byte Folded Reload
+; X32-NEXT:    movl %eax, (%esp) # 4-byte Spill
+; X32-NEXT:    adcl %ebp, %ecx
+; X32-NEXT:    movl %esi, %eax
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edx, %ebp
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    movl %eax, %edi
-; X32-NEXT:    addl %ebx, %edi
-; X32-NEXT:    adcl $0, %ecx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    mull %ebx
+; X32-NEXT:    movl %ebx, %eax
+; X32-NEXT:    mull %edi
 ; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    addl %edi, %eax
+; X32-NEXT:    movl %eax, %ebx
+; X32-NEXT:    addl %ebp, %ebx
+; X32-NEXT:    adcl $0, %esi
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X32-NEXT:    mull %ebp
+; X32-NEXT:    movl %edx, %edi
+; X32-NEXT:    addl %ebx, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %ecx, %esi
-; X32-NEXT:    setb %cl
+; X32-NEXT:    adcl %esi, %edi
+; X32-NEXT:    setb %bl
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    addl %esi, %eax
-; X32-NEXT:    movzbl %cl, %ecx
-; X32-NEXT:    adcl %ecx, %edx
+; X32-NEXT:    mull %ebp
+; X32-NEXT:    addl %edi, %eax
+; X32-NEXT:    movzbl %bl, %esi
+; X32-NEXT:    adcl %esi, %edx
 ; X32-NEXT:    addl (%esp), %eax # 4-byte Folded Reload
 ; X32-NEXT:    movl %eax, (%esp) # 4-byte Spill
-; X32-NEXT:    adcl %ebp, %edx
+; X32-NEXT:    adcl %ecx, %edx
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %ebx
 ; X32-NEXT:    movl 28(%ebx), %ecx
@@ -348,7 +291,7 @@ define void @test(i256* %a, i256* %b, i256* %out) #0 {
 ; X32-NEXT:    movl %esi, 20(%ecx)
 ; X32-NEXT:    movl %eax, 24(%ecx)
 ; X32-NEXT:    movl %edx, 28(%ecx)
-; X32-NEXT:    addl $88, %esp
+; X32-NEXT:    addl $72, %esp
 ; X32-NEXT:    .cfi_def_cfa_offset 20
 ; X32-NEXT:    popl %esi
 ; X32-NEXT:    .cfi_def_cfa_offset 16
diff --git a/llvm/test/CodeGen/X86/mul-i512.ll b/llvm/test/CodeGen/X86/mul-i512.ll
index a0ed13517..a5050467a 100644
--- a/llvm/test/CodeGen/X86/mul-i512.ll
+++ b/llvm/test/CodeGen/X86/mul-i512.ll
@@ -9,453 +9,262 @@ define void @test_512(i512* %a, i512* %b, i512* %out) nounwind {
 ; X32-NEXT:    pushl %ebx
 ; X32-NEXT:    pushl %edi
 ; X32-NEXT:    pushl %esi
-; X32-NEXT:    subl $244, %esp
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    movl 20(%ecx), %edi
-; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl 16(%ecx), %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %ecx, %ebp
-; X32-NEXT:    xorl %ebx, %ebx
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    movl %eax, %esi
-; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    movl %eax, %edi
-; X32-NEXT:    addl %ecx, %edi
-; X32-NEXT:    movl %edx, %ebx
-; X32-NEXT:    adcl $0, %ebx
-; X32-NEXT:    addl %esi, %edi
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %ecx, %ebx
-; X32-NEXT:    movl %ecx, %edi
+; X32-NEXT:    subl $180, %esp
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT:    movl 28(%edx), %ecx
 ; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    setb %cl
-; X32-NEXT:    addl %eax, %ebx
-; X32-NEXT:    movzbl %cl, %ecx
-; X32-NEXT:    adcl %edx, %ecx
-; X32-NEXT:    movl 24(%ebp), %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    xorl %edx, %edx
-; X32-NEXT:    mull %edx
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    addl %eax, %esi
-; X32-NEXT:    adcl %edx, %edi
-; X32-NEXT:    addl %ebx, %esi
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %ecx, %edi
+; X32-NEXT:    movl 24(%edx), %ebp
+; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl (%eax), %edi
 ; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    movl (%ecx), %eax
+; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edx, %esi
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    xorl %ebx, %ebx
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    movl %eax, %ebp
+; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    mull %edi
 ; X32-NEXT:    movl %edx, %edi
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl 4(%ecx), %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %ecx, %esi
-; X32-NEXT:    mull %ebx
 ; X32-NEXT:    movl %eax, %ebx
-; X32-NEXT:    addl %edi, %ebx
-; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    adcl $0, %ecx
-; X32-NEXT:    addl %ebp, %ebx
-; X32-NEXT:    movl %ebx, (%esp) # 4-byte Spill
-; X32-NEXT:    adcl %edi, %ecx
-; X32-NEXT:    setb %bl
-; X32-NEXT:    addl %eax, %ecx
-; X32-NEXT:    movzbl %bl, %ebx
-; X32-NEXT:    adcl %edx, %ebx
-; X32-NEXT:    movl 8(%esi), %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    xorl %edx, %edx
-; X32-NEXT:    mull %edx
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %ebp, %esi
-; X32-NEXT:    addl %eax, %esi
-; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    adcl %edx, %eax
-; X32-NEXT:    addl %ecx, %esi
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %ebx, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    addl %ebp, %eax
-; X32-NEXT:    movl %ebp, %esi
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %edi, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    movl (%ecx), %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    xorl %ebp, %ebp
-; X32-NEXT:    mull %ebp
-; X32-NEXT:    movl %edx, %ebx
-; X32-NEXT:    movl %eax, %ecx
-; X32-NEXT:    movl %eax, %edx
-; X32-NEXT:    addl %esi, %edx
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    adcl %edi, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    addl %esi, %ebx
+; X32-NEXT:    adcl $0, %edi
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl 16(%eax), %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    mull %ebp
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %ecx, %edi
+; X32-NEXT:    movl 4(%eax), %ecx
+; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    mull %ecx
 ; X32-NEXT:    movl %ecx, %ebp
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    addl %eax, %edi
-; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    adcl %edx, %eax
+; X32-NEXT:    movl %edx, %esi
+; X32-NEXT:    addl %ebx, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl %edi, %esi
+; X32-NEXT:    setb %cl
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    mull %ebp
 ; X32-NEXT:    addl %esi, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl (%esp), %eax # 4-byte Folded Reload
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl 4(%eax), %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    xorl %edi, %edi
-; X32-NEXT:    mull %edi
-; X32-NEXT:    movl %eax, %esi
-; X32-NEXT:    addl %ebx, %esi
-; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    adcl $0, %ecx
-; X32-NEXT:    addl %ebp, %esi
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %ebx, %ecx
-; X32-NEXT:    movl %ebx, %esi
-; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    setb %bl
-; X32-NEXT:    addl %eax, %ecx
-; X32-NEXT:    movzbl %bl, %ebx
-; X32-NEXT:    adcl %edx, %ebx
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl 8(%eax), %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    mull %edi
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movzbl %cl, %eax
+; X32-NEXT:    adcl %eax, %edx
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %ebp, %edi
-; X32-NEXT:    addl %eax, %edi
-; X32-NEXT:    adcl %edx, %esi
-; X32-NEXT:    addl %ecx, %edi
-; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %ebx, %esi
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %eax, (%esp) # 4-byte Folded Spill
-; X32-NEXT:    adcl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X32-NEXT:    movl 20(%esi), %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    xorl %ecx, %ecx
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %eax, %ebx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    addl %edi, %ebx
-; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    adcl $0, %ecx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    addl %ebp, %ebx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT:    movl 16(%ecx), %ebx
+; X32-NEXT:    movl %ebx, %eax
 ; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edi, %ecx
-; X32-NEXT:    setb %bl
-; X32-NEXT:    addl %eax, %ecx
-; X32-NEXT:    movzbl %bl, %ebx
-; X32-NEXT:    adcl %edx, %ebx
-; X32-NEXT:    movl 24(%esi), %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    xorl %edx, %edx
-; X32-NEXT:    mull %edx
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %ebp, %esi
-; X32-NEXT:    addl %eax, %esi
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    adcl %edx, %edi
-; X32-NEXT:    addl %ecx, %esi
-; X32-NEXT:    adcl %ebx, %edi
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    addl %ebp, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %ebp, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    adcl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    addl %edx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    addl %edx, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    movl %ecx, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
 ; X32-NEXT:    mull %edi
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %edx, %ebp
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl 28(%eax), %esi
-; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl %edx, %esi
+; X32-NEXT:    movl 20(%ecx), %eax
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    mull %edi
 ; X32-NEXT:    movl %edx, %edi
-; X32-NEXT:    movl %eax, %ebx
-; X32-NEXT:    addl %ebp, %ebx
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    addl %esi, %ecx
 ; X32-NEXT:    adcl $0, %edi
-; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %ebx, %eax
+; X32-NEXT:    movl %ebp, %esi
+; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    mull %ebp
 ; X32-NEXT:    movl %edx, %ebp
-; X32-NEXT:    addl %ebx, %eax
+; X32-NEXT:    addl %ecx, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    adcl %edi, %ebp
-; X32-NEXT:    setb %bl
-; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    addl %ebp, %eax
-; X32-NEXT:    movzbl %bl, %ecx
-; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %ecx
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edx, %esi
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X32-NEXT:    setb %cl
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X32-NEXT:    movl %ebx, %eax
 ; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %eax, %esi
+; X32-NEXT:    addl %ebp, %esi
+; X32-NEXT:    movzbl %cl, %eax
+; X32-NEXT:    adcl %eax, %edx
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl 8(%eax), %ebp
+; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
 ; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %ebx
-; X32-NEXT:    movl %eax, %ebp
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    adcl $0, %ebx
-; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    mull %esi
+; X32-NEXT:    mull %ebp
 ; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    addl %ebp, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %ebx, %ecx
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl %edi, %ebp
-; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %edi
+; X32-NEXT:    movl %ebx, %eax
+; X32-NEXT:    mull %ebp
+; X32-NEXT:    movl %edx, %ebp
 ; X32-NEXT:    movl %eax, %ebx
 ; X32-NEXT:    addl %ecx, %ebx
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
-; X32-NEXT:    adcl %eax, %edi
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    movl %eax, %ebp
-; X32-NEXT:    addl %ecx, %ebp
-; X32-NEXT:    adcl $0, %esi
+; X32-NEXT:    adcl $0, %ebp
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-NEXT:    movl 12(%eax), %ecx
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    movl %edi, %eax
 ; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    addl %ebp, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %esi, %ecx
+; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl %edx, %edi
+; X32-NEXT:    addl %ebx, %eax
+; X32-NEXT:    movl %eax, %ebx
+; X32-NEXT:    adcl %ebp, %edi
 ; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
-; X32-NEXT:    addl %ecx, %eax
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
-; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %ebp
-; X32-NEXT:    adcl %edx, %ecx
-; X32-NEXT:    addl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl $0, %ebp
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    addl %edi, %ecx
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; X32-NEXT:    adcl %eax, %edx
+; X32-NEXT:    addl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    adcl $0, %ecx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X32-NEXT:    adcl $0, %edx
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    movl %ebx, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %esi, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NEXT:    mull %edi
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %eax, %edi
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edx, %ebp
+; X32-NEXT:    movl %eax, %ebx
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X32-NEXT:    adcl $0, %ebp
+; X32-NEXT:    movl %esi, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edx, %esi
+; X32-NEXT:    addl %ebx, %eax
+; X32-NEXT:    movl %eax, %ebx
+; X32-NEXT:    adcl %ebp, %esi
+; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    mull %edi
+; X32-NEXT:    addl %esi, %eax
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 1-byte Folded Reload
+; X32-NEXT:    adcl %esi, %edx
+; X32-NEXT:    addl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
+; X32-NEXT:    adcl %ecx, %eax
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    adcl $0, %edx
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT:    movl 8(%ecx), %ebx
 ; X32-NEXT:    movl %ebx, %eax
+; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %ebx
-; X32-NEXT:    addl %edi, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    movl %edx, %ebp
+; X32-NEXT:    movl 12(%ecx), %ecx
+; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    mull %esi
-; X32-NEXT:    addl %ebx, %eax
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 1-byte Folded Reload
-; X32-NEXT:    adcl %esi, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT:    movl %edx, %esi
+; X32-NEXT:    movl %eax, %edi
+; X32-NEXT:    addl %ebp, %edi
+; X32-NEXT:    adcl $0, %esi
+; X32-NEXT:    movl %ebx, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %edi
-; X32-NEXT:    adcl %edx, %ebx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X32-NEXT:    addl %ebp, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    adcl %ecx, %esi
+; X32-NEXT:    mull %ebx
+; X32-NEXT:    movl %edx, %ebp
+; X32-NEXT:    addl %edi, %eax
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl %esi, %ebp
+; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    mull %ebx
+; X32-NEXT:    addl %ebp, %eax
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
-; X32-NEXT:    adcl %eax, %edi
-; X32-NEXT:    adcl $0, %ebx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT:    adcl %eax, %edx
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT:    movl (%ecx), %esi
+; X32-NEXT:    movl %esi, %eax
 ; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    movl %ecx, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
 ; X32-NEXT:    mull %edi
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl 12(%eax), %eax
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl 4(%ecx), %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    mull %edi
 ; X32-NEXT:    movl %edx, %edi
 ; X32-NEXT:    movl %eax, %ebp
-; X32-NEXT:    addl %esi, %ebp
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
 ; X32-NEXT:    adcl $0, %edi
-; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X32-NEXT:    movl %esi, %eax
 ; X32-NEXT:    mull %ebx
-; X32-NEXT:    movl %edx, %esi
+; X32-NEXT:    movl %ebx, %esi
+; X32-NEXT:    movl %edx, %ecx
 ; X32-NEXT:    addl %ebp, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edi, %esi
-; X32-NEXT:    setb %cl
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    addl %esi, %eax
-; X32-NEXT:    movzbl %cl, %ecx
-; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %ecx
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edx, %esi
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    movl %esi, %eax
+; X32-NEXT:    adcl %edi, %ecx
+; X32-NEXT:    setb %bl
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %eax, %esi
+; X32-NEXT:    addl %ecx, %esi
+; X32-NEXT:    movzbl %bl, %eax
+; X32-NEXT:    adcl %eax, %edx
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NEXT:    mull %edi
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    movl %ebx, %eax
+; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edx, %ebp
+; X32-NEXT:    movl %eax, %ebx
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X32-NEXT:    adcl $0, %ebp
+; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X32-NEXT:    mull %ecx
 ; X32-NEXT:    movl %edx, %edi
-; X32-NEXT:    movl %eax, %ebp
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    adcl $0, %edi
-; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    addl %ebp, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edi, %ecx
+; X32-NEXT:    addl %ebx, %eax
+; X32-NEXT:    movl %eax, %ebx
+; X32-NEXT:    adcl %ebp, %edi
 ; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl %ebx, %edi
-; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    mull %esi
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    mull %ecx
 ; X32-NEXT:    movl %eax, %ebp
-; X32-NEXT:    addl %ecx, %ebp
+; X32-NEXT:    addl %edi, %ebp
 ; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
 ; X32-NEXT:    adcl %eax, %edx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT:    addl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, %ebp
+; X32-NEXT:    adcl $0, %edx
 ; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    mull %ebx
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    mull %ecx
 ; X32-NEXT:    movl %edx, %edi
 ; X32-NEXT:    movl %eax, %ebx
-; X32-NEXT:    addl %ecx, %ebx
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
 ; X32-NEXT:    adcl $0, %edi
 ; X32-NEXT:    movl %esi, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
@@ -469,112 +278,32 @@ define void @test_512(i512* %a, i512* %b, i512* %out) nounwind {
 ; X32-NEXT:    mull %ecx
 ; X32-NEXT:    addl %esi, %eax
 ; X32-NEXT:    movzbl %bl, %ecx
-; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X32-NEXT:    movl %edx, %esi
+; X32-NEXT:    adcl %ecx, %esi
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %ebx
-; X32-NEXT:    adcl %edx, %ecx
-; X32-NEXT:    addl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl $0, %ebx
-; X32-NEXT:    adcl $0, %ecx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %eax, %edi
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    adcl $0, %edx
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %ebp
-; X32-NEXT:    addl %edi, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    addl %ebp, %eax
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 1-byte Folded Reload
-; X32-NEXT:    adcl %esi, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %ebp
-; X32-NEXT:    adcl %edx, %edi
+; X32-NEXT:    addl %ebp, %ecx
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X32-NEXT:    addl %ebx, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    adcl %ecx, %esi
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
-; X32-NEXT:    adcl %eax, %ebp
-; X32-NEXT:    adcl $0, %edi
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    adcl (%esp), %esi # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 1-byte Folded Reload
+; X32-NEXT:    adcl %edi, %eax
+; X32-NEXT:    adcl $0, %esi
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
 ; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
 ; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
 ; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
 ; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    movl %eax, (%esp) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %edi
-; X32-NEXT:    movl %eax, %ecx
-; X32-NEXT:    addl %esi, %ecx
-; X32-NEXT:    adcl $0, %edi
-; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    addl %ecx, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edi, %esi
-; X32-NEXT:    setb %cl
-; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    addl %esi, %eax
-; X32-NEXT:    movzbl %cl, %ecx
-; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %ecx
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edx, %esi
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl 16(%eax), %esi
 ; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
 ; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X32-NEXT:    mull %esi
 ; X32-NEXT:    movl %edx, %ecx
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
@@ -585,9 +314,12 @@ define void @test_512(i512* %a, i512* %b, i512* %out) nounwind {
 ; X32-NEXT:    movl %eax, %edi
 ; X32-NEXT:    addl %ecx, %edi
 ; X32-NEXT:    adcl $0, %esi
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl 20(%eax), %ecx
 ; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    mull %ebx
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %ecx, %ebx
+; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl %edx, %ecx
 ; X32-NEXT:    addl %edi, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
@@ -595,272 +327,267 @@ define void @test_512(i512* %a, i512* %b, i512* %out) nounwind {
 ; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
 ; X32-NEXT:    movl %ebp, %eax
 ; X32-NEXT:    mull %ebx
-; X32-NEXT:    movl %eax, %edi
-; X32-NEXT:    addl %ecx, %edi
+; X32-NEXT:    addl %ecx, %eax
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
 ; X32-NEXT:    adcl %eax, %edx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    addl (%esp), %edi # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    movl %ebx, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, %esi
+; X32-NEXT:    movl %eax, %ebx
+; X32-NEXT:    addl %ecx, %ebx
+; X32-NEXT:    adcl $0, %esi
+; X32-NEXT:    movl %ebp, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, (%esp) # 4-byte Spill
+; X32-NEXT:    movl %edx, %ebp
+; X32-NEXT:    addl %ebx, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    adcl %esi, %ebp
+; X32-NEXT:    setb %bl
+; X32-NEXT:    movl %edi, %eax
 ; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %esi
 ; X32-NEXT:    movl %eax, %ecx
-; X32-NEXT:    addl (%esp), %ecx # 4-byte Folded Reload
-; X32-NEXT:    adcl $0, %esi
+; X32-NEXT:    addl %ebp, %ecx
+; X32-NEXT:    movzbl %bl, %eax
+; X32-NEXT:    adcl %eax, %edx
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl 28(%eax), %ebp
-; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    mull %ebp
-; X32-NEXT:    movl %ebp, (%esp) # 4-byte Spill
-; X32-NEXT:    movl %edx, %ebx
-; X32-NEXT:    addl %ecx, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %esi, %ebx
-; X32-NEXT:    setb %cl
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %ebp
-; X32-NEXT:    addl %ebx, %eax
-; X32-NEXT:    movzbl %cl, %ecx
-; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %ebp
-; X32-NEXT:    adcl %edx, %ecx
-; X32-NEXT:    addl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl $0, %ebp
-; X32-NEXT:    adcl $0, %ecx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X32-NEXT:    movl 24(%eax), %ebx
+; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    mull %edi
+; X32-NEXT:    mull %ebx
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    mull %ebx
+; X32-NEXT:    movl %edx, %ebp
+; X32-NEXT:    movl %eax, %edi
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT:    adcl $0, %ebp
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl 28(%eax), %ebx
+; X32-NEXT:    movl %esi, %eax
+; X32-NEXT:    mull %ebx
+; X32-NEXT:    movl %edx, %esi
+; X32-NEXT:    addl %edi, %eax
+; X32-NEXT:    movl %eax, %edi
+; X32-NEXT:    adcl %ebp, %esi
+; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %edi
-; X32-NEXT:    movl %edx, %edi
+; X32-NEXT:    mull %ebx
+; X32-NEXT:    movl %ebx, %ebp
 ; X32-NEXT:    movl %eax, %ebx
+; X32-NEXT:    addl %esi, %ebx
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; X32-NEXT:    adcl %eax, %edx
+; X32-NEXT:    addl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, %ebx
+; X32-NEXT:    adcl $0, %edx
 ; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    adcl $0, %edi
-; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    movl (%esp), %esi # 4-byte Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT:    movl %edx, (%esp) # 4-byte Spill
+; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X32-NEXT:    mull %esi
-; X32-NEXT:    addl %ebx, %eax
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edi, %edx
-; X32-NEXT:    movl %edx, %edi
-; X32-NEXT:    setb %bl
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, %esi
+; X32-NEXT:    movl %eax, %edi
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT:    adcl $0, %esi
+; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    movl %ebp, %ecx
+; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    mull %ebp
+; X32-NEXT:    movl %edx, %ebp
 ; X32-NEXT:    addl %edi, %eax
-; X32-NEXT:    movzbl %bl, %esi
-; X32-NEXT:    adcl %esi, %edx
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl %esi, %ebp
+; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    addl %ebp, %eax
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
+; X32-NEXT:    adcl %ecx, %edx
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT:    addl %ebx, %edi
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %edi
-; X32-NEXT:    adcl %edx, %ebx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X32-NEXT:    addl %ebp, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    adcl %ecx, %esi
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
-; X32-NEXT:    adcl %eax, %edi
-; X32-NEXT:    adcl $0, %ebx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    addl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl (%esp), %ebx # 4-byte Folded Reload
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
+; X32-NEXT:    adcl %ecx, %eax
+; X32-NEXT:    adcl $0, %edx
+; X32-NEXT:    movl %edx, %esi
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    addl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    adcl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    adcl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    adcl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl %edi, %edx
 ; X32-NEXT:    adcl $0, %edx
+; X32-NEXT:    movl %ebx, %ecx
+; X32-NEXT:    adcl $0, %ecx
+; X32-NEXT:    adcl $0, %eax
 ; X32-NEXT:    adcl $0, %esi
-; X32-NEXT:    adcl $0, %edi
-; X32-NEXT:    adcl $0, %ebx
 ; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
 ; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    movl %esi, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    mull %edi
-; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    mull %edi
-; X32-NEXT:    movl %edx, %ebx
-; X32-NEXT:    movl %eax, %edi
-; X32-NEXT:    addl %ecx, %edi
-; X32-NEXT:    adcl $0, %ebx
-; X32-NEXT:    movl %esi, %eax
+; X32-NEXT:    movl %edi, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X32-NEXT:    mull %esi
 ; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    addl %edi, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %ebx, %ecx
-; X32-NEXT:    setb %bl
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
 ; X32-NEXT:    movl %ebp, %eax
 ; X32-NEXT:    mull %esi
-; X32-NEXT:    addl %ecx, %eax
-; X32-NEXT:    movzbl %bl, %ecx
-; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %esi
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edx, %ecx
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    movl %edx, %esi
+; X32-NEXT:    movl %eax, %ebx
+; X32-NEXT:    addl %ecx, %ebx
+; X32-NEXT:    adcl $0, %esi
+; X32-NEXT:    movl %edi, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
 ; X32-NEXT:    mull %edi
-; X32-NEXT:    movl %edx, %esi
+; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    addl %ebx, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X32-NEXT:    adcl %esi, %ecx
+; X32-NEXT:    setb %bl
 ; X32-NEXT:    movl %ebp, %eax
 ; X32-NEXT:    mull %edi
-; X32-NEXT:    movl %edx, %edi
-; X32-NEXT:    movl %eax, %ebx
-; X32-NEXT:    addl %esi, %ebx
-; X32-NEXT:    adcl $0, %edi
-; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    addl %ecx, %eax
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movzbl %bl, %eax
+; X32-NEXT:    adcl %eax, %edx
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NEXT:    movl %edi, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X32-NEXT:    mull %esi
 ; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    addl %ebx, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edi, %ecx
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
 ; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    movl %ebp, %edi
 ; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, %esi
 ; X32-NEXT:    movl %eax, %ebx
 ; X32-NEXT:    addl %ecx, %ebx
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; X32-NEXT:    adcl $0, %esi
+; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %edx, %edi
+; X32-NEXT:    addl %ebx, %eax
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl %esi, %edi
+; X32-NEXT:    setb %bl
+; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %eax, %esi
+; X32-NEXT:    addl %edi, %esi
+; X32-NEXT:    movzbl %bl, %eax
 ; X32-NEXT:    adcl %eax, %edx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
 ; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    movl %ebp, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %esi
+; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edx, (%esp) # 4-byte Spill
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %edi
-; X32-NEXT:    movl %eax, %ecx
-; X32-NEXT:    addl %esi, %ecx
-; X32-NEXT:    adcl $0, %edi
 ; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    movl (%esp), %esi # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %ebp
-; X32-NEXT:    addl %ecx, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edi, %ebp
-; X32-NEXT:    setb %cl
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %esi
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edx, %ebx
+; X32-NEXT:    movl %eax, %ebp
+; X32-NEXT:    addl (%esp), %ebp # 4-byte Folded Reload
+; X32-NEXT:    adcl $0, %ebx
+; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edx, %ecx
 ; X32-NEXT:    addl %ebp, %eax
-; X32-NEXT:    movzbl %cl, %ecx
-; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %esi
-; X32-NEXT:    adcl %edx, %ecx
-; X32-NEXT:    addl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl %eax, %ebp
+; X32-NEXT:    adcl %ebx, %ecx
+; X32-NEXT:    setb (%esp) # 1-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl $0, %esi
-; X32-NEXT:    adcl $0, %ecx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edi, %ebx
+; X32-NEXT:    movl %eax, %edi
+; X32-NEXT:    addl %ecx, %edi
+; X32-NEXT:    movzbl (%esp), %eax # 1-byte Folded Reload
+; X32-NEXT:    adcl %eax, %edx
+; X32-NEXT:    addl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, %edi
+; X32-NEXT:    adcl $0, %edx
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT:    movl %edx, (%esp) # 4-byte Spill
 ; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    movl %ecx, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X32-NEXT:    mull %esi
 ; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    movl %eax, %edi
-; X32-NEXT:    addl %ecx, %edi
+; X32-NEXT:    movl %eax, %ebp
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
 ; X32-NEXT:    adcl $0, %esi
-; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    movl (%esp), %ebx # 4-byte Reload
+; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    movl %ebx, %ecx
 ; X32-NEXT:    mull %ebx
-; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    movl %edx, %ebx
+; X32-NEXT:    addl %ebp, %eax
 ; X32-NEXT:    movl %eax, %ebp
-; X32-NEXT:    addl %edi, %ebp
-; X32-NEXT:    adcl %esi, %ecx
+; X32-NEXT:    adcl %esi, %ebx
 ; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    addl %ecx, %eax
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    addl %ebx, %eax
 ; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
 ; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %esi
-; X32-NEXT:    adcl %edx, %ecx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
-; X32-NEXT:    adcl %eax, %esi
-; X32-NEXT:    adcl $0, %ecx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X32-NEXT:    addl %edi, %ecx
+; X32-NEXT:    adcl (%esp), %ebp # 4-byte Folded Reload
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 1-byte Folded Reload
+; X32-NEXT:    adcl %esi, %eax
+; X32-NEXT:    movl %eax, %esi
+; X32-NEXT:    adcl $0, %edx
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X32-NEXT:    addl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
@@ -870,232 +597,193 @@ define void @test_512(i512* %a, i512* %b, i512* %out) nounwind {
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
 ; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
-; X32-NEXT:    adcl %eax, %edx
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl %eax, %ecx
+; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    adcl $0, %ebp
 ; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    adcl $0, %esi
 ; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl $0, %ecx
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    movl 32(%ecx), %edi
+; X32-NEXT:    adcl $0, %edx
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl 32(%eax), %edi
 ; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X32-NEXT:    movl %esi, %eax
 ; X32-NEXT:    mull %edi
-; X32-NEXT:    movl %edx, %esi
+; X32-NEXT:    movl %edx, %ecx
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
 ; X32-NEXT:    movl %ebp, %eax
 ; X32-NEXT:    mull %edi
 ; X32-NEXT:    movl %edx, %edi
 ; X32-NEXT:    movl %eax, %ebx
-; X32-NEXT:    addl %esi, %ebx
+; X32-NEXT:    addl %ecx, %ebx
 ; X32-NEXT:    adcl $0, %edi
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl 36(%eax), %esi
-; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %esi, %ecx
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %edx, %esi
+; X32-NEXT:    movl 36(%eax), %ecx
+; X32-NEXT:    movl %esi, %eax
+; X32-NEXT:    movl %ecx, %esi
+; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %edx, %ecx
 ; X32-NEXT:    addl %ebx, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %edi, %esi
+; X32-NEXT:    adcl %edi, %ecx
 ; X32-NEXT:    setb %bl
 ; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    movl %eax, %ebp
-; X32-NEXT:    addl %esi, %ebp
-; X32-NEXT:    movzbl %bl, %eax
-; X32-NEXT:    adcl %eax, %ecx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    xorl %edx, %edx
-; X32-NEXT:    mull %edx
+; X32-NEXT:    mull %esi
+; X32-NEXT:    addl %ecx, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movzbl %bl, %eax
+; X32-NEXT:    adcl %eax, %edx
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    addl %eax, %edi
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    adcl %edx, %esi
-; X32-NEXT:    addl %ebp, %edi
-; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %ecx, %esi
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    movl %ebx, %esi
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    movl %edx, %edi
+; X32-NEXT:    movl %esi, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edx, %ecx
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
 ; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    movl %eax, %ebp
-; X32-NEXT:    addl %edi, %ebp
-; X32-NEXT:    adcl $0, %esi
-; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
 ; X32-NEXT:    mull %edi
-; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    movl %edx, %edi
+; X32-NEXT:    movl %eax, %ebp
+; X32-NEXT:    addl %ecx, %ebp
+; X32-NEXT:    adcl $0, %edi
+; X32-NEXT:    movl %esi, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %edx, %esi
 ; X32-NEXT:    addl %ebp, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %esi, %ecx
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X32-NEXT:    adcl %edi, %esi
+; X32-NEXT:    setb (%esp) # 1-byte Folded Spill
 ; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    mull %edi
-; X32-NEXT:    addl %ecx, %eax
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
-; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %edi
-; X32-NEXT:    adcl %edx, %ecx
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %eax, %edi
+; X32-NEXT:    addl %esi, %edi
+; X32-NEXT:    movzbl (%esp), %eax # 1-byte Folded Reload
+; X32-NEXT:    adcl %eax, %edx
 ; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
 ; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl 40(%eax), %ebp
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    mull %ebp
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl 40(%eax), %esi
+; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, (%esp) # 4-byte Spill
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    mull %ebp
-; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    movl %eax, %ecx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    adcl $0, %esi
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, %ebp
+; X32-NEXT:    movl %eax, %ebx
+; X32-NEXT:    addl (%esp), %ebx # 4-byte Folded Reload
+; X32-NEXT:    adcl $0, %ebp
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl 44(%eax), %ebx
-; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    movl %edx, %edi
-; X32-NEXT:    addl %ecx, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %esi, %edi
+; X32-NEXT:    movl 44(%eax), %esi
+; X32-NEXT:    movl %esi, (%esp) # 4-byte Spill
+; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    addl %ebx, %eax
+; X32-NEXT:    movl %eax, %ebx
+; X32-NEXT:    adcl %ebp, %ecx
 ; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, %ebp
 ; X32-NEXT:    movl %eax, %esi
-; X32-NEXT:    addl %edi, %esi
+; X32-NEXT:    addl %ecx, %esi
 ; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
-; X32-NEXT:    adcl %eax, %ecx
-; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    xorl %edx, %edx
-; X32-NEXT:    mull %edx
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    addl %eax, %edi
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %edx, %eax
-; X32-NEXT:    addl %esi, %edi
-; X32-NEXT:    adcl %ecx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    addl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl $0, %edi
-; X32-NEXT:    adcl $0, %eax
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl %eax, %ebp
+; X32-NEXT:    addl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, %esi
+; X32-NEXT:    adcl $0, %ebp
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
 ; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    mull %ebp
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NEXT:    mull %edi
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    mull %ebp
-; X32-NEXT:    movl %edx, %esi
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edx, %edi
 ; X32-NEXT:    movl %eax, %ebx
 ; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    adcl $0, %esi
+; X32-NEXT:    adcl $0, %edi
 ; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    mull %ebp
+; X32-NEXT:    mull (%esp) # 4-byte Folded Reload
 ; X32-NEXT:    movl %edx, %ecx
 ; X32-NEXT:    addl %ebx, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %esi, %ecx
-; X32-NEXT:    setb %bl
-; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    mull %ebp
+; X32-NEXT:    movl %eax, %ebx
+; X32-NEXT:    adcl %edi, %ecx
+; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    movl (%esp), %edi # 4-byte Reload
+; X32-NEXT:    mull %edi
 ; X32-NEXT:    addl %ecx, %eax
-; X32-NEXT:    movzbl %bl, %ecx
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
 ; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %esi
-; X32-NEXT:    adcl %edx, %ecx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    addl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    addl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl %ebp, %ebx
+; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
+; X32-NEXT:    adcl %ecx, %eax
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, %edx
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
-; X32-NEXT:    adcl %eax, %esi
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl $0, %ecx
+; X32-NEXT:    movl %edi, %esi
+; X32-NEXT:    imull %eax, %esi
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %eax, (%esp) # 4-byte Spill
+; X32-NEXT:    addl %esi, %edx
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X32-NEXT:    imull %ebx, %ecx
+; X32-NEXT:    addl %edx, %ecx
 ; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    imull %eax, %ebp
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    addl %ebp, %edx
-; X32-NEXT:    imull {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    addl %edx, %esi
-; X32-NEXT:    movl %esi, %ebp
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X32-NEXT:    movl %eax, %esi
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    imull %ebx, %esi
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    mull %edi
+; X32-NEXT:    imull %edi, %esi
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X32-NEXT:    mull %ebp
 ; X32-NEXT:    movl %eax, %ecx
 ; X32-NEXT:    addl %esi, %edx
-; X32-NEXT:    movl (%esp), %esi # 4-byte Reload
-; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    imull %edi, %esi
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    imull %ebp, %esi
 ; X32-NEXT:    addl %edx, %esi
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl %ecx, %edi
-; X32-NEXT:    adcl %ebp, %esi
-; X32-NEXT:    movl %esi, (%esp) # 4-byte Spill
+; X32-NEXT:    addl (%esp), %ecx # 4-byte Folded Reload
+; X32-NEXT:    movl %ecx, (%esp) # 4-byte Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X32-NEXT:    mull %ecx
 ; X32-NEXT:    movl %edx, %ebp
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %ebx, %eax
+; X32-NEXT:    movl %edi, %eax
 ; X32-NEXT:    mull %ecx
 ; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    movl %eax, %ebx
-; X32-NEXT:    addl %ebp, %ebx
+; X32-NEXT:    movl %eax, %edi
+; X32-NEXT:    addl %ebp, %edi
 ; X32-NEXT:    adcl $0, %esi
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    mull %ebp
+; X32-NEXT:    mull %ebx
+; X32-NEXT:    movl %ebx, %ebp
 ; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    addl %ebx, %eax
+; X32-NEXT:    addl %edi, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    adcl %esi, %ecx
 ; X32-NEXT:    setb %bl
@@ -1104,351 +792,312 @@ define void @test_512(i512* %a, i512* %b, i512* %out) nounwind {
 ; X32-NEXT:    addl %ecx, %eax
 ; X32-NEXT:    movzbl %bl, %ecx
 ; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    addl %edi, %eax
+; X32-NEXT:    addl (%esp), %eax # 4-byte Folded Reload
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl (%esp), %edx # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %edi
 ; X32-NEXT:    movl 60(%edi), %ecx
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X32-NEXT:    imull %eax, %ecx
 ; X32-NEXT:    movl 56(%edi), %esi
+; X32-NEXT:    movl %edi, %ebx
 ; X32-NEXT:    mull %esi
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    addl %ecx, %edx
 ; X32-NEXT:    imull {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
 ; X32-NEXT:    addl %edx, %esi
-; X32-NEXT:    movl 48(%edi), %ebx
-; X32-NEXT:    movl 52(%edi), %ebp
+; X32-NEXT:    movl 48(%edi), %edi
+; X32-NEXT:    movl 52(%ebx), %ebp
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    movl %eax, %edi
-; X32-NEXT:    imull %ebp, %edi
+; X32-NEXT:    movl %eax, %ebx
+; X32-NEXT:    imull %ebp, %ebx
 ; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    addl %edi, %edx
+; X32-NEXT:    mull %edi
+; X32-NEXT:    addl %ebx, %edx
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    imull %ebx, %ecx
+; X32-NEXT:    imull %edi, %ecx
 ; X32-NEXT:    addl %edx, %ecx
 ; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    adcl %esi, %ecx
 ; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %ebx, %eax
+; X32-NEXT:    movl %edi, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X32-NEXT:    mull %ecx
 ; X32-NEXT:    movl %edx, %esi
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl %ebp, %eax
 ; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %ebp
-; X32-NEXT:    movl %eax, %edi
-; X32-NEXT:    addl %esi, %edi
-; X32-NEXT:    adcl $0, %ebp
-; X32-NEXT:    movl %ebx, %eax
+; X32-NEXT:    movl %edx, %ebx
+; X32-NEXT:    movl %eax, %ebp
+; X32-NEXT:    addl %esi, %ebp
+; X32-NEXT:    adcl $0, %ebx
+; X32-NEXT:    movl %edi, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X32-NEXT:    mull %esi
 ; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    movl %eax, %ebx
-; X32-NEXT:    addl %edi, %ebx
-; X32-NEXT:    adcl %ebp, %ecx
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    addl %ecx, %eax
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
-; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X32-NEXT:    movl %esi, %ecx
-; X32-NEXT:    movl 40(%esi), %ebx
-; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %edx, %ebp
-; X32-NEXT:    movl 44(%ecx), %ecx
-; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %esi
 ; X32-NEXT:    movl %eax, %edi
 ; X32-NEXT:    addl %ebp, %edi
-; X32-NEXT:    adcl $0, %esi
-; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    mull %ebp
-; X32-NEXT:    movl %edx, %ebx
-; X32-NEXT:    addl %edi, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %esi, %ebx
-; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    mull %ebp
-; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    movl %eax, %edi
-; X32-NEXT:    addl %ebx, %edi
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
-; X32-NEXT:    adcl %eax, %esi
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    xorl %ecx, %ecx
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl %ebx, %ecx
+; X32-NEXT:    setb %bl
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    mull %esi
+; X32-NEXT:    addl %ecx, %eax
+; X32-NEXT:    movzbl %bl, %ecx
+; X32-NEXT:    adcl %ecx, %edx
 ; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT:    addl %edi, %eax
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %esi, %edx
-; X32-NEXT:    movl %edx, (%esp) # 4-byte Spill
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X32-NEXT:    movl 32(%esi), %edi
-; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
 ; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X32-NEXT:    movl 40(%edi), %ebp
+; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X32-NEXT:    mull %ecx
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl 36(%esi), %esi
-; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl %edx, %esi
+; X32-NEXT:    movl 44(%edi), %edi
+; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    mull %ecx
 ; X32-NEXT:    movl %edx, %ebx
-; X32-NEXT:    movl %eax, %ebp
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    addl %esi, %ecx
 ; X32-NEXT:    adcl $0, %ebx
+; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X32-NEXT:    mull %ebp
+; X32-NEXT:    movl %edx, %esi
+; X32-NEXT:    addl %ecx, %eax
+; X32-NEXT:    movl %eax, (%esp) # 4-byte Spill
+; X32-NEXT:    adcl %ebx, %esi
+; X32-NEXT:    setb %cl
 ; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %edi
-; X32-NEXT:    addl %ebp, %eax
+; X32-NEXT:    mull %ebp
+; X32-NEXT:    addl %esi, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %ebx, %edi
-; X32-NEXT:    setb %bl
+; X32-NEXT:    movzbl %cl, %eax
+; X32-NEXT:    adcl %eax, %edx
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT:    movl 32(%ecx), %esi
 ; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %ebp
-; X32-NEXT:    movl %eax, %esi
-; X32-NEXT:    addl %edi, %esi
-; X32-NEXT:    movzbl %bl, %eax
-; X32-NEXT:    adcl %eax, %ebp
+; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    xorl %ecx, %ecx
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    mull %edi
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    addl %eax, %ecx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    adcl %edx, %eax
-; X32-NEXT:    addl %esi, %ecx
-; X32-NEXT:    adcl %ebp, %eax
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl 36(%ecx), %ebp
+; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edx, %ebx
+; X32-NEXT:    movl %eax, %ecx
 ; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X32-NEXT:    adcl $0, %ebx
+; X32-NEXT:    movl %esi, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, %edi
+; X32-NEXT:    addl %ecx, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl %ebx, %edi
+; X32-NEXT:    setb %cl
+; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %eax, %ebx
+; X32-NEXT:    addl %edi, %ebx
+; X32-NEXT:    movzbl %cl, %eax
+; X32-NEXT:    adcl %eax, %edx
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X32-NEXT:    adcl (%esp), %edx # 4-byte Folded Reload
+; X32-NEXT:    movl %edx, (%esp) # 4-byte Spill
 ; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl $0, (%esp) # 4-byte Folded Spill
-; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT:    movl %ecx, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    movl %ebx, %eax
+; X32-NEXT:    movl %ebp, %eax
 ; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %esi
+; X32-NEXT:    movl %edx, %edi
 ; X32-NEXT:    movl %eax, %ebp
-; X32-NEXT:    addl %ecx, %ebp
-; X32-NEXT:    adcl $0, %esi
-; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X32-NEXT:    adcl $0, %edi
+; X32-NEXT:    movl %ecx, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %edi
+; X32-NEXT:    movl %edx, %esi
 ; X32-NEXT:    addl %ebp, %eax
 ; X32-NEXT:    movl %eax, %ebp
-; X32-NEXT:    adcl %esi, %edi
+; X32-NEXT:    adcl %edi, %esi
 ; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %ecx, %ebx
-; X32-NEXT:    addl %edi, %eax
-; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
-; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %esi
-; X32-NEXT:    adcl %edx, %ecx
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    addl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl $0, %esi
+; X32-NEXT:    mull %ecx
+; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    movl %eax, %edi
+; X32-NEXT:    addl %esi, %edi
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; X32-NEXT:    adcl %eax, %ecx
+; X32-NEXT:    addl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl (%esp), %ebp # 4-byte Folded Reload
+; X32-NEXT:    movl %ebp, (%esp) # 4-byte Spill
+; X32-NEXT:    adcl $0, %edi
 ; X32-NEXT:    adcl $0, %ecx
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl (%esp), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
 ; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    movl %ecx, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, (%esp) # 4-byte Spill
+; X32-NEXT:    movl %esi, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X32-NEXT:    mull %ebx
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    mull %esi
-; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    movl %eax, %ebp
-; X32-NEXT:    addl (%esp), %ebp # 4-byte Folded Reload
-; X32-NEXT:    adcl $0, %esi
-; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X32-NEXT:    mull %ebx
-; X32-NEXT:    movl %edx, %ecx
+; X32-NEXT:    movl %edx, %ebx
+; X32-NEXT:    movl %eax, %ebp
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X32-NEXT:    adcl $0, %ebx
+; X32-NEXT:    movl %esi, %eax
+; X32-NEXT:    mull {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; X32-NEXT:    movl %edx, %esi
 ; X32-NEXT:    addl %ebp, %eax
 ; X32-NEXT:    movl %eax, %ebp
-; X32-NEXT:    adcl %esi, %ecx
-; X32-NEXT:    setb (%esp) # 1-byte Folded Spill
-; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    addl %ecx, %eax
-; X32-NEXT:    movzbl (%esp), %ecx # 1-byte Folded Reload
-; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    addl %eax, %edi
-; X32-NEXT:    adcl %edx, %ecx
+; X32-NEXT:    adcl %ebx, %esi
+; X32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    addl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X32-NEXT:    mull %ebx
+; X32-NEXT:    addl %esi, %eax
+; X32-NEXT:    movl %eax, %esi
+; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; X32-NEXT:    adcl %eax, %edx
+; X32-NEXT:    addl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NEXT:    adcl %ecx, %ebp
 ; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
-; X32-NEXT:    adcl %eax, %edi
-; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl $0, %ecx
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl %eax, %esi
+; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl $0, %edx
+; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    movl 48(%ecx), %ebp
-; X32-NEXT:    imull %ebp, %ebx
-; X32-NEXT:    movl %ebp, %eax
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    mull %edi
+; X32-NEXT:    movl 48(%ecx), %edi
+; X32-NEXT:    imull %edi, %ebx
+; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X32-NEXT:    mull %esi
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    addl %ebx, %edx
 ; X32-NEXT:    movl 52(%ecx), %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    imull %eax, %edi
-; X32-NEXT:    addl %edx, %edi
-; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    imull %eax, %esi
+; X32-NEXT:    addl %edx, %esi
+; X32-NEXT:    movl %esi, %ebp
 ; X32-NEXT:    movl 56(%ecx), %eax
-; X32-NEXT:    movl %ecx, %ebx
-; X32-NEXT:    movl %eax, %edi
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT:    imull %esi, %edi
+; X32-NEXT:    movl %eax, %esi
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X32-NEXT:    imull %ebx, %esi
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X32-NEXT:    mull %ecx
+; X32-NEXT:    addl %esi, %edx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT:    movl 60(%esi), %esi
+; X32-NEXT:    imull %ecx, %esi
+; X32-NEXT:    addl %edx, %esi
+; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    addl %edi, %edx
-; X32-NEXT:    movl 60(%ebx), %ebx
+; X32-NEXT:    adcl %ebp, %esi
 ; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    imull %ecx, %ebx
-; X32-NEXT:    addl %edx, %ebx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    addl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NEXT:    mull %ebp
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    mull %edi
+; X32-NEXT:    movl %edx, %ebp
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    mull %ebp
+; X32-NEXT:    movl %ebx, %eax
+; X32-NEXT:    mull %edi
 ; X32-NEXT:    movl %edx, %ecx
-; X32-NEXT:    movl %eax, %edi
-; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT:    movl %eax, %ebx
+; X32-NEXT:    addl %ebp, %ebx
 ; X32-NEXT:    adcl $0, %ecx
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
 ; X32-NEXT:    mull %ebp
-; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    addl %edi, %eax
+; X32-NEXT:    movl %edx, %edi
+; X32-NEXT:    addl %ebx, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %ecx, %esi
+; X32-NEXT:    adcl %ecx, %edi
 ; X32-NEXT:    setb %cl
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X32-NEXT:    mull %ebp
-; X32-NEXT:    addl %esi, %eax
+; X32-NEXT:    addl %edi, %eax
 ; X32-NEXT:    movzbl %cl, %ecx
 ; X32-NEXT:    adcl %ecx, %edx
 ; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl %ebx, %edx
+; X32-NEXT:    adcl %esi, %edx
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT:    imull %ebp, %edi
-; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X32-NEXT:    imull %esi, %edi
+; X32-NEXT:    movl %esi, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %eax, %esi
+; X32-NEXT:    movl %eax, %ebp
 ; X32-NEXT:    addl %edi, %edx
 ; X32-NEXT:    imull {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
 ; X32-NEXT:    addl %edx, %ecx
 ; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X32-NEXT:    movl %eax, %ecx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    imull %ebx, %ecx
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    mull %edi
+; X32-NEXT:    imull %edi, %ecx
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X32-NEXT:    mull %ebx
 ; X32-NEXT:    addl %ecx, %edx
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    imull %edi, %ecx
+; X32-NEXT:    imull %ebx, %ecx
 ; X32-NEXT:    addl %edx, %ecx
-; X32-NEXT:    addl %esi, %eax
+; X32-NEXT:    addl %ebp, %eax
 ; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
 ; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    movl %ebx, %eax
+; X32-NEXT:    movl %ebx, %ebp
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, %ebx
+; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    mull %ebp
+; X32-NEXT:    mull %esi
 ; X32-NEXT:    movl %edx, %esi
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    mull %ebp
-; X32-NEXT:    movl %edx, %edi
 ; X32-NEXT:    movl %eax, %ecx
-; X32-NEXT:    addl %esi, %ecx
-; X32-NEXT:    adcl $0, %edi
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT:    addl %ebx, %ecx
+; X32-NEXT:    adcl $0, %esi
+; X32-NEXT:    movl %ebp, %eax
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
 ; X32-NEXT:    mull %ebx
-; X32-NEXT:    movl %edx, %esi
+; X32-NEXT:    movl %edx, %edi
 ; X32-NEXT:    movl %eax, %ebp
 ; X32-NEXT:    addl %ecx, %ebp
-; X32-NEXT:    adcl %edi, %esi
+; X32-NEXT:    adcl %esi, %edi
 ; X32-NEXT:    setb %cl
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X32-NEXT:    mull %ebx
 ; X32-NEXT:    movl %edx, %ebx
-; X32-NEXT:    addl %esi, %eax
+; X32-NEXT:    addl %edi, %eax
 ; X32-NEXT:    movzbl %cl, %ecx
 ; X32-NEXT:    adcl %ecx, %ebx
 ; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
@@ -1466,12 +1115,12 @@ define void @test_512(i512* %a, i512* %b, i512* %out) nounwind {
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT:    movl (%esp), %edi # 4-byte Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
 ; X32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
@@ -1481,45 +1130,45 @@ define void @test_512(i512* %a, i512* %b, i512* %out) nounwind {
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
 ; X32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
 ; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X32-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
 ; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
 ; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
 ; X32-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    movl %edi, (%ecx)
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    movl %edi, 4(%ecx)
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    movl %edi, 8(%ecx)
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    movl %edi, 12(%ecx)
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    movl %edi, 16(%ecx)
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    movl %edi, 20(%ecx)
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    movl %edi, 24(%ecx)
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    movl %edi, 28(%ecx)
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    movl %edi, 32(%ecx)
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    movl %edi, 36(%ecx)
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT:    movl %edi, 40(%ecx)
-; X32-NEXT:    movl %esi, 44(%ecx)
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X32-NEXT:    movl %esi, (%ecx)
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X32-NEXT:    movl %esi, 4(%ecx)
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X32-NEXT:    movl %esi, 8(%ecx)
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X32-NEXT:    movl %esi, 12(%ecx)
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X32-NEXT:    movl %esi, 16(%ecx)
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X32-NEXT:    movl %esi, 20(%ecx)
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X32-NEXT:    movl %esi, 24(%ecx)
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X32-NEXT:    movl %esi, 28(%ecx)
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X32-NEXT:    movl %esi, 32(%ecx)
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X32-NEXT:    movl %esi, 36(%ecx)
+; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X32-NEXT:    movl %esi, 40(%ecx)
+; X32-NEXT:    movl %edi, 44(%ecx)
 ; X32-NEXT:    movl %edx, 48(%ecx)
 ; X32-NEXT:    movl %ebp, 52(%ecx)
 ; X32-NEXT:    movl %eax, 56(%ecx)
 ; X32-NEXT:    movl %ebx, 60(%ecx)
-; X32-NEXT:    addl $244, %esp
+; X32-NEXT:    addl $180, %esp
 ; X32-NEXT:    popl %esi
 ; X32-NEXT:    popl %edi
 ; X32-NEXT:    popl %ebx
@@ -1536,292 +1185,250 @@ define void @test_512(i512* %a, i512* %b, i512* %out) nounwind {
 ; X64-NEXT:    pushq %rbx
 ; X64-NEXT:    pushq %rax
 ; X64-NEXT:    movq %rdx, (%rsp) # 8-byte Spill
-; X64-NEXT:    movq 24(%rdi), %r11
-; X64-NEXT:    movq 16(%rdi), %r15
+; X64-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    movq (%rdi), %r9
+; X64-NEXT:    movq 8(%rdi), %r15
+; X64-NEXT:    movq 24(%rdi), %r12
+; X64-NEXT:    movq 16(%rdi), %rax
 ; X64-NEXT:    movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    movq (%rsi), %rdx
-; X64-NEXT:    movq 8(%rsi), %rbp
-; X64-NEXT:    movq %r15, %rax
-; X64-NEXT:    movq %rdx, %rsi
-; X64-NEXT:    mulq %rdx
-; X64-NEXT:    movq %rdx, %r9
-; X64-NEXT:    movq %rax, %r8
-; X64-NEXT:    movq %r11, %rax
-; X64-NEXT:    movq %r11, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    mulq %rsi
-; X64-NEXT:    movq %rsi, %r10
-; X64-NEXT:    movq %rdx, %rbx
+; X64-NEXT:    movq (%rsi), %rdi
+; X64-NEXT:    movq 8(%rsi), %r14
 ; X64-NEXT:    movq %rax, %rsi
-; X64-NEXT:    addq %r9, %rsi
-; X64-NEXT:    adcq $0, %rbx
-; X64-NEXT:    movq %r15, %rax
-; X64-NEXT:    movq %r15, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    mulq %rbp
-; X64-NEXT:    movq %rdx, %rcx
-; X64-NEXT:    movq %rax, %r9
-; X64-NEXT:    addq %rsi, %r9
-; X64-NEXT:    adcq %rbx, %rcx
-; X64-NEXT:    setb %al
-; X64-NEXT:    movzbl %al, %ebx
-; X64-NEXT:    movq %r11, %rax
-; X64-NEXT:    mulq %rbp
-; X64-NEXT:    movq %rbp, %r14
-; X64-NEXT:    movq %rbp, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    movq %rdx, %rsi
-; X64-NEXT:    movq %rax, %rbp
-; X64-NEXT:    addq %rcx, %rbp
-; X64-NEXT:    adcq %rbx, %rsi
-; X64-NEXT:    xorl %ecx, %ecx
-; X64-NEXT:    movq %r10, %rbx
-; X64-NEXT:    movq %r10, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    movq %r10, %rax
-; X64-NEXT:    mulq %rcx
-; X64-NEXT:    movq %rdx, %r13
-; X64-NEXT:    movq %rax, %r10
-; X64-NEXT:    movq %r15, %rax
-; X64-NEXT:    mulq %rcx
-; X64-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    movq %rax, %r15
-; X64-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    addq %r10, %r15
-; X64-NEXT:    adcq %r13, %rdx
-; X64-NEXT:    addq %rbp, %r15
-; X64-NEXT:    adcq %rsi, %rdx
-; X64-NEXT:    movq %rdx, %r12
-; X64-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    movq (%rdi), %rcx
-; X64-NEXT:    movq %rcx, %rax
-; X64-NEXT:    mulq %rbx
 ; X64-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    movq %rdx, %r11
-; X64-NEXT:    movq 8(%rdi), %rdi
-; X64-NEXT:    movq %rdi, %rax
-; X64-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    mulq %rbx
+; X64-NEXT:    mulq %rdi
+; X64-NEXT:    movq %rdx, %rcx
+; X64-NEXT:    movq %rax, %r11
+; X64-NEXT:    movq %r12, %rax
+; X64-NEXT:    movq %r12, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    mulq %rdi
 ; X64-NEXT:    movq %rdx, %rbp
-; X64-NEXT:    movq %rax, %rsi
-; X64-NEXT:    addq %r11, %rsi
+; X64-NEXT:    movq %rax, %rbx
+; X64-NEXT:    addq %rcx, %rbx
 ; X64-NEXT:    adcq $0, %rbp
-; X64-NEXT:    movq %rcx, %rax
+; X64-NEXT:    movq %rsi, %rax
+; X64-NEXT:    mulq %r14
+; X64-NEXT:    movq %rdx, %rcx
+; X64-NEXT:    movq %rax, %r10
+; X64-NEXT:    addq %rbx, %r10
+; X64-NEXT:    adcq %rbp, %rcx
+; X64-NEXT:    setb %al
+; X64-NEXT:    movzbl %al, %esi
+; X64-NEXT:    movq %r12, %rax
 ; X64-NEXT:    mulq %r14
+; X64-NEXT:    movq %r14, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    movq %rdx, %r8
+; X64-NEXT:    movq %rax, %r13
+; X64-NEXT:    addq %rcx, %r13
+; X64-NEXT:    adcq %rsi, %r8
+; X64-NEXT:    movq %r9, %rax
+; X64-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    mulq %rdi
+; X64-NEXT:    movq %rdx, %rcx
+; X64-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    movq %r15, %rax
+; X64-NEXT:    mulq %rdi
 ; X64-NEXT:    movq %rdx, %rbx
-; X64-NEXT:    addq %rsi, %rax
+; X64-NEXT:    movq %rax, %rbp
+; X64-NEXT:    addq %rcx, %rbp
+; X64-NEXT:    adcq $0, %rbx
+; X64-NEXT:    movq %r9, %rax
+; X64-NEXT:    movq %r9, %r12
+; X64-NEXT:    movq %r9, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    mulq %r14
+; X64-NEXT:    movq %rdx, %rcx
+; X64-NEXT:    addq %rbp, %rax
 ; X64-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    adcq %rbp, %rbx
-; X64-NEXT:    setb %r11b
-; X64-NEXT:    movq %rdi, %rax
+; X64-NEXT:    adcq %rbx, %rcx
+; X64-NEXT:    setb %sil
+; X64-NEXT:    movq %r15, %rdi
+; X64-NEXT:    movq %r15, %rax
 ; X64-NEXT:    mulq %r14
-; X64-NEXT:    movq %rdx, %rsi
-; X64-NEXT:    movq %rax, %rbp
-; X64-NEXT:    addq %rbx, %rbp
-; X64-NEXT:    movzbl %r11b, %eax
-; X64-NEXT:    adcq %rax, %rsi
-; X64-NEXT:    movq %rcx, %rax
-; X64-NEXT:    xorl %edx, %edx
-; X64-NEXT:    mulq %rdx
-; X64-NEXT:    movq %rdx, %r14
-; X64-NEXT:    movq %rax, %r11
-; X64-NEXT:    addq %rax, %r10
-; X64-NEXT:    adcq %rdx, %r13
-; X64-NEXT:    addq %rbp, %r10
-; X64-NEXT:    adcq %rsi, %r13
-; X64-NEXT:    addq %r8, %r10
-; X64-NEXT:    adcq %r9, %r13
-; X64-NEXT:    adcq $0, %r15
-; X64-NEXT:    adcq $0, %r12
-; X64-NEXT:    movq %r12, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    movq %rdx, %r15
+; X64-NEXT:    movq %rax, %rbx
+; X64-NEXT:    addq %rcx, %rbx
+; X64-NEXT:    movzbl %sil, %eax
+; X64-NEXT:    adcq %rax, %r15
+; X64-NEXT:    addq %r11, %rbx
+; X64-NEXT:    adcq %r10, %r15
+; X64-NEXT:    adcq $0, %r13
+; X64-NEXT:    movq %r8, %r14
+; X64-NEXT:    adcq $0, %r14
 ; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
 ; X64-NEXT:    movq 16(%rsi), %r8
-; X64-NEXT:    movq %rcx, %rax
-; X64-NEXT:    movq %rcx, %r9
-; X64-NEXT:    movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    movq %r9, %rax
 ; X64-NEXT:    mulq %r8
-; X64-NEXT:    movq %rdx, %rdi
-; X64-NEXT:    movq %rax, %r12
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
-; X64-NEXT:    movq %rcx, %rax
+; X64-NEXT:    movq %rdx, %r10
+; X64-NEXT:    movq %rax, %r9
+; X64-NEXT:    movq %rdi, %rax
+; X64-NEXT:    movq %rdi, %r11
+; X64-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; X64-NEXT:    mulq %r8
-; X64-NEXT:    movq %rdx, %rbp
-; X64-NEXT:    movq %rax, %rbx
-; X64-NEXT:    addq %rdi, %rbx
-; X64-NEXT:    adcq $0, %rbp
+; X64-NEXT:    movq %rdx, %rcx
+; X64-NEXT:    movq %rax, %rbp
+; X64-NEXT:    addq %r10, %rbp
+; X64-NEXT:    adcq $0, %rcx
 ; X64-NEXT:    movq 24(%rsi), %rdi
-; X64-NEXT:    movq %r9, %rax
+; X64-NEXT:    movq %r12, %rax
 ; X64-NEXT:    mulq %rdi
 ; X64-NEXT:    movq %rdx, %rsi
-; X64-NEXT:    addq %rbx, %rax
-; X64-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    adcq %rbp, %rsi
-; X64-NEXT:    setb %bpl
-; X64-NEXT:    movq %rcx, %rax
-; X64-NEXT:    mulq %rdi
-; X64-NEXT:    movq %rdx, %rbx
-; X64-NEXT:    movq %rax, %r9
-; X64-NEXT:    addq %rsi, %r9
-; X64-NEXT:    movzbl %bpl, %eax
-; X64-NEXT:    adcq %rax, %rbx
-; X64-NEXT:    movq %r8, %rax
-; X64-NEXT:    xorl %ecx, %ecx
-; X64-NEXT:    mulq %rcx
-; X64-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    addq %rbp, %rax
 ; X64-NEXT:    movq %rax, %rbp
-; X64-NEXT:    addq %rax, %r11
-; X64-NEXT:    adcq %rdx, %r14
-; X64-NEXT:    addq %r9, %r11
-; X64-NEXT:    adcq %rbx, %r14
-; X64-NEXT:    addq %r10, %r12
-; X64-NEXT:    movq %r12, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    adcq %r13, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
+; X64-NEXT:    adcq %rcx, %rsi
+; X64-NEXT:    setb %cl
+; X64-NEXT:    movq %r11, %rax
+; X64-NEXT:    mulq %rdi
+; X64-NEXT:    movq %rdx, %r11
+; X64-NEXT:    movq %rax, %r12
+; X64-NEXT:    addq %rsi, %r12
+; X64-NEXT:    movzbl %cl, %eax
+; X64-NEXT:    adcq %rax, %r11
+; X64-NEXT:    addq %rbx, %r9
+; X64-NEXT:    movq %r9, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    adcq %r15, %rbp
+; X64-NEXT:    movq %rbp, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    adcq $0, %r12
 ; X64-NEXT:    adcq $0, %r11
-; X64-NEXT:    adcq $0, %r14
-; X64-NEXT:    addq %r15, %r11
-; X64-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %r14 # 8-byte Folded Reload
+; X64-NEXT:    addq %r13, %r12
+; X64-NEXT:    adcq %r14, %r11
 ; X64-NEXT:    setb %r9b
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
-; X64-NEXT:    movq %rcx, %rax
+; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rbx # 8-byte Reload
+; X64-NEXT:    movq %rbx, %rax
 ; X64-NEXT:    mulq %r8
-; X64-NEXT:    movq %rdx, %r10
-; X64-NEXT:    movq %rax, %r12
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r15 # 8-byte Reload
-; X64-NEXT:    movq %r15, %rax
+; X64-NEXT:    movq %rdx, %rcx
+; X64-NEXT:    movq %rax, %r14
+; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r10 # 8-byte Reload
+; X64-NEXT:    movq %r10, %rax
 ; X64-NEXT:    mulq %r8
 ; X64-NEXT:    movq %rdx, %rsi
-; X64-NEXT:    movq %rax, %rbx
-; X64-NEXT:    addq %r10, %rbx
+; X64-NEXT:    movq %rax, %rbp
+; X64-NEXT:    addq %rcx, %rbp
 ; X64-NEXT:    adcq $0, %rsi
-; X64-NEXT:    movq %rcx, %rax
+; X64-NEXT:    movq %rbx, %rax
 ; X64-NEXT:    mulq %rdi
 ; X64-NEXT:    movq %rdx, %rcx
-; X64-NEXT:    addq %rbx, %rax
-; X64-NEXT:    movq %rax, %rbx
+; X64-NEXT:    addq %rbp, %rax
+; X64-NEXT:    movq %rax, %rbp
 ; X64-NEXT:    adcq %rsi, %rcx
-; X64-NEXT:    setb %sil
-; X64-NEXT:    movq %r15, %rax
+; X64-NEXT:    setb %bl
+; X64-NEXT:    movq %r10, %rax
 ; X64-NEXT:    mulq %rdi
 ; X64-NEXT:    addq %rcx, %rax
-; X64-NEXT:    movzbl %sil, %ecx
+; X64-NEXT:    movzbl %bl, %ecx
 ; X64-NEXT:    adcq %rcx, %rdx
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
-; X64-NEXT:    addq %rbp, %rsi
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
-; X64-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Folded Reload
-; X64-NEXT:    addq %rax, %rsi
-; X64-NEXT:    adcq %rdx, %rcx
-; X64-NEXT:    addq %r11, %r12
-; X64-NEXT:    movq %r12, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    adcq %r14, %rbx
-; X64-NEXT:    movq %rbx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    movzbl %r9b, %eax
-; X64-NEXT:    adcq %rax, %rsi
-; X64-NEXT:    movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    adcq $0, %rcx
-; X64-NEXT:    movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    addq %r12, %r14
+; X64-NEXT:    movq %r14, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    adcq %r11, %rbp
+; X64-NEXT:    movq %rbp, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    movzbl %r9b, %ecx
+; X64-NEXT:    adcq %rcx, %rax
+; X64-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    adcq $0, %rdx
+; X64-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
-; X64-NEXT:    movq 32(%rcx), %rsi
-; X64-NEXT:    imulq %rsi, %rdi
-; X64-NEXT:    movq %rsi, %rax
+; X64-NEXT:    movq 32(%rcx), %r10
+; X64-NEXT:    imulq %r10, %rdi
+; X64-NEXT:    movq %r10, %rax
 ; X64-NEXT:    mulq %r8
-; X64-NEXT:    movq %rax, %r10
+; X64-NEXT:    movq %rax, %r11
 ; X64-NEXT:    addq %rdi, %rdx
 ; X64-NEXT:    movq 40(%rcx), %r9
 ; X64-NEXT:    imulq %r9, %r8
 ; X64-NEXT:    addq %rdx, %r8
 ; X64-NEXT:    movq 48(%rcx), %rax
-; X64-NEXT:    movq %rcx, %rbx
 ; X64-NEXT:    movq %rax, %rdi
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
-; X64-NEXT:    imulq %rcx, %rdi
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rbp # 8-byte Reload
-; X64-NEXT:    mulq %rbp
-; X64-NEXT:    movq %rax, %r14
-; X64-NEXT:    addq %rdi, %rdx
-; X64-NEXT:    movq 56(%rbx), %rbx
-; X64-NEXT:    imulq %rbp, %rbx
-; X64-NEXT:    addq %rdx, %rbx
-; X64-NEXT:    addq %r10, %r14
-; X64-NEXT:    adcq %r8, %rbx
-; X64-NEXT:    movq %rbp, %rax
-; X64-NEXT:    movq %rbp, %r10
+; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rbx # 8-byte Reload
+; X64-NEXT:    imulq %rbx, %rdi
+; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
 ; X64-NEXT:    mulq %rsi
+; X64-NEXT:    movq %rax, %r12
+; X64-NEXT:    addq %rdi, %rdx
+; X64-NEXT:    movq 56(%rcx), %rbp
+; X64-NEXT:    imulq %rsi, %rbp
+; X64-NEXT:    addq %rdx, %rbp
+; X64-NEXT:    addq %r11, %r12
+; X64-NEXT:    adcq %r8, %rbp
+; X64-NEXT:    movq %rsi, %rax
+; X64-NEXT:    movq %rsi, %rcx
+; X64-NEXT:    mulq %r10
 ; X64-NEXT:    movq %rdx, %rdi
 ; X64-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    movq %rcx, %rax
-; X64-NEXT:    movq %rcx, %r8
-; X64-NEXT:    mulq %rsi
+; X64-NEXT:    movq %rbx, %rax
+; X64-NEXT:    movq %rbx, %r11
+; X64-NEXT:    mulq %r10
 ; X64-NEXT:    movq %rdx, %rsi
-; X64-NEXT:    movq %rax, %rbp
-; X64-NEXT:    addq %rdi, %rbp
+; X64-NEXT:    movq %rax, %rbx
+; X64-NEXT:    addq %rdi, %rbx
 ; X64-NEXT:    adcq $0, %rsi
-; X64-NEXT:    movq %r10, %rax
+; X64-NEXT:    movq %rcx, %rax
 ; X64-NEXT:    mulq %r9
 ; X64-NEXT:    movq %rdx, %rdi
-; X64-NEXT:    movq %rax, %r13
-; X64-NEXT:    addq %rbp, %r13
+; X64-NEXT:    movq %rax, %r10
+; X64-NEXT:    addq %rbx, %r10
 ; X64-NEXT:    adcq %rsi, %rdi
-; X64-NEXT:    setb %cl
-; X64-NEXT:    movq %r8, %rax
+; X64-NEXT:    setb %bl
+; X64-NEXT:    movq %r11, %rax
 ; X64-NEXT:    mulq %r9
-; X64-NEXT:    movq %rdx, %r11
-; X64-NEXT:    movq %rax, %r9
-; X64-NEXT:    addq %rdi, %r9
-; X64-NEXT:    movzbl %cl, %eax
-; X64-NEXT:    adcq %rax, %r11
-; X64-NEXT:    addq %r14, %r9
-; X64-NEXT:    adcq %rbx, %r11
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rbp # 8-byte Reload
-; X64-NEXT:    movq 56(%rbp), %rcx
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r10 # 8-byte Reload
-; X64-NEXT:    imulq %r10, %rcx
-; X64-NEXT:    movq 48(%rbp), %rbx
-; X64-NEXT:    movq %r10, %rax
-; X64-NEXT:    mulq %rbx
-; X64-NEXT:    movq %rax, %rsi
-; X64-NEXT:    addq %rcx, %rdx
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r15 # 8-byte Reload
-; X64-NEXT:    imulq %r15, %rbx
-; X64-NEXT:    addq %rdx, %rbx
-; X64-NEXT:    movq 32(%rbp), %rdi
-; X64-NEXT:    movq 40(%rbp), %r8
+; X64-NEXT:    movq %rdx, %r15
+; X64-NEXT:    movq %rax, %r13
+; X64-NEXT:    addq %rdi, %r13
+; X64-NEXT:    movzbl %bl, %eax
+; X64-NEXT:    adcq %rax, %r15
+; X64-NEXT:    addq %r12, %r13
+; X64-NEXT:    adcq %rbp, %r15
+; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
+; X64-NEXT:    movq 56(%rdx), %rcx
 ; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
-; X64-NEXT:    movq %rax, %rcx
-; X64-NEXT:    imulq %r8, %rcx
-; X64-NEXT:    mulq %rdi
-; X64-NEXT:    movq %rax, %r14
+; X64-NEXT:    imulq %rax, %rcx
+; X64-NEXT:    movq 48(%rdx), %rbp
+; X64-NEXT:    movq %rdx, %rsi
+; X64-NEXT:    movq %rax, %r11
+; X64-NEXT:    mulq %rbp
+; X64-NEXT:    movq %rax, %r12
 ; X64-NEXT:    addq %rcx, %rdx
+; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Reload
+; X64-NEXT:    imulq %r8, %rbp
+; X64-NEXT:    addq %rdx, %rbp
+; X64-NEXT:    movq 32(%rsi), %rdi
+; X64-NEXT:    movq 40(%rsi), %rbx
 ; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
-; X64-NEXT:    imulq %rdi, %rax
-; X64-NEXT:    addq %rdx, %rax
-; X64-NEXT:    addq %rsi, %r14
-; X64-NEXT:    adcq %rbx, %rax
-; X64-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    movq %rax, %rsi
+; X64-NEXT:    imulq %rbx, %rsi
+; X64-NEXT:    mulq %rdi
+; X64-NEXT:    movq %rax, %rcx
+; X64-NEXT:    addq %rsi, %rdx
+; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r14 # 8-byte Reload
+; X64-NEXT:    imulq %rdi, %r14
+; X64-NEXT:    addq %rdx, %r14
+; X64-NEXT:    addq %r12, %rcx
+; X64-NEXT:    adcq %rbp, %r14
 ; X64-NEXT:    movq %rdi, %rax
-; X64-NEXT:    mulq %r10
-; X64-NEXT:    movq %rdx, %r12
+; X64-NEXT:    mulq %r11
+; X64-NEXT:    movq %rdx, %rbp
 ; X64-NEXT:    movq %rax, %rsi
-; X64-NEXT:    movq %r8, %rax
-; X64-NEXT:    mulq %r10
-; X64-NEXT:    movq %rdx, %rcx
-; X64-NEXT:    movq %rax, %rbx
-; X64-NEXT:    addq %r12, %rbx
-; X64-NEXT:    adcq $0, %rcx
+; X64-NEXT:    movq %rbx, %rax
+; X64-NEXT:    mulq %r11
+; X64-NEXT:    movq %rdx, %r11
+; X64-NEXT:    movq %rax, %r12
+; X64-NEXT:    addq %rbp, %r12
+; X64-NEXT:    adcq $0, %r11
 ; X64-NEXT:    movq %rdi, %rax
-; X64-NEXT:    mulq %r15
+; X64-NEXT:    mulq %r8
 ; X64-NEXT:    movq %rdx, %rbp
 ; X64-NEXT:    movq %rax, %rdi
-; X64-NEXT:    addq %rbx, %rdi
-; X64-NEXT:    adcq %rcx, %rbp
-; X64-NEXT:    setb %cl
-; X64-NEXT:    movq %r8, %rax
-; X64-NEXT:    mulq %r15
+; X64-NEXT:    addq %r12, %rdi
+; X64-NEXT:    adcq %r11, %rbp
+; X64-NEXT:    setb %r9b
+; X64-NEXT:    movq %rbx, %rax
+; X64-NEXT:    mulq %r8
 ; X64-NEXT:    addq %rbp, %rax
-; X64-NEXT:    movzbl %cl, %ecx
-; X64-NEXT:    adcq %rcx, %rdx
-; X64-NEXT:    addq %r14, %rax
-; X64-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Folded Reload
+; X64-NEXT:    movzbl %r9b, %ebp
+; X64-NEXT:    adcq %rbp, %rdx
+; X64-NEXT:    addq %rcx, %rax
+; X64-NEXT:    adcq %r14, %rdx
 ; X64-NEXT:    addq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Folded Reload
-; X64-NEXT:    adcq %r13, %rdi
-; X64-NEXT:    adcq %r9, %rax
-; X64-NEXT:    adcq %r11, %rdx
+; X64-NEXT:    adcq %r10, %rdi
+; X64-NEXT:    adcq %r13, %rax
+; X64-NEXT:    adcq %r15, %rdx
 ; X64-NEXT:    addq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Folded Reload
 ; X64-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Folded Reload
 ; X64-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Folded Reload
diff --git a/llvm/test/CodeGen/X86/mul-remat.ll b/llvm/test/CodeGen/X86/mul-remat.ll
index 87921324f..1a5d88aba 100644
--- a/llvm/test/CodeGen/X86/mul-remat.ll
+++ b/llvm/test/CodeGen/X86/mul-remat.ll
@@ -1,7 +1,13 @@
-; RUN: llc < %s -mtriple=i686-- | grep mov | count 1
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- | FileCheck %s
 ; PR1874
-	
+
 define i32 @test(i32 %a, i32 %b) {
+; CHECK-LABEL: test:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    imull {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    retl
 entry:
 	%tmp3 = mul i32 %b, %a
 	ret i32 %tmp3
diff --git a/llvm/test/CodeGen/X86/mul-shift-reassoc.ll b/llvm/test/CodeGen/X86/mul-shift-reassoc.ll
index c1139b01f..74ae97676 100644
--- a/llvm/test/CodeGen/X86/mul-shift-reassoc.ll
+++ b/llvm/test/CodeGen/X86/mul-shift-reassoc.ll
@@ -1,9 +1,16 @@
-; RUN: llc < %s -mtriple=i686-- | grep lea
-; RUN: llc < %s -mtriple=i686-- | not grep add
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- | FileCheck %s
 
 define i32 @test(i32 %X, i32 %Y) {
 	; Push the shl through the mul to allow an LEA to be formed, instead
         ; of using a shift and add separately.
+; CHECK-LABEL: test:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT:    imull %eax, %ecx
+; CHECK-NEXT:    leal (%eax,%ecx,2), %eax
+; CHECK-NEXT:    retl
         %tmp.2 = shl i32 %X, 1          ; <i32> [#uses=1]
         %tmp.3 = mul i32 %tmp.2, %Y             ; <i32> [#uses=1]
         %tmp.5 = add i32 %tmp.3, %Y             ; <i32> [#uses=1]
diff --git a/llvm/test/CodeGen/X86/mul128.ll b/llvm/test/CodeGen/X86/mul128.ll
index 2bea5b749..66d2f3b4e 100644
--- a/llvm/test/CodeGen/X86/mul128.ll
+++ b/llvm/test/CodeGen/X86/mul128.ll
@@ -119,41 +119,11 @@ define void @PR13897() nounwind {
 ;
 ; X86-LABEL: PR13897:
 ; X86:       # %bb.0: # %"0x0"
-; X86-NEXT:    pushl %ebp
-; X86-NEXT:    pushl %ebx
-; X86-NEXT:    pushl %edi
-; X86-NEXT:    pushl %esi
-; X86-NEXT:    pushl %eax
 ; X86-NEXT:    movl bbb, %eax
-; X86-NEXT:    movl %eax, (%esp) # 4-byte Spill
-; X86-NEXT:    movl $1, %ebx
-; X86-NEXT:    mull %ebx
-; X86-NEXT:    movl %edx, %esi
-; X86-NEXT:    movl %eax, %edi
-; X86-NEXT:    xorl %eax, %eax
-; X86-NEXT:    mull %ebx
-; X86-NEXT:    movl %eax, %ebx
-; X86-NEXT:    addl %esi, %ebx
-; X86-NEXT:    movl %edx, %ebp
-; X86-NEXT:    adcl $0, %ebp
-; X86-NEXT:    addl %edi, %ebx
-; X86-NEXT:    adcl %esi, %ebp
-; X86-NEXT:    setb %cl
-; X86-NEXT:    addl %eax, %ebp
-; X86-NEXT:    movzbl %cl, %eax
-; X86-NEXT:    adcl %edx, %eax
-; X86-NEXT:    addl %edi, %ebp
-; X86-NEXT:    adcl %esi, %eax
-; X86-NEXT:    addl (%esp), %eax # 4-byte Folded Reload
-; X86-NEXT:    movl %edi, aaa
-; X86-NEXT:    movl %ebx, aaa+4
-; X86-NEXT:    movl %ebp, aaa+8
 ; X86-NEXT:    movl %eax, aaa+12
-; X86-NEXT:    addl $4, %esp
-; X86-NEXT:    popl %esi
-; X86-NEXT:    popl %edi
-; X86-NEXT:    popl %ebx
-; X86-NEXT:    popl %ebp
+; X86-NEXT:    movl %eax, aaa+8
+; X86-NEXT:    movl %eax, aaa+4
+; X86-NEXT:    movl %eax, aaa
 ; X86-NEXT:    retl
 "0x0":
   %0 = load i128, i128* @bbb
diff --git a/llvm/test/CodeGen/X86/mulfix_combine.ll b/llvm/test/CodeGen/X86/mulfix_combine.ll
new file mode 100644
index 000000000..81a0135ce
--- /dev/null
+++ b/llvm/test/CodeGen/X86/mulfix_combine.ll
@@ -0,0 +1,156 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-linux -o - | FileCheck %s
+
+declare i32 @llvm.smul.fix.i32(i32, i32, i32 immarg)
+declare i32 @llvm.umul.fix.i32(i32, i32, i32 immarg)
+declare i32 @llvm.smul.fix.sat.i32(i32, i32, i32 immarg)
+declare i32 @llvm.umul.fix.sat.i32(i32, i32, i32 immarg)
+
+declare <4 x i32> @llvm.smul.fix.v4i32(<4 x i32>, <4 x i32>, i32 immarg)
+declare <4 x i32> @llvm.umul.fix.v4i32(<4 x i32>, <4 x i32>, i32 immarg)
+declare <4 x i32> @llvm.smul.fix.sat.v4i32(<4 x i32>, <4 x i32>, i32 immarg)
+declare <4 x i32> @llvm.umul.fix.sat.v4i32(<4 x i32>, <4 x i32>, i32 immarg)
+
+define i32 @smulfix_undef(i32 %y) nounwind {
+; CHECK-LABEL: smulfix_undef:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    retq
+  %tmp = call i32 @llvm.smul.fix.i32(i32 undef, i32 %y, i32 2)
+  ret i32 %tmp
+}
+
+define i32 @smulfix_zero(i32 %y) nounwind {
+; CHECK-LABEL: smulfix_zero:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    retq
+  %tmp = call i32 @llvm.smul.fix.i32(i32 0, i32 %y, i32 2)
+  ret i32 %tmp
+}
+
+define i32 @umulfix_undef(i32 %y) nounwind {
+; CHECK-LABEL: umulfix_undef:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    retq
+  %tmp = call i32 @llvm.umul.fix.i32(i32 undef, i32 %y, i32 2)
+  ret i32 %tmp
+}
+
+define i32 @umulfix_zero(i32 %y) nounwind {
+; CHECK-LABEL: umulfix_zero:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    retq
+  %tmp = call i32 @llvm.umul.fix.i32(i32 0, i32 %y, i32 2)
+  ret i32 %tmp
+}
+
+define i32 @smulfixsat_undef(i32 %y) nounwind {
+; CHECK-LABEL: smulfixsat_undef:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    retq
+  %tmp = call i32 @llvm.smul.fix.sat.i32(i32 undef, i32 %y, i32 2)
+  ret i32 %tmp
+}
+
+define i32 @smulfixsat_zero(i32 %y) nounwind {
+; CHECK-LABEL: smulfixsat_zero:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    retq
+  %tmp = call i32 @llvm.smul.fix.sat.i32(i32 0, i32 %y, i32 2)
+  ret i32 %tmp
+}
+
+define i32 @umulfixsat_undef(i32 %y) nounwind {
+; CHECK-LABEL: umulfixsat_undef:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    retq
+  %tmp = call i32 @llvm.umul.fix.sat.i32(i32 undef, i32 %y, i32 2)
+  ret i32 %tmp
+}
+
+define i32 @umulfixsat_zero(i32 %y) nounwind {
+; CHECK-LABEL: umulfixsat_zero:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    retq
+  %tmp = call i32 @llvm.umul.fix.sat.i32(i32 0, i32 %y, i32 2)
+  ret i32 %tmp
+}
+
+define <4 x i32> @vec_smulfix_undef(<4 x i32> %y) nounwind {
+; CHECK-LABEL: vec_smulfix_undef:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xorps %xmm0, %xmm0
+; CHECK-NEXT:    retq
+  %tmp = call <4 x i32> @llvm.smul.fix.v4i32(<4 x i32> undef, <4 x i32> %y, i32 2)
+  ret <4 x i32> %tmp
+}
+
+define <4 x i32> @vec_smulfix_zero(<4 x i32> %y) nounwind {
+; CHECK-LABEL: vec_smulfix_zero:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xorps %xmm0, %xmm0
+; CHECK-NEXT:    retq
+  %tmp = call <4 x i32> @llvm.smul.fix.v4i32(<4 x i32> <i32 0, i32 0, i32 0, i32 0>, <4 x i32> %y, i32 2)
+  ret <4 x i32> %tmp
+}
+
+define <4 x i32> @vec_umulfix_undef(<4 x i32> %y) nounwind {
+; CHECK-LABEL: vec_umulfix_undef:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xorps %xmm0, %xmm0
+; CHECK-NEXT:    retq
+  %tmp = call <4 x i32> @llvm.umul.fix.v4i32(<4 x i32> undef, <4 x i32> %y, i32 2)
+  ret <4 x i32> %tmp
+}
+
+define <4 x i32> @vec_umulfix_zero(<4 x i32> %y) nounwind {
+; CHECK-LABEL: vec_umulfix_zero:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xorps %xmm0, %xmm0
+; CHECK-NEXT:    retq
+  %tmp = call <4 x i32> @llvm.umul.fix.v4i32(<4 x i32> <i32 0, i32 0, i32 0, i32 0>, <4 x i32> %y, i32 2)
+  ret <4 x i32> %tmp
+}
+
+define <4 x i32> @vec_smulfixsat_undef(<4 x i32> %y) nounwind {
+; CHECK-LABEL: vec_smulfixsat_undef:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xorps %xmm0, %xmm0
+; CHECK-NEXT:    retq
+  %tmp = call <4 x i32> @llvm.smul.fix.sat.v4i32(<4 x i32> undef, <4 x i32> %y, i32 2)
+  ret <4 x i32> %tmp
+}
+
+define <4 x i32> @vec_smulfixsat_zero(<4 x i32> %y) nounwind {
+; CHECK-LABEL: vec_smulfixsat_zero:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xorps %xmm0, %xmm0
+; CHECK-NEXT:    retq
+  %tmp = call <4 x i32> @llvm.smul.fix.sat.v4i32(<4 x i32> <i32 0, i32 0, i32 0, i32 0>, <4 x i32> %y, i32 2)
+  ret <4 x i32> %tmp
+}
+
+define <4 x i32> @vec_umulfixsat_undef(<4 x i32> %y) nounwind {
+; CHECK-LABEL: vec_umulfixsat_undef:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xorps %xmm0, %xmm0
+; CHECK-NEXT:    retq
+  %tmp = call <4 x i32> @llvm.umul.fix.sat.v4i32(<4 x i32> undef, <4 x i32> %y, i32 2)
+  ret <4 x i32> %tmp
+}
+
+define <4 x i32> @vec_umulfixsat_zero(<4 x i32> %y) nounwind {
+; CHECK-LABEL: vec_umulfixsat_zero:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xorps %xmm0, %xmm0
+; CHECK-NEXT:    retq
+  %tmp = call <4 x i32> @llvm.umul.fix.sat.v4i32(<4 x i32> <i32 0, i32 0, i32 0, i32 0>, <4 x i32> %y, i32 2)
+  ret <4 x i32> %tmp
+}
diff --git a/llvm/test/CodeGen/X86/mulo-pow2.ll b/llvm/test/CodeGen/X86/mulo-pow2.ll
index f5be144ff..de7dc5673 100644
--- a/llvm/test/CodeGen/X86/mulo-pow2.ll
+++ b/llvm/test/CodeGen/X86/mulo-pow2.ll
@@ -98,15 +98,10 @@ define <4 x i32> @smul_v4i32_1(<4 x i32> %a, <4 x i32> %b) nounwind {
 define <4 x i32> @smul_v4i32_2(<4 x i32> %a, <4 x i32> %b) nounwind {
 ; AVX-LABEL: smul_v4i32_2:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX-NEXT:    vpcmpgtd %xmm0, %xmm2, %xmm3
-; AVX-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX-NEXT:    vpxor %xmm4, %xmm3, %xmm3
-; AVX-NEXT:    vpaddd %xmm0, %xmm0, %xmm0
-; AVX-NEXT:    vpcmpgtd %xmm0, %xmm2, %xmm2
-; AVX-NEXT:    vpxor %xmm4, %xmm2, %xmm2
-; AVX-NEXT:    vpcmpeqd %xmm2, %xmm3, %xmm2
-; AVX-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    vpaddd %xmm0, %xmm0, %xmm2
+; AVX-NEXT:    vpcmpgtd %xmm2, %xmm0, %xmm3
+; AVX-NEXT:    vpxor %xmm3, %xmm0, %xmm0
+; AVX-NEXT:    vblendvps %xmm0, %xmm1, %xmm2, %xmm0
 ; AVX-NEXT:    retq
     %x = call { <4 x i32>, <4 x i1> } @llvm.smul.with.overflow.v4i32(<4 x i32> %a, <4 x i32> <i32 2, i32 2, i32 2, i32 2>)
     %y = extractvalue { <4 x i32>, <4 x i1> } %x, 0
diff --git a/llvm/test/CodeGen/X86/mulvi32.ll b/llvm/test/CodeGen/X86/mulvi32.ll
index ebcbd62b8..d05c26e98 100644
--- a/llvm/test/CodeGen/X86/mulvi32.ll
+++ b/llvm/test/CodeGen/X86/mulvi32.ll
@@ -7,36 +7,39 @@
 ; PR6399
 
 define <2 x i32> @_mul2xi32a(<2 x i32>, <2 x i32>) {
-; SSE-LABEL: _mul2xi32a:
-; SSE:       # %bb.0:
-; SSE-NEXT:    pmuludq %xmm1, %xmm0
-; SSE-NEXT:    retq
+; SSE2-LABEL: _mul2xi32a:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pmuludq %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pmuludq %xmm2, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT:    retq
+;
+; SSE42-LABEL: _mul2xi32a:
+; SSE42:       # %bb.0:
+; SSE42-NEXT:    pmulld %xmm1, %xmm0
+; SSE42-NEXT:    retq
 ;
 ; AVX-LABEL: _mul2xi32a:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    retq
   %r = mul <2 x i32> %0, %1
   ret <2 x i32> %r
 }
 
 define <2 x i32> @_mul2xi32b(<2 x i32>, <2 x i32>) {
-; SSE2-LABEL: _mul2xi32b:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    pmuludq %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,1,3]
-; SSE2-NEXT:    retq
-;
-; SSE42-LABEL: _mul2xi32b:
-; SSE42:       # %bb.0:
-; SSE42-NEXT:    pmuludq %xmm1, %xmm0
-; SSE42-NEXT:    pmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
-; SSE42-NEXT:    retq
+; SSE-LABEL: _mul2xi32b:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pmuludq %xmm1, %xmm0
+; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: _mul2xi32b:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
 ; AVX-NEXT:    retq
   %factor0 = shufflevector <2 x i32> %0, <2 x i32> undef, <4 x i32> <i32 0, i32 undef, i32 2, i32 undef>
   %factor1 = shufflevector <2 x i32> %1, <2 x i32> undef, <4 x i32> <i32 0, i32 undef, i32 2, i32 undef>
@@ -153,8 +156,8 @@ define <4 x i64> @_mul4xi32toi64a(<4 x i32>, <4 x i32>) {
 ;
 ; AVX1-LABEL: _mul4xi32toi64a:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[2,2,3,3]
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[2,2,3,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[2,1,3,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[2,1,3,3]
 ; AVX1-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2
 ; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
 ; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
diff --git a/llvm/test/CodeGen/X86/musttail-fastcall.ll b/llvm/test/CodeGen/X86/musttail-fastcall.ll
index a95e0ff12..22a2ec60d 100644
--- a/llvm/test/CodeGen/X86/musttail-fastcall.ll
+++ b/llvm/test/CodeGen/X86/musttail-fastcall.ll
@@ -1,6 +1,7 @@
 ; RUN: llc < %s -mtriple=i686-pc-win32 -mattr=+sse2 | FileCheck %s --check-prefix=CHECK --check-prefix=SSE2
 ; RUN: llc < %s -mtriple=i686-pc-win32 -mattr=+sse2,+avx | FileCheck %s --check-prefix=CHECK --check-prefix=AVX
-; RUN: llc < %s -mtriple=i686-pc-win32 -mattr=+sse2,+avx,+avx512f | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512
+; RUN: llc < %s -mtriple=i686-pc-win32 -mattr=+sse2,+avx,+avx512f | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512F
+; RUN: llc < %s -mtriple=i686-pc-win32 -mattr=+sse2,+avx,+avx512vl | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512VL
 
 ; While we don't support varargs with fastcall, we do support forwarding.
 
@@ -107,3 +108,90 @@ define x86_vectorcallcc i32 @vector_target(i32 inreg %a, i32 inreg %b, i32 %c) {
   %a1 = add i32 %a0, %c
   ret i32 %a1
 }
+
+; Repeat the test for vectorcall, which has XMM registers.
+
+define i32 @call_vector_thunk_prefer256() "min-legal-vector-width"="256" "prefer-vector-width"="256" {
+  %r = call x86_vectorcallcc i32 (...) @vector_thunk_prefer256(i32 inreg 1, i32 inreg 2, i32 3)
+  ret i32 %r
+}
+
+define x86_vectorcallcc i32 @vector_thunk_prefer256(...) "min-legal-vector-width"="256" "prefer-vector-width"="256" {
+  call void @puts(i8* getelementptr ([4 x i8], [4 x i8]* @asdf, i32 0, i32 0))
+  %r = musttail call x86_vectorcallcc i32 (...) bitcast (i32 (i32, i32, i32)* @vector_target_prefer256 to i32 (...)*) (...)
+  ret i32 %r
+}
+
+; Check that we spill and fill SSE registers around the call to puts.
+
+; CHECK-LABEL: vector_thunk_prefer256@@0:
+; CHECK-DAG: movl %ecx, {{.*}}
+; CHECK-DAG: movl %edx, {{.*}}
+
+; SSE2-DAG: movups %xmm0, {{.*}}
+; SSE2-DAG: movups %xmm1, {{.*}}
+; SSE2-DAG: movups %xmm2, {{.*}}
+; SSE2-DAG: movups %xmm3, {{.*}}
+; SSE2-DAG: movups %xmm4, {{.*}}
+; SSE2-DAG: movups %xmm5, {{.*}}
+
+; AVX-DAG: vmovups %ymm0, {{.*}}
+; AVX-DAG: vmovups %ymm1, {{.*}}
+; AVX-DAG: vmovups %ymm2, {{.*}}
+; AVX-DAG: vmovups %ymm3, {{.*}}
+; AVX-DAG: vmovups %ymm4, {{.*}}
+; AVX-DAG: vmovups %ymm5, {{.*}}
+
+; AVX512F-DAG: vmovups %zmm0, {{.*}}
+; AVX512F-DAG: vmovups %zmm1, {{.*}}
+; AVX512F-DAG: vmovups %zmm2, {{.*}}
+; AVX512F-DAG: vmovups %zmm3, {{.*}}
+; AVX512F-DAG: vmovups %zmm4, {{.*}}
+; AVX512F-DAG: vmovups %zmm5, {{.*}}
+
+; AVX512VL-DAG: vmovups %ymm0, {{.*}}
+; AVX512VL-DAG: vmovups %ymm1, {{.*}}
+; AVX512VL-DAG: vmovups %ymm2, {{.*}}
+; AVX512VL-DAG: vmovups %ymm3, {{.*}}
+; AVX512VL-DAG: vmovups %ymm4, {{.*}}
+; AVX512VL-DAG: vmovups %ymm5, {{.*}}
+
+; CHECK: calll _puts
+
+; SSE2-DAG: movups {{.*}}, %xmm0
+; SSE2-DAG: movups {{.*}}, %xmm1
+; SSE2-DAG: movups {{.*}}, %xmm2
+; SSE2-DAG: movups {{.*}}, %xmm3
+; SSE2-DAG: movups {{.*}}, %xmm4
+; SSE2-DAG: movups {{.*}}, %xmm5
+
+; AVX-DAG: vmovups {{.*}}, %ymm0
+; AVX-DAG: vmovups {{.*}}, %ymm1
+; AVX-DAG: vmovups {{.*}}, %ymm2
+; AVX-DAG: vmovups {{.*}}, %ymm3
+; AVX-DAG: vmovups {{.*}}, %ymm4
+; AVX-DAG: vmovups {{.*}}, %ymm5
+
+; AVX512F-DAG: vmovups {{.*}}, %zmm0
+; AVX512F-DAG: vmovups {{.*}}, %zmm1
+; AVX512F-DAG: vmovups {{.*}}, %zmm2
+; AVX512F-DAG: vmovups {{.*}}, %zmm3
+; AVX512F-DAG: vmovups {{.*}}, %zmm4
+; AVX512F-DAG: vmovups {{.*}}, %zmm5
+
+; AVX512VL-DAG: vmovups {{.*}}, %ymm0
+; AVX512VL-DAG: vmovups {{.*}}, %ymm1
+; AVX512VL-DAG: vmovups {{.*}}, %ymm2
+; AVX512VL-DAG: vmovups {{.*}}, %ymm3
+; AVX512VL-DAG: vmovups {{.*}}, %ymm4
+; AVX512VL-DAG: vmovups {{.*}}, %ymm5
+
+; CHECK-DAG: movl {{.*}}, %ecx
+; CHECK-DAG: movl {{.*}}, %edx
+; CHECK: jmp vector_target_prefer256@@12
+
+define x86_vectorcallcc i32 @vector_target_prefer256(i32 inreg %a, i32 inreg %b, i32 %c) "min-legal-vector-width"="256" "prefer-vector-width"="256" {
+  %a0 = add i32 %a, %b
+  %a1 = add i32 %a0, %c
+  ret i32 %a1
+}
diff --git a/llvm/test/CodeGen/X86/musttail-inalloca.ll b/llvm/test/CodeGen/X86/musttail-inalloca.ll
new file mode 100644
index 000000000..c17d29c4c
--- /dev/null
+++ b/llvm/test/CodeGen/X86/musttail-inalloca.ll
@@ -0,0 +1,38 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc %s -o - | FileCheck %s
+
+; Previously, we would accidentally leave behind SP adjustments to setup a call
+; frame for the musttail call target, and SP adjustments would end up
+; unbalanced. Reported as https://crbug.com/1026882.
+
+target datalayout = "e-m:x-p:32:32-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:32-n8:16:32-a:0:32-S32"
+target triple = "i386-pc-windows-msvc19.16.0"
+
+; 20 bytes of memory.
+%struct.Args = type { i32, i32, i32, i32, i32 }
+
+declare dso_local x86_thiscallcc void @methodWithVtorDisp(i8* nocapture readonly, <{ %struct.Args }>* inalloca)
+
+define dso_local x86_thiscallcc void @methodWithVtorDisp_thunk(i8* %0, <{ %struct.Args }>* inalloca %1) #0 {
+; CHECK-LABEL: methodWithVtorDisp_thunk:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushl %esi
+; CHECK-NEXT:    movl %ecx, %esi
+; CHECK-NEXT:    subl -4(%ecx), %esi
+; CHECK-NEXT:    pushl {{[0-9]+}}(%esp)
+; CHECK-NEXT:    pushl $_methodWithVtorDisp_thunk
+; CHECK-NEXT:    calll ___cyg_profile_func_exit
+; CHECK-NEXT:    addl $8, %esp
+; CHECK-NEXT:    movl %esi, %ecx
+; CHECK-NEXT:    popl %esi
+; CHECK-NEXT:    jmp _methodWithVtorDisp # TAILCALL
+  %3 = getelementptr inbounds i8, i8* %0, i32 -4
+  %4 = bitcast i8* %3 to i32*
+  %5 = load i32, i32* %4, align 4
+  %6 = sub i32 0, %5
+  %7 = getelementptr i8, i8* %0, i32 %6
+  musttail call x86_thiscallcc void @methodWithVtorDisp(i8* %7, <{ %struct.Args }>* inalloca nonnull %1)
+  ret void
+}
+
+attributes #0 = { nounwind optsize "instrument-function-exit-inlined"="__cyg_profile_func_exit"  }
diff --git a/llvm/test/CodeGen/X86/musttail-tailcc.ll b/llvm/test/CodeGen/X86/musttail-tailcc.ll
new file mode 100644
index 000000000..4de8d4e2b
--- /dev/null
+++ b/llvm/test/CodeGen/X86/musttail-tailcc.ll
@@ -0,0 +1,92 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s -check-prefix=X64
+; RUN: llc < %s -mtriple=i686-unknown-unknown   | FileCheck %s -check-prefix=X32
+
+; tailcc will turn all of these musttail calls into tail calls.
+
+declare tailcc i32 @tailcallee(i32 %a1, i32 %a2)
+
+define tailcc i32 @tailcaller(i32 %in1, i32 %in2) nounwind {
+; X64-LABEL: tailcaller:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    jmp tailcallee # TAILCALL
+;
+; X32-LABEL: tailcaller:
+; X32:       # %bb.0: # %entry
+; X32-NEXT:    jmp tailcallee # TAILCALL
+entry:
+  %tmp11 = musttail call tailcc i32 @tailcallee(i32 %in1, i32 %in2)
+  ret i32 %tmp11
+}
+
+declare tailcc i8* @alias_callee()
+
+define tailcc noalias i8* @noalias_caller() nounwind {
+; X64-LABEL: noalias_caller:
+; X64:       # %bb.0:
+; X64-NEXT:    jmp alias_callee # TAILCALL
+;
+; X32-LABEL: noalias_caller:
+; X32:       # %bb.0:
+; X32-NEXT:    jmp alias_callee # TAILCALL
+  %p = musttail call tailcc i8* @alias_callee()
+  ret i8* %p
+}
+
+declare tailcc noalias i8* @noalias_callee()
+
+define tailcc i8* @alias_caller() nounwind {
+; X64-LABEL: alias_caller:
+; X64:       # %bb.0:
+; X64-NEXT:    jmp noalias_callee # TAILCALL
+;
+; X32-LABEL: alias_caller:
+; X32:       # %bb.0:
+; X32-NEXT:    jmp noalias_callee # TAILCALL
+  %p = musttail call tailcc noalias i8* @noalias_callee()
+  ret i8* %p
+}
+
+define tailcc void @void_test(i32, i32, i32, i32) {
+; X64-LABEL: void_test:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    jmp void_test # TAILCALL
+;
+; X32-LABEL: void_test:
+; X32:       # %bb.0: # %entry
+; X32-NEXT:    pushl %esi
+; X32-NEXT:    .cfi_def_cfa_offset 8
+; X32-NEXT:    .cfi_offset %esi, -8
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X32-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X32-NEXT:    popl %esi
+; X32-NEXT:    .cfi_def_cfa_offset 4
+; X32-NEXT:    jmp void_test # TAILCALL
+  entry:
+   musttail call tailcc void @void_test( i32 %0, i32 %1, i32 %2, i32 %3)
+   ret void
+}
+
+define tailcc i1 @i1test(i32, i32, i32, i32) {
+; X64-LABEL: i1test:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    jmp i1test # TAILCALL
+;
+; X32-LABEL: i1test:
+; X32:       # %bb.0: # %entry
+; X32-NEXT:    pushl %esi
+; X32-NEXT:    .cfi_def_cfa_offset 8
+; X32-NEXT:    .cfi_offset %esi, -8
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X32-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X32-NEXT:    popl %esi
+; X32-NEXT:    .cfi_def_cfa_offset 4
+; X32-NEXT:    jmp i1test # TAILCALL
+  entry:
+  %4 = musttail call tailcc i1 @i1test( i32 %0, i32 %1, i32 %2, i32 %3)
+  ret i1 %4
+}
diff --git a/llvm/test/CodeGen/X86/musttail-varargs.ll b/llvm/test/CodeGen/X86/musttail-varargs.ll
index b62343fc8..5d88f0dd1 100644
--- a/llvm/test/CodeGen/X86/musttail-varargs.ll
+++ b/llvm/test/CodeGen/X86/musttail-varargs.ll
@@ -206,42 +206,36 @@ define void @f_thunk(i8* %this, ...) {
 ; WINDOWS-LABEL: f_thunk:
 ; WINDOWS:       # %bb.0:
 ; WINDOWS-NEXT:    pushq %r14
-; WINDOWS-NEXT:    .seh_pushreg 14
+; WINDOWS-NEXT:    .seh_pushreg %r14
 ; WINDOWS-NEXT:    pushq %rsi
-; WINDOWS-NEXT:    .seh_pushreg 6
+; WINDOWS-NEXT:    .seh_pushreg %rsi
 ; WINDOWS-NEXT:    pushq %rdi
-; WINDOWS-NEXT:    .seh_pushreg 7
-; WINDOWS-NEXT:    pushq %rbp
-; WINDOWS-NEXT:    .seh_pushreg 5
+; WINDOWS-NEXT:    .seh_pushreg %rdi
 ; WINDOWS-NEXT:    pushq %rbx
-; WINDOWS-NEXT:    .seh_pushreg 3
-; WINDOWS-NEXT:    subq $64, %rsp
-; WINDOWS-NEXT:    .seh_stackalloc 64
+; WINDOWS-NEXT:    .seh_pushreg %rbx
+; WINDOWS-NEXT:    subq $72, %rsp
+; WINDOWS-NEXT:    .seh_stackalloc 72
 ; WINDOWS-NEXT:    .seh_endprologue
-; WINDOWS-NEXT:    movl %eax, %r14d
-; WINDOWS-NEXT:    movq %r9, %rsi
+; WINDOWS-NEXT:    movq %r9, %r14
 ; WINDOWS-NEXT:    movq %r8, %rdi
 ; WINDOWS-NEXT:    movq %rdx, %rbx
-; WINDOWS-NEXT:    movq %rcx, %rbp
+; WINDOWS-NEXT:    movq %rcx, %rsi
 ; WINDOWS-NEXT:    movq %rdx, {{[0-9]+}}(%rsp)
 ; WINDOWS-NEXT:    movq %r8, {{[0-9]+}}(%rsp)
 ; WINDOWS-NEXT:    movq %r9, {{[0-9]+}}(%rsp)
 ; WINDOWS-NEXT:    leaq {{[0-9]+}}(%rsp), %rax
 ; WINDOWS-NEXT:    movq %rax, {{[0-9]+}}(%rsp)
 ; WINDOWS-NEXT:    callq get_f
-; WINDOWS-NEXT:    movq %rax, %r10
-; WINDOWS-NEXT:    movq %rbp, %rcx
+; WINDOWS-NEXT:    movq %rsi, %rcx
 ; WINDOWS-NEXT:    movq %rbx, %rdx
 ; WINDOWS-NEXT:    movq %rdi, %r8
-; WINDOWS-NEXT:    movq %rsi, %r9
-; WINDOWS-NEXT:    movl %r14d, %eax
-; WINDOWS-NEXT:    addq $64, %rsp
+; WINDOWS-NEXT:    movq %r14, %r9
+; WINDOWS-NEXT:    addq $72, %rsp
 ; WINDOWS-NEXT:    popq %rbx
-; WINDOWS-NEXT:    popq %rbp
 ; WINDOWS-NEXT:    popq %rdi
 ; WINDOWS-NEXT:    popq %rsi
 ; WINDOWS-NEXT:    popq %r14
-; WINDOWS-NEXT:    rex64 jmpq *%r10 # TAILCALL
+; WINDOWS-NEXT:    rex64 jmpq *%rax # TAILCALL
 ; WINDOWS-NEXT:    .seh_handlerdata
 ; WINDOWS-NEXT:    .text
 ; WINDOWS-NEXT:    .seh_endproc
@@ -252,12 +246,13 @@ define void @f_thunk(i8* %this, ...) {
 ; X86-NOSSE-NEXT:    movl %esp, %ebp
 ; X86-NOSSE-NEXT:    pushl %esi
 ; X86-NOSSE-NEXT:    andl $-16, %esp
-; X86-NOSSE-NEXT:    subl $48, %esp
+; X86-NOSSE-NEXT:    subl $32, %esp
 ; X86-NOSSE-NEXT:    movl 8(%ebp), %esi
 ; X86-NOSSE-NEXT:    leal 12(%ebp), %eax
-; X86-NOSSE-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; X86-NOSSE-NEXT:    movl %esi, (%esp)
+; X86-NOSSE-NEXT:    movl %eax, (%esp)
+; X86-NOSSE-NEXT:    pushl %esi
 ; X86-NOSSE-NEXT:    calll _get_f
+; X86-NOSSE-NEXT:    addl $4, %esp
 ; X86-NOSSE-NEXT:    movl %esi, 8(%ebp)
 ; X86-NOSSE-NEXT:    leal -4(%ebp), %esp
 ; X86-NOSSE-NEXT:    popl %esi
@@ -270,17 +265,18 @@ define void @f_thunk(i8* %this, ...) {
 ; X86-SSE-NEXT:    movl %esp, %ebp
 ; X86-SSE-NEXT:    pushl %esi
 ; X86-SSE-NEXT:    andl $-16, %esp
-; X86-SSE-NEXT:    subl $96, %esp
+; X86-SSE-NEXT:    subl $80, %esp
 ; X86-SSE-NEXT:    movaps %xmm2, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
 ; X86-SSE-NEXT:    movaps %xmm1, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
-; X86-SSE-NEXT:    movaps %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; X86-SSE-NEXT:    movaps %xmm0, (%esp) # 16-byte Spill
 ; X86-SSE-NEXT:    movl 8(%ebp), %esi
 ; X86-SSE-NEXT:    leal 12(%ebp), %eax
 ; X86-SSE-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; X86-SSE-NEXT:    movl %esi, (%esp)
+; X86-SSE-NEXT:    pushl %esi
 ; X86-SSE-NEXT:    calll _get_f
+; X86-SSE-NEXT:    addl $4, %esp
 ; X86-SSE-NEXT:    movl %esi, 8(%ebp)
-; X86-SSE-NEXT:    movaps {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
+; X86-SSE-NEXT:    movaps (%esp), %xmm0 # 16-byte Reload
 ; X86-SSE-NEXT:    movaps {{[-0-9]+}}(%e{{[sb]}}p), %xmm1 # 16-byte Reload
 ; X86-SSE-NEXT:    movaps {{[-0-9]+}}(%e{{[sb]}}p), %xmm2 # 16-byte Reload
 ; X86-SSE-NEXT:    leal -4(%ebp), %esp
@@ -306,38 +302,21 @@ define void @f_thunk(i8* %this, ...) {
 define void @g_thunk(i8* %fptr_i8, ...) {
 ; LINUX-LABEL: g_thunk:
 ; LINUX:       # %bb.0:
-; LINUX-NEXT:    pushq %rax
-; LINUX-NEXT:    .cfi_def_cfa_offset 16
-; LINUX-NEXT:    popq %r11
-; LINUX-NEXT:    .cfi_def_cfa_offset 8
 ; LINUX-NEXT:    jmpq *%rdi # TAILCALL
 ;
 ; LINUX-X32-LABEL: g_thunk:
 ; LINUX-X32:       # %bb.0:
-; LINUX-X32-NEXT:    pushq %rax
-; LINUX-X32-NEXT:    .cfi_def_cfa_offset 16
 ; LINUX-X32-NEXT:    movl %edi, %r11d
-; LINUX-X32-NEXT:    addl $8, %esp
-; LINUX-X32-NEXT:    .cfi_def_cfa_offset 8
 ; LINUX-X32-NEXT:    jmpq *%r11 # TAILCALL
 ;
 ; WINDOWS-LABEL: g_thunk:
 ; WINDOWS:       # %bb.0:
-; WINDOWS-NEXT:    subq $40, %rsp
-; WINDOWS-NEXT:    .seh_stackalloc 40
-; WINDOWS-NEXT:    .seh_endprologue
-; WINDOWS-NEXT:    addq $40, %rsp
 ; WINDOWS-NEXT:    rex64 jmpq *%rcx # TAILCALL
-; WINDOWS-NEXT:    .seh_handlerdata
-; WINDOWS-NEXT:    .text
-; WINDOWS-NEXT:    .seh_endproc
 ;
 ; X86-LABEL: g_thunk:
 ; X86:       # %bb.0:
-; X86-NEXT:    pushl %eax
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; X86-NEXT:    popl %ecx
 ; X86-NEXT:    jmpl *%eax # TAILCALL
   %fptr = bitcast i8* %fptr_i8 to void (i8*, ...)*
   musttail call void (i8*, ...) %fptr(i8* %fptr_i8, ...)
@@ -353,78 +332,53 @@ define void @g_thunk(i8* %fptr_i8, ...) {
 define void @h_thunk(%struct.Foo* %this, ...) {
 ; LINUX-LABEL: h_thunk:
 ; LINUX:       # %bb.0:
-; LINUX-NEXT:    pushq %rax
-; LINUX-NEXT:    .cfi_def_cfa_offset 16
 ; LINUX-NEXT:    cmpb $1, (%rdi)
 ; LINUX-NEXT:    jne .LBB2_2
 ; LINUX-NEXT:  # %bb.1: # %then
 ; LINUX-NEXT:    movq 8(%rdi), %r11
-; LINUX-NEXT:    addq $8, %rsp
-; LINUX-NEXT:    .cfi_def_cfa_offset 8
 ; LINUX-NEXT:    jmpq *%r11 # TAILCALL
 ; LINUX-NEXT:  .LBB2_2: # %else
-; LINUX-NEXT:    .cfi_def_cfa_offset 16
 ; LINUX-NEXT:    movq 16(%rdi), %r11
 ; LINUX-NEXT:    movl $42, {{.*}}(%rip)
-; LINUX-NEXT:    addq $8, %rsp
-; LINUX-NEXT:    .cfi_def_cfa_offset 8
 ; LINUX-NEXT:    jmpq *%r11 # TAILCALL
 ;
 ; LINUX-X32-LABEL: h_thunk:
 ; LINUX-X32:       # %bb.0:
-; LINUX-X32-NEXT:    pushq %rax
-; LINUX-X32-NEXT:    .cfi_def_cfa_offset 16
 ; LINUX-X32-NEXT:    cmpb $1, (%edi)
 ; LINUX-X32-NEXT:    jne .LBB2_2
 ; LINUX-X32-NEXT:  # %bb.1: # %then
 ; LINUX-X32-NEXT:    movl 4(%edi), %r11d
-; LINUX-X32-NEXT:    addl $8, %esp
-; LINUX-X32-NEXT:    .cfi_def_cfa_offset 8
 ; LINUX-X32-NEXT:    jmpq *%r11 # TAILCALL
 ; LINUX-X32-NEXT:  .LBB2_2: # %else
-; LINUX-X32-NEXT:    .cfi_def_cfa_offset 16
 ; LINUX-X32-NEXT:    movl 8(%edi), %r11d
 ; LINUX-X32-NEXT:    movl $42, {{.*}}(%rip)
-; LINUX-X32-NEXT:    addl $8, %esp
-; LINUX-X32-NEXT:    .cfi_def_cfa_offset 8
 ; LINUX-X32-NEXT:    jmpq *%r11 # TAILCALL
 ;
 ; WINDOWS-LABEL: h_thunk:
 ; WINDOWS:       # %bb.0:
-; WINDOWS-NEXT:    subq $40, %rsp
-; WINDOWS-NEXT:    .seh_stackalloc 40
-; WINDOWS-NEXT:    .seh_endprologue
 ; WINDOWS-NEXT:    cmpb $1, (%rcx)
 ; WINDOWS-NEXT:    jne .LBB2_2
 ; WINDOWS-NEXT:  # %bb.1: # %then
-; WINDOWS-NEXT:    movq 8(%rcx), %r10
-; WINDOWS-NEXT:    addq $40, %rsp
-; WINDOWS-NEXT:    rex64 jmpq *%r10 # TAILCALL
+; WINDOWS-NEXT:    movq 8(%rcx), %rax
+; WINDOWS-NEXT:    rex64 jmpq *%rax # TAILCALL
 ; WINDOWS-NEXT:  .LBB2_2: # %else
-; WINDOWS-NEXT:    movq 16(%rcx), %r10
+; WINDOWS-NEXT:    movq 16(%rcx), %rax
 ; WINDOWS-NEXT:    movl $42, {{.*}}(%rip)
-; WINDOWS-NEXT:    addq $40, %rsp
-; WINDOWS-NEXT:    rex64 jmpq *%r10 # TAILCALL
-; WINDOWS-NEXT:    .seh_handlerdata
-; WINDOWS-NEXT:    .text
-; WINDOWS-NEXT:    .seh_endproc
+; WINDOWS-NEXT:    rex64 jmpq *%rax # TAILCALL
 ;
 ; X86-LABEL: h_thunk:
 ; X86:       # %bb.0:
-; X86-NEXT:    pushl %eax
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    cmpb $1, (%eax)
 ; X86-NEXT:    jne LBB2_2
 ; X86-NEXT:  # %bb.1: # %then
 ; X86-NEXT:    movl 4(%eax), %ecx
 ; X86-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; X86-NEXT:    popl %eax
 ; X86-NEXT:    jmpl *%ecx # TAILCALL
 ; X86-NEXT:  LBB2_2: # %else
 ; X86-NEXT:    movl 8(%eax), %ecx
 ; X86-NEXT:    movl $42, _g
 ; X86-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; X86-NEXT:    popl %eax
 ; X86-NEXT:    jmpl *%ecx # TAILCALL
   %cond_p = getelementptr %struct.Foo, %struct.Foo* %this, i32 0, i32 0
   %cond = load i1, i1* %cond_p
diff --git a/llvm/test/CodeGen/X86/mxcsr-reg-usage.ll b/llvm/test/CodeGen/X86/mxcsr-reg-usage.ll
new file mode 100644
index 000000000..851828612
--- /dev/null
+++ b/llvm/test/CodeGen/X86/mxcsr-reg-usage.ll
@@ -0,0 +1,52 @@
+; RUN: llc -mtriple=x86_64-unknown-unknown -mattr=+mmx,+fma,+f16c,+avx512f -stop-after finalize-isel -o - %s | FileCheck %s
+; This test ensures that the MXCSR is implicitly used by MMX FP instructions.
+
+define x86_mmx @mxcsr_mmx(<4 x float> %a0) {
+; CHECK: MMX_CVTPS2PIirr %{{[0-9]}}, implicit $mxcsr
+; CHECK: MMX_CVTPI2PSirr %{{[0-9]}}, killed %{{[0-9]}}, implicit $mxcsr
+; CHECK: MMX_CVTTPS2PIirr killed %{{[0-9]}}, implicit $mxcsr
+; CHECK: MMX_CVTPI2PDirr killed %{{[0-9]$}}
+; CHECK: MMX_CVTPD2PIirr killed %{{[0-9]}}, implicit $mxcsr
+  %1 = call x86_mmx @llvm.x86.sse.cvtps2pi(<4 x float> %a0)
+  %2 = call <4 x float> @llvm.x86.sse.cvtpi2ps(<4 x float> %a0, x86_mmx %1)
+  %3 = call x86_mmx @llvm.x86.sse.cvttps2pi(<4 x float> %2)
+  %4 = call <2 x double> @llvm.x86.sse.cvtpi2pd(x86_mmx %3)
+  %5 = call x86_mmx @llvm.x86.sse.cvtpd2pi(<2 x double> %4)
+  ret x86_mmx %5
+}
+
+define half @mxcsr_f16c(float %a) {
+; CHECK: VCVTPS2PH{{.*}}mxcsr
+; CHECK: VCVTPH2PS{{.*}}mxcsr
+  %res = fptrunc float %a to half
+  ret half %res
+}
+
+define <4 x float> @mxcsr_fma_ss(<4 x float> %a, <4 x float> %b) {
+; CHECK: VFMADD{{.*}}mxcsr
+  %res = call <4 x float> @llvm.x86.fma.vfmadd.ss(<4 x float> %b, <4 x float> %a, <4 x float>
+%a)
+  ret <4 x float> %res
+}
+
+define <4 x float> @mxcsr_fma_ps(<4 x float> %a, <4 x float> %b) {
+; CHECK: VFMADD{{.*}}mxcsr
+  %res = call <4 x float> @llvm.x86.fma.vfmadd.ps(<4 x float> %b, <4 x float> %a, <4 x float>
+%a)
+  ret <4 x float> %res
+}
+
+define <8 x double> @mxcsr_fma_sae(<8 x double> %a, <8 x double> %b, <8 x double> %c) {
+; CHECK: VFMADD{{.*}}mxcsr
+  %res = call <8 x double> @llvm.x86.avx512.mask.vfmadd.pd.512(<8 x double> %a, <8 x double> %b, <8 x double> %c, i8 -1, i32 10)
+  ret <8 x double> %res
+}
+
+declare x86_mmx @llvm.x86.sse.cvtps2pi(<4 x float>)
+declare<4 x float> @llvm.x86.sse.cvtpi2ps(<4 x float>, x86_mmx)
+declare x86_mmx @llvm.x86.sse.cvttps2pi(<4 x float>)
+declare <2 x double> @llvm.x86.sse.cvtpi2pd(x86_mmx)
+declare x86_mmx @llvm.x86.sse.cvtpd2pi(<2 x double>)
+declare <4 x float> @llvm.x86.fma.vfmadd.ss(<4 x float>, <4 x float>, <4 x float>)
+declare <4 x float> @llvm.x86.fma.vfmadd.ps(<4 x float>, <4 x float>, <4 x float>)
+declare <8 x double> @llvm.x86.avx512.mask.vfmadd.pd.512(<8 x double>, <8 x double>, <8 x double>, i8, i32)
diff --git a/llvm/test/CodeGen/X86/neg-shl-add.ll b/llvm/test/CodeGen/X86/neg-shl-add.ll
index 71d65074f..9a4bde074 100644
--- a/llvm/test/CodeGen/X86/neg-shl-add.ll
+++ b/llvm/test/CodeGen/X86/neg-shl-add.ll
@@ -1,15 +1,31 @@
-; RUN: llc -mtriple=x86_64-- < %s | not grep negq
-
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=x86_64-- < %s | FileCheck %s
 ; These sequences don't need neg instructions; they can be done with
 ; a single shift and sub each.
 
 define i64 @foo(i64 %x, i64 %y, i64 %n) nounwind {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %rdx, %rcx
+; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    # kill: def $cl killed $cl killed $rcx
+; CHECK-NEXT:    shlq %cl, %rsi
+; CHECK-NEXT:    subq %rsi, %rax
+; CHECK-NEXT:    retq
   %a = sub i64 0, %y
   %b = shl i64 %a, %n
   %c = add i64 %b, %x
   ret i64 %c
 }
 define i64 @boo(i64 %x, i64 %y, i64 %n) nounwind {
+; CHECK-LABEL: boo:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %rdx, %rcx
+; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    # kill: def $cl killed $cl killed $rcx
+; CHECK-NEXT:    shlq %cl, %rsi
+; CHECK-NEXT:    subq %rsi, %rax
+; CHECK-NEXT:    retq
   %a = sub i64 0, %y
   %b = shl i64 %a, %n
   %c = add i64 %x, %b
diff --git a/llvm/test/CodeGen/X86/neg_fp.ll b/llvm/test/CodeGen/X86/neg_fp.ll
index 9cfe686b2..3c04aafce 100644
--- a/llvm/test/CodeGen/X86/neg_fp.ll
+++ b/llvm/test/CodeGen/X86/neg_fp.ll
@@ -1,12 +1,56 @@
-; RUN: llc < %s -mtriple=i686-- -mattr=+sse4.1 -o %t
-; RUN: grep xorps %t | count 1
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- -mattr=+sse4.1 | FileCheck %s
 
 ; Test that when we don't -enable-unsafe-fp-math, we don't do the optimization
 ; -0 - (A - B) to (B - A) because A==B, -0 != 0
 
-define float @negfp(float %a, float %b) {
+define float @negfp(float %a, float %b) nounwind {
+; CHECK-LABEL: negfp:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushl %eax
+; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; CHECK-NEXT:    subss {{[0-9]+}}(%esp), %xmm0
+; CHECK-NEXT:    xorps {{\.LCPI.*}}, %xmm0
+; CHECK-NEXT:    movss %xmm0, (%esp)
+; CHECK-NEXT:    flds (%esp)
+; CHECK-NEXT:    popl %eax
+; CHECK-NEXT:    retl
 entry:
 	%sub = fsub float %a, %b		; <float> [#uses=1]
 	%neg = fsub float -0.000000e+00, %sub		; <float> [#uses=1]
 	ret float %neg
 }
+
+; This may infinite loop if isNegatibleForFree and getNegatedExpression are conflicted.
+
+define double @negation_propagation(double* %arg, double %arg1, double %arg2) nounwind {
+; CHECK-LABEL: negation_propagation:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushl %ebp
+; CHECK-NEXT:    movl %esp, %ebp
+; CHECK-NEXT:    andl $-8, %esp
+; CHECK-NEXT:    subl $8, %esp
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    divsd 12(%ebp), %xmm0
+; CHECK-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
+; CHECK-NEXT:    mulsd %xmm0, %xmm1
+; CHECK-NEXT:    movapd %xmm0, %xmm2
+; CHECK-NEXT:    mulsd %xmm0, %xmm2
+; CHECK-NEXT:    mulsd %xmm0, %xmm2
+; CHECK-NEXT:    subsd %xmm2, %xmm1
+; CHECK-NEXT:    movsd %xmm1, (%esp)
+; CHECK-NEXT:    fldl (%esp)
+; CHECK-NEXT:    movl %ebp, %esp
+; CHECK-NEXT:    popl %ebp
+; CHECK-NEXT:    retl
+  %t = fdiv double 1.0, %arg1
+  %t7 = fmul double %t, %arg2
+  %t10 = fneg double %t7
+  %t11 = fmul double %t, %t
+  %t13 = fsub double %t11, %t
+  %t14 = fneg double %t
+  %t15 = fmul double %t, %t14
+  %t16 = fmul double %t, %t15
+  %t18 = fadd double %t16, %t7
+  ret double %t18
+}
diff --git a/llvm/test/CodeGen/X86/negate-add-zero.ll b/llvm/test/CodeGen/X86/negate-add-zero.ll
index 78a4cf89a..26fac9b51 100644
--- a/llvm/test/CodeGen/X86/negate-add-zero.ll
+++ b/llvm/test/CodeGen/X86/negate-add-zero.ll
@@ -1,4 +1,5 @@
-; RUN: llc < %s -enable-unsafe-fp-math | not grep xor
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -enable-unsafe-fp-math | FileCheck %s
 ; PR3374
 
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
@@ -824,6 +825,52 @@ declare void @_ZN11MatrixTools9transposeI11FixedMatrixIdLi6ELi6ELi0ELi0EEEENT_13
 declare void @_ZN21HNodeTranslateRotate311toCartesianEv(%struct.HNodeTranslateRotate3*)
 
 define linkonce void @_ZN21HNodeTranslateRotate36setVelERK9CDSVectorIdLi1EN3CDS12DefaultAllocEE(%struct.HNodeTranslateRotate3* %this, %"struct.CDSVector<double,0,CDS::DefaultAlloc>"* %velv) {
+; CHECK-LABEL: _ZN21HNodeTranslateRotate36setVelERK9CDSVectorIdLi1EN3CDS12DefaultAllocEE:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT:    fldl 0
+; CHECK-NEXT:    fldl 3184(%ecx)
+; CHECK-NEXT:    fld %st(1)
+; CHECK-NEXT:    fmull 3176(%ecx)
+; CHECK-NEXT:    fldz
+; CHECK-NEXT:    fld %st(1)
+; CHECK-NEXT:    fadd %st(1), %st
+; CHECK-NEXT:    fld %st(3)
+; CHECK-NEXT:    fmul %st(5), %st
+; CHECK-NEXT:    fadd %st(2), %st
+; CHECK-NEXT:    fxch %st(5)
+; CHECK-NEXT:    fmul %st, %st(0)
+; CHECK-NEXT:    fadd %st, %st(5)
+; CHECK-NEXT:    fsubr %st, %st(5)
+; CHECK-NEXT:    fxch %st(4)
+; CHECK-NEXT:    fmull -8
+; CHECK-NEXT:    fxch %st(5)
+; CHECK-NEXT:    fstl 8
+; CHECK-NEXT:    fxch %st(2)
+; CHECK-NEXT:    fsubp %st, %st(5)
+; CHECK-NEXT:    fxch %st(4)
+; CHECK-NEXT:    fsubp %st, %st(2)
+; CHECK-NEXT:    fxch %st(1)
+; CHECK-NEXT:    fadd %st(2), %st
+; CHECK-NEXT:    faddp %st, %st(2)
+; CHECK-NEXT:    fxch %st(1)
+; CHECK-NEXT:    fstl 16
+; CHECK-NEXT:    fxch %st(2)
+; CHECK-NEXT:    fadd %st, %st(0)
+; CHECK-NEXT:    fxch %st(1)
+; CHECK-NEXT:    fadd %st, %st(0)
+; CHECK-NEXT:    fxch %st(2)
+; CHECK-NEXT:    fadd %st, %st(0)
+; CHECK-NEXT:    fxch %st(1)
+; CHECK-NEXT:    fstpl 2056(%ecx)
+; CHECK-NEXT:    fxch %st(1)
+; CHECK-NEXT:    fstpl 2064(%ecx)
+; CHECK-NEXT:    fstpl 2072(%ecx)
+; CHECK-NEXT:    movl %eax, 0
+; CHECK-NEXT:    movl $4, 4
+; CHECK-NEXT:    movl $3, 8
+; CHECK-NEXT:    ud2
 	%1 = getelementptr double, double* null, i32 -1		; <double*> [#uses=1]
 	%2 = load double, double* %1, align 8		; <double> [#uses=1]
 	%3 = load double, double* null, align 8		; <double> [#uses=2]
diff --git a/llvm/test/CodeGen/X86/negative-stride-fptosi-user.ll b/llvm/test/CodeGen/X86/negative-stride-fptosi-user.ll
index d42b34444..18441ee65 100644
--- a/llvm/test/CodeGen/X86/negative-stride-fptosi-user.ll
+++ b/llvm/test/CodeGen/X86/negative-stride-fptosi-user.ll
@@ -1,4 +1,5 @@
-; RUN: llc < %s -mtriple=x86_64-- | grep cvtsi2sd
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-- | FileCheck %s
 
 ; LSR previously eliminated the sitofp by introducing an induction
 ; variable which stepped by a bogus ((double)UINT32_C(-1)). It's theoretically
@@ -6,6 +7,30 @@
 ; test should be changed if that is done.
 
 define void @foo(i32 %N) nounwind {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    testl %edi, %edi
+; CHECK-NEXT:    jns .LBB0_3
+; CHECK-NEXT:  # %bb.1: # %bb.preheader
+; CHECK-NEXT:    movl %edi, %ebx
+; CHECK-NEXT:    xorl %ebp, %ebp
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB0_2: # %bb
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    xorps %xmm0, %xmm0
+; CHECK-NEXT:    cvtsi2sd %ebp, %xmm0
+; CHECK-NEXT:    callq bar
+; CHECK-NEXT:    decl %ebp
+; CHECK-NEXT:    cmpl %ebp, %ebx
+; CHECK-NEXT:    jne .LBB0_2
+; CHECK-NEXT:  .LBB0_3: # %return
+; CHECK-NEXT:    addq $8, %rsp
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    retq
 entry:
   %0 = icmp slt i32 %N, 0                         ; <i1> [#uses=1]
   br i1 %0, label %bb, label %return
diff --git a/llvm/test/CodeGen/X86/no-plt-libcalls.ll b/llvm/test/CodeGen/X86/no-plt-libcalls.ll
index 876c8f10e..615b7bc33 100644
--- a/llvm/test/CodeGen/X86/no-plt-libcalls.ll
+++ b/llvm/test/CodeGen/X86/no-plt-libcalls.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; Check if "RtLibUseGOT" works correctly when lib calls are simplified.
 ; RUN: opt < %s -instcombine -S | FileCheck %s
 
@@ -5,10 +6,13 @@
 @hello_world = constant [13 x i8] c"hello world\0A\00"
 declare i32 @printf(i8*, ...)
 define void @printf_call() {
+; CHECK-LABEL: @printf_call(
+; CHECK-NEXT:    [[PUTS:%.*]] = call i32 @puts(i8* nonnull dereferenceable(1) getelementptr inbounds ([13 x i8], [13 x i8]* @hello_world, i64 0, i64 0))
+; CHECK-NEXT:    ret void
+;
   %fmt = getelementptr [4 x i8], [4 x i8]* @percent_s, i32 0, i32 0
   %str = getelementptr [13 x i8], [13 x i8]* @hello_world, i32 0, i32 0
   call i32 (i8*, ...) @printf(i8* %fmt, i8* %str)
-; CHECK:  call i32 @puts(i8* getelementptr inbounds ([13 x i8], [13 x i8]* @hello_world, i64 0, i64 0))
   ret void
 }
 
diff --git a/llvm/test/CodeGen/X86/no-sse-win64.ll b/llvm/test/CodeGen/X86/no-sse-win64.ll
new file mode 100644
index 000000000..c220b9606
--- /dev/null
+++ b/llvm/test/CodeGen/X86/no-sse-win64.ll
@@ -0,0 +1,129 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=x86_64-windows-msvc < %s -mattr=-sse | FileCheck %s
+; RUN: llc -mtriple=x86_64-windows-gnu < %s -mattr=-sse | FileCheck %s
+
+define void @recv_double(double %v, double* %p) {
+; CHECK-LABEL: recv_double:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %rcx, (%rdx)
+; CHECK-NEXT:    retq
+  store double %v, double* %p
+  ret void
+}
+
+define void @recv_float(float %v, float* %p) {
+; CHECK-LABEL: recv_float:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %ecx, (%rdx)
+; CHECK-NEXT:    retq
+  store float %v, float* %p
+  ret void
+}
+
+define dso_local double @ret_double(double* %p) {
+; CHECK-LABEL: ret_double:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movq (%rcx), %rax
+; CHECK-NEXT:    retq
+entry:
+  %v = load double, double* %p
+  ret double %v
+}
+
+define dso_local float @ret_float(float* %p) {
+; CHECK-LABEL: ret_float:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movl (%rcx), %eax
+; CHECK-NEXT:    retq
+entry:
+  %v = load float, float* %p
+  ret float %v
+}
+
+declare void @take_double(double)
+declare void @take_float(float)
+
+define void @pass_double(double* %p) {
+; CHECK-LABEL: pass_double:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subq $40, %rsp
+; CHECK-NEXT:    .seh_stackalloc 40
+; CHECK-NEXT:    .seh_endprologue
+; CHECK-NEXT:    movq (%rcx), %rcx
+; CHECK-NEXT:    callq take_double
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    addq $40, %rsp
+; CHECK-NEXT:    retq
+; CHECK-NEXT:    .seh_handlerdata
+; CHECK-NEXT:    .text
+; CHECK-NEXT:    .seh_endproc
+  %v = load double, double* %p
+  call void @take_double(double %v)
+  ret void
+}
+
+define void @pass_float(float* %p) {
+; CHECK-LABEL: pass_float:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subq $40, %rsp
+; CHECK-NEXT:    .seh_stackalloc 40
+; CHECK-NEXT:    .seh_endprologue
+; CHECK-NEXT:    movl (%rcx), %ecx
+; CHECK-NEXT:    callq take_float
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    addq $40, %rsp
+; CHECK-NEXT:    retq
+; CHECK-NEXT:    .seh_handlerdata
+; CHECK-NEXT:    .text
+; CHECK-NEXT:    .seh_endproc
+  %v = load float, float* %p
+  call void @take_float(float %v)
+  ret void
+}
+
+declare double @produce_double()
+declare float @produce_float()
+
+define void @call_double(double* %p) {
+; CHECK-LABEL: call_double:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rsi
+; CHECK-NEXT:    .seh_pushreg %rsi
+; CHECK-NEXT:    subq $32, %rsp
+; CHECK-NEXT:    .seh_stackalloc 32
+; CHECK-NEXT:    .seh_endprologue
+; CHECK-NEXT:    movq %rcx, %rsi
+; CHECK-NEXT:    callq produce_double
+; CHECK-NEXT:    movq %rax, (%rsi)
+; CHECK-NEXT:    addq $32, %rsp
+; CHECK-NEXT:    popq %rsi
+; CHECK-NEXT:    retq
+; CHECK-NEXT:    .seh_handlerdata
+; CHECK-NEXT:    .text
+; CHECK-NEXT:    .seh_endproc
+  %v = call double @produce_double()
+  store double %v, double* %p
+  ret void
+}
+
+define void @call_float(float* %p) {
+; CHECK-LABEL: call_float:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rsi
+; CHECK-NEXT:    .seh_pushreg %rsi
+; CHECK-NEXT:    subq $32, %rsp
+; CHECK-NEXT:    .seh_stackalloc 32
+; CHECK-NEXT:    .seh_endprologue
+; CHECK-NEXT:    movq %rcx, %rsi
+; CHECK-NEXT:    callq produce_float
+; CHECK-NEXT:    movl %eax, (%rsi)
+; CHECK-NEXT:    addq $32, %rsp
+; CHECK-NEXT:    popq %rsi
+; CHECK-NEXT:    retq
+; CHECK-NEXT:    .seh_handlerdata
+; CHECK-NEXT:    .text
+; CHECK-NEXT:    .seh_endproc
+  %v = call float @produce_float()
+  store float %v, float* %p
+  ret void
+}
diff --git a/llvm/test/CodeGen/X86/no-sse-x86.ll b/llvm/test/CodeGen/X86/no-sse-x86.ll
new file mode 100644
index 000000000..45fea53af
--- /dev/null
+++ b/llvm/test/CodeGen/X86/no-sse-x86.ll
@@ -0,0 +1,76 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mcpu=i686 -mattr=+sse | FileCheck %s
+; RUN: llc < %s -mcpu=i686 -mattr=-sse 2>&1 | FileCheck --check-prefix NOSSE %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32"
+target triple = "i386-unknown-linux-gnu"
+@f = external global float
+@d = external global double
+
+define void @test() nounwind {
+; CHECK-LABEL: test:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    subl $12, %esp
+; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; CHECK-NEXT:    movss %xmm0, (%esp)
+; CHECK-NEXT:    calll foo1
+; CHECK-NEXT:    fstps f
+; CHECK-NEXT:    fldl d
+; CHECK-NEXT:    fstpl (%esp)
+; CHECK-NEXT:    calll foo2
+; CHECK-NEXT:    fstpl d
+; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; CHECK-NEXT:    movss %xmm0, (%esp)
+; CHECK-NEXT:    calll foo3
+; CHECK-NEXT:    fstps f
+; CHECK-NEXT:    fldl d
+; CHECK-NEXT:    fstpl (%esp)
+; CHECK-NEXT:    calll foo4
+; CHECK-NEXT:    fstpl d
+; CHECK-NEXT:    addl $12, %esp
+; CHECK-NEXT:    retl
+;
+; NOSSE-LABEL: test:
+; NOSSE:       # %bb.0: # %entry
+; NOSSE-NEXT:    subl $12, %esp
+; NOSSE-NEXT:    flds f
+; NOSSE-NEXT:    fstps (%esp)
+; NOSSE-NEXT:    calll foo1
+; NOSSE-NEXT:    fstps f
+; NOSSE-NEXT:    fldl d
+; NOSSE-NEXT:    fstpl (%esp)
+; NOSSE-NEXT:    calll foo2
+; NOSSE-NEXT:    fstpl d
+; NOSSE-NEXT:    flds f
+; NOSSE-NEXT:    fstps (%esp)
+; NOSSE-NEXT:    calll foo3
+; NOSSE-NEXT:    fstps f
+; NOSSE-NEXT:    fldl d
+; NOSSE-NEXT:    fstpl (%esp)
+; NOSSE-NEXT:    calll foo4
+; NOSSE-NEXT:    fstpl d
+; NOSSE-NEXT:    addl $12, %esp
+; NOSSE-NEXT:    retl
+entry:
+  %0 = load float, float* @f, align 4
+  %1 = tail call inreg float @foo1(float inreg %0) nounwind
+  store float %1, float* @f, align 4
+  %2 = load double, double* @d, align 8
+  %3 = tail call inreg double @foo2(double inreg %2) nounwind
+  store double %3, double* @d, align 8
+  %4 = load float, float* @f, align 4
+  %5 = tail call inreg float @foo3(float inreg %4) nounwind
+  store float %5, float* @f, align 4
+  %6 = load double, double* @d, align 8
+  %7 = tail call inreg double @foo4(double inreg %6) nounwind
+  store double %7, double* @d, align 8
+  ret void
+}
+
+declare inreg float @foo1(float inreg)
+
+declare inreg double @foo2(double inreg)
+
+declare inreg float @foo3(float inreg)
+
+declare inreg double @foo4(double inreg)
diff --git a/llvm/test/CodeGen/X86/nobt.ll b/llvm/test/CodeGen/X86/nobt.ll
index b60723e9c..b994b7f95 100644
--- a/llvm/test/CodeGen/X86/nobt.ll
+++ b/llvm/test/CodeGen/X86/nobt.ll
@@ -1,9 +1,19 @@
-; RUN: llc < %s -mtriple=i686-- | not grep btl
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- | FileCheck %s
 
 ; This tests some cases where BT must not be generated.  See also bt.ll.
 ; Fixes 20040709-[12].c in gcc testsuite.
 
 define void @test2(i32 %x, i32 %n) nounwind {
+; CHECK-LABEL: test2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    testb %al, %al
+; CHECK-NEXT:    jne .LBB0_2
+; CHECK-NEXT:  # %bb.1: # %bb
+; CHECK-NEXT:    calll foo
+; CHECK-NEXT:  .LBB0_2: # %UnifiedReturnBlock
+; CHECK-NEXT:    retl
 entry:
         %tmp1 = and i32 %x, 1
         %tmp2 = urem i32 %tmp1, 15
@@ -20,6 +30,15 @@ UnifiedReturnBlock:		; preds = %entry
 }
 
 define void @test3(i32 %x, i32 %n) nounwind {
+; CHECK-LABEL: test3:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    testb %al, %al
+; CHECK-NEXT:    jne .LBB1_2
+; CHECK-NEXT:  # %bb.1: # %bb
+; CHECK-NEXT:    calll foo
+; CHECK-NEXT:  .LBB1_2: # %UnifiedReturnBlock
+; CHECK-NEXT:    retl
 entry:
         %tmp1 = and i32 %x, 1
         %tmp2 = urem i32 %tmp1, 15
@@ -36,6 +55,15 @@ UnifiedReturnBlock:		; preds = %entry
 }
 
 define void @test4(i32 %x, i32 %n) nounwind {
+; CHECK-LABEL: test4:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movb $1, %al
+; CHECK-NEXT:    testb %al, %al
+; CHECK-NEXT:    jne .LBB2_2
+; CHECK-NEXT:  # %bb.1: # %bb
+; CHECK-NEXT:    calll foo
+; CHECK-NEXT:  .LBB2_2: # %UnifiedReturnBlock
+; CHECK-NEXT:    retl
 entry:
         %tmp1 = and i32 %x, 1
         %tmp2 = urem i32 %tmp1, 15
@@ -52,6 +80,15 @@ UnifiedReturnBlock:		; preds = %entry
 }
 
 define void @test5(i32 %x, i32 %n) nounwind {
+; CHECK-LABEL: test5:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movb $1, %al
+; CHECK-NEXT:    testb %al, %al
+; CHECK-NEXT:    jne .LBB3_2
+; CHECK-NEXT:  # %bb.1: # %bb
+; CHECK-NEXT:    calll foo
+; CHECK-NEXT:  .LBB3_2: # %UnifiedReturnBlock
+; CHECK-NEXT:    retl
 entry:
         %tmp1 = and i32 %x, 1
         %tmp2 = urem i32 %tmp1, 15
diff --git a/llvm/test/CodeGen/X86/nocf_check.ll b/llvm/test/CodeGen/X86/nocf_check.ll
index 994d20503..c8d18eaa6 100644
--- a/llvm/test/CodeGen/X86/nocf_check.ll
+++ b/llvm/test/CodeGen/X86/nocf_check.ll
@@ -38,7 +38,7 @@ entry:
   ret void
 }
 
-attributes #0 = { noinline nocf_check nounwind optnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-features"="+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noinline nocf_check nounwind optnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-features"="+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nocf_check }
 
 !llvm.module.flags = !{!0}
diff --git a/llvm/test/CodeGen/X86/nomovtopush.ll b/llvm/test/CodeGen/X86/nomovtopush.ll
new file mode 100644
index 000000000..f690c2377
--- /dev/null
+++ b/llvm/test/CodeGen/X86/nomovtopush.ll
@@ -0,0 +1,59 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i386-pc-windows-msvc | FileCheck %s
+
+target datalayout = "e-m:x-p:32:32-i64:64-f80:32-n8:16:32-a:0:32-S32"
+target triple = "i386-pc-windows-msvc"
+
+%struct._param_str = type { i32, i32, [4096 x i32], i32 }
+
+@g_d = common dso_local local_unnamed_addr global i32 0, align 4
+@g_c = common dso_local local_unnamed_addr global i32 0, align 4
+@g_b = common dso_local local_unnamed_addr global i32 0, align 4
+@g_a = common dso_local local_unnamed_addr global i32 0, align 4
+@g_param = common dso_local global %struct._param_str zeroinitializer, align 4
+
+; Function Attrs: nounwind
+define dso_local i32 @test() local_unnamed_addr {
+; CHECK-LABEL: test:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushl %edi
+; CHECK-NEXT:    pushl %esi
+; CHECK-NEXT:    movl $16396, %eax # imm = 0x400C
+; CHECK-NEXT:    calll __chkstk
+; CHECK-NEXT:    movl _g_d, %eax
+; CHECK-NEXT:    movl _g_c, %ecx
+; CHECK-NEXT:    movl _g_b, %edx
+; CHECK-NEXT:    movl _g_a, %esi
+; CHECK-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movl %esi, (%esp)
+; CHECK-NEXT:    calll _bar
+; CHECK-NEXT:    movl $4099, %ecx # imm = 0x1003
+; CHECK-NEXT:    movl %esp, %edi
+; CHECK-NEXT:    movl $_g_param, %esi
+; CHECK-NEXT:    rep;movsl (%esi), %es:(%edi)
+; CHECK-NEXT:    calll _foo
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    addl $16396, %esp # imm = 0x400C
+; CHECK-NEXT:    popl %esi
+; CHECK-NEXT:    popl %edi
+; CHECK-NEXT:    retl
+entry:
+  %0 = load i32, i32* @g_d, align 4, !tbaa !3
+  %1 = load i32, i32* @g_c, align 4, !tbaa !3
+  %2 = load i32, i32* @g_b, align 4, !tbaa !3
+  %3 = load i32, i32* @g_a, align 4, !tbaa !3
+  %call = tail call i32 @bar(i32 %3, i32 %2, i32 %1, i32 %0) #2
+  tail call void @foo(%struct._param_str* byval nonnull align 4 @g_param) #2
+  ret i32 0
+}
+
+declare dso_local i32 @bar(i32, i32, i32, i32) local_unnamed_addr
+
+declare dso_local void @foo(%struct._param_str* byval align 4) local_unnamed_addr
+
+!3 = !{!4, !4, i64 0}
+!4 = !{!"int", !5, i64 0}
+!5 = !{!"omnipotent char", !6, i64 0}
+!6 = !{!"Simple C/C++ TBAA"}
diff --git a/llvm/test/CodeGen/X86/nontemporal-3.ll b/llvm/test/CodeGen/X86/nontemporal-3.ll
index f0a2f6fac..457e5e19d 100644
--- a/llvm/test/CodeGen/X86/nontemporal-3.ll
+++ b/llvm/test/CodeGen/X86/nontemporal-3.ll
@@ -37,27 +37,12 @@ define void @test_zero_v2f64_align1(<2 x double>* %dst) nounwind {
 }
 
 define void @test_zero_v4f32_align1(<4 x float>* %dst) nounwind {
-; SSE2-LABEL: test_zero_v4f32_align1:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    xorl %eax, %eax
-; SSE2-NEXT:    movntiq %rax, 8(%rdi)
-; SSE2-NEXT:    movntiq %rax, (%rdi)
-; SSE2-NEXT:    retq
-;
-; SSE4A-LABEL: test_zero_v4f32_align1:
-; SSE4A:       # %bb.0:
-; SSE4A-NEXT:    xorl %eax, %eax
-; SSE4A-NEXT:    movntiq %rax, 8(%rdi)
-; SSE4A-NEXT:    xorps %xmm0, %xmm0
-; SSE4A-NEXT:    movntsd %xmm0, (%rdi)
-; SSE4A-NEXT:    retq
-;
-; SSE41-LABEL: test_zero_v4f32_align1:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    xorl %eax, %eax
-; SSE41-NEXT:    movntiq %rax, 8(%rdi)
-; SSE41-NEXT:    movntiq %rax, (%rdi)
-; SSE41-NEXT:    retq
+; SSE-LABEL: test_zero_v4f32_align1:
+; SSE:       # %bb.0:
+; SSE-NEXT:    xorl %eax, %eax
+; SSE-NEXT:    movntiq %rax, 8(%rdi)
+; SSE-NEXT:    movntiq %rax, (%rdi)
+; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: test_zero_v4f32_align1:
 ; AVX:       # %bb.0:
@@ -77,27 +62,12 @@ define void @test_zero_v4f32_align1(<4 x float>* %dst) nounwind {
 }
 
 define void @test_zero_v2i64_align1(<2 x i64>* %dst) nounwind {
-; SSE2-LABEL: test_zero_v2i64_align1:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    xorl %eax, %eax
-; SSE2-NEXT:    movntiq %rax, 8(%rdi)
-; SSE2-NEXT:    movntiq %rax, (%rdi)
-; SSE2-NEXT:    retq
-;
-; SSE4A-LABEL: test_zero_v2i64_align1:
-; SSE4A:       # %bb.0:
-; SSE4A-NEXT:    xorl %eax, %eax
-; SSE4A-NEXT:    movntiq %rax, 8(%rdi)
-; SSE4A-NEXT:    xorps %xmm0, %xmm0
-; SSE4A-NEXT:    movntsd %xmm0, (%rdi)
-; SSE4A-NEXT:    retq
-;
-; SSE41-LABEL: test_zero_v2i64_align1:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    xorl %eax, %eax
-; SSE41-NEXT:    movntiq %rax, 8(%rdi)
-; SSE41-NEXT:    movntiq %rax, (%rdi)
-; SSE41-NEXT:    retq
+; SSE-LABEL: test_zero_v2i64_align1:
+; SSE:       # %bb.0:
+; SSE-NEXT:    xorl %eax, %eax
+; SSE-NEXT:    movntiq %rax, 8(%rdi)
+; SSE-NEXT:    movntiq %rax, (%rdi)
+; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: test_zero_v2i64_align1:
 ; AVX:       # %bb.0:
@@ -117,27 +87,12 @@ define void @test_zero_v2i64_align1(<2 x i64>* %dst) nounwind {
 }
 
 define void @test_zero_v4i32_align1(<4 x i32>* %dst) nounwind {
-; SSE2-LABEL: test_zero_v4i32_align1:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    xorl %eax, %eax
-; SSE2-NEXT:    movntiq %rax, 8(%rdi)
-; SSE2-NEXT:    movntiq %rax, (%rdi)
-; SSE2-NEXT:    retq
-;
-; SSE4A-LABEL: test_zero_v4i32_align1:
-; SSE4A:       # %bb.0:
-; SSE4A-NEXT:    xorl %eax, %eax
-; SSE4A-NEXT:    movntiq %rax, 8(%rdi)
-; SSE4A-NEXT:    xorps %xmm0, %xmm0
-; SSE4A-NEXT:    movntsd %xmm0, (%rdi)
-; SSE4A-NEXT:    retq
-;
-; SSE41-LABEL: test_zero_v4i32_align1:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    xorl %eax, %eax
-; SSE41-NEXT:    movntiq %rax, 8(%rdi)
-; SSE41-NEXT:    movntiq %rax, (%rdi)
-; SSE41-NEXT:    retq
+; SSE-LABEL: test_zero_v4i32_align1:
+; SSE:       # %bb.0:
+; SSE-NEXT:    xorl %eax, %eax
+; SSE-NEXT:    movntiq %rax, 8(%rdi)
+; SSE-NEXT:    movntiq %rax, (%rdi)
+; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: test_zero_v4i32_align1:
 ; AVX:       # %bb.0:
@@ -157,27 +112,12 @@ define void @test_zero_v4i32_align1(<4 x i32>* %dst) nounwind {
 }
 
 define void @test_zero_v8i16_align1(<8 x i16>* %dst) nounwind {
-; SSE2-LABEL: test_zero_v8i16_align1:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    xorl %eax, %eax
-; SSE2-NEXT:    movntiq %rax, 8(%rdi)
-; SSE2-NEXT:    movntiq %rax, (%rdi)
-; SSE2-NEXT:    retq
-;
-; SSE4A-LABEL: test_zero_v8i16_align1:
-; SSE4A:       # %bb.0:
-; SSE4A-NEXT:    xorl %eax, %eax
-; SSE4A-NEXT:    movntiq %rax, 8(%rdi)
-; SSE4A-NEXT:    xorps %xmm0, %xmm0
-; SSE4A-NEXT:    movntsd %xmm0, (%rdi)
-; SSE4A-NEXT:    retq
-;
-; SSE41-LABEL: test_zero_v8i16_align1:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    xorl %eax, %eax
-; SSE41-NEXT:    movntiq %rax, 8(%rdi)
-; SSE41-NEXT:    movntiq %rax, (%rdi)
-; SSE41-NEXT:    retq
+; SSE-LABEL: test_zero_v8i16_align1:
+; SSE:       # %bb.0:
+; SSE-NEXT:    xorl %eax, %eax
+; SSE-NEXT:    movntiq %rax, 8(%rdi)
+; SSE-NEXT:    movntiq %rax, (%rdi)
+; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: test_zero_v8i16_align1:
 ; AVX:       # %bb.0:
@@ -197,27 +137,12 @@ define void @test_zero_v8i16_align1(<8 x i16>* %dst) nounwind {
 }
 
 define void @test_zero_v16i8_align1(<16 x i8>* %dst) nounwind {
-; SSE2-LABEL: test_zero_v16i8_align1:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    xorl %eax, %eax
-; SSE2-NEXT:    movntiq %rax, 8(%rdi)
-; SSE2-NEXT:    movntiq %rax, (%rdi)
-; SSE2-NEXT:    retq
-;
-; SSE4A-LABEL: test_zero_v16i8_align1:
-; SSE4A:       # %bb.0:
-; SSE4A-NEXT:    xorl %eax, %eax
-; SSE4A-NEXT:    movntiq %rax, 8(%rdi)
-; SSE4A-NEXT:    xorps %xmm0, %xmm0
-; SSE4A-NEXT:    movntsd %xmm0, (%rdi)
-; SSE4A-NEXT:    retq
-;
-; SSE41-LABEL: test_zero_v16i8_align1:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    xorl %eax, %eax
-; SSE41-NEXT:    movntiq %rax, 8(%rdi)
-; SSE41-NEXT:    movntiq %rax, (%rdi)
-; SSE41-NEXT:    retq
+; SSE-LABEL: test_zero_v16i8_align1:
+; SSE:       # %bb.0:
+; SSE-NEXT:    xorl %eax, %eax
+; SSE-NEXT:    movntiq %rax, 8(%rdi)
+; SSE-NEXT:    movntiq %rax, (%rdi)
+; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: test_zero_v16i8_align1:
 ; AVX:       # %bb.0:
@@ -281,10 +206,11 @@ define void @test_zero_v8f32_align1(<8 x float>* %dst) nounwind {
 ;
 ; SSE4A-LABEL: test_zero_v8f32_align1:
 ; SSE4A:       # %bb.0:
+; SSE4A-NEXT:    xorl %eax, %eax
+; SSE4A-NEXT:    movntiq %rax, 8(%rdi)
+; SSE4A-NEXT:    movntiq %rax, 24(%rdi)
 ; SSE4A-NEXT:    xorps %xmm0, %xmm0
-; SSE4A-NEXT:    movntsd %xmm0, 8(%rdi)
 ; SSE4A-NEXT:    movntsd %xmm0, (%rdi)
-; SSE4A-NEXT:    movntsd %xmm0, 24(%rdi)
 ; SSE4A-NEXT:    movntsd %xmm0, 16(%rdi)
 ; SSE4A-NEXT:    retq
 ;
@@ -725,14 +651,15 @@ define void @test_zero_v16f32_align1(<16 x float>* %dst) nounwind {
 ;
 ; SSE4A-LABEL: test_zero_v16f32_align1:
 ; SSE4A:       # %bb.0:
+; SSE4A-NEXT:    xorl %eax, %eax
+; SSE4A-NEXT:    movntiq %rax, 24(%rdi)
+; SSE4A-NEXT:    movntiq %rax, 8(%rdi)
+; SSE4A-NEXT:    movntiq %rax, 56(%rdi)
+; SSE4A-NEXT:    movntiq %rax, 40(%rdi)
 ; SSE4A-NEXT:    xorps %xmm0, %xmm0
-; SSE4A-NEXT:    movntsd %xmm0, 24(%rdi)
 ; SSE4A-NEXT:    movntsd %xmm0, 16(%rdi)
-; SSE4A-NEXT:    movntsd %xmm0, 8(%rdi)
 ; SSE4A-NEXT:    movntsd %xmm0, (%rdi)
-; SSE4A-NEXT:    movntsd %xmm0, 56(%rdi)
 ; SSE4A-NEXT:    movntsd %xmm0, 48(%rdi)
-; SSE4A-NEXT:    movntsd %xmm0, 40(%rdi)
 ; SSE4A-NEXT:    movntsd %xmm0, 32(%rdi)
 ; SSE4A-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/nontemporal-loads-2.ll b/llvm/test/CodeGen/X86/nontemporal-loads-2.ll
index 0441d3a49..a141db3c0 100644
--- a/llvm/test/CodeGen/X86/nontemporal-loads-2.ll
+++ b/llvm/test/CodeGen/X86/nontemporal-loads-2.ll
@@ -511,16 +511,10 @@ define <32 x i16> @test_v32i16_align1(<32 x i16>* %src) nounwind {
 ; AVX2-NEXT:    vmovups 32(%rdi), %ymm1
 ; AVX2-NEXT:    retq
 ;
-; AVX512DQ-LABEL: test_v32i16_align1:
-; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vmovups (%rdi), %ymm0
-; AVX512DQ-NEXT:    vmovups 32(%rdi), %ymm1
-; AVX512DQ-NEXT:    retq
-;
-; AVX512BW-LABEL: test_v32i16_align1:
-; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vmovups (%rdi), %zmm0
-; AVX512BW-NEXT:    retq
+; AVX512-LABEL: test_v32i16_align1:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovups (%rdi), %zmm0
+; AVX512-NEXT:    retq
   %1 = load <32 x i16>, <32 x i16>* %src, align 1, !nontemporal !1
   ret <32 x i16> %1
 }
@@ -546,16 +540,10 @@ define <64 x i8> @test_v64i8_align1(<64 x i8>* %src) nounwind {
 ; AVX2-NEXT:    vmovups 32(%rdi), %ymm1
 ; AVX2-NEXT:    retq
 ;
-; AVX512DQ-LABEL: test_v64i8_align1:
-; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vmovups (%rdi), %ymm0
-; AVX512DQ-NEXT:    vmovups 32(%rdi), %ymm1
-; AVX512DQ-NEXT:    retq
-;
-; AVX512BW-LABEL: test_v64i8_align1:
-; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vmovups (%rdi), %zmm0
-; AVX512BW-NEXT:    retq
+; AVX512-LABEL: test_v64i8_align1:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovups (%rdi), %zmm0
+; AVX512-NEXT:    retq
   %1 = load <64 x i8>, <64 x i8>* %src, align 1, !nontemporal !1
   ret <64 x i8> %1
 }
@@ -948,7 +936,7 @@ define <32 x i16> @test_v32i16_align16(<32 x i16>* %src) nounwind {
 ; AVX512DQ-NEXT:    vmovntdqa 32(%rdi), %xmm0
 ; AVX512DQ-NEXT:    vmovdqa %xmm0, {{[0-9]+}}(%rsp)
 ; AVX512DQ-NEXT:    vmovaps (%rsp), %ymm0
-; AVX512DQ-NEXT:    vmovaps {{[0-9]+}}(%rsp), %ymm1
+; AVX512DQ-NEXT:    vinsertf64x4 $1, {{[0-9]+}}(%rsp), %zmm0, %zmm0
 ; AVX512DQ-NEXT:    movq %rbp, %rsp
 ; AVX512DQ-NEXT:    popq %rbp
 ; AVX512DQ-NEXT:    retq
@@ -1047,7 +1035,7 @@ define <64 x i8> @test_v64i8_align16(<64 x i8>* %src) nounwind {
 ; AVX512DQ-NEXT:    vmovntdqa 32(%rdi), %xmm0
 ; AVX512DQ-NEXT:    vmovdqa %xmm0, {{[0-9]+}}(%rsp)
 ; AVX512DQ-NEXT:    vmovaps (%rsp), %ymm0
-; AVX512DQ-NEXT:    vmovaps {{[0-9]+}}(%rsp), %ymm1
+; AVX512DQ-NEXT:    vinsertf64x4 $1, {{[0-9]+}}(%rsp), %zmm0, %zmm0
 ; AVX512DQ-NEXT:    movq %rbp, %rsp
 ; AVX512DQ-NEXT:    popq %rbp
 ; AVX512DQ-NEXT:    retq
@@ -1315,6 +1303,7 @@ define <32 x i16> @test_v32i16_align32(<32 x i16>* %src) nounwind {
 ; AVX512DQ:       # %bb.0:
 ; AVX512DQ-NEXT:    vmovntdqa (%rdi), %ymm0
 ; AVX512DQ-NEXT:    vmovntdqa 32(%rdi), %ymm1
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512BW-LABEL: test_v32i16_align32:
@@ -1372,6 +1361,7 @@ define <64 x i8> @test_v64i8_align32(<64 x i8>* %src) nounwind {
 ; AVX512DQ:       # %bb.0:
 ; AVX512DQ-NEXT:    vmovntdqa (%rdi), %ymm0
 ; AVX512DQ-NEXT:    vmovntdqa 32(%rdi), %ymm1
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512BW-LABEL: test_v64i8_align32:
diff --git a/llvm/test/CodeGen/X86/nontemporal-loads.ll b/llvm/test/CodeGen/X86/nontemporal-loads.ll
index 8f0118d39..5dfdb917a 100644
--- a/llvm/test/CodeGen/X86/nontemporal-loads.ll
+++ b/llvm/test/CodeGen/X86/nontemporal-loads.ll
@@ -550,22 +550,10 @@ define <32 x i16> @test_v32i16(<32 x i16>* %src) {
 ; AVX2-NEXT:    vmovntdqa 32(%rdi), %ymm1
 ; AVX2-NEXT:    retq
 ;
-; AVX512F-LABEL: test_v32i16:
-; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovntdqa (%rdi), %ymm0
-; AVX512F-NEXT:    vmovntdqa 32(%rdi), %ymm1
-; AVX512F-NEXT:    retq
-;
-; AVX512BW-LABEL: test_v32i16:
-; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vmovntdqa (%rdi), %zmm0
-; AVX512BW-NEXT:    retq
-;
-; AVX512VL-LABEL: test_v32i16:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vmovntdqa (%rdi), %ymm0
-; AVX512VL-NEXT:    vmovntdqa 32(%rdi), %ymm1
-; AVX512VL-NEXT:    retq
+; AVX512-LABEL: test_v32i16:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovntdqa (%rdi), %zmm0
+; AVX512-NEXT:    retq
   %1 = load <32 x i16>, <32 x i16>* %src, align 64, !nontemporal !1
   ret <32 x i16> %1
 }
@@ -603,22 +591,10 @@ define <64 x i8> @test_v64i8(<64 x i8>* %src) {
 ; AVX2-NEXT:    vmovntdqa 32(%rdi), %ymm1
 ; AVX2-NEXT:    retq
 ;
-; AVX512F-LABEL: test_v64i8:
-; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovntdqa (%rdi), %ymm0
-; AVX512F-NEXT:    vmovntdqa 32(%rdi), %ymm1
-; AVX512F-NEXT:    retq
-;
-; AVX512BW-LABEL: test_v64i8:
-; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vmovntdqa (%rdi), %zmm0
-; AVX512BW-NEXT:    retq
-;
-; AVX512VL-LABEL: test_v64i8:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vmovntdqa (%rdi), %ymm0
-; AVX512VL-NEXT:    vmovntdqa 32(%rdi), %ymm1
-; AVX512VL-NEXT:    retq
+; AVX512-LABEL: test_v64i8:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovntdqa (%rdi), %zmm0
+; AVX512-NEXT:    retq
   %1 = load <64 x i8>, <64 x i8>* %src, align 64, !nontemporal !1
   ret <64 x i8> %1
 }
@@ -1299,10 +1275,12 @@ define <32 x i16> @test_arg_v32i16(<32 x i16> %arg, <32 x i16>* %src) {
 ;
 ; AVX512F-LABEL: test_arg_v32i16:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovntdqa 32(%rdi), %ymm2
-; AVX512F-NEXT:    vmovntdqa (%rdi), %ymm3
-; AVX512F-NEXT:    vpaddw %ymm3, %ymm0, %ymm0
-; AVX512F-NEXT:    vpaddw %ymm2, %ymm1, %ymm1
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
+; AVX512F-NEXT:    vmovntdqa (%rdi), %ymm2
+; AVX512F-NEXT:    vmovntdqa 32(%rdi), %ymm3
+; AVX512F-NEXT:    vpaddw %ymm3, %ymm1, %ymm1
+; AVX512F-NEXT:    vpaddw %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: test_arg_v32i16:
@@ -1313,10 +1291,12 @@ define <32 x i16> @test_arg_v32i16(<32 x i16> %arg, <32 x i16>* %src) {
 ;
 ; AVX512VL-LABEL: test_arg_v32i16:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vmovntdqa 32(%rdi), %ymm2
-; AVX512VL-NEXT:    vmovntdqa (%rdi), %ymm3
-; AVX512VL-NEXT:    vpaddw %ymm3, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpaddw %ymm2, %ymm1, %ymm1
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
+; AVX512VL-NEXT:    vmovntdqa (%rdi), %ymm2
+; AVX512VL-NEXT:    vmovntdqa 32(%rdi), %ymm3
+; AVX512VL-NEXT:    vpaddw %ymm3, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpaddw %ymm2, %ymm0, %ymm0
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512VL-NEXT:    retq
   %1 = load <32 x i16>, <32 x i16>* %src, align 64, !nontemporal !1
   %2 = add <32 x i16> %arg, %1
@@ -1370,10 +1350,12 @@ define <64 x i8> @test_arg_v64i8(<64 x i8> %arg, <64 x i8>* %src) {
 ;
 ; AVX512F-LABEL: test_arg_v64i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovntdqa 32(%rdi), %ymm2
-; AVX512F-NEXT:    vmovntdqa (%rdi), %ymm3
-; AVX512F-NEXT:    vpaddb %ymm3, %ymm0, %ymm0
-; AVX512F-NEXT:    vpaddb %ymm2, %ymm1, %ymm1
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
+; AVX512F-NEXT:    vmovntdqa (%rdi), %ymm2
+; AVX512F-NEXT:    vmovntdqa 32(%rdi), %ymm3
+; AVX512F-NEXT:    vpaddb %ymm3, %ymm1, %ymm1
+; AVX512F-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: test_arg_v64i8:
@@ -1384,10 +1366,12 @@ define <64 x i8> @test_arg_v64i8(<64 x i8> %arg, <64 x i8>* %src) {
 ;
 ; AVX512VL-LABEL: test_arg_v64i8:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vmovntdqa 32(%rdi), %ymm2
-; AVX512VL-NEXT:    vmovntdqa (%rdi), %ymm3
-; AVX512VL-NEXT:    vpaddb %ymm3, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpaddb %ymm2, %ymm1, %ymm1
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
+; AVX512VL-NEXT:    vmovntdqa (%rdi), %ymm2
+; AVX512VL-NEXT:    vmovntdqa 32(%rdi), %ymm3
+; AVX512VL-NEXT:    vpaddb %ymm3, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512VL-NEXT:    retq
   %1 = load <64 x i8>, <64 x i8>* %src, align 64, !nontemporal !1
   %2 = add <64 x i8> %arg, %1
@@ -1742,22 +1726,10 @@ define <32 x i16> @test_unaligned_v32i16(<32 x i16>* %src) {
 ; AVX-NEXT:    vmovups 32(%rdi), %ymm1
 ; AVX-NEXT:    retq
 ;
-; AVX512F-LABEL: test_unaligned_v32i16:
-; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovups (%rdi), %ymm0
-; AVX512F-NEXT:    vmovups 32(%rdi), %ymm1
-; AVX512F-NEXT:    retq
-;
-; AVX512BW-LABEL: test_unaligned_v32i16:
-; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vmovups (%rdi), %zmm0
-; AVX512BW-NEXT:    retq
-;
-; AVX512VL-LABEL: test_unaligned_v32i16:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vmovups (%rdi), %ymm0
-; AVX512VL-NEXT:    vmovups 32(%rdi), %ymm1
-; AVX512VL-NEXT:    retq
+; AVX512-LABEL: test_unaligned_v32i16:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovups (%rdi), %zmm0
+; AVX512-NEXT:    retq
   %1 = load <32 x i16>, <32 x i16>* %src, align 1, !nontemporal !1
   ret <32 x i16> %1
 }
@@ -1777,22 +1749,10 @@ define <64 x i8> @test_unaligned_v64i8(<64 x i8>* %src) {
 ; AVX-NEXT:    vmovups 32(%rdi), %ymm1
 ; AVX-NEXT:    retq
 ;
-; AVX512F-LABEL: test_unaligned_v64i8:
-; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovups (%rdi), %ymm0
-; AVX512F-NEXT:    vmovups 32(%rdi), %ymm1
-; AVX512F-NEXT:    retq
-;
-; AVX512BW-LABEL: test_unaligned_v64i8:
-; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vmovups (%rdi), %zmm0
-; AVX512BW-NEXT:    retq
-;
-; AVX512VL-LABEL: test_unaligned_v64i8:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vmovups (%rdi), %ymm0
-; AVX512VL-NEXT:    vmovups 32(%rdi), %ymm1
-; AVX512VL-NEXT:    retq
+; AVX512-LABEL: test_unaligned_v64i8:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovups (%rdi), %zmm0
+; AVX512-NEXT:    retq
   %1 = load <64 x i8>, <64 x i8>* %src, align 1, !nontemporal !1
   ret <64 x i8> %1
 }
@@ -1852,25 +1812,20 @@ define <16 x i32> @test_masked_v16i32(i8 * %addr, <16 x i32> %old, <16 x i32> %m
 ; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm4
 ; AVX1-NEXT:    vpxor %xmm5, %xmm5, %xmm5
 ; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm6, %xmm6, %xmm6
-; AVX1-NEXT:    vpxor %xmm6, %xmm4, %xmm4
 ; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm6, %xmm3, %xmm3
 ; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
 ; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm4
 ; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vpxor %xmm6, %xmm4, %xmm4
 ; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm2, %xmm2
-; AVX1-NEXT:    vpxor %xmm6, %xmm2, %xmm2
 ; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm2, %ymm2
 ; AVX1-NEXT:    vmovntdqa 32(%rdi), %xmm4
 ; AVX1-NEXT:    vmovntdqa 48(%rdi), %xmm5
 ; AVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm4, %ymm4
-; AVX1-NEXT:    vblendvps %ymm3, %ymm4, %ymm1, %ymm1
+; AVX1-NEXT:    vblendvps %ymm3, %ymm1, %ymm4, %ymm1
 ; AVX1-NEXT:    vmovntdqa (%rdi), %xmm3
 ; AVX1-NEXT:    vmovntdqa 16(%rdi), %xmm4
 ; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
-; AVX1-NEXT:    vblendvps %ymm2, %ymm3, %ymm0, %ymm0
+; AVX1-NEXT:    vblendvps %ymm2, %ymm0, %ymm3, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: test_masked_v16i32:
diff --git a/llvm/test/CodeGen/X86/noreturn-call-linux.ll b/llvm/test/CodeGen/X86/noreturn-call-linux.ll
new file mode 100644
index 000000000..da4694690
--- /dev/null
+++ b/llvm/test/CodeGen/X86/noreturn-call-linux.ll
@@ -0,0 +1,59 @@
+; RUN: llc < %s -mtriple=x86_64-linux-gnu | FileCheck %s
+
+; PR43155, we used to emit dead stack adjustments for noreturn calls with stack
+; arguments.
+
+; Original source code:
+; __attribute__((noreturn)) void exit_manyarg(int, int, int, int, int, int, int, int, int, int);
+; struct ByVal {
+;   int vals[10];
+; };
+; struct ByVal getbyval();
+; void make_push_unprofitable(struct ByVal);
+; int foo(int c) {
+;   if (c)
+;     exit_manyarg(1, 2, 3, 4, 5, 6, 7, 8, 9, 10);
+;   make_push_unprofitable(getbyval());
+;   make_push_unprofitable(getbyval());
+;   make_push_unprofitable(getbyval());
+;   return 0;
+; }
+
+%struct.ByVal = type { [10 x i32] }
+
+define dso_local i32 @foo(i32 %c) {
+entry:
+  %agg.tmp = alloca %struct.ByVal, align 8
+  %agg.tmp1 = alloca %struct.ByVal, align 8
+  %agg.tmp2 = alloca %struct.ByVal, align 8
+  %tobool = icmp eq i32 %c, 0
+  br i1 %tobool, label %if.end, label %if.then
+
+if.then:                                          ; preds = %entry
+  tail call void @exit_manyarg(i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10) #3
+  unreachable
+
+if.end:                                           ; preds = %entry
+  call void @getbyval(%struct.ByVal* nonnull sret %agg.tmp) #4
+  call void @make_push_unprofitable(%struct.ByVal* nonnull byval(%struct.ByVal) align 8 %agg.tmp) #4
+  call void @getbyval(%struct.ByVal* nonnull sret %agg.tmp1) #4
+  call void @make_push_unprofitable(%struct.ByVal* nonnull byval(%struct.ByVal) align 8 %agg.tmp1) #4
+  call void @getbyval(%struct.ByVal* nonnull sret %agg.tmp2) #4
+  call void @make_push_unprofitable(%struct.ByVal* nonnull byval(%struct.ByVal) align 8 %agg.tmp2) #4
+  ret i32 0
+}
+
+; CHECK-LABEL: foo:
+;   The main body is not important.
+; CHECK: callq exit_manyarg
+; CHECK-NOT: sub
+; CHECK-NOT: add
+; CHECK: # -- End function
+
+; Function Attrs: noreturn
+declare dso_local void @exit_manyarg(i32, i32, i32, i32, i32, i32, i32, i32, i32, i32) noreturn
+
+declare dso_local void @make_push_unprofitable(%struct.ByVal* byval(%struct.ByVal) align 8)
+
+declare dso_local void @getbyval(%struct.ByVal* sret)
+
diff --git a/llvm/test/CodeGen/X86/noreturn-call-win64.ll b/llvm/test/CodeGen/X86/noreturn-call-win64.ll
new file mode 100644
index 000000000..6289eef6b
--- /dev/null
+++ b/llvm/test/CodeGen/X86/noreturn-call-win64.ll
@@ -0,0 +1,53 @@
+; RUN: llc < %s -mtriple=x86_64-windows-msvc | FileCheck %s
+
+; Function Attrs: noinline nounwind optnone uwtable
+define dso_local i32 @foo() {
+entry:
+  %call = call i32 @cond()
+  %tobool = icmp ne i32 %call, 0
+  br i1 %tobool, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  call void @abort1()
+  unreachable
+
+if.end:                                           ; preds = %entry
+  %call1 = call i32 @cond()
+  %tobool2 = icmp ne i32 %call1, 0
+  br i1 %tobool2, label %if.then3, label %if.end4
+
+if.then3:                                         ; preds = %if.end
+  call void @abort2()
+  unreachable
+
+if.end4:                                          ; preds = %if.end
+  %call5 = call i32 @cond()
+  %tobool6 = icmp ne i32 %call5, 0
+  br i1 %tobool6, label %if.then7, label %if.end8
+
+if.then7:                                         ; preds = %if.end4
+  call void @abort3()
+  unreachable
+
+if.end8:                                          ; preds = %if.end4
+  ret i32 0
+}
+
+; CHECK-LABEL: foo:
+; CHECK: callq cond
+; CHECK: callq cond
+; CHECK: callq cond
+;   We don't need int3's between these calls to abort, since they won't confuse
+;   the unwinder.
+; CHECK: callq abort1
+; CHECK-NEXT:   # %if.then3
+; CHECK: callq abort2
+; CHECK-NEXT:   # %if.then7
+; CHECK: callq abort3
+; CHECK-NEXT: int3
+
+declare dso_local i32 @cond()
+
+declare dso_local void @abort1() noreturn
+declare dso_local void @abort2() noreturn
+declare dso_local void @abort3() noreturn
diff --git a/llvm/test/CodeGen/X86/noreturn-call.ll b/llvm/test/CodeGen/X86/noreturn-call.ll
index 89781816d..6f877b6d1 100644
--- a/llvm/test/CodeGen/X86/noreturn-call.ll
+++ b/llvm/test/CodeGen/X86/noreturn-call.ll
@@ -46,3 +46,59 @@ if.then:
 declare void @crash(i8*) noreturn
 declare void @crash2(i8*)
 declare void @g(i8*)
+
+%struct.ByVal = type { [10 x i32] }
+
+define dso_local i32 @pr43155() {
+entry:
+  %agg.tmp = alloca %struct.ByVal, align 4
+  %agg.tmp5 = alloca %struct.ByVal, align 4
+  %agg.tmp6 = alloca %struct.ByVal, align 4
+  %call = tail call i32 @cond()
+  %tobool = icmp eq i32 %call, 0
+  br i1 %tobool, label %if.end, label %if.then
+
+if.then:                                          ; preds = %entry
+  tail call x86_stdcallcc void @stdcall_abort(i32 12, i32 2)
+  unreachable
+
+if.end:                                           ; preds = %entry
+  %call1 = tail call i32 @cond()
+  %tobool2 = icmp eq i32 %call1, 0
+  br i1 %tobool2, label %if.end4, label %if.then3
+
+if.then3:                                         ; preds = %if.end
+  tail call x86_stdcallcc void @stdcall_abort(i32 15, i32 2)
+  unreachable
+
+if.end4:                                          ; preds = %if.end
+  call void @getbyval(%struct.ByVal* nonnull sret %agg.tmp)
+  call void @make_push_unprofitable(%struct.ByVal* nonnull byval(%struct.ByVal) align 4 %agg.tmp)
+  call void @getbyval(%struct.ByVal* nonnull sret %agg.tmp5)
+  call void @make_push_unprofitable(%struct.ByVal* nonnull byval(%struct.ByVal) align 4 %agg.tmp5)
+  call void @getbyval(%struct.ByVal* nonnull sret %agg.tmp6)
+  call void @make_push_unprofitable(%struct.ByVal* nonnull byval(%struct.ByVal) align 4 %agg.tmp6)
+  ret i32 0
+}
+
+;   Check that there are no stack adjustments after stdcall_abort.
+; CHECK-LABEL: pr43155:
+;   The main function body contents are not important.
+; CHECK: retl
+; CHECK:  # %if.then
+; CHECK: calll _stdcall_abort@8
+; CHECK-NOT: sub
+; CHECK-NOT: add
+; CHECK:  # %if.then3
+; CHECK: calll _stdcall_abort@8
+; CHECK-NOT: sub
+; CHECK-NOT: add
+; CHECK: # -- End function
+
+declare dso_local i32 @cond()
+
+declare dso_local x86_stdcallcc void @stdcall_abort(i32, i32) noreturn
+
+declare dso_local void @make_push_unprofitable(%struct.ByVal* byval(%struct.ByVal) align 4)
+
+declare dso_local void @getbyval(%struct.ByVal* sret)
diff --git a/llvm/test/CodeGen/X86/not-of-dec.ll b/llvm/test/CodeGen/X86/not-of-dec.ll
new file mode 100644
index 000000000..eb1eaa6e4
--- /dev/null
+++ b/llvm/test/CodeGen/X86/not-of-dec.ll
@@ -0,0 +1,117 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=i686-unknown-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,X86
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,X64
+
+; Fold
+;   ~(X - 1)
+; To
+;   - X
+;
+; This needs to be a backend-level fold because only by now pointers
+; are just registers; in middle-end IR this can only be done via @llvm.ptrmask()
+; intrinsic which is not sufficiently widely-spread yet.
+;
+; https://bugs.llvm.org/show_bug.cgi?id=44448
+
+; The basic positive tests
+
+define i32 @t0_32(i32 %alignment) nounwind {
+; X86-LABEL: t0_32:
+; X86:       # %bb.0:
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    subl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: t0_32:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    negl %eax
+; X64-NEXT:    retq
+  %mask = add i32 %alignment, -1
+  %invmask = xor i32 %mask, -1
+  ret i32 %invmask
+}
+define i64 @t1_64(i64 %alignment) nounwind {
+; X86-LABEL: t1_64:
+; X86:       # %bb.0:
+; X86-NEXT:    xorl %edx, %edx
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    subl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    sbbl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    retl
+;
+; X64-LABEL: t1_64:
+; X64:       # %bb.0:
+; X64-NEXT:    movq %rdi, %rax
+; X64-NEXT:    negq %rax
+; X64-NEXT:    retq
+  %mask = add i64 %alignment, -1
+  %invmask = xor i64 %mask, -1
+  ret i64 %invmask
+}
+
+; Extra use test
+
+define i32 @t2_extrause(i32 %alignment, i32* %mask_storage) nounwind {
+; X86-LABEL: t2_extrause:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    leal -1(%eax), %edx
+; X86-NEXT:    movl %edx, (%ecx)
+; X86-NEXT:    negl %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: t2_extrause:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    leal -1(%rax), %ecx
+; X64-NEXT:    movl %ecx, (%rsi)
+; X64-NEXT:    negl %eax
+; X64-NEXT:    # kill: def $eax killed $eax killed $rax
+; X64-NEXT:    retq
+  %mask = add i32 %alignment, -1
+  store i32 %mask, i32* %mask_storage
+  %invmask = xor i32 %mask, -1
+  ret i32 %invmask
+}
+
+; Negative tests
+
+define i32 @n3_not_dec(i32 %alignment) nounwind {
+; X86-LABEL: n3_not_dec:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    incl %eax
+; X86-NEXT:    notl %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: n3_not_dec:
+; X64:       # %bb.0:
+; X64-NEXT:    # kill: def $edi killed $edi def $rdi
+; X64-NEXT:    leal 1(%rdi), %eax
+; X64-NEXT:    notl %eax
+; X64-NEXT:    retq
+  %mask = add i32 %alignment, 1 ; not -1
+  %invmask = xor i32 %mask, -1
+  ret i32 %invmask
+}
+
+define i32 @n4_not_not(i32 %alignment) nounwind {
+; X86-LABEL: n4_not_not:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    decl %eax
+; X86-NEXT:    xorl $1, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: n4_not_not:
+; X64:       # %bb.0:
+; X64-NEXT:    # kill: def $edi killed $edi def $rdi
+; X64-NEXT:    leal -1(%rdi), %eax
+; X64-NEXT:    xorl $1, %eax
+; X64-NEXT:    retq
+  %mask = add i32 %alignment, -1
+  %invmask = xor i32 %mask, 1 ; not -1
+  ret i32 %invmask
+}
diff --git a/llvm/test/CodeGen/X86/oddshuffles.ll b/llvm/test/CodeGen/X86/oddshuffles.ll
index 2c36385e7..8460e8666 100644
--- a/llvm/test/CodeGen/X86/oddshuffles.ll
+++ b/llvm/test/CodeGen/X86/oddshuffles.ll
@@ -68,7 +68,7 @@ define void @v3f64(<2 x double> %a, <2 x double> %b, <3 x double>* %p) nounwind
 define void @v3i32(<2 x i32> %a, <2 x i32> %b, <3 x i32>* %p) nounwind {
 ; SSE2-LABEL: v3i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; SSE2-NEXT:    movd %xmm2, 8(%rdi)
 ; SSE2-NEXT:    movq %xmm0, (%rdi)
@@ -76,7 +76,7 @@ define void @v3i32(<2 x i32> %a, <2 x i32> %b, <3 x i32>* %p) nounwind {
 ;
 ; SSE42-LABEL: v3i32:
 ; SSE42:       # %bb.0:
-; SSE42-NEXT:    extractps $2, %xmm0, 8(%rdi)
+; SSE42-NEXT:    extractps $1, %xmm0, 8(%rdi)
 ; SSE42-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; SSE42-NEXT:    movlps %xmm0, (%rdi)
 ; SSE42-NEXT:    retq
@@ -84,14 +84,14 @@ define void @v3i32(<2 x i32> %a, <2 x i32> %b, <3 x i32>* %p) nounwind {
 ; AVX-LABEL: v3i32:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; AVX-NEXT:    vextractps $2, %xmm0, 8(%rdi)
+; AVX-NEXT:    vextractps $1, %xmm0, 8(%rdi)
 ; AVX-NEXT:    vmovlps %xmm1, (%rdi)
 ; AVX-NEXT:    retq
 ;
 ; XOP-LABEL: v3i32:
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; XOP-NEXT:    vextractps $2, %xmm0, 8(%rdi)
+; XOP-NEXT:    vextractps $1, %xmm0, 8(%rdi)
 ; XOP-NEXT:    vmovlps %xmm1, (%rdi)
 ; XOP-NEXT:    retq
   %r = shufflevector <2 x i32> %a, <2 x i32> %b, <3 x i32> <i32 0, i32 2, i32 1>
@@ -102,58 +102,34 @@ define void @v3i32(<2 x i32> %a, <2 x i32> %b, <3 x i32>* %p) nounwind {
 define void @v5i16(<4 x i16> %a, <4 x i16> %b, <5 x i16>* %p) nounwind {
 ; SSE2-LABEL: v5i16:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm0[0,2,2,3,4,5,6,7]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,1,2,3]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[2,0,2,3,4,5,6,7]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; SSE2-NEXT:    pextrw $6, %xmm0, %eax
+; SSE2-NEXT:    psrlq $16, %xmm1
+; SSE2-NEXT:    pextrw $3, %xmm0, %eax
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
 ; SSE2-NEXT:    movw %ax, 8(%rdi)
-; SSE2-NEXT:    movq %xmm2, (%rdi)
+; SSE2-NEXT:    movq %xmm0, (%rdi)
 ; SSE2-NEXT:    retq
 ;
 ; SSE42-LABEL: v5i16:
 ; SSE42:       # %bb.0:
-; SSE42-NEXT:    pshuflw {{.*#+}} xmm2 = xmm0[0,2,2,3,4,5,6,7]
-; SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,1,2,3]
-; SSE42-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[2,0,2,3,4,5,6,7]
-; SSE42-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; SSE42-NEXT:    pextrw $6, %xmm0, 8(%rdi)
-; SSE42-NEXT:    movq %xmm2, (%rdi)
+; SSE42-NEXT:    psrlq $16, %xmm1
+; SSE42-NEXT:    pextrw $3, %xmm0, 8(%rdi)
+; SSE42-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSE42-NEXT:    movq %xmm0, (%rdi)
 ; SSE42-NEXT:    retq
 ;
-; AVX1-LABEL: v5i16:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm0[0,2,2,3,4,5,6,7]
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,1,2,3]
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[2,0,2,3,4,5,6,7]
-; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX1-NEXT:    vpextrw $6, %xmm0, 8(%rdi)
-; AVX1-NEXT:    vmovq %xmm1, (%rdi)
-; AVX1-NEXT:    retq
-;
-; AVX2-SLOW-LABEL: v5i16:
-; AVX2-SLOW:       # %bb.0:
-; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm0[0,2,2,3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,1,2,3]
-; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[2,0,2,3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX2-SLOW-NEXT:    vpextrw $6, %xmm0, 8(%rdi)
-; AVX2-SLOW-NEXT:    vmovq %xmm1, (%rdi)
-; AVX2-SLOW-NEXT:    retq
-;
-; AVX2-FAST-LABEL: v5i16:
-; AVX2-FAST:       # %bb.0:
-; AVX2-FAST-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[4,5,8,9,4,5,6,7,8,9,10,11,12,13,14,15]
-; AVX2-FAST-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm0[0,2,2,3,4,5,6,7]
-; AVX2-FAST-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX2-FAST-NEXT:    vpextrw $6, %xmm0, 8(%rdi)
-; AVX2-FAST-NEXT:    vmovq %xmm1, (%rdi)
-; AVX2-FAST-NEXT:    retq
+; AVX-LABEL: v5i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpsrlq $16, %xmm1, %xmm1
+; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX-NEXT:    vpextrw $3, %xmm0, 8(%rdi)
+; AVX-NEXT:    vmovq %xmm1, (%rdi)
+; AVX-NEXT:    retq
 ;
 ; XOP-LABEL: v5i16:
 ; XOP:       # %bb.0:
-; XOP-NEXT:    vpperm {{.*#+}} xmm1 = xmm0[0,1],xmm1[4,5],xmm0[4,5],xmm1[8,9],xmm0[4,5],xmm1[4,5],xmm0[6,7],xmm1[6,7]
-; XOP-NEXT:    vpextrw $6, %xmm0, 8(%rdi)
+; XOP-NEXT:    vpsrlq $16, %xmm1, %xmm1
+; XOP-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; XOP-NEXT:    vpextrw $3, %xmm0, 8(%rdi)
 ; XOP-NEXT:    vmovq %xmm1, (%rdi)
 ; XOP-NEXT:    retq
   %r = shufflevector <4 x i16> %a, <4 x i16> %b, <5 x i32> <i32 0, i32 5, i32 1, i32 6, i32 3>
@@ -251,42 +227,35 @@ define void @v5f32(<4 x float> %a, <4 x float> %b, <5 x float>* %p) nounwind {
 define void @v7i8(<4 x i8> %a, <4 x i8> %b, <7 x i8>* %p) nounwind {
 ; SSE2-LABEL: v7i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,3,1,3]
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [0,65535,0,65535,0,65535,65535,65535]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,1,0,3]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,0,2,0,4,5,6,7]
-; SSE2-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,6,4,7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,3,1,3,4,5,6,7]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [0,255,0,255,0,255,255,255,255,255,255,255,255,255,255,255]
+; SSE2-NEXT:    movdqa %xmm1, -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[2,2,3,0,4,5,6,7]
 ; SSE2-NEXT:    pand %xmm2, %xmm1
 ; SSE2-NEXT:    pandn %xmm0, %xmm2
 ; SSE2-NEXT:    por %xmm1, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255]
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    packuswb %xmm0, %xmm0
-; SSE2-NEXT:    movdqa %xmm2, -{{[0-9]+}}(%rsp)
 ; SSE2-NEXT:    movb -{{[0-9]+}}(%rsp), %al
 ; SSE2-NEXT:    movb %al, 6(%rdi)
-; SSE2-NEXT:    movd %xmm0, (%rdi)
-; SSE2-NEXT:    pextrw $2, %xmm0, %eax
+; SSE2-NEXT:    movd %xmm2, (%rdi)
+; SSE2-NEXT:    pextrw $2, %xmm2, %eax
 ; SSE2-NEXT:    movw %ax, 4(%rdi)
 ; SSE2-NEXT:    retq
 ;
 ; SSE42-LABEL: v7i8:
 ; SSE42:       # %bb.0:
-; SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,3,1,3]
 ; SSE42-NEXT:    pextrb $0, %xmm1, 6(%rdi)
-; SSE42-NEXT:    pshufb {{.*#+}} xmm1 = xmm1[8,9,8,9,4,5,8,9,0,1,12,13,0,1,14,15]
-; SSE42-NEXT:    pblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5,6,7]
-; SSE42-NEXT:    pshufb {{.*#+}} xmm1 = xmm1[0,2,4,6,8,10,12,u,u,u,u,u,u,u,u,u]
+; SSE42-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE42-NEXT:    pshufb {{.*#+}} xmm1 = xmm1[1,4,7,4,3,6,0,u,u,u,u,u,u,u,u,u]
 ; SSE42-NEXT:    pextrw $2, %xmm1, 4(%rdi)
 ; SSE42-NEXT:    movd %xmm1, (%rdi)
 ; SSE42-NEXT:    retq
 ;
 ; AVX-LABEL: v7i8:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,3,1,3]
-; AVX-NEXT:    vpshufb {{.*#+}} xmm2 = xmm1[8,9,8,9,4,5,8,9,0,1,12,13,0,1,14,15]
-; AVX-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3],xmm0[4],xmm2[5,6,7]
-; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,u,u,u,u,u,u,u,u,u]
+; AVX-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[1,4,7,4,3,6,0,u,u,u,u,u,u,u,u,u]
 ; AVX-NEXT:    vpextrb $0, %xmm1, 6(%rdi)
 ; AVX-NEXT:    vpextrw $2, %xmm0, 4(%rdi)
 ; AVX-NEXT:    vmovd %xmm0, (%rdi)
@@ -294,7 +263,7 @@ define void @v7i8(<4 x i8> %a, <4 x i8> %b, <7 x i8>* %p) nounwind {
 ;
 ; XOP-LABEL: v7i8:
 ; XOP:       # %bb.0:
-; XOP-NEXT:    vpperm {{.*#+}} xmm0 = xmm0[0],xmm1[8],xmm0[12],xmm1[8],xmm0[4],xmm1[12,0,u,u,u,u,u,u,u,u,u]
+; XOP-NEXT:    vpperm {{.*#+}} xmm0 = xmm0[0],xmm1[2],xmm0[3],xmm1[2],xmm0[1],xmm1[3,0,u,u,u,u,u,u,u,u,u]
 ; XOP-NEXT:    vpextrb $0, %xmm1, 6(%rdi)
 ; XOP-NEXT:    vpextrw $2, %xmm0, 4(%rdi)
 ; XOP-NEXT:    vmovd %xmm0, (%rdi)
@@ -307,36 +276,32 @@ define void @v7i8(<4 x i8> %a, <4 x i8> %b, <7 x i8>* %p) nounwind {
 define void @v7i16(<4 x i16> %a, <4 x i16> %b, <7 x i16>* %p) nounwind {
 ; SSE2-LABEL: v7i16:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,3,1,3]
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [0,65535,0,65535,0,65535,65535,65535]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,1,0,3]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm3[0,0,2,0,4,5,6,7]
-; SSE2-NEXT:    pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,6,4,7]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pandn %xmm0, %xmm2
-; SSE2-NEXT:    por %xmm3, %xmm2
 ; SSE2-NEXT:    movd %xmm1, %eax
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm1[0,1,0,3,4,5,6,7]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,7,6,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[1,2,3,2,4,5,6,7]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm1 = xmm0[0,1,2,3,5,6,4,7]
 ; SSE2-NEXT:    movw %ax, 12(%rdi)
-; SSE2-NEXT:    movq %xmm2, (%rdi)
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE2-NEXT:    movq %xmm0, (%rdi)
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
 ; SSE2-NEXT:    movd %xmm0, 8(%rdi)
 ; SSE2-NEXT:    retq
 ;
 ; SSE42-LABEL: v7i16:
 ; SSE42:       # %bb.0:
-; SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,3,1,3]
 ; SSE42-NEXT:    pextrw $0, %xmm1, 12(%rdi)
-; SSE42-NEXT:    pshufb {{.*#+}} xmm1 = xmm1[8,9,8,9,4,5,8,9,0,1,12,13,0,1,14,15]
-; SSE42-NEXT:    pblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5,6,7]
+; SSE42-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; SSE42-NEXT:    pshufb {{.*#+}} xmm1 = xmm1[2,3,8,9,14,15,8,9,6,7,12,13,0,1,14,15]
 ; SSE42-NEXT:    pextrd $2, %xmm1, 8(%rdi)
 ; SSE42-NEXT:    movq %xmm1, (%rdi)
 ; SSE42-NEXT:    retq
 ;
 ; AVX-LABEL: v7i16:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,3,1,3]
-; AVX-NEXT:    vpshufb {{.*#+}} xmm2 = xmm1[8,9,8,9,4,5,8,9,0,1,12,13,0,1,14,15]
-; AVX-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3],xmm0[4],xmm2[5,6,7]
+; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[2,3,8,9,14,15,8,9,6,7,12,13,0,1,14,15]
 ; AVX-NEXT:    vpextrw $0, %xmm1, 12(%rdi)
 ; AVX-NEXT:    vpextrd $2, %xmm0, 8(%rdi)
 ; AVX-NEXT:    vmovq %xmm0, (%rdi)
@@ -344,7 +309,7 @@ define void @v7i16(<4 x i16> %a, <4 x i16> %b, <7 x i16>* %p) nounwind {
 ;
 ; XOP-LABEL: v7i16:
 ; XOP:       # %bb.0:
-; XOP-NEXT:    vpperm {{.*#+}} xmm0 = xmm0[0,1],xmm1[8,9],xmm0[12,13],xmm1[8,9],xmm0[4,5],xmm1[12,13,0,1,14,15]
+; XOP-NEXT:    vpperm {{.*#+}} xmm0 = xmm0[0,1],xmm1[4,5],xmm0[6,7],xmm1[4,5],xmm0[2,3],xmm1[6,7,0,1],xmm0[6,7]
 ; XOP-NEXT:    vpextrw $0, %xmm1, 12(%rdi)
 ; XOP-NEXT:    vpextrd $2, %xmm0, 8(%rdi)
 ; XOP-NEXT:    vmovq %xmm0, (%rdi)
@@ -410,19 +375,19 @@ define void @v7i32(<4 x i32> %a, <4 x i32> %b, <7 x i32>* %p) nounwind {
 define void @v12i8(<8 x i8> %a, <8 x i8> %b, <12 x i8>* %p) nounwind {
 ; SSE2-LABEL: v12i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,0,2,3,4,5,6,7]
-; SSE2-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,6,6,7]
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,0,255,255,0,255,255,0,255,255,0,255,255,255,255]
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[3,1,2,3]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm3[0,3,1,3,4,5,6,7]
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,1,2,3]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,3,1,3,4,5,6,7]
 ; SSE2-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,4,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,3,2,1]
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,1,4,5,6,7]
 ; SSE2-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,5,6,4]
-; SSE2-NEXT:    packuswb %xmm3, %xmm0
+; SSE2-NEXT:    packuswb %xmm2, %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,0,255,255,0,255,255,0,255,255,0,255,255,255,255]
 ; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,0,1,1,4,5,6,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,1,3]
 ; SSE2-NEXT:    pandn %xmm1, %xmm2
 ; SSE2-NEXT:    por %xmm0, %xmm2
 ; SSE2-NEXT:    movq %xmm2, (%rdi)
@@ -432,27 +397,23 @@ define void @v12i8(<8 x i8> %a, <8 x i8> %b, <12 x i8>* %p) nounwind {
 ;
 ; SSE42-LABEL: v12i8:
 ; SSE42:       # %bb.0:
-; SSE42-NEXT:    pshufb {{.*#+}} xmm1 = zero,zero,xmm1[0],zero,zero,xmm1[2],zero,zero,xmm1[4],zero,zero,xmm1[6,u,u,u,u]
-; SSE42-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,8],zero,xmm0[2,10],zero,xmm0[4,12],zero,xmm0[6,14],zero,xmm0[u,u,u,u]
-; SSE42-NEXT:    por %xmm1, %xmm0
+; SSE42-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSE42-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,8,1,2,10,3,4,12,5,6,14,7,u,u,u,u]
 ; SSE42-NEXT:    pextrd $2, %xmm0, 8(%rdi)
 ; SSE42-NEXT:    movq %xmm0, (%rdi)
 ; SSE42-NEXT:    retq
 ;
 ; AVX-LABEL: v12i8:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufb {{.*#+}} xmm1 = zero,zero,xmm1[0],zero,zero,xmm1[2],zero,zero,xmm1[4],zero,zero,xmm1[6,u,u,u,u]
-; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,8],zero,xmm0[2,10],zero,xmm0[4,12],zero,xmm0[6,14],zero,xmm0[u,u,u,u]
-; AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,8,1,2,10,3,4,12,5,6,14,7,u,u,u,u]
 ; AVX-NEXT:    vpextrd $2, %xmm0, 8(%rdi)
 ; AVX-NEXT:    vmovq %xmm0, (%rdi)
 ; AVX-NEXT:    retq
 ;
 ; XOP-LABEL: v12i8:
 ; XOP:       # %bb.0:
-; XOP-NEXT:    vpshufb {{.*#+}} xmm1 = zero,zero,xmm1[0],zero,zero,xmm1[2],zero,zero,xmm1[4],zero,zero,xmm1[6,u,u,u,u]
-; XOP-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,8],zero,xmm0[2,10],zero,xmm0[4,12],zero,xmm0[6,14],zero,xmm0[u,u,u,u]
-; XOP-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; XOP-NEXT:    vpperm {{.*#+}} xmm0 = xmm0[0,4],xmm1[0],xmm0[1,5],xmm1[1],xmm0[2,6],xmm1[2],xmm0[3,7],xmm1[3],xmm0[u,u,u,u]
 ; XOP-NEXT:    vpextrd $2, %xmm0, 8(%rdi)
 ; XOP-NEXT:    vmovq %xmm0, (%rdi)
 ; XOP-NEXT:    retq
@@ -659,11 +620,7 @@ define void @v12i32(<8 x i32> %a, <8 x i32> %b, <12 x i32>* %p) nounwind {
 define void @pr29025(<4 x i8> %a, <4 x i8> %b, <4 x i8> %c, <12 x i8> *%p) nounwind {
 ; SSE2-LABEL: pr29025:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [255,255,255,255]
-; SSE2-NEXT:    pand %xmm3, %xmm1
-; SSE2-NEXT:    pand %xmm3, %xmm0
-; SSE2-NEXT:    packuswb %xmm1, %xmm0
-; SSE2-NEXT:    packuswb %xmm0, %xmm0
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; SSE2-NEXT:    pxor %xmm1, %xmm1
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,1,2,3]
@@ -675,9 +632,6 @@ define void @pr29025(<4 x i8> %a, <4 x i8> %b, <4 x i8> %c, <12 x i8> *%p) nounw
 ; SSE2-NEXT:    packuswb %xmm1, %xmm0
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,0,255,255,0,255,255,0,255,255,0,255,255,255,255]
 ; SSE2-NEXT:    pand %xmm1, %xmm0
-; SSE2-NEXT:    pand %xmm3, %xmm2
-; SSE2-NEXT:    packuswb %xmm2, %xmm2
-; SSE2-NEXT:    packuswb %xmm2, %xmm2
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,0,1,1,4,5,6,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,0,1,3]
 ; SSE2-NEXT:    pandn %xmm2, %xmm1
@@ -689,11 +643,7 @@ define void @pr29025(<4 x i8> %a, <4 x i8> %b, <4 x i8> %c, <12 x i8> *%p) nounw
 ;
 ; SSE42-LABEL: pr29025:
 ; SSE42:       # %bb.0:
-; SSE42-NEXT:    movdqa {{.*#+}} xmm3 = <0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u>
-; SSE42-NEXT:    pshufb %xmm3, %xmm1
-; SSE42-NEXT:    pshufb %xmm3, %xmm0
 ; SSE42-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; SSE42-NEXT:    pshufb %xmm3, %xmm2
 ; SSE42-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
 ; SSE42-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,8,1,2,10,3,4,12,5,6,14,7,u,u,u,u]
 ; SSE42-NEXT:    pextrd $2, %xmm0, 8(%rdi)
@@ -702,12 +652,8 @@ define void @pr29025(<4 x i8> %a, <4 x i8> %b, <4 x i8> %c, <12 x i8> *%p) nounw
 ;
 ; AVX-LABEL: pr29025:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vmovdqa {{.*#+}} xmm3 = <0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
-; AVX-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
 ; AVX-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; AVX-NEXT:    vpshufb %xmm3, %xmm2, %xmm1
-; AVX-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; AVX-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
 ; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,8,1,2,10,3,4,12,5,6,14,7,u,u,u,u]
 ; AVX-NEXT:    vpextrd $2, %xmm0, 8(%rdi)
 ; AVX-NEXT:    vmovq %xmm0, (%rdi)
@@ -715,8 +661,8 @@ define void @pr29025(<4 x i8> %a, <4 x i8> %b, <4 x i8> %c, <12 x i8> *%p) nounw
 ;
 ; XOP-LABEL: pr29025:
 ; XOP:       # %bb.0:
-; XOP-NEXT:    vpperm {{.*#+}} xmm0 = xmm0[0,4,8,12],xmm1[0,4,8,12],xmm0[u,u,u,u,u,u,u,u]
-; XOP-NEXT:    vpperm {{.*#+}} xmm0 = xmm0[0,4],xmm2[0],xmm0[1,5],xmm2[4],xmm0[2,6],xmm2[8],xmm0[3,7],xmm2[12],xmm0[u,u,u,u]
+; XOP-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; XOP-NEXT:    vpperm {{.*#+}} xmm0 = xmm0[0,4],xmm2[0],xmm0[1,5],xmm2[1],xmm0[2,6],xmm2[2],xmm0[3,7],xmm2[3],xmm0[u,u,u,u]
 ; XOP-NEXT:    vpextrd $2, %xmm0, 8(%rdi)
 ; XOP-NEXT:    vmovq %xmm0, (%rdi)
 ; XOP-NEXT:    retq
@@ -751,39 +697,39 @@ define void @interleave_24i8_out(<24 x i8>* %p, <8 x i8>* %q1, <8 x i8>* %q2, <8
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm4 = xmm4[0,3,2,1,4,5,6,7]
 ; SSE2-NEXT:    pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,6,5,4,7]
 ; SSE2-NEXT:    packuswb %xmm0, %xmm4
-; SSE2-NEXT:    movq %xmm4, (%rsi)
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [0,255,255,0,255,255,0,255,255,255,255,255,255,255,255,255]
-; SSE2-NEXT:    movdqa %xmm0, %xmm5
-; SSE2-NEXT:    pand %xmm4, %xmm5
-; SSE2-NEXT:    pandn %xmm1, %xmm4
-; SSE2-NEXT:    por %xmm5, %xmm4
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3],xmm4[4],xmm2[4],xmm4[5],xmm2[5],xmm4[6],xmm2[6],xmm4[7],xmm2[7]
-; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [65535,65535,0,65535,65535,0,65535,65535]
-; SSE2-NEXT:    pand %xmm5, %xmm4
-; SSE2-NEXT:    pandn %xmm3, %xmm5
-; SSE2-NEXT:    por %xmm4, %xmm5
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm4 = xmm5[2,1,0,3,4,5,6,7]
-; SSE2-NEXT:    pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,6,5,4,7]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[0,3,2,1]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm4 = xmm4[1,2,3,0,4,5,6,7]
-; SSE2-NEXT:    pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,5,6,7,4]
-; SSE2-NEXT:    packuswb %xmm0, %xmm4
-; SSE2-NEXT:    movq %xmm4, (%rdx)
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [255,0,255,255,0,255,255,0,255,255,255,255,255,255,255,255]
-; SSE2-NEXT:    pand %xmm4, %xmm0
-; SSE2-NEXT:    pandn %xmm1, %xmm4
-; SSE2-NEXT:    por %xmm0, %xmm4
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3],xmm4[4],xmm2[4],xmm4[5],xmm2[5],xmm4[6],xmm2[6],xmm4[7],xmm2[7]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [0,255,255,0,255,255,0,255,255,255,255,255,255,255,255,255]
+; SSE2-NEXT:    movdqa %xmm0, %xmm6
+; SSE2-NEXT:    pand %xmm5, %xmm6
+; SSE2-NEXT:    pandn %xmm1, %xmm5
+; SSE2-NEXT:    por %xmm6, %xmm5
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3],xmm5[4],xmm2[4],xmm5[5],xmm2[5],xmm5[6],xmm2[6],xmm5[7],xmm2[7]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm6 = [65535,65535,0,65535,65535,0,65535,65535]
+; SSE2-NEXT:    pand %xmm6, %xmm5
+; SSE2-NEXT:    pandn %xmm3, %xmm6
+; SSE2-NEXT:    por %xmm5, %xmm6
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm5 = xmm6[2,1,0,3,4,5,6,7]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,6,5,4,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[0,3,2,1]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm5 = xmm5[1,2,3,0,4,5,6,7]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,5,6,7,4]
+; SSE2-NEXT:    packuswb %xmm0, %xmm5
+; SSE2-NEXT:    movdqa {{.*#+}} xmm6 = [255,0,255,255,0,255,255,0,255,255,255,255,255,255,255,255]
+; SSE2-NEXT:    pand %xmm6, %xmm0
+; SSE2-NEXT:    pandn %xmm1, %xmm6
+; SSE2-NEXT:    por %xmm0, %xmm6
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3],xmm6[4],xmm2[4],xmm6[5],xmm2[5],xmm6[6],xmm2[6],xmm6[7],xmm2[7]
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [0,65535,65535,0,65535,65535,0,65535]
-; SSE2-NEXT:    pand %xmm0, %xmm4
+; SSE2-NEXT:    pand %xmm0, %xmm6
 ; SSE2-NEXT:    pandn %xmm3, %xmm0
-; SSE2-NEXT:    por %xmm4, %xmm0
+; SSE2-NEXT:    por %xmm6, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,0]
 ; SSE2-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,7,6,5]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,0]
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[2,1,0,3,4,5,6,7]
 ; SSE2-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,4,7]
 ; SSE2-NEXT:    packuswb %xmm0, %xmm0
+; SSE2-NEXT:    movq %xmm4, (%rsi)
+; SSE2-NEXT:    movq %xmm5, (%rdx)
 ; SSE2-NEXT:    movq %xmm0, (%rcx)
 ; SSE2-NEXT:    retq
 ;
@@ -796,16 +742,16 @@ define void @interleave_24i8_out(<24 x i8>* %p, <8 x i8>* %q1, <8 x i8>* %q2, <8
 ; SSE42-NEXT:    movdqa %xmm0, %xmm3
 ; SSE42-NEXT:    pshufb {{.*#+}} xmm3 = xmm3[0,3,6,9,12,15],zero,zero,xmm3[u,u,u,u,u,u,u,u]
 ; SSE42-NEXT:    por %xmm2, %xmm3
-; SSE42-NEXT:    movq %xmm3, (%rsi)
 ; SSE42-NEXT:    movdqa %xmm1, %xmm2
 ; SSE42-NEXT:    pshufb {{.*#+}} xmm2 = zero,zero,zero,zero,zero,xmm2[0,3,6,u,u,u,u,u,u,u,u]
-; SSE42-NEXT:    movdqa %xmm0, %xmm3
-; SSE42-NEXT:    pshufb {{.*#+}} xmm3 = xmm3[1,4,7,10,13],zero,zero,zero,xmm3[u,u,u,u,u,u,u,u]
-; SSE42-NEXT:    por %xmm2, %xmm3
-; SSE42-NEXT:    movq %xmm3, (%rdx)
+; SSE42-NEXT:    movdqa %xmm0, %xmm4
+; SSE42-NEXT:    pshufb {{.*#+}} xmm4 = xmm4[1,4,7,10,13],zero,zero,zero,xmm4[u,u,u,u,u,u,u,u]
+; SSE42-NEXT:    por %xmm2, %xmm4
 ; SSE42-NEXT:    pshufb {{.*#+}} xmm1 = zero,zero,zero,zero,zero,xmm1[1,4,7,u,u,u,u,u,u,u,u]
 ; SSE42-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[2,5,8,11,14],zero,zero,zero,xmm0[u,u,u,u,u,u,u,u]
 ; SSE42-NEXT:    por %xmm1, %xmm0
+; SSE42-NEXT:    movq %xmm3, (%rsi)
+; SSE42-NEXT:    movq %xmm4, (%rdx)
 ; SSE42-NEXT:    movq %xmm0, (%rcx)
 ; SSE42-NEXT:    retq
 ;
@@ -816,14 +762,14 @@ define void @interleave_24i8_out(<24 x i8>* %p, <8 x i8>* %q1, <8 x i8>* %q2, <8
 ; AVX-NEXT:    vpshufb {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,xmm1[2,5,u,u,u,u,u,u,u,u]
 ; AVX-NEXT:    vpshufb {{.*#+}} xmm3 = xmm0[0,3,6,9,12,15],zero,zero,xmm0[u,u,u,u,u,u,u,u]
 ; AVX-NEXT:    vpor %xmm2, %xmm3, %xmm2
-; AVX-NEXT:    vmovq %xmm2, (%rsi)
-; AVX-NEXT:    vpshufb {{.*#+}} xmm2 = zero,zero,zero,zero,zero,xmm1[0,3,6,u,u,u,u,u,u,u,u]
-; AVX-NEXT:    vpshufb {{.*#+}} xmm3 = xmm0[1,4,7,10,13],zero,zero,zero,xmm0[u,u,u,u,u,u,u,u]
-; AVX-NEXT:    vpor %xmm2, %xmm3, %xmm2
-; AVX-NEXT:    vmovq %xmm2, (%rdx)
+; AVX-NEXT:    vpshufb {{.*#+}} xmm3 = zero,zero,zero,zero,zero,xmm1[0,3,6,u,u,u,u,u,u,u,u]
+; AVX-NEXT:    vpshufb {{.*#+}} xmm4 = xmm0[1,4,7,10,13],zero,zero,zero,xmm0[u,u,u,u,u,u,u,u]
+; AVX-NEXT:    vpor %xmm3, %xmm4, %xmm3
 ; AVX-NEXT:    vpshufb {{.*#+}} xmm1 = zero,zero,zero,zero,zero,xmm1[1,4,7,u,u,u,u,u,u,u,u]
 ; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[2,5,8,11,14],zero,zero,zero,xmm0[u,u,u,u,u,u,u,u]
 ; AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vmovq %xmm2, (%rsi)
+; AVX-NEXT:    vmovq %xmm3, (%rdx)
 ; AVX-NEXT:    vmovq %xmm0, (%rcx)
 ; AVX-NEXT:    retq
 ;
@@ -834,14 +780,14 @@ define void @interleave_24i8_out(<24 x i8>* %p, <8 x i8>* %q1, <8 x i8>* %q2, <8
 ; XOP-NEXT:    vpshufb {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,xmm1[2,5,u,u,u,u,u,u,u,u]
 ; XOP-NEXT:    vpshufb {{.*#+}} xmm3 = xmm0[0,3,6,9,12,15],zero,zero,xmm0[u,u,u,u,u,u,u,u]
 ; XOP-NEXT:    vpor %xmm2, %xmm3, %xmm2
-; XOP-NEXT:    vmovq %xmm2, (%rsi)
-; XOP-NEXT:    vpshufb {{.*#+}} xmm2 = zero,zero,zero,zero,zero,xmm1[0,3,6,u,u,u,u,u,u,u,u]
-; XOP-NEXT:    vpshufb {{.*#+}} xmm3 = xmm0[1,4,7,10,13],zero,zero,zero,xmm0[u,u,u,u,u,u,u,u]
-; XOP-NEXT:    vpor %xmm2, %xmm3, %xmm2
-; XOP-NEXT:    vmovq %xmm2, (%rdx)
+; XOP-NEXT:    vpshufb {{.*#+}} xmm3 = zero,zero,zero,zero,zero,xmm1[0,3,6,u,u,u,u,u,u,u,u]
+; XOP-NEXT:    vpshufb {{.*#+}} xmm4 = xmm0[1,4,7,10,13],zero,zero,zero,xmm0[u,u,u,u,u,u,u,u]
+; XOP-NEXT:    vpor %xmm3, %xmm4, %xmm3
 ; XOP-NEXT:    vpshufb {{.*#+}} xmm1 = zero,zero,zero,zero,zero,xmm1[1,4,7,u,u,u,u,u,u,u,u]
 ; XOP-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[2,5,8,11,14],zero,zero,zero,xmm0[u,u,u,u,u,u,u,u]
 ; XOP-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; XOP-NEXT:    vmovq %xmm2, (%rsi)
+; XOP-NEXT:    vmovq %xmm3, (%rdx)
 ; XOP-NEXT:    vmovq %xmm0, (%rcx)
 ; XOP-NEXT:    retq
   %wide.vec = load <24 x i8>, <24 x i8>* %p, align 4
@@ -1056,7 +1002,7 @@ define void @interleave_24i16_out(<24 x i16>* %p, <8 x i16>* %q1, <8 x i16>* %q2
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm4 = xmm5[0,1,2,3,4],xmm4[5,6,7]
 ; AVX1-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,6,7,8,9,2,3,8,9,14,15]
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2],xmm1[3,4],xmm0[5],xmm1[6,7]
-; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[4,5,10,11,0,1,6,7,12,13,14,15,0,1,2,3]
+; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[4,5,10,11,0,1,6,7,12,13,u,u,u,u,u,u]
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4],xmm2[5,6,7]
 ; AVX1-NEXT:    vmovdqu %xmm3, (%rsi)
 ; AVX1-NEXT:    vmovdqu %xmm4, (%rdx)
@@ -1094,8 +1040,8 @@ define void @interleave_24i16_out(<24 x i16>* %p, <8 x i16>* %q1, <8 x i16>* %q2
 ; XOP-NEXT:    vpperm {{.*#+}} xmm3 = xmm3[0,1,6,7,12,13,2,3,8,9,14,15],xmm2[4,5,10,11]
 ; XOP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm0[0,1],xmm1[2],xmm0[3,4],xmm1[5],xmm0[6,7]
 ; XOP-NEXT:    vpperm {{.*#+}} xmm4 = xmm4[2,3,8,9,14,15,4,5,10,11],xmm2[0,1,6,7,12,13]
-; XOP-NEXT:    vpperm {{.*#+}} xmm0 = xmm0[4,5,10,11],xmm1[0,1,6,7,12,13,14,15,0,1,2,3]
-; XOP-NEXT:    vpperm {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,6,7,8,9],xmm2[2,3,8,9,14,15]
+; XOP-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2],xmm1[3,4],xmm0[5],xmm1[6,7]
+; XOP-NEXT:    vpperm {{.*#+}} xmm0 = xmm0[4,5,10,11,0,1,6,7,12,13],xmm2[2,3,8,9,14,15]
 ; XOP-NEXT:    vmovdqu %xmm3, (%rsi)
 ; XOP-NEXT:    vmovdqu %xmm4, (%rdx)
 ; XOP-NEXT:    vmovdqu %xmm0, (%rcx)
@@ -1187,7 +1133,7 @@ define void @interleave_24i16_in(<24 x i16>* %p, <8 x i16>* %q1, <8 x i16>* %q2,
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm3[1,1,2,2]
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm4[0],xmm2[1,2],xmm4[3],xmm2[4,5],xmm4[6],xmm2[7]
 ; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; AVX1-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[4,5,10,11,10,11,8,9,8,9,14,15,12,13,14,15]
+; AVX1-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[4,5,u,u,10,11,8,9,u,u,14,15,12,13,u,u]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm3[2,2,3,3]
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0],xmm5[1],xmm4[2,3],xmm5[4],xmm4[5,6],xmm5[7]
 ; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
@@ -1233,9 +1179,8 @@ define void @interleave_24i16_in(<24 x i16>* %p, <8 x i16>* %q1, <8 x i16>* %q2,
 ; XOP-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
 ; XOP-NEXT:    vpperm {{.*#+}} xmm4 = xmm4[0,1,2,3],xmm2[0,1],xmm4[4,5,6,7],xmm2[2,3],xmm4[8,9,10,11]
 ; XOP-NEXT:    vinsertf128 $1, %xmm3, %ymm4, %ymm3
-; XOP-NEXT:    vpperm {{.*#+}} xmm0 = xmm1[10,11],xmm0[12,13,12,13],xmm1[12,13,12,13],xmm0[14,15],xmm1[14,15],xmm0[14,15]
-; XOP-NEXT:    vpshufd {{.*#+}} xmm1 = xmm2[2,2,3,3]
-; XOP-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3],xmm1[4],xmm0[5,6],xmm1[7]
+; XOP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; XOP-NEXT:    vpperm {{.*#+}} xmm0 = xmm0[4,5],xmm2[10,11],xmm0[10,11,8,9],xmm2[12,13],xmm0[14,15,12,13],xmm2[14,15]
 ; XOP-NEXT:    vmovdqu %xmm0, 32(%rdi)
 ; XOP-NEXT:    vmovups %ymm3, (%rdi)
 ; XOP-NEXT:    vzeroupper
@@ -1295,40 +1240,38 @@ define void @interleave_24i32_out(<24 x i32>* %p, <8 x i32>* %q1, <8 x i32>* %q2
 ;
 ; SSE42-LABEL: interleave_24i32_out:
 ; SSE42:       # %bb.0:
-; SSE42-NEXT:    movdqu 80(%rdi), %xmm9
-; SSE42-NEXT:    movdqu 64(%rdi), %xmm10
+; SSE42-NEXT:    movups 80(%rdi), %xmm8
+; SSE42-NEXT:    movdqu 64(%rdi), %xmm9
 ; SSE42-NEXT:    movdqu (%rdi), %xmm4
 ; SSE42-NEXT:    movdqu 16(%rdi), %xmm2
-; SSE42-NEXT:    movdqu 32(%rdi), %xmm11
+; SSE42-NEXT:    movups 32(%rdi), %xmm10
 ; SSE42-NEXT:    movdqu 48(%rdi), %xmm5
-; SSE42-NEXT:    pshufd {{.*#+}} xmm8 = xmm11[0,1,0,1]
-; SSE42-NEXT:    movdqa %xmm2, %xmm7
-; SSE42-NEXT:    pblendw {{.*#+}} xmm7 = xmm7[0,1],xmm4[2,3],xmm7[4,5,6,7]
-; SSE42-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[2,3,0,1]
+; SSE42-NEXT:    movdqa %xmm2, %xmm6
+; SSE42-NEXT:    pblendw {{.*#+}} xmm6 = xmm6[0,1],xmm4[2,3],xmm6[4,5,6,7]
+; SSE42-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[2,3,0,1]
 ; SSE42-NEXT:    shufps {{.*#+}} xmm4 = xmm4[0,3],xmm2[2,3]
-; SSE42-NEXT:    blendps {{.*#+}} xmm4 = xmm4[0,1,2],xmm8[3]
-; SSE42-NEXT:    movdqa %xmm10, %xmm1
+; SSE42-NEXT:    insertps {{.*#+}} xmm4 = xmm4[0,1,2],xmm10[1]
+; SSE42-NEXT:    movdqa %xmm9, %xmm1
 ; SSE42-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm5[2,3],xmm1[4,5,6,7]
 ; SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[2,3,0,1]
-; SSE42-NEXT:    shufps {{.*#+}} xmm5 = xmm5[0,3],xmm10[2,3]
-; SSE42-NEXT:    pshufd {{.*#+}} xmm3 = xmm9[0,1,0,1]
-; SSE42-NEXT:    pblendw {{.*#+}} xmm3 = xmm5[0,1,2,3,4,5],xmm3[6,7]
-; SSE42-NEXT:    pshufd {{.*#+}} xmm5 = xmm11[0,1,2,2]
-; SSE42-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,0,3,3]
-; SSE42-NEXT:    pblendw {{.*#+}} xmm7 = xmm7[0,1,2,3,4,5],xmm5[6,7]
+; SSE42-NEXT:    shufps {{.*#+}} xmm5 = xmm5[0,3],xmm9[2,3]
+; SSE42-NEXT:    insertps {{.*#+}} xmm5 = xmm5[0,1,2],xmm8[1]
+; SSE42-NEXT:    pshufd {{.*#+}} xmm3 = xmm10[0,1,2,2]
+; SSE42-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,0,3,3]
+; SSE42-NEXT:    pblendw {{.*#+}} xmm6 = xmm6[0,1,2,3,4,5],xmm3[6,7]
 ; SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,0,3,3]
-; SSE42-NEXT:    pshufd {{.*#+}} xmm5 = xmm9[0,1,2,2]
-; SSE42-NEXT:    pblendw {{.*#+}} xmm5 = xmm1[0,1,2,3,4,5],xmm5[6,7]
-; SSE42-NEXT:    pblendw {{.*#+}} xmm6 = xmm6[0,1],xmm2[2,3],xmm6[4,5,6,7]
-; SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm11[0,1,0,3]
-; SSE42-NEXT:    pblendw {{.*#+}} xmm1 = xmm6[0,1,2,3],xmm1[4,5,6,7]
-; SSE42-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm10[2,3],xmm0[4,5,6,7]
-; SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm9[0,1,0,3]
+; SSE42-NEXT:    pshufd {{.*#+}} xmm3 = xmm8[0,1,2,2]
+; SSE42-NEXT:    pblendw {{.*#+}} xmm3 = xmm1[0,1,2,3,4,5],xmm3[6,7]
+; SSE42-NEXT:    pblendw {{.*#+}} xmm7 = xmm7[0,1],xmm2[2,3],xmm7[4,5,6,7]
+; SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm10[0,1,0,3]
+; SSE42-NEXT:    pblendw {{.*#+}} xmm1 = xmm7[0,1,2,3],xmm1[4,5,6,7]
+; SSE42-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm9[2,3],xmm0[4,5,6,7]
+; SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm8[0,1,0,3]
 ; SSE42-NEXT:    pblendw {{.*#+}} xmm2 = xmm0[0,1,2,3],xmm2[4,5,6,7]
-; SSE42-NEXT:    movdqu %xmm3, 16(%rsi)
+; SSE42-NEXT:    movups %xmm5, 16(%rsi)
 ; SSE42-NEXT:    movups %xmm4, (%rsi)
-; SSE42-NEXT:    movdqu %xmm5, 16(%rdx)
-; SSE42-NEXT:    movdqu %xmm7, (%rdx)
+; SSE42-NEXT:    movdqu %xmm3, 16(%rdx)
+; SSE42-NEXT:    movdqu %xmm6, (%rdx)
 ; SSE42-NEXT:    movdqu %xmm2, 16(%rcx)
 ; SSE42-NEXT:    movdqu %xmm1, (%rcx)
 ; SSE42-NEXT:    retq
@@ -1570,34 +1513,34 @@ define void @interleave_24i32_in(<24 x i32>* %p, <8 x i32>* %q1, <8 x i32>* %q2,
 ; AVX1-LABEL: interleave_24i32_in:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vmovupd (%rsi), %ymm0
-; AVX1-NEXT:    vmovups 16(%rcx), %xmm1
-; AVX1-NEXT:    vmovups (%rdx), %xmm2
-; AVX1-NEXT:    vmovups 16(%rdx), %xmm3
-; AVX1-NEXT:    vshufps {{.*#+}} xmm4 = xmm3[3,0],xmm1[3,0]
-; AVX1-NEXT:    vshufps {{.*#+}} xmm4 = xmm1[2,1],xmm4[0,2]
-; AVX1-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[1,0],xmm3[1,0]
-; AVX1-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[2,0],xmm3[2,2]
-; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm1, %ymm1
-; AVX1-NEXT:    vpermilpd {{.*#+}} ymm3 = ymm0[1,1,3,3]
-; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm3 = ymm3[2,3,2,3]
-; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1],ymm3[2],ymm1[3,4],ymm3[5],ymm1[6,7]
+; AVX1-NEXT:    vmovups (%rdx), %xmm1
+; AVX1-NEXT:    vmovups 16(%rdx), %xmm2
 ; AVX1-NEXT:    vmovups (%rsi), %xmm3
-; AVX1-NEXT:    vshufps {{.*#+}} xmm4 = xmm3[2,0],xmm2[2,0]
-; AVX1-NEXT:    vshufps {{.*#+}} xmm4 = xmm2[1,1],xmm4[0,2]
-; AVX1-NEXT:    vshufps {{.*#+}} xmm2 = xmm2[0,0],xmm3[0,0]
-; AVX1-NEXT:    vshufps {{.*#+}} xmm2 = xmm2[2,0],xmm3[2,1]
-; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm2, %ymm2
+; AVX1-NEXT:    vshufps {{.*#+}} xmm4 = xmm3[2,0],xmm1[2,0]
+; AVX1-NEXT:    vshufps {{.*#+}} xmm4 = xmm1[1,1],xmm4[0,2]
+; AVX1-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[0,0],xmm3[0,0]
+; AVX1-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[2,0],xmm3[2,1]
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm1, %ymm1
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm3 = mem[0,1,0,1]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm3, %ymm3
+; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1],ymm3[2],ymm1[3,4],ymm3[5],ymm1[6,7]
+; AVX1-NEXT:    vmovups 16(%rcx), %xmm3
+; AVX1-NEXT:    vshufps {{.*#+}} xmm4 = xmm2[3,0],xmm3[3,0]
+; AVX1-NEXT:    vshufps {{.*#+}} xmm4 = xmm3[2,1],xmm4[0,2]
+; AVX1-NEXT:    vshufps {{.*#+}} xmm3 = xmm3[1,0],xmm2[1,0]
+; AVX1-NEXT:    vshufps {{.*#+}} xmm2 = xmm3[2,0],xmm2[2,2]
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm2, %ymm2
+; AVX1-NEXT:    vpermilpd {{.*#+}} ymm3 = ymm0[1,1,3,3]
+; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm3 = ymm3[2,3,2,3]
 ; AVX1-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1],ymm3[2],ymm2[3,4],ymm3[5],ymm2[6,7]
-; AVX1-NEXT:    vpermilpd {{.*#+}} ymm0 = ymm0[1,0,2,2]
 ; AVX1-NEXT:    vpermilpd {{.*#+}} ymm3 = mem[1,1,2,2]
+; AVX1-NEXT:    vpermilpd {{.*#+}} ymm0 = ymm0[1,0,2,2]
 ; AVX1-NEXT:    vblendps {{.*#+}} ymm0 = ymm3[0],ymm0[1],ymm3[2,3],ymm0[4],ymm3[5,6],ymm0[7]
 ; AVX1-NEXT:    vpermilps {{.*#+}} ymm3 = mem[0,0,3,3,4,4,7,7]
 ; AVX1-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm3[2],ymm0[3,4],ymm3[5],ymm0[6,7]
 ; AVX1-NEXT:    vmovups %ymm0, 32(%rdi)
-; AVX1-NEXT:    vmovups %ymm2, (%rdi)
-; AVX1-NEXT:    vmovups %ymm1, 64(%rdi)
+; AVX1-NEXT:    vmovups %ymm2, 64(%rdi)
+; AVX1-NEXT:    vmovups %ymm1, (%rdi)
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
@@ -1606,17 +1549,17 @@ define void @interleave_24i32_in(<24 x i32>* %p, <8 x i32>* %q1, <8 x i32>* %q2,
 ; AVX2-SLOW-NEXT:    vmovups (%rsi), %ymm0
 ; AVX2-SLOW-NEXT:    vmovups (%rdx), %ymm1
 ; AVX2-SLOW-NEXT:    vmovups (%rcx), %ymm2
-; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm3 = ymm2[2,1,3,3]
-; AVX2-SLOW-NEXT:    vpermilps {{.*#+}} ymm4 = ymm1[1,2,3,3,5,6,7,7]
-; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm4 = ymm4[2,2,2,3]
+; AVX2-SLOW-NEXT:    vpermilps {{.*#+}} xmm3 = mem[1,0,2,2]
+; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm3 = ymm3[0,1,0,1]
+; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm4 = ymm0[0,0,2,1]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm3 = ymm4[0],ymm3[1],ymm4[2,3],ymm3[4],ymm4[5,6],ymm3[7]
-; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm4 = ymm0[0,3,3,3]
+; AVX2-SLOW-NEXT:    vbroadcastsd (%rcx), %ymm4
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1],ymm4[2],ymm3[3,4],ymm4[5],ymm3[6,7]
-; AVX2-SLOW-NEXT:    vpermilps {{.*#+}} xmm4 = mem[1,0,2,2]
-; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm4 = ymm4[0,1,0,1]
-; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm5 = ymm0[0,0,2,1]
+; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm4 = ymm2[2,1,3,3]
+; AVX2-SLOW-NEXT:    vpermilps {{.*#+}} ymm5 = ymm1[1,2,3,3,5,6,7,7]
+; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm5 = ymm5[2,2,2,3]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm4 = ymm5[0],ymm4[1],ymm5[2,3],ymm4[4],ymm5[5,6],ymm4[7]
-; AVX2-SLOW-NEXT:    vbroadcastsd (%rcx), %ymm5
+; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm5 = ymm0[0,3,3,3]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1],ymm5[2],ymm4[3,4],ymm5[5],ymm4[6,7]
 ; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[1,1,2,2]
 ; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[1,1,2,2]
@@ -1624,8 +1567,8 @@ define void @interleave_24i32_in(<24 x i32>* %p, <8 x i32>* %q1, <8 x i32>* %q2,
 ; AVX2-SLOW-NEXT:    vpermilps {{.*#+}} ymm1 = ymm1[0,0,3,3,4,4,7,7]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]
 ; AVX2-SLOW-NEXT:    vmovups %ymm0, 32(%rdi)
-; AVX2-SLOW-NEXT:    vmovups %ymm4, (%rdi)
-; AVX2-SLOW-NEXT:    vmovups %ymm3, 64(%rdi)
+; AVX2-SLOW-NEXT:    vmovups %ymm4, 64(%rdi)
+; AVX2-SLOW-NEXT:    vmovups %ymm3, (%rdi)
 ; AVX2-SLOW-NEXT:    vzeroupper
 ; AVX2-SLOW-NEXT:    retq
 ;
@@ -1634,27 +1577,27 @@ define void @interleave_24i32_in(<24 x i32>* %p, <8 x i32>* %q1, <8 x i32>* %q2,
 ; AVX2-FAST-NEXT:    vmovups (%rsi), %ymm0
 ; AVX2-FAST-NEXT:    vmovups (%rdx), %ymm1
 ; AVX2-FAST-NEXT:    vmovups (%rcx), %ymm2
-; AVX2-FAST-NEXT:    vmovaps {{.*#+}} ymm3 = [5,6,5,6,5,6,7,7]
+; AVX2-FAST-NEXT:    vbroadcastf128 {{.*#+}} ymm3 = [1,0,2,2,1,0,2,2]
+; AVX2-FAST-NEXT:    # ymm3 = mem[0,1,0,1]
 ; AVX2-FAST-NEXT:    vpermps %ymm1, %ymm3, %ymm3
-; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm4 = ymm2[2,1,3,3]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0],ymm4[1],ymm3[2,3],ymm4[4],ymm3[5,6],ymm4[7]
-; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm4 = ymm0[0,3,3,3]
+; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm4 = ymm0[0,0,2,1]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm3 = ymm4[0],ymm3[1],ymm4[2,3],ymm3[4],ymm4[5,6],ymm3[7]
+; AVX2-FAST-NEXT:    vbroadcastsd (%rcx), %ymm4
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1],ymm4[2],ymm3[3,4],ymm4[5],ymm3[6,7]
-; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm4 = ymm0[1,1,2,2]
+; AVX2-FAST-NEXT:    vmovaps {{.*#+}} ymm4 = [5,6,5,6,5,6,7,7]
+; AVX2-FAST-NEXT:    vpermps %ymm1, %ymm4, %ymm4
+; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm5 = ymm2[2,1,3,3]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0],ymm5[1],ymm4[2,3],ymm5[4],ymm4[5,6],ymm5[7]
+; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm5 = ymm0[0,3,3,3]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1],ymm5[2],ymm4[3,4],ymm5[5],ymm4[6,7]
+; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[1,1,2,2]
 ; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[1,1,2,2]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0],ymm4[1],ymm2[2,3],ymm4[4],ymm2[5,6],ymm4[7]
-; AVX2-FAST-NEXT:    vpermilps {{.*#+}} ymm4 = ymm1[0,0,3,3,4,4,7,7]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1],ymm4[2],ymm2[3,4],ymm4[5],ymm2[6,7]
-; AVX2-FAST-NEXT:    vbroadcastf128 {{.*#+}} ymm4 = [1,0,2,2,1,0,2,2]
-; AVX2-FAST-NEXT:    # ymm4 = mem[0,1,0,1]
-; AVX2-FAST-NEXT:    vpermps %ymm1, %ymm4, %ymm1
-; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,0,2,1]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5,6],ymm1[7]
-; AVX2-FAST-NEXT:    vbroadcastsd (%rcx), %ymm1
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm2[0],ymm0[1],ymm2[2,3],ymm0[4],ymm2[5,6],ymm0[7]
+; AVX2-FAST-NEXT:    vpermilps {{.*#+}} ymm1 = ymm1[0,0,3,3,4,4,7,7]
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]
-; AVX2-FAST-NEXT:    vmovups %ymm0, (%rdi)
-; AVX2-FAST-NEXT:    vmovups %ymm2, 32(%rdi)
-; AVX2-FAST-NEXT:    vmovups %ymm3, 64(%rdi)
+; AVX2-FAST-NEXT:    vmovups %ymm0, 32(%rdi)
+; AVX2-FAST-NEXT:    vmovups %ymm4, 64(%rdi)
+; AVX2-FAST-NEXT:    vmovups %ymm3, (%rdi)
 ; AVX2-FAST-NEXT:    vzeroupper
 ; AVX2-FAST-NEXT:    retq
 ;
@@ -1662,32 +1605,32 @@ define void @interleave_24i32_in(<24 x i32>* %p, <8 x i32>* %q1, <8 x i32>* %q2,
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vmovupd (%rsi), %ymm0
 ; XOP-NEXT:    vmovups (%rcx), %ymm1
-; XOP-NEXT:    vmovups 16(%rcx), %xmm2
-; XOP-NEXT:    vmovups (%rdx), %xmm3
-; XOP-NEXT:    vmovups 16(%rdx), %xmm4
-; XOP-NEXT:    vshufps {{.*#+}} xmm5 = xmm4[3,0],xmm2[3,0]
-; XOP-NEXT:    vshufps {{.*#+}} xmm5 = xmm2[2,1],xmm5[0,2]
-; XOP-NEXT:    vshufps {{.*#+}} xmm2 = xmm2[1,0],xmm4[1,0]
-; XOP-NEXT:    vshufps {{.*#+}} xmm2 = xmm2[2,0],xmm4[2,2]
-; XOP-NEXT:    vinsertf128 $1, %xmm5, %ymm2, %ymm2
-; XOP-NEXT:    vpermilpd {{.*#+}} ymm4 = ymm0[1,1,3,3]
-; XOP-NEXT:    vperm2f128 {{.*#+}} ymm4 = ymm4[2,3,2,3]
-; XOP-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1],ymm4[2],ymm2[3,4],ymm4[5],ymm2[6,7]
+; XOP-NEXT:    vmovups (%rdx), %xmm2
+; XOP-NEXT:    vmovups 16(%rdx), %xmm3
 ; XOP-NEXT:    vmovups (%rsi), %xmm4
-; XOP-NEXT:    vshufps {{.*#+}} xmm5 = xmm4[2,0],xmm3[2,0]
-; XOP-NEXT:    vshufps {{.*#+}} xmm5 = xmm3[1,1],xmm5[0,2]
-; XOP-NEXT:    vshufps {{.*#+}} xmm3 = xmm3[0,0],xmm4[0,0]
-; XOP-NEXT:    vshufps {{.*#+}} xmm3 = xmm3[2,0],xmm4[2,1]
-; XOP-NEXT:    vinsertf128 $1, %xmm5, %ymm3, %ymm3
+; XOP-NEXT:    vshufps {{.*#+}} xmm5 = xmm4[2,0],xmm2[2,0]
+; XOP-NEXT:    vshufps {{.*#+}} xmm5 = xmm2[1,1],xmm5[0,2]
+; XOP-NEXT:    vshufps {{.*#+}} xmm2 = xmm2[0,0],xmm4[0,0]
+; XOP-NEXT:    vshufps {{.*#+}} xmm2 = xmm2[2,0],xmm4[2,1]
+; XOP-NEXT:    vinsertf128 $1, %xmm5, %ymm2, %ymm2
 ; XOP-NEXT:    vpermilps {{.*#+}} xmm4 = mem[0,1,0,1]
 ; XOP-NEXT:    vinsertf128 $1, %xmm4, %ymm4, %ymm4
+; XOP-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1],ymm4[2],ymm2[3,4],ymm4[5],ymm2[6,7]
+; XOP-NEXT:    vmovups 16(%rcx), %xmm4
+; XOP-NEXT:    vshufps {{.*#+}} xmm5 = xmm3[3,0],xmm4[3,0]
+; XOP-NEXT:    vshufps {{.*#+}} xmm5 = xmm4[2,1],xmm5[0,2]
+; XOP-NEXT:    vshufps {{.*#+}} xmm4 = xmm4[1,0],xmm3[1,0]
+; XOP-NEXT:    vshufps {{.*#+}} xmm3 = xmm4[2,0],xmm3[2,2]
+; XOP-NEXT:    vinsertf128 $1, %xmm5, %ymm3, %ymm3
+; XOP-NEXT:    vpermilpd {{.*#+}} ymm4 = ymm0[1,1,3,3]
+; XOP-NEXT:    vperm2f128 {{.*#+}} ymm4 = ymm4[2,3,2,3]
 ; XOP-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1],ymm4[2],ymm3[3,4],ymm4[5],ymm3[6,7]
 ; XOP-NEXT:    vpermil2ps {{.*#+}} ymm0 = ymm1[2],ymm0[3],ymm1[2,3],ymm0[4],ymm1[5,4],ymm0[5]
 ; XOP-NEXT:    vpermilps {{.*#+}} ymm1 = mem[0,0,3,3,4,4,7,7]
 ; XOP-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]
 ; XOP-NEXT:    vmovups %ymm0, 32(%rdi)
-; XOP-NEXT:    vmovups %ymm3, (%rdi)
-; XOP-NEXT:    vmovups %ymm2, 64(%rdi)
+; XOP-NEXT:    vmovups %ymm3, 64(%rdi)
+; XOP-NEXT:    vmovups %ymm2, (%rdi)
 ; XOP-NEXT:    vzeroupper
 ; XOP-NEXT:    retq
   %s1 = load <8 x i32>, <8 x i32>* %q1, align 4
diff --git a/llvm/test/CodeGen/X86/oddsubvector.ll b/llvm/test/CodeGen/X86/oddsubvector.ll
index 2db39b1b9..e4cc9b847 100644
--- a/llvm/test/CodeGen/X86/oddsubvector.ll
+++ b/llvm/test/CodeGen/X86/oddsubvector.ll
@@ -12,19 +12,12 @@ define void @insert_v7i8_v2i16_2(<7 x i8> *%a0, <2 x i16> *%a1) nounwind {
 ; SSE2-LABEL: insert_v7i8_v2i16_2:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm1[0,3]
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[2,0,1,3]
-; SSE2-NEXT:    movaps {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
-; SSE2-NEXT:    andps %xmm0, %xmm1
-; SSE2-NEXT:    packuswb %xmm1, %xmm1
-; SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movb -{{[0-9]+}}(%rsp), %al
-; SSE2-NEXT:    movb %al, 6(%rdi)
+; SSE2-NEXT:    pextrw $3, %xmm1, %eax
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
 ; SSE2-NEXT:    movd %xmm1, (%rdi)
-; SSE2-NEXT:    pextrw $2, %xmm1, %eax
+; SSE2-NEXT:    movb %al, 6(%rdi)
+; SSE2-NEXT:    pextrw $1, %xmm0, %eax
 ; SSE2-NEXT:    movw %ax, 4(%rdi)
 ; SSE2-NEXT:    retq
 ;
@@ -32,52 +25,40 @@ define void @insert_v7i8_v2i16_2(<7 x i8> *%a0, <2 x i16> *%a1) nounwind {
 ; SSE42:       # %bb.0:
 ; SSE42-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; SSE42-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
-; SSE42-NEXT:    pmovzxbw {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
-; SSE42-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[3],zero,zero,zero
-; SSE42-NEXT:    pblendw {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3,4,5],xmm2[6,7]
-; SSE42-NEXT:    packuswb %xmm0, %xmm0
 ; SSE42-NEXT:    pextrb $6, %xmm1, 6(%rdi)
-; SSE42-NEXT:    pextrw $2, %xmm0, 4(%rdi)
-; SSE42-NEXT:    movd %xmm0, (%rdi)
+; SSE42-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; SSE42-NEXT:    pextrw $1, %xmm0, 4(%rdi)
+; SSE42-NEXT:    movd %xmm1, (%rdi)
 ; SSE42-NEXT:    retq
 ;
 ; AVX1-LABEL: insert_v7i8_v2i16_2:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; AVX1-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
-; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[3],zero,zero,zero
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3,4,5],xmm2[6,7]
-; AVX1-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
+; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
 ; AVX1-NEXT:    vpextrb $6, %xmm1, 6(%rdi)
-; AVX1-NEXT:    vpextrw $2, %xmm0, 4(%rdi)
-; AVX1-NEXT:    vmovd %xmm0, (%rdi)
+; AVX1-NEXT:    vpextrw $1, %xmm0, 4(%rdi)
+; AVX1-NEXT:    vmovd %xmm2, (%rdi)
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: insert_v7i8_v2i16_2:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; AVX2-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
-; AVX2-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
-; AVX2-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[3],zero,zero,zero
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm2[0],xmm0[1,2],xmm2[3]
-; AVX2-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
+; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
 ; AVX2-NEXT:    vpextrb $6, %xmm1, 6(%rdi)
-; AVX2-NEXT:    vpextrw $2, %xmm0, 4(%rdi)
-; AVX2-NEXT:    vmovd %xmm0, (%rdi)
+; AVX2-NEXT:    vpextrw $1, %xmm0, 4(%rdi)
+; AVX2-NEXT:    vmovd %xmm2, (%rdi)
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: insert_v7i8_v2i16_2:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; AVX512-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
-; AVX512-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
-; AVX512-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[3],zero,zero,zero
-; AVX512-NEXT:    vpblendd {{.*#+}} xmm0 = xmm2[0],xmm0[1,2],xmm2[3]
-; AVX512-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
 ; AVX512-NEXT:    vpextrb $6, %xmm1, 6(%rdi)
-; AVX512-NEXT:    vpextrw $2, %xmm0, 4(%rdi)
-; AVX512-NEXT:    vmovd %xmm0, (%rdi)
+; AVX512-NEXT:    vpextrw $1, %xmm0, 4(%rdi)
+; AVX512-NEXT:    vmovd %xmm2, (%rdi)
 ; AVX512-NEXT:    retq
 ;
 ; XOP-LABEL: insert_v7i8_v2i16_2:
diff --git a/llvm/test/CodeGen/X86/offset-operator.ll b/llvm/test/CodeGen/X86/offset-operator.ll
new file mode 100644
index 000000000..126e0c429
--- /dev/null
+++ b/llvm/test/CodeGen/X86/offset-operator.ll
@@ -0,0 +1,15 @@
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu -x86-asm-syntax=intel -relocation-model=static < %s | FileCheck %s
+
+; Test we are emitting the 'offset' operator upon an immediate reference of a label:
+; The emitted 'att-equivalent' of this one is "movl $.L.str, %eax"
+
+@.str = private unnamed_addr constant [1 x i8] zeroinitializer, align 1
+
+define i8* @test_offset_operator() {
+; CHECK-LABEL: test_offset_operator:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    mov eax, offset .L.str
+; CHECK-NEXT:    ret
+entry:
+  ret i8* getelementptr inbounds ([1 x i8], [1 x i8]* @.str, i64 0, i64 0)
+}
diff --git a/llvm/test/CodeGen/X86/optimize-max-0.ll b/llvm/test/CodeGen/X86/optimize-max-0.ll
index b5e8627a8..5fb2371cb 100644
--- a/llvm/test/CodeGen/X86/optimize-max-0.ll
+++ b/llvm/test/CodeGen/X86/optimize-max-0.ll
@@ -1,4 +1,5 @@
-; RUN: llc < %s | not grep cmov
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s | FileCheck %s
 
 ; LSR should be able to eliminate the max computations by
 ; making the loops use slt/ult comparisons instead of ne comparisons.
@@ -7,6 +8,219 @@ target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f3
 target triple = "i386-apple-darwin9"
 
 define void @foo(i8* %r, i32 %s, i32 %w, i32 %x, i8* %j, i32 %d) nounwind {
+; CHECK-LABEL: foo:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    pushl %ebp
+; CHECK-NEXT:    pushl %ebx
+; CHECK-NEXT:    pushl %edi
+; CHECK-NEXT:    pushl %esi
+; CHECK-NEXT:    subl $28, %esp
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; CHECK-NEXT:    movl %ebx, %eax
+; CHECK-NEXT:    imull %edi, %eax
+; CHECK-NEXT:    cmpl $1, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) ## 4-byte Spill
+; CHECK-NEXT:    je LBB0_19
+; CHECK-NEXT:  ## %bb.1: ## %bb10.preheader
+; CHECK-NEXT:    movl %eax, %ebp
+; CHECK-NEXT:    sarl $31, %ebp
+; CHECK-NEXT:    shrl $30, %ebp
+; CHECK-NEXT:    addl %eax, %ebp
+; CHECK-NEXT:    sarl $2, %ebp
+; CHECK-NEXT:    testl %ebx, %ebx
+; CHECK-NEXT:    jle LBB0_12
+; CHECK-NEXT:  ## %bb.2: ## %bb.nph9
+; CHECK-NEXT:    testl %edi, %edi
+; CHECK-NEXT:    jle LBB0_12
+; CHECK-NEXT:  ## %bb.3: ## %bb.nph9.split
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    incl %eax
+; CHECK-NEXT:    xorl %ecx, %ecx
+; CHECK-NEXT:    xorl %esi, %esi
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  LBB0_4: ## %bb6
+; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movzbl (%eax,%esi,2), %ebx
+; CHECK-NEXT:    movb %bl, (%edx,%esi)
+; CHECK-NEXT:    incl %esi
+; CHECK-NEXT:    cmpl %edi, %esi
+; CHECK-NEXT:    jl LBB0_4
+; CHECK-NEXT:  ## %bb.5: ## %bb9
+; CHECK-NEXT:    ## in Loop: Header=BB0_4 Depth=1
+; CHECK-NEXT:    incl %ecx
+; CHECK-NEXT:    addl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    addl %edi, %edx
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; CHECK-NEXT:    cmpl %ebx, %ecx
+; CHECK-NEXT:    je LBB0_12
+; CHECK-NEXT:  ## %bb.6: ## %bb7.preheader
+; CHECK-NEXT:    ## in Loop: Header=BB0_4 Depth=1
+; CHECK-NEXT:    xorl %esi, %esi
+; CHECK-NEXT:    jmp LBB0_4
+; CHECK-NEXT:  LBB0_12: ## %bb18.loopexit
+; CHECK-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) ## 4-byte Spill
+; CHECK-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax ## 4-byte Reload
+; CHECK-NEXT:    addl %ebp, %eax
+; CHECK-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) ## 4-byte Spill
+; CHECK-NEXT:    cmpl $1, %ebx
+; CHECK-NEXT:    jle LBB0_13
+; CHECK-NEXT:  ## %bb.7: ## %bb.nph5
+; CHECK-NEXT:    cmpl $2, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    jl LBB0_13
+; CHECK-NEXT:  ## %bb.8: ## %bb.nph5.split
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    movl %eax, %edx
+; CHECK-NEXT:    shrl $31, %edx
+; CHECK-NEXT:    addl %eax, %edx
+; CHECK-NEXT:    sarl %edx
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    movl %eax, %ecx
+; CHECK-NEXT:    shrl $31, %ecx
+; CHECK-NEXT:    addl %eax, %ecx
+; CHECK-NEXT:    sarl %ecx
+; CHECK-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) ## 4-byte Spill
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax ## 4-byte Reload
+; CHECK-NEXT:    addl %ecx, %eax
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; CHECK-NEXT:    leal 2(%esi), %esi
+; CHECK-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) ## 4-byte Spill
+; CHECK-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi ## 4-byte Reload
+; CHECK-NEXT:    addl %esi, %ecx
+; CHECK-NEXT:    xorl %ebx, %ebx
+; CHECK-NEXT:    xorl %ebp, %ebp
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  LBB0_9: ## %bb13
+; CHECK-NEXT:    ## =>This Loop Header: Depth=1
+; CHECK-NEXT:    ## Child Loop BB0_10 Depth 2
+; CHECK-NEXT:    movl %ebp, %esi
+; CHECK-NEXT:    shrl $31, %esi
+; CHECK-NEXT:    addl %ebp, %esi
+; CHECK-NEXT:    andl $-2, %esi
+; CHECK-NEXT:    movl %ebp, %edi
+; CHECK-NEXT:    subl %esi, %edi
+; CHECK-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) ## 4-byte Spill
+; CHECK-NEXT:    addl %ebx, %edi
+; CHECK-NEXT:    imull {{[0-9]+}}(%esp), %edi
+; CHECK-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi ## 4-byte Folded Reload
+; CHECK-NEXT:    xorl %esi, %esi
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  LBB0_10: ## %bb14
+; CHECK-NEXT:    ## Parent Loop BB0_9 Depth=1
+; CHECK-NEXT:    ## => This Inner Loop Header: Depth=2
+; CHECK-NEXT:    movzbl -2(%edi,%esi,4), %ebx
+; CHECK-NEXT:    movb %bl, (%ecx,%esi)
+; CHECK-NEXT:    movzbl (%edi,%esi,4), %ebx
+; CHECK-NEXT:    movb %bl, (%eax,%esi)
+; CHECK-NEXT:    incl %esi
+; CHECK-NEXT:    cmpl %edx, %esi
+; CHECK-NEXT:    jl LBB0_10
+; CHECK-NEXT:  ## %bb.11: ## %bb17
+; CHECK-NEXT:    ## in Loop: Header=BB0_9 Depth=1
+; CHECK-NEXT:    incl %ebp
+; CHECK-NEXT:    addl %edx, %eax
+; CHECK-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx ## 4-byte Reload
+; CHECK-NEXT:    addl $2, %ebx
+; CHECK-NEXT:    addl %edx, %ecx
+; CHECK-NEXT:    cmpl {{[-0-9]+}}(%e{{[sb]}}p), %ebp ## 4-byte Folded Reload
+; CHECK-NEXT:    jl LBB0_9
+; CHECK-NEXT:  LBB0_13: ## %bb20
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; CHECK-NEXT:    cmpl $1, %edx
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; CHECK-NEXT:    je LBB0_19
+; CHECK-NEXT:  ## %bb.14: ## %bb20
+; CHECK-NEXT:    cmpl $3, %edx
+; CHECK-NEXT:    jne LBB0_24
+; CHECK-NEXT:  ## %bb.15: ## %bb22
+; CHECK-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi ## 4-byte Reload
+; CHECK-NEXT:    addl %esi, {{[-0-9]+}}(%e{{[sb]}}p) ## 4-byte Folded Spill
+; CHECK-NEXT:    testl %ebx, %ebx
+; CHECK-NEXT:    jle LBB0_18
+; CHECK-NEXT:  ## %bb.16: ## %bb.nph
+; CHECK-NEXT:    leal 15(%ebx), %eax
+; CHECK-NEXT:    andl $-16, %eax
+; CHECK-NEXT:    imull {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    addl %eax, %ebp
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    addl $15, %eax
+; CHECK-NEXT:    andl $-16, %eax
+; CHECK-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) ## 4-byte Spill
+; CHECK-NEXT:    addl %esi, %esi
+; CHECK-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %esi ## 4-byte Folded Reload
+; CHECK-NEXT:    addl {{[0-9]+}}(%esp), %esi
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  LBB0_17: ## %bb23
+; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    subl $4, %esp
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; CHECK-NEXT:    pushl %edi
+; CHECK-NEXT:    pushl %ebp
+; CHECK-NEXT:    pushl %esi
+; CHECK-NEXT:    calll _memcpy
+; CHECK-NEXT:    addl $16, %esp
+; CHECK-NEXT:    addl %edi, %esi
+; CHECK-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp ## 4-byte Folded Reload
+; CHECK-NEXT:    decl %ebx
+; CHECK-NEXT:    jne LBB0_17
+; CHECK-NEXT:  LBB0_18: ## %bb26
+; CHECK-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx ## 4-byte Reload
+; CHECK-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi ## 4-byte Reload
+; CHECK-NEXT:    addl %ecx, %esi
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; CHECK-NEXT:    addl %esi, %edx
+; CHECK-NEXT:    jmp LBB0_23
+; CHECK-NEXT:  LBB0_19: ## %bb29
+; CHECK-NEXT:    testl %ebx, %ebx
+; CHECK-NEXT:    jle LBB0_22
+; CHECK-NEXT:  ## %bb.20: ## %bb.nph11
+; CHECK-NEXT:    movl %edi, %esi
+; CHECK-NEXT:    leal 15(%edi), %eax
+; CHECK-NEXT:    andl $-16, %eax
+; CHECK-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) ## 4-byte Spill
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  LBB0_21: ## %bb30
+; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    subl $4, %esp
+; CHECK-NEXT:    pushl %esi
+; CHECK-NEXT:    pushl %ebp
+; CHECK-NEXT:    pushl %edi
+; CHECK-NEXT:    calll _memcpy
+; CHECK-NEXT:    addl $16, %esp
+; CHECK-NEXT:    addl %esi, %edi
+; CHECK-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp ## 4-byte Folded Reload
+; CHECK-NEXT:    decl %ebx
+; CHECK-NEXT:    jne LBB0_21
+; CHECK-NEXT:  LBB0_22: ## %bb33
+; CHECK-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx ## 4-byte Reload
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; CHECK-NEXT:    addl %ecx, %edx
+; CHECK-NEXT:  LBB0_23: ## %bb33
+; CHECK-NEXT:    movl %ecx, %eax
+; CHECK-NEXT:    shrl $31, %eax
+; CHECK-NEXT:    addl %ecx, %eax
+; CHECK-NEXT:    sarl %eax
+; CHECK-NEXT:    subl $4, %esp
+; CHECK-NEXT:    pushl %eax
+; CHECK-NEXT:    pushl $128
+; CHECK-NEXT:    pushl %edx
+; CHECK-NEXT:    calll _memset
+; CHECK-NEXT:    addl $44, %esp
+; CHECK-NEXT:  LBB0_25: ## %return
+; CHECK-NEXT:    popl %esi
+; CHECK-NEXT:    popl %edi
+; CHECK-NEXT:    popl %ebx
+; CHECK-NEXT:    popl %ebp
+; CHECK-NEXT:    retl
+; CHECK-NEXT:  LBB0_24: ## %return
+; CHECK-NEXT:    addl $28, %esp
+; CHECK-NEXT:    jmp LBB0_25
 entry:
   %0 = mul i32 %x, %w
   %1 = mul i32 %x, %w
@@ -232,6 +446,208 @@ return:                                           ; preds = %bb20
 }
 
 define void @bar(i8* %r, i32 %s, i32 %w, i32 %x, i8* %j, i32 %d) nounwind {
+; CHECK-LABEL: bar:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    pushl %ebp
+; CHECK-NEXT:    pushl %ebx
+; CHECK-NEXT:    pushl %edi
+; CHECK-NEXT:    pushl %esi
+; CHECK-NEXT:    subl $28, %esp
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; CHECK-NEXT:    movl %ebp, %eax
+; CHECK-NEXT:    imull %ecx, %eax
+; CHECK-NEXT:    cmpl $1, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) ## 4-byte Spill
+; CHECK-NEXT:    je LBB1_19
+; CHECK-NEXT:  ## %bb.1: ## %bb10.preheader
+; CHECK-NEXT:    shrl $2, %eax
+; CHECK-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) ## 4-byte Spill
+; CHECK-NEXT:    testl %ebp, %ebp
+; CHECK-NEXT:    je LBB1_12
+; CHECK-NEXT:  ## %bb.2: ## %bb.nph9
+; CHECK-NEXT:    cmpl $0, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    je LBB1_12
+; CHECK-NEXT:  ## %bb.3: ## %bb.nph9.split
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    incl %eax
+; CHECK-NEXT:    xorl %ecx, %ecx
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  LBB1_6: ## %bb7.preheader
+; CHECK-NEXT:    ## =>This Loop Header: Depth=1
+; CHECK-NEXT:    ## Child Loop BB1_4 Depth 2
+; CHECK-NEXT:    xorl %esi, %esi
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  LBB1_4: ## %bb6
+; CHECK-NEXT:    ## Parent Loop BB1_6 Depth=1
+; CHECK-NEXT:    ## => This Inner Loop Header: Depth=2
+; CHECK-NEXT:    movzbl (%eax,%esi,2), %ebx
+; CHECK-NEXT:    movb %bl, (%edx,%esi)
+; CHECK-NEXT:    incl %esi
+; CHECK-NEXT:    cmpl %edi, %esi
+; CHECK-NEXT:    jb LBB1_4
+; CHECK-NEXT:  ## %bb.5: ## %bb9
+; CHECK-NEXT:    ## in Loop: Header=BB1_6 Depth=1
+; CHECK-NEXT:    incl %ecx
+; CHECK-NEXT:    addl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    addl %edi, %edx
+; CHECK-NEXT:    cmpl %ebp, %ecx
+; CHECK-NEXT:    jne LBB1_6
+; CHECK-NEXT:  LBB1_12: ## %bb18.loopexit
+; CHECK-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax ## 4-byte Reload
+; CHECK-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx ## 4-byte Reload
+; CHECK-NEXT:    addl %ecx, %eax
+; CHECK-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) ## 4-byte Spill
+; CHECK-NEXT:    cmpl $1, %ebp
+; CHECK-NEXT:    jbe LBB1_13
+; CHECK-NEXT:  ## %bb.7: ## %bb.nph5
+; CHECK-NEXT:    cmpl $2, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    jb LBB1_13
+; CHECK-NEXT:  ## %bb.8: ## %bb.nph5.split
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; CHECK-NEXT:    shrl %ebp
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    shrl %eax
+; CHECK-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) ## 4-byte Spill
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx ## 4-byte Reload
+; CHECK-NEXT:    addl %eax, %ecx
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; CHECK-NEXT:    leal 2(%edx), %edx
+; CHECK-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) ## 4-byte Spill
+; CHECK-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx ## 4-byte Reload
+; CHECK-NEXT:    addl %edx, %eax
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    xorl %edi, %edi
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  LBB1_9: ## %bb13
+; CHECK-NEXT:    ## =>This Loop Header: Depth=1
+; CHECK-NEXT:    ## Child Loop BB1_10 Depth 2
+; CHECK-NEXT:    movl %edi, %ebx
+; CHECK-NEXT:    andl $1, %ebx
+; CHECK-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) ## 4-byte Spill
+; CHECK-NEXT:    addl %edx, %ebx
+; CHECK-NEXT:    imull {{[0-9]+}}(%esp), %ebx
+; CHECK-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx ## 4-byte Folded Reload
+; CHECK-NEXT:    xorl %esi, %esi
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  LBB1_10: ## %bb14
+; CHECK-NEXT:    ## Parent Loop BB1_9 Depth=1
+; CHECK-NEXT:    ## => This Inner Loop Header: Depth=2
+; CHECK-NEXT:    movzbl -2(%ebx,%esi,4), %edx
+; CHECK-NEXT:    movb %dl, (%eax,%esi)
+; CHECK-NEXT:    movzbl (%ebx,%esi,4), %edx
+; CHECK-NEXT:    movb %dl, (%ecx,%esi)
+; CHECK-NEXT:    incl %esi
+; CHECK-NEXT:    cmpl %ebp, %esi
+; CHECK-NEXT:    jb LBB1_10
+; CHECK-NEXT:  ## %bb.11: ## %bb17
+; CHECK-NEXT:    ## in Loop: Header=BB1_9 Depth=1
+; CHECK-NEXT:    incl %edi
+; CHECK-NEXT:    addl %ebp, %ecx
+; CHECK-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx ## 4-byte Reload
+; CHECK-NEXT:    addl $2, %edx
+; CHECK-NEXT:    addl %ebp, %eax
+; CHECK-NEXT:    cmpl {{[-0-9]+}}(%e{{[sb]}}p), %edi ## 4-byte Folded Reload
+; CHECK-NEXT:    jb LBB1_9
+; CHECK-NEXT:  LBB1_13: ## %bb20
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; CHECK-NEXT:    cmpl $1, %edx
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; CHECK-NEXT:    je LBB1_19
+; CHECK-NEXT:  ## %bb.14: ## %bb20
+; CHECK-NEXT:    cmpl $3, %edx
+; CHECK-NEXT:    jne LBB1_24
+; CHECK-NEXT:  ## %bb.15: ## %bb22
+; CHECK-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax ## 4-byte Reload
+; CHECK-NEXT:    addl %eax, {{[-0-9]+}}(%e{{[sb]}}p) ## 4-byte Folded Spill
+; CHECK-NEXT:    testl %ebp, %ebp
+; CHECK-NEXT:    je LBB1_18
+; CHECK-NEXT:  ## %bb.16: ## %bb.nph
+; CHECK-NEXT:    movl %ebp, %esi
+; CHECK-NEXT:    leal 15(%ebp), %eax
+; CHECK-NEXT:    andl $-16, %eax
+; CHECK-NEXT:    imull {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    leal 15(%ecx), %ebx
+; CHECK-NEXT:    andl $-16, %ebx
+; CHECK-NEXT:    addl %eax, %edi
+; CHECK-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax ## 4-byte Reload
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; CHECK-NEXT:    leal (%edx,%eax), %ebp
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  LBB1_17: ## %bb23
+; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    subl $4, %esp
+; CHECK-NEXT:    pushl %ecx
+; CHECK-NEXT:    pushl %edi
+; CHECK-NEXT:    pushl %ebp
+; CHECK-NEXT:    calll _memcpy
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT:    addl $16, %esp
+; CHECK-NEXT:    addl %ecx, %ebp
+; CHECK-NEXT:    addl %ebx, %edi
+; CHECK-NEXT:    decl %esi
+; CHECK-NEXT:    jne LBB1_17
+; CHECK-NEXT:  LBB1_18: ## %bb26
+; CHECK-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax ## 4-byte Reload
+; CHECK-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx ## 4-byte Reload
+; CHECK-NEXT:    addl %ecx, %eax
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; CHECK-NEXT:    addl %eax, %edx
+; CHECK-NEXT:    shrl %ecx
+; CHECK-NEXT:    subl $4, %esp
+; CHECK-NEXT:    pushl %ecx
+; CHECK-NEXT:    pushl $128
+; CHECK-NEXT:    pushl %edx
+; CHECK-NEXT:    jmp LBB1_23
+; CHECK-NEXT:  LBB1_19: ## %bb29
+; CHECK-NEXT:    testl %ebp, %ebp
+; CHECK-NEXT:    je LBB1_22
+; CHECK-NEXT:  ## %bb.20: ## %bb.nph11
+; CHECK-NEXT:    movl %ebp, %esi
+; CHECK-NEXT:    leal 15(%ecx), %ebx
+; CHECK-NEXT:    andl $-16, %ebx
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  LBB1_21: ## %bb30
+; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    subl $4, %esp
+; CHECK-NEXT:    pushl %ecx
+; CHECK-NEXT:    pushl %edi
+; CHECK-NEXT:    pushl %ebp
+; CHECK-NEXT:    calll _memcpy
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT:    addl $16, %esp
+; CHECK-NEXT:    addl %ecx, %ebp
+; CHECK-NEXT:    addl %ebx, %edi
+; CHECK-NEXT:    decl %esi
+; CHECK-NEXT:    jne LBB1_21
+; CHECK-NEXT:  LBB1_22: ## %bb33
+; CHECK-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax ## 4-byte Reload
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT:    addl %eax, %ecx
+; CHECK-NEXT:    shrl %eax
+; CHECK-NEXT:    subl $4, %esp
+; CHECK-NEXT:    pushl %eax
+; CHECK-NEXT:    pushl $128
+; CHECK-NEXT:    pushl %ecx
+; CHECK-NEXT:  LBB1_23: ## %bb33
+; CHECK-NEXT:    calll _memset
+; CHECK-NEXT:    addl $44, %esp
+; CHECK-NEXT:  LBB1_25: ## %return
+; CHECK-NEXT:    popl %esi
+; CHECK-NEXT:    popl %edi
+; CHECK-NEXT:    popl %ebx
+; CHECK-NEXT:    popl %ebp
+; CHECK-NEXT:    retl
+; CHECK-NEXT:  LBB1_24: ## %return
+; CHECK-NEXT:    addl $28, %esp
+; CHECK-NEXT:    jmp LBB1_25
 entry:
   %0 = mul i32 %x, %w
   %1 = mul i32 %x, %w
diff --git a/llvm/test/CodeGen/X86/overflow.ll b/llvm/test/CodeGen/X86/overflow.ll
index a9fd19d4f..6e7850068 100644
--- a/llvm/test/CodeGen/X86/overflow.ll
+++ b/llvm/test/CodeGen/X86/overflow.ll
@@ -10,42 +10,29 @@ define i128 @mulhioverflow(i64 %a, i64 %b, i64 %c) nounwind {
 ; X32-NEXT:    pushl %edi
 ; X32-NEXT:    pushl %esi
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %ebp
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    mull %ebp
-; X32-NEXT:    movl %edx, %ebx
-; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    mull %ebp
-; X32-NEXT:    movl %edx, %ebp
-; X32-NEXT:    movl %eax, %esi
-; X32-NEXT:    addl %ebx, %esi
-; X32-NEXT:    adcl $0, %ebp
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    mull %edi
+; X32-NEXT:    mull %ebx
+; X32-NEXT:    movl %edx, %edi
+; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    mull %ebx
 ; X32-NEXT:    movl %edx, %ebx
-; X32-NEXT:    addl %esi, %eax
-; X32-NEXT:    adcl %ebp, %ebx
-; X32-NEXT:    setb %al
-; X32-NEXT:    movzbl %al, %ecx
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    mull %edi
-; X32-NEXT:    movl %edx, %esi
 ; X32-NEXT:    movl %eax, %ebp
-; X32-NEXT:    addl %ebx, %ebp
-; X32-NEXT:    adcl %ecx, %esi
-; X32-NEXT:    xorl %ecx, %ecx
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    movl %edx, %edi
-; X32-NEXT:    movl %eax, %ebx
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    mull %ecx
-; X32-NEXT:    addl %ebx, %eax
-; X32-NEXT:    adcl %edi, %edx
+; X32-NEXT:    addl %edi, %ebp
+; X32-NEXT:    adcl $0, %ebx
+; X32-NEXT:    movl %ecx, %eax
+; X32-NEXT:    mull %esi
+; X32-NEXT:    movl %edx, %ecx
 ; X32-NEXT:    addl %ebp, %eax
-; X32-NEXT:    adcl %esi, %edx
+; X32-NEXT:    adcl %ebx, %ecx
+; X32-NEXT:    setb %bl
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    mull %esi
+; X32-NEXT:    addl %ecx, %eax
+; X32-NEXT:    movzbl %bl, %ecx
+; X32-NEXT:    adcl %ecx, %edx
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X32-NEXT:    andl $1, %ecx
 ; X32-NEXT:    addl %eax, %ecx
diff --git a/llvm/test/CodeGen/X86/overlap-shift.ll b/llvm/test/CodeGen/X86/overlap-shift.ll
index c1e15f989..7f51baea4 100644
--- a/llvm/test/CodeGen/X86/overlap-shift.ll
+++ b/llvm/test/CodeGen/X86/overlap-shift.ll
@@ -1,3 +1,6 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- -x86-asm-syntax=intel | FileCheck %s
+
 ;; X's live range extends beyond the shift, so the register allocator
 ;; cannot coalesce it with Y.  Because of this, a copy needs to be
 ;; emitted before the shift to save the register value before it is
@@ -6,12 +9,15 @@
 
 ; Check that the shift gets turned into an LEA.
 
-; RUN: llc < %s -mtriple=i686-- -x86-asm-syntax=intel | \
-; RUN:   not grep "mov E.X, E.X"
-
 @G = external global i32                ; <i32*> [#uses=1]
 
 define i32 @test1(i32 %X) {
+; CHECK-LABEL: test1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mov eax, dword ptr [esp + 4]
+; CHECK-NEXT:    lea ecx, [4*eax]
+; CHECK-NEXT:    mov dword ptr [G], ecx
+; CHECK-NEXT:    ret
         %Z = shl i32 %X, 2              ; <i32> [#uses=1]
         store volatile i32 %Z, i32* @G
         ret i32 %X
diff --git a/llvm/test/CodeGen/X86/packed_struct.ll b/llvm/test/CodeGen/X86/packed_struct.ll
index e00a772ec..b0c4ea0e2 100644
--- a/llvm/test/CodeGen/X86/packed_struct.ll
+++ b/llvm/test/CodeGen/X86/packed_struct.ll
@@ -1,9 +1,5 @@
-; RUN: llc < %s > %t
-; RUN: grep foos+5 %t
-; RUN: grep foos+1 %t
-; RUN: grep foos+9 %t
-; RUN: grep bara+19 %t
-; RUN: grep bara+4 %t
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s | FileCheck %s
 
 ; make sure we compute the correct offset for a packed structure
 
@@ -16,6 +12,12 @@ target triple = "i686-pc-linux-gnu"
 @bara = weak global [4 x <{ i32, i8 }>] zeroinitializer		; <[4 x <{ i32, i8 }>]*> [#uses=2]
 
 define i32 @foo() nounwind {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movl foos+5, %eax
+; CHECK-NEXT:    addl foos+1, %eax
+; CHECK-NEXT:    addl foos+9, %eax
+; CHECK-NEXT:    retl
 entry:
 	%tmp = load i32, i32* getelementptr (%struct.anon, %struct.anon* @foos, i32 0, i32 1)		; <i32> [#uses=1]
 	%tmp3 = load i32, i32* getelementptr (%struct.anon, %struct.anon* @foos, i32 0, i32 2)		; <i32> [#uses=1]
@@ -26,6 +28,11 @@ entry:
 }
 
 define i8 @bar() nounwind {
+; CHECK-LABEL: bar:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movb bara+19, %al
+; CHECK-NEXT:    addb bara+4, %al
+; CHECK-NEXT:    retl
 entry:
 	%tmp = load i8, i8* getelementptr ([4 x <{ i32, i8 }>], [4 x <{ i32, i8 }>]* @bara, i32 0, i32 0, i32 1)		; <i8> [#uses=1]
 	%tmp4 = load i8, i8* getelementptr ([4 x <{ i32, i8 }>], [4 x <{ i32, i8 }>]* @bara, i32 0, i32 3, i32 1)		; <i8> [#uses=1]
diff --git a/llvm/test/CodeGen/X86/packss.ll b/llvm/test/CodeGen/X86/packss.ll
index 7c4e1d7ec..e3bd9d9e6 100644
--- a/llvm/test/CodeGen/X86/packss.ll
+++ b/llvm/test/CodeGen/X86/packss.ll
@@ -172,19 +172,18 @@ define <8 x i16> @trunc_ashr_v4i64_demandedelts(<4 x i64> %a0) {
 ;
 ; X86-AVX1-LABEL: trunc_ashr_v4i64_demandedelts:
 ; X86-AVX1:       # %bb.0:
-; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; X86-AVX1-NEXT:    vpsllq $63, %xmm1, %xmm2
-; X86-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1,2,3],xmm1[4,5,6,7]
-; X86-AVX1-NEXT:    vpsllq $63, %xmm0, %xmm2
-; X86-AVX1-NEXT:    vpsrlq $63, %xmm1, %xmm1
-; X86-AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [1,0,0,2147483648]
-; X86-AVX1-NEXT:    vpxor %xmm3, %xmm1, %xmm1
-; X86-AVX1-NEXT:    vpsubq %xmm3, %xmm1, %xmm1
+; X86-AVX1-NEXT:    vpsllq $63, %xmm0, %xmm1
+; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; X86-AVX1-NEXT:    vpsllq $63, %xmm2, %xmm2
 ; X86-AVX1-NEXT:    vpsrlq $63, %xmm2, %xmm2
-; X86-AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm2[0,1,2,3],xmm0[4,5,6,7]
+; X86-AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [1,0,0,2147483648]
+; X86-AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
+; X86-AVX1-NEXT:    vpsubq %xmm3, %xmm2, %xmm2
+; X86-AVX1-NEXT:    vpsrlq $63, %xmm1, %xmm1
+; X86-AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4,5,6,7]
 ; X86-AVX1-NEXT:    vpxor %xmm3, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vpsubq %xmm3, %xmm0, %xmm0
-; X86-AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; X86-AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; X86-AVX1-NEXT:    vpermilps {{.*#+}} ymm0 = ymm0[0,0,0,0,4,4,4,4]
 ; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; X86-AVX1-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
@@ -225,19 +224,18 @@ define <8 x i16> @trunc_ashr_v4i64_demandedelts(<4 x i64> %a0) {
 ;
 ; X64-AVX1-LABEL: trunc_ashr_v4i64_demandedelts:
 ; X64-AVX1:       # %bb.0:
-; X64-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; X64-AVX1-NEXT:    vpsllq $63, %xmm1, %xmm2
-; X64-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1,2,3],xmm1[4,5,6,7]
-; X64-AVX1-NEXT:    vpsllq $63, %xmm0, %xmm2
-; X64-AVX1-NEXT:    vpsrlq $63, %xmm1, %xmm1
-; X64-AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [1,9223372036854775808]
-; X64-AVX1-NEXT:    vpxor %xmm3, %xmm1, %xmm1
-; X64-AVX1-NEXT:    vpsubq %xmm3, %xmm1, %xmm1
+; X64-AVX1-NEXT:    vpsllq $63, %xmm0, %xmm1
+; X64-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; X64-AVX1-NEXT:    vpsllq $63, %xmm2, %xmm2
 ; X64-AVX1-NEXT:    vpsrlq $63, %xmm2, %xmm2
-; X64-AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm2[0,1,2,3],xmm0[4,5,6,7]
+; X64-AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [1,9223372036854775808]
+; X64-AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
+; X64-AVX1-NEXT:    vpsubq %xmm3, %xmm2, %xmm2
+; X64-AVX1-NEXT:    vpsrlq $63, %xmm1, %xmm1
+; X64-AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4,5,6,7]
 ; X64-AVX1-NEXT:    vpxor %xmm3, %xmm0, %xmm0
 ; X64-AVX1-NEXT:    vpsubq %xmm3, %xmm0, %xmm0
-; X64-AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; X64-AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; X64-AVX1-NEXT:    vpermilps {{.*#+}} ymm0 = ymm0[0,0,0,0,4,4,4,4]
 ; X64-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; X64-AVX1-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
@@ -265,20 +263,19 @@ define <8 x i16> @trunc_ashr_v4i64_demandedelts(<4 x i64> %a0) {
   ret <8 x i16> %5
 }
 
-define <16 x i8> @packsswb_icmp_128_zero(<8 x i16> %a0) {
-; SSE-LABEL: packsswb_icmp_128_zero:
+define <16 x i8> @packsswb_icmp_zero_128(<8 x i16> %a0) {
+; SSE-LABEL: packsswb_icmp_zero_128:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pxor %xmm1, %xmm1
-; SSE-NEXT:    pcmpeqw %xmm0, %xmm1
-; SSE-NEXT:    packsswb %xmm0, %xmm1
-; SSE-NEXT:    movq {{.*#+}} xmm0 = xmm1[0],zero
+; SSE-NEXT:    pcmpeqw %xmm1, %xmm0
+; SSE-NEXT:    packsswb %xmm1, %xmm0
 ; SSE-NEXT:    ret{{[l|q]}}
 ;
-; AVX-LABEL: packsswb_icmp_128_zero:
+; AVX-LABEL: packsswb_icmp_zero_128:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX-NEXT:    vpcmpeqw %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14],zero,zero,zero,zero,zero,zero,zero,zero
+; AVX-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    ret{{[l|q]}}
   %1 = icmp eq <8 x i16> %a0, zeroinitializer
   %2 = sext <8 x i1> %1 to <8 x i8>
@@ -286,42 +283,106 @@ define <16 x i8> @packsswb_icmp_128_zero(<8 x i16> %a0) {
   ret <16 x i8> %3
 }
 
+define <16 x i8> @packsswb_icmp_zero_trunc_128(<8 x i16> %a0) {
+; SSE-LABEL: packsswb_icmp_zero_trunc_128:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pxor %xmm1, %xmm1
+; SSE-NEXT:    pcmpeqw %xmm1, %xmm0
+; SSE-NEXT:    packsswb %xmm1, %xmm0
+; SSE-NEXT:    ret{{[l|q]}}
+;
+; AVX-LABEL: packsswb_icmp_zero_trunc_128:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vpcmpeqw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    ret{{[l|q]}}
+  %1 = icmp eq <8 x i16> %a0, zeroinitializer
+  %2 = sext <8 x i1> %1 to <8 x i16>
+  %3 = shufflevector <8 x i16> %2, <8 x i16> zeroinitializer, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %4 = trunc <16 x i16> %3 to <16 x i8>
+  ret <16 x i8> %4
+}
+
 define <32 x i8> @packsswb_icmp_zero_256(<16 x i16> %a0) {
 ; SSE-LABEL: packsswb_icmp_zero_256:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pxor %xmm2, %xmm2
 ; SSE-NEXT:    pcmpeqw %xmm2, %xmm1
 ; SSE-NEXT:    pcmpeqw %xmm2, %xmm0
-; SSE-NEXT:    packsswb %xmm1, %xmm0
-; SSE-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,1],xmm0[2,3]
-; SSE-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6,7]
-; SSE-NEXT:    movaps %xmm2, %xmm1
+; SSE-NEXT:    pxor %xmm3, %xmm3
+; SSE-NEXT:    packsswb %xmm0, %xmm3
+; SSE-NEXT:    packsswb %xmm1, %xmm2
+; SSE-NEXT:    movdqa %xmm3, %xmm0
+; SSE-NEXT:    movdqa %xmm2, %xmm1
 ; SSE-NEXT:    ret{{[l|q]}}
 ;
 ; AVX1-LABEL: packsswb_icmp_zero_256:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqw %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpacksswb %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3],ymm1[4,5],ymm0[6,7]
+; AVX1-NEXT:    vpcmpeqw %xmm1, %xmm0, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vpcmpeqw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpacksswb %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vpacksswb %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    ret{{[l|q]}}
 ;
 ; AVX2-LABEL: packsswb_icmp_zero_256:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX2-NEXT:    vpcmpeqw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm2
-; AVX2-NEXT:    vpacksswb %xmm2, %xmm0, %xmm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,2,1]
-; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3],ymm1[4,5],ymm0[6,7]
+; AVX2-NEXT:    vpacksswb %ymm0, %ymm1, %ymm0
+; AVX2-NEXT:    ret{{[l|q]}}
+  %1 = icmp eq <16 x i16> %a0, zeroinitializer
+  %2 = sext <16 x i1> %1 to <16 x i16>
+  %3 = bitcast <16 x i16> %2 to <32 x i8>
+  %4 = shufflevector <32 x i8> zeroinitializer, <32 x i8> %3, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 32, i32 34, i32 36, i32 38, i32 40, i32 42, i32 44, i32 46, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 48, i32 50, i32 52, i32 54, i32 56, i32 58, i32 60, i32 62>
+  ret <32 x i8> %4
+}
+
+define <32 x i8> @packsswb_icmp_zero_trunc_256(<16 x i16> %a0) {
+; SSE-LABEL: packsswb_icmp_zero_trunc_256:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pxor %xmm2, %xmm2
+; SSE-NEXT:    pcmpeqw %xmm2, %xmm1
+; SSE-NEXT:    pcmpeqw %xmm2, %xmm0
+; SSE-NEXT:    pxor %xmm3, %xmm3
+; SSE-NEXT:    packsswb %xmm0, %xmm3
+; SSE-NEXT:    packsswb %xmm1, %xmm2
+; SSE-NEXT:    movdqa %xmm3, %xmm0
+; SSE-NEXT:    movdqa %xmm2, %xmm1
+; SSE-NEXT:    ret{{[l|q]}}
+;
+; AVX1-LABEL: packsswb_icmp_zero_trunc_256:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpeqw %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpcmpeqw %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm2 = zero,zero,ymm0[0,1]
+; AVX1-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm1
+; AVX1-NEXT:    vpacksswb %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX1-NEXT:    ret{{[l|q]}}
+;
+; AVX2-LABEL: packsswb_icmp_zero_trunc_256:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT:    vpcmpeqw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm0[4,5,6,7]
+; AVX2-NEXT:    vperm2i128 {{.*#+}} ymm0 = zero,zero,ymm0[0,1]
+; AVX2-NEXT:    vpacksswb %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    ret{{[l|q]}}
   %1 = icmp eq <16 x i16> %a0, zeroinitializer
-  %2 = sext <16 x i1> %1 to <16 x i8>
-  %3 = shufflevector <16 x i8> zeroinitializer, <16 x i8> %2, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-  ret <32 x i8> %3
+  %2 = sext <16 x i1> %1 to <16 x i16>
+  %3 = shufflevector <16 x i16> zeroinitializer, <16 x i16> %2, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+  %4 = trunc <32 x i16> %3 to <32 x i8>
+  ret <32 x i8> %4
 }
diff --git a/llvm/test/CodeGen/X86/paddus.ll b/llvm/test/CodeGen/X86/paddus.ll
index 63ef093fd..c14826fe8 100644
--- a/llvm/test/CodeGen/X86/paddus.ll
+++ b/llvm/test/CodeGen/X86/paddus.ll
@@ -1527,30 +1527,13 @@ define void @addus_v8i8(<8 x i8>* %p1, <8 x i8>* %p2) {
 ; SSE-NEXT:    movq %xmm1, (%rdi)
 ; SSE-NEXT:    retq
 ;
-; AVX1-LABEL: addus_v8i8:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
-; AVX1-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
-; AVX1-NEXT:    vpaddusb %xmm0, %xmm1, %xmm0
-; AVX1-NEXT:    vmovq %xmm0, (%rdi)
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: addus_v8i8:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
-; AVX2-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
-; AVX2-NEXT:    vpaddusb %xmm0, %xmm1, %xmm0
-; AVX2-NEXT:    vmovq %xmm0, (%rdi)
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: addus_v8i8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
-; AVX512-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
-; AVX512-NEXT:    vpaddusb %xmm0, %xmm1, %xmm0
-; AVX512-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX512-NEXT:    vpmovwb %xmm0, (%rdi)
-; AVX512-NEXT:    retq
+; AVX-LABEL: addus_v8i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
+; AVX-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
+; AVX-NEXT:    vpaddusb %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    vmovq %xmm0, (%rdi)
+; AVX-NEXT:    retq
   %ld1 = load <8 x i8>, <8 x i8>* %p1, align 8
   %ld2 = load <8 x i8>, <8 x i8>* %p2, align 8
   %1 = add <8 x i8> %ld2, %ld1
@@ -1569,30 +1552,13 @@ define void @addus_v4i8(<4 x i8>* %p1, <4 x i8>* %p2) {
 ; SSE-NEXT:    movd %xmm1, (%rdi)
 ; SSE-NEXT:    retq
 ;
-; AVX1-LABEL: addus_v4i8:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX1-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX1-NEXT:    vpaddusb %xmm0, %xmm1, %xmm0
-; AVX1-NEXT:    vmovd %xmm0, (%rdi)
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: addus_v4i8:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX2-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX2-NEXT:    vpaddusb %xmm0, %xmm1, %xmm0
-; AVX2-NEXT:    vmovd %xmm0, (%rdi)
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: addus_v4i8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX512-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX512-NEXT:    vpaddusb %xmm0, %xmm1, %xmm0
-; AVX512-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; AVX512-NEXT:    vpmovdb %xmm0, (%rdi)
-; AVX512-NEXT:    retq
+; AVX-LABEL: addus_v4i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX-NEXT:    vpaddusb %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    vmovd %xmm0, (%rdi)
+; AVX-NEXT:    retq
   %ld1 = load <4 x i8>, <4 x i8>* %p1, align 4
   %ld2 = load <4 x i8>, <4 x i8>* %p2, align 4
   %1 = add <4 x i8> %ld2, %ld1
@@ -1635,36 +1601,15 @@ define void @addus_v2i8(<2 x i8>* %p1, <2 x i8>* %p2) {
 ; SSE41-NEXT:    pextrw $0, %xmm1, (%rdi)
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: addus_v2i8:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    movzwl (%rdi), %eax
-; AVX1-NEXT:    vmovd %eax, %xmm0
-; AVX1-NEXT:    movzwl (%rsi), %eax
-; AVX1-NEXT:    vmovd %eax, %xmm1
-; AVX1-NEXT:    vpaddusb %xmm0, %xmm1, %xmm0
-; AVX1-NEXT:    vpextrw $0, %xmm0, (%rdi)
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: addus_v2i8:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    movzwl (%rdi), %eax
-; AVX2-NEXT:    vmovd %eax, %xmm0
-; AVX2-NEXT:    movzwl (%rsi), %eax
-; AVX2-NEXT:    vmovd %eax, %xmm1
-; AVX2-NEXT:    vpaddusb %xmm0, %xmm1, %xmm0
-; AVX2-NEXT:    vpextrw $0, %xmm0, (%rdi)
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: addus_v2i8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    movzwl (%rdi), %eax
-; AVX512-NEXT:    vmovd %eax, %xmm0
-; AVX512-NEXT:    movzwl (%rsi), %eax
-; AVX512-NEXT:    vmovd %eax, %xmm1
-; AVX512-NEXT:    vpaddusb %xmm0, %xmm1, %xmm0
-; AVX512-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT:    vpmovqb %xmm0, (%rdi)
-; AVX512-NEXT:    retq
+; AVX-LABEL: addus_v2i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    movzwl (%rdi), %eax
+; AVX-NEXT:    vmovd %eax, %xmm0
+; AVX-NEXT:    movzwl (%rsi), %eax
+; AVX-NEXT:    vmovd %eax, %xmm1
+; AVX-NEXT:    vpaddusb %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    vpextrw $0, %xmm0, (%rdi)
+; AVX-NEXT:    retq
   %ld1 = load <2 x i8>, <2 x i8>* %p1, align 2
   %ld2 = load <2 x i8>, <2 x i8>* %p2, align 2
   %1 = add <2 x i8> %ld2, %ld1
@@ -1683,30 +1628,13 @@ define void @addus_v4i16(<4 x i16>* %p1, <4 x i16>* %p2) {
 ; SSE-NEXT:    movq %xmm1, (%rdi)
 ; SSE-NEXT:    retq
 ;
-; AVX1-LABEL: addus_v4i16:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
-; AVX1-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
-; AVX1-NEXT:    vpaddusw %xmm0, %xmm1, %xmm0
-; AVX1-NEXT:    vmovq %xmm0, (%rdi)
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: addus_v4i16:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
-; AVX2-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
-; AVX2-NEXT:    vpaddusw %xmm0, %xmm1, %xmm0
-; AVX2-NEXT:    vmovq %xmm0, (%rdi)
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: addus_v4i16:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
-; AVX512-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
-; AVX512-NEXT:    vpaddusw %xmm0, %xmm1, %xmm0
-; AVX512-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; AVX512-NEXT:    vpmovdw %xmm0, (%rdi)
-; AVX512-NEXT:    retq
+; AVX-LABEL: addus_v4i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
+; AVX-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
+; AVX-NEXT:    vpaddusw %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    vmovq %xmm0, (%rdi)
+; AVX-NEXT:    retq
   %ld1 = load <4 x i16>, <4 x i16>* %p1, align 4
   %ld2 = load <4 x i16>, <4 x i16>* %p2, align 4
   %1 = add <4 x i16> %ld2, %ld1
@@ -1725,30 +1653,13 @@ define void @addus_v2i16(<2 x i16>* %p1, <2 x i16>* %p2) {
 ; SSE-NEXT:    movd %xmm1, (%rdi)
 ; SSE-NEXT:    retq
 ;
-; AVX1-LABEL: addus_v2i16:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX1-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX1-NEXT:    vpaddusw %xmm0, %xmm1, %xmm0
-; AVX1-NEXT:    vmovd %xmm0, (%rdi)
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: addus_v2i16:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX2-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX2-NEXT:    vpaddusw %xmm0, %xmm1, %xmm0
-; AVX2-NEXT:    vmovd %xmm0, (%rdi)
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: addus_v2i16:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX512-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX512-NEXT:    vpaddusw %xmm0, %xmm1, %xmm0
-; AVX512-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; AVX512-NEXT:    vpmovqw %xmm0, (%rdi)
-; AVX512-NEXT:    retq
+; AVX-LABEL: addus_v2i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX-NEXT:    vpaddusw %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    vmovd %xmm0, (%rdi)
+; AVX-NEXT:    retq
   %ld1 = load <2 x i16>, <2 x i16>* %p1, align 2
   %ld2 = load <2 x i16>, <2 x i16>* %p2, align 2
   %1 = add <2 x i16> %ld2, %ld1
diff --git a/llvm/test/CodeGen/X86/patchable-function-entry-ibt.ll b/llvm/test/CodeGen/X86/patchable-function-entry-ibt.ll
new file mode 100644
index 000000000..3def5a378
--- /dev/null
+++ b/llvm/test/CodeGen/X86/patchable-function-entry-ibt.ll
@@ -0,0 +1,88 @@
+; RUN: llc -mtriple=i686 %s -o - | FileCheck --check-prefixes=CHECK,32 %s
+; RUN: llc -mtriple=x86_64 %s -o - | FileCheck --check-prefixes=CHECK,64 %s
+
+;; -fpatchable-function-entry=0 -fcf-protection=branch
+define void @f0() "patchable-function-entry"="0" {
+; CHECK-LABEL: f0:
+; CHECK-NEXT: .Lfunc_begin0:
+; CHECK-NEXT: .cfi_startproc
+; CHECK-NEXT: # %bb.0:
+; 32-NEXT:     endbr32
+; 64-NEXT:     endbr64
+; CHECK-NEXT:  ret
+; CHECK-NOT:  .section __patchable_function_entries
+  ret void
+}
+
+;; -fpatchable-function-entry=1 -fcf-protection=branch
+;; For M=0, place the label .Lpatch0 after the initial ENDBR.
+;; .cfi_startproc should be placed at the function entry.
+define void @f1() "patchable-function-entry"="1" {
+; CHECK-LABEL: f1:
+; CHECK-NEXT: .Lfunc_begin1:
+; CHECK-NEXT: .cfi_startproc
+; CHECK-NEXT: # %bb.0:
+; 32-NEXT:     endbr32
+; 64-NEXT:     endbr64
+; CHECK-NEXT: .Lpatch0:
+; CHECK-NEXT:  nop
+; CHECK-NEXT:  ret
+; CHECK:      .section __patchable_function_entries,"awo",@progbits,f1,unique,0
+; 32-NEXT:    .p2align 2
+; 32-NEXT:    .long .Lpatch0
+; 64-NEXT:    .p2align 3
+; 64-NEXT:    .quad .Lpatch0
+  ret void
+}
+
+;; -fpatchable-function-entry=2,1 -fcf-protection=branch
+define void @f2_1() "patchable-function-entry"="1" "patchable-function-prefix"="1" {
+; CHECK-LABEL: .type f2_1,@function
+; CHECK-NEXT: .Ltmp0:
+; CHECK-NEXT:  nop
+; CHECK-NEXT: f2_1:
+; CHECK-NEXT: .Lfunc_begin2:
+; CHECK-NEXT: .cfi_startproc
+; CHECK-NEXT: # %bb.0:
+; 32-NEXT:     endbr32
+; 64-NEXT:     endbr64
+; CHECK-NEXT:  nop
+; CHECK-NEXT:  ret
+; CHECK:      .Lfunc_end2:
+; CHECK-NEXT: .size f2_1, .Lfunc_end2-f2_1
+; CHECK:      .section __patchable_function_entries,"awo",@progbits,f1,unique,0
+; 32-NEXT:    .p2align 2
+; 32-NEXT:    .long .Ltmp0
+; 64-NEXT:    .p2align 3
+; 64-NEXT:    .quad .Ltmp0
+  ret void
+}
+
+;; -fpatchable-function-entry=1 -fcf-protection=branch
+;; For M=0, don't create .Lpatch0 if the initial instruction is not ENDBR,
+;; even if other basic blocks may have ENDBR.
+@buf = internal global [5 x i8*] zeroinitializer
+declare i32 @llvm.eh.sjlj.setjmp(i8*)
+
+define internal void @f1i() "patchable-function-entry"="1" {
+; CHECK-LABEL: f1i:
+; CHECK-NEXT: .Lfunc_begin3:
+; CHECK-NEXT: .cfi_startproc
+; CHECK-NEXT: # %bb.0:
+; CHECK-NEXT:  nop
+; CHECK-NOT:  .Lpatch0:
+;; Another basic block has ENDBR, but it doesn't affect our decision to not create .Lpatch0
+; CHECK:       endbr
+; CHECK:      .section __patchable_function_entries,"awo",@progbits,f1,unique,0
+; 32-NEXT:    .p2align 2
+; 32-NEXT:    .long .Lfunc_begin3
+; 64-NEXT:    .p2align 3
+; 64-NEXT:    .quad .Lfunc_begin3
+entry:
+  tail call i32 @llvm.eh.sjlj.setjmp(i8* bitcast ([5 x i8*]* @buf to i8*))
+  ret void
+}
+
+!llvm.module.flags = !{!0}
+
+!0 = !{i32 4, !"cf-protection-branch", i32 1}
diff --git a/llvm/test/CodeGen/X86/patchable-function-entry.ll b/llvm/test/CodeGen/X86/patchable-function-entry.ll
new file mode 100644
index 000000000..dcf14df7b
--- /dev/null
+++ b/llvm/test/CodeGen/X86/patchable-function-entry.ll
@@ -0,0 +1,102 @@
+; RUN: llc -mtriple=i386 %s -o - | FileCheck --check-prefixes=CHECK,NOFSECT,32 %s
+; RUN: llc -mtriple=x86_64 %s -o - | FileCheck --check-prefixes=CHECK,NOFSECT,64 %s
+; RUN: llc -mtriple=x86_64 -function-sections %s -o - | FileCheck --check-prefixes=CHECK,FSECT,64 %s
+
+define void @f0() "patchable-function-entry"="0" {
+; CHECK-LABEL: f0:
+; CHECK-NEXT: .Lfunc_begin0:
+; CHECK-NOT:   nop
+; CHECK:       ret
+; CHECK-NOT:   .section __patchable_function_entries
+  ret void
+}
+
+define void @f1() "patchable-function-entry"="1" {
+; CHECK-LABEL: f1:
+; CHECK-NEXT: .Lfunc_begin1:
+; CHECK:       nop
+; CHECK-NEXT:  ret
+; CHECK:       .section __patchable_function_entries,"awo",@progbits,f1,unique,0
+; 32:          .p2align 2
+; 32-NEXT:     .long .Lfunc_begin1
+; 64:          .p2align 3
+; 64-NEXT:     .quad .Lfunc_begin1
+  ret void
+}
+
+;; Without -function-sections, f2 is in the same text section as f1.
+;; They share the __patchable_function_entries section.
+;; With -function-sections, f1 and f2 are in different text sections.
+;; Use separate __patchable_function_entries.
+define void @f2() "patchable-function-entry"="2" {
+; CHECK-LABEL: f2:
+; CHECK-NEXT: .Lfunc_begin2:
+; 32-COUNT-2:  nop
+; 64:          xchgw %ax, %ax
+; CHECK-NEXT:  ret
+; NOFSECT:     .section __patchable_function_entries,"awo",@progbits,f1,unique,0
+; FSECT:       .section __patchable_function_entries,"awo",@progbits,f2,unique,1
+; 32:          .p2align 2
+; 32-NEXT:     .long .Lfunc_begin2
+; 64:          .p2align 3
+; 64-NEXT:     .quad .Lfunc_begin2
+  ret void
+}
+
+$f3 = comdat any
+define void @f3() "patchable-function-entry"="3" comdat {
+; CHECK-LABEL: f3:
+; CHECK-NEXT: .Lfunc_begin3:
+; 32-COUNT-3:  nop
+; 64:          nopl (%rax)
+; CHECK:       ret
+; NOFSECT:     .section __patchable_function_entries,"aGwo",@progbits,f3,comdat,f3,unique,1
+; FSECT:       .section __patchable_function_entries,"aGwo",@progbits,f3,comdat,f3,unique,2
+; 32:          .p2align 2
+; 32-NEXT:     .long .Lfunc_begin3
+; 64:          .p2align 3
+; 64-NEXT:     .quad .Lfunc_begin3
+  ret void
+}
+
+$f5 = comdat any
+define void @f5() "patchable-function-entry"="5" comdat {
+; CHECK-LABEL: f5:
+; CHECK-NEXT: .Lfunc_begin4:
+; 32-COUNT-5:  nop
+; 64:          nopl 8(%rax,%rax)
+; CHECK-NEXT:  ret
+; NOFSECT      .section __patchable_function_entries,"aGwo",@progbits,f5,comdat,f5,unique,2
+; FSECT:       .section __patchable_function_entries,"aGwo",@progbits,f5,comdat,f5,unique,3
+; 32:          .p2align 2
+; 32-NEXT:     .long .Lfunc_begin4
+; 64:          .p2align 3
+; 64-NEXT:     .quad .Lfunc_begin4
+  ret void
+}
+
+;; -fpatchable-function-entry=3,2
+;; "patchable-function-prefix" emits data before the function entry label.
+;; We emit 1-byte NOPs before the function entry, so that with a partial patch,
+;; the remaining instructions do not need to be modified.
+define void @f3_2() "patchable-function-entry"="1" "patchable-function-prefix"="2" {
+; CHECK-LABEL: .type f3_2,@function
+; CHECK-NEXT: .Ltmp0: # @f3_2
+; CHECK-NEXT:  nop
+; CHECK-NEXT:  nop
+; CHECK-NEXT: f3_2:
+; CHECK:      # %bb.0:
+; CHECK-NEXT:  nop
+; CHECK-NEXT:  ret
+;; .size does not include the prefix.
+; CHECK:      .Lfunc_end5:
+; CHECK-NEXT: .size f3_2, .Lfunc_end5-f3_2
+; NOFSECT     .section __patchable_function_entries,"awo",@progbits,f0,unique,0
+; FSECT:      .section __patchable_function_entries,"awo",@progbits,f3_2,unique,4
+; 32:         .p2align 2
+; 32-NEXT:    .long .Ltmp0
+; 64:         .p2align 3
+; 64-NEXT:    .quad .Ltmp0
+  %frame = alloca i8, i32 16
+  ret void
+}
diff --git a/llvm/test/CodeGen/X86/patchable-prologue.ll b/llvm/test/CodeGen/X86/patchable-prologue.ll
index 3779bc395..222243019 100644
--- a/llvm/test/CodeGen/X86/patchable-prologue.ll
+++ b/llvm/test/CodeGen/X86/patchable-prologue.ll
@@ -13,7 +13,7 @@ define void @f0() "patchable-function"="prologue-short-redirect" {
   ret void
 }
 
-define void @f1() "patchable-function"="prologue-short-redirect" "no-frame-pointer-elim"="true" {
+define void @f1() "patchable-function"="prologue-short-redirect" "frame-pointer"="all" {
 ; CHECK-LABEL: _f1
 ; CHECK-NEXT: ff f5 	pushq	%rbp
 
diff --git a/llvm/test/CodeGen/X86/peep-test-0.ll b/llvm/test/CodeGen/X86/peep-test-0.ll
index 2d5e7a148..72619f6cf 100644
--- a/llvm/test/CodeGen/X86/peep-test-0.ll
+++ b/llvm/test/CodeGen/X86/peep-test-0.ll
@@ -1,8 +1,24 @@
-; RUN: llc < %s -mtriple=x86_64-- > %t
-; RUN: not grep cmp %t
-; RUN: not grep test %t
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-- | FileCheck %s
 
 define void @loop(i64 %n, double* nocapture %d) nounwind {
+; CHECK-LABEL: loop:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    shlq $4, %rax
+; CHECK-NEXT:    addq %rsi, %rax
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB0_1: # %bb
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
+; CHECK-NEXT:    mulsd %xmm0, %xmm1
+; CHECK-NEXT:    movsd %xmm1, (%rax)
+; CHECK-NEXT:    addq $8, %rax
+; CHECK-NEXT:    incq %rdi
+; CHECK-NEXT:    jne .LBB0_1
+; CHECK-NEXT:  # %bb.2: # %return
+; CHECK-NEXT:    retq
 entry:
 	br label %bb
 
diff --git a/llvm/test/CodeGen/X86/peep-test-1.ll b/llvm/test/CodeGen/X86/peep-test-1.ll
index e3d34d30f..88762cac1 100644
--- a/llvm/test/CodeGen/X86/peep-test-1.ll
+++ b/llvm/test/CodeGen/X86/peep-test-1.ll
@@ -1,9 +1,21 @@
-; RUN: llc < %s -mtriple=i686-- > %t
-; RUN: grep dec %t | count 1
-; RUN: not grep test %t
-; RUN: not grep cmp %t
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- | FileCheck %s
 
 define void @foo(i32 %n, double* nocapture %p) nounwind {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB0_1: # %bb
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    fldl (%eax,%ecx,8)
+; CHECK-NEXT:    fmull {{\.LCPI.*}}
+; CHECK-NEXT:    fstpl (%eax,%ecx,8)
+; CHECK-NEXT:    decl %ecx
+; CHECK-NEXT:    js .LBB0_1
+; CHECK-NEXT:  # %bb.2: # %return
+; CHECK-NEXT:    retl
 	br label %bb
 
 bb:
diff --git a/llvm/test/CodeGen/X86/phaddsub-extract.ll b/llvm/test/CodeGen/X86/phaddsub-extract.ll
index 2a7039e93..b7af19b7b 100644
--- a/llvm/test/CodeGen/X86/phaddsub-extract.ll
+++ b/llvm/test/CodeGen/X86/phaddsub-extract.ll
@@ -1699,8 +1699,7 @@ define i32 @partial_reduction_add_v8i32(<8 x i32> %x) {
 ;
 ; AVX-FAST-LABEL: partial_reduction_add_v8i32:
 ; AVX-FAST:       # %bb.0:
-; AVX-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX-FAST-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX-FAST-NEXT:    vphaddd %xmm0, %xmm0, %xmm0
 ; AVX-FAST-NEXT:    vphaddd %xmm0, %xmm0, %xmm0
 ; AVX-FAST-NEXT:    vmovd %xmm0, %eax
 ; AVX-FAST-NEXT:    vzeroupper
@@ -1741,34 +1740,13 @@ define i32 @partial_reduction_add_v16i32(<16 x i32> %x) {
 ; AVX-SLOW-NEXT:    vzeroupper
 ; AVX-SLOW-NEXT:    retq
 ;
-; AVX1-FAST-LABEL: partial_reduction_add_v16i32:
-; AVX1-FAST:       # %bb.0:
-; AVX1-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-FAST-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-FAST-NEXT:    vphaddd %xmm0, %xmm0, %xmm0
-; AVX1-FAST-NEXT:    vmovd %xmm0, %eax
-; AVX1-FAST-NEXT:    vzeroupper
-; AVX1-FAST-NEXT:    retq
-;
-; AVX2-FAST-LABEL: partial_reduction_add_v16i32:
-; AVX2-FAST:       # %bb.0:
-; AVX2-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-FAST-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX2-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-FAST-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX2-FAST-NEXT:    vmovd %xmm0, %eax
-; AVX2-FAST-NEXT:    vzeroupper
-; AVX2-FAST-NEXT:    retq
-;
-; AVX512-FAST-LABEL: partial_reduction_add_v16i32:
-; AVX512-FAST:       # %bb.0:
-; AVX512-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-FAST-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX512-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-FAST-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX512-FAST-NEXT:    vmovd %xmm0, %eax
-; AVX512-FAST-NEXT:    vzeroupper
-; AVX512-FAST-NEXT:    retq
+; AVX-FAST-LABEL: partial_reduction_add_v16i32:
+; AVX-FAST:       # %bb.0:
+; AVX-FAST-NEXT:    vphaddd %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    vphaddd %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    vmovd %xmm0, %eax
+; AVX-FAST-NEXT:    vzeroupper
+; AVX-FAST-NEXT:    retq
   %x23 = shufflevector <16 x i32> %x, <16 x i32> undef, <16 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
   %x0213 = add <16 x i32> %x, %x23
   %x13 = shufflevector <16 x i32> %x0213, <16 x i32> undef, <16 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -2010,8 +1988,7 @@ define i32 @hadd32_8(<8 x i32> %x225) {
 ;
 ; AVX-FAST-LABEL: hadd32_8:
 ; AVX-FAST:       # %bb.0:
-; AVX-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX-FAST-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX-FAST-NEXT:    vphaddd %xmm0, %xmm0, %xmm0
 ; AVX-FAST-NEXT:    vphaddd %xmm0, %xmm0, %xmm0
 ; AVX-FAST-NEXT:    vmovd %xmm0, %eax
 ; AVX-FAST-NEXT:    vzeroupper
@@ -2052,34 +2029,13 @@ define i32 @hadd32_16(<16 x i32> %x225) {
 ; AVX-SLOW-NEXT:    vzeroupper
 ; AVX-SLOW-NEXT:    retq
 ;
-; AVX1-FAST-LABEL: hadd32_16:
-; AVX1-FAST:       # %bb.0:
-; AVX1-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-FAST-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-FAST-NEXT:    vphaddd %xmm0, %xmm0, %xmm0
-; AVX1-FAST-NEXT:    vmovd %xmm0, %eax
-; AVX1-FAST-NEXT:    vzeroupper
-; AVX1-FAST-NEXT:    retq
-;
-; AVX2-FAST-LABEL: hadd32_16:
-; AVX2-FAST:       # %bb.0:
-; AVX2-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-FAST-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX2-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-FAST-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX2-FAST-NEXT:    vmovd %xmm0, %eax
-; AVX2-FAST-NEXT:    vzeroupper
-; AVX2-FAST-NEXT:    retq
-;
-; AVX512-FAST-LABEL: hadd32_16:
-; AVX512-FAST:       # %bb.0:
-; AVX512-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-FAST-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX512-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-FAST-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX512-FAST-NEXT:    vmovd %xmm0, %eax
-; AVX512-FAST-NEXT:    vzeroupper
-; AVX512-FAST-NEXT:    retq
+; AVX-FAST-LABEL: hadd32_16:
+; AVX-FAST:       # %bb.0:
+; AVX-FAST-NEXT:    vphaddd %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    vphaddd %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    vmovd %xmm0, %eax
+; AVX-FAST-NEXT:    vzeroupper
+; AVX-FAST-NEXT:    retq
   %x226 = shufflevector <16 x i32> %x225, <16 x i32> undef, <16 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
   %x227 = add <16 x i32> %x225, %x226
   %x228 = shufflevector <16 x i32> %x227, <16 x i32> undef, <16 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -2149,8 +2105,7 @@ define i32 @hadd32_8_optsize(<8 x i32> %x225) optsize {
 ;
 ; AVX-LABEL: hadd32_8_optsize:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vphaddd %xmm0, %xmm0, %xmm0
 ; AVX-NEXT:    vphaddd %xmm0, %xmm0, %xmm0
 ; AVX-NEXT:    vmovd %xmm0, %eax
 ; AVX-NEXT:    vzeroupper
@@ -2172,63 +2127,13 @@ define i32 @hadd32_16_optsize(<16 x i32> %x225) optsize {
 ; SSE3-NEXT:    movd %xmm1, %eax
 ; SSE3-NEXT:    retq
 ;
-; AVX1-SLOW-LABEL: hadd32_16_optsize:
-; AVX1-SLOW:       # %bb.0:
-; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-SLOW-NEXT:    vphaddd %xmm0, %xmm0, %xmm0
-; AVX1-SLOW-NEXT:    vmovd %xmm0, %eax
-; AVX1-SLOW-NEXT:    vzeroupper
-; AVX1-SLOW-NEXT:    retq
-;
-; AVX1-FAST-LABEL: hadd32_16_optsize:
-; AVX1-FAST:       # %bb.0:
-; AVX1-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-FAST-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-FAST-NEXT:    vphaddd %xmm0, %xmm0, %xmm0
-; AVX1-FAST-NEXT:    vmovd %xmm0, %eax
-; AVX1-FAST-NEXT:    vzeroupper
-; AVX1-FAST-NEXT:    retq
-;
-; AVX2-SLOW-LABEL: hadd32_16_optsize:
-; AVX2-SLOW:       # %bb.0:
-; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX2-SLOW-NEXT:    vmovd %xmm0, %eax
-; AVX2-SLOW-NEXT:    vzeroupper
-; AVX2-SLOW-NEXT:    retq
-;
-; AVX2-FAST-LABEL: hadd32_16_optsize:
-; AVX2-FAST:       # %bb.0:
-; AVX2-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-FAST-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX2-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-FAST-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX2-FAST-NEXT:    vmovd %xmm0, %eax
-; AVX2-FAST-NEXT:    vzeroupper
-; AVX2-FAST-NEXT:    retq
-;
-; AVX512-SLOW-LABEL: hadd32_16_optsize:
-; AVX512-SLOW:       # %bb.0:
-; AVX512-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX512-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX512-SLOW-NEXT:    vmovd %xmm0, %eax
-; AVX512-SLOW-NEXT:    vzeroupper
-; AVX512-SLOW-NEXT:    retq
-;
-; AVX512-FAST-LABEL: hadd32_16_optsize:
-; AVX512-FAST:       # %bb.0:
-; AVX512-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-FAST-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX512-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-FAST-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX512-FAST-NEXT:    vmovd %xmm0, %eax
-; AVX512-FAST-NEXT:    vzeroupper
-; AVX512-FAST-NEXT:    retq
+; AVX-LABEL: hadd32_16_optsize:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vphaddd %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vphaddd %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vmovd %xmm0, %eax
+; AVX-NEXT:    vzeroupper
+; AVX-NEXT:    retq
   %x226 = shufflevector <16 x i32> %x225, <16 x i32> undef, <16 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
   %x227 = add <16 x i32> %x225, %x226
   %x228 = shufflevector <16 x i32> %x227, <16 x i32> undef, <16 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
diff --git a/llvm/test/CodeGen/X86/pic-load-remat.ll b/llvm/test/CodeGen/X86/pic-load-remat.ll
index 61849c895..81db54451 100644
--- a/llvm/test/CodeGen/X86/pic-load-remat.ll
+++ b/llvm/test/CodeGen/X86/pic-load-remat.ll
@@ -1,6 +1,23 @@
-; RUN: llc < %s -mtriple=i686-apple-darwin -mattr=+sse2 -relocation-model=pic | grep psllw | grep pb
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-apple-darwin -mattr=+sse2 -relocation-model=pic | FileCheck %s
 
 define void @f() nounwind  {
+; CHECK-LABEL: f:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    calll L0$pb
+; CHECK-NEXT:  L0$pb:
+; CHECK-NEXT:    popl %eax
+; CHECK-NEXT:    pxor %xmm0, %xmm0
+; CHECK-NEXT:    movdqa {{.*#+}} xmm1 = [1,1,1,1,1,1,1,1]
+; CHECK-NEXT:    psllw LCPI0_1-L0$pb(%eax), %xmm1
+; CHECK-NEXT:    pavgw LCPI0_2-L0$pb(%eax), %xmm0
+; CHECK-NEXT:    paddsw %xmm0, %xmm0
+; CHECK-NEXT:    paddw %xmm1, %xmm0
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  LBB0_1: ## %bb
+; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movdqa %xmm0, 0
+; CHECK-NEXT:    jmp LBB0_1
 entry:
 	br label %bb
 
@@ -38,10 +55,10 @@ bb:		; preds = %bb, %entry
 	br label %bb
 }
 
-declare <8 x i16> @llvm.x86.sse2.psll.w(<8 x i16>, <8 x i16>) nounwind readnone 
+declare <8 x i16> @llvm.x86.sse2.psll.w(<8 x i16>, <8 x i16>) nounwind readnone
 
-declare <8 x i16> @llvm.x86.sse2.pavg.w(<8 x i16>, <8 x i16>) nounwind readnone 
+declare <8 x i16> @llvm.x86.sse2.pavg.w(<8 x i16>, <8 x i16>) nounwind readnone
 
-declare <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16>, <8 x i16>) nounwind readnone 
+declare <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16>, <8 x i16>) nounwind readnone
 
-declare <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16>, <8 x i16>) nounwind readnone 
+declare <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16>, <8 x i16>) nounwind readnone
diff --git a/llvm/test/CodeGen/X86/pmaddubsw.ll b/llvm/test/CodeGen/X86/pmaddubsw.ll
index 342242900..48da43c63 100644
--- a/llvm/test/CodeGen/X86/pmaddubsw.ll
+++ b/llvm/test/CodeGen/X86/pmaddubsw.ll
@@ -324,23 +324,23 @@ define <8 x i16> @pmaddubsw_bad_extend(<16 x i8>* %Aptr, <16 x i8>* %Bptr) {
 ; AVX1-NEXT:    vmovdqa (%rsi), %xmm1
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
 ; AVX1-NEXT:    vpshufb %xmm2, %xmm0, %xmm3
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = <1,3,5,7,9,11,13,15,u,u,u,u,u,u,u,u>
+; AVX1-NEXT:    vpshufb %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufb %xmm2, %xmm1, %xmm2
+; AVX1-NEXT:    vpshufb %xmm4, %xmm1, %xmm1
 ; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm4
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
-; AVX1-NEXT:    vpshufb %xmm2, %xmm1, %xmm2
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm5 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
 ; AVX1-NEXT:    vpmulld %xmm5, %xmm4, %xmm4
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,2,3]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
 ; AVX1-NEXT:    vpmulld %xmm2, %xmm3, %xmm2
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = <1,3,5,7,9,11,13,15,u,u,u,u,u,u,u,u>
-; AVX1-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
-; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm5 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm3 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; AVX1-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm3
-; AVX1-NEXT:    vpmulld %xmm3, %xmm5, %xmm3
+; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm5
+; AVX1-NEXT:    vpmulld %xmm5, %xmm3, %xmm3
 ; AVX1-NEXT:    vpaddd %xmm3, %xmm4, %xmm3
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
@@ -349,53 +349,27 @@ define <8 x i16> @pmaddubsw_bad_extend(<16 x i8>* %Aptr, <16 x i8>* %Bptr) {
 ; AVX1-NEXT:    vpackssdw %xmm0, %xmm3, %xmm0
 ; AVX1-NEXT:    retq
 ;
-; AVX2-LABEL: pmaddubsw_bad_extend:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX2-NEXT:    vmovdqa (%rsi), %xmm1
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
-; AVX2-NEXT:    vpshufb %xmm2, %xmm0, %xmm3
-; AVX2-NEXT:    vpmovsxbd %xmm3, %ymm3
-; AVX2-NEXT:    vpshufb %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero
-; AVX2-NEXT:    vpmulld %ymm2, %ymm3, %ymm2
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm3 = <1,3,5,7,9,11,13,15,u,u,u,u,u,u,u,u>
-; AVX2-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
-; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
-; AVX2-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
-; AVX2-NEXT:    vpmovsxbd %xmm1, %ymm1
-; AVX2-NEXT:    vpmulld %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpaddd %ymm0, %ymm2, %ymm0
-; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vzeroupper
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: pmaddubsw_bad_extend:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512-NEXT:    vmovdqa (%rsi), %xmm1
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm2 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
-; AVX512-NEXT:    vpshufb %xmm2, %xmm0, %xmm3
-; AVX512-NEXT:    vpmovsxbd %xmm3, %ymm3
-; AVX512-NEXT:    vpshufb %xmm2, %xmm1, %xmm2
-; AVX512-NEXT:    vpmovzxbd {{.*#+}} ymm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero
-; AVX512-NEXT:    vpmulld %ymm2, %ymm3, %ymm2
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm3 = <1,3,5,7,9,11,13,15,u,u,u,u,u,u,u,u>
-; AVX512-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
-; AVX512-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
-; AVX512-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
-; AVX512-NEXT:    vpmovsxbd %xmm1, %ymm1
-; AVX512-NEXT:    vpmulld %ymm1, %ymm0, %ymm0
-; AVX512-NEXT:    vpaddd %ymm0, %ymm2, %ymm0
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [4294934528,4294934528,4294934528,4294934528,4294934528,4294934528,4294934528,4294934528]
-; AVX512-NEXT:    vpmaxsd %ymm1, %ymm0, %ymm0
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [32767,32767,32767,32767,32767,32767,32767,32767]
-; AVX512-NEXT:    vpminsd %ymm1, %ymm0, %ymm0
-; AVX512-NEXT:    vpmovdw %zmm0, %ymm0
-; AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
-; AVX512-NEXT:    vzeroupper
-; AVX512-NEXT:    retq
+; AVX256-LABEL: pmaddubsw_bad_extend:
+; AVX256:       # %bb.0:
+; AVX256-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX256-NEXT:    vmovdqa (%rsi), %xmm1
+; AVX256-NEXT:    vmovdqa {{.*#+}} xmm2 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
+; AVX256-NEXT:    vpshufb %xmm2, %xmm0, %xmm3
+; AVX256-NEXT:    vmovdqa {{.*#+}} xmm4 = <1,3,5,7,9,11,13,15,u,u,u,u,u,u,u,u>
+; AVX256-NEXT:    vpshufb %xmm4, %xmm0, %xmm0
+; AVX256-NEXT:    vpshufb %xmm2, %xmm1, %xmm2
+; AVX256-NEXT:    vpshufb %xmm4, %xmm1, %xmm1
+; AVX256-NEXT:    vpmovsxbd %xmm3, %ymm3
+; AVX256-NEXT:    vpmovzxbd {{.*#+}} ymm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero
+; AVX256-NEXT:    vpmulld %ymm2, %ymm3, %ymm2
+; AVX256-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
+; AVX256-NEXT:    vpmovsxbd %xmm1, %ymm1
+; AVX256-NEXT:    vpmulld %ymm1, %ymm0, %ymm0
+; AVX256-NEXT:    vpaddd %ymm0, %ymm2, %ymm0
+; AVX256-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX256-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
+; AVX256-NEXT:    vzeroupper
+; AVX256-NEXT:    retq
   %A = load <16 x i8>, <16 x i8>* %Aptr
   %B = load <16 x i8>, <16 x i8>* %Bptr
   %A_even = shufflevector <16 x i8> %A, <16 x i8> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
@@ -452,23 +426,23 @@ define <8 x i16> @pmaddubsw_bad_indices(<16 x i8>* %Aptr, <16 x i8>* %Bptr) {
 ; AVX1-NEXT:    vmovdqa (%rdi), %xmm0
 ; AVX1-NEXT:    vmovdqa (%rsi), %xmm1
 ; AVX1-NEXT:    vpshufb {{.*#+}} xmm2 = xmm0[1,2,5,6,9,10,13,14,u,u,u,u,u,u,u,u]
-; AVX1-NEXT:    vpmovsxbd %xmm2, %xmm3
+; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,3,4,7,8,11,12,15,u,u,u,u,u,u,u,u]
+; AVX1-NEXT:    vpshufb {{.*#+}} xmm3 = xmm1[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
+; AVX1-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[1,3,5,7,9,11,13,15,u,u,u,u,u,u,u,u]
+; AVX1-NEXT:    vpmovsxbd %xmm2, %xmm4
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm2, %xmm2
-; AVX1-NEXT:    vpshufb {{.*#+}} xmm4 = xmm1[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
-; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm5 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero
-; AVX1-NEXT:    vpmulld %xmm5, %xmm3, %xmm3
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[1,1,2,3]
-; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero
-; AVX1-NEXT:    vpmulld %xmm4, %xmm2, %xmm2
-; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,3,4,7,8,11,12,15,u,u,u,u,u,u,u,u]
-; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm4
+; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm5 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
+; AVX1-NEXT:    vpmulld %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,2,3]
+; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
+; AVX1-NEXT:    vpmulld %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm3
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm0
-; AVX1-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[1,3,5,7,9,11,13,15,u,u,u,u,u,u,u,u]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm5 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
-; AVX1-NEXT:    vpmulld %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vpaddd %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpmulld %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vpaddd %xmm3, %xmm4, %xmm3
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
@@ -476,49 +450,25 @@ define <8 x i16> @pmaddubsw_bad_indices(<16 x i8>* %Aptr, <16 x i8>* %Bptr) {
 ; AVX1-NEXT:    vpackssdw %xmm0, %xmm3, %xmm0
 ; AVX1-NEXT:    retq
 ;
-; AVX2-LABEL: pmaddubsw_bad_indices:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX2-NEXT:    vmovdqa (%rsi), %xmm1
-; AVX2-NEXT:    vpshufb {{.*#+}} xmm2 = xmm0[1,2,5,6,9,10,13,14,u,u,u,u,u,u,u,u]
-; AVX2-NEXT:    vpmovsxbd %xmm2, %ymm2
-; AVX2-NEXT:    vpshufb {{.*#+}} xmm3 = xmm1[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
-; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero,xmm3[4],zero,zero,zero,xmm3[5],zero,zero,zero,xmm3[6],zero,zero,zero,xmm3[7],zero,zero,zero
-; AVX2-NEXT:    vpmulld %ymm3, %ymm2, %ymm2
-; AVX2-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,3,4,7,8,11,12,15,u,u,u,u,u,u,u,u]
-; AVX2-NEXT:    vpmovsxbd %xmm0, %ymm0
-; AVX2-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[1,3,5,7,9,11,13,15,u,u,u,u,u,u,u,u]
-; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero
-; AVX2-NEXT:    vpmulld %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpaddd %ymm0, %ymm2, %ymm0
-; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vzeroupper
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: pmaddubsw_bad_indices:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512-NEXT:    vmovdqa (%rsi), %xmm1
-; AVX512-NEXT:    vpshufb {{.*#+}} xmm2 = xmm0[1,2,5,6,9,10,13,14,u,u,u,u,u,u,u,u]
-; AVX512-NEXT:    vpmovsxbd %xmm2, %ymm2
-; AVX512-NEXT:    vpshufb {{.*#+}} xmm3 = xmm1[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
-; AVX512-NEXT:    vpmovzxbd {{.*#+}} ymm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero,xmm3[4],zero,zero,zero,xmm3[5],zero,zero,zero,xmm3[6],zero,zero,zero,xmm3[7],zero,zero,zero
-; AVX512-NEXT:    vpmulld %ymm3, %ymm2, %ymm2
-; AVX512-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,3,4,7,8,11,12,15,u,u,u,u,u,u,u,u]
-; AVX512-NEXT:    vpmovsxbd %xmm0, %ymm0
-; AVX512-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[1,3,5,7,9,11,13,15,u,u,u,u,u,u,u,u]
-; AVX512-NEXT:    vpmovzxbd {{.*#+}} ymm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero
-; AVX512-NEXT:    vpmulld %ymm1, %ymm0, %ymm0
-; AVX512-NEXT:    vpaddd %ymm0, %ymm2, %ymm0
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [4294934528,4294934528,4294934528,4294934528,4294934528,4294934528,4294934528,4294934528]
-; AVX512-NEXT:    vpmaxsd %ymm1, %ymm0, %ymm0
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [32767,32767,32767,32767,32767,32767,32767,32767]
-; AVX512-NEXT:    vpminsd %ymm1, %ymm0, %ymm0
-; AVX512-NEXT:    vpmovdw %zmm0, %ymm0
-; AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
-; AVX512-NEXT:    vzeroupper
-; AVX512-NEXT:    retq
+; AVX256-LABEL: pmaddubsw_bad_indices:
+; AVX256:       # %bb.0:
+; AVX256-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX256-NEXT:    vmovdqa (%rsi), %xmm1
+; AVX256-NEXT:    vpshufb {{.*#+}} xmm2 = xmm0[1,2,5,6,9,10,13,14,u,u,u,u,u,u,u,u]
+; AVX256-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,3,4,7,8,11,12,15,u,u,u,u,u,u,u,u]
+; AVX256-NEXT:    vpshufb {{.*#+}} xmm3 = xmm1[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
+; AVX256-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[1,3,5,7,9,11,13,15,u,u,u,u,u,u,u,u]
+; AVX256-NEXT:    vpmovsxbd %xmm2, %ymm2
+; AVX256-NEXT:    vpmovzxbd {{.*#+}} ymm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero,xmm3[4],zero,zero,zero,xmm3[5],zero,zero,zero,xmm3[6],zero,zero,zero,xmm3[7],zero,zero,zero
+; AVX256-NEXT:    vpmulld %ymm3, %ymm2, %ymm2
+; AVX256-NEXT:    vpmovsxbd %xmm0, %ymm0
+; AVX256-NEXT:    vpmovzxbd {{.*#+}} ymm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero
+; AVX256-NEXT:    vpmulld %ymm1, %ymm0, %ymm0
+; AVX256-NEXT:    vpaddd %ymm0, %ymm2, %ymm0
+; AVX256-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX256-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
+; AVX256-NEXT:    vzeroupper
+; AVX256-NEXT:    retq
   %A = load <16 x i8>, <16 x i8>* %Aptr
   %B = load <16 x i8>, <16 x i8>* %Bptr
   %A_even = shufflevector <16 x i8> %A, <16 x i8> undef, <8 x i32> <i32 1, i32 2, i32 5, i32 6, i32 9, i32 10, i32 13, i32 14> ;indices aren't all even
diff --git a/llvm/test/CodeGen/X86/pmul.ll b/llvm/test/CodeGen/X86/pmul.ll
index 8e6b75fa6..3b8563143 100644
--- a/llvm/test/CodeGen/X86/pmul.ll
+++ b/llvm/test/CodeGen/X86/pmul.ll
@@ -818,22 +818,24 @@ define <64 x i8> @mul_v64i8c(<64 x i8> %i) nounwind  {
 ;
 ; AVX512F-LABEL: mul_v64i8c:
 ; AVX512F:       # %bb.0: # %entry
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31]
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm3 = [117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117]
 ; AVX512F-NEXT:    vpmullw %ymm3, %ymm2, %ymm2
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
 ; AVX512F-NEXT:    vpand %ymm4, %ymm2, %ymm2
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512F-NEXT:    vpmullw %ymm3, %ymm0, %ymm0
-; AVX512F-NEXT:    vpand %ymm4, %ymm0, %ymm0
-; AVX512F-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31]
-; AVX512F-NEXT:    vpmullw %ymm3, %ymm2, %ymm2
-; AVX512F-NEXT:    vpand %ymm4, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
 ; AVX512F-NEXT:    vpmullw %ymm3, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpand %ymm4, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpackuswb %ymm2, %ymm1, %ymm1
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX512F-NEXT:    vpmullw %ymm3, %ymm2, %ymm2
+; AVX512F-NEXT:    vpand %ymm4, %ymm2, %ymm2
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512F-NEXT:    vpmullw %ymm3, %ymm0, %ymm0
+; AVX512F-NEXT:    vpand %ymm4, %ymm0, %ymm0
+; AVX512F-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: mul_v64i8c:
@@ -973,25 +975,28 @@ define <64 x i8> @mul_v64i8(<64 x i8> %i, <64 x i8> %j) nounwind  {
 ;
 ; AVX512F-LABEL: mul_v64i8:
 ; AVX512F:       # %bb.0: # %entry
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11],ymm2[12],ymm0[12],ymm2[13],ymm0[13],ymm2[14],ymm0[14],ymm2[15],ymm0[15],ymm2[24],ymm0[24],ymm2[25],ymm0[25],ymm2[26],ymm0[26],ymm2[27],ymm0[27],ymm2[28],ymm0[28],ymm2[29],ymm0[29],ymm2[30],ymm0[30],ymm2[31],ymm0[31]
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm3[8],ymm0[8],ymm3[9],ymm0[9],ymm3[10],ymm0[10],ymm3[11],ymm0[11],ymm3[12],ymm0[12],ymm3[13],ymm0[13],ymm3[14],ymm0[14],ymm3[15],ymm0[15],ymm3[24],ymm0[24],ymm3[25],ymm0[25],ymm3[26],ymm0[26],ymm3[27],ymm0[27],ymm3[28],ymm0[28],ymm3[29],ymm0[29],ymm3[30],ymm0[30],ymm3[31],ymm0[31]
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11],ymm2[12],ymm0[12],ymm2[13],ymm0[13],ymm2[14],ymm0[14],ymm2[15],ymm0[15],ymm2[24],ymm0[24],ymm2[25],ymm0[25],ymm2[26],ymm0[26],ymm2[27],ymm0[27],ymm2[28],ymm0[28],ymm2[29],ymm0[29],ymm2[30],ymm0[30],ymm2[31],ymm0[31]
 ; AVX512F-NEXT:    vpmullw %ymm4, %ymm5, %ymm4
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm5 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
 ; AVX512F-NEXT:    vpand %ymm5, %ymm4, %ymm4
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23]
 ; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512F-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
-; AVX512F-NEXT:    vpand %ymm5, %ymm0, %ymm0
-; AVX512F-NEXT:    vpackuswb %ymm4, %ymm0, %ymm0
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm3[8],ymm0[8],ymm3[9],ymm0[9],ymm3[10],ymm0[10],ymm3[11],ymm0[11],ymm3[12],ymm0[12],ymm3[13],ymm0[13],ymm3[14],ymm0[14],ymm3[15],ymm0[15],ymm3[24],ymm0[24],ymm3[25],ymm0[25],ymm3[26],ymm0[26],ymm3[27],ymm0[27],ymm3[28],ymm0[28],ymm3[29],ymm0[29],ymm3[30],ymm0[30],ymm3[31],ymm0[31]
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31]
-; AVX512F-NEXT:    vpmullw %ymm2, %ymm4, %ymm2
+; AVX512F-NEXT:    vpmullw %ymm3, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpand %ymm5, %ymm2, %ymm2
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23]
+; AVX512F-NEXT:    vpackuswb %ymm4, %ymm2, %ymm2
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31]
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX512F-NEXT:    vpmullw %ymm3, %ymm4, %ymm3
+; AVX512F-NEXT:    vpand %ymm5, %ymm3, %ymm3
 ; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
-; AVX512F-NEXT:    vpmullw %ymm3, %ymm1, %ymm1
-; AVX512F-NEXT:    vpand %ymm5, %ymm1, %ymm1
-; AVX512F-NEXT:    vpackuswb %ymm2, %ymm1, %ymm1
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512F-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vpand %ymm5, %ymm0, %ymm0
+; AVX512F-NEXT:    vpackuswb %ymm3, %ymm0, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: mul_v64i8:
@@ -1119,10 +1124,8 @@ entry:
 define <4 x i32> @mul_v4i64_zero_lower(<4 x i32> %val1, <4 x i64> %val2) {
 ; SSE2-LABEL: mul_v4i64_zero_lower:
 ; SSE2:       # %bb.0: # %entry
-; SSE2-NEXT:    pxor %xmm4, %xmm4
-; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
-; SSE2-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm4[2],xmm0[3],xmm4[3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[0,1,1,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,1,3,3]
 ; SSE2-NEXT:    psrlq $32, %xmm2
 ; SSE2-NEXT:    pmuludq %xmm0, %xmm2
 ; SSE2-NEXT:    psrlq $32, %xmm1
@@ -1133,14 +1136,13 @@ define <4 x i32> @mul_v4i64_zero_lower(<4 x i32> %val1, <4 x i64> %val2) {
 ;
 ; SSE41-LABEL: mul_v4i64_zero_lower:
 ; SSE41:       # %bb.0: # %entry
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,0,1]
-; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero
-; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
+; SSE41-NEXT:    psrlq $32, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,2,3,3]
+; SSE41-NEXT:    pmuludq %xmm2, %xmm3
 ; SSE41-NEXT:    psrlq $32, %xmm1
+; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
 ; SSE41-NEXT:    pmuludq %xmm1, %xmm0
-; SSE41-NEXT:    psrlq $32, %xmm2
-; SSE41-NEXT:    pmuludq %xmm3, %xmm2
-; SSE41-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm2[0,2]
+; SSE41-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm3[0,2]
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: mul_v4i64_zero_lower:
diff --git a/llvm/test/CodeGen/X86/pmulh.ll b/llvm/test/CodeGen/X86/pmulh.ll
index 706865011..7281e4c2c 100644
--- a/llvm/test/CodeGen/X86/pmulh.ll
+++ b/llvm/test/CodeGen/X86/pmulh.ll
@@ -1,52 +1,19 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s --check-prefix=SSE --check-prefix=SSE2 --check-prefix=SSE2-PROMOTE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -x86-experimental-vector-widening-legalization | FileCheck %s --check-prefix=SSE --check-prefix=SSE2 --check-prefix=SSE2-WIDEN
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=SSE --check-prefix=SSE41 --check-prefix=SSE41-PROMOTE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 -x86-experimental-vector-widening-legalization | FileCheck %s --check-prefix=SSE --check-prefix=SSE41 --check-prefix=SSE41-WIDEN
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=AVX --check-prefix=AVX2
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefix=AVX --check-prefix=AVX512 --check-prefix=AVX512F
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw | FileCheck %s --check-prefix=AVX --check-prefix=AVX512 --check-prefix=AVX512BW
 
 define <4 x i16> @mulhuw_v4i16(<4 x i16> %a, <4 x i16> %b) {
-; SSE2-PROMOTE-LABEL: mulhuw_v4i16:
-; SSE2-PROMOTE:       # %bb.0:
-; SSE2-PROMOTE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
-; SSE2-PROMOTE-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,6,6,7]
-; SSE2-PROMOTE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; SSE2-PROMOTE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
-; SSE2-PROMOTE-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,6,6,7]
-; SSE2-PROMOTE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; SSE2-PROMOTE-NEXT:    pmulhuw %xmm1, %xmm0
-; SSE2-PROMOTE-NEXT:    pxor %xmm1, %xmm1
-; SSE2-PROMOTE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; SSE2-PROMOTE-NEXT:    retq
-;
-; SSE2-WIDEN-LABEL: mulhuw_v4i16:
-; SSE2-WIDEN:       # %bb.0:
-; SSE2-WIDEN-NEXT:    pmulhuw %xmm1, %xmm0
-; SSE2-WIDEN-NEXT:    retq
-;
-; SSE41-PROMOTE-LABEL: mulhuw_v4i16:
-; SSE41-PROMOTE:       # %bb.0:
-; SSE41-PROMOTE-NEXT:    pxor %xmm2, %xmm2
-; SSE41-PROMOTE-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3],xmm0[4],xmm2[5],xmm0[6],xmm2[7]
-; SSE41-PROMOTE-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
-; SSE41-PROMOTE-NEXT:    pmulld %xmm1, %xmm0
-; SSE41-PROMOTE-NEXT:    psrld $16, %xmm0
-; SSE41-PROMOTE-NEXT:    retq
-;
-; SSE41-WIDEN-LABEL: mulhuw_v4i16:
-; SSE41-WIDEN:       # %bb.0:
-; SSE41-WIDEN-NEXT:    pmulhuw %xmm1, %xmm0
-; SSE41-WIDEN-NEXT:    retq
+; SSE-LABEL: mulhuw_v4i16:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pmulhuw %xmm1, %xmm0
+; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: mulhuw_v4i16:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3],xmm0[4],xmm2[5],xmm0[6],xmm2[7]
-; AVX-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
-; AVX-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vpsrld $16, %xmm0, %xmm0
+; AVX-NEXT:    vpmulhuw %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    retq
   %a1 = zext <4 x i16> %a to <4 x i32>
   %b1 = zext <4 x i16> %b to <4 x i32>
@@ -57,47 +24,14 @@ define <4 x i16> @mulhuw_v4i16(<4 x i16> %a, <4 x i16> %b) {
 }
 
 define <4 x i16> @mulhw_v4i16(<4 x i16> %a, <4 x i16> %b) {
-; SSE2-PROMOTE-LABEL: mulhw_v4i16:
-; SSE2-PROMOTE:       # %bb.0:
-; SSE2-PROMOTE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
-; SSE2-PROMOTE-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,6,6,7]
-; SSE2-PROMOTE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; SSE2-PROMOTE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
-; SSE2-PROMOTE-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,6,6,7]
-; SSE2-PROMOTE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; SSE2-PROMOTE-NEXT:    pmulhw %xmm1, %xmm0
-; SSE2-PROMOTE-NEXT:    pxor %xmm1, %xmm1
-; SSE2-PROMOTE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; SSE2-PROMOTE-NEXT:    retq
-;
-; SSE2-WIDEN-LABEL: mulhw_v4i16:
-; SSE2-WIDEN:       # %bb.0:
-; SSE2-WIDEN-NEXT:    pmulhw %xmm1, %xmm0
-; SSE2-WIDEN-NEXT:    retq
-;
-; SSE41-PROMOTE-LABEL: mulhw_v4i16:
-; SSE41-PROMOTE:       # %bb.0:
-; SSE41-PROMOTE-NEXT:    pslld $16, %xmm0
-; SSE41-PROMOTE-NEXT:    psrad $16, %xmm0
-; SSE41-PROMOTE-NEXT:    pslld $16, %xmm1
-; SSE41-PROMOTE-NEXT:    psrad $16, %xmm1
-; SSE41-PROMOTE-NEXT:    pmulld %xmm1, %xmm0
-; SSE41-PROMOTE-NEXT:    psrld $16, %xmm0
-; SSE41-PROMOTE-NEXT:    retq
-;
-; SSE41-WIDEN-LABEL: mulhw_v4i16:
-; SSE41-WIDEN:       # %bb.0:
-; SSE41-WIDEN-NEXT:    pmulhw %xmm1, %xmm0
-; SSE41-WIDEN-NEXT:    retq
+; SSE-LABEL: mulhw_v4i16:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pmulhw %xmm1, %xmm0
+; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: mulhw_v4i16:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpslld $16, %xmm0, %xmm0
-; AVX-NEXT:    vpsrad $16, %xmm0, %xmm0
-; AVX-NEXT:    vpslld $16, %xmm1, %xmm1
-; AVX-NEXT:    vpsrad $16, %xmm1, %xmm1
-; AVX-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vpsrld $16, %xmm0, %xmm0
+; AVX-NEXT:    vpmulhw %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    retq
   %a1 = sext <4 x i16> %a to <4 x i32>
   %b1 = sext <4 x i16> %b to <4 x i32>
@@ -198,8 +132,11 @@ define <32 x i16> @mulhuw_v32i16(<32 x i16> %a, <32 x i16> %b) {
 ;
 ; AVX512F-LABEL: mulhuw_v32i16:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpmulhuw %ymm2, %ymm0, %ymm0
-; AVX512F-NEXT:    vpmulhuw %ymm3, %ymm1, %ymm1
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
+; AVX512F-NEXT:    vpmulhuw %ymm2, %ymm3, %ymm2
+; AVX512F-NEXT:    vpmulhuw %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: mulhuw_v32i16:
@@ -231,8 +168,11 @@ define <32 x i16> @mulhw_v32i16(<32 x i16> %a, <32 x i16> %b) {
 ;
 ; AVX512F-LABEL: mulhw_v32i16:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpmulhw %ymm2, %ymm0, %ymm0
-; AVX512F-NEXT:    vpmulhw %ymm3, %ymm1, %ymm1
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
+; AVX512F-NEXT:    vpmulhw %ymm2, %ymm3, %ymm2
+; AVX512F-NEXT:    vpmulhw %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: mulhw_v32i16:
diff --git a/llvm/test/CodeGen/X86/pointer-vector.ll b/llvm/test/CodeGen/X86/pointer-vector.ll
index 739e66c7b..67b389d2e 100644
--- a/llvm/test/CodeGen/X86/pointer-vector.ll
+++ b/llvm/test/CodeGen/X86/pointer-vector.ll
@@ -117,7 +117,7 @@ define <2 x i32*> @BITCAST1(<2 x i8*>* %p) nounwind {
 ; CHECK-LABEL: BITCAST1:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; CHECK-NEXT:    pmovzxdq {{.*#+}} xmm0 = mem[0],zero,mem[1],zero
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; CHECK-NEXT:    retl
 entry:
   %G = load <2 x i8*>, <2 x i8*>* %p
diff --git a/llvm/test/CodeGen/X86/postalloc-coalescing.ll b/llvm/test/CodeGen/X86/postalloc-coalescing.ll
index 83c435e5e..7f190a342 100644
--- a/llvm/test/CodeGen/X86/postalloc-coalescing.ll
+++ b/llvm/test/CodeGen/X86/postalloc-coalescing.ll
@@ -1,6 +1,23 @@
-; RUN: llc < %s -mtriple=i686-- | grep mov | count 3
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- | FileCheck %s
 
 define fastcc i32 @_Z18yy_get_next_bufferv() nounwind {
+; CHECK-LABEL: _Z18yy_get_next_bufferv:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movl $42, %eax
+; CHECK-NEXT:    cmpl $-1, %eax
+; CHECK-NEXT:    je .LBB0_3
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB0_1: # %bb116
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movb %al, 0
+; CHECK-NEXT:    cmpl $-1, %eax
+; CHECK-NEXT:    jne .LBB0_1
+; CHECK-NEXT:  .LBB0_3: # %bb158
+; CHECK-NEXT:    movb %al, 0
+; CHECK-NEXT:    cmpl $-1, %eax
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    retl
 entry:
 	br label %bb131
 
diff --git a/llvm/test/CodeGen/X86/powi-windows.ll b/llvm/test/CodeGen/X86/powi-windows.ll
new file mode 100644
index 000000000..804071ba1
--- /dev/null
+++ b/llvm/test/CodeGen/X86/powi-windows.ll
@@ -0,0 +1,46 @@
+; RUN: llc -mtriple x86_64-windows < %s | FileCheck %s
+
+declare double @llvm.powi.f64(double, i32)
+declare float @llvm.powi.f32(float, i32)
+
+define double @d(double %d, i32 %i) {
+entry:
+  %0 = tail call double @llvm.powi.f64(double %d, i32 %i)
+  ret double %0
+}
+
+; CHECK-LABEL: d:
+; CHECK: cvtsi2sd %edx, %xmm1
+; CHECK-NEXT: jmp pow
+
+define float @f(float %f, i32 %i) {
+entry:
+  %0 = tail call float @llvm.powi.f32(float %f, i32 %i)
+  ret float %0
+}
+
+; CHECK-LABEL: f:
+; CHECK: cvtsi2ss %edx, %xmm1
+; CHECK-NEXT: jmp powf
+
+define float @g(double %d, i32 %i) {
+entry:
+  %0 = tail call double @llvm.powi.f64(double %d, i32 %i)
+  %conv = fptrunc double %0 to float
+  ret float %conv
+}
+
+; CHECK-LABEL: g:
+; CHECK: cvtsi2sd %edx, %xmm1
+; CHECK-NEXT: callq pow
+
+define double @h(float %f, i32 %i) {
+entry:
+  %0 = tail call float @llvm.powi.f32(float %f, i32 %i)
+  %conv = fpext float %0 to double
+  ret double %conv
+}
+
+; CHECK-LABEL: h:
+; CHECK: cvtsi2ss %edx, %xmm1
+; CHECK-NEXT: callq powf
diff --git a/llvm/test/CodeGen/X86/powi.ll b/llvm/test/CodeGen/X86/powi.ll
index fe2cfb0bd..7a5282337 100644
--- a/llvm/test/CodeGen/X86/powi.ll
+++ b/llvm/test/CodeGen/X86/powi.ll
@@ -86,6 +86,39 @@ define double @pow_wrapper_optsize(double %a) optsize {
   ret double %ret
 }
 
+define double @pow_wrapper_pgso(double %a) !prof !14 {
+; X86-X87-LABEL: pow_wrapper_pgso:
+; X86-X87:       # %bb.0:
+; X86-X87-NEXT:    subl $12, %esp
+; X86-X87-NEXT:    .cfi_def_cfa_offset 16
+; X86-X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X86-X87-NEXT:    fstpl (%esp)
+; X86-X87-NEXT:    movl $15, {{[0-9]+}}(%esp)
+; X86-X87-NEXT:    calll __powidf2
+; X86-X87-NEXT:    addl $12, %esp
+; X86-X87-NEXT:    .cfi_def_cfa_offset 4
+; X86-X87-NEXT:    retl
+;
+; X86-SSE-LABEL: pow_wrapper_pgso:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    subl $12, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 16
+; X86-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE-NEXT:    movsd %xmm0, (%esp)
+; X86-SSE-NEXT:    movl $15, {{[0-9]+}}(%esp)
+; X86-SSE-NEXT:    calll __powidf2
+; X86-SSE-NEXT:    addl $12, %esp
+; X86-SSE-NEXT:    .cfi_def_cfa_offset 4
+; X86-SSE-NEXT:    retl
+;
+; X64-LABEL: pow_wrapper_pgso:
+; X64:       # %bb.0:
+; X64-NEXT:    movl $15, %edi
+; X64-NEXT:    jmp __powidf2 # TAILCALL
+  %ret = tail call double @llvm.powi.f64(double %a, i32 15) nounwind ; <double> [#uses=1]
+  ret double %ret
+}
+
 define double @pow_wrapper_minsize(double %a) minsize {
 ; X86-X87-LABEL: pow_wrapper_minsize:
 ; X86-X87:       # %bb.0:
@@ -124,3 +157,19 @@ define double @pow_wrapper_minsize(double %a) minsize {
 
 declare double @llvm.powi.f64(double, i32) nounwind readonly
 
+!llvm.module.flags = !{!0}
+!0 = !{i32 1, !"ProfileSummary", !1}
+!1 = !{!2, !3, !4, !5, !6, !7, !8, !9}
+!2 = !{!"ProfileFormat", !"InstrProf"}
+!3 = !{!"TotalCount", i64 10000}
+!4 = !{!"MaxCount", i64 10}
+!5 = !{!"MaxInternalCount", i64 1}
+!6 = !{!"MaxFunctionCount", i64 1000}
+!7 = !{!"NumCounts", i64 3}
+!8 = !{!"NumFunctions", i64 3}
+!9 = !{!"DetailedSummary", !10}
+!10 = !{!11, !12, !13}
+!11 = !{i32 10000, i64 100, i32 1}
+!12 = !{i32 999000, i64 100, i32 1}
+!13 = !{i32 999999, i64 1, i32 2}
+!14 = !{!"function_entry_count", i64 0}
diff --git a/llvm/test/CodeGen/X86/pr14161.ll b/llvm/test/CodeGen/X86/pr14161.ll
index ef8cd918f..152f3d350 100644
--- a/llvm/test/CodeGen/X86/pr14161.ll
+++ b/llvm/test/CodeGen/X86/pr14161.ll
@@ -7,7 +7,6 @@ define <2 x i16> @good(<4 x i32>*, <4 x i8>*) {
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    movdqa (%rdi), %xmm0
 ; CHECK-NEXT:    pminud {{.*}}(%rip), %xmm0
-; CHECK-NEXT:    pmovzxwq %xmm0, %xmm0
 ; CHECK-NEXT:    retq
 entry:
   %2 = load <4 x i32>, <4 x i32>* %0, align 16
@@ -27,7 +26,6 @@ define <2 x i16> @bad(<4 x i32>*, <4 x i8>*) {
 ; CHECK-NEXT:    movdqa (%rdi), %xmm0
 ; CHECK-NEXT:    pminud {{.*}}(%rip), %xmm0
 ; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; CHECK-NEXT:    pmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
 ; CHECK-NEXT:    retq
 entry:
   %2 = load <4 x i32>, <4 x i32>* %0, align 16
diff --git a/llvm/test/CodeGen/X86/pr1489.ll b/llvm/test/CodeGen/X86/pr1489.ll
index 36d30e8be..6226ea6ca 100644
--- a/llvm/test/CodeGen/X86/pr1489.ll
+++ b/llvm/test/CodeGen/X86/pr1489.ll
@@ -1,5 +1,5 @@
-; RUN: llc < %s -frame-pointer=all -O0 -mcpu=i486 | grep 1082126238 | count 3
-; RUN: llc < %s -frame-pointer=all -O0 -mcpu=i486 | grep -- -1236950581 | count 1
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -frame-pointer=all -O0 -mcpu=i486 | FileCheck %s
 ;; magic constants are 3.999f and half of 3.999
 ; ModuleID = '1489.c'
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64"
@@ -7,6 +7,21 @@ target triple = "i686-apple-darwin8"
 @.str = internal constant [13 x i8] c"%d %d %d %d\0A\00"		; <[13 x i8]*> [#uses=1]
 
 define i32 @quux() nounwind {
+; CHECK-LABEL: quux:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    pushl %ebp
+; CHECK-NEXT:    movl %esp, %ebp
+; CHECK-NEXT:    subl $8, %esp
+; CHECK-NEXT:    movl %esp, %eax
+; CHECK-NEXT:    movl $1082126238, (%eax) ## imm = 0x407FEF9E
+; CHECK-NEXT:    calll _lrintf
+; CHECK-NEXT:    cmpl $1, %eax
+; CHECK-NEXT:    setl %cl
+; CHECK-NEXT:    andb $1, %cl
+; CHECK-NEXT:    movzbl %cl, %eax
+; CHECK-NEXT:    addl $8, %esp
+; CHECK-NEXT:    popl %ebp
+; CHECK-NEXT:    retl
 entry:
 	%tmp1 = tail call i32 @lrintf( float 0x400FFDF3C0000000 )		; <i32> [#uses=1]
 	%tmp2 = icmp slt i32 %tmp1, 1		; <i1> [#uses=1]
@@ -17,6 +32,22 @@ entry:
 declare i32 @lrintf(float)
 
 define i32 @foo() nounwind {
+; CHECK-LABEL: foo:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    pushl %ebp
+; CHECK-NEXT:    movl %esp, %ebp
+; CHECK-NEXT:    subl $8, %esp
+; CHECK-NEXT:    movl %esp, %eax
+; CHECK-NEXT:    movl $1074789875, 4(%eax) ## imm = 0x400FFDF3
+; CHECK-NEXT:    movl $-1236950581, (%eax) ## imm = 0xB645A1CB
+; CHECK-NEXT:    calll _lrint
+; CHECK-NEXT:    cmpl $1, %eax
+; CHECK-NEXT:    setl %cl
+; CHECK-NEXT:    andb $1, %cl
+; CHECK-NEXT:    movzbl %cl, %eax
+; CHECK-NEXT:    addl $8, %esp
+; CHECK-NEXT:    popl %ebp
+; CHECK-NEXT:    retl
 entry:
 	%tmp1 = tail call i32 @lrint( double 3.999000e+00 )		; <i32> [#uses=1]
 	%tmp2 = icmp slt i32 %tmp1, 1		; <i1> [#uses=1]
@@ -27,6 +58,21 @@ entry:
 declare i32 @lrint(double)
 
 define i32 @bar() nounwind {
+; CHECK-LABEL: bar:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    pushl %ebp
+; CHECK-NEXT:    movl %esp, %ebp
+; CHECK-NEXT:    subl $8, %esp
+; CHECK-NEXT:    movl %esp, %eax
+; CHECK-NEXT:    movl $1082126238, (%eax) ## imm = 0x407FEF9E
+; CHECK-NEXT:    calll _lrintf
+; CHECK-NEXT:    cmpl $1, %eax
+; CHECK-NEXT:    setl %cl
+; CHECK-NEXT:    andb $1, %cl
+; CHECK-NEXT:    movzbl %cl, %eax
+; CHECK-NEXT:    addl $8, %esp
+; CHECK-NEXT:    popl %ebp
+; CHECK-NEXT:    retl
 entry:
 	%tmp1 = tail call i32 @lrintf( float 0x400FFDF3C0000000 )		; <i32> [#uses=1]
 	%tmp2 = icmp slt i32 %tmp1, 1		; <i1> [#uses=1]
@@ -35,6 +81,21 @@ entry:
 }
 
 define i32 @baz() nounwind {
+; CHECK-LABEL: baz:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    pushl %ebp
+; CHECK-NEXT:    movl %esp, %ebp
+; CHECK-NEXT:    subl $8, %esp
+; CHECK-NEXT:    movl %esp, %eax
+; CHECK-NEXT:    movl $1082126238, (%eax) ## imm = 0x407FEF9E
+; CHECK-NEXT:    calll _lrintf
+; CHECK-NEXT:    cmpl $1, %eax
+; CHECK-NEXT:    setl %cl
+; CHECK-NEXT:    andb $1, %cl
+; CHECK-NEXT:    movzbl %cl, %eax
+; CHECK-NEXT:    addl $8, %esp
+; CHECK-NEXT:    popl %ebp
+; CHECK-NEXT:    retl
 entry:
 	%tmp1 = tail call i32 @lrintf( float 0x400FFDF3C0000000 )		; <i32> [#uses=1]
 	%tmp2 = icmp slt i32 %tmp1, 1		; <i1> [#uses=1]
@@ -43,6 +104,38 @@ entry:
 }
 
 define i32 @main() nounwind {
+; CHECK-LABEL: main:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    pushl %ebp
+; CHECK-NEXT:    movl %esp, %ebp
+; CHECK-NEXT:    pushl %edi
+; CHECK-NEXT:    pushl %esi
+; CHECK-NEXT:    subl $48, %esp
+; CHECK-NEXT:    calll _baz
+; CHECK-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) ## 4-byte Spill
+; CHECK-NEXT:    calll _bar
+; CHECK-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) ## 4-byte Spill
+; CHECK-NEXT:    calll _foo
+; CHECK-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) ## 4-byte Spill
+; CHECK-NEXT:    calll _quux
+; CHECK-NEXT:    movl %esp, %ecx
+; CHECK-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx ## 4-byte Reload
+; CHECK-NEXT:    movl %edx, 16(%ecx)
+; CHECK-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi ## 4-byte Reload
+; CHECK-NEXT:    movl %esi, 12(%ecx)
+; CHECK-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi ## 4-byte Reload
+; CHECK-NEXT:    movl %edi, 8(%ecx)
+; CHECK-NEXT:    movl %eax, 4(%ecx)
+; CHECK-NEXT:    movl $_.str, (%ecx)
+; CHECK-NEXT:    calll _printf
+; CHECK-NEXT:    ## implicit-def: $ecx
+; CHECK-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) ## 4-byte Spill
+; CHECK-NEXT:    movl %ecx, %eax
+; CHECK-NEXT:    addl $48, %esp
+; CHECK-NEXT:    popl %esi
+; CHECK-NEXT:    popl %edi
+; CHECK-NEXT:    popl %ebp
+; CHECK-NEXT:    retl
 entry:
 	%tmp = tail call i32 @baz( )		; <i32> [#uses=1]
 	%tmp1 = tail call i32 @bar( )		; <i32> [#uses=1]
diff --git a/llvm/test/CodeGen/X86/pr1505.ll b/llvm/test/CodeGen/X86/pr1505.ll
index 883a806f3..2611b6c49 100644
--- a/llvm/test/CodeGen/X86/pr1505.ll
+++ b/llvm/test/CodeGen/X86/pr1505.ll
@@ -1,4 +1,5 @@
-; RUN: llc < %s -mcpu=i486 | not grep fldl
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mcpu=i486 | FileCheck %s
 ; PR1505
 
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64"
@@ -6,6 +7,12 @@ target triple = "i686-apple-darwin8"
 @G = weak global float 0.000000e+00		; <float*> [#uses=1]
 
 define void @t1(float %F) {
+; CHECK-LABEL: t1:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    flds {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movl L_G$non_lazy_ptr, %eax
+; CHECK-NEXT:    fstps (%eax)
+; CHECK-NEXT:    retl
 entry:
 	store float %F, float* @G
 	ret void
diff --git a/llvm/test/CodeGen/X86/pr15309.ll b/llvm/test/CodeGen/X86/pr15309.ll
index 871735337..e154af43b 100644
--- a/llvm/test/CodeGen/X86/pr15309.ll
+++ b/llvm/test/CodeGen/X86/pr15309.ll
@@ -17,16 +17,12 @@ define void @test_convert_float2_ulong2(<2 x i64>* nocapture %src, <2 x float>*
 ; CHECK-NEXT:    movl %edi, (%esp)
 ; CHECK-NEXT:    movl %esi, {{[0-9]+}}(%esp)
 ; CHECK-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; CHECK-NEXT:    xorl %edx, %edx
-; CHECK-NEXT:    testl %ecx, %ecx
-; CHECK-NEXT:    setns %dl
+; CHECK-NEXT:    shrl $31, %ecx
 ; CHECK-NEXT:    fildll (%esp)
-; CHECK-NEXT:    fadds {{\.LCPI.*}}(,%edx,4)
-; CHECK-NEXT:    xorl %ecx, %ecx
-; CHECK-NEXT:    testl %esi, %esi
-; CHECK-NEXT:    setns %cl
-; CHECK-NEXT:    fildll {{[0-9]+}}(%esp)
 ; CHECK-NEXT:    fadds {{\.LCPI.*}}(,%ecx,4)
+; CHECK-NEXT:    shrl $31, %esi
+; CHECK-NEXT:    fildll {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fadds {{\.LCPI.*}}(,%esi,4)
 ; CHECK-NEXT:    fstps 84(%eax)
 ; CHECK-NEXT:    fstps 80(%eax)
 ; CHECK-NEXT:    addl $20, %esp
diff --git a/llvm/test/CodeGen/X86/pr15705.ll b/llvm/test/CodeGen/X86/pr15705.ll
index 0c72359b7..3dd4aabe2 100644
--- a/llvm/test/CodeGen/X86/pr15705.ll
+++ b/llvm/test/CodeGen/X86/pr15705.ll
@@ -45,4 +45,4 @@ return:
   ret i32 %retval.0
 }
 
-attributes #0 = { nounwind readnone ssp "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind readnone ssp "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/X86/pr18846.ll b/llvm/test/CodeGen/X86/pr18846.ll
index 02c33fed8..2d9be1c65 100644
--- a/llvm/test/CodeGen/X86/pr18846.ll
+++ b/llvm/test/CodeGen/X86/pr18846.ll
@@ -126,7 +126,7 @@ for.body65:                                       ; preds = %for.body29
 ; Function Attrs: nounwind
 declare void @llvm.x86.avx.storeu.ps.256(i8*, <8 x float>) #1
 
-attributes #0 = { uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { uwtable "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind }
 
 !llvm.ident = !{!0}
diff --git a/llvm/test/CodeGen/X86/pr20020.ll b/llvm/test/CodeGen/X86/pr20020.ll
index 56c12faaf..2f554d111 100644
--- a/llvm/test/CodeGen/X86/pr20020.ll
+++ b/llvm/test/CodeGen/X86/pr20020.ll
@@ -70,4 +70,4 @@ for.end38:                                        ; preds = %for.inc20, %entry
 ; Function Attrs: nounwind
 declare double @sqrt(double) #1
 
-attributes #0 = { "no-frame-pointer-elim-non-leaf" }
+attributes #0 = { "frame-pointer"="non-leaf" }
diff --git a/llvm/test/CodeGen/X86/pr2326.ll b/llvm/test/CodeGen/X86/pr2326.ll
index ec2844c42..4b9cb35c0 100644
--- a/llvm/test/CodeGen/X86/pr2326.ll
+++ b/llvm/test/CodeGen/X86/pr2326.ll
@@ -1,7 +1,18 @@
-; RUN: llc < %s -mtriple=i686-- | grep sete
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- | FileCheck %s
 ; PR2326
 
 define i32 @func_59(i32 %p_60) nounwind  {
+; CHECK-LABEL: func_59:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushl %eax
+; CHECK-NEXT:    movl 0, %eax
+; CHECK-NEXT:    xorl %ecx, %ecx
+; CHECK-NEXT:    cmpl (%esp), %eax
+; CHECK-NEXT:    sete %cl
+; CHECK-NEXT:    pushl $0
+; CHECK-NEXT:    pushl %ecx
+; CHECK-NEXT:    calll func_15
 entry:
 	%l_108 = alloca i32		; <i32*> [#uses=2]
 	%tmp15 = load i32, i32* null, align 4		; <i32> [#uses=1]
diff --git a/llvm/test/CodeGen/X86/pr24374.ll b/llvm/test/CodeGen/X86/pr24374.ll
index dab3b3f41..7ad05e8b7 100644
--- a/llvm/test/CodeGen/X86/pr24374.ll
+++ b/llvm/test/CodeGen/X86/pr24374.ll
@@ -31,6 +31,6 @@ define void @g() {
   unreachable
 }
 ; CHECK-LABEL: g:
-; CHECK: ud2
+; CHECK: nop
 
 attributes #0 = { nounwind }
diff --git a/llvm/test/CodeGen/X86/pr28472.ll b/llvm/test/CodeGen/X86/pr28472.ll
index 603549a73..f58a01b48 100644
--- a/llvm/test/CodeGen/X86/pr28472.ll
+++ b/llvm/test/CodeGen/X86/pr28472.ll
@@ -1,9 +1,10 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
 
-; CHECK-LABEL: {{^}}same_dynamic_index_fp_vector_type:
-; CHECK: # %bb.0:
-; CHECK-NEXT: retq
 define float @same_dynamic_index_fp_vector_type(float %val, i32 %idx) {
+; CHECK-LABEL: same_dynamic_index_fp_vector_type:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    retq
 bb:
   %tmp0 = insertelement <4 x float> undef, float %val, i32 %idx
   %tmp1 = extractelement <4 x float> %tmp0, i32 %idx
diff --git a/llvm/test/CodeGen/X86/pr29112.ll b/llvm/test/CodeGen/X86/pr29112.ll
index 6ce9da9ca..0a2dfef43 100644
--- a/llvm/test/CodeGen/X86/pr29112.ll
+++ b/llvm/test/CodeGen/X86/pr29112.ll
@@ -11,9 +11,9 @@ define <4 x float> @bar(<4 x float>* %a1p, <4 x float>* %a2p, <4 x float> %a3, <
 ; CHECK-NEXT:    subq $72, %rsp
 ; CHECK-NEXT:    .cfi_def_cfa_offset 80
 ; CHECK-NEXT:    vmovaps %xmm1, %xmm9
-; CHECK-NEXT:    vbroadcasti32x4 {{.*#+}} zmm15 = [4,22,1,17,4,22,1,17,4,22,1,17,4,22,1,17]
-; CHECK-NEXT:    # zmm15 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; CHECK-NEXT:    vpermi2ps %zmm3, %zmm2, %zmm15
+; CHECK-NEXT:    vbroadcasti32x4 {{.*#+}} zmm14 = [4,22,1,17,4,22,1,17,4,22,1,17,4,22,1,17]
+; CHECK-NEXT:    # zmm14 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; CHECK-NEXT:    vpermi2ps %zmm3, %zmm2, %zmm14
 ; CHECK-NEXT:    vbroadcasti32x4 {{.*#+}} zmm10 = [4,30,1,22,4,30,1,22,4,30,1,22,4,30,1,22]
 ; CHECK-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
 ; CHECK-NEXT:    vpermi2ps %zmm3, %zmm2, %zmm10
@@ -27,7 +27,7 @@ define <4 x float> @bar(<4 x float>* %a1p, <4 x float>* %a2p, <4 x float> %a3, <
 ; CHECK-NEXT:    vextractf128 $1, %ymm2, %xmm6
 ; CHECK-NEXT:    vunpcklps {{.*#+}} xmm11 = xmm6[0],xmm5[0],xmm6[1],xmm5[1]
 ; CHECK-NEXT:    vinsertps {{.*#+}} xmm1 = xmm11[0,1],xmm2[1],xmm11[3]
-; CHECK-NEXT:    vinsertps {{.*#+}} xmm14 = xmm1[0,1,2],xmm3[1]
+; CHECK-NEXT:    vinsertps {{.*#+}} xmm13 = xmm1[0,1,2],xmm3[1]
 ; CHECK-NEXT:    vinsertps {{.*#+}} xmm6 = xmm4[0,1,2],xmm3[1]
 ; CHECK-NEXT:    vmovaps %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; CHECK-NEXT:    vextractf32x4 $2, %zmm3, %xmm4
@@ -39,20 +39,19 @@ define <4 x float> @bar(<4 x float>* %a1p, <4 x float>* %a2p, <4 x float> %a3, <
 ; CHECK-NEXT:    vinsertps {{.*#+}} xmm0 = xmm7[0,1],xmm2[1],xmm7[3]
 ; CHECK-NEXT:    vblendps {{.*#+}} xmm7 = xmm0[0,1,2],xmm3[3]
 ; CHECK-NEXT:    vblendps {{.*#+}} xmm12 = xmm1[0,1,2],xmm3[3]
-; CHECK-NEXT:    vpermilpd {{.*#+}} xmm13 = xmm3[1,0]
 ; CHECK-NEXT:    vinsertps {{.*#+}} xmm1 = xmm8[0,1,2],xmm3[1]
-; CHECK-NEXT:    vinsertps {{.*#+}} xmm3 = xmm0[0,1,2],xmm3[1]
-; CHECK-NEXT:    vaddps %xmm1, %xmm3, %xmm8
+; CHECK-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm3[1]
+; CHECK-NEXT:    vaddps %xmm1, %xmm0, %xmm8
 ; CHECK-NEXT:    vinsertps {{.*#+}} xmm2 = xmm11[0,1],xmm2[3],xmm11[3]
-; CHECK-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1,2],xmm13[0]
-; CHECK-NEXT:    vaddps %xmm15, %xmm2, %xmm2
-; CHECK-NEXT:    vmovaps %xmm14, %xmm1
-; CHECK-NEXT:    vmovaps %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; CHECK-NEXT:    vaddps %xmm10, %xmm14, %xmm10
-; CHECK-NEXT:    vaddps %xmm14, %xmm14, %xmm3
-; CHECK-NEXT:    vaddps %xmm12, %xmm15, %xmm0
-; CHECK-NEXT:    vaddps %xmm8, %xmm0, %xmm0
-; CHECK-NEXT:    vaddps %xmm0, %xmm14, %xmm0
+; CHECK-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1,2],xmm3[2]
+; CHECK-NEXT:    vaddps %xmm2, %xmm14, %xmm2
+; CHECK-NEXT:    vmovaps %xmm13, %xmm1
+; CHECK-NEXT:    vmovaps %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    vaddps %xmm10, %xmm13, %xmm10
+; CHECK-NEXT:    vaddps %xmm13, %xmm13, %xmm3
+; CHECK-NEXT:    vaddps %xmm12, %xmm14, %xmm0
+; CHECK-NEXT:    vaddps %xmm0, %xmm8, %xmm0
+; CHECK-NEXT:    vaddps %xmm0, %xmm13, %xmm0
 ; CHECK-NEXT:    vmovaps %xmm3, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    vmovaps %xmm10, (%rsp)
 ; CHECK-NEXT:    vmovaps %xmm9, %xmm3
@@ -65,7 +64,6 @@ define <4 x float> @bar(<4 x float>* %a1p, <4 x float>* %a2p, <4 x float> %a3, <
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK-NEXT:    retq
   %a1 = shufflevector <16 x float>%c1, <16 x float>%c2, <4 x i32> <i32 4, i32 20, i32 1, i32 17>
-
   %a2 = shufflevector <16 x float>%c1, <16 x float>%c2, <4 x i32> <i32 4, i32 21, i32 1, i32 17>
   %a5 = shufflevector <16 x float>%c1, <16 x float>%c2, <4 x i32> <i32 4, i32 20, i32 1, i32 27>
   %a6 = shufflevector <16 x float>%c1, <16 x float>%c2, <4 x i32> <i32 3, i32 20, i32 1, i32 17>
diff --git a/llvm/test/CodeGen/X86/pr31045.ll b/llvm/test/CodeGen/X86/pr31045.ll
index d706d583f..395c82753 100644
--- a/llvm/test/CodeGen/X86/pr31045.ll
+++ b/llvm/test/CodeGen/X86/pr31045.ll
@@ -73,4 +73,4 @@ entry:
   ret void
 }
 
-attributes #0 = { norecurse nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { norecurse nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/X86/pr32610.ll b/llvm/test/CodeGen/X86/pr32610.ll
index e8b90ee9c..0386dbf4b 100644
--- a/llvm/test/CodeGen/X86/pr32610.ll
+++ b/llvm/test/CodeGen/X86/pr32610.ll
@@ -50,7 +50,7 @@ entry:
   ret void
 }
 
-attributes #0 = { norecurse nounwind optsize ssp "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+fxsr,+mmx,+sse,+sse2,+sse3,+sse4.1,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { norecurse nounwind optsize ssp "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+fxsr,+mmx,+sse,+sse2,+sse3,+sse4.1,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.ident = !{!0}
 
diff --git a/llvm/test/CodeGen/X86/pr3366.ll b/llvm/test/CodeGen/X86/pr3366.ll
index f72a35185..cde6bb4de 100644
--- a/llvm/test/CodeGen/X86/pr3366.ll
+++ b/llvm/test/CodeGen/X86/pr3366.ll
@@ -1,7 +1,15 @@
-; RUN: llc < %s -mtriple=i686-- -disable-cgp-branch-opts | grep movzbl
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- -disable-cgp-branch-opts | FileCheck %s
 ; PR3366
 
 define void @_ada_c34002a() nounwind {
+; CHECK-LABEL: _ada_c34002a:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movb $90, %al
+; CHECK-NEXT:    movzbl %al, %eax
+; CHECK-NEXT:    idivb 0
+; CHECK-NEXT:    cmpb $3, %al
+; CHECK-NEXT:  # %bb.1: # %bb457
 entry:
   %0 = load i8, i8* null, align 1
   %1 = sdiv i8 90, %0
diff --git a/llvm/test/CodeGen/X86/pr34080-2.ll b/llvm/test/CodeGen/X86/pr34080-2.ll
index cb3e22d5e..b09f2a274 100644
--- a/llvm/test/CodeGen/X86/pr34080-2.ll
+++ b/llvm/test/CodeGen/X86/pr34080-2.ll
@@ -62,13 +62,13 @@ define void @computeJD(%struct.DateTime*) nounwind {
 ; CHECK-NEXT:    imull $60000, 24(%ebx), %ecx # imm = 0xEA60
 ; CHECK-NEXT:    addl %eax, %ecx
 ; CHECK-NEXT:    fldl 28(%ebx)
+; CHECK-NEXT:    fmuls {{\.LCPI.*}}
 ; CHECK-NEXT:    fnstcw {{[0-9]+}}(%esp)
 ; CHECK-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
 ; CHECK-NEXT:    orl $3072, %eax # imm = 0xC00
 ; CHECK-NEXT:    movw %ax, {{[0-9]+}}(%esp)
 ; CHECK-NEXT:    movl %ecx, %eax
 ; CHECK-NEXT:    sarl $31, %eax
-; CHECK-NEXT:    fmuls {{\.LCPI.*}}
 ; CHECK-NEXT:    fldcw {{[0-9]+}}(%esp)
 ; CHECK-NEXT:    fistpll {{[0-9]+}}(%esp)
 ; CHECK-NEXT:    fldcw {{[0-9]+}}(%esp)
@@ -133,4 +133,4 @@ define void @computeJD(%struct.DateTime*) nounwind {
   ret void
 }
 
-attributes #0 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="i486" "target-features"="+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="i486" "target-features"="+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/X86/pr34080.ll b/llvm/test/CodeGen/X86/pr34080.ll
index 8783daafe..6c293cc8a 100644
--- a/llvm/test/CodeGen/X86/pr34080.ll
+++ b/llvm/test/CodeGen/X86/pr34080.ll
@@ -164,4 +164,4 @@ entry:
   ret void
 }
 
-attributes #0 = { noinline uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noinline uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/X86/pr34177.ll b/llvm/test/CodeGen/X86/pr34177.ll
index 056682bb2..f8ead6352 100644
--- a/llvm/test/CodeGen/X86/pr34177.ll
+++ b/llvm/test/CodeGen/X86/pr34177.ll
@@ -6,45 +6,88 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define void @test(<4 x i64> %a, <4 x x86_fp80> %b, <8 x x86_fp80>* %c) local_unnamed_addr {
-; CHECK-LABEL: test:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmovq %xmm0, %rax
-; CHECK-NEXT:    vpextrq $1, %xmm0, %rcx
-; CHECK-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; CHECK-NEXT:    vmovq %xmm0, %rdx
-; CHECK-NEXT:    vpextrq $1, %xmm0, %rsi
-; CHECK-NEXT:    cmpq $3, %rsi
-; CHECK-NEXT:    fld1
-; CHECK-NEXT:    fldz
-; CHECK-NEXT:    fld %st(0)
-; CHECK-NEXT:    fcmove %st(2), %st
-; CHECK-NEXT:    cmpq $2, %rdx
-; CHECK-NEXT:    fld %st(1)
-; CHECK-NEXT:    fcmove %st(3), %st
-; CHECK-NEXT:    cmpq $1, %rcx
-; CHECK-NEXT:    fld %st(2)
-; CHECK-NEXT:    fcmove %st(4), %st
-; CHECK-NEXT:    testq %rax, %rax
-; CHECK-NEXT:    fxch %st(3)
-; CHECK-NEXT:    fcmove %st(4), %st
-; CHECK-NEXT:    fstp %st(4)
-; CHECK-NEXT:    fldt {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    fstpt 70(%rdi)
-; CHECK-NEXT:    fldt {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    fstpt 50(%rdi)
-; CHECK-NEXT:    fldt {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    fstpt 30(%rdi)
-; CHECK-NEXT:    fldt {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    fstpt 10(%rdi)
-; CHECK-NEXT:    fxch %st(1)
-; CHECK-NEXT:    fadd %st, %st(0)
-; CHECK-NEXT:    fstpt 60(%rdi)
-; CHECK-NEXT:    fadd %st, %st(0)
-; CHECK-NEXT:    fstpt 40(%rdi)
-; CHECK-NEXT:    fadd %st, %st(0)
-; CHECK-NEXT:    fstpt 20(%rdi)
-; CHECK-NEXT:    fadd %st, %st(0)
-; CHECK-NEXT:    fstpt (%rdi)
+; AVX512F-LABEL: test:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vmovq %xmm0, %rax
+; AVX512F-NEXT:    vpextrq $1, %xmm0, %rcx
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm0
+; AVX512F-NEXT:    vmovq %xmm0, %rdx
+; AVX512F-NEXT:    vpextrq $1, %xmm0, %rsi
+; AVX512F-NEXT:    cmpq $3, %rsi
+; AVX512F-NEXT:    fld1
+; AVX512F-NEXT:    fldz
+; AVX512F-NEXT:    fld %st(0)
+; AVX512F-NEXT:    fcmove %st(2), %st
+; AVX512F-NEXT:    cmpq $2, %rdx
+; AVX512F-NEXT:    fld %st(1)
+; AVX512F-NEXT:    fcmove %st(3), %st
+; AVX512F-NEXT:    cmpq $1, %rcx
+; AVX512F-NEXT:    fld %st(2)
+; AVX512F-NEXT:    fcmove %st(4), %st
+; AVX512F-NEXT:    testq %rax, %rax
+; AVX512F-NEXT:    fxch %st(3)
+; AVX512F-NEXT:    fcmove %st(4), %st
+; AVX512F-NEXT:    fstp %st(4)
+; AVX512F-NEXT:    fldt {{[0-9]+}}(%rsp)
+; AVX512F-NEXT:    fstpt 70(%rdi)
+; AVX512F-NEXT:    fldt {{[0-9]+}}(%rsp)
+; AVX512F-NEXT:    fstpt 50(%rdi)
+; AVX512F-NEXT:    fldt {{[0-9]+}}(%rsp)
+; AVX512F-NEXT:    fstpt 30(%rdi)
+; AVX512F-NEXT:    fldt {{[0-9]+}}(%rsp)
+; AVX512F-NEXT:    fstpt 10(%rdi)
+; AVX512F-NEXT:    fxch %st(1)
+; AVX512F-NEXT:    fadd %st, %st(0)
+; AVX512F-NEXT:    fstpt 60(%rdi)
+; AVX512F-NEXT:    fadd %st, %st(0)
+; AVX512F-NEXT:    fstpt 40(%rdi)
+; AVX512F-NEXT:    fadd %st, %st(0)
+; AVX512F-NEXT:    fstpt 20(%rdi)
+; AVX512F-NEXT:    fadd %st, %st(0)
+; AVX512F-NEXT:    fstpt (%rdi)
+;
+; AVX512VL-LABEL: test:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpcmpeqq {{.*}}(%rip), %ymm0, %k0
+; AVX512VL-NEXT:    kshiftrb $2, %k0, %k1
+; AVX512VL-NEXT:    kshiftrb $1, %k0, %k2
+; AVX512VL-NEXT:    kmovd %k0, %eax
+; AVX512VL-NEXT:    testb $1, %al
+; AVX512VL-NEXT:    fld1
+; AVX512VL-NEXT:    fldz
+; AVX512VL-NEXT:    fld %st(0)
+; AVX512VL-NEXT:    fcmovne %st(2), %st
+; AVX512VL-NEXT:    kshiftrb $1, %k1, %k0
+; AVX512VL-NEXT:    kmovd %k0, %eax
+; AVX512VL-NEXT:    testb $1, %al
+; AVX512VL-NEXT:    fld %st(1)
+; AVX512VL-NEXT:    fcmovne %st(3), %st
+; AVX512VL-NEXT:    kmovd %k1, %eax
+; AVX512VL-NEXT:    testb $1, %al
+; AVX512VL-NEXT:    fld %st(2)
+; AVX512VL-NEXT:    fcmovne %st(4), %st
+; AVX512VL-NEXT:    kmovd %k2, %eax
+; AVX512VL-NEXT:    testb $1, %al
+; AVX512VL-NEXT:    fxch %st(3)
+; AVX512VL-NEXT:    fcmovne %st(4), %st
+; AVX512VL-NEXT:    fstp %st(4)
+; AVX512VL-NEXT:    fldt {{[0-9]+}}(%rsp)
+; AVX512VL-NEXT:    fstpt 70(%rdi)
+; AVX512VL-NEXT:    fldt {{[0-9]+}}(%rsp)
+; AVX512VL-NEXT:    fstpt 50(%rdi)
+; AVX512VL-NEXT:    fldt {{[0-9]+}}(%rsp)
+; AVX512VL-NEXT:    fstpt 30(%rdi)
+; AVX512VL-NEXT:    fldt {{[0-9]+}}(%rsp)
+; AVX512VL-NEXT:    fstpt 10(%rdi)
+; AVX512VL-NEXT:    fxch %st(1)
+; AVX512VL-NEXT:    fadd %st, %st(0)
+; AVX512VL-NEXT:    fstpt (%rdi)
+; AVX512VL-NEXT:    fadd %st, %st(0)
+; AVX512VL-NEXT:    fstpt 60(%rdi)
+; AVX512VL-NEXT:    fadd %st, %st(0)
+; AVX512VL-NEXT:    fstpt 40(%rdi)
+; AVX512VL-NEXT:    fadd %st, %st(0)
+; AVX512VL-NEXT:    fstpt 20(%rdi)
   %1 = icmp eq <4 x i64> <i64 0, i64 1, i64 2, i64 3>, %a
   %2 = select <4 x i1> %1, <4 x x86_fp80> <x86_fp80 0xK3FFF8000000000000000, x86_fp80 0xK3FFF8000000000000000, x86_fp80 0xK3FFF8000000000000000, x86_fp80 0xK3FFF8000000000000000>, <4 x x86_fp80> zeroinitializer
   %3 = fadd <4 x x86_fp80> %2, %2
diff --git a/llvm/test/CodeGen/X86/pr3457.ll b/llvm/test/CodeGen/X86/pr3457.ll
index d4c0020cb..39c46928b 100644
--- a/llvm/test/CodeGen/X86/pr3457.ll
+++ b/llvm/test/CodeGen/X86/pr3457.ll
@@ -1,8 +1,30 @@
-; RUN: llc < %s -mtriple=i386-apple-darwin -mcpu=corei7 | not grep fstpt
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i386-apple-darwin -mcpu=corei7 | FileCheck %s
 ; PR3457
 ; rdar://6548010
 
 define void @foo(double* nocapture %P) nounwind {
+; CHECK-LABEL: foo:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    pushl %esi
+; CHECK-NEXT:    subl $24, %esp
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; CHECK-NEXT:    calll _test
+; CHECK-NEXT:    fstpl {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    movsd %xmm0, (%esp) ## 8-byte Spill
+; CHECK-NEXT:    calll _test
+; CHECK-NEXT:    fstpl {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    movsd (%esp), %xmm1 ## 8-byte Reload
+; CHECK-NEXT:    ## xmm1 = mem[0],zero
+; CHECK-NEXT:    mulsd %xmm1, %xmm1
+; CHECK-NEXT:    mulsd %xmm0, %xmm0
+; CHECK-NEXT:    addsd %xmm1, %xmm0
+; CHECK-NEXT:    movsd %xmm0, (%esi)
+; CHECK-NEXT:    addl $24, %esp
+; CHECK-NEXT:    popl %esi
+; CHECK-NEXT:    retl
 entry:
 	%0 = tail call double (...) @test() nounwind		; <double> [#uses=2]
 	%1 = tail call double (...) @test() nounwind		; <double> [#uses=2]
diff --git a/llvm/test/CodeGen/X86/pr34629.ll b/llvm/test/CodeGen/X86/pr34629.ll
index 55084b425..fcd33c80a 100644
--- a/llvm/test/CodeGen/X86/pr34629.ll
+++ b/llvm/test/CodeGen/X86/pr34629.ll
@@ -38,7 +38,7 @@ if.end:                                           ; preds = %entry, %if.then
   ret void
 }
 
-attributes #0 = { norecurse nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { norecurse nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.module.flags = !{!0}
 !llvm.ident = !{!1}
diff --git a/llvm/test/CodeGen/X86/pr34634.ll b/llvm/test/CodeGen/X86/pr34634.ll
index 9ed78a28d..02b34da45 100644
--- a/llvm/test/CodeGen/X86/pr34634.ll
+++ b/llvm/test/CodeGen/X86/pr34634.ll
@@ -54,7 +54,7 @@ entry:
   ret i32 0
 }
 
-attributes #0 = { norecurse nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { norecurse nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.module.flags = !{!0}
 !llvm.ident = !{!1}
diff --git a/llvm/test/CodeGen/X86/pr34657.ll b/llvm/test/CodeGen/X86/pr34657.ll
index d8b72920f..9761927dc 100644
--- a/llvm/test/CodeGen/X86/pr34657.ll
+++ b/llvm/test/CodeGen/X86/pr34657.ll
@@ -5,13 +5,12 @@ define <112 x i8> @pr34657(<112 x i8>* %src) local_unnamed_addr {
 ; CHECK-LABEL: pr34657:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    movq %rdi, %rax
-; CHECK-NEXT:    vmovups 64(%rsi), %ymm0
-; CHECK-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
-; CHECK-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm1
-; CHECK-NEXT:    vmovups (%rsi), %zmm2
-; CHECK-NEXT:    vmovaps %ymm0, 64(%rdi)
-; CHECK-NEXT:    vmovaps %zmm2, (%rdi)
-; CHECK-NEXT:    vextractf32x4 $2, %zmm1, 96(%rdi)
+; CHECK-NEXT:    vmovups (%rsi), %zmm0
+; CHECK-NEXT:    vmovups 64(%rsi), %ymm1
+; CHECK-NEXT:    vmovups 96(%rsi), %xmm2
+; CHECK-NEXT:    vmovaps %xmm2, 96(%rdi)
+; CHECK-NEXT:    vmovaps %ymm1, 64(%rdi)
+; CHECK-NEXT:    vmovaps %zmm0, (%rdi)
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
 entry:
diff --git a/llvm/test/CodeGen/X86/pr35918.ll b/llvm/test/CodeGen/X86/pr35918.ll
index 5b9e7a614..373f72272 100644
--- a/llvm/test/CodeGen/X86/pr35918.ll
+++ b/llvm/test/CodeGen/X86/pr35918.ll
@@ -5,79 +5,31 @@
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=skx | FileCheck %s --check-prefixes=X64,X64-SKX
 
 define void @fetch_r16g16_snorm_unorm8(<4 x i8>*, i8*, i32, i32, { [2048 x i32], [128 x i64] }*) nounwind {
-; X86-SKYLAKE-LABEL: fetch_r16g16_snorm_unorm8:
-; X86-SKYLAKE:       # %bb.0: # %entry
-; X86-SKYLAKE-NEXT:    subl $8, %esp
-; X86-SKYLAKE-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-SKYLAKE-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-SKYLAKE-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X86-SKYLAKE-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,1,4,5,6,7]
-; X86-SKYLAKE-NEXT:    vpsrad $16, %xmm0, %xmm0
-; X86-SKYLAKE-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; X86-SKYLAKE-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
-; X86-SKYLAKE-NEXT:    vpsrld $7, %xmm0, %xmm0
-; X86-SKYLAKE-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4],zero,xmm0[u,u,u,u,u,u,u,u,u,u,u,u,u]
-; X86-SKYLAKE-NEXT:    vmovd %xmm0, %ecx
-; X86-SKYLAKE-NEXT:    orl $-16777216, %ecx # imm = 0xFF000000
-; X86-SKYLAKE-NEXT:    movl %ecx, (%eax)
-; X86-SKYLAKE-NEXT:    addl $8, %esp
-; X86-SKYLAKE-NEXT:    retl
+; X86-LABEL: fetch_r16g16_snorm_unorm8:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; X86-NEXT:    vpmaxsw %xmm1, %xmm0, %xmm0
+; X86-NEXT:    vpsrlw $7, %xmm0, %xmm0
+; X86-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2],zero,xmm0[u,u,u,u,u,u,u,u,u,u,u,u,u]
+; X86-NEXT:    vmovd %xmm0, %ecx
+; X86-NEXT:    orl $-16777216, %ecx # imm = 0xFF000000
+; X86-NEXT:    movl %ecx, (%eax)
+; X86-NEXT:    retl
 ;
-; X86-SKX-LABEL: fetch_r16g16_snorm_unorm8:
-; X86-SKX:       # %bb.0: # %entry
-; X86-SKX-NEXT:    subl $8, %esp
-; X86-SKX-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-SKX-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X86-SKX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,0,1,u,u,2,3,u,u,u,u,u,u,u,u]
-; X86-SKX-NEXT:    vpsrad $16, %xmm0, %xmm0
-; X86-SKX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; X86-SKX-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
-; X86-SKX-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7]
-; X86-SKX-NEXT:    vpsrld $7, %xmm0, %xmm0
-; X86-SKX-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
-; X86-SKX-NEXT:    vpmovqw %xmm0, {{[0-9]+}}(%esp)
-; X86-SKX-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X86-SKX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; X86-SKX-NEXT:    vpmovdb %xmm0, (%esp)
-; X86-SKX-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-SKX-NEXT:    movzwl (%esp), %ecx
-; X86-SKX-NEXT:    orl $-16777216, %ecx # imm = 0xFF000000
-; X86-SKX-NEXT:    movl %ecx, (%eax)
-; X86-SKX-NEXT:    addl $8, %esp
-; X86-SKX-NEXT:    retl
-;
-; X64-SKYLAKE-LABEL: fetch_r16g16_snorm_unorm8:
-; X64-SKYLAKE:       # %bb.0: # %entry
-; X64-SKYLAKE-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X64-SKYLAKE-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,1,4,5,6,7]
-; X64-SKYLAKE-NEXT:    vpsrad $16, %xmm0, %xmm0
-; X64-SKYLAKE-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; X64-SKYLAKE-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
-; X64-SKYLAKE-NEXT:    vpsrld $7, %xmm0, %xmm0
-; X64-SKYLAKE-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4],zero,xmm0[u,u,u,u,u,u,u,u,u,u,u,u,u]
-; X64-SKYLAKE-NEXT:    vmovd %xmm0, %eax
-; X64-SKYLAKE-NEXT:    orl $-16777216, %eax # imm = 0xFF000000
-; X64-SKYLAKE-NEXT:    movl %eax, (%rdi)
-; X64-SKYLAKE-NEXT:    retq
-;
-; X64-SKX-LABEL: fetch_r16g16_snorm_unorm8:
-; X64-SKX:       # %bb.0: # %entry
-; X64-SKX-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X64-SKX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,0,1,u,u,2,3,u,u,u,u,u,u,u,u]
-; X64-SKX-NEXT:    vpsrad $16, %xmm0, %xmm0
-; X64-SKX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; X64-SKX-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
-; X64-SKX-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7]
-; X64-SKX-NEXT:    vpsrld $7, %xmm0, %xmm0
-; X64-SKX-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
-; X64-SKX-NEXT:    vpmovqw %xmm0, -{{[0-9]+}}(%rsp)
-; X64-SKX-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X64-SKX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; X64-SKX-NEXT:    vpmovdb %xmm0, -{{[0-9]+}}(%rsp)
-; X64-SKX-NEXT:    movzwl -{{[0-9]+}}(%rsp), %eax
-; X64-SKX-NEXT:    orl $-16777216, %eax # imm = 0xFF000000
-; X64-SKX-NEXT:    movl %eax, (%rdi)
-; X64-SKX-NEXT:    retq
+; X64-LABEL: fetch_r16g16_snorm_unorm8:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X64-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; X64-NEXT:    vpmaxsw %xmm1, %xmm0, %xmm0
+; X64-NEXT:    vpsrlw $7, %xmm0, %xmm0
+; X64-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2],zero,xmm0[u,u,u,u,u,u,u,u,u,u,u,u,u]
+; X64-NEXT:    vmovd %xmm0, %eax
+; X64-NEXT:    orl $-16777216, %eax # imm = 0xFF000000
+; X64-NEXT:    movl %eax, (%rdi)
+; X64-NEXT:    retq
 entry:
   %5 = bitcast i8* %1 to <2 x i16>*
   %6 = load <2 x i16>, <2 x i16>* %5, align 2
diff --git a/llvm/test/CodeGen/X86/pr37063.ll b/llvm/test/CodeGen/X86/pr37063.ll
index cf5e1fa56..f7f8d622d 100644
--- a/llvm/test/CodeGen/X86/pr37063.ll
+++ b/llvm/test/CodeGen/X86/pr37063.ll
@@ -6,9 +6,9 @@ declare void @bar()
 define void @foo(i64*) {
 ; CHECK-LABEL: foo:
 ; CHECK:       # %bb.0: # %start
-; CHECK-NEXT:    movq (%rdi), %rax
-; CHECK-NEXT:    andl $-2, %eax
-; CHECK-NEXT:    cmpq $4, %rax
+; CHECK-NEXT:    movl (%rdi), %eax
+; CHECK-NEXT:    andl $6, %eax
+; CHECK-NEXT:    cmpl $4, %eax
 ; CHECK-NEXT:    jne .LBB0_2
 ; CHECK-NEXT:  # %bb.1: # %bb1
 ; CHECK-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/pr37916.ll b/llvm/test/CodeGen/X86/pr37916.ll
index 2da9413a9..484104da9 100644
--- a/llvm/test/CodeGen/X86/pr37916.ll
+++ b/llvm/test/CodeGen/X86/pr37916.ll
@@ -7,7 +7,6 @@
 define void @fn1() local_unnamed_addr {
 ; CHECK-LABEL: fn1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    .p2align 4, 0x90
 ; CHECK-NEXT:  .LBB0_1: # %if.end
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movl a+4, %eax
diff --git a/llvm/test/CodeGen/X86/pr38795.ll b/llvm/test/CodeGen/X86/pr38795.ll
index 11534a920..d805dcad8 100644
--- a/llvm/test/CodeGen/X86/pr38795.ll
+++ b/llvm/test/CodeGen/X86/pr38795.ll
@@ -70,8 +70,15 @@ define dso_local void @fn() {
 ; CHECK-NEXT:    movb %dl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
 ; CHECK-NEXT:    movb %dh, %dl
 ; CHECK-NEXT:    jne .LBB0_16
+; CHECK-NEXT:    jmp .LBB0_6
 ; CHECK-NEXT:    .p2align 4, 0x90
-; CHECK-NEXT:  # %bb.6: # %for.cond35
+; CHECK-NEXT:  .LBB0_3: # %if.then
+; CHECK-NEXT:    # in Loop: Header=BB0_1 Depth=1
+; CHECK-NEXT:    movl $.str, (%esp)
+; CHECK-NEXT:    calll printf
+; CHECK-NEXT:    movb {{[-0-9]+}}(%e{{[sb]}}p), %dl # 1-byte Reload
+; CHECK-NEXT:    # implicit-def: $eax
+; CHECK-NEXT:  .LBB0_6: # %for.cond35
 ; CHECK-NEXT:    # in Loop: Header=BB0_1 Depth=1
 ; CHECK-NEXT:    testl %edi, %edi
 ; CHECK-NEXT:    je .LBB0_7
@@ -96,22 +103,10 @@ define dso_local void @fn() {
 ; CHECK-NEXT:    # implicit-def: $ebp
 ; CHECK-NEXT:    jmp .LBB0_20
 ; CHECK-NEXT:    .p2align 4, 0x90
-; CHECK-NEXT:  .LBB0_3: # %if.then
-; CHECK-NEXT:    # in Loop: Header=BB0_1 Depth=1
-; CHECK-NEXT:    movl $.str, (%esp)
-; CHECK-NEXT:    calll printf
-; CHECK-NEXT:    movb {{[-0-9]+}}(%e{{[sb]}}p), %dl # 1-byte Reload
-; CHECK-NEXT:    # implicit-def: $eax
-; CHECK-NEXT:    testl %edi, %edi
-; CHECK-NEXT:    jne .LBB0_11
-; CHECK-NEXT:    jmp .LBB0_7
-; CHECK-NEXT:    .p2align 4, 0x90
 ; CHECK-NEXT:  .LBB0_8: # %if.end21
 ; CHECK-NEXT:    # in Loop: Header=BB0_1 Depth=1
 ; CHECK-NEXT:    # implicit-def: $ebp
-; CHECK-NEXT:    testb %bl, %bl
-; CHECK-NEXT:    je .LBB0_13
-; CHECK-NEXT:    jmp .LBB0_10
+; CHECK-NEXT:    jmp .LBB0_9
 ; CHECK-NEXT:    .p2align 4, 0x90
 ; CHECK-NEXT:  .LBB0_7: # in Loop: Header=BB0_1 Depth=1
 ; CHECK-NEXT:    xorl %edi, %edi
@@ -127,11 +122,11 @@ define dso_local void @fn() {
 ; CHECK-NEXT:    # in Loop: Header=BB0_20 Depth=2
 ; CHECK-NEXT:    testb %bl, %bl
 ; CHECK-NEXT:    jne .LBB0_20
-; CHECK-NEXT:  # %bb.9: # %ae
+; CHECK-NEXT:  .LBB0_9: # %ae
 ; CHECK-NEXT:    # in Loop: Header=BB0_1 Depth=1
 ; CHECK-NEXT:    testb %bl, %bl
 ; CHECK-NEXT:    jne .LBB0_10
-; CHECK-NEXT:  .LBB0_13: # %if.end26
+; CHECK-NEXT:  # %bb.13: # %if.end26
 ; CHECK-NEXT:    # in Loop: Header=BB0_1 Depth=1
 ; CHECK-NEXT:    xorl %ecx, %ecx
 ; CHECK-NEXT:    testb %dl, %dl
diff --git a/llvm/test/CodeGen/X86/pr38803.ll b/llvm/test/CodeGen/X86/pr38803.ll
index a1ca27d89..a2fc19e0c 100644
--- a/llvm/test/CodeGen/X86/pr38803.ll
+++ b/llvm/test/CodeGen/X86/pr38803.ll
@@ -13,15 +13,14 @@ define float @_Z3fn2v() {
 ; CHECK-NEXT:    callq _Z1av
 ; CHECK-NEXT:    # kill: def $al killed $al def $eax
 ; CHECK-NEXT:    kmovd %eax, %k1
-; CHECK-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; CHECK-NEXT:    vmovss %xmm0, %xmm0, %xmm0 {%k1} {z}
+; CHECK-NEXT:    vmovss {{.*}}(%rip), %xmm0 {%k1} {z}
 ; CHECK-NEXT:    cmpl $0, {{.*}}(%rip)
 ; CHECK-NEXT:    je .LBB0_2
 ; CHECK-NEXT:  # %bb.1: # %if.then
 ; CHECK-NEXT:    vcvtsi2ssl {{.*}}(%rip), %xmm1, %xmm1
 ; CHECK-NEXT:    kmovd %eax, %k1
 ; CHECK-NEXT:    vxorps %xmm2, %xmm2, %xmm2
-; CHECK-NEXT:    vmovss %xmm2, %xmm0, %xmm1 {%k1}
+; CHECK-NEXT:    vmovss %xmm2, %xmm1, %xmm1 {%k1}
 ; CHECK-NEXT:    vmovss %xmm1, {{.*}}(%rip)
 ; CHECK-NEXT:  .LBB0_2: # %if.end
 ; CHECK-NEXT:    popq %rax
diff --git a/llvm/test/CodeGen/X86/pr39666.ll b/llvm/test/CodeGen/X86/pr39666.ll
index 949e8ab13..d2ef05922 100644
--- a/llvm/test/CodeGen/X86/pr39666.ll
+++ b/llvm/test/CodeGen/X86/pr39666.ll
@@ -3,7 +3,7 @@
 
 define <2 x i64> @test5(i64* %base, <2 x i64> %src0) {
 ; CHECK-LABEL: test5:
-; CHECK:       # %bb.0: # %else
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vpinsrq $1, (%rdi), %xmm0, %xmm0
 ; CHECK-NEXT:    retq
  %res = call <2 x i64> @llvm.masked.expandload.v2i64(i64* %base, <2 x i1> <i1 false, i1 true>, <2 x i64> %src0)
@@ -14,19 +14,22 @@ declare <2 x i64> @llvm.masked.expandload.v2i64(i64*, <2 x i1>, <2 x i64>)
 define void @test11(i64* %base, <2 x i64> %V, <2 x i1> %mask) {
 ; CHECK-LABEL: test11:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpextrb $0, %xmm1, %eax
+; CHECK-NEXT:    vpsllq $63, %xmm1, %xmm1
+; CHECK-NEXT:    vmovmskpd %xmm1, %eax
 ; CHECK-NEXT:    testb $1, %al
-; CHECK-NEXT:    je .LBB1_2
-; CHECK-NEXT:  # %bb.1: # %cond.store
+; CHECK-NEXT:    jne .LBB1_1
+; CHECK-NEXT:  # %bb.2: # %else
+; CHECK-NEXT:    testb $2, %al
+; CHECK-NEXT:    jne .LBB1_3
+; CHECK-NEXT:  .LBB1_4: # %else2
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  .LBB1_1: # %cond.store
 ; CHECK-NEXT:    vmovq %xmm0, (%rdi)
 ; CHECK-NEXT:    addq $8, %rdi
-; CHECK-NEXT:  .LBB1_2: # %else
-; CHECK-NEXT:    vpextrb $8, %xmm1, %eax
-; CHECK-NEXT:    testb $1, %al
+; CHECK-NEXT:    testb $2, %al
 ; CHECK-NEXT:    je .LBB1_4
-; CHECK-NEXT:  # %bb.3: # %cond.store1
+; CHECK-NEXT:  .LBB1_3: # %cond.store1
 ; CHECK-NEXT:    vpextrq $1, %xmm0, (%rdi)
-; CHECK-NEXT:  .LBB1_4: # %else2
 ; CHECK-NEXT:    retq
  call void @llvm.masked.compressstore.v2i64(<2 x i64> %V, i64* %base, <2 x i1> %mask)
  ret void
diff --git a/llvm/test/CodeGen/X86/pr40631_deadstore_elision.ll b/llvm/test/CodeGen/X86/pr40631_deadstore_elision.ll
index f330e0f15..c742ce4bd 100644
--- a/llvm/test/CodeGen/X86/pr40631_deadstore_elision.ll
+++ b/llvm/test/CodeGen/X86/pr40631_deadstore_elision.ll
@@ -12,12 +12,13 @@ define i32 @ipt_do_table(%struct.sk_buff* noalias nocapture readonly) {
 ; CHECK-NEXT:    movq (%rdi), %rax
 ; CHECK-NEXT:    xorps %xmm0, %xmm0
 ; CHECK-NEXT:    movaps %xmm0, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movq $170, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movq $0, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movaps {{.*#+}} xmm0 = [12297829382473034410,12297829382473034410]
 ; CHECK-NEXT:    movaps %xmm0, (%rsp)
 ; CHECK-NEXT:    movabsq $-6148914691236517206, %rcx # imm = 0xAAAAAAAAAAAAAAAA
 ; CHECK-NEXT:    movq %rcx, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movq %rcx, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movb $-86, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movzwl 2(%rax), %ecx
 ; CHECK-NEXT:    andl $8191, %ecx # imm = 0x1FFF
 ; CHECK-NEXT:    movl %ecx, {{[0-9]+}}(%rsp)
diff --git a/llvm/test/CodeGen/X86/pr40994.ll b/llvm/test/CodeGen/X86/pr40994.ll
index 66d55949f..7ad494d75 100644
--- a/llvm/test/CodeGen/X86/pr40994.ll
+++ b/llvm/test/CodeGen/X86/pr40994.ll
@@ -3,7 +3,7 @@
 
 define <8 x i8> @foo(<16 x i8> %a) {
 ; CHECK-LABEL: foo:
-; CHECK:       # %bb.0: # %cond.store
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    pextrb $0, %xmm0, -{{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    pextrb $2, %xmm0, -{{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    pextrb $4, %xmm0, -{{[0-9]+}}(%rsp)
@@ -12,7 +12,7 @@ define <8 x i8> @foo(<16 x i8> %a) {
 ; CHECK-NEXT:    pextrb $10, %xmm0, -{{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    pextrb $12, %xmm0, -{{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    pextrb $14, %xmm0, -{{[0-9]+}}(%rsp)
-; CHECK-NEXT:    pmovzxbw {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; CHECK-NEXT:    retq
   %v = alloca i8, i32 8, align 16
   call void @llvm.masked.compressstore.v16i8(<16 x i8> %a, i8* %v, <16 x i1> <i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false>)
diff --git a/llvm/test/CodeGen/X86/pr42727.ll b/llvm/test/CodeGen/X86/pr42727.ll
new file mode 100644
index 000000000..6b4a967f3
--- /dev/null
+++ b/llvm/test/CodeGen/X86/pr42727.ll
@@ -0,0 +1,34 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-w64-windows-gnu | FileCheck %s
+
+%union.c_v256.26.65.104.143.962.1248 = type { [4 x i64] }
+
+define void @_ZN14simd_test_avx216c_imm_v256_alignILi1EEE6c_v256S1_S1_(%union.c_v256.26.65.104.143.962.1248* byval(%union.c_v256.26.65.104.143.962.1248) align 4) #0 {
+; CHECK-LABEL: _ZN14simd_test_avx216c_imm_v256_alignILi1EEE6c_v256S1_S1_:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmovdqu {{[0-9]+}}(%esp), %xmm0
+; CHECK-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; CHECK-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; CHECK-NEXT:    vpsllq $56, %ymm0, %ymm0
+; CHECK-NEXT:    vmovdqu %ymm0, (%eax)
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retl
+entry:
+  %b.sroa.0.sroa.2.0.b.sroa.0.0..sroa_cast.sroa_idx38 = getelementptr inbounds %union.c_v256.26.65.104.143.962.1248, %union.c_v256.26.65.104.143.962.1248* %0, i32 0, i32 0, i32 1
+  %1 = bitcast i64* %b.sroa.0.sroa.2.0.b.sroa.0.0..sroa_cast.sroa_idx38 to <2 x i64>*
+  %2 = load <2 x i64>, <2 x i64>* %1, align 4
+  %b.sroa.0.sroa.4.0.copyload = load i64, i64* undef, align 4
+  %3 = extractelement <2 x i64> %2, i32 0
+  %4 = extractelement <2 x i64> %2, i32 1
+  %5 = insertelement <4 x i64> undef, i64 %3, i32 0
+  %6 = insertelement <4 x i64> %5, i64 %4, i32 1
+  %7 = insertelement <4 x i64> %6, i64 %b.sroa.0.sroa.4.0.copyload, i32 2
+  %8 = insertelement <4 x i64> %7, i64 undef, i32 3
+  %9 = shl <4 x i64> %8, <i64 56, i64 56, i64 56, i64 56>
+  %10 = or <4 x i64> %9, zeroinitializer
+  store <4 x i64> %10, <4 x i64>* undef, align 8
+  ret void
+}
+
+attributes #0 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+avx,+avx2,+cx8,+fxsr,+mmx,+popcnt,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87,+xsave" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
diff --git a/llvm/test/CodeGen/X86/pr42870.ll b/llvm/test/CodeGen/X86/pr42870.ll
new file mode 100644
index 000000000..575a2653a
--- /dev/null
+++ b/llvm/test/CodeGen/X86/pr42870.ll
@@ -0,0 +1,31 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-apple-darwin -mattr=sse | FileCheck %s
+
+define i32 @foo(<4 x float>* %a) {
+; CHECK-LABEL: foo:
+; CHECK:       ## %bb.0: ## %start
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    movaps (%eax), %xmm0
+; CHECK-NEXT:    movmskps %xmm0, %eax
+; CHECK-NEXT:    retl
+start:
+  %0 = bitcast <4 x float>* %a to <4 x i32>*
+  %1 = load <4 x i32>, <4 x i32>* %0, align 16
+  %2 = icmp slt <4 x i32> %1, zeroinitializer
+  %3 = bitcast <4 x i1> %2 to i4
+  %4 = zext i4 %3 to i32
+  ret i32 %4
+}
+
+define i32 @bar(<4 x float> %a) {
+; CHECK-LABEL: bar:
+; CHECK:       ## %bb.0: ## %start
+; CHECK-NEXT:    movmskps %xmm0, %eax
+; CHECK-NEXT:    retl
+start:
+  %0 = bitcast <4 x float> %a to <4 x i32>
+  %1 = icmp slt <4 x i32> %0, zeroinitializer
+  %2 = bitcast <4 x i1> %1 to i4
+  %3 = zext i4 %2 to i32
+  ret i32 %3
+}
diff --git a/llvm/test/CodeGen/X86/pr42905.ll b/llvm/test/CodeGen/X86/pr42905.ll
new file mode 100644
index 000000000..310a173f8
--- /dev/null
+++ b/llvm/test/CodeGen/X86/pr42905.ll
@@ -0,0 +1,26 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu | FileCheck %s
+
+define <4 x double> @autogen_SD30452(i1 %L230) {
+; CHECK-LABEL: autogen_SD30452:
+; CHECK:       # %bb.0: # %BB
+; CHECK-NEXT:    movdqa {{.*#+}} xmm1 = [151829,151829]
+; CHECK-NEXT:    movq %xmm0, %rax
+; CHECK-NEXT:    cvtsi2sd %rax, %xmm0
+; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; CHECK-NEXT:    movq %xmm2, %rax
+; CHECK-NEXT:    xorps %xmm2, %xmm2
+; CHECK-NEXT:    cvtsi2sd %rax, %xmm2
+; CHECK-NEXT:    unpcklpd {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; CHECK-NEXT:    cvtdq2pd %xmm1, %xmm1
+; CHECK-NEXT:    retq
+BB:
+  %I = insertelement <4 x i64> zeroinitializer, i64 151829, i32 3
+  %Shuff7 = shufflevector <4 x i64> %I, <4 x i64> zeroinitializer, <4 x i32> <i32 undef, i32 undef, i32 3, i32 undef>
+  br label %CF242
+
+CF242:                                            ; preds = %CF242, %BB
+  %FC125 = sitofp <4 x i64> %Shuff7 to <4 x double>
+  ret <4 x double> %FC125
+}
diff --git a/llvm/test/CodeGen/X86/pr42909.ll b/llvm/test/CodeGen/X86/pr42909.ll
new file mode 100644
index 000000000..48338904a
--- /dev/null
+++ b/llvm/test/CodeGen/X86/pr42909.ll
@@ -0,0 +1,21 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=corei7 | FileCheck %s
+
+define void @autogen_SD31033(i16* %a0) {
+; CHECK-LABEL: autogen_SD31033:
+; CHECK:       # %bb.0: # %BB
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB0_1: # %CF
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    jmp .LBB0_1
+BB:
+  %L5 = load i16, i16* %a0
+  %I8 = insertelement <4 x i16> zeroinitializer, i16 %L5, i32 1
+  %Tr = trunc <4 x i16> %I8 to <4 x i1>
+  %Shuff28 = shufflevector <4 x i1> zeroinitializer, <4 x i1> %Tr, <4 x i32> <i32 undef, i32 3, i32 5, i32 undef>
+  br label %CF
+
+CF:                                               ; preds = %CF, %BB
+  %E42 = extractelement <4 x i1> %Shuff28, i32 3
+  br label %CF
+}
diff --git a/llvm/test/CodeGen/X86/pr42998.ll b/llvm/test/CodeGen/X86/pr42998.ll
new file mode 100644
index 000000000..e3d6a560a
--- /dev/null
+++ b/llvm/test/CodeGen/X86/pr42998.ll
@@ -0,0 +1,63 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=skx | FileCheck %s --check-prefixes=CHECK,FAST-INCDEC
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=goldmont | FileCheck %s --check-prefixes=CHECK,SLOW-INCDEC
+
+define i64 @imm1_Oz(i32 %x, i32 %y) minsize nounwind {
+; CHECK-LABEL: imm1_Oz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    # kill: def $esi killed $esi def $rsi
+; CHECK-NEXT:    # kill: def $edi killed $edi def $rdi
+; CHECK-NEXT:    leal 1(%rdi), %eax
+; CHECK-NEXT:    incl %esi
+; CHECK-NEXT:    addq %rsi, %rax
+; CHECK-NEXT:    retq
+  %x1 = add i32 %x, 1
+  %y1 = add i32 %y, 1
+  %x1z = zext i32 %x1 to i64
+  %y1z = zext i32 %y1 to i64
+  %r = add i64 %x1z, %y1z
+  ret i64 %r
+}
+
+define i64 @imm1_Os(i32 %x, i32 %y) optsize nounwind {
+; CHECK-LABEL: imm1_Os:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    # kill: def $esi killed $esi def $rsi
+; CHECK-NEXT:    # kill: def $edi killed $edi def $rdi
+; CHECK-NEXT:    leal 1(%rdi), %eax
+; CHECK-NEXT:    incl %esi
+; CHECK-NEXT:    addq %rsi, %rax
+; CHECK-NEXT:    retq
+  %x1 = add i32 %x, 1
+  %y1 = add i32 %y, 1
+  %x1z = zext i32 %x1 to i64
+  %y1z = zext i32 %y1 to i64
+  %r = add i64 %x1z, %y1z
+  ret i64 %r
+}
+
+define i64 @imm1_O2(i32 %x, i32 %y) nounwind {
+; FAST-INCDEC-LABEL: imm1_O2:
+; FAST-INCDEC:       # %bb.0:
+; FAST-INCDEC-NEXT:    # kill: def $esi killed $esi def $rsi
+; FAST-INCDEC-NEXT:    # kill: def $edi killed $edi def $rdi
+; FAST-INCDEC-NEXT:    leal 1(%rdi), %eax
+; FAST-INCDEC-NEXT:    incl %esi
+; FAST-INCDEC-NEXT:    addq %rsi, %rax
+; FAST-INCDEC-NEXT:    retq
+;
+; SLOW-INCDEC-LABEL: imm1_O2:
+; SLOW-INCDEC:       # %bb.0:
+; SLOW-INCDEC-NEXT:    # kill: def $esi killed $esi def $rsi
+; SLOW-INCDEC-NEXT:    # kill: def $edi killed $edi def $rdi
+; SLOW-INCDEC-NEXT:    leal 1(%rdi), %eax
+; SLOW-INCDEC-NEXT:    addl $1, %esi
+; SLOW-INCDEC-NEXT:    addq %rsi, %rax
+; SLOW-INCDEC-NEXT:    retq
+  %x1 = add i32 %x, 1
+  %y1 = add i32 %y, 1
+  %x1z = zext i32 %x1 to i64
+  %y1z = zext i32 %y1 to i64
+  %r = add i64 %x1z, %y1z
+  ret i64 %r
+}
diff --git a/llvm/test/CodeGen/X86/pr43157.ll b/llvm/test/CodeGen/X86/pr43157.ll
new file mode 100644
index 000000000..9510ed81b
--- /dev/null
+++ b/llvm/test/CodeGen/X86/pr43157.ll
@@ -0,0 +1,20 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-pc-linux -o - -mattr=+mmx | FileCheck %s
+
+define void @foo(fp128 %x) {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    movaps {{.*}}(%rip), %xmm1
+; CHECK-NEXT:    callq __multf3
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
+entry:
+  %mul = fmul fp128 %x, 0xL00000000000000003FFF800000000000
+  tail call void asm sideeffect "", "x,~{dirflag},~{fpsr},~{flags}"(fp128 %mul)
+  ret void
+}
diff --git a/llvm/test/CodeGen/X86/pr43507.ll b/llvm/test/CodeGen/X86/pr43507.ll
new file mode 100644
index 000000000..ec18d3c13
--- /dev/null
+++ b/llvm/test/CodeGen/X86/pr43507.ll
@@ -0,0 +1,18 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=skx | FileCheck %s
+
+define <8 x i1> @ham(i64 %arg) {
+; CHECK-LABEL: ham:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    testb $1, %dil
+; CHECK-NEXT:    movl $255, %ecx
+; CHECK-NEXT:    cmovel %eax, %ecx
+; CHECK-NEXT:    kmovd %ecx, %k0
+; CHECK-NEXT:    vpmovm2w %k0, %xmm0
+; CHECK-NEXT:    retq
+  %tmp = trunc i64 %arg to i1
+  %tmp1 = insertelement <8 x i1> undef, i1 %tmp, i32 0
+  %tmp2 = shufflevector <8 x i1> %tmp1, <8 x i1> undef, <8 x i32> zeroinitializer
+  ret <8 x i1> %tmp2
+}
diff --git a/llvm/test/CodeGen/X86/pr43509.ll b/llvm/test/CodeGen/X86/pr43509.ll
new file mode 100644
index 000000000..4243764d0
--- /dev/null
+++ b/llvm/test/CodeGen/X86/pr43509.ll
@@ -0,0 +1,25 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=skx | FileCheck %s
+
+define <8 x i8> @foo(<8 x float> %arg) {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    vcmpgtps {{.*}}(%rip){1to8}, %ymm0, %k0
+; CHECK-NEXT:    vpmovm2b %k0, %xmm1
+; CHECK-NEXT:    vxorps %xmm2, %xmm2, %xmm2
+; CHECK-NEXT:    vcmpltps %ymm2, %ymm0, %k1
+; CHECK-NEXT:    vmovdqu8 {{.*}}(%rip), %xmm0 {%k1} {z}
+; CHECK-NEXT:    vpand %xmm0, %xmm1, %xmm0
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  %tmp = xor <8 x i8> zeroinitializer, <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
+  %tmp1 = fcmp reassoc nsz contract ogt <8 x float> %arg, <float 8.000000e+00, float 8.000000e+00, float 8.000000e+00, float 8.000000e+00, float 8.000000e+00, float 8.000000e+00, float 8.000000e+00, float 8.000000e+00>
+  %tmp2 = zext <8 x i1> %tmp1 to <8 x i8>
+  %tmp3 = and <8 x i8> %tmp, %tmp2
+  %tmp4 = fcmp reassoc nsz contract ogt <8 x float> zeroinitializer, %arg
+  %tmp5 = or <8 x i1> zeroinitializer, %tmp4
+  %tmp6 = zext <8 x i1> %tmp5 to <8 x i8>
+  %tmp7 = and <8 x i8> %tmp3, %tmp6
+  ret <8 x i8> %tmp7
+}
diff --git a/llvm/test/CodeGen/X86/pr43529.ll b/llvm/test/CodeGen/X86/pr43529.ll
new file mode 100644
index 000000000..afccf5e46
--- /dev/null
+++ b/llvm/test/CodeGen/X86/pr43529.ll
@@ -0,0 +1,39 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i386-unknown-linux-gnu | FileCheck %s
+
+define i32 @a() nounwind {
+; CHECK-LABEL: a:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushl %esi
+; CHECK-NEXT:    subl $8, %esp
+; CHECK-NEXT:    leal {{[0-9]+}}(%esp), %esi
+; CHECK-NEXT:    movl %esi, %eax
+; CHECK-NEXT:    subl $a, %eax
+; CHECK-NEXT:    calll d
+; CHECK-NEXT:    cmpl $a, %esi
+; CHECK-NEXT:    jbe .LBB0_2
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB0_1: # %for.cond
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    jmp .LBB0_1
+; CHECK-NEXT:  .LBB0_2: # %for.end.split
+; CHECK-NEXT:    addl $8, %esp
+; CHECK-NEXT:    popl %esi
+; CHECK-NEXT:    retl
+entry:
+  %b = alloca i32, align 4
+  %0 = bitcast i32* %b to i8*
+  %1 = ptrtoint i32* %b to i32
+  %sub = sub nsw i32 %1, ptrtoint (i32 ()* @a to i32)
+  %call = call i32 bitcast (i32 (...)* @d to i32 (i32)*)(i32 inreg %sub)
+  %cmp = icmp ugt i32* %b, bitcast (i32 ()* @a to i32*)
+  br i1 %cmp, label %for.cond, label %for.end.split
+
+for.cond:                                         ; preds = %entry, %for.cond
+  br label %for.cond
+
+for.end.split:                                    ; preds = %entry
+  ret i32 undef
+}
+
+declare i32 @d(...)
diff --git a/llvm/test/CodeGen/X86/pr43575.ll b/llvm/test/CodeGen/X86/pr43575.ll
new file mode 100644
index 000000000..00c70c697
--- /dev/null
+++ b/llvm/test/CodeGen/X86/pr43575.ll
@@ -0,0 +1,14 @@
+; RUN: llc < %s -mtriple=x86_64-apple-macosx10.14.0 -O0 | FileCheck %s
+
+define void @exit(i32 %status)
+; CHECK-LABEL: exit:
+; CHECK:       ## %bb.0:
+; CHECK:    ## InlineAsm Start
+; CHECK:    movq $60, %rax
+; CHECK:    syscall
+; CHECK:    ## InlineAsm End
+; CHECK:    retq
+{
+    call void asm sideeffect inteldialect "mov rax, 60; syscall", ""()
+    ret void
+}
diff --git a/llvm/test/CodeGen/X86/pr43820.ll b/llvm/test/CodeGen/X86/pr43820.ll
new file mode 100644
index 000000000..5bdf7872d
--- /dev/null
+++ b/llvm/test/CodeGen/X86/pr43820.ll
@@ -0,0 +1,383 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu | FileCheck %s
+
+define i1000 @square(i1000 %A) nounwind {
+; CHECK-LABEL: square:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    movq {{[0-9]+}}(%rsp), %r10
+; CHECK-NEXT:    movq {{[0-9]+}}(%rsp), %rbx
+; CHECK-NEXT:    bswapq %rbx
+; CHECK-NEXT:    movabsq $1085102592571150095, %rdi # imm = 0xF0F0F0F0F0F0F0F
+; CHECK-NEXT:    movq %rbx, %rbp
+; CHECK-NEXT:    andq %rdi, %rbp
+; CHECK-NEXT:    shlq $4, %rbp
+; CHECK-NEXT:    movabsq $-1085102592571150096, %r11 # imm = 0xF0F0F0F0F0F0F0F0
+; CHECK-NEXT:    andq %r11, %rbx
+; CHECK-NEXT:    movq %r11, %rax
+; CHECK-NEXT:    shrq $4, %rbx
+; CHECK-NEXT:    orq %rbp, %rbx
+; CHECK-NEXT:    movabsq $3689348814741910323, %r11 # imm = 0x3333333333333333
+; CHECK-NEXT:    movq %rbx, %r14
+; CHECK-NEXT:    andq %r11, %r14
+; CHECK-NEXT:    movabsq $-3689348814741910324, %rbp # imm = 0xCCCCCCCCCCCCCCCC
+; CHECK-NEXT:    andq %rbp, %rbx
+; CHECK-NEXT:    movq %rbp, %r15
+; CHECK-NEXT:    shrq $2, %rbx
+; CHECK-NEXT:    leaq (%rbx,%r14,4), %r14
+; CHECK-NEXT:    movabsq $6148914691230924800, %rbx # imm = 0x5555555555000000
+; CHECK-NEXT:    andq %r14, %rbx
+; CHECK-NEXT:    movabsq $-6148914691247702016, %rbp # imm = 0xAAAAAAAAAA000000
+; CHECK-NEXT:    andq %r14, %rbp
+; CHECK-NEXT:    shrq %rbp
+; CHECK-NEXT:    leaq (%rbp,%rbx,2), %rbx
+; CHECK-NEXT:    movq %rbx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    bswapq %r10
+; CHECK-NEXT:    movq %r10, %rbx
+; CHECK-NEXT:    andq %rdi, %rbx
+; CHECK-NEXT:    shlq $4, %rbx
+; CHECK-NEXT:    andq %rax, %r10
+; CHECK-NEXT:    shrq $4, %r10
+; CHECK-NEXT:    orq %rbx, %r10
+; CHECK-NEXT:    movq %r10, %rbx
+; CHECK-NEXT:    andq %r11, %rbx
+; CHECK-NEXT:    andq %r15, %r10
+; CHECK-NEXT:    shrq $2, %r10
+; CHECK-NEXT:    leaq (%r10,%rbx,4), %rbp
+; CHECK-NEXT:    movabsq $6148914691236517205, %rbx # imm = 0x5555555555555555
+; CHECK-NEXT:    movq %rbp, %r10
+; CHECK-NEXT:    andq %rbx, %r10
+; CHECK-NEXT:    movabsq $-6148914691236517206, %r13 # imm = 0xAAAAAAAAAAAAAAAA
+; CHECK-NEXT:    andq %r13, %rbp
+; CHECK-NEXT:    shrq %rbp
+; CHECK-NEXT:    leaq (%rbp,%r10,2), %rbp
+; CHECK-NEXT:    movq %rbp, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    movq {{[0-9]+}}(%rsp), %rbp
+; CHECK-NEXT:    bswapq %rbp
+; CHECK-NEXT:    movq %rbp, %r10
+; CHECK-NEXT:    andq %rdi, %r10
+; CHECK-NEXT:    shlq $4, %r10
+; CHECK-NEXT:    andq %rax, %rbp
+; CHECK-NEXT:    shrq $4, %rbp
+; CHECK-NEXT:    orq %r10, %rbp
+; CHECK-NEXT:    movq %rbp, %r10
+; CHECK-NEXT:    andq %r11, %r10
+; CHECK-NEXT:    andq %r15, %rbp
+; CHECK-NEXT:    shrq $2, %rbp
+; CHECK-NEXT:    leaq (%rbp,%r10,4), %rbp
+; CHECK-NEXT:    movq %rbp, %r10
+; CHECK-NEXT:    andq %rbx, %r10
+; CHECK-NEXT:    andq %r13, %rbp
+; CHECK-NEXT:    shrq %rbp
+; CHECK-NEXT:    leaq (%rbp,%r10,2), %rbp
+; CHECK-NEXT:    movq %rbp, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    movq {{[0-9]+}}(%rsp), %rbp
+; CHECK-NEXT:    bswapq %rbp
+; CHECK-NEXT:    movq %rbp, %r10
+; CHECK-NEXT:    andq %rdi, %r10
+; CHECK-NEXT:    shlq $4, %r10
+; CHECK-NEXT:    andq %rax, %rbp
+; CHECK-NEXT:    shrq $4, %rbp
+; CHECK-NEXT:    orq %r10, %rbp
+; CHECK-NEXT:    movq %rbp, %r10
+; CHECK-NEXT:    andq %r11, %r10
+; CHECK-NEXT:    andq %r15, %rbp
+; CHECK-NEXT:    shrq $2, %rbp
+; CHECK-NEXT:    leaq (%rbp,%r10,4), %rbp
+; CHECK-NEXT:    movq %rbp, %r10
+; CHECK-NEXT:    andq %rbx, %r10
+; CHECK-NEXT:    andq %r13, %rbp
+; CHECK-NEXT:    shrq %rbp
+; CHECK-NEXT:    leaq (%rbp,%r10,2), %rbp
+; CHECK-NEXT:    movq %rbp, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    movq {{[0-9]+}}(%rsp), %rbp
+; CHECK-NEXT:    bswapq %rbp
+; CHECK-NEXT:    movq %rbp, %r10
+; CHECK-NEXT:    andq %rdi, %r10
+; CHECK-NEXT:    shlq $4, %r10
+; CHECK-NEXT:    andq %rax, %rbp
+; CHECK-NEXT:    shrq $4, %rbp
+; CHECK-NEXT:    orq %r10, %rbp
+; CHECK-NEXT:    movq %rbp, %r10
+; CHECK-NEXT:    andq %r11, %r10
+; CHECK-NEXT:    andq %r15, %rbp
+; CHECK-NEXT:    shrq $2, %rbp
+; CHECK-NEXT:    leaq (%rbp,%r10,4), %rbp
+; CHECK-NEXT:    movq %rbp, %r10
+; CHECK-NEXT:    andq %rbx, %r10
+; CHECK-NEXT:    andq %r13, %rbp
+; CHECK-NEXT:    shrq %rbp
+; CHECK-NEXT:    leaq (%rbp,%r10,2), %rbp
+; CHECK-NEXT:    movq %rbp, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    movq {{[0-9]+}}(%rsp), %rbp
+; CHECK-NEXT:    bswapq %rbp
+; CHECK-NEXT:    movq %rbp, %r10
+; CHECK-NEXT:    andq %rdi, %r10
+; CHECK-NEXT:    shlq $4, %r10
+; CHECK-NEXT:    andq %rax, %rbp
+; CHECK-NEXT:    movq %rax, %r14
+; CHECK-NEXT:    shrq $4, %rbp
+; CHECK-NEXT:    orq %r10, %rbp
+; CHECK-NEXT:    movq %rbp, %r10
+; CHECK-NEXT:    andq %r11, %r10
+; CHECK-NEXT:    andq %r15, %rbp
+; CHECK-NEXT:    shrq $2, %rbp
+; CHECK-NEXT:    leaq (%rbp,%r10,4), %rbp
+; CHECK-NEXT:    movq %rbp, %r10
+; CHECK-NEXT:    andq %rbx, %r10
+; CHECK-NEXT:    andq %r13, %rbp
+; CHECK-NEXT:    shrq %rbp
+; CHECK-NEXT:    leaq (%rbp,%r10,2), %rax
+; CHECK-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    movq {{[0-9]+}}(%rsp), %rbp
+; CHECK-NEXT:    bswapq %rbp
+; CHECK-NEXT:    movq %rbp, %r10
+; CHECK-NEXT:    andq %rdi, %r10
+; CHECK-NEXT:    shlq $4, %r10
+; CHECK-NEXT:    andq %r14, %rbp
+; CHECK-NEXT:    shrq $4, %rbp
+; CHECK-NEXT:    orq %r10, %rbp
+; CHECK-NEXT:    movq %rbp, %r10
+; CHECK-NEXT:    andq %r11, %r10
+; CHECK-NEXT:    andq %r15, %rbp
+; CHECK-NEXT:    shrq $2, %rbp
+; CHECK-NEXT:    leaq (%rbp,%r10,4), %rbp
+; CHECK-NEXT:    movq %rbp, %r10
+; CHECK-NEXT:    andq %rbx, %r10
+; CHECK-NEXT:    andq %r13, %rbp
+; CHECK-NEXT:    shrq %rbp
+; CHECK-NEXT:    leaq (%rbp,%r10,2), %rbp
+; CHECK-NEXT:    movq %rbp, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    movq {{[0-9]+}}(%rsp), %rbp
+; CHECK-NEXT:    bswapq %rbp
+; CHECK-NEXT:    movq %rbp, %r10
+; CHECK-NEXT:    andq %rdi, %r10
+; CHECK-NEXT:    shlq $4, %r10
+; CHECK-NEXT:    andq %r14, %rbp
+; CHECK-NEXT:    shrq $4, %rbp
+; CHECK-NEXT:    orq %r10, %rbp
+; CHECK-NEXT:    movq %rbp, %r10
+; CHECK-NEXT:    andq %r11, %r10
+; CHECK-NEXT:    andq %r15, %rbp
+; CHECK-NEXT:    shrq $2, %rbp
+; CHECK-NEXT:    leaq (%rbp,%r10,4), %rbp
+; CHECK-NEXT:    movq %rbp, %r10
+; CHECK-NEXT:    andq %rbx, %r10
+; CHECK-NEXT:    andq %r13, %rbp
+; CHECK-NEXT:    shrq %rbp
+; CHECK-NEXT:    leaq (%rbp,%r10,2), %rbp
+; CHECK-NEXT:    movq %rbp, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    movq {{[0-9]+}}(%rsp), %rbp
+; CHECK-NEXT:    bswapq %rbp
+; CHECK-NEXT:    movq %rbp, %r10
+; CHECK-NEXT:    andq %rdi, %r10
+; CHECK-NEXT:    shlq $4, %r10
+; CHECK-NEXT:    andq %r14, %rbp
+; CHECK-NEXT:    shrq $4, %rbp
+; CHECK-NEXT:    orq %r10, %rbp
+; CHECK-NEXT:    movq %rbp, %r10
+; CHECK-NEXT:    andq %r11, %r10
+; CHECK-NEXT:    andq %r15, %rbp
+; CHECK-NEXT:    shrq $2, %rbp
+; CHECK-NEXT:    leaq (%rbp,%r10,4), %rbp
+; CHECK-NEXT:    movq %rbp, %r10
+; CHECK-NEXT:    andq %rbx, %r10
+; CHECK-NEXT:    andq %r13, %rbp
+; CHECK-NEXT:    shrq %rbp
+; CHECK-NEXT:    leaq (%rbp,%r10,2), %rbp
+; CHECK-NEXT:    movq %rbp, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    movq {{[0-9]+}}(%rsp), %rbp
+; CHECK-NEXT:    bswapq %rbp
+; CHECK-NEXT:    movq %rbp, %r10
+; CHECK-NEXT:    andq %rdi, %r10
+; CHECK-NEXT:    shlq $4, %r10
+; CHECK-NEXT:    andq %r14, %rbp
+; CHECK-NEXT:    shrq $4, %rbp
+; CHECK-NEXT:    orq %r10, %rbp
+; CHECK-NEXT:    movq %rbp, %r10
+; CHECK-NEXT:    andq %r11, %r10
+; CHECK-NEXT:    andq %r15, %rbp
+; CHECK-NEXT:    shrq $2, %rbp
+; CHECK-NEXT:    leaq (%rbp,%r10,4), %rbp
+; CHECK-NEXT:    movq %rbp, %r10
+; CHECK-NEXT:    andq %rbx, %r10
+; CHECK-NEXT:    andq %r13, %rbp
+; CHECK-NEXT:    shrq %rbp
+; CHECK-NEXT:    leaq (%rbp,%r10,2), %rbp
+; CHECK-NEXT:    movq %rbp, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    movq {{[0-9]+}}(%rsp), %rbp
+; CHECK-NEXT:    bswapq %rbp
+; CHECK-NEXT:    movq %rbp, %r10
+; CHECK-NEXT:    andq %rdi, %r10
+; CHECK-NEXT:    shlq $4, %r10
+; CHECK-NEXT:    andq %r14, %rbp
+; CHECK-NEXT:    shrq $4, %rbp
+; CHECK-NEXT:    orq %r10, %rbp
+; CHECK-NEXT:    movq %rbp, %r10
+; CHECK-NEXT:    andq %r11, %r10
+; CHECK-NEXT:    andq %r15, %rbp
+; CHECK-NEXT:    shrq $2, %rbp
+; CHECK-NEXT:    leaq (%rbp,%r10,4), %rbp
+; CHECK-NEXT:    movq %rbp, %r10
+; CHECK-NEXT:    andq %rbx, %r10
+; CHECK-NEXT:    andq %r13, %rbp
+; CHECK-NEXT:    shrq %rbp
+; CHECK-NEXT:    leaq (%rbp,%r10,2), %rbp
+; CHECK-NEXT:    movq %rbp, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    bswapq %r9
+; CHECK-NEXT:    movq %r9, %rbp
+; CHECK-NEXT:    andq %rdi, %rbp
+; CHECK-NEXT:    shlq $4, %rbp
+; CHECK-NEXT:    andq %r14, %r9
+; CHECK-NEXT:    shrq $4, %r9
+; CHECK-NEXT:    orq %rbp, %r9
+; CHECK-NEXT:    movq %r9, %rbp
+; CHECK-NEXT:    andq %r11, %rbp
+; CHECK-NEXT:    andq %r15, %r9
+; CHECK-NEXT:    shrq $2, %r9
+; CHECK-NEXT:    leaq (%r9,%rbp,4), %rbp
+; CHECK-NEXT:    movq %rbp, %r9
+; CHECK-NEXT:    andq %rbx, %r9
+; CHECK-NEXT:    andq %r13, %rbp
+; CHECK-NEXT:    shrq %rbp
+; CHECK-NEXT:    leaq (%rbp,%r9,2), %rbp
+; CHECK-NEXT:    movq %rbp, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    bswapq %r8
+; CHECK-NEXT:    movq %r8, %rbp
+; CHECK-NEXT:    andq %rdi, %rbp
+; CHECK-NEXT:    shlq $4, %rbp
+; CHECK-NEXT:    andq %r14, %r8
+; CHECK-NEXT:    shrq $4, %r8
+; CHECK-NEXT:    orq %rbp, %r8
+; CHECK-NEXT:    movq %r8, %rbp
+; CHECK-NEXT:    andq %r11, %rbp
+; CHECK-NEXT:    andq %r15, %r8
+; CHECK-NEXT:    movq %r15, %r9
+; CHECK-NEXT:    shrq $2, %r8
+; CHECK-NEXT:    leaq (%r8,%rbp,4), %rbp
+; CHECK-NEXT:    movq %rbp, %r8
+; CHECK-NEXT:    andq %rbx, %r8
+; CHECK-NEXT:    andq %r13, %rbp
+; CHECK-NEXT:    shrq %rbp
+; CHECK-NEXT:    leaq (%rbp,%r8,2), %rbp
+; CHECK-NEXT:    movq %rbp, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    bswapq %rcx
+; CHECK-NEXT:    movq %rcx, %rbp
+; CHECK-NEXT:    andq %rdi, %rbp
+; CHECK-NEXT:    shlq $4, %rbp
+; CHECK-NEXT:    andq %r14, %rcx
+; CHECK-NEXT:    shrq $4, %rcx
+; CHECK-NEXT:    orq %rbp, %rcx
+; CHECK-NEXT:    movq %rcx, %rbp
+; CHECK-NEXT:    andq %r11, %rbp
+; CHECK-NEXT:    andq %r15, %rcx
+; CHECK-NEXT:    shrq $2, %rcx
+; CHECK-NEXT:    leaq (%rcx,%rbp,4), %rcx
+; CHECK-NEXT:    movq %rcx, %rbp
+; CHECK-NEXT:    andq %rbx, %rbp
+; CHECK-NEXT:    andq %r13, %rcx
+; CHECK-NEXT:    shrq %rcx
+; CHECK-NEXT:    leaq (%rcx,%rbp,2), %r15
+; CHECK-NEXT:    bswapq %rdx
+; CHECK-NEXT:    movq %rdx, %rbp
+; CHECK-NEXT:    andq %rdi, %rbp
+; CHECK-NEXT:    shlq $4, %rbp
+; CHECK-NEXT:    andq %r14, %rdx
+; CHECK-NEXT:    shrq $4, %rdx
+; CHECK-NEXT:    orq %rbp, %rdx
+; CHECK-NEXT:    movq %rdx, %rbp
+; CHECK-NEXT:    andq %r11, %rbp
+; CHECK-NEXT:    andq %r9, %rdx
+; CHECK-NEXT:    shrq $2, %rdx
+; CHECK-NEXT:    leaq (%rdx,%rbp,4), %rdx
+; CHECK-NEXT:    movq %rdx, %rbp
+; CHECK-NEXT:    andq %rbx, %rbp
+; CHECK-NEXT:    andq %r13, %rdx
+; CHECK-NEXT:    shrq %rdx
+; CHECK-NEXT:    leaq (%rdx,%rbp,2), %rdx
+; CHECK-NEXT:    bswapq %rsi
+; CHECK-NEXT:    andq %rsi, %rdi
+; CHECK-NEXT:    andq %r14, %rsi
+; CHECK-NEXT:    shlq $4, %rdi
+; CHECK-NEXT:    shrq $4, %rsi
+; CHECK-NEXT:    orq %rdi, %rsi
+; CHECK-NEXT:    andq %rsi, %r11
+; CHECK-NEXT:    andq %r9, %rsi
+; CHECK-NEXT:    shrq $2, %rsi
+; CHECK-NEXT:    leaq (%rsi,%r11,4), %rsi
+; CHECK-NEXT:    andq %rsi, %rbx
+; CHECK-NEXT:    andq %r13, %rsi
+; CHECK-NEXT:    shrq %rsi
+; CHECK-NEXT:    leaq (%rsi,%rbx,2), %r13
+; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r11 # 8-byte Reload
+; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; CHECK-NEXT:    shrdq $24, %rax, %r11
+; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
+; CHECK-NEXT:    shrdq $24, %rcx, %rax
+; CHECK-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rbp # 8-byte Reload
+; CHECK-NEXT:    shrdq $24, %rbp, %rcx
+; CHECK-NEXT:    movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r12 # 8-byte Reload
+; CHECK-NEXT:    shrdq $24, %r12, %rbp
+; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r14 # 8-byte Reload
+; CHECK-NEXT:    shrdq $24, %r14, %r12
+; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rbx # 8-byte Reload
+; CHECK-NEXT:    shrdq $24, %rbx, %r14
+; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r10 # 8-byte Reload
+; CHECK-NEXT:    shrdq $24, %r10, %rbx
+; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r9 # 8-byte Reload
+; CHECK-NEXT:    shrdq $24, %r9, %r10
+; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Reload
+; CHECK-NEXT:    shrdq $24, %r8, %r9
+; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
+; CHECK-NEXT:    shrdq $24, %rdi, %r8
+; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
+; CHECK-NEXT:    shrdq $24, %rsi, %rdi
+; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; CHECK-NEXT:    shrdq $24, %rax, %rsi
+; CHECK-NEXT:    shrdq $24, %r15, %rax
+; CHECK-NEXT:    movq %rax, %rcx
+; CHECK-NEXT:    shrdq $24, %rdx, %r15
+; CHECK-NEXT:    shrdq $24, %r13, %rdx
+; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; CHECK-NEXT:    movq %rdx, 112(%rax)
+; CHECK-NEXT:    movq %r15, 104(%rax)
+; CHECK-NEXT:    movq %rcx, 96(%rax)
+; CHECK-NEXT:    movq %rsi, 88(%rax)
+; CHECK-NEXT:    movq %rdi, 80(%rax)
+; CHECK-NEXT:    movq %r8, 72(%rax)
+; CHECK-NEXT:    movq %r9, 64(%rax)
+; CHECK-NEXT:    movq %r10, 56(%rax)
+; CHECK-NEXT:    movq %rbx, 48(%rax)
+; CHECK-NEXT:    movq %r14, 40(%rax)
+; CHECK-NEXT:    movq %r12, 32(%rax)
+; CHECK-NEXT:    movq %rbp, 24(%rax)
+; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
+; CHECK-NEXT:    movq %rcx, 16(%rax)
+; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
+; CHECK-NEXT:    movq %rcx, 8(%rax)
+; CHECK-NEXT:    movq %r11, (%rax)
+; CHECK-NEXT:    movq %r13, %rcx
+; CHECK-NEXT:    shrq $56, %r13
+; CHECK-NEXT:    movb %r13b, 124(%rax)
+; CHECK-NEXT:    shrq $24, %rcx
+; CHECK-NEXT:    movl %ecx, 120(%rax)
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    retq
+  %Z = call i1000 @llvm.bitreverse.i1000(i1000 %A)
+  ret i1000 %Z
+}
+
+declare i1000 @llvm.bitreverse.i1000(i1000)
diff --git a/llvm/test/CodeGen/X86/pr43866.ll b/llvm/test/CodeGen/X86/pr43866.ll
new file mode 100644
index 000000000..a430975c4
--- /dev/null
+++ b/llvm/test/CodeGen/X86/pr43866.ll
@@ -0,0 +1,37 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx | FileCheck %s
+
+@v2_0 = global <2 x i32> zeroinitializer, align 8
+
+define void @test()  {
+; CHECK-LABEL: test:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movq %rsp, %rbp
+; CHECK-NEXT:    .cfi_def_cfa_register %rbp
+; CHECK-NEXT:    andq $-32, %rsp
+; CHECK-NEXT:    subq $64, %rsp
+; CHECK-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; CHECK-NEXT:    vxorps %xmm2, %xmm2, %xmm2
+; CHECK-NEXT:    vcmptrueps %ymm2, %ymm2, %ymm2
+; CHECK-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
+; CHECK-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[1,0],ymm0[1,0],ymm1[5,4],ymm0[5,4]
+; CHECK-NEXT:    vshufps {{.*#+}} ymm0 = ymm1[2,0],ymm0[0,0],ymm1[6,4],ymm0[4,4]
+; CHECK-NEXT:    vmovaps %ymm0, (%rsp)
+; CHECK-NEXT:    movq %rbp, %rsp
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa %rsp, 8
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+entry:
+  %v8_0 = alloca <8 x i32>, align 32
+  %v8_0.0.v8_0.0..sroa_cast = bitcast <8 x i32>* %v8_0 to i8*
+  %0 = load <2 x i32>, <2 x i32>* @v2_0, align 8
+  %shuffle = shufflevector <2 x i32> %0, <2 x i32> <i32 -1, i32 -1>, <8 x i32> <i32 1, i32 3, i32 0, i32 0, i32 3, i32 3, i32 2, i32 2>
+  store volatile <8 x i32> %shuffle, <8 x i32>* %v8_0, align 32
+  ret void
+}
diff --git a/llvm/test/CodeGen/X86/pr43952.ll b/llvm/test/CodeGen/X86/pr43952.ll
new file mode 100644
index 000000000..9c6b51136
--- /dev/null
+++ b/llvm/test/CodeGen/X86/pr43952.ll
@@ -0,0 +1,16 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-apple-macosx10.14.0 | FileCheck %s
+
+define void @x() {
+; CHECK-LABEL: x:
+; CHECK:       ## %bb.0: ## %start
+; CHECK-NEXT:    ## InlineAsm Start
+; CHECK-EMPTY:
+; CHECK-NEXT:    vpbroadcastq {{.*}}(%rip), %zmm0 {%k1}
+; CHECK-EMPTY:
+; CHECK-NEXT:    ## InlineAsm End
+; CHECK-NEXT:    retq
+start:
+  tail call void asm sideeffect inteldialect "vpbroadcastq  zmm0{k1}, $0", "m,~{dirflag},~{fpsr},~{flags}"(i64 123)
+  ret void
+}
diff --git a/llvm/test/CodeGen/X86/pr44140.ll b/llvm/test/CodeGen/X86/pr44140.ll
new file mode 100644
index 000000000..941f45d2d
--- /dev/null
+++ b/llvm/test/CodeGen/X86/pr44140.ll
@@ -0,0 +1,97 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=znver1 | FileCheck %s
+
+define win64cc void @opaque() {
+; CHECK-LABEL: opaque:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    retq
+  ret void
+}
+
+; We need xmm6 to be live from the loop header across all iterations of the loop.
+; We shouldn't clobber ymm6 inside the loop.
+define i32 @main() {
+; CHECK-LABEL: main:
+; CHECK:       # %bb.0: # %start
+; CHECK-NEXT:    subq $584, %rsp # imm = 0x248
+; CHECK-NEXT:    .cfi_def_cfa_offset 592
+; CHECK-NEXT:    vmovaps {{.*#+}} xmm6 = [1010101010101010101,2020202020202020202]
+; CHECK-NEXT:    xorl %esi, %esi
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB1_1: # %fake-loop
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovups {{[0-9]+}}(%rsp), %ymm0
+; CHECK-NEXT:    vmovups {{[0-9]+}}(%rsp), %ymm1
+; CHECK-NEXT:    vmovups {{[0-9]+}}(%rsp), %ymm7
+; CHECK-NEXT:    vmovups {{[0-9]+}}(%rsp), %ymm2
+; CHECK-NEXT:    vmovups {{[0-9]+}}(%rsp), %ymm3
+; CHECK-NEXT:    vmovups %ymm0, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    vmovups %ymm1, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    vmovups {{[0-9]+}}(%rsp), %ymm1
+; CHECK-NEXT:    vmovups %ymm3, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    vmovups %ymm2, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    vmovups %ymm7, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    vmovups %ymm3, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    vmovups %ymm2, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    vmovups %ymm7, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    vmovups %ymm1, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    vmovups %ymm1, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    vmovups {{[0-9]+}}(%rsp), %ymm5
+; CHECK-NEXT:    vmovups {{[0-9]+}}(%rsp), %ymm4
+; CHECK-NEXT:    vmovups %ymm5, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    vmovups %ymm4, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    callq opaque
+; CHECK-NEXT:    vmovaps %xmm6, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    testb %sil, %sil
+; CHECK-NEXT:    jne .LBB1_1
+; CHECK-NEXT:  # %bb.2: # %exit
+; CHECK-NEXT:    movabsq $1010101010101010101, %rcx # imm = 0xE04998456557EB5
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    cmpq %rcx, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    sete %al
+; CHECK-NEXT:    decl %eax
+; CHECK-NEXT:    addq $584, %rsp # imm = 0x248
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
+start:
+  %dummy0 = alloca [22 x i64], align 8
+  %dummy1 = alloca [22 x i64], align 8
+  %dummy2 = alloca [22 x i64], align 8
+
+  %data = alloca <2 x i64>, align 8
+
+  br label %fake-loop
+
+fake-loop:                                        ; preds = %fake-loop, %start
+  %dummy0.cast = bitcast [22 x i64]* %dummy0 to i8*
+  %dummy1.cast = bitcast [22 x i64]* %dummy1 to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 8 %dummy1.cast, i8* nonnull align 8 %dummy0.cast, i64 176, i1 false)
+
+  %dummy1.cast.copy = bitcast [22 x i64]* %dummy1 to i8*
+  %dummy2.cast = bitcast [22 x i64]* %dummy2 to i8*
+  call void @llvm.lifetime.start.p0i8(i64 176, i8* nonnull %dummy2.cast)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 8 %dummy2.cast, i8* nonnull align 8 %dummy1.cast.copy, i64 176, i1 false)
+
+  call win64cc void @opaque()
+
+  store <2 x i64> <i64 1010101010101010101, i64 2020202020202020202>, <2 x i64>* %data, align 8
+
+  %opaque-false = icmp eq i8 0, 1
+  br i1 %opaque-false, label %fake-loop, label %exit
+
+exit:                                             ; preds = %fake-loop
+  %data.cast = bitcast <2 x i64>* %data to i64*
+  %0 = load i64, i64* %data.cast, align 8
+  %1 = icmp eq i64 %0, 1010101010101010101
+  %2 = select i1 %1, i32 0, i32 -1
+  ret i32 %2
+}
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture writeonly, i8* nocapture readonly, i64, i1 immarg) #0
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.lifetime.start.p0i8(i64 immarg, i8* nocapture) #0
+
+attributes #0 = { argmemonly nounwind }
diff --git a/llvm/test/CodeGen/X86/pr44396.ll b/llvm/test/CodeGen/X86/pr44396.ll
new file mode 100644
index 000000000..d42d7513a
--- /dev/null
+++ b/llvm/test/CodeGen/X86/pr44396.ll
@@ -0,0 +1,45 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i386-apple-macosx10.15.0 -mattr=+cmov | FileCheck %s
+
+@b = global i32 0, align 4
+@a = global i64 0, align 8
+
+define double @c() nounwind {
+; CHECK-LABEL: c:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    pushl %esi
+; CHECK-NEXT:    subl $16, %esp
+; CHECK-NEXT:    movl _b, %eax
+; CHECK-NEXT:    movl %eax, %ecx
+; CHECK-NEXT:    sarl $31, %ecx
+; CHECK-NEXT:    movl _a+4, %edx
+; CHECK-NEXT:    movl _a, %esi
+; CHECK-NEXT:    subl %eax, %esi
+; CHECK-NEXT:    sbbl %ecx, %edx
+; CHECK-NEXT:    setb %al
+; CHECK-NEXT:    movl %esi, (%esp)
+; CHECK-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    shrl $31, %edx
+; CHECK-NEXT:    fildll (%esp)
+; CHECK-NEXT:    fadds LCPI0_0(,%edx,4)
+; CHECK-NEXT:    fstpl {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fldl {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fldz
+; CHECK-NEXT:    testb %al, %al
+; CHECK-NEXT:    fxch %st(1)
+; CHECK-NEXT:    fcmovne %st(1), %st
+; CHECK-NEXT:    fstp %st(1)
+; CHECK-NEXT:    addl $16, %esp
+; CHECK-NEXT:    popl %esi
+; CHECK-NEXT:    retl
+entry:
+  %0 = load i32, i32* @b, align 4
+  %conv = sext i32 %0 to i64
+  %1 = load i64, i64* @a, align 8
+  %cmp = icmp ult i64 %1, %conv
+  %sub = sub i64 %1, %conv
+  %conv3 = uitofp i64 %sub to double
+  %cond = select i1 %cmp, double 0.000000e+00, double %conv3
+  ret double %cond
+}
+
diff --git a/llvm/test/CodeGen/X86/pr44412.ll b/llvm/test/CodeGen/X86/pr44412.ll
new file mode 100644
index 000000000..dbd0a4e6e
--- /dev/null
+++ b/llvm/test/CodeGen/X86/pr44412.ll
@@ -0,0 +1,68 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+;RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu | FileCheck %s
+
+define void @bar(i32 %0, i32 %1) nounwind {
+; CHECK-LABEL: bar:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    testl %edi, %edi
+; CHECK-NEXT:    je .LBB0_3
+; CHECK-NEXT:  # %bb.1: # %.preheader
+; CHECK-NEXT:    movl %edi, %ebx
+; CHECK-NEXT:    decl %ebx
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB0_2: # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movl %ebx, %edi
+; CHECK-NEXT:    callq foo
+; CHECK-NEXT:    addl $-1, %ebx
+; CHECK-NEXT:    jb .LBB0_2
+; CHECK-NEXT:  .LBB0_3:
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    retq
+  %3 = icmp eq i32 %0, 0
+  br i1 %3, label %8, label %4
+
+4:                                                ; preds = %2, %4
+  %5 = phi i32 [ %6, %4 ], [ %0, %2 ]
+  %6 = add nsw i32 %5, -1
+  tail call void @foo(i32 %6)
+  %7 = icmp eq i32 %6, 0
+  br i1 %7, label %8, label %4
+
+8:                                                ; preds = %4, %2
+  ret void
+}
+
+define void @baz(i32 %0, i32 %1) nounwind {
+; CHECK-LABEL: baz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    testl %edi, %edi
+; CHECK-NEXT:    je .LBB1_3
+; CHECK-NEXT:  # %bb.1: # %.preheader
+; CHECK-NEXT:    movl %edi, %ebx
+; CHECK-NEXT:    decl %ebx
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB1_2: # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movl %ebx, %edi
+; CHECK-NEXT:    callq foo
+; CHECK-NEXT:    addl $-1, %ebx
+; CHECK-NEXT:    jae .LBB1_2
+; CHECK-NEXT:  .LBB1_3:
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    retq
+  %3 = icmp eq i32 %0, 0
+  br i1 %3, label %8, label %4
+
+4:                                                ; preds = %2, %4
+  %5 = phi i32 [ %6, %4 ], [ %0, %2 ]
+  %6 = add nsw i32 %5, -1
+  tail call void @foo(i32 %6)
+  %7 = icmp ne i32 %6, 0
+  br i1 %7, label %8, label %4
+
+8:                                                ; preds = %4, %2
+  ret void
+}
+
+declare void @foo(i32)
diff --git a/llvm/test/CodeGen/X86/pr44812.ll b/llvm/test/CodeGen/X86/pr44812.ll
new file mode 100644
index 000000000..7c4dc67dc
--- /dev/null
+++ b/llvm/test/CodeGen/X86/pr44812.ll
@@ -0,0 +1,24 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i386-unknown-linux-gnu -mattr=cmov | FileCheck %s
+
+define <2 x i32> @foo(<2 x i32> %tmp)  {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    leal 7(%eax), %ecx
+; CHECK-NEXT:    testl %eax, %eax
+; CHECK-NEXT:    cmovnsl %eax, %ecx
+; CHECK-NEXT:    sarl $3, %ecx
+; CHECK-NEXT:    movl $1717986919, %eax # imm = 0x66666667
+; CHECK-NEXT:    imull {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movl %edx, %eax
+; CHECK-NEXT:    shrl $31, %eax
+; CHECK-NEXT:    sarl $2, %edx
+; CHECK-NEXT:    addl %edx, %eax
+; CHECK-NEXT:    movl %ecx, %edx
+; CHECK-NEXT:    retl
+entry:
+  %tmp1 = sdiv <2 x i32> %tmp, <i32 10, i32 8>
+  ret <2 x i32> %tmp1
+}
+
diff --git a/llvm/test/CodeGen/X86/prefer-avx256-mask-shuffle.ll b/llvm/test/CodeGen/X86/prefer-avx256-mask-shuffle.ll
index 7abb3a043..bf5ba184f 100644
--- a/llvm/test/CodeGen/X86/prefer-avx256-mask-shuffle.ll
+++ b/llvm/test/CodeGen/X86/prefer-avx256-mask-shuffle.ll
@@ -153,9 +153,7 @@ define <32 x i1> @shuf32i1_3_6_22_12_3_7_7_0_3_6_1_13_3_21_7_0_3_6_22_12_3_7_7_0
 ; AVX256VL-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[6,7,12,13,u,u,8,9,6,7,14,15,14,15,0,1,22,23,28,29,18,19,26,27,22,23,u,u,30,31,16,17]
 ; AVX256VL-NEXT:    vmovdqa32 %ymm0, %ymm2 {%k1} {z}
 ; AVX256VL-NEXT:    vpmovdw %ymm2, %xmm2
-; AVX256VL-NEXT:    kshiftrw $8, %k1, %k1
-; AVX256VL-NEXT:    vmovdqa32 %ymm0, %ymm3 {%k1} {z}
-; AVX256VL-NEXT:    vpmovdw %ymm3, %xmm3
+; AVX256VL-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
 ; AVX256VL-NEXT:    vinserti128 $1, %xmm3, %ymm2, %ymm2
 ; AVX256VL-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[1,1,2,1]
 ; AVX256VL-NEXT:    vmovdqa {{.*#+}} ymm3 = [255,255,255,255,0,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,255,255,255,255]
diff --git a/llvm/test/CodeGen/X86/prolog-push-seq.ll b/llvm/test/CodeGen/X86/prolog-push-seq.ll
index 99095104d..82662adf1 100644
--- a/llvm/test/CodeGen/X86/prolog-push-seq.ll
+++ b/llvm/test/CodeGen/X86/prolog-push-seq.ll
@@ -16,4 +16,4 @@ define fastcc void @foo(i32 %a, i32 %b) #0 {
   ret void
 }
 
-attributes #0 = { nounwind optsize "no-frame-pointer-elim-non-leaf"}
+attributes #0 = { nounwind optsize "frame-pointer"="all" }
diff --git a/llvm/test/CodeGen/X86/promote-vec3.ll b/llvm/test/CodeGen/X86/promote-vec3.ll
index cb4e99b3b..19eda22b5 100644
--- a/llvm/test/CodeGen/X86/promote-vec3.ll
+++ b/llvm/test/CodeGen/X86/promote-vec3.ll
@@ -11,12 +11,6 @@ define <3 x i16> @zext_i8(<3 x i8>) {
 ; SSE3-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
 ; SSE3-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
 ; SSE3-NEXT:    movzbl {{[0-9]+}}(%esp), %edx
-; SSE3-NEXT:    movd %edx, %xmm0
-; SSE3-NEXT:    pinsrw $1, %ecx, %xmm0
-; SSE3-NEXT:    pinsrw $2, %eax, %xmm0
-; SSE3-NEXT:    pextrw $0, %xmm0, %eax
-; SSE3-NEXT:    pextrw $1, %xmm0, %edx
-; SSE3-NEXT:    pextrw $2, %xmm0, %ecx
 ; SSE3-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE3-NEXT:    # kill: def $dx killed $dx killed $edx
 ; SSE3-NEXT:    # kill: def $cx killed $cx killed $ecx
@@ -24,13 +18,13 @@ define <3 x i16> @zext_i8(<3 x i8>) {
 ;
 ; SSE41-LABEL: zext_i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    pinsrb $0, {{[0-9]+}}(%esp), %xmm0
-; SSE41-NEXT:    pinsrb $4, {{[0-9]+}}(%esp), %xmm0
-; SSE41-NEXT:    pinsrb $8, {{[0-9]+}}(%esp), %xmm0
+; SSE41-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE41-NEXT:    pinsrb $1, {{[0-9]+}}(%esp), %xmm0
+; SSE41-NEXT:    pinsrb $2, {{[0-9]+}}(%esp), %xmm0
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; SSE41-NEXT:    movd %xmm0, %eax
-; SSE41-NEXT:    pextrw $2, %xmm0, %edx
-; SSE41-NEXT:    pextrw $4, %xmm0, %ecx
+; SSE41-NEXT:    pextrw $1, %xmm0, %edx
+; SSE41-NEXT:    pextrw $2, %xmm0, %ecx
 ; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE41-NEXT:    # kill: def $dx killed $dx killed $edx
 ; SSE41-NEXT:    # kill: def $cx killed $cx killed $ecx
@@ -38,13 +32,13 @@ define <3 x i16> @zext_i8(<3 x i8>) {
 ;
 ; AVX-32-LABEL: zext_i8:
 ; AVX-32:       # %bb.0:
-; AVX-32-NEXT:    vpxor %xmm0, %xmm0, %xmm0
-; AVX-32-NEXT:    vpinsrb $0, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT:    vpinsrb $4, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT:    vpinsrb $8, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vpinsrb $1, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrb $2, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; AVX-32-NEXT:    vmovd %xmm0, %eax
-; AVX-32-NEXT:    vpextrw $2, %xmm0, %edx
-; AVX-32-NEXT:    vpextrw $4, %xmm0, %ecx
+; AVX-32-NEXT:    vpextrw $1, %xmm0, %edx
+; AVX-32-NEXT:    vpextrw $2, %xmm0, %ecx
 ; AVX-32-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX-32-NEXT:    # kill: def $dx killed $dx killed $edx
 ; AVX-32-NEXT:    # kill: def $cx killed $cx killed $ecx
@@ -53,12 +47,12 @@ define <3 x i16> @zext_i8(<3 x i8>) {
 ; AVX-64-LABEL: zext_i8:
 ; AVX-64:       # %bb.0:
 ; AVX-64-NEXT:    vmovd %edi, %xmm0
-; AVX-64-NEXT:    vpinsrd $1, %esi, %xmm0, %xmm0
-; AVX-64-NEXT:    vpinsrd $2, %edx, %xmm0, %xmm0
-; AVX-64-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX-64-NEXT:    vpinsrb $1, %esi, %xmm0, %xmm0
+; AVX-64-NEXT:    vpinsrb $2, %edx, %xmm0, %xmm0
+; AVX-64-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; AVX-64-NEXT:    vmovd %xmm0, %eax
-; AVX-64-NEXT:    vpextrw $2, %xmm0, %edx
-; AVX-64-NEXT:    vpextrw $4, %xmm0, %ecx
+; AVX-64-NEXT:    vpextrw $1, %xmm0, %edx
+; AVX-64-NEXT:    vpextrw $2, %xmm0, %ecx
 ; AVX-64-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX-64-NEXT:    # kill: def $dx killed $dx killed $edx
 ; AVX-64-NEXT:    # kill: def $cx killed $cx killed $ecx
@@ -71,13 +65,16 @@ define <3 x i16> @sext_i8(<3 x i8>) {
 ; SSE3-LABEL: sext_i8:
 ; SSE3:       # %bb.0:
 ; SSE3-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; SSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE3-NEXT:    shll $8, %eax
+; SSE3-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; SSE3-NEXT:    shll $8, %ecx
+; SSE3-NEXT:    movd %ecx, %xmm0
 ; SSE3-NEXT:    pinsrw $1, %eax, %xmm0
 ; SSE3-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; SSE3-NEXT:    shll $8, %eax
 ; SSE3-NEXT:    pinsrw $2, %eax, %xmm0
-; SSE3-NEXT:    psllw $8, %xmm0
 ; SSE3-NEXT:    psraw $8, %xmm0
-; SSE3-NEXT:    pextrw $0, %xmm0, %eax
+; SSE3-NEXT:    movd %xmm0, %eax
 ; SSE3-NEXT:    pextrw $1, %xmm0, %edx
 ; SSE3-NEXT:    pextrw $2, %xmm0, %ecx
 ; SSE3-NEXT:    # kill: def $ax killed $ax killed $eax
@@ -88,13 +85,12 @@ define <3 x i16> @sext_i8(<3 x i8>) {
 ; SSE41-LABEL: sext_i8:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE41-NEXT:    pinsrb $4, {{[0-9]+}}(%esp), %xmm0
-; SSE41-NEXT:    pinsrb $8, {{[0-9]+}}(%esp), %xmm0
-; SSE41-NEXT:    pslld $24, %xmm0
-; SSE41-NEXT:    psrad $24, %xmm0
+; SSE41-NEXT:    pinsrb $1, {{[0-9]+}}(%esp), %xmm0
+; SSE41-NEXT:    pinsrb $2, {{[0-9]+}}(%esp), %xmm0
+; SSE41-NEXT:    pmovsxbw %xmm0, %xmm0
 ; SSE41-NEXT:    movd %xmm0, %eax
-; SSE41-NEXT:    pextrw $2, %xmm0, %edx
-; SSE41-NEXT:    pextrw $4, %xmm0, %ecx
+; SSE41-NEXT:    pextrw $1, %xmm0, %edx
+; SSE41-NEXT:    pextrw $2, %xmm0, %ecx
 ; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE41-NEXT:    # kill: def $dx killed $dx killed $edx
 ; SSE41-NEXT:    # kill: def $cx killed $cx killed $ecx
@@ -103,13 +99,12 @@ define <3 x i16> @sext_i8(<3 x i8>) {
 ; AVX-32-LABEL: sext_i8:
 ; AVX-32:       # %bb.0:
 ; AVX-32-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX-32-NEXT:    vpinsrb $4, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT:    vpinsrb $8, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT:    vpslld $24, %xmm0, %xmm0
-; AVX-32-NEXT:    vpsrad $24, %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrb $1, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrb $2, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vpmovsxbw %xmm0, %xmm0
 ; AVX-32-NEXT:    vmovd %xmm0, %eax
-; AVX-32-NEXT:    vpextrw $2, %xmm0, %edx
-; AVX-32-NEXT:    vpextrw $4, %xmm0, %ecx
+; AVX-32-NEXT:    vpextrw $1, %xmm0, %edx
+; AVX-32-NEXT:    vpextrw $2, %xmm0, %ecx
 ; AVX-32-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX-32-NEXT:    # kill: def $dx killed $dx killed $edx
 ; AVX-32-NEXT:    # kill: def $cx killed $cx killed $ecx
@@ -118,13 +113,12 @@ define <3 x i16> @sext_i8(<3 x i8>) {
 ; AVX-64-LABEL: sext_i8:
 ; AVX-64:       # %bb.0:
 ; AVX-64-NEXT:    vmovd %edi, %xmm0
-; AVX-64-NEXT:    vpinsrd $1, %esi, %xmm0, %xmm0
-; AVX-64-NEXT:    vpinsrd $2, %edx, %xmm0, %xmm0
-; AVX-64-NEXT:    vpslld $24, %xmm0, %xmm0
-; AVX-64-NEXT:    vpsrad $24, %xmm0, %xmm0
+; AVX-64-NEXT:    vpinsrb $1, %esi, %xmm0, %xmm0
+; AVX-64-NEXT:    vpinsrb $2, %edx, %xmm0, %xmm0
+; AVX-64-NEXT:    vpmovsxbw %xmm0, %xmm0
 ; AVX-64-NEXT:    vmovd %xmm0, %eax
-; AVX-64-NEXT:    vpextrw $2, %xmm0, %edx
-; AVX-64-NEXT:    vpextrw $4, %xmm0, %ecx
+; AVX-64-NEXT:    vpextrw $1, %xmm0, %edx
+; AVX-64-NEXT:    vpextrw $2, %xmm0, %ecx
 ; AVX-64-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX-64-NEXT:    # kill: def $dx killed $dx killed $edx
 ; AVX-64-NEXT:    # kill: def $cx killed $cx killed $ecx
diff --git a/llvm/test/CodeGen/X86/promote.ll b/llvm/test/CodeGen/X86/promote.ll
index da180d556..a5cff7629 100644
--- a/llvm/test/CodeGen/X86/promote.ll
+++ b/llvm/test/CodeGen/X86/promote.ll
@@ -6,18 +6,19 @@ define i32 @mul_f(<4 x i8>* %A) {
 ; X86-LABEL: mul_f:
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    pmovzxbd {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
-; X86-NEXT:    pmaddwd %xmm0, %xmm0
-; X86-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; X86-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; X86-NEXT:    pmullw %xmm0, %xmm0
+; X86-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,u,u,u,u,u,u,u,u,u,u,u,u]
 ; X86-NEXT:    movd %xmm0, (%eax)
 ; X86-NEXT:    xorl %eax, %eax
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: mul_f:
 ; X64:       # %bb.0: # %entry
-; X64-NEXT:    pmovzxbd {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
-; X64-NEXT:    pmaddwd %xmm0, %xmm0
-; X64-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; X64-NEXT:    pmovzxbw {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
+; X64-NEXT:    pmullw %xmm0, %xmm0
+; X64-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,u,u,u,u,u,u,u,u,u,u,u,u]
 ; X64-NEXT:    movd %xmm0, (%rax)
 ; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    retq
@@ -32,18 +33,16 @@ define i32 @shuff_f(<4 x i8>* %A) {
 ; X86-LABEL: shuff_f:
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    pmovzxbd {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
-; X86-NEXT:    paddd %xmm0, %xmm0
-; X86-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; X86-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-NEXT:    paddb %xmm0, %xmm0
 ; X86-NEXT:    movd %xmm0, (%eax)
 ; X86-NEXT:    xorl %eax, %eax
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: shuff_f:
 ; X64:       # %bb.0: # %entry
-; X64-NEXT:    pmovzxbd {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
-; X64-NEXT:    paddd %xmm0, %xmm0
-; X64-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; X64-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; X64-NEXT:    paddb %xmm0, %xmm0
 ; X64-NEXT:    movd %xmm0, (%rax)
 ; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/psadbw.ll b/llvm/test/CodeGen/X86/psadbw.ll
new file mode 100644
index 000000000..753e88c3d
--- /dev/null
+++ b/llvm/test/CodeGen/X86/psadbw.ll
@@ -0,0 +1,43 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=CHECK,X86
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=CHECK,X64
+
+; Only bottom 16 bits are set - upper 48 bits are zero.
+define <2 x i64> @combine_psadbw_shift(<16 x i8> %0, <16 x i8> %1) {
+; CHECK-LABEL: combine_psadbw_shift:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xorps %xmm0, %xmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %3 = tail call <2 x i64> @llvm.x86.sse2.psad.bw(<16 x i8> %0, <16 x i8> %1)
+  %4 = lshr <2 x i64> %3, <i64 48, i64 48>
+  ret <2 x i64> %4
+}
+
+; Propagate the demanded result elements to the 8 aliasing source elements.
+define i64 @combine_psadbw_demandedelt(<16 x i8> %0, <16 x i8> %1) {
+; X86-LABEL: combine_psadbw_demandedelt:
+; X86:       # %bb.0:
+; X86-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,3,2]
+; X86-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,1,3,2]
+; X86-NEXT:    psadbw %xmm0, %xmm1
+; X86-NEXT:    movd %xmm1, %eax
+; X86-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X86-NEXT:    movd %xmm0, %edx
+; X86-NEXT:    retl
+;
+; X64-LABEL: combine_psadbw_demandedelt:
+; X64:       # %bb.0:
+; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,3,2]
+; X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,1,3,2]
+; X64-NEXT:    psadbw %xmm0, %xmm1
+; X64-NEXT:    movq %xmm1, %rax
+; X64-NEXT:    retq
+  %3 = shufflevector <16 x i8> %0, <16 x i8> %0, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11>
+  %4 = shufflevector <16 x i8> %1, <16 x i8> %1, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11>
+  %5 = tail call <2 x i64> @llvm.x86.sse2.psad.bw(<16 x i8> %3, <16 x i8> %4)
+  %6 = extractelement <2 x i64> %5, i32 0
+  ret i64 %6
+}
+
+declare <2 x i64> @llvm.x86.sse2.psad.bw(<16 x i8>, <16 x i8>)
+
diff --git a/llvm/test/CodeGen/X86/psubus.ll b/llvm/test/CodeGen/X86/psubus.ll
index 1009b1ef9..c765412a7 100644
--- a/llvm/test/CodeGen/X86/psubus.ll
+++ b/llvm/test/CodeGen/X86/psubus.ll
@@ -622,7 +622,7 @@ define <16 x i8> @test14(<16 x i8> %x, <16 x i32> %y) nounwind {
 ; SSE2-NEXT:    psubd %xmm5, %xmm4
 ; SSE2-NEXT:    por %xmm0, %xmm5
 ; SSE2-NEXT:    pcmpgtd %xmm9, %xmm5
-; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [255,255,255,255]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0]
 ; SSE2-NEXT:    pand %xmm9, %xmm5
 ; SSE2-NEXT:    movdqa %xmm3, %xmm7
 ; SSE2-NEXT:    pxor %xmm0, %xmm7
@@ -1671,55 +1671,54 @@ define <8 x i16> @psubus_8i64_max(<8 x i16> %x, <8 x i64> %y) nounwind {
 ; SSE41-LABEL: psubus_8i64_max:
 ; SSE41:       # %bb.0: # %vector.ph
 ; SSE41-NEXT:    movdqa %xmm0, %xmm8
-; SSE41-NEXT:    movdqa {{.*#+}} xmm6 = [9223372039002259456,9223372039002259456]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm9 = [9223372039002259456,9223372039002259456]
 ; SSE41-NEXT:    movdqa %xmm4, %xmm0
-; SSE41-NEXT:    pxor %xmm6, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm9 = [9223372039002324991,9223372039002324991]
-; SSE41-NEXT:    movdqa %xmm9, %xmm7
+; SSE41-NEXT:    pxor %xmm9, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm5 = [9223372039002324991,9223372039002324991]
+; SSE41-NEXT:    movdqa %xmm5, %xmm7
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm7
+; SSE41-NEXT:    movdqa %xmm5, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
+; SSE41-NEXT:    pand %xmm7, %xmm0
+; SSE41-NEXT:    por %xmm6, %xmm0
+; SSE41-NEXT:    movapd {{.*#+}} xmm6 = [65535,65535]
+; SSE41-NEXT:    movapd %xmm6, %xmm10
+; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm10
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    pxor %xmm9, %xmm0
+; SSE41-NEXT:    movdqa %xmm5, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT:    movdqa %xmm5, %xmm7
 ; SSE41-NEXT:    pcmpgtd %xmm0, %xmm7
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm7[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm9, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
 ; SSE41-NEXT:    por %xmm7, %xmm0
-; SSE41-NEXT:    movapd {{.*#+}} xmm7 = [65535,65535]
-; SSE41-NEXT:    movapd %xmm7, %xmm5
-; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm5
-; SSE41-NEXT:    movdqa %xmm3, %xmm0
-; SSE41-NEXT:    pxor %xmm6, %xmm0
-; SSE41-NEXT:    movdqa %xmm9, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm10 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm9, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm10, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
-; SSE41-NEXT:    movapd %xmm7, %xmm4
+; SSE41-NEXT:    movapd %xmm6, %xmm4
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm4
-; SSE41-NEXT:    packusdw %xmm5, %xmm4
+; SSE41-NEXT:    packusdw %xmm10, %xmm4
 ; SSE41-NEXT:    movdqa %xmm2, %xmm0
-; SSE41-NEXT:    pxor %xmm6, %xmm0
-; SSE41-NEXT:    movdqa %xmm9, %xmm3
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm9, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm0
-; SSE41-NEXT:    por %xmm3, %xmm0
-; SSE41-NEXT:    movapd %xmm7, %xmm3
+; SSE41-NEXT:    pxor %xmm9, %xmm0
+; SSE41-NEXT:    movdqa %xmm5, %xmm3
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm3
+; SSE41-NEXT:    movdqa %xmm5, %xmm7
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm7
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[0,0,2,2]
+; SSE41-NEXT:    pand %xmm3, %xmm0
+; SSE41-NEXT:    por %xmm7, %xmm0
+; SSE41-NEXT:    movapd %xmm6, %xmm3
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm3
-; SSE41-NEXT:    pxor %xmm1, %xmm6
-; SSE41-NEXT:    movdqa %xmm9, %xmm2
-; SSE41-NEXT:    pcmpgtd %xmm6, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm9, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm0
-; SSE41-NEXT:    por %xmm2, %xmm0
-; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm7
-; SSE41-NEXT:    packusdw %xmm3, %xmm7
-; SSE41-NEXT:    packusdw %xmm4, %xmm7
-; SSE41-NEXT:    psubusw %xmm7, %xmm8
+; SSE41-NEXT:    pxor %xmm1, %xmm9
+; SSE41-NEXT:    movdqa %xmm5, %xmm2
+; SSE41-NEXT:    pcmpeqd %xmm9, %xmm2
+; SSE41-NEXT:    pcmpgtd %xmm9, %xmm5
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2]
+; SSE41-NEXT:    pand %xmm2, %xmm0
+; SSE41-NEXT:    por %xmm5, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm6
+; SSE41-NEXT:    packusdw %xmm3, %xmm6
+; SSE41-NEXT:    packusdw %xmm4, %xmm6
+; SSE41-NEXT:    psubusw %xmm6, %xmm8
 ; SSE41-NEXT:    movdqa %xmm8, %xmm0
 ; SSE41-NEXT:    retq
 ;
@@ -2185,30 +2184,13 @@ define void @subus_v8i8(<8 x i8>* %p1, <8 x i8>* %p2) {
 ; SSE-NEXT:    movq %xmm0, (%rdi)
 ; SSE-NEXT:    retq
 ;
-; AVX1-LABEL: subus_v8i8:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
-; AVX1-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
-; AVX1-NEXT:    vpsubusb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vmovq %xmm0, (%rdi)
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: subus_v8i8:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
-; AVX2-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
-; AVX2-NEXT:    vpsubusb %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vmovq %xmm0, (%rdi)
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: subus_v8i8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
-; AVX512-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
-; AVX512-NEXT:    vpsubusb %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX512-NEXT:    vpmovwb %xmm0, (%rdi)
-; AVX512-NEXT:    retq
+; AVX-LABEL: subus_v8i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
+; AVX-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
+; AVX-NEXT:    vpsubusb %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vmovq %xmm0, (%rdi)
+; AVX-NEXT:    retq
   %ld1 = load <8 x i8>, <8 x i8>* %p1, align 8
   %ld2 = load <8 x i8>, <8 x i8>* %p2, align 8
   %1 = sub <8 x i8> %ld1, %ld2
@@ -2221,36 +2203,19 @@ define void @subus_v8i8(<8 x i8>* %p1, <8 x i8>* %p2) {
 define void @subus_v4i8(<4 x i8>* %p1, <4 x i8>* %p2) {
 ; SSE-LABEL: subus_v4i8:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; SSE-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
 ; SSE-NEXT:    psubusb %xmm1, %xmm0
 ; SSE-NEXT:    movd %xmm0, (%rdi)
 ; SSE-NEXT:    retq
 ;
-; AVX1-LABEL: subus_v4i8:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX1-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX1-NEXT:    vpsubusb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vmovd %xmm0, (%rdi)
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: subus_v4i8:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX2-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX2-NEXT:    vpsubusb %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vmovd %xmm0, (%rdi)
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: subus_v4i8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX512-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX512-NEXT:    vpsubusb %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; AVX512-NEXT:    vpmovdb %xmm0, (%rdi)
-; AVX512-NEXT:    retq
+; AVX-LABEL: subus_v4i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
+; AVX-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
+; AVX-NEXT:    vpsubusb %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vmovd %xmm0, (%rdi)
+; AVX-NEXT:    retq
   %ld1 = load <4 x i8>, <4 x i8>* %p1, align 8
   %ld2 = load <4 x i8>, <4 x i8>* %p2, align 8
   %1 = sub <4 x i8> %ld1, %ld2
@@ -2263,8 +2228,8 @@ define void @subus_v4i8(<4 x i8>* %p1, <4 x i8>* %p2) {
 define void @subus_v2i8(<2 x i8>* %p1, <2 x i8>* %p2) {
 ; SSE2-LABEL: subus_v2i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE2-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; SSE2-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
 ; SSE2-NEXT:    psubusb %xmm1, %xmm0
 ; SSE2-NEXT:    movd %xmm0, %eax
 ; SSE2-NEXT:    movw %ax, (%rdi)
@@ -2272,8 +2237,8 @@ define void @subus_v2i8(<2 x i8>* %p1, <2 x i8>* %p2) {
 ;
 ; SSSE3-LABEL: subus_v2i8:
 ; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSSE3-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSSE3-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; SSSE3-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
 ; SSSE3-NEXT:    psubusb %xmm1, %xmm0
 ; SSSE3-NEXT:    movd %xmm0, %eax
 ; SSSE3-NEXT:    movw %ax, (%rdi)
@@ -2281,36 +2246,19 @@ define void @subus_v2i8(<2 x i8>* %p1, <2 x i8>* %p2) {
 ;
 ; SSE41-LABEL: subus_v2i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE41-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE41-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; SSE41-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
 ; SSE41-NEXT:    psubusb %xmm1, %xmm0
 ; SSE41-NEXT:    pextrw $0, %xmm0, (%rdi)
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: subus_v2i8:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX1-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX1-NEXT:    vpsubusb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpextrw $0, %xmm0, (%rdi)
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: subus_v2i8:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX2-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX2-NEXT:    vpsubusb %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpextrw $0, %xmm0, (%rdi)
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: subus_v2i8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX512-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX512-NEXT:    vpsubusb %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT:    vpmovqb %xmm0, (%rdi)
-; AVX512-NEXT:    retq
+; AVX-LABEL: subus_v2i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
+; AVX-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
+; AVX-NEXT:    vpsubusb %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpextrw $0, %xmm0, (%rdi)
+; AVX-NEXT:    retq
   %ld1 = load <2 x i8>, <2 x i8>* %p1, align 8
   %ld2 = load <2 x i8>, <2 x i8>* %p2, align 8
   %1 = sub <2 x i8> %ld1, %ld2
@@ -2329,30 +2277,13 @@ define void @subus_v4i16(<4 x i16>* %p1, <4 x i16>* %p2) {
 ; SSE-NEXT:    movq %xmm0, (%rdi)
 ; SSE-NEXT:    retq
 ;
-; AVX1-LABEL: subus_v4i16:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
-; AVX1-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
-; AVX1-NEXT:    vpsubusw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vmovq %xmm0, (%rdi)
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: subus_v4i16:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
-; AVX2-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
-; AVX2-NEXT:    vpsubusw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vmovq %xmm0, (%rdi)
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: subus_v4i16:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
-; AVX512-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
-; AVX512-NEXT:    vpsubusw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; AVX512-NEXT:    vpmovdw %xmm0, (%rdi)
-; AVX512-NEXT:    retq
+; AVX-LABEL: subus_v4i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
+; AVX-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
+; AVX-NEXT:    vpsubusw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vmovq %xmm0, (%rdi)
+; AVX-NEXT:    retq
   %ld1 = load <4 x i16>, <4 x i16>* %p1, align 8
   %ld2 = load <4 x i16>, <4 x i16>* %p2, align 8
   %1 = sub <4 x i16> %ld1, %ld2
@@ -2365,36 +2296,19 @@ define void @subus_v4i16(<4 x i16>* %p1, <4 x i16>* %p2) {
 define void @subus_v2i16(<2 x i16>* %p1, <2 x i16>* %p2) {
 ; SSE-LABEL: subus_v2i16:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; SSE-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
 ; SSE-NEXT:    psubusw %xmm1, %xmm0
 ; SSE-NEXT:    movd %xmm0, (%rdi)
 ; SSE-NEXT:    retq
 ;
-; AVX1-LABEL: subus_v2i16:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX1-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX1-NEXT:    vpsubusw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vmovd %xmm0, (%rdi)
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: subus_v2i16:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX2-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX2-NEXT:    vpsubusw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vmovd %xmm0, (%rdi)
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: subus_v2i16:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX512-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX512-NEXT:    vpsubusw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; AVX512-NEXT:    vpmovqw %xmm0, (%rdi)
-; AVX512-NEXT:    retq
+; AVX-LABEL: subus_v2i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
+; AVX-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
+; AVX-NEXT:    vpsubusw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vmovd %xmm0, (%rdi)
+; AVX-NEXT:    retq
   %ld1 = load <2 x i16>, <2 x i16>* %p1, align 8
   %ld2 = load <2 x i16>, <2 x i16>* %p2, align 8
   %1 = sub <2 x i16> %ld1, %ld2
diff --git a/llvm/test/CodeGen/X86/ptr-rotate.ll b/llvm/test/CodeGen/X86/ptr-rotate.ll
index fbd13b503..c2365c607 100644
--- a/llvm/test/CodeGen/X86/ptr-rotate.ll
+++ b/llvm/test/CodeGen/X86/ptr-rotate.ll
@@ -1,11 +1,16 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=i386-apple-darwin -mcpu=corei7 -o - < %s | FileCheck %s
 
 define i32 @func(i8* %A) nounwind readnone {
+; CHECK-LABEL: func:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    roll $27, %eax
+; CHECK-NEXT:    retl
 entry:
   %tmp = ptrtoint i8* %A to i32
   %shr = lshr i32 %tmp, 5
   %shl = shl i32 %tmp, 27
   %or = or i32 %shr, %shl
-; CHECK: roll  $27
   ret i32 %or
 }
diff --git a/llvm/test/CodeGen/X86/push-cfi-obj.ll b/llvm/test/CodeGen/X86/push-cfi-obj.ll
index aca5803cb..4b608ef94 100644
--- a/llvm/test/CodeGen/X86/push-cfi-obj.ll
+++ b/llvm/test/CodeGen/X86/push-cfi-obj.ll
@@ -42,4 +42,4 @@ cleanup:
   ret void
 }
 
-attributes #0 = { optsize "no-frame-pointer-elim"="true" }
+attributes #0 = { optsize "frame-pointer"="all" }
diff --git a/llvm/test/CodeGen/X86/push-cfi.ll b/llvm/test/CodeGen/X86/push-cfi.ll
index 97f5630f7..d1e393053 100644
--- a/llvm/test/CodeGen/X86/push-cfi.ll
+++ b/llvm/test/CodeGen/X86/push-cfi.ll
@@ -289,4 +289,4 @@ cleanup:
 }
 
 attributes #0 = { optsize }
-attributes #1 = { optsize "no-frame-pointer-elim"="true" }
+attributes #1 = { optsize "frame-pointer"="all" }
diff --git a/llvm/test/CodeGen/X86/ragreedy-hoist-spill.ll b/llvm/test/CodeGen/X86/ragreedy-hoist-spill.ll
index 9238ab0bf..670477c4c 100644
--- a/llvm/test/CodeGen/X86/ragreedy-hoist-spill.ll
+++ b/llvm/test/CodeGen/X86/ragreedy-hoist-spill.ll
@@ -29,8 +29,8 @@ define i8* @SyFgets(i8* %line, i64 %length, i64 %fid) {
 ; CHECK-NEXT:    .cfi_def_cfa_offset 48
 ; CHECK-NEXT:    pushq %rbx
 ; CHECK-NEXT:    .cfi_def_cfa_offset 56
-; CHECK-NEXT:    subq $536, %rsp ## imm = 0x218
-; CHECK-NEXT:    .cfi_def_cfa_offset 592
+; CHECK-NEXT:    subq $552, %rsp ## imm = 0x228
+; CHECK-NEXT:    .cfi_def_cfa_offset 608
 ; CHECK-NEXT:    .cfi_offset %rbx, -56
 ; CHECK-NEXT:    .cfi_offset %r12, -48
 ; CHECK-NEXT:    .cfi_offset %r13, -40
@@ -54,7 +54,7 @@ define i8* @SyFgets(i8* %line, i64 %length, i64 %fid) {
 ; CHECK-NEXT:    testb %al, %al
 ; CHECK-NEXT:    je LBB0_55
 ; CHECK-NEXT:  LBB0_4: ## %cleanup
-; CHECK-NEXT:    addq $536, %rsp ## imm = 0x218
+; CHECK-NEXT:    addq $552, %rsp ## imm = 0x228
 ; CHECK-NEXT:    popq %rbx
 ; CHECK-NEXT:    popq %r12
 ; CHECK-NEXT:    popq %r13
@@ -68,7 +68,7 @@ define i8* @SyFgets(i8* %line, i64 %length, i64 %fid) {
 ; CHECK-NEXT:    je LBB0_55
 ; CHECK-NEXT:  ## %bb.6: ## %SyTime.exit2720
 ; CHECK-NEXT:    movq %rdx, %rbx
-; CHECK-NEXT:    movq %rdi, %rbp
+; CHECK-NEXT:    movq %rdi, %r14
 ; CHECK-NEXT:    leaq {{[0-9]+}}(%rsp), %rax
 ; CHECK-NEXT:    leaq {{[0-9]+}}(%rsp), %rcx
 ; CHECK-NEXT:    cmpq %rax, %rcx
@@ -78,10 +78,10 @@ define i8* @SyFgets(i8* %line, i64 %length, i64 %fid) {
 ; CHECK-NEXT:    movl $32, %esi
 ; CHECK-NEXT:    callq _memset
 ; CHECK-NEXT:  LBB0_8: ## %while.body.preheader
-; CHECK-NEXT:    movq %rbp, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
 ; CHECK-NEXT:    imulq $1040, %rbx, %rax ## imm = 0x410
 ; CHECK-NEXT:    movq _syBuf@{{.*}}(%rip), %rcx
-; CHECK-NEXT:    leaq 8(%rcx,%rax), %rbx
+; CHECK-NEXT:    leaq 8(%rcx,%rax), %rax
+; CHECK-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
 ; CHECK-NEXT:    movl $1, %r15d
 ; CHECK-NEXT:    movq _syCTRO@{{.*}}(%rip), %rax
 ; CHECK-NEXT:    movb $1, %cl
@@ -92,98 +92,96 @@ define i8* @SyFgets(i8* %line, i64 %length, i64 %fid) {
 ; CHECK-NEXT:    testb %cl, %cl
 ; CHECK-NEXT:    jne LBB0_9
 ; CHECK-NEXT:  ## %bb.10: ## %do.end
-; CHECK-NEXT:    xorl %r14d, %r14d
-; CHECK-NEXT:    testb %r14b, %r14b
+; CHECK-NEXT:    xorl %ebp, %ebp
+; CHECK-NEXT:    testb %bpl, %bpl
 ; CHECK-NEXT:    jne LBB0_11
 ; CHECK-NEXT:  ## %bb.12: ## %while.body200.preheader
-; CHECK-NEXT:    xorl %edx, %edx
-; CHECK-NEXT:    leaq {{.*}}(%rip), %rsi
-; CHECK-NEXT:    leaq {{.*}}(%rip), %rdi
-; CHECK-NEXT:    xorl %ebp, %ebp
-; CHECK-NEXT:    xorl %r13d, %r13d
+; CHECK-NEXT:    xorl %ebx, %ebx
+; CHECK-NEXT:    leaq {{.*}}(%rip), %r13
+; CHECK-NEXT:    movl $0, {{[-0-9]+}}(%r{{[sb]}}p) ## 4-byte Folded Spill
+; CHECK-NEXT:    xorl %r12d, %r12d
+; CHECK-NEXT:    movq %r14, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
 ; CHECK-NEXT:    jmp LBB0_13
 ; CHECK-NEXT:    .p2align 4, 0x90
 ; CHECK-NEXT:  LBB0_20: ## %sw.bb256
 ; CHECK-NEXT:    ## in Loop: Header=BB0_13 Depth=1
-; CHECK-NEXT:    movl %r14d, %r13d
+; CHECK-NEXT:    movl %ebp, %r12d
 ; CHECK-NEXT:  LBB0_21: ## %while.cond197.backedge
 ; CHECK-NEXT:    ## in Loop: Header=BB0_13 Depth=1
 ; CHECK-NEXT:    decl %r15d
 ; CHECK-NEXT:    testl %r15d, %r15d
-; CHECK-NEXT:    movl %r13d, %r14d
+; CHECK-NEXT:    movl %r12d, %ebp
 ; CHECK-NEXT:    jle LBB0_22
 ; CHECK-NEXT:  LBB0_13: ## %while.body200
 ; CHECK-NEXT:    ## =>This Loop Header: Depth=1
-; CHECK-NEXT:    ## Child Loop BB0_30 Depth 2
+; CHECK-NEXT:    ## Child Loop BB0_29 Depth 2
 ; CHECK-NEXT:    ## Child Loop BB0_38 Depth 2
-; CHECK-NEXT:    leal -268(%r14), %eax
+; CHECK-NEXT:    leal -268(%rbp), %eax
 ; CHECK-NEXT:    cmpl $105, %eax
 ; CHECK-NEXT:    ja LBB0_14
 ; CHECK-NEXT:  ## %bb.56: ## %while.body200
 ; CHECK-NEXT:    ## in Loop: Header=BB0_13 Depth=1
-; CHECK-NEXT:    movslq (%rdi,%rax,4), %rax
-; CHECK-NEXT:    addq %rdi, %rax
+; CHECK-NEXT:    movslq (%r13,%rax,4), %rax
+; CHECK-NEXT:    addq %r13, %rax
 ; CHECK-NEXT:    jmpq *%rax
 ; CHECK-NEXT:  LBB0_44: ## %while.cond1037.preheader
 ; CHECK-NEXT:    ## in Loop: Header=BB0_13 Depth=1
-; CHECK-NEXT:    testb %dl, %dl
-; CHECK-NEXT:    movl %r14d, %r13d
+; CHECK-NEXT:    testb %bl, %bl
+; CHECK-NEXT:    movl %ebp, %r12d
 ; CHECK-NEXT:    jne LBB0_21
 ; CHECK-NEXT:    jmp LBB0_55
 ; CHECK-NEXT:    .p2align 4, 0x90
 ; CHECK-NEXT:  LBB0_14: ## %while.body200
 ; CHECK-NEXT:    ## in Loop: Header=BB0_13 Depth=1
-; CHECK-NEXT:    leal 1(%r14), %eax
+; CHECK-NEXT:    leal 1(%rbp), %eax
 ; CHECK-NEXT:    cmpl $21, %eax
 ; CHECK-NEXT:    ja LBB0_20
 ; CHECK-NEXT:  ## %bb.15: ## %while.body200
 ; CHECK-NEXT:    ## in Loop: Header=BB0_13 Depth=1
-; CHECK-NEXT:    movl $-1, %r13d
-; CHECK-NEXT:    movslq (%rsi,%rax,4), %rax
-; CHECK-NEXT:    addq %rsi, %rax
+; CHECK-NEXT:    movl $-1, %r12d
+; CHECK-NEXT:    leaq {{.*}}(%rip), %rcx
+; CHECK-NEXT:    movslq (%rcx,%rax,4), %rax
+; CHECK-NEXT:    addq %rcx, %rax
 ; CHECK-NEXT:    jmpq *%rax
 ; CHECK-NEXT:  LBB0_18: ## %while.cond201.preheader
 ; CHECK-NEXT:    ## in Loop: Header=BB0_13 Depth=1
-; CHECK-NEXT:    movl $1, %r13d
+; CHECK-NEXT:    movl $1, %r12d
 ; CHECK-NEXT:    jmp LBB0_21
 ; CHECK-NEXT:  LBB0_26: ## %sw.bb474
 ; CHECK-NEXT:    ## in Loop: Header=BB0_13 Depth=1
-; CHECK-NEXT:    testb %dl, %dl
-; CHECK-NEXT:    ## implicit-def: $r12
+; CHECK-NEXT:    testb %bl, %bl
+; CHECK-NEXT:    ## implicit-def: $r14
 ; CHECK-NEXT:    jne LBB0_34
 ; CHECK-NEXT:  ## %bb.27: ## %do.body479.preheader
 ; CHECK-NEXT:    ## in Loop: Header=BB0_13 Depth=1
-; CHECK-NEXT:    testb %dl, %dl
-; CHECK-NEXT:    ## implicit-def: $r12
+; CHECK-NEXT:    testb %bl, %bl
+; CHECK-NEXT:    ## implicit-def: $r14
 ; CHECK-NEXT:    jne LBB0_34
 ; CHECK-NEXT:  ## %bb.28: ## %land.rhs485.preheader
 ; CHECK-NEXT:    ## in Loop: Header=BB0_13 Depth=1
 ; CHECK-NEXT:    ## implicit-def: $rax
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jns LBB0_30
-; CHECK-NEXT:    jmp LBB0_55
+; CHECK-NEXT:    jmp LBB0_29
 ; CHECK-NEXT:    .p2align 4, 0x90
 ; CHECK-NEXT:  LBB0_32: ## %do.body479.backedge
-; CHECK-NEXT:    ## in Loop: Header=BB0_30 Depth=2
-; CHECK-NEXT:    leaq 1(%r12), %rax
-; CHECK-NEXT:    testb %dl, %dl
+; CHECK-NEXT:    ## in Loop: Header=BB0_29 Depth=2
+; CHECK-NEXT:    leaq 1(%r14), %rax
+; CHECK-NEXT:    testb %bl, %bl
 ; CHECK-NEXT:    je LBB0_33
-; CHECK-NEXT:  ## %bb.29: ## %land.rhs485
-; CHECK-NEXT:    ## in Loop: Header=BB0_30 Depth=2
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    js LBB0_55
-; CHECK-NEXT:  LBB0_30: ## %cond.true.i.i2780
+; CHECK-NEXT:  LBB0_29: ## %land.rhs485
 ; CHECK-NEXT:    ## Parent Loop BB0_13 Depth=1
 ; CHECK-NEXT:    ## => This Inner Loop Header: Depth=2
-; CHECK-NEXT:    movq %rax, %r12
-; CHECK-NEXT:    testb %dl, %dl
+; CHECK-NEXT:    testb %al, %al
+; CHECK-NEXT:    js LBB0_55
+; CHECK-NEXT:  ## %bb.30: ## %cond.true.i.i2780
+; CHECK-NEXT:    ## in Loop: Header=BB0_29 Depth=2
+; CHECK-NEXT:    movq %rax, %r14
+; CHECK-NEXT:    testb %bl, %bl
 ; CHECK-NEXT:    jne LBB0_32
 ; CHECK-NEXT:  ## %bb.31: ## %lor.rhs500
-; CHECK-NEXT:    ## in Loop: Header=BB0_30 Depth=2
+; CHECK-NEXT:    ## in Loop: Header=BB0_29 Depth=2
 ; CHECK-NEXT:    movl $256, %esi ## imm = 0x100
 ; CHECK-NEXT:    callq ___maskrune
-; CHECK-NEXT:    xorl %edx, %edx
-; CHECK-NEXT:    testb %dl, %dl
+; CHECK-NEXT:    testb %bl, %bl
 ; CHECK-NEXT:    jne LBB0_32
 ; CHECK-NEXT:    jmp LBB0_34
 ; CHECK-NEXT:  LBB0_45: ## %sw.bb1134
@@ -193,23 +191,23 @@ define i8* @SyFgets(i8* %line, i64 %length, i64 %fid) {
 ; CHECK-NEXT:    cmpq %rax, %rcx
 ; CHECK-NEXT:    jb LBB0_55
 ; CHECK-NEXT:  ## %bb.46: ## in Loop: Header=BB0_13 Depth=1
-; CHECK-NEXT:    xorl %ebp, %ebp
-; CHECK-NEXT:    movl $268, %r13d ## imm = 0x10C
+; CHECK-NEXT:    movl $0, {{[-0-9]+}}(%r{{[sb]}}p) ## 4-byte Folded Spill
+; CHECK-NEXT:    movl $268, %r12d ## imm = 0x10C
 ; CHECK-NEXT:    jmp LBB0_21
-; CHECK-NEXT:  LBB0_19: ## %sw.bb243
+; CHECK-NEXT:  LBB0_40: ## %sw.bb566
 ; CHECK-NEXT:    ## in Loop: Header=BB0_13 Depth=1
-; CHECK-NEXT:    movl $2, %r13d
+; CHECK-NEXT:    movl $20, %r12d
 ; CHECK-NEXT:    jmp LBB0_21
-; CHECK-NEXT:  LBB0_40: ## %sw.bb566
+; CHECK-NEXT:  LBB0_19: ## %sw.bb243
 ; CHECK-NEXT:    ## in Loop: Header=BB0_13 Depth=1
-; CHECK-NEXT:    movl $20, %r13d
+; CHECK-NEXT:    movl $2, %r12d
 ; CHECK-NEXT:    jmp LBB0_21
 ; CHECK-NEXT:  LBB0_33: ## %if.end517.loopexitsplit
 ; CHECK-NEXT:    ## in Loop: Header=BB0_13 Depth=1
-; CHECK-NEXT:    incq %r12
+; CHECK-NEXT:    incq %r14
 ; CHECK-NEXT:  LBB0_34: ## %if.end517
 ; CHECK-NEXT:    ## in Loop: Header=BB0_13 Depth=1
-; CHECK-NEXT:    leal -324(%r13), %eax
+; CHECK-NEXT:    leal -324(%r12), %eax
 ; CHECK-NEXT:    cmpl $59, %eax
 ; CHECK-NEXT:    ja LBB0_35
 ; CHECK-NEXT:  ## %bb.57: ## %if.end517
@@ -219,11 +217,11 @@ define i8* @SyFgets(i8* %line, i64 %length, i64 %fid) {
 ; CHECK-NEXT:    jb LBB0_38
 ; CHECK-NEXT:  LBB0_35: ## %if.end517
 ; CHECK-NEXT:    ## in Loop: Header=BB0_13 Depth=1
-; CHECK-NEXT:    cmpl $11, %r13d
+; CHECK-NEXT:    cmpl $11, %r12d
 ; CHECK-NEXT:    je LBB0_38
 ; CHECK-NEXT:  ## %bb.36: ## %if.end517
 ; CHECK-NEXT:    ## in Loop: Header=BB0_13 Depth=1
-; CHECK-NEXT:    cmpl $24, %r13d
+; CHECK-NEXT:    cmpl $24, %r12d
 ; CHECK-NEXT:    je LBB0_38
 ; CHECK-NEXT:  ## %bb.37: ## %if.then532
 ; CHECK-NEXT:    ## in Loop: Header=BB0_13 Depth=1
@@ -233,15 +231,14 @@ define i8* @SyFgets(i8* %line, i64 %length, i64 %fid) {
 ; CHECK-NEXT:  LBB0_38: ## %for.cond534
 ; CHECK-NEXT:    ## Parent Loop BB0_13 Depth=1
 ; CHECK-NEXT:    ## => This Inner Loop Header: Depth=2
-; CHECK-NEXT:    testb %dl, %dl
+; CHECK-NEXT:    testb %bl, %bl
 ; CHECK-NEXT:    jne LBB0_38
 ; CHECK-NEXT:  ## %bb.39: ## %for.cond542.preheader
 ; CHECK-NEXT:    ## in Loop: Header=BB0_13 Depth=1
-; CHECK-NEXT:    testb %dl, %dl
-; CHECK-NEXT:    movb $0, (%r12)
-; CHECK-NEXT:    movl %r14d, %r13d
-; CHECK-NEXT:    leaq {{.*}}(%rip), %rsi
-; CHECK-NEXT:    leaq {{.*}}(%rip), %rdi
+; CHECK-NEXT:    testb %bl, %bl
+; CHECK-NEXT:    movb $0, (%r14)
+; CHECK-NEXT:    movl %ebp, %r12d
+; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r14 ## 8-byte Reload
 ; CHECK-NEXT:    jmp LBB0_21
 ; CHECK-NEXT:    .p2align 4, 0x90
 ; CHECK-NEXT:  LBB0_42: ## %while.cond864
@@ -256,30 +253,44 @@ define i8* @SyFgets(i8* %line, i64 %length, i64 %fid) {
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    jmp LBB0_25
 ; CHECK-NEXT:  LBB0_11:
-; CHECK-NEXT:    xorl %ebp, %ebp
-; CHECK-NEXT:    xorl %r13d, %r13d
+; CHECK-NEXT:    movl $0, {{[-0-9]+}}(%r{{[sb]}}p) ## 4-byte Folded Spill
+; CHECK-NEXT:    xorl %r12d, %r12d
 ; CHECK-NEXT:  LBB0_22: ## %while.end1465
-; CHECK-NEXT:    incl %r13d
-; CHECK-NEXT:    cmpl $16, %r13d
+; CHECK-NEXT:    incl %r12d
+; CHECK-NEXT:    cmpl $16, %r12d
 ; CHECK-NEXT:    ja LBB0_50
 ; CHECK-NEXT:  ## %bb.23: ## %while.end1465
 ; CHECK-NEXT:    movl $83969, %eax ## imm = 0x14801
-; CHECK-NEXT:    btl %r13d, %eax
+; CHECK-NEXT:    btl %r12d, %eax
 ; CHECK-NEXT:    jae LBB0_50
 ; CHECK-NEXT:  ## %bb.24:
-; CHECK-NEXT:    xorl %ebp, %ebp
-; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rbx ## 8-byte Reload
+; CHECK-NEXT:    xorl %ebx, %ebx
 ; CHECK-NEXT:  LBB0_48: ## %if.then1477
 ; CHECK-NEXT:    movl $1, %edx
 ; CHECK-NEXT:    callq _write
-; CHECK-NEXT:    subq %rbp, %rbx
+; CHECK-NEXT:    subq %rbx, %r14
 ; CHECK-NEXT:    movq _syHistory@{{.*}}(%rip), %rax
-; CHECK-NEXT:    leaq 8189(%rbx,%rax), %rax
+; CHECK-NEXT:    leaq 8189(%r14,%rax), %rax
 ; CHECK-NEXT:    .p2align 4, 0x90
 ; CHECK-NEXT:  LBB0_49: ## %for.body1723
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    decq %rax
 ; CHECK-NEXT:    jmp LBB0_49
+; CHECK-NEXT:  LBB0_47: ## %if.then1477.loopexit
+; CHECK-NEXT:    movq %r14, %rbx
+; CHECK-NEXT:    jmp LBB0_48
+; CHECK-NEXT:  LBB0_16: ## %while.cond635.preheader
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    testb %al, %al
+; CHECK-NEXT:    je LBB0_41
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  LBB0_17: ## %for.body643.us
+; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    jmp LBB0_17
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  LBB0_41: ## %while.cond661
+; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    jmp LBB0_41
 ; CHECK-NEXT:  LBB0_50: ## %for.cond1480.preheader
 ; CHECK-NEXT:    movl $512, %eax ## imm = 0x200
 ; CHECK-NEXT:    cmpq %rax, %rax
@@ -289,14 +300,15 @@ define i8* @SyFgets(i8* %line, i64 %length, i64 %fid) {
 ; CHECK-NEXT:    testb %al, %al
 ; CHECK-NEXT:    jne LBB0_54
 ; CHECK-NEXT:  ## %bb.52: ## %while.body1679.preheader
-; CHECK-NEXT:    incl %ebp
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    incl {{[-0-9]+}}(%r{{[sb]}}p) ## 4-byte Folded Spill
 ; CHECK-NEXT:  LBB0_53: ## %while.body1679
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    movq (%rbx), %rdi
+; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax ## 8-byte Reload
+; CHECK-NEXT:    movq (%rax), %rdi
 ; CHECK-NEXT:    callq _fileno
-; CHECK-NEXT:    movslq %ebp, %rax
-; CHECK-NEXT:    leal 1(%rax), %ebp
+; CHECK-NEXT:    movslq {{[-0-9]+}}(%r{{[sb]}}p), %rax ## 4-byte Folded Reload
+; CHECK-NEXT:    leal 1(%rax), %ecx
+; CHECK-NEXT:    movl %ecx, {{[-0-9]+}}(%r{{[sb]}}p) ## 4-byte Spill
 ; CHECK-NEXT:    cmpq %rax, %rax
 ; CHECK-NEXT:    jl LBB0_53
 ; CHECK-NEXT:  LBB0_54: ## %while.cond1683.preheader
@@ -304,22 +316,6 @@ define i8* @SyFgets(i8* %line, i64 %length, i64 %fid) {
 ; CHECK-NEXT:    testb %al, %al
 ; CHECK-NEXT:  LBB0_55: ## %if.then.i
 ; CHECK-NEXT:    ud2
-; CHECK-NEXT:  LBB0_47: ## %if.then1477.loopexit
-; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rbx ## 8-byte Reload
-; CHECK-NEXT:    movq %rbx, %rbp
-; CHECK-NEXT:    jmp LBB0_48
-; CHECK-NEXT:  LBB0_16: ## %while.cond635.preheader
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    je LBB0_41
-; CHECK-NEXT:    .p2align 4, 0x90
-; CHECK-NEXT:  LBB0_17: ## %for.body643.us
-; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    jmp LBB0_17
-; CHECK-NEXT:    .p2align 4, 0x90
-; CHECK-NEXT:  LBB0_41: ## %while.cond661
-; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    jmp LBB0_41
 entry:
   %sub.ptr.rhs.cast646 = ptrtoint i8* %line to i64
   %old = alloca [512 x i8], align 16
diff --git a/llvm/test/CodeGen/X86/ragreedy-last-chance-recoloring.ll b/llvm/test/CodeGen/X86/ragreedy-last-chance-recoloring.ll
index f32875581..a0c915c09 100644
--- a/llvm/test/CodeGen/X86/ragreedy-last-chance-recoloring.ll
+++ b/llvm/test/CodeGen/X86/ragreedy-last-chance-recoloring.ll
@@ -180,4 +180,4 @@ bb439:                                            ; preds = %bb222, %bb85
   ret void
 }
 
-attributes #0 = { nounwind ssp "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind ssp "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/X86/recip-fastmath.ll b/llvm/test/CodeGen/X86/recip-fastmath.ll
index c8908b54c..5d05bd401 100644
--- a/llvm/test/CodeGen/X86/recip-fastmath.ll
+++ b/llvm/test/CodeGen/X86/recip-fastmath.ll
@@ -60,15 +60,15 @@ define float @f32_one_step(float %x) #1 {
 ; FMA-RECIP-LABEL: f32_one_step:
 ; FMA-RECIP:       # %bb.0:
 ; FMA-RECIP-NEXT:    vrcpss %xmm0, %xmm0, %xmm1
-; FMA-RECIP-NEXT:    vfnmadd213ss {{.*#+}} xmm0 = -(xmm1 * xmm0) + mem
-; FMA-RECIP-NEXT:    vfmadd132ss {{.*#+}} xmm0 = (xmm0 * xmm1) + xmm1
+; FMA-RECIP-NEXT:    vfmadd213ss {{.*#+}} xmm0 = (xmm1 * xmm0) + mem
+; FMA-RECIP-NEXT:    vfnmadd132ss {{.*#+}} xmm0 = -(xmm0 * xmm1) + xmm1
 ; FMA-RECIP-NEXT:    retq
 ;
 ; BDVER2-LABEL: f32_one_step:
 ; BDVER2:       # %bb.0:
 ; BDVER2-NEXT:    vrcpss %xmm0, %xmm0, %xmm1
-; BDVER2-NEXT:    vfnmaddss {{.*}}(%rip), %xmm1, %xmm0, %xmm0
-; BDVER2-NEXT:    vfmaddss %xmm1, %xmm0, %xmm1, %xmm0
+; BDVER2-NEXT:    vfmaddss {{.*}}(%rip), %xmm1, %xmm0, %xmm0
+; BDVER2-NEXT:    vfnmaddss %xmm1, %xmm0, %xmm1, %xmm0
 ; BDVER2-NEXT:    retq
 ;
 ; BTVER2-LABEL: f32_one_step:
@@ -94,8 +94,8 @@ define float @f32_one_step(float %x) #1 {
 ; HASWELL-LABEL: f32_one_step:
 ; HASWELL:       # %bb.0:
 ; HASWELL-NEXT:    vrcpss %xmm0, %xmm0, %xmm1
-; HASWELL-NEXT:    vfnmadd213ss {{.*#+}} xmm0 = -(xmm1 * xmm0) + mem
-; HASWELL-NEXT:    vfmadd132ss {{.*#+}} xmm0 = (xmm0 * xmm1) + xmm1
+; HASWELL-NEXT:    vfmadd213ss {{.*#+}} xmm0 = (xmm1 * xmm0) + mem
+; HASWELL-NEXT:    vfnmadd132ss {{.*#+}} xmm0 = -(xmm0 * xmm1) + xmm1
 ; HASWELL-NEXT:    retq
 ;
 ; HASWELL-NO-FMA-LABEL: f32_one_step:
@@ -111,13 +111,103 @@ define float @f32_one_step(float %x) #1 {
 ; AVX512-LABEL: f32_one_step:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vrcpss %xmm0, %xmm0, %xmm1
-; AVX512-NEXT:    vfnmadd213ss {{.*#+}} xmm0 = -(xmm1 * xmm0) + mem
-; AVX512-NEXT:    vfmadd132ss {{.*#+}} xmm0 = (xmm0 * xmm1) + xmm1
+; AVX512-NEXT:    vfmadd213ss {{.*#+}} xmm0 = (xmm1 * xmm0) + mem
+; AVX512-NEXT:    vfnmadd132ss {{.*#+}} xmm0 = -(xmm0 * xmm1) + xmm1
 ; AVX512-NEXT:    retq
   %div = fdiv fast float 1.0, %x
   ret float %div
 }
 
+define float @f32_one_step_variables(float %x, float %y) #1 {
+; SSE-LABEL: f32_one_step_variables:
+; SSE:       # %bb.0:
+; SSE-NEXT:    rcpss %xmm1, %xmm2
+; SSE-NEXT:    movaps %xmm0, %xmm3
+; SSE-NEXT:    mulss %xmm2, %xmm3
+; SSE-NEXT:    mulss %xmm3, %xmm1
+; SSE-NEXT:    subss %xmm1, %xmm0
+; SSE-NEXT:    mulss %xmm2, %xmm0
+; SSE-NEXT:    addss %xmm3, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-RECIP-LABEL: f32_one_step_variables:
+; AVX-RECIP:       # %bb.0:
+; AVX-RECIP-NEXT:    vrcpss %xmm1, %xmm1, %xmm2
+; AVX-RECIP-NEXT:    vmulss %xmm2, %xmm0, %xmm3
+; AVX-RECIP-NEXT:    vmulss %xmm3, %xmm1, %xmm1
+; AVX-RECIP-NEXT:    vsubss %xmm1, %xmm0, %xmm0
+; AVX-RECIP-NEXT:    vmulss %xmm0, %xmm2, %xmm0
+; AVX-RECIP-NEXT:    vaddss %xmm0, %xmm3, %xmm0
+; AVX-RECIP-NEXT:    retq
+;
+; FMA-RECIP-LABEL: f32_one_step_variables:
+; FMA-RECIP:       # %bb.0:
+; FMA-RECIP-NEXT:    vrcpss %xmm1, %xmm1, %xmm2
+; FMA-RECIP-NEXT:    vmulss %xmm2, %xmm0, %xmm3
+; FMA-RECIP-NEXT:    vfnmadd213ss {{.*#+}} xmm1 = -(xmm3 * xmm1) + xmm0
+; FMA-RECIP-NEXT:    vfmadd213ss {{.*#+}} xmm2 = (xmm1 * xmm2) + xmm3
+; FMA-RECIP-NEXT:    vmovaps %xmm2, %xmm0
+; FMA-RECIP-NEXT:    retq
+;
+; BDVER2-LABEL: f32_one_step_variables:
+; BDVER2:       # %bb.0:
+; BDVER2-NEXT:    vrcpss %xmm1, %xmm1, %xmm2
+; BDVER2-NEXT:    vmulss %xmm2, %xmm0, %xmm3
+; BDVER2-NEXT:    vfnmaddss %xmm0, %xmm3, %xmm1, %xmm0
+; BDVER2-NEXT:    vfmaddss %xmm3, %xmm0, %xmm2, %xmm0
+; BDVER2-NEXT:    retq
+;
+; BTVER2-LABEL: f32_one_step_variables:
+; BTVER2:       # %bb.0:
+; BTVER2-NEXT:    vrcpss %xmm1, %xmm1, %xmm2
+; BTVER2-NEXT:    vmulss %xmm2, %xmm0, %xmm3
+; BTVER2-NEXT:    vmulss %xmm3, %xmm1, %xmm1
+; BTVER2-NEXT:    vsubss %xmm1, %xmm0, %xmm0
+; BTVER2-NEXT:    vmulss %xmm0, %xmm2, %xmm0
+; BTVER2-NEXT:    vaddss %xmm0, %xmm3, %xmm0
+; BTVER2-NEXT:    retq
+;
+; SANDY-LABEL: f32_one_step_variables:
+; SANDY:       # %bb.0:
+; SANDY-NEXT:    vrcpss %xmm1, %xmm1, %xmm2
+; SANDY-NEXT:    vmulss %xmm2, %xmm0, %xmm3
+; SANDY-NEXT:    vmulss %xmm3, %xmm1, %xmm1
+; SANDY-NEXT:    vsubss %xmm1, %xmm0, %xmm0
+; SANDY-NEXT:    vmulss %xmm0, %xmm2, %xmm0
+; SANDY-NEXT:    vaddss %xmm0, %xmm3, %xmm0
+; SANDY-NEXT:    retq
+;
+; HASWELL-LABEL: f32_one_step_variables:
+; HASWELL:       # %bb.0:
+; HASWELL-NEXT:    vrcpss %xmm1, %xmm1, %xmm2
+; HASWELL-NEXT:    vmulss %xmm2, %xmm0, %xmm3
+; HASWELL-NEXT:    vfnmadd213ss {{.*#+}} xmm1 = -(xmm3 * xmm1) + xmm0
+; HASWELL-NEXT:    vfmadd213ss {{.*#+}} xmm2 = (xmm1 * xmm2) + xmm3
+; HASWELL-NEXT:    vmovaps %xmm2, %xmm0
+; HASWELL-NEXT:    retq
+;
+; HASWELL-NO-FMA-LABEL: f32_one_step_variables:
+; HASWELL-NO-FMA:       # %bb.0:
+; HASWELL-NO-FMA-NEXT:    vrcpss %xmm1, %xmm1, %xmm2
+; HASWELL-NO-FMA-NEXT:    vmulss %xmm2, %xmm0, %xmm3
+; HASWELL-NO-FMA-NEXT:    vmulss %xmm3, %xmm1, %xmm1
+; HASWELL-NO-FMA-NEXT:    vsubss %xmm1, %xmm0, %xmm0
+; HASWELL-NO-FMA-NEXT:    vmulss %xmm0, %xmm2, %xmm0
+; HASWELL-NO-FMA-NEXT:    vaddss %xmm0, %xmm3, %xmm0
+; HASWELL-NO-FMA-NEXT:    retq
+;
+; AVX512-LABEL: f32_one_step_variables:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vrcpss %xmm1, %xmm1, %xmm2
+; AVX512-NEXT:    vmulss %xmm2, %xmm0, %xmm3
+; AVX512-NEXT:    vfnmadd213ss {{.*#+}} xmm1 = -(xmm3 * xmm1) + xmm0
+; AVX512-NEXT:    vfmadd213ss {{.*#+}} xmm2 = (xmm1 * xmm2) + xmm3
+; AVX512-NEXT:    vmovaps %xmm2, %xmm0
+; AVX512-NEXT:    retq
+  %div = fdiv fast float %x, %y
+  ret float %div
+}
+
 define float @f32_two_step(float %x) #2 {
 ; SSE-LABEL: f32_two_step:
 ; SSE:       # %bb.0:
@@ -389,6 +479,96 @@ define <4 x float> @v4f32_one_step(<4 x float> %x) #1 {
   ret <4 x float> %div
 }
 
+define <4 x float> @v4f32_one_step_variables(<4 x float> %x, <4 x float> %y) #1 {
+; SSE-LABEL: v4f32_one_step_variables:
+; SSE:       # %bb.0:
+; SSE-NEXT:    rcpps %xmm1, %xmm2
+; SSE-NEXT:    movaps %xmm0, %xmm3
+; SSE-NEXT:    mulps %xmm2, %xmm3
+; SSE-NEXT:    mulps %xmm3, %xmm1
+; SSE-NEXT:    subps %xmm1, %xmm0
+; SSE-NEXT:    mulps %xmm2, %xmm0
+; SSE-NEXT:    addps %xmm3, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-RECIP-LABEL: v4f32_one_step_variables:
+; AVX-RECIP:       # %bb.0:
+; AVX-RECIP-NEXT:    vrcpps %xmm1, %xmm2
+; AVX-RECIP-NEXT:    vmulps %xmm2, %xmm0, %xmm3
+; AVX-RECIP-NEXT:    vmulps %xmm3, %xmm1, %xmm1
+; AVX-RECIP-NEXT:    vsubps %xmm1, %xmm0, %xmm0
+; AVX-RECIP-NEXT:    vmulps %xmm0, %xmm2, %xmm0
+; AVX-RECIP-NEXT:    vaddps %xmm0, %xmm3, %xmm0
+; AVX-RECIP-NEXT:    retq
+;
+; FMA-RECIP-LABEL: v4f32_one_step_variables:
+; FMA-RECIP:       # %bb.0:
+; FMA-RECIP-NEXT:    vrcpps %xmm1, %xmm2
+; FMA-RECIP-NEXT:    vmulps %xmm2, %xmm0, %xmm3
+; FMA-RECIP-NEXT:    vfnmadd213ps {{.*#+}} xmm1 = -(xmm3 * xmm1) + xmm0
+; FMA-RECIP-NEXT:    vfmadd213ps {{.*#+}} xmm2 = (xmm1 * xmm2) + xmm3
+; FMA-RECIP-NEXT:    vmovaps %xmm2, %xmm0
+; FMA-RECIP-NEXT:    retq
+;
+; BDVER2-LABEL: v4f32_one_step_variables:
+; BDVER2:       # %bb.0:
+; BDVER2-NEXT:    vrcpps %xmm1, %xmm2
+; BDVER2-NEXT:    vmulps %xmm2, %xmm0, %xmm3
+; BDVER2-NEXT:    vfnmaddps %xmm0, %xmm3, %xmm1, %xmm0
+; BDVER2-NEXT:    vfmaddps %xmm3, %xmm0, %xmm2, %xmm0
+; BDVER2-NEXT:    retq
+;
+; BTVER2-LABEL: v4f32_one_step_variables:
+; BTVER2:       # %bb.0:
+; BTVER2-NEXT:    vrcpps %xmm1, %xmm2
+; BTVER2-NEXT:    vmulps %xmm2, %xmm0, %xmm3
+; BTVER2-NEXT:    vmulps %xmm3, %xmm1, %xmm1
+; BTVER2-NEXT:    vsubps %xmm1, %xmm0, %xmm0
+; BTVER2-NEXT:    vmulps %xmm0, %xmm2, %xmm0
+; BTVER2-NEXT:    vaddps %xmm0, %xmm3, %xmm0
+; BTVER2-NEXT:    retq
+;
+; SANDY-LABEL: v4f32_one_step_variables:
+; SANDY:       # %bb.0:
+; SANDY-NEXT:    vrcpps %xmm1, %xmm2
+; SANDY-NEXT:    vmulps %xmm2, %xmm0, %xmm3
+; SANDY-NEXT:    vmulps %xmm3, %xmm1, %xmm1
+; SANDY-NEXT:    vsubps %xmm1, %xmm0, %xmm0
+; SANDY-NEXT:    vmulps %xmm0, %xmm2, %xmm0
+; SANDY-NEXT:    vaddps %xmm0, %xmm3, %xmm0
+; SANDY-NEXT:    retq
+;
+; HASWELL-LABEL: v4f32_one_step_variables:
+; HASWELL:       # %bb.0:
+; HASWELL-NEXT:    vrcpps %xmm1, %xmm2
+; HASWELL-NEXT:    vmulps %xmm2, %xmm0, %xmm3
+; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} xmm1 = -(xmm3 * xmm1) + xmm0
+; HASWELL-NEXT:    vfmadd213ps {{.*#+}} xmm2 = (xmm1 * xmm2) + xmm3
+; HASWELL-NEXT:    vmovaps %xmm2, %xmm0
+; HASWELL-NEXT:    retq
+;
+; HASWELL-NO-FMA-LABEL: v4f32_one_step_variables:
+; HASWELL-NO-FMA:       # %bb.0:
+; HASWELL-NO-FMA-NEXT:    vrcpps %xmm1, %xmm2
+; HASWELL-NO-FMA-NEXT:    vmulps %xmm2, %xmm0, %xmm3
+; HASWELL-NO-FMA-NEXT:    vmulps %xmm3, %xmm1, %xmm1
+; HASWELL-NO-FMA-NEXT:    vsubps %xmm1, %xmm0, %xmm0
+; HASWELL-NO-FMA-NEXT:    vmulps %xmm0, %xmm2, %xmm0
+; HASWELL-NO-FMA-NEXT:    vaddps %xmm0, %xmm3, %xmm0
+; HASWELL-NO-FMA-NEXT:    retq
+;
+; AVX512-LABEL: v4f32_one_step_variables:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vrcpps %xmm1, %xmm2
+; AVX512-NEXT:    vmulps %xmm2, %xmm0, %xmm3
+; AVX512-NEXT:    vfnmadd213ps {{.*#+}} xmm1 = -(xmm3 * xmm1) + xmm0
+; AVX512-NEXT:    vfmadd213ps {{.*#+}} xmm2 = (xmm1 * xmm2) + xmm3
+; AVX512-NEXT:    vmovaps %xmm2, %xmm0
+; AVX512-NEXT:    retq
+  %div = fdiv fast <4 x float> %x, %y
+  ret <4 x float> %div
+}
+
 define <4 x float> @v4f32_two_step(<4 x float> %x) #2 {
 ; SSE-LABEL: v4f32_two_step:
 ; SSE:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/recip-fastmath2.ll b/llvm/test/CodeGen/X86/recip-fastmath2.ll
index de511b411..c5e364280 100644
--- a/llvm/test/CodeGen/X86/recip-fastmath2.ll
+++ b/llvm/test/CodeGen/X86/recip-fastmath2.ll
@@ -32,89 +32,94 @@ define float @f32_one_step_2(float %x) #1 {
 ; SSE-LABEL: f32_one_step_2:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    rcpss %xmm0, %xmm2
-; SSE-NEXT:    mulss %xmm2, %xmm0
 ; SSE-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE-NEXT:    movaps %xmm2, %xmm3
+; SSE-NEXT:    mulss %xmm1, %xmm3
+; SSE-NEXT:    mulss %xmm3, %xmm0
 ; SSE-NEXT:    subss %xmm0, %xmm1
 ; SSE-NEXT:    mulss %xmm2, %xmm1
-; SSE-NEXT:    addss %xmm2, %xmm1
-; SSE-NEXT:    mulss {{.*}}(%rip), %xmm1
+; SSE-NEXT:    addss %xmm3, %xmm1
 ; SSE-NEXT:    movaps %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-RECIP-LABEL: f32_one_step_2:
 ; AVX-RECIP:       # %bb.0:
 ; AVX-RECIP-NEXT:    vrcpss %xmm0, %xmm0, %xmm1
-; AVX-RECIP-NEXT:    vmulss %xmm1, %xmm0, %xmm0
 ; AVX-RECIP-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; AVX-RECIP-NEXT:    vmulss %xmm2, %xmm1, %xmm3
+; AVX-RECIP-NEXT:    vmulss %xmm3, %xmm0, %xmm0
 ; AVX-RECIP-NEXT:    vsubss %xmm0, %xmm2, %xmm0
 ; AVX-RECIP-NEXT:    vmulss %xmm0, %xmm1, %xmm0
-; AVX-RECIP-NEXT:    vaddss %xmm0, %xmm1, %xmm0
-; AVX-RECIP-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm0
+; AVX-RECIP-NEXT:    vaddss %xmm0, %xmm3, %xmm0
 ; AVX-RECIP-NEXT:    retq
 ;
 ; FMA-RECIP-LABEL: f32_one_step_2:
 ; FMA-RECIP:       # %bb.0:
 ; FMA-RECIP-NEXT:    vrcpss %xmm0, %xmm0, %xmm1
-; FMA-RECIP-NEXT:    vfnmadd213ss {{.*#+}} xmm0 = -(xmm1 * xmm0) + mem
-; FMA-RECIP-NEXT:    vfmadd132ss {{.*#+}} xmm0 = (xmm0 * xmm1) + xmm1
-; FMA-RECIP-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm0
+; FMA-RECIP-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; FMA-RECIP-NEXT:    vmulss %xmm2, %xmm1, %xmm3
+; FMA-RECIP-NEXT:    vfnmadd213ss {{.*#+}} xmm0 = -(xmm3 * xmm0) + xmm2
+; FMA-RECIP-NEXT:    vfmadd213ss {{.*#+}} xmm0 = (xmm1 * xmm0) + xmm3
 ; FMA-RECIP-NEXT:    retq
 ;
 ; BDVER2-LABEL: f32_one_step_2:
 ; BDVER2:       # %bb.0:
 ; BDVER2-NEXT:    vrcpss %xmm0, %xmm0, %xmm1
-; BDVER2-NEXT:    vfnmaddss {{.*}}(%rip), %xmm1, %xmm0, %xmm0
-; BDVER2-NEXT:    vfmaddss %xmm1, %xmm0, %xmm1, %xmm0
-; BDVER2-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm0
+; BDVER2-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; BDVER2-NEXT:    vmulss %xmm2, %xmm1, %xmm3
+; BDVER2-NEXT:    vfnmaddss %xmm2, %xmm3, %xmm0, %xmm0
+; BDVER2-NEXT:    vfmaddss %xmm3, %xmm0, %xmm1, %xmm0
 ; BDVER2-NEXT:    retq
 ;
 ; BTVER2-LABEL: f32_one_step_2:
 ; BTVER2:       # %bb.0:
 ; BTVER2-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
 ; BTVER2-NEXT:    vrcpss %xmm0, %xmm0, %xmm1
-; BTVER2-NEXT:    vmulss %xmm1, %xmm0, %xmm0
+; BTVER2-NEXT:    vmulss %xmm2, %xmm1, %xmm3
+; BTVER2-NEXT:    vmulss %xmm3, %xmm0, %xmm0
 ; BTVER2-NEXT:    vsubss %xmm0, %xmm2, %xmm0
 ; BTVER2-NEXT:    vmulss %xmm0, %xmm1, %xmm0
-; BTVER2-NEXT:    vaddss %xmm0, %xmm1, %xmm0
-; BTVER2-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm0
+; BTVER2-NEXT:    vaddss %xmm0, %xmm3, %xmm0
 ; BTVER2-NEXT:    retq
 ;
 ; SANDY-LABEL: f32_one_step_2:
 ; SANDY:       # %bb.0:
 ; SANDY-NEXT:    vrcpss %xmm0, %xmm0, %xmm1
-; SANDY-NEXT:    vmulss %xmm1, %xmm0, %xmm0
 ; SANDY-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SANDY-NEXT:    vmulss %xmm2, %xmm1, %xmm3
+; SANDY-NEXT:    vmulss %xmm3, %xmm0, %xmm0
 ; SANDY-NEXT:    vsubss %xmm0, %xmm2, %xmm0
 ; SANDY-NEXT:    vmulss %xmm0, %xmm1, %xmm0
-; SANDY-NEXT:    vaddss %xmm0, %xmm1, %xmm0
-; SANDY-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm0
+; SANDY-NEXT:    vaddss %xmm0, %xmm3, %xmm0
 ; SANDY-NEXT:    retq
 ;
 ; HASWELL-LABEL: f32_one_step_2:
 ; HASWELL:       # %bb.0:
 ; HASWELL-NEXT:    vrcpss %xmm0, %xmm0, %xmm1
-; HASWELL-NEXT:    vfnmadd213ss {{.*#+}} xmm0 = -(xmm1 * xmm0) + mem
-; HASWELL-NEXT:    vfmadd132ss {{.*#+}} xmm0 = (xmm0 * xmm1) + xmm1
-; HASWELL-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm0
+; HASWELL-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; HASWELL-NEXT:    vmulss %xmm2, %xmm1, %xmm3
+; HASWELL-NEXT:    vfnmadd213ss {{.*#+}} xmm0 = -(xmm3 * xmm0) + xmm2
+; HASWELL-NEXT:    vfmadd213ss {{.*#+}} xmm0 = (xmm1 * xmm0) + xmm3
 ; HASWELL-NEXT:    retq
 ;
 ; HASWELL-NO-FMA-LABEL: f32_one_step_2:
 ; HASWELL-NO-FMA:       # %bb.0:
 ; HASWELL-NO-FMA-NEXT:    vrcpss %xmm0, %xmm0, %xmm1
-; HASWELL-NO-FMA-NEXT:    vmulss %xmm1, %xmm0, %xmm0
 ; HASWELL-NO-FMA-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; HASWELL-NO-FMA-NEXT:    vmulss %xmm2, %xmm1, %xmm3
+; HASWELL-NO-FMA-NEXT:    vmulss %xmm3, %xmm0, %xmm0
 ; HASWELL-NO-FMA-NEXT:    vsubss %xmm0, %xmm2, %xmm0
 ; HASWELL-NO-FMA-NEXT:    vmulss %xmm0, %xmm1, %xmm0
-; HASWELL-NO-FMA-NEXT:    vaddss %xmm0, %xmm1, %xmm0
-; HASWELL-NO-FMA-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm0
+; HASWELL-NO-FMA-NEXT:    vaddss %xmm0, %xmm3, %xmm0
 ; HASWELL-NO-FMA-NEXT:    retq
 ;
 ; AVX512-LABEL: f32_one_step_2:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vrcpss %xmm0, %xmm0, %xmm1
-; AVX512-NEXT:    vfnmadd213ss {{.*#+}} xmm0 = -(xmm1 * xmm0) + mem
-; AVX512-NEXT:    vfmadd132ss {{.*#+}} xmm0 = (xmm0 * xmm1) + xmm1
-; AVX512-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm0
+; AVX512-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; AVX512-NEXT:    vmulss %xmm2, %xmm1, %xmm3
+; AVX512-NEXT:    vfnmadd213ss {{.*#+}} xmm0 = -(xmm3 * xmm0) + xmm2
+; AVX512-NEXT:    vfmadd213ss {{.*#+}} xmm0 = (xmm1 * xmm0) + xmm3
 ; AVX512-NEXT:    retq
   %div = fdiv fast float 3456.0, %x
   ret float %div
@@ -149,8 +154,8 @@ define float @f32_one_step_2_divs(float %x) #1 {
 ; FMA-RECIP-LABEL: f32_one_step_2_divs:
 ; FMA-RECIP:       # %bb.0:
 ; FMA-RECIP-NEXT:    vrcpss %xmm0, %xmm0, %xmm1
-; FMA-RECIP-NEXT:    vfnmadd213ss {{.*#+}} xmm0 = -(xmm1 * xmm0) + mem
-; FMA-RECIP-NEXT:    vfmadd132ss {{.*#+}} xmm0 = (xmm0 * xmm1) + xmm1
+; FMA-RECIP-NEXT:    vfmadd213ss {{.*#+}} xmm0 = (xmm1 * xmm0) + mem
+; FMA-RECIP-NEXT:    vfnmadd132ss {{.*#+}} xmm0 = -(xmm0 * xmm1) + xmm1
 ; FMA-RECIP-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm1
 ; FMA-RECIP-NEXT:    vmulss %xmm0, %xmm1, %xmm0
 ; FMA-RECIP-NEXT:    retq
@@ -158,8 +163,8 @@ define float @f32_one_step_2_divs(float %x) #1 {
 ; BDVER2-LABEL: f32_one_step_2_divs:
 ; BDVER2:       # %bb.0:
 ; BDVER2-NEXT:    vrcpss %xmm0, %xmm0, %xmm1
-; BDVER2-NEXT:    vfnmaddss {{.*}}(%rip), %xmm1, %xmm0, %xmm0
-; BDVER2-NEXT:    vfmaddss %xmm1, %xmm0, %xmm1, %xmm0
+; BDVER2-NEXT:    vfmaddss {{.*}}(%rip), %xmm1, %xmm0, %xmm0
+; BDVER2-NEXT:    vfnmaddss %xmm1, %xmm0, %xmm1, %xmm0
 ; BDVER2-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm1
 ; BDVER2-NEXT:    vmulss %xmm0, %xmm1, %xmm0
 ; BDVER2-NEXT:    retq
@@ -191,8 +196,8 @@ define float @f32_one_step_2_divs(float %x) #1 {
 ; HASWELL-LABEL: f32_one_step_2_divs:
 ; HASWELL:       # %bb.0:
 ; HASWELL-NEXT:    vrcpss %xmm0, %xmm0, %xmm1
-; HASWELL-NEXT:    vfnmadd213ss {{.*#+}} xmm0 = -(xmm1 * xmm0) + mem
-; HASWELL-NEXT:    vfmadd132ss {{.*#+}} xmm0 = (xmm0 * xmm1) + xmm1
+; HASWELL-NEXT:    vfmadd213ss {{.*#+}} xmm0 = (xmm1 * xmm0) + mem
+; HASWELL-NEXT:    vfnmadd132ss {{.*#+}} xmm0 = -(xmm0 * xmm1) + xmm1
 ; HASWELL-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm1
 ; HASWELL-NEXT:    vmulss %xmm0, %xmm1, %xmm0
 ; HASWELL-NEXT:    retq
@@ -212,8 +217,8 @@ define float @f32_one_step_2_divs(float %x) #1 {
 ; AVX512-LABEL: f32_one_step_2_divs:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vrcpss %xmm0, %xmm0, %xmm1
-; AVX512-NEXT:    vfnmadd213ss {{.*#+}} xmm0 = -(xmm1 * xmm0) + mem
-; AVX512-NEXT:    vfmadd132ss {{.*#+}} xmm0 = (xmm0 * xmm1) + xmm1
+; AVX512-NEXT:    vfmadd213ss {{.*#+}} xmm0 = (xmm1 * xmm0) + mem
+; AVX512-NEXT:    vfnmadd132ss {{.*#+}} xmm0 = -(xmm0 * xmm1) + xmm1
 ; AVX512-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm1
 ; AVX512-NEXT:    vmulss %xmm0, %xmm1, %xmm0
 ; AVX512-NEXT:    retq
@@ -225,19 +230,20 @@ define float @f32_one_step_2_divs(float %x) #1 {
 define float @f32_two_step_2(float %x) #2 {
 ; SSE-LABEL: f32_two_step_2:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    rcpss %xmm0, %xmm2
-; SSE-NEXT:    movaps %xmm0, %xmm3
-; SSE-NEXT:    mulss %xmm2, %xmm3
+; SSE-NEXT:    rcpss %xmm0, %xmm1
+; SSE-NEXT:    movaps %xmm0, %xmm2
+; SSE-NEXT:    mulss %xmm1, %xmm2
+; SSE-NEXT:    movss {{.*#+}} xmm3 = mem[0],zero,zero,zero
+; SSE-NEXT:    subss %xmm2, %xmm3
+; SSE-NEXT:    mulss %xmm1, %xmm3
+; SSE-NEXT:    addss %xmm1, %xmm3
 ; SSE-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; SSE-NEXT:    movaps %xmm1, %xmm4
-; SSE-NEXT:    subss %xmm3, %xmm4
-; SSE-NEXT:    mulss %xmm2, %xmm4
-; SSE-NEXT:    addss %xmm2, %xmm4
-; SSE-NEXT:    mulss %xmm4, %xmm0
+; SSE-NEXT:    movaps %xmm3, %xmm2
+; SSE-NEXT:    mulss %xmm1, %xmm2
+; SSE-NEXT:    mulss %xmm2, %xmm0
 ; SSE-NEXT:    subss %xmm0, %xmm1
-; SSE-NEXT:    mulss %xmm4, %xmm1
-; SSE-NEXT:    addss %xmm4, %xmm1
-; SSE-NEXT:    mulss {{.*}}(%rip), %xmm1
+; SSE-NEXT:    mulss %xmm3, %xmm1
+; SSE-NEXT:    addss %xmm2, %xmm1
 ; SSE-NEXT:    movaps %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
@@ -249,49 +255,51 @@ define float @f32_two_step_2(float %x) #2 {
 ; AVX-RECIP-NEXT:    vsubss %xmm2, %xmm3, %xmm2
 ; AVX-RECIP-NEXT:    vmulss %xmm2, %xmm1, %xmm2
 ; AVX-RECIP-NEXT:    vaddss %xmm2, %xmm1, %xmm1
-; AVX-RECIP-NEXT:    vmulss %xmm1, %xmm0, %xmm0
-; AVX-RECIP-NEXT:    vsubss %xmm0, %xmm3, %xmm0
+; AVX-RECIP-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; AVX-RECIP-NEXT:    vmulss %xmm2, %xmm1, %xmm3
+; AVX-RECIP-NEXT:    vmulss %xmm3, %xmm0, %xmm0
+; AVX-RECIP-NEXT:    vsubss %xmm0, %xmm2, %xmm0
 ; AVX-RECIP-NEXT:    vmulss %xmm0, %xmm1, %xmm0
-; AVX-RECIP-NEXT:    vaddss %xmm0, %xmm1, %xmm0
-; AVX-RECIP-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm0
+; AVX-RECIP-NEXT:    vaddss %xmm0, %xmm3, %xmm0
 ; AVX-RECIP-NEXT:    retq
 ;
 ; FMA-RECIP-LABEL: f32_two_step_2:
 ; FMA-RECIP:       # %bb.0:
 ; FMA-RECIP-NEXT:    vrcpss %xmm0, %xmm0, %xmm1
 ; FMA-RECIP-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; FMA-RECIP-NEXT:    vmovaps %xmm1, %xmm3
-; FMA-RECIP-NEXT:    vfnmadd213ss {{.*#+}} xmm3 = -(xmm0 * xmm3) + xmm2
-; FMA-RECIP-NEXT:    vfmadd132ss {{.*#+}} xmm3 = (xmm3 * xmm1) + xmm1
-; FMA-RECIP-NEXT:    vfnmadd213ss {{.*#+}} xmm0 = -(xmm3 * xmm0) + xmm2
-; FMA-RECIP-NEXT:    vfmadd132ss {{.*#+}} xmm0 = (xmm0 * xmm3) + xmm3
-; FMA-RECIP-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm0
+; FMA-RECIP-NEXT:    vfmadd231ss {{.*#+}} xmm2 = (xmm0 * xmm1) + xmm2
+; FMA-RECIP-NEXT:    vfnmadd132ss {{.*#+}} xmm2 = -(xmm2 * xmm1) + xmm1
+; FMA-RECIP-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; FMA-RECIP-NEXT:    vmulss %xmm1, %xmm2, %xmm3
+; FMA-RECIP-NEXT:    vfnmadd213ss {{.*#+}} xmm0 = -(xmm3 * xmm0) + xmm1
+; FMA-RECIP-NEXT:    vfmadd213ss {{.*#+}} xmm0 = (xmm2 * xmm0) + xmm3
 ; FMA-RECIP-NEXT:    retq
 ;
 ; BDVER2-LABEL: f32_two_step_2:
 ; BDVER2:       # %bb.0:
 ; BDVER2-NEXT:    vrcpss %xmm0, %xmm0, %xmm1
-; BDVER2-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; BDVER2-NEXT:    vfnmaddss %xmm2, %xmm1, %xmm0, %xmm3
-; BDVER2-NEXT:    vfmaddss %xmm1, %xmm3, %xmm1, %xmm1
-; BDVER2-NEXT:    vfnmaddss %xmm2, %xmm1, %xmm0, %xmm0
-; BDVER2-NEXT:    vfmaddss %xmm1, %xmm0, %xmm1, %xmm0
-; BDVER2-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm0
+; BDVER2-NEXT:    vfmaddss {{.*}}(%rip), %xmm1, %xmm0, %xmm2
+; BDVER2-NEXT:    vmovss {{.*#+}} xmm4 = mem[0],zero,zero,zero
+; BDVER2-NEXT:    vfnmaddss %xmm1, %xmm2, %xmm1, %xmm1
+; BDVER2-NEXT:    vmulss %xmm4, %xmm1, %xmm3
+; BDVER2-NEXT:    vfnmaddss %xmm4, %xmm3, %xmm0, %xmm0
+; BDVER2-NEXT:    vfmaddss %xmm3, %xmm0, %xmm1, %xmm0
 ; BDVER2-NEXT:    retq
 ;
 ; BTVER2-LABEL: f32_two_step_2:
 ; BTVER2:       # %bb.0:
 ; BTVER2-NEXT:    vmovss {{.*#+}} xmm3 = mem[0],zero,zero,zero
 ; BTVER2-NEXT:    vrcpss %xmm0, %xmm0, %xmm1
+; BTVER2-NEXT:    vmovss {{.*#+}} xmm4 = mem[0],zero,zero,zero
 ; BTVER2-NEXT:    vmulss %xmm1, %xmm0, %xmm2
 ; BTVER2-NEXT:    vsubss %xmm2, %xmm3, %xmm2
 ; BTVER2-NEXT:    vmulss %xmm2, %xmm1, %xmm2
 ; BTVER2-NEXT:    vaddss %xmm2, %xmm1, %xmm1
-; BTVER2-NEXT:    vmulss %xmm1, %xmm0, %xmm0
-; BTVER2-NEXT:    vsubss %xmm0, %xmm3, %xmm0
+; BTVER2-NEXT:    vmulss %xmm4, %xmm1, %xmm3
+; BTVER2-NEXT:    vmulss %xmm3, %xmm0, %xmm0
+; BTVER2-NEXT:    vsubss %xmm0, %xmm4, %xmm0
 ; BTVER2-NEXT:    vmulss %xmm0, %xmm1, %xmm0
-; BTVER2-NEXT:    vaddss %xmm0, %xmm1, %xmm0
-; BTVER2-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm0
+; BTVER2-NEXT:    vaddss %xmm0, %xmm3, %xmm0
 ; BTVER2-NEXT:    retq
 ;
 ; SANDY-LABEL: f32_two_step_2:
@@ -302,23 +310,24 @@ define float @f32_two_step_2(float %x) #2 {
 ; SANDY-NEXT:    vsubss %xmm2, %xmm3, %xmm2
 ; SANDY-NEXT:    vmulss %xmm2, %xmm1, %xmm2
 ; SANDY-NEXT:    vaddss %xmm2, %xmm1, %xmm1
-; SANDY-NEXT:    vmulss %xmm1, %xmm0, %xmm0
-; SANDY-NEXT:    vsubss %xmm0, %xmm3, %xmm0
+; SANDY-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SANDY-NEXT:    vmulss %xmm2, %xmm1, %xmm3
+; SANDY-NEXT:    vmulss %xmm3, %xmm0, %xmm0
+; SANDY-NEXT:    vsubss %xmm0, %xmm2, %xmm0
 ; SANDY-NEXT:    vmulss %xmm0, %xmm1, %xmm0
-; SANDY-NEXT:    vaddss %xmm0, %xmm1, %xmm0
-; SANDY-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm0
+; SANDY-NEXT:    vaddss %xmm0, %xmm3, %xmm0
 ; SANDY-NEXT:    retq
 ;
 ; HASWELL-LABEL: f32_two_step_2:
 ; HASWELL:       # %bb.0:
 ; HASWELL-NEXT:    vrcpss %xmm0, %xmm0, %xmm1
 ; HASWELL-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; HASWELL-NEXT:    vmovaps %xmm1, %xmm3
-; HASWELL-NEXT:    vfnmadd213ss {{.*#+}} xmm3 = -(xmm0 * xmm3) + xmm2
-; HASWELL-NEXT:    vfmadd132ss {{.*#+}} xmm3 = (xmm3 * xmm1) + xmm1
-; HASWELL-NEXT:    vfnmadd213ss {{.*#+}} xmm0 = -(xmm3 * xmm0) + xmm2
-; HASWELL-NEXT:    vfmadd132ss {{.*#+}} xmm0 = (xmm0 * xmm3) + xmm3
-; HASWELL-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm0
+; HASWELL-NEXT:    vfmadd231ss {{.*#+}} xmm2 = (xmm0 * xmm1) + xmm2
+; HASWELL-NEXT:    vfnmadd132ss {{.*#+}} xmm2 = -(xmm2 * xmm1) + xmm1
+; HASWELL-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; HASWELL-NEXT:    vmulss %xmm1, %xmm2, %xmm3
+; HASWELL-NEXT:    vfnmadd213ss {{.*#+}} xmm0 = -(xmm3 * xmm0) + xmm1
+; HASWELL-NEXT:    vfmadd213ss {{.*#+}} xmm0 = (xmm2 * xmm0) + xmm3
 ; HASWELL-NEXT:    retq
 ;
 ; HASWELL-NO-FMA-LABEL: f32_two_step_2:
@@ -329,23 +338,24 @@ define float @f32_two_step_2(float %x) #2 {
 ; HASWELL-NO-FMA-NEXT:    vsubss %xmm2, %xmm3, %xmm2
 ; HASWELL-NO-FMA-NEXT:    vmulss %xmm2, %xmm1, %xmm2
 ; HASWELL-NO-FMA-NEXT:    vaddss %xmm2, %xmm1, %xmm1
-; HASWELL-NO-FMA-NEXT:    vmulss %xmm1, %xmm0, %xmm0
-; HASWELL-NO-FMA-NEXT:    vsubss %xmm0, %xmm3, %xmm0
+; HASWELL-NO-FMA-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; HASWELL-NO-FMA-NEXT:    vmulss %xmm2, %xmm1, %xmm3
+; HASWELL-NO-FMA-NEXT:    vmulss %xmm3, %xmm0, %xmm0
+; HASWELL-NO-FMA-NEXT:    vsubss %xmm0, %xmm2, %xmm0
 ; HASWELL-NO-FMA-NEXT:    vmulss %xmm0, %xmm1, %xmm0
-; HASWELL-NO-FMA-NEXT:    vaddss %xmm0, %xmm1, %xmm0
-; HASWELL-NO-FMA-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm0
+; HASWELL-NO-FMA-NEXT:    vaddss %xmm0, %xmm3, %xmm0
 ; HASWELL-NO-FMA-NEXT:    retq
 ;
 ; AVX512-LABEL: f32_two_step_2:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vrcpss %xmm0, %xmm0, %xmm1
 ; AVX512-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; AVX512-NEXT:    vmovaps %xmm1, %xmm3
-; AVX512-NEXT:    vfnmadd213ss {{.*#+}} xmm3 = -(xmm0 * xmm3) + xmm2
-; AVX512-NEXT:    vfmadd132ss {{.*#+}} xmm3 = (xmm3 * xmm1) + xmm1
-; AVX512-NEXT:    vfnmadd213ss {{.*#+}} xmm0 = -(xmm3 * xmm0) + xmm2
-; AVX512-NEXT:    vfmadd132ss {{.*#+}} xmm0 = (xmm0 * xmm3) + xmm3
-; AVX512-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm0
+; AVX512-NEXT:    vfmadd231ss {{.*#+}} xmm2 = (xmm0 * xmm1) + xmm2
+; AVX512-NEXT:    vfnmadd132ss {{.*#+}} xmm2 = -(xmm2 * xmm1) + xmm1
+; AVX512-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX512-NEXT:    vmulss %xmm1, %xmm2, %xmm3
+; AVX512-NEXT:    vfnmadd213ss {{.*#+}} xmm0 = -(xmm3 * xmm0) + xmm1
+; AVX512-NEXT:    vfmadd213ss {{.*#+}} xmm0 = (xmm2 * xmm0) + xmm3
 ; AVX512-NEXT:    retq
   %div = fdiv fast float 6789.0, %x
   ret float %div
@@ -355,100 +365,95 @@ define <4 x float> @v4f32_one_step2(<4 x float> %x) #1 {
 ; SSE-LABEL: v4f32_one_step2:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    rcpps %xmm0, %xmm2
-; SSE-NEXT:    mulps %xmm2, %xmm0
-; SSE-NEXT:    movaps {{.*#+}} xmm1 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; SSE-NEXT:    movaps {{.*#+}} xmm1 = [1.0E+0,2.0E+0,3.0E+0,4.0E+0]
+; SSE-NEXT:    movaps %xmm2, %xmm3
+; SSE-NEXT:    mulps %xmm1, %xmm3
+; SSE-NEXT:    mulps %xmm3, %xmm0
 ; SSE-NEXT:    subps %xmm0, %xmm1
 ; SSE-NEXT:    mulps %xmm2, %xmm1
-; SSE-NEXT:    addps %xmm2, %xmm1
-; SSE-NEXT:    mulps {{.*}}(%rip), %xmm1
+; SSE-NEXT:    addps %xmm3, %xmm1
 ; SSE-NEXT:    movaps %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-RECIP-LABEL: v4f32_one_step2:
 ; AVX-RECIP:       # %bb.0:
 ; AVX-RECIP-NEXT:    vrcpps %xmm0, %xmm1
-; AVX-RECIP-NEXT:    vmulps %xmm1, %xmm0, %xmm0
-; AVX-RECIP-NEXT:    vmovaps {{.*#+}} xmm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; AVX-RECIP-NEXT:    vmovaps {{.*#+}} xmm2 = [1.0E+0,2.0E+0,3.0E+0,4.0E+0]
+; AVX-RECIP-NEXT:    vmulps %xmm2, %xmm1, %xmm3
+; AVX-RECIP-NEXT:    vmulps %xmm3, %xmm0, %xmm0
 ; AVX-RECIP-NEXT:    vsubps %xmm0, %xmm2, %xmm0
 ; AVX-RECIP-NEXT:    vmulps %xmm0, %xmm1, %xmm0
-; AVX-RECIP-NEXT:    vaddps %xmm0, %xmm1, %xmm0
-; AVX-RECIP-NEXT:    vmulps {{.*}}(%rip), %xmm0, %xmm0
+; AVX-RECIP-NEXT:    vaddps %xmm0, %xmm3, %xmm0
 ; AVX-RECIP-NEXT:    retq
 ;
 ; FMA-RECIP-LABEL: v4f32_one_step2:
 ; FMA-RECIP:       # %bb.0:
 ; FMA-RECIP-NEXT:    vrcpps %xmm0, %xmm1
-; FMA-RECIP-NEXT:    vfnmadd213ps {{.*#+}} xmm0 = -(xmm1 * xmm0) + mem
-; FMA-RECIP-NEXT:    vfmadd132ps {{.*#+}} xmm0 = (xmm0 * xmm1) + xmm1
-; FMA-RECIP-NEXT:    vmulps {{.*}}(%rip), %xmm0, %xmm0
+; FMA-RECIP-NEXT:    vmovaps {{.*#+}} xmm2 = [1.0E+0,2.0E+0,3.0E+0,4.0E+0]
+; FMA-RECIP-NEXT:    vmulps %xmm2, %xmm1, %xmm3
+; FMA-RECIP-NEXT:    vfnmadd213ps {{.*#+}} xmm0 = -(xmm3 * xmm0) + xmm2
+; FMA-RECIP-NEXT:    vfmadd213ps {{.*#+}} xmm0 = (xmm1 * xmm0) + xmm3
 ; FMA-RECIP-NEXT:    retq
 ;
 ; BDVER2-LABEL: v4f32_one_step2:
 ; BDVER2:       # %bb.0:
 ; BDVER2-NEXT:    vrcpps %xmm0, %xmm1
-; BDVER2-NEXT:    vfnmaddps {{.*}}(%rip), %xmm1, %xmm0, %xmm0
-; BDVER2-NEXT:    vfmaddps %xmm1, %xmm0, %xmm1, %xmm0
-; BDVER2-NEXT:    vmulps {{.*}}(%rip), %xmm0, %xmm0
+; BDVER2-NEXT:    vmovaps {{.*#+}} xmm2 = [1.0E+0,2.0E+0,3.0E+0,4.0E+0]
+; BDVER2-NEXT:    vmulps %xmm2, %xmm1, %xmm3
+; BDVER2-NEXT:    vfnmaddps %xmm2, %xmm3, %xmm0, %xmm0
+; BDVER2-NEXT:    vfmaddps %xmm3, %xmm0, %xmm1, %xmm0
 ; BDVER2-NEXT:    retq
 ;
 ; BTVER2-LABEL: v4f32_one_step2:
 ; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmovaps {{.*#+}} xmm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; BTVER2-NEXT:    vmovaps {{.*#+}} xmm2 = [1.0E+0,2.0E+0,3.0E+0,4.0E+0]
 ; BTVER2-NEXT:    vrcpps %xmm0, %xmm1
-; BTVER2-NEXT:    vmulps %xmm1, %xmm0, %xmm0
+; BTVER2-NEXT:    vmulps %xmm2, %xmm1, %xmm3
+; BTVER2-NEXT:    vmulps %xmm3, %xmm0, %xmm0
 ; BTVER2-NEXT:    vsubps %xmm0, %xmm2, %xmm0
 ; BTVER2-NEXT:    vmulps %xmm0, %xmm1, %xmm0
-; BTVER2-NEXT:    vaddps %xmm0, %xmm1, %xmm0
-; BTVER2-NEXT:    vmulps {{.*}}(%rip), %xmm0, %xmm0
+; BTVER2-NEXT:    vaddps %xmm0, %xmm3, %xmm0
 ; BTVER2-NEXT:    retq
 ;
 ; SANDY-LABEL: v4f32_one_step2:
 ; SANDY:       # %bb.0:
 ; SANDY-NEXT:    vrcpps %xmm0, %xmm1
-; SANDY-NEXT:    vmulps %xmm1, %xmm0, %xmm0
-; SANDY-NEXT:    vmovaps {{.*#+}} xmm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; SANDY-NEXT:    vmovaps {{.*#+}} xmm2 = [1.0E+0,2.0E+0,3.0E+0,4.0E+0]
+; SANDY-NEXT:    vmulps %xmm2, %xmm1, %xmm3
+; SANDY-NEXT:    vmulps %xmm3, %xmm0, %xmm0
 ; SANDY-NEXT:    vsubps %xmm0, %xmm2, %xmm0
 ; SANDY-NEXT:    vmulps %xmm0, %xmm1, %xmm0
-; SANDY-NEXT:    vaddps %xmm0, %xmm1, %xmm0
-; SANDY-NEXT:    vmulps {{.*}}(%rip), %xmm0, %xmm0
+; SANDY-NEXT:    vaddps %xmm0, %xmm3, %xmm0
 ; SANDY-NEXT:    retq
 ;
 ; HASWELL-LABEL: v4f32_one_step2:
 ; HASWELL:       # %bb.0:
 ; HASWELL-NEXT:    vrcpps %xmm0, %xmm1
-; HASWELL-NEXT:    vbroadcastss {{.*#+}} xmm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} xmm0 = -(xmm1 * xmm0) + xmm2
-; HASWELL-NEXT:    vfmadd132ps {{.*#+}} xmm0 = (xmm0 * xmm1) + xmm1
-; HASWELL-NEXT:    vmulps {{.*}}(%rip), %xmm0, %xmm0
+; HASWELL-NEXT:    vmovaps {{.*#+}} xmm2 = [1.0E+0,2.0E+0,3.0E+0,4.0E+0]
+; HASWELL-NEXT:    vmulps %xmm2, %xmm1, %xmm3
+; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} xmm0 = -(xmm3 * xmm0) + xmm2
+; HASWELL-NEXT:    vfmadd213ps {{.*#+}} xmm0 = (xmm1 * xmm0) + xmm3
 ; HASWELL-NEXT:    retq
 ;
 ; HASWELL-NO-FMA-LABEL: v4f32_one_step2:
 ; HASWELL-NO-FMA:       # %bb.0:
 ; HASWELL-NO-FMA-NEXT:    vrcpps %xmm0, %xmm1
-; HASWELL-NO-FMA-NEXT:    vmulps %xmm1, %xmm0, %xmm0
-; HASWELL-NO-FMA-NEXT:    vbroadcastss {{.*#+}} xmm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; HASWELL-NO-FMA-NEXT:    vmovaps {{.*#+}} xmm2 = [1.0E+0,2.0E+0,3.0E+0,4.0E+0]
+; HASWELL-NO-FMA-NEXT:    vmulps %xmm2, %xmm1, %xmm3
+; HASWELL-NO-FMA-NEXT:    vmulps %xmm3, %xmm0, %xmm0
 ; HASWELL-NO-FMA-NEXT:    vsubps %xmm0, %xmm2, %xmm0
 ; HASWELL-NO-FMA-NEXT:    vmulps %xmm0, %xmm1, %xmm0
-; HASWELL-NO-FMA-NEXT:    vaddps %xmm0, %xmm1, %xmm0
-; HASWELL-NO-FMA-NEXT:    vmulps {{.*}}(%rip), %xmm0, %xmm0
+; HASWELL-NO-FMA-NEXT:    vaddps %xmm0, %xmm3, %xmm0
 ; HASWELL-NO-FMA-NEXT:    retq
 ;
-; KNL-LABEL: v4f32_one_step2:
-; KNL:       # %bb.0:
-; KNL-NEXT:    vrcpps %xmm0, %xmm1
-; KNL-NEXT:    vbroadcastss {{.*#+}} xmm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; KNL-NEXT:    vfnmadd213ps {{.*#+}} xmm0 = -(xmm1 * xmm0) + xmm2
-; KNL-NEXT:    vfmadd132ps {{.*#+}} xmm0 = (xmm0 * xmm1) + xmm1
-; KNL-NEXT:    vmulps {{.*}}(%rip), %xmm0, %xmm0
-; KNL-NEXT:    retq
-;
-; SKX-LABEL: v4f32_one_step2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vrcpps %xmm0, %xmm1
-; SKX-NEXT:    vfnmadd213ps {{.*#+}} xmm0 = -(xmm1 * xmm0) + mem
-; SKX-NEXT:    vfmadd132ps {{.*#+}} xmm0 = (xmm0 * xmm1) + xmm1
-; SKX-NEXT:    vmulps {{.*}}(%rip), %xmm0, %xmm0
-; SKX-NEXT:    retq
+; AVX512-LABEL: v4f32_one_step2:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vrcpps %xmm0, %xmm1
+; AVX512-NEXT:    vmovaps {{.*#+}} xmm2 = [1.0E+0,2.0E+0,3.0E+0,4.0E+0]
+; AVX512-NEXT:    vmulps %xmm2, %xmm1, %xmm3
+; AVX512-NEXT:    vfnmadd213ps {{.*#+}} xmm0 = -(xmm3 * xmm0) + xmm2
+; AVX512-NEXT:    vfmadd213ps {{.*#+}} xmm0 = (xmm1 * xmm0) + xmm3
+; AVX512-NEXT:    retq
   %div = fdiv fast <4 x float> <float 1.0, float 2.0, float 3.0, float 4.0>, %x
   ret <4 x float> %div
 }
@@ -569,19 +574,20 @@ define <4 x float> @v4f32_one_step_2_divs(<4 x float> %x) #1 {
 define <4 x float> @v4f32_two_step2(<4 x float> %x) #2 {
 ; SSE-LABEL: v4f32_two_step2:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    rcpps %xmm0, %xmm2
-; SSE-NEXT:    movaps %xmm0, %xmm3
-; SSE-NEXT:    mulps %xmm2, %xmm3
-; SSE-NEXT:    movaps {{.*#+}} xmm1 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; SSE-NEXT:    movaps %xmm1, %xmm4
-; SSE-NEXT:    subps %xmm3, %xmm4
-; SSE-NEXT:    mulps %xmm2, %xmm4
-; SSE-NEXT:    addps %xmm2, %xmm4
-; SSE-NEXT:    mulps %xmm4, %xmm0
+; SSE-NEXT:    rcpps %xmm0, %xmm1
+; SSE-NEXT:    movaps %xmm0, %xmm2
+; SSE-NEXT:    mulps %xmm1, %xmm2
+; SSE-NEXT:    movaps {{.*#+}} xmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; SSE-NEXT:    subps %xmm2, %xmm3
+; SSE-NEXT:    mulps %xmm1, %xmm3
+; SSE-NEXT:    addps %xmm1, %xmm3
+; SSE-NEXT:    movaps {{.*#+}} xmm1 = [1.0E+0,2.0E+0,3.0E+0,4.0E+0]
+; SSE-NEXT:    movaps %xmm3, %xmm2
+; SSE-NEXT:    mulps %xmm1, %xmm2
+; SSE-NEXT:    mulps %xmm2, %xmm0
 ; SSE-NEXT:    subps %xmm0, %xmm1
-; SSE-NEXT:    mulps %xmm4, %xmm1
-; SSE-NEXT:    addps %xmm4, %xmm1
-; SSE-NEXT:    mulps {{.*}}(%rip), %xmm1
+; SSE-NEXT:    mulps %xmm3, %xmm1
+; SSE-NEXT:    addps %xmm2, %xmm1
 ; SSE-NEXT:    movaps %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
@@ -593,49 +599,51 @@ define <4 x float> @v4f32_two_step2(<4 x float> %x) #2 {
 ; AVX-RECIP-NEXT:    vsubps %xmm2, %xmm3, %xmm2
 ; AVX-RECIP-NEXT:    vmulps %xmm2, %xmm1, %xmm2
 ; AVX-RECIP-NEXT:    vaddps %xmm2, %xmm1, %xmm1
-; AVX-RECIP-NEXT:    vmulps %xmm1, %xmm0, %xmm0
-; AVX-RECIP-NEXT:    vsubps %xmm0, %xmm3, %xmm0
+; AVX-RECIP-NEXT:    vmovaps {{.*#+}} xmm2 = [1.0E+0,2.0E+0,3.0E+0,4.0E+0]
+; AVX-RECIP-NEXT:    vmulps %xmm2, %xmm1, %xmm3
+; AVX-RECIP-NEXT:    vmulps %xmm3, %xmm0, %xmm0
+; AVX-RECIP-NEXT:    vsubps %xmm0, %xmm2, %xmm0
 ; AVX-RECIP-NEXT:    vmulps %xmm0, %xmm1, %xmm0
-; AVX-RECIP-NEXT:    vaddps %xmm0, %xmm1, %xmm0
-; AVX-RECIP-NEXT:    vmulps {{.*}}(%rip), %xmm0, %xmm0
+; AVX-RECIP-NEXT:    vaddps %xmm0, %xmm3, %xmm0
 ; AVX-RECIP-NEXT:    retq
 ;
 ; FMA-RECIP-LABEL: v4f32_two_step2:
 ; FMA-RECIP:       # %bb.0:
 ; FMA-RECIP-NEXT:    vrcpps %xmm0, %xmm1
-; FMA-RECIP-NEXT:    vmovaps {{.*#+}} xmm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; FMA-RECIP-NEXT:    vmovaps %xmm1, %xmm3
-; FMA-RECIP-NEXT:    vfnmadd213ps {{.*#+}} xmm3 = -(xmm0 * xmm3) + xmm2
-; FMA-RECIP-NEXT:    vfmadd132ps {{.*#+}} xmm3 = (xmm3 * xmm1) + xmm1
-; FMA-RECIP-NEXT:    vfnmadd213ps {{.*#+}} xmm0 = -(xmm3 * xmm0) + xmm2
-; FMA-RECIP-NEXT:    vfmadd132ps {{.*#+}} xmm0 = (xmm0 * xmm3) + xmm3
-; FMA-RECIP-NEXT:    vmulps {{.*}}(%rip), %xmm0, %xmm0
+; FMA-RECIP-NEXT:    vmovaps {{.*#+}} xmm2 = [-1.0E+0,-1.0E+0,-1.0E+0,-1.0E+0]
+; FMA-RECIP-NEXT:    vfmadd231ps {{.*#+}} xmm2 = (xmm0 * xmm1) + xmm2
+; FMA-RECIP-NEXT:    vfnmadd132ps {{.*#+}} xmm2 = -(xmm2 * xmm1) + xmm1
+; FMA-RECIP-NEXT:    vmovaps {{.*#+}} xmm1 = [1.0E+0,2.0E+0,3.0E+0,4.0E+0]
+; FMA-RECIP-NEXT:    vmulps %xmm1, %xmm2, %xmm3
+; FMA-RECIP-NEXT:    vfnmadd213ps {{.*#+}} xmm0 = -(xmm3 * xmm0) + xmm1
+; FMA-RECIP-NEXT:    vfmadd213ps {{.*#+}} xmm0 = (xmm2 * xmm0) + xmm3
 ; FMA-RECIP-NEXT:    retq
 ;
 ; BDVER2-LABEL: v4f32_two_step2:
 ; BDVER2:       # %bb.0:
 ; BDVER2-NEXT:    vrcpps %xmm0, %xmm1
-; BDVER2-NEXT:    vmovaps {{.*#+}} xmm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; BDVER2-NEXT:    vfnmaddps %xmm2, %xmm1, %xmm0, %xmm3
-; BDVER2-NEXT:    vfmaddps %xmm1, %xmm3, %xmm1, %xmm1
-; BDVER2-NEXT:    vfnmaddps %xmm2, %xmm1, %xmm0, %xmm0
-; BDVER2-NEXT:    vfmaddps %xmm1, %xmm0, %xmm1, %xmm0
-; BDVER2-NEXT:    vmulps {{.*}}(%rip), %xmm0, %xmm0
+; BDVER2-NEXT:    vfmaddps {{.*}}(%rip), %xmm1, %xmm0, %xmm2
+; BDVER2-NEXT:    vmovaps {{.*#+}} xmm4 = [1.0E+0,2.0E+0,3.0E+0,4.0E+0]
+; BDVER2-NEXT:    vfnmaddps %xmm1, %xmm2, %xmm1, %xmm1
+; BDVER2-NEXT:    vmulps %xmm4, %xmm1, %xmm3
+; BDVER2-NEXT:    vfnmaddps %xmm4, %xmm3, %xmm0, %xmm0
+; BDVER2-NEXT:    vfmaddps %xmm3, %xmm0, %xmm1, %xmm0
 ; BDVER2-NEXT:    retq
 ;
 ; BTVER2-LABEL: v4f32_two_step2:
 ; BTVER2:       # %bb.0:
 ; BTVER2-NEXT:    vmovaps {{.*#+}} xmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0]
 ; BTVER2-NEXT:    vrcpps %xmm0, %xmm1
+; BTVER2-NEXT:    vmovaps {{.*#+}} xmm4 = [1.0E+0,2.0E+0,3.0E+0,4.0E+0]
 ; BTVER2-NEXT:    vmulps %xmm1, %xmm0, %xmm2
 ; BTVER2-NEXT:    vsubps %xmm2, %xmm3, %xmm2
 ; BTVER2-NEXT:    vmulps %xmm2, %xmm1, %xmm2
 ; BTVER2-NEXT:    vaddps %xmm2, %xmm1, %xmm1
-; BTVER2-NEXT:    vmulps %xmm1, %xmm0, %xmm0
-; BTVER2-NEXT:    vsubps %xmm0, %xmm3, %xmm0
+; BTVER2-NEXT:    vmulps %xmm4, %xmm1, %xmm3
+; BTVER2-NEXT:    vmulps %xmm3, %xmm0, %xmm0
+; BTVER2-NEXT:    vsubps %xmm0, %xmm4, %xmm0
 ; BTVER2-NEXT:    vmulps %xmm0, %xmm1, %xmm0
-; BTVER2-NEXT:    vaddps %xmm0, %xmm1, %xmm0
-; BTVER2-NEXT:    vmulps {{.*}}(%rip), %xmm0, %xmm0
+; BTVER2-NEXT:    vaddps %xmm0, %xmm3, %xmm0
 ; BTVER2-NEXT:    retq
 ;
 ; SANDY-LABEL: v4f32_two_step2:
@@ -646,23 +654,24 @@ define <4 x float> @v4f32_two_step2(<4 x float> %x) #2 {
 ; SANDY-NEXT:    vsubps %xmm2, %xmm3, %xmm2
 ; SANDY-NEXT:    vmulps %xmm2, %xmm1, %xmm2
 ; SANDY-NEXT:    vaddps %xmm2, %xmm1, %xmm1
-; SANDY-NEXT:    vmulps %xmm1, %xmm0, %xmm0
-; SANDY-NEXT:    vsubps %xmm0, %xmm3, %xmm0
+; SANDY-NEXT:    vmovaps {{.*#+}} xmm2 = [1.0E+0,2.0E+0,3.0E+0,4.0E+0]
+; SANDY-NEXT:    vmulps %xmm2, %xmm1, %xmm3
+; SANDY-NEXT:    vmulps %xmm3, %xmm0, %xmm0
+; SANDY-NEXT:    vsubps %xmm0, %xmm2, %xmm0
 ; SANDY-NEXT:    vmulps %xmm0, %xmm1, %xmm0
-; SANDY-NEXT:    vaddps %xmm0, %xmm1, %xmm0
-; SANDY-NEXT:    vmulps {{.*}}(%rip), %xmm0, %xmm0
+; SANDY-NEXT:    vaddps %xmm0, %xmm3, %xmm0
 ; SANDY-NEXT:    retq
 ;
 ; HASWELL-LABEL: v4f32_two_step2:
 ; HASWELL:       # %bb.0:
 ; HASWELL-NEXT:    vrcpps %xmm0, %xmm1
-; HASWELL-NEXT:    vbroadcastss {{.*#+}} xmm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; HASWELL-NEXT:    vmovaps %xmm1, %xmm3
-; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} xmm3 = -(xmm0 * xmm3) + xmm2
-; HASWELL-NEXT:    vfmadd132ps {{.*#+}} xmm3 = (xmm3 * xmm1) + xmm1
-; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} xmm0 = -(xmm3 * xmm0) + xmm2
-; HASWELL-NEXT:    vfmadd132ps {{.*#+}} xmm0 = (xmm0 * xmm3) + xmm3
-; HASWELL-NEXT:    vmulps {{.*}}(%rip), %xmm0, %xmm0
+; HASWELL-NEXT:    vbroadcastss {{.*#+}} xmm2 = [-1.0E+0,-1.0E+0,-1.0E+0,-1.0E+0]
+; HASWELL-NEXT:    vfmadd231ps {{.*#+}} xmm2 = (xmm0 * xmm1) + xmm2
+; HASWELL-NEXT:    vfnmadd132ps {{.*#+}} xmm2 = -(xmm2 * xmm1) + xmm1
+; HASWELL-NEXT:    vmovaps {{.*#+}} xmm1 = [1.0E+0,2.0E+0,3.0E+0,4.0E+0]
+; HASWELL-NEXT:    vmulps %xmm1, %xmm2, %xmm3
+; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} xmm0 = -(xmm3 * xmm0) + xmm1
+; HASWELL-NEXT:    vfmadd213ps {{.*#+}} xmm0 = (xmm2 * xmm0) + xmm3
 ; HASWELL-NEXT:    retq
 ;
 ; HASWELL-NO-FMA-LABEL: v4f32_two_step2:
@@ -673,23 +682,24 @@ define <4 x float> @v4f32_two_step2(<4 x float> %x) #2 {
 ; HASWELL-NO-FMA-NEXT:    vsubps %xmm2, %xmm3, %xmm2
 ; HASWELL-NO-FMA-NEXT:    vmulps %xmm2, %xmm1, %xmm2
 ; HASWELL-NO-FMA-NEXT:    vaddps %xmm2, %xmm1, %xmm1
-; HASWELL-NO-FMA-NEXT:    vmulps %xmm1, %xmm0, %xmm0
-; HASWELL-NO-FMA-NEXT:    vsubps %xmm0, %xmm3, %xmm0
+; HASWELL-NO-FMA-NEXT:    vmovaps {{.*#+}} xmm2 = [1.0E+0,2.0E+0,3.0E+0,4.0E+0]
+; HASWELL-NO-FMA-NEXT:    vmulps %xmm2, %xmm1, %xmm3
+; HASWELL-NO-FMA-NEXT:    vmulps %xmm3, %xmm0, %xmm0
+; HASWELL-NO-FMA-NEXT:    vsubps %xmm0, %xmm2, %xmm0
 ; HASWELL-NO-FMA-NEXT:    vmulps %xmm0, %xmm1, %xmm0
-; HASWELL-NO-FMA-NEXT:    vaddps %xmm0, %xmm1, %xmm0
-; HASWELL-NO-FMA-NEXT:    vmulps {{.*}}(%rip), %xmm0, %xmm0
+; HASWELL-NO-FMA-NEXT:    vaddps %xmm0, %xmm3, %xmm0
 ; HASWELL-NO-FMA-NEXT:    retq
 ;
 ; AVX512-LABEL: v4f32_two_step2:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vrcpps %xmm0, %xmm1
-; AVX512-NEXT:    vbroadcastss {{.*#+}} xmm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; AVX512-NEXT:    vmovaps %xmm1, %xmm3
-; AVX512-NEXT:    vfnmadd213ps {{.*#+}} xmm3 = -(xmm0 * xmm3) + xmm2
-; AVX512-NEXT:    vfmadd132ps {{.*#+}} xmm3 = (xmm3 * xmm1) + xmm1
-; AVX512-NEXT:    vfnmadd213ps {{.*#+}} xmm0 = -(xmm3 * xmm0) + xmm2
-; AVX512-NEXT:    vfmadd132ps {{.*#+}} xmm0 = (xmm0 * xmm3) + xmm3
-; AVX512-NEXT:    vmulps {{.*}}(%rip), %xmm0, %xmm0
+; AVX512-NEXT:    vbroadcastss {{.*#+}} xmm2 = [-1.0E+0,-1.0E+0,-1.0E+0,-1.0E+0]
+; AVX512-NEXT:    vfmadd231ps {{.*#+}} xmm2 = (xmm0 * xmm1) + xmm2
+; AVX512-NEXT:    vfnmadd132ps {{.*#+}} xmm2 = -(xmm2 * xmm1) + xmm1
+; AVX512-NEXT:    vmovaps {{.*#+}} xmm1 = [1.0E+0,2.0E+0,3.0E+0,4.0E+0]
+; AVX512-NEXT:    vmulps %xmm1, %xmm2, %xmm3
+; AVX512-NEXT:    vfnmadd213ps {{.*#+}} xmm0 = -(xmm3 * xmm0) + xmm1
+; AVX512-NEXT:    vfmadd213ps {{.*#+}} xmm0 = (xmm2 * xmm0) + xmm3
 ; AVX512-NEXT:    retq
   %div = fdiv fast <4 x float> <float 1.0, float 2.0, float 3.0, float 4.0>, %x
   ret <4 x float> %div
@@ -698,20 +708,22 @@ define <4 x float> @v4f32_two_step2(<4 x float> %x) #2 {
 define <8 x float> @v8f32_one_step2(<8 x float> %x) #1 {
 ; SSE-LABEL: v8f32_one_step2:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    rcpps %xmm1, %xmm4
+; SSE-NEXT:    rcpps %xmm0, %xmm3
+; SSE-NEXT:    movaps {{.*#+}} xmm2 = [1.0E+0,2.0E+0,3.0E+0,4.0E+0]
+; SSE-NEXT:    movaps %xmm3, %xmm4
+; SSE-NEXT:    mulps %xmm2, %xmm4
+; SSE-NEXT:    mulps %xmm4, %xmm0
+; SSE-NEXT:    subps %xmm0, %xmm2
+; SSE-NEXT:    mulps %xmm3, %xmm2
+; SSE-NEXT:    addps %xmm4, %xmm2
+; SSE-NEXT:    rcpps %xmm1, %xmm0
+; SSE-NEXT:    movaps {{.*#+}} xmm3 = [5.0E+0,6.0E+0,7.0E+0,8.0E+0]
+; SSE-NEXT:    movaps %xmm0, %xmm4
+; SSE-NEXT:    mulps %xmm3, %xmm4
 ; SSE-NEXT:    mulps %xmm4, %xmm1
-; SSE-NEXT:    movaps {{.*#+}} xmm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; SSE-NEXT:    movaps %xmm2, %xmm3
 ; SSE-NEXT:    subps %xmm1, %xmm3
-; SSE-NEXT:    mulps %xmm4, %xmm3
+; SSE-NEXT:    mulps %xmm0, %xmm3
 ; SSE-NEXT:    addps %xmm4, %xmm3
-; SSE-NEXT:    rcpps %xmm0, %xmm1
-; SSE-NEXT:    mulps %xmm1, %xmm0
-; SSE-NEXT:    subps %xmm0, %xmm2
-; SSE-NEXT:    mulps %xmm1, %xmm2
-; SSE-NEXT:    addps %xmm1, %xmm2
-; SSE-NEXT:    mulps {{.*}}(%rip), %xmm2
-; SSE-NEXT:    mulps {{.*}}(%rip), %xmm3
 ; SSE-NEXT:    movaps %xmm2, %xmm0
 ; SSE-NEXT:    movaps %xmm3, %xmm1
 ; SSE-NEXT:    retq
@@ -719,88 +731,82 @@ define <8 x float> @v8f32_one_step2(<8 x float> %x) #1 {
 ; AVX-RECIP-LABEL: v8f32_one_step2:
 ; AVX-RECIP:       # %bb.0:
 ; AVX-RECIP-NEXT:    vrcpps %ymm0, %ymm1
-; AVX-RECIP-NEXT:    vmulps %ymm1, %ymm0, %ymm0
-; AVX-RECIP-NEXT:    vmovaps {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; AVX-RECIP-NEXT:    vmovaps {{.*#+}} ymm2 = [1.0E+0,2.0E+0,3.0E+0,4.0E+0,5.0E+0,6.0E+0,7.0E+0,8.0E+0]
+; AVX-RECIP-NEXT:    vmulps %ymm2, %ymm1, %ymm3
+; AVX-RECIP-NEXT:    vmulps %ymm3, %ymm0, %ymm0
 ; AVX-RECIP-NEXT:    vsubps %ymm0, %ymm2, %ymm0
 ; AVX-RECIP-NEXT:    vmulps %ymm0, %ymm1, %ymm0
-; AVX-RECIP-NEXT:    vaddps %ymm0, %ymm1, %ymm0
-; AVX-RECIP-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
+; AVX-RECIP-NEXT:    vaddps %ymm0, %ymm3, %ymm0
 ; AVX-RECIP-NEXT:    retq
 ;
 ; FMA-RECIP-LABEL: v8f32_one_step2:
 ; FMA-RECIP:       # %bb.0:
 ; FMA-RECIP-NEXT:    vrcpps %ymm0, %ymm1
-; FMA-RECIP-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm1 * ymm0) + mem
-; FMA-RECIP-NEXT:    vfmadd132ps {{.*#+}} ymm0 = (ymm0 * ymm1) + ymm1
-; FMA-RECIP-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
+; FMA-RECIP-NEXT:    vmovaps {{.*#+}} ymm2 = [1.0E+0,2.0E+0,3.0E+0,4.0E+0,5.0E+0,6.0E+0,7.0E+0,8.0E+0]
+; FMA-RECIP-NEXT:    vmulps %ymm2, %ymm1, %ymm3
+; FMA-RECIP-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm3 * ymm0) + ymm2
+; FMA-RECIP-NEXT:    vfmadd213ps {{.*#+}} ymm0 = (ymm1 * ymm0) + ymm3
 ; FMA-RECIP-NEXT:    retq
 ;
 ; BDVER2-LABEL: v8f32_one_step2:
 ; BDVER2:       # %bb.0:
 ; BDVER2-NEXT:    vrcpps %ymm0, %ymm1
-; BDVER2-NEXT:    vfnmaddps {{.*}}(%rip), %ymm1, %ymm0, %ymm0
-; BDVER2-NEXT:    vfmaddps %ymm1, %ymm0, %ymm1, %ymm0
-; BDVER2-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
+; BDVER2-NEXT:    vmovaps {{.*#+}} ymm2 = [1.0E+0,2.0E+0,3.0E+0,4.0E+0,5.0E+0,6.0E+0,7.0E+0,8.0E+0]
+; BDVER2-NEXT:    vmulps %ymm2, %ymm1, %ymm3
+; BDVER2-NEXT:    vfnmaddps %ymm2, %ymm3, %ymm0, %ymm0
+; BDVER2-NEXT:    vfmaddps %ymm3, %ymm0, %ymm1, %ymm0
 ; BDVER2-NEXT:    retq
 ;
 ; BTVER2-LABEL: v8f32_one_step2:
 ; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmovaps {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; BTVER2-NEXT:    vmovaps {{.*#+}} ymm2 = [1.0E+0,2.0E+0,3.0E+0,4.0E+0,5.0E+0,6.0E+0,7.0E+0,8.0E+0]
 ; BTVER2-NEXT:    vrcpps %ymm0, %ymm1
-; BTVER2-NEXT:    vmulps %ymm1, %ymm0, %ymm0
+; BTVER2-NEXT:    vmulps %ymm2, %ymm1, %ymm3
+; BTVER2-NEXT:    vmulps %ymm3, %ymm0, %ymm0
 ; BTVER2-NEXT:    vsubps %ymm0, %ymm2, %ymm0
 ; BTVER2-NEXT:    vmulps %ymm0, %ymm1, %ymm0
-; BTVER2-NEXT:    vaddps %ymm0, %ymm1, %ymm0
-; BTVER2-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
+; BTVER2-NEXT:    vaddps %ymm0, %ymm3, %ymm0
 ; BTVER2-NEXT:    retq
 ;
 ; SANDY-LABEL: v8f32_one_step2:
 ; SANDY:       # %bb.0:
 ; SANDY-NEXT:    vrcpps %ymm0, %ymm1
-; SANDY-NEXT:    vmulps %ymm1, %ymm0, %ymm0
-; SANDY-NEXT:    vmovaps {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; SANDY-NEXT:    vmovaps {{.*#+}} ymm2 = [1.0E+0,2.0E+0,3.0E+0,4.0E+0,5.0E+0,6.0E+0,7.0E+0,8.0E+0]
+; SANDY-NEXT:    vmulps %ymm2, %ymm1, %ymm3
+; SANDY-NEXT:    vmulps %ymm3, %ymm0, %ymm0
 ; SANDY-NEXT:    vsubps %ymm0, %ymm2, %ymm0
 ; SANDY-NEXT:    vmulps %ymm0, %ymm1, %ymm0
-; SANDY-NEXT:    vaddps %ymm0, %ymm1, %ymm0
-; SANDY-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
+; SANDY-NEXT:    vaddps %ymm0, %ymm3, %ymm0
 ; SANDY-NEXT:    retq
 ;
 ; HASWELL-LABEL: v8f32_one_step2:
 ; HASWELL:       # %bb.0:
 ; HASWELL-NEXT:    vrcpps %ymm0, %ymm1
-; HASWELL-NEXT:    vbroadcastss {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm1 * ymm0) + ymm2
-; HASWELL-NEXT:    vfmadd132ps {{.*#+}} ymm0 = (ymm0 * ymm1) + ymm1
-; HASWELL-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
+; HASWELL-NEXT:    vmovaps {{.*#+}} ymm2 = [1.0E+0,2.0E+0,3.0E+0,4.0E+0,5.0E+0,6.0E+0,7.0E+0,8.0E+0]
+; HASWELL-NEXT:    vmulps %ymm2, %ymm1, %ymm3
+; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm3 * ymm0) + ymm2
+; HASWELL-NEXT:    vfmadd213ps {{.*#+}} ymm0 = (ymm1 * ymm0) + ymm3
 ; HASWELL-NEXT:    retq
 ;
 ; HASWELL-NO-FMA-LABEL: v8f32_one_step2:
 ; HASWELL-NO-FMA:       # %bb.0:
 ; HASWELL-NO-FMA-NEXT:    vrcpps %ymm0, %ymm1
-; HASWELL-NO-FMA-NEXT:    vmulps %ymm1, %ymm0, %ymm0
-; HASWELL-NO-FMA-NEXT:    vbroadcastss {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; HASWELL-NO-FMA-NEXT:    vmovaps {{.*#+}} ymm2 = [1.0E+0,2.0E+0,3.0E+0,4.0E+0,5.0E+0,6.0E+0,7.0E+0,8.0E+0]
+; HASWELL-NO-FMA-NEXT:    vmulps %ymm2, %ymm1, %ymm3
+; HASWELL-NO-FMA-NEXT:    vmulps %ymm3, %ymm0, %ymm0
 ; HASWELL-NO-FMA-NEXT:    vsubps %ymm0, %ymm2, %ymm0
 ; HASWELL-NO-FMA-NEXT:    vmulps %ymm0, %ymm1, %ymm0
-; HASWELL-NO-FMA-NEXT:    vaddps %ymm0, %ymm1, %ymm0
-; HASWELL-NO-FMA-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
+; HASWELL-NO-FMA-NEXT:    vaddps %ymm0, %ymm3, %ymm0
 ; HASWELL-NO-FMA-NEXT:    retq
 ;
-; KNL-LABEL: v8f32_one_step2:
-; KNL:       # %bb.0:
-; KNL-NEXT:    vrcpps %ymm0, %ymm1
-; KNL-NEXT:    vbroadcastss {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; KNL-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm1 * ymm0) + ymm2
-; KNL-NEXT:    vfmadd132ps {{.*#+}} ymm0 = (ymm0 * ymm1) + ymm1
-; KNL-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
-; KNL-NEXT:    retq
-;
-; SKX-LABEL: v8f32_one_step2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vrcpps %ymm0, %ymm1
-; SKX-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm1 * ymm0) + mem
-; SKX-NEXT:    vfmadd132ps {{.*#+}} ymm0 = (ymm0 * ymm1) + ymm1
-; SKX-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
-; SKX-NEXT:    retq
+; AVX512-LABEL: v8f32_one_step2:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vrcpps %ymm0, %ymm1
+; AVX512-NEXT:    vmovaps {{.*#+}} ymm2 = [1.0E+0,2.0E+0,3.0E+0,4.0E+0,5.0E+0,6.0E+0,7.0E+0,8.0E+0]
+; AVX512-NEXT:    vmulps %ymm2, %ymm1, %ymm3
+; AVX512-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm3 * ymm0) + ymm2
+; AVX512-NEXT:    vfmadd213ps {{.*#+}} ymm0 = (ymm1 * ymm0) + ymm3
+; AVX512-NEXT:    retq
   %div = fdiv fast <8 x float> <float 1.0, float 2.0, float 3.0, float 4.0, float 5.0, float 6.0, float 7.0, float 8.0>, %x
   ret <8 x float> %div
 }
@@ -930,33 +936,35 @@ define <8 x float> @v8f32_one_step_2_divs(<8 x float> %x) #1 {
 define <8 x float> @v8f32_two_step2(<8 x float> %x) #2 {
 ; SSE-LABEL: v8f32_two_step2:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    movaps %xmm0, %xmm2
-; SSE-NEXT:    rcpps %xmm1, %xmm3
-; SSE-NEXT:    movaps %xmm1, %xmm4
-; SSE-NEXT:    mulps %xmm3, %xmm4
-; SSE-NEXT:    movaps {{.*#+}} xmm0 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; SSE-NEXT:    movaps %xmm0, %xmm5
-; SSE-NEXT:    subps %xmm4, %xmm5
-; SSE-NEXT:    mulps %xmm3, %xmm5
-; SSE-NEXT:    addps %xmm3, %xmm5
-; SSE-NEXT:    mulps %xmm5, %xmm1
+; SSE-NEXT:    rcpps %xmm0, %xmm2
 ; SSE-NEXT:    movaps %xmm0, %xmm3
-; SSE-NEXT:    subps %xmm1, %xmm3
-; SSE-NEXT:    mulps %xmm5, %xmm3
-; SSE-NEXT:    addps %xmm5, %xmm3
-; SSE-NEXT:    rcpps %xmm2, %xmm1
-; SSE-NEXT:    movaps %xmm2, %xmm4
-; SSE-NEXT:    mulps %xmm1, %xmm4
-; SSE-NEXT:    movaps %xmm0, %xmm5
-; SSE-NEXT:    subps %xmm4, %xmm5
-; SSE-NEXT:    mulps %xmm1, %xmm5
-; SSE-NEXT:    addps %xmm1, %xmm5
+; SSE-NEXT:    mulps %xmm2, %xmm3
+; SSE-NEXT:    movaps {{.*#+}} xmm4 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; SSE-NEXT:    movaps %xmm4, %xmm5
+; SSE-NEXT:    subps %xmm3, %xmm5
+; SSE-NEXT:    mulps %xmm2, %xmm5
+; SSE-NEXT:    addps %xmm2, %xmm5
+; SSE-NEXT:    movaps {{.*#+}} xmm2 = [1.0E+0,2.0E+0,3.0E+0,4.0E+0]
+; SSE-NEXT:    movaps %xmm5, %xmm3
+; SSE-NEXT:    mulps %xmm2, %xmm3
+; SSE-NEXT:    mulps %xmm3, %xmm0
+; SSE-NEXT:    subps %xmm0, %xmm2
 ; SSE-NEXT:    mulps %xmm5, %xmm2
-; SSE-NEXT:    subps %xmm2, %xmm0
-; SSE-NEXT:    mulps %xmm5, %xmm0
-; SSE-NEXT:    addps %xmm5, %xmm0
-; SSE-NEXT:    mulps {{.*}}(%rip), %xmm0
-; SSE-NEXT:    mulps {{.*}}(%rip), %xmm3
+; SSE-NEXT:    addps %xmm3, %xmm2
+; SSE-NEXT:    rcpps %xmm1, %xmm0
+; SSE-NEXT:    movaps %xmm1, %xmm3
+; SSE-NEXT:    mulps %xmm0, %xmm3
+; SSE-NEXT:    subps %xmm3, %xmm4
+; SSE-NEXT:    mulps %xmm0, %xmm4
+; SSE-NEXT:    addps %xmm0, %xmm4
+; SSE-NEXT:    movaps {{.*#+}} xmm3 = [5.0E+0,6.0E+0,7.0E+0,8.0E+0]
+; SSE-NEXT:    movaps %xmm4, %xmm0
+; SSE-NEXT:    mulps %xmm3, %xmm0
+; SSE-NEXT:    mulps %xmm0, %xmm1
+; SSE-NEXT:    subps %xmm1, %xmm3
+; SSE-NEXT:    mulps %xmm4, %xmm3
+; SSE-NEXT:    addps %xmm0, %xmm3
+; SSE-NEXT:    movaps %xmm2, %xmm0
 ; SSE-NEXT:    movaps %xmm3, %xmm1
 ; SSE-NEXT:    retq
 ;
@@ -968,49 +976,51 @@ define <8 x float> @v8f32_two_step2(<8 x float> %x) #2 {
 ; AVX-RECIP-NEXT:    vsubps %ymm2, %ymm3, %ymm2
 ; AVX-RECIP-NEXT:    vmulps %ymm2, %ymm1, %ymm2
 ; AVX-RECIP-NEXT:    vaddps %ymm2, %ymm1, %ymm1
-; AVX-RECIP-NEXT:    vmulps %ymm1, %ymm0, %ymm0
-; AVX-RECIP-NEXT:    vsubps %ymm0, %ymm3, %ymm0
+; AVX-RECIP-NEXT:    vmovaps {{.*#+}} ymm2 = [1.0E+0,2.0E+0,3.0E+0,4.0E+0,5.0E+0,6.0E+0,7.0E+0,8.0E+0]
+; AVX-RECIP-NEXT:    vmulps %ymm2, %ymm1, %ymm3
+; AVX-RECIP-NEXT:    vmulps %ymm3, %ymm0, %ymm0
+; AVX-RECIP-NEXT:    vsubps %ymm0, %ymm2, %ymm0
 ; AVX-RECIP-NEXT:    vmulps %ymm0, %ymm1, %ymm0
-; AVX-RECIP-NEXT:    vaddps %ymm0, %ymm1, %ymm0
-; AVX-RECIP-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
+; AVX-RECIP-NEXT:    vaddps %ymm0, %ymm3, %ymm0
 ; AVX-RECIP-NEXT:    retq
 ;
 ; FMA-RECIP-LABEL: v8f32_two_step2:
 ; FMA-RECIP:       # %bb.0:
 ; FMA-RECIP-NEXT:    vrcpps %ymm0, %ymm1
-; FMA-RECIP-NEXT:    vmovaps {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; FMA-RECIP-NEXT:    vmovaps %ymm1, %ymm3
-; FMA-RECIP-NEXT:    vfnmadd213ps {{.*#+}} ymm3 = -(ymm0 * ymm3) + ymm2
-; FMA-RECIP-NEXT:    vfmadd132ps {{.*#+}} ymm3 = (ymm3 * ymm1) + ymm1
-; FMA-RECIP-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm3 * ymm0) + ymm2
-; FMA-RECIP-NEXT:    vfmadd132ps {{.*#+}} ymm0 = (ymm0 * ymm3) + ymm3
-; FMA-RECIP-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
+; FMA-RECIP-NEXT:    vmovaps {{.*#+}} ymm2 = [-1.0E+0,-1.0E+0,-1.0E+0,-1.0E+0,-1.0E+0,-1.0E+0,-1.0E+0,-1.0E+0]
+; FMA-RECIP-NEXT:    vfmadd231ps {{.*#+}} ymm2 = (ymm0 * ymm1) + ymm2
+; FMA-RECIP-NEXT:    vfnmadd132ps {{.*#+}} ymm2 = -(ymm2 * ymm1) + ymm1
+; FMA-RECIP-NEXT:    vmovaps {{.*#+}} ymm1 = [1.0E+0,2.0E+0,3.0E+0,4.0E+0,5.0E+0,6.0E+0,7.0E+0,8.0E+0]
+; FMA-RECIP-NEXT:    vmulps %ymm1, %ymm2, %ymm3
+; FMA-RECIP-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm3 * ymm0) + ymm1
+; FMA-RECIP-NEXT:    vfmadd213ps {{.*#+}} ymm0 = (ymm2 * ymm0) + ymm3
 ; FMA-RECIP-NEXT:    retq
 ;
 ; BDVER2-LABEL: v8f32_two_step2:
 ; BDVER2:       # %bb.0:
 ; BDVER2-NEXT:    vrcpps %ymm0, %ymm1
-; BDVER2-NEXT:    vmovaps {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; BDVER2-NEXT:    vfnmaddps %ymm2, %ymm1, %ymm0, %ymm3
-; BDVER2-NEXT:    vfmaddps %ymm1, %ymm3, %ymm1, %ymm1
-; BDVER2-NEXT:    vfnmaddps %ymm2, %ymm1, %ymm0, %ymm0
-; BDVER2-NEXT:    vfmaddps %ymm1, %ymm0, %ymm1, %ymm0
-; BDVER2-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
+; BDVER2-NEXT:    vfmaddps {{.*}}(%rip), %ymm1, %ymm0, %ymm2
+; BDVER2-NEXT:    vmovaps {{.*#+}} ymm4 = [1.0E+0,2.0E+0,3.0E+0,4.0E+0,5.0E+0,6.0E+0,7.0E+0,8.0E+0]
+; BDVER2-NEXT:    vfnmaddps %ymm1, %ymm2, %ymm1, %ymm1
+; BDVER2-NEXT:    vmulps %ymm4, %ymm1, %ymm3
+; BDVER2-NEXT:    vfnmaddps %ymm4, %ymm3, %ymm0, %ymm0
+; BDVER2-NEXT:    vfmaddps %ymm3, %ymm0, %ymm1, %ymm0
 ; BDVER2-NEXT:    retq
 ;
 ; BTVER2-LABEL: v8f32_two_step2:
 ; BTVER2:       # %bb.0:
 ; BTVER2-NEXT:    vmovaps {{.*#+}} ymm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
 ; BTVER2-NEXT:    vrcpps %ymm0, %ymm1
+; BTVER2-NEXT:    vmovaps {{.*#+}} ymm4 = [1.0E+0,2.0E+0,3.0E+0,4.0E+0,5.0E+0,6.0E+0,7.0E+0,8.0E+0]
 ; BTVER2-NEXT:    vmulps %ymm1, %ymm0, %ymm2
 ; BTVER2-NEXT:    vsubps %ymm2, %ymm3, %ymm2
 ; BTVER2-NEXT:    vmulps %ymm2, %ymm1, %ymm2
 ; BTVER2-NEXT:    vaddps %ymm2, %ymm1, %ymm1
-; BTVER2-NEXT:    vmulps %ymm1, %ymm0, %ymm0
-; BTVER2-NEXT:    vsubps %ymm0, %ymm3, %ymm0
+; BTVER2-NEXT:    vmulps %ymm4, %ymm1, %ymm3
+; BTVER2-NEXT:    vmulps %ymm3, %ymm0, %ymm0
+; BTVER2-NEXT:    vsubps %ymm0, %ymm4, %ymm0
 ; BTVER2-NEXT:    vmulps %ymm0, %ymm1, %ymm0
-; BTVER2-NEXT:    vaddps %ymm0, %ymm1, %ymm0
-; BTVER2-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
+; BTVER2-NEXT:    vaddps %ymm0, %ymm3, %ymm0
 ; BTVER2-NEXT:    retq
 ;
 ; SANDY-LABEL: v8f32_two_step2:
@@ -1021,23 +1031,24 @@ define <8 x float> @v8f32_two_step2(<8 x float> %x) #2 {
 ; SANDY-NEXT:    vsubps %ymm2, %ymm3, %ymm2
 ; SANDY-NEXT:    vmulps %ymm2, %ymm1, %ymm2
 ; SANDY-NEXT:    vaddps %ymm2, %ymm1, %ymm1
-; SANDY-NEXT:    vmulps %ymm1, %ymm0, %ymm0
-; SANDY-NEXT:    vsubps %ymm0, %ymm3, %ymm0
+; SANDY-NEXT:    vmovaps {{.*#+}} ymm2 = [1.0E+0,2.0E+0,3.0E+0,4.0E+0,5.0E+0,6.0E+0,7.0E+0,8.0E+0]
+; SANDY-NEXT:    vmulps %ymm2, %ymm1, %ymm3
+; SANDY-NEXT:    vmulps %ymm3, %ymm0, %ymm0
+; SANDY-NEXT:    vsubps %ymm0, %ymm2, %ymm0
 ; SANDY-NEXT:    vmulps %ymm0, %ymm1, %ymm0
-; SANDY-NEXT:    vaddps %ymm0, %ymm1, %ymm0
-; SANDY-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
+; SANDY-NEXT:    vaddps %ymm0, %ymm3, %ymm0
 ; SANDY-NEXT:    retq
 ;
 ; HASWELL-LABEL: v8f32_two_step2:
 ; HASWELL:       # %bb.0:
 ; HASWELL-NEXT:    vrcpps %ymm0, %ymm1
-; HASWELL-NEXT:    vbroadcastss {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; HASWELL-NEXT:    vmovaps %ymm1, %ymm3
-; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} ymm3 = -(ymm0 * ymm3) + ymm2
-; HASWELL-NEXT:    vfmadd132ps {{.*#+}} ymm3 = (ymm3 * ymm1) + ymm1
-; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm3 * ymm0) + ymm2
-; HASWELL-NEXT:    vfmadd132ps {{.*#+}} ymm0 = (ymm0 * ymm3) + ymm3
-; HASWELL-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
+; HASWELL-NEXT:    vbroadcastss {{.*#+}} ymm2 = [-1.0E+0,-1.0E+0,-1.0E+0,-1.0E+0,-1.0E+0,-1.0E+0,-1.0E+0,-1.0E+0]
+; HASWELL-NEXT:    vfmadd231ps {{.*#+}} ymm2 = (ymm0 * ymm1) + ymm2
+; HASWELL-NEXT:    vfnmadd132ps {{.*#+}} ymm2 = -(ymm2 * ymm1) + ymm1
+; HASWELL-NEXT:    vmovaps {{.*#+}} ymm1 = [1.0E+0,2.0E+0,3.0E+0,4.0E+0,5.0E+0,6.0E+0,7.0E+0,8.0E+0]
+; HASWELL-NEXT:    vmulps %ymm1, %ymm2, %ymm3
+; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm3 * ymm0) + ymm1
+; HASWELL-NEXT:    vfmadd213ps {{.*#+}} ymm0 = (ymm2 * ymm0) + ymm3
 ; HASWELL-NEXT:    retq
 ;
 ; HASWELL-NO-FMA-LABEL: v8f32_two_step2:
@@ -1048,23 +1059,24 @@ define <8 x float> @v8f32_two_step2(<8 x float> %x) #2 {
 ; HASWELL-NO-FMA-NEXT:    vsubps %ymm2, %ymm3, %ymm2
 ; HASWELL-NO-FMA-NEXT:    vmulps %ymm2, %ymm1, %ymm2
 ; HASWELL-NO-FMA-NEXT:    vaddps %ymm2, %ymm1, %ymm1
-; HASWELL-NO-FMA-NEXT:    vmulps %ymm1, %ymm0, %ymm0
-; HASWELL-NO-FMA-NEXT:    vsubps %ymm0, %ymm3, %ymm0
+; HASWELL-NO-FMA-NEXT:    vmovaps {{.*#+}} ymm2 = [1.0E+0,2.0E+0,3.0E+0,4.0E+0,5.0E+0,6.0E+0,7.0E+0,8.0E+0]
+; HASWELL-NO-FMA-NEXT:    vmulps %ymm2, %ymm1, %ymm3
+; HASWELL-NO-FMA-NEXT:    vmulps %ymm3, %ymm0, %ymm0
+; HASWELL-NO-FMA-NEXT:    vsubps %ymm0, %ymm2, %ymm0
 ; HASWELL-NO-FMA-NEXT:    vmulps %ymm0, %ymm1, %ymm0
-; HASWELL-NO-FMA-NEXT:    vaddps %ymm0, %ymm1, %ymm0
-; HASWELL-NO-FMA-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
+; HASWELL-NO-FMA-NEXT:    vaddps %ymm0, %ymm3, %ymm0
 ; HASWELL-NO-FMA-NEXT:    retq
 ;
 ; AVX512-LABEL: v8f32_two_step2:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vrcpps %ymm0, %ymm1
-; AVX512-NEXT:    vbroadcastss {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; AVX512-NEXT:    vmovaps %ymm1, %ymm3
-; AVX512-NEXT:    vfnmadd213ps {{.*#+}} ymm3 = -(ymm0 * ymm3) + ymm2
-; AVX512-NEXT:    vfmadd132ps {{.*#+}} ymm3 = (ymm3 * ymm1) + ymm1
-; AVX512-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm3 * ymm0) + ymm2
-; AVX512-NEXT:    vfmadd132ps {{.*#+}} ymm0 = (ymm0 * ymm3) + ymm3
-; AVX512-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
+; AVX512-NEXT:    vbroadcastss {{.*#+}} ymm2 = [-1.0E+0,-1.0E+0,-1.0E+0,-1.0E+0,-1.0E+0,-1.0E+0,-1.0E+0,-1.0E+0]
+; AVX512-NEXT:    vfmadd231ps {{.*#+}} ymm2 = (ymm0 * ymm1) + ymm2
+; AVX512-NEXT:    vfnmadd132ps {{.*#+}} ymm2 = -(ymm2 * ymm1) + ymm1
+; AVX512-NEXT:    vmovaps {{.*#+}} ymm1 = [1.0E+0,2.0E+0,3.0E+0,4.0E+0,5.0E+0,6.0E+0,7.0E+0,8.0E+0]
+; AVX512-NEXT:    vmulps %ymm1, %ymm2, %ymm3
+; AVX512-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm3 * ymm0) + ymm1
+; AVX512-NEXT:    vfmadd213ps {{.*#+}} ymm0 = (ymm2 * ymm0) + ymm3
 ; AVX512-NEXT:    retq
   %div = fdiv fast <8 x float> <float 1.0, float 2.0, float 3.0, float 4.0, float 5.0, float 6.0, float 7.0, float 8.0>, %x
   ret <8 x float> %div
@@ -1088,9 +1100,9 @@ define <8 x float> @v8f32_no_step(<8 x float> %x) #3 {
 define <8 x float> @v8f32_no_step2(<8 x float> %x) #3 {
 ; SSE-LABEL: v8f32_no_step2:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    rcpps %xmm1, %xmm1
 ; SSE-NEXT:    rcpps %xmm0, %xmm0
 ; SSE-NEXT:    mulps {{.*}}(%rip), %xmm0
+; SSE-NEXT:    rcpps %xmm1, %xmm1
 ; SSE-NEXT:    mulps {{.*}}(%rip), %xmm1
 ; SSE-NEXT:    retq
 ;
@@ -1106,153 +1118,165 @@ define <8 x float> @v8f32_no_step2(<8 x float> %x) #3 {
 define <16 x float> @v16f32_one_step2(<16 x float> %x) #1 {
 ; SSE-LABEL: v16f32_one_step2:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    movaps %xmm3, %xmm4
-; SSE-NEXT:    movaps %xmm2, %xmm5
-; SSE-NEXT:    movaps %xmm0, %xmm6
-; SSE-NEXT:    rcpps %xmm3, %xmm2
-; SSE-NEXT:    mulps %xmm2, %xmm4
-; SSE-NEXT:    movaps {{.*#+}} xmm0 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; SSE-NEXT:    movaps %xmm0, %xmm3
-; SSE-NEXT:    subps %xmm4, %xmm3
-; SSE-NEXT:    mulps %xmm2, %xmm3
-; SSE-NEXT:    addps %xmm2, %xmm3
-; SSE-NEXT:    rcpps %xmm5, %xmm4
-; SSE-NEXT:    mulps %xmm4, %xmm5
-; SSE-NEXT:    movaps %xmm0, %xmm2
-; SSE-NEXT:    subps %xmm5, %xmm2
-; SSE-NEXT:    mulps %xmm4, %xmm2
-; SSE-NEXT:    addps %xmm4, %xmm2
-; SSE-NEXT:    rcpps %xmm1, %xmm5
+; SSE-NEXT:    movaps %xmm1, %xmm4
+; SSE-NEXT:    movaps %xmm0, %xmm1
+; SSE-NEXT:    rcpps %xmm0, %xmm5
+; SSE-NEXT:    movaps {{.*#+}} xmm0 = [1.0E+0,2.0E+0,3.0E+0,4.0E+0]
+; SSE-NEXT:    movaps %xmm5, %xmm6
+; SSE-NEXT:    mulps %xmm0, %xmm6
+; SSE-NEXT:    mulps %xmm6, %xmm1
+; SSE-NEXT:    subps %xmm1, %xmm0
+; SSE-NEXT:    mulps %xmm5, %xmm0
+; SSE-NEXT:    addps %xmm6, %xmm0
+; SSE-NEXT:    rcpps %xmm4, %xmm5
+; SSE-NEXT:    movaps {{.*#+}} xmm1 = [5.0E+0,6.0E+0,7.0E+0,8.0E+0]
+; SSE-NEXT:    movaps %xmm5, %xmm6
+; SSE-NEXT:    mulps %xmm1, %xmm6
+; SSE-NEXT:    mulps %xmm6, %xmm4
+; SSE-NEXT:    subps %xmm4, %xmm1
 ; SSE-NEXT:    mulps %xmm5, %xmm1
-; SSE-NEXT:    movaps %xmm0, %xmm4
-; SSE-NEXT:    subps %xmm1, %xmm4
+; SSE-NEXT:    addps %xmm6, %xmm1
+; SSE-NEXT:    rcpps %xmm2, %xmm5
+; SSE-NEXT:    movaps {{.*#+}} xmm4 = [9.0E+0,1.0E+1,1.1E+1,1.2E+1]
+; SSE-NEXT:    movaps %xmm5, %xmm6
+; SSE-NEXT:    mulps %xmm4, %xmm6
+; SSE-NEXT:    mulps %xmm6, %xmm2
+; SSE-NEXT:    subps %xmm2, %xmm4
 ; SSE-NEXT:    mulps %xmm5, %xmm4
-; SSE-NEXT:    addps %xmm5, %xmm4
-; SSE-NEXT:    rcpps %xmm6, %xmm1
-; SSE-NEXT:    mulps %xmm1, %xmm6
-; SSE-NEXT:    subps %xmm6, %xmm0
-; SSE-NEXT:    mulps %xmm1, %xmm0
-; SSE-NEXT:    addps %xmm1, %xmm0
-; SSE-NEXT:    mulps {{.*}}(%rip), %xmm0
-; SSE-NEXT:    mulps {{.*}}(%rip), %xmm4
-; SSE-NEXT:    mulps {{.*}}(%rip), %xmm2
-; SSE-NEXT:    mulps {{.*}}(%rip), %xmm3
-; SSE-NEXT:    movaps %xmm4, %xmm1
+; SSE-NEXT:    addps %xmm6, %xmm4
+; SSE-NEXT:    rcpps %xmm3, %xmm2
+; SSE-NEXT:    movaps {{.*#+}} xmm5 = [1.3E+1,1.4E+1,1.5E+1,1.6E+1]
+; SSE-NEXT:    movaps %xmm2, %xmm6
+; SSE-NEXT:    mulps %xmm5, %xmm6
+; SSE-NEXT:    mulps %xmm6, %xmm3
+; SSE-NEXT:    subps %xmm3, %xmm5
+; SSE-NEXT:    mulps %xmm2, %xmm5
+; SSE-NEXT:    addps %xmm6, %xmm5
+; SSE-NEXT:    movaps %xmm4, %xmm2
+; SSE-NEXT:    movaps %xmm5, %xmm3
 ; SSE-NEXT:    retq
 ;
 ; AVX-RECIP-LABEL: v16f32_one_step2:
 ; AVX-RECIP:       # %bb.0:
-; AVX-RECIP-NEXT:    vrcpps %ymm1, %ymm2
-; AVX-RECIP-NEXT:    vmulps %ymm2, %ymm1, %ymm1
-; AVX-RECIP-NEXT:    vmovaps {{.*#+}} ymm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; AVX-RECIP-NEXT:    vsubps %ymm1, %ymm3, %ymm1
-; AVX-RECIP-NEXT:    vmulps %ymm1, %ymm2, %ymm1
-; AVX-RECIP-NEXT:    vaddps %ymm1, %ymm2, %ymm1
 ; AVX-RECIP-NEXT:    vrcpps %ymm0, %ymm2
-; AVX-RECIP-NEXT:    vmulps %ymm2, %ymm0, %ymm0
+; AVX-RECIP-NEXT:    vmovaps {{.*#+}} ymm3 = [1.0E+0,2.0E+0,3.0E+0,4.0E+0,5.0E+0,6.0E+0,7.0E+0,8.0E+0]
+; AVX-RECIP-NEXT:    vmulps %ymm3, %ymm2, %ymm4
+; AVX-RECIP-NEXT:    vmulps %ymm4, %ymm0, %ymm0
 ; AVX-RECIP-NEXT:    vsubps %ymm0, %ymm3, %ymm0
 ; AVX-RECIP-NEXT:    vmulps %ymm0, %ymm2, %ymm0
-; AVX-RECIP-NEXT:    vaddps %ymm0, %ymm2, %ymm0
-; AVX-RECIP-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
-; AVX-RECIP-NEXT:    vmulps {{.*}}(%rip), %ymm1, %ymm1
+; AVX-RECIP-NEXT:    vaddps %ymm0, %ymm4, %ymm0
+; AVX-RECIP-NEXT:    vrcpps %ymm1, %ymm2
+; AVX-RECIP-NEXT:    vmovaps {{.*#+}} ymm3 = [9.0E+0,1.0E+1,1.1E+1,1.2E+1,1.3E+1,1.4E+1,1.5E+1,1.6E+1]
+; AVX-RECIP-NEXT:    vmulps %ymm3, %ymm2, %ymm4
+; AVX-RECIP-NEXT:    vmulps %ymm4, %ymm1, %ymm1
+; AVX-RECIP-NEXT:    vsubps %ymm1, %ymm3, %ymm1
+; AVX-RECIP-NEXT:    vmulps %ymm1, %ymm2, %ymm1
+; AVX-RECIP-NEXT:    vaddps %ymm1, %ymm4, %ymm1
 ; AVX-RECIP-NEXT:    retq
 ;
 ; FMA-RECIP-LABEL: v16f32_one_step2:
 ; FMA-RECIP:       # %bb.0:
-; FMA-RECIP-NEXT:    vrcpps %ymm1, %ymm2
-; FMA-RECIP-NEXT:    vmovaps {{.*#+}} ymm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; FMA-RECIP-NEXT:    vfnmadd213ps {{.*#+}} ymm1 = -(ymm2 * ymm1) + ymm3
-; FMA-RECIP-NEXT:    vfmadd132ps {{.*#+}} ymm1 = (ymm1 * ymm2) + ymm2
 ; FMA-RECIP-NEXT:    vrcpps %ymm0, %ymm2
-; FMA-RECIP-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm2 * ymm0) + ymm3
-; FMA-RECIP-NEXT:    vfmadd132ps {{.*#+}} ymm0 = (ymm0 * ymm2) + ymm2
-; FMA-RECIP-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
-; FMA-RECIP-NEXT:    vmulps {{.*}}(%rip), %ymm1, %ymm1
+; FMA-RECIP-NEXT:    vmovaps {{.*#+}} ymm3 = [1.0E+0,2.0E+0,3.0E+0,4.0E+0,5.0E+0,6.0E+0,7.0E+0,8.0E+0]
+; FMA-RECIP-NEXT:    vmulps %ymm3, %ymm2, %ymm4
+; FMA-RECIP-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm4 * ymm0) + ymm3
+; FMA-RECIP-NEXT:    vfmadd213ps {{.*#+}} ymm0 = (ymm2 * ymm0) + ymm4
+; FMA-RECIP-NEXT:    vrcpps %ymm1, %ymm2
+; FMA-RECIP-NEXT:    vmovaps {{.*#+}} ymm3 = [9.0E+0,1.0E+1,1.1E+1,1.2E+1,1.3E+1,1.4E+1,1.5E+1,1.6E+1]
+; FMA-RECIP-NEXT:    vmulps %ymm3, %ymm2, %ymm4
+; FMA-RECIP-NEXT:    vfnmadd213ps {{.*#+}} ymm1 = -(ymm4 * ymm1) + ymm3
+; FMA-RECIP-NEXT:    vfmadd213ps {{.*#+}} ymm1 = (ymm2 * ymm1) + ymm4
 ; FMA-RECIP-NEXT:    retq
 ;
 ; BDVER2-LABEL: v16f32_one_step2:
 ; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vrcpps %ymm1, %ymm2
-; BDVER2-NEXT:    vmovaps {{.*#+}} ymm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; BDVER2-NEXT:    vrcpps %ymm0, %ymm4
-; BDVER2-NEXT:    vfnmaddps %ymm3, %ymm2, %ymm1, %ymm1
+; BDVER2-NEXT:    vrcpps %ymm0, %ymm2
+; BDVER2-NEXT:    vmovaps {{.*#+}} ymm3 = [1.0E+0,2.0E+0,3.0E+0,4.0E+0,5.0E+0,6.0E+0,7.0E+0,8.0E+0]
+; BDVER2-NEXT:    vrcpps %ymm1, %ymm5
+; BDVER2-NEXT:    vmulps %ymm3, %ymm2, %ymm4
 ; BDVER2-NEXT:    vfnmaddps %ymm3, %ymm4, %ymm0, %ymm0
-; BDVER2-NEXT:    vfmaddps %ymm2, %ymm1, %ymm2, %ymm1
-; BDVER2-NEXT:    vfmaddps %ymm4, %ymm0, %ymm4, %ymm0
-; BDVER2-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
-; BDVER2-NEXT:    vmulps {{.*}}(%rip), %ymm1, %ymm1
+; BDVER2-NEXT:    vmovaps {{.*#+}} ymm3 = [9.0E+0,1.0E+1,1.1E+1,1.2E+1,1.3E+1,1.4E+1,1.5E+1,1.6E+1]
+; BDVER2-NEXT:    vfmaddps %ymm4, %ymm0, %ymm2, %ymm0
+; BDVER2-NEXT:    vmulps %ymm3, %ymm5, %ymm4
+; BDVER2-NEXT:    vfnmaddps %ymm3, %ymm4, %ymm1, %ymm1
+; BDVER2-NEXT:    vfmaddps %ymm4, %ymm1, %ymm5, %ymm1
 ; BDVER2-NEXT:    retq
 ;
 ; BTVER2-LABEL: v16f32_one_step2:
 ; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmovaps {{.*#+}} ymm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; BTVER2-NEXT:    vrcpps %ymm1, %ymm2
-; BTVER2-NEXT:    vrcpps %ymm0, %ymm4
-; BTVER2-NEXT:    vmulps %ymm2, %ymm1, %ymm1
+; BTVER2-NEXT:    vmovaps {{.*#+}} ymm3 = [1.0E+0,2.0E+0,3.0E+0,4.0E+0,5.0E+0,6.0E+0,7.0E+0,8.0E+0]
+; BTVER2-NEXT:    vrcpps %ymm0, %ymm2
+; BTVER2-NEXT:    vmulps %ymm3, %ymm2, %ymm4
 ; BTVER2-NEXT:    vmulps %ymm4, %ymm0, %ymm0
-; BTVER2-NEXT:    vsubps %ymm1, %ymm3, %ymm1
 ; BTVER2-NEXT:    vsubps %ymm0, %ymm3, %ymm0
-; BTVER2-NEXT:    vmulps %ymm1, %ymm2, %ymm1
-; BTVER2-NEXT:    vmulps %ymm0, %ymm4, %ymm0
-; BTVER2-NEXT:    vaddps %ymm1, %ymm2, %ymm1
+; BTVER2-NEXT:    vmovaps {{.*#+}} ymm3 = [9.0E+0,1.0E+1,1.1E+1,1.2E+1,1.3E+1,1.4E+1,1.5E+1,1.6E+1]
+; BTVER2-NEXT:    vmulps %ymm0, %ymm2, %ymm0
+; BTVER2-NEXT:    vrcpps %ymm1, %ymm2
+; BTVER2-NEXT:    vmulps %ymm3, %ymm2, %ymm5
 ; BTVER2-NEXT:    vaddps %ymm0, %ymm4, %ymm0
-; BTVER2-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
-; BTVER2-NEXT:    vmulps {{.*}}(%rip), %ymm1, %ymm1
+; BTVER2-NEXT:    vmulps %ymm5, %ymm1, %ymm1
+; BTVER2-NEXT:    vsubps %ymm1, %ymm3, %ymm1
+; BTVER2-NEXT:    vmulps %ymm1, %ymm2, %ymm1
+; BTVER2-NEXT:    vaddps %ymm1, %ymm5, %ymm1
 ; BTVER2-NEXT:    retq
 ;
 ; SANDY-LABEL: v16f32_one_step2:
 ; SANDY:       # %bb.0:
-; SANDY-NEXT:    vrcpps %ymm1, %ymm2
-; SANDY-NEXT:    vmulps %ymm2, %ymm1, %ymm1
-; SANDY-NEXT:    vmovaps {{.*#+}} ymm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; SANDY-NEXT:    vsubps %ymm1, %ymm3, %ymm1
-; SANDY-NEXT:    vmulps %ymm1, %ymm2, %ymm1
-; SANDY-NEXT:    vaddps %ymm1, %ymm2, %ymm1
 ; SANDY-NEXT:    vrcpps %ymm0, %ymm2
-; SANDY-NEXT:    vmulps %ymm2, %ymm0, %ymm0
+; SANDY-NEXT:    vmovaps {{.*#+}} ymm3 = [1.0E+0,2.0E+0,3.0E+0,4.0E+0,5.0E+0,6.0E+0,7.0E+0,8.0E+0]
+; SANDY-NEXT:    vmulps %ymm3, %ymm2, %ymm4
+; SANDY-NEXT:    vmulps %ymm4, %ymm0, %ymm0
 ; SANDY-NEXT:    vsubps %ymm0, %ymm3, %ymm0
+; SANDY-NEXT:    vrcpps %ymm1, %ymm3
 ; SANDY-NEXT:    vmulps %ymm0, %ymm2, %ymm0
-; SANDY-NEXT:    vaddps %ymm0, %ymm2, %ymm0
-; SANDY-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
-; SANDY-NEXT:    vmulps {{.*}}(%rip), %ymm1, %ymm1
+; SANDY-NEXT:    vaddps %ymm0, %ymm4, %ymm0
+; SANDY-NEXT:    vmovaps {{.*#+}} ymm2 = [9.0E+0,1.0E+1,1.1E+1,1.2E+1,1.3E+1,1.4E+1,1.5E+1,1.6E+1]
+; SANDY-NEXT:    vmulps %ymm2, %ymm3, %ymm4
+; SANDY-NEXT:    vmulps %ymm4, %ymm1, %ymm1
+; SANDY-NEXT:    vsubps %ymm1, %ymm2, %ymm1
+; SANDY-NEXT:    vmulps %ymm1, %ymm3, %ymm1
+; SANDY-NEXT:    vaddps %ymm1, %ymm4, %ymm1
 ; SANDY-NEXT:    retq
 ;
 ; HASWELL-LABEL: v16f32_one_step2:
 ; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vrcpps %ymm1, %ymm2
-; HASWELL-NEXT:    vbroadcastss {{.*#+}} ymm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; HASWELL-NEXT:    vrcpps %ymm0, %ymm4
-; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} ymm1 = -(ymm2 * ymm1) + ymm3
-; HASWELL-NEXT:    vfmadd132ps {{.*#+}} ymm1 = (ymm1 * ymm2) + ymm2
+; HASWELL-NEXT:    vrcpps %ymm0, %ymm2
+; HASWELL-NEXT:    vmovaps {{.*#+}} ymm3 = [1.0E+0,2.0E+0,3.0E+0,4.0E+0,5.0E+0,6.0E+0,7.0E+0,8.0E+0]
+; HASWELL-NEXT:    vmulps %ymm3, %ymm2, %ymm4
 ; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm4 * ymm0) + ymm3
-; HASWELL-NEXT:    vfmadd132ps {{.*#+}} ymm0 = (ymm0 * ymm4) + ymm4
-; HASWELL-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
-; HASWELL-NEXT:    vmulps {{.*}}(%rip), %ymm1, %ymm1
+; HASWELL-NEXT:    vfmadd213ps {{.*#+}} ymm0 = (ymm2 * ymm0) + ymm4
+; HASWELL-NEXT:    vrcpps %ymm1, %ymm2
+; HASWELL-NEXT:    vmovaps {{.*#+}} ymm3 = [9.0E+0,1.0E+1,1.1E+1,1.2E+1,1.3E+1,1.4E+1,1.5E+1,1.6E+1]
+; HASWELL-NEXT:    vmulps %ymm3, %ymm2, %ymm4
+; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} ymm1 = -(ymm4 * ymm1) + ymm3
+; HASWELL-NEXT:    vfmadd213ps {{.*#+}} ymm1 = (ymm2 * ymm1) + ymm4
 ; HASWELL-NEXT:    retq
 ;
 ; HASWELL-NO-FMA-LABEL: v16f32_one_step2:
 ; HASWELL-NO-FMA:       # %bb.0:
-; HASWELL-NO-FMA-NEXT:    vrcpps %ymm1, %ymm2
-; HASWELL-NO-FMA-NEXT:    vmulps %ymm2, %ymm1, %ymm1
-; HASWELL-NO-FMA-NEXT:    vbroadcastss {{.*#+}} ymm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; HASWELL-NO-FMA-NEXT:    vsubps %ymm1, %ymm3, %ymm1
-; HASWELL-NO-FMA-NEXT:    vmulps %ymm1, %ymm2, %ymm1
-; HASWELL-NO-FMA-NEXT:    vaddps %ymm1, %ymm2, %ymm1
 ; HASWELL-NO-FMA-NEXT:    vrcpps %ymm0, %ymm2
-; HASWELL-NO-FMA-NEXT:    vmulps %ymm2, %ymm0, %ymm0
+; HASWELL-NO-FMA-NEXT:    vmovaps {{.*#+}} ymm3 = [1.0E+0,2.0E+0,3.0E+0,4.0E+0,5.0E+0,6.0E+0,7.0E+0,8.0E+0]
+; HASWELL-NO-FMA-NEXT:    vmulps %ymm3, %ymm2, %ymm4
+; HASWELL-NO-FMA-NEXT:    vmulps %ymm4, %ymm0, %ymm0
 ; HASWELL-NO-FMA-NEXT:    vsubps %ymm0, %ymm3, %ymm0
+; HASWELL-NO-FMA-NEXT:    vrcpps %ymm1, %ymm3
 ; HASWELL-NO-FMA-NEXT:    vmulps %ymm0, %ymm2, %ymm0
-; HASWELL-NO-FMA-NEXT:    vaddps %ymm0, %ymm2, %ymm0
-; HASWELL-NO-FMA-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
-; HASWELL-NO-FMA-NEXT:    vmulps {{.*}}(%rip), %ymm1, %ymm1
+; HASWELL-NO-FMA-NEXT:    vaddps %ymm0, %ymm4, %ymm0
+; HASWELL-NO-FMA-NEXT:    vmovaps {{.*#+}} ymm2 = [9.0E+0,1.0E+1,1.1E+1,1.2E+1,1.3E+1,1.4E+1,1.5E+1,1.6E+1]
+; HASWELL-NO-FMA-NEXT:    vmulps %ymm2, %ymm3, %ymm4
+; HASWELL-NO-FMA-NEXT:    vmulps %ymm4, %ymm1, %ymm1
+; HASWELL-NO-FMA-NEXT:    vsubps %ymm1, %ymm2, %ymm1
+; HASWELL-NO-FMA-NEXT:    vmulps %ymm1, %ymm3, %ymm1
+; HASWELL-NO-FMA-NEXT:    vaddps %ymm1, %ymm4, %ymm1
 ; HASWELL-NO-FMA-NEXT:    retq
 ;
 ; AVX512-LABEL: v16f32_one_step2:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vrcp14ps %zmm0, %zmm1
-; AVX512-NEXT:    vfnmadd213ps {{.*#+}} zmm0 = -(zmm1 * zmm0) + mem
-; AVX512-NEXT:    vfmadd132ps {{.*#+}} zmm0 = (zmm0 * zmm1) + zmm1
-; AVX512-NEXT:    vmulps {{.*}}(%rip), %zmm0, %zmm0
+; AVX512-NEXT:    vmovaps {{.*#+}} zmm2 = [1.0E+0,2.0E+0,3.0E+0,4.0E+0,5.0E+0,6.0E+0,7.0E+0,8.0E+0,9.0E+0,1.0E+1,1.1E+1,1.2E+1,1.3E+1,1.4E+1,1.5E+1,1.6E+1]
+; AVX512-NEXT:    vmulps %zmm2, %zmm1, %zmm3
+; AVX512-NEXT:    vfnmadd213ps {{.*#+}} zmm0 = -(zmm3 * zmm0) + zmm2
+; AVX512-NEXT:    vfmadd213ps {{.*#+}} zmm0 = (zmm1 * zmm0) + zmm3
 ; AVX512-NEXT:    retq
   %div = fdiv fast <16 x float> <float 1.0, float 2.0, float 3.0, float 4.0, float 5.0, float 6.0, float 7.0, float 8.0, float 9.0, float 10.0, float 11.0, float 12.0, float 13.0, float 14.0, float 15.0, float 16.0>, %x
   ret <16 x float> %div
@@ -1436,228 +1460,245 @@ define <16 x float> @v16f32_one_step_2_divs(<16 x float> %x) #1 {
 define <16 x float> @v16f32_two_step2(<16 x float> %x) #2 {
 ; SSE-LABEL: v16f32_two_step2:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    movaps %xmm3, %xmm6
-; SSE-NEXT:    movaps %xmm2, %xmm5
-; SSE-NEXT:    movaps %xmm0, %xmm4
-; SSE-NEXT:    rcpps %xmm3, %xmm2
-; SSE-NEXT:    mulps %xmm2, %xmm3
-; SSE-NEXT:    movaps {{.*#+}} xmm0 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; SSE-NEXT:    movaps %xmm0, %xmm7
-; SSE-NEXT:    subps %xmm3, %xmm7
-; SSE-NEXT:    mulps %xmm2, %xmm7
-; SSE-NEXT:    addps %xmm2, %xmm7
-; SSE-NEXT:    mulps %xmm7, %xmm6
-; SSE-NEXT:    movaps %xmm0, %xmm3
-; SSE-NEXT:    subps %xmm6, %xmm3
-; SSE-NEXT:    mulps %xmm7, %xmm3
-; SSE-NEXT:    addps %xmm7, %xmm3
-; SSE-NEXT:    rcpps %xmm5, %xmm2
-; SSE-NEXT:    movaps %xmm5, %xmm6
-; SSE-NEXT:    mulps %xmm2, %xmm6
-; SSE-NEXT:    movaps %xmm0, %xmm7
-; SSE-NEXT:    subps %xmm6, %xmm7
-; SSE-NEXT:    mulps %xmm2, %xmm7
-; SSE-NEXT:    addps %xmm2, %xmm7
-; SSE-NEXT:    mulps %xmm7, %xmm5
-; SSE-NEXT:    movaps %xmm0, %xmm2
-; SSE-NEXT:    subps %xmm5, %xmm2
-; SSE-NEXT:    mulps %xmm7, %xmm2
-; SSE-NEXT:    addps %xmm7, %xmm2
-; SSE-NEXT:    rcpps %xmm1, %xmm5
-; SSE-NEXT:    movaps %xmm1, %xmm6
-; SSE-NEXT:    mulps %xmm5, %xmm6
-; SSE-NEXT:    movaps %xmm0, %xmm7
-; SSE-NEXT:    subps %xmm6, %xmm7
-; SSE-NEXT:    mulps %xmm5, %xmm7
-; SSE-NEXT:    addps %xmm5, %xmm7
-; SSE-NEXT:    mulps %xmm7, %xmm1
-; SSE-NEXT:    movaps %xmm0, %xmm5
-; SSE-NEXT:    subps %xmm1, %xmm5
-; SSE-NEXT:    mulps %xmm7, %xmm5
-; SSE-NEXT:    addps %xmm7, %xmm5
+; SSE-NEXT:    movaps %xmm1, %xmm4
+; SSE-NEXT:    movaps %xmm0, %xmm1
+; SSE-NEXT:    rcpps %xmm0, %xmm0
+; SSE-NEXT:    movaps %xmm1, %xmm5
+; SSE-NEXT:    mulps %xmm0, %xmm5
+; SSE-NEXT:    movaps {{.*#+}} xmm6 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; SSE-NEXT:    movaps %xmm6, %xmm7
+; SSE-NEXT:    subps %xmm5, %xmm7
+; SSE-NEXT:    mulps %xmm0, %xmm7
+; SSE-NEXT:    addps %xmm0, %xmm7
+; SSE-NEXT:    movaps {{.*#+}} xmm0 = [1.0E+0,2.0E+0,3.0E+0,4.0E+0]
+; SSE-NEXT:    movaps %xmm7, %xmm5
+; SSE-NEXT:    mulps %xmm0, %xmm5
+; SSE-NEXT:    mulps %xmm5, %xmm1
+; SSE-NEXT:    subps %xmm1, %xmm0
+; SSE-NEXT:    mulps %xmm7, %xmm0
+; SSE-NEXT:    addps %xmm5, %xmm0
 ; SSE-NEXT:    rcpps %xmm4, %xmm1
-; SSE-NEXT:    movaps %xmm4, %xmm6
-; SSE-NEXT:    mulps %xmm1, %xmm6
-; SSE-NEXT:    movaps %xmm0, %xmm7
-; SSE-NEXT:    subps %xmm6, %xmm7
+; SSE-NEXT:    movaps %xmm4, %xmm5
+; SSE-NEXT:    mulps %xmm1, %xmm5
+; SSE-NEXT:    movaps %xmm6, %xmm7
+; SSE-NEXT:    subps %xmm5, %xmm7
 ; SSE-NEXT:    mulps %xmm1, %xmm7
 ; SSE-NEXT:    addps %xmm1, %xmm7
+; SSE-NEXT:    movaps {{.*#+}} xmm1 = [5.0E+0,6.0E+0,7.0E+0,8.0E+0]
+; SSE-NEXT:    movaps %xmm7, %xmm5
+; SSE-NEXT:    mulps %xmm1, %xmm5
+; SSE-NEXT:    mulps %xmm5, %xmm4
+; SSE-NEXT:    subps %xmm4, %xmm1
+; SSE-NEXT:    mulps %xmm7, %xmm1
+; SSE-NEXT:    addps %xmm5, %xmm1
+; SSE-NEXT:    rcpps %xmm2, %xmm4
+; SSE-NEXT:    movaps %xmm2, %xmm5
+; SSE-NEXT:    mulps %xmm4, %xmm5
+; SSE-NEXT:    movaps %xmm6, %xmm7
+; SSE-NEXT:    subps %xmm5, %xmm7
+; SSE-NEXT:    mulps %xmm4, %xmm7
+; SSE-NEXT:    addps %xmm4, %xmm7
+; SSE-NEXT:    movaps {{.*#+}} xmm4 = [9.0E+0,1.0E+1,1.1E+1,1.2E+1]
+; SSE-NEXT:    movaps %xmm7, %xmm5
+; SSE-NEXT:    mulps %xmm4, %xmm5
+; SSE-NEXT:    mulps %xmm5, %xmm2
+; SSE-NEXT:    subps %xmm2, %xmm4
 ; SSE-NEXT:    mulps %xmm7, %xmm4
-; SSE-NEXT:    subps %xmm4, %xmm0
-; SSE-NEXT:    mulps %xmm7, %xmm0
-; SSE-NEXT:    addps %xmm7, %xmm0
-; SSE-NEXT:    mulps {{.*}}(%rip), %xmm0
-; SSE-NEXT:    mulps {{.*}}(%rip), %xmm5
-; SSE-NEXT:    mulps {{.*}}(%rip), %xmm2
-; SSE-NEXT:    mulps {{.*}}(%rip), %xmm3
-; SSE-NEXT:    movaps %xmm5, %xmm1
+; SSE-NEXT:    addps %xmm5, %xmm4
+; SSE-NEXT:    rcpps %xmm3, %xmm2
+; SSE-NEXT:    movaps %xmm3, %xmm5
+; SSE-NEXT:    mulps %xmm2, %xmm5
+; SSE-NEXT:    subps %xmm5, %xmm6
+; SSE-NEXT:    mulps %xmm2, %xmm6
+; SSE-NEXT:    addps %xmm2, %xmm6
+; SSE-NEXT:    movaps {{.*#+}} xmm5 = [1.3E+1,1.4E+1,1.5E+1,1.6E+1]
+; SSE-NEXT:    movaps %xmm6, %xmm2
+; SSE-NEXT:    mulps %xmm5, %xmm2
+; SSE-NEXT:    mulps %xmm2, %xmm3
+; SSE-NEXT:    subps %xmm3, %xmm5
+; SSE-NEXT:    mulps %xmm6, %xmm5
+; SSE-NEXT:    addps %xmm2, %xmm5
+; SSE-NEXT:    movaps %xmm4, %xmm2
+; SSE-NEXT:    movaps %xmm5, %xmm3
 ; SSE-NEXT:    retq
 ;
 ; AVX-RECIP-LABEL: v16f32_two_step2:
 ; AVX-RECIP:       # %bb.0:
-; AVX-RECIP-NEXT:    vrcpps %ymm1, %ymm2
-; AVX-RECIP-NEXT:    vmulps %ymm2, %ymm1, %ymm3
+; AVX-RECIP-NEXT:    vrcpps %ymm0, %ymm2
+; AVX-RECIP-NEXT:    vmulps %ymm2, %ymm0, %ymm3
 ; AVX-RECIP-NEXT:    vmovaps {{.*#+}} ymm4 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
 ; AVX-RECIP-NEXT:    vsubps %ymm3, %ymm4, %ymm3
 ; AVX-RECIP-NEXT:    vmulps %ymm3, %ymm2, %ymm3
 ; AVX-RECIP-NEXT:    vaddps %ymm3, %ymm2, %ymm2
-; AVX-RECIP-NEXT:    vmulps %ymm2, %ymm1, %ymm1
-; AVX-RECIP-NEXT:    vsubps %ymm1, %ymm4, %ymm1
-; AVX-RECIP-NEXT:    vmulps %ymm1, %ymm2, %ymm1
-; AVX-RECIP-NEXT:    vaddps %ymm1, %ymm2, %ymm1
-; AVX-RECIP-NEXT:    vrcpps %ymm0, %ymm2
-; AVX-RECIP-NEXT:    vmulps %ymm2, %ymm0, %ymm3
+; AVX-RECIP-NEXT:    vmovaps {{.*#+}} ymm3 = [1.0E+0,2.0E+0,3.0E+0,4.0E+0,5.0E+0,6.0E+0,7.0E+0,8.0E+0]
+; AVX-RECIP-NEXT:    vmulps %ymm3, %ymm2, %ymm5
+; AVX-RECIP-NEXT:    vmulps %ymm5, %ymm0, %ymm0
+; AVX-RECIP-NEXT:    vsubps %ymm0, %ymm3, %ymm0
+; AVX-RECIP-NEXT:    vmulps %ymm0, %ymm2, %ymm0
+; AVX-RECIP-NEXT:    vaddps %ymm0, %ymm5, %ymm0
+; AVX-RECIP-NEXT:    vrcpps %ymm1, %ymm2
+; AVX-RECIP-NEXT:    vmulps %ymm2, %ymm1, %ymm3
 ; AVX-RECIP-NEXT:    vsubps %ymm3, %ymm4, %ymm3
 ; AVX-RECIP-NEXT:    vmulps %ymm3, %ymm2, %ymm3
 ; AVX-RECIP-NEXT:    vaddps %ymm3, %ymm2, %ymm2
-; AVX-RECIP-NEXT:    vmulps %ymm2, %ymm0, %ymm0
-; AVX-RECIP-NEXT:    vsubps %ymm0, %ymm4, %ymm0
-; AVX-RECIP-NEXT:    vmulps %ymm0, %ymm2, %ymm0
-; AVX-RECIP-NEXT:    vaddps %ymm0, %ymm2, %ymm0
-; AVX-RECIP-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
-; AVX-RECIP-NEXT:    vmulps {{.*}}(%rip), %ymm1, %ymm1
+; AVX-RECIP-NEXT:    vmovaps {{.*#+}} ymm3 = [9.0E+0,1.0E+1,1.1E+1,1.2E+1,1.3E+1,1.4E+1,1.5E+1,1.6E+1]
+; AVX-RECIP-NEXT:    vmulps %ymm3, %ymm2, %ymm4
+; AVX-RECIP-NEXT:    vmulps %ymm4, %ymm1, %ymm1
+; AVX-RECIP-NEXT:    vsubps %ymm1, %ymm3, %ymm1
+; AVX-RECIP-NEXT:    vmulps %ymm1, %ymm2, %ymm1
+; AVX-RECIP-NEXT:    vaddps %ymm1, %ymm4, %ymm1
 ; AVX-RECIP-NEXT:    retq
 ;
 ; FMA-RECIP-LABEL: v16f32_two_step2:
 ; FMA-RECIP:       # %bb.0:
-; FMA-RECIP-NEXT:    vrcpps %ymm1, %ymm2
-; FMA-RECIP-NEXT:    vmovaps {{.*#+}} ymm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; FMA-RECIP-NEXT:    vmovaps %ymm2, %ymm4
-; FMA-RECIP-NEXT:    vfnmadd213ps {{.*#+}} ymm4 = -(ymm1 * ymm4) + ymm3
-; FMA-RECIP-NEXT:    vfmadd132ps {{.*#+}} ymm4 = (ymm4 * ymm2) + ymm2
-; FMA-RECIP-NEXT:    vfnmadd213ps {{.*#+}} ymm1 = -(ymm4 * ymm1) + ymm3
-; FMA-RECIP-NEXT:    vfmadd132ps {{.*#+}} ymm1 = (ymm1 * ymm4) + ymm4
 ; FMA-RECIP-NEXT:    vrcpps %ymm0, %ymm2
+; FMA-RECIP-NEXT:    vmovaps {{.*#+}} ymm3 = [-1.0E+0,-1.0E+0,-1.0E+0,-1.0E+0,-1.0E+0,-1.0E+0,-1.0E+0,-1.0E+0]
 ; FMA-RECIP-NEXT:    vmovaps %ymm2, %ymm4
-; FMA-RECIP-NEXT:    vfnmadd213ps {{.*#+}} ymm4 = -(ymm0 * ymm4) + ymm3
-; FMA-RECIP-NEXT:    vfmadd132ps {{.*#+}} ymm4 = (ymm4 * ymm2) + ymm2
-; FMA-RECIP-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm4 * ymm0) + ymm3
-; FMA-RECIP-NEXT:    vfmadd132ps {{.*#+}} ymm0 = (ymm0 * ymm4) + ymm4
-; FMA-RECIP-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
-; FMA-RECIP-NEXT:    vmulps {{.*}}(%rip), %ymm1, %ymm1
+; FMA-RECIP-NEXT:    vfmadd213ps {{.*#+}} ymm4 = (ymm0 * ymm4) + ymm3
+; FMA-RECIP-NEXT:    vfnmadd132ps {{.*#+}} ymm4 = -(ymm4 * ymm2) + ymm2
+; FMA-RECIP-NEXT:    vmovaps {{.*#+}} ymm2 = [1.0E+0,2.0E+0,3.0E+0,4.0E+0,5.0E+0,6.0E+0,7.0E+0,8.0E+0]
+; FMA-RECIP-NEXT:    vmulps %ymm2, %ymm4, %ymm5
+; FMA-RECIP-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm5 * ymm0) + ymm2
+; FMA-RECIP-NEXT:    vfmadd213ps {{.*#+}} ymm0 = (ymm4 * ymm0) + ymm5
+; FMA-RECIP-NEXT:    vrcpps %ymm1, %ymm2
+; FMA-RECIP-NEXT:    vfmadd231ps {{.*#+}} ymm3 = (ymm1 * ymm2) + ymm3
+; FMA-RECIP-NEXT:    vfnmadd132ps {{.*#+}} ymm3 = -(ymm3 * ymm2) + ymm2
+; FMA-RECIP-NEXT:    vmovaps {{.*#+}} ymm2 = [9.0E+0,1.0E+1,1.1E+1,1.2E+1,1.3E+1,1.4E+1,1.5E+1,1.6E+1]
+; FMA-RECIP-NEXT:    vmulps %ymm2, %ymm3, %ymm4
+; FMA-RECIP-NEXT:    vfnmadd213ps {{.*#+}} ymm1 = -(ymm4 * ymm1) + ymm2
+; FMA-RECIP-NEXT:    vfmadd213ps {{.*#+}} ymm1 = (ymm3 * ymm1) + ymm4
 ; FMA-RECIP-NEXT:    retq
 ;
 ; BDVER2-LABEL: v16f32_two_step2:
 ; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vrcpps %ymm1, %ymm2
-; BDVER2-NEXT:    vmovaps {{.*#+}} ymm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; BDVER2-NEXT:    vfnmaddps %ymm3, %ymm2, %ymm1, %ymm4
-; BDVER2-NEXT:    vfmaddps %ymm2, %ymm4, %ymm2, %ymm2
-; BDVER2-NEXT:    vfnmaddps %ymm3, %ymm2, %ymm1, %ymm1
-; BDVER2-NEXT:    vfmaddps %ymm2, %ymm1, %ymm2, %ymm1
 ; BDVER2-NEXT:    vrcpps %ymm0, %ymm2
-; BDVER2-NEXT:    vmulps {{.*}}(%rip), %ymm1, %ymm1
-; BDVER2-NEXT:    vfnmaddps %ymm3, %ymm2, %ymm0, %ymm4
-; BDVER2-NEXT:    vfmaddps %ymm2, %ymm4, %ymm2, %ymm2
-; BDVER2-NEXT:    vfnmaddps %ymm3, %ymm2, %ymm0, %ymm0
-; BDVER2-NEXT:    vfmaddps %ymm2, %ymm0, %ymm2, %ymm0
-; BDVER2-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
+; BDVER2-NEXT:    vmovaps {{.*#+}} ymm3 = [-1.0E+0,-1.0E+0,-1.0E+0,-1.0E+0,-1.0E+0,-1.0E+0,-1.0E+0,-1.0E+0]
+; BDVER2-NEXT:    vfmaddps %ymm3, %ymm2, %ymm0, %ymm4
+; BDVER2-NEXT:    vfnmaddps %ymm2, %ymm4, %ymm2, %ymm2
+; BDVER2-NEXT:    vmovaps {{.*#+}} ymm4 = [1.0E+0,2.0E+0,3.0E+0,4.0E+0,5.0E+0,6.0E+0,7.0E+0,8.0E+0]
+; BDVER2-NEXT:    vmulps %ymm4, %ymm2, %ymm5
+; BDVER2-NEXT:    vfnmaddps %ymm4, %ymm5, %ymm0, %ymm0
+; BDVER2-NEXT:    vfmaddps %ymm5, %ymm0, %ymm2, %ymm0
+; BDVER2-NEXT:    vrcpps %ymm1, %ymm2
+; BDVER2-NEXT:    vmovaps {{.*#+}} ymm5 = [9.0E+0,1.0E+1,1.1E+1,1.2E+1,1.3E+1,1.4E+1,1.5E+1,1.6E+1]
+; BDVER2-NEXT:    vfmaddps %ymm3, %ymm2, %ymm1, %ymm3
+; BDVER2-NEXT:    vfnmaddps %ymm2, %ymm3, %ymm2, %ymm2
+; BDVER2-NEXT:    vmulps %ymm5, %ymm2, %ymm4
+; BDVER2-NEXT:    vfnmaddps %ymm5, %ymm4, %ymm1, %ymm1
+; BDVER2-NEXT:    vfmaddps %ymm4, %ymm1, %ymm2, %ymm1
 ; BDVER2-NEXT:    retq
 ;
 ; BTVER2-LABEL: v16f32_two_step2:
 ; BTVER2:       # %bb.0:
 ; BTVER2-NEXT:    vmovaps {{.*#+}} ymm4 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; BTVER2-NEXT:    vrcpps %ymm1, %ymm2
-; BTVER2-NEXT:    vmulps %ymm2, %ymm1, %ymm3
-; BTVER2-NEXT:    vsubps %ymm3, %ymm4, %ymm3
-; BTVER2-NEXT:    vmulps %ymm3, %ymm2, %ymm3
-; BTVER2-NEXT:    vaddps %ymm3, %ymm2, %ymm2
-; BTVER2-NEXT:    vmulps %ymm2, %ymm1, %ymm1
-; BTVER2-NEXT:    vsubps %ymm1, %ymm4, %ymm1
-; BTVER2-NEXT:    vmulps %ymm1, %ymm2, %ymm1
-; BTVER2-NEXT:    vaddps %ymm1, %ymm2, %ymm1
 ; BTVER2-NEXT:    vrcpps %ymm0, %ymm2
-; BTVER2-NEXT:    vmulps {{.*}}(%rip), %ymm1, %ymm1
 ; BTVER2-NEXT:    vmulps %ymm2, %ymm0, %ymm3
 ; BTVER2-NEXT:    vsubps %ymm3, %ymm4, %ymm3
 ; BTVER2-NEXT:    vmulps %ymm3, %ymm2, %ymm3
 ; BTVER2-NEXT:    vaddps %ymm3, %ymm2, %ymm2
-; BTVER2-NEXT:    vmulps %ymm2, %ymm0, %ymm0
-; BTVER2-NEXT:    vsubps %ymm0, %ymm4, %ymm0
+; BTVER2-NEXT:    vmovaps {{.*#+}} ymm3 = [1.0E+0,2.0E+0,3.0E+0,4.0E+0,5.0E+0,6.0E+0,7.0E+0,8.0E+0]
+; BTVER2-NEXT:    vmulps %ymm3, %ymm2, %ymm5
+; BTVER2-NEXT:    vmulps %ymm5, %ymm0, %ymm0
+; BTVER2-NEXT:    vsubps %ymm0, %ymm3, %ymm0
 ; BTVER2-NEXT:    vmulps %ymm0, %ymm2, %ymm0
-; BTVER2-NEXT:    vaddps %ymm0, %ymm2, %ymm0
-; BTVER2-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
+; BTVER2-NEXT:    vrcpps %ymm1, %ymm2
+; BTVER2-NEXT:    vmulps %ymm2, %ymm1, %ymm3
+; BTVER2-NEXT:    vaddps %ymm0, %ymm5, %ymm0
+; BTVER2-NEXT:    vmovaps {{.*#+}} ymm5 = [9.0E+0,1.0E+1,1.1E+1,1.2E+1,1.3E+1,1.4E+1,1.5E+1,1.6E+1]
+; BTVER2-NEXT:    vsubps %ymm3, %ymm4, %ymm3
+; BTVER2-NEXT:    vmulps %ymm3, %ymm2, %ymm3
+; BTVER2-NEXT:    vaddps %ymm3, %ymm2, %ymm2
+; BTVER2-NEXT:    vmulps %ymm5, %ymm2, %ymm4
+; BTVER2-NEXT:    vmulps %ymm4, %ymm1, %ymm1
+; BTVER2-NEXT:    vsubps %ymm1, %ymm5, %ymm1
+; BTVER2-NEXT:    vmulps %ymm1, %ymm2, %ymm1
+; BTVER2-NEXT:    vaddps %ymm1, %ymm4, %ymm1
 ; BTVER2-NEXT:    retq
 ;
 ; SANDY-LABEL: v16f32_two_step2:
 ; SANDY:       # %bb.0:
-; SANDY-NEXT:    vrcpps %ymm1, %ymm2
-; SANDY-NEXT:    vmulps %ymm2, %ymm1, %ymm3
-; SANDY-NEXT:    vmovaps {{.*#+}} ymm4 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; SANDY-NEXT:    vsubps %ymm3, %ymm4, %ymm3
-; SANDY-NEXT:    vmulps %ymm3, %ymm2, %ymm3
-; SANDY-NEXT:    vaddps %ymm3, %ymm2, %ymm2
-; SANDY-NEXT:    vmulps %ymm2, %ymm1, %ymm1
-; SANDY-NEXT:    vsubps %ymm1, %ymm4, %ymm1
-; SANDY-NEXT:    vmulps %ymm1, %ymm2, %ymm1
-; SANDY-NEXT:    vaddps %ymm1, %ymm2, %ymm1
 ; SANDY-NEXT:    vrcpps %ymm0, %ymm2
 ; SANDY-NEXT:    vmulps %ymm2, %ymm0, %ymm3
+; SANDY-NEXT:    vmovaps {{.*#+}} ymm4 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
 ; SANDY-NEXT:    vsubps %ymm3, %ymm4, %ymm3
 ; SANDY-NEXT:    vmulps %ymm3, %ymm2, %ymm3
 ; SANDY-NEXT:    vaddps %ymm3, %ymm2, %ymm2
-; SANDY-NEXT:    vmulps %ymm2, %ymm0, %ymm0
-; SANDY-NEXT:    vsubps %ymm0, %ymm4, %ymm0
+; SANDY-NEXT:    vmovaps {{.*#+}} ymm3 = [1.0E+0,2.0E+0,3.0E+0,4.0E+0,5.0E+0,6.0E+0,7.0E+0,8.0E+0]
+; SANDY-NEXT:    vmulps %ymm3, %ymm2, %ymm5
+; SANDY-NEXT:    vmulps %ymm5, %ymm0, %ymm0
+; SANDY-NEXT:    vsubps %ymm0, %ymm3, %ymm0
+; SANDY-NEXT:    vrcpps %ymm1, %ymm3
 ; SANDY-NEXT:    vmulps %ymm0, %ymm2, %ymm0
-; SANDY-NEXT:    vaddps %ymm0, %ymm2, %ymm0
-; SANDY-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
-; SANDY-NEXT:    vmulps {{.*}}(%rip), %ymm1, %ymm1
+; SANDY-NEXT:    vaddps %ymm0, %ymm5, %ymm0
+; SANDY-NEXT:    vmulps %ymm3, %ymm1, %ymm2
+; SANDY-NEXT:    vsubps %ymm2, %ymm4, %ymm2
+; SANDY-NEXT:    vmulps %ymm2, %ymm3, %ymm2
+; SANDY-NEXT:    vaddps %ymm2, %ymm3, %ymm2
+; SANDY-NEXT:    vmovaps {{.*#+}} ymm3 = [9.0E+0,1.0E+1,1.1E+1,1.2E+1,1.3E+1,1.4E+1,1.5E+1,1.6E+1]
+; SANDY-NEXT:    vmulps %ymm3, %ymm2, %ymm4
+; SANDY-NEXT:    vmulps %ymm4, %ymm1, %ymm1
+; SANDY-NEXT:    vsubps %ymm1, %ymm3, %ymm1
+; SANDY-NEXT:    vmulps %ymm1, %ymm2, %ymm1
+; SANDY-NEXT:    vaddps %ymm1, %ymm4, %ymm1
 ; SANDY-NEXT:    retq
 ;
 ; HASWELL-LABEL: v16f32_two_step2:
 ; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vrcpps %ymm1, %ymm2
-; HASWELL-NEXT:    vbroadcastss {{.*#+}} ymm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; HASWELL-NEXT:    vmovaps %ymm2, %ymm4
-; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} ymm4 = -(ymm1 * ymm4) + ymm3
-; HASWELL-NEXT:    vfmadd132ps {{.*#+}} ymm4 = (ymm4 * ymm2) + ymm2
-; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} ymm1 = -(ymm4 * ymm1) + ymm3
-; HASWELL-NEXT:    vfmadd132ps {{.*#+}} ymm1 = (ymm1 * ymm4) + ymm4
 ; HASWELL-NEXT:    vrcpps %ymm0, %ymm2
+; HASWELL-NEXT:    vbroadcastss {{.*#+}} ymm3 = [-1.0E+0,-1.0E+0,-1.0E+0,-1.0E+0,-1.0E+0,-1.0E+0,-1.0E+0,-1.0E+0]
 ; HASWELL-NEXT:    vmovaps %ymm2, %ymm4
-; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} ymm4 = -(ymm0 * ymm4) + ymm3
-; HASWELL-NEXT:    vfmadd132ps {{.*#+}} ymm4 = (ymm4 * ymm2) + ymm2
-; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm4 * ymm0) + ymm3
-; HASWELL-NEXT:    vfmadd132ps {{.*#+}} ymm0 = (ymm0 * ymm4) + ymm4
-; HASWELL-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
-; HASWELL-NEXT:    vmulps {{.*}}(%rip), %ymm1, %ymm1
+; HASWELL-NEXT:    vfmadd213ps {{.*#+}} ymm4 = (ymm0 * ymm4) + ymm3
+; HASWELL-NEXT:    vfnmadd132ps {{.*#+}} ymm4 = -(ymm4 * ymm2) + ymm2
+; HASWELL-NEXT:    vmovaps {{.*#+}} ymm2 = [1.0E+0,2.0E+0,3.0E+0,4.0E+0,5.0E+0,6.0E+0,7.0E+0,8.0E+0]
+; HASWELL-NEXT:    vmulps %ymm2, %ymm4, %ymm5
+; HASWELL-NEXT:    vrcpps %ymm1, %ymm6
+; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm5 * ymm0) + ymm2
+; HASWELL-NEXT:    vfmadd213ps {{.*#+}} ymm0 = (ymm4 * ymm0) + ymm5
+; HASWELL-NEXT:    vfmadd231ps {{.*#+}} ymm3 = (ymm1 * ymm6) + ymm3
+; HASWELL-NEXT:    vfnmadd132ps {{.*#+}} ymm3 = -(ymm3 * ymm6) + ymm6
+; HASWELL-NEXT:    vmovaps {{.*#+}} ymm2 = [9.0E+0,1.0E+1,1.1E+1,1.2E+1,1.3E+1,1.4E+1,1.5E+1,1.6E+1]
+; HASWELL-NEXT:    vmulps %ymm2, %ymm3, %ymm4
+; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} ymm1 = -(ymm4 * ymm1) + ymm2
+; HASWELL-NEXT:    vfmadd213ps {{.*#+}} ymm1 = (ymm3 * ymm1) + ymm4
 ; HASWELL-NEXT:    retq
 ;
 ; HASWELL-NO-FMA-LABEL: v16f32_two_step2:
 ; HASWELL-NO-FMA:       # %bb.0:
-; HASWELL-NO-FMA-NEXT:    vrcpps %ymm1, %ymm2
-; HASWELL-NO-FMA-NEXT:    vmulps %ymm2, %ymm1, %ymm3
-; HASWELL-NO-FMA-NEXT:    vbroadcastss {{.*#+}} ymm4 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; HASWELL-NO-FMA-NEXT:    vsubps %ymm3, %ymm4, %ymm3
-; HASWELL-NO-FMA-NEXT:    vmulps %ymm3, %ymm2, %ymm3
-; HASWELL-NO-FMA-NEXT:    vaddps %ymm3, %ymm2, %ymm2
-; HASWELL-NO-FMA-NEXT:    vmulps %ymm2, %ymm1, %ymm1
-; HASWELL-NO-FMA-NEXT:    vsubps %ymm1, %ymm4, %ymm1
-; HASWELL-NO-FMA-NEXT:    vmulps %ymm1, %ymm2, %ymm1
-; HASWELL-NO-FMA-NEXT:    vaddps %ymm1, %ymm2, %ymm1
 ; HASWELL-NO-FMA-NEXT:    vrcpps %ymm0, %ymm2
 ; HASWELL-NO-FMA-NEXT:    vmulps %ymm2, %ymm0, %ymm3
+; HASWELL-NO-FMA-NEXT:    vbroadcastss {{.*#+}} ymm4 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
 ; HASWELL-NO-FMA-NEXT:    vsubps %ymm3, %ymm4, %ymm3
 ; HASWELL-NO-FMA-NEXT:    vmulps %ymm3, %ymm2, %ymm3
 ; HASWELL-NO-FMA-NEXT:    vaddps %ymm3, %ymm2, %ymm2
-; HASWELL-NO-FMA-NEXT:    vmulps %ymm2, %ymm0, %ymm0
-; HASWELL-NO-FMA-NEXT:    vsubps %ymm0, %ymm4, %ymm0
+; HASWELL-NO-FMA-NEXT:    vmovaps {{.*#+}} ymm3 = [1.0E+0,2.0E+0,3.0E+0,4.0E+0,5.0E+0,6.0E+0,7.0E+0,8.0E+0]
+; HASWELL-NO-FMA-NEXT:    vmulps %ymm3, %ymm2, %ymm5
+; HASWELL-NO-FMA-NEXT:    vmulps %ymm5, %ymm0, %ymm0
+; HASWELL-NO-FMA-NEXT:    vsubps %ymm0, %ymm3, %ymm0
+; HASWELL-NO-FMA-NEXT:    vrcpps %ymm1, %ymm3
 ; HASWELL-NO-FMA-NEXT:    vmulps %ymm0, %ymm2, %ymm0
-; HASWELL-NO-FMA-NEXT:    vaddps %ymm0, %ymm2, %ymm0
-; HASWELL-NO-FMA-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
-; HASWELL-NO-FMA-NEXT:    vmulps {{.*}}(%rip), %ymm1, %ymm1
+; HASWELL-NO-FMA-NEXT:    vaddps %ymm0, %ymm5, %ymm0
+; HASWELL-NO-FMA-NEXT:    vmulps %ymm3, %ymm1, %ymm2
+; HASWELL-NO-FMA-NEXT:    vsubps %ymm2, %ymm4, %ymm2
+; HASWELL-NO-FMA-NEXT:    vmulps %ymm2, %ymm3, %ymm2
+; HASWELL-NO-FMA-NEXT:    vaddps %ymm2, %ymm3, %ymm2
+; HASWELL-NO-FMA-NEXT:    vmovaps {{.*#+}} ymm3 = [9.0E+0,1.0E+1,1.1E+1,1.2E+1,1.3E+1,1.4E+1,1.5E+1,1.6E+1]
+; HASWELL-NO-FMA-NEXT:    vmulps %ymm3, %ymm2, %ymm4
+; HASWELL-NO-FMA-NEXT:    vmulps %ymm4, %ymm1, %ymm1
+; HASWELL-NO-FMA-NEXT:    vsubps %ymm1, %ymm3, %ymm1
+; HASWELL-NO-FMA-NEXT:    vmulps %ymm1, %ymm2, %ymm1
+; HASWELL-NO-FMA-NEXT:    vaddps %ymm1, %ymm4, %ymm1
 ; HASWELL-NO-FMA-NEXT:    retq
 ;
 ; AVX512-LABEL: v16f32_two_step2:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vrcp14ps %zmm0, %zmm1
-; AVX512-NEXT:    vbroadcastss {{.*#+}} zmm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; AVX512-NEXT:    vmovaps %zmm1, %zmm3
-; AVX512-NEXT:    vfnmadd213ps {{.*#+}} zmm3 = -(zmm0 * zmm3) + zmm2
-; AVX512-NEXT:    vfmadd132ps {{.*#+}} zmm3 = (zmm3 * zmm1) + zmm1
-; AVX512-NEXT:    vfnmadd213ps {{.*#+}} zmm0 = -(zmm3 * zmm0) + zmm2
-; AVX512-NEXT:    vfmadd132ps {{.*#+}} zmm0 = (zmm0 * zmm3) + zmm3
-; AVX512-NEXT:    vmulps {{.*}}(%rip), %zmm0, %zmm0
+; AVX512-NEXT:    vbroadcastss {{.*#+}} zmm2 = [-1.0E+0,-1.0E+0,-1.0E+0,-1.0E+0,-1.0E+0,-1.0E+0,-1.0E+0,-1.0E+0,-1.0E+0,-1.0E+0,-1.0E+0,-1.0E+0,-1.0E+0,-1.0E+0,-1.0E+0,-1.0E+0]
+; AVX512-NEXT:    vfmadd231ps {{.*#+}} zmm2 = (zmm0 * zmm1) + zmm2
+; AVX512-NEXT:    vfnmadd132ps {{.*#+}} zmm2 = -(zmm2 * zmm1) + zmm1
+; AVX512-NEXT:    vmovaps {{.*#+}} zmm1 = [1.0E+0,2.0E+0,3.0E+0,4.0E+0,5.0E+0,6.0E+0,7.0E+0,8.0E+0,9.0E+0,1.0E+1,1.1E+1,1.2E+1,1.3E+1,1.4E+1,1.5E+1,1.6E+1]
+; AVX512-NEXT:    vmulps %zmm1, %zmm2, %zmm3
+; AVX512-NEXT:    vfnmadd213ps {{.*#+}} zmm0 = -(zmm3 * zmm0) + zmm1
+; AVX512-NEXT:    vfmadd213ps {{.*#+}} zmm0 = (zmm2 * zmm0) + zmm3
 ; AVX512-NEXT:    retq
   %div = fdiv fast <16 x float> <float 1.0, float 2.0, float 3.0, float 4.0, float 5.0, float 6.0, float 7.0, float 8.0, float 9.0, float 10.0, float 11.0, float 12.0, float 13.0, float 14.0, float 15.0, float 16.0>, %x
   ret <16 x float> %div
@@ -1725,68 +1766,68 @@ define <16 x float> @v16f32_no_step(<16 x float> %x) #3 {
 define <16 x float> @v16f32_no_step2(<16 x float> %x) #3 {
 ; SSE-LABEL: v16f32_no_step2:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    rcpps %xmm3, %xmm3
-; SSE-NEXT:    rcpps %xmm2, %xmm2
-; SSE-NEXT:    rcpps %xmm1, %xmm1
 ; SSE-NEXT:    rcpps %xmm0, %xmm0
 ; SSE-NEXT:    mulps {{.*}}(%rip), %xmm0
+; SSE-NEXT:    rcpps %xmm1, %xmm1
 ; SSE-NEXT:    mulps {{.*}}(%rip), %xmm1
+; SSE-NEXT:    rcpps %xmm2, %xmm2
 ; SSE-NEXT:    mulps {{.*}}(%rip), %xmm2
+; SSE-NEXT:    rcpps %xmm3, %xmm3
 ; SSE-NEXT:    mulps {{.*}}(%rip), %xmm3
 ; SSE-NEXT:    retq
 ;
 ; AVX-RECIP-LABEL: v16f32_no_step2:
 ; AVX-RECIP:       # %bb.0:
-; AVX-RECIP-NEXT:    vrcpps %ymm1, %ymm1
 ; AVX-RECIP-NEXT:    vrcpps %ymm0, %ymm0
 ; AVX-RECIP-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
+; AVX-RECIP-NEXT:    vrcpps %ymm1, %ymm1
 ; AVX-RECIP-NEXT:    vmulps {{.*}}(%rip), %ymm1, %ymm1
 ; AVX-RECIP-NEXT:    retq
 ;
 ; FMA-RECIP-LABEL: v16f32_no_step2:
 ; FMA-RECIP:       # %bb.0:
-; FMA-RECIP-NEXT:    vrcpps %ymm1, %ymm1
 ; FMA-RECIP-NEXT:    vrcpps %ymm0, %ymm0
 ; FMA-RECIP-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
+; FMA-RECIP-NEXT:    vrcpps %ymm1, %ymm1
 ; FMA-RECIP-NEXT:    vmulps {{.*}}(%rip), %ymm1, %ymm1
 ; FMA-RECIP-NEXT:    retq
 ;
 ; BDVER2-LABEL: v16f32_no_step2:
 ; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vrcpps %ymm1, %ymm1
 ; BDVER2-NEXT:    vrcpps %ymm0, %ymm0
+; BDVER2-NEXT:    vrcpps %ymm1, %ymm1
 ; BDVER2-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
 ; BDVER2-NEXT:    vmulps {{.*}}(%rip), %ymm1, %ymm1
 ; BDVER2-NEXT:    retq
 ;
 ; BTVER2-LABEL: v16f32_no_step2:
 ; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vrcpps %ymm1, %ymm1
 ; BTVER2-NEXT:    vrcpps %ymm0, %ymm0
+; BTVER2-NEXT:    vrcpps %ymm1, %ymm1
 ; BTVER2-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
 ; BTVER2-NEXT:    vmulps {{.*}}(%rip), %ymm1, %ymm1
 ; BTVER2-NEXT:    retq
 ;
 ; SANDY-LABEL: v16f32_no_step2:
 ; SANDY:       # %bb.0:
-; SANDY-NEXT:    vrcpps %ymm1, %ymm1
 ; SANDY-NEXT:    vrcpps %ymm0, %ymm0
+; SANDY-NEXT:    vrcpps %ymm1, %ymm1
 ; SANDY-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
 ; SANDY-NEXT:    vmulps {{.*}}(%rip), %ymm1, %ymm1
 ; SANDY-NEXT:    retq
 ;
 ; HASWELL-LABEL: v16f32_no_step2:
 ; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vrcpps %ymm1, %ymm1
 ; HASWELL-NEXT:    vrcpps %ymm0, %ymm0
+; HASWELL-NEXT:    vrcpps %ymm1, %ymm1
 ; HASWELL-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
 ; HASWELL-NEXT:    vmulps {{.*}}(%rip), %ymm1, %ymm1
 ; HASWELL-NEXT:    retq
 ;
 ; HASWELL-NO-FMA-LABEL: v16f32_no_step2:
 ; HASWELL-NO-FMA:       # %bb.0:
-; HASWELL-NO-FMA-NEXT:    vrcpps %ymm1, %ymm1
 ; HASWELL-NO-FMA-NEXT:    vrcpps %ymm0, %ymm0
+; HASWELL-NO-FMA-NEXT:    vrcpps %ymm1, %ymm1
 ; HASWELL-NO-FMA-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
 ; HASWELL-NO-FMA-NEXT:    vmulps {{.*}}(%rip), %ymm1, %ymm1
 ; HASWELL-NO-FMA-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/regalloc-advanced-split-cost.ll b/llvm/test/CodeGen/X86/regalloc-advanced-split-cost.ll
index 4c50288db..d9f390838 100644
--- a/llvm/test/CodeGen/X86/regalloc-advanced-split-cost.ll
+++ b/llvm/test/CodeGen/X86/regalloc-advanced-split-cost.ll
@@ -72,7 +72,7 @@ if.end:                                           ; preds = %if.else, %if.then
   ret i32 %add
 }
 
-attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-features"="+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-features"="+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind }
 
 !llvm.module.flags = !{!0, !1}
diff --git a/llvm/test/CodeGen/X86/regparm.ll b/llvm/test/CodeGen/X86/regparm.ll
index 01a734f9f..fe96a562e 100644
--- a/llvm/test/CodeGen/X86/regparm.ll
+++ b/llvm/test/CodeGen/X86/regparm.ll
@@ -38,7 +38,7 @@ entry:
 declare void @llvm.memset.p0i8.i32(i8* nocapture writeonly, i8, i32, i1) #1
 
 
-attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { argmemonly nounwind }
 
 !llvm.module.flags = !{!0}
diff --git a/llvm/test/CodeGen/X86/relptr-rodata.ll b/llvm/test/CodeGen/X86/relptr-rodata.ll
new file mode 100644
index 000000000..b2635abab
--- /dev/null
+++ b/llvm/test/CodeGen/X86/relptr-rodata.ll
@@ -0,0 +1,21 @@
+; RUN: llc -relocation-model=pic -data-sections -o - %s | FileCheck %s
+
+target triple = "x86_64-unknown-linux-gnu"
+
+@hidden = external hidden global i8
+@default = external global i8
+
+; CHECK: .section .rodata.rodata
+; CHECK: rodata:
+; CHECK: .long hidden-rodata
+@rodata = hidden constant i32 trunc (i64 sub (i64 ptrtoint (i8* @hidden to i64), i64 ptrtoint (i32* @rodata to i64)) to i32)
+
+; CHECK: .section .data.rel.ro.relro1
+; CHECK: relro1:
+; CHECK: .long default-relro1
+@relro1 = hidden constant i32 trunc (i64 sub (i64 ptrtoint (i8* @default to i64), i64 ptrtoint (i32* @relro1 to i64)) to i32)
+
+; CHECK: .section .data.rel.ro.relro2
+; CHECK: relro2:
+; CHECK: .long hidden-relro2
+@relro2 = constant i32 trunc (i64 sub (i64 ptrtoint (i8* @hidden to i64), i64 ptrtoint (i32* @relro2 to i64)) to i32)
diff --git a/llvm/test/CodeGen/X86/rem.ll b/llvm/test/CodeGen/X86/rem.ll
index 5f2cc199b..c3f6d61f4 100644
--- a/llvm/test/CodeGen/X86/rem.ll
+++ b/llvm/test/CodeGen/X86/rem.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=i386-unknown-unknown | FileCheck %s
+; RUN: llc < %s -mtriple=i386-unknown-unknown -mattr=+cmov | FileCheck %s
 
 define i32 @test1(i32 %X) {
 ; CHECK-LABEL: test1:
@@ -27,10 +27,9 @@ define i32 @test2(i32 %X) {
 ; CHECK-LABEL: test2:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; CHECK-NEXT:    movl %eax, %ecx
-; CHECK-NEXT:    sarl $31, %ecx
-; CHECK-NEXT:    shrl $24, %ecx
-; CHECK-NEXT:    addl %eax, %ecx
+; CHECK-NEXT:    leal 255(%eax), %ecx
+; CHECK-NEXT:    testl %eax, %eax
+; CHECK-NEXT:    cmovnsl %eax, %ecx
 ; CHECK-NEXT:    andl $-256, %ecx
 ; CHECK-NEXT:    subl %ecx, %eax
 ; CHECK-NEXT:    retl
diff --git a/llvm/test/CodeGen/X86/remarks-section.ll b/llvm/test/CodeGen/X86/remarks-section.ll
index 6d7957a16..dba20d428 100644
--- a/llvm/test/CodeGen/X86/remarks-section.ll
+++ b/llvm/test/CodeGen/X86/remarks-section.ll
@@ -1,79 +1,32 @@
-; RUN: llc < %s -mtriple=x86_64-linux -remarks-section -pass-remarks-output=%/t.yaml | FileCheck -DPATH=%/t.yaml %s
 ; RUN: llc < %s -mtriple=x86_64-darwin -remarks-section -pass-remarks-output=%/t.yaml | FileCheck --check-prefix=CHECK-DARWIN -DPATH=%/t.yaml %s
-; RUN: llc < %s -mtriple=x86_64-darwin -remarks-section -remarks-yaml-string-table -pass-remarks-output=%/t.yaml | FileCheck --check-prefix=CHECK-DARWIN-STRTAB -DPATH=%/t.yaml %s
+; RUN: llc < %s -mtriple=x86_64-darwin --pass-remarks-format=yaml-strtab -remarks-section -pass-remarks-output=%/t.yaml | FileCheck --check-prefix=CHECK-DARWIN-STRTAB -DPATH=%/t.yaml %s
 
-; CHECK-LABEL: func1:
-
-; CHECK: .section .remarks,"e",@progbits
-; The magic number:
-; CHECK-NEXT: .ascii "REMARKS"
-; Null-terminator:
-; CHECK-NEXT: .byte 0
-; The version:
-; CHECK-NEXT: .byte 0x00, 0x00, 0x00, 0x00
-; CHECK-NEXT: .byte 0x00, 0x00, 0x00, 0x00
-; The string table size:
-; CHECK-NEXT: .byte 0x00, 0x00, 0x00, 0x00
-; CHECK-NEXT: .byte 0x00, 0x00, 0x00, 0x00
-; The string table:
-; EMPTY
-; The remark file path:
-; CHECK-NEXT: .ascii "[[PATH]]"
-; Null-terminator:
-; CHECK-NEXT: .byte 0
+; RUN: llc < %s -mtriple=x86_64-darwin -pass-remarks-output=%/t.yaml | FileCheck --check-prefix=CHECK-DARWIN-DEFAULT %s
+; RUN: llc < %s -mtriple=x86_64-darwin --pass-remarks-format=yaml-strtab -pass-remarks-output=%/t.yaml | FileCheck --check-prefix=CHECK-DARWIN-DEFAULT-YAML-STRTAB %s
+; RUN: llc < %s -mtriple=x86_64-darwin --pass-remarks-format=bitstream -pass-remarks-output=%/t.yaml | FileCheck --check-prefix=CHECK-DARWIN-DEFAULT-BITSTREAM %s
+; RUN: llc < %s -mtriple=x86_64-darwin --pass-remarks-format=bitstream -remarks-section=false -pass-remarks-output=%/t.yaml | FileCheck --check-prefix=CHECK-DARWIN-OVERRIDE-BITSTREAM %s
+; RUN: llc < %s -mtriple=x86_64-darwin --pass-remarks-format=yaml -remarks-section=true -pass-remarks-output=%/t.yaml | FileCheck --check-prefix=CHECK-DARWIN-OVERRIDE-YAML %s
 
 ; CHECK-DARWIN: .section __LLVM,__remarks,regular,debug
-; The magic number:
-; CHECK-DARWIN-NEXT: .ascii "REMARKS"
-; Null-terminator:
-; CHECK-DARWIN-NEXT: .byte 0
-; The version:
-; CHECK-DARWIN-NEXT: .byte 0x00, 0x00, 0x00, 0x00
-; CHECK-DARWIN-NEXT: .byte 0x00, 0x00, 0x00, 0x00
-; The string table size:
-; CHECK-DARWIN-NEXT: .byte 0x00, 0x00, 0x00, 0x00
-; CHECK-DARWIN-NEXT: .byte 0x00, 0x00, 0x00, 0x00
-; The string table:
-; EMPTY
-; The remark file path:
-; CHECK-DARWIN-NEXT: .ascii "[[PATH]]"
-; Null-terminator:
-; CHECK-DARWIN-NEXT: .byte 0
+; CHECK-DARWIN-NEXT: .byte
 
 ; CHECK-DARWIN-STRTAB: .section __LLVM,__remarks,regular,debug
-; The magic number:
-; CHECK-DARWIN-STRTAB-NEXT: .ascii "REMARKS"
-; Null-terminator:
-; CHECK-DARWIN-STRTAB-NEXT: .byte 0
-; The version:
-; CHECK-DARWIN-STRTAB-NEXT: .byte 0x00, 0x00, 0x00, 0x00
-; CHECK-DARWIN-STRTAB-NEXT: .byte 0x00, 0x00, 0x00, 0x00
-; The size of the string table:
-; CHECK-DARWIN-STRTAB-NEXT: .byte 0x71, 0x00, 0x00, 0x00
-; CHECK-DARWIN-STRTAB-NEXT: .byte 0x00, 0x00, 0x00, 0x00
-; The string table:
-; CHECK-DARWIN-STRTAB-NEXT: .ascii "prologepilog"
-; CHECK-DARWIN-STRTAB-NEXT: .byte 0
-; CHECK-DARWIN-STRTAB-NEXT: .ascii "StackSize"
-; CHECK-DARWIN-STRTAB-NEXT: .byte 0
-; CHECK-DARWIN-STRTAB-NEXT: .ascii "func1"
-; CHECK-DARWIN-STRTAB-NEXT: .byte 0
-; CHECK-DARWIN-STRTAB-NEXT: .byte 48
-; CHECK-DARWIN-STRTAB-NEXT: .byte 0
-; CHECK-DARWIN-STRTAB-NEXT: .ascii " stack bytes in function"
-; CHECK-DARWIN-STRTAB-NEXT: .byte 0
-; CHECK-DARWIN-STRTAB-NEXT: .ascii "asm-printer"
-; CHECK-DARWIN-STRTAB-NEXT: .byte 0
-; CHECK-DARWIN-STRTAB-NEXT: .ascii "InstructionCount"
-; CHECK-DARWIN-STRTAB-NEXT: .byte 0
-; CHECK-DARWIN-STRTAB-NEXT: .byte 49
-; CHECK-DARWIN-STRTAB-NEXT: .byte 0
-; CHECK-DARWIN-STRTAB-NEXT: .ascii " instructions in function"
-; CHECK-DARWIN-STRTAB-NEXT: .byte 0
-; The remark file path:
-; CHECK-DARWIN-STRTAB-NEXT: .ascii "[[PATH]]"
-; Null-terminator:
-; CHECK-DARWIN-STRTAB-NEXT: .byte 0
+; CHECK-DARWIN-STRTAB-NEXT: .byte
+
+; By default, the format is YAML which does not need a section.
+; CHECK-DARWIN-DEFAULT-NOT: .section __LLVM,__remarks
+
+; yaml-strtab needs a section.
+; CHECK-DARWIN-DEFAULT-YAML-STRTAB: .section __LLVM,__remarks
+
+; bitstream needs a section.
+; CHECK-DARWIN-DEFAULT-BITSTREAM: .section __LLVM,__remarks
+
+; -remarks-section should force disable the section.
+; CHECK-DARWIN-OVERRIDE-BITSTREAM-NOT: .section __LLVM,__remarks
+
+; -remarks-section should also force enable the section.
+; CHECK-DARWIN-OVERRIDE-YAML: .section __LLVM,__remarks
 define void @func1() {
   ret void
 }
diff --git a/llvm/test/CodeGen/X86/remat-constant.ll b/llvm/test/CodeGen/X86/remat-constant.ll
index 5a6826f38..332d635f8 100644
--- a/llvm/test/CodeGen/X86/remat-constant.ll
+++ b/llvm/test/CodeGen/X86/remat-constant.ll
@@ -1,12 +1,23 @@
-; RUN: llc < %s -mtriple=x86_64-linux -relocation-model=static | grep xmm | count 2
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-linux -relocation-model=static | FileCheck %s
 
 declare void @bar() nounwind
 
 @a = external constant float
 
-declare void @qux(float %f) nounwind 
+declare void @qux(float %f) nounwind
 
 define void @foo() nounwind  {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    callq bar
+; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; CHECK-NEXT:    callq qux
+; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; CHECK-NEXT:    callq qux
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    retq
   %f = load float, float* @a
   call void @bar()
   call void @qux(float %f)
diff --git a/llvm/test/CodeGen/X86/ret-addr.ll b/llvm/test/CodeGen/X86/ret-addr.ll
index d90f639ba..04dc2b7fc 100644
--- a/llvm/test/CodeGen/X86/ret-addr.ll
+++ b/llvm/test/CodeGen/X86/ret-addr.ll
@@ -1,7 +1,27 @@
-; RUN: llc < %s -frame-pointer=all -mtriple=i686-- | not grep xor
-; RUN: llc < %s -frame-pointer=all -mtriple=x86_64-- | not grep xor
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -frame-pointer=all -mtriple=i686-- | FileCheck %s --check-prefix=i686
+; RUN: llc < %s -frame-pointer=all -mtriple=x86_64-- | FileCheck %s --check-prefix=x86_64
 
 define i8* @h() nounwind readnone optsize {
+; i686-LABEL: h:
+; i686:       # %bb.0: # %entry
+; i686-NEXT:    pushl %ebp
+; i686-NEXT:    movl %esp, %ebp
+; i686-NEXT:    movl (%ebp), %eax
+; i686-NEXT:    movl (%eax), %eax
+; i686-NEXT:    movl 4(%eax), %eax
+; i686-NEXT:    popl %ebp
+; i686-NEXT:    retl
+;
+; x86_64-LABEL: h:
+; x86_64:       # %bb.0: # %entry
+; x86_64-NEXT:    pushq %rbp
+; x86_64-NEXT:    movq %rsp, %rbp
+; x86_64-NEXT:    movq (%rbp), %rax
+; x86_64-NEXT:    movq (%rax), %rax
+; x86_64-NEXT:    movq 8(%rax), %rax
+; x86_64-NEXT:    popq %rbp
+; x86_64-NEXT:    retq
 entry:
 	%0 = tail call i8* @llvm.returnaddress(i32 2)		; <i8*> [#uses=1]
 	ret i8* %0
@@ -10,12 +30,44 @@ entry:
 declare i8* @llvm.returnaddress(i32) nounwind readnone
 
 define i8* @g() nounwind readnone optsize {
+; i686-LABEL: g:
+; i686:       # %bb.0: # %entry
+; i686-NEXT:    pushl %ebp
+; i686-NEXT:    movl %esp, %ebp
+; i686-NEXT:    movl (%ebp), %eax
+; i686-NEXT:    movl 4(%eax), %eax
+; i686-NEXT:    popl %ebp
+; i686-NEXT:    retl
+;
+; x86_64-LABEL: g:
+; x86_64:       # %bb.0: # %entry
+; x86_64-NEXT:    pushq %rbp
+; x86_64-NEXT:    movq %rsp, %rbp
+; x86_64-NEXT:    movq (%rbp), %rax
+; x86_64-NEXT:    movq 8(%rax), %rax
+; x86_64-NEXT:    popq %rbp
+; x86_64-NEXT:    retq
 entry:
 	%0 = tail call i8* @llvm.returnaddress(i32 1)		; <i8*> [#uses=1]
 	ret i8* %0
 }
 
 define i8* @f() nounwind readnone optsize {
+; i686-LABEL: f:
+; i686:       # %bb.0: # %entry
+; i686-NEXT:    pushl %ebp
+; i686-NEXT:    movl %esp, %ebp
+; i686-NEXT:    movl 4(%ebp), %eax
+; i686-NEXT:    popl %ebp
+; i686-NEXT:    retl
+;
+; x86_64-LABEL: f:
+; x86_64:       # %bb.0: # %entry
+; x86_64-NEXT:    pushq %rbp
+; x86_64-NEXT:    movq %rsp, %rbp
+; x86_64-NEXT:    movq 8(%rbp), %rax
+; x86_64-NEXT:    popq %rbp
+; x86_64-NEXT:    retq
 entry:
 	%0 = tail call i8* @llvm.returnaddress(i32 0)		; <i8*> [#uses=1]
 	ret i8* %0
diff --git a/llvm/test/CodeGen/X86/ret-i64-0.ll b/llvm/test/CodeGen/X86/ret-i64-0.ll
index be82129de..c281de0a5 100644
--- a/llvm/test/CodeGen/X86/ret-i64-0.ll
+++ b/llvm/test/CodeGen/X86/ret-i64-0.ll
@@ -1,5 +1,11 @@
-; RUN: llc < %s -mtriple=i686-- | grep xor | count 2
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- | FileCheck %s
 
 define i64 @foo() nounwind {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    retl
   ret i64 0
 }
diff --git a/llvm/test/CodeGen/X86/ret-mmx.ll b/llvm/test/CodeGen/X86/ret-mmx.ll
index 6a9e59193..3797b6db4 100644
--- a/llvm/test/CodeGen/X86/ret-mmx.ll
+++ b/llvm/test/CodeGen/X86/ret-mmx.ll
@@ -33,7 +33,7 @@ define <2 x i32> @t3() nounwind {
 ; CHECK-LABEL: t3:
 ; CHECK:       ## %bb.0:
 ; CHECK-NEXT:    movl $1, %eax
-; CHECK-NEXT:    movq %rax, %xmm0
+; CHECK-NEXT:    movd %eax, %xmm0
 ; CHECK-NEXT:    retq
   ret <2 x i32> <i32 1, i32 0>
 }
diff --git a/llvm/test/CodeGen/X86/reverse_branches.ll b/llvm/test/CodeGen/X86/reverse_branches.ll
index fabde1679..170fc6a76 100644
--- a/llvm/test/CodeGen/X86/reverse_branches.ll
+++ b/llvm/test/CodeGen/X86/reverse_branches.ll
@@ -36,24 +36,24 @@ define i32 @test_branches_order() uwtable ssp {
 ; CHECK-NEXT:    xorl %r12d, %r12d
 ; CHECK-NEXT:    leaq -{{[0-9]+}}(%rsp), %r14
 ; CHECK-NEXT:    movq %rsp, %r15
-; CHECK-NEXT:    cmpl $999, %r12d ## imm = 0x3E7
-; CHECK-NEXT:    jle LBB0_2
-; CHECK-NEXT:    jmp LBB0_7
+; CHECK-NEXT:    jmp LBB0_1
 ; CHECK-NEXT:    .p2align 4, 0x90
 ; CHECK-NEXT:  LBB0_6: ## %for.inc9
-; CHECK-NEXT:    ## in Loop: Header=BB0_2 Depth=1
+; CHECK-NEXT:    ## in Loop: Header=BB0_1 Depth=1
 ; CHECK-NEXT:    incl %r12d
-; CHECK-NEXT:    cmpl $999, %r12d ## imm = 0x3E7
-; CHECK-NEXT:    jg LBB0_7
-; CHECK-NEXT:  LBB0_2: ## %for.cond1.preheader
+; CHECK-NEXT:  LBB0_1: ## %for.cond
 ; CHECK-NEXT:    ## =>This Loop Header: Depth=1
 ; CHECK-NEXT:    ## Child Loop BB0_3 Depth 2
+; CHECK-NEXT:    cmpl $999, %r12d ## imm = 0x3E7
+; CHECK-NEXT:    jg LBB0_7
+; CHECK-NEXT:  ## %bb.2: ## %for.cond1.preheader
+; CHECK-NEXT:    ## in Loop: Header=BB0_1 Depth=1
 ; CHECK-NEXT:    movl $-1, %r13d
 ; CHECK-NEXT:    movq %r15, %rbx
 ; CHECK-NEXT:    movq %r14, %rbp
 ; CHECK-NEXT:    .p2align 4, 0x90
 ; CHECK-NEXT:  LBB0_3: ## %for.cond1
-; CHECK-NEXT:    ## Parent Loop BB0_2 Depth=1
+; CHECK-NEXT:    ## Parent Loop BB0_1 Depth=1
 ; CHECK-NEXT:    ## => This Inner Loop Header: Depth=2
 ; CHECK-NEXT:    incl %r13d
 ; CHECK-NEXT:    cmpl $999, %r13d ## imm = 0x3E7
@@ -74,47 +74,45 @@ define i32 @test_branches_order() uwtable ssp {
 ; CHECK-NEXT:    callq _puts
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    movq %rsp, %rcx
-; CHECK-NEXT:    cmpl $999, %eax ## imm = 0x3E7
-; CHECK-NEXT:    jle LBB0_9
-; CHECK-NEXT:    jmp LBB0_16
+; CHECK-NEXT:    jmp LBB0_8
 ; CHECK-NEXT:    .p2align 4, 0x90
 ; CHECK-NEXT:  LBB0_15: ## %for.inc38
-; CHECK-NEXT:    ## in Loop: Header=BB0_9 Depth=1
+; CHECK-NEXT:    ## in Loop: Header=BB0_8 Depth=1
 ; CHECK-NEXT:    incl %eax
-; CHECK-NEXT:    cmpl $999, %eax ## imm = 0x3E7
-; CHECK-NEXT:    jg LBB0_16
-; CHECK-NEXT:  LBB0_9: ## %for.cond18.preheader
+; CHECK-NEXT:  LBB0_8: ## %for.cond14
 ; CHECK-NEXT:    ## =>This Loop Header: Depth=1
-; CHECK-NEXT:    ## Child Loop BB0_11 Depth 2
+; CHECK-NEXT:    ## Child Loop BB0_10 Depth 2
 ; CHECK-NEXT:    ## Child Loop BB0_12 Depth 3
+; CHECK-NEXT:    cmpl $999, %eax ## imm = 0x3E7
+; CHECK-NEXT:    jg LBB0_16
+; CHECK-NEXT:  ## %bb.9: ## %for.cond18.preheader
+; CHECK-NEXT:    ## in Loop: Header=BB0_8 Depth=1
 ; CHECK-NEXT:    movq %rcx, %rdx
 ; CHECK-NEXT:    xorl %esi, %esi
 ; CHECK-NEXT:    xorl %edi, %edi
-; CHECK-NEXT:    cmpl $999, %edi ## imm = 0x3E7
-; CHECK-NEXT:    jle LBB0_11
-; CHECK-NEXT:    jmp LBB0_15
+; CHECK-NEXT:    jmp LBB0_10
 ; CHECK-NEXT:    .p2align 4, 0x90
 ; CHECK-NEXT:  LBB0_14: ## %exit
-; CHECK-NEXT:    ## in Loop: Header=BB0_11 Depth=2
+; CHECK-NEXT:    ## in Loop: Header=BB0_10 Depth=2
 ; CHECK-NEXT:    addq %rsi, %rbp
 ; CHECK-NEXT:    incq %rdi
 ; CHECK-NEXT:    decq %rsi
 ; CHECK-NEXT:    addq $1001, %rdx ## imm = 0x3E9
 ; CHECK-NEXT:    cmpq $-1000, %rbp ## imm = 0xFC18
 ; CHECK-NEXT:    jne LBB0_5
-; CHECK-NEXT:  ## %bb.10: ## %for.cond18
-; CHECK-NEXT:    ## in Loop: Header=BB0_11 Depth=2
-; CHECK-NEXT:    cmpl $999, %edi ## imm = 0x3E7
-; CHECK-NEXT:    jg LBB0_15
-; CHECK-NEXT:  LBB0_11: ## %for.body20
-; CHECK-NEXT:    ## Parent Loop BB0_9 Depth=1
+; CHECK-NEXT:  LBB0_10: ## %for.cond18
+; CHECK-NEXT:    ## Parent Loop BB0_8 Depth=1
 ; CHECK-NEXT:    ## => This Loop Header: Depth=2
 ; CHECK-NEXT:    ## Child Loop BB0_12 Depth 3
+; CHECK-NEXT:    cmpl $999, %edi ## imm = 0x3E7
+; CHECK-NEXT:    jg LBB0_15
+; CHECK-NEXT:  ## %bb.11: ## %for.body20
+; CHECK-NEXT:    ## in Loop: Header=BB0_10 Depth=2
 ; CHECK-NEXT:    movq $-1000, %rbp ## imm = 0xFC18
 ; CHECK-NEXT:    .p2align 4, 0x90
 ; CHECK-NEXT:  LBB0_12: ## %do.body.i
-; CHECK-NEXT:    ## Parent Loop BB0_9 Depth=1
-; CHECK-NEXT:    ## Parent Loop BB0_11 Depth=2
+; CHECK-NEXT:    ## Parent Loop BB0_8 Depth=1
+; CHECK-NEXT:    ## Parent Loop BB0_10 Depth=2
 ; CHECK-NEXT:    ## => This Inner Loop Header: Depth=3
 ; CHECK-NEXT:    cmpb $120, 1000(%rdx,%rbp)
 ; CHECK-NEXT:    je LBB0_14
diff --git a/llvm/test/CodeGen/X86/rot16.ll b/llvm/test/CodeGen/X86/rot16.ll
index b0221c485..5a1a8da3c 100644
--- a/llvm/test/CodeGen/X86/rot16.ll
+++ b/llvm/test/CodeGen/X86/rot16.ll
@@ -177,3 +177,56 @@ define i16 @xbu(i16 %x, i16 %y, i16 %z) nounwind {
 	%t2 = or i16 %t0, %t1
 	ret i16 %t2
 }
+
+define i32 @rot16_demandedbits(i32 %x, i32 %y) nounwind {
+; X32-LABEL: rot16_demandedbits:
+; X32:       # %bb.0:
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    shrl $11, %ecx
+; X32-NEXT:    shll $5, %eax
+; X32-NEXT:    orl %ecx, %eax
+; X32-NEXT:    movzwl %ax, %eax
+; X32-NEXT:    retl
+;
+; X64-LABEL: rot16_demandedbits:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    shrl $11, %eax
+; X64-NEXT:    shll $5, %edi
+; X64-NEXT:    orl %eax, %edi
+; X64-NEXT:    movzwl %di, %eax
+; X64-NEXT:    retq
+	%t0 = lshr i32 %x, 11
+	%t1 = shl i32 %x, 5
+	%t2 = or i32 %t0, %t1
+	%t3 = and i32 %t2, 65535
+	ret i32 %t3
+}
+
+define i16 @rot16_trunc(i32 %x, i32 %y) nounwind {
+; X32-LABEL: rot16_trunc:
+; X32:       # %bb.0:
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    shrl $11, %ecx
+; X32-NEXT:    shll $5, %eax
+; X32-NEXT:    orl %ecx, %eax
+; X32-NEXT:    # kill: def $ax killed $ax killed $eax
+; X32-NEXT:    retl
+;
+; X64-LABEL: rot16_trunc:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    movl %edi, %ecx
+; X64-NEXT:    shrl $11, %ecx
+; X64-NEXT:    shll $5, %eax
+; X64-NEXT:    orl %ecx, %eax
+; X64-NEXT:    # kill: def $ax killed $ax killed $eax
+; X64-NEXT:    retq
+	%t0 = lshr i32 %x, 11
+	%t1 = shl i32 %x, 5
+	%t2 = or i32 %t0, %t1
+	%t3 = trunc i32 %t2 to i16
+	ret i16 %t3
+}
diff --git a/llvm/test/CodeGen/X86/rotate-extract-vector.ll b/llvm/test/CodeGen/X86/rotate-extract-vector.ll
index 4959de711..7b7feb337 100644
--- a/llvm/test/CodeGen/X86/rotate-extract-vector.ll
+++ b/llvm/test/CodeGen/X86/rotate-extract-vector.ll
@@ -280,3 +280,45 @@ define <2 x i64> @no_extract_udiv(<2 x i64> %i) nounwind {
   %out = or <2 x i64> %lhs_shift, %rhs_div
   ret <2 x i64> %out
 }
+
+; DAGCombiner transforms shl X, 1 into add X, X.
+define <4 x i32> @extract_add_1(<4 x i32> %i) nounwind {
+; CHECK-LABEL: extract_add_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; CHECK-NEXT:    vprold $1, %zmm0, %zmm0
+; CHECK-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    ret{{[l|q]}}
+  %ii = add <4 x i32> %i, %i
+  %rhs = lshr <4 x i32> %i, <i32 31, i32 31, i32 31, i32 31>
+  %out = or <4 x i32> %ii, %rhs
+  ret <4 x i32> %out
+}
+
+define <4 x i32> @extract_add_1_comut(<4 x i32> %i) nounwind {
+; CHECK-LABEL: extract_add_1_comut:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; CHECK-NEXT:    vprold $1, %zmm0, %zmm0
+; CHECK-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    ret{{[l|q]}}
+  %ii = add <4 x i32> %i, %i
+  %lhs = lshr <4 x i32> %i, <i32 31, i32 31, i32 31, i32 31>
+  %out = or <4 x i32> %lhs, %ii
+  ret <4 x i32> %out
+}
+
+define <4 x i32> @no_extract_add_1(<4 x i32> %i) nounwind {
+; CHECK-LABEL: no_extract_add_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpaddd %xmm0, %xmm0, %xmm1
+; CHECK-NEXT:    vpsrld $27, %xmm0, %xmm0
+; CHECK-NEXT:    vpor %xmm0, %xmm1, %xmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %ii = add <4 x i32> %i, %i
+  %rhs = lshr <4 x i32> %i, <i32 27, i32 27, i32 27, i32 27>
+  %out = or <4 x i32> %ii, %rhs
+  ret <4 x i32> %out
+}
diff --git a/llvm/test/CodeGen/X86/rotate-extract.ll b/llvm/test/CodeGen/X86/rotate-extract.ll
index e5228d271..a70577359 100644
--- a/llvm/test/CodeGen/X86/rotate-extract.ll
+++ b/llvm/test/CodeGen/X86/rotate-extract.ll
@@ -265,3 +265,62 @@ define i8 @no_extract_udiv(i8 %i) nounwind {
   %out = or i8 %lhs_shift, %rhs_div
   ret i8 %out
 }
+
+; DAGCombiner transforms shl X, 1 into add X, X.
+define i32 @extract_add_1(i32 %i) nounwind {
+; X86-LABEL: extract_add_1:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    roll %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: extract_add_1:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    roll %eax
+; X64-NEXT:    retq
+  %ii = add i32 %i, %i
+  %rhs = lshr i32 %i, 31
+  %out = or i32 %ii, %rhs
+  ret i32 %out
+}
+
+define i32 @extract_add_1_comut(i32 %i) nounwind {
+; X86-LABEL: extract_add_1_comut:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    roll %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: extract_add_1_comut:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    roll %eax
+; X64-NEXT:    retq
+  %ii = add i32 %i, %i
+  %lhs = lshr i32 %i, 31
+  %out = or i32 %lhs, %ii
+  ret i32 %out
+}
+
+define i32 @no_extract_add_1(i32 %i) nounwind {
+; X86-LABEL: no_extract_add_1:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    leal (%ecx,%ecx), %eax
+; X86-NEXT:    shrl $27, %ecx
+; X86-NEXT:    orl %ecx, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: no_extract_add_1:
+; X64:       # %bb.0:
+; X64-NEXT:    # kill: def $edi killed $edi def $rdi
+; X64-NEXT:    leal (%rdi,%rdi), %eax
+; X64-NEXT:    shrl $27, %edi
+; X64-NEXT:    orl %edi, %eax
+; X64-NEXT:    retq
+  %ii = add i32 %i, %i
+  %rhs = lshr i32 %i, 27
+  %out = or i32 %ii, %rhs
+  ret i32 %out
+}
diff --git a/llvm/test/CodeGen/X86/rounding-ops.ll b/llvm/test/CodeGen/X86/rounding-ops.ll
index ded613b7f..00a5f2d0b 100644
--- a/llvm/test/CodeGen/X86/rounding-ops.ll
+++ b/llvm/test/CodeGen/X86/rounding-ops.ll
@@ -252,3 +252,60 @@ define double @test12(double* %xptr) nounwind optsize {
   %call = tail call double @trunc(double %x) nounwind readnone
   ret double %call
 }
+
+define float @test11_pgso(float* %xptr) nounwind !prof !14 {
+; CHECK-SSE-LABEL: test11_pgso:
+; CHECK-SSE:       ## %bb.0:
+; CHECK-SSE-NEXT:    roundss $11, (%rdi), %xmm0
+; CHECK-SSE-NEXT:    retq
+;
+; CHECK-AVX-LABEL: test11_pgso:
+; CHECK-AVX:       ## %bb.0:
+; CHECK-AVX-NEXT:    vroundss $11, (%rdi), %xmm0, %xmm0
+; CHECK-AVX-NEXT:    retq
+;
+; CHECK-AVX512-LABEL: test11_pgso:
+; CHECK-AVX512:       ## %bb.0:
+; CHECK-AVX512-NEXT:    vroundss $11, (%rdi), %xmm0, %xmm0
+; CHECK-AVX512-NEXT:    retq
+  %x = load float, float* %xptr
+  %call = tail call float @truncf(float %x) nounwind readnone
+  ret float %call
+}
+
+define double @test12_pgso(double* %xptr) nounwind !prof !14 {
+; CHECK-SSE-LABEL: test12_pgso:
+; CHECK-SSE:       ## %bb.0:
+; CHECK-SSE-NEXT:    roundsd $11, (%rdi), %xmm0
+; CHECK-SSE-NEXT:    retq
+;
+; CHECK-AVX-LABEL: test12_pgso:
+; CHECK-AVX:       ## %bb.0:
+; CHECK-AVX-NEXT:    vroundsd $11, (%rdi), %xmm0, %xmm0
+; CHECK-AVX-NEXT:    retq
+;
+; CHECK-AVX512-LABEL: test12_pgso:
+; CHECK-AVX512:       ## %bb.0:
+; CHECK-AVX512-NEXT:    vroundsd $11, (%rdi), %xmm0, %xmm0
+; CHECK-AVX512-NEXT:    retq
+  %x = load double, double* %xptr
+  %call = tail call double @trunc(double %x) nounwind readnone
+  ret double %call
+}
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 1, !"ProfileSummary", !1}
+!1 = !{!2, !3, !4, !5, !6, !7, !8, !9}
+!2 = !{!"ProfileFormat", !"InstrProf"}
+!3 = !{!"TotalCount", i64 10000}
+!4 = !{!"MaxCount", i64 10}
+!5 = !{!"MaxInternalCount", i64 1}
+!6 = !{!"MaxFunctionCount", i64 1000}
+!7 = !{!"NumCounts", i64 3}
+!8 = !{!"NumFunctions", i64 3}
+!9 = !{!"DetailedSummary", !10}
+!10 = !{!11, !12, !13}
+!11 = !{i32 10000, i64 100, i32 1}
+!12 = !{i32 999000, i64 100, i32 1}
+!13 = !{i32 999999, i64 1, i32 2}
+!14 = !{!"function_entry_count", i64 0}
diff --git a/llvm/test/CodeGen/X86/sad.ll b/llvm/test/CodeGen/X86/sad.ll
index 0bbae5c85..72b24929c 100644
--- a/llvm/test/CodeGen/X86/sad.ll
+++ b/llvm/test/CodeGen/X86/sad.ll
@@ -786,49 +786,49 @@ define i32 @sad_avx64i8() nounwind {
 ; AVX1-NEXT:    vextractf128 $1, %ymm11, %xmm7
 ; AVX1-NEXT:    vpaddd %xmm7, %xmm2, %xmm2
 ; AVX1-NEXT:    vpabsd %xmm1, %xmm1
-; AVX1-NEXT:    vpaddd %xmm11, %xmm1, %xmm1
+; AVX1-NEXT:    vpaddd %xmm1, %xmm11, %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm7
 ; AVX1-NEXT:    vpabsd %xmm6, %xmm1
 ; AVX1-NEXT:    vextractf128 $1, %ymm15, %xmm2
 ; AVX1-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
 ; AVX1-NEXT:    vpabsd %xmm5, %xmm2
-; AVX1-NEXT:    vpaddd %xmm15, %xmm2, %xmm2
+; AVX1-NEXT:    vpaddd %xmm2, %xmm15, %xmm2
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm15
 ; AVX1-NEXT:    vpabsd %xmm0, %xmm1
 ; AVX1-NEXT:    vextractf128 $1, %ymm14, %xmm2
 ; AVX1-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
 ; AVX1-NEXT:    vpabsd %xmm3, %xmm2
-; AVX1-NEXT:    vpaddd %xmm14, %xmm2, %xmm2
+; AVX1-NEXT:    vpaddd %xmm2, %xmm14, %xmm2
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm14
 ; AVX1-NEXT:    vpabsd %xmm4, %xmm1
 ; AVX1-NEXT:    vextractf128 $1, %ymm13, %xmm2
 ; AVX1-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
 ; AVX1-NEXT:    vpabsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
-; AVX1-NEXT:    vpaddd %xmm13, %xmm0, %xmm0
+; AVX1-NEXT:    vpaddd %xmm0, %xmm13, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm13
 ; AVX1-NEXT:    vpabsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
 ; AVX1-NEXT:    vextractf128 $1, %ymm8, %xmm1
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpabsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
-; AVX1-NEXT:    vpaddd %xmm8, %xmm1, %xmm1
+; AVX1-NEXT:    vpaddd %xmm1, %xmm8, %xmm1
 ; AVX1-NEXT:    vpabsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm8
 ; AVX1-NEXT:    vextractf128 $1, %ymm9, %xmm0
 ; AVX1-NEXT:    vpaddd %xmm0, %xmm2, %xmm0
 ; AVX1-NEXT:    vpabsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
-; AVX1-NEXT:    vpaddd %xmm9, %xmm1, %xmm1
+; AVX1-NEXT:    vpaddd %xmm1, %xmm9, %xmm1
 ; AVX1-NEXT:    vpabsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm9
 ; AVX1-NEXT:    vextractf128 $1, %ymm10, %xmm0
 ; AVX1-NEXT:    vpaddd %xmm0, %xmm2, %xmm0
 ; AVX1-NEXT:    vpabsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
-; AVX1-NEXT:    vpaddd %xmm10, %xmm1, %xmm1
+; AVX1-NEXT:    vpaddd %xmm1, %xmm10, %xmm1
 ; AVX1-NEXT:    vpabsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm10
 ; AVX1-NEXT:    vextractf128 $1, %ymm12, %xmm0
 ; AVX1-NEXT:    vpaddd %xmm0, %xmm2, %xmm0
 ; AVX1-NEXT:    vpabsd (%rsp), %xmm1 # 16-byte Folded Reload
-; AVX1-NEXT:    vpaddd %xmm12, %xmm1, %xmm1
+; AVX1-NEXT:    vpaddd %xmm1, %xmm12, %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm12
 ; AVX1-NEXT:    addq $4, %rax
 ; AVX1-NEXT:    jne .LBB2_1
@@ -849,7 +849,7 @@ define i32 @sad_avx64i8() nounwind {
 ; AVX1-NEXT:    vpaddd %xmm0, %xmm2, %xmm0
 ; AVX1-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
 ; AVX1-NEXT:    vpaddd %xmm12, %xmm13, %xmm1
-; AVX1-NEXT:    vpaddd %xmm10, %xmm7, %xmm2
+; AVX1-NEXT:    vpaddd %xmm7, %xmm10, %xmm2
 ; AVX1-NEXT:    vpaddd %xmm2, %xmm8, %xmm2
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm9, %xmm1
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm15, %xmm1
@@ -1074,12 +1074,12 @@ define i32 @sad_2i8() nounwind {
 ; SSE2-NEXT:    pand %xmm1, %xmm3
 ; SSE2-NEXT:    pand %xmm1, %xmm2
 ; SSE2-NEXT:    psadbw %xmm3, %xmm2
-; SSE2-NEXT:    paddq %xmm2, %xmm0
+; SSE2-NEXT:    paddd %xmm2, %xmm0
 ; SSE2-NEXT:    addq $4, %rax
 ; SSE2-NEXT:    jne .LBB3_1
 ; SSE2-NEXT:  # %bb.2: # %middle.block
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; SSE2-NEXT:    paddq %xmm0, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE2-NEXT:    paddd %xmm0, %xmm1
 ; SSE2-NEXT:    movd %xmm1, %eax
 ; SSE2-NEXT:    retq
 ;
@@ -1096,12 +1096,12 @@ define i32 @sad_2i8() nounwind {
 ; AVX-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0],xmm0[1,2,3,4,5,6,7]
 ; AVX-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0],xmm0[1,2,3,4,5,6,7]
 ; AVX-NEXT:    vpsadbw %xmm3, %xmm2, %xmm2
-; AVX-NEXT:    vpaddq %xmm1, %xmm2, %xmm1
+; AVX-NEXT:    vpaddd %xmm1, %xmm2, %xmm1
 ; AVX-NEXT:    addq $4, %rax
 ; AVX-NEXT:    jne .LBB3_1
 ; AVX-NEXT:  # %bb.2: # %middle.block
-; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
-; AVX-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; AVX-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    vmovd %xmm0, %eax
 ; AVX-NEXT:    retq
 entry:
@@ -1403,18 +1403,18 @@ define i32 @sad_unroll_nonzero_initial(<16 x i8>* %arg, <16 x i8>* %arg1, <16 x
 ; SSE2-NEXT:    movdqu (%rdi), %xmm0
 ; SSE2-NEXT:    movdqu (%rsi), %xmm1
 ; SSE2-NEXT:    psadbw %xmm0, %xmm1
-; SSE2-NEXT:    movdqu (%rdx), %xmm0
-; SSE2-NEXT:    movdqu (%rcx), %xmm2
-; SSE2-NEXT:    psadbw %xmm0, %xmm2
 ; SSE2-NEXT:    movl $1, %eax
 ; SSE2-NEXT:    movd %eax, %xmm0
-; SSE2-NEXT:    paddd %xmm2, %xmm0
-; SSE2-NEXT:    paddd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    movdqu (%rdx), %xmm2
+; SSE2-NEXT:    movdqu (%rcx), %xmm3
+; SSE2-NEXT:    psadbw %xmm2, %xmm3
+; SSE2-NEXT:    paddd %xmm0, %xmm3
+; SSE2-NEXT:    paddd %xmm1, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[2,3,0,1]
+; SSE2-NEXT:    paddd %xmm3, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
-; SSE2-NEXT:    paddd %xmm1, %xmm0
-; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    movd %xmm1, %eax
 ; SSE2-NEXT:    retq
 ;
 ; AVX1-LABEL: sad_unroll_nonzero_initial:
@@ -1442,7 +1442,7 @@ define i32 @sad_unroll_nonzero_initial(<16 x i8>* %arg, <16 x i8>* %arg1, <16 x
 ; AVX2-NEXT:    vmovd %eax, %xmm1
 ; AVX2-NEXT:    vmovdqu (%rdx), %xmm2
 ; AVX2-NEXT:    vpsadbw (%rcx), %xmm2, %xmm2
-; AVX2-NEXT:    vpaddd %xmm1, %xmm2, %xmm1
+; AVX2-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -1455,10 +1455,10 @@ define i32 @sad_unroll_nonzero_initial(<16 x i8>* %arg, <16 x i8>* %arg1, <16 x
 ; AVX512:       # %bb.0: # %bb
 ; AVX512-NEXT:    vmovdqu (%rdi), %xmm0
 ; AVX512-NEXT:    vpsadbw (%rsi), %xmm0, %xmm0
-; AVX512-NEXT:    vmovdqu (%rdx), %xmm1
-; AVX512-NEXT:    vpsadbw (%rcx), %xmm1, %xmm1
 ; AVX512-NEXT:    movl $1, %eax
-; AVX512-NEXT:    vmovd %eax, %xmm2
+; AVX512-NEXT:    vmovd %eax, %xmm1
+; AVX512-NEXT:    vmovdqu (%rdx), %xmm2
+; AVX512-NEXT:    vpsadbw (%rcx), %xmm2, %xmm2
 ; AVX512-NEXT:    vpaddd %zmm2, %zmm1, %zmm1
 ; AVX512-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
@@ -1526,9 +1526,9 @@ define i32 @sad_double_reduction(<16 x i8>* %arg, <16 x i8>* %arg1, <16 x i8>* %
 ; AVX:       # %bb.0: # %bb
 ; AVX-NEXT:    vmovdqu (%rdi), %xmm0
 ; AVX-NEXT:    vmovdqu (%rdx), %xmm1
-; AVX-NEXT:    vpsadbw (%rsi), %xmm0, %xmm0
 ; AVX-NEXT:    vpsadbw (%rcx), %xmm1, %xmm1
-; AVX-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    vpsadbw (%rsi), %xmm0, %xmm0
+; AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
diff --git a/llvm/test/CodeGen/X86/sadd_sat.ll b/llvm/test/CodeGen/X86/sadd_sat.ll
index e395bb4e0..e0bf68a6a 100644
--- a/llvm/test/CodeGen/X86/sadd_sat.ll
+++ b/llvm/test/CodeGen/X86/sadd_sat.ll
@@ -2,10 +2,12 @@
 ; RUN: llc < %s -mtriple=i686 -mattr=cmov | FileCheck %s --check-prefixes=CHECK,X86
 ; RUN: llc < %s -mtriple=x86_64-linux | FileCheck %s --check-prefixes=CHECK,X64
 
-declare  i4  @llvm.sadd.sat.i4   (i4,  i4)
-declare  i32 @llvm.sadd.sat.i32  (i32, i32)
-declare  i64 @llvm.sadd.sat.i64  (i64, i64)
-declare  <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32>, <4 x i32>)
+declare i4 @llvm.sadd.sat.i4(i4, i4)
+declare i8 @llvm.sadd.sat.i8(i8, i8)
+declare i16 @llvm.sadd.sat.i16(i16, i16)
+declare i32 @llvm.sadd.sat.i32(i32, i32)
+declare i64 @llvm.sadd.sat.i64(i64, i64)
+declare <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32>, <4 x i32>)
 
 define i32 @func(i32 %x, i32 %y) nounwind {
 ; X86-LABEL: func:
@@ -89,13 +91,43 @@ define i64 @func2(i64 %x, i64 %y) nounwind {
   ret i64 %tmp;
 }
 
-define i4 @func3(i4 %x, i4 %y) nounwind {
-; X86-LABEL: func3:
+define signext i16 @func16(i16 signext %x, i16 signext %y) nounwind {
+; X86-LABEL: func16:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    xorl %ecx, %ecx
+; X86-NEXT:    movl %eax, %esi
+; X86-NEXT:    addw %dx, %si
+; X86-NEXT:    setns %cl
+; X86-NEXT:    addl $32767, %ecx # imm = 0x7FFF
+; X86-NEXT:    addw %dx, %ax
+; X86-NEXT:    cmovol %ecx, %eax
+; X86-NEXT:    # kill: def $ax killed $ax killed $eax
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
+;
+; X64-LABEL: func16:
+; X64:       # %bb.0:
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    movl %edi, %ecx
+; X64-NEXT:    addw %si, %cx
+; X64-NEXT:    setns %al
+; X64-NEXT:    addl $32767, %eax # imm = 0x7FFF
+; X64-NEXT:    addw %si, %di
+; X64-NEXT:    cmovnol %edi, %eax
+; X64-NEXT:    # kill: def $ax killed $ax killed $eax
+; X64-NEXT:    retq
+  %tmp = call i16 @llvm.sadd.sat.i16(i16 %x, i16 %y)
+  ret i16 %tmp
+}
+
+define signext i8 @func8(i8 signext %x, i8 signext %y) nounwind {
+; X86-LABEL: func8:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
 ; X86-NEXT:    movb {{[0-9]+}}(%esp), %dl
-; X86-NEXT:    shlb $4, %dl
-; X86-NEXT:    shlb $4, %al
 ; X86-NEXT:    xorl %ecx, %ecx
 ; X86-NEXT:    movb %al, %ah
 ; X86-NEXT:    addb %dl, %ah
@@ -104,14 +136,11 @@ define i4 @func3(i4 %x, i4 %y) nounwind {
 ; X86-NEXT:    addb %dl, %al
 ; X86-NEXT:    movzbl %al, %eax
 ; X86-NEXT:    cmovol %ecx, %eax
-; X86-NEXT:    sarb $4, %al
 ; X86-NEXT:    # kill: def $al killed $al killed $eax
 ; X86-NEXT:    retl
 ;
-; X64-LABEL: func3:
+; X64-LABEL: func8:
 ; X64:       # %bb.0:
-; X64-NEXT:    shlb $4, %sil
-; X64-NEXT:    shlb $4, %dil
 ; X64-NEXT:    xorl %ecx, %ecx
 ; X64-NEXT:    movl %edi, %eax
 ; X64-NEXT:    addb %sil, %al
@@ -120,8 +149,38 @@ define i4 @func3(i4 %x, i4 %y) nounwind {
 ; X64-NEXT:    addb %sil, %dil
 ; X64-NEXT:    movzbl %dil, %eax
 ; X64-NEXT:    cmovol %ecx, %eax
-; X64-NEXT:    sarb $4, %al
 ; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    retq
+  %tmp = call i8 @llvm.sadd.sat.i8(i8 %x, i8 %y)
+  ret i8 %tmp
+}
+
+define signext i4 @func3(i4 signext %x, i4 signext %y) nounwind {
+; X86-LABEL: func3:
+; X86:       # %bb.0:
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    addb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    movzbl %al, %ecx
+; X86-NEXT:    cmpb $7, %al
+; X86-NEXT:    movl $7, %eax
+; X86-NEXT:    cmovll %ecx, %eax
+; X86-NEXT:    cmpb $-8, %al
+; X86-NEXT:    movl $248, %ecx
+; X86-NEXT:    cmovgl %eax, %ecx
+; X86-NEXT:    movsbl %cl, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: func3:
+; X64:       # %bb.0:
+; X64-NEXT:    addb %sil, %dil
+; X64-NEXT:    movzbl %dil, %eax
+; X64-NEXT:    cmpb $7, %al
+; X64-NEXT:    movl $7, %ecx
+; X64-NEXT:    cmovll %eax, %ecx
+; X64-NEXT:    cmpb $-8, %cl
+; X64-NEXT:    movl $248, %eax
+; X64-NEXT:    cmovgl %ecx, %eax
+; X64-NEXT:    movsbl %al, %eax
 ; X64-NEXT:    retq
   %tmp = call i4 @llvm.sadd.sat.i4(i4 %x, i4 %y);
   ret i4 %tmp;
@@ -183,28 +242,20 @@ define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
 ;
 ; X64-LABEL: vec:
 ; X64:       # %bb.0:
+; X64-NEXT:    pxor %xmm2, %xmm2
 ; X64-NEXT:    pxor %xmm3, %xmm3
-; X64-NEXT:    pxor %xmm4, %xmm4
-; X64-NEXT:    pcmpgtd %xmm1, %xmm4
-; X64-NEXT:    pcmpeqd %xmm2, %xmm2
-; X64-NEXT:    pxor %xmm2, %xmm4
-; X64-NEXT:    pxor %xmm5, %xmm5
-; X64-NEXT:    pcmpgtd %xmm0, %xmm5
-; X64-NEXT:    pxor %xmm2, %xmm5
-; X64-NEXT:    pcmpeqd %xmm5, %xmm4
-; X64-NEXT:    paddd %xmm1, %xmm0
-; X64-NEXT:    pcmpgtd %xmm0, %xmm3
-; X64-NEXT:    pxor %xmm3, %xmm2
-; X64-NEXT:    pcmpeqd %xmm5, %xmm2
-; X64-NEXT:    pandn %xmm4, %xmm2
-; X64-NEXT:    movdqa %xmm3, %xmm1
-; X64-NEXT:    pandn {{.*}}(%rip), %xmm1
-; X64-NEXT:    psrld $1, %xmm3
-; X64-NEXT:    por %xmm1, %xmm3
-; X64-NEXT:    pand %xmm2, %xmm3
-; X64-NEXT:    pandn %xmm0, %xmm2
+; X64-NEXT:    pcmpgtd %xmm1, %xmm3
+; X64-NEXT:    paddd %xmm0, %xmm1
+; X64-NEXT:    pcmpgtd %xmm1, %xmm0
+; X64-NEXT:    pxor %xmm3, %xmm0
+; X64-NEXT:    movdqa %xmm1, %xmm3
+; X64-NEXT:    pandn {{.*}}(%rip), %xmm3
+; X64-NEXT:    pcmpgtd %xmm1, %xmm2
+; X64-NEXT:    psrld $1, %xmm2
 ; X64-NEXT:    por %xmm3, %xmm2
-; X64-NEXT:    movdqa %xmm2, %xmm0
+; X64-NEXT:    pand %xmm0, %xmm2
+; X64-NEXT:    pandn %xmm1, %xmm0
+; X64-NEXT:    por %xmm2, %xmm0
 ; X64-NEXT:    retq
   %tmp = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> %x, <4 x i32> %y);
   ret <4 x i32> %tmp;
diff --git a/llvm/test/CodeGen/X86/sadd_sat_plus.ll b/llvm/test/CodeGen/X86/sadd_sat_plus.ll
new file mode 100644
index 000000000..2c902be80
--- /dev/null
+++ b/llvm/test/CodeGen/X86/sadd_sat_plus.ll
@@ -0,0 +1,207 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686 -mattr=cmov | FileCheck %s --check-prefixes=CHECK,X86
+; RUN: llc < %s -mtriple=x86_64-linux | FileCheck %s --check-prefixes=CHECK,X64
+
+declare i4 @llvm.sadd.sat.i4(i4, i4)
+declare i8 @llvm.sadd.sat.i8(i8, i8)
+declare i16 @llvm.sadd.sat.i16(i16, i16)
+declare i32 @llvm.sadd.sat.i32(i32, i32)
+declare i64 @llvm.sadd.sat.i64(i64, i64)
+
+define i32 @func32(i32 %x, i32 %y, i32 %z) nounwind {
+; X86-LABEL: func32:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    imull {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    xorl %ecx, %ecx
+; X86-NEXT:    movl %eax, %esi
+; X86-NEXT:    addl %edx, %esi
+; X86-NEXT:    setns %cl
+; X86-NEXT:    addl $2147483647, %ecx # imm = 0x7FFFFFFF
+; X86-NEXT:    addl %edx, %eax
+; X86-NEXT:    cmovol %ecx, %eax
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
+;
+; X64-LABEL: func32:
+; X64:       # %bb.0:
+; X64-NEXT:    imull %edx, %esi
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    movl %edi, %ecx
+; X64-NEXT:    addl %esi, %ecx
+; X64-NEXT:    setns %al
+; X64-NEXT:    addl $2147483647, %eax # imm = 0x7FFFFFFF
+; X64-NEXT:    addl %edi, %esi
+; X64-NEXT:    cmovnol %esi, %eax
+; X64-NEXT:    retq
+  %a = mul i32 %y, %z
+  %tmp = call i32 @llvm.sadd.sat.i32(i32 %x, i32 %a)
+  ret i32 %tmp
+}
+
+define i64 @func64(i64 %x, i64 %y, i64 %z) nounwind {
+; X86-LABEL: func64:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT:    addl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl %ebx, %ebp
+; X86-NEXT:    adcl %esi, %ebp
+; X86-NEXT:    movl %ebp, %eax
+; X86-NEXT:    sarl $31, %eax
+; X86-NEXT:    xorl %ecx, %ecx
+; X86-NEXT:    testl %ebp, %ebp
+; X86-NEXT:    setns %cl
+; X86-NEXT:    movl %ecx, %edx
+; X86-NEXT:    addl $2147483647, %edx # imm = 0x7FFFFFFF
+; X86-NEXT:    testl %ebx, %ebx
+; X86-NEXT:    setns %bl
+; X86-NEXT:    cmpb %cl, %bl
+; X86-NEXT:    setne %cl
+; X86-NEXT:    testl %esi, %esi
+; X86-NEXT:    setns %ch
+; X86-NEXT:    cmpb %ch, %bl
+; X86-NEXT:    sete %ch
+; X86-NEXT:    testb %cl, %ch
+; X86-NEXT:    cmovel %ebp, %edx
+; X86-NEXT:    cmovel %edi, %eax
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl
+;
+; X64-LABEL: func64:
+; X64:       # %bb.0:
+; X64-NEXT:    xorl %ecx, %ecx
+; X64-NEXT:    movq %rdi, %rax
+; X64-NEXT:    addq %rdx, %rax
+; X64-NEXT:    setns %cl
+; X64-NEXT:    movabsq $9223372036854775807, %rax # imm = 0x7FFFFFFFFFFFFFFF
+; X64-NEXT:    addq %rcx, %rax
+; X64-NEXT:    addq %rdx, %rdi
+; X64-NEXT:    cmovnoq %rdi, %rax
+; X64-NEXT:    retq
+  %a = mul i64 %y, %z
+  %tmp = call i64 @llvm.sadd.sat.i64(i64 %x, i64 %z)
+  ret i64 %tmp
+}
+
+define signext i16 @func16(i16 signext %x, i16 signext %y, i16 signext %z) nounwind {
+; X86-LABEL: func16:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    imulw {{[0-9]+}}(%esp), %ax
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    xorl %ecx, %ecx
+; X86-NEXT:    movl %eax, %esi
+; X86-NEXT:    addw %dx, %si
+; X86-NEXT:    setns %cl
+; X86-NEXT:    addl $32767, %ecx # imm = 0x7FFF
+; X86-NEXT:    addw %dx, %ax
+; X86-NEXT:    cmovol %ecx, %eax
+; X86-NEXT:    # kill: def $ax killed $ax killed $eax
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
+;
+; X64-LABEL: func16:
+; X64:       # %bb.0:
+; X64-NEXT:    imull %edx, %esi
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    movl %edi, %ecx
+; X64-NEXT:    addw %si, %cx
+; X64-NEXT:    setns %al
+; X64-NEXT:    addl $32767, %eax # imm = 0x7FFF
+; X64-NEXT:    addw %si, %di
+; X64-NEXT:    cmovnol %edi, %eax
+; X64-NEXT:    # kill: def $ax killed $ax killed $eax
+; X64-NEXT:    retq
+  %a = mul i16 %y, %z
+  %tmp = call i16 @llvm.sadd.sat.i16(i16 %x, i16 %a)
+  ret i16 %tmp
+}
+
+define signext i8 @func8(i8 signext %x, i8 signext %y, i8 signext %z) nounwind {
+; X86-LABEL: func8:
+; X86:       # %bb.0:
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    mulb {{[0-9]+}}(%esp)
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %dl
+; X86-NEXT:    xorl %ecx, %ecx
+; X86-NEXT:    movb %al, %ah
+; X86-NEXT:    addb %dl, %ah
+; X86-NEXT:    setns %cl
+; X86-NEXT:    addl $127, %ecx
+; X86-NEXT:    addb %dl, %al
+; X86-NEXT:    movzbl %al, %eax
+; X86-NEXT:    cmovol %ecx, %eax
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: func8:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %esi, %eax
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    mulb %dl
+; X64-NEXT:    xorl %ecx, %ecx
+; X64-NEXT:    movl %edi, %edx
+; X64-NEXT:    addb %al, %dl
+; X64-NEXT:    setns %cl
+; X64-NEXT:    addl $127, %ecx
+; X64-NEXT:    addb %al, %dil
+; X64-NEXT:    movzbl %dil, %eax
+; X64-NEXT:    cmovol %ecx, %eax
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    retq
+  %a = mul i8 %y, %z
+  %tmp = call i8 @llvm.sadd.sat.i8(i8 %x, i8 %a)
+  ret i8 %tmp
+}
+
+define signext i4 @func4(i4 signext %x, i4 signext %y, i4 signext %z) nounwind {
+; X86-LABEL: func4:
+; X86:       # %bb.0:
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    mulb {{[0-9]+}}(%esp)
+; X86-NEXT:    shlb $4, %al
+; X86-NEXT:    sarb $4, %al
+; X86-NEXT:    addb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    movzbl %al, %ecx
+; X86-NEXT:    cmpb $7, %al
+; X86-NEXT:    movl $7, %eax
+; X86-NEXT:    cmovll %ecx, %eax
+; X86-NEXT:    cmpb $-8, %al
+; X86-NEXT:    movl $248, %ecx
+; X86-NEXT:    cmovgl %eax, %ecx
+; X86-NEXT:    movsbl %cl, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: func4:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %esi, %eax
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    mulb %dl
+; X64-NEXT:    shlb $4, %al
+; X64-NEXT:    sarb $4, %al
+; X64-NEXT:    addb %dil, %al
+; X64-NEXT:    movzbl %al, %eax
+; X64-NEXT:    cmpb $7, %al
+; X64-NEXT:    movl $7, %ecx
+; X64-NEXT:    cmovll %eax, %ecx
+; X64-NEXT:    cmpb $-8, %cl
+; X64-NEXT:    movl $248, %eax
+; X64-NEXT:    cmovgl %ecx, %eax
+; X64-NEXT:    movsbl %al, %eax
+; X64-NEXT:    retq
+  %a = mul i4 %y, %z
+  %tmp = call i4 @llvm.sadd.sat.i4(i4 %x, i4 %a)
+  ret i4 %tmp
+}
diff --git a/llvm/test/CodeGen/X86/sadd_sat_vec.ll b/llvm/test/CodeGen/X86/sadd_sat_vec.ll
index b0f5bfa31..25e9a09c4 100644
--- a/llvm/test/CodeGen/X86/sadd_sat_vec.ll
+++ b/llvm/test/CodeGen/X86/sadd_sat_vec.ll
@@ -210,30 +210,13 @@ define void @v8i8(<8 x i8>* %px, <8 x i8>* %py, <8 x i8>* %pz) nounwind {
 ; SSE-NEXT:    movq %xmm1, (%rdx)
 ; SSE-NEXT:    retq
 ;
-; AVX1-LABEL: v8i8:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
-; AVX1-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
-; AVX1-NEXT:    vpaddsb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vmovq %xmm0, (%rdx)
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: v8i8:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
-; AVX2-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
-; AVX2-NEXT:    vpaddsb %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vmovq %xmm0, (%rdx)
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: v8i8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
-; AVX512-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
-; AVX512-NEXT:    vpaddsb %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX512-NEXT:    vpmovwb %xmm0, (%rdx)
-; AVX512-NEXT:    retq
+; AVX-LABEL: v8i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
+; AVX-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
+; AVX-NEXT:    vpaddsb %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vmovq %xmm0, (%rdx)
+; AVX-NEXT:    retq
   %x = load <8 x i8>, <8 x i8>* %px
   %y = load <8 x i8>, <8 x i8>* %py
   %z = call <8 x i8> @llvm.sadd.sat.v8i8(<8 x i8> %x, <8 x i8> %y)
@@ -250,30 +233,13 @@ define void @v4i8(<4 x i8>* %px, <4 x i8>* %py, <4 x i8>* %pz) nounwind {
 ; SSE-NEXT:    movd %xmm1, (%rdx)
 ; SSE-NEXT:    retq
 ;
-; AVX1-LABEL: v4i8:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX1-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX1-NEXT:    vpaddsb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vmovd %xmm0, (%rdx)
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: v4i8:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX2-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX2-NEXT:    vpaddsb %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vmovd %xmm0, (%rdx)
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: v4i8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX512-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX512-NEXT:    vpaddsb %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; AVX512-NEXT:    vpmovdb %xmm0, (%rdx)
-; AVX512-NEXT:    retq
+; AVX-LABEL: v4i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX-NEXT:    vpaddsb %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vmovd %xmm0, (%rdx)
+; AVX-NEXT:    retq
   %x = load <4 x i8>, <4 x i8>* %px
   %y = load <4 x i8>, <4 x i8>* %py
   %z = call <4 x i8> @llvm.sadd.sat.v4i8(<4 x i8> %x, <4 x i8> %y)
@@ -314,36 +280,15 @@ define void @v2i8(<2 x i8>* %px, <2 x i8>* %py, <2 x i8>* %pz) nounwind {
 ; SSE41-NEXT:    pextrw $0, %xmm1, (%rdx)
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: v2i8:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    movzwl (%rdi), %eax
-; AVX1-NEXT:    vmovd %eax, %xmm0
-; AVX1-NEXT:    movzwl (%rsi), %eax
-; AVX1-NEXT:    vmovd %eax, %xmm1
-; AVX1-NEXT:    vpaddsb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpextrw $0, %xmm0, (%rdx)
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: v2i8:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    movzwl (%rdi), %eax
-; AVX2-NEXT:    vmovd %eax, %xmm0
-; AVX2-NEXT:    movzwl (%rsi), %eax
-; AVX2-NEXT:    vmovd %eax, %xmm1
-; AVX2-NEXT:    vpaddsb %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpextrw $0, %xmm0, (%rdx)
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: v2i8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    movzwl (%rdi), %eax
-; AVX512-NEXT:    vmovd %eax, %xmm0
-; AVX512-NEXT:    movzwl (%rsi), %eax
-; AVX512-NEXT:    vmovd %eax, %xmm1
-; AVX512-NEXT:    vpaddsb %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT:    vpmovqb %xmm0, (%rdx)
-; AVX512-NEXT:    retq
+; AVX-LABEL: v2i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    movzwl (%rdi), %eax
+; AVX-NEXT:    vmovd %eax, %xmm0
+; AVX-NEXT:    movzwl (%rsi), %eax
+; AVX-NEXT:    vmovd %eax, %xmm1
+; AVX-NEXT:    vpaddsb %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpextrw $0, %xmm0, (%rdx)
+; AVX-NEXT:    retq
   %x = load <2 x i8>, <2 x i8>* %px
   %y = load <2 x i8>, <2 x i8>* %py
   %z = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> %x, <2 x i8> %y)
@@ -360,30 +305,13 @@ define void @v4i16(<4 x i16>* %px, <4 x i16>* %py, <4 x i16>* %pz) nounwind {
 ; SSE-NEXT:    movq %xmm1, (%rdx)
 ; SSE-NEXT:    retq
 ;
-; AVX1-LABEL: v4i16:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
-; AVX1-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
-; AVX1-NEXT:    vpaddsw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vmovq %xmm0, (%rdx)
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: v4i16:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
-; AVX2-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
-; AVX2-NEXT:    vpaddsw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vmovq %xmm0, (%rdx)
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: v4i16:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
-; AVX512-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
-; AVX512-NEXT:    vpaddsw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; AVX512-NEXT:    vpmovdw %xmm0, (%rdx)
-; AVX512-NEXT:    retq
+; AVX-LABEL: v4i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
+; AVX-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
+; AVX-NEXT:    vpaddsw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vmovq %xmm0, (%rdx)
+; AVX-NEXT:    retq
   %x = load <4 x i16>, <4 x i16>* %px
   %y = load <4 x i16>, <4 x i16>* %py
   %z = call <4 x i16> @llvm.sadd.sat.v4i16(<4 x i16> %x, <4 x i16> %y)
@@ -400,30 +328,13 @@ define void @v2i16(<2 x i16>* %px, <2 x i16>* %py, <2 x i16>* %pz) nounwind {
 ; SSE-NEXT:    movd %xmm1, (%rdx)
 ; SSE-NEXT:    retq
 ;
-; AVX1-LABEL: v2i16:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX1-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX1-NEXT:    vpaddsw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vmovd %xmm0, (%rdx)
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: v2i16:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX2-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX2-NEXT:    vpaddsw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vmovd %xmm0, (%rdx)
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: v2i16:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX512-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX512-NEXT:    vpaddsw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; AVX512-NEXT:    vpmovqw %xmm0, (%rdx)
-; AVX512-NEXT:    retq
+; AVX-LABEL: v2i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX-NEXT:    vpaddsw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vmovd %xmm0, (%rdx)
+; AVX-NEXT:    retq
   %x = load <2 x i16>, <2 x i16>* %px
   %y = load <2 x i16>, <2 x i16>* %py
   %z = call <2 x i16> @llvm.sadd.sat.v2i16(<2 x i16> %x, <2 x i16> %y)
@@ -658,240 +569,88 @@ define <16 x i1> @v16i1(<16 x i1> %x, <16 x i1> %y) nounwind {
 define <2 x i32> @v2i32(<2 x i32> %x, <2 x i32> %y) nounwind {
 ; SSE2-LABEL: v2i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    psllq $32, %xmm1
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
-; SSE2-NEXT:    psllq $32, %xmm0
-; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    paddq %xmm1, %xmm0
-; SSE2-NEXT:    por %xmm2, %xmm1
-; SSE2-NEXT:    movdqa %xmm2, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pand %xmm5, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm4
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pxor %xmm1, %xmm4
-; SSE2-NEXT:    por %xmm2, %xmm3
-; SSE2-NEXT:    movdqa %xmm2, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm5
-; SSE2-NEXT:    pxor %xmm1, %xmm5
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,0,3,2]
-; SSE2-NEXT:    pand %xmm4, %xmm3
-; SSE2-NEXT:    movdqa %xmm0, %xmm4
-; SSE2-NEXT:    por %xmm2, %xmm4
-; SSE2-NEXT:    movdqa %xmm2, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm4, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3]
-; SSE2-NEXT:    pand %xmm7, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[1,1,3,3]
-; SSE2-NEXT:    por %xmm2, %xmm4
-; SSE2-NEXT:    pxor %xmm4, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,0,3,2]
-; SSE2-NEXT:    pand %xmm1, %xmm2
-; SSE2-NEXT:    pandn %xmm3, %xmm2
-; SSE2-NEXT:    movdqa %xmm4, %xmm1
-; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm1
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm4
-; SSE2-NEXT:    por %xmm1, %xmm4
-; SSE2-NEXT:    pand %xmm2, %xmm4
-; SSE2-NEXT:    pandn %xmm0, %xmm2
-; SSE2-NEXT:    por %xmm4, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,3,2,3]
-; SSE2-NEXT:    psrad $31, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,3,2,3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
+; SSE2-NEXT:    paddd %xmm0, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
+; SSE2-NEXT:    pxor %xmm3, %xmm0
+; SSE2-NEXT:    movdqa %xmm1, %xmm3
+; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSE2-NEXT:    psrld $1, %xmm2
+; SSE2-NEXT:    por %xmm3, %xmm2
+; SSE2-NEXT:    pand %xmm0, %xmm2
+; SSE2-NEXT:    pandn %xmm1, %xmm0
+; SSE2-NEXT:    por %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: v2i32:
 ; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    psllq $32, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
-; SSSE3-NEXT:    psllq $32, %xmm0
-; SSSE3-NEXT:    movdqa %xmm0, %xmm3
-; SSSE3-NEXT:    paddq %xmm1, %xmm0
-; SSSE3-NEXT:    por %xmm2, %xmm1
-; SSSE3-NEXT:    movdqa %xmm2, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm5, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    por %xmm1, %xmm4
-; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSSE3-NEXT:    pxor %xmm1, %xmm4
-; SSSE3-NEXT:    por %xmm2, %xmm3
-; SSSE3-NEXT:    movdqa %xmm2, %xmm5
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm5
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm6, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSSE3-NEXT:    por %xmm3, %xmm5
-; SSSE3-NEXT:    pxor %xmm1, %xmm5
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm4, %xmm3
-; SSSE3-NEXT:    movdqa %xmm0, %xmm4
-; SSSE3-NEXT:    por %xmm2, %xmm4
-; SSSE3-NEXT:    movdqa %xmm2, %xmm6
-; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm7, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm4
-; SSSE3-NEXT:    pxor %xmm4, %xmm1
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pandn %xmm3, %xmm2
-; SSSE3-NEXT:    movdqa %xmm4, %xmm1
-; SSSE3-NEXT:    pandn {{.*}}(%rip), %xmm1
-; SSSE3-NEXT:    pand {{.*}}(%rip), %xmm4
-; SSSE3-NEXT:    por %xmm1, %xmm4
-; SSSE3-NEXT:    pand %xmm2, %xmm4
-; SSSE3-NEXT:    pandn %xmm0, %xmm2
-; SSSE3-NEXT:    por %xmm4, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,3,2,3]
-; SSSE3-NEXT:    psrad $31, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,3,2,3]
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSSE3-NEXT:    pxor %xmm2, %xmm2
+; SSSE3-NEXT:    pxor %xmm3, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm3
+; SSSE3-NEXT:    paddd %xmm0, %xmm1
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
+; SSSE3-NEXT:    pxor %xmm3, %xmm0
+; SSSE3-NEXT:    movdqa %xmm1, %xmm3
+; SSSE3-NEXT:    pandn {{.*}}(%rip), %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSSE3-NEXT:    psrld $1, %xmm2
+; SSSE3-NEXT:    por %xmm3, %xmm2
+; SSSE3-NEXT:    pand %xmm0, %xmm2
+; SSSE3-NEXT:    pandn %xmm1, %xmm0
+; SSSE3-NEXT:    por %xmm2, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: v2i32:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    psllq $32, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483648,2147483648]
-; SSE41-NEXT:    psllq $32, %xmm2
-; SSE41-NEXT:    movdqa %xmm2, %xmm3
-; SSE41-NEXT:    paddq %xmm1, %xmm2
-; SSE41-NEXT:    por %xmm0, %xmm1
-; SSE41-NEXT:    movdqa %xmm0, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm1, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE41-NEXT:    por %xmm1, %xmm4
-; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE41-NEXT:    pxor %xmm1, %xmm4
-; SSE41-NEXT:    por %xmm0, %xmm3
-; SSE41-NEXT:    movdqa %xmm0, %xmm5
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pand %xmm6, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE41-NEXT:    por %xmm3, %xmm5
-; SSE41-NEXT:    pxor %xmm1, %xmm5
-; SSE41-NEXT:    pcmpeqq %xmm5, %xmm4
-; SSE41-NEXT:    movdqa %xmm2, %xmm3
-; SSE41-NEXT:    por %xmm0, %xmm3
-; SSE41-NEXT:    movdqa %xmm0, %xmm6
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pand %xmm7, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3]
-; SSE41-NEXT:    por %xmm3, %xmm0
-; SSE41-NEXT:    pxor %xmm0, %xmm1
-; SSE41-NEXT:    pcmpeqq %xmm5, %xmm1
-; SSE41-NEXT:    pandn %xmm4, %xmm1
-; SSE41-NEXT:    movapd {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808]
-; SSE41-NEXT:    blendvpd %xmm0, {{.*}}(%rip), %xmm3
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3]
-; SSE41-NEXT:    psrad $31, %xmm2
-; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
+; SSE41-NEXT:    movdqa %xmm0, %xmm3
+; SSE41-NEXT:    paddd %xmm1, %xmm3
+; SSE41-NEXT:    movaps {{.*#+}} xmm4 = [2147483648,2147483648,2147483648,2147483648]
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, {{.*}}(%rip), %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm2
+; SSE41-NEXT:    pxor %xmm1, %xmm2
 ; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm4, %xmm3
+; SSE41-NEXT:    movaps %xmm3, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: v2i32:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpsllq $32, %xmm1, %xmm1
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm2, %xmm3
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
-; AVX1-NEXT:    vpsllq $32, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm5
-; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX1-NEXT:    vpcmpeqq %xmm3, %xmm5, %xmm3
-; AVX1-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm4, %xmm1, %xmm2
-; AVX1-NEXT:    vpcmpeqq %xmm2, %xmm5, %xmm2
-; AVX1-NEXT:    vpandn %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vmovapd {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808]
-; AVX1-NEXT:    vblendvpd %xmm1, {{.*}}(%rip), %xmm3, %xmm1
-; AVX1-NEXT:    vblendvpd %xmm2, %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrad $31, %xmm0, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
+; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm2
+; AVX1-NEXT:    vmovaps {{.*#+}} xmm3 = [2147483648,2147483648,2147483648,2147483648]
+; AVX1-NEXT:    vblendvps %xmm2, {{.*}}(%rip), %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vblendvps %xmm0, %xmm3, %xmm2, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: v2i32:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpsllq $32, %xmm1, %xmm1
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtq %xmm1, %xmm2, %xmm3
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpxor %xmm4, %xmm3, %xmm3
-; AVX2-NEXT:    vpsllq $32, %xmm0, %xmm0
-; AVX2-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm5
-; AVX2-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX2-NEXT:    vpcmpeqq %xmm3, %xmm5, %xmm3
-; AVX2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm1
-; AVX2-NEXT:    vpxor %xmm4, %xmm1, %xmm2
-; AVX2-NEXT:    vpcmpeqq %xmm2, %xmm5, %xmm2
-; AVX2-NEXT:    vpandn %xmm3, %xmm2, %xmm2
-; AVX2-NEXT:    vmovapd {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808]
-; AVX2-NEXT:    vblendvpd %xmm1, {{.*}}(%rip), %xmm3, %xmm1
-; AVX2-NEXT:    vblendvpd %xmm2, %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrad $31, %xmm0, %xmm1
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm2
+; AVX2-NEXT:    vbroadcastss {{.*#+}} xmm3 = [2147483647,2147483647,2147483647,2147483647]
+; AVX2-NEXT:    vbroadcastss {{.*#+}} xmm4 = [2147483648,2147483648,2147483648,2147483648]
+; AVX2-NEXT:    vblendvps %xmm2, %xmm3, %xmm4, %xmm3
+; AVX2-NEXT:    vpcmpgtd %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor %xmm0, %xmm1, %xmm0
+; AVX2-NEXT:    vblendvps %xmm0, %xmm3, %xmm2, %xmm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: v2i32:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsllq $32, %xmm1, %xmm1
 ; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpcmpnltq %xmm2, %xmm1, %k0
-; AVX512-NEXT:    vpsllq $32, %xmm0, %xmm0
-; AVX512-NEXT:    vpcmpnltq %xmm2, %xmm0, %k1
-; AVX512-NEXT:    kxorw %k0, %k1, %k0
-; AVX512-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpcmpnltq %xmm2, %xmm0, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
-; AVX512-NEXT:    kandnw %k1, %k0, %k1
-; AVX512-NEXT:    vpcmpgtq %xmm0, %xmm2, %k2
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = [9223372036854775808,9223372036854775808]
-; AVX512-NEXT:    vmovdqa64 {{.*}}(%rip), %xmm1 {%k2}
-; AVX512-NEXT:    vmovdqa64 %xmm1, %xmm0 {%k1}
-; AVX512-NEXT:    vpsraq $32, %xmm0, %xmm0
+; AVX512-NEXT:    vpcmpgtd %xmm1, %xmm2, %k0
+; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm1
+; AVX512-NEXT:    vpcmpgtd %xmm1, %xmm0, %k1
+; AVX512-NEXT:    kxorw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpgtd %xmm1, %xmm2, %k2
+; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm0 = [2147483648,2147483648,2147483648,2147483648]
+; AVX512-NEXT:    vpbroadcastd {{.*}}(%rip), %xmm0 {%k2}
+; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm1 {%k1}
+; AVX512-NEXT:    vmovdqa %xmm1, %xmm0
 ; AVX512-NEXT:    retq
   %z = call <2 x i32> @llvm.sadd.sat.v2i32(<2 x i32> %x, <2 x i32> %y)
   ret <2 x i32> %z
@@ -900,133 +659,88 @@ define <2 x i32> @v2i32(<2 x i32> %x, <2 x i32> %y) nounwind {
 define <4 x i32> @v4i32(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; SSE2-LABEL: v4i32:
 ; SSE2:       # %bb.0:
+; SSE2-NEXT:    pxor %xmm2, %xmm2
 ; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    pxor %xmm4, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm4
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT:    pxor %xmm2, %xmm4
-; SSE2-NEXT:    pxor %xmm5, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm5
-; SSE2-NEXT:    pxor %xmm2, %xmm5
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm4
-; SSE2-NEXT:    paddd %xmm1, %xmm0
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSE2-NEXT:    pxor %xmm3, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm2
-; SSE2-NEXT:    pandn %xmm4, %xmm2
-; SSE2-NEXT:    movdqa %xmm3, %xmm1
-; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm1
-; SSE2-NEXT:    psrld $1, %xmm3
-; SSE2-NEXT:    por %xmm1, %xmm3
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pandn %xmm0, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
+; SSE2-NEXT:    paddd %xmm0, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
+; SSE2-NEXT:    pxor %xmm3, %xmm0
+; SSE2-NEXT:    movdqa %xmm1, %xmm3
+; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSE2-NEXT:    psrld $1, %xmm2
 ; SSE2-NEXT:    por %xmm3, %xmm2
-; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    pand %xmm0, %xmm2
+; SSE2-NEXT:    pandn %xmm1, %xmm0
+; SSE2-NEXT:    por %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: v4i32:
 ; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    pxor %xmm2, %xmm2
 ; SSSE3-NEXT:    pxor %xmm3, %xmm3
-; SSSE3-NEXT:    pxor %xmm4, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm4
-; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSSE3-NEXT:    pxor %xmm2, %xmm4
-; SSSE3-NEXT:    pxor %xmm5, %xmm5
-; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm5
-; SSSE3-NEXT:    pxor %xmm2, %xmm5
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm4
-; SSSE3-NEXT:    paddd %xmm1, %xmm0
-; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSSE3-NEXT:    pxor %xmm3, %xmm2
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm2
-; SSSE3-NEXT:    pandn %xmm4, %xmm2
-; SSSE3-NEXT:    movdqa %xmm3, %xmm1
-; SSSE3-NEXT:    pandn {{.*}}(%rip), %xmm1
-; SSSE3-NEXT:    psrld $1, %xmm3
-; SSSE3-NEXT:    por %xmm1, %xmm3
-; SSSE3-NEXT:    pand %xmm2, %xmm3
-; SSSE3-NEXT:    pandn %xmm0, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm3
+; SSSE3-NEXT:    paddd %xmm0, %xmm1
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
+; SSSE3-NEXT:    pxor %xmm3, %xmm0
+; SSSE3-NEXT:    movdqa %xmm1, %xmm3
+; SSSE3-NEXT:    pandn {{.*}}(%rip), %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSSE3-NEXT:    psrld $1, %xmm2
 ; SSSE3-NEXT:    por %xmm3, %xmm2
-; SSSE3-NEXT:    movdqa %xmm2, %xmm0
+; SSSE3-NEXT:    pand %xmm0, %xmm2
+; SSSE3-NEXT:    pandn %xmm1, %xmm0
+; SSSE3-NEXT:    por %xmm2, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: v4i32:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    pxor %xmm4, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm1, %xmm4
-; SSE41-NEXT:    pcmpeqd %xmm3, %xmm3
-; SSE41-NEXT:    pxor %xmm3, %xmm4
-; SSE41-NEXT:    pxor %xmm5, %xmm5
-; SSE41-NEXT:    pcmpgtd %xmm2, %xmm5
-; SSE41-NEXT:    pxor %xmm3, %xmm5
-; SSE41-NEXT:    pcmpeqd %xmm5, %xmm4
-; SSE41-NEXT:    paddd %xmm1, %xmm2
-; SSE41-NEXT:    pcmpgtd %xmm2, %xmm0
-; SSE41-NEXT:    pxor %xmm0, %xmm3
-; SSE41-NEXT:    pcmpeqd %xmm5, %xmm3
-; SSE41-NEXT:    pandn %xmm4, %xmm3
-; SSE41-NEXT:    movaps {{.*#+}} xmm1 = [2147483648,2147483648,2147483648,2147483648]
-; SSE41-NEXT:    blendvps %xmm0, {{.*}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa %xmm0, %xmm3
+; SSE41-NEXT:    paddd %xmm1, %xmm3
+; SSE41-NEXT:    movaps {{.*#+}} xmm4 = [2147483648,2147483648,2147483648,2147483648]
 ; SSE41-NEXT:    movdqa %xmm3, %xmm0
-; SSE41-NEXT:    blendvps %xmm0, %xmm1, %xmm2
-; SSE41-NEXT:    movaps %xmm2, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, {{.*}}(%rip), %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm2
+; SSE41-NEXT:    pxor %xmm1, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm4, %xmm3
+; SSE41-NEXT:    movaps %xmm3, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: v4i32:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm3
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm2, %xmm5
-; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm5, %xmm3
-; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm4, %xmm1, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm5, %xmm2
-; AVX1-NEXT:    vpandn %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm2
 ; AVX1-NEXT:    vmovaps {{.*#+}} xmm3 = [2147483648,2147483648,2147483648,2147483648]
-; AVX1-NEXT:    vblendvps %xmm1, {{.*}}(%rip), %xmm3, %xmm1
-; AVX1-NEXT:    vblendvps %xmm2, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vblendvps %xmm2, {{.*}}(%rip), %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vblendvps %xmm0, %xmm3, %xmm2, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: v4i32:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm3
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpxor %xmm4, %xmm3, %xmm3
-; AVX2-NEXT:    vpcmpgtd %xmm0, %xmm2, %xmm5
-; AVX2-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm5, %xmm3
-; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpcmpgtd %xmm0, %xmm2, %xmm1
-; AVX2-NEXT:    vpxor %xmm4, %xmm1, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm5, %xmm2
-; AVX2-NEXT:    vpandn %xmm3, %xmm2, %xmm2
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm2
 ; AVX2-NEXT:    vbroadcastss {{.*#+}} xmm3 = [2147483647,2147483647,2147483647,2147483647]
 ; AVX2-NEXT:    vbroadcastss {{.*#+}} xmm4 = [2147483648,2147483648,2147483648,2147483648]
-; AVX2-NEXT:    vblendvps %xmm1, %xmm3, %xmm4, %xmm1
-; AVX2-NEXT:    vblendvps %xmm2, %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vblendvps %xmm2, %xmm3, %xmm4, %xmm3
+; AVX2-NEXT:    vpcmpgtd %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor %xmm0, %xmm1, %xmm0
+; AVX2-NEXT:    vblendvps %xmm0, %xmm3, %xmm2, %xmm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: v4i32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpcmpnltd %xmm2, %xmm1, %k0
-; AVX512-NEXT:    vpcmpnltd %xmm2, %xmm0, %k1
-; AVX512-NEXT:    kxorw %k0, %k1, %k0
-; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpcmpnltd %xmm2, %xmm0, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
-; AVX512-NEXT:    kandnw %k1, %k0, %k1
-; AVX512-NEXT:    vpcmpgtd %xmm0, %xmm2, %k2
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [2147483648,2147483648,2147483648,2147483648]
-; AVX512-NEXT:    vpbroadcastd {{.*}}(%rip), %xmm1 {%k2}
-; AVX512-NEXT:    vmovdqa32 %xmm1, %xmm0 {%k1}
+; AVX512-NEXT:    vpcmpgtd %xmm1, %xmm2, %k0
+; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm1
+; AVX512-NEXT:    vpcmpgtd %xmm1, %xmm0, %k1
+; AVX512-NEXT:    kxorw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpgtd %xmm1, %xmm2, %k2
+; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm0 = [2147483648,2147483648,2147483648,2147483648]
+; AVX512-NEXT:    vpbroadcastd {{.*}}(%rip), %xmm0 {%k2}
+; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm1 {%k1}
+; AVX512-NEXT:    vmovdqa %xmm1, %xmm0
 ; AVX512-NEXT:    retq
   %z = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> %x, <4 x i32> %y)
   ret <4 x i32> %z
@@ -1035,214 +749,135 @@ define <4 x i32> @v4i32(<4 x i32> %x, <4 x i32> %y) nounwind {
 define <8 x i32> @v8i32(<8 x i32> %x, <8 x i32> %y) nounwind {
 ; SSE2-LABEL: v8i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa %xmm0, %xmm8
-; SSE2-NEXT:    pxor %xmm6, %xmm6
-; SSE2-NEXT:    pxor %xmm7, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm7
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm5
-; SSE2-NEXT:    pxor %xmm5, %xmm7
 ; SSE2-NEXT:    pxor %xmm4, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE2-NEXT:    pxor %xmm5, %xmm4
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm7
-; SSE2-NEXT:    paddd %xmm2, %xmm8
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpgtd %xmm8, %xmm2
-; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    pxor %xmm5, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm5
+; SSE2-NEXT:    paddd %xmm0, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm0
 ; SSE2-NEXT:    pxor %xmm5, %xmm0
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm0
-; SSE2-NEXT:    pandn %xmm7, %xmm0
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [2147483648,2147483648,2147483648,2147483648]
-; SSE2-NEXT:    movdqa %xmm2, %xmm7
-; SSE2-NEXT:    pandn %xmm4, %xmm7
-; SSE2-NEXT:    psrld $1, %xmm2
-; SSE2-NEXT:    por %xmm7, %xmm2
-; SSE2-NEXT:    pand %xmm0, %xmm2
-; SSE2-NEXT:    pandn %xmm8, %xmm0
-; SSE2-NEXT:    por %xmm2, %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [2147483648,2147483648,2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm2, %xmm6
+; SSE2-NEXT:    pandn %xmm5, %xmm6
+; SSE2-NEXT:    pxor %xmm7, %xmm7
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm7
+; SSE2-NEXT:    psrld $1, %xmm7
+; SSE2-NEXT:    por %xmm6, %xmm7
+; SSE2-NEXT:    pand %xmm0, %xmm7
+; SSE2-NEXT:    pandn %xmm2, %xmm0
+; SSE2-NEXT:    por %xmm7, %xmm0
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
 ; SSE2-NEXT:    pcmpgtd %xmm3, %xmm2
-; SSE2-NEXT:    pxor %xmm5, %xmm2
-; SSE2-NEXT:    pxor %xmm7, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm7
-; SSE2-NEXT:    pxor %xmm5, %xmm7
-; SSE2-NEXT:    pcmpeqd %xmm7, %xmm2
-; SSE2-NEXT:    paddd %xmm3, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm6
-; SSE2-NEXT:    pxor %xmm6, %xmm5
-; SSE2-NEXT:    pcmpeqd %xmm7, %xmm5
-; SSE2-NEXT:    pandn %xmm2, %xmm5
-; SSE2-NEXT:    movdqa %xmm6, %xmm2
-; SSE2-NEXT:    pandn %xmm4, %xmm2
-; SSE2-NEXT:    psrld $1, %xmm6
-; SSE2-NEXT:    por %xmm2, %xmm6
-; SSE2-NEXT:    pand %xmm5, %xmm6
-; SSE2-NEXT:    pandn %xmm1, %xmm5
-; SSE2-NEXT:    por %xmm6, %xmm5
-; SSE2-NEXT:    movdqa %xmm5, %xmm1
+; SSE2-NEXT:    paddd %xmm1, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm1
+; SSE2-NEXT:    pxor %xmm2, %xmm1
+; SSE2-NEXT:    movdqa %xmm3, %xmm2
+; SSE2-NEXT:    pandn %xmm5, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm4
+; SSE2-NEXT:    psrld $1, %xmm4
+; SSE2-NEXT:    por %xmm2, %xmm4
+; SSE2-NEXT:    pand %xmm1, %xmm4
+; SSE2-NEXT:    pandn %xmm3, %xmm1
+; SSE2-NEXT:    por %xmm4, %xmm1
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: v8i32:
 ; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    movdqa %xmm0, %xmm8
-; SSSE3-NEXT:    pxor %xmm6, %xmm6
-; SSSE3-NEXT:    pxor %xmm7, %xmm7
-; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm7
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm5
-; SSSE3-NEXT:    pxor %xmm5, %xmm7
 ; SSSE3-NEXT:    pxor %xmm4, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSSE3-NEXT:    pxor %xmm5, %xmm4
-; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm7
-; SSSE3-NEXT:    paddd %xmm2, %xmm8
-; SSSE3-NEXT:    pxor %xmm2, %xmm2
-; SSSE3-NEXT:    pcmpgtd %xmm8, %xmm2
-; SSSE3-NEXT:    movdqa %xmm2, %xmm0
+; SSSE3-NEXT:    pxor %xmm5, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm5
+; SSSE3-NEXT:    paddd %xmm0, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm0
 ; SSSE3-NEXT:    pxor %xmm5, %xmm0
-; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm0
-; SSSE3-NEXT:    pandn %xmm7, %xmm0
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm4 = [2147483648,2147483648,2147483648,2147483648]
-; SSSE3-NEXT:    movdqa %xmm2, %xmm7
-; SSSE3-NEXT:    pandn %xmm4, %xmm7
-; SSSE3-NEXT:    psrld $1, %xmm2
-; SSSE3-NEXT:    por %xmm7, %xmm2
-; SSSE3-NEXT:    pand %xmm0, %xmm2
-; SSSE3-NEXT:    pandn %xmm8, %xmm0
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm5 = [2147483648,2147483648,2147483648,2147483648]
+; SSSE3-NEXT:    movdqa %xmm2, %xmm6
+; SSSE3-NEXT:    pandn %xmm5, %xmm6
+; SSSE3-NEXT:    pxor %xmm7, %xmm7
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm7
+; SSSE3-NEXT:    psrld $1, %xmm7
+; SSSE3-NEXT:    por %xmm6, %xmm7
+; SSSE3-NEXT:    pand %xmm0, %xmm7
+; SSSE3-NEXT:    pandn %xmm2, %xmm0
+; SSSE3-NEXT:    por %xmm7, %xmm0
 ; SSSE3-NEXT:    pxor %xmm2, %xmm2
 ; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm2
-; SSSE3-NEXT:    pxor %xmm5, %xmm2
-; SSSE3-NEXT:    pxor %xmm7, %xmm7
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm7
-; SSSE3-NEXT:    pxor %xmm5, %xmm7
-; SSSE3-NEXT:    pcmpeqd %xmm7, %xmm2
-; SSSE3-NEXT:    paddd %xmm3, %xmm1
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm6
-; SSSE3-NEXT:    pxor %xmm6, %xmm5
-; SSSE3-NEXT:    pcmpeqd %xmm7, %xmm5
-; SSSE3-NEXT:    pandn %xmm2, %xmm5
-; SSSE3-NEXT:    movdqa %xmm6, %xmm2
-; SSSE3-NEXT:    pandn %xmm4, %xmm2
-; SSSE3-NEXT:    psrld $1, %xmm6
-; SSSE3-NEXT:    por %xmm2, %xmm6
-; SSSE3-NEXT:    pand %xmm5, %xmm6
-; SSSE3-NEXT:    pandn %xmm1, %xmm5
-; SSSE3-NEXT:    por %xmm6, %xmm5
-; SSSE3-NEXT:    movdqa %xmm5, %xmm1
+; SSSE3-NEXT:    paddd %xmm1, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm1
+; SSSE3-NEXT:    pxor %xmm2, %xmm1
+; SSSE3-NEXT:    movdqa %xmm3, %xmm2
+; SSSE3-NEXT:    pandn %xmm5, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm4
+; SSSE3-NEXT:    psrld $1, %xmm4
+; SSSE3-NEXT:    por %xmm2, %xmm4
+; SSSE3-NEXT:    pand %xmm1, %xmm4
+; SSSE3-NEXT:    pandn %xmm3, %xmm1
+; SSSE3-NEXT:    por %xmm4, %xmm1
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: v8i32:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa %xmm0, %xmm6
-; SSE41-NEXT:    pxor %xmm8, %xmm8
-; SSE41-NEXT:    pxor %xmm7, %xmm7
-; SSE41-NEXT:    pcmpgtd %xmm2, %xmm7
-; SSE41-NEXT:    pcmpeqd %xmm4, %xmm4
-; SSE41-NEXT:    pxor %xmm4, %xmm7
-; SSE41-NEXT:    pxor %xmm5, %xmm5
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm5
-; SSE41-NEXT:    pxor %xmm4, %xmm5
-; SSE41-NEXT:    pcmpeqd %xmm5, %xmm7
-; SSE41-NEXT:    paddd %xmm2, %xmm6
-; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    pcmpgtd %xmm6, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    pxor %xmm4, %xmm2
-; SSE41-NEXT:    pcmpeqd %xmm5, %xmm2
-; SSE41-NEXT:    pandn %xmm7, %xmm2
-; SSE41-NEXT:    movaps {{.*#+}} xmm9 = [2147483647,2147483647,2147483647,2147483647]
-; SSE41-NEXT:    movaps {{.*#+}} xmm5 = [2147483648,2147483648,2147483648,2147483648]
-; SSE41-NEXT:    movaps %xmm5, %xmm7
-; SSE41-NEXT:    blendvps %xmm0, %xmm9, %xmm7
-; SSE41-NEXT:    movdqa %xmm2, %xmm0
-; SSE41-NEXT:    blendvps %xmm0, %xmm7, %xmm6
-; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE41-NEXT:    pxor %xmm4, %xmm0
-; SSE41-NEXT:    pxor %xmm2, %xmm2
-; SSE41-NEXT:    pcmpgtd %xmm1, %xmm2
-; SSE41-NEXT:    pxor %xmm4, %xmm2
-; SSE41-NEXT:    pcmpeqd %xmm2, %xmm0
-; SSE41-NEXT:    paddd %xmm3, %xmm1
-; SSE41-NEXT:    pcmpgtd %xmm1, %xmm8
-; SSE41-NEXT:    pxor %xmm8, %xmm4
-; SSE41-NEXT:    pcmpeqd %xmm2, %xmm4
-; SSE41-NEXT:    pandn %xmm0, %xmm4
-; SSE41-NEXT:    movdqa %xmm8, %xmm0
-; SSE41-NEXT:    blendvps %xmm0, %xmm9, %xmm5
+; SSE41-NEXT:    movdqa %xmm0, %xmm4
+; SSE41-NEXT:    movdqa %xmm0, %xmm5
+; SSE41-NEXT:    paddd %xmm2, %xmm5
+; SSE41-NEXT:    movaps {{.*#+}} xmm8 = [2147483647,2147483647,2147483647,2147483647]
+; SSE41-NEXT:    movaps {{.*#+}} xmm6 = [2147483648,2147483648,2147483648,2147483648]
+; SSE41-NEXT:    movaps %xmm6, %xmm7
+; SSE41-NEXT:    movdqa %xmm5, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm8, %xmm7
+; SSE41-NEXT:    pcmpgtd %xmm5, %xmm4
+; SSE41-NEXT:    pxor %xmm2, %xmm4
 ; SSE41-NEXT:    movdqa %xmm4, %xmm0
-; SSE41-NEXT:    blendvps %xmm0, %xmm5, %xmm1
-; SSE41-NEXT:    movaps %xmm6, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm7, %xmm5
+; SSE41-NEXT:    movdqa %xmm1, %xmm2
+; SSE41-NEXT:    paddd %xmm3, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm8, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm2, %xmm1
+; SSE41-NEXT:    pxor %xmm3, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm6, %xmm2
+; SSE41-NEXT:    movaps %xmm5, %xmm0
+; SSE41-NEXT:    movaps %xmm2, %xmm1
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: v8i32:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm3, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
-; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm6
-; AVX1-NEXT:    vpcmpgtd %xmm6, %xmm3, %xmm7
-; AVX1-NEXT:    vpxor %xmm5, %xmm7, %xmm7
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm7, %xmm8
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm3, %xmm4
-; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm9
-; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm3, %xmm4
-; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm9, %xmm4, %xmm9
-; AVX1-NEXT:    vinsertf128 $1, %xmm8, %ymm9, %ymm8
-; AVX1-NEXT:    vpaddd %xmm2, %xmm6, %xmm9
-; AVX1-NEXT:    vpcmpgtd %xmm9, %xmm3, %xmm6
-; AVX1-NEXT:    vpxor %xmm5, %xmm6, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm7, %xmm2
-; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm3, %xmm1
-; AVX1-NEXT:    vpxor %xmm5, %xmm1, %xmm3
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm4, %xmm3
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
-; AVX1-NEXT:    vandnps %ymm8, %ymm2, %ymm2
-; AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm1, %ymm1
-; AVX1-NEXT:    vmovaps {{.*#+}} ymm3 = [2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648]
-; AVX1-NEXT:    vblendvps %ymm1, {{.*}}(%rip), %ymm3, %ymm1
-; AVX1-NEXT:    vinsertf128 $1, %xmm9, %ymm0, %ymm0
-; AVX1-NEXT:    vblendvps %ymm2, %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; AVX1-NEXT:    vpaddd %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm4
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm4, %ymm5
+; AVX1-NEXT:    vmovaps {{.*#+}} ymm6 = [2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648]
+; AVX1-NEXT:    vblendvps %ymm5, {{.*}}(%rip), %ymm6, %ymm6
+; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vpcmpgtd %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    vxorps %ymm0, %ymm1, %ymm0
+; AVX1-NEXT:    vblendvps %ymm0, %ymm6, %ymm5, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: v8i32:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm2, %ymm3
-; AVX2-NEXT:    vpcmpeqd %ymm4, %ymm4, %ymm4
-; AVX2-NEXT:    vpxor %ymm4, %ymm3, %ymm3
-; AVX2-NEXT:    vpcmpgtd %ymm0, %ymm2, %ymm5
-; AVX2-NEXT:    vpxor %ymm4, %ymm5, %ymm5
-; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm5, %ymm3
-; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpcmpgtd %ymm0, %ymm2, %ymm1
-; AVX2-NEXT:    vpxor %ymm4, %ymm1, %ymm2
-; AVX2-NEXT:    vpcmpeqd %ymm2, %ymm5, %ymm2
-; AVX2-NEXT:    vpandn %ymm3, %ymm2, %ymm2
+; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm2
 ; AVX2-NEXT:    vbroadcastss {{.*#+}} ymm3 = [2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647]
 ; AVX2-NEXT:    vbroadcastss {{.*#+}} ymm4 = [2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648]
-; AVX2-NEXT:    vblendvps %ymm1, %ymm3, %ymm4, %ymm1
-; AVX2-NEXT:    vblendvps %ymm2, %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vblendvps %ymm2, %ymm3, %ymm4, %ymm3
+; AVX2-NEXT:    vpcmpgtd %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %ymm0, %ymm1, %ymm0
+; AVX2-NEXT:    vblendvps %ymm0, %ymm3, %ymm2, %ymm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: v8i32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpcmpnltd %ymm2, %ymm1, %k0
-; AVX512-NEXT:    vpcmpnltd %ymm2, %ymm0, %k1
-; AVX512-NEXT:    kxorw %k0, %k1, %k0
-; AVX512-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
-; AVX512-NEXT:    vpcmpnltd %ymm2, %ymm0, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
-; AVX512-NEXT:    kandnw %k1, %k0, %k1
-; AVX512-NEXT:    vpcmpgtd %ymm0, %ymm2, %k2
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648]
-; AVX512-NEXT:    vpbroadcastd {{.*}}(%rip), %ymm1 {%k2}
-; AVX512-NEXT:    vmovdqa32 %ymm1, %ymm0 {%k1}
+; AVX512-NEXT:    vpcmpgtd %ymm1, %ymm2, %k0
+; AVX512-NEXT:    vpaddd %ymm1, %ymm0, %ymm1
+; AVX512-NEXT:    vpcmpgtd %ymm1, %ymm0, %k1
+; AVX512-NEXT:    kxorw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpgtd %ymm1, %ymm2, %k2
+; AVX512-NEXT:    vpbroadcastd {{.*#+}} ymm0 = [2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648]
+; AVX512-NEXT:    vpbroadcastd {{.*}}(%rip), %ymm0 {%k2}
+; AVX512-NEXT:    vmovdqa32 %ymm0, %ymm1 {%k1}
+; AVX512-NEXT:    vmovdqa %ymm1, %ymm0
 ; AVX512-NEXT:    retq
   %z = call <8 x i32> @llvm.sadd.sat.v8i32(<8 x i32> %x, <8 x i32> %y)
   ret <8 x i32> %z
@@ -1251,378 +886,230 @@ define <8 x i32> @v8i32(<8 x i32> %x, <8 x i32> %y) nounwind {
 define <16 x i32> @v16i32(<16 x i32> %x, <16 x i32> %y) nounwind {
 ; SSE2-LABEL: v16i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa %xmm1, %xmm8
-; SSE2-NEXT:    movdqa %xmm0, %xmm11
-; SSE2-NEXT:    pxor %xmm10, %xmm10
-; SSE2-NEXT:    pxor %xmm12, %xmm12
-; SSE2-NEXT:    pcmpgtd %xmm4, %xmm12
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm9
-; SSE2-NEXT:    pxor %xmm9, %xmm12
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
-; SSE2-NEXT:    pxor %xmm9, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm12
-; SSE2-NEXT:    paddd %xmm4, %xmm11
-; SSE2-NEXT:    pxor %xmm4, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm11, %xmm4
-; SSE2-NEXT:    movdqa %xmm4, %xmm0
+; SSE2-NEXT:    pxor %xmm8, %xmm8
+; SSE2-NEXT:    pxor %xmm9, %xmm9
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm9
+; SSE2-NEXT:    paddd %xmm0, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm0
 ; SSE2-NEXT:    pxor %xmm9, %xmm0
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
-; SSE2-NEXT:    pandn %xmm12, %xmm0
-; SSE2-NEXT:    movdqa {{.*#+}} xmm12 = [2147483648,2147483648,2147483648,2147483648]
-; SSE2-NEXT:    movdqa %xmm4, %xmm1
-; SSE2-NEXT:    pandn %xmm12, %xmm1
-; SSE2-NEXT:    psrld $1, %xmm4
-; SSE2-NEXT:    por %xmm1, %xmm4
-; SSE2-NEXT:    pand %xmm0, %xmm4
-; SSE2-NEXT:    pandn %xmm11, %xmm0
-; SSE2-NEXT:    por %xmm4, %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [2147483648,2147483648,2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm4, %xmm10
+; SSE2-NEXT:    pandn %xmm9, %xmm10
 ; SSE2-NEXT:    pxor %xmm11, %xmm11
-; SSE2-NEXT:    pcmpgtd %xmm5, %xmm11
-; SSE2-NEXT:    pxor %xmm9, %xmm11
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm11
+; SSE2-NEXT:    psrld $1, %xmm11
+; SSE2-NEXT:    por %xmm10, %xmm11
+; SSE2-NEXT:    pand %xmm0, %xmm11
+; SSE2-NEXT:    pandn %xmm4, %xmm0
+; SSE2-NEXT:    por %xmm11, %xmm0
 ; SSE2-NEXT:    pxor %xmm4, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm8, %xmm4
-; SSE2-NEXT:    pxor %xmm9, %xmm4
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm11
-; SSE2-NEXT:    paddd %xmm5, %xmm8
-; SSE2-NEXT:    pxor %xmm5, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm8, %xmm5
-; SSE2-NEXT:    movdqa %xmm5, %xmm1
-; SSE2-NEXT:    pxor %xmm9, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm1
-; SSE2-NEXT:    pandn %xmm11, %xmm1
-; SSE2-NEXT:    movdqa %xmm5, %xmm4
-; SSE2-NEXT:    pandn %xmm12, %xmm4
-; SSE2-NEXT:    psrld $1, %xmm5
-; SSE2-NEXT:    por %xmm4, %xmm5
-; SSE2-NEXT:    pand %xmm1, %xmm5
-; SSE2-NEXT:    pandn %xmm8, %xmm1
-; SSE2-NEXT:    por %xmm5, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm5, %xmm4
+; SSE2-NEXT:    paddd %xmm1, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm5, %xmm1
+; SSE2-NEXT:    pxor %xmm4, %xmm1
+; SSE2-NEXT:    movdqa %xmm5, %xmm10
+; SSE2-NEXT:    pandn %xmm9, %xmm10
+; SSE2-NEXT:    pxor %xmm4, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm5, %xmm4
+; SSE2-NEXT:    psrld $1, %xmm4
+; SSE2-NEXT:    por %xmm10, %xmm4
+; SSE2-NEXT:    pand %xmm1, %xmm4
+; SSE2-NEXT:    pandn %xmm5, %xmm1
+; SSE2-NEXT:    por %xmm4, %xmm1
 ; SSE2-NEXT:    pxor %xmm4, %xmm4
 ; SSE2-NEXT:    pcmpgtd %xmm6, %xmm4
-; SSE2-NEXT:    pxor %xmm9, %xmm4
-; SSE2-NEXT:    pxor %xmm5, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm5
-; SSE2-NEXT:    pxor %xmm9, %xmm5
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm4
-; SSE2-NEXT:    paddd %xmm6, %xmm2
-; SSE2-NEXT:    pxor %xmm6, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm6
-; SSE2-NEXT:    movdqa %xmm6, %xmm8
-; SSE2-NEXT:    pxor %xmm9, %xmm8
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm8
-; SSE2-NEXT:    pandn %xmm4, %xmm8
+; SSE2-NEXT:    paddd %xmm2, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm6, %xmm2
+; SSE2-NEXT:    pxor %xmm4, %xmm2
 ; SSE2-NEXT:    movdqa %xmm6, %xmm4
-; SSE2-NEXT:    pandn %xmm12, %xmm4
-; SSE2-NEXT:    psrld $1, %xmm6
-; SSE2-NEXT:    por %xmm4, %xmm6
-; SSE2-NEXT:    pand %xmm8, %xmm6
-; SSE2-NEXT:    pandn %xmm2, %xmm8
-; SSE2-NEXT:    por %xmm6, %xmm8
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpgtd %xmm7, %xmm2
-; SSE2-NEXT:    pxor %xmm9, %xmm2
+; SSE2-NEXT:    pandn %xmm9, %xmm4
+; SSE2-NEXT:    pxor %xmm5, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm6, %xmm5
+; SSE2-NEXT:    psrld $1, %xmm5
+; SSE2-NEXT:    por %xmm4, %xmm5
+; SSE2-NEXT:    pand %xmm2, %xmm5
+; SSE2-NEXT:    pandn %xmm6, %xmm2
+; SSE2-NEXT:    por %xmm5, %xmm2
 ; SSE2-NEXT:    pxor %xmm4, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm4
-; SSE2-NEXT:    pxor %xmm9, %xmm4
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm2
-; SSE2-NEXT:    paddd %xmm7, %xmm3
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm10
-; SSE2-NEXT:    pxor %xmm10, %xmm9
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm9
-; SSE2-NEXT:    pandn %xmm2, %xmm9
-; SSE2-NEXT:    movdqa %xmm10, %xmm2
-; SSE2-NEXT:    pandn %xmm12, %xmm2
-; SSE2-NEXT:    psrld $1, %xmm10
-; SSE2-NEXT:    por %xmm2, %xmm10
-; SSE2-NEXT:    pand %xmm9, %xmm10
-; SSE2-NEXT:    pandn %xmm3, %xmm9
-; SSE2-NEXT:    por %xmm10, %xmm9
-; SSE2-NEXT:    movdqa %xmm8, %xmm2
-; SSE2-NEXT:    movdqa %xmm9, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm7, %xmm4
+; SSE2-NEXT:    paddd %xmm3, %xmm7
+; SSE2-NEXT:    pcmpgtd %xmm7, %xmm3
+; SSE2-NEXT:    pxor %xmm4, %xmm3
+; SSE2-NEXT:    movdqa %xmm7, %xmm4
+; SSE2-NEXT:    pandn %xmm9, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm7, %xmm8
+; SSE2-NEXT:    psrld $1, %xmm8
+; SSE2-NEXT:    por %xmm4, %xmm8
+; SSE2-NEXT:    pand %xmm3, %xmm8
+; SSE2-NEXT:    pandn %xmm7, %xmm3
+; SSE2-NEXT:    por %xmm8, %xmm3
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: v16i32:
 ; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    movdqa %xmm1, %xmm8
-; SSSE3-NEXT:    movdqa %xmm0, %xmm11
-; SSSE3-NEXT:    pxor %xmm10, %xmm10
-; SSSE3-NEXT:    pxor %xmm12, %xmm12
-; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm12
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm9
-; SSSE3-NEXT:    pxor %xmm9, %xmm12
-; SSSE3-NEXT:    pxor %xmm1, %xmm1
-; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm1
-; SSSE3-NEXT:    pxor %xmm9, %xmm1
-; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm12
-; SSSE3-NEXT:    paddd %xmm4, %xmm11
-; SSSE3-NEXT:    pxor %xmm4, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm11, %xmm4
-; SSSE3-NEXT:    movdqa %xmm4, %xmm0
+; SSSE3-NEXT:    pxor %xmm8, %xmm8
+; SSSE3-NEXT:    pxor %xmm9, %xmm9
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm9
+; SSSE3-NEXT:    paddd %xmm0, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm0
 ; SSSE3-NEXT:    pxor %xmm9, %xmm0
-; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm0
-; SSSE3-NEXT:    pandn %xmm12, %xmm0
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm12 = [2147483648,2147483648,2147483648,2147483648]
-; SSSE3-NEXT:    movdqa %xmm4, %xmm1
-; SSSE3-NEXT:    pandn %xmm12, %xmm1
-; SSSE3-NEXT:    psrld $1, %xmm4
-; SSSE3-NEXT:    por %xmm1, %xmm4
-; SSSE3-NEXT:    pand %xmm0, %xmm4
-; SSSE3-NEXT:    pandn %xmm11, %xmm0
-; SSSE3-NEXT:    por %xmm4, %xmm0
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm9 = [2147483648,2147483648,2147483648,2147483648]
+; SSSE3-NEXT:    movdqa %xmm4, %xmm10
+; SSSE3-NEXT:    pandn %xmm9, %xmm10
 ; SSSE3-NEXT:    pxor %xmm11, %xmm11
-; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm11
-; SSSE3-NEXT:    pxor %xmm9, %xmm11
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm11
+; SSSE3-NEXT:    psrld $1, %xmm11
+; SSSE3-NEXT:    por %xmm10, %xmm11
+; SSSE3-NEXT:    pand %xmm0, %xmm11
+; SSSE3-NEXT:    pandn %xmm4, %xmm0
+; SSSE3-NEXT:    por %xmm11, %xmm0
 ; SSSE3-NEXT:    pxor %xmm4, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm8, %xmm4
-; SSSE3-NEXT:    pxor %xmm9, %xmm4
-; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm11
-; SSSE3-NEXT:    paddd %xmm5, %xmm8
-; SSSE3-NEXT:    pxor %xmm5, %xmm5
-; SSSE3-NEXT:    pcmpgtd %xmm8, %xmm5
-; SSSE3-NEXT:    movdqa %xmm5, %xmm1
-; SSSE3-NEXT:    pxor %xmm9, %xmm1
-; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm1
-; SSSE3-NEXT:    pandn %xmm11, %xmm1
-; SSSE3-NEXT:    movdqa %xmm5, %xmm4
-; SSSE3-NEXT:    pandn %xmm12, %xmm4
-; SSSE3-NEXT:    psrld $1, %xmm5
-; SSSE3-NEXT:    por %xmm4, %xmm5
-; SSSE3-NEXT:    pand %xmm1, %xmm5
-; SSSE3-NEXT:    pandn %xmm8, %xmm1
-; SSSE3-NEXT:    por %xmm5, %xmm1
+; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm4
+; SSSE3-NEXT:    paddd %xmm1, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm1
+; SSSE3-NEXT:    pxor %xmm4, %xmm1
+; SSSE3-NEXT:    movdqa %xmm5, %xmm10
+; SSSE3-NEXT:    pandn %xmm9, %xmm10
+; SSSE3-NEXT:    pxor %xmm4, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm4
+; SSSE3-NEXT:    psrld $1, %xmm4
+; SSSE3-NEXT:    por %xmm10, %xmm4
+; SSSE3-NEXT:    pand %xmm1, %xmm4
+; SSSE3-NEXT:    pandn %xmm5, %xmm1
+; SSSE3-NEXT:    por %xmm4, %xmm1
 ; SSSE3-NEXT:    pxor %xmm4, %xmm4
 ; SSSE3-NEXT:    pcmpgtd %xmm6, %xmm4
-; SSSE3-NEXT:    pxor %xmm9, %xmm4
-; SSSE3-NEXT:    pxor %xmm5, %xmm5
-; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm5
-; SSSE3-NEXT:    pxor %xmm9, %xmm5
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm4
-; SSSE3-NEXT:    paddd %xmm6, %xmm2
-; SSSE3-NEXT:    pxor %xmm6, %xmm6
-; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm6
-; SSSE3-NEXT:    movdqa %xmm6, %xmm8
-; SSSE3-NEXT:    pxor %xmm9, %xmm8
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm8
-; SSSE3-NEXT:    pandn %xmm4, %xmm8
+; SSSE3-NEXT:    paddd %xmm2, %xmm6
+; SSSE3-NEXT:    pcmpgtd %xmm6, %xmm2
+; SSSE3-NEXT:    pxor %xmm4, %xmm2
 ; SSSE3-NEXT:    movdqa %xmm6, %xmm4
-; SSSE3-NEXT:    pandn %xmm12, %xmm4
-; SSSE3-NEXT:    psrld $1, %xmm6
-; SSSE3-NEXT:    por %xmm4, %xmm6
-; SSSE3-NEXT:    pand %xmm8, %xmm6
-; SSSE3-NEXT:    pandn %xmm2, %xmm8
-; SSSE3-NEXT:    por %xmm6, %xmm8
-; SSSE3-NEXT:    pxor %xmm2, %xmm2
-; SSSE3-NEXT:    pcmpgtd %xmm7, %xmm2
-; SSSE3-NEXT:    pxor %xmm9, %xmm2
+; SSSE3-NEXT:    pandn %xmm9, %xmm4
+; SSSE3-NEXT:    pxor %xmm5, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm6, %xmm5
+; SSSE3-NEXT:    psrld $1, %xmm5
+; SSSE3-NEXT:    por %xmm4, %xmm5
+; SSSE3-NEXT:    pand %xmm2, %xmm5
+; SSSE3-NEXT:    pandn %xmm6, %xmm2
+; SSSE3-NEXT:    por %xmm5, %xmm2
 ; SSSE3-NEXT:    pxor %xmm4, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm4
-; SSSE3-NEXT:    pxor %xmm9, %xmm4
-; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm2
-; SSSE3-NEXT:    paddd %xmm7, %xmm3
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm10
-; SSSE3-NEXT:    pxor %xmm10, %xmm9
-; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm9
-; SSSE3-NEXT:    pandn %xmm2, %xmm9
-; SSSE3-NEXT:    movdqa %xmm10, %xmm2
-; SSSE3-NEXT:    pandn %xmm12, %xmm2
-; SSSE3-NEXT:    psrld $1, %xmm10
-; SSSE3-NEXT:    por %xmm2, %xmm10
-; SSSE3-NEXT:    pand %xmm9, %xmm10
-; SSSE3-NEXT:    pandn %xmm3, %xmm9
-; SSSE3-NEXT:    por %xmm10, %xmm9
-; SSSE3-NEXT:    movdqa %xmm8, %xmm2
-; SSSE3-NEXT:    movdqa %xmm9, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm7, %xmm4
+; SSSE3-NEXT:    paddd %xmm3, %xmm7
+; SSSE3-NEXT:    pcmpgtd %xmm7, %xmm3
+; SSSE3-NEXT:    pxor %xmm4, %xmm3
+; SSSE3-NEXT:    movdqa %xmm7, %xmm4
+; SSSE3-NEXT:    pandn %xmm9, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm7, %xmm8
+; SSSE3-NEXT:    psrld $1, %xmm8
+; SSSE3-NEXT:    por %xmm4, %xmm8
+; SSSE3-NEXT:    pand %xmm3, %xmm8
+; SSSE3-NEXT:    pandn %xmm7, %xmm3
+; SSSE3-NEXT:    por %xmm8, %xmm3
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: v16i32:
 ; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm3, %xmm8
+; SSE41-NEXT:    movdqa %xmm2, %xmm12
+; SSE41-NEXT:    movdqa %xmm0, %xmm3
 ; SSE41-NEXT:    movdqa %xmm0, %xmm9
-; SSE41-NEXT:    pxor %xmm8, %xmm8
-; SSE41-NEXT:    pxor %xmm11, %xmm11
-; SSE41-NEXT:    pcmpgtd %xmm4, %xmm11
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm10
-; SSE41-NEXT:    pxor %xmm10, %xmm11
-; SSE41-NEXT:    pxor %xmm12, %xmm12
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm12
-; SSE41-NEXT:    pxor %xmm10, %xmm12
-; SSE41-NEXT:    pcmpeqd %xmm12, %xmm11
 ; SSE41-NEXT:    paddd %xmm4, %xmm9
-; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    pcmpgtd %xmm9, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm4
-; SSE41-NEXT:    pxor %xmm10, %xmm4
-; SSE41-NEXT:    pcmpeqd %xmm12, %xmm4
-; SSE41-NEXT:    pandn %xmm11, %xmm4
-; SSE41-NEXT:    movaps {{.*#+}} xmm12 = [2147483647,2147483647,2147483647,2147483647]
-; SSE41-NEXT:    movaps {{.*#+}} xmm11 = [2147483648,2147483648,2147483648,2147483648]
-; SSE41-NEXT:    movaps %xmm11, %xmm13
-; SSE41-NEXT:    blendvps %xmm0, %xmm12, %xmm13
-; SSE41-NEXT:    movdqa %xmm4, %xmm0
-; SSE41-NEXT:    blendvps %xmm0, %xmm13, %xmm9
-; SSE41-NEXT:    xorps %xmm13, %xmm13
-; SSE41-NEXT:    pcmpgtd %xmm5, %xmm13
-; SSE41-NEXT:    pxor %xmm10, %xmm13
-; SSE41-NEXT:    pxor %xmm14, %xmm14
-; SSE41-NEXT:    pcmpgtd %xmm1, %xmm14
-; SSE41-NEXT:    pxor %xmm10, %xmm14
-; SSE41-NEXT:    pcmpeqd %xmm14, %xmm13
-; SSE41-NEXT:    paddd %xmm5, %xmm1
-; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm4
-; SSE41-NEXT:    pxor %xmm10, %xmm4
-; SSE41-NEXT:    pcmpeqd %xmm14, %xmm4
-; SSE41-NEXT:    pandn %xmm13, %xmm4
-; SSE41-NEXT:    movaps %xmm11, %xmm5
-; SSE41-NEXT:    blendvps %xmm0, %xmm12, %xmm5
-; SSE41-NEXT:    movdqa %xmm4, %xmm0
-; SSE41-NEXT:    blendvps %xmm0, %xmm5, %xmm1
-; SSE41-NEXT:    pxor %xmm13, %xmm13
-; SSE41-NEXT:    pcmpgtd %xmm6, %xmm13
-; SSE41-NEXT:    pxor %xmm10, %xmm13
-; SSE41-NEXT:    xorps %xmm5, %xmm5
-; SSE41-NEXT:    pcmpgtd %xmm2, %xmm5
-; SSE41-NEXT:    pxor %xmm10, %xmm5
-; SSE41-NEXT:    pcmpeqd %xmm5, %xmm13
-; SSE41-NEXT:    paddd %xmm6, %xmm2
-; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    pcmpgtd %xmm2, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm4
-; SSE41-NEXT:    pxor %xmm10, %xmm4
-; SSE41-NEXT:    pcmpeqd %xmm5, %xmm4
-; SSE41-NEXT:    pandn %xmm13, %xmm4
-; SSE41-NEXT:    movaps %xmm11, %xmm5
-; SSE41-NEXT:    blendvps %xmm0, %xmm12, %xmm5
+; SSE41-NEXT:    movaps {{.*#+}} xmm11 = [2147483647,2147483647,2147483647,2147483647]
+; SSE41-NEXT:    movaps {{.*#+}} xmm10 = [2147483648,2147483648,2147483648,2147483648]
+; SSE41-NEXT:    movaps %xmm10, %xmm2
+; SSE41-NEXT:    movdqa %xmm9, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm11, %xmm2
+; SSE41-NEXT:    pcmpgtd %xmm9, %xmm3
+; SSE41-NEXT:    pxor %xmm4, %xmm3
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm2, %xmm9
+; SSE41-NEXT:    movdqa %xmm1, %xmm4
+; SSE41-NEXT:    paddd %xmm5, %xmm4
+; SSE41-NEXT:    movaps %xmm10, %xmm2
 ; SSE41-NEXT:    movdqa %xmm4, %xmm0
-; SSE41-NEXT:    blendvps %xmm0, %xmm5, %xmm2
-; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    pcmpgtd %xmm7, %xmm0
-; SSE41-NEXT:    pxor %xmm10, %xmm0
-; SSE41-NEXT:    pxor %xmm4, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm4
-; SSE41-NEXT:    pxor %xmm10, %xmm4
-; SSE41-NEXT:    pcmpeqd %xmm4, %xmm0
-; SSE41-NEXT:    paddd %xmm7, %xmm3
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm8
-; SSE41-NEXT:    pxor %xmm8, %xmm10
-; SSE41-NEXT:    pcmpeqd %xmm4, %xmm10
-; SSE41-NEXT:    pandn %xmm0, %xmm10
+; SSE41-NEXT:    blendvps %xmm0, %xmm11, %xmm2
+; SSE41-NEXT:    pcmpgtd %xmm4, %xmm1
+; SSE41-NEXT:    pxor %xmm5, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm2, %xmm4
+; SSE41-NEXT:    movdqa %xmm12, %xmm3
+; SSE41-NEXT:    paddd %xmm6, %xmm3
+; SSE41-NEXT:    movaps %xmm10, %xmm1
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm11, %xmm1
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm12
+; SSE41-NEXT:    pxor %xmm6, %xmm12
+; SSE41-NEXT:    movdqa %xmm12, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm1, %xmm3
+; SSE41-NEXT:    movdqa %xmm8, %xmm5
+; SSE41-NEXT:    paddd %xmm7, %xmm5
+; SSE41-NEXT:    movdqa %xmm5, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm11, %xmm10
+; SSE41-NEXT:    pcmpgtd %xmm5, %xmm8
+; SSE41-NEXT:    pxor %xmm7, %xmm8
 ; SSE41-NEXT:    movdqa %xmm8, %xmm0
-; SSE41-NEXT:    blendvps %xmm0, %xmm12, %xmm11
-; SSE41-NEXT:    movdqa %xmm10, %xmm0
-; SSE41-NEXT:    blendvps %xmm0, %xmm11, %xmm3
+; SSE41-NEXT:    blendvps %xmm0, %xmm10, %xmm5
 ; SSE41-NEXT:    movaps %xmm9, %xmm0
+; SSE41-NEXT:    movaps %xmm4, %xmm1
+; SSE41-NEXT:    movaps %xmm3, %xmm2
+; SSE41-NEXT:    movaps %xmm5, %xmm3
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: v16i32:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm9
-; AVX1-NEXT:    vpxor %xmm12, %xmm12, %xmm12
-; AVX1-NEXT:    vpcmpgtd %xmm9, %xmm12, %xmm7
-; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
-; AVX1-NEXT:    vpxor %xmm5, %xmm7, %xmm8
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm7
-; AVX1-NEXT:    vpcmpgtd %xmm7, %xmm12, %xmm6
-; AVX1-NEXT:    vpxor %xmm5, %xmm6, %xmm10
-; AVX1-NEXT:    vpcmpeqd %xmm8, %xmm10, %xmm8
-; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm12, %xmm6
-; AVX1-NEXT:    vpxor %xmm5, %xmm6, %xmm11
-; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm12, %xmm6
-; AVX1-NEXT:    vpxor %xmm5, %xmm6, %xmm6
-; AVX1-NEXT:    vpcmpeqd %xmm11, %xmm6, %xmm11
-; AVX1-NEXT:    vinsertf128 $1, %xmm8, %ymm11, %ymm8
-; AVX1-NEXT:    vpaddd %xmm9, %xmm7, %xmm9
-; AVX1-NEXT:    vpcmpgtd %xmm9, %xmm12, %xmm7
-; AVX1-NEXT:    vpxor %xmm5, %xmm7, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm10, %xmm10
-; AVX1-NEXT:    vpaddd %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm12, %xmm2
-; AVX1-NEXT:    vpxor %xmm5, %xmm2, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm6, %xmm4
-; AVX1-NEXT:    vinsertf128 $1, %xmm10, %ymm4, %ymm4
-; AVX1-NEXT:    vandnps %ymm8, %ymm4, %ymm4
-; AVX1-NEXT:    vinsertf128 $1, %xmm7, %ymm2, %ymm7
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm4
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm5
+; AVX1-NEXT:    vpaddd %xmm4, %xmm5, %xmm4
+; AVX1-NEXT:    vpaddd %xmm2, %xmm0, %xmm6
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm6, %ymm7
 ; AVX1-NEXT:    vmovaps {{.*#+}} ymm8 = [2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647]
-; AVX1-NEXT:    vmovaps {{.*#+}} ymm10 = [2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648]
-; AVX1-NEXT:    vblendvps %ymm7, %ymm8, %ymm10, %ymm7
-; AVX1-NEXT:    vinsertf128 $1, %xmm9, %ymm0, %ymm0
-; AVX1-NEXT:    vblendvps %ymm4, %ymm7, %ymm0, %ymm0
-; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm4
-; AVX1-NEXT:    vpcmpgtd %xmm4, %xmm12, %xmm7
-; AVX1-NEXT:    vpxor %xmm5, %xmm7, %xmm7
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm12, %xmm6
-; AVX1-NEXT:    vpxor %xmm5, %xmm6, %xmm6
-; AVX1-NEXT:    vpcmpeqd %xmm7, %xmm6, %xmm9
-; AVX1-NEXT:    vpcmpgtd %xmm3, %xmm12, %xmm7
-; AVX1-NEXT:    vpxor %xmm5, %xmm7, %xmm11
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm12, %xmm7
-; AVX1-NEXT:    vpxor %xmm5, %xmm7, %xmm7
-; AVX1-NEXT:    vpcmpeqd %xmm11, %xmm7, %xmm11
-; AVX1-NEXT:    vinsertf128 $1, %xmm9, %ymm11, %ymm9
-; AVX1-NEXT:    vpaddd %xmm4, %xmm2, %xmm11
-; AVX1-NEXT:    vpcmpgtd %xmm11, %xmm12, %xmm4
-; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm6, %xmm2
-; AVX1-NEXT:    vpaddd %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm12, %xmm3
-; AVX1-NEXT:    vpxor %xmm5, %xmm3, %xmm5
-; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm7, %xmm5
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm5, %ymm2
-; AVX1-NEXT:    vandnps %ymm9, %ymm2, %ymm2
-; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
-; AVX1-NEXT:    vblendvps %ymm3, %ymm8, %ymm10, %ymm3
-; AVX1-NEXT:    vinsertf128 $1, %xmm11, %ymm1, %ymm1
-; AVX1-NEXT:    vblendvps %ymm2, %ymm3, %ymm1, %ymm1
+; AVX1-NEXT:    vmovaps {{.*#+}} ymm9 = [2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648]
+; AVX1-NEXT:    vblendvps %ymm7, %ymm8, %ymm9, %ymm10
+; AVX1-NEXT:    vpcmpgtd %xmm4, %xmm5, %xmm4
+; AVX1-NEXT:    vpcmpgtd %xmm6, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
+; AVX1-NEXT:    vxorps %ymm0, %ymm2, %ymm0
+; AVX1-NEXT:    vblendvps %ymm0, %ymm10, %ymm7, %ymm0
+; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm4
+; AVX1-NEXT:    vpaddd %xmm2, %xmm4, %xmm2
+; AVX1-NEXT:    vpaddd %xmm3, %xmm1, %xmm5
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm5, %ymm6
+; AVX1-NEXT:    vblendvps %ymm6, %ymm8, %ymm9, %ymm7
+; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm4, %xmm2
+; AVX1-NEXT:    vpcmpgtd %xmm5, %xmm1, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; AVX1-NEXT:    vxorps %ymm1, %ymm3, %ymm1
+; AVX1-NEXT:    vblendvps %ymm1, %ymm7, %ymm6, %ymm1
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: v16i32:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpcmpgtd %ymm2, %ymm4, %ymm5
-; AVX2-NEXT:    vpcmpeqd %ymm6, %ymm6, %ymm6
-; AVX2-NEXT:    vpxor %ymm6, %ymm5, %ymm5
-; AVX2-NEXT:    vpcmpgtd %ymm0, %ymm4, %ymm7
-; AVX2-NEXT:    vpxor %ymm6, %ymm7, %ymm7
-; AVX2-NEXT:    vpcmpeqd %ymm5, %ymm7, %ymm5
-; AVX2-NEXT:    vpaddd %ymm2, %ymm0, %ymm0
-; AVX2-NEXT:    vpcmpgtd %ymm0, %ymm4, %ymm2
-; AVX2-NEXT:    vpxor %ymm6, %ymm2, %ymm8
-; AVX2-NEXT:    vpcmpeqd %ymm8, %ymm7, %ymm7
-; AVX2-NEXT:    vpandn %ymm5, %ymm7, %ymm5
-; AVX2-NEXT:    vbroadcastss {{.*#+}} ymm7 = [2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647]
-; AVX2-NEXT:    vbroadcastss {{.*#+}} ymm8 = [2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648]
-; AVX2-NEXT:    vblendvps %ymm2, %ymm7, %ymm8, %ymm2
-; AVX2-NEXT:    vblendvps %ymm5, %ymm2, %ymm0, %ymm0
-; AVX2-NEXT:    vpcmpgtd %ymm3, %ymm4, %ymm2
-; AVX2-NEXT:    vpxor %ymm6, %ymm2, %ymm2
-; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm4, %ymm5
-; AVX2-NEXT:    vpxor %ymm6, %ymm5, %ymm5
-; AVX2-NEXT:    vpcmpeqd %ymm2, %ymm5, %ymm2
-; AVX2-NEXT:    vpaddd %ymm3, %ymm1, %ymm1
-; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm4, %ymm3
-; AVX2-NEXT:    vpxor %ymm6, %ymm3, %ymm4
-; AVX2-NEXT:    vpcmpeqd %ymm4, %ymm5, %ymm4
-; AVX2-NEXT:    vpandn %ymm2, %ymm4, %ymm2
-; AVX2-NEXT:    vblendvps %ymm3, %ymm7, %ymm8, %ymm3
-; AVX2-NEXT:    vblendvps %ymm2, %ymm3, %ymm1, %ymm1
+; AVX2-NEXT:    vpaddd %ymm2, %ymm0, %ymm4
+; AVX2-NEXT:    vbroadcastss {{.*#+}} ymm5 = [2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647]
+; AVX2-NEXT:    vbroadcastss {{.*#+}} ymm6 = [2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648]
+; AVX2-NEXT:    vblendvps %ymm4, %ymm5, %ymm6, %ymm7
+; AVX2-NEXT:    vpcmpgtd %ymm4, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %ymm0, %ymm2, %ymm0
+; AVX2-NEXT:    vblendvps %ymm0, %ymm7, %ymm4, %ymm0
+; AVX2-NEXT:    vpaddd %ymm3, %ymm1, %ymm2
+; AVX2-NEXT:    vblendvps %ymm2, %ymm5, %ymm6, %ymm4
+; AVX2-NEXT:    vpcmpgtd %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    vpxor %ymm1, %ymm3, %ymm1
+; AVX2-NEXT:    vblendvps %ymm1, %ymm4, %ymm2, %ymm1
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: v16i32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpcmpnltd %zmm2, %zmm1, %k0
-; AVX512-NEXT:    vpcmpnltd %zmm2, %zmm0, %k1
-; AVX512-NEXT:    kxorw %k0, %k1, %k0
-; AVX512-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
-; AVX512-NEXT:    vpcmpnltd %zmm2, %zmm0, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
-; AVX512-NEXT:    kandnw %k1, %k0, %k1
-; AVX512-NEXT:    vpcmpgtd %zmm0, %zmm2, %k2
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} zmm1 = [2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648]
-; AVX512-NEXT:    vpbroadcastd {{.*}}(%rip), %zmm1 {%k2}
-; AVX512-NEXT:    vmovdqa32 %zmm1, %zmm0 {%k1}
+; AVX512-NEXT:    vpcmpgtd %zmm1, %zmm2, %k0
+; AVX512-NEXT:    vpaddd %zmm1, %zmm0, %zmm1
+; AVX512-NEXT:    vpcmpgtd %zmm1, %zmm0, %k1
+; AVX512-NEXT:    kxorw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpgtd %zmm1, %zmm2, %k2
+; AVX512-NEXT:    vpbroadcastd {{.*#+}} zmm0 = [2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648]
+; AVX512-NEXT:    vpbroadcastd {{.*}}(%rip), %zmm0 {%k2}
+; AVX512-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
+; AVX512-NEXT:    vmovdqa64 %zmm1, %zmm0
 ; AVX512-NEXT:    retq
   %z = call <16 x i32> @llvm.sadd.sat.v16i32(<16 x i32> %x, <16 x i32> %y)
   ret <16 x i32> %z
@@ -1633,210 +1120,119 @@ define <2 x i64> @v2i64(<2 x i64> %x, <2 x i64> %y) nounwind {
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm3
+; SSE2-NEXT:    pxor %xmm2, %xmm3
 ; SSE2-NEXT:    paddq %xmm1, %xmm0
-; SSE2-NEXT:    pxor %xmm2, %xmm1
-; SSE2-NEXT:    movdqa %xmm2, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm4
+; SSE2-NEXT:    pxor %xmm0, %xmm2
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm4
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pand %xmm5, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm4
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm5
-; SSE2-NEXT:    pxor %xmm5, %xmm4
-; SSE2-NEXT:    pxor %xmm2, %xmm3
-; SSE2-NEXT:    movdqa %xmm2, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm1
-; SSE2-NEXT:    pxor %xmm5, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,0,3,2]
-; SSE2-NEXT:    pand %xmm4, %xmm3
-; SSE2-NEXT:    movdqa %xmm0, %xmm4
-; SSE2-NEXT:    pxor %xmm2, %xmm4
-; SSE2-NEXT:    movdqa %xmm2, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm4, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3]
-; SSE2-NEXT:    pand %xmm7, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[1,1,3,3]
-; SSE2-NEXT:    por %xmm2, %xmm4
-; SSE2-NEXT:    pxor %xmm4, %xmm5
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm5[1,0,3,2]
-; SSE2-NEXT:    pand %xmm5, %xmm1
-; SSE2-NEXT:    pandn %xmm3, %xmm1
-; SSE2-NEXT:    movdqa %xmm4, %xmm2
-; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm2
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm4
-; SSE2-NEXT:    por %xmm2, %xmm4
-; SSE2-NEXT:    pand %xmm1, %xmm4
-; SSE2-NEXT:    pandn %xmm0, %xmm1
-; SSE2-NEXT:    por %xmm4, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    pcmpeqd %xmm3, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE2-NEXT:    pand %xmm5, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm2, %xmm3
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    pxor %xmm4, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,1,3,3]
+; SSE2-NEXT:    pxor %xmm3, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm3
+; SSE2-NEXT:    pandn %xmm0, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm2
+; SSE2-NEXT:    por %xmm2, %xmm0
+; SSE2-NEXT:    pand %xmm1, %xmm0
+; SSE2-NEXT:    por %xmm3, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: v2i64:
 ; SSSE3:       # %bb.0:
 ; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
 ; SSSE3-NEXT:    movdqa %xmm0, %xmm3
+; SSSE3-NEXT:    pxor %xmm2, %xmm3
 ; SSSE3-NEXT:    paddq %xmm1, %xmm0
-; SSSE3-NEXT:    pxor %xmm2, %xmm1
-; SSSE3-NEXT:    movdqa %xmm2, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm4
+; SSSE3-NEXT:    pxor %xmm0, %xmm2
+; SSSE3-NEXT:    movdqa %xmm3, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm4
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm5, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    por %xmm1, %xmm4
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm5
-; SSSE3-NEXT:    pxor %xmm5, %xmm4
-; SSSE3-NEXT:    pxor %xmm2, %xmm3
-; SSSE3-NEXT:    movdqa %xmm2, %xmm1
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm6, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    por %xmm3, %xmm1
-; SSSE3-NEXT:    pxor %xmm5, %xmm1
-; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm4, %xmm3
-; SSSE3-NEXT:    movdqa %xmm0, %xmm4
-; SSSE3-NEXT:    pxor %xmm2, %xmm4
-; SSSE3-NEXT:    movdqa %xmm2, %xmm6
-; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm7, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm4
-; SSSE3-NEXT:    pxor %xmm4, %xmm5
-; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm5
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm5[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm5, %xmm1
-; SSSE3-NEXT:    pandn %xmm3, %xmm1
-; SSSE3-NEXT:    movdqa %xmm4, %xmm2
-; SSSE3-NEXT:    pandn {{.*}}(%rip), %xmm2
-; SSSE3-NEXT:    pand {{.*}}(%rip), %xmm4
-; SSSE3-NEXT:    por %xmm2, %xmm4
-; SSSE3-NEXT:    pand %xmm1, %xmm4
-; SSSE3-NEXT:    pandn %xmm0, %xmm1
-; SSSE3-NEXT:    por %xmm4, %xmm1
-; SSSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm5, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm2, %xmm3
+; SSSE3-NEXT:    pxor %xmm2, %xmm2
+; SSSE3-NEXT:    pxor %xmm4, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    pxor %xmm3, %xmm1
+; SSSE3-NEXT:    movdqa %xmm1, %xmm3
+; SSSE3-NEXT:    pandn %xmm0, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSSE3-NEXT:    pandn {{.*}}(%rip), %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    pand {{.*}}(%rip), %xmm2
+; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    pand %xmm1, %xmm0
+; SSSE3-NEXT:    por %xmm3, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: v2i64:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movdqa %xmm0, %xmm2
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483648,2147483648]
-; SSE41-NEXT:    movdqa %xmm2, %xmm3
+; SSE41-NEXT:    movdqa %xmm2, %xmm4
+; SSE41-NEXT:    pxor %xmm0, %xmm4
 ; SSE41-NEXT:    paddq %xmm1, %xmm2
-; SSE41-NEXT:    pxor %xmm0, %xmm1
-; SSE41-NEXT:    movdqa %xmm0, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm1, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE41-NEXT:    por %xmm1, %xmm4
-; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE41-NEXT:    pxor %xmm1, %xmm4
-; SSE41-NEXT:    pxor %xmm0, %xmm3
-; SSE41-NEXT:    movdqa %xmm0, %xmm5
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pand %xmm6, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE41-NEXT:    por %xmm3, %xmm5
-; SSE41-NEXT:    pxor %xmm1, %xmm5
-; SSE41-NEXT:    pcmpeqq %xmm5, %xmm4
-; SSE41-NEXT:    movdqa %xmm2, %xmm3
-; SSE41-NEXT:    pxor %xmm0, %xmm3
-; SSE41-NEXT:    movdqa %xmm0, %xmm6
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pand %xmm7, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3]
-; SSE41-NEXT:    por %xmm3, %xmm0
-; SSE41-NEXT:    pxor %xmm0, %xmm1
-; SSE41-NEXT:    pcmpeqq %xmm5, %xmm1
-; SSE41-NEXT:    pandn %xmm4, %xmm1
-; SSE41-NEXT:    movapd {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808]
-; SSE41-NEXT:    blendvpd %xmm0, {{.*}}(%rip), %xmm3
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm2
+; SSE41-NEXT:    pxor %xmm2, %xmm0
+; SSE41-NEXT:    movdqa %xmm4, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm3
+; SSE41-NEXT:    por %xmm4, %xmm3
+; SSE41-NEXT:    pxor %xmm1, %xmm3
+; SSE41-NEXT:    movapd {{.*#+}} xmm1 = [9223372036854775808,9223372036854775808]
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, {{.*}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
 ; SSE41-NEXT:    movapd %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: v2i64:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm2, %xmm3
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm5
-; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX1-NEXT:    vpcmpeqq %xmm3, %xmm5, %xmm3
-; AVX1-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm4, %xmm1, %xmm2
-; AVX1-NEXT:    vpcmpeqq %xmm2, %xmm5, %xmm2
-; AVX1-NEXT:    vpandn %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpaddq %xmm1, %xmm0, %xmm2
 ; AVX1-NEXT:    vmovapd {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808]
-; AVX1-NEXT:    vblendvpd %xmm1, {{.*}}(%rip), %xmm3, %xmm1
-; AVX1-NEXT:    vblendvpd %xmm2, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vblendvpd %xmm2, {{.*}}(%rip), %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vblendvpd %xmm0, %xmm3, %xmm2, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: v2i64:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtq %xmm1, %xmm2, %xmm3
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpxor %xmm4, %xmm3, %xmm3
-; AVX2-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm5
-; AVX2-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX2-NEXT:    vpcmpeqq %xmm3, %xmm5, %xmm3
-; AVX2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm1
-; AVX2-NEXT:    vpxor %xmm4, %xmm1, %xmm2
-; AVX2-NEXT:    vpcmpeqq %xmm2, %xmm5, %xmm2
-; AVX2-NEXT:    vpandn %xmm3, %xmm2, %xmm2
+; AVX2-NEXT:    vpaddq %xmm1, %xmm0, %xmm2
 ; AVX2-NEXT:    vmovapd {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808]
-; AVX2-NEXT:    vblendvpd %xmm1, {{.*}}(%rip), %xmm3, %xmm1
-; AVX2-NEXT:    vblendvpd %xmm2, %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vblendvpd %xmm2, {{.*}}(%rip), %xmm3, %xmm3
+; AVX2-NEXT:    vpcmpgtq %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor %xmm0, %xmm1, %xmm0
+; AVX2-NEXT:    vblendvpd %xmm0, %xmm3, %xmm2, %xmm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: v2i64:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpcmpnltq %xmm2, %xmm1, %k0
-; AVX512-NEXT:    vpcmpnltq %xmm2, %xmm0, %k1
-; AVX512-NEXT:    kxorw %k0, %k1, %k0
-; AVX512-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpcmpnltq %xmm2, %xmm0, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
-; AVX512-NEXT:    kandnw %k1, %k0, %k1
-; AVX512-NEXT:    vpcmpgtq %xmm0, %xmm2, %k2
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = [9223372036854775808,9223372036854775808]
-; AVX512-NEXT:    vmovdqa64 {{.*}}(%rip), %xmm1 {%k2}
-; AVX512-NEXT:    vmovdqa64 %xmm1, %xmm0 {%k1}
+; AVX512-NEXT:    vpcmpgtq %xmm1, %xmm2, %k0
+; AVX512-NEXT:    vpaddq %xmm1, %xmm0, %xmm1
+; AVX512-NEXT:    vpcmpgtq %xmm1, %xmm0, %k1
+; AVX512-NEXT:    kxorw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpgtq %xmm1, %xmm2, %k2
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm0 = [9223372036854775808,9223372036854775808]
+; AVX512-NEXT:    vmovdqa64 {{.*}}(%rip), %xmm0 {%k2}
+; AVX512-NEXT:    vmovdqa64 %xmm0, %xmm1 {%k1}
+; AVX512-NEXT:    vmovdqa %xmm1, %xmm0
 ; AVX512-NEXT:    retq
   %z = call <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64> %x, <2 x i64> %y)
   ret <2 x i64> %z
@@ -1845,369 +1241,203 @@ define <2 x i64> @v2i64(<2 x i64> %x, <2 x i64> %y) nounwind {
 define <4 x i64> @v4i64(<4 x i64> %x, <4 x i64> %y) nounwind {
 ; SSE2-LABEL: v4i64:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa %xmm0, %xmm9
-; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [2147483648,2147483648]
-; SSE2-NEXT:    paddq %xmm2, %xmm9
-; SSE2-NEXT:    pxor %xmm10, %xmm2
-; SSE2-NEXT:    movdqa %xmm10, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm10, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT:    pand %xmm7, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[1,1,3,3]
-; SSE2-NEXT:    por %xmm2, %xmm7
-; SSE2-NEXT:    pcmpeqd %xmm6, %xmm6
-; SSE2-NEXT:    pxor %xmm6, %xmm7
-; SSE2-NEXT:    pxor %xmm10, %xmm0
-; SSE2-NEXT:    movdqa %xmm10, %xmm2
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm10, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pand %xmm4, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT:    por %xmm0, %xmm2
-; SSE2-NEXT:    pxor %xmm6, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm7[1,0,3,2]
-; SSE2-NEXT:    pand %xmm7, %xmm4
-; SSE2-NEXT:    movdqa %xmm9, %xmm0
-; SSE2-NEXT:    pxor %xmm10, %xmm0
-; SSE2-NEXT:    movdqa %xmm10, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm7[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm10, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pand %xmm5, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm7[1,1,3,3]
-; SSE2-NEXT:    por %xmm0, %xmm5
+; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm0, %xmm5
+; SSE2-NEXT:    pxor %xmm4, %xmm5
+; SSE2-NEXT:    paddq %xmm2, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm6
+; SSE2-NEXT:    pxor %xmm4, %xmm6
 ; SSE2-NEXT:    movdqa %xmm5, %xmm7
+; SSE2-NEXT:    pcmpgtd %xmm6, %xmm7
+; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm6[1,1,3,3]
+; SSE2-NEXT:    pand %xmm8, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm7[1,1,3,3]
+; SSE2-NEXT:    por %xmm5, %xmm6
+; SSE2-NEXT:    pxor %xmm5, %xmm5
+; SSE2-NEXT:    pxor %xmm7, %xmm7
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm7
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
 ; SSE2-NEXT:    pxor %xmm6, %xmm7
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[1,0,3,2]
+; SSE2-NEXT:    movdqa %xmm7, %xmm2
+; SSE2-NEXT:    pandn %xmm0, %xmm2
+; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [9223372036854775808,9223372036854775808]
+; SSE2-NEXT:    pxor %xmm6, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm6
+; SSE2-NEXT:    pandn %xmm8, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [9223372036854775807,9223372036854775807]
+; SSE2-NEXT:    pand %xmm9, %xmm6
+; SSE2-NEXT:    por %xmm6, %xmm0
 ; SSE2-NEXT:    pand %xmm7, %xmm0
-; SSE2-NEXT:    pandn %xmm4, %xmm0
-; SSE2-NEXT:    movdqa {{.*#+}} xmm11 = [9223372036854775808,9223372036854775808]
-; SSE2-NEXT:    movdqa %xmm5, %xmm2
-; SSE2-NEXT:    pandn %xmm11, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [9223372036854775807,9223372036854775807]
-; SSE2-NEXT:    pand %xmm8, %xmm5
-; SSE2-NEXT:    por %xmm2, %xmm5
-; SSE2-NEXT:    pand %xmm0, %xmm5
-; SSE2-NEXT:    pandn %xmm9, %xmm0
-; SSE2-NEXT:    por %xmm5, %xmm0
+; SSE2-NEXT:    por %xmm2, %xmm0
 ; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    pxor %xmm4, %xmm2
 ; SSE2-NEXT:    paddq %xmm3, %xmm1
-; SSE2-NEXT:    pxor %xmm10, %xmm3
-; SSE2-NEXT:    movdqa %xmm10, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm10, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE2-NEXT:    pand %xmm5, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm4
-; SSE2-NEXT:    pxor %xmm6, %xmm4
-; SSE2-NEXT:    pxor %xmm10, %xmm2
-; SSE2-NEXT:    movdqa %xmm10, %xmm3
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm10, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT:    pand %xmm5, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE2-NEXT:    por %xmm2, %xmm3
-; SSE2-NEXT:    pxor %xmm6, %xmm3
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[1,0,3,2]
-; SSE2-NEXT:    pand %xmm4, %xmm5
-; SSE2-NEXT:    movdqa %xmm1, %xmm2
-; SSE2-NEXT:    pxor %xmm10, %xmm2
-; SSE2-NEXT:    movdqa %xmm10, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm10, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE2-NEXT:    pxor %xmm1, %xmm4
+; SSE2-NEXT:    movdqa %xmm2, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm7, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[1,1,3,3]
 ; SSE2-NEXT:    por %xmm2, %xmm4
-; SSE2-NEXT:    pxor %xmm4, %xmm6
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[1,0,3,2]
-; SSE2-NEXT:    pand %xmm6, %xmm2
-; SSE2-NEXT:    pandn %xmm5, %xmm2
-; SSE2-NEXT:    movdqa %xmm4, %xmm3
-; SSE2-NEXT:    pandn %xmm11, %xmm3
-; SSE2-NEXT:    pand %xmm8, %xmm4
-; SSE2-NEXT:    por %xmm3, %xmm4
-; SSE2-NEXT:    pand %xmm2, %xmm4
-; SSE2-NEXT:    pandn %xmm1, %xmm2
-; SSE2-NEXT:    por %xmm4, %xmm2
-; SSE2-NEXT:    movdqa %xmm2, %xmm1
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE2-NEXT:    pxor %xmm4, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm3
+; SSE2-NEXT:    pandn %xmm1, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm5
+; SSE2-NEXT:    pandn %xmm8, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm5[1,1,3,3]
+; SSE2-NEXT:    pand %xmm9, %xmm4
+; SSE2-NEXT:    por %xmm4, %xmm1
+; SSE2-NEXT:    pand %xmm2, %xmm1
+; SSE2-NEXT:    por %xmm3, %xmm1
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: v4i64:
 ; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    movdqa %xmm0, %xmm9
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm10 = [2147483648,2147483648]
-; SSSE3-NEXT:    paddq %xmm2, %xmm9
-; SSSE3-NEXT:    pxor %xmm10, %xmm2
-; SSSE3-NEXT:    movdqa %xmm10, %xmm6
-; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm7, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm7
-; SSSE3-NEXT:    pcmpeqd %xmm6, %xmm6
-; SSSE3-NEXT:    pxor %xmm6, %xmm7
-; SSSE3-NEXT:    pxor %xmm10, %xmm0
-; SSSE3-NEXT:    movdqa %xmm10, %xmm2
-; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm4, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSSE3-NEXT:    por %xmm0, %xmm2
-; SSSE3-NEXT:    pxor %xmm6, %xmm2
-; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm7
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm7[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm7, %xmm4
-; SSSE3-NEXT:    movdqa %xmm9, %xmm0
-; SSSE3-NEXT:    pxor %xmm10, %xmm0
-; SSSE3-NEXT:    movdqa %xmm10, %xmm7
-; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm7
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm7[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm5, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm7[1,1,3,3]
-; SSSE3-NEXT:    por %xmm0, %xmm5
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm4 = [2147483648,2147483648]
+; SSSE3-NEXT:    movdqa %xmm0, %xmm5
+; SSSE3-NEXT:    pxor %xmm4, %xmm5
+; SSSE3-NEXT:    paddq %xmm2, %xmm0
+; SSSE3-NEXT:    movdqa %xmm0, %xmm6
+; SSSE3-NEXT:    pxor %xmm4, %xmm6
 ; SSSE3-NEXT:    movdqa %xmm5, %xmm7
+; SSSE3-NEXT:    pcmpgtd %xmm6, %xmm7
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm6[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm8, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm7[1,1,3,3]
+; SSSE3-NEXT:    por %xmm5, %xmm6
+; SSSE3-NEXT:    pxor %xmm5, %xmm5
+; SSSE3-NEXT:    pxor %xmm7, %xmm7
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm7
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
 ; SSSE3-NEXT:    pxor %xmm6, %xmm7
-; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm7
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[1,0,3,2]
+; SSSE3-NEXT:    movdqa %xmm7, %xmm2
+; SSSE3-NEXT:    pandn %xmm0, %xmm2
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm8 = [9223372036854775808,9223372036854775808]
+; SSSE3-NEXT:    pxor %xmm6, %xmm6
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm6
+; SSSE3-NEXT:    pandn %xmm8, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm9 = [9223372036854775807,9223372036854775807]
+; SSSE3-NEXT:    pand %xmm9, %xmm6
+; SSSE3-NEXT:    por %xmm6, %xmm0
 ; SSSE3-NEXT:    pand %xmm7, %xmm0
-; SSSE3-NEXT:    pandn %xmm4, %xmm0
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm11 = [9223372036854775808,9223372036854775808]
-; SSSE3-NEXT:    movdqa %xmm5, %xmm2
-; SSSE3-NEXT:    pandn %xmm11, %xmm2
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm8 = [9223372036854775807,9223372036854775807]
-; SSSE3-NEXT:    pand %xmm8, %xmm5
-; SSSE3-NEXT:    por %xmm2, %xmm5
-; SSSE3-NEXT:    pand %xmm0, %xmm5
-; SSSE3-NEXT:    pandn %xmm9, %xmm0
-; SSSE3-NEXT:    por %xmm5, %xmm0
+; SSSE3-NEXT:    por %xmm2, %xmm0
 ; SSSE3-NEXT:    movdqa %xmm1, %xmm2
+; SSSE3-NEXT:    pxor %xmm4, %xmm2
 ; SSSE3-NEXT:    paddq %xmm3, %xmm1
-; SSSE3-NEXT:    pxor %xmm10, %xmm3
-; SSSE3-NEXT:    movdqa %xmm10, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm5, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    por %xmm3, %xmm4
-; SSSE3-NEXT:    pxor %xmm6, %xmm4
-; SSSE3-NEXT:    pxor %xmm10, %xmm2
-; SSSE3-NEXT:    movdqa %xmm10, %xmm3
-; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm5, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm3
-; SSSE3-NEXT:    pxor %xmm6, %xmm3
-; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm4, %xmm5
-; SSSE3-NEXT:    movdqa %xmm1, %xmm2
-; SSSE3-NEXT:    pxor %xmm10, %xmm2
-; SSSE3-NEXT:    movdqa %xmm10, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    pxor %xmm1, %xmm4
+; SSSE3-NEXT:    movdqa %xmm2, %xmm6
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3]
 ; SSSE3-NEXT:    pand %xmm7, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[1,1,3,3]
 ; SSSE3-NEXT:    por %xmm2, %xmm4
-; SSSE3-NEXT:    pxor %xmm4, %xmm6
-; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm6, %xmm2
-; SSSE3-NEXT:    pandn %xmm5, %xmm2
-; SSSE3-NEXT:    movdqa %xmm4, %xmm3
-; SSSE3-NEXT:    pandn %xmm11, %xmm3
-; SSSE3-NEXT:    pand %xmm8, %xmm4
-; SSSE3-NEXT:    por %xmm3, %xmm4
-; SSSE3-NEXT:    pand %xmm2, %xmm4
-; SSSE3-NEXT:    pandn %xmm1, %xmm2
-; SSSE3-NEXT:    por %xmm4, %xmm2
-; SSSE3-NEXT:    movdqa %xmm2, %xmm1
+; SSSE3-NEXT:    pxor %xmm2, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    pxor %xmm4, %xmm2
+; SSSE3-NEXT:    movdqa %xmm2, %xmm3
+; SSSE3-NEXT:    pandn %xmm1, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm5
+; SSSE3-NEXT:    pandn %xmm8, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm5[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm9, %xmm4
+; SSSE3-NEXT:    por %xmm4, %xmm1
+; SSSE3-NEXT:    pand %xmm2, %xmm1
+; SSSE3-NEXT:    por %xmm3, %xmm1
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: v4i64:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa %xmm0, %xmm9
-; SSE41-NEXT:    movdqa {{.*#+}} xmm10 = [2147483648,2147483648]
-; SSE41-NEXT:    paddq %xmm2, %xmm9
-; SSE41-NEXT:    pxor %xmm10, %xmm2
-; SSE41-NEXT:    movdqa %xmm10, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm2, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE41-NEXT:    pand %xmm7, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm7
-; SSE41-NEXT:    pcmpeqd %xmm4, %xmm4
-; SSE41-NEXT:    pxor %xmm4, %xmm7
-; SSE41-NEXT:    pxor %xmm10, %xmm0
-; SSE41-NEXT:    movdqa %xmm10, %xmm2
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[1,1,3,3]
+; SSE41-NEXT:    movdqa %xmm0, %xmm4
+; SSE41-NEXT:    movdqa {{.*#+}} xmm6 = [2147483648,2147483648]
+; SSE41-NEXT:    pxor %xmm6, %xmm0
+; SSE41-NEXT:    paddq %xmm2, %xmm4
+; SSE41-NEXT:    movdqa %xmm4, %xmm5
+; SSE41-NEXT:    pxor %xmm6, %xmm5
+; SSE41-NEXT:    movdqa %xmm0, %xmm7
+; SSE41-NEXT:    pcmpeqd %xmm5, %xmm7
+; SSE41-NEXT:    pcmpgtd %xmm5, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[0,0,2,2]
+; SSE41-NEXT:    pand %xmm7, %xmm5
 ; SSE41-NEXT:    por %xmm0, %xmm5
-; SSE41-NEXT:    pxor %xmm4, %xmm5
-; SSE41-NEXT:    pcmpeqq %xmm5, %xmm7
-; SSE41-NEXT:    movdqa %xmm9, %xmm0
-; SSE41-NEXT:    pxor %xmm10, %xmm0
-; SSE41-NEXT:    movdqa %xmm10, %xmm2
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm8 = xmm2[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm8, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3]
-; SSE41-NEXT:    por %xmm6, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    pxor %xmm4, %xmm2
-; SSE41-NEXT:    pcmpeqq %xmm5, %xmm2
-; SSE41-NEXT:    pandn %xmm7, %xmm2
+; SSE41-NEXT:    pxor %xmm2, %xmm5
 ; SSE41-NEXT:    movapd {{.*#+}} xmm8 = [9223372036854775807,9223372036854775807]
-; SSE41-NEXT:    movapd {{.*#+}} xmm11 = [9223372036854775808,9223372036854775808]
-; SSE41-NEXT:    movapd %xmm11, %xmm5
-; SSE41-NEXT:    blendvpd %xmm0, %xmm8, %xmm5
-; SSE41-NEXT:    movdqa %xmm2, %xmm0
-; SSE41-NEXT:    blendvpd %xmm0, %xmm5, %xmm9
+; SSE41-NEXT:    movapd {{.*#+}} xmm7 = [9223372036854775808,9223372036854775808]
+; SSE41-NEXT:    movapd %xmm7, %xmm2
+; SSE41-NEXT:    movdqa %xmm4, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm8, %xmm2
+; SSE41-NEXT:    movdqa %xmm5, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm4
 ; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pxor %xmm6, %xmm0
 ; SSE41-NEXT:    paddq %xmm3, %xmm1
-; SSE41-NEXT:    pxor %xmm10, %xmm3
-; SSE41-NEXT:    movdqa %xmm10, %xmm2
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE41-NEXT:    por %xmm3, %xmm2
-; SSE41-NEXT:    pxor %xmm4, %xmm2
-; SSE41-NEXT:    pxor %xmm10, %xmm0
-; SSE41-NEXT:    movdqa %xmm10, %xmm3
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm0, %xmm3
-; SSE41-NEXT:    pxor %xmm4, %xmm3
-; SSE41-NEXT:    pcmpeqq %xmm3, %xmm2
+; SSE41-NEXT:    pxor %xmm1, %xmm6
+; SSE41-NEXT:    movdqa %xmm0, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm6, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm6, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm2
+; SSE41-NEXT:    por %xmm0, %xmm2
+; SSE41-NEXT:    pxor %xmm3, %xmm2
 ; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    pxor %xmm10, %xmm0
-; SSE41-NEXT:    movdqa %xmm10, %xmm5
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm6, %xmm7
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[1,1,3,3]
-; SSE41-NEXT:    por %xmm7, %xmm0
-; SSE41-NEXT:    pxor %xmm0, %xmm4
-; SSE41-NEXT:    pcmpeqq %xmm3, %xmm4
-; SSE41-NEXT:    pandn %xmm2, %xmm4
-; SSE41-NEXT:    blendvpd %xmm0, %xmm8, %xmm11
-; SSE41-NEXT:    movdqa %xmm4, %xmm0
-; SSE41-NEXT:    blendvpd %xmm0, %xmm11, %xmm1
-; SSE41-NEXT:    movapd %xmm9, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm8, %xmm7
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm7, %xmm1
+; SSE41-NEXT:    movapd %xmm4, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: v4i64:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm3, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
-; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm6
-; AVX1-NEXT:    vpcmpgtq %xmm6, %xmm3, %xmm7
-; AVX1-NEXT:    vpxor %xmm5, %xmm7, %xmm7
-; AVX1-NEXT:    vpcmpeqq %xmm4, %xmm7, %xmm8
-; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm3, %xmm4
-; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm9
-; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm3, %xmm4
-; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vpcmpeqq %xmm9, %xmm4, %xmm9
-; AVX1-NEXT:    vinsertf128 $1, %xmm8, %ymm9, %ymm8
-; AVX1-NEXT:    vpaddq %xmm2, %xmm6, %xmm9
-; AVX1-NEXT:    vpcmpgtq %xmm9, %xmm3, %xmm6
-; AVX1-NEXT:    vpxor %xmm5, %xmm6, %xmm2
-; AVX1-NEXT:    vpcmpeqq %xmm2, %xmm7, %xmm2
-; AVX1-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm3, %xmm1
-; AVX1-NEXT:    vpxor %xmm5, %xmm1, %xmm3
-; AVX1-NEXT:    vpcmpeqq %xmm3, %xmm4, %xmm3
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
-; AVX1-NEXT:    vandnpd %ymm8, %ymm2, %ymm2
-; AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm1, %ymm1
-; AVX1-NEXT:    vmovapd {{.*#+}} ymm3 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
-; AVX1-NEXT:    vblendvpd %ymm1, {{.*}}(%rip), %ymm3, %ymm1
-; AVX1-NEXT:    vinsertf128 $1, %xmm9, %ymm0, %ymm0
-; AVX1-NEXT:    vblendvpd %ymm2, %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; AVX1-NEXT:    vpaddq %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vpaddq %xmm1, %xmm0, %xmm4
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm4, %ymm5
+; AVX1-NEXT:    vmovapd {{.*#+}} ymm6 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
+; AVX1-NEXT:    vblendvpd %ymm5, {{.*}}(%rip), %ymm6, %ymm6
+; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    vxorpd %ymm0, %ymm1, %ymm0
+; AVX1-NEXT:    vblendvpd %ymm0, %ymm6, %ymm5, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: v4i64:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtq %ymm1, %ymm2, %ymm3
-; AVX2-NEXT:    vpcmpeqd %ymm4, %ymm4, %ymm4
-; AVX2-NEXT:    vpxor %ymm4, %ymm3, %ymm3
-; AVX2-NEXT:    vpcmpgtq %ymm0, %ymm2, %ymm5
-; AVX2-NEXT:    vpxor %ymm4, %ymm5, %ymm5
-; AVX2-NEXT:    vpcmpeqq %ymm3, %ymm5, %ymm3
-; AVX2-NEXT:    vpaddq %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpcmpgtq %ymm0, %ymm2, %ymm1
-; AVX2-NEXT:    vpxor %ymm4, %ymm1, %ymm2
-; AVX2-NEXT:    vpcmpeqq %ymm2, %ymm5, %ymm2
-; AVX2-NEXT:    vpandn %ymm3, %ymm2, %ymm2
+; AVX2-NEXT:    vpaddq %ymm1, %ymm0, %ymm2
 ; AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm3 = [9223372036854775807,9223372036854775807,9223372036854775807,9223372036854775807]
 ; AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm4 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
-; AVX2-NEXT:    vblendvpd %ymm1, %ymm3, %ymm4, %ymm1
-; AVX2-NEXT:    vblendvpd %ymm2, %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vblendvpd %ymm2, %ymm3, %ymm4, %ymm3
+; AVX2-NEXT:    vpcmpgtq %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %ymm0, %ymm1, %ymm0
+; AVX2-NEXT:    vblendvpd %ymm0, %ymm3, %ymm2, %ymm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: v4i64:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpcmpnltq %ymm2, %ymm1, %k0
-; AVX512-NEXT:    vpcmpnltq %ymm2, %ymm0, %k1
-; AVX512-NEXT:    kxorw %k0, %k1, %k0
-; AVX512-NEXT:    vpaddq %ymm1, %ymm0, %ymm0
-; AVX512-NEXT:    vpcmpnltq %ymm2, %ymm0, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
-; AVX512-NEXT:    kandnw %k1, %k0, %k1
-; AVX512-NEXT:    vpcmpgtq %ymm0, %ymm2, %k2
-; AVX512-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
-; AVX512-NEXT:    vpbroadcastq {{.*}}(%rip), %ymm1 {%k2}
-; AVX512-NEXT:    vmovdqa64 %ymm1, %ymm0 {%k1}
+; AVX512-NEXT:    vpcmpgtq %ymm1, %ymm2, %k0
+; AVX512-NEXT:    vpaddq %ymm1, %ymm0, %ymm1
+; AVX512-NEXT:    vpcmpgtq %ymm1, %ymm0, %k1
+; AVX512-NEXT:    kxorw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpgtq %ymm1, %ymm2, %k2
+; AVX512-NEXT:    vpbroadcastq {{.*#+}} ymm0 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
+; AVX512-NEXT:    vpbroadcastq {{.*}}(%rip), %ymm0 {%k2}
+; AVX512-NEXT:    vmovdqa64 %ymm0, %ymm1 {%k1}
+; AVX512-NEXT:    vmovdqa %ymm1, %ymm0
 ; AVX512-NEXT:    retq
   %z = call <4 x i64> @llvm.sadd.sat.v4i64(<4 x i64> %x, <4 x i64> %y)
   ret <4 x i64> %z
@@ -2216,688 +1446,362 @@ define <4 x i64> @v4i64(<4 x i64> %x, <4 x i64> %y) nounwind {
 define <8 x i64> @v8i64(<8 x i64> %x, <8 x i64> %y) nounwind {
 ; SSE2-LABEL: v8i64:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa %xmm1, %xmm8
-; SSE2-NEXT:    movdqa %xmm0, %xmm13
-; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [2147483648,2147483648]
-; SSE2-NEXT:    paddq %xmm4, %xmm13
-; SSE2-NEXT:    pxor %xmm9, %xmm4
-; SSE2-NEXT:    movdqa %xmm9, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm4, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    pand %xmm10, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm4, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm10, %xmm10
-; SSE2-NEXT:    pxor %xmm10, %xmm1
-; SSE2-NEXT:    pxor %xmm9, %xmm0
-; SSE2-NEXT:    movdqa %xmm9, %xmm4
+; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm0, %xmm9
+; SSE2-NEXT:    pxor %xmm8, %xmm9
+; SSE2-NEXT:    paddq %xmm4, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm10
+; SSE2-NEXT:    pxor %xmm8, %xmm10
+; SSE2-NEXT:    movdqa %xmm9, %xmm11
+; SSE2-NEXT:    pcmpgtd %xmm10, %xmm11
+; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm11[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm9, %xmm10
+; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm10[1,1,3,3]
+; SSE2-NEXT:    pand %xmm12, %xmm9
+; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm11[1,1,3,3]
+; SSE2-NEXT:    por %xmm9, %xmm10
+; SSE2-NEXT:    pxor %xmm9, %xmm9
+; SSE2-NEXT:    pxor %xmm11, %xmm11
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm11
+; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm11[1,1,3,3]
+; SSE2-NEXT:    pxor %xmm10, %xmm13
+; SSE2-NEXT:    movdqa %xmm13, %xmm12
+; SSE2-NEXT:    pandn %xmm0, %xmm12
+; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [9223372036854775808,9223372036854775808]
+; SSE2-NEXT:    pxor %xmm4, %xmm4
 ; SSE2-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pand %xmm11, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm14 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm0, %xmm14
-; SSE2-NEXT:    pxor %xmm10, %xmm14
-; SSE2-NEXT:    pcmpeqd %xmm14, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm1[1,0,3,2]
-; SSE2-NEXT:    pand %xmm1, %xmm11
-; SSE2-NEXT:    movdqa %xmm13, %xmm0
-; SSE2-NEXT:    pxor %xmm9, %xmm0
-; SSE2-NEXT:    movdqa %xmm9, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pand %xmm12, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm0, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm4
-; SSE2-NEXT:    pxor %xmm10, %xmm4
-; SSE2-NEXT:    pcmpeqd %xmm14, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[1,0,3,2]
-; SSE2-NEXT:    pand %xmm4, %xmm0
-; SSE2-NEXT:    pandn %xmm11, %xmm0
-; SSE2-NEXT:    movdqa {{.*#+}} xmm11 = [9223372036854775808,9223372036854775808]
+; SSE2-NEXT:    pandn %xmm10, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm11 = [9223372036854775807,9223372036854775807]
+; SSE2-NEXT:    pand %xmm11, %xmm4
+; SSE2-NEXT:    por %xmm4, %xmm0
+; SSE2-NEXT:    pand %xmm13, %xmm0
+; SSE2-NEXT:    por %xmm12, %xmm0
+; SSE2-NEXT:    movdqa %xmm1, %xmm12
+; SSE2-NEXT:    pxor %xmm8, %xmm12
+; SSE2-NEXT:    paddq %xmm5, %xmm1
 ; SSE2-NEXT:    movdqa %xmm1, %xmm4
-; SSE2-NEXT:    pandn %xmm11, %xmm4
-; SSE2-NEXT:    movdqa {{.*#+}} xmm12 = [9223372036854775807,9223372036854775807]
-; SSE2-NEXT:    pand %xmm12, %xmm1
-; SSE2-NEXT:    por %xmm4, %xmm1
-; SSE2-NEXT:    pand %xmm0, %xmm1
-; SSE2-NEXT:    pandn %xmm13, %xmm0
-; SSE2-NEXT:    por %xmm1, %xmm0
-; SSE2-NEXT:    movdqa %xmm8, %xmm1
-; SSE2-NEXT:    paddq %xmm5, %xmm8
-; SSE2-NEXT:    pxor %xmm9, %xmm5
-; SSE2-NEXT:    movdqa %xmm9, %xmm4
+; SSE2-NEXT:    pxor %xmm8, %xmm4
+; SSE2-NEXT:    movdqa %xmm12, %xmm13
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm13
+; SSE2-NEXT:    pshufd {{.*#+}} xmm14 = xmm13[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm12, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    pand %xmm14, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm13[1,1,3,3]
+; SSE2-NEXT:    por %xmm4, %xmm12
+; SSE2-NEXT:    pxor %xmm4, %xmm4
 ; SSE2-NEXT:    pcmpgtd %xmm5, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE2-NEXT:    pand %xmm13, %xmm5
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm5, %xmm4
-; SSE2-NEXT:    pxor %xmm10, %xmm4
-; SSE2-NEXT:    pxor %xmm9, %xmm1
-; SSE2-NEXT:    movdqa %xmm9, %xmm5
+; SSE2-NEXT:    pxor %xmm12, %xmm4
+; SSE2-NEXT:    movdqa %xmm4, %xmm12
+; SSE2-NEXT:    pandn %xmm1, %xmm12
+; SSE2-NEXT:    pxor %xmm5, %xmm5
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm5[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pand %xmm13, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm15 = xmm5[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm15
-; SSE2-NEXT:    pxor %xmm10, %xmm15
-; SSE2-NEXT:    pcmpeqd %xmm15, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm4[1,0,3,2]
-; SSE2-NEXT:    pand %xmm4, %xmm13
-; SSE2-NEXT:    movdqa %xmm8, %xmm1
-; SSE2-NEXT:    pxor %xmm9, %xmm1
-; SSE2-NEXT:    movdqa %xmm9, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm14 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pand %xmm14, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm4
-; SSE2-NEXT:    movdqa %xmm4, %xmm5
-; SSE2-NEXT:    pxor %xmm10, %xmm5
-; SSE2-NEXT:    pcmpeqd %xmm15, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm5[1,0,3,2]
-; SSE2-NEXT:    pand %xmm5, %xmm1
-; SSE2-NEXT:    pandn %xmm13, %xmm1
-; SSE2-NEXT:    movdqa %xmm4, %xmm5
-; SSE2-NEXT:    pandn %xmm11, %xmm5
-; SSE2-NEXT:    pand %xmm12, %xmm4
-; SSE2-NEXT:    por %xmm5, %xmm4
-; SSE2-NEXT:    pand %xmm1, %xmm4
-; SSE2-NEXT:    pandn %xmm8, %xmm1
-; SSE2-NEXT:    por %xmm4, %xmm1
-; SSE2-NEXT:    movdqa %xmm2, %xmm4
-; SSE2-NEXT:    paddq %xmm6, %xmm2
-; SSE2-NEXT:    pxor %xmm9, %xmm6
-; SSE2-NEXT:    movdqa %xmm9, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm6, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm5[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT:    pand %xmm8, %xmm6
+; SSE2-NEXT:    pandn %xmm10, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE2-NEXT:    por %xmm6, %xmm5
-; SSE2-NEXT:    pxor %xmm10, %xmm5
-; SSE2-NEXT:    pxor %xmm9, %xmm4
-; SSE2-NEXT:    movdqa %xmm9, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm4, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    pand %xmm8, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT:    por %xmm4, %xmm6
-; SSE2-NEXT:    pxor %xmm10, %xmm6
-; SSE2-NEXT:    pcmpeqd %xmm6, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm5[1,0,3,2]
-; SSE2-NEXT:    pand %xmm5, %xmm8
+; SSE2-NEXT:    pand %xmm11, %xmm5
+; SSE2-NEXT:    por %xmm5, %xmm1
+; SSE2-NEXT:    pand %xmm4, %xmm1
+; SSE2-NEXT:    por %xmm12, %xmm1
+; SSE2-NEXT:    movdqa %xmm2, %xmm12
+; SSE2-NEXT:    pxor %xmm8, %xmm12
+; SSE2-NEXT:    paddq %xmm6, %xmm2
 ; SSE2-NEXT:    movdqa %xmm2, %xmm5
-; SSE2-NEXT:    pxor %xmm9, %xmm5
-; SSE2-NEXT:    movdqa %xmm9, %xmm4
+; SSE2-NEXT:    pxor %xmm8, %xmm5
+; SSE2-NEXT:    movdqa %xmm12, %xmm4
 ; SSE2-NEXT:    pcmpgtd %xmm5, %xmm4
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm5
+; SSE2-NEXT:    pcmpeqd %xmm12, %xmm5
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm13, %xmm5
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
 ; SSE2-NEXT:    por %xmm5, %xmm4
+; SSE2-NEXT:    pxor %xmm5, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm6, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSE2-NEXT:    pxor %xmm4, %xmm5
+; SSE2-NEXT:    movdqa %xmm5, %xmm4
+; SSE2-NEXT:    pandn %xmm2, %xmm4
+; SSE2-NEXT:    pxor %xmm6, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm6
+; SSE2-NEXT:    pandn %xmm10, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSE2-NEXT:    pand %xmm11, %xmm6
+; SSE2-NEXT:    por %xmm6, %xmm2
+; SSE2-NEXT:    pand %xmm5, %xmm2
+; SSE2-NEXT:    por %xmm4, %xmm2
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    pxor %xmm8, %xmm4
+; SSE2-NEXT:    paddq %xmm7, %xmm3
+; SSE2-NEXT:    pxor %xmm3, %xmm8
 ; SSE2-NEXT:    movdqa %xmm4, %xmm5
-; SSE2-NEXT:    pxor %xmm10, %xmm5
-; SSE2-NEXT:    pcmpeqd %xmm6, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[1,0,3,2]
-; SSE2-NEXT:    pand %xmm5, %xmm6
-; SSE2-NEXT:    pandn %xmm8, %xmm6
-; SSE2-NEXT:    movdqa %xmm4, %xmm5
-; SSE2-NEXT:    pandn %xmm11, %xmm5
-; SSE2-NEXT:    pand %xmm12, %xmm4
-; SSE2-NEXT:    por %xmm5, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm8, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm4, %xmm8
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm8[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm6, %xmm4
-; SSE2-NEXT:    pandn %xmm2, %xmm6
-; SSE2-NEXT:    por %xmm4, %xmm6
-; SSE2-NEXT:    movdqa %xmm3, %xmm2
-; SSE2-NEXT:    paddq %xmm7, %xmm3
-; SSE2-NEXT:    pxor %xmm9, %xmm7
-; SSE2-NEXT:    movdqa %xmm9, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm7, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT:    pand %xmm5, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm7, %xmm4
-; SSE2-NEXT:    pxor %xmm10, %xmm4
-; SSE2-NEXT:    pxor %xmm9, %xmm2
-; SSE2-NEXT:    movdqa %xmm9, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm5[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT:    pand %xmm7, %xmm2
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE2-NEXT:    por %xmm2, %xmm5
-; SSE2-NEXT:    pxor %xmm10, %xmm5
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,0,3,2]
-; SSE2-NEXT:    pand %xmm4, %xmm2
-; SSE2-NEXT:    movdqa %xmm3, %xmm4
-; SSE2-NEXT:    pxor %xmm9, %xmm4
-; SSE2-NEXT:    movdqa %xmm9, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm4, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm4
+; SSE2-NEXT:    por %xmm4, %xmm5
+; SSE2-NEXT:    pxor %xmm4, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm7, %xmm4
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    pand %xmm8, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT:    por %xmm4, %xmm7
-; SSE2-NEXT:    pxor %xmm7, %xmm10
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm10
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm10[1,0,3,2]
-; SSE2-NEXT:    pand %xmm10, %xmm5
-; SSE2-NEXT:    pandn %xmm2, %xmm5
-; SSE2-NEXT:    movdqa %xmm7, %xmm2
-; SSE2-NEXT:    pandn %xmm11, %xmm2
-; SSE2-NEXT:    pand %xmm12, %xmm7
-; SSE2-NEXT:    por %xmm2, %xmm7
-; SSE2-NEXT:    pand %xmm5, %xmm7
+; SSE2-NEXT:    pxor %xmm5, %xmm4
+; SSE2-NEXT:    movdqa %xmm4, %xmm5
 ; SSE2-NEXT:    pandn %xmm3, %xmm5
-; SSE2-NEXT:    por %xmm7, %xmm5
-; SSE2-NEXT:    movdqa %xmm6, %xmm2
-; SSE2-NEXT:    movdqa %xmm5, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm9
+; SSE2-NEXT:    pandn %xmm10, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm9[1,1,3,3]
+; SSE2-NEXT:    pand %xmm11, %xmm6
+; SSE2-NEXT:    por %xmm6, %xmm3
+; SSE2-NEXT:    pand %xmm4, %xmm3
+; SSE2-NEXT:    por %xmm5, %xmm3
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: v8i64:
 ; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    movdqa %xmm1, %xmm8
-; SSSE3-NEXT:    movdqa %xmm0, %xmm13
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm9 = [2147483648,2147483648]
-; SSSE3-NEXT:    paddq %xmm4, %xmm13
-; SSSE3-NEXT:    pxor %xmm9, %xmm4
-; SSSE3-NEXT:    movdqa %xmm9, %xmm1
-; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm10 = xmm1[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm10, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    por %xmm4, %xmm1
-; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm10
-; SSSE3-NEXT:    pxor %xmm10, %xmm1
-; SSSE3-NEXT:    pxor %xmm9, %xmm0
-; SSSE3-NEXT:    movdqa %xmm9, %xmm4
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm8 = [2147483648,2147483648]
+; SSSE3-NEXT:    movdqa %xmm0, %xmm9
+; SSSE3-NEXT:    pxor %xmm8, %xmm9
+; SSSE3-NEXT:    paddq %xmm4, %xmm0
+; SSSE3-NEXT:    movdqa %xmm0, %xmm10
+; SSSE3-NEXT:    pxor %xmm8, %xmm10
+; SSSE3-NEXT:    movdqa %xmm9, %xmm11
+; SSSE3-NEXT:    pcmpgtd %xmm10, %xmm11
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm12 = xmm11[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm10
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm9 = xmm10[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm12, %xmm9
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm10 = xmm11[1,1,3,3]
+; SSSE3-NEXT:    por %xmm9, %xmm10
+; SSSE3-NEXT:    pxor %xmm9, %xmm9
+; SSSE3-NEXT:    pxor %xmm11, %xmm11
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm11
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm13 = xmm11[1,1,3,3]
+; SSSE3-NEXT:    pxor %xmm10, %xmm13
+; SSSE3-NEXT:    movdqa %xmm13, %xmm12
+; SSSE3-NEXT:    pandn %xmm0, %xmm12
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm10 = [9223372036854775808,9223372036854775808]
+; SSSE3-NEXT:    pxor %xmm4, %xmm4
 ; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm11 = xmm4[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm11, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm14 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    por %xmm0, %xmm14
-; SSSE3-NEXT:    pxor %xmm10, %xmm14
-; SSSE3-NEXT:    pcmpeqd %xmm14, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm11 = xmm1[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm11
-; SSSE3-NEXT:    movdqa %xmm13, %xmm0
-; SSSE3-NEXT:    pxor %xmm9, %xmm0
-; SSSE3-NEXT:    movdqa %xmm9, %xmm1
-; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm12 = xmm1[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm12, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    por %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa %xmm1, %xmm4
-; SSSE3-NEXT:    pxor %xmm10, %xmm4
-; SSSE3-NEXT:    pcmpeqd %xmm14, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm4, %xmm0
-; SSSE3-NEXT:    pandn %xmm11, %xmm0
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm11 = [9223372036854775808,9223372036854775808]
+; SSSE3-NEXT:    pandn %xmm10, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm11 = [9223372036854775807,9223372036854775807]
+; SSSE3-NEXT:    pand %xmm11, %xmm4
+; SSSE3-NEXT:    por %xmm4, %xmm0
+; SSSE3-NEXT:    pand %xmm13, %xmm0
+; SSSE3-NEXT:    por %xmm12, %xmm0
+; SSSE3-NEXT:    movdqa %xmm1, %xmm12
+; SSSE3-NEXT:    pxor %xmm8, %xmm12
+; SSSE3-NEXT:    paddq %xmm5, %xmm1
 ; SSSE3-NEXT:    movdqa %xmm1, %xmm4
-; SSSE3-NEXT:    pandn %xmm11, %xmm4
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm12 = [9223372036854775807,9223372036854775807]
-; SSSE3-NEXT:    pand %xmm12, %xmm1
-; SSSE3-NEXT:    por %xmm4, %xmm1
-; SSSE3-NEXT:    pand %xmm0, %xmm1
-; SSSE3-NEXT:    pandn %xmm13, %xmm0
-; SSSE3-NEXT:    por %xmm1, %xmm0
-; SSSE3-NEXT:    movdqa %xmm8, %xmm1
-; SSSE3-NEXT:    paddq %xmm5, %xmm8
-; SSSE3-NEXT:    pxor %xmm9, %xmm5
-; SSSE3-NEXT:    movdqa %xmm9, %xmm4
+; SSSE3-NEXT:    pxor %xmm8, %xmm4
+; SSSE3-NEXT:    movdqa %xmm12, %xmm13
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm13
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm14 = xmm13[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm12, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm14, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm12 = xmm13[1,1,3,3]
+; SSSE3-NEXT:    por %xmm4, %xmm12
+; SSSE3-NEXT:    pxor %xmm4, %xmm4
 ; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm13 = xmm4[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm5
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm13, %xmm5
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    por %xmm5, %xmm4
-; SSSE3-NEXT:    pxor %xmm10, %xmm4
-; SSSE3-NEXT:    pxor %xmm9, %xmm1
-; SSSE3-NEXT:    movdqa %xmm9, %xmm5
+; SSSE3-NEXT:    pxor %xmm12, %xmm4
+; SSSE3-NEXT:    movdqa %xmm4, %xmm12
+; SSSE3-NEXT:    pandn %xmm1, %xmm12
+; SSSE3-NEXT:    pxor %xmm5, %xmm5
 ; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm5
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm13 = xmm5[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm13, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm15 = xmm5[1,1,3,3]
-; SSSE3-NEXT:    por %xmm1, %xmm15
-; SSSE3-NEXT:    pxor %xmm10, %xmm15
-; SSSE3-NEXT:    pcmpeqd %xmm15, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm13 = xmm4[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm4, %xmm13
-; SSSE3-NEXT:    movdqa %xmm8, %xmm1
-; SSSE3-NEXT:    pxor %xmm9, %xmm1
-; SSSE3-NEXT:    movdqa %xmm9, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm14 = xmm4[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm14, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    por %xmm1, %xmm4
-; SSSE3-NEXT:    movdqa %xmm4, %xmm5
-; SSSE3-NEXT:    pxor %xmm10, %xmm5
-; SSSE3-NEXT:    pcmpeqd %xmm15, %xmm5
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm5[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm5, %xmm1
-; SSSE3-NEXT:    pandn %xmm13, %xmm1
-; SSSE3-NEXT:    movdqa %xmm4, %xmm5
-; SSSE3-NEXT:    pandn %xmm11, %xmm5
-; SSSE3-NEXT:    pand %xmm12, %xmm4
-; SSSE3-NEXT:    por %xmm5, %xmm4
-; SSSE3-NEXT:    pand %xmm1, %xmm4
-; SSSE3-NEXT:    pandn %xmm8, %xmm1
-; SSSE3-NEXT:    por %xmm4, %xmm1
-; SSSE3-NEXT:    movdqa %xmm2, %xmm4
-; SSSE3-NEXT:    paddq %xmm6, %xmm2
-; SSSE3-NEXT:    pxor %xmm9, %xmm6
-; SSSE3-NEXT:    movdqa %xmm9, %xmm5
-; SSSE3-NEXT:    pcmpgtd %xmm6, %xmm5
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm5[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm8, %xmm6
+; SSSE3-NEXT:    pandn %xmm10, %xmm1
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSSE3-NEXT:    por %xmm6, %xmm5
-; SSSE3-NEXT:    pxor %xmm10, %xmm5
-; SSSE3-NEXT:    pxor %xmm9, %xmm4
-; SSSE3-NEXT:    movdqa %xmm9, %xmm6
-; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm6[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm8, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSSE3-NEXT:    por %xmm4, %xmm6
-; SSSE3-NEXT:    pxor %xmm10, %xmm6
-; SSSE3-NEXT:    pcmpeqd %xmm6, %xmm5
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm5[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm5, %xmm8
+; SSSE3-NEXT:    pand %xmm11, %xmm5
+; SSSE3-NEXT:    por %xmm5, %xmm1
+; SSSE3-NEXT:    pand %xmm4, %xmm1
+; SSSE3-NEXT:    por %xmm12, %xmm1
+; SSSE3-NEXT:    movdqa %xmm2, %xmm12
+; SSSE3-NEXT:    pxor %xmm8, %xmm12
+; SSSE3-NEXT:    paddq %xmm6, %xmm2
 ; SSSE3-NEXT:    movdqa %xmm2, %xmm5
-; SSSE3-NEXT:    pxor %xmm9, %xmm5
-; SSSE3-NEXT:    movdqa %xmm9, %xmm4
+; SSSE3-NEXT:    pxor %xmm8, %xmm5
+; SSSE3-NEXT:    movdqa %xmm12, %xmm4
 ; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm4
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm13 = xmm4[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm5
+; SSSE3-NEXT:    pcmpeqd %xmm12, %xmm5
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
 ; SSSE3-NEXT:    pand %xmm13, %xmm5
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
 ; SSSE3-NEXT:    por %xmm5, %xmm4
+; SSSE3-NEXT:    pxor %xmm5, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm6, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSSE3-NEXT:    pxor %xmm4, %xmm5
+; SSSE3-NEXT:    movdqa %xmm5, %xmm4
+; SSSE3-NEXT:    pandn %xmm2, %xmm4
+; SSSE3-NEXT:    pxor %xmm6, %xmm6
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm6
+; SSSE3-NEXT:    pandn %xmm10, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm11, %xmm6
+; SSSE3-NEXT:    por %xmm6, %xmm2
+; SSSE3-NEXT:    pand %xmm5, %xmm2
+; SSSE3-NEXT:    por %xmm4, %xmm2
+; SSSE3-NEXT:    movdqa %xmm3, %xmm4
+; SSSE3-NEXT:    pxor %xmm8, %xmm4
+; SSSE3-NEXT:    paddq %xmm7, %xmm3
+; SSSE3-NEXT:    pxor %xmm3, %xmm8
 ; SSSE3-NEXT:    movdqa %xmm4, %xmm5
-; SSSE3-NEXT:    pxor %xmm10, %xmm5
-; SSSE3-NEXT:    pcmpeqd %xmm6, %xmm5
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm5, %xmm6
-; SSSE3-NEXT:    pandn %xmm8, %xmm6
-; SSSE3-NEXT:    movdqa %xmm4, %xmm5
-; SSSE3-NEXT:    pandn %xmm11, %xmm5
-; SSSE3-NEXT:    pand %xmm12, %xmm4
-; SSSE3-NEXT:    por %xmm5, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm8, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm8
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm8[1,1,3,3]
 ; SSSE3-NEXT:    pand %xmm6, %xmm4
-; SSSE3-NEXT:    pandn %xmm2, %xmm6
-; SSSE3-NEXT:    por %xmm4, %xmm6
-; SSSE3-NEXT:    movdqa %xmm3, %xmm2
-; SSSE3-NEXT:    paddq %xmm7, %xmm3
-; SSSE3-NEXT:    pxor %xmm9, %xmm7
-; SSSE3-NEXT:    movdqa %xmm9, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm7, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm7
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm5, %xmm7
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    por %xmm7, %xmm4
-; SSSE3-NEXT:    pxor %xmm10, %xmm4
-; SSSE3-NEXT:    pxor %xmm9, %xmm2
-; SSSE3-NEXT:    movdqa %xmm9, %xmm5
-; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm5
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm5[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm7, %xmm2
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm5
-; SSSE3-NEXT:    pxor %xmm10, %xmm5
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm4, %xmm2
-; SSSE3-NEXT:    movdqa %xmm3, %xmm4
-; SSSE3-NEXT:    pxor %xmm9, %xmm4
-; SSSE3-NEXT:    movdqa %xmm9, %xmm7
-; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm7
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm4
+; SSSE3-NEXT:    por %xmm4, %xmm5
+; SSSE3-NEXT:    pxor %xmm4, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm7, %xmm4
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm8, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSSE3-NEXT:    por %xmm4, %xmm7
-; SSSE3-NEXT:    pxor %xmm7, %xmm10
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm10
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm10[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm10, %xmm5
-; SSSE3-NEXT:    pandn %xmm2, %xmm5
-; SSSE3-NEXT:    movdqa %xmm7, %xmm2
-; SSSE3-NEXT:    pandn %xmm11, %xmm2
-; SSSE3-NEXT:    pand %xmm12, %xmm7
-; SSSE3-NEXT:    por %xmm2, %xmm7
-; SSSE3-NEXT:    pand %xmm5, %xmm7
+; SSSE3-NEXT:    pxor %xmm5, %xmm4
+; SSSE3-NEXT:    movdqa %xmm4, %xmm5
 ; SSSE3-NEXT:    pandn %xmm3, %xmm5
-; SSSE3-NEXT:    por %xmm7, %xmm5
-; SSSE3-NEXT:    movdqa %xmm6, %xmm2
-; SSSE3-NEXT:    movdqa %xmm5, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm9
+; SSSE3-NEXT:    pandn %xmm10, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm9[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm11, %xmm6
+; SSSE3-NEXT:    por %xmm6, %xmm3
+; SSSE3-NEXT:    pand %xmm4, %xmm3
+; SSSE3-NEXT:    por %xmm5, %xmm3
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: v8i64:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movdqa %xmm0, %xmm8
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm10 = [2147483648,2147483648]
-; SSE41-NEXT:    movdqa %xmm0, %xmm11
+; SSE41-NEXT:    pxor %xmm10, %xmm0
 ; SSE41-NEXT:    paddq %xmm4, %xmm8
-; SSE41-NEXT:    pxor %xmm10, %xmm4
-; SSE41-NEXT:    movdqa %xmm10, %xmm0
-; SSE41-NEXT:    pcmpgtd %xmm4, %xmm0
+; SSE41-NEXT:    movdqa %xmm8, %xmm9
+; SSE41-NEXT:    pxor %xmm10, %xmm9
+; SSE41-NEXT:    movdqa %xmm0, %xmm11
+; SSE41-NEXT:    pcmpeqd %xmm9, %xmm11
+; SSE41-NEXT:    pcmpgtd %xmm9, %xmm0
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm9 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE41-NEXT:    pand %xmm9, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm12 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm4, %xmm12
-; SSE41-NEXT:    pcmpeqd %xmm9, %xmm9
-; SSE41-NEXT:    pxor %xmm9, %xmm12
-; SSE41-NEXT:    pxor %xmm10, %xmm11
-; SSE41-NEXT:    movdqa %xmm10, %xmm0
-; SSE41-NEXT:    pcmpgtd %xmm11, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm13 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm11
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm11[1,1,3,3]
-; SSE41-NEXT:    pand %xmm13, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm11 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm4, %xmm11
-; SSE41-NEXT:    pxor %xmm9, %xmm11
-; SSE41-NEXT:    pcmpeqq %xmm11, %xmm12
-; SSE41-NEXT:    movdqa %xmm8, %xmm0
-; SSE41-NEXT:    pxor %xmm10, %xmm0
-; SSE41-NEXT:    movdqa %xmm10, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm13 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm14 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm13, %xmm14
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3]
-; SSE41-NEXT:    por %xmm14, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm4
-; SSE41-NEXT:    pxor %xmm9, %xmm4
-; SSE41-NEXT:    pcmpeqq %xmm11, %xmm4
-; SSE41-NEXT:    pandn %xmm12, %xmm4
+; SSE41-NEXT:    pand %xmm11, %xmm9
+; SSE41-NEXT:    por %xmm0, %xmm9
+; SSE41-NEXT:    pxor %xmm4, %xmm9
 ; SSE41-NEXT:    movapd {{.*#+}} xmm12 = [9223372036854775807,9223372036854775807]
 ; SSE41-NEXT:    movapd {{.*#+}} xmm11 = [9223372036854775808,9223372036854775808]
-; SSE41-NEXT:    movapd %xmm11, %xmm13
-; SSE41-NEXT:    blendvpd %xmm0, %xmm12, %xmm13
-; SSE41-NEXT:    movdqa %xmm4, %xmm0
-; SSE41-NEXT:    blendvpd %xmm0, %xmm13, %xmm8
-; SSE41-NEXT:    movdqa %xmm1, %xmm14
-; SSE41-NEXT:    paddq %xmm5, %xmm1
-; SSE41-NEXT:    pxor %xmm10, %xmm5
-; SSE41-NEXT:    movdqa %xmm10, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm5, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm13 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[1,1,3,3]
-; SSE41-NEXT:    pand %xmm13, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm15 = xmm4[1,1,3,3]
-; SSE41-NEXT:    por %xmm0, %xmm15
-; SSE41-NEXT:    pxor %xmm9, %xmm15
-; SSE41-NEXT:    pxor %xmm10, %xmm14
-; SSE41-NEXT:    movdqa %xmm10, %xmm0
-; SSE41-NEXT:    pcmpgtd %xmm14, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm13 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm14
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm14[1,1,3,3]
-; SSE41-NEXT:    pand %xmm13, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm4, %xmm5
-; SSE41-NEXT:    pxor %xmm9, %xmm5
-; SSE41-NEXT:    pcmpeqq %xmm5, %xmm15
+; SSE41-NEXT:    movapd %xmm11, %xmm4
+; SSE41-NEXT:    movdqa %xmm8, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm12, %xmm4
+; SSE41-NEXT:    movdqa %xmm9, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm8
 ; SSE41-NEXT:    movdqa %xmm1, %xmm0
 ; SSE41-NEXT:    pxor %xmm10, %xmm0
-; SSE41-NEXT:    movdqa %xmm10, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm13 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm14 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm13, %xmm14
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3]
-; SSE41-NEXT:    por %xmm14, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm4
-; SSE41-NEXT:    pxor %xmm9, %xmm4
-; SSE41-NEXT:    pcmpeqq %xmm5, %xmm4
-; SSE41-NEXT:    pandn %xmm15, %xmm4
+; SSE41-NEXT:    paddq %xmm5, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm4
+; SSE41-NEXT:    pxor %xmm10, %xmm4
+; SSE41-NEXT:    movdqa %xmm0, %xmm9
+; SSE41-NEXT:    pcmpeqd %xmm4, %xmm9
+; SSE41-NEXT:    pcmpgtd %xmm4, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[0,0,2,2]
+; SSE41-NEXT:    pand %xmm9, %xmm4
+; SSE41-NEXT:    por %xmm0, %xmm4
+; SSE41-NEXT:    pxor %xmm5, %xmm4
 ; SSE41-NEXT:    movapd %xmm11, %xmm5
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm12, %xmm5
 ; SSE41-NEXT:    movdqa %xmm4, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm5, %xmm1
 ; SSE41-NEXT:    movdqa %xmm2, %xmm0
-; SSE41-NEXT:    paddq %xmm6, %xmm2
-; SSE41-NEXT:    pxor %xmm10, %xmm6
-; SSE41-NEXT:    movdqa %xmm10, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm6, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[1,1,3,3]
-; SSE41-NEXT:    por %xmm6, %xmm5
-; SSE41-NEXT:    pxor %xmm9, %xmm5
 ; SSE41-NEXT:    pxor %xmm10, %xmm0
-; SSE41-NEXT:    movdqa %xmm10, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm6, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[1,1,3,3]
-; SSE41-NEXT:    por %xmm0, %xmm6
-; SSE41-NEXT:    pxor %xmm9, %xmm6
-; SSE41-NEXT:    pcmpeqq %xmm6, %xmm5
-; SSE41-NEXT:    movdqa %xmm2, %xmm0
-; SSE41-NEXT:    pxor %xmm10, %xmm0
-; SSE41-NEXT:    movdqa %xmm10, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm13 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm14 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm13, %xmm14
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3]
-; SSE41-NEXT:    por %xmm14, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm4
-; SSE41-NEXT:    pxor %xmm9, %xmm4
-; SSE41-NEXT:    pcmpeqq %xmm6, %xmm4
-; SSE41-NEXT:    pandn %xmm5, %xmm4
+; SSE41-NEXT:    paddq %xmm6, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm4
+; SSE41-NEXT:    pxor %xmm10, %xmm4
+; SSE41-NEXT:    movdqa %xmm0, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm4, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm4, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm4
+; SSE41-NEXT:    por %xmm0, %xmm4
+; SSE41-NEXT:    pxor %xmm6, %xmm4
 ; SSE41-NEXT:    movapd %xmm11, %xmm5
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm12, %xmm5
 ; SSE41-NEXT:    movdqa %xmm4, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm5, %xmm2
 ; SSE41-NEXT:    movdqa %xmm3, %xmm0
-; SSE41-NEXT:    paddq %xmm7, %xmm3
-; SSE41-NEXT:    pxor %xmm10, %xmm7
-; SSE41-NEXT:    movdqa %xmm10, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm7, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm7
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm7[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE41-NEXT:    por %xmm6, %xmm4
-; SSE41-NEXT:    pxor %xmm9, %xmm4
 ; SSE41-NEXT:    pxor %xmm10, %xmm0
-; SSE41-NEXT:    movdqa %xmm10, %xmm5
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm6, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE41-NEXT:    por %xmm0, %xmm5
-; SSE41-NEXT:    pxor %xmm9, %xmm5
-; SSE41-NEXT:    pcmpeqq %xmm5, %xmm4
+; SSE41-NEXT:    paddq %xmm7, %xmm3
+; SSE41-NEXT:    pxor %xmm3, %xmm10
+; SSE41-NEXT:    movdqa %xmm0, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm10, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm10, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm4
+; SSE41-NEXT:    por %xmm0, %xmm4
+; SSE41-NEXT:    pxor %xmm7, %xmm4
 ; SSE41-NEXT:    movdqa %xmm3, %xmm0
-; SSE41-NEXT:    pxor %xmm10, %xmm0
-; SSE41-NEXT:    movdqa %xmm10, %xmm6
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm13 = xmm6[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm13, %xmm7
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3]
-; SSE41-NEXT:    por %xmm7, %xmm0
-; SSE41-NEXT:    pxor %xmm0, %xmm9
-; SSE41-NEXT:    pcmpeqq %xmm5, %xmm9
-; SSE41-NEXT:    pandn %xmm4, %xmm9
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm12, %xmm11
-; SSE41-NEXT:    movdqa %xmm9, %xmm0
+; SSE41-NEXT:    movdqa %xmm4, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm11, %xmm3
 ; SSE41-NEXT:    movapd %xmm8, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: v8i64:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm9
-; AVX1-NEXT:    vpxor %xmm12, %xmm12, %xmm12
-; AVX1-NEXT:    vpcmpgtq %xmm9, %xmm12, %xmm7
-; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
-; AVX1-NEXT:    vpxor %xmm5, %xmm7, %xmm8
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm7
-; AVX1-NEXT:    vpcmpgtq %xmm7, %xmm12, %xmm6
-; AVX1-NEXT:    vpxor %xmm5, %xmm6, %xmm10
-; AVX1-NEXT:    vpcmpeqq %xmm8, %xmm10, %xmm8
-; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm12, %xmm6
-; AVX1-NEXT:    vpxor %xmm5, %xmm6, %xmm11
-; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm12, %xmm6
-; AVX1-NEXT:    vpxor %xmm5, %xmm6, %xmm6
-; AVX1-NEXT:    vpcmpeqq %xmm11, %xmm6, %xmm11
-; AVX1-NEXT:    vinsertf128 $1, %xmm8, %ymm11, %ymm8
-; AVX1-NEXT:    vpaddq %xmm9, %xmm7, %xmm9
-; AVX1-NEXT:    vpcmpgtq %xmm9, %xmm12, %xmm7
-; AVX1-NEXT:    vpxor %xmm5, %xmm7, %xmm4
-; AVX1-NEXT:    vpcmpeqq %xmm4, %xmm10, %xmm10
-; AVX1-NEXT:    vpaddq %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm12, %xmm2
-; AVX1-NEXT:    vpxor %xmm5, %xmm2, %xmm4
-; AVX1-NEXT:    vpcmpeqq %xmm4, %xmm6, %xmm4
-; AVX1-NEXT:    vinsertf128 $1, %xmm10, %ymm4, %ymm4
-; AVX1-NEXT:    vandnpd %ymm8, %ymm4, %ymm4
-; AVX1-NEXT:    vinsertf128 $1, %xmm7, %ymm2, %ymm7
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm4
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm5
+; AVX1-NEXT:    vpaddq %xmm4, %xmm5, %xmm4
+; AVX1-NEXT:    vpaddq %xmm2, %xmm0, %xmm6
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm6, %ymm7
 ; AVX1-NEXT:    vmovapd {{.*#+}} ymm8 = [9223372036854775807,9223372036854775807,9223372036854775807,9223372036854775807]
-; AVX1-NEXT:    vmovapd {{.*#+}} ymm10 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
-; AVX1-NEXT:    vblendvpd %ymm7, %ymm8, %ymm10, %ymm7
-; AVX1-NEXT:    vinsertf128 $1, %xmm9, %ymm0, %ymm0
-; AVX1-NEXT:    vblendvpd %ymm4, %ymm7, %ymm0, %ymm0
-; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm4
-; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm12, %xmm7
-; AVX1-NEXT:    vpxor %xmm5, %xmm7, %xmm7
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm12, %xmm6
-; AVX1-NEXT:    vpxor %xmm5, %xmm6, %xmm6
-; AVX1-NEXT:    vpcmpeqq %xmm7, %xmm6, %xmm9
-; AVX1-NEXT:    vpcmpgtq %xmm3, %xmm12, %xmm7
-; AVX1-NEXT:    vpxor %xmm5, %xmm7, %xmm11
-; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm12, %xmm7
-; AVX1-NEXT:    vpxor %xmm5, %xmm7, %xmm7
-; AVX1-NEXT:    vpcmpeqq %xmm11, %xmm7, %xmm11
-; AVX1-NEXT:    vinsertf128 $1, %xmm9, %ymm11, %ymm9
-; AVX1-NEXT:    vpaddq %xmm4, %xmm2, %xmm11
-; AVX1-NEXT:    vpcmpgtq %xmm11, %xmm12, %xmm4
-; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm2
-; AVX1-NEXT:    vpcmpeqq %xmm2, %xmm6, %xmm2
-; AVX1-NEXT:    vpaddq %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm12, %xmm3
-; AVX1-NEXT:    vpxor %xmm5, %xmm3, %xmm5
-; AVX1-NEXT:    vpcmpeqq %xmm5, %xmm7, %xmm5
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm5, %ymm2
-; AVX1-NEXT:    vandnpd %ymm9, %ymm2, %ymm2
-; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
-; AVX1-NEXT:    vblendvpd %ymm3, %ymm8, %ymm10, %ymm3
-; AVX1-NEXT:    vinsertf128 $1, %xmm11, %ymm1, %ymm1
-; AVX1-NEXT:    vblendvpd %ymm2, %ymm3, %ymm1, %ymm1
+; AVX1-NEXT:    vmovapd {{.*#+}} ymm9 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
+; AVX1-NEXT:    vblendvpd %ymm7, %ymm8, %ymm9, %ymm10
+; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm5, %xmm4
+; AVX1-NEXT:    vpcmpgtq %xmm6, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
+; AVX1-NEXT:    vxorpd %ymm0, %ymm2, %ymm0
+; AVX1-NEXT:    vblendvpd %ymm0, %ymm10, %ymm7, %ymm0
+; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm4
+; AVX1-NEXT:    vpaddq %xmm2, %xmm4, %xmm2
+; AVX1-NEXT:    vpaddq %xmm3, %xmm1, %xmm5
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm5, %ymm6
+; AVX1-NEXT:    vblendvpd %ymm6, %ymm8, %ymm9, %ymm7
+; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm4, %xmm2
+; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm1, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; AVX1-NEXT:    vxorpd %ymm1, %ymm3, %ymm1
+; AVX1-NEXT:    vblendvpd %ymm1, %ymm7, %ymm6, %ymm1
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: v8i64:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpcmpgtq %ymm2, %ymm4, %ymm5
-; AVX2-NEXT:    vpcmpeqd %ymm6, %ymm6, %ymm6
-; AVX2-NEXT:    vpxor %ymm6, %ymm5, %ymm5
-; AVX2-NEXT:    vpcmpgtq %ymm0, %ymm4, %ymm7
-; AVX2-NEXT:    vpxor %ymm6, %ymm7, %ymm7
-; AVX2-NEXT:    vpcmpeqq %ymm5, %ymm7, %ymm5
-; AVX2-NEXT:    vpaddq %ymm2, %ymm0, %ymm0
-; AVX2-NEXT:    vpcmpgtq %ymm0, %ymm4, %ymm2
-; AVX2-NEXT:    vpxor %ymm6, %ymm2, %ymm8
-; AVX2-NEXT:    vpcmpeqq %ymm8, %ymm7, %ymm7
-; AVX2-NEXT:    vpandn %ymm5, %ymm7, %ymm5
-; AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm7 = [9223372036854775807,9223372036854775807,9223372036854775807,9223372036854775807]
-; AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm8 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
-; AVX2-NEXT:    vblendvpd %ymm2, %ymm7, %ymm8, %ymm2
-; AVX2-NEXT:    vblendvpd %ymm5, %ymm2, %ymm0, %ymm0
-; AVX2-NEXT:    vpcmpgtq %ymm3, %ymm4, %ymm2
-; AVX2-NEXT:    vpxor %ymm6, %ymm2, %ymm2
-; AVX2-NEXT:    vpcmpgtq %ymm1, %ymm4, %ymm5
-; AVX2-NEXT:    vpxor %ymm6, %ymm5, %ymm5
-; AVX2-NEXT:    vpcmpeqq %ymm2, %ymm5, %ymm2
-; AVX2-NEXT:    vpaddq %ymm3, %ymm1, %ymm1
-; AVX2-NEXT:    vpcmpgtq %ymm1, %ymm4, %ymm3
-; AVX2-NEXT:    vpxor %ymm6, %ymm3, %ymm4
-; AVX2-NEXT:    vpcmpeqq %ymm4, %ymm5, %ymm4
-; AVX2-NEXT:    vpandn %ymm2, %ymm4, %ymm2
-; AVX2-NEXT:    vblendvpd %ymm3, %ymm7, %ymm8, %ymm3
-; AVX2-NEXT:    vblendvpd %ymm2, %ymm3, %ymm1, %ymm1
+; AVX2-NEXT:    vpaddq %ymm2, %ymm0, %ymm4
+; AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm5 = [9223372036854775807,9223372036854775807,9223372036854775807,9223372036854775807]
+; AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm6 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
+; AVX2-NEXT:    vblendvpd %ymm4, %ymm5, %ymm6, %ymm7
+; AVX2-NEXT:    vpcmpgtq %ymm4, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %ymm0, %ymm2, %ymm0
+; AVX2-NEXT:    vblendvpd %ymm0, %ymm7, %ymm4, %ymm0
+; AVX2-NEXT:    vpaddq %ymm3, %ymm1, %ymm2
+; AVX2-NEXT:    vblendvpd %ymm2, %ymm5, %ymm6, %ymm4
+; AVX2-NEXT:    vpcmpgtq %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    vpxor %ymm1, %ymm3, %ymm1
+; AVX2-NEXT:    vblendvpd %ymm1, %ymm4, %ymm2, %ymm1
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: v8i64:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpcmpnltq %zmm2, %zmm1, %k0
-; AVX512-NEXT:    vpcmpnltq %zmm2, %zmm0, %k1
-; AVX512-NEXT:    kxorw %k0, %k1, %k0
-; AVX512-NEXT:    vpaddq %zmm1, %zmm0, %zmm0
-; AVX512-NEXT:    vpcmpnltq %zmm2, %zmm0, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
-; AVX512-NEXT:    kandnw %k1, %k0, %k1
-; AVX512-NEXT:    vpcmpgtq %zmm0, %zmm2, %k2
-; AVX512-NEXT:    vpbroadcastq {{.*#+}} zmm1 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
-; AVX512-NEXT:    vpbroadcastq {{.*}}(%rip), %zmm1 {%k2}
-; AVX512-NEXT:    vmovdqa64 %zmm1, %zmm0 {%k1}
+; AVX512-NEXT:    vpcmpgtq %zmm1, %zmm2, %k0
+; AVX512-NEXT:    vpaddq %zmm1, %zmm0, %zmm1
+; AVX512-NEXT:    vpcmpgtq %zmm1, %zmm0, %k1
+; AVX512-NEXT:    kxorw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpgtq %zmm1, %zmm2, %k2
+; AVX512-NEXT:    vpbroadcastq {{.*#+}} zmm0 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
+; AVX512-NEXT:    vpbroadcastq {{.*}}(%rip), %zmm0 {%k2}
+; AVX512-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
+; AVX512-NEXT:    vmovdqa64 %zmm1, %zmm0
 ; AVX512-NEXT:    retq
   %z = call <8 x i64> @llvm.sadd.sat.v8i64(<8 x i64> %x, <8 x i64> %y)
   ret <8 x i64> %z
diff --git a/llvm/test/CodeGen/X86/saddo-redundant-add.ll b/llvm/test/CodeGen/X86/saddo-redundant-add.ll
index c56c68674..e89d81a84 100644
--- a/llvm/test/CodeGen/X86/saddo-redundant-add.ll
+++ b/llvm/test/CodeGen/X86/saddo-redundant-add.ll
@@ -1,12 +1,24 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-apple-darwin | FileCheck %s
 
 define void @redundant_add(i64 %n) {
 ; Check that we don't create two additions for the sadd.with.overflow.
-; CHECK-LABEL: redundant_add
-; CHECK-NOT:  leaq
-; CHECK-NOT:  addq
-; CHECK:      incq
-; CHECK-NEXT: jno
+; CHECK-LABEL: redundant_add:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  LBB0_1: ## %exit_check
+; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    cmpq %rdi, %rax
+; CHECK-NEXT:    jge LBB0_4
+; CHECK-NEXT:  ## %bb.2: ## %loop
+; CHECK-NEXT:    ## in Loop: Header=BB0_1 Depth=1
+; CHECK-NEXT:    incq %rax
+; CHECK-NEXT:    jno LBB0_1
+; CHECK-NEXT:  ## %bb.3: ## %overflow
+; CHECK-NEXT:    ud2
+; CHECK-NEXT:  LBB0_4: ## %exit
+; CHECK-NEXT:    retq
 entry:
   br label %exit_check
 
diff --git a/llvm/test/CodeGen/X86/sat-add.ll b/llvm/test/CodeGen/X86/sat-add.ll
index 70d102667..4f5ec6fbe 100644
--- a/llvm/test/CodeGen/X86/sat-add.ll
+++ b/llvm/test/CodeGen/X86/sat-add.ll
@@ -581,12 +581,11 @@ define <2 x i64> @unsigned_sat_constant_v2i64_using_min(<2 x i64> %x) {
 ; SSE41-NEXT:    pxor %xmm1, %xmm0
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [9223372034707292117,9223372034707292117]
 ; SSE41-NEXT:    movdqa %xmm3, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm3, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
 ; SSE41-NEXT:    paddq {{.*}}(%rip), %xmm2
 ; SSE41-NEXT:    movdqa %xmm2, %xmm0
@@ -877,12 +876,11 @@ define <2 x i64> @unsigned_sat_variable_v2i64_using_min(<2 x i64> %x, <2 x i64>
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [9223372034707292159,9223372034707292159]
 ; SSE41-NEXT:    pxor %xmm1, %xmm4
 ; SSE41-NEXT:    movdqa %xmm4, %xmm5
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3]
-; SSE41-NEXT:    pand %xmm6, %xmm0
-; SSE41-NEXT:    por %xmm5, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm0
+; SSE41-NEXT:    por %xmm4, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm3
 ; SSE41-NEXT:    paddq %xmm1, %xmm3
 ; SSE41-NEXT:    movdqa %xmm3, %xmm0
diff --git a/llvm/test/CodeGen/X86/scalar-extract.ll b/llvm/test/CodeGen/X86/scalar-extract.ll
index dd1b9a55e..d09754d30 100644
--- a/llvm/test/CodeGen/X86/scalar-extract.ll
+++ b/llvm/test/CodeGen/X86/scalar-extract.ll
@@ -1,10 +1,17 @@
-; RUN: llc < %s -mtriple=i686-- -mattr=+mmx -o %t
-; RUN: not grep movq  %t
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- -mattr=+mmx | FileCheck %s
 
 ; Check that widening doesn't introduce a mmx register in this case when
 ; a simple load/store would suffice.
 
 define void @foo(<2 x i16>* %A, <2 x i16>* %B) {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT:    movl (%ecx), %ecx
+; CHECK-NEXT:    movl %ecx, (%eax)
+; CHECK-NEXT:    retl
 entry:
 	%tmp1 = load <2 x i16>, <2 x i16>* %A		; <<2 x i16>> [#uses=1]
 	store <2 x i16> %tmp1, <2 x i16>* %B
diff --git a/llvm/test/CodeGen/X86/scalar-fp-to-i32.ll b/llvm/test/CodeGen/X86/scalar-fp-to-i32.ll
new file mode 100644
index 000000000..4ca20a778
--- /dev/null
+++ b/llvm/test/CodeGen/X86/scalar-fp-to-i32.ll
@@ -0,0 +1,1024 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i386-pc-windows-msvc     -mattr=+avx512f,+avx512dq,+avx512vl | FileCheck %s --check-prefixes=CHECK,X32,AVX512_32,AVX512_32_WIN,AVX512DQVL_32_WIN
+; RUN: llc < %s -mtriple=i386-unknown-linux-gnu   -mattr=+avx512f,+avx512dq,+avx512vl | FileCheck %s --check-prefixes=CHECK,X32,AVX512_32,AVX512_32_LIN,AVX512DQVL_32_LIN
+; RUN: llc < %s -mtriple=x86_64-pc-windows-msvc   -mattr=+avx512f,+avx512dq,+avx512vl | FileCheck %s --check-prefixes=CHECK,X64,AVX512_64,AVX512_64_WIN,AVX512DQVL_64_WIN
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512f,+avx512dq,+avx512vl | FileCheck %s --check-prefixes=CHECK,X32,AVX512_64,AVX512_64_LIN,AVX512DQVL_64_LIN
+; RUN: llc < %s -mtriple=i386-pc-windows-msvc     -mattr=+avx512f,+avx512dq | FileCheck %s --check-prefixes=CHECK,X32,AVX512_32,AVX512_32_WIN,AVX512DQ_32_WIN
+; RUN: llc < %s -mtriple=i386-unknown-linux-gnu   -mattr=+avx512f,+avx512dq | FileCheck %s --check-prefixes=CHECK,X32,AVX512_32,AVX512_32_LIN,AVX512DQ_32_LIN
+; RUN: llc < %s -mtriple=x86_64-pc-windows-msvc   -mattr=+avx512f,+avx512dq | FileCheck %s --check-prefixes=CHECK,X64,AVX512_64,AVX512_64_WIN,AVX512DQ_64_WIN
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512f,+avx512dq | FileCheck %s --check-prefixes=CHECK,X32,AVX512_64,AVX512_64_LIN,AVX512DQ_64_LIN
+; RUN: llc < %s -mtriple=i386-pc-windows-msvc     -mattr=+avx512f | FileCheck %s --check-prefixes=CHECK,X32,AVX512_32,AVX512_32_WIN,AVX512F_32_WIN
+; RUN: llc < %s -mtriple=i386-unknown-linux-gnu   -mattr=+avx512f | FileCheck %s --check-prefixes=CHECK,X32,AVX512_32,AVX512_32_LIN,AVX512F_32_LIN
+; RUN: llc < %s -mtriple=x86_64-pc-windows-msvc   -mattr=+avx512f | FileCheck %s --check-prefixes=CHECK,X64,AVX512_64,AVX512_64_WIN,AVX512F_64_WIN
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512f | FileCheck %s --check-prefixes=CHECK,X32,AVX512_64,AVX512_64_LIN,AVX512F_64_LIN
+; RUN: llc < %s -mtriple=i386-pc-windows-msvc     -mattr=+sse3 | FileCheck %s --check-prefixes=CHECK,X32,SSE3_32,SSE3_32_WIN
+; RUN: llc < %s -mtriple=i386-unknown-linux-gnu   -mattr=+sse3 | FileCheck %s --check-prefixes=CHECK,X32,SSE3_32,SSE3_32_LIN
+; RUN: llc < %s -mtriple=x86_64-pc-windows-msvc   -mattr=+sse3 | FileCheck %s --check-prefixes=CHECK,X64,SSE3_64,SSE3_64_WIN
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+sse3 | FileCheck %s --check-prefixes=CHECK,X64,SSE3_64,SSE3_64_LIN
+; RUN: llc < %s -mtriple=i386-pc-windows-msvc     -mattr=+sse2 | FileCheck %s --check-prefixes=CHECK,X32,SSE2_32,SSE2_32_WIN
+; RUN: llc < %s -mtriple=i386-unknown-linux-gnu   -mattr=+sse2 | FileCheck %s --check-prefixes=CHECK,X32,SSE2_32,SSE2_32_LIN
+; RUN: llc < %s -mtriple=x86_64-pc-windows-msvc   -mattr=+sse2 | FileCheck %s --check-prefixes=CHECK,X64,SSE2_64,SSE2_64_WIN
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+sse2 | FileCheck %s --check-prefixes=CHECK,X64,SSE2_64,SSE2_64_LIN
+; RUN: llc < %s -mtriple=i386-pc-windows-msvc     -mattr=+sse | FileCheck %s --check-prefixes=CHECK,X32,SSE_32,SSE_32_WIN
+; RUN: llc < %s -mtriple=i386-unknown-linux-gnu   -mattr=+sse | FileCheck %s --check-prefixes=CHECK,X32,SSE_32,SSE_32_LIN
+; RUN: llc < %s -mtriple=i386-pc-windows-msvc     -mattr=-sse  | FileCheck %s --check-prefixes=CHECK,X32,X87,X87_WIN
+; RUN: llc < %s -mtriple=i386-unknown-linux-gnu   -mattr=-sse  | FileCheck %s --check-prefixes=CHECK,X32,X87,X87_LIN
+
+; Check that scalar FP conversions to signed and unsigned int32 are using
+; reasonable sequences, across platforms and target switches.
+
+define i32 @f_to_u32(float %a) nounwind {
+; AVX512_32-LABEL: f_to_u32:
+; AVX512_32:       # %bb.0:
+; AVX512_32-NEXT:    vcvttss2usi {{[0-9]+}}(%esp), %eax
+; AVX512_32-NEXT:    retl
+;
+; AVX512_64-LABEL: f_to_u32:
+; AVX512_64:       # %bb.0:
+; AVX512_64-NEXT:    vcvttss2usi %xmm0, %eax
+; AVX512_64-NEXT:    retq
+;
+; SSE3_32_WIN-LABEL: f_to_u32:
+; SSE3_32_WIN:       # %bb.0:
+; SSE3_32_WIN-NEXT:    pushl %ebp
+; SSE3_32_WIN-NEXT:    movl %esp, %ebp
+; SSE3_32_WIN-NEXT:    andl $-8, %esp
+; SSE3_32_WIN-NEXT:    subl $8, %esp
+; SSE3_32_WIN-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE3_32_WIN-NEXT:    movss %xmm0, (%esp)
+; SSE3_32_WIN-NEXT:    flds (%esp)
+; SSE3_32_WIN-NEXT:    fisttpll (%esp)
+; SSE3_32_WIN-NEXT:    movl (%esp), %eax
+; SSE3_32_WIN-NEXT:    movl %ebp, %esp
+; SSE3_32_WIN-NEXT:    popl %ebp
+; SSE3_32_WIN-NEXT:    retl
+;
+; SSE3_32_LIN-LABEL: f_to_u32:
+; SSE3_32_LIN:       # %bb.0:
+; SSE3_32_LIN-NEXT:    subl $12, %esp
+; SSE3_32_LIN-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE3_32_LIN-NEXT:    movss %xmm0, (%esp)
+; SSE3_32_LIN-NEXT:    flds (%esp)
+; SSE3_32_LIN-NEXT:    fisttpll (%esp)
+; SSE3_32_LIN-NEXT:    movl (%esp), %eax
+; SSE3_32_LIN-NEXT:    addl $12, %esp
+; SSE3_32_LIN-NEXT:    retl
+;
+; SSE3_64-LABEL: f_to_u32:
+; SSE3_64:       # %bb.0:
+; SSE3_64-NEXT:    cvttss2si %xmm0, %rax
+; SSE3_64-NEXT:    # kill: def $eax killed $eax killed $rax
+; SSE3_64-NEXT:    retq
+;
+; SSE2_32-LABEL: f_to_u32:
+; SSE2_32:       # %bb.0:
+; SSE2_32-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2_32-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE2_32-NEXT:    movaps %xmm0, %xmm2
+; SSE2_32-NEXT:    subss %xmm1, %xmm2
+; SSE2_32-NEXT:    cvttss2si %xmm2, %ecx
+; SSE2_32-NEXT:    xorl $-2147483648, %ecx # imm = 0x80000000
+; SSE2_32-NEXT:    cvttss2si %xmm0, %eax
+; SSE2_32-NEXT:    ucomiss %xmm0, %xmm1
+; SSE2_32-NEXT:    cmovbel %ecx, %eax
+; SSE2_32-NEXT:    retl
+;
+; SSE2_64-LABEL: f_to_u32:
+; SSE2_64:       # %bb.0:
+; SSE2_64-NEXT:    cvttss2si %xmm0, %rax
+; SSE2_64-NEXT:    # kill: def $eax killed $eax killed $rax
+; SSE2_64-NEXT:    retq
+;
+; SSE_32-LABEL: f_to_u32:
+; SSE_32:       # %bb.0:
+; SSE_32-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE_32-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE_32-NEXT:    movaps %xmm0, %xmm2
+; SSE_32-NEXT:    subss %xmm1, %xmm2
+; SSE_32-NEXT:    cvttss2si %xmm2, %ecx
+; SSE_32-NEXT:    xorl $-2147483648, %ecx # imm = 0x80000000
+; SSE_32-NEXT:    cvttss2si %xmm0, %eax
+; SSE_32-NEXT:    ucomiss %xmm0, %xmm1
+; SSE_32-NEXT:    cmovbel %ecx, %eax
+; SSE_32-NEXT:    retl
+;
+; X87_WIN-LABEL: f_to_u32:
+; X87_WIN:       # %bb.0:
+; X87_WIN-NEXT:    pushl %ebp
+; X87_WIN-NEXT:    movl %esp, %ebp
+; X87_WIN-NEXT:    andl $-8, %esp
+; X87_WIN-NEXT:    subl $16, %esp
+; X87_WIN-NEXT:    flds 8(%ebp)
+; X87_WIN-NEXT:    fnstcw {{[0-9]+}}(%esp)
+; X87_WIN-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X87_WIN-NEXT:    orl $3072, %eax # imm = 0xC00
+; X87_WIN-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; X87_WIN-NEXT:    fldcw {{[0-9]+}}(%esp)
+; X87_WIN-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X87_WIN-NEXT:    fldcw {{[0-9]+}}(%esp)
+; X87_WIN-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X87_WIN-NEXT:    movl %ebp, %esp
+; X87_WIN-NEXT:    popl %ebp
+; X87_WIN-NEXT:    retl
+;
+; X87_LIN-LABEL: f_to_u32:
+; X87_LIN:       # %bb.0:
+; X87_LIN-NEXT:    subl $20, %esp
+; X87_LIN-NEXT:    flds {{[0-9]+}}(%esp)
+; X87_LIN-NEXT:    fnstcw {{[0-9]+}}(%esp)
+; X87_LIN-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X87_LIN-NEXT:    orl $3072, %eax # imm = 0xC00
+; X87_LIN-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; X87_LIN-NEXT:    fldcw {{[0-9]+}}(%esp)
+; X87_LIN-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X87_LIN-NEXT:    fldcw {{[0-9]+}}(%esp)
+; X87_LIN-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X87_LIN-NEXT:    addl $20, %esp
+; X87_LIN-NEXT:    retl
+  %r = fptoui float %a to i32
+  ret i32 %r
+}
+
+define i32 @f_to_s32(float %a) nounwind {
+; AVX512_32-LABEL: f_to_s32:
+; AVX512_32:       # %bb.0:
+; AVX512_32-NEXT:    vcvttss2si {{[0-9]+}}(%esp), %eax
+; AVX512_32-NEXT:    retl
+;
+; AVX512_64-LABEL: f_to_s32:
+; AVX512_64:       # %bb.0:
+; AVX512_64-NEXT:    vcvttss2si %xmm0, %eax
+; AVX512_64-NEXT:    retq
+;
+; SSE3_32-LABEL: f_to_s32:
+; SSE3_32:       # %bb.0:
+; SSE3_32-NEXT:    cvttss2si {{[0-9]+}}(%esp), %eax
+; SSE3_32-NEXT:    retl
+;
+; SSE3_64-LABEL: f_to_s32:
+; SSE3_64:       # %bb.0:
+; SSE3_64-NEXT:    cvttss2si %xmm0, %eax
+; SSE3_64-NEXT:    retq
+;
+; SSE2_32-LABEL: f_to_s32:
+; SSE2_32:       # %bb.0:
+; SSE2_32-NEXT:    cvttss2si {{[0-9]+}}(%esp), %eax
+; SSE2_32-NEXT:    retl
+;
+; SSE2_64-LABEL: f_to_s32:
+; SSE2_64:       # %bb.0:
+; SSE2_64-NEXT:    cvttss2si %xmm0, %eax
+; SSE2_64-NEXT:    retq
+;
+; SSE_32-LABEL: f_to_s32:
+; SSE_32:       # %bb.0:
+; SSE_32-NEXT:    cvttss2si {{[0-9]+}}(%esp), %eax
+; SSE_32-NEXT:    retl
+;
+; X87-LABEL: f_to_s32:
+; X87:       # %bb.0:
+; X87-NEXT:    subl $8, %esp
+; X87-NEXT:    flds {{[0-9]+}}(%esp)
+; X87-NEXT:    fnstcw (%esp)
+; X87-NEXT:    movzwl (%esp), %eax
+; X87-NEXT:    orl $3072, %eax # imm = 0xC00
+; X87-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; X87-NEXT:    fldcw {{[0-9]+}}(%esp)
+; X87-NEXT:    fistpl {{[0-9]+}}(%esp)
+; X87-NEXT:    fldcw (%esp)
+; X87-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    addl $8, %esp
+; X87-NEXT:    retl
+  %r = fptosi float %a to i32
+  ret i32 %r
+}
+
+define i32 @d_to_u32(double %a) nounwind {
+; AVX512_32-LABEL: d_to_u32:
+; AVX512_32:       # %bb.0:
+; AVX512_32-NEXT:    vcvttsd2usi {{[0-9]+}}(%esp), %eax
+; AVX512_32-NEXT:    retl
+;
+; AVX512_64-LABEL: d_to_u32:
+; AVX512_64:       # %bb.0:
+; AVX512_64-NEXT:    vcvttsd2usi %xmm0, %eax
+; AVX512_64-NEXT:    retq
+;
+; SSE3_32_WIN-LABEL: d_to_u32:
+; SSE3_32_WIN:       # %bb.0:
+; SSE3_32_WIN-NEXT:    pushl %ebp
+; SSE3_32_WIN-NEXT:    movl %esp, %ebp
+; SSE3_32_WIN-NEXT:    andl $-8, %esp
+; SSE3_32_WIN-NEXT:    subl $8, %esp
+; SSE3_32_WIN-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE3_32_WIN-NEXT:    movsd %xmm0, (%esp)
+; SSE3_32_WIN-NEXT:    fldl (%esp)
+; SSE3_32_WIN-NEXT:    fisttpll (%esp)
+; SSE3_32_WIN-NEXT:    movl (%esp), %eax
+; SSE3_32_WIN-NEXT:    movl %ebp, %esp
+; SSE3_32_WIN-NEXT:    popl %ebp
+; SSE3_32_WIN-NEXT:    retl
+;
+; SSE3_32_LIN-LABEL: d_to_u32:
+; SSE3_32_LIN:       # %bb.0:
+; SSE3_32_LIN-NEXT:    subl $12, %esp
+; SSE3_32_LIN-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE3_32_LIN-NEXT:    movsd %xmm0, (%esp)
+; SSE3_32_LIN-NEXT:    fldl (%esp)
+; SSE3_32_LIN-NEXT:    fisttpll (%esp)
+; SSE3_32_LIN-NEXT:    movl (%esp), %eax
+; SSE3_32_LIN-NEXT:    addl $12, %esp
+; SSE3_32_LIN-NEXT:    retl
+;
+; SSE3_64-LABEL: d_to_u32:
+; SSE3_64:       # %bb.0:
+; SSE3_64-NEXT:    cvttsd2si %xmm0, %rax
+; SSE3_64-NEXT:    # kill: def $eax killed $eax killed $rax
+; SSE3_64-NEXT:    retq
+;
+; SSE2_32-LABEL: d_to_u32:
+; SSE2_32:       # %bb.0:
+; SSE2_32-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE2_32-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
+; SSE2_32-NEXT:    movapd %xmm0, %xmm2
+; SSE2_32-NEXT:    subsd %xmm1, %xmm2
+; SSE2_32-NEXT:    cvttsd2si %xmm2, %ecx
+; SSE2_32-NEXT:    xorl $-2147483648, %ecx # imm = 0x80000000
+; SSE2_32-NEXT:    cvttsd2si %xmm0, %eax
+; SSE2_32-NEXT:    ucomisd %xmm0, %xmm1
+; SSE2_32-NEXT:    cmovbel %ecx, %eax
+; SSE2_32-NEXT:    retl
+;
+; SSE2_64-LABEL: d_to_u32:
+; SSE2_64:       # %bb.0:
+; SSE2_64-NEXT:    cvttsd2si %xmm0, %rax
+; SSE2_64-NEXT:    # kill: def $eax killed $eax killed $rax
+; SSE2_64-NEXT:    retq
+;
+; SSE_32_WIN-LABEL: d_to_u32:
+; SSE_32_WIN:       # %bb.0:
+; SSE_32_WIN-NEXT:    pushl %ebp
+; SSE_32_WIN-NEXT:    movl %esp, %ebp
+; SSE_32_WIN-NEXT:    andl $-8, %esp
+; SSE_32_WIN-NEXT:    subl $16, %esp
+; SSE_32_WIN-NEXT:    fldl 8(%ebp)
+; SSE_32_WIN-NEXT:    fnstcw {{[0-9]+}}(%esp)
+; SSE_32_WIN-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; SSE_32_WIN-NEXT:    orl $3072, %eax # imm = 0xC00
+; SSE_32_WIN-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; SSE_32_WIN-NEXT:    fldcw {{[0-9]+}}(%esp)
+; SSE_32_WIN-NEXT:    fistpll {{[0-9]+}}(%esp)
+; SSE_32_WIN-NEXT:    fldcw {{[0-9]+}}(%esp)
+; SSE_32_WIN-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; SSE_32_WIN-NEXT:    movl %ebp, %esp
+; SSE_32_WIN-NEXT:    popl %ebp
+; SSE_32_WIN-NEXT:    retl
+;
+; SSE_32_LIN-LABEL: d_to_u32:
+; SSE_32_LIN:       # %bb.0:
+; SSE_32_LIN-NEXT:    subl $20, %esp
+; SSE_32_LIN-NEXT:    fldl {{[0-9]+}}(%esp)
+; SSE_32_LIN-NEXT:    fnstcw {{[0-9]+}}(%esp)
+; SSE_32_LIN-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; SSE_32_LIN-NEXT:    orl $3072, %eax # imm = 0xC00
+; SSE_32_LIN-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; SSE_32_LIN-NEXT:    fldcw {{[0-9]+}}(%esp)
+; SSE_32_LIN-NEXT:    fistpll {{[0-9]+}}(%esp)
+; SSE_32_LIN-NEXT:    fldcw {{[0-9]+}}(%esp)
+; SSE_32_LIN-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; SSE_32_LIN-NEXT:    addl $20, %esp
+; SSE_32_LIN-NEXT:    retl
+;
+; X87_WIN-LABEL: d_to_u32:
+; X87_WIN:       # %bb.0:
+; X87_WIN-NEXT:    pushl %ebp
+; X87_WIN-NEXT:    movl %esp, %ebp
+; X87_WIN-NEXT:    andl $-8, %esp
+; X87_WIN-NEXT:    subl $16, %esp
+; X87_WIN-NEXT:    fldl 8(%ebp)
+; X87_WIN-NEXT:    fnstcw {{[0-9]+}}(%esp)
+; X87_WIN-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X87_WIN-NEXT:    orl $3072, %eax # imm = 0xC00
+; X87_WIN-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; X87_WIN-NEXT:    fldcw {{[0-9]+}}(%esp)
+; X87_WIN-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X87_WIN-NEXT:    fldcw {{[0-9]+}}(%esp)
+; X87_WIN-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X87_WIN-NEXT:    movl %ebp, %esp
+; X87_WIN-NEXT:    popl %ebp
+; X87_WIN-NEXT:    retl
+;
+; X87_LIN-LABEL: d_to_u32:
+; X87_LIN:       # %bb.0:
+; X87_LIN-NEXT:    subl $20, %esp
+; X87_LIN-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87_LIN-NEXT:    fnstcw {{[0-9]+}}(%esp)
+; X87_LIN-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X87_LIN-NEXT:    orl $3072, %eax # imm = 0xC00
+; X87_LIN-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; X87_LIN-NEXT:    fldcw {{[0-9]+}}(%esp)
+; X87_LIN-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X87_LIN-NEXT:    fldcw {{[0-9]+}}(%esp)
+; X87_LIN-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X87_LIN-NEXT:    addl $20, %esp
+; X87_LIN-NEXT:    retl
+  %r = fptoui double %a to i32
+  ret i32 %r
+}
+
+define i32 @d_to_s32(double %a) nounwind {
+; AVX512_32-LABEL: d_to_s32:
+; AVX512_32:       # %bb.0:
+; AVX512_32-NEXT:    vcvttsd2si {{[0-9]+}}(%esp), %eax
+; AVX512_32-NEXT:    retl
+;
+; AVX512_64-LABEL: d_to_s32:
+; AVX512_64:       # %bb.0:
+; AVX512_64-NEXT:    vcvttsd2si %xmm0, %eax
+; AVX512_64-NEXT:    retq
+;
+; SSE3_32-LABEL: d_to_s32:
+; SSE3_32:       # %bb.0:
+; SSE3_32-NEXT:    cvttsd2si {{[0-9]+}}(%esp), %eax
+; SSE3_32-NEXT:    retl
+;
+; SSE3_64-LABEL: d_to_s32:
+; SSE3_64:       # %bb.0:
+; SSE3_64-NEXT:    cvttsd2si %xmm0, %eax
+; SSE3_64-NEXT:    retq
+;
+; SSE2_32-LABEL: d_to_s32:
+; SSE2_32:       # %bb.0:
+; SSE2_32-NEXT:    cvttsd2si {{[0-9]+}}(%esp), %eax
+; SSE2_32-NEXT:    retl
+;
+; SSE2_64-LABEL: d_to_s32:
+; SSE2_64:       # %bb.0:
+; SSE2_64-NEXT:    cvttsd2si %xmm0, %eax
+; SSE2_64-NEXT:    retq
+;
+; SSE_32-LABEL: d_to_s32:
+; SSE_32:       # %bb.0:
+; SSE_32-NEXT:    subl $8, %esp
+; SSE_32-NEXT:    fldl {{[0-9]+}}(%esp)
+; SSE_32-NEXT:    fnstcw (%esp)
+; SSE_32-NEXT:    movzwl (%esp), %eax
+; SSE_32-NEXT:    orl $3072, %eax # imm = 0xC00
+; SSE_32-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; SSE_32-NEXT:    fldcw {{[0-9]+}}(%esp)
+; SSE_32-NEXT:    fistpl {{[0-9]+}}(%esp)
+; SSE_32-NEXT:    fldcw (%esp)
+; SSE_32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; SSE_32-NEXT:    addl $8, %esp
+; SSE_32-NEXT:    retl
+;
+; X87-LABEL: d_to_s32:
+; X87:       # %bb.0:
+; X87-NEXT:    subl $8, %esp
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    fnstcw (%esp)
+; X87-NEXT:    movzwl (%esp), %eax
+; X87-NEXT:    orl $3072, %eax # imm = 0xC00
+; X87-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; X87-NEXT:    fldcw {{[0-9]+}}(%esp)
+; X87-NEXT:    fistpl {{[0-9]+}}(%esp)
+; X87-NEXT:    fldcw (%esp)
+; X87-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    addl $8, %esp
+; X87-NEXT:    retl
+  %r = fptosi double %a to i32
+  ret i32 %r
+}
+
+define i32 @x_to_u32(x86_fp80 %a) nounwind {
+; AVX512_32_WIN-LABEL: x_to_u32:
+; AVX512_32_WIN:       # %bb.0:
+; AVX512_32_WIN-NEXT:    pushl %ebp
+; AVX512_32_WIN-NEXT:    movl %esp, %ebp
+; AVX512_32_WIN-NEXT:    andl $-8, %esp
+; AVX512_32_WIN-NEXT:    subl $8, %esp
+; AVX512_32_WIN-NEXT:    fldt 8(%ebp)
+; AVX512_32_WIN-NEXT:    fisttpll (%esp)
+; AVX512_32_WIN-NEXT:    movl (%esp), %eax
+; AVX512_32_WIN-NEXT:    movl %ebp, %esp
+; AVX512_32_WIN-NEXT:    popl %ebp
+; AVX512_32_WIN-NEXT:    retl
+;
+; AVX512_32_LIN-LABEL: x_to_u32:
+; AVX512_32_LIN:       # %bb.0:
+; AVX512_32_LIN-NEXT:    subl $12, %esp
+; AVX512_32_LIN-NEXT:    fldt {{[0-9]+}}(%esp)
+; AVX512_32_LIN-NEXT:    fisttpll (%esp)
+; AVX512_32_LIN-NEXT:    movl (%esp), %eax
+; AVX512_32_LIN-NEXT:    addl $12, %esp
+; AVX512_32_LIN-NEXT:    retl
+;
+; AVX512_64_WIN-LABEL: x_to_u32:
+; AVX512_64_WIN:       # %bb.0:
+; AVX512_64_WIN-NEXT:    pushq %rax
+; AVX512_64_WIN-NEXT:    fldt (%rcx)
+; AVX512_64_WIN-NEXT:    fisttpll (%rsp)
+; AVX512_64_WIN-NEXT:    movl (%rsp), %eax
+; AVX512_64_WIN-NEXT:    popq %rcx
+; AVX512_64_WIN-NEXT:    retq
+;
+; AVX512_64_LIN-LABEL: x_to_u32:
+; AVX512_64_LIN:       # %bb.0:
+; AVX512_64_LIN-NEXT:    fldt {{[0-9]+}}(%rsp)
+; AVX512_64_LIN-NEXT:    fisttpll -{{[0-9]+}}(%rsp)
+; AVX512_64_LIN-NEXT:    movl -{{[0-9]+}}(%rsp), %eax
+; AVX512_64_LIN-NEXT:    retq
+;
+; SSE3_32_WIN-LABEL: x_to_u32:
+; SSE3_32_WIN:       # %bb.0:
+; SSE3_32_WIN-NEXT:    pushl %ebp
+; SSE3_32_WIN-NEXT:    movl %esp, %ebp
+; SSE3_32_WIN-NEXT:    andl $-8, %esp
+; SSE3_32_WIN-NEXT:    subl $8, %esp
+; SSE3_32_WIN-NEXT:    fldt 8(%ebp)
+; SSE3_32_WIN-NEXT:    fisttpll (%esp)
+; SSE3_32_WIN-NEXT:    movl (%esp), %eax
+; SSE3_32_WIN-NEXT:    movl %ebp, %esp
+; SSE3_32_WIN-NEXT:    popl %ebp
+; SSE3_32_WIN-NEXT:    retl
+;
+; SSE3_32_LIN-LABEL: x_to_u32:
+; SSE3_32_LIN:       # %bb.0:
+; SSE3_32_LIN-NEXT:    subl $12, %esp
+; SSE3_32_LIN-NEXT:    fldt {{[0-9]+}}(%esp)
+; SSE3_32_LIN-NEXT:    fisttpll (%esp)
+; SSE3_32_LIN-NEXT:    movl (%esp), %eax
+; SSE3_32_LIN-NEXT:    addl $12, %esp
+; SSE3_32_LIN-NEXT:    retl
+;
+; SSE3_64_WIN-LABEL: x_to_u32:
+; SSE3_64_WIN:       # %bb.0:
+; SSE3_64_WIN-NEXT:    pushq %rax
+; SSE3_64_WIN-NEXT:    fldt (%rcx)
+; SSE3_64_WIN-NEXT:    fisttpll (%rsp)
+; SSE3_64_WIN-NEXT:    movl (%rsp), %eax
+; SSE3_64_WIN-NEXT:    popq %rcx
+; SSE3_64_WIN-NEXT:    retq
+;
+; SSE3_64_LIN-LABEL: x_to_u32:
+; SSE3_64_LIN:       # %bb.0:
+; SSE3_64_LIN-NEXT:    fldt {{[0-9]+}}(%rsp)
+; SSE3_64_LIN-NEXT:    fisttpll -{{[0-9]+}}(%rsp)
+; SSE3_64_LIN-NEXT:    movl -{{[0-9]+}}(%rsp), %eax
+; SSE3_64_LIN-NEXT:    retq
+;
+; SSE2_32_WIN-LABEL: x_to_u32:
+; SSE2_32_WIN:       # %bb.0:
+; SSE2_32_WIN-NEXT:    pushl %ebp
+; SSE2_32_WIN-NEXT:    movl %esp, %ebp
+; SSE2_32_WIN-NEXT:    andl $-8, %esp
+; SSE2_32_WIN-NEXT:    subl $16, %esp
+; SSE2_32_WIN-NEXT:    fldt 8(%ebp)
+; SSE2_32_WIN-NEXT:    fnstcw {{[0-9]+}}(%esp)
+; SSE2_32_WIN-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; SSE2_32_WIN-NEXT:    orl $3072, %eax # imm = 0xC00
+; SSE2_32_WIN-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; SSE2_32_WIN-NEXT:    fldcw {{[0-9]+}}(%esp)
+; SSE2_32_WIN-NEXT:    fistpll {{[0-9]+}}(%esp)
+; SSE2_32_WIN-NEXT:    fldcw {{[0-9]+}}(%esp)
+; SSE2_32_WIN-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; SSE2_32_WIN-NEXT:    movl %ebp, %esp
+; SSE2_32_WIN-NEXT:    popl %ebp
+; SSE2_32_WIN-NEXT:    retl
+;
+; SSE2_32_LIN-LABEL: x_to_u32:
+; SSE2_32_LIN:       # %bb.0:
+; SSE2_32_LIN-NEXT:    subl $20, %esp
+; SSE2_32_LIN-NEXT:    fldt {{[0-9]+}}(%esp)
+; SSE2_32_LIN-NEXT:    fnstcw {{[0-9]+}}(%esp)
+; SSE2_32_LIN-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; SSE2_32_LIN-NEXT:    orl $3072, %eax # imm = 0xC00
+; SSE2_32_LIN-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; SSE2_32_LIN-NEXT:    fldcw {{[0-9]+}}(%esp)
+; SSE2_32_LIN-NEXT:    fistpll {{[0-9]+}}(%esp)
+; SSE2_32_LIN-NEXT:    fldcw {{[0-9]+}}(%esp)
+; SSE2_32_LIN-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; SSE2_32_LIN-NEXT:    addl $20, %esp
+; SSE2_32_LIN-NEXT:    retl
+;
+; SSE2_64_WIN-LABEL: x_to_u32:
+; SSE2_64_WIN:       # %bb.0:
+; SSE2_64_WIN-NEXT:    subq $16, %rsp
+; SSE2_64_WIN-NEXT:    fldt (%rcx)
+; SSE2_64_WIN-NEXT:    fnstcw {{[0-9]+}}(%rsp)
+; SSE2_64_WIN-NEXT:    movzwl {{[0-9]+}}(%rsp), %eax
+; SSE2_64_WIN-NEXT:    orl $3072, %eax # imm = 0xC00
+; SSE2_64_WIN-NEXT:    movw %ax, {{[0-9]+}}(%rsp)
+; SSE2_64_WIN-NEXT:    fldcw {{[0-9]+}}(%rsp)
+; SSE2_64_WIN-NEXT:    fistpll {{[0-9]+}}(%rsp)
+; SSE2_64_WIN-NEXT:    fldcw {{[0-9]+}}(%rsp)
+; SSE2_64_WIN-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; SSE2_64_WIN-NEXT:    addq $16, %rsp
+; SSE2_64_WIN-NEXT:    retq
+;
+; SSE2_64_LIN-LABEL: x_to_u32:
+; SSE2_64_LIN:       # %bb.0:
+; SSE2_64_LIN-NEXT:    fldt {{[0-9]+}}(%rsp)
+; SSE2_64_LIN-NEXT:    fnstcw -{{[0-9]+}}(%rsp)
+; SSE2_64_LIN-NEXT:    movzwl -{{[0-9]+}}(%rsp), %eax
+; SSE2_64_LIN-NEXT:    orl $3072, %eax # imm = 0xC00
+; SSE2_64_LIN-NEXT:    movw %ax, -{{[0-9]+}}(%rsp)
+; SSE2_64_LIN-NEXT:    fldcw -{{[0-9]+}}(%rsp)
+; SSE2_64_LIN-NEXT:    fistpll -{{[0-9]+}}(%rsp)
+; SSE2_64_LIN-NEXT:    fldcw -{{[0-9]+}}(%rsp)
+; SSE2_64_LIN-NEXT:    movl -{{[0-9]+}}(%rsp), %eax
+; SSE2_64_LIN-NEXT:    retq
+;
+; SSE_32_WIN-LABEL: x_to_u32:
+; SSE_32_WIN:       # %bb.0:
+; SSE_32_WIN-NEXT:    pushl %ebp
+; SSE_32_WIN-NEXT:    movl %esp, %ebp
+; SSE_32_WIN-NEXT:    andl $-8, %esp
+; SSE_32_WIN-NEXT:    subl $16, %esp
+; SSE_32_WIN-NEXT:    fldt 8(%ebp)
+; SSE_32_WIN-NEXT:    fnstcw {{[0-9]+}}(%esp)
+; SSE_32_WIN-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; SSE_32_WIN-NEXT:    orl $3072, %eax # imm = 0xC00
+; SSE_32_WIN-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; SSE_32_WIN-NEXT:    fldcw {{[0-9]+}}(%esp)
+; SSE_32_WIN-NEXT:    fistpll {{[0-9]+}}(%esp)
+; SSE_32_WIN-NEXT:    fldcw {{[0-9]+}}(%esp)
+; SSE_32_WIN-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; SSE_32_WIN-NEXT:    movl %ebp, %esp
+; SSE_32_WIN-NEXT:    popl %ebp
+; SSE_32_WIN-NEXT:    retl
+;
+; SSE_32_LIN-LABEL: x_to_u32:
+; SSE_32_LIN:       # %bb.0:
+; SSE_32_LIN-NEXT:    subl $20, %esp
+; SSE_32_LIN-NEXT:    fldt {{[0-9]+}}(%esp)
+; SSE_32_LIN-NEXT:    fnstcw {{[0-9]+}}(%esp)
+; SSE_32_LIN-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; SSE_32_LIN-NEXT:    orl $3072, %eax # imm = 0xC00
+; SSE_32_LIN-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; SSE_32_LIN-NEXT:    fldcw {{[0-9]+}}(%esp)
+; SSE_32_LIN-NEXT:    fistpll {{[0-9]+}}(%esp)
+; SSE_32_LIN-NEXT:    fldcw {{[0-9]+}}(%esp)
+; SSE_32_LIN-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; SSE_32_LIN-NEXT:    addl $20, %esp
+; SSE_32_LIN-NEXT:    retl
+;
+; X87_WIN-LABEL: x_to_u32:
+; X87_WIN:       # %bb.0:
+; X87_WIN-NEXT:    pushl %ebp
+; X87_WIN-NEXT:    movl %esp, %ebp
+; X87_WIN-NEXT:    andl $-8, %esp
+; X87_WIN-NEXT:    subl $16, %esp
+; X87_WIN-NEXT:    fldt 8(%ebp)
+; X87_WIN-NEXT:    fnstcw {{[0-9]+}}(%esp)
+; X87_WIN-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X87_WIN-NEXT:    orl $3072, %eax # imm = 0xC00
+; X87_WIN-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; X87_WIN-NEXT:    fldcw {{[0-9]+}}(%esp)
+; X87_WIN-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X87_WIN-NEXT:    fldcw {{[0-9]+}}(%esp)
+; X87_WIN-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X87_WIN-NEXT:    movl %ebp, %esp
+; X87_WIN-NEXT:    popl %ebp
+; X87_WIN-NEXT:    retl
+;
+; X87_LIN-LABEL: x_to_u32:
+; X87_LIN:       # %bb.0:
+; X87_LIN-NEXT:    subl $20, %esp
+; X87_LIN-NEXT:    fldt {{[0-9]+}}(%esp)
+; X87_LIN-NEXT:    fnstcw {{[0-9]+}}(%esp)
+; X87_LIN-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X87_LIN-NEXT:    orl $3072, %eax # imm = 0xC00
+; X87_LIN-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; X87_LIN-NEXT:    fldcw {{[0-9]+}}(%esp)
+; X87_LIN-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X87_LIN-NEXT:    fldcw {{[0-9]+}}(%esp)
+; X87_LIN-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X87_LIN-NEXT:    addl $20, %esp
+; X87_LIN-NEXT:    retl
+  %r = fptoui x86_fp80 %a to i32
+  ret i32 %r
+}
+
+define i32 @x_to_s32(x86_fp80 %a) nounwind {
+; AVX512_32-LABEL: x_to_s32:
+; AVX512_32:       # %bb.0:
+; AVX512_32-NEXT:    pushl %eax
+; AVX512_32-NEXT:    fldt {{[0-9]+}}(%esp)
+; AVX512_32-NEXT:    fisttpl (%esp)
+; AVX512_32-NEXT:    movl (%esp), %eax
+; AVX512_32-NEXT:    popl %ecx
+; AVX512_32-NEXT:    retl
+;
+; AVX512_64_WIN-LABEL: x_to_s32:
+; AVX512_64_WIN:       # %bb.0:
+; AVX512_64_WIN-NEXT:    pushq %rax
+; AVX512_64_WIN-NEXT:    fldt (%rcx)
+; AVX512_64_WIN-NEXT:    fisttpl {{[0-9]+}}(%rsp)
+; AVX512_64_WIN-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; AVX512_64_WIN-NEXT:    popq %rcx
+; AVX512_64_WIN-NEXT:    retq
+;
+; AVX512_64_LIN-LABEL: x_to_s32:
+; AVX512_64_LIN:       # %bb.0:
+; AVX512_64_LIN-NEXT:    fldt {{[0-9]+}}(%rsp)
+; AVX512_64_LIN-NEXT:    fisttpl -{{[0-9]+}}(%rsp)
+; AVX512_64_LIN-NEXT:    movl -{{[0-9]+}}(%rsp), %eax
+; AVX512_64_LIN-NEXT:    retq
+;
+; SSE3_32-LABEL: x_to_s32:
+; SSE3_32:       # %bb.0:
+; SSE3_32-NEXT:    pushl %eax
+; SSE3_32-NEXT:    fldt {{[0-9]+}}(%esp)
+; SSE3_32-NEXT:    fisttpl (%esp)
+; SSE3_32-NEXT:    movl (%esp), %eax
+; SSE3_32-NEXT:    popl %ecx
+; SSE3_32-NEXT:    retl
+;
+; SSE3_64_WIN-LABEL: x_to_s32:
+; SSE3_64_WIN:       # %bb.0:
+; SSE3_64_WIN-NEXT:    pushq %rax
+; SSE3_64_WIN-NEXT:    fldt (%rcx)
+; SSE3_64_WIN-NEXT:    fisttpl {{[0-9]+}}(%rsp)
+; SSE3_64_WIN-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; SSE3_64_WIN-NEXT:    popq %rcx
+; SSE3_64_WIN-NEXT:    retq
+;
+; SSE3_64_LIN-LABEL: x_to_s32:
+; SSE3_64_LIN:       # %bb.0:
+; SSE3_64_LIN-NEXT:    fldt {{[0-9]+}}(%rsp)
+; SSE3_64_LIN-NEXT:    fisttpl -{{[0-9]+}}(%rsp)
+; SSE3_64_LIN-NEXT:    movl -{{[0-9]+}}(%rsp), %eax
+; SSE3_64_LIN-NEXT:    retq
+;
+; SSE2_32-LABEL: x_to_s32:
+; SSE2_32:       # %bb.0:
+; SSE2_32-NEXT:    subl $8, %esp
+; SSE2_32-NEXT:    fldt {{[0-9]+}}(%esp)
+; SSE2_32-NEXT:    fnstcw (%esp)
+; SSE2_32-NEXT:    movzwl (%esp), %eax
+; SSE2_32-NEXT:    orl $3072, %eax # imm = 0xC00
+; SSE2_32-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; SSE2_32-NEXT:    fldcw {{[0-9]+}}(%esp)
+; SSE2_32-NEXT:    fistpl {{[0-9]+}}(%esp)
+; SSE2_32-NEXT:    fldcw (%esp)
+; SSE2_32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; SSE2_32-NEXT:    addl $8, %esp
+; SSE2_32-NEXT:    retl
+;
+; SSE2_64_WIN-LABEL: x_to_s32:
+; SSE2_64_WIN:       # %bb.0:
+; SSE2_64_WIN-NEXT:    pushq %rax
+; SSE2_64_WIN-NEXT:    fldt (%rcx)
+; SSE2_64_WIN-NEXT:    fnstcw (%rsp)
+; SSE2_64_WIN-NEXT:    movzwl (%rsp), %eax
+; SSE2_64_WIN-NEXT:    orl $3072, %eax # imm = 0xC00
+; SSE2_64_WIN-NEXT:    movw %ax, {{[0-9]+}}(%rsp)
+; SSE2_64_WIN-NEXT:    fldcw {{[0-9]+}}(%rsp)
+; SSE2_64_WIN-NEXT:    fistpl {{[0-9]+}}(%rsp)
+; SSE2_64_WIN-NEXT:    fldcw (%rsp)
+; SSE2_64_WIN-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; SSE2_64_WIN-NEXT:    popq %rcx
+; SSE2_64_WIN-NEXT:    retq
+;
+; SSE2_64_LIN-LABEL: x_to_s32:
+; SSE2_64_LIN:       # %bb.0:
+; SSE2_64_LIN-NEXT:    fldt {{[0-9]+}}(%rsp)
+; SSE2_64_LIN-NEXT:    fnstcw -{{[0-9]+}}(%rsp)
+; SSE2_64_LIN-NEXT:    movzwl -{{[0-9]+}}(%rsp), %eax
+; SSE2_64_LIN-NEXT:    orl $3072, %eax # imm = 0xC00
+; SSE2_64_LIN-NEXT:    movw %ax, -{{[0-9]+}}(%rsp)
+; SSE2_64_LIN-NEXT:    fldcw -{{[0-9]+}}(%rsp)
+; SSE2_64_LIN-NEXT:    fistpl -{{[0-9]+}}(%rsp)
+; SSE2_64_LIN-NEXT:    fldcw -{{[0-9]+}}(%rsp)
+; SSE2_64_LIN-NEXT:    movl -{{[0-9]+}}(%rsp), %eax
+; SSE2_64_LIN-NEXT:    retq
+;
+; SSE_32-LABEL: x_to_s32:
+; SSE_32:       # %bb.0:
+; SSE_32-NEXT:    subl $8, %esp
+; SSE_32-NEXT:    fldt {{[0-9]+}}(%esp)
+; SSE_32-NEXT:    fnstcw (%esp)
+; SSE_32-NEXT:    movzwl (%esp), %eax
+; SSE_32-NEXT:    orl $3072, %eax # imm = 0xC00
+; SSE_32-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; SSE_32-NEXT:    fldcw {{[0-9]+}}(%esp)
+; SSE_32-NEXT:    fistpl {{[0-9]+}}(%esp)
+; SSE_32-NEXT:    fldcw (%esp)
+; SSE_32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; SSE_32-NEXT:    addl $8, %esp
+; SSE_32-NEXT:    retl
+;
+; X87-LABEL: x_to_s32:
+; X87:       # %bb.0:
+; X87-NEXT:    subl $8, %esp
+; X87-NEXT:    fldt {{[0-9]+}}(%esp)
+; X87-NEXT:    fnstcw (%esp)
+; X87-NEXT:    movzwl (%esp), %eax
+; X87-NEXT:    orl $3072, %eax # imm = 0xC00
+; X87-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; X87-NEXT:    fldcw {{[0-9]+}}(%esp)
+; X87-NEXT:    fistpl {{[0-9]+}}(%esp)
+; X87-NEXT:    fldcw (%esp)
+; X87-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X87-NEXT:    addl $8, %esp
+; X87-NEXT:    retl
+  %r = fptosi x86_fp80 %a to i32
+  ret i32 %r
+}
+
+define i32 @t_to_u32(fp128 %a) nounwind {
+; AVX512_32_WIN-LABEL: t_to_u32:
+; AVX512_32_WIN:       # %bb.0:
+; AVX512_32_WIN-NEXT:    subl $16, %esp
+; AVX512_32_WIN-NEXT:    vmovups {{[0-9]+}}(%esp), %xmm0
+; AVX512_32_WIN-NEXT:    vmovups %xmm0, (%esp)
+; AVX512_32_WIN-NEXT:    calll ___fixunstfsi
+; AVX512_32_WIN-NEXT:    addl $16, %esp
+; AVX512_32_WIN-NEXT:    retl
+;
+; AVX512_32_LIN-LABEL: t_to_u32:
+; AVX512_32_LIN:       # %bb.0:
+; AVX512_32_LIN-NEXT:    subl $28, %esp
+; AVX512_32_LIN-NEXT:    vmovaps {{[0-9]+}}(%esp), %xmm0
+; AVX512_32_LIN-NEXT:    vmovups %xmm0, (%esp)
+; AVX512_32_LIN-NEXT:    calll __fixunstfsi
+; AVX512_32_LIN-NEXT:    addl $28, %esp
+; AVX512_32_LIN-NEXT:    retl
+;
+; AVX512_64_WIN-LABEL: t_to_u32:
+; AVX512_64_WIN:       # %bb.0:
+; AVX512_64_WIN-NEXT:    subq $40, %rsp
+; AVX512_64_WIN-NEXT:    callq __fixunstfsi
+; AVX512_64_WIN-NEXT:    addq $40, %rsp
+; AVX512_64_WIN-NEXT:    retq
+;
+; AVX512_64_LIN-LABEL: t_to_u32:
+; AVX512_64_LIN:       # %bb.0:
+; AVX512_64_LIN-NEXT:    pushq %rax
+; AVX512_64_LIN-NEXT:    callq __fixunstfsi
+; AVX512_64_LIN-NEXT:    popq %rcx
+; AVX512_64_LIN-NEXT:    retq
+;
+; SSE3_32_WIN-LABEL: t_to_u32:
+; SSE3_32_WIN:       # %bb.0:
+; SSE3_32_WIN-NEXT:    pushl {{[0-9]+}}(%esp)
+; SSE3_32_WIN-NEXT:    pushl {{[0-9]+}}(%esp)
+; SSE3_32_WIN-NEXT:    pushl {{[0-9]+}}(%esp)
+; SSE3_32_WIN-NEXT:    pushl {{[0-9]+}}(%esp)
+; SSE3_32_WIN-NEXT:    calll ___fixunstfsi
+; SSE3_32_WIN-NEXT:    addl $16, %esp
+; SSE3_32_WIN-NEXT:    retl
+;
+; SSE3_32_LIN-LABEL: t_to_u32:
+; SSE3_32_LIN:       # %bb.0:
+; SSE3_32_LIN-NEXT:    subl $12, %esp
+; SSE3_32_LIN-NEXT:    pushl {{[0-9]+}}(%esp)
+; SSE3_32_LIN-NEXT:    pushl {{[0-9]+}}(%esp)
+; SSE3_32_LIN-NEXT:    pushl {{[0-9]+}}(%esp)
+; SSE3_32_LIN-NEXT:    pushl {{[0-9]+}}(%esp)
+; SSE3_32_LIN-NEXT:    calll __fixunstfsi
+; SSE3_32_LIN-NEXT:    addl $28, %esp
+; SSE3_32_LIN-NEXT:    retl
+;
+; SSE3_64_WIN-LABEL: t_to_u32:
+; SSE3_64_WIN:       # %bb.0:
+; SSE3_64_WIN-NEXT:    subq $40, %rsp
+; SSE3_64_WIN-NEXT:    callq __fixunstfsi
+; SSE3_64_WIN-NEXT:    addq $40, %rsp
+; SSE3_64_WIN-NEXT:    retq
+;
+; SSE3_64_LIN-LABEL: t_to_u32:
+; SSE3_64_LIN:       # %bb.0:
+; SSE3_64_LIN-NEXT:    pushq %rax
+; SSE3_64_LIN-NEXT:    callq __fixunstfsi
+; SSE3_64_LIN-NEXT:    popq %rcx
+; SSE3_64_LIN-NEXT:    retq
+;
+; SSE2_32_WIN-LABEL: t_to_u32:
+; SSE2_32_WIN:       # %bb.0:
+; SSE2_32_WIN-NEXT:    pushl {{[0-9]+}}(%esp)
+; SSE2_32_WIN-NEXT:    pushl {{[0-9]+}}(%esp)
+; SSE2_32_WIN-NEXT:    pushl {{[0-9]+}}(%esp)
+; SSE2_32_WIN-NEXT:    pushl {{[0-9]+}}(%esp)
+; SSE2_32_WIN-NEXT:    calll ___fixunstfsi
+; SSE2_32_WIN-NEXT:    addl $16, %esp
+; SSE2_32_WIN-NEXT:    retl
+;
+; SSE2_32_LIN-LABEL: t_to_u32:
+; SSE2_32_LIN:       # %bb.0:
+; SSE2_32_LIN-NEXT:    subl $12, %esp
+; SSE2_32_LIN-NEXT:    pushl {{[0-9]+}}(%esp)
+; SSE2_32_LIN-NEXT:    pushl {{[0-9]+}}(%esp)
+; SSE2_32_LIN-NEXT:    pushl {{[0-9]+}}(%esp)
+; SSE2_32_LIN-NEXT:    pushl {{[0-9]+}}(%esp)
+; SSE2_32_LIN-NEXT:    calll __fixunstfsi
+; SSE2_32_LIN-NEXT:    addl $28, %esp
+; SSE2_32_LIN-NEXT:    retl
+;
+; SSE2_64_WIN-LABEL: t_to_u32:
+; SSE2_64_WIN:       # %bb.0:
+; SSE2_64_WIN-NEXT:    subq $40, %rsp
+; SSE2_64_WIN-NEXT:    callq __fixunstfsi
+; SSE2_64_WIN-NEXT:    addq $40, %rsp
+; SSE2_64_WIN-NEXT:    retq
+;
+; SSE2_64_LIN-LABEL: t_to_u32:
+; SSE2_64_LIN:       # %bb.0:
+; SSE2_64_LIN-NEXT:    pushq %rax
+; SSE2_64_LIN-NEXT:    callq __fixunstfsi
+; SSE2_64_LIN-NEXT:    popq %rcx
+; SSE2_64_LIN-NEXT:    retq
+;
+; SSE_32_WIN-LABEL: t_to_u32:
+; SSE_32_WIN:       # %bb.0:
+; SSE_32_WIN-NEXT:    pushl {{[0-9]+}}(%esp)
+; SSE_32_WIN-NEXT:    pushl {{[0-9]+}}(%esp)
+; SSE_32_WIN-NEXT:    pushl {{[0-9]+}}(%esp)
+; SSE_32_WIN-NEXT:    pushl {{[0-9]+}}(%esp)
+; SSE_32_WIN-NEXT:    calll ___fixunstfsi
+; SSE_32_WIN-NEXT:    addl $16, %esp
+; SSE_32_WIN-NEXT:    retl
+;
+; SSE_32_LIN-LABEL: t_to_u32:
+; SSE_32_LIN:       # %bb.0:
+; SSE_32_LIN-NEXT:    subl $12, %esp
+; SSE_32_LIN-NEXT:    pushl {{[0-9]+}}(%esp)
+; SSE_32_LIN-NEXT:    pushl {{[0-9]+}}(%esp)
+; SSE_32_LIN-NEXT:    pushl {{[0-9]+}}(%esp)
+; SSE_32_LIN-NEXT:    pushl {{[0-9]+}}(%esp)
+; SSE_32_LIN-NEXT:    calll __fixunstfsi
+; SSE_32_LIN-NEXT:    addl $28, %esp
+; SSE_32_LIN-NEXT:    retl
+;
+; X87_WIN-LABEL: t_to_u32:
+; X87_WIN:       # %bb.0:
+; X87_WIN-NEXT:    pushl {{[0-9]+}}(%esp)
+; X87_WIN-NEXT:    pushl {{[0-9]+}}(%esp)
+; X87_WIN-NEXT:    pushl {{[0-9]+}}(%esp)
+; X87_WIN-NEXT:    pushl {{[0-9]+}}(%esp)
+; X87_WIN-NEXT:    calll ___fixunstfsi
+; X87_WIN-NEXT:    addl $16, %esp
+; X87_WIN-NEXT:    retl
+;
+; X87_LIN-LABEL: t_to_u32:
+; X87_LIN:       # %bb.0:
+; X87_LIN-NEXT:    subl $12, %esp
+; X87_LIN-NEXT:    pushl {{[0-9]+}}(%esp)
+; X87_LIN-NEXT:    pushl {{[0-9]+}}(%esp)
+; X87_LIN-NEXT:    pushl {{[0-9]+}}(%esp)
+; X87_LIN-NEXT:    pushl {{[0-9]+}}(%esp)
+; X87_LIN-NEXT:    calll __fixunstfsi
+; X87_LIN-NEXT:    addl $28, %esp
+; X87_LIN-NEXT:    retl
+  %r = fptoui fp128 %a to i32
+  ret i32 %r
+}
+
+define i32 @t_to_s32(fp128 %a) nounwind {
+; AVX512_32_WIN-LABEL: t_to_s32:
+; AVX512_32_WIN:       # %bb.0:
+; AVX512_32_WIN-NEXT:    subl $16, %esp
+; AVX512_32_WIN-NEXT:    vmovups {{[0-9]+}}(%esp), %xmm0
+; AVX512_32_WIN-NEXT:    vmovups %xmm0, (%esp)
+; AVX512_32_WIN-NEXT:    calll ___fixtfsi
+; AVX512_32_WIN-NEXT:    addl $16, %esp
+; AVX512_32_WIN-NEXT:    retl
+;
+; AVX512_32_LIN-LABEL: t_to_s32:
+; AVX512_32_LIN:       # %bb.0:
+; AVX512_32_LIN-NEXT:    subl $28, %esp
+; AVX512_32_LIN-NEXT:    vmovaps {{[0-9]+}}(%esp), %xmm0
+; AVX512_32_LIN-NEXT:    vmovups %xmm0, (%esp)
+; AVX512_32_LIN-NEXT:    calll __fixtfsi
+; AVX512_32_LIN-NEXT:    addl $28, %esp
+; AVX512_32_LIN-NEXT:    retl
+;
+; AVX512_64_WIN-LABEL: t_to_s32:
+; AVX512_64_WIN:       # %bb.0:
+; AVX512_64_WIN-NEXT:    subq $40, %rsp
+; AVX512_64_WIN-NEXT:    callq __fixtfsi
+; AVX512_64_WIN-NEXT:    addq $40, %rsp
+; AVX512_64_WIN-NEXT:    retq
+;
+; AVX512_64_LIN-LABEL: t_to_s32:
+; AVX512_64_LIN:       # %bb.0:
+; AVX512_64_LIN-NEXT:    pushq %rax
+; AVX512_64_LIN-NEXT:    callq __fixtfsi
+; AVX512_64_LIN-NEXT:    popq %rcx
+; AVX512_64_LIN-NEXT:    retq
+;
+; SSE3_32_WIN-LABEL: t_to_s32:
+; SSE3_32_WIN:       # %bb.0:
+; SSE3_32_WIN-NEXT:    pushl {{[0-9]+}}(%esp)
+; SSE3_32_WIN-NEXT:    pushl {{[0-9]+}}(%esp)
+; SSE3_32_WIN-NEXT:    pushl {{[0-9]+}}(%esp)
+; SSE3_32_WIN-NEXT:    pushl {{[0-9]+}}(%esp)
+; SSE3_32_WIN-NEXT:    calll ___fixtfsi
+; SSE3_32_WIN-NEXT:    addl $16, %esp
+; SSE3_32_WIN-NEXT:    retl
+;
+; SSE3_32_LIN-LABEL: t_to_s32:
+; SSE3_32_LIN:       # %bb.0:
+; SSE3_32_LIN-NEXT:    subl $12, %esp
+; SSE3_32_LIN-NEXT:    pushl {{[0-9]+}}(%esp)
+; SSE3_32_LIN-NEXT:    pushl {{[0-9]+}}(%esp)
+; SSE3_32_LIN-NEXT:    pushl {{[0-9]+}}(%esp)
+; SSE3_32_LIN-NEXT:    pushl {{[0-9]+}}(%esp)
+; SSE3_32_LIN-NEXT:    calll __fixtfsi
+; SSE3_32_LIN-NEXT:    addl $28, %esp
+; SSE3_32_LIN-NEXT:    retl
+;
+; SSE3_64_WIN-LABEL: t_to_s32:
+; SSE3_64_WIN:       # %bb.0:
+; SSE3_64_WIN-NEXT:    subq $40, %rsp
+; SSE3_64_WIN-NEXT:    callq __fixtfsi
+; SSE3_64_WIN-NEXT:    addq $40, %rsp
+; SSE3_64_WIN-NEXT:    retq
+;
+; SSE3_64_LIN-LABEL: t_to_s32:
+; SSE3_64_LIN:       # %bb.0:
+; SSE3_64_LIN-NEXT:    pushq %rax
+; SSE3_64_LIN-NEXT:    callq __fixtfsi
+; SSE3_64_LIN-NEXT:    popq %rcx
+; SSE3_64_LIN-NEXT:    retq
+;
+; SSE2_32_WIN-LABEL: t_to_s32:
+; SSE2_32_WIN:       # %bb.0:
+; SSE2_32_WIN-NEXT:    pushl {{[0-9]+}}(%esp)
+; SSE2_32_WIN-NEXT:    pushl {{[0-9]+}}(%esp)
+; SSE2_32_WIN-NEXT:    pushl {{[0-9]+}}(%esp)
+; SSE2_32_WIN-NEXT:    pushl {{[0-9]+}}(%esp)
+; SSE2_32_WIN-NEXT:    calll ___fixtfsi
+; SSE2_32_WIN-NEXT:    addl $16, %esp
+; SSE2_32_WIN-NEXT:    retl
+;
+; SSE2_32_LIN-LABEL: t_to_s32:
+; SSE2_32_LIN:       # %bb.0:
+; SSE2_32_LIN-NEXT:    subl $12, %esp
+; SSE2_32_LIN-NEXT:    pushl {{[0-9]+}}(%esp)
+; SSE2_32_LIN-NEXT:    pushl {{[0-9]+}}(%esp)
+; SSE2_32_LIN-NEXT:    pushl {{[0-9]+}}(%esp)
+; SSE2_32_LIN-NEXT:    pushl {{[0-9]+}}(%esp)
+; SSE2_32_LIN-NEXT:    calll __fixtfsi
+; SSE2_32_LIN-NEXT:    addl $28, %esp
+; SSE2_32_LIN-NEXT:    retl
+;
+; SSE2_64_WIN-LABEL: t_to_s32:
+; SSE2_64_WIN:       # %bb.0:
+; SSE2_64_WIN-NEXT:    subq $40, %rsp
+; SSE2_64_WIN-NEXT:    callq __fixtfsi
+; SSE2_64_WIN-NEXT:    addq $40, %rsp
+; SSE2_64_WIN-NEXT:    retq
+;
+; SSE2_64_LIN-LABEL: t_to_s32:
+; SSE2_64_LIN:       # %bb.0:
+; SSE2_64_LIN-NEXT:    pushq %rax
+; SSE2_64_LIN-NEXT:    callq __fixtfsi
+; SSE2_64_LIN-NEXT:    popq %rcx
+; SSE2_64_LIN-NEXT:    retq
+;
+; SSE_32_WIN-LABEL: t_to_s32:
+; SSE_32_WIN:       # %bb.0:
+; SSE_32_WIN-NEXT:    pushl {{[0-9]+}}(%esp)
+; SSE_32_WIN-NEXT:    pushl {{[0-9]+}}(%esp)
+; SSE_32_WIN-NEXT:    pushl {{[0-9]+}}(%esp)
+; SSE_32_WIN-NEXT:    pushl {{[0-9]+}}(%esp)
+; SSE_32_WIN-NEXT:    calll ___fixtfsi
+; SSE_32_WIN-NEXT:    addl $16, %esp
+; SSE_32_WIN-NEXT:    retl
+;
+; SSE_32_LIN-LABEL: t_to_s32:
+; SSE_32_LIN:       # %bb.0:
+; SSE_32_LIN-NEXT:    subl $12, %esp
+; SSE_32_LIN-NEXT:    pushl {{[0-9]+}}(%esp)
+; SSE_32_LIN-NEXT:    pushl {{[0-9]+}}(%esp)
+; SSE_32_LIN-NEXT:    pushl {{[0-9]+}}(%esp)
+; SSE_32_LIN-NEXT:    pushl {{[0-9]+}}(%esp)
+; SSE_32_LIN-NEXT:    calll __fixtfsi
+; SSE_32_LIN-NEXT:    addl $28, %esp
+; SSE_32_LIN-NEXT:    retl
+;
+; X87_WIN-LABEL: t_to_s32:
+; X87_WIN:       # %bb.0:
+; X87_WIN-NEXT:    pushl {{[0-9]+}}(%esp)
+; X87_WIN-NEXT:    pushl {{[0-9]+}}(%esp)
+; X87_WIN-NEXT:    pushl {{[0-9]+}}(%esp)
+; X87_WIN-NEXT:    pushl {{[0-9]+}}(%esp)
+; X87_WIN-NEXT:    calll ___fixtfsi
+; X87_WIN-NEXT:    addl $16, %esp
+; X87_WIN-NEXT:    retl
+;
+; X87_LIN-LABEL: t_to_s32:
+; X87_LIN:       # %bb.0:
+; X87_LIN-NEXT:    subl $12, %esp
+; X87_LIN-NEXT:    pushl {{[0-9]+}}(%esp)
+; X87_LIN-NEXT:    pushl {{[0-9]+}}(%esp)
+; X87_LIN-NEXT:    pushl {{[0-9]+}}(%esp)
+; X87_LIN-NEXT:    pushl {{[0-9]+}}(%esp)
+; X87_LIN-NEXT:    calll __fixtfsi
+; X87_LIN-NEXT:    addl $28, %esp
+; X87_LIN-NEXT:    retl
+  %r = fptosi fp128 %a to i32
+  ret i32 %r
+}
diff --git a/llvm/test/CodeGen/X86/scalar-fp-to-i64.ll b/llvm/test/CodeGen/X86/scalar-fp-to-i64.ll
index 7dc44f182..30e499605 100644
--- a/llvm/test/CodeGen/X86/scalar-fp-to-i64.ll
+++ b/llvm/test/CodeGen/X86/scalar-fp-to-i64.ll
@@ -42,19 +42,17 @@ define i64 @f_to_u64(float %a) nounwind {
 ; AVX512DQVL_32_WIN-LABEL: f_to_u64:
 ; AVX512DQVL_32_WIN:       # %bb.0:
 ; AVX512DQVL_32_WIN-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX512DQVL_32_WIN-NEXT:    vcvttps2uqq %xmm0, %ymm0
+; AVX512DQVL_32_WIN-NEXT:    vcvttps2uqq %xmm0, %xmm0
 ; AVX512DQVL_32_WIN-NEXT:    vmovd %xmm0, %eax
 ; AVX512DQVL_32_WIN-NEXT:    vpextrd $1, %xmm0, %edx
-; AVX512DQVL_32_WIN-NEXT:    vzeroupper
 ; AVX512DQVL_32_WIN-NEXT:    retl
 ;
 ; AVX512DQVL_32_LIN-LABEL: f_to_u64:
 ; AVX512DQVL_32_LIN:       # %bb.0:
 ; AVX512DQVL_32_LIN-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX512DQVL_32_LIN-NEXT:    vcvttps2uqq %xmm0, %ymm0
+; AVX512DQVL_32_LIN-NEXT:    vcvttps2uqq %xmm0, %xmm0
 ; AVX512DQVL_32_LIN-NEXT:    vmovd %xmm0, %eax
 ; AVX512DQVL_32_LIN-NEXT:    vpextrd $1, %xmm0, %edx
-; AVX512DQVL_32_LIN-NEXT:    vzeroupper
 ; AVX512DQVL_32_LIN-NEXT:    retl
 ;
 ; AVX512_64-LABEL: f_to_u64:
@@ -89,13 +87,14 @@ define i64 @f_to_u64(float %a) nounwind {
 ; AVX512F_32_WIN-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; AVX512F_32_WIN-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; AVX512F_32_WIN-NEXT:    vcmpltss %xmm1, %xmm0, %k1
-; AVX512F_32_WIN-NEXT:    vsubss %xmm1, %xmm0, %xmm2
-; AVX512F_32_WIN-NEXT:    vmovss %xmm0, %xmm0, %xmm2 {%k1}
-; AVX512F_32_WIN-NEXT:    vmovss %xmm2, (%esp)
-; AVX512F_32_WIN-NEXT:    flds (%esp)
-; AVX512F_32_WIN-NEXT:    fisttpll (%esp)
+; AVX512F_32_WIN-NEXT:    vxorps %xmm2, %xmm2, %xmm2
 ; AVX512F_32_WIN-NEXT:    xorl %edx, %edx
 ; AVX512F_32_WIN-NEXT:    vucomiss %xmm0, %xmm1
+; AVX512F_32_WIN-NEXT:    vmovss %xmm2, %xmm1, %xmm1 {%k1}
+; AVX512F_32_WIN-NEXT:    vsubss %xmm1, %xmm0, %xmm0
+; AVX512F_32_WIN-NEXT:    vmovss %xmm0, (%esp)
+; AVX512F_32_WIN-NEXT:    flds (%esp)
+; AVX512F_32_WIN-NEXT:    fisttpll (%esp)
 ; AVX512F_32_WIN-NEXT:    setbe %dl
 ; AVX512F_32_WIN-NEXT:    shll $31, %edx
 ; AVX512F_32_WIN-NEXT:    xorl {{[0-9]+}}(%esp), %edx
@@ -110,13 +109,14 @@ define i64 @f_to_u64(float %a) nounwind {
 ; AVX512F_32_LIN-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; AVX512F_32_LIN-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; AVX512F_32_LIN-NEXT:    vcmpltss %xmm1, %xmm0, %k1
-; AVX512F_32_LIN-NEXT:    vsubss %xmm1, %xmm0, %xmm2
-; AVX512F_32_LIN-NEXT:    vmovss %xmm0, %xmm0, %xmm2 {%k1}
-; AVX512F_32_LIN-NEXT:    vmovss %xmm2, (%esp)
-; AVX512F_32_LIN-NEXT:    flds (%esp)
-; AVX512F_32_LIN-NEXT:    fisttpll (%esp)
+; AVX512F_32_LIN-NEXT:    vxorps %xmm2, %xmm2, %xmm2
 ; AVX512F_32_LIN-NEXT:    xorl %edx, %edx
 ; AVX512F_32_LIN-NEXT:    vucomiss %xmm0, %xmm1
+; AVX512F_32_LIN-NEXT:    vmovss %xmm2, %xmm1, %xmm1 {%k1}
+; AVX512F_32_LIN-NEXT:    vsubss %xmm1, %xmm0, %xmm0
+; AVX512F_32_LIN-NEXT:    vmovss %xmm0, (%esp)
+; AVX512F_32_LIN-NEXT:    flds (%esp)
+; AVX512F_32_LIN-NEXT:    fisttpll (%esp)
 ; AVX512F_32_LIN-NEXT:    setbe %dl
 ; AVX512F_32_LIN-NEXT:    shll $31, %edx
 ; AVX512F_32_LIN-NEXT:    xorl {{[0-9]+}}(%esp), %edx
@@ -133,14 +133,11 @@ define i64 @f_to_u64(float %a) nounwind {
 ; SSE3_32_WIN-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; SSE3_32_WIN-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; SSE3_32_WIN-NEXT:    movaps %xmm0, %xmm2
-; SSE3_32_WIN-NEXT:    cmpltss %xmm1, %xmm2
-; SSE3_32_WIN-NEXT:    movaps %xmm2, %xmm3
-; SSE3_32_WIN-NEXT:    andps %xmm0, %xmm2
 ; SSE3_32_WIN-NEXT:    xorl %edx, %edx
 ; SSE3_32_WIN-NEXT:    ucomiss %xmm0, %xmm1
-; SSE3_32_WIN-NEXT:    subss %xmm1, %xmm0
-; SSE3_32_WIN-NEXT:    andnps %xmm0, %xmm3
-; SSE3_32_WIN-NEXT:    orps %xmm3, %xmm2
+; SSE3_32_WIN-NEXT:    cmpltss %xmm1, %xmm0
+; SSE3_32_WIN-NEXT:    andnps %xmm1, %xmm0
+; SSE3_32_WIN-NEXT:    subss %xmm0, %xmm2
 ; SSE3_32_WIN-NEXT:    movss %xmm2, (%esp)
 ; SSE3_32_WIN-NEXT:    flds (%esp)
 ; SSE3_32_WIN-NEXT:    fisttpll (%esp)
@@ -158,14 +155,11 @@ define i64 @f_to_u64(float %a) nounwind {
 ; SSE3_32_LIN-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; SSE3_32_LIN-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; SSE3_32_LIN-NEXT:    movaps %xmm0, %xmm2
-; SSE3_32_LIN-NEXT:    cmpltss %xmm1, %xmm2
-; SSE3_32_LIN-NEXT:    movaps %xmm2, %xmm3
-; SSE3_32_LIN-NEXT:    andps %xmm0, %xmm2
 ; SSE3_32_LIN-NEXT:    xorl %edx, %edx
 ; SSE3_32_LIN-NEXT:    ucomiss %xmm0, %xmm1
-; SSE3_32_LIN-NEXT:    subss %xmm1, %xmm0
-; SSE3_32_LIN-NEXT:    andnps %xmm0, %xmm3
-; SSE3_32_LIN-NEXT:    orps %xmm3, %xmm2
+; SSE3_32_LIN-NEXT:    cmpltss %xmm1, %xmm0
+; SSE3_32_LIN-NEXT:    andnps %xmm1, %xmm0
+; SSE3_32_LIN-NEXT:    subss %xmm0, %xmm2
 ; SSE3_32_LIN-NEXT:    movss %xmm2, (%esp)
 ; SSE3_32_LIN-NEXT:    flds (%esp)
 ; SSE3_32_LIN-NEXT:    fisttpll (%esp)
@@ -198,13 +192,10 @@ define i64 @f_to_u64(float %a) nounwind {
 ; SSE2_32_WIN-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; SSE2_32_WIN-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; SSE2_32_WIN-NEXT:    movaps %xmm0, %xmm2
-; SSE2_32_WIN-NEXT:    subss %xmm1, %xmm2
+; SSE2_32_WIN-NEXT:    cmpltss %xmm1, %xmm2
+; SSE2_32_WIN-NEXT:    andnps %xmm1, %xmm2
 ; SSE2_32_WIN-NEXT:    movaps %xmm0, %xmm3
-; SSE2_32_WIN-NEXT:    cmpltss %xmm1, %xmm3
-; SSE2_32_WIN-NEXT:    movaps %xmm3, %xmm4
-; SSE2_32_WIN-NEXT:    andnps %xmm2, %xmm4
-; SSE2_32_WIN-NEXT:    andps %xmm0, %xmm3
-; SSE2_32_WIN-NEXT:    orps %xmm4, %xmm3
+; SSE2_32_WIN-NEXT:    subss %xmm2, %xmm3
 ; SSE2_32_WIN-NEXT:    movss %xmm3, {{[0-9]+}}(%esp)
 ; SSE2_32_WIN-NEXT:    flds {{[0-9]+}}(%esp)
 ; SSE2_32_WIN-NEXT:    fnstcw {{[0-9]+}}(%esp)
@@ -230,13 +221,10 @@ define i64 @f_to_u64(float %a) nounwind {
 ; SSE2_32_LIN-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; SSE2_32_LIN-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; SSE2_32_LIN-NEXT:    movaps %xmm0, %xmm2
-; SSE2_32_LIN-NEXT:    subss %xmm1, %xmm2
+; SSE2_32_LIN-NEXT:    cmpltss %xmm1, %xmm2
+; SSE2_32_LIN-NEXT:    andnps %xmm1, %xmm2
 ; SSE2_32_LIN-NEXT:    movaps %xmm0, %xmm3
-; SSE2_32_LIN-NEXT:    cmpltss %xmm1, %xmm3
-; SSE2_32_LIN-NEXT:    movaps %xmm3, %xmm4
-; SSE2_32_LIN-NEXT:    andnps %xmm2, %xmm4
-; SSE2_32_LIN-NEXT:    andps %xmm0, %xmm3
-; SSE2_32_LIN-NEXT:    orps %xmm4, %xmm3
+; SSE2_32_LIN-NEXT:    subss %xmm2, %xmm3
 ; SSE2_32_LIN-NEXT:    movss %xmm3, {{[0-9]+}}(%esp)
 ; SSE2_32_LIN-NEXT:    flds {{[0-9]+}}(%esp)
 ; SSE2_32_LIN-NEXT:    fnstcw {{[0-9]+}}(%esp)
@@ -276,20 +264,21 @@ define i64 @f_to_u64(float %a) nounwind {
 ; X87_WIN-NEXT:    subl $16, %esp
 ; X87_WIN-NEXT:    flds 8(%ebp)
 ; X87_WIN-NEXT:    flds __real@5f000000
-; X87_WIN-NEXT:    fld %st(1)
-; X87_WIN-NEXT:    fsub %st(1), %st
-; X87_WIN-NEXT:    fxch %st(1)
-; X87_WIN-NEXT:    fucomp %st(2)
+; X87_WIN-NEXT:    fucom %st(1)
 ; X87_WIN-NEXT:    fnstsw %ax
+; X87_WIN-NEXT:    xorl %edx, %edx
 ; X87_WIN-NEXT:    # kill: def $ah killed $ah killed $ax
 ; X87_WIN-NEXT:    sahf
+; X87_WIN-NEXT:    setbe %al
+; X87_WIN-NEXT:    fldz
 ; X87_WIN-NEXT:    ja LBB0_2
 ; X87_WIN-NEXT:  # %bb.1:
-; X87_WIN-NEXT:    fstp %st(1)
+; X87_WIN-NEXT:    fstp %st(0)
 ; X87_WIN-NEXT:    fldz
+; X87_WIN-NEXT:    fxch %st(1)
 ; X87_WIN-NEXT:  LBB0_2:
-; X87_WIN-NEXT:    fstp %st(0)
-; X87_WIN-NEXT:    setbe %al
+; X87_WIN-NEXT:    fstp %st(1)
+; X87_WIN-NEXT:    fsubrp %st, %st(1)
 ; X87_WIN-NEXT:    fnstcw {{[0-9]+}}(%esp)
 ; X87_WIN-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
 ; X87_WIN-NEXT:    orl $3072, %ecx # imm = 0xC00
@@ -297,7 +286,7 @@ define i64 @f_to_u64(float %a) nounwind {
 ; X87_WIN-NEXT:    fldcw {{[0-9]+}}(%esp)
 ; X87_WIN-NEXT:    fistpll {{[0-9]+}}(%esp)
 ; X87_WIN-NEXT:    fldcw {{[0-9]+}}(%esp)
-; X87_WIN-NEXT:    movzbl %al, %edx
+; X87_WIN-NEXT:    movb %al, %dl
 ; X87_WIN-NEXT:    shll $31, %edx
 ; X87_WIN-NEXT:    xorl {{[0-9]+}}(%esp), %edx
 ; X87_WIN-NEXT:    movl {{[0-9]+}}(%esp), %eax
@@ -310,20 +299,21 @@ define i64 @f_to_u64(float %a) nounwind {
 ; X87_LIN-NEXT:    subl $20, %esp
 ; X87_LIN-NEXT:    flds {{[0-9]+}}(%esp)
 ; X87_LIN-NEXT:    flds {{\.LCPI.*}}
-; X87_LIN-NEXT:    fld %st(1)
-; X87_LIN-NEXT:    fsub %st(1), %st
-; X87_LIN-NEXT:    fxch %st(1)
-; X87_LIN-NEXT:    fucomp %st(2)
+; X87_LIN-NEXT:    fucom %st(1)
 ; X87_LIN-NEXT:    fnstsw %ax
+; X87_LIN-NEXT:    xorl %edx, %edx
 ; X87_LIN-NEXT:    # kill: def $ah killed $ah killed $ax
 ; X87_LIN-NEXT:    sahf
+; X87_LIN-NEXT:    setbe %al
+; X87_LIN-NEXT:    fldz
 ; X87_LIN-NEXT:    ja .LBB0_2
 ; X87_LIN-NEXT:  # %bb.1:
-; X87_LIN-NEXT:    fstp %st(1)
+; X87_LIN-NEXT:    fstp %st(0)
 ; X87_LIN-NEXT:    fldz
+; X87_LIN-NEXT:    fxch %st(1)
 ; X87_LIN-NEXT:  .LBB0_2:
-; X87_LIN-NEXT:    fstp %st(0)
-; X87_LIN-NEXT:    setbe %al
+; X87_LIN-NEXT:    fstp %st(1)
+; X87_LIN-NEXT:    fsubrp %st, %st(1)
 ; X87_LIN-NEXT:    fnstcw {{[0-9]+}}(%esp)
 ; X87_LIN-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
 ; X87_LIN-NEXT:    orl $3072, %ecx # imm = 0xC00
@@ -331,7 +321,7 @@ define i64 @f_to_u64(float %a) nounwind {
 ; X87_LIN-NEXT:    fldcw {{[0-9]+}}(%esp)
 ; X87_LIN-NEXT:    fistpll {{[0-9]+}}(%esp)
 ; X87_LIN-NEXT:    fldcw {{[0-9]+}}(%esp)
-; X87_LIN-NEXT:    movzbl %al, %edx
+; X87_LIN-NEXT:    movb %al, %dl
 ; X87_LIN-NEXT:    shll $31, %edx
 ; X87_LIN-NEXT:    xorl {{[0-9]+}}(%esp), %edx
 ; X87_LIN-NEXT:    movl {{[0-9]+}}(%esp), %eax
@@ -345,19 +335,17 @@ define i64 @f_to_s64(float %a) nounwind {
 ; AVX512DQVL_32_WIN-LABEL: f_to_s64:
 ; AVX512DQVL_32_WIN:       # %bb.0:
 ; AVX512DQVL_32_WIN-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX512DQVL_32_WIN-NEXT:    vcvttps2qq %xmm0, %ymm0
+; AVX512DQVL_32_WIN-NEXT:    vcvttps2qq %xmm0, %xmm0
 ; AVX512DQVL_32_WIN-NEXT:    vmovd %xmm0, %eax
 ; AVX512DQVL_32_WIN-NEXT:    vpextrd $1, %xmm0, %edx
-; AVX512DQVL_32_WIN-NEXT:    vzeroupper
 ; AVX512DQVL_32_WIN-NEXT:    retl
 ;
 ; AVX512DQVL_32_LIN-LABEL: f_to_s64:
 ; AVX512DQVL_32_LIN:       # %bb.0:
 ; AVX512DQVL_32_LIN-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX512DQVL_32_LIN-NEXT:    vcvttps2qq %xmm0, %ymm0
+; AVX512DQVL_32_LIN-NEXT:    vcvttps2qq %xmm0, %xmm0
 ; AVX512DQVL_32_LIN-NEXT:    vmovd %xmm0, %eax
 ; AVX512DQVL_32_LIN-NEXT:    vpextrd $1, %xmm0, %edx
-; AVX512DQVL_32_LIN-NEXT:    vzeroupper
 ; AVX512DQVL_32_LIN-NEXT:    retl
 ;
 ; AVX512_64-LABEL: f_to_s64:
@@ -532,19 +520,17 @@ define i64 @d_to_u64(double %a) nounwind {
 ; AVX512DQVL_32_WIN-LABEL: d_to_u64:
 ; AVX512DQVL_32_WIN:       # %bb.0:
 ; AVX512DQVL_32_WIN-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; AVX512DQVL_32_WIN-NEXT:    vcvttpd2uqq %ymm0, %ymm0
+; AVX512DQVL_32_WIN-NEXT:    vcvttpd2uqq %xmm0, %xmm0
 ; AVX512DQVL_32_WIN-NEXT:    vmovd %xmm0, %eax
 ; AVX512DQVL_32_WIN-NEXT:    vpextrd $1, %xmm0, %edx
-; AVX512DQVL_32_WIN-NEXT:    vzeroupper
 ; AVX512DQVL_32_WIN-NEXT:    retl
 ;
 ; AVX512DQVL_32_LIN-LABEL: d_to_u64:
 ; AVX512DQVL_32_LIN:       # %bb.0:
 ; AVX512DQVL_32_LIN-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; AVX512DQVL_32_LIN-NEXT:    vcvttpd2uqq %ymm0, %ymm0
+; AVX512DQVL_32_LIN-NEXT:    vcvttpd2uqq %xmm0, %xmm0
 ; AVX512DQVL_32_LIN-NEXT:    vmovd %xmm0, %eax
 ; AVX512DQVL_32_LIN-NEXT:    vpextrd $1, %xmm0, %edx
-; AVX512DQVL_32_LIN-NEXT:    vzeroupper
 ; AVX512DQVL_32_LIN-NEXT:    retl
 ;
 ; AVX512_64-LABEL: d_to_u64:
@@ -579,13 +565,14 @@ define i64 @d_to_u64(double %a) nounwind {
 ; AVX512F_32_WIN-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; AVX512F_32_WIN-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
 ; AVX512F_32_WIN-NEXT:    vcmpltsd %xmm1, %xmm0, %k1
-; AVX512F_32_WIN-NEXT:    vsubsd %xmm1, %xmm0, %xmm2
-; AVX512F_32_WIN-NEXT:    vmovsd %xmm0, %xmm0, %xmm2 {%k1}
-; AVX512F_32_WIN-NEXT:    vmovsd %xmm2, (%esp)
-; AVX512F_32_WIN-NEXT:    fldl (%esp)
-; AVX512F_32_WIN-NEXT:    fisttpll (%esp)
+; AVX512F_32_WIN-NEXT:    vxorpd %xmm2, %xmm2, %xmm2
 ; AVX512F_32_WIN-NEXT:    xorl %edx, %edx
 ; AVX512F_32_WIN-NEXT:    vucomisd %xmm0, %xmm1
+; AVX512F_32_WIN-NEXT:    vmovsd %xmm2, %xmm1, %xmm1 {%k1}
+; AVX512F_32_WIN-NEXT:    vsubsd %xmm1, %xmm0, %xmm0
+; AVX512F_32_WIN-NEXT:    vmovsd %xmm0, (%esp)
+; AVX512F_32_WIN-NEXT:    fldl (%esp)
+; AVX512F_32_WIN-NEXT:    fisttpll (%esp)
 ; AVX512F_32_WIN-NEXT:    setbe %dl
 ; AVX512F_32_WIN-NEXT:    shll $31, %edx
 ; AVX512F_32_WIN-NEXT:    xorl {{[0-9]+}}(%esp), %edx
@@ -600,13 +587,14 @@ define i64 @d_to_u64(double %a) nounwind {
 ; AVX512F_32_LIN-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; AVX512F_32_LIN-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
 ; AVX512F_32_LIN-NEXT:    vcmpltsd %xmm1, %xmm0, %k1
-; AVX512F_32_LIN-NEXT:    vsubsd %xmm1, %xmm0, %xmm2
-; AVX512F_32_LIN-NEXT:    vmovsd %xmm0, %xmm0, %xmm2 {%k1}
-; AVX512F_32_LIN-NEXT:    vmovsd %xmm2, (%esp)
-; AVX512F_32_LIN-NEXT:    fldl (%esp)
-; AVX512F_32_LIN-NEXT:    fisttpll (%esp)
+; AVX512F_32_LIN-NEXT:    vxorpd %xmm2, %xmm2, %xmm2
 ; AVX512F_32_LIN-NEXT:    xorl %edx, %edx
 ; AVX512F_32_LIN-NEXT:    vucomisd %xmm0, %xmm1
+; AVX512F_32_LIN-NEXT:    vmovsd %xmm2, %xmm1, %xmm1 {%k1}
+; AVX512F_32_LIN-NEXT:    vsubsd %xmm1, %xmm0, %xmm0
+; AVX512F_32_LIN-NEXT:    vmovsd %xmm0, (%esp)
+; AVX512F_32_LIN-NEXT:    fldl (%esp)
+; AVX512F_32_LIN-NEXT:    fisttpll (%esp)
 ; AVX512F_32_LIN-NEXT:    setbe %dl
 ; AVX512F_32_LIN-NEXT:    shll $31, %edx
 ; AVX512F_32_LIN-NEXT:    xorl {{[0-9]+}}(%esp), %edx
@@ -623,15 +611,12 @@ define i64 @d_to_u64(double %a) nounwind {
 ; SSE3_32_WIN-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; SSE3_32_WIN-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
 ; SSE3_32_WIN-NEXT:    movapd %xmm0, %xmm2
-; SSE3_32_WIN-NEXT:    cmpltsd %xmm1, %xmm2
-; SSE3_32_WIN-NEXT:    movapd %xmm2, %xmm3
-; SSE3_32_WIN-NEXT:    andpd %xmm0, %xmm2
 ; SSE3_32_WIN-NEXT:    xorl %edx, %edx
 ; SSE3_32_WIN-NEXT:    ucomisd %xmm0, %xmm1
-; SSE3_32_WIN-NEXT:    subsd %xmm1, %xmm0
-; SSE3_32_WIN-NEXT:    andnpd %xmm0, %xmm3
-; SSE3_32_WIN-NEXT:    orpd %xmm3, %xmm2
-; SSE3_32_WIN-NEXT:    movlpd %xmm2, (%esp)
+; SSE3_32_WIN-NEXT:    cmpltsd %xmm1, %xmm0
+; SSE3_32_WIN-NEXT:    andnpd %xmm1, %xmm0
+; SSE3_32_WIN-NEXT:    subsd %xmm0, %xmm2
+; SSE3_32_WIN-NEXT:    movsd %xmm2, (%esp)
 ; SSE3_32_WIN-NEXT:    fldl (%esp)
 ; SSE3_32_WIN-NEXT:    fisttpll (%esp)
 ; SSE3_32_WIN-NEXT:    setbe %dl
@@ -648,15 +633,12 @@ define i64 @d_to_u64(double %a) nounwind {
 ; SSE3_32_LIN-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; SSE3_32_LIN-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
 ; SSE3_32_LIN-NEXT:    movapd %xmm0, %xmm2
-; SSE3_32_LIN-NEXT:    cmpltsd %xmm1, %xmm2
-; SSE3_32_LIN-NEXT:    movapd %xmm2, %xmm3
-; SSE3_32_LIN-NEXT:    andpd %xmm0, %xmm2
 ; SSE3_32_LIN-NEXT:    xorl %edx, %edx
 ; SSE3_32_LIN-NEXT:    ucomisd %xmm0, %xmm1
-; SSE3_32_LIN-NEXT:    subsd %xmm1, %xmm0
-; SSE3_32_LIN-NEXT:    andnpd %xmm0, %xmm3
-; SSE3_32_LIN-NEXT:    orpd %xmm3, %xmm2
-; SSE3_32_LIN-NEXT:    movlpd %xmm2, (%esp)
+; SSE3_32_LIN-NEXT:    cmpltsd %xmm1, %xmm0
+; SSE3_32_LIN-NEXT:    andnpd %xmm1, %xmm0
+; SSE3_32_LIN-NEXT:    subsd %xmm0, %xmm2
+; SSE3_32_LIN-NEXT:    movsd %xmm2, (%esp)
 ; SSE3_32_LIN-NEXT:    fldl (%esp)
 ; SSE3_32_LIN-NEXT:    fisttpll (%esp)
 ; SSE3_32_LIN-NEXT:    setbe %dl
@@ -688,14 +670,11 @@ define i64 @d_to_u64(double %a) nounwind {
 ; SSE2_32_WIN-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; SSE2_32_WIN-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
 ; SSE2_32_WIN-NEXT:    movapd %xmm0, %xmm2
-; SSE2_32_WIN-NEXT:    subsd %xmm1, %xmm2
+; SSE2_32_WIN-NEXT:    cmpltsd %xmm1, %xmm2
+; SSE2_32_WIN-NEXT:    andnpd %xmm1, %xmm2
 ; SSE2_32_WIN-NEXT:    movapd %xmm0, %xmm3
-; SSE2_32_WIN-NEXT:    cmpltsd %xmm1, %xmm3
-; SSE2_32_WIN-NEXT:    movapd %xmm3, %xmm4
-; SSE2_32_WIN-NEXT:    andnpd %xmm2, %xmm4
-; SSE2_32_WIN-NEXT:    andpd %xmm0, %xmm3
-; SSE2_32_WIN-NEXT:    orpd %xmm4, %xmm3
-; SSE2_32_WIN-NEXT:    movlpd %xmm3, {{[0-9]+}}(%esp)
+; SSE2_32_WIN-NEXT:    subsd %xmm2, %xmm3
+; SSE2_32_WIN-NEXT:    movsd %xmm3, {{[0-9]+}}(%esp)
 ; SSE2_32_WIN-NEXT:    fldl {{[0-9]+}}(%esp)
 ; SSE2_32_WIN-NEXT:    fnstcw {{[0-9]+}}(%esp)
 ; SSE2_32_WIN-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
@@ -720,14 +699,11 @@ define i64 @d_to_u64(double %a) nounwind {
 ; SSE2_32_LIN-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; SSE2_32_LIN-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
 ; SSE2_32_LIN-NEXT:    movapd %xmm0, %xmm2
-; SSE2_32_LIN-NEXT:    subsd %xmm1, %xmm2
+; SSE2_32_LIN-NEXT:    cmpltsd %xmm1, %xmm2
+; SSE2_32_LIN-NEXT:    andnpd %xmm1, %xmm2
 ; SSE2_32_LIN-NEXT:    movapd %xmm0, %xmm3
-; SSE2_32_LIN-NEXT:    cmpltsd %xmm1, %xmm3
-; SSE2_32_LIN-NEXT:    movapd %xmm3, %xmm4
-; SSE2_32_LIN-NEXT:    andnpd %xmm2, %xmm4
-; SSE2_32_LIN-NEXT:    andpd %xmm0, %xmm3
-; SSE2_32_LIN-NEXT:    orpd %xmm4, %xmm3
-; SSE2_32_LIN-NEXT:    movlpd %xmm3, {{[0-9]+}}(%esp)
+; SSE2_32_LIN-NEXT:    subsd %xmm2, %xmm3
+; SSE2_32_LIN-NEXT:    movsd %xmm3, {{[0-9]+}}(%esp)
 ; SSE2_32_LIN-NEXT:    fldl {{[0-9]+}}(%esp)
 ; SSE2_32_LIN-NEXT:    fnstcw {{[0-9]+}}(%esp)
 ; SSE2_32_LIN-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
@@ -766,20 +742,21 @@ define i64 @d_to_u64(double %a) nounwind {
 ; X87_WIN-NEXT:    subl $16, %esp
 ; X87_WIN-NEXT:    fldl 8(%ebp)
 ; X87_WIN-NEXT:    flds __real@5f000000
-; X87_WIN-NEXT:    fld %st(1)
-; X87_WIN-NEXT:    fsub %st(1), %st
-; X87_WIN-NEXT:    fxch %st(1)
-; X87_WIN-NEXT:    fucomp %st(2)
+; X87_WIN-NEXT:    fucom %st(1)
 ; X87_WIN-NEXT:    fnstsw %ax
+; X87_WIN-NEXT:    xorl %edx, %edx
 ; X87_WIN-NEXT:    # kill: def $ah killed $ah killed $ax
 ; X87_WIN-NEXT:    sahf
+; X87_WIN-NEXT:    setbe %al
+; X87_WIN-NEXT:    fldz
 ; X87_WIN-NEXT:    ja LBB2_2
 ; X87_WIN-NEXT:  # %bb.1:
-; X87_WIN-NEXT:    fstp %st(1)
+; X87_WIN-NEXT:    fstp %st(0)
 ; X87_WIN-NEXT:    fldz
+; X87_WIN-NEXT:    fxch %st(1)
 ; X87_WIN-NEXT:  LBB2_2:
-; X87_WIN-NEXT:    fstp %st(0)
-; X87_WIN-NEXT:    setbe %al
+; X87_WIN-NEXT:    fstp %st(1)
+; X87_WIN-NEXT:    fsubrp %st, %st(1)
 ; X87_WIN-NEXT:    fnstcw {{[0-9]+}}(%esp)
 ; X87_WIN-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
 ; X87_WIN-NEXT:    orl $3072, %ecx # imm = 0xC00
@@ -787,7 +764,7 @@ define i64 @d_to_u64(double %a) nounwind {
 ; X87_WIN-NEXT:    fldcw {{[0-9]+}}(%esp)
 ; X87_WIN-NEXT:    fistpll {{[0-9]+}}(%esp)
 ; X87_WIN-NEXT:    fldcw {{[0-9]+}}(%esp)
-; X87_WIN-NEXT:    movzbl %al, %edx
+; X87_WIN-NEXT:    movb %al, %dl
 ; X87_WIN-NEXT:    shll $31, %edx
 ; X87_WIN-NEXT:    xorl {{[0-9]+}}(%esp), %edx
 ; X87_WIN-NEXT:    movl {{[0-9]+}}(%esp), %eax
@@ -800,20 +777,21 @@ define i64 @d_to_u64(double %a) nounwind {
 ; X87_LIN-NEXT:    subl $20, %esp
 ; X87_LIN-NEXT:    fldl {{[0-9]+}}(%esp)
 ; X87_LIN-NEXT:    flds {{\.LCPI.*}}
-; X87_LIN-NEXT:    fld %st(1)
-; X87_LIN-NEXT:    fsub %st(1), %st
-; X87_LIN-NEXT:    fxch %st(1)
-; X87_LIN-NEXT:    fucomp %st(2)
+; X87_LIN-NEXT:    fucom %st(1)
 ; X87_LIN-NEXT:    fnstsw %ax
+; X87_LIN-NEXT:    xorl %edx, %edx
 ; X87_LIN-NEXT:    # kill: def $ah killed $ah killed $ax
 ; X87_LIN-NEXT:    sahf
+; X87_LIN-NEXT:    setbe %al
+; X87_LIN-NEXT:    fldz
 ; X87_LIN-NEXT:    ja .LBB2_2
 ; X87_LIN-NEXT:  # %bb.1:
-; X87_LIN-NEXT:    fstp %st(1)
+; X87_LIN-NEXT:    fstp %st(0)
 ; X87_LIN-NEXT:    fldz
+; X87_LIN-NEXT:    fxch %st(1)
 ; X87_LIN-NEXT:  .LBB2_2:
-; X87_LIN-NEXT:    fstp %st(0)
-; X87_LIN-NEXT:    setbe %al
+; X87_LIN-NEXT:    fstp %st(1)
+; X87_LIN-NEXT:    fsubrp %st, %st(1)
 ; X87_LIN-NEXT:    fnstcw {{[0-9]+}}(%esp)
 ; X87_LIN-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
 ; X87_LIN-NEXT:    orl $3072, %ecx # imm = 0xC00
@@ -821,7 +799,7 @@ define i64 @d_to_u64(double %a) nounwind {
 ; X87_LIN-NEXT:    fldcw {{[0-9]+}}(%esp)
 ; X87_LIN-NEXT:    fistpll {{[0-9]+}}(%esp)
 ; X87_LIN-NEXT:    fldcw {{[0-9]+}}(%esp)
-; X87_LIN-NEXT:    movzbl %al, %edx
+; X87_LIN-NEXT:    movb %al, %dl
 ; X87_LIN-NEXT:    shll $31, %edx
 ; X87_LIN-NEXT:    xorl {{[0-9]+}}(%esp), %edx
 ; X87_LIN-NEXT:    movl {{[0-9]+}}(%esp), %eax
@@ -835,19 +813,17 @@ define i64 @d_to_s64(double %a) nounwind {
 ; AVX512DQVL_32_WIN-LABEL: d_to_s64:
 ; AVX512DQVL_32_WIN:       # %bb.0:
 ; AVX512DQVL_32_WIN-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; AVX512DQVL_32_WIN-NEXT:    vcvttpd2qq %ymm0, %ymm0
+; AVX512DQVL_32_WIN-NEXT:    vcvttpd2qq %xmm0, %xmm0
 ; AVX512DQVL_32_WIN-NEXT:    vmovd %xmm0, %eax
 ; AVX512DQVL_32_WIN-NEXT:    vpextrd $1, %xmm0, %edx
-; AVX512DQVL_32_WIN-NEXT:    vzeroupper
 ; AVX512DQVL_32_WIN-NEXT:    retl
 ;
 ; AVX512DQVL_32_LIN-LABEL: d_to_s64:
 ; AVX512DQVL_32_LIN:       # %bb.0:
 ; AVX512DQVL_32_LIN-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; AVX512DQVL_32_LIN-NEXT:    vcvttpd2qq %ymm0, %ymm0
+; AVX512DQVL_32_LIN-NEXT:    vcvttpd2qq %xmm0, %xmm0
 ; AVX512DQVL_32_LIN-NEXT:    vmovd %xmm0, %eax
 ; AVX512DQVL_32_LIN-NEXT:    vpextrd $1, %xmm0, %edx
-; AVX512DQVL_32_LIN-NEXT:    vzeroupper
 ; AVX512DQVL_32_LIN-NEXT:    retl
 ;
 ; AVX512_64-LABEL: d_to_s64:
@@ -1027,13 +1003,13 @@ define i64 @x_to_u64(x86_fp80 %a) nounwind {
 ; AVX512_32_WIN-NEXT:    subl $8, %esp
 ; AVX512_32_WIN-NEXT:    fldt 8(%ebp)
 ; AVX512_32_WIN-NEXT:    flds __real@5f000000
-; AVX512_32_WIN-NEXT:    fld %st(1)
-; AVX512_32_WIN-NEXT:    fsub %st(1), %st
 ; AVX512_32_WIN-NEXT:    xorl %edx, %edx
+; AVX512_32_WIN-NEXT:    fucomi %st(1), %st
+; AVX512_32_WIN-NEXT:    fldz
 ; AVX512_32_WIN-NEXT:    fxch %st(1)
-; AVX512_32_WIN-NEXT:    fucompi %st(2), %st
 ; AVX512_32_WIN-NEXT:    fcmovnbe %st(1), %st
 ; AVX512_32_WIN-NEXT:    fstp %st(1)
+; AVX512_32_WIN-NEXT:    fsubrp %st, %st(1)
 ; AVX512_32_WIN-NEXT:    fisttpll (%esp)
 ; AVX512_32_WIN-NEXT:    setbe %dl
 ; AVX512_32_WIN-NEXT:    shll $31, %edx
@@ -1048,13 +1024,13 @@ define i64 @x_to_u64(x86_fp80 %a) nounwind {
 ; AVX512_32_LIN-NEXT:    subl $12, %esp
 ; AVX512_32_LIN-NEXT:    fldt {{[0-9]+}}(%esp)
 ; AVX512_32_LIN-NEXT:    flds {{\.LCPI.*}}
-; AVX512_32_LIN-NEXT:    fld %st(1)
-; AVX512_32_LIN-NEXT:    fsub %st(1), %st
 ; AVX512_32_LIN-NEXT:    xorl %edx, %edx
+; AVX512_32_LIN-NEXT:    fucomi %st(1), %st
+; AVX512_32_LIN-NEXT:    fldz
 ; AVX512_32_LIN-NEXT:    fxch %st(1)
-; AVX512_32_LIN-NEXT:    fucompi %st(2), %st
 ; AVX512_32_LIN-NEXT:    fcmovnbe %st(1), %st
 ; AVX512_32_LIN-NEXT:    fstp %st(1)
+; AVX512_32_LIN-NEXT:    fsubrp %st, %st(1)
 ; AVX512_32_LIN-NEXT:    fisttpll (%esp)
 ; AVX512_32_LIN-NEXT:    setbe %dl
 ; AVX512_32_LIN-NEXT:    shll $31, %edx
@@ -1068,13 +1044,13 @@ define i64 @x_to_u64(x86_fp80 %a) nounwind {
 ; AVX512_64_WIN-NEXT:    pushq %rax
 ; AVX512_64_WIN-NEXT:    fldt (%rcx)
 ; AVX512_64_WIN-NEXT:    flds __real@{{.*}}(%rip)
-; AVX512_64_WIN-NEXT:    fld %st(1)
-; AVX512_64_WIN-NEXT:    fsub %st(1), %st
 ; AVX512_64_WIN-NEXT:    xorl %eax, %eax
+; AVX512_64_WIN-NEXT:    fucomi %st(1), %st
+; AVX512_64_WIN-NEXT:    fldz
 ; AVX512_64_WIN-NEXT:    fxch %st(1)
-; AVX512_64_WIN-NEXT:    fucompi %st(2), %st
 ; AVX512_64_WIN-NEXT:    fcmovnbe %st(1), %st
 ; AVX512_64_WIN-NEXT:    fstp %st(1)
+; AVX512_64_WIN-NEXT:    fsubrp %st, %st(1)
 ; AVX512_64_WIN-NEXT:    fisttpll (%rsp)
 ; AVX512_64_WIN-NEXT:    setbe %al
 ; AVX512_64_WIN-NEXT:    shlq $63, %rax
@@ -1086,13 +1062,13 @@ define i64 @x_to_u64(x86_fp80 %a) nounwind {
 ; AVX512_64_LIN:       # %bb.0:
 ; AVX512_64_LIN-NEXT:    fldt {{[0-9]+}}(%rsp)
 ; AVX512_64_LIN-NEXT:    flds {{.*}}(%rip)
-; AVX512_64_LIN-NEXT:    fld %st(1)
-; AVX512_64_LIN-NEXT:    fsub %st(1), %st
 ; AVX512_64_LIN-NEXT:    xorl %eax, %eax
+; AVX512_64_LIN-NEXT:    fucomi %st(1), %st
+; AVX512_64_LIN-NEXT:    fldz
 ; AVX512_64_LIN-NEXT:    fxch %st(1)
-; AVX512_64_LIN-NEXT:    fucompi %st(2), %st
 ; AVX512_64_LIN-NEXT:    fcmovnbe %st(1), %st
 ; AVX512_64_LIN-NEXT:    fstp %st(1)
+; AVX512_64_LIN-NEXT:    fsubrp %st, %st(1)
 ; AVX512_64_LIN-NEXT:    fisttpll -{{[0-9]+}}(%rsp)
 ; AVX512_64_LIN-NEXT:    setbe %al
 ; AVX512_64_LIN-NEXT:    shlq $63, %rax
@@ -1107,13 +1083,13 @@ define i64 @x_to_u64(x86_fp80 %a) nounwind {
 ; SSE3_32_WIN-NEXT:    subl $8, %esp
 ; SSE3_32_WIN-NEXT:    fldt 8(%ebp)
 ; SSE3_32_WIN-NEXT:    flds __real@5f000000
-; SSE3_32_WIN-NEXT:    fld %st(1)
-; SSE3_32_WIN-NEXT:    fsub %st(1), %st
 ; SSE3_32_WIN-NEXT:    xorl %edx, %edx
+; SSE3_32_WIN-NEXT:    fucomi %st(1), %st
+; SSE3_32_WIN-NEXT:    fldz
 ; SSE3_32_WIN-NEXT:    fxch %st(1)
-; SSE3_32_WIN-NEXT:    fucompi %st(2), %st
 ; SSE3_32_WIN-NEXT:    fcmovnbe %st(1), %st
 ; SSE3_32_WIN-NEXT:    fstp %st(1)
+; SSE3_32_WIN-NEXT:    fsubrp %st, %st(1)
 ; SSE3_32_WIN-NEXT:    fisttpll (%esp)
 ; SSE3_32_WIN-NEXT:    setbe %dl
 ; SSE3_32_WIN-NEXT:    shll $31, %edx
@@ -1128,13 +1104,13 @@ define i64 @x_to_u64(x86_fp80 %a) nounwind {
 ; SSE3_32_LIN-NEXT:    subl $12, %esp
 ; SSE3_32_LIN-NEXT:    fldt {{[0-9]+}}(%esp)
 ; SSE3_32_LIN-NEXT:    flds {{\.LCPI.*}}
-; SSE3_32_LIN-NEXT:    fld %st(1)
-; SSE3_32_LIN-NEXT:    fsub %st(1), %st
 ; SSE3_32_LIN-NEXT:    xorl %edx, %edx
+; SSE3_32_LIN-NEXT:    fucomi %st(1), %st
+; SSE3_32_LIN-NEXT:    fldz
 ; SSE3_32_LIN-NEXT:    fxch %st(1)
-; SSE3_32_LIN-NEXT:    fucompi %st(2), %st
 ; SSE3_32_LIN-NEXT:    fcmovnbe %st(1), %st
 ; SSE3_32_LIN-NEXT:    fstp %st(1)
+; SSE3_32_LIN-NEXT:    fsubrp %st, %st(1)
 ; SSE3_32_LIN-NEXT:    fisttpll (%esp)
 ; SSE3_32_LIN-NEXT:    setbe %dl
 ; SSE3_32_LIN-NEXT:    shll $31, %edx
@@ -1148,13 +1124,13 @@ define i64 @x_to_u64(x86_fp80 %a) nounwind {
 ; SSE3_64_WIN-NEXT:    pushq %rax
 ; SSE3_64_WIN-NEXT:    fldt (%rcx)
 ; SSE3_64_WIN-NEXT:    flds __real@{{.*}}(%rip)
-; SSE3_64_WIN-NEXT:    fld %st(1)
-; SSE3_64_WIN-NEXT:    fsub %st(1), %st
 ; SSE3_64_WIN-NEXT:    xorl %eax, %eax
+; SSE3_64_WIN-NEXT:    fucomi %st(1), %st
+; SSE3_64_WIN-NEXT:    fldz
 ; SSE3_64_WIN-NEXT:    fxch %st(1)
-; SSE3_64_WIN-NEXT:    fucompi %st(2), %st
 ; SSE3_64_WIN-NEXT:    fcmovnbe %st(1), %st
 ; SSE3_64_WIN-NEXT:    fstp %st(1)
+; SSE3_64_WIN-NEXT:    fsubrp %st, %st(1)
 ; SSE3_64_WIN-NEXT:    fisttpll (%rsp)
 ; SSE3_64_WIN-NEXT:    setbe %al
 ; SSE3_64_WIN-NEXT:    shlq $63, %rax
@@ -1166,13 +1142,13 @@ define i64 @x_to_u64(x86_fp80 %a) nounwind {
 ; SSE3_64_LIN:       # %bb.0:
 ; SSE3_64_LIN-NEXT:    fldt {{[0-9]+}}(%rsp)
 ; SSE3_64_LIN-NEXT:    flds {{.*}}(%rip)
-; SSE3_64_LIN-NEXT:    fld %st(1)
-; SSE3_64_LIN-NEXT:    fsub %st(1), %st
 ; SSE3_64_LIN-NEXT:    xorl %eax, %eax
+; SSE3_64_LIN-NEXT:    fucomi %st(1), %st
+; SSE3_64_LIN-NEXT:    fldz
 ; SSE3_64_LIN-NEXT:    fxch %st(1)
-; SSE3_64_LIN-NEXT:    fucompi %st(2), %st
 ; SSE3_64_LIN-NEXT:    fcmovnbe %st(1), %st
 ; SSE3_64_LIN-NEXT:    fstp %st(1)
+; SSE3_64_LIN-NEXT:    fsubrp %st, %st(1)
 ; SSE3_64_LIN-NEXT:    fisttpll -{{[0-9]+}}(%rsp)
 ; SSE3_64_LIN-NEXT:    setbe %al
 ; SSE3_64_LIN-NEXT:    shlq $63, %rax
@@ -1187,14 +1163,14 @@ define i64 @x_to_u64(x86_fp80 %a) nounwind {
 ; SSE2_32_WIN-NEXT:    subl $16, %esp
 ; SSE2_32_WIN-NEXT:    fldt 8(%ebp)
 ; SSE2_32_WIN-NEXT:    flds __real@5f000000
-; SSE2_32_WIN-NEXT:    fld %st(1)
-; SSE2_32_WIN-NEXT:    fsub %st(1), %st
 ; SSE2_32_WIN-NEXT:    xorl %edx, %edx
+; SSE2_32_WIN-NEXT:    fucomi %st(1), %st
+; SSE2_32_WIN-NEXT:    setbe %dl
+; SSE2_32_WIN-NEXT:    fldz
 ; SSE2_32_WIN-NEXT:    fxch %st(1)
-; SSE2_32_WIN-NEXT:    fucompi %st(2), %st
 ; SSE2_32_WIN-NEXT:    fcmovnbe %st(1), %st
 ; SSE2_32_WIN-NEXT:    fstp %st(1)
-; SSE2_32_WIN-NEXT:    setbe %dl
+; SSE2_32_WIN-NEXT:    fsubrp %st, %st(1)
 ; SSE2_32_WIN-NEXT:    fnstcw {{[0-9]+}}(%esp)
 ; SSE2_32_WIN-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
 ; SSE2_32_WIN-NEXT:    orl $3072, %eax # imm = 0xC00
@@ -1214,14 +1190,14 @@ define i64 @x_to_u64(x86_fp80 %a) nounwind {
 ; SSE2_32_LIN-NEXT:    subl $20, %esp
 ; SSE2_32_LIN-NEXT:    fldt {{[0-9]+}}(%esp)
 ; SSE2_32_LIN-NEXT:    flds {{\.LCPI.*}}
-; SSE2_32_LIN-NEXT:    fld %st(1)
-; SSE2_32_LIN-NEXT:    fsub %st(1), %st
 ; SSE2_32_LIN-NEXT:    xorl %edx, %edx
+; SSE2_32_LIN-NEXT:    fucomi %st(1), %st
+; SSE2_32_LIN-NEXT:    setbe %dl
+; SSE2_32_LIN-NEXT:    fldz
 ; SSE2_32_LIN-NEXT:    fxch %st(1)
-; SSE2_32_LIN-NEXT:    fucompi %st(2), %st
 ; SSE2_32_LIN-NEXT:    fcmovnbe %st(1), %st
 ; SSE2_32_LIN-NEXT:    fstp %st(1)
-; SSE2_32_LIN-NEXT:    setbe %dl
+; SSE2_32_LIN-NEXT:    fsubrp %st, %st(1)
 ; SSE2_32_LIN-NEXT:    fnstcw {{[0-9]+}}(%esp)
 ; SSE2_32_LIN-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
 ; SSE2_32_LIN-NEXT:    orl $3072, %eax # imm = 0xC00
@@ -1240,14 +1216,14 @@ define i64 @x_to_u64(x86_fp80 %a) nounwind {
 ; SSE2_64_WIN-NEXT:    subq $16, %rsp
 ; SSE2_64_WIN-NEXT:    fldt (%rcx)
 ; SSE2_64_WIN-NEXT:    flds __real@{{.*}}(%rip)
-; SSE2_64_WIN-NEXT:    fld %st(1)
-; SSE2_64_WIN-NEXT:    fsub %st(1), %st
 ; SSE2_64_WIN-NEXT:    xorl %eax, %eax
+; SSE2_64_WIN-NEXT:    fucomi %st(1), %st
+; SSE2_64_WIN-NEXT:    setbe %al
+; SSE2_64_WIN-NEXT:    fldz
 ; SSE2_64_WIN-NEXT:    fxch %st(1)
-; SSE2_64_WIN-NEXT:    fucompi %st(2), %st
 ; SSE2_64_WIN-NEXT:    fcmovnbe %st(1), %st
 ; SSE2_64_WIN-NEXT:    fstp %st(1)
-; SSE2_64_WIN-NEXT:    setbe %al
+; SSE2_64_WIN-NEXT:    fsubrp %st, %st(1)
 ; SSE2_64_WIN-NEXT:    fnstcw {{[0-9]+}}(%rsp)
 ; SSE2_64_WIN-NEXT:    movzwl {{[0-9]+}}(%rsp), %ecx
 ; SSE2_64_WIN-NEXT:    orl $3072, %ecx # imm = 0xC00
@@ -1264,14 +1240,14 @@ define i64 @x_to_u64(x86_fp80 %a) nounwind {
 ; SSE2_64_LIN:       # %bb.0:
 ; SSE2_64_LIN-NEXT:    fldt {{[0-9]+}}(%rsp)
 ; SSE2_64_LIN-NEXT:    flds {{.*}}(%rip)
-; SSE2_64_LIN-NEXT:    fld %st(1)
-; SSE2_64_LIN-NEXT:    fsub %st(1), %st
 ; SSE2_64_LIN-NEXT:    xorl %eax, %eax
+; SSE2_64_LIN-NEXT:    fucomi %st(1), %st
+; SSE2_64_LIN-NEXT:    setbe %al
+; SSE2_64_LIN-NEXT:    fldz
 ; SSE2_64_LIN-NEXT:    fxch %st(1)
-; SSE2_64_LIN-NEXT:    fucompi %st(2), %st
 ; SSE2_64_LIN-NEXT:    fcmovnbe %st(1), %st
 ; SSE2_64_LIN-NEXT:    fstp %st(1)
-; SSE2_64_LIN-NEXT:    setbe %al
+; SSE2_64_LIN-NEXT:    fsubrp %st, %st(1)
 ; SSE2_64_LIN-NEXT:    fnstcw -{{[0-9]+}}(%rsp)
 ; SSE2_64_LIN-NEXT:    movzwl -{{[0-9]+}}(%rsp), %ecx
 ; SSE2_64_LIN-NEXT:    orl $3072, %ecx # imm = 0xC00
@@ -1291,20 +1267,21 @@ define i64 @x_to_u64(x86_fp80 %a) nounwind {
 ; X87_WIN-NEXT:    subl $16, %esp
 ; X87_WIN-NEXT:    fldt 8(%ebp)
 ; X87_WIN-NEXT:    flds __real@5f000000
-; X87_WIN-NEXT:    fld %st(1)
-; X87_WIN-NEXT:    fsub %st(1), %st
-; X87_WIN-NEXT:    fxch %st(1)
-; X87_WIN-NEXT:    fucomp %st(2)
+; X87_WIN-NEXT:    fucom %st(1)
 ; X87_WIN-NEXT:    fnstsw %ax
+; X87_WIN-NEXT:    xorl %edx, %edx
 ; X87_WIN-NEXT:    # kill: def $ah killed $ah killed $ax
 ; X87_WIN-NEXT:    sahf
+; X87_WIN-NEXT:    setbe %al
+; X87_WIN-NEXT:    fldz
 ; X87_WIN-NEXT:    ja LBB4_2
 ; X87_WIN-NEXT:  # %bb.1:
-; X87_WIN-NEXT:    fstp %st(1)
+; X87_WIN-NEXT:    fstp %st(0)
 ; X87_WIN-NEXT:    fldz
+; X87_WIN-NEXT:    fxch %st(1)
 ; X87_WIN-NEXT:  LBB4_2:
-; X87_WIN-NEXT:    fstp %st(0)
-; X87_WIN-NEXT:    setbe %al
+; X87_WIN-NEXT:    fstp %st(1)
+; X87_WIN-NEXT:    fsubrp %st, %st(1)
 ; X87_WIN-NEXT:    fnstcw {{[0-9]+}}(%esp)
 ; X87_WIN-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
 ; X87_WIN-NEXT:    orl $3072, %ecx # imm = 0xC00
@@ -1312,7 +1289,7 @@ define i64 @x_to_u64(x86_fp80 %a) nounwind {
 ; X87_WIN-NEXT:    fldcw {{[0-9]+}}(%esp)
 ; X87_WIN-NEXT:    fistpll {{[0-9]+}}(%esp)
 ; X87_WIN-NEXT:    fldcw {{[0-9]+}}(%esp)
-; X87_WIN-NEXT:    movzbl %al, %edx
+; X87_WIN-NEXT:    movb %al, %dl
 ; X87_WIN-NEXT:    shll $31, %edx
 ; X87_WIN-NEXT:    xorl {{[0-9]+}}(%esp), %edx
 ; X87_WIN-NEXT:    movl {{[0-9]+}}(%esp), %eax
@@ -1325,20 +1302,21 @@ define i64 @x_to_u64(x86_fp80 %a) nounwind {
 ; X87_LIN-NEXT:    subl $20, %esp
 ; X87_LIN-NEXT:    fldt {{[0-9]+}}(%esp)
 ; X87_LIN-NEXT:    flds {{\.LCPI.*}}
-; X87_LIN-NEXT:    fld %st(1)
-; X87_LIN-NEXT:    fsub %st(1), %st
-; X87_LIN-NEXT:    fxch %st(1)
-; X87_LIN-NEXT:    fucomp %st(2)
+; X87_LIN-NEXT:    fucom %st(1)
 ; X87_LIN-NEXT:    fnstsw %ax
+; X87_LIN-NEXT:    xorl %edx, %edx
 ; X87_LIN-NEXT:    # kill: def $ah killed $ah killed $ax
 ; X87_LIN-NEXT:    sahf
+; X87_LIN-NEXT:    setbe %al
+; X87_LIN-NEXT:    fldz
 ; X87_LIN-NEXT:    ja .LBB4_2
 ; X87_LIN-NEXT:  # %bb.1:
-; X87_LIN-NEXT:    fstp %st(1)
+; X87_LIN-NEXT:    fstp %st(0)
 ; X87_LIN-NEXT:    fldz
+; X87_LIN-NEXT:    fxch %st(1)
 ; X87_LIN-NEXT:  .LBB4_2:
-; X87_LIN-NEXT:    fstp %st(0)
-; X87_LIN-NEXT:    setbe %al
+; X87_LIN-NEXT:    fstp %st(1)
+; X87_LIN-NEXT:    fsubrp %st, %st(1)
 ; X87_LIN-NEXT:    fnstcw {{[0-9]+}}(%esp)
 ; X87_LIN-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
 ; X87_LIN-NEXT:    orl $3072, %ecx # imm = 0xC00
@@ -1346,7 +1324,7 @@ define i64 @x_to_u64(x86_fp80 %a) nounwind {
 ; X87_LIN-NEXT:    fldcw {{[0-9]+}}(%esp)
 ; X87_LIN-NEXT:    fistpll {{[0-9]+}}(%esp)
 ; X87_LIN-NEXT:    fldcw {{[0-9]+}}(%esp)
-; X87_LIN-NEXT:    movzbl %al, %edx
+; X87_LIN-NEXT:    movb %al, %dl
 ; X87_LIN-NEXT:    shll $31, %edx
 ; X87_LIN-NEXT:    xorl {{[0-9]+}}(%esp), %edx
 ; X87_LIN-NEXT:    movl {{[0-9]+}}(%esp), %eax
diff --git a/llvm/test/CodeGen/X86/scalar-int-to-fp.ll b/llvm/test/CodeGen/X86/scalar-int-to-fp.ll
index 8a46b9605..29e8df668 100644
--- a/llvm/test/CodeGen/X86/scalar-int-to-fp.ll
+++ b/llvm/test/CodeGen/X86/scalar-int-to-fp.ll
@@ -214,68 +214,26 @@ define double @s32_to_d(i32 %a) nounwind {
 }
 
 define x86_fp80 @u32_to_x(i32 %a) nounwind {
-; AVX512_32-LABEL: u32_to_x:
-; AVX512_32:       # %bb.0:
-; AVX512_32-NEXT:    pushl %ebp
-; AVX512_32-NEXT:    movl %esp, %ebp
-; AVX512_32-NEXT:    andl $-8, %esp
-; AVX512_32-NEXT:    subl $8, %esp
-; AVX512_32-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; AVX512_32-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX512_32-NEXT:    vorpd %xmm0, %xmm1, %xmm1
-; AVX512_32-NEXT:    vsubsd %xmm0, %xmm1, %xmm0
-; AVX512_32-NEXT:    vmovsd %xmm0, (%esp)
-; AVX512_32-NEXT:    fldl (%esp)
-; AVX512_32-NEXT:    movl %ebp, %esp
-; AVX512_32-NEXT:    popl %ebp
-; AVX512_32-NEXT:    retl
-;
-; AVX512_64-LABEL: u32_to_x:
-; AVX512_64:       # %bb.0:
-; AVX512_64-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
-; AVX512_64-NEXT:    vmovd %edi, %xmm1
-; AVX512_64-NEXT:    vpor %xmm0, %xmm1, %xmm1
-; AVX512_64-NEXT:    vsubsd %xmm0, %xmm1, %xmm0
-; AVX512_64-NEXT:    vmovsd %xmm0, -{{[0-9]+}}(%rsp)
-; AVX512_64-NEXT:    fldl -{{[0-9]+}}(%rsp)
-; AVX512_64-NEXT:    retq
-;
-; SSE2_32-LABEL: u32_to_x:
-; SSE2_32:       # %bb.0:
-; SSE2_32-NEXT:    pushl %ebp
-; SSE2_32-NEXT:    movl %esp, %ebp
-; SSE2_32-NEXT:    andl $-8, %esp
-; SSE2_32-NEXT:    subl $8, %esp
-; SSE2_32-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; SSE2_32-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; SSE2_32-NEXT:    orpd %xmm0, %xmm1
-; SSE2_32-NEXT:    subsd %xmm0, %xmm1
-; SSE2_32-NEXT:    movsd %xmm1, (%esp)
-; SSE2_32-NEXT:    fldl (%esp)
-; SSE2_32-NEXT:    movl %ebp, %esp
-; SSE2_32-NEXT:    popl %ebp
-; SSE2_32-NEXT:    retl
-;
-; SSE2_64-LABEL: u32_to_x:
-; SSE2_64:       # %bb.0:
-; SSE2_64-NEXT:    movl %edi, %eax
-; SSE2_64-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
-; SSE2_64-NEXT:    fildll -{{[0-9]+}}(%rsp)
-; SSE2_64-NEXT:    retq
+; CHECK32-LABEL: u32_to_x:
+; CHECK32:       # %bb.0:
+; CHECK32-NEXT:    pushl %ebp
+; CHECK32-NEXT:    movl %esp, %ebp
+; CHECK32-NEXT:    andl $-8, %esp
+; CHECK32-NEXT:    subl $8, %esp
+; CHECK32-NEXT:    movl 8(%ebp), %eax
+; CHECK32-NEXT:    movl %eax, (%esp)
+; CHECK32-NEXT:    movl $0, {{[0-9]+}}(%esp)
+; CHECK32-NEXT:    fildll (%esp)
+; CHECK32-NEXT:    movl %ebp, %esp
+; CHECK32-NEXT:    popl %ebp
+; CHECK32-NEXT:    retl
 ;
-; X87-LABEL: u32_to_x:
-; X87:       # %bb.0:
-; X87-NEXT:    pushl %ebp
-; X87-NEXT:    movl %esp, %ebp
-; X87-NEXT:    andl $-8, %esp
-; X87-NEXT:    subl $8, %esp
-; X87-NEXT:    movl 8(%ebp), %eax
-; X87-NEXT:    movl %eax, (%esp)
-; X87-NEXT:    movl $0, {{[0-9]+}}(%esp)
-; X87-NEXT:    fildll (%esp)
-; X87-NEXT:    movl %ebp, %esp
-; X87-NEXT:    popl %ebp
-; X87-NEXT:    retl
+; CHECK64-LABEL: u32_to_x:
+; CHECK64:       # %bb.0:
+; CHECK64-NEXT:    movl %edi, %eax
+; CHECK64-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
+; CHECK64-NEXT:    fildll -{{[0-9]+}}(%rsp)
+; CHECK64-NEXT:    retq
   %r = uitofp i32 %a to x86_fp80
   ret x86_fp80 %r
 }
@@ -333,11 +291,10 @@ define float @u64_to_f(i64 %a) nounwind {
 ; AVX512F_32-NEXT:    movl %esp, %ebp
 ; AVX512F_32-NEXT:    andl $-8, %esp
 ; AVX512F_32-NEXT:    subl $16, %esp
+; AVX512F_32-NEXT:    movl 12(%ebp), %eax
 ; AVX512F_32-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; AVX512F_32-NEXT:    vmovlps %xmm0, {{[0-9]+}}(%esp)
-; AVX512F_32-NEXT:    xorl %eax, %eax
-; AVX512F_32-NEXT:    cmpl $0, 12(%ebp)
-; AVX512F_32-NEXT:    setns %al
+; AVX512F_32-NEXT:    shrl $31, %eax
 ; AVX512F_32-NEXT:    fildll {{[0-9]+}}(%esp)
 ; AVX512F_32-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
 ; AVX512F_32-NEXT:    fstps {{[0-9]+}}(%esp)
@@ -354,11 +311,10 @@ define float @u64_to_f(i64 %a) nounwind {
 ; SSE2_32-NEXT:    movl %esp, %ebp
 ; SSE2_32-NEXT:    andl $-8, %esp
 ; SSE2_32-NEXT:    subl $16, %esp
+; SSE2_32-NEXT:    movl 12(%ebp), %eax
 ; SSE2_32-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; SSE2_32-NEXT:    movlps %xmm0, {{[0-9]+}}(%esp)
-; SSE2_32-NEXT:    xorl %eax, %eax
-; SSE2_32-NEXT:    cmpl $0, 12(%ebp)
-; SSE2_32-NEXT:    setns %al
+; SSE2_32-NEXT:    shrl $31, %eax
 ; SSE2_32-NEXT:    fildll {{[0-9]+}}(%esp)
 ; SSE2_32-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
 ; SSE2_32-NEXT:    fstps {{[0-9]+}}(%esp)
@@ -395,11 +351,9 @@ define float @u64_to_f(i64 %a) nounwind {
 ; X87-NEXT:    movl 12(%ebp), %ecx
 ; X87-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
 ; X87-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; X87-NEXT:    xorl %eax, %eax
-; X87-NEXT:    testl %ecx, %ecx
-; X87-NEXT:    setns %al
+; X87-NEXT:    shrl $31, %ecx
 ; X87-NEXT:    fildll {{[0-9]+}}(%esp)
-; X87-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; X87-NEXT:    fadds {{\.LCPI.*}}(,%ecx,4)
 ; X87-NEXT:    fstps {{[0-9]+}}(%esp)
 ; X87-NEXT:    flds {{[0-9]+}}(%esp)
 ; X87-NEXT:    movl %ebp, %esp
@@ -620,8 +574,9 @@ define double @u64_to_d(i64 %a) nounwind {
 ; AVX512F_32-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; AVX512F_32-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
 ; AVX512F_32-NEXT:    vsubpd {{\.LCPI.*}}, %xmm0, %xmm0
-; AVX512F_32-NEXT:    vhaddpd %xmm0, %xmm0, %xmm0
-; AVX512F_32-NEXT:    vmovlpd %xmm0, (%esp)
+; AVX512F_32-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
+; AVX512F_32-NEXT:    vaddsd %xmm0, %xmm1, %xmm0
+; AVX512F_32-NEXT:    vmovsd %xmm0, (%esp)
 ; AVX512F_32-NEXT:    fldl (%esp)
 ; AVX512F_32-NEXT:    movl %ebp, %esp
 ; AVX512F_32-NEXT:    popl %ebp
@@ -665,11 +620,111 @@ define double @u64_to_d(i64 %a) nounwind {
 ; X87-NEXT:    movl 12(%ebp), %ecx
 ; X87-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
 ; X87-NEXT:    movl %eax, (%esp)
-; X87-NEXT:    xorl %eax, %eax
-; X87-NEXT:    testl %ecx, %ecx
-; X87-NEXT:    setns %al
+; X87-NEXT:    shrl $31, %ecx
+; X87-NEXT:    fildll (%esp)
+; X87-NEXT:    fadds {{\.LCPI.*}}(,%ecx,4)
+; X87-NEXT:    fstpl {{[0-9]+}}(%esp)
+; X87-NEXT:    fldl {{[0-9]+}}(%esp)
+; X87-NEXT:    movl %ebp, %esp
+; X87-NEXT:    popl %ebp
+; X87-NEXT:    retl
+  %r = uitofp i64 %a to double
+  ret double %r
+}
+
+define double @u64_to_d_optsize(i64 %a) nounwind optsize {
+; AVX512DQVL_32-LABEL: u64_to_d_optsize:
+; AVX512DQVL_32:       # %bb.0:
+; AVX512DQVL_32-NEXT:    pushl %ebp
+; AVX512DQVL_32-NEXT:    movl %esp, %ebp
+; AVX512DQVL_32-NEXT:    andl $-8, %esp
+; AVX512DQVL_32-NEXT:    subl $8, %esp
+; AVX512DQVL_32-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX512DQVL_32-NEXT:    vcvtuqq2pd %ymm0, %ymm0
+; AVX512DQVL_32-NEXT:    vmovlps %xmm0, (%esp)
+; AVX512DQVL_32-NEXT:    fldl (%esp)
+; AVX512DQVL_32-NEXT:    movl %ebp, %esp
+; AVX512DQVL_32-NEXT:    popl %ebp
+; AVX512DQVL_32-NEXT:    vzeroupper
+; AVX512DQVL_32-NEXT:    retl
+;
+; AVX512_64-LABEL: u64_to_d_optsize:
+; AVX512_64:       # %bb.0:
+; AVX512_64-NEXT:    vcvtusi2sd %rdi, %xmm0, %xmm0
+; AVX512_64-NEXT:    retq
+;
+; AVX512DQ_32-LABEL: u64_to_d_optsize:
+; AVX512DQ_32:       # %bb.0:
+; AVX512DQ_32-NEXT:    pushl %ebp
+; AVX512DQ_32-NEXT:    movl %esp, %ebp
+; AVX512DQ_32-NEXT:    andl $-8, %esp
+; AVX512DQ_32-NEXT:    subl $8, %esp
+; AVX512DQ_32-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX512DQ_32-NEXT:    vcvtuqq2pd %zmm0, %zmm0
+; AVX512DQ_32-NEXT:    vmovlps %xmm0, (%esp)
+; AVX512DQ_32-NEXT:    fldl (%esp)
+; AVX512DQ_32-NEXT:    movl %ebp, %esp
+; AVX512DQ_32-NEXT:    popl %ebp
+; AVX512DQ_32-NEXT:    vzeroupper
+; AVX512DQ_32-NEXT:    retl
+;
+; AVX512F_32-LABEL: u64_to_d_optsize:
+; AVX512F_32:       # %bb.0:
+; AVX512F_32-NEXT:    pushl %ebp
+; AVX512F_32-NEXT:    movl %esp, %ebp
+; AVX512F_32-NEXT:    andl $-8, %esp
+; AVX512F_32-NEXT:    subl $8, %esp
+; AVX512F_32-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX512F_32-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
+; AVX512F_32-NEXT:    vsubpd {{\.LCPI.*}}, %xmm0, %xmm0
+; AVX512F_32-NEXT:    vhaddpd %xmm0, %xmm0, %xmm0
+; AVX512F_32-NEXT:    vmovlpd %xmm0, (%esp)
+; AVX512F_32-NEXT:    fldl (%esp)
+; AVX512F_32-NEXT:    movl %ebp, %esp
+; AVX512F_32-NEXT:    popl %ebp
+; AVX512F_32-NEXT:    retl
+;
+; SSE2_32-LABEL: u64_to_d_optsize:
+; SSE2_32:       # %bb.0:
+; SSE2_32-NEXT:    pushl %ebp
+; SSE2_32-NEXT:    movl %esp, %ebp
+; SSE2_32-NEXT:    andl $-8, %esp
+; SSE2_32-NEXT:    subl $8, %esp
+; SSE2_32-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE2_32-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
+; SSE2_32-NEXT:    subpd {{\.LCPI.*}}, %xmm0
+; SSE2_32-NEXT:    movapd %xmm0, %xmm1
+; SSE2_32-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
+; SSE2_32-NEXT:    addsd %xmm0, %xmm1
+; SSE2_32-NEXT:    movsd %xmm1, (%esp)
+; SSE2_32-NEXT:    fldl (%esp)
+; SSE2_32-NEXT:    movl %ebp, %esp
+; SSE2_32-NEXT:    popl %ebp
+; SSE2_32-NEXT:    retl
+;
+; SSE2_64-LABEL: u64_to_d_optsize:
+; SSE2_64:       # %bb.0:
+; SSE2_64-NEXT:    movq %rdi, %xmm1
+; SSE2_64-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]
+; SSE2_64-NEXT:    subpd {{.*}}(%rip), %xmm1
+; SSE2_64-NEXT:    movapd %xmm1, %xmm0
+; SSE2_64-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1]
+; SSE2_64-NEXT:    addsd %xmm1, %xmm0
+; SSE2_64-NEXT:    retq
+;
+; X87-LABEL: u64_to_d_optsize:
+; X87:       # %bb.0:
+; X87-NEXT:    pushl %ebp
+; X87-NEXT:    movl %esp, %ebp
+; X87-NEXT:    andl $-8, %esp
+; X87-NEXT:    subl $16, %esp
+; X87-NEXT:    movl 8(%ebp), %eax
+; X87-NEXT:    movl 12(%ebp), %ecx
+; X87-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X87-NEXT:    movl %eax, (%esp)
+; X87-NEXT:    shrl $31, %ecx
 ; X87-NEXT:    fildll (%esp)
-; X87-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; X87-NEXT:    fadds {{\.LCPI.*}}(,%ecx,4)
 ; X87-NEXT:    fstpl {{[0-9]+}}(%esp)
 ; X87-NEXT:    fldl {{[0-9]+}}(%esp)
 ; X87-NEXT:    movl %ebp, %esp
@@ -880,11 +935,9 @@ define x86_fp80 @u64_to_x(i64 %a) nounwind {
 ; CHECK32-NEXT:    movl 12(%ebp), %ecx
 ; CHECK32-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
 ; CHECK32-NEXT:    movl %eax, (%esp)
-; CHECK32-NEXT:    xorl %eax, %eax
-; CHECK32-NEXT:    testl %ecx, %ecx
-; CHECK32-NEXT:    setns %al
+; CHECK32-NEXT:    shrl $31, %ecx
 ; CHECK32-NEXT:    fildll (%esp)
-; CHECK32-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; CHECK32-NEXT:    fadds {{\.LCPI.*}}(,%ecx,4)
 ; CHECK32-NEXT:    movl %ebp, %esp
 ; CHECK32-NEXT:    popl %ebp
 ; CHECK32-NEXT:    retl
@@ -894,7 +947,7 @@ define x86_fp80 @u64_to_x(i64 %a) nounwind {
 ; CHECK64-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
 ; CHECK64-NEXT:    xorl %eax, %eax
 ; CHECK64-NEXT:    testq %rdi, %rdi
-; CHECK64-NEXT:    setns %al
+; CHECK64-NEXT:    sets %al
 ; CHECK64-NEXT:    fildll -{{[0-9]+}}(%rsp)
 ; CHECK64-NEXT:    fadds {{\.LCPI.*}}(,%rax,4)
 ; CHECK64-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/scalar_widen_div.ll b/llvm/test/CodeGen/X86/scalar_widen_div.ll
index 11dda4bc7..c6deb686e 100644
--- a/llvm/test/CodeGen/X86/scalar_widen_div.ll
+++ b/llvm/test/CodeGen/X86/scalar_widen_div.ll
@@ -56,21 +56,17 @@ entry:
 define <3 x i8> @test_char_div(<3 x i8> %num, <3 x i8> %div) {
 ; CHECK-LABEL: test_char_div:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    movl %edx, %r10d
-; CHECK-NEXT:    movl %edi, %eax
-; CHECK-NEXT:    # kill: def $al killed $al killed $eax
-; CHECK-NEXT:    cbtw
+; CHECK-NEXT:    movsbl %dil, %eax
 ; CHECK-NEXT:    idivb %cl
 ; CHECK-NEXT:    movl %eax, %edi
-; CHECK-NEXT:    movl %esi, %eax
-; CHECK-NEXT:    cbtw
+; CHECK-NEXT:    movsbl %sil, %eax
 ; CHECK-NEXT:    idivb %r8b
-; CHECK-NEXT:    movl %eax, %edx
-; CHECK-NEXT:    movl %r10d, %eax
-; CHECK-NEXT:    cbtw
+; CHECK-NEXT:    movl %eax, %esi
+; CHECK-NEXT:    movsbl %dl, %eax
 ; CHECK-NEXT:    idivb %r9b
 ; CHECK-NEXT:    movl %eax, %ecx
 ; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    movl %esi, %edx
 ; CHECK-NEXT:    retq
   %div.r = sdiv <3 x i8> %num, %div
   ret <3 x i8>  %div.r
@@ -81,15 +77,12 @@ define <3 x i8> @test_uchar_div(<3 x i8> %num, <3 x i8> %div) {
 ; CHECK-LABEL: test_uchar_div:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    movzbl %dil, %eax
-; CHECK-NEXT:    # kill: def $eax killed $eax def $ax
 ; CHECK-NEXT:    divb %cl
 ; CHECK-NEXT:    movl %eax, %edi
 ; CHECK-NEXT:    movzbl %sil, %eax
-; CHECK-NEXT:    # kill: def $eax killed $eax def $ax
 ; CHECK-NEXT:    divb %r8b
 ; CHECK-NEXT:    movl %eax, %esi
 ; CHECK-NEXT:    movzbl %dl, %eax
-; CHECK-NEXT:    # kill: def $eax killed $eax def $ax
 ; CHECK-NEXT:    divb %r9b
 ; CHECK-NEXT:    movl %eax, %ecx
 ; CHECK-NEXT:    movl %edi, %eax
@@ -147,30 +140,34 @@ define <5 x i16> @test_short_div(<5 x i16> %num, <5 x i16> %div) {
 define <4 x i16> @test_ushort_div(<4 x i16> %num, <4 x i16> %div) {
 ; CHECK-LABEL: test_ushort_div:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pxor %xmm2, %xmm2
-; CHECK-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
-; CHECK-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3],xmm0[4],xmm2[5],xmm0[6],xmm2[7]
-; CHECK-NEXT:    pextrd $1, %xmm0, %eax
-; CHECK-NEXT:    pextrd $1, %xmm1, %ecx
+; CHECK-NEXT:    pextrw $1, %xmm0, %eax
+; CHECK-NEXT:    pextrw $1, %xmm1, %ecx
+; CHECK-NEXT:    # kill: def $ax killed $ax killed $eax
 ; CHECK-NEXT:    xorl %edx, %edx
-; CHECK-NEXT:    divl %ecx
+; CHECK-NEXT:    divw %cx
 ; CHECK-NEXT:    movl %eax, %ecx
 ; CHECK-NEXT:    movd %xmm0, %eax
 ; CHECK-NEXT:    movd %xmm1, %esi
+; CHECK-NEXT:    # kill: def $ax killed $ax killed $eax
 ; CHECK-NEXT:    xorl %edx, %edx
-; CHECK-NEXT:    divl %esi
+; CHECK-NEXT:    divw %si
+; CHECK-NEXT:    # kill: def $ax killed $ax def $eax
 ; CHECK-NEXT:    movd %eax, %xmm2
-; CHECK-NEXT:    pinsrd $1, %ecx, %xmm2
-; CHECK-NEXT:    pextrd $2, %xmm0, %eax
-; CHECK-NEXT:    pextrd $2, %xmm1, %ecx
+; CHECK-NEXT:    pinsrw $1, %ecx, %xmm2
+; CHECK-NEXT:    pextrw $2, %xmm0, %eax
+; CHECK-NEXT:    pextrw $2, %xmm1, %ecx
+; CHECK-NEXT:    # kill: def $ax killed $ax killed $eax
 ; CHECK-NEXT:    xorl %edx, %edx
-; CHECK-NEXT:    divl %ecx
-; CHECK-NEXT:    pinsrd $2, %eax, %xmm2
-; CHECK-NEXT:    pextrd $3, %xmm0, %eax
-; CHECK-NEXT:    pextrd $3, %xmm1, %ecx
+; CHECK-NEXT:    divw %cx
+; CHECK-NEXT:    # kill: def $ax killed $ax def $eax
+; CHECK-NEXT:    pinsrw $2, %eax, %xmm2
+; CHECK-NEXT:    pextrw $3, %xmm0, %eax
+; CHECK-NEXT:    pextrw $3, %xmm1, %ecx
+; CHECK-NEXT:    # kill: def $ax killed $ax killed $eax
 ; CHECK-NEXT:    xorl %edx, %edx
-; CHECK-NEXT:    divl %ecx
-; CHECK-NEXT:    pinsrd $3, %eax, %xmm2
+; CHECK-NEXT:    divw %cx
+; CHECK-NEXT:    # kill: def $ax killed $ax def $eax
+; CHECK-NEXT:    pinsrw $3, %eax, %xmm2
 ; CHECK-NEXT:    movdqa %xmm2, %xmm0
 ; CHECK-NEXT:    retq
   %div.r = udiv <4 x i16> %num, %div
@@ -257,31 +254,30 @@ define <3 x i64> @test_ulong_div(<3 x i64> %num, <3 x i64> %div) {
 define <4 x i8> @test_char_rem(<4 x i8> %num, <4 x i8> %rem) {
 ; CHECK-LABEL: test_char_rem:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pslld $24, %xmm1
-; CHECK-NEXT:    psrad $24, %xmm1
-; CHECK-NEXT:    pslld $24, %xmm0
-; CHECK-NEXT:    psrad $24, %xmm0
-; CHECK-NEXT:    pextrd $1, %xmm0, %eax
-; CHECK-NEXT:    pextrd $1, %xmm1, %ecx
-; CHECK-NEXT:    cltd
-; CHECK-NEXT:    idivl %ecx
-; CHECK-NEXT:    movl %edx, %ecx
-; CHECK-NEXT:    movd %xmm0, %eax
-; CHECK-NEXT:    movd %xmm1, %esi
-; CHECK-NEXT:    cltd
-; CHECK-NEXT:    idivl %esi
-; CHECK-NEXT:    movd %edx, %xmm2
-; CHECK-NEXT:    pinsrd $1, %ecx, %xmm2
-; CHECK-NEXT:    pextrd $2, %xmm0, %eax
-; CHECK-NEXT:    pextrd $2, %xmm1, %ecx
-; CHECK-NEXT:    cltd
-; CHECK-NEXT:    idivl %ecx
-; CHECK-NEXT:    pinsrd $2, %edx, %xmm2
-; CHECK-NEXT:    pextrd $3, %xmm0, %eax
-; CHECK-NEXT:    pextrd $3, %xmm1, %ecx
-; CHECK-NEXT:    cltd
-; CHECK-NEXT:    idivl %ecx
-; CHECK-NEXT:    pinsrd $3, %edx, %xmm2
+; CHECK-NEXT:    pextrb $1, %xmm1, %ecx
+; CHECK-NEXT:    pextrb $1, %xmm0, %eax
+; CHECK-NEXT:    cbtw
+; CHECK-NEXT:    idivb %cl
+; CHECK-NEXT:    movsbl %ah, %ecx
+; CHECK-NEXT:    pextrb $0, %xmm1, %edx
+; CHECK-NEXT:    pextrb $0, %xmm0, %eax
+; CHECK-NEXT:    cbtw
+; CHECK-NEXT:    idivb %dl
+; CHECK-NEXT:    movsbl %ah, %eax
+; CHECK-NEXT:    movd %eax, %xmm2
+; CHECK-NEXT:    pinsrb $1, %ecx, %xmm2
+; CHECK-NEXT:    pextrb $2, %xmm1, %ecx
+; CHECK-NEXT:    pextrb $2, %xmm0, %eax
+; CHECK-NEXT:    cbtw
+; CHECK-NEXT:    idivb %cl
+; CHECK-NEXT:    movsbl %ah, %eax
+; CHECK-NEXT:    pinsrb $2, %eax, %xmm2
+; CHECK-NEXT:    pextrb $3, %xmm1, %ecx
+; CHECK-NEXT:    pextrb $3, %xmm0, %eax
+; CHECK-NEXT:    cbtw
+; CHECK-NEXT:    idivb %cl
+; CHECK-NEXT:    movsbl %ah, %eax
+; CHECK-NEXT:    pinsrb $3, %eax, %xmm2
 ; CHECK-NEXT:    movdqa %xmm2, %xmm0
 ; CHECK-NEXT:    retq
   %rem.r = srem <4 x i8> %num, %rem
diff --git a/llvm/test/CodeGen/X86/scalarize-fp.ll b/llvm/test/CodeGen/X86/scalarize-fp.ll
index 650b948d3..5b98272fd 100644
--- a/llvm/test/CodeGen/X86/scalarize-fp.ll
+++ b/llvm/test/CodeGen/X86/scalarize-fp.ll
@@ -773,3 +773,26 @@ define <8 x float> @splat0_fdiv_const_op0_v8f32(<8 x float> %vx) {
   %r = shufflevector <8 x float> %b, <8 x float> undef, <8 x i32> zeroinitializer
   ret <8 x float> %r
 }
+
+define <4 x float> @multi_use_binop(<4 x float> %x, <4 x float> %y) {
+; SSE-LABEL: multi_use_binop:
+; SSE:       # %bb.0:
+; SSE-NEXT:    mulps %xmm1, %xmm0
+; SSE-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1,2,0]
+; SSE-NEXT:    addps %xmm1, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: multi_use_binop:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmulps %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[0,1,2,0]
+; AVX-NEXT:    vmovddup {{.*#+}} xmm0 = xmm0[0,0]
+; AVX-NEXT:    vaddps %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    retq
+  %mul = fmul <4 x float> %x, %y
+  %mul0 = shufflevector <4 x float> %mul, <4 x float> undef, <4 x i32> <i32 undef, i32 undef, i32 undef, i32 0>
+  %mul1 = shufflevector <4 x float> %mul, <4 x float> undef, <4 x i32> <i32 undef, i32 undef, i32 undef, i32 1>
+  %r = fadd <4 x float> %mul0, %mul1
+  ret <4 x float> %r
+}
diff --git a/llvm/test/CodeGen/X86/sdiv_fix.ll b/llvm/test/CodeGen/X86/sdiv_fix.ll
new file mode 100644
index 000000000..13f10b23d
--- /dev/null
+++ b/llvm/test/CodeGen/X86/sdiv_fix.ll
@@ -0,0 +1,713 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-linux | FileCheck %s --check-prefix=X64
+; RUN: llc < %s -mtriple=i686 -mattr=cmov | FileCheck %s --check-prefix=X86
+
+declare  i4  @llvm.sdiv.fix.i4   (i4,  i4,  i32)
+declare  i15 @llvm.sdiv.fix.i15  (i15, i15, i32)
+declare  i16 @llvm.sdiv.fix.i16  (i16, i16, i32)
+declare  i18 @llvm.sdiv.fix.i18  (i18, i18, i32)
+declare  i64 @llvm.sdiv.fix.i64  (i64, i64, i32)
+declare  <4 x i32> @llvm.sdiv.fix.v4i32(<4 x i32>, <4 x i32>, i32)
+
+define i16 @func(i16 %x, i16 %y) nounwind {
+; X64-LABEL: func:
+; X64:       # %bb.0:
+; X64-NEXT:    movswl %si, %esi
+; X64-NEXT:    movswl %di, %ecx
+; X64-NEXT:    shll $7, %ecx
+; X64-NEXT:    movl %ecx, %eax
+; X64-NEXT:    cltd
+; X64-NEXT:    idivl %esi
+; X64-NEXT:    # kill: def $eax killed $eax def $rax
+; X64-NEXT:    leal -1(%rax), %edi
+; X64-NEXT:    testl %esi, %esi
+; X64-NEXT:    sets %sil
+; X64-NEXT:    testl %ecx, %ecx
+; X64-NEXT:    sets %cl
+; X64-NEXT:    xorb %sil, %cl
+; X64-NEXT:    testl %edx, %edx
+; X64-NEXT:    setne %dl
+; X64-NEXT:    testb %cl, %dl
+; X64-NEXT:    cmovnel %edi, %eax
+; X64-NEXT:    # kill: def $ax killed $ax killed $rax
+; X64-NEXT:    retq
+;
+; X86-LABEL: func:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movswl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movswl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    shll $7, %ecx
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    cltd
+; X86-NEXT:    idivl %esi
+; X86-NEXT:    leal -1(%eax), %edi
+; X86-NEXT:    testl %esi, %esi
+; X86-NEXT:    sets %bl
+; X86-NEXT:    testl %ecx, %ecx
+; X86-NEXT:    sets %cl
+; X86-NEXT:    xorb %bl, %cl
+; X86-NEXT:    testl %edx, %edx
+; X86-NEXT:    setne %dl
+; X86-NEXT:    testb %cl, %dl
+; X86-NEXT:    cmovnel %edi, %eax
+; X86-NEXT:    # kill: def $ax killed $ax killed $eax
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    retl
+  %tmp = call i16 @llvm.sdiv.fix.i16(i16 %x, i16 %y, i32 7)
+  ret i16 %tmp
+}
+
+define i16 @func2(i8 %x, i8 %y) nounwind {
+; X64-LABEL: func2:
+; X64:       # %bb.0:
+; X64-NEXT:    movsbl %dil, %eax
+; X64-NEXT:    movsbl %sil, %ecx
+; X64-NEXT:    movswl %cx, %esi
+; X64-NEXT:    movswl %ax, %ecx
+; X64-NEXT:    shll $14, %ecx
+; X64-NEXT:    movl %ecx, %eax
+; X64-NEXT:    cltd
+; X64-NEXT:    idivl %esi
+; X64-NEXT:    # kill: def $eax killed $eax def $rax
+; X64-NEXT:    leal -1(%rax), %edi
+; X64-NEXT:    testl %esi, %esi
+; X64-NEXT:    sets %sil
+; X64-NEXT:    testl %ecx, %ecx
+; X64-NEXT:    sets %cl
+; X64-NEXT:    xorb %sil, %cl
+; X64-NEXT:    testl %edx, %edx
+; X64-NEXT:    setne %dl
+; X64-NEXT:    testb %cl, %dl
+; X64-NEXT:    cmovel %eax, %edi
+; X64-NEXT:    addl %edi, %edi
+; X64-NEXT:    movswl %di, %eax
+; X64-NEXT:    shrl %eax
+; X64-NEXT:    # kill: def $ax killed $ax killed $eax
+; X64-NEXT:    retq
+;
+; X86-LABEL: func2:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movsbl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movsbl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    shll $14, %ecx
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    cltd
+; X86-NEXT:    idivl %esi
+; X86-NEXT:    leal -1(%eax), %edi
+; X86-NEXT:    testl %esi, %esi
+; X86-NEXT:    sets %bl
+; X86-NEXT:    testl %ecx, %ecx
+; X86-NEXT:    sets %cl
+; X86-NEXT:    xorb %bl, %cl
+; X86-NEXT:    testl %edx, %edx
+; X86-NEXT:    setne %dl
+; X86-NEXT:    testb %cl, %dl
+; X86-NEXT:    cmovel %eax, %edi
+; X86-NEXT:    addl %edi, %edi
+; X86-NEXT:    movswl %di, %eax
+; X86-NEXT:    shrl %eax
+; X86-NEXT:    # kill: def $ax killed $ax killed $eax
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    retl
+  %x2 = sext i8 %x to i15
+  %y2 = sext i8 %y to i15
+  %tmp = call i15 @llvm.sdiv.fix.i15(i15 %x2, i15 %y2, i32 14)
+  %tmp2 = sext i15 %tmp to i16
+  ret i16 %tmp2
+}
+
+define i16 @func3(i15 %x, i8 %y) nounwind {
+; X64-LABEL: func3:
+; X64:       # %bb.0:
+; X64-NEXT:    shll $8, %esi
+; X64-NEXT:    movswl %si, %ecx
+; X64-NEXT:    addl %edi, %edi
+; X64-NEXT:    shrl $4, %ecx
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    cwtd
+; X64-NEXT:    idivw %cx
+; X64-NEXT:    # kill: def $ax killed $ax def $rax
+; X64-NEXT:    leal -1(%rax), %esi
+; X64-NEXT:    testw %di, %di
+; X64-NEXT:    sets %dil
+; X64-NEXT:    testw %cx, %cx
+; X64-NEXT:    sets %cl
+; X64-NEXT:    xorb %dil, %cl
+; X64-NEXT:    testw %dx, %dx
+; X64-NEXT:    setne %dl
+; X64-NEXT:    testb %cl, %dl
+; X64-NEXT:    cmovel %eax, %esi
+; X64-NEXT:    addl %esi, %esi
+; X64-NEXT:    movswl %si, %eax
+; X64-NEXT:    shrl %eax
+; X64-NEXT:    # kill: def $ax killed $ax killed $eax
+; X64-NEXT:    retq
+;
+; X86-LABEL: func3:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    shll $8, %eax
+; X86-NEXT:    movswl %ax, %esi
+; X86-NEXT:    addl %ecx, %ecx
+; X86-NEXT:    shrl $4, %esi
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    cwtd
+; X86-NEXT:    idivw %si
+; X86-NEXT:    # kill: def $ax killed $ax def $eax
+; X86-NEXT:    leal -1(%eax), %edi
+; X86-NEXT:    testw %cx, %cx
+; X86-NEXT:    sets %cl
+; X86-NEXT:    testw %si, %si
+; X86-NEXT:    sets %ch
+; X86-NEXT:    xorb %cl, %ch
+; X86-NEXT:    testw %dx, %dx
+; X86-NEXT:    setne %cl
+; X86-NEXT:    testb %ch, %cl
+; X86-NEXT:    cmovel %eax, %edi
+; X86-NEXT:    addl %edi, %edi
+; X86-NEXT:    movswl %di, %eax
+; X86-NEXT:    shrl %eax
+; X86-NEXT:    # kill: def $ax killed $ax killed $eax
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    retl
+  %y2 = sext i8 %y to i15
+  %y3 = shl i15 %y2, 7
+  %tmp = call i15 @llvm.sdiv.fix.i15(i15 %x, i15 %y3, i32 4)
+  %tmp2 = sext i15 %tmp to i16
+  ret i16 %tmp2
+}
+
+define i4 @func4(i4 %x, i4 %y) nounwind {
+; X64-LABEL: func4:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rbx
+; X64-NEXT:    shlb $4, %sil
+; X64-NEXT:    sarb $4, %sil
+; X64-NEXT:    shlb $4, %dil
+; X64-NEXT:    sarb $4, %dil
+; X64-NEXT:    shlb $2, %dil
+; X64-NEXT:    movsbl %dil, %ecx
+; X64-NEXT:    movl %ecx, %eax
+; X64-NEXT:    idivb %sil
+; X64-NEXT:    movsbl %ah, %ebx
+; X64-NEXT:    movzbl %al, %edi
+; X64-NEXT:    leal -1(%rdi), %eax
+; X64-NEXT:    movzbl %al, %eax
+; X64-NEXT:    testb %sil, %sil
+; X64-NEXT:    sets %dl
+; X64-NEXT:    testb %cl, %cl
+; X64-NEXT:    sets %cl
+; X64-NEXT:    xorb %dl, %cl
+; X64-NEXT:    testb %bl, %bl
+; X64-NEXT:    setne %dl
+; X64-NEXT:    testb %cl, %dl
+; X64-NEXT:    cmovel %edi, %eax
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    popq %rbx
+; X64-NEXT:    retq
+;
+; X86-LABEL: func4:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %dl
+; X86-NEXT:    shlb $4, %dl
+; X86-NEXT:    sarb $4, %dl
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %dh
+; X86-NEXT:    shlb $4, %dh
+; X86-NEXT:    sarb $4, %dh
+; X86-NEXT:    shlb $2, %dh
+; X86-NEXT:    movsbl %dh, %eax
+; X86-NEXT:    idivb %dl
+; X86-NEXT:    movsbl %ah, %ecx
+; X86-NEXT:    movzbl %al, %esi
+; X86-NEXT:    decb %al
+; X86-NEXT:    movzbl %al, %eax
+; X86-NEXT:    testb %dl, %dl
+; X86-NEXT:    sets %dl
+; X86-NEXT:    testb %dh, %dh
+; X86-NEXT:    sets %dh
+; X86-NEXT:    xorb %dl, %dh
+; X86-NEXT:    testb %cl, %cl
+; X86-NEXT:    setne %cl
+; X86-NEXT:    testb %dh, %cl
+; X86-NEXT:    cmovel %esi, %eax
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
+  %tmp = call i4 @llvm.sdiv.fix.i4(i4 %x, i4 %y, i32 2)
+  ret i4 %tmp
+}
+
+define i64 @func5(i64 %x, i64 %y) nounwind {
+; X64-LABEL: func5:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rbp
+; X64-NEXT:    pushq %r15
+; X64-NEXT:    pushq %r14
+; X64-NEXT:    pushq %r13
+; X64-NEXT:    pushq %r12
+; X64-NEXT:    pushq %rbx
+; X64-NEXT:    subq $24, %rsp
+; X64-NEXT:    movq %rsi, %r14
+; X64-NEXT:    movq %rdi, %r15
+; X64-NEXT:    movq %rdi, %rax
+; X64-NEXT:    shrq $33, %rax
+; X64-NEXT:    movq %rdi, %rbx
+; X64-NEXT:    sarq $63, %rbx
+; X64-NEXT:    shlq $31, %rbx
+; X64-NEXT:    orq %rax, %rbx
+; X64-NEXT:    sets {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Folded Spill
+; X64-NEXT:    shlq $31, %r15
+; X64-NEXT:    movq %rsi, %r12
+; X64-NEXT:    sarq $63, %r12
+; X64-NEXT:    movq %r15, %rdi
+; X64-NEXT:    movq %rbx, %rsi
+; X64-NEXT:    movq %r14, %rdx
+; X64-NEXT:    movq %r12, %rcx
+; X64-NEXT:    callq __divti3
+; X64-NEXT:    movq %rax, %r13
+; X64-NEXT:    decq %rax
+; X64-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    testq %r12, %r12
+; X64-NEXT:    sets %bpl
+; X64-NEXT:    xorb {{[-0-9]+}}(%r{{[sb]}}p), %bpl # 1-byte Folded Reload
+; X64-NEXT:    movq %r15, %rdi
+; X64-NEXT:    movq %rbx, %rsi
+; X64-NEXT:    movq %r14, %rdx
+; X64-NEXT:    movq %r12, %rcx
+; X64-NEXT:    callq __modti3
+; X64-NEXT:    orq %rax, %rdx
+; X64-NEXT:    setne %al
+; X64-NEXT:    testb %bpl, %al
+; X64-NEXT:    cmovneq {{[-0-9]+}}(%r{{[sb]}}p), %r13 # 8-byte Folded Reload
+; X64-NEXT:    movq %r13, %rax
+; X64-NEXT:    addq $24, %rsp
+; X64-NEXT:    popq %rbx
+; X64-NEXT:    popq %r12
+; X64-NEXT:    popq %r13
+; X64-NEXT:    popq %r14
+; X64-NEXT:    popq %r15
+; X64-NEXT:    popq %rbp
+; X64-NEXT:    retq
+;
+; X86-LABEL: func5:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    movl %esp, %ebp
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    andl $-8, %esp
+; X86-NEXT:    subl $72, %esp
+; X86-NEXT:    movl 8(%ebp), %ecx
+; X86-NEXT:    movl 12(%ebp), %edx
+; X86-NEXT:    movl 20(%ebp), %ebx
+; X86-NEXT:    sarl $31, %ebx
+; X86-NEXT:    movl %edx, %eax
+; X86-NEXT:    shldl $31, %ecx, %eax
+; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    shll $31, %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl %edx, %esi
+; X86-NEXT:    sarl $31, %esi
+; X86-NEXT:    movl %esi, %edi
+; X86-NEXT:    shldl $31, %edx, %esi
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    rorl %edi
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl 20(%ebp)
+; X86-NEXT:    pushl 16(%ebp)
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    pushl %ecx
+; X86-NEXT:    pushl %edx
+; X86-NEXT:    calll __divti3
+; X86-NEXT:    addl $32, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    subl $1, %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    sbbl $0, %eax
+; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    testl %ebx, %ebx
+; X86-NEXT:    sets %al
+; X86-NEXT:    testl %edi, %edi
+; X86-NEXT:    sets %cl
+; X86-NEXT:    xorb %al, %cl
+; X86-NEXT:    movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl 20(%ebp)
+; X86-NEXT:    pushl 16(%ebp)
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    pushl {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; X86-NEXT:    pushl {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    calll __modti3
+; X86-NEXT:    addl $32, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    orl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    orl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    orl %eax, %ecx
+; X86-NEXT:    setne %al
+; X86-NEXT:    testb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Reload
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT:    cmovel {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT:    cmovel {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X86-NEXT:    leal -12(%ebp), %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl
+  %tmp = call i64 @llvm.sdiv.fix.i64(i64 %x, i64 %y, i32 31)
+  ret i64 %tmp
+}
+
+define i18 @func6(i16 %x, i16 %y) nounwind {
+; X64-LABEL: func6:
+; X64:       # %bb.0:
+; X64-NEXT:    movswl %di, %ecx
+; X64-NEXT:    movswl %si, %esi
+; X64-NEXT:    shll $7, %ecx
+; X64-NEXT:    movl %ecx, %eax
+; X64-NEXT:    cltd
+; X64-NEXT:    idivl %esi
+; X64-NEXT:    # kill: def $eax killed $eax def $rax
+; X64-NEXT:    leal -1(%rax), %edi
+; X64-NEXT:    testl %esi, %esi
+; X64-NEXT:    sets %sil
+; X64-NEXT:    testl %ecx, %ecx
+; X64-NEXT:    sets %cl
+; X64-NEXT:    xorb %sil, %cl
+; X64-NEXT:    testl %edx, %edx
+; X64-NEXT:    setne %dl
+; X64-NEXT:    testb %cl, %dl
+; X64-NEXT:    cmovnel %edi, %eax
+; X64-NEXT:    # kill: def $eax killed $eax killed $rax
+; X64-NEXT:    retq
+;
+; X86-LABEL: func6:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movswl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movswl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    shll $7, %ecx
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    cltd
+; X86-NEXT:    idivl %esi
+; X86-NEXT:    leal -1(%eax), %edi
+; X86-NEXT:    testl %esi, %esi
+; X86-NEXT:    sets %bl
+; X86-NEXT:    testl %ecx, %ecx
+; X86-NEXT:    sets %cl
+; X86-NEXT:    xorb %bl, %cl
+; X86-NEXT:    testl %edx, %edx
+; X86-NEXT:    setne %dl
+; X86-NEXT:    testb %cl, %dl
+; X86-NEXT:    cmovnel %edi, %eax
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    retl
+  %x2 = sext i16 %x to i18
+  %y2 = sext i16 %y to i18
+  %tmp = call i18 @llvm.sdiv.fix.i18(i18 %x2, i18 %y2, i32 7)
+  ret i18 %tmp
+}
+
+define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
+; X64-LABEL: vec:
+; X64:       # %bb.0:
+; X64-NEXT:    pxor %xmm2, %xmm2
+; X64-NEXT:    pcmpgtd %xmm1, %xmm2
+; X64-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,0,1]
+; X64-NEXT:    movdqa %xmm1, %xmm4
+; X64-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
+; X64-NEXT:    movq %xmm4, %rcx
+; X64-NEXT:    pxor %xmm2, %xmm2
+; X64-NEXT:    pcmpgtd %xmm0, %xmm2
+; X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; X64-NEXT:    psllq $31, %xmm0
+; X64-NEXT:    movq %xmm0, %rax
+; X64-NEXT:    cqto
+; X64-NEXT:    idivq %rcx
+; X64-NEXT:    movq %rax, %r8
+; X64-NEXT:    movq %rdx, %r11
+; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[2,3,0,1]
+; X64-NEXT:    movq %xmm2, %rcx
+; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; X64-NEXT:    movq %xmm2, %rax
+; X64-NEXT:    cqto
+; X64-NEXT:    idivq %rcx
+; X64-NEXT:    movq %rax, %r10
+; X64-NEXT:    movq %rdx, %rcx
+; X64-NEXT:    pxor %xmm2, %xmm2
+; X64-NEXT:    pcmpgtd %xmm3, %xmm2
+; X64-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; X64-NEXT:    movq %xmm3, %rdi
+; X64-NEXT:    pxor %xmm2, %xmm2
+; X64-NEXT:    pcmpgtd %xmm1, %xmm2
+; X64-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; X64-NEXT:    psllq $31, %xmm1
+; X64-NEXT:    movq %xmm1, %rax
+; X64-NEXT:    cqto
+; X64-NEXT:    idivq %rdi
+; X64-NEXT:    movq %rax, %r9
+; X64-NEXT:    movq %rdx, %rdi
+; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,3,0,1]
+; X64-NEXT:    movq %xmm2, %rsi
+; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; X64-NEXT:    movq %xmm2, %rax
+; X64-NEXT:    cqto
+; X64-NEXT:    idivq %rsi
+; X64-NEXT:    movq %r11, %xmm2
+; X64-NEXT:    movq %rcx, %xmm5
+; X64-NEXT:    pxor %xmm6, %xmm6
+; X64-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm5[0]
+; X64-NEXT:    pcmpeqd %xmm6, %xmm2
+; X64-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[1,0,3,2]
+; X64-NEXT:    pand %xmm2, %xmm5
+; X64-NEXT:    pxor %xmm2, %xmm2
+; X64-NEXT:    pcmpgtd %xmm4, %xmm2
+; X64-NEXT:    pxor %xmm4, %xmm4
+; X64-NEXT:    pcmpgtd %xmm0, %xmm4
+; X64-NEXT:    movq %r8, %xmm0
+; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; X64-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; X64-NEXT:    pxor %xmm2, %xmm4
+; X64-NEXT:    movq %r10, %xmm2
+; X64-NEXT:    pandn %xmm4, %xmm5
+; X64-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; X64-NEXT:    movdqa %xmm5, %xmm2
+; X64-NEXT:    pandn %xmm0, %xmm2
+; X64-NEXT:    pcmpeqd %xmm4, %xmm4
+; X64-NEXT:    paddq %xmm4, %xmm0
+; X64-NEXT:    pand %xmm5, %xmm0
+; X64-NEXT:    por %xmm2, %xmm0
+; X64-NEXT:    movq %rdi, %xmm2
+; X64-NEXT:    movq %rdx, %xmm5
+; X64-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm5[0]
+; X64-NEXT:    pcmpeqd %xmm6, %xmm2
+; X64-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[1,0,3,2]
+; X64-NEXT:    pand %xmm2, %xmm5
+; X64-NEXT:    pxor %xmm2, %xmm2
+; X64-NEXT:    pcmpgtd %xmm3, %xmm2
+; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; X64-NEXT:    pcmpgtd %xmm1, %xmm6
+; X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm6[1,1,3,3]
+; X64-NEXT:    pxor %xmm2, %xmm1
+; X64-NEXT:    pandn %xmm1, %xmm5
+; X64-NEXT:    movq %r9, %xmm1
+; X64-NEXT:    movq %rax, %xmm2
+; X64-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; X64-NEXT:    movdqa %xmm5, %xmm2
+; X64-NEXT:    pandn %xmm1, %xmm2
+; X64-NEXT:    paddq %xmm4, %xmm1
+; X64-NEXT:    pand %xmm5, %xmm1
+; X64-NEXT:    por %xmm2, %xmm1
+; X64-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
+; X64-NEXT:    retq
+;
+; X86-LABEL: vec:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $64, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, %edx
+; X86-NEXT:    sarl $31, %edx
+; X86-NEXT:    movl %edi, %esi
+; X86-NEXT:    shll $31, %esi
+; X86-NEXT:    movl %ebx, %eax
+; X86-NEXT:    shrl %eax
+; X86-NEXT:    andl $-2147483648, %ebx # imm = 0x80000000
+; X86-NEXT:    orl %eax, %ebx
+; X86-NEXT:    sets {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X86-NEXT:    movl %ebp, %eax
+; X86-NEXT:    shrl %eax
+; X86-NEXT:    andl $-2147483648, %ebp # imm = 0x80000000
+; X86-NEXT:    orl %eax, %ebp
+; X86-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    sets {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    shrl %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-NEXT:    andl $-2147483648, %ebp # imm = 0x80000000
+; X86-NEXT:    orl %eax, %ebp
+; X86-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    sets (%esp) # 1-byte Folded Spill
+; X86-NEXT:    movl %edi, %eax
+; X86-NEXT:    shrl %eax
+; X86-NEXT:    andl $-2147483648, %edi # imm = 0x80000000
+; X86-NEXT:    orl %eax, %edi
+; X86-NEXT:    sets {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X86-NEXT:    pushl %edx
+; X86-NEXT:    movl %edx, %ebp
+; X86-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    pushl %ecx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    calll __moddi3
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    calll __divdi3
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    shll $31, %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl %edx, %eax
+; X86-NEXT:    sarl $31, %eax
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    movl %eax, %esi
+; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    pushl %edx
+; X86-NEXT:    movl %edx, %ebp
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %ecx
+; X86-NEXT:    movl %ecx, %edi
+; X86-NEXT:    calll __moddi3
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    calll __divdi3
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    shll $31, %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, %edx
+; X86-NEXT:    sarl $31, %edx
+; X86-NEXT:    pushl %edx
+; X86-NEXT:    movl %edx, %ebp
+; X86-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    pushl %ecx
+; X86-NEXT:    movl %ecx, %edi
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    movl %eax, %esi
+; X86-NEXT:    calll __moddi3
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    calll __divdi3
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    shll $31, %eax
+; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, %ebp
+; X86-NEXT:    sarl $31, %ebp
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    pushl %ecx
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    calll __moddi3
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl %edx, %edi
+; X86-NEXT:    testl %ebp, %ebp
+; X86-NEXT:    sets %bl
+; X86-NEXT:    xorb (%esp), %bl # 1-byte Folded Reload
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    pushl {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; X86-NEXT:    calll __divdi3
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X86-NEXT:    setne %cl
+; X86-NEXT:    testb %bl, %cl
+; X86-NEXT:    leal -1(%eax), %ecx
+; X86-NEXT:    cmovel %eax, %ecx
+; X86-NEXT:    cmpl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; X86-NEXT:    sets %al
+; X86-NEXT:    xorb {{[-0-9]+}}(%e{{[sb]}}p), %al # 1-byte Folded Reload
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X86-NEXT:    setne %dl
+; X86-NEXT:    testb %al, %dl
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT:    leal -1(%eax), %edi
+; X86-NEXT:    cmovel %eax, %edi
+; X86-NEXT:    cmpl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; X86-NEXT:    sets %dl
+; X86-NEXT:    xorb {{[-0-9]+}}(%e{{[sb]}}p), %dl # 1-byte Folded Reload
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-NEXT:    setne %dh
+; X86-NEXT:    testb %dl, %dh
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT:    leal -1(%eax), %edx
+; X86-NEXT:    cmovel %eax, %edx
+; X86-NEXT:    cmpl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; X86-NEXT:    sets %bl
+; X86-NEXT:    xorb {{[-0-9]+}}(%e{{[sb]}}p), %bl # 1-byte Folded Reload
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-NEXT:    setne %bh
+; X86-NEXT:    testb %bl, %bh
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT:    leal -1(%eax), %esi
+; X86-NEXT:    cmovel %eax, %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl %esi, 12(%eax)
+; X86-NEXT:    movl %edx, 8(%eax)
+; X86-NEXT:    movl %edi, 4(%eax)
+; X86-NEXT:    movl %ecx, (%eax)
+; X86-NEXT:    addl $64, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl $4
+  %tmp = call <4 x i32> @llvm.sdiv.fix.v4i32(<4 x i32> %x, <4 x i32> %y, i32 31)
+  ret <4 x i32> %tmp
+}
diff --git a/llvm/test/CodeGen/X86/segmented-stacks.ll b/llvm/test/CodeGen/X86/segmented-stacks.ll
index fac9a3339..c4539f7b1 100644
--- a/llvm/test/CodeGen/X86/segmented-stacks.ll
+++ b/llvm/test/CodeGen/X86/segmented-stacks.ll
@@ -611,7 +611,7 @@ define void @test_nostack() #0 {
 ; X32-Linux-NOT:   calll __morestack
 
 ; X64-Linux-LABEL: test_nostack:
-; X32-Linux-NOT:   callq __morestack
+; X64-Linux-NOT:   callq __morestack
 
 ; X32ABI-LABEL: test_nostack:
 ; X32ABI-NOT:   callq __morestack
@@ -688,6 +688,45 @@ define i32 @test_sibling_call_empty_frame(i32 %x) #0 {
 
 }
 
+; Test that unused nested argument doesn't need saving/restoring.
+
+define i32 @test_nested_unused(i32 * nest %unused) #0 {
+       %mem = alloca i32, i32 10
+       call void @dummy_use (i32* %mem, i32 10)
+       ret i32 123
+
+; X64-Linux-LABEL: test_nested_unused:
+; X64-Linux-NOT:   movq %r10, %rax
+; X64-Linux:       callq __morestack
+; X64-Linux-NOT:   movq %rax, %r10
+
+; X64-Darwin-LABEL: test_nested_unused:
+; X64-Darwin-NOT:   movq %r10, %rax
+; X64-Darwin:       callq ___morestack
+; X64-Darwin-NOT:   movq %rax, %r10
+
+; X64-FreeBSD-LABEL: test_nested_unused:
+; X64-FreeBSD-NOT:   movq %r10, %rax
+; X64-FreeBSD:       callq __morestack
+; X64-FreeBSD-NOT:   movq %rax, %r10
+
+; X64-DFlyBSD-LABEL: test_nested_unused:
+; X64-DFlyBSD-NOT:   movq %r10, %rax
+; X64-DFlyBSD:       callq __morestack
+; X64-DFlyBSD-NOT:   movq %rax, %r10
+
+; X64-MinGW-LABEL: test_nested_unused:
+; X64-MinGW-NOT:   movq %r10, %rax
+; X64-MinGW:       callq __morestack
+; X64-MinGW-NOT:   movq %rax, %r10
+
+; X32ABI-LABEL: test_nested_unused:
+; X32ABI-NOT:   movl %r10d, %eax
+; X32ABI:       callq __morestack
+; X32ABI-NOT:   movq %rax, %r10
+
+}
+
 attributes #0 = { "split-stack" }
 
 ; X64-Linux-Large: .rodata
diff --git a/llvm/test/CodeGen/X86/seh-catchpad.ll b/llvm/test/CodeGen/X86/seh-catchpad.ll
index 7dc935315..cb558e23c 100644
--- a/llvm/test/CodeGen/X86/seh-catchpad.ll
+++ b/llvm/test/CodeGen/X86/seh-catchpad.ll
@@ -87,11 +87,11 @@ __except.ret:                                     ; preds = %catch.dispatch.7
 ; CHECK: .seh_proc main
 ; CHECK:         .seh_handler __C_specific_handler, @unwind, @except
 ; CHECK:         pushq   %rbp
-; CHECK:         .seh_pushreg 5
+; CHECK:         .seh_pushreg %rbp
 ; CHECK:         subq    $32, %rsp
 ; CHECK:         .seh_stackalloc 32
 ; CHECK:         leaq    32(%rsp), %rbp
-; CHECK:         .seh_setframe 5, 32
+; CHECK:         .seh_setframe %rbp, 32
 ; CHECK:         .seh_endprologue
 ; CHECK: .Ltmp0:
 ; CHECK:         movl    $1, %ecx
@@ -151,7 +151,7 @@ __except.ret:                                     ; preds = %catch.dispatch.7
 ; CHECK: .LBB1_[[finbb]]:                                # %ehcleanup
 ; CHECK:         movq    %rdx, 16(%rsp)
 ; CHECK:         pushq   %rbp
-; CHECK:         .seh_pushreg 5
+; CHECK:         .seh_pushreg %rbp
 ; CHECK:         subq    $32, %rsp
 ; CHECK:         .seh_stackalloc 32
 ; CHECK:         leaq    32(%rdx), %rbp
@@ -187,9 +187,9 @@ entry:
 ; Function Attrs: nounwind
 declare i32 @puts(i8* nocapture readonly) #3
 
-attributes #0 = { nounwind readnone "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-features"="+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-features"="+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #2 = { noinline nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-features"="+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #3 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-features"="+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind readnone "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-features"="+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-features"="+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { noinline nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-features"="+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-features"="+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #4 = { noinline }
 attributes #5 = { nounwind }
diff --git a/llvm/test/CodeGen/X86/seh-except-finally.ll b/llvm/test/CodeGen/X86/seh-except-finally.ll
index b29788cd0..0a1e02217 100644
--- a/llvm/test/CodeGen/X86/seh-except-finally.ll
+++ b/llvm/test/CodeGen/X86/seh-except-finally.ll
@@ -137,10 +137,10 @@ declare i8* @llvm.localaddress() #4
 ; Function Attrs: nounwind readnone
 declare i32 @llvm.eh.typeid.for(i8*) #4
 
-attributes #0 = { noinline nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #2 = { noinline nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #3 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noinline nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { noinline nounwind "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #4 = { nounwind readnone }
 attributes #5 = { noinline }
 attributes #6 = { nounwind }
diff --git a/llvm/test/CodeGen/X86/seh-no-invokes.ll b/llvm/test/CodeGen/X86/seh-no-invokes.ll
index f32ab5c1b..213d608f9 100644
--- a/llvm/test/CodeGen/X86/seh-no-invokes.ll
+++ b/llvm/test/CodeGen/X86/seh-no-invokes.ll
@@ -65,8 +65,8 @@ declare i32 @_except_handler3(...)
 ; Function Attrs: nounwind
 declare void @llvm.localescape(...) #3
 
-attributes #0 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind readnone }
 attributes #3 = { nounwind }
 
diff --git a/llvm/test/CodeGen/X86/select-1-or-neg1.ll b/llvm/test/CodeGen/X86/select-1-or-neg1.ll
index b0244fe7d..c85cc08f8 100644
--- a/llvm/test/CodeGen/X86/select-1-or-neg1.ll
+++ b/llvm/test/CodeGen/X86/select-1-or-neg1.ll
@@ -19,8 +19,8 @@ define i32 @PR28968(i32 %x) {
 ; SLOWLEA3-NEXT:    xorl %eax, %eax
 ; SLOWLEA3-NEXT:    cmpl $1, %edi
 ; SLOWLEA3-NEXT:    sete %al
-; SLOWLEA3-NEXT:    leal (%rax,%rax), %eax
-; SLOWLEA3-NEXT:    addl $-1, %eax
+; SLOWLEA3-NEXT:    addl %eax, %eax
+; SLOWLEA3-NEXT:    decl %eax
 ; SLOWLEA3-NEXT:    retq
   %cmp = icmp eq i32 %x, 1
   %sel = select i1 %cmp, i32 1, i32 -1
diff --git a/llvm/test/CodeGen/X86/select-of-fp-constants.ll b/llvm/test/CodeGen/X86/select-of-fp-constants.ll
index cc9ac9e56..f492b255d 100644
--- a/llvm/test/CodeGen/X86/select-of-fp-constants.ll
+++ b/llvm/test/CodeGen/X86/select-of-fp-constants.ll
@@ -61,7 +61,7 @@ define float @fcmp_select_fp_constants(float %x) nounwind readnone {
 ; X32_AVX512F-LABEL: fcmp_select_fp_constants:
 ; X32_AVX512F:       # %bb.0:
 ; X32_AVX512F-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X32_AVX512F-NEXT:    vcmpneqss {{[0-9]+}}(%esp), %xmm0, %k0
+; X32_AVX512F-NEXT:    vcmpneqss {{\.LCPI.*}}, %xmm0, %k0
 ; X32_AVX512F-NEXT:    kmovw %k0, %eax
 ; X32_AVX512F-NEXT:    flds {{\.LCPI.*}}(,%eax,4)
 ; X32_AVX512F-NEXT:    retl
@@ -84,10 +84,9 @@ define float @fcmp_select_fp_constants(float %x) nounwind readnone {
 ;
 ; X64_AVX512F-LABEL: fcmp_select_fp_constants:
 ; X64_AVX512F:       # %bb.0:
-; X64_AVX512F-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; X64_AVX512F-NEXT:    vcmpneqss {{.*}}(%rip), %xmm0, %k1
 ; X64_AVX512F-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X64_AVX512F-NEXT:    vmovss %xmm1, %xmm0, %xmm0 {%k1}
+; X64_AVX512F-NEXT:    vmovss {{.*}}(%rip), %xmm0 {%k1}
 ; X64_AVX512F-NEXT:    retq
  %c = fcmp une float %x, -4.0
  %r = select i1 %c, float 42.0, float 23.0
diff --git a/llvm/test/CodeGen/X86/select-sra.ll b/llvm/test/CodeGen/X86/select-sra.ll
new file mode 100644
index 000000000..4d984aab0
--- /dev/null
+++ b/llvm/test/CodeGen/X86/select-sra.ll
@@ -0,0 +1,196 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-- | FileCheck %s
+
+define i8 @isnonneg_i8(i8 %x) {
+; CHECK-LABEL: isnonneg_i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    sarb $7, %al
+; CHECK-NEXT:    orb $42, %al
+; CHECK-NEXT:    # kill: def $al killed $al killed $eax
+; CHECK-NEXT:    retq
+  %cond = icmp sgt i8 %x, -1
+  %r = select i1 %cond, i8 42, i8 -1
+  ret i8 %r
+}
+
+define i16 @isnonneg_i16(i16 %x) {
+; CHECK-LABEL: isnonneg_i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movswl %di, %eax
+; CHECK-NEXT:    sarl $15, %eax
+; CHECK-NEXT:    orl $542, %eax # imm = 0x21E
+; CHECK-NEXT:    # kill: def $ax killed $ax killed $eax
+; CHECK-NEXT:    retq
+  %cond = icmp sgt i16 %x, -1
+  %r = select i1 %cond, i16 542, i16 -1
+  ret i16 %r
+}
+
+define i32 @isnonneg_i32(i32 %x) {
+; CHECK-LABEL: isnonneg_i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    sarl $31, %eax
+; CHECK-NEXT:    orl $-42, %eax
+; CHECK-NEXT:    retq
+  %cond = icmp sgt i32 %x, -1
+  %r = select i1 %cond, i32 -42, i32 -1
+  ret i32 %r
+}
+
+define i64 @isnonneg_i64(i64 %x) {
+; CHECK-LABEL: isnonneg_i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    sarq $63, %rax
+; CHECK-NEXT:    orq $2342342, %rax # imm = 0x23BDC6
+; CHECK-NEXT:    retq
+  %cond = icmp sgt i64 %x, -1
+  %r = select i1 %cond, i64 2342342, i64 -1
+  ret i64 %r
+}
+
+define <16 x i8> @isnonneg_v16i8(<16 x i8> %x) {
+; CHECK-LABEL: isnonneg_v16i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pxor %xmm1, %xmm1
+; CHECK-NEXT:    pcmpgtb %xmm0, %xmm1
+; CHECK-NEXT:    por {{.*}}(%rip), %xmm1
+; CHECK-NEXT:    movdqa %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %cond = icmp sgt <16 x i8> %x, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
+  %r = select <16 x i1> %cond, <16 x i8> <i8 12, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42>, <16 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
+  ret <16 x i8> %r
+}
+
+define <8 x i16> @isnonneg_v8i16(<8 x i16> %x) {
+; CHECK-LABEL: isnonneg_v8i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    psraw $15, %xmm0
+; CHECK-NEXT:    por {{.*}}(%rip), %xmm0
+; CHECK-NEXT:    retq
+  %cond = icmp sgt <8 x i16> %x, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>
+  %r = select <8 x i1> %cond, <8 x i16> <i16 1, i16 542, i16 542, i16 542, i16 542, i16 542, i16 542, i16 1>, <8 x i16> <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>
+  ret <8 x i16> %r
+}
+
+define <4 x i32> @isnonneg_v4i32(<4 x i32> %x) {
+; CHECK-LABEL: isnonneg_v4i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    psrad $31, %xmm0
+; CHECK-NEXT:    por {{.*}}(%rip), %xmm0
+; CHECK-NEXT:    retq
+  %cond = icmp sgt <4 x i32> %x, <i32 -1, i32 -1, i32 -1, i32 -1>
+  %r = select <4 x i1> %cond, <4 x i32> <i32 0, i32 42, i32 -42, i32 1>, <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>
+  ret <4 x i32> %r
+}
+
+define <2 x i64> @isnonneg_v2i64(<2 x i64> %x) {
+; CHECK-LABEL: isnonneg_v2i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    psrad $31, %xmm0
+; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; CHECK-NEXT:    por {{.*}}(%rip), %xmm0
+; CHECK-NEXT:    retq
+  %cond = icmp sgt <2 x i64> %x, <i64 -1, i64 -1>
+  %r = select <2 x i1> %cond, <2 x i64> <i64 2342342, i64 12>, <2 x i64> <i64 -1, i64 -1>
+  ret <2 x i64> %r
+}
+
+define i8 @isneg_i8(i8 %x) {
+; CHECK-LABEL: isneg_i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    sarb $7, %al
+; CHECK-NEXT:    andb $42, %al
+; CHECK-NEXT:    # kill: def $al killed $al killed $eax
+; CHECK-NEXT:    retq
+  %cond = icmp slt i8 %x, 0
+  %r = select i1 %cond, i8 42, i8 0
+  ret i8 %r
+}
+
+define i16 @isneg_i16(i16 %x) {
+; CHECK-LABEL: isneg_i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movswl %di, %eax
+; CHECK-NEXT:    shrl $15, %eax
+; CHECK-NEXT:    andl $542, %eax # imm = 0x21E
+; CHECK-NEXT:    # kill: def $ax killed $ax killed $eax
+; CHECK-NEXT:    retq
+  %cond = icmp slt i16 %x, 0
+  %r = select i1 %cond, i16 542, i16 0
+  ret i16 %r
+}
+
+define i32 @isneg_i32(i32 %x) {
+; CHECK-LABEL: isneg_i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    sarl $31, %eax
+; CHECK-NEXT:    andl $-42, %eax
+; CHECK-NEXT:    retq
+  %cond = icmp slt i32 %x, 0
+  %r = select i1 %cond, i32 -42, i32 0
+  ret i32 %r
+}
+
+define i64 @isneg_i64(i64 %x) {
+; CHECK-LABEL: isneg_i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    sarq $63, %rax
+; CHECK-NEXT:    andl $2342342, %eax # imm = 0x23BDC6
+; CHECK-NEXT:    retq
+  %cond = icmp slt i64 %x, 0
+  %r = select i1 %cond, i64 2342342, i64 0
+  ret i64 %r
+}
+
+define <16 x i8> @isneg_v16i8(<16 x i8> %x) {
+; CHECK-LABEL: isneg_v16i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pxor %xmm1, %xmm1
+; CHECK-NEXT:    pcmpgtb %xmm0, %xmm1
+; CHECK-NEXT:    pand {{.*}}(%rip), %xmm1
+; CHECK-NEXT:    movdqa %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %cond = icmp slt <16 x i8> %x, zeroinitializer
+  %r = select <16 x i1> %cond, <16 x i8> <i8 12, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42>, <16 x i8> zeroinitializer
+  ret <16 x i8> %r
+}
+
+define <8 x i16> @isneg_v8i16(<8 x i16> %x) {
+; CHECK-LABEL: isneg_v8i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    psraw $15, %xmm0
+; CHECK-NEXT:    pand {{.*}}(%rip), %xmm0
+; CHECK-NEXT:    retq
+  %cond = icmp slt <8 x i16> %x, zeroinitializer
+  %r = select <8 x i1> %cond, <8 x i16> <i16 1, i16 542, i16 542, i16 542, i16 542, i16 542, i16 542, i16 1>, <8 x i16> zeroinitializer
+  ret <8 x i16> %r
+}
+
+define <4 x i32> @isneg_v4i32(<4 x i32> %x) {
+; CHECK-LABEL: isneg_v4i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    psrad $31, %xmm0
+; CHECK-NEXT:    pand {{.*}}(%rip), %xmm0
+; CHECK-NEXT:    retq
+  %cond = icmp slt <4 x i32> %x, zeroinitializer
+  %r = select <4 x i1> %cond, <4 x i32> <i32 0, i32 42, i32 -42, i32 1>, <4 x i32> zeroinitializer
+  ret <4 x i32> %r
+}
+
+define <2 x i64> @isneg_v2i64(<2 x i64> %x) {
+; CHECK-LABEL: isneg_v2i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    psrad $31, %xmm0
+; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; CHECK-NEXT:    pand {{.*}}(%rip), %xmm0
+; CHECK-NEXT:    retq
+  %cond = icmp slt <2 x i64> %x, zeroinitializer
+  %r = select <2 x i1> %cond, <2 x i64> <i64 2342342, i64 12>, <2 x i64> zeroinitializer
+  ret <2 x i64> %r
+}
diff --git a/llvm/test/CodeGen/X86/select-testb-volatile-load.ll b/llvm/test/CodeGen/X86/select-testb-volatile-load.ll
new file mode 100644
index 000000000..69cfe17b4
--- /dev/null
+++ b/llvm/test/CodeGen/X86/select-testb-volatile-load.ll
@@ -0,0 +1,33 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=x86_64-apple-darwin < %s | FileCheck %s
+
+; This test checks that we don't try to narrow the volatile load by selecting
+; the pattern below into a testb instruction.
+
+define void @testb_volatile(i32 **%ptrptr) {
+; CHECK-LABEL: testb_volatile:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    movq (%rdi), %rax
+; CHECK-NEXT:    testl $1, (%rax)
+; CHECK-NEXT:    jne LBB0_1
+; CHECK-NEXT:  ## %bb.2: ## %exit
+; CHECK-NEXT:    movl $1, (%rax)
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  LBB0_1: ## %bb2
+; CHECK-NEXT:    movl $0, (%rax)
+; CHECK-NEXT:    retq
+entry:
+  %ptr = load i32*, i32** %ptrptr, align 8
+  %vol_load = load volatile i32, i32* %ptr, align 4
+  %and = and i32 %vol_load, 1
+  %cmp = icmp eq i32 %and, 0
+  br i1 %cmp, label %exit, label %bb2
+
+bb2:
+  store i32 0, i32 *%ptr, align 4
+  ret void
+
+exit:
+  store i32 1, i32 *%ptr, align 4
+  ret void
+}
diff --git a/llvm/test/CodeGen/X86/select.ll b/llvm/test/CodeGen/X86/select.ll
index d174951c2..7344d38b5 100644
--- a/llvm/test/CodeGen/X86/select.ll
+++ b/llvm/test/CodeGen/X86/select.ll
@@ -215,17 +215,27 @@ entry:
 }
 
 define void @test5(i1 %c, <2 x i16> %a, <2 x i16> %b, <2 x i16>* %p) nounwind {
-; CHECK-LABEL: test5:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    testb $1, %dil
-; CHECK-NEXT:    jne LBB4_2
-; CHECK-NEXT:  ## %bb.1:
-; CHECK-NEXT:    movdqa %xmm1, %xmm0
-; CHECK-NEXT:  LBB4_2:
-; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; CHECK-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
-; CHECK-NEXT:    movd %xmm0, (%rsi)
-; CHECK-NEXT:    retq
+; GENERIC-LABEL: test5:
+; GENERIC:       ## %bb.0:
+; GENERIC-NEXT:    testb $1, %dil
+; GENERIC-NEXT:    jne LBB4_2
+; GENERIC-NEXT:  ## %bb.1:
+; GENERIC-NEXT:    movaps %xmm1, %xmm0
+; GENERIC-NEXT:  LBB4_2:
+; GENERIC-NEXT:    movss %xmm0, (%rsi)
+; GENERIC-NEXT:    retq
+;
+; ATOM-LABEL: test5:
+; ATOM:       ## %bb.0:
+; ATOM-NEXT:    testb $1, %dil
+; ATOM-NEXT:    jne LBB4_2
+; ATOM-NEXT:  ## %bb.1:
+; ATOM-NEXT:    movaps %xmm1, %xmm0
+; ATOM-NEXT:  LBB4_2:
+; ATOM-NEXT:    movss %xmm0, (%rsi)
+; ATOM-NEXT:    nop
+; ATOM-NEXT:    nop
+; ATOM-NEXT:    retq
 ;
 ; ATHLON-LABEL: test5:
 ; ATHLON:       ## %bb.0:
diff --git a/llvm/test/CodeGen/X86/selectcc-to-shiftand.ll b/llvm/test/CodeGen/X86/selectcc-to-shiftand.ll
index b6bbce92c..8e8e1e806 100644
--- a/llvm/test/CodeGen/X86/selectcc-to-shiftand.ll
+++ b/llvm/test/CodeGen/X86/selectcc-to-shiftand.ll
@@ -1,23 +1,16 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=x86_64-unknown-linux-gnu -mattr=-bmi < %s | FileCheck %s --check-prefix=CHECK-NOBMI
-; RUN: llc -mtriple=x86_64-unknown-linux-gnu -mattr=+bmi < %s | FileCheck %s --check-prefix=CHECK-BMI
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu -mattr=-bmi < %s | FileCheck %s --check-prefixes=ANY,CHECK-NOBMI
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu -mattr=+bmi < %s | FileCheck %s --check-prefixes=ANY,CHECK-BMI
 
 ; Compare if negative and select of constants where one constant is zero.
 
 define i32 @neg_sel_constants(i32 %a) {
-; CHECK-NOBMI-LABEL: neg_sel_constants:
-; CHECK-NOBMI:       # %bb.0:
-; CHECK-NOBMI-NEXT:    movl %edi, %eax
-; CHECK-NOBMI-NEXT:    sarl $31, %eax
-; CHECK-NOBMI-NEXT:    andl $5, %eax
-; CHECK-NOBMI-NEXT:    retq
-;
-; CHECK-BMI-LABEL: neg_sel_constants:
-; CHECK-BMI:       # %bb.0:
-; CHECK-BMI-NEXT:    movl %edi, %eax
-; CHECK-BMI-NEXT:    sarl $31, %eax
-; CHECK-BMI-NEXT:    andl $5, %eax
-; CHECK-BMI-NEXT:    retq
+; ANY-LABEL: neg_sel_constants:
+; ANY:       # %bb.0:
+; ANY-NEXT:    movl %edi, %eax
+; ANY-NEXT:    sarl $31, %eax
+; ANY-NEXT:    andl $5, %eax
+; ANY-NEXT:    retq
   %tmp.1 = icmp slt i32 %a, 0
   %retval = select i1 %tmp.1, i32 5, i32 0
   ret i32 %retval
@@ -26,19 +19,12 @@ define i32 @neg_sel_constants(i32 %a) {
 ; Compare if negative and select of constants where one constant is zero and the other is a single bit.
 
 define i32 @neg_sel_special_constant(i32 %a) {
-; CHECK-NOBMI-LABEL: neg_sel_special_constant:
-; CHECK-NOBMI:       # %bb.0:
-; CHECK-NOBMI-NEXT:    movl %edi, %eax
-; CHECK-NOBMI-NEXT:    shrl $22, %eax
-; CHECK-NOBMI-NEXT:    andl $512, %eax # imm = 0x200
-; CHECK-NOBMI-NEXT:    retq
-;
-; CHECK-BMI-LABEL: neg_sel_special_constant:
-; CHECK-BMI:       # %bb.0:
-; CHECK-BMI-NEXT:    movl %edi, %eax
-; CHECK-BMI-NEXT:    shrl $22, %eax
-; CHECK-BMI-NEXT:    andl $512, %eax # imm = 0x200
-; CHECK-BMI-NEXT:    retq
+; ANY-LABEL: neg_sel_special_constant:
+; ANY:       # %bb.0:
+; ANY-NEXT:    movl %edi, %eax
+; ANY-NEXT:    shrl $22, %eax
+; ANY-NEXT:    andl $512, %eax # imm = 0x200
+; ANY-NEXT:    retq
   %tmp.1 = icmp slt i32 %a, 0
   %retval = select i1 %tmp.1, i32 512, i32 0
   ret i32 %retval
@@ -47,19 +33,12 @@ define i32 @neg_sel_special_constant(i32 %a) {
 ; Compare if negative and select variable or zero.
 
 define i32 @neg_sel_variable_and_zero(i32 %a, i32 %b) {
-; CHECK-NOBMI-LABEL: neg_sel_variable_and_zero:
-; CHECK-NOBMI:       # %bb.0:
-; CHECK-NOBMI-NEXT:    movl %edi, %eax
-; CHECK-NOBMI-NEXT:    sarl $31, %eax
-; CHECK-NOBMI-NEXT:    andl %esi, %eax
-; CHECK-NOBMI-NEXT:    retq
-;
-; CHECK-BMI-LABEL: neg_sel_variable_and_zero:
-; CHECK-BMI:       # %bb.0:
-; CHECK-BMI-NEXT:    movl %edi, %eax
-; CHECK-BMI-NEXT:    sarl $31, %eax
-; CHECK-BMI-NEXT:    andl %esi, %eax
-; CHECK-BMI-NEXT:    retq
+; ANY-LABEL: neg_sel_variable_and_zero:
+; ANY:       # %bb.0:
+; ANY-NEXT:    movl %edi, %eax
+; ANY-NEXT:    sarl $31, %eax
+; ANY-NEXT:    andl %esi, %eax
+; ANY-NEXT:    retq
   %tmp.1 = icmp slt i32 %a, 0
   %retval = select i1 %tmp.1, i32 %b, i32 0
   ret i32 %retval
@@ -68,19 +47,12 @@ define i32 @neg_sel_variable_and_zero(i32 %a, i32 %b) {
 ; Compare if not positive and select the same variable as being compared: smin(a, 0).
 
 define i32 @not_pos_sel_same_variable(i32 %a) {
-; CHECK-NOBMI-LABEL: not_pos_sel_same_variable:
-; CHECK-NOBMI:       # %bb.0:
-; CHECK-NOBMI-NEXT:    movl %edi, %eax
-; CHECK-NOBMI-NEXT:    sarl $31, %eax
-; CHECK-NOBMI-NEXT:    andl %edi, %eax
-; CHECK-NOBMI-NEXT:    retq
-;
-; CHECK-BMI-LABEL: not_pos_sel_same_variable:
-; CHECK-BMI:       # %bb.0:
-; CHECK-BMI-NEXT:    movl %edi, %eax
-; CHECK-BMI-NEXT:    sarl $31, %eax
-; CHECK-BMI-NEXT:    andl %edi, %eax
-; CHECK-BMI-NEXT:    retq
+; ANY-LABEL: not_pos_sel_same_variable:
+; ANY:       # %bb.0:
+; ANY-NEXT:    movl %edi, %eax
+; ANY-NEXT:    sarl $31, %eax
+; ANY-NEXT:    andl %edi, %eax
+; ANY-NEXT:    retq
   %tmp = icmp slt i32 %a, 1
   %min = select i1 %tmp, i32 %a, i32 0
   ret i32 %min
@@ -91,21 +63,13 @@ define i32 @not_pos_sel_same_variable(i32 %a) {
 ; Compare if positive and select of constants where one constant is zero.
 
 define i32 @pos_sel_constants(i32 %a) {
-; CHECK-NOBMI-LABEL: pos_sel_constants:
-; CHECK-NOBMI:       # %bb.0:
-; CHECK-NOBMI-NEXT:    # kill: def $edi killed $edi def $rdi
-; CHECK-NOBMI-NEXT:    notl %edi
-; CHECK-NOBMI-NEXT:    shrl $31, %edi
-; CHECK-NOBMI-NEXT:    leal (%rdi,%rdi,4), %eax
-; CHECK-NOBMI-NEXT:    retq
-;
-; CHECK-BMI-LABEL: pos_sel_constants:
-; CHECK-BMI:       # %bb.0:
-; CHECK-BMI-NEXT:    # kill: def $edi killed $edi def $rdi
-; CHECK-BMI-NEXT:    notl %edi
-; CHECK-BMI-NEXT:    shrl $31, %edi
-; CHECK-BMI-NEXT:    leal (%rdi,%rdi,4), %eax
-; CHECK-BMI-NEXT:    retq
+; ANY-LABEL: pos_sel_constants:
+; ANY:       # %bb.0:
+; ANY-NEXT:    # kill: def $edi killed $edi def $rdi
+; ANY-NEXT:    notl %edi
+; ANY-NEXT:    shrl $31, %edi
+; ANY-NEXT:    leal (%rdi,%rdi,4), %eax
+; ANY-NEXT:    retq
   %tmp.1 = icmp sgt i32 %a, -1
   %retval = select i1 %tmp.1, i32 5, i32 0
   ret i32 %retval
@@ -114,21 +78,13 @@ define i32 @pos_sel_constants(i32 %a) {
 ; Compare if positive and select of constants where one constant is zero and the other is a single bit.
 
 define i32 @pos_sel_special_constant(i32 %a) {
-; CHECK-NOBMI-LABEL: pos_sel_special_constant:
-; CHECK-NOBMI:       # %bb.0:
-; CHECK-NOBMI-NEXT:    movl %edi, %eax
-; CHECK-NOBMI-NEXT:    notl %eax
-; CHECK-NOBMI-NEXT:    shrl $22, %eax
-; CHECK-NOBMI-NEXT:    andl $512, %eax # imm = 0x200
-; CHECK-NOBMI-NEXT:    retq
-;
-; CHECK-BMI-LABEL: pos_sel_special_constant:
-; CHECK-BMI:       # %bb.0:
-; CHECK-BMI-NEXT:    movl %edi, %eax
-; CHECK-BMI-NEXT:    notl %eax
-; CHECK-BMI-NEXT:    shrl $22, %eax
-; CHECK-BMI-NEXT:    andl $512, %eax # imm = 0x200
-; CHECK-BMI-NEXT:    retq
+; ANY-LABEL: pos_sel_special_constant:
+; ANY:       # %bb.0:
+; ANY-NEXT:    movl %edi, %eax
+; ANY-NEXT:    notl %eax
+; ANY-NEXT:    shrl $22, %eax
+; ANY-NEXT:    andl $512, %eax # imm = 0x200
+; ANY-NEXT:    retq
   %tmp.1 = icmp sgt i32 %a, -1
   %retval = select i1 %tmp.1, i32 512, i32 0
   ret i32 %retval
@@ -198,3 +154,88 @@ define i32 @PR31175(i32 %x, i32 %y) {
   %sel = select i1 %cmp, i32 %sub, i32 0
   ret i32 %sel
 }
+
+define i8 @sel_shift_bool_i8(i1 %t) {
+; ANY-LABEL: sel_shift_bool_i8:
+; ANY:       # %bb.0:
+; ANY-NEXT:    movl %edi, %eax
+; ANY-NEXT:    shlb $7, %al
+; ANY-NEXT:    # kill: def $al killed $al killed $eax
+; ANY-NEXT:    retq
+  %shl = select i1 %t, i8 128, i8 0
+  ret i8 %shl
+}
+
+define i16 @sel_shift_bool_i16(i1 %t) {
+; ANY-LABEL: sel_shift_bool_i16:
+; ANY:       # %bb.0:
+; ANY-NEXT:    movl %edi, %eax
+; ANY-NEXT:    andl $1, %eax
+; ANY-NEXT:    shll $7, %eax
+; ANY-NEXT:    # kill: def $ax killed $ax killed $eax
+; ANY-NEXT:    retq
+  %shl = select i1 %t, i16 128, i16 0
+  ret i16 %shl
+}
+
+define i32 @sel_shift_bool_i32(i1 %t) {
+; ANY-LABEL: sel_shift_bool_i32:
+; ANY:       # %bb.0:
+; ANY-NEXT:    movl %edi, %eax
+; ANY-NEXT:    andl $1, %eax
+; ANY-NEXT:    shll $6, %eax
+; ANY-NEXT:    retq
+  %shl = select i1 %t, i32 64, i32 0
+  ret i32 %shl
+}
+
+define i64 @sel_shift_bool_i64(i1 %t) {
+; ANY-LABEL: sel_shift_bool_i64:
+; ANY:       # %bb.0:
+; ANY-NEXT:    movl %edi, %eax
+; ANY-NEXT:    andl $1, %eax
+; ANY-NEXT:    shlq $16, %rax
+; ANY-NEXT:    retq
+  %shl = select i1 %t, i64 65536, i64 0
+  ret i64 %shl
+}
+
+define <16 x i8> @sel_shift_bool_v16i8(<16 x i1> %t) {
+; ANY-LABEL: sel_shift_bool_v16i8:
+; ANY:       # %bb.0:
+; ANY-NEXT:    psllw $7, %xmm0
+; ANY-NEXT:    pand {{.*}}(%rip), %xmm0
+; ANY-NEXT:    retq
+  %shl = select <16 x i1> %t, <16 x i8> <i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128>, <16 x i8> zeroinitializer
+  ret <16 x i8> %shl
+}
+
+define <8 x i16> @sel_shift_bool_v8i16(<8 x i1> %t) {
+; ANY-LABEL: sel_shift_bool_v8i16:
+; ANY:       # %bb.0:
+; ANY-NEXT:    pand {{.*}}(%rip), %xmm0
+; ANY-NEXT:    psllw $7, %xmm0
+; ANY-NEXT:    retq
+  %shl= select <8 x i1> %t, <8 x i16> <i16 128, i16 128, i16 128, i16 128, i16 128, i16 128, i16 128, i16 128>, <8 x i16> zeroinitializer
+  ret <8 x i16> %shl
+}
+
+define <4 x i32> @sel_shift_bool_v4i32(<4 x i1> %t) {
+; ANY-LABEL: sel_shift_bool_v4i32:
+; ANY:       # %bb.0:
+; ANY-NEXT:    pand {{.*}}(%rip), %xmm0
+; ANY-NEXT:    pslld $6, %xmm0
+; ANY-NEXT:    retq
+  %shl = select <4 x i1> %t, <4 x i32> <i32 64, i32 64, i32 64, i32 64>, <4 x i32> zeroinitializer
+  ret <4 x i32> %shl
+}
+
+define <2 x i64> @sel_shift_bool_v2i64(<2 x i1> %t) {
+; ANY-LABEL: sel_shift_bool_v2i64:
+; ANY:       # %bb.0:
+; ANY-NEXT:    pand {{.*}}(%rip), %xmm0
+; ANY-NEXT:    psllq $16, %xmm0
+; ANY-NEXT:    retq
+  %shl = select <2 x i1> %t, <2 x i64> <i64 65536, i64 65536>, <2 x i64> zeroinitializer
+  ret <2 x i64> %shl
+}
diff --git a/llvm/test/CodeGen/X86/setcc-wide-types.ll b/llvm/test/CodeGen/X86/setcc-wide-types.ll
index d8176e488..74a6ae58f 100644
--- a/llvm/test/CodeGen/X86/setcc-wide-types.ll
+++ b/llvm/test/CodeGen/X86/setcc-wide-types.ll
@@ -1,9 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=sse2     | FileCheck %s --check-prefix=ANY --check-prefix=NO512 --check-prefix=SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=sse4.1   | FileCheck %s --check-prefix=ANY --check-prefix=NO512 --check-prefix=SSE41
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx      | FileCheck %s --check-prefix=ANY --check-prefix=NO512 --check-prefix=AVXANY --check-prefix=AVX1
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx2     | FileCheck %s --check-prefix=ANY --check-prefix=NO512 --check-prefix=AVXANY --check-prefix=AVX256 --check-prefix=AVX2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512f  | FileCheck %s --check-prefix=ANY --check-prefix=AVXANY --check-prefix=AVX256 --check-prefix=AVX512 --check-prefix=AVX512F
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512bw | FileCheck %s --check-prefix=ANY --check-prefix=AVXANY --check-prefix=AVX256 --check-prefix=AVX512 --check-prefix=AVX512BW
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx2     | FileCheck %s --check-prefix=ANY --check-prefix=NO512 --check-prefix=AVXANY --check-prefix=AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512f  | FileCheck %s --check-prefix=ANY --check-prefix=AVXANY --check-prefix=AVX512 --check-prefix=AVX512F
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512bw | FileCheck %s --check-prefix=ANY --check-prefix=AVXANY --check-prefix=AVX512 --check-prefix=AVX512BW
 
 ; Equality checks of 128/256-bit values can use PMOVMSK or PTEST to avoid scalarization.
 
@@ -17,12 +18,19 @@ define i32 @ne_i128(<2 x i64> %x, <2 x i64> %y) {
 ; SSE2-NEXT:    setne %al
 ; SSE2-NEXT:    retq
 ;
+; SSE41-LABEL: ne_i128:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    xorl %eax, %eax
+; SSE41-NEXT:    ptest %xmm0, %xmm0
+; SSE41-NEXT:    setne %al
+; SSE41-NEXT:    retq
+;
 ; AVXANY-LABEL: ne_i128:
 ; AVXANY:       # %bb.0:
-; AVXANY-NEXT:    vpcmpeqb %xmm1, %xmm0, %xmm0
-; AVXANY-NEXT:    vpmovmskb %xmm0, %ecx
+; AVXANY-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVXANY-NEXT:    xorl %eax, %eax
-; AVXANY-NEXT:    cmpl $65535, %ecx # imm = 0xFFFF
+; AVXANY-NEXT:    vptest %xmm0, %xmm0
 ; AVXANY-NEXT:    setne %al
 ; AVXANY-NEXT:    retq
   %bcx = bitcast <2 x i64> %x to i128
@@ -42,12 +50,19 @@ define i32 @eq_i128(<2 x i64> %x, <2 x i64> %y) {
 ; SSE2-NEXT:    sete %al
 ; SSE2-NEXT:    retq
 ;
+; SSE41-LABEL: eq_i128:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    xorl %eax, %eax
+; SSE41-NEXT:    ptest %xmm0, %xmm0
+; SSE41-NEXT:    sete %al
+; SSE41-NEXT:    retq
+;
 ; AVXANY-LABEL: eq_i128:
 ; AVXANY:       # %bb.0:
-; AVXANY-NEXT:    vpcmpeqb %xmm1, %xmm0, %xmm0
-; AVXANY-NEXT:    vpmovmskb %xmm0, %ecx
+; AVXANY-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVXANY-NEXT:    xorl %eax, %eax
-; AVXANY-NEXT:    cmpl $65535, %ecx # imm = 0xFFFF
+; AVXANY-NEXT:    vptest %xmm0, %xmm0
 ; AVXANY-NEXT:    sete %al
 ; AVXANY-NEXT:    retq
   %bcx = bitcast <2 x i64> %x to i128
@@ -83,39 +98,53 @@ define i32 @ne_i256(<4 x i64> %x, <4 x i64> %y) {
 ; SSE2-NEXT:    setne %al
 ; SSE2-NEXT:    retq
 ;
+; SSE41-LABEL: ne_i256:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movq %xmm0, %rax
+; SSE41-NEXT:    movq %xmm1, %rcx
+; SSE41-NEXT:    pextrq $1, %xmm0, %rdx
+; SSE41-NEXT:    pextrq $1, %xmm1, %r8
+; SSE41-NEXT:    movq %xmm2, %rdi
+; SSE41-NEXT:    xorq %rax, %rdi
+; SSE41-NEXT:    movq %xmm3, %rsi
+; SSE41-NEXT:    xorq %rcx, %rsi
+; SSE41-NEXT:    orq %rdi, %rsi
+; SSE41-NEXT:    pextrq $1, %xmm2, %rax
+; SSE41-NEXT:    xorq %rdx, %rax
+; SSE41-NEXT:    pextrq $1, %xmm3, %rcx
+; SSE41-NEXT:    xorq %r8, %rcx
+; SSE41-NEXT:    orq %rax, %rcx
+; SSE41-NEXT:    xorl %eax, %eax
+; SSE41-NEXT:    orq %rsi, %rcx
+; SSE41-NEXT:    setne %al
+; SSE41-NEXT:    retq
+;
 ; AVX1-LABEL: ne_i256:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovq %xmm0, %rax
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX1-NEXT:    vmovq %xmm2, %rcx
-; AVX1-NEXT:    vpextrq $1, %xmm0, %rdx
-; AVX1-NEXT:    vpextrq $1, %xmm2, %r8
-; AVX1-NEXT:    vmovq %xmm1, %rdi
-; AVX1-NEXT:    xorq %rax, %rdi
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm0
-; AVX1-NEXT:    vmovq %xmm0, %rsi
-; AVX1-NEXT:    xorq %rcx, %rsi
-; AVX1-NEXT:    orq %rdi, %rsi
-; AVX1-NEXT:    vpextrq $1, %xmm1, %rax
-; AVX1-NEXT:    xorq %rdx, %rax
-; AVX1-NEXT:    vpextrq $1, %xmm0, %rcx
-; AVX1-NEXT:    xorq %r8, %rcx
-; AVX1-NEXT:    orq %rax, %rcx
+; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    xorl %eax, %eax
-; AVX1-NEXT:    orq %rsi, %rcx
+; AVX1-NEXT:    vptest %ymm0, %ymm0
 ; AVX1-NEXT:    setne %al
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
-; AVX256-LABEL: ne_i256:
-; AVX256:       # %bb.0:
-; AVX256-NEXT:    vpcmpeqb %ymm1, %ymm0, %ymm0
-; AVX256-NEXT:    vpmovmskb %ymm0, %ecx
-; AVX256-NEXT:    xorl %eax, %eax
-; AVX256-NEXT:    cmpl $-1, %ecx
-; AVX256-NEXT:    setne %al
-; AVX256-NEXT:    vzeroupper
-; AVX256-NEXT:    retq
+; AVX2-LABEL: ne_i256:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    xorl %eax, %eax
+; AVX2-NEXT:    vptest %ymm0, %ymm0
+; AVX2-NEXT:    setne %al
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: ne_i256:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    xorl %eax, %eax
+; AVX512-NEXT:    vptest %ymm0, %ymm0
+; AVX512-NEXT:    setne %al
+; AVX512-NEXT:    vzeroupper
+; AVX512-NEXT:    retq
   %bcx = bitcast <4 x i64> %x to i256
   %bcy = bitcast <4 x i64> %y to i256
   %cmp = icmp ne i256 %bcx, %bcy
@@ -149,39 +178,53 @@ define i32 @eq_i256(<4 x i64> %x, <4 x i64> %y) {
 ; SSE2-NEXT:    sete %al
 ; SSE2-NEXT:    retq
 ;
+; SSE41-LABEL: eq_i256:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movq %xmm0, %rax
+; SSE41-NEXT:    movq %xmm1, %rcx
+; SSE41-NEXT:    pextrq $1, %xmm0, %rdx
+; SSE41-NEXT:    pextrq $1, %xmm1, %r8
+; SSE41-NEXT:    movq %xmm2, %rdi
+; SSE41-NEXT:    xorq %rax, %rdi
+; SSE41-NEXT:    movq %xmm3, %rsi
+; SSE41-NEXT:    xorq %rcx, %rsi
+; SSE41-NEXT:    orq %rdi, %rsi
+; SSE41-NEXT:    pextrq $1, %xmm2, %rax
+; SSE41-NEXT:    xorq %rdx, %rax
+; SSE41-NEXT:    pextrq $1, %xmm3, %rcx
+; SSE41-NEXT:    xorq %r8, %rcx
+; SSE41-NEXT:    orq %rax, %rcx
+; SSE41-NEXT:    xorl %eax, %eax
+; SSE41-NEXT:    orq %rsi, %rcx
+; SSE41-NEXT:    sete %al
+; SSE41-NEXT:    retq
+;
 ; AVX1-LABEL: eq_i256:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovq %xmm0, %rax
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX1-NEXT:    vmovq %xmm2, %rcx
-; AVX1-NEXT:    vpextrq $1, %xmm0, %rdx
-; AVX1-NEXT:    vpextrq $1, %xmm2, %r8
-; AVX1-NEXT:    vmovq %xmm1, %rdi
-; AVX1-NEXT:    xorq %rax, %rdi
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm0
-; AVX1-NEXT:    vmovq %xmm0, %rsi
-; AVX1-NEXT:    xorq %rcx, %rsi
-; AVX1-NEXT:    orq %rdi, %rsi
-; AVX1-NEXT:    vpextrq $1, %xmm1, %rax
-; AVX1-NEXT:    xorq %rdx, %rax
-; AVX1-NEXT:    vpextrq $1, %xmm0, %rcx
-; AVX1-NEXT:    xorq %r8, %rcx
-; AVX1-NEXT:    orq %rax, %rcx
+; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    xorl %eax, %eax
-; AVX1-NEXT:    orq %rsi, %rcx
+; AVX1-NEXT:    vptest %ymm0, %ymm0
 ; AVX1-NEXT:    sete %al
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
-; AVX256-LABEL: eq_i256:
-; AVX256:       # %bb.0:
-; AVX256-NEXT:    vpcmpeqb %ymm1, %ymm0, %ymm0
-; AVX256-NEXT:    vpmovmskb %ymm0, %ecx
-; AVX256-NEXT:    xorl %eax, %eax
-; AVX256-NEXT:    cmpl $-1, %ecx
-; AVX256-NEXT:    sete %al
-; AVX256-NEXT:    vzeroupper
-; AVX256-NEXT:    retq
+; AVX2-LABEL: eq_i256:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    xorl %eax, %eax
+; AVX2-NEXT:    vptest %ymm0, %ymm0
+; AVX2-NEXT:    sete %al
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: eq_i256:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    xorl %eax, %eax
+; AVX512-NEXT:    vptest %ymm0, %ymm0
+; AVX512-NEXT:    sete %al
+; AVX512-NEXT:    vzeroupper
+; AVX512-NEXT:    retq
   %bcx = bitcast <4 x i64> %x to i256
   %bcy = bitcast <4 x i64> %y to i256
   %cmp = icmp eq i256 %bcx, %bcy
@@ -235,6 +278,43 @@ define i32 @ne_i512(<8 x i64> %x, <8 x i64> %y) {
 ; SSE2-NEXT:    setne %al
 ; SSE2-NEXT:    retq
 ;
+; SSE41-LABEL: ne_i512:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movq %xmm0, %rax
+; SSE41-NEXT:    movq %xmm2, %rcx
+; SSE41-NEXT:    movq %xmm1, %rdx
+; SSE41-NEXT:    movq %xmm3, %rsi
+; SSE41-NEXT:    pextrq $1, %xmm0, %r11
+; SSE41-NEXT:    pextrq $1, %xmm2, %r8
+; SSE41-NEXT:    pextrq $1, %xmm1, %r9
+; SSE41-NEXT:    pextrq $1, %xmm3, %r10
+; SSE41-NEXT:    movq %xmm4, %rdi
+; SSE41-NEXT:    xorq %rax, %rdi
+; SSE41-NEXT:    movq %xmm6, %rax
+; SSE41-NEXT:    xorq %rcx, %rax
+; SSE41-NEXT:    movq %xmm5, %rcx
+; SSE41-NEXT:    xorq %rdx, %rcx
+; SSE41-NEXT:    movq %xmm7, %rdx
+; SSE41-NEXT:    xorq %rsi, %rdx
+; SSE41-NEXT:    orq %rcx, %rdx
+; SSE41-NEXT:    orq %rax, %rdx
+; SSE41-NEXT:    orq %rdi, %rdx
+; SSE41-NEXT:    pextrq $1, %xmm4, %rax
+; SSE41-NEXT:    xorq %r11, %rax
+; SSE41-NEXT:    pextrq $1, %xmm6, %rcx
+; SSE41-NEXT:    xorq %r8, %rcx
+; SSE41-NEXT:    pextrq $1, %xmm5, %rsi
+; SSE41-NEXT:    xorq %r9, %rsi
+; SSE41-NEXT:    pextrq $1, %xmm7, %rdi
+; SSE41-NEXT:    xorq %r10, %rdi
+; SSE41-NEXT:    orq %rsi, %rdi
+; SSE41-NEXT:    orq %rcx, %rdi
+; SSE41-NEXT:    orq %rax, %rdi
+; SSE41-NEXT:    xorl %eax, %eax
+; SSE41-NEXT:    orq %rdx, %rdi
+; SSE41-NEXT:    setne %al
+; SSE41-NEXT:    retq
+;
 ; AVX1-LABEL: ne_i512:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vmovq %xmm0, %rax
@@ -319,14 +399,23 @@ define i32 @ne_i512(<8 x i64> %x, <8 x i64> %y) {
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
-; AVX512-LABEL: ne_i512:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpcmpeqd %zmm1, %zmm0, %k0
-; AVX512-NEXT:    xorl %eax, %eax
-; AVX512-NEXT:    kortestw %k0, %k0
-; AVX512-NEXT:    setae %al
-; AVX512-NEXT:    vzeroupper
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: ne_i512:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpcmpneqd %zmm1, %zmm0, %k0
+; AVX512F-NEXT:    xorl %eax, %eax
+; AVX512F-NEXT:    kortestw %k0, %k0
+; AVX512F-NEXT:    setne %al
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: ne_i512:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpcmpneqb %zmm1, %zmm0, %k0
+; AVX512BW-NEXT:    xorl %eax, %eax
+; AVX512BW-NEXT:    kortestq %k0, %k0
+; AVX512BW-NEXT:    setne %al
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
   %bcx = bitcast <8 x i64> %x to i512
   %bcy = bitcast <8 x i64> %y to i512
   %cmp = icmp ne i512 %bcx, %bcy
@@ -380,6 +469,43 @@ define i32 @eq_i512(<8 x i64> %x, <8 x i64> %y) {
 ; SSE2-NEXT:    sete %al
 ; SSE2-NEXT:    retq
 ;
+; SSE41-LABEL: eq_i512:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movq %xmm0, %rax
+; SSE41-NEXT:    movq %xmm2, %rcx
+; SSE41-NEXT:    movq %xmm1, %rdx
+; SSE41-NEXT:    movq %xmm3, %rsi
+; SSE41-NEXT:    pextrq $1, %xmm0, %r11
+; SSE41-NEXT:    pextrq $1, %xmm2, %r8
+; SSE41-NEXT:    pextrq $1, %xmm1, %r9
+; SSE41-NEXT:    pextrq $1, %xmm3, %r10
+; SSE41-NEXT:    movq %xmm4, %rdi
+; SSE41-NEXT:    xorq %rax, %rdi
+; SSE41-NEXT:    movq %xmm6, %rax
+; SSE41-NEXT:    xorq %rcx, %rax
+; SSE41-NEXT:    movq %xmm5, %rcx
+; SSE41-NEXT:    xorq %rdx, %rcx
+; SSE41-NEXT:    movq %xmm7, %rdx
+; SSE41-NEXT:    xorq %rsi, %rdx
+; SSE41-NEXT:    orq %rcx, %rdx
+; SSE41-NEXT:    orq %rax, %rdx
+; SSE41-NEXT:    orq %rdi, %rdx
+; SSE41-NEXT:    pextrq $1, %xmm4, %rax
+; SSE41-NEXT:    xorq %r11, %rax
+; SSE41-NEXT:    pextrq $1, %xmm6, %rcx
+; SSE41-NEXT:    xorq %r8, %rcx
+; SSE41-NEXT:    pextrq $1, %xmm5, %rsi
+; SSE41-NEXT:    xorq %r9, %rsi
+; SSE41-NEXT:    pextrq $1, %xmm7, %rdi
+; SSE41-NEXT:    xorq %r10, %rdi
+; SSE41-NEXT:    orq %rsi, %rdi
+; SSE41-NEXT:    orq %rcx, %rdi
+; SSE41-NEXT:    orq %rax, %rdi
+; SSE41-NEXT:    xorl %eax, %eax
+; SSE41-NEXT:    orq %rdx, %rdi
+; SSE41-NEXT:    sete %al
+; SSE41-NEXT:    retq
+;
 ; AVX1-LABEL: eq_i512:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vmovq %xmm0, %rax
@@ -464,14 +590,23 @@ define i32 @eq_i512(<8 x i64> %x, <8 x i64> %y) {
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
-; AVX512-LABEL: eq_i512:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpcmpeqd %zmm1, %zmm0, %k0
-; AVX512-NEXT:    xorl %eax, %eax
-; AVX512-NEXT:    kortestw %k0, %k0
-; AVX512-NEXT:    setb %al
-; AVX512-NEXT:    vzeroupper
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: eq_i512:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpcmpneqd %zmm1, %zmm0, %k0
+; AVX512F-NEXT:    xorl %eax, %eax
+; AVX512F-NEXT:    kortestw %k0, %k0
+; AVX512F-NEXT:    sete %al
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: eq_i512:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpcmpneqb %zmm1, %zmm0, %k0
+; AVX512BW-NEXT:    xorl %eax, %eax
+; AVX512BW-NEXT:    kortestq %k0, %k0
+; AVX512BW-NEXT:    sete %al
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
   %bcx = bitcast <8 x i64> %x to i512
   %bcy = bitcast <8 x i64> %y to i512
   %cmp = icmp eq i512 %bcx, %bcy
@@ -498,16 +633,29 @@ define i32 @ne_i128_pair(i128* %a, i128* %b) {
 ; SSE2-NEXT:    setne %al
 ; SSE2-NEXT:    retq
 ;
+; SSE41-LABEL: ne_i128_pair:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqu (%rdi), %xmm0
+; SSE41-NEXT:    movdqu 16(%rdi), %xmm1
+; SSE41-NEXT:    movdqu (%rsi), %xmm2
+; SSE41-NEXT:    pxor %xmm0, %xmm2
+; SSE41-NEXT:    movdqu 16(%rsi), %xmm0
+; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    xorl %eax, %eax
+; SSE41-NEXT:    ptest %xmm0, %xmm0
+; SSE41-NEXT:    setne %al
+; SSE41-NEXT:    retq
+;
 ; AVXANY-LABEL: ne_i128_pair:
 ; AVXANY:       # %bb.0:
 ; AVXANY-NEXT:    vmovdqu (%rdi), %xmm0
 ; AVXANY-NEXT:    vmovdqu 16(%rdi), %xmm1
-; AVXANY-NEXT:    vpcmpeqb 16(%rsi), %xmm1, %xmm1
-; AVXANY-NEXT:    vpcmpeqb (%rsi), %xmm0, %xmm0
-; AVXANY-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVXANY-NEXT:    vpmovmskb %xmm0, %ecx
+; AVXANY-NEXT:    vpxor 16(%rsi), %xmm1, %xmm1
+; AVXANY-NEXT:    vpxor (%rsi), %xmm0, %xmm0
+; AVXANY-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVXANY-NEXT:    xorl %eax, %eax
-; AVXANY-NEXT:    cmpl $65535, %ecx # imm = 0xFFFF
+; AVXANY-NEXT:    vptest %xmm0, %xmm0
 ; AVXANY-NEXT:    setne %al
 ; AVXANY-NEXT:    retq
   %a0 = load i128, i128* %a
@@ -543,16 +691,29 @@ define i32 @eq_i128_pair(i128* %a, i128* %b) {
 ; SSE2-NEXT:    sete %al
 ; SSE2-NEXT:    retq
 ;
+; SSE41-LABEL: eq_i128_pair:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqu (%rdi), %xmm0
+; SSE41-NEXT:    movdqu 16(%rdi), %xmm1
+; SSE41-NEXT:    movdqu (%rsi), %xmm2
+; SSE41-NEXT:    pxor %xmm0, %xmm2
+; SSE41-NEXT:    movdqu 16(%rsi), %xmm0
+; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    xorl %eax, %eax
+; SSE41-NEXT:    ptest %xmm0, %xmm0
+; SSE41-NEXT:    sete %al
+; SSE41-NEXT:    retq
+;
 ; AVXANY-LABEL: eq_i128_pair:
 ; AVXANY:       # %bb.0:
 ; AVXANY-NEXT:    vmovdqu (%rdi), %xmm0
 ; AVXANY-NEXT:    vmovdqu 16(%rdi), %xmm1
-; AVXANY-NEXT:    vpcmpeqb 16(%rsi), %xmm1, %xmm1
-; AVXANY-NEXT:    vpcmpeqb (%rsi), %xmm0, %xmm0
-; AVXANY-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVXANY-NEXT:    vpmovmskb %xmm0, %ecx
+; AVXANY-NEXT:    vpxor 16(%rsi), %xmm1, %xmm1
+; AVXANY-NEXT:    vpxor (%rsi), %xmm0, %xmm0
+; AVXANY-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVXANY-NEXT:    xorl %eax, %eax
-; AVXANY-NEXT:    cmpl $65535, %ecx # imm = 0xFFFF
+; AVXANY-NEXT:    vptest %xmm0, %xmm0
 ; AVXANY-NEXT:    sete %al
 ; AVXANY-NEXT:    retq
   %a0 = load i128, i128* %a
@@ -602,48 +763,73 @@ define i32 @ne_i256_pair(i256* %a, i256* %b) {
 ; SSE2-NEXT:    setne %al
 ; SSE2-NEXT:    retq
 ;
+; SSE41-LABEL: ne_i256_pair:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movq 16(%rdi), %r9
+; SSE41-NEXT:    movq 24(%rdi), %r11
+; SSE41-NEXT:    movq (%rdi), %r8
+; SSE41-NEXT:    movq 8(%rdi), %r10
+; SSE41-NEXT:    xorq 8(%rsi), %r10
+; SSE41-NEXT:    xorq 24(%rsi), %r11
+; SSE41-NEXT:    xorq (%rsi), %r8
+; SSE41-NEXT:    xorq 16(%rsi), %r9
+; SSE41-NEXT:    movq 48(%rdi), %rdx
+; SSE41-NEXT:    movq 32(%rdi), %rax
+; SSE41-NEXT:    movq 56(%rdi), %rcx
+; SSE41-NEXT:    movq 40(%rdi), %rdi
+; SSE41-NEXT:    xorq 40(%rsi), %rdi
+; SSE41-NEXT:    xorq 56(%rsi), %rcx
+; SSE41-NEXT:    orq %r11, %rcx
+; SSE41-NEXT:    orq %rdi, %rcx
+; SSE41-NEXT:    orq %r10, %rcx
+; SSE41-NEXT:    xorq 32(%rsi), %rax
+; SSE41-NEXT:    xorq 48(%rsi), %rdx
+; SSE41-NEXT:    orq %r9, %rdx
+; SSE41-NEXT:    orq %rax, %rdx
+; SSE41-NEXT:    orq %r8, %rdx
+; SSE41-NEXT:    xorl %eax, %eax
+; SSE41-NEXT:    orq %rcx, %rdx
+; SSE41-NEXT:    setne %al
+; SSE41-NEXT:    retq
+;
 ; AVX1-LABEL: ne_i256_pair:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    movq 16(%rdi), %r9
-; AVX1-NEXT:    movq 24(%rdi), %r11
-; AVX1-NEXT:    movq (%rdi), %r8
-; AVX1-NEXT:    movq 8(%rdi), %r10
-; AVX1-NEXT:    xorq 8(%rsi), %r10
-; AVX1-NEXT:    xorq 24(%rsi), %r11
-; AVX1-NEXT:    xorq (%rsi), %r8
-; AVX1-NEXT:    xorq 16(%rsi), %r9
-; AVX1-NEXT:    movq 48(%rdi), %rdx
-; AVX1-NEXT:    movq 32(%rdi), %rax
-; AVX1-NEXT:    movq 56(%rdi), %rcx
-; AVX1-NEXT:    movq 40(%rdi), %rdi
-; AVX1-NEXT:    xorq 40(%rsi), %rdi
-; AVX1-NEXT:    xorq 56(%rsi), %rcx
-; AVX1-NEXT:    orq %r11, %rcx
-; AVX1-NEXT:    orq %rdi, %rcx
-; AVX1-NEXT:    orq %r10, %rcx
-; AVX1-NEXT:    xorq 32(%rsi), %rax
-; AVX1-NEXT:    xorq 48(%rsi), %rdx
-; AVX1-NEXT:    orq %r9, %rdx
-; AVX1-NEXT:    orq %rax, %rdx
-; AVX1-NEXT:    orq %r8, %rdx
+; AVX1-NEXT:    vmovups (%rdi), %ymm0
+; AVX1-NEXT:    vmovups 32(%rdi), %ymm1
+; AVX1-NEXT:    vxorps 32(%rsi), %ymm1, %ymm1
+; AVX1-NEXT:    vxorps (%rsi), %ymm0, %ymm0
+; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    xorl %eax, %eax
-; AVX1-NEXT:    orq %rcx, %rdx
+; AVX1-NEXT:    vptest %ymm0, %ymm0
 ; AVX1-NEXT:    setne %al
+; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
-; AVX256-LABEL: ne_i256_pair:
-; AVX256:       # %bb.0:
-; AVX256-NEXT:    vmovdqu (%rdi), %ymm0
-; AVX256-NEXT:    vmovdqu 32(%rdi), %ymm1
-; AVX256-NEXT:    vpcmpeqb 32(%rsi), %ymm1, %ymm1
-; AVX256-NEXT:    vpcmpeqb (%rsi), %ymm0, %ymm0
-; AVX256-NEXT:    vpand %ymm1, %ymm0, %ymm0
-; AVX256-NEXT:    vpmovmskb %ymm0, %ecx
-; AVX256-NEXT:    xorl %eax, %eax
-; AVX256-NEXT:    cmpl $-1, %ecx
-; AVX256-NEXT:    setne %al
-; AVX256-NEXT:    vzeroupper
-; AVX256-NEXT:    retq
+; AVX2-LABEL: ne_i256_pair:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vmovdqu (%rdi), %ymm0
+; AVX2-NEXT:    vmovdqu 32(%rdi), %ymm1
+; AVX2-NEXT:    vpxor 32(%rsi), %ymm1, %ymm1
+; AVX2-NEXT:    vpxor (%rsi), %ymm0, %ymm0
+; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    xorl %eax, %eax
+; AVX2-NEXT:    vptest %ymm0, %ymm0
+; AVX2-NEXT:    setne %al
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: ne_i256_pair:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovdqu (%rdi), %ymm0
+; AVX512-NEXT:    vmovdqu 32(%rdi), %ymm1
+; AVX512-NEXT:    vpxor 32(%rsi), %ymm1, %ymm1
+; AVX512-NEXT:    vpxor (%rsi), %ymm0, %ymm0
+; AVX512-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    xorl %eax, %eax
+; AVX512-NEXT:    vptest %ymm0, %ymm0
+; AVX512-NEXT:    setne %al
+; AVX512-NEXT:    vzeroupper
+; AVX512-NEXT:    retq
   %a0 = load i256, i256* %a
   %b0 = load i256, i256* %b
   %xor1 = xor i256 %a0, %b0
@@ -691,48 +877,73 @@ define i32 @eq_i256_pair(i256* %a, i256* %b) {
 ; SSE2-NEXT:    sete %al
 ; SSE2-NEXT:    retq
 ;
+; SSE41-LABEL: eq_i256_pair:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movq 16(%rdi), %r9
+; SSE41-NEXT:    movq 24(%rdi), %r11
+; SSE41-NEXT:    movq (%rdi), %r8
+; SSE41-NEXT:    movq 8(%rdi), %r10
+; SSE41-NEXT:    xorq 8(%rsi), %r10
+; SSE41-NEXT:    xorq 24(%rsi), %r11
+; SSE41-NEXT:    xorq (%rsi), %r8
+; SSE41-NEXT:    xorq 16(%rsi), %r9
+; SSE41-NEXT:    movq 48(%rdi), %rdx
+; SSE41-NEXT:    movq 32(%rdi), %rax
+; SSE41-NEXT:    movq 56(%rdi), %rcx
+; SSE41-NEXT:    movq 40(%rdi), %rdi
+; SSE41-NEXT:    xorq 40(%rsi), %rdi
+; SSE41-NEXT:    xorq 56(%rsi), %rcx
+; SSE41-NEXT:    orq %r11, %rcx
+; SSE41-NEXT:    orq %rdi, %rcx
+; SSE41-NEXT:    orq %r10, %rcx
+; SSE41-NEXT:    xorq 32(%rsi), %rax
+; SSE41-NEXT:    xorq 48(%rsi), %rdx
+; SSE41-NEXT:    orq %r9, %rdx
+; SSE41-NEXT:    orq %rax, %rdx
+; SSE41-NEXT:    orq %r8, %rdx
+; SSE41-NEXT:    xorl %eax, %eax
+; SSE41-NEXT:    orq %rcx, %rdx
+; SSE41-NEXT:    sete %al
+; SSE41-NEXT:    retq
+;
 ; AVX1-LABEL: eq_i256_pair:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    movq 16(%rdi), %r9
-; AVX1-NEXT:    movq 24(%rdi), %r11
-; AVX1-NEXT:    movq (%rdi), %r8
-; AVX1-NEXT:    movq 8(%rdi), %r10
-; AVX1-NEXT:    xorq 8(%rsi), %r10
-; AVX1-NEXT:    xorq 24(%rsi), %r11
-; AVX1-NEXT:    xorq (%rsi), %r8
-; AVX1-NEXT:    xorq 16(%rsi), %r9
-; AVX1-NEXT:    movq 48(%rdi), %rdx
-; AVX1-NEXT:    movq 32(%rdi), %rax
-; AVX1-NEXT:    movq 56(%rdi), %rcx
-; AVX1-NEXT:    movq 40(%rdi), %rdi
-; AVX1-NEXT:    xorq 40(%rsi), %rdi
-; AVX1-NEXT:    xorq 56(%rsi), %rcx
-; AVX1-NEXT:    orq %r11, %rcx
-; AVX1-NEXT:    orq %rdi, %rcx
-; AVX1-NEXT:    orq %r10, %rcx
-; AVX1-NEXT:    xorq 32(%rsi), %rax
-; AVX1-NEXT:    xorq 48(%rsi), %rdx
-; AVX1-NEXT:    orq %r9, %rdx
-; AVX1-NEXT:    orq %rax, %rdx
-; AVX1-NEXT:    orq %r8, %rdx
+; AVX1-NEXT:    vmovups (%rdi), %ymm0
+; AVX1-NEXT:    vmovups 32(%rdi), %ymm1
+; AVX1-NEXT:    vxorps 32(%rsi), %ymm1, %ymm1
+; AVX1-NEXT:    vxorps (%rsi), %ymm0, %ymm0
+; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    xorl %eax, %eax
-; AVX1-NEXT:    orq %rcx, %rdx
+; AVX1-NEXT:    vptest %ymm0, %ymm0
 ; AVX1-NEXT:    sete %al
+; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
-; AVX256-LABEL: eq_i256_pair:
-; AVX256:       # %bb.0:
-; AVX256-NEXT:    vmovdqu (%rdi), %ymm0
-; AVX256-NEXT:    vmovdqu 32(%rdi), %ymm1
-; AVX256-NEXT:    vpcmpeqb 32(%rsi), %ymm1, %ymm1
-; AVX256-NEXT:    vpcmpeqb (%rsi), %ymm0, %ymm0
-; AVX256-NEXT:    vpand %ymm1, %ymm0, %ymm0
-; AVX256-NEXT:    vpmovmskb %ymm0, %ecx
-; AVX256-NEXT:    xorl %eax, %eax
-; AVX256-NEXT:    cmpl $-1, %ecx
-; AVX256-NEXT:    sete %al
-; AVX256-NEXT:    vzeroupper
-; AVX256-NEXT:    retq
+; AVX2-LABEL: eq_i256_pair:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vmovdqu (%rdi), %ymm0
+; AVX2-NEXT:    vmovdqu 32(%rdi), %ymm1
+; AVX2-NEXT:    vpxor 32(%rsi), %ymm1, %ymm1
+; AVX2-NEXT:    vpxor (%rsi), %ymm0, %ymm0
+; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    xorl %eax, %eax
+; AVX2-NEXT:    vptest %ymm0, %ymm0
+; AVX2-NEXT:    sete %al
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: eq_i256_pair:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovdqu (%rdi), %ymm0
+; AVX512-NEXT:    vmovdqu 32(%rdi), %ymm1
+; AVX512-NEXT:    vpxor 32(%rsi), %ymm1, %ymm1
+; AVX512-NEXT:    vpxor (%rsi), %ymm0, %ymm0
+; AVX512-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    xorl %eax, %eax
+; AVX512-NEXT:    vptest %ymm0, %ymm0
+; AVX512-NEXT:    sete %al
+; AVX512-NEXT:    vzeroupper
+; AVX512-NEXT:    retq
   %a0 = load i256, i256* %a
   %b0 = load i256, i256* %b
   %xor1 = xor i256 %a0, %b0
@@ -804,17 +1015,29 @@ define i32 @ne_i512_pair(i512* %a, i512* %b) {
 ; NO512-NEXT:    setne %al
 ; NO512-NEXT:    retq
 ;
-; AVX512-LABEL: ne_i512_pair:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vmovdqu64 (%rdi), %zmm0
-; AVX512-NEXT:    vmovdqu64 64(%rdi), %zmm1
-; AVX512-NEXT:    vpcmpeqd (%rsi), %zmm0, %k1
-; AVX512-NEXT:    vpcmpeqd 64(%rsi), %zmm1, %k0 {%k1}
-; AVX512-NEXT:    xorl %eax, %eax
-; AVX512-NEXT:    kortestw %k0, %k0
-; AVX512-NEXT:    setae %al
-; AVX512-NEXT:    vzeroupper
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: ne_i512_pair:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vmovdqu64 (%rdi), %zmm0
+; AVX512F-NEXT:    vmovdqu64 64(%rdi), %zmm1
+; AVX512F-NEXT:    vpcmpneqd 64(%rsi), %zmm1, %k0
+; AVX512F-NEXT:    vpcmpneqd (%rsi), %zmm0, %k1
+; AVX512F-NEXT:    xorl %eax, %eax
+; AVX512F-NEXT:    kortestw %k0, %k1
+; AVX512F-NEXT:    setne %al
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: ne_i512_pair:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vmovdqu64 (%rdi), %zmm0
+; AVX512BW-NEXT:    vmovdqu64 64(%rdi), %zmm1
+; AVX512BW-NEXT:    vpcmpneqb 64(%rsi), %zmm1, %k0
+; AVX512BW-NEXT:    vpcmpneqb (%rsi), %zmm0, %k1
+; AVX512BW-NEXT:    xorl %eax, %eax
+; AVX512BW-NEXT:    kortestq %k0, %k1
+; AVX512BW-NEXT:    setne %al
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
   %a0 = load i512, i512* %a
   %b0 = load i512, i512* %b
   %xor1 = xor i512 %a0, %b0
@@ -886,17 +1109,29 @@ define i32 @eq_i512_pair(i512* %a, i512* %b) {
 ; NO512-NEXT:    sete %al
 ; NO512-NEXT:    retq
 ;
-; AVX512-LABEL: eq_i512_pair:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vmovdqu64 (%rdi), %zmm0
-; AVX512-NEXT:    vmovdqu64 64(%rdi), %zmm1
-; AVX512-NEXT:    vpcmpeqd (%rsi), %zmm0, %k1
-; AVX512-NEXT:    vpcmpeqd 64(%rsi), %zmm1, %k0 {%k1}
-; AVX512-NEXT:    xorl %eax, %eax
-; AVX512-NEXT:    kortestw %k0, %k0
-; AVX512-NEXT:    setb %al
-; AVX512-NEXT:    vzeroupper
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: eq_i512_pair:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vmovdqu64 (%rdi), %zmm0
+; AVX512F-NEXT:    vmovdqu64 64(%rdi), %zmm1
+; AVX512F-NEXT:    vpcmpneqd 64(%rsi), %zmm1, %k0
+; AVX512F-NEXT:    vpcmpneqd (%rsi), %zmm0, %k1
+; AVX512F-NEXT:    xorl %eax, %eax
+; AVX512F-NEXT:    kortestw %k0, %k1
+; AVX512F-NEXT:    sete %al
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: eq_i512_pair:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vmovdqu64 (%rdi), %zmm0
+; AVX512BW-NEXT:    vmovdqu64 64(%rdi), %zmm1
+; AVX512BW-NEXT:    vpcmpneqb 64(%rsi), %zmm1, %k0
+; AVX512BW-NEXT:    vpcmpneqb (%rsi), %zmm0, %k1
+; AVX512BW-NEXT:    xorl %eax, %eax
+; AVX512BW-NEXT:    kortestq %k0, %k1
+; AVX512BW-NEXT:    sete %al
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
   %a0 = load i512, i512* %a
   %b0 = load i512, i512* %b
   %xor1 = xor i512 %a0, %b0
diff --git a/llvm/test/CodeGen/X86/setcc.ll b/llvm/test/CodeGen/X86/setcc.ll
index 648935dd0..f9ebeff7e 100644
--- a/llvm/test/CodeGen/X86/setcc.ll
+++ b/llvm/test/CodeGen/X86/setcc.ll
@@ -89,4 +89,18 @@ define zeroext i1 @t6(i32 %a) #0 {
   ret i1 %.not
 }
 
+define i16 @shift_and(i16 %a) {
+; CHECK-LABEL: shift_and:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    shrl $10, %eax
+; CHECK-NEXT:    andl $1, %eax
+; CHECK-NEXT:    ## kill: def $ax killed $ax killed $eax
+; CHECK-NEXT:    retq
+  %and = and i16 %a, 1024
+  %cmp = icmp ne i16 %and, 0
+  %conv = zext i1 %cmp to i16
+  ret i16 %conv
+}
+
 attributes #0 = { "target-cpu"="skylake-avx512" }
diff --git a/llvm/test/CodeGen/X86/setuge.ll b/llvm/test/CodeGen/X86/setuge.ll
index 96187198a..ea5562dbd 100644
--- a/llvm/test/CodeGen/X86/setuge.ll
+++ b/llvm/test/CodeGen/X86/setuge.ll
@@ -1,8 +1,26 @@
-; RUN: llc < %s -mtriple=i686--  | not grep set
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- | FileCheck %s
 
 declare i1 @llvm.isunordered.f32(float, float)
 
 define float @cmp(float %A, float %B, float %C, float %D) nounwind {
+; CHECK-LABEL: cmp:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    flds {{[0-9]+}}(%esp)
+; CHECK-NEXT:    flds {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fucompp
+; CHECK-NEXT:    fnstsw %ax
+; CHECK-NEXT:    # kill: def $ah killed $ah killed $ax
+; CHECK-NEXT:    sahf
+; CHECK-NEXT:    jbe .LBB0_1
+; CHECK-NEXT:  # %bb.2: # %entry
+; CHECK-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    flds (%eax)
+; CHECK-NEXT:    retl
+; CHECK-NEXT:  .LBB0_1:
+; CHECK-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    flds (%eax)
+; CHECK-NEXT:    retl
 entry:
         %tmp.1 = fcmp uno float %A, %B          ; <i1> [#uses=1]
         %tmp.2 = fcmp oge float %A, %B          ; <i1> [#uses=1]
diff --git a/llvm/test/CodeGen/X86/shadow-stack.ll b/llvm/test/CodeGen/X86/shadow-stack.ll
index 6fa413781..bb5461062 100644
--- a/llvm/test/CodeGen/X86/shadow-stack.ll
+++ b/llvm/test/CodeGen/X86/shadow-stack.ll
@@ -141,15 +141,14 @@ define i32 @foo(i32 %i) local_unnamed_addr {
 ; X86_64-NEXT:    #EH_SjLj_Setup LBB1_4
 ; X86_64-NEXT:  ## %bb.1: ## %entry
 ; X86_64-NEXT:    xorl %eax, %eax
-; X86_64-NEXT:    testl %eax, %eax
-; X86_64-NEXT:    jne LBB1_3
-; X86_64-NEXT:    jmp LBB1_5
+; X86_64-NEXT:    jmp LBB1_2
 ; X86_64-NEXT:  LBB1_4: ## Block address taken
 ; X86_64-NEXT:    ## %entry
 ; X86_64-NEXT:    movl $1, %eax
+; X86_64-NEXT:  LBB1_2: ## %entry
 ; X86_64-NEXT:    testl %eax, %eax
 ; X86_64-NEXT:    je LBB1_5
-; X86_64-NEXT:  LBB1_3: ## %if.end
+; X86_64-NEXT:  ## %bb.3: ## %if.end
 ; X86_64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax ## 8-byte Reload
 ; X86_64-NEXT:    shll $2, %eax
 ; X86_64-NEXT:    leal (%rax,%rax,2), %eax
@@ -190,15 +189,14 @@ define i32 @foo(i32 %i) local_unnamed_addr {
 ; X86-NEXT:    #EH_SjLj_Setup LBB1_4
 ; X86-NEXT:  ## %bb.1: ## %entry
 ; X86-NEXT:    xorl %eax, %eax
-; X86-NEXT:    testl %eax, %eax
-; X86-NEXT:    jne LBB1_3
-; X86-NEXT:    jmp LBB1_5
+; X86-NEXT:    jmp LBB1_2
 ; X86-NEXT:  LBB1_4: ## Block address taken
 ; X86-NEXT:    ## %entry
 ; X86-NEXT:    movl $1, %eax
+; X86-NEXT:  LBB1_2: ## %entry
 ; X86-NEXT:    testl %eax, %eax
 ; X86-NEXT:    je LBB1_5
-; X86-NEXT:  LBB1_3: ## %if.end
+; X86-NEXT:  ## %bb.3: ## %if.end
 ; X86-NEXT:    movl 8(%ebp), %eax
 ; X86-NEXT:    shll $2, %eax
 ; X86-NEXT:    leal (%eax,%eax,2), %eax
diff --git a/llvm/test/CodeGen/X86/shift-amount-mod.ll b/llvm/test/CodeGen/X86/shift-amount-mod.ll
index 70fa32027..6c0527c91 100644
--- a/llvm/test/CodeGen/X86/shift-amount-mod.ll
+++ b/llvm/test/CodeGen/X86/shift-amount-mod.ll
@@ -1552,3 +1552,25 @@ define i64 @reg64_lshr_by_masked_negated_unfolded_add_b(i64 %val, i64 %a, i64 %b
   %shifted = lshr i64 %val, %negaaddbitwidthaddb
   ret i64 %shifted
 }
+
+define i16 @sh_trunc_sh(i64 %x) {
+; X32-LABEL: sh_trunc_sh:
+; X32:       # %bb.0:
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    shrl $4, %eax
+; X32-NEXT:    andl $15, %eax
+; X32-NEXT:    # kill: def $ax killed $ax killed $eax
+; X32-NEXT:    retl
+;
+; X64-LABEL: sh_trunc_sh:
+; X64:       # %bb.0:
+; X64-NEXT:    movq %rdi, %rax
+; X64-NEXT:    shrq $36, %rax
+; X64-NEXT:    andl $15, %eax
+; X64-NEXT:    # kill: def $ax killed $ax killed $rax
+; X64-NEXT:    retq
+  %s = lshr i64 %x, 24
+  %t = trunc i64 %s to i16
+  %r = lshr i16 %t, 12
+  ret i16 %r
+}
diff --git a/llvm/test/CodeGen/X86/shift-avx2-crash.ll b/llvm/test/CodeGen/X86/shift-avx2-crash.ll
index b23d6e335..868c013bf 100644
--- a/llvm/test/CodeGen/X86/shift-avx2-crash.ll
+++ b/llvm/test/CodeGen/X86/shift-avx2-crash.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s  -mcpu=core-avx2 > /dev/null
 ; This test crashed on variable shift creation on AVX2
 
@@ -29,7 +30,7 @@ if_then18.i:
   br label %__fast_idiv___UM_vyuunu.exit
 
 
-__fast_idiv___UM_vyuunu.exit:                   
+__fast_idiv___UM_vyuunu.exit:
   %calltmp_to_float = uitofp <8 x i32> %bitop22_to_uint32.i to <8 x float>
   %ptrcast = bitcast float* %RET to <8 x float>*
   store <8 x float> %calltmp_to_float, <8 x float>* %ptrcast, align 4
diff --git a/llvm/test/CodeGen/X86/shift-by-signext.ll b/llvm/test/CodeGen/X86/shift-by-signext.ll
new file mode 100644
index 000000000..85ca2bd6e
--- /dev/null
+++ b/llvm/test/CodeGen/X86/shift-by-signext.ll
@@ -0,0 +1,148 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+bmi,+bmi2,+sse,+sse2,+avx,+avx2 | FileCheck %s --check-prefixes=CHECK,X86
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+bmi,+bmi2,+sse,+sse2,+avx,+avx2 | FileCheck %s --check-prefixes=CHECK,X64
+
+; If we have a shift by sign-extended value, we can replace sign-extension
+; with zero-extension.
+
+define i32 @t0_shl(i32 %x, i8 %shamt) nounwind {
+; X86-LABEL: t0_shl:
+; X86:       # %bb.0:
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    shlxl %eax, {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: t0_shl:
+; X64:       # %bb.0:
+; X64-NEXT:    shlxl %esi, %edi, %eax
+; X64-NEXT:    retq
+  %shamt_wide = sext i8 %shamt to i32
+  %r = shl i32 %x, %shamt_wide
+  ret i32 %r
+}
+define i32 @t1_lshr(i32 %x, i8 %shamt) nounwind {
+; X86-LABEL: t1_lshr:
+; X86:       # %bb.0:
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    shrxl %eax, {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: t1_lshr:
+; X64:       # %bb.0:
+; X64-NEXT:    shrxl %esi, %edi, %eax
+; X64-NEXT:    retq
+  %shamt_wide = sext i8 %shamt to i32
+  %r = lshr i32 %x, %shamt_wide
+  ret i32 %r
+}
+define i32 @t2_ashr(i32 %x, i8 %shamt) nounwind {
+; X86-LABEL: t2_ashr:
+; X86:       # %bb.0:
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    sarxl %eax, {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: t2_ashr:
+; X64:       # %bb.0:
+; X64-NEXT:    sarxl %esi, %edi, %eax
+; X64-NEXT:    retq
+  %shamt_wide = sext i8 %shamt to i32
+  %r = ashr i32 %x, %shamt_wide
+  ret i32 %r
+}
+
+define <4 x i32> @t3_vec_shl(<4 x i32> %x, <4 x i8> %shamt) nounwind {
+; CHECK-LABEL: t3_vec_shl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmovsxbd %xmm1, %xmm1
+; CHECK-NEXT:    vpsllvd %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %shamt_wide = sext <4 x i8> %shamt to <4 x i32>
+  %r = shl <4 x i32> %x, %shamt_wide
+  ret <4 x i32> %r
+}
+define <4 x i32> @t4_vec_lshr(<4 x i32> %x, <4 x i8> %shamt) nounwind {
+; CHECK-LABEL: t4_vec_lshr:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmovsxbd %xmm1, %xmm1
+; CHECK-NEXT:    vpsrlvd %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %shamt_wide = sext <4 x i8> %shamt to <4 x i32>
+  %r = lshr <4 x i32> %x, %shamt_wide
+  ret <4 x i32> %r
+}
+define <4 x i32> @t5_vec_ashr(<4 x i32> %x, <4 x i8> %shamt) nounwind {
+; CHECK-LABEL: t5_vec_ashr:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmovsxbd %xmm1, %xmm1
+; CHECK-NEXT:    vpsravd %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %shamt_wide = sext <4 x i8> %shamt to <4 x i32>
+  %r = ashr <4 x i32> %x, %shamt_wide
+  ret <4 x i32> %r
+}
+
+; This is not valid for funnel shifts
+declare i32 @llvm.fshl.i32(i32 %a, i32 %b, i32 %c)
+declare i32 @llvm.fshr.i32(i32 %a, i32 %b, i32 %c)
+define i32 @n6_fshl(i32 %x, i32 %y, i8 %shamt) nounwind {
+; X86-LABEL: n6_fshl:
+; X86:       # %bb.0:
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    shldl %cl, %edx, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: n6_fshl:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edx, %ecx
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-NEXT:    shldl %cl, %esi, %eax
+; X64-NEXT:    retq
+  %shamt_wide = sext i8 %shamt to i32
+  %r = call i32 @llvm.fshl.i32(i32 %x, i32 %y, i32 %shamt_wide)
+  ret i32 %r
+}
+define i32 @n7_fshr(i32 %x, i32 %y, i8 %shamt) nounwind {
+; X86-LABEL: n7_fshr:
+; X86:       # %bb.0:
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    shrdl %cl, %edx, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: n7_fshr:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edx, %ecx
+; X64-NEXT:    movl %esi, %eax
+; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-NEXT:    shrdl %cl, %edi, %eax
+; X64-NEXT:    retq
+  %shamt_wide = sext i8 %shamt to i32
+  %r = call i32 @llvm.fshr.i32(i32 %x, i32 %y, i32 %shamt_wide)
+  ret i32 %r
+}
+
+define i32 @n8_extrause(i32 %x, i8 %shamt, i32* %shamt_wide_store) nounwind {
+; X86-LABEL: n8_extrause:
+; X86:       # %bb.0:
+; X86-NEXT:    movsbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %eax, (%ecx)
+; X86-NEXT:    shlxl %eax, {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: n8_extrause:
+; X64:       # %bb.0:
+; X64-NEXT:    movsbl %sil, %eax
+; X64-NEXT:    movl %eax, (%rdx)
+; X64-NEXT:    shlxl %eax, %edi, %eax
+; X64-NEXT:    retq
+  %shamt_wide = sext i8 %shamt to i32
+  store i32 %shamt_wide, i32* %shamt_wide_store, align 4
+  %r = shl i32 %x, %shamt_wide
+  ret i32 %r
+}
diff --git a/llvm/test/CodeGen/X86/shift-coalesce.ll b/llvm/test/CodeGen/X86/shift-coalesce.ll
index 7f2c3b5a0..5eda5eb3b 100644
--- a/llvm/test/CodeGen/X86/shift-coalesce.ll
+++ b/llvm/test/CodeGen/X86/shift-coalesce.ll
@@ -1,11 +1,30 @@
-; RUN: llc < %s -mtriple=i686-- -x86-asm-syntax=intel | \
-; RUN:   grep "shld.*cl"
-; RUN: llc < %s -mtriple=i686-- -x86-asm-syntax=intel | \
-; RUN:   not grep "mov cl, bl"
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- -x86-asm-syntax=intel | FileCheck %s
 
 ; PR687
 
 define i64 @foo(i64 %x, i64* %X) {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    push esi
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    .cfi_offset esi, -8
+; CHECK-NEXT:    mov esi, dword ptr [esp + 8]
+; CHECK-NEXT:    mov edx, dword ptr [esp + 12]
+; CHECK-NEXT:    mov eax, dword ptr [esp + 16]
+; CHECK-NEXT:    mov cl, byte ptr [eax]
+; CHECK-NEXT:    mov eax, esi
+; CHECK-NEXT:    shl eax, cl
+; CHECK-NEXT:    shld edx, esi, cl
+; CHECK-NEXT:    test cl, 32
+; CHECK-NEXT:    je .LBB0_2
+; CHECK-NEXT:  # %bb.1:
+; CHECK-NEXT:    mov edx, eax
+; CHECK-NEXT:    xor eax, eax
+; CHECK-NEXT:  .LBB0_2:
+; CHECK-NEXT:    pop esi
+; CHECK-NEXT:    .cfi_def_cfa_offset 4
+; CHECK-NEXT:    ret
         %tmp.1 = load i64, i64* %X           ; <i64> [#uses=1]
         %tmp.3 = trunc i64 %tmp.1 to i8         ; <i8> [#uses=1]
         %shift.upgrd.1 = zext i8 %tmp.3 to i64          ; <i64> [#uses=1]
diff --git a/llvm/test/CodeGen/X86/shift-combine-crash.ll b/llvm/test/CodeGen/X86/shift-combine-crash.ll
index a69a907d4..5db2ac978 100644
--- a/llvm/test/CodeGen/X86/shift-combine-crash.ll
+++ b/llvm/test/CodeGen/X86/shift-combine-crash.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=corei7 > /dev/null
 
 ; Verify that DAGCombiner doesn't crash with an assertion failure in the
diff --git a/llvm/test/CodeGen/X86/shift-combine.ll b/llvm/test/CodeGen/X86/shift-combine.ll
index bdbf3379e..59fe62c0e 100644
--- a/llvm/test/CodeGen/X86/shift-combine.ll
+++ b/llvm/test/CodeGen/X86/shift-combine.ll
@@ -154,3 +154,273 @@ define i32* @test_exact6(i32 %a, i32 %b, i32* %x)  {
   %gep = getelementptr inbounds i32, i32* %x, i32 %shr
   ret i32* %gep
 }
+
+; PR42644 - https://bugs.llvm.org/show_bug.cgi?id=42644
+
+define i64 @ashr_add_shl_i32(i64 %r) nounwind {
+; X32-LABEL: ashr_add_shl_i32:
+; X32:       # %bb.0:
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    incl %eax
+; X32-NEXT:    movl %eax, %edx
+; X32-NEXT:    sarl $31, %edx
+; X32-NEXT:    retl
+;
+; X64-LABEL: ashr_add_shl_i32:
+; X64:       # %bb.0:
+; X64-NEXT:    incl %edi
+; X64-NEXT:    movslq %edi, %rax
+; X64-NEXT:    retq
+  %conv = shl i64 %r, 32
+  %sext = add i64 %conv, 4294967296
+  %conv1 = ashr i64 %sext, 32
+  ret i64 %conv1
+}
+
+define i64 @ashr_add_shl_i8(i64 %r) nounwind {
+; X32-LABEL: ashr_add_shl_i8:
+; X32:       # %bb.0:
+; X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X32-NEXT:    addb $2, %al
+; X32-NEXT:    movsbl %al, %eax
+; X32-NEXT:    movl %eax, %edx
+; X32-NEXT:    sarl $31, %edx
+; X32-NEXT:    retl
+;
+; X64-LABEL: ashr_add_shl_i8:
+; X64:       # %bb.0:
+; X64-NEXT:    addb $2, %dil
+; X64-NEXT:    movsbq %dil, %rax
+; X64-NEXT:    retq
+  %conv = shl i64 %r, 56
+  %sext = add i64 %conv, 144115188075855872
+  %conv1 = ashr i64 %sext, 56
+  ret i64 %conv1
+}
+
+define <4 x i32> @ashr_add_shl_v4i8(<4 x i32> %r) nounwind {
+; X32-LABEL: ashr_add_shl_v4i8:
+; X32:       # %bb.0:
+; X32-NEXT:    pushl %edi
+; X32-NEXT:    pushl %esi
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X32-NEXT:    movb {{[0-9]+}}(%esp), %dl
+; X32-NEXT:    movb {{[0-9]+}}(%esp), %ch
+; X32-NEXT:    movb {{[0-9]+}}(%esp), %dh
+; X32-NEXT:    incb %dh
+; X32-NEXT:    movsbl %dh, %esi
+; X32-NEXT:    incb %ch
+; X32-NEXT:    movsbl %ch, %edi
+; X32-NEXT:    incb %dl
+; X32-NEXT:    movsbl %dl, %edx
+; X32-NEXT:    incb %cl
+; X32-NEXT:    movsbl %cl, %ecx
+; X32-NEXT:    movl %ecx, 12(%eax)
+; X32-NEXT:    movl %edx, 8(%eax)
+; X32-NEXT:    movl %edi, 4(%eax)
+; X32-NEXT:    movl %esi, (%eax)
+; X32-NEXT:    popl %esi
+; X32-NEXT:    popl %edi
+; X32-NEXT:    retl $4
+;
+; X64-LABEL: ashr_add_shl_v4i8:
+; X64:       # %bb.0:
+; X64-NEXT:    pand {{.*}}(%rip), %xmm0
+; X64-NEXT:    packuswb %xmm0, %xmm0
+; X64-NEXT:    packuswb %xmm0, %xmm0
+; X64-NEXT:    pcmpeqd %xmm1, %xmm1
+; X64-NEXT:    psubb %xmm1, %xmm0
+; X64-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X64-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; X64-NEXT:    psrad $24, %xmm0
+; X64-NEXT:    retq
+  %conv = shl <4 x i32> %r, <i32 24, i32 24, i32 24, i32 24>
+  %sext = add <4 x i32> %conv, <i32 16777216, i32 16777216, i32 16777216, i32 16777216>
+  %conv1 = ashr <4 x i32> %sext, <i32 24, i32 24, i32 24, i32 24>
+  ret <4 x i32> %conv1
+}
+
+define i64 @ashr_add_shl_i36(i64 %r) nounwind {
+; X32-LABEL: ashr_add_shl_i36:
+; X32:       # %bb.0:
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT:    shll $4, %edx
+; X32-NEXT:    movl %edx, %eax
+; X32-NEXT:    sarl $4, %eax
+; X32-NEXT:    sarl $31, %edx
+; X32-NEXT:    retl
+;
+; X64-LABEL: ashr_add_shl_i36:
+; X64:       # %bb.0:
+; X64-NEXT:    movq %rdi, %rax
+; X64-NEXT:    shlq $36, %rax
+; X64-NEXT:    sarq $36, %rax
+; X64-NEXT:    retq
+  %conv = shl i64 %r, 36
+  %sext = add i64 %conv, 4294967296
+  %conv1 = ashr i64 %sext, 36
+  ret i64 %conv1
+}
+
+define i64 @ashr_add_shl_mismatch_shifts1(i64 %r) nounwind {
+; X32-LABEL: ashr_add_shl_mismatch_shifts1:
+; X32:       # %bb.0:
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    incl %eax
+; X32-NEXT:    movl %eax, %edx
+; X32-NEXT:    sarl $31, %edx
+; X32-NEXT:    retl
+;
+; X64-LABEL: ashr_add_shl_mismatch_shifts1:
+; X64:       # %bb.0:
+; X64-NEXT:    shlq $8, %rdi
+; X64-NEXT:    movabsq $4294967296, %rax # imm = 0x100000000
+; X64-NEXT:    addq %rdi, %rax
+; X64-NEXT:    sarq $32, %rax
+; X64-NEXT:    retq
+  %conv = shl i64 %r, 8
+  %sext = add i64 %conv, 4294967296
+  %conv1 = ashr i64 %sext, 32
+  ret i64 %conv1
+}
+
+define i64 @ashr_add_shl_mismatch_shifts2(i64 %r) nounwind {
+; X32-LABEL: ashr_add_shl_mismatch_shifts2:
+; X32:       # %bb.0:
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT:    shrdl $8, %edx, %eax
+; X32-NEXT:    shrl $8, %edx
+; X32-NEXT:    incl %edx
+; X32-NEXT:    shrdl $8, %edx, %eax
+; X32-NEXT:    shrl $8, %edx
+; X32-NEXT:    retl
+;
+; X64-LABEL: ashr_add_shl_mismatch_shifts2:
+; X64:       # %bb.0:
+; X64-NEXT:    shrq $8, %rdi
+; X64-NEXT:    movabsq $4294967296, %rax # imm = 0x100000000
+; X64-NEXT:    addq %rdi, %rax
+; X64-NEXT:    shrq $8, %rax
+; X64-NEXT:    retq
+  %conv = lshr i64 %r, 8
+  %sext = add i64 %conv, 4294967296
+  %conv1 = ashr i64 %sext, 8
+  ret i64 %conv1
+}
+
+define i32 @ashr_add_shl_i32_i8_extra_use1(i32 %r, i32* %p) nounwind {
+; X32-LABEL: ashr_add_shl_i32_i8_extra_use1:
+; X32:       # %bb.0:
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    shll $24, %eax
+; X32-NEXT:    addl $33554432, %eax # imm = 0x2000000
+; X32-NEXT:    movl %eax, (%ecx)
+; X32-NEXT:    sarl $24, %eax
+; X32-NEXT:    retl
+;
+; X64-LABEL: ashr_add_shl_i32_i8_extra_use1:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    shll $24, %eax
+; X64-NEXT:    addl $33554432, %eax # imm = 0x2000000
+; X64-NEXT:    movl %eax, (%rsi)
+; X64-NEXT:    sarl $24, %eax
+; X64-NEXT:    retq
+  %conv = shl i32 %r, 24
+  %sext = add i32 %conv, 33554432
+  store i32 %sext, i32* %p
+  %conv1 = ashr i32 %sext, 24
+  ret i32 %conv1
+}
+
+define i32 @ashr_add_shl_i32_i8_extra_use2(i32 %r, i32* %p) nounwind {
+; X32-LABEL: ashr_add_shl_i32_i8_extra_use2:
+; X32:       # %bb.0:
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    shll $24, %eax
+; X32-NEXT:    movl %eax, (%ecx)
+; X32-NEXT:    addl $33554432, %eax # imm = 0x2000000
+; X32-NEXT:    sarl $24, %eax
+; X32-NEXT:    retl
+;
+; X64-LABEL: ashr_add_shl_i32_i8_extra_use2:
+; X64:       # %bb.0:
+; X64-NEXT:    # kill: def $edi killed $edi def $rdi
+; X64-NEXT:    shll $24, %edi
+; X64-NEXT:    movl %edi, (%rsi)
+; X64-NEXT:    leal 33554432(%rdi), %eax
+; X64-NEXT:    sarl $24, %eax
+; X64-NEXT:    retq
+  %conv = shl i32 %r, 24
+  store i32 %conv, i32* %p
+  %sext = add i32 %conv, 33554432
+  %conv1 = ashr i32 %sext, 24
+  ret i32 %conv1
+}
+
+define i32 @ashr_add_shl_i32_i8_extra_use3(i32 %r, i32* %p1, i32* %p2) nounwind {
+; X32-LABEL: ashr_add_shl_i32_i8_extra_use3:
+; X32:       # %bb.0:
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    shll $24, %eax
+; X32-NEXT:    movl %eax, (%edx)
+; X32-NEXT:    addl $33554432, %eax # imm = 0x2000000
+; X32-NEXT:    movl %eax, (%ecx)
+; X32-NEXT:    sarl $24, %eax
+; X32-NEXT:    retl
+;
+; X64-LABEL: ashr_add_shl_i32_i8_extra_use3:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    shll $24, %eax
+; X64-NEXT:    movl %eax, (%rsi)
+; X64-NEXT:    addl $33554432, %eax # imm = 0x2000000
+; X64-NEXT:    movl %eax, (%rdx)
+; X64-NEXT:    sarl $24, %eax
+; X64-NEXT:    retq
+  %conv = shl i32 %r, 24
+  store i32 %conv, i32* %p1
+  %sext = add i32 %conv, 33554432
+  store i32 %sext, i32* %p2
+  %conv1 = ashr i32 %sext, 24
+  ret i32 %conv1
+}
+
+%"class.QPainterPath" = type { double, double, i32 }
+
+define void @PR42880(i32 %t0) {
+; X32-LABEL: PR42880:
+; X32:       # %bb.0:
+; X32-NEXT:    xorl %eax, %eax
+; X32-NEXT:    testb %al, %al
+; X32-NEXT:    je .LBB16_1
+; X32-NEXT:  # %bb.2: # %if
+; X32-NEXT:  .LBB16_1: # %then
+;
+; X64-LABEL: PR42880:
+; X64:       # %bb.0:
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    testb %al, %al
+; X64-NEXT:    je .LBB16_1
+; X64-NEXT:  # %bb.2: # %if
+; X64-NEXT:  .LBB16_1: # %then
+  %sub = add nsw i32 %t0, -1
+  %add.ptr.i94 = getelementptr inbounds %"class.QPainterPath", %"class.QPainterPath"* null, i32 %sub
+  %x = ptrtoint %"class.QPainterPath"* %add.ptr.i94 to i32
+  %sub2 = sub i32 %x, 0
+  %div = sdiv exact i32 %sub2, 24
+  br i1 undef, label %if, label %then
+
+then:
+  %t1 = xor i32 %div, -1
+  unreachable
+
+if:
+  unreachable
+}
diff --git a/llvm/test/CodeGen/X86/shift-i128.ll b/llvm/test/CodeGen/X86/shift-i128.ll
index 9c69aab5b..4dffbb795 100644
--- a/llvm/test/CodeGen/X86/shift-i128.ll
+++ b/llvm/test/CodeGen/X86/shift-i128.ll
@@ -1,11 +1,142 @@
-; RUN: llc < %s -mtriple=i686--
-; RUN: llc < %s -mtriple=x86_64--
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- | FileCheck %s --check-prefixes=ALL,i686
+; RUN: llc < %s -mtriple=x86_64-- | FileCheck %s --check-prefixes=ALL,x86_64
 
 ;
 ; Scalars
 ;
 
 define void @test_lshr_i128(i128 %x, i128 %a, i128* nocapture %r) nounwind {
+; i686-LABEL: test_lshr_i128:
+; i686:       # %bb.0: # %entry
+; i686-NEXT:    pushl %ebp
+; i686-NEXT:    pushl %ebx
+; i686-NEXT:    pushl %edi
+; i686-NEXT:    pushl %esi
+; i686-NEXT:    subl $20, %esp
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; i686-NEXT:    movb {{[0-9]+}}(%esp), %al
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; i686-NEXT:    movl %ebp, %esi
+; i686-NEXT:    movl %eax, %ecx
+; i686-NEXT:    shrdl %cl, %edi, %esi
+; i686-NEXT:    shrl %cl, %edx
+; i686-NEXT:    shrl %cl, %edi
+; i686-NEXT:    movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; i686-NEXT:    testb $32, %al
+; i686-NEXT:    jne .LBB0_1
+; i686-NEXT:  # %bb.2: # %entry
+; i686-NEXT:    movl %edx, (%esp) # 4-byte Spill
+; i686-NEXT:    jmp .LBB0_3
+; i686-NEXT:  .LBB0_1:
+; i686-NEXT:    movl %edi, %esi
+; i686-NEXT:    movl $0, (%esp) # 4-byte Folded Spill
+; i686-NEXT:    xorl %edi, %edi
+; i686-NEXT:  .LBB0_3: # %entry
+; i686-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    movl %eax, %edx
+; i686-NEXT:    subb $64, %dl
+; i686-NEXT:    jb .LBB0_5
+; i686-NEXT:  # %bb.4: # %entry
+; i686-NEXT:    xorl %edi, %edi
+; i686-NEXT:  .LBB0_5: # %entry
+; i686-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    negb %dl
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; i686-NEXT:    movl %edx, %ecx
+; i686-NEXT:    shldl %cl, %ebp, %edi
+; i686-NEXT:    movl %ebp, %esi
+; i686-NEXT:    shll %cl, %esi
+; i686-NEXT:    testb $32, %dl
+; i686-NEXT:    movl %esi, %ebx
+; i686-NEXT:    jne .LBB0_7
+; i686-NEXT:  # %bb.6: # %entry
+; i686-NEXT:    movl %edi, %ebx
+; i686-NEXT:  .LBB0_7: # %entry
+; i686-NEXT:    movb %al, %ah
+; i686-NEXT:    addb $-64, %ah
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; i686-NEXT:    movb %ah, %cl
+; i686-NEXT:    shrl %cl, %edi
+; i686-NEXT:    testb $32, %ah
+; i686-NEXT:    movl $0, %ecx
+; i686-NEXT:    jne .LBB0_9
+; i686-NEXT:  # %bb.8: # %entry
+; i686-NEXT:    movl %edi, %ecx
+; i686-NEXT:  .LBB0_9: # %entry
+; i686-NEXT:    cmpb $64, %al
+; i686-NEXT:    jb .LBB0_10
+; i686-NEXT:  # %bb.11: # %entry
+; i686-NEXT:    movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; i686-NEXT:    jmp .LBB0_12
+; i686-NEXT:  .LBB0_10:
+; i686-NEXT:    movl (%esp), %ecx # 4-byte Reload
+; i686-NEXT:    orl %ebx, %ecx
+; i686-NEXT:  .LBB0_12: # %entry
+; i686-NEXT:    movl %ecx, (%esp) # 4-byte Spill
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; i686-NEXT:    testb $32, %dl
+; i686-NEXT:    jne .LBB0_14
+; i686-NEXT:  # %bb.13: # %entry
+; i686-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:  .LBB0_14: # %entry
+; i686-NEXT:    movl %ebx, %edx
+; i686-NEXT:    movl %eax, %ecx
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; i686-NEXT:    shrdl %cl, %esi, %edx
+; i686-NEXT:    testb $32, %al
+; i686-NEXT:    jne .LBB0_16
+; i686-NEXT:  # %bb.15: # %entry
+; i686-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:  .LBB0_16: # %entry
+; i686-NEXT:    movb %ah, %cl
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; i686-NEXT:    shrdl %cl, %edx, %ebp
+; i686-NEXT:    testb $32, %ah
+; i686-NEXT:    jne .LBB0_18
+; i686-NEXT:  # %bb.17: # %entry
+; i686-NEXT:    movl %ebp, %edi
+; i686-NEXT:  .LBB0_18: # %entry
+; i686-NEXT:    cmpb $64, %al
+; i686-NEXT:    jae .LBB0_20
+; i686-NEXT:  # %bb.19:
+; i686-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; i686-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; i686-NEXT:  .LBB0_20: # %entry
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; i686-NEXT:    testb %al, %al
+; i686-NEXT:    je .LBB0_22
+; i686-NEXT:  # %bb.21: # %entry
+; i686-NEXT:    movl %edi, %ebx
+; i686-NEXT:    movl (%esp), %esi # 4-byte Reload
+; i686-NEXT:  .LBB0_22: # %entry
+; i686-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; i686-NEXT:    movl %eax, 12(%ecx)
+; i686-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; i686-NEXT:    movl %eax, 8(%ecx)
+; i686-NEXT:    movl %esi, 4(%ecx)
+; i686-NEXT:    movl %ebx, (%ecx)
+; i686-NEXT:    addl $20, %esp
+; i686-NEXT:    popl %esi
+; i686-NEXT:    popl %edi
+; i686-NEXT:    popl %ebx
+; i686-NEXT:    popl %ebp
+; i686-NEXT:    retl
+;
+; x86_64-LABEL: test_lshr_i128:
+; x86_64:       # %bb.0: # %entry
+; x86_64-NEXT:    movq %rdx, %rcx
+; x86_64-NEXT:    shrdq %cl, %rsi, %rdi
+; x86_64-NEXT:    shrq %cl, %rsi
+; x86_64-NEXT:    xorl %eax, %eax
+; x86_64-NEXT:    testb $64, %cl
+; x86_64-NEXT:    cmovneq %rsi, %rdi
+; x86_64-NEXT:    cmoveq %rsi, %rax
+; x86_64-NEXT:    movq %rax, 8(%r8)
+; x86_64-NEXT:    movq %rdi, (%r8)
+; x86_64-NEXT:    retq
 entry:
 	%0 = lshr i128 %x, %a
 	store i128 %0, i128* %r, align 16
@@ -13,6 +144,141 @@ entry:
 }
 
 define void @test_ashr_i128(i128 %x, i128 %a, i128* nocapture %r) nounwind {
+; i686-LABEL: test_ashr_i128:
+; i686:       # %bb.0: # %entry
+; i686-NEXT:    pushl %ebp
+; i686-NEXT:    pushl %ebx
+; i686-NEXT:    pushl %edi
+; i686-NEXT:    pushl %esi
+; i686-NEXT:    subl $24, %esp
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; i686-NEXT:    movb {{[0-9]+}}(%esp), %al
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; i686-NEXT:    movl %ebp, %esi
+; i686-NEXT:    movl %eax, %ecx
+; i686-NEXT:    shrdl %cl, %ebx, %esi
+; i686-NEXT:    shrl %cl, %edx
+; i686-NEXT:    movl %ebx, %edi
+; i686-NEXT:    sarl %cl, %edi
+; i686-NEXT:    sarl $31, %ebx
+; i686-NEXT:    movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; i686-NEXT:    testb $32, %al
+; i686-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    jne .LBB1_1
+; i686-NEXT:  # %bb.2: # %entry
+; i686-NEXT:    movl %edx, (%esp) # 4-byte Spill
+; i686-NEXT:    jmp .LBB1_3
+; i686-NEXT:  .LBB1_1:
+; i686-NEXT:    movl %edi, %esi
+; i686-NEXT:    movl $0, (%esp) # 4-byte Folded Spill
+; i686-NEXT:    movl %ebx, %edi
+; i686-NEXT:  .LBB1_3: # %entry
+; i686-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    movl %eax, %edx
+; i686-NEXT:    subb $64, %dl
+; i686-NEXT:    jb .LBB1_5
+; i686-NEXT:  # %bb.4: # %entry
+; i686-NEXT:    movl %ebx, %edi
+; i686-NEXT:  .LBB1_5: # %entry
+; i686-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    negb %dl
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; i686-NEXT:    movl %edx, %ecx
+; i686-NEXT:    shldl %cl, %ebp, %edi
+; i686-NEXT:    movl %ebp, %esi
+; i686-NEXT:    shll %cl, %esi
+; i686-NEXT:    testb $32, %dl
+; i686-NEXT:    movl %esi, %ecx
+; i686-NEXT:    jne .LBB1_7
+; i686-NEXT:  # %bb.6: # %entry
+; i686-NEXT:    movl %edi, %ecx
+; i686-NEXT:  .LBB1_7: # %entry
+; i686-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    movb %al, %ah
+; i686-NEXT:    addb $-64, %ah
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; i686-NEXT:    movb %ah, %cl
+; i686-NEXT:    sarl %cl, %edi
+; i686-NEXT:    testb $32, %ah
+; i686-NEXT:    movl %ebx, %ecx
+; i686-NEXT:    jne .LBB1_9
+; i686-NEXT:  # %bb.8: # %entry
+; i686-NEXT:    movl %edi, %ecx
+; i686-NEXT:  .LBB1_9: # %entry
+; i686-NEXT:    cmpb $64, %al
+; i686-NEXT:    jb .LBB1_10
+; i686-NEXT:  # %bb.11: # %entry
+; i686-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    jmp .LBB1_12
+; i686-NEXT:  .LBB1_10:
+; i686-NEXT:    movl (%esp), %ecx # 4-byte Reload
+; i686-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; i686-NEXT:  .LBB1_12: # %entry
+; i686-NEXT:    movl %ecx, (%esp) # 4-byte Spill
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; i686-NEXT:    testb $32, %dl
+; i686-NEXT:    jne .LBB1_14
+; i686-NEXT:  # %bb.13: # %entry
+; i686-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:  .LBB1_14: # %entry
+; i686-NEXT:    movl %ebx, %edx
+; i686-NEXT:    movl %eax, %ecx
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; i686-NEXT:    shrdl %cl, %esi, %edx
+; i686-NEXT:    testb $32, %al
+; i686-NEXT:    jne .LBB1_16
+; i686-NEXT:  # %bb.15: # %entry
+; i686-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:  .LBB1_16: # %entry
+; i686-NEXT:    movb %ah, %cl
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; i686-NEXT:    shrdl %cl, %edx, %ebp
+; i686-NEXT:    testb $32, %ah
+; i686-NEXT:    jne .LBB1_18
+; i686-NEXT:  # %bb.17: # %entry
+; i686-NEXT:    movl %ebp, %edi
+; i686-NEXT:  .LBB1_18: # %entry
+; i686-NEXT:    cmpb $64, %al
+; i686-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; i686-NEXT:    jae .LBB1_20
+; i686-NEXT:  # %bb.19:
+; i686-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; i686-NEXT:    movl %ecx, %edi
+; i686-NEXT:  .LBB1_20: # %entry
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; i686-NEXT:    testb %al, %al
+; i686-NEXT:    je .LBB1_22
+; i686-NEXT:  # %bb.21: # %entry
+; i686-NEXT:    movl %edi, %ebx
+; i686-NEXT:    movl (%esp), %esi # 4-byte Reload
+; i686-NEXT:  .LBB1_22: # %entry
+; i686-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; i686-NEXT:    movl %eax, 12(%ecx)
+; i686-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; i686-NEXT:    movl %eax, 8(%ecx)
+; i686-NEXT:    movl %esi, 4(%ecx)
+; i686-NEXT:    movl %ebx, (%ecx)
+; i686-NEXT:    addl $24, %esp
+; i686-NEXT:    popl %esi
+; i686-NEXT:    popl %edi
+; i686-NEXT:    popl %ebx
+; i686-NEXT:    popl %ebp
+; i686-NEXT:    retl
+;
+; x86_64-LABEL: test_ashr_i128:
+; x86_64:       # %bb.0: # %entry
+; x86_64-NEXT:    movq %rdx, %rcx
+; x86_64-NEXT:    shrdq %cl, %rsi, %rdi
+; x86_64-NEXT:    movq %rsi, %rax
+; x86_64-NEXT:    sarq %cl, %rax
+; x86_64-NEXT:    sarq $63, %rsi
+; x86_64-NEXT:    testb $64, %cl
+; x86_64-NEXT:    cmovneq %rax, %rdi
+; x86_64-NEXT:    cmoveq %rax, %rsi
+; x86_64-NEXT:    movq %rsi, 8(%r8)
+; x86_64-NEXT:    movq %rdi, (%r8)
+; x86_64-NEXT:    retq
 entry:
 	%0 = ashr i128 %x, %a
 	store i128 %0, i128* %r, align 16
@@ -20,6 +286,137 @@ entry:
 }
 
 define void @test_shl_i128(i128 %x, i128 %a, i128* nocapture %r) nounwind {
+; i686-LABEL: test_shl_i128:
+; i686:       # %bb.0: # %entry
+; i686-NEXT:    pushl %ebp
+; i686-NEXT:    pushl %ebx
+; i686-NEXT:    pushl %edi
+; i686-NEXT:    pushl %esi
+; i686-NEXT:    subl $20, %esp
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; i686-NEXT:    movb {{[0-9]+}}(%esp), %al
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; i686-NEXT:    movl %eax, %ecx
+; i686-NEXT:    shll %cl, %ebx
+; i686-NEXT:    movl %ebp, %esi
+; i686-NEXT:    shll %cl, %esi
+; i686-NEXT:    movl %edi, %edx
+; i686-NEXT:    shldl %cl, %ebp, %edx
+; i686-NEXT:    movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; i686-NEXT:    testb $32, %al
+; i686-NEXT:    jne .LBB2_1
+; i686-NEXT:  # %bb.2: # %entry
+; i686-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    movl %ebx, (%esp) # 4-byte Spill
+; i686-NEXT:    jmp .LBB2_3
+; i686-NEXT:  .LBB2_1:
+; i686-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    movl $0, (%esp) # 4-byte Folded Spill
+; i686-NEXT:    xorl %esi, %esi
+; i686-NEXT:  .LBB2_3: # %entry
+; i686-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    movl %eax, %edx
+; i686-NEXT:    subb $64, %dl
+; i686-NEXT:    jb .LBB2_5
+; i686-NEXT:  # %bb.4: # %entry
+; i686-NEXT:    xorl %esi, %esi
+; i686-NEXT:  .LBB2_5: # %entry
+; i686-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    negb %dl
+; i686-NEXT:    movl %edi, %esi
+; i686-NEXT:    movl %edx, %ecx
+; i686-NEXT:    shrl %cl, %esi
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; i686-NEXT:    shrdl %cl, %edi, %ebx
+; i686-NEXT:    testb $32, %dl
+; i686-NEXT:    movl %esi, %ebp
+; i686-NEXT:    jne .LBB2_7
+; i686-NEXT:  # %bb.6: # %entry
+; i686-NEXT:    movl %ebx, %ebp
+; i686-NEXT:  .LBB2_7: # %entry
+; i686-NEXT:    movb %al, %ah
+; i686-NEXT:    addb $-64, %ah
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; i686-NEXT:    movb %ah, %cl
+; i686-NEXT:    shll %cl, %ebx
+; i686-NEXT:    testb $32, %ah
+; i686-NEXT:    movl $0, %ecx
+; i686-NEXT:    jne .LBB2_9
+; i686-NEXT:  # %bb.8: # %entry
+; i686-NEXT:    movl %ebx, %ecx
+; i686-NEXT:  .LBB2_9: # %entry
+; i686-NEXT:    cmpb $64, %al
+; i686-NEXT:    jb .LBB2_10
+; i686-NEXT:  # %bb.11: # %entry
+; i686-NEXT:    movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; i686-NEXT:    jmp .LBB2_12
+; i686-NEXT:  .LBB2_10:
+; i686-NEXT:    movl (%esp), %ecx # 4-byte Reload
+; i686-NEXT:    orl %ebp, %ecx
+; i686-NEXT:  .LBB2_12: # %entry
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; i686-NEXT:    movl %ecx, (%esp) # 4-byte Spill
+; i686-NEXT:    testb $32, %dl
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; i686-NEXT:    jne .LBB2_14
+; i686-NEXT:  # %bb.13: # %entry
+; i686-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:  .LBB2_14: # %entry
+; i686-NEXT:    movl %edx, %esi
+; i686-NEXT:    movl %eax, %ecx
+; i686-NEXT:    shldl %cl, %ebp, %esi
+; i686-NEXT:    testb $32, %al
+; i686-NEXT:    jne .LBB2_16
+; i686-NEXT:  # %bb.15: # %entry
+; i686-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:  .LBB2_16: # %entry
+; i686-NEXT:    movb %ah, %cl
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; i686-NEXT:    shldl %cl, %esi, %edi
+; i686-NEXT:    testb $32, %ah
+; i686-NEXT:    jne .LBB2_18
+; i686-NEXT:  # %bb.17: # %entry
+; i686-NEXT:    movl %edi, %ebx
+; i686-NEXT:  .LBB2_18: # %entry
+; i686-NEXT:    cmpb $64, %al
+; i686-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; i686-NEXT:    jae .LBB2_20
+; i686-NEXT:  # %bb.19:
+; i686-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; i686-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; i686-NEXT:  .LBB2_20: # %entry
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; i686-NEXT:    testb %al, %al
+; i686-NEXT:    je .LBB2_22
+; i686-NEXT:  # %bb.21: # %entry
+; i686-NEXT:    movl %ebx, %edx
+; i686-NEXT:    movl (%esp), %ebp # 4-byte Reload
+; i686-NEXT:  .LBB2_22: # %entry
+; i686-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; i686-NEXT:    movl %eax, 4(%ecx)
+; i686-NEXT:    movl %esi, (%ecx)
+; i686-NEXT:    movl %edx, 12(%ecx)
+; i686-NEXT:    movl %ebp, 8(%ecx)
+; i686-NEXT:    addl $20, %esp
+; i686-NEXT:    popl %esi
+; i686-NEXT:    popl %edi
+; i686-NEXT:    popl %ebx
+; i686-NEXT:    popl %ebp
+; i686-NEXT:    retl
+;
+; x86_64-LABEL: test_shl_i128:
+; x86_64:       # %bb.0: # %entry
+; x86_64-NEXT:    movq %rdx, %rcx
+; x86_64-NEXT:    shldq %cl, %rdi, %rsi
+; x86_64-NEXT:    shlq %cl, %rdi
+; x86_64-NEXT:    xorl %eax, %eax
+; x86_64-NEXT:    testb $64, %cl
+; x86_64-NEXT:    cmovneq %rdi, %rsi
+; x86_64-NEXT:    cmoveq %rdi, %rax
+; x86_64-NEXT:    movq %rsi, 8(%r8)
+; x86_64-NEXT:    movq %rax, (%r8)
+; x86_64-NEXT:    retq
 entry:
 	%0 = shl i128 %x, %a
 	store i128 %0, i128* %r, align 16
@@ -27,6 +424,9 @@ entry:
 }
 
 define void @test_lshr_i128_outofrange(i128 %x, i128* nocapture %r) nounwind {
+; ALL-LABEL: test_lshr_i128_outofrange:
+; ALL:       # %bb.0: # %entry
+; ALL-NEXT:    ret{{[l|q]}}
 entry:
 	%0 = lshr i128 %x, -1
 	store i128 %0, i128* %r, align 16
@@ -34,6 +434,9 @@ entry:
 }
 
 define void @test_ashr_i128_outofrange(i128 %x, i128* nocapture %r) nounwind {
+; ALL-LABEL: test_ashr_i128_outofrange:
+; ALL:       # %bb.0: # %entry
+; ALL-NEXT:    ret{{[l|q]}}
 entry:
 	%0 = ashr i128 %x, -1
 	store i128 %0, i128* %r, align 16
@@ -41,6 +444,9 @@ entry:
 }
 
 define void @test_shl_i128_outofrange(i128 %x, i128* nocapture %r) nounwind {
+; ALL-LABEL: test_shl_i128_outofrange:
+; ALL:       # %bb.0: # %entry
+; ALL-NEXT:    ret{{[l|q]}}
 entry:
 	%0 = shl i128 %x, -1
 	store i128 %0, i128* %r, align 16
@@ -52,6 +458,290 @@ entry:
 ;
 
 define void @test_lshr_v2i128(<2 x i128> %x, <2 x i128> %a, <2 x i128>* nocapture %r) nounwind {
+; i686-LABEL: test_lshr_v2i128:
+; i686:       # %bb.0: # %entry
+; i686-NEXT:    pushl %ebp
+; i686-NEXT:    pushl %ebx
+; i686-NEXT:    pushl %edi
+; i686-NEXT:    pushl %esi
+; i686-NEXT:    subl $68, %esp
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; i686-NEXT:    movl %ebx, %edi
+; i686-NEXT:    movl %eax, %ecx
+; i686-NEXT:    shrl %cl, %edi
+; i686-NEXT:    movl %esi, %ebp
+; i686-NEXT:    shrl %cl, %ebp
+; i686-NEXT:    shrdl %cl, %esi, %edx
+; i686-NEXT:    testb $32, %al
+; i686-NEXT:    jne .LBB6_1
+; i686-NEXT:  # %bb.2: # %entry
+; i686-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    jmp .LBB6_3
+; i686-NEXT:  .LBB6_1:
+; i686-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; i686-NEXT:    movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; i686-NEXT:  .LBB6_3: # %entry
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; i686-NEXT:    movl %eax, %ecx
+; i686-NEXT:    shrdl %cl, %ebx, %esi
+; i686-NEXT:    testb $32, %al
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; i686-NEXT:    jne .LBB6_5
+; i686-NEXT:  # %bb.4: # %entry
+; i686-NEXT:    movl %esi, %edi
+; i686-NEXT:  .LBB6_5: # %entry
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; i686-NEXT:    movl %edx, %ecx
+; i686-NEXT:    shrl %cl, %ebx
+; i686-NEXT:    shrl %cl, %ebp
+; i686-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    movl %edx, %ecx
+; i686-NEXT:    subl $64, %ecx
+; i686-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; i686-NEXT:    sbbl $0, %ecx
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; i686-NEXT:    sbbl $0, %ecx
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; i686-NEXT:    sbbl $0, %ecx
+; i686-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; i686-NEXT:    testb $32, %dl
+; i686-NEXT:    movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; i686-NEXT:    movl $0, %ecx
+; i686-NEXT:    jne .LBB6_7
+; i686-NEXT:  # %bb.6: # %entry
+; i686-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; i686-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    movl %ebx, %ecx
+; i686-NEXT:  .LBB6_7: # %entry
+; i686-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    movl %edx, %ecx
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; i686-NEXT:    shrdl %cl, %ebp, %esi
+; i686-NEXT:    testb $32, %dl
+; i686-NEXT:    jne .LBB6_9
+; i686-NEXT:  # %bb.8: # %entry
+; i686-NEXT:    movl %esi, %ebx
+; i686-NEXT:  .LBB6_9: # %entry
+; i686-NEXT:    movl %edi, %esi
+; i686-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; i686-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; i686-NEXT:    shrl %cl, %ebp
+; i686-NEXT:    testb $32, %cl
+; i686-NEXT:    movl $0, %ecx
+; i686-NEXT:    jne .LBB6_11
+; i686-NEXT:  # %bb.10: # %entry
+; i686-NEXT:    movl %ebp, %ecx
+; i686-NEXT:  .LBB6_11: # %entry
+; i686-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    movb $64, %cl
+; i686-NEXT:    subb %dl, %cl
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; i686-NEXT:    shldl %cl, %ebx, %edi
+; i686-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    movl %ebx, %edi
+; i686-NEXT:    shll %cl, %edi
+; i686-NEXT:    testb $32, %cl
+; i686-NEXT:    movb $64, %bl
+; i686-NEXT:    jne .LBB6_12
+; i686-NEXT:  # %bb.13: # %entry
+; i686-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    jmp .LBB6_14
+; i686-NEXT:  .LBB6_12:
+; i686-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; i686-NEXT:  .LBB6_14: # %entry
+; i686-NEXT:    movl %esi, %edi
+; i686-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; i686-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; i686-NEXT:    movl %edx, %ecx
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; i686-NEXT:    shrdl %cl, %ebp, %esi
+; i686-NEXT:    testb $32, %dl
+; i686-NEXT:    jne .LBB6_16
+; i686-NEXT:  # %bb.15: # %entry
+; i686-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:  .LBB6_16: # %entry
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; i686-NEXT:    subb %al, %bl
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; i686-NEXT:    movl %ebx, %ecx
+; i686-NEXT:    shll %cl, %ebp
+; i686-NEXT:    testb $32, %bl
+; i686-NEXT:    movl $0, %ecx
+; i686-NEXT:    jne .LBB6_18
+; i686-NEXT:  # %bb.17: # %entry
+; i686-NEXT:    movl %ebp, %ecx
+; i686-NEXT:  .LBB6_18: # %entry
+; i686-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    movl %eax, %ecx
+; i686-NEXT:    subl $64, %ecx
+; i686-NEXT:    sbbl $0, %esi
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; i686-NEXT:    sbbl $0, %esi
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; i686-NEXT:    sbbl $0, %esi
+; i686-NEXT:    setae %bh
+; i686-NEXT:    jb .LBB6_20
+; i686-NEXT:  # %bb.19: # %entry
+; i686-NEXT:    xorl %edi, %edi
+; i686-NEXT:    movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; i686-NEXT:  .LBB6_20: # %entry
+; i686-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; i686-NEXT:    shrdl %cl, %esi, %edi
+; i686-NEXT:    shrl %cl, %esi
+; i686-NEXT:    testb $32, %cl
+; i686-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    jne .LBB6_22
+; i686-NEXT:  # %bb.21: # %entry
+; i686-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:  .LBB6_22: # %entry
+; i686-NEXT:    testb %bh, %bh
+; i686-NEXT:    jne .LBB6_24
+; i686-NEXT:  # %bb.23:
+; i686-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; i686-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; i686-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:  .LBB6_24: # %entry
+; i686-NEXT:    testb $32, %cl
+; i686-NEXT:    movl $0, %ecx
+; i686-NEXT:    jne .LBB6_26
+; i686-NEXT:  # %bb.25: # %entry
+; i686-NEXT:    movl %esi, %ecx
+; i686-NEXT:  .LBB6_26: # %entry
+; i686-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    movl %ebx, %ecx
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; i686-NEXT:    shldl %cl, %edi, %esi
+; i686-NEXT:    testb $32, %bl
+; i686-NEXT:    jne .LBB6_28
+; i686-NEXT:  # %bb.27: # %entry
+; i686-NEXT:    movl %esi, %ebp
+; i686-NEXT:  .LBB6_28: # %entry
+; i686-NEXT:    testb %bh, %bh
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; i686-NEXT:    jne .LBB6_30
+; i686-NEXT:  # %bb.29:
+; i686-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; i686-NEXT:    orl %ebp, %ecx
+; i686-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:  .LBB6_30: # %entry
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; i686-NEXT:    cmpb $0, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Reload
+; i686-NEXT:    jne .LBB6_32
+; i686-NEXT:  # %bb.31: # %entry
+; i686-NEXT:    movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; i686-NEXT:    movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; i686-NEXT:  .LBB6_32: # %entry
+; i686-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; i686-NEXT:    shrdl %cl, %ebp, %edi
+; i686-NEXT:    movl %edi, %ebp
+; i686-NEXT:    testb $32, %cl
+; i686-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; i686-NEXT:    je .LBB6_33
+; i686-NEXT:  # %bb.34: # %entry
+; i686-NEXT:    cmpb $0, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Reload
+; i686-NEXT:    jne .LBB6_35
+; i686-NEXT:  .LBB6_36: # %entry
+; i686-NEXT:    cmpb $0, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Reload
+; i686-NEXT:    je .LBB6_38
+; i686-NEXT:  .LBB6_37:
+; i686-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; i686-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; i686-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:  .LBB6_38: # %entry
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; i686-NEXT:    orl {{[0-9]+}}(%esp), %ecx
+; i686-NEXT:    orl {{[0-9]+}}(%esp), %edx
+; i686-NEXT:    orl %ecx, %edx
+; i686-NEXT:    je .LBB6_40
+; i686-NEXT:  # %bb.39: # %entry
+; i686-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; i686-NEXT:  .LBB6_40: # %entry
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; i686-NEXT:    orl {{[0-9]+}}(%esp), %edx
+; i686-NEXT:    orl {{[0-9]+}}(%esp), %eax
+; i686-NEXT:    orl %edx, %eax
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; i686-NEXT:    je .LBB6_42
+; i686-NEXT:  # %bb.41: # %entry
+; i686-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; i686-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; i686-NEXT:  .LBB6_42: # %entry
+; i686-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; i686-NEXT:    movl %edx, 28(%ecx)
+; i686-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; i686-NEXT:    movl %edx, 24(%ecx)
+; i686-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; i686-NEXT:    movl %edx, 12(%ecx)
+; i686-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; i686-NEXT:    movl %edx, 8(%ecx)
+; i686-NEXT:    movl %esi, 20(%ecx)
+; i686-NEXT:    movl %eax, 16(%ecx)
+; i686-NEXT:    movl %ebx, 4(%ecx)
+; i686-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; i686-NEXT:    movl %eax, (%ecx)
+; i686-NEXT:    addl $68, %esp
+; i686-NEXT:    popl %esi
+; i686-NEXT:    popl %edi
+; i686-NEXT:    popl %ebx
+; i686-NEXT:    popl %ebp
+; i686-NEXT:    retl
+; i686-NEXT:  .LBB6_33: # %entry
+; i686-NEXT:    movl %ebp, %edi
+; i686-NEXT:    cmpb $0, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Reload
+; i686-NEXT:    je .LBB6_36
+; i686-NEXT:  .LBB6_35:
+; i686-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; i686-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; i686-NEXT:    movl %ecx, %edi
+; i686-NEXT:    cmpb $0, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Reload
+; i686-NEXT:    jne .LBB6_37
+; i686-NEXT:    jmp .LBB6_38
+;
+; x86_64-LABEL: test_lshr_v2i128:
+; x86_64:       # %bb.0: # %entry
+; x86_64-NEXT:    movq %rcx, %rax
+; x86_64-NEXT:    movq {{[0-9]+}}(%rsp), %r10
+; x86_64-NEXT:    movb {{[0-9]+}}(%rsp), %r9b
+; x86_64-NEXT:    movl %r9d, %ecx
+; x86_64-NEXT:    shrdq %cl, %rax, %rdx
+; x86_64-NEXT:    movl %r8d, %ecx
+; x86_64-NEXT:    shrdq %cl, %rsi, %rdi
+; x86_64-NEXT:    shrq %cl, %rsi
+; x86_64-NEXT:    xorl %r11d, %r11d
+; x86_64-NEXT:    testb $64, %r8b
+; x86_64-NEXT:    cmovneq %rsi, %rdi
+; x86_64-NEXT:    cmovneq %r11, %rsi
+; x86_64-NEXT:    movl %r9d, %ecx
+; x86_64-NEXT:    shrq %cl, %rax
+; x86_64-NEXT:    testb $64, %r9b
+; x86_64-NEXT:    cmovneq %rax, %rdx
+; x86_64-NEXT:    cmovneq %r11, %rax
+; x86_64-NEXT:    movq %rax, 24(%r10)
+; x86_64-NEXT:    movq %rdx, 16(%r10)
+; x86_64-NEXT:    movq %rsi, 8(%r10)
+; x86_64-NEXT:    movq %rdi, (%r10)
+; x86_64-NEXT:    retq
 entry:
 	%0 = lshr <2 x i128> %x, %a
 	store <2 x i128> %0, <2 x i128>* %r, align 16
@@ -59,6 +749,296 @@ entry:
 }
 
 define void @test_ashr_v2i128(<2 x i128> %x, <2 x i128> %a, <2 x i128>* nocapture %r) nounwind {
+; i686-LABEL: test_ashr_v2i128:
+; i686:       # %bb.0: # %entry
+; i686-NEXT:    pushl %ebp
+; i686-NEXT:    pushl %ebx
+; i686-NEXT:    pushl %edi
+; i686-NEXT:    pushl %esi
+; i686-NEXT:    subl $80, %esp
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; i686-NEXT:    movl %ebp, %ebx
+; i686-NEXT:    movl %eax, %ecx
+; i686-NEXT:    sarl %cl, %ebx
+; i686-NEXT:    movl %esi, %edi
+; i686-NEXT:    shrl %cl, %edi
+; i686-NEXT:    shrdl %cl, %esi, %edx
+; i686-NEXT:    sarl $31, %ebp
+; i686-NEXT:    movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; i686-NEXT:    testb $32, %al
+; i686-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    jne .LBB7_1
+; i686-NEXT:  # %bb.2: # %entry
+; i686-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    jmp .LBB7_3
+; i686-NEXT:  .LBB7_1:
+; i686-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; i686-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:  .LBB7_3: # %entry
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; i686-NEXT:    movl %eax, %ecx
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; i686-NEXT:    shrdl %cl, %edx, %edi
+; i686-NEXT:    testb $32, %al
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; i686-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; i686-NEXT:    jne .LBB7_5
+; i686-NEXT:  # %bb.4: # %entry
+; i686-NEXT:    movl %edi, %ebx
+; i686-NEXT:  .LBB7_5: # %entry
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; i686-NEXT:    movl %ebp, %edi
+; i686-NEXT:    movl %edx, %ecx
+; i686-NEXT:    sarl %cl, %edi
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; i686-NEXT:    shrl %cl, %esi
+; i686-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    sarl $31, %ebp
+; i686-NEXT:    movl %edx, %ecx
+; i686-NEXT:    subl $64, %ecx
+; i686-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; i686-NEXT:    sbbl $0, %ecx
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; i686-NEXT:    sbbl $0, %ecx
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; i686-NEXT:    sbbl $0, %ecx
+; i686-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; i686-NEXT:    testb $32, %dl
+; i686-NEXT:    movl $0, %esi
+; i686-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    movl %ebp, %ecx
+; i686-NEXT:    jne .LBB7_7
+; i686-NEXT:  # %bb.6: # %entry
+; i686-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; i686-NEXT:    movl %edi, %ecx
+; i686-NEXT:  .LBB7_7: # %entry
+; i686-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; i686-NEXT:    movl %edx, %ecx
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; i686-NEXT:    shrdl %cl, %ebp, %esi
+; i686-NEXT:    testb $32, %dl
+; i686-NEXT:    jne .LBB7_9
+; i686-NEXT:  # %bb.8: # %entry
+; i686-NEXT:    movl %esi, %edi
+; i686-NEXT:  .LBB7_9: # %entry
+; i686-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; i686-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; i686-NEXT:    sarl %cl, %esi
+; i686-NEXT:    testb $32, %cl
+; i686-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; i686-NEXT:    jne .LBB7_11
+; i686-NEXT:  # %bb.10: # %entry
+; i686-NEXT:    movl %esi, %ecx
+; i686-NEXT:  .LBB7_11: # %entry
+; i686-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; i686-NEXT:    movb $64, %cl
+; i686-NEXT:    subb %dl, %cl
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; i686-NEXT:    shldl %cl, %ebx, %ebp
+; i686-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    movl %ebx, %ebp
+; i686-NEXT:    shll %cl, %ebp
+; i686-NEXT:    testb $32, %cl
+; i686-NEXT:    movb $64, %bl
+; i686-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    je .LBB7_13
+; i686-NEXT:  # %bb.12:
+; i686-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    xorl %ebp, %ebp
+; i686-NEXT:  .LBB7_13: # %entry
+; i686-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    movl %edx, %ecx
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; i686-NEXT:    shrdl %cl, %edi, %esi
+; i686-NEXT:    testb $32, %dl
+; i686-NEXT:    jne .LBB7_15
+; i686-NEXT:  # %bb.14: # %entry
+; i686-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:  .LBB7_15: # %entry
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; i686-NEXT:    subb %al, %bl
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; i686-NEXT:    movl %ebx, %ecx
+; i686-NEXT:    shll %cl, %ebp
+; i686-NEXT:    testb $32, %bl
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; i686-NEXT:    jne .LBB7_17
+; i686-NEXT:  # %bb.16: # %entry
+; i686-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:  .LBB7_17: # %entry
+; i686-NEXT:    movl %eax, %ecx
+; i686-NEXT:    subl $64, %ecx
+; i686-NEXT:    sbbl $0, %esi
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; i686-NEXT:    sbbl $0, %esi
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; i686-NEXT:    sbbl $0, %esi
+; i686-NEXT:    setae %bh
+; i686-NEXT:    jb .LBB7_19
+; i686-NEXT:  # %bb.18: # %entry
+; i686-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; i686-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:  .LBB7_19: # %entry
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; i686-NEXT:    shrdl %cl, %edi, %esi
+; i686-NEXT:    sarl %cl, %edi
+; i686-NEXT:    testb $32, %cl
+; i686-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    je .LBB7_20
+; i686-NEXT:  # %bb.21: # %entry
+; i686-NEXT:    testb %bh, %bh
+; i686-NEXT:    je .LBB7_22
+; i686-NEXT:  .LBB7_23: # %entry
+; i686-NEXT:    testb $32, %cl
+; i686-NEXT:    jne .LBB7_25
+; i686-NEXT:  .LBB7_24: # %entry
+; i686-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:  .LBB7_25: # %entry
+; i686-NEXT:    movl %ebx, %ecx
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; i686-NEXT:    shldl %cl, %esi, %edi
+; i686-NEXT:    testb $32, %bl
+; i686-NEXT:    jne .LBB7_27
+; i686-NEXT:  # %bb.26: # %entry
+; i686-NEXT:    movl %edi, %ebp
+; i686-NEXT:  .LBB7_27: # %entry
+; i686-NEXT:    testb %bh, %bh
+; i686-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; i686-NEXT:    jne .LBB7_29
+; i686-NEXT:  # %bb.28:
+; i686-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; i686-NEXT:    orl %ebp, %ebx
+; i686-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:  .LBB7_29: # %entry
+; i686-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; i686-NEXT:    cmpb $0, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Reload
+; i686-NEXT:    jne .LBB7_31
+; i686-NEXT:  # %bb.30: # %entry
+; i686-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; i686-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:  .LBB7_31: # %entry
+; i686-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; i686-NEXT:    shrdl %cl, %ebp, %ebx
+; i686-NEXT:    testb $32, %cl
+; i686-NEXT:    jne .LBB7_33
+; i686-NEXT:  # %bb.32: # %entry
+; i686-NEXT:    movl %ebx, %esi
+; i686-NEXT:  .LBB7_33: # %entry
+; i686-NEXT:    cmpb $0, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Reload
+; i686-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; i686-NEXT:    je .LBB7_35
+; i686-NEXT:  # %bb.34:
+; i686-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; i686-NEXT:    orl %ebx, %ecx
+; i686-NEXT:    movl %ecx, %esi
+; i686-NEXT:  .LBB7_35: # %entry
+; i686-NEXT:    cmpb $0, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Reload
+; i686-NEXT:    je .LBB7_37
+; i686-NEXT:  # %bb.36:
+; i686-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; i686-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; i686-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:  .LBB7_37: # %entry
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; i686-NEXT:    orl {{[0-9]+}}(%esp), %ecx
+; i686-NEXT:    orl {{[0-9]+}}(%esp), %edx
+; i686-NEXT:    orl %ecx, %edx
+; i686-NEXT:    je .LBB7_39
+; i686-NEXT:  # %bb.38: # %entry
+; i686-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; i686-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:  .LBB7_39: # %entry
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; i686-NEXT:    orl {{[0-9]+}}(%esp), %edx
+; i686-NEXT:    orl {{[0-9]+}}(%esp), %eax
+; i686-NEXT:    orl %edx, %eax
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; i686-NEXT:    je .LBB7_41
+; i686-NEXT:  # %bb.40: # %entry
+; i686-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; i686-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; i686-NEXT:  .LBB7_41: # %entry
+; i686-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; i686-NEXT:    movl %edx, 28(%ecx)
+; i686-NEXT:    movl %edi, 24(%ecx)
+; i686-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; i686-NEXT:    movl %edx, 12(%ecx)
+; i686-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; i686-NEXT:    movl %edx, 8(%ecx)
+; i686-NEXT:    movl %esi, 20(%ecx)
+; i686-NEXT:    movl %eax, 16(%ecx)
+; i686-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; i686-NEXT:    movl %eax, 4(%ecx)
+; i686-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; i686-NEXT:    movl %eax, (%ecx)
+; i686-NEXT:    addl $80, %esp
+; i686-NEXT:    popl %esi
+; i686-NEXT:    popl %edi
+; i686-NEXT:    popl %ebx
+; i686-NEXT:    popl %ebp
+; i686-NEXT:    retl
+; i686-NEXT:  .LBB7_20: # %entry
+; i686-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    testb %bh, %bh
+; i686-NEXT:    jne .LBB7_23
+; i686-NEXT:  .LBB7_22:
+; i686-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; i686-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; i686-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    testb $32, %cl
+; i686-NEXT:    je .LBB7_24
+; i686-NEXT:    jmp .LBB7_25
+;
+; x86_64-LABEL: test_ashr_v2i128:
+; x86_64:       # %bb.0: # %entry
+; x86_64-NEXT:    movq %rcx, %r11
+; x86_64-NEXT:    movq {{[0-9]+}}(%rsp), %r10
+; x86_64-NEXT:    movb {{[0-9]+}}(%rsp), %r9b
+; x86_64-NEXT:    movl %r9d, %ecx
+; x86_64-NEXT:    shrdq %cl, %r11, %rdx
+; x86_64-NEXT:    movl %r8d, %ecx
+; x86_64-NEXT:    shrdq %cl, %rsi, %rdi
+; x86_64-NEXT:    movq %rsi, %rax
+; x86_64-NEXT:    sarq %cl, %rax
+; x86_64-NEXT:    sarq $63, %rsi
+; x86_64-NEXT:    testb $64, %r8b
+; x86_64-NEXT:    cmovneq %rax, %rdi
+; x86_64-NEXT:    cmoveq %rax, %rsi
+; x86_64-NEXT:    movq %r11, %rax
+; x86_64-NEXT:    movl %r9d, %ecx
+; x86_64-NEXT:    sarq %cl, %rax
+; x86_64-NEXT:    sarq $63, %r11
+; x86_64-NEXT:    testb $64, %r9b
+; x86_64-NEXT:    cmovneq %rax, %rdx
+; x86_64-NEXT:    cmoveq %rax, %r11
+; x86_64-NEXT:    movq %r11, 24(%r10)
+; x86_64-NEXT:    movq %rdx, 16(%r10)
+; x86_64-NEXT:    movq %rsi, 8(%r10)
+; x86_64-NEXT:    movq %rdi, (%r10)
+; x86_64-NEXT:    retq
 entry:
 	%0 = ashr <2 x i128> %x, %a
 	store <2 x i128> %0, <2 x i128>* %r, align 16
@@ -66,6 +1046,307 @@ entry:
 }
 
 define void @test_shl_v2i128(<2 x i128> %x, <2 x i128> %a, <2 x i128>* nocapture %r) nounwind {
+; i686-LABEL: test_shl_v2i128:
+; i686:       # %bb.0: # %entry
+; i686-NEXT:    pushl %ebp
+; i686-NEXT:    pushl %ebx
+; i686-NEXT:    pushl %edi
+; i686-NEXT:    pushl %esi
+; i686-NEXT:    subl $72, %esp
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; i686-NEXT:    movl %ebx, %ecx
+; i686-NEXT:    shll %cl, %ebp
+; i686-NEXT:    shll %cl, %esi
+; i686-NEXT:    movl %edx, %eax
+; i686-NEXT:    subl $64, %eax
+; i686-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; i686-NEXT:    sbbl $0, %eax
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; i686-NEXT:    sbbl $0, %eax
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; i686-NEXT:    sbbl $0, %eax
+; i686-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; i686-NEXT:    movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; i686-NEXT:    testb $32, %bl
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; i686-NEXT:    movl $0, %eax
+; i686-NEXT:    movl $0, %ecx
+; i686-NEXT:    jne .LBB8_2
+; i686-NEXT:  # %bb.1: # %entry
+; i686-NEXT:    movl %esi, %eax
+; i686-NEXT:    movl %ebp, %ecx
+; i686-NEXT:  .LBB8_2: # %entry
+; i686-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    movl %edi, %eax
+; i686-NEXT:    movl %ebx, %ecx
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; i686-NEXT:    shldl %cl, %edi, %eax
+; i686-NEXT:    testb $32, %bl
+; i686-NEXT:    jne .LBB8_4
+; i686-NEXT:  # %bb.3: # %entry
+; i686-NEXT:    movl %eax, %esi
+; i686-NEXT:  .LBB8_4: # %entry
+; i686-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    movb $64, %cl
+; i686-NEXT:    subb %bl, %cl
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; i686-NEXT:    movl %edi, %esi
+; i686-NEXT:    shrl %cl, %esi
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; i686-NEXT:    shrdl %cl, %edi, %eax
+; i686-NEXT:    testb $32, %cl
+; i686-NEXT:    jne .LBB8_5
+; i686-NEXT:  # %bb.6: # %entry
+; i686-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    jmp .LBB8_7
+; i686-NEXT:  .LBB8_5:
+; i686-NEXT:    movl %esi, %eax
+; i686-NEXT:    movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; i686-NEXT:  .LBB8_7: # %entry
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; i686-NEXT:    movl %ebx, %ecx
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; i686-NEXT:    shldl %cl, %esi, %edi
+; i686-NEXT:    testb $32, %bl
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; i686-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; i686-NEXT:    jne .LBB8_9
+; i686-NEXT:  # %bb.8: # %entry
+; i686-NEXT:    movl %edi, %ebp
+; i686-NEXT:  .LBB8_9: # %entry
+; i686-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    movl %ecx, %ebp
+; i686-NEXT:    movl %edx, %ecx
+; i686-NEXT:    shll %cl, %ebp
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; i686-NEXT:    shll %cl, %esi
+; i686-NEXT:    testb $32, %dl
+; i686-NEXT:    movl $0, %edi
+; i686-NEXT:    movl $0, %ecx
+; i686-NEXT:    jne .LBB8_11
+; i686-NEXT:  # %bb.10: # %entry
+; i686-NEXT:    movl %esi, %edi
+; i686-NEXT:    movl %ebp, %ecx
+; i686-NEXT:  .LBB8_11: # %entry
+; i686-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; i686-NEXT:    movl %edx, %ecx
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; i686-NEXT:    shldl %cl, %ebx, %edi
+; i686-NEXT:    testb $32, %dl
+; i686-NEXT:    jne .LBB8_13
+; i686-NEXT:  # %bb.12: # %entry
+; i686-NEXT:    movl %edi, %ebp
+; i686-NEXT:  .LBB8_13: # %entry
+; i686-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; i686-NEXT:    movb $64, %cl
+; i686-NEXT:    subb %dl, %cl
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; i686-NEXT:    shrl %cl, %ebx
+; i686-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; i686-NEXT:    testb $32, %cl
+; i686-NEXT:    movl $0, %ecx
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; i686-NEXT:    jne .LBB8_15
+; i686-NEXT:  # %bb.14: # %entry
+; i686-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; i686-NEXT:  .LBB8_15: # %entry
+; i686-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    movl %edx, %ecx
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; i686-NEXT:    shldl %cl, %ebp, %edi
+; i686-NEXT:    testb $32, %dl
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; i686-NEXT:    jne .LBB8_17
+; i686-NEXT:  # %bb.16: # %entry
+; i686-NEXT:    movl %edi, %esi
+; i686-NEXT:  .LBB8_17: # %entry
+; i686-NEXT:    orl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; i686-NEXT:    movl %ebx, %eax
+; i686-NEXT:    subl $64, %eax
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; i686-NEXT:    sbbl $0, %ecx
+; i686-NEXT:    movl %ebp, %ecx
+; i686-NEXT:    sbbl $0, %ecx
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; i686-NEXT:    sbbl $0, %ecx
+; i686-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; i686-NEXT:    jb .LBB8_19
+; i686-NEXT:  # %bb.18: # %entry
+; i686-NEXT:    movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; i686-NEXT:  .LBB8_19: # %entry
+; i686-NEXT:    jb .LBB8_21
+; i686-NEXT:  # %bb.20: # %entry
+; i686-NEXT:    movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; i686-NEXT:  .LBB8_21: # %entry
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; i686-NEXT:    movl %ebp, %ebx
+; i686-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; i686-NEXT:    shll %cl, %ebx
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; i686-NEXT:    shldl %cl, %ebp, %edi
+; i686-NEXT:    testb $32, %cl
+; i686-NEXT:    movl %ebx, %ecx
+; i686-NEXT:    jne .LBB8_23
+; i686-NEXT:  # %bb.22: # %entry
+; i686-NEXT:    movl %edi, %ecx
+; i686-NEXT:  .LBB8_23: # %entry
+; i686-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; i686-NEXT:    movl %eax, %ecx
+; i686-NEXT:    shll %cl, %edi
+; i686-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    testb $32, %al
+; i686-NEXT:    movl $0, %edi
+; i686-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; i686-NEXT:    jne .LBB8_25
+; i686-NEXT:  # %bb.24: # %entry
+; i686-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; i686-NEXT:  .LBB8_25: # %entry
+; i686-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    cmpb $0, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Reload
+; i686-NEXT:    jne .LBB8_27
+; i686-NEXT:  # %bb.26: # %entry
+; i686-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:  .LBB8_27: # %entry
+; i686-NEXT:    movl %eax, %ecx
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; i686-NEXT:    shldl %cl, %edi, %esi
+; i686-NEXT:    testb $32, %al
+; i686-NEXT:    jne .LBB8_29
+; i686-NEXT:  # %bb.28: # %entry
+; i686-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:  .LBB8_29: # %entry
+; i686-NEXT:    cmpb $0, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Reload
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; i686-NEXT:    movb {{[-0-9]+}}(%e{{[sb]}}p), %al # 1-byte Reload
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; i686-NEXT:    jne .LBB8_30
+; i686-NEXT:  # %bb.31: # %entry
+; i686-NEXT:    testb %al, %al
+; i686-NEXT:    je .LBB8_32
+; i686-NEXT:  .LBB8_33: # %entry
+; i686-NEXT:    testb $32, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Reload
+; i686-NEXT:    jne .LBB8_35
+; i686-NEXT:  .LBB8_34: # %entry
+; i686-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:  .LBB8_35: # %entry
+; i686-NEXT:    movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; i686-NEXT:    shrdl %cl, %ebx, %esi
+; i686-NEXT:    testb $32, %cl
+; i686-NEXT:    jne .LBB8_37
+; i686-NEXT:  # %bb.36: # %entry
+; i686-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:  .LBB8_37: # %entry
+; i686-NEXT:    testb %al, %al
+; i686-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; i686-NEXT:    jne .LBB8_38
+; i686-NEXT:  # %bb.39: # %entry
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; i686-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; i686-NEXT:    testb %al, %al
+; i686-NEXT:    jne .LBB8_41
+; i686-NEXT:    jmp .LBB8_42
+; i686-NEXT:  .LBB8_30:
+; i686-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; i686-NEXT:    orl %ebp, %ecx
+; i686-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    testb %al, %al
+; i686-NEXT:    jne .LBB8_33
+; i686-NEXT:  .LBB8_32: # %entry
+; i686-NEXT:    movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; i686-NEXT:    movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; i686-NEXT:    testb $32, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Reload
+; i686-NEXT:    je .LBB8_34
+; i686-NEXT:    jmp .LBB8_35
+; i686-NEXT:  .LBB8_38:
+; i686-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; i686-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; i686-NEXT:    testb %al, %al
+; i686-NEXT:    je .LBB8_42
+; i686-NEXT:  .LBB8_41:
+; i686-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; i686-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; i686-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:  .LBB8_42: # %entry
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; i686-NEXT:    orl {{[0-9]+}}(%esp), %eax
+; i686-NEXT:    orl {{[0-9]+}}(%esp), %edx
+; i686-NEXT:    orl %eax, %edx
+; i686-NEXT:    je .LBB8_44
+; i686-NEXT:  # %bb.43: # %entry
+; i686-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; i686-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; i686-NEXT:  .LBB8_44: # %entry
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; i686-NEXT:    orl {{[0-9]+}}(%esp), %edx
+; i686-NEXT:    orl {{[0-9]+}}(%esp), %ebx
+; i686-NEXT:    orl %edx, %ebx
+; i686-NEXT:    je .LBB8_46
+; i686-NEXT:  # %bb.45: # %entry
+; i686-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; i686-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; i686-NEXT:  .LBB8_46: # %entry
+; i686-NEXT:    movl %esi, 20(%eax)
+; i686-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; i686-NEXT:    movl %edx, 16(%eax)
+; i686-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; i686-NEXT:    movl %edx, 4(%eax)
+; i686-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; i686-NEXT:    movl %edx, (%eax)
+; i686-NEXT:    movl %edi, 28(%eax)
+; i686-NEXT:    movl %ecx, 24(%eax)
+; i686-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; i686-NEXT:    movl %ecx, 12(%eax)
+; i686-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; i686-NEXT:    movl %ecx, 8(%eax)
+; i686-NEXT:    addl $72, %esp
+; i686-NEXT:    popl %esi
+; i686-NEXT:    popl %edi
+; i686-NEXT:    popl %ebx
+; i686-NEXT:    popl %ebp
+; i686-NEXT:    retl
+;
+; x86_64-LABEL: test_shl_v2i128:
+; x86_64:       # %bb.0: # %entry
+; x86_64-NEXT:    movq %rcx, %rax
+; x86_64-NEXT:    movq {{[0-9]+}}(%rsp), %r10
+; x86_64-NEXT:    movb {{[0-9]+}}(%rsp), %r9b
+; x86_64-NEXT:    movl %r9d, %ecx
+; x86_64-NEXT:    shldq %cl, %rdx, %rax
+; x86_64-NEXT:    movl %r8d, %ecx
+; x86_64-NEXT:    shldq %cl, %rdi, %rsi
+; x86_64-NEXT:    shlq %cl, %rdi
+; x86_64-NEXT:    xorl %r11d, %r11d
+; x86_64-NEXT:    testb $64, %r8b
+; x86_64-NEXT:    cmovneq %rdi, %rsi
+; x86_64-NEXT:    cmovneq %r11, %rdi
+; x86_64-NEXT:    movl %r9d, %ecx
+; x86_64-NEXT:    shlq %cl, %rdx
+; x86_64-NEXT:    testb $64, %r9b
+; x86_64-NEXT:    cmovneq %rdx, %rax
+; x86_64-NEXT:    cmovneq %r11, %rdx
+; x86_64-NEXT:    movq %rax, 24(%r10)
+; x86_64-NEXT:    movq %rdx, 16(%r10)
+; x86_64-NEXT:    movq %rsi, 8(%r10)
+; x86_64-NEXT:    movq %rdi, (%r10)
+; x86_64-NEXT:    retq
 entry:
 	%0 = shl <2 x i128> %x, %a
 	store <2 x i128> %0, <2 x i128>* %r, align 16
@@ -73,6 +1354,9 @@ entry:
 }
 
 define void @test_lshr_v2i128_outofrange(<2 x i128> %x, <2 x i128>* nocapture %r) nounwind {
+; ALL-LABEL: test_lshr_v2i128_outofrange:
+; ALL:       # %bb.0: # %entry
+; ALL-NEXT:    ret{{[l|q]}}
 entry:
 	%0 = lshr <2 x i128> %x, <i128 -1, i128 -1>
 	store <2 x i128> %0, <2 x i128>* %r, align 16
@@ -80,6 +1364,9 @@ entry:
 }
 
 define void @test_ashr_v2i128_outofrange(<2 x i128> %x, <2 x i128>* nocapture %r) nounwind {
+; ALL-LABEL: test_ashr_v2i128_outofrange:
+; ALL:       # %bb.0: # %entry
+; ALL-NEXT:    ret{{[l|q]}}
 entry:
 	%0 = ashr <2 x i128> %x, <i128 -1, i128 -1>
 	store <2 x i128> %0, <2 x i128>* %r, align 16
@@ -87,6 +1374,9 @@ entry:
 }
 
 define void @test_shl_v2i128_outofrange(<2 x i128> %x, <2 x i128>* nocapture %r) nounwind {
+; ALL-LABEL: test_shl_v2i128_outofrange:
+; ALL:       # %bb.0: # %entry
+; ALL-NEXT:    ret{{[l|q]}}
 entry:
 	%0 = shl <2 x i128> %x, <i128 -1, i128 -1>
 	store <2 x i128> %0, <2 x i128>* %r, align 16
@@ -94,6 +1384,25 @@ entry:
 }
 
 define void @test_lshr_v2i128_outofrange_sum(<2 x i128> %x, <2 x i128>* nocapture %r) nounwind {
+; i686-LABEL: test_lshr_v2i128_outofrange_sum:
+; i686:       # %bb.0: # %entry
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; i686-NEXT:    movl $0, 28(%eax)
+; i686-NEXT:    movl $0, 24(%eax)
+; i686-NEXT:    movl $0, 20(%eax)
+; i686-NEXT:    movl $0, 16(%eax)
+; i686-NEXT:    movl $0, 12(%eax)
+; i686-NEXT:    movl $0, 8(%eax)
+; i686-NEXT:    movl $0, 4(%eax)
+; i686-NEXT:    movl $0, (%eax)
+; i686-NEXT:    retl
+;
+; x86_64-LABEL: test_lshr_v2i128_outofrange_sum:
+; x86_64:       # %bb.0: # %entry
+; x86_64-NEXT:    xorps %xmm0, %xmm0
+; x86_64-NEXT:    movaps %xmm0, 16(%r8)
+; x86_64-NEXT:    movaps %xmm0, (%r8)
+; x86_64-NEXT:    retq
 entry:
 	%0 = lshr <2 x i128> %x, <i128 -1, i128 -1>
 	%1 = lshr <2 x i128> %0, <i128  1, i128  1>
@@ -102,6 +1411,25 @@ entry:
 }
 
 define void @test_ashr_v2i128_outofrange_sum(<2 x i128> %x, <2 x i128>* nocapture %r) nounwind {
+; i686-LABEL: test_ashr_v2i128_outofrange_sum:
+; i686:       # %bb.0: # %entry
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; i686-NEXT:    movl $0, 28(%eax)
+; i686-NEXT:    movl $0, 24(%eax)
+; i686-NEXT:    movl $0, 20(%eax)
+; i686-NEXT:    movl $0, 16(%eax)
+; i686-NEXT:    movl $0, 12(%eax)
+; i686-NEXT:    movl $0, 8(%eax)
+; i686-NEXT:    movl $0, 4(%eax)
+; i686-NEXT:    movl $0, (%eax)
+; i686-NEXT:    retl
+;
+; x86_64-LABEL: test_ashr_v2i128_outofrange_sum:
+; x86_64:       # %bb.0: # %entry
+; x86_64-NEXT:    xorps %xmm0, %xmm0
+; x86_64-NEXT:    movaps %xmm0, 16(%r8)
+; x86_64-NEXT:    movaps %xmm0, (%r8)
+; x86_64-NEXT:    retq
 entry:
 	%0 = ashr <2 x i128> %x, <i128 -1, i128 -1>
 	%1 = ashr <2 x i128> %0, <i128  1, i128  1>
@@ -110,6 +1438,25 @@ entry:
 }
 
 define void @test_shl_v2i128_outofrange_sum(<2 x i128> %x, <2 x i128>* nocapture %r) nounwind {
+; i686-LABEL: test_shl_v2i128_outofrange_sum:
+; i686:       # %bb.0: # %entry
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; i686-NEXT:    movl $0, 28(%eax)
+; i686-NEXT:    movl $0, 24(%eax)
+; i686-NEXT:    movl $0, 20(%eax)
+; i686-NEXT:    movl $0, 16(%eax)
+; i686-NEXT:    movl $0, 12(%eax)
+; i686-NEXT:    movl $0, 8(%eax)
+; i686-NEXT:    movl $0, 4(%eax)
+; i686-NEXT:    movl $0, (%eax)
+; i686-NEXT:    retl
+;
+; x86_64-LABEL: test_shl_v2i128_outofrange_sum:
+; x86_64:       # %bb.0: # %entry
+; x86_64-NEXT:    xorps %xmm0, %xmm0
+; x86_64-NEXT:    movaps %xmm0, 16(%r8)
+; x86_64-NEXT:    movaps %xmm0, (%r8)
+; x86_64-NEXT:    retq
 entry:
 	%0 = shl <2 x i128> %x, <i128 -1, i128 -1>
 	%1 = shl <2 x i128> %0, <i128  1, i128  1>
@@ -122,6 +1469,36 @@ entry:
 ;
 
 define <2 x i256> @shl_sext_shl_outofrange(<2 x i128> %a0) {
+; i686-LABEL: shl_sext_shl_outofrange:
+; i686:       # %bb.0:
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; i686-NEXT:    movl $0, 60(%eax)
+; i686-NEXT:    movl $0, 56(%eax)
+; i686-NEXT:    movl $0, 52(%eax)
+; i686-NEXT:    movl $0, 48(%eax)
+; i686-NEXT:    movl $0, 44(%eax)
+; i686-NEXT:    movl $0, 40(%eax)
+; i686-NEXT:    movl $0, 36(%eax)
+; i686-NEXT:    movl $0, 32(%eax)
+; i686-NEXT:    movl $0, 28(%eax)
+; i686-NEXT:    movl $0, 24(%eax)
+; i686-NEXT:    movl $0, 20(%eax)
+; i686-NEXT:    movl $0, 16(%eax)
+; i686-NEXT:    movl $0, 12(%eax)
+; i686-NEXT:    movl $0, 8(%eax)
+; i686-NEXT:    movl $0, 4(%eax)
+; i686-NEXT:    movl $0, (%eax)
+; i686-NEXT:    retl $4
+;
+; x86_64-LABEL: shl_sext_shl_outofrange:
+; x86_64:       # %bb.0:
+; x86_64-NEXT:    movq %rdi, %rax
+; x86_64-NEXT:    xorps %xmm0, %xmm0
+; x86_64-NEXT:    movaps %xmm0, 48(%rdi)
+; x86_64-NEXT:    movaps %xmm0, 32(%rdi)
+; x86_64-NEXT:    movaps %xmm0, 16(%rdi)
+; x86_64-NEXT:    movaps %xmm0, (%rdi)
+; x86_64-NEXT:    retq
   %1 = shl <2 x i128> %a0, <i128 -1, i128 -1>
   %2 = sext <2 x i128> %1 to <2 x i256>
   %3 = shl <2 x i256> %2, <i256 128, i256 128>
@@ -129,6 +1506,36 @@ define <2 x i256> @shl_sext_shl_outofrange(<2 x i128> %a0) {
 }
 
 define <2 x i256> @shl_zext_shl_outofrange(<2 x i128> %a0) {
+; i686-LABEL: shl_zext_shl_outofrange:
+; i686:       # %bb.0:
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; i686-NEXT:    movl $0, 60(%eax)
+; i686-NEXT:    movl $0, 56(%eax)
+; i686-NEXT:    movl $0, 52(%eax)
+; i686-NEXT:    movl $0, 48(%eax)
+; i686-NEXT:    movl $0, 44(%eax)
+; i686-NEXT:    movl $0, 40(%eax)
+; i686-NEXT:    movl $0, 36(%eax)
+; i686-NEXT:    movl $0, 32(%eax)
+; i686-NEXT:    movl $0, 28(%eax)
+; i686-NEXT:    movl $0, 24(%eax)
+; i686-NEXT:    movl $0, 20(%eax)
+; i686-NEXT:    movl $0, 16(%eax)
+; i686-NEXT:    movl $0, 12(%eax)
+; i686-NEXT:    movl $0, 8(%eax)
+; i686-NEXT:    movl $0, 4(%eax)
+; i686-NEXT:    movl $0, (%eax)
+; i686-NEXT:    retl $4
+;
+; x86_64-LABEL: shl_zext_shl_outofrange:
+; x86_64:       # %bb.0:
+; x86_64-NEXT:    movq %rdi, %rax
+; x86_64-NEXT:    xorps %xmm0, %xmm0
+; x86_64-NEXT:    movaps %xmm0, 48(%rdi)
+; x86_64-NEXT:    movaps %xmm0, 32(%rdi)
+; x86_64-NEXT:    movaps %xmm0, 16(%rdi)
+; x86_64-NEXT:    movaps %xmm0, (%rdi)
+; x86_64-NEXT:    retq
   %1 = shl <2 x i128> %a0, <i128 -1, i128 -1>
   %2 = zext <2 x i128> %1 to <2 x i256>
   %3 = shl <2 x i256> %2, <i256 128, i256 128>
@@ -136,6 +1543,36 @@ define <2 x i256> @shl_zext_shl_outofrange(<2 x i128> %a0) {
 }
 
 define <2 x i256> @shl_zext_lshr_outofrange(<2 x i128> %a0) {
+; i686-LABEL: shl_zext_lshr_outofrange:
+; i686:       # %bb.0:
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; i686-NEXT:    movl $0, 60(%eax)
+; i686-NEXT:    movl $0, 56(%eax)
+; i686-NEXT:    movl $0, 52(%eax)
+; i686-NEXT:    movl $0, 48(%eax)
+; i686-NEXT:    movl $0, 44(%eax)
+; i686-NEXT:    movl $0, 40(%eax)
+; i686-NEXT:    movl $0, 36(%eax)
+; i686-NEXT:    movl $0, 32(%eax)
+; i686-NEXT:    movl $0, 28(%eax)
+; i686-NEXT:    movl $0, 24(%eax)
+; i686-NEXT:    movl $0, 20(%eax)
+; i686-NEXT:    movl $0, 16(%eax)
+; i686-NEXT:    movl $0, 12(%eax)
+; i686-NEXT:    movl $0, 8(%eax)
+; i686-NEXT:    movl $0, 4(%eax)
+; i686-NEXT:    movl $0, (%eax)
+; i686-NEXT:    retl $4
+;
+; x86_64-LABEL: shl_zext_lshr_outofrange:
+; x86_64:       # %bb.0:
+; x86_64-NEXT:    movq %rdi, %rax
+; x86_64-NEXT:    xorps %xmm0, %xmm0
+; x86_64-NEXT:    movaps %xmm0, 48(%rdi)
+; x86_64-NEXT:    movaps %xmm0, 32(%rdi)
+; x86_64-NEXT:    movaps %xmm0, 16(%rdi)
+; x86_64-NEXT:    movaps %xmm0, (%rdi)
+; x86_64-NEXT:    retq
   %1 = lshr <2 x i128> %a0, <i128 -1, i128 -1>
   %2 = zext <2 x i128> %1 to <2 x i256>
   %3 = shl <2 x i256> %2, <i256 128, i256 128>
@@ -143,6 +1580,36 @@ define <2 x i256> @shl_zext_lshr_outofrange(<2 x i128> %a0) {
 }
 
 define i128 @lshr_shl_mask(i128 %a0) {
+; i686-LABEL: lshr_shl_mask:
+; i686:       # %bb.0:
+; i686-NEXT:    pushl %edi
+; i686-NEXT:    .cfi_def_cfa_offset 8
+; i686-NEXT:    pushl %esi
+; i686-NEXT:    .cfi_def_cfa_offset 12
+; i686-NEXT:    .cfi_offset %esi, -12
+; i686-NEXT:    .cfi_offset %edi, -8
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; i686-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; i686-NEXT:    movl $2147483647, %edi # imm = 0x7FFFFFFF
+; i686-NEXT:    andl {{[0-9]+}}(%esp), %edi
+; i686-NEXT:    movl %edi, 12(%eax)
+; i686-NEXT:    movl %esi, 8(%eax)
+; i686-NEXT:    movl %edx, 4(%eax)
+; i686-NEXT:    movl %ecx, (%eax)
+; i686-NEXT:    popl %esi
+; i686-NEXT:    .cfi_def_cfa_offset 8
+; i686-NEXT:    popl %edi
+; i686-NEXT:    .cfi_def_cfa_offset 4
+; i686-NEXT:    retl $4
+;
+; x86_64-LABEL: lshr_shl_mask:
+; x86_64:       # %bb.0:
+; x86_64-NEXT:    movq %rdi, %rax
+; x86_64-NEXT:    movabsq $9223372036854775807, %rdx # imm = 0x7FFFFFFFFFFFFFFF
+; x86_64-NEXT:    andq %rsi, %rdx
+; x86_64-NEXT:    retq
   %1 = shl i128 %a0, 1
   %2 = lshr i128 %1, 1
   ret i128 %2
diff --git a/llvm/test/CodeGen/X86/shift-logic.ll b/llvm/test/CodeGen/X86/shift-logic.ll
new file mode 100644
index 000000000..12e2328f7
--- /dev/null
+++ b/llvm/test/CodeGen/X86/shift-logic.ll
@@ -0,0 +1,189 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-- | FileCheck %s
+
+define i8 @shl_and(i8 %x, i8 %y) nounwind {
+; CHECK-LABEL: shl_and:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    shlb $2, %sil
+; CHECK-NEXT:    shlb $5, %al
+; CHECK-NEXT:    andb %sil, %al
+; CHECK-NEXT:    # kill: def $al killed $al killed $eax
+; CHECK-NEXT:    retq
+  %sh0 = shl i8 %x, 3
+  %r = and i8 %sh0, %y
+  %sh1 = shl i8 %r, 2
+  ret i8 %sh1
+}
+
+define i16 @shl_or(i16 %x, i16 %y) nounwind {
+; CHECK-LABEL: shl_or:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    shll $7, %esi
+; CHECK-NEXT:    shll $12, %eax
+; CHECK-NEXT:    orl %esi, %eax
+; CHECK-NEXT:    # kill: def $ax killed $ax killed $eax
+; CHECK-NEXT:    retq
+  %sh0 = shl i16 %x, 5
+  %r = or i16 %y, %sh0
+  %sh1 = shl i16 %r, 7
+  ret i16 %sh1
+}
+
+define i32 @shl_xor(i32 %x, i32 %y) nounwind {
+; CHECK-LABEL: shl_xor:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    shll $7, %esi
+; CHECK-NEXT:    shll $12, %eax
+; CHECK-NEXT:    xorl %esi, %eax
+; CHECK-NEXT:    retq
+  %sh0 = shl i32 %x, 5
+  %r = xor i32 %sh0, %y
+  %sh1 = shl i32 %r, 7
+  ret i32 %sh1
+}
+
+define i64 @lshr_and(i64 %x, i64 %y) nounwind {
+; CHECK-LABEL: lshr_and:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    shrq $7, %rsi
+; CHECK-NEXT:    shrq $12, %rax
+; CHECK-NEXT:    andq %rsi, %rax
+; CHECK-NEXT:    retq
+  %sh0 = lshr i64 %x, 5
+  %r = and i64 %y, %sh0
+  %sh1 = lshr i64 %r, 7
+  ret i64 %sh1
+}
+
+define <4 x i32> @lshr_or(<4 x i32> %x, <4 x i32> %y) nounwind {
+; CHECK-LABEL: lshr_or:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    psrld $7, %xmm1
+; CHECK-NEXT:    psrld $12, %xmm0
+; CHECK-NEXT:    por %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %sh0 = lshr <4 x i32> %x, <i32 5, i32 5, i32 5, i32 5>
+  %r = or <4 x i32> %sh0, %y
+  %sh1 = lshr <4 x i32> %r, <i32 7, i32 7, i32 7, i32 7>
+  ret <4 x i32> %sh1
+}
+
+define <8 x i16> @lshr_xor(<8 x i16> %x, <8 x i16> %y) nounwind {
+; CHECK-LABEL: lshr_xor:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    psrlw $7, %xmm1
+; CHECK-NEXT:    psrlw $12, %xmm0
+; CHECK-NEXT:    pxor %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %sh0 = lshr <8 x i16> %x, <i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5>
+  %r = xor <8 x i16> %y, %sh0
+  %sh1 = lshr <8 x i16> %r, <i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7>
+  ret <8 x i16> %sh1
+}
+
+
+define <16 x i8> @ashr_and(<16 x i8> %x, <16 x i8> %y) nounwind {
+; CHECK-LABEL: ashr_and:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    psrlw $2, %xmm1
+; CHECK-NEXT:    pand {{.*}}(%rip), %xmm1
+; CHECK-NEXT:    movdqa {{.*#+}} xmm2 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
+; CHECK-NEXT:    pxor %xmm2, %xmm1
+; CHECK-NEXT:    psubb %xmm2, %xmm1
+; CHECK-NEXT:    psrlw $5, %xmm0
+; CHECK-NEXT:    pand {{.*}}(%rip), %xmm0
+; CHECK-NEXT:    movdqa {{.*#+}} xmm2 = [4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4]
+; CHECK-NEXT:    pxor %xmm2, %xmm0
+; CHECK-NEXT:    psubb %xmm2, %xmm0
+; CHECK-NEXT:    pand %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %sh0 = ashr <16 x i8> %x, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
+  %r = and <16 x i8> %y, %sh0
+  %sh1 = ashr <16 x i8> %r, <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>
+  ret <16 x i8> %sh1
+}
+
+define <2 x i64> @ashr_or(<2 x i64> %x, <2 x i64> %y) nounwind {
+; CHECK-LABEL: ashr_or:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movdqa %xmm1, %xmm2
+; CHECK-NEXT:    psrad $7, %xmm2
+; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
+; CHECK-NEXT:    psrlq $7, %xmm1
+; CHECK-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; CHECK-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; CHECK-NEXT:    movdqa %xmm0, %xmm2
+; CHECK-NEXT:    psrad $12, %xmm2
+; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
+; CHECK-NEXT:    psrlq $12, %xmm0
+; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; CHECK-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; CHECK-NEXT:    por %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %sh0 = ashr <2 x i64> %x, <i64 5, i64 5>
+  %r = or <2 x i64> %sh0, %y
+  %sh1 = ashr <2 x i64> %r, <i64 7, i64 7>
+  ret <2 x i64> %sh1
+}
+
+define i32 @ashr_xor(i32 %x, i32 %y) nounwind {
+; CHECK-LABEL: ashr_xor:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    sarl $7, %esi
+; CHECK-NEXT:    sarl $12, %eax
+; CHECK-NEXT:    xorl %esi, %eax
+; CHECK-NEXT:    retq
+  %sh0 = ashr i32 %x, 5
+  %r = xor i32 %y, %sh0
+  %sh1 = ashr i32 %r, 7
+  ret i32 %sh1
+}
+
+define i32 @shr_mismatch_xor(i32 %x, i32 %y) nounwind {
+; CHECK-LABEL: shr_mismatch_xor:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    sarl $5, %eax
+; CHECK-NEXT:    xorl %esi, %eax
+; CHECK-NEXT:    shrl $7, %eax
+; CHECK-NEXT:    retq
+  %sh0 = ashr i32 %x, 5
+  %r = xor i32 %y, %sh0
+  %sh1 = lshr i32 %r, 7
+  ret i32 %sh1
+}
+
+define i32 @ashr_overshift_xor(i32 %x, i32 %y) nounwind {
+; CHECK-LABEL: ashr_overshift_xor:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    sarl $15, %eax
+; CHECK-NEXT:    xorl %esi, %eax
+; CHECK-NEXT:    sarl $17, %eax
+; CHECK-NEXT:    retq
+  %sh0 = ashr i32 %x, 15
+  %r = xor i32 %y, %sh0
+  %sh1 = ashr i32 %r, 17
+  ret i32 %sh1
+}
+
+define i32 @lshr_or_extra_use(i32 %x, i32 %y, i32* %p) nounwind {
+; CHECK-LABEL: lshr_or_extra_use:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    shrl $5, %eax
+; CHECK-NEXT:    orl %esi, %eax
+; CHECK-NEXT:    movl %eax, (%rdx)
+; CHECK-NEXT:    shrl $7, %eax
+; CHECK-NEXT:    retq
+  %sh0 = lshr i32 %x, 5
+  %r = or i32 %sh0, %y
+  store i32 %r, i32* %p
+  %sh1 = lshr i32 %r, 7
+  ret i32 %sh1
+}
diff --git a/llvm/test/CodeGen/X86/shift-one.ll b/llvm/test/CodeGen/X86/shift-one.ll
index d961eb145..c4c50afb2 100644
--- a/llvm/test/CodeGen/X86/shift-one.ll
+++ b/llvm/test/CodeGen/X86/shift-one.ll
@@ -1,8 +1,14 @@
-; RUN: llc < %s -mtriple=i686-- | not grep leal
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- | FileCheck %s
 
 @x = external global i32                ; <i32*> [#uses=1]
 
 define i32 @test() {
+; CHECK-LABEL: test:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl x, %eax
+; CHECK-NEXT:    addl %eax, %eax
+; CHECK-NEXT:    retl
         %tmp.0 = load i32, i32* @x           ; <i32> [#uses=1]
         %tmp.1 = shl i32 %tmp.0, 1              ; <i32> [#uses=1]
         ret i32 %tmp.1
diff --git a/llvm/test/CodeGen/X86/shift-parts.ll b/llvm/test/CodeGen/X86/shift-parts.ll
index 678866979..2dc35f6bf 100644
--- a/llvm/test/CodeGen/X86/shift-parts.ll
+++ b/llvm/test/CodeGen/X86/shift-parts.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=x86_64-- < %s | FileCheck %s
 ; PR4736
 
@@ -5,9 +6,25 @@
 
 @g_144 = external global %0, align 8              ; <%0*> [#uses=1]
 
-; CHECK: shrdq
-
 define i32 @int87(i32 %uint64p_8, i1 %cond) nounwind {
+; CHECK-LABEL: int87:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movq g_144+{{.*}}(%rip), %rax
+; CHECK-NEXT:    movq g_144+{{.*}}(%rip), %rdx
+; CHECK-NEXT:    movzbl %sil, %ecx
+; CHECK-NEXT:    shll $6, %ecx
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB0_1: # %for.cond
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movq %rdx, %rsi
+; CHECK-NEXT:    shrdq %cl, %rax, %rsi
+; CHECK-NEXT:    testb $64, %cl
+; CHECK-NEXT:    cmovneq %rax, %rsi
+; CHECK-NEXT:    orl $0, %esi
+; CHECK-NEXT:    je .LBB0_1
+; CHECK-NEXT:  # %bb.2: # %if.then
+; CHECK-NEXT:    movl $1, %eax
+; CHECK-NEXT:    retq
 entry:
   %srcval4 = load i320, i320* bitcast (%0* @g_144 to i320*), align 8 ; <i320> [#uses=1]
   br label %for.cond
diff --git a/llvm/test/CodeGen/X86/shl_elim.ll b/llvm/test/CodeGen/X86/shl_elim.ll
index ed67a09f0..2eb29fe4c 100644
--- a/llvm/test/CodeGen/X86/shl_elim.ll
+++ b/llvm/test/CodeGen/X86/shl_elim.ll
@@ -1,16 +1,18 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=i686-- | FileCheck %s
 
 define i32 @test1(i64 %a) nounwind {
+; CHECK-LABEL: test1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    shrl %eax
+; CHECK-NEXT:    cwtl
+; CHECK-NEXT:    retl
         %tmp29 = lshr i64 %a, 24                ; <i64> [#uses=1]
         %tmp23 = trunc i64 %tmp29 to i32                ; <i32> [#uses=1]
         %tmp410 = lshr i32 %tmp23, 9            ; <i32> [#uses=1]
         %tmp45 = trunc i32 %tmp410 to i16               ; <i16> [#uses=1]
         %tmp456 = sext i16 %tmp45 to i32                ; <i32> [#uses=1]
         ret i32 %tmp456
-
-; CHECK-LABEL: test1:
-; CHECK: movl 8(%esp), %eax
-; CHECK: shrl %eax
-; CHECK: cwtl
 }
 
diff --git a/llvm/test/CodeGen/X86/shrink-compare-pgso.ll b/llvm/test/CodeGen/X86/shrink-compare-pgso.ll
new file mode 100644
index 000000000..337f7d11e
--- /dev/null
+++ b/llvm/test/CodeGen/X86/shrink-compare-pgso.ll
@@ -0,0 +1,321 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-linux | FileCheck %s
+
+declare void @bar()
+
+define void @test1(i32* nocapture %X) nounwind !prof !14 {
+; CHECK-LABEL: test1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cmpb $47, (%rdi)
+; CHECK-NEXT:    je bar # TAILCALL
+; CHECK-NEXT:  # %bb.1: # %if.end
+; CHECK-NEXT:    retq
+entry:
+  %tmp1 = load i32, i32* %X, align 4
+  %and = and i32 %tmp1, 255
+  %cmp = icmp eq i32 %and, 47
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  tail call void @bar() nounwind
+  br label %if.end
+
+if.end:
+  ret void
+}
+
+define void @test2(i32 %X) nounwind !prof !14 {
+; CHECK-LABEL: test2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cmpb $47, %dil
+; CHECK-NEXT:    je bar # TAILCALL
+; CHECK-NEXT:  # %bb.1: # %if.end
+; CHECK-NEXT:    retq
+entry:
+  %and = and i32 %X, 255
+  %cmp = icmp eq i32 %and, 47
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  tail call void @bar() nounwind
+  br label %if.end
+
+if.end:
+  ret void
+}
+
+define void @test3(i32 %X) nounwind !prof !14 {
+; CHECK-LABEL: test3:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cmpb $-1, %dil
+; CHECK-NEXT:    je bar # TAILCALL
+; CHECK-NEXT:  # %bb.1: # %if.end
+; CHECK-NEXT:    retq
+entry:
+  %and = and i32 %X, 255
+  %cmp = icmp eq i32 %and, 255
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  tail call void @bar() nounwind
+  br label %if.end
+
+if.end:
+  ret void
+}
+
+; PR16083
+define i1 @test4(i64 %a, i32 %b) {
+; CHECK-LABEL: test4:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movb $1, %al
+; CHECK-NEXT:    testl %esi, %esi
+; CHECK-NEXT:    je .LBB3_1
+; CHECK-NEXT:  # %bb.2: # %lor.end
+; CHECK-NEXT:    # kill: def $al killed $al killed $eax
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  .LBB3_1: # %lor.rhs
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    # kill: def $al killed $al killed $eax
+; CHECK-NEXT:    retq
+entry:
+  %tobool = icmp ne i32 %b, 0
+  br i1 %tobool, label %lor.end, label %lor.rhs
+
+lor.rhs:                                          ; preds = %entry
+  %and = and i64 0, %a
+  %tobool1 = icmp ne i64 %and, 0
+  br label %lor.end
+
+lor.end:                                          ; preds = %lor.rhs, %entry
+  %p = phi i1 [ true, %entry ], [ %tobool1, %lor.rhs ]
+  ret i1 %p
+}
+
+@x = global { i8, i8, i8, i8, i8, i8, i8, i8 } { i8 1, i8 0, i8 0, i8 0, i8 1, i8 0, i8 0, i8 1 }, align 4
+
+; PR16551
+define void @test5(i32 %X) nounwind !prof !14 {
+; CHECK-LABEL: test5:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movzbl x+{{.*}}(%rip), %eax
+; CHECK-NEXT:    shll $16, %eax
+; CHECK-NEXT:    movzwl x+{{.*}}(%rip), %ecx
+; CHECK-NEXT:    orl %eax, %ecx
+; CHECK-NEXT:    cmpl $1, %ecx
+; CHECK-NEXT:    jne bar # TAILCALL
+; CHECK-NEXT:  # %bb.1: # %if.end
+; CHECK-NEXT:    retq
+entry:
+  %bf.load = load i56, i56* bitcast ({ i8, i8, i8, i8, i8, i8, i8, i8 }* @x to i56*), align 4
+  %bf.lshr = lshr i56 %bf.load, 32
+  %bf.cast = trunc i56 %bf.lshr to i32
+  %cmp = icmp ne i32 %bf.cast, 1
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  tail call void @bar() nounwind
+  br label %if.end
+
+if.end:
+  ret void
+}
+
+define void @test2_1(i32 %X) nounwind !prof !14 {
+; CHECK-LABEL: test2_1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    cmpl $256, %eax # imm = 0x100
+; CHECK-NEXT:    je bar # TAILCALL
+; CHECK-NEXT:  # %bb.1: # %if.end
+; CHECK-NEXT:    retq
+entry:
+  %and = and i32 %X, 255
+  %cmp = icmp eq i32 %and, 256
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  tail call void @bar() nounwind
+  br label %if.end
+
+if.end:
+  ret void
+}
+
+define void @test_sext_i8_icmp_1(i8 %x) nounwind !prof !14 {
+; CHECK-LABEL: test_sext_i8_icmp_1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cmpb $1, %dil
+; CHECK-NEXT:    je bar # TAILCALL
+; CHECK-NEXT:  # %bb.1: # %if.end
+; CHECK-NEXT:    retq
+entry:
+  %sext = sext i8 %x to i32
+  %cmp = icmp eq i32 %sext, 1
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  tail call void @bar() nounwind
+  br label %if.end
+
+if.end:
+  ret void
+}
+
+define void @test_sext_i8_icmp_47(i8 %x) nounwind !prof !14 {
+; CHECK-LABEL: test_sext_i8_icmp_47:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cmpb $47, %dil
+; CHECK-NEXT:    je bar # TAILCALL
+; CHECK-NEXT:  # %bb.1: # %if.end
+; CHECK-NEXT:    retq
+entry:
+  %sext = sext i8 %x to i32
+  %cmp = icmp eq i32 %sext, 47
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  tail call void @bar() nounwind
+  br label %if.end
+
+if.end:
+  ret void
+}
+
+define void @test_sext_i8_icmp_127(i8 %x) nounwind !prof !14 {
+; CHECK-LABEL: test_sext_i8_icmp_127:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cmpb $127, %dil
+; CHECK-NEXT:    je bar # TAILCALL
+; CHECK-NEXT:  # %bb.1: # %if.end
+; CHECK-NEXT:    retq
+entry:
+  %sext = sext i8 %x to i32
+  %cmp = icmp eq i32 %sext, 127
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  tail call void @bar() nounwind
+  br label %if.end
+
+if.end:
+  ret void
+}
+
+define void @test_sext_i8_icmp_neg1(i8 %x) nounwind !prof !14 {
+; CHECK-LABEL: test_sext_i8_icmp_neg1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cmpb $-1, %dil
+; CHECK-NEXT:    je bar # TAILCALL
+; CHECK-NEXT:  # %bb.1: # %if.end
+; CHECK-NEXT:    retq
+entry:
+  %sext = sext i8 %x to i32
+  %cmp = icmp eq i32 %sext, -1
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  tail call void @bar() nounwind
+  br label %if.end
+
+if.end:
+  ret void
+}
+
+define void @test_sext_i8_icmp_neg2(i8 %x) nounwind !prof !14 {
+; CHECK-LABEL: test_sext_i8_icmp_neg2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cmpb $-2, %dil
+; CHECK-NEXT:    je bar # TAILCALL
+; CHECK-NEXT:  # %bb.1: # %if.end
+; CHECK-NEXT:    retq
+entry:
+  %sext = sext i8 %x to i32
+  %cmp = icmp eq i32 %sext, -2
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  tail call void @bar() nounwind
+  br label %if.end
+
+if.end:
+  ret void
+}
+
+define void @test_sext_i8_icmp_neg127(i8 %x) nounwind !prof !14 {
+; CHECK-LABEL: test_sext_i8_icmp_neg127:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cmpb $-127, %dil
+; CHECK-NEXT:    je bar # TAILCALL
+; CHECK-NEXT:  # %bb.1: # %if.end
+; CHECK-NEXT:    retq
+entry:
+  %sext = sext i8 %x to i32
+  %cmp = icmp eq i32 %sext, -127
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  tail call void @bar() nounwind
+  br label %if.end
+
+if.end:
+  ret void
+}
+
+define void @test_sext_i8_icmp_neg128(i8 %x) nounwind !prof !14 {
+; CHECK-LABEL: test_sext_i8_icmp_neg128:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cmpb $-128, %dil
+; CHECK-NEXT:    je bar # TAILCALL
+; CHECK-NEXT:  # %bb.1: # %if.end
+; CHECK-NEXT:    retq
+entry:
+  %sext = sext i8 %x to i32
+  %cmp = icmp eq i32 %sext, -128
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  tail call void @bar() nounwind
+  br label %if.end
+
+if.end:
+  ret void
+}
+
+define void @test_sext_i8_icmp_255(i8 %x) nounwind !prof !14 {
+; CHECK-LABEL: test_sext_i8_icmp_255:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movb $1, %al
+; CHECK-NEXT:    testb %al, %al
+; CHECK-NEXT:    je bar # TAILCALL
+; CHECK-NEXT:  # %bb.1: # %if.end
+; CHECK-NEXT:    retq
+entry:
+  %sext = sext i8 %x to i32
+  %cmp = icmp eq i32 %sext, 255
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  tail call void @bar() nounwind
+  br label %if.end
+
+if.end:
+  ret void
+}
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 1, !"ProfileSummary", !1}
+!1 = !{!2, !3, !4, !5, !6, !7, !8, !9}
+!2 = !{!"ProfileFormat", !"InstrProf"}
+!3 = !{!"TotalCount", i64 10000}
+!4 = !{!"MaxCount", i64 10}
+!5 = !{!"MaxInternalCount", i64 1}
+!6 = !{!"MaxFunctionCount", i64 1000}
+!7 = !{!"NumCounts", i64 3}
+!8 = !{!"NumFunctions", i64 3}
+!9 = !{!"DetailedSummary", !10}
+!10 = !{!11, !12, !13}
+!11 = !{i32 10000, i64 100, i32 1}
+!12 = !{i32 999000, i64 100, i32 1}
+!13 = !{i32 999999, i64 1, i32 2}
+!14 = !{!"function_entry_count", i64 0}
diff --git a/llvm/test/CodeGen/X86/shrink-fp-const1.ll b/llvm/test/CodeGen/X86/shrink-fp-const1.ll
index ba769c194..b633cc23b 100644
--- a/llvm/test/CodeGen/X86/shrink-fp-const1.ll
+++ b/llvm/test/CodeGen/X86/shrink-fp-const1.ll
@@ -1,7 +1,12 @@
-; RUN: llc < %s -mtriple=x86_64-- -mattr=+sse2 | not grep cvtss2sd
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+sse2 | FileCheck %s
 ; PR1264
 
 define double @foo(double %x) {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mulsd {{.*}}(%rip), %xmm0
+; CHECK-NEXT:    retq
         %y = fmul double %x, 5.000000e-01
         ret double %y
 }
diff --git a/llvm/test/CodeGen/X86/shrink-fp-const2.ll b/llvm/test/CodeGen/X86/shrink-fp-const2.ll
index b62a69cab..670f268b6 100644
--- a/llvm/test/CodeGen/X86/shrink-fp-const2.ll
+++ b/llvm/test/CodeGen/X86/shrink-fp-const2.ll
@@ -1,6 +1,11 @@
-; RUN: llc < %s -mtriple=i686-- | grep flds
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- | FileCheck %s
 ; This should be a flds, not fldt.
 define x86_fp80 @test2() nounwind  {
+; CHECK-LABEL: test2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    flds {{\.LCPI.*}}
+; CHECK-NEXT:    retl
 entry:
 	ret x86_fp80 0xK3FFFC000000000000000
 }
diff --git a/llvm/test/CodeGen/X86/shrink_vmul.ll b/llvm/test/CodeGen/X86/shrink_vmul.ll
index 5e952472f..1c1032a4c 100644
--- a/llvm/test/CodeGen/X86/shrink_vmul.ll
+++ b/llvm/test/CodeGen/X86/shrink_vmul.ll
@@ -42,10 +42,13 @@ define void @mul_2xi8(i8* nocapture readonly %a, i8* nocapture readonly %b, i64
 ; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-AVX-NEXT:    movl c, %esi
-; X86-AVX-NEXT:    vpmovzxbq {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero
-; X86-AVX-NEXT:    vpmovzxbq {{.*#+}} xmm1 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero
-; X86-AVX-NEXT:    vpmuludq %xmm0, %xmm1, %xmm0
-; X86-AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X86-AVX-NEXT:    movzwl (%edx,%ecx), %edx
+; X86-AVX-NEXT:    vmovd %edx, %xmm0
+; X86-AVX-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; X86-AVX-NEXT:    movzwl (%eax,%ecx), %eax
+; X86-AVX-NEXT:    vmovd %eax, %xmm1
+; X86-AVX-NEXT:    vpmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
+; X86-AVX-NEXT:    vpmaddwd %xmm0, %xmm1, %xmm0
 ; X86-AVX-NEXT:    vmovq %xmm0, (%esi,%ecx,4)
 ; X86-AVX-NEXT:    popl %esi
 ; X86-AVX-NEXT:    retl
@@ -68,10 +71,13 @@ define void @mul_2xi8(i8* nocapture readonly %a, i8* nocapture readonly %b, i64
 ; X64-AVX-LABEL: mul_2xi8:
 ; X64-AVX:       # %bb.0: # %entry
 ; X64-AVX-NEXT:    movq {{.*}}(%rip), %rax
-; X64-AVX-NEXT:    vpmovzxbq {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero
-; X64-AVX-NEXT:    vpmovzxbq {{.*#+}} xmm1 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero
-; X64-AVX-NEXT:    vpmuludq %xmm0, %xmm1, %xmm0
-; X64-AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X64-AVX-NEXT:    movzwl (%rdi,%rdx), %ecx
+; X64-AVX-NEXT:    vmovd %ecx, %xmm0
+; X64-AVX-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; X64-AVX-NEXT:    movzwl (%rsi,%rdx), %ecx
+; X64-AVX-NEXT:    vmovd %ecx, %xmm1
+; X64-AVX-NEXT:    vpmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
+; X64-AVX-NEXT:    vpmaddwd %xmm0, %xmm1, %xmm0
 ; X64-AVX-NEXT:    vmovq %xmm0, (%rax,%rdx,4)
 ; X64-AVX-NEXT:    retq
 entry:
@@ -475,10 +481,11 @@ define void @mul_2xi16(i8* nocapture readonly %a, i8* nocapture readonly %b, i64
 ; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-AVX-NEXT:    movl c, %esi
-; X86-AVX-NEXT:    vpmovzxwq {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero
-; X86-AVX-NEXT:    vpmovzxwq {{.*#+}} xmm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero
-; X86-AVX-NEXT:    vpmuludq %xmm0, %xmm1, %xmm0
-; X86-AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X86-AVX-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-AVX-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; X86-AVX-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; X86-AVX-NEXT:    vpmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; X86-AVX-NEXT:    vpmulld %xmm0, %xmm1, %xmm0
 ; X86-AVX-NEXT:    vmovq %xmm0, (%esi,%ecx,4)
 ; X86-AVX-NEXT:    popl %esi
 ; X86-AVX-NEXT:    retl
@@ -498,10 +505,11 @@ define void @mul_2xi16(i8* nocapture readonly %a, i8* nocapture readonly %b, i64
 ; X64-AVX-LABEL: mul_2xi16:
 ; X64-AVX:       # %bb.0: # %entry
 ; X64-AVX-NEXT:    movq {{.*}}(%rip), %rax
-; X64-AVX-NEXT:    vpmovzxwq {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero
-; X64-AVX-NEXT:    vpmovzxwq {{.*#+}} xmm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero
-; X64-AVX-NEXT:    vpmuludq %xmm0, %xmm1, %xmm0
-; X64-AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X64-AVX-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X64-AVX-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; X64-AVX-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; X64-AVX-NEXT:    vpmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; X64-AVX-NEXT:    vpmulld %xmm0, %xmm1, %xmm0
 ; X64-AVX-NEXT:    vmovq %xmm0, (%rax,%rdx,4)
 ; X64-AVX-NEXT:    retq
 entry:
@@ -904,10 +912,13 @@ define void @mul_2xi8_sext(i8* nocapture readonly %a, i8* nocapture readonly %b,
 ; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-AVX-NEXT:    movl c, %esi
-; X86-AVX-NEXT:    vpmovsxbq (%edx,%ecx), %xmm0
-; X86-AVX-NEXT:    vpmovsxbq (%eax,%ecx), %xmm1
-; X86-AVX-NEXT:    vpmuludq %xmm0, %xmm1, %xmm0
-; X86-AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X86-AVX-NEXT:    movzwl (%edx,%ecx), %edx
+; X86-AVX-NEXT:    vmovd %edx, %xmm0
+; X86-AVX-NEXT:    vpmovsxbd %xmm0, %xmm0
+; X86-AVX-NEXT:    movzwl (%eax,%ecx), %eax
+; X86-AVX-NEXT:    vmovd %eax, %xmm1
+; X86-AVX-NEXT:    vpmovsxbd %xmm1, %xmm1
+; X86-AVX-NEXT:    vpmulld %xmm0, %xmm1, %xmm0
 ; X86-AVX-NEXT:    vmovq %xmm0, (%esi,%ecx,4)
 ; X86-AVX-NEXT:    popl %esi
 ; X86-AVX-NEXT:    retl
@@ -932,10 +943,13 @@ define void @mul_2xi8_sext(i8* nocapture readonly %a, i8* nocapture readonly %b,
 ; X64-AVX-LABEL: mul_2xi8_sext:
 ; X64-AVX:       # %bb.0: # %entry
 ; X64-AVX-NEXT:    movq {{.*}}(%rip), %rax
-; X64-AVX-NEXT:    vpmovsxbq (%rdi,%rdx), %xmm0
-; X64-AVX-NEXT:    vpmovsxbq (%rsi,%rdx), %xmm1
-; X64-AVX-NEXT:    vpmuludq %xmm0, %xmm1, %xmm0
-; X64-AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X64-AVX-NEXT:    movzwl (%rdi,%rdx), %ecx
+; X64-AVX-NEXT:    vmovd %ecx, %xmm0
+; X64-AVX-NEXT:    vpmovsxbd %xmm0, %xmm0
+; X64-AVX-NEXT:    movzwl (%rsi,%rdx), %ecx
+; X64-AVX-NEXT:    vmovd %ecx, %xmm1
+; X64-AVX-NEXT:    vpmovsxbd %xmm1, %xmm1
+; X64-AVX-NEXT:    vpmulld %xmm0, %xmm1, %xmm0
 ; X64-AVX-NEXT:    vmovq %xmm0, (%rax,%rdx,4)
 ; X64-AVX-NEXT:    retq
 entry:
@@ -992,10 +1006,13 @@ define void @mul_2xi8_sext_zext(i8* nocapture readonly %a, i8* nocapture readonl
 ; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-AVX-NEXT:    movl c, %esi
-; X86-AVX-NEXT:    vpmovsxbq (%edx,%ecx), %xmm0
-; X86-AVX-NEXT:    vpmovzxbq {{.*#+}} xmm1 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero
-; X86-AVX-NEXT:    vpmuludq %xmm0, %xmm1, %xmm0
-; X86-AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X86-AVX-NEXT:    movzwl (%edx,%ecx), %edx
+; X86-AVX-NEXT:    vmovd %edx, %xmm0
+; X86-AVX-NEXT:    vpmovsxbd %xmm0, %xmm0
+; X86-AVX-NEXT:    movzwl (%eax,%ecx), %eax
+; X86-AVX-NEXT:    vmovd %eax, %xmm1
+; X86-AVX-NEXT:    vpmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
+; X86-AVX-NEXT:    vpmulld %xmm0, %xmm1, %xmm0
 ; X86-AVX-NEXT:    vmovq %xmm0, (%esi,%ecx,4)
 ; X86-AVX-NEXT:    popl %esi
 ; X86-AVX-NEXT:    retl
@@ -1021,10 +1038,13 @@ define void @mul_2xi8_sext_zext(i8* nocapture readonly %a, i8* nocapture readonl
 ; X64-AVX-LABEL: mul_2xi8_sext_zext:
 ; X64-AVX:       # %bb.0: # %entry
 ; X64-AVX-NEXT:    movq {{.*}}(%rip), %rax
-; X64-AVX-NEXT:    vpmovsxbq (%rdi,%rdx), %xmm0
-; X64-AVX-NEXT:    vpmovzxbq {{.*#+}} xmm1 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero
-; X64-AVX-NEXT:    vpmuludq %xmm0, %xmm1, %xmm0
-; X64-AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X64-AVX-NEXT:    movzwl (%rdi,%rdx), %ecx
+; X64-AVX-NEXT:    vmovd %ecx, %xmm0
+; X64-AVX-NEXT:    vpmovsxbd %xmm0, %xmm0
+; X64-AVX-NEXT:    movzwl (%rsi,%rdx), %ecx
+; X64-AVX-NEXT:    vmovd %ecx, %xmm1
+; X64-AVX-NEXT:    vpmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
+; X64-AVX-NEXT:    vpmulld %xmm0, %xmm1, %xmm0
 ; X64-AVX-NEXT:    vmovq %xmm0, (%rax,%rdx,4)
 ; X64-AVX-NEXT:    retq
 entry:
@@ -1075,10 +1095,11 @@ define void @mul_2xi16_sext(i8* nocapture readonly %a, i8* nocapture readonly %b
 ; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-AVX-NEXT:    movl c, %esi
-; X86-AVX-NEXT:    vpmovsxwq (%edx,%ecx), %xmm0
-; X86-AVX-NEXT:    vpmovsxwq (%eax,%ecx), %xmm1
-; X86-AVX-NEXT:    vpmuludq %xmm0, %xmm1, %xmm0
-; X86-AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X86-AVX-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-AVX-NEXT:    vpmovsxwd %xmm0, %xmm0
+; X86-AVX-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; X86-AVX-NEXT:    vpmovsxwd %xmm1, %xmm1
+; X86-AVX-NEXT:    vpmulld %xmm0, %xmm1, %xmm0
 ; X86-AVX-NEXT:    vmovq %xmm0, (%esi,%ecx,4)
 ; X86-AVX-NEXT:    popl %esi
 ; X86-AVX-NEXT:    retl
@@ -1098,10 +1119,11 @@ define void @mul_2xi16_sext(i8* nocapture readonly %a, i8* nocapture readonly %b
 ; X64-AVX-LABEL: mul_2xi16_sext:
 ; X64-AVX:       # %bb.0: # %entry
 ; X64-AVX-NEXT:    movq {{.*}}(%rip), %rax
-; X64-AVX-NEXT:    vpmovsxwq (%rdi,%rdx), %xmm0
-; X64-AVX-NEXT:    vpmovsxwq (%rsi,%rdx), %xmm1
-; X64-AVX-NEXT:    vpmuludq %xmm0, %xmm1, %xmm0
-; X64-AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X64-AVX-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X64-AVX-NEXT:    vpmovsxwd %xmm0, %xmm0
+; X64-AVX-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; X64-AVX-NEXT:    vpmovsxwd %xmm1, %xmm1
+; X64-AVX-NEXT:    vpmulld %xmm0, %xmm1, %xmm0
 ; X64-AVX-NEXT:    vmovq %xmm0, (%rax,%rdx,4)
 ; X64-AVX-NEXT:    retq
 entry:
@@ -1138,14 +1160,15 @@ define void @mul_2xi16_sext_zext(i8* nocapture readonly %a, i8* nocapture readon
 ; X86-SSE-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; X86-SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,2,1,4,5,6,7]
 ; X86-SSE-NEXT:    psrad $16, %xmm0
-; X86-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,1,3]
 ; X86-SSE-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; X86-SSE-NEXT:    pxor %xmm2, %xmm2
 ; X86-SSE-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
-; X86-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,1,1,3]
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
 ; X86-SSE-NEXT:    pmuludq %xmm0, %xmm1
-; X86-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
-; X86-SSE-NEXT:    movq %xmm0, (%esi,%ecx,4)
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; X86-SSE-NEXT:    pmuludq %xmm2, %xmm0
+; X86-SSE-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; X86-SSE-NEXT:    movq %xmm1, (%esi,%ecx,4)
 ; X86-SSE-NEXT:    popl %esi
 ; X86-SSE-NEXT:    retl
 ;
@@ -1156,10 +1179,11 @@ define void @mul_2xi16_sext_zext(i8* nocapture readonly %a, i8* nocapture readon
 ; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-AVX-NEXT:    movl c, %esi
-; X86-AVX-NEXT:    vpmovsxwq (%edx,%ecx), %xmm0
-; X86-AVX-NEXT:    vpmovzxwq {{.*#+}} xmm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero
-; X86-AVX-NEXT:    vpmuludq %xmm0, %xmm1, %xmm0
-; X86-AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X86-AVX-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-AVX-NEXT:    vpmovsxwd %xmm0, %xmm0
+; X86-AVX-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; X86-AVX-NEXT:    vpmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; X86-AVX-NEXT:    vpmulld %xmm0, %xmm1, %xmm0
 ; X86-AVX-NEXT:    vmovq %xmm0, (%esi,%ecx,4)
 ; X86-AVX-NEXT:    popl %esi
 ; X86-AVX-NEXT:    retl
@@ -1170,23 +1194,25 @@ define void @mul_2xi16_sext_zext(i8* nocapture readonly %a, i8* nocapture readon
 ; X64-SSE-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; X64-SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,2,1,4,5,6,7]
 ; X64-SSE-NEXT:    psrad $16, %xmm0
-; X64-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,1,3]
 ; X64-SSE-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; X64-SSE-NEXT:    pxor %xmm2, %xmm2
 ; X64-SSE-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
-; X64-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,1,1,3]
+; X64-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
 ; X64-SSE-NEXT:    pmuludq %xmm0, %xmm1
-; X64-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
-; X64-SSE-NEXT:    movq %xmm0, (%rax,%rdx,4)
+; X64-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; X64-SSE-NEXT:    pmuludq %xmm2, %xmm0
+; X64-SSE-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; X64-SSE-NEXT:    movq %xmm1, (%rax,%rdx,4)
 ; X64-SSE-NEXT:    retq
 ;
 ; X64-AVX-LABEL: mul_2xi16_sext_zext:
 ; X64-AVX:       # %bb.0: # %entry
 ; X64-AVX-NEXT:    movq {{.*}}(%rip), %rax
-; X64-AVX-NEXT:    vpmovsxwq (%rdi,%rdx), %xmm0
-; X64-AVX-NEXT:    vpmovzxwq {{.*#+}} xmm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero
-; X64-AVX-NEXT:    vpmuludq %xmm0, %xmm1, %xmm0
-; X64-AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X64-AVX-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X64-AVX-NEXT:    vpmovsxwd %xmm0, %xmm0
+; X64-AVX-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; X64-AVX-NEXT:    vpmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; X64-AVX-NEXT:    vpmulld %xmm0, %xmm1, %xmm0
 ; X64-AVX-NEXT:    vmovq %xmm0, (%rax,%rdx,4)
 ; X64-AVX-NEXT:    retq
 entry:
@@ -1379,8 +1405,8 @@ define void @mul_2xi8_varconst1(i8* nocapture readonly %a, i64 %index) {
 ; X86-SSE-NEXT:    movd %ecx, %xmm0
 ; X86-SSE-NEXT:    pxor %xmm1, %xmm1
 ; X86-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
-; X86-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
 ; X86-SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; X86-SSE-NEXT:    pmaddwd {{\.LCPI.*}}, %xmm0
 ; X86-SSE-NEXT:    movq %xmm0, (%edx,%eax,4)
 ; X86-SSE-NEXT:    retl
 ;
@@ -1389,9 +1415,10 @@ define void @mul_2xi8_varconst1(i8* nocapture readonly %a, i64 %index) {
 ; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-AVX-NEXT:    movl c, %edx
-; X86-AVX-NEXT:    vpmovzxbq {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero
-; X86-AVX-NEXT:    vpmuludq {{\.LCPI.*}}, %xmm0, %xmm0
-; X86-AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X86-AVX-NEXT:    movzwl (%ecx,%eax), %ecx
+; X86-AVX-NEXT:    vmovd %ecx, %xmm0
+; X86-AVX-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; X86-AVX-NEXT:    vpmaddwd {{\.LCPI.*}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vmovq %xmm0, (%edx,%eax,4)
 ; X86-AVX-NEXT:    retl
 ;
@@ -1402,17 +1429,18 @@ define void @mul_2xi8_varconst1(i8* nocapture readonly %a, i64 %index) {
 ; X64-SSE-NEXT:    movd %ecx, %xmm0
 ; X64-SSE-NEXT:    pxor %xmm1, %xmm1
 ; X64-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
-; X64-SSE-NEXT:    pmullw {{.*}}(%rip), %xmm0
 ; X64-SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; X64-SSE-NEXT:    pmaddwd {{.*}}(%rip), %xmm0
 ; X64-SSE-NEXT:    movq %xmm0, (%rax,%rsi,4)
 ; X64-SSE-NEXT:    retq
 ;
 ; X64-AVX-LABEL: mul_2xi8_varconst1:
 ; X64-AVX:       # %bb.0: # %entry
 ; X64-AVX-NEXT:    movq {{.*}}(%rip), %rax
-; X64-AVX-NEXT:    vpmovzxbq {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero
-; X64-AVX-NEXT:    vpmuludq {{.*}}(%rip), %xmm0, %xmm0
-; X64-AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X64-AVX-NEXT:    movzwl (%rdi,%rsi), %ecx
+; X64-AVX-NEXT:    vmovd %ecx, %xmm0
+; X64-AVX-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; X64-AVX-NEXT:    vpmaddwd {{.*}}(%rip), %xmm0, %xmm0
 ; X64-AVX-NEXT:    vmovq %xmm0, (%rax,%rsi,4)
 ; X64-AVX-NEXT:    retq
 entry:
@@ -1454,9 +1482,10 @@ define void @mul_2xi8_varconst2(i8* nocapture readonly %a, i64 %index) {
 ; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-AVX-NEXT:    movl c, %edx
-; X86-AVX-NEXT:    vpmovsxbq (%ecx,%eax), %xmm0
-; X86-AVX-NEXT:    vpmuludq {{\.LCPI.*}}, %xmm0, %xmm0
-; X86-AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X86-AVX-NEXT:    movzwl (%ecx,%eax), %ecx
+; X86-AVX-NEXT:    vmovd %ecx, %xmm0
+; X86-AVX-NEXT:    vpmovsxbd %xmm0, %xmm0
+; X86-AVX-NEXT:    vpmulld {{\.LCPI.*}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vmovq %xmm0, (%edx,%eax,4)
 ; X86-AVX-NEXT:    retl
 ;
@@ -1476,9 +1505,10 @@ define void @mul_2xi8_varconst2(i8* nocapture readonly %a, i64 %index) {
 ; X64-AVX-LABEL: mul_2xi8_varconst2:
 ; X64-AVX:       # %bb.0: # %entry
 ; X64-AVX-NEXT:    movq {{.*}}(%rip), %rax
-; X64-AVX-NEXT:    vpmovsxbq (%rdi,%rsi), %xmm0
-; X64-AVX-NEXT:    vpmuludq {{.*}}(%rip), %xmm0, %xmm0
-; X64-AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X64-AVX-NEXT:    movzwl (%rdi,%rsi), %ecx
+; X64-AVX-NEXT:    vmovd %ecx, %xmm0
+; X64-AVX-NEXT:    vpmovsxbd %xmm0, %xmm0
+; X64-AVX-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
 ; X64-AVX-NEXT:    vmovq %xmm0, (%rax,%rsi,4)
 ; X64-AVX-NEXT:    retq
 entry:
@@ -1509,11 +1539,8 @@ define void @mul_2xi8_varconst3(i8* nocapture readonly %a, i64 %index) {
 ; X86-SSE-NEXT:    movd %ecx, %xmm0
 ; X86-SSE-NEXT:    pxor %xmm1, %xmm1
 ; X86-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
-; X86-SSE-NEXT:    movdqa {{.*#+}} xmm1 = <0,256,u,u,u,u,u,u>
-; X86-SSE-NEXT:    movdqa %xmm0, %xmm2
-; X86-SSE-NEXT:    pmulhw %xmm1, %xmm2
-; X86-SSE-NEXT:    pmullw %xmm1, %xmm0
-; X86-SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; X86-SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; X86-SSE-NEXT:    pmaddwd {{\.LCPI.*}}, %xmm0
 ; X86-SSE-NEXT:    movq %xmm0, (%edx,%eax,4)
 ; X86-SSE-NEXT:    retl
 ;
@@ -1522,9 +1549,10 @@ define void @mul_2xi8_varconst3(i8* nocapture readonly %a, i64 %index) {
 ; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-AVX-NEXT:    movl c, %edx
-; X86-AVX-NEXT:    vpmovzxbq {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero
-; X86-AVX-NEXT:    vpmuludq {{\.LCPI.*}}, %xmm0, %xmm0
-; X86-AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X86-AVX-NEXT:    movzwl (%ecx,%eax), %ecx
+; X86-AVX-NEXT:    vmovd %ecx, %xmm0
+; X86-AVX-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; X86-AVX-NEXT:    vpmaddwd {{\.LCPI.*}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vmovq %xmm0, (%edx,%eax,4)
 ; X86-AVX-NEXT:    retl
 ;
@@ -1535,20 +1563,18 @@ define void @mul_2xi8_varconst3(i8* nocapture readonly %a, i64 %index) {
 ; X64-SSE-NEXT:    movd %ecx, %xmm0
 ; X64-SSE-NEXT:    pxor %xmm1, %xmm1
 ; X64-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
-; X64-SSE-NEXT:    movdqa {{.*#+}} xmm1 = <0,256,u,u,u,u,u,u>
-; X64-SSE-NEXT:    movdqa %xmm0, %xmm2
-; X64-SSE-NEXT:    pmulhw %xmm1, %xmm2
-; X64-SSE-NEXT:    pmullw %xmm1, %xmm0
-; X64-SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; X64-SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; X64-SSE-NEXT:    pmaddwd {{.*}}(%rip), %xmm0
 ; X64-SSE-NEXT:    movq %xmm0, (%rax,%rsi,4)
 ; X64-SSE-NEXT:    retq
 ;
 ; X64-AVX-LABEL: mul_2xi8_varconst3:
 ; X64-AVX:       # %bb.0: # %entry
 ; X64-AVX-NEXT:    movq {{.*}}(%rip), %rax
-; X64-AVX-NEXT:    vpmovzxbq {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero
-; X64-AVX-NEXT:    vpmuludq {{.*}}(%rip), %xmm0, %xmm0
-; X64-AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X64-AVX-NEXT:    movzwl (%rdi,%rsi), %ecx
+; X64-AVX-NEXT:    vmovd %ecx, %xmm0
+; X64-AVX-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; X64-AVX-NEXT:    vpmaddwd {{.*}}(%rip), %xmm0, %xmm0
 ; X64-AVX-NEXT:    vmovq %xmm0, (%rax,%rsi,4)
 ; X64-AVX-NEXT:    retq
 entry:
@@ -1592,9 +1618,10 @@ define void @mul_2xi8_varconst4(i8* nocapture readonly %a, i64 %index) {
 ; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-AVX-NEXT:    movl c, %edx
-; X86-AVX-NEXT:    vpmovzxbq {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero
-; X86-AVX-NEXT:    vpmuludq {{\.LCPI.*}}, %xmm0, %xmm0
-; X86-AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X86-AVX-NEXT:    movzwl (%ecx,%eax), %ecx
+; X86-AVX-NEXT:    vmovd %ecx, %xmm0
+; X86-AVX-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; X86-AVX-NEXT:    vpmulld {{\.LCPI.*}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vmovq %xmm0, (%edx,%eax,4)
 ; X86-AVX-NEXT:    retl
 ;
@@ -1616,9 +1643,10 @@ define void @mul_2xi8_varconst4(i8* nocapture readonly %a, i64 %index) {
 ; X64-AVX-LABEL: mul_2xi8_varconst4:
 ; X64-AVX:       # %bb.0: # %entry
 ; X64-AVX-NEXT:    movq {{.*}}(%rip), %rax
-; X64-AVX-NEXT:    vpmovzxbq {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero
-; X64-AVX-NEXT:    vpmuludq {{.*}}(%rip), %xmm0, %xmm0
-; X64-AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X64-AVX-NEXT:    movzwl (%rdi,%rsi), %ecx
+; X64-AVX-NEXT:    vmovd %ecx, %xmm0
+; X64-AVX-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; X64-AVX-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
 ; X64-AVX-NEXT:    vmovq %xmm0, (%rax,%rsi,4)
 ; X64-AVX-NEXT:    retq
 entry:
@@ -1662,9 +1690,10 @@ define void @mul_2xi8_varconst5(i8* nocapture readonly %a, i64 %index) {
 ; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-AVX-NEXT:    movl c, %edx
-; X86-AVX-NEXT:    vpmovsxbq (%ecx,%eax), %xmm0
-; X86-AVX-NEXT:    vpmuludq {{\.LCPI.*}}, %xmm0, %xmm0
-; X86-AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X86-AVX-NEXT:    movzwl (%ecx,%eax), %ecx
+; X86-AVX-NEXT:    vmovd %ecx, %xmm0
+; X86-AVX-NEXT:    vpmovsxbd %xmm0, %xmm0
+; X86-AVX-NEXT:    vpmulld {{\.LCPI.*}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vmovq %xmm0, (%edx,%eax,4)
 ; X86-AVX-NEXT:    retl
 ;
@@ -1686,9 +1715,10 @@ define void @mul_2xi8_varconst5(i8* nocapture readonly %a, i64 %index) {
 ; X64-AVX-LABEL: mul_2xi8_varconst5:
 ; X64-AVX:       # %bb.0: # %entry
 ; X64-AVX-NEXT:    movq {{.*}}(%rip), %rax
-; X64-AVX-NEXT:    vpmovsxbq (%rdi,%rsi), %xmm0
-; X64-AVX-NEXT:    vpmuludq {{.*}}(%rip), %xmm0, %xmm0
-; X64-AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X64-AVX-NEXT:    movzwl (%rdi,%rsi), %ecx
+; X64-AVX-NEXT:    vmovd %ecx, %xmm0
+; X64-AVX-NEXT:    vpmovsxbd %xmm0, %xmm0
+; X64-AVX-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
 ; X64-AVX-NEXT:    vmovq %xmm0, (%rax,%rsi,4)
 ; X64-AVX-NEXT:    retq
 entry:
@@ -1732,9 +1762,10 @@ define void @mul_2xi8_varconst6(i8* nocapture readonly %a, i64 %index) {
 ; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-AVX-NEXT:    movl c, %edx
-; X86-AVX-NEXT:    vpmovsxbq (%ecx,%eax), %xmm0
-; X86-AVX-NEXT:    vpmuludq {{\.LCPI.*}}, %xmm0, %xmm0
-; X86-AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X86-AVX-NEXT:    movzwl (%ecx,%eax), %ecx
+; X86-AVX-NEXT:    vmovd %ecx, %xmm0
+; X86-AVX-NEXT:    vpmovsxbd %xmm0, %xmm0
+; X86-AVX-NEXT:    vpmulld {{\.LCPI.*}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vmovq %xmm0, (%edx,%eax,4)
 ; X86-AVX-NEXT:    retl
 ;
@@ -1756,9 +1787,10 @@ define void @mul_2xi8_varconst6(i8* nocapture readonly %a, i64 %index) {
 ; X64-AVX-LABEL: mul_2xi8_varconst6:
 ; X64-AVX:       # %bb.0: # %entry
 ; X64-AVX-NEXT:    movq {{.*}}(%rip), %rax
-; X64-AVX-NEXT:    vpmovsxbq (%rdi,%rsi), %xmm0
-; X64-AVX-NEXT:    vpmuludq {{.*}}(%rip), %xmm0, %xmm0
-; X64-AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X64-AVX-NEXT:    movzwl (%rdi,%rsi), %ecx
+; X64-AVX-NEXT:    vmovd %ecx, %xmm0
+; X64-AVX-NEXT:    vpmovsxbd %xmm0, %xmm0
+; X64-AVX-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
 ; X64-AVX-NEXT:    vmovq %xmm0, (%rax,%rsi,4)
 ; X64-AVX-NEXT:    retq
 entry:
@@ -1799,9 +1831,9 @@ define void @mul_2xi16_varconst1(i8* nocapture readonly %a, i64 %index) {
 ; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-AVX-NEXT:    movl c, %edx
-; X86-AVX-NEXT:    vpmovzxwq {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero
-; X86-AVX-NEXT:    vpmuludq {{\.LCPI.*}}, %xmm0, %xmm0
-; X86-AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X86-AVX-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-AVX-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; X86-AVX-NEXT:    vpmulld {{\.LCPI.*}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vmovq %xmm0, (%edx,%eax,4)
 ; X86-AVX-NEXT:    retl
 ;
@@ -1820,9 +1852,9 @@ define void @mul_2xi16_varconst1(i8* nocapture readonly %a, i64 %index) {
 ; X64-AVX-LABEL: mul_2xi16_varconst1:
 ; X64-AVX:       # %bb.0: # %entry
 ; X64-AVX-NEXT:    movq {{.*}}(%rip), %rax
-; X64-AVX-NEXT:    vpmovzxwq {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero
-; X64-AVX-NEXT:    vpmuludq {{.*}}(%rip), %xmm0, %xmm0
-; X64-AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X64-AVX-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X64-AVX-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; X64-AVX-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
 ; X64-AVX-NEXT:    vmovq %xmm0, (%rax,%rsi,4)
 ; X64-AVX-NEXT:    retq
 entry:
@@ -1863,9 +1895,9 @@ define void @mul_2xi16_varconst2(i8* nocapture readonly %a, i64 %index) {
 ; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-AVX-NEXT:    movl c, %edx
-; X86-AVX-NEXT:    vpmovsxwq (%ecx,%eax), %xmm0
-; X86-AVX-NEXT:    vpmuludq {{\.LCPI.*}}, %xmm0, %xmm0
-; X86-AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X86-AVX-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-AVX-NEXT:    vpmovsxwd %xmm0, %xmm0
+; X86-AVX-NEXT:    vpmulld {{\.LCPI.*}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vmovq %xmm0, (%edx,%eax,4)
 ; X86-AVX-NEXT:    retl
 ;
@@ -1884,9 +1916,9 @@ define void @mul_2xi16_varconst2(i8* nocapture readonly %a, i64 %index) {
 ; X64-AVX-LABEL: mul_2xi16_varconst2:
 ; X64-AVX:       # %bb.0: # %entry
 ; X64-AVX-NEXT:    movq {{.*}}(%rip), %rax
-; X64-AVX-NEXT:    vpmovsxwq (%rdi,%rsi), %xmm0
-; X64-AVX-NEXT:    vpmuludq {{.*}}(%rip), %xmm0, %xmm0
-; X64-AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X64-AVX-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X64-AVX-NEXT:    vpmovsxwd %xmm0, %xmm0
+; X64-AVX-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
 ; X64-AVX-NEXT:    vmovq %xmm0, (%rax,%rsi,4)
 ; X64-AVX-NEXT:    retq
 entry:
@@ -1916,9 +1948,12 @@ define void @mul_2xi16_varconst3(i8* nocapture readonly %a, i64 %index) {
 ; X86-SSE-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; X86-SSE-NEXT:    pxor %xmm1, %xmm1
 ; X86-SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; X86-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,1,3]
-; X86-SSE-NEXT:    pmuludq {{\.LCPI.*}}, %xmm0
-; X86-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X86-SSE-NEXT:    movdqa {{.*#+}} xmm1 = <0,65536,u,u>
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; X86-SSE-NEXT:    pmuludq %xmm1, %xmm0
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; X86-SSE-NEXT:    pmuludq %xmm2, %xmm1
+; X86-SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; X86-SSE-NEXT:    movq %xmm0, (%edx,%eax,4)
 ; X86-SSE-NEXT:    retl
 ;
@@ -1927,9 +1962,9 @@ define void @mul_2xi16_varconst3(i8* nocapture readonly %a, i64 %index) {
 ; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-AVX-NEXT:    movl c, %edx
-; X86-AVX-NEXT:    vpmovzxwq {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero
-; X86-AVX-NEXT:    vpmuludq {{\.LCPI.*}}, %xmm0, %xmm0
-; X86-AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X86-AVX-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-AVX-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; X86-AVX-NEXT:    vpmulld {{\.LCPI.*}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vmovq %xmm0, (%edx,%eax,4)
 ; X86-AVX-NEXT:    retl
 ;
@@ -1939,18 +1974,21 @@ define void @mul_2xi16_varconst3(i8* nocapture readonly %a, i64 %index) {
 ; X64-SSE-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; X64-SSE-NEXT:    pxor %xmm1, %xmm1
 ; X64-SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; X64-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,1,3]
-; X64-SSE-NEXT:    pmuludq {{.*}}(%rip), %xmm0
-; X64-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X64-SSE-NEXT:    movdqa {{.*#+}} xmm1 = <0,65536,u,u>
+; X64-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; X64-SSE-NEXT:    pmuludq %xmm1, %xmm0
+; X64-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; X64-SSE-NEXT:    pmuludq %xmm2, %xmm1
+; X64-SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; X64-SSE-NEXT:    movq %xmm0, (%rax,%rsi,4)
 ; X64-SSE-NEXT:    retq
 ;
 ; X64-AVX-LABEL: mul_2xi16_varconst3:
 ; X64-AVX:       # %bb.0: # %entry
 ; X64-AVX-NEXT:    movq {{.*}}(%rip), %rax
-; X64-AVX-NEXT:    vpmovzxwq {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero
-; X64-AVX-NEXT:    vpmuludq {{.*}}(%rip), %xmm0, %xmm0
-; X64-AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X64-AVX-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X64-AVX-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; X64-AVX-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
 ; X64-AVX-NEXT:    vmovq %xmm0, (%rax,%rsi,4)
 ; X64-AVX-NEXT:    retq
 entry:
@@ -1980,9 +2018,12 @@ define void @mul_2xi16_varconst4(i8* nocapture readonly %a, i64 %index) {
 ; X86-SSE-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; X86-SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,2,1,4,5,6,7]
 ; X86-SSE-NEXT:    psrad $16, %xmm0
-; X86-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,1,3]
-; X86-SSE-NEXT:    pmuludq {{\.LCPI.*}}, %xmm0
-; X86-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X86-SSE-NEXT:    movdqa {{.*#+}} xmm1 = <0,32768,u,u>
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; X86-SSE-NEXT:    pmuludq %xmm1, %xmm0
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; X86-SSE-NEXT:    pmuludq %xmm2, %xmm1
+; X86-SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; X86-SSE-NEXT:    movq %xmm0, (%edx,%eax,4)
 ; X86-SSE-NEXT:    retl
 ;
@@ -1991,9 +2032,9 @@ define void @mul_2xi16_varconst4(i8* nocapture readonly %a, i64 %index) {
 ; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-AVX-NEXT:    movl c, %edx
-; X86-AVX-NEXT:    vpmovsxwq (%ecx,%eax), %xmm0
-; X86-AVX-NEXT:    vpmuludq {{\.LCPI.*}}, %xmm0, %xmm0
-; X86-AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X86-AVX-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-AVX-NEXT:    vpmovsxwd %xmm0, %xmm0
+; X86-AVX-NEXT:    vpmulld {{\.LCPI.*}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vmovq %xmm0, (%edx,%eax,4)
 ; X86-AVX-NEXT:    retl
 ;
@@ -2003,18 +2044,21 @@ define void @mul_2xi16_varconst4(i8* nocapture readonly %a, i64 %index) {
 ; X64-SSE-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; X64-SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,2,1,4,5,6,7]
 ; X64-SSE-NEXT:    psrad $16, %xmm0
-; X64-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,1,3]
-; X64-SSE-NEXT:    pmuludq {{.*}}(%rip), %xmm0
-; X64-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X64-SSE-NEXT:    movdqa {{.*#+}} xmm1 = <0,32768,u,u>
+; X64-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; X64-SSE-NEXT:    pmuludq %xmm1, %xmm0
+; X64-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; X64-SSE-NEXT:    pmuludq %xmm2, %xmm1
+; X64-SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; X64-SSE-NEXT:    movq %xmm0, (%rax,%rsi,4)
 ; X64-SSE-NEXT:    retq
 ;
 ; X64-AVX-LABEL: mul_2xi16_varconst4:
 ; X64-AVX:       # %bb.0: # %entry
 ; X64-AVX-NEXT:    movq {{.*}}(%rip), %rax
-; X64-AVX-NEXT:    vpmovsxwq (%rdi,%rsi), %xmm0
-; X64-AVX-NEXT:    vpmuludq {{.*}}(%rip), %xmm0, %xmm0
-; X64-AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X64-AVX-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X64-AVX-NEXT:    vpmovsxwd %xmm0, %xmm0
+; X64-AVX-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
 ; X64-AVX-NEXT:    vmovq %xmm0, (%rax,%rsi,4)
 ; X64-AVX-NEXT:    retq
 entry:
@@ -2041,85 +2085,88 @@ define void @PR34947(<9 x i16>* %p0, <9 x i32>* %p1) nounwind {
 ; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-SSE-NEXT:    movdqa (%eax), %xmm5
-; X86-SSE-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; X86-SSE-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; X86-SSE-NEXT:    movdqa (%ecx), %xmm2
 ; X86-SSE-NEXT:    movdqa 16(%ecx), %xmm6
-; X86-SSE-NEXT:    pxor %xmm0, %xmm0
-; X86-SSE-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; X86-SSE-NEXT:    pxor %xmm1, %xmm1
+; X86-SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
 ; X86-SSE-NEXT:    movdqa %xmm5, %xmm4
-; X86-SSE-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
-; X86-SSE-NEXT:    punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm0[4],xmm5[5],xmm0[5],xmm5[6],xmm0[6],xmm5[7],xmm0[7]
-; X86-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[3,1,2,3]
-; X86-SSE-NEXT:    movd %xmm0, %eax
-; X86-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[3,1,2,3]
-; X86-SSE-NEXT:    movd %xmm0, %esi
+; X86-SSE-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
+; X86-SSE-NEXT:    movdqa %xmm5, %xmm3
+; X86-SSE-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
+; X86-SSE-NEXT:    movdqa %xmm5, %xmm1
+; X86-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X86-SSE-NEXT:    movd %xmm1, %eax
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm6[3,1,2,3]
+; X86-SSE-NEXT:    movd %xmm1, %esi
 ; X86-SSE-NEXT:    xorl %edx, %edx
 ; X86-SSE-NEXT:    divl %esi
-; X86-SSE-NEXT:    movd %edx, %xmm0
-; X86-SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm5[2,3,0,1]
-; X86-SSE-NEXT:    movd %xmm3, %eax
-; X86-SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm6[2,3,0,1]
-; X86-SSE-NEXT:    movd %xmm3, %esi
+; X86-SSE-NEXT:    movd %edx, %xmm1
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm7 = xmm3[2,3,0,1]
+; X86-SSE-NEXT:    movd %xmm7, %eax
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[2,3,0,1]
+; X86-SSE-NEXT:    movd %xmm7, %esi
 ; X86-SSE-NEXT:    xorl %edx, %edx
 ; X86-SSE-NEXT:    divl %esi
 ; X86-SSE-NEXT:    movd %edx, %xmm7
-; X86-SSE-NEXT:    punpckldq {{.*#+}} xmm7 = xmm7[0],xmm0[0],xmm7[1],xmm0[1]
-; X86-SSE-NEXT:    movd %xmm5, %eax
+; X86-SSE-NEXT:    punpckldq {{.*#+}} xmm7 = xmm7[0],xmm1[0],xmm7[1],xmm1[1]
+; X86-SSE-NEXT:    movd %xmm3, %eax
 ; X86-SSE-NEXT:    movd %xmm6, %esi
 ; X86-SSE-NEXT:    xorl %edx, %edx
 ; X86-SSE-NEXT:    divl %esi
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,2,3]
+; X86-SSE-NEXT:    movd %xmm3, %eax
 ; X86-SSE-NEXT:    movd %edx, %xmm3
-; X86-SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,2,3]
-; X86-SSE-NEXT:    movd %xmm5, %eax
-; X86-SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm6[1,1,2,3]
-; X86-SSE-NEXT:    movd %xmm5, %esi
-; X86-SSE-NEXT:    xorl %edx, %edx
-; X86-SSE-NEXT:    divl %esi
-; X86-SSE-NEXT:    movd %edx, %xmm5
-; X86-SSE-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1]
-; X86-SSE-NEXT:    punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm7[0]
-; X86-SSE-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[3,1,2,3]
-; X86-SSE-NEXT:    movd %xmm6, %eax
-; X86-SSE-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[3,1,2,3]
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,2,3]
 ; X86-SSE-NEXT:    movd %xmm6, %esi
 ; X86-SSE-NEXT:    xorl %edx, %edx
 ; X86-SSE-NEXT:    divl %esi
 ; X86-SSE-NEXT:    movd %edx, %xmm6
-; X86-SSE-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[2,3,0,1]
+; X86-SSE-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm6[0],xmm3[1],xmm6[1]
+; X86-SSE-NEXT:    punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm7[0]
+; X86-SSE-NEXT:    movdqa %xmm5, %xmm7
+; X86-SSE-NEXT:    psrld $16, %xmm7
 ; X86-SSE-NEXT:    movd %xmm7, %eax
-; X86-SSE-NEXT:    pshufd {{.*#+}} xmm7 = xmm2[2,3,0,1]
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm7 = xmm2[1,1,2,3]
 ; X86-SSE-NEXT:    movd %xmm7, %esi
 ; X86-SSE-NEXT:    xorl %edx, %edx
 ; X86-SSE-NEXT:    divl %esi
 ; X86-SSE-NEXT:    movd %edx, %xmm7
-; X86-SSE-NEXT:    punpckldq {{.*#+}} xmm7 = xmm7[0],xmm6[0],xmm7[1],xmm6[1]
 ; X86-SSE-NEXT:    movd %xmm4, %eax
 ; X86-SSE-NEXT:    movd %xmm2, %esi
 ; X86-SSE-NEXT:    xorl %edx, %edx
 ; X86-SSE-NEXT:    divl %esi
-; X86-SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,2,3]
+; X86-SSE-NEXT:    psrlq $48, %xmm5
+; X86-SSE-NEXT:    movd %xmm5, %eax
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[3,1,2,3]
+; X86-SSE-NEXT:    movd %xmm5, %esi
+; X86-SSE-NEXT:    movd %edx, %xmm5
+; X86-SSE-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm7[0],xmm5[1],xmm7[1]
+; X86-SSE-NEXT:    xorl %edx, %edx
+; X86-SSE-NEXT:    divl %esi
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[2,3,0,1]
 ; X86-SSE-NEXT:    movd %xmm4, %eax
 ; X86-SSE-NEXT:    movd %edx, %xmm4
-; X86-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,2,3]
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
 ; X86-SSE-NEXT:    movd %xmm2, %esi
 ; X86-SSE-NEXT:    xorl %edx, %edx
 ; X86-SSE-NEXT:    divl %esi
 ; X86-SSE-NEXT:    movd %edx, %xmm2
+; X86-SSE-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
+; X86-SSE-NEXT:    punpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm2[0]
+; X86-SSE-NEXT:    movd %xmm0, %eax
+; X86-SSE-NEXT:    shufps {{.*#+}} xmm7 = xmm7[0,0],xmm4[0,0]
+; X86-SSE-NEXT:    movdqa {{.*#+}} xmm0 = [8199,8199,8199,8199]
+; X86-SSE-NEXT:    pmuludq %xmm0, %xmm7
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm7[0,2,2,3]
+; X86-SSE-NEXT:    pmuludq %xmm0, %xmm5
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm5[0,2,2,3]
 ; X86-SSE-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
-; X86-SSE-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm7[0]
-; X86-SSE-NEXT:    movd %xmm1, %eax
-; X86-SSE-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,0],xmm6[0,0]
-; X86-SSE-NEXT:    movdqa {{.*#+}} xmm1 = [8199,8199,8199,8199]
-; X86-SSE-NEXT:    pmuludq %xmm1, %xmm4
-; X86-SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[0,2,2,3]
-; X86-SSE-NEXT:    pmuludq %xmm1, %xmm2
-; X86-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
-; X86-SSE-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
-; X86-SSE-NEXT:    shufps {{.*#+}} xmm5 = xmm5[0,0],xmm0[0,0]
-; X86-SSE-NEXT:    pmuludq %xmm1, %xmm3
-; X86-SSE-NEXT:    pmuludq %xmm1, %xmm5
+; X86-SSE-NEXT:    shufps {{.*#+}} xmm6 = xmm6[0,0],xmm1[0,0]
+; X86-SSE-NEXT:    pmuludq %xmm0, %xmm3
+; X86-SSE-NEXT:    pmuludq %xmm0, %xmm6
 ; X86-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,2,2,3]
-; X86-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm5[0,2,2,3]
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm6[0,2,2,3]
 ; X86-SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; X86-SSE-NEXT:    xorl %edx, %edx
 ; X86-SSE-NEXT:    divl 32(%ecx)
@@ -2280,92 +2327,95 @@ define void @PR34947(<9 x i16>* %p0, <9 x i32>* %p1) nounwind {
 ; X64-SSE-LABEL: PR34947:
 ; X64-SSE:       # %bb.0:
 ; X64-SSE-NEXT:    movdqa (%rdi), %xmm5
-; X64-SSE-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; X64-SSE-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; X64-SSE-NEXT:    movdqa (%rsi), %xmm2
 ; X64-SSE-NEXT:    movdqa 16(%rsi), %xmm6
-; X64-SSE-NEXT:    pxor %xmm0, %xmm0
-; X64-SSE-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; X64-SSE-NEXT:    pxor %xmm1, %xmm1
+; X64-SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
 ; X64-SSE-NEXT:    movdqa %xmm5, %xmm3
-; X64-SSE-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3]
-; X64-SSE-NEXT:    punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm0[4],xmm5[5],xmm0[5],xmm5[6],xmm0[6],xmm5[7],xmm0[7]
-; X64-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[3,1,2,3]
-; X64-SSE-NEXT:    movd %xmm0, %eax
-; X64-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[3,1,2,3]
-; X64-SSE-NEXT:    movd %xmm0, %ecx
+; X64-SSE-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
+; X64-SSE-NEXT:    movdqa %xmm5, %xmm7
+; X64-SSE-NEXT:    punpckhwd {{.*#+}} xmm7 = xmm7[4],xmm1[4],xmm7[5],xmm1[5],xmm7[6],xmm1[6],xmm7[7],xmm1[7]
+; X64-SSE-NEXT:    movdqa %xmm5, %xmm1
+; X64-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X64-SSE-NEXT:    movd %xmm1, %eax
+; X64-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm6[3,1,2,3]
+; X64-SSE-NEXT:    movd %xmm1, %ecx
 ; X64-SSE-NEXT:    xorl %edx, %edx
 ; X64-SSE-NEXT:    divl %ecx
 ; X64-SSE-NEXT:    movd %edx, %xmm8
-; X64-SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm5[2,3,0,1]
-; X64-SSE-NEXT:    movd %xmm4, %eax
-; X64-SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[2,3,0,1]
-; X64-SSE-NEXT:    movd %xmm4, %ecx
+; X64-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm7[2,3,0,1]
+; X64-SSE-NEXT:    movd %xmm1, %eax
+; X64-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm6[2,3,0,1]
+; X64-SSE-NEXT:    movd %xmm1, %ecx
 ; X64-SSE-NEXT:    xorl %edx, %edx
 ; X64-SSE-NEXT:    divl %ecx
-; X64-SSE-NEXT:    movd %edx, %xmm7
-; X64-SSE-NEXT:    punpckldq {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1]
-; X64-SSE-NEXT:    movd %xmm5, %eax
+; X64-SSE-NEXT:    movd %edx, %xmm1
+; X64-SSE-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm8[0],xmm1[1],xmm8[1]
+; X64-SSE-NEXT:    movd %xmm7, %eax
 ; X64-SSE-NEXT:    movd %xmm6, %ecx
 ; X64-SSE-NEXT:    xorl %edx, %edx
 ; X64-SSE-NEXT:    divl %ecx
 ; X64-SSE-NEXT:    movd %edx, %xmm4
-; X64-SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,2,3]
-; X64-SSE-NEXT:    movd %xmm5, %eax
-; X64-SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm6[1,1,2,3]
-; X64-SSE-NEXT:    movd %xmm5, %ecx
-; X64-SSE-NEXT:    xorl %edx, %edx
-; X64-SSE-NEXT:    divl %ecx
-; X64-SSE-NEXT:    movd %edx, %xmm5
-; X64-SSE-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
-; X64-SSE-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm7[0]
-; X64-SSE-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[3,1,2,3]
-; X64-SSE-NEXT:    movd %xmm6, %eax
-; X64-SSE-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[3,1,2,3]
+; X64-SSE-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,2,3]
+; X64-SSE-NEXT:    movd %xmm7, %eax
+; X64-SSE-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,2,3]
 ; X64-SSE-NEXT:    movd %xmm6, %ecx
 ; X64-SSE-NEXT:    xorl %edx, %edx
 ; X64-SSE-NEXT:    divl %ecx
 ; X64-SSE-NEXT:    movd %edx, %xmm6
-; X64-SSE-NEXT:    pshufd {{.*#+}} xmm7 = xmm3[2,3,0,1]
-; X64-SSE-NEXT:    movd %xmm7, %eax
-; X64-SSE-NEXT:    pshufd {{.*#+}} xmm7 = xmm2[2,3,0,1]
-; X64-SSE-NEXT:    movd %xmm7, %ecx
+; X64-SSE-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm6[0],xmm4[1],xmm6[1]
+; X64-SSE-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm1[0]
+; X64-SSE-NEXT:    movdqa %xmm5, %xmm1
+; X64-SSE-NEXT:    psrld $16, %xmm1
+; X64-SSE-NEXT:    movd %xmm1, %eax
+; X64-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,2,3]
+; X64-SSE-NEXT:    movd %xmm1, %ecx
 ; X64-SSE-NEXT:    xorl %edx, %edx
 ; X64-SSE-NEXT:    divl %ecx
 ; X64-SSE-NEXT:    movd %edx, %xmm7
-; X64-SSE-NEXT:    punpckldq {{.*#+}} xmm7 = xmm7[0],xmm6[0],xmm7[1],xmm6[1]
 ; X64-SSE-NEXT:    movd %xmm3, %eax
 ; X64-SSE-NEXT:    movd %xmm2, %ecx
 ; X64-SSE-NEXT:    xorl %edx, %edx
 ; X64-SSE-NEXT:    divl %ecx
-; X64-SSE-NEXT:    movd %edx, %xmm0
-; X64-SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,2,3]
+; X64-SSE-NEXT:    movd %edx, %xmm1
+; X64-SSE-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm7[0],xmm1[1],xmm7[1]
+; X64-SSE-NEXT:    psrlq $48, %xmm5
+; X64-SSE-NEXT:    movd %xmm5, %eax
+; X64-SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[3,1,2,3]
+; X64-SSE-NEXT:    movd %xmm5, %ecx
+; X64-SSE-NEXT:    xorl %edx, %edx
+; X64-SSE-NEXT:    divl %ecx
+; X64-SSE-NEXT:    movd %edx, %xmm5
+; X64-SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[2,3,0,1]
 ; X64-SSE-NEXT:    movd %xmm3, %eax
-; X64-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,2,3]
+; X64-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
 ; X64-SSE-NEXT:    movd %xmm2, %ecx
 ; X64-SSE-NEXT:    xorl %edx, %edx
 ; X64-SSE-NEXT:    divl %ecx
 ; X64-SSE-NEXT:    movd %edx, %xmm2
-; X64-SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; X64-SSE-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm7[0]
-; X64-SSE-NEXT:    movd %xmm1, %eax
+; X64-SSE-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
+; X64-SSE-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; X64-SSE-NEXT:    movd %xmm0, %eax
 ; X64-SSE-NEXT:    xorl %edx, %edx
 ; X64-SSE-NEXT:    divl 32(%rsi)
-; X64-SSE-NEXT:    movdqa {{.*#+}} xmm1 = [8199,8199,8199,8199]
-; X64-SSE-NEXT:    pmuludq %xmm1, %xmm0
-; X64-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; X64-SSE-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,0],xmm6[0,0]
-; X64-SSE-NEXT:    pmuludq %xmm1, %xmm2
-; X64-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
-; X64-SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; X64-SSE-NEXT:    pmuludq %xmm1, %xmm4
+; X64-SSE-NEXT:    shufps {{.*#+}} xmm7 = xmm7[0,0],xmm5[0,0]
+; X64-SSE-NEXT:    movdqa {{.*#+}} xmm0 = [8199,8199,8199,8199]
+; X64-SSE-NEXT:    pmuludq %xmm0, %xmm7
+; X64-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm7[0,2,2,3]
+; X64-SSE-NEXT:    pmuludq %xmm0, %xmm1
+; X64-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; X64-SSE-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; X64-SSE-NEXT:    pmuludq %xmm0, %xmm4
 ; X64-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[0,2,2,3]
-; X64-SSE-NEXT:    shufps {{.*#+}} xmm5 = xmm5[0,0],xmm8[0,0]
-; X64-SSE-NEXT:    pmuludq %xmm1, %xmm5
-; X64-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm5[0,2,2,3]
-; X64-SSE-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; X64-SSE-NEXT:    shufps {{.*#+}} xmm6 = xmm6[0,0],xmm8[0,0]
+; X64-SSE-NEXT:    pmuludq %xmm0, %xmm6
+; X64-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,2,2,3]
+; X64-SSE-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
 ; X64-SSE-NEXT:    imull $8199, %edx, %eax # imm = 0x2007
 ; X64-SSE-NEXT:    movl %eax, (%rax)
 ; X64-SSE-NEXT:    movdqa %xmm2, (%rax)
-; X64-SSE-NEXT:    movdqa %xmm0, (%rax)
+; X64-SSE-NEXT:    movdqa %xmm1, (%rax)
 ; X64-SSE-NEXT:    retq
 ;
 ; X64-AVX1-LABEL: PR34947:
diff --git a/llvm/test/CodeGen/X86/shrinkwrap-hang.ll b/llvm/test/CodeGen/X86/shrinkwrap-hang.ll
index e1e4eefb3..00b076f72 100644
--- a/llvm/test/CodeGen/X86/shrinkwrap-hang.ll
+++ b/llvm/test/CodeGen/X86/shrinkwrap-hang.ll
@@ -29,4 +29,4 @@ if.end3:                                          ; preds = %if.end
   ret void
 }
 
-attributes #0 = { norecurse nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { norecurse nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/X86/shuffle-combine-crash.ll b/llvm/test/CodeGen/X86/shuffle-combine-crash.ll
index 06fcaa973..4141e7eda 100644
--- a/llvm/test/CodeGen/X86/shuffle-combine-crash.ll
+++ b/llvm/test/CodeGen/X86/shuffle-combine-crash.ll
@@ -1,4 +1,5 @@
-; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=corei7 
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=corei7 | FileCheck %s
 
 ; Verify that DAGCombiner does not crash when checking if it is
 ; safe to fold the shuffles in function @sample_test according to rule
@@ -15,6 +16,18 @@
 ; As a consequence, compiling the function below would have caused a crash.
 
 define void @sample_test() {
+; CHECK-LABEL: sample_test:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    testb %al, %al
+; CHECK-NEXT:    jne .LBB0_2
+; CHECK-NEXT:  # %bb.1:
+; CHECK-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; CHECK-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; CHECK-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; CHECK-NEXT:    movd %xmm0, (%rax)
+; CHECK-NEXT:  .LBB0_2:
+; CHECK-NEXT:    retq
   br i1 undef, label %5, label %1
 
 ; <label>:1                                       ; preds = %0
diff --git a/llvm/test/CodeGen/X86/shuffle-strided-with-offset-128.ll b/llvm/test/CodeGen/X86/shuffle-strided-with-offset-128.ll
index 20e3432f9..710f5bae9 100644
--- a/llvm/test/CodeGen/X86/shuffle-strided-with-offset-128.ll
+++ b/llvm/test/CodeGen/X86/shuffle-strided-with-offset-128.ll
@@ -44,32 +44,12 @@ define void @shuffle_v16i8_to_v8i8_1(<16 x i8>* %L, <8 x i8>* %S) nounwind {
 ; AVX-NEXT:    vmovq %xmm0, (%rsi)
 ; AVX-NEXT:    retq
 ;
-; AVX512F-LABEL: shuffle_v16i8_to_v8i8_1:
-; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[1,3,5,7,9,11,13,15,u,u,u,u,u,u,u,u]
-; AVX512F-NEXT:    vmovq %xmm0, (%rsi)
-; AVX512F-NEXT:    retq
-;
-; AVX512VL-LABEL: shuffle_v16i8_to_v8i8_1:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512VL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[1,3,5,7,9,11,13,15,u,u,u,u,u,u,u,u]
-; AVX512VL-NEXT:    vmovq %xmm0, (%rsi)
-; AVX512VL-NEXT:    retq
-;
-; AVX512BW-LABEL: shuffle_v16i8_to_v8i8_1:
-; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[1,3,5,7,9,11,13,15,u,u,u,u,u,u,u,u]
-; AVX512BW-NEXT:    vmovq %xmm0, (%rsi)
-; AVX512BW-NEXT:    retq
-;
-; AVX512BWVL-LABEL: shuffle_v16i8_to_v8i8_1:
-; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vpsrlw $8, (%rdi), %xmm0
-; AVX512BWVL-NEXT:    vpmovwb %xmm0, (%rsi)
-; AVX512BWVL-NEXT:    retq
+; AVX512-LABEL: shuffle_v16i8_to_v8i8_1:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[1,3,5,7,9,11,13,15,u,u,u,u,u,u,u,u]
+; AVX512-NEXT:    vmovq %xmm0, (%rsi)
+; AVX512-NEXT:    retq
   %vec = load <16 x i8>, <16 x i8>* %L
   %strided.vec = shufflevector <16 x i8> %vec, <16 x i8> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
   store <8 x i8> %strided.vec, <8 x i8>* %S
@@ -100,31 +80,12 @@ define void @shuffle_v8i16_to_v4i16_1(<8 x i16>* %L, <4 x i16>* %S) nounwind {
 ; AVX-NEXT:    vmovq %xmm0, (%rsi)
 ; AVX-NEXT:    retq
 ;
-; AVX512F-LABEL: shuffle_v8i16_to_v4i16_1:
-; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[2,3,6,7,10,11,14,15,14,15,10,11,12,13,14,15]
-; AVX512F-NEXT:    vmovq %xmm0, (%rsi)
-; AVX512F-NEXT:    retq
-;
-; AVX512VL-LABEL: shuffle_v8i16_to_v4i16_1:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsrld $16, (%rdi), %xmm0
-; AVX512VL-NEXT:    vpmovdw %xmm0, (%rsi)
-; AVX512VL-NEXT:    retq
-;
-; AVX512BW-LABEL: shuffle_v8i16_to_v4i16_1:
-; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[2,3,6,7,10,11,14,15,14,15,10,11,12,13,14,15]
-; AVX512BW-NEXT:    vmovq %xmm0, (%rsi)
-; AVX512BW-NEXT:    retq
-;
-; AVX512BWVL-LABEL: shuffle_v8i16_to_v4i16_1:
-; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vpsrld $16, (%rdi), %xmm0
-; AVX512BWVL-NEXT:    vpmovdw %xmm0, (%rsi)
-; AVX512BWVL-NEXT:    retq
+; AVX512-LABEL: shuffle_v8i16_to_v4i16_1:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[2,3,6,7,10,11,14,15,14,15,10,11,12,13,14,15]
+; AVX512-NEXT:    vmovq %xmm0, (%rsi)
+; AVX512-NEXT:    retq
   %vec = load <8 x i16>, <8 x i16>* %L
   %strided.vec = shufflevector <8 x i16> %vec, <8 x i16> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
   store <4 x i16> %strided.vec, <4 x i16>* %S
@@ -144,29 +105,11 @@ define void @shuffle_v4i32_to_v2i32_1(<4 x i32>* %L, <2 x i32>* %S) nounwind {
 ; AVX-NEXT:    vmovlps %xmm0, (%rsi)
 ; AVX-NEXT:    retq
 ;
-; AVX512F-LABEL: shuffle_v4i32_to_v2i32_1:
-; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpermilps {{.*#+}} xmm0 = mem[1,3,2,3]
-; AVX512F-NEXT:    vmovlps %xmm0, (%rsi)
-; AVX512F-NEXT:    retq
-;
-; AVX512VL-LABEL: shuffle_v4i32_to_v2i32_1:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpshufd {{.*#+}} xmm0 = mem[1,1,3,3]
-; AVX512VL-NEXT:    vpmovqd %xmm0, (%rsi)
-; AVX512VL-NEXT:    retq
-;
-; AVX512BW-LABEL: shuffle_v4i32_to_v2i32_1:
-; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpermilps {{.*#+}} xmm0 = mem[1,3,2,3]
-; AVX512BW-NEXT:    vmovlps %xmm0, (%rsi)
-; AVX512BW-NEXT:    retq
-;
-; AVX512BWVL-LABEL: shuffle_v4i32_to_v2i32_1:
-; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm0 = mem[1,1,3,3]
-; AVX512BWVL-NEXT:    vpmovqd %xmm0, (%rsi)
-; AVX512BWVL-NEXT:    retq
+; AVX512-LABEL: shuffle_v4i32_to_v2i32_1:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpermilps {{.*#+}} xmm0 = mem[1,3,2,3]
+; AVX512-NEXT:    vmovlps %xmm0, (%rsi)
+; AVX512-NEXT:    retq
   %vec = load <4 x i32>, <4 x i32>* %L
   %strided.vec = shufflevector <4 x i32> %vec, <4 x i32> undef, <2 x i32> <i32 1, i32 3>
   store <2 x i32> %strided.vec, <2 x i32>* %S
@@ -204,32 +147,12 @@ define void @shuffle_v16i8_to_v4i8_1(<16 x i8>* %L, <4 x i8>* %S) nounwind {
 ; AVX-NEXT:    vmovd %xmm0, (%rsi)
 ; AVX-NEXT:    retq
 ;
-; AVX512F-LABEL: shuffle_v16i8_to_v4i8_1:
-; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[1,5,9,13,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512F-NEXT:    vmovd %xmm0, (%rsi)
-; AVX512F-NEXT:    retq
-;
-; AVX512VL-LABEL: shuffle_v16i8_to_v4i8_1:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512VL-NEXT:    vpsrlw $8, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpmovdb %xmm0, (%rsi)
-; AVX512VL-NEXT:    retq
-;
-; AVX512BW-LABEL: shuffle_v16i8_to_v4i8_1:
-; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[1,5,9,13,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512BW-NEXT:    vmovd %xmm0, (%rsi)
-; AVX512BW-NEXT:    retq
-;
-; AVX512BWVL-LABEL: shuffle_v16i8_to_v4i8_1:
-; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vpsrlw $8, (%rdi), %xmm0
-; AVX512BWVL-NEXT:    vpmovdb %xmm0, (%rsi)
-; AVX512BWVL-NEXT:    retq
+; AVX512-LABEL: shuffle_v16i8_to_v4i8_1:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[1,5,9,13,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512-NEXT:    vmovd %xmm0, (%rsi)
+; AVX512-NEXT:    retq
   %vec = load <16 x i8>, <16 x i8>* %L
   %strided.vec = shufflevector <16 x i8> %vec, <16 x i8> undef, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
   store <4 x i8> %strided.vec, <4 x i8>* %S
@@ -263,31 +186,12 @@ define void @shuffle_v16i8_to_v4i8_2(<16 x i8>* %L, <4 x i8>* %S) nounwind {
 ; AVX-NEXT:    vmovd %xmm0, (%rsi)
 ; AVX-NEXT:    retq
 ;
-; AVX512F-LABEL: shuffle_v16i8_to_v4i8_2:
-; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[2,6,10,14,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512F-NEXT:    vmovd %xmm0, (%rsi)
-; AVX512F-NEXT:    retq
-;
-; AVX512VL-LABEL: shuffle_v16i8_to_v4i8_2:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsrld $16, (%rdi), %xmm0
-; AVX512VL-NEXT:    vpmovdb %xmm0, (%rsi)
-; AVX512VL-NEXT:    retq
-;
-; AVX512BW-LABEL: shuffle_v16i8_to_v4i8_2:
-; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[2,6,10,14,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512BW-NEXT:    vmovd %xmm0, (%rsi)
-; AVX512BW-NEXT:    retq
-;
-; AVX512BWVL-LABEL: shuffle_v16i8_to_v4i8_2:
-; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vpsrld $16, (%rdi), %xmm0
-; AVX512BWVL-NEXT:    vpmovdb %xmm0, (%rsi)
-; AVX512BWVL-NEXT:    retq
+; AVX512-LABEL: shuffle_v16i8_to_v4i8_2:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[2,6,10,14,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512-NEXT:    vmovd %xmm0, (%rsi)
+; AVX512-NEXT:    retq
   %vec = load <16 x i8>, <16 x i8>* %L
   %strided.vec = shufflevector <16 x i8> %vec, <16 x i8> undef, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
   store <4 x i8> %strided.vec, <4 x i8>* %S
@@ -325,31 +229,12 @@ define void @shuffle_v16i8_to_v4i8_3(<16 x i8>* %L, <4 x i8>* %S) nounwind {
 ; AVX-NEXT:    vmovd %xmm0, (%rsi)
 ; AVX-NEXT:    retq
 ;
-; AVX512F-LABEL: shuffle_v16i8_to_v4i8_3:
-; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[3,7,11,15,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512F-NEXT:    vmovd %xmm0, (%rsi)
-; AVX512F-NEXT:    retq
-;
-; AVX512VL-LABEL: shuffle_v16i8_to_v4i8_3:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsrld $24, (%rdi), %xmm0
-; AVX512VL-NEXT:    vpmovdb %xmm0, (%rsi)
-; AVX512VL-NEXT:    retq
-;
-; AVX512BW-LABEL: shuffle_v16i8_to_v4i8_3:
-; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[3,7,11,15,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512BW-NEXT:    vmovd %xmm0, (%rsi)
-; AVX512BW-NEXT:    retq
-;
-; AVX512BWVL-LABEL: shuffle_v16i8_to_v4i8_3:
-; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vpsrld $24, (%rdi), %xmm0
-; AVX512BWVL-NEXT:    vpmovdb %xmm0, (%rsi)
-; AVX512BWVL-NEXT:    retq
+; AVX512-LABEL: shuffle_v16i8_to_v4i8_3:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[3,7,11,15,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512-NEXT:    vmovd %xmm0, (%rsi)
+; AVX512-NEXT:    retq
   %vec = load <16 x i8>, <16 x i8>* %L
   %strided.vec = shufflevector <16 x i8> %vec, <16 x i8> undef, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
   store <4 x i8> %strided.vec, <4 x i8>* %S
@@ -394,8 +279,9 @@ define void @shuffle_v8i16_to_v2i16_1(<8 x i16>* %L, <2 x i16>* %S) nounwind {
 ;
 ; AVX512VL-LABEL: shuffle_v8i16_to_v2i16_1:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsrld $16, (%rdi), %xmm0
-; AVX512VL-NEXT:    vpmovqw %xmm0, (%rsi)
+; AVX512VL-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512VL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[2,3,10,11,8,9,10,11,8,9,10,11,12,13,14,15]
+; AVX512VL-NEXT:    vmovd %xmm0, (%rsi)
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: shuffle_v8i16_to_v2i16_1:
@@ -407,8 +293,9 @@ define void @shuffle_v8i16_to_v2i16_1(<8 x i16>* %L, <2 x i16>* %S) nounwind {
 ;
 ; AVX512BWVL-LABEL: shuffle_v8i16_to_v2i16_1:
 ; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vpsrld $16, (%rdi), %xmm0
-; AVX512BWVL-NEXT:    vpmovqw %xmm0, (%rsi)
+; AVX512BWVL-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512BWVL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[2,3,10,11,8,9,10,11,8,9,10,11,12,13,14,15]
+; AVX512BWVL-NEXT:    vmovd %xmm0, (%rsi)
 ; AVX512BWVL-NEXT:    retq
   %vec = load <8 x i16>, <8 x i16>* %L
   %strided.vec = shufflevector <8 x i16> %vec, <8 x i16> undef, <2 x i32> <i32 1, i32 5>
@@ -454,8 +341,9 @@ define void @shuffle_v8i16_to_v2i16_2(<8 x i16>* %L, <2 x i16>* %S) nounwind {
 ;
 ; AVX512VL-LABEL: shuffle_v8i16_to_v2i16_2:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpshufd {{.*#+}} xmm0 = mem[1,1,3,3]
-; AVX512VL-NEXT:    vpmovqw %xmm0, (%rsi)
+; AVX512VL-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512VL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[4,5,12,13,4,5,6,7,8,9,10,11,12,13,14,15]
+; AVX512VL-NEXT:    vmovd %xmm0, (%rsi)
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: shuffle_v8i16_to_v2i16_2:
@@ -467,8 +355,9 @@ define void @shuffle_v8i16_to_v2i16_2(<8 x i16>* %L, <2 x i16>* %S) nounwind {
 ;
 ; AVX512BWVL-LABEL: shuffle_v8i16_to_v2i16_2:
 ; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm0 = mem[1,1,3,3]
-; AVX512BWVL-NEXT:    vpmovqw %xmm0, (%rsi)
+; AVX512BWVL-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512BWVL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[4,5,12,13,4,5,6,7,8,9,10,11,12,13,14,15]
+; AVX512BWVL-NEXT:    vmovd %xmm0, (%rsi)
 ; AVX512BWVL-NEXT:    retq
   %vec = load <8 x i16>, <8 x i16>* %L
   %strided.vec = shufflevector <8 x i16> %vec, <8 x i16> undef, <2 x i32> <i32 2, i32 6>
@@ -514,8 +403,9 @@ define void @shuffle_v8i16_to_v2i16_3(<8 x i16>* %L, <2 x i16>* %S) nounwind {
 ;
 ; AVX512VL-LABEL: shuffle_v8i16_to_v2i16_3:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsrlq $48, (%rdi), %xmm0
-; AVX512VL-NEXT:    vpmovqw %xmm0, (%rsi)
+; AVX512VL-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512VL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[6,7,14,15,4,5,6,7,8,9,10,11,12,13,14,15]
+; AVX512VL-NEXT:    vmovd %xmm0, (%rsi)
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: shuffle_v8i16_to_v2i16_3:
@@ -527,8 +417,9 @@ define void @shuffle_v8i16_to_v2i16_3(<8 x i16>* %L, <2 x i16>* %S) nounwind {
 ;
 ; AVX512BWVL-LABEL: shuffle_v8i16_to_v2i16_3:
 ; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vpsrlq $48, (%rdi), %xmm0
-; AVX512BWVL-NEXT:    vpmovqw %xmm0, (%rsi)
+; AVX512BWVL-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512BWVL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[6,7,14,15,4,5,6,7,8,9,10,11,12,13,14,15]
+; AVX512BWVL-NEXT:    vmovd %xmm0, (%rsi)
 ; AVX512BWVL-NEXT:    retq
   %vec = load <8 x i16>, <8 x i16>* %L
   %strided.vec = shufflevector <8 x i16> %vec, <8 x i16> undef, <2 x i32> <i32 3, i32 7>
@@ -565,32 +456,12 @@ define void @shuffle_v16i8_to_v2i8_1(<16 x i8>* %L, <2 x i8>* %S) nounwind {
 ; AVX-NEXT:    vpextrw $0, %xmm0, (%rsi)
 ; AVX-NEXT:    retq
 ;
-; AVX512F-LABEL: shuffle_v16i8_to_v2i8_1:
-; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[1,9,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512F-NEXT:    vpextrw $0, %xmm0, (%rsi)
-; AVX512F-NEXT:    retq
-;
-; AVX512VL-LABEL: shuffle_v16i8_to_v2i8_1:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512VL-NEXT:    vpsrlw $8, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpmovqb %xmm0, (%rsi)
-; AVX512VL-NEXT:    retq
-;
-; AVX512BW-LABEL: shuffle_v16i8_to_v2i8_1:
-; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[1,9,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512BW-NEXT:    vpextrw $0, %xmm0, (%rsi)
-; AVX512BW-NEXT:    retq
-;
-; AVX512BWVL-LABEL: shuffle_v16i8_to_v2i8_1:
-; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vpsrlw $8, (%rdi), %xmm0
-; AVX512BWVL-NEXT:    vpmovqb %xmm0, (%rsi)
-; AVX512BWVL-NEXT:    retq
+; AVX512-LABEL: shuffle_v16i8_to_v2i8_1:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[1,9,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512-NEXT:    vpextrw $0, %xmm0, (%rsi)
+; AVX512-NEXT:    retq
   %vec = load <16 x i8>, <16 x i8>* %L
   %strided.vec = shufflevector <16 x i8> %vec, <16 x i8> undef, <2 x i32> <i32 1, i32 9>
   store <2 x i8> %strided.vec, <2 x i8>* %S
@@ -623,31 +494,12 @@ define void @shuffle_v16i8_to_v2i8_2(<16 x i8>* %L, <2 x i8>* %S) nounwind {
 ; AVX-NEXT:    vpextrw $0, %xmm0, (%rsi)
 ; AVX-NEXT:    retq
 ;
-; AVX512F-LABEL: shuffle_v16i8_to_v2i8_2:
-; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[2,10,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512F-NEXT:    vpextrw $0, %xmm0, (%rsi)
-; AVX512F-NEXT:    retq
-;
-; AVX512VL-LABEL: shuffle_v16i8_to_v2i8_2:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsrld $16, (%rdi), %xmm0
-; AVX512VL-NEXT:    vpmovqb %xmm0, (%rsi)
-; AVX512VL-NEXT:    retq
-;
-; AVX512BW-LABEL: shuffle_v16i8_to_v2i8_2:
-; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[2,10,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512BW-NEXT:    vpextrw $0, %xmm0, (%rsi)
-; AVX512BW-NEXT:    retq
-;
-; AVX512BWVL-LABEL: shuffle_v16i8_to_v2i8_2:
-; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vpsrld $16, (%rdi), %xmm0
-; AVX512BWVL-NEXT:    vpmovqb %xmm0, (%rsi)
-; AVX512BWVL-NEXT:    retq
+; AVX512-LABEL: shuffle_v16i8_to_v2i8_2:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[2,10,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512-NEXT:    vpextrw $0, %xmm0, (%rsi)
+; AVX512-NEXT:    retq
   %vec = load <16 x i8>, <16 x i8>* %L
   %strided.vec = shufflevector <16 x i8> %vec, <16 x i8> undef, <2 x i32> <i32 2, i32 10>
   store <2 x i8> %strided.vec, <2 x i8>* %S
@@ -683,31 +535,12 @@ define void @shuffle_v16i8_to_v2i8_3(<16 x i8>* %L, <2 x i8>* %S) nounwind {
 ; AVX-NEXT:    vpextrw $0, %xmm0, (%rsi)
 ; AVX-NEXT:    retq
 ;
-; AVX512F-LABEL: shuffle_v16i8_to_v2i8_3:
-; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[3,11,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512F-NEXT:    vpextrw $0, %xmm0, (%rsi)
-; AVX512F-NEXT:    retq
-;
-; AVX512VL-LABEL: shuffle_v16i8_to_v2i8_3:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsrld $24, (%rdi), %xmm0
-; AVX512VL-NEXT:    vpmovqb %xmm0, (%rsi)
-; AVX512VL-NEXT:    retq
-;
-; AVX512BW-LABEL: shuffle_v16i8_to_v2i8_3:
-; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[3,11,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512BW-NEXT:    vpextrw $0, %xmm0, (%rsi)
-; AVX512BW-NEXT:    retq
-;
-; AVX512BWVL-LABEL: shuffle_v16i8_to_v2i8_3:
-; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vpsrld $24, (%rdi), %xmm0
-; AVX512BWVL-NEXT:    vpmovqb %xmm0, (%rsi)
-; AVX512BWVL-NEXT:    retq
+; AVX512-LABEL: shuffle_v16i8_to_v2i8_3:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[3,11,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512-NEXT:    vpextrw $0, %xmm0, (%rsi)
+; AVX512-NEXT:    retq
   %vec = load <16 x i8>, <16 x i8>* %L
   %strided.vec = shufflevector <16 x i8> %vec, <16 x i8> undef, <2 x i32> <i32 3, i32 11>
   store <2 x i8> %strided.vec, <2 x i8>* %S
@@ -740,31 +573,12 @@ define void @shuffle_v16i8_to_v2i8_4(<16 x i8>* %L, <2 x i8>* %S) nounwind {
 ; AVX-NEXT:    vpextrw $0, %xmm0, (%rsi)
 ; AVX-NEXT:    retq
 ;
-; AVX512F-LABEL: shuffle_v16i8_to_v2i8_4:
-; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[4,12,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512F-NEXT:    vpextrw $0, %xmm0, (%rsi)
-; AVX512F-NEXT:    retq
-;
-; AVX512VL-LABEL: shuffle_v16i8_to_v2i8_4:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpshufd {{.*#+}} xmm0 = mem[1,1,3,3]
-; AVX512VL-NEXT:    vpmovqb %xmm0, (%rsi)
-; AVX512VL-NEXT:    retq
-;
-; AVX512BW-LABEL: shuffle_v16i8_to_v2i8_4:
-; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[4,12,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512BW-NEXT:    vpextrw $0, %xmm0, (%rsi)
-; AVX512BW-NEXT:    retq
-;
-; AVX512BWVL-LABEL: shuffle_v16i8_to_v2i8_4:
-; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm0 = mem[1,1,3,3]
-; AVX512BWVL-NEXT:    vpmovqb %xmm0, (%rsi)
-; AVX512BWVL-NEXT:    retq
+; AVX512-LABEL: shuffle_v16i8_to_v2i8_4:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[4,12,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512-NEXT:    vpextrw $0, %xmm0, (%rsi)
+; AVX512-NEXT:    retq
   %vec = load <16 x i8>, <16 x i8>* %L
   %strided.vec = shufflevector <16 x i8> %vec, <16 x i8> undef, <2 x i32> <i32 4, i32 12>
   store <2 x i8> %strided.vec, <2 x i8>* %S
@@ -800,31 +614,12 @@ define void @shuffle_v16i8_to_v2i8_5(<16 x i8>* %L, <2 x i8>* %S) nounwind {
 ; AVX-NEXT:    vpextrw $0, %xmm0, (%rsi)
 ; AVX-NEXT:    retq
 ;
-; AVX512F-LABEL: shuffle_v16i8_to_v2i8_5:
-; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[5,13,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512F-NEXT:    vpextrw $0, %xmm0, (%rsi)
-; AVX512F-NEXT:    retq
-;
-; AVX512VL-LABEL: shuffle_v16i8_to_v2i8_5:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsrlq $40, (%rdi), %xmm0
-; AVX512VL-NEXT:    vpmovqb %xmm0, (%rsi)
-; AVX512VL-NEXT:    retq
-;
-; AVX512BW-LABEL: shuffle_v16i8_to_v2i8_5:
-; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[5,13,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512BW-NEXT:    vpextrw $0, %xmm0, (%rsi)
-; AVX512BW-NEXT:    retq
-;
-; AVX512BWVL-LABEL: shuffle_v16i8_to_v2i8_5:
-; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vpsrlq $40, (%rdi), %xmm0
-; AVX512BWVL-NEXT:    vpmovqb %xmm0, (%rsi)
-; AVX512BWVL-NEXT:    retq
+; AVX512-LABEL: shuffle_v16i8_to_v2i8_5:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[5,13,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512-NEXT:    vpextrw $0, %xmm0, (%rsi)
+; AVX512-NEXT:    retq
   %vec = load <16 x i8>, <16 x i8>* %L
   %strided.vec = shufflevector <16 x i8> %vec, <16 x i8> undef, <2 x i32> <i32 5, i32 13>
   store <2 x i8> %strided.vec, <2 x i8>* %S
@@ -857,31 +652,12 @@ define void @shuffle_v16i8_to_v2i8_6(<16 x i8>* %L, <2 x i8>* %S) nounwind {
 ; AVX-NEXT:    vpextrw $0, %xmm0, (%rsi)
 ; AVX-NEXT:    retq
 ;
-; AVX512F-LABEL: shuffle_v16i8_to_v2i8_6:
-; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[6,14,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512F-NEXT:    vpextrw $0, %xmm0, (%rsi)
-; AVX512F-NEXT:    retq
-;
-; AVX512VL-LABEL: shuffle_v16i8_to_v2i8_6:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsrlq $48, (%rdi), %xmm0
-; AVX512VL-NEXT:    vpmovqb %xmm0, (%rsi)
-; AVX512VL-NEXT:    retq
-;
-; AVX512BW-LABEL: shuffle_v16i8_to_v2i8_6:
-; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[6,14,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512BW-NEXT:    vpextrw $0, %xmm0, (%rsi)
-; AVX512BW-NEXT:    retq
-;
-; AVX512BWVL-LABEL: shuffle_v16i8_to_v2i8_6:
-; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vpsrlq $48, (%rdi), %xmm0
-; AVX512BWVL-NEXT:    vpmovqb %xmm0, (%rsi)
-; AVX512BWVL-NEXT:    retq
+; AVX512-LABEL: shuffle_v16i8_to_v2i8_6:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[6,14,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512-NEXT:    vpextrw $0, %xmm0, (%rsi)
+; AVX512-NEXT:    retq
   %vec = load <16 x i8>, <16 x i8>* %L
   %strided.vec = shufflevector <16 x i8> %vec, <16 x i8> undef, <2 x i32> <i32 6, i32 14>
   store <2 x i8> %strided.vec, <2 x i8>* %S
@@ -917,31 +693,12 @@ define void @shuffle_v16i8_to_v2i8_7(<16 x i8>* %L, <2 x i8>* %S) nounwind {
 ; AVX-NEXT:    vpextrw $0, %xmm0, (%rsi)
 ; AVX-NEXT:    retq
 ;
-; AVX512F-LABEL: shuffle_v16i8_to_v2i8_7:
-; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[7,15,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512F-NEXT:    vpextrw $0, %xmm0, (%rsi)
-; AVX512F-NEXT:    retq
-;
-; AVX512VL-LABEL: shuffle_v16i8_to_v2i8_7:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsrlq $56, (%rdi), %xmm0
-; AVX512VL-NEXT:    vpmovqb %xmm0, (%rsi)
-; AVX512VL-NEXT:    retq
-;
-; AVX512BW-LABEL: shuffle_v16i8_to_v2i8_7:
-; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[7,15,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512BW-NEXT:    vpextrw $0, %xmm0, (%rsi)
-; AVX512BW-NEXT:    retq
-;
-; AVX512BWVL-LABEL: shuffle_v16i8_to_v2i8_7:
-; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vpsrlq $56, (%rdi), %xmm0
-; AVX512BWVL-NEXT:    vpmovqb %xmm0, (%rsi)
-; AVX512BWVL-NEXT:    retq
+; AVX512-LABEL: shuffle_v16i8_to_v2i8_7:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[7,15,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512-NEXT:    vpextrw $0, %xmm0, (%rsi)
+; AVX512-NEXT:    retq
   %vec = load <16 x i8>, <16 x i8>* %L
   %strided.vec = shufflevector <16 x i8> %vec, <16 x i8> undef, <2 x i32> <i32 7, i32 15>
   store <2 x i8> %strided.vec, <2 x i8>* %S
diff --git a/llvm/test/CodeGen/X86/shuffle-strided-with-offset-256.ll b/llvm/test/CodeGen/X86/shuffle-strided-with-offset-256.ll
index 8cbcfc4a1..8c2e5d971 100644
--- a/llvm/test/CodeGen/X86/shuffle-strided-with-offset-256.ll
+++ b/llvm/test/CodeGen/X86/shuffle-strided-with-offset-256.ll
@@ -125,49 +125,16 @@ define void @shuffle_v32i8_to_v8i8_1(<32 x i8>* %L, <8 x i8>* %S) nounwind {
 ; AVX-NEXT:    vmovq %xmm0, (%rsi)
 ; AVX-NEXT:    retq
 ;
-; AVX512F-LABEL: shuffle_v32i8_to_v8i8_1:
-; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512F-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm2 = <1,5,9,13,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512F-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX512F-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX512F-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; AVX512F-NEXT:    vmovq %xmm0, (%rsi)
-; AVX512F-NEXT:    retq
-;
-; AVX512VL-LABEL: shuffle_v32i8_to_v8i8_1:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512VL-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm2 = <1,5,9,13,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512VL-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; AVX512VL-NEXT:    vmovq %xmm0, (%rsi)
-; AVX512VL-NEXT:    retq
-;
-; AVX512BW-LABEL: shuffle_v32i8_to_v8i8_1:
-; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512BW-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm2 = <1,5,9,13,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512BW-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; AVX512BW-NEXT:    vmovq %xmm0, (%rsi)
-; AVX512BW-NEXT:    retq
-;
-; AVX512BWVL-LABEL: shuffle_v32i8_to_v8i8_1:
-; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512BWVL-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512BWVL-NEXT:    vmovdqa {{.*#+}} xmm2 = [1,1,5,5,9,9,13,13,13,13,5,5,12,12,13,13]
-; AVX512BWVL-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX512BWVL-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX512BWVL-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; AVX512BWVL-NEXT:    vpmovwb %xmm0, (%rsi)
-; AVX512BWVL-NEXT:    retq
+; AVX512-LABEL: shuffle_v32i8_to_v8i8_1:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512-NEXT:    vmovdqa 16(%rdi), %xmm1
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm2 = <1,5,9,13,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX512-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
+; AVX512-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
+; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX512-NEXT:    vmovq %xmm0, (%rsi)
+; AVX512-NEXT:    retq
   %vec = load <32 x i8>, <32 x i8>* %L
   %strided.vec = shufflevector <32 x i8> %vec, <32 x i8> undef, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>
   store <8 x i8> %strided.vec, <8 x i8>* %S
@@ -186,46 +153,16 @@ define void @shuffle_v32i8_to_v8i8_2(<32 x i8>* %L, <8 x i8>* %S) nounwind {
 ; AVX-NEXT:    vmovq %xmm0, (%rsi)
 ; AVX-NEXT:    retq
 ;
-; AVX512F-LABEL: shuffle_v32i8_to_v8i8_2:
-; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512F-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm2 = <2,6,10,14,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512F-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX512F-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX512F-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; AVX512F-NEXT:    vmovq %xmm0, (%rsi)
-; AVX512F-NEXT:    retq
-;
-; AVX512VL-LABEL: shuffle_v32i8_to_v8i8_2:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512VL-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm2 = <2,6,10,14,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512VL-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; AVX512VL-NEXT:    vmovq %xmm0, (%rsi)
-; AVX512VL-NEXT:    retq
-;
-; AVX512BW-LABEL: shuffle_v32i8_to_v8i8_2:
-; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512BW-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm2 = <2,6,10,14,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512BW-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; AVX512BW-NEXT:    vmovq %xmm0, (%rsi)
-; AVX512BW-NEXT:    retq
-;
-; AVX512BWVL-LABEL: shuffle_v32i8_to_v8i8_2:
-; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512BWVL-NEXT:    vmovdqa {{.*#+}} xmm1 = [1,3,5,7,9,11,13,15]
-; AVX512BWVL-NEXT:    vpermi2w 16(%rdi), %xmm0, %xmm1
-; AVX512BWVL-NEXT:    vpmovwb %xmm1, (%rsi)
-; AVX512BWVL-NEXT:    retq
+; AVX512-LABEL: shuffle_v32i8_to_v8i8_2:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512-NEXT:    vmovdqa 16(%rdi), %xmm1
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm2 = <2,6,10,14,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX512-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
+; AVX512-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
+; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX512-NEXT:    vmovq %xmm0, (%rsi)
+; AVX512-NEXT:    retq
   %vec = load <32 x i8>, <32 x i8>* %L
   %strided.vec = shufflevector <32 x i8> %vec, <32 x i8> undef, <8 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30>
   store <8 x i8> %strided.vec, <8 x i8>* %S
@@ -244,49 +181,16 @@ define void @shuffle_v32i8_to_v8i8_3(<32 x i8>* %L, <8 x i8>* %S) nounwind {
 ; AVX-NEXT:    vmovq %xmm0, (%rsi)
 ; AVX-NEXT:    retq
 ;
-; AVX512F-LABEL: shuffle_v32i8_to_v8i8_3:
-; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512F-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm2 = <3,7,11,15,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512F-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX512F-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX512F-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; AVX512F-NEXT:    vmovq %xmm0, (%rsi)
-; AVX512F-NEXT:    retq
-;
-; AVX512VL-LABEL: shuffle_v32i8_to_v8i8_3:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512VL-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm2 = <3,7,11,15,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512VL-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; AVX512VL-NEXT:    vmovq %xmm0, (%rsi)
-; AVX512VL-NEXT:    retq
-;
-; AVX512BW-LABEL: shuffle_v32i8_to_v8i8_3:
-; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512BW-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm2 = <3,7,11,15,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512BW-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; AVX512BW-NEXT:    vmovq %xmm0, (%rsi)
-; AVX512BW-NEXT:    retq
-;
-; AVX512BWVL-LABEL: shuffle_v32i8_to_v8i8_3:
-; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512BWVL-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512BWVL-NEXT:    vmovdqa {{.*#+}} xmm2 = [3,3,7,7,11,11,15,15,7,7,15,15,6,6,7,7]
-; AVX512BWVL-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX512BWVL-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX512BWVL-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; AVX512BWVL-NEXT:    vpmovwb %xmm0, (%rsi)
-; AVX512BWVL-NEXT:    retq
+; AVX512-LABEL: shuffle_v32i8_to_v8i8_3:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512-NEXT:    vmovdqa 16(%rdi), %xmm1
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm2 = <3,7,11,15,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX512-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
+; AVX512-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
+; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX512-NEXT:    vmovq %xmm0, (%rsi)
+; AVX512-NEXT:    retq
   %vec = load <32 x i8>, <32 x i8>* %L
   %strided.vec = shufflevector <32 x i8> %vec, <32 x i8> undef, <8 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31>
   store <8 x i8> %strided.vec, <8 x i8>* %S
@@ -339,11 +243,11 @@ define void @shuffle_v16i16_to_v4i16_1(<16 x i16>* %L, <4 x i16>* %S) nounwind {
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vmovdqa (%rdi), %xmm0
 ; AVX512VL-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm2 = [2,3,2,3,10,11,10,11,8,9,10,11,12,13,14,15]
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm2 = [2,3,10,11,8,9,10,11,8,9,10,11,12,13,14,15]
 ; AVX512VL-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
 ; AVX512VL-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; AVX512VL-NEXT:    vpmovdw %xmm0, (%rsi)
+; AVX512VL-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX512VL-NEXT:    vmovq %xmm0, (%rsi)
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: shuffle_v16i16_to_v4i16_1:
@@ -360,9 +264,9 @@ define void @shuffle_v16i16_to_v4i16_1(<16 x i16>* %L, <4 x i16>* %S) nounwind {
 ; AVX512BWVL-LABEL: shuffle_v16i16_to_v4i16_1:
 ; AVX512BWVL:       # %bb.0:
 ; AVX512BWVL-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512BWVL-NEXT:    vmovdqa {{.*#+}} xmm1 = [1,1,5,5,9,9,13,13]
+; AVX512BWVL-NEXT:    vmovdqa {{.*#+}} xmm1 = [1,5,9,13,4,5,12,13]
 ; AVX512BWVL-NEXT:    vpermi2w 16(%rdi), %xmm0, %xmm1
-; AVX512BWVL-NEXT:    vpmovdw %xmm1, (%rsi)
+; AVX512BWVL-NEXT:    vmovq %xmm1, (%rsi)
 ; AVX512BWVL-NEXT:    retq
   %vec = load <16 x i16>, <16 x i16>* %L
   %strided.vec = shufflevector <16 x i16> %vec, <16 x i16> undef, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
@@ -414,9 +318,13 @@ define void @shuffle_v16i16_to_v4i16_2(<16 x i16>* %L, <4 x i16>* %S) nounwind {
 ;
 ; AVX512VL-LABEL: shuffle_v16i16_to_v4i16_2:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vmovaps (%rdi), %xmm0
-; AVX512VL-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[1,3],mem[1,3]
-; AVX512VL-NEXT:    vpmovdw %xmm0, (%rsi)
+; AVX512VL-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512VL-NEXT:    vmovdqa 16(%rdi), %xmm1
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm2 = [4,5,12,13,4,5,6,7,8,9,10,11,12,13,14,15]
+; AVX512VL-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
+; AVX512VL-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX512VL-NEXT:    vmovq %xmm0, (%rsi)
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: shuffle_v16i16_to_v4i16_2:
@@ -432,9 +340,10 @@ define void @shuffle_v16i16_to_v4i16_2(<16 x i16>* %L, <4 x i16>* %S) nounwind {
 ;
 ; AVX512BWVL-LABEL: shuffle_v16i16_to_v4i16_2:
 ; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vmovaps (%rdi), %xmm0
-; AVX512BWVL-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[1,3],mem[1,3]
-; AVX512BWVL-NEXT:    vpmovdw %xmm0, (%rsi)
+; AVX512BWVL-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512BWVL-NEXT:    vmovdqa {{.*#+}} xmm1 = [2,6,10,14,2,3,10,11]
+; AVX512BWVL-NEXT:    vpermi2w 16(%rdi), %xmm0, %xmm1
+; AVX512BWVL-NEXT:    vmovq %xmm1, (%rsi)
 ; AVX512BWVL-NEXT:    retq
   %vec = load <16 x i16>, <16 x i16>* %L
   %strided.vec = shufflevector <16 x i16> %vec, <16 x i16> undef, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
@@ -488,11 +397,11 @@ define void @shuffle_v16i16_to_v4i16_3(<16 x i16>* %L, <4 x i16>* %S) nounwind {
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vmovdqa (%rdi), %xmm0
 ; AVX512VL-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm2 = [6,7,14,15,14,15,6,7,8,9,10,11,12,13,14,15]
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm2 = [6,7,14,15,4,5,6,7,8,9,10,11,12,13,14,15]
 ; AVX512VL-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
 ; AVX512VL-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; AVX512VL-NEXT:    vpmovdw %xmm0, (%rsi)
+; AVX512VL-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX512VL-NEXT:    vmovq %xmm0, (%rsi)
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: shuffle_v16i16_to_v4i16_3:
@@ -509,9 +418,9 @@ define void @shuffle_v16i16_to_v4i16_3(<16 x i16>* %L, <4 x i16>* %S) nounwind {
 ; AVX512BWVL-LABEL: shuffle_v16i16_to_v4i16_3:
 ; AVX512BWVL:       # %bb.0:
 ; AVX512BWVL-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512BWVL-NEXT:    vmovdqa {{.*#+}} xmm1 = [3,7,7,3,11,15,15,11]
+; AVX512BWVL-NEXT:    vmovdqa {{.*#+}} xmm1 = [3,7,11,15,2,3,10,11]
 ; AVX512BWVL-NEXT:    vpermi2w 16(%rdi), %xmm0, %xmm1
-; AVX512BWVL-NEXT:    vpmovdw %xmm1, (%rsi)
+; AVX512BWVL-NEXT:    vmovq %xmm1, (%rsi)
 ; AVX512BWVL-NEXT:    retq
   %vec = load <16 x i16>, <16 x i16>* %L
   %strided.vec = shufflevector <16 x i16> %vec, <16 x i16> undef, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
@@ -531,49 +440,16 @@ define void @shuffle_v32i8_to_v4i8_1(<32 x i8>* %L, <4 x i8>* %S) nounwind {
 ; AVX-NEXT:    vmovd %xmm0, (%rsi)
 ; AVX-NEXT:    retq
 ;
-; AVX512F-LABEL: shuffle_v32i8_to_v4i8_1:
-; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512F-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm2 = <1,9,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512F-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX512F-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX512F-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX512F-NEXT:    vmovd %xmm0, (%rsi)
-; AVX512F-NEXT:    retq
-;
-; AVX512VL-LABEL: shuffle_v32i8_to_v4i8_1:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512VL-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm2 = [1,1,255,255,9,9,255,255,9,9,255,255,11,11,255,255]
-; AVX512VL-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; AVX512VL-NEXT:    vpmovdb %xmm0, (%rsi)
-; AVX512VL-NEXT:    retq
-;
-; AVX512BW-LABEL: shuffle_v32i8_to_v4i8_1:
-; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512BW-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm2 = <1,9,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512BW-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX512BW-NEXT:    vmovd %xmm0, (%rsi)
-; AVX512BW-NEXT:    retq
-;
-; AVX512BWVL-LABEL: shuffle_v32i8_to_v4i8_1:
-; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512BWVL-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512BWVL-NEXT:    vmovdqa {{.*#+}} xmm2 = [1,1,255,255,9,9,255,255,9,9,255,255,11,11,255,255]
-; AVX512BWVL-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX512BWVL-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX512BWVL-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; AVX512BWVL-NEXT:    vpmovdb %xmm0, (%rsi)
-; AVX512BWVL-NEXT:    retq
+; AVX512-LABEL: shuffle_v32i8_to_v4i8_1:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512-NEXT:    vmovdqa 16(%rdi), %xmm1
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm2 = <1,9,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX512-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
+; AVX512-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX512-NEXT:    vmovd %xmm0, (%rsi)
+; AVX512-NEXT:    retq
   %vec = load <32 x i8>, <32 x i8>* %L
   %strided.vec = shufflevector <32 x i8> %vec, <32 x i8> undef, <4 x i32> <i32 1, i32 9, i32 17, i32 25>
   store <4 x i8> %strided.vec, <4 x i8>* %S
@@ -592,46 +468,16 @@ define void @shuffle_v32i8_to_v4i8_2(<32 x i8>* %L, <4 x i8>* %S) nounwind {
 ; AVX-NEXT:    vmovd %xmm0, (%rsi)
 ; AVX-NEXT:    retq
 ;
-; AVX512F-LABEL: shuffle_v32i8_to_v4i8_2:
-; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512F-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm2 = <2,10,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512F-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX512F-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX512F-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX512F-NEXT:    vmovd %xmm0, (%rsi)
-; AVX512F-NEXT:    retq
-;
-; AVX512VL-LABEL: shuffle_v32i8_to_v4i8_2:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512VL-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm2 = [2,3,2,3,10,11,10,11,8,9,10,11,12,13,14,15]
-; AVX512VL-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; AVX512VL-NEXT:    vpmovdb %xmm0, (%rsi)
-; AVX512VL-NEXT:    retq
-;
-; AVX512BW-LABEL: shuffle_v32i8_to_v4i8_2:
-; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512BW-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm2 = <2,10,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512BW-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX512BW-NEXT:    vmovd %xmm0, (%rsi)
-; AVX512BW-NEXT:    retq
-;
-; AVX512BWVL-LABEL: shuffle_v32i8_to_v4i8_2:
-; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512BWVL-NEXT:    vmovdqa {{.*#+}} xmm1 = [1,1,5,5,9,9,13,13]
-; AVX512BWVL-NEXT:    vpermi2w 16(%rdi), %xmm0, %xmm1
-; AVX512BWVL-NEXT:    vpmovdb %xmm1, (%rsi)
-; AVX512BWVL-NEXT:    retq
+; AVX512-LABEL: shuffle_v32i8_to_v4i8_2:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512-NEXT:    vmovdqa 16(%rdi), %xmm1
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm2 = <2,10,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX512-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
+; AVX512-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX512-NEXT:    vmovd %xmm0, (%rsi)
+; AVX512-NEXT:    retq
   %vec = load <32 x i8>, <32 x i8>* %L
   %strided.vec = shufflevector <32 x i8> %vec, <32 x i8> undef, <4 x i32> <i32 2, i32 10, i32 18, i32 26>
   store <4 x i8> %strided.vec, <4 x i8>* %S
@@ -650,49 +496,16 @@ define void @shuffle_v32i8_to_v4i8_3(<32 x i8>* %L, <4 x i8>* %S) nounwind {
 ; AVX-NEXT:    vmovd %xmm0, (%rsi)
 ; AVX-NEXT:    retq
 ;
-; AVX512F-LABEL: shuffle_v32i8_to_v4i8_3:
-; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512F-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm2 = <3,11,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512F-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX512F-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX512F-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX512F-NEXT:    vmovd %xmm0, (%rsi)
-; AVX512F-NEXT:    retq
-;
-; AVX512VL-LABEL: shuffle_v32i8_to_v4i8_3:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512VL-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm2 = [3,3,11,11,11,11,3,3,8,8,9,9,10,10,11,11]
-; AVX512VL-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; AVX512VL-NEXT:    vpmovdb %xmm0, (%rsi)
-; AVX512VL-NEXT:    retq
-;
-; AVX512BW-LABEL: shuffle_v32i8_to_v4i8_3:
-; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512BW-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm2 = <3,11,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512BW-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX512BW-NEXT:    vmovd %xmm0, (%rsi)
-; AVX512BW-NEXT:    retq
-;
-; AVX512BWVL-LABEL: shuffle_v32i8_to_v4i8_3:
-; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512BWVL-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512BWVL-NEXT:    vmovdqa {{.*#+}} xmm2 = [3,3,11,11,11,11,3,3,8,8,9,9,10,10,11,11]
-; AVX512BWVL-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX512BWVL-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX512BWVL-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; AVX512BWVL-NEXT:    vpmovdb %xmm0, (%rsi)
-; AVX512BWVL-NEXT:    retq
+; AVX512-LABEL: shuffle_v32i8_to_v4i8_3:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512-NEXT:    vmovdqa 16(%rdi), %xmm1
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm2 = <3,11,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX512-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
+; AVX512-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX512-NEXT:    vmovd %xmm0, (%rsi)
+; AVX512-NEXT:    retq
   %vec = load <32 x i8>, <32 x i8>* %L
   %strided.vec = shufflevector <32 x i8> %vec, <32 x i8> undef, <4 x i32> <i32 3, i32 11, i32 19, i32 27>
   store <4 x i8> %strided.vec, <4 x i8>* %S
@@ -711,41 +524,16 @@ define void @shuffle_v32i8_to_v4i8_4(<32 x i8>* %L, <4 x i8>* %S) nounwind {
 ; AVX-NEXT:    vmovd %xmm0, (%rsi)
 ; AVX-NEXT:    retq
 ;
-; AVX512F-LABEL: shuffle_v32i8_to_v4i8_4:
-; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512F-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm2 = <4,12,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512F-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX512F-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX512F-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX512F-NEXT:    vmovd %xmm0, (%rsi)
-; AVX512F-NEXT:    retq
-;
-; AVX512VL-LABEL: shuffle_v32i8_to_v4i8_4:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vmovaps (%rdi), %xmm0
-; AVX512VL-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[1,3],mem[1,3]
-; AVX512VL-NEXT:    vpmovdb %xmm0, (%rsi)
-; AVX512VL-NEXT:    retq
-;
-; AVX512BW-LABEL: shuffle_v32i8_to_v4i8_4:
-; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512BW-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm2 = <4,12,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512BW-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX512BW-NEXT:    vmovd %xmm0, (%rsi)
-; AVX512BW-NEXT:    retq
-;
-; AVX512BWVL-LABEL: shuffle_v32i8_to_v4i8_4:
-; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vmovaps (%rdi), %xmm0
-; AVX512BWVL-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[1,3],mem[1,3]
-; AVX512BWVL-NEXT:    vpmovdb %xmm0, (%rsi)
-; AVX512BWVL-NEXT:    retq
+; AVX512-LABEL: shuffle_v32i8_to_v4i8_4:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512-NEXT:    vmovdqa 16(%rdi), %xmm1
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm2 = <4,12,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX512-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
+; AVX512-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX512-NEXT:    vmovd %xmm0, (%rsi)
+; AVX512-NEXT:    retq
   %vec = load <32 x i8>, <32 x i8>* %L
   %strided.vec = shufflevector <32 x i8> %vec, <32 x i8> undef, <4 x i32> <i32 4, i32 12, i32 20, i32 28>
   store <4 x i8> %strided.vec, <4 x i8>* %S
@@ -764,49 +552,16 @@ define void @shuffle_v32i8_to_v4i8_5(<32 x i8>* %L, <4 x i8>* %S) nounwind {
 ; AVX-NEXT:    vmovd %xmm0, (%rsi)
 ; AVX-NEXT:    retq
 ;
-; AVX512F-LABEL: shuffle_v32i8_to_v4i8_5:
-; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512F-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm2 = <5,13,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512F-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX512F-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX512F-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX512F-NEXT:    vmovd %xmm0, (%rsi)
-; AVX512F-NEXT:    retq
-;
-; AVX512VL-LABEL: shuffle_v32i8_to_v4i8_5:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512VL-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm2 = [5,5,13,13,13,13,5,5,4,4,5,5,6,6,7,7]
-; AVX512VL-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; AVX512VL-NEXT:    vpmovdb %xmm0, (%rsi)
-; AVX512VL-NEXT:    retq
-;
-; AVX512BW-LABEL: shuffle_v32i8_to_v4i8_5:
-; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512BW-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm2 = <5,13,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512BW-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX512BW-NEXT:    vmovd %xmm0, (%rsi)
-; AVX512BW-NEXT:    retq
-;
-; AVX512BWVL-LABEL: shuffle_v32i8_to_v4i8_5:
-; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512BWVL-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512BWVL-NEXT:    vmovdqa {{.*#+}} xmm2 = [5,5,13,13,13,13,5,5,4,4,5,5,6,6,7,7]
-; AVX512BWVL-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX512BWVL-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX512BWVL-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; AVX512BWVL-NEXT:    vpmovdb %xmm0, (%rsi)
-; AVX512BWVL-NEXT:    retq
+; AVX512-LABEL: shuffle_v32i8_to_v4i8_5:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512-NEXT:    vmovdqa 16(%rdi), %xmm1
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm2 = <5,13,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX512-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
+; AVX512-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX512-NEXT:    vmovd %xmm0, (%rsi)
+; AVX512-NEXT:    retq
   %vec = load <32 x i8>, <32 x i8>* %L
   %strided.vec = shufflevector <32 x i8> %vec, <32 x i8> undef, <4 x i32> <i32 5, i32 13, i32 21, i32 29>
   store <4 x i8> %strided.vec, <4 x i8>* %S
@@ -825,46 +580,16 @@ define void @shuffle_v32i8_to_v4i8_6(<32 x i8>* %L, <4 x i8>* %S) nounwind {
 ; AVX-NEXT:    vmovd %xmm0, (%rsi)
 ; AVX-NEXT:    retq
 ;
-; AVX512F-LABEL: shuffle_v32i8_to_v4i8_6:
-; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512F-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm2 = <6,14,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512F-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX512F-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX512F-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX512F-NEXT:    vmovd %xmm0, (%rsi)
-; AVX512F-NEXT:    retq
-;
-; AVX512VL-LABEL: shuffle_v32i8_to_v4i8_6:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512VL-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm2 = [6,7,14,15,14,15,6,7,8,9,10,11,12,13,14,15]
-; AVX512VL-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; AVX512VL-NEXT:    vpmovdb %xmm0, (%rsi)
-; AVX512VL-NEXT:    retq
-;
-; AVX512BW-LABEL: shuffle_v32i8_to_v4i8_6:
-; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512BW-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm2 = <6,14,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512BW-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX512BW-NEXT:    vmovd %xmm0, (%rsi)
-; AVX512BW-NEXT:    retq
-;
-; AVX512BWVL-LABEL: shuffle_v32i8_to_v4i8_6:
-; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512BWVL-NEXT:    vmovdqa {{.*#+}} xmm1 = [3,7,7,3,11,15,15,11]
-; AVX512BWVL-NEXT:    vpermi2w 16(%rdi), %xmm0, %xmm1
-; AVX512BWVL-NEXT:    vpmovdb %xmm1, (%rsi)
-; AVX512BWVL-NEXT:    retq
+; AVX512-LABEL: shuffle_v32i8_to_v4i8_6:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512-NEXT:    vmovdqa 16(%rdi), %xmm1
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm2 = <6,14,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX512-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
+; AVX512-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX512-NEXT:    vmovd %xmm0, (%rsi)
+; AVX512-NEXT:    retq
   %vec = load <32 x i8>, <32 x i8>* %L
   %strided.vec = shufflevector <32 x i8> %vec, <32 x i8> undef, <4 x i32> <i32 6, i32 14, i32 22, i32 30>
   store <4 x i8> %strided.vec, <4 x i8>* %S
@@ -883,49 +608,16 @@ define void @shuffle_v32i8_to_v4i8_7(<32 x i8>* %L, <4 x i8>* %S) nounwind {
 ; AVX-NEXT:    vmovd %xmm0, (%rsi)
 ; AVX-NEXT:    retq
 ;
-; AVX512F-LABEL: shuffle_v32i8_to_v4i8_7:
-; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512F-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm2 = <7,15,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512F-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX512F-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX512F-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX512F-NEXT:    vmovd %xmm0, (%rsi)
-; AVX512F-NEXT:    retq
-;
-; AVX512VL-LABEL: shuffle_v32i8_to_v4i8_7:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512VL-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm2 = [7,7,14,14,15,15,14,14,15,15,4,4,5,5,6,6]
-; AVX512VL-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; AVX512VL-NEXT:    vpmovdb %xmm0, (%rsi)
-; AVX512VL-NEXT:    retq
-;
-; AVX512BW-LABEL: shuffle_v32i8_to_v4i8_7:
-; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512BW-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm2 = <7,15,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512BW-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX512BW-NEXT:    vmovd %xmm0, (%rsi)
-; AVX512BW-NEXT:    retq
-;
-; AVX512BWVL-LABEL: shuffle_v32i8_to_v4i8_7:
-; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512BWVL-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512BWVL-NEXT:    vmovdqa {{.*#+}} xmm2 = [7,7,14,14,15,15,14,14,15,15,4,4,5,5,6,6]
-; AVX512BWVL-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX512BWVL-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX512BWVL-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; AVX512BWVL-NEXT:    vpmovdb %xmm0, (%rsi)
-; AVX512BWVL-NEXT:    retq
+; AVX512-LABEL: shuffle_v32i8_to_v4i8_7:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512-NEXT:    vmovdqa 16(%rdi), %xmm1
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm2 = <7,15,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX512-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
+; AVX512-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX512-NEXT:    vmovd %xmm0, (%rsi)
+; AVX512-NEXT:    retq
   %vec = load <32 x i8>, <32 x i8>* %L
   %strided.vec = shufflevector <32 x i8> %vec, <32 x i8> undef, <4 x i32> <i32 7, i32 15, i32 23, i32 31>
   store <4 x i8> %strided.vec, <4 x i8>* %S
diff --git a/llvm/test/CodeGen/X86/shuffle-strided-with-offset-512.ll b/llvm/test/CodeGen/X86/shuffle-strided-with-offset-512.ll
index 2837974ee..b73e50e09 100644
--- a/llvm/test/CodeGen/X86/shuffle-strided-with-offset-512.ll
+++ b/llvm/test/CodeGen/X86/shuffle-strided-with-offset-512.ll
@@ -425,77 +425,23 @@ define void @shuffle_v32i16_to_v8i16_3(<32 x i16>* %L, <8 x i16>* %S) nounwind {
 }
 
 define void @shuffle_v64i8_to_v8i8_1(<64 x i8>* %L, <8 x i8>* %S) nounwind {
-; AVX512F-LABEL: shuffle_v64i8_to_v8i8_1:
-; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512F-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512F-NEXT:    vmovdqa 32(%rdi), %xmm2
-; AVX512F-NEXT:    vmovdqa 48(%rdi), %xmm3
-; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm4 = <u,u,1,9,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512F-NEXT:    vpshufb %xmm4, %xmm3, %xmm3
-; AVX512F-NEXT:    vpshufb %xmm4, %xmm2, %xmm2
-; AVX512F-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
-; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm3 = <1,9,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512F-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
-; AVX512F-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
-; AVX512F-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX512F-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2,3]
-; AVX512F-NEXT:    vmovq %xmm0, (%rsi)
-; AVX512F-NEXT:    retq
-;
-; AVX512VL-LABEL: shuffle_v64i8_to_v8i8_1:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512VL-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512VL-NEXT:    vmovdqa 32(%rdi), %xmm2
-; AVX512VL-NEXT:    vmovdqa 48(%rdi), %xmm3
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm4 = <u,u,1,9,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512VL-NEXT:    vpshufb %xmm4, %xmm3, %xmm3
-; AVX512VL-NEXT:    vpshufb %xmm4, %xmm2, %xmm2
-; AVX512VL-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm3 = <1,9,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512VL-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX512VL-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2,3]
-; AVX512VL-NEXT:    vmovq %xmm0, (%rsi)
-; AVX512VL-NEXT:    retq
-;
-; AVX512BW-LABEL: shuffle_v64i8_to_v8i8_1:
-; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512BW-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512BW-NEXT:    vmovdqa 32(%rdi), %xmm2
-; AVX512BW-NEXT:    vmovdqa 48(%rdi), %xmm3
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm4 = <u,u,1,9,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512BW-NEXT:    vpshufb %xmm4, %xmm3, %xmm3
-; AVX512BW-NEXT:    vpshufb %xmm4, %xmm2, %xmm2
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm3 = <1,9,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512BW-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2,3]
-; AVX512BW-NEXT:    vmovq %xmm0, (%rsi)
-; AVX512BW-NEXT:    retq
-;
-; AVX512BWVL-LABEL: shuffle_v64i8_to_v8i8_1:
-; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512BWVL-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512BWVL-NEXT:    vmovdqa 32(%rdi), %xmm2
-; AVX512BWVL-NEXT:    vmovdqa 48(%rdi), %xmm3
-; AVX512BWVL-NEXT:    vmovdqa {{.*#+}} xmm4 = [0,0,1,1,1,1,9,9,8,8,9,9,10,10,11,11]
-; AVX512BWVL-NEXT:    vpshufb %xmm4, %xmm3, %xmm3
-; AVX512BWVL-NEXT:    vpshufb %xmm4, %xmm2, %xmm2
-; AVX512BWVL-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; AVX512BWVL-NEXT:    vmovdqa {{.*#+}} xmm3 = [1,1,9,9,8,8,9,9,8,8,9,9,10,10,11,11]
-; AVX512BWVL-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
-; AVX512BWVL-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
-; AVX512BWVL-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; AVX512BWVL-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3]
-; AVX512BWVL-NEXT:    vpmovwb %xmm0, (%rsi)
-; AVX512BWVL-NEXT:    retq
+; AVX512-LABEL: shuffle_v64i8_to_v8i8_1:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512-NEXT:    vmovdqa 16(%rdi), %xmm1
+; AVX512-NEXT:    vmovdqa 32(%rdi), %xmm2
+; AVX512-NEXT:    vmovdqa 48(%rdi), %xmm3
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm4 = <u,u,1,9,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX512-NEXT:    vpshufb %xmm4, %xmm3, %xmm3
+; AVX512-NEXT:    vpshufb %xmm4, %xmm2, %xmm2
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm3 = <1,9,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX512-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
+; AVX512-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX512-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2,3]
+; AVX512-NEXT:    vmovq %xmm0, (%rsi)
+; AVX512-NEXT:    retq
   %vec = load <64 x i8>, <64 x i8>* %L
   %strided.vec = shufflevector <64 x i8> %vec, <64 x i8> undef, <8 x i32> <i32 1, i32 9, i32 17, i32 25, i32 33, i32 41, i32 49, i32 57>
   store <8 x i8> %strided.vec, <8 x i8>* %S
@@ -503,68 +449,23 @@ define void @shuffle_v64i8_to_v8i8_1(<64 x i8>* %L, <8 x i8>* %S) nounwind {
 }
 
 define void @shuffle_v64i8_to_v8i8_2(<64 x i8>* %L, <8 x i8>* %S) nounwind {
-; AVX512F-LABEL: shuffle_v64i8_to_v8i8_2:
-; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512F-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512F-NEXT:    vmovdqa 32(%rdi), %xmm2
-; AVX512F-NEXT:    vmovdqa 48(%rdi), %xmm3
-; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm4 = <u,u,2,10,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512F-NEXT:    vpshufb %xmm4, %xmm3, %xmm3
-; AVX512F-NEXT:    vpshufb %xmm4, %xmm2, %xmm2
-; AVX512F-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
-; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm3 = <2,10,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512F-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
-; AVX512F-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
-; AVX512F-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX512F-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2,3]
-; AVX512F-NEXT:    vmovq %xmm0, (%rsi)
-; AVX512F-NEXT:    retq
-;
-; AVX512VL-LABEL: shuffle_v64i8_to_v8i8_2:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512VL-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512VL-NEXT:    vmovdqa 32(%rdi), %xmm2
-; AVX512VL-NEXT:    vmovdqa 48(%rdi), %xmm3
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm4 = <u,u,2,10,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512VL-NEXT:    vpshufb %xmm4, %xmm3, %xmm3
-; AVX512VL-NEXT:    vpshufb %xmm4, %xmm2, %xmm2
-; AVX512VL-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm3 = <2,10,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512VL-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX512VL-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2,3]
-; AVX512VL-NEXT:    vmovq %xmm0, (%rsi)
-; AVX512VL-NEXT:    retq
-;
-; AVX512BW-LABEL: shuffle_v64i8_to_v8i8_2:
-; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512BW-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512BW-NEXT:    vmovdqa 32(%rdi), %xmm2
-; AVX512BW-NEXT:    vmovdqa 48(%rdi), %xmm3
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm4 = <u,u,2,10,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512BW-NEXT:    vpshufb %xmm4, %xmm3, %xmm3
-; AVX512BW-NEXT:    vpshufb %xmm4, %xmm2, %xmm2
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm3 = <2,10,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512BW-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2,3]
-; AVX512BW-NEXT:    vmovq %xmm0, (%rsi)
-; AVX512BW-NEXT:    retq
-;
-; AVX512BWVL-LABEL: shuffle_v64i8_to_v8i8_2:
-; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vmovdqa {{.*#+}} xmm0 = [1,5,9,13,17,21,25,29]
-; AVX512BWVL-NEXT:    vmovdqa (%rdi), %ymm1
-; AVX512BWVL-NEXT:    vpermt2w 32(%rdi), %ymm0, %ymm1
-; AVX512BWVL-NEXT:    vpmovwb %xmm1, (%rsi)
-; AVX512BWVL-NEXT:    vzeroupper
-; AVX512BWVL-NEXT:    retq
+; AVX512-LABEL: shuffle_v64i8_to_v8i8_2:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512-NEXT:    vmovdqa 16(%rdi), %xmm1
+; AVX512-NEXT:    vmovdqa 32(%rdi), %xmm2
+; AVX512-NEXT:    vmovdqa 48(%rdi), %xmm3
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm4 = <u,u,2,10,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX512-NEXT:    vpshufb %xmm4, %xmm3, %xmm3
+; AVX512-NEXT:    vpshufb %xmm4, %xmm2, %xmm2
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm3 = <2,10,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX512-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
+; AVX512-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX512-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2,3]
+; AVX512-NEXT:    vmovq %xmm0, (%rsi)
+; AVX512-NEXT:    retq
   %vec = load <64 x i8>, <64 x i8>* %L
   %strided.vec = shufflevector <64 x i8> %vec, <64 x i8> undef, <8 x i32> <i32 2, i32 10, i32 18, i32 26, i32 34, i32 42, i32 50, i32 58>
   store <8 x i8> %strided.vec, <8 x i8>* %S
@@ -572,77 +473,23 @@ define void @shuffle_v64i8_to_v8i8_2(<64 x i8>* %L, <8 x i8>* %S) nounwind {
 }
 
 define void @shuffle_v64i8_to_v8i8_3(<64 x i8>* %L, <8 x i8>* %S) nounwind {
-; AVX512F-LABEL: shuffle_v64i8_to_v8i8_3:
-; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512F-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512F-NEXT:    vmovdqa 32(%rdi), %xmm2
-; AVX512F-NEXT:    vmovdqa 48(%rdi), %xmm3
-; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm4 = <u,u,3,11,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512F-NEXT:    vpshufb %xmm4, %xmm3, %xmm3
-; AVX512F-NEXT:    vpshufb %xmm4, %xmm2, %xmm2
-; AVX512F-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
-; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm3 = <3,11,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512F-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
-; AVX512F-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
-; AVX512F-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX512F-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2,3]
-; AVX512F-NEXT:    vmovq %xmm0, (%rsi)
-; AVX512F-NEXT:    retq
-;
-; AVX512VL-LABEL: shuffle_v64i8_to_v8i8_3:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512VL-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512VL-NEXT:    vmovdqa 32(%rdi), %xmm2
-; AVX512VL-NEXT:    vmovdqa 48(%rdi), %xmm3
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm4 = <u,u,3,11,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512VL-NEXT:    vpshufb %xmm4, %xmm3, %xmm3
-; AVX512VL-NEXT:    vpshufb %xmm4, %xmm2, %xmm2
-; AVX512VL-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm3 = <3,11,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512VL-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX512VL-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2,3]
-; AVX512VL-NEXT:    vmovq %xmm0, (%rsi)
-; AVX512VL-NEXT:    retq
-;
-; AVX512BW-LABEL: shuffle_v64i8_to_v8i8_3:
-; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512BW-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512BW-NEXT:    vmovdqa 32(%rdi), %xmm2
-; AVX512BW-NEXT:    vmovdqa 48(%rdi), %xmm3
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm4 = <u,u,3,11,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512BW-NEXT:    vpshufb %xmm4, %xmm3, %xmm3
-; AVX512BW-NEXT:    vpshufb %xmm4, %xmm2, %xmm2
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm3 = <3,11,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512BW-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2,3]
-; AVX512BW-NEXT:    vmovq %xmm0, (%rsi)
-; AVX512BW-NEXT:    retq
-;
-; AVX512BWVL-LABEL: shuffle_v64i8_to_v8i8_3:
-; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512BWVL-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512BWVL-NEXT:    vmovdqa 32(%rdi), %xmm2
-; AVX512BWVL-NEXT:    vmovdqa 48(%rdi), %xmm3
-; AVX512BWVL-NEXT:    vmovdqa {{.*#+}} xmm4 = [10,10,11,11,3,3,11,11,8,8,9,9,10,10,11,11]
-; AVX512BWVL-NEXT:    vpshufb %xmm4, %xmm3, %xmm3
-; AVX512BWVL-NEXT:    vpshufb %xmm4, %xmm2, %xmm2
-; AVX512BWVL-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; AVX512BWVL-NEXT:    vmovdqa {{.*#+}} xmm3 = [3,3,11,11,2,2,3,3,8,8,9,9,10,10,11,11]
-; AVX512BWVL-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
-; AVX512BWVL-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
-; AVX512BWVL-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; AVX512BWVL-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3]
-; AVX512BWVL-NEXT:    vpmovwb %xmm0, (%rsi)
-; AVX512BWVL-NEXT:    retq
+; AVX512-LABEL: shuffle_v64i8_to_v8i8_3:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512-NEXT:    vmovdqa 16(%rdi), %xmm1
+; AVX512-NEXT:    vmovdqa 32(%rdi), %xmm2
+; AVX512-NEXT:    vmovdqa 48(%rdi), %xmm3
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm4 = <u,u,3,11,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX512-NEXT:    vpshufb %xmm4, %xmm3, %xmm3
+; AVX512-NEXT:    vpshufb %xmm4, %xmm2, %xmm2
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm3 = <3,11,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX512-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
+; AVX512-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX512-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2,3]
+; AVX512-NEXT:    vmovq %xmm0, (%rsi)
+; AVX512-NEXT:    retq
   %vec = load <64 x i8>, <64 x i8>* %L
   %strided.vec = shufflevector <64 x i8> %vec, <64 x i8> undef, <8 x i32> <i32 3, i32 11, i32 19, i32 27, i32 35, i32 43, i32 51, i32 59>
   store <8 x i8> %strided.vec, <8 x i8>* %S
@@ -650,68 +497,23 @@ define void @shuffle_v64i8_to_v8i8_3(<64 x i8>* %L, <8 x i8>* %S) nounwind {
 }
 
 define void @shuffle_v64i8_to_v8i8_4(<64 x i8>* %L, <8 x i8>* %S) nounwind {
-; AVX512F-LABEL: shuffle_v64i8_to_v8i8_4:
-; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512F-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512F-NEXT:    vmovdqa 32(%rdi), %xmm2
-; AVX512F-NEXT:    vmovdqa 48(%rdi), %xmm3
-; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm4 = <u,u,4,12,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512F-NEXT:    vpshufb %xmm4, %xmm3, %xmm3
-; AVX512F-NEXT:    vpshufb %xmm4, %xmm2, %xmm2
-; AVX512F-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
-; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm3 = <4,12,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512F-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
-; AVX512F-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
-; AVX512F-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX512F-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2,3]
-; AVX512F-NEXT:    vmovq %xmm0, (%rsi)
-; AVX512F-NEXT:    retq
-;
-; AVX512VL-LABEL: shuffle_v64i8_to_v8i8_4:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512VL-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512VL-NEXT:    vmovdqa 32(%rdi), %xmm2
-; AVX512VL-NEXT:    vmovdqa 48(%rdi), %xmm3
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm4 = <u,u,4,12,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512VL-NEXT:    vpshufb %xmm4, %xmm3, %xmm3
-; AVX512VL-NEXT:    vpshufb %xmm4, %xmm2, %xmm2
-; AVX512VL-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm3 = <4,12,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512VL-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX512VL-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2,3]
-; AVX512VL-NEXT:    vmovq %xmm0, (%rsi)
-; AVX512VL-NEXT:    retq
-;
-; AVX512BW-LABEL: shuffle_v64i8_to_v8i8_4:
-; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512BW-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512BW-NEXT:    vmovdqa 32(%rdi), %xmm2
-; AVX512BW-NEXT:    vmovdqa 48(%rdi), %xmm3
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm4 = <u,u,4,12,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512BW-NEXT:    vpshufb %xmm4, %xmm3, %xmm3
-; AVX512BW-NEXT:    vpshufb %xmm4, %xmm2, %xmm2
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm3 = <4,12,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512BW-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2,3]
-; AVX512BW-NEXT:    vmovq %xmm0, (%rsi)
-; AVX512BW-NEXT:    retq
-;
-; AVX512BWVL-LABEL: shuffle_v64i8_to_v8i8_4:
-; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vmovdqa {{.*#+}} xmm0 = [2,6,10,14,18,22,26,30]
-; AVX512BWVL-NEXT:    vmovdqa (%rdi), %ymm1
-; AVX512BWVL-NEXT:    vpermt2w 32(%rdi), %ymm0, %ymm1
-; AVX512BWVL-NEXT:    vpmovwb %xmm1, (%rsi)
-; AVX512BWVL-NEXT:    vzeroupper
-; AVX512BWVL-NEXT:    retq
+; AVX512-LABEL: shuffle_v64i8_to_v8i8_4:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512-NEXT:    vmovdqa 16(%rdi), %xmm1
+; AVX512-NEXT:    vmovdqa 32(%rdi), %xmm2
+; AVX512-NEXT:    vmovdqa 48(%rdi), %xmm3
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm4 = <u,u,4,12,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX512-NEXT:    vpshufb %xmm4, %xmm3, %xmm3
+; AVX512-NEXT:    vpshufb %xmm4, %xmm2, %xmm2
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm3 = <4,12,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX512-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
+; AVX512-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX512-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2,3]
+; AVX512-NEXT:    vmovq %xmm0, (%rsi)
+; AVX512-NEXT:    retq
   %vec = load <64 x i8>, <64 x i8>* %L
   %strided.vec = shufflevector <64 x i8> %vec, <64 x i8> undef, <8 x i32> <i32 4, i32 12, i32 20, i32 28, i32 36, i32 44, i32 52, i32 60>
   store <8 x i8> %strided.vec, <8 x i8>* %S
@@ -719,77 +521,23 @@ define void @shuffle_v64i8_to_v8i8_4(<64 x i8>* %L, <8 x i8>* %S) nounwind {
 }
 
 define void @shuffle_v64i8_to_v8i8_5(<64 x i8>* %L, <8 x i8>* %S) nounwind {
-; AVX512F-LABEL: shuffle_v64i8_to_v8i8_5:
-; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512F-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512F-NEXT:    vmovdqa 32(%rdi), %xmm2
-; AVX512F-NEXT:    vmovdqa 48(%rdi), %xmm3
-; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm4 = <u,u,5,13,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512F-NEXT:    vpshufb %xmm4, %xmm3, %xmm3
-; AVX512F-NEXT:    vpshufb %xmm4, %xmm2, %xmm2
-; AVX512F-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
-; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm3 = <5,13,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512F-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
-; AVX512F-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
-; AVX512F-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX512F-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2,3]
-; AVX512F-NEXT:    vmovq %xmm0, (%rsi)
-; AVX512F-NEXT:    retq
-;
-; AVX512VL-LABEL: shuffle_v64i8_to_v8i8_5:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512VL-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512VL-NEXT:    vmovdqa 32(%rdi), %xmm2
-; AVX512VL-NEXT:    vmovdqa 48(%rdi), %xmm3
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm4 = <u,u,5,13,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512VL-NEXT:    vpshufb %xmm4, %xmm3, %xmm3
-; AVX512VL-NEXT:    vpshufb %xmm4, %xmm2, %xmm2
-; AVX512VL-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm3 = <5,13,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512VL-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX512VL-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2,3]
-; AVX512VL-NEXT:    vmovq %xmm0, (%rsi)
-; AVX512VL-NEXT:    retq
-;
-; AVX512BW-LABEL: shuffle_v64i8_to_v8i8_5:
-; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512BW-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512BW-NEXT:    vmovdqa 32(%rdi), %xmm2
-; AVX512BW-NEXT:    vmovdqa 48(%rdi), %xmm3
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm4 = <u,u,5,13,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512BW-NEXT:    vpshufb %xmm4, %xmm3, %xmm3
-; AVX512BW-NEXT:    vpshufb %xmm4, %xmm2, %xmm2
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm3 = <5,13,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512BW-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2,3]
-; AVX512BW-NEXT:    vmovq %xmm0, (%rsi)
-; AVX512BW-NEXT:    retq
-;
-; AVX512BWVL-LABEL: shuffle_v64i8_to_v8i8_5:
-; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512BWVL-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512BWVL-NEXT:    vmovdqa 32(%rdi), %xmm2
-; AVX512BWVL-NEXT:    vmovdqa 48(%rdi), %xmm3
-; AVX512BWVL-NEXT:    vmovdqa {{.*#+}} xmm4 = [12,12,13,13,5,5,13,13,4,4,5,5,6,6,7,7]
-; AVX512BWVL-NEXT:    vpshufb %xmm4, %xmm3, %xmm3
-; AVX512BWVL-NEXT:    vpshufb %xmm4, %xmm2, %xmm2
-; AVX512BWVL-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; AVX512BWVL-NEXT:    vmovdqa {{.*#+}} xmm3 = [5,5,13,13,4,4,5,5,4,4,5,5,6,6,7,7]
-; AVX512BWVL-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
-; AVX512BWVL-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
-; AVX512BWVL-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; AVX512BWVL-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3]
-; AVX512BWVL-NEXT:    vpmovwb %xmm0, (%rsi)
-; AVX512BWVL-NEXT:    retq
+; AVX512-LABEL: shuffle_v64i8_to_v8i8_5:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512-NEXT:    vmovdqa 16(%rdi), %xmm1
+; AVX512-NEXT:    vmovdqa 32(%rdi), %xmm2
+; AVX512-NEXT:    vmovdqa 48(%rdi), %xmm3
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm4 = <u,u,5,13,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX512-NEXT:    vpshufb %xmm4, %xmm3, %xmm3
+; AVX512-NEXT:    vpshufb %xmm4, %xmm2, %xmm2
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm3 = <5,13,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX512-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
+; AVX512-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX512-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2,3]
+; AVX512-NEXT:    vmovq %xmm0, (%rsi)
+; AVX512-NEXT:    retq
   %vec = load <64 x i8>, <64 x i8>* %L
   %strided.vec = shufflevector <64 x i8> %vec, <64 x i8> undef, <8 x i32> <i32 5, i32 13, i32 21, i32 29, i32 37, i32 45, i32 53, i32 61>
   store <8 x i8> %strided.vec, <8 x i8>* %S
@@ -797,68 +545,23 @@ define void @shuffle_v64i8_to_v8i8_5(<64 x i8>* %L, <8 x i8>* %S) nounwind {
 }
 
 define void @shuffle_v64i8_to_v8i8_6(<64 x i8>* %L, <8 x i8>* %S) nounwind {
-; AVX512F-LABEL: shuffle_v64i8_to_v8i8_6:
-; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512F-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512F-NEXT:    vmovdqa 32(%rdi), %xmm2
-; AVX512F-NEXT:    vmovdqa 48(%rdi), %xmm3
-; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm4 = <u,u,6,14,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512F-NEXT:    vpshufb %xmm4, %xmm3, %xmm3
-; AVX512F-NEXT:    vpshufb %xmm4, %xmm2, %xmm2
-; AVX512F-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
-; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm3 = <6,14,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512F-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
-; AVX512F-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
-; AVX512F-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX512F-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2,3]
-; AVX512F-NEXT:    vmovq %xmm0, (%rsi)
-; AVX512F-NEXT:    retq
-;
-; AVX512VL-LABEL: shuffle_v64i8_to_v8i8_6:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512VL-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512VL-NEXT:    vmovdqa 32(%rdi), %xmm2
-; AVX512VL-NEXT:    vmovdqa 48(%rdi), %xmm3
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm4 = <u,u,6,14,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512VL-NEXT:    vpshufb %xmm4, %xmm3, %xmm3
-; AVX512VL-NEXT:    vpshufb %xmm4, %xmm2, %xmm2
-; AVX512VL-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm3 = <6,14,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512VL-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX512VL-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2,3]
-; AVX512VL-NEXT:    vmovq %xmm0, (%rsi)
-; AVX512VL-NEXT:    retq
-;
-; AVX512BW-LABEL: shuffle_v64i8_to_v8i8_6:
-; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512BW-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512BW-NEXT:    vmovdqa 32(%rdi), %xmm2
-; AVX512BW-NEXT:    vmovdqa 48(%rdi), %xmm3
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm4 = <u,u,6,14,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512BW-NEXT:    vpshufb %xmm4, %xmm3, %xmm3
-; AVX512BW-NEXT:    vpshufb %xmm4, %xmm2, %xmm2
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm3 = <6,14,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512BW-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2,3]
-; AVX512BW-NEXT:    vmovq %xmm0, (%rsi)
-; AVX512BW-NEXT:    retq
-;
-; AVX512BWVL-LABEL: shuffle_v64i8_to_v8i8_6:
-; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vmovdqa {{.*#+}} xmm0 = [3,7,11,15,19,23,27,31]
-; AVX512BWVL-NEXT:    vmovdqa (%rdi), %ymm1
-; AVX512BWVL-NEXT:    vpermt2w 32(%rdi), %ymm0, %ymm1
-; AVX512BWVL-NEXT:    vpmovwb %xmm1, (%rsi)
-; AVX512BWVL-NEXT:    vzeroupper
-; AVX512BWVL-NEXT:    retq
+; AVX512-LABEL: shuffle_v64i8_to_v8i8_6:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512-NEXT:    vmovdqa 16(%rdi), %xmm1
+; AVX512-NEXT:    vmovdqa 32(%rdi), %xmm2
+; AVX512-NEXT:    vmovdqa 48(%rdi), %xmm3
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm4 = <u,u,6,14,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX512-NEXT:    vpshufb %xmm4, %xmm3, %xmm3
+; AVX512-NEXT:    vpshufb %xmm4, %xmm2, %xmm2
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm3 = <6,14,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX512-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
+; AVX512-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX512-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2,3]
+; AVX512-NEXT:    vmovq %xmm0, (%rsi)
+; AVX512-NEXT:    retq
   %vec = load <64 x i8>, <64 x i8>* %L
   %strided.vec = shufflevector <64 x i8> %vec, <64 x i8> undef, <8 x i32> <i32 6, i32 14, i32 22, i32 30, i32 38, i32 46, i32 54, i32 62>
   store <8 x i8> %strided.vec, <8 x i8>* %S
@@ -866,77 +569,23 @@ define void @shuffle_v64i8_to_v8i8_6(<64 x i8>* %L, <8 x i8>* %S) nounwind {
 }
 
 define void @shuffle_v64i8_to_v8i8_7(<64 x i8>* %L, <8 x i8>* %S) nounwind {
-; AVX512F-LABEL: shuffle_v64i8_to_v8i8_7:
-; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512F-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512F-NEXT:    vmovdqa 32(%rdi), %xmm2
-; AVX512F-NEXT:    vmovdqa 48(%rdi), %xmm3
-; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm4 = <u,u,7,15,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512F-NEXT:    vpshufb %xmm4, %xmm3, %xmm3
-; AVX512F-NEXT:    vpshufb %xmm4, %xmm2, %xmm2
-; AVX512F-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
-; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm3 = <7,15,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512F-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
-; AVX512F-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
-; AVX512F-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX512F-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2,3]
-; AVX512F-NEXT:    vmovq %xmm0, (%rsi)
-; AVX512F-NEXT:    retq
-;
-; AVX512VL-LABEL: shuffle_v64i8_to_v8i8_7:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512VL-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512VL-NEXT:    vmovdqa 32(%rdi), %xmm2
-; AVX512VL-NEXT:    vmovdqa 48(%rdi), %xmm3
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm4 = <u,u,7,15,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512VL-NEXT:    vpshufb %xmm4, %xmm3, %xmm3
-; AVX512VL-NEXT:    vpshufb %xmm4, %xmm2, %xmm2
-; AVX512VL-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm3 = <7,15,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512VL-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX512VL-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2,3]
-; AVX512VL-NEXT:    vmovq %xmm0, (%rsi)
-; AVX512VL-NEXT:    retq
-;
-; AVX512BW-LABEL: shuffle_v64i8_to_v8i8_7:
-; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512BW-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512BW-NEXT:    vmovdqa 32(%rdi), %xmm2
-; AVX512BW-NEXT:    vmovdqa 48(%rdi), %xmm3
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm4 = <u,u,7,15,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512BW-NEXT:    vpshufb %xmm4, %xmm3, %xmm3
-; AVX512BW-NEXT:    vpshufb %xmm4, %xmm2, %xmm2
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm3 = <7,15,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512BW-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2,3]
-; AVX512BW-NEXT:    vmovq %xmm0, (%rsi)
-; AVX512BW-NEXT:    retq
-;
-; AVX512BWVL-LABEL: shuffle_v64i8_to_v8i8_7:
-; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512BWVL-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512BWVL-NEXT:    vmovdqa 32(%rdi), %xmm2
-; AVX512BWVL-NEXT:    vmovdqa 48(%rdi), %xmm3
-; AVX512BWVL-NEXT:    vmovdqa {{.*#+}} xmm4 = [14,14,15,15,7,7,15,15,4,4,5,5,6,6,7,7]
-; AVX512BWVL-NEXT:    vpshufb %xmm4, %xmm3, %xmm3
-; AVX512BWVL-NEXT:    vpshufb %xmm4, %xmm2, %xmm2
-; AVX512BWVL-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; AVX512BWVL-NEXT:    vmovdqa {{.*#+}} xmm3 = [7,7,15,15,6,6,7,7,4,4,5,5,6,6,7,7]
-; AVX512BWVL-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
-; AVX512BWVL-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
-; AVX512BWVL-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; AVX512BWVL-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3]
-; AVX512BWVL-NEXT:    vpmovwb %xmm0, (%rsi)
-; AVX512BWVL-NEXT:    retq
+; AVX512-LABEL: shuffle_v64i8_to_v8i8_7:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512-NEXT:    vmovdqa 16(%rdi), %xmm1
+; AVX512-NEXT:    vmovdqa 32(%rdi), %xmm2
+; AVX512-NEXT:    vmovdqa 48(%rdi), %xmm3
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm4 = <u,u,7,15,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX512-NEXT:    vpshufb %xmm4, %xmm3, %xmm3
+; AVX512-NEXT:    vpshufb %xmm4, %xmm2, %xmm2
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm3 = <7,15,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX512-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
+; AVX512-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX512-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2,3]
+; AVX512-NEXT:    vmovq %xmm0, (%rsi)
+; AVX512-NEXT:    retq
   %vec = load <64 x i8>, <64 x i8>* %L
   %strided.vec = shufflevector <64 x i8> %vec, <64 x i8> undef, <8 x i32> <i32 7, i32 15, i32 23, i32 31, i32 39, i32 47, i32 55, i32 63>
   store <8 x i8> %strided.vec, <8 x i8>* %S
diff --git a/llvm/test/CodeGen/X86/shuffle-vs-trunc-128.ll b/llvm/test/CodeGen/X86/shuffle-vs-trunc-128.ll
index 9c5344562..7805c46e5 100644
--- a/llvm/test/CodeGen/X86/shuffle-vs-trunc-128.ll
+++ b/llvm/test/CodeGen/X86/shuffle-vs-trunc-128.ll
@@ -37,32 +37,12 @@ define void @shuffle_v16i8_to_v8i8(<16 x i8>* %L, <8 x i8>* %S) nounwind {
 ; AVX-NEXT:    vmovq %xmm0, (%rsi)
 ; AVX-NEXT:    retq
 ;
-; AVX512F-LABEL: shuffle_v16i8_to_v8i8:
-; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
-; AVX512F-NEXT:    vmovq %xmm0, (%rsi)
-; AVX512F-NEXT:    retq
-;
-; AVX512VL-LABEL: shuffle_v16i8_to_v8i8:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512VL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
-; AVX512VL-NEXT:    vmovq %xmm0, (%rsi)
-; AVX512VL-NEXT:    retq
-;
-; AVX512BW-LABEL: shuffle_v16i8_to_v8i8:
-; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
-; AVX512BW-NEXT:    vmovq %xmm0, (%rsi)
-; AVX512BW-NEXT:    retq
-;
-; AVX512BWVL-LABEL: shuffle_v16i8_to_v8i8:
-; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512BWVL-NEXT:    vpmovwb %xmm0, (%rsi)
-; AVX512BWVL-NEXT:    retq
+; AVX512-LABEL: shuffle_v16i8_to_v8i8:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
+; AVX512-NEXT:    vmovq %xmm0, (%rsi)
+; AVX512-NEXT:    retq
   %vec = load <16 x i8>, <16 x i8>* %L
   %strided.vec = shufflevector <16 x i8> %vec, <16 x i8> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
   store <8 x i8> %strided.vec, <8 x i8>* %S
@@ -148,31 +128,12 @@ define void @shuffle_v8i16_to_v4i16(<8 x i16>* %L, <4 x i16>* %S) nounwind {
 ; AVX-NEXT:    vmovq %xmm0, (%rsi)
 ; AVX-NEXT:    retq
 ;
-; AVX512F-LABEL: shuffle_v8i16_to_v4i16:
-; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
-; AVX512F-NEXT:    vmovq %xmm0, (%rsi)
-; AVX512F-NEXT:    retq
-;
-; AVX512VL-LABEL: shuffle_v8i16_to_v4i16:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512VL-NEXT:    vpmovdw %xmm0, (%rsi)
-; AVX512VL-NEXT:    retq
-;
-; AVX512BW-LABEL: shuffle_v8i16_to_v4i16:
-; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
-; AVX512BW-NEXT:    vmovq %xmm0, (%rsi)
-; AVX512BW-NEXT:    retq
-;
-; AVX512BWVL-LABEL: shuffle_v8i16_to_v4i16:
-; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512BWVL-NEXT:    vpmovdw %xmm0, (%rsi)
-; AVX512BWVL-NEXT:    retq
+; AVX512-LABEL: shuffle_v8i16_to_v4i16:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
+; AVX512-NEXT:    vmovq %xmm0, (%rsi)
+; AVX512-NEXT:    retq
   %vec = load <8 x i16>, <8 x i16>* %L
   %strided.vec = shufflevector <8 x i16> %vec, <8 x i16> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
   store <4 x i16> %strided.vec, <4 x i16>* %S
@@ -247,29 +208,11 @@ define void @shuffle_v4i32_to_v2i32(<4 x i32>* %L, <2 x i32>* %S) nounwind {
 ; AVX-NEXT:    vmovlps %xmm0, (%rsi)
 ; AVX-NEXT:    retq
 ;
-; AVX512F-LABEL: shuffle_v4i32_to_v2i32:
-; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpermilps {{.*#+}} xmm0 = mem[0,2,2,3]
-; AVX512F-NEXT:    vmovlps %xmm0, (%rsi)
-; AVX512F-NEXT:    retq
-;
-; AVX512VL-LABEL: shuffle_v4i32_to_v2i32:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512VL-NEXT:    vpmovqd %xmm0, (%rsi)
-; AVX512VL-NEXT:    retq
-;
-; AVX512BW-LABEL: shuffle_v4i32_to_v2i32:
-; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpermilps {{.*#+}} xmm0 = mem[0,2,2,3]
-; AVX512BW-NEXT:    vmovlps %xmm0, (%rsi)
-; AVX512BW-NEXT:    retq
-;
-; AVX512BWVL-LABEL: shuffle_v4i32_to_v2i32:
-; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512BWVL-NEXT:    vpmovqd %xmm0, (%rsi)
-; AVX512BWVL-NEXT:    retq
+; AVX512-LABEL: shuffle_v4i32_to_v2i32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpermilps {{.*#+}} xmm0 = mem[0,2,2,3]
+; AVX512-NEXT:    vmovlps %xmm0, (%rsi)
+; AVX512-NEXT:    retq
   %vec = load <4 x i32>, <4 x i32>* %L
   %strided.vec = shufflevector <4 x i32> %vec, <4 x i32> undef, <2 x i32> <i32 0, i32 2>
   store <2 x i32> %strided.vec, <2 x i32>* %S
@@ -343,31 +286,12 @@ define void @shuffle_v16i8_to_v4i8(<16 x i8>* %L, <4 x i8>* %S) nounwind {
 ; AVX-NEXT:    vmovd %xmm0, (%rsi)
 ; AVX-NEXT:    retq
 ;
-; AVX512F-LABEL: shuffle_v16i8_to_v4i8:
-; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512F-NEXT:    vmovd %xmm0, (%rsi)
-; AVX512F-NEXT:    retq
-;
-; AVX512VL-LABEL: shuffle_v16i8_to_v4i8:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512VL-NEXT:    vpmovdb %xmm0, (%rsi)
-; AVX512VL-NEXT:    retq
-;
-; AVX512BW-LABEL: shuffle_v16i8_to_v4i8:
-; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512BW-NEXT:    vmovd %xmm0, (%rsi)
-; AVX512BW-NEXT:    retq
-;
-; AVX512BWVL-LABEL: shuffle_v16i8_to_v4i8:
-; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512BWVL-NEXT:    vpmovdb %xmm0, (%rsi)
-; AVX512BWVL-NEXT:    retq
+; AVX512-LABEL: shuffle_v16i8_to_v4i8:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512-NEXT:    vmovd %xmm0, (%rsi)
+; AVX512-NEXT:    retq
   %vec = load <16 x i8>, <16 x i8>* %L
   %strided.vec = shufflevector <16 x i8> %vec, <16 x i8> undef, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
   store <4 x i8> %strided.vec, <4 x i8>* %S
@@ -469,7 +393,8 @@ define void @shuffle_v8i16_to_v2i16(<8 x i16>* %L, <2 x i16>* %S) nounwind {
 ; AVX512VL-LABEL: shuffle_v8i16_to_v2i16:
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512VL-NEXT:    vpmovqw %xmm0, (%rsi)
+; AVX512VL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,8,9,8,9,10,11,8,9,10,11,12,13,14,15]
+; AVX512VL-NEXT:    vmovd %xmm0, (%rsi)
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: shuffle_v8i16_to_v2i16:
@@ -482,7 +407,8 @@ define void @shuffle_v8i16_to_v2i16(<8 x i16>* %L, <2 x i16>* %S) nounwind {
 ; AVX512BWVL-LABEL: shuffle_v8i16_to_v2i16:
 ; AVX512BWVL:       # %bb.0:
 ; AVX512BWVL-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512BWVL-NEXT:    vpmovqw %xmm0, (%rsi)
+; AVX512BWVL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,8,9,8,9,10,11,8,9,10,11,12,13,14,15]
+; AVX512BWVL-NEXT:    vmovd %xmm0, (%rsi)
 ; AVX512BWVL-NEXT:    retq
   %vec = load <8 x i16>, <8 x i16>* %L
   %strided.vec = shufflevector <8 x i16> %vec, <8 x i16> undef, <2 x i32> <i32 0, i32 4>
@@ -577,31 +503,12 @@ define void @shuffle_v16i8_to_v2i8(<16 x i8>* %L, <2 x i8>* %S) nounwind {
 ; AVX-NEXT:    vpextrw $0, %xmm0, (%rsi)
 ; AVX-NEXT:    retq
 ;
-; AVX512F-LABEL: shuffle_v16i8_to_v2i8:
-; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512F-NEXT:    vpextrw $0, %xmm0, (%rsi)
-; AVX512F-NEXT:    retq
-;
-; AVX512VL-LABEL: shuffle_v16i8_to_v2i8:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512VL-NEXT:    vpmovqb %xmm0, (%rsi)
-; AVX512VL-NEXT:    retq
-;
-; AVX512BW-LABEL: shuffle_v16i8_to_v2i8:
-; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512BW-NEXT:    vpextrw $0, %xmm0, (%rsi)
-; AVX512BW-NEXT:    retq
-;
-; AVX512BWVL-LABEL: shuffle_v16i8_to_v2i8:
-; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512BWVL-NEXT:    vpmovqb %xmm0, (%rsi)
-; AVX512BWVL-NEXT:    retq
+; AVX512-LABEL: shuffle_v16i8_to_v2i8:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512-NEXT:    vpextrw $0, %xmm0, (%rsi)
+; AVX512-NEXT:    retq
   %vec = load <16 x i8>, <16 x i8>* %L
   %strided.vec = shufflevector <16 x i8> %vec, <16 x i8> undef, <2 x i32> <i32 0, i32 8>
   store <2 x i8> %strided.vec, <2 x i8>* %S
diff --git a/llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll b/llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll
index fd9787beb..01890e2ad 100644
--- a/llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll
+++ b/llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll
@@ -408,17 +408,20 @@ define void @shuffle_v32i8_to_v8i8(<32 x i8>* %L, <8 x i8>* %S) nounwind {
 ; AVX512BWVL-LABEL: shuffle_v32i8_to_v8i8:
 ; AVX512BWVL:       # %bb.0:
 ; AVX512BWVL-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512BWVL-NEXT:    vmovdqa {{.*#+}} xmm1 = [0,2,4,6,8,10,12,14]
-; AVX512BWVL-NEXT:    vpermi2w 16(%rdi), %xmm0, %xmm1
-; AVX512BWVL-NEXT:    vpmovwb %xmm1, (%rsi)
+; AVX512BWVL-NEXT:    vmovdqa 16(%rdi), %xmm1
+; AVX512BWVL-NEXT:    vmovdqa {{.*#+}} xmm2 = <0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX512BWVL-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
+; AVX512BWVL-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
+; AVX512BWVL-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX512BWVL-NEXT:    vmovq %xmm0, (%rsi)
 ; AVX512BWVL-NEXT:    retq
 ;
 ; AVX512VBMIVL-LABEL: shuffle_v32i8_to_v8i8:
 ; AVX512VBMIVL:       # %bb.0:
 ; AVX512VBMIVL-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512VBMIVL-NEXT:    vmovdqa {{.*#+}} xmm1 = [0,2,4,6,8,10,12,14]
-; AVX512VBMIVL-NEXT:    vpermi2w 16(%rdi), %xmm0, %xmm1
-; AVX512VBMIVL-NEXT:    vpmovwb %xmm1, (%rsi)
+; AVX512VBMIVL-NEXT:    vpbroadcastq {{.*#+}} xmm1 = [2024390091656922112,2024390091656922112]
+; AVX512VBMIVL-NEXT:    vpermi2b 16(%rdi), %xmm0, %xmm1
+; AVX512VBMIVL-NEXT:    vmovq %xmm1, (%rsi)
 ; AVX512VBMIVL-NEXT:    retq
   %vec = load <32 x i8>, <32 x i8>* %L
   %strided.vec = shufflevector <32 x i8> %vec, <32 x i8> undef, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
@@ -427,32 +430,21 @@ define void @shuffle_v32i8_to_v8i8(<32 x i8>* %L, <8 x i8>* %S) nounwind {
 }
 
 define void @trunc_v8i32_to_v8i8(<32 x i8>* %L, <8 x i8>* %S) nounwind {
-; AVX1-LABEL: trunc_v8i32_to_v8i8:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX1-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = <0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX1-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; AVX1-NEXT:    vmovq %xmm0, (%rsi)
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: trunc_v8i32_to_v8i8:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vmovdqa (%rdi), %ymm0
-; AVX2-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
-; AVX2-NEXT:    vmovq %xmm0, (%rsi)
-; AVX2-NEXT:    vzeroupper
-; AVX2-NEXT:    retq
+; AVX-LABEL: trunc_v8i32_to_v8i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX-NEXT:    vmovdqa 16(%rdi), %xmm1
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm2 = <0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
+; AVX-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX-NEXT:    vmovq %xmm0, (%rsi)
+; AVX-NEXT:    retq
 ;
 ; AVX512F-LABEL: trunc_v8i32_to_v8i8:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovdqa (%rdi), %ymm0
-; AVX512F-NEXT:    vpmovdw %zmm0, %ymm0
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
+; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
 ; AVX512F-NEXT:    vmovq %xmm0, (%rsi)
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
@@ -467,8 +459,7 @@ define void @trunc_v8i32_to_v8i8(<32 x i8>* %L, <8 x i8>* %S) nounwind {
 ; AVX512BW-LABEL: trunc_v8i32_to_v8i8:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    vmovdqa (%rdi), %ymm0
-; AVX512BW-NEXT:    vpmovdw %zmm0, %ymm0
-; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
+; AVX512BW-NEXT:    vpmovdb %zmm0, %xmm0
 ; AVX512BW-NEXT:    vmovq %xmm0, (%rsi)
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -499,53 +490,59 @@ define <2 x i64> @trunc_v8i32_to_v8i8_return_v2i64(<8 x i32> %vec) nounwind {
 ; AVX1-LABEL: trunc_v8i32_to_v8i8_return_v2i64:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = <0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u>
 ; AVX1-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
 ; AVX1-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14],zero,zero,zero,zero,zero,zero,zero,zero
+; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX1-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: trunc_v8i32_to_v8i8_return_v2i64:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14],zero,zero,zero,zero,zero,zero,zero,zero
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = <0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX2-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX2-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: trunc_v8i32_to_v8i8_return_v2i64:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; AVX512F-NEXT:    vpmovdw %zmm0, %ymm0
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14],zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512F-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: trunc_v8i32_to_v8i8_return_v2i64:
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vpmovdb %ymm0, %xmm0
+; AVX512VL-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: trunc_v8i32_to_v8i8_return_v2i64:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; AVX512BW-NEXT:    vpmovdw %zmm0, %ymm0
-; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14],zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512BW-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512BW-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512BWVL-LABEL: trunc_v8i32_to_v8i8_return_v2i64:
 ; AVX512BWVL:       # %bb.0:
 ; AVX512BWVL-NEXT:    vpmovdb %ymm0, %xmm0
+; AVX512BWVL-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
 ;
 ; AVX512VBMIVL-LABEL: trunc_v8i32_to_v8i8_return_v2i64:
 ; AVX512VBMIVL:       # %bb.0:
 ; AVX512VBMIVL-NEXT:    vpmovdb %ymm0, %xmm0
+; AVX512VBMIVL-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; AVX512VBMIVL-NEXT:    vzeroupper
 ; AVX512VBMIVL-NEXT:    retq
   %truncated.vec = trunc <8 x i32> %vec to <8 x i8>
@@ -677,53 +674,59 @@ define <16 x i8> @trunc_v8i32_to_v8i8_return_v16i8(<8 x i32> %vec) nounwind {
 ; AVX1-LABEL: trunc_v8i32_to_v8i8_return_v16i8:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = <0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u>
 ; AVX1-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
 ; AVX1-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14],zero,zero,zero,zero,zero,zero,zero,zero
+; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX1-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: trunc_v8i32_to_v8i8_return_v16i8:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14],zero,zero,zero,zero,zero,zero,zero,zero
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = <0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX2-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX2-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: trunc_v8i32_to_v8i8_return_v16i8:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; AVX512F-NEXT:    vpmovdw %zmm0, %ymm0
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14],zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512F-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: trunc_v8i32_to_v8i8_return_v16i8:
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vpmovdb %ymm0, %xmm0
+; AVX512VL-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: trunc_v8i32_to_v8i8_return_v16i8:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; AVX512BW-NEXT:    vpmovdw %zmm0, %ymm0
-; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14],zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512BW-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512BW-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512BWVL-LABEL: trunc_v8i32_to_v8i8_return_v16i8:
 ; AVX512BWVL:       # %bb.0:
 ; AVX512BWVL-NEXT:    vpmovdb %ymm0, %xmm0
+; AVX512BWVL-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
 ;
 ; AVX512VBMIVL-LABEL: trunc_v8i32_to_v8i8_return_v16i8:
 ; AVX512VBMIVL:       # %bb.0:
 ; AVX512VBMIVL-NEXT:    vpmovdb %ymm0, %xmm0
+; AVX512VBMIVL-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; AVX512VBMIVL-NEXT:    vzeroupper
 ; AVX512VBMIVL-NEXT:    retq
   %truncated = trunc <8 x i32> %vec to <8 x i8>
@@ -737,58 +740,72 @@ define <2 x i64> @trunc_v4i64_to_v4i16_return_v2i64(<4 x i64> %vec) nounwind {
 ; AVX1-LABEL: trunc_v4i64_to_v4i16_return_v2i64:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
-; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,8,9,12,13],zero,zero,zero,zero,zero,zero,zero,zero
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX1-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-SLOW-LABEL: trunc_v4i64_to_v4i16_return_v2i64:
 ; AVX2-SLOW:       # %bb.0:
-; AVX2-SLOW-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
-; AVX2-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,8,9,12,13],zero,zero,zero,zero,zero,zero,zero,zero
+; AVX2-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX2-SLOW-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; AVX2-SLOW-NEXT:    vzeroupper
 ; AVX2-SLOW-NEXT:    retq
 ;
 ; AVX2-FAST-LABEL: trunc_v4i64_to_v4i16_return_v2i64:
 ; AVX2-FAST:       # %bb.0:
-; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,2,4,6,4,6,6,7]
-; AVX2-FAST-NEXT:    vpermd %ymm0, %ymm1, %ymm0
-; AVX2-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,8,9,12,13],zero,zero,zero,zero,zero,zero,zero,zero
+; AVX2-FAST-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,1,8,9,8,9,10,11,8,9,10,11,12,13,14,15]
+; AVX2-FAST-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
+; AVX2-FAST-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
+; AVX2-FAST-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX2-FAST-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; AVX2-FAST-NEXT:    vzeroupper
 ; AVX2-FAST-NEXT:    retq
 ;
 ; AVX512F-LABEL: trunc_v4i64_to_v4i16_return_v2i64:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; AVX512F-NEXT:    vpmovqd %zmm0, %ymm0
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,8,9,12,13],zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512F-NEXT:    vpmovqw %zmm0, %xmm0
+; AVX512F-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: trunc_v4i64_to_v4i16_return_v2i64:
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vpmovqw %ymm0, %xmm0
+; AVX512VL-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: trunc_v4i64_to_v4i16_return_v2i64:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; AVX512BW-NEXT:    vpmovqd %zmm0, %ymm0
-; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,8,9,12,13],zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512BW-NEXT:    vpmovqw %zmm0, %xmm0
+; AVX512BW-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512BWVL-LABEL: trunc_v4i64_to_v4i16_return_v2i64:
 ; AVX512BWVL:       # %bb.0:
 ; AVX512BWVL-NEXT:    vpmovqw %ymm0, %xmm0
+; AVX512BWVL-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
 ;
 ; AVX512VBMIVL-LABEL: trunc_v4i64_to_v4i16_return_v2i64:
 ; AVX512VBMIVL:       # %bb.0:
 ; AVX512VBMIVL-NEXT:    vpmovqw %ymm0, %xmm0
+; AVX512VBMIVL-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; AVX512VBMIVL-NEXT:    vzeroupper
 ; AVX512VBMIVL-NEXT:    retq
   %truncated = trunc <4 x i64> %vec to <4 x i16>
@@ -930,58 +947,72 @@ define <8 x i16> @trunc_v4i64_to_v4i16_return_v8i16(<4 x i64> %vec) nounwind {
 ; AVX1-LABEL: trunc_v4i64_to_v4i16_return_v8i16:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
-; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,8,9,12,13],zero,zero,zero,zero,zero,zero,zero,zero
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX1-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-SLOW-LABEL: trunc_v4i64_to_v4i16_return_v8i16:
 ; AVX2-SLOW:       # %bb.0:
-; AVX2-SLOW-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
-; AVX2-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,8,9,12,13],zero,zero,zero,zero,zero,zero,zero,zero
+; AVX2-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX2-SLOW-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; AVX2-SLOW-NEXT:    vzeroupper
 ; AVX2-SLOW-NEXT:    retq
 ;
 ; AVX2-FAST-LABEL: trunc_v4i64_to_v4i16_return_v8i16:
 ; AVX2-FAST:       # %bb.0:
-; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,2,4,6,4,6,6,7]
-; AVX2-FAST-NEXT:    vpermd %ymm0, %ymm1, %ymm0
-; AVX2-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,8,9,12,13],zero,zero,zero,zero,zero,zero,zero,zero
+; AVX2-FAST-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,1,8,9,8,9,10,11,8,9,10,11,12,13,14,15]
+; AVX2-FAST-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
+; AVX2-FAST-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
+; AVX2-FAST-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX2-FAST-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; AVX2-FAST-NEXT:    vzeroupper
 ; AVX2-FAST-NEXT:    retq
 ;
 ; AVX512F-LABEL: trunc_v4i64_to_v4i16_return_v8i16:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; AVX512F-NEXT:    vpmovqd %zmm0, %ymm0
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,8,9,12,13],zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512F-NEXT:    vpmovqw %zmm0, %xmm0
+; AVX512F-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: trunc_v4i64_to_v4i16_return_v8i16:
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vpmovqw %ymm0, %xmm0
+; AVX512VL-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: trunc_v4i64_to_v4i16_return_v8i16:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; AVX512BW-NEXT:    vpmovqd %zmm0, %ymm0
-; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,8,9,12,13],zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512BW-NEXT:    vpmovqw %zmm0, %xmm0
+; AVX512BW-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512BWVL-LABEL: trunc_v4i64_to_v4i16_return_v8i16:
 ; AVX512BWVL:       # %bb.0:
 ; AVX512BWVL-NEXT:    vpmovqw %ymm0, %xmm0
+; AVX512BWVL-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
 ;
 ; AVX512VBMIVL-LABEL: trunc_v4i64_to_v4i16_return_v8i16:
 ; AVX512VBMIVL:       # %bb.0:
 ; AVX512VBMIVL-NEXT:    vpmovqw %ymm0, %xmm0
+; AVX512VBMIVL-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; AVX512VBMIVL-NEXT:    vzeroupper
 ; AVX512VBMIVL-NEXT:    retq
   %truncated = trunc <4 x i64> %vec to <4 x i16>
@@ -993,58 +1024,66 @@ define <16 x i8> @trunc_v4i64_to_v4i8_return_v16i8(<4 x i64> %vec) nounwind {
 ; AVX1-LABEL: trunc_v4i64_to_v4i8_return_v16i8:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
-; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[u],zero
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = <0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX1-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7]
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
-; AVX2-SLOW-LABEL: trunc_v4i64_to_v4i8_return_v16i8:
-; AVX2-SLOW:       # %bb.0:
-; AVX2-SLOW-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
-; AVX2-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[u],zero
-; AVX2-SLOW-NEXT:    vzeroupper
-; AVX2-SLOW-NEXT:    retq
-;
-; AVX2-FAST-LABEL: trunc_v4i64_to_v4i8_return_v16i8:
-; AVX2-FAST:       # %bb.0:
-; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,2,4,6,4,6,6,7]
-; AVX2-FAST-NEXT:    vpermd %ymm0, %ymm1, %ymm0
-; AVX2-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[u],zero
-; AVX2-FAST-NEXT:    vzeroupper
-; AVX2-FAST-NEXT:    retq
+; AVX2-LABEL: trunc_v4i64_to_v4i8_return_v16i8:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = <0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX2-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7]
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: trunc_v4i64_to_v4i8_return_v16i8:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; AVX512F-NEXT:    vpmovqd %zmm0, %ymm0
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[u],zero
+; AVX512F-NEXT:    vpmovqb %zmm0, %xmm0
+; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7]
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: trunc_v4i64_to_v4i8_return_v16i8:
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vpmovqb %ymm0, %xmm0
+; AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512VL-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7]
 ; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: trunc_v4i64_to_v4i8_return_v16i8:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; AVX512BW-NEXT:    vpmovqd %zmm0, %ymm0
-; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[u],zero
+; AVX512BW-NEXT:    vpmovqb %zmm0, %xmm0
+; AVX512BW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512BW-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7]
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512BWVL-LABEL: trunc_v4i64_to_v4i8_return_v16i8:
 ; AVX512BWVL:       # %bb.0:
 ; AVX512BWVL-NEXT:    vpmovqb %ymm0, %xmm0
+; AVX512BWVL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512BWVL-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7]
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
 ;
 ; AVX512VBMIVL-LABEL: trunc_v4i64_to_v4i8_return_v16i8:
 ; AVX512VBMIVL:       # %bb.0:
 ; AVX512VBMIVL-NEXT:    vpmovqb %ymm0, %xmm0
+; AVX512VBMIVL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512VBMIVL-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7]
 ; AVX512VBMIVL-NEXT:    vzeroupper
 ; AVX512VBMIVL-NEXT:    retq
   %truncated = trunc <4 x i64> %vec to <4 x i8>
@@ -1096,9 +1135,13 @@ define void @shuffle_v16i16_to_v4i16(<16 x i16>* %L, <4 x i16>* %S) nounwind {
 ;
 ; AVX512VL-LABEL: shuffle_v16i16_to_v4i16:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vmovaps (%rdi), %xmm0
-; AVX512VL-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],mem[0,2]
-; AVX512VL-NEXT:    vpmovdw %xmm0, (%rsi)
+; AVX512VL-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512VL-NEXT:    vmovdqa 16(%rdi), %xmm1
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,1,8,9,8,9,10,11,8,9,10,11,12,13,14,15]
+; AVX512VL-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
+; AVX512VL-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX512VL-NEXT:    vmovq %xmm0, (%rsi)
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: shuffle_v16i16_to_v4i16:
@@ -1114,16 +1157,18 @@ define void @shuffle_v16i16_to_v4i16(<16 x i16>* %L, <4 x i16>* %S) nounwind {
 ;
 ; AVX512BWVL-LABEL: shuffle_v16i16_to_v4i16:
 ; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vmovaps (%rdi), %xmm0
-; AVX512BWVL-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],mem[0,2]
-; AVX512BWVL-NEXT:    vpmovdw %xmm0, (%rsi)
+; AVX512BWVL-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512BWVL-NEXT:    vmovdqa {{.*#+}} xmm1 = [0,4,8,12,4,5,12,13]
+; AVX512BWVL-NEXT:    vpermi2w 16(%rdi), %xmm0, %xmm1
+; AVX512BWVL-NEXT:    vmovq %xmm1, (%rsi)
 ; AVX512BWVL-NEXT:    retq
 ;
 ; AVX512VBMIVL-LABEL: shuffle_v16i16_to_v4i16:
 ; AVX512VBMIVL:       # %bb.0:
-; AVX512VBMIVL-NEXT:    vmovaps (%rdi), %xmm0
-; AVX512VBMIVL-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],mem[0,2]
-; AVX512VBMIVL-NEXT:    vpmovdw %xmm0, (%rsi)
+; AVX512VBMIVL-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512VBMIVL-NEXT:    vmovdqa {{.*#+}} xmm1 = [0,4,8,12,4,5,12,13]
+; AVX512VBMIVL-NEXT:    vpermi2w 16(%rdi), %xmm0, %xmm1
+; AVX512VBMIVL-NEXT:    vmovq %xmm1, (%rsi)
 ; AVX512VBMIVL-NEXT:    retq
   %vec = load <16 x i16>, <16 x i16>* %L
   %strided.vec = shufflevector <16 x i16> %vec, <16 x i16> undef, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
@@ -1134,34 +1179,39 @@ define void @shuffle_v16i16_to_v4i16(<16 x i16>* %L, <4 x i16>* %S) nounwind {
 define void @trunc_v4i64_to_v4i16(<16 x i16>* %L, <4 x i16>* %S) nounwind {
 ; AVX1-LABEL: trunc_v4i64_to_v4i16:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovaps (%rdi), %xmm0
-; AVX1-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],mem[0,2]
-; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = mem[0,2,2,3]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = mem[0,2,2,3]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
+; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
 ; AVX1-NEXT:    vmovq %xmm0, (%rsi)
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-SLOW-LABEL: trunc_v4i64_to_v4i16:
 ; AVX2-SLOW:       # %bb.0:
-; AVX2-SLOW-NEXT:    vmovaps (%rdi), %xmm0
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],mem[0,2]
-; AVX2-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
+; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = mem[0,2,2,3]
+; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = mem[0,2,2,3]
+; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
 ; AVX2-SLOW-NEXT:    vmovq %xmm0, (%rsi)
 ; AVX2-SLOW-NEXT:    retq
 ;
 ; AVX2-FAST-LABEL: trunc_v4i64_to_v4i16:
 ; AVX2-FAST:       # %bb.0:
-; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} ymm0 = [0,2,4,6,4,6,6,7]
-; AVX2-FAST-NEXT:    vpermd (%rdi), %ymm0, %ymm0
-; AVX2-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
+; AVX2-FAST-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX2-FAST-NEXT:    vmovdqa 16(%rdi), %xmm1
+; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,1,8,9,8,9,10,11,8,9,10,11,12,13,14,15]
+; AVX2-FAST-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
+; AVX2-FAST-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
+; AVX2-FAST-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; AVX2-FAST-NEXT:    vmovq %xmm0, (%rsi)
-; AVX2-FAST-NEXT:    vzeroupper
 ; AVX2-FAST-NEXT:    retq
 ;
 ; AVX512F-LABEL: trunc_v4i64_to_v4i16:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovdqa (%rdi), %ymm0
-; AVX512F-NEXT:    vpmovqd %zmm0, %ymm0
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
+; AVX512F-NEXT:    vpmovqw %zmm0, %xmm0
 ; AVX512F-NEXT:    vmovq %xmm0, (%rsi)
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
@@ -1176,8 +1226,7 @@ define void @trunc_v4i64_to_v4i16(<16 x i16>* %L, <4 x i16>* %S) nounwind {
 ; AVX512BW-LABEL: trunc_v4i64_to_v4i16:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    vmovdqa (%rdi), %ymm0
-; AVX512BW-NEXT:    vpmovqd %zmm0, %ymm0
-; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
+; AVX512BW-NEXT:    vpmovqw %zmm0, %xmm0
 ; AVX512BW-NEXT:    vmovq %xmm0, (%rsi)
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -1227,9 +1276,13 @@ define void @shuffle_v32i8_to_v4i8(<32 x i8>* %L, <4 x i8>* %S) nounwind {
 ;
 ; AVX512VL-LABEL: shuffle_v32i8_to_v4i8:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vmovaps (%rdi), %xmm0
-; AVX512VL-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],mem[0,2]
-; AVX512VL-NEXT:    vpmovdb %xmm0, (%rsi)
+; AVX512VL-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512VL-NEXT:    vmovdqa 16(%rdi), %xmm1
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm2 = <0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX512VL-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
+; AVX512VL-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX512VL-NEXT:    vmovd %xmm0, (%rsi)
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: shuffle_v32i8_to_v4i8:
@@ -1245,16 +1298,21 @@ define void @shuffle_v32i8_to_v4i8(<32 x i8>* %L, <4 x i8>* %S) nounwind {
 ;
 ; AVX512BWVL-LABEL: shuffle_v32i8_to_v4i8:
 ; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vmovaps (%rdi), %xmm0
-; AVX512BWVL-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],mem[0,2]
-; AVX512BWVL-NEXT:    vpmovdb %xmm0, (%rsi)
+; AVX512BWVL-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512BWVL-NEXT:    vmovdqa 16(%rdi), %xmm1
+; AVX512BWVL-NEXT:    vmovdqa {{.*#+}} xmm2 = <0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX512BWVL-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
+; AVX512BWVL-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
+; AVX512BWVL-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX512BWVL-NEXT:    vmovd %xmm0, (%rsi)
 ; AVX512BWVL-NEXT:    retq
 ;
 ; AVX512VBMIVL-LABEL: shuffle_v32i8_to_v4i8:
 ; AVX512VBMIVL:       # %bb.0:
-; AVX512VBMIVL-NEXT:    vmovaps (%rdi), %xmm0
-; AVX512VBMIVL-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],mem[0,2]
-; AVX512VBMIVL-NEXT:    vpmovdb %xmm0, (%rsi)
+; AVX512VBMIVL-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512VBMIVL-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [403703808,403703808,403703808,403703808]
+; AVX512VBMIVL-NEXT:    vpermi2b 16(%rdi), %xmm0, %xmm1
+; AVX512VBMIVL-NEXT:    vmovd %xmm1, (%rsi)
 ; AVX512VBMIVL-NEXT:    retq
   %vec = load <32 x i8>, <32 x i8>* %L
   %strided.vec = shufflevector <32 x i8> %vec, <32 x i8> undef, <4 x i32> <i32 0, i32 8, i32 16, i32 24>
@@ -1263,36 +1321,21 @@ define void @shuffle_v32i8_to_v4i8(<32 x i8>* %L, <4 x i8>* %S) nounwind {
 }
 
 define void @trunc_v4i64_to_v4i8(<32 x i8>* %L, <4 x i8>* %S) nounwind {
-; AVX1-LABEL: trunc_v4i64_to_v4i8:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovaps (%rdi), %xmm0
-; AVX1-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],mem[0,2]
-; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX1-NEXT:    vmovd %xmm0, (%rsi)
-; AVX1-NEXT:    retq
-;
-; AVX2-SLOW-LABEL: trunc_v4i64_to_v4i8:
-; AVX2-SLOW:       # %bb.0:
-; AVX2-SLOW-NEXT:    vmovaps (%rdi), %xmm0
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],mem[0,2]
-; AVX2-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX2-SLOW-NEXT:    vmovd %xmm0, (%rsi)
-; AVX2-SLOW-NEXT:    retq
-;
-; AVX2-FAST-LABEL: trunc_v4i64_to_v4i8:
-; AVX2-FAST:       # %bb.0:
-; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} ymm0 = [0,2,4,6,4,6,6,7]
-; AVX2-FAST-NEXT:    vpermd (%rdi), %ymm0, %ymm0
-; AVX2-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX2-FAST-NEXT:    vmovd %xmm0, (%rsi)
-; AVX2-FAST-NEXT:    vzeroupper
-; AVX2-FAST-NEXT:    retq
+; AVX-LABEL: trunc_v4i64_to_v4i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX-NEXT:    vmovdqa 16(%rdi), %xmm1
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm2 = <0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
+; AVX-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX-NEXT:    vmovd %xmm0, (%rsi)
+; AVX-NEXT:    retq
 ;
 ; AVX512F-LABEL: trunc_v4i64_to_v4i8:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovdqa (%rdi), %ymm0
-; AVX512F-NEXT:    vpmovqd %zmm0, %ymm0
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512F-NEXT:    vpmovqb %zmm0, %xmm0
 ; AVX512F-NEXT:    vmovd %xmm0, (%rsi)
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
@@ -1307,8 +1350,7 @@ define void @trunc_v4i64_to_v4i8(<32 x i8>* %L, <4 x i8>* %S) nounwind {
 ; AVX512BW-LABEL: trunc_v4i64_to_v4i8:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    vmovdqa (%rdi), %ymm0
-; AVX512BW-NEXT:    vpmovqd %zmm0, %ymm0
-; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512BW-NEXT:    vpmovqb %zmm0, %xmm0
 ; AVX512BW-NEXT:    vmovd %xmm0, (%rsi)
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/shuffle-vs-trunc-512.ll b/llvm/test/CodeGen/X86/shuffle-vs-trunc-512.ll
index 8e0c1d796..7f00b49b8 100644
--- a/llvm/test/CodeGen/X86/shuffle-vs-trunc-512.ll
+++ b/llvm/test/CodeGen/X86/shuffle-vs-trunc-512.ll
@@ -549,20 +549,13 @@ define void @shuffle_v64i8_to_v8i8(<64 x i8>* %L, <8 x i8>* %S) nounwind {
 ;
 ; AVX512VL-LABEL: shuffle_v64i8_to_v8i8:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512VL-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512VL-NEXT:    vmovdqa 32(%rdi), %xmm2
-; AVX512VL-NEXT:    vmovdqa 48(%rdi), %xmm3
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm4 = <u,u,0,8,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512VL-NEXT:    vpshufb %xmm4, %xmm3, %xmm3
-; AVX512VL-NEXT:    vpshufb %xmm4, %xmm2, %xmm2
-; AVX512VL-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm3 = <0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512VL-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX512VL-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2,3]
+; AVX512VL-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX512VL-NEXT:    vmovdqa 32(%rdi), %ymm1
+; AVX512VL-NEXT:    vpmovqb %ymm1, %xmm1
+; AVX512VL-NEXT:    vpmovqb %ymm0, %xmm0
+; AVX512VL-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; AVX512VL-NEXT:    vmovq %xmm0, (%rsi)
+; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: shuffle_v64i8_to_v8i8:
@@ -585,10 +578,12 @@ define void @shuffle_v64i8_to_v8i8(<64 x i8>* %L, <8 x i8>* %S) nounwind {
 ;
 ; AVX512BWVL-LABEL: shuffle_v64i8_to_v8i8:
 ; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vmovdqa {{.*#+}} xmm0 = [0,4,8,12,16,20,24,28]
-; AVX512BWVL-NEXT:    vmovdqa (%rdi), %ymm1
-; AVX512BWVL-NEXT:    vpermt2w 32(%rdi), %ymm0, %ymm1
-; AVX512BWVL-NEXT:    vpmovwb %xmm1, (%rsi)
+; AVX512BWVL-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX512BWVL-NEXT:    vmovdqa 32(%rdi), %ymm1
+; AVX512BWVL-NEXT:    vpmovqb %ymm1, %xmm1
+; AVX512BWVL-NEXT:    vpmovqb %ymm0, %xmm0
+; AVX512BWVL-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX512BWVL-NEXT:    vmovq %xmm0, (%rsi)
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
 ;
@@ -612,10 +607,10 @@ define void @shuffle_v64i8_to_v8i8(<64 x i8>* %L, <8 x i8>* %S) nounwind {
 ;
 ; AVX512VBMIVL-LABEL: shuffle_v64i8_to_v8i8:
 ; AVX512VBMIVL:       # %bb.0:
-; AVX512VBMIVL-NEXT:    vmovdqa {{.*#+}} xmm0 = [0,4,8,12,16,20,24,28]
-; AVX512VBMIVL-NEXT:    vmovdqa (%rdi), %ymm1
-; AVX512VBMIVL-NEXT:    vpermt2w 32(%rdi), %ymm0, %ymm1
-; AVX512VBMIVL-NEXT:    vpmovwb %xmm1, (%rsi)
+; AVX512VBMIVL-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX512VBMIVL-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [4048780183313844224,4048780183313844224,4048780183313844224,4048780183313844224]
+; AVX512VBMIVL-NEXT:    vpermi2b 32(%rdi), %ymm0, %ymm1
+; AVX512VBMIVL-NEXT:    vmovq %xmm1, (%rsi)
 ; AVX512VBMIVL-NEXT:    vzeroupper
 ; AVX512VBMIVL-NEXT:    retq
   %vec = load <64 x i8>, <64 x i8>* %L
@@ -641,6 +636,7 @@ define void @trunc_v8i64_to_v8i8(<64 x i8>* %L, <8 x i8>* %S) nounwind {
 define <16 x i8> @trunc_shuffle_v64i8_01_05_09_13_17_21_25_29_33_37_41_45_49_53_57_61(<64 x i8> %x) {
 ; AVX512F-LABEL: trunc_shuffle_v64i8_01_05_09_13_17_21_25_29_33_37_41_45_49_53_57_61:
 ; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm2
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm3 = <u,u,u,u,1,5,9,13,u,u,u,u,u,u,u,u>
 ; AVX512F-NEXT:    vpshufb %xmm3, %xmm2, %xmm2
@@ -657,6 +653,7 @@ define <16 x i8> @trunc_shuffle_v64i8_01_05_09_13_17_21_25_29_33_37_41_45_49_53_
 ;
 ; AVX512VL-LABEL: trunc_shuffle_v64i8_01_05_09_13_17_21_25_29_33_37_41_45_49_53_57_61:
 ; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512VL-NEXT:    vextracti128 $1, %ymm1, %xmm2
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm3 = <u,u,u,u,1,5,9,13,u,u,u,u,u,u,u,u>
 ; AVX512VL-NEXT:    vpshufb %xmm3, %xmm2, %xmm2
@@ -728,31 +725,33 @@ define <16 x i8> @trunc_shuffle_v64i8_01_05_09_13_17_21_25_29_33_37_41_45_49_53_
 define <16 x i8> @trunc_shuffle_v64i8_01_05_09_13_17_21_25_29_33_37_41_45_49_53_57_62(<64 x i8> %x) {
 ; AVX512F-LABEL: trunc_shuffle_v64i8_01_05_09_13_17_21_25_29_33_37_41_45_49_53_57_62:
 ; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm2 = <1,5,9,13,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX512F-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
+; AVX512F-NEXT:    vpshufb %xmm2, %xmm0, %xmm2
+; AVX512F-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
 ; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm2
-; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm3 = <1,5,9,13,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512F-NEXT:    vpshufb %xmm3, %xmm2, %xmm2
-; AVX512F-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
-; AVX512F-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm2
 ; AVX512F-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u,u,u,u,1,5,9,14,u,u,u,u,u,u,u,u]
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[u,u,u,u,1,5,9,13,u,u,u,u,u,u,u,u]
-; AVX512F-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
-; AVX512F-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
+; AVX512F-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,1,5,9,13,u,u,u,u,u,u,u,u]
+; AVX512F-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; AVX512F-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: trunc_shuffle_v64i8_01_05_09_13_17_21_25_29_33_37_41_45_49_53_57_62:
 ; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm2 = <1,5,9,13,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX512VL-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
+; AVX512VL-NEXT:    vpshufb %xmm2, %xmm0, %xmm2
+; AVX512VL-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
 ; AVX512VL-NEXT:    vextracti128 $1, %ymm0, %xmm2
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm3 = <1,5,9,13,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512VL-NEXT:    vpshufb %xmm3, %xmm2, %xmm2
-; AVX512VL-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; AVX512VL-NEXT:    vextracti128 $1, %ymm1, %xmm2
 ; AVX512VL-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u,u,u,u,1,5,9,14,u,u,u,u,u,u,u,u]
-; AVX512VL-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[u,u,u,u,1,5,9,13,u,u,u,u,u,u,u,u]
-; AVX512VL-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
-; AVX512VL-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
+; AVX512VL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,1,5,9,13,u,u,u,u,u,u,u,u]
+; AVX512VL-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; AVX512VL-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
 ; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq
 ;
@@ -815,7 +814,7 @@ define <4 x double> @PR34175(<32 x i16>* %p) {
 ; AVX512F-NEXT:    vmovdqu 32(%rdi), %xmm1
 ; AVX512F-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3]
 ; AVX512F-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1],xmm1[2],mem[2],xmm1[3],mem[3]
-; AVX512F-NEXT:    vpbroadcastd %xmm1, %xmm1
+; AVX512F-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,1,1]
 ; AVX512F-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3]
 ; AVX512F-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; AVX512F-NEXT:    vcvtdq2pd %xmm0, %ymm0
@@ -839,7 +838,7 @@ define <4 x double> @PR34175(<32 x i16>* %p) {
 ; AVX512BW-NEXT:    vmovdqu 32(%rdi), %xmm1
 ; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3]
 ; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1],xmm1[2],mem[2],xmm1[3],mem[3]
-; AVX512BW-NEXT:    vpbroadcastd %xmm1, %xmm1
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,1,1]
 ; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3]
 ; AVX512BW-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; AVX512BW-NEXT:    vcvtdq2pd %xmm0, %ymm0
@@ -860,7 +859,7 @@ define <4 x double> @PR34175(<32 x i16>* %p) {
 ; AVX512VBMI-NEXT:    vmovdqu 32(%rdi), %xmm1
 ; AVX512VBMI-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3]
 ; AVX512VBMI-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1],xmm1[2],mem[2],xmm1[3],mem[3]
-; AVX512VBMI-NEXT:    vpbroadcastd %xmm1, %xmm1
+; AVX512VBMI-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,1,1]
 ; AVX512VBMI-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3]
 ; AVX512VBMI-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; AVX512VBMI-NEXT:    vcvtdq2pd %xmm0, %ymm0
@@ -884,6 +883,7 @@ define <16 x i8> @trunc_v8i64_to_v8i8_return_v16i8(<8 x i64> %vec) nounwind {
 ; AVX512-LABEL: trunc_v8i64_to_v8i8_return_v16i8:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpmovqb %zmm0, %xmm0
+; AVX512-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
   %truncated = trunc <8 x i64> %vec to <8 x i8>
diff --git a/llvm/test/CodeGen/X86/sincos-opt.ll b/llvm/test/CodeGen/X86/sincos-opt.ll
index b64450863..821f2b144 100644
--- a/llvm/test/CodeGen/X86/sincos-opt.ll
+++ b/llvm/test/CodeGen/X86/sincos-opt.ll
@@ -5,6 +5,7 @@
 ; RUN: llc < %s -mtriple=x86_64-pc-linux-gnux32 -mcpu=core2 -enable-unsafe-fp-math | FileCheck %s --check-prefix=GNU_SINCOS_FASTMATH
 ; RUN: llc < %s -mtriple=x86_64-fuchsia -mcpu=core2 | FileCheck %s --check-prefix=GNU_SINCOS
 ; RUN: llc < %s -mtriple=x86_64-fuchsia -mcpu=core2 -enable-unsafe-fp-math | FileCheck %s --check-prefix=GNU_SINCOS_FASTMATH
+; RUN: llc < %s -mtriple=x86_64-scei-ps4 -mcpu=btver2 | FileCheck %s --check-prefix=PS4_SINCOS
 
 ; Combine sin / cos into a single call unless they may write errno (as
 ; captured by readnone attrbiute, controlled by clang -fmath-errno
@@ -32,6 +33,11 @@ entry:
 ; OSX_NOOPT-LABEL: test1:
 ; OSX_NOOPT: callq _sinf
 ; OSX_NOOPT: callq _cosf
+
+; PS4_SINCOS-LABEL: test1:
+; PS4_SINCOS: callq sincosf
+; PS4_SINCOS: vmovss 4(%rsp), %xmm0
+; PS4_SINCOS: vaddss (%rsp), %xmm0, %xmm0
   %call = tail call float @sinf(float %x) readnone
   %call1 = tail call float @cosf(float %x) readnone
   %add = fadd float %call, %call1
@@ -55,6 +61,10 @@ entry:
 ; OSX_NOOPT-LABEL: test1_errno:
 ; OSX_NOOPT: callq _sinf
 ; OSX_NOOPT: callq _cosf
+
+; PS4_SINCOS-LABEL: test1_errno:
+; PS4_SINCOS: callq sinf
+; PS4_SINCOS: callq cosf
   %call = tail call float @sinf(float %x)
   %call1 = tail call float @cosf(float %x)
   %add = fadd float %call, %call1
@@ -80,6 +90,11 @@ entry:
 ; OSX_NOOPT-LABEL: test2:
 ; OSX_NOOPT: callq _sin
 ; OSX_NOOPT: callq _cos
+
+; PS4_SINCOS-LABEL: test2:
+; PS4_SINCOS: callq sincos
+; PS4_SINCOS: vmovsd 16(%rsp), %xmm0
+; PS4_SINCOS: vaddsd 8(%rsp), %xmm0, %xmm0
   %call = tail call double @sin(double %x) readnone
   %call1 = tail call double @cos(double %x) readnone
   %add = fadd double %call, %call1
@@ -103,6 +118,10 @@ entry:
 ; OSX_NOOPT-LABEL: test2_errno:
 ; OSX_NOOPT: callq _sin
 ; OSX_NOOPT: callq _cos
+
+; PS4_SINCOS-LABEL: test2_errno:
+; PS4_SINCOS: callq sin
+; PS4_SINCOS: callq cos
   %call = tail call double @sin(double %x)
   %call1 = tail call double @cos(double %x)
   %add = fadd double %call, %call1
@@ -122,6 +141,10 @@ entry:
 ; GNU_SINCOS_FASTMATH: fldt 16(%{{[re]}}sp)
 ; GNU_SINCOS_FASTMATH: fldt 32(%{{[re]}}sp)
 ; GNU_SINCOS_FASTMATH: faddp %st, %st(1)
+
+; PS4_SINCOS-LABEL: test3:
+; PS4_SINCOS: callq sinl
+; PS4_SINCOS: callq cosl
   %call = tail call x86_fp80 @sinl(x86_fp80 %x) readnone
   %call1 = tail call x86_fp80 @cosl(x86_fp80 %x) readnone
   %add = fadd x86_fp80 %call, %call1
@@ -137,6 +160,10 @@ entry:
 ; GNU_SINCOS_FASTMATH-LABEL: test3_errno:
 ; GNU_SINCOS_FASTMATH: callq sinl
 ; GNU_SINCOS_FASTMATH: callq cosl
+
+; PS4_SINCOS-LABEL: test3_errno:
+; PS4_SINCOS: callq sinl
+; PS4_SINCOS: callq cosl
   %call = tail call x86_fp80 @sinl(x86_fp80 %x)
   %call1 = tail call x86_fp80 @cosl(x86_fp80 %x)
   %add = fadd x86_fp80 %call, %call1
diff --git a/llvm/test/CodeGen/X86/sjlj-baseptr.ll b/llvm/test/CodeGen/X86/sjlj-baseptr.ll
index e439ff4db..93ff0eb8a 100644
--- a/llvm/test/CodeGen/X86/sjlj-baseptr.ll
+++ b/llvm/test/CodeGen/X86/sjlj-baseptr.ll
@@ -9,7 +9,7 @@ declare i32 @llvm.eh.sjlj.setjmp(i8*) nounwind
 
 declare void @whatever(i64, %Foo*, i8**, i8*, i8*, i32)  #0
 
-attributes #0 = { nounwind uwtable "no-frame-pointer-elim"="true" }
+attributes #0 = { nounwind uwtable "frame-pointer"="all" }
 
 define i32 @test1(i64 %n, %Foo* byval nocapture readnone align 8 %f) #0 {
 entry:
diff --git a/llvm/test/CodeGen/X86/slow-incdec.ll b/llvm/test/CodeGen/X86/slow-incdec.ll
index 83bf250d4..6f4ad98a5 100644
--- a/llvm/test/CodeGen/X86/slow-incdec.ll
+++ b/llvm/test/CodeGen/X86/slow-incdec.ll
@@ -54,6 +54,26 @@ define i32 @dec_size(i32 %x) optsize {
   ret i32 %r
 }
 
+define i32 @inc_pgso(i32 %x) !prof !14 {
+; CHECK-LABEL: inc_pgso:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    incl %eax
+; CHECK-NEXT:    retl
+  %r = add i32 %x, 1
+  ret i32 %r
+}
+
+define i32 @dec_pgso(i32 %x) !prof !14 {
+; CHECK-LABEL: dec_pgso:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    decl %eax
+; CHECK-NEXT:    retl
+  %r = add i32 %x, -1
+  ret i32 %r
+}
+
 declare {i32, i1} @llvm.uadd.with.overflow.i32(i32, i32)
 declare void @other(i32* ) nounwind;
 
@@ -62,20 +82,20 @@ define void @cond_ae_to_cond_ne(i32* %p) nounwind {
 ; INCDEC:       # %bb.0: # %entry
 ; INCDEC-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; INCDEC-NEXT:    incl (%eax)
-; INCDEC-NEXT:    jne .LBB4_1
+; INCDEC-NEXT:    jne .LBB6_1
 ; INCDEC-NEXT:  # %bb.2: # %if.end4
 ; INCDEC-NEXT:    jmp other # TAILCALL
-; INCDEC-NEXT:  .LBB4_1: # %return
+; INCDEC-NEXT:  .LBB6_1: # %return
 ; INCDEC-NEXT:    retl
 ;
 ; ADD-LABEL: cond_ae_to_cond_ne:
 ; ADD:       # %bb.0: # %entry
 ; ADD-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; ADD-NEXT:    addl $1, (%eax)
-; ADD-NEXT:    jne .LBB4_1
+; ADD-NEXT:    jne .LBB6_1
 ; ADD-NEXT:  # %bb.2: # %if.end4
 ; ADD-NEXT:    jmp other # TAILCALL
-; ADD-NEXT:  .LBB4_1: # %return
+; ADD-NEXT:  .LBB6_1: # %return
 ; ADD-NEXT:    retl
 entry:
   %t0 = load i32, i32* %p, align 8
@@ -109,10 +129,10 @@ define void @test_tail_call(i32* %ptr) nounwind {
 ; INCDEC-NEXT:    incb a
 ; INCDEC-NEXT:    sete d
 ; INCDEC-NEXT:    testb %al, %al
-; INCDEC-NEXT:    jne .LBB5_2
+; INCDEC-NEXT:    jne .LBB7_2
 ; INCDEC-NEXT:  # %bb.1: # %then
 ; INCDEC-NEXT:    jmp external_a # TAILCALL
-; INCDEC-NEXT:  .LBB5_2: # %else
+; INCDEC-NEXT:  .LBB7_2: # %else
 ; INCDEC-NEXT:    jmp external_b # TAILCALL
 ;
 ; ADD-LABEL: test_tail_call:
@@ -123,10 +143,10 @@ define void @test_tail_call(i32* %ptr) nounwind {
 ; ADD-NEXT:    addb $1, a
 ; ADD-NEXT:    sete d
 ; ADD-NEXT:    testb %al, %al
-; ADD-NEXT:    jne .LBB5_2
+; ADD-NEXT:    jne .LBB7_2
 ; ADD-NEXT:  # %bb.1: # %then
 ; ADD-NEXT:    jmp external_a # TAILCALL
-; ADD-NEXT:  .LBB5_2: # %else
+; ADD-NEXT:  .LBB7_2: # %else
 ; ADD-NEXT:    jmp external_b # TAILCALL
 entry:
   %val = load i32, i32* %ptr
@@ -152,3 +172,19 @@ else:
   ret void
 }
 
+!llvm.module.flags = !{!0}
+!0 = !{i32 1, !"ProfileSummary", !1}
+!1 = !{!2, !3, !4, !5, !6, !7, !8, !9}
+!2 = !{!"ProfileFormat", !"InstrProf"}
+!3 = !{!"TotalCount", i64 10000}
+!4 = !{!"MaxCount", i64 10}
+!5 = !{!"MaxInternalCount", i64 1}
+!6 = !{!"MaxFunctionCount", i64 1000}
+!7 = !{!"NumCounts", i64 3}
+!8 = !{!"NumFunctions", i64 3}
+!9 = !{!"DetailedSummary", !10}
+!10 = !{!11, !12, !13}
+!11 = !{i32 10000, i64 100, i32 1}
+!12 = !{i32 999000, i64 100, i32 1}
+!13 = !{i32 999999, i64 1, i32 2}
+!14 = !{!"function_entry_count", i64 0}
diff --git a/llvm/test/CodeGen/X86/slow-pmulld.ll b/llvm/test/CodeGen/X86/slow-pmulld.ll
index 007531fca..0b79b62f8 100644
--- a/llvm/test/CodeGen/X86/slow-pmulld.ll
+++ b/llvm/test/CodeGen/X86/slow-pmulld.ll
@@ -20,74 +20,74 @@
 define <4 x i32> @test_mul_v4i32_v4i8(<4 x i8> %A) {
 ; CHECK32-LABEL: test_mul_v4i32_v4i8:
 ; CHECK32:       # %bb.0:
-; CHECK32-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; CHECK32-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; CHECK32-NEXT:    pmaddwd {{\.LCPI.*}}, %xmm0
 ; CHECK32-NEXT:    retl
 ;
 ; CHECK64-LABEL: test_mul_v4i32_v4i8:
 ; CHECK64:       # %bb.0:
-; CHECK64-NEXT:    pand {{.*}}(%rip), %xmm0
+; CHECK64-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; CHECK64-NEXT:    pmaddwd {{.*}}(%rip), %xmm0
 ; CHECK64-NEXT:    retq
 ;
 ; SSE4-32-LABEL: test_mul_v4i32_v4i8:
 ; SSE4-32:       # %bb.0:
-; SSE4-32-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; SSE4-32-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SSE4-32-NEXT:    pmaddwd {{\.LCPI.*}}, %xmm0
 ; SSE4-32-NEXT:    retl
 ;
 ; SSE4-64-LABEL: test_mul_v4i32_v4i8:
 ; SSE4-64:       # %bb.0:
-; SSE4-64-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE4-64-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SSE4-64-NEXT:    pmaddwd {{.*}}(%rip), %xmm0
 ; SSE4-64-NEXT:    retq
 ;
 ; AVX2-32-LABEL: test_mul_v4i32_v4i8:
 ; AVX2-32:       # %bb.0:
-; AVX2-32-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; AVX2-32-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; AVX2-32-NEXT:    vpmaddwd {{\.LCPI.*}}, %xmm0, %xmm0
 ; AVX2-32-NEXT:    retl
 ;
 ; AVX2-64-LABEL: test_mul_v4i32_v4i8:
 ; AVX2-64:       # %bb.0:
-; AVX2-64-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX2-64-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; AVX2-64-NEXT:    vpmaddwd {{.*}}(%rip), %xmm0, %xmm0
 ; AVX2-64-NEXT:    retq
 ;
 ; AVX512DQ-32-LABEL: test_mul_v4i32_v4i8:
 ; AVX512DQ-32:       # %bb.0:
-; AVX512DQ-32-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; AVX512DQ-32-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; AVX512DQ-32-NEXT:    vpmaddwd {{\.LCPI.*}}, %xmm0, %xmm0
 ; AVX512DQ-32-NEXT:    retl
 ;
 ; AVX512DQ-64-LABEL: test_mul_v4i32_v4i8:
 ; AVX512DQ-64:       # %bb.0:
-; AVX512DQ-64-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX512DQ-64-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; AVX512DQ-64-NEXT:    vpmaddwd {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512DQ-64-NEXT:    retq
 ;
 ; AVX512BW-32-LABEL: test_mul_v4i32_v4i8:
 ; AVX512BW-32:       # %bb.0:
-; AVX512BW-32-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; AVX512BW-32-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; AVX512BW-32-NEXT:    vpmaddwd {{\.LCPI.*}}, %xmm0, %xmm0
 ; AVX512BW-32-NEXT:    retl
 ;
 ; AVX512BW-64-LABEL: test_mul_v4i32_v4i8:
 ; AVX512BW-64:       # %bb.0:
-; AVX512BW-64-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX512BW-64-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; AVX512BW-64-NEXT:    vpmaddwd {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512BW-64-NEXT:    retq
 ;
 ; KNL-32-LABEL: test_mul_v4i32_v4i8:
 ; KNL-32:       # %bb.0:
-; KNL-32-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; KNL-32-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; KNL-32-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [18778,18778,18778,18778]
 ; KNL-32-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; KNL-32-NEXT:    retl
 ;
 ; KNL-64-LABEL: test_mul_v4i32_v4i8:
 ; KNL-64:       # %bb.0:
-; KNL-64-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; KNL-64-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; KNL-64-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [18778,18778,18778,18778]
 ; KNL-64-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; KNL-64-NEXT:    retq
@@ -99,34 +99,31 @@ define <4 x i32> @test_mul_v4i32_v4i8(<4 x i8> %A) {
 define <8 x i32> @test_mul_v8i32_v8i8(<8 x i8> %A) {
 ; SLM32-LABEL: test_mul_v8i32_v8i8:
 ; SLM32:       # %bb.0:
-; SLM32-NEXT:    movdqa %xmm0, %xmm1
-; SLM32-NEXT:    pand {{\.LCPI.*}}, %xmm1
+; SLM32-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; SLM32-NEXT:    movdqa {{.*#+}} xmm0 = [18778,18778,18778,18778,18778,18778,18778,18778]
 ; SLM32-NEXT:    movdqa %xmm1, %xmm2
 ; SLM32-NEXT:    pmullw %xmm0, %xmm1
 ; SLM32-NEXT:    pmulhw %xmm0, %xmm2
 ; SLM32-NEXT:    movdqa %xmm1, %xmm0
-; SLM32-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
 ; SLM32-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; SLM32-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
 ; SLM32-NEXT:    retl
 ;
 ; SLM64-LABEL: test_mul_v8i32_v8i8:
 ; SLM64:       # %bb.0:
-; SLM64-NEXT:    movdqa %xmm0, %xmm1
-; SLM64-NEXT:    pand {{.*}}(%rip), %xmm1
+; SLM64-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; SLM64-NEXT:    movdqa {{.*#+}} xmm0 = [18778,18778,18778,18778,18778,18778,18778,18778]
 ; SLM64-NEXT:    movdqa %xmm1, %xmm2
 ; SLM64-NEXT:    pmullw %xmm0, %xmm1
 ; SLM64-NEXT:    pmulhw %xmm0, %xmm2
 ; SLM64-NEXT:    movdqa %xmm1, %xmm0
-; SLM64-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
 ; SLM64-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; SLM64-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
 ; SLM64-NEXT:    retq
 ;
 ; SLOW32-LABEL: test_mul_v8i32_v8i8:
 ; SLOW32:       # %bb.0:
-; SLOW32-NEXT:    movdqa %xmm0, %xmm1
-; SLOW32-NEXT:    pand {{\.LCPI.*}}, %xmm1
+; SLOW32-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; SLOW32-NEXT:    movdqa {{.*#+}} xmm0 = [18778,18778,18778,18778,18778,18778,18778,18778]
 ; SLOW32-NEXT:    movdqa %xmm1, %xmm2
 ; SLOW32-NEXT:    pmulhw %xmm0, %xmm2
@@ -138,8 +135,7 @@ define <8 x i32> @test_mul_v8i32_v8i8(<8 x i8> %A) {
 ;
 ; SLOW64-LABEL: test_mul_v8i32_v8i8:
 ; SLOW64:       # %bb.0:
-; SLOW64-NEXT:    movdqa %xmm0, %xmm1
-; SLOW64-NEXT:    pand {{.*}}(%rip), %xmm1
+; SLOW64-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; SLOW64-NEXT:    movdqa {{.*#+}} xmm0 = [18778,18778,18778,18778,18778,18778,18778,18778]
 ; SLOW64-NEXT:    movdqa %xmm1, %xmm2
 ; SLOW64-NEXT:    pmulhw %xmm0, %xmm2
@@ -151,10 +147,9 @@ define <8 x i32> @test_mul_v8i32_v8i8(<8 x i8> %A) {
 ;
 ; SSE4-32-LABEL: test_mul_v8i32_v8i8:
 ; SSE4-32:       # %bb.0:
-; SSE4-32-NEXT:    pand {{\.LCPI.*}}, %xmm0
-; SSE4-32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; SSE4-32-NEXT:    pmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
-; SSE4-32-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; SSE4-32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE4-32-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
+; SSE4-32-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SSE4-32-NEXT:    movdqa {{.*#+}} xmm2 = [18778,18778,18778,18778]
 ; SSE4-32-NEXT:    pmaddwd %xmm2, %xmm0
 ; SSE4-32-NEXT:    pmaddwd %xmm2, %xmm1
@@ -162,10 +157,9 @@ define <8 x i32> @test_mul_v8i32_v8i8(<8 x i8> %A) {
 ;
 ; SSE4-64-LABEL: test_mul_v8i32_v8i8:
 ; SSE4-64:       # %bb.0:
-; SSE4-64-NEXT:    pand {{.*}}(%rip), %xmm0
-; SSE4-64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; SSE4-64-NEXT:    pmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
-; SSE4-64-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; SSE4-64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE4-64-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
+; SSE4-64-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SSE4-64-NEXT:    movdqa {{.*#+}} xmm2 = [18778,18778,18778,18778]
 ; SSE4-64-NEXT:    pmaddwd %xmm2, %xmm0
 ; SSE4-64-NEXT:    pmaddwd %xmm2, %xmm1
@@ -173,58 +167,50 @@ define <8 x i32> @test_mul_v8i32_v8i8(<8 x i8> %A) {
 ;
 ; AVX2-32-LABEL: test_mul_v8i32_v8i8:
 ; AVX2-32:       # %bb.0:
-; AVX2-32-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
-; AVX2-32-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2-32-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
 ; AVX2-32-NEXT:    vpmaddwd {{\.LCPI.*}}, %ymm0, %ymm0
 ; AVX2-32-NEXT:    retl
 ;
 ; AVX2-64-LABEL: test_mul_v8i32_v8i8:
 ; AVX2-64:       # %bb.0:
-; AVX2-64-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX2-64-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2-64-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
 ; AVX2-64-NEXT:    vpmaddwd {{.*}}(%rip), %ymm0, %ymm0
 ; AVX2-64-NEXT:    retq
 ;
 ; AVX512DQ-32-LABEL: test_mul_v8i32_v8i8:
 ; AVX512DQ-32:       # %bb.0:
-; AVX512DQ-32-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
-; AVX512DQ-32-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512DQ-32-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
 ; AVX512DQ-32-NEXT:    vpmaddwd {{\.LCPI.*}}, %ymm0, %ymm0
 ; AVX512DQ-32-NEXT:    retl
 ;
 ; AVX512DQ-64-LABEL: test_mul_v8i32_v8i8:
 ; AVX512DQ-64:       # %bb.0:
-; AVX512DQ-64-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX512DQ-64-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512DQ-64-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
 ; AVX512DQ-64-NEXT:    vpmaddwd {{.*}}(%rip), %ymm0, %ymm0
 ; AVX512DQ-64-NEXT:    retq
 ;
 ; AVX512BW-32-LABEL: test_mul_v8i32_v8i8:
 ; AVX512BW-32:       # %bb.0:
-; AVX512BW-32-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
-; AVX512BW-32-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512BW-32-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
 ; AVX512BW-32-NEXT:    vpmaddwd {{\.LCPI.*}}, %ymm0, %ymm0
 ; AVX512BW-32-NEXT:    retl
 ;
 ; AVX512BW-64-LABEL: test_mul_v8i32_v8i8:
 ; AVX512BW-64:       # %bb.0:
-; AVX512BW-64-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX512BW-64-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512BW-64-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
 ; AVX512BW-64-NEXT:    vpmaddwd {{.*}}(%rip), %ymm0, %ymm0
 ; AVX512BW-64-NEXT:    retq
 ;
 ; KNL-32-LABEL: test_mul_v8i32_v8i8:
 ; KNL-32:       # %bb.0:
-; KNL-32-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
-; KNL-32-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; KNL-32-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
 ; KNL-32-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [18778,18778,18778,18778,18778,18778,18778,18778]
 ; KNL-32-NEXT:    vpmulld %ymm1, %ymm0, %ymm0
 ; KNL-32-NEXT:    retl
 ;
 ; KNL-64-LABEL: test_mul_v8i32_v8i8:
 ; KNL-64:       # %bb.0:
-; KNL-64-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; KNL-64-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; KNL-64-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
 ; KNL-64-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [18778,18778,18778,18778,18778,18778,18778,18778]
 ; KNL-64-NEXT:    vpmulld %ymm1, %ymm0, %ymm0
 ; KNL-64-NEXT:    retq
@@ -409,72 +395,46 @@ define <16 x i32> @test_mul_v16i32_v16i8(<16 x i8> %A) {
 }
 
 define <4 x i32> @test_mul_v4i32_v4i16(<4 x i16> %A) {
-; SLM32-LABEL: test_mul_v4i32_v4i16:
-; SLM32:       # %bb.0:
-; SLM32-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
-; SLM32-NEXT:    movdqa {{.*#+}} xmm1 = <18778,18778,18778,18778,u,u,u,u>
-; SLM32-NEXT:    movdqa %xmm0, %xmm2
-; SLM32-NEXT:    pmullw %xmm1, %xmm0
-; SLM32-NEXT:    pmulhuw %xmm1, %xmm2
-; SLM32-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
-; SLM32-NEXT:    retl
-;
-; SLM64-LABEL: test_mul_v4i32_v4i16:
-; SLM64:       # %bb.0:
-; SLM64-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
-; SLM64-NEXT:    movdqa {{.*#+}} xmm1 = <18778,18778,18778,18778,u,u,u,u>
-; SLM64-NEXT:    movdqa %xmm0, %xmm2
-; SLM64-NEXT:    pmullw %xmm1, %xmm0
-; SLM64-NEXT:    pmulhuw %xmm1, %xmm2
-; SLM64-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
-; SLM64-NEXT:    retq
-;
-; SLOW32-LABEL: test_mul_v4i32_v4i16:
-; SLOW32:       # %bb.0:
-; SLOW32-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
-; SLOW32-NEXT:    movdqa {{.*#+}} xmm1 = <18778,18778,18778,18778,u,u,u,u>
-; SLOW32-NEXT:    movdqa %xmm0, %xmm2
-; SLOW32-NEXT:    pmulhuw %xmm1, %xmm2
-; SLOW32-NEXT:    pmullw %xmm1, %xmm0
-; SLOW32-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
-; SLOW32-NEXT:    retl
+; CHECK32-LABEL: test_mul_v4i32_v4i16:
+; CHECK32:       # %bb.0:
+; CHECK32-NEXT:    movdqa {{.*#+}} xmm1 = <18778,18778,18778,18778,u,u,u,u>
+; CHECK32-NEXT:    movdqa %xmm0, %xmm2
+; CHECK32-NEXT:    pmulhuw %xmm1, %xmm2
+; CHECK32-NEXT:    pmullw %xmm1, %xmm0
+; CHECK32-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; CHECK32-NEXT:    retl
 ;
-; SLOW64-LABEL: test_mul_v4i32_v4i16:
-; SLOW64:       # %bb.0:
-; SLOW64-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
-; SLOW64-NEXT:    movdqa {{.*#+}} xmm1 = <18778,18778,18778,18778,u,u,u,u>
-; SLOW64-NEXT:    movdqa %xmm0, %xmm2
-; SLOW64-NEXT:    pmulhuw %xmm1, %xmm2
-; SLOW64-NEXT:    pmullw %xmm1, %xmm0
-; SLOW64-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
-; SLOW64-NEXT:    retq
+; CHECK64-LABEL: test_mul_v4i32_v4i16:
+; CHECK64:       # %bb.0:
+; CHECK64-NEXT:    movdqa {{.*#+}} xmm1 = <18778,18778,18778,18778,u,u,u,u>
+; CHECK64-NEXT:    movdqa %xmm0, %xmm2
+; CHECK64-NEXT:    pmulhuw %xmm1, %xmm2
+; CHECK64-NEXT:    pmullw %xmm1, %xmm0
+; CHECK64-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; CHECK64-NEXT:    retq
 ;
 ; SSE4-32-LABEL: test_mul_v4i32_v4i16:
 ; SSE4-32:       # %bb.0:
-; SSE4-32-NEXT:    pxor %xmm1, %xmm1
-; SSE4-32-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7]
+; SSE4-32-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; SSE4-32-NEXT:    pmulld {{\.LCPI.*}}, %xmm0
 ; SSE4-32-NEXT:    retl
 ;
 ; SSE4-64-LABEL: test_mul_v4i32_v4i16:
 ; SSE4-64:       # %bb.0:
-; SSE4-64-NEXT:    pxor %xmm1, %xmm1
-; SSE4-64-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7]
+; SSE4-64-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; SSE4-64-NEXT:    pmulld {{.*}}(%rip), %xmm0
 ; SSE4-64-NEXT:    retq
 ;
 ; AVX-32-LABEL: test_mul_v4i32_v4i16:
 ; AVX-32:       # %bb.0:
-; AVX-32-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX-32-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7]
+; AVX-32-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; AVX-32-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [18778,18778,18778,18778]
 ; AVX-32-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX-32-NEXT:    retl
 ;
 ; AVX-64-LABEL: test_mul_v4i32_v4i16:
 ; AVX-64:       # %bb.0:
-; AVX-64-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX-64-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7]
+; AVX-64-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; AVX-64-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [18778,18778,18778,18778]
 ; AVX-64-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX-64-NEXT:    retq
@@ -722,74 +682,74 @@ define <16 x i32> @test_mul_v16i32_v16i16(<16 x i16> %A) {
 define <4 x i32> @test_mul_v4i32_v4i8_minsize(<4 x i8> %A) minsize {
 ; CHECK32-LABEL: test_mul_v4i32_v4i8_minsize:
 ; CHECK32:       # %bb.0:
-; CHECK32-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; CHECK32-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; CHECK32-NEXT:    pmaddwd {{\.LCPI.*}}, %xmm0
 ; CHECK32-NEXT:    retl
 ;
 ; CHECK64-LABEL: test_mul_v4i32_v4i8_minsize:
 ; CHECK64:       # %bb.0:
-; CHECK64-NEXT:    pand {{.*}}(%rip), %xmm0
+; CHECK64-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; CHECK64-NEXT:    pmaddwd {{.*}}(%rip), %xmm0
 ; CHECK64-NEXT:    retq
 ;
 ; SSE4-32-LABEL: test_mul_v4i32_v4i8_minsize:
 ; SSE4-32:       # %bb.0:
-; SSE4-32-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; SSE4-32-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SSE4-32-NEXT:    pmaddwd {{\.LCPI.*}}, %xmm0
 ; SSE4-32-NEXT:    retl
 ;
 ; SSE4-64-LABEL: test_mul_v4i32_v4i8_minsize:
 ; SSE4-64:       # %bb.0:
-; SSE4-64-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE4-64-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SSE4-64-NEXT:    pmaddwd {{.*}}(%rip), %xmm0
 ; SSE4-64-NEXT:    retq
 ;
 ; AVX2-32-LABEL: test_mul_v4i32_v4i8_minsize:
 ; AVX2-32:       # %bb.0:
-; AVX2-32-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; AVX2-32-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; AVX2-32-NEXT:    vpmaddwd {{\.LCPI.*}}, %xmm0, %xmm0
 ; AVX2-32-NEXT:    retl
 ;
 ; AVX2-64-LABEL: test_mul_v4i32_v4i8_minsize:
 ; AVX2-64:       # %bb.0:
-; AVX2-64-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX2-64-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; AVX2-64-NEXT:    vpmaddwd {{.*}}(%rip), %xmm0, %xmm0
 ; AVX2-64-NEXT:    retq
 ;
 ; AVX512DQ-32-LABEL: test_mul_v4i32_v4i8_minsize:
 ; AVX512DQ-32:       # %bb.0:
-; AVX512DQ-32-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; AVX512DQ-32-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; AVX512DQ-32-NEXT:    vpmaddwd {{\.LCPI.*}}, %xmm0, %xmm0
 ; AVX512DQ-32-NEXT:    retl
 ;
 ; AVX512DQ-64-LABEL: test_mul_v4i32_v4i8_minsize:
 ; AVX512DQ-64:       # %bb.0:
-; AVX512DQ-64-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX512DQ-64-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; AVX512DQ-64-NEXT:    vpmaddwd {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512DQ-64-NEXT:    retq
 ;
 ; AVX512BW-32-LABEL: test_mul_v4i32_v4i8_minsize:
 ; AVX512BW-32:       # %bb.0:
-; AVX512BW-32-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; AVX512BW-32-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; AVX512BW-32-NEXT:    vpmaddwd {{\.LCPI.*}}, %xmm0, %xmm0
 ; AVX512BW-32-NEXT:    retl
 ;
 ; AVX512BW-64-LABEL: test_mul_v4i32_v4i8_minsize:
 ; AVX512BW-64:       # %bb.0:
-; AVX512BW-64-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX512BW-64-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; AVX512BW-64-NEXT:    vpmaddwd {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512BW-64-NEXT:    retq
 ;
 ; KNL-32-LABEL: test_mul_v4i32_v4i8_minsize:
 ; KNL-32:       # %bb.0:
-; KNL-32-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; KNL-32-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; KNL-32-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [18778,18778,18778,18778]
 ; KNL-32-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; KNL-32-NEXT:    retl
 ;
 ; KNL-64-LABEL: test_mul_v4i32_v4i8_minsize:
 ; KNL-64:       # %bb.0:
-; KNL-64-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; KNL-64-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; KNL-64-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [18778,18778,18778,18778]
 ; KNL-64-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; KNL-64-NEXT:    retq
@@ -801,32 +761,29 @@ define <4 x i32> @test_mul_v4i32_v4i8_minsize(<4 x i8> %A) minsize {
 define <8 x i32> @test_mul_v8i32_v8i8_minsize(<8 x i8> %A) minsize {
 ; SLM32-LABEL: test_mul_v8i32_v8i8_minsize:
 ; SLM32:       # %bb.0:
-; SLM32-NEXT:    pand {{\.LCPI.*}}, %xmm0
 ; SLM32-NEXT:    movdqa {{.*#+}} xmm2 = [18778,18778,18778,18778]
-; SLM32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; SLM32-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; SLM32-NEXT:    pmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; SLM32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SLM32-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; SLM32-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; SLM32-NEXT:    pmaddwd %xmm2, %xmm0
 ; SLM32-NEXT:    pmaddwd %xmm2, %xmm1
 ; SLM32-NEXT:    retl
 ;
 ; SLM64-LABEL: test_mul_v8i32_v8i8_minsize:
 ; SLM64:       # %bb.0:
-; SLM64-NEXT:    pand {{.*}}(%rip), %xmm0
 ; SLM64-NEXT:    movdqa {{.*#+}} xmm2 = [18778,18778,18778,18778]
-; SLM64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; SLM64-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; SLM64-NEXT:    pmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; SLM64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SLM64-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; SLM64-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; SLM64-NEXT:    pmaddwd %xmm2, %xmm0
 ; SLM64-NEXT:    pmaddwd %xmm2, %xmm1
 ; SLM64-NEXT:    retq
 ;
 ; SLOW32-LABEL: test_mul_v8i32_v8i8_minsize:
 ; SLOW32:       # %bb.0:
-; SLOW32-NEXT:    pand {{\.LCPI.*}}, %xmm0
-; SLOW32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; SLOW32-NEXT:    pmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
-; SLOW32-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; SLOW32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SLOW32-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
+; SLOW32-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SLOW32-NEXT:    movdqa {{.*#+}} xmm2 = [18778,18778,18778,18778]
 ; SLOW32-NEXT:    pmaddwd %xmm2, %xmm0
 ; SLOW32-NEXT:    pmaddwd %xmm2, %xmm1
@@ -834,10 +791,9 @@ define <8 x i32> @test_mul_v8i32_v8i8_minsize(<8 x i8> %A) minsize {
 ;
 ; SLOW64-LABEL: test_mul_v8i32_v8i8_minsize:
 ; SLOW64:       # %bb.0:
-; SLOW64-NEXT:    pand {{.*}}(%rip), %xmm0
-; SLOW64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; SLOW64-NEXT:    pmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
-; SLOW64-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; SLOW64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SLOW64-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
+; SLOW64-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SLOW64-NEXT:    movdqa {{.*#+}} xmm2 = [18778,18778,18778,18778]
 ; SLOW64-NEXT:    pmaddwd %xmm2, %xmm0
 ; SLOW64-NEXT:    pmaddwd %xmm2, %xmm1
@@ -845,10 +801,9 @@ define <8 x i32> @test_mul_v8i32_v8i8_minsize(<8 x i8> %A) minsize {
 ;
 ; SSE4-32-LABEL: test_mul_v8i32_v8i8_minsize:
 ; SSE4-32:       # %bb.0:
-; SSE4-32-NEXT:    pand {{\.LCPI.*}}, %xmm0
-; SSE4-32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; SSE4-32-NEXT:    pmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
-; SSE4-32-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; SSE4-32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE4-32-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
+; SSE4-32-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SSE4-32-NEXT:    movdqa {{.*#+}} xmm2 = [18778,18778,18778,18778]
 ; SSE4-32-NEXT:    pmaddwd %xmm2, %xmm0
 ; SSE4-32-NEXT:    pmaddwd %xmm2, %xmm1
@@ -856,10 +811,9 @@ define <8 x i32> @test_mul_v8i32_v8i8_minsize(<8 x i8> %A) minsize {
 ;
 ; SSE4-64-LABEL: test_mul_v8i32_v8i8_minsize:
 ; SSE4-64:       # %bb.0:
-; SSE4-64-NEXT:    pand {{.*}}(%rip), %xmm0
-; SSE4-64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; SSE4-64-NEXT:    pmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
-; SSE4-64-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; SSE4-64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE4-64-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
+; SSE4-64-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SSE4-64-NEXT:    movdqa {{.*#+}} xmm2 = [18778,18778,18778,18778]
 ; SSE4-64-NEXT:    pmaddwd %xmm2, %xmm0
 ; SSE4-64-NEXT:    pmaddwd %xmm2, %xmm1
@@ -867,58 +821,50 @@ define <8 x i32> @test_mul_v8i32_v8i8_minsize(<8 x i8> %A) minsize {
 ;
 ; AVX2-32-LABEL: test_mul_v8i32_v8i8_minsize:
 ; AVX2-32:       # %bb.0:
-; AVX2-32-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
-; AVX2-32-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2-32-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
 ; AVX2-32-NEXT:    vpmaddwd {{\.LCPI.*}}, %ymm0, %ymm0
 ; AVX2-32-NEXT:    retl
 ;
 ; AVX2-64-LABEL: test_mul_v8i32_v8i8_minsize:
 ; AVX2-64:       # %bb.0:
-; AVX2-64-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX2-64-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2-64-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
 ; AVX2-64-NEXT:    vpmaddwd {{.*}}(%rip), %ymm0, %ymm0
 ; AVX2-64-NEXT:    retq
 ;
 ; AVX512DQ-32-LABEL: test_mul_v8i32_v8i8_minsize:
 ; AVX512DQ-32:       # %bb.0:
-; AVX512DQ-32-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
-; AVX512DQ-32-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512DQ-32-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
 ; AVX512DQ-32-NEXT:    vpmaddwd {{\.LCPI.*}}, %ymm0, %ymm0
 ; AVX512DQ-32-NEXT:    retl
 ;
 ; AVX512DQ-64-LABEL: test_mul_v8i32_v8i8_minsize:
 ; AVX512DQ-64:       # %bb.0:
-; AVX512DQ-64-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX512DQ-64-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512DQ-64-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
 ; AVX512DQ-64-NEXT:    vpmaddwd {{.*}}(%rip), %ymm0, %ymm0
 ; AVX512DQ-64-NEXT:    retq
 ;
 ; AVX512BW-32-LABEL: test_mul_v8i32_v8i8_minsize:
 ; AVX512BW-32:       # %bb.0:
-; AVX512BW-32-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
-; AVX512BW-32-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512BW-32-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
 ; AVX512BW-32-NEXT:    vpmaddwd {{\.LCPI.*}}, %ymm0, %ymm0
 ; AVX512BW-32-NEXT:    retl
 ;
 ; AVX512BW-64-LABEL: test_mul_v8i32_v8i8_minsize:
 ; AVX512BW-64:       # %bb.0:
-; AVX512BW-64-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX512BW-64-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512BW-64-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
 ; AVX512BW-64-NEXT:    vpmaddwd {{.*}}(%rip), %ymm0, %ymm0
 ; AVX512BW-64-NEXT:    retq
 ;
 ; KNL-32-LABEL: test_mul_v8i32_v8i8_minsize:
 ; KNL-32:       # %bb.0:
-; KNL-32-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
-; KNL-32-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; KNL-32-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
 ; KNL-32-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [18778,18778,18778,18778,18778,18778,18778,18778]
 ; KNL-32-NEXT:    vpmulld %ymm1, %ymm0, %ymm0
 ; KNL-32-NEXT:    retl
 ;
 ; KNL-64-LABEL: test_mul_v8i32_v8i8_minsize:
 ; KNL-64:       # %bb.0:
-; KNL-64-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; KNL-64-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; KNL-64-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
 ; KNL-64-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [18778,18778,18778,18778,18778,18778,18778,18778]
 ; KNL-64-NEXT:    vpmulld %ymm1, %ymm0, %ymm0
 ; KNL-64-NEXT:    retq
@@ -1087,44 +1033,38 @@ define <16 x i32> @test_mul_v16i32_v16i8_minsize(<16 x i8> %A) minsize {
 define <4 x i32> @test_mul_v4i32_v4i16_minsize(<4 x i16> %A) minsize {
 ; CHECK32-LABEL: test_mul_v4i32_v4i16_minsize:
 ; CHECK32:       # %bb.0:
-; CHECK32-NEXT:    pxor %xmm1, %xmm1
-; CHECK32-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7]
+; CHECK32-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; CHECK32-NEXT:    pmulld {{\.LCPI.*}}, %xmm0
 ; CHECK32-NEXT:    retl
 ;
 ; CHECK64-LABEL: test_mul_v4i32_v4i16_minsize:
 ; CHECK64:       # %bb.0:
-; CHECK64-NEXT:    pxor %xmm1, %xmm1
-; CHECK64-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7]
+; CHECK64-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; CHECK64-NEXT:    pmulld {{.*}}(%rip), %xmm0
 ; CHECK64-NEXT:    retq
 ;
 ; SSE4-32-LABEL: test_mul_v4i32_v4i16_minsize:
 ; SSE4-32:       # %bb.0:
-; SSE4-32-NEXT:    pxor %xmm1, %xmm1
-; SSE4-32-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7]
+; SSE4-32-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; SSE4-32-NEXT:    pmulld {{\.LCPI.*}}, %xmm0
 ; SSE4-32-NEXT:    retl
 ;
 ; SSE4-64-LABEL: test_mul_v4i32_v4i16_minsize:
 ; SSE4-64:       # %bb.0:
-; SSE4-64-NEXT:    pxor %xmm1, %xmm1
-; SSE4-64-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7]
+; SSE4-64-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; SSE4-64-NEXT:    pmulld {{.*}}(%rip), %xmm0
 ; SSE4-64-NEXT:    retq
 ;
 ; AVX-32-LABEL: test_mul_v4i32_v4i16_minsize:
 ; AVX-32:       # %bb.0:
-; AVX-32-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX-32-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7]
+; AVX-32-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; AVX-32-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [18778,18778,18778,18778]
 ; AVX-32-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX-32-NEXT:    retl
 ;
 ; AVX-64-LABEL: test_mul_v4i32_v4i16_minsize:
 ; AVX-64:       # %bb.0:
-; AVX-64-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX-64-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7]
+; AVX-64-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; AVX-64-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [18778,18778,18778,18778]
 ; AVX-64-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX-64-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/smul_fix.ll b/llvm/test/CodeGen/X86/smul_fix.ll
index 30db0d3b4..c34730b11 100644
--- a/llvm/test/CodeGen/X86/smul_fix.ll
+++ b/llvm/test/CodeGen/X86/smul_fix.ll
@@ -25,8 +25,8 @@ define i32 @func(i32 %x, i32 %y) nounwind {
 ; X86-NEXT:    imull {{[0-9]+}}(%esp)
 ; X86-NEXT:    shrdl $2, %edx, %eax
 ; X86-NEXT:    retl
-  %tmp = call i32 @llvm.smul.fix.i32(i32 %x, i32 %y, i32 2);
-  ret i32 %tmp;
+  %tmp = call i32 @llvm.smul.fix.i32(i32 %x, i32 %y, i32 2)
+  ret i32 %tmp
 }
 
 define i64 @func2(i64 %x, i64 %y) {
@@ -90,8 +90,8 @@ define i64 @func2(i64 %x, i64 %y) {
 ; X86-NEXT:    popl %ebp
 ; X86-NEXT:    .cfi_def_cfa_offset 4
 ; X86-NEXT:    retl
-  %tmp = call i64 @llvm.smul.fix.i64(i64 %x, i64 %y, i32 2);
-  ret i64 %tmp;
+  %tmp = call i64 @llvm.smul.fix.i64(i64 %x, i64 %y, i32 2)
+  ret i64 %tmp
 }
 
 define i4 @func3(i4 %x, i4 %y) nounwind {
@@ -128,8 +128,8 @@ define i4 @func3(i4 %x, i4 %y) nounwind {
 ; X86-NEXT:    orb %ah, %al
 ; X86-NEXT:    # kill: def $al killed $al killed $eax
 ; X86-NEXT:    retl
-  %tmp = call i4 @llvm.smul.fix.i4(i4 %x, i4 %y, i32 2);
-  ret i4 %tmp;
+  %tmp = call i4 @llvm.smul.fix.i4(i4 %x, i4 %y, i32 2)
+  ret i4 %tmp
 }
 
 define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
@@ -193,8 +193,8 @@ define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; X86-NEXT:    popl %ebx
 ; X86-NEXT:    popl %ebp
 ; X86-NEXT:    retl $4
-  %tmp = call <4 x i32> @llvm.smul.fix.v4i32(<4 x i32> %x, <4 x i32> %y, i32 2);
-  ret <4 x i32> %tmp;
+  %tmp = call <4 x i32> @llvm.smul.fix.v4i32(<4 x i32> %x, <4 x i32> %y, i32 2)
+  ret <4 x i32> %tmp
 }
 
 ; These result in regular integer multiplication
@@ -210,8 +210,8 @@ define i32 @func4(i32 %x, i32 %y) nounwind {
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    imull {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    retl
-  %tmp = call i32 @llvm.smul.fix.i32(i32 %x, i32 %y, i32 0);
-  ret i32 %tmp;
+  %tmp = call i32 @llvm.smul.fix.i32(i32 %x, i32 %y, i32 0)
+  ret i32 %tmp
 }
 
 define i64 @func5(i64 %x, i64 %y) {
@@ -237,8 +237,8 @@ define i64 @func5(i64 %x, i64 %y) {
 ; X86-NEXT:    popl %esi
 ; X86-NEXT:    .cfi_def_cfa_offset 4
 ; X86-NEXT:    retl
-  %tmp = call i64 @llvm.smul.fix.i64(i64 %x, i64 %y, i32 0);
-  ret i64 %tmp;
+  %tmp = call i64 @llvm.smul.fix.i64(i64 %x, i64 %y, i32 0)
+  ret i64 %tmp
 }
 
 define i4 @func6(i4 %x, i4 %y) nounwind {
@@ -263,8 +263,8 @@ define i4 @func6(i4 %x, i4 %y) nounwind {
 ; X86-NEXT:    sarb $4, %cl
 ; X86-NEXT:    mulb %cl
 ; X86-NEXT:    retl
-  %tmp = call i4 @llvm.smul.fix.i4(i4 %x, i4 %y, i32 0);
-  ret i4 %tmp;
+  %tmp = call i4 @llvm.smul.fix.i4(i4 %x, i4 %y, i32 0)
+  ret i4 %tmp
 }
 
 define <4 x i32> @vec2(<4 x i32> %x, <4 x i32> %y) nounwind {
@@ -299,8 +299,8 @@ define <4 x i32> @vec2(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; X86-NEXT:    popl %esi
 ; X86-NEXT:    popl %edi
 ; X86-NEXT:    retl $4
-  %tmp = call <4 x i32> @llvm.smul.fix.v4i32(<4 x i32> %x, <4 x i32> %y, i32 0);
-  ret <4 x i32> %tmp;
+  %tmp = call <4 x i32> @llvm.smul.fix.v4i32(<4 x i32> %x, <4 x i32> %y, i32 0)
+  ret <4 x i32> %tmp
 }
 
 define i64 @func7(i64 %x, i64 %y) nounwind {
@@ -348,8 +348,8 @@ define i64 @func7(i64 %x, i64 %y) nounwind {
 ; X86-NEXT:    popl %ebx
 ; X86-NEXT:    popl %ebp
 ; X86-NEXT:    retl
-  %tmp = call i64 @llvm.smul.fix.i64(i64 %x, i64 %y, i32 32);
-  ret i64 %tmp;
+  %tmp = call i64 @llvm.smul.fix.i64(i64 %x, i64 %y, i32 32)
+  ret i64 %tmp
 }
 
 define i64 @func8(i64 %x, i64 %y) nounwind {
@@ -374,26 +374,25 @@ define i64 @func8(i64 %x, i64 %y) nounwind {
 ; X86-NEXT:    movl %eax, %ebx
 ; X86-NEXT:    movl %ecx, %eax
 ; X86-NEXT:    mull {{[0-9]+}}(%esp)
-; X86-NEXT:    movl %edx, %ebp
-; X86-NEXT:    addl %ebx, %ebp
+; X86-NEXT:    addl %edx, %ebx
 ; X86-NEXT:    adcl $0, %edi
 ; X86-NEXT:    movl %esi, %eax
 ; X86-NEXT:    imull {{[0-9]+}}(%esp)
-; X86-NEXT:    movl %edx, %ebx
+; X86-NEXT:    movl %edx, %ebp
 ; X86-NEXT:    movl %eax, %ecx
 ; X86-NEXT:    movl %esi, %eax
 ; X86-NEXT:    mull {{[0-9]+}}(%esp)
-; X86-NEXT:    addl %ebp, %eax
+; X86-NEXT:    addl %ebx, %eax
 ; X86-NEXT:    adcl %edi, %edx
-; X86-NEXT:    adcl $0, %ebx
+; X86-NEXT:    adcl $0, %ebp
 ; X86-NEXT:    addl %ecx, %edx
-; X86-NEXT:    adcl $0, %ebx
+; X86-NEXT:    adcl $0, %ebp
 ; X86-NEXT:    movl %edx, %ecx
 ; X86-NEXT:    subl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl %ebx, %esi
+; X86-NEXT:    movl %ebp, %esi
 ; X86-NEXT:    sbbl $0, %esi
 ; X86-NEXT:    cmpl $0, {{[0-9]+}}(%esp)
-; X86-NEXT:    cmovnsl %ebx, %esi
+; X86-NEXT:    cmovnsl %ebp, %esi
 ; X86-NEXT:    cmovnsl %edx, %ecx
 ; X86-NEXT:    movl %ecx, %edi
 ; X86-NEXT:    subl {{[0-9]+}}(%esp), %edi
@@ -409,6 +408,6 @@ define i64 @func8(i64 %x, i64 %y) nounwind {
 ; X86-NEXT:    popl %ebx
 ; X86-NEXT:    popl %ebp
 ; X86-NEXT:    retl
-  %tmp = call i64 @llvm.smul.fix.i64(i64 %x, i64 %y, i32 63);
-  ret i64 %tmp;
+  %tmp = call i64 @llvm.smul.fix.i64(i64 %x, i64 %y, i32 63)
+  ret i64 %tmp
 }
diff --git a/llvm/test/CodeGen/X86/smul_fix_sat.ll b/llvm/test/CodeGen/X86/smul_fix_sat.ll
index 09dc11a8c..284f51d74 100644
--- a/llvm/test/CodeGen/X86/smul_fix_sat.ll
+++ b/llvm/test/CodeGen/X86/smul_fix_sat.ll
@@ -36,8 +36,8 @@ define i32 @func(i32 %x, i32 %y) nounwind {
 ; X86-NEXT:    movl $-2147483648, %ecx # imm = 0x80000000
 ; X86-NEXT:    cmovll %ecx, %eax
 ; X86-NEXT:    retl
-  %tmp = call i32 @llvm.smul.fix.sat.i32(i32 %x, i32 %y, i32 2);
-  ret i32 %tmp;
+  %tmp = call i32 @llvm.smul.fix.sat.i32(i32 %x, i32 %y, i32 2)
+  ret i32 %tmp
 }
 
 define i64 @func2(i64 %x, i64 %y) nounwind {
@@ -60,7 +60,6 @@ define i64 @func2(i64 %x, i64 %y) nounwind {
 ; X86-NEXT:    pushl %ebx
 ; X86-NEXT:    pushl %edi
 ; X86-NEXT:    pushl %esi
-; X86-NEXT:    subl $8, %esp
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-NEXT:    movl %ecx, %eax
@@ -69,71 +68,68 @@ define i64 @func2(i64 %x, i64 %y) nounwind {
 ; X86-NEXT:    movl %eax, %ebx
 ; X86-NEXT:    movl %ecx, %eax
 ; X86-NEXT:    mull {{[0-9]+}}(%esp)
-; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl %eax, %ecx
 ; X86-NEXT:    movl %edx, %ebp
 ; X86-NEXT:    addl %ebx, %ebp
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    adcl $0, %edi
-; X86-NEXT:    movl %ebx, %eax
 ; X86-NEXT:    imull %esi
-; X86-NEXT:    movl %edx, %ecx
-; X86-NEXT:    movl %eax, %esi
-; X86-NEXT:    movl %ebx, %eax
+; X86-NEXT:    movl %edx, %esi
+; X86-NEXT:    movl %eax, %ebx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    mull {{[0-9]+}}(%esp)
 ; X86-NEXT:    addl %ebp, %eax
 ; X86-NEXT:    adcl %edi, %edx
-; X86-NEXT:    adcl $0, %ecx
-; X86-NEXT:    addl %esi, %edx
-; X86-NEXT:    adcl $0, %ecx
-; X86-NEXT:    movl %edx, %esi
-; X86-NEXT:    subl {{[0-9]+}}(%esp), %esi
-; X86-NEXT:    movl %ecx, %edi
-; X86-NEXT:    sbbl $0, %edi
-; X86-NEXT:    testl %ebx, %ebx
-; X86-NEXT:    cmovnsl %ecx, %edi
-; X86-NEXT:    cmovnsl %edx, %esi
-; X86-NEXT:    movl %esi, %ecx
-; X86-NEXT:    subl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    adcl $0, %esi
+; X86-NEXT:    addl %ebx, %edx
+; X86-NEXT:    adcl $0, %esi
+; X86-NEXT:    movl %edx, %edi
+; X86-NEXT:    subl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl %esi, %ebx
+; X86-NEXT:    sbbl $0, %ebx
+; X86-NEXT:    cmpl $0, {{[0-9]+}}(%esp)
+; X86-NEXT:    cmovnsl %esi, %ebx
+; X86-NEXT:    cmovnsl %edx, %edi
 ; X86-NEXT:    movl %edi, %ebp
-; X86-NEXT:    sbbl $0, %ebp
+; X86-NEXT:    subl {{[0-9]+}}(%esp), %ebp
+; X86-NEXT:    movl %ebx, %esi
+; X86-NEXT:    sbbl $0, %esi
 ; X86-NEXT:    cmpl $0, {{[0-9]+}}(%esp)
+; X86-NEXT:    cmovnsl %ebx, %esi
 ; X86-NEXT:    cmovnsl %edi, %ebp
-; X86-NEXT:    cmovnsl %esi, %ecx
-; X86-NEXT:    testl %ebp, %ebp
-; X86-NEXT:    setg %bh
-; X86-NEXT:    sete {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X86-NEXT:    cmpl $1, %ecx
-; X86-NEXT:    seta %bl
-; X86-NEXT:    movl %ecx, %edx
-; X86-NEXT:    shldl $30, %eax, %edx
-; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT:    shldl $30, %esi, %eax
-; X86-NEXT:    andb {{[-0-9]+}}(%e{{[sb]}}p), %bl # 1-byte Folded Reload
-; X86-NEXT:    orb %bh, %bl
-; X86-NEXT:    testb %bl, %bl
-; X86-NEXT:    movl $2147483647, %esi # imm = 0x7FFFFFFF
-; X86-NEXT:    cmovnel %esi, %edx
-; X86-NEXT:    movl $-1, %esi
-; X86-NEXT:    cmovnel %esi, %eax
-; X86-NEXT:    cmpl $-1, %ebp
-; X86-NEXT:    setl %bl
+; X86-NEXT:    testl %esi, %esi
+; X86-NEXT:    setg %bl
 ; X86-NEXT:    sete %bh
-; X86-NEXT:    cmpl $-2, %ecx
-; X86-NEXT:    setb %cl
-; X86-NEXT:    andb %bh, %cl
-; X86-NEXT:    xorl %esi, %esi
-; X86-NEXT:    orb %bl, %cl
-; X86-NEXT:    cmovnel %esi, %eax
-; X86-NEXT:    movl $-2147483648, %ecx # imm = 0x80000000
-; X86-NEXT:    cmovnel %ecx, %edx
-; X86-NEXT:    addl $8, %esp
+; X86-NEXT:    cmpl $1, %ebp
+; X86-NEXT:    seta %dl
+; X86-NEXT:    andb %bh, %dl
+; X86-NEXT:    orb %bl, %dl
+; X86-NEXT:    shrdl $2, %eax, %ecx
+; X86-NEXT:    shrdl $2, %ebp, %eax
+; X86-NEXT:    testb %dl, %dl
+; X86-NEXT:    movl $2147483647, %edi # imm = 0x7FFFFFFF
+; X86-NEXT:    cmovel %eax, %edi
+; X86-NEXT:    movl $-1, %eax
+; X86-NEXT:    cmovnel %eax, %ecx
+; X86-NEXT:    cmpl $-1, %esi
+; X86-NEXT:    setl %al
+; X86-NEXT:    sete %dl
+; X86-NEXT:    cmpl $-2, %ebp
+; X86-NEXT:    setb %ah
+; X86-NEXT:    andb %dl, %ah
+; X86-NEXT:    xorl %edx, %edx
+; X86-NEXT:    orb %al, %ah
+; X86-NEXT:    cmovnel %edx, %ecx
+; X86-NEXT:    movl $-2147483648, %edx # imm = 0x80000000
+; X86-NEXT:    cmovel %edi, %edx
+; X86-NEXT:    movl %ecx, %eax
 ; X86-NEXT:    popl %esi
 ; X86-NEXT:    popl %edi
 ; X86-NEXT:    popl %ebx
 ; X86-NEXT:    popl %ebp
 ; X86-NEXT:    retl
-  %tmp = call i64 @llvm.smul.fix.sat.i64(i64 %x, i64 %y, i32 2);
-  ret i64 %tmp;
+  %tmp = call i64 @llvm.smul.fix.sat.i64(i64 %x, i64 %y, i32 2)
+  ret i64 %tmp
 }
 
 define i4 @func3(i4 %x, i4 %y) nounwind {
@@ -186,8 +182,8 @@ define i4 @func3(i4 %x, i4 %y) nounwind {
 ; X86-NEXT:    sarb $4, %al
 ; X86-NEXT:    # kill: def $al killed $al killed $eax
 ; X86-NEXT:    retl
-  %tmp = call i4 @llvm.smul.fix.sat.i4(i4 %x, i4 %y, i32 2);
-  ret i4 %tmp;
+  %tmp = call i4 @llvm.smul.fix.sat.i4(i4 %x, i4 %y, i32 2)
+  ret i4 %tmp
 }
 
 define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
@@ -311,8 +307,8 @@ define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; X86-NEXT:    popl %ebx
 ; X86-NEXT:    popl %ebp
 ; X86-NEXT:    retl $4
-  %tmp = call <4 x i32> @llvm.smul.fix.sat.v4i32(<4 x i32> %x, <4 x i32> %y, i32 2);
-  ret <4 x i32> %tmp;
+  %tmp = call <4 x i32> @llvm.smul.fix.sat.v4i32(<4 x i32> %x, <4 x i32> %y, i32 2)
+  ret <4 x i32> %tmp
 }
 
 ; These result in regular integer multiplication
@@ -344,8 +340,8 @@ define i32 @func4(i32 %x, i32 %y) nounwind {
 ; X86-NEXT:    cmovol %ecx, %eax
 ; X86-NEXT:    popl %esi
 ; X86-NEXT:    retl
-  %tmp = call i32 @llvm.smul.fix.sat.i32(i32 %x, i32 %y, i32 0);
-  ret i32 %tmp;
+  %tmp = call i32 @llvm.smul.fix.sat.i32(i32 %x, i32 %y, i32 0)
+  ret i32 %tmp
 }
 
 define i64 @func5(i64 %x, i64 %y) {
@@ -407,8 +403,8 @@ define i64 @func5(i64 %x, i64 %y) {
 ; X86-NEXT:    popl %edi
 ; X86-NEXT:    .cfi_def_cfa_offset 4
 ; X86-NEXT:    retl
-  %tmp = call i64 @llvm.smul.fix.sat.i64(i64 %x, i64 %y, i32 0);
-  ret i64 %tmp;
+  %tmp = call i64 @llvm.smul.fix.sat.i64(i64 %x, i64 %y, i32 0)
+  ret i64 %tmp
 }
 
 define i4 @func6(i4 %x, i4 %y) nounwind {
@@ -451,8 +447,8 @@ define i4 @func6(i4 %x, i4 %y) nounwind {
 ; X86-NEXT:    sarb $4, %al
 ; X86-NEXT:    # kill: def $al killed $al killed $eax
 ; X86-NEXT:    retl
-  %tmp = call i4 @llvm.smul.fix.sat.i4(i4 %x, i4 %y, i32 0);
-  ret i4 %tmp;
+  %tmp = call i4 @llvm.smul.fix.sat.i4(i4 %x, i4 %y, i32 0)
+  ret i4 %tmp
 }
 
 define <4 x i32> @vec2(<4 x i32> %x, <4 x i32> %y) nounwind {
@@ -569,8 +565,8 @@ define <4 x i32> @vec2(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; X86-NEXT:    popl %ebx
 ; X86-NEXT:    popl %ebp
 ; X86-NEXT:    retl $4
-  %tmp = call <4 x i32> @llvm.smul.fix.sat.v4i32(<4 x i32> %x, <4 x i32> %y, i32 0);
-  ret <4 x i32> %tmp;
+  %tmp = call <4 x i32> @llvm.smul.fix.sat.v4i32(<4 x i32> %x, <4 x i32> %y, i32 0)
+  ret <4 x i32> %tmp
 }
 
 define i64 @func7(i64 %x, i64 %y) nounwind {
@@ -654,8 +650,8 @@ define i64 @func7(i64 %x, i64 %y) nounwind {
 ; X86-NEXT:    popl %ebx
 ; X86-NEXT:    popl %ebp
 ; X86-NEXT:    retl
-  %tmp = call i64 @llvm.smul.fix.sat.i64(i64 %x, i64 %y, i32 32);
-  ret i64 %tmp;
+  %tmp = call i64 @llvm.smul.fix.sat.i64(i64 %x, i64 %y, i32 32)
+  ret i64 %tmp
 }
 
 define i64 @func8(i64 %x, i64 %y) nounwind {
@@ -688,44 +684,42 @@ define i64 @func8(i64 %x, i64 %y) nounwind {
 ; X86-NEXT:    movl %eax, %ebx
 ; X86-NEXT:    movl %ecx, %eax
 ; X86-NEXT:    mull {{[0-9]+}}(%esp)
-; X86-NEXT:    movl %edx, %ebp
-; X86-NEXT:    addl %ebx, %ebp
+; X86-NEXT:    addl %edx, %ebx
 ; X86-NEXT:    adcl $0, %edi
 ; X86-NEXT:    movl %esi, %eax
 ; X86-NEXT:    imull {{[0-9]+}}(%esp)
-; X86-NEXT:    movl %edx, %ebx
+; X86-NEXT:    movl %edx, %ebp
 ; X86-NEXT:    movl %eax, %ecx
 ; X86-NEXT:    movl %esi, %eax
 ; X86-NEXT:    mull {{[0-9]+}}(%esp)
-; X86-NEXT:    addl %ebp, %eax
+; X86-NEXT:    addl %ebx, %eax
 ; X86-NEXT:    adcl %edi, %edx
-; X86-NEXT:    adcl $0, %ebx
+; X86-NEXT:    adcl $0, %ebp
 ; X86-NEXT:    addl %ecx, %edx
-; X86-NEXT:    adcl $0, %ebx
+; X86-NEXT:    adcl $0, %ebp
 ; X86-NEXT:    movl %edx, %ecx
 ; X86-NEXT:    subl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl %ebx, %esi
+; X86-NEXT:    movl %ebp, %esi
 ; X86-NEXT:    sbbl $0, %esi
 ; X86-NEXT:    cmpl $0, {{[0-9]+}}(%esp)
-; X86-NEXT:    cmovnsl %ebx, %esi
+; X86-NEXT:    cmovnsl %ebp, %esi
 ; X86-NEXT:    cmovnsl %edx, %ecx
-; X86-NEXT:    movl %ecx, %edi
-; X86-NEXT:    subl {{[0-9]+}}(%esp), %edi
-; X86-NEXT:    movl %esi, %ebx
-; X86-NEXT:    sbbl $0, %ebx
+; X86-NEXT:    movl %ecx, %edx
+; X86-NEXT:    subl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl %esi, %edi
+; X86-NEXT:    sbbl $0, %edi
 ; X86-NEXT:    cmpl $0, {{[0-9]+}}(%esp)
-; X86-NEXT:    cmovnsl %esi, %ebx
-; X86-NEXT:    cmovnsl %ecx, %edi
-; X86-NEXT:    movl %ebx, %edx
-; X86-NEXT:    shldl $1, %edi, %edx
-; X86-NEXT:    shrdl $31, %edi, %eax
-; X86-NEXT:    cmpl $1073741823, %ebx # imm = 0x3FFFFFFF
+; X86-NEXT:    cmovnsl %esi, %edi
+; X86-NEXT:    cmovnsl %ecx, %edx
+; X86-NEXT:    shrdl $31, %edx, %eax
+; X86-NEXT:    shrdl $31, %edi, %edx
+; X86-NEXT:    cmpl $1073741823, %edi # imm = 0x3FFFFFFF
 ; X86-NEXT:    movl $2147483647, %ecx # imm = 0x7FFFFFFF
 ; X86-NEXT:    cmovgl %ecx, %edx
 ; X86-NEXT:    movl $-1, %ecx
 ; X86-NEXT:    cmovgl %ecx, %eax
 ; X86-NEXT:    xorl %ecx, %ecx
-; X86-NEXT:    cmpl $-1073741824, %ebx # imm = 0xC0000000
+; X86-NEXT:    cmpl $-1073741824, %edi # imm = 0xC0000000
 ; X86-NEXT:    cmovll %ecx, %eax
 ; X86-NEXT:    movl $-2147483648, %ecx # imm = 0x80000000
 ; X86-NEXT:    cmovll %ecx, %edx
@@ -734,6 +728,6 @@ define i64 @func8(i64 %x, i64 %y) nounwind {
 ; X86-NEXT:    popl %ebx
 ; X86-NEXT:    popl %ebp
 ; X86-NEXT:    retl
-  %tmp = call i64 @llvm.smul.fix.sat.i64(i64 %x, i64 %y, i32 63);
-  ret i64 %tmp;
+  %tmp = call i64 @llvm.smul.fix.sat.i64(i64 %x, i64 %y, i32 63)
+  ret i64 %tmp
 }
diff --git a/llvm/test/CodeGen/X86/smul_fix_sat_constants.ll b/llvm/test/CodeGen/X86/smul_fix_sat_constants.ll
index 53c2074de..e92b8410e 100644
--- a/llvm/test/CodeGen/X86/smul_fix_sat_constants.ll
+++ b/llvm/test/CodeGen/X86/smul_fix_sat_constants.ll
@@ -23,8 +23,8 @@ define i64 @func() nounwind {
 ; X64-NEXT:    movabsq $-9223372036854775808, %rcx # imm = 0x8000000000000000
 ; X64-NEXT:    cmovlq %rcx, %rax
 ; X64-NEXT:    retq
-  %tmp = call i64 @llvm.smul.fix.sat.i64(i64 3, i64 2, i32 2);
-  ret i64 %tmp;
+  %tmp = call i64 @llvm.smul.fix.sat.i64(i64 3, i64 2, i32 2)
+  ret i64 %tmp
 }
 
 define i64 @func2() nounwind {
@@ -40,8 +40,8 @@ define i64 @func2() nounwind {
 ; X64-NEXT:    imulq $2, %rax, %rax
 ; X64-NEXT:    cmovoq %rcx, %rax
 ; X64-NEXT:    retq
-  %tmp = call i64 @llvm.smul.fix.sat.i64(i64 3, i64 2, i32 0);
-  ret i64 %tmp;
+  %tmp = call i64 @llvm.smul.fix.sat.i64(i64 3, i64 2, i32 0)
+  ret i64 %tmp
 }
 
 define i64 @func3() nounwind {
@@ -58,8 +58,8 @@ define i64 @func3() nounwind {
 ; X64-NEXT:    movabsq $-9223372036854775808, %rcx # imm = 0x8000000000000000
 ; X64-NEXT:    cmovlq %rcx, %rax
 ; X64-NEXT:    retq
-  %tmp = call i64 @llvm.smul.fix.sat.i64(i64 9223372036854775807, i64 2, i32 2);
-  ret i64 %tmp;
+  %tmp = call i64 @llvm.smul.fix.sat.i64(i64 9223372036854775807, i64 2, i32 2)
+  ret i64 %tmp
 }
 
 define i64 @func4() nounwind {
@@ -76,8 +76,8 @@ define i64 @func4() nounwind {
 ; X64-NEXT:    movabsq $-9223372036854775808, %rcx # imm = 0x8000000000000000
 ; X64-NEXT:    cmovlq %rcx, %rax
 ; X64-NEXT:    retq
-  %tmp = call i64 @llvm.smul.fix.sat.i64(i64 9223372036854775807, i64 2, i32 32);
-  ret i64 %tmp;
+  %tmp = call i64 @llvm.smul.fix.sat.i64(i64 9223372036854775807, i64 2, i32 32)
+  ret i64 %tmp
 }
 
 define i64 @func5() nounwind {
@@ -96,6 +96,6 @@ define i64 @func5() nounwind {
 ; X64-NEXT:    movabsq $-9223372036854775808, %rcx # imm = 0x8000000000000000
 ; X64-NEXT:    cmovlq %rcx, %rax
 ; X64-NEXT:    retq
-  %tmp = call i64 @llvm.smul.fix.sat.i64(i64 9223372036854775807, i64 2, i32 63);
-  ret i64 %tmp;
+  %tmp = call i64 @llvm.smul.fix.sat.i64(i64 9223372036854775807, i64 2, i32 63)
+  ret i64 %tmp
 }
diff --git a/llvm/test/CodeGen/X86/soft-fp.ll b/llvm/test/CodeGen/X86/soft-fp.ll
index 134b24c73..09093606b 100644
--- a/llvm/test/CodeGen/X86/soft-fp.ll
+++ b/llvm/test/CodeGen/X86/soft-fp.ll
@@ -53,5 +53,8 @@ entry:
 ; SOFT2-NOT:   xmm{{[0-9]+}}
 ; SSE1:        xmm{{[0-9]+}}
 ; SSE2:        xmm{{[0-9]+}}
-; CHECK:       ret{{[lq]}}
+; SOFT1:       ret{{[lq]}}
+; SOFT2:       ret{{[lq]}}
+; SSE1:       jmp __addtf3
+; SSE2:       jmp __addtf3
 }
diff --git a/llvm/test/CodeGen/X86/speculative-load-hardening.ll b/llvm/test/CodeGen/X86/speculative-load-hardening.ll
index 934581e13..c20964fcd 100644
--- a/llvm/test/CodeGen/X86/speculative-load-hardening.ll
+++ b/llvm/test/CodeGen/X86/speculative-load-hardening.ll
@@ -411,8 +411,18 @@ define void @test_basic_nested_loop(i32 %a, i32 %b, i32 %c, i32* %ptr1, i32* %pt
 ; X64-LFENCE-NEXT:    pushq %rbx
 ; X64-LFENCE-NEXT:    pushq %rax
 ; X64-LFENCE-NEXT:    testl %edi, %edi
-; X64-LFENCE-NEXT:    jne .LBB3_6
-; X64-LFENCE-NEXT:  # %bb.1: # %l1.header.preheader
+; X64-LFENCE-NEXT:    je .LBB3_1
+; X64-LFENCE-NEXT:  .LBB3_6: # %exit
+; X64-LFENCE-NEXT:    lfence
+; X64-LFENCE-NEXT:    addq $8, %rsp
+; X64-LFENCE-NEXT:    popq %rbx
+; X64-LFENCE-NEXT:    popq %r12
+; X64-LFENCE-NEXT:    popq %r13
+; X64-LFENCE-NEXT:    popq %r14
+; X64-LFENCE-NEXT:    popq %r15
+; X64-LFENCE-NEXT:    popq %rbp
+; X64-LFENCE-NEXT:    retq
+; X64-LFENCE-NEXT:  .LBB3_1: # %l1.header.preheader
 ; X64-LFENCE-NEXT:    movq %r8, %r14
 ; X64-LFENCE-NEXT:    movq %rcx, %rbx
 ; X64-LFENCE-NEXT:    movl %edx, %r13d
@@ -452,16 +462,6 @@ define void @test_basic_nested_loop(i32 %a, i32 %b, i32 %c, i32* %ptr1, i32* %pt
 ; X64-LFENCE-NEXT:    cmpl %r13d, %ebp
 ; X64-LFENCE-NEXT:    jl .LBB3_4
 ; X64-LFENCE-NEXT:    jmp .LBB3_5
-; X64-LFENCE-NEXT:  .LBB3_6: # %exit
-; X64-LFENCE-NEXT:    lfence
-; X64-LFENCE-NEXT:    addq $8, %rsp
-; X64-LFENCE-NEXT:    popq %rbx
-; X64-LFENCE-NEXT:    popq %r12
-; X64-LFENCE-NEXT:    popq %r13
-; X64-LFENCE-NEXT:    popq %r14
-; X64-LFENCE-NEXT:    popq %r15
-; X64-LFENCE-NEXT:    popq %rbp
-; X64-LFENCE-NEXT:    retq
 entry:
   %a.cmp = icmp eq i32 %a, 0
   br i1 %a.cmp, label %l1.header, label %exit
diff --git a/llvm/test/CodeGen/X86/splat-for-size.ll b/llvm/test/CodeGen/X86/splat-for-size.ll
index 7aae59080..da68f069b 100644
--- a/llvm/test/CodeGen/X86/splat-for-size.ll
+++ b/llvm/test/CodeGen/X86/splat-for-size.ll
@@ -17,6 +17,17 @@ define <2 x double> @splat_v2f64(<2 x double> %x) #0 {
   ret <2 x double> %add
 }
 
+define <2 x double> @splat_v2f64_pgso(<2 x double> %x) !prof !14 {
+; CHECK-LABEL: splat_v2f64_pgso:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovddup {{.*#+}} xmm1 = [1.0E+0,1.0E+0]
+; CHECK-NEXT:    # xmm1 = mem[0,0]
+; CHECK-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    retq
+  %add = fadd <2 x double> %x, <double 1.0, double 1.0>
+  ret <2 x double> %add
+}
+
 define <4 x double> @splat_v4f64(<4 x double> %x) #1 {
 ; CHECK-LABEL: splat_v4f64:
 ; CHECK:       # %bb.0:
@@ -27,6 +38,16 @@ define <4 x double> @splat_v4f64(<4 x double> %x) #1 {
   ret <4 x double> %add
 }
 
+define <4 x double> @splat_v4f64_pgso(<4 x double> %x) !prof !14 {
+; CHECK-LABEL: splat_v4f64_pgso:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vbroadcastsd {{.*#+}} ymm1 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; CHECK-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    retq
+  %add = fadd <4 x double> %x, <double 1.0, double 1.0, double 1.0, double 1.0>
+  ret <4 x double> %add
+}
+
 define <4 x float> @splat_v4f32(<4 x float> %x) #0 {
 ; CHECK-LABEL: splat_v4f32:
 ; CHECK:       # %bb.0:
@@ -37,6 +58,16 @@ define <4 x float> @splat_v4f32(<4 x float> %x) #0 {
   ret <4 x float> %add
 }
 
+define <4 x float> @splat_v4f32_pgso(<4 x float> %x) !prof !14 {
+; CHECK-LABEL: splat_v4f32_pgso:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vbroadcastss {{.*#+}} xmm1 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; CHECK-NEXT:    vaddps %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    retq
+  %add = fadd <4 x float> %x, <float 1.0, float 1.0, float 1.0, float 1.0>
+  ret <4 x float> %add
+}
+
 define <8 x float> @splat_v8f32(<8 x float> %x) #1 {
 ; CHECK-LABEL: splat_v8f32:
 ; CHECK:       # %bb.0:
@@ -47,6 +78,16 @@ define <8 x float> @splat_v8f32(<8 x float> %x) #1 {
   ret <8 x float> %add
 }
 
+define <8 x float> @splat_v8f32_pgso(<8 x float> %x) !prof !14 {
+; CHECK-LABEL: splat_v8f32_pgso:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vbroadcastss {{.*#+}} ymm1 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; CHECK-NEXT:    vaddps %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    retq
+  %add = fadd <8 x float> %x, <float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>
+  ret <8 x float> %add
+}
+
 ; AVX can't do integer splats, so fake it: use vmovddup to splat 64-bit value.
 ; We also generate vmovddup for AVX2 because it's one byte smaller than vpbroadcastq.
 define <2 x i64> @splat_v2i64(<2 x i64> %x) #1 {
@@ -66,6 +107,23 @@ define <2 x i64> @splat_v2i64(<2 x i64> %x) #1 {
   ret <2 x i64> %add
 }
 
+define <2 x i64> @splat_v2i64_pgso(<2 x i64> %x) !prof !14 {
+; AVX-LABEL: splat_v2i64_pgso:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovddup {{.*#+}} xmm1 = [2,2]
+; AVX-NEXT:    # xmm1 = mem[0,0]
+; AVX-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; AVX2-LABEL: splat_v2i64_pgso:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} xmm1 = [2,2]
+; AVX2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    retq
+  %add = add <2 x i64> %x, <i64 2, i64 2>
+  ret <2 x i64> %add
+}
+
 ; AVX can't do 256-bit integer ops, so we split this into two 128-bit vectors,
 ; and then we fake it: use vmovddup to splat 64-bit value.
 define <4 x i64> @splat_v4i64(<4 x i64> %x) #0 {
@@ -88,6 +146,26 @@ define <4 x i64> @splat_v4i64(<4 x i64> %x) #0 {
   ret <4 x i64> %add
 }
 
+define <4 x i64> @splat_v4i64_pgso(<4 x i64> %x) !prof !14 {
+; AVX-LABEL: splat_v4i64_pgso:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX-NEXT:    vmovddup {{.*#+}} xmm2 = [2,2]
+; AVX-NEXT:    # xmm2 = mem[0,0]
+; AVX-NEXT:    vpaddq %xmm2, %xmm1, %xmm1
+; AVX-NEXT:    vpaddq %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX-NEXT:    retq
+;
+; AVX2-LABEL: splat_v4i64_pgso:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [2,2,2,2]
+; AVX2-NEXT:    vpaddq %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %add = add <4 x i64> %x, <i64 2, i64 2, i64 2, i64 2>
+  ret <4 x i64> %add
+}
+
 ; AVX can't do integer splats, so fake it: use vbroadcastss to splat 32-bit value.
 define <4 x i32> @splat_v4i32(<4 x i32> %x) #1 {
 ; AVX-LABEL: splat_v4i32:
@@ -105,6 +183,22 @@ define <4 x i32> @splat_v4i32(<4 x i32> %x) #1 {
   ret <4 x i32> %add
 }
 
+define <4 x i32> @splat_v4i32_pgso(<4 x i32> %x) !prof !14 {
+; AVX-LABEL: splat_v4i32_pgso:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vbroadcastss {{.*#+}} xmm1 = [2,2,2,2]
+; AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; AVX2-LABEL: splat_v4i32_pgso:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [2,2,2,2]
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    retq
+  %add = add <4 x i32> %x, <i32 2, i32 2, i32 2, i32 2>
+  ret <4 x i32> %add
+}
+
 ; AVX can't do integer splats, so fake it: use vbroadcastss to splat 32-bit value.
 define <8 x i32> @splat_v8i32(<8 x i32> %x) #0 {
 ; AVX-LABEL: splat_v8i32:
@@ -125,6 +219,25 @@ define <8 x i32> @splat_v8i32(<8 x i32> %x) #0 {
   ret <8 x i32> %add
 }
 
+define <8 x i32> @splat_v8i32_pgso(<8 x i32> %x) !prof !14 {
+; AVX-LABEL: splat_v8i32_pgso:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX-NEXT:    vbroadcastss {{.*#+}} xmm2 = [2,2,2,2]
+; AVX-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; AVX-NEXT:    vpaddd %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX-NEXT:    retq
+;
+; AVX2-LABEL: splat_v8i32_pgso:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [2,2,2,2,2,2,2,2]
+; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %add = add <8 x i32> %x, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  ret <8 x i32> %add
+}
+
 ; AVX can't do integer splats, and there's no broadcast fakery for 16-bit. Could use pshuflw, etc?
 define <8 x i16> @splat_v8i16(<8 x i16> %x) #1 {
 ; AVX-LABEL: splat_v8i16:
@@ -141,6 +254,21 @@ define <8 x i16> @splat_v8i16(<8 x i16> %x) #1 {
   ret <8 x i16> %add
 }
 
+define <8 x i16> @splat_v8i16_pgso(<8 x i16> %x) !prof !14 {
+; AVX-LABEL: splat_v8i16_pgso:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpaddw {{.*}}(%rip), %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; AVX2-LABEL: splat_v8i16_pgso:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpbroadcastw {{.*#+}} xmm1 = [2,2,2,2,2,2,2,2]
+; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    retq
+  %add = add <8 x i16> %x, <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
+  ret <8 x i16> %add
+}
+
 ; AVX can't do integer splats, and there's no broadcast fakery for 16-bit. Could use pshuflw, etc?
 define <16 x i16> @splat_v16i16(<16 x i16> %x) #0 {
 ; AVX-LABEL: splat_v16i16:
@@ -161,6 +289,25 @@ define <16 x i16> @splat_v16i16(<16 x i16> %x) #0 {
   ret <16 x i16> %add
 }
 
+define <16 x i16> @splat_v16i16_pgso(<16 x i16> %x) !prof !14 {
+; AVX-LABEL: splat_v16i16_pgso:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm2 = [2,2,2,2,2,2,2,2]
+; AVX-NEXT:    vpaddw %xmm2, %xmm1, %xmm1
+; AVX-NEXT:    vpaddw %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX-NEXT:    retq
+;
+; AVX2-LABEL: splat_v16i16_pgso:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpbroadcastw {{.*#+}} ymm1 = [2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2]
+; AVX2-NEXT:    vpaddw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %add = add <16 x i16> %x, <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
+  ret <16 x i16> %add
+}
+
 ; AVX can't do integer splats, and there's no broadcast fakery for 8-bit. Could use pshufb, etc?
 define <16 x i8> @splat_v16i8(<16 x i8> %x) #1 {
 ; AVX-LABEL: splat_v16i8:
@@ -177,6 +324,21 @@ define <16 x i8> @splat_v16i8(<16 x i8> %x) #1 {
   ret <16 x i8> %add
 }
 
+define <16 x i8> @splat_v16i8_pgso(<16 x i8> %x) !prof !14 {
+; AVX-LABEL: splat_v16i8_pgso:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpaddb {{.*}}(%rip), %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; AVX2-LABEL: splat_v16i8_pgso:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpbroadcastb {{.*#+}} xmm1 = [2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2]
+; AVX2-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    retq
+  %add = add <16 x i8> %x, <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>
+  ret <16 x i8> %add
+}
+
 ; AVX can't do integer splats, and there's no broadcast fakery for 8-bit. Could use pshufb, etc?
 define <32 x i8> @splat_v32i8(<32 x i8> %x) #0 {
 ; AVX-LABEL: splat_v32i8:
@@ -197,6 +359,25 @@ define <32 x i8> @splat_v32i8(<32 x i8> %x) #0 {
   ret <32 x i8> %add
 }
 
+define <32 x i8> @splat_v32i8_pgso(<32 x i8> %x) !prof !14 {
+; AVX-LABEL: splat_v32i8_pgso:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm2 = [2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2]
+; AVX-NEXT:    vpaddb %xmm2, %xmm1, %xmm1
+; AVX-NEXT:    vpaddb %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX-NEXT:    retq
+;
+; AVX2-LABEL: splat_v32i8_pgso:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpbroadcastb {{.*#+}} ymm1 = [2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2]
+; AVX2-NEXT:    vpaddb %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %add = add <32 x i8> %x, <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>
+  ret <32 x i8> %add
+}
+
 ; PR23259: Verify that ISel doesn't crash with a 'fatal error in backend'
 ; due to a missing AVX pattern to select a v2i64 X86ISD::BROADCAST of a
 ; loadi64 with multiple uses.
@@ -238,3 +419,20 @@ entry:
 
 attributes #0 = { optsize }
 attributes #1 = { minsize }
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 1, !"ProfileSummary", !1}
+!1 = !{!2, !3, !4, !5, !6, !7, !8, !9}
+!2 = !{!"ProfileFormat", !"InstrProf"}
+!3 = !{!"TotalCount", i64 10000}
+!4 = !{!"MaxCount", i64 10}
+!5 = !{!"MaxInternalCount", i64 1}
+!6 = !{!"MaxFunctionCount", i64 1000}
+!7 = !{!"NumCounts", i64 3}
+!8 = !{!"NumFunctions", i64 3}
+!9 = !{!"DetailedSummary", !10}
+!10 = !{!11, !12, !13}
+!11 = !{i32 10000, i64 100, i32 1}
+!12 = !{i32 999000, i64 100, i32 1}
+!13 = !{i32 999999, i64 1, i32 2}
+!14 = !{!"function_entry_count", i64 0}
diff --git a/llvm/test/CodeGen/X86/split-eh-lpad-edges.ll b/llvm/test/CodeGen/X86/split-eh-lpad-edges.ll
index 82dd3b767..441d0ce99 100644
--- a/llvm/test/CodeGen/X86/split-eh-lpad-edges.ll
+++ b/llvm/test/CodeGen/X86/split-eh-lpad-edges.ll
@@ -1,4 +1,5 @@
-; RUN: llc < %s -mtriple=x86_64-apple-darwin | not grep jmp
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-apple-darwin | FileCheck %s
 ; rdar://6647639
 
 	%struct.FetchPlanHeader = type { i8*, i8*, i32, i8*, i8*, i8*, i8*, i8*, %struct.NSObject* (%struct.NSObject*, %struct.objc_selector*, ...)*, %struct.__attributeDescriptionFlags }
@@ -11,6 +12,36 @@
 @"\01l_objc_msgSend_fixup_alloc" = external global %struct._message_ref_t, align 16		; <%struct._message_ref_t*> [#uses=2]
 
 define %struct.NSArray* @newFetchedRowsForFetchPlan_MT(%struct.FetchPlanHeader* %fetchPlan, %struct.objc_selector* %selectionMethod, %struct.NSObject* %selectionParameter) ssp personality i32 (...)* @__gxx_personality_v0 {
+; CHECK-LABEL: newFetchedRowsForFetchPlan_MT:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:  Ltmp0:
+; CHECK-NEXT:    movq l_objc_msgSend_fixup_alloc@{{.*}}(%rip), %rsi
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    xorl %edi, %edi
+; CHECK-NEXT:    callq *%rax
+; CHECK-NEXT:  Ltmp1:
+; CHECK-NEXT:  ## %bb.1: ## %invcont
+; CHECK-NEXT:  Ltmp2:
+; CHECK-NEXT:    movq %rax, %rdi
+; CHECK-NEXT:    xorl %esi, %esi
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    callq _objc_msgSend
+; CHECK-NEXT:  Ltmp3:
+; CHECK-NEXT:  ## %bb.2: ## %invcont26
+; CHECK-NEXT:  Ltmp4:
+; CHECK-NEXT:    movq l_objc_msgSend_fixup_alloc@{{.*}}(%rip), %rsi
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    xorl %edi, %edi
+; CHECK-NEXT:    callq *%rax
+; CHECK-NEXT:  Ltmp5:
+; CHECK-NEXT:  ## %bb.3: ## %invcont27
+; CHECK-NEXT:    ud2
+; CHECK-NEXT:  LBB0_4: ## %lpad
+; CHECK-NEXT:  Ltmp6:
+; CHECK-NEXT:    ud2
+; CHECK-NEXT:  Lfunc_end0:
 entry:
 	%0 = invoke %struct.NSObject* null(%struct.NSObject* null, %struct._message_ref_t* @"\01l_objc_msgSend_fixup_alloc")
 			to label %invcont unwind label %lpad		; <%struct.NSObject*> [#uses=1]
diff --git a/llvm/test/CodeGen/X86/split-vector-bitcast.ll b/llvm/test/CodeGen/X86/split-vector-bitcast.ll
index 1ad716003..267a910b2 100644
--- a/llvm/test/CodeGen/X86/split-vector-bitcast.ll
+++ b/llvm/test/CodeGen/X86/split-vector-bitcast.ll
@@ -1,8 +1,30 @@
-; RUN: llc < %s -mtriple=i686-- -mattr=-sse2,+sse | grep addps
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- -mattr=-sse2,+sse | FileCheck %s
 
 ; PR10497 + another isel issue with sse2 disabled
 ; (This is primarily checking that this construct doesn't crash.)
 define void @a(<2 x float>* %a, <2 x i32>* %b) {
+; CHECK-LABEL: a:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subl $8, %esp
+; CHECK-NEXT:    .cfi_def_cfa_offset 12
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT:    xorps %xmm0, %xmm0
+; CHECK-NEXT:    movlps {{.*#+}} xmm0 = mem[0,1],xmm0[2,3]
+; CHECK-NEXT:    addps %xmm0, %xmm0
+; CHECK-NEXT:    movss %xmm0, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; CHECK-NEXT:    movss %xmm0, (%esp)
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT:    movl (%esp), %edx
+; CHECK-NEXT:    addl %edx, %edx
+; CHECK-NEXT:    addl %ecx, %ecx
+; CHECK-NEXT:    movl %ecx, (%eax)
+; CHECK-NEXT:    movl %edx, 4(%eax)
+; CHECK-NEXT:    addl $8, %esp
+; CHECK-NEXT:    .cfi_def_cfa_offset 4
+; CHECK-NEXT:    retl
   %cc = load <2 x float>, <2 x float>* %a
   %c = fadd <2 x float> %cc, %cc
   %dd = bitcast <2 x float> %c to <2 x i32>
diff --git a/llvm/test/CodeGen/X86/split-vector-rem.ll b/llvm/test/CodeGen/X86/split-vector-rem.ll
index 0e51dbf78..ef03075ac 100644
--- a/llvm/test/CodeGen/X86/split-vector-rem.ll
+++ b/llvm/test/CodeGen/X86/split-vector-rem.ll
@@ -1,15 +1,207 @@
-; RUN: llc < %s -mtriple=x86_64-- | grep div | count 16
-; RUN: llc < %s -mtriple=x86_64-- | grep fmodf | count 8
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-- | FileCheck %s
 
 define <8 x i32> @foo(<8 x i32> %t, <8 x i32> %u) {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movdqa %xmm0, %xmm4
+; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; CHECK-NEXT:    movd %xmm0, %eax
+; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[3,1,2,3]
+; CHECK-NEXT:    movd %xmm0, %ecx
+; CHECK-NEXT:    cltd
+; CHECK-NEXT:    idivl %ecx
+; CHECK-NEXT:    movd %edx, %xmm0
+; CHECK-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[2,3,0,1]
+; CHECK-NEXT:    movd %xmm5, %eax
+; CHECK-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[2,3,0,1]
+; CHECK-NEXT:    movd %xmm5, %ecx
+; CHECK-NEXT:    cltd
+; CHECK-NEXT:    idivl %ecx
+; CHECK-NEXT:    movd %edx, %xmm5
+; CHECK-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1]
+; CHECK-NEXT:    movd %xmm4, %eax
+; CHECK-NEXT:    movd %xmm2, %ecx
+; CHECK-NEXT:    cltd
+; CHECK-NEXT:    idivl %ecx
+; CHECK-NEXT:    movd %edx, %xmm0
+; CHECK-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,2,3]
+; CHECK-NEXT:    movd %xmm4, %eax
+; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,2,3]
+; CHECK-NEXT:    movd %xmm2, %ecx
+; CHECK-NEXT:    cltd
+; CHECK-NEXT:    idivl %ecx
+; CHECK-NEXT:    movd %edx, %xmm2
+; CHECK-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; CHECK-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm5[0]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,1,2,3]
+; CHECK-NEXT:    movd %xmm2, %eax
+; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[3,1,2,3]
+; CHECK-NEXT:    movd %xmm2, %ecx
+; CHECK-NEXT:    cltd
+; CHECK-NEXT:    idivl %ecx
+; CHECK-NEXT:    movd %edx, %xmm2
+; CHECK-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[2,3,0,1]
+; CHECK-NEXT:    movd %xmm4, %eax
+; CHECK-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[2,3,0,1]
+; CHECK-NEXT:    movd %xmm4, %ecx
+; CHECK-NEXT:    cltd
+; CHECK-NEXT:    idivl %ecx
+; CHECK-NEXT:    movd %edx, %xmm4
+; CHECK-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
+; CHECK-NEXT:    movd %xmm1, %eax
+; CHECK-NEXT:    movd %xmm3, %ecx
+; CHECK-NEXT:    cltd
+; CHECK-NEXT:    idivl %ecx
+; CHECK-NEXT:    movd %edx, %xmm2
+; CHECK-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; CHECK-NEXT:    movd %xmm1, %eax
+; CHECK-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,2,3]
+; CHECK-NEXT:    movd %xmm1, %ecx
+; CHECK-NEXT:    cltd
+; CHECK-NEXT:    idivl %ecx
+; CHECK-NEXT:    movd %edx, %xmm1
+; CHECK-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; CHECK-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
+; CHECK-NEXT:    movdqa %xmm2, %xmm1
+; CHECK-NEXT:    retq
 	%m = srem <8 x i32> %t, %u
 	ret <8 x i32> %m
 }
 define <8 x i32> @bar(<8 x i32> %t, <8 x i32> %u) {
+; CHECK-LABEL: bar:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movdqa %xmm0, %xmm4
+; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; CHECK-NEXT:    movd %xmm0, %eax
+; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[3,1,2,3]
+; CHECK-NEXT:    movd %xmm0, %ecx
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    divl %ecx
+; CHECK-NEXT:    movd %edx, %xmm0
+; CHECK-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[2,3,0,1]
+; CHECK-NEXT:    movd %xmm5, %eax
+; CHECK-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[2,3,0,1]
+; CHECK-NEXT:    movd %xmm5, %ecx
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    divl %ecx
+; CHECK-NEXT:    movd %edx, %xmm5
+; CHECK-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1]
+; CHECK-NEXT:    movd %xmm4, %eax
+; CHECK-NEXT:    movd %xmm2, %ecx
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    divl %ecx
+; CHECK-NEXT:    movd %edx, %xmm0
+; CHECK-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,2,3]
+; CHECK-NEXT:    movd %xmm4, %eax
+; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,2,3]
+; CHECK-NEXT:    movd %xmm2, %ecx
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    divl %ecx
+; CHECK-NEXT:    movd %edx, %xmm2
+; CHECK-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; CHECK-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm5[0]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,1,2,3]
+; CHECK-NEXT:    movd %xmm2, %eax
+; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[3,1,2,3]
+; CHECK-NEXT:    movd %xmm2, %ecx
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    divl %ecx
+; CHECK-NEXT:    movd %edx, %xmm2
+; CHECK-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[2,3,0,1]
+; CHECK-NEXT:    movd %xmm4, %eax
+; CHECK-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[2,3,0,1]
+; CHECK-NEXT:    movd %xmm4, %ecx
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    divl %ecx
+; CHECK-NEXT:    movd %edx, %xmm4
+; CHECK-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
+; CHECK-NEXT:    movd %xmm1, %eax
+; CHECK-NEXT:    movd %xmm3, %ecx
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    divl %ecx
+; CHECK-NEXT:    movd %edx, %xmm2
+; CHECK-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; CHECK-NEXT:    movd %xmm1, %eax
+; CHECK-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,2,3]
+; CHECK-NEXT:    movd %xmm1, %ecx
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    divl %ecx
+; CHECK-NEXT:    movd %edx, %xmm1
+; CHECK-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; CHECK-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
+; CHECK-NEXT:    movdqa %xmm2, %xmm1
+; CHECK-NEXT:    retq
 	%m = urem <8 x i32> %t, %u
 	ret <8 x i32> %m
 }
 define <8 x float> @qux(<8 x float> %t, <8 x float> %u) {
+; CHECK-LABEL: qux:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subq $104, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 112
+; CHECK-NEXT:    movaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps %xmm2, (%rsp) # 16-byte Spill
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; CHECK-NEXT:    movaps %xmm2, %xmm1
+; CHECK-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,1],xmm2[2,3]
+; CHECK-NEXT:    callq fmodf
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
+; CHECK-NEXT:    movaps (%rsp), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    movhlps {{.*#+}} xmm1 = xmm1[1,1]
+; CHECK-NEXT:    callq fmodf
+; CHECK-NEXT:    unpcklps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    movaps (%rsp), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    callq fmodf
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; CHECK-NEXT:    movaps (%rsp), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; CHECK-NEXT:    callq fmodf
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; CHECK-NEXT:    unpcklpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm1 = xmm1[0],mem[0]
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; CHECK-NEXT:    callq fmodf
+; CHECK-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    movhlps {{.*#+}} xmm1 = xmm1[1,1]
+; CHECK-NEXT:    callq fmodf
+; CHECK-NEXT:    unpcklps (%rsp), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
+; CHECK-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    callq fmodf
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; CHECK-NEXT:    callq fmodf
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; CHECK-NEXT:    unpcklpd (%rsp), %xmm1 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm1 = xmm1[0],mem[0]
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    addq $104, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
 	%m = frem <8 x float> %t, %u
 	ret <8 x float> %m
 }
diff --git a/llvm/test/CodeGen/X86/sqrt-fastmath-mir.ll b/llvm/test/CodeGen/X86/sqrt-fastmath-mir.ll
index 687079232..6abf34c60 100644
--- a/llvm/test/CodeGen/X86/sqrt-fastmath-mir.ll
+++ b/llvm/test/CodeGen/X86/sqrt-fastmath-mir.ll
@@ -7,18 +7,18 @@ define float @foo(float %f) #0 {
 ; CHECK: body:
 ; CHECK:     %0:fr32 = COPY $xmm0
 ; CHECK:     %1:fr32 = VRSQRTSSr killed %2, %0
-; CHECK:     %3:fr32 = VMULSSrr %0, %1
+; CHECK:     %3:fr32 = nofpexcept VMULSSrr %0, %1
 ; CHECK:     %4:fr32 = VMOVSSrm
-; CHECK:     %5:fr32 = VFMADD213SSr %1, killed %3, %4
+; CHECK:     %5:fr32 = nofpexcept VFMADD213SSr %1, killed %3, %4
 ; CHECK:     %6:fr32 = VMOVSSrm
-; CHECK:     %7:fr32 = VMULSSrr %1, %6
-; CHECK:     %8:fr32 = VMULSSrr killed %7, killed %5
-; CHECK:     %9:fr32 = VMULSSrr %0, %8
-; CHECK:     %10:fr32 = VFMADD213SSr %8, %9, %4
-; CHECK:     %11:fr32 = VMULSSrr %9, %6
-; CHECK:     %12:fr32 = VMULSSrr killed %11, killed %10
+; CHECK:     %7:fr32 = nofpexcept VMULSSrr %1, %6
+; CHECK:     %8:fr32 = nofpexcept VMULSSrr killed %7, killed %5
+; CHECK:     %9:fr32 = nofpexcept VMULSSrr %0, %8
+; CHECK:     %10:fr32 = nofpexcept VFMADD213SSr %8, %9, %4
+; CHECK:     %11:fr32 = nofpexcept VMULSSrr %9, %6
+; CHECK:     %12:fr32 = nofpexcept VMULSSrr killed %11, killed %10
 ; CHECK:     %14:fr32 = FsFLD0SS
-; CHECK:     %15:fr32 = VCMPSSrr %0, killed %14, 0
+; CHECK:     %15:fr32 = nofpexcept VCMPSSrr %0, killed %14, 0
 ; CHECK:     %17:vr128 = VPANDNrr killed %16, killed %13
 ; CHECK:     $xmm0 = COPY %18
 ; CHECK:     RET 0, $xmm0
@@ -31,16 +31,16 @@ define float @rfoo(float %f) #0 {
 ; CHECK: body:             |
 ; CHECK:     %0:fr32 = COPY $xmm0
 ; CHECK:     %1:fr32 = VRSQRTSSr killed %2, %0
-; CHECK:     %3:fr32 = nnan ninf nsz arcp contract afn reassoc VMULSSrr %0, %1
+; CHECK:     %3:fr32 = nnan ninf nsz arcp contract afn reassoc nofpexcept VMULSSrr %0, %1
 ; CHECK:     %4:fr32 = VMOVSSrm
-; CHECK:     %5:fr32 = nnan ninf nsz arcp contract afn reassoc VFMADD213SSr %1, killed %3, %4
+; CHECK:     %5:fr32 = nnan ninf nsz arcp contract afn reassoc nofpexcept VFMADD213SSr %1, killed %3, %4
 ; CHECK:     %6:fr32 = VMOVSSrm
-; CHECK:     %7:fr32 = nnan ninf nsz arcp contract afn reassoc VMULSSrr %1, %6
-; CHECK:     %8:fr32 = nnan ninf nsz arcp contract afn reassoc VMULSSrr killed %7, killed %5
-; CHECK:     %9:fr32 = nnan ninf nsz arcp contract afn reassoc VMULSSrr %0, %8
-; CHECK:     %10:fr32 = nnan ninf nsz arcp contract afn reassoc VFMADD213SSr %8, killed %9, %4
-; CHECK:     %11:fr32 = nnan ninf nsz arcp contract afn reassoc VMULSSrr %8, %6
-; CHECK:     %12:fr32 = nnan ninf nsz arcp contract afn reassoc VMULSSrr killed %11, killed %10
+; CHECK:     %7:fr32 = nnan ninf nsz arcp contract afn reassoc nofpexcept VMULSSrr %1, %6
+; CHECK:     %8:fr32 = nnan ninf nsz arcp contract afn reassoc nofpexcept VMULSSrr killed %7, killed %5
+; CHECK:     %9:fr32 = nnan ninf nsz arcp contract afn reassoc nofpexcept VMULSSrr %0, %8
+; CHECK:     %10:fr32 = nnan ninf nsz arcp contract afn reassoc nofpexcept VFMADD213SSr %8, killed %9, %4
+; CHECK:     %11:fr32 = nnan ninf nsz arcp contract afn reassoc nofpexcept VMULSSrr %8, %6
+; CHECK:     %12:fr32 = nnan ninf nsz arcp contract afn reassoc nofpexcept VMULSSrr killed %11, killed %10
 ; CHECK:     $xmm0 = COPY %12
 ; CHECK:     RET 0, $xmm0
   %sqrt = tail call float @llvm.sqrt.f32(float %f)
diff --git a/llvm/test/CodeGen/X86/sqrt-fastmath.ll b/llvm/test/CodeGen/X86/sqrt-fastmath.ll
index 6e0273d51..6aad4e8f6 100644
--- a/llvm/test/CodeGen/X86/sqrt-fastmath.ll
+++ b/llvm/test/CodeGen/X86/sqrt-fastmath.ll
@@ -94,7 +94,7 @@ define float @finite_f32_estimate(float %f) #1 {
 ; AVX512-NEXT:    vmulss %xmm1, %xmm2, %xmm1
 ; AVX512-NEXT:    vxorps %xmm2, %xmm2, %xmm2
 ; AVX512-NEXT:    vcmpeqss %xmm2, %xmm0, %k1
-; AVX512-NEXT:    vmovss %xmm2, %xmm0, %xmm1 {%k1}
+; AVX512-NEXT:    vmovss %xmm2, %xmm1, %xmm1 {%k1}
 ; AVX512-NEXT:    vmovaps %xmm1, %xmm0
 ; AVX512-NEXT:    retq
   %call = tail call float @__sqrtf_finite(float %f) #2
@@ -165,7 +165,7 @@ define float @sqrtf_check_denorms(float %x) #3 {
 ; AVX512-NEXT:    vandps %xmm2, %xmm0, %xmm0
 ; AVX512-NEXT:    vcmpltss {{.*}}(%rip), %xmm0, %k1
 ; AVX512-NEXT:    vxorps %xmm0, %xmm0, %xmm0
-; AVX512-NEXT:    vmovss %xmm0, %xmm0, %xmm1 {%k1}
+; AVX512-NEXT:    vmovss %xmm0, %xmm1, %xmm1 {%k1}
 ; AVX512-NEXT:    vmovaps %xmm1, %xmm0
 ; AVX512-NEXT:    retq
   %call = tail call float @__sqrtf_finite(float %x) #2
diff --git a/llvm/test/CodeGen/X86/sqrt-partial.ll b/llvm/test/CodeGen/X86/sqrt-partial.ll
index 6f0d52490..48914d8ed 100644
--- a/llvm/test/CodeGen/X86/sqrt-partial.ll
+++ b/llvm/test/CodeGen/X86/sqrt-partial.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=CHECK,AVX
 
 ; PR31455 - https://bugs.llvm.org/show_bug.cgi?id=31455
 ; We have to assume that errno can be set, so we have to make a libcall in that case.
@@ -9,35 +10,100 @@
 ; for that), but we're checking the final asm to make sure that comes out as expected too.
 
 define float @f(float %val) nounwind {
-; CHECK-LABEL: f:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    xorps %xmm1, %xmm1
-; CHECK-NEXT:    ucomiss %xmm1, %xmm0
-; CHECK-NEXT:    jb .LBB0_2
-; CHECK-NEXT:  # %bb.1: # %.split
-; CHECK-NEXT:    sqrtss %xmm0, %xmm0
-; CHECK-NEXT:    retq
-; CHECK-NEXT:  .LBB0_2: # %call.sqrt
-; CHECK-NEXT:    jmp sqrtf # TAILCALL
+; SSE-LABEL: f:
+; SSE:       # %bb.0:
+; SSE-NEXT:    xorps %xmm1, %xmm1
+; SSE-NEXT:    ucomiss %xmm1, %xmm0
+; SSE-NEXT:    jb .LBB0_2
+; SSE-NEXT:  # %bb.1: # %.split
+; SSE-NEXT:    sqrtss %xmm0, %xmm0
+; SSE-NEXT:    retq
+; SSE-NEXT:  .LBB0_2: # %call.sqrt
+; SSE-NEXT:    jmp sqrtf # TAILCALL
+;
+; AVX-LABEL: f:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vucomiss %xmm1, %xmm0
+; AVX-NEXT:    jb .LBB0_2
+; AVX-NEXT:  # %bb.1: # %.split
+; AVX-NEXT:    vsqrtss %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    retq
+; AVX-NEXT:  .LBB0_2: # %call.sqrt
+; AVX-NEXT:    jmp sqrtf # TAILCALL
   %res = tail call float @sqrtf(float %val)
   ret float %res
 }
 
 define double @d(double %val) nounwind {
-; CHECK-LABEL: d:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    xorps %xmm1, %xmm1
-; CHECK-NEXT:    ucomisd %xmm1, %xmm0
-; CHECK-NEXT:    jb .LBB1_2
-; CHECK-NEXT:  # %bb.1: # %.split
-; CHECK-NEXT:    sqrtsd %xmm0, %xmm0
-; CHECK-NEXT:    retq
-; CHECK-NEXT:  .LBB1_2: # %call.sqrt
-; CHECK-NEXT:    jmp sqrt # TAILCALL
+; SSE-LABEL: d:
+; SSE:       # %bb.0:
+; SSE-NEXT:    xorpd %xmm1, %xmm1
+; SSE-NEXT:    ucomisd %xmm1, %xmm0
+; SSE-NEXT:    jb .LBB1_2
+; SSE-NEXT:  # %bb.1: # %.split
+; SSE-NEXT:    sqrtsd %xmm0, %xmm0
+; SSE-NEXT:    retq
+; SSE-NEXT:  .LBB1_2: # %call.sqrt
+; SSE-NEXT:    jmp sqrt # TAILCALL
+;
+; AVX-LABEL: d:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vucomisd %xmm1, %xmm0
+; AVX-NEXT:    jb .LBB1_2
+; AVX-NEXT:  # %bb.1: # %.split
+; AVX-NEXT:    vsqrtsd %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    retq
+; AVX-NEXT:  .LBB1_2: # %call.sqrt
+; AVX-NEXT:    jmp sqrt # TAILCALL
   %res = tail call double @sqrt(double %val)
   ret double %res
 }
 
+define double @minsize(double %x, double %y) minsize {
+; SSE-LABEL: minsize:
+; SSE:       # %bb.0:
+; SSE-NEXT:    mulsd %xmm0, %xmm0
+; SSE-NEXT:    mulsd %xmm1, %xmm1
+; SSE-NEXT:    addsd %xmm0, %xmm1
+; SSE-NEXT:    sqrtsd %xmm1, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: minsize:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmulsd %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vmulsd %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vaddsd %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vsqrtsd %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %t3 = fmul fast double %x, %x
+  %t4 = fmul fast double %y, %y
+  %t5 = fadd fast double %t3, %t4
+  %t6 = tail call fast double @llvm.sqrt.f64(double %t5)
+  ret double %t6
+}
+
+; Partial reg avoidance may involve register allocation
+; rather than adding an instruction.
+
+define double @partial_dep_minsize(double %x, double %y) minsize {
+; SSE-LABEL: partial_dep_minsize:
+; SSE:       # %bb.0:
+; SSE-NEXT:    sqrtsd %xmm1, %xmm0
+; SSE-NEXT:    addsd %xmm1, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: partial_dep_minsize:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vsqrtsd %xmm1, %xmm1, %xmm0
+; AVX-NEXT:    vaddsd %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %t6 = tail call fast double @llvm.sqrt.f64(double %y)
+  %t = fadd fast double %t6, %y
+  ret double %t
+}
+
 declare float @sqrtf(float)
 declare double @sqrt(double)
-
+declare double @llvm.sqrt.f64(double)
diff --git a/llvm/test/CodeGen/X86/sqrt.ll b/llvm/test/CodeGen/X86/sqrt.ll
index b22c04125..becdbb18f 100644
--- a/llvm/test/CodeGen/X86/sqrt.ll
+++ b/llvm/test/CodeGen/X86/sqrt.ll
@@ -1,23 +1,34 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=-avx,+sse2                             | FileCheck %s --check-prefix=SSE2
 ; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=-avx,+sse2 -fast-isel -fast-isel-abort=1 | FileCheck %s --check-prefix=SSE2
 ; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=-avx2,+avx                             | FileCheck %s --check-prefix=AVX
 ; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=-avx2,+avx -fast-isel -fast-isel-abort=1 | FileCheck %s --check-prefix=AVX
 
 define float @test_sqrt_f32(float %a) {
-; SSE2-LABEL: test_sqrt_f32
-; SSE2:       sqrtss %xmm0, %xmm0
-; AVX-LABEL:  test_sqrt_f32
-; AVX:        vsqrtss %xmm0, %xmm0
+; SSE2-LABEL: test_sqrt_f32:
+; SSE2:       ## %bb.0:
+; SSE2-NEXT:    sqrtss %xmm0, %xmm0
+; SSE2-NEXT:    retq
+;
+; AVX-LABEL: test_sqrt_f32:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    vsqrtss %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    retq
   %res = call float @llvm.sqrt.f32(float %a)
   ret float %res
 }
 declare float @llvm.sqrt.f32(float) nounwind readnone
 
 define double @test_sqrt_f64(double %a) {
-; SSE2-LABEL: test_sqrt_f64
-; SSE2:       sqrtsd %xmm0, %xmm0
-; AVX-LABEL:  test_sqrt_f64
-; AVX:        vsqrtsd %xmm0, %xmm0
+; SSE2-LABEL: test_sqrt_f64:
+; SSE2:       ## %bb.0:
+; SSE2-NEXT:    sqrtsd %xmm0, %xmm0
+; SSE2-NEXT:    retq
+;
+; AVX-LABEL: test_sqrt_f64:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    vsqrtsd %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    retq
   %res = call double @llvm.sqrt.f64(double %a)
   ret double %res
 }
diff --git a/llvm/test/CodeGen/X86/srem-lkk.ll b/llvm/test/CodeGen/X86/srem-lkk.ll
new file mode 100644
index 000000000..ae30ae446
--- /dev/null
+++ b/llvm/test/CodeGen/X86/srem-lkk.ll
@@ -0,0 +1,159 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s --check-prefix=CHECK
+
+define i32 @fold_srem_positive_odd(i32 %x) {
+; CHECK-LABEL: fold_srem_positive_odd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movslq %edi, %rax
+; CHECK-NEXT:    imulq $-1401515643, %rax, %rcx # imm = 0xAC769185
+; CHECK-NEXT:    shrq $32, %rcx
+; CHECK-NEXT:    addl %eax, %ecx
+; CHECK-NEXT:    movl %ecx, %edx
+; CHECK-NEXT:    shrl $31, %edx
+; CHECK-NEXT:    sarl $6, %ecx
+; CHECK-NEXT:    addl %edx, %ecx
+; CHECK-NEXT:    imull $95, %ecx, %ecx
+; CHECK-NEXT:    subl %ecx, %eax
+; CHECK-NEXT:    # kill: def $eax killed $eax killed $rax
+; CHECK-NEXT:    retq
+  %1 = srem i32 %x, 95
+  ret i32 %1
+}
+
+
+define i32 @fold_srem_positive_even(i32 %x) {
+; CHECK-LABEL: fold_srem_positive_even:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movslq %edi, %rax
+; CHECK-NEXT:    imulq $1037275121, %rax, %rcx # imm = 0x3DD38FF1
+; CHECK-NEXT:    movq %rcx, %rdx
+; CHECK-NEXT:    shrq $63, %rdx
+; CHECK-NEXT:    sarq $40, %rcx
+; CHECK-NEXT:    addl %edx, %ecx
+; CHECK-NEXT:    imull $1060, %ecx, %ecx # imm = 0x424
+; CHECK-NEXT:    subl %ecx, %eax
+; CHECK-NEXT:    # kill: def $eax killed $eax killed $rax
+; CHECK-NEXT:    retq
+  %1 = srem i32 %x, 1060
+  ret i32 %1
+}
+
+
+define i32 @fold_srem_negative_odd(i32 %x) {
+; CHECK-LABEL: fold_srem_negative_odd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movslq %edi, %rax
+; CHECK-NEXT:    imulq $-1520762971, %rax, %rcx # imm = 0xA55AFFA5
+; CHECK-NEXT:    movq %rcx, %rdx
+; CHECK-NEXT:    shrq $63, %rdx
+; CHECK-NEXT:    sarq $40, %rcx
+; CHECK-NEXT:    addl %edx, %ecx
+; CHECK-NEXT:    imull $-723, %ecx, %ecx # imm = 0xFD2D
+; CHECK-NEXT:    subl %ecx, %eax
+; CHECK-NEXT:    # kill: def $eax killed $eax killed $rax
+; CHECK-NEXT:    retq
+  %1 = srem i32 %x, -723
+  ret i32 %1
+}
+
+
+define i32 @fold_srem_negative_even(i32 %x) {
+; CHECK-LABEL: fold_srem_negative_even:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movslq %edi, %rax
+; CHECK-NEXT:    imulq $-47844377, %rax, %rcx # imm = 0xFD25F3E7
+; CHECK-NEXT:    movq %rcx, %rdx
+; CHECK-NEXT:    shrq $63, %rdx
+; CHECK-NEXT:    sarq $40, %rcx
+; CHECK-NEXT:    addl %edx, %ecx
+; CHECK-NEXT:    imull $-22981, %ecx, %ecx # imm = 0xA63B
+; CHECK-NEXT:    subl %ecx, %eax
+; CHECK-NEXT:    # kill: def $eax killed $eax killed $rax
+; CHECK-NEXT:    retq
+  %1 = srem i32 %x, -22981
+  ret i32 %1
+}
+
+
+; Don't fold if we can combine srem with sdiv.
+define i32 @combine_srem_sdiv(i32 %x) {
+; CHECK-LABEL: combine_srem_sdiv:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movslq %edi, %rax
+; CHECK-NEXT:    imulq $-1401515643, %rax, %rcx # imm = 0xAC769185
+; CHECK-NEXT:    shrq $32, %rcx
+; CHECK-NEXT:    addl %eax, %ecx
+; CHECK-NEXT:    movl %ecx, %edx
+; CHECK-NEXT:    shrl $31, %edx
+; CHECK-NEXT:    sarl $6, %ecx
+; CHECK-NEXT:    addl %edx, %ecx
+; CHECK-NEXT:    imull $95, %ecx, %edx
+; CHECK-NEXT:    subl %edx, %eax
+; CHECK-NEXT:    addl %ecx, %eax
+; CHECK-NEXT:    # kill: def $eax killed $eax killed $rax
+; CHECK-NEXT:    retq
+  %1 = srem i32 %x, 95
+  %2 = sdiv i32 %x, 95
+  %3 = add i32 %1, %2
+  ret i32 %3
+}
+
+; Don't fold for divisors that are a power of two.
+define i32 @dont_fold_srem_power_of_two(i32 %x) {
+; CHECK-LABEL: dont_fold_srem_power_of_two:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    leal 63(%rax), %ecx
+; CHECK-NEXT:    testl %edi, %edi
+; CHECK-NEXT:    cmovnsl %edi, %ecx
+; CHECK-NEXT:    andl $-64, %ecx
+; CHECK-NEXT:    subl %ecx, %eax
+; CHECK-NEXT:    # kill: def $eax killed $eax killed $rax
+; CHECK-NEXT:    retq
+  %1 = srem i32 %x, 64
+  ret i32 %1
+}
+
+; Don't fold if the divisor is one.
+define i32 @dont_fold_srem_one(i32 %x) {
+; CHECK-LABEL: dont_fold_srem_one:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    retq
+  %1 = srem i32 %x, 1
+  ret i32 %1
+}
+
+; Don't fold if the divisor is 2^31.
+define i32 @dont_fold_srem_i32_smax(i32 %x) {
+; CHECK-LABEL: dont_fold_srem_i32_smax:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    # kill: def $edi killed $edi def $rdi
+; CHECK-NEXT:    leal 2147483647(%rdi), %eax
+; CHECK-NEXT:    testl %edi, %edi
+; CHECK-NEXT:    cmovnsl %edi, %eax
+; CHECK-NEXT:    andl $-2147483648, %eax # imm = 0x80000000
+; CHECK-NEXT:    addl %edi, %eax
+; CHECK-NEXT:    retq
+  %1 = srem i32 %x, 2147483648
+  ret i32 %1
+}
+
+; Don't fold i64 srem
+define i64 @dont_fold_srem_i64(i64 %x) {
+; CHECK-LABEL: dont_fold_srem_i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movabsq $6023426636313322977, %rcx # imm = 0x5397829CBC14E5E1
+; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    imulq %rcx
+; CHECK-NEXT:    movq %rdx, %rax
+; CHECK-NEXT:    shrq $63, %rax
+; CHECK-NEXT:    sarq $5, %rdx
+; CHECK-NEXT:    addq %rax, %rdx
+; CHECK-NEXT:    imulq $98, %rdx, %rax
+; CHECK-NEXT:    subq %rax, %rdi
+; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    retq
+  %1 = srem i64 %x, 98
+  ret i64 %1
+}
diff --git a/llvm/test/CodeGen/X86/srem-seteq-optsize.ll b/llvm/test/CodeGen/X86/srem-seteq-optsize.ll
new file mode 100644
index 000000000..a1fcdc915
--- /dev/null
+++ b/llvm/test/CodeGen/X86/srem-seteq-optsize.ll
@@ -0,0 +1,73 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=i686-unknown-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,X86
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,X64
+
+; On X86, division in expensive. BuildRemEqFold should therefore run even
+; when optimizing for size. Only optimizing for minimum size retains a plain div.
+
+define i32 @test_minsize(i32 %X) optsize minsize nounwind readnone {
+; X86-LABEL: test_minsize:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    pushl $5
+; X86-NEXT:    popl %ecx
+; X86-NEXT:    cltd
+; X86-NEXT:    idivl %ecx
+; X86-NEXT:    testl %edx, %edx
+; X86-NEXT:    je .LBB0_1
+; X86-NEXT:  # %bb.2:
+; X86-NEXT:    pushl $-10
+; X86-NEXT:    popl %eax
+; X86-NEXT:    retl
+; X86-NEXT:  .LBB0_1:
+; X86-NEXT:    pushl $42
+; X86-NEXT:    popl %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: test_minsize:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    pushq $5
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    cltd
+; X64-NEXT:    idivl %ecx
+; X64-NEXT:    testl %edx, %edx
+; X64-NEXT:    pushq $42
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    pushq $-10
+; X64-NEXT:    popq %rax
+; X64-NEXT:    cmovel %ecx, %eax
+; X64-NEXT:    retq
+  %rem = srem i32 %X, 5
+  %cmp = icmp eq i32 %rem, 0
+  %ret = select i1 %cmp, i32 42, i32 -10
+  ret i32 %ret
+}
+
+define i32 @test_optsize(i32 %X) optsize nounwind readnone {
+; X86-LABEL: test_optsize:
+; X86:       # %bb.0:
+; X86-NEXT:    imull $-858993459, {{[0-9]+}}(%esp), %eax # imm = 0xCCCCCCCD
+; X86-NEXT:    addl $429496729, %eax # imm = 0x19999999
+; X86-NEXT:    cmpl $858993459, %eax # imm = 0x33333333
+; X86-NEXT:    movl $42, %eax
+; X86-NEXT:    jb .LBB1_2
+; X86-NEXT:  # %bb.1:
+; X86-NEXT:    movl $-10, %eax
+; X86-NEXT:  .LBB1_2:
+; X86-NEXT:    retl
+;
+; X64-LABEL: test_optsize:
+; X64:       # %bb.0:
+; X64-NEXT:    imull $-858993459, %edi, %eax # imm = 0xCCCCCCCD
+; X64-NEXT:    addl $429496729, %eax # imm = 0x19999999
+; X64-NEXT:    cmpl $858993459, %eax # imm = 0x33333333
+; X64-NEXT:    movl $42, %ecx
+; X64-NEXT:    movl $-10, %eax
+; X64-NEXT:    cmovbl %ecx, %eax
+; X64-NEXT:    retq
+  %rem = srem i32 %X, 5
+  %cmp = icmp eq i32 %rem, 0
+  %ret = select i1 %cmp, i32 42, i32 -10
+  ret i32 %ret
+}
diff --git a/llvm/test/CodeGen/X86/srem-seteq-vec-nonsplat.ll b/llvm/test/CodeGen/X86/srem-seteq-vec-nonsplat.ll
new file mode 100644
index 000000000..b23921963
--- /dev/null
+++ b/llvm/test/CodeGen/X86/srem-seteq-vec-nonsplat.ll
@@ -0,0 +1,3370 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu -mattr=+sse2 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-SSE,CHECK-SSE2
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu -mattr=+sse4.1 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-SSE,CHECK-SSE41
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu -mattr=+avx < %s | FileCheck %s --check-prefixes=CHECK,CHECK-AVX,CHECK-AVX1
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu -mattr=+avx2 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-AVX,CHECK-AVX2
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512f,+avx512vl < %s | FileCheck %s --check-prefixes=CHECK,CHECK-AVX,CHECK-AVX512VL
+
+; Odd+Even divisors
+define <4 x i32> @test_srem_odd_even(<4 x i32> %X) nounwind {
+; CHECK-SSE2-LABEL: test_srem_odd_even:
+; CHECK-SSE2:       # %bb.0:
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [0,1,0,0]
+; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm2
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm4, %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [1717986919,2454267027,1374389535,1374389535]
+; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm1
+; CHECK-SSE2-NEXT:    pmuludq %xmm5, %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,3,2,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm5[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm4, %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; CHECK-SSE2-NEXT:    pxor %xmm1, %xmm1
+; CHECK-SSE2-NEXT:    pxor %xmm4, %xmm4
+; CHECK-SSE2-NEXT:    pcmpgtd %xmm0, %xmm4
+; CHECK-SSE2-NEXT:    pand %xmm5, %xmm4
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [0,4294967295,0,0]
+; CHECK-SSE2-NEXT:    pand %xmm0, %xmm5
+; CHECK-SSE2-NEXT:    paddd %xmm4, %xmm5
+; CHECK-SSE2-NEXT:    psubd %xmm5, %xmm2
+; CHECK-SSE2-NEXT:    paddd %xmm3, %xmm2
+; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm3
+; CHECK-SSE2-NEXT:    psrad $5, %xmm3
+; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm4
+; CHECK-SSE2-NEXT:    psrad $3, %xmm4
+; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm5
+; CHECK-SSE2-NEXT:    psrad $1, %xmm5
+; CHECK-SSE2-NEXT:    punpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm4[0]
+; CHECK-SSE2-NEXT:    punpckhqdq {{.*#+}} xmm4 = xmm4[1],xmm3[1]
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm5 = xmm5[0,3],xmm4[0,3]
+; CHECK-SSE2-NEXT:    psrld $31, %xmm2
+; CHECK-SSE2-NEXT:    paddd %xmm5, %xmm2
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [5,14,25,100]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm4, %xmm3
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; CHECK-SSE2-NEXT:    psubd %xmm2, %xmm0
+; CHECK-SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-SSE2-NEXT:    psrld $31, %xmm0
+; CHECK-SSE2-NEXT:    retq
+;
+; CHECK-SSE41-LABEL: test_srem_odd_even:
+; CHECK-SSE41:       # %bb.0:
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [1717986919,2454267027,1374389535,1374389535]
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; CHECK-SSE41-NEXT:    pmuldq %xmm2, %xmm3
+; CHECK-SSE41-NEXT:    pmuldq %xmm0, %xmm1
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5],xmm3[6,7]
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [0,1,0,0]
+; CHECK-SSE41-NEXT:    pmulld %xmm0, %xmm2
+; CHECK-SSE41-NEXT:    paddd %xmm1, %xmm2
+; CHECK-SSE41-NEXT:    movdqa %xmm2, %xmm1
+; CHECK-SSE41-NEXT:    psrad $5, %xmm1
+; CHECK-SSE41-NEXT:    movdqa %xmm2, %xmm3
+; CHECK-SSE41-NEXT:    psrad $3, %xmm3
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm3[0,1,2,3],xmm1[4,5,6,7]
+; CHECK-SSE41-NEXT:    movdqa %xmm2, %xmm4
+; CHECK-SSE41-NEXT:    psrad $1, %xmm4
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1,2,3],xmm3[4,5,6,7]
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1],xmm1[2,3],xmm4[4,5],xmm1[6,7]
+; CHECK-SSE41-NEXT:    psrld $31, %xmm2
+; CHECK-SSE41-NEXT:    paddd %xmm4, %xmm2
+; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm2
+; CHECK-SSE41-NEXT:    psubd %xmm2, %xmm0
+; CHECK-SSE41-NEXT:    pxor %xmm1, %xmm1
+; CHECK-SSE41-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-SSE41-NEXT:    psrld $31, %xmm0
+; CHECK-SSE41-NEXT:    retq
+;
+; CHECK-AVX1-LABEL: test_srem_odd_even:
+; CHECK-AVX1:       # %bb.0:
+; CHECK-AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [1717986919,2454267027,1374389535,1374389535]
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpmuldq %xmm2, %xmm3, %xmm2
+; CHECK-AVX1-NEXT:    vpmuldq %xmm1, %xmm0, %xmm1
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
+; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm2
+; CHECK-AVX1-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpsrad $5, %xmm1, %xmm2
+; CHECK-AVX1-NEXT:    vpsrad $3, %xmm1, %xmm3
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
+; CHECK-AVX1-NEXT:    vpsrad $1, %xmm1, %xmm4
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm4[0,1,2,3],xmm3[4,5,6,7]
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpaddd %xmm1, %xmm2, %xmm1
+; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    retq
+;
+; CHECK-AVX2-LABEL: test_srem_odd_even:
+; CHECK-AVX2:       # %bb.0:
+; CHECK-AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [1717986919,2454267027,1374389535,1374389535]
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpmuldq %xmm2, %xmm3, %xmm2
+; CHECK-AVX2-NEXT:    vpmuldq %xmm1, %xmm0, %xmm1
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
+; CHECK-AVX2-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm2
+; CHECK-AVX2-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpsrld $31, %xmm1, %xmm2
+; CHECK-AVX2-NEXT:    vpsravd {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    retq
+;
+; CHECK-AVX512VL-LABEL: test_srem_odd_even:
+; CHECK-AVX512VL:       # %bb.0:
+; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpaddd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vprorvd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
+; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    retq
+  %srem = srem <4 x i32> %X, <i32 5, i32 14, i32 25, i32 100>
+  %cmp = icmp eq <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+;==============================================================================;
+
+; One all-ones divisor in odd divisor
+define <4 x i32> @test_srem_odd_allones_eq(<4 x i32> %X) nounwind {
+; CHECK-SSE2-LABEL: test_srem_odd_allones_eq:
+; CHECK-SSE2:       # %bb.0:
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [3435973837,3435973837,3435973837,3435973837]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm0
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; CHECK-SSE2-NEXT:    paddd {{.*}}(%rip), %xmm0
+; CHECK-SSE2-NEXT:    pxor {{.*}}(%rip), %xmm0
+; CHECK-SSE2-NEXT:    pcmpgtd {{.*}}(%rip), %xmm0
+; CHECK-SSE2-NEXT:    pandn {{.*}}(%rip), %xmm0
+; CHECK-SSE2-NEXT:    retq
+;
+; CHECK-SSE41-LABEL: test_srem_odd_allones_eq:
+; CHECK-SSE41:       # %bb.0:
+; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm0
+; CHECK-SSE41-NEXT:    paddd {{.*}}(%rip), %xmm0
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [858993458,858993458,4294967295,858993458]
+; CHECK-SSE41-NEXT:    pminud %xmm0, %xmm1
+; CHECK-SSE41-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-SSE41-NEXT:    psrld $31, %xmm0
+; CHECK-SSE41-NEXT:    retq
+;
+; CHECK-AVX1-LABEL: test_srem_odd_allones_eq:
+; CHECK-AVX1:       # %bb.0:
+; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpaddd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
+; CHECK-AVX1-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    retq
+;
+; CHECK-AVX2-LABEL: test_srem_odd_allones_eq:
+; CHECK-AVX2:       # %bb.0:
+; CHECK-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [3435973837,3435973837,3435973837,3435973837]
+; CHECK-AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [429496729,429496729,429496729,429496729]
+; CHECK-AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
+; CHECK-AVX2-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    retq
+;
+; CHECK-AVX512VL-LABEL: test_srem_odd_allones_eq:
+; CHECK-AVX512VL:       # %bb.0:
+; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip){1to4}, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpaddd {{.*}}(%rip){1to4}, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
+; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    retq
+  %srem = srem <4 x i32> %X, <i32 5, i32 5, i32 4294967295, i32 5>
+  %cmp = icmp eq <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+define <4 x i32> @test_srem_odd_allones_ne(<4 x i32> %X) nounwind {
+; CHECK-SSE2-LABEL: test_srem_odd_allones_ne:
+; CHECK-SSE2:       # %bb.0:
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [3435973837,3435973837,3435973837,3435973837]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm0
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; CHECK-SSE2-NEXT:    paddd {{.*}}(%rip), %xmm0
+; CHECK-SSE2-NEXT:    pxor {{.*}}(%rip), %xmm0
+; CHECK-SSE2-NEXT:    pcmpgtd {{.*}}(%rip), %xmm0
+; CHECK-SSE2-NEXT:    psrld $31, %xmm0
+; CHECK-SSE2-NEXT:    retq
+;
+; CHECK-SSE41-LABEL: test_srem_odd_allones_ne:
+; CHECK-SSE41:       # %bb.0:
+; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm0
+; CHECK-SSE41-NEXT:    paddd {{.*}}(%rip), %xmm0
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [858993458,858993458,4294967295,858993458]
+; CHECK-SSE41-NEXT:    pminud %xmm0, %xmm1
+; CHECK-SSE41-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-SSE41-NEXT:    pandn {{.*}}(%rip), %xmm0
+; CHECK-SSE41-NEXT:    retq
+;
+; CHECK-AVX1-LABEL: test_srem_odd_allones_ne:
+; CHECK-AVX1:       # %bb.0:
+; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpaddd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
+; CHECK-AVX1-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpandn {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    retq
+;
+; CHECK-AVX2-LABEL: test_srem_odd_allones_ne:
+; CHECK-AVX2:       # %bb.0:
+; CHECK-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [3435973837,3435973837,3435973837,3435973837]
+; CHECK-AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [429496729,429496729,429496729,429496729]
+; CHECK-AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
+; CHECK-AVX2-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [1,1,1,1]
+; CHECK-AVX2-NEXT:    vpandn %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    retq
+;
+; CHECK-AVX512VL-LABEL: test_srem_odd_allones_ne:
+; CHECK-AVX512VL:       # %bb.0:
+; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip){1to4}, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpaddd {{.*}}(%rip){1to4}, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
+; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpandnd {{.*}}(%rip){1to4}, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    retq
+  %srem = srem <4 x i32> %X, <i32 5, i32 5, i32 4294967295, i32 5>
+  %cmp = icmp ne <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; One all-ones divisor in even divisor
+define <4 x i32> @test_srem_even_allones_eq(<4 x i32> %X) nounwind {
+; CHECK-SSE2-LABEL: test_srem_even_allones_eq:
+; CHECK-SSE2:       # %bb.0:
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [4294967295,4294967295,0,4294967295]
+; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm3
+; CHECK-SSE2-NEXT:    pand %xmm2, %xmm3
+; CHECK-SSE2-NEXT:    pxor %xmm1, %xmm1
+; CHECK-SSE2-NEXT:    pxor %xmm4, %xmm4
+; CHECK-SSE2-NEXT:    pcmpgtd %xmm0, %xmm4
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [2454267027,2454267027,0,2454267027]
+; CHECK-SSE2-NEXT:    pand %xmm5, %xmm4
+; CHECK-SSE2-NEXT:    paddd %xmm3, %xmm4
+; CHECK-SSE2-NEXT:    pmuludq %xmm0, %xmm5
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm5[1,3,2,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[1,1,3,3]
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm6 = [2454267027,2454267027,2454267027,2454267027]
+; CHECK-SSE2-NEXT:    pmuludq %xmm5, %xmm6
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,3,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm6[0],xmm3[1],xmm6[1]
+; CHECK-SSE2-NEXT:    psubd %xmm4, %xmm3
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm4 = <1,u,4294967295,u>
+; CHECK-SSE2-NEXT:    pmuludq %xmm0, %xmm4
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[0,2,2,3]
+; CHECK-SSE2-NEXT:    pmuludq {{.*}}(%rip), %xmm5
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
+; CHECK-SSE2-NEXT:    paddd %xmm3, %xmm4
+; CHECK-SSE2-NEXT:    movdqa %xmm4, %xmm3
+; CHECK-SSE2-NEXT:    psrad $3, %xmm3
+; CHECK-SSE2-NEXT:    movdqa %xmm4, %xmm5
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm5 = xmm5[2,0],xmm3[3,0]
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,1],xmm5[0,2]
+; CHECK-SSE2-NEXT:    psrld $31, %xmm4
+; CHECK-SSE2-NEXT:    pand %xmm2, %xmm4
+; CHECK-SSE2-NEXT:    paddd %xmm3, %xmm4
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq {{.*}}(%rip), %xmm4
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[0,2,2,3]
+; CHECK-SSE2-NEXT:    pmuludq {{.*}}(%rip), %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; CHECK-SSE2-NEXT:    psubd %xmm3, %xmm0
+; CHECK-SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-SSE2-NEXT:    psrld $31, %xmm0
+; CHECK-SSE2-NEXT:    retq
+;
+; CHECK-SSE41-LABEL: test_srem_even_allones_eq:
+; CHECK-SSE41:       # %bb.0:
+; CHECK-SSE41-NEXT:    movl $-1840700269, %eax # imm = 0x92492493
+; CHECK-SSE41-NEXT:    movd %eax, %xmm1
+; CHECK-SSE41-NEXT:    pmuldq %xmm0, %xmm1
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; CHECK-SSE41-NEXT:    pmuldq {{.*}}(%rip), %xmm2
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [1,1,4294967295,1]
+; CHECK-SSE41-NEXT:    pmulld %xmm0, %xmm1
+; CHECK-SSE41-NEXT:    paddd %xmm2, %xmm1
+; CHECK-SSE41-NEXT:    movdqa %xmm1, %xmm2
+; CHECK-SSE41-NEXT:    psrad $3, %xmm2
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm1[4,5],xmm2[6,7]
+; CHECK-SSE41-NEXT:    psrld $31, %xmm1
+; CHECK-SSE41-NEXT:    pxor %xmm3, %xmm3
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm3[4,5],xmm1[6,7]
+; CHECK-SSE41-NEXT:    paddd %xmm2, %xmm1
+; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm1
+; CHECK-SSE41-NEXT:    psubd %xmm1, %xmm0
+; CHECK-SSE41-NEXT:    pcmpeqd %xmm3, %xmm0
+; CHECK-SSE41-NEXT:    psrld $31, %xmm0
+; CHECK-SSE41-NEXT:    retq
+;
+; CHECK-AVX1-LABEL: test_srem_even_allones_eq:
+; CHECK-AVX1:       # %bb.0:
+; CHECK-AVX1-NEXT:    movl $-1840700269, %eax # imm = 0x92492493
+; CHECK-AVX1-NEXT:    vmovd %eax, %xmm1
+; CHECK-AVX1-NEXT:    vpmuldq %xmm1, %xmm0, %xmm1
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpmuldq {{.*}}(%rip), %xmm2, %xmm2
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
+; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm2
+; CHECK-AVX1-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpsrad $3, %xmm1, %xmm2
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm1[4,5],xmm2[6,7]
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm3[4,5],xmm1[6,7]
+; CHECK-AVX1-NEXT:    vpaddd %xmm1, %xmm2, %xmm1
+; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpcmpeqd %xmm3, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    retq
+;
+; CHECK-AVX2-LABEL: test_srem_even_allones_eq:
+; CHECK-AVX2:       # %bb.0:
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [2454267027,2454267027,2454267027,2454267027]
+; CHECK-AVX2-NEXT:    vpmuldq %xmm2, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    movl $-1840700269, %eax # imm = 0x92492493
+; CHECK-AVX2-NEXT:    vmovd %eax, %xmm2
+; CHECK-AVX2-NEXT:    vpmuldq %xmm2, %xmm0, %xmm2
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2],xmm1[3]
+; CHECK-AVX2-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm2
+; CHECK-AVX2-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpsrld $31, %xmm1, %xmm2
+; CHECK-AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm2 = xmm2[0,1],xmm3[2],xmm2[3]
+; CHECK-AVX2-NEXT:    vpsravd {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpcmpeqd %xmm3, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    retq
+;
+; CHECK-AVX512VL-LABEL: test_srem_even_allones_eq:
+; CHECK-AVX512VL:       # %bb.0:
+; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip){1to4}, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpaddd {{.*}}(%rip){1to4}, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vprord $1, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
+; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    retq
+  %srem = srem <4 x i32> %X, <i32 14, i32 14, i32 4294967295, i32 14>
+  %cmp = icmp eq <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+define <4 x i32> @test_srem_even_allones_ne(<4 x i32> %X) nounwind {
+; CHECK-SSE2-LABEL: test_srem_even_allones_ne:
+; CHECK-SSE2:       # %bb.0:
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [4294967295,4294967295,0,4294967295]
+; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm3
+; CHECK-SSE2-NEXT:    pand %xmm2, %xmm3
+; CHECK-SSE2-NEXT:    pxor %xmm1, %xmm1
+; CHECK-SSE2-NEXT:    pxor %xmm4, %xmm4
+; CHECK-SSE2-NEXT:    pcmpgtd %xmm0, %xmm4
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [2454267027,2454267027,0,2454267027]
+; CHECK-SSE2-NEXT:    pand %xmm5, %xmm4
+; CHECK-SSE2-NEXT:    paddd %xmm3, %xmm4
+; CHECK-SSE2-NEXT:    pmuludq %xmm0, %xmm5
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,3,2,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm0[1,1,3,3]
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [2454267027,2454267027,2454267027,2454267027]
+; CHECK-SSE2-NEXT:    pmuludq %xmm6, %xmm3
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,3,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]
+; CHECK-SSE2-NEXT:    psubd %xmm4, %xmm5
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [1,1,1,1]
+; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm6
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[0,2,2,3]
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm6 = <1,u,4294967295,u>
+; CHECK-SSE2-NEXT:    pmuludq %xmm0, %xmm6
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm6 = xmm6[0],xmm4[0],xmm6[1],xmm4[1]
+; CHECK-SSE2-NEXT:    paddd %xmm5, %xmm6
+; CHECK-SSE2-NEXT:    movdqa %xmm6, %xmm4
+; CHECK-SSE2-NEXT:    psrad $3, %xmm4
+; CHECK-SSE2-NEXT:    movdqa %xmm6, %xmm5
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm5 = xmm5[2,0],xmm4[3,0]
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[0,1],xmm5[0,2]
+; CHECK-SSE2-NEXT:    psrld $31, %xmm6
+; CHECK-SSE2-NEXT:    pand %xmm2, %xmm6
+; CHECK-SSE2-NEXT:    paddd %xmm4, %xmm6
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq {{.*}}(%rip), %xmm6
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[0,2,2,3]
+; CHECK-SSE2-NEXT:    pmuludq {{.*}}(%rip), %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
+; CHECK-SSE2-NEXT:    psubd %xmm4, %xmm0
+; CHECK-SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-SSE2-NEXT:    pandn %xmm3, %xmm0
+; CHECK-SSE2-NEXT:    retq
+;
+; CHECK-SSE41-LABEL: test_srem_even_allones_ne:
+; CHECK-SSE41:       # %bb.0:
+; CHECK-SSE41-NEXT:    movl $-1840700269, %eax # imm = 0x92492493
+; CHECK-SSE41-NEXT:    movd %eax, %xmm1
+; CHECK-SSE41-NEXT:    pmuldq %xmm0, %xmm1
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; CHECK-SSE41-NEXT:    pmuldq {{.*}}(%rip), %xmm2
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [1,1,4294967295,1]
+; CHECK-SSE41-NEXT:    pmulld %xmm0, %xmm1
+; CHECK-SSE41-NEXT:    paddd %xmm2, %xmm1
+; CHECK-SSE41-NEXT:    movdqa %xmm1, %xmm2
+; CHECK-SSE41-NEXT:    psrad $3, %xmm2
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm1[4,5],xmm2[6,7]
+; CHECK-SSE41-NEXT:    psrld $31, %xmm1
+; CHECK-SSE41-NEXT:    pxor %xmm3, %xmm3
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm3[4,5],xmm1[6,7]
+; CHECK-SSE41-NEXT:    paddd %xmm2, %xmm1
+; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm1
+; CHECK-SSE41-NEXT:    psubd %xmm1, %xmm0
+; CHECK-SSE41-NEXT:    pcmpeqd %xmm3, %xmm0
+; CHECK-SSE41-NEXT:    pandn {{.*}}(%rip), %xmm0
+; CHECK-SSE41-NEXT:    retq
+;
+; CHECK-AVX1-LABEL: test_srem_even_allones_ne:
+; CHECK-AVX1:       # %bb.0:
+; CHECK-AVX1-NEXT:    movl $-1840700269, %eax # imm = 0x92492493
+; CHECK-AVX1-NEXT:    vmovd %eax, %xmm1
+; CHECK-AVX1-NEXT:    vpmuldq %xmm1, %xmm0, %xmm1
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpmuldq {{.*}}(%rip), %xmm2, %xmm2
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
+; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm2
+; CHECK-AVX1-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpsrad $3, %xmm1, %xmm2
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm1[4,5],xmm2[6,7]
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm3[4,5],xmm1[6,7]
+; CHECK-AVX1-NEXT:    vpaddd %xmm1, %xmm2, %xmm1
+; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpcmpeqd %xmm3, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpandn {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    retq
+;
+; CHECK-AVX2-LABEL: test_srem_even_allones_ne:
+; CHECK-AVX2:       # %bb.0:
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [2454267027,2454267027,2454267027,2454267027]
+; CHECK-AVX2-NEXT:    vpmuldq %xmm2, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    movl $-1840700269, %eax # imm = 0x92492493
+; CHECK-AVX2-NEXT:    vmovd %eax, %xmm2
+; CHECK-AVX2-NEXT:    vpmuldq %xmm2, %xmm0, %xmm2
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2],xmm1[3]
+; CHECK-AVX2-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm2
+; CHECK-AVX2-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpsrld $31, %xmm1, %xmm2
+; CHECK-AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm2 = xmm2[0,1],xmm3[2],xmm2[3]
+; CHECK-AVX2-NEXT:    vpsravd {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpcmpeqd %xmm3, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [1,1,1,1]
+; CHECK-AVX2-NEXT:    vpandn %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    retq
+;
+; CHECK-AVX512VL-LABEL: test_srem_even_allones_ne:
+; CHECK-AVX512VL:       # %bb.0:
+; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip){1to4}, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpaddd {{.*}}(%rip){1to4}, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vprord $1, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
+; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpandnd {{.*}}(%rip){1to4}, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    retq
+  %srem = srem <4 x i32> %X, <i32 14, i32 14, i32 4294967295, i32 14>
+  %cmp = icmp ne <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; One all-ones divisor in odd+even divisor
+define <4 x i32> @test_srem_odd_even_allones_eq(<4 x i32> %X) nounwind {
+; CHECK-SSE2-LABEL: test_srem_odd_even_allones_eq:
+; CHECK-SSE2:       # %bb.0:
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [0,1,4294967295,0]
+; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm2
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm4, %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [1717986919,2454267027,0,1374389535]
+; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm1
+; CHECK-SSE2-NEXT:    pmuludq %xmm5, %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,3,2,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm5[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm4, %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; CHECK-SSE2-NEXT:    pxor %xmm1, %xmm1
+; CHECK-SSE2-NEXT:    pxor %xmm4, %xmm4
+; CHECK-SSE2-NEXT:    pcmpgtd %xmm0, %xmm4
+; CHECK-SSE2-NEXT:    pand %xmm5, %xmm4
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [0,4294967295,0,0]
+; CHECK-SSE2-NEXT:    pand %xmm0, %xmm5
+; CHECK-SSE2-NEXT:    paddd %xmm4, %xmm5
+; CHECK-SSE2-NEXT:    psubd %xmm5, %xmm2
+; CHECK-SSE2-NEXT:    paddd %xmm3, %xmm2
+; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm3
+; CHECK-SSE2-NEXT:    psrad $5, %xmm3
+; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm4
+; CHECK-SSE2-NEXT:    punpckhqdq {{.*#+}} xmm4 = xmm4[1],xmm3[1]
+; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm3
+; CHECK-SSE2-NEXT:    psrad $3, %xmm3
+; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm5
+; CHECK-SSE2-NEXT:    psrad $1, %xmm5
+; CHECK-SSE2-NEXT:    punpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm3[0]
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm5 = xmm5[0,3],xmm4[0,3]
+; CHECK-SSE2-NEXT:    psrld $31, %xmm2
+; CHECK-SSE2-NEXT:    pand {{.*}}(%rip), %xmm2
+; CHECK-SSE2-NEXT:    paddd %xmm5, %xmm2
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [5,14,4294967295,100]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm4, %xmm3
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; CHECK-SSE2-NEXT:    psubd %xmm2, %xmm0
+; CHECK-SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-SSE2-NEXT:    psrld $31, %xmm0
+; CHECK-SSE2-NEXT:    retq
+;
+; CHECK-SSE41-LABEL: test_srem_odd_even_allones_eq:
+; CHECK-SSE41:       # %bb.0:
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [1717986919,2454267027,0,1374389535]
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; CHECK-SSE41-NEXT:    pmuldq %xmm2, %xmm3
+; CHECK-SSE41-NEXT:    pmuldq %xmm0, %xmm1
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5],xmm3[6,7]
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [0,1,4294967295,0]
+; CHECK-SSE41-NEXT:    pmulld %xmm0, %xmm2
+; CHECK-SSE41-NEXT:    paddd %xmm1, %xmm2
+; CHECK-SSE41-NEXT:    movdqa %xmm2, %xmm1
+; CHECK-SSE41-NEXT:    psrad $5, %xmm1
+; CHECK-SSE41-NEXT:    movdqa %xmm2, %xmm3
+; CHECK-SSE41-NEXT:    psrad $3, %xmm3
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm1[4,5,6,7]
+; CHECK-SSE41-NEXT:    movdqa %xmm2, %xmm1
+; CHECK-SSE41-NEXT:    psrad $1, %xmm1
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm2[4,5,6,7]
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5],xmm3[6,7]
+; CHECK-SSE41-NEXT:    psrld $31, %xmm2
+; CHECK-SSE41-NEXT:    pxor %xmm3, %xmm3
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm3[4,5],xmm2[6,7]
+; CHECK-SSE41-NEXT:    paddd %xmm1, %xmm2
+; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm2
+; CHECK-SSE41-NEXT:    psubd %xmm2, %xmm0
+; CHECK-SSE41-NEXT:    pcmpeqd %xmm3, %xmm0
+; CHECK-SSE41-NEXT:    psrld $31, %xmm0
+; CHECK-SSE41-NEXT:    retq
+;
+; CHECK-AVX1-LABEL: test_srem_odd_even_allones_eq:
+; CHECK-AVX1:       # %bb.0:
+; CHECK-AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [1717986919,2454267027,0,1374389535]
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpmuldq %xmm2, %xmm3, %xmm2
+; CHECK-AVX1-NEXT:    vpmuldq %xmm1, %xmm0, %xmm1
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
+; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm2
+; CHECK-AVX1-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpsrad $5, %xmm1, %xmm2
+; CHECK-AVX1-NEXT:    vpsrad $3, %xmm1, %xmm3
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
+; CHECK-AVX1-NEXT:    vpsrad $1, %xmm1, %xmm3
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm1[4,5,6,7]
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm3[4,5],xmm1[6,7]
+; CHECK-AVX1-NEXT:    vpaddd %xmm1, %xmm2, %xmm1
+; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpcmpeqd %xmm3, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    retq
+;
+; CHECK-AVX2-LABEL: test_srem_odd_even_allones_eq:
+; CHECK-AVX2:       # %bb.0:
+; CHECK-AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [1717986919,2454267027,0,1374389535]
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpmuldq %xmm2, %xmm3, %xmm2
+; CHECK-AVX2-NEXT:    vpmuldq %xmm1, %xmm0, %xmm1
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
+; CHECK-AVX2-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm2
+; CHECK-AVX2-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpsrld $31, %xmm1, %xmm2
+; CHECK-AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm2 = xmm2[0,1],xmm3[2],xmm2[3]
+; CHECK-AVX2-NEXT:    vpsravd {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpcmpeqd %xmm3, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    retq
+;
+; CHECK-AVX512VL-LABEL: test_srem_odd_even_allones_eq:
+; CHECK-AVX512VL:       # %bb.0:
+; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpaddd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vprorvd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
+; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    retq
+  %srem = srem <4 x i32> %X, <i32 5, i32 14, i32 4294967295, i32 100>
+  %cmp = icmp eq <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+define <4 x i32> @test_srem_odd_even_allones_ne(<4 x i32> %X) nounwind {
+; CHECK-SSE2-LABEL: test_srem_odd_even_allones_ne:
+; CHECK-SSE2:       # %bb.0:
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [0,1,4294967295,0]
+; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm2
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm4, %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [1717986919,2454267027,0,1374389535]
+; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm1
+; CHECK-SSE2-NEXT:    pmuludq %xmm5, %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,3,2,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm5[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm4, %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; CHECK-SSE2-NEXT:    pxor %xmm1, %xmm1
+; CHECK-SSE2-NEXT:    pxor %xmm4, %xmm4
+; CHECK-SSE2-NEXT:    pcmpgtd %xmm0, %xmm4
+; CHECK-SSE2-NEXT:    pand %xmm5, %xmm4
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [0,4294967295,0,0]
+; CHECK-SSE2-NEXT:    pand %xmm0, %xmm5
+; CHECK-SSE2-NEXT:    paddd %xmm4, %xmm5
+; CHECK-SSE2-NEXT:    psubd %xmm5, %xmm2
+; CHECK-SSE2-NEXT:    paddd %xmm3, %xmm2
+; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm3
+; CHECK-SSE2-NEXT:    psrad $5, %xmm3
+; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm4
+; CHECK-SSE2-NEXT:    punpckhqdq {{.*#+}} xmm4 = xmm4[1],xmm3[1]
+; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm3
+; CHECK-SSE2-NEXT:    psrad $3, %xmm3
+; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm5
+; CHECK-SSE2-NEXT:    psrad $1, %xmm5
+; CHECK-SSE2-NEXT:    punpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm3[0]
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm5 = xmm5[0,3],xmm4[0,3]
+; CHECK-SSE2-NEXT:    psrld $31, %xmm2
+; CHECK-SSE2-NEXT:    pand {{.*}}(%rip), %xmm2
+; CHECK-SSE2-NEXT:    paddd %xmm5, %xmm2
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [5,14,4294967295,100]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm4, %xmm3
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; CHECK-SSE2-NEXT:    psubd %xmm2, %xmm0
+; CHECK-SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-SSE2-NEXT:    pandn {{.*}}(%rip), %xmm0
+; CHECK-SSE2-NEXT:    retq
+;
+; CHECK-SSE41-LABEL: test_srem_odd_even_allones_ne:
+; CHECK-SSE41:       # %bb.0:
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [1717986919,2454267027,0,1374389535]
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; CHECK-SSE41-NEXT:    pmuldq %xmm2, %xmm3
+; CHECK-SSE41-NEXT:    pmuldq %xmm0, %xmm1
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5],xmm3[6,7]
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [0,1,4294967295,0]
+; CHECK-SSE41-NEXT:    pmulld %xmm0, %xmm2
+; CHECK-SSE41-NEXT:    paddd %xmm1, %xmm2
+; CHECK-SSE41-NEXT:    movdqa %xmm2, %xmm1
+; CHECK-SSE41-NEXT:    psrad $5, %xmm1
+; CHECK-SSE41-NEXT:    movdqa %xmm2, %xmm3
+; CHECK-SSE41-NEXT:    psrad $3, %xmm3
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm1[4,5,6,7]
+; CHECK-SSE41-NEXT:    movdqa %xmm2, %xmm1
+; CHECK-SSE41-NEXT:    psrad $1, %xmm1
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm2[4,5,6,7]
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5],xmm3[6,7]
+; CHECK-SSE41-NEXT:    psrld $31, %xmm2
+; CHECK-SSE41-NEXT:    pxor %xmm3, %xmm3
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm3[4,5],xmm2[6,7]
+; CHECK-SSE41-NEXT:    paddd %xmm1, %xmm2
+; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm2
+; CHECK-SSE41-NEXT:    psubd %xmm2, %xmm0
+; CHECK-SSE41-NEXT:    pcmpeqd %xmm3, %xmm0
+; CHECK-SSE41-NEXT:    pandn {{.*}}(%rip), %xmm0
+; CHECK-SSE41-NEXT:    retq
+;
+; CHECK-AVX1-LABEL: test_srem_odd_even_allones_ne:
+; CHECK-AVX1:       # %bb.0:
+; CHECK-AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [1717986919,2454267027,0,1374389535]
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpmuldq %xmm2, %xmm3, %xmm2
+; CHECK-AVX1-NEXT:    vpmuldq %xmm1, %xmm0, %xmm1
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
+; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm2
+; CHECK-AVX1-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpsrad $5, %xmm1, %xmm2
+; CHECK-AVX1-NEXT:    vpsrad $3, %xmm1, %xmm3
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
+; CHECK-AVX1-NEXT:    vpsrad $1, %xmm1, %xmm3
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm1[4,5,6,7]
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm3[4,5],xmm1[6,7]
+; CHECK-AVX1-NEXT:    vpaddd %xmm1, %xmm2, %xmm1
+; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpcmpeqd %xmm3, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpandn {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    retq
+;
+; CHECK-AVX2-LABEL: test_srem_odd_even_allones_ne:
+; CHECK-AVX2:       # %bb.0:
+; CHECK-AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [1717986919,2454267027,0,1374389535]
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpmuldq %xmm2, %xmm3, %xmm2
+; CHECK-AVX2-NEXT:    vpmuldq %xmm1, %xmm0, %xmm1
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
+; CHECK-AVX2-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm2
+; CHECK-AVX2-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpsrld $31, %xmm1, %xmm2
+; CHECK-AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm2 = xmm2[0,1],xmm3[2],xmm2[3]
+; CHECK-AVX2-NEXT:    vpsravd {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpcmpeqd %xmm3, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [1,1,1,1]
+; CHECK-AVX2-NEXT:    vpandn %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    retq
+;
+; CHECK-AVX512VL-LABEL: test_srem_odd_even_allones_ne:
+; CHECK-AVX512VL:       # %bb.0:
+; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpaddd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vprorvd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
+; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpandnd {{.*}}(%rip){1to4}, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    retq
+  %srem = srem <4 x i32> %X, <i32 5, i32 14, i32 4294967295, i32 100>
+  %cmp = icmp ne <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+;------------------------------------------------------------------------------;
+
+; One power-of-two divisor in odd divisor
+define <4 x i32> @test_srem_odd_poweroftwo(<4 x i32> %X) nounwind {
+; CHECK-SSE2-LABEL: test_srem_odd_poweroftwo:
+; CHECK-SSE2:       # %bb.0:
+; CHECK-SSE2-NEXT:    pxor %xmm1, %xmm1
+; CHECK-SSE2-NEXT:    pxor %xmm2, %xmm2
+; CHECK-SSE2-NEXT:    pcmpgtd %xmm0, %xmm2
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [1717986919,1717986919,2147483649,1717986919]
+; CHECK-SSE2-NEXT:    pand %xmm3, %xmm2
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [0,0,4294967295,0]
+; CHECK-SSE2-NEXT:    pand %xmm0, %xmm4
+; CHECK-SSE2-NEXT:    paddd %xmm2, %xmm4
+; CHECK-SSE2-NEXT:    pmuludq %xmm0, %xmm3
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,3,2,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq {{.*}}(%rip), %xmm3
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,3,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; CHECK-SSE2-NEXT:    psubd %xmm4, %xmm2
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm3 = <0,u,1,u>
+; CHECK-SSE2-NEXT:    pmuludq %xmm0, %xmm3
+; CHECK-SSE2-NEXT:    pand {{.*}}(%rip), %xmm3
+; CHECK-SSE2-NEXT:    paddd %xmm2, %xmm3
+; CHECK-SSE2-NEXT:    movdqa %xmm3, %xmm2
+; CHECK-SSE2-NEXT:    psrad $1, %xmm2
+; CHECK-SSE2-NEXT:    movdqa %xmm3, %xmm4
+; CHECK-SSE2-NEXT:    psrad $3, %xmm4
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[2,0],xmm2[3,0]
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,1],xmm4[0,2]
+; CHECK-SSE2-NEXT:    psrld $31, %xmm3
+; CHECK-SSE2-NEXT:    paddd %xmm2, %xmm3
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq {{.*}}(%rip), %xmm3
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
+; CHECK-SSE2-NEXT:    pmuludq {{.*}}(%rip), %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; CHECK-SSE2-NEXT:    psubd %xmm3, %xmm0
+; CHECK-SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-SSE2-NEXT:    psrld $31, %xmm0
+; CHECK-SSE2-NEXT:    retq
+;
+; CHECK-SSE41-LABEL: test_srem_odd_poweroftwo:
+; CHECK-SSE41:       # %bb.0:
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; CHECK-SSE41-NEXT:    pmuldq {{.*}}(%rip), %xmm1
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm2 = <1717986919,u,2147483649,u>
+; CHECK-SSE41-NEXT:    pmuldq %xmm0, %xmm2
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [0,0,1,0]
+; CHECK-SSE41-NEXT:    pmulld %xmm0, %xmm1
+; CHECK-SSE41-NEXT:    paddd %xmm2, %xmm1
+; CHECK-SSE41-NEXT:    movdqa %xmm1, %xmm2
+; CHECK-SSE41-NEXT:    psrad $3, %xmm2
+; CHECK-SSE41-NEXT:    movdqa %xmm1, %xmm3
+; CHECK-SSE41-NEXT:    psrad $1, %xmm3
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm2[4,5],xmm3[6,7]
+; CHECK-SSE41-NEXT:    psrld $31, %xmm1
+; CHECK-SSE41-NEXT:    paddd %xmm3, %xmm1
+; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm1
+; CHECK-SSE41-NEXT:    psubd %xmm1, %xmm0
+; CHECK-SSE41-NEXT:    pxor %xmm1, %xmm1
+; CHECK-SSE41-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-SSE41-NEXT:    psrld $31, %xmm0
+; CHECK-SSE41-NEXT:    retq
+;
+; CHECK-AVX1-LABEL: test_srem_odd_poweroftwo:
+; CHECK-AVX1:       # %bb.0:
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpmuldq {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpmuldq {{.*}}(%rip), %xmm0, %xmm2
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
+; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm2
+; CHECK-AVX1-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpsrad $3, %xmm1, %xmm2
+; CHECK-AVX1-NEXT:    vpsrad $1, %xmm1, %xmm3
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5],xmm3[6,7]
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpaddd %xmm1, %xmm2, %xmm1
+; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    retq
+;
+; CHECK-AVX2-LABEL: test_srem_odd_poweroftwo:
+; CHECK-AVX2:       # %bb.0:
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [1717986919,1717986919,1717986919,1717986919]
+; CHECK-AVX2-NEXT:    vpmuldq %xmm2, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpmuldq {{.*}}(%rip), %xmm0, %xmm2
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2],xmm1[3]
+; CHECK-AVX2-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm2
+; CHECK-AVX2-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpsrld $31, %xmm1, %xmm2
+; CHECK-AVX2-NEXT:    vpsravd {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    retq
+;
+; CHECK-AVX512VL-LABEL: test_srem_odd_poweroftwo:
+; CHECK-AVX512VL:       # %bb.0:
+; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpaddd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vprorvd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
+; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    retq
+  %srem = srem <4 x i32> %X, <i32 5, i32 5, i32 16, i32 5>
+  %cmp = icmp eq <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; One power-of-two divisor in even divisor
+define <4 x i32> @test_srem_even_poweroftwo(<4 x i32> %X) nounwind {
+; CHECK-SSE2-LABEL: test_srem_even_poweroftwo:
+; CHECK-SSE2:       # %bb.0:
+; CHECK-SSE2-NEXT:    pxor %xmm1, %xmm1
+; CHECK-SSE2-NEXT:    pxor %xmm2, %xmm2
+; CHECK-SSE2-NEXT:    pcmpgtd %xmm0, %xmm2
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [2454267027,2454267027,2147483649,2454267027]
+; CHECK-SSE2-NEXT:    pand %xmm3, %xmm2
+; CHECK-SSE2-NEXT:    paddd %xmm0, %xmm2
+; CHECK-SSE2-NEXT:    pmuludq %xmm0, %xmm3
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,3,2,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq {{.*}}(%rip), %xmm4
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,3,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
+; CHECK-SSE2-NEXT:    psubd %xmm2, %xmm3
+; CHECK-SSE2-NEXT:    paddd %xmm0, %xmm3
+; CHECK-SSE2-NEXT:    movdqa %xmm3, %xmm2
+; CHECK-SSE2-NEXT:    psrld $31, %xmm2
+; CHECK-SSE2-NEXT:    psrad $3, %xmm3
+; CHECK-SSE2-NEXT:    paddd %xmm2, %xmm3
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq {{.*}}(%rip), %xmm3
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
+; CHECK-SSE2-NEXT:    pmuludq {{.*}}(%rip), %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; CHECK-SSE2-NEXT:    psubd %xmm3, %xmm0
+; CHECK-SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-SSE2-NEXT:    psrld $31, %xmm0
+; CHECK-SSE2-NEXT:    retq
+;
+; CHECK-SSE41-LABEL: test_srem_even_poweroftwo:
+; CHECK-SSE41:       # %bb.0:
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; CHECK-SSE41-NEXT:    pmuldq {{.*}}(%rip), %xmm1
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm2 = <2454267027,u,2147483649,u>
+; CHECK-SSE41-NEXT:    pmuldq %xmm0, %xmm2
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
+; CHECK-SSE41-NEXT:    paddd %xmm0, %xmm2
+; CHECK-SSE41-NEXT:    movdqa %xmm2, %xmm1
+; CHECK-SSE41-NEXT:    psrld $31, %xmm1
+; CHECK-SSE41-NEXT:    psrad $3, %xmm2
+; CHECK-SSE41-NEXT:    paddd %xmm1, %xmm2
+; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm2
+; CHECK-SSE41-NEXT:    psubd %xmm2, %xmm0
+; CHECK-SSE41-NEXT:    pxor %xmm1, %xmm1
+; CHECK-SSE41-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-SSE41-NEXT:    psrld $31, %xmm0
+; CHECK-SSE41-NEXT:    retq
+;
+; CHECK-AVX1-LABEL: test_srem_even_poweroftwo:
+; CHECK-AVX1:       # %bb.0:
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpmuldq {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpmuldq {{.*}}(%rip), %xmm0, %xmm2
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
+; CHECK-AVX1-NEXT:    vpaddd %xmm0, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm1, %xmm2
+; CHECK-AVX1-NEXT:    vpsrad $3, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    retq
+;
+; CHECK-AVX2-LABEL: test_srem_even_poweroftwo:
+; CHECK-AVX2:       # %bb.0:
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [2454267027,2454267027,2454267027,2454267027]
+; CHECK-AVX2-NEXT:    vpmuldq %xmm2, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpmuldq {{.*}}(%rip), %xmm0, %xmm2
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2],xmm1[3]
+; CHECK-AVX2-NEXT:    vpaddd %xmm0, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpsrld $31, %xmm1, %xmm2
+; CHECK-AVX2-NEXT:    vpsrad $3, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    retq
+;
+; CHECK-AVX512VL-LABEL: test_srem_even_poweroftwo:
+; CHECK-AVX512VL:       # %bb.0:
+; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpaddd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vprorvd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
+; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    retq
+  %srem = srem <4 x i32> %X, <i32 14, i32 14, i32 16, i32 14>
+  %cmp = icmp eq <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; One power-of-two divisor in odd+even divisor
+define <4 x i32> @test_srem_odd_even_poweroftwo(<4 x i32> %X) nounwind {
+; CHECK-SSE2-LABEL: test_srem_odd_even_poweroftwo:
+; CHECK-SSE2:       # %bb.0:
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [0,1,1,0]
+; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm2
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,2,3,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm4, %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [1717986919,2454267027,2147483649,1374389535]
+; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm1
+; CHECK-SSE2-NEXT:    pmuludq %xmm5, %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,3,2,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm5[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm4, %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; CHECK-SSE2-NEXT:    pxor %xmm1, %xmm1
+; CHECK-SSE2-NEXT:    pxor %xmm4, %xmm4
+; CHECK-SSE2-NEXT:    pcmpgtd %xmm0, %xmm4
+; CHECK-SSE2-NEXT:    pand %xmm5, %xmm4
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [0,4294967295,4294967295,0]
+; CHECK-SSE2-NEXT:    pand %xmm0, %xmm5
+; CHECK-SSE2-NEXT:    paddd %xmm4, %xmm5
+; CHECK-SSE2-NEXT:    psubd %xmm5, %xmm2
+; CHECK-SSE2-NEXT:    paddd %xmm3, %xmm2
+; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm3
+; CHECK-SSE2-NEXT:    psrad $5, %xmm3
+; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm4
+; CHECK-SSE2-NEXT:    psrad $3, %xmm4
+; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm5
+; CHECK-SSE2-NEXT:    psrad $1, %xmm5
+; CHECK-SSE2-NEXT:    punpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm4[0]
+; CHECK-SSE2-NEXT:    punpckhqdq {{.*#+}} xmm4 = xmm4[1],xmm3[1]
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm5 = xmm5[0,3],xmm4[0,3]
+; CHECK-SSE2-NEXT:    psrld $31, %xmm2
+; CHECK-SSE2-NEXT:    paddd %xmm5, %xmm2
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [5,14,16,100]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm4, %xmm3
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; CHECK-SSE2-NEXT:    psubd %xmm2, %xmm0
+; CHECK-SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-SSE2-NEXT:    psrld $31, %xmm0
+; CHECK-SSE2-NEXT:    retq
+;
+; CHECK-SSE41-LABEL: test_srem_odd_even_poweroftwo:
+; CHECK-SSE41:       # %bb.0:
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [1717986919,2454267027,2147483649,1374389535]
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; CHECK-SSE41-NEXT:    pmuldq %xmm2, %xmm3
+; CHECK-SSE41-NEXT:    pmuldq %xmm0, %xmm1
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5],xmm3[6,7]
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [0,1,1,0]
+; CHECK-SSE41-NEXT:    pmulld %xmm0, %xmm2
+; CHECK-SSE41-NEXT:    paddd %xmm1, %xmm2
+; CHECK-SSE41-NEXT:    movdqa %xmm2, %xmm1
+; CHECK-SSE41-NEXT:    psrad $5, %xmm1
+; CHECK-SSE41-NEXT:    movdqa %xmm2, %xmm3
+; CHECK-SSE41-NEXT:    psrad $3, %xmm3
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm3[0,1,2,3],xmm1[4,5,6,7]
+; CHECK-SSE41-NEXT:    movdqa %xmm2, %xmm4
+; CHECK-SSE41-NEXT:    psrad $1, %xmm4
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1,2,3],xmm3[4,5,6,7]
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1],xmm1[2,3],xmm4[4,5],xmm1[6,7]
+; CHECK-SSE41-NEXT:    psrld $31, %xmm2
+; CHECK-SSE41-NEXT:    paddd %xmm4, %xmm2
+; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm2
+; CHECK-SSE41-NEXT:    psubd %xmm2, %xmm0
+; CHECK-SSE41-NEXT:    pxor %xmm1, %xmm1
+; CHECK-SSE41-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-SSE41-NEXT:    psrld $31, %xmm0
+; CHECK-SSE41-NEXT:    retq
+;
+; CHECK-AVX1-LABEL: test_srem_odd_even_poweroftwo:
+; CHECK-AVX1:       # %bb.0:
+; CHECK-AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [1717986919,2454267027,2147483649,1374389535]
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpmuldq %xmm2, %xmm3, %xmm2
+; CHECK-AVX1-NEXT:    vpmuldq %xmm1, %xmm0, %xmm1
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
+; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm2
+; CHECK-AVX1-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpsrad $5, %xmm1, %xmm2
+; CHECK-AVX1-NEXT:    vpsrad $3, %xmm1, %xmm3
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
+; CHECK-AVX1-NEXT:    vpsrad $1, %xmm1, %xmm4
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm4[0,1,2,3],xmm3[4,5,6,7]
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpaddd %xmm1, %xmm2, %xmm1
+; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    retq
+;
+; CHECK-AVX2-LABEL: test_srem_odd_even_poweroftwo:
+; CHECK-AVX2:       # %bb.0:
+; CHECK-AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [1717986919,2454267027,2147483649,1374389535]
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpmuldq %xmm2, %xmm3, %xmm2
+; CHECK-AVX2-NEXT:    vpmuldq %xmm1, %xmm0, %xmm1
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
+; CHECK-AVX2-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm2
+; CHECK-AVX2-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpsrld $31, %xmm1, %xmm2
+; CHECK-AVX2-NEXT:    vpsravd {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    retq
+;
+; CHECK-AVX512VL-LABEL: test_srem_odd_even_poweroftwo:
+; CHECK-AVX512VL:       # %bb.0:
+; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpaddd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vprorvd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
+; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    retq
+  %srem = srem <4 x i32> %X, <i32 5, i32 14, i32 16, i32 100>
+  %cmp = icmp eq <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+;------------------------------------------------------------------------------;
+
+; One one divisor in odd divisor
+define <4 x i32> @test_srem_odd_one(<4 x i32> %X) nounwind {
+; CHECK-SSE2-LABEL: test_srem_odd_one:
+; CHECK-SSE2:       # %bb.0:
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [3435973837,3435973837,3435973837,3435973837]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm0
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; CHECK-SSE2-NEXT:    paddd {{.*}}(%rip), %xmm0
+; CHECK-SSE2-NEXT:    pxor {{.*}}(%rip), %xmm0
+; CHECK-SSE2-NEXT:    pcmpgtd {{.*}}(%rip), %xmm0
+; CHECK-SSE2-NEXT:    pandn {{.*}}(%rip), %xmm0
+; CHECK-SSE2-NEXT:    retq
+;
+; CHECK-SSE41-LABEL: test_srem_odd_one:
+; CHECK-SSE41:       # %bb.0:
+; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm0
+; CHECK-SSE41-NEXT:    paddd {{.*}}(%rip), %xmm0
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [858993458,858993458,4294967295,858993458]
+; CHECK-SSE41-NEXT:    pminud %xmm0, %xmm1
+; CHECK-SSE41-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-SSE41-NEXT:    psrld $31, %xmm0
+; CHECK-SSE41-NEXT:    retq
+;
+; CHECK-AVX1-LABEL: test_srem_odd_one:
+; CHECK-AVX1:       # %bb.0:
+; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpaddd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
+; CHECK-AVX1-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    retq
+;
+; CHECK-AVX2-LABEL: test_srem_odd_one:
+; CHECK-AVX2:       # %bb.0:
+; CHECK-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [3435973837,3435973837,3435973837,3435973837]
+; CHECK-AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [429496729,429496729,429496729,429496729]
+; CHECK-AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
+; CHECK-AVX2-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    retq
+;
+; CHECK-AVX512VL-LABEL: test_srem_odd_one:
+; CHECK-AVX512VL:       # %bb.0:
+; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip){1to4}, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpaddd {{.*}}(%rip){1to4}, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
+; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    retq
+  %srem = srem <4 x i32> %X, <i32 5, i32 5, i32 1, i32 5>
+  %cmp = icmp eq <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; One one divisor in even divisor
+define <4 x i32> @test_srem_even_one(<4 x i32> %X) nounwind {
+; CHECK-SSE2-LABEL: test_srem_even_one:
+; CHECK-SSE2:       # %bb.0:
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [4294967295,4294967295,0,4294967295]
+; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm3
+; CHECK-SSE2-NEXT:    pand %xmm2, %xmm3
+; CHECK-SSE2-NEXT:    pxor %xmm1, %xmm1
+; CHECK-SSE2-NEXT:    pxor %xmm4, %xmm4
+; CHECK-SSE2-NEXT:    pcmpgtd %xmm0, %xmm4
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [2454267027,2454267027,0,2454267027]
+; CHECK-SSE2-NEXT:    pand %xmm5, %xmm4
+; CHECK-SSE2-NEXT:    paddd %xmm3, %xmm4
+; CHECK-SSE2-NEXT:    pmuludq %xmm0, %xmm5
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm5[1,3,2,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq {{.*}}(%rip), %xmm5
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,3,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1]
+; CHECK-SSE2-NEXT:    psubd %xmm4, %xmm3
+; CHECK-SSE2-NEXT:    paddd %xmm0, %xmm3
+; CHECK-SSE2-NEXT:    movdqa %xmm3, %xmm4
+; CHECK-SSE2-NEXT:    psrad $3, %xmm4
+; CHECK-SSE2-NEXT:    movdqa %xmm3, %xmm5
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm5 = xmm5[2,0],xmm4[3,0]
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[0,1],xmm5[0,2]
+; CHECK-SSE2-NEXT:    psrld $31, %xmm3
+; CHECK-SSE2-NEXT:    pand %xmm2, %xmm3
+; CHECK-SSE2-NEXT:    paddd %xmm4, %xmm3
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq {{.*}}(%rip), %xmm3
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
+; CHECK-SSE2-NEXT:    pmuludq {{.*}}(%rip), %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; CHECK-SSE2-NEXT:    psubd %xmm3, %xmm0
+; CHECK-SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-SSE2-NEXT:    psrld $31, %xmm0
+; CHECK-SSE2-NEXT:    retq
+;
+; CHECK-SSE41-LABEL: test_srem_even_one:
+; CHECK-SSE41:       # %bb.0:
+; CHECK-SSE41-NEXT:    movl $-1840700269, %eax # imm = 0x92492493
+; CHECK-SSE41-NEXT:    movd %eax, %xmm1
+; CHECK-SSE41-NEXT:    pmuldq %xmm0, %xmm1
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; CHECK-SSE41-NEXT:    pmuldq {{.*}}(%rip), %xmm2
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
+; CHECK-SSE41-NEXT:    paddd %xmm0, %xmm2
+; CHECK-SSE41-NEXT:    movdqa %xmm2, %xmm1
+; CHECK-SSE41-NEXT:    psrad $3, %xmm1
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm2[4,5],xmm1[6,7]
+; CHECK-SSE41-NEXT:    psrld $31, %xmm2
+; CHECK-SSE41-NEXT:    pxor %xmm3, %xmm3
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm3[4,5],xmm2[6,7]
+; CHECK-SSE41-NEXT:    paddd %xmm1, %xmm2
+; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm2
+; CHECK-SSE41-NEXT:    psubd %xmm2, %xmm0
+; CHECK-SSE41-NEXT:    pcmpeqd %xmm3, %xmm0
+; CHECK-SSE41-NEXT:    psrld $31, %xmm0
+; CHECK-SSE41-NEXT:    retq
+;
+; CHECK-AVX1-LABEL: test_srem_even_one:
+; CHECK-AVX1:       # %bb.0:
+; CHECK-AVX1-NEXT:    movl $-1840700269, %eax # imm = 0x92492493
+; CHECK-AVX1-NEXT:    vmovd %eax, %xmm1
+; CHECK-AVX1-NEXT:    vpmuldq %xmm1, %xmm0, %xmm1
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpmuldq {{.*}}(%rip), %xmm2, %xmm2
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
+; CHECK-AVX1-NEXT:    vpaddd %xmm0, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpsrad $3, %xmm1, %xmm2
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm1[4,5],xmm2[6,7]
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm3[4,5],xmm1[6,7]
+; CHECK-AVX1-NEXT:    vpaddd %xmm1, %xmm2, %xmm1
+; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpcmpeqd %xmm3, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    retq
+;
+; CHECK-AVX2-LABEL: test_srem_even_one:
+; CHECK-AVX2:       # %bb.0:
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [2454267027,2454267027,2454267027,2454267027]
+; CHECK-AVX2-NEXT:    vpmuldq %xmm2, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    movl $-1840700269, %eax # imm = 0x92492493
+; CHECK-AVX2-NEXT:    vmovd %eax, %xmm2
+; CHECK-AVX2-NEXT:    vpmuldq %xmm2, %xmm0, %xmm2
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2],xmm1[3]
+; CHECK-AVX2-NEXT:    vpaddd %xmm0, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpsrld $31, %xmm1, %xmm2
+; CHECK-AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm2 = xmm2[0,1],xmm3[2],xmm2[3]
+; CHECK-AVX2-NEXT:    vpsravd {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpcmpeqd %xmm3, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    retq
+;
+; CHECK-AVX512VL-LABEL: test_srem_even_one:
+; CHECK-AVX512VL:       # %bb.0:
+; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip){1to4}, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpaddd {{.*}}(%rip){1to4}, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vprord $1, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
+; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    retq
+  %srem = srem <4 x i32> %X, <i32 14, i32 14, i32 1, i32 14>
+  %cmp = icmp eq <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; One one divisor in odd+even divisor
+define <4 x i32> @test_srem_odd_even_one(<4 x i32> %X) nounwind {
+; CHECK-SSE2-LABEL: test_srem_odd_even_one:
+; CHECK-SSE2:       # %bb.0:
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [0,1,1,0]
+; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm2
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,2,3,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm4, %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [1717986919,2454267027,0,1374389535]
+; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm1
+; CHECK-SSE2-NEXT:    pmuludq %xmm5, %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,3,2,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm5[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm4, %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; CHECK-SSE2-NEXT:    pxor %xmm1, %xmm1
+; CHECK-SSE2-NEXT:    pxor %xmm4, %xmm4
+; CHECK-SSE2-NEXT:    pcmpgtd %xmm0, %xmm4
+; CHECK-SSE2-NEXT:    pand %xmm5, %xmm4
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [0,4294967295,0,0]
+; CHECK-SSE2-NEXT:    pand %xmm0, %xmm5
+; CHECK-SSE2-NEXT:    paddd %xmm4, %xmm5
+; CHECK-SSE2-NEXT:    psubd %xmm5, %xmm2
+; CHECK-SSE2-NEXT:    paddd %xmm3, %xmm2
+; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm3
+; CHECK-SSE2-NEXT:    psrad $5, %xmm3
+; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm4
+; CHECK-SSE2-NEXT:    punpckhqdq {{.*#+}} xmm4 = xmm4[1],xmm3[1]
+; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm3
+; CHECK-SSE2-NEXT:    psrad $3, %xmm3
+; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm5
+; CHECK-SSE2-NEXT:    psrad $1, %xmm5
+; CHECK-SSE2-NEXT:    punpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm3[0]
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm5 = xmm5[0,3],xmm4[0,3]
+; CHECK-SSE2-NEXT:    psrld $31, %xmm2
+; CHECK-SSE2-NEXT:    pand {{.*}}(%rip), %xmm2
+; CHECK-SSE2-NEXT:    paddd %xmm5, %xmm2
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [5,14,1,100]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm4, %xmm3
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; CHECK-SSE2-NEXT:    psubd %xmm2, %xmm0
+; CHECK-SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-SSE2-NEXT:    psrld $31, %xmm0
+; CHECK-SSE2-NEXT:    retq
+;
+; CHECK-SSE41-LABEL: test_srem_odd_even_one:
+; CHECK-SSE41:       # %bb.0:
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [1717986919,2454267027,0,1374389535]
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; CHECK-SSE41-NEXT:    pmuldq %xmm2, %xmm3
+; CHECK-SSE41-NEXT:    pmuldq %xmm0, %xmm1
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5],xmm3[6,7]
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [0,1,1,0]
+; CHECK-SSE41-NEXT:    pmulld %xmm0, %xmm2
+; CHECK-SSE41-NEXT:    paddd %xmm1, %xmm2
+; CHECK-SSE41-NEXT:    movdqa %xmm2, %xmm1
+; CHECK-SSE41-NEXT:    psrad $5, %xmm1
+; CHECK-SSE41-NEXT:    movdqa %xmm2, %xmm3
+; CHECK-SSE41-NEXT:    psrad $3, %xmm3
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm1[4,5,6,7]
+; CHECK-SSE41-NEXT:    movdqa %xmm2, %xmm1
+; CHECK-SSE41-NEXT:    psrad $1, %xmm1
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm2[4,5,6,7]
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5],xmm3[6,7]
+; CHECK-SSE41-NEXT:    psrld $31, %xmm2
+; CHECK-SSE41-NEXT:    pxor %xmm3, %xmm3
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm3[4,5],xmm2[6,7]
+; CHECK-SSE41-NEXT:    paddd %xmm1, %xmm2
+; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm2
+; CHECK-SSE41-NEXT:    psubd %xmm2, %xmm0
+; CHECK-SSE41-NEXT:    pcmpeqd %xmm3, %xmm0
+; CHECK-SSE41-NEXT:    psrld $31, %xmm0
+; CHECK-SSE41-NEXT:    retq
+;
+; CHECK-AVX1-LABEL: test_srem_odd_even_one:
+; CHECK-AVX1:       # %bb.0:
+; CHECK-AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [1717986919,2454267027,0,1374389535]
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpmuldq %xmm2, %xmm3, %xmm2
+; CHECK-AVX1-NEXT:    vpmuldq %xmm1, %xmm0, %xmm1
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
+; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm2
+; CHECK-AVX1-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpsrad $5, %xmm1, %xmm2
+; CHECK-AVX1-NEXT:    vpsrad $3, %xmm1, %xmm3
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
+; CHECK-AVX1-NEXT:    vpsrad $1, %xmm1, %xmm3
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm1[4,5,6,7]
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm3[4,5],xmm1[6,7]
+; CHECK-AVX1-NEXT:    vpaddd %xmm1, %xmm2, %xmm1
+; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpcmpeqd %xmm3, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    retq
+;
+; CHECK-AVX2-LABEL: test_srem_odd_even_one:
+; CHECK-AVX2:       # %bb.0:
+; CHECK-AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [1717986919,2454267027,0,1374389535]
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpmuldq %xmm2, %xmm3, %xmm2
+; CHECK-AVX2-NEXT:    vpmuldq %xmm1, %xmm0, %xmm1
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
+; CHECK-AVX2-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm2
+; CHECK-AVX2-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpsrld $31, %xmm1, %xmm2
+; CHECK-AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm2 = xmm2[0,1],xmm3[2],xmm2[3]
+; CHECK-AVX2-NEXT:    vpsravd {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpcmpeqd %xmm3, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    retq
+;
+; CHECK-AVX512VL-LABEL: test_srem_odd_even_one:
+; CHECK-AVX512VL:       # %bb.0:
+; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpaddd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vprorvd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
+; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    retq
+  %srem = srem <4 x i32> %X, <i32 5, i32 14, i32 1, i32 100>
+  %cmp = icmp eq <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+;------------------------------------------------------------------------------;
+
+; One INT_MIN divisor in odd divisor
+define <4 x i32> @test_srem_odd_INT_MIN(<4 x i32> %X) nounwind {
+; CHECK-SSE2-LABEL: test_srem_odd_INT_MIN:
+; CHECK-SSE2:       # %bb.0:
+; CHECK-SSE2-NEXT:    pxor %xmm1, %xmm1
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483647,2147483647,2147483647,2147483647]
+; CHECK-SSE2-NEXT:    pand %xmm0, %xmm2
+; CHECK-SSE2-NEXT:    pcmpeqd %xmm1, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq {{.*}}(%rip), %xmm0
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[0,2,2,3]
+; CHECK-SSE2-NEXT:    pmuludq {{.*}}(%rip), %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1]
+; CHECK-SSE2-NEXT:    paddd {{.*}}(%rip), %xmm3
+; CHECK-SSE2-NEXT:    pxor {{.*}}(%rip), %xmm3
+; CHECK-SSE2-NEXT:    pcmpgtd {{.*}}(%rip), %xmm3
+; CHECK-SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
+; CHECK-SSE2-NEXT:    pxor %xmm3, %xmm0
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[2,0],xmm0[3,0]
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm2[0,2]
+; CHECK-SSE2-NEXT:    psrld $31, %xmm0
+; CHECK-SSE2-NEXT:    retq
+;
+; CHECK-SSE41-LABEL: test_srem_odd_INT_MIN:
+; CHECK-SSE41:       # %bb.0:
+; CHECK-SSE41-NEXT:    pxor %xmm1, %xmm1
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [2147483647,2147483647,2147483647,2147483647]
+; CHECK-SSE41-NEXT:    pand %xmm0, %xmm2
+; CHECK-SSE41-NEXT:    pcmpeqd %xmm1, %xmm2
+; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm0
+; CHECK-SSE41-NEXT:    paddd {{.*}}(%rip), %xmm0
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [858993458,858993458,0,858993458]
+; CHECK-SSE41-NEXT:    pminud %xmm0, %xmm1
+; CHECK-SSE41-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm2[4,5],xmm0[6,7]
+; CHECK-SSE41-NEXT:    psrld $31, %xmm0
+; CHECK-SSE41-NEXT:    retq
+;
+; CHECK-AVX1-LABEL: test_srem_odd_INT_MIN:
+; CHECK-AVX1:       # %bb.0:
+; CHECK-AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm2
+; CHECK-AVX1-NEXT:    vpcmpeqd %xmm1, %xmm2, %xmm1
+; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpaddd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm2
+; CHECK-AVX1-NEXT:    vpcmpeqd %xmm2, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4,5],xmm0[6,7]
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    retq
+;
+; CHECK-AVX2-LABEL: test_srem_odd_INT_MIN:
+; CHECK-AVX2:       # %bb.0:
+; CHECK-AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [2147483647,2147483647,2147483647,2147483647]
+; CHECK-AVX2-NEXT:    vpand %xmm2, %xmm0, %xmm2
+; CHECK-AVX2-NEXT:    vpcmpeqd %xmm1, %xmm2, %xmm1
+; CHECK-AVX2-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpaddd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm2
+; CHECK-AVX2-NEXT:    vpcmpeqd %xmm2, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm1[2],xmm0[3]
+; CHECK-AVX2-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    retq
+;
+; CHECK-AVX512VL-LABEL: test_srem_odd_INT_MIN:
+; CHECK-AVX512VL:       # %bb.0:
+; CHECK-AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX512VL-NEXT:    vpandd {{.*}}(%rip){1to4}, %xmm0, %xmm2
+; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm2, %xmm1
+; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpaddd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm2
+; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm2, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm1[2],xmm0[3]
+; CHECK-AVX512VL-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    retq
+  %srem = srem <4 x i32> %X, <i32 5, i32 5, i32 2147483648, i32 5>
+  %cmp = icmp eq <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; One INT_MIN divisor in even divisor
+define <4 x i32> @test_srem_even_INT_MIN(<4 x i32> %X) nounwind {
+; CHECK-SSE2-LABEL: test_srem_even_INT_MIN:
+; CHECK-SSE2:       # %bb.0:
+; CHECK-SSE2-NEXT:    pxor %xmm1, %xmm1
+; CHECK-SSE2-NEXT:    pxor %xmm2, %xmm2
+; CHECK-SSE2-NEXT:    pcmpgtd %xmm0, %xmm2
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [2454267027,2454267027,2147483647,2454267027]
+; CHECK-SSE2-NEXT:    pand %xmm3, %xmm2
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [4294967295,4294967295,0,4294967295]
+; CHECK-SSE2-NEXT:    pand %xmm0, %xmm4
+; CHECK-SSE2-NEXT:    paddd %xmm2, %xmm4
+; CHECK-SSE2-NEXT:    pmuludq %xmm0, %xmm3
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,3,2,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [2454267027,2454267027,2454267027,2454267027]
+; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm5
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,3,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
+; CHECK-SSE2-NEXT:    psubd %xmm4, %xmm2
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm4 = <1,u,4294967295,u>
+; CHECK-SSE2-NEXT:    pmuludq %xmm0, %xmm4
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[0,2,2,3]
+; CHECK-SSE2-NEXT:    pmuludq {{.*}}(%rip), %xmm3
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
+; CHECK-SSE2-NEXT:    paddd %xmm2, %xmm4
+; CHECK-SSE2-NEXT:    movdqa %xmm4, %xmm2
+; CHECK-SSE2-NEXT:    psrad $3, %xmm2
+; CHECK-SSE2-NEXT:    movdqa %xmm4, %xmm3
+; CHECK-SSE2-NEXT:    psrad $30, %xmm3
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[2,0],xmm2[3,0]
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,1],xmm3[0,2]
+; CHECK-SSE2-NEXT:    psrld $31, %xmm4
+; CHECK-SSE2-NEXT:    paddd %xmm2, %xmm4
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq {{.*}}(%rip), %xmm4
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[0,2,2,3]
+; CHECK-SSE2-NEXT:    pmuludq {{.*}}(%rip), %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; CHECK-SSE2-NEXT:    psubd %xmm3, %xmm0
+; CHECK-SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-SSE2-NEXT:    psrld $31, %xmm0
+; CHECK-SSE2-NEXT:    retq
+;
+; CHECK-SSE41-LABEL: test_srem_even_INT_MIN:
+; CHECK-SSE41:       # %bb.0:
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; CHECK-SSE41-NEXT:    pmuldq {{.*}}(%rip), %xmm1
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm2 = <2454267027,u,2147483647,u>
+; CHECK-SSE41-NEXT:    pmuldq %xmm0, %xmm2
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [1,1,4294967295,1]
+; CHECK-SSE41-NEXT:    pmulld %xmm0, %xmm1
+; CHECK-SSE41-NEXT:    paddd %xmm2, %xmm1
+; CHECK-SSE41-NEXT:    movdqa %xmm1, %xmm2
+; CHECK-SSE41-NEXT:    psrad $30, %xmm2
+; CHECK-SSE41-NEXT:    movdqa %xmm1, %xmm3
+; CHECK-SSE41-NEXT:    psrad $3, %xmm3
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm2[4,5],xmm3[6,7]
+; CHECK-SSE41-NEXT:    psrld $31, %xmm1
+; CHECK-SSE41-NEXT:    paddd %xmm3, %xmm1
+; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm1
+; CHECK-SSE41-NEXT:    psubd %xmm1, %xmm0
+; CHECK-SSE41-NEXT:    pxor %xmm1, %xmm1
+; CHECK-SSE41-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-SSE41-NEXT:    psrld $31, %xmm0
+; CHECK-SSE41-NEXT:    retq
+;
+; CHECK-AVX1-LABEL: test_srem_even_INT_MIN:
+; CHECK-AVX1:       # %bb.0:
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpmuldq {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpmuldq {{.*}}(%rip), %xmm0, %xmm2
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
+; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm2
+; CHECK-AVX1-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpsrad $30, %xmm1, %xmm2
+; CHECK-AVX1-NEXT:    vpsrad $3, %xmm1, %xmm3
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5],xmm3[6,7]
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpaddd %xmm1, %xmm2, %xmm1
+; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    retq
+;
+; CHECK-AVX2-LABEL: test_srem_even_INT_MIN:
+; CHECK-AVX2:       # %bb.0:
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [2454267027,2454267027,2454267027,2454267027]
+; CHECK-AVX2-NEXT:    vpmuldq %xmm2, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpmuldq {{.*}}(%rip), %xmm0, %xmm2
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2],xmm1[3]
+; CHECK-AVX2-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm2
+; CHECK-AVX2-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpsrld $31, %xmm1, %xmm2
+; CHECK-AVX2-NEXT:    vpsravd {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    retq
+;
+; CHECK-AVX512VL-LABEL: test_srem_even_INT_MIN:
+; CHECK-AVX512VL:       # %bb.0:
+; CHECK-AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm2
+; CHECK-AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm3 = [306783378,306783378,0,306783378]
+; CHECK-AVX512VL-NEXT:    vpaddd %xmm3, %xmm2, %xmm2
+; CHECK-AVX512VL-NEXT:    vprorvd {{.*}}(%rip), %xmm2, %xmm2
+; CHECK-AVX512VL-NEXT:    vpminud %xmm3, %xmm2, %xmm3
+; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
+; CHECK-AVX512VL-NEXT:    vpandd {{.*}}(%rip){1to4}, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2],xmm2[3]
+; CHECK-AVX512VL-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    retq
+  %srem = srem <4 x i32> %X, <i32 14, i32 14, i32 2147483648, i32 14>
+  %cmp = icmp eq <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; One INT_MIN divisor in odd+even divisor
+define <4 x i32> @test_srem_odd_even_INT_MIN(<4 x i32> %X) nounwind {
+; CHECK-SSE2-LABEL: test_srem_odd_even_INT_MIN:
+; CHECK-SSE2:       # %bb.0:
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [0,1,4294967295,0]
+; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm2
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm4, %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [1717986919,2454267027,2147483647,1374389535]
+; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm1
+; CHECK-SSE2-NEXT:    pmuludq %xmm5, %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,3,2,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm5[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm4, %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; CHECK-SSE2-NEXT:    pxor %xmm1, %xmm1
+; CHECK-SSE2-NEXT:    pxor %xmm4, %xmm4
+; CHECK-SSE2-NEXT:    pcmpgtd %xmm0, %xmm4
+; CHECK-SSE2-NEXT:    pand %xmm5, %xmm4
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [0,4294967295,0,0]
+; CHECK-SSE2-NEXT:    pand %xmm0, %xmm5
+; CHECK-SSE2-NEXT:    paddd %xmm4, %xmm5
+; CHECK-SSE2-NEXT:    psubd %xmm5, %xmm2
+; CHECK-SSE2-NEXT:    paddd %xmm3, %xmm2
+; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm3
+; CHECK-SSE2-NEXT:    psrad $5, %xmm3
+; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm4
+; CHECK-SSE2-NEXT:    psrad $30, %xmm4
+; CHECK-SSE2-NEXT:    punpckhqdq {{.*#+}} xmm4 = xmm4[1],xmm3[1]
+; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm3
+; CHECK-SSE2-NEXT:    psrad $3, %xmm3
+; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm5
+; CHECK-SSE2-NEXT:    psrad $1, %xmm5
+; CHECK-SSE2-NEXT:    punpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm3[0]
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm5 = xmm5[0,3],xmm4[0,3]
+; CHECK-SSE2-NEXT:    psrld $31, %xmm2
+; CHECK-SSE2-NEXT:    paddd %xmm5, %xmm2
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [5,14,2147483648,100]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm4, %xmm3
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; CHECK-SSE2-NEXT:    psubd %xmm2, %xmm0
+; CHECK-SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-SSE2-NEXT:    psrld $31, %xmm0
+; CHECK-SSE2-NEXT:    retq
+;
+; CHECK-SSE41-LABEL: test_srem_odd_even_INT_MIN:
+; CHECK-SSE41:       # %bb.0:
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [1717986919,2454267027,2147483647,1374389535]
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; CHECK-SSE41-NEXT:    pmuldq %xmm2, %xmm3
+; CHECK-SSE41-NEXT:    pmuldq %xmm0, %xmm1
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5],xmm3[6,7]
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [0,1,4294967295,0]
+; CHECK-SSE41-NEXT:    pmulld %xmm0, %xmm2
+; CHECK-SSE41-NEXT:    paddd %xmm1, %xmm2
+; CHECK-SSE41-NEXT:    movdqa %xmm2, %xmm1
+; CHECK-SSE41-NEXT:    psrad $5, %xmm1
+; CHECK-SSE41-NEXT:    movdqa %xmm2, %xmm3
+; CHECK-SSE41-NEXT:    psrad $3, %xmm3
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm1[4,5,6,7]
+; CHECK-SSE41-NEXT:    movdqa %xmm2, %xmm1
+; CHECK-SSE41-NEXT:    psrad $30, %xmm1
+; CHECK-SSE41-NEXT:    movdqa %xmm2, %xmm4
+; CHECK-SSE41-NEXT:    psrad $1, %xmm4
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1,2,3],xmm1[4,5,6,7]
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1],xmm3[2,3],xmm4[4,5],xmm3[6,7]
+; CHECK-SSE41-NEXT:    psrld $31, %xmm2
+; CHECK-SSE41-NEXT:    paddd %xmm4, %xmm2
+; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm2
+; CHECK-SSE41-NEXT:    psubd %xmm2, %xmm0
+; CHECK-SSE41-NEXT:    pxor %xmm1, %xmm1
+; CHECK-SSE41-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-SSE41-NEXT:    psrld $31, %xmm0
+; CHECK-SSE41-NEXT:    retq
+;
+; CHECK-AVX1-LABEL: test_srem_odd_even_INT_MIN:
+; CHECK-AVX1:       # %bb.0:
+; CHECK-AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [1717986919,2454267027,2147483647,1374389535]
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpmuldq %xmm2, %xmm3, %xmm2
+; CHECK-AVX1-NEXT:    vpmuldq %xmm1, %xmm0, %xmm1
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
+; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm2
+; CHECK-AVX1-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpsrad $5, %xmm1, %xmm2
+; CHECK-AVX1-NEXT:    vpsrad $3, %xmm1, %xmm3
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
+; CHECK-AVX1-NEXT:    vpsrad $30, %xmm1, %xmm3
+; CHECK-AVX1-NEXT:    vpsrad $1, %xmm1, %xmm4
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm4[0,1,2,3],xmm3[4,5,6,7]
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpaddd %xmm1, %xmm2, %xmm1
+; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    retq
+;
+; CHECK-AVX2-LABEL: test_srem_odd_even_INT_MIN:
+; CHECK-AVX2:       # %bb.0:
+; CHECK-AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [1717986919,2454267027,2147483647,1374389535]
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpmuldq %xmm2, %xmm3, %xmm2
+; CHECK-AVX2-NEXT:    vpmuldq %xmm1, %xmm0, %xmm1
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
+; CHECK-AVX2-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm2
+; CHECK-AVX2-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpsrld $31, %xmm1, %xmm2
+; CHECK-AVX2-NEXT:    vpsravd {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    retq
+;
+; CHECK-AVX512VL-LABEL: test_srem_odd_even_INT_MIN:
+; CHECK-AVX512VL:       # %bb.0:
+; CHECK-AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX512VL-NEXT:    vpandd {{.*}}(%rip){1to4}, %xmm0, %xmm2
+; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm2, %xmm1
+; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpaddd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vprorvd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm2
+; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm2, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm1[2],xmm0[3]
+; CHECK-AVX512VL-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    retq
+  %srem = srem <4 x i32> %X, <i32 5, i32 14, i32 2147483648, i32 100>
+  %cmp = icmp eq <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+;==============================================================================;
+
+; One all-ones divisor and power-of-two divisor divisor in odd divisor
+define <4 x i32> @test_srem_odd_allones_and_poweroftwo(<4 x i32> %X) nounwind {
+; CHECK-SSE2-LABEL: test_srem_odd_allones_and_poweroftwo:
+; CHECK-SSE2:       # %bb.0:
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [0,4294967295,1,0]
+; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm2
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm4, %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [1717986919,0,2147483649,1717986919]
+; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm1
+; CHECK-SSE2-NEXT:    pmuludq %xmm5, %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,3,2,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm5[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm4, %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; CHECK-SSE2-NEXT:    pxor %xmm1, %xmm1
+; CHECK-SSE2-NEXT:    pxor %xmm4, %xmm4
+; CHECK-SSE2-NEXT:    pcmpgtd %xmm0, %xmm4
+; CHECK-SSE2-NEXT:    pand %xmm5, %xmm4
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [0,0,4294967295,0]
+; CHECK-SSE2-NEXT:    pand %xmm0, %xmm5
+; CHECK-SSE2-NEXT:    paddd %xmm4, %xmm5
+; CHECK-SSE2-NEXT:    psubd %xmm5, %xmm2
+; CHECK-SSE2-NEXT:    paddd %xmm3, %xmm2
+; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm3
+; CHECK-SSE2-NEXT:    psrad $1, %xmm3
+; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm4
+; CHECK-SSE2-NEXT:    psrad $3, %xmm4
+; CHECK-SSE2-NEXT:    punpckhqdq {{.*#+}} xmm4 = xmm4[1],xmm3[1]
+; CHECK-SSE2-NEXT:    punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm2[0]
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,3],xmm4[0,3]
+; CHECK-SSE2-NEXT:    psrld $31, %xmm2
+; CHECK-SSE2-NEXT:    pand {{.*}}(%rip), %xmm2
+; CHECK-SSE2-NEXT:    paddd %xmm3, %xmm2
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [5,4294967295,16,5]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm4, %xmm3
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; CHECK-SSE2-NEXT:    psubd %xmm2, %xmm0
+; CHECK-SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-SSE2-NEXT:    psrld $31, %xmm0
+; CHECK-SSE2-NEXT:    retq
+;
+; CHECK-SSE41-LABEL: test_srem_odd_allones_and_poweroftwo:
+; CHECK-SSE41:       # %bb.0:
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [1717986919,0,2147483649,1717986919]
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; CHECK-SSE41-NEXT:    pmuldq %xmm2, %xmm3
+; CHECK-SSE41-NEXT:    pmuldq %xmm0, %xmm1
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5],xmm3[6,7]
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [0,4294967295,1,0]
+; CHECK-SSE41-NEXT:    pmulld %xmm0, %xmm2
+; CHECK-SSE41-NEXT:    paddd %xmm1, %xmm2
+; CHECK-SSE41-NEXT:    movdqa %xmm2, %xmm1
+; CHECK-SSE41-NEXT:    psrad $1, %xmm1
+; CHECK-SSE41-NEXT:    movdqa %xmm2, %xmm3
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm1[4,5,6,7]
+; CHECK-SSE41-NEXT:    movdqa %xmm2, %xmm4
+; CHECK-SSE41-NEXT:    psrad $3, %xmm4
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm4 = xmm1[0,1,2,3],xmm4[4,5,6,7]
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1],xmm3[2,3],xmm4[4,5],xmm3[6,7]
+; CHECK-SSE41-NEXT:    psrld $31, %xmm2
+; CHECK-SSE41-NEXT:    pxor %xmm1, %xmm1
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm1[2,3],xmm2[4,5,6,7]
+; CHECK-SSE41-NEXT:    paddd %xmm4, %xmm2
+; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm2
+; CHECK-SSE41-NEXT:    psubd %xmm2, %xmm0
+; CHECK-SSE41-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-SSE41-NEXT:    psrld $31, %xmm0
+; CHECK-SSE41-NEXT:    retq
+;
+; CHECK-AVX1-LABEL: test_srem_odd_allones_and_poweroftwo:
+; CHECK-AVX1:       # %bb.0:
+; CHECK-AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [1717986919,0,2147483649,1717986919]
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpmuldq %xmm2, %xmm3, %xmm2
+; CHECK-AVX1-NEXT:    vpmuldq %xmm1, %xmm0, %xmm1
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
+; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm2
+; CHECK-AVX1-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpsrad $1, %xmm1, %xmm2
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm1[0,1,2,3],xmm2[4,5,6,7]
+; CHECK-AVX1-NEXT:    vpsrad $3, %xmm1, %xmm4
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm4[4,5,6,7]
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3],xmm2[4,5],xmm3[6,7]
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5,6,7]
+; CHECK-AVX1-NEXT:    vpaddd %xmm1, %xmm2, %xmm1
+; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpcmpeqd %xmm3, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    retq
+;
+; CHECK-AVX2-LABEL: test_srem_odd_allones_and_poweroftwo:
+; CHECK-AVX2:       # %bb.0:
+; CHECK-AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [1717986919,0,2147483649,1717986919]
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpmuldq %xmm2, %xmm3, %xmm2
+; CHECK-AVX2-NEXT:    vpmuldq %xmm1, %xmm0, %xmm1
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
+; CHECK-AVX2-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm2
+; CHECK-AVX2-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpsrld $31, %xmm1, %xmm2
+; CHECK-AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm2 = xmm2[0],xmm3[1],xmm2[2,3]
+; CHECK-AVX2-NEXT:    vpsravd {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpcmpeqd %xmm3, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    retq
+;
+; CHECK-AVX512VL-LABEL: test_srem_odd_allones_and_poweroftwo:
+; CHECK-AVX512VL:       # %bb.0:
+; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpaddd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vprorvd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
+; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    retq
+  %srem = srem <4 x i32> %X, <i32 5, i32 4294967295, i32 16, i32 5>
+  %cmp = icmp eq <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; One all-ones divisor and power-of-two divisor divisor in even divisor
+define <4 x i32> @test_srem_even_allones_and_poweroftwo(<4 x i32> %X) nounwind {
+; CHECK-SSE2-LABEL: test_srem_even_allones_and_poweroftwo:
+; CHECK-SSE2:       # %bb.0:
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [4294967295,0,4294967295,4294967295]
+; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm3
+; CHECK-SSE2-NEXT:    pand %xmm2, %xmm3
+; CHECK-SSE2-NEXT:    pxor %xmm1, %xmm1
+; CHECK-SSE2-NEXT:    pxor %xmm4, %xmm4
+; CHECK-SSE2-NEXT:    pcmpgtd %xmm0, %xmm4
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [2454267027,0,2147483649,2454267027]
+; CHECK-SSE2-NEXT:    pand %xmm5, %xmm4
+; CHECK-SSE2-NEXT:    paddd %xmm3, %xmm4
+; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm3
+; CHECK-SSE2-NEXT:    pmuludq %xmm5, %xmm3
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,3,2,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm0[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm6, %xmm5
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,3,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1]
+; CHECK-SSE2-NEXT:    psubd %xmm4, %xmm3
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [1,4294967295,1,1]
+; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm5
+; CHECK-SSE2-NEXT:    pmuludq %xmm4, %xmm5
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[0,2,2,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm6, %xmm4
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
+; CHECK-SSE2-NEXT:    paddd %xmm3, %xmm5
+; CHECK-SSE2-NEXT:    movdqa %xmm5, %xmm3
+; CHECK-SSE2-NEXT:    psrad $3, %xmm3
+; CHECK-SSE2-NEXT:    movdqa %xmm5, %xmm4
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[1,0],xmm3[0,0]
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[2,0],xmm3[2,3]
+; CHECK-SSE2-NEXT:    psrld $31, %xmm5
+; CHECK-SSE2-NEXT:    pand %xmm2, %xmm5
+; CHECK-SSE2-NEXT:    paddd %xmm4, %xmm5
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [14,4294967295,16,14]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm5[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm2, %xmm5
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm5[0,2,2,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
+; CHECK-SSE2-NEXT:    psubd %xmm4, %xmm0
+; CHECK-SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-SSE2-NEXT:    psrld $31, %xmm0
+; CHECK-SSE2-NEXT:    retq
+;
+; CHECK-SSE41-LABEL: test_srem_even_allones_and_poweroftwo:
+; CHECK-SSE41:       # %bb.0:
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [2454267027,0,2147483649,2454267027]
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; CHECK-SSE41-NEXT:    pmuldq %xmm2, %xmm3
+; CHECK-SSE41-NEXT:    pmuldq %xmm0, %xmm1
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5],xmm3[6,7]
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [1,4294967295,1,1]
+; CHECK-SSE41-NEXT:    pmulld %xmm0, %xmm2
+; CHECK-SSE41-NEXT:    paddd %xmm1, %xmm2
+; CHECK-SSE41-NEXT:    movdqa %xmm2, %xmm1
+; CHECK-SSE41-NEXT:    psrad $3, %xmm1
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5,6,7]
+; CHECK-SSE41-NEXT:    psrld $31, %xmm2
+; CHECK-SSE41-NEXT:    pxor %xmm3, %xmm3
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3],xmm2[4,5,6,7]
+; CHECK-SSE41-NEXT:    paddd %xmm1, %xmm2
+; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm2
+; CHECK-SSE41-NEXT:    psubd %xmm2, %xmm0
+; CHECK-SSE41-NEXT:    pcmpeqd %xmm3, %xmm0
+; CHECK-SSE41-NEXT:    psrld $31, %xmm0
+; CHECK-SSE41-NEXT:    retq
+;
+; CHECK-AVX1-LABEL: test_srem_even_allones_and_poweroftwo:
+; CHECK-AVX1:       # %bb.0:
+; CHECK-AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [2454267027,0,2147483649,2454267027]
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpmuldq %xmm2, %xmm3, %xmm2
+; CHECK-AVX1-NEXT:    vpmuldq %xmm1, %xmm0, %xmm1
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
+; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm2
+; CHECK-AVX1-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpsrad $3, %xmm1, %xmm2
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1],xmm1[2,3],xmm2[4,5,6,7]
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5,6,7]
+; CHECK-AVX1-NEXT:    vpaddd %xmm1, %xmm2, %xmm1
+; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpcmpeqd %xmm3, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    retq
+;
+; CHECK-AVX2-LABEL: test_srem_even_allones_and_poweroftwo:
+; CHECK-AVX2:       # %bb.0:
+; CHECK-AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [2454267027,0,2147483649,2454267027]
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpmuldq %xmm2, %xmm3, %xmm2
+; CHECK-AVX2-NEXT:    vpmuldq %xmm1, %xmm0, %xmm1
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
+; CHECK-AVX2-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm2
+; CHECK-AVX2-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpsrld $31, %xmm1, %xmm2
+; CHECK-AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm2 = xmm2[0],xmm3[1],xmm2[2,3]
+; CHECK-AVX2-NEXT:    vpsravd {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpcmpeqd %xmm3, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    retq
+;
+; CHECK-AVX512VL-LABEL: test_srem_even_allones_and_poweroftwo:
+; CHECK-AVX512VL:       # %bb.0:
+; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpaddd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vprorvd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
+; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    retq
+  %srem = srem <4 x i32> %X, <i32 14, i32 4294967295, i32 16, i32 14>
+  %cmp = icmp eq <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; One all-ones divisor and power-of-two divisor divisor in odd+even divisor
+define <4 x i32> @test_srem_odd_even_allones_and_poweroftwo(<4 x i32> %X) nounwind {
+; CHECK-SSE2-LABEL: test_srem_odd_even_allones_and_poweroftwo:
+; CHECK-SSE2:       # %bb.0:
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [0,4294967295,1,0]
+; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm2
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm4, %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [1717986919,0,2147483649,1374389535]
+; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm1
+; CHECK-SSE2-NEXT:    pmuludq %xmm5, %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,3,2,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm5[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm4, %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; CHECK-SSE2-NEXT:    pxor %xmm1, %xmm1
+; CHECK-SSE2-NEXT:    pxor %xmm4, %xmm4
+; CHECK-SSE2-NEXT:    pcmpgtd %xmm0, %xmm4
+; CHECK-SSE2-NEXT:    pand %xmm5, %xmm4
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [0,0,4294967295,0]
+; CHECK-SSE2-NEXT:    pand %xmm0, %xmm5
+; CHECK-SSE2-NEXT:    paddd %xmm4, %xmm5
+; CHECK-SSE2-NEXT:    psubd %xmm5, %xmm2
+; CHECK-SSE2-NEXT:    paddd %xmm3, %xmm2
+; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm3
+; CHECK-SSE2-NEXT:    psrad $5, %xmm3
+; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm4
+; CHECK-SSE2-NEXT:    psrad $3, %xmm4
+; CHECK-SSE2-NEXT:    punpckhqdq {{.*#+}} xmm4 = xmm4[1],xmm3[1]
+; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm3
+; CHECK-SSE2-NEXT:    psrad $1, %xmm3
+; CHECK-SSE2-NEXT:    punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm2[0]
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,3],xmm4[0,3]
+; CHECK-SSE2-NEXT:    psrld $31, %xmm2
+; CHECK-SSE2-NEXT:    pand {{.*}}(%rip), %xmm2
+; CHECK-SSE2-NEXT:    paddd %xmm3, %xmm2
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [5,4294967295,16,100]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm4, %xmm3
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; CHECK-SSE2-NEXT:    psubd %xmm2, %xmm0
+; CHECK-SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-SSE2-NEXT:    psrld $31, %xmm0
+; CHECK-SSE2-NEXT:    retq
+;
+; CHECK-SSE41-LABEL: test_srem_odd_even_allones_and_poweroftwo:
+; CHECK-SSE41:       # %bb.0:
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [1717986919,0,2147483649,1374389535]
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; CHECK-SSE41-NEXT:    pmuldq %xmm2, %xmm3
+; CHECK-SSE41-NEXT:    pmuldq %xmm0, %xmm1
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5],xmm3[6,7]
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [0,4294967295,1,0]
+; CHECK-SSE41-NEXT:    pmulld %xmm0, %xmm2
+; CHECK-SSE41-NEXT:    paddd %xmm1, %xmm2
+; CHECK-SSE41-NEXT:    movdqa %xmm2, %xmm1
+; CHECK-SSE41-NEXT:    psrad $5, %xmm1
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm2[0,1,2,3],xmm1[4,5,6,7]
+; CHECK-SSE41-NEXT:    movdqa %xmm2, %xmm3
+; CHECK-SSE41-NEXT:    psrad $3, %xmm3
+; CHECK-SSE41-NEXT:    movdqa %xmm2, %xmm4
+; CHECK-SSE41-NEXT:    psrad $1, %xmm4
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1,2,3],xmm3[4,5,6,7]
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1],xmm1[2,3],xmm4[4,5],xmm1[6,7]
+; CHECK-SSE41-NEXT:    psrld $31, %xmm2
+; CHECK-SSE41-NEXT:    pxor %xmm1, %xmm1
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm1[2,3],xmm2[4,5,6,7]
+; CHECK-SSE41-NEXT:    paddd %xmm4, %xmm2
+; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm2
+; CHECK-SSE41-NEXT:    psubd %xmm2, %xmm0
+; CHECK-SSE41-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-SSE41-NEXT:    psrld $31, %xmm0
+; CHECK-SSE41-NEXT:    retq
+;
+; CHECK-AVX1-LABEL: test_srem_odd_even_allones_and_poweroftwo:
+; CHECK-AVX1:       # %bb.0:
+; CHECK-AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [1717986919,0,2147483649,1374389535]
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpmuldq %xmm2, %xmm3, %xmm2
+; CHECK-AVX1-NEXT:    vpmuldq %xmm1, %xmm0, %xmm1
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
+; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm2
+; CHECK-AVX1-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpsrad $5, %xmm1, %xmm2
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm1[0,1,2,3],xmm2[4,5,6,7]
+; CHECK-AVX1-NEXT:    vpsrad $3, %xmm1, %xmm3
+; CHECK-AVX1-NEXT:    vpsrad $1, %xmm1, %xmm4
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm4[0,1,2,3],xmm3[4,5,6,7]
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5,6,7]
+; CHECK-AVX1-NEXT:    vpaddd %xmm1, %xmm2, %xmm1
+; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpcmpeqd %xmm3, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    retq
+;
+; CHECK-AVX2-LABEL: test_srem_odd_even_allones_and_poweroftwo:
+; CHECK-AVX2:       # %bb.0:
+; CHECK-AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [1717986919,0,2147483649,1374389535]
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpmuldq %xmm2, %xmm3, %xmm2
+; CHECK-AVX2-NEXT:    vpmuldq %xmm1, %xmm0, %xmm1
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
+; CHECK-AVX2-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm2
+; CHECK-AVX2-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpsrld $31, %xmm1, %xmm2
+; CHECK-AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm2 = xmm2[0],xmm3[1],xmm2[2,3]
+; CHECK-AVX2-NEXT:    vpsravd {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpcmpeqd %xmm3, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    retq
+;
+; CHECK-AVX512VL-LABEL: test_srem_odd_even_allones_and_poweroftwo:
+; CHECK-AVX512VL:       # %bb.0:
+; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpaddd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vprorvd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
+; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    retq
+  %srem = srem <4 x i32> %X, <i32 5, i32 4294967295, i32 16, i32 100>
+  %cmp = icmp eq <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+;------------------------------------------------------------------------------;
+
+; One all-ones divisor and one one divisor in odd divisor
+define <4 x i32> @test_srem_odd_allones_and_one(<4 x i32> %X) nounwind {
+; CHECK-SSE2-LABEL: test_srem_odd_allones_and_one:
+; CHECK-SSE2:       # %bb.0:
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [3435973837,3435973837,3435973837,3435973837]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm0
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; CHECK-SSE2-NEXT:    paddd {{.*}}(%rip), %xmm0
+; CHECK-SSE2-NEXT:    pxor {{.*}}(%rip), %xmm0
+; CHECK-SSE2-NEXT:    pcmpgtd {{.*}}(%rip), %xmm0
+; CHECK-SSE2-NEXT:    pandn {{.*}}(%rip), %xmm0
+; CHECK-SSE2-NEXT:    retq
+;
+; CHECK-SSE41-LABEL: test_srem_odd_allones_and_one:
+; CHECK-SSE41:       # %bb.0:
+; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm0
+; CHECK-SSE41-NEXT:    paddd {{.*}}(%rip), %xmm0
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [858993458,4294967295,4294967295,858993458]
+; CHECK-SSE41-NEXT:    pminud %xmm0, %xmm1
+; CHECK-SSE41-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-SSE41-NEXT:    psrld $31, %xmm0
+; CHECK-SSE41-NEXT:    retq
+;
+; CHECK-AVX1-LABEL: test_srem_odd_allones_and_one:
+; CHECK-AVX1:       # %bb.0:
+; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpaddd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
+; CHECK-AVX1-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    retq
+;
+; CHECK-AVX2-LABEL: test_srem_odd_allones_and_one:
+; CHECK-AVX2:       # %bb.0:
+; CHECK-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [3435973837,3435973837,3435973837,3435973837]
+; CHECK-AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [429496729,429496729,429496729,429496729]
+; CHECK-AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
+; CHECK-AVX2-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    retq
+;
+; CHECK-AVX512VL-LABEL: test_srem_odd_allones_and_one:
+; CHECK-AVX512VL:       # %bb.0:
+; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip){1to4}, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpaddd {{.*}}(%rip){1to4}, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
+; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    retq
+  %srem = srem <4 x i32> %X, <i32 5, i32 4294967295, i32 1, i32 5>
+  %cmp = icmp eq <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; One all-ones divisor and one one divisor in even divisor
+define <4 x i32> @test_srem_even_allones_and_one(<4 x i32> %X) nounwind {
+; CHECK-SSE2-LABEL: test_srem_even_allones_and_one:
+; CHECK-SSE2:       # %bb.0:
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [4294967295,0,0,4294967295]
+; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm3
+; CHECK-SSE2-NEXT:    pand %xmm2, %xmm3
+; CHECK-SSE2-NEXT:    pxor %xmm1, %xmm1
+; CHECK-SSE2-NEXT:    pxor %xmm4, %xmm4
+; CHECK-SSE2-NEXT:    pcmpgtd %xmm0, %xmm4
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [2454267027,0,0,2454267027]
+; CHECK-SSE2-NEXT:    pand %xmm5, %xmm4
+; CHECK-SSE2-NEXT:    paddd %xmm3, %xmm4
+; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm3
+; CHECK-SSE2-NEXT:    pmuludq %xmm5, %xmm3
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,3,2,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[2,2,3,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm0[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm6, %xmm5
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,3,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1]
+; CHECK-SSE2-NEXT:    psubd %xmm4, %xmm3
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [1,4294967295,1,1]
+; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm5
+; CHECK-SSE2-NEXT:    pmuludq %xmm4, %xmm5
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[0,2,2,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm6, %xmm4
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
+; CHECK-SSE2-NEXT:    paddd %xmm3, %xmm5
+; CHECK-SSE2-NEXT:    movdqa %xmm5, %xmm3
+; CHECK-SSE2-NEXT:    psrad $3, %xmm3
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,3],xmm5[1,2]
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,2,3,1]
+; CHECK-SSE2-NEXT:    psrld $31, %xmm5
+; CHECK-SSE2-NEXT:    pand %xmm2, %xmm5
+; CHECK-SSE2-NEXT:    paddd %xmm3, %xmm5
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [14,4294967295,1,14]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm5[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm2, %xmm5
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm5[0,2,2,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
+; CHECK-SSE2-NEXT:    psubd %xmm4, %xmm0
+; CHECK-SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-SSE2-NEXT:    psrld $31, %xmm0
+; CHECK-SSE2-NEXT:    retq
+;
+; CHECK-SSE41-LABEL: test_srem_even_allones_and_one:
+; CHECK-SSE41:       # %bb.0:
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [2454267027,0,0,2454267027]
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,2,3,3]
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; CHECK-SSE41-NEXT:    pmuldq %xmm2, %xmm3
+; CHECK-SSE41-NEXT:    pmuldq %xmm0, %xmm1
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5],xmm3[6,7]
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [1,4294967295,1,1]
+; CHECK-SSE41-NEXT:    pmulld %xmm0, %xmm2
+; CHECK-SSE41-NEXT:    paddd %xmm1, %xmm2
+; CHECK-SSE41-NEXT:    movdqa %xmm2, %xmm1
+; CHECK-SSE41-NEXT:    psrad $3, %xmm1
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3,4,5],xmm1[6,7]
+; CHECK-SSE41-NEXT:    psrld $31, %xmm2
+; CHECK-SSE41-NEXT:    pxor %xmm3, %xmm3
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3,4,5],xmm2[6,7]
+; CHECK-SSE41-NEXT:    paddd %xmm1, %xmm2
+; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm2
+; CHECK-SSE41-NEXT:    psubd %xmm2, %xmm0
+; CHECK-SSE41-NEXT:    pcmpeqd %xmm3, %xmm0
+; CHECK-SSE41-NEXT:    psrld $31, %xmm0
+; CHECK-SSE41-NEXT:    retq
+;
+; CHECK-AVX1-LABEL: test_srem_even_allones_and_one:
+; CHECK-AVX1:       # %bb.0:
+; CHECK-AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [2454267027,0,0,2454267027]
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[2,2,3,3]
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpmuldq %xmm2, %xmm3, %xmm2
+; CHECK-AVX1-NEXT:    vpmuldq %xmm1, %xmm0, %xmm1
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
+; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm2
+; CHECK-AVX1-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpsrad $3, %xmm1, %xmm2
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1],xmm1[2,3,4,5],xmm2[6,7]
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3,4,5],xmm1[6,7]
+; CHECK-AVX1-NEXT:    vpaddd %xmm1, %xmm2, %xmm1
+; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpcmpeqd %xmm3, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    retq
+;
+; CHECK-AVX2-LABEL: test_srem_even_allones_and_one:
+; CHECK-AVX2:       # %bb.0:
+; CHECK-AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [2454267027,0,0,2454267027]
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[2,2,3,3]
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpmuldq %xmm2, %xmm3, %xmm2
+; CHECK-AVX2-NEXT:    vpmuldq %xmm1, %xmm0, %xmm1
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
+; CHECK-AVX2-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm2
+; CHECK-AVX2-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpsrld $31, %xmm1, %xmm2
+; CHECK-AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm2 = xmm2[0],xmm3[1,2],xmm2[3]
+; CHECK-AVX2-NEXT:    vpsravd {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpcmpeqd %xmm3, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    retq
+;
+; CHECK-AVX512VL-LABEL: test_srem_even_allones_and_one:
+; CHECK-AVX512VL:       # %bb.0:
+; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip){1to4}, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpaddd {{.*}}(%rip){1to4}, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vprord $1, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
+; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    retq
+  %srem = srem <4 x i32> %X, <i32 14, i32 4294967295, i32 1, i32 14>
+  %cmp = icmp eq <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; One all-ones divisor and one one divisor in odd+even divisor
+define <4 x i32> @test_srem_odd_even_allones_and_one(<4 x i32> %X) nounwind {
+; CHECK-SSE2-LABEL: test_srem_odd_even_allones_and_one:
+; CHECK-SSE2:       # %bb.0:
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [0,4294967295,1,0]
+; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm2
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm4, %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [1717986919,0,0,1374389535]
+; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm1
+; CHECK-SSE2-NEXT:    pmuludq %xmm5, %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,3,2,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm5[2,2,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm4, %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; CHECK-SSE2-NEXT:    pxor %xmm1, %xmm1
+; CHECK-SSE2-NEXT:    pxor %xmm4, %xmm4
+; CHECK-SSE2-NEXT:    pcmpgtd %xmm0, %xmm4
+; CHECK-SSE2-NEXT:    pand %xmm5, %xmm4
+; CHECK-SSE2-NEXT:    psubd %xmm4, %xmm2
+; CHECK-SSE2-NEXT:    paddd %xmm3, %xmm2
+; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm3
+; CHECK-SSE2-NEXT:    psrad $5, %xmm3
+; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm4
+; CHECK-SSE2-NEXT:    punpckhqdq {{.*#+}} xmm4 = xmm4[1],xmm3[1]
+; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm3
+; CHECK-SSE2-NEXT:    psrad $1, %xmm3
+; CHECK-SSE2-NEXT:    punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm2[0]
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,3],xmm4[0,3]
+; CHECK-SSE2-NEXT:    psrld $31, %xmm2
+; CHECK-SSE2-NEXT:    pand {{.*}}(%rip), %xmm2
+; CHECK-SSE2-NEXT:    paddd %xmm3, %xmm2
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [5,4294967295,1,100]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm4, %xmm3
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; CHECK-SSE2-NEXT:    psubd %xmm2, %xmm0
+; CHECK-SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-SSE2-NEXT:    psrld $31, %xmm0
+; CHECK-SSE2-NEXT:    retq
+;
+; CHECK-SSE41-LABEL: test_srem_odd_even_allones_and_one:
+; CHECK-SSE41:       # %bb.0:
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [1717986919,0,0,1374389535]
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,2,3,3]
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; CHECK-SSE41-NEXT:    pmuldq %xmm2, %xmm3
+; CHECK-SSE41-NEXT:    pmuldq %xmm0, %xmm1
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5],xmm3[6,7]
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [0,4294967295,1,0]
+; CHECK-SSE41-NEXT:    pmulld %xmm0, %xmm2
+; CHECK-SSE41-NEXT:    paddd %xmm1, %xmm2
+; CHECK-SSE41-NEXT:    movdqa %xmm2, %xmm1
+; CHECK-SSE41-NEXT:    psrad $5, %xmm1
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm2[0,1,2,3],xmm1[4,5,6,7]
+; CHECK-SSE41-NEXT:    movdqa %xmm2, %xmm3
+; CHECK-SSE41-NEXT:    psrad $1, %xmm3
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm2[4,5,6,7]
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm3 = xmm3[0,1],xmm1[2,3],xmm3[4,5],xmm1[6,7]
+; CHECK-SSE41-NEXT:    psrld $31, %xmm2
+; CHECK-SSE41-NEXT:    pxor %xmm1, %xmm1
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm1[2,3,4,5],xmm2[6,7]
+; CHECK-SSE41-NEXT:    paddd %xmm3, %xmm2
+; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm2
+; CHECK-SSE41-NEXT:    psubd %xmm2, %xmm0
+; CHECK-SSE41-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-SSE41-NEXT:    psrld $31, %xmm0
+; CHECK-SSE41-NEXT:    retq
+;
+; CHECK-AVX1-LABEL: test_srem_odd_even_allones_and_one:
+; CHECK-AVX1:       # %bb.0:
+; CHECK-AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [1717986919,0,0,1374389535]
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[2,2,3,3]
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpmuldq %xmm2, %xmm3, %xmm2
+; CHECK-AVX1-NEXT:    vpmuldq %xmm1, %xmm0, %xmm1
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
+; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm2
+; CHECK-AVX1-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpsrad $5, %xmm1, %xmm2
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm1[0,1,2,3],xmm2[4,5,6,7]
+; CHECK-AVX1-NEXT:    vpsrad $1, %xmm1, %xmm3
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm1[4,5,6,7]
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3,4,5],xmm1[6,7]
+; CHECK-AVX1-NEXT:    vpaddd %xmm1, %xmm2, %xmm1
+; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpcmpeqd %xmm3, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    retq
+;
+; CHECK-AVX2-LABEL: test_srem_odd_even_allones_and_one:
+; CHECK-AVX2:       # %bb.0:
+; CHECK-AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [1717986919,0,0,1374389535]
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[2,2,3,3]
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpmuldq %xmm2, %xmm3, %xmm2
+; CHECK-AVX2-NEXT:    vpmuldq %xmm1, %xmm0, %xmm1
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
+; CHECK-AVX2-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm2
+; CHECK-AVX2-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpsrld $31, %xmm1, %xmm2
+; CHECK-AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm2 = xmm2[0],xmm3[1,2],xmm2[3]
+; CHECK-AVX2-NEXT:    vpsravd {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpcmpeqd %xmm3, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    retq
+;
+; CHECK-AVX512VL-LABEL: test_srem_odd_even_allones_and_one:
+; CHECK-AVX512VL:       # %bb.0:
+; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpaddd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vprorvd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
+; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    retq
+  %srem = srem <4 x i32> %X, <i32 5, i32 4294967295, i32 1, i32 100>
+  %cmp = icmp eq <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+;------------------------------------------------------------------------------;
+
+; One power-of-two divisor divisor and one divisor in odd divisor
+define <4 x i32> @test_srem_odd_poweroftwo_and_one(<4 x i32> %X) nounwind {
+; CHECK-SSE2-LABEL: test_srem_odd_poweroftwo_and_one:
+; CHECK-SSE2:       # %bb.0:
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [0,1,1,0]
+; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm2
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,2,3,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm4, %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [1717986919,2147483649,0,1717986919]
+; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm1
+; CHECK-SSE2-NEXT:    pmuludq %xmm5, %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,3,2,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm5[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm4, %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; CHECK-SSE2-NEXT:    pxor %xmm1, %xmm1
+; CHECK-SSE2-NEXT:    pxor %xmm4, %xmm4
+; CHECK-SSE2-NEXT:    pcmpgtd %xmm0, %xmm4
+; CHECK-SSE2-NEXT:    pand %xmm5, %xmm4
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [0,4294967295,0,0]
+; CHECK-SSE2-NEXT:    pand %xmm0, %xmm5
+; CHECK-SSE2-NEXT:    paddd %xmm4, %xmm5
+; CHECK-SSE2-NEXT:    psubd %xmm5, %xmm2
+; CHECK-SSE2-NEXT:    paddd %xmm3, %xmm2
+; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm3
+; CHECK-SSE2-NEXT:    psrad $1, %xmm3
+; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm4
+; CHECK-SSE2-NEXT:    punpckhqdq {{.*#+}} xmm4 = xmm4[1],xmm3[1]
+; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm5
+; CHECK-SSE2-NEXT:    psrad $3, %xmm5
+; CHECK-SSE2-NEXT:    punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,3],xmm4[0,3]
+; CHECK-SSE2-NEXT:    psrld $31, %xmm2
+; CHECK-SSE2-NEXT:    pand {{.*}}(%rip), %xmm2
+; CHECK-SSE2-NEXT:    paddd %xmm3, %xmm2
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [5,16,1,5]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm4, %xmm3
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; CHECK-SSE2-NEXT:    psubd %xmm2, %xmm0
+; CHECK-SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-SSE2-NEXT:    psrld $31, %xmm0
+; CHECK-SSE2-NEXT:    retq
+;
+; CHECK-SSE41-LABEL: test_srem_odd_poweroftwo_and_one:
+; CHECK-SSE41:       # %bb.0:
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [1717986919,2147483649,0,1717986919]
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; CHECK-SSE41-NEXT:    pmuldq %xmm2, %xmm3
+; CHECK-SSE41-NEXT:    pmuldq %xmm0, %xmm1
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5],xmm3[6,7]
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [0,1,1,0]
+; CHECK-SSE41-NEXT:    pmulld %xmm0, %xmm2
+; CHECK-SSE41-NEXT:    paddd %xmm1, %xmm2
+; CHECK-SSE41-NEXT:    movdqa %xmm2, %xmm1
+; CHECK-SSE41-NEXT:    psrad $1, %xmm1
+; CHECK-SSE41-NEXT:    movdqa %xmm2, %xmm3
+; CHECK-SSE41-NEXT:    psrad $3, %xmm3
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm1[4,5,6,7]
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm2[4,5,6,7]
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5],xmm3[6,7]
+; CHECK-SSE41-NEXT:    psrld $31, %xmm2
+; CHECK-SSE41-NEXT:    pxor %xmm3, %xmm3
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm3[4,5],xmm2[6,7]
+; CHECK-SSE41-NEXT:    paddd %xmm1, %xmm2
+; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm2
+; CHECK-SSE41-NEXT:    psubd %xmm2, %xmm0
+; CHECK-SSE41-NEXT:    pcmpeqd %xmm3, %xmm0
+; CHECK-SSE41-NEXT:    psrld $31, %xmm0
+; CHECK-SSE41-NEXT:    retq
+;
+; CHECK-AVX1-LABEL: test_srem_odd_poweroftwo_and_one:
+; CHECK-AVX1:       # %bb.0:
+; CHECK-AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [1717986919,2147483649,0,1717986919]
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpmuldq %xmm2, %xmm3, %xmm2
+; CHECK-AVX1-NEXT:    vpmuldq %xmm1, %xmm0, %xmm1
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
+; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm2
+; CHECK-AVX1-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpsrad $1, %xmm1, %xmm2
+; CHECK-AVX1-NEXT:    vpsrad $3, %xmm1, %xmm3
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm2[4,5,6,7]
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm1[4,5,6,7]
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3],xmm2[4,5],xmm3[6,7]
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm3[4,5],xmm1[6,7]
+; CHECK-AVX1-NEXT:    vpaddd %xmm1, %xmm2, %xmm1
+; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpcmpeqd %xmm3, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    retq
+;
+; CHECK-AVX2-LABEL: test_srem_odd_poweroftwo_and_one:
+; CHECK-AVX2:       # %bb.0:
+; CHECK-AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [1717986919,2147483649,0,1717986919]
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpmuldq %xmm2, %xmm3, %xmm2
+; CHECK-AVX2-NEXT:    vpmuldq %xmm1, %xmm0, %xmm1
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
+; CHECK-AVX2-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm2
+; CHECK-AVX2-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpsrld $31, %xmm1, %xmm2
+; CHECK-AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm2 = xmm2[0,1],xmm3[2],xmm2[3]
+; CHECK-AVX2-NEXT:    vpsravd {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpcmpeqd %xmm3, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    retq
+;
+; CHECK-AVX512VL-LABEL: test_srem_odd_poweroftwo_and_one:
+; CHECK-AVX512VL:       # %bb.0:
+; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpaddd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vprorvd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
+; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    retq
+  %srem = srem <4 x i32> %X, <i32 5, i32 16, i32 1, i32 5>
+  %cmp = icmp eq <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; One power-of-two divisor divisor and one divisor in even divisor
+define <4 x i32> @test_srem_even_poweroftwo_and_one(<4 x i32> %X) nounwind {
+; CHECK-SSE2-LABEL: test_srem_even_poweroftwo_and_one:
+; CHECK-SSE2:       # %bb.0:
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [4294967295,4294967295,0,4294967295]
+; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm3
+; CHECK-SSE2-NEXT:    pand %xmm2, %xmm3
+; CHECK-SSE2-NEXT:    pxor %xmm1, %xmm1
+; CHECK-SSE2-NEXT:    pxor %xmm4, %xmm4
+; CHECK-SSE2-NEXT:    pcmpgtd %xmm0, %xmm4
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [2454267027,2147483649,0,2454267027]
+; CHECK-SSE2-NEXT:    pand %xmm5, %xmm4
+; CHECK-SSE2-NEXT:    paddd %xmm3, %xmm4
+; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm3
+; CHECK-SSE2-NEXT:    pmuludq %xmm5, %xmm3
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,3,2,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm0[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm5, %xmm6
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm6[1,3,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1]
+; CHECK-SSE2-NEXT:    psubd %xmm4, %xmm3
+; CHECK-SSE2-NEXT:    paddd %xmm0, %xmm3
+; CHECK-SSE2-NEXT:    movdqa %xmm3, %xmm4
+; CHECK-SSE2-NEXT:    psrad $3, %xmm4
+; CHECK-SSE2-NEXT:    movdqa %xmm3, %xmm5
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm5 = xmm5[2,0],xmm4[3,0]
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[0,1],xmm5[0,2]
+; CHECK-SSE2-NEXT:    psrld $31, %xmm3
+; CHECK-SSE2-NEXT:    pand %xmm2, %xmm3
+; CHECK-SSE2-NEXT:    paddd %xmm4, %xmm3
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [14,16,1,14]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm2, %xmm3
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm4, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; CHECK-SSE2-NEXT:    psubd %xmm3, %xmm0
+; CHECK-SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-SSE2-NEXT:    psrld $31, %xmm0
+; CHECK-SSE2-NEXT:    retq
+;
+; CHECK-SSE41-LABEL: test_srem_even_poweroftwo_and_one:
+; CHECK-SSE41:       # %bb.0:
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [2454267027,2147483649,0,2454267027]
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; CHECK-SSE41-NEXT:    pmuldq %xmm2, %xmm3
+; CHECK-SSE41-NEXT:    pmuldq %xmm0, %xmm1
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5],xmm3[6,7]
+; CHECK-SSE41-NEXT:    paddd %xmm0, %xmm1
+; CHECK-SSE41-NEXT:    movdqa %xmm1, %xmm2
+; CHECK-SSE41-NEXT:    psrad $3, %xmm2
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm1[4,5],xmm2[6,7]
+; CHECK-SSE41-NEXT:    psrld $31, %xmm1
+; CHECK-SSE41-NEXT:    pxor %xmm3, %xmm3
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm3[4,5],xmm1[6,7]
+; CHECK-SSE41-NEXT:    paddd %xmm2, %xmm1
+; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm1
+; CHECK-SSE41-NEXT:    psubd %xmm1, %xmm0
+; CHECK-SSE41-NEXT:    pcmpeqd %xmm3, %xmm0
+; CHECK-SSE41-NEXT:    psrld $31, %xmm0
+; CHECK-SSE41-NEXT:    retq
+;
+; CHECK-AVX1-LABEL: test_srem_even_poweroftwo_and_one:
+; CHECK-AVX1:       # %bb.0:
+; CHECK-AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [2454267027,2147483649,0,2454267027]
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpmuldq %xmm2, %xmm3, %xmm2
+; CHECK-AVX1-NEXT:    vpmuldq %xmm1, %xmm0, %xmm1
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
+; CHECK-AVX1-NEXT:    vpaddd %xmm0, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpsrad $3, %xmm1, %xmm2
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm1[4,5],xmm2[6,7]
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm3[4,5],xmm1[6,7]
+; CHECK-AVX1-NEXT:    vpaddd %xmm1, %xmm2, %xmm1
+; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpcmpeqd %xmm3, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    retq
+;
+; CHECK-AVX2-LABEL: test_srem_even_poweroftwo_and_one:
+; CHECK-AVX2:       # %bb.0:
+; CHECK-AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [2454267027,2147483649,0,2454267027]
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpmuldq %xmm2, %xmm3, %xmm2
+; CHECK-AVX2-NEXT:    vpmuldq %xmm1, %xmm0, %xmm1
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
+; CHECK-AVX2-NEXT:    vpaddd %xmm0, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpsrld $31, %xmm1, %xmm2
+; CHECK-AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm2 = xmm2[0,1],xmm3[2],xmm2[3]
+; CHECK-AVX2-NEXT:    vpsravd {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpcmpeqd %xmm3, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    retq
+;
+; CHECK-AVX512VL-LABEL: test_srem_even_poweroftwo_and_one:
+; CHECK-AVX512VL:       # %bb.0:
+; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpaddd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vprorvd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
+; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    retq
+  %srem = srem <4 x i32> %X, <i32 14, i32 16, i32 1, i32 14>
+  %cmp = icmp eq <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; One power-of-two divisor divisor and one divisor in odd+even divisor
+define <4 x i32> @test_srem_odd_even_poweroftwo_and_one(<4 x i32> %X) nounwind {
+; CHECK-SSE2-LABEL: test_srem_odd_even_poweroftwo_and_one:
+; CHECK-SSE2:       # %bb.0:
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [0,1,1,0]
+; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm2
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,2,3,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm4, %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [1717986919,2147483649,0,1374389535]
+; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm1
+; CHECK-SSE2-NEXT:    pmuludq %xmm5, %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,3,2,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm5[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm4, %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; CHECK-SSE2-NEXT:    pxor %xmm1, %xmm1
+; CHECK-SSE2-NEXT:    pxor %xmm4, %xmm4
+; CHECK-SSE2-NEXT:    pcmpgtd %xmm0, %xmm4
+; CHECK-SSE2-NEXT:    pand %xmm5, %xmm4
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [0,4294967295,0,0]
+; CHECK-SSE2-NEXT:    pand %xmm0, %xmm5
+; CHECK-SSE2-NEXT:    paddd %xmm4, %xmm5
+; CHECK-SSE2-NEXT:    psubd %xmm5, %xmm2
+; CHECK-SSE2-NEXT:    paddd %xmm3, %xmm2
+; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm3
+; CHECK-SSE2-NEXT:    psrad $5, %xmm3
+; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm4
+; CHECK-SSE2-NEXT:    punpckhqdq {{.*#+}} xmm4 = xmm4[1],xmm3[1]
+; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm3
+; CHECK-SSE2-NEXT:    psrad $3, %xmm3
+; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm5
+; CHECK-SSE2-NEXT:    psrad $1, %xmm5
+; CHECK-SSE2-NEXT:    punpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm3[0]
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm5 = xmm5[0,3],xmm4[0,3]
+; CHECK-SSE2-NEXT:    psrld $31, %xmm2
+; CHECK-SSE2-NEXT:    pand {{.*}}(%rip), %xmm2
+; CHECK-SSE2-NEXT:    paddd %xmm5, %xmm2
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [5,16,1,100]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm4, %xmm3
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; CHECK-SSE2-NEXT:    psubd %xmm2, %xmm0
+; CHECK-SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-SSE2-NEXT:    psrld $31, %xmm0
+; CHECK-SSE2-NEXT:    retq
+;
+; CHECK-SSE41-LABEL: test_srem_odd_even_poweroftwo_and_one:
+; CHECK-SSE41:       # %bb.0:
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [1717986919,2147483649,0,1374389535]
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; CHECK-SSE41-NEXT:    pmuldq %xmm2, %xmm3
+; CHECK-SSE41-NEXT:    pmuldq %xmm0, %xmm1
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5],xmm3[6,7]
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [0,1,1,0]
+; CHECK-SSE41-NEXT:    pmulld %xmm0, %xmm2
+; CHECK-SSE41-NEXT:    paddd %xmm1, %xmm2
+; CHECK-SSE41-NEXT:    movdqa %xmm2, %xmm1
+; CHECK-SSE41-NEXT:    psrad $5, %xmm1
+; CHECK-SSE41-NEXT:    movdqa %xmm2, %xmm3
+; CHECK-SSE41-NEXT:    psrad $3, %xmm3
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm1[4,5,6,7]
+; CHECK-SSE41-NEXT:    movdqa %xmm2, %xmm1
+; CHECK-SSE41-NEXT:    psrad $1, %xmm1
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm2[4,5,6,7]
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5],xmm3[6,7]
+; CHECK-SSE41-NEXT:    psrld $31, %xmm2
+; CHECK-SSE41-NEXT:    pxor %xmm3, %xmm3
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm3[4,5],xmm2[6,7]
+; CHECK-SSE41-NEXT:    paddd %xmm1, %xmm2
+; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm2
+; CHECK-SSE41-NEXT:    psubd %xmm2, %xmm0
+; CHECK-SSE41-NEXT:    pcmpeqd %xmm3, %xmm0
+; CHECK-SSE41-NEXT:    psrld $31, %xmm0
+; CHECK-SSE41-NEXT:    retq
+;
+; CHECK-AVX1-LABEL: test_srem_odd_even_poweroftwo_and_one:
+; CHECK-AVX1:       # %bb.0:
+; CHECK-AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [1717986919,2147483649,0,1374389535]
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpmuldq %xmm2, %xmm3, %xmm2
+; CHECK-AVX1-NEXT:    vpmuldq %xmm1, %xmm0, %xmm1
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
+; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm2
+; CHECK-AVX1-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpsrad $5, %xmm1, %xmm2
+; CHECK-AVX1-NEXT:    vpsrad $3, %xmm1, %xmm3
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
+; CHECK-AVX1-NEXT:    vpsrad $1, %xmm1, %xmm3
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm1[4,5,6,7]
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm3[4,5],xmm1[6,7]
+; CHECK-AVX1-NEXT:    vpaddd %xmm1, %xmm2, %xmm1
+; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpcmpeqd %xmm3, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    retq
+;
+; CHECK-AVX2-LABEL: test_srem_odd_even_poweroftwo_and_one:
+; CHECK-AVX2:       # %bb.0:
+; CHECK-AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [1717986919,2147483649,0,1374389535]
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpmuldq %xmm2, %xmm3, %xmm2
+; CHECK-AVX2-NEXT:    vpmuldq %xmm1, %xmm0, %xmm1
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
+; CHECK-AVX2-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm2
+; CHECK-AVX2-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpsrld $31, %xmm1, %xmm2
+; CHECK-AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm2 = xmm2[0,1],xmm3[2],xmm2[3]
+; CHECK-AVX2-NEXT:    vpsravd {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpcmpeqd %xmm3, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    retq
+;
+; CHECK-AVX512VL-LABEL: test_srem_odd_even_poweroftwo_and_one:
+; CHECK-AVX512VL:       # %bb.0:
+; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpaddd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vprorvd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
+; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    retq
+  %srem = srem <4 x i32> %X, <i32 5, i32 16, i32 1, i32 100>
+  %cmp = icmp eq <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+;------------------------------------------------------------------------------;
+
+define <4 x i32> @test_srem_odd_allones_and_poweroftwo_and_one(<4 x i32> %X) nounwind {
+; CHECK-SSE2-LABEL: test_srem_odd_allones_and_poweroftwo_and_one:
+; CHECK-SSE2:       # %bb.0:
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [0,4294967295,1,1]
+; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm2
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; CHECK-SSE2-NEXT:    pxor %xmm1, %xmm1
+; CHECK-SSE2-NEXT:    pxor %xmm4, %xmm4
+; CHECK-SSE2-NEXT:    pcmpgtd %xmm0, %xmm4
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [1717986919,0,2147483649,0]
+; CHECK-SSE2-NEXT:    pand %xmm2, %xmm4
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [0,0,4294967295,0]
+; CHECK-SSE2-NEXT:    pand %xmm0, %xmm5
+; CHECK-SSE2-NEXT:    paddd %xmm4, %xmm5
+; CHECK-SSE2-NEXT:    pmuludq %xmm0, %xmm2
+; CHECK-SSE2-NEXT:    psrlq $32, %xmm2
+; CHECK-SSE2-NEXT:    psubd %xmm5, %xmm2
+; CHECK-SSE2-NEXT:    paddd %xmm3, %xmm2
+; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm3
+; CHECK-SSE2-NEXT:    psrad $3, %xmm3
+; CHECK-SSE2-NEXT:    punpckhqdq {{.*#+}} xmm3 = xmm3[1],xmm2[1]
+; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm4
+; CHECK-SSE2-NEXT:    psrad $1, %xmm4
+; CHECK-SSE2-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm2[0]
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[0,3],xmm3[0,3]
+; CHECK-SSE2-NEXT:    psrld $31, %xmm2
+; CHECK-SSE2-NEXT:    pand {{.*}}(%rip), %xmm2
+; CHECK-SSE2-NEXT:    paddd %xmm4, %xmm2
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [5,4294967295,16,1]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm4, %xmm3
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; CHECK-SSE2-NEXT:    psubd %xmm2, %xmm0
+; CHECK-SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-SSE2-NEXT:    psrld $31, %xmm0
+; CHECK-SSE2-NEXT:    retq
+;
+; CHECK-SSE41-LABEL: test_srem_odd_allones_and_poweroftwo_and_one:
+; CHECK-SSE41:       # %bb.0:
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [0,4294967295,1,1]
+; CHECK-SSE41-NEXT:    pmulld %xmm0, %xmm1
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm2 = <1717986919,u,2147483649,u>
+; CHECK-SSE41-NEXT:    pmuldq %xmm0, %xmm2
+; CHECK-SSE41-NEXT:    psrlq $32, %xmm2
+; CHECK-SSE41-NEXT:    paddd %xmm1, %xmm2
+; CHECK-SSE41-NEXT:    movdqa %xmm2, %xmm1
+; CHECK-SSE41-NEXT:    psrad $3, %xmm1
+; CHECK-SSE41-NEXT:    movdqa %xmm2, %xmm3
+; CHECK-SSE41-NEXT:    psrad $1, %xmm3
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm1[4,5,6,7]
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm3 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
+; CHECK-SSE41-NEXT:    psrld $31, %xmm2
+; CHECK-SSE41-NEXT:    pxor %xmm1, %xmm1
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
+; CHECK-SSE41-NEXT:    paddd %xmm3, %xmm2
+; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm2
+; CHECK-SSE41-NEXT:    psubd %xmm2, %xmm0
+; CHECK-SSE41-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-SSE41-NEXT:    psrld $31, %xmm0
+; CHECK-SSE41-NEXT:    retq
+;
+; CHECK-AVX1-LABEL: test_srem_odd_allones_and_poweroftwo_and_one:
+; CHECK-AVX1:       # %bb.0:
+; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm1
+; CHECK-AVX1-NEXT:    vpmuldq {{.*}}(%rip), %xmm0, %xmm2
+; CHECK-AVX1-NEXT:    vpsrlq $32, %xmm2, %xmm2
+; CHECK-AVX1-NEXT:    vpaddd %xmm1, %xmm2, %xmm1
+; CHECK-AVX1-NEXT:    vpsrad $3, %xmm1, %xmm2
+; CHECK-AVX1-NEXT:    vpsrad $1, %xmm1, %xmm3
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5],xmm3[6,7]
+; CHECK-AVX1-NEXT:    vpaddd %xmm1, %xmm2, %xmm1
+; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpcmpeqd %xmm3, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    retq
+;
+; CHECK-AVX2-LABEL: test_srem_odd_allones_and_poweroftwo_and_one:
+; CHECK-AVX2:       # %bb.0:
+; CHECK-AVX2-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm1
+; CHECK-AVX2-NEXT:    vpmuldq {{.*}}(%rip), %xmm0, %xmm2
+; CHECK-AVX2-NEXT:    vpsrlq $32, %xmm2, %xmm2
+; CHECK-AVX2-NEXT:    vpaddd %xmm1, %xmm2, %xmm1
+; CHECK-AVX2-NEXT:    vpsrld $31, %xmm1, %xmm2
+; CHECK-AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm2 = xmm2[0],xmm3[1],xmm2[2],xmm3[3]
+; CHECK-AVX2-NEXT:    vpsravd {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpcmpeqd %xmm3, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    retq
+;
+; CHECK-AVX512VL-LABEL: test_srem_odd_allones_and_poweroftwo_and_one:
+; CHECK-AVX512VL:       # %bb.0:
+; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpaddd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vprorvd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
+; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    retq
+  %srem = srem <4 x i32> %X, <i32 5, i32 4294967295, i32 16, i32 1>
+  %cmp = icmp eq <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+define <4 x i32> @test_srem_even_allones_and_poweroftwo_and_one(<4 x i32> %X) nounwind {
+; CHECK-SSE2-LABEL: test_srem_even_allones_and_poweroftwo_and_one:
+; CHECK-SSE2:       # %bb.0:
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [1,4294967295,1,1]
+; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm2
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm3
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [4294967295,0,4294967295,0]
+; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm4
+; CHECK-SSE2-NEXT:    pand %xmm3, %xmm4
+; CHECK-SSE2-NEXT:    pxor %xmm1, %xmm1
+; CHECK-SSE2-NEXT:    pxor %xmm5, %xmm5
+; CHECK-SSE2-NEXT:    pcmpgtd %xmm0, %xmm5
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm6 = [2454267027,0,2147483649,0]
+; CHECK-SSE2-NEXT:    pand %xmm6, %xmm5
+; CHECK-SSE2-NEXT:    paddd %xmm4, %xmm5
+; CHECK-SSE2-NEXT:    pmuludq %xmm0, %xmm6
+; CHECK-SSE2-NEXT:    psrlq $32, %xmm6
+; CHECK-SSE2-NEXT:    psubd %xmm5, %xmm6
+; CHECK-SSE2-NEXT:    paddd %xmm2, %xmm6
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[1,3,2,3]
+; CHECK-SSE2-NEXT:    movdqa %xmm6, %xmm4
+; CHECK-SSE2-NEXT:    psrad $3, %xmm4
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
+; CHECK-SSE2-NEXT:    psrld $31, %xmm6
+; CHECK-SSE2-NEXT:    pand %xmm3, %xmm6
+; CHECK-SSE2-NEXT:    paddd %xmm4, %xmm6
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [14,4294967295,16,1]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm6[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm2, %xmm6
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[0,2,2,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
+; CHECK-SSE2-NEXT:    psubd %xmm4, %xmm0
+; CHECK-SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-SSE2-NEXT:    psrld $31, %xmm0
+; CHECK-SSE2-NEXT:    retq
+;
+; CHECK-SSE41-LABEL: test_srem_even_allones_and_poweroftwo_and_one:
+; CHECK-SSE41:       # %bb.0:
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [1,4294967295,1,1]
+; CHECK-SSE41-NEXT:    pmulld %xmm0, %xmm1
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm2 = <2454267027,u,2147483649,u>
+; CHECK-SSE41-NEXT:    pmuldq %xmm0, %xmm2
+; CHECK-SSE41-NEXT:    psrlq $32, %xmm2
+; CHECK-SSE41-NEXT:    paddd %xmm1, %xmm2
+; CHECK-SSE41-NEXT:    movdqa %xmm2, %xmm1
+; CHECK-SSE41-NEXT:    psrad $3, %xmm1
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
+; CHECK-SSE41-NEXT:    psrld $31, %xmm2
+; CHECK-SSE41-NEXT:    pxor %xmm3, %xmm3
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3],xmm2[4,5],xmm3[6,7]
+; CHECK-SSE41-NEXT:    paddd %xmm1, %xmm2
+; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm2
+; CHECK-SSE41-NEXT:    psubd %xmm2, %xmm0
+; CHECK-SSE41-NEXT:    pcmpeqd %xmm3, %xmm0
+; CHECK-SSE41-NEXT:    psrld $31, %xmm0
+; CHECK-SSE41-NEXT:    retq
+;
+; CHECK-AVX1-LABEL: test_srem_even_allones_and_poweroftwo_and_one:
+; CHECK-AVX1:       # %bb.0:
+; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm1
+; CHECK-AVX1-NEXT:    vpmuldq {{.*}}(%rip), %xmm0, %xmm2
+; CHECK-AVX1-NEXT:    vpsrlq $32, %xmm2, %xmm2
+; CHECK-AVX1-NEXT:    vpaddd %xmm1, %xmm2, %xmm1
+; CHECK-AVX1-NEXT:    vpsrad $3, %xmm1, %xmm2
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5],xmm3[6,7]
+; CHECK-AVX1-NEXT:    vpaddd %xmm1, %xmm2, %xmm1
+; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpcmpeqd %xmm3, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    retq
+;
+; CHECK-AVX2-LABEL: test_srem_even_allones_and_poweroftwo_and_one:
+; CHECK-AVX2:       # %bb.0:
+; CHECK-AVX2-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm1
+; CHECK-AVX2-NEXT:    vpmuldq {{.*}}(%rip), %xmm0, %xmm2
+; CHECK-AVX2-NEXT:    vpsrlq $32, %xmm2, %xmm2
+; CHECK-AVX2-NEXT:    vpaddd %xmm1, %xmm2, %xmm1
+; CHECK-AVX2-NEXT:    vpsrld $31, %xmm1, %xmm2
+; CHECK-AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm2 = xmm2[0],xmm3[1],xmm2[2],xmm3[3]
+; CHECK-AVX2-NEXT:    vpsravd {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpcmpeqd %xmm3, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    retq
+;
+; CHECK-AVX512VL-LABEL: test_srem_even_allones_and_poweroftwo_and_one:
+; CHECK-AVX512VL:       # %bb.0:
+; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpaddd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vprorvd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
+; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    retq
+  %srem = srem <4 x i32> %X, <i32 14, i32 4294967295, i32 16, i32 1>
+  %cmp = icmp eq <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
diff --git a/llvm/test/CodeGen/X86/srem-seteq-vec-splat.ll b/llvm/test/CodeGen/X86/srem-seteq-vec-splat.ll
new file mode 100644
index 000000000..1d1f10f4d
--- /dev/null
+++ b/llvm/test/CodeGen/X86/srem-seteq-vec-splat.ll
@@ -0,0 +1,752 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu -mattr=+sse2 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-SSE,CHECK-SSE2
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu -mattr=+sse4.1 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-SSE,CHECK-SSE41
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu -mattr=+avx < %s | FileCheck %s --check-prefixes=CHECK,CHECK-AVX,CHECK-AVX1
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu -mattr=+avx2 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-AVX,CHECK-AVX2
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512f,+avx512vl < %s | FileCheck %s --check-prefixes=CHECK,CHECK-AVX,CHECK-AVX512VL
+
+; Odd divisor
+define <4 x i32> @test_srem_odd_25(<4 x i32> %X) nounwind {
+; CHECK-SSE2-LABEL: test_srem_odd_25:
+; CHECK-SSE2:       # %bb.0:
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [3264175145,3264175145,3264175145,3264175145]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm0
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; CHECK-SSE2-NEXT:    paddd {{.*}}(%rip), %xmm0
+; CHECK-SSE2-NEXT:    pxor {{.*}}(%rip), %xmm0
+; CHECK-SSE2-NEXT:    pcmpgtd {{.*}}(%rip), %xmm0
+; CHECK-SSE2-NEXT:    pandn {{.*}}(%rip), %xmm0
+; CHECK-SSE2-NEXT:    retq
+;
+; CHECK-SSE41-LABEL: test_srem_odd_25:
+; CHECK-SSE41:       # %bb.0:
+; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm0
+; CHECK-SSE41-NEXT:    paddd {{.*}}(%rip), %xmm0
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [171798690,171798690,171798690,171798690]
+; CHECK-SSE41-NEXT:    pminud %xmm0, %xmm1
+; CHECK-SSE41-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-SSE41-NEXT:    psrld $31, %xmm0
+; CHECK-SSE41-NEXT:    retq
+;
+; CHECK-AVX1-LABEL: test_srem_odd_25:
+; CHECK-AVX1:       # %bb.0:
+; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpaddd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
+; CHECK-AVX1-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    retq
+;
+; CHECK-AVX2-LABEL: test_srem_odd_25:
+; CHECK-AVX2:       # %bb.0:
+; CHECK-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [3264175145,3264175145,3264175145,3264175145]
+; CHECK-AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [85899345,85899345,85899345,85899345]
+; CHECK-AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [171798690,171798690,171798690,171798690]
+; CHECK-AVX2-NEXT:    vpminud %xmm1, %xmm0, %xmm1
+; CHECK-AVX2-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    retq
+;
+; CHECK-AVX512VL-LABEL: test_srem_odd_25:
+; CHECK-AVX512VL:       # %bb.0:
+; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip){1to4}, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpaddd {{.*}}(%rip){1to4}, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpminud {{.*}}(%rip){1to4}, %xmm0, %xmm1
+; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    retq
+  %srem = srem <4 x i32> %X, <i32 25, i32 25, i32 25, i32 25>
+  %cmp = icmp eq <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Even divisors
+define <4 x i32> @test_srem_even_100(<4 x i32> %X) nounwind {
+; CHECK-SSE2-LABEL: test_srem_even_100:
+; CHECK-SSE2:       # %bb.0:
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [1374389535,1374389535,1374389535,1374389535]
+; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm2
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm3
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,3,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; CHECK-SSE2-NEXT:    pxor %xmm3, %xmm3
+; CHECK-SSE2-NEXT:    pxor %xmm4, %xmm4
+; CHECK-SSE2-NEXT:    pcmpgtd %xmm0, %xmm4
+; CHECK-SSE2-NEXT:    pand %xmm1, %xmm4
+; CHECK-SSE2-NEXT:    psubd %xmm4, %xmm2
+; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm1
+; CHECK-SSE2-NEXT:    psrld $31, %xmm1
+; CHECK-SSE2-NEXT:    psrad $5, %xmm2
+; CHECK-SSE2-NEXT:    paddd %xmm1, %xmm2
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [100,100,100,100]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm4
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; CHECK-SSE2-NEXT:    psubd %xmm2, %xmm0
+; CHECK-SSE2-NEXT:    pcmpeqd %xmm3, %xmm0
+; CHECK-SSE2-NEXT:    psrld $31, %xmm0
+; CHECK-SSE2-NEXT:    retq
+;
+; CHECK-SSE41-LABEL: test_srem_even_100:
+; CHECK-SSE41:       # %bb.0:
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [1374389535,1374389535,1374389535,1374389535]
+; CHECK-SSE41-NEXT:    pmuldq %xmm2, %xmm1
+; CHECK-SSE41-NEXT:    pmuldq %xmm0, %xmm2
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
+; CHECK-SSE41-NEXT:    movdqa %xmm2, %xmm1
+; CHECK-SSE41-NEXT:    psrld $31, %xmm1
+; CHECK-SSE41-NEXT:    psrad $5, %xmm2
+; CHECK-SSE41-NEXT:    paddd %xmm1, %xmm2
+; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm2
+; CHECK-SSE41-NEXT:    psubd %xmm2, %xmm0
+; CHECK-SSE41-NEXT:    pxor %xmm1, %xmm1
+; CHECK-SSE41-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-SSE41-NEXT:    psrld $31, %xmm0
+; CHECK-SSE41-NEXT:    retq
+;
+; CHECK-AVX1-LABEL: test_srem_even_100:
+; CHECK-AVX1:       # %bb.0:
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; CHECK-AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [1374389535,1374389535,1374389535,1374389535]
+; CHECK-AVX1-NEXT:    vpmuldq %xmm2, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpmuldq %xmm2, %xmm0, %xmm2
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm1, %xmm2
+; CHECK-AVX1-NEXT:    vpsrad $5, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    retq
+;
+; CHECK-AVX2-LABEL: test_srem_even_100:
+; CHECK-AVX2:       # %bb.0:
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [1374389535,1374389535,1374389535,1374389535]
+; CHECK-AVX2-NEXT:    vpmuldq %xmm2, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpmuldq %xmm2, %xmm0, %xmm2
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2],xmm1[3]
+; CHECK-AVX2-NEXT:    vpsrld $31, %xmm1, %xmm2
+; CHECK-AVX2-NEXT:    vpsrad $5, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [100,100,100,100]
+; CHECK-AVX2-NEXT:    vpmulld %xmm2, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    retq
+;
+; CHECK-AVX512VL-LABEL: test_srem_even_100:
+; CHECK-AVX512VL:       # %bb.0:
+; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip){1to4}, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpaddd {{.*}}(%rip){1to4}, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vprord $2, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpminud {{.*}}(%rip){1to4}, %xmm0, %xmm1
+; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    retq
+  %srem = srem <4 x i32> %X, <i32 100, i32 100, i32 100, i32 100>
+  %cmp = icmp eq <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Negative divisors should be negated, and thus this is still splat vectors.
+
+; Odd divisor
+define <4 x i32> @test_srem_odd_neg25(<4 x i32> %X) nounwind {
+; CHECK-SSE2-LABEL: test_srem_odd_neg25:
+; CHECK-SSE2:       # %bb.0:
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [3264175145,3264175145,3264175145,3264175145]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm0
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; CHECK-SSE2-NEXT:    paddd {{.*}}(%rip), %xmm0
+; CHECK-SSE2-NEXT:    pxor {{.*}}(%rip), %xmm0
+; CHECK-SSE2-NEXT:    pcmpgtd {{.*}}(%rip), %xmm0
+; CHECK-SSE2-NEXT:    pandn {{.*}}(%rip), %xmm0
+; CHECK-SSE2-NEXT:    retq
+;
+; CHECK-SSE41-LABEL: test_srem_odd_neg25:
+; CHECK-SSE41:       # %bb.0:
+; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm0
+; CHECK-SSE41-NEXT:    paddd {{.*}}(%rip), %xmm0
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [171798690,171798690,171798690,171798690]
+; CHECK-SSE41-NEXT:    pminud %xmm0, %xmm1
+; CHECK-SSE41-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-SSE41-NEXT:    psrld $31, %xmm0
+; CHECK-SSE41-NEXT:    retq
+;
+; CHECK-AVX1-LABEL: test_srem_odd_neg25:
+; CHECK-AVX1:       # %bb.0:
+; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpaddd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
+; CHECK-AVX1-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    retq
+;
+; CHECK-AVX2-LABEL: test_srem_odd_neg25:
+; CHECK-AVX2:       # %bb.0:
+; CHECK-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [3264175145,3264175145,3264175145,3264175145]
+; CHECK-AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [85899345,85899345,85899345,85899345]
+; CHECK-AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [171798690,171798690,171798690,171798690]
+; CHECK-AVX2-NEXT:    vpminud %xmm1, %xmm0, %xmm1
+; CHECK-AVX2-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    retq
+;
+; CHECK-AVX512VL-LABEL: test_srem_odd_neg25:
+; CHECK-AVX512VL:       # %bb.0:
+; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip){1to4}, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpaddd {{.*}}(%rip){1to4}, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpminud {{.*}}(%rip){1to4}, %xmm0, %xmm1
+; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    retq
+  %srem = srem <4 x i32> %X, <i32 25, i32 -25, i32 -25, i32 25>
+  %cmp = icmp eq <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Even divisors
+define <4 x i32> @test_srem_even_neg100(<4 x i32> %X) nounwind {
+; CHECK-SSE2-LABEL: test_srem_even_neg100:
+; CHECK-SSE2:       # %bb.0:
+; CHECK-SSE2-NEXT:    pxor %xmm1, %xmm1
+; CHECK-SSE2-NEXT:    pxor %xmm2, %xmm2
+; CHECK-SSE2-NEXT:    pcmpgtd %xmm0, %xmm2
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [2920577761,1374389535,2920577761,1374389535]
+; CHECK-SSE2-NEXT:    pand %xmm3, %xmm2
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [4294967295,0,4294967295,0]
+; CHECK-SSE2-NEXT:    pand %xmm0, %xmm4
+; CHECK-SSE2-NEXT:    paddd %xmm2, %xmm4
+; CHECK-SSE2-NEXT:    pmuludq %xmm0, %xmm3
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,3,2,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq {{.*}}(%rip), %xmm3
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,3,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; CHECK-SSE2-NEXT:    psubd %xmm4, %xmm2
+; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm3
+; CHECK-SSE2-NEXT:    psrld $31, %xmm3
+; CHECK-SSE2-NEXT:    psrad $5, %xmm2
+; CHECK-SSE2-NEXT:    paddd %xmm3, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq {{.*}}(%rip), %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    pmuludq {{.*}}(%rip), %xmm3
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; CHECK-SSE2-NEXT:    psubd %xmm2, %xmm0
+; CHECK-SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-SSE2-NEXT:    psrld $31, %xmm0
+; CHECK-SSE2-NEXT:    retq
+;
+; CHECK-SSE41-LABEL: test_srem_even_neg100:
+; CHECK-SSE41:       # %bb.0:
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; CHECK-SSE41-NEXT:    pmuldq {{.*}}(%rip), %xmm1
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm2 = <2920577761,u,2920577761,u>
+; CHECK-SSE41-NEXT:    pmuldq %xmm0, %xmm2
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
+; CHECK-SSE41-NEXT:    movdqa %xmm2, %xmm1
+; CHECK-SSE41-NEXT:    psrld $31, %xmm1
+; CHECK-SSE41-NEXT:    psrad $5, %xmm2
+; CHECK-SSE41-NEXT:    paddd %xmm1, %xmm2
+; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm2
+; CHECK-SSE41-NEXT:    psubd %xmm2, %xmm0
+; CHECK-SSE41-NEXT:    pxor %xmm1, %xmm1
+; CHECK-SSE41-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-SSE41-NEXT:    psrld $31, %xmm0
+; CHECK-SSE41-NEXT:    retq
+;
+; CHECK-AVX1-LABEL: test_srem_even_neg100:
+; CHECK-AVX1:       # %bb.0:
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpmuldq {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpmuldq {{.*}}(%rip), %xmm0, %xmm2
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm1, %xmm2
+; CHECK-AVX1-NEXT:    vpsrad $5, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    retq
+;
+; CHECK-AVX2-LABEL: test_srem_even_neg100:
+; CHECK-AVX2:       # %bb.0:
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [1374389535,1374389535,1374389535,1374389535]
+; CHECK-AVX2-NEXT:    vpmuldq %xmm2, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [2920577761,2920577761,2920577761,2920577761]
+; CHECK-AVX2-NEXT:    vpmuldq %xmm2, %xmm0, %xmm2
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2],xmm1[3]
+; CHECK-AVX2-NEXT:    vpsrld $31, %xmm1, %xmm2
+; CHECK-AVX2-NEXT:    vpsrad $5, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    retq
+;
+; CHECK-AVX512VL-LABEL: test_srem_even_neg100:
+; CHECK-AVX512VL:       # %bb.0:
+; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip){1to4}, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpaddd {{.*}}(%rip){1to4}, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vprord $2, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpminud {{.*}}(%rip){1to4}, %xmm0, %xmm1
+; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    retq
+  %srem = srem <4 x i32> %X, <i32 -100, i32 100, i32 -100, i32 100>
+  %cmp = icmp eq <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+;------------------------------------------------------------------------------;
+; Comparison constant has undef elements.
+;------------------------------------------------------------------------------;
+
+define <4 x i32> @test_srem_odd_undef1(<4 x i32> %X) nounwind {
+; CHECK-SSE2-LABEL: test_srem_odd_undef1:
+; CHECK-SSE2:       # %bb.0:
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [1374389535,1374389535,1374389535,1374389535]
+; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm2
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm3
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,3,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; CHECK-SSE2-NEXT:    pxor %xmm3, %xmm3
+; CHECK-SSE2-NEXT:    pxor %xmm4, %xmm4
+; CHECK-SSE2-NEXT:    pcmpgtd %xmm0, %xmm4
+; CHECK-SSE2-NEXT:    pand %xmm1, %xmm4
+; CHECK-SSE2-NEXT:    psubd %xmm4, %xmm2
+; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm1
+; CHECK-SSE2-NEXT:    psrld $31, %xmm1
+; CHECK-SSE2-NEXT:    psrad $3, %xmm2
+; CHECK-SSE2-NEXT:    paddd %xmm1, %xmm2
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [25,25,25,25]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm4
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; CHECK-SSE2-NEXT:    psubd %xmm2, %xmm0
+; CHECK-SSE2-NEXT:    pcmpeqd %xmm3, %xmm0
+; CHECK-SSE2-NEXT:    psrld $31, %xmm0
+; CHECK-SSE2-NEXT:    retq
+;
+; CHECK-SSE41-LABEL: test_srem_odd_undef1:
+; CHECK-SSE41:       # %bb.0:
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [1374389535,1374389535,1374389535,1374389535]
+; CHECK-SSE41-NEXT:    pmuldq %xmm2, %xmm1
+; CHECK-SSE41-NEXT:    pmuldq %xmm0, %xmm2
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
+; CHECK-SSE41-NEXT:    movdqa %xmm2, %xmm1
+; CHECK-SSE41-NEXT:    psrld $31, %xmm1
+; CHECK-SSE41-NEXT:    psrad $3, %xmm2
+; CHECK-SSE41-NEXT:    paddd %xmm1, %xmm2
+; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm2
+; CHECK-SSE41-NEXT:    psubd %xmm2, %xmm0
+; CHECK-SSE41-NEXT:    pxor %xmm1, %xmm1
+; CHECK-SSE41-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-SSE41-NEXT:    psrld $31, %xmm0
+; CHECK-SSE41-NEXT:    retq
+;
+; CHECK-AVX1-LABEL: test_srem_odd_undef1:
+; CHECK-AVX1:       # %bb.0:
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; CHECK-AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [1374389535,1374389535,1374389535,1374389535]
+; CHECK-AVX1-NEXT:    vpmuldq %xmm2, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpmuldq %xmm2, %xmm0, %xmm2
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm1, %xmm2
+; CHECK-AVX1-NEXT:    vpsrad $3, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    retq
+;
+; CHECK-AVX2-LABEL: test_srem_odd_undef1:
+; CHECK-AVX2:       # %bb.0:
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [1374389535,1374389535,1374389535,1374389535]
+; CHECK-AVX2-NEXT:    vpmuldq %xmm2, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpmuldq %xmm2, %xmm0, %xmm2
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2],xmm1[3]
+; CHECK-AVX2-NEXT:    vpsrld $31, %xmm1, %xmm2
+; CHECK-AVX2-NEXT:    vpsrad $3, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [25,25,25,25]
+; CHECK-AVX2-NEXT:    vpmulld %xmm2, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    retq
+;
+; CHECK-AVX512VL-LABEL: test_srem_odd_undef1:
+; CHECK-AVX512VL:       # %bb.0:
+; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; CHECK-AVX512VL-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [1374389535,1374389535,1374389535,1374389535]
+; CHECK-AVX512VL-NEXT:    vpmuldq %xmm2, %xmm1, %xmm1
+; CHECK-AVX512VL-NEXT:    vpmuldq %xmm2, %xmm0, %xmm2
+; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; CHECK-AVX512VL-NEXT:    vpblendd {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2],xmm1[3]
+; CHECK-AVX512VL-NEXT:    vpsrld $31, %xmm1, %xmm2
+; CHECK-AVX512VL-NEXT:    vpsrad $3, %xmm1, %xmm1
+; CHECK-AVX512VL-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip){1to4}, %xmm1, %xmm1
+; CHECK-AVX512VL-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    retq
+  %srem = srem <4 x i32> %X, <i32 25, i32 25, i32 25, i32 25>
+  %cmp = icmp eq <4 x i32> %srem, <i32 0, i32 0, i32 undef, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+define <4 x i32> @test_srem_even_undef1(<4 x i32> %X) nounwind {
+; CHECK-SSE2-LABEL: test_srem_even_undef1:
+; CHECK-SSE2:       # %bb.0:
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [1374389535,1374389535,1374389535,1374389535]
+; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm2
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm3
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,3,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; CHECK-SSE2-NEXT:    pxor %xmm3, %xmm3
+; CHECK-SSE2-NEXT:    pxor %xmm4, %xmm4
+; CHECK-SSE2-NEXT:    pcmpgtd %xmm0, %xmm4
+; CHECK-SSE2-NEXT:    pand %xmm1, %xmm4
+; CHECK-SSE2-NEXT:    psubd %xmm4, %xmm2
+; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm1
+; CHECK-SSE2-NEXT:    psrld $31, %xmm1
+; CHECK-SSE2-NEXT:    psrad $5, %xmm2
+; CHECK-SSE2-NEXT:    paddd %xmm1, %xmm2
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [100,100,100,100]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm4
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; CHECK-SSE2-NEXT:    psubd %xmm2, %xmm0
+; CHECK-SSE2-NEXT:    pcmpeqd %xmm3, %xmm0
+; CHECK-SSE2-NEXT:    psrld $31, %xmm0
+; CHECK-SSE2-NEXT:    retq
+;
+; CHECK-SSE41-LABEL: test_srem_even_undef1:
+; CHECK-SSE41:       # %bb.0:
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [1374389535,1374389535,1374389535,1374389535]
+; CHECK-SSE41-NEXT:    pmuldq %xmm2, %xmm1
+; CHECK-SSE41-NEXT:    pmuldq %xmm0, %xmm2
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
+; CHECK-SSE41-NEXT:    movdqa %xmm2, %xmm1
+; CHECK-SSE41-NEXT:    psrld $31, %xmm1
+; CHECK-SSE41-NEXT:    psrad $5, %xmm2
+; CHECK-SSE41-NEXT:    paddd %xmm1, %xmm2
+; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm2
+; CHECK-SSE41-NEXT:    psubd %xmm2, %xmm0
+; CHECK-SSE41-NEXT:    pxor %xmm1, %xmm1
+; CHECK-SSE41-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-SSE41-NEXT:    psrld $31, %xmm0
+; CHECK-SSE41-NEXT:    retq
+;
+; CHECK-AVX1-LABEL: test_srem_even_undef1:
+; CHECK-AVX1:       # %bb.0:
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; CHECK-AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [1374389535,1374389535,1374389535,1374389535]
+; CHECK-AVX1-NEXT:    vpmuldq %xmm2, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpmuldq %xmm2, %xmm0, %xmm2
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm1, %xmm2
+; CHECK-AVX1-NEXT:    vpsrad $5, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    retq
+;
+; CHECK-AVX2-LABEL: test_srem_even_undef1:
+; CHECK-AVX2:       # %bb.0:
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [1374389535,1374389535,1374389535,1374389535]
+; CHECK-AVX2-NEXT:    vpmuldq %xmm2, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpmuldq %xmm2, %xmm0, %xmm2
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2],xmm1[3]
+; CHECK-AVX2-NEXT:    vpsrld $31, %xmm1, %xmm2
+; CHECK-AVX2-NEXT:    vpsrad $5, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [100,100,100,100]
+; CHECK-AVX2-NEXT:    vpmulld %xmm2, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    retq
+;
+; CHECK-AVX512VL-LABEL: test_srem_even_undef1:
+; CHECK-AVX512VL:       # %bb.0:
+; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; CHECK-AVX512VL-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [1374389535,1374389535,1374389535,1374389535]
+; CHECK-AVX512VL-NEXT:    vpmuldq %xmm2, %xmm1, %xmm1
+; CHECK-AVX512VL-NEXT:    vpmuldq %xmm2, %xmm0, %xmm2
+; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; CHECK-AVX512VL-NEXT:    vpblendd {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2],xmm1[3]
+; CHECK-AVX512VL-NEXT:    vpsrld $31, %xmm1, %xmm2
+; CHECK-AVX512VL-NEXT:    vpsrad $5, %xmm1, %xmm1
+; CHECK-AVX512VL-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip){1to4}, %xmm1, %xmm1
+; CHECK-AVX512VL-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    retq
+  %srem = srem <4 x i32> %X, <i32 100, i32 100, i32 100, i32 100>
+  %cmp = icmp eq <4 x i32> %srem, <i32 0, i32 0, i32 undef, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+;------------------------------------------------------------------------------;
+; Negative tests
+;------------------------------------------------------------------------------;
+
+define <4 x i32> @test_srem_one_eq(<4 x i32> %X) nounwind {
+; CHECK-SSE-LABEL: test_srem_one_eq:
+; CHECK-SSE:       # %bb.0:
+; CHECK-SSE-NEXT:    movaps {{.*#+}} xmm0 = [1,1,1,1]
+; CHECK-SSE-NEXT:    retq
+;
+; CHECK-AVX1-LABEL: test_srem_one_eq:
+; CHECK-AVX1:       # %bb.0:
+; CHECK-AVX1-NEXT:    vmovaps {{.*#+}} xmm0 = [1,1,1,1]
+; CHECK-AVX1-NEXT:    retq
+;
+; CHECK-AVX2-LABEL: test_srem_one_eq:
+; CHECK-AVX2:       # %bb.0:
+; CHECK-AVX2-NEXT:    vbroadcastss {{.*#+}} xmm0 = [1,1,1,1]
+; CHECK-AVX2-NEXT:    retq
+;
+; CHECK-AVX512VL-LABEL: test_srem_one_eq:
+; CHECK-AVX512VL:       # %bb.0:
+; CHECK-AVX512VL-NEXT:    vbroadcastss {{.*#+}} xmm0 = [1,1,1,1]
+; CHECK-AVX512VL-NEXT:    retq
+  %srem = srem <4 x i32> %X, <i32 1, i32 1, i32 1, i32 1>
+  %cmp = icmp eq <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+define <4 x i32> @test_srem_one_ne(<4 x i32> %X) nounwind {
+; CHECK-SSE-LABEL: test_srem_one_ne:
+; CHECK-SSE:       # %bb.0:
+; CHECK-SSE-NEXT:    xorps %xmm0, %xmm0
+; CHECK-SSE-NEXT:    retq
+;
+; CHECK-AVX-LABEL: test_srem_one_ne:
+; CHECK-AVX:       # %bb.0:
+; CHECK-AVX-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; CHECK-AVX-NEXT:    retq
+  %srem = srem <4 x i32> %X, <i32 1, i32 1, i32 1, i32 1>
+  %cmp = icmp ne <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; We can lower remainder of division by powers of two much better elsewhere.
+define <4 x i32> @test_srem_pow2(<4 x i32> %X) nounwind {
+; CHECK-SSE-LABEL: test_srem_pow2:
+; CHECK-SSE:       # %bb.0:
+; CHECK-SSE-NEXT:    movdqa %xmm0, %xmm1
+; CHECK-SSE-NEXT:    psrad $31, %xmm1
+; CHECK-SSE-NEXT:    psrld $28, %xmm1
+; CHECK-SSE-NEXT:    paddd %xmm0, %xmm1
+; CHECK-SSE-NEXT:    pand {{.*}}(%rip), %xmm1
+; CHECK-SSE-NEXT:    psubd %xmm1, %xmm0
+; CHECK-SSE-NEXT:    pxor %xmm1, %xmm1
+; CHECK-SSE-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-SSE-NEXT:    psrld $31, %xmm0
+; CHECK-SSE-NEXT:    retq
+;
+; CHECK-AVX1-LABEL: test_srem_pow2:
+; CHECK-AVX1:       # %bb.0:
+; CHECK-AVX1-NEXT:    vpsrad $31, %xmm0, %xmm1
+; CHECK-AVX1-NEXT:    vpsrld $28, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm1
+; CHECK-AVX1-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    retq
+;
+; CHECK-AVX2-LABEL: test_srem_pow2:
+; CHECK-AVX2:       # %bb.0:
+; CHECK-AVX2-NEXT:    vpsrad $31, %xmm0, %xmm1
+; CHECK-AVX2-NEXT:    vpsrld $28, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm1
+; CHECK-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [4294967280,4294967280,4294967280,4294967280]
+; CHECK-AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    retq
+;
+; CHECK-AVX512VL-LABEL: test_srem_pow2:
+; CHECK-AVX512VL:       # %bb.0:
+; CHECK-AVX512VL-NEXT:    vpsrad $31, %xmm0, %xmm1
+; CHECK-AVX512VL-NEXT:    vpsrld $28, %xmm1, %xmm1
+; CHECK-AVX512VL-NEXT:    vpaddd %xmm1, %xmm0, %xmm1
+; CHECK-AVX512VL-NEXT:    vpandd {{.*}}(%rip){1to4}, %xmm1, %xmm1
+; CHECK-AVX512VL-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    retq
+  %srem = srem <4 x i32> %X, <i32 16, i32 16, i32 16, i32 16>
+  %cmp = icmp eq <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; We could lower remainder of division by INT_MIN much better elsewhere.
+define <4 x i32> @test_srem_int_min(<4 x i32> %X) nounwind {
+; CHECK-SSE-LABEL: test_srem_int_min:
+; CHECK-SSE:       # %bb.0:
+; CHECK-SSE-NEXT:    movdqa %xmm0, %xmm1
+; CHECK-SSE-NEXT:    psrad $31, %xmm1
+; CHECK-SSE-NEXT:    psrld $1, %xmm1
+; CHECK-SSE-NEXT:    paddd %xmm0, %xmm1
+; CHECK-SSE-NEXT:    pand {{.*}}(%rip), %xmm1
+; CHECK-SSE-NEXT:    psubd %xmm1, %xmm0
+; CHECK-SSE-NEXT:    pxor %xmm1, %xmm1
+; CHECK-SSE-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-SSE-NEXT:    psrld $31, %xmm0
+; CHECK-SSE-NEXT:    retq
+;
+; CHECK-AVX1-LABEL: test_srem_int_min:
+; CHECK-AVX1:       # %bb.0:
+; CHECK-AVX1-NEXT:    vpsrad $31, %xmm0, %xmm1
+; CHECK-AVX1-NEXT:    vpsrld $1, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm1
+; CHECK-AVX1-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    retq
+;
+; CHECK-AVX2-LABEL: test_srem_int_min:
+; CHECK-AVX2:       # %bb.0:
+; CHECK-AVX2-NEXT:    vpsrad $31, %xmm0, %xmm1
+; CHECK-AVX2-NEXT:    vpsrld $1, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm1
+; CHECK-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648]
+; CHECK-AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    retq
+;
+; CHECK-AVX512VL-LABEL: test_srem_int_min:
+; CHECK-AVX512VL:       # %bb.0:
+; CHECK-AVX512VL-NEXT:    vpsrad $31, %xmm0, %xmm1
+; CHECK-AVX512VL-NEXT:    vpsrld $1, %xmm1, %xmm1
+; CHECK-AVX512VL-NEXT:    vpaddd %xmm1, %xmm0, %xmm1
+; CHECK-AVX512VL-NEXT:    vpandd {{.*}}(%rip){1to4}, %xmm1, %xmm1
+; CHECK-AVX512VL-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    retq
+  %srem = srem <4 x i32> %X, <i32 2147483648, i32 2147483648, i32 2147483648, i32 2147483648>
+  %cmp = icmp eq <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; We could lower remainder of division by all-ones much better elsewhere.
+define <4 x i32> @test_srem_allones(<4 x i32> %X) nounwind {
+; CHECK-SSE-LABEL: test_srem_allones:
+; CHECK-SSE:       # %bb.0:
+; CHECK-SSE-NEXT:    movaps {{.*#+}} xmm0 = [1,1,1,1]
+; CHECK-SSE-NEXT:    retq
+;
+; CHECK-AVX1-LABEL: test_srem_allones:
+; CHECK-AVX1:       # %bb.0:
+; CHECK-AVX1-NEXT:    vmovaps {{.*#+}} xmm0 = [1,1,1,1]
+; CHECK-AVX1-NEXT:    retq
+;
+; CHECK-AVX2-LABEL: test_srem_allones:
+; CHECK-AVX2:       # %bb.0:
+; CHECK-AVX2-NEXT:    vbroadcastss {{.*#+}} xmm0 = [1,1,1,1]
+; CHECK-AVX2-NEXT:    retq
+;
+; CHECK-AVX512VL-LABEL: test_srem_allones:
+; CHECK-AVX512VL:       # %bb.0:
+; CHECK-AVX512VL-NEXT:    vbroadcastss {{.*#+}} xmm0 = [1,1,1,1]
+; CHECK-AVX512VL-NEXT:    retq
+  %srem = srem <4 x i32> %X, <i32 4294967295, i32 4294967295, i32 4294967295, i32 4294967295>
+  %cmp = icmp eq <4 x i32> %srem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
diff --git a/llvm/test/CodeGen/X86/srem-seteq.ll b/llvm/test/CodeGen/X86/srem-seteq.ll
new file mode 100644
index 000000000..67fe5f4c5
--- /dev/null
+++ b/llvm/test/CodeGen/X86/srem-seteq.ll
@@ -0,0 +1,388 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=i686-unknown-linux-gnu -mattr=+cmov < %s | FileCheck %s --check-prefixes=CHECK,X86
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,X64
+
+;------------------------------------------------------------------------------;
+; Odd divisors
+;------------------------------------------------------------------------------;
+
+define i32 @test_srem_odd(i32 %X) nounwind {
+; X86-LABEL: test_srem_odd:
+; X86:       # %bb.0:
+; X86-NEXT:    imull $-858993459, {{[0-9]+}}(%esp), %ecx # imm = 0xCCCCCCCD
+; X86-NEXT:    addl $429496729, %ecx # imm = 0x19999999
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    cmpl $858993459, %ecx # imm = 0x33333333
+; X86-NEXT:    setb %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: test_srem_odd:
+; X64:       # %bb.0:
+; X64-NEXT:    imull $-858993459, %edi, %ecx # imm = 0xCCCCCCCD
+; X64-NEXT:    addl $429496729, %ecx # imm = 0x19999999
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpl $858993459, %ecx # imm = 0x33333333
+; X64-NEXT:    setb %al
+; X64-NEXT:    retq
+  %srem = srem i32 %X, 5
+  %cmp = icmp eq i32 %srem, 0
+  %ret = zext i1 %cmp to i32
+  ret i32 %ret
+}
+
+define i32 @test_srem_odd_25(i32 %X) nounwind {
+; X86-LABEL: test_srem_odd_25:
+; X86:       # %bb.0:
+; X86-NEXT:    imull $-1030792151, {{[0-9]+}}(%esp), %ecx # imm = 0xC28F5C29
+; X86-NEXT:    addl $85899345, %ecx # imm = 0x51EB851
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    cmpl $171798691, %ecx # imm = 0xA3D70A3
+; X86-NEXT:    setb %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: test_srem_odd_25:
+; X64:       # %bb.0:
+; X64-NEXT:    imull $-1030792151, %edi, %ecx # imm = 0xC28F5C29
+; X64-NEXT:    addl $85899345, %ecx # imm = 0x51EB851
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpl $171798691, %ecx # imm = 0xA3D70A3
+; X64-NEXT:    setb %al
+; X64-NEXT:    retq
+  %srem = srem i32 %X, 25
+  %cmp = icmp eq i32 %srem, 0
+  %ret = zext i1 %cmp to i32
+  ret i32 %ret
+}
+
+; This is like test_srem_odd, except the divisor has bit 30 set.
+define i32 @test_srem_odd_bit30(i32 %X) nounwind {
+; X86-LABEL: test_srem_odd_bit30:
+; X86:       # %bb.0:
+; X86-NEXT:    imull $1789569707, {{[0-9]+}}(%esp), %ecx # imm = 0x6AAAAAAB
+; X86-NEXT:    incl %ecx
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    cmpl $3, %ecx
+; X86-NEXT:    setb %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: test_srem_odd_bit30:
+; X64:       # %bb.0:
+; X64-NEXT:    imull $1789569707, %edi, %ecx # imm = 0x6AAAAAAB
+; X64-NEXT:    incl %ecx
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpl $3, %ecx
+; X64-NEXT:    setb %al
+; X64-NEXT:    retq
+  %srem = srem i32 %X, 1073741827
+  %cmp = icmp eq i32 %srem, 0
+  %ret = zext i1 %cmp to i32
+  ret i32 %ret
+}
+
+; This is like test_srem_odd, except the divisor has bit 31 set.
+define i32 @test_srem_odd_bit31(i32 %X) nounwind {
+; X86-LABEL: test_srem_odd_bit31:
+; X86:       # %bb.0:
+; X86-NEXT:    imull $-715827883, {{[0-9]+}}(%esp), %ecx # imm = 0xD5555555
+; X86-NEXT:    incl %ecx
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    cmpl $3, %ecx
+; X86-NEXT:    setb %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: test_srem_odd_bit31:
+; X64:       # %bb.0:
+; X64-NEXT:    imull $-715827883, %edi, %ecx # imm = 0xD5555555
+; X64-NEXT:    incl %ecx
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpl $3, %ecx
+; X64-NEXT:    setb %al
+; X64-NEXT:    retq
+  %srem = srem i32 %X, 2147483651
+  %cmp = icmp eq i32 %srem, 0
+  %ret = zext i1 %cmp to i32
+  ret i32 %ret
+}
+
+;------------------------------------------------------------------------------;
+; Even divisors
+;------------------------------------------------------------------------------;
+
+define i16 @test_srem_even(i16 %X) nounwind {
+; X86-LABEL: test_srem_even:
+; X86:       # %bb.0:
+; X86-NEXT:    imull $28087, {{[0-9]+}}(%esp), %eax # imm = 0x6DB7
+; X86-NEXT:    addl $4680, %eax # imm = 0x1248
+; X86-NEXT:    rorw %ax
+; X86-NEXT:    movzwl %ax, %ecx
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    cmpl $4680, %ecx # imm = 0x1248
+; X86-NEXT:    seta %al
+; X86-NEXT:    # kill: def $ax killed $ax killed $eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: test_srem_even:
+; X64:       # %bb.0:
+; X64-NEXT:    imull $28087, %edi, %eax # imm = 0x6DB7
+; X64-NEXT:    addl $4680, %eax # imm = 0x1248
+; X64-NEXT:    rorw %ax
+; X64-NEXT:    movzwl %ax, %ecx
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpl $4680, %ecx # imm = 0x1248
+; X64-NEXT:    seta %al
+; X64-NEXT:    # kill: def $ax killed $ax killed $eax
+; X64-NEXT:    retq
+  %srem = srem i16 %X, 14
+  %cmp = icmp ne i16 %srem, 0
+  %ret = zext i1 %cmp to i16
+  ret i16 %ret
+}
+
+define i32 @test_srem_even_100(i32 %X) nounwind {
+; X86-LABEL: test_srem_even_100:
+; X86:       # %bb.0:
+; X86-NEXT:    imull $-1030792151, {{[0-9]+}}(%esp), %ecx # imm = 0xC28F5C29
+; X86-NEXT:    addl $85899344, %ecx # imm = 0x51EB850
+; X86-NEXT:    rorl $2, %ecx
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    cmpl $42949673, %ecx # imm = 0x28F5C29
+; X86-NEXT:    setb %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: test_srem_even_100:
+; X64:       # %bb.0:
+; X64-NEXT:    imull $-1030792151, %edi, %ecx # imm = 0xC28F5C29
+; X64-NEXT:    addl $85899344, %ecx # imm = 0x51EB850
+; X64-NEXT:    rorl $2, %ecx
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpl $42949673, %ecx # imm = 0x28F5C29
+; X64-NEXT:    setb %al
+; X64-NEXT:    retq
+  %srem = srem i32 %X, 100
+  %cmp = icmp eq i32 %srem, 0
+  %ret = zext i1 %cmp to i32
+  ret i32 %ret
+}
+
+; This is like test_srem_even, except the divisor has bit 30 set.
+define i32 @test_srem_even_bit30(i32 %X) nounwind {
+; X86-LABEL: test_srem_even_bit30:
+; X86:       # %bb.0:
+; X86-NEXT:    imull $-51622203, {{[0-9]+}}(%esp), %ecx # imm = 0xFCEC4EC5
+; X86-NEXT:    addl $8, %ecx
+; X86-NEXT:    rorl $3, %ecx
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    cmpl $3, %ecx
+; X86-NEXT:    setb %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: test_srem_even_bit30:
+; X64:       # %bb.0:
+; X64-NEXT:    imull $-51622203, %edi, %ecx # imm = 0xFCEC4EC5
+; X64-NEXT:    addl $8, %ecx
+; X64-NEXT:    rorl $3, %ecx
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpl $3, %ecx
+; X64-NEXT:    setb %al
+; X64-NEXT:    retq
+  %srem = srem i32 %X, 1073741928
+  %cmp = icmp eq i32 %srem, 0
+  %ret = zext i1 %cmp to i32
+  ret i32 %ret
+}
+
+; This is like test_srem_odd, except the divisor has bit 31 set.
+define i32 @test_srem_even_bit31(i32 %X) nounwind {
+; X86-LABEL: test_srem_even_bit31:
+; X86:       # %bb.0:
+; X86-NEXT:    imull $-989526779, {{[0-9]+}}(%esp), %ecx # imm = 0xC5050505
+; X86-NEXT:    addl $2, %ecx
+; X86-NEXT:    rorl %ecx
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    cmpl $3, %ecx
+; X86-NEXT:    setb %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: test_srem_even_bit31:
+; X64:       # %bb.0:
+; X64-NEXT:    imull $-989526779, %edi, %ecx # imm = 0xC5050505
+; X64-NEXT:    addl $2, %ecx
+; X64-NEXT:    rorl %ecx
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpl $3, %ecx
+; X64-NEXT:    setb %al
+; X64-NEXT:    retq
+  %srem = srem i32 %X, 2147483750
+  %cmp = icmp eq i32 %srem, 0
+  %ret = zext i1 %cmp to i32
+  ret i32 %ret
+}
+
+;------------------------------------------------------------------------------;
+; Special case
+;------------------------------------------------------------------------------;
+
+; 'NE' predicate is fine too.
+define i32 @test_srem_odd_setne(i32 %X) nounwind {
+; X86-LABEL: test_srem_odd_setne:
+; X86:       # %bb.0:
+; X86-NEXT:    imull $-858993459, {{[0-9]+}}(%esp), %ecx # imm = 0xCCCCCCCD
+; X86-NEXT:    addl $429496729, %ecx # imm = 0x19999999
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    cmpl $858993458, %ecx # imm = 0x33333332
+; X86-NEXT:    seta %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: test_srem_odd_setne:
+; X64:       # %bb.0:
+; X64-NEXT:    imull $-858993459, %edi, %ecx # imm = 0xCCCCCCCD
+; X64-NEXT:    addl $429496729, %ecx # imm = 0x19999999
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpl $858993458, %ecx # imm = 0x33333332
+; X64-NEXT:    seta %al
+; X64-NEXT:    retq
+  %srem = srem i32 %X, 5
+  %cmp = icmp ne i32 %srem, 0
+  %ret = zext i1 %cmp to i32
+  ret i32 %ret
+}
+
+; The fold is only valid for positive divisors, negative-ones should be negated.
+define i32 @test_srem_negative_odd(i32 %X) nounwind {
+; X86-LABEL: test_srem_negative_odd:
+; X86:       # %bb.0:
+; X86-NEXT:    imull $-858993459, {{[0-9]+}}(%esp), %ecx # imm = 0xCCCCCCCD
+; X86-NEXT:    addl $429496729, %ecx # imm = 0x19999999
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    cmpl $858993458, %ecx # imm = 0x33333332
+; X86-NEXT:    seta %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: test_srem_negative_odd:
+; X64:       # %bb.0:
+; X64-NEXT:    imull $-858993459, %edi, %ecx # imm = 0xCCCCCCCD
+; X64-NEXT:    addl $429496729, %ecx # imm = 0x19999999
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpl $858993458, %ecx # imm = 0x33333332
+; X64-NEXT:    seta %al
+; X64-NEXT:    retq
+  %srem = srem i32 %X, -5
+  %cmp = icmp ne i32 %srem, 0
+  %ret = zext i1 %cmp to i32
+  ret i32 %ret
+}
+define i32 @test_srem_negative_even(i32 %X) nounwind {
+; X86-LABEL: test_srem_negative_even:
+; X86:       # %bb.0:
+; X86-NEXT:    imull $-1227133513, {{[0-9]+}}(%esp), %ecx # imm = 0xB6DB6DB7
+; X86-NEXT:    addl $306783378, %ecx # imm = 0x12492492
+; X86-NEXT:    rorl %ecx
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    cmpl $306783378, %ecx # imm = 0x12492492
+; X86-NEXT:    seta %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: test_srem_negative_even:
+; X64:       # %bb.0:
+; X64-NEXT:    imull $-1227133513, %edi, %ecx # imm = 0xB6DB6DB7
+; X64-NEXT:    addl $306783378, %ecx # imm = 0x12492492
+; X64-NEXT:    rorl %ecx
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpl $306783378, %ecx # imm = 0x12492492
+; X64-NEXT:    seta %al
+; X64-NEXT:    retq
+  %srem = srem i32 %X, -14
+  %cmp = icmp ne i32 %srem, 0
+  %ret = zext i1 %cmp to i32
+  ret i32 %ret
+}
+
+;------------------------------------------------------------------------------;
+; Negative tests
+;------------------------------------------------------------------------------;
+
+; We can lower remainder of division by one much better elsewhere.
+define i32 @test_srem_one(i32 %X) nounwind {
+; CHECK-LABEL: test_srem_one:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl $1, %eax
+; CHECK-NEXT:    ret{{[l|q]}}
+  %srem = srem i32 %X, 1
+  %cmp = icmp eq i32 %srem, 0
+  %ret = zext i1 %cmp to i32
+  ret i32 %ret
+}
+
+; We can lower remainder of division by powers of two much better elsewhere.
+define i32 @test_srem_pow2(i32 %X) nounwind {
+; X86-LABEL: test_srem_pow2:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    leal 15(%ecx), %edx
+; X86-NEXT:    testl %ecx, %ecx
+; X86-NEXT:    cmovnsl %ecx, %edx
+; X86-NEXT:    andl $-16, %edx
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    cmpl %edx, %ecx
+; X86-NEXT:    sete %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: test_srem_pow2:
+; X64:       # %bb.0:
+; X64-NEXT:    # kill: def $edi killed $edi def $rdi
+; X64-NEXT:    leal 15(%rdi), %ecx
+; X64-NEXT:    testl %edi, %edi
+; X64-NEXT:    cmovnsl %edi, %ecx
+; X64-NEXT:    andl $-16, %ecx
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpl %ecx, %edi
+; X64-NEXT:    sete %al
+; X64-NEXT:    retq
+  %srem = srem i32 %X, 16
+  %cmp = icmp eq i32 %srem, 0
+  %ret = zext i1 %cmp to i32
+  ret i32 %ret
+}
+
+; The fold is only valid for positive divisors, and we can't negate INT_MIN.
+define i32 @test_srem_int_min(i32 %X) nounwind {
+; X86-LABEL: test_srem_int_min:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    leal 2147483647(%ecx), %edx
+; X86-NEXT:    testl %ecx, %ecx
+; X86-NEXT:    cmovnsl %ecx, %edx
+; X86-NEXT:    andl $-2147483648, %edx # imm = 0x80000000
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    addl %ecx, %edx
+; X86-NEXT:    sete %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: test_srem_int_min:
+; X64:       # %bb.0:
+; X64-NEXT:    # kill: def $edi killed $edi def $rdi
+; X64-NEXT:    leal 2147483647(%rdi), %ecx
+; X64-NEXT:    testl %edi, %edi
+; X64-NEXT:    cmovnsl %edi, %ecx
+; X64-NEXT:    andl $-2147483648, %ecx # imm = 0x80000000
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    addl %edi, %ecx
+; X64-NEXT:    sete %al
+; X64-NEXT:    retq
+  %srem = srem i32 %X, 2147483648
+  %cmp = icmp eq i32 %srem, 0
+  %ret = zext i1 %cmp to i32
+  ret i32 %ret
+}
+
+; We can lower remainder of division by all-ones much better elsewhere.
+define i32 @test_srem_allones(i32 %X) nounwind {
+; CHECK-LABEL: test_srem_allones:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl $1, %eax
+; CHECK-NEXT:    ret{{[l|q]}}
+  %srem = srem i32 %X, 4294967295
+  %cmp = icmp eq i32 %srem, 0
+  %ret = zext i1 %cmp to i32
+  ret i32 %ret
+}
diff --git a/llvm/test/CodeGen/X86/srem-vector-lkk.ll b/llvm/test/CodeGen/X86/srem-vector-lkk.ll
new file mode 100644
index 000000000..19f793274
--- /dev/null
+++ b/llvm/test/CodeGen/X86/srem-vector-lkk.ll
@@ -0,0 +1,556 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx  | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX1
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
+
+define <4 x i16> @fold_srem_vec_1(<4 x i16> %x) {
+; SSE-LABEL: fold_srem_vec_1:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pextrw $3, %xmm0, %eax
+; SSE-NEXT:    movswl %ax, %ecx
+; SSE-NEXT:    imull $32081, %ecx, %ecx # imm = 0x7D51
+; SSE-NEXT:    shrl $16, %ecx
+; SSE-NEXT:    subl %eax, %ecx
+; SSE-NEXT:    movzwl %cx, %ecx
+; SSE-NEXT:    movswl %cx, %edx
+; SSE-NEXT:    shrl $15, %ecx
+; SSE-NEXT:    sarl $9, %edx
+; SSE-NEXT:    addl %ecx, %edx
+; SSE-NEXT:    imull $-1003, %edx, %ecx # imm = 0xFC15
+; SSE-NEXT:    subl %ecx, %eax
+; SSE-NEXT:    movd %xmm0, %ecx
+; SSE-NEXT:    movswl %cx, %edx
+; SSE-NEXT:    imull $-21385, %edx, %edx # imm = 0xAC77
+; SSE-NEXT:    shrl $16, %edx
+; SSE-NEXT:    addl %ecx, %edx
+; SSE-NEXT:    movzwl %dx, %edx
+; SSE-NEXT:    movswl %dx, %esi
+; SSE-NEXT:    shrl $15, %edx
+; SSE-NEXT:    sarl $6, %esi
+; SSE-NEXT:    addl %edx, %esi
+; SSE-NEXT:    imull $95, %esi, %edx
+; SSE-NEXT:    subl %edx, %ecx
+; SSE-NEXT:    movd %ecx, %xmm1
+; SSE-NEXT:    pextrw $1, %xmm0, %ecx
+; SSE-NEXT:    movswl %cx, %edx
+; SSE-NEXT:    imull $-16913, %edx, %edx # imm = 0xBDEF
+; SSE-NEXT:    movl %edx, %esi
+; SSE-NEXT:    shrl $31, %esi
+; SSE-NEXT:    sarl $21, %edx
+; SSE-NEXT:    addl %esi, %edx
+; SSE-NEXT:    imull $-124, %edx, %edx
+; SSE-NEXT:    subl %edx, %ecx
+; SSE-NEXT:    pinsrw $1, %ecx, %xmm1
+; SSE-NEXT:    pextrw $2, %xmm0, %ecx
+; SSE-NEXT:    movswl %cx, %edx
+; SSE-NEXT:    imull $2675, %edx, %edx # imm = 0xA73
+; SSE-NEXT:    movl %edx, %esi
+; SSE-NEXT:    shrl $31, %esi
+; SSE-NEXT:    sarl $18, %edx
+; SSE-NEXT:    addl %esi, %edx
+; SSE-NEXT:    imull $98, %edx, %edx
+; SSE-NEXT:    subl %edx, %ecx
+; SSE-NEXT:    pinsrw $2, %ecx, %xmm1
+; SSE-NEXT:    pinsrw $3, %eax, %xmm1
+; SSE-NEXT:    movdqa %xmm1, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: fold_srem_vec_1:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpextrw $3, %xmm0, %eax
+; AVX-NEXT:    movswl %ax, %ecx
+; AVX-NEXT:    imull $32081, %ecx, %ecx # imm = 0x7D51
+; AVX-NEXT:    shrl $16, %ecx
+; AVX-NEXT:    subl %eax, %ecx
+; AVX-NEXT:    movzwl %cx, %ecx
+; AVX-NEXT:    movswl %cx, %edx
+; AVX-NEXT:    shrl $15, %ecx
+; AVX-NEXT:    sarl $9, %edx
+; AVX-NEXT:    addl %ecx, %edx
+; AVX-NEXT:    imull $-1003, %edx, %ecx # imm = 0xFC15
+; AVX-NEXT:    subl %ecx, %eax
+; AVX-NEXT:    vmovd %xmm0, %ecx
+; AVX-NEXT:    movswl %cx, %edx
+; AVX-NEXT:    imull $-21385, %edx, %edx # imm = 0xAC77
+; AVX-NEXT:    shrl $16, %edx
+; AVX-NEXT:    addl %ecx, %edx
+; AVX-NEXT:    movzwl %dx, %edx
+; AVX-NEXT:    movswl %dx, %esi
+; AVX-NEXT:    shrl $15, %edx
+; AVX-NEXT:    sarl $6, %esi
+; AVX-NEXT:    addl %edx, %esi
+; AVX-NEXT:    imull $95, %esi, %edx
+; AVX-NEXT:    subl %edx, %ecx
+; AVX-NEXT:    vmovd %ecx, %xmm1
+; AVX-NEXT:    vpextrw $1, %xmm0, %ecx
+; AVX-NEXT:    movswl %cx, %edx
+; AVX-NEXT:    imull $-16913, %edx, %edx # imm = 0xBDEF
+; AVX-NEXT:    movl %edx, %esi
+; AVX-NEXT:    shrl $31, %esi
+; AVX-NEXT:    sarl $21, %edx
+; AVX-NEXT:    addl %esi, %edx
+; AVX-NEXT:    imull $-124, %edx, %edx
+; AVX-NEXT:    subl %edx, %ecx
+; AVX-NEXT:    vpinsrw $1, %ecx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrw $2, %xmm0, %ecx
+; AVX-NEXT:    movswl %cx, %edx
+; AVX-NEXT:    imull $2675, %edx, %edx # imm = 0xA73
+; AVX-NEXT:    movl %edx, %esi
+; AVX-NEXT:    shrl $31, %esi
+; AVX-NEXT:    sarl $18, %edx
+; AVX-NEXT:    addl %esi, %edx
+; AVX-NEXT:    imull $98, %edx, %edx
+; AVX-NEXT:    subl %edx, %ecx
+; AVX-NEXT:    vpinsrw $2, %ecx, %xmm1, %xmm0
+; AVX-NEXT:    vpinsrw $3, %eax, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %1 = srem <4 x i16> %x, <i16 95, i16 -124, i16 98, i16 -1003>
+  ret <4 x i16> %1
+}
+
+define <4 x i16> @fold_srem_vec_2(<4 x i16> %x) {
+; SSE-LABEL: fold_srem_vec_2:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqa {{.*#+}} xmm1 = [44151,44151,44151,44151,44151,44151,44151,44151]
+; SSE-NEXT:    pmulhw %xmm0, %xmm1
+; SSE-NEXT:    paddw %xmm0, %xmm1
+; SSE-NEXT:    movdqa %xmm1, %xmm2
+; SSE-NEXT:    psrlw $15, %xmm2
+; SSE-NEXT:    psraw $6, %xmm1
+; SSE-NEXT:    paddw %xmm2, %xmm1
+; SSE-NEXT:    pmullw {{.*}}(%rip), %xmm1
+; SSE-NEXT:    psubw %xmm1, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: fold_srem_vec_2:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpmulhw {{.*}}(%rip), %xmm0, %xmm1
+; AVX-NEXT:    vpaddw %xmm0, %xmm1, %xmm1
+; AVX-NEXT:    vpsrlw $15, %xmm1, %xmm2
+; AVX-NEXT:    vpsraw $6, %xmm1, %xmm1
+; AVX-NEXT:    vpaddw %xmm2, %xmm1, %xmm1
+; AVX-NEXT:    vpmullw {{.*}}(%rip), %xmm1, %xmm1
+; AVX-NEXT:    vpsubw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %1 = srem <4 x i16> %x, <i16 95, i16 95, i16 95, i16 95>
+  ret <4 x i16> %1
+}
+
+
+; Don't fold if we can combine srem with sdiv.
+define <4 x i16> @combine_srem_sdiv(<4 x i16> %x) {
+; SSE-LABEL: combine_srem_sdiv:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqa {{.*#+}} xmm1 = [44151,44151,44151,44151,44151,44151,44151,44151]
+; SSE-NEXT:    pmulhw %xmm0, %xmm1
+; SSE-NEXT:    paddw %xmm0, %xmm1
+; SSE-NEXT:    movdqa %xmm1, %xmm2
+; SSE-NEXT:    psrlw $15, %xmm2
+; SSE-NEXT:    psraw $6, %xmm1
+; SSE-NEXT:    paddw %xmm2, %xmm1
+; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [95,95,95,95,95,95,95,95]
+; SSE-NEXT:    pmullw %xmm1, %xmm2
+; SSE-NEXT:    psubw %xmm2, %xmm0
+; SSE-NEXT:    paddw %xmm1, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: combine_srem_sdiv:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpmulhw {{.*}}(%rip), %xmm0, %xmm1
+; AVX-NEXT:    vpaddw %xmm0, %xmm1, %xmm1
+; AVX-NEXT:    vpsrlw $15, %xmm1, %xmm2
+; AVX-NEXT:    vpsraw $6, %xmm1, %xmm1
+; AVX-NEXT:    vpaddw %xmm2, %xmm1, %xmm1
+; AVX-NEXT:    vpmullw {{.*}}(%rip), %xmm1, %xmm2
+; AVX-NEXT:    vpsubw %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %1 = srem <4 x i16> %x, <i16 95, i16 95, i16 95, i16 95>
+  %2 = sdiv <4 x i16> %x, <i16 95, i16 95, i16 95, i16 95>
+  %3 = add <4 x i16> %1, %2
+  ret <4 x i16> %3
+}
+
+; Don't fold for divisors that are a power of two.
+define <4 x i16> @dont_fold_srem_power_of_two(<4 x i16> %x) {
+; SSE-LABEL: dont_fold_srem_power_of_two:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pextrw $1, %xmm0, %eax
+; SSE-NEXT:    leal 31(%rax), %ecx
+; SSE-NEXT:    testw %ax, %ax
+; SSE-NEXT:    cmovnsl %eax, %ecx
+; SSE-NEXT:    andl $-32, %ecx
+; SSE-NEXT:    subl %ecx, %eax
+; SSE-NEXT:    movd %xmm0, %ecx
+; SSE-NEXT:    leal 63(%rcx), %edx
+; SSE-NEXT:    testw %cx, %cx
+; SSE-NEXT:    cmovnsl %ecx, %edx
+; SSE-NEXT:    andl $-64, %edx
+; SSE-NEXT:    subl %edx, %ecx
+; SSE-NEXT:    movd %ecx, %xmm1
+; SSE-NEXT:    pinsrw $1, %eax, %xmm1
+; SSE-NEXT:    pextrw $2, %xmm0, %eax
+; SSE-NEXT:    leal 7(%rax), %ecx
+; SSE-NEXT:    testw %ax, %ax
+; SSE-NEXT:    cmovnsl %eax, %ecx
+; SSE-NEXT:    andl $-8, %ecx
+; SSE-NEXT:    subl %ecx, %eax
+; SSE-NEXT:    pinsrw $2, %eax, %xmm1
+; SSE-NEXT:    pextrw $3, %xmm0, %eax
+; SSE-NEXT:    movswl %ax, %ecx
+; SSE-NEXT:    imull $-21385, %ecx, %ecx # imm = 0xAC77
+; SSE-NEXT:    shrl $16, %ecx
+; SSE-NEXT:    addl %eax, %ecx
+; SSE-NEXT:    movzwl %cx, %ecx
+; SSE-NEXT:    movswl %cx, %edx
+; SSE-NEXT:    shrl $15, %ecx
+; SSE-NEXT:    sarl $6, %edx
+; SSE-NEXT:    addl %ecx, %edx
+; SSE-NEXT:    imull $95, %edx, %ecx
+; SSE-NEXT:    subl %ecx, %eax
+; SSE-NEXT:    pinsrw $3, %eax, %xmm1
+; SSE-NEXT:    movdqa %xmm1, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: dont_fold_srem_power_of_two:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpextrw $1, %xmm0, %eax
+; AVX-NEXT:    leal 31(%rax), %ecx
+; AVX-NEXT:    testw %ax, %ax
+; AVX-NEXT:    cmovnsl %eax, %ecx
+; AVX-NEXT:    andl $-32, %ecx
+; AVX-NEXT:    subl %ecx, %eax
+; AVX-NEXT:    vmovd %xmm0, %ecx
+; AVX-NEXT:    leal 63(%rcx), %edx
+; AVX-NEXT:    testw %cx, %cx
+; AVX-NEXT:    cmovnsl %ecx, %edx
+; AVX-NEXT:    andl $-64, %edx
+; AVX-NEXT:    subl %edx, %ecx
+; AVX-NEXT:    vmovd %ecx, %xmm1
+; AVX-NEXT:    vpinsrw $1, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrw $2, %xmm0, %eax
+; AVX-NEXT:    leal 7(%rax), %ecx
+; AVX-NEXT:    testw %ax, %ax
+; AVX-NEXT:    cmovnsl %eax, %ecx
+; AVX-NEXT:    andl $-8, %ecx
+; AVX-NEXT:    subl %ecx, %eax
+; AVX-NEXT:    vpinsrw $2, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrw $3, %xmm0, %eax
+; AVX-NEXT:    movswl %ax, %ecx
+; AVX-NEXT:    imull $-21385, %ecx, %ecx # imm = 0xAC77
+; AVX-NEXT:    shrl $16, %ecx
+; AVX-NEXT:    addl %eax, %ecx
+; AVX-NEXT:    movzwl %cx, %ecx
+; AVX-NEXT:    movswl %cx, %edx
+; AVX-NEXT:    shrl $15, %ecx
+; AVX-NEXT:    sarl $6, %edx
+; AVX-NEXT:    addl %ecx, %edx
+; AVX-NEXT:    imull $95, %edx, %ecx
+; AVX-NEXT:    subl %ecx, %eax
+; AVX-NEXT:    vpinsrw $3, %eax, %xmm1, %xmm0
+; AVX-NEXT:    retq
+  %1 = srem <4 x i16> %x, <i16 64, i16 32, i16 8, i16 95>
+  ret <4 x i16> %1
+}
+
+; Don't fold if the divisor is one.
+define <4 x i16> @dont_fold_srem_one(<4 x i16> %x) {
+; SSE-LABEL: dont_fold_srem_one:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pextrw $2, %xmm0, %eax
+; SSE-NEXT:    movswl %ax, %ecx
+; SSE-NEXT:    imull $-19945, %ecx, %ecx # imm = 0xB217
+; SSE-NEXT:    shrl $16, %ecx
+; SSE-NEXT:    addl %eax, %ecx
+; SSE-NEXT:    movzwl %cx, %ecx
+; SSE-NEXT:    movswl %cx, %edx
+; SSE-NEXT:    shrl $15, %ecx
+; SSE-NEXT:    sarl $4, %edx
+; SSE-NEXT:    addl %ecx, %edx
+; SSE-NEXT:    leal (%rdx,%rdx,2), %ecx
+; SSE-NEXT:    shll $3, %ecx
+; SSE-NEXT:    subl %ecx, %edx
+; SSE-NEXT:    addl %eax, %edx
+; SSE-NEXT:    pextrw $1, %xmm0, %eax
+; SSE-NEXT:    movswl %ax, %ecx
+; SSE-NEXT:    imull $12827, %ecx, %ecx # imm = 0x321B
+; SSE-NEXT:    movl %ecx, %esi
+; SSE-NEXT:    shrl $31, %esi
+; SSE-NEXT:    sarl $23, %ecx
+; SSE-NEXT:    addl %esi, %ecx
+; SSE-NEXT:    imull $654, %ecx, %ecx # imm = 0x28E
+; SSE-NEXT:    subl %ecx, %eax
+; SSE-NEXT:    pxor %xmm1, %xmm1
+; SSE-NEXT:    pinsrw $1, %eax, %xmm1
+; SSE-NEXT:    pinsrw $2, %edx, %xmm1
+; SSE-NEXT:    pextrw $3, %xmm0, %eax
+; SSE-NEXT:    movswl %ax, %ecx
+; SSE-NEXT:    imull $12375, %ecx, %ecx # imm = 0x3057
+; SSE-NEXT:    movl %ecx, %edx
+; SSE-NEXT:    shrl $31, %edx
+; SSE-NEXT:    sarl $26, %ecx
+; SSE-NEXT:    addl %edx, %ecx
+; SSE-NEXT:    imull $5423, %ecx, %ecx # imm = 0x152F
+; SSE-NEXT:    subl %ecx, %eax
+; SSE-NEXT:    pinsrw $3, %eax, %xmm1
+; SSE-NEXT:    movdqa %xmm1, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: dont_fold_srem_one:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpextrw $2, %xmm0, %eax
+; AVX-NEXT:    movswl %ax, %ecx
+; AVX-NEXT:    imull $-19945, %ecx, %ecx # imm = 0xB217
+; AVX-NEXT:    shrl $16, %ecx
+; AVX-NEXT:    addl %eax, %ecx
+; AVX-NEXT:    movzwl %cx, %ecx
+; AVX-NEXT:    movswl %cx, %edx
+; AVX-NEXT:    shrl $15, %ecx
+; AVX-NEXT:    sarl $4, %edx
+; AVX-NEXT:    addl %ecx, %edx
+; AVX-NEXT:    leal (%rdx,%rdx,2), %ecx
+; AVX-NEXT:    shll $3, %ecx
+; AVX-NEXT:    subl %ecx, %edx
+; AVX-NEXT:    addl %eax, %edx
+; AVX-NEXT:    vpextrw $1, %xmm0, %eax
+; AVX-NEXT:    movswl %ax, %ecx
+; AVX-NEXT:    imull $12827, %ecx, %ecx # imm = 0x321B
+; AVX-NEXT:    movl %ecx, %esi
+; AVX-NEXT:    shrl $31, %esi
+; AVX-NEXT:    sarl $23, %ecx
+; AVX-NEXT:    addl %esi, %ecx
+; AVX-NEXT:    imull $654, %ecx, %ecx # imm = 0x28E
+; AVX-NEXT:    subl %ecx, %eax
+; AVX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vpinsrw $1, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpinsrw $2, %edx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrw $3, %xmm0, %eax
+; AVX-NEXT:    movswl %ax, %ecx
+; AVX-NEXT:    imull $12375, %ecx, %ecx # imm = 0x3057
+; AVX-NEXT:    movl %ecx, %edx
+; AVX-NEXT:    shrl $31, %edx
+; AVX-NEXT:    sarl $26, %ecx
+; AVX-NEXT:    addl %edx, %ecx
+; AVX-NEXT:    imull $5423, %ecx, %ecx # imm = 0x152F
+; AVX-NEXT:    subl %ecx, %eax
+; AVX-NEXT:    vpinsrw $3, %eax, %xmm1, %xmm0
+; AVX-NEXT:    retq
+  %1 = srem <4 x i16> %x, <i16 1, i16 654, i16 23, i16 5423>
+  ret <4 x i16> %1
+}
+
+; Don't fold if the divisor is 2^15.
+define <4 x i16> @dont_fold_urem_i16_smax(<4 x i16> %x) {
+; SSE-LABEL: dont_fold_urem_i16_smax:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pextrw $2, %xmm0, %eax
+; SSE-NEXT:    movswl %ax, %ecx
+; SSE-NEXT:    imull $-19945, %ecx, %ecx # imm = 0xB217
+; SSE-NEXT:    shrl $16, %ecx
+; SSE-NEXT:    addl %eax, %ecx
+; SSE-NEXT:    movzwl %cx, %ecx
+; SSE-NEXT:    movswl %cx, %edx
+; SSE-NEXT:    shrl $15, %ecx
+; SSE-NEXT:    sarl $4, %edx
+; SSE-NEXT:    addl %ecx, %edx
+; SSE-NEXT:    leal (%rdx,%rdx,2), %ecx
+; SSE-NEXT:    shll $3, %ecx
+; SSE-NEXT:    subl %ecx, %edx
+; SSE-NEXT:    addl %eax, %edx
+; SSE-NEXT:    pextrw $1, %xmm0, %eax
+; SSE-NEXT:    leal 32767(%rax), %ecx
+; SSE-NEXT:    testw %ax, %ax
+; SSE-NEXT:    cmovnsl %eax, %ecx
+; SSE-NEXT:    andl $-32768, %ecx # imm = 0x8000
+; SSE-NEXT:    addl %eax, %ecx
+; SSE-NEXT:    pxor %xmm1, %xmm1
+; SSE-NEXT:    pinsrw $1, %ecx, %xmm1
+; SSE-NEXT:    pinsrw $2, %edx, %xmm1
+; SSE-NEXT:    pextrw $3, %xmm0, %eax
+; SSE-NEXT:    movswl %ax, %ecx
+; SSE-NEXT:    imull $12375, %ecx, %ecx # imm = 0x3057
+; SSE-NEXT:    movl %ecx, %edx
+; SSE-NEXT:    shrl $31, %edx
+; SSE-NEXT:    sarl $26, %ecx
+; SSE-NEXT:    addl %edx, %ecx
+; SSE-NEXT:    imull $5423, %ecx, %ecx # imm = 0x152F
+; SSE-NEXT:    subl %ecx, %eax
+; SSE-NEXT:    pinsrw $3, %eax, %xmm1
+; SSE-NEXT:    movdqa %xmm1, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: dont_fold_urem_i16_smax:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpextrw $2, %xmm0, %eax
+; AVX-NEXT:    movswl %ax, %ecx
+; AVX-NEXT:    imull $-19945, %ecx, %ecx # imm = 0xB217
+; AVX-NEXT:    shrl $16, %ecx
+; AVX-NEXT:    addl %eax, %ecx
+; AVX-NEXT:    movzwl %cx, %ecx
+; AVX-NEXT:    movswl %cx, %edx
+; AVX-NEXT:    shrl $15, %ecx
+; AVX-NEXT:    sarl $4, %edx
+; AVX-NEXT:    addl %ecx, %edx
+; AVX-NEXT:    leal (%rdx,%rdx,2), %ecx
+; AVX-NEXT:    shll $3, %ecx
+; AVX-NEXT:    subl %ecx, %edx
+; AVX-NEXT:    addl %eax, %edx
+; AVX-NEXT:    vpextrw $1, %xmm0, %eax
+; AVX-NEXT:    leal 32767(%rax), %ecx
+; AVX-NEXT:    testw %ax, %ax
+; AVX-NEXT:    cmovnsl %eax, %ecx
+; AVX-NEXT:    andl $-32768, %ecx # imm = 0x8000
+; AVX-NEXT:    addl %eax, %ecx
+; AVX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vpinsrw $1, %ecx, %xmm1, %xmm1
+; AVX-NEXT:    vpinsrw $2, %edx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrw $3, %xmm0, %eax
+; AVX-NEXT:    movswl %ax, %ecx
+; AVX-NEXT:    imull $12375, %ecx, %ecx # imm = 0x3057
+; AVX-NEXT:    movl %ecx, %edx
+; AVX-NEXT:    shrl $31, %edx
+; AVX-NEXT:    sarl $26, %ecx
+; AVX-NEXT:    addl %edx, %ecx
+; AVX-NEXT:    imull $5423, %ecx, %ecx # imm = 0x152F
+; AVX-NEXT:    subl %ecx, %eax
+; AVX-NEXT:    vpinsrw $3, %eax, %xmm1, %xmm0
+; AVX-NEXT:    retq
+  %1 = srem <4 x i16> %x, <i16 1, i16 32768, i16 23, i16 5423>
+  ret <4 x i16> %1
+}
+
+; Don't fold i64 srem.
+define <4 x i64> @dont_fold_srem_i64(<4 x i64> %x) {
+; SSE-LABEL: dont_fold_srem_i64:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqa %xmm1, %xmm2
+; SSE-NEXT:    movq %xmm1, %rcx
+; SSE-NEXT:    movabsq $-5614226457215950491, %rdx # imm = 0xB21642C8590B2165
+; SSE-NEXT:    movq %rcx, %rax
+; SSE-NEXT:    imulq %rdx
+; SSE-NEXT:    addq %rcx, %rdx
+; SSE-NEXT:    movq %rdx, %rax
+; SSE-NEXT:    shrq $63, %rax
+; SSE-NEXT:    sarq $4, %rdx
+; SSE-NEXT:    addq %rax, %rdx
+; SSE-NEXT:    leaq (%rdx,%rdx,2), %rax
+; SSE-NEXT:    shlq $3, %rax
+; SSE-NEXT:    subq %rax, %rdx
+; SSE-NEXT:    addq %rcx, %rdx
+; SSE-NEXT:    movq %rdx, %xmm1
+; SSE-NEXT:    pextrq $1, %xmm2, %rcx
+; SSE-NEXT:    movabsq $6966426675817289639, %rdx # imm = 0x60ADB826E5E517A7
+; SSE-NEXT:    movq %rcx, %rax
+; SSE-NEXT:    imulq %rdx
+; SSE-NEXT:    movq %rdx, %rax
+; SSE-NEXT:    shrq $63, %rax
+; SSE-NEXT:    sarq $11, %rdx
+; SSE-NEXT:    addq %rax, %rdx
+; SSE-NEXT:    imulq $5423, %rdx, %rax # imm = 0x152F
+; SSE-NEXT:    subq %rax, %rcx
+; SSE-NEXT:    movq %rcx, %xmm2
+; SSE-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; SSE-NEXT:    pextrq $1, %xmm0, %rcx
+; SSE-NEXT:    movabsq $7220743857598845893, %rdx # imm = 0x64353C48064353C5
+; SSE-NEXT:    movq %rcx, %rax
+; SSE-NEXT:    imulq %rdx
+; SSE-NEXT:    movq %rdx, %rax
+; SSE-NEXT:    shrq $63, %rax
+; SSE-NEXT:    sarq $8, %rdx
+; SSE-NEXT:    addq %rax, %rdx
+; SSE-NEXT:    imulq $654, %rdx, %rax # imm = 0x28E
+; SSE-NEXT:    subq %rax, %rcx
+; SSE-NEXT:    movq %rcx, %xmm0
+; SSE-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6,7]
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: dont_fold_srem_i64:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vmovq %xmm1, %rcx
+; AVX1-NEXT:    movabsq $-5614226457215950491, %rdx # imm = 0xB21642C8590B2165
+; AVX1-NEXT:    movq %rcx, %rax
+; AVX1-NEXT:    imulq %rdx
+; AVX1-NEXT:    addq %rcx, %rdx
+; AVX1-NEXT:    movq %rdx, %rax
+; AVX1-NEXT:    shrq $63, %rax
+; AVX1-NEXT:    sarq $4, %rdx
+; AVX1-NEXT:    addq %rax, %rdx
+; AVX1-NEXT:    leaq (%rdx,%rdx,2), %rax
+; AVX1-NEXT:    shlq $3, %rax
+; AVX1-NEXT:    subq %rax, %rdx
+; AVX1-NEXT:    addq %rcx, %rdx
+; AVX1-NEXT:    vmovq %rdx, %xmm2
+; AVX1-NEXT:    vpextrq $1, %xmm1, %rcx
+; AVX1-NEXT:    movabsq $6966426675817289639, %rdx # imm = 0x60ADB826E5E517A7
+; AVX1-NEXT:    movq %rcx, %rax
+; AVX1-NEXT:    imulq %rdx
+; AVX1-NEXT:    movq %rdx, %rax
+; AVX1-NEXT:    shrq $63, %rax
+; AVX1-NEXT:    sarq $11, %rdx
+; AVX1-NEXT:    addq %rax, %rdx
+; AVX1-NEXT:    imulq $5423, %rdx, %rax # imm = 0x152F
+; AVX1-NEXT:    subq %rax, %rcx
+; AVX1-NEXT:    vmovq %rcx, %xmm1
+; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX1-NEXT:    vpextrq $1, %xmm0, %rcx
+; AVX1-NEXT:    movabsq $7220743857598845893, %rdx # imm = 0x64353C48064353C5
+; AVX1-NEXT:    movq %rcx, %rax
+; AVX1-NEXT:    imulq %rdx
+; AVX1-NEXT:    movq %rdx, %rax
+; AVX1-NEXT:    shrq $63, %rax
+; AVX1-NEXT:    sarq $8, %rdx
+; AVX1-NEXT:    addq %rax, %rdx
+; AVX1-NEXT:    imulq $654, %rdx, %rax # imm = 0x28E
+; AVX1-NEXT:    subq %rax, %rcx
+; AVX1-NEXT:    vmovq %rcx, %xmm0
+; AVX1-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6,7]
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: dont_fold_srem_i64:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vmovq %xmm1, %rcx
+; AVX2-NEXT:    movabsq $-5614226457215950491, %rdx # imm = 0xB21642C8590B2165
+; AVX2-NEXT:    movq %rcx, %rax
+; AVX2-NEXT:    imulq %rdx
+; AVX2-NEXT:    addq %rcx, %rdx
+; AVX2-NEXT:    movq %rdx, %rax
+; AVX2-NEXT:    shrq $63, %rax
+; AVX2-NEXT:    sarq $4, %rdx
+; AVX2-NEXT:    addq %rax, %rdx
+; AVX2-NEXT:    leaq (%rdx,%rdx,2), %rax
+; AVX2-NEXT:    shlq $3, %rax
+; AVX2-NEXT:    subq %rax, %rdx
+; AVX2-NEXT:    addq %rcx, %rdx
+; AVX2-NEXT:    vmovq %rdx, %xmm2
+; AVX2-NEXT:    vpextrq $1, %xmm1, %rcx
+; AVX2-NEXT:    movabsq $6966426675817289639, %rdx # imm = 0x60ADB826E5E517A7
+; AVX2-NEXT:    movq %rcx, %rax
+; AVX2-NEXT:    imulq %rdx
+; AVX2-NEXT:    movq %rdx, %rax
+; AVX2-NEXT:    shrq $63, %rax
+; AVX2-NEXT:    sarq $11, %rdx
+; AVX2-NEXT:    addq %rax, %rdx
+; AVX2-NEXT:    imulq $5423, %rdx, %rax # imm = 0x152F
+; AVX2-NEXT:    subq %rax, %rcx
+; AVX2-NEXT:    vmovq %rcx, %xmm1
+; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX2-NEXT:    vpextrq $1, %xmm0, %rcx
+; AVX2-NEXT:    movabsq $7220743857598845893, %rdx # imm = 0x64353C48064353C5
+; AVX2-NEXT:    movq %rcx, %rax
+; AVX2-NEXT:    imulq %rdx
+; AVX2-NEXT:    movq %rdx, %rax
+; AVX2-NEXT:    shrq $63, %rax
+; AVX2-NEXT:    sarq $8, %rdx
+; AVX2-NEXT:    addq %rax, %rdx
+; AVX2-NEXT:    imulq $654, %rdx, %rax # imm = 0x28E
+; AVX2-NEXT:    subq %rax, %rcx
+; AVX2-NEXT:    vmovq %rcx, %xmm0
+; AVX2-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6,7]
+; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %1 = srem <4 x i64> %x, <i64 1, i64 654, i64 23, i64 5423>
+  ret <4 x i64> %1
+}
diff --git a/llvm/test/CodeGen/X86/sse-align-1.ll b/llvm/test/CodeGen/X86/sse-align-1.ll
index 378100d69..f58e1204d 100644
--- a/llvm/test/CodeGen/X86/sse-align-1.ll
+++ b/llvm/test/CodeGen/X86/sse-align-1.ll
@@ -1,10 +1,19 @@
-; RUN: llc < %s -mtriple=x86_64-- | grep movap | count 2
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-- | FileCheck %s
 
 define <4 x float> @foo(<4 x float>* %p) nounwind {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps (%rdi), %xmm0
+; CHECK-NEXT:    retq
   %t = load <4 x float>, <4 x float>* %p
   ret <4 x float> %t
 }
 define <2 x double> @bar(<2 x double>* %p) nounwind {
+; CHECK-LABEL: bar:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps (%rdi), %xmm0
+; CHECK-NEXT:    retq
   %t = load <2 x double>, <2 x double>* %p
   ret <2 x double> %t
 }
diff --git a/llvm/test/CodeGen/X86/sse-align-11.ll b/llvm/test/CodeGen/X86/sse-align-11.ll
index 857ea4508..e6b5f85e6 100644
--- a/llvm/test/CodeGen/X86/sse-align-11.ll
+++ b/llvm/test/CodeGen/X86/sse-align-11.ll
@@ -1,13 +1,23 @@
-; RUN: llc < %s -mcpu=yonah -mtriple=i686-apple-darwin8 | grep movaps
-; RUN: llc < %s -mcpu=yonah -mtriple=i686-linux-gnu | grep movaps
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mcpu=yonah -mtriple=i686-apple-darwin8 | FileCheck %s --check-prefix=i686-apple-darwin8
+; RUN: llc < %s -mcpu=yonah -mtriple=i686-linux-gnu | FileCheck %s --check-prefix=i686-linux-gnu
 ; PR8969 - make 32-bit linux have a 16-byte aligned stack
 
 define <4 x float> @foo(float %a, float %b, float %c, float %d) nounwind {
+; i686-apple-darwin8-LABEL: foo:
+; i686-apple-darwin8:       ## %bb.0: ## %entry
+; i686-apple-darwin8-NEXT:    movaps {{[0-9]+}}(%esp), %xmm0
+; i686-apple-darwin8-NEXT:    retl
+;
+; i686-linux-gnu-LABEL: foo:
+; i686-linux-gnu:       # %bb.0: # %entry
+; i686-linux-gnu-NEXT:    movaps {{[0-9]+}}(%esp), %xmm0
+; i686-linux-gnu-NEXT:    retl
 entry:
-        %tmp6 = insertelement <4 x float> undef, float %a, i32 0               
-        %tmp7 = insertelement <4 x float> %tmp6, float %b, i32 1               
-        %tmp8 = insertelement <4 x float> %tmp7, float %c, i32 2               
-        %tmp9 = insertelement <4 x float> %tmp8, float %d, i32 3               
+        %tmp6 = insertelement <4 x float> undef, float %a, i32 0
+        %tmp7 = insertelement <4 x float> %tmp6, float %b, i32 1
+        %tmp8 = insertelement <4 x float> %tmp7, float %c, i32 2
+        %tmp9 = insertelement <4 x float> %tmp8, float %d, i32 3
         ret <4 x float> %tmp9
 }
 
diff --git a/llvm/test/CodeGen/X86/sse-align-4.ll b/llvm/test/CodeGen/X86/sse-align-4.ll
index 8edafd969..cc924b930 100644
--- a/llvm/test/CodeGen/X86/sse-align-4.ll
+++ b/llvm/test/CodeGen/X86/sse-align-4.ll
@@ -1,10 +1,19 @@
-; RUN: llc < %s -mtriple=x86_64-- | grep movup | count 2
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-- | FileCheck %s
 
 define void @foo(<4 x float>* %p, <4 x float> %x) nounwind {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movups %xmm0, (%rdi)
+; CHECK-NEXT:    retq
   store <4 x float> %x, <4 x float>* %p, align 4
   ret void
 }
 define void @bar(<2 x double>* %p, <2 x double> %x) nounwind {
+; CHECK-LABEL: bar:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movups %xmm0, (%rdi)
+; CHECK-NEXT:    retq
   store <2 x double> %x, <2 x double>* %p, align 8
   ret void
 }
diff --git a/llvm/test/CodeGen/X86/sse-align-5.ll b/llvm/test/CodeGen/X86/sse-align-5.ll
index 4429d4579..ebba1b946 100644
--- a/llvm/test/CodeGen/X86/sse-align-5.ll
+++ b/llvm/test/CodeGen/X86/sse-align-5.ll
@@ -1,6 +1,11 @@
-; RUN: llc < %s -mtriple=x86_64-- | grep movaps | count 1
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-- | FileCheck %s
 
 define <2 x i64> @bar(<2 x i64>* %p) nounwind {
+; CHECK-LABEL: bar:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps (%rdi), %xmm0
+; CHECK-NEXT:    retq
   %t = load <2 x i64>, <2 x i64>* %p
   ret <2 x i64> %t
 }
diff --git a/llvm/test/CodeGen/X86/sse-align-6.ll b/llvm/test/CodeGen/X86/sse-align-6.ll
index 343832c74..20cfa41e4 100644
--- a/llvm/test/CodeGen/X86/sse-align-6.ll
+++ b/llvm/test/CodeGen/X86/sse-align-6.ll
@@ -1,6 +1,21 @@
-; RUN: llc < %s -mtriple=x86_64-- | grep movdqu | count 1
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-- | FileCheck %s
 
 define <2 x i64> @bar(<2 x i64>* %p, <2 x i64> %x) nounwind {
+; CHECK-LABEL: bar:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movdqu (%rdi), %xmm1
+; CHECK-NEXT:    movdqa %xmm0, %xmm2
+; CHECK-NEXT:    psrlq $32, %xmm2
+; CHECK-NEXT:    pmuludq %xmm1, %xmm2
+; CHECK-NEXT:    movdqa %xmm1, %xmm3
+; CHECK-NEXT:    psrlq $32, %xmm3
+; CHECK-NEXT:    pmuludq %xmm0, %xmm3
+; CHECK-NEXT:    paddq %xmm2, %xmm3
+; CHECK-NEXT:    psllq $32, %xmm3
+; CHECK-NEXT:    pmuludq %xmm1, %xmm0
+; CHECK-NEXT:    paddq %xmm3, %xmm0
+; CHECK-NEXT:    retq
   %t = load <2 x i64>, <2 x i64>* %p, align 8
   %z = mul <2 x i64> %t, %x
   ret <2 x i64> %z
diff --git a/llvm/test/CodeGen/X86/sse-align-8.ll b/llvm/test/CodeGen/X86/sse-align-8.ll
index 60aa22b7e..a6f1bb4e8 100644
--- a/llvm/test/CodeGen/X86/sse-align-8.ll
+++ b/llvm/test/CodeGen/X86/sse-align-8.ll
@@ -1,6 +1,11 @@
-; RUN: llc < %s -mtriple=x86_64-- | grep movups | count 1
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-- | FileCheck %s
 
 define void @bar(<2 x i64>* %p, <2 x i64> %x) nounwind {
+; CHECK-LABEL: bar:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movups %xmm0, (%rdi)
+; CHECK-NEXT:    retq
   store <2 x i64> %x, <2 x i64>* %p, align 8
   ret void
 }
diff --git a/llvm/test/CodeGen/X86/sse-align-9.ll b/llvm/test/CodeGen/X86/sse-align-9.ll
index afec18ef5..d39e5d997 100644
--- a/llvm/test/CodeGen/X86/sse-align-9.ll
+++ b/llvm/test/CodeGen/X86/sse-align-9.ll
@@ -1,10 +1,19 @@
-; RUN: llc < %s -mtriple=x86_64-- | grep movup | count 2
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-- | FileCheck %s
 
 define <4 x float> @foo(<4 x float>* %p) nounwind {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movups (%rdi), %xmm0
+; CHECK-NEXT:    retq
   %t = load <4 x float>, <4 x float>* %p, align 4
   ret <4 x float> %t
 }
 define <2 x double> @bar(<2 x double>* %p) nounwind {
+; CHECK-LABEL: bar:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movups (%rdi), %xmm0
+; CHECK-NEXT:    retq
   %t = load <2 x double>, <2 x double>* %p, align 8
   ret <2 x double> %t
 }
diff --git a/llvm/test/CodeGen/X86/sse-load-ret.ll b/llvm/test/CodeGen/X86/sse-load-ret.ll
index a84201a9a..841410bf8 100644
--- a/llvm/test/CodeGen/X86/sse-load-ret.ll
+++ b/llvm/test/CodeGen/X86/sse-load-ret.ll
@@ -1,19 +1,50 @@
-; RUN: llc < %s -mtriple=i686-- -mcpu=yonah | not grep movss
-; RUN: llc < %s -mtriple=i686-- -mcpu=yonah | not grep xmm
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- -mcpu=yonah | FileCheck %s
 
 define double @test1(double* %P) {
+; CHECK-LABEL: test1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    fldl (%eax)
+; CHECK-NEXT:    retl
         %X = load double, double* %P            ; <double> [#uses=1]
         ret double %X
 }
 
 define double @test2() {
+; CHECK-LABEL: test2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fldl {{\.LCPI.*}}
+; CHECK-NEXT:    retl
         ret double 1.234560e+03
 }
 
 
-; FIXME: Todo
-;double %test3(bool %B) {
-;	%C = select bool %B, double 123.412, double 523.01123123
-;	ret double %C
-;}
+define double @test3(i1 %B) {
+; CHECK-LABEL: test3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushl %ebp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    .cfi_offset %ebp, -8
+; CHECK-NEXT:    movl %esp, %ebp
+; CHECK-NEXT:    .cfi_def_cfa_register %ebp
+; CHECK-NEXT:    andl $-8, %esp
+; CHECK-NEXT:    subl $8, %esp
+; CHECK-NEXT:    testb $1, 8(%ebp)
+; CHECK-NEXT:    jne .LBB2_1
+; CHECK-NEXT:  # %bb.2:
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    jmp .LBB2_3
+; CHECK-NEXT:  .LBB2_1:
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:  .LBB2_3:
+; CHECK-NEXT:    movsd %xmm0, (%esp)
+; CHECK-NEXT:    fldl (%esp)
+; CHECK-NEXT:    movl %ebp, %esp
+; CHECK-NEXT:    popl %ebp
+; CHECK-NEXT:    .cfi_def_cfa %esp, 4
+; CHECK-NEXT:    retl
+	%C = select i1 %B, double 123.412, double 523.01123123
+	ret double %C
+}
 
diff --git a/llvm/test/CodeGen/X86/sse-minmax.ll b/llvm/test/CodeGen/X86/sse-minmax.ll
index 178a74ba5..0f703d695 100644
--- a/llvm/test/CodeGen/X86/sse-minmax.ll
+++ b/llvm/test/CodeGen/X86/sse-minmax.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=sse4.2                                                 | FileCheck %s --check-prefix=ALL --check-prefix=STRICT
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=sse4.2 -enable-unsafe-fp-math -enable-no-nans-fp-math  | FileCheck %s --check-prefix=ALL --check-prefix=RELAX --check-prefix=UNSAFE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=sse4.2 -enable-no-nans-fp-math                         | FileCheck %s --check-prefix=ALL --check-prefix=RELAX --check-prefix=FINITE
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=sse4.2                                                          | FileCheck %s --check-prefix=ALL --check-prefix=STRICT
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=sse4.2 -enable-no-signed-zeros-fp-math -enable-no-nans-fp-math  | FileCheck %s --check-prefix=ALL --check-prefix=RELAX --check-prefix=UNSAFE
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=sse4.2 -enable-no-nans-fp-math                                  | FileCheck %s --check-prefix=ALL --check-prefix=RELAX --check-prefix=FINITE
 
 ; Some of these patterns can be matched as SSE min or max. Some of
 ; them can be matched provided that the operands are swapped.
diff --git a/llvm/test/CodeGen/X86/sse-varargs.ll b/llvm/test/CodeGen/X86/sse-varargs.ll
index 3d1bec88e..6f13dfb1d 100644
--- a/llvm/test/CodeGen/X86/sse-varargs.ll
+++ b/llvm/test/CodeGen/X86/sse-varargs.ll
@@ -1,8 +1,22 @@
-; RUN: llc < %s -mtriple=i686-- -mattr=+sse2 | grep xmm | grep esp
-
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- -mattr=+sse2 | FileCheck %s
 define i32 @t() nounwind  {
+; CHECK-LABEL: t:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushl %ebp
+; CHECK-NEXT:    movl %esp, %ebp
+; CHECK-NEXT:    andl $-16, %esp
+; CHECK-NEXT:    subl $48, %esp
+; CHECK-NEXT:    movaps {{.*#+}} xmm0 = [10,11,12,13]
+; CHECK-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movl $1, (%esp)
+; CHECK-NEXT:    calll foo
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    movl %ebp, %esp
+; CHECK-NEXT:    popl %ebp
+; CHECK-NEXT:    retl
 entry:
-	tail call void (i32, ...) @foo( i32 1, <4 x i32> < i32 10, i32 11, i32 12, i32 13 > ) nounwind 
+	tail call void (i32, ...) @foo( i32 1, <4 x i32> < i32 10, i32 11, i32 12, i32 13 > ) nounwind
 	ret i32 0
 }
 
diff --git a/llvm/test/CodeGen/X86/sse2-intrinsics-canonical.ll b/llvm/test/CodeGen/X86/sse2-intrinsics-canonical.ll
index 506fb9eb1..4721cb480 100644
--- a/llvm/test/CodeGen/X86/sse2-intrinsics-canonical.ll
+++ b/llvm/test/CodeGen/X86/sse2-intrinsics-canonical.ll
@@ -93,40 +93,17 @@ define <8 x i16> @test_x86_sse2_psubus_w(<8 x i16> %a0, <8 x i16> %a1) {
 define <8 x i8> @test_x86_sse2_paddus_b_64(<8 x i8> %a0, <8 x i8> %a1) {
 ; SSE-LABEL: test_x86_sse2_paddus_b_64:
 ; SSE:       ## %bb.0:
-; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
-; SSE-NEXT:    ## encoding: [0x66,0x0f,0x6f,0x15,A,A,A,A]
-; SSE-NEXT:    ## fixup A - offset: 4, value: LCPI4_0, kind: FK_Data_4
-; SSE-NEXT:    pand %xmm2, %xmm1 ## encoding: [0x66,0x0f,0xdb,0xca]
-; SSE-NEXT:    packuswb %xmm1, %xmm1 ## encoding: [0x66,0x0f,0x67,0xc9]
-; SSE-NEXT:    pand %xmm2, %xmm0 ## encoding: [0x66,0x0f,0xdb,0xc2]
-; SSE-NEXT:    packuswb %xmm0, %xmm0 ## encoding: [0x66,0x0f,0x67,0xc0]
 ; SSE-NEXT:    paddusb %xmm1, %xmm0 ## encoding: [0x66,0x0f,0xdc,0xc1]
-; SSE-NEXT:    punpcklbw %xmm0, %xmm0 ## encoding: [0x66,0x0f,0x60,0xc0]
-; SSE-NEXT:    ## xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; SSE-NEXT:    retl ## encoding: [0xc3]
 ;
 ; AVX2-LABEL: test_x86_sse2_paddus_b_64:
 ; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
-; AVX2-NEXT:    ## encoding: [0xc5,0xf9,0x6f,0x15,A,A,A,A]
-; AVX2-NEXT:    ## fixup A - offset: 4, value: LCPI4_0, kind: FK_Data_4
-; AVX2-NEXT:    vpshufb %xmm2, %xmm1, %xmm1 ## encoding: [0xc4,0xe2,0x71,0x00,0xca]
-; AVX2-NEXT:    vpshufb %xmm2, %xmm0, %xmm0 ## encoding: [0xc4,0xe2,0x79,0x00,0xc2]
 ; AVX2-NEXT:    vpaddusb %xmm1, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xdc,0xc1]
-; AVX2-NEXT:    vpmovzxbw %xmm0, %xmm0 ## encoding: [0xc4,0xe2,0x79,0x30,0xc0]
-; AVX2-NEXT:    ## xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; AVX2-NEXT:    retl ## encoding: [0xc3]
 ;
 ; SKX-LABEL: test_x86_sse2_paddus_b_64:
 ; SKX:       ## %bb.0:
-; SKX-NEXT:    vmovdqa LCPI4_0, %xmm2 ## EVEX TO VEX Compression xmm2 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
-; SKX-NEXT:    ## encoding: [0xc5,0xf9,0x6f,0x15,A,A,A,A]
-; SKX-NEXT:    ## fixup A - offset: 4, value: LCPI4_0, kind: FK_Data_4
-; SKX-NEXT:    vpshufb %xmm2, %xmm1, %xmm1 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x71,0x00,0xca]
-; SKX-NEXT:    vpshufb %xmm2, %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x00,0xc2]
 ; SKX-NEXT:    vpaddusb %xmm1, %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0xdc,0xc1]
-; SKX-NEXT:    vpmovzxbw %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x30,0xc0]
-; SKX-NEXT:    ## xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; SKX-NEXT:    retl ## encoding: [0xc3]
   %1 = add <8 x i8> %a0, %a1
   %2 = icmp ugt <8 x i8> %a0, %1
@@ -137,45 +114,17 @@ define <8 x i8> @test_x86_sse2_paddus_b_64(<8 x i8> %a0, <8 x i8> %a1) {
 define <4 x i16> @test_x86_sse2_paddus_w_64(<4 x i16> %a0, <4 x i16> %a1) {
 ; SSE-LABEL: test_x86_sse2_paddus_w_64:
 ; SSE:       ## %bb.0:
-; SSE-NEXT:    pshuflw $232, %xmm1, %xmm1 ## encoding: [0xf2,0x0f,0x70,0xc9,0xe8]
-; SSE-NEXT:    ## xmm1 = xmm1[0,2,2,3,4,5,6,7]
-; SSE-NEXT:    pshufhw $232, %xmm1, %xmm1 ## encoding: [0xf3,0x0f,0x70,0xc9,0xe8]
-; SSE-NEXT:    ## xmm1 = xmm1[0,1,2,3,4,6,6,7]
-; SSE-NEXT:    pshufd $232, %xmm1, %xmm1 ## encoding: [0x66,0x0f,0x70,0xc9,0xe8]
-; SSE-NEXT:    ## xmm1 = xmm1[0,2,2,3]
-; SSE-NEXT:    pshuflw $232, %xmm0, %xmm0 ## encoding: [0xf2,0x0f,0x70,0xc0,0xe8]
-; SSE-NEXT:    ## xmm0 = xmm0[0,2,2,3,4,5,6,7]
-; SSE-NEXT:    pshufhw $232, %xmm0, %xmm0 ## encoding: [0xf3,0x0f,0x70,0xc0,0xe8]
-; SSE-NEXT:    ## xmm0 = xmm0[0,1,2,3,4,6,6,7]
-; SSE-NEXT:    pshufd $232, %xmm0, %xmm0 ## encoding: [0x66,0x0f,0x70,0xc0,0xe8]
-; SSE-NEXT:    ## xmm0 = xmm0[0,2,2,3]
 ; SSE-NEXT:    paddusw %xmm1, %xmm0 ## encoding: [0x66,0x0f,0xdd,0xc1]
-; SSE-NEXT:    punpcklwd %xmm0, %xmm0 ## encoding: [0x66,0x0f,0x61,0xc0]
-; SSE-NEXT:    ## xmm0 = xmm0[0,0,1,1,2,2,3,3]
 ; SSE-NEXT:    retl ## encoding: [0xc3]
 ;
 ; AVX2-LABEL: test_x86_sse2_paddus_w_64:
 ; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
-; AVX2-NEXT:    ## encoding: [0xc5,0xf9,0x6f,0x15,A,A,A,A]
-; AVX2-NEXT:    ## fixup A - offset: 4, value: LCPI5_0, kind: FK_Data_4
-; AVX2-NEXT:    vpshufb %xmm2, %xmm1, %xmm1 ## encoding: [0xc4,0xe2,0x71,0x00,0xca]
-; AVX2-NEXT:    vpshufb %xmm2, %xmm0, %xmm0 ## encoding: [0xc4,0xe2,0x79,0x00,0xc2]
 ; AVX2-NEXT:    vpaddusw %xmm1, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xdd,0xc1]
-; AVX2-NEXT:    vpmovzxwd %xmm0, %xmm0 ## encoding: [0xc4,0xe2,0x79,0x33,0xc0]
-; AVX2-NEXT:    ## xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; AVX2-NEXT:    retl ## encoding: [0xc3]
 ;
 ; SKX-LABEL: test_x86_sse2_paddus_w_64:
 ; SKX:       ## %bb.0:
-; SKX-NEXT:    vmovdqa LCPI5_0, %xmm2 ## EVEX TO VEX Compression xmm2 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
-; SKX-NEXT:    ## encoding: [0xc5,0xf9,0x6f,0x15,A,A,A,A]
-; SKX-NEXT:    ## fixup A - offset: 4, value: LCPI5_0, kind: FK_Data_4
-; SKX-NEXT:    vpshufb %xmm2, %xmm1, %xmm1 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x71,0x00,0xca]
-; SKX-NEXT:    vpshufb %xmm2, %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x00,0xc2]
 ; SKX-NEXT:    vpaddusw %xmm1, %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0xdd,0xc1]
-; SKX-NEXT:    vpmovzxwd %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x33,0xc0]
-; SKX-NEXT:    ## xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; SKX-NEXT:    retl ## encoding: [0xc3]
   %1 = add <4 x i16> %a0, %a1
   %2 = icmp ugt <4 x i16> %a0, %1
@@ -186,36 +135,17 @@ define <4 x i16> @test_x86_sse2_paddus_w_64(<4 x i16> %a0, <4 x i16> %a1) {
 define <8 x i8> @test_x86_sse2_psubus_b_64(<8 x i8> %a0, <8 x i8> %a1) {
 ; SSE-LABEL: test_x86_sse2_psubus_b_64:
 ; SSE:       ## %bb.0:
-; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
-; SSE-NEXT:    ## encoding: [0x66,0x0f,0x6f,0x15,A,A,A,A]
-; SSE-NEXT:    ## fixup A - offset: 4, value: LCPI6_0, kind: FK_Data_4
-; SSE-NEXT:    movdqa %xmm1, %xmm3 ## encoding: [0x66,0x0f,0x6f,0xd9]
-; SSE-NEXT:    pand %xmm2, %xmm3 ## encoding: [0x66,0x0f,0xdb,0xda]
-; SSE-NEXT:    pand %xmm2, %xmm0 ## encoding: [0x66,0x0f,0xdb,0xc2]
-; SSE-NEXT:    pmaxsw %xmm3, %xmm0 ## encoding: [0x66,0x0f,0xee,0xc3]
-; SSE-NEXT:    psubw %xmm1, %xmm0 ## encoding: [0x66,0x0f,0xf9,0xc1]
+; SSE-NEXT:    psubusb %xmm1, %xmm0 ## encoding: [0x66,0x0f,0xd8,0xc1]
 ; SSE-NEXT:    retl ## encoding: [0xc3]
 ;
 ; AVX2-LABEL: test_x86_sse2_psubus_b_64:
 ; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpbroadcastw {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
-; AVX2-NEXT:    ## encoding: [0xc4,0xe2,0x79,0x79,0x15,A,A,A,A]
-; AVX2-NEXT:    ## fixup A - offset: 5, value: LCPI6_0, kind: FK_Data_4
-; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm3 ## encoding: [0xc5,0xf1,0xdb,0xda]
-; AVX2-NEXT:    vpand %xmm2, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xdb,0xc2]
-; AVX2-NEXT:    vpmaxuw %xmm3, %xmm0, %xmm0 ## encoding: [0xc4,0xe2,0x79,0x3e,0xc3]
-; AVX2-NEXT:    vpsubw %xmm1, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xf9,0xc1]
+; AVX2-NEXT:    vpsubusb %xmm1, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xd8,0xc1]
 ; AVX2-NEXT:    retl ## encoding: [0xc3]
 ;
 ; SKX-LABEL: test_x86_sse2_psubus_b_64:
 ; SKX:       ## %bb.0:
-; SKX-NEXT:    vpbroadcastw LCPI6_0, %xmm2 ## EVEX TO VEX Compression xmm2 = [255,255,255,255,255,255,255,255]
-; SKX-NEXT:    ## encoding: [0xc4,0xe2,0x79,0x79,0x15,A,A,A,A]
-; SKX-NEXT:    ## fixup A - offset: 5, value: LCPI6_0, kind: FK_Data_4
-; SKX-NEXT:    vpand %xmm2, %xmm1, %xmm3 ## EVEX TO VEX Compression encoding: [0xc5,0xf1,0xdb,0xda]
-; SKX-NEXT:    vpand %xmm2, %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0xdb,0xc2]
-; SKX-NEXT:    vpmaxuw %xmm3, %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x3e,0xc3]
-; SKX-NEXT:    vpsubw %xmm1, %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0xf9,0xc1]
+; SKX-NEXT:    vpsubusb %xmm1, %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd8,0xc1]
 ; SKX-NEXT:    retl ## encoding: [0xc3]
   %cmp = icmp ugt <8 x i8> %a0, %a1
   %sel = select <8 x i1> %cmp, <8 x i8> %a0, <8 x i8> %a1
@@ -226,41 +156,17 @@ define <8 x i8> @test_x86_sse2_psubus_b_64(<8 x i8> %a0, <8 x i8> %a1) {
 define <4 x i16> @test_x86_sse2_psubus_w_64(<4 x i16> %a0, <4 x i16> %a1) {
 ; SSE-LABEL: test_x86_sse2_psubus_w_64:
 ; SSE:       ## %bb.0:
-; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [65535,0,65535,0,65535,0,65535,0]
-; SSE-NEXT:    ## encoding: [0x66,0x0f,0x6f,0x15,A,A,A,A]
-; SSE-NEXT:    ## fixup A - offset: 4, value: LCPI7_0, kind: FK_Data_4
-; SSE-NEXT:    movdqa %xmm1, %xmm3 ## encoding: [0x66,0x0f,0x6f,0xd9]
-; SSE-NEXT:    pand %xmm2, %xmm3 ## encoding: [0x66,0x0f,0xdb,0xda]
-; SSE-NEXT:    pand %xmm2, %xmm0 ## encoding: [0x66,0x0f,0xdb,0xc2]
-; SSE-NEXT:    movdqa %xmm0, %xmm2 ## encoding: [0x66,0x0f,0x6f,0xd0]
-; SSE-NEXT:    pcmpgtd %xmm3, %xmm2 ## encoding: [0x66,0x0f,0x66,0xd3]
-; SSE-NEXT:    pand %xmm2, %xmm0 ## encoding: [0x66,0x0f,0xdb,0xc2]
-; SSE-NEXT:    pandn %xmm3, %xmm2 ## encoding: [0x66,0x0f,0xdf,0xd3]
-; SSE-NEXT:    por %xmm0, %xmm2 ## encoding: [0x66,0x0f,0xeb,0xd0]
-; SSE-NEXT:    psubd %xmm1, %xmm2 ## encoding: [0x66,0x0f,0xfa,0xd1]
-; SSE-NEXT:    movdqa %xmm2, %xmm0 ## encoding: [0x66,0x0f,0x6f,0xc2]
+; SSE-NEXT:    psubusw %xmm1, %xmm0 ## encoding: [0x66,0x0f,0xd9,0xc1]
 ; SSE-NEXT:    retl ## encoding: [0xc3]
 ;
 ; AVX2-LABEL: test_x86_sse2_psubus_w_64:
 ; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2 ## encoding: [0xc5,0xe9,0xef,0xd2]
-; AVX2-NEXT:    vpblendw $170, %xmm2, %xmm1, %xmm3 ## encoding: [0xc4,0xe3,0x71,0x0e,0xda,0xaa]
-; AVX2-NEXT:    ## xmm3 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
-; AVX2-NEXT:    vpblendw $170, %xmm2, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x0e,0xc2,0xaa]
-; AVX2-NEXT:    ## xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3],xmm0[4],xmm2[5],xmm0[6],xmm2[7]
-; AVX2-NEXT:    vpmaxud %xmm3, %xmm0, %xmm0 ## encoding: [0xc4,0xe2,0x79,0x3f,0xc3]
-; AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xfa,0xc1]
+; AVX2-NEXT:    vpsubusw %xmm1, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xd9,0xc1]
 ; AVX2-NEXT:    retl ## encoding: [0xc3]
 ;
 ; SKX-LABEL: test_x86_sse2_psubus_w_64:
 ; SKX:       ## %bb.0:
-; SKX-NEXT:    vpxor %xmm2, %xmm2, %xmm2 ## EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xd2]
-; SKX-NEXT:    vpblendw $170, %xmm2, %xmm1, %xmm3 ## encoding: [0xc4,0xe3,0x71,0x0e,0xda,0xaa]
-; SKX-NEXT:    ## xmm3 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
-; SKX-NEXT:    vpblendw $170, %xmm2, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x0e,0xc2,0xaa]
-; SKX-NEXT:    ## xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3],xmm0[4],xmm2[5],xmm0[6],xmm2[7]
-; SKX-NEXT:    vpmaxud %xmm3, %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x3f,0xc3]
-; SKX-NEXT:    vpsubd %xmm1, %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfa,0xc1]
+; SKX-NEXT:    vpsubusw %xmm1, %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd9,0xc1]
 ; SKX-NEXT:    retl ## encoding: [0xc3]
   %cmp = icmp ugt <4 x i16> %a0, %a1
   %sel = select <4 x i1> %cmp, <4 x i16> %a0, <4 x i16> %a1
diff --git a/llvm/test/CodeGen/X86/sse2-vector-shifts.ll b/llvm/test/CodeGen/X86/sse2-vector-shifts.ll
index 696f99220..24d0453c1 100644
--- a/llvm/test/CodeGen/X86/sse2-vector-shifts.ll
+++ b/llvm/test/CodeGen/X86/sse2-vector-shifts.ll
@@ -321,8 +321,9 @@ define <4 x i32> @shl_srl_v4i32(<4 x i32> %x) nounwind {
 define <4 x i32> @shl_zext_srl_v4i32(<4 x i16> %x) nounwind {
 ; CHECK-LABEL: shl_zext_srl_v4i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    andps {{.*}}(%rip), %xmm0
-; CHECK-NEXT:    andps {{.*}}(%rip), %xmm0
+; CHECK-NEXT:    pand {{.*}}(%rip), %xmm0
+; CHECK-NEXT:    pxor %xmm1, %xmm1
+; CHECK-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
 ; CHECK-NEXT:    retq
   %srl = lshr <4 x i16> %x, <i16 2, i16 2, i16 2, i16 2>
   %zext = zext <4 x i16> %srl to <4 x i32>
@@ -334,6 +335,7 @@ define <4 x i16> @sra_trunc_srl_v4i32(<4 x i32> %x) nounwind {
 ; CHECK-LABEL: sra_trunc_srl_v4i32:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    psrad $19, %xmm0
+; CHECK-NEXT:    packssdw %xmm0, %xmm0
 ; CHECK-NEXT:    retq
   %srl = lshr <4 x i32> %x, <i32 16, i32 16, i32 16, i32 16>
   %trunc = trunc <4 x i32> %srl to <4 x i16>
@@ -344,6 +346,7 @@ define <4 x i16> @sra_trunc_srl_v4i32(<4 x i32> %x) nounwind {
 define <4 x i32> @shl_zext_shl_v4i32(<4 x i16> %x) nounwind {
 ; CHECK-LABEL: shl_zext_shl_v4i32:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
 ; CHECK-NEXT:    pslld $19, %xmm0
 ; CHECK-NEXT:    retq
   %shl0 = shl <4 x i16> %x, <i16 2, i16 2, i16 2, i16 2>
diff --git a/llvm/test/CodeGen/X86/sse41.ll b/llvm/test/CodeGen/X86/sse41.ll
index b5c372634..4e80f8f92 100644
--- a/llvm/test/CodeGen/X86/sse41.ll
+++ b/llvm/test/CodeGen/X86/sse41.ll
@@ -656,7 +656,8 @@ define <4 x i32> @pinsrd_from_shufflevector_i32(<4 x i32> %a, <4 x i32>* nocaptu
 ; X86-AVX512-LABEL: pinsrd_from_shufflevector_i32:
 ; X86-AVX512:       ## %bb.0: ## %entry
 ; X86-AVX512-NEXT:    movl {{[0-9]+}}(%esp), %eax ## encoding: [0x8b,0x44,0x24,0x04]
-; X86-AVX512-NEXT:    vbroadcastss (%eax), %xmm1 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x18,0x08]
+; X86-AVX512-NEXT:    vpermilps $36, (%eax), %xmm1 ## EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x04,0x08,0x24]
+; X86-AVX512-NEXT:    ## xmm1 = mem[0,1,2,0]
 ; X86-AVX512-NEXT:    vblendps $8, %xmm1, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x0c,0xc1,0x08]
 ; X86-AVX512-NEXT:    ## xmm0 = xmm0[0,1,2],xmm1[3]
 ; X86-AVX512-NEXT:    retl ## encoding: [0xc3]
@@ -679,7 +680,8 @@ define <4 x i32> @pinsrd_from_shufflevector_i32(<4 x i32> %a, <4 x i32>* nocaptu
 ;
 ; X64-AVX512-LABEL: pinsrd_from_shufflevector_i32:
 ; X64-AVX512:       ## %bb.0: ## %entry
-; X64-AVX512-NEXT:    vbroadcastss (%rdi), %xmm1 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x18,0x0f]
+; X64-AVX512-NEXT:    vpermilps $36, (%rdi), %xmm1 ## EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x04,0x0f,0x24]
+; X64-AVX512-NEXT:    ## xmm1 = mem[0,1,2,0]
 ; X64-AVX512-NEXT:    vblendps $8, %xmm1, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x0c,0xc1,0x08]
 ; X64-AVX512-NEXT:    ## xmm0 = xmm0[0,1,2],xmm1[3]
 ; X64-AVX512-NEXT:    retq ## encoding: [0xc3]
@@ -1224,7 +1226,8 @@ define <4 x i32> @i32_shuf_X00A(<4 x i32> %x, <4 x i32> %a) {
 ; AVX512-NEXT:    vxorps %xmm2, %xmm2, %xmm2 ## encoding: [0xc5,0xe8,0x57,0xd2]
 ; AVX512-NEXT:    vblendps $1, %xmm0, %xmm2, %xmm0 ## encoding: [0xc4,0xe3,0x69,0x0c,0xc0,0x01]
 ; AVX512-NEXT:    ## xmm0 = xmm0[0],xmm2[1,2,3]
-; AVX512-NEXT:    vbroadcastss %xmm1, %xmm1 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x18,0xc9]
+; AVX512-NEXT:    vpermilps $36, %xmm1, %xmm1 ## EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x04,0xc9,0x24]
+; AVX512-NEXT:    ## xmm1 = xmm1[0,1,2,0]
 ; AVX512-NEXT:    vblendps $8, %xmm1, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x0c,0xc1,0x08]
 ; AVX512-NEXT:    ## xmm0 = xmm0[0,1,2],xmm1[3]
 ; AVX512-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
@@ -1556,9 +1559,8 @@ define <4 x float> @insertps_from_broadcast_loadf32(<4 x float> %a, float* nocap
 ; X86-AVX512:       ## %bb.0:
 ; X86-AVX512-NEXT:    movl {{[0-9]+}}(%esp), %eax ## encoding: [0x8b,0x44,0x24,0x08]
 ; X86-AVX512-NEXT:    movl {{[0-9]+}}(%esp), %ecx ## encoding: [0x8b,0x4c,0x24,0x04]
-; X86-AVX512-NEXT:    vbroadcastss (%ecx,%eax,4), %xmm1 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x18,0x0c,0x81]
-; X86-AVX512-NEXT:    vinsertps $48, %xmm1, %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x21,0xc1,0x30]
-; X86-AVX512-NEXT:    ## xmm0 = xmm0[0,1,2],xmm1[0]
+; X86-AVX512-NEXT:    vinsertps $48, (%ecx,%eax,4), %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x21,0x04,0x81,0x30]
+; X86-AVX512-NEXT:    ## xmm0 = xmm0[0,1,2],mem[0]
 ; X86-AVX512-NEXT:    retl ## encoding: [0xc3]
 ;
 ; X64-SSE-LABEL: insertps_from_broadcast_loadf32:
@@ -1575,9 +1577,8 @@ define <4 x float> @insertps_from_broadcast_loadf32(<4 x float> %a, float* nocap
 ;
 ; X64-AVX512-LABEL: insertps_from_broadcast_loadf32:
 ; X64-AVX512:       ## %bb.0:
-; X64-AVX512-NEXT:    vbroadcastss (%rdi,%rsi,4), %xmm1 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x18,0x0c,0xb7]
-; X64-AVX512-NEXT:    vinsertps $48, %xmm1, %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x21,0xc1,0x30]
-; X64-AVX512-NEXT:    ## xmm0 = xmm0[0,1,2],xmm1[0]
+; X64-AVX512-NEXT:    vinsertps $48, (%rdi,%rsi,4), %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x21,0x04,0xb7,0x30]
+; X64-AVX512-NEXT:    ## xmm0 = xmm0[0,1,2],mem[0]
 ; X64-AVX512-NEXT:    retq ## encoding: [0xc3]
   %1 = getelementptr inbounds float, float* %fb, i64 %index
   %2 = load float, float* %1, align 4
@@ -1608,9 +1609,8 @@ define <4 x float> @insertps_from_broadcast_loadv4f32(<4 x float> %a, <4 x float
 ; X86-AVX512-LABEL: insertps_from_broadcast_loadv4f32:
 ; X86-AVX512:       ## %bb.0:
 ; X86-AVX512-NEXT:    movl {{[0-9]+}}(%esp), %eax ## encoding: [0x8b,0x44,0x24,0x04]
-; X86-AVX512-NEXT:    vbroadcastss (%eax), %xmm1 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x18,0x08]
-; X86-AVX512-NEXT:    vinsertps $48, %xmm1, %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x21,0xc1,0x30]
-; X86-AVX512-NEXT:    ## xmm0 = xmm0[0,1,2],xmm1[0]
+; X86-AVX512-NEXT:    vinsertps $48, (%eax), %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x21,0x00,0x30]
+; X86-AVX512-NEXT:    ## xmm0 = xmm0[0,1,2],mem[0]
 ; X86-AVX512-NEXT:    retl ## encoding: [0xc3]
 ;
 ; X64-SSE-LABEL: insertps_from_broadcast_loadv4f32:
@@ -1628,9 +1628,8 @@ define <4 x float> @insertps_from_broadcast_loadv4f32(<4 x float> %a, <4 x float
 ;
 ; X64-AVX512-LABEL: insertps_from_broadcast_loadv4f32:
 ; X64-AVX512:       ## %bb.0:
-; X64-AVX512-NEXT:    vbroadcastss (%rdi), %xmm1 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x18,0x0f]
-; X64-AVX512-NEXT:    vinsertps $48, %xmm1, %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x21,0xc1,0x30]
-; X64-AVX512-NEXT:    ## xmm0 = xmm0[0,1,2],xmm1[0]
+; X64-AVX512-NEXT:    vinsertps $48, (%rdi), %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x21,0x07,0x30]
+; X64-AVX512-NEXT:    ## xmm0 = xmm0[0,1,2],mem[0]
 ; X64-AVX512-NEXT:    retq ## encoding: [0xc3]
   %1 = load <4 x float>, <4 x float>* %b, align 4
   %2 = extractelement <4 x float> %1, i32 0
diff --git a/llvm/test/CodeGen/X86/ssub_sat.ll b/llvm/test/CodeGen/X86/ssub_sat.ll
index 9a1633df4..1cecaab68 100644
--- a/llvm/test/CodeGen/X86/ssub_sat.ll
+++ b/llvm/test/CodeGen/X86/ssub_sat.ll
@@ -2,39 +2,37 @@
 ; RUN: llc < %s -mtriple=i686 -mattr=cmov | FileCheck %s --check-prefixes=CHECK,X86
 ; RUN: llc < %s -mtriple=x86_64-linux | FileCheck %s --check-prefixes=CHECK,X64
 
-declare  i4  @llvm.ssub.sat.i4   (i4,  i4)
-declare  i32 @llvm.ssub.sat.i32  (i32, i32)
-declare  i64 @llvm.ssub.sat.i64  (i64, i64)
-declare  <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32>, <4 x i32>)
+declare i4 @llvm.ssub.sat.i4(i4, i4)
+declare i8 @llvm.ssub.sat.i8(i8, i8)
+declare i16 @llvm.ssub.sat.i16(i16, i16)
+declare i32 @llvm.ssub.sat.i32(i32, i32)
+declare i64 @llvm.ssub.sat.i64(i64, i64)
+declare <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32>, <4 x i32>)
 
 define i32 @func(i32 %x, i32 %y) nounwind {
 ; X86-LABEL: func:
 ; X86:       # %bb.0:
-; X86-NEXT:    pushl %esi
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-NEXT:    xorl %ecx, %ecx
-; X86-NEXT:    movl %eax, %esi
-; X86-NEXT:    subl %edx, %esi
+; X86-NEXT:    cmpl %edx, %eax
 ; X86-NEXT:    setns %cl
 ; X86-NEXT:    addl $2147483647, %ecx # imm = 0x7FFFFFFF
 ; X86-NEXT:    subl %edx, %eax
 ; X86-NEXT:    cmovol %ecx, %eax
-; X86-NEXT:    popl %esi
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: func:
 ; X64:       # %bb.0:
 ; X64-NEXT:    xorl %eax, %eax
-; X64-NEXT:    movl %edi, %ecx
-; X64-NEXT:    subl %esi, %ecx
+; X64-NEXT:    cmpl %esi, %edi
 ; X64-NEXT:    setns %al
 ; X64-NEXT:    addl $2147483647, %eax # imm = 0x7FFFFFFF
 ; X64-NEXT:    subl %esi, %edi
 ; X64-NEXT:    cmovnol %edi, %eax
 ; X64-NEXT:    retq
-  %tmp = call i32 @llvm.ssub.sat.i32(i32 %x, i32 %y);
-  ret i32 %tmp;
+  %tmp = call i32 @llvm.ssub.sat.i32(i32 %x, i32 %y)
+  ret i32 %tmp
 }
 
 define i64 @func2(i64 %x, i64 %y) nounwind {
@@ -77,68 +75,116 @@ define i64 @func2(i64 %x, i64 %y) nounwind {
 ; X64-LABEL: func2:
 ; X64:       # %bb.0:
 ; X64-NEXT:    xorl %ecx, %ecx
-; X64-NEXT:    movq %rdi, %rax
-; X64-NEXT:    subq %rsi, %rax
+; X64-NEXT:    cmpq %rsi, %rdi
 ; X64-NEXT:    setns %cl
 ; X64-NEXT:    movabsq $9223372036854775807, %rax # imm = 0x7FFFFFFFFFFFFFFF
 ; X64-NEXT:    addq %rcx, %rax
 ; X64-NEXT:    subq %rsi, %rdi
 ; X64-NEXT:    cmovnoq %rdi, %rax
 ; X64-NEXT:    retq
-  %tmp = call i64 @llvm.ssub.sat.i64(i64 %x, i64 %y);
-  ret i64 %tmp;
+  %tmp = call i64 @llvm.ssub.sat.i64(i64 %x, i64 %y)
+  ret i64 %tmp
 }
 
-define i4 @func3(i4 %x, i4 %y) nounwind {
-; X86-LABEL: func3:
+define signext i16 @func16(i16 signext %x, i16 signext %y) nounwind {
+; X86-LABEL: func16:
+; X86:       # %bb.0:
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    xorl %ecx, %ecx
+; X86-NEXT:    cmpw %dx, %ax
+; X86-NEXT:    setns %cl
+; X86-NEXT:    addl $32767, %ecx # imm = 0x7FFF
+; X86-NEXT:    subw %dx, %ax
+; X86-NEXT:    cmovol %ecx, %eax
+; X86-NEXT:    # kill: def $ax killed $ax killed $eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: func16:
+; X64:       # %bb.0:
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpw %si, %di
+; X64-NEXT:    setns %al
+; X64-NEXT:    addl $32767, %eax # imm = 0x7FFF
+; X64-NEXT:    subw %si, %di
+; X64-NEXT:    cmovnol %edi, %eax
+; X64-NEXT:    # kill: def $ax killed $ax killed $eax
+; X64-NEXT:    retq
+  %tmp = call i16 @llvm.ssub.sat.i16(i16 %x, i16 %y)
+  ret i16 %tmp
+}
+
+define signext i8 @func8(i8 signext %x, i8 signext %y) nounwind {
+; X86-LABEL: func8:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
 ; X86-NEXT:    movb {{[0-9]+}}(%esp), %dl
-; X86-NEXT:    shlb $4, %dl
-; X86-NEXT:    shlb $4, %al
 ; X86-NEXT:    xorl %ecx, %ecx
-; X86-NEXT:    movb %al, %ah
-; X86-NEXT:    subb %dl, %ah
+; X86-NEXT:    cmpb %dl, %al
 ; X86-NEXT:    setns %cl
 ; X86-NEXT:    addl $127, %ecx
 ; X86-NEXT:    subb %dl, %al
 ; X86-NEXT:    movzbl %al, %eax
 ; X86-NEXT:    cmovol %ecx, %eax
-; X86-NEXT:    sarb $4, %al
 ; X86-NEXT:    # kill: def $al killed $al killed $eax
 ; X86-NEXT:    retl
 ;
-; X64-LABEL: func3:
+; X64-LABEL: func8:
 ; X64:       # %bb.0:
-; X64-NEXT:    shlb $4, %sil
-; X64-NEXT:    shlb $4, %dil
 ; X64-NEXT:    xorl %ecx, %ecx
-; X64-NEXT:    movl %edi, %eax
-; X64-NEXT:    subb %sil, %al
+; X64-NEXT:    cmpb %sil, %dil
 ; X64-NEXT:    setns %cl
 ; X64-NEXT:    addl $127, %ecx
 ; X64-NEXT:    subb %sil, %dil
 ; X64-NEXT:    movzbl %dil, %eax
 ; X64-NEXT:    cmovol %ecx, %eax
-; X64-NEXT:    sarb $4, %al
 ; X64-NEXT:    # kill: def $al killed $al killed $eax
 ; X64-NEXT:    retq
-  %tmp = call i4 @llvm.ssub.sat.i4(i4 %x, i4 %y);
-  ret i4 %tmp;
+  %tmp = call i8 @llvm.ssub.sat.i8(i8 %x, i8 %y)
+  ret i8 %tmp
+}
+
+define signext i4 @func3(i4 signext %x, i4 signext %y) nounwind {
+; X86-LABEL: func3:
+; X86:       # %bb.0:
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    subb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    movzbl %al, %ecx
+; X86-NEXT:    cmpb $7, %al
+; X86-NEXT:    movl $7, %eax
+; X86-NEXT:    cmovll %ecx, %eax
+; X86-NEXT:    cmpb $-8, %al
+; X86-NEXT:    movl $248, %ecx
+; X86-NEXT:    cmovgl %eax, %ecx
+; X86-NEXT:    movsbl %cl, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: func3:
+; X64:       # %bb.0:
+; X64-NEXT:    subb %sil, %dil
+; X64-NEXT:    movzbl %dil, %eax
+; X64-NEXT:    cmpb $7, %al
+; X64-NEXT:    movl $7, %ecx
+; X64-NEXT:    cmovll %eax, %ecx
+; X64-NEXT:    cmpb $-8, %cl
+; X64-NEXT:    movl $248, %eax
+; X64-NEXT:    cmovgl %ecx, %eax
+; X64-NEXT:    movsbl %al, %eax
+; X64-NEXT:    retq
+  %tmp = call i4 @llvm.ssub.sat.i4(i4 %x, i4 %y)
+  ret i4 %tmp
 }
 
 define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; X86-LABEL: vec:
 ; X86:       # %bb.0:
-; X86-NEXT:    pushl %ebp
 ; X86-NEXT:    pushl %ebx
 ; X86-NEXT:    pushl %edi
 ; X86-NEXT:    pushl %esi
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-NEXT:    xorl %eax, %eax
-; X86-NEXT:    movl %ecx, %esi
-; X86-NEXT:    subl %edx, %esi
+; X86-NEXT:    cmpl %edx, %ecx
 ; X86-NEXT:    setns %al
 ; X86-NEXT:    addl $2147483647, %eax # imm = 0x7FFFFFFF
 ; X86-NEXT:    subl %edx, %ecx
@@ -146,8 +192,7 @@ define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; X86-NEXT:    cmovol %eax, %ecx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-NEXT:    xorl %eax, %eax
-; X86-NEXT:    movl %edx, %edi
-; X86-NEXT:    subl %esi, %edi
+; X86-NEXT:    cmpl %esi, %edx
 ; X86-NEXT:    setns %al
 ; X86-NEXT:    addl $2147483647, %eax # imm = 0x7FFFFFFF
 ; X86-NEXT:    subl %esi, %edx
@@ -155,8 +200,7 @@ define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; X86-NEXT:    cmovol %eax, %edx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
 ; X86-NEXT:    xorl %eax, %eax
-; X86-NEXT:    movl %esi, %ebx
-; X86-NEXT:    subl %edi, %ebx
+; X86-NEXT:    cmpl %edi, %esi
 ; X86-NEXT:    setns %al
 ; X86-NEXT:    addl $2147483647, %eax # imm = 0x7FFFFFFF
 ; X86-NEXT:    subl %edi, %esi
@@ -164,8 +208,7 @@ define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; X86-NEXT:    cmovol %eax, %esi
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    xorl %ebx, %ebx
-; X86-NEXT:    movl %edi, %ebp
-; X86-NEXT:    subl %eax, %ebp
+; X86-NEXT:    cmpl %eax, %edi
 ; X86-NEXT:    setns %bl
 ; X86-NEXT:    addl $2147483647, %ebx # imm = 0x7FFFFFFF
 ; X86-NEXT:    subl %eax, %edi
@@ -178,36 +221,25 @@ define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; X86-NEXT:    popl %esi
 ; X86-NEXT:    popl %edi
 ; X86-NEXT:    popl %ebx
-; X86-NEXT:    popl %ebp
 ; X86-NEXT:    retl $4
 ;
 ; X64-LABEL: vec:
 ; X64:       # %bb.0:
-; X64-NEXT:    movdqa %xmm0, %xmm2
-; X64-NEXT:    pxor %xmm3, %xmm3
-; X64-NEXT:    pxor %xmm0, %xmm0
-; X64-NEXT:    pcmpgtd %xmm1, %xmm0
-; X64-NEXT:    pcmpeqd %xmm4, %xmm4
-; X64-NEXT:    pxor %xmm4, %xmm0
-; X64-NEXT:    pxor %xmm5, %xmm5
-; X64-NEXT:    pcmpgtd %xmm2, %xmm5
-; X64-NEXT:    pxor %xmm4, %xmm5
-; X64-NEXT:    pcmpeqd %xmm5, %xmm0
-; X64-NEXT:    psubd %xmm1, %xmm2
-; X64-NEXT:    pcmpgtd %xmm2, %xmm3
-; X64-NEXT:    movdqa %xmm3, %xmm1
-; X64-NEXT:    pxor %xmm4, %xmm1
-; X64-NEXT:    pcmpeqd %xmm5, %xmm1
-; X64-NEXT:    pxor %xmm4, %xmm1
-; X64-NEXT:    pandn %xmm1, %xmm0
+; X64-NEXT:    pxor %xmm2, %xmm2
+; X64-NEXT:    movdqa %xmm0, %xmm3
+; X64-NEXT:    psubd %xmm1, %xmm3
+; X64-NEXT:    pcmpgtd %xmm2, %xmm1
+; X64-NEXT:    pcmpgtd %xmm3, %xmm0
+; X64-NEXT:    pxor %xmm1, %xmm0
 ; X64-NEXT:    movdqa %xmm3, %xmm1
 ; X64-NEXT:    pandn {{.*}}(%rip), %xmm1
-; X64-NEXT:    psrld $1, %xmm3
-; X64-NEXT:    por %xmm1, %xmm3
-; X64-NEXT:    pand %xmm0, %xmm3
-; X64-NEXT:    pandn %xmm2, %xmm0
-; X64-NEXT:    por %xmm3, %xmm0
+; X64-NEXT:    pcmpgtd %xmm3, %xmm2
+; X64-NEXT:    psrld $1, %xmm2
+; X64-NEXT:    por %xmm2, %xmm1
+; X64-NEXT:    pand %xmm0, %xmm1
+; X64-NEXT:    pandn %xmm3, %xmm0
+; X64-NEXT:    por %xmm1, %xmm0
 ; X64-NEXT:    retq
-  %tmp = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> %x, <4 x i32> %y);
-  ret <4 x i32> %tmp;
+  %tmp = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> %x, <4 x i32> %y)
+  ret <4 x i32> %tmp
 }
diff --git a/llvm/test/CodeGen/X86/ssub_sat_plus.ll b/llvm/test/CodeGen/X86/ssub_sat_plus.ll
new file mode 100644
index 000000000..12c587f1a
--- /dev/null
+++ b/llvm/test/CodeGen/X86/ssub_sat_plus.ll
@@ -0,0 +1,197 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686 -mattr=cmov | FileCheck %s --check-prefixes=CHECK,X86
+; RUN: llc < %s -mtriple=x86_64-linux | FileCheck %s --check-prefixes=CHECK,X64
+
+declare i4 @llvm.ssub.sat.i4(i4, i4)
+declare i8 @llvm.ssub.sat.i8(i8, i8)
+declare i16 @llvm.ssub.sat.i16(i16, i16)
+declare i32 @llvm.ssub.sat.i32(i32, i32)
+declare i64 @llvm.ssub.sat.i64(i64, i64)
+
+define i32 @func32(i32 %x, i32 %y, i32 %z) nounwind {
+; X86-LABEL: func32:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    imull {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    xorl %ecx, %ecx
+; X86-NEXT:    cmpl %edx, %eax
+; X86-NEXT:    setns %cl
+; X86-NEXT:    addl $2147483647, %ecx # imm = 0x7FFFFFFF
+; X86-NEXT:    subl %edx, %eax
+; X86-NEXT:    cmovol %ecx, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: func32:
+; X64:       # %bb.0:
+; X64-NEXT:    imull %edx, %esi
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpl %esi, %edi
+; X64-NEXT:    setns %al
+; X64-NEXT:    addl $2147483647, %eax # imm = 0x7FFFFFFF
+; X64-NEXT:    subl %esi, %edi
+; X64-NEXT:    cmovnol %edi, %eax
+; X64-NEXT:    retq
+  %a = mul i32 %y, %z
+  %tmp = call i32 @llvm.ssub.sat.i32(i32 %x, i32 %a)
+  ret i32 %tmp
+}
+
+define i64 @func64(i64 %x, i64 %y, i64 %z) nounwind {
+; X86-LABEL: func64:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT:    subl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl %ebx, %ebp
+; X86-NEXT:    sbbl %esi, %ebp
+; X86-NEXT:    movl %ebp, %eax
+; X86-NEXT:    sarl $31, %eax
+; X86-NEXT:    xorl %ecx, %ecx
+; X86-NEXT:    testl %ebp, %ebp
+; X86-NEXT:    setns %cl
+; X86-NEXT:    movl %ecx, %edx
+; X86-NEXT:    addl $2147483647, %edx # imm = 0x7FFFFFFF
+; X86-NEXT:    testl %ebx, %ebx
+; X86-NEXT:    setns %bl
+; X86-NEXT:    cmpb %cl, %bl
+; X86-NEXT:    setne %cl
+; X86-NEXT:    testl %esi, %esi
+; X86-NEXT:    setns %ch
+; X86-NEXT:    cmpb %ch, %bl
+; X86-NEXT:    setne %ch
+; X86-NEXT:    testb %cl, %ch
+; X86-NEXT:    cmovel %ebp, %edx
+; X86-NEXT:    cmovel %edi, %eax
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl
+;
+; X64-LABEL: func64:
+; X64:       # %bb.0:
+; X64-NEXT:    xorl %ecx, %ecx
+; X64-NEXT:    cmpq %rdx, %rdi
+; X64-NEXT:    setns %cl
+; X64-NEXT:    movabsq $9223372036854775807, %rax # imm = 0x7FFFFFFFFFFFFFFF
+; X64-NEXT:    addq %rcx, %rax
+; X64-NEXT:    subq %rdx, %rdi
+; X64-NEXT:    cmovnoq %rdi, %rax
+; X64-NEXT:    retq
+  %a = mul i64 %y, %z
+  %tmp = call i64 @llvm.ssub.sat.i64(i64 %x, i64 %z)
+  ret i64 %tmp
+}
+
+define signext i16 @func16(i16 signext %x, i16 signext %y, i16 signext %z) nounwind {
+; X86-LABEL: func16:
+; X86:       # %bb.0:
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    imulw {{[0-9]+}}(%esp), %dx
+; X86-NEXT:    xorl %ecx, %ecx
+; X86-NEXT:    cmpw %dx, %ax
+; X86-NEXT:    setns %cl
+; X86-NEXT:    addl $32767, %ecx # imm = 0x7FFF
+; X86-NEXT:    subw %dx, %ax
+; X86-NEXT:    cmovol %ecx, %eax
+; X86-NEXT:    # kill: def $ax killed $ax killed $eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: func16:
+; X64:       # %bb.0:
+; X64-NEXT:    imull %edx, %esi
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpw %si, %di
+; X64-NEXT:    setns %al
+; X64-NEXT:    addl $32767, %eax # imm = 0x7FFF
+; X64-NEXT:    subw %si, %di
+; X64-NEXT:    cmovnol %edi, %eax
+; X64-NEXT:    # kill: def $ax killed $ax killed $eax
+; X64-NEXT:    retq
+  %a = mul i16 %y, %z
+  %tmp = call i16 @llvm.ssub.sat.i16(i16 %x, i16 %a)
+  ret i16 %tmp
+}
+
+define signext i8 @func8(i8 signext %x, i8 signext %y, i8 signext %z) nounwind {
+; X86-LABEL: func8:
+; X86:       # %bb.0:
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %dl
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    mulb {{[0-9]+}}(%esp)
+; X86-NEXT:    xorl %ecx, %ecx
+; X86-NEXT:    cmpb %al, %dl
+; X86-NEXT:    setns %cl
+; X86-NEXT:    addl $127, %ecx
+; X86-NEXT:    subb %al, %dl
+; X86-NEXT:    movzbl %dl, %eax
+; X86-NEXT:    cmovol %ecx, %eax
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: func8:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %esi, %eax
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    mulb %dl
+; X64-NEXT:    xorl %ecx, %ecx
+; X64-NEXT:    cmpb %al, %dil
+; X64-NEXT:    setns %cl
+; X64-NEXT:    addl $127, %ecx
+; X64-NEXT:    subb %al, %dil
+; X64-NEXT:    movzbl %dil, %eax
+; X64-NEXT:    cmovol %ecx, %eax
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    retq
+  %a = mul i8 %y, %z
+  %tmp = call i8 @llvm.ssub.sat.i8(i8 %x, i8 %a)
+  ret i8 %tmp
+}
+
+define signext i4 @func4(i4 signext %x, i4 signext %y, i4 signext %z) nounwind {
+; X86-LABEL: func4:
+; X86:       # %bb.0:
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    mulb {{[0-9]+}}(%esp)
+; X86-NEXT:    shlb $4, %al
+; X86-NEXT:    sarb $4, %al
+; X86-NEXT:    subb %al, %cl
+; X86-NEXT:    movzbl %cl, %eax
+; X86-NEXT:    cmpb $7, %cl
+; X86-NEXT:    movl $7, %ecx
+; X86-NEXT:    cmovll %eax, %ecx
+; X86-NEXT:    cmpb $-8, %cl
+; X86-NEXT:    movl $248, %eax
+; X86-NEXT:    cmovgl %ecx, %eax
+; X86-NEXT:    movsbl %al, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: func4:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %esi, %eax
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    mulb %dl
+; X64-NEXT:    shlb $4, %al
+; X64-NEXT:    sarb $4, %al
+; X64-NEXT:    subb %al, %dil
+; X64-NEXT:    movzbl %dil, %eax
+; X64-NEXT:    cmpb $7, %al
+; X64-NEXT:    movl $7, %ecx
+; X64-NEXT:    cmovll %eax, %ecx
+; X64-NEXT:    cmpb $-8, %cl
+; X64-NEXT:    movl $248, %eax
+; X64-NEXT:    cmovgl %ecx, %eax
+; X64-NEXT:    movsbl %al, %eax
+; X64-NEXT:    retq
+  %a = mul i4 %y, %z
+  %tmp = call i4 @llvm.ssub.sat.i4(i4 %x, i4 %a)
+  ret i4 %tmp
+}
diff --git a/llvm/test/CodeGen/X86/ssub_sat_vec.ll b/llvm/test/CodeGen/X86/ssub_sat_vec.ll
index 0fb927c50..c3612a7a5 100644
--- a/llvm/test/CodeGen/X86/ssub_sat_vec.ll
+++ b/llvm/test/CodeGen/X86/ssub_sat_vec.ll
@@ -210,30 +210,13 @@ define void @v8i8(<8 x i8>* %px, <8 x i8>* %py, <8 x i8>* %pz) nounwind {
 ; SSE-NEXT:    movq %xmm0, (%rdx)
 ; SSE-NEXT:    retq
 ;
-; AVX1-LABEL: v8i8:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
-; AVX1-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
-; AVX1-NEXT:    vpsubsb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vmovq %xmm0, (%rdx)
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: v8i8:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
-; AVX2-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
-; AVX2-NEXT:    vpsubsb %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vmovq %xmm0, (%rdx)
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: v8i8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
-; AVX512-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
-; AVX512-NEXT:    vpsubsb %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX512-NEXT:    vpmovwb %xmm0, (%rdx)
-; AVX512-NEXT:    retq
+; AVX-LABEL: v8i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
+; AVX-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
+; AVX-NEXT:    vpsubsb %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vmovq %xmm0, (%rdx)
+; AVX-NEXT:    retq
   %x = load <8 x i8>, <8 x i8>* %px
   %y = load <8 x i8>, <8 x i8>* %py
   %z = call <8 x i8> @llvm.ssub.sat.v8i8(<8 x i8> %x, <8 x i8> %y)
@@ -250,30 +233,13 @@ define void @v4i8(<4 x i8>* %px, <4 x i8>* %py, <4 x i8>* %pz) nounwind {
 ; SSE-NEXT:    movd %xmm0, (%rdx)
 ; SSE-NEXT:    retq
 ;
-; AVX1-LABEL: v4i8:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX1-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX1-NEXT:    vpsubsb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vmovd %xmm0, (%rdx)
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: v4i8:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX2-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX2-NEXT:    vpsubsb %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vmovd %xmm0, (%rdx)
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: v4i8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX512-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX512-NEXT:    vpsubsb %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; AVX512-NEXT:    vpmovdb %xmm0, (%rdx)
-; AVX512-NEXT:    retq
+; AVX-LABEL: v4i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX-NEXT:    vpsubsb %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vmovd %xmm0, (%rdx)
+; AVX-NEXT:    retq
   %x = load <4 x i8>, <4 x i8>* %px
   %y = load <4 x i8>, <4 x i8>* %py
   %z = call <4 x i8> @llvm.ssub.sat.v4i8(<4 x i8> %x, <4 x i8> %y)
@@ -314,36 +280,15 @@ define void @v2i8(<2 x i8>* %px, <2 x i8>* %py, <2 x i8>* %pz) nounwind {
 ; SSE41-NEXT:    pextrw $0, %xmm0, (%rdx)
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: v2i8:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    movzwl (%rdi), %eax
-; AVX1-NEXT:    vmovd %eax, %xmm0
-; AVX1-NEXT:    movzwl (%rsi), %eax
-; AVX1-NEXT:    vmovd %eax, %xmm1
-; AVX1-NEXT:    vpsubsb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpextrw $0, %xmm0, (%rdx)
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: v2i8:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    movzwl (%rdi), %eax
-; AVX2-NEXT:    vmovd %eax, %xmm0
-; AVX2-NEXT:    movzwl (%rsi), %eax
-; AVX2-NEXT:    vmovd %eax, %xmm1
-; AVX2-NEXT:    vpsubsb %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpextrw $0, %xmm0, (%rdx)
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: v2i8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    movzwl (%rdi), %eax
-; AVX512-NEXT:    vmovd %eax, %xmm0
-; AVX512-NEXT:    movzwl (%rsi), %eax
-; AVX512-NEXT:    vmovd %eax, %xmm1
-; AVX512-NEXT:    vpsubsb %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT:    vpmovqb %xmm0, (%rdx)
-; AVX512-NEXT:    retq
+; AVX-LABEL: v2i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    movzwl (%rdi), %eax
+; AVX-NEXT:    vmovd %eax, %xmm0
+; AVX-NEXT:    movzwl (%rsi), %eax
+; AVX-NEXT:    vmovd %eax, %xmm1
+; AVX-NEXT:    vpsubsb %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpextrw $0, %xmm0, (%rdx)
+; AVX-NEXT:    retq
   %x = load <2 x i8>, <2 x i8>* %px
   %y = load <2 x i8>, <2 x i8>* %py
   %z = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> %x, <2 x i8> %y)
@@ -360,30 +305,13 @@ define void @v4i16(<4 x i16>* %px, <4 x i16>* %py, <4 x i16>* %pz) nounwind {
 ; SSE-NEXT:    movq %xmm0, (%rdx)
 ; SSE-NEXT:    retq
 ;
-; AVX1-LABEL: v4i16:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
-; AVX1-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
-; AVX1-NEXT:    vpsubsw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vmovq %xmm0, (%rdx)
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: v4i16:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
-; AVX2-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
-; AVX2-NEXT:    vpsubsw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vmovq %xmm0, (%rdx)
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: v4i16:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
-; AVX512-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
-; AVX512-NEXT:    vpsubsw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; AVX512-NEXT:    vpmovdw %xmm0, (%rdx)
-; AVX512-NEXT:    retq
+; AVX-LABEL: v4i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
+; AVX-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
+; AVX-NEXT:    vpsubsw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vmovq %xmm0, (%rdx)
+; AVX-NEXT:    retq
   %x = load <4 x i16>, <4 x i16>* %px
   %y = load <4 x i16>, <4 x i16>* %py
   %z = call <4 x i16> @llvm.ssub.sat.v4i16(<4 x i16> %x, <4 x i16> %y)
@@ -400,30 +328,13 @@ define void @v2i16(<2 x i16>* %px, <2 x i16>* %py, <2 x i16>* %pz) nounwind {
 ; SSE-NEXT:    movd %xmm0, (%rdx)
 ; SSE-NEXT:    retq
 ;
-; AVX1-LABEL: v2i16:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX1-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX1-NEXT:    vpsubsw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vmovd %xmm0, (%rdx)
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: v2i16:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX2-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX2-NEXT:    vpsubsw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vmovd %xmm0, (%rdx)
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: v2i16:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX512-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX512-NEXT:    vpsubsw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; AVX512-NEXT:    vpmovqw %xmm0, (%rdx)
-; AVX512-NEXT:    retq
+; AVX-LABEL: v2i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX-NEXT:    vpsubsw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vmovd %xmm0, (%rdx)
+; AVX-NEXT:    retq
   %x = load <2 x i16>, <2 x i16>* %px
   %y = load <2 x i16>, <2 x i16>* %py
   %z = call <2 x i16> @llvm.ssub.sat.v2i16(<2 x i16> %x, <2 x i16> %y)
@@ -497,30 +408,28 @@ define void @v12i16(<12 x i16>* %px, <12 x i16>* %py, <12 x i16>* %pz) nounwind
 define void @v1i8(<1 x i8>* %px, <1 x i8>* %py, <1 x i8>* %pz) nounwind {
 ; SSE-LABEL: v1i8:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    movb (%rdi), %cl
-; SSE-NEXT:    movb (%rsi), %dil
+; SSE-NEXT:    movb (%rdi), %al
+; SSE-NEXT:    movb (%rsi), %cl
 ; SSE-NEXT:    xorl %esi, %esi
-; SSE-NEXT:    movl %ecx, %eax
-; SSE-NEXT:    subb %dil, %al
+; SSE-NEXT:    cmpb %cl, %al
 ; SSE-NEXT:    setns %sil
 ; SSE-NEXT:    addl $127, %esi
-; SSE-NEXT:    subb %dil, %cl
-; SSE-NEXT:    movzbl %cl, %eax
+; SSE-NEXT:    subb %cl, %al
+; SSE-NEXT:    movzbl %al, %eax
 ; SSE-NEXT:    cmovol %esi, %eax
 ; SSE-NEXT:    movb %al, (%rdx)
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: v1i8:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    movb (%rdi), %cl
-; AVX-NEXT:    movb (%rsi), %dil
+; AVX-NEXT:    movb (%rdi), %al
+; AVX-NEXT:    movb (%rsi), %cl
 ; AVX-NEXT:    xorl %esi, %esi
-; AVX-NEXT:    movl %ecx, %eax
-; AVX-NEXT:    subb %dil, %al
+; AVX-NEXT:    cmpb %cl, %al
 ; AVX-NEXT:    setns %sil
 ; AVX-NEXT:    addl $127, %esi
-; AVX-NEXT:    subb %dil, %cl
-; AVX-NEXT:    movzbl %cl, %eax
+; AVX-NEXT:    subb %cl, %al
+; AVX-NEXT:    movzbl %al, %eax
 ; AVX-NEXT:    cmovol %esi, %eax
 ; AVX-NEXT:    movb %al, (%rdx)
 ; AVX-NEXT:    retq
@@ -537,8 +446,7 @@ define void @v1i16(<1 x i16>* %px, <1 x i16>* %py, <1 x i16>* %pz) nounwind {
 ; SSE-NEXT:    movzwl (%rdi), %eax
 ; SSE-NEXT:    movzwl (%rsi), %ecx
 ; SSE-NEXT:    xorl %esi, %esi
-; SSE-NEXT:    movl %eax, %edi
-; SSE-NEXT:    subw %cx, %di
+; SSE-NEXT:    cmpw %cx, %ax
 ; SSE-NEXT:    setns %sil
 ; SSE-NEXT:    addl $32767, %esi # imm = 0x7FFF
 ; SSE-NEXT:    subw %cx, %ax
@@ -551,8 +459,7 @@ define void @v1i16(<1 x i16>* %px, <1 x i16>* %py, <1 x i16>* %pz) nounwind {
 ; AVX-NEXT:    movzwl (%rdi), %eax
 ; AVX-NEXT:    movzwl (%rsi), %ecx
 ; AVX-NEXT:    xorl %esi, %esi
-; AVX-NEXT:    movl %eax, %edi
-; AVX-NEXT:    subw %cx, %di
+; AVX-NEXT:    cmpw %cx, %ax
 ; AVX-NEXT:    setns %sil
 ; AVX-NEXT:    addl $32767, %esi # imm = 0x7FFF
 ; AVX-NEXT:    subw %cx, %ax
@@ -658,248 +565,94 @@ define <16 x i1> @v16i1(<16 x i1> %x, <16 x i1> %y) nounwind {
 define <2 x i32> @v2i32(<2 x i32> %x, <2 x i32> %y) nounwind {
 ; SSE2-LABEL: v2i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    psllq $32, %xmm1
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
-; SSE2-NEXT:    psllq $32, %xmm0
+; SSE2-NEXT:    pxor %xmm2, %xmm2
 ; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    psubq %xmm1, %xmm0
-; SSE2-NEXT:    por %xmm2, %xmm1
-; SSE2-NEXT:    movdqa %xmm2, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pand %xmm5, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm4
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pxor %xmm1, %xmm4
-; SSE2-NEXT:    por %xmm2, %xmm3
-; SSE2-NEXT:    movdqa %xmm2, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm5
-; SSE2-NEXT:    pxor %xmm1, %xmm5
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,0,3,2]
-; SSE2-NEXT:    pand %xmm4, %xmm3
-; SSE2-NEXT:    movdqa %xmm0, %xmm4
-; SSE2-NEXT:    por %xmm2, %xmm4
-; SSE2-NEXT:    movdqa %xmm2, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm4, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3]
-; SSE2-NEXT:    pand %xmm7, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[1,1,3,3]
-; SSE2-NEXT:    por %xmm2, %xmm4
-; SSE2-NEXT:    movdqa %xmm4, %xmm2
-; SSE2-NEXT:    pxor %xmm1, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[1,0,3,2]
-; SSE2-NEXT:    pand %xmm2, %xmm5
-; SSE2-NEXT:    pxor %xmm1, %xmm5
-; SSE2-NEXT:    pandn %xmm5, %xmm3
-; SSE2-NEXT:    movdqa %xmm4, %xmm1
+; SSE2-NEXT:    psubd %xmm1, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm0
+; SSE2-NEXT:    movdqa %xmm3, %xmm1
 ; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm1
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm4
-; SSE2-NEXT:    por %xmm1, %xmm4
-; SSE2-NEXT:    pand %xmm3, %xmm4
-; SSE2-NEXT:    pandn %xmm0, %xmm3
-; SSE2-NEXT:    por %xmm4, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,3,2,3]
-; SSE2-NEXT:    psrad $31, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,3,2,3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm2
+; SSE2-NEXT:    psrld $1, %xmm2
+; SSE2-NEXT:    por %xmm2, %xmm1
+; SSE2-NEXT:    pand %xmm0, %xmm1
+; SSE2-NEXT:    pandn %xmm3, %xmm0
+; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: v2i32:
 ; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    psllq $32, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
-; SSSE3-NEXT:    psllq $32, %xmm0
+; SSSE3-NEXT:    pxor %xmm2, %xmm2
 ; SSSE3-NEXT:    movdqa %xmm0, %xmm3
-; SSSE3-NEXT:    psubq %xmm1, %xmm0
-; SSSE3-NEXT:    por %xmm2, %xmm1
-; SSSE3-NEXT:    movdqa %xmm2, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm5, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    por %xmm1, %xmm4
-; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSSE3-NEXT:    pxor %xmm1, %xmm4
-; SSSE3-NEXT:    por %xmm2, %xmm3
-; SSSE3-NEXT:    movdqa %xmm2, %xmm5
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm5
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm6, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSSE3-NEXT:    por %xmm3, %xmm5
-; SSSE3-NEXT:    pxor %xmm1, %xmm5
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm4, %xmm3
-; SSSE3-NEXT:    movdqa %xmm0, %xmm4
-; SSSE3-NEXT:    por %xmm2, %xmm4
-; SSSE3-NEXT:    movdqa %xmm2, %xmm6
-; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm7, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm4
-; SSSE3-NEXT:    movdqa %xmm4, %xmm2
-; SSSE3-NEXT:    pxor %xmm1, %xmm2
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm2, %xmm5
-; SSSE3-NEXT:    pxor %xmm1, %xmm5
-; SSSE3-NEXT:    pandn %xmm5, %xmm3
-; SSSE3-NEXT:    movdqa %xmm4, %xmm1
+; SSSE3-NEXT:    psubd %xmm1, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm1
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
+; SSSE3-NEXT:    pxor %xmm1, %xmm0
+; SSSE3-NEXT:    movdqa %xmm3, %xmm1
 ; SSSE3-NEXT:    pandn {{.*}}(%rip), %xmm1
-; SSSE3-NEXT:    pand {{.*}}(%rip), %xmm4
-; SSSE3-NEXT:    por %xmm1, %xmm4
-; SSSE3-NEXT:    pand %xmm3, %xmm4
-; SSSE3-NEXT:    pandn %xmm0, %xmm3
-; SSSE3-NEXT:    por %xmm4, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,3,2,3]
-; SSSE3-NEXT:    psrad $31, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,3,2,3]
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm2
+; SSSE3-NEXT:    psrld $1, %xmm2
+; SSSE3-NEXT:    por %xmm2, %xmm1
+; SSSE3-NEXT:    pand %xmm0, %xmm1
+; SSSE3-NEXT:    pandn %xmm3, %xmm0
+; SSSE3-NEXT:    por %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: v2i32:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    psllq $32, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483648,2147483648]
-; SSE41-NEXT:    psllq $32, %xmm2
-; SSE41-NEXT:    movdqa %xmm2, %xmm3
-; SSE41-NEXT:    psubq %xmm1, %xmm2
-; SSE41-NEXT:    por %xmm0, %xmm1
-; SSE41-NEXT:    movdqa %xmm0, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm1, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,1,3,3]
-; SSE41-NEXT:    por %xmm6, %xmm1
-; SSE41-NEXT:    pcmpeqd %xmm4, %xmm4
-; SSE41-NEXT:    pxor %xmm4, %xmm1
-; SSE41-NEXT:    por %xmm0, %xmm3
-; SSE41-NEXT:    movdqa %xmm0, %xmm5
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pand %xmm6, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE41-NEXT:    por %xmm3, %xmm5
-; SSE41-NEXT:    pxor %xmm4, %xmm5
-; SSE41-NEXT:    pcmpeqq %xmm5, %xmm1
+; SSE41-NEXT:    pxor %xmm0, %xmm0
 ; SSE41-NEXT:    movdqa %xmm2, %xmm3
-; SSE41-NEXT:    por %xmm0, %xmm3
-; SSE41-NEXT:    movdqa %xmm0, %xmm6
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pand %xmm7, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3]
-; SSE41-NEXT:    por %xmm3, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm3
-; SSE41-NEXT:    pxor %xmm4, %xmm3
-; SSE41-NEXT:    pcmpeqq %xmm5, %xmm3
-; SSE41-NEXT:    pxor %xmm4, %xmm3
-; SSE41-NEXT:    pandn %xmm3, %xmm1
-; SSE41-NEXT:    movapd {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808]
-; SSE41-NEXT:    blendvpd %xmm0, {{.*}}(%rip), %xmm3
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3]
-; SSE41-NEXT:    psrad $31, %xmm2
-; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
+; SSE41-NEXT:    psubd %xmm1, %xmm3
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm1
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm2
+; SSE41-NEXT:    pxor %xmm1, %xmm2
+; SSE41-NEXT:    movaps {{.*#+}} xmm1 = [2147483648,2147483648,2147483648,2147483648]
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, {{.*}}(%rip), %xmm1
 ; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm1, %xmm3
+; SSE41-NEXT:    movaps %xmm3, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: v2i32:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpsllq $32, %xmm1, %xmm1
 ; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm2, %xmm3
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
-; AVX1-NEXT:    vpsllq $32, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm5
-; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX1-NEXT:    vpcmpeqq %xmm3, %xmm5, %xmm3
-; AVX1-NEXT:    vpsubq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm4, %xmm1, %xmm2
-; AVX1-NEXT:    vpcmpeqq %xmm2, %xmm5, %xmm2
-; AVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm2
-; AVX1-NEXT:    vpandn %xmm2, %xmm3, %xmm2
-; AVX1-NEXT:    vmovapd {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808]
-; AVX1-NEXT:    vblendvpd %xmm1, {{.*}}(%rip), %xmm3, %xmm1
-; AVX1-NEXT:    vblendvpd %xmm2, %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrad $31, %xmm0, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
+; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm1, %xmm2
+; AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor %xmm0, %xmm2, %xmm0
+; AVX1-NEXT:    vmovaps {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648]
+; AVX1-NEXT:    vblendvps %xmm1, {{.*}}(%rip), %xmm2, %xmm2
+; AVX1-NEXT:    vblendvps %xmm0, %xmm2, %xmm1, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: v2i32:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpsllq $32, %xmm1, %xmm1
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtq %xmm1, %xmm2, %xmm3
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpxor %xmm4, %xmm3, %xmm3
-; AVX2-NEXT:    vpsllq $32, %xmm0, %xmm0
-; AVX2-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm5
-; AVX2-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX2-NEXT:    vpcmpeqq %xmm3, %xmm5, %xmm3
-; AVX2-NEXT:    vpsubq %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm1
-; AVX2-NEXT:    vpxor %xmm4, %xmm1, %xmm2
-; AVX2-NEXT:    vpcmpeqq %xmm2, %xmm5, %xmm2
-; AVX2-NEXT:    vpxor %xmm4, %xmm2, %xmm2
-; AVX2-NEXT:    vpandn %xmm2, %xmm3, %xmm2
-; AVX2-NEXT:    vmovapd {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808]
-; AVX2-NEXT:    vblendvpd %xmm1, {{.*}}(%rip), %xmm3, %xmm1
-; AVX2-NEXT:    vblendvpd %xmm2, %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrad $31, %xmm0, %xmm1
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
+; AVX2-NEXT:    vpcmpgtd %xmm2, %xmm1, %xmm2
+; AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm1
+; AVX2-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor %xmm0, %xmm2, %xmm0
+; AVX2-NEXT:    vbroadcastss {{.*#+}} xmm2 = [2147483647,2147483647,2147483647,2147483647]
+; AVX2-NEXT:    vbroadcastss {{.*#+}} xmm3 = [2147483648,2147483648,2147483648,2147483648]
+; AVX2-NEXT:    vblendvps %xmm1, %xmm2, %xmm3, %xmm2
+; AVX2-NEXT:    vblendvps %xmm0, %xmm2, %xmm1, %xmm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: v2i32:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsllq $32, %xmm1, %xmm1
 ; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpcmpnltq %xmm2, %xmm1, %k0
-; AVX512-NEXT:    vpsllq $32, %xmm0, %xmm0
-; AVX512-NEXT:    vpcmpnltq %xmm2, %xmm0, %k1
-; AVX512-NEXT:    kxorw %k0, %k1, %k0
-; AVX512-NEXT:    vpsubq %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpcmpnltq %xmm2, %xmm0, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
-; AVX512-NEXT:    kandw %k1, %k0, %k1
-; AVX512-NEXT:    vpcmpgtq %xmm0, %xmm2, %k2
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = [9223372036854775808,9223372036854775808]
-; AVX512-NEXT:    vmovdqa64 {{.*}}(%rip), %xmm1 {%k2}
-; AVX512-NEXT:    vmovdqa64 %xmm1, %xmm0 {%k1}
-; AVX512-NEXT:    vpsraq $32, %xmm0, %xmm0
+; AVX512-NEXT:    vpcmpgtd %xmm2, %xmm1, %k0
+; AVX512-NEXT:    vpsubd %xmm1, %xmm0, %xmm1
+; AVX512-NEXT:    vpcmpgtd %xmm1, %xmm0, %k1
+; AVX512-NEXT:    kxorw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpgtd %xmm1, %xmm2, %k2
+; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm0 = [2147483648,2147483648,2147483648,2147483648]
+; AVX512-NEXT:    vpbroadcastd {{.*}}(%rip), %xmm0 {%k2}
+; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm1 {%k1}
+; AVX512-NEXT:    vmovdqa %xmm1, %xmm0
 ; AVX512-NEXT:    retq
   %z = call <2 x i32> @llvm.ssub.sat.v2i32(<2 x i32> %x, <2 x i32> %y)
   ret <2 x i32> %z
@@ -908,141 +661,94 @@ define <2 x i32> @v2i32(<2 x i32> %x, <2 x i32> %y) nounwind {
 define <4 x i32> @v4i32(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; SSE2-LABEL: v4i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm4
-; SSE2-NEXT:    pxor %xmm4, %xmm0
-; SSE2-NEXT:    pxor %xmm5, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm5
-; SSE2-NEXT:    pxor %xmm4, %xmm5
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm0
-; SSE2-NEXT:    psubd %xmm1, %xmm2
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm3
-; SSE2-NEXT:    movdqa %xmm3, %xmm1
-; SSE2-NEXT:    pxor %xmm4, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm1
-; SSE2-NEXT:    pxor %xmm4, %xmm1
-; SSE2-NEXT:    pandn %xmm1, %xmm0
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    movdqa %xmm0, %xmm3
+; SSE2-NEXT:    psubd %xmm1, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm0
 ; SSE2-NEXT:    movdqa %xmm3, %xmm1
 ; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm1
-; SSE2-NEXT:    psrld $1, %xmm3
-; SSE2-NEXT:    por %xmm1, %xmm3
-; SSE2-NEXT:    pand %xmm0, %xmm3
-; SSE2-NEXT:    pandn %xmm2, %xmm0
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm2
+; SSE2-NEXT:    psrld $1, %xmm2
+; SSE2-NEXT:    por %xmm2, %xmm1
+; SSE2-NEXT:    pand %xmm0, %xmm1
+; SSE2-NEXT:    pandn %xmm3, %xmm0
+; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: v4i32:
 ; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    movdqa %xmm0, %xmm2
-; SSSE3-NEXT:    pxor %xmm3, %xmm3
-; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm4
-; SSSE3-NEXT:    pxor %xmm4, %xmm0
-; SSSE3-NEXT:    pxor %xmm5, %xmm5
-; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm5
-; SSSE3-NEXT:    pxor %xmm4, %xmm5
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm0
-; SSSE3-NEXT:    psubd %xmm1, %xmm2
-; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm3
-; SSSE3-NEXT:    movdqa %xmm3, %xmm1
-; SSSE3-NEXT:    pxor %xmm4, %xmm1
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm1
-; SSSE3-NEXT:    pxor %xmm4, %xmm1
-; SSSE3-NEXT:    pandn %xmm1, %xmm0
+; SSSE3-NEXT:    pxor %xmm2, %xmm2
+; SSSE3-NEXT:    movdqa %xmm0, %xmm3
+; SSSE3-NEXT:    psubd %xmm1, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm1
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
+; SSSE3-NEXT:    pxor %xmm1, %xmm0
 ; SSSE3-NEXT:    movdqa %xmm3, %xmm1
 ; SSSE3-NEXT:    pandn {{.*}}(%rip), %xmm1
-; SSSE3-NEXT:    psrld $1, %xmm3
-; SSSE3-NEXT:    por %xmm1, %xmm3
-; SSSE3-NEXT:    pand %xmm0, %xmm3
-; SSSE3-NEXT:    pandn %xmm2, %xmm0
-; SSSE3-NEXT:    por %xmm3, %xmm0
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm2
+; SSSE3-NEXT:    psrld $1, %xmm2
+; SSSE3-NEXT:    por %xmm2, %xmm1
+; SSSE3-NEXT:    pand %xmm0, %xmm1
+; SSSE3-NEXT:    pandn %xmm3, %xmm0
+; SSSE3-NEXT:    por %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: v4i32:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movdqa %xmm0, %xmm2
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    pxor %xmm3, %xmm3
-; SSE41-NEXT:    pcmpgtd %xmm1, %xmm3
-; SSE41-NEXT:    pcmpeqd %xmm4, %xmm4
-; SSE41-NEXT:    pxor %xmm4, %xmm3
-; SSE41-NEXT:    pxor %xmm5, %xmm5
-; SSE41-NEXT:    pcmpgtd %xmm2, %xmm5
-; SSE41-NEXT:    pxor %xmm4, %xmm5
-; SSE41-NEXT:    pcmpeqd %xmm5, %xmm3
-; SSE41-NEXT:    psubd %xmm1, %xmm2
-; SSE41-NEXT:    pcmpgtd %xmm2, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    pxor %xmm4, %xmm1
-; SSE41-NEXT:    pcmpeqd %xmm5, %xmm1
-; SSE41-NEXT:    pxor %xmm4, %xmm1
-; SSE41-NEXT:    pandn %xmm1, %xmm3
+; SSE41-NEXT:    movdqa %xmm2, %xmm3
+; SSE41-NEXT:    psubd %xmm1, %xmm3
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm1
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm2
+; SSE41-NEXT:    pxor %xmm1, %xmm2
 ; SSE41-NEXT:    movaps {{.*#+}} xmm1 = [2147483648,2147483648,2147483648,2147483648]
-; SSE41-NEXT:    blendvps %xmm0, {{.*}}(%rip), %xmm1
 ; SSE41-NEXT:    movdqa %xmm3, %xmm0
-; SSE41-NEXT:    blendvps %xmm0, %xmm1, %xmm2
-; SSE41-NEXT:    movaps %xmm2, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, {{.*}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm1, %xmm3
+; SSE41-NEXT:    movaps %xmm3, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: v4i32:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm3
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm2, %xmm5
-; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm5, %xmm3
-; AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm4, %xmm1, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm5, %xmm2
-; AVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm2
-; AVX1-NEXT:    vpandn %xmm2, %xmm3, %xmm2
-; AVX1-NEXT:    vmovaps {{.*#+}} xmm3 = [2147483648,2147483648,2147483648,2147483648]
-; AVX1-NEXT:    vblendvps %xmm1, {{.*}}(%rip), %xmm3, %xmm1
-; AVX1-NEXT:    vblendvps %xmm2, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm1, %xmm2
+; AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor %xmm0, %xmm2, %xmm0
+; AVX1-NEXT:    vmovaps {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648]
+; AVX1-NEXT:    vblendvps %xmm1, {{.*}}(%rip), %xmm2, %xmm2
+; AVX1-NEXT:    vblendvps %xmm0, %xmm2, %xmm1, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: v4i32:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm3
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpxor %xmm4, %xmm3, %xmm3
-; AVX2-NEXT:    vpcmpgtd %xmm0, %xmm2, %xmm5
-; AVX2-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm5, %xmm3
-; AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpcmpgtd %xmm0, %xmm2, %xmm1
-; AVX2-NEXT:    vpxor %xmm4, %xmm1, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm5, %xmm2
-; AVX2-NEXT:    vpxor %xmm4, %xmm2, %xmm2
-; AVX2-NEXT:    vpandn %xmm2, %xmm3, %xmm2
-; AVX2-NEXT:    vbroadcastss {{.*#+}} xmm3 = [2147483647,2147483647,2147483647,2147483647]
-; AVX2-NEXT:    vbroadcastss {{.*#+}} xmm4 = [2147483648,2147483648,2147483648,2147483648]
-; AVX2-NEXT:    vblendvps %xmm1, %xmm3, %xmm4, %xmm1
-; AVX2-NEXT:    vblendvps %xmm2, %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpcmpgtd %xmm2, %xmm1, %xmm2
+; AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm1
+; AVX2-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor %xmm0, %xmm2, %xmm0
+; AVX2-NEXT:    vbroadcastss {{.*#+}} xmm2 = [2147483647,2147483647,2147483647,2147483647]
+; AVX2-NEXT:    vbroadcastss {{.*#+}} xmm3 = [2147483648,2147483648,2147483648,2147483648]
+; AVX2-NEXT:    vblendvps %xmm1, %xmm2, %xmm3, %xmm2
+; AVX2-NEXT:    vblendvps %xmm0, %xmm2, %xmm1, %xmm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: v4i32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpcmpnltd %xmm2, %xmm1, %k0
-; AVX512-NEXT:    vpcmpnltd %xmm2, %xmm0, %k1
-; AVX512-NEXT:    kxorw %k0, %k1, %k0
-; AVX512-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpcmpnltd %xmm2, %xmm0, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
-; AVX512-NEXT:    kandw %k1, %k0, %k1
-; AVX512-NEXT:    vpcmpgtd %xmm0, %xmm2, %k2
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [2147483648,2147483648,2147483648,2147483648]
-; AVX512-NEXT:    vpbroadcastd {{.*}}(%rip), %xmm1 {%k2}
-; AVX512-NEXT:    vmovdqa32 %xmm1, %xmm0 {%k1}
+; AVX512-NEXT:    vpcmpgtd %xmm2, %xmm1, %k0
+; AVX512-NEXT:    vpsubd %xmm1, %xmm0, %xmm1
+; AVX512-NEXT:    vpcmpgtd %xmm1, %xmm0, %k1
+; AVX512-NEXT:    kxorw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpgtd %xmm1, %xmm2, %k2
+; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm0 = [2147483648,2147483648,2147483648,2147483648]
+; AVX512-NEXT:    vpbroadcastd {{.*}}(%rip), %xmm0 {%k2}
+; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm1 {%k1}
+; AVX512-NEXT:    vmovdqa %xmm1, %xmm0
 ; AVX512-NEXT:    retq
   %z = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> %x, <4 x i32> %y)
   ret <4 x i32> %z
@@ -1051,226 +757,144 @@ define <4 x i32> @v4i32(<4 x i32> %x, <4 x i32> %y) nounwind {
 define <8 x i32> @v8i32(<8 x i32> %x, <8 x i32> %y) nounwind {
 ; SSE2-LABEL: v8i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa %xmm0, %xmm4
-; SSE2-NEXT:    pxor %xmm5, %xmm5
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm0
-; SSE2-NEXT:    pcmpeqd %xmm8, %xmm8
-; SSE2-NEXT:    pxor %xmm8, %xmm0
+; SSE2-NEXT:    pxor %xmm4, %xmm4
+; SSE2-NEXT:    movdqa %xmm0, %xmm5
+; SSE2-NEXT:    psubd %xmm2, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm5, %xmm0
+; SSE2-NEXT:    pxor %xmm2, %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm5, %xmm6
+; SSE2-NEXT:    pandn %xmm2, %xmm6
 ; SSE2-NEXT:    pxor %xmm7, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm4, %xmm7
-; SSE2-NEXT:    pxor %xmm8, %xmm7
-; SSE2-NEXT:    pcmpeqd %xmm7, %xmm0
-; SSE2-NEXT:    psubd %xmm2, %xmm4
-; SSE2-NEXT:    pxor %xmm6, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm4, %xmm6
-; SSE2-NEXT:    movdqa %xmm6, %xmm2
-; SSE2-NEXT:    pxor %xmm8, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm7, %xmm2
-; SSE2-NEXT:    pxor %xmm8, %xmm2
-; SSE2-NEXT:    pandn %xmm2, %xmm0
-; SSE2-NEXT:    movdqa {{.*#+}} xmm7 = [2147483648,2147483648,2147483648,2147483648]
-; SSE2-NEXT:    movdqa %xmm6, %xmm2
-; SSE2-NEXT:    pandn %xmm7, %xmm2
-; SSE2-NEXT:    psrld $1, %xmm6
-; SSE2-NEXT:    por %xmm2, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm5, %xmm7
+; SSE2-NEXT:    psrld $1, %xmm7
+; SSE2-NEXT:    por %xmm7, %xmm6
 ; SSE2-NEXT:    pand %xmm0, %xmm6
-; SSE2-NEXT:    pandn %xmm4, %xmm0
+; SSE2-NEXT:    pandn %xmm5, %xmm0
 ; SSE2-NEXT:    por %xmm6, %xmm0
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm2
-; SSE2-NEXT:    pxor %xmm8, %xmm2
-; SSE2-NEXT:    pxor %xmm4, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm4
-; SSE2-NEXT:    pxor %xmm8, %xmm4
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm2
-; SSE2-NEXT:    psubd %xmm3, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm5
-; SSE2-NEXT:    movdqa %xmm5, %xmm3
-; SSE2-NEXT:    pxor %xmm8, %xmm3
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm3
-; SSE2-NEXT:    pxor %xmm8, %xmm3
-; SSE2-NEXT:    pandn %xmm3, %xmm2
+; SSE2-NEXT:    movdqa %xmm1, %xmm5
+; SSE2-NEXT:    psubd %xmm3, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm5, %xmm1
+; SSE2-NEXT:    pxor %xmm3, %xmm1
 ; SSE2-NEXT:    movdqa %xmm5, %xmm3
-; SSE2-NEXT:    pandn %xmm7, %xmm3
-; SSE2-NEXT:    psrld $1, %xmm5
-; SSE2-NEXT:    por %xmm3, %xmm5
-; SSE2-NEXT:    pand %xmm2, %xmm5
-; SSE2-NEXT:    pandn %xmm1, %xmm2
-; SSE2-NEXT:    por %xmm5, %xmm2
-; SSE2-NEXT:    movdqa %xmm2, %xmm1
+; SSE2-NEXT:    pandn %xmm2, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm5, %xmm4
+; SSE2-NEXT:    psrld $1, %xmm4
+; SSE2-NEXT:    por %xmm4, %xmm3
+; SSE2-NEXT:    pand %xmm1, %xmm3
+; SSE2-NEXT:    pandn %xmm5, %xmm1
+; SSE2-NEXT:    por %xmm3, %xmm1
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: v8i32:
 ; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    movdqa %xmm0, %xmm4
-; SSSE3-NEXT:    pxor %xmm5, %xmm5
-; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm0
-; SSSE3-NEXT:    pcmpeqd %xmm8, %xmm8
-; SSSE3-NEXT:    pxor %xmm8, %xmm0
+; SSSE3-NEXT:    pxor %xmm4, %xmm4
+; SSSE3-NEXT:    movdqa %xmm0, %xmm5
+; SSSE3-NEXT:    psubd %xmm2, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm0
+; SSSE3-NEXT:    pxor %xmm2, %xmm0
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648]
+; SSSE3-NEXT:    movdqa %xmm5, %xmm6
+; SSSE3-NEXT:    pandn %xmm2, %xmm6
 ; SSSE3-NEXT:    pxor %xmm7, %xmm7
-; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm7
-; SSSE3-NEXT:    pxor %xmm8, %xmm7
-; SSSE3-NEXT:    pcmpeqd %xmm7, %xmm0
-; SSSE3-NEXT:    psubd %xmm2, %xmm4
-; SSSE3-NEXT:    pxor %xmm6, %xmm6
-; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm6
-; SSSE3-NEXT:    movdqa %xmm6, %xmm2
-; SSSE3-NEXT:    pxor %xmm8, %xmm2
-; SSSE3-NEXT:    pcmpeqd %xmm7, %xmm2
-; SSSE3-NEXT:    pxor %xmm8, %xmm2
-; SSSE3-NEXT:    pandn %xmm2, %xmm0
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm7 = [2147483648,2147483648,2147483648,2147483648]
-; SSSE3-NEXT:    movdqa %xmm6, %xmm2
-; SSSE3-NEXT:    pandn %xmm7, %xmm2
-; SSSE3-NEXT:    psrld $1, %xmm6
-; SSSE3-NEXT:    por %xmm2, %xmm6
+; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm7
+; SSSE3-NEXT:    psrld $1, %xmm7
+; SSSE3-NEXT:    por %xmm7, %xmm6
 ; SSSE3-NEXT:    pand %xmm0, %xmm6
-; SSSE3-NEXT:    pandn %xmm4, %xmm0
+; SSSE3-NEXT:    pandn %xmm5, %xmm0
 ; SSSE3-NEXT:    por %xmm6, %xmm0
-; SSSE3-NEXT:    pxor %xmm2, %xmm2
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm2
-; SSSE3-NEXT:    pxor %xmm8, %xmm2
-; SSSE3-NEXT:    pxor %xmm4, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm4
-; SSSE3-NEXT:    pxor %xmm8, %xmm4
-; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm2
-; SSSE3-NEXT:    psubd %xmm3, %xmm1
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm5
-; SSSE3-NEXT:    movdqa %xmm5, %xmm3
-; SSSE3-NEXT:    pxor %xmm8, %xmm3
-; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm3
-; SSSE3-NEXT:    pxor %xmm8, %xmm3
-; SSSE3-NEXT:    pandn %xmm3, %xmm2
+; SSSE3-NEXT:    movdqa %xmm1, %xmm5
+; SSSE3-NEXT:    psubd %xmm3, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm1
+; SSSE3-NEXT:    pxor %xmm3, %xmm1
 ; SSSE3-NEXT:    movdqa %xmm5, %xmm3
-; SSSE3-NEXT:    pandn %xmm7, %xmm3
-; SSSE3-NEXT:    psrld $1, %xmm5
-; SSSE3-NEXT:    por %xmm3, %xmm5
-; SSSE3-NEXT:    pand %xmm2, %xmm5
-; SSSE3-NEXT:    pandn %xmm1, %xmm2
-; SSSE3-NEXT:    por %xmm5, %xmm2
-; SSSE3-NEXT:    movdqa %xmm2, %xmm1
+; SSSE3-NEXT:    pandn %xmm2, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm4
+; SSSE3-NEXT:    psrld $1, %xmm4
+; SSSE3-NEXT:    por %xmm4, %xmm3
+; SSSE3-NEXT:    pand %xmm1, %xmm3
+; SSSE3-NEXT:    pandn %xmm5, %xmm1
+; SSSE3-NEXT:    por %xmm3, %xmm1
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: v8i32:
 ; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm4
+; SSE41-NEXT:    pxor %xmm8, %xmm8
 ; SSE41-NEXT:    movdqa %xmm0, %xmm5
-; SSE41-NEXT:    pxor %xmm4, %xmm4
-; SSE41-NEXT:    pxor %xmm6, %xmm6
-; SSE41-NEXT:    pcmpgtd %xmm2, %xmm6
-; SSE41-NEXT:    pcmpeqd %xmm8, %xmm8
-; SSE41-NEXT:    pxor %xmm8, %xmm6
-; SSE41-NEXT:    pxor %xmm7, %xmm7
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm7
-; SSE41-NEXT:    pxor %xmm8, %xmm7
-; SSE41-NEXT:    pcmpeqd %xmm7, %xmm6
 ; SSE41-NEXT:    psubd %xmm2, %xmm5
-; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    pcmpgtd %xmm5, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    pxor %xmm8, %xmm2
-; SSE41-NEXT:    pcmpeqd %xmm7, %xmm2
-; SSE41-NEXT:    pxor %xmm8, %xmm2
-; SSE41-NEXT:    pandn %xmm2, %xmm6
-; SSE41-NEXT:    movaps {{.*#+}} xmm9 = [2147483647,2147483647,2147483647,2147483647]
-; SSE41-NEXT:    movaps {{.*#+}} xmm7 = [2147483648,2147483648,2147483648,2147483648]
-; SSE41-NEXT:    movaps %xmm7, %xmm2
-; SSE41-NEXT:    blendvps %xmm0, %xmm9, %xmm2
-; SSE41-NEXT:    movdqa %xmm6, %xmm0
-; SSE41-NEXT:    blendvps %xmm0, %xmm2, %xmm5
-; SSE41-NEXT:    xorps %xmm2, %xmm2
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm2
-; SSE41-NEXT:    pxor %xmm8, %xmm2
-; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE41-NEXT:    pxor %xmm8, %xmm0
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE41-NEXT:    psubd %xmm3, %xmm1
-; SSE41-NEXT:    pcmpgtd %xmm1, %xmm4
-; SSE41-NEXT:    movdqa %xmm4, %xmm3
-; SSE41-NEXT:    pxor %xmm8, %xmm3
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm3
-; SSE41-NEXT:    pxor %xmm8, %xmm3
-; SSE41-NEXT:    pandn %xmm3, %xmm2
+; SSE41-NEXT:    pcmpgtd %xmm8, %xmm2
+; SSE41-NEXT:    pcmpgtd %xmm5, %xmm4
+; SSE41-NEXT:    pxor %xmm2, %xmm4
+; SSE41-NEXT:    movaps {{.*#+}} xmm7 = [2147483647,2147483647,2147483647,2147483647]
+; SSE41-NEXT:    movaps {{.*#+}} xmm6 = [2147483648,2147483648,2147483648,2147483648]
+; SSE41-NEXT:    movaps %xmm6, %xmm2
+; SSE41-NEXT:    movdqa %xmm5, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm7, %xmm2
 ; SSE41-NEXT:    movdqa %xmm4, %xmm0
-; SSE41-NEXT:    blendvps %xmm0, %xmm9, %xmm7
+; SSE41-NEXT:    blendvps %xmm0, %xmm2, %xmm5
+; SSE41-NEXT:    movdqa %xmm1, %xmm2
+; SSE41-NEXT:    psubd %xmm3, %xmm2
+; SSE41-NEXT:    pcmpgtd %xmm8, %xmm3
+; SSE41-NEXT:    pcmpgtd %xmm2, %xmm1
+; SSE41-NEXT:    pxor %xmm3, %xmm1
 ; SSE41-NEXT:    movdqa %xmm2, %xmm0
-; SSE41-NEXT:    blendvps %xmm0, %xmm7, %xmm1
+; SSE41-NEXT:    blendvps %xmm0, %xmm7, %xmm6
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm6, %xmm2
 ; SSE41-NEXT:    movaps %xmm5, %xmm0
+; SSE41-NEXT:    movaps %xmm2, %xmm1
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: v8i32:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
 ; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm3, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
-; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm6
-; AVX1-NEXT:    vpcmpgtd %xmm6, %xmm3, %xmm7
-; AVX1-NEXT:    vpxor %xmm5, %xmm7, %xmm7
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm7, %xmm8
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm3, %xmm4
-; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm9
-; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm3, %xmm4
-; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm9, %xmm4, %xmm9
-; AVX1-NEXT:    vinsertf128 $1, %xmm8, %ymm9, %ymm8
-; AVX1-NEXT:    vpsubd %xmm2, %xmm6, %xmm9
-; AVX1-NEXT:    vpcmpgtd %xmm9, %xmm3, %xmm6
-; AVX1-NEXT:    vpxor %xmm5, %xmm6, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm7, %xmm2
-; AVX1-NEXT:    vpxor %xmm5, %xmm2, %xmm2
-; AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm3, %xmm1
-; AVX1-NEXT:    vpxor %xmm5, %xmm1, %xmm3
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm4, %xmm3
-; AVX1-NEXT:    vpxor %xmm5, %xmm3, %xmm3
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
-; AVX1-NEXT:    vandnps %ymm2, %ymm8, %ymm2
-; AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm1, %ymm1
-; AVX1-NEXT:    vmovaps {{.*#+}} ymm3 = [2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648]
-; AVX1-NEXT:    vblendvps %ymm1, {{.*}}(%rip), %ymm3, %ymm1
-; AVX1-NEXT:    vinsertf128 $1, %xmm9, %ymm0, %ymm0
-; AVX1-NEXT:    vblendvps %ymm2, %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vpcmpgtd %xmm3, %xmm2, %xmm4
+; AVX1-NEXT:    vpcmpgtd %xmm3, %xmm1, %xmm3
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
+; AVX1-NEXT:    vpsubd %xmm2, %xmm4, %xmm2
+; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm4, %xmm4
+; AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
+; AVX1-NEXT:    vxorps %ymm0, %ymm3, %ymm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; AVX1-NEXT:    vmovaps {{.*#+}} ymm2 = [2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648]
+; AVX1-NEXT:    vblendvps %ymm1, {{.*}}(%rip), %ymm2, %ymm2
+; AVX1-NEXT:    vblendvps %ymm0, %ymm2, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: v8i32:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm2, %ymm3
-; AVX2-NEXT:    vpcmpeqd %ymm4, %ymm4, %ymm4
-; AVX2-NEXT:    vpxor %ymm4, %ymm3, %ymm3
-; AVX2-NEXT:    vpcmpgtd %ymm0, %ymm2, %ymm5
-; AVX2-NEXT:    vpxor %ymm4, %ymm5, %ymm5
-; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm5, %ymm3
-; AVX2-NEXT:    vpsubd %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpcmpgtd %ymm0, %ymm2, %ymm1
-; AVX2-NEXT:    vpxor %ymm4, %ymm1, %ymm2
-; AVX2-NEXT:    vpcmpeqd %ymm2, %ymm5, %ymm2
-; AVX2-NEXT:    vpxor %ymm4, %ymm2, %ymm2
-; AVX2-NEXT:    vpandn %ymm2, %ymm3, %ymm2
-; AVX2-NEXT:    vbroadcastss {{.*#+}} ymm3 = [2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647]
-; AVX2-NEXT:    vbroadcastss {{.*#+}} ymm4 = [2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648]
-; AVX2-NEXT:    vblendvps %ymm1, %ymm3, %ymm4, %ymm1
-; AVX2-NEXT:    vblendvps %ymm2, %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpcmpgtd %ymm2, %ymm1, %ymm2
+; AVX2-NEXT:    vpsubd %ymm1, %ymm0, %ymm1
+; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %ymm0, %ymm2, %ymm0
+; AVX2-NEXT:    vbroadcastss {{.*#+}} ymm2 = [2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647]
+; AVX2-NEXT:    vbroadcastss {{.*#+}} ymm3 = [2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648]
+; AVX2-NEXT:    vblendvps %ymm1, %ymm2, %ymm3, %ymm2
+; AVX2-NEXT:    vblendvps %ymm0, %ymm2, %ymm1, %ymm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: v8i32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpcmpnltd %ymm2, %ymm1, %k0
-; AVX512-NEXT:    vpcmpnltd %ymm2, %ymm0, %k1
-; AVX512-NEXT:    kxorw %k0, %k1, %k0
-; AVX512-NEXT:    vpsubd %ymm1, %ymm0, %ymm0
-; AVX512-NEXT:    vpcmpnltd %ymm2, %ymm0, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
-; AVX512-NEXT:    kandw %k1, %k0, %k1
-; AVX512-NEXT:    vpcmpgtd %ymm0, %ymm2, %k2
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648]
-; AVX512-NEXT:    vpbroadcastd {{.*}}(%rip), %ymm1 {%k2}
-; AVX512-NEXT:    vmovdqa32 %ymm1, %ymm0 {%k1}
+; AVX512-NEXT:    vpcmpgtd %ymm2, %ymm1, %k0
+; AVX512-NEXT:    vpsubd %ymm1, %ymm0, %ymm1
+; AVX512-NEXT:    vpcmpgtd %ymm1, %ymm0, %k1
+; AVX512-NEXT:    kxorw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpgtd %ymm1, %ymm2, %k2
+; AVX512-NEXT:    vpbroadcastd {{.*#+}} ymm0 = [2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648]
+; AVX512-NEXT:    vpbroadcastd {{.*}}(%rip), %ymm0 {%k2}
+; AVX512-NEXT:    vmovdqa32 %ymm0, %ymm1 {%k1}
+; AVX512-NEXT:    vmovdqa %ymm1, %ymm0
 ; AVX512-NEXT:    retq
   %z = call <8 x i32> @llvm.ssub.sat.v8i32(<8 x i32> %x, <8 x i32> %y)
   ret <8 x i32> %z
@@ -1279,399 +903,244 @@ define <8 x i32> @v8i32(<8 x i32> %x, <8 x i32> %y) nounwind {
 define <16 x i32> @v16i32(<16 x i32> %x, <16 x i32> %y) nounwind {
 ; SSE2-LABEL: v16i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa %xmm1, %xmm8
-; SSE2-NEXT:    movdqa %xmm0, %xmm12
-; SSE2-NEXT:    pxor %xmm9, %xmm9
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpgtd %xmm4, %xmm0
-; SSE2-NEXT:    pcmpeqd %xmm10, %xmm10
-; SSE2-NEXT:    pxor %xmm10, %xmm0
-; SSE2-NEXT:    pxor %xmm11, %xmm11
-; SSE2-NEXT:    pcmpgtd %xmm12, %xmm11
-; SSE2-NEXT:    pxor %xmm10, %xmm11
-; SSE2-NEXT:    pcmpeqd %xmm11, %xmm0
-; SSE2-NEXT:    psubd %xmm4, %xmm12
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm12, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm4
-; SSE2-NEXT:    pxor %xmm10, %xmm4
-; SSE2-NEXT:    pcmpeqd %xmm11, %xmm4
-; SSE2-NEXT:    pxor %xmm10, %xmm4
-; SSE2-NEXT:    pandn %xmm4, %xmm0
-; SSE2-NEXT:    movdqa {{.*#+}} xmm11 = [2147483648,2147483648,2147483648,2147483648]
-; SSE2-NEXT:    movdqa %xmm1, %xmm4
-; SSE2-NEXT:    pandn %xmm11, %xmm4
-; SSE2-NEXT:    psrld $1, %xmm1
-; SSE2-NEXT:    por %xmm4, %xmm1
-; SSE2-NEXT:    pand %xmm0, %xmm1
-; SSE2-NEXT:    pandn %xmm12, %xmm0
-; SSE2-NEXT:    por %xmm1, %xmm0
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm5, %xmm1
-; SSE2-NEXT:    pxor %xmm10, %xmm1
-; SSE2-NEXT:    pxor %xmm12, %xmm12
-; SSE2-NEXT:    pcmpgtd %xmm8, %xmm12
-; SSE2-NEXT:    pxor %xmm10, %xmm12
-; SSE2-NEXT:    pcmpeqd %xmm12, %xmm1
-; SSE2-NEXT:    psubd %xmm5, %xmm8
-; SSE2-NEXT:    pxor %xmm4, %xmm4
+; SSE2-NEXT:    pxor %xmm8, %xmm8
+; SSE2-NEXT:    movdqa %xmm0, %xmm9
+; SSE2-NEXT:    psubd %xmm4, %xmm9
 ; SSE2-NEXT:    pcmpgtd %xmm8, %xmm4
-; SSE2-NEXT:    movdqa %xmm4, %xmm5
-; SSE2-NEXT:    pxor %xmm10, %xmm5
-; SSE2-NEXT:    pcmpeqd %xmm12, %xmm5
-; SSE2-NEXT:    pxor %xmm10, %xmm5
-; SSE2-NEXT:    pandn %xmm5, %xmm1
-; SSE2-NEXT:    movdqa %xmm4, %xmm5
-; SSE2-NEXT:    pandn %xmm11, %xmm5
-; SSE2-NEXT:    psrld $1, %xmm4
-; SSE2-NEXT:    por %xmm5, %xmm4
-; SSE2-NEXT:    pand %xmm1, %xmm4
-; SSE2-NEXT:    pandn %xmm8, %xmm1
-; SSE2-NEXT:    por %xmm4, %xmm1
-; SSE2-NEXT:    pxor %xmm8, %xmm8
-; SSE2-NEXT:    pcmpgtd %xmm6, %xmm8
-; SSE2-NEXT:    pxor %xmm10, %xmm8
-; SSE2-NEXT:    pxor %xmm5, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm5
-; SSE2-NEXT:    pxor %xmm10, %xmm5
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm8
-; SSE2-NEXT:    psubd %xmm6, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm9, %xmm0
+; SSE2-NEXT:    pxor %xmm4, %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [2147483648,2147483648,2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm9, %xmm11
+; SSE2-NEXT:    pandn %xmm10, %xmm11
 ; SSE2-NEXT:    pxor %xmm4, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm4
-; SSE2-NEXT:    movdqa %xmm4, %xmm6
-; SSE2-NEXT:    pxor %xmm10, %xmm6
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm6
-; SSE2-NEXT:    pxor %xmm10, %xmm6
-; SSE2-NEXT:    pandn %xmm6, %xmm8
-; SSE2-NEXT:    movdqa %xmm4, %xmm5
-; SSE2-NEXT:    pandn %xmm11, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm9, %xmm4
 ; SSE2-NEXT:    psrld $1, %xmm4
-; SSE2-NEXT:    por %xmm5, %xmm4
-; SSE2-NEXT:    pand %xmm8, %xmm4
-; SSE2-NEXT:    pandn %xmm2, %xmm8
-; SSE2-NEXT:    por %xmm4, %xmm8
-; SSE2-NEXT:    pxor %xmm5, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm7, %xmm5
-; SSE2-NEXT:    pxor %xmm10, %xmm5
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm2
-; SSE2-NEXT:    pxor %xmm10, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm5
-; SSE2-NEXT:    psubd %xmm7, %xmm3
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm9
-; SSE2-NEXT:    movdqa %xmm9, %xmm4
-; SSE2-NEXT:    pxor %xmm10, %xmm4
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm4
-; SSE2-NEXT:    pxor %xmm10, %xmm4
-; SSE2-NEXT:    pandn %xmm4, %xmm5
-; SSE2-NEXT:    movdqa %xmm9, %xmm2
-; SSE2-NEXT:    pandn %xmm11, %xmm2
-; SSE2-NEXT:    psrld $1, %xmm9
-; SSE2-NEXT:    por %xmm2, %xmm9
-; SSE2-NEXT:    pand %xmm5, %xmm9
-; SSE2-NEXT:    pandn %xmm3, %xmm5
-; SSE2-NEXT:    por %xmm9, %xmm5
-; SSE2-NEXT:    movdqa %xmm8, %xmm2
-; SSE2-NEXT:    movdqa %xmm5, %xmm3
+; SSE2-NEXT:    por %xmm4, %xmm11
+; SSE2-NEXT:    pand %xmm0, %xmm11
+; SSE2-NEXT:    pandn %xmm9, %xmm0
+; SSE2-NEXT:    por %xmm11, %xmm0
+; SSE2-NEXT:    movdqa %xmm1, %xmm9
+; SSE2-NEXT:    psubd %xmm5, %xmm9
+; SSE2-NEXT:    pcmpgtd %xmm8, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm9, %xmm1
+; SSE2-NEXT:    pxor %xmm5, %xmm1
+; SSE2-NEXT:    movdqa %xmm9, %xmm5
+; SSE2-NEXT:    pandn %xmm10, %xmm5
+; SSE2-NEXT:    pxor %xmm4, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm9, %xmm4
+; SSE2-NEXT:    psrld $1, %xmm4
+; SSE2-NEXT:    por %xmm4, %xmm5
+; SSE2-NEXT:    pand %xmm1, %xmm5
+; SSE2-NEXT:    pandn %xmm9, %xmm1
+; SSE2-NEXT:    por %xmm5, %xmm1
+; SSE2-NEXT:    movdqa %xmm2, %xmm4
+; SSE2-NEXT:    psubd %xmm6, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm8, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm2
+; SSE2-NEXT:    pxor %xmm6, %xmm2
+; SSE2-NEXT:    movdqa %xmm4, %xmm5
+; SSE2-NEXT:    pandn %xmm10, %xmm5
+; SSE2-NEXT:    pxor %xmm6, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm6
+; SSE2-NEXT:    psrld $1, %xmm6
+; SSE2-NEXT:    por %xmm6, %xmm5
+; SSE2-NEXT:    pand %xmm2, %xmm5
+; SSE2-NEXT:    pandn %xmm4, %xmm2
+; SSE2-NEXT:    por %xmm5, %xmm2
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    psubd %xmm7, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm8, %xmm7
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm3
+; SSE2-NEXT:    pxor %xmm7, %xmm3
+; SSE2-NEXT:    movdqa %xmm4, %xmm5
+; SSE2-NEXT:    pandn %xmm10, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm8
+; SSE2-NEXT:    psrld $1, %xmm8
+; SSE2-NEXT:    por %xmm8, %xmm5
+; SSE2-NEXT:    pand %xmm3, %xmm5
+; SSE2-NEXT:    pandn %xmm4, %xmm3
+; SSE2-NEXT:    por %xmm5, %xmm3
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: v16i32:
 ; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    movdqa %xmm1, %xmm8
-; SSSE3-NEXT:    movdqa %xmm0, %xmm12
-; SSSE3-NEXT:    pxor %xmm9, %xmm9
-; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm0
-; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm10
-; SSSE3-NEXT:    pxor %xmm10, %xmm0
-; SSSE3-NEXT:    pxor %xmm11, %xmm11
-; SSSE3-NEXT:    pcmpgtd %xmm12, %xmm11
-; SSSE3-NEXT:    pxor %xmm10, %xmm11
-; SSSE3-NEXT:    pcmpeqd %xmm11, %xmm0
-; SSSE3-NEXT:    psubd %xmm4, %xmm12
-; SSSE3-NEXT:    pxor %xmm1, %xmm1
-; SSSE3-NEXT:    pcmpgtd %xmm12, %xmm1
-; SSSE3-NEXT:    movdqa %xmm1, %xmm4
-; SSSE3-NEXT:    pxor %xmm10, %xmm4
-; SSSE3-NEXT:    pcmpeqd %xmm11, %xmm4
-; SSSE3-NEXT:    pxor %xmm10, %xmm4
-; SSSE3-NEXT:    pandn %xmm4, %xmm0
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm11 = [2147483648,2147483648,2147483648,2147483648]
-; SSSE3-NEXT:    movdqa %xmm1, %xmm4
-; SSSE3-NEXT:    pandn %xmm11, %xmm4
-; SSSE3-NEXT:    psrld $1, %xmm1
-; SSSE3-NEXT:    por %xmm4, %xmm1
-; SSSE3-NEXT:    pand %xmm0, %xmm1
-; SSSE3-NEXT:    pandn %xmm12, %xmm0
-; SSSE3-NEXT:    por %xmm1, %xmm0
-; SSSE3-NEXT:    pxor %xmm1, %xmm1
-; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm1
-; SSSE3-NEXT:    pxor %xmm10, %xmm1
-; SSSE3-NEXT:    pxor %xmm12, %xmm12
-; SSSE3-NEXT:    pcmpgtd %xmm8, %xmm12
-; SSSE3-NEXT:    pxor %xmm10, %xmm12
-; SSSE3-NEXT:    pcmpeqd %xmm12, %xmm1
-; SSSE3-NEXT:    psubd %xmm5, %xmm8
-; SSSE3-NEXT:    pxor %xmm4, %xmm4
+; SSSE3-NEXT:    pxor %xmm8, %xmm8
+; SSSE3-NEXT:    movdqa %xmm0, %xmm9
+; SSSE3-NEXT:    psubd %xmm4, %xmm9
 ; SSSE3-NEXT:    pcmpgtd %xmm8, %xmm4
-; SSSE3-NEXT:    movdqa %xmm4, %xmm5
-; SSSE3-NEXT:    pxor %xmm10, %xmm5
-; SSSE3-NEXT:    pcmpeqd %xmm12, %xmm5
-; SSSE3-NEXT:    pxor %xmm10, %xmm5
-; SSSE3-NEXT:    pandn %xmm5, %xmm1
-; SSSE3-NEXT:    movdqa %xmm4, %xmm5
-; SSSE3-NEXT:    pandn %xmm11, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm9, %xmm0
+; SSSE3-NEXT:    pxor %xmm4, %xmm0
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm10 = [2147483648,2147483648,2147483648,2147483648]
+; SSSE3-NEXT:    movdqa %xmm9, %xmm11
+; SSSE3-NEXT:    pandn %xmm10, %xmm11
+; SSSE3-NEXT:    pxor %xmm4, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm9, %xmm4
 ; SSSE3-NEXT:    psrld $1, %xmm4
-; SSSE3-NEXT:    por %xmm5, %xmm4
-; SSSE3-NEXT:    pand %xmm1, %xmm4
-; SSSE3-NEXT:    pandn %xmm8, %xmm1
-; SSSE3-NEXT:    por %xmm4, %xmm1
-; SSSE3-NEXT:    pxor %xmm8, %xmm8
-; SSSE3-NEXT:    pcmpgtd %xmm6, %xmm8
-; SSSE3-NEXT:    pxor %xmm10, %xmm8
-; SSSE3-NEXT:    pxor %xmm5, %xmm5
-; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm5
-; SSSE3-NEXT:    pxor %xmm10, %xmm5
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm8
-; SSSE3-NEXT:    psubd %xmm6, %xmm2
+; SSSE3-NEXT:    por %xmm4, %xmm11
+; SSSE3-NEXT:    pand %xmm0, %xmm11
+; SSSE3-NEXT:    pandn %xmm9, %xmm0
+; SSSE3-NEXT:    por %xmm11, %xmm0
+; SSSE3-NEXT:    movdqa %xmm1, %xmm9
+; SSSE3-NEXT:    psubd %xmm5, %xmm9
+; SSSE3-NEXT:    pcmpgtd %xmm8, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm9, %xmm1
+; SSSE3-NEXT:    pxor %xmm5, %xmm1
+; SSSE3-NEXT:    movdqa %xmm9, %xmm5
+; SSSE3-NEXT:    pandn %xmm10, %xmm5
 ; SSSE3-NEXT:    pxor %xmm4, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm4
-; SSSE3-NEXT:    movdqa %xmm4, %xmm6
-; SSSE3-NEXT:    pxor %xmm10, %xmm6
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm6
-; SSSE3-NEXT:    pxor %xmm10, %xmm6
-; SSSE3-NEXT:    pandn %xmm6, %xmm8
-; SSSE3-NEXT:    movdqa %xmm4, %xmm5
-; SSSE3-NEXT:    pandn %xmm11, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm9, %xmm4
 ; SSSE3-NEXT:    psrld $1, %xmm4
-; SSSE3-NEXT:    por %xmm5, %xmm4
-; SSSE3-NEXT:    pand %xmm8, %xmm4
-; SSSE3-NEXT:    pandn %xmm2, %xmm8
-; SSSE3-NEXT:    por %xmm4, %xmm8
-; SSSE3-NEXT:    pxor %xmm5, %xmm5
-; SSSE3-NEXT:    pcmpgtd %xmm7, %xmm5
-; SSSE3-NEXT:    pxor %xmm10, %xmm5
-; SSSE3-NEXT:    pxor %xmm2, %xmm2
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm2
-; SSSE3-NEXT:    pxor %xmm10, %xmm2
-; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm5
-; SSSE3-NEXT:    psubd %xmm7, %xmm3
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm9
-; SSSE3-NEXT:    movdqa %xmm9, %xmm4
-; SSSE3-NEXT:    pxor %xmm10, %xmm4
-; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm4
-; SSSE3-NEXT:    pxor %xmm10, %xmm4
-; SSSE3-NEXT:    pandn %xmm4, %xmm5
-; SSSE3-NEXT:    movdqa %xmm9, %xmm2
-; SSSE3-NEXT:    pandn %xmm11, %xmm2
-; SSSE3-NEXT:    psrld $1, %xmm9
-; SSSE3-NEXT:    por %xmm2, %xmm9
-; SSSE3-NEXT:    pand %xmm5, %xmm9
-; SSSE3-NEXT:    pandn %xmm3, %xmm5
-; SSSE3-NEXT:    por %xmm9, %xmm5
-; SSSE3-NEXT:    movdqa %xmm8, %xmm2
-; SSSE3-NEXT:    movdqa %xmm5, %xmm3
+; SSSE3-NEXT:    por %xmm4, %xmm5
+; SSSE3-NEXT:    pand %xmm1, %xmm5
+; SSSE3-NEXT:    pandn %xmm9, %xmm1
+; SSSE3-NEXT:    por %xmm5, %xmm1
+; SSSE3-NEXT:    movdqa %xmm2, %xmm4
+; SSSE3-NEXT:    psubd %xmm6, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm8, %xmm6
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm2
+; SSSE3-NEXT:    pxor %xmm6, %xmm2
+; SSSE3-NEXT:    movdqa %xmm4, %xmm5
+; SSSE3-NEXT:    pandn %xmm10, %xmm5
+; SSSE3-NEXT:    pxor %xmm6, %xmm6
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm6
+; SSSE3-NEXT:    psrld $1, %xmm6
+; SSSE3-NEXT:    por %xmm6, %xmm5
+; SSSE3-NEXT:    pand %xmm2, %xmm5
+; SSSE3-NEXT:    pandn %xmm4, %xmm2
+; SSSE3-NEXT:    por %xmm5, %xmm2
+; SSSE3-NEXT:    movdqa %xmm3, %xmm4
+; SSSE3-NEXT:    psubd %xmm7, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm8, %xmm7
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm3
+; SSSE3-NEXT:    pxor %xmm7, %xmm3
+; SSSE3-NEXT:    movdqa %xmm4, %xmm5
+; SSSE3-NEXT:    pandn %xmm10, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm8
+; SSSE3-NEXT:    psrld $1, %xmm8
+; SSSE3-NEXT:    por %xmm8, %xmm5
+; SSSE3-NEXT:    pand %xmm3, %xmm5
+; SSSE3-NEXT:    pandn %xmm4, %xmm3
+; SSSE3-NEXT:    por %xmm5, %xmm3
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: v16i32:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa %xmm0, %xmm9
-; SSE41-NEXT:    pxor %xmm8, %xmm8
+; SSE41-NEXT:    movdqa %xmm3, %xmm8
+; SSE41-NEXT:    movdqa %xmm0, %xmm3
 ; SSE41-NEXT:    pxor %xmm10, %xmm10
-; SSE41-NEXT:    pcmpgtd %xmm4, %xmm10
-; SSE41-NEXT:    pcmpeqd %xmm11, %xmm11
-; SSE41-NEXT:    pxor %xmm11, %xmm10
-; SSE41-NEXT:    pxor %xmm12, %xmm12
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm12
-; SSE41-NEXT:    pxor %xmm11, %xmm12
-; SSE41-NEXT:    pcmpeqd %xmm12, %xmm10
+; SSE41-NEXT:    movdqa %xmm0, %xmm9
 ; SSE41-NEXT:    psubd %xmm4, %xmm9
-; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    pcmpgtd %xmm9, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm4
-; SSE41-NEXT:    pxor %xmm11, %xmm4
-; SSE41-NEXT:    pcmpeqd %xmm12, %xmm4
-; SSE41-NEXT:    pxor %xmm11, %xmm4
-; SSE41-NEXT:    pandn %xmm4, %xmm10
-; SSE41-NEXT:    movaps {{.*#+}} xmm13 = [2147483647,2147483647,2147483647,2147483647]
-; SSE41-NEXT:    movaps {{.*#+}} xmm12 = [2147483648,2147483648,2147483648,2147483648]
-; SSE41-NEXT:    movaps %xmm12, %xmm4
-; SSE41-NEXT:    blendvps %xmm0, %xmm13, %xmm4
-; SSE41-NEXT:    movdqa %xmm10, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm10, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm9, %xmm3
+; SSE41-NEXT:    pxor %xmm4, %xmm3
+; SSE41-NEXT:    movaps {{.*#+}} xmm12 = [2147483647,2147483647,2147483647,2147483647]
+; SSE41-NEXT:    movaps {{.*#+}} xmm11 = [2147483648,2147483648,2147483648,2147483648]
+; SSE41-NEXT:    movaps %xmm11, %xmm4
+; SSE41-NEXT:    movdqa %xmm9, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm12, %xmm4
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
 ; SSE41-NEXT:    blendvps %xmm0, %xmm4, %xmm9
-; SSE41-NEXT:    xorps %xmm4, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm5, %xmm4
-; SSE41-NEXT:    pxor %xmm11, %xmm4
-; SSE41-NEXT:    pxor %xmm10, %xmm10
-; SSE41-NEXT:    pcmpgtd %xmm1, %xmm10
-; SSE41-NEXT:    pxor %xmm11, %xmm10
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm4
-; SSE41-NEXT:    psubd %xmm5, %xmm1
-; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm5
-; SSE41-NEXT:    pxor %xmm11, %xmm5
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm5
-; SSE41-NEXT:    pxor %xmm11, %xmm5
-; SSE41-NEXT:    pandn %xmm5, %xmm4
-; SSE41-NEXT:    movaps %xmm12, %xmm5
-; SSE41-NEXT:    blendvps %xmm0, %xmm13, %xmm5
-; SSE41-NEXT:    movdqa %xmm4, %xmm0
-; SSE41-NEXT:    blendvps %xmm0, %xmm5, %xmm1
-; SSE41-NEXT:    pxor %xmm4, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm6, %xmm4
-; SSE41-NEXT:    pxor %xmm11, %xmm4
-; SSE41-NEXT:    xorps %xmm5, %xmm5
-; SSE41-NEXT:    pcmpgtd %xmm2, %xmm5
-; SSE41-NEXT:    pxor %xmm11, %xmm5
-; SSE41-NEXT:    pcmpeqd %xmm5, %xmm4
-; SSE41-NEXT:    psubd %xmm6, %xmm2
-; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    pcmpgtd %xmm2, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm6
-; SSE41-NEXT:    pxor %xmm11, %xmm6
-; SSE41-NEXT:    pcmpeqd %xmm5, %xmm6
-; SSE41-NEXT:    pxor %xmm11, %xmm6
-; SSE41-NEXT:    pandn %xmm6, %xmm4
-; SSE41-NEXT:    movaps %xmm12, %xmm5
-; SSE41-NEXT:    blendvps %xmm0, %xmm13, %xmm5
+; SSE41-NEXT:    movdqa %xmm1, %xmm4
+; SSE41-NEXT:    psubd %xmm5, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm10, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm4, %xmm1
+; SSE41-NEXT:    pxor %xmm5, %xmm1
+; SSE41-NEXT:    movaps %xmm11, %xmm3
 ; SSE41-NEXT:    movdqa %xmm4, %xmm0
-; SSE41-NEXT:    blendvps %xmm0, %xmm5, %xmm2
-; SSE41-NEXT:    pxor %xmm4, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm7, %xmm4
-; SSE41-NEXT:    pxor %xmm11, %xmm4
-; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE41-NEXT:    pxor %xmm11, %xmm0
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
-; SSE41-NEXT:    psubd %xmm7, %xmm3
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm8
+; SSE41-NEXT:    blendvps %xmm0, %xmm12, %xmm3
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm3, %xmm4
+; SSE41-NEXT:    movdqa %xmm2, %xmm3
+; SSE41-NEXT:    psubd %xmm6, %xmm3
+; SSE41-NEXT:    pcmpgtd %xmm10, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm2
+; SSE41-NEXT:    pxor %xmm6, %xmm2
+; SSE41-NEXT:    movaps %xmm11, %xmm1
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm12, %xmm1
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm1, %xmm3
 ; SSE41-NEXT:    movdqa %xmm8, %xmm5
-; SSE41-NEXT:    pxor %xmm11, %xmm5
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm5
-; SSE41-NEXT:    pxor %xmm11, %xmm5
-; SSE41-NEXT:    pandn %xmm5, %xmm4
+; SSE41-NEXT:    psubd %xmm7, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm10, %xmm7
+; SSE41-NEXT:    pcmpgtd %xmm5, %xmm8
+; SSE41-NEXT:    pxor %xmm7, %xmm8
+; SSE41-NEXT:    movdqa %xmm5, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm12, %xmm11
 ; SSE41-NEXT:    movdqa %xmm8, %xmm0
-; SSE41-NEXT:    blendvps %xmm0, %xmm13, %xmm12
-; SSE41-NEXT:    movdqa %xmm4, %xmm0
-; SSE41-NEXT:    blendvps %xmm0, %xmm12, %xmm3
+; SSE41-NEXT:    blendvps %xmm0, %xmm11, %xmm5
 ; SSE41-NEXT:    movaps %xmm9, %xmm0
+; SSE41-NEXT:    movaps %xmm4, %xmm1
+; SSE41-NEXT:    movaps %xmm3, %xmm2
+; SSE41-NEXT:    movaps %xmm5, %xmm3
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: v16i32:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm9
-; AVX1-NEXT:    vpxor %xmm10, %xmm10, %xmm10
-; AVX1-NEXT:    vpcmpgtd %xmm9, %xmm10, %xmm7
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm8
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm4
+; AVX1-NEXT:    vpxor %xmm5, %xmm5, %xmm5
+; AVX1-NEXT:    vpcmpgtd %xmm5, %xmm4, %xmm6
+; AVX1-NEXT:    vpcmpgtd %xmm5, %xmm2, %xmm7
+; AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm7, %ymm6
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm7
-; AVX1-NEXT:    vpcmpgtd %xmm7, %xmm10, %xmm6
-; AVX1-NEXT:    vpxor %xmm4, %xmm6, %xmm12
-; AVX1-NEXT:    vpcmpeqd %xmm8, %xmm12, %xmm8
-; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm10, %xmm5
-; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm11
-; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm10, %xmm5
-; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX1-NEXT:    vpcmpeqd %xmm11, %xmm5, %xmm11
-; AVX1-NEXT:    vinsertf128 $1, %xmm8, %ymm11, %ymm8
-; AVX1-NEXT:    vpsubd %xmm9, %xmm7, %xmm9
-; AVX1-NEXT:    vpcmpgtd %xmm9, %xmm10, %xmm7
-; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm6
-; AVX1-NEXT:    vpcmpeqd %xmm6, %xmm12, %xmm6
-; AVX1-NEXT:    vpxor %xmm4, %xmm6, %xmm11
-; AVX1-NEXT:    vpsubd %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm10, %xmm2
-; AVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm6
-; AVX1-NEXT:    vpcmpeqd %xmm6, %xmm5, %xmm5
-; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX1-NEXT:    vinsertf128 $1, %xmm11, %ymm5, %ymm5
-; AVX1-NEXT:    vandnps %ymm5, %ymm8, %ymm5
-; AVX1-NEXT:    vinsertf128 $1, %xmm7, %ymm2, %ymm7
-; AVX1-NEXT:    vmovaps {{.*#+}} ymm8 = [2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647]
-; AVX1-NEXT:    vmovaps {{.*#+}} ymm11 = [2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648]
-; AVX1-NEXT:    vblendvps %ymm7, %ymm8, %ymm11, %ymm7
-; AVX1-NEXT:    vinsertf128 $1, %xmm9, %ymm0, %ymm0
-; AVX1-NEXT:    vblendvps %ymm5, %ymm7, %ymm0, %ymm0
-; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm5
-; AVX1-NEXT:    vpcmpgtd %xmm5, %xmm10, %xmm7
-; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm7
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm10, %xmm6
-; AVX1-NEXT:    vpxor %xmm4, %xmm6, %xmm6
-; AVX1-NEXT:    vpcmpeqd %xmm7, %xmm6, %xmm9
-; AVX1-NEXT:    vpcmpgtd %xmm3, %xmm10, %xmm7
-; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm12
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm10, %xmm7
-; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm7
-; AVX1-NEXT:    vpcmpeqd %xmm12, %xmm7, %xmm12
-; AVX1-NEXT:    vinsertf128 $1, %xmm9, %ymm12, %ymm9
-; AVX1-NEXT:    vpsubd %xmm5, %xmm2, %xmm12
-; AVX1-NEXT:    vpcmpgtd %xmm12, %xmm10, %xmm5
-; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm6, %xmm2
-; AVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm2
-; AVX1-NEXT:    vpsubd %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm10, %xmm3
-; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm6
-; AVX1-NEXT:    vpcmpeqd %xmm6, %xmm7, %xmm6
-; AVX1-NEXT:    vpxor %xmm4, %xmm6, %xmm4
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm4, %ymm2
-; AVX1-NEXT:    vandnps %ymm2, %ymm9, %ymm2
-; AVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm3, %ymm3
-; AVX1-NEXT:    vblendvps %ymm3, %ymm8, %ymm11, %ymm3
-; AVX1-NEXT:    vinsertf128 $1, %xmm12, %ymm1, %ymm1
-; AVX1-NEXT:    vblendvps %ymm2, %ymm3, %ymm1, %ymm1
+; AVX1-NEXT:    vpsubd %xmm4, %xmm7, %xmm4
+; AVX1-NEXT:    vpcmpgtd %xmm4, %xmm7, %xmm7
+; AVX1-NEXT:    vpsubd %xmm2, %xmm0, %xmm2
+; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm7, %ymm0, %ymm0
+; AVX1-NEXT:    vxorps %ymm0, %ymm6, %ymm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm2, %ymm2
+; AVX1-NEXT:    vmovaps {{.*#+}} ymm4 = [2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647]
+; AVX1-NEXT:    vmovaps {{.*#+}} ymm6 = [2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648]
+; AVX1-NEXT:    vblendvps %ymm2, %ymm4, %ymm6, %ymm7
+; AVX1-NEXT:    vblendvps %ymm0, %ymm7, %ymm2, %ymm0
+; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm2
+; AVX1-NEXT:    vpcmpgtd %xmm5, %xmm2, %xmm7
+; AVX1-NEXT:    vpcmpgtd %xmm5, %xmm3, %xmm5
+; AVX1-NEXT:    vinsertf128 $1, %xmm7, %ymm5, %ymm5
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm7
+; AVX1-NEXT:    vpsubd %xmm2, %xmm7, %xmm2
+; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm7, %xmm7
+; AVX1-NEXT:    vpsubd %xmm3, %xmm1, %xmm3
+; AVX1-NEXT:    vpcmpgtd %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm7, %ymm1, %ymm1
+; AVX1-NEXT:    vxorps %ymm1, %ymm5, %ymm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
+; AVX1-NEXT:    vblendvps %ymm2, %ymm4, %ymm6, %ymm3
+; AVX1-NEXT:    vblendvps %ymm1, %ymm3, %ymm2, %ymm1
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: v16i32:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpcmpgtd %ymm2, %ymm4, %ymm5
-; AVX2-NEXT:    vpcmpeqd %ymm6, %ymm6, %ymm6
-; AVX2-NEXT:    vpxor %ymm6, %ymm5, %ymm5
-; AVX2-NEXT:    vpcmpgtd %ymm0, %ymm4, %ymm7
-; AVX2-NEXT:    vpxor %ymm6, %ymm7, %ymm7
-; AVX2-NEXT:    vpcmpeqd %ymm5, %ymm7, %ymm5
-; AVX2-NEXT:    vpsubd %ymm2, %ymm0, %ymm0
-; AVX2-NEXT:    vpcmpgtd %ymm0, %ymm4, %ymm2
-; AVX2-NEXT:    vpxor %ymm6, %ymm2, %ymm8
-; AVX2-NEXT:    vpcmpeqd %ymm8, %ymm7, %ymm7
-; AVX2-NEXT:    vpxor %ymm6, %ymm7, %ymm7
-; AVX2-NEXT:    vpandn %ymm7, %ymm5, %ymm5
-; AVX2-NEXT:    vbroadcastss {{.*#+}} ymm7 = [2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647]
-; AVX2-NEXT:    vbroadcastss {{.*#+}} ymm8 = [2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648]
-; AVX2-NEXT:    vblendvps %ymm2, %ymm7, %ymm8, %ymm2
-; AVX2-NEXT:    vblendvps %ymm5, %ymm2, %ymm0, %ymm0
-; AVX2-NEXT:    vpcmpgtd %ymm3, %ymm4, %ymm2
-; AVX2-NEXT:    vpxor %ymm6, %ymm2, %ymm2
-; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm4, %ymm5
-; AVX2-NEXT:    vpxor %ymm6, %ymm5, %ymm5
-; AVX2-NEXT:    vpcmpeqd %ymm2, %ymm5, %ymm2
-; AVX2-NEXT:    vpsubd %ymm3, %ymm1, %ymm1
-; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm4, %ymm3
-; AVX2-NEXT:    vpxor %ymm6, %ymm3, %ymm4
-; AVX2-NEXT:    vpcmpeqd %ymm4, %ymm5, %ymm4
-; AVX2-NEXT:    vpxor %ymm6, %ymm4, %ymm4
-; AVX2-NEXT:    vpandn %ymm4, %ymm2, %ymm2
-; AVX2-NEXT:    vblendvps %ymm3, %ymm7, %ymm8, %ymm3
-; AVX2-NEXT:    vblendvps %ymm2, %ymm3, %ymm1, %ymm1
+; AVX2-NEXT:    vpcmpgtd %ymm4, %ymm2, %ymm5
+; AVX2-NEXT:    vpsubd %ymm2, %ymm0, %ymm2
+; AVX2-NEXT:    vpcmpgtd %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %ymm0, %ymm5, %ymm0
+; AVX2-NEXT:    vbroadcastss {{.*#+}} ymm5 = [2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647]
+; AVX2-NEXT:    vbroadcastss {{.*#+}} ymm6 = [2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648]
+; AVX2-NEXT:    vblendvps %ymm2, %ymm5, %ymm6, %ymm7
+; AVX2-NEXT:    vblendvps %ymm0, %ymm7, %ymm2, %ymm0
+; AVX2-NEXT:    vpcmpgtd %ymm4, %ymm3, %ymm2
+; AVX2-NEXT:    vpsubd %ymm3, %ymm1, %ymm3
+; AVX2-NEXT:    vpcmpgtd %ymm3, %ymm1, %ymm1
+; AVX2-NEXT:    vpxor %ymm1, %ymm2, %ymm1
+; AVX2-NEXT:    vblendvps %ymm3, %ymm5, %ymm6, %ymm2
+; AVX2-NEXT:    vblendvps %ymm1, %ymm2, %ymm3, %ymm1
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: v16i32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpcmpnltd %zmm2, %zmm1, %k0
-; AVX512-NEXT:    vpcmpnltd %zmm2, %zmm0, %k1
-; AVX512-NEXT:    kxorw %k0, %k1, %k0
-; AVX512-NEXT:    vpsubd %zmm1, %zmm0, %zmm0
-; AVX512-NEXT:    vpcmpnltd %zmm2, %zmm0, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
-; AVX512-NEXT:    kandw %k1, %k0, %k1
-; AVX512-NEXT:    vpcmpgtd %zmm0, %zmm2, %k2
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} zmm1 = [2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648]
-; AVX512-NEXT:    vpbroadcastd {{.*}}(%rip), %zmm1 {%k2}
-; AVX512-NEXT:    vmovdqa32 %zmm1, %zmm0 {%k1}
+; AVX512-NEXT:    vpcmpgtd %zmm2, %zmm1, %k0
+; AVX512-NEXT:    vpsubd %zmm1, %zmm0, %zmm1
+; AVX512-NEXT:    vpcmpgtd %zmm1, %zmm0, %k1
+; AVX512-NEXT:    kxorw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpgtd %zmm1, %zmm2, %k2
+; AVX512-NEXT:    vpbroadcastd {{.*#+}} zmm0 = [2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648]
+; AVX512-NEXT:    vpbroadcastd {{.*}}(%rip), %zmm0 {%k2}
+; AVX512-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
+; AVX512-NEXT:    vmovdqa64 %zmm1, %zmm0
 ; AVX512-NEXT:    retq
   %z = call <16 x i32> @llvm.ssub.sat.v16i32(<16 x i32> %x, <16 x i32> %y)
   ret <16 x i32> %z
@@ -1682,55 +1151,38 @@ define <2 x i64> @v2i64(<2 x i64> %x, <2 x i64> %y) nounwind {
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm3
+; SSE2-NEXT:    pxor %xmm2, %xmm3
 ; SSE2-NEXT:    psubq %xmm1, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm4
+; SSE2-NEXT:    pxor %xmm2, %xmm4
+; SSE2-NEXT:    movdqa %xmm3, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm3, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
+; SSE2-NEXT:    pand %xmm6, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm5[1,1,3,3]
+; SSE2-NEXT:    por %xmm3, %xmm4
 ; SSE2-NEXT:    pxor %xmm2, %xmm1
-; SSE2-NEXT:    movdqa %xmm2, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE2-NEXT:    movdqa %xmm1, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[0,0,2,2]
 ; SSE2-NEXT:    pcmpeqd %xmm2, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm5, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm4
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm5
-; SSE2-NEXT:    pxor %xmm5, %xmm4
-; SSE2-NEXT:    pxor %xmm2, %xmm3
-; SSE2-NEXT:    movdqa %xmm2, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm6
-; SSE2-NEXT:    pxor %xmm5, %xmm6
-; SSE2-NEXT:    pcmpeqd %xmm6, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,0,3,2]
-; SSE2-NEXT:    pand %xmm4, %xmm1
-; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    pxor %xmm2, %xmm3
-; SSE2-NEXT:    movdqa %xmm2, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm3
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,1,3,3]
-; SSE2-NEXT:    pand %xmm7, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm2, %xmm3
-; SSE2-NEXT:    movdqa %xmm3, %xmm2
-; SSE2-NEXT:    pxor %xmm5, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm6, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,0,3,2]
-; SSE2-NEXT:    pand %xmm2, %xmm4
-; SSE2-NEXT:    pxor %xmm5, %xmm4
-; SSE2-NEXT:    pandn %xmm4, %xmm1
-; SSE2-NEXT:    movdqa %xmm3, %xmm2
-; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm2
+; SSE2-NEXT:    por %xmm1, %xmm2
+; SSE2-NEXT:    pxor %xmm4, %xmm2
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm1
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
 ; SSE2-NEXT:    pand {{.*}}(%rip), %xmm3
-; SSE2-NEXT:    por %xmm2, %xmm3
-; SSE2-NEXT:    pand %xmm1, %xmm3
-; SSE2-NEXT:    pandn %xmm0, %xmm1
 ; SSE2-NEXT:    por %xmm3, %xmm1
+; SSE2-NEXT:    pand %xmm2, %xmm1
+; SSE2-NEXT:    pandn %xmm0, %xmm2
+; SSE2-NEXT:    por %xmm2, %xmm1
 ; SSE2-NEXT:    movdqa %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
@@ -1738,55 +1190,38 @@ define <2 x i64> @v2i64(<2 x i64> %x, <2 x i64> %y) nounwind {
 ; SSSE3:       # %bb.0:
 ; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
 ; SSSE3-NEXT:    movdqa %xmm0, %xmm3
+; SSSE3-NEXT:    pxor %xmm2, %xmm3
 ; SSSE3-NEXT:    psubq %xmm1, %xmm0
+; SSSE3-NEXT:    movdqa %xmm0, %xmm4
+; SSSE3-NEXT:    pxor %xmm2, %xmm4
+; SSSE3-NEXT:    movdqa %xmm3, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm6, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm5[1,1,3,3]
+; SSSE3-NEXT:    por %xmm3, %xmm4
 ; SSSE3-NEXT:    pxor %xmm2, %xmm1
-; SSSE3-NEXT:    movdqa %xmm2, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    movdqa %xmm1, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[0,0,2,2]
 ; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm1
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
 ; SSSE3-NEXT:    pand %xmm5, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    por %xmm1, %xmm4
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm5
-; SSSE3-NEXT:    pxor %xmm5, %xmm4
-; SSSE3-NEXT:    pxor %xmm2, %xmm3
-; SSSE3-NEXT:    movdqa %xmm2, %xmm1
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm6, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    por %xmm3, %xmm6
-; SSSE3-NEXT:    pxor %xmm5, %xmm6
-; SSSE3-NEXT:    pcmpeqd %xmm6, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm4, %xmm1
-; SSSE3-NEXT:    movdqa %xmm0, %xmm3
-; SSSE3-NEXT:    pxor %xmm2, %xmm3
-; SSSE3-NEXT:    movdqa %xmm2, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm3
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm7, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm3
-; SSSE3-NEXT:    movdqa %xmm3, %xmm2
-; SSSE3-NEXT:    pxor %xmm5, %xmm2
-; SSSE3-NEXT:    pcmpeqd %xmm6, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm2, %xmm4
-; SSSE3-NEXT:    pxor %xmm5, %xmm4
-; SSSE3-NEXT:    pandn %xmm4, %xmm1
-; SSSE3-NEXT:    movdqa %xmm3, %xmm2
-; SSSE3-NEXT:    pandn {{.*}}(%rip), %xmm2
+; SSSE3-NEXT:    por %xmm1, %xmm2
+; SSSE3-NEXT:    pxor %xmm4, %xmm2
+; SSSE3-NEXT:    movdqa %xmm0, %xmm1
+; SSSE3-NEXT:    pandn {{.*}}(%rip), %xmm1
+; SSSE3-NEXT:    pxor %xmm3, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
 ; SSSE3-NEXT:    pand {{.*}}(%rip), %xmm3
-; SSSE3-NEXT:    por %xmm2, %xmm3
-; SSSE3-NEXT:    pand %xmm1, %xmm3
-; SSSE3-NEXT:    pandn %xmm0, %xmm1
 ; SSSE3-NEXT:    por %xmm3, %xmm1
+; SSSE3-NEXT:    pand %xmm2, %xmm1
+; SSSE3-NEXT:    pandn %xmm0, %xmm2
+; SSSE3-NEXT:    por %xmm2, %xmm1
 ; SSSE3-NEXT:    movdqa %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
@@ -1795,105 +1230,68 @@ define <2 x i64> @v2i64(<2 x i64> %x, <2 x i64> %y) nounwind {
 ; SSE41-NEXT:    movdqa %xmm0, %xmm2
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483648,2147483648]
 ; SSE41-NEXT:    movdqa %xmm2, %xmm3
+; SSE41-NEXT:    pxor %xmm0, %xmm3
 ; SSE41-NEXT:    psubq %xmm1, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm4
+; SSE41-NEXT:    pxor %xmm0, %xmm4
+; SSE41-NEXT:    movdqa %xmm3, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm4, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm4, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm4
+; SSE41-NEXT:    por %xmm3, %xmm4
 ; SSE41-NEXT:    pxor %xmm0, %xmm1
-; SSE41-NEXT:    movdqa %xmm0, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm1, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,1,3,3]
-; SSE41-NEXT:    por %xmm6, %xmm1
-; SSE41-NEXT:    pcmpeqd %xmm4, %xmm4
-; SSE41-NEXT:    pxor %xmm4, %xmm1
-; SSE41-NEXT:    pxor %xmm0, %xmm3
-; SSE41-NEXT:    movdqa %xmm0, %xmm5
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pand %xmm6, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE41-NEXT:    por %xmm3, %xmm5
-; SSE41-NEXT:    pxor %xmm4, %xmm5
-; SSE41-NEXT:    pcmpeqq %xmm5, %xmm1
-; SSE41-NEXT:    movdqa %xmm2, %xmm3
-; SSE41-NEXT:    pxor %xmm0, %xmm3
-; SSE41-NEXT:    movdqa %xmm0, %xmm6
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pand %xmm7, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3]
-; SSE41-NEXT:    por %xmm3, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm3
-; SSE41-NEXT:    pxor %xmm4, %xmm3
-; SSE41-NEXT:    pcmpeqq %xmm5, %xmm3
+; SSE41-NEXT:    movdqa %xmm1, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm3
+; SSE41-NEXT:    por %xmm1, %xmm3
 ; SSE41-NEXT:    pxor %xmm4, %xmm3
-; SSE41-NEXT:    pandn %xmm3, %xmm1
-; SSE41-NEXT:    movapd {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808]
-; SSE41-NEXT:    blendvpd %xmm0, {{.*}}(%rip), %xmm3
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm2
+; SSE41-NEXT:    movapd {{.*#+}} xmm1 = [9223372036854775808,9223372036854775808]
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, {{.*}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
 ; SSE41-NEXT:    movapd %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: v2i64:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm2, %xmm3
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm5
-; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX1-NEXT:    vpcmpeqq %xmm3, %xmm5, %xmm3
-; AVX1-NEXT:    vpsubq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm4, %xmm1, %xmm2
-; AVX1-NEXT:    vpcmpeqq %xmm2, %xmm5, %xmm2
-; AVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm2
-; AVX1-NEXT:    vpandn %xmm2, %xmm3, %xmm2
-; AVX1-NEXT:    vmovapd {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808]
-; AVX1-NEXT:    vblendvpd %xmm1, {{.*}}(%rip), %xmm3, %xmm1
-; AVX1-NEXT:    vblendvpd %xmm2, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm1, %xmm2
+; AVX1-NEXT:    vpsubq %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor %xmm0, %xmm2, %xmm0
+; AVX1-NEXT:    vmovapd {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
+; AVX1-NEXT:    vblendvpd %xmm1, {{.*}}(%rip), %xmm2, %xmm2
+; AVX1-NEXT:    vblendvpd %xmm0, %xmm2, %xmm1, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: v2i64:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtq %xmm1, %xmm2, %xmm3
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpxor %xmm4, %xmm3, %xmm3
-; AVX2-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm5
-; AVX2-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX2-NEXT:    vpcmpeqq %xmm3, %xmm5, %xmm3
-; AVX2-NEXT:    vpsubq %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm1
-; AVX2-NEXT:    vpxor %xmm4, %xmm1, %xmm2
-; AVX2-NEXT:    vpcmpeqq %xmm2, %xmm5, %xmm2
-; AVX2-NEXT:    vpxor %xmm4, %xmm2, %xmm2
-; AVX2-NEXT:    vpandn %xmm2, %xmm3, %xmm2
-; AVX2-NEXT:    vmovapd {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808]
-; AVX2-NEXT:    vblendvpd %xmm1, {{.*}}(%rip), %xmm3, %xmm1
-; AVX2-NEXT:    vblendvpd %xmm2, %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpcmpgtq %xmm2, %xmm1, %xmm2
+; AVX2-NEXT:    vpsubq %xmm1, %xmm0, %xmm1
+; AVX2-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor %xmm0, %xmm2, %xmm0
+; AVX2-NEXT:    vmovapd {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
+; AVX2-NEXT:    vblendvpd %xmm1, {{.*}}(%rip), %xmm2, %xmm2
+; AVX2-NEXT:    vblendvpd %xmm0, %xmm2, %xmm1, %xmm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: v2i64:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpcmpnltq %xmm2, %xmm1, %k0
-; AVX512-NEXT:    vpcmpnltq %xmm2, %xmm0, %k1
-; AVX512-NEXT:    kxorw %k0, %k1, %k0
-; AVX512-NEXT:    vpsubq %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpcmpnltq %xmm2, %xmm0, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
-; AVX512-NEXT:    kandw %k1, %k0, %k1
-; AVX512-NEXT:    vpcmpgtq %xmm0, %xmm2, %k2
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = [9223372036854775808,9223372036854775808]
-; AVX512-NEXT:    vmovdqa64 {{.*}}(%rip), %xmm1 {%k2}
-; AVX512-NEXT:    vmovdqa64 %xmm1, %xmm0 {%k1}
+; AVX512-NEXT:    vpcmpgtq %xmm2, %xmm1, %k0
+; AVX512-NEXT:    vpsubq %xmm1, %xmm0, %xmm1
+; AVX512-NEXT:    vpcmpgtq %xmm1, %xmm0, %k1
+; AVX512-NEXT:    kxorw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpgtq %xmm1, %xmm2, %k2
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm0 = [9223372036854775808,9223372036854775808]
+; AVX512-NEXT:    vmovdqa64 {{.*}}(%rip), %xmm0 {%k2}
+; AVX512-NEXT:    vmovdqa64 %xmm0, %xmm1 {%k1}
+; AVX512-NEXT:    vmovdqa %xmm1, %xmm0
 ; AVX512-NEXT:    retq
   %z = call <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64> %x, <2 x i64> %y)
   ret <2 x i64> %z
@@ -1902,381 +1300,250 @@ define <2 x i64> @v2i64(<2 x i64> %x, <2 x i64> %y) nounwind {
 define <4 x i64> @v4i64(<4 x i64> %x, <4 x i64> %y) nounwind {
 ; SSE2-LABEL: v4i64:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa %xmm0, %xmm10
-; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [2147483648,2147483648]
-; SSE2-NEXT:    psubq %xmm2, %xmm10
-; SSE2-NEXT:    pxor %xmm5, %xmm2
-; SSE2-NEXT:    movdqa %xmm5, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT:    pand %xmm7, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[1,1,3,3]
-; SSE2-NEXT:    por %xmm2, %xmm7
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm9
-; SSE2-NEXT:    pxor %xmm9, %xmm7
-; SSE2-NEXT:    pxor %xmm5, %xmm0
-; SSE2-NEXT:    movdqa %xmm5, %xmm2
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT:    por %xmm0, %xmm2
-; SSE2-NEXT:    pxor %xmm9, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[1,0,3,2]
-; SSE2-NEXT:    pand %xmm7, %xmm0
-; SSE2-NEXT:    movdqa %xmm10, %xmm6
-; SSE2-NEXT:    pxor %xmm5, %xmm6
+; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm0, %xmm5
+; SSE2-NEXT:    pxor %xmm8, %xmm5
+; SSE2-NEXT:    psubq %xmm2, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm6
+; SSE2-NEXT:    pxor %xmm8, %xmm6
 ; SSE2-NEXT:    movdqa %xmm5, %xmm7
 ; SSE2-NEXT:    pcmpgtd %xmm6, %xmm7
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm7[0,0,2,2]
 ; SSE2-NEXT:    pcmpeqd %xmm5, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT:    pand %xmm4, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm6[1,1,3,3]
+; SSE2-NEXT:    pand %xmm4, %xmm5
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm7[1,1,3,3]
-; SSE2-NEXT:    por %xmm6, %xmm4
-; SSE2-NEXT:    movdqa %xmm4, %xmm6
-; SSE2-NEXT:    pxor %xmm9, %xmm6
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[1,0,3,2]
+; SSE2-NEXT:    por %xmm5, %xmm4
+; SSE2-NEXT:    pxor %xmm8, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm8, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm6, %xmm2
-; SSE2-NEXT:    pxor %xmm9, %xmm2
-; SSE2-NEXT:    pandn %xmm2, %xmm0
-; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [9223372036854775808,9223372036854775808]
-; SSE2-NEXT:    movdqa %xmm4, %xmm2
-; SSE2-NEXT:    pandn %xmm8, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm11 = [9223372036854775807,9223372036854775807]
-; SSE2-NEXT:    pand %xmm11, %xmm4
-; SSE2-NEXT:    por %xmm2, %xmm4
-; SSE2-NEXT:    pand %xmm0, %xmm4
-; SSE2-NEXT:    pandn %xmm10, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm5[1,1,3,3]
+; SSE2-NEXT:    por %xmm2, %xmm7
+; SSE2-NEXT:    pxor %xmm4, %xmm7
+; SSE2-NEXT:    movdqa %xmm7, %xmm4
+; SSE2-NEXT:    pandn %xmm0, %xmm4
+; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [9223372036854775808,9223372036854775808]
+; SSE2-NEXT:    pxor %xmm5, %xmm5
+; SSE2-NEXT:    pxor %xmm6, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm6
+; SSE2-NEXT:    pandn %xmm9, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[1,1,3,3]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [9223372036854775807,9223372036854775807]
+; SSE2-NEXT:    pand %xmm10, %xmm2
+; SSE2-NEXT:    por %xmm2, %xmm0
+; SSE2-NEXT:    pand %xmm7, %xmm0
 ; SSE2-NEXT:    por %xmm4, %xmm0
 ; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    pxor %xmm8, %xmm2
 ; SSE2-NEXT:    psubq %xmm3, %xmm1
-; SSE2-NEXT:    pxor %xmm5, %xmm3
-; SSE2-NEXT:    movdqa %xmm5, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm3
+; SSE2-NEXT:    movdqa %xmm1, %xmm4
+; SSE2-NEXT:    pxor %xmm8, %xmm4
+; SSE2-NEXT:    movdqa %xmm2, %xmm7
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm7
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm7[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3]
+; SSE2-NEXT:    pand %xmm6, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm7[1,1,3,3]
+; SSE2-NEXT:    por %xmm2, %xmm4
+; SSE2-NEXT:    pxor %xmm8, %xmm3
+; SSE2-NEXT:    movdqa %xmm3, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm8, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm3
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm6, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm4
-; SSE2-NEXT:    pxor %xmm9, %xmm4
-; SSE2-NEXT:    pxor %xmm5, %xmm2
-; SSE2-NEXT:    movdqa %xmm5, %xmm3
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm2
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE2-NEXT:    por %xmm2, %xmm3
-; SSE2-NEXT:    pxor %xmm9, %xmm3
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,0,3,2]
-; SSE2-NEXT:    pand %xmm4, %xmm2
-; SSE2-NEXT:    movdqa %xmm1, %xmm4
-; SSE2-NEXT:    pxor %xmm5, %xmm4
-; SSE2-NEXT:    movdqa %xmm5, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm4, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    pand %xmm7, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm6[1,1,3,3]
-; SSE2-NEXT:    por %xmm4, %xmm5
-; SSE2-NEXT:    movdqa %xmm5, %xmm4
-; SSE2-NEXT:    pxor %xmm9, %xmm4
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,0,3,2]
-; SSE2-NEXT:    pand %xmm4, %xmm3
-; SSE2-NEXT:    pxor %xmm9, %xmm3
-; SSE2-NEXT:    pandn %xmm3, %xmm2
-; SSE2-NEXT:    movdqa %xmm5, %xmm3
-; SSE2-NEXT:    pandn %xmm8, %xmm3
-; SSE2-NEXT:    pand %xmm11, %xmm5
-; SSE2-NEXT:    por %xmm3, %xmm5
-; SSE2-NEXT:    pand %xmm2, %xmm5
-; SSE2-NEXT:    pandn %xmm1, %xmm2
-; SSE2-NEXT:    por %xmm5, %xmm2
-; SSE2-NEXT:    movdqa %xmm2, %xmm1
+; SSE2-NEXT:    por %xmm3, %xmm2
+; SSE2-NEXT:    pxor %xmm4, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm3
+; SSE2-NEXT:    pandn %xmm1, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm5
+; SSE2-NEXT:    pandn %xmm9, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm5[1,1,3,3]
+; SSE2-NEXT:    pand %xmm10, %xmm4
+; SSE2-NEXT:    por %xmm4, %xmm1
+; SSE2-NEXT:    pand %xmm2, %xmm1
+; SSE2-NEXT:    por %xmm3, %xmm1
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: v4i64:
 ; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    movdqa %xmm0, %xmm10
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm5 = [2147483648,2147483648]
-; SSSE3-NEXT:    psubq %xmm2, %xmm10
-; SSSE3-NEXT:    pxor %xmm5, %xmm2
-; SSSE3-NEXT:    movdqa %xmm5, %xmm6
-; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm7, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm7
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm9
-; SSSE3-NEXT:    pxor %xmm9, %xmm7
-; SSSE3-NEXT:    pxor %xmm5, %xmm0
-; SSSE3-NEXT:    movdqa %xmm5, %xmm2
-; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm6, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSSE3-NEXT:    por %xmm0, %xmm2
-; SSSE3-NEXT:    pxor %xmm9, %xmm2
-; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm7
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm7, %xmm0
-; SSSE3-NEXT:    movdqa %xmm10, %xmm6
-; SSSE3-NEXT:    pxor %xmm5, %xmm6
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm8 = [2147483648,2147483648]
+; SSSE3-NEXT:    movdqa %xmm0, %xmm5
+; SSSE3-NEXT:    pxor %xmm8, %xmm5
+; SSSE3-NEXT:    psubq %xmm2, %xmm0
+; SSSE3-NEXT:    movdqa %xmm0, %xmm6
+; SSSE3-NEXT:    pxor %xmm8, %xmm6
 ; SSSE3-NEXT:    movdqa %xmm5, %xmm7
 ; SSSE3-NEXT:    pcmpgtd %xmm6, %xmm7
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm7[0,0,2,2]
 ; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm4, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm6[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm4, %xmm5
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm7[1,1,3,3]
-; SSSE3-NEXT:    por %xmm6, %xmm4
-; SSSE3-NEXT:    movdqa %xmm4, %xmm6
-; SSSE3-NEXT:    pxor %xmm9, %xmm6
-; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[1,0,3,2]
+; SSSE3-NEXT:    por %xmm5, %xmm4
+; SSSE3-NEXT:    pxor %xmm8, %xmm2
+; SSSE3-NEXT:    movdqa %xmm2, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm8, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm8, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
 ; SSSE3-NEXT:    pand %xmm6, %xmm2
-; SSSE3-NEXT:    pxor %xmm9, %xmm2
-; SSSE3-NEXT:    pandn %xmm2, %xmm0
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm8 = [9223372036854775808,9223372036854775808]
-; SSSE3-NEXT:    movdqa %xmm4, %xmm2
-; SSSE3-NEXT:    pandn %xmm8, %xmm2
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm11 = [9223372036854775807,9223372036854775807]
-; SSSE3-NEXT:    pand %xmm11, %xmm4
-; SSSE3-NEXT:    por %xmm2, %xmm4
-; SSSE3-NEXT:    pand %xmm0, %xmm4
-; SSSE3-NEXT:    pandn %xmm10, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm5[1,1,3,3]
+; SSSE3-NEXT:    por %xmm2, %xmm7
+; SSSE3-NEXT:    pxor %xmm4, %xmm7
+; SSSE3-NEXT:    movdqa %xmm7, %xmm4
+; SSSE3-NEXT:    pandn %xmm0, %xmm4
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm9 = [9223372036854775808,9223372036854775808]
+; SSSE3-NEXT:    pxor %xmm5, %xmm5
+; SSSE3-NEXT:    pxor %xmm6, %xmm6
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm6
+; SSSE3-NEXT:    pandn %xmm9, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[1,1,3,3]
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm10 = [9223372036854775807,9223372036854775807]
+; SSSE3-NEXT:    pand %xmm10, %xmm2
+; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    pand %xmm7, %xmm0
 ; SSSE3-NEXT:    por %xmm4, %xmm0
 ; SSSE3-NEXT:    movdqa %xmm1, %xmm2
+; SSSE3-NEXT:    pxor %xmm8, %xmm2
 ; SSSE3-NEXT:    psubq %xmm3, %xmm1
-; SSSE3-NEXT:    pxor %xmm5, %xmm3
-; SSSE3-NEXT:    movdqa %xmm5, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm6, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    por %xmm3, %xmm4
-; SSSE3-NEXT:    pxor %xmm9, %xmm4
-; SSSE3-NEXT:    pxor %xmm5, %xmm2
-; SSSE3-NEXT:    movdqa %xmm5, %xmm3
-; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm6, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm3
-; SSSE3-NEXT:    pxor %xmm9, %xmm3
-; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm4, %xmm2
 ; SSSE3-NEXT:    movdqa %xmm1, %xmm4
-; SSSE3-NEXT:    pxor %xmm5, %xmm4
-; SSSE3-NEXT:    movdqa %xmm5, %xmm6
-; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm7, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm6[1,1,3,3]
-; SSSE3-NEXT:    por %xmm4, %xmm5
-; SSSE3-NEXT:    movdqa %xmm5, %xmm4
-; SSSE3-NEXT:    pxor %xmm9, %xmm4
-; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm4, %xmm3
-; SSSE3-NEXT:    pxor %xmm9, %xmm3
-; SSSE3-NEXT:    pandn %xmm3, %xmm2
-; SSSE3-NEXT:    movdqa %xmm5, %xmm3
-; SSSE3-NEXT:    pandn %xmm8, %xmm3
-; SSSE3-NEXT:    pand %xmm11, %xmm5
-; SSSE3-NEXT:    por %xmm3, %xmm5
-; SSSE3-NEXT:    pand %xmm2, %xmm5
-; SSSE3-NEXT:    pandn %xmm1, %xmm2
-; SSSE3-NEXT:    por %xmm5, %xmm2
-; SSSE3-NEXT:    movdqa %xmm2, %xmm1
+; SSSE3-NEXT:    pxor %xmm8, %xmm4
+; SSSE3-NEXT:    movdqa %xmm2, %xmm7
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm7
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm7[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm6, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm7[1,1,3,3]
+; SSSE3-NEXT:    por %xmm2, %xmm4
+; SSSE3-NEXT:    pxor %xmm8, %xmm3
+; SSSE3-NEXT:    movdqa %xmm3, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm8, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm8, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm6, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    por %xmm3, %xmm2
+; SSSE3-NEXT:    pxor %xmm4, %xmm2
+; SSSE3-NEXT:    movdqa %xmm2, %xmm3
+; SSSE3-NEXT:    pandn %xmm1, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm5
+; SSSE3-NEXT:    pandn %xmm9, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm5[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm10, %xmm4
+; SSSE3-NEXT:    por %xmm4, %xmm1
+; SSSE3-NEXT:    pand %xmm2, %xmm1
+; SSSE3-NEXT:    por %xmm3, %xmm1
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: v4i64:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa %xmm0, %xmm9
-; SSE41-NEXT:    movdqa {{.*#+}} xmm11 = [2147483648,2147483648]
-; SSE41-NEXT:    psubq %xmm2, %xmm9
-; SSE41-NEXT:    pxor %xmm11, %xmm2
-; SSE41-NEXT:    movdqa %xmm11, %xmm6
-; SSE41-NEXT:    pcmpgtd %xmm2, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm11, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,1,3,3]
-; SSE41-NEXT:    pand %xmm7, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[1,1,3,3]
-; SSE41-NEXT:    por %xmm4, %xmm2
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm10
-; SSE41-NEXT:    pxor %xmm10, %xmm2
-; SSE41-NEXT:    pxor %xmm11, %xmm0
-; SSE41-NEXT:    movdqa %xmm11, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm11, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm7, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE41-NEXT:    por %xmm0, %xmm4
-; SSE41-NEXT:    pxor %xmm10, %xmm4
-; SSE41-NEXT:    pcmpeqq %xmm4, %xmm2
-; SSE41-NEXT:    movdqa %xmm9, %xmm0
-; SSE41-NEXT:    pxor %xmm11, %xmm0
-; SSE41-NEXT:    movdqa %xmm11, %xmm7
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm7
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm7[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm11, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm6, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[1,1,3,3]
-; SSE41-NEXT:    por %xmm5, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm5
-; SSE41-NEXT:    pxor %xmm10, %xmm5
-; SSE41-NEXT:    pcmpeqq %xmm4, %xmm5
-; SSE41-NEXT:    pxor %xmm10, %xmm5
-; SSE41-NEXT:    pandn %xmm5, %xmm2
+; SSE41-NEXT:    movdqa %xmm0, %xmm4
+; SSE41-NEXT:    movdqa {{.*#+}} xmm9 = [2147483648,2147483648]
+; SSE41-NEXT:    pxor %xmm9, %xmm0
+; SSE41-NEXT:    psubq %xmm2, %xmm4
+; SSE41-NEXT:    movdqa %xmm4, %xmm5
+; SSE41-NEXT:    pxor %xmm9, %xmm5
+; SSE41-NEXT:    movdqa %xmm0, %xmm7
+; SSE41-NEXT:    pcmpeqd %xmm5, %xmm7
+; SSE41-NEXT:    pcmpgtd %xmm5, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm0[0,0,2,2]
+; SSE41-NEXT:    pand %xmm7, %xmm6
+; SSE41-NEXT:    por %xmm0, %xmm6
+; SSE41-NEXT:    pxor %xmm9, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm9, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm9, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[0,0,2,2]
+; SSE41-NEXT:    pand %xmm0, %xmm5
+; SSE41-NEXT:    por %xmm2, %xmm5
+; SSE41-NEXT:    pxor %xmm6, %xmm5
 ; SSE41-NEXT:    movapd {{.*#+}} xmm8 = [9223372036854775807,9223372036854775807]
 ; SSE41-NEXT:    movapd {{.*#+}} xmm7 = [9223372036854775808,9223372036854775808]
-; SSE41-NEXT:    movapd %xmm7, %xmm4
-; SSE41-NEXT:    blendvpd %xmm0, %xmm8, %xmm4
-; SSE41-NEXT:    movdqa %xmm2, %xmm0
-; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm9
+; SSE41-NEXT:    movapd %xmm7, %xmm2
+; SSE41-NEXT:    movdqa %xmm4, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm8, %xmm2
+; SSE41-NEXT:    movdqa %xmm5, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm4
 ; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pxor %xmm9, %xmm0
 ; SSE41-NEXT:    psubq %xmm3, %xmm1
-; SSE41-NEXT:    pxor %xmm11, %xmm3
-; SSE41-NEXT:    movdqa %xmm11, %xmm2
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm11, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pand %xmm4, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE41-NEXT:    movdqa %xmm1, %xmm2
+; SSE41-NEXT:    pxor %xmm9, %xmm2
+; SSE41-NEXT:    movdqa %xmm0, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm2, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm2, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm0[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm6
+; SSE41-NEXT:    por %xmm0, %xmm6
+; SSE41-NEXT:    pxor %xmm9, %xmm3
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm9, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm9, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pand %xmm0, %xmm2
 ; SSE41-NEXT:    por %xmm3, %xmm2
-; SSE41-NEXT:    pxor %xmm10, %xmm2
-; SSE41-NEXT:    pxor %xmm11, %xmm0
-; SSE41-NEXT:    movdqa %xmm11, %xmm3
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm11, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm4, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm0, %xmm3
-; SSE41-NEXT:    pxor %xmm10, %xmm3
-; SSE41-NEXT:    pcmpeqq %xmm3, %xmm2
+; SSE41-NEXT:    pxor %xmm6, %xmm2
 ; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    pxor %xmm11, %xmm0
-; SSE41-NEXT:    movdqa %xmm11, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm11, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3]
-; SSE41-NEXT:    por %xmm6, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm4
-; SSE41-NEXT:    pxor %xmm10, %xmm4
-; SSE41-NEXT:    pcmpeqq %xmm3, %xmm4
-; SSE41-NEXT:    pxor %xmm10, %xmm4
-; SSE41-NEXT:    pandn %xmm4, %xmm2
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm8, %xmm7
 ; SSE41-NEXT:    movdqa %xmm2, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm7, %xmm1
-; SSE41-NEXT:    movapd %xmm9, %xmm0
+; SSE41-NEXT:    movapd %xmm4, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: v4i64:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
 ; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm3, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
-; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm6
-; AVX1-NEXT:    vpcmpgtq %xmm6, %xmm3, %xmm7
-; AVX1-NEXT:    vpxor %xmm5, %xmm7, %xmm7
-; AVX1-NEXT:    vpcmpeqq %xmm4, %xmm7, %xmm8
-; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm3, %xmm4
-; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm9
-; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm3, %xmm4
-; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vpcmpeqq %xmm9, %xmm4, %xmm9
-; AVX1-NEXT:    vinsertf128 $1, %xmm8, %ymm9, %ymm8
-; AVX1-NEXT:    vpsubq %xmm2, %xmm6, %xmm9
-; AVX1-NEXT:    vpcmpgtq %xmm9, %xmm3, %xmm6
-; AVX1-NEXT:    vpxor %xmm5, %xmm6, %xmm2
-; AVX1-NEXT:    vpcmpeqq %xmm2, %xmm7, %xmm2
-; AVX1-NEXT:    vpxor %xmm5, %xmm2, %xmm2
-; AVX1-NEXT:    vpsubq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm3, %xmm1
-; AVX1-NEXT:    vpxor %xmm5, %xmm1, %xmm3
-; AVX1-NEXT:    vpcmpeqq %xmm3, %xmm4, %xmm3
-; AVX1-NEXT:    vpxor %xmm5, %xmm3, %xmm3
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
-; AVX1-NEXT:    vandnpd %ymm2, %ymm8, %ymm2
-; AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm1, %ymm1
-; AVX1-NEXT:    vmovapd {{.*#+}} ymm3 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
-; AVX1-NEXT:    vblendvpd %ymm1, {{.*}}(%rip), %ymm3, %ymm1
-; AVX1-NEXT:    vinsertf128 $1, %xmm9, %ymm0, %ymm0
-; AVX1-NEXT:    vblendvpd %ymm2, %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vpcmpgtq %xmm3, %xmm2, %xmm4
+; AVX1-NEXT:    vpcmpgtq %xmm3, %xmm1, %xmm3
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
+; AVX1-NEXT:    vpsubq %xmm2, %xmm4, %xmm2
+; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm4, %xmm4
+; AVX1-NEXT:    vpsubq %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
+; AVX1-NEXT:    vxorpd %ymm0, %ymm3, %ymm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; AVX1-NEXT:    vmovapd {{.*#+}} ymm2 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
+; AVX1-NEXT:    vblendvpd %ymm1, {{.*}}(%rip), %ymm2, %ymm2
+; AVX1-NEXT:    vblendvpd %ymm0, %ymm2, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: v4i64:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtq %ymm1, %ymm2, %ymm3
-; AVX2-NEXT:    vpcmpeqd %ymm4, %ymm4, %ymm4
-; AVX2-NEXT:    vpxor %ymm4, %ymm3, %ymm3
-; AVX2-NEXT:    vpcmpgtq %ymm0, %ymm2, %ymm5
-; AVX2-NEXT:    vpxor %ymm4, %ymm5, %ymm5
-; AVX2-NEXT:    vpcmpeqq %ymm3, %ymm5, %ymm3
-; AVX2-NEXT:    vpsubq %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpcmpgtq %ymm0, %ymm2, %ymm1
-; AVX2-NEXT:    vpxor %ymm4, %ymm1, %ymm2
-; AVX2-NEXT:    vpcmpeqq %ymm2, %ymm5, %ymm2
-; AVX2-NEXT:    vpxor %ymm4, %ymm2, %ymm2
-; AVX2-NEXT:    vpandn %ymm2, %ymm3, %ymm2
-; AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm3 = [9223372036854775807,9223372036854775807,9223372036854775807,9223372036854775807]
-; AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm4 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
-; AVX2-NEXT:    vblendvpd %ymm1, %ymm3, %ymm4, %ymm1
-; AVX2-NEXT:    vblendvpd %ymm2, %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpcmpgtq %ymm2, %ymm1, %ymm2
+; AVX2-NEXT:    vpsubq %ymm1, %ymm0, %ymm1
+; AVX2-NEXT:    vpcmpgtq %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %ymm0, %ymm2, %ymm0
+; AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm2 = [9223372036854775807,9223372036854775807,9223372036854775807,9223372036854775807]
+; AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm3 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
+; AVX2-NEXT:    vblendvpd %ymm1, %ymm2, %ymm3, %ymm2
+; AVX2-NEXT:    vblendvpd %ymm0, %ymm2, %ymm1, %ymm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: v4i64:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpcmpnltq %ymm2, %ymm1, %k0
-; AVX512-NEXT:    vpcmpnltq %ymm2, %ymm0, %k1
-; AVX512-NEXT:    kxorw %k0, %k1, %k0
-; AVX512-NEXT:    vpsubq %ymm1, %ymm0, %ymm0
-; AVX512-NEXT:    vpcmpnltq %ymm2, %ymm0, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
-; AVX512-NEXT:    kandw %k1, %k0, %k1
-; AVX512-NEXT:    vpcmpgtq %ymm0, %ymm2, %k2
-; AVX512-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
-; AVX512-NEXT:    vpbroadcastq {{.*}}(%rip), %ymm1 {%k2}
-; AVX512-NEXT:    vmovdqa64 %ymm1, %ymm0 {%k1}
+; AVX512-NEXT:    vpcmpgtq %ymm2, %ymm1, %k0
+; AVX512-NEXT:    vpsubq %ymm1, %ymm0, %ymm1
+; AVX512-NEXT:    vpcmpgtq %ymm1, %ymm0, %k1
+; AVX512-NEXT:    kxorw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpgtq %ymm1, %ymm2, %k2
+; AVX512-NEXT:    vpbroadcastq {{.*#+}} ymm0 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
+; AVX512-NEXT:    vpbroadcastq {{.*}}(%rip), %ymm0 {%k2}
+; AVX512-NEXT:    vmovdqa64 %ymm0, %ymm1 {%k1}
+; AVX512-NEXT:    vmovdqa %ymm1, %ymm0
 ; AVX512-NEXT:    retq
   %z = call <4 x i64> @llvm.ssub.sat.v4i64(<4 x i64> %x, <4 x i64> %y)
   ret <4 x i64> %z
@@ -2285,589 +1552,379 @@ define <4 x i64> @v4i64(<4 x i64> %x, <4 x i64> %y) nounwind {
 define <8 x i64> @v8i64(<8 x i64> %x, <8 x i64> %y) nounwind {
 ; SSE2-LABEL: v8i64:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa %xmm1, %xmm8
-; SSE2-NEXT:    movdqa %xmm0, %xmm13
-; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [2147483648,2147483648]
-; SSE2-NEXT:    psubq %xmm4, %xmm13
-; SSE2-NEXT:    pxor %xmm9, %xmm4
-; SSE2-NEXT:    movdqa %xmm9, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm4, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    pand %xmm10, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm4, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm10, %xmm10
-; SSE2-NEXT:    pxor %xmm10, %xmm1
-; SSE2-NEXT:    pxor %xmm9, %xmm0
-; SSE2-NEXT:    movdqa %xmm9, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pand %xmm11, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm15 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm0, %xmm15
-; SSE2-NEXT:    pxor %xmm10, %xmm15
-; SSE2-NEXT:    pcmpeqd %xmm15, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,0,3,2]
-; SSE2-NEXT:    pand %xmm1, %xmm0
-; SSE2-NEXT:    movdqa %xmm13, %xmm1
-; SSE2-NEXT:    pxor %xmm9, %xmm1
+; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm0, %xmm9
+; SSE2-NEXT:    pxor %xmm8, %xmm9
+; SSE2-NEXT:    psubq %xmm4, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm10
+; SSE2-NEXT:    pxor %xmm8, %xmm10
 ; SSE2-NEXT:    movdqa %xmm9, %xmm11
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm11
+; SSE2-NEXT:    pcmpgtd %xmm10, %xmm11
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm11[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm14 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pand %xmm12, %xmm14
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm11[1,1,3,3]
-; SSE2-NEXT:    por %xmm14, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm11
-; SSE2-NEXT:    pxor %xmm10, %xmm11
-; SSE2-NEXT:    pcmpeqd %xmm15, %xmm11
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm11[1,0,3,2]
-; SSE2-NEXT:    pand %xmm11, %xmm4
+; SSE2-NEXT:    pcmpeqd %xmm9, %xmm10
+; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm10[1,1,3,3]
+; SSE2-NEXT:    pand %xmm12, %xmm9
+; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm11[1,1,3,3]
+; SSE2-NEXT:    por %xmm9, %xmm10
+; SSE2-NEXT:    pxor %xmm8, %xmm4
+; SSE2-NEXT:    movdqa %xmm4, %xmm9
+; SSE2-NEXT:    pcmpgtd %xmm8, %xmm9
+; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm9[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm4[1,1,3,3]
+; SSE2-NEXT:    pand %xmm11, %xmm12
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm9[1,1,3,3]
+; SSE2-NEXT:    por %xmm12, %xmm4
 ; SSE2-NEXT:    pxor %xmm10, %xmm4
-; SSE2-NEXT:    pandn %xmm4, %xmm0
-; SSE2-NEXT:    movdqa {{.*#+}} xmm11 = [9223372036854775808,9223372036854775808]
+; SSE2-NEXT:    movdqa %xmm4, %xmm12
+; SSE2-NEXT:    pandn %xmm0, %xmm12
+; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [9223372036854775808,9223372036854775808]
+; SSE2-NEXT:    pxor %xmm11, %xmm11
+; SSE2-NEXT:    pxor %xmm10, %xmm10
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm10
+; SSE2-NEXT:    pandn %xmm9, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm10[1,1,3,3]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [9223372036854775807,9223372036854775807]
+; SSE2-NEXT:    pand %xmm10, %xmm13
+; SSE2-NEXT:    por %xmm13, %xmm0
+; SSE2-NEXT:    pand %xmm4, %xmm0
+; SSE2-NEXT:    por %xmm12, %xmm0
+; SSE2-NEXT:    movdqa %xmm1, %xmm12
+; SSE2-NEXT:    pxor %xmm8, %xmm12
+; SSE2-NEXT:    psubq %xmm5, %xmm1
 ; SSE2-NEXT:    movdqa %xmm1, %xmm4
-; SSE2-NEXT:    pandn %xmm11, %xmm4
-; SSE2-NEXT:    movdqa {{.*#+}} xmm12 = [9223372036854775807,9223372036854775807]
-; SSE2-NEXT:    pand %xmm12, %xmm1
-; SSE2-NEXT:    por %xmm4, %xmm1
-; SSE2-NEXT:    pand %xmm0, %xmm1
-; SSE2-NEXT:    pandn %xmm13, %xmm0
-; SSE2-NEXT:    por %xmm1, %xmm0
-; SSE2-NEXT:    movdqa %xmm8, %xmm1
-; SSE2-NEXT:    psubq %xmm5, %xmm8
-; SSE2-NEXT:    pxor %xmm9, %xmm5
-; SSE2-NEXT:    movdqa %xmm9, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm5, %xmm4
+; SSE2-NEXT:    pxor %xmm8, %xmm4
+; SSE2-NEXT:    movdqa %xmm12, %xmm13
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm13
+; SSE2-NEXT:    pshufd {{.*#+}} xmm14 = xmm13[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm12, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    pand %xmm14, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm13[1,1,3,3]
+; SSE2-NEXT:    por %xmm4, %xmm12
+; SSE2-NEXT:    pxor %xmm8, %xmm5
+; SSE2-NEXT:    movdqa %xmm5, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm8, %xmm4
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm5
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm5
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm13, %xmm5
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
 ; SSE2-NEXT:    por %xmm5, %xmm4
-; SSE2-NEXT:    pxor %xmm10, %xmm4
-; SSE2-NEXT:    pxor %xmm9, %xmm1
-; SSE2-NEXT:    movdqa %xmm9, %xmm5
+; SSE2-NEXT:    pxor %xmm12, %xmm4
+; SSE2-NEXT:    movdqa %xmm4, %xmm12
+; SSE2-NEXT:    pandn %xmm1, %xmm12
+; SSE2-NEXT:    pxor %xmm5, %xmm5
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm5[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pand %xmm13, %xmm1
+; SSE2-NEXT:    pandn %xmm9, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm5
-; SSE2-NEXT:    pxor %xmm10, %xmm5
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,0,3,2]
+; SSE2-NEXT:    pand %xmm10, %xmm5
+; SSE2-NEXT:    por %xmm5, %xmm1
 ; SSE2-NEXT:    pand %xmm4, %xmm1
-; SSE2-NEXT:    movdqa %xmm8, %xmm4
-; SSE2-NEXT:    pxor %xmm9, %xmm4
-; SSE2-NEXT:    movdqa %xmm9, %xmm13
-; SSE2-NEXT:    pcmpgtd %xmm4, %xmm13
-; SSE2-NEXT:    pshufd {{.*#+}} xmm14 = xmm13[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm15 = xmm4[1,1,3,3]
-; SSE2-NEXT:    pand %xmm14, %xmm15
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm13[1,1,3,3]
-; SSE2-NEXT:    por %xmm15, %xmm4
-; SSE2-NEXT:    movdqa %xmm4, %xmm13
-; SSE2-NEXT:    pxor %xmm10, %xmm13
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm13
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm13[1,0,3,2]
-; SSE2-NEXT:    pand %xmm13, %xmm5
-; SSE2-NEXT:    pxor %xmm10, %xmm5
-; SSE2-NEXT:    pandn %xmm5, %xmm1
-; SSE2-NEXT:    movdqa %xmm4, %xmm5
-; SSE2-NEXT:    pandn %xmm11, %xmm5
-; SSE2-NEXT:    pand %xmm12, %xmm4
-; SSE2-NEXT:    por %xmm5, %xmm4
-; SSE2-NEXT:    pand %xmm1, %xmm4
-; SSE2-NEXT:    pandn %xmm8, %xmm1
-; SSE2-NEXT:    por %xmm4, %xmm1
-; SSE2-NEXT:    movdqa %xmm2, %xmm4
+; SSE2-NEXT:    por %xmm12, %xmm1
+; SSE2-NEXT:    movdqa %xmm2, %xmm12
+; SSE2-NEXT:    pxor %xmm8, %xmm12
 ; SSE2-NEXT:    psubq %xmm6, %xmm2
-; SSE2-NEXT:    pxor %xmm9, %xmm6
-; SSE2-NEXT:    movdqa %xmm9, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm6, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm5[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT:    pand %xmm8, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE2-NEXT:    por %xmm6, %xmm5
-; SSE2-NEXT:    pxor %xmm10, %xmm5
-; SSE2-NEXT:    pxor %xmm9, %xmm4
-; SSE2-NEXT:    movdqa %xmm9, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm4, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    pand %xmm8, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT:    por %xmm4, %xmm6
-; SSE2-NEXT:    pxor %xmm10, %xmm6
-; SSE2-NEXT:    pcmpeqd %xmm6, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm5[1,0,3,2]
-; SSE2-NEXT:    pand %xmm5, %xmm8
 ; SSE2-NEXT:    movdqa %xmm2, %xmm5
-; SSE2-NEXT:    pxor %xmm9, %xmm5
-; SSE2-NEXT:    movdqa %xmm9, %xmm4
+; SSE2-NEXT:    pxor %xmm8, %xmm5
+; SSE2-NEXT:    movdqa %xmm12, %xmm4
 ; SSE2-NEXT:    pcmpgtd %xmm5, %xmm4
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm5
+; SSE2-NEXT:    pcmpeqd %xmm12, %xmm5
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm13, %xmm5
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
 ; SSE2-NEXT:    por %xmm5, %xmm4
-; SSE2-NEXT:    movdqa %xmm4, %xmm5
-; SSE2-NEXT:    pxor %xmm10, %xmm5
-; SSE2-NEXT:    pcmpeqd %xmm6, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[1,0,3,2]
-; SSE2-NEXT:    pand %xmm5, %xmm6
-; SSE2-NEXT:    pxor %xmm10, %xmm6
-; SSE2-NEXT:    pandn %xmm6, %xmm8
-; SSE2-NEXT:    movdqa %xmm4, %xmm5
-; SSE2-NEXT:    pandn %xmm11, %xmm5
-; SSE2-NEXT:    pand %xmm12, %xmm4
-; SSE2-NEXT:    por %xmm5, %xmm4
-; SSE2-NEXT:    pand %xmm8, %xmm4
-; SSE2-NEXT:    pandn %xmm2, %xmm8
-; SSE2-NEXT:    por %xmm4, %xmm8
-; SSE2-NEXT:    movdqa %xmm3, %xmm2
+; SSE2-NEXT:    pxor %xmm8, %xmm6
+; SSE2-NEXT:    movdqa %xmm6, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm8, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm5[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSE2-NEXT:    pand %xmm12, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSE2-NEXT:    por %xmm6, %xmm5
+; SSE2-NEXT:    pxor %xmm4, %xmm5
+; SSE2-NEXT:    movdqa %xmm5, %xmm4
+; SSE2-NEXT:    pandn %xmm2, %xmm4
+; SSE2-NEXT:    pxor %xmm6, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm6
+; SSE2-NEXT:    pandn %xmm9, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSE2-NEXT:    pand %xmm10, %xmm6
+; SSE2-NEXT:    por %xmm6, %xmm2
+; SSE2-NEXT:    pand %xmm5, %xmm2
+; SSE2-NEXT:    por %xmm4, %xmm2
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    pxor %xmm8, %xmm4
 ; SSE2-NEXT:    psubq %xmm7, %xmm3
-; SSE2-NEXT:    pxor %xmm9, %xmm7
-; SSE2-NEXT:    movdqa %xmm9, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm7, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm7[1,1,3,3]
-; SSE2-NEXT:    pand %xmm5, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm6, %xmm4
-; SSE2-NEXT:    pxor %xmm10, %xmm4
-; SSE2-NEXT:    pxor %xmm9, %xmm2
-; SSE2-NEXT:    movdqa %xmm9, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[1,1,3,3]
-; SSE2-NEXT:    por %xmm2, %xmm6
-; SSE2-NEXT:    pxor %xmm10, %xmm6
-; SSE2-NEXT:    pcmpeqd %xmm6, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[1,0,3,2]
-; SSE2-NEXT:    pand %xmm4, %xmm5
-; SSE2-NEXT:    movdqa %xmm3, %xmm2
-; SSE2-NEXT:    pxor %xmm9, %xmm2
-; SSE2-NEXT:    movdqa %xmm9, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT:    pand %xmm7, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm2, %xmm4
-; SSE2-NEXT:    movdqa %xmm4, %xmm2
-; SSE2-NEXT:    pxor %xmm10, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm6, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[1,0,3,2]
-; SSE2-NEXT:    pand %xmm2, %xmm6
-; SSE2-NEXT:    pxor %xmm10, %xmm6
-; SSE2-NEXT:    pandn %xmm6, %xmm5
-; SSE2-NEXT:    movdqa %xmm4, %xmm2
-; SSE2-NEXT:    pandn %xmm11, %xmm2
+; SSE2-NEXT:    movdqa %xmm3, %xmm5
+; SSE2-NEXT:    pxor %xmm8, %xmm5
+; SSE2-NEXT:    movdqa %xmm4, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm5, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm6[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm4, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm5[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm12, %xmm4
-; SSE2-NEXT:    por %xmm2, %xmm4
-; SSE2-NEXT:    pand %xmm5, %xmm4
-; SSE2-NEXT:    pandn %xmm3, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm6[1,1,3,3]
 ; SSE2-NEXT:    por %xmm4, %xmm5
-; SSE2-NEXT:    movdqa %xmm8, %xmm2
-; SSE2-NEXT:    movdqa %xmm5, %xmm3
+; SSE2-NEXT:    pxor %xmm8, %xmm7
+; SSE2-NEXT:    movdqa %xmm7, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm8, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm7
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
+; SSE2-NEXT:    pand %xmm6, %xmm7
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm7, %xmm4
+; SSE2-NEXT:    pxor %xmm5, %xmm4
+; SSE2-NEXT:    movdqa %xmm4, %xmm5
+; SSE2-NEXT:    pandn %xmm3, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm11
+; SSE2-NEXT:    pandn %xmm9, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm11[1,1,3,3]
+; SSE2-NEXT:    pand %xmm10, %xmm6
+; SSE2-NEXT:    por %xmm6, %xmm3
+; SSE2-NEXT:    pand %xmm4, %xmm3
+; SSE2-NEXT:    por %xmm5, %xmm3
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: v8i64:
 ; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    movdqa %xmm1, %xmm8
-; SSSE3-NEXT:    movdqa %xmm0, %xmm13
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm9 = [2147483648,2147483648]
-; SSSE3-NEXT:    psubq %xmm4, %xmm13
-; SSSE3-NEXT:    pxor %xmm9, %xmm4
-; SSSE3-NEXT:    movdqa %xmm9, %xmm1
-; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm10 = xmm1[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm10, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    por %xmm4, %xmm1
-; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm10
-; SSSE3-NEXT:    pxor %xmm10, %xmm1
-; SSSE3-NEXT:    pxor %xmm9, %xmm0
-; SSSE3-NEXT:    movdqa %xmm9, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm11 = xmm4[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm11, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm15 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    por %xmm0, %xmm15
-; SSSE3-NEXT:    pxor %xmm10, %xmm15
-; SSSE3-NEXT:    pcmpeqd %xmm15, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm0
-; SSSE3-NEXT:    movdqa %xmm13, %xmm1
-; SSSE3-NEXT:    pxor %xmm9, %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm8 = [2147483648,2147483648]
+; SSSE3-NEXT:    movdqa %xmm0, %xmm9
+; SSSE3-NEXT:    pxor %xmm8, %xmm9
+; SSSE3-NEXT:    psubq %xmm4, %xmm0
+; SSSE3-NEXT:    movdqa %xmm0, %xmm10
+; SSSE3-NEXT:    pxor %xmm8, %xmm10
 ; SSSE3-NEXT:    movdqa %xmm9, %xmm11
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm11
+; SSSE3-NEXT:    pcmpgtd %xmm10, %xmm11
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm12 = xmm11[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm14 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm12, %xmm14
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm11[1,1,3,3]
-; SSSE3-NEXT:    por %xmm14, %xmm1
-; SSSE3-NEXT:    movdqa %xmm1, %xmm11
-; SSSE3-NEXT:    pxor %xmm10, %xmm11
-; SSSE3-NEXT:    pcmpeqd %xmm15, %xmm11
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm11[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm11, %xmm4
+; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm10
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm9 = xmm10[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm12, %xmm9
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm10 = xmm11[1,1,3,3]
+; SSSE3-NEXT:    por %xmm9, %xmm10
+; SSSE3-NEXT:    pxor %xmm8, %xmm4
+; SSSE3-NEXT:    movdqa %xmm4, %xmm9
+; SSSE3-NEXT:    pcmpgtd %xmm8, %xmm9
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm11 = xmm9[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm8, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm12 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm11, %xmm12
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm9[1,1,3,3]
+; SSSE3-NEXT:    por %xmm12, %xmm4
 ; SSSE3-NEXT:    pxor %xmm10, %xmm4
-; SSSE3-NEXT:    pandn %xmm4, %xmm0
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm11 = [9223372036854775808,9223372036854775808]
+; SSSE3-NEXT:    movdqa %xmm4, %xmm12
+; SSSE3-NEXT:    pandn %xmm0, %xmm12
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm9 = [9223372036854775808,9223372036854775808]
+; SSSE3-NEXT:    pxor %xmm11, %xmm11
+; SSSE3-NEXT:    pxor %xmm10, %xmm10
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm10
+; SSSE3-NEXT:    pandn %xmm9, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm13 = xmm10[1,1,3,3]
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm10 = [9223372036854775807,9223372036854775807]
+; SSSE3-NEXT:    pand %xmm10, %xmm13
+; SSSE3-NEXT:    por %xmm13, %xmm0
+; SSSE3-NEXT:    pand %xmm4, %xmm0
+; SSSE3-NEXT:    por %xmm12, %xmm0
+; SSSE3-NEXT:    movdqa %xmm1, %xmm12
+; SSSE3-NEXT:    pxor %xmm8, %xmm12
+; SSSE3-NEXT:    psubq %xmm5, %xmm1
 ; SSSE3-NEXT:    movdqa %xmm1, %xmm4
-; SSSE3-NEXT:    pandn %xmm11, %xmm4
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm12 = [9223372036854775807,9223372036854775807]
-; SSSE3-NEXT:    pand %xmm12, %xmm1
-; SSSE3-NEXT:    por %xmm4, %xmm1
-; SSSE3-NEXT:    pand %xmm0, %xmm1
-; SSSE3-NEXT:    pandn %xmm13, %xmm0
-; SSSE3-NEXT:    por %xmm1, %xmm0
-; SSSE3-NEXT:    movdqa %xmm8, %xmm1
-; SSSE3-NEXT:    psubq %xmm5, %xmm8
-; SSSE3-NEXT:    pxor %xmm9, %xmm5
-; SSSE3-NEXT:    movdqa %xmm9, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm4
+; SSSE3-NEXT:    pxor %xmm8, %xmm4
+; SSSE3-NEXT:    movdqa %xmm12, %xmm13
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm13
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm14 = xmm13[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm12, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm14, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm12 = xmm13[1,1,3,3]
+; SSSE3-NEXT:    por %xmm4, %xmm12
+; SSSE3-NEXT:    pxor %xmm8, %xmm5
+; SSSE3-NEXT:    movdqa %xmm5, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm8, %xmm4
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm13 = xmm4[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm5
+; SSSE3-NEXT:    pcmpeqd %xmm8, %xmm5
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
 ; SSSE3-NEXT:    pand %xmm13, %xmm5
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
 ; SSSE3-NEXT:    por %xmm5, %xmm4
-; SSSE3-NEXT:    pxor %xmm10, %xmm4
-; SSSE3-NEXT:    pxor %xmm9, %xmm1
-; SSSE3-NEXT:    movdqa %xmm9, %xmm5
+; SSSE3-NEXT:    pxor %xmm12, %xmm4
+; SSSE3-NEXT:    movdqa %xmm4, %xmm12
+; SSSE3-NEXT:    pandn %xmm1, %xmm12
+; SSSE3-NEXT:    pxor %xmm5, %xmm5
 ; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm5
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm13 = xmm5[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm13, %xmm1
+; SSSE3-NEXT:    pandn %xmm9, %xmm1
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSSE3-NEXT:    por %xmm1, %xmm5
-; SSSE3-NEXT:    pxor %xmm10, %xmm5
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,0,3,2]
+; SSSE3-NEXT:    pand %xmm10, %xmm5
+; SSSE3-NEXT:    por %xmm5, %xmm1
 ; SSSE3-NEXT:    pand %xmm4, %xmm1
-; SSSE3-NEXT:    movdqa %xmm8, %xmm4
-; SSSE3-NEXT:    pxor %xmm9, %xmm4
-; SSSE3-NEXT:    movdqa %xmm9, %xmm13
-; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm13
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm14 = xmm13[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm15 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm14, %xmm15
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm13[1,1,3,3]
-; SSSE3-NEXT:    por %xmm15, %xmm4
-; SSSE3-NEXT:    movdqa %xmm4, %xmm13
-; SSSE3-NEXT:    pxor %xmm10, %xmm13
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm13
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm13[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm13, %xmm5
-; SSSE3-NEXT:    pxor %xmm10, %xmm5
-; SSSE3-NEXT:    pandn %xmm5, %xmm1
-; SSSE3-NEXT:    movdqa %xmm4, %xmm5
-; SSSE3-NEXT:    pandn %xmm11, %xmm5
-; SSSE3-NEXT:    pand %xmm12, %xmm4
-; SSSE3-NEXT:    por %xmm5, %xmm4
-; SSSE3-NEXT:    pand %xmm1, %xmm4
-; SSSE3-NEXT:    pandn %xmm8, %xmm1
-; SSSE3-NEXT:    por %xmm4, %xmm1
-; SSSE3-NEXT:    movdqa %xmm2, %xmm4
+; SSSE3-NEXT:    por %xmm12, %xmm1
+; SSSE3-NEXT:    movdqa %xmm2, %xmm12
+; SSSE3-NEXT:    pxor %xmm8, %xmm12
 ; SSSE3-NEXT:    psubq %xmm6, %xmm2
-; SSSE3-NEXT:    pxor %xmm9, %xmm6
-; SSSE3-NEXT:    movdqa %xmm9, %xmm5
-; SSSE3-NEXT:    pcmpgtd %xmm6, %xmm5
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm5[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm8, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSSE3-NEXT:    por %xmm6, %xmm5
-; SSSE3-NEXT:    pxor %xmm10, %xmm5
-; SSSE3-NEXT:    pxor %xmm9, %xmm4
-; SSSE3-NEXT:    movdqa %xmm9, %xmm6
-; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm6[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm8, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSSE3-NEXT:    por %xmm4, %xmm6
-; SSSE3-NEXT:    pxor %xmm10, %xmm6
-; SSSE3-NEXT:    pcmpeqd %xmm6, %xmm5
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm5[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm5, %xmm8
 ; SSSE3-NEXT:    movdqa %xmm2, %xmm5
-; SSSE3-NEXT:    pxor %xmm9, %xmm5
-; SSSE3-NEXT:    movdqa %xmm9, %xmm4
+; SSSE3-NEXT:    pxor %xmm8, %xmm5
+; SSSE3-NEXT:    movdqa %xmm12, %xmm4
 ; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm4
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm13 = xmm4[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm5
+; SSSE3-NEXT:    pcmpeqd %xmm12, %xmm5
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
 ; SSSE3-NEXT:    pand %xmm13, %xmm5
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
 ; SSSE3-NEXT:    por %xmm5, %xmm4
-; SSSE3-NEXT:    movdqa %xmm4, %xmm5
-; SSSE3-NEXT:    pxor %xmm10, %xmm5
-; SSSE3-NEXT:    pcmpeqd %xmm6, %xmm5
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm5, %xmm6
-; SSSE3-NEXT:    pxor %xmm10, %xmm6
-; SSSE3-NEXT:    pandn %xmm6, %xmm8
-; SSSE3-NEXT:    movdqa %xmm4, %xmm5
-; SSSE3-NEXT:    pandn %xmm11, %xmm5
-; SSSE3-NEXT:    pand %xmm12, %xmm4
-; SSSE3-NEXT:    por %xmm5, %xmm4
-; SSSE3-NEXT:    pand %xmm8, %xmm4
-; SSSE3-NEXT:    pandn %xmm2, %xmm8
-; SSSE3-NEXT:    por %xmm4, %xmm8
-; SSSE3-NEXT:    movdqa %xmm3, %xmm2
+; SSSE3-NEXT:    pxor %xmm8, %xmm6
+; SSSE3-NEXT:    movdqa %xmm6, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm8, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm12 = xmm5[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm8, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm12, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSSE3-NEXT:    por %xmm6, %xmm5
+; SSSE3-NEXT:    pxor %xmm4, %xmm5
+; SSSE3-NEXT:    movdqa %xmm5, %xmm4
+; SSSE3-NEXT:    pandn %xmm2, %xmm4
+; SSSE3-NEXT:    pxor %xmm6, %xmm6
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm6
+; SSSE3-NEXT:    pandn %xmm9, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm10, %xmm6
+; SSSE3-NEXT:    por %xmm6, %xmm2
+; SSSE3-NEXT:    pand %xmm5, %xmm2
+; SSSE3-NEXT:    por %xmm4, %xmm2
+; SSSE3-NEXT:    movdqa %xmm3, %xmm4
+; SSSE3-NEXT:    pxor %xmm8, %xmm4
 ; SSSE3-NEXT:    psubq %xmm7, %xmm3
-; SSSE3-NEXT:    pxor %xmm9, %xmm7
-; SSSE3-NEXT:    movdqa %xmm9, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm7, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm7
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm7[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm5, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    por %xmm6, %xmm4
-; SSSE3-NEXT:    pxor %xmm10, %xmm4
-; SSSE3-NEXT:    pxor %xmm9, %xmm2
-; SSSE3-NEXT:    movdqa %xmm9, %xmm5
-; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm5
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm6, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm6
-; SSSE3-NEXT:    pxor %xmm10, %xmm6
-; SSSE3-NEXT:    pcmpeqd %xmm6, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm4, %xmm5
-; SSSE3-NEXT:    movdqa %xmm3, %xmm2
-; SSSE3-NEXT:    pxor %xmm9, %xmm2
-; SSSE3-NEXT:    movdqa %xmm9, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm7, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm4
-; SSSE3-NEXT:    movdqa %xmm4, %xmm2
-; SSSE3-NEXT:    pxor %xmm10, %xmm2
-; SSSE3-NEXT:    pcmpeqd %xmm6, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm2, %xmm6
-; SSSE3-NEXT:    pxor %xmm10, %xmm6
-; SSSE3-NEXT:    pandn %xmm6, %xmm5
-; SSSE3-NEXT:    movdqa %xmm4, %xmm2
-; SSSE3-NEXT:    pandn %xmm11, %xmm2
+; SSSE3-NEXT:    movdqa %xmm3, %xmm5
+; SSSE3-NEXT:    pxor %xmm8, %xmm5
+; SSSE3-NEXT:    movdqa %xmm4, %xmm6
+; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm12 = xmm6[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm5[1,1,3,3]
 ; SSSE3-NEXT:    pand %xmm12, %xmm4
-; SSSE3-NEXT:    por %xmm2, %xmm4
-; SSSE3-NEXT:    pand %xmm5, %xmm4
-; SSSE3-NEXT:    pandn %xmm3, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm6[1,1,3,3]
 ; SSSE3-NEXT:    por %xmm4, %xmm5
-; SSSE3-NEXT:    movdqa %xmm8, %xmm2
-; SSSE3-NEXT:    movdqa %xmm5, %xmm3
+; SSSE3-NEXT:    pxor %xmm8, %xmm7
+; SSSE3-NEXT:    movdqa %xmm7, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm8, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm8, %xmm7
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm6, %xmm7
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm7, %xmm4
+; SSSE3-NEXT:    pxor %xmm5, %xmm4
+; SSSE3-NEXT:    movdqa %xmm4, %xmm5
+; SSSE3-NEXT:    pandn %xmm3, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm11
+; SSSE3-NEXT:    pandn %xmm9, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm11[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm10, %xmm6
+; SSSE3-NEXT:    por %xmm6, %xmm3
+; SSSE3-NEXT:    pand %xmm4, %xmm3
+; SSSE3-NEXT:    por %xmm5, %xmm3
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: v8i64:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movdqa %xmm0, %xmm8
-; SSE41-NEXT:    movdqa {{.*#+}} xmm9 = [2147483648,2147483648]
-; SSE41-NEXT:    movdqa %xmm0, %xmm11
+; SSE41-NEXT:    movdqa {{.*#+}} xmm10 = [2147483648,2147483648]
+; SSE41-NEXT:    pxor %xmm10, %xmm0
 ; SSE41-NEXT:    psubq %xmm4, %xmm8
-; SSE41-NEXT:    pxor %xmm9, %xmm4
-; SSE41-NEXT:    movdqa %xmm9, %xmm0
-; SSE41-NEXT:    pcmpgtd %xmm4, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm10 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm9, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm12 = xmm4[1,1,3,3]
-; SSE41-NEXT:    pand %xmm10, %xmm12
-; SSE41-NEXT:    pshufd {{.*#+}} xmm15 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm12, %xmm15
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm10
-; SSE41-NEXT:    pxor %xmm10, %xmm15
-; SSE41-NEXT:    pxor %xmm9, %xmm11
-; SSE41-NEXT:    movdqa %xmm9, %xmm0
-; SSE41-NEXT:    pcmpgtd %xmm11, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm12 = xmm0[0,0,2,2]
+; SSE41-NEXT:    movdqa %xmm8, %xmm9
+; SSE41-NEXT:    pxor %xmm10, %xmm9
+; SSE41-NEXT:    movdqa %xmm0, %xmm11
 ; SSE41-NEXT:    pcmpeqd %xmm9, %xmm11
-; SSE41-NEXT:    pshufd {{.*#+}} xmm11 = xmm11[1,1,3,3]
-; SSE41-NEXT:    pand %xmm12, %xmm11
-; SSE41-NEXT:    pshufd {{.*#+}} xmm12 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm11, %xmm12
-; SSE41-NEXT:    pxor %xmm10, %xmm12
-; SSE41-NEXT:    pcmpeqq %xmm12, %xmm15
-; SSE41-NEXT:    movdqa %xmm8, %xmm0
-; SSE41-NEXT:    pxor %xmm9, %xmm0
-; SSE41-NEXT:    movdqa %xmm9, %xmm11
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm11
-; SSE41-NEXT:    pshufd {{.*#+}} xmm13 = xmm11[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm9, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm14 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm13, %xmm14
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm11[1,1,3,3]
-; SSE41-NEXT:    por %xmm14, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm4
-; SSE41-NEXT:    pxor %xmm10, %xmm4
-; SSE41-NEXT:    pcmpeqq %xmm12, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm9, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm12 = xmm0[0,0,2,2]
+; SSE41-NEXT:    pand %xmm11, %xmm12
+; SSE41-NEXT:    por %xmm0, %xmm12
 ; SSE41-NEXT:    pxor %xmm10, %xmm4
-; SSE41-NEXT:    pandn %xmm4, %xmm15
+; SSE41-NEXT:    movdqa %xmm4, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm10, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm10, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm9 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pand %xmm0, %xmm9
+; SSE41-NEXT:    por %xmm4, %xmm9
+; SSE41-NEXT:    pxor %xmm12, %xmm9
 ; SSE41-NEXT:    movapd {{.*#+}} xmm12 = [9223372036854775807,9223372036854775807]
 ; SSE41-NEXT:    movapd {{.*#+}} xmm11 = [9223372036854775808,9223372036854775808]
 ; SSE41-NEXT:    movapd %xmm11, %xmm4
+; SSE41-NEXT:    movdqa %xmm8, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm12, %xmm4
-; SSE41-NEXT:    movdqa %xmm15, %xmm0
+; SSE41-NEXT:    movdqa %xmm9, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm8
 ; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pxor %xmm10, %xmm0
 ; SSE41-NEXT:    psubq %xmm5, %xmm1
-; SSE41-NEXT:    pxor %xmm9, %xmm5
-; SSE41-NEXT:    movdqa %xmm9, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm5, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm13 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm9, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE41-NEXT:    pand %xmm13, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE41-NEXT:    por %xmm5, %xmm4
+; SSE41-NEXT:    movdqa %xmm1, %xmm4
 ; SSE41-NEXT:    pxor %xmm10, %xmm4
-; SSE41-NEXT:    pxor %xmm9, %xmm0
-; SSE41-NEXT:    movdqa %xmm9, %xmm5
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm13 = xmm5[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm9, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm13, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm14 = xmm5[1,1,3,3]
-; SSE41-NEXT:    por %xmm0, %xmm14
-; SSE41-NEXT:    pxor %xmm10, %xmm14
-; SSE41-NEXT:    pcmpeqq %xmm14, %xmm4
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    pxor %xmm9, %xmm0
-; SSE41-NEXT:    movdqa %xmm9, %xmm13
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm13
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm13[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm9, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm15 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm15
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm13[1,1,3,3]
-; SSE41-NEXT:    por %xmm15, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm5
-; SSE41-NEXT:    pxor %xmm10, %xmm5
-; SSE41-NEXT:    pcmpeqq %xmm14, %xmm5
+; SSE41-NEXT:    movdqa %xmm0, %xmm9
+; SSE41-NEXT:    pcmpeqd %xmm4, %xmm9
+; SSE41-NEXT:    pcmpgtd %xmm4, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm13 = xmm0[0,0,2,2]
+; SSE41-NEXT:    pand %xmm9, %xmm13
+; SSE41-NEXT:    por %xmm0, %xmm13
 ; SSE41-NEXT:    pxor %xmm10, %xmm5
-; SSE41-NEXT:    pandn %xmm5, %xmm4
+; SSE41-NEXT:    movdqa %xmm5, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm10, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm10, %xmm5
+; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm5[0,0,2,2]
+; SSE41-NEXT:    pand %xmm0, %xmm4
+; SSE41-NEXT:    por %xmm5, %xmm4
+; SSE41-NEXT:    pxor %xmm13, %xmm4
 ; SSE41-NEXT:    movapd %xmm11, %xmm5
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm12, %xmm5
 ; SSE41-NEXT:    movdqa %xmm4, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm5, %xmm1
 ; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    pxor %xmm10, %xmm0
 ; SSE41-NEXT:    psubq %xmm6, %xmm2
-; SSE41-NEXT:    pxor %xmm9, %xmm6
-; SSE41-NEXT:    movdqa %xmm9, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm6, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm9, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE41-NEXT:    por %xmm6, %xmm4
+; SSE41-NEXT:    movdqa %xmm2, %xmm4
 ; SSE41-NEXT:    pxor %xmm10, %xmm4
-; SSE41-NEXT:    pxor %xmm9, %xmm0
-; SSE41-NEXT:    movdqa %xmm9, %xmm5
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm9, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm6, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSE41-NEXT:    movdqa %xmm0, %xmm9
+; SSE41-NEXT:    pcmpeqd %xmm4, %xmm9
+; SSE41-NEXT:    pcmpgtd %xmm4, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[0,0,2,2]
+; SSE41-NEXT:    pand %xmm9, %xmm5
 ; SSE41-NEXT:    por %xmm0, %xmm5
-; SSE41-NEXT:    pxor %xmm10, %xmm5
-; SSE41-NEXT:    pcmpeqq %xmm5, %xmm4
-; SSE41-NEXT:    movdqa %xmm2, %xmm0
-; SSE41-NEXT:    pxor %xmm9, %xmm0
-; SSE41-NEXT:    movdqa %xmm9, %xmm6
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm13 = xmm6[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm9, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm14 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm13, %xmm14
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3]
-; SSE41-NEXT:    por %xmm14, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm6
-; SSE41-NEXT:    pxor %xmm10, %xmm6
-; SSE41-NEXT:    pcmpeqq %xmm5, %xmm6
 ; SSE41-NEXT:    pxor %xmm10, %xmm6
-; SSE41-NEXT:    pandn %xmm6, %xmm4
+; SSE41-NEXT:    movdqa %xmm6, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm10, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm10, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[0,0,2,2]
+; SSE41-NEXT:    pand %xmm0, %xmm4
+; SSE41-NEXT:    por %xmm6, %xmm4
+; SSE41-NEXT:    pxor %xmm5, %xmm4
 ; SSE41-NEXT:    movapd %xmm11, %xmm5
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm12, %xmm5
 ; SSE41-NEXT:    movdqa %xmm4, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm5, %xmm2
 ; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    pxor %xmm10, %xmm0
 ; SSE41-NEXT:    psubq %xmm7, %xmm3
-; SSE41-NEXT:    pxor %xmm9, %xmm7
-; SSE41-NEXT:    movdqa %xmm9, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm7, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm9, %xmm7
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm7[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE41-NEXT:    por %xmm6, %xmm4
+; SSE41-NEXT:    movdqa %xmm3, %xmm4
 ; SSE41-NEXT:    pxor %xmm10, %xmm4
-; SSE41-NEXT:    pxor %xmm9, %xmm0
-; SSE41-NEXT:    movdqa %xmm9, %xmm5
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm9, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm6, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE41-NEXT:    por %xmm0, %xmm5
-; SSE41-NEXT:    pxor %xmm10, %xmm5
-; SSE41-NEXT:    pcmpeqq %xmm5, %xmm4
+; SSE41-NEXT:    movdqa %xmm0, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm4, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm4, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm0[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm6
+; SSE41-NEXT:    por %xmm0, %xmm6
+; SSE41-NEXT:    pxor %xmm10, %xmm7
+; SSE41-NEXT:    movdqa %xmm7, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm10, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm10, %xmm7
+; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm7[0,0,2,2]
+; SSE41-NEXT:    pand %xmm0, %xmm4
+; SSE41-NEXT:    por %xmm7, %xmm4
+; SSE41-NEXT:    pxor %xmm6, %xmm4
 ; SSE41-NEXT:    movdqa %xmm3, %xmm0
-; SSE41-NEXT:    pxor %xmm9, %xmm0
-; SSE41-NEXT:    movdqa %xmm9, %xmm6
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm13 = xmm6[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm9, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm13, %xmm7
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3]
-; SSE41-NEXT:    por %xmm7, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm6
-; SSE41-NEXT:    pxor %xmm10, %xmm6
-; SSE41-NEXT:    pcmpeqq %xmm5, %xmm6
-; SSE41-NEXT:    pxor %xmm10, %xmm6
-; SSE41-NEXT:    pandn %xmm6, %xmm4
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm12, %xmm11
 ; SSE41-NEXT:    movdqa %xmm4, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm11, %xmm3
@@ -2876,118 +1933,70 @@ define <8 x i64> @v8i64(<8 x i64> %x, <8 x i64> %y) nounwind {
 ;
 ; AVX1-LABEL: v8i64:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm9
-; AVX1-NEXT:    vpxor %xmm10, %xmm10, %xmm10
-; AVX1-NEXT:    vpcmpgtq %xmm9, %xmm10, %xmm7
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm8
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm4
+; AVX1-NEXT:    vpxor %xmm5, %xmm5, %xmm5
+; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm4, %xmm6
+; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm2, %xmm7
+; AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm7, %ymm6
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm7
-; AVX1-NEXT:    vpcmpgtq %xmm7, %xmm10, %xmm6
-; AVX1-NEXT:    vpxor %xmm4, %xmm6, %xmm12
-; AVX1-NEXT:    vpcmpeqq %xmm8, %xmm12, %xmm8
-; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm10, %xmm5
-; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm11
-; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm10, %xmm5
-; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX1-NEXT:    vpcmpeqq %xmm11, %xmm5, %xmm11
-; AVX1-NEXT:    vinsertf128 $1, %xmm8, %ymm11, %ymm8
-; AVX1-NEXT:    vpsubq %xmm9, %xmm7, %xmm9
-; AVX1-NEXT:    vpcmpgtq %xmm9, %xmm10, %xmm7
-; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm6
-; AVX1-NEXT:    vpcmpeqq %xmm6, %xmm12, %xmm6
-; AVX1-NEXT:    vpxor %xmm4, %xmm6, %xmm11
-; AVX1-NEXT:    vpsubq %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm10, %xmm2
-; AVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm6
-; AVX1-NEXT:    vpcmpeqq %xmm6, %xmm5, %xmm5
-; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX1-NEXT:    vinsertf128 $1, %xmm11, %ymm5, %ymm5
-; AVX1-NEXT:    vandnpd %ymm5, %ymm8, %ymm5
-; AVX1-NEXT:    vinsertf128 $1, %xmm7, %ymm2, %ymm7
-; AVX1-NEXT:    vmovapd {{.*#+}} ymm8 = [9223372036854775807,9223372036854775807,9223372036854775807,9223372036854775807]
-; AVX1-NEXT:    vmovapd {{.*#+}} ymm11 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
-; AVX1-NEXT:    vblendvpd %ymm7, %ymm8, %ymm11, %ymm7
-; AVX1-NEXT:    vinsertf128 $1, %xmm9, %ymm0, %ymm0
-; AVX1-NEXT:    vblendvpd %ymm5, %ymm7, %ymm0, %ymm0
-; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm5
-; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm10, %xmm7
-; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm7
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm10, %xmm6
-; AVX1-NEXT:    vpxor %xmm4, %xmm6, %xmm6
-; AVX1-NEXT:    vpcmpeqq %xmm7, %xmm6, %xmm9
-; AVX1-NEXT:    vpcmpgtq %xmm3, %xmm10, %xmm7
-; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm12
-; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm10, %xmm7
-; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm7
-; AVX1-NEXT:    vpcmpeqq %xmm12, %xmm7, %xmm12
-; AVX1-NEXT:    vinsertf128 $1, %xmm9, %ymm12, %ymm9
-; AVX1-NEXT:    vpsubq %xmm5, %xmm2, %xmm12
-; AVX1-NEXT:    vpcmpgtq %xmm12, %xmm10, %xmm5
-; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm2
-; AVX1-NEXT:    vpcmpeqq %xmm2, %xmm6, %xmm2
-; AVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm2
-; AVX1-NEXT:    vpsubq %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm10, %xmm3
-; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm6
-; AVX1-NEXT:    vpcmpeqq %xmm6, %xmm7, %xmm6
-; AVX1-NEXT:    vpxor %xmm4, %xmm6, %xmm4
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm4, %ymm2
-; AVX1-NEXT:    vandnpd %ymm2, %ymm9, %ymm2
-; AVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm3, %ymm3
-; AVX1-NEXT:    vblendvpd %ymm3, %ymm8, %ymm11, %ymm3
-; AVX1-NEXT:    vinsertf128 $1, %xmm12, %ymm1, %ymm1
-; AVX1-NEXT:    vblendvpd %ymm2, %ymm3, %ymm1, %ymm1
+; AVX1-NEXT:    vpsubq %xmm4, %xmm7, %xmm4
+; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm7, %xmm7
+; AVX1-NEXT:    vpsubq %xmm2, %xmm0, %xmm2
+; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm7, %ymm0, %ymm0
+; AVX1-NEXT:    vxorpd %ymm0, %ymm6, %ymm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm2, %ymm2
+; AVX1-NEXT:    vmovapd {{.*#+}} ymm4 = [9223372036854775807,9223372036854775807,9223372036854775807,9223372036854775807]
+; AVX1-NEXT:    vmovapd {{.*#+}} ymm6 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
+; AVX1-NEXT:    vblendvpd %ymm2, %ymm4, %ymm6, %ymm7
+; AVX1-NEXT:    vblendvpd %ymm0, %ymm7, %ymm2, %ymm0
+; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm2
+; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm2, %xmm7
+; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm3, %xmm5
+; AVX1-NEXT:    vinsertf128 $1, %xmm7, %ymm5, %ymm5
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm7
+; AVX1-NEXT:    vpsubq %xmm2, %xmm7, %xmm2
+; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm7, %xmm7
+; AVX1-NEXT:    vpsubq %xmm3, %xmm1, %xmm3
+; AVX1-NEXT:    vpcmpgtq %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm7, %ymm1, %ymm1
+; AVX1-NEXT:    vxorpd %ymm1, %ymm5, %ymm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
+; AVX1-NEXT:    vblendvpd %ymm2, %ymm4, %ymm6, %ymm3
+; AVX1-NEXT:    vblendvpd %ymm1, %ymm3, %ymm2, %ymm1
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: v8i64:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpcmpgtq %ymm2, %ymm4, %ymm5
-; AVX2-NEXT:    vpcmpeqd %ymm6, %ymm6, %ymm6
-; AVX2-NEXT:    vpxor %ymm6, %ymm5, %ymm5
-; AVX2-NEXT:    vpcmpgtq %ymm0, %ymm4, %ymm7
-; AVX2-NEXT:    vpxor %ymm6, %ymm7, %ymm7
-; AVX2-NEXT:    vpcmpeqq %ymm5, %ymm7, %ymm5
-; AVX2-NEXT:    vpsubq %ymm2, %ymm0, %ymm0
-; AVX2-NEXT:    vpcmpgtq %ymm0, %ymm4, %ymm2
-; AVX2-NEXT:    vpxor %ymm6, %ymm2, %ymm8
-; AVX2-NEXT:    vpcmpeqq %ymm8, %ymm7, %ymm7
-; AVX2-NEXT:    vpxor %ymm6, %ymm7, %ymm7
-; AVX2-NEXT:    vpandn %ymm7, %ymm5, %ymm5
-; AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm7 = [9223372036854775807,9223372036854775807,9223372036854775807,9223372036854775807]
-; AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm8 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
-; AVX2-NEXT:    vblendvpd %ymm2, %ymm7, %ymm8, %ymm2
-; AVX2-NEXT:    vblendvpd %ymm5, %ymm2, %ymm0, %ymm0
-; AVX2-NEXT:    vpcmpgtq %ymm3, %ymm4, %ymm2
-; AVX2-NEXT:    vpxor %ymm6, %ymm2, %ymm2
-; AVX2-NEXT:    vpcmpgtq %ymm1, %ymm4, %ymm5
-; AVX2-NEXT:    vpxor %ymm6, %ymm5, %ymm5
-; AVX2-NEXT:    vpcmpeqq %ymm2, %ymm5, %ymm2
-; AVX2-NEXT:    vpsubq %ymm3, %ymm1, %ymm1
-; AVX2-NEXT:    vpcmpgtq %ymm1, %ymm4, %ymm3
-; AVX2-NEXT:    vpxor %ymm6, %ymm3, %ymm4
-; AVX2-NEXT:    vpcmpeqq %ymm4, %ymm5, %ymm4
-; AVX2-NEXT:    vpxor %ymm6, %ymm4, %ymm4
-; AVX2-NEXT:    vpandn %ymm4, %ymm2, %ymm2
-; AVX2-NEXT:    vblendvpd %ymm3, %ymm7, %ymm8, %ymm3
-; AVX2-NEXT:    vblendvpd %ymm2, %ymm3, %ymm1, %ymm1
+; AVX2-NEXT:    vpcmpgtq %ymm4, %ymm2, %ymm5
+; AVX2-NEXT:    vpsubq %ymm2, %ymm0, %ymm2
+; AVX2-NEXT:    vpcmpgtq %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %ymm0, %ymm5, %ymm0
+; AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm5 = [9223372036854775807,9223372036854775807,9223372036854775807,9223372036854775807]
+; AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm6 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
+; AVX2-NEXT:    vblendvpd %ymm2, %ymm5, %ymm6, %ymm7
+; AVX2-NEXT:    vblendvpd %ymm0, %ymm7, %ymm2, %ymm0
+; AVX2-NEXT:    vpcmpgtq %ymm4, %ymm3, %ymm2
+; AVX2-NEXT:    vpsubq %ymm3, %ymm1, %ymm3
+; AVX2-NEXT:    vpcmpgtq %ymm3, %ymm1, %ymm1
+; AVX2-NEXT:    vpxor %ymm1, %ymm2, %ymm1
+; AVX2-NEXT:    vblendvpd %ymm3, %ymm5, %ymm6, %ymm2
+; AVX2-NEXT:    vblendvpd %ymm1, %ymm2, %ymm3, %ymm1
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: v8i64:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpcmpnltq %zmm2, %zmm1, %k0
-; AVX512-NEXT:    vpcmpnltq %zmm2, %zmm0, %k1
-; AVX512-NEXT:    kxorw %k0, %k1, %k0
-; AVX512-NEXT:    vpsubq %zmm1, %zmm0, %zmm0
-; AVX512-NEXT:    vpcmpnltq %zmm2, %zmm0, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
-; AVX512-NEXT:    kandw %k1, %k0, %k1
-; AVX512-NEXT:    vpcmpgtq %zmm0, %zmm2, %k2
-; AVX512-NEXT:    vpbroadcastq {{.*#+}} zmm1 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
-; AVX512-NEXT:    vpbroadcastq {{.*}}(%rip), %zmm1 {%k2}
-; AVX512-NEXT:    vmovdqa64 %zmm1, %zmm0 {%k1}
+; AVX512-NEXT:    vpcmpgtq %zmm2, %zmm1, %k0
+; AVX512-NEXT:    vpsubq %zmm1, %zmm0, %zmm1
+; AVX512-NEXT:    vpcmpgtq %zmm1, %zmm0, %k1
+; AVX512-NEXT:    kxorw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpgtq %zmm1, %zmm2, %k2
+; AVX512-NEXT:    vpbroadcastq {{.*#+}} zmm0 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
+; AVX512-NEXT:    vpbroadcastq {{.*}}(%rip), %zmm0 {%k2}
+; AVX512-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
+; AVX512-NEXT:    vmovdqa64 %zmm1, %zmm0
 ; AVX512-NEXT:    retq
   %z = call <8 x i64> @llvm.ssub.sat.v8i64(<8 x i64> %x, <8 x i64> %y)
   ret <8 x i64> %z
diff --git a/llvm/test/CodeGen/X86/stack-align.ll b/llvm/test/CodeGen/X86/stack-align.ll
index 338ced0eb..5987828f6 100644
--- a/llvm/test/CodeGen/X86/stack-align.ll
+++ b/llvm/test/CodeGen/X86/stack-align.ll
@@ -88,4 +88,4 @@ declare void @llvm.memcpy.p0i8.p0i8.i32(i8* nocapture, i8* nocapture readonly, i
 
 declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture) argmemonly nounwind
 
-attributes #0 = { nounwind alignstack=16 "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" }
+attributes #0 = { nounwind alignstack=16 "frame-pointer"="all" }
diff --git a/llvm/test/CodeGen/X86/stack-folding-3dnow.ll b/llvm/test/CodeGen/X86/stack-folding-3dnow.ll
index 955bf4499..1cbd61567 100644
--- a/llvm/test/CodeGen/X86/stack-folding-3dnow.ll
+++ b/llvm/test/CodeGen/X86/stack-folding-3dnow.ll
@@ -1,8 +1,16 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+3dnow | FileCheck %s
 
 define x86_mmx @stack_fold_pavgusb(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_pavgusb
-  ;CHECK:       pavgusb {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_pavgusb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pavgusb {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.3dnow.pavgusb(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -10,8 +18,15 @@ define x86_mmx @stack_fold_pavgusb(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.3dnow.pavgusb(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_pf2id(x86_mmx %a) {
-  ;CHECK-LABEL: stack_fold_pf2id
-  ;CHECK:       pf2id {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_pf2id:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pf2id {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm1},~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.3dnow.pf2id(x86_mmx %a) nounwind readnone
   ret x86_mmx %2
@@ -19,8 +34,15 @@ define x86_mmx @stack_fold_pf2id(x86_mmx %a) {
 declare x86_mmx @llvm.x86.3dnow.pf2id(x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_pf2iw(x86_mmx %a) {
-  ;CHECK-LABEL: stack_fold_pf2iw
-  ;CHECK:       pf2iw {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_pf2iw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pf2iw {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm1},~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.3dnowa.pf2iw(x86_mmx %a) nounwind readnone
   ret x86_mmx %2
@@ -28,8 +50,15 @@ define x86_mmx @stack_fold_pf2iw(x86_mmx %a) {
 declare x86_mmx @llvm.x86.3dnowa.pf2iw(x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_pfacc(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_pfacc
-  ;CHECK:       pfacc {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_pfacc:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pfacc {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.3dnow.pfacc(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -37,8 +66,15 @@ define x86_mmx @stack_fold_pfacc(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.3dnow.pfacc(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_pfadd(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_pfadd
-  ;CHECK:       pfadd {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_pfadd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pfadd {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.3dnow.pfadd(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -46,8 +82,15 @@ define x86_mmx @stack_fold_pfadd(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.3dnow.pfadd(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_pfcmpeq(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_pfcmpeq
-  ;CHECK:       pfcmpeq {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_pfcmpeq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pfcmpeq {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.3dnow.pfcmpeq(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -55,8 +98,15 @@ define x86_mmx @stack_fold_pfcmpeq(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.3dnow.pfcmpeq(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_pfcmpge(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_pfcmpge
-  ;CHECK:       pfcmpge {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_pfcmpge:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pfcmpge {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.3dnow.pfcmpge(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -64,8 +114,15 @@ define x86_mmx @stack_fold_pfcmpge(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.3dnow.pfcmpge(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_pfcmpgt(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_pfcmpgt
-  ;CHECK:       pfcmpgt {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_pfcmpgt:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pfcmpgt {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.3dnow.pfcmpgt(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -73,8 +130,15 @@ define x86_mmx @stack_fold_pfcmpgt(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.3dnow.pfcmpgt(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_pfmax(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_pfmax
-  ;CHECK:       pfmax {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_pfmax:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pfmax {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.3dnow.pfmax(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -82,8 +146,15 @@ define x86_mmx @stack_fold_pfmax(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.3dnow.pfmax(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_pfmin(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_pfmin
-  ;CHECK:       pfmin {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_pfmin:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pfmin {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.3dnow.pfmin(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -91,8 +162,15 @@ define x86_mmx @stack_fold_pfmin(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.3dnow.pfmin(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_pfmul(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_pfmul
-  ;CHECK:       pfmul {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_pfmul:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pfmul {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.3dnow.pfmul(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -100,8 +178,15 @@ define x86_mmx @stack_fold_pfmul(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.3dnow.pfmul(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_pfnacc(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_pfnacc
-  ;CHECK:       pfnacc {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_pfnacc:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pfnacc {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.3dnowa.pfnacc(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -109,8 +194,15 @@ define x86_mmx @stack_fold_pfnacc(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.3dnowa.pfnacc(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_pfpnacc(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_pfpnacc
-  ;CHECK:       pfpnacc {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_pfpnacc:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pfpnacc {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.3dnowa.pfpnacc(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -118,8 +210,15 @@ define x86_mmx @stack_fold_pfpnacc(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.3dnowa.pfpnacc(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_pfrcp(x86_mmx %a) {
-  ;CHECK-LABEL: stack_fold_pfrcp
-  ;CHECK:       pfrcp {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_pfrcp:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pfrcp {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm1},~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.3dnow.pfrcp(x86_mmx %a) nounwind readnone
   ret x86_mmx %2
@@ -127,8 +226,15 @@ define x86_mmx @stack_fold_pfrcp(x86_mmx %a) {
 declare x86_mmx @llvm.x86.3dnow.pfrcp(x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_pfrcpit1(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_pfrcpit1
-  ;CHECK:       pfrcpit1 {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_pfrcpit1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pfrcpit1 {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.3dnow.pfrcpit1(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -136,8 +242,15 @@ define x86_mmx @stack_fold_pfrcpit1(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.3dnow.pfrcpit1(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_pfrcpit2(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_pfrcpit2
-  ;CHECK:       pfrcpit2 {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_pfrcpit2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pfrcpit2 {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.3dnow.pfrcpit2(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -145,8 +258,15 @@ define x86_mmx @stack_fold_pfrcpit2(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.3dnow.pfrcpit2(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_pfrsqit1(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_pfrsqit1
-  ;CHECK:       pfrsqit1 {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_pfrsqit1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pfrsqit1 {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.3dnow.pfrsqit1(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -154,8 +274,15 @@ define x86_mmx @stack_fold_pfrsqit1(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.3dnow.pfrsqit1(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_pfrsqrt(x86_mmx %a) {
-  ;CHECK-LABEL: stack_fold_pfrsqrt
-  ;CHECK:       pfrsqrt {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_pfrsqrt:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pfrsqrt {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm1},~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.3dnow.pfrsqrt(x86_mmx %a) nounwind readnone
   ret x86_mmx %2
@@ -163,8 +290,15 @@ define x86_mmx @stack_fold_pfrsqrt(x86_mmx %a) {
 declare x86_mmx @llvm.x86.3dnow.pfrsqrt(x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_pfsub(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_pfsub
-  ;CHECK:       pfsub {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_pfsub:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pfsub {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.3dnow.pfsub(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -172,8 +306,15 @@ define x86_mmx @stack_fold_pfsub(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.3dnow.pfsub(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_pfsubr(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_pfsubr
-  ;CHECK:       pfsubr {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_pfsubr:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pfsubr {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.3dnow.pfsubr(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -181,8 +322,15 @@ define x86_mmx @stack_fold_pfsubr(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.3dnow.pfsubr(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_pi2fd(x86_mmx %a) {
-  ;CHECK-LABEL: stack_fold_pi2fd
-  ;CHECK:       pi2fd {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_pi2fd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pi2fd {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm1},~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.3dnow.pi2fd(x86_mmx %a) nounwind readnone
   ret x86_mmx %2
@@ -190,8 +338,15 @@ define x86_mmx @stack_fold_pi2fd(x86_mmx %a) {
 declare x86_mmx @llvm.x86.3dnow.pi2fd(x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_pi2fw(x86_mmx %a) {
-  ;CHECK-LABEL: stack_fold_pi2fw
-  ;CHECK:       pi2fw {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_pi2fw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pi2fw {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm1},~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.3dnowa.pi2fw(x86_mmx %a) nounwind readnone
   ret x86_mmx %2
@@ -199,8 +354,15 @@ define x86_mmx @stack_fold_pi2fw(x86_mmx %a) {
 declare x86_mmx @llvm.x86.3dnowa.pi2fw(x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_pmulhrw(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_pmulhrw
-  ;CHECK:       pmulhrw {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmulhrw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pmulhrw {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.3dnow.pmulhrw(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -208,8 +370,16 @@ define x86_mmx @stack_fold_pmulhrw(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.3dnow.pmulhrw(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_pswapd(x86_mmx %a) {
-  ;CHECK-LABEL: stack_fold_pswapd
-  ;CHECK:       pswapd {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_pswapd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pswapd {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    # mm0 = mem[1,0]
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm1},~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.3dnowa.pswapd(x86_mmx %a) nounwind readnone
   ret x86_mmx %2
diff --git a/llvm/test/CodeGen/X86/stack-folding-adx-x86_64.ll b/llvm/test/CodeGen/X86/stack-folding-adx-x86_64.ll
index fa00b7cd7..801169750 100644
--- a/llvm/test/CodeGen/X86/stack-folding-adx-x86_64.ll
+++ b/llvm/test/CodeGen/X86/stack-folding-adx-x86_64.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -O3 -disable-peephole -mtriple=x86_64-unknown-unknown -mattr=+adx < %s | FileCheck %s --check-prefix=CHECK --check-prefix=ADX
 ; RUN: llc -O3 -disable-peephole -mtriple=x86_64-unknown-unknown -mattr=-adx < %s | FileCheck %s --check-prefix=CHECK --check-prefix=NOADX
 
@@ -10,8 +11,53 @@ target triple = "x86_64-unknown-unknown"
 ; relevant registers and check that the reload is correctly folded into the instruction.
 
 define i8 @stack_fold_addcarry_u32(i8 %a0, i32 %a1, i32 %a2, i8* %a3) {
-  ;CHECK-LABEL: stack_fold_addcarry_u32
-  ;CHECK:       adcl {{-?[0-9]*}}(%rsp), %{{.*}} {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_addcarry_u32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    movl %edx, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    movl %esi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
+; CHECK-NEXT:    addb $-1, %al
+; CHECK-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %edx # 4-byte Reload
+; CHECK-NEXT:    adcl {{[-0-9]+}}(%r{{[sb]}}p), %edx # 4-byte Folded Reload
+; CHECK-NEXT:    setb %al
+; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
+; CHECK-NEXT:    movl %edx, (%rcx)
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call i64 asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = call { i8, i32 } @llvm.x86.addcarry.32(i8 %a0, i32 %a1, i32 %a2)
   %3 = extractvalue { i8, i32 } %2, 1
@@ -22,8 +68,53 @@ define i8 @stack_fold_addcarry_u32(i8 %a0, i32 %a1, i32 %a2, i8* %a3) {
 }
 
 define i8 @stack_fold_addcarry_u64(i8 %a0, i64 %a1, i64 %a2, i8* %a3) {
-  ;CHECK-LABEL: stack_fold_addcarry_u64
-  ;CHECK:       adcq {{-?[0-9]*}}(%rsp), %{{.*}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_addcarry_u64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
+; CHECK-NEXT:    addb $-1, %al
+; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
+; CHECK-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Folded Reload
+; CHECK-NEXT:    setb %al
+; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
+; CHECK-NEXT:    movq %rdx, (%rcx)
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call i64 asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = call { i8, i64 } @llvm.x86.addcarry.64(i8 %a0, i64 %a1, i64 %a2)
   %3 = extractvalue { i8, i64 } %2, 1
@@ -34,8 +125,53 @@ define i8 @stack_fold_addcarry_u64(i8 %a0, i64 %a1, i64 %a2, i8* %a3) {
 }
 
 define i8 @stack_fold_addcarryx_u32(i8 %a0, i32 %a1, i32 %a2, i8* %a3) {
-  ;CHECK-LABEL: stack_fold_addcarryx_u32
-  ;CHECK:       adcl {{-?[0-9]*}}(%rsp), %{{.*}} {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_addcarryx_u32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    movl %edx, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    movl %esi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
+; CHECK-NEXT:    addb $-1, %al
+; CHECK-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %edx # 4-byte Reload
+; CHECK-NEXT:    adcl {{[-0-9]+}}(%r{{[sb]}}p), %edx # 4-byte Folded Reload
+; CHECK-NEXT:    setb %al
+; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
+; CHECK-NEXT:    movl %edx, (%rcx)
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call i64 asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = call { i8, i32 } @llvm.x86.addcarry.32(i8 %a0, i32 %a1, i32 %a2)
   %3 = extractvalue { i8, i32 } %2, 1
@@ -46,8 +182,53 @@ define i8 @stack_fold_addcarryx_u32(i8 %a0, i32 %a1, i32 %a2, i8* %a3) {
 }
 
 define i8 @stack_fold_addcarryx_u64(i8 %a0, i64 %a1, i64 %a2, i8* %a3) {
-  ;CHECK-LABEL: stack_fold_addcarryx_u64
-  ;CHECK:       adcq {{-?[0-9]*}}(%rsp), %{{.*}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_addcarryx_u64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
+; CHECK-NEXT:    addb $-1, %al
+; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
+; CHECK-NEXT:    adcq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Folded Reload
+; CHECK-NEXT:    setb %al
+; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
+; CHECK-NEXT:    movq %rdx, (%rcx)
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call i64 asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = call { i8, i64 } @llvm.x86.addcarry.64(i8 %a0, i64 %a1, i64 %a2)
   %3 = extractvalue { i8, i64 } %2, 1
@@ -58,8 +239,53 @@ define i8 @stack_fold_addcarryx_u64(i8 %a0, i64 %a1, i64 %a2, i8* %a3) {
 }
 
 define i8 @stack_fold_subborrow_u32(i8 %a0, i32 %a1, i32 %a2, i8* %a3) {
-  ;CHECK-LABEL: stack_fold_subborrow_u32
-  ;CHECK:       sbbl {{-?[0-9]*}}(%rsp), %{{.*}} {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_subborrow_u32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    movl %edx, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    movl %esi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
+; CHECK-NEXT:    addb $-1, %al
+; CHECK-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %edx # 4-byte Reload
+; CHECK-NEXT:    sbbl {{[-0-9]+}}(%r{{[sb]}}p), %edx # 4-byte Folded Reload
+; CHECK-NEXT:    setb %al
+; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
+; CHECK-NEXT:    movl %edx, (%rcx)
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call i64 asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = call { i8, i32 } @llvm.x86.subborrow.32(i8 %a0, i32 %a1, i32 %a2)
   %3 = extractvalue { i8, i32 } %2, 1
@@ -70,8 +296,53 @@ define i8 @stack_fold_subborrow_u32(i8 %a0, i32 %a1, i32 %a2, i8* %a3) {
 }
 
 define i8 @stack_fold_subborrow_u64(i8 %a0, i64 %a1, i64 %a2, i8* %a3) {
-  ;CHECK-LABEL: stack_fold_subborrow_u64
-  ;CHECK:       sbbq {{-?[0-9]*}}(%rsp), %{{.*}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_subborrow_u64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
+; CHECK-NEXT:    addb $-1, %al
+; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
+; CHECK-NEXT:    sbbq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Folded Reload
+; CHECK-NEXT:    setb %al
+; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
+; CHECK-NEXT:    movq %rdx, (%rcx)
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call i64 asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = call { i8, i64 } @llvm.x86.subborrow.64(i8 %a0, i64 %a1, i64 %a2)
   %3 = extractvalue { i8, i64 } %2, 1
diff --git a/llvm/test/CodeGen/X86/stack-folding-avx512bf16.ll b/llvm/test/CodeGen/X86/stack-folding-avx512bf16.ll
index 16c0c5be0..e368b0879 100644
--- a/llvm/test/CodeGen/X86/stack-folding-avx512bf16.ll
+++ b/llvm/test/CodeGen/X86/stack-folding-avx512bf16.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -O3 -disable-peephole -mtriple=x86_64-unknown-unknown -mattr=+avx512bf16,+avx512vl < %s | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -9,8 +10,14 @@ target triple = "x86_64-unknown-unknown"
 ; relevant registers and check that the reload is correctly folded into the instruction.
 
 define <32 x i16> @stack_fold_cvtne2ps2bf16(<16 x float> %a0, <16 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_cvtne2ps2bf16:
-  ;CHECK:       vcvtne2ps2bf16 {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtne2ps2bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcvtne2ps2bf16 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <32 x i16> @llvm.x86.avx512bf16.cvtne2ps2bf16.512(<16 x float> %a0, <16 x float> %a1)
   ret <32 x i16> %2
@@ -19,7 +26,16 @@ declare <32 x i16> @llvm.x86.avx512bf16.cvtne2ps2bf16.512(<16 x float>, <16 x fl
 
 define <32 x i16> @stack_fold_cvtne2ps2bf16_mask(<16 x float> %a0, <16 x float> %a1, <32 x i16>* %passthru, i32 %U) {
 ; CHECK-LABEL: stack_fold_cvtne2ps2bf16_mask:
-; CHECK:       vcvtne2ps2bf16 {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm2
+; CHECK-NEXT:    vcvtne2ps2bf16 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <32 x i16> @llvm.x86.avx512bf16.cvtne2ps2bf16.512(<16 x float> %a0, <16 x float> %a1)
   %3 = bitcast i32 %U to <32 x i1>
@@ -31,7 +47,14 @@ define <32 x i16> @stack_fold_cvtne2ps2bf16_mask(<16 x float> %a0, <16 x float>
 
 define <32 x i16> @stack_fold_cvtne2ps2bf16_maskz(<16 x float> %a0, <16 x float> %a1, i32 %U) {
 ; CHECK-LABEL: stack_fold_cvtne2ps2bf16_maskz:
-; CHECK:       vcvtne2ps2bf16 {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vcvtne2ps2bf16 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <32 x i16> @llvm.x86.avx512bf16.cvtne2ps2bf16.512(<16 x float> %a0, <16 x float> %a1)
   %3 = bitcast i32 %U to <32 x i1>
@@ -41,7 +64,13 @@ define <32 x i16> @stack_fold_cvtne2ps2bf16_maskz(<16 x float> %a0, <16 x float>
 
 define <16 x i16> @stack_fold_cvtneps2bf16(<16 x float> %a0) {
 ; CHECK-LABEL: stack_fold_cvtneps2bf16:
-; CHECK:       vcvtneps2bf16 {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcvtneps2bf16 {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = tail call <16 x i16> @llvm.x86.avx512bf16.cvtneps2bf16.512(<16 x float> %a0)
   ret <16 x i16> %2
@@ -50,7 +79,16 @@ declare <16 x i16> @llvm.x86.avx512bf16.cvtneps2bf16.512(<16 x float>)
 
 define <16 x i16> @stack_fold_cvtneps2bf16_mask(<16 x float> %a0, <16 x i16>* %passthru, i16 %U) {
 ; CHECK-LABEL: stack_fold_cvtneps2bf16_mask:
-; CHECK:       vcvtneps2bf16 {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vmovaps (%rdi), %ymm1
+; CHECK-NEXT:    vcvtneps2bf16 {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovaps %ymm1, %ymm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = tail call <16 x i16> @llvm.x86.avx512bf16.cvtneps2bf16.512(<16 x float> %a0)
   %3 = bitcast i16 %U to <16 x i1>
@@ -62,7 +100,14 @@ define <16 x i16> @stack_fold_cvtneps2bf16_mask(<16 x float> %a0, <16 x i16>* %p
 
 define <16 x i16> @stack_fold_cvtneps2bf16_maskz(<16 x float> %a0, i16 %U) {
 ; CHECK-LABEL: stack_fold_cvtneps2bf16_maskz:
-; CHECK:       vcvtneps2bf16 {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vcvtneps2bf16 {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = tail call <16 x i16> @llvm.x86.avx512bf16.cvtneps2bf16.512(<16 x float> %a0)
   %3 = bitcast i16 %U to <16 x i1>
@@ -72,7 +117,13 @@ define <16 x i16> @stack_fold_cvtneps2bf16_maskz(<16 x float> %a0, i16 %U) {
 
 define <16 x float> @stack_fold_vdpbf16ps(<16 x float> %a0, <16 x i32> %a1, <16 x i32> %a2) {
 ; CHECK-LABEL: stack_fold_vdpbf16ps:
-; CHECK:       vdpbf16ps {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vdpbf16ps {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = tail call <16 x float> @llvm.x86.avx512bf16.dpbf16ps.512(<16 x float> %a0, <16 x i32> %a1, <16 x i32> %a2)
   ret <16 x float> %2
@@ -81,7 +132,16 @@ declare <16 x float> @llvm.x86.avx512bf16.dpbf16ps.512(<16 x float>, <16 x i32>,
 
 define <16 x float> @stack_fold_vdpbf16ps_mask(<16 x float>* %a0, <16 x i32> %a1, <16 x i32> %a2, <16 x float>* %passthru, i16 %U) {
 ; CHECK-LABEL: stack_fold_vdpbf16ps_mask:
-; CHECK:       vdpbf16ps {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovaps (%rdi), %zmm2
+; CHECK-NEXT:    kmovd %edx, %k1
+; CHECK-NEXT:    vdpbf16ps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovaps %zmm2, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   ; load needed to keep the operation from being scheduled above the asm block
   %2 = load <16 x float>, <16 x float>* %a0
@@ -93,7 +153,14 @@ define <16 x float> @stack_fold_vdpbf16ps_mask(<16 x float>* %a0, <16 x i32> %a1
 
 define <16 x float> @stack_fold_vdpbf16ps_maskz(<16 x float> %a0, <16 x i32> %a1, <16 x i32> %a2, i16* %U) {
 ; CHECK-LABEL: stack_fold_vdpbf16ps_maskz:
-; CHECK:       vdpbf16ps {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovw (%rdi), %k1
+; CHECK-NEXT:    vdpbf16ps {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = tail call <16 x float> @llvm.x86.avx512bf16.dpbf16ps.512(<16 x float> %a0, <16 x i32> %a1, <16 x i32> %a2)
   %3 = load i16, i16* %U
@@ -105,8 +172,14 @@ define <16 x float> @stack_fold_vdpbf16ps_maskz(<16 x float> %a0, <16 x i32> %a1
 
 
 define <16 x i16> @stack_fold_cvtne2ps2bf16_ymm(<8 x float> %a0, <8 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_cvtne2ps2bf16_ymm:
-  ;CHECK:       vcvtne2ps2bf16 {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtne2ps2bf16_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcvtne2ps2bf16 {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x i16> @llvm.x86.avx512bf16.cvtne2ps2bf16.256(<8 x float> %a0, <8 x float> %a1)
   ret <16 x i16> %2
@@ -115,7 +188,16 @@ declare <16 x i16> @llvm.x86.avx512bf16.cvtne2ps2bf16.256(<8 x float>, <8 x floa
 
 define <16 x i16> @stack_fold_cvtne2ps2bf16_mask_ymm(<8 x float> %a0, <8 x float> %a1, <16 x i16>* %passthru, i16 %U) {
 ; CHECK-LABEL: stack_fold_cvtne2ps2bf16_mask_ymm:
-; CHECK:       vcvtne2ps2bf16 {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 32-byte Folded Reload
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vmovdqa (%rdi), %ymm2
+; CHECK-NEXT:    vcvtne2ps2bf16 {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm2 {%k1} # 32-byte Folded Reload
+; CHECK-NEXT:    vmovdqa %ymm2, %ymm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x i16> @llvm.x86.avx512bf16.cvtne2ps2bf16.256(<8 x float> %a0, <8 x float> %a1)
   %3 = bitcast i16 %U to <16 x i1>
@@ -127,7 +209,14 @@ define <16 x i16> @stack_fold_cvtne2ps2bf16_mask_ymm(<8 x float> %a0, <8 x float
 
 define <16 x i16> @stack_fold_cvtne2ps2bf16_maskz_ymm(<8 x float> %a0, <8 x float> %a1, i16 %U) {
 ; CHECK-LABEL: stack_fold_cvtne2ps2bf16_maskz_ymm:
-; CHECK:       vcvtne2ps2bf16 {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 32-byte Folded Reload
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vcvtne2ps2bf16 {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 {%k1} {z} # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x i16> @llvm.x86.avx512bf16.cvtne2ps2bf16.256(<8 x float> %a0, <8 x float> %a1)
   %3 = bitcast i16 %U to <16 x i1>
@@ -137,7 +226,14 @@ define <16 x i16> @stack_fold_cvtne2ps2bf16_maskz_ymm(<8 x float> %a0, <8 x floa
 
 define <8 x i16> @stack_fold_cvtneps2bf16_ymm(<8 x float> %a0) {
 ; CHECK-LABEL: stack_fold_cvtneps2bf16_ymm:
-; CHECK:       vcvtneps2bf16y {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcvtneps2bf16y {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 32-byte Folded Reload
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = tail call <8 x i16> @llvm.x86.avx512bf16.cvtneps2bf16.256(<8 x float> %a0)
   ret <8 x i16> %2
@@ -146,7 +242,17 @@ declare <8 x i16> @llvm.x86.avx512bf16.cvtneps2bf16.256(<8 x float>)
 
 define <8 x i16> @stack_fold_cvtneps2bf16_mask_ymm(<8 x float> %a0, <8 x i16>* %passthru, i8 %U) {
 ; CHECK-LABEL: stack_fold_cvtneps2bf16_mask_ymm:
-; CHECK:       vcvtneps2bf16y {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 32-byte Folded Reload
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vmovaps (%rdi), %xmm1
+; CHECK-NEXT:    vcvtneps2bf16y {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 {%k1} # 32-byte Folded Reload
+; CHECK-NEXT:    vmovaps %xmm1, %xmm0
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = tail call <8 x i16> @llvm.x86.avx512bf16.cvtneps2bf16.256(<8 x float> %a0)
   %3 = bitcast i8 %U to <8 x i1>
@@ -158,7 +264,15 @@ define <8 x i16> @stack_fold_cvtneps2bf16_mask_ymm(<8 x float> %a0, <8 x i16>* %
 
 define <8 x i16> @stack_fold_cvtneps2bf16_maskz_ymm(<8 x float> %a0, i8 %U) {
 ; CHECK-LABEL: stack_fold_cvtneps2bf16_maskz_ymm:
-; CHECK:       vcvtneps2bf16y {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 32-byte Folded Reload
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vcvtneps2bf16y {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 {%k1} {z} # 32-byte Folded Reload
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = tail call <8 x i16> @llvm.x86.avx512bf16.cvtneps2bf16.256(<8 x float> %a0)
   %3 = bitcast i8 %U to <8 x i1>
@@ -168,7 +282,13 @@ define <8 x i16> @stack_fold_cvtneps2bf16_maskz_ymm(<8 x float> %a0, i8 %U) {
 
 define <8 x float> @stack_fold_vdpbf16ps_ymm(<8 x float> %a0, <8 x i32> %a1, <8 x i32> %a2) {
 ; CHECK-LABEL: stack_fold_vdpbf16ps_ymm:
-; CHECK:       vdpbf16ps {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vdpbf16ps {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = tail call <8 x float> @llvm.x86.avx512bf16.dpbf16ps.256(<8 x float> %a0, <8 x i32> %a1, <8 x i32> %a2)
   ret <8 x float> %2
@@ -177,7 +297,16 @@ declare <8 x float> @llvm.x86.avx512bf16.dpbf16ps.256(<8 x float>, <8 x i32>, <8
 
 define <8 x float> @stack_fold_vdpbf16ps_mask_ymm(<8 x float>* %a0, <8 x i32> %a1, <8 x i32> %a2, <8 x float>* %passthru, i8 %U) {
 ; CHECK-LABEL: stack_fold_vdpbf16ps_mask_ymm:
-; CHECK:       vdpbf16ps {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 32-byte Folded Reload
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovaps (%rdi), %ymm2
+; CHECK-NEXT:    kmovd %edx, %k1
+; CHECK-NEXT:    vdpbf16ps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm2 {%k1} # 32-byte Folded Reload
+; CHECK-NEXT:    vmovaps %ymm2, %ymm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   ; load needed to keep the operation from being scheduled above the asm block
   %2 = load <8 x float>, <8 x float>* %a0
@@ -189,7 +318,15 @@ define <8 x float> @stack_fold_vdpbf16ps_mask_ymm(<8 x float>* %a0, <8 x i32> %a
 
 define <8 x float> @stack_fold_vdpbf16ps_maskz_ymm(<8 x float> %a0, <8 x i32> %a1, <8 x i32> %a2, i8* %U) {
 ; CHECK-LABEL: stack_fold_vdpbf16ps_maskz_ymm:
-; CHECK:       vdpbf16ps {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 32-byte Folded Reload
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    movzbl (%rdi), %eax
+; CHECK-NEXT:    kmovd %eax, %k1
+; CHECK-NEXT:    vdpbf16ps {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm0 {%k1} {z} # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = tail call <8 x float> @llvm.x86.avx512bf16.dpbf16ps.256(<8 x float> %a0, <8 x i32> %a1, <8 x i32> %a2)
   %3 = load i8, i8* %U
@@ -202,8 +339,14 @@ define <8 x float> @stack_fold_vdpbf16ps_maskz_ymm(<8 x float> %a0, <8 x i32> %a
 
 
 define <8 x i16> @stack_fold_cvtne2ps2bf16_xmm(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_cvtne2ps2bf16_xmm:
-  ;CHECK:       vcvtne2ps2bf16 {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtne2ps2bf16_xmm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcvtne2ps2bf16 {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x i16> @llvm.x86.avx512bf16.cvtne2ps2bf16.128(<4 x float> %a0, <4 x float> %a1)
   ret <8 x i16> %2
@@ -212,7 +355,16 @@ declare <8 x i16> @llvm.x86.avx512bf16.cvtne2ps2bf16.128(<4 x float>, <4 x float
 
 define <8 x i16> @stack_fold_cvtne2ps2bf16_mask_xmm(<4 x float> %a0, <4 x float> %a1, <8 x i16>* %passthru, i8 %U) {
 ; CHECK-LABEL: stack_fold_cvtne2ps2bf16_mask_xmm:
-; CHECK:       vcvtne2ps2bf16 {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 16-byte Folded Reload
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vmovdqa (%rdi), %xmm2
+; CHECK-NEXT:    vcvtne2ps2bf16 {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm2 {%k1} # 16-byte Folded Reload
+; CHECK-NEXT:    vmovdqa %xmm2, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x i16> @llvm.x86.avx512bf16.cvtne2ps2bf16.128(<4 x float> %a0, <4 x float> %a1)
   %3 = bitcast i8 %U to <8 x i1>
@@ -224,7 +376,14 @@ define <8 x i16> @stack_fold_cvtne2ps2bf16_mask_xmm(<4 x float> %a0, <4 x float>
 
 define <8 x i16> @stack_fold_cvtne2ps2bf16_maskz_xmm(<4 x float> %a0, <4 x float> %a1, i8 %U) {
 ; CHECK-LABEL: stack_fold_cvtne2ps2bf16_maskz_xmm:
-; CHECK:       vcvtne2ps2bf16 {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 16-byte Folded Reload
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vcvtne2ps2bf16 {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 {%k1} {z} # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x i16> @llvm.x86.avx512bf16.cvtne2ps2bf16.128(<4 x float> %a0, <4 x float> %a1)
   %3 = bitcast i8 %U to <8 x i1>
@@ -234,7 +393,13 @@ define <8 x i16> @stack_fold_cvtne2ps2bf16_maskz_xmm(<4 x float> %a0, <4 x float
 
 define <8 x i16> @stack_fold_cvtneps2bf16_xmm(<4 x float> %a0) {
 ; CHECK-LABEL: stack_fold_cvtneps2bf16_xmm:
-; CHECK:       vcvtneps2bf16x {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcvtneps2bf16x {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = tail call <8 x i16> @llvm.x86.avx512bf16.mask.cvtneps2bf16.128(<4 x float> %a0, <8 x i16> undef, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
   ret <8 x i16> %2
@@ -243,7 +408,16 @@ declare <8 x i16> @llvm.x86.avx512bf16.mask.cvtneps2bf16.128(<4 x float>, <8 x i
 
 define <8 x i16> @stack_fold_cvtneps2bf16_mask_xmm(<4 x float> %a0, <8 x i16>* %passthru, i8 %U) {
 ; CHECK-LABEL: stack_fold_cvtneps2bf16_mask_xmm:
-; CHECK:       vcvtneps2bf16x {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 16-byte Folded Reload
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovaps (%rdi), %xmm1
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vcvtneps2bf16x {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 {%k1} # 16-byte Folded Reload
+; CHECK-NEXT:    vmovaps %xmm1, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = load <8 x i16>, <8 x i16>* %passthru
   %3 = bitcast i8 %U to <8 x i1>
@@ -254,7 +428,14 @@ define <8 x i16> @stack_fold_cvtneps2bf16_mask_xmm(<4 x float> %a0, <8 x i16>* %
 
 define <8 x i16> @stack_fold_cvtneps2bf16_maskz_xmm(<4 x float> %a0, i8 %U) {
 ; CHECK-LABEL: stack_fold_cvtneps2bf16_maskz_xmm:
-; CHECK:       vcvtneps2bf16x {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 16-byte Folded Reload
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vcvtneps2bf16x {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 {%k1} {z} # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = bitcast i8 %U to <8 x i1>
   %3 = shufflevector <8 x i1> %2, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -264,7 +445,13 @@ define <8 x i16> @stack_fold_cvtneps2bf16_maskz_xmm(<4 x float> %a0, i8 %U) {
 
 define <4 x float> @stack_fold_vdpbf16ps_xmm(<4 x float> %a0, <4 x i32> %a1, <4 x i32> %a2) {
 ; CHECK-LABEL: stack_fold_vdpbf16ps_xmm:
-; CHECK:       vdpbf16ps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vdpbf16ps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = tail call <4 x float> @llvm.x86.avx512bf16.dpbf16ps.128(<4 x float> %a0, <4 x i32> %a1, <4 x i32> %a2)
   ret <4 x float> %2
@@ -273,7 +460,16 @@ declare <4 x float> @llvm.x86.avx512bf16.dpbf16ps.128(<4 x float>, <4 x i32>, <4
 
 define <4 x float> @stack_fold_vdpbf16ps_mask_xmm(<4 x float>* %a0, <4 x i32> %a1, <4 x i32> %a2, <4 x float>* %passthru, i8 %U) {
 ; CHECK-LABEL: stack_fold_vdpbf16ps_mask_xmm:
-; CHECK:       vdpbf16ps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 16-byte Folded Reload
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovaps (%rdi), %xmm2
+; CHECK-NEXT:    kmovd %edx, %k1
+; CHECK-NEXT:    vdpbf16ps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm2 {%k1} # 16-byte Folded Reload
+; CHECK-NEXT:    vmovaps %xmm2, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   ; load needed to keep the operation from being scheduled above the asm block
   %2 = load <4 x float>, <4 x float>* %a0
@@ -286,7 +482,15 @@ define <4 x float> @stack_fold_vdpbf16ps_mask_xmm(<4 x float>* %a0, <4 x i32> %a
 
 define <4 x float> @stack_fold_vdpbf16ps_maskz_xmm(<4 x float> %a0, <4 x i32> %a1, <4 x i32> %a2, i8* %U) {
 ; CHECK-LABEL: stack_fold_vdpbf16ps_maskz_xmm:
-; CHECK:       vdpbf16ps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 16-byte Folded Reload
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    movzbl (%rdi), %eax
+; CHECK-NEXT:    kmovd %eax, %k1
+; CHECK-NEXT:    vdpbf16ps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} {z} # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = tail call <4 x float> @llvm.x86.avx512bf16.dpbf16ps.128(<4 x float> %a0, <4 x i32> %a1, <4 x i32> %a2)
   %3 = load i8, i8* %U
diff --git a/llvm/test/CodeGen/X86/stack-folding-avx512vp2intersect.ll b/llvm/test/CodeGen/X86/stack-folding-avx512vp2intersect.ll
index 1d60504b5..839dd1c70 100644
--- a/llvm/test/CodeGen/X86/stack-folding-avx512vp2intersect.ll
+++ b/llvm/test/CodeGen/X86/stack-folding-avx512vp2intersect.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -O3 -disable-peephole -mtriple=x86_64-unknown-unknown -mattr=+avx512vp2intersect,+avx512vl < %s | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -5,7 +6,17 @@ target triple = "x86_64-unknown-unknown"
 
 define void @stack_fold_vp2intersectd(<16 x i32>* %a, <16 x i32> %b, <16 x i1>* nocapture %m0, <16 x i1>* nocapture %m1) {
 ; CHECK-LABEL: stack_fold_vp2intersectd:
-; CHECK:    vp2intersectd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%k[0-7]}} {{.*#+}} 64-byte Folded Reload
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovaps (%rdi), %zmm0
+; CHECK-NEXT:    vp2intersectd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %k0 # 64-byte Folded Reload
+; CHECK-NEXT:    kmovw %k0, (%rsi)
+; CHECK-NEXT:    kmovw %k1, (%rdx)
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = load <16 x i32>, <16 x i32>* %a
   %3 = tail call { <16 x i1>, <16 x i1> } @llvm.x86.avx512.vp2intersect.d.512(<16 x i32> %2, <16 x i32> %b)
@@ -19,7 +30,19 @@ declare { <16 x i1>, <16 x i1> } @llvm.x86.avx512.vp2intersect.d.512(<16 x i32>,
 
 define void @stack_fold_vp2intersectq(<8 x i64>* %a, <8 x i64> %b, <8 x i1>* nocapture %m0, <8 x i1>* nocapture %m1) {
 ; CHECK-LABEL: stack_fold_vp2intersectq:
-; CHECK:    vp2intersectq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%k[0-7]}} {{.*#+}} 64-byte Folded Reload
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovaps (%rdi), %zmm0
+; CHECK-NEXT:    vp2intersectq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %k0 # 64-byte Folded Reload
+; CHECK-NEXT:    kmovw %k1, %eax
+; CHECK-NEXT:    kmovw %k0, %ecx
+; CHECK-NEXT:    movb %cl, (%rsi)
+; CHECK-NEXT:    movb %al, (%rdx)
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = load <8 x i64>, <8 x i64>* %a
   %3 = tail call { <8 x i1>, <8 x i1> } @llvm.x86.avx512.vp2intersect.q.512(<8 x i64> %2, <8 x i64> %b)
@@ -33,7 +56,19 @@ declare { <8 x i1>, <8 x i1> } @llvm.x86.avx512.vp2intersect.q.512(<8 x i64>, <8
 
 define void @stack_fold_vp2intersectd_256(<8 x i32>* %a, <8 x i32> %b, <8 x i1>* nocapture %m0, <8 x i1>* nocapture %m1) {
 ; CHECK-LABEL: stack_fold_vp2intersectd_256:
-; CHECK:    vp2intersectd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%k[0-7]}} {{.*#+}} 32-byte Folded Reload
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovaps (%rdi), %ymm0
+; CHECK-NEXT:    vp2intersectd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %k0 # 32-byte Folded Reload
+; CHECK-NEXT:    kmovw %k1, %eax
+; CHECK-NEXT:    kmovw %k0, %ecx
+; CHECK-NEXT:    movb %cl, (%rsi)
+; CHECK-NEXT:    movb %al, (%rdx)
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = load <8 x i32>, <8 x i32>* %a
   %3 = tail call { <8 x i1>, <8 x i1> } @llvm.x86.avx512.vp2intersect.d.256(<8 x i32> %2, <8 x i32> %b)
@@ -47,7 +82,19 @@ declare { <8 x i1>, <8 x i1> } @llvm.x86.avx512.vp2intersect.d.256(<8 x i32>, <8
 
 define void @stack_fold_vp2intersectq_256(<4 x i64>* %a, <4 x i64> %b, <4 x i1>* nocapture %m0, <4 x i1>* nocapture %m1) {
 ; CHECK-LABEL: stack_fold_vp2intersectq_256:
-; CHECK:    vp2intersectq {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%k[0-7]}} {{.*#+}} 32-byte Folded Reload
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovaps (%rdi), %ymm0
+; CHECK-NEXT:    vp2intersectq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %k0 # 32-byte Folded Reload
+; CHECK-NEXT:    kmovw %k1, %eax
+; CHECK-NEXT:    kmovw %k0, %ecx
+; CHECK-NEXT:    movb %cl, (%rsi)
+; CHECK-NEXT:    movb %al, (%rdx)
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = load <4 x i64>, <4 x i64>* %a
   %3 = tail call { <4 x i1>, <4 x i1> } @llvm.x86.avx512.vp2intersect.q.256(<4 x i64> %2, <4 x i64> %b)
@@ -61,7 +108,18 @@ declare { <4 x i1>, <4 x i1> } @llvm.x86.avx512.vp2intersect.q.256(<4 x i64>, <4
 
 define void @stack_fold_vp2intersectd_128(<4 x i32>* %a, <4 x i32> %b, <4 x i1>* nocapture %m0, <4 x i1>* nocapture %m1) {
 ; CHECK-LABEL: stack_fold_vp2intersectd_128:
-; CHECK:    vp2intersectd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%k[0-7]}} {{.*#+}} 16-byte Folded Reload
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovaps (%rdi), %xmm0
+; CHECK-NEXT:    vp2intersectd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %k0 # 16-byte Folded Reload
+; CHECK-NEXT:    kmovw %k1, %eax
+; CHECK-NEXT:    kmovw %k0, %ecx
+; CHECK-NEXT:    movb %cl, (%rsi)
+; CHECK-NEXT:    movb %al, (%rdx)
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = load <4 x i32>, <4 x i32>* %a
   %3 = tail call { <4 x i1>, <4 x i1> } @llvm.x86.avx512.vp2intersect.d.128(<4 x i32> %2, <4 x i32> %b)
@@ -75,7 +133,18 @@ declare { <4 x i1>, <4 x i1> } @llvm.x86.avx512.vp2intersect.d.128(<4 x i32>, <4
 
 define void @stack_fold_vp2intersectq_128(<2 x i64>* %a, <2 x i64> %b, <2 x i1>* nocapture %m0, <2 x i1>* nocapture %m1) {
 ; CHECK-LABEL: stack_fold_vp2intersectq_128:
-; CHECK:    vp2intersectq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%k[0-7]}} {{.*#+}} 16-byte Folded Reload
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovaps (%rdi), %xmm0
+; CHECK-NEXT:    vp2intersectq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %k0 # 16-byte Folded Reload
+; CHECK-NEXT:    kmovw %k1, %eax
+; CHECK-NEXT:    kmovw %k0, %ecx
+; CHECK-NEXT:    movb %cl, (%rsi)
+; CHECK-NEXT:    movb %al, (%rdx)
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = load <2 x i64>, <2 x i64>* %a
   %3 = tail call { <2 x i1>, <2 x i1> } @llvm.x86.avx512.vp2intersect.q.128(<2 x i64> %2, <2 x i64> %b)
diff --git a/llvm/test/CodeGen/X86/stack-folding-bmi.ll b/llvm/test/CodeGen/X86/stack-folding-bmi.ll
index 0bc6ef8f9..569482f16 100644
--- a/llvm/test/CodeGen/X86/stack-folding-bmi.ll
+++ b/llvm/test/CodeGen/X86/stack-folding-bmi.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -O3 -disable-peephole -mtriple=x86_64-unknown-unknown -mattr=+bmi < %s | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -9,8 +10,46 @@ target triple = "x86_64-unknown-unknown"
 ; relevant registers and check that the reload is correctly folded into the instruction.
 
 define i32 @stack_fold_andn_u32(i32 %a0, i32 %a1) {
-  ;CHECK-LABEL: stack_fold_andn_u32
-  ;CHECK:       andnl {{-?[0-9]*}}(%rsp), %eax, %eax {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_andn_u32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movl %esi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
+; CHECK-NEXT:    andnl {{[-0-9]+}}(%r{{[sb]}}p), %eax, %eax # 4-byte Folded Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call i64 asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = xor i32 %a0, -1
   %3 = and i32 %a1, %2
@@ -18,8 +57,46 @@ define i32 @stack_fold_andn_u32(i32 %a0, i32 %a1) {
 }
 
 define i64 @stack_fold_andn_u64(i64 %a0, i64 %a1) {
-  ;CHECK-LABEL: stack_fold_andn_u64
-  ;CHECK:       andnq {{-?[0-9]*}}(%rsp), %rax, %rax {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_andn_u64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; CHECK-NEXT:    andnq {{[-0-9]+}}(%r{{[sb]}}p), %rax, %rax # 8-byte Folded Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call i64 asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = xor i64 %a0, -1
   %3 = and i64 %a1, %2
@@ -27,9 +104,46 @@ define i64 @stack_fold_andn_u64(i64 %a0, i64 %a1) {
 }
 
 define i32 @stack_fold_bextr_u32(i32 %a0, i32 %a1) {
-  ;CHECK-LABEL: stack_fold_bextr_u32
-  ;CHECK:       # %bb.0:
-  ;CHECK:       bextrl %eax, {{-?[0-9]*}}(%rsp), %eax {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_bextr_u32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movl %esi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
+; CHECK-NEXT:    bextrl %eax, {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Folded Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call i64 asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = tail call i32 @llvm.x86.bmi.bextr.32(i32 %a0, i32 %a1)
   ret i32 %2
@@ -37,9 +151,46 @@ define i32 @stack_fold_bextr_u32(i32 %a0, i32 %a1) {
 declare i32 @llvm.x86.bmi.bextr.32(i32, i32)
 
 define i64 @stack_fold_bextr_u64(i64 %a0, i64 %a1) {
-  ;CHECK-LABEL: stack_fold_bextr_u64
-  ;CHECK:       # %bb.0:
-  ;CHECK:       bextrq %rax, {{-?[0-9]*}}(%rsp), %rax {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_bextr_u64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; CHECK-NEXT:    bextrq %rax, {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Folded Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call i64 asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = tail call i64 @llvm.x86.bmi.bextr.64(i64 %a0, i64 %a1)
   ret i64 %2
@@ -47,8 +198,44 @@ define i64 @stack_fold_bextr_u64(i64 %a0, i64 %a1) {
 declare i64 @llvm.x86.bmi.bextr.64(i64, i64)
 
 define i32 @stack_fold_blsi_u32(i32 %a0) {
-  ;CHECK-LABEL: stack_fold_blsi_u32
-  ;CHECK:       blsil {{-?[0-9]*}}(%rsp), %eax {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_blsi_u32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    blsil {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Folded Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call i64 asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = sub i32 0, %a0
   %3 = and i32 %2, %a0
@@ -56,8 +243,44 @@ define i32 @stack_fold_blsi_u32(i32 %a0) {
 }
 
 define i64 @stack_fold_blsi_u64(i64 %a0) {
-  ;CHECK-LABEL: stack_fold_blsi_u64
-  ;CHECK:       blsiq {{-?[0-9]*}}(%rsp), %rax {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_blsi_u64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    blsiq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Folded Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call i64 asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = sub i64 0, %a0
   %3 = and i64 %2, %a0
@@ -65,8 +288,44 @@ define i64 @stack_fold_blsi_u64(i64 %a0) {
 }
 
 define i32 @stack_fold_blsmsk_u32(i32 %a0) {
-  ;CHECK-LABEL: stack_fold_blsmsk_u32
-  ;CHECK:       blsmskl {{-?[0-9]*}}(%rsp), %eax {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_blsmsk_u32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    blsmskl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Folded Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call i64 asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = sub i32 %a0, 1
   %3 = xor i32 %2, %a0
@@ -74,8 +333,44 @@ define i32 @stack_fold_blsmsk_u32(i32 %a0) {
 }
 
 define i64 @stack_fold_blsmsk_u64(i64 %a0) {
-  ;CHECK-LABEL: stack_fold_blsmsk_u64
-  ;CHECK:       blsmskq {{-?[0-9]*}}(%rsp), %rax {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_blsmsk_u64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    blsmskq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Folded Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call i64 asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = sub i64 %a0, 1
   %3 = xor i64 %2, %a0
@@ -83,8 +378,44 @@ define i64 @stack_fold_blsmsk_u64(i64 %a0) {
 }
 
 define i32 @stack_fold_blsr_u32(i32 %a0) {
-  ;CHECK-LABEL: stack_fold_blsr_u32
-  ;CHECK:       blsrl {{-?[0-9]*}}(%rsp), %eax {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_blsr_u32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    blsrl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Folded Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call i64 asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = sub i32 %a0, 1
   %3 = and i32 %2, %a0
@@ -92,8 +423,44 @@ define i32 @stack_fold_blsr_u32(i32 %a0) {
 }
 
 define i64 @stack_fold_blsr_u64(i64 %a0) {
-  ;CHECK-LABEL: stack_fold_blsr_u64
-  ;CHECK:       blsrq {{-?[0-9]*}}(%rsp), %rax {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_blsr_u64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    blsrq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Folded Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call i64 asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = sub i64 %a0, 1
   %3 = and i64 %2, %a0
@@ -103,8 +470,44 @@ define i64 @stack_fold_blsr_u64(i64 %a0) {
 ;TODO stack_fold_tzcnt_u16
 
 define i32 @stack_fold_tzcnt_u32(i32 %a0) {
-  ;CHECK-LABEL: stack_fold_tzcnt_u32
-  ;CHECK:       tzcntl {{-?[0-9]*}}(%rsp), %eax {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_tzcnt_u32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    tzcntl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Folded Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call i64 asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = tail call i32 @llvm.cttz.i32(i32 %a0, i1 0)
   ret i32 %2
@@ -112,8 +515,44 @@ define i32 @stack_fold_tzcnt_u32(i32 %a0) {
 declare i32 @llvm.cttz.i32(i32, i1)
 
 define i64 @stack_fold_tzcnt_u64(i64 %a0) {
-  ;CHECK-LABEL: stack_fold_tzcnt_u64
-  ;CHECK:       tzcntq {{-?[0-9]*}}(%rsp), %rax {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_tzcnt_u64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    tzcntq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Folded Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call i64 asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = tail call i64 @llvm.cttz.i64(i64 %a0, i1 0)
   ret i64 %2
diff --git a/llvm/test/CodeGen/X86/stack-folding-bmi2.ll b/llvm/test/CodeGen/X86/stack-folding-bmi2.ll
index 0d0be6cd6..ce2f99ca8 100644
--- a/llvm/test/CodeGen/X86/stack-folding-bmi2.ll
+++ b/llvm/test/CodeGen/X86/stack-folding-bmi2.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -O3 -disable-peephole -mtriple=x86_64-unknown-unknown -mattr=+bmi,+bmi2 < %s | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -9,8 +10,46 @@ target triple = "x86_64-unknown-unknown"
 ; relevant registers and check that the reload is correctly folded into the instruction.
 
 define i32 @stack_fold_bzhi_u32(i32 %a0, i32 %a1)   {
-  ;CHECK-LABEL: stack_fold_bzhi_u32
-  ;CHECK:       bzhil %eax, {{-?[0-9]*}}(%rsp), %eax {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_bzhi_u32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movl %esi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
+; CHECK-NEXT:    bzhil %eax, {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Folded Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call i32 asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = tail call i32 @llvm.x86.bmi.bzhi.32(i32 %a0, i32 %a1)
   ret i32 %2
@@ -18,8 +57,46 @@ define i32 @stack_fold_bzhi_u32(i32 %a0, i32 %a1)   {
 declare i32 @llvm.x86.bmi.bzhi.32(i32, i32)
 
 define i64 @stack_fold_bzhi_u64(i64 %a0, i64 %a1)   {
-  ;CHECK-LABEL: stack_fold_bzhi_u64
-  ;CHECK:       bzhiq %rax, {{-?[0-9]*}}(%rsp), %rax {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_bzhi_u64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; CHECK-NEXT:    bzhiq %rax, {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Folded Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call i64 asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = tail call i64 @llvm.x86.bmi.bzhi.64(i64 %a0, i64 %a1)
   ret i64 %2
@@ -27,8 +104,46 @@ define i64 @stack_fold_bzhi_u64(i64 %a0, i64 %a1)   {
 declare i64 @llvm.x86.bmi.bzhi.64(i64, i64)
 
 define i32 @stack_fold_pdep_u32(i32 %a0, i32 %a1)   {
-  ;CHECK-LABEL: stack_fold_pdep_u32
-  ;CHECK:       pdepl {{-?[0-9]*}}(%rsp), %eax, %eax {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_pdep_u32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movl %esi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
+; CHECK-NEXT:    pdepl {{[-0-9]+}}(%r{{[sb]}}p), %eax, %eax # 4-byte Folded Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call i32 asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = tail call i32 @llvm.x86.bmi.pdep.32(i32 %a0, i32 %a1)
   ret i32 %2
@@ -36,8 +151,46 @@ define i32 @stack_fold_pdep_u32(i32 %a0, i32 %a1)   {
 declare i32 @llvm.x86.bmi.pdep.32(i32, i32)
 
 define i64 @stack_fold_pdep_u64(i64 %a0, i64 %a1)   {
-  ;CHECK-LABEL: stack_fold_pdep_u64
-  ;CHECK:       pdepq {{-?[0-9]*}}(%rsp), %rax, %rax {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_pdep_u64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; CHECK-NEXT:    pdepq {{[-0-9]+}}(%r{{[sb]}}p), %rax, %rax # 8-byte Folded Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call i64 asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = tail call i64 @llvm.x86.bmi.pdep.64(i64 %a0, i64 %a1)
   ret i64 %2
@@ -45,8 +198,46 @@ define i64 @stack_fold_pdep_u64(i64 %a0, i64 %a1)   {
 declare i64 @llvm.x86.bmi.pdep.64(i64, i64)
 
 define i32 @stack_fold_pext_u32(i32 %a0, i32 %a1)   {
-  ;CHECK-LABEL: stack_fold_pext_u32
-  ;CHECK:       pextl {{-?[0-9]*}}(%rsp), %eax, %eax {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_pext_u32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movl %esi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
+; CHECK-NEXT:    pextl {{[-0-9]+}}(%r{{[sb]}}p), %eax, %eax # 4-byte Folded Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call i32 asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = tail call i32 @llvm.x86.bmi.pext.32(i32 %a0, i32 %a1)
   ret i32 %2
@@ -54,8 +245,46 @@ define i32 @stack_fold_pext_u32(i32 %a0, i32 %a1)   {
 declare i32 @llvm.x86.bmi.pext.32(i32, i32)
 
 define i64 @stack_fold_pext_u64(i64 %a0, i64 %a1)   {
-  ;CHECK-LABEL: stack_fold_pext_u64
-  ;CHECK:       pextq {{-?[0-9]*}}(%rsp), %rax, %rax {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_pext_u64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; CHECK-NEXT:    pextq {{[-0-9]+}}(%r{{[sb]}}p), %rax, %rax # 8-byte Folded Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call i64 asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = tail call i64 @llvm.x86.bmi.pext.64(i64 %a0, i64 %a1)
   ret i64 %2
diff --git a/llvm/test/CodeGen/X86/stack-folding-fp-avx1.ll b/llvm/test/CodeGen/X86/stack-folding-fp-avx1.ll
index cbeeb04f4..6625cc4f0 100644
--- a/llvm/test/CodeGen/X86/stack-folding-fp-avx1.ll
+++ b/llvm/test/CodeGen/X86/stack-folding-fp-avx1.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -O3 -verify-machineinstrs -disable-peephole -mtriple=x86_64-unknown-unknown -mattr=+avx,+f16c < %s | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -9,48 +10,84 @@ target triple = "x86_64-unknown-unknown"
 ; relevant registers and check that the reload is correctly folded into the instruction.
 
 define <2 x double> @stack_fold_addpd(<2 x double> %a0, <2 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_addpd
-  ;CHECK:       vaddpd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_addpd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vaddpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fadd <2 x double> %a0, %a1
   ret <2 x double> %2
 }
 
 define <4 x double> @stack_fold_addpd_ymm(<4 x double> %a0, <4 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_addpd_ymm
-  ;CHECK:       vaddpd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_addpd_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vaddpd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fadd <4 x double> %a0, %a1
   ret <4 x double> %2
 }
 
 define <4 x float> @stack_fold_addps(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_addps
-  ;CHECK:       vaddps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_addps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vaddps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fadd <4 x float> %a0, %a1
   ret <4 x float> %2
 }
 
 define <8 x float> @stack_fold_addps_ymm(<8 x float> %a0, <8 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_addps_ymm
-  ;CHECK:       vaddps {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_addps_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vaddps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fadd <8 x float> %a0, %a1
   ret <8 x float> %2
 }
 
 define double @stack_fold_addsd(double %a0, double %a1) {
-  ;CHECK-LABEL: stack_fold_addsd
-  ;CHECK:       vaddsd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_addsd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovsd %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vaddsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 8-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fadd double %a0, %a1
   ret double %2
 }
 
 define <2 x double> @stack_fold_addsd_int(<2 x double> %a0, <2 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_addsd_int
-  ;CHECK:       vaddsd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_addsd_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vaddsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = extractelement <2 x double> %a0, i32 0
   %3 = extractelement <2 x double> %a1, i32 0
@@ -61,16 +98,28 @@ define <2 x double> @stack_fold_addsd_int(<2 x double> %a0, <2 x double> %a1) {
 declare <2 x double> @llvm.x86.sse2.add.sd(<2 x double>, <2 x double>) nounwind readnone
 
 define float @stack_fold_addss(float %a0, float %a1) {
-  ;CHECK-LABEL: stack_fold_addss
-  ;CHECK:       vaddss {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_addss:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vaddss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 4-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fadd float %a0, %a1
   ret float %2
 }
 
 define <4 x float> @stack_fold_addss_int(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_addss_int
-  ;CHECK:       vaddss {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_addss_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vaddss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = extractelement <4 x float> %a0, i32 0
   %3 = extractelement <4 x float> %a1, i32 0
@@ -81,8 +130,14 @@ define <4 x float> @stack_fold_addss_int(<4 x float> %a0, <4 x float> %a1) {
 declare <4 x float> @llvm.x86.sse.add.ss(<4 x float>, <4 x float>) nounwind readnone
 
 define <2 x double> @stack_fold_addsubpd(<2 x double> %a0, <2 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_addsubpd
-  ;CHECK:       vaddsubpd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_addsubpd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vaddsubpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x double> @llvm.x86.sse3.addsub.pd(<2 x double> %a0, <2 x double> %a1)
   ret <2 x double> %2
@@ -90,8 +145,14 @@ define <2 x double> @stack_fold_addsubpd(<2 x double> %a0, <2 x double> %a1) {
 declare <2 x double> @llvm.x86.sse3.addsub.pd(<2 x double>, <2 x double>) nounwind readnone
 
 define <4 x double> @stack_fold_addsubpd_ymm(<4 x double> %a0, <4 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_addsubpd_ymm
-  ;CHECK:       vaddsubpd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_addsubpd_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vaddsubpd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x double> @llvm.x86.avx.addsub.pd.256(<4 x double> %a0, <4 x double> %a1)
   ret <4 x double> %2
@@ -99,8 +160,14 @@ define <4 x double> @stack_fold_addsubpd_ymm(<4 x double> %a0, <4 x double> %a1)
 declare <4 x double> @llvm.x86.avx.addsub.pd.256(<4 x double>, <4 x double>) nounwind readnone
 
 define <4 x float> @stack_fold_addsubps(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_addsubps
-  ;CHECK:       vaddsubps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_addsubps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vaddsubps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x float> @llvm.x86.sse3.addsub.ps(<4 x float> %a0, <4 x float> %a1)
   ret <4 x float> %2
@@ -108,8 +175,14 @@ define <4 x float> @stack_fold_addsubps(<4 x float> %a0, <4 x float> %a1) {
 declare <4 x float> @llvm.x86.sse3.addsub.ps(<4 x float>, <4 x float>) nounwind readnone
 
 define <8 x float> @stack_fold_addsubps_ymm(<8 x float> %a0, <8 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_addsubps_ymm
-  ;CHECK:       vaddsubps {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_addsubps_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vaddsubps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x float> @llvm.x86.avx.addsub.ps.256(<8 x float> %a0, <8 x float> %a1)
   ret <8 x float> %2
@@ -117,8 +190,16 @@ define <8 x float> @stack_fold_addsubps_ymm(<8 x float> %a0, <8 x float> %a1) {
 declare <8 x float> @llvm.x86.avx.addsub.ps.256(<8 x float>, <8 x float>) nounwind readnone
 
 define <2 x double> @stack_fold_andnpd(<2 x double> %a0, <2 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_andnpd
-  ;CHECK:       vandnpd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_andnpd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vandnpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = bitcast <2 x double> %a0 to <2 x i64>
   %3 = bitcast <2 x double> %a1 to <2 x i64>
@@ -131,8 +212,16 @@ define <2 x double> @stack_fold_andnpd(<2 x double> %a0, <2 x double> %a1) {
 }
 
 define <4 x double> @stack_fold_andnpd_ymm(<4 x double> %a0, <4 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_andnpd_ymm
-  ;CHECK:       vandnpd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_andnpd_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vandnpd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = bitcast <4 x double> %a0 to <4 x i64>
   %3 = bitcast <4 x double> %a1 to <4 x i64>
@@ -145,8 +234,16 @@ define <4 x double> @stack_fold_andnpd_ymm(<4 x double> %a0, <4 x double> %a1) {
 }
 
 define <4 x float> @stack_fold_andnps(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_andnps
-  ;CHECK:       vandnps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_andnps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vandnps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vaddps %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = bitcast <4 x float> %a0 to <2 x i64>
   %3 = bitcast <4 x float> %a1 to <2 x i64>
@@ -159,8 +256,16 @@ define <4 x float> @stack_fold_andnps(<4 x float> %a0, <4 x float> %a1) {
 }
 
 define <8 x float> @stack_fold_andnps_ymm(<8 x float> %a0, <8 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_andnps_ymm
-  ;CHECK:       vandnps {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_andnps_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vandnps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vaddps %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = bitcast <8 x float> %a0 to <4 x i64>
   %3 = bitcast <8 x float> %a1 to <4 x i64>
@@ -173,8 +278,16 @@ define <8 x float> @stack_fold_andnps_ymm(<8 x float> %a0, <8 x float> %a1) {
 }
 
 define <2 x double> @stack_fold_andpd(<2 x double> %a0, <2 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_andpd
-  ;CHECK:       vandpd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_andpd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vandpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = bitcast <2 x double> %a0 to <2 x i64>
   %3 = bitcast <2 x double> %a1 to <2 x i64>
@@ -186,8 +299,16 @@ define <2 x double> @stack_fold_andpd(<2 x double> %a0, <2 x double> %a1) {
 }
 
 define <4 x double> @stack_fold_andpd_ymm(<4 x double> %a0, <4 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_andpd_ymm
-  ;CHECK:       vandpd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_andpd_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vandpd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = bitcast <4 x double> %a0 to <4 x i64>
   %3 = bitcast <4 x double> %a1 to <4 x i64>
@@ -199,8 +320,16 @@ define <4 x double> @stack_fold_andpd_ymm(<4 x double> %a0, <4 x double> %a1) {
 }
 
 define <4 x float> @stack_fold_andps(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_andps
-  ;CHECK:       vandps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_andps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vandps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vaddps %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = bitcast <4 x float> %a0 to <2 x i64>
   %3 = bitcast <4 x float> %a1 to <2 x i64>
@@ -212,8 +341,16 @@ define <4 x float> @stack_fold_andps(<4 x float> %a0, <4 x float> %a1) {
 }
 
 define <8 x float> @stack_fold_andps_ymm(<8 x float> %a0, <8 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_andps_ymm
-  ;CHECK:       vandps {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_andps_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vandps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vaddps %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = bitcast <8 x float> %a0 to <4 x i64>
   %3 = bitcast <8 x float> %a1 to <4 x i64>
@@ -225,8 +362,17 @@ define <8 x float> @stack_fold_andps_ymm(<8 x float> %a0, <8 x float> %a1) {
 }
 
 define <2 x double> @stack_fold_blendpd(<2 x double> %a0, <2 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_blendpd
-  ;CHECK:       vblendpd $2, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_blendpd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vblendpd $2, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = xmm0[0],mem[1]
+; CHECK-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = select <2 x i1> <i1 1, i1 0>, <2 x double> %a0, <2 x double> %a1
   ; fadd forces execution domain
@@ -235,8 +381,17 @@ define <2 x double> @stack_fold_blendpd(<2 x double> %a0, <2 x double> %a1) {
 }
 
 define <4 x double> @stack_fold_blendpd_ymm(<4 x double> %a0, <4 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_blendpd_ymm
-  ;CHECK:       vblendpd $6, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_blendpd_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vblendpd $6, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    # ymm0 = ymm0[0],mem[1,2],ymm0[3]
+; CHECK-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = select <4 x i1> <i1 1, i1 0, i1 0, i1 1>, <4 x double> %a0, <4 x double> %a1
   ; fadd forces execution domain
@@ -244,8 +399,6 @@ define <4 x double> @stack_fold_blendpd_ymm(<4 x double> %a0, <4 x double> %a1)
   ret <4 x double> %3}
 
 define <4 x float> @stack_fold_blendps(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_blendps
-  ;CHECK:       vblendps $6, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = select <4 x i1> <i1 1, i1 0, i1 0, i1 1>, <4 x float> %a0, <4 x float> %a1
   ; fadd forces execution domain
@@ -254,8 +407,17 @@ define <4 x float> @stack_fold_blendps(<4 x float> %a0, <4 x float> %a1) {
 }
 
 define <8 x float> @stack_fold_blendps_ymm(<8 x float> %a0, <8 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_blendps_ymm
-  ;CHECK:       vblendps $102, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_blendps_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vblendps $102, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    # ymm0 = ymm0[0],mem[1,2],ymm0[3,4],mem[5,6],ymm0[7]
+; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vaddps %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = select <8 x i1> <i1 1, i1 0, i1 0, i1 1, i1 1, i1 0, i1 0, i1 1>, <8 x float> %a0, <8 x float> %a1
   ; fadd forces execution domain
@@ -264,8 +426,14 @@ define <8 x float> @stack_fold_blendps_ymm(<8 x float> %a0, <8 x float> %a1) {
 }
 
 define <2 x double> @stack_fold_blendvpd(<2 x double> %a0, <2 x double> %a1, <2 x double> %c) {
-  ;CHECK-LABEL: stack_fold_blendvpd
-  ;CHECK:       vblendvpd {{%xmm[0-9][0-9]*}}, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_blendvpd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vblendvpd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x double> @llvm.x86.sse41.blendvpd(<2 x double> %a1, <2 x double> %c, <2 x double> %a0)
   ret <2 x double> %2
@@ -273,8 +441,14 @@ define <2 x double> @stack_fold_blendvpd(<2 x double> %a0, <2 x double> %a1, <2
 declare <2 x double> @llvm.x86.sse41.blendvpd(<2 x double>, <2 x double>, <2 x double>) nounwind readnone
 
 define <4 x double> @stack_fold_blendvpd_ymm(<4 x double> %a0, <4 x double> %a1, <4 x double> %c) {
-  ;CHECK-LABEL: stack_fold_blendvpd_ymm
-  ;CHECK:       vblendvpd {{%ymm[0-9][0-9]*}}, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_blendvpd_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vblendvpd %ymm0, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x double> @llvm.x86.avx.blendv.pd.256(<4 x double> %a1, <4 x double> %c, <4 x double> %a0)
   ret <4 x double> %2
@@ -282,8 +456,14 @@ define <4 x double> @stack_fold_blendvpd_ymm(<4 x double> %a0, <4 x double> %a1,
 declare <4 x double> @llvm.x86.avx.blendv.pd.256(<4 x double>, <4 x double>, <4 x double>) nounwind readnone
 
 define <4 x float> @stack_fold_blendvps(<4 x float> %a0, <4 x float> %a1, <4 x float> %c) {
-  ;CHECK-LABEL: stack_fold_blendvps
-  ;CHECK:       vblendvps {{%xmm[0-9][0-9]*}}, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_blendvps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vblendvps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x float> @llvm.x86.sse41.blendvps(<4 x float> %a1, <4 x float> %c, <4 x float> %a0)
   ret <4 x float> %2
@@ -291,8 +471,14 @@ define <4 x float> @stack_fold_blendvps(<4 x float> %a0, <4 x float> %a1, <4 x f
 declare <4 x float> @llvm.x86.sse41.blendvps(<4 x float>, <4 x float>, <4 x float>) nounwind readnone
 
 define <8 x float> @stack_fold_blendvps_ymm(<8 x float> %a0, <8 x float> %a1, <8 x float> %c) {
-  ;CHECK-LABEL: stack_fold_blendvps_ymm
-  ;CHECK:       vblendvps {{%ymm[0-9][0-9]*}}, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_blendvps_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vblendvps %ymm0, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x float> @llvm.x86.avx.blendv.ps.256(<8 x float> %a1, <8 x float> %c, <8 x float> %a0)
   ret <8 x float> %2
@@ -300,8 +486,14 @@ define <8 x float> @stack_fold_blendvps_ymm(<8 x float> %a0, <8 x float> %a1, <8
 declare <8 x float> @llvm.x86.avx.blendv.ps.256(<8 x float>, <8 x float>, <8 x float>) nounwind readnone
 
 define <2 x double> @stack_fold_cmppd(<2 x double> %a0, <2 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_cmppd
-  ;CHECK:       vcmpeqpd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_cmppd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcmpeqpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x double> @llvm.x86.sse2.cmp.pd(<2 x double> %a0, <2 x double> %a1, i8 0)
   ret <2 x double> %2
@@ -309,8 +501,14 @@ define <2 x double> @stack_fold_cmppd(<2 x double> %a0, <2 x double> %a1) {
 declare <2 x double> @llvm.x86.sse2.cmp.pd(<2 x double>, <2 x double>, i8) nounwind readnone
 
 define <4 x double> @stack_fold_cmppd_ymm(<4 x double> %a0, <4 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_cmppd_ymm
-  ;CHECK:       vcmpeqpd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_cmppd_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcmpeqpd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x double> @llvm.x86.avx.cmp.pd.256(<4 x double> %a0, <4 x double> %a1, i8 0)
   ret <4 x double> %2
@@ -318,8 +516,14 @@ define <4 x double> @stack_fold_cmppd_ymm(<4 x double> %a0, <4 x double> %a1) {
 declare <4 x double> @llvm.x86.avx.cmp.pd.256(<4 x double>, <4 x double>, i8) nounwind readnone
 
 define <4 x float> @stack_fold_cmpps(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_cmpps
-  ;CHECK:       vcmpeqps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_cmpps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcmpeqps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x float> @llvm.x86.sse.cmp.ps(<4 x float> %a0, <4 x float> %a1, i8 0)
   ret <4 x float> %2
@@ -327,8 +531,14 @@ define <4 x float> @stack_fold_cmpps(<4 x float> %a0, <4 x float> %a1) {
 declare <4 x float> @llvm.x86.sse.cmp.ps(<4 x float>, <4 x float>, i8) nounwind readnone
 
 define <8 x float> @stack_fold_cmpps_ymm(<8 x float> %a0, <8 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_cmpps_ymm
-  ;CHECK:       vcmpeqps {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_cmpps_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcmpeqps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float> %a0, <8 x float> %a1, i8 0)
   ret <8 x float> %2
@@ -336,8 +546,17 @@ define <8 x float> @stack_fold_cmpps_ymm(<8 x float> %a0, <8 x float> %a1) {
 declare <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float>, <8 x float>, i8) nounwind readnone
 
 define i32 @stack_fold_cmpsd(double %a0, double %a1) {
-  ;CHECK-LABEL: stack_fold_cmpsd
-  ;CHECK:       vcmpeqsd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_cmpsd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovsd %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcmpeqsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 8-byte Folded Reload
+; CHECK-NEXT:    vmovq %xmm0, %rax
+; CHECK-NEXT:    andl $1, %eax
+; CHECK-NEXT:    # kill: def $eax killed $eax killed $rax
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fcmp oeq double %a0, %a1
   %3 = zext i1 %2 to i32
@@ -345,8 +564,14 @@ define i32 @stack_fold_cmpsd(double %a0, double %a1) {
 }
 
 define <2 x double> @stack_fold_cmpsd_int(<2 x double> %a0, <2 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_cmpsd_int
-  ;CHECK:       vcmpeqsd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_cmpsd_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcmpeqsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x double> @llvm.x86.sse2.cmp.sd(<2 x double> %a0, <2 x double> %a1, i8 0)
   ret <2 x double> %2
@@ -354,8 +579,16 @@ define <2 x double> @stack_fold_cmpsd_int(<2 x double> %a0, <2 x double> %a1) {
 declare <2 x double> @llvm.x86.sse2.cmp.sd(<2 x double>, <2 x double>, i8) nounwind readnone
 
 define i32 @stack_fold_cmpss(float %a0, float %a1) {
-  ;CHECK-LABEL: stack_fold_cmpss
-  ;CHECK:       vcmpeqss {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_cmpss:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcmpeqss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 4-byte Folded Reload
+; CHECK-NEXT:    vmovd %xmm0, %eax
+; CHECK-NEXT:    andl $1, %eax
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fcmp oeq float %a0, %a1
   %3 = zext i1 %2 to i32
@@ -363,8 +596,14 @@ define i32 @stack_fold_cmpss(float %a0, float %a1) {
 }
 
 define <4 x float> @stack_fold_cmpss_int(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_cmpss_int
-  ;CHECK:       vcmpeqss {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_cmpss_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcmpeqss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x float> @llvm.x86.sse.cmp.ss(<4 x float> %a0, <4 x float> %a1, i8 0)
   ret <4 x float> %2
@@ -374,8 +613,18 @@ declare <4 x float> @llvm.x86.sse.cmp.ss(<4 x float>, <4 x float>, i8) nounwind
 ; TODO stack_fold_comisd
 
 define i32 @stack_fold_comisd_int(<2 x double> %a0, <2 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_comisd_int
-  ;CHECK:       vcomisd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_comisd_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcomisd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    setnp %al
+; CHECK-NEXT:    sete %cl
+; CHECK-NEXT:    andb %al, %cl
+; CHECK-NEXT:    movzbl %cl, %eax
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call i32 @llvm.x86.sse2.comieq.sd(<2 x double> %a0, <2 x double> %a1)
   ret i32 %2
@@ -385,8 +634,18 @@ declare i32 @llvm.x86.sse2.comieq.sd(<2 x double>, <2 x double>) nounwind readno
 ; TODO stack_fold_comiss
 
 define i32 @stack_fold_comiss_int(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_comiss_int
-  ;CHECK:       vcomiss {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_comiss_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcomiss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    setnp %al
+; CHECK-NEXT:    sete %cl
+; CHECK-NEXT:    andb %al, %cl
+; CHECK-NEXT:    movzbl %cl, %eax
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call i32 @llvm.x86.sse.comieq.ss(<4 x float> %a0, <4 x float> %a1)
   ret i32 %2
@@ -394,16 +653,28 @@ define i32 @stack_fold_comiss_int(<4 x float> %a0, <4 x float> %a1) {
 declare i32 @llvm.x86.sse.comieq.ss(<4 x float>, <4 x float>) nounwind readnone
 
 define <2 x double> @stack_fold_cvtdq2pd(<4 x i32> %a0) {
-  ;CHECK-LABEL: stack_fold_cvtdq2pd
-  ;CHECK:   vcvtdq2pd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtdq2pd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcvtdq2pd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <4 x i32> %a0, <4 x i32> undef, <2 x i32> <i32 0, i32 1>
   %3 = sitofp <2 x i32> %2 to <2 x double>
   ret <2 x double> %3
 }
 define <2 x double> @stack_fold_cvtdq2pd_int(<4 x i32> %a0) {
-  ;CHECK-LABEL: stack_fold_cvtdq2pd_int
-  ;CHECK:   vcvtdq2pd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtdq2pd_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcvtdq2pd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <4 x i32> %a0, <4 x i32> %a0, <2 x i32> <i32 0, i32 1>
   %cvt = sitofp <2 x i32> %2 to <2 x double>
@@ -411,40 +682,70 @@ define <2 x double> @stack_fold_cvtdq2pd_int(<4 x i32> %a0) {
 }
 
 define <4 x double> @stack_fold_cvtdq2pd_ymm(<4 x i32> %a0) {
-  ;CHECK-LABEL: stack_fold_cvtdq2pd_ymm
-  ;CHECK:   vcvtdq2pd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtdq2pd_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcvtdq2pd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = sitofp <4 x i32> %a0 to <4 x double>
   ret <4 x double> %2
 }
 
 define <4 x double> @stack_fold_cvtdq2pd_ymm_int(<4 x i32> %a0) {
-  ;CHECK-LABEL: stack_fold_cvtdq2pd_ymm_int
-  ;CHECK:   vcvtdq2pd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtdq2pd_ymm_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcvtdq2pd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %cvt = sitofp <4 x i32> %a0 to <4 x double>
   ret <4 x double> %cvt
 }
 
 define <4 x float> @stack_fold_cvtdq2ps(<4 x i32> %a0) {
-  ;CHECK-LABEL: stack_fold_cvtdq2ps
-  ;CHECK:   vcvtdq2ps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtdq2ps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcvtdq2ps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = sitofp <4 x i32> %a0 to <4 x float>
   ret <4 x float> %2
 }
 
 define <8 x float> @stack_fold_cvtdq2ps_ymm(<8 x i32> %a0) {
-  ;CHECK-LABEL: stack_fold_cvtdq2ps_ymm
-  ;CHECK:   vcvtdq2ps {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtdq2ps_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcvtdq2ps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = sitofp <8 x i32> %a0 to <8 x float>
   ret <8 x float> %2
 }
 
 define <4 x i32> @stack_fold_cvtpd2dq(<2 x double> %a0) {
-  ;CHECK-LABEL: stack_fold_cvtpd2dq
-  ;CHECK:   vcvtpd2dqx {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtpd2dq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcvtpd2dqx {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x i32> @llvm.x86.sse2.cvtpd2dq(<2 x double> %a0)
   ret <4 x i32> %2
@@ -452,8 +753,15 @@ define <4 x i32> @stack_fold_cvtpd2dq(<2 x double> %a0) {
 declare <4 x i32> @llvm.x86.sse2.cvtpd2dq(<2 x double>) nounwind readnone
 
 define <4 x i32> @stack_fold_cvtpd2dq_ymm(<4 x double> %a0) {
-  ;CHECK-LABEL: stack_fold_cvtpd2dq_ymm
-  ;CHECK:   vcvtpd2dqy {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtpd2dq_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcvtpd2dqy {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 32-byte Folded Reload
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x i32> @llvm.x86.avx.cvt.pd2dq.256(<4 x double> %a0)
   ret <4 x i32> %2
@@ -461,24 +769,43 @@ define <4 x i32> @stack_fold_cvtpd2dq_ymm(<4 x double> %a0) {
 declare <4 x i32> @llvm.x86.avx.cvt.pd2dq.256(<4 x double>) nounwind readnone
 
 define <2 x float> @stack_fold_cvtpd2ps(<2 x double> %a0) {
-  ;CHECK-LABEL: stack_fold_cvtpd2ps
-  ;CHECK:   vcvtpd2psx {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtpd2ps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcvtpd2psx {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fptrunc <2 x double> %a0 to <2 x float>
   ret <2 x float> %2
 }
 
 define <4 x float> @stack_fold_cvtpd2ps_ymm(<4 x double> %a0) {
-  ;CHECK-LABEL: stack_fold_cvtpd2ps_ymm
-  ;CHECK:   vcvtpd2psy {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtpd2ps_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcvtpd2psy {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 32-byte Folded Reload
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fptrunc <4 x double> %a0 to <4 x float>
   ret <4 x float> %2
 }
 
 define <4 x float> @stack_fold_cvtph2ps(<8 x i16> %a0) {
-  ;CHECK-LABEL: stack_fold_cvtph2ps
-  ;CHECK:   vcvtph2ps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtph2ps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcvtph2ps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x float> @llvm.x86.vcvtph2ps.128(<8 x i16> %a0)
   ret <4 x float> %2
@@ -486,8 +813,14 @@ define <4 x float> @stack_fold_cvtph2ps(<8 x i16> %a0) {
 declare <4 x float> @llvm.x86.vcvtph2ps.128(<8 x i16>) nounwind readonly
 
 define <8 x float> @stack_fold_cvtph2ps_ymm(<8 x i16> %a0) {
-  ;CHECK-LABEL: stack_fold_cvtph2ps_ymm
-  ;CHECK:   vcvtph2ps {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtph2ps_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcvtph2ps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x float> @llvm.x86.vcvtph2ps.256(<8 x i16> %a0)
   ret <8 x float> %2
@@ -495,8 +828,14 @@ define <8 x float> @stack_fold_cvtph2ps_ymm(<8 x i16> %a0) {
 declare <8 x float> @llvm.x86.vcvtph2ps.256(<8 x i16>) nounwind readonly
 
 define <4 x i32> @stack_fold_cvtps2dq(<4 x float> %a0) {
-  ;CHECK-LABEL: stack_fold_cvtps2dq
-  ;CHECK:  vcvtps2dq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtps2dq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcvtps2dq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x i32> @llvm.x86.sse2.cvtps2dq(<4 x float> %a0)
   ret <4 x i32> %2
@@ -504,8 +843,14 @@ define <4 x i32> @stack_fold_cvtps2dq(<4 x float> %a0) {
 declare <4 x i32> @llvm.x86.sse2.cvtps2dq(<4 x float>) nounwind readnone
 
 define <8 x i32> @stack_fold_cvtps2dq_ymm(<8 x float> %a0) {
-  ;CHECK-LABEL: stack_fold_cvtps2dq_ymm
-  ;CHECK:  vcvtps2dq {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtps2dq_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcvtps2dq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x i32> @llvm.x86.avx.cvt.ps2dq.256(<8 x float> %a0)
   ret <8 x i32> %2
@@ -513,8 +858,14 @@ define <8 x i32> @stack_fold_cvtps2dq_ymm(<8 x float> %a0) {
 declare <8 x i32> @llvm.x86.avx.cvt.ps2dq.256(<8 x float>) nounwind readnone
 
 define <2 x double> @stack_fold_cvtps2pd(<4 x float> %a0) {
-  ;CHECK-LABEL: stack_fold_cvtps2pd
-  ;CHECK:   vcvtps2pd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtps2pd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcvtps2pd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <4 x float> %a0, <4 x float> undef, <2 x i32> <i32 0, i32 1>
   %3 = fpext <2 x float> %2 to <2 x double>
@@ -522,8 +873,14 @@ define <2 x double> @stack_fold_cvtps2pd(<4 x float> %a0) {
 }
 
 define <2 x double> @stack_fold_cvtps2pd_int(<4 x float> %a0) {
-  ;CHECK-LABEL: stack_fold_cvtps2pd_int
-  ;CHECK:   vcvtps2pd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtps2pd_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcvtps2pd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <4 x float> %a0, <4 x float> %a0, <2 x i32> <i32 0, i32 1>
   %cvtps2pd = fpext <2 x float> %2 to <2 x double>
@@ -531,24 +888,43 @@ define <2 x double> @stack_fold_cvtps2pd_int(<4 x float> %a0) {
 }
 
 define <4 x double> @stack_fold_cvtps2pd_ymm(<4 x float> %a0) {
-  ;CHECK-LABEL: stack_fold_cvtps2pd_ymm
-  ;CHECK:   vcvtps2pd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtps2pd_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcvtps2pd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fpext <4 x float> %a0 to <4 x double>
   ret <4 x double> %2
 }
 
 define <4 x double> @stack_fold_cvtps2pd_ymm_int(<4 x float> %a0) {
-  ;CHECK-LABEL: stack_fold_cvtps2pd_ymm_int
-  ;CHECK:   vcvtps2pd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtps2pd_ymm_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcvtps2pd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %cvtps2pd = fpext <4 x float> %a0 to <4 x double>
   ret <4 x double> %cvtps2pd
 }
 
 define <8 x i16> @stack_fold_cvtps2ph_ymm(<8 x float> %a0) {
-  ;CHECK-LABEL: stack_fold_cvtps2ph_ymm
-  ;CHECK:   vcvtps2ph $0, {{%ymm[0-9][0-9]*}}, {{-?[0-9]*}}(%rsp) {{.*#+}} 16-byte Folded Spill
+; CHECK-LABEL: stack_fold_cvtps2ph_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vcvtps2ph $0, %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Folded Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
   %1 = call <8 x i16> @llvm.x86.vcvtps2ph.256(<8 x float> %a0, i32 0)
   %2 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   ret <8 x i16> %1
@@ -558,8 +934,14 @@ declare <8 x i16> @llvm.x86.vcvtps2ph.256(<8 x float>, i32) nounwind readonly
 ; TODO stack_fold_cvtsd2si
 
 define i32 @stack_fold_cvtsd2si_int(<2 x double> %a0) {
-  ;CHECK-LABEL: stack_fold_cvtsd2si_int
-  ;CHECK:  vcvtsd2si {{-?[0-9]*}}(%rsp), %eax {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtsd2si_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcvtsd2si {{[-0-9]+}}(%r{{[sb]}}p), %eax # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call i32 @llvm.x86.sse2.cvtsd2si(<2 x double> %a0)
   ret i32 %2
@@ -569,8 +951,14 @@ declare i32 @llvm.x86.sse2.cvtsd2si(<2 x double>) nounwind readnone
 ; TODO stack_fold_cvtsd2si64
 
 define i64 @stack_fold_cvtsd2si64_int(<2 x double> %a0) {
-  ;CHECK-LABEL: stack_fold_cvtsd2si64_int
-  ;CHECK:  vcvtsd2si {{-?[0-9]*}}(%rsp), %rax {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtsd2si64_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcvtsd2si {{[-0-9]+}}(%r{{[sb]}}p), %rax # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call i64 @llvm.x86.sse2.cvtsd2si64(<2 x double> %a0)
   ret i64 %2
@@ -578,16 +966,89 @@ define i64 @stack_fold_cvtsd2si64_int(<2 x double> %a0) {
 declare i64 @llvm.x86.sse2.cvtsd2si64(<2 x double>) nounwind readnone
 
 define double @stack_fold_cvtsi2sd(i32 %a0) {
-  ;CHECK-LABEL: stack_fold_cvtsi2sd
-  ;CHECK:  vcvtsi2sdl {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtsi2sd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcvtsi2sdl {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 # 4-byte Folded Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = sitofp i32 %a0 to double
   ret double %2
 }
 
 define <2 x double> @stack_fold_cvtsi2sd_int(i32 %a0) {
-  ;CHECK-LABEL: stack_fold_cvtsi2sd_int
-  ;CHECK:  vcvtsi2sdl {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtsi2sd_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcvtsi2sdl {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 # 4-byte Folded Reload
+; CHECK-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = sitofp i32 %a0 to double
   %3 = insertelement <2 x double> zeroinitializer, double %2, i64 0
@@ -595,16 +1056,89 @@ define <2 x double> @stack_fold_cvtsi2sd_int(i32 %a0) {
 }
 
 define double @stack_fold_cvtsi642sd(i64 %a0) {
-  ;CHECK-LABEL: stack_fold_cvtsi642sd
-  ;CHECK:  vcvtsi2sdq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtsi642sd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcvtsi2sdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 # 8-byte Folded Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = sitofp i64 %a0 to double
   ret double %2
 }
 
 define <2 x double> @stack_fold_cvtsi642sd_int(i64 %a0) {
-  ;CHECK-LABEL: stack_fold_cvtsi642sd_int
-  ;CHECK:  vcvtsi2sdq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtsi642sd_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcvtsi2sdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 # 8-byte Folded Reload
+; CHECK-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = sitofp i64 %a0 to double
   %3 = insertelement <2 x double> zeroinitializer, double %2, i64 0
@@ -612,16 +1146,90 @@ define <2 x double> @stack_fold_cvtsi642sd_int(i64 %a0) {
 }
 
 define float @stack_fold_cvtsi2ss(i32 %a0) {
-  ;CHECK-LABEL: stack_fold_cvtsi2ss
-  ;CHECK:  vcvtsi2ssl {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtsi2ss:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcvtsi2ssl {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 # 4-byte Folded Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = sitofp i32 %a0 to float
   ret float %2
 }
 
 define <4 x float> @stack_fold_cvtsi2ss_int(i32 %a0) {
-  ;CHECK-LABEL: stack_fold_cvtsi2ss_int
-  ;CHECK:  vcvtsi2ssl {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtsi2ss_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcvtsi2ssl {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 # 4-byte Folded Reload
+; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = sitofp i32 %a0 to float
   %3 = insertelement <4 x float> zeroinitializer, float %2, i64 0
@@ -629,16 +1237,90 @@ define <4 x float> @stack_fold_cvtsi2ss_int(i32 %a0) {
 }
 
 define float @stack_fold_cvtsi642ss(i64 %a0) {
-  ;CHECK-LABEL: stack_fold_cvtsi642ss
-  ;CHECK:  vcvtsi2ssq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtsi642ss:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcvtsi2ssq {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 # 8-byte Folded Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = sitofp i64 %a0 to float
   ret float %2
 }
 
 define <4 x float> @stack_fold_cvtsi642ss_int(i64 %a0) {
-  ;CHECK-LABEL: stack_fold_cvtsi642ss_int
-  ;CHECK:  vcvtsi2ssq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtsi642ss_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcvtsi2ssq {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 # 8-byte Folded Reload
+; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = sitofp i64 %a0 to float
   %3 = insertelement <4 x float> zeroinitializer, float %2, i64 0
@@ -648,8 +1330,14 @@ define <4 x float> @stack_fold_cvtsi642ss_int(i64 %a0) {
 ; TODO stack_fold_cvtss2si
 
 define i32 @stack_fold_cvtss2si_int(<4 x float> %a0) {
-  ;CHECK-LABEL: stack_fold_cvtss2si_int
-  ;CHECK:  vcvtss2si {{-?[0-9]*}}(%rsp), %eax {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtss2si_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcvtss2si {{[-0-9]+}}(%r{{[sb]}}p), %eax # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call i32 @llvm.x86.sse.cvtss2si(<4 x float> %a0)
   ret i32 %2
@@ -659,8 +1347,14 @@ declare i32 @llvm.x86.sse.cvtss2si(<4 x float>) nounwind readnone
 ; TODO stack_fold_cvtss2si64
 
 define i64 @stack_fold_cvtss2si64_int(<4 x float> %a0) {
-  ;CHECK-LABEL: stack_fold_cvtss2si64_int
-  ;CHECK:  vcvtss2si {{-?[0-9]*}}(%rsp), %rax {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtss2si64_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcvtss2si {{[-0-9]+}}(%r{{[sb]}}p), %rax # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call i64 @llvm.x86.sse.cvtss2si64(<4 x float> %a0)
   ret i64 %2
@@ -668,8 +1362,14 @@ define i64 @stack_fold_cvtss2si64_int(<4 x float> %a0) {
 declare i64 @llvm.x86.sse.cvtss2si64(<4 x float>) nounwind readnone
 
 define <4 x i32> @stack_fold_cvttpd2dq(<2 x double> %a0) {
-  ;CHECK-LABEL: stack_fold_cvttpd2dq
-  ;CHECK:  vcvttpd2dqx {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvttpd2dq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcvttpd2dqx {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x i32> @llvm.x86.sse2.cvttpd2dq(<2 x double> %a0)
   ret <4 x i32> %2
@@ -677,40 +1377,71 @@ define <4 x i32> @stack_fold_cvttpd2dq(<2 x double> %a0) {
 declare <4 x i32> @llvm.x86.sse2.cvttpd2dq(<2 x double>) nounwind readnone
 
 define <4 x i32> @stack_fold_cvttpd2dq_ymm(<4 x double> %a0) {
-  ;CHECK-LABEL: stack_fold_cvttpd2dq_ymm
-  ;CHECK:  vcvttpd2dqy {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvttpd2dq_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcvttpd2dqy {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 32-byte Folded Reload
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fptosi <4 x double> %a0 to <4 x i32>
   ret <4 x i32> %2
 }
 
 define <4 x i32> @stack_fold_cvttps2dq(<4 x float> %a0) {
-  ;CHECK-LABEL: stack_fold_cvttps2dq
-  ;CHECK:  vcvttps2dq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvttps2dq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcvttps2dq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fptosi <4 x float> %a0 to <4 x i32>
   ret <4 x i32> %2
 }
 
 define <8 x i32> @stack_fold_cvttps2dq_ymm(<8 x float> %a0) {
-  ;CHECK-LABEL: stack_fold_cvttps2dq_ymm
-  ;CHECK:  vcvttps2dq {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvttps2dq_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcvttps2dq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fptosi <8 x float> %a0 to <8 x i32>
   ret <8 x i32> %2
 }
 
 define i32 @stack_fold_cvttsd2si(double %a0) {
-  ;CHECK-LABEL: stack_fold_cvttsd2si
-  ;CHECK:  vcvttsd2si {{-?[0-9]*}}(%rsp), %eax {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvttsd2si:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovsd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcvttsd2si {{[-0-9]+}}(%r{{[sb]}}p), %eax # 8-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fptosi double %a0 to i32
   ret i32 %2
 }
 
 define i32 @stack_fold_cvttsd2si_int(<2 x double> %a0) {
-  ;CHECK-LABEL: stack_fold_cvttsd2si_int
-  ;CHECK:  vcvttsd2si {{-?[0-9]*}}(%rsp), %eax {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvttsd2si_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcvttsd2si {{[-0-9]+}}(%r{{[sb]}}p), %eax # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call i32 @llvm.x86.sse2.cvttsd2si(<2 x double> %a0)
   ret i32 %2
@@ -718,16 +1449,28 @@ define i32 @stack_fold_cvttsd2si_int(<2 x double> %a0) {
 declare i32 @llvm.x86.sse2.cvttsd2si(<2 x double>) nounwind readnone
 
 define i64 @stack_fold_cvttsd2si64(double %a0) {
-  ;CHECK-LABEL: stack_fold_cvttsd2si64
-  ;CHECK:  vcvttsd2si {{-?[0-9]*}}(%rsp), %rax {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvttsd2si64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovsd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcvttsd2si {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fptosi double %a0 to i64
   ret i64 %2
 }
 
 define i64 @stack_fold_cvttsd2si64_int(<2 x double> %a0) {
-  ;CHECK-LABEL: stack_fold_cvttsd2si64_int
-  ;CHECK:  vcvttsd2si {{-?[0-9]*}}(%rsp), %rax {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvttsd2si64_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcvttsd2si {{[-0-9]+}}(%r{{[sb]}}p), %rax # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call i64 @llvm.x86.sse2.cvttsd2si64(<2 x double> %a0)
   ret i64 %2
@@ -735,16 +1478,28 @@ define i64 @stack_fold_cvttsd2si64_int(<2 x double> %a0) {
 declare i64 @llvm.x86.sse2.cvttsd2si64(<2 x double>) nounwind readnone
 
 define i32 @stack_fold_cvttss2si(float %a0) {
-  ;CHECK-LABEL: stack_fold_cvttss2si
-  ;CHECK:  vcvttss2si {{-?[0-9]*}}(%rsp), %eax {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvttss2si:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovss %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcvttss2si {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fptosi float %a0 to i32
   ret i32 %2
 }
 
 define i32 @stack_fold_cvttss2si_int(<4 x float> %a0) {
-  ;CHECK-LABEL: stack_fold_cvttss2si_int
-  ;CHECK:  vcvttss2si {{-?[0-9]*}}(%rsp), %eax {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvttss2si_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcvttss2si {{[-0-9]+}}(%r{{[sb]}}p), %eax # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call i32 @llvm.x86.sse.cvttss2si(<4 x float> %a0)
   ret i32 %2
@@ -752,16 +1507,28 @@ define i32 @stack_fold_cvttss2si_int(<4 x float> %a0) {
 declare i32 @llvm.x86.sse.cvttss2si(<4 x float>) nounwind readnone
 
 define i64 @stack_fold_cvttss2si64(float %a0) {
-  ;CHECK-LABEL: stack_fold_cvttss2si64
-  ;CHECK:  vcvttss2si {{-?[0-9]*}}(%rsp), %rax {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvttss2si64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovss %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcvttss2si {{[-0-9]+}}(%r{{[sb]}}p), %rax # 4-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fptosi float %a0 to i64
   ret i64 %2
 }
 
 define i64 @stack_fold_cvttss2si64_int(<4 x float> %a0) {
-  ;CHECK-LABEL: stack_fold_cvttss2si64_int
-  ;CHECK:  vcvttss2si {{-?[0-9]*}}(%rsp), %rax {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvttss2si64_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcvttss2si {{[-0-9]+}}(%r{{[sb]}}p), %rax # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call i64 @llvm.x86.sse.cvttss2si64(<4 x float> %a0)
   ret i64 %2
@@ -769,48 +1536,84 @@ define i64 @stack_fold_cvttss2si64_int(<4 x float> %a0) {
 declare i64 @llvm.x86.sse.cvttss2si64(<4 x float>) nounwind readnone
 
 define <2 x double> @stack_fold_divpd(<2 x double> %a0, <2 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_divpd
-  ;CHECK:       vdivpd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_divpd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vdivpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fdiv <2 x double> %a0, %a1
   ret <2 x double> %2
 }
 
 define <4 x double> @stack_fold_divpd_ymm(<4 x double> %a0, <4 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_divpd_ymm
-  ;CHECK:       vdivpd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_divpd_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vdivpd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fdiv <4 x double> %a0, %a1
   ret <4 x double> %2
 }
 
 define <4 x float> @stack_fold_divps(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_divps
-  ;CHECK:       vdivps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_divps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vdivps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fdiv <4 x float> %a0, %a1
   ret <4 x float> %2
 }
 
 define <8 x float> @stack_fold_divps_ymm(<8 x float> %a0, <8 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_divps_ymm
-  ;CHECK:       vdivps {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_divps_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vdivps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fdiv <8 x float> %a0, %a1
   ret <8 x float> %2
 }
 
 define double @stack_fold_divsd(double %a0, double %a1) {
-  ;CHECK-LABEL: stack_fold_divsd
-  ;CHECK:       vdivsd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_divsd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovsd %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vdivsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 8-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fdiv double %a0, %a1
   ret double %2
 }
 
 define <2 x double> @stack_fold_divsd_int(<2 x double> %a0, <2 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_divsd_int
-  ;CHECK:       vdivsd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_divsd_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vdivsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = extractelement <2 x double> %a0, i32 0
   %3 = extractelement <2 x double> %a1, i32 0
@@ -820,16 +1623,28 @@ define <2 x double> @stack_fold_divsd_int(<2 x double> %a0, <2 x double> %a1) {
 }
 
 define float @stack_fold_divss(float %a0, float %a1) {
-  ;CHECK-LABEL: stack_fold_divss
-  ;CHECK:       vdivss {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_divss:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vdivss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 4-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fdiv float %a0, %a1
   ret float %2
 }
 
 define <4 x float> @stack_fold_divss_int(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_divss_int
-  ;CHECK:       vdivss {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_divss_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vdivss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = extractelement <4 x float> %a0, i32 0
   %3 = extractelement <4 x float> %a1, i32 0
@@ -839,8 +1654,14 @@ define <4 x float> @stack_fold_divss_int(<4 x float> %a0, <4 x float> %a1) {
 }
 
 define <2 x double> @stack_fold_dppd(<2 x double> %a0, <2 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_dppd
-  ;CHECK:       vdppd $7, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_dppd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vdppd $7, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x double> @llvm.x86.sse41.dppd(<2 x double> %a0, <2 x double> %a1, i8 7)
   ret <2 x double> %2
@@ -848,8 +1669,14 @@ define <2 x double> @stack_fold_dppd(<2 x double> %a0, <2 x double> %a1) {
 declare <2 x double> @llvm.x86.sse41.dppd(<2 x double>, <2 x double>, i8) nounwind readnone
 
 define <4 x float> @stack_fold_dpps(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_dpps
-  ;CHECK:       vdpps $7, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_dpps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vdpps $7, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x float> @llvm.x86.sse41.dpps(<4 x float> %a0, <4 x float> %a1, i8 7)
   ret <4 x float> %2
@@ -857,8 +1684,14 @@ define <4 x float> @stack_fold_dpps(<4 x float> %a0, <4 x float> %a1) {
 declare <4 x float> @llvm.x86.sse41.dpps(<4 x float>, <4 x float>, i8) nounwind readnone
 
 define <8 x float> @stack_fold_dpps_ymm(<8 x float> %a0, <8 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_dpps_ymm
-  ;CHECK:       vdpps $7, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_dpps_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vdpps $7, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x float> @llvm.x86.avx.dp.ps.256(<8 x float> %a0, <8 x float> %a1, i8 7)
   ret <8 x float> %2
@@ -866,17 +1699,60 @@ define <8 x float> @stack_fold_dpps_ymm(<8 x float> %a0, <8 x float> %a1) {
 declare <8 x float> @llvm.x86.avx.dp.ps.256(<8 x float>, <8 x float>, i8) nounwind readnone
 
 define <4 x float> @stack_fold_extractf128(<8 x float> %a0, <8 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_extractf128
-  ;CHECK:       vextractf128 $1, {{%ymm[0-9][0-9]*}}, {{-?[0-9]*}}(%rsp) {{.*#+}} 16-byte Folded Spill
+; CHECK-LABEL: stack_fold_extractf128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vextractf128 $1, %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Folded Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
   %1 = shufflevector <8 x float> %a0, <8 x float> %a1, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
   %2 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   ret <4 x float> %1
 }
 
 define i32 @stack_fold_extractps(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_extractps
-  ;CHECK:       vextractps $1, {{%xmm[0-9][0-9]*}}, {{-?[0-9]*}}(%rsp) {{.*#+}} 4-byte Folded Spill
-  ;CHECK:       movl    {{-?[0-9]*}}(%rsp), %eax {{.*#+}} 4-byte Reload
+; CHECK-LABEL: stack_fold_extractps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    vaddps %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    vextractps $1, %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   ; fadd forces execution domain
   %1 = fadd <4 x float> %a0, %a1
   %2 = extractelement <4 x float> %1, i32 1
@@ -886,8 +1762,14 @@ define i32 @stack_fold_extractps(<4 x float> %a0, <4 x float> %a1) {
 }
 
 define <2 x double> @stack_fold_haddpd(<2 x double> %a0, <2 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_haddpd
-  ;CHECK:       vhaddpd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_haddpd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vhaddpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x double> @llvm.x86.sse3.hadd.pd(<2 x double> %a0, <2 x double> %a1)
   ret <2 x double> %2
@@ -895,8 +1777,14 @@ define <2 x double> @stack_fold_haddpd(<2 x double> %a0, <2 x double> %a1) {
 declare <2 x double> @llvm.x86.sse3.hadd.pd(<2 x double>, <2 x double>) nounwind readnone
 
 define <4 x double> @stack_fold_haddpd_ymm(<4 x double> %a0, <4 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_haddpd_ymm
-  ;CHECK:       vhaddpd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_haddpd_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vhaddpd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x double> @llvm.x86.avx.hadd.pd.256(<4 x double> %a0, <4 x double> %a1)
   ret <4 x double> %2
@@ -904,8 +1792,14 @@ define <4 x double> @stack_fold_haddpd_ymm(<4 x double> %a0, <4 x double> %a1) {
 declare <4 x double> @llvm.x86.avx.hadd.pd.256(<4 x double>, <4 x double>) nounwind readnone
 
 define <4 x float> @stack_fold_haddps(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_haddps
-  ;CHECK:       vhaddps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_haddps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vhaddps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x float> @llvm.x86.sse3.hadd.ps(<4 x float> %a0, <4 x float> %a1)
   ret <4 x float> %2
@@ -913,8 +1807,14 @@ define <4 x float> @stack_fold_haddps(<4 x float> %a0, <4 x float> %a1) {
 declare <4 x float> @llvm.x86.sse3.hadd.ps(<4 x float>, <4 x float>) nounwind readnone
 
 define <8 x float> @stack_fold_haddps_ymm(<8 x float> %a0, <8 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_haddps_ymm
-  ;CHECK:       vhaddps {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_haddps_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vhaddps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x float> @llvm.x86.avx.hadd.ps.256(<8 x float> %a0, <8 x float> %a1)
   ret <8 x float> %2
@@ -922,8 +1822,14 @@ define <8 x float> @stack_fold_haddps_ymm(<8 x float> %a0, <8 x float> %a1) {
 declare <8 x float> @llvm.x86.avx.hadd.ps.256(<8 x float>, <8 x float>) nounwind readnone
 
 define <2 x double> @stack_fold_hsubpd(<2 x double> %a0, <2 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_hsubpd
-  ;CHECK:       vhsubpd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_hsubpd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vhsubpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x double> @llvm.x86.sse3.hsub.pd(<2 x double> %a0, <2 x double> %a1)
   ret <2 x double> %2
@@ -931,8 +1837,14 @@ define <2 x double> @stack_fold_hsubpd(<2 x double> %a0, <2 x double> %a1) {
 declare <2 x double> @llvm.x86.sse3.hsub.pd(<2 x double>, <2 x double>) nounwind readnone
 
 define <4 x double> @stack_fold_hsubpd_ymm(<4 x double> %a0, <4 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_hsubpd_ymm
-  ;CHECK:       vhsubpd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_hsubpd_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vhsubpd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x double> @llvm.x86.avx.hsub.pd.256(<4 x double> %a0, <4 x double> %a1)
   ret <4 x double> %2
@@ -940,8 +1852,14 @@ define <4 x double> @stack_fold_hsubpd_ymm(<4 x double> %a0, <4 x double> %a1) {
 declare <4 x double> @llvm.x86.avx.hsub.pd.256(<4 x double>, <4 x double>) nounwind readnone
 
 define <4 x float> @stack_fold_hsubps(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_hsubps
-  ;CHECK:       vhsubps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_hsubps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vhsubps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x float> @llvm.x86.sse3.hsub.ps(<4 x float> %a0, <4 x float> %a1)
   ret <4 x float> %2
@@ -949,8 +1867,14 @@ define <4 x float> @stack_fold_hsubps(<4 x float> %a0, <4 x float> %a1) {
 declare <4 x float> @llvm.x86.sse3.hsub.ps(<4 x float>, <4 x float>) nounwind readnone
 
 define <8 x float> @stack_fold_hsubps_ymm(<8 x float> %a0, <8 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_hsubps_ymm
-  ;CHECK:       vhsubps {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_hsubps_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vhsubps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x float> @llvm.x86.avx.hsub.ps.256(<8 x float> %a0, <8 x float> %a1)
   ret <8 x float> %2
@@ -958,17 +1882,30 @@ define <8 x float> @stack_fold_hsubps_ymm(<8 x float> %a0, <8 x float> %a1) {
 declare <8 x float> @llvm.x86.avx.hsub.ps.256(<8 x float>, <8 x float>) nounwind readnone
 
 define <8 x float> @stack_fold_insertf128(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_insertf128
-  ;CHECK:       vinsertf128 $1, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_insertf128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vinsertf128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <4 x float> %a0, <4 x float> %a1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   ret <8 x float> %2
 }
 
 define <4 x float> @stack_fold_insertps(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_insertps
-  ;CHECK:       vinsertps $17, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
-  ;CHECK-NEXT:                                                                              {{.*#+}} xmm0 = zero,mem[0],xmm0[2,3]
+; CHECK-LABEL: stack_fold_insertps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vinsertps $17, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = zero,mem[0],xmm0[2,3]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x float> @llvm.x86.sse41.insertps(<4 x float> %a0, <4 x float> %a1, i8 209)
   ret <4 x float> %2
@@ -976,8 +1913,14 @@ define <4 x float> @stack_fold_insertps(<4 x float> %a0, <4 x float> %a1) {
 declare <4 x float> @llvm.x86.sse41.insertps(<4 x float>, <4 x float>, i8) nounwind readnone
 
 define <2 x double> @stack_fold_maxpd(<2 x double> %a0, <2 x double> %a1) #0 {
-  ;CHECK-LABEL: stack_fold_maxpd
-  ;CHECK:       vmaxpd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_maxpd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmaxpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x double> @llvm.x86.sse2.max.pd(<2 x double> %a0, <2 x double> %a1)
   ret <2 x double> %2
@@ -985,16 +1928,28 @@ define <2 x double> @stack_fold_maxpd(<2 x double> %a0, <2 x double> %a1) #0 {
 declare <2 x double> @llvm.x86.sse2.max.pd(<2 x double>, <2 x double>) nounwind readnone
 
 define <2 x double> @stack_fold_maxpd_commutable(<2 x double> %a0, <2 x double> %a1) #1 {
-  ;CHECK-LABEL: stack_fold_maxpd_commutable
-  ;CHECK:       vmaxpd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_maxpd_commutable:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmaxpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x double> @llvm.x86.sse2.max.pd(<2 x double> %a0, <2 x double> %a1)
   ret <2 x double> %2
 }
 
 define <4 x double> @stack_fold_maxpd_ymm(<4 x double> %a0, <4 x double> %a1) #0 {
-  ;CHECK-LABEL: stack_fold_maxpd_ymm
-  ;CHECK:       vmaxpd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_maxpd_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmaxpd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x double> @llvm.x86.avx.max.pd.256(<4 x double> %a0, <4 x double> %a1)
   ret <4 x double> %2
@@ -1002,16 +1957,28 @@ define <4 x double> @stack_fold_maxpd_ymm(<4 x double> %a0, <4 x double> %a1) #0
 declare <4 x double> @llvm.x86.avx.max.pd.256(<4 x double>, <4 x double>) nounwind readnone
 
 define <4 x double> @stack_fold_maxpd_ymm_commutable(<4 x double> %a0, <4 x double> %a1) #1 {
-  ;CHECK-LABEL: stack_fold_maxpd_ymm_commutable
-  ;CHECK:       vmaxpd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_maxpd_ymm_commutable:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmaxpd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x double> @llvm.x86.avx.max.pd.256(<4 x double> %a0, <4 x double> %a1)
   ret <4 x double> %2
 }
 
 define <4 x float> @stack_fold_maxps(<4 x float> %a0, <4 x float> %a1) #0 {
-  ;CHECK-LABEL: stack_fold_maxps
-  ;CHECK:       vmaxps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_maxps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmaxps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x float> @llvm.x86.sse.max.ps(<4 x float> %a0, <4 x float> %a1)
   ret <4 x float> %2
@@ -1019,16 +1986,28 @@ define <4 x float> @stack_fold_maxps(<4 x float> %a0, <4 x float> %a1) #0 {
 declare <4 x float> @llvm.x86.sse.max.ps(<4 x float>, <4 x float>) nounwind readnone
 
 define <4 x float> @stack_fold_maxps_commutable(<4 x float> %a0, <4 x float> %a1) #1 {
-  ;CHECK-LABEL: stack_fold_maxps_commutable
-  ;CHECK:       vmaxps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_maxps_commutable:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmaxps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x float> @llvm.x86.sse.max.ps(<4 x float> %a0, <4 x float> %a1)
   ret <4 x float> %2
 }
 
 define <8 x float> @stack_fold_maxps_ymm(<8 x float> %a0, <8 x float> %a1) #0 {
-  ;CHECK-LABEL: stack_fold_maxps_ymm
-  ;CHECK:       vmaxps {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_maxps_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmaxps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x float> @llvm.x86.avx.max.ps.256(<8 x float> %a0, <8 x float> %a1)
   ret <8 x float> %2
@@ -1036,16 +2015,28 @@ define <8 x float> @stack_fold_maxps_ymm(<8 x float> %a0, <8 x float> %a1) #0 {
 declare <8 x float> @llvm.x86.avx.max.ps.256(<8 x float>, <8 x float>) nounwind readnone
 
 define <8 x float> @stack_fold_maxps_ymm_commutable(<8 x float> %a0, <8 x float> %a1) #1 {
-  ;CHECK-LABEL: stack_fold_maxps_ymm_commutable
-  ;CHECK:       vmaxps {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_maxps_ymm_commutable:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmaxps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x float> @llvm.x86.avx.max.ps.256(<8 x float> %a0, <8 x float> %a1)
   ret <8 x float> %2
 }
 
 define double @stack_fold_maxsd(double %a0, double %a1) #0 {
-  ;CHECK-LABEL: stack_fold_maxsd
-  ;CHECK:       vmaxsd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_maxsd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovsd %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmaxsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 8-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fcmp ogt double %a0, %a1
   %3 = select i1 %2, double %a0, double %a1
@@ -1053,8 +2044,14 @@ define double @stack_fold_maxsd(double %a0, double %a1) #0 {
 }
 
 define double @stack_fold_maxsd_commutable(double %a0, double %a1) #1 {
-  ;CHECK-LABEL: stack_fold_maxsd_commutable
-  ;CHECK:       vmaxsd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_maxsd_commutable:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovsd %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmaxsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 8-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fcmp ogt double %a0, %a1
   %3 = select i1 %2, double %a0, double %a1
@@ -1062,8 +2059,14 @@ define double @stack_fold_maxsd_commutable(double %a0, double %a1) #1 {
 }
 
 define <2 x double> @stack_fold_maxsd_int(<2 x double> %a0, <2 x double> %a1) #0 {
-  ;CHECK-LABEL: stack_fold_maxsd_int
-  ;CHECK:       vmaxsd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_maxsd_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmaxsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x double> @llvm.x86.sse2.max.sd(<2 x double> %a0, <2 x double> %a1)
   ret <2 x double> %2
@@ -1071,8 +2074,14 @@ define <2 x double> @stack_fold_maxsd_int(<2 x double> %a0, <2 x double> %a1) #0
 declare <2 x double> @llvm.x86.sse2.max.sd(<2 x double>, <2 x double>) nounwind readnone
 
 define float @stack_fold_maxss(float %a0, float %a1) #0 {
-  ;CHECK-LABEL: stack_fold_maxss
-  ;CHECK:       vmaxss {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_maxss:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmaxss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 4-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fcmp ogt float %a0, %a1
   %3 = select i1 %2, float %a0, float %a1
@@ -1080,8 +2089,14 @@ define float @stack_fold_maxss(float %a0, float %a1) #0 {
 }
 
 define float @stack_fold_maxss_commutable(float %a0, float %a1) #1 {
-  ;CHECK-LABEL: stack_fold_maxss_commutable
-  ;CHECK:       vmaxss {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_maxss_commutable:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmaxss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 4-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fcmp ogt float %a0, %a1
   %3 = select i1 %2, float %a0, float %a1
@@ -1089,8 +2104,14 @@ define float @stack_fold_maxss_commutable(float %a0, float %a1) #1 {
 }
 
 define <4 x float> @stack_fold_maxss_int(<4 x float> %a0, <4 x float> %a1) #0 {
-  ;CHECK-LABEL: stack_fold_maxss_int
-  ;CHECK:       vmaxss {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_maxss_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmaxss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x float> @llvm.x86.sse.max.ss(<4 x float> %a0, <4 x float> %a1)
   ret <4 x float> %2
@@ -1098,8 +2119,14 @@ define <4 x float> @stack_fold_maxss_int(<4 x float> %a0, <4 x float> %a1) #0 {
 declare <4 x float> @llvm.x86.sse.max.ss(<4 x float>, <4 x float>) nounwind readnone
 
 define <2 x double> @stack_fold_minpd(<2 x double> %a0, <2 x double> %a1) #0 {
-  ;CHECK-LABEL: stack_fold_minpd
-  ;CHECK:       vminpd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_minpd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vminpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x double> @llvm.x86.sse2.min.pd(<2 x double> %a0, <2 x double> %a1)
   ret <2 x double> %2
@@ -1107,16 +2134,28 @@ define <2 x double> @stack_fold_minpd(<2 x double> %a0, <2 x double> %a1) #0 {
 declare <2 x double> @llvm.x86.sse2.min.pd(<2 x double>, <2 x double>) nounwind readnone
 
 define <2 x double> @stack_fold_minpd_commutable(<2 x double> %a0, <2 x double> %a1) #1 {
-  ;CHECK-LABEL: stack_fold_minpd_commutable
-  ;CHECK:       vminpd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_minpd_commutable:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vminpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x double> @llvm.x86.sse2.min.pd(<2 x double> %a0, <2 x double> %a1)
   ret <2 x double> %2
 }
 
 define <4 x double> @stack_fold_minpd_ymm(<4 x double> %a0, <4 x double> %a1) #0 {
-  ;CHECK-LABEL: stack_fold_minpd_ymm
-  ;CHECK:       vminpd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_minpd_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vminpd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x double> @llvm.x86.avx.min.pd.256(<4 x double> %a0, <4 x double> %a1)
   ret <4 x double> %2
@@ -1124,16 +2163,28 @@ define <4 x double> @stack_fold_minpd_ymm(<4 x double> %a0, <4 x double> %a1) #0
 declare <4 x double> @llvm.x86.avx.min.pd.256(<4 x double>, <4 x double>) nounwind readnone
 
 define <4 x double> @stack_fold_minpd_ymm_commutable(<4 x double> %a0, <4 x double> %a1) #1 {
-  ;CHECK-LABEL: stack_fold_minpd_ymm_commutable
-  ;CHECK:       vminpd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_minpd_ymm_commutable:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vminpd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x double> @llvm.x86.avx.min.pd.256(<4 x double> %a0, <4 x double> %a1)
   ret <4 x double> %2
 }
 
 define <4 x float> @stack_fold_minps(<4 x float> %a0, <4 x float> %a1) #0 {
-  ;CHECK-LABEL: stack_fold_minps
-  ;CHECK:       vminps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_minps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vminps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x float> @llvm.x86.sse.min.ps(<4 x float> %a0, <4 x float> %a1)
   ret <4 x float> %2
@@ -1141,16 +2192,28 @@ define <4 x float> @stack_fold_minps(<4 x float> %a0, <4 x float> %a1) #0 {
 declare <4 x float> @llvm.x86.sse.min.ps(<4 x float>, <4 x float>) nounwind readnone
 
 define <4 x float> @stack_fold_minps_commutable(<4 x float> %a0, <4 x float> %a1) #1 {
-  ;CHECK-LABEL: stack_fold_minps_commutable
-  ;CHECK:       vminps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_minps_commutable:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vminps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x float> @llvm.x86.sse.min.ps(<4 x float> %a0, <4 x float> %a1)
   ret <4 x float> %2
 }
 
 define <8 x float> @stack_fold_minps_ymm(<8 x float> %a0, <8 x float> %a1) #0 {
-  ;CHECK-LABEL: stack_fold_minps_ymm
-  ;CHECK:       vminps {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_minps_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vminps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x float> @llvm.x86.avx.min.ps.256(<8 x float> %a0, <8 x float> %a1)
   ret <8 x float> %2
@@ -1158,16 +2221,28 @@ define <8 x float> @stack_fold_minps_ymm(<8 x float> %a0, <8 x float> %a1) #0 {
 declare <8 x float> @llvm.x86.avx.min.ps.256(<8 x float>, <8 x float>) nounwind readnone
 
 define <8 x float> @stack_fold_minps_ymm_commutable(<8 x float> %a0, <8 x float> %a1) #1 {
-  ;CHECK-LABEL: stack_fold_minps_ymm_commutable
-  ;CHECK:       vminps {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_minps_ymm_commutable:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vminps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x float> @llvm.x86.avx.min.ps.256(<8 x float> %a0, <8 x float> %a1)
   ret <8 x float> %2
 }
 
 define double @stack_fold_minsd(double %a0, double %a1) #0 {
-  ;CHECK-LABEL: stack_fold_minsd
-  ;CHECK:       vminsd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_minsd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovsd %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vminsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 8-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fcmp olt double %a0, %a1
   %3 = select i1 %2, double %a0, double %a1
@@ -1175,8 +2250,14 @@ define double @stack_fold_minsd(double %a0, double %a1) #0 {
 }
 
 define double @stack_fold_minsd_commutable(double %a0, double %a1) #1 {
-  ;CHECK-LABEL: stack_fold_minsd_commutable
-  ;CHECK:       vminsd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_minsd_commutable:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovsd %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vminsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 8-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fcmp olt double %a0, %a1
   %3 = select i1 %2, double %a0, double %a1
@@ -1184,8 +2265,14 @@ define double @stack_fold_minsd_commutable(double %a0, double %a1) #1 {
 }
 
 define <2 x double> @stack_fold_minsd_int(<2 x double> %a0, <2 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_minsd_int
-  ;CHECK:       vminsd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_minsd_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vminsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x double> @llvm.x86.sse2.min.sd(<2 x double> %a0, <2 x double> %a1)
   ret <2 x double> %2
@@ -1193,8 +2280,14 @@ define <2 x double> @stack_fold_minsd_int(<2 x double> %a0, <2 x double> %a1) {
 declare <2 x double> @llvm.x86.sse2.min.sd(<2 x double>, <2 x double>) nounwind readnone
 
 define float @stack_fold_minss(float %a0, float %a1) #0 {
-  ;CHECK-LABEL: stack_fold_minss
-  ;CHECK:       vminss {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_minss:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vminss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 4-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fcmp olt float %a0, %a1
   %3 = select i1 %2, float %a0, float %a1
@@ -1202,8 +2295,14 @@ define float @stack_fold_minss(float %a0, float %a1) #0 {
 }
 
 define float @stack_fold_minss_commutable(float %a0, float %a1) #1 {
-  ;CHECK-LABEL: stack_fold_minss_commutable
-  ;CHECK:       vminss {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_minss_commutable:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vminss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 4-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fcmp olt float %a0, %a1
   %3 = select i1 %2, float %a0, float %a1
@@ -1211,8 +2310,14 @@ define float @stack_fold_minss_commutable(float %a0, float %a1) #1 {
 }
 
 define <4 x float> @stack_fold_minss_int(<4 x float> %a0, <4 x float> %a1) #0 {
-  ;CHECK-LABEL: stack_fold_minss_int
-  ;CHECK:       vminss {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_minss_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vminss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x float> @llvm.x86.sse.min.ss(<4 x float> %a0, <4 x float> %a1)
   ret <4 x float> %2
@@ -1220,16 +2325,30 @@ define <4 x float> @stack_fold_minss_int(<4 x float> %a0, <4 x float> %a1) #0 {
 declare <4 x float> @llvm.x86.sse.min.ss(<4 x float>, <4 x float>) nounwind readnone
 
 define <2 x double> @stack_fold_movddup(<2 x double> %a0) {
-  ;CHECK-LABEL: stack_fold_movddup
-  ;CHECK:   vmovddup {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_movddup:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovddup {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = mem[0,0]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <2 x double> %a0, <2 x double> undef, <2 x i32> <i32 0, i32 0>
   ret <2 x double> %2
 }
 
 define <4 x double> @stack_fold_movddup_ymm(<4 x double> %a0) {
-  ;CHECK-LABEL: stack_fold_movddup_ymm
-  ;CHECK:   vmovddup {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_movddup_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovddup {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    # ymm0 = mem[0,0,2,2]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <4 x double> %a0, <4 x double> undef, <4 x i32> <i32 0, i32 0, i32 2, i32 2>
   ret <4 x double> %2
@@ -1242,80 +2361,144 @@ define <4 x double> @stack_fold_movddup_ymm(<4 x double> %a0) {
 ; TODO stack_fold_movlps (load / store)
 
 define <4 x float> @stack_fold_movshdup(<4 x float> %a0) {
-  ;CHECK-LABEL: stack_fold_movshdup
-  ;CHECK:   vmovshdup {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_movshdup:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovshdup {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = mem[1,1,3,3]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <4 x float> %a0, <4 x float> undef, <4 x i32> <i32 1, i32 1, i32 3, i32 3>
   ret <4 x float> %2
 }
 
 define <8 x float> @stack_fold_movshdup_ymm(<8 x float> %a0) {
-  ;CHECK-LABEL: stack_fold_movshdup_ymm
-  ;CHECK:   vmovshdup {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_movshdup_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovshdup {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    # ymm0 = mem[1,1,3,3,5,5,7,7]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <8 x float> %a0, <8 x float> undef, <8 x i32> <i32 1, i32 1, i32 3, i32 3, i32 5, i32 5, i32 7, i32 7>
   ret <8 x float> %2
 }
 
 define <4 x float> @stack_fold_movsldup(<4 x float> %a0) {
-  ;CHECK-LABEL: stack_fold_movsldup
-  ;CHECK:   vmovsldup {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_movsldup:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovsldup {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = mem[0,0,2,2]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <4 x float> %a0, <4 x float> undef, <4 x i32> <i32 0, i32 0, i32 2, i32 2>
   ret <4 x float> %2
 }
 
 define <8 x float> @stack_fold_movsldup_ymm(<8 x float> %a0) {
-  ;CHECK-LABEL: stack_fold_movsldup_ymm
-  ;CHECK:   vmovsldup {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_movsldup_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovsldup {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    # ymm0 = mem[0,0,2,2,4,4,6,6]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <8 x float> %a0, <8 x float> undef, <8 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6>
   ret <8 x float> %2
 }
 
 define <2 x double> @stack_fold_mulpd(<2 x double> %a0, <2 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_mulpd
-  ;CHECK:       vmulpd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_mulpd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmulpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fmul <2 x double> %a0, %a1
   ret <2 x double> %2
 }
 
 define <4 x double> @stack_fold_mulpd_ymm(<4 x double> %a0, <4 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_mulpd_ymm
-  ;CHECK:       vmulpd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_mulpd_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmulpd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fmul <4 x double> %a0, %a1
   ret <4 x double> %2
 }
 
 define <4 x float> @stack_fold_mulps(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_mulps
-  ;CHECK:       vmulps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_mulps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmulps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fmul <4 x float> %a0, %a1
   ret <4 x float> %2
 }
 
 define <8 x float> @stack_fold_mulps_ymm(<8 x float> %a0, <8 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_mulps_ymm
-  ;CHECK:       vmulps {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_mulps_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmulps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fmul <8 x float> %a0, %a1
   ret <8 x float> %2
 }
 
 define double @stack_fold_mulsd(double %a0, double %a1) {
-  ;CHECK-LABEL: stack_fold_mulsd
-  ;CHECK:       vmulsd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_mulsd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovsd %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmulsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 8-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fmul double %a0, %a1
   ret double %2
 }
 
 define <2 x double> @stack_fold_mulsd_int(<2 x double> %a0, <2 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_mulsd_int
-  ;CHECK:       vmulsd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_mulsd_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmulsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = extractelement <2 x double> %a0, i32 0
   %3 = extractelement <2 x double> %a1, i32 0
@@ -1325,16 +2508,28 @@ define <2 x double> @stack_fold_mulsd_int(<2 x double> %a0, <2 x double> %a1) {
 }
 
 define float @stack_fold_mulss(float %a0, float %a1) {
-  ;CHECK-LABEL: stack_fold_mulss
-  ;CHECK:       vmulss {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_mulss:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmulss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 4-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fmul float %a0, %a1
   ret float %2
 }
 
 define <4 x float> @stack_fold_mulss_int(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_mulss_int
-  ;CHECK:       vmulss {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_mulss_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmulss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = extractelement <4 x float> %a0, i32 0
   %3 = extractelement <4 x float> %a1, i32 0
@@ -1344,8 +2539,16 @@ define <4 x float> @stack_fold_mulss_int(<4 x float> %a0, <4 x float> %a1) {
 }
 
 define <2 x double> @stack_fold_orpd(<2 x double> %a0, <2 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_orpd
-  ;CHECK:       vorpd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_orpd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vorpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = bitcast <2 x double> %a0 to <2 x i64>
   %3 = bitcast <2 x double> %a1 to <2 x i64>
@@ -1357,8 +2560,16 @@ define <2 x double> @stack_fold_orpd(<2 x double> %a0, <2 x double> %a1) {
 }
 
 define <4 x double> @stack_fold_orpd_ymm(<4 x double> %a0, <4 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_orpd_ymm
-  ;CHECK:       vorpd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_orpd_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vorpd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = bitcast <4 x double> %a0 to <4 x i64>
   %3 = bitcast <4 x double> %a1 to <4 x i64>
@@ -1370,8 +2581,16 @@ define <4 x double> @stack_fold_orpd_ymm(<4 x double> %a0, <4 x double> %a1) {
 }
 
 define <4 x float> @stack_fold_orps(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_orps
-  ;CHECK:       vorps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_orps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vorps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vaddps %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = bitcast <4 x float> %a0 to <2 x i64>
   %3 = bitcast <4 x float> %a1 to <2 x i64>
@@ -1383,8 +2602,16 @@ define <4 x float> @stack_fold_orps(<4 x float> %a0, <4 x float> %a1) {
 }
 
 define <8 x float> @stack_fold_orps_ymm(<8 x float> %a0, <8 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_orps_ymm
-  ;CHECK:       vorps {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_orps_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vorps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vaddps %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = bitcast <8 x float> %a0 to <4 x i64>
   %3 = bitcast <8 x float> %a1 to <4 x i64>
@@ -1396,32 +2623,59 @@ define <8 x float> @stack_fold_orps_ymm(<8 x float> %a0, <8 x float> %a1) {
 }
 
 define <8 x float> @stack_fold_perm2f128(<8 x float> %a0, <8 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_perm2f128
-  ;CHECK:   vperm2f128 $33, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_perm2f128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vperm2f128 $33, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    # ymm0 = ymm0[2,3],mem[0,1]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <8 x float> %a0, <8 x float> %a1, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
   ret <8 x float> %2
 }
 
 define <2 x double> @stack_fold_permilpd(<2 x double> %a0) {
-  ;CHECK-LABEL: stack_fold_permilpd
-  ;CHECK:   vpermilpd $1, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_permilpd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = mem[1,0]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <2 x double> %a0, <2 x double> undef, <2 x i32> <i32 1, i32 0>
   ret <2 x double> %2
 }
 
 define <4 x double> @stack_fold_permilpd_ymm(<4 x double> %a0) {
-  ;CHECK-LABEL: stack_fold_permilpd_ymm
-  ;CHECK:   vpermilpd $5, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_permilpd_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermilpd $5, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    # ymm0 = mem[1,0,3,2]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <4 x double> %a0, <4 x double> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
   ret <4 x double> %2
 }
 
 define <2 x double> @stack_fold_permilpdvar(<2 x double> %a0, <2 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_permilpdvar
-  ;CHECK:       vpermilpd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_permilpdvar:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermilpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x double> @llvm.x86.avx.vpermilvar.pd(<2 x double> %a0, <2 x i64> %a1)
   ret <2 x double> %2
@@ -1429,8 +2683,14 @@ define <2 x double> @stack_fold_permilpdvar(<2 x double> %a0, <2 x i64> %a1) {
 declare <2 x double> @llvm.x86.avx.vpermilvar.pd(<2 x double>, <2 x i64>) nounwind readnone
 
 define <4 x double> @stack_fold_permilpdvar_ymm(<4 x double> %a0, <4 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_permilpdvar_ymm
-  ;CHECK:       vpermilpd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_permilpdvar_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermilpd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x double> @llvm.x86.avx.vpermilvar.pd.256(<4 x double> %a0, <4 x i64> %a1)
   ret <4 x double> %2
@@ -1438,24 +2698,44 @@ define <4 x double> @stack_fold_permilpdvar_ymm(<4 x double> %a0, <4 x i64> %a1)
 declare <4 x double> @llvm.x86.avx.vpermilvar.pd.256(<4 x double>, <4 x i64>) nounwind readnone
 
 define <4 x float> @stack_fold_permilps(<4 x float> %a0) {
-  ;CHECK-LABEL: stack_fold_permilps
-  ;CHECK:   vpermilps $27, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_permilps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermilps $27, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = mem[3,2,1,0]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <4 x float> %a0, <4 x float> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
   ret <4 x float> %2
 }
 
 define <8 x float> @stack_fold_permilps_ymm(<8 x float> %a0) {
-  ;CHECK-LABEL: stack_fold_permilps_ymm
-  ;CHECK:   vpermilps $27, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_permilps_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermilps $27, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    # ymm0 = mem[3,2,1,0,7,6,5,4]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <8 x float> %a0, <8 x float> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
   ret <8 x float> %2
 }
 
 define <4 x float> @stack_fold_permilpsvar(<4 x float> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_permilpsvar
-  ;CHECK:       vpermilps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_permilpsvar:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermilps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float> %a0, <4 x i32> %a1)
   ret <4 x float> %2
@@ -1463,8 +2743,14 @@ define <4 x float> @stack_fold_permilpsvar(<4 x float> %a0, <4 x i32> %a1) {
 declare <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float>, <4 x i32>) nounwind readnone
 
 define <8 x float> @stack_fold_permilpsvar_ymm(<8 x float> %a0, <8 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_permilpsvar_ymm
-  ;CHECK:       vpermilps {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_permilpsvar_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermilps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x float> @llvm.x86.avx.vpermilvar.ps.256(<8 x float> %a0, <8 x i32> %a1)
   ret <8 x float> %2
@@ -1474,8 +2760,14 @@ declare <8 x float> @llvm.x86.avx.vpermilvar.ps.256(<8 x float>, <8 x i32>) noun
 ; TODO stack_fold_rcpps
 
 define <4 x float> @stack_fold_rcpps_int(<4 x float> %a0) {
-  ;CHECK-LABEL: stack_fold_rcpps_int
-  ;CHECK:       vrcpps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_rcpps_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vrcpps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x float> @llvm.x86.sse.rcp.ps(<4 x float> %a0)
   ret <4 x float> %2
@@ -1485,8 +2777,14 @@ declare <4 x float> @llvm.x86.sse.rcp.ps(<4 x float>) nounwind readnone
 ; TODO stack_fold_rcpps_ymm
 
 define <8 x float> @stack_fold_rcpps_ymm_int(<8 x float> %a0) {
-  ;CHECK-LABEL: stack_fold_rcpps_ymm_int
-  ;CHECK:       vrcpps {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_rcpps_ymm_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vrcpps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x float> @llvm.x86.avx.rcp.ps.256(<8 x float> %a0)
   ret <8 x float> %2
@@ -1497,8 +2795,14 @@ declare <8 x float> @llvm.x86.avx.rcp.ps.256(<8 x float>) nounwind readnone
 ; TODO stack_fold_rcpss_int
 
 define <2 x double> @stack_fold_roundpd(<2 x double> %a0) {
-  ;CHECK-LABEL: stack_fold_roundpd
-  ;CHECK:  vroundpd $7, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_roundpd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vroundpd $7, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x double> @llvm.x86.sse41.round.pd(<2 x double> %a0, i32 7)
   ret <2 x double> %2
@@ -1506,8 +2810,14 @@ define <2 x double> @stack_fold_roundpd(<2 x double> %a0) {
 declare <2 x double> @llvm.x86.sse41.round.pd(<2 x double>, i32) nounwind readnone
 
 define <4 x double> @stack_fold_roundpd_ymm(<4 x double> %a0) {
-  ;CHECK-LABEL: stack_fold_roundpd_ymm
-  ;CHECK:  vroundpd $7, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_roundpd_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vroundpd $7, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x double> @llvm.x86.avx.round.pd.256(<4 x double> %a0, i32 7)
   ret <4 x double> %2
@@ -1515,8 +2825,14 @@ define <4 x double> @stack_fold_roundpd_ymm(<4 x double> %a0) {
 declare <4 x double> @llvm.x86.avx.round.pd.256(<4 x double>, i32) nounwind readnone
 
 define <4 x float> @stack_fold_roundps(<4 x float> %a0) {
-  ;CHECK-LABEL: stack_fold_roundps
-  ;CHECK:  vroundps $7, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_roundps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vroundps $7, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x float> @llvm.x86.sse41.round.ps(<4 x float> %a0, i32 7)
   ret <4 x float> %2
@@ -1524,8 +2840,14 @@ define <4 x float> @stack_fold_roundps(<4 x float> %a0) {
 declare <4 x float> @llvm.x86.sse41.round.ps(<4 x float>, i32) nounwind readnone
 
 define <8 x float> @stack_fold_roundps_ymm(<8 x float> %a0) {
-  ;CHECK-LABEL: stack_fold_roundps_ymm
-  ;CHECK:  vroundps $7, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_roundps_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vroundps $7, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x float> @llvm.x86.avx.round.ps.256(<8 x float> %a0, i32 7)
   ret <8 x float> %2
@@ -1533,8 +2855,29 @@ define <8 x float> @stack_fold_roundps_ymm(<8 x float> %a0) {
 declare <8 x float> @llvm.x86.avx.round.ps.256(<8 x float>, i32) nounwind readnone
 
 define double @stack_fold_roundsd(double %a0) optsize {
-  ;CHECK-LABEL: stack_fold_roundsd
-  ;CHECK:       vroundsd $9, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_roundsd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovsd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; CHECK-NEXT:    vroundsd $9, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 8-byte Folded Reload
+; CHECK-NEXT:    retq
+  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
+  %2 = call double @llvm.floor.f64(double %a0)
+  ret double %2
+}
+
+define double @stack_fold_roundsd_minsize(double %a0) minsize {
+; CHECK-LABEL: stack_fold_roundsd_minsize:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovsd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vroundsd $9, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 8-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call double @llvm.floor.f64(double %a0)
   ret double %2
@@ -1542,8 +2885,16 @@ define double @stack_fold_roundsd(double %a0) optsize {
 declare double @llvm.floor.f64(double) nounwind readnone
 
 define <2 x double> @stack_fold_roundsd_int(<2 x double> %a0, <2 x double> %a1) optsize {
-  ;CHECK-LABEL: stack_fold_roundsd_int
-  ;CHECK:       vroundsd $7, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_roundsd_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovapd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    vroundsd $7, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x double> @llvm.x86.sse41.round.sd(<2 x double> %a0, <2 x double> %a1, i32 7)
   ret <2 x double> %2
@@ -1551,8 +2902,15 @@ define <2 x double> @stack_fold_roundsd_int(<2 x double> %a0, <2 x double> %a1)
 declare <2 x double> @llvm.x86.sse41.round.sd(<2 x double>, <2 x double>, i32) nounwind readnone
 
 define float @stack_fold_roundss(float %a0) optsize {
-  ;CHECK-LABEL: stack_fold_roundss
-  ;CHECK:       vroundss $9, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_roundss:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovss %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; CHECK-NEXT:    vroundss $9, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 4-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call float @llvm.floor.f32(float %a0)
   ret float %2
@@ -1560,8 +2918,16 @@ define float @stack_fold_roundss(float %a0) optsize {
 declare float @llvm.floor.f32(float) nounwind readnone
 
 define <4 x float> @stack_fold_roundss_int(<4 x float> %a0, <4 x float> %a1) optsize {
-  ;CHECK-LABEL: stack_fold_roundss_int
-  ;CHECK:       vroundss $7, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_roundss_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    vroundss $7, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x float> @llvm.x86.sse41.round.ss(<4 x float> %a0, <4 x float> %a1, i32 7)
   ret <4 x float> %2
@@ -1571,8 +2937,14 @@ declare <4 x float> @llvm.x86.sse41.round.ss(<4 x float>, <4 x float>, i32) noun
 ; TODO stack_fold_rsqrtps
 
 define <4 x float> @stack_fold_rsqrtps_int(<4 x float> %a0) {
-  ;CHECK-LABEL: stack_fold_rsqrtps_int
-  ;CHECK:       vrsqrtps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_rsqrtps_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vrsqrtps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x float> @llvm.x86.sse.rsqrt.ps(<4 x float> %a0)
   ret <4 x float> %2
@@ -1582,8 +2954,14 @@ declare <4 x float> @llvm.x86.sse.rsqrt.ps(<4 x float>) nounwind readnone
 ; TODO stack_fold_rsqrtps_ymm
 
 define <8 x float> @stack_fold_rsqrtps_ymm_int(<8 x float> %a0) {
-  ;CHECK-LABEL: stack_fold_rsqrtps_ymm_int
-  ;CHECK:       vrsqrtps {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_rsqrtps_ymm_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vrsqrtps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x float> @llvm.x86.avx.rsqrt.ps.256(<8 x float> %a0)
   ret <8 x float> %2
@@ -1594,8 +2972,17 @@ declare <8 x float> @llvm.x86.avx.rsqrt.ps.256(<8 x float>) nounwind readnone
 ; TODO stack_fold_rsqrtss_int
 
 define <2 x double> @stack_fold_shufpd(<2 x double> %a0, <2 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_shufpd
-  ;CHECK:       vshufpd $1, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_shufpd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vshufpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = xmm0[1],mem[0]
+; CHECK-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <2 x double> %a0, <2 x double> %a1, <2 x i32> <i32 1, i32 2>
   ; fadd forces execution domain
@@ -1604,8 +2991,17 @@ define <2 x double> @stack_fold_shufpd(<2 x double> %a0, <2 x double> %a1) {
 }
 
 define <4 x double> @stack_fold_shufpd_ymm(<4 x double> %a0, <4 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_shufpd_ymm
-  ;CHECK:       vshufpd $5, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_shufpd_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vshufpd $5, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    # ymm0 = ymm0[1],mem[0],ymm0[3],mem[2]
+; CHECK-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <4 x double> %a0, <4 x double> %a1, <4 x i32> <i32 1, i32 4, i32 3, i32 6>
   ; fadd forces execution domain
@@ -1614,24 +3010,44 @@ define <4 x double> @stack_fold_shufpd_ymm(<4 x double> %a0, <4 x double> %a1) {
 }
 
 define <4 x float> @stack_fold_shufps(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_shufps
-  ;CHECK:       vshufps $200, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_shufps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vshufps $200, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = xmm0[0,2],mem[0,3]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <4 x float> %a0, <4 x float> %a1, <4 x i32> <i32 0, i32 2, i32 4, i32 7>
   ret <4 x float> %2
 }
 
 define <8 x float> @stack_fold_shufps_ymm(<8 x float> %a0, <8 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_shufps_ymm
-  ;CHECK:       vshufps $148, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_shufps_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vshufps $148, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    # ymm0 = ymm0[0,1],mem[1,2],ymm0[4,5],mem[5,6]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <8 x float> %a0, <8 x float> %a1, <8 x i32> <i32 0, i32 1, i32 9, i32 10, i32 4, i32 5, i32 13, i32 14>
   ret <8 x float> %2
 }
 
 define <2 x double> @stack_fold_sqrtpd(<2 x double> %a0) {
-  ;CHECK-LABEL: stack_fold_sqrtpd
-  ;CHECK:       vsqrtpd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_sqrtpd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vsqrtpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x double> @llvm.sqrt.v2f64(<2 x double> %a0)
   ret <2 x double> %2
@@ -1639,8 +3055,14 @@ define <2 x double> @stack_fold_sqrtpd(<2 x double> %a0) {
 declare <2 x double> @llvm.sqrt.v2f64(<2 x double>)
 
 define <4 x double> @stack_fold_sqrtpd_ymm(<4 x double> %a0) {
-  ;CHECK-LABEL: stack_fold_sqrtpd_ymm
-  ;CHECK:       vsqrtpd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_sqrtpd_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vsqrtpd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x double> @llvm.sqrt.v4f64(<4 x double> %a0)
   ret <4 x double> %2
@@ -1648,8 +3070,14 @@ define <4 x double> @stack_fold_sqrtpd_ymm(<4 x double> %a0) {
 declare <4 x double> @llvm.sqrt.v4f64(<4 x double>)
 
 define <4 x float> @stack_fold_sqrtps(<4 x float> %a0) {
-  ;CHECK-LABEL: stack_fold_sqrtps
-  ;CHECK:       vsqrtps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_sqrtps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vsqrtps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x float> @llvm.sqrt.v4f32(<4 x float> %a0)
   ret <4 x float> %2
@@ -1657,8 +3085,14 @@ define <4 x float> @stack_fold_sqrtps(<4 x float> %a0) {
 declare <4 x float> @llvm.sqrt.v4f32(<4 x float>)
 
 define <8 x float> @stack_fold_sqrtps_ymm(<8 x float> %a0) {
-  ;CHECK-LABEL: stack_fold_sqrtps_ymm
-  ;CHECK:       vsqrtps {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_sqrtps_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vsqrtps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x float> @llvm.sqrt.v8f32(<8 x float> %a0)
   ret <8 x float> %2
@@ -1666,8 +3100,15 @@ define <8 x float> @stack_fold_sqrtps_ymm(<8 x float> %a0) {
 declare <8 x float> @llvm.sqrt.v8f32(<8 x float>)
 
 define double @stack_fold_sqrtsd(double %a0) optsize {
-  ;CHECK-LABEL: stack_fold_sqrtsd
-  ;CHECK:       vsqrtsd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_sqrtsd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovsd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; CHECK-NEXT:    vsqrtsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 8-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call double @llvm.sqrt.f64(double %a0)
   ret double %2
@@ -1677,8 +3118,15 @@ declare double @llvm.sqrt.f64(double) nounwind readnone
 ; TODO stack_fold_sqrtsd_int
 
 define float @stack_fold_sqrtss(float %a0) optsize {
-  ;CHECK-LABEL: stack_fold_sqrtss
-  ;CHECK:       vsqrtss {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_sqrtss:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovss %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; CHECK-NEXT:    vsqrtss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 4-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call float @llvm.sqrt.f32(float %a0)
   ret float %2
@@ -1688,48 +3136,84 @@ declare float @llvm.sqrt.f32(float) nounwind readnone
 ; TODO stack_fold_sqrtss_int
 
 define <2 x double> @stack_fold_subpd(<2 x double> %a0, <2 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_subpd
-  ;CHECK:       vsubpd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_subpd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vsubpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fsub <2 x double> %a0, %a1
   ret <2 x double> %2
 }
 
 define <4 x double> @stack_fold_subpd_ymm(<4 x double> %a0, <4 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_subpd_ymm
-  ;CHECK:       vsubpd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_subpd_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vsubpd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fsub <4 x double> %a0, %a1
   ret <4 x double> %2
 }
 
 define <4 x float> @stack_fold_subps(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_subps
-  ;CHECK:       vsubps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_subps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vsubps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fsub <4 x float> %a0, %a1
   ret <4 x float> %2
 }
 
 define <8 x float> @stack_fold_subps_ymm(<8 x float> %a0, <8 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_subps_ymm
-  ;CHECK:       vsubps {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_subps_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vsubps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fsub <8 x float> %a0, %a1
   ret <8 x float> %2
 }
 
 define double @stack_fold_subsd(double %a0, double %a1) {
-  ;CHECK-LABEL: stack_fold_subsd
-  ;CHECK:       vsubsd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_subsd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovsd %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vsubsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 8-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fsub double %a0, %a1
   ret double %2
 }
 
 define <2 x double> @stack_fold_subsd_int(<2 x double> %a0, <2 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_subsd_int
-  ;CHECK:       vsubsd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_subsd_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vsubsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = extractelement <2 x double> %a0, i32 0
   %3 = extractelement <2 x double> %a1, i32 0
@@ -1739,16 +3223,28 @@ define <2 x double> @stack_fold_subsd_int(<2 x double> %a0, <2 x double> %a1) {
 }
 
 define float @stack_fold_subss(float %a0, float %a1) {
-  ;CHECK-LABEL: stack_fold_subss
-  ;CHECK:       vsubss {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_subss:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vsubss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 4-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fsub float %a0, %a1
   ret float %2
 }
 
 define <4 x float> @stack_fold_subss_int(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_subss_int
-  ;CHECK:       vsubss {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_subss_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vsubss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = extractelement <4 x float> %a0, i32 0
   %3 = extractelement <4 x float> %a1, i32 0
@@ -1758,8 +3254,16 @@ define <4 x float> @stack_fold_subss_int(<4 x float> %a0, <4 x float> %a1) {
 }
 
 define i32 @stack_fold_testpd(<2 x double> %a0, <2 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_testpd
-  ;CHECK:       vtestpd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_testpd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    vtestpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    setb %al
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call i32 @llvm.x86.avx.vtestc.pd(<2 x double> %a0, <2 x double> %a1)
   ret i32 %2
@@ -1767,8 +3271,17 @@ define i32 @stack_fold_testpd(<2 x double> %a0, <2 x double> %a1) {
 declare i32 @llvm.x86.avx.vtestc.pd(<2 x double>, <2 x double>) nounwind readnone
 
 define i32 @stack_fold_testpd_ymm(<4 x double> %a0, <4 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_testpd_ymm
-  ;CHECK:       vtestpd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_testpd_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    vtestpd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    setb %al
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call i32 @llvm.x86.avx.vtestc.pd.256(<4 x double> %a0, <4 x double> %a1)
   ret i32 %2
@@ -1776,8 +3289,16 @@ define i32 @stack_fold_testpd_ymm(<4 x double> %a0, <4 x double> %a1) {
 declare i32 @llvm.x86.avx.vtestc.pd.256(<4 x double>, <4 x double>) nounwind readnone
 
 define i32 @stack_fold_testps(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_testps
-  ;CHECK:       vtestps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_testps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    vtestps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    setb %al
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call i32 @llvm.x86.avx.vtestc.ps(<4 x float> %a0, <4 x float> %a1)
   ret i32 %2
@@ -1785,8 +3306,17 @@ define i32 @stack_fold_testps(<4 x float> %a0, <4 x float> %a1) {
 declare i32 @llvm.x86.avx.vtestc.ps(<4 x float>, <4 x float>) nounwind readnone
 
 define i32 @stack_fold_testps_ymm(<8 x float> %a0, <8 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_testps_ymm
-  ;CHECK:       vtestps {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_testps_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    vtestps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    setb %al
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call i32 @llvm.x86.avx.vtestc.ps.256(<8 x float> %a0, <8 x float> %a1)
   ret i32 %2
@@ -1794,8 +3324,17 @@ define i32 @stack_fold_testps_ymm(<8 x float> %a0, <8 x float> %a1) {
 declare i32 @llvm.x86.avx.vtestc.ps.256(<8 x float>, <8 x float>) nounwind readnone
 
 define i32 @stack_fold_ucomisd(double %a0, double %a1) {
-  ;CHECK-LABEL: stack_fold_ucomisd
-  ;CHECK:       vucomisd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_ucomisd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovsd %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    vucomisd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Folded Reload
+; CHECK-NEXT:    sete %al
+; CHECK-NEXT:    leal -1(%rax,%rax), %eax
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fcmp ueq double %a0, %a1
   %3 = select i1 %2, i32 1, i32 -1
@@ -1803,8 +3342,18 @@ define i32 @stack_fold_ucomisd(double %a0, double %a1) {
 }
 
 define i32 @stack_fold_ucomisd_int(<2 x double> %a0, <2 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_ucomisd_int
-  ;CHECK:       vucomisd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_ucomisd_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vucomisd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    setnp %al
+; CHECK-NEXT:    sete %cl
+; CHECK-NEXT:    andb %al, %cl
+; CHECK-NEXT:    movzbl %cl, %eax
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call i32 @llvm.x86.sse2.ucomieq.sd(<2 x double> %a0, <2 x double> %a1)
   ret i32 %2
@@ -1812,8 +3361,17 @@ define i32 @stack_fold_ucomisd_int(<2 x double> %a0, <2 x double> %a1) {
 declare i32 @llvm.x86.sse2.ucomieq.sd(<2 x double>, <2 x double>) nounwind readnone
 
 define i32 @stack_fold_ucomiss(float %a0, float %a1) {
-  ;CHECK-LABEL: stack_fold_ucomiss
-  ;CHECK:       vucomiss {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_ucomiss:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    vucomiss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
+; CHECK-NEXT:    sete %al
+; CHECK-NEXT:    leal -1(%rax,%rax), %eax
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fcmp ueq float %a0, %a1
   %3 = select i1 %2, i32 1, i32 -1
@@ -1821,8 +3379,18 @@ define i32 @stack_fold_ucomiss(float %a0, float %a1) {
 }
 
 define i32 @stack_fold_ucomiss_int(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_ucomiss_int
-  ;CHECK:       vucomiss {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_ucomiss_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vucomiss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    setnp %al
+; CHECK-NEXT:    sete %cl
+; CHECK-NEXT:    andb %al, %cl
+; CHECK-NEXT:    movzbl %cl, %eax
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call i32 @llvm.x86.sse.ucomieq.ss(<4 x float> %a0, <4 x float> %a1)
   ret i32 %2
@@ -1830,8 +3398,17 @@ define i32 @stack_fold_ucomiss_int(<4 x float> %a0, <4 x float> %a1) {
 declare i32 @llvm.x86.sse.ucomieq.ss(<4 x float>, <4 x float>) nounwind readnone
 
 define <2 x double> @stack_fold_unpckhpd(<2 x double> %a0, <2 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_unpckhpd
-  ;CHECK:       vunpckhpd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_unpckhpd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vunpckhpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = xmm0[1],mem[1]
+; CHECK-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <2 x double> %a0, <2 x double> %a1, <2 x i32> <i32 1, i32 3>
   ; fadd forces execution domain
@@ -1840,8 +3417,17 @@ define <2 x double> @stack_fold_unpckhpd(<2 x double> %a0, <2 x double> %a1) {
 }
 
 define <4 x double> @stack_fold_unpckhpd_ymm(<4 x double> %a0, <4 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_unpckhpd_ymm
-  ;CHECK:       vunpckhpd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_unpckhpd_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vunpckhpd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    # ymm0 = ymm0[1],mem[1],ymm0[3],mem[3]
+; CHECK-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <4 x double> %a0, <4 x double> %a1, <4 x i32> <i32 1, i32 5, i32 3, i32 7>
   ; fadd forces execution domain
@@ -1850,8 +3436,17 @@ define <4 x double> @stack_fold_unpckhpd_ymm(<4 x double> %a0, <4 x double> %a1)
 }
 
 define <4 x float> @stack_fold_unpckhps(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_unpckhps
-  ;CHECK:       vunpckhps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_unpckhps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = xmm0[2],mem[2],xmm0[3],mem[3]
+; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vaddps %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <4 x float> %a0, <4 x float> %a1, <4 x i32> <i32 2, i32 6, i32 3, i32 7>
   ; fadd forces execution domain
@@ -1860,8 +3455,17 @@ define <4 x float> @stack_fold_unpckhps(<4 x float> %a0, <4 x float> %a1) {
 }
 
 define <8 x float> @stack_fold_unpckhps_ymm(<8 x float> %a0, <8 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_unpckhps_ymm
-  ;CHECK:       vunpckhps {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_unpckhps_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    # ymm0 = ymm0[2],mem[2],ymm0[3],mem[3],ymm0[6],mem[6],ymm0[7],mem[7]
+; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vaddps %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <8 x float> %a0, <8 x float> %a1, <8 x i32> <i32 2, i32 10, i32 3, i32 11, i32 6, i32 14, i32 7, i32 15>
   ; fadd forces execution domain
@@ -1870,8 +3474,17 @@ define <8 x float> @stack_fold_unpckhps_ymm(<8 x float> %a0, <8 x float> %a1) {
 }
 
 define <2 x double> @stack_fold_unpcklpd(<2 x double> %a0, <2 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_unpcklpd
-  ;CHECK:       vunpcklpd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_unpcklpd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vunpcklpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = xmm0[0],mem[0]
+; CHECK-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <2 x double> %a0, <2 x double> %a1, <2 x i32> <i32 0, i32 2>
   ; fadd forces execution domain
@@ -1880,8 +3493,17 @@ define <2 x double> @stack_fold_unpcklpd(<2 x double> %a0, <2 x double> %a1) {
 }
 
 define <4 x double> @stack_fold_unpcklpd_ymm(<4 x double> %a0, <4 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_unpcklpd_ymm
-  ;CHECK:       vunpcklpd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_unpcklpd_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vunpcklpd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    # ymm0 = ymm0[0],mem[0],ymm0[2],mem[2]
+; CHECK-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <4 x double> %a0, <4 x double> %a1, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
   ; fadd forces execution domain
@@ -1890,8 +3512,17 @@ define <4 x double> @stack_fold_unpcklpd_ymm(<4 x double> %a0, <4 x double> %a1)
 }
 
 define <4 x float> @stack_fold_unpcklps(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_unpcklps
-  ;CHECK:       vunpcklps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_unpcklps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vunpcklps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
+; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vaddps %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <4 x float> %a0, <4 x float> %a1, <4 x i32> <i32 0, i32 4, i32 1, i32 5>
   ; fadd forces execution domain
@@ -1900,8 +3531,17 @@ define <4 x float> @stack_fold_unpcklps(<4 x float> %a0, <4 x float> %a1) {
 }
 
 define <8 x float> @stack_fold_unpcklps_ymm(<8 x float> %a0, <8 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_unpcklps_ymm
-  ;CHECK:       vunpcklps {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_unpcklps_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vunpcklps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    # ymm0 = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[4],mem[4],ymm0[5],mem[5]
+; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vaddps %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <8 x float> %a0, <8 x float> %a1, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 4, i32 12, i32 5, i32 13>
   ; fadd forces execution domain
@@ -1910,8 +3550,16 @@ define <8 x float> @stack_fold_unpcklps_ymm(<8 x float> %a0, <8 x float> %a1) {
 }
 
 define <2 x double> @stack_fold_xorpd(<2 x double> %a0, <2 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_xorpd
-  ;CHECK:       vxorpd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_xorpd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vxorpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = bitcast <2 x double> %a0 to <2 x i64>
   %3 = bitcast <2 x double> %a1 to <2 x i64>
@@ -1923,8 +3571,16 @@ define <2 x double> @stack_fold_xorpd(<2 x double> %a0, <2 x double> %a1) {
 }
 
 define <4 x double> @stack_fold_xorpd_ymm(<4 x double> %a0, <4 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_xorpd_ymm
-  ;CHECK:       vxorpd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_xorpd_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vxorpd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = bitcast <4 x double> %a0 to <4 x i64>
   %3 = bitcast <4 x double> %a1 to <4 x i64>
@@ -1936,8 +3592,16 @@ define <4 x double> @stack_fold_xorpd_ymm(<4 x double> %a0, <4 x double> %a1) {
 }
 
 define <4 x float> @stack_fold_xorps(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_xorps
-  ;CHECK:       vxorps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_xorps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vxorps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vaddps %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = bitcast <4 x float> %a0 to <2 x i64>
   %3 = bitcast <4 x float> %a1 to <2 x i64>
@@ -1949,8 +3613,16 @@ define <4 x float> @stack_fold_xorps(<4 x float> %a0, <4 x float> %a1) {
 }
 
 define <8 x float> @stack_fold_xorps_ymm(<8 x float> %a0, <8 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_xorps_ymm
-  ;CHECK:       vxorps {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_xorps_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vxorps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vaddps %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = bitcast <8 x float> %a0 to <4 x i64>
   %3 = bitcast <8 x float> %a1 to <4 x i64>
diff --git a/llvm/test/CodeGen/X86/stack-folding-fp-avx512.ll b/llvm/test/CodeGen/X86/stack-folding-fp-avx512.ll
index 6e926e698..9ad8e7b2b 100644
--- a/llvm/test/CodeGen/X86/stack-folding-fp-avx512.ll
+++ b/llvm/test/CodeGen/X86/stack-folding-fp-avx512.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -O3 -disable-peephole -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512dq < %s | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -9,16 +10,31 @@ target triple = "x86_64-unknown-unknown"
 ; relevant registers and check that the reload is correctly folded into the instruction.
 
 define <8 x double> @stack_fold_addpd_zmm(<8 x double> %a0, <8 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_addpd_zmm
-  ;CHECK:       vaddpd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_addpd_zmm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vaddpd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = fadd <8 x double> %a0, %a1
   ret <8 x double> %2
 }
 
 define <8 x double> @stack_fold_addpd_zmm_k(<8 x double> %a0, <8 x double> %a1, i8 %mask, <8 x double>* %passthru) {
-  ;CHECK-LABEL: stack_fold_addpd_zmm_k:
-  ;CHECK:       vaddpd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[1-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_addpd_zmm_k:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    vmovapd (%rsi), %zmm2
+; CHECK-NEXT:    vaddpd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovapd %zmm2, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = fadd <8 x double> %a0, %a1
   %3 = bitcast i8 %mask to <8 x i1>
@@ -28,8 +44,17 @@ define <8 x double> @stack_fold_addpd_zmm_k(<8 x double> %a0, <8 x double> %a1,
 }
 
 define <8 x double> @stack_fold_addpd_zmm_k_commuted(<8 x double> %a0, <8 x double> %a1, i8 %mask, <8 x double>* %passthru) {
-  ;CHECK-LABEL: stack_fold_addpd_zmm_k_commuted:
-  ;CHECK:       vaddpd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[1-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_addpd_zmm_k_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    vmovapd (%rsi), %zmm2
+; CHECK-NEXT:    vaddpd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovapd %zmm2, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = fadd <8 x double> %a1, %a0
   %3 = bitcast i8 %mask to <8 x i1>
@@ -39,8 +64,15 @@ define <8 x double> @stack_fold_addpd_zmm_k_commuted(<8 x double> %a0, <8 x doub
 }
 
 define <8 x double> @stack_fold_addpd_zmm_kz(<8 x double> %a0, <8 x double> %a1, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_addpd_zmm_kz
-  ;CHECK:       vaddpd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[1-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_addpd_zmm_kz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    vaddpd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = fadd <8 x double> %a1, %a0
   %3 = bitcast i8 %mask to <8 x i1>
@@ -49,16 +81,31 @@ define <8 x double> @stack_fold_addpd_zmm_kz(<8 x double> %a0, <8 x double> %a1,
 }
 
 define <16 x float> @stack_fold_addps_zmm(<16 x float> %a0, <16 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_addps_zmm
-  ;CHECK:       vaddps {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_addps_zmm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vaddps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = fadd <16 x float> %a0, %a1
   ret <16 x float> %2
 }
 
 define <16 x float> @stack_fold_addps_zmm_k(<16 x float> %a0, <16 x float> %a1, i16 %mask, <16 x float>* %passthru) {
-  ;CHECK-LABEL: stack_fold_addps_zmm_k:
-  ;CHECK:       vaddps {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[1-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_addps_zmm_k:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    vmovaps (%rsi), %zmm2
+; CHECK-NEXT:    vaddps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovaps %zmm2, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = fadd <16 x float> %a0, %a1
   %3 = bitcast i16 %mask to <16 x i1>
@@ -68,8 +115,17 @@ define <16 x float> @stack_fold_addps_zmm_k(<16 x float> %a0, <16 x float> %a1,
 }
 
 define <16 x float> @stack_fold_addps_zmm_k_commuted(<16 x float> %a0, <16 x float> %a1, i16 %mask, <16 x float>* %passthru) {
-  ;CHECK-LABEL: stack_fold_addps_zmm_k_commuted:
-  ;CHECK:       vaddps {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[1-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_addps_zmm_k_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    vmovaps (%rsi), %zmm2
+; CHECK-NEXT:    vaddps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovaps %zmm2, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = fadd <16 x float> %a1, %a0
   %3 = bitcast i16 %mask to <16 x i1>
@@ -79,8 +135,15 @@ define <16 x float> @stack_fold_addps_zmm_k_commuted(<16 x float> %a0, <16 x flo
 }
 
 define <16 x float> @stack_fold_addps_zmm_kz(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
-  ;CHECK-LABEL: stack_fold_addps_zmm_kz
-  ;CHECK:       vaddps {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[1-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_addps_zmm_kz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    vaddps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = fadd <16 x float> %a1, %a0
   %3 = bitcast i16 %mask to <16 x i1>
@@ -89,16 +152,28 @@ define <16 x float> @stack_fold_addps_zmm_kz(<16 x float> %a0, <16 x float> %a1,
 }
 
 define double @stack_fold_addsd(double %a0, double %a1) {
-  ;CHECK-LABEL: stack_fold_addsd
-  ;CHECK:       vaddsd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_addsd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovsd %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vaddsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 8-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = fadd double %a0, %a1
   ret double %2
 }
 
 define <2 x double> @stack_fold_addsd_int(<2 x double> %a0, <2 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_addsd_int
-  ;CHECK:       vaddsd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_addsd_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vaddsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = extractelement <2 x double> %a0, i32 0
   %3 = extractelement <2 x double> %a1, i32 0
@@ -108,16 +183,28 @@ define <2 x double> @stack_fold_addsd_int(<2 x double> %a0, <2 x double> %a1) {
 }
 
 define float @stack_fold_addss(float %a0, float %a1) {
-  ;CHECK-LABEL: stack_fold_addss
-  ;CHECK:       vaddss {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_addss:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vaddss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 4-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = fadd float %a0, %a1
   ret float %2
 }
 
 define <4 x float> @stack_fold_addss_int(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_addss_int
-  ;CHECK:       vaddss {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_addss_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vaddss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = extractelement <4 x float> %a0, i32 0
   %3 = extractelement <4 x float> %a1, i32 0
@@ -127,8 +214,16 @@ define <4 x float> @stack_fold_addss_int(<4 x float> %a0, <4 x float> %a1) {
 }
 
 define <8 x double> @stack_fold_andnpd_zmm(<8 x double> %a0, <8 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_andnpd_zmm
-  ;CHECK:       vandnpd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_andnpd_zmm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vandnpd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vaddpd %zmm1, %zmm0, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = bitcast <8 x double> %a0 to <8 x i64>
   %3 = bitcast <8 x double> %a1 to <8 x i64>
@@ -141,8 +236,16 @@ define <8 x double> @stack_fold_andnpd_zmm(<8 x double> %a0, <8 x double> %a1) {
 }
 
 define <16 x float> @stack_fold_andnps_zmm(<16 x float> %a0, <16 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_andnps_zmm
-  ;CHECK:       vandnps {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_andnps_zmm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vandnps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vaddps %zmm1, %zmm0, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = bitcast <16 x float> %a0 to <16 x i32>
   %3 = bitcast <16 x float> %a1 to <16 x i32>
@@ -155,8 +258,16 @@ define <16 x float> @stack_fold_andnps_zmm(<16 x float> %a0, <16 x float> %a1) {
 }
 
 define <8 x double> @stack_fold_andpd_zmm(<8 x double> %a0, <8 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_andpd_zmm
-  ;CHECK:       vandpd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_andpd_zmm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vandpd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vaddpd %zmm1, %zmm0, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = bitcast <8 x double> %a0 to <8 x i64>
   %3 = bitcast <8 x double> %a1 to <8 x i64>
@@ -168,8 +279,16 @@ define <8 x double> @stack_fold_andpd_zmm(<8 x double> %a0, <8 x double> %a1) {
 }
 
 define <16 x float> @stack_fold_andps_zmm(<16 x float> %a0, <16 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_andps_zmm
-  ;CHECK:       vandps {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_andps_zmm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vandps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vaddps %zmm1, %zmm0, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = bitcast <16 x float> %a0 to <16 x i32>
   %3 = bitcast <16 x float> %a1 to <16 x i32>
@@ -181,8 +300,17 @@ define <16 x float> @stack_fold_andps_zmm(<16 x float> %a0, <16 x float> %a1) {
 }
 
 define i8 @stack_fold_cmppd(<8 x double> %a0, <8 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_cmppd
-  ;CHECK:       vcmpeqpd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%k[0-9]}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_cmppd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcmpeqpd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %k0 # 64-byte Folded Reload
+; CHECK-NEXT:    kmovw %k0, %eax
+; CHECK-NEXT:    # kill: def $al killed $al killed $eax
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %res = call <8 x i1> @llvm.x86.avx512.cmp.pd.512(<8 x double> %a0, <8 x double> %a1, i32 0, i32 4)
   %2 = bitcast <8 x i1> %res to i8
@@ -191,8 +319,26 @@ define i8 @stack_fold_cmppd(<8 x double> %a0, <8 x double> %a1) {
 declare <8 x i1> @llvm.x86.avx512.cmp.pd.512(<8 x double>, <8 x double>, i32, i32)
 
 define <8 x double> @stack_fold_cmppd_mask(<8 x double> %a0, <8 x double> %a1, <8 x double>* %a2, i8 %mask, <8 x double> %b0, <8 x double> %b1) {
-  ;CHECK-LABEL: stack_fold_cmppd_mask:
-  ;CHECK:       vcmpeqpd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%k[0-7]}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_cmppd_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subq $184, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 192
+; CHECK-NEXT:    vmovups %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovups %zmm2, (%rsp) # 64-byte Spill
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovupd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; CHECK-NEXT:    vaddpd (%rdi), %zmm0, %zmm0
+; CHECK-NEXT:    kmovw %esi, %k1
+; CHECK-NEXT:    vcmpeqpd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %k1 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovupd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; CHECK-NEXT:    vblendmpd (%rsp), %zmm0, %zmm0 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    addq $184, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   ; load and fadd are here to keep the operations below the side effecting block and to avoid folding the wrong load
   %2 = load <8 x double>, <8 x double>* %a2
@@ -205,8 +351,26 @@ define <8 x double> @stack_fold_cmppd_mask(<8 x double> %a0, <8 x double> %a1, <
 }
 
 define <8 x double> @stack_fold_cmppd_mask_commuted(<8 x double> %a0, <8 x double> %a1, <8 x double>* %a2, i8 %mask, <8 x double> %b0, <8 x double> %b1) {
-  ;CHECK-LABEL: stack_fold_cmppd_mask_commuted:
-  ;CHECK:       vcmpeqpd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%k[0-7]}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_cmppd_mask_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subq $184, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 192
+; CHECK-NEXT:    vmovups %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovups %zmm2, (%rsp) # 64-byte Spill
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovupd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; CHECK-NEXT:    vaddpd (%rdi), %zmm0, %zmm0
+; CHECK-NEXT:    kmovw %esi, %k1
+; CHECK-NEXT:    vcmpeqpd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %k1 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovupd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; CHECK-NEXT:    vblendmpd (%rsp), %zmm0, %zmm0 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    addq $184, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   ; load and fadd are here to keep the operations below the side effecting block and to avoid folding the wrong load
   %2 = load <8 x double>, <8 x double>* %a2
@@ -219,8 +383,17 @@ define <8 x double> @stack_fold_cmppd_mask_commuted(<8 x double> %a0, <8 x doubl
 }
 
 define i16 @stack_fold_cmpps(<16 x float> %a0, <16 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_cmpps
-  ;CHECK:       vcmpeqps {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%k[0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_cmpps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcmpeqps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %k0 # 64-byte Folded Reload
+; CHECK-NEXT:    kmovw %k0, %eax
+; CHECK-NEXT:    # kill: def $ax killed $ax killed $eax
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %res = call <16 x i1> @llvm.x86.avx512.cmp.ps.512(<16 x float> %a0, <16 x float> %a1, i32 0, i32 4)
   %2 = bitcast <16 x i1> %res to i16
@@ -229,8 +402,26 @@ define i16 @stack_fold_cmpps(<16 x float> %a0, <16 x float> %a1) {
 declare <16 x i1> @llvm.x86.avx512.cmp.ps.512(<16 x float>, <16 x float>, i32, i32)
 
 define <16 x float> @stack_fold_cmpps_mask(<16 x float> %a0, <16 x float> %a1, <16 x float>* %a2, i16 %mask, <16 x float> %b0, <16 x float> %b1) {
-  ;CHECK-LABEL: stack_fold_cmpps_mask:
-  ;CHECK:       vcmpeqps {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%k[0-7]}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_cmpps_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subq $184, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 192
+; CHECK-NEXT:    vmovups %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovups %zmm2, (%rsp) # 64-byte Spill
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; CHECK-NEXT:    vaddps (%rdi), %zmm0, %zmm0
+; CHECK-NEXT:    kmovw %esi, %k1
+; CHECK-NEXT:    vcmpeqps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %k1 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; CHECK-NEXT:    vblendmps (%rsp), %zmm0, %zmm0 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    addq $184, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   ; load and fadd are here to keep the operations below the side effecting block and to avoid folding the wrong load
   %2 = load <16 x float>, <16 x float>* %a2
@@ -243,8 +434,26 @@ define <16 x float> @stack_fold_cmpps_mask(<16 x float> %a0, <16 x float> %a1, <
 }
 
 define <16 x float> @stack_fold_cmpps_mask_commuted(<16 x float> %a0, <16 x float> %a1, <16 x float>* %a2, i16 %mask, <16 x float> %b0, <16 x float> %b1) {
-  ;CHECK-LABEL: stack_fold_cmpps_mask_commuted:
-  ;CHECK:       vcmpeqps {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%k[0-7]}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_cmpps_mask_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subq $184, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 192
+; CHECK-NEXT:    vmovups %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovups %zmm2, (%rsp) # 64-byte Spill
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; CHECK-NEXT:    vaddps (%rdi), %zmm0, %zmm0
+; CHECK-NEXT:    kmovw %esi, %k1
+; CHECK-NEXT:    vcmpeqps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %k1 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; CHECK-NEXT:    vblendmps (%rsp), %zmm0, %zmm0 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    addq $184, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   ; load and fadd are here to keep the operations below the side effecting block and to avoid folding the wrong load
   %2 = load <16 x float>, <16 x float>* %a2
@@ -257,8 +466,14 @@ define <16 x float> @stack_fold_cmpps_mask_commuted(<16 x float> %a0, <16 x floa
 }
 
 define <2 x double> @stack_fold_divsd_int(<2 x double> %a0, <2 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_divsd_int
-  ;CHECK:       vdivsd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_divsd_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vdivsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = extractelement <2 x double> %a0, i32 0
   %3 = extractelement <2 x double> %a1, i32 0
@@ -268,16 +483,28 @@ define <2 x double> @stack_fold_divsd_int(<2 x double> %a0, <2 x double> %a1) {
 }
 
 define float @stack_fold_divss(float %a0, float %a1) {
-  ;CHECK-LABEL: stack_fold_divss
-  ;CHECK:       vdivss {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_divss:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vdivss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 4-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = fdiv float %a0, %a1
   ret float %2
 }
 
 define <4 x float> @stack_fold_divss_int(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_divss_int
-  ;CHECK:       vdivss {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_divss_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vdivss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = extractelement <4 x float> %a0, i32 0
   %3 = extractelement <4 x float> %a1, i32 0
@@ -287,32 +514,56 @@ define <4 x float> @stack_fold_divss_int(<4 x float> %a0, <4 x float> %a1) {
 }
 
 define <8 x double> @stack_fold_cvtdq2pd(<8 x i32> %a0) {
-  ;CHECK-LABEL: stack_fold_cvtdq2pd
-  ;CHECK:   vcvtdq2pd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtdq2pd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcvtdq2pd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = sitofp <8 x i32> %a0 to <8 x double>
   ret <8 x double> %2
 }
 
 define <8 x double> @stack_fold_cvtudq2pd(<8 x i32> %a0) {
-  ;CHECK-LABEL: stack_fold_cvtudq2pd
-  ;CHECK:   vcvtudq2pd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtudq2pd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcvtudq2pd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = uitofp <8 x i32> %a0 to <8 x double>
   ret <8 x double> %2
 }
 
 define <8 x float> @stack_fold_cvtpd2ps(<8 x double> %a0) {
-  ;CHECK-LABEL: stack_fold_cvtpd2ps
-  ;CHECK:   vcvtpd2ps {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtpd2ps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcvtpd2ps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = fptrunc <8 x double> %a0 to <8 x float>
   ret <8 x float> %2
 }
 
 define <16 x float> @stack_fold_cvtph2ps(<16 x i16> %a0) {
-  ;CHECK-LABEL: stack_fold_cvtph2ps
-  ;CHECK:   vcvtph2ps {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtph2ps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcvtph2ps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x float> @llvm.x86.avx512.mask.vcvtph2ps.512(<16 x i16> %a0, <16 x float> undef, i16 -1, i32 4)
   ret <16 x float> %2
@@ -320,8 +571,14 @@ define <16 x float> @stack_fold_cvtph2ps(<16 x i16> %a0) {
 declare <16 x float> @llvm.x86.avx512.mask.vcvtph2ps.512(<16 x i16>, <16 x float>, i16, i32) nounwind readonly
 
 define <16 x i16> @stack_fold_cvtps2ph(<16 x float> %a0) {
-  ;CHECK-LABEL: stack_fold_cvtps2ph
-  ;CHECK:   vcvtps2ph $0, {{%zmm[0-9][0-9]*}}, {{-?[0-9]*}}(%rsp) {{.*#+}} 32-byte Folded Spill
+; CHECK-LABEL: stack_fold_cvtps2ph:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vcvtps2ph $0, %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Folded Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; CHECK-NEXT:    retq
   %1 = call <16 x i16> @llvm.x86.avx512.mask.vcvtps2ph.512(<16 x float> %a0, i32 0, <16 x i16> undef, i16 -1)
   %2 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   ret <16 x i16> %1
@@ -329,9 +586,15 @@ define <16 x i16> @stack_fold_cvtps2ph(<16 x float> %a0) {
 declare <16 x i16> @llvm.x86.avx512.mask.vcvtps2ph.512(<16 x float>, i32, <16 x i16>, i16) nounwind readonly
 
 define <4 x float> @stack_fold_insertps(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_insertps
-  ;CHECK:       vinsertps $17, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
-  ;CHECK-NEXT:                                                                              {{.*#+}} xmm0 = zero,mem[0],xmm0[2,3]
+; CHECK-LABEL: stack_fold_insertps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vinsertps $17, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = zero,mem[0],xmm0[2,3]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <4 x float> @llvm.x86.sse41.insertps(<4 x float> %a0, <4 x float> %a1, i8 209)
   ret <4 x float> %2
@@ -339,8 +602,14 @@ define <4 x float> @stack_fold_insertps(<4 x float> %a0, <4 x float> %a1) {
 declare <4 x float> @llvm.x86.sse41.insertps(<4 x float>, <4 x float>, i8) nounwind readnone
 
 define <8 x double> @stack_fold_maxpd_zmm(<8 x double> %a0, <8 x double> %a1) #0 {
-  ;CHECK-LABEL: stack_fold_maxpd_zmm
-  ;CHECK:       vmaxpd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_maxpd_zmm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmaxpd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x double> @llvm.x86.avx512.max.pd.512(<8 x double> %a0, <8 x double> %a1, i32 4)
   ret <8 x double> %2
@@ -348,16 +617,31 @@ define <8 x double> @stack_fold_maxpd_zmm(<8 x double> %a0, <8 x double> %a1) #0
 declare <8 x double> @llvm.x86.avx512.max.pd.512(<8 x double>, <8 x double>, i32) nounwind readnone
 
 define <8 x double> @stack_fold_maxpd_zmm_commutable(<8 x double> %a0, <8 x double> %a1) #1 {
-  ;CHECK-LABEL: stack_fold_maxpd_zmm_commutable
-  ;CHECK:       vmaxpd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_maxpd_zmm_commutable:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmaxpd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x double> @llvm.x86.avx512.max.pd.512(<8 x double> %a0, <8 x double> %a1, i32 4)
   ret <8 x double> %2
 }
 
 define <8 x double> @stack_fold_maxpd_zmm_commutable_k(<8 x double> %a0, <8 x double> %a1, i8 %mask, <8 x double>* %passthru) #1 {
-  ;CHECK-LABEL: stack_fold_maxpd_zmm_commutable_k:
-  ;CHECK:       vmaxpd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[1-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_maxpd_zmm_commutable_k:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    vmovapd (%rsi), %zmm2
+; CHECK-NEXT:    vmaxpd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovapd %zmm2, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x double> @llvm.x86.avx512.max.pd.512(<8 x double> %a0, <8 x double> %a1, i32 4)
   %3 = bitcast i8 %mask to <8 x i1>
@@ -367,8 +651,17 @@ define <8 x double> @stack_fold_maxpd_zmm_commutable_k(<8 x double> %a0, <8 x do
 }
 
 define <8 x double> @stack_fold_maxpd_zmm_commutable_k_commuted(<8 x double> %a0, <8 x double> %a1, i8 %mask, <8 x double>* %passthru) #1 {
-  ;CHECK-LABEL: stack_fold_maxpd_zmm_commutable_k_commuted:
-  ;CHECK:       vmaxpd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[1-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_maxpd_zmm_commutable_k_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    vmovapd (%rsi), %zmm2
+; CHECK-NEXT:    vmaxpd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovapd %zmm2, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x double> @llvm.x86.avx512.max.pd.512(<8 x double> %a1, <8 x double> %a0, i32 4)
   %3 = bitcast i8 %mask to <8 x i1>
@@ -378,8 +671,15 @@ define <8 x double> @stack_fold_maxpd_zmm_commutable_k_commuted(<8 x double> %a0
 }
 
 define <8 x double> @stack_fold_maxpd_zmm_commutable_kz(<8 x double> %a0, <8 x double> %a1, i8 %mask) #1 {
-  ;CHECK-LABEL: stack_fold_maxpd_zmm_commutable_kz
-  ;CHECK:       vmaxpd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[1-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_maxpd_zmm_commutable_kz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    vmaxpd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x double> @llvm.x86.avx512.max.pd.512(<8 x double> %a1, <8 x double> %a0, i32 4)
   %3 = bitcast i8 %mask to <8 x i1>
@@ -388,8 +688,14 @@ define <8 x double> @stack_fold_maxpd_zmm_commutable_kz(<8 x double> %a0, <8 x d
 }
 
 define <16 x float> @stack_fold_maxps_zmm(<16 x float> %a0, <16 x float> %a1) #0 {
-  ;CHECK-LABEL: stack_fold_maxps_zmm
-  ;CHECK:       vmaxps {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_maxps_zmm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmaxps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x float> @llvm.x86.avx512.max.ps.512(<16 x float> %a0, <16 x float> %a1, i32 4)
   ret <16 x float> %2
@@ -397,16 +703,31 @@ define <16 x float> @stack_fold_maxps_zmm(<16 x float> %a0, <16 x float> %a1) #0
 declare <16 x float> @llvm.x86.avx512.max.ps.512(<16 x float>, <16 x float>, i32) nounwind readnone
 
 define <16 x float> @stack_fold_maxps_zmm_commutable(<16 x float> %a0, <16 x float> %a1) #1 {
-  ;CHECK-LABEL: stack_fold_maxps_zmm_commutable
-  ;CHECK:       vmaxps {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_maxps_zmm_commutable:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmaxps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x float> @llvm.x86.avx512.max.ps.512(<16 x float> %a0, <16 x float> %a1, i32 4)
   ret <16 x float> %2
 }
 
 define <16 x float> @stack_fold_maxps_zmm_commutable_k(<16 x float> %a0, <16 x float> %a1, i16 %mask, <16 x float>* %passthru) #1 {
-  ;CHECK-LABEL: stack_fold_maxps_zmm_commutable_k:
-  ;CHECK:       vmaxps {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[1-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_maxps_zmm_commutable_k:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    vmovaps (%rsi), %zmm2
+; CHECK-NEXT:    vmaxps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovaps %zmm2, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x float> @llvm.x86.avx512.max.ps.512(<16 x float> %a0, <16 x float> %a1, i32 4)
   %3 = bitcast i16 %mask to <16 x i1>
@@ -416,8 +737,17 @@ define <16 x float> @stack_fold_maxps_zmm_commutable_k(<16 x float> %a0, <16 x f
 }
 
 define <16 x float> @stack_fold_maxps_zmm_commutable_k_commuted(<16 x float> %a0, <16 x float> %a1, i16 %mask, <16 x float>* %passthru) #1 {
-  ;CHECK-LABEL: stack_fold_maxps_zmm_commutable_k_commuted:
-  ;CHECK:       vmaxps {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[1-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_maxps_zmm_commutable_k_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    vmovaps (%rsi), %zmm2
+; CHECK-NEXT:    vmaxps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovaps %zmm2, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x float> @llvm.x86.avx512.max.ps.512(<16 x float> %a1, <16 x float> %a0, i32 4)
   %3 = bitcast i16 %mask to <16 x i1>
@@ -427,8 +757,15 @@ define <16 x float> @stack_fold_maxps_zmm_commutable_k_commuted(<16 x float> %a0
 }
 
 define <16 x float> @stack_fold_maxps_zmm_commutable_kz(<16 x float> %a0, <16 x float> %a1, i16 %mask) #1 {
-  ;CHECK-LABEL: stack_fold_maxps_zmm_commutable_kz
-  ;CHECK:       vmaxps {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[1-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_maxps_zmm_commutable_kz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    vmaxps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x float> @llvm.x86.avx512.max.ps.512(<16 x float> %a1, <16 x float> %a0, i32 4)
   %3 = bitcast i16 %mask to <16 x i1>
@@ -437,8 +774,14 @@ define <16 x float> @stack_fold_maxps_zmm_commutable_kz(<16 x float> %a0, <16 x
 }
 
 define <8 x double> @stack_fold_minpd_zmm(<8 x double> %a0, <8 x double> %a1) #0 {
-  ;CHECK-LABEL: stack_fold_minpd_zmm
-  ;CHECK:       vminpd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_minpd_zmm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vminpd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x double> @llvm.x86.avx512.min.pd.512(<8 x double> %a0, <8 x double> %a1, i32 4)
   ret <8 x double> %2
@@ -446,16 +789,31 @@ define <8 x double> @stack_fold_minpd_zmm(<8 x double> %a0, <8 x double> %a1) #0
 declare <8 x double> @llvm.x86.avx512.min.pd.512(<8 x double>, <8 x double>, i32) nounwind readnone
 
 define <8 x double> @stack_fold_minpd_zmm_commutable(<8 x double> %a0, <8 x double> %a1) #1 {
-  ;CHECK-LABEL: stack_fold_minpd_zmm_commutable
-  ;CHECK:       vminpd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_minpd_zmm_commutable:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vminpd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x double> @llvm.x86.avx512.min.pd.512(<8 x double> %a0, <8 x double> %a1, i32 4)
   ret <8 x double> %2
 }
 
 define <8 x double> @stack_fold_minpd_zmm_commutable_k(<8 x double> %a0, <8 x double> %a1, i8 %mask, <8 x double>* %passthru) #1 {
-  ;CHECK-LABEL: stack_fold_minpd_zmm_commutable_k:
-  ;CHECK:       vminpd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[1-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_minpd_zmm_commutable_k:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    vmovapd (%rsi), %zmm2
+; CHECK-NEXT:    vminpd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovapd %zmm2, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x double> @llvm.x86.avx512.min.pd.512(<8 x double> %a0, <8 x double> %a1, i32 4)
   %3 = bitcast i8 %mask to <8 x i1>
@@ -465,8 +823,17 @@ define <8 x double> @stack_fold_minpd_zmm_commutable_k(<8 x double> %a0, <8 x do
 }
 
 define <8 x double> @stack_fold_minpd_zmm_commutable_k_commuted(<8 x double> %a0, <8 x double> %a1, i8 %mask, <8 x double>* %passthru) #1 {
-  ;CHECK-LABEL: stack_fold_minpd_zmm_commutable_k_commuted:
-  ;CHECK:       vminpd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[1-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_minpd_zmm_commutable_k_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    vmovapd (%rsi), %zmm2
+; CHECK-NEXT:    vminpd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovapd %zmm2, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x double> @llvm.x86.avx512.min.pd.512(<8 x double> %a1, <8 x double> %a0, i32 4)
   %3 = bitcast i8 %mask to <8 x i1>
@@ -476,8 +843,15 @@ define <8 x double> @stack_fold_minpd_zmm_commutable_k_commuted(<8 x double> %a0
 }
 
 define <8 x double> @stack_fold_minpd_zmm_commutable_kz(<8 x double> %a0, <8 x double> %a1, i8 %mask) #1 {
-  ;CHECK-LABEL: stack_fold_minpd_zmm_commutable_kz
-  ;CHECK:       vminpd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[1-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_minpd_zmm_commutable_kz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    vminpd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x double> @llvm.x86.avx512.min.pd.512(<8 x double> %a1, <8 x double> %a0, i32 4)
   %3 = bitcast i8 %mask to <8 x i1>
@@ -486,8 +860,14 @@ define <8 x double> @stack_fold_minpd_zmm_commutable_kz(<8 x double> %a0, <8 x d
 }
 
 define <16 x float> @stack_fold_minps_zmm(<16 x float> %a0, <16 x float> %a1) #0 {
-  ;CHECK-LABEL: stack_fold_minps_zmm
-  ;CHECK:       vminps {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_minps_zmm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vminps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x float> @llvm.x86.avx512.min.ps.512(<16 x float> %a0, <16 x float> %a1, i32 4)
   ret <16 x float> %2
@@ -495,16 +875,31 @@ define <16 x float> @stack_fold_minps_zmm(<16 x float> %a0, <16 x float> %a1) #0
 declare <16 x float> @llvm.x86.avx512.min.ps.512(<16 x float>, <16 x float>, i32) nounwind readnone
 
 define <16 x float> @stack_fold_minps_zmm_commutable(<16 x float> %a0, <16 x float> %a1) #1 {
-  ;CHECK-LABEL: stack_fold_minps_zmm_commutable
-  ;CHECK:       vminps {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_minps_zmm_commutable:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vminps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x float> @llvm.x86.avx512.min.ps.512(<16 x float> %a0, <16 x float> %a1, i32 4)
   ret <16 x float> %2
 }
 
 define <16 x float> @stack_fold_minps_zmm_commutable_k(<16 x float> %a0, <16 x float> %a1, i16 %mask, <16 x float>* %passthru) #1 {
-  ;CHECK-LABEL: stack_fold_minps_zmm_commutable_k:
-  ;CHECK:       vminps {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[1-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_minps_zmm_commutable_k:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    vmovaps (%rsi), %zmm2
+; CHECK-NEXT:    vminps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovaps %zmm2, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x float> @llvm.x86.avx512.min.ps.512(<16 x float> %a0, <16 x float> %a1, i32 4)
   %3 = bitcast i16 %mask to <16 x i1>
@@ -514,8 +909,17 @@ define <16 x float> @stack_fold_minps_zmm_commutable_k(<16 x float> %a0, <16 x f
 }
 
 define <16 x float> @stack_fold_minps_zmm_commutable_k_commuted(<16 x float> %a0, <16 x float> %a1, i16 %mask, <16 x float>* %passthru) #1 {
-  ;CHECK-LABEL: stack_fold_minps_zmm_commutable_k_commuted:
-  ;CHECK:       vminps {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[1-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_minps_zmm_commutable_k_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    vmovaps (%rsi), %zmm2
+; CHECK-NEXT:    vminps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovaps %zmm2, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x float> @llvm.x86.avx512.min.ps.512(<16 x float> %a1, <16 x float> %a0, i32 4)
   %3 = bitcast i16 %mask to <16 x i1>
@@ -525,8 +929,15 @@ define <16 x float> @stack_fold_minps_zmm_commutable_k_commuted(<16 x float> %a0
 }
 
 define <16 x float> @stack_fold_minps_zmm_commutable_kz(<16 x float> %a0, <16 x float> %a1, i16 %mask) #1 {
-  ;CHECK-LABEL: stack_fold_minps_zmm_commutable_kz
-  ;CHECK:       vminps {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[1-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_minps_zmm_commutable_kz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    vminps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x float> @llvm.x86.avx512.min.ps.512(<16 x float> %a1, <16 x float> %a0, i32 4)
   %3 = bitcast i16 %mask to <16 x i1>
@@ -535,16 +946,31 @@ define <16 x float> @stack_fold_minps_zmm_commutable_kz(<16 x float> %a0, <16 x
 }
 
 define <8 x double> @stack_fold_mulpd_zmm(<8 x double> %a0, <8 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_mulpd_zmm
-  ;CHECK:       vmulpd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_mulpd_zmm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmulpd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = fmul <8 x double> %a0, %a1
   ret <8 x double> %2
 }
 
 define <8 x double> @stack_fold_mulpd_zmm_k(<8 x double> %a0, <8 x double> %a1, i8 %mask, <8 x double>* %passthru) {
-  ;CHECK-LABEL: stack_fold_mulpd_zmm_k:
-  ;CHECK:       vmulpd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[1-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_mulpd_zmm_k:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    vmovapd (%rsi), %zmm2
+; CHECK-NEXT:    vmulpd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovapd %zmm2, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = fmul <8 x double> %a0, %a1
   %3 = bitcast i8 %mask to <8 x i1>
@@ -554,8 +980,17 @@ define <8 x double> @stack_fold_mulpd_zmm_k(<8 x double> %a0, <8 x double> %a1,
 }
 
 define <8 x double> @stack_fold_mulpd_zmm_k_commuted(<8 x double> %a0, <8 x double> %a1, i8 %mask, <8 x double>* %passthru) {
-  ;CHECK-LABEL: stack_fold_mulpd_zmm_k_commuted:
-  ;CHECK:       vmulpd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[1-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_mulpd_zmm_k_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    vmovapd (%rsi), %zmm2
+; CHECK-NEXT:    vmulpd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovapd %zmm2, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = fmul <8 x double> %a1, %a0
   %3 = bitcast i8 %mask to <8 x i1>
@@ -565,8 +1000,15 @@ define <8 x double> @stack_fold_mulpd_zmm_k_commuted(<8 x double> %a0, <8 x doub
 }
 
 define <8 x double> @stack_fold_mulpd_zmm_kz(<8 x double> %a0, <8 x double> %a1, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_mulpd_zmm_kz
-  ;CHECK:       vmulpd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[1-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_mulpd_zmm_kz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    vmulpd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = fmul <8 x double> %a1, %a0
   %3 = bitcast i8 %mask to <8 x i1>
@@ -575,16 +1017,31 @@ define <8 x double> @stack_fold_mulpd_zmm_kz(<8 x double> %a0, <8 x double> %a1,
 }
 
 define <16 x float> @stack_fold_mulps_zmm(<16 x float> %a0, <16 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_mulps_zmm
-  ;CHECK:       vmulps {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_mulps_zmm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmulps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = fmul <16 x float> %a0, %a1
   ret <16 x float> %2
 }
 
 define <16 x float> @stack_fold_mulps_zmm_k(<16 x float> %a0, <16 x float> %a1, i16 %mask, <16 x float>* %passthru) {
-  ;CHECK-LABEL: stack_fold_mulps_zmm_k:
-  ;CHECK:       vmulps {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[1-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_mulps_zmm_k:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    vmovaps (%rsi), %zmm2
+; CHECK-NEXT:    vmulps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovaps %zmm2, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = fmul <16 x float> %a0, %a1
   %3 = bitcast i16 %mask to <16 x i1>
@@ -594,8 +1051,17 @@ define <16 x float> @stack_fold_mulps_zmm_k(<16 x float> %a0, <16 x float> %a1,
 }
 
 define <16 x float> @stack_fold_mulps_zmm_k_commuted(<16 x float> %a0, <16 x float> %a1, i16 %mask, <16 x float>* %passthru) {
-  ;CHECK-LABEL: stack_fold_mulps_zmm_k_commuted:
-  ;CHECK:       vmulps {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[1-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_mulps_zmm_k_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    vmovaps (%rsi), %zmm2
+; CHECK-NEXT:    vmulps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovaps %zmm2, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = fmul <16 x float> %a1, %a0
   %3 = bitcast i16 %mask to <16 x i1>
@@ -605,8 +1071,15 @@ define <16 x float> @stack_fold_mulps_zmm_k_commuted(<16 x float> %a0, <16 x flo
 }
 
 define <16 x float> @stack_fold_mulps_zmm_kz(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
-  ;CHECK-LABEL: stack_fold_mulps_zmm_kz
-  ;CHECK:       vmulps {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[1-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_mulps_zmm_kz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    vmulps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = fmul <16 x float> %a1, %a0
   %3 = bitcast i16 %mask to <16 x i1>
@@ -615,16 +1088,28 @@ define <16 x float> @stack_fold_mulps_zmm_kz(<16 x float> %a0, <16 x float> %a1,
 }
 
 define double @stack_fold_mulsd(double %a0, double %a1) {
-  ;CHECK-LABEL: stack_fold_mulsd
-  ;CHECK:       vmulsd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_mulsd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovsd %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmulsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 8-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = fmul double %a0, %a1
   ret double %2
 }
 
 define <2 x double> @stack_fold_mulsd_int(<2 x double> %a0, <2 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_mulsd_int
-  ;CHECK:       vmulsd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_mulsd_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmulsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = extractelement <2 x double> %a0, i32 0
   %3 = extractelement <2 x double> %a1, i32 0
@@ -634,16 +1119,28 @@ define <2 x double> @stack_fold_mulsd_int(<2 x double> %a0, <2 x double> %a1) {
 }
 
 define float @stack_fold_mulss(float %a0, float %a1) {
-  ;CHECK-LABEL: stack_fold_mulss
-  ;CHECK:       vmulss {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_mulss:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmulss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 4-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = fmul float %a0, %a1
   ret float %2
 }
 
 define <4 x float> @stack_fold_mulss_int(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_mulss_int
-  ;CHECK:       vmulss {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_mulss_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmulss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = extractelement <4 x float> %a0, i32 0
   %3 = extractelement <4 x float> %a1, i32 0
@@ -653,8 +1150,16 @@ define <4 x float> @stack_fold_mulss_int(<4 x float> %a0, <4 x float> %a1) {
 }
 
 define <8 x double> @stack_fold_orpd_zmm(<8 x double> %a0, <8 x double> %a1) #0 {
-  ;CHECK-LABEL: stack_fold_orpd_zmm
-  ;CHECK:       vorpd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_orpd_zmm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vorpd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vaddpd %zmm1, %zmm0, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = bitcast <8 x double> %a0 to <8 x i64>
   %3 = bitcast <8 x double> %a1 to <8 x i64>
@@ -666,8 +1171,16 @@ define <8 x double> @stack_fold_orpd_zmm(<8 x double> %a0, <8 x double> %a1) #0
 }
 
 define <16 x float> @stack_fold_orps_zmm(<16 x float> %a0, <16 x float> %a1) #0 {
-  ;CHECK-LABEL: stack_fold_orps_zmm
-  ;CHECK:       vorps {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_orps_zmm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vorps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vaddps %zmm1, %zmm0, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = bitcast <16 x float> %a0 to <16 x i32>
   %3 = bitcast <16 x float> %a1 to <16 x i32>
@@ -679,16 +1192,45 @@ define <16 x float> @stack_fold_orps_zmm(<16 x float> %a0, <16 x float> %a1) #0
 }
 
 define <8 x double> @stack_fold_shuff64x2(<8 x double> %a, <8 x double> %b) {
-  ;CHECK-LABEL: stack_fold_shuff64x2
-  ;CHECK:   vshuff64x2 $24, {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_shuff64x2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subq $56, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovupd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; CHECK-NEXT:    vshuff64x2 $24, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    # zmm0 = zmm0[0,1,4,5],mem[2,3,0,1]
+; CHECK-NEXT:    addq $56, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 0, i32 1, i32 4, i32 5, i32 10, i32 11, i32 8, i32 9>
   ret <8 x double> %2
 }
 
 define <8 x double> @stack_fold_shuff64x2_mask(<8 x double> %a, <8 x double> %b, i8 %mask, <8 x double>* %passthru) {
-  ;CHECK-LABEL: stack_fold_shuff64x2_mask
-  ;CHECK:   vshuff64x2 $24, {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[1-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_shuff64x2_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subq $56, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    vmovapd (%rsi), %zmm1
+; CHECK-NEXT:    vmovupd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; CHECK-NEXT:    vshuff64x2 $24, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm1 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    # zmm1 {%k1} = zmm0[0,1,4,5],mem[2,3,0,1]
+; CHECK-NEXT:    vmovapd %zmm1, %zmm0
+; CHECK-NEXT:    addq $56, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 0, i32 1, i32 4, i32 5, i32 10, i32 11, i32 8, i32 9>
   %3 = bitcast i8 %mask to <8 x i1>
@@ -699,8 +1241,22 @@ define <8 x double> @stack_fold_shuff64x2_mask(<8 x double> %a, <8 x double> %b,
 }
 
 define <8 x double> @stack_fold_shuff64x2_maskz(<8 x double> %a, <8 x double> %b, i8 %mask, <8 x double>* %passthru) {
-  ;CHECK-LABEL: stack_fold_shuff64x2_maskz
-  ;CHECK:   vshuff64x2 $24, {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[1-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_shuff64x2_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subq $56, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    vmovupd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; CHECK-NEXT:    vshuff64x2 $24, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    # zmm0 {%k1} {z} = zmm0[0,1,4,5],mem[2,3,0,1]
+; CHECK-NEXT:    addq $56, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 0, i32 1, i32 4, i32 5, i32 10, i32 11, i32 8, i32 9>
   %3 = bitcast i8 %mask to <8 x i1>
@@ -709,8 +1265,24 @@ define <8 x double> @stack_fold_shuff64x2_maskz(<8 x double> %a, <8 x double> %b
 }
 
 define <16 x float> @stack_fold_shuff32x4_mask(<16 x float> %a, <16 x float> %b, i16 %mask, <16 x float>* %passthru) {
-  ;CHECK-LABEL: stack_fold_shuff32x4_mask
-  ;CHECK:   vshuff32x4 $20, {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[1-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_shuff32x4_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subq $56, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    vmovaps (%rsi), %zmm1
+; CHECK-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; CHECK-NEXT:    vshuff32x4 $20, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm1 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    # zmm1 {%k1} = zmm0[0,1,2,3,4,5,6,7],mem[4,5,6,7,0,1,2,3]
+; CHECK-NEXT:    vmovaps %zmm1, %zmm0
+; CHECK-NEXT:    addq $56, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <16 x float> %a, <16 x float> %b, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 20, i32 21, i32 22, i32 23, i32 16, i32 17, i32 18, i32 19>
   %3 = bitcast i16 %mask to <16 x i1>
@@ -721,8 +1293,22 @@ define <16 x float> @stack_fold_shuff32x4_mask(<16 x float> %a, <16 x float> %b,
 }
 
 define <16 x float> @stack_fold_shuff32x4_maskz(<16 x float> %a, <16 x float> %b, i16 %mask) {
-  ;CHECK-LABEL: stack_fold_shuff32x4_maskz
-  ;CHECK:   vshuff32x4 $20, {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[1-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_shuff32x4_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subq $56, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; CHECK-NEXT:    vshuff32x4 $20, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    # zmm0 {%k1} {z} = zmm0[0,1,2,3,4,5,6,7],mem[4,5,6,7,0,1,2,3]
+; CHECK-NEXT:    addq $56, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <16 x float> %a, <16 x float> %b, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 20, i32 21, i32 22, i32 23, i32 16, i32 17, i32 18, i32 19>
   %3 = bitcast i16 %mask to <16 x i1>
@@ -731,32 +1317,56 @@ define <16 x float> @stack_fold_shuff32x4_maskz(<16 x float> %a, <16 x float> %b
 }
 
 define <8 x double> @stack_fold_subpd_zmm(<8 x double> %a0, <8 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_subpd_zmm
-  ;CHECK:       vsubpd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_subpd_zmm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vsubpd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = fsub <8 x double> %a0, %a1
   ret <8 x double> %2
 }
 
 define <16 x float> @stack_fold_subps_zmm(<16 x float> %a0, <16 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_subps_zmm
-  ;CHECK:       vsubps {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_subps_zmm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vsubps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = fsub <16 x float> %a0, %a1
   ret <16 x float> %2
 }
 
 define double @stack_fold_subsd(double %a0, double %a1) {
-  ;CHECK-LABEL: stack_fold_subsd
-  ;CHECK:       vsubsd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_subsd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovsd %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vsubsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 8-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = fsub double %a0, %a1
   ret double %2
 }
 
 define <2 x double> @stack_fold_subsd_int(<2 x double> %a0, <2 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_subsd_int
-  ;CHECK:       vsubsd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_subsd_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vsubsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = extractelement <2 x double> %a0, i32 0
   %3 = extractelement <2 x double> %a1, i32 0
@@ -766,16 +1376,28 @@ define <2 x double> @stack_fold_subsd_int(<2 x double> %a0, <2 x double> %a1) {
 }
 
 define float @stack_fold_subss(float %a0, float %a1) {
-  ;CHECK-LABEL: stack_fold_subss
-  ;CHECK:       vsubss {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_subss:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vsubss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 4-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = fsub float %a0, %a1
   ret float %2
 }
 
 define <4 x float> @stack_fold_subss_int(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_subss_int
-  ;CHECK:       vsubss {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_subss_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vsubss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = extractelement <4 x float> %a0, i32 0
   %3 = extractelement <4 x float> %a1, i32 0
@@ -785,8 +1407,16 @@ define <4 x float> @stack_fold_subss_int(<4 x float> %a0, <4 x float> %a1) {
 }
 
 define <8 x double> @stack_fold_xorpd_zmm(<8 x double> %a0, <8 x double> %a1) #0 {
-  ;CHECK-LABEL: stack_fold_xorpd_zmm
-  ;CHECK:       vxorpd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_xorpd_zmm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vxorpd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vaddpd %zmm1, %zmm0, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = bitcast <8 x double> %a0 to <8 x i64>
   %3 = bitcast <8 x double> %a1 to <8 x i64>
@@ -798,8 +1428,16 @@ define <8 x double> @stack_fold_xorpd_zmm(<8 x double> %a0, <8 x double> %a1) #0
 }
 
 define <16 x float> @stack_fold_xorps_zmm(<16 x float> %a0, <16 x float> %a1) #0 {
-  ;CHECK-LABEL: stack_fold_xorps_zmm
-  ;CHECK:       vxorps {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_xorps_zmm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vxorps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vaddps %zmm1, %zmm0, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = bitcast <16 x float> %a0 to <16 x i32>
   %3 = bitcast <16 x float> %a1 to <16 x i32>
@@ -811,9 +1449,44 @@ define <16 x float> @stack_fold_xorps_zmm(<16 x float> %a0, <16 x float> %a1) #0
 }
 
 define i32 @stack_fold_extractps(<4 x float> %a0) {
-  ;CHECK-LABEL: stack_fold_extractps
-  ;CHECK:       vextractps $1, {{%xmm[0-9][0-9]*}}, {{-?[0-9]*}}(%rsp) {{.*#+}} 4-byte Folded Spill
-  ;CHECK:       movl    {{-?[0-9]*}}(%rsp), %eax {{.*#+}} 4-byte Reload
+; CHECK-LABEL: stack_fold_extractps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    vextractps $1, %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = extractelement <4 x float> %a0, i32 1
   %2 = bitcast float %1 to i32
   %3 = tail call <2 x i64> asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
@@ -821,56 +1494,106 @@ define i32 @stack_fold_extractps(<4 x float> %a0) {
 }
 
 define <4 x float> @stack_fold_extracti32x4(<16 x float> %a0) {
-  ;CHECK-LABEL: stack_fold_extracti32x4
-  ;CHECK:       vextractf32x4 $3, {{%zmm[0-9][0-9]*}}, {{-?[0-9]*}}(%rsp) {{.*#+}} 16-byte Folded Spill
+; CHECK-LABEL: stack_fold_extracti32x4:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vextractf32x4 $3, %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Folded Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
   %1 = shufflevector <16 x float> %a0, <16 x float> undef, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
   %2 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   ret <4 x float> %1
 }
 
 define <2 x double> @stack_fold_extractf64x2(<8 x double> %a0) {
-  ;CHECK-LABEL: stack_fold_extractf64x2
-  ;CHECK:       vextractf32x4 $3, {{%zmm[0-9][0-9]*}}, {{-?[0-9]*}}(%rsp) {{.*#+}} 16-byte Folded Spill
+; CHECK-LABEL: stack_fold_extractf64x2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vextractf32x4 $3, %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Folded Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
   %1 = shufflevector <8 x double> %a0, <8 x double> undef, <2 x i32> <i32 6, i32 7>
   %2 = tail call <2 x double> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   ret <2 x double> %1
 }
 
 define <8 x float> @stack_fold_extracti32x8(<16 x float> %a0) {
-  ;CHECK-LABEL: stack_fold_extracti32x8
-  ;CHECK:       vextractf64x4 $1, {{%zmm[0-9][0-9]*}}, {{-?[0-9]*}}(%rsp) {{.*#+}} 32-byte Folded Spill
+; CHECK-LABEL: stack_fold_extracti32x8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vextractf64x4 $1, %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Folded Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; CHECK-NEXT:    retq
   %1 = shufflevector <16 x float> %a0, <16 x float> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
   %2 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   ret <8 x float> %1
 }
 
 define <4 x double> @stack_fold_extractf64x4(<8 x double> %a0) {
-  ;CHECK-LABEL: stack_fold_extractf64x4
-  ;CHECK:       vextractf64x4 $1, {{%zmm[0-9][0-9]*}}, {{-?[0-9]*}}(%rsp) {{.*#+}} 32-byte Folded Spill
+; CHECK-LABEL: stack_fold_extractf64x4:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vextractf64x4 $1, %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Folded Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; CHECK-NEXT:    retq
   %1 = shufflevector <8 x double> %a0, <8 x double> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
   %2 = tail call <2 x double> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   ret <4 x double> %1
 }
 
 define <16 x float> @stack_fold_insertf32x8(<8 x float> %a0, <8 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_insertf32x8
-  ;CHECK:       vinsertf64x4 $1, {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_insertf32x8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vinsertf64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <8 x float> %a0, <8 x float> %a1, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
   ret <16 x float> %2
 }
 
 define <8 x double> @stack_fold_insertf64x4(<4 x double> %a0, <4 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_insertf64x4
-  ;CHECK:       vinsertf64x4 $1, {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_insertf64x4:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vinsertf64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <4 x double> %a0, <4 x double> %a1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   ret <8 x double> %2
 }
 
 define <8 x double> @stack_fold_insertf64x4_mask(<8 x double>* %passthru, <4 x double> %a0, <4 x double> %a1, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_insertf64x4_mask
-  ;CHECK:       vinsertf64x4 $1, {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[1-7]}}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_insertf64x4_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovw %esi, %k1
+; CHECK-NEXT:    vmovapd (%rdi), %zmm2
+; CHECK-NEXT:    vinsertf64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 32-byte Folded Reload
+; CHECK-NEXT:    vmovapd %zmm2, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <4 x double> %a0, <4 x double> %a1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   %3 = bitcast i8 %mask to <8 x i1>
@@ -880,8 +1603,16 @@ define <8 x double> @stack_fold_insertf64x4_mask(<8 x double>* %passthru, <4 x d
 }
 
 define <8 x double> @stack_fold_insertf64x4_maskz(<4 x double> %a0, <4 x double> %a1, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_insertf64x4_maskz
-  ;CHECK:       vinsertf64x4 $1, {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[1-7]}}} {z} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_insertf64x4_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    vinsertf64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <4 x double> %a0, <4 x double> %a1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   %3 = bitcast i8 %mask to <8 x i1>
@@ -890,8 +1621,14 @@ define <8 x double> @stack_fold_insertf64x4_maskz(<4 x double> %a0, <4 x double>
 }
 
 define <16 x float> @stack_fold_vpermt2ps(<16 x float> %x0, <16 x i32> %x1, <16 x float> %x2) {
-  ;CHECK-LABEL: stack_fold_vpermt2ps
-  ;CHECK:       vpermt2ps {{-?[0-9]*}}(%rsp), %zmm1, %zmm0 # 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpermt2ps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermt2ps {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x float> @llvm.x86.avx512.vpermi2var.ps.512(<16 x float> %x0, <16 x i32> %x1, <16 x float> %x2)
   ret <16 x float> %2
@@ -899,16 +1636,31 @@ define <16 x float> @stack_fold_vpermt2ps(<16 x float> %x0, <16 x i32> %x1, <16
 declare <16 x float> @llvm.x86.avx512.vpermi2var.ps.512(<16 x float>, <16 x i32>, <16 x float>)
 
 define <16 x float> @stack_fold_vpermi2ps(<16 x i32> %x0, <16 x float> %x1, <16 x float> %x2) {
-  ;CHECK-LABEL: stack_fold_vpermi2ps
-  ;CHECK:       vpermi2ps {{-?[0-9]*}}(%rsp), %zmm1, %zmm0 # 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpermi2ps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermi2ps {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x float> @llvm.x86.avx512.vpermi2var.ps.512(<16 x float> %x1, <16 x i32> %x0, <16 x float> %x2)
   ret <16 x float> %2
 }
 
 define <16 x float> @stack_fold_vpermi2ps_mask(<16 x float> %x0, <16 x i32>* %x1, <16 x float> %x2, i16 %mask) {
-  ;CHECK-LABEL: stack_fold_vpermi2ps_mask
-  ;CHECK:       vpermi2ps {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpermi2ps_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovaps (%rdi), %zmm2
+; CHECK-NEXT:    kmovw %esi, %k1
+; CHECK-NEXT:    vpermi2ps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovaps %zmm2, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %x1b = load <16 x i32>, <16 x i32>* %x1
   %2 = call <16 x float> @llvm.x86.avx512.vpermi2var.ps.512(<16 x float> %x0, <16 x i32> %x1b, <16 x float> %x2)
@@ -919,8 +1671,16 @@ define <16 x float> @stack_fold_vpermi2ps_mask(<16 x float> %x0, <16 x i32>* %x1
 }
 
 define <16 x float> @stack_fold_vpermt2ps_mask(<16 x i32>* %x0, <16 x float> %x1, <16 x float> %x2, i16 %mask) {
-  ;CHECK-LABEL: stack_fold_vpermt2ps_mask
-  ;CHECK:       vpermt2ps {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpermt2ps_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovaps (%rdi), %zmm1
+; CHECK-NEXT:    kmovw %esi, %k1
+; CHECK-NEXT:    vpermt2ps {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %x0b = load <16 x i32>, <16 x i32>* %x0
   %2 = call <16 x float> @llvm.x86.avx512.vpermi2var.ps.512(<16 x float> %x1, <16 x i32> %x0b, <16 x float> %x2)
@@ -930,8 +1690,16 @@ define <16 x float> @stack_fold_vpermt2ps_mask(<16 x i32>* %x0, <16 x float> %x1
 }
 
 define <16 x float> @stack_fold_vpermt2ps_maskz(<16 x i32>* %x0, <16 x float> %x1, <16 x float> %x2, i16 %mask) {
-  ;CHECK-LABEL: stack_fold_vpermt2ps_maskz
-  ;CHECK:       vpermt2ps {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpermt2ps_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovaps (%rdi), %zmm1
+; CHECK-NEXT:    kmovw %esi, %k1
+; CHECK-NEXT:    vpermt2ps {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %x0b = load <16 x i32>, <16 x i32>* %x0
   %2 = call <16 x float> @llvm.x86.avx512.vpermi2var.ps.512(<16 x float> %x1, <16 x i32> %x0b, <16 x float> %x2)
@@ -941,8 +1709,14 @@ define <16 x float> @stack_fold_vpermt2ps_maskz(<16 x i32>* %x0, <16 x float> %x
 }
 
 define <8 x double> @stack_fold_vpermt2pd(<8 x double> %x0, <8 x i64> %x1, <8 x double> %x2) {
-  ;CHECK-LABEL: stack_fold_vpermt2pd
-  ;CHECK:       vpermt2pd {{-?[0-9]*}}(%rsp), %zmm1, %zmm0 # 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpermt2pd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermt2pd {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x double> @llvm.x86.avx512.vpermi2var.pd.512(<8 x double> %x0, <8 x i64> %x1, <8 x double> %x2)
   %3 = bitcast <8 x i64> %x1 to <8 x double>
@@ -951,16 +1725,31 @@ define <8 x double> @stack_fold_vpermt2pd(<8 x double> %x0, <8 x i64> %x1, <8 x
 declare <8 x double> @llvm.x86.avx512.vpermi2var.pd.512(<8 x double>, <8 x i64>, <8 x double>)
 
 define <8 x double> @stack_fold_vpermi2pd(<8 x i64> %x0, <8 x double> %x1, <8 x double> %x2) {
-  ;CHECK-LABEL: stack_fold_vpermi2pd
-  ;CHECK:       vpermi2pd {{-?[0-9]*}}(%rsp), %zmm1, %zmm0 # 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpermi2pd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermi2pd {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x double> @llvm.x86.avx512.vpermi2var.pd.512(<8 x double> %x1, <8 x i64> %x0, <8 x double> %x2)
   ret <8 x double> %2
 }
 
 define <8 x double> @stack_fold_permpd(<8 x double> %a0) {
-  ;CHECK-LABEL: stack_fold_permpd
-  ;CHECK:   vpermpd $235, {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_permpd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermpd $235, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    # zmm0 = mem[3,2,2,3,7,6,6,7]
+; CHECK-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vaddpd %zmm1, %zmm0, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <8 x double> %a0, <8 x double> undef, <8 x i32> <i32 3, i32 2, i32 2, i32 3, i32 7, i32 6, i32 6, i32 7>
   ; fadd forces execution domain
@@ -969,8 +1758,19 @@ define <8 x double> @stack_fold_permpd(<8 x double> %a0) {
 }
 
 define <8 x double> @stack_fold_permpd_mask(<8 x double>* %passthru, <8 x double> %a0, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_permpd_mask
-  ;CHECK:   vpermpd $235, {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_permpd_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovw %esi, %k1
+; CHECK-NEXT:    vmovapd (%rdi), %zmm0
+; CHECK-NEXT:    vpermpd $235, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    # zmm0 {%k1} = mem[3,2,2,3,7,6,6,7]
+; CHECK-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vaddpd %zmm1, %zmm0, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <8 x double> %a0, <8 x double> undef, <8 x i32> <i32 3, i32 2, i32 2, i32 3, i32 7, i32 6, i32 6, i32 7>
   %3 = bitcast i8 %mask to <8 x i1>
@@ -983,8 +1783,16 @@ define <8 x double> @stack_fold_permpd_mask(<8 x double>* %passthru, <8 x double
 }
 
 define <8 x double> @stack_fold_permpd_maskz(<8 x double> %a0, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_permpd_maskz
-  ;CHECK:   vpermpd $235, {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_permpd_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    vpermpd $235, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    # zmm0 {%k1} {z} = mem[3,2,2,3,7,6,6,7]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <8 x double> %a0, <8 x double> undef, <8 x i32> <i32 3, i32 2, i32 2, i32 3, i32 7, i32 6, i32 6, i32 7>
   %3 = bitcast i8 %mask to <8 x i1>
@@ -993,8 +1801,16 @@ define <8 x double> @stack_fold_permpd_maskz(<8 x double> %a0, i8 %mask) {
 }
 
 define <8 x double> @stack_fold_permpdvar(<8 x i64> %a0, <8 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_permpdvar
-  ;CHECK:   vpermpd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_permpdvar:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermpd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vaddpd %zmm1, %zmm0, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x double> @llvm.x86.avx512.permvar.df.512(<8 x double> %a1, <8 x i64> %a0)
   ; fadd forces execution domain
@@ -1004,8 +1820,14 @@ define <8 x double> @stack_fold_permpdvar(<8 x i64> %a0, <8 x double> %a1) {
 declare <8 x double> @llvm.x86.avx512.permvar.df.512(<8 x double>, <8 x i64>) nounwind readonly
 
 define <16 x float> @stack_fold_permps(<16 x i32> %a0, <16 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_permps
-  ;CHECK:       vpermps {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_permps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x float> @llvm.x86.avx512.permvar.sf.512(<16 x float> %a1, <16 x i32> %a0)
   ret <16 x float> %2
@@ -1013,16 +1835,33 @@ define <16 x float> @stack_fold_permps(<16 x i32> %a0, <16 x float> %a1) {
 declare <16 x float> @llvm.x86.avx512.permvar.sf.512(<16 x float>, <16 x i32>) nounwind readonly
 
 define <8 x double> @stack_fold_permilpd_zmm(<8 x double> %a0) {
-  ;CHECK-LABEL: stack_fold_permilpd_zmm
-  ;CHECK:   vpermilpd $85, {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_permilpd_zmm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermilpd $85, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    # zmm0 = mem[1,0,3,2,5,4,7,6]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <8 x double> %a0, <8 x double> undef, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
   ret <8 x double> %2
 }
 
 define <8 x double> @stack_fold_permilpd_zmm_mask(<8 x double>* %passthru, <8 x double> %a0, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_permilpd_zmm_mask
-  ;CHECK:   vpermilpd $85, {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_permilpd_zmm_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovw %esi, %k1
+; CHECK-NEXT:    vmovapd (%rdi), %zmm1
+; CHECK-NEXT:    vpermilpd $85, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    # zmm1 {%k1} = mem[1,0,3,2,5,4,7,6]
+; CHECK-NEXT:    vmovapd %zmm1, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <8 x double> %a0, <8 x double> undef, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
   %3 = bitcast i8 %mask to <8 x i1>
@@ -1033,8 +1872,16 @@ define <8 x double> @stack_fold_permilpd_zmm_mask(<8 x double>* %passthru, <8 x
 }
 
 define <8 x double> @stack_fold_permilpd_zmm_maskz(<8 x double> %a0, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_permilpd_zmm_maskz
-  ;CHECK:   vpermilpd $85, {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_permilpd_zmm_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    vpermilpd $85, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    # zmm0 {%k1} {z} = mem[1,0,3,2,5,4,7,6]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <8 x double> %a0, <8 x double> undef, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
   %3 = bitcast i8 %mask to <8 x i1>
@@ -1043,8 +1890,14 @@ define <8 x double> @stack_fold_permilpd_zmm_maskz(<8 x double> %a0, i8 %mask) {
 }
 
 define <8 x double> @stack_fold_permilpdvar_zmm(<8 x double> %a0, <8 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_permilpdvar_zmm
-  ;CHECK:       vpermilpd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_permilpdvar_zmm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermilpd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double> %a0, <8 x i64> %a1)
   ret <8 x double> %2
@@ -1052,8 +1905,17 @@ define <8 x double> @stack_fold_permilpdvar_zmm(<8 x double> %a0, <8 x i64> %a1)
 declare <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double>, <8 x i64>) nounwind readnone
 
 define <8 x double> @stack_fold_permilpdvar_zmm_mask(<8 x double>* %passthru, <8 x double> %a0, <8 x i64> %a1, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_permilpdvar_zmm_mask
-  ;CHECK:       vpermilpd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_permilpdvar_zmm_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovw %esi, %k1
+; CHECK-NEXT:    vmovapd (%rdi), %zmm2
+; CHECK-NEXT:    vpermilpd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovapd %zmm2, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double> %a0, <8 x i64> %a1)
   %3 = bitcast i8 %mask to <8 x i1>
@@ -1064,8 +1926,15 @@ define <8 x double> @stack_fold_permilpdvar_zmm_mask(<8 x double>* %passthru, <8
 }
 
 define <8 x double> @stack_fold_permilpdvar_zmm_maskz(<8 x double> %a0, <8 x i64> %a1, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_permilpdvar_zmm_maskz
-  ;CHECK:       vpermilpd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_permilpdvar_zmm_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    vpermilpd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double> %a0, <8 x i64> %a1)
   %3 = bitcast i8 %mask to <8 x i1>
@@ -1074,16 +1943,33 @@ define <8 x double> @stack_fold_permilpdvar_zmm_maskz(<8 x double> %a0, <8 x i64
 }
 
 define <16 x float> @stack_fold_permilps_zmm(<16 x float> %a0) {
-  ;CHECK-LABEL: stack_fold_permilps_zmm
-  ;CHECK:   vpermilps $27, {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_permilps_zmm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermilps $27, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    # zmm0 = mem[3,2,1,0,7,6,5,4,11,10,9,8,15,14,13,12]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <16 x float> %a0, <16 x float> undef, <16 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 11, i32 10, i32 9, i32 8, i32 15, i32 14, i32 13, i32 12>
   ret <16 x float> %2
 }
 
 define <16 x float> @stack_fold_permilps_zmm_mask(<16 x float>* %passthru, <16 x float> %a0, i16 %mask) {
-  ;CHECK-LABEL: stack_fold_permilps_zmm_mask
-  ;CHECK:   vpermilps $27, {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_permilps_zmm_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovw %esi, %k1
+; CHECK-NEXT:    vmovaps (%rdi), %zmm1
+; CHECK-NEXT:    vpermilps $27, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    # zmm1 {%k1} = mem[3,2,1,0,7,6,5,4,11,10,9,8,15,14,13,12]
+; CHECK-NEXT:    vmovaps %zmm1, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <16 x float> %a0, <16 x float> undef, <16 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 11, i32 10, i32 9, i32 8, i32 15, i32 14, i32 13, i32 12>
   %3 = bitcast i16 %mask to <16 x i1>
@@ -1094,8 +1980,16 @@ define <16 x float> @stack_fold_permilps_zmm_mask(<16 x float>* %passthru, <16 x
 }
 
 define <16 x float> @stack_fold_permilps_zmm_maskz(<16 x float> %a0, i16 %mask) {
-  ;CHECK-LABEL: stack_fold_permilps_zmm_maskz
-  ;CHECK:   vpermilps $27, {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_permilps_zmm_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    vpermilps $27, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    # zmm0 {%k1} {z} = mem[3,2,1,0,7,6,5,4,11,10,9,8,15,14,13,12]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <16 x float> %a0, <16 x float> undef, <16 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 11, i32 10, i32 9, i32 8, i32 15, i32 14, i32 13, i32 12>
   %3 = bitcast i16 %mask to <16 x i1>
@@ -1104,8 +1998,14 @@ define <16 x float> @stack_fold_permilps_zmm_maskz(<16 x float> %a0, i16 %mask)
 }
 
 define <16 x float> @stack_fold_permilpsvar_zmm(<16 x float> %a0, <16 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_permilpsvar_zmm
-  ;CHECK:       vpermilps {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_permilpsvar_zmm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermilps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> %a0, <16 x i32> %a1)
   ret <16 x float> %2
@@ -1113,8 +2013,17 @@ define <16 x float> @stack_fold_permilpsvar_zmm(<16 x float> %a0, <16 x i32> %a1
 declare <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float>, <16 x i32>) nounwind readnone
 
 define <16 x float> @stack_fold_permilpsvar_zmm_mask(<16 x float>* %passthru, <16 x float> %a0, <16 x i32> %a1, i16 %mask) {
-  ;CHECK-LABEL: stack_fold_permilpsvar_zmm_mask
-  ;CHECK:       vpermilps {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_permilpsvar_zmm_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovw %esi, %k1
+; CHECK-NEXT:    vmovaps (%rdi), %zmm2
+; CHECK-NEXT:    vpermilps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovaps %zmm2, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> %a0, <16 x i32> %a1)
   %3 = bitcast i16 %mask to <16 x i1>
@@ -1125,8 +2034,15 @@ define <16 x float> @stack_fold_permilpsvar_zmm_mask(<16 x float>* %passthru, <1
 }
 
 define <16 x float> @stack_fold_permilpsvar_zmm_maskz(<16 x float> %a0, <16 x i32> %a1, i16 %mask) {
-  ;CHECK-LABEL: stack_fold_permilpsvar_zmm_maskz
-  ;CHECK:       vpermilps {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_permilpsvar_zmm_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    vpermilps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> %a0, <16 x i32> %a1)
   %3 = bitcast i16 %mask to <16 x i1>
@@ -1135,4 +2051,4 @@ define <16 x float> @stack_fold_permilpsvar_zmm_maskz(<16 x float> %a0, <16 x i3
 }
 
 attributes #0 = { "unsafe-fp-math"="false" }
-attributes #1 = { "unsafe-fp-math"="true" }
+attributes #1 = { "unsafe-fp-math"="true" "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="true" }
diff --git a/llvm/test/CodeGen/X86/stack-folding-fp-avx512vl.ll b/llvm/test/CodeGen/X86/stack-folding-fp-avx512vl.ll
index 0cb72d162..09a06e12b 100644
--- a/llvm/test/CodeGen/X86/stack-folding-fp-avx512vl.ll
+++ b/llvm/test/CodeGen/X86/stack-folding-fp-avx512vl.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -O3 -disable-peephole -mtriple=x86_64-unknown-unknown -mattr=+avx512vl,+avx512dq < %s | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -9,40 +10,72 @@ target triple = "x86_64-unknown-unknown"
 ; relevant registers and check that the reload is correctly folded into the instruction.
 
 define <2 x double> @stack_fold_addpd(<2 x double> %a0, <2 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_addpd
-  ;CHECK:       vaddpd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_addpd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vaddpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = fadd <2 x double> %a0, %a1
   ret <2 x double> %2
 }
 
 define <4 x double> @stack_fold_addpd_ymm(<4 x double> %a0, <4 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_addpd_ymm
-  ;CHECK:       vaddpd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_addpd_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vaddpd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = fadd <4 x double> %a0, %a1
   ret <4 x double> %2
 }
 
 define <4 x float> @stack_fold_addps(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_addps
-  ;CHECK:       vaddps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_addps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vaddps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = fadd <4 x float> %a0, %a1
   ret <4 x float> %2
 }
 
 define <8 x float> @stack_fold_addps_ymm(<8 x float> %a0, <8 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_addps_ymm
-  ;CHECK:       vaddps {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_addps_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vaddps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = fadd <8 x float> %a0, %a1
   ret <8 x float> %2
 }
 
 define <2 x double> @stack_fold_andnpd(<2 x double> %a0, <2 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_andnpd
-  ;CHECK:       vandnpd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_andnpd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vandnpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = bitcast <2 x double> %a0 to <2 x i64>
   %3 = bitcast <2 x double> %a1 to <2 x i64>
@@ -55,8 +88,16 @@ define <2 x double> @stack_fold_andnpd(<2 x double> %a0, <2 x double> %a1) {
 }
 
 define <4 x double> @stack_fold_andnpd_ymm(<4 x double> %a0, <4 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_andnpd_ymm
-  ;CHECK:       vandnpd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_andnpd_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vandnpd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = bitcast <4 x double> %a0 to <4 x i64>
   %3 = bitcast <4 x double> %a1 to <4 x i64>
@@ -69,8 +110,16 @@ define <4 x double> @stack_fold_andnpd_ymm(<4 x double> %a0, <4 x double> %a1) {
 }
 
 define <4 x float> @stack_fold_andnps(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_andnps
-  ;CHECK:       vandnps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_andnps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vandnps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vaddps %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = bitcast <4 x float> %a0 to <2 x i64>
   %3 = bitcast <4 x float> %a1 to <2 x i64>
@@ -83,8 +132,16 @@ define <4 x float> @stack_fold_andnps(<4 x float> %a0, <4 x float> %a1) {
 }
 
 define <8 x float> @stack_fold_andnps_ymm(<8 x float> %a0, <8 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_andnps_ymm
-  ;CHECK:       vandnps {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_andnps_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vandnps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vaddps %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = bitcast <8 x float> %a0 to <4 x i64>
   %3 = bitcast <8 x float> %a1 to <4 x i64>
@@ -97,8 +154,16 @@ define <8 x float> @stack_fold_andnps_ymm(<8 x float> %a0, <8 x float> %a1) {
 }
 
 define <2 x double> @stack_fold_andpd(<2 x double> %a0, <2 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_andpd
-  ;CHECK:       vandpd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_andpd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vandpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = bitcast <2 x double> %a0 to <2 x i64>
   %3 = bitcast <2 x double> %a1 to <2 x i64>
@@ -110,8 +175,16 @@ define <2 x double> @stack_fold_andpd(<2 x double> %a0, <2 x double> %a1) {
 }
 
 define <4 x double> @stack_fold_andpd_ymm(<4 x double> %a0, <4 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_andpd_ymm
-  ;CHECK:       vandpd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_andpd_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vandpd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = bitcast <4 x double> %a0 to <4 x i64>
   %3 = bitcast <4 x double> %a1 to <4 x i64>
@@ -123,8 +196,16 @@ define <4 x double> @stack_fold_andpd_ymm(<4 x double> %a0, <4 x double> %a1) {
 }
 
 define <4 x float> @stack_fold_andps(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_andps
-  ;CHECK:       vandps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_andps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vandps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vaddps %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = bitcast <4 x float> %a0 to <4 x i32>
   %3 = bitcast <4 x float> %a1 to <4 x i32>
@@ -136,8 +217,16 @@ define <4 x float> @stack_fold_andps(<4 x float> %a0, <4 x float> %a1) {
 }
 
 define <8 x float> @stack_fold_andps_ymm(<8 x float> %a0, <8 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_andps_ymm
-  ;CHECK:       vandps {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_andps_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vandps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vaddps %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = bitcast <8 x float> %a0 to <8 x i32>
   %3 = bitcast <8 x float> %a1 to <8 x i32>
@@ -149,8 +238,16 @@ define <8 x float> @stack_fold_andps_ymm(<8 x float> %a0, <8 x float> %a1) {
 }
 
 define i8 @stack_fold_cmppd(<2 x double> %a0, <2 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_cmppd
-  ;CHECK:       vcmpeqpd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%k[0-9]}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_cmppd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcmpeqpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %k0 # 16-byte Folded Reload
+; CHECK-NEXT:    kmovw %k0, %eax
+; CHECK-NEXT:    # kill: def $al killed $al killed $eax
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %res = call <2 x i1> @llvm.x86.avx512.cmp.pd.128(<2 x double> %a0, <2 x double> %a1, i32 0)
   %2 = shufflevector <2 x i1> %res, <2 x i1> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 2, i32 3, i32 2, i32 3>
@@ -160,8 +257,17 @@ define i8 @stack_fold_cmppd(<2 x double> %a0, <2 x double> %a1) {
 declare <2 x i1> @llvm.x86.avx512.cmp.pd.128(<2 x double>, <2 x double>, i32)
 
 define i8 @stack_fold_cmppd_ymm(<4 x double> %a0, <4 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_cmppd_ymm
-  ;CHECK:       vcmpeqpd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%k[0-9]}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_cmppd_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcmpeqpd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %k0 # 32-byte Folded Reload
+; CHECK-NEXT:    kmovw %k0, %eax
+; CHECK-NEXT:    # kill: def $al killed $al killed $eax
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %res = call <4 x i1> @llvm.x86.avx512.cmp.pd.256(<4 x double> %a0, <4 x double> %a1, i32 0)
   %2 = shufflevector <4 x i1> %res, <4 x i1> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -171,8 +277,16 @@ define i8 @stack_fold_cmppd_ymm(<4 x double> %a0, <4 x double> %a1) {
 declare <4 x i1> @llvm.x86.avx512.cmp.pd.256(<4 x double>, <4 x double>, i32)
 
 define i8 @stack_fold_cmpps(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_cmpps
-  ;CHECK:       vcmpeqps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%k[0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_cmpps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcmpeqps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %k0 # 16-byte Folded Reload
+; CHECK-NEXT:    kmovw %k0, %eax
+; CHECK-NEXT:    # kill: def $al killed $al killed $eax
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %res = call <4 x i1> @llvm.x86.avx512.cmp.ps.128(<4 x float> %a0, <4 x float> %a1, i32 0)
   %2 = shufflevector <4 x i1> %res, <4 x i1> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -182,8 +296,17 @@ define i8 @stack_fold_cmpps(<4 x float> %a0, <4 x float> %a1) {
 declare <4 x i1> @llvm.x86.avx512.cmp.ps.128(<4 x float>, <4 x float>, i32)
 
 define i8 @stack_fold_cmpps_ymm(<8 x float> %a0, <8 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_cmpps_ymm
-  ;CHECK:       vcmpeqps {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%k[0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_cmpps_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcmpeqps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %k0 # 32-byte Folded Reload
+; CHECK-NEXT:    kmovw %k0, %eax
+; CHECK-NEXT:    # kill: def $al killed $al killed $eax
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %res = call <8 x i1> @llvm.x86.avx512.cmp.ps.256(<8 x float> %a0, <8 x float> %a1, i32 0)
   %2 = bitcast <8 x i1> %res to i8
@@ -192,40 +315,70 @@ define i8 @stack_fold_cmpps_ymm(<8 x float> %a0, <8 x float> %a1) {
 declare <8 x i1> @llvm.x86.avx512.cmp.ps.256(<8 x float>, <8 x float>, i32)
 
 define <2 x double> @stack_fold_divpd(<2 x double> %a0, <2 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_divpd
-  ;CHECK:       vdivpd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_divpd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vdivpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = fdiv <2 x double> %a0, %a1
   ret <2 x double> %2
 }
 
 define <4 x double> @stack_fold_divpd_ymm(<4 x double> %a0, <4 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_divpd_ymm
-  ;CHECK:       vdivpd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_divpd_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vdivpd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = fdiv <4 x double> %a0, %a1
   ret <4 x double> %2
 }
 
 define <4 x float> @stack_fold_divps(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_divps
-  ;CHECK:       vdivps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_divps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vdivps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = fdiv <4 x float> %a0, %a1
   ret <4 x float> %2
 }
 
 define <8 x float> @stack_fold_divps_ymm(<8 x float> %a0, <8 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_divps_ymm
-  ;CHECK:       vdivps {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_divps_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vdivps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = fdiv <8 x float> %a0, %a1
   ret <8 x float> %2
 }
 
 define <2 x double> @stack_fold_cvtdq2pd(<4 x i32> %a0) {
-  ;CHECK-LABEL: stack_fold_cvtdq2pd
-  ;CHECK:   vcvtdq2pd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtdq2pd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcvtdq2pd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <4 x i32> %a0, <4 x i32> undef, <2 x i32> <i32 0, i32 1>
   %3 = sitofp <2 x i32> %2 to <2 x double>
@@ -233,16 +386,28 @@ define <2 x double> @stack_fold_cvtdq2pd(<4 x i32> %a0) {
 }
 
 define <4 x double> @stack_fold_cvtdq2pd_ymm(<4 x i32> %a0) {
-  ;CHECK-LABEL: stack_fold_cvtdq2pd_ymm
-  ;CHECK:   vcvtdq2pd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtdq2pd_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcvtdq2pd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = sitofp <4 x i32> %a0 to <4 x double>
   ret <4 x double> %2
 }
 
 define <2 x double> @stack_fold_cvtudq2pd(<4 x i32> %a0) {
-  ;CHECK-LABEL: stack_fold_cvtudq2pd
-  ;CHECK:   vcvtudq2pd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtudq2pd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcvtudq2pd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <4 x i32> %a0, <4 x i32> undef, <2 x i32> <i32 0, i32 1>
   %3 = uitofp <2 x i32> %2 to <2 x double>
@@ -250,32 +415,57 @@ define <2 x double> @stack_fold_cvtudq2pd(<4 x i32> %a0) {
 }
 
 define <4 x double> @stack_fold_cvtudq2pd_ymm(<4 x i32> %a0) {
-  ;CHECK-LABEL: stack_fold_cvtudq2pd_ymm
-  ;CHECK:   vcvtudq2pd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtudq2pd_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcvtudq2pd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = uitofp <4 x i32> %a0 to <4 x double>
   ret <4 x double> %2
 }
 
 define <2 x float> @stack_fold_cvtpd2ps(<2 x double> %a0) {
-  ;CHECK-LABEL: stack_fold_cvtpd2ps
-  ;CHECK:   vcvtpd2psx {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtpd2ps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcvtpd2psx {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = fptrunc <2 x double> %a0 to <2 x float>
   ret <2 x float> %2
 }
 
 define <4 x float> @stack_fold_cvtpd2ps_ymm(<4 x double> %a0) {
-  ;CHECK-LABEL: stack_fold_cvtpd2ps_ymm
-  ;CHECK:   vcvtpd2psy {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtpd2ps_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vcvtpd2psy {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 32-byte Folded Reload
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = fptrunc <4 x double> %a0 to <4 x float>
   ret <4 x float> %2
 }
 
 define <2 x double> @stack_fold_maxpd(<2 x double> %a0, <2 x double> %a1) #0 {
-  ;CHECK-LABEL: stack_fold_maxpd
-  ;CHECK:       vmaxpd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_maxpd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmaxpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <2 x double> @llvm.x86.sse2.max.pd(<2 x double> %a0, <2 x double> %a1)
   ret <2 x double> %2
@@ -283,16 +473,28 @@ define <2 x double> @stack_fold_maxpd(<2 x double> %a0, <2 x double> %a1) #0 {
 declare <2 x double> @llvm.x86.sse2.max.pd(<2 x double>, <2 x double>) nounwind readnone
 
 define <2 x double> @stack_fold_maxpd_commutable(<2 x double> %a0, <2 x double> %a1) #1 {
-  ;CHECK-LABEL: stack_fold_maxpd_commutable
-  ;CHECK:       vmaxpd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_maxpd_commutable:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmaxpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <2 x double> @llvm.x86.sse2.max.pd(<2 x double> %a0, <2 x double> %a1)
   ret <2 x double> %2
 }
 
 define <4 x double> @stack_fold_maxpd_ymm(<4 x double> %a0, <4 x double> %a1) #0 {
-  ;CHECK-LABEL: stack_fold_maxpd_ymm
-  ;CHECK:       vmaxpd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_maxpd_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmaxpd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <4 x double> @llvm.x86.avx.max.pd.256(<4 x double> %a0, <4 x double> %a1)
   ret <4 x double> %2
@@ -300,16 +502,28 @@ define <4 x double> @stack_fold_maxpd_ymm(<4 x double> %a0, <4 x double> %a1) #0
 declare <4 x double> @llvm.x86.avx.max.pd.256(<4 x double>, <4 x double>) nounwind readnone
 
 define <4 x double> @stack_fold_maxpd_ymm_commutable(<4 x double> %a0, <4 x double> %a1) #1 {
-  ;CHECK-LABEL: stack_fold_maxpd_ymm_commutable
-  ;CHECK:       vmaxpd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_maxpd_ymm_commutable:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmaxpd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <4 x double> @llvm.x86.avx.max.pd.256(<4 x double> %a0, <4 x double> %a1)
   ret <4 x double> %2
 }
 
 define <4 x float> @stack_fold_maxps(<4 x float> %a0, <4 x float> %a1) #0 {
-  ;CHECK-LABEL: stack_fold_maxps
-  ;CHECK:       vmaxps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_maxps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmaxps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <4 x float> @llvm.x86.sse.max.ps(<4 x float> %a0, <4 x float> %a1)
   ret <4 x float> %2
@@ -317,16 +531,28 @@ define <4 x float> @stack_fold_maxps(<4 x float> %a0, <4 x float> %a1) #0 {
 declare <4 x float> @llvm.x86.sse.max.ps(<4 x float>, <4 x float>) nounwind readnone
 
 define <4 x float> @stack_fold_maxps_commutable(<4 x float> %a0, <4 x float> %a1) #1 {
-  ;CHECK-LABEL: stack_fold_maxps_commutable
-  ;CHECK:       vmaxps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_maxps_commutable:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmaxps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <4 x float> @llvm.x86.sse.max.ps(<4 x float> %a0, <4 x float> %a1)
   ret <4 x float> %2
 }
 
 define <8 x float> @stack_fold_maxps_ymm(<8 x float> %a0, <8 x float> %a1) #0 {
-  ;CHECK-LABEL: stack_fold_maxps_ymm
-  ;CHECK:       vmaxps {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_maxps_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmaxps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x float> @llvm.x86.avx.max.ps.256(<8 x float> %a0, <8 x float> %a1)
   ret <8 x float> %2
@@ -334,16 +560,28 @@ define <8 x float> @stack_fold_maxps_ymm(<8 x float> %a0, <8 x float> %a1) #0 {
 declare <8 x float> @llvm.x86.avx.max.ps.256(<8 x float>, <8 x float>) nounwind readnone
 
 define <8 x float> @stack_fold_maxps_ymm_commutable(<8 x float> %a0, <8 x float> %a1) #1 {
-  ;CHECK-LABEL: stack_fold_maxps_ymm_commutable
-  ;CHECK:       vmaxps {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_maxps_ymm_commutable:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmaxps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x float> @llvm.x86.avx.max.ps.256(<8 x float> %a0, <8 x float> %a1)
   ret <8 x float> %2
 }
 
 define <4 x float> @stack_fold_minps(<4 x float> %a0, <4 x float> %a1) #0 {
-  ;CHECK-LABEL: stack_fold_minps
-  ;CHECK:       vminps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_minps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vminps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <4 x float> @llvm.x86.sse.min.ps(<4 x float> %a0, <4 x float> %a1)
   ret <4 x float> %2
@@ -351,16 +589,28 @@ define <4 x float> @stack_fold_minps(<4 x float> %a0, <4 x float> %a1) #0 {
 declare <4 x float> @llvm.x86.sse.min.ps(<4 x float>, <4 x float>) nounwind readnone
 
 define <4 x float> @stack_fold_minps_commutable(<4 x float> %a0, <4 x float> %a1) #1 {
-  ;CHECK-LABEL: stack_fold_minps_commutable
-  ;CHECK:       vminps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_minps_commutable:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vminps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <4 x float> @llvm.x86.sse.min.ps(<4 x float> %a0, <4 x float> %a1)
   ret <4 x float> %2
 }
 
 define <8 x float> @stack_fold_minps_ymm(<8 x float> %a0, <8 x float> %a1) #0 {
-  ;CHECK-LABEL: stack_fold_minps_ymm
-  ;CHECK:       vminps {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_minps_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vminps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x float> @llvm.x86.avx.min.ps.256(<8 x float> %a0, <8 x float> %a1)
   ret <8 x float> %2
@@ -368,48 +618,86 @@ define <8 x float> @stack_fold_minps_ymm(<8 x float> %a0, <8 x float> %a1) #0 {
 declare <8 x float> @llvm.x86.avx.min.ps.256(<8 x float>, <8 x float>) nounwind readnone
 
 define <8 x float> @stack_fold_minps_ymm_commutable(<8 x float> %a0, <8 x float> %a1) #1 {
-  ;CHECK-LABEL: stack_fold_minps_ymm_commutable
-  ;CHECK:       vminps {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_minps_ymm_commutable:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vminps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x float> @llvm.x86.avx.min.ps.256(<8 x float> %a0, <8 x float> %a1)
   ret <8 x float> %2
 }
 
 define <2 x double> @stack_fold_mulpd(<2 x double> %a0, <2 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_mulpd
-  ;CHECK:       vmulpd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_mulpd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmulpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = fmul <2 x double> %a0, %a1
   ret <2 x double> %2
 }
 
 define <4 x double> @stack_fold_mulpd_ymm(<4 x double> %a0, <4 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_mulpd_ymm
-  ;CHECK:       vmulpd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_mulpd_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmulpd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = fmul <4 x double> %a0, %a1
   ret <4 x double> %2
 }
 
 define <4 x float> @stack_fold_mulps(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_mulps
-  ;CHECK:       vmulps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_mulps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmulps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = fmul <4 x float> %a0, %a1
   ret <4 x float> %2
 }
 
 define <8 x float> @stack_fold_mulps_ymm(<8 x float> %a0, <8 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_mulps_ymm
-  ;CHECK:       vmulps {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_mulps_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmulps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = fmul <8 x float> %a0, %a1
   ret <8 x float> %2
 }
 
 define <2 x double> @stack_fold_orpd(<2 x double> %a0, <2 x double> %a1) #0 {
-  ;CHECK-LABEL: stack_fold_orpd
-  ;CHECK:       vorpd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_orpd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vorpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = bitcast <2 x double> %a0 to <2 x i64>
   %3 = bitcast <2 x double> %a1 to <2 x i64>
@@ -421,8 +709,16 @@ define <2 x double> @stack_fold_orpd(<2 x double> %a0, <2 x double> %a1) #0 {
 }
 
 define <4 x double> @stack_fold_orpd_ymm(<4 x double> %a0, <4 x double> %a1) #0 {
-  ;CHECK-LABEL: stack_fold_orpd_ymm
-  ;CHECK:       vorpd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_orpd_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vorpd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = bitcast <4 x double> %a0 to <4 x i64>
   %3 = bitcast <4 x double> %a1 to <4 x i64>
@@ -434,8 +730,16 @@ define <4 x double> @stack_fold_orpd_ymm(<4 x double> %a0, <4 x double> %a1) #0
 }
 
 define <4 x float> @stack_fold_orps(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_orps
-  ;CHECK:       vorps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_orps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vorps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vaddps %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = bitcast <4 x float> %a0 to <4 x i32>
   %3 = bitcast <4 x float> %a1 to <4 x i32>
@@ -447,8 +751,16 @@ define <4 x float> @stack_fold_orps(<4 x float> %a0, <4 x float> %a1) {
 }
 
 define <8 x float> @stack_fold_orps_ymm(<8 x float> %a0, <8 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_orps_ymm
-  ;CHECK:       vorps {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_orps_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vorps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vaddps %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = bitcast <8 x float> %a0 to <8 x i32>
   %3 = bitcast <8 x float> %a1 to <8 x i32>
@@ -460,8 +772,18 @@ define <8 x float> @stack_fold_orps_ymm(<8 x float> %a0, <8 x float> %a1) {
 }
 
 define <4 x double> @stack_fold_shuff64x2_maskz(<4 x double> %a, <4 x double> %b, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_shuff64x2_maskz
-  ;CHECK:   vshuff64x2 $1, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{{%k[1-7]}}} {z} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_shuff64x2_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    vmovupd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; CHECK-NEXT:    vshuff64x2 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 {%k1} {z} # 32-byte Folded Reload
+; CHECK-NEXT:    # ymm0 {%k1} {z} = ymm0[2,3],mem[0,1]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <4 x double> %a, <4 x double> %b, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
   %3 = bitcast i8 %mask to <8 x i1>
@@ -471,8 +793,18 @@ define <4 x double> @stack_fold_shuff64x2_maskz(<4 x double> %a, <4 x double> %b
 }
 
 define <8 x float> @stack_fold_shuff32x4_maskz(<8 x float> %a, <8 x float> %b, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_shuff32x4_maskz
-  ;CHECK:   vshuff32x4 $1, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{{%k[1-7]}}} {z} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_shuff32x4_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; CHECK-NEXT:    vshuff32x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 {%k1} {z} # 32-byte Folded Reload
+; CHECK-NEXT:    # ymm0 {%k1} {z} = ymm0[4,5,6,7],mem[0,1,2,3]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
   %3 = bitcast i8 %mask to <8 x i1>
@@ -481,16 +813,33 @@ define <8 x float> @stack_fold_shuff32x4_maskz(<8 x float> %a, <8 x float> %b, i
 }
 
 define <4 x float> @stack_fold_shufps(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_shufps
-  ;CHECK:       vshufps $200, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_shufps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vshufps $200, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = xmm0[0,2],mem[0,3]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <4 x float> %a0, <4 x float> %a1, <4 x i32> <i32 0, i32 2, i32 4, i32 7>
   ret <4 x float> %2
 }
 
 define <4 x float> @stack_fold_shufps_mask(<4 x float>* %passthru, <4 x float> %a0, <4 x float> %a1, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_shufps_mask
-  ;CHECK:       vshufps $200, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_shufps_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovw %esi, %k1
+; CHECK-NEXT:    vmovaps (%rdi), %xmm2
+; CHECK-NEXT:    vshufps $200, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm2 {%k1} # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm2 {%k1} = xmm0[0,2],mem[0,3]
+; CHECK-NEXT:    vmovaps %xmm2, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <4 x float> %a0, <4 x float> %a1, <4 x i32> <i32 0, i32 2, i32 4, i32 7>
   %3 = bitcast i8 %mask to <8 x i1>
@@ -501,8 +850,16 @@ define <4 x float> @stack_fold_shufps_mask(<4 x float>* %passthru, <4 x float> %
 }
 
 define <4 x float> @stack_fold_shufps_maskz(<4 x float> %a0, <4 x float> %a1, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_shufps_maskz
-  ;CHECK:       vshufps $200, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_shufps_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    vshufps $200, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 {%k1} {z} # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 {%k1} {z} = xmm0[0,2],mem[0,3]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <4 x float> %a0, <4 x float> %a1, <4 x i32> <i32 0, i32 2, i32 4, i32 7>
   %3 = bitcast i8 %mask to <8 x i1>
@@ -512,48 +869,87 @@ define <4 x float> @stack_fold_shufps_maskz(<4 x float> %a0, <4 x float> %a1, i8
 }
 
 define <8 x float> @stack_fold_shufps_ymm(<8 x float> %a0, <8 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_shufps_ymm
-  ;CHECK:       vshufps $148, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_shufps_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vshufps $148, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    # ymm0 = ymm0[0,1],mem[1,2],ymm0[4,5],mem[5,6]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <8 x float> %a0, <8 x float> %a1, <8 x i32> <i32 0, i32 1, i32 9, i32 10, i32 4, i32 5, i32 13, i32 14>
   ret <8 x float> %2
 }
 
 define <2 x double> @stack_fold_subpd(<2 x double> %a0, <2 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_subpd
-  ;CHECK:       vsubpd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_subpd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vsubpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = fsub <2 x double> %a0, %a1
   ret <2 x double> %2
 }
 
 define <4 x double> @stack_fold_subpd_ymm(<4 x double> %a0, <4 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_subpd_ymm
-  ;CHECK:       vsubpd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_subpd_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vsubpd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = fsub <4 x double> %a0, %a1
   ret <4 x double> %2
 }
 
 define <4 x float> @stack_fold_subps(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_subps
-  ;CHECK:       vsubps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_subps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vsubps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = fsub <4 x float> %a0, %a1
   ret <4 x float> %2
 }
 
 define <8 x float> @stack_fold_subps_ymm(<8 x float> %a0, <8 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_subps_ymm
-  ;CHECK:       vsubps {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_subps_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vsubps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = fsub <8 x float> %a0, %a1
   ret <8 x float> %2
 }
 
 define <2 x double> @stack_fold_xorpd(<2 x double> %a0, <2 x double> %a1) #0 {
-  ;CHECK-LABEL: stack_fold_xorpd
-  ;CHECK:       vxorpd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_xorpd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vxorpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = bitcast <2 x double> %a0 to <2 x i64>
   %3 = bitcast <2 x double> %a1 to <2 x i64>
@@ -565,8 +961,16 @@ define <2 x double> @stack_fold_xorpd(<2 x double> %a0, <2 x double> %a1) #0 {
 }
 
 define <4 x double> @stack_fold_xorpd_ymm(<4 x double> %a0, <4 x double> %a1) #0 {
-  ;CHECK-LABEL: stack_fold_xorpd_ymm
-  ;CHECK:       vxorpd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_xorpd_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vxorpd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = bitcast <4 x double> %a0 to <4 x i64>
   %3 = bitcast <4 x double> %a1 to <4 x i64>
@@ -578,8 +982,16 @@ define <4 x double> @stack_fold_xorpd_ymm(<4 x double> %a0, <4 x double> %a1) #0
 }
 
 define <4 x float> @stack_fold_xorps(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_xorps
-  ;CHECK:       vxorps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_xorps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vxorps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vaddps %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = bitcast <4 x float> %a0 to <4 x i32>
   %3 = bitcast <4 x float> %a1 to <4 x i32>
@@ -591,8 +1003,16 @@ define <4 x float> @stack_fold_xorps(<4 x float> %a0, <4 x float> %a1) {
 }
 
 define <8 x float> @stack_fold_xorps_ymm(<8 x float> %a0, <8 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_xorps_ymm
-  ;CHECK:       vxorps {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_xorps_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vxorps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vaddps %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = bitcast <8 x float> %a0 to <8 x i32>
   %3 = bitcast <8 x float> %a1 to <8 x i32>
@@ -604,104 +1024,189 @@ define <8 x float> @stack_fold_xorps_ymm(<8 x float> %a0, <8 x float> %a1) {
 }
 
 define <4 x float> @stack_fold_extractf32x4(<8 x float> %a0, <8 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_extractf32x4
-  ;CHECK:       vextractf128 $1, {{%ymm[0-9][0-9]*}}, {{-?[0-9]*}}(%rsp) {{.*#+}} 16-byte Folded Spill
+; CHECK-LABEL: stack_fold_extractf32x4:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vextractf128 $1, %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Folded Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
   %1 = shufflevector <8 x float> %a0, <8 x float> %a1, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
   %2 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   ret <4 x float> %1
 }
 
 define <2 x double> @stack_fold_extractf64x2(<4 x double> %a0, <4 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_extractf64x2
-  ;CHECK:       vextractf128 $1, {{%ymm[0-9][0-9]*}}, {{-?[0-9]*}}(%rsp) {{.*#+}} 16-byte Folded Spill
+; CHECK-LABEL: stack_fold_extractf64x2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vextractf128 $1, %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Folded Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
   %1 = shufflevector <4 x double> %a0, <4 x double> %a1, <2 x i32> <i32 2, i32 3>
   %2 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   ret <2 x double> %1
 }
 
 define <8 x float> @stack_fold_insertf32x4(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_insertf32x4
-  ;CHECK:       vinsertf128 $1, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_insertf32x4:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vinsertf128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <4 x float> %a0, <4 x float> %a1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   ret <8 x float> %2
 }
 
 define <4 x double> @stack_fold_insertf64x2(<2 x double> %a0, <2 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_insertf64x2
-  ;CHECK:       vinsertf128 $1, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_insertf64x2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vinsertf128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <2 x double> %a0, <2 x double> %a1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   ret <4 x double> %2
 }
 
 define <4 x float> @stack_fold_vpermt2ps(<4 x float> %x0, <4 x i32> %x1, <4 x float> %x2) {
-  ;CHECK-LABEL: stack_fold_vpermt2ps
-  ;CHECK:       vpermt2ps {{-?[0-9]*}}(%rsp), %xmm1, %xmm0 # 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpermt2ps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermt2ps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <4 x float> @llvm.x86.avx512.vpermi2var.ps.128(<4 x float> %x0, <4 x i32> %x1, <4 x float> %x2)
   ret <4 x float> %2
 }
 
 define <4 x float> @stack_fold_vpermi2ps(<4 x i32> %x0, <4 x float> %x1, <4 x float> %x2) {
-  ;CHECK-LABEL: stack_fold_vpermi2ps
-  ;CHECK:       vpermi2ps {{-?[0-9]*}}(%rsp), %xmm1, %xmm0 # 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpermi2ps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermi2ps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <4 x float> @llvm.x86.avx512.vpermi2var.ps.128(<4 x float> %x1, <4 x i32> %x0, <4 x float> %x2)
   ret <4 x float> %2
 }
 
 define <2 x double> @stack_fold_vpermt2pd(<2 x double> %x0, <2 x i64> %x1, <2 x double> %x2) {
-  ;CHECK-LABEL: stack_fold_vpermt2pd
-  ;CHECK:       vpermt2pd {{-?[0-9]*}}(%rsp), %xmm1, %xmm0 # 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpermt2pd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermt2pd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <2 x double> @llvm.x86.avx512.vpermi2var.pd.128(<2 x double> %x0, <2 x i64> %x1, <2 x double> %x2)
   ret <2 x double> %2
 }
 
 define <2 x double> @stack_fold_vpermi2pd(<2 x i64> %x0, <2 x double> %x1, <2 x double> %x2) {
-  ;CHECK-LABEL: stack_fold_vpermi2pd
-  ;CHECK:       vpermi2pd {{-?[0-9]*}}(%rsp), %xmm1, %xmm0 # 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpermi2pd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermi2pd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <2 x double> @llvm.x86.avx512.vpermi2var.pd.128(<2 x double> %x1, <2 x i64> %x0, <2 x double> %x2)
   ret <2 x double> %2
 }
 
 define <8 x float> @stack_fold_vpermt2ps_ymm(<8 x float> %x0, <8 x i32> %x1, <8 x float> %x2) {
-  ;CHECK-LABEL: stack_fold_vpermt2ps_ymm
-  ;CHECK:       vpermt2ps {{-?[0-9]*}}(%rsp), %ymm1, %ymm0 # 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpermt2ps_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermt2ps {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <4 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x float> @llvm.x86.avx512.vpermi2var.ps.256(<8 x float> %x0, <8 x i32> %x1, <8 x float> %x2)
   ret <8 x float> %2
 }
 
 define <8 x float> @stack_fold_vpermi2ps_ymm(<8 x i32> %x0, <8 x float> %x1, <8 x float> %x2) {
-  ;CHECK-LABEL: stack_fold_vpermi2ps_ymm
-  ;CHECK:       vpermi2ps {{-?[0-9]*}}(%rsp), %ymm1, %ymm0 # 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpermi2ps_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermi2ps {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <4 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x float> @llvm.x86.avx512.vpermi2var.ps.256(<8 x float> %x1, <8 x i32> %x0, <8 x float> %x2)
   ret <8 x float> %2
 }
 
 define <4 x double> @stack_fold_vpermt2pd_ymm(<4 x double> %x0, <4 x i64> %x1, <4 x double> %x2) {
-  ;CHECK-LABEL: stack_fold_vpermt2pd_ymm
-  ;CHECK:       vpermt2pd {{-?[0-9]*}}(%rsp), %ymm1, %ymm0 # 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpermt2pd_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermt2pd {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <4 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <4 x double> @llvm.x86.avx512.vpermi2var.pd.256(<4 x double> %x0, <4 x i64> %x1, <4 x double> %x2)
   ret <4 x double> %2
 }
 
 define <4 x double> @stack_fold_vpermi2pd_ymm(<4 x i64> %x0, <4 x double> %x1, <4 x double> %x2) {
-  ;CHECK-LABEL: stack_fold_vpermi2pd_ymm
-  ;CHECK:       vpermi2pd {{-?[0-9]*}}(%rsp), %ymm1, %ymm0 # 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpermi2pd_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermi2pd {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <4 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <4 x double> @llvm.x86.avx512.vpermi2var.pd.256(<4 x double> %x1, <4 x i64> %x0, <4 x double> %x2)
   ret <4 x double> %2
 }
 
 define <4 x double> @stack_fold_permpd(<4 x double> %a0) {
-  ;CHECK-LABEL: stack_fold_permpd
-  ;CHECK:   vpermpd $235, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_permpd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermpd $235, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    # ymm0 = mem[3,2,2,3]
+; CHECK-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <4 x double> %a0, <4 x double> undef, <4 x i32> <i32 3, i32 2, i32 2, i32 3>
   ; fadd forces execution domain
@@ -710,8 +1215,18 @@ define <4 x double> @stack_fold_permpd(<4 x double> %a0) {
 }
 
 define <4 x double> @stack_fold_permpdvar(<4 x i64> %a0, <4 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_permpdvar
-  ;CHECK:   vpermpd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_permpdvar:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovupd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; CHECK-NEXT:    vpermpd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <4 x double> @llvm.x86.avx512.permvar.df.256(<4 x double> %a1, <4 x i64> %a0)
   ; fadd forces execution domain
@@ -721,8 +1236,14 @@ define <4 x double> @stack_fold_permpdvar(<4 x i64> %a0, <4 x double> %a1) {
 declare <4 x double> @llvm.x86.avx512.permvar.df.256(<4 x double>, <4 x i64>) nounwind readonly
 
 define <8 x float> @stack_fold_permps(<8 x i32> %a0, <8 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_permps
-  ;CHECK:       vpermps {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_permps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x float> @llvm.x86.avx2.permps(<8 x float> %a1, <8 x i32> %a0)
   ret <8 x float> %2
@@ -730,24 +1251,44 @@ define <8 x float> @stack_fold_permps(<8 x i32> %a0, <8 x float> %a1) {
 declare <8 x float> @llvm.x86.avx2.permps(<8 x float>, <8 x i32>) nounwind readonly
 
 define <2 x double> @stack_fold_permilpd(<2 x double> %a0) {
-  ;CHECK-LABEL: stack_fold_permilpd
-  ;CHECK:   vpermilpd $1, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_permilpd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = mem[1,0]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <2 x double> %a0, <2 x double> undef, <2 x i32> <i32 1, i32 0>
   ret <2 x double> %2
 }
 
 define <4 x double> @stack_fold_permilpd_ymm(<4 x double> %a0) {
-  ;CHECK-LABEL: stack_fold_permilpd_ymm
-  ;CHECK:   vpermilpd $5, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_permilpd_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermilpd $5, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    # ymm0 = mem[1,0,3,2]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <4 x double> %a0, <4 x double> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
   ret <4 x double> %2
 }
 
 define <2 x double> @stack_fold_permilpdvar(<2 x double> %a0, <2 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_permilpdvar
-  ;CHECK:       vpermilpd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_permilpdvar:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermilpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <2 x double> @llvm.x86.avx.vpermilvar.pd(<2 x double> %a0, <2 x i64> %a1)
   ret <2 x double> %2
@@ -755,8 +1296,14 @@ define <2 x double> @stack_fold_permilpdvar(<2 x double> %a0, <2 x i64> %a1) {
 declare <2 x double> @llvm.x86.avx.vpermilvar.pd(<2 x double>, <2 x i64>) nounwind readnone
 
 define <4 x double> @stack_fold_permilpdvar_ymm(<4 x double> %a0, <4 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_permilpdvar_ymm
-  ;CHECK:       vpermilpd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_permilpdvar_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermilpd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <4 x double> @llvm.x86.avx.vpermilvar.pd.256(<4 x double> %a0, <4 x i64> %a1)
   ret <4 x double> %2
@@ -764,24 +1311,44 @@ define <4 x double> @stack_fold_permilpdvar_ymm(<4 x double> %a0, <4 x i64> %a1)
 declare <4 x double> @llvm.x86.avx.vpermilvar.pd.256(<4 x double>, <4 x i64>) nounwind readnone
 
 define <4 x float> @stack_fold_permilps(<4 x float> %a0) {
-  ;CHECK-LABEL: stack_fold_permilps
-  ;CHECK:   vpermilps $27, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_permilps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermilps $27, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = mem[3,2,1,0]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <4 x float> %a0, <4 x float> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
   ret <4 x float> %2
 }
 
 define <8 x float> @stack_fold_permilps_ymm(<8 x float> %a0) {
-  ;CHECK-LABEL: stack_fold_permilps_ymm
-  ;CHECK:   vpermilps $27, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_permilps_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermilps $27, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    # ymm0 = mem[3,2,1,0,7,6,5,4]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <8 x float> %a0, <8 x float> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
   ret <8 x float> %2
 }
 
 define <4 x float> @stack_fold_permilpsvar(<4 x float> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_permilpsvar
-  ;CHECK:       vpermilps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_permilpsvar:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermilps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float> %a0, <4 x i32> %a1)
   ret <4 x float> %2
@@ -789,8 +1356,14 @@ define <4 x float> @stack_fold_permilpsvar(<4 x float> %a0, <4 x i32> %a1) {
 declare <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float>, <4 x i32>) nounwind readnone
 
 define <8 x float> @stack_fold_permilpsvar_ymm(<8 x float> %a0, <8 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_permilpsvar_ymm
-  ;CHECK:       vpermilps {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_permilpsvar_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermilps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x float> @llvm.x86.avx.vpermilvar.ps.256(<8 x float> %a0, <8 x i32> %a1)
   ret <8 x float> %2
@@ -798,8 +1371,15 @@ define <8 x float> @stack_fold_permilpsvar_ymm(<8 x float> %a0, <8 x i32> %a1) {
 declare <8 x float> @llvm.x86.avx.vpermilvar.ps.256(<8 x float>, <8 x i32>) nounwind readnone
 
 define <8 x float> @stack_fold_permilpsvar_ymm_maskz(<8 x float> %a0, <8 x i32> %a1, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_permilpsvar_ymm_maskz
-  ;CHECK:       vpermilps {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_permilpsvar_ymm_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    vpermilps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 {%k1} {z} # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x float> @llvm.x86.avx.vpermilvar.ps.256(<8 x float> %a0, <8 x i32> %a1)
   %3 = bitcast i8 %mask to <8 x i1>
diff --git a/llvm/test/CodeGen/X86/stack-folding-fp-sse42.ll b/llvm/test/CodeGen/X86/stack-folding-fp-sse42.ll
index 37f235cc7..306ee3109 100644
--- a/llvm/test/CodeGen/X86/stack-folding-fp-sse42.ll
+++ b/llvm/test/CodeGen/X86/stack-folding-fp-sse42.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -O3 -verify-machineinstrs -disable-peephole -mtriple=x86_64-unknown-unknown -mattr=+sse4.2 < %s | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -9,32 +10,56 @@ target triple = "x86_64-unknown-unknown"
 ; relevant registers and check that the reload is correctly folded into the instruction.
 
 define <2 x double> @stack_fold_addpd(<2 x double> %a0, <2 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_addpd
-  ;CHECK:       addpd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_addpd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    addpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fadd <2 x double> %a0, %a1
   ret <2 x double> %2
 }
 
 define <4 x float> @stack_fold_addps(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_addps
-  ;CHECK:       addps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_addps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    addps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fadd <4 x float> %a0, %a1
   ret <4 x float> %2
 }
 
 define double @stack_fold_addsd(double %a0, double %a1) {
-  ;CHECK-LABEL: stack_fold_addsd
-  ;CHECK:       addsd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_addsd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movsd %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    addsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fadd double %a0, %a1
   ret double %2
 }
 
 define <2 x double> @stack_fold_addsd_int(<2 x double> %a0, <2 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_addsd_int
-  ;CHECK:       addsd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_addsd_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    addsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = extractelement <2 x double> %a0, i32 0
   %3 = extractelement <2 x double> %a1, i32 0
@@ -44,16 +69,28 @@ define <2 x double> @stack_fold_addsd_int(<2 x double> %a0, <2 x double> %a1) {
 }
 
 define float @stack_fold_addss(float %a0, float %a1) {
-  ;CHECK-LABEL: stack_fold_addss
-  ;CHECK:       addss {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_addss:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    addss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fadd float %a0, %a1
   ret float %2
 }
 
 define <4 x float> @stack_fold_addss_int(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_addss_int
-  ;CHECK:       addss {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_addss_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    addss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = extractelement <4 x float> %a0, i32 0
   %3 = extractelement <4 x float> %a1, i32 0
@@ -63,8 +100,14 @@ define <4 x float> @stack_fold_addss_int(<4 x float> %a0, <4 x float> %a1) {
 }
 
 define <2 x double> @stack_fold_addsubpd(<2 x double> %a0, <2 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_addsubpd
-  ;CHECK:       addsubpd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_addsubpd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    addsubpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x double> @llvm.x86.sse3.addsub.pd(<2 x double> %a0, <2 x double> %a1)
   ret <2 x double> %2
@@ -72,8 +115,14 @@ define <2 x double> @stack_fold_addsubpd(<2 x double> %a0, <2 x double> %a1) {
 declare <2 x double> @llvm.x86.sse3.addsub.pd(<2 x double>, <2 x double>) nounwind readnone
 
 define <4 x float> @stack_fold_addsubps(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_addsubps
-  ;CHECK:       addsubps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_addsubps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    addsubps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x float> @llvm.x86.sse3.addsub.ps(<4 x float> %a0, <4 x float> %a1)
   ret <4 x float> %2
@@ -81,8 +130,16 @@ define <4 x float> @stack_fold_addsubps(<4 x float> %a0, <4 x float> %a1) {
 declare <4 x float> @llvm.x86.sse3.addsub.ps(<4 x float>, <4 x float>) nounwind readnone
 
 define <2 x double> @stack_fold_andnpd(<2 x double> %a0, <2 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_andnpd
-  ;CHECK:       andnpd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_andnpd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    andnpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    xorpd %xmm1, %xmm1
+; CHECK-NEXT:    addpd %xmm1, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = bitcast <2 x double> %a0 to <2 x i64>
   %3 = bitcast <2 x double> %a1 to <2 x i64>
@@ -95,8 +152,16 @@ define <2 x double> @stack_fold_andnpd(<2 x double> %a0, <2 x double> %a1) {
 }
 
 define <4 x float> @stack_fold_andnps(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_andnps
-  ;CHECK:       andnps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_andnps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    andnps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    xorps %xmm1, %xmm1
+; CHECK-NEXT:    addps %xmm1, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = bitcast <4 x float> %a0 to <2 x i64>
   %3 = bitcast <4 x float> %a1 to <2 x i64>
@@ -109,8 +174,16 @@ define <4 x float> @stack_fold_andnps(<4 x float> %a0, <4 x float> %a1) {
 }
 
 define <2 x double> @stack_fold_andpd(<2 x double> %a0, <2 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_andpd
-  ;CHECK:       andpd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_andpd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    andpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    xorpd %xmm1, %xmm1
+; CHECK-NEXT:    addpd %xmm1, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = bitcast <2 x double> %a0 to <2 x i64>
   %3 = bitcast <2 x double> %a1 to <2 x i64>
@@ -122,8 +195,16 @@ define <2 x double> @stack_fold_andpd(<2 x double> %a0, <2 x double> %a1) {
 }
 
 define <4 x float> @stack_fold_andps(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_andps
-  ;CHECK:       andps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_andps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    andps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    xorps %xmm1, %xmm1
+; CHECK-NEXT:    addps %xmm1, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = bitcast <4 x float> %a0 to <2 x i64>
   %3 = bitcast <4 x float> %a1 to <2 x i64>
@@ -135,8 +216,17 @@ define <4 x float> @stack_fold_andps(<4 x float> %a0, <4 x float> %a1) {
 }
 
 define <2 x double> @stack_fold_blendpd(<2 x double> %a0, <2 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_blendpd
-  ;CHECK:       blendpd $2, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_blendpd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    blendpd $2, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = xmm0[0],mem[1]
+; CHECK-NEXT:    xorpd %xmm1, %xmm1
+; CHECK-NEXT:    addpd %xmm1, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = select <2 x i1> <i1 1, i1 0>, <2 x double> %a0, <2 x double> %a1
   ; fadd forces execution domain
@@ -145,8 +235,17 @@ define <2 x double> @stack_fold_blendpd(<2 x double> %a0, <2 x double> %a1) {
 }
 
 define <4 x float> @stack_fold_blendps(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_blendps
-  ;CHECK:       blendps $6, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_blendps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    blendps $6, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = xmm0[0],mem[1,2],xmm0[3]
+; CHECK-NEXT:    xorps %xmm1, %xmm1
+; CHECK-NEXT:    addps %xmm1, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = select <4 x i1> <i1 1, i1 0, i1 0, i1 1>, <4 x float> %a0, <4 x float> %a1
   ; fadd forces execution domain
@@ -155,8 +254,16 @@ define <4 x float> @stack_fold_blendps(<4 x float> %a0, <4 x float> %a1) {
 }
 
 define <2 x double> @stack_fold_blendvpd(<2 x double> %a0, <2 x double> %a1, <2 x double> %c) {
-  ;CHECK-LABEL: stack_fold_blendvpd
-  ;CHECK:       blendvpd %xmm0, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_blendvpd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movapd %xmm1, %xmm2
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    blendvpd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
+; CHECK-NEXT:    movapd %xmm2, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x double> @llvm.x86.sse41.blendvpd(<2 x double> %a1, <2 x double> %c, <2 x double> %a0)
   ret <2 x double> %2
@@ -164,8 +271,16 @@ define <2 x double> @stack_fold_blendvpd(<2 x double> %a0, <2 x double> %a1, <2
 declare <2 x double> @llvm.x86.sse41.blendvpd(<2 x double>, <2 x double>, <2 x double>) nounwind readnone
 
 define <4 x float> @stack_fold_blendvps(<4 x float> %a0, <4 x float> %a1, <4 x float> %c) {
-  ;CHECK-LABEL: stack_fold_blendvps
-  ;CHECK:       blendvps %xmm0, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_blendvps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps %xmm1, %xmm2
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    blendvps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
+; CHECK-NEXT:    movaps %xmm2, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x float> @llvm.x86.sse41.blendvps(<4 x float> %a1, <4 x float> %c, <4 x float> %a0)
   ret <4 x float> %2
@@ -173,8 +288,14 @@ define <4 x float> @stack_fold_blendvps(<4 x float> %a0, <4 x float> %a1, <4 x f
 declare <4 x float> @llvm.x86.sse41.blendvps(<4 x float>, <4 x float>, <4 x float>) nounwind readnone
 
 define <2 x double> @stack_fold_cmppd(<2 x double> %a0, <2 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_cmppd
-  ;CHECK:       cmpeqpd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_cmppd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    cmpeqpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x double> @llvm.x86.sse2.cmp.pd(<2 x double> %a0, <2 x double> %a1, i8 0)
   ret <2 x double> %2
@@ -182,8 +303,14 @@ define <2 x double> @stack_fold_cmppd(<2 x double> %a0, <2 x double> %a1) {
 declare <2 x double> @llvm.x86.sse2.cmp.pd(<2 x double>, <2 x double>, i8) nounwind readnone
 
 define <4 x float> @stack_fold_cmpps(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_cmpps
-  ;CHECK:       cmpeqps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_cmpps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    cmpeqps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x float> @llvm.x86.sse.cmp.ps(<4 x float> %a0, <4 x float> %a1, i8 0)
   ret <4 x float> %2
@@ -191,8 +318,17 @@ define <4 x float> @stack_fold_cmpps(<4 x float> %a0, <4 x float> %a1) {
 declare <4 x float> @llvm.x86.sse.cmp.ps(<4 x float>, <4 x float>, i8) nounwind readnone
 
 define i32 @stack_fold_cmpsd(double %a0, double %a1) {
-  ;CHECK-LABEL: stack_fold_cmpsd
-  ;CHECK:       cmpeqsd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_cmpsd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movsd %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    cmpeqsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq %xmm0, %rax
+; CHECK-NEXT:    andl $1, %eax
+; CHECK-NEXT:    # kill: def $eax killed $eax killed $rax
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fcmp oeq double %a0, %a1
   %3 = zext i1 %2 to i32
@@ -200,8 +336,14 @@ define i32 @stack_fold_cmpsd(double %a0, double %a1) {
 }
 
 define <2 x double> @stack_fold_cmpsd_int(<2 x double> %a0, <2 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_cmpsd_int
-  ;CHECK:       cmpeqsd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_cmpsd_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    cmpeqsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x double> @llvm.x86.sse2.cmp.sd(<2 x double> %a0, <2 x double> %a1, i8 0)
   ret <2 x double> %2
@@ -209,8 +351,16 @@ define <2 x double> @stack_fold_cmpsd_int(<2 x double> %a0, <2 x double> %a1) {
 declare <2 x double> @llvm.x86.sse2.cmp.sd(<2 x double>, <2 x double>, i8) nounwind readnone
 
 define i32 @stack_fold_cmpss(float %a0, float %a1) {
-  ;CHECK-LABEL: stack_fold_cmpss
-  ;CHECK:       cmpeqss {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_cmpss:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    cmpeqss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
+; CHECK-NEXT:    movd %xmm0, %eax
+; CHECK-NEXT:    andl $1, %eax
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fcmp oeq float %a0, %a1
   %3 = zext i1 %2 to i32
@@ -218,8 +368,14 @@ define i32 @stack_fold_cmpss(float %a0, float %a1) {
 }
 
 define <4 x float> @stack_fold_cmpss_int(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_cmpss_int
-  ;CHECK:       cmpeqss {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_cmpss_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    cmpeqss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x float> @llvm.x86.sse.cmp.ss(<4 x float> %a0, <4 x float> %a1, i8 0)
   ret <4 x float> %2
@@ -229,8 +385,18 @@ declare <4 x float> @llvm.x86.sse.cmp.ss(<4 x float>, <4 x float>, i8) nounwind
 ; TODO stack_fold_comisd
 
 define i32 @stack_fold_comisd_int(<2 x double> %a0, <2 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_comisd_int
-  ;CHECK:       comisd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_comisd_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    comisd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    setnp %al
+; CHECK-NEXT:    sete %cl
+; CHECK-NEXT:    andb %al, %cl
+; CHECK-NEXT:    movzbl %cl, %eax
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call i32 @llvm.x86.sse2.comieq.sd(<2 x double> %a0, <2 x double> %a1)
   ret i32 %2
@@ -240,8 +406,18 @@ declare i32 @llvm.x86.sse2.comieq.sd(<2 x double>, <2 x double>) nounwind readno
 ; TODO stack_fold_comiss
 
 define i32 @stack_fold_comiss_int(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_comiss_int
-  ;CHECK:       comiss {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_comiss_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    comiss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    setnp %al
+; CHECK-NEXT:    sete %cl
+; CHECK-NEXT:    andb %al, %cl
+; CHECK-NEXT:    movzbl %cl, %eax
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call i32 @llvm.x86.sse.comieq.ss(<4 x float> %a0, <4 x float> %a1)
   ret i32 %2
@@ -249,8 +425,14 @@ define i32 @stack_fold_comiss_int(<4 x float> %a0, <4 x float> %a1) {
 declare i32 @llvm.x86.sse.comieq.ss(<4 x float>, <4 x float>) nounwind readnone
 
 define <2 x double> @stack_fold_cvtdq2pd(<4 x i32> %a0) {
-  ;CHECK-LABEL: stack_fold_cvtdq2pd
-  ;CHECK:       cvtdq2pd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtdq2pd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    cvtdq2pd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <4 x i32> %a0, <4 x i32> undef, <2 x i32> <i32 0, i32 1>
   %3 = sitofp <2 x i32> %2 to <2 x double>
@@ -258,8 +440,14 @@ define <2 x double> @stack_fold_cvtdq2pd(<4 x i32> %a0) {
 }
 
 define <2 x double> @stack_fold_cvtdq2pd_int(<4 x i32> %a0) {
-  ;CHECK-LABEL: stack_fold_cvtdq2pd_int
-  ;CHECK:       cvtdq2pd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtdq2pd_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    cvtdq2pd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <4 x i32> %a0, <4 x i32> %a0, <2 x i32> <i32 0, i32 1>
   %cvt = sitofp <2 x i32> %2 to <2 x double>
@@ -267,16 +455,28 @@ define <2 x double> @stack_fold_cvtdq2pd_int(<4 x i32> %a0) {
 }
 
 define <4 x float> @stack_fold_cvtdq2ps(<4 x i32> %a0) {
-  ;CHECK-LABEL: stack_fold_cvtdq2ps
-  ;CHECK:       cvtdq2ps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtdq2ps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    cvtdq2ps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = sitofp <4 x i32> %a0 to <4 x float>
   ret <4 x float> %2
 }
 
 define <4 x i32> @stack_fold_cvtpd2dq(<2 x double> %a0) {
-  ;CHECK-LABEL: stack_fold_cvtpd2dq
-  ;CHECK:       cvtpd2dq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtpd2dq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    cvtpd2dq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x i32> @llvm.x86.sse2.cvtpd2dq(<2 x double> %a0)
   ret <4 x i32> %2
@@ -284,16 +484,28 @@ define <4 x i32> @stack_fold_cvtpd2dq(<2 x double> %a0) {
 declare <4 x i32> @llvm.x86.sse2.cvtpd2dq(<2 x double>) nounwind readnone
 
 define <2 x float> @stack_fold_cvtpd2ps(<2 x double> %a0) {
-  ;CHECK-LABEL: stack_fold_cvtpd2ps
-  ;CHECK:       cvtpd2ps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtpd2ps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    cvtpd2ps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fptrunc <2 x double> %a0 to <2 x float>
   ret <2 x float> %2
 }
 
 define <4 x i32> @stack_fold_cvtps2dq(<4 x float> %a0) {
-  ;CHECK-LABEL: stack_fold_cvtps2dq
-  ;CHECK:       cvtps2dq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtps2dq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    cvtps2dq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x i32> @llvm.x86.sse2.cvtps2dq(<4 x float> %a0)
   ret <4 x i32> %2
@@ -301,8 +513,14 @@ define <4 x i32> @stack_fold_cvtps2dq(<4 x float> %a0) {
 declare <4 x i32> @llvm.x86.sse2.cvtps2dq(<4 x float>) nounwind readnone
 
 define <2 x double> @stack_fold_cvtps2pd(<4 x float> %a0) {
-  ;CHECK-LABEL: stack_fold_cvtps2pd
-  ;CHECK:       cvtps2pd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtps2pd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    cvtps2pd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <4 x float> %a0, <4 x float> undef, <2 x i32> <i32 0, i32 1>
   %3 = fpext <2 x float> %2 to <2 x double>
@@ -310,8 +528,14 @@ define <2 x double> @stack_fold_cvtps2pd(<4 x float> %a0) {
 }
 
 define <2 x double> @stack_fold_cvtps2pd_int(<4 x float> %a0) {
-  ;CHECK-LABEL: stack_fold_cvtps2pd_int
-  ;CHECK:       cvtps2pd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtps2pd_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    cvtps2pd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <4 x float> %a0, <4 x float> %a0, <2 x i32> <i32 0, i32 1>
   %cvtps2pd = fpext <2 x float> %2 to <2 x double>
@@ -321,8 +545,14 @@ define <2 x double> @stack_fold_cvtps2pd_int(<4 x float> %a0) {
 ; TODO stack_fold_cvtsd2si
 
 define i32 @stack_fold_cvtsd2si_int(<2 x double> %a0) {
-  ;CHECK-LABEL: stack_fold_cvtsd2si_int
-  ;CHECK:       cvtsd2si {{-?[0-9]*}}(%rsp), %eax {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtsd2si_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    cvtsd2si {{[-0-9]+}}(%r{{[sb]}}p), %eax # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call i32 @llvm.x86.sse2.cvtsd2si(<2 x double> %a0)
   ret i32 %2
@@ -332,8 +562,14 @@ declare i32 @llvm.x86.sse2.cvtsd2si(<2 x double>) nounwind readnone
 ; TODO stack_fold_cvtsd2si64
 
 define i64 @stack_fold_cvtsd2si64_int(<2 x double> %a0) {
-  ;CHECK-LABEL: stack_fold_cvtsd2si64_int
-  ;CHECK:       cvtsd2si {{-?[0-9]*}}(%rsp), %rax {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtsd2si64_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    cvtsd2si {{[-0-9]+}}(%r{{[sb]}}p), %rax # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call i64 @llvm.x86.sse2.cvtsd2si64(<2 x double> %a0)
   ret i64 %2
@@ -341,16 +577,30 @@ define i64 @stack_fold_cvtsd2si64_int(<2 x double> %a0) {
 declare i64 @llvm.x86.sse2.cvtsd2si64(<2 x double>) nounwind readnone
 
 define float @stack_fold_cvtsd2ss(double %a0) minsize {
-  ;CHECK-LABEL: stack_fold_cvtsd2ss
-  ;CHECK:       cvtsd2ss {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtsd2ss:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movsd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    cvtsd2ss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fptrunc double %a0 to float
   ret float %2
 }
 
 define <4 x float> @stack_fold_cvtsd2ss_int(<2 x double> %a0) optsize {
-  ;CHECK-LABEL: stack_fold_cvtsd2ss_int
-  ;CHECK:       cvtsd2ss {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtsd2ss_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    xorps %xmm1, %xmm1
+; CHECK-NEXT:    cvtsd2ss {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
+; CHECK-NEXT:    movaps %xmm1, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x float> @llvm.x86.sse2.cvtsd2ss(<4 x float> <float 0x0, float 0x0, float 0x0, float 0x0>, <2 x double> %a0)
   ret <4 x float> %2
@@ -358,16 +608,89 @@ define <4 x float> @stack_fold_cvtsd2ss_int(<2 x double> %a0) optsize {
 declare <4 x float> @llvm.x86.sse2.cvtsd2ss(<4 x float>, <2 x double>) nounwind readnone
 
 define double @stack_fold_cvtsi2sd(i32 %a0) {
-  ;CHECK-LABEL: stack_fold_cvtsi2sd
-  ;CHECK:       cvtsi2sdl {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtsi2sd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    xorps %xmm0, %xmm0
+; CHECK-NEXT:    cvtsi2sdl {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = sitofp i32 %a0 to double
   ret double %2
 }
 
 define <2 x double> @stack_fold_cvtsi2sd_int(i32 %a0, <2 x double> %b0) {
-  ;CHECK-LABEL: stack_fold_cvtsi2sd_int
-  ;CHECK:       cvtsi2sdl {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtsi2sd_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    cvtsi2sdl {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = sitofp i32 %a0 to double
   %3 = insertelement <2 x double> %b0, double %2, i64 0
@@ -375,16 +698,89 @@ define <2 x double> @stack_fold_cvtsi2sd_int(i32 %a0, <2 x double> %b0) {
 }
 
 define double @stack_fold_cvtsi642sd(i64 %a0) {
-  ;CHECK-LABEL: stack_fold_cvtsi642sd
-  ;CHECK:       cvtsi2sdq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtsi642sd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    xorps %xmm0, %xmm0
+; CHECK-NEXT:    cvtsi2sdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Folded Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = sitofp i64 %a0 to double
   ret double %2
 }
 
 define <2 x double> @stack_fold_cvtsi642sd_int(i64 %a0, <2 x double> %b0) {
-  ;CHECK-LABEL: stack_fold_cvtsi642sd_int
-  ;CHECK:       cvtsi2sdq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtsi642sd_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    cvtsi2sdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Folded Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = sitofp i64 %a0 to double
   %3 = insertelement <2 x double> %b0, double %2, i64 0
@@ -392,16 +788,89 @@ define <2 x double> @stack_fold_cvtsi642sd_int(i64 %a0, <2 x double> %b0) {
 }
 
 define float @stack_fold_cvtsi2ss(i32 %a0) {
-  ;CHECK-LABEL: stack_fold_cvtsi2ss
-  ;CHECK:       cvtsi2ssl {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtsi2ss:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    xorps %xmm0, %xmm0
+; CHECK-NEXT:    cvtsi2ssl {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = sitofp i32 %a0 to float
   ret float %2
 }
 
 define <4 x float> @stack_fold_cvtsi2ss_int(i32 %a0, <4 x float> %b0) {
-  ;CHECK-LABEL: stack_fold_cvtsi2ss_int
-  ;CHECK:  cvtsi2ssl {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtsi2ss_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    cvtsi2ssl {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = sitofp i32 %a0 to float
   %3 = insertelement <4 x float> %b0, float %2, i64 0
@@ -409,16 +878,89 @@ define <4 x float> @stack_fold_cvtsi2ss_int(i32 %a0, <4 x float> %b0) {
 }
 
 define float @stack_fold_cvtsi642ss(i64 %a0) {
-  ;CHECK-LABEL: stack_fold_cvtsi642ss
-  ;CHECK:       cvtsi2ssq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtsi642ss:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    xorps %xmm0, %xmm0
+; CHECK-NEXT:    cvtsi2ssq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Folded Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = sitofp i64 %a0 to float
   ret float %2
 }
 
 define <4 x float> @stack_fold_cvtsi642ss_int(i64 %a0, <4 x float> %b0) {
-  ;CHECK-LABEL: stack_fold_cvtsi642ss_int
-  ;CHECK:  cvtsi2ssq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtsi642ss_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    cvtsi2ssq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Folded Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = sitofp i64 %a0 to float
   %3 = insertelement <4 x float> %b0, float %2, i64 0
@@ -426,16 +968,30 @@ define <4 x float> @stack_fold_cvtsi642ss_int(i64 %a0, <4 x float> %b0) {
 }
 
 define double @stack_fold_cvtss2sd(float %a0) minsize {
-  ;CHECK-LABEL: stack_fold_cvtss2sd
-  ;CHECK:       cvtss2sd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtss2sd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movss %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    cvtss2sd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fpext float %a0 to double
   ret double %2
 }
 
 define <2 x double> @stack_fold_cvtss2sd_int(<4 x float> %a0) optsize {
-  ;CHECK-LABEL: stack_fold_cvtss2sd_int
-  ;CHECK:       cvtss2sd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtss2sd_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    xorps %xmm0, %xmm0
+; CHECK-NEXT:    cvtss2sd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    movq {{.*#+}} xmm0 = xmm0[0],zero
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = extractelement <4 x float> %a0, i64 0
   %3 = fpext float %2 to double
@@ -446,8 +1002,14 @@ define <2 x double> @stack_fold_cvtss2sd_int(<4 x float> %a0) optsize {
 ; TODO stack_fold_cvtss2si
 
 define i32 @stack_fold_cvtss2si_int(<4 x float> %a0) {
-  ;CHECK-LABEL: stack_fold_cvtss2si_int
-  ;CHECK:       cvtss2si {{-?[0-9]*}}(%rsp), %eax {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtss2si_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    cvtss2si {{[-0-9]+}}(%r{{[sb]}}p), %eax # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call i32 @llvm.x86.sse.cvtss2si(<4 x float> %a0)
   ret i32 %2
@@ -457,8 +1019,14 @@ declare i32 @llvm.x86.sse.cvtss2si(<4 x float>) nounwind readnone
 ; TODO stack_fold_cvtss2si64
 
 define i64 @stack_fold_cvtss2si64_int(<4 x float> %a0) {
-  ;CHECK-LABEL: stack_fold_cvtss2si64_int
-  ;CHECK:       cvtss2si {{-?[0-9]*}}(%rsp), %rax {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtss2si64_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    cvtss2si {{[-0-9]+}}(%r{{[sb]}}p), %rax # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call i64 @llvm.x86.sse.cvtss2si64(<4 x float> %a0)
   ret i64 %2
@@ -466,8 +1034,14 @@ define i64 @stack_fold_cvtss2si64_int(<4 x float> %a0) {
 declare i64 @llvm.x86.sse.cvtss2si64(<4 x float>) nounwind readnone
 
 define <4 x i32> @stack_fold_cvttpd2dq(<2 x double> %a0) {
-  ;CHECK-LABEL: stack_fold_cvttpd2dq
-  ;CHECK:       cvttpd2dq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvttpd2dq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    cvttpd2dq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x i32> @llvm.x86.sse2.cvttpd2dq(<2 x double> %a0)
   ret <4 x i32> %2
@@ -475,24 +1049,42 @@ define <4 x i32> @stack_fold_cvttpd2dq(<2 x double> %a0) {
 declare <4 x i32> @llvm.x86.sse2.cvttpd2dq(<2 x double>) nounwind readnone
 
 define <4 x i32> @stack_fold_cvttps2dq(<4 x float> %a0) {
-  ;CHECK-LABEL: stack_fold_cvttps2dq
-  ;CHECK:       cvttps2dq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvttps2dq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    cvttps2dq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fptosi <4 x float> %a0 to <4 x i32>
   ret <4 x i32> %2
 }
 
 define i32 @stack_fold_cvttsd2si(double %a0) {
-  ;CHECK-LABEL: stack_fold_cvttsd2si
-  ;CHECK:       cvttsd2si {{-?[0-9]*}}(%rsp), %eax {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvttsd2si:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movsd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    cvttsd2si {{[-0-9]+}}(%r{{[sb]}}p), %eax # 8-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fptosi double %a0 to i32
   ret i32 %2
 }
 
 define i32 @stack_fold_cvttsd2si_int(<2 x double> %a0) {
-  ;CHECK-LABEL: stack_fold_cvttsd2si_int
-  ;CHECK:       cvttsd2si {{-?[0-9]*}}(%rsp), %eax {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvttsd2si_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    cvttsd2si {{[-0-9]+}}(%r{{[sb]}}p), %eax # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call i32 @llvm.x86.sse2.cvttsd2si(<2 x double> %a0)
   ret i32 %2
@@ -500,16 +1092,28 @@ define i32 @stack_fold_cvttsd2si_int(<2 x double> %a0) {
 declare i32 @llvm.x86.sse2.cvttsd2si(<2 x double>) nounwind readnone
 
 define i64 @stack_fold_cvttsd2si64(double %a0) {
-  ;CHECK-LABEL: stack_fold_cvttsd2si64
-  ;CHECK:       cvttsd2si {{-?[0-9]*}}(%rsp), %rax {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvttsd2si64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movsd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    cvttsd2si {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fptosi double %a0 to i64
   ret i64 %2
 }
 
 define i64 @stack_fold_cvttsd2si64_int(<2 x double> %a0) {
-  ;CHECK-LABEL: stack_fold_cvttsd2si64_int
-  ;CHECK:       cvttsd2si {{-?[0-9]*}}(%rsp), %rax {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvttsd2si64_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    cvttsd2si {{[-0-9]+}}(%r{{[sb]}}p), %rax # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call i64 @llvm.x86.sse2.cvttsd2si64(<2 x double> %a0)
   ret i64 %2
@@ -517,16 +1121,28 @@ define i64 @stack_fold_cvttsd2si64_int(<2 x double> %a0) {
 declare i64 @llvm.x86.sse2.cvttsd2si64(<2 x double>) nounwind readnone
 
 define i32 @stack_fold_cvttss2si(float %a0) {
-  ;CHECK-LABEL: stack_fold_cvttss2si
-  ;CHECK:       cvttss2si {{-?[0-9]*}}(%rsp), %eax {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvttss2si:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movss %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    cvttss2si {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fptosi float %a0 to i32
   ret i32 %2
 }
 
 define i32 @stack_fold_cvttss2si_int(<4 x float> %a0) {
-  ;CHECK-LABEL: stack_fold_cvttss2si_int
-  ;CHECK:       cvttss2si {{-?[0-9]*}}(%rsp), %eax {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvttss2si_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    cvttss2si {{[-0-9]+}}(%r{{[sb]}}p), %eax # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call i32 @llvm.x86.sse.cvttss2si(<4 x float> %a0)
   ret i32 %2
@@ -534,16 +1150,28 @@ define i32 @stack_fold_cvttss2si_int(<4 x float> %a0) {
 declare i32 @llvm.x86.sse.cvttss2si(<4 x float>) nounwind readnone
 
 define i64 @stack_fold_cvttss2si64(float %a0) {
-  ;CHECK-LABEL: stack_fold_cvttss2si64
-  ;CHECK:       cvttss2si {{-?[0-9]*}}(%rsp), %rax {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvttss2si64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movss %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    cvttss2si {{[-0-9]+}}(%r{{[sb]}}p), %rax # 4-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fptosi float %a0 to i64
   ret i64 %2
 }
 
 define i64 @stack_fold_cvttss2si64_int(<4 x float> %a0) {
-  ;CHECK-LABEL: stack_fold_cvttss2si64_int
-  ;CHECK:       cvttss2si {{-?[0-9]*}}(%rsp), %rax {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvttss2si64_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    cvttss2si {{[-0-9]+}}(%r{{[sb]}}p), %rax # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call i64 @llvm.x86.sse.cvttss2si64(<4 x float> %a0)
   ret i64 %2
@@ -551,32 +1179,56 @@ define i64 @stack_fold_cvttss2si64_int(<4 x float> %a0) {
 declare i64 @llvm.x86.sse.cvttss2si64(<4 x float>) nounwind readnone
 
 define <2 x double> @stack_fold_divpd(<2 x double> %a0, <2 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_divpd
-  ;CHECK:       divpd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_divpd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    divpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fdiv <2 x double> %a0, %a1
   ret <2 x double> %2
 }
 
 define <4 x float> @stack_fold_divps(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_divps
-  ;CHECK:       divps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_divps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    divps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fdiv <4 x float> %a0, %a1
   ret <4 x float> %2
 }
 
 define double @stack_fold_divsd(double %a0, double %a1) {
-  ;CHECK-LABEL: stack_fold_divsd
-  ;CHECK:       divsd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_divsd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movsd %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    divsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fdiv double %a0, %a1
   ret double %2
 }
 
 define <2 x double> @stack_fold_divsd_int(<2 x double> %a0, <2 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_divsd_int
-  ;CHECK:       divsd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_divsd_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    divsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = extractelement <2 x double> %a0, i32 0
   %3 = extractelement <2 x double> %a1, i32 0
@@ -587,16 +1239,28 @@ define <2 x double> @stack_fold_divsd_int(<2 x double> %a0, <2 x double> %a1) {
 declare <2 x double> @llvm.x86.sse2.div.sd(<2 x double>, <2 x double>) nounwind readnone
 
 define float @stack_fold_divss(float %a0, float %a1) {
-  ;CHECK-LABEL: stack_fold_divss
-  ;CHECK:       divss {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_divss:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    divss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fdiv float %a0, %a1
   ret float %2
 }
 
 define <4 x float> @stack_fold_divss_int(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_divss_int
-  ;CHECK:       divss {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_divss_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    divss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = extractelement <4 x float> %a0, i32 0
   %3 = extractelement <4 x float> %a1, i32 0
@@ -607,8 +1271,14 @@ define <4 x float> @stack_fold_divss_int(<4 x float> %a0, <4 x float> %a1) {
 declare <4 x float> @llvm.x86.sse.div.ss(<4 x float>, <4 x float>) nounwind readnone
 
 define <2 x double> @stack_fold_dppd(<2 x double> %a0, <2 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_dppd
-  ;CHECK:       dppd $7, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_dppd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    dppd $7, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x double> @llvm.x86.sse41.dppd(<2 x double> %a0, <2 x double> %a1, i8 7)
   ret <2 x double> %2
@@ -616,8 +1286,14 @@ define <2 x double> @stack_fold_dppd(<2 x double> %a0, <2 x double> %a1) {
 declare <2 x double> @llvm.x86.sse41.dppd(<2 x double>, <2 x double>, i8) nounwind readnone
 
 define <4 x float> @stack_fold_dpps(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_dpps
-  ;CHECK:       dpps $7, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_dpps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    dpps $7, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x float> @llvm.x86.sse41.dpps(<4 x float> %a0, <4 x float> %a1, i8 7)
   ret <4 x float> %2
@@ -625,9 +1301,45 @@ define <4 x float> @stack_fold_dpps(<4 x float> %a0, <4 x float> %a1) {
 declare <4 x float> @llvm.x86.sse41.dpps(<4 x float>, <4 x float>, i8) nounwind readnone
 
 define i32 @stack_fold_extractps(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_extractps
-  ;CHECK:       extractps $1, {{%xmm[0-9][0-9]*}}, {{-?[0-9]*}}(%rsp) {{.*#+}} 4-byte Folded Spill
-  ;CHECK:       movl    {{-?[0-9]*}}(%rsp), %eax {{.*#+}} 4-byte Reload
+; CHECK-LABEL: stack_fold_extractps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    addps %xmm1, %xmm0
+; CHECK-NEXT:    extractps $1, %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   ; fadd forces execution domain
   %1 = fadd <4 x float> %a0, %a1
   %2 = extractelement <4 x float> %1, i32 1
@@ -637,8 +1349,14 @@ define i32 @stack_fold_extractps(<4 x float> %a0, <4 x float> %a1) {
 }
 
 define <2 x double> @stack_fold_haddpd(<2 x double> %a0, <2 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_haddpd
-  ;CHECK:       haddpd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_haddpd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    haddpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x double> @llvm.x86.sse3.hadd.pd(<2 x double> %a0, <2 x double> %a1)
   ret <2 x double> %2
@@ -646,8 +1364,14 @@ define <2 x double> @stack_fold_haddpd(<2 x double> %a0, <2 x double> %a1) {
 declare <2 x double> @llvm.x86.sse3.hadd.pd(<2 x double>, <2 x double>) nounwind readnone
 
 define <4 x float> @stack_fold_haddps(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_haddps
-  ;CHECK:       haddps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_haddps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    haddps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x float> @llvm.x86.sse3.hadd.ps(<4 x float> %a0, <4 x float> %a1)
   ret <4 x float> %2
@@ -655,8 +1379,14 @@ define <4 x float> @stack_fold_haddps(<4 x float> %a0, <4 x float> %a1) {
 declare <4 x float> @llvm.x86.sse3.hadd.ps(<4 x float>, <4 x float>) nounwind readnone
 
 define <2 x double> @stack_fold_hsubpd(<2 x double> %a0, <2 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_hsubpd
-  ;CHECK:       hsubpd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_hsubpd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    hsubpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x double> @llvm.x86.sse3.hsub.pd(<2 x double> %a0, <2 x double> %a1)
   ret <2 x double> %2
@@ -664,8 +1394,14 @@ define <2 x double> @stack_fold_hsubpd(<2 x double> %a0, <2 x double> %a1) {
 declare <2 x double> @llvm.x86.sse3.hsub.pd(<2 x double>, <2 x double>) nounwind readnone
 
 define <4 x float> @stack_fold_hsubps(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_hsubps
-  ;CHECK:       hsubps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_hsubps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    hsubps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x float> @llvm.x86.sse3.hsub.ps(<4 x float> %a0, <4 x float> %a1)
   ret <4 x float> %2
@@ -673,9 +1409,15 @@ define <4 x float> @stack_fold_hsubps(<4 x float> %a0, <4 x float> %a1) {
 declare <4 x float> @llvm.x86.sse3.hsub.ps(<4 x float>, <4 x float>) nounwind readnone
 
 define <4 x float> @stack_fold_insertps(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_insertps
-  ;CHECK:       insertps $17, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
-  ;CHECK-NEXT:                                                        {{.*#+}} xmm0 = zero,mem[0],xmm0[2,3]
+; CHECK-LABEL: stack_fold_insertps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    insertps $17, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = zero,mem[0],xmm0[2,3]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x float> @llvm.x86.sse41.insertps(<4 x float> %a0, <4 x float> %a1, i8 209)
   ret <4 x float> %2
@@ -683,8 +1425,14 @@ define <4 x float> @stack_fold_insertps(<4 x float> %a0, <4 x float> %a1) {
 declare <4 x float> @llvm.x86.sse41.insertps(<4 x float>, <4 x float>, i8) nounwind readnone
 
 define <2 x double> @stack_fold_maxpd(<2 x double> %a0, <2 x double> %a1) #0 {
-  ;CHECK-LABEL: stack_fold_maxpd
-  ;CHECK:       maxpd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_maxpd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    maxpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x double> @llvm.x86.sse2.max.pd(<2 x double> %a0, <2 x double> %a1)
   ret <2 x double> %2
@@ -692,16 +1440,28 @@ define <2 x double> @stack_fold_maxpd(<2 x double> %a0, <2 x double> %a1) #0 {
 declare <2 x double> @llvm.x86.sse2.max.pd(<2 x double>, <2 x double>) nounwind readnone
 
 define <2 x double> @stack_fold_maxpd_commutable(<2 x double> %a0, <2 x double> %a1) #1 {
-  ;CHECK-LABEL: stack_fold_maxpd_commutable
-  ;CHECK:       maxpd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_maxpd_commutable:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    maxpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x double> @llvm.x86.sse2.max.pd(<2 x double> %a0, <2 x double> %a1)
   ret <2 x double> %2
 }
 
 define <4 x float> @stack_fold_maxps(<4 x float> %a0, <4 x float> %a1) #0 {
-  ;CHECK-LABEL: stack_fold_maxps
-  ;CHECK:       maxps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_maxps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    maxps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x float> @llvm.x86.sse.max.ps(<4 x float> %a0, <4 x float> %a1)
   ret <4 x float> %2
@@ -709,16 +1469,28 @@ define <4 x float> @stack_fold_maxps(<4 x float> %a0, <4 x float> %a1) #0 {
 declare <4 x float> @llvm.x86.sse.max.ps(<4 x float>, <4 x float>) nounwind readnone
 
 define <4 x float> @stack_fold_maxps_commutable(<4 x float> %a0, <4 x float> %a1) #1 {
-  ;CHECK-LABEL: stack_fold_maxps_commutable
-  ;CHECK:       maxps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_maxps_commutable:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    maxps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x float> @llvm.x86.sse.max.ps(<4 x float> %a0, <4 x float> %a1)
   ret <4 x float> %2
 }
 
 define double @stack_fold_maxsd(double %a0, double %a1) #0 {
-  ;CHECK-LABEL: stack_fold_maxsd
-  ;CHECK:       maxsd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_maxsd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movsd %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    maxsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fcmp ogt double %a0, %a1
   %3 = select i1 %2, double %a0, double %a1
@@ -726,8 +1498,14 @@ define double @stack_fold_maxsd(double %a0, double %a1) #0 {
 }
 
 define double @stack_fold_maxsd_commutable(double %a0, double %a1) #1 {
-  ;CHECK-LABEL: stack_fold_maxsd_commutable
-  ;CHECK:       maxsd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_maxsd_commutable:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movsd %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    maxsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fcmp ogt double %a0, %a1
   %3 = select i1 %2, double %a0, double %a1
@@ -735,8 +1513,14 @@ define double @stack_fold_maxsd_commutable(double %a0, double %a1) #1 {
 }
 
 define <2 x double> @stack_fold_maxsd_int(<2 x double> %a0, <2 x double> %a1) #0 {
-  ;CHECK-LABEL: stack_fold_maxsd_int
-  ;CHECK:       maxsd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_maxsd_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    maxsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x double> @llvm.x86.sse2.max.sd(<2 x double> %a0, <2 x double> %a1)
   ret <2 x double> %2
@@ -744,8 +1528,14 @@ define <2 x double> @stack_fold_maxsd_int(<2 x double> %a0, <2 x double> %a1) #0
 declare <2 x double> @llvm.x86.sse2.max.sd(<2 x double>, <2 x double>) nounwind readnone
 
 define float @stack_fold_maxss(float %a0, float %a1) #0 {
-  ;CHECK-LABEL: stack_fold_maxss
-  ;CHECK:       maxss {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_maxss:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    maxss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fcmp ogt float %a0, %a1
   %3 = select i1 %2, float %a0, float %a1
@@ -753,8 +1543,14 @@ define float @stack_fold_maxss(float %a0, float %a1) #0 {
 }
 
 define float @stack_fold_maxss_commutable(float %a0, float %a1) #1 {
-  ;CHECK-LABEL: stack_fold_maxss_commutable
-  ;CHECK:       maxss {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_maxss_commutable:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    maxss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fcmp ogt float %a0, %a1
   %3 = select i1 %2, float %a0, float %a1
@@ -762,8 +1558,14 @@ define float @stack_fold_maxss_commutable(float %a0, float %a1) #1 {
 }
 
 define <4 x float> @stack_fold_maxss_int(<4 x float> %a0, <4 x float> %a1) #0 {
-  ;CHECK-LABEL: stack_fold_maxss_int
-  ;CHECK:       maxss {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_maxss_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    maxss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x float> @llvm.x86.sse.max.ss(<4 x float> %a0, <4 x float> %a1)
   ret <4 x float> %2
@@ -771,8 +1573,14 @@ define <4 x float> @stack_fold_maxss_int(<4 x float> %a0, <4 x float> %a1) #0 {
 declare <4 x float> @llvm.x86.sse.max.ss(<4 x float>, <4 x float>) nounwind readnone
 
 define <2 x double> @stack_fold_minpd(<2 x double> %a0, <2 x double> %a1) #0 {
-  ;CHECK-LABEL: stack_fold_minpd
-  ;CHECK:       minpd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_minpd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    minpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x double> @llvm.x86.sse2.min.pd(<2 x double> %a0, <2 x double> %a1)
   ret <2 x double> %2
@@ -780,16 +1588,28 @@ define <2 x double> @stack_fold_minpd(<2 x double> %a0, <2 x double> %a1) #0 {
 declare <2 x double> @llvm.x86.sse2.min.pd(<2 x double>, <2 x double>) nounwind readnone
 
 define <2 x double> @stack_fold_minpd_commutable(<2 x double> %a0, <2 x double> %a1) #1 {
-  ;CHECK-LABEL: stack_fold_minpd_commutable
-  ;CHECK:       minpd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_minpd_commutable:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    minpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x double> @llvm.x86.sse2.min.pd(<2 x double> %a0, <2 x double> %a1)
   ret <2 x double> %2
 }
 
 define <4 x float> @stack_fold_minps(<4 x float> %a0, <4 x float> %a1) #0 {
-  ;CHECK-LABEL: stack_fold_minps
-  ;CHECK:       minps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_minps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    minps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x float> @llvm.x86.sse.min.ps(<4 x float> %a0, <4 x float> %a1)
   ret <4 x float> %2
@@ -797,16 +1617,28 @@ define <4 x float> @stack_fold_minps(<4 x float> %a0, <4 x float> %a1) #0 {
 declare <4 x float> @llvm.x86.sse.min.ps(<4 x float>, <4 x float>) nounwind readnone
 
 define <4 x float> @stack_fold_minps_commutable(<4 x float> %a0, <4 x float> %a1) #1 {
-  ;CHECK-LABEL: stack_fold_minps_commutable
-  ;CHECK:       minps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_minps_commutable:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    minps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x float> @llvm.x86.sse.min.ps(<4 x float> %a0, <4 x float> %a1)
   ret <4 x float> %2
 }
 
 define double @stack_fold_minsd(double %a0, double %a1) #0 {
-  ;CHECK-LABEL: stack_fold_minsd
-  ;CHECK:       minsd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_minsd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movsd %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    minsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fcmp olt double %a0, %a1
   %3 = select i1 %2, double %a0, double %a1
@@ -814,8 +1646,14 @@ define double @stack_fold_minsd(double %a0, double %a1) #0 {
 }
 
 define double @stack_fold_minsd_commutable(double %a0, double %a1) #1 {
-  ;CHECK-LABEL: stack_fold_minsd_commutable
-  ;CHECK:       minsd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_minsd_commutable:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movsd %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    minsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fcmp olt double %a0, %a1
   %3 = select i1 %2, double %a0, double %a1
@@ -823,8 +1661,14 @@ define double @stack_fold_minsd_commutable(double %a0, double %a1) #1 {
 }
 
 define <2 x double> @stack_fold_minsd_int(<2 x double> %a0, <2 x double> %a1) #0 {
-  ;CHECK-LABEL: stack_fold_minsd_int
-  ;CHECK:       minsd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_minsd_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    minsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x double> @llvm.x86.sse2.min.sd(<2 x double> %a0, <2 x double> %a1)
   ret <2 x double> %2
@@ -832,8 +1676,14 @@ define <2 x double> @stack_fold_minsd_int(<2 x double> %a0, <2 x double> %a1) #0
 declare <2 x double> @llvm.x86.sse2.min.sd(<2 x double>, <2 x double>) nounwind readnone
 
 define float @stack_fold_minss(float %a0, float %a1) #0 {
-  ;CHECK-LABEL: stack_fold_minss
-  ;CHECK:       minss {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_minss:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    minss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fcmp olt float %a0, %a1
   %3 = select i1 %2, float %a0, float %a1
@@ -841,8 +1691,14 @@ define float @stack_fold_minss(float %a0, float %a1) #0 {
 }
 
 define float @stack_fold_minss_commutable(float %a0, float %a1) #1 {
-  ;CHECK-LABEL: stack_fold_minss_commutable
-  ;CHECK:       minss {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_minss_commutable:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    minss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fcmp olt float %a0, %a1
   %3 = select i1 %2, float %a0, float %a1
@@ -850,8 +1706,14 @@ define float @stack_fold_minss_commutable(float %a0, float %a1) #1 {
 }
 
 define <4 x float> @stack_fold_minss_int(<4 x float> %a0, <4 x float> %a1) #0 {
-  ;CHECK-LABEL: stack_fold_minss_int
-  ;CHECK:       minss {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_minss_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    minss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x float> @llvm.x86.sse.min.ss(<4 x float> %a0, <4 x float> %a1)
   ret <4 x float> %2
@@ -859,8 +1721,15 @@ define <4 x float> @stack_fold_minss_int(<4 x float> %a0, <4 x float> %a1) #0 {
 declare <4 x float> @llvm.x86.sse.min.ss(<4 x float>, <4 x float>) nounwind readnone
 
 define <2 x double> @stack_fold_movddup(<2 x double> %a0) {
-  ;CHECK-LABEL: stack_fold_movddup
-  ;CHECK:   movddup {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_movddup:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    movddup {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = mem[0,0]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <2 x double> %a0, <2 x double> undef, <2 x i32> <i32 0, i32 0>
   ret <2 x double> %2
@@ -872,48 +1741,86 @@ define <2 x double> @stack_fold_movddup(<2 x double> %a0) {
 ; TODO stack_fold_movlps (load / store)
 
 define <4 x float> @stack_fold_movshdup(<4 x float> %a0) {
-  ;CHECK-LABEL: stack_fold_movshdup
-  ;CHECK:       movshdup {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_movshdup:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    movshdup {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = mem[1,1,3,3]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <4 x float> %a0, <4 x float> undef, <4 x i32> <i32 1, i32 1, i32 3, i32 3>
   ret <4 x float> %2
 }
 
 define <4 x float> @stack_fold_movsldup(<4 x float> %a0) {
-  ;CHECK-LABEL: stack_fold_movsldup
-  ;CHECK:       movsldup {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_movsldup:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    movsldup {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = mem[0,0,2,2]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <4 x float> %a0, <4 x float> undef, <4 x i32> <i32 0, i32 0, i32 2, i32 2>
   ret <4 x float> %2
 }
 
 define <2 x double> @stack_fold_mulpd(<2 x double> %a0, <2 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_mulpd
-  ;CHECK:       mulpd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_mulpd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    mulpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fmul <2 x double> %a0, %a1
   ret <2 x double> %2
 }
 
 define <4 x float> @stack_fold_mulps(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_mulps
-  ;CHECK:       mulps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_mulps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    mulps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fmul <4 x float> %a0, %a1
   ret <4 x float> %2
 }
 
 define double @stack_fold_mulsd(double %a0, double %a1) {
-  ;CHECK-LABEL: stack_fold_mulsd
-  ;CHECK:       mulsd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_mulsd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movsd %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    mulsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fmul double %a0, %a1
   ret double %2
 }
 
 define <2 x double> @stack_fold_mulsd_int(<2 x double> %a0, <2 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_mulsd_int
-  ;CHECK:       mulsd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_mulsd_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    mulsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = extractelement <2 x double> %a0, i32 0
   %3 = extractelement <2 x double> %a1, i32 0
@@ -923,16 +1830,28 @@ define <2 x double> @stack_fold_mulsd_int(<2 x double> %a0, <2 x double> %a1) {
 }
 
 define float @stack_fold_mulss(float %a0, float %a1) {
-  ;CHECK-LABEL: stack_fold_mulss
-  ;CHECK:       mulss {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_mulss:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    mulss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fmul float %a0, %a1
   ret float %2
 }
 
 define <4 x float> @stack_fold_mulss_int(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_mulss_int
-  ;CHECK:       mulss {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_mulss_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    mulss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = extractelement <4 x float> %a0, i32 0
   %3 = extractelement <4 x float> %a1, i32 0
@@ -942,8 +1861,16 @@ define <4 x float> @stack_fold_mulss_int(<4 x float> %a0, <4 x float> %a1) {
 }
 
 define <2 x double> @stack_fold_orpd(<2 x double> %a0, <2 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_orpd
-  ;CHECK:       orpd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_orpd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    orpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    xorpd %xmm1, %xmm1
+; CHECK-NEXT:    addpd %xmm1, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = bitcast <2 x double> %a0 to <2 x i64>
   %3 = bitcast <2 x double> %a1 to <2 x i64>
@@ -955,8 +1882,16 @@ define <2 x double> @stack_fold_orpd(<2 x double> %a0, <2 x double> %a1) {
 }
 
 define <4 x float> @stack_fold_orps(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_orps
-  ;CHECK:       orps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_orps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    orps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    xorps %xmm1, %xmm1
+; CHECK-NEXT:    addps %xmm1, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = bitcast <4 x float> %a0 to <2 x i64>
   %3 = bitcast <4 x float> %a1 to <2 x i64>
@@ -970,8 +1905,14 @@ define <4 x float> @stack_fold_orps(<4 x float> %a0, <4 x float> %a1) {
 ; TODO stack_fold_rcpps
 
 define <4 x float> @stack_fold_rcpps_int(<4 x float> %a0) {
-  ;CHECK-LABEL: stack_fold_rcpps_int
-  ;CHECK:       rcpps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_rcpps_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    rcpps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x float> @llvm.x86.sse.rcp.ps(<4 x float> %a0)
   ret <4 x float> %2
@@ -981,8 +1922,16 @@ declare <4 x float> @llvm.x86.sse.rcp.ps(<4 x float>) nounwind readnone
 ; TODO stack_fold_rcpss
 
 define <4 x float> @stack_fold_rcpss_int(<4 x float> %a0, <4 x float> %a1) optsize {
-  ;CHECK-LABEL: stack_fold_rcpss_int
-  ;CHECK:       rcpss {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_rcpss_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    rcpss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x float> @llvm.x86.sse.rcp.ss(<4 x float> %a1)
   %3 = extractelement <4 x float> %2, i32 0
@@ -992,8 +1941,14 @@ define <4 x float> @stack_fold_rcpss_int(<4 x float> %a0, <4 x float> %a1) optsi
 declare <4 x float> @llvm.x86.sse.rcp.ss(<4 x float>)
 
 define <2 x double> @stack_fold_roundpd(<2 x double> %a0) {
-  ;CHECK-LABEL: stack_fold_roundpd
-  ;CHECK:       roundpd $7, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_roundpd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    roundpd $7, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x double> @llvm.x86.sse41.round.pd(<2 x double> %a0, i32 7)
   ret <2 x double> %2
@@ -1001,8 +1956,14 @@ define <2 x double> @stack_fold_roundpd(<2 x double> %a0) {
 declare <2 x double> @llvm.x86.sse41.round.pd(<2 x double>, i32) nounwind readnone
 
 define <4 x float> @stack_fold_roundps(<4 x float> %a0) {
-  ;CHECK-LABEL: stack_fold_roundps
-  ;CHECK:       roundps $7, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_roundps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    roundps $7, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x float> @llvm.x86.sse41.round.ps(<4 x float> %a0, i32 7)
   ret <4 x float> %2
@@ -1010,8 +1971,15 @@ define <4 x float> @stack_fold_roundps(<4 x float> %a0) {
 declare <4 x float> @llvm.x86.sse41.round.ps(<4 x float>, i32) nounwind readnone
 
 define double @stack_fold_roundsd(double %a0) optsize {
-  ;CHECK-LABEL: stack_fold_roundsd
-  ;CHECK:       roundsd $9, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_roundsd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movsd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    xorps %xmm0, %xmm0
+; CHECK-NEXT:    roundsd $9, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call double @llvm.floor.f64(double %a0)
   ret double %2
@@ -1019,8 +1987,16 @@ define double @stack_fold_roundsd(double %a0) optsize {
 declare double @llvm.floor.f64(double) nounwind readnone
 
 define <2 x double> @stack_fold_roundsd_int(<2 x double> %a0, <2 x double> %a1) optsize {
-  ;CHECK-LABEL: stack_fold_roundsd_int
-  ;CHECK:       roundsd $7, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_roundsd_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    movapd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    roundsd $7, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x double> @llvm.x86.sse41.round.sd(<2 x double> %a0, <2 x double> %a1, i32 7)
   ret <2 x double> %2
@@ -1028,8 +2004,14 @@ define <2 x double> @stack_fold_roundsd_int(<2 x double> %a0, <2 x double> %a1)
 declare <2 x double> @llvm.x86.sse41.round.sd(<2 x double>, <2 x double>, i32) nounwind readnone
 
 define float @stack_fold_roundss(float %a0) minsize {
-  ;CHECK-LABEL: stack_fold_roundss
-  ;CHECK:       roundss $9, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_roundss:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movss %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    roundss $9, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call float @llvm.floor.f32(float %a0)
   ret float %2
@@ -1037,8 +2019,16 @@ define float @stack_fold_roundss(float %a0) minsize {
 declare float @llvm.floor.f32(float) nounwind readnone
 
 define <4 x float> @stack_fold_roundss_int(<4 x float> %a0, <4 x float> %a1) optsize {
-  ;CHECK-LABEL: stack_fold_roundss_int
-  ;CHECK:       roundss $7, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_roundss_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    roundss $7, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x float> @llvm.x86.sse41.round.ss(<4 x float> %a0, <4 x float> %a1, i32 7)
   ret <4 x float> %2
@@ -1048,8 +2038,14 @@ declare <4 x float> @llvm.x86.sse41.round.ss(<4 x float>, <4 x float>, i32) noun
 ; TODO stack_fold_rsqrtps
 
 define <4 x float> @stack_fold_rsqrtps_int(<4 x float> %a0) {
-  ;CHECK-LABEL: stack_fold_rsqrtps_int
-  ;CHECK:       rsqrtps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_rsqrtps_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    rsqrtps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x float> @llvm.x86.sse.rsqrt.ps(<4 x float> %a0)
   ret <4 x float> %2
@@ -1059,8 +2055,16 @@ declare <4 x float> @llvm.x86.sse.rsqrt.ps(<4 x float>) nounwind readnone
 ; TODO stack_fold_rsqrtss
 
 define <4 x float> @stack_fold_rsqrtss_int(<4 x float> %a0, <4 x float> %a1) optsize {
-  ;CHECK-LABEL: stack_fold_rsqrtss_int
-  ;CHECK:       rsqrtss {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_rsqrtss_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    rsqrtss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x float> @llvm.x86.sse.rsqrt.ss(<4 x float> %a1)
   %3 = extractelement <4 x float> %2, i32 0
@@ -1070,8 +2074,17 @@ define <4 x float> @stack_fold_rsqrtss_int(<4 x float> %a0, <4 x float> %a1) opt
 declare <4 x float> @llvm.x86.sse.rsqrt.ss(<4 x float>)
 
 define <2 x double> @stack_fold_shufpd(<2 x double> %a0, <2 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_shufpd
-  ;CHECK:       shufpd $1, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_shufpd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    shufpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = xmm0[1],mem[0]
+; CHECK-NEXT:    xorpd %xmm1, %xmm1
+; CHECK-NEXT:    addpd %xmm1, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <2 x double> %a0, <2 x double> %a1, <2 x i32> <i32 1, i32 2>
   ; fadd forces execution domain
@@ -1080,32 +2093,58 @@ define <2 x double> @stack_fold_shufpd(<2 x double> %a0, <2 x double> %a1) {
 }
 
 define <4 x float> @stack_fold_shufps(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_shufps
-  ;CHECK:       shufps $200, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_shufps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    shufps $200, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = xmm0[0,2],mem[0,3]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <4 x float> %a0, <4 x float> %a1, <4 x i32> <i32 0, i32 2, i32 4, i32 7>
   ret <4 x float> %2
 }
 
 define <2 x double> @stack_fold_sqrtpd(<2 x double> %a0) {
-  ;CHECK-LABEL: stack_fold_sqrtpd
-  ;CHECK:       sqrtpd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_sqrtpd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    sqrtpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x double> @llvm.sqrt.v2f64(<2 x double> %a0)
   ret <2 x double> %2
 }
 
 define <4 x float> @stack_fold_sqrtps(<4 x float> %a0) {
-  ;CHECK-LABEL: stack_fold_sqrtps
-  ;CHECK:       sqrtps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_sqrtps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    sqrtps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x float> @llvm.sqrt.v4f32(<4 x float> %a0)
   ret <4 x float> %2
 }
 
 define double @stack_fold_sqrtsd(double %a0) optsize {
-  ;CHECK-LABEL: stack_fold_sqrtsd
-  ;CHECK:       sqrtsd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_sqrtsd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movsd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    xorps %xmm0, %xmm0
+; CHECK-NEXT:    sqrtsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call double @llvm.sqrt.f64(double %a0)
   ret double %2
@@ -1113,8 +2152,16 @@ define double @stack_fold_sqrtsd(double %a0) optsize {
 declare double @llvm.sqrt.f64(double) nounwind readnone
 
 define <2 x double> @stack_fold_sqrtsd_int(<2 x double> %a0, <2 x double> %a1) optsize {
-  ;CHECK-LABEL: stack_fold_sqrtsd_int
-  ;CHECK:       sqrtsd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_sqrtsd_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    movapd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    sqrtsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = extractelement <2 x double> %a1, i64 0
   %3 = call double @llvm.sqrt.f64(double %2)
@@ -1125,8 +2172,14 @@ define <2 x double> @stack_fold_sqrtsd_int(<2 x double> %a0, <2 x double> %a1) o
 }
 
 define float @stack_fold_sqrtss(float %a0) minsize {
-  ;CHECK-LABEL: stack_fold_sqrtss
-  ;CHECK:       sqrtss {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_sqrtss:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movss %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    sqrtss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call float @llvm.sqrt.f32(float %a0)
   ret float %2
@@ -1134,8 +2187,16 @@ define float @stack_fold_sqrtss(float %a0) minsize {
 declare float @llvm.sqrt.f32(float) nounwind readnone
 
 define <4 x float> @stack_fold_sqrtss_int(<4 x float> %a0, <4 x float> %a1) optsize {
-  ;CHECK-LABEL: stack_fold_sqrtss_int
-  ;CHECK:       sqrtss {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_sqrtss_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    sqrtss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = extractelement <4 x float> %a1, i64 0
   %3 = call float @llvm.sqrt.f32(float %2)
@@ -1146,32 +2207,56 @@ define <4 x float> @stack_fold_sqrtss_int(<4 x float> %a0, <4 x float> %a1) opts
 }
 
 define <2 x double> @stack_fold_subpd(<2 x double> %a0, <2 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_subpd
-  ;CHECK:       subpd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_subpd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    subpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fsub <2 x double> %a0, %a1
   ret <2 x double> %2
 }
 
 define <4 x float> @stack_fold_subps(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_subps
-  ;CHECK:       subps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_subps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    subps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fsub <4 x float> %a0, %a1
   ret <4 x float> %2
 }
 
 define double @stack_fold_subsd(double %a0, double %a1) {
-  ;CHECK-LABEL: stack_fold_subsd
-  ;CHECK:       subsd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_subsd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movsd %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    subsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fsub double %a0, %a1
   ret double %2
 }
 
 define <2 x double> @stack_fold_subsd_int(<2 x double> %a0, <2 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_subsd_int
-  ;CHECK:       subsd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_subsd_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    subsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = extractelement <2 x double> %a0, i32 0
   %3 = extractelement <2 x double> %a1, i32 0
@@ -1181,16 +2266,28 @@ define <2 x double> @stack_fold_subsd_int(<2 x double> %a0, <2 x double> %a1) {
 }
 
 define float @stack_fold_subss(float %a0, float %a1) {
-  ;CHECK-LABEL: stack_fold_subss
-  ;CHECK:       subss {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_subss:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    subss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fsub float %a0, %a1
   ret float %2
 }
 
 define <4 x float> @stack_fold_subss_int(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_subss_int
-  ;CHECK:       subss {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_subss_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    subss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = extractelement <4 x float> %a0, i32 0
   %3 = extractelement <4 x float> %a1, i32 0
@@ -1200,8 +2297,17 @@ define <4 x float> @stack_fold_subss_int(<4 x float> %a0, <4 x float> %a1) {
 }
 
 define i32 @stack_fold_ucomisd(double %a0, double %a1) {
-  ;CHECK-LABEL: stack_fold_ucomisd
-  ;CHECK:       ucomisd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_ucomisd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movsd %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    ucomisd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Folded Reload
+; CHECK-NEXT:    sete %al
+; CHECK-NEXT:    leal -1(%rax,%rax), %eax
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fcmp ueq double %a0, %a1
   %3 = select i1 %2, i32 1, i32 -1
@@ -1209,8 +2315,18 @@ define i32 @stack_fold_ucomisd(double %a0, double %a1) {
 }
 
 define i32 @stack_fold_ucomisd_int(<2 x double> %a0, <2 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_ucomisd_int
-  ;CHECK:       ucomisd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_ucomisd_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    ucomisd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    setnp %al
+; CHECK-NEXT:    sete %cl
+; CHECK-NEXT:    andb %al, %cl
+; CHECK-NEXT:    movzbl %cl, %eax
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call i32 @llvm.x86.sse2.ucomieq.sd(<2 x double> %a0, <2 x double> %a1)
   ret i32 %2
@@ -1218,8 +2334,17 @@ define i32 @stack_fold_ucomisd_int(<2 x double> %a0, <2 x double> %a1) {
 declare i32 @llvm.x86.sse2.ucomieq.sd(<2 x double>, <2 x double>) nounwind readnone
 
 define i32 @stack_fold_ucomiss(float %a0, float %a1) {
-  ;CHECK-LABEL: stack_fold_ucomiss
-  ;CHECK:       ucomiss {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_ucomiss:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    ucomiss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
+; CHECK-NEXT:    sete %al
+; CHECK-NEXT:    leal -1(%rax,%rax), %eax
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fcmp ueq float %a0, %a1
   %3 = select i1 %2, i32 1, i32 -1
@@ -1227,8 +2352,18 @@ define i32 @stack_fold_ucomiss(float %a0, float %a1) {
 }
 
 define i32 @stack_fold_ucomiss_int(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_ucomiss_int
-  ;CHECK:       ucomiss {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_ucomiss_int:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    ucomiss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    setnp %al
+; CHECK-NEXT:    sete %cl
+; CHECK-NEXT:    andb %al, %cl
+; CHECK-NEXT:    movzbl %cl, %eax
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call i32 @llvm.x86.sse.ucomieq.ss(<4 x float> %a0, <4 x float> %a1)
   ret i32 %2
@@ -1236,8 +2371,17 @@ define i32 @stack_fold_ucomiss_int(<4 x float> %a0, <4 x float> %a1) {
 declare i32 @llvm.x86.sse.ucomieq.ss(<4 x float>, <4 x float>) nounwind readnone
 
 define <2 x double> @stack_fold_unpckhpd(<2 x double> %a0, <2 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_unpckhpd
-  ;CHECK:       unpckhpd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_unpckhpd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    unpckhpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = xmm0[1],mem[1]
+; CHECK-NEXT:    xorpd %xmm1, %xmm1
+; CHECK-NEXT:    addpd %xmm1, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <2 x double> %a0, <2 x double> %a1, <2 x i32> <i32 1, i32 3>
   ; fadd forces execution domain
@@ -1246,8 +2390,17 @@ define <2 x double> @stack_fold_unpckhpd(<2 x double> %a0, <2 x double> %a1) {
 }
 
 define <4 x float> @stack_fold_unpckhps(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_unpckhps
-  ;CHECK:       unpckhps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_unpckhps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    unpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = xmm0[2],mem[2],xmm0[3],mem[3]
+; CHECK-NEXT:    xorps %xmm1, %xmm1
+; CHECK-NEXT:    addps %xmm1, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <4 x float> %a0, <4 x float> %a1, <4 x i32> <i32 2, i32 6, i32 3, i32 7>
   ; fadd forces execution domain
@@ -1256,8 +2409,17 @@ define <4 x float> @stack_fold_unpckhps(<4 x float> %a0, <4 x float> %a1) {
 }
 
 define <2 x double> @stack_fold_unpcklpd(<2 x double> %a0, <2 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_unpcklpd
-  ;CHECK:       unpcklpd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_unpcklpd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    unpcklpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = xmm0[0],mem[0]
+; CHECK-NEXT:    xorpd %xmm1, %xmm1
+; CHECK-NEXT:    addpd %xmm1, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <2 x double> %a0, <2 x double> %a1, <2 x i32> <i32 0, i32 2>
   ; fadd forces execution domain
@@ -1266,8 +2428,17 @@ define <2 x double> @stack_fold_unpcklpd(<2 x double> %a0, <2 x double> %a1) {
 }
 
 define <4 x float> @stack_fold_unpcklps(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_unpcklps
-  ;CHECK:       unpcklps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_unpcklps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    unpcklps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
+; CHECK-NEXT:    xorps %xmm1, %xmm1
+; CHECK-NEXT:    addps %xmm1, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <4 x float> %a0, <4 x float> %a1, <4 x i32> <i32 0, i32 4, i32 1, i32 5>
   ; fadd forces execution domain
@@ -1276,8 +2447,16 @@ define <4 x float> @stack_fold_unpcklps(<4 x float> %a0, <4 x float> %a1) {
 }
 
 define <2 x double> @stack_fold_xorpd(<2 x double> %a0, <2 x double> %a1) {
-  ;CHECK-LABEL: stack_fold_xorpd
-  ;CHECK:       xorpd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_xorpd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    xorpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    xorpd %xmm1, %xmm1
+; CHECK-NEXT:    addpd %xmm1, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = bitcast <2 x double> %a0 to <2 x i64>
   %3 = bitcast <2 x double> %a1 to <2 x i64>
@@ -1289,8 +2468,16 @@ define <2 x double> @stack_fold_xorpd(<2 x double> %a0, <2 x double> %a1) {
 }
 
 define <4 x float> @stack_fold_xorps(<4 x float> %a0, <4 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_xorps
-  ;CHECK:       xorps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_xorps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    xorps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    xorps %xmm1, %xmm1
+; CHECK-NEXT:    addps %xmm1, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = bitcast <4 x float> %a0 to <2 x i64>
   %3 = bitcast <4 x float> %a1 to <2 x i64>
diff --git a/llvm/test/CodeGen/X86/stack-folding-int-avx1.ll b/llvm/test/CodeGen/X86/stack-folding-int-avx1.ll
index 04d62d4bc..46fd1ab5d 100644
--- a/llvm/test/CodeGen/X86/stack-folding-int-avx1.ll
+++ b/llvm/test/CodeGen/X86/stack-folding-int-avx1.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -O3 -disable-peephole -mtriple=x86_64-unknown-unknown -mattr=+avx,+aes,+pclmul < %s | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -9,8 +10,14 @@ target triple = "x86_64-unknown-unknown"
 ; relevant registers and check that the reload is correctly folded into the instruction.
 
 define <2 x i64> @stack_fold_aesdec(<2 x i64> %a0, <2 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_aesdec
-  ;CHECK:       vaesdec {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_aesdec:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vaesdec {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x i64> @llvm.x86.aesni.aesdec(<2 x i64> %a0, <2 x i64> %a1)
   ret <2 x i64> %2
@@ -18,8 +25,14 @@ define <2 x i64> @stack_fold_aesdec(<2 x i64> %a0, <2 x i64> %a1) {
 declare <2 x i64> @llvm.x86.aesni.aesdec(<2 x i64>, <2 x i64>) nounwind readnone
 
 define <2 x i64> @stack_fold_aesdeclast(<2 x i64> %a0, <2 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_aesdeclast
-  ;CHECK:       vaesdeclast {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_aesdeclast:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vaesdeclast {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x i64> @llvm.x86.aesni.aesdeclast(<2 x i64> %a0, <2 x i64> %a1)
   ret <2 x i64> %2
@@ -27,8 +40,14 @@ define <2 x i64> @stack_fold_aesdeclast(<2 x i64> %a0, <2 x i64> %a1) {
 declare <2 x i64> @llvm.x86.aesni.aesdeclast(<2 x i64>, <2 x i64>) nounwind readnone
 
 define <2 x i64> @stack_fold_aesenc(<2 x i64> %a0, <2 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_aesenc
-  ;CHECK:       vaesenc {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_aesenc:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vaesenc {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x i64> @llvm.x86.aesni.aesenc(<2 x i64> %a0, <2 x i64> %a1)
   ret <2 x i64> %2
@@ -36,8 +55,14 @@ define <2 x i64> @stack_fold_aesenc(<2 x i64> %a0, <2 x i64> %a1) {
 declare <2 x i64> @llvm.x86.aesni.aesenc(<2 x i64>, <2 x i64>) nounwind readnone
 
 define <2 x i64> @stack_fold_aesenclast(<2 x i64> %a0, <2 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_aesenclast
-  ;CHECK:       vaesenclast {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_aesenclast:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vaesenclast {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x i64> @llvm.x86.aesni.aesenclast(<2 x i64> %a0, <2 x i64> %a1)
   ret <2 x i64> %2
@@ -45,8 +70,14 @@ define <2 x i64> @stack_fold_aesenclast(<2 x i64> %a0, <2 x i64> %a1) {
 declare <2 x i64> @llvm.x86.aesni.aesenclast(<2 x i64>, <2 x i64>) nounwind readnone
 
 define <2 x i64> @stack_fold_aesimc(<2 x i64> %a0) {
-  ;CHECK-LABEL: stack_fold_aesimc
-  ;CHECK:       vaesimc {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_aesimc:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vaesimc {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x i64> @llvm.x86.aesni.aesimc(<2 x i64> %a0)
   ret <2 x i64> %2
@@ -54,8 +85,14 @@ define <2 x i64> @stack_fold_aesimc(<2 x i64> %a0) {
 declare <2 x i64> @llvm.x86.aesni.aesimc(<2 x i64>) nounwind readnone
 
 define <2 x i64> @stack_fold_aeskeygenassist(<2 x i64> %a0) {
-  ;CHECK-LABEL: stack_fold_aeskeygenassist
-  ;CHECK:       vaeskeygenassist $7, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_aeskeygenassist:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vaeskeygenassist $7, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x i64> @llvm.x86.aesni.aeskeygenassist(<2 x i64> %a0, i8 7)
   ret <2 x i64> %2
@@ -63,8 +100,47 @@ define <2 x i64> @stack_fold_aeskeygenassist(<2 x i64> %a0) {
 declare <2 x i64> @llvm.x86.aesni.aeskeygenassist(<2 x i64>, i8) nounwind readnone
 
 define <4 x i32> @stack_fold_movd_load(i32 %a0) {
-  ;CHECK-LABEL: stack_fold_movd_load
-  ;CHECK:       movd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_movd_load:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = mem[0],zero,zero,zero
+; CHECK-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = insertelement <4 x i32> zeroinitializer, i32 %a0, i32 0
   ; add forces execution domain
@@ -73,8 +149,45 @@ define <4 x i32> @stack_fold_movd_load(i32 %a0) {
 }
 
 define i32 @stack_fold_movd_store(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_movd_store
-  ;CHECK:       movd {{%xmm[0-9][0-9]*}}, {{-?[0-9]*}}(%rsp) {{.*#+}} 4-byte Folded Spill
+; CHECK-LABEL: stack_fold_movd_store:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    vmovd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   ; add forces execution domain
   %1 = add <4 x i32> %a0, %a1
   %2 = extractelement <4 x i32> %1, i32 0
@@ -83,8 +196,17 @@ define i32 @stack_fold_movd_store(<4 x i32> %a0, <4 x i32> %a1) {
 }
 
 define <2 x i64> @stack_fold_movq_load(<2 x i64> %a0) {
-  ;CHECK-LABEL: stack_fold_movq_load
-  ;CHECK:       movq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_movq_load:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = mem[0],zero
+; CHECK-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vpsubq %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <2 x i64> %a0, <2 x i64> zeroinitializer, <2 x i32> <i32 0, i32 2>
   ; add forces execution domain
@@ -93,8 +215,45 @@ define <2 x i64> @stack_fold_movq_load(<2 x i64> %a0) {
 }
 
 define i64 @stack_fold_movq_store(<2 x i64> %a0, <2 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_movq_store
-  ;CHECK:       movq {{%xmm[0-9][0-9]*}}, {{-?[0-9]*}}(%rsp) {{.*#+}} 8-byte Folded Spill
+; CHECK-LABEL: stack_fold_movq_store:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    vmovq %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   ; add forces execution domain
   %1 = add <2 x i64> %a0, %a1
   %2 = extractelement <2 x i64> %1, i32 0
@@ -103,8 +262,14 @@ define i64 @stack_fold_movq_store(<2 x i64> %a0, <2 x i64> %a1) {
 }
 
 define <8 x i16> @stack_fold_mpsadbw(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_mpsadbw
-  ;CHECK:       vmpsadbw $7, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_mpsadbw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmpsadbw $7, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x i16> @llvm.x86.sse41.mpsadbw(<16 x i8> %a0, <16 x i8> %a1, i8 7)
   ret <8 x i16> %2
@@ -112,8 +277,14 @@ define <8 x i16> @stack_fold_mpsadbw(<16 x i8> %a0, <16 x i8> %a1) {
 declare <8 x i16> @llvm.x86.sse41.mpsadbw(<16 x i8>, <16 x i8>, i8) nounwind readnone
 
 define <16 x i8> @stack_fold_pabsb(<16 x i8> %a0) {
-  ;CHECK-LABEL: stack_fold_pabsb
-  ;CHECK:       vpabsb {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pabsb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpabsb {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp sgt <16 x i8> %a0, zeroinitializer
   %3 = sub <16 x i8> zeroinitializer, %a0
@@ -122,8 +293,14 @@ define <16 x i8> @stack_fold_pabsb(<16 x i8> %a0) {
 }
 
 define <4 x i32> @stack_fold_pabsd(<4 x i32> %a0) {
-  ;CHECK-LABEL: stack_fold_pabsd
-  ;CHECK:       vpabsd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pabsd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpabsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp sgt <4 x i32> %a0, zeroinitializer
   %3 = sub <4 x i32> zeroinitializer, %a0
@@ -132,8 +309,14 @@ define <4 x i32> @stack_fold_pabsd(<4 x i32> %a0) {
 }
 
 define <8 x i16> @stack_fold_pabsw(<8 x i16> %a0) {
-  ;CHECK-LABEL: stack_fold_pabsw
-  ;CHECK:       vpabsw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pabsw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpabsw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp sgt <8 x i16> %a0, zeroinitializer
   %3 = sub <8 x i16> zeroinitializer, %a0
@@ -142,8 +325,14 @@ define <8 x i16> @stack_fold_pabsw(<8 x i16> %a0) {
 }
 
 define <8 x i16> @stack_fold_packssdw(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_packssdw
-  ;CHECK:       vpackssdw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_packssdw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpackssdw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x i16> @llvm.x86.sse2.packssdw.128(<4 x i32> %a0, <4 x i32> %a1)
   ret <8 x i16> %2
@@ -151,8 +340,14 @@ define <8 x i16> @stack_fold_packssdw(<4 x i32> %a0, <4 x i32> %a1) {
 declare <8 x i16> @llvm.x86.sse2.packssdw.128(<4 x i32>, <4 x i32>) nounwind readnone
 
 define <16 x i8> @stack_fold_packsswb(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_packsswb
-  ;CHECK:       vpacksswb {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_packsswb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpacksswb {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <16 x i8> @llvm.x86.sse2.packsswb.128(<8 x i16> %a0, <8 x i16> %a1)
   ret <16 x i8> %2
@@ -160,8 +355,14 @@ define <16 x i8> @stack_fold_packsswb(<8 x i16> %a0, <8 x i16> %a1) {
 declare <16 x i8> @llvm.x86.sse2.packsswb.128(<8 x i16>, <8 x i16>) nounwind readnone
 
 define <8 x i16> @stack_fold_packusdw(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_packusdw
-  ;CHECK:       vpackusdw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_packusdw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpackusdw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x i16> @llvm.x86.sse41.packusdw(<4 x i32> %a0, <4 x i32> %a1)
   ret <8 x i16> %2
@@ -169,8 +370,14 @@ define <8 x i16> @stack_fold_packusdw(<4 x i32> %a0, <4 x i32> %a1) {
 declare <8 x i16> @llvm.x86.sse41.packusdw(<4 x i32>, <4 x i32>) nounwind readnone
 
 define <16 x i8> @stack_fold_packuswb(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_packuswb
-  ;CHECK:       vpackuswb {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_packuswb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpackuswb {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <16 x i8> @llvm.x86.sse2.packuswb.128(<8 x i16> %a0, <8 x i16> %a1)
   ret <16 x i8> %2
@@ -178,32 +385,56 @@ define <16 x i8> @stack_fold_packuswb(<8 x i16> %a0, <8 x i16> %a1) {
 declare <16 x i8> @llvm.x86.sse2.packuswb.128(<8 x i16>, <8 x i16>) nounwind readnone
 
 define <16 x i8> @stack_fold_paddb(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_paddb
-  ;CHECK:       vpaddb {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpaddb {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = add <16 x i8> %a0, %a1
   ret <16 x i8> %2
 }
 
 define <4 x i32> @stack_fold_paddd(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_paddd
-  ;CHECK:       vpaddd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpaddd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = add <4 x i32> %a0, %a1
   ret <4 x i32> %2
 }
 
 define <2 x i64> @stack_fold_paddq(<2 x i64> %a0, <2 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_paddq
-  ;CHECK:       vpaddq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpaddq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = add <2 x i64> %a0, %a1
   ret <2 x i64> %2
 }
 
 define <16 x i8> @stack_fold_paddsb(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_paddsb
-  ;CHECK:       vpaddsb {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddsb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpaddsb {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> %a0, <16 x i8> %a1)
   ret <16 x i8> %2
@@ -211,8 +442,14 @@ define <16 x i8> @stack_fold_paddsb(<16 x i8> %a0, <16 x i8> %a1) {
 declare <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8>, <16 x i8>) nounwind readnone
 
 define <8 x i16> @stack_fold_paddsw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_paddsw
-  ;CHECK:       vpaddsw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddsw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpaddsw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> %a0, <8 x i16> %a1)
   ret <8 x i16> %2
@@ -220,8 +457,14 @@ define <8 x i16> @stack_fold_paddsw(<8 x i16> %a0, <8 x i16> %a1) {
 declare <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16>, <8 x i16>) nounwind readnone
 
 define <16 x i8> @stack_fold_paddusb(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_paddusb
-  ;CHECK:       vpaddusb {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddusb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpaddusb {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> %a0, <16 x i8> %a1)
   ret <16 x i8> %2
@@ -229,8 +472,14 @@ define <16 x i8> @stack_fold_paddusb(<16 x i8> %a0, <16 x i8> %a1) {
 declare <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8>, <16 x i8>) nounwind readnone
 
 define <8 x i16> @stack_fold_paddusw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_paddusw
-  ;CHECK:       vpaddusw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddusw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpaddusw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> %a0, <8 x i16> %a1)
   ret <8 x i16> %2
@@ -238,24 +487,45 @@ define <8 x i16> @stack_fold_paddusw(<8 x i16> %a0, <8 x i16> %a1) {
 declare <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16>, <8 x i16>) nounwind readnone
 
 define <8 x i16> @stack_fold_paddw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_paddw
-  ;CHECK:       vpaddw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpaddw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = add <8 x i16> %a0, %a1
   ret <8 x i16> %2
 }
 
 define <16 x i8> @stack_fold_palignr(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_palignr
-  ;CHECK:       vpalignr $1, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_palignr:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpalignr $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = mem[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],xmm0[0]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <16 x i8> %a1, <16 x i8> %a0, <16 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16>
   ret <16 x i8> %2
 }
 
 define <16 x i8> @stack_fold_pand(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pand
-  ;CHECK:       vpand {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pand:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpand {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = and <16 x i8> %a0, %a1
   ; add forces execution domain
@@ -264,8 +534,16 @@ define <16 x i8> @stack_fold_pand(<16 x i8> %a0, <16 x i8> %a1) {
 }
 
 define <16 x i8> @stack_fold_pandn(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pandn
-  ;CHECK:       vpandn {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pandn:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpandn {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = xor <16 x i8> %a0, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
   %3 = and <16 x i8> %2, %a1
@@ -275,8 +553,14 @@ define <16 x i8> @stack_fold_pandn(<16 x i8> %a0, <16 x i8> %a1) {
 }
 
 define <16 x i8> @stack_fold_pavgb(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pavgb
-  ;CHECK:       vpavgb {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pavgb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpavgb {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = zext <16 x i8> %a0 to <16 x i16>
   %3 = zext <16 x i8> %a1 to <16 x i16>
@@ -288,8 +572,14 @@ define <16 x i8> @stack_fold_pavgb(<16 x i8> %a0, <16 x i8> %a1) {
 }
 
 define <8 x i16> @stack_fold_pavgw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_pavgw
-  ;CHECK:       vpavgw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pavgw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpavgw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = zext <8 x i16> %a0 to <8 x i32>
   %3 = zext <8 x i16> %a1 to <8 x i32>
@@ -301,8 +591,14 @@ define <8 x i16> @stack_fold_pavgw(<8 x i16> %a0, <8 x i16> %a1) {
 }
 
 define <16 x i8> @stack_fold_pblendvb(<16 x i8> %a0, <16 x i8> %a1, <16 x i8> %c) {
-  ;CHECK-LABEL: stack_fold_pblendvb
-  ;CHECK:       vpblendvb {{%xmm[0-9][0-9]*}}, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pblendvb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpblendvb %xmm0, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <16 x i8> @llvm.x86.sse41.pblendvb(<16 x i8> %a1, <16 x i8> %c, <16 x i8> %a0)
   ret <16 x i8> %2
@@ -310,16 +606,29 @@ define <16 x i8> @stack_fold_pblendvb(<16 x i8> %a0, <16 x i8> %a1, <16 x i8> %c
 declare <16 x i8> @llvm.x86.sse41.pblendvb(<16 x i8>, <16 x i8>, <16 x i8>) nounwind readnone
 
 define <8 x i16> @stack_fold_pblendw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_pblendw
-  ;CHECK:       vpblendw $7, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pblendw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpblendw $7, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = mem[0,1,2],xmm0[3,4,5,6,7]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <8 x i16> %a0, <8 x i16> %a1, <8 x i32> <i32 8, i32 9, i32 10, i32 3, i32 4, i32 5, i32 6, i32 7>
   ret <8 x i16> %2
 }
 
 define <2 x i64> @stack_fold_pclmulqdq(<2 x i64> %a0, <2 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_pclmulqdq
-  ;CHECK:       vpclmulqdq $0, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pclmulqdq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpclmulqdq $0, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x i64> @llvm.x86.pclmulqdq(<2 x i64> %a0, <2 x i64> %a1, i8 0)
   ret <2 x i64> %2
@@ -327,8 +636,14 @@ define <2 x i64> @stack_fold_pclmulqdq(<2 x i64> %a0, <2 x i64> %a1) {
 declare <2 x i64> @llvm.x86.pclmulqdq(<2 x i64>, <2 x i64>, i8) nounwind readnone
 
 define <16 x i8> @stack_fold_pcmpeqb(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pcmpeqb
-  ;CHECK:       vpcmpeqb {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pcmpeqb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpcmpeqb {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp eq <16 x i8> %a0, %a1
   %3 = sext <16 x i1> %2 to <16 x i8>
@@ -336,8 +651,14 @@ define <16 x i8> @stack_fold_pcmpeqb(<16 x i8> %a0, <16 x i8> %a1) {
 }
 
 define <4 x i32> @stack_fold_pcmpeqd(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_pcmpeqd
-  ;CHECK:       vpcmpeqd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pcmpeqd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpcmpeqd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp eq <4 x i32> %a0, %a1
   %3 = sext <4 x i1> %2 to <4 x i32>
@@ -345,8 +666,14 @@ define <4 x i32> @stack_fold_pcmpeqd(<4 x i32> %a0, <4 x i32> %a1) {
 }
 
 define <2 x i64> @stack_fold_pcmpeqq(<2 x i64> %a0, <2 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_pcmpeqq
-  ;CHECK:       vpcmpeqq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pcmpeqq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpcmpeqq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp eq <2 x i64> %a0, %a1
   %3 = sext <2 x i1> %2 to <2 x i64>
@@ -354,8 +681,14 @@ define <2 x i64> @stack_fold_pcmpeqq(<2 x i64> %a0, <2 x i64> %a1) {
 }
 
 define <8 x i16> @stack_fold_pcmpeqw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_pcmpeqw
-  ;CHECK:       vpcmpeqw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pcmpeqw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpcmpeqw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp eq <8 x i16> %a0, %a1
   %3 = sext <8 x i1> %2 to <8 x i16>
@@ -363,8 +696,17 @@ define <8 x i16> @stack_fold_pcmpeqw(<8 x i16> %a0, <8 x i16> %a1) {
 }
 
 define i32 @stack_fold_pcmpestri(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pcmpestri
-  ;CHECK:       vpcmpestri $7, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pcmpestri:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    movl $7, %eax
+; CHECK-NEXT:    movl $7, %edx
+; CHECK-NEXT:    vpcmpestri $7, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    movl %ecx, %eax
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{rax},~{flags}"()
   %2 = call i32 @llvm.x86.sse42.pcmpestri128(<16 x i8> %a0, i32 7, <16 x i8> %a1, i32 7, i8 7)
   ret i32 %2
@@ -372,8 +714,16 @@ define i32 @stack_fold_pcmpestri(<16 x i8> %a0, <16 x i8> %a1) {
 declare i32 @llvm.x86.sse42.pcmpestri128(<16 x i8>, i32, <16 x i8>, i32, i8) nounwind readnone
 
 define <16 x i8> @stack_fold_pcmpestrm(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pcmpestrm
-  ;CHECK:       vpcmpestrm $7, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pcmpestrm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    movl $7, %eax
+; CHECK-NEXT:    movl $7, %edx
+; CHECK-NEXT:    vpcmpestrm $7, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{rax},~{flags}"()
   %2 = call <16 x i8> @llvm.x86.sse42.pcmpestrm128(<16 x i8> %a0, i32 7, <16 x i8> %a1, i32 7, i8 7)
   ret <16 x i8> %2
@@ -381,8 +731,14 @@ define <16 x i8> @stack_fold_pcmpestrm(<16 x i8> %a0, <16 x i8> %a1) {
 declare <16 x i8> @llvm.x86.sse42.pcmpestrm128(<16 x i8>, i32, <16 x i8>, i32, i8) nounwind readnone
 
 define <16 x i8> @stack_fold_pcmpgtb(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pcmpgtb
-  ;CHECK:       vpcmpgtb {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pcmpgtb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpcmpgtb {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp sgt <16 x i8> %a0, %a1
   %3 = sext <16 x i1> %2 to <16 x i8>
@@ -390,8 +746,14 @@ define <16 x i8> @stack_fold_pcmpgtb(<16 x i8> %a0, <16 x i8> %a1) {
 }
 
 define <4 x i32> @stack_fold_pcmpgtd(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_pcmpgtd
-  ;CHECK:       vpcmpgtd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pcmpgtd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpcmpgtd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp sgt <4 x i32> %a0, %a1
   %3 = sext <4 x i1> %2 to <4 x i32>
@@ -399,8 +761,14 @@ define <4 x i32> @stack_fold_pcmpgtd(<4 x i32> %a0, <4 x i32> %a1) {
 }
 
 define <2 x i64> @stack_fold_pcmpgtq(<2 x i64> %a0, <2 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_pcmpgtq
-  ;CHECK:       vpcmpgtq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pcmpgtq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpcmpgtq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp sgt <2 x i64> %a0, %a1
   %3 = sext <2 x i1> %2 to <2 x i64>
@@ -408,8 +776,14 @@ define <2 x i64> @stack_fold_pcmpgtq(<2 x i64> %a0, <2 x i64> %a1) {
 }
 
 define <8 x i16> @stack_fold_pcmpgtw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_pcmpgtw
-  ;CHECK:       vpcmpgtw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pcmpgtw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpcmpgtw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp sgt <8 x i16> %a0, %a1
   %3 = sext <8 x i1> %2 to <8 x i16>
@@ -417,8 +791,15 @@ define <8 x i16> @stack_fold_pcmpgtw(<8 x i16> %a0, <8 x i16> %a1) {
 }
 
 define i32 @stack_fold_pcmpistri(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pcmpistri
-  ;CHECK:       vpcmpistri $7, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pcmpistri:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpcmpistri $7, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    movl %ecx, %eax
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call i32 @llvm.x86.sse42.pcmpistri128(<16 x i8> %a0, <16 x i8> %a1, i8 7)
   ret i32 %2
@@ -426,8 +807,14 @@ define i32 @stack_fold_pcmpistri(<16 x i8> %a0, <16 x i8> %a1) {
 declare i32 @llvm.x86.sse42.pcmpistri128(<16 x i8>, <16 x i8>, i8) nounwind readnone
 
 define <16 x i8> @stack_fold_pcmpistrm(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pcmpistrm
-  ;CHECK:       vpcmpistrm $7, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pcmpistrm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpcmpistrm $7, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <16 x i8> @llvm.x86.sse42.pcmpistrm128(<16 x i8> %a0, <16 x i8> %a1, i8 7)
   ret <16 x i8> %2
@@ -437,9 +824,45 @@ declare <16 x i8> @llvm.x86.sse42.pcmpistrm128(<16 x i8>, <16 x i8>, i8) nounwin
 ; TODO stack_fold_pextrb
 
 define i32 @stack_fold_pextrd(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_pextrd
-  ;CHECK:       vpextrd $1, {{%xmm[0-9][0-9]*}}, {{-?[0-9]*}}(%rsp) {{.*#+}} 4-byte Folded Spill
-  ;CHECK:       movl    {{-?[0-9]*}}(%rsp), %eax {{.*#+}} 4-byte Reload
+; CHECK-LABEL: stack_fold_pextrd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    vpextrd $1, %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   ; add forces execution domain
   %1 = add <4 x i32> %a0, %a1
   %2 = extractelement <4 x i32> %1, i32 1
@@ -448,9 +871,44 @@ define i32 @stack_fold_pextrd(<4 x i32> %a0, <4 x i32> %a1) {
 }
 
 define i64 @stack_fold_pextrq(<2 x i64> %a0) {
-  ;CHECK-LABEL: stack_fold_pextrq
-  ;CHECK:       vpextrq $1, {{%xmm[0-9][0-9]*}}, {{-?[0-9]*}}(%rsp) {{.*#+}} 8-byte Folded Spill
-  ;CHECK:       movq    {{-?[0-9]*}}(%rsp), %rax {{.*#+}} 8-byte Reload
+; CHECK-LABEL: stack_fold_pextrq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    vpextrq $1, %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = extractelement <2 x i64> %a0, i32 1
   %2 = tail call <2 x i64> asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   ret i64 %1
@@ -459,8 +917,14 @@ define i64 @stack_fold_pextrq(<2 x i64> %a0) {
 ; TODO stack_fold_pextrw
 
 define <4 x i32> @stack_fold_phaddd(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_phaddd
-  ;CHECK:       vphaddd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_phaddd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vphaddd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x i32> @llvm.x86.ssse3.phadd.d.128(<4 x i32> %a0, <4 x i32> %a1)
   ret <4 x i32> %2
@@ -468,8 +932,14 @@ define <4 x i32> @stack_fold_phaddd(<4 x i32> %a0, <4 x i32> %a1) {
 declare <4 x i32> @llvm.x86.ssse3.phadd.d.128(<4 x i32>, <4 x i32>) nounwind readnone
 
 define <8 x i16> @stack_fold_phaddsw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_phaddsw
-  ;CHECK:       vphaddsw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_phaddsw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vphaddsw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x i16> @llvm.x86.ssse3.phadd.sw.128(<8 x i16> %a0, <8 x i16> %a1)
   ret <8 x i16> %2
@@ -477,8 +947,14 @@ define <8 x i16> @stack_fold_phaddsw(<8 x i16> %a0, <8 x i16> %a1) {
 declare <8 x i16> @llvm.x86.ssse3.phadd.sw.128(<8 x i16>, <8 x i16>) nounwind readnone
 
 define <8 x i16> @stack_fold_phaddw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_phaddw
-  ;CHECK:       vphaddw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_phaddw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vphaddw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x i16> @llvm.x86.ssse3.phadd.w.128(<8 x i16> %a0, <8 x i16> %a1)
   ret <8 x i16> %2
@@ -486,8 +962,14 @@ define <8 x i16> @stack_fold_phaddw(<8 x i16> %a0, <8 x i16> %a1) {
 declare <8 x i16> @llvm.x86.ssse3.phadd.w.128(<8 x i16>, <8 x i16>) nounwind readnone
 
 define <8 x i16> @stack_fold_phminposuw(<8 x i16> %a0) {
-  ;CHECK-LABEL: stack_fold_phminposuw
-  ;CHECK:       vphminposuw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_phminposuw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vphminposuw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x i16> @llvm.x86.sse41.phminposuw(<8 x i16> %a0)
   ret <8 x i16> %2
@@ -495,8 +977,14 @@ define <8 x i16> @stack_fold_phminposuw(<8 x i16> %a0) {
 declare <8 x i16> @llvm.x86.sse41.phminposuw(<8 x i16>) nounwind readnone
 
 define <4 x i32> @stack_fold_phsubd(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_phsubd
-  ;CHECK:       vphsubd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_phsubd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vphsubd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x i32> @llvm.x86.ssse3.phsub.d.128(<4 x i32> %a0, <4 x i32> %a1)
   ret <4 x i32> %2
@@ -504,8 +992,14 @@ define <4 x i32> @stack_fold_phsubd(<4 x i32> %a0, <4 x i32> %a1) {
 declare <4 x i32> @llvm.x86.ssse3.phsub.d.128(<4 x i32>, <4 x i32>) nounwind readnone
 
 define <8 x i16> @stack_fold_phsubsw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_phsubsw
-  ;CHECK:       vphsubsw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_phsubsw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vphsubsw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x i16> @llvm.x86.ssse3.phsub.sw.128(<8 x i16> %a0, <8 x i16> %a1)
   ret <8 x i16> %2
@@ -513,8 +1007,14 @@ define <8 x i16> @stack_fold_phsubsw(<8 x i16> %a0, <8 x i16> %a1) {
 declare <8 x i16> @llvm.x86.ssse3.phsub.sw.128(<8 x i16>, <8 x i16>) nounwind readnone
 
 define <8 x i16> @stack_fold_phsubw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_phsubw
-  ;CHECK:       vphsubw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_phsubw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vphsubw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x i16> @llvm.x86.ssse3.phsub.w.128(<8 x i16> %a0, <8 x i16> %a1)
   ret <8 x i16> %2
@@ -522,40 +1022,190 @@ define <8 x i16> @stack_fold_phsubw(<8 x i16> %a0, <8 x i16> %a1) {
 declare <8 x i16> @llvm.x86.ssse3.phsub.w.128(<8 x i16>, <8 x i16>) nounwind readnone
 
 define <16 x i8> @stack_fold_pinsrb(<16 x i8> %a0, i8 %a1) {
-  ;CHECK-LABEL: stack_fold_pinsrb
-  ;CHECK:       vpinsrb $1, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_pinsrb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpinsrb $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 4-byte Folded Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = insertelement <16 x i8> %a0, i8 %a1, i32 1
   ret <16 x i8> %2
 }
 
 define <4 x i32> @stack_fold_pinsrd(<4 x i32> %a0, i32 %a1) {
-  ;CHECK-LABEL: stack_fold_pinsrd
-  ;CHECK:       vpinsrd $1, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_pinsrd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpinsrd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 4-byte Folded Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = insertelement <4 x i32> %a0, i32 %a1, i32 1
   ret <4 x i32> %2
 }
 
 define <2 x i64> @stack_fold_pinsrq(<2 x i64> %a0, i64 %a1) {
-  ;CHECK-LABEL: stack_fold_pinsrq
-  ;CHECK:       vpinsrq $1, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_pinsrq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpinsrq $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 8-byte Folded Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = insertelement <2 x i64> %a0, i64 %a1, i32 1
   ret <2 x i64> %2
 }
 
 define <8 x i16> @stack_fold_pinsrw(<8 x i16> %a0, i16 %a1) {
-  ;CHECK-LABEL: stack_fold_pinsrw
-  ;CHECK:       vpinsrw $1, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_pinsrw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpinsrw $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 4-byte Folded Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = insertelement <8 x i16> %a0, i16 %a1, i32 1
   ret <8 x i16> %2
 }
 
 define <8 x i16> @stack_fold_pmaddubsw(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pmaddubsw
-  ;CHECK:       vpmaddubsw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaddubsw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmaddubsw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8> %a0, <16 x i8> %a1)
   ret <8 x i16> %2
@@ -563,8 +1213,14 @@ define <8 x i16> @stack_fold_pmaddubsw(<16 x i8> %a0, <16 x i8> %a1) {
 declare <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8>, <16 x i8>) nounwind readnone
 
 define <4 x i32> @stack_fold_pmaddwd(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_pmaddwd
-  ;CHECK:       vpmaddwd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaddwd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmaddwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> %a0, <8 x i16> %a1)
   ret <4 x i32> %2
@@ -572,8 +1228,14 @@ define <4 x i32> @stack_fold_pmaddwd(<8 x i16> %a0, <8 x i16> %a1) {
 declare <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16>, <8 x i16>) nounwind readnone
 
 define <16 x i8> @stack_fold_pmaxsb(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pmaxsb
-  ;CHECK:       vpmaxsb {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxsb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmaxsb {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp sgt <16 x i8> %a0, %a1
   %3 = select <16 x i1> %2, <16 x i8> %a0, <16 x i8> %a1
@@ -581,8 +1243,14 @@ define <16 x i8> @stack_fold_pmaxsb(<16 x i8> %a0, <16 x i8> %a1) {
 }
 
 define <4 x i32> @stack_fold_pmaxsd(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_pmaxsd
-  ;CHECK:       vpmaxsd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxsd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmaxsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp sgt <4 x i32> %a0, %a1
   %3 = select <4 x i1> %2, <4 x i32> %a0, <4 x i32> %a1
@@ -590,8 +1258,14 @@ define <4 x i32> @stack_fold_pmaxsd(<4 x i32> %a0, <4 x i32> %a1) {
 }
 
 define <8 x i16> @stack_fold_pmaxsw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_pmaxsw
-  ;CHECK:       vpmaxsw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxsw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmaxsw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp sgt <8 x i16> %a0, %a1
   %3 = select <8 x i1> %2, <8 x i16> %a0, <8 x i16> %a1
@@ -599,8 +1273,14 @@ define <8 x i16> @stack_fold_pmaxsw(<8 x i16> %a0, <8 x i16> %a1) {
 }
 
 define <16 x i8> @stack_fold_pmaxub(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pmaxub
-  ;CHECK:       vpmaxub {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxub:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmaxub {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp ugt <16 x i8> %a0, %a1
   %3 = select <16 x i1> %2, <16 x i8> %a0, <16 x i8> %a1
@@ -608,8 +1288,14 @@ define <16 x i8> @stack_fold_pmaxub(<16 x i8> %a0, <16 x i8> %a1) {
 }
 
 define <4 x i32> @stack_fold_pmaxud(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_pmaxud
-  ;CHECK:       vpmaxud {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxud:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmaxud {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp ugt <4 x i32> %a0, %a1
   %3 = select <4 x i1> %2, <4 x i32> %a0, <4 x i32> %a1
@@ -617,8 +1303,14 @@ define <4 x i32> @stack_fold_pmaxud(<4 x i32> %a0, <4 x i32> %a1) {
 }
 
 define <8 x i16> @stack_fold_pmaxuw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_pmaxuw
-  ;CHECK:       vpmaxuw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxuw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmaxuw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp ugt <8 x i16> %a0, %a1
   %3 = select <8 x i1> %2, <8 x i16> %a0, <8 x i16> %a1
@@ -626,8 +1318,14 @@ define <8 x i16> @stack_fold_pmaxuw(<8 x i16> %a0, <8 x i16> %a1) {
 }
 
 define <16 x i8> @stack_fold_pminsb(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pminsb
-  ;CHECK:       vpminsb {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminsb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpminsb {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp slt <16 x i8> %a0, %a1
   %3 = select <16 x i1> %2, <16 x i8> %a0, <16 x i8> %a1
@@ -635,8 +1333,14 @@ define <16 x i8> @stack_fold_pminsb(<16 x i8> %a0, <16 x i8> %a1) {
 }
 
 define <4 x i32> @stack_fold_pminsd(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_pminsd
-  ;CHECK:       vpminsd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminsd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpminsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp slt <4 x i32> %a0, %a1
   %3 = select <4 x i1> %2, <4 x i32> %a0, <4 x i32> %a1
@@ -644,8 +1348,14 @@ define <4 x i32> @stack_fold_pminsd(<4 x i32> %a0, <4 x i32> %a1) {
 }
 
 define <8 x i16> @stack_fold_pminsw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_pminsw
-  ;CHECK:       vpminsw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminsw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpminsw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp slt <8 x i16> %a0, %a1
   %3 = select <8 x i1> %2, <8 x i16> %a0, <8 x i16> %a1
@@ -653,8 +1363,14 @@ define <8 x i16> @stack_fold_pminsw(<8 x i16> %a0, <8 x i16> %a1) {
 }
 
 define <16 x i8> @stack_fold_pminub(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pminub
-  ;CHECK:       vpminub {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminub:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpminub {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp ult <16 x i8> %a0, %a1
   %3 = select <16 x i1> %2, <16 x i8> %a0, <16 x i8> %a1
@@ -662,8 +1378,14 @@ define <16 x i8> @stack_fold_pminub(<16 x i8> %a0, <16 x i8> %a1) {
 }
 
 define <4 x i32> @stack_fold_pminud(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_pminud
-  ;CHECK:       vpminud {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminud:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpminud {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp ult <4 x i32> %a0, %a1
   %3 = select <4 x i1> %2, <4 x i32> %a0, <4 x i32> %a1
@@ -671,8 +1393,14 @@ define <4 x i32> @stack_fold_pminud(<4 x i32> %a0, <4 x i32> %a1) {
 }
 
 define <8 x i16> @stack_fold_pminuw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_pminuw
-  ;CHECK:       vpminuw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminuw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpminuw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp ult <8 x i16> %a0, %a1
   %3 = select <8 x i1> %2, <8 x i16> %a0, <8 x i16> %a1
@@ -680,8 +1408,14 @@ define <8 x i16> @stack_fold_pminuw(<8 x i16> %a0, <8 x i16> %a1) {
 }
 
 define <2 x i64> @stack_fold_pmuldq(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_pmuldq
-  ;CHECK:       vpmuldq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmuldq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmuldq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = bitcast <4 x i32> %a0 to <2 x i64>
   %3 = bitcast <4 x i32> %a1 to <2 x i64>
@@ -694,8 +1428,14 @@ define <2 x i64> @stack_fold_pmuldq(<4 x i32> %a0, <4 x i32> %a1) {
 }
 
 define <8 x i16> @stack_fold_pmulhrsw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_pmulhrsw
-  ;CHECK:       vpmulhrsw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmulhrsw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmulhrsw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x i16> @llvm.x86.ssse3.pmul.hr.sw.128(<8 x i16> %a0, <8 x i16> %a1)
   ret <8 x i16> %2
@@ -703,8 +1443,14 @@ define <8 x i16> @stack_fold_pmulhrsw(<8 x i16> %a0, <8 x i16> %a1) {
 declare <8 x i16> @llvm.x86.ssse3.pmul.hr.sw.128(<8 x i16>, <8 x i16>) nounwind readnone
 
 define <8 x i16> @stack_fold_pmulhuw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_pmulhuw
-  ;CHECK:       vpmulhuw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmulhuw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmulhuw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x i16> @llvm.x86.sse2.pmulhu.w(<8 x i16> %a0, <8 x i16> %a1)
   ret <8 x i16> %2
@@ -712,8 +1458,14 @@ define <8 x i16> @stack_fold_pmulhuw(<8 x i16> %a0, <8 x i16> %a1) {
 declare <8 x i16> @llvm.x86.sse2.pmulhu.w(<8 x i16>, <8 x i16>) nounwind readnone
 
 define <8 x i16> @stack_fold_pmulhw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_pmulhw
-  ;CHECK:       vpmulhw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmulhw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmulhw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x i16> @llvm.x86.sse2.pmulh.w(<8 x i16> %a0, <8 x i16> %a1)
   ret <8 x i16> %2
@@ -721,24 +1473,42 @@ define <8 x i16> @stack_fold_pmulhw(<8 x i16> %a0, <8 x i16> %a1) {
 declare <8 x i16> @llvm.x86.sse2.pmulh.w(<8 x i16>, <8 x i16>) nounwind readnone
 
 define <4 x i32> @stack_fold_pmulld(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_pmulld
-  ;CHECK:       vpmulld {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmulld:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmulld {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = mul <4 x i32> %a0, %a1
   ret <4 x i32> %2
 }
 
 define <8 x i16> @stack_fold_pmullw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_pmullw
-  ;CHECK:       vpmullw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmullw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmullw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = mul <8 x i16> %a0, %a1
   ret <8 x i16> %2
 }
 
 define <2 x i64> @stack_fold_pmuludq(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_pmuludq
-  ;CHECK:       vpmuludq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmuludq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmuludq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = bitcast <4 x i32> %a0 to <2 x i64>
   %3 = bitcast <4 x i32> %a1 to <2 x i64>
@@ -749,8 +1519,16 @@ define <2 x i64> @stack_fold_pmuludq(<4 x i32> %a0, <4 x i32> %a1) {
 }
 
 define <16 x i8> @stack_fold_por(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_por
-  ;CHECK:       vpor {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_por:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpor {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = or <16 x i8> %a0, %a1
   ; add forces execution domain
@@ -759,8 +1537,14 @@ define <16 x i8> @stack_fold_por(<16 x i8> %a0, <16 x i8> %a1) {
 }
 
 define <2 x i64> @stack_fold_psadbw(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_psadbw
-  ;CHECK:       vpsadbw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psadbw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsadbw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x i64> @llvm.x86.sse2.psad.bw(<16 x i8> %a0, <16 x i8> %a1)
   ret <2 x i64> %2
@@ -768,8 +1552,14 @@ define <2 x i64> @stack_fold_psadbw(<16 x i8> %a0, <16 x i8> %a1) {
 declare <2 x i64> @llvm.x86.sse2.psad.bw(<16 x i8>, <16 x i8>) nounwind readnone
 
 define <16 x i8> @stack_fold_pshufb(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pshufb
-  ;CHECK:       vpshufb {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pshufb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpshufb {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %a0, <16 x i8> %a1)
   ret <16 x i8> %2
@@ -777,8 +1567,17 @@ define <16 x i8> @stack_fold_pshufb(<16 x i8> %a0, <16 x i8> %a1) {
 declare <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8>, <16 x i8>) nounwind readnone
 
 define <4 x i32> @stack_fold_pshufd(<4 x i32> %a0) {
-  ;CHECK-LABEL: stack_fold_pshufd
-  ;CHECK:       vpshufd $27, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pshufd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpshufd $27, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = mem[3,2,1,0]
+; CHECK-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <4 x i32> %a0, <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
   ; add forces execution domain
@@ -787,24 +1586,44 @@ define <4 x i32> @stack_fold_pshufd(<4 x i32> %a0) {
 }
 
 define <8 x i16> @stack_fold_pshufhw(<8 x i16> %a0) {
-  ;CHECK-LABEL: stack_fold_pshufhw
-  ;CHECK:       vpshufhw $11, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pshufhw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpshufhw $11, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = mem[0,1,2,3,7,6,4,4]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <8 x i16> %a0, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 7, i32 6, i32 4, i32 4>
   ret <8 x i16> %2
 }
 
 define <8 x i16> @stack_fold_pshuflw(<8 x i16> %a0) {
-  ;CHECK-LABEL: stack_fold_pshuflw
-  ;CHECK:       vpshuflw $27, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pshuflw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpshuflw $27, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = mem[3,2,1,0,4,5,6,7]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <8 x i16> %a0, <8 x i16> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 4, i32 5, i32 6, i32 7>
   ret <8 x i16> %2
 }
 
 define <16 x i8> @stack_fold_psignb(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_psignb
-  ;CHECK:       vpsignb {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psignb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsignb {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <16 x i8> @llvm.x86.ssse3.psign.b.128(<16 x i8> %a0, <16 x i8> %a1)
   ret <16 x i8> %2
@@ -812,8 +1631,14 @@ define <16 x i8> @stack_fold_psignb(<16 x i8> %a0, <16 x i8> %a1) {
 declare <16 x i8> @llvm.x86.ssse3.psign.b.128(<16 x i8>, <16 x i8>) nounwind readnone
 
 define <4 x i32> @stack_fold_psignd(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_psignd
-  ;CHECK:       vpsignd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psignd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsignd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x i32> @llvm.x86.ssse3.psign.d.128(<4 x i32> %a0, <4 x i32> %a1)
   ret <4 x i32> %2
@@ -821,8 +1646,14 @@ define <4 x i32> @stack_fold_psignd(<4 x i32> %a0, <4 x i32> %a1) {
 declare <4 x i32> @llvm.x86.ssse3.psign.d.128(<4 x i32>, <4 x i32>) nounwind readnone
 
 define <8 x i16> @stack_fold_psignw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_psignw
-  ;CHECK:       vpsignw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psignw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsignw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x i16> @llvm.x86.ssse3.psign.w.128(<8 x i16> %a0, <8 x i16> %a1)
   ret <8 x i16> %2
@@ -830,8 +1661,14 @@ define <8 x i16> @stack_fold_psignw(<8 x i16> %a0, <8 x i16> %a1) {
 declare <8 x i16> @llvm.x86.ssse3.psign.w.128(<8 x i16>, <8 x i16>) nounwind readnone
 
 define <4 x i32> @stack_fold_pslld(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_pslld
-  ;CHECK:       vpslld {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pslld:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpslld {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x i32> @llvm.x86.sse2.psll.d(<4 x i32> %a0, <4 x i32> %a1)
   ret <4 x i32> %2
@@ -839,8 +1676,14 @@ define <4 x i32> @stack_fold_pslld(<4 x i32> %a0, <4 x i32> %a1) {
 declare <4 x i32> @llvm.x86.sse2.psll.d(<4 x i32>, <4 x i32>) nounwind readnone
 
 define <2 x i64> @stack_fold_psllq(<2 x i64> %a0, <2 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_psllq
-  ;CHECK:       vpsllq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psllq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsllq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x i64> @llvm.x86.sse2.psll.q(<2 x i64> %a0, <2 x i64> %a1)
   ret <2 x i64> %2
@@ -848,8 +1691,14 @@ define <2 x i64> @stack_fold_psllq(<2 x i64> %a0, <2 x i64> %a1) {
 declare <2 x i64> @llvm.x86.sse2.psll.q(<2 x i64>, <2 x i64>) nounwind readnone
 
 define <8 x i16> @stack_fold_psllw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_psllw
-  ;CHECK:       vpsllw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psllw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsllw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x i16> @llvm.x86.sse2.psll.w(<8 x i16> %a0, <8 x i16> %a1)
   ret <8 x i16> %2
@@ -857,8 +1706,14 @@ define <8 x i16> @stack_fold_psllw(<8 x i16> %a0, <8 x i16> %a1) {
 declare <8 x i16> @llvm.x86.sse2.psll.w(<8 x i16>, <8 x i16>) nounwind readnone
 
 define <4 x i32> @stack_fold_psrad(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_psrad
-  ;CHECK:       vpsrad {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psrad:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsrad {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x i32> @llvm.x86.sse2.psra.d(<4 x i32> %a0, <4 x i32> %a1)
   ret <4 x i32> %2
@@ -866,8 +1721,14 @@ define <4 x i32> @stack_fold_psrad(<4 x i32> %a0, <4 x i32> %a1) {
 declare <4 x i32> @llvm.x86.sse2.psra.d(<4 x i32>, <4 x i32>) nounwind readnone
 
 define <8 x i16> @stack_fold_psraw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_psraw
-  ;CHECK:       vpsraw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psraw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsraw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x i16> @llvm.x86.sse2.psra.w(<8 x i16> %a0, <8 x i16> %a1)
   ret <8 x i16> %2
@@ -875,8 +1736,14 @@ define <8 x i16> @stack_fold_psraw(<8 x i16> %a0, <8 x i16> %a1) {
 declare <8 x i16> @llvm.x86.sse2.psra.w(<8 x i16>, <8 x i16>) nounwind readnone
 
 define <4 x i32> @stack_fold_psrld(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_psrld
-  ;CHECK:       vpsrld {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psrld:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsrld {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x i32> @llvm.x86.sse2.psrl.d(<4 x i32> %a0, <4 x i32> %a1)
   ret <4 x i32> %2
@@ -884,8 +1751,14 @@ define <4 x i32> @stack_fold_psrld(<4 x i32> %a0, <4 x i32> %a1) {
 declare <4 x i32> @llvm.x86.sse2.psrl.d(<4 x i32>, <4 x i32>) nounwind readnone
 
 define <2 x i64> @stack_fold_psrlq(<2 x i64> %a0, <2 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_psrlq
-  ;CHECK:       vpsrlq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psrlq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsrlq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x i64> @llvm.x86.sse2.psrl.q(<2 x i64> %a0, <2 x i64> %a1)
   ret <2 x i64> %2
@@ -893,8 +1766,14 @@ define <2 x i64> @stack_fold_psrlq(<2 x i64> %a0, <2 x i64> %a1) {
 declare <2 x i64> @llvm.x86.sse2.psrl.q(<2 x i64>, <2 x i64>) nounwind readnone
 
 define <8 x i16> @stack_fold_psrlw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_psrlw
-  ;CHECK:       vpsrlw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psrlw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsrlw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x i16> @llvm.x86.sse2.psrl.w(<8 x i16> %a0, <8 x i16> %a1)
   ret <8 x i16> %2
@@ -902,32 +1781,56 @@ define <8 x i16> @stack_fold_psrlw(<8 x i16> %a0, <8 x i16> %a1) {
 declare <8 x i16> @llvm.x86.sse2.psrl.w(<8 x i16>, <8 x i16>) nounwind readnone
 
 define <16 x i8> @stack_fold_psubb(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_psubb
-  ;CHECK:       vpsubb {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psubb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsubb {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = sub <16 x i8> %a0, %a1
   ret <16 x i8> %2
 }
 
 define <4 x i32> @stack_fold_psubd(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_psubd
-  ;CHECK:       vpsubd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psubd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsubd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = sub <4 x i32> %a0, %a1
   ret <4 x i32> %2
 }
 
 define <2 x i64> @stack_fold_psubq(<2 x i64> %a0, <2 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_psubq
-  ;CHECK:       vpsubq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psubq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsubq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = sub <2 x i64> %a0, %a1
   ret <2 x i64> %2
 }
 
 define <16 x i8> @stack_fold_psubsb(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_psubsb
-  ;CHECK:       vpsubsb {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psubsb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsubsb {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> %a0, <16 x i8> %a1)
   ret <16 x i8> %2
@@ -935,8 +1838,14 @@ define <16 x i8> @stack_fold_psubsb(<16 x i8> %a0, <16 x i8> %a1) {
 declare <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8>, <16 x i8>) nounwind readnone
 
 define <8 x i16> @stack_fold_psubsw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_psubsw
-  ;CHECK:       vpsubsw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psubsw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsubsw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> %a0, <8 x i16> %a1)
   ret <8 x i16> %2
@@ -944,8 +1853,14 @@ define <8 x i16> @stack_fold_psubsw(<8 x i16> %a0, <8 x i16> %a1) {
 declare <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16>, <8 x i16>) nounwind readnone
 
 define <16 x i8> @stack_fold_psubusb(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_psubusb
-  ;CHECK:       vpsubusb {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psubusb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsubusb {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> %a0, <16 x i8> %a1)
   ret <16 x i8> %2
@@ -953,8 +1868,14 @@ define <16 x i8> @stack_fold_psubusb(<16 x i8> %a0, <16 x i8> %a1) {
 declare <16 x i8> @llvm.usub.sat.v16i8(<16 x i8>, <16 x i8>) nounwind readnone
 
 define <8 x i16> @stack_fold_psubusw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_psubusw
-  ;CHECK:       vpsubusw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psubusw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsubusw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> %a0, <8 x i16> %a1)
   ret <8 x i16> %2
@@ -962,16 +1883,30 @@ define <8 x i16> @stack_fold_psubusw(<8 x i16> %a0, <8 x i16> %a1) {
 declare <8 x i16> @llvm.usub.sat.v8i16(<8 x i16>, <8 x i16>) nounwind readnone
 
 define <8 x i16> @stack_fold_psubw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_psubw
-  ;CHECK:       vpsubw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psubw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsubw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = sub <8 x i16> %a0, %a1
   ret <8 x i16> %2
 }
 
 define i32 @stack_fold_ptest(<2 x i64> %a0, <2 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_ptest
-  ;CHECK:       vptest {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_ptest:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    vptest {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    setb %al
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call i32 @llvm.x86.sse41.ptestc(<2 x i64> %a0, <2 x i64> %a1)
   ret i32 %2
@@ -979,8 +1914,17 @@ define i32 @stack_fold_ptest(<2 x i64> %a0, <2 x i64> %a1) {
 declare i32 @llvm.x86.sse41.ptestc(<2 x i64>, <2 x i64>) nounwind readnone
 
 define i32 @stack_fold_ptest_ymm(<4 x i64> %a0, <4 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_ptest_ymm
-  ;CHECK:       vptest {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_ptest_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    vptest {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    setb %al
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call i32 @llvm.x86.avx.ptestc.256(<4 x i64> %a0, <4 x i64> %a1)
   ret i32 %2
@@ -988,16 +1932,32 @@ define i32 @stack_fold_ptest_ymm(<4 x i64> %a0, <4 x i64> %a1) {
 declare i32 @llvm.x86.avx.ptestc.256(<4 x i64>, <4 x i64>) nounwind readnone
 
 define <16 x i8> @stack_fold_punpckhbw(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_punpckhbw
-  ;CHECK:       vpunpckhbw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_punpckhbw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpunpckhbw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = xmm0[8],mem[8],xmm0[9],mem[9],xmm0[10],mem[10],xmm0[11],mem[11],xmm0[12],mem[12],xmm0[13],mem[13],xmm0[14],mem[14],xmm0[15],mem[15]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <16 x i8> %a0, <16 x i8> %a1, <16 x i32> <i32 8, i32 24, i32 9, i32 25, i32 10, i32 26, i32 11, i32 27, i32 12, i32 28, i32 13, i32 29, i32 14, i32 30, i32 15, i32 31>
   ret <16 x i8> %2
 }
 
 define <4 x i32> @stack_fold_punpckhdq(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_punpckhdq
-  ;CHECK:       vpunpckhdq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_punpckhdq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpunpckhdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = xmm0[2],mem[2],xmm0[3],mem[3]
+; CHECK-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <4 x i32> %a0, <4 x i32> %a1, <4 x i32> <i32 2, i32 6, i32 3, i32 7>
   ; add forces execution domain
@@ -1006,8 +1966,17 @@ define <4 x i32> @stack_fold_punpckhdq(<4 x i32> %a0, <4 x i32> %a1) {
 }
 
 define <2 x i64> @stack_fold_punpckhqdq(<2 x i64> %a0, <2 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_punpckhqdq
-  ;CHECK:       vpunpckhqdq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_punpckhqdq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = xmm0[1],mem[1]
+; CHECK-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vpsubq %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <2 x i64> %a0, <2 x i64> %a1, <2 x i32> <i32 1, i32 3>
   ; add forces execution domain
@@ -1016,24 +1985,47 @@ define <2 x i64> @stack_fold_punpckhqdq(<2 x i64> %a0, <2 x i64> %a1) {
 }
 
 define <8 x i16> @stack_fold_punpckhwd(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_punpckhwd
-  ;CHECK:       vpunpckhwd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_punpckhwd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <8 x i16> %a0, <8 x i16> %a1, <8 x i32> <i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
   ret <8 x i16> %2
 }
 
 define <16 x i8> @stack_fold_punpcklbw(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_punpcklbw
-  ;CHECK:       vpunpcklbw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_punpcklbw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpunpcklbw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3],xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <16 x i8> %a0, <16 x i8> %a1, <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23>
   ret <16 x i8> %2
 }
 
 define <4 x i32> @stack_fold_punpckldq(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_punpckldq
-  ;CHECK:       vpunpckldq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_punpckldq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpunpckldq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
+; CHECK-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <4 x i32> %a0, <4 x i32> %a1, <4 x i32> <i32 0, i32 4, i32 1, i32 5>
   ; add forces execution domain
@@ -1042,8 +2034,17 @@ define <4 x i32> @stack_fold_punpckldq(<4 x i32> %a0, <4 x i32> %a1) {
 }
 
 define <2 x i64> @stack_fold_punpcklqdq(<2 x i64> %a0, <2 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_punpcklqdq
-  ;CHECK:       vpunpcklqdq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_punpcklqdq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpunpcklqdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = xmm0[0],mem[0]
+; CHECK-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vpsubq %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <2 x i64> %a0, <2 x i64> %a1, <2 x i32> <i32 0, i32 2>
   ; add forces execution domain
@@ -1052,16 +2053,31 @@ define <2 x i64> @stack_fold_punpcklqdq(<2 x i64> %a0, <2 x i64> %a1) {
 }
 
 define <8 x i16> @stack_fold_punpcklwd(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_punpcklwd
-  ;CHECK:       vpunpcklwd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_punpcklwd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <8 x i16> %a0, <8 x i16> %a1, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11>
   ret <8 x i16> %2
 }
 
 define <16 x i8> @stack_fold_pxor(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pxor
-  ;CHECK:       vpxor {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pxor:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpxor {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = xor <16 x i8> %a0, %a1
   ; add forces execution domain
diff --git a/llvm/test/CodeGen/X86/stack-folding-int-avx2.ll b/llvm/test/CodeGen/X86/stack-folding-int-avx2.ll
index 471b13422..38d87beb5 100644
--- a/llvm/test/CodeGen/X86/stack-folding-int-avx2.ll
+++ b/llvm/test/CodeGen/X86/stack-folding-int-avx2.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -O3 -disable-peephole -mtriple=x86_64-unknown-unknown -mattr=+avx2 < %s | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -9,8 +10,16 @@ target triple = "x86_64-unknown-unknown"
 ; relevant registers and check that the reload is correctly folded into the instruction.
 
 define <4 x double> @stack_fold_broadcastsd_ymm(<2 x double> %a0) {
-  ;CHECK-LABEL: stack_fold_broadcastsd_ymm
-  ;CHECK:       vbroadcastsd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_broadcastsd_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vbroadcastsd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 16-byte Folded Reload
+; CHECK-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
+; CHECK-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <2 x double> %a0, <2 x double> undef, <4 x i32> zeroinitializer
   ; fadd forces execution domain
@@ -19,8 +28,16 @@ define <4 x double> @stack_fold_broadcastsd_ymm(<2 x double> %a0) {
 }
 
 define <4 x float> @stack_fold_broadcastss(<4 x float> %a0) {
-  ;CHECK-LABEL: stack_fold_broadcastss
-  ;CHECK:       vbroadcastss {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_broadcastss:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vbroadcastss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; CHECK-NEXT:    vaddps %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <4 x float> %a0, <4 x float> undef, <4 x i32> zeroinitializer
   ; fadd forces execution domain
@@ -29,8 +46,16 @@ define <4 x float> @stack_fold_broadcastss(<4 x float> %a0) {
 }
 
 define <8 x float> @stack_fold_broadcastss_ymm(<4 x float> %a0) {
-  ;CHECK-LABEL: stack_fold_broadcastss_ymm
-  ;CHECK:       vbroadcastss {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_broadcastss_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vbroadcastss {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 16-byte Folded Reload
+; CHECK-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; CHECK-NEXT:    vaddps %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <4 x float> %a0, <4 x float> undef, <8 x i32> zeroinitializer
   ; fadd forces execution domain
@@ -39,8 +64,16 @@ define <8 x float> @stack_fold_broadcastss_ymm(<4 x float> %a0) {
 }
 
 define <4 x i32> @stack_fold_extracti128(<8 x i16> %a0, <8 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_extracti128
-  ;CHECK:       vextracti128 $1, {{%ymm[0-9][0-9]*}}, {{-?[0-9]*}}(%rsp) {{.*#+}} 16-byte Folded Spill
+; CHECK-LABEL: stack_fold_extracti128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; CHECK-NEXT:    vextracti128 $1, %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Folded Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
   ; zext forces execution domain
   %t1 = zext <8 x i16> %a0 to <8 x i32>
   %t2 = shufflevector <8 x i32> %t1, <8 x i32> %a1, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
@@ -49,8 +82,16 @@ define <4 x i32> @stack_fold_extracti128(<8 x i16> %a0, <8 x i32> %a1) {
 }
 
 define <8 x i32> @stack_fold_inserti128(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_inserti128
-  ;CHECK:       vinserti128 $1, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_inserti128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vinserti128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 16-byte Folded Reload
+; CHECK-NEXT:    vpaddd {{.*}}(%rip), %ymm0, %ymm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <4 x i32> %a0, <4 x i32> %a1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   ; add forces execution domain
@@ -59,8 +100,14 @@ define <8 x i32> @stack_fold_inserti128(<4 x i32> %a0, <4 x i32> %a1) {
 }
 
 define <16 x i16> @stack_fold_mpsadbw(<32 x i8> %a0, <32 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_mpsadbw
-  ;CHECK:       vmpsadbw $7, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_mpsadbw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmpsadbw $7, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <16 x i16> @llvm.x86.avx2.mpsadbw(<32 x i8> %a0, <32 x i8> %a1, i8 7)
   ret <16 x i16> %2
@@ -68,8 +115,14 @@ define <16 x i16> @stack_fold_mpsadbw(<32 x i8> %a0, <32 x i8> %a1) {
 declare <16 x i16> @llvm.x86.avx2.mpsadbw(<32 x i8>, <32 x i8>, i8) nounwind readnone
 
 define <32 x i8> @stack_fold_pabsb(<32 x i8> %a0) {
-  ;CHECK-LABEL: stack_fold_pabsb
-  ;CHECK:       vpabsb {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pabsb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpabsb {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp sgt <32 x i8> %a0, zeroinitializer
   %3 = sub <32 x i8> zeroinitializer, %a0
@@ -78,8 +131,14 @@ define <32 x i8> @stack_fold_pabsb(<32 x i8> %a0) {
 }
 
 define <8 x i32> @stack_fold_pabsd(<8 x i32> %a0) {
-  ;CHECK-LABEL: stack_fold_pabsd
-  ;CHECK:       vpabsd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pabsd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpabsd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp sgt <8 x i32> %a0, zeroinitializer
   %3 = sub <8 x i32> zeroinitializer, %a0
@@ -88,8 +147,14 @@ define <8 x i32> @stack_fold_pabsd(<8 x i32> %a0) {
 }
 
 define <16 x i16> @stack_fold_pabsw(<16 x i16> %a0) {
-  ;CHECK-LABEL: stack_fold_pabsw
-  ;CHECK:       vpabsw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pabsw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpabsw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp sgt <16 x i16> %a0, zeroinitializer
   %3 = sub <16 x i16> zeroinitializer, %a0
@@ -98,8 +163,14 @@ define <16 x i16> @stack_fold_pabsw(<16 x i16> %a0) {
 }
 
 define <16 x i16> @stack_fold_packssdw(<8 x i32> %a0, <8 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_packssdw
-  ;CHECK:       vpackssdw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_packssdw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpackssdw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <16 x i16> @llvm.x86.avx2.packssdw(<8 x i32> %a0, <8 x i32> %a1)
   ret <16 x i16> %2
@@ -107,8 +178,14 @@ define <16 x i16> @stack_fold_packssdw(<8 x i32> %a0, <8 x i32> %a1) {
 declare <16 x i16> @llvm.x86.avx2.packssdw(<8 x i32>, <8 x i32>) nounwind readnone
 
 define <32 x i8> @stack_fold_packsswb(<16 x i16> %a0, <16 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_packsswb
-  ;CHECK:       vpacksswb {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_packsswb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpacksswb {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <32 x i8> @llvm.x86.avx2.packsswb(<16 x i16> %a0, <16 x i16> %a1)
   ret <32 x i8> %2
@@ -116,8 +193,14 @@ define <32 x i8> @stack_fold_packsswb(<16 x i16> %a0, <16 x i16> %a1) {
 declare <32 x i8> @llvm.x86.avx2.packsswb(<16 x i16>, <16 x i16>) nounwind readnone
 
 define <16 x i16> @stack_fold_packusdw(<8 x i32> %a0, <8 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_packusdw
-  ;CHECK:       vpackusdw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_packusdw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpackusdw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <16 x i16> @llvm.x86.avx2.packusdw(<8 x i32> %a0, <8 x i32> %a1)
   ret <16 x i16> %2
@@ -125,8 +208,14 @@ define <16 x i16> @stack_fold_packusdw(<8 x i32> %a0, <8 x i32> %a1) {
 declare <16 x i16> @llvm.x86.avx2.packusdw(<8 x i32>, <8 x i32>) nounwind readnone
 
 define <32 x i8> @stack_fold_packuswb(<16 x i16> %a0, <16 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_packuswb
-  ;CHECK:       vpackuswb {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_packuswb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpackuswb {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <32 x i8> @llvm.x86.avx2.packuswb(<16 x i16> %a0, <16 x i16> %a1)
   ret <32 x i8> %2
@@ -134,32 +223,56 @@ define <32 x i8> @stack_fold_packuswb(<16 x i16> %a0, <16 x i16> %a1) {
 declare <32 x i8> @llvm.x86.avx2.packuswb(<16 x i16>, <16 x i16>) nounwind readnone
 
 define <32 x i8> @stack_fold_paddb(<32 x i8> %a0, <32 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_paddb
-  ;CHECK:       vpaddb {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpaddb {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = add <32 x i8> %a0, %a1
   ret <32 x i8> %2
 }
 
 define <8 x i32> @stack_fold_paddd(<8 x i32> %a0, <8 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_paddd
-  ;CHECK:       vpaddd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpaddd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = add <8 x i32> %a0, %a1
   ret <8 x i32> %2
 }
 
 define <4 x i64> @stack_fold_paddq(<4 x i64> %a0, <4 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_paddq
-  ;CHECK:       vpaddq {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpaddq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = add <4 x i64> %a0, %a1
   ret <4 x i64> %2
 }
 
 define <32 x i8> @stack_fold_paddsb(<32 x i8> %a0, <32 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_paddsb
-  ;CHECK:       vpaddsb {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddsb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpaddsb {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <32 x i8> @llvm.sadd.sat.v32i8(<32 x i8> %a0, <32 x i8> %a1)
   ret <32 x i8> %2
@@ -167,8 +280,14 @@ define <32 x i8> @stack_fold_paddsb(<32 x i8> %a0, <32 x i8> %a1) {
 declare <32 x i8> @llvm.sadd.sat.v32i8(<32 x i8>, <32 x i8>) nounwind readnone
 
 define <16 x i16> @stack_fold_paddsw(<16 x i16> %a0, <16 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_paddsw
-  ;CHECK:       vpaddsw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddsw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpaddsw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16> %a0, <16 x i16> %a1)
   ret <16 x i16> %2
@@ -176,8 +295,14 @@ define <16 x i16> @stack_fold_paddsw(<16 x i16> %a0, <16 x i16> %a1) {
 declare <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16>, <16 x i16>) nounwind readnone
 
 define <32 x i8> @stack_fold_paddusb(<32 x i8> %a0, <32 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_paddusb
-  ;CHECK:       vpaddusb {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddusb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpaddusb {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <32 x i8> @llvm.uadd.sat.v32i8(<32 x i8> %a0, <32 x i8> %a1)
   ret <32 x i8> %2
@@ -185,8 +310,14 @@ define <32 x i8> @stack_fold_paddusb(<32 x i8> %a0, <32 x i8> %a1) {
 declare <32 x i8> @llvm.uadd.sat.v32i8(<32 x i8>, <32 x i8>) nounwind readnone
 
 define <16 x i16> @stack_fold_paddusw(<16 x i16> %a0, <16 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_paddusw
-  ;CHECK:       vpaddusw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddusw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpaddusw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <16 x i16> @llvm.uadd.sat.v16i16(<16 x i16> %a0, <16 x i16> %a1)
   ret <16 x i16> %2
@@ -194,24 +325,44 @@ define <16 x i16> @stack_fold_paddusw(<16 x i16> %a0, <16 x i16> %a1) {
 declare <16 x i16> @llvm.uadd.sat.v16i16(<16 x i16>, <16 x i16>) nounwind readnone
 
 define <16 x i16> @stack_fold_paddw(<16 x i16> %a0, <16 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_paddw
-  ;CHECK:       vpaddw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpaddw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = add <16 x i16> %a0, %a1
   ret <16 x i16> %2
 }
 
 define <32 x i8> @stack_fold_palignr(<32 x i8> %a0, <32 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_palignr
-  ;CHECK:       vpalignr $1, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_palignr:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpalignr $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    # ymm0 = mem[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],ymm0[0],mem[17,18,19,20,21,22,23,24,25,26,27,28,29,30,31],ymm0[16]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <32 x i8> %a1, <32 x i8> %a0, <32 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 32, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 48>
   ret <32 x i8> %2
 }
 
 define <32 x i8> @stack_fold_pand(<32 x i8> %a0, <32 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pand
-  ;CHECK:       vpand {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pand:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpand {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    vpaddb {{.*}}(%rip), %ymm0, %ymm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = and <32 x i8> %a0, %a1
   ; add forces execution domain
@@ -220,8 +371,15 @@ define <32 x i8> @stack_fold_pand(<32 x i8> %a0, <32 x i8> %a1) {
 }
 
 define <32 x i8> @stack_fold_pandn(<32 x i8> %a0, <32 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pandn
-  ;CHECK:       vpandn {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pandn:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpandn {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    vpaddb {{.*}}(%rip), %ymm0, %ymm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = xor <32 x i8> %a0, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
   %3 = and <32 x i8> %2, %a1
@@ -231,8 +389,14 @@ define <32 x i8> @stack_fold_pandn(<32 x i8> %a0, <32 x i8> %a1) {
 }
 
 define <32 x i8> @stack_fold_pavgb(<32 x i8> %a0, <32 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pavgb
-  ;CHECK:       vpavgb {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pavgb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpavgb {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = zext <32 x i8> %a0 to <32 x i16>
   %3 = zext <32 x i8> %a1 to <32 x i16>
@@ -244,8 +408,14 @@ define <32 x i8> @stack_fold_pavgb(<32 x i8> %a0, <32 x i8> %a1) {
 }
 
 define <16 x i16> @stack_fold_pavgw(<16 x i16> %a0, <16 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_pavgw
-  ;CHECK:       vpavgw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pavgw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpavgw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = zext <16 x i16> %a0 to <16 x i32>
   %3 = zext <16 x i16> %a1 to <16 x i32>
@@ -257,8 +427,17 @@ define <16 x i16> @stack_fold_pavgw(<16 x i16> %a0, <16 x i16> %a1) {
 }
 
 define <4 x i32> @stack_fold_pblendd(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_pblendd
-  ;CHECK:       vpblendd $7, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pblendd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpblendd $7, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = mem[0,1,2],xmm0[3]
+; CHECK-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <4 x i32> %a0, <4 x i32> %a1, <4 x i32> <i32 4, i32 5, i32 6, i32 3>
   ; add forces execution domain
@@ -267,8 +446,16 @@ define <4 x i32> @stack_fold_pblendd(<4 x i32> %a0, <4 x i32> %a1) {
 }
 
 define <8 x i32> @stack_fold_pblendd_ymm(<8 x i32> %a0, <8 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_pblendd_ymm
-  ;CHECK:       vpblendd $7, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pblendd_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpblendd $7, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    # ymm0 = mem[0,1,2],ymm0[3,4,5,6,7]
+; CHECK-NEXT:    vpaddd {{.*}}(%rip), %ymm0, %ymm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <8 x i32> %a0, <8 x i32> %a1, <8 x i32> <i32 8, i32 9, i32 10, i32 3, i32 4, i32 5, i32 6, i32 7>
   ; add forces execution domain
@@ -277,8 +464,14 @@ define <8 x i32> @stack_fold_pblendd_ymm(<8 x i32> %a0, <8 x i32> %a1) {
 }
 
 define <32 x i8> @stack_fold_pblendvb(<32 x i8> %a0, <32 x i8> %a1, <32 x i8> %c) {
-  ;CHECK-LABEL: stack_fold_pblendvb
-  ;CHECK:       vpblendvb {{%ymm[0-9][0-9]*}}, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pblendvb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpblendvb %ymm0, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <32 x i8> @llvm.x86.avx2.pblendvb(<32 x i8> %a1, <32 x i8> %c, <32 x i8> %a0)
   ret <32 x i8> %2
@@ -286,32 +479,58 @@ define <32 x i8> @stack_fold_pblendvb(<32 x i8> %a0, <32 x i8> %a1, <32 x i8> %c
 declare <32 x i8> @llvm.x86.avx2.pblendvb(<32 x i8>, <32 x i8>, <32 x i8>) nounwind readnone
 
 define <16 x i16> @stack_fold_pblendw(<16 x i16> %a0, <16 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_pblendw
-  ;CHECK:       vpblendw $7, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pblendw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpblendw $7, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    # ymm0 = mem[0,1,2],ymm0[3,4,5,6,7],mem[8,9,10],ymm0[11,12,13,14,15]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <16 x i16> %a0, <16 x i16> %a1, <16 x i32> <i32 16, i32 17, i32 18, i32 3, i32 4, i32 5, i32 6, i32 7, i32 24, i32 25, i32 26, i32 11, i32 12, i32 13, i32 14, i32 15>
   ret <16 x i16> %2
 }
 
 define <16 x i8> @stack_fold_pbroadcastb(<16 x i8> %a0) {
-  ;CHECK-LABEL: stack_fold_pbroadcastb
-  ;CHECK:       vpbroadcastb {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pbroadcastb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpbroadcastb {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <16 x i8> %a0, <16 x i8> undef, <16 x i32> zeroinitializer
   ret <16 x i8> %2
 }
 
 define <32 x i8> @stack_fold_pbroadcastb_ymm(<16 x i8> %a0) {
-  ;CHECK-LABEL: stack_fold_pbroadcastb_ymm
-  ;CHECK:       vpbroadcastb {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pbroadcastb_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpbroadcastb {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <16 x i8> %a0, <16 x i8> undef, <32 x i32> zeroinitializer
   ret <32 x i8> %2
 }
 
 define <4 x i32> @stack_fold_pbroadcastd(<4 x i32> %a0) {
-  ;CHECK-LABEL: stack_fold_pbroadcastd
-  ;CHECK:       vpbroadcastd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pbroadcastd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpbroadcastd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    vpaddd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <4 x i32> %a0, <4 x i32> undef, <4 x i32> zeroinitializer
   ; add forces execution domain
@@ -320,8 +539,15 @@ define <4 x i32> @stack_fold_pbroadcastd(<4 x i32> %a0) {
 }
 
 define <8 x i32> @stack_fold_pbroadcastd_ymm(<4 x i32> %a0) {
-  ;CHECK-LABEL: stack_fold_pbroadcastd_ymm
-  ;CHECK:       vpbroadcastd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pbroadcastd_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpbroadcastd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 16-byte Folded Reload
+; CHECK-NEXT:    vpaddd {{.*}}(%rip), %ymm0, %ymm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <4 x i32> %a0, <4 x i32> undef, <8 x i32> zeroinitializer
   ; add forces execution domain
@@ -330,8 +556,15 @@ define <8 x i32> @stack_fold_pbroadcastd_ymm(<4 x i32> %a0) {
 }
 
 define <2 x i64> @stack_fold_pbroadcastq(<2 x i64> %a0) {
-  ;CHECK-LABEL: stack_fold_pbroadcastq
-  ;CHECK:       vpbroadcastq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pbroadcastq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpbroadcastq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    vpaddq {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <2 x i64> %a0, <2 x i64> undef, <2 x i32> zeroinitializer
   ; add forces execution domain
@@ -340,8 +573,15 @@ define <2 x i64> @stack_fold_pbroadcastq(<2 x i64> %a0) {
 }
 
 define <4 x i64> @stack_fold_pbroadcastq_ymm(<2 x i64> %a0) {
-  ;CHECK-LABEL: stack_fold_pbroadcastq_ymm
-  ;CHECK:       vpbroadcastq {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pbroadcastq_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpbroadcastq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 16-byte Folded Reload
+; CHECK-NEXT:    vpaddq {{.*}}(%rip), %ymm0, %ymm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <2 x i64> %a0, <2 x i64> undef, <4 x i32> zeroinitializer
   ; add forces execution domain
@@ -350,24 +590,42 @@ define <4 x i64> @stack_fold_pbroadcastq_ymm(<2 x i64> %a0) {
 }
 
 define <8 x i16> @stack_fold_pbroadcastw(<8 x i16> %a0) {
-  ;CHECK-LABEL: stack_fold_pbroadcastw
-  ;CHECK:       vpbroadcastw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pbroadcastw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpbroadcastw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <8 x i16> %a0, <8 x i16> undef, <8 x i32> zeroinitializer
   ret <8 x i16> %2
 }
 
 define <16 x i16> @stack_fold_pbroadcastw_ymm(<8 x i16> %a0) {
-  ;CHECK-LABEL: stack_fold_pbroadcastw_ymm
-  ;CHECK:       vpbroadcastw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pbroadcastw_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpbroadcastw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <8 x i16> %a0, <8 x i16> undef, <16 x i32> zeroinitializer
   ret <16 x i16> %2
 }
 
 define <32 x i8> @stack_fold_pcmpeqb(<32 x i8> %a0, <32 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pcmpeqb
-  ;CHECK:       vpcmpeqb {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pcmpeqb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpcmpeqb {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp eq <32 x i8> %a0, %a1
   %3 = sext <32 x i1> %2 to <32 x i8>
@@ -375,8 +633,14 @@ define <32 x i8> @stack_fold_pcmpeqb(<32 x i8> %a0, <32 x i8> %a1) {
 }
 
 define <8 x i32> @stack_fold_pcmpeqd(<8 x i32> %a0, <8 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_pcmpeqd
-  ;CHECK:       vpcmpeqd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pcmpeqd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpcmpeqd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp eq <8 x i32> %a0, %a1
   %3 = sext <8 x i1> %2 to <8 x i32>
@@ -384,8 +648,14 @@ define <8 x i32> @stack_fold_pcmpeqd(<8 x i32> %a0, <8 x i32> %a1) {
 }
 
 define <4 x i64> @stack_fold_pcmpeqq(<4 x i64> %a0, <4 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_pcmpeqq
-  ;CHECK:       vpcmpeqq {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pcmpeqq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpcmpeqq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp eq <4 x i64> %a0, %a1
   %3 = sext <4 x i1> %2 to <4 x i64>
@@ -393,8 +663,14 @@ define <4 x i64> @stack_fold_pcmpeqq(<4 x i64> %a0, <4 x i64> %a1) {
 }
 
 define <16 x i16> @stack_fold_pcmpeqw(<16 x i16> %a0, <16 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_pcmpeqw
-  ;CHECK:       vpcmpeqw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pcmpeqw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpcmpeqw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp eq <16 x i16> %a0, %a1
   %3 = sext <16 x i1> %2 to <16 x i16>
@@ -402,8 +678,14 @@ define <16 x i16> @stack_fold_pcmpeqw(<16 x i16> %a0, <16 x i16> %a1) {
 }
 
 define <32 x i8> @stack_fold_pcmpgtb(<32 x i8> %a0, <32 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pcmpgtb
-  ;CHECK:       vpcmpgtb {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pcmpgtb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpcmpgtb {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp sgt <32 x i8> %a0, %a1
   %3 = sext <32 x i1> %2 to <32 x i8>
@@ -411,8 +693,14 @@ define <32 x i8> @stack_fold_pcmpgtb(<32 x i8> %a0, <32 x i8> %a1) {
 }
 
 define <8 x i32> @stack_fold_pcmpgtd(<8 x i32> %a0, <8 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_pcmpgtd
-  ;CHECK:       vpcmpgtd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pcmpgtd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpcmpgtd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp sgt <8 x i32> %a0, %a1
   %3 = sext <8 x i1> %2 to <8 x i32>
@@ -420,8 +708,14 @@ define <8 x i32> @stack_fold_pcmpgtd(<8 x i32> %a0, <8 x i32> %a1) {
 }
 
 define <4 x i64> @stack_fold_pcmpgtq(<4 x i64> %a0, <4 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_pcmpgtq
-  ;CHECK:       vpcmpgtq {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pcmpgtq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpcmpgtq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp sgt <4 x i64> %a0, %a1
   %3 = sext <4 x i1> %2 to <4 x i64>
@@ -429,8 +723,14 @@ define <4 x i64> @stack_fold_pcmpgtq(<4 x i64> %a0, <4 x i64> %a1) {
 }
 
 define <16 x i16> @stack_fold_pcmpgtw(<16 x i16> %a0, <16 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_pcmpgtw
-  ;CHECK:       vpcmpgtw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pcmpgtw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpcmpgtw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp sgt <16 x i16> %a0, %a1
   %3 = sext <16 x i1> %2 to <16 x i16>
@@ -438,8 +738,16 @@ define <16 x i16> @stack_fold_pcmpgtw(<16 x i16> %a0, <16 x i16> %a1) {
 }
 
 define <8 x i32> @stack_fold_perm2i128(<8 x i32> %a0, <8 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_perm2i128
-  ;CHECK:   vperm2i128 $33, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_perm2i128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vperm2i128 $33, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    # ymm0 = ymm0[2,3],mem[0,1]
+; CHECK-NEXT:    vpaddd {{.*}}(%rip), %ymm0, %ymm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <8 x i32> %a0, <8 x i32> %a1, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
   ; add forces execution domain
@@ -448,8 +756,15 @@ define <8 x i32> @stack_fold_perm2i128(<8 x i32> %a0, <8 x i32> %a1) {
 }
 
 define <8 x i32> @stack_fold_permd(<8 x i32> %a0, <8 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_permd
-  ;CHECK:   vpermd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_permd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    vpaddd {{.*}}(%rip), %ymm0, %ymm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x i32> @llvm.x86.avx2.permd(<8 x i32> %a1, <8 x i32> %a0)
   ; add forces execution domain
@@ -459,8 +774,17 @@ define <8 x i32> @stack_fold_permd(<8 x i32> %a0, <8 x i32> %a1) {
 declare <8 x i32> @llvm.x86.avx2.permd(<8 x i32>, <8 x i32>) nounwind readonly
 
 define <4 x double> @stack_fold_permpd(<4 x double> %a0) {
-  ;CHECK-LABEL: stack_fold_permpd
-  ;CHECK:   vpermpd $235, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_permpd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermpd $235, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    # ymm0 = mem[3,2,2,3]
+; CHECK-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <4 x double> %a0, <4 x double> undef, <4 x i32> <i32 3, i32 2, i32 2, i32 3>
   ; fadd forces execution domain
@@ -469,8 +793,14 @@ define <4 x double> @stack_fold_permpd(<4 x double> %a0) {
 }
 
 define <8 x float> @stack_fold_permps(<8 x i32> %a0, <8 x float> %a1) {
-  ;CHECK-LABEL: stack_fold_permps
-  ;CHECK:       vpermps {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_permps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x float> @llvm.x86.avx2.permps(<8 x float> %a1, <8 x i32> %a0)
   ret <8 x float> %2
@@ -478,8 +808,16 @@ define <8 x float> @stack_fold_permps(<8 x i32> %a0, <8 x float> %a1) {
 declare <8 x float> @llvm.x86.avx2.permps(<8 x float>, <8 x i32>) nounwind readonly
 
 define <4 x i64> @stack_fold_permq(<4 x i64> %a0) {
-  ;CHECK-LABEL: stack_fold_permq
-  ;CHECK:   vpermq $235, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_permq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermq $235, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    # ymm0 = mem[3,2,2,3]
+; CHECK-NEXT:    vpaddq {{.*}}(%rip), %ymm0, %ymm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <4 x i64> %a0, <4 x i64> undef, <4 x i32> <i32 3, i32 2, i32 2, i32 3>
   ; add forces execution domain
@@ -488,8 +826,14 @@ define <4 x i64> @stack_fold_permq(<4 x i64> %a0) {
 }
 
 define <8 x i32> @stack_fold_phaddd(<8 x i32> %a0, <8 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_phaddd
-  ;CHECK:       vphaddd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_phaddd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vphaddd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x i32> @llvm.x86.avx2.phadd.d(<8 x i32> %a0, <8 x i32> %a1)
   ret <8 x i32> %2
@@ -497,8 +841,14 @@ define <8 x i32> @stack_fold_phaddd(<8 x i32> %a0, <8 x i32> %a1) {
 declare <8 x i32> @llvm.x86.avx2.phadd.d(<8 x i32>, <8 x i32>) nounwind readnone
 
 define <16 x i16> @stack_fold_phaddsw(<16 x i16> %a0, <16 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_phaddsw
-  ;CHECK:       vphaddsw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_phaddsw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vphaddsw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <16 x i16> @llvm.x86.avx2.phadd.sw(<16 x i16> %a0, <16 x i16> %a1)
   ret <16 x i16> %2
@@ -506,8 +856,14 @@ define <16 x i16> @stack_fold_phaddsw(<16 x i16> %a0, <16 x i16> %a1) {
 declare <16 x i16> @llvm.x86.avx2.phadd.sw(<16 x i16>, <16 x i16>) nounwind readnone
 
 define <16 x i16> @stack_fold_phaddw(<16 x i16> %a0, <16 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_phaddw
-  ;CHECK:       vphaddw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_phaddw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vphaddw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <16 x i16> @llvm.x86.avx2.phadd.w(<16 x i16> %a0, <16 x i16> %a1)
   ret <16 x i16> %2
@@ -515,8 +871,14 @@ define <16 x i16> @stack_fold_phaddw(<16 x i16> %a0, <16 x i16> %a1) {
 declare <16 x i16> @llvm.x86.avx2.phadd.w(<16 x i16>, <16 x i16>) nounwind readnone
 
 define <8 x i32> @stack_fold_phsubd(<8 x i32> %a0, <8 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_phsubd
-  ;CHECK:       vphsubd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_phsubd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vphsubd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x i32> @llvm.x86.avx2.phsub.d(<8 x i32> %a0, <8 x i32> %a1)
   ret <8 x i32> %2
@@ -524,8 +886,14 @@ define <8 x i32> @stack_fold_phsubd(<8 x i32> %a0, <8 x i32> %a1) {
 declare <8 x i32> @llvm.x86.avx2.phsub.d(<8 x i32>, <8 x i32>) nounwind readnone
 
 define <16 x i16> @stack_fold_phsubsw(<16 x i16> %a0, <16 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_phsubsw
-  ;CHECK:       vphsubsw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_phsubsw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vphsubsw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <16 x i16> @llvm.x86.avx2.phsub.sw(<16 x i16> %a0, <16 x i16> %a1)
   ret <16 x i16> %2
@@ -533,8 +901,14 @@ define <16 x i16> @stack_fold_phsubsw(<16 x i16> %a0, <16 x i16> %a1) {
 declare <16 x i16> @llvm.x86.avx2.phsub.sw(<16 x i16>, <16 x i16>) nounwind readnone
 
 define <16 x i16> @stack_fold_phsubw(<16 x i16> %a0, <16 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_phsubw
-  ;CHECK:       vphsubw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_phsubw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vphsubw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <16 x i16> @llvm.x86.avx2.phsub.w(<16 x i16> %a0, <16 x i16> %a1)
   ret <16 x i16> %2
@@ -542,8 +916,14 @@ define <16 x i16> @stack_fold_phsubw(<16 x i16> %a0, <16 x i16> %a1) {
 declare <16 x i16> @llvm.x86.avx2.phsub.w(<16 x i16>, <16 x i16>) nounwind readnone
 
 define <16 x i16> @stack_fold_pmaddubsw(<32 x i8> %a0, <32 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pmaddubsw
-  ;CHECK:       vpmaddubsw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaddubsw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmaddubsw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <16 x i16> @llvm.x86.avx2.pmadd.ub.sw(<32 x i8> %a0, <32 x i8> %a1)
   ret <16 x i16> %2
@@ -551,8 +931,14 @@ define <16 x i16> @stack_fold_pmaddubsw(<32 x i8> %a0, <32 x i8> %a1) {
 declare <16 x i16> @llvm.x86.avx2.pmadd.ub.sw(<32 x i8>, <32 x i8>) nounwind readnone
 
 define <8 x i32> @stack_fold_pmaddwd(<16 x i16> %a0, <16 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_pmaddwd
-  ;CHECK:       vpmaddwd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaddwd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmaddwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x i32> @llvm.x86.avx2.pmadd.wd(<16 x i16> %a0, <16 x i16> %a1)
   ret <8 x i32> %2
@@ -560,8 +946,14 @@ define <8 x i32> @stack_fold_pmaddwd(<16 x i16> %a0, <16 x i16> %a1) {
 declare <8 x i32> @llvm.x86.avx2.pmadd.wd(<16 x i16>, <16 x i16>) nounwind readnone
 
 define <32 x i8> @stack_fold_pmaxsb(<32 x i8> %a0, <32 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pmaxsb
-  ;CHECK:       vpmaxsb {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxsb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmaxsb {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp sgt <32 x i8> %a0, %a1
   %3 = select <32 x i1> %2, <32 x i8> %a0, <32 x i8> %a1
@@ -569,8 +961,14 @@ define <32 x i8> @stack_fold_pmaxsb(<32 x i8> %a0, <32 x i8> %a1) {
 }
 
 define <8 x i32> @stack_fold_pmaxsd(<8 x i32> %a0, <8 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_pmaxsd
-  ;CHECK:       vpmaxsd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxsd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmaxsd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp sgt <8 x i32> %a0, %a1
   %3 = select <8 x i1> %2, <8 x i32> %a0, <8 x i32> %a1
@@ -578,8 +976,14 @@ define <8 x i32> @stack_fold_pmaxsd(<8 x i32> %a0, <8 x i32> %a1) {
 }
 
 define <16 x i16> @stack_fold_pmaxsw(<16 x i16> %a0, <16 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_pmaxsw
-  ;CHECK:       vpmaxsw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxsw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmaxsw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp sgt <16 x i16> %a0, %a1
   %3 = select <16 x i1> %2, <16 x i16> %a0, <16 x i16> %a1
@@ -587,8 +991,14 @@ define <16 x i16> @stack_fold_pmaxsw(<16 x i16> %a0, <16 x i16> %a1) {
 }
 
 define <32 x i8> @stack_fold_pmaxub(<32 x i8> %a0, <32 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pmaxub
-  ;CHECK:       vpmaxub {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxub:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmaxub {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp ugt <32 x i8> %a0, %a1
   %3 = select <32 x i1> %2, <32 x i8> %a0, <32 x i8> %a1
@@ -596,8 +1006,14 @@ define <32 x i8> @stack_fold_pmaxub(<32 x i8> %a0, <32 x i8> %a1) {
 }
 
 define <8 x i32> @stack_fold_pmaxud(<8 x i32> %a0, <8 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_pmaxud
-  ;CHECK:       vpmaxud {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxud:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmaxud {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp ugt <8 x i32> %a0, %a1
   %3 = select <8 x i1> %2, <8 x i32> %a0, <8 x i32> %a1
@@ -605,8 +1021,14 @@ define <8 x i32> @stack_fold_pmaxud(<8 x i32> %a0, <8 x i32> %a1) {
 }
 
 define <16 x i16> @stack_fold_pmaxuw(<16 x i16> %a0, <16 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_pmaxuw
-  ;CHECK:       vpmaxuw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxuw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmaxuw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp ugt <16 x i16> %a0, %a1
   %3 = select <16 x i1> %2, <16 x i16> %a0, <16 x i16> %a1
@@ -614,8 +1036,14 @@ define <16 x i16> @stack_fold_pmaxuw(<16 x i16> %a0, <16 x i16> %a1) {
 }
 
 define <32 x i8> @stack_fold_pminsb(<32 x i8> %a0, <32 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pminsb
-  ;CHECK:       vpminsb {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminsb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpminsb {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp slt <32 x i8> %a0, %a1
   %3 = select <32 x i1> %2, <32 x i8> %a0, <32 x i8> %a1
@@ -623,8 +1051,14 @@ define <32 x i8> @stack_fold_pminsb(<32 x i8> %a0, <32 x i8> %a1) {
 }
 
 define <8 x i32> @stack_fold_pminsd(<8 x i32> %a0, <8 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_pminsd
-  ;CHECK:       vpminsd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminsd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpminsd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp slt <8 x i32> %a0, %a1
   %3 = select <8 x i1> %2, <8 x i32> %a0, <8 x i32> %a1
@@ -632,8 +1066,14 @@ define <8 x i32> @stack_fold_pminsd(<8 x i32> %a0, <8 x i32> %a1) {
 }
 
 define <16 x i16> @stack_fold_pminsw(<16 x i16> %a0, <16 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_pminsw
-  ;CHECK:       vpminsw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminsw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpminsw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp slt <16 x i16> %a0, %a1
   %3 = select <16 x i1> %2, <16 x i16> %a0, <16 x i16> %a1
@@ -641,8 +1081,14 @@ define <16 x i16> @stack_fold_pminsw(<16 x i16> %a0, <16 x i16> %a1) {
 }
 
 define <32 x i8> @stack_fold_pminub(<32 x i8> %a0, <32 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pminub
-  ;CHECK:       vpminub {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminub:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpminub {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp ult <32 x i8> %a0, %a1
   %3 = select <32 x i1> %2, <32 x i8> %a0, <32 x i8> %a1
@@ -650,8 +1096,14 @@ define <32 x i8> @stack_fold_pminub(<32 x i8> %a0, <32 x i8> %a1) {
 }
 
 define <8 x i32> @stack_fold_pminud(<8 x i32> %a0, <8 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_pminud
-  ;CHECK:       vpminud {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminud:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpminud {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp ult <8 x i32> %a0, %a1
   %3 = select <8 x i1> %2, <8 x i32> %a0, <8 x i32> %a1
@@ -659,8 +1111,14 @@ define <8 x i32> @stack_fold_pminud(<8 x i32> %a0, <8 x i32> %a1) {
 }
 
 define <16 x i16> @stack_fold_pminuw(<16 x i16> %a0, <16 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_pminuw
-  ;CHECK:       vpminuw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminuw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpminuw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp ult <16 x i16> %a0, %a1
   %3 = select <16 x i1> %2, <16 x i16> %a0, <16 x i16> %a1
@@ -668,8 +1126,14 @@ define <16 x i16> @stack_fold_pminuw(<16 x i16> %a0, <16 x i16> %a1) {
 }
 
 define <8 x i32> @stack_fold_pmovsxbd(<16 x i8> %a0) {
-  ;CHECK-LABEL: stack_fold_pmovsxbd
-  ;CHECK:       vpmovsxbd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmovsxbd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmovsxbd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <16 x i8> %a0, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   %3 = sext <8 x i8> %2 to <8 x i32>
@@ -677,8 +1141,14 @@ define <8 x i32> @stack_fold_pmovsxbd(<16 x i8> %a0) {
 }
 
 define <4 x i64> @stack_fold_pmovsxbq(<16 x i8> %a0) {
-  ;CHECK-LABEL: stack_fold_pmovsxbq
-  ;CHECK:       pmovsxbq {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmovsxbq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmovsxbq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <16 x i8> %a0, <16 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %3 = sext <4 x i8> %2 to <4 x i64>
@@ -686,32 +1156,56 @@ define <4 x i64> @stack_fold_pmovsxbq(<16 x i8> %a0) {
 }
 
 define <16 x i16> @stack_fold_pmovsxbw(<16 x i8> %a0) {
-  ;CHECK-LABEL: stack_fold_pmovsxbw
-  ;CHECK:       vpmovsxbw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmovsxbw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmovsxbw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = sext <16 x i8> %a0 to <16 x i16>
   ret <16 x i16> %2
 }
 
 define <4 x i64> @stack_fold_pmovsxdq(<4 x i32> %a0) {
-  ;CHECK-LABEL: stack_fold_pmovsxdq
-  ;CHECK:       vpmovsxdq {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmovsxdq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmovsxdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = sext <4 x i32> %a0 to <4 x i64>
   ret <4 x i64> %2
 }
 
 define <8 x i32> @stack_fold_pmovsxwd(<8 x i16> %a0) {
-  ;CHECK-LABEL: stack_fold_pmovsxwd
-  ;CHECK:       vpmovsxwd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmovsxwd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmovsxwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = sext <8 x i16> %a0 to <8 x i32>
   ret <8 x i32> %2
 }
 
 define <4 x i64> @stack_fold_pmovsxwq(<8 x i16> %a0) {
-  ;CHECK-LABEL: stack_fold_pmovsxwq
-  ;CHECK:       vpmovsxwq {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmovsxwq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmovsxwq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <8 x i16> %a0, <8 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %3 = sext <4 x i16> %2 to <4 x i64>
@@ -719,8 +1213,15 @@ define <4 x i64> @stack_fold_pmovsxwq(<8 x i16> %a0) {
 }
 
 define <8 x i32> @stack_fold_pmovzxbd(<16 x i8> %a0) {
-  ;CHECK-LABEL: stack_fold_pmovzxbd
-  ;CHECK:       vpmovzxbd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmovzxbd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmovzxbd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # ymm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <16 x i8> %a0, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   %3 = zext <8 x i8> %2 to <8 x i32>
@@ -728,8 +1229,15 @@ define <8 x i32> @stack_fold_pmovzxbd(<16 x i8> %a0) {
 }
 
 define <4 x i64> @stack_fold_pmovzxbq(<16 x i8> %a0) {
-  ;CHECK-LABEL: stack_fold_pmovzxbq
-  ;CHECK:       vpmovzxbq {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmovzxbq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmovzxbq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # ymm0 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero,mem[2],zero,zero,zero,zero,zero,zero,zero,mem[3],zero,zero,zero,zero,zero,zero,zero
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <16 x i8> %a0, <16 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %3 = zext <4 x i8> %2 to <4 x i64>
@@ -737,32 +1245,60 @@ define <4 x i64> @stack_fold_pmovzxbq(<16 x i8> %a0) {
 }
 
 define <16 x i16> @stack_fold_pmovzxbw(<16 x i8> %a0) {
-  ;CHECK-LABEL: stack_fold_pmovzxbw
-  ;CHECK:       vpmovzxbw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmovzxbw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmovzxbw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # ymm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = zext <16 x i8> %a0 to <16 x i16>
   ret <16 x i16> %2
 }
 
 define <4 x i64> @stack_fold_pmovzxdq(<4 x i32> %a0) {
-  ;CHECK-LABEL: stack_fold_pmovzxdq
-  ;CHECK:       vpmovzxdq {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmovzxdq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmovzxdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # ymm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = zext <4 x i32> %a0 to <4 x i64>
   ret <4 x i64> %2
 }
 
 define <8 x i32> @stack_fold_pmovzxwd(<8 x i16> %a0) {
-  ;CHECK-LABEL: stack_fold_pmovzxwd
-  ;CHECK:       vpmovzxwd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmovzxwd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmovzxwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # ymm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = zext <8 x i16> %a0 to <8 x i32>
   ret <8 x i32> %2
 }
 
 define <4 x i64> @stack_fold_pmovzxwq(<8 x i16> %a0) {
-  ;CHECK-LABEL: stack_fold_pmovzxwq
-  ;CHECK:       vpmovzxwq {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmovzxwq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmovzxwq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # ymm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <8 x i16> %a0, <8 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %3 = zext <4 x i16> %2 to <4 x i64>
@@ -770,8 +1306,14 @@ define <4 x i64> @stack_fold_pmovzxwq(<8 x i16> %a0) {
 }
 
 define <4 x i64> @stack_fold_pmuldq(<8 x i32> %a0, <8 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_pmuldq
-  ;CHECK:       vpmuldq {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmuldq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmuldq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = bitcast <8 x i32> %a0 to <4 x i64>
   %3 = bitcast <8 x i32> %a1 to <4 x i64>
@@ -784,8 +1326,14 @@ define <4 x i64> @stack_fold_pmuldq(<8 x i32> %a0, <8 x i32> %a1) {
 }
 
 define <16 x i16> @stack_fold_pmulhrsw(<16 x i16> %a0, <16 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_pmulhrsw
-  ;CHECK:       vpmulhrsw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmulhrsw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmulhrsw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <16 x i16> @llvm.x86.avx2.pmul.hr.sw(<16 x i16> %a0, <16 x i16> %a1)
   ret <16 x i16> %2
@@ -793,8 +1341,14 @@ define <16 x i16> @stack_fold_pmulhrsw(<16 x i16> %a0, <16 x i16> %a1) {
 declare <16 x i16> @llvm.x86.avx2.pmul.hr.sw(<16 x i16>, <16 x i16>) nounwind readnone
 
 define <16 x i16> @stack_fold_pmulhuw(<16 x i16> %a0, <16 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_pmulhuw
-  ;CHECK:       vpmulhuw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmulhuw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmulhuw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <16 x i16> @llvm.x86.avx2.pmulhu.w(<16 x i16> %a0, <16 x i16> %a1)
   ret <16 x i16> %2
@@ -802,8 +1356,14 @@ define <16 x i16> @stack_fold_pmulhuw(<16 x i16> %a0, <16 x i16> %a1) {
 declare <16 x i16> @llvm.x86.avx2.pmulhu.w(<16 x i16>, <16 x i16>) nounwind readnone
 
 define <16 x i16> @stack_fold_pmulhw(<16 x i16> %a0, <16 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_pmulhw
-  ;CHECK:       vpmulhw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmulhw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmulhw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <16 x i16> @llvm.x86.avx2.pmulh.w(<16 x i16> %a0, <16 x i16> %a1)
   ret <16 x i16> %2
@@ -811,24 +1371,42 @@ define <16 x i16> @stack_fold_pmulhw(<16 x i16> %a0, <16 x i16> %a1) {
 declare <16 x i16> @llvm.x86.avx2.pmulh.w(<16 x i16>, <16 x i16>) nounwind readnone
 
 define <8 x i32> @stack_fold_pmulld(<8 x i32> %a0, <8 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_pmulld
-  ;CHECK:       vpmulld {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmulld:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmulld {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = mul <8 x i32> %a0, %a1
   ret <8 x i32> %2
 }
 
 define <16 x i16> @stack_fold_pmullw(<16 x i16> %a0, <16 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_pmullw
-  ;CHECK:       vpmullw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmullw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmullw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = mul <16 x i16> %a0, %a1
   ret <16 x i16> %2
 }
 
 define <4 x i64> @stack_fold_pmuludq(<8 x i32> %a0, <8 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_pmuludq
-  ;CHECK:       vpmuludq {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmuludq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmuludq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = bitcast <8 x i32> %a0 to <4 x i64>
   %3 = bitcast <8 x i32> %a1 to <4 x i64>
@@ -839,8 +1417,15 @@ define <4 x i64> @stack_fold_pmuludq(<8 x i32> %a0, <8 x i32> %a1) {
 }
 
 define <32 x i8> @stack_fold_por(<32 x i8> %a0, <32 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_por
-  ;CHECK:       vpor {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_por:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpor {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    vpaddb {{.*}}(%rip), %ymm0, %ymm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = or <32 x i8> %a0, %a1
   ; add forces execution domain
@@ -849,8 +1434,14 @@ define <32 x i8> @stack_fold_por(<32 x i8> %a0, <32 x i8> %a1) {
 }
 
 define <4 x i64> @stack_fold_psadbw(<32 x i8> %a0, <32 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_psadbw
-  ;CHECK:       vpsadbw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_psadbw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsadbw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x i64> @llvm.x86.avx2.psad.bw(<32 x i8> %a0, <32 x i8> %a1)
   ret <4 x i64> %2
@@ -858,8 +1449,14 @@ define <4 x i64> @stack_fold_psadbw(<32 x i8> %a0, <32 x i8> %a1) {
 declare <4 x i64> @llvm.x86.avx2.psad.bw(<32 x i8>, <32 x i8>) nounwind readnone
 
 define <32 x i8> @stack_fold_pshufb(<32 x i8> %a0, <32 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pshufb
-  ;CHECK:       vpshufb {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pshufb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpshufb {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> %a0, <32 x i8> %a1)
   ret <32 x i8> %2
@@ -867,8 +1464,16 @@ define <32 x i8> @stack_fold_pshufb(<32 x i8> %a0, <32 x i8> %a1) {
 declare <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8>, <32 x i8>) nounwind readnone
 
 define <8 x i32> @stack_fold_pshufd(<8 x i32> %a0) {
-  ;CHECK-LABEL: stack_fold_pshufd
-  ;CHECK:       vpshufd $27, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pshufd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpshufd $27, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    # ymm0 = mem[3,2,1,0,7,6,5,4]
+; CHECK-NEXT:    vpaddd {{.*}}(%rip), %ymm0, %ymm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <8 x i32> %a0, <8 x i32> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
   ; add forces execution domain
@@ -877,24 +1482,44 @@ define <8 x i32> @stack_fold_pshufd(<8 x i32> %a0) {
 }
 
 define <16 x i16> @stack_fold_vpshufhw(<16 x i16> %a0) {
-  ;CHECK-LABEL: stack_fold_vpshufhw
-  ;CHECK:       vpshufhw $27, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpshufhw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpshufhw $27, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    # ymm0 = mem[0,1,2,3,7,6,5,4,8,9,10,11,15,14,13,12]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <16 x i16> %a0, <16 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 7, i32 6, i32 5, i32 4, i32 8, i32 9, i32 10, i32 11, i32 15, i32 14, i32 13, i32 12>
   ret <16 x i16> %2
 }
 
 define <16 x i16> @stack_fold_vpshuflw(<16 x i16> %a0) {
-  ;CHECK-LABEL: stack_fold_vpshuflw
-  ;CHECK:       vpshuflw $27, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpshuflw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpshuflw $27, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    # ymm0 = mem[3,2,1,0,4,5,6,7,11,10,9,8,12,13,14,15]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <16 x i16> %a0, <16 x i16> undef, <16 x i32> <i32 3, i32 2, i32 1, i32 0, i32 4, i32 5, i32 6, i32 7, i32 11, i32 10, i32 9, i32 8, i32 12, i32 13, i32 14, i32 15>
   ret <16 x i16> %2
 }
 
 define <32 x i8> @stack_fold_psignb(<32 x i8> %a0, <32 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_psignb
-  ;CHECK:       vpsignb {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_psignb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsignb {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <32 x i8> @llvm.x86.avx2.psign.b(<32 x i8> %a0, <32 x i8> %a1)
   ret <32 x i8> %2
@@ -902,8 +1527,14 @@ define <32 x i8> @stack_fold_psignb(<32 x i8> %a0, <32 x i8> %a1) {
 declare <32 x i8> @llvm.x86.avx2.psign.b(<32 x i8>, <32 x i8>) nounwind readnone
 
 define <8 x i32> @stack_fold_psignd(<8 x i32> %a0, <8 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_psignd
-  ;CHECK:       vpsignd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_psignd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsignd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x i32> @llvm.x86.avx2.psign.d(<8 x i32> %a0, <8 x i32> %a1)
   ret <8 x i32> %2
@@ -911,8 +1542,14 @@ define <8 x i32> @stack_fold_psignd(<8 x i32> %a0, <8 x i32> %a1) {
 declare <8 x i32> @llvm.x86.avx2.psign.d(<8 x i32>, <8 x i32>) nounwind readnone
 
 define <16 x i16> @stack_fold_psignw(<16 x i16> %a0, <16 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_psignw
-  ;CHECK:       vpsignw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_psignw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsignw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <16 x i16> @llvm.x86.avx2.psign.w(<16 x i16> %a0, <16 x i16> %a1)
   ret <16 x i16> %2
@@ -920,8 +1557,14 @@ define <16 x i16> @stack_fold_psignw(<16 x i16> %a0, <16 x i16> %a1) {
 declare <16 x i16> @llvm.x86.avx2.psign.w(<16 x i16>, <16 x i16>) nounwind readnone
 
 define <8 x i32> @stack_fold_pslld(<8 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_pslld
-  ;CHECK:       vpslld {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pslld:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpslld {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x i32> @llvm.x86.avx2.psll.d(<8 x i32> %a0, <4 x i32> %a1)
   ret <8 x i32> %2
@@ -929,8 +1572,14 @@ define <8 x i32> @stack_fold_pslld(<8 x i32> %a0, <4 x i32> %a1) {
 declare <8 x i32> @llvm.x86.avx2.psll.d(<8 x i32>, <4 x i32>) nounwind readnone
 
 define <4 x i64> @stack_fold_psllq(<4 x i64> %a0, <2 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_psllq
-  ;CHECK:       vpsllq {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psllq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsllq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x i64> @llvm.x86.avx2.psll.q(<4 x i64> %a0, <2 x i64> %a1)
   ret <4 x i64> %2
@@ -938,8 +1587,14 @@ define <4 x i64> @stack_fold_psllq(<4 x i64> %a0, <2 x i64> %a1) {
 declare <4 x i64> @llvm.x86.avx2.psll.q(<4 x i64>, <2 x i64>) nounwind readnone
 
 define <4 x i32> @stack_fold_psllvd(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_psllvd
-  ;CHECK:       vpsllvd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psllvd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsllvd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x i32> @llvm.x86.avx2.psllv.d(<4 x i32> %a0, <4 x i32> %a1)
   ret <4 x i32> %2
@@ -947,8 +1602,14 @@ define <4 x i32> @stack_fold_psllvd(<4 x i32> %a0, <4 x i32> %a1) {
 declare <4 x i32> @llvm.x86.avx2.psllv.d(<4 x i32>, <4 x i32>) nounwind readnone
 
 define <8 x i32> @stack_fold_psllvd_ymm(<8 x i32> %a0, <8 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_psllvd_ymm
-  ;CHECK:       vpsllvd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_psllvd_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsllvd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x i32> @llvm.x86.avx2.psllv.d.256(<8 x i32> %a0, <8 x i32> %a1)
   ret <8 x i32> %2
@@ -956,8 +1617,14 @@ define <8 x i32> @stack_fold_psllvd_ymm(<8 x i32> %a0, <8 x i32> %a1) {
 declare <8 x i32> @llvm.x86.avx2.psllv.d.256(<8 x i32>, <8 x i32>) nounwind readnone
 
 define <2 x i64> @stack_fold_psllvq(<2 x i64> %a0, <2 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_psllvq
-  ;CHECK:       vpsllvq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psllvq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsllvq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x i64> @llvm.x86.avx2.psllv.q(<2 x i64> %a0, <2 x i64> %a1)
   ret <2 x i64> %2
@@ -965,8 +1632,14 @@ define <2 x i64> @stack_fold_psllvq(<2 x i64> %a0, <2 x i64> %a1) {
 declare <2 x i64> @llvm.x86.avx2.psllv.q(<2 x i64>, <2 x i64>) nounwind readnone
 
 define <4 x i64> @stack_fold_psllvq_ymm(<4 x i64> %a0, <4 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_psllvq_ymm
-  ;CHECK:       vpsllvq {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_psllvq_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsllvq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x i64> @llvm.x86.avx2.psllv.q.256(<4 x i64> %a0, <4 x i64> %a1)
   ret <4 x i64> %2
@@ -974,8 +1647,14 @@ define <4 x i64> @stack_fold_psllvq_ymm(<4 x i64> %a0, <4 x i64> %a1) {
 declare <4 x i64> @llvm.x86.avx2.psllv.q.256(<4 x i64>, <4 x i64>) nounwind readnone
 
 define <16 x i16> @stack_fold_psllw(<16 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_psllw
-  ;CHECK:       vpsllw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psllw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsllw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <16 x i16> @llvm.x86.avx2.psll.w(<16 x i16> %a0, <8 x i16> %a1)
   ret <16 x i16> %2
@@ -983,8 +1662,14 @@ define <16 x i16> @stack_fold_psllw(<16 x i16> %a0, <8 x i16> %a1) {
 declare <16 x i16> @llvm.x86.avx2.psll.w(<16 x i16>, <8 x i16>) nounwind readnone
 
 define <8 x i32> @stack_fold_psrad(<8 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_psrad
-  ;CHECK:       vpsrad {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psrad:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsrad {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x i32> @llvm.x86.avx2.psra.d(<8 x i32> %a0, <4 x i32> %a1)
   ret <8 x i32> %2
@@ -992,8 +1677,14 @@ define <8 x i32> @stack_fold_psrad(<8 x i32> %a0, <4 x i32> %a1) {
 declare <8 x i32> @llvm.x86.avx2.psra.d(<8 x i32>, <4 x i32>) nounwind readnone
 
 define <4 x i32> @stack_fold_psravd(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_psravd
-  ;CHECK:       vpsravd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psravd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsravd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x i32> @llvm.x86.avx2.psrav.d(<4 x i32> %a0, <4 x i32> %a1)
   ret <4 x i32> %2
@@ -1001,8 +1692,14 @@ define <4 x i32> @stack_fold_psravd(<4 x i32> %a0, <4 x i32> %a1) {
 declare <4 x i32> @llvm.x86.avx2.psrav.d(<4 x i32>, <4 x i32>) nounwind readnone
 
 define <8 x i32> @stack_fold_psravd_ymm(<8 x i32> %a0, <8 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_psravd_ymm
-  ;CHECK:       vpsravd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_psravd_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsravd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x i32> @llvm.x86.avx2.psrav.d.256(<8 x i32> %a0, <8 x i32> %a1)
   ret <8 x i32> %2
@@ -1010,8 +1707,14 @@ define <8 x i32> @stack_fold_psravd_ymm(<8 x i32> %a0, <8 x i32> %a1) {
 declare <8 x i32> @llvm.x86.avx2.psrav.d.256(<8 x i32>, <8 x i32>) nounwind readnone
 
 define <16 x i16> @stack_fold_psraw(<16 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_psraw
-  ;CHECK:       vpsraw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psraw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsraw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <16 x i16> @llvm.x86.avx2.psra.w(<16 x i16> %a0, <8 x i16> %a1)
   ret <16 x i16> %2
@@ -1019,8 +1722,14 @@ define <16 x i16> @stack_fold_psraw(<16 x i16> %a0, <8 x i16> %a1) {
 declare <16 x i16> @llvm.x86.avx2.psra.w(<16 x i16>, <8 x i16>) nounwind readnone
 
 define <8 x i32> @stack_fold_psrld(<8 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_psrld
-  ;CHECK:       vpsrld {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psrld:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsrld {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x i32> @llvm.x86.avx2.psrl.d(<8 x i32> %a0, <4 x i32> %a1)
   ret <8 x i32> %2
@@ -1028,8 +1737,14 @@ define <8 x i32> @stack_fold_psrld(<8 x i32> %a0, <4 x i32> %a1) {
 declare <8 x i32> @llvm.x86.avx2.psrl.d(<8 x i32>, <4 x i32>) nounwind readnone
 
 define <4 x i64> @stack_fold_psrlq(<4 x i64> %a0, <2 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_psrlq
-  ;CHECK:       vpsrlq {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psrlq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsrlq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x i64> @llvm.x86.avx2.psrl.q(<4 x i64> %a0, <2 x i64> %a1)
   ret <4 x i64> %2
@@ -1037,8 +1752,14 @@ define <4 x i64> @stack_fold_psrlq(<4 x i64> %a0, <2 x i64> %a1) {
 declare <4 x i64> @llvm.x86.avx2.psrl.q(<4 x i64>, <2 x i64>) nounwind readnone
 
 define <4 x i32> @stack_fold_psrlvd(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_psrlvd
-  ;CHECK:       vpsrlvd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psrlvd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsrlvd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x i32> @llvm.x86.avx2.psrlv.d(<4 x i32> %a0, <4 x i32> %a1)
   ret <4 x i32> %2
@@ -1046,8 +1767,14 @@ define <4 x i32> @stack_fold_psrlvd(<4 x i32> %a0, <4 x i32> %a1) {
 declare <4 x i32> @llvm.x86.avx2.psrlv.d(<4 x i32>, <4 x i32>) nounwind readnone
 
 define <8 x i32> @stack_fold_psrlvd_ymm(<8 x i32> %a0, <8 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_psrlvd_ymm
-  ;CHECK:       vpsrlvd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_psrlvd_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsrlvd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x i32> @llvm.x86.avx2.psrlv.d.256(<8 x i32> %a0, <8 x i32> %a1)
   ret <8 x i32> %2
@@ -1055,8 +1782,14 @@ define <8 x i32> @stack_fold_psrlvd_ymm(<8 x i32> %a0, <8 x i32> %a1) {
 declare <8 x i32> @llvm.x86.avx2.psrlv.d.256(<8 x i32>, <8 x i32>) nounwind readnone
 
 define <2 x i64> @stack_fold_psrlvq(<2 x i64> %a0, <2 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_psrlvq
-  ;CHECK:       vpsrlvq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psrlvq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsrlvq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x i64> @llvm.x86.avx2.psrlv.q(<2 x i64> %a0, <2 x i64> %a1)
   ret <2 x i64> %2
@@ -1064,8 +1797,14 @@ define <2 x i64> @stack_fold_psrlvq(<2 x i64> %a0, <2 x i64> %a1) {
 declare <2 x i64> @llvm.x86.avx2.psrlv.q(<2 x i64>, <2 x i64>) nounwind readnone
 
 define <4 x i64> @stack_fold_psrlvq_ymm(<4 x i64> %a0, <4 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_psrlvq_ymm
-  ;CHECK:       vpsrlvq {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_psrlvq_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsrlvq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x i64> @llvm.x86.avx2.psrlv.q.256(<4 x i64> %a0, <4 x i64> %a1)
   ret <4 x i64> %2
@@ -1073,8 +1812,14 @@ define <4 x i64> @stack_fold_psrlvq_ymm(<4 x i64> %a0, <4 x i64> %a1) {
 declare <4 x i64> @llvm.x86.avx2.psrlv.q.256(<4 x i64>, <4 x i64>) nounwind readnone
 
 define <16 x i16> @stack_fold_psrlw(<16 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_psrlw
-  ;CHECK:       vpsrlw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psrlw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsrlw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <16 x i16> @llvm.x86.avx2.psrl.w(<16 x i16> %a0, <8 x i16> %a1)
   ret <16 x i16> %2
@@ -1082,32 +1827,56 @@ define <16 x i16> @stack_fold_psrlw(<16 x i16> %a0, <8 x i16> %a1) {
 declare <16 x i16> @llvm.x86.avx2.psrl.w(<16 x i16>, <8 x i16>) nounwind readnone
 
 define <32 x i8> @stack_fold_psubb(<32 x i8> %a0, <32 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_psubb
-  ;CHECK:       vpsubb {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_psubb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsubb {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = sub <32 x i8> %a0, %a1
   ret <32 x i8> %2
 }
 
 define <8 x i32> @stack_fold_psubd(<8 x i32> %a0, <8 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_psubd
-  ;CHECK:       vpsubd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_psubd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsubd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = sub <8 x i32> %a0, %a1
   ret <8 x i32> %2
 }
 
 define <4 x i64> @stack_fold_psubq(<4 x i64> %a0, <4 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_psubq
-  ;CHECK:       vpsubq {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_psubq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsubq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = sub <4 x i64> %a0, %a1
   ret <4 x i64> %2
 }
 
 define <32 x i8> @stack_fold_psubsb(<32 x i8> %a0, <32 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_psubsb
-  ;CHECK:       vpsubsb {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_psubsb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsubsb {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <32 x i8> @llvm.ssub.sat.v32i8(<32 x i8> %a0, <32 x i8> %a1)
   ret <32 x i8> %2
@@ -1115,8 +1884,14 @@ define <32 x i8> @stack_fold_psubsb(<32 x i8> %a0, <32 x i8> %a1) {
 declare <32 x i8> @llvm.ssub.sat.v32i8(<32 x i8>, <32 x i8>) nounwind readnone
 
 define <16 x i16> @stack_fold_psubsw(<16 x i16> %a0, <16 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_psubsw
-  ;CHECK:       vpsubsw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_psubsw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsubsw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16> %a0, <16 x i16> %a1)
   ret <16 x i16> %2
@@ -1124,8 +1899,14 @@ define <16 x i16> @stack_fold_psubsw(<16 x i16> %a0, <16 x i16> %a1) {
 declare <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16>, <16 x i16>) nounwind readnone
 
 define <32 x i8> @stack_fold_psubusb(<32 x i8> %a0, <32 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_psubusb
-  ;CHECK:       vpsubusb {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_psubusb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsubusb {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <32 x i8> @llvm.usub.sat.v32i8(<32 x i8> %a0, <32 x i8> %a1)
   ret <32 x i8> %2
@@ -1133,8 +1914,14 @@ define <32 x i8> @stack_fold_psubusb(<32 x i8> %a0, <32 x i8> %a1) {
 declare <32 x i8> @llvm.usub.sat.v32i8(<32 x i8>, <32 x i8>) nounwind readnone
 
 define <16 x i16> @stack_fold_psubusw(<16 x i16> %a0, <16 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_psubusw
-  ;CHECK:       vpsubusw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_psubusw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsubusw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <16 x i16> @llvm.usub.sat.v16i16(<16 x i16> %a0, <16 x i16> %a1)
   ret <16 x i16> %2
@@ -1142,24 +1929,45 @@ define <16 x i16> @stack_fold_psubusw(<16 x i16> %a0, <16 x i16> %a1) {
 declare <16 x i16> @llvm.usub.sat.v16i16(<16 x i16>, <16 x i16>) nounwind readnone
 
 define <16 x i16> @stack_fold_psubw(<16 x i16> %a0, <16 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_psubw
-  ;CHECK:       vpsubw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_psubw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsubw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = sub <16 x i16> %a0, %a1
   ret <16 x i16> %2
 }
 
 define <32 x i8> @stack_fold_punpckhbw(<32 x i8> %a0, <32 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_punpckhbw
-  ;CHECK:       vpunpckhbw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_punpckhbw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpunpckhbw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    # ymm0 = ymm0[8],mem[8],ymm0[9],mem[9],ymm0[10],mem[10],ymm0[11],mem[11],ymm0[12],mem[12],ymm0[13],mem[13],ymm0[14],mem[14],ymm0[15],mem[15],ymm0[24],mem[24],ymm0[25],mem[25],ymm0[26],mem[26],ymm0[27],mem[27],ymm0[28],mem[28],ymm0[29],mem[29],ymm0[30],mem[30],ymm0[31],mem[31]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <32 x i8> %a0, <32 x i8> %a1, <32 x i32> <i32 8, i32 40, i32 9, i32 41, i32 10, i32 42, i32 11, i32 43, i32 12, i32 44, i32 13, i32 45, i32 14, i32 46, i32 15, i32 47, i32 24, i32 56, i32 25, i32 57, i32 26, i32 58, i32 27, i32 59, i32 28, i32 60, i32 29, i32 61, i32 30, i32 62, i32 31, i32 63>
   ret <32 x i8> %2
 }
 
 define <8 x i32> @stack_fold_punpckhdq(<8 x i32> %a0, <8 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_punpckhdq
-  ;CHECK:       vpunpckhdq {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_punpckhdq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpunpckhdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    # ymm0 = ymm0[2],mem[2],ymm0[3],mem[3],ymm0[6],mem[6],ymm0[7],mem[7]
+; CHECK-NEXT:    vpaddd {{.*}}(%rip), %ymm0, %ymm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <8 x i32> %a0, <8 x i32> %a1, <8 x i32> <i32 2, i32 10, i32 3, i32 11, i32 6, i32 14, i32 7, i32 15>
   ; add forces execution domain
@@ -1168,8 +1976,16 @@ define <8 x i32> @stack_fold_punpckhdq(<8 x i32> %a0, <8 x i32> %a1) {
 }
 
 define <4 x i64> @stack_fold_punpckhqdq(<4 x i64> %a0, <4 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_punpckhqdq
-  ;CHECK:       vpunpckhqdq {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_punpckhqdq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    # ymm0 = ymm0[1],mem[1],ymm0[3],mem[3]
+; CHECK-NEXT:    vpaddq {{.*}}(%rip), %ymm0, %ymm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <4 x i64> %a0, <4 x i64> %a1, <4 x i32> <i32 1, i32 5, i32 3, i32 7>
   ; add forces execution domain
@@ -1178,24 +1994,46 @@ define <4 x i64> @stack_fold_punpckhqdq(<4 x i64> %a0, <4 x i64> %a1) {
 }
 
 define <16 x i16> @stack_fold_punpckhwd(<16 x i16> %a0, <16 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_punpckhwd
-  ;CHECK:       vpunpckhwd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_punpckhwd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    # ymm0 = ymm0[4],mem[4],ymm0[5],mem[5],ymm0[6],mem[6],ymm0[7],mem[7],ymm0[12],mem[12],ymm0[13],mem[13],ymm0[14],mem[14],ymm0[15],mem[15]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <16 x i16> %a0, <16 x i16> %a1, <16 x i32> <i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23, i32 12, i32 28, i32 13, i32 29, i32 14, i32 30, i32 15, i32 31>
   ret <16 x i16> %2
 }
 
 define <32 x i8> @stack_fold_punpcklbw(<32 x i8> %a0, <32 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_punpcklbw
-  ;CHECK:       vpunpcklbw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_punpcklbw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpunpcklbw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    # ymm0 = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[2],mem[2],ymm0[3],mem[3],ymm0[4],mem[4],ymm0[5],mem[5],ymm0[6],mem[6],ymm0[7],mem[7],ymm0[16],mem[16],ymm0[17],mem[17],ymm0[18],mem[18],ymm0[19],mem[19],ymm0[20],mem[20],ymm0[21],mem[21],ymm0[22],mem[22],ymm0[23],mem[23]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <32 x i8> %a0, <32 x i8> %a1, <32 x i32> <i32 0, i32 32, i32 1, i32 33, i32 2, i32 34, i32 3, i32 35, i32 4, i32 36, i32 5, i32 37, i32 6, i32 38, i32 7, i32 39, i32 16, i32 48, i32 17, i32 49, i32 18, i32 50, i32 19, i32 51, i32 20, i32 52, i32 21, i32 53, i32 22, i32 54, i32 23, i32 55>
   ret <32 x i8> %2
 }
 
 define <8 x i32> @stack_fold_punpckldq(<8 x i32> %a0, <8 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_punpckldq
-  ;CHECK:       vpunpckldq {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_punpckldq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpunpckldq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    # ymm0 = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[4],mem[4],ymm0[5],mem[5]
+; CHECK-NEXT:    vpaddd {{.*}}(%rip), %ymm0, %ymm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <8 x i32> %a0, <8 x i32> %a1, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 4, i32 12, i32 5, i32 13>
   ; add forces execution domain
@@ -1204,8 +2042,16 @@ define <8 x i32> @stack_fold_punpckldq(<8 x i32> %a0, <8 x i32> %a1) {
 }
 
 define <4 x i64> @stack_fold_punpcklqdq(<4 x i64> %a0, <4 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_punpcklqdq
-  ;CHECK:       vpunpcklqdq {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_punpcklqdq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpunpcklqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    # ymm0 = ymm0[0],mem[0],ymm0[2],mem[2]
+; CHECK-NEXT:    vpaddq {{.*}}(%rip), %ymm0, %ymm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <4 x i64> %a0, <4 x i64> %a1, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
   ; add forces execution domain
@@ -1214,16 +2060,30 @@ define <4 x i64> @stack_fold_punpcklqdq(<4 x i64> %a0, <4 x i64> %a1) {
 }
 
 define <16 x i16> @stack_fold_punpcklwd(<16 x i16> %a0, <16 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_punpcklwd
-  ;CHECK:       vpunpcklwd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_punpcklwd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    # ymm0 = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[2],mem[2],ymm0[3],mem[3],ymm0[8],mem[8],ymm0[9],mem[9],ymm0[10],mem[10],ymm0[11],mem[11]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <16 x i16> %a0, <16 x i16> %a1, <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 8, i32 24, i32 9, i32 25, i32 10, i32 26, i32 11, i32 27>
   ret <16 x i16> %2
 }
 
 define <32 x i8> @stack_fold_pxor(<32 x i8> %a0, <32 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pxor
-  ;CHECK:       vpxor {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pxor:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpxor {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    vpaddb {{.*}}(%rip), %ymm0, %ymm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = xor <32 x i8> %a0, %a1
   ; add forces execution domain
diff --git a/llvm/test/CodeGen/X86/stack-folding-int-avx512.ll b/llvm/test/CodeGen/X86/stack-folding-int-avx512.ll
index f8a2973f9..e176f3a32 100644
--- a/llvm/test/CodeGen/X86/stack-folding-int-avx512.ll
+++ b/llvm/test/CodeGen/X86/stack-folding-int-avx512.ll
@@ -1,4 +1,5 @@
-; RUN: llc -O3 -disable-peephole -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw,+avx512dq,+avx512vbmi,+avx512cd,+avx512vpopcntdq < %s | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -O3 -disable-peephole -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw,+avx512dq,+avx512vbmi,+avx512cd,+avx512vpopcntdq,+avx512vnni < %s | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-unknown"
@@ -9,16 +10,45 @@ target triple = "x86_64-unknown-unknown"
 ; relevant registers and check that the reload is correctly folded into the instruction.
 
 define <16 x i32> @stack_fold_valignd(<16 x i32> %a, <16 x i32> %b) {
-  ;CHECK-LABEL: stack_fold_valignd
-  ;CHECK:   valignd $1, {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_valignd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subq $56, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; CHECK-NEXT:    valignd $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    # zmm0 = mem[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],zmm0[0]
+; CHECK-NEXT:    addq $56, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <16 x i32> %a, <16 x i32> %b, <16 x i32><i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16>
   ret <16 x i32> %2
 }
 
 define <16 x i32> @stack_fold_valignd_mask(<16 x i32> %a, <16 x i32> %b, <16 x i32>* %passthru, i16 %mask) {
-  ;CHECK-LABEL: stack_fold_valignd_mask
-  ;CHECK:   valignd $1, {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_valignd_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subq $56, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm1
+; CHECK-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; CHECK-NEXT:    valignd $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm1 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    # zmm1 {%k1} = mem[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],zmm0[0]
+; CHECK-NEXT:    vmovdqa64 %zmm1, %zmm0
+; CHECK-NEXT:    addq $56, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <16 x i32> %a, <16 x i32> %b, <16 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16>
   %3 = bitcast i16 %mask to <16 x i1>
@@ -28,8 +58,22 @@ define <16 x i32> @stack_fold_valignd_mask(<16 x i32> %a, <16 x i32> %b, <16 x i
 }
 
 define <16 x i32> @stack_fold_valignd_maskz(<16 x i32> %a, <16 x i32> %b, i16 %mask) {
-  ;CHECK-LABEL: stack_fold_valignd_maskz
-  ;CHECK:   valignd $1, {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_valignd_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subq $56, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; CHECK-NEXT:    valignd $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    # zmm0 {%k1} {z} = mem[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],zmm0[0]
+; CHECK-NEXT:    addq $56, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <16 x i32> %a, <16 x i32> %b, <16 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16>
   %3 = bitcast i16 %mask to <16 x i1>
@@ -38,16 +82,45 @@ define <16 x i32> @stack_fold_valignd_maskz(<16 x i32> %a, <16 x i32> %b, i16 %m
 }
 
 define <8 x i64> @stack_fold_valignq(<8 x i64> %a, <8 x i64> %b) {
-  ;CHECK-LABEL: stack_fold_valignq
-  ;CHECK:   valignq $1, {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_valignq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subq $56, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; CHECK-NEXT:    valignq $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    # zmm0 = mem[1,2,3,4,5,6,7],zmm0[0]
+; CHECK-NEXT:    addq $56, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
   ret <8 x i64> %2
 }
 
 define <8 x i64> @stack_fold_valignq_mask(<8 x i64> %a, <8 x i64> %b, <8 x i64>* %passthru, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_valignq_mask
-  ;CHECK:   valignq $1, {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_valignq_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subq $56, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm1
+; CHECK-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; CHECK-NEXT:    valignq $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm1 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    # zmm1 {%k1} = mem[1,2,3,4,5,6,7],zmm0[0]
+; CHECK-NEXT:    vmovdqa64 %zmm1, %zmm0
+; CHECK-NEXT:    addq $56, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
   %3 = bitcast i8 %mask to <8 x i1>
@@ -57,8 +130,22 @@ define <8 x i64> @stack_fold_valignq_mask(<8 x i64> %a, <8 x i64> %b, <8 x i64>*
 }
 
 define <8 x i64> @stack_fold_valignq_maskz(<8 x i64> %a, <8 x i64> %b, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_valignq_maskz
-  ;CHECK:   valignq $1, {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_valignq_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subq $56, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; CHECK-NEXT:    valignq $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    # zmm0 {%k1} {z} = mem[1,2,3,4,5,6,7],zmm0[0]
+; CHECK-NEXT:    addq $56, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
   %3 = bitcast i8 %mask to <8 x i1>
@@ -67,8 +154,14 @@ define <8 x i64> @stack_fold_valignq_maskz(<8 x i64> %a, <8 x i64> %b, i8 %mask)
 }
 
 define <64 x i8> @stack_fold_pavgb(<64 x i8> %a0, <64 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pavgb
-  ;CHECK:       vpavgb {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pavgb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpavgb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <64 x i8> @llvm.x86.avx512.pavg.b.512(<64 x i8> %a0, <64 x i8> %a1)
   ret <64 x i8> %2
@@ -76,16 +169,31 @@ define <64 x i8> @stack_fold_pavgb(<64 x i8> %a0, <64 x i8> %a1) {
 declare <64 x i8> @llvm.x86.avx512.pavg.b.512(<64 x i8>, <64 x i8>)
 
 define <64 x i8> @stack_fold_pavgb_commuted(<64 x i8> %a0, <64 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pavgb_commuted:
-  ;CHECK:       vpavgb {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pavgb_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpavgb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <64 x i8> @llvm.x86.avx512.pavg.b.512(<64 x i8> %a1, <64 x i8> %a0)
   ret <64 x i8> %2
 }
 
 define <64 x i8> @stack_fold_pavgb_mask(<64 x i8> %a0, <64 x i8> %a1, <64 x i8>* %a2, i64 %mask) {
-  ;CHECK-LABEL: stack_fold_pavgb_mask
-  ;CHECK:       vpavgb {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pavgb_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm1
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovq %rsi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm0
+; CHECK-NEXT:    vpavgb {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <64 x i8> @llvm.x86.avx512.pavg.b.512(<64 x i8> %a0, <64 x i8> %a1)
   %3 = bitcast i64 %mask to <64 x i1>
@@ -96,8 +204,17 @@ define <64 x i8> @stack_fold_pavgb_mask(<64 x i8> %a0, <64 x i8> %a1, <64 x i8>*
 }
 
 define <64 x i8> @stack_fold_pavgb_mask_commuted(<64 x i8> %a0, <64 x i8> %a1, <64 x i8>* %a2, i64 %mask) {
-  ;CHECK-LABEL: stack_fold_pavgb_mask_commuted:
-  ;CHECK:       vpavgb {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pavgb_mask_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm1
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovq %rsi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm0
+; CHECK-NEXT:    vpavgb {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <64 x i8> @llvm.x86.avx512.pavg.b.512(<64 x i8> %a1, <64 x i8> %a0)
   %3 = bitcast i64 %mask to <64 x i1>
@@ -108,8 +225,15 @@ define <64 x i8> @stack_fold_pavgb_mask_commuted(<64 x i8> %a0, <64 x i8> %a1, <
 }
 
 define <64 x i8> @stack_fold_pavgb_maskz(<64 x i8> %a0, <64 x i8> %a1, i64 %mask) {
-  ;CHECK-LABEL: stack_fold_pavgb_maskz
-  ;CHECK:       vpavgb {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pavgb_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovq %rdi, %k1
+; CHECK-NEXT:    vpavgb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <64 x i8> @llvm.x86.avx512.pavg.b.512(<64 x i8> %a0, <64 x i8> %a1)
   %3 = bitcast i64 %mask to <64 x i1>
@@ -118,8 +242,15 @@ define <64 x i8> @stack_fold_pavgb_maskz(<64 x i8> %a0, <64 x i8> %a1, i64 %mask
 }
 
 define <64 x i8> @stack_fold_pavgb_maskz_commuted(<64 x i8> %a0, <64 x i8> %a1, i64 %mask) {
-  ;CHECK-LABEL: stack_fold_pavgb_maskz_commuted:
-  ;CHECK:       vpavgb {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pavgb_maskz_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovq %rdi, %k1
+; CHECK-NEXT:    vpavgb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <64 x i8> @llvm.x86.avx512.pavg.b.512(<64 x i8> %a1, <64 x i8> %a0)
   %3 = bitcast i64 %mask to <64 x i1>
@@ -128,8 +259,14 @@ define <64 x i8> @stack_fold_pavgb_maskz_commuted(<64 x i8> %a0, <64 x i8> %a1,
 }
 
 define <32 x i16> @stack_fold_pavgw(<32 x i16> %a0, <32 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_pavgw
-  ;CHECK:       vpavgw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pavgw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpavgw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <32 x i16> @llvm.x86.avx512.pavg.w.512(<32 x i16> %a0, <32 x i16> %a1)
   ret <32 x i16> %2
@@ -137,16 +274,31 @@ define <32 x i16> @stack_fold_pavgw(<32 x i16> %a0, <32 x i16> %a1) {
 declare <32 x i16> @llvm.x86.avx512.pavg.w.512(<32 x i16>, <32 x i16>)
 
 define <32 x i16> @stack_fold_pavgw_commuted(<32 x i16> %a0, <32 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_pavgw_commuted:
-  ;CHECK:       vpavgw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pavgw_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpavgw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <32 x i16> @llvm.x86.avx512.pavg.w.512(<32 x i16> %a1, <32 x i16> %a0)
   ret <32 x i16> %2
 }
 
 define <32 x i16> @stack_fold_pavgw_mask(<32 x i16> %a0, <32 x i16> %a1, <32 x i16>* %a2, i32 %mask) {
-  ;CHECK-LABEL: stack_fold_pavgw_mask
-  ;CHECK:       vpavgw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pavgw_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm1
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm0
+; CHECK-NEXT:    vpavgw {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <32 x i16> @llvm.x86.avx512.pavg.w.512(<32 x i16> %a0, <32 x i16> %a1)
   %3 = bitcast i32 %mask to <32 x i1>
@@ -157,8 +309,17 @@ define <32 x i16> @stack_fold_pavgw_mask(<32 x i16> %a0, <32 x i16> %a1, <32 x i
 }
 
 define <32 x i16> @stack_fold_pavgw_mask_commuted(<32 x i16> %a0, <32 x i16> %a1, <32 x i16>* %a2, i32 %mask) {
-  ;CHECK-LABEL: stack_fold_pavgw_mask_commuted:
-  ;CHECK:       vpavgw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pavgw_mask_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm1
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm0
+; CHECK-NEXT:    vpavgw {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <32 x i16> @llvm.x86.avx512.pavg.w.512(<32 x i16> %a1, <32 x i16> %a0)
   %3 = bitcast i32 %mask to <32 x i1>
@@ -169,8 +330,15 @@ define <32 x i16> @stack_fold_pavgw_mask_commuted(<32 x i16> %a0, <32 x i16> %a1
 }
 
 define <32 x i16> @stack_fold_pavgw_maskz(<32 x i16> %a0, <32 x i16> %a1, i32 %mask) {
-  ;CHECK-LABEL: stack_fold_pavgw_maskz
-  ;CHECK:       vpavgw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pavgw_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpavgw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <32 x i16> @llvm.x86.avx512.pavg.w.512(<32 x i16> %a0, <32 x i16> %a1)
   %3 = bitcast i32 %mask to <32 x i1>
@@ -179,8 +347,15 @@ define <32 x i16> @stack_fold_pavgw_maskz(<32 x i16> %a0, <32 x i16> %a1, i32 %m
 }
 
 define <32 x i16> @stack_fold_pavgw_maskz_commuted(<32 x i16> %a0, <32 x i16> %a1, i32 %mask) {
-  ;CHECK-LABEL: stack_fold_pavgw_maskz_commuted:
-  ;CHECK:       vpavgw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pavgw_maskz_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpavgw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <32 x i16> @llvm.x86.avx512.pavg.w.512(<32 x i16> %a1, <32 x i16> %a0)
   %3 = bitcast i32 %mask to <32 x i1>
@@ -189,8 +364,16 @@ define <32 x i16> @stack_fold_pavgw_maskz_commuted(<32 x i16> %a0, <32 x i16> %a
 }
 
 define <4 x i32> @stack_fold_extracti32x4(<16 x i16> %a0, <16 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_extracti32x4
-  ;CHECK:       vextracti32x4 $3, {{%zmm[0-9][0-9]*}}, {{-?[0-9]*}}(%rsp) {{.*#+}} 16-byte Folded Spill
+; CHECK-LABEL: stack_fold_extracti32x4:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
+; CHECK-NEXT:    vextracti32x4 $3, %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Folded Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
   ; zext forces execution domain
   %1 = zext <16 x i16> %a0 to <16 x i32>
   %2 = shufflevector <16 x i32> %1, <16 x i32> %a1, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
@@ -199,8 +382,16 @@ define <4 x i32> @stack_fold_extracti32x4(<16 x i16> %a0, <16 x i32> %a1) {
 }
 
 define <2 x i64> @stack_fold_extracti64x2(<8 x i32> %a0, <8 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_extracti64x2
-  ;CHECK:       vextracti32x4 $3, {{%zmm[0-9][0-9]*}}, {{-?[0-9]*}}(%rsp) {{.*#+}} 16-byte Folded Spill
+; CHECK-LABEL: stack_fold_extracti64x2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmovzxdq {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero
+; CHECK-NEXT:    vextracti32x4 $3, %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Folded Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
   ; zext forces execution domain
   %1 = zext <8 x i32> %a0 to <8 x i64>
   %2 = shufflevector <8 x i64> %1, <8 x i64> %a1, <2 x i32> <i32 6, i32 7>
@@ -209,8 +400,15 @@ define <2 x i64> @stack_fold_extracti64x2(<8 x i32> %a0, <8 x i64> %a1) {
 }
 
 define <8 x i32> @stack_fold_extracti32x8(<16 x i16> %a0, <16 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_extracti32x8
-  ;CHECK:       vextracti64x4 $1, {{%zmm[0-9][0-9]*}}, {{-?[0-9]*}}(%rsp) {{.*#+}} 32-byte Folded Spill
+; CHECK-LABEL: stack_fold_extracti32x8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
+; CHECK-NEXT:    vextracti64x4 $1, %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Folded Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; CHECK-NEXT:    retq
   ; zext forces execution domain
   %1 = zext <16 x i16> %a0 to <16 x i32>
   %2 = shufflevector <16 x i32> %1, <16 x i32> %a1, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -219,8 +417,15 @@ define <8 x i32> @stack_fold_extracti32x8(<16 x i16> %a0, <16 x i32> %a1) {
 }
 
 define <4 x i64> @stack_fold_extracti64x4(<8 x i32> %a0, <8 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_extracti64x4
-  ;CHECK:       vextracti64x4 $1, {{%zmm[0-9][0-9]*}}, {{-?[0-9]*}}(%rsp) {{.*#+}} 32-byte Folded Spill
+; CHECK-LABEL: stack_fold_extracti64x4:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmovzxdq {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero
+; CHECK-NEXT:    vextracti64x4 $1, %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Folded Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; CHECK-NEXT:    retq
   ; zext forces execution domain
   %1 = zext <8 x i32> %a0 to <8 x i64>
   %2 = shufflevector <8 x i64> %1, <8 x i64> %a1, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
@@ -229,8 +434,17 @@ define <4 x i64> @stack_fold_extracti64x4(<8 x i32> %a0, <8 x i64> %a1) {
 }
 
 define <16 x i32> @stack_fold_inserti32x8(<8 x i32> %a0, <8 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_inserti32x8
-  ;CHECK:       vinserti64x4 $1, {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_inserti32x8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 32-byte Folded Reload
+; CHECK-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1
+; CHECK-NEXT:    vpsubd %zmm1, %zmm0, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <8 x i32> %a0, <8 x i32> %a1, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
   ; add forces execution domain
@@ -239,8 +453,17 @@ define <16 x i32> @stack_fold_inserti32x8(<8 x i32> %a0, <8 x i32> %a1) {
 }
 
 define <8 x i64> @stack_fold_inserti64x4(<4 x i64> %a0, <4 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_inserti64x4
-  ;CHECK:       vinserti64x4 $1, {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_inserti64x4:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 32-byte Folded Reload
+; CHECK-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1
+; CHECK-NEXT:    vpsubq %zmm1, %zmm0, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <4 x i64> %a0, <4 x i64> %a1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   ; add forces execution domain
@@ -249,8 +472,14 @@ define <8 x i64> @stack_fold_inserti64x4(<4 x i64> %a0, <4 x i64> %a1) {
 }
 
 define <64 x i8> @stack_fold_pabsb(<64 x i8> %a0) {
-  ;CHECK-LABEL: stack_fold_pabsb
-  ;CHECK:       vpabsb {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pabsb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpabsb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp sgt <64 x i8> %a0, zeroinitializer
   %3 = sub <64 x i8> zeroinitializer, %a0
@@ -259,8 +488,21 @@ define <64 x i8> @stack_fold_pabsb(<64 x i8> %a0) {
 }
 
 define <64 x i8> @stack_fold_pabsb_mask(<64 x i8> %passthru, <64 x i8> %a0, i64 %mask) {
-  ;CHECK-LABEL: stack_fold_pabsb_mask
-  ;CHECK:       vpabsb {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pabsb_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subq $56, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovq %rdi, %k1
+; CHECK-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; CHECK-NEXT:    vpabsb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    addq $56, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp sgt <64 x i8> %a0, zeroinitializer
   %3 = sub <64 x i8> zeroinitializer, %a0
@@ -271,8 +513,15 @@ define <64 x i8> @stack_fold_pabsb_mask(<64 x i8> %passthru, <64 x i8> %a0, i64
 }
 
 define <64 x i8> @stack_fold_pabsb_maskz(<64 x i8> %a0, i64 %mask) {
-  ;CHECK-LABEL: stack_fold_pabsb_maskz
-  ;CHECK:       vpabsb {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pabsb_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovq %rdi, %k1
+; CHECK-NEXT:    vpabsb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp sgt <64 x i8> %a0, zeroinitializer
   %3 = sub <64 x i8> zeroinitializer, %a0
@@ -283,8 +532,14 @@ define <64 x i8> @stack_fold_pabsb_maskz(<64 x i8> %a0, i64 %mask) {
 }
 
 define <16 x i32> @stack_fold_pabsd(<16 x i32> %a0) {
-  ;CHECK-LABEL: stack_fold_pabsd
-  ;CHECK:       vpabsd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pabsd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpabsd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp sgt <16 x i32> %a0, zeroinitializer
   %3 = sub <16 x i32> zeroinitializer, %a0
@@ -293,8 +548,21 @@ define <16 x i32> @stack_fold_pabsd(<16 x i32> %a0) {
 }
 
 define <16 x i32> @stack_fold_pabsd_mask(<16 x i32> %passthru, <16 x i32> %a0, i16 %mask) {
-  ;CHECK-LABEL: stack_fold_pabsd_mask
-  ;CHECK:       vpabsd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pabsd_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subq $56, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; CHECK-NEXT:    vpabsd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    addq $56, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp sgt <16 x i32> %a0, zeroinitializer
   %3 = sub <16 x i32> zeroinitializer, %a0
@@ -305,8 +573,15 @@ define <16 x i32> @stack_fold_pabsd_mask(<16 x i32> %passthru, <16 x i32> %a0, i
 }
 
 define <16 x i32> @stack_fold_pabsd_maskz(<16 x i32> %a0, i16 %mask) {
-  ;CHECK-LABEL: stack_fold_pabsd_maskz
-  ;CHECK:       vpabsd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pabsd_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpabsd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp sgt <16 x i32> %a0, zeroinitializer
   %3 = sub <16 x i32> zeroinitializer, %a0
@@ -317,8 +592,14 @@ define <16 x i32> @stack_fold_pabsd_maskz(<16 x i32> %a0, i16 %mask) {
 }
 
 define <8 x i64> @stack_fold_pabsq(<8 x i64> %a0) {
-  ;CHECK-LABEL: stack_fold_pabsq
-  ;CHECK:       vpabsq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pabsq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpabsq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp sgt <8 x i64> %a0, zeroinitializer
   %3 = sub <8 x i64> zeroinitializer, %a0
@@ -327,8 +608,21 @@ define <8 x i64> @stack_fold_pabsq(<8 x i64> %a0) {
 }
 
 define <8 x i64> @stack_fold_pabsq_mask(<8 x i64> %passthru, <8 x i64> %a0, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_pabsq_mask
-  ;CHECK:       vpabsq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pabsq_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subq $56, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; CHECK-NEXT:    vpabsq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    addq $56, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp sgt <8 x i64> %a0, zeroinitializer
   %3 = sub <8 x i64> zeroinitializer, %a0
@@ -339,8 +633,15 @@ define <8 x i64> @stack_fold_pabsq_mask(<8 x i64> %passthru, <8 x i64> %a0, i8 %
 }
 
 define <8 x i64> @stack_fold_pabsq_maskz(<8 x i64> %a0, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_pabsq_maskz
-  ;CHECK:       vpabsq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pabsq_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpabsq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp sgt <8 x i64> %a0, zeroinitializer
   %3 = sub <8 x i64> zeroinitializer, %a0
@@ -351,8 +652,14 @@ define <8 x i64> @stack_fold_pabsq_maskz(<8 x i64> %a0, i8 %mask) {
 }
 
 define <32 x i16> @stack_fold_pabsw(<32 x i16> %a0) {
-  ;CHECK-LABEL: stack_fold_pabsw
-  ;CHECK:       vpabsw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pabsw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpabsw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp sgt <32 x i16> %a0, zeroinitializer
   %3 = sub <32 x i16> zeroinitializer, %a0
@@ -361,8 +668,21 @@ define <32 x i16> @stack_fold_pabsw(<32 x i16> %a0) {
 }
 
 define <32 x i16> @stack_fold_pabsw_mask(<32 x i16> %passthru, <32 x i16> %a0, i32 %mask) {
-  ;CHECK-LABEL: stack_fold_pabsw_mask
-  ;CHECK:       vpabsw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pabsw_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subq $56, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; CHECK-NEXT:    vpabsw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    addq $56, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp sgt <32 x i16> %a0, zeroinitializer
   %3 = sub <32 x i16> zeroinitializer, %a0
@@ -373,8 +693,15 @@ define <32 x i16> @stack_fold_pabsw_mask(<32 x i16> %passthru, <32 x i16> %a0, i
 }
 
 define <32 x i16> @stack_fold_pabsw_maskz(<32 x i16> %a0, i32 %mask) {
-  ;CHECK-LABEL: stack_fold_pabsw_maskz
-  ;CHECK:       vpabsw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pabsw_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpabsw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp sgt <32 x i16> %a0, zeroinitializer
   %3 = sub <32 x i16> zeroinitializer, %a0
@@ -385,8 +712,14 @@ define <32 x i16> @stack_fold_pabsw_maskz(<32 x i16> %a0, i32 %mask) {
 }
 
 define <32 x i16> @stack_fold_packssdw(<16 x i32> %a0, <16 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_packssdw
-  ;CHECK:       vpackssdw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_packssdw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpackssdw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <32 x i16> @llvm.x86.avx512.packssdw.512(<16 x i32> %a0, <16 x i32> %a1)
   ret <32 x i16> %2
@@ -394,8 +727,14 @@ define <32 x i16> @stack_fold_packssdw(<16 x i32> %a0, <16 x i32> %a1) {
 declare <32 x i16> @llvm.x86.avx512.packssdw.512(<16 x i32>, <16 x i32>) nounwind readnone
 
 define <64 x i8> @stack_fold_packsswb(<32 x i16> %a0, <32 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_packsswb
-  ;CHECK:       vpacksswb {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_packsswb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpacksswb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <64 x i8> @llvm.x86.avx512.packsswb.512(<32 x i16> %a0, <32 x i16> %a1)
   ret <64 x i8> %2
@@ -403,8 +742,14 @@ define <64 x i8> @stack_fold_packsswb(<32 x i16> %a0, <32 x i16> %a1) {
 declare <64 x i8> @llvm.x86.avx512.packsswb.512(<32 x i16>, <32 x i16>) nounwind readnone
 
 define <32 x i16> @stack_fold_packusdw(<16 x i32> %a0, <16 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_packusdw
-  ;CHECK:       vpackusdw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_packusdw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpackusdw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <32 x i16> @llvm.x86.avx512.packusdw.512(<16 x i32> %a0, <16 x i32> %a1)
   ret <32 x i16> %2
@@ -412,8 +757,17 @@ define <32 x i16> @stack_fold_packusdw(<16 x i32> %a0, <16 x i32> %a1) {
 declare <32 x i16> @llvm.x86.avx512.packusdw.512(<16 x i32>, <16 x i32>) nounwind readnone
 
 define <32 x i16> @stack_fold_packusdw_mask(<32 x i16>* %passthru, <16 x i32> %a0, <16 x i32> %a1, i32 %mask) {
-  ;CHECK-LABEL: stack_fold_packusdw_mask
-  ;CHECK:       vpackusdw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_packusdw_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm2
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vpackusdw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = load <32 x i16>, <32 x i16>* %passthru
   %3 = call <32 x i16> @llvm.x86.avx512.packusdw.512(<16 x i32> %a0, <16 x i32> %a1)
@@ -423,8 +777,15 @@ define <32 x i16> @stack_fold_packusdw_mask(<32 x i16>* %passthru, <16 x i32> %a
 }
 
 define <32 x i16> @stack_fold_packusdw_maskz(<16 x i32> %a0, <16 x i32> %a1, i32 %mask) {
-  ;CHECK-LABEL: stack_fold_packusdw_maskz
-  ;CHECK:       vpackusdw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_packusdw_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpackusdw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <32 x i16> @llvm.x86.avx512.packusdw.512(<16 x i32> %a0, <16 x i32> %a1)
   %3 = bitcast i32 %mask to <32 x i1>
@@ -433,8 +794,14 @@ define <32 x i16> @stack_fold_packusdw_maskz(<16 x i32> %a0, <16 x i32> %a1, i32
 }
 
 define <64 x i8> @stack_fold_packuswb(<32 x i16> %a0, <32 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_packuswb
-  ;CHECK:       vpackuswb {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_packuswb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpackuswb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <64 x i8> @llvm.x86.avx512.packuswb.512(<32 x i16> %a0, <32 x i16> %a1)
   ret <64 x i8> %2
@@ -442,24 +809,45 @@ define <64 x i8> @stack_fold_packuswb(<32 x i16> %a0, <32 x i16> %a1) {
 declare <64 x i8> @llvm.x86.avx512.packuswb.512(<32 x i16>, <32 x i16>) nounwind readnone
 
 define <64 x i8> @stack_fold_paddb(<64 x i8> %a0, <64 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_paddb
-  ;CHECK:       vpaddb {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpaddb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = add <64 x i8> %a0, %a1
   ret <64 x i8> %2
 }
 
 define <64 x i8> @stack_fold_paddb_commuted(<64 x i8> %a0, <64 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_paddb_commuted:
-  ;CHECK:       vpaddb {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddb_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpaddb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = add <64 x i8> %a1, %a0
   ret <64 x i8> %2
 }
 
 define <64 x i8> @stack_fold_paddb_mask(<64 x i8> %a0, <64 x i8> %a1, <64 x i8>* %a2, i64 %mask) {
-  ;CHECK-LABEL: stack_fold_paddb_mask
-  ;CHECK:       vpaddb {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddb_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm1
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovq %rsi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm0
+; CHECK-NEXT:    vpaddb {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = add <64 x i8> %a0, %a1
   %3 = bitcast i64 %mask to <64 x i1>
@@ -470,8 +858,17 @@ define <64 x i8> @stack_fold_paddb_mask(<64 x i8> %a0, <64 x i8> %a1, <64 x i8>*
 }
 
 define <64 x i8> @stack_fold_paddb_mask_commuted(<64 x i8> %a0, <64 x i8> %a1, <64 x i8>* %a2, i64 %mask) {
-  ;CHECK-LABEL: stack_fold_paddb_mask_commuted:
-  ;CHECK:       vpaddb {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddb_mask_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm1
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovq %rsi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm0
+; CHECK-NEXT:    vpaddb {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = add <64 x i8> %a1, %a0
   %3 = bitcast i64 %mask to <64 x i1>
@@ -482,8 +879,15 @@ define <64 x i8> @stack_fold_paddb_mask_commuted(<64 x i8> %a0, <64 x i8> %a1, <
 }
 
 define <64 x i8> @stack_fold_paddb_maskz(<64 x i8> %a0, <64 x i8> %a1, i64 %mask) {
-  ;CHECK-LABEL: stack_fold_paddb_maskz
-  ;CHECK:       vpaddb {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddb_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovq %rdi, %k1
+; CHECK-NEXT:    vpaddb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = add <64 x i8> %a0, %a1
   %3 = bitcast i64 %mask to <64 x i1>
@@ -492,8 +896,15 @@ define <64 x i8> @stack_fold_paddb_maskz(<64 x i8> %a0, <64 x i8> %a1, i64 %mask
 }
 
 define <64 x i8> @stack_fold_paddb_maskz_commuted(<64 x i8> %a0, <64 x i8> %a1, i64 %mask) {
-  ;CHECK-LABEL: stack_fold_paddb_maskz_commuted:
-  ;CHECK:       vpaddb {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddb_maskz_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovq %rdi, %k1
+; CHECK-NEXT:    vpaddb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = add <64 x i8> %a1, %a0
   %3 = bitcast i64 %mask to <64 x i1>
@@ -502,24 +913,45 @@ define <64 x i8> @stack_fold_paddb_maskz_commuted(<64 x i8> %a0, <64 x i8> %a1,
 }
 
 define <16 x i32> @stack_fold_paddd(<16 x i32> %a0, <16 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_paddd
-  ;CHECK:       vpaddd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpaddd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = add <16 x i32> %a0, %a1
   ret <16 x i32> %2
 }
 
 define <16 x i32> @stack_fold_paddd_commuted(<16 x i32> %a0, <16 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_paddd_commuted:
-  ;CHECK:       vpaddd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddd_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpaddd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = add <16 x i32> %a1, %a0
   ret <16 x i32> %2
 }
 
 define <16 x i32> @stack_fold_paddd_mask(<16 x i32> %a0, <16 x i32> %a1, <16 x i32>* %a2, i16 %mask) {
-  ;CHECK-LABEL: stack_fold_paddd_mask
-  ;CHECK:       vpaddd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddd_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm1
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm0
+; CHECK-NEXT:    vpaddd {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = add <16 x i32> %a0, %a1
   %3 = bitcast i16 %mask to <16 x i1>
@@ -530,8 +962,17 @@ define <16 x i32> @stack_fold_paddd_mask(<16 x i32> %a0, <16 x i32> %a1, <16 x i
 }
 
 define <16 x i32> @stack_fold_paddd_mask_commuted(<16 x i32> %a0, <16 x i32> %a1, <16 x i32>* %a2, i16 %mask) {
-  ;CHECK-LABEL: stack_fold_paddd_mask_commuted:
-  ;CHECK:       vpaddd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddd_mask_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm1
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm0
+; CHECK-NEXT:    vpaddd {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = add <16 x i32> %a1, %a0
   %3 = bitcast i16 %mask to <16 x i1>
@@ -542,8 +983,15 @@ define <16 x i32> @stack_fold_paddd_mask_commuted(<16 x i32> %a0, <16 x i32> %a1
 }
 
 define <16 x i32> @stack_fold_paddd_maskz(<16 x i32> %a0, <16 x i32> %a1, i16 %mask) {
-  ;CHECK-LABEL: stack_fold_paddd_maskz
-  ;CHECK:       vpaddd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddd_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpaddd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = add <16 x i32> %a0, %a1
   %3 = bitcast i16 %mask to <16 x i1>
@@ -552,8 +1000,15 @@ define <16 x i32> @stack_fold_paddd_maskz(<16 x i32> %a0, <16 x i32> %a1, i16 %m
 }
 
 define <16 x i32> @stack_fold_paddd_maskz_commuted(<16 x i32> %a0, <16 x i32> %a1, i16 %mask) {
-  ;CHECK-LABEL: stack_fold_paddd_maskz_commuted:
-  ;CHECK:       vpaddd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddd_maskz_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpaddd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = add <16 x i32> %a1, %a0
   %3 = bitcast i16 %mask to <16 x i1>
@@ -562,24 +1017,45 @@ define <16 x i32> @stack_fold_paddd_maskz_commuted(<16 x i32> %a0, <16 x i32> %a
 }
 
 define <8 x i64> @stack_fold_paddq(<8 x i64> %a0, <8 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_paddq
-  ;CHECK:       vpaddq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpaddq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = add <8 x i64> %a0, %a1
   ret <8 x i64> %2
 }
 
 define <8 x i64> @stack_fold_paddq_commuted(<8 x i64> %a0, <8 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_paddq_commuted:
-  ;CHECK:       vpaddq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddq_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpaddq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = add <8 x i64> %a1, %a0
   ret <8 x i64> %2
 }
 
 define <8 x i64> @stack_fold_paddq_mask(<8 x i64> %a0, <8 x i64> %a1, <8 x i64>* %a2, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_paddq_mask
-  ;CHECK:       vpaddq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddq_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm1
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm0
+; CHECK-NEXT:    vpaddq {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = add <8 x i64> %a0, %a1
   %3 = bitcast i8 %mask to <8 x i1>
@@ -590,8 +1066,17 @@ define <8 x i64> @stack_fold_paddq_mask(<8 x i64> %a0, <8 x i64> %a1, <8 x i64>*
 }
 
 define <8 x i64> @stack_fold_paddq_mask_commuted(<8 x i64> %a0, <8 x i64> %a1, <8 x i64>* %a2, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_paddq_mask_commuted:
-  ;CHECK:       vpaddq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddq_mask_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm1
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm0
+; CHECK-NEXT:    vpaddq {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = add <8 x i64> %a1, %a0
   %3 = bitcast i8 %mask to <8 x i1>
@@ -602,8 +1087,15 @@ define <8 x i64> @stack_fold_paddq_mask_commuted(<8 x i64> %a0, <8 x i64> %a1, <
 }
 
 define <8 x i64> @stack_fold_paddq_maskz(<8 x i64> %a0, <8 x i64> %a1, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_paddq_maskz
-  ;CHECK:       vpaddq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddq_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpaddq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = add <8 x i64> %a0, %a1
   %3 = bitcast i8 %mask to <8 x i1>
@@ -612,8 +1104,15 @@ define <8 x i64> @stack_fold_paddq_maskz(<8 x i64> %a0, <8 x i64> %a1, i8 %mask)
 }
 
 define <8 x i64> @stack_fold_paddq_maskz_commuted(<8 x i64> %a0, <8 x i64> %a1, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_paddq_maskz_commuted:
-  ;CHECK:       vpaddq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddq_maskz_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpaddq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = add <8 x i64> %a1, %a0
   %3 = bitcast i8 %mask to <8 x i1>
@@ -622,24 +1121,45 @@ define <8 x i64> @stack_fold_paddq_maskz_commuted(<8 x i64> %a0, <8 x i64> %a1,
 }
 
 define <64 x i8> @stack_fold_paddsb(<64 x i8> %a0, <64 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_paddsb
-  ;CHECK:       vpaddsb {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddsb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpaddsb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <64 x i8> @llvm.sadd.sat.v64i8(<64 x i8> %a0, <64 x i8> %a1)
   ret <64 x i8> %2
 }
 
 define <64 x i8> @stack_fold_paddsb_commuted(<64 x i8> %a0, <64 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_paddsb_commuted:
-  ;CHECK:       vpaddsb {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddsb_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpaddsb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <64 x i8> @llvm.sadd.sat.v64i8(<64 x i8> %a1, <64 x i8> %a0)
   ret <64 x i8> %2
 }
 
 define <64 x i8> @stack_fold_paddsb_mask(<64 x i8> %a0, <64 x i8> %a1, <64 x i8>* %a2, i64 %mask) {
-  ;CHECK-LABEL: stack_fold_paddsb_mask
-  ;CHECK:       vpaddsb {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddsb_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm1
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovq %rsi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm0
+; CHECK-NEXT:    vpaddsb {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <64 x i8> @llvm.sadd.sat.v64i8(<64 x i8> %a0, <64 x i8> %a1)
   %3 = bitcast i64 %mask to <64 x i1>
@@ -650,8 +1170,17 @@ define <64 x i8> @stack_fold_paddsb_mask(<64 x i8> %a0, <64 x i8> %a1, <64 x i8>
 }
 
 define <64 x i8> @stack_fold_paddsb_mask_commuted(<64 x i8> %a0, <64 x i8> %a1, <64 x i8>* %a2, i64 %mask) {
-  ;CHECK-LABEL: stack_fold_paddsb_mask_commuted:
-  ;CHECK:       vpaddsb {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddsb_mask_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm1
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovq %rsi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm0
+; CHECK-NEXT:    vpaddsb {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <64 x i8> @llvm.sadd.sat.v64i8(<64 x i8> %a1, <64 x i8> %a0)
   %3 = bitcast i64 %mask to <64 x i1>
@@ -662,8 +1191,15 @@ define <64 x i8> @stack_fold_paddsb_mask_commuted(<64 x i8> %a0, <64 x i8> %a1,
 }
 
 define <64 x i8> @stack_fold_paddsb_maskz(<64 x i8> %a0, <64 x i8> %a1, i64 %mask) {
-  ;CHECK-LABEL: stack_fold_paddsb_maskz
-  ;CHECK:       vpaddsb {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddsb_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovq %rdi, %k1
+; CHECK-NEXT:    vpaddsb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <64 x i8> @llvm.sadd.sat.v64i8(<64 x i8> %a0, <64 x i8> %a1)
   %3 = bitcast i64 %mask to <64 x i1>
@@ -672,8 +1208,15 @@ define <64 x i8> @stack_fold_paddsb_maskz(<64 x i8> %a0, <64 x i8> %a1, i64 %mas
 }
 
 define <64 x i8> @stack_fold_paddsb_maskz_commuted(<64 x i8> %a0, <64 x i8> %a1, i64 %mask) {
-  ;CHECK-LABEL: stack_fold_paddsb_maskz_commuted:
-  ;CHECK:       vpaddsb {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddsb_maskz_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovq %rdi, %k1
+; CHECK-NEXT:    vpaddsb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <64 x i8> @llvm.sadd.sat.v64i8(<64 x i8> %a1, <64 x i8> %a0)
   %3 = bitcast i64 %mask to <64 x i1>
@@ -682,24 +1225,45 @@ define <64 x i8> @stack_fold_paddsb_maskz_commuted(<64 x i8> %a0, <64 x i8> %a1,
 }
 
 define <32 x i16> @stack_fold_paddsw(<32 x i16> %a0, <32 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_paddsw
-  ;CHECK:       vpaddsw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddsw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpaddsw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16> %a0, <32 x i16> %a1)
   ret <32 x i16> %2
 }
 
 define <32 x i16> @stack_fold_paddsw_commuted(<32 x i16> %a0, <32 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_paddsw_commuted:
-  ;CHECK:       vpaddsw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddsw_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpaddsw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16> %a1, <32 x i16> %a0)
   ret <32 x i16> %2
 }
 
 define <32 x i16> @stack_fold_paddsw_mask(<32 x i16> %a0, <32 x i16> %a1, <32 x i16>* %a2, i32 %mask) {
-  ;CHECK-LABEL: stack_fold_paddsw_mask
-  ;CHECK:       vpaddsw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddsw_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm1
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm0
+; CHECK-NEXT:    vpaddsw {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16> %a0, <32 x i16> %a1)
   %3 = bitcast i32 %mask to <32 x i1>
@@ -710,8 +1274,17 @@ define <32 x i16> @stack_fold_paddsw_mask(<32 x i16> %a0, <32 x i16> %a1, <32 x
 }
 
 define <32 x i16> @stack_fold_paddsw_mask_commuted(<32 x i16> %a0, <32 x i16> %a1, <32 x i16>* %a2, i32 %mask) {
-  ;CHECK-LABEL: stack_fold_paddsw_mask_commuted:
-  ;CHECK:       vpaddsw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddsw_mask_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm1
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm0
+; CHECK-NEXT:    vpaddsw {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16> %a1, <32 x i16> %a0)
   %3 = bitcast i32 %mask to <32 x i1>
@@ -722,8 +1295,15 @@ define <32 x i16> @stack_fold_paddsw_mask_commuted(<32 x i16> %a0, <32 x i16> %a
 }
 
 define <32 x i16> @stack_fold_paddsw_maskz(<32 x i16> %a0, <32 x i16> %a1, i32 %mask) {
-  ;CHECK-LABEL: stack_fold_paddsw_maskz
-  ;CHECK:       vpaddsw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddsw_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpaddsw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16> %a0, <32 x i16> %a1)
   %3 = bitcast i32 %mask to <32 x i1>
@@ -732,8 +1312,15 @@ define <32 x i16> @stack_fold_paddsw_maskz(<32 x i16> %a0, <32 x i16> %a1, i32 %
 }
 
 define <32 x i16> @stack_fold_paddsw_maskz_commuted(<32 x i16> %a0, <32 x i16> %a1, i32 %mask) {
-  ;CHECK-LABEL: stack_fold_paddsw_maskz_commuted:
-  ;CHECK:       vpaddsw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddsw_maskz_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpaddsw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16> %a1, <32 x i16> %a0)
   %3 = bitcast i32 %mask to <32 x i1>
@@ -742,24 +1329,45 @@ define <32 x i16> @stack_fold_paddsw_maskz_commuted(<32 x i16> %a0, <32 x i16> %
 }
 
 define <64 x i8> @stack_fold_paddusb(<64 x i8> %a0, <64 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_paddusb
-  ;CHECK:       vpaddusb {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddusb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpaddusb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <64 x i8> @llvm.uadd.sat.v64i8(<64 x i8> %a0, <64 x i8> %a1)
   ret <64 x i8> %2
 }
 
 define <64 x i8> @stack_fold_paddusb_commuted(<64 x i8> %a0, <64 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_paddusb_commuted:
-  ;CHECK:       vpaddusb {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddusb_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpaddusb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <64 x i8> @llvm.uadd.sat.v64i8(<64 x i8> %a1, <64 x i8> %a0)
   ret <64 x i8> %2
 }
 
 define <64 x i8> @stack_fold_paddusb_mask(<64 x i8> %a0, <64 x i8> %a1, <64 x i8>* %a2, i64 %mask) {
-  ;CHECK-LABEL: stack_fold_paddusb_mask
-  ;CHECK:       vpaddusb {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddusb_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm1
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovq %rsi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm0
+; CHECK-NEXT:    vpaddusb {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <64 x i8> @llvm.uadd.sat.v64i8(<64 x i8> %a0, <64 x i8> %a1)
   %3 = bitcast i64 %mask to <64 x i1>
@@ -770,8 +1378,17 @@ define <64 x i8> @stack_fold_paddusb_mask(<64 x i8> %a0, <64 x i8> %a1, <64 x i8
 }
 
 define <64 x i8> @stack_fold_paddusb_mask_commuted(<64 x i8> %a0, <64 x i8> %a1, <64 x i8>* %a2, i64 %mask) {
-  ;CHECK-LABEL: stack_fold_paddusb_mask_commuted:
-  ;CHECK:       vpaddusb {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddusb_mask_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm1
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovq %rsi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm0
+; CHECK-NEXT:    vpaddusb {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <64 x i8> @llvm.uadd.sat.v64i8(<64 x i8> %a1, <64 x i8> %a0)
   %3 = bitcast i64 %mask to <64 x i1>
@@ -782,8 +1399,15 @@ define <64 x i8> @stack_fold_paddusb_mask_commuted(<64 x i8> %a0, <64 x i8> %a1,
 }
 
 define <64 x i8> @stack_fold_paddusb_maskz(<64 x i8> %a0, <64 x i8> %a1, i64 %mask) {
-  ;CHECK-LABEL: stack_fold_paddusb_maskz
-  ;CHECK:       vpaddusb {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddusb_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovq %rdi, %k1
+; CHECK-NEXT:    vpaddusb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <64 x i8> @llvm.uadd.sat.v64i8(<64 x i8> %a0, <64 x i8> %a1)
   %3 = bitcast i64 %mask to <64 x i1>
@@ -792,8 +1416,15 @@ define <64 x i8> @stack_fold_paddusb_maskz(<64 x i8> %a0, <64 x i8> %a1, i64 %ma
 }
 
 define <64 x i8> @stack_fold_paddusb_maskz_commuted(<64 x i8> %a0, <64 x i8> %a1, i64 %mask) {
-  ;CHECK-LABEL: stack_fold_paddusb_maskz_commuted:
-  ;CHECK:       vpaddusb {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddusb_maskz_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovq %rdi, %k1
+; CHECK-NEXT:    vpaddusb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <64 x i8> @llvm.uadd.sat.v64i8(<64 x i8> %a1, <64 x i8> %a0)
   %3 = bitcast i64 %mask to <64 x i1>
@@ -802,24 +1433,45 @@ define <64 x i8> @stack_fold_paddusb_maskz_commuted(<64 x i8> %a0, <64 x i8> %a1
 }
 
 define <32 x i16> @stack_fold_paddusw(<32 x i16> %a0, <32 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_paddusw
-  ;CHECK:       vpaddusw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddusw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpaddusw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <32 x i16> @llvm.uadd.sat.v32i16(<32 x i16> %a0, <32 x i16> %a1)
   ret <32 x i16> %2
 }
 
 define <32 x i16> @stack_fold_paddusw_commuted(<32 x i16> %a0, <32 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_paddusw_commuted:
-  ;CHECK:       vpaddusw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddusw_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpaddusw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <32 x i16> @llvm.uadd.sat.v32i16(<32 x i16> %a1, <32 x i16> %a0)
   ret <32 x i16> %2
 }
 
 define <32 x i16> @stack_fold_paddusw_mask(<32 x i16> %a0, <32 x i16> %a1, <32 x i16>* %a2, i32 %mask) {
-  ;CHECK-LABEL: stack_fold_paddusw_mask
-  ;CHECK:       vpaddusw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddusw_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm1
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm0
+; CHECK-NEXT:    vpaddusw {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <32 x i16> @llvm.uadd.sat.v32i16(<32 x i16> %a0, <32 x i16> %a1)
   %3 = bitcast i32 %mask to <32 x i1>
@@ -830,8 +1482,17 @@ define <32 x i16> @stack_fold_paddusw_mask(<32 x i16> %a0, <32 x i16> %a1, <32 x
 }
 
 define <32 x i16> @stack_fold_paddusw_mask_commuted(<32 x i16> %a0, <32 x i16> %a1, <32 x i16>* %a2, i32 %mask) {
-  ;CHECK-LABEL: stack_fold_paddusw_mask_commuted:
-  ;CHECK:       vpaddusw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddusw_mask_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm1
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm0
+; CHECK-NEXT:    vpaddusw {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <32 x i16> @llvm.uadd.sat.v32i16(<32 x i16> %a1, <32 x i16> %a0)
   %3 = bitcast i32 %mask to <32 x i1>
@@ -842,8 +1503,15 @@ define <32 x i16> @stack_fold_paddusw_mask_commuted(<32 x i16> %a0, <32 x i16> %
 }
 
 define <32 x i16> @stack_fold_paddusw_maskz(<32 x i16> %a0, <32 x i16> %a1, i32 %mask) {
-  ;CHECK-LABEL: stack_fold_paddusw_maskz
-  ;CHECK:       vpaddusw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddusw_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpaddusw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <32 x i16> @llvm.uadd.sat.v32i16(<32 x i16> %a0, <32 x i16> %a1)
   %3 = bitcast i32 %mask to <32 x i1>
@@ -852,8 +1520,15 @@ define <32 x i16> @stack_fold_paddusw_maskz(<32 x i16> %a0, <32 x i16> %a1, i32
 }
 
 define <32 x i16> @stack_fold_paddusw_maskz_commuted(<32 x i16> %a0, <32 x i16> %a1, i32 %mask) {
-  ;CHECK-LABEL: stack_fold_paddusw_maskz_commuted:
-  ;CHECK:       vpaddusw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddusw_maskz_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpaddusw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <32 x i16> @llvm.uadd.sat.v32i16(<32 x i16> %a1, <32 x i16> %a0)
   %3 = bitcast i32 %mask to <32 x i1>
@@ -862,24 +1537,45 @@ define <32 x i16> @stack_fold_paddusw_maskz_commuted(<32 x i16> %a0, <32 x i16>
 }
 
 define <32 x i16> @stack_fold_paddw(<32 x i16> %a0, <32 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_paddw
-  ;CHECK:       vpaddw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpaddw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = add <32 x i16> %a0, %a1
   ret <32 x i16> %2
 }
 
 define <32 x i16> @stack_fold_paddw_commuted(<32 x i16> %a0, <32 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_paddw_commuted:
-  ;CHECK:       vpaddw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddw_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpaddw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = add <32 x i16> %a1, %a0
   ret <32 x i16> %2
 }
 
 define <32 x i16> @stack_fold_paddw_mask(<32 x i16> %a0, <32 x i16> %a1, <32 x i16>* %a2, i32 %mask) {
-  ;CHECK-LABEL: stack_fold_paddw_mask
-  ;CHECK:       vpaddw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddw_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm1
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm0
+; CHECK-NEXT:    vpaddw {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = add <32 x i16> %a0, %a1
   %3 = bitcast i32 %mask to <32 x i1>
@@ -890,8 +1586,17 @@ define <32 x i16> @stack_fold_paddw_mask(<32 x i16> %a0, <32 x i16> %a1, <32 x i
 }
 
 define <32 x i16> @stack_fold_paddw_mask_commuted(<32 x i16> %a0, <32 x i16> %a1, <32 x i16>* %a2, i32 %mask) {
-  ;CHECK-LABEL: stack_fold_paddw_mask_commuted:
-  ;CHECK:       vpaddw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddw_mask_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm1
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm0
+; CHECK-NEXT:    vpaddw {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = add <32 x i16> %a1, %a0
   %3 = bitcast i32 %mask to <32 x i1>
@@ -902,8 +1607,15 @@ define <32 x i16> @stack_fold_paddw_mask_commuted(<32 x i16> %a0, <32 x i16> %a1
 }
 
 define <32 x i16> @stack_fold_paddw_maskz(<32 x i16> %a0, <32 x i16> %a1, i32 %mask) {
-  ;CHECK-LABEL: stack_fold_paddw_maskz
-  ;CHECK:       vpaddw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddw_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpaddw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = add <32 x i16> %a0, %a1
   %3 = bitcast i32 %mask to <32 x i1>
@@ -912,8 +1624,15 @@ define <32 x i16> @stack_fold_paddw_maskz(<32 x i16> %a0, <32 x i16> %a1, i32 %m
 }
 
 define <32 x i16> @stack_fold_paddw_maskz_commuted(<32 x i16> %a0, <32 x i16> %a1, i32 %mask) {
-  ;CHECK-LABEL: stack_fold_paddw_maskz_commuted:
-  ;CHECK:       vpaddw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddw_maskz_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpaddw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = add <32 x i16> %a1, %a0
   %3 = bitcast i32 %mask to <32 x i1>
@@ -922,16 +1641,45 @@ define <32 x i16> @stack_fold_paddw_maskz_commuted(<32 x i16> %a0, <32 x i16> %a
 }
 
 define <64 x i8> @stack_fold_palignr(<64 x i8> %a0, <64 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_palignr
-  ;CHECK:       vpalignr $1, {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_palignr:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subq $56, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; CHECK-NEXT:    vpalignr $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    # zmm0 = mem[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],zmm0[0],mem[17,18,19,20,21,22,23,24,25,26,27,28,29,30,31],zmm0[16],mem[33,34,35,36,37,38,39,40,41,42,43,44,45,46,47],zmm0[32],mem[49,50,51,52,53,54,55,56,57,58,59,60,61,62,63],zmm0[48]
+; CHECK-NEXT:    addq $56, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <64 x i8> %a1, <64 x i8> %a0, <64 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 64, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 80, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 96, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 112>
   ret <64 x i8> %2
 }
 
 define <64 x i8> @stack_fold_palignr_mask(<64 x i8> %a0, <64 x i8> %a1, <64 x i8>* %passthru, i64 %mask) {
-  ;CHECK-LABEL: stack_fold_palignr_mask
-  ;CHECK:       vpalignr $1, {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_palignr_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subq $56, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovq %rsi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm1
+; CHECK-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; CHECK-NEXT:    vpalignr $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm1 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    # zmm1 {%k1} = mem[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],zmm0[0],mem[17,18,19,20,21,22,23,24,25,26,27,28,29,30,31],zmm0[16],mem[33,34,35,36,37,38,39,40,41,42,43,44,45,46,47],zmm0[32],mem[49,50,51,52,53,54,55,56,57,58,59,60,61,62,63],zmm0[48]
+; CHECK-NEXT:    vmovdqa64 %zmm1, %zmm0
+; CHECK-NEXT:    addq $56, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <64 x i8> %a1, <64 x i8> %a0, <64 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 64, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 80, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 96, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 112>
   %3 = bitcast i64 %mask to <64 x i1>
@@ -941,8 +1689,22 @@ define <64 x i8> @stack_fold_palignr_mask(<64 x i8> %a0, <64 x i8> %a1, <64 x i8
 }
 
 define <64 x i8> @stack_fold_palignr_maskz(<64 x i8> %a0, <64 x i8> %a1, i64 %mask) {
-  ;CHECK-LABEL: stack_fold_palignr_maskz
-  ;CHECK:       vpalignr $1, {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_palignr_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subq $56, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovq %rdi, %k1
+; CHECK-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; CHECK-NEXT:    vpalignr $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    # zmm0 {%k1} {z} = mem[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],zmm0[0],mem[17,18,19,20,21,22,23,24,25,26,27,28,29,30,31],zmm0[16],mem[33,34,35,36,37,38,39,40,41,42,43,44,45,46,47],zmm0[32],mem[49,50,51,52,53,54,55,56,57,58,59,60,61,62,63],zmm0[48]
+; CHECK-NEXT:    addq $56, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <64 x i8> %a1, <64 x i8> %a0, <64 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 64, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 80, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 96, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 112>
   %3 = bitcast i64 %mask to <64 x i1>
@@ -951,24 +1713,45 @@ define <64 x i8> @stack_fold_palignr_maskz(<64 x i8> %a0, <64 x i8> %a1, i64 %ma
 }
 
 define <16 x i32> @stack_fold_pandd(<16 x i32> %a0, <16 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_pandd
-  ;CHECK:       vandps {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pandd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vandps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = and <16 x i32> %a0, %a1
   ret <16 x i32> %2
 }
 
 define <16 x i32> @stack_fold_pandd_commuted(<16 x i32> %a0, <16 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_pandd_commuted:
-  ;CHECK:       vandps {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pandd_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vandps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = and <16 x i32> %a1, %a0
   ret <16 x i32> %2
 }
 
 define <16 x i32> @stack_fold_pandd_mask(<16 x i32> %a0, <16 x i32> %a1, <16 x i32>* %a2, i16 %mask) {
-  ;CHECK-LABEL: stack_fold_pandd_mask
-  ;CHECK:       vandps {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pandd_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovaps %zmm0, %zmm1
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vmovaps (%rdi), %zmm0
+; CHECK-NEXT:    vandps {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = and <16 x i32> %a0, %a1
   %3 = bitcast i16 %mask to <16 x i1>
@@ -979,8 +1762,17 @@ define <16 x i32> @stack_fold_pandd_mask(<16 x i32> %a0, <16 x i32> %a1, <16 x i
 }
 
 define <16 x i32> @stack_fold_pandd_mask_commuted(<16 x i32> %a0, <16 x i32> %a1, <16 x i32>* %a2, i16 %mask) {
-  ;CHECK-LABEL: stack_fold_pandd_mask_commuted:
-  ;CHECK:       vandps {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pandd_mask_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovaps %zmm0, %zmm1
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vmovaps (%rdi), %zmm0
+; CHECK-NEXT:    vandps {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = and <16 x i32> %a1, %a0
   %3 = bitcast i16 %mask to <16 x i1>
@@ -991,8 +1783,15 @@ define <16 x i32> @stack_fold_pandd_mask_commuted(<16 x i32> %a0, <16 x i32> %a1
 }
 
 define <16 x i32> @stack_fold_pandd_maskz(<16 x i32> %a0, <16 x i32> %a1, i16 %mask) {
-  ;CHECK-LABEL: stack_fold_pandd_maskz
-  ;CHECK:       vandps {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pandd_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vandps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = and <16 x i32> %a0, %a1
   %3 = bitcast i16 %mask to <16 x i1>
@@ -1001,8 +1800,15 @@ define <16 x i32> @stack_fold_pandd_maskz(<16 x i32> %a0, <16 x i32> %a1, i16 %m
 }
 
 define <16 x i32> @stack_fold_pandd_maskz_commuted(<16 x i32> %a0, <16 x i32> %a1, i16 %mask) {
-  ;CHECK-LABEL: stack_fold_pandd_maskz_commuted:
-  ;CHECK:       vandps {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pandd_maskz_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vandps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = and <16 x i32> %a1, %a0
   %3 = bitcast i16 %mask to <16 x i1>
@@ -1011,24 +1817,45 @@ define <16 x i32> @stack_fold_pandd_maskz_commuted(<16 x i32> %a0, <16 x i32> %a
 }
 
 define <8 x i64> @stack_fold_pandq(<8 x i64> %a0, <8 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_pandq
-  ;CHECK:       vandps {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pandq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vandps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = and <8 x i64> %a0, %a1
   ret <8 x i64> %2
 }
 
 define <8 x i64> @stack_fold_pandq_commuted(<8 x i64> %a0, <8 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_pandq_commuted:
-  ;CHECK:       vandps {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pandq_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vandps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = and <8 x i64> %a1, %a0
   ret <8 x i64> %2
 }
 
 define <8 x i64> @stack_fold_pandq_mask(<8 x i64> %a0, <8 x i64> %a1, <8 x i64>* %a2, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_pandq_mask
-  ;CHECK:       vandpd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pandq_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovapd %zmm0, %zmm1
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vmovapd (%rdi), %zmm0
+; CHECK-NEXT:    vandpd {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = and <8 x i64> %a0, %a1
   %3 = bitcast i8 %mask to <8 x i1>
@@ -1039,8 +1866,17 @@ define <8 x i64> @stack_fold_pandq_mask(<8 x i64> %a0, <8 x i64> %a1, <8 x i64>*
 }
 
 define <8 x i64> @stack_fold_pandq_mask_commuted(<8 x i64> %a0, <8 x i64> %a1, <8 x i64>* %a2, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_pandq_mask_commuted:
-  ;CHECK:       vandpd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pandq_mask_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovapd %zmm0, %zmm1
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vmovapd (%rdi), %zmm0
+; CHECK-NEXT:    vandpd {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = and <8 x i64> %a1, %a0
   %3 = bitcast i8 %mask to <8 x i1>
@@ -1051,8 +1887,15 @@ define <8 x i64> @stack_fold_pandq_mask_commuted(<8 x i64> %a0, <8 x i64> %a1, <
 }
 
 define <8 x i64> @stack_fold_pandq_maskz(<8 x i64> %a0, <8 x i64> %a1, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_pandq_maskz
-  ;CHECK:       vandpd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pandq_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vandpd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = and <8 x i64> %a0, %a1
   %3 = bitcast i8 %mask to <8 x i1>
@@ -1061,8 +1904,15 @@ define <8 x i64> @stack_fold_pandq_maskz(<8 x i64> %a0, <8 x i64> %a1, i8 %mask)
 }
 
 define <8 x i64> @stack_fold_pandq_maskz_commuted(<8 x i64> %a0, <8 x i64> %a1, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_pandq_maskz_commuted:
-  ;CHECK:       vandpd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pandq_maskz_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vandpd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = and <8 x i64> %a1, %a0
   %3 = bitcast i8 %mask to <8 x i1>
@@ -1071,8 +1921,14 @@ define <8 x i64> @stack_fold_pandq_maskz_commuted(<8 x i64> %a0, <8 x i64> %a1,
 }
 
 define <16 x i32> @stack_fold_vpconflictd(<16 x i32> %a0) {
-  ;CHECK-LABEL: stack_fold_vpconflictd
-  ;CHECK:       vpconflictd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpconflictd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpconflictd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x i32> @llvm.x86.avx512.conflict.d.512(<16 x i32> %a0)
   ret <16 x i32> %2
@@ -1080,8 +1936,14 @@ define <16 x i32> @stack_fold_vpconflictd(<16 x i32> %a0) {
 declare <16 x i32> @llvm.x86.avx512.mask.conflict.d.512(<16 x i32>, <16 x i32>, i16) nounwind readonly
 
 define <8 x i64> @stack_fold_vpconflictq(<8 x i64> %a0) {
-  ;CHECK-LABEL: stack_fold_vpconflictq
-  ;CHECK:       vpconflictq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpconflictq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpconflictq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x i64> @llvm.x86.avx512.conflict.q.512(<8 x i64> %a0)
   ret <8 x i64> %2
@@ -1089,8 +1951,16 @@ define <8 x i64> @stack_fold_vpconflictq(<8 x i64> %a0) {
 declare <8 x i64> @llvm.x86.avx512.mask.conflict.q.512(<8 x i64>, <8 x i64>, i8) nounwind readnone
 
 define i64 @stack_fold_pcmpeqb(<64 x i8> %a0, <64 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pcmpeqb
-  ;CHECK:       vpcmpeqb {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%k[0-7]}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pcmpeqb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpcmpeqb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %k0 # 64-byte Folded Reload
+; CHECK-NEXT:    kmovq %k0, %rax
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp eq <64 x i8> %a0, %a1
   %3 = bitcast <64 x i1> %2 to i64
@@ -1098,8 +1968,17 @@ define i64 @stack_fold_pcmpeqb(<64 x i8> %a0, <64 x i8> %a1) {
 }
 
 define i16 @stack_fold_pcmpeqd(<16 x i32> %a0, <16 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_pcmpeqd
-  ;CHECK:       vpcmpeqd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%k[0-7]}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pcmpeqd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpcmpeqd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %k0 # 64-byte Folded Reload
+; CHECK-NEXT:    kmovd %k0, %eax
+; CHECK-NEXT:    # kill: def $ax killed $ax killed $eax
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp eq <16 x i32> %a0, %a1
   %3 = bitcast <16 x i1> %2 to i16
@@ -1107,8 +1986,17 @@ define i16 @stack_fold_pcmpeqd(<16 x i32> %a0, <16 x i32> %a1) {
 }
 
 define i8 @stack_fold_pcmpeqq(<8 x i64> %a0, <8 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_pcmpeqq
-  ;CHECK:       vpcmpeqq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%k[0-7]}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pcmpeqq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpcmpeqq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %k0 # 64-byte Folded Reload
+; CHECK-NEXT:    kmovd %k0, %eax
+; CHECK-NEXT:    # kill: def $al killed $al killed $eax
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp eq <8 x i64> %a0, %a1
   %3 = bitcast <8 x i1> %2 to i8
@@ -1116,8 +2004,16 @@ define i8 @stack_fold_pcmpeqq(<8 x i64> %a0, <8 x i64> %a1) {
 }
 
 define i32 @stack_fold_pcmpeqw(<32 x i16> %a0, <32 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_pcmpeqw
-  ;CHECK:       vpcmpeqw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%k[0-7]}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pcmpeqw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpcmpeqw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %k0 # 64-byte Folded Reload
+; CHECK-NEXT:    kmovd %k0, %eax
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp eq <32 x i16> %a0, %a1
   %3 = bitcast <32 x i1> %2 to i32
@@ -1125,8 +2021,26 @@ define i32 @stack_fold_pcmpeqw(<32 x i16> %a0, <32 x i16> %a1) {
 }
 
 define <16 x i32> @stack_fold_pcmpeqd_mask(<16 x i32> %a0, <16 x i32> %a1, <16 x i32>* %a2, i16 %mask, <16 x i32> %b0, <16 x i32> %b1) {
-  ;CHECK-LABEL: stack_fold_pcmpeqd_mask
-  ;CHECK:       vpcmpeqd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%k[0-7]}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pcmpeqd_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subq $184, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 192
+; CHECK-NEXT:    vmovups %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovups %zmm2, (%rsp) # 64-byte Spill
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; CHECK-NEXT:    vpaddd (%rdi), %zmm0, %zmm0
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vpcmpeqd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %k1 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; CHECK-NEXT:    vpblendmd (%rsp), %zmm0, %zmm0 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    addq $184, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   ; load and add are here to keep the operations below the side effecting block and to avoid folding the wrong load
   %2 = load <16 x i32>, <16 x i32>* %a2
@@ -1139,8 +2053,26 @@ define <16 x i32> @stack_fold_pcmpeqd_mask(<16 x i32> %a0, <16 x i32> %a1, <16 x
 }
 
 define <16 x i32> @stack_fold_pcmpeqd_mask_commuted(<16 x i32> %a0, <16 x i32> %a1, <16 x i32>* %a2, i16 %mask, <16 x i32> %b0, <16 x i32> %b1) {
-  ;CHECK-LABEL: stack_fold_pcmpeqd_mask_commuted
-  ;CHECK:       vpcmpeqd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%k[0-7]}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pcmpeqd_mask_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subq $184, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 192
+; CHECK-NEXT:    vmovups %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovups %zmm2, (%rsp) # 64-byte Spill
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; CHECK-NEXT:    vpaddd (%rdi), %zmm0, %zmm0
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vpcmpeqd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %k1 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; CHECK-NEXT:    vpblendmd (%rsp), %zmm0, %zmm0 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    addq $184, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   ; load and add are here to keep the operations below the side effecting block and to avoid folding the wrong load
   %2 = load <16 x i32>, <16 x i32>* %a2
@@ -1153,8 +2085,26 @@ define <16 x i32> @stack_fold_pcmpeqd_mask_commuted(<16 x i32> %a0, <16 x i32> %
 }
 
 define <16 x i32> @stack_fold_pcmpled_mask(<16 x i32> %a0, <16 x i32> %a1, <16 x i32>* %a2, i16 %mask, <16 x i32> %b0, <16 x i32> %b1) {
-  ;CHECK-LABEL: stack_fold_pcmpled_mask
-  ;CHECK:       vpcmpled {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%k[0-7]}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pcmpled_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subq $184, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 192
+; CHECK-NEXT:    vmovups %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovups %zmm2, (%rsp) # 64-byte Spill
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; CHECK-NEXT:    vpaddd (%rdi), %zmm0, %zmm0
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vpcmpled {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %k1 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; CHECK-NEXT:    vpblendmd (%rsp), %zmm0, %zmm0 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    addq $184, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   ; load and add are here to keep the operations below the side effecting block and to avoid folding the wrong load
   %2 = load <16 x i32>, <16 x i32>* %a2
@@ -1167,8 +2117,25 @@ define <16 x i32> @stack_fold_pcmpled_mask(<16 x i32> %a0, <16 x i32> %a1, <16 x
 }
 
 define i16 @stack_fold_pcmpleud(<16 x i32> %a0, <16 x i32> %a1, <16 x i32>* %a2, i16 %mask) {
-  ;CHECK-LABEL: stack_fold_pcmpleud
-  ;CHECK:       vpcmpleud {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%k[0-7]}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pcmpleud:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subq $56, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; CHECK-NEXT:    vpaddd (%rdi), %zmm0, %zmm0
+; CHECK-NEXT:    vpcmpleud {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %k0 # 64-byte Folded Reload
+; CHECK-NEXT:    kmovd %k0, %eax
+; CHECK-NEXT:    andl %esi, %eax
+; CHECK-NEXT:    # kill: def $ax killed $ax killed $eax
+; CHECK-NEXT:    addq $56, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = load <16 x i32>, <16 x i32>* %a2
   %3 = add <16 x i32> %a1, %2
@@ -1180,8 +2147,14 @@ define i16 @stack_fold_pcmpleud(<16 x i32> %a0, <16 x i32> %a1, <16 x i32>* %a2,
 }
 
 define <64 x i8> @stack_fold_permbvar(<64 x i8> %a0, <64 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_permbvar
-  ;CHECK:   vpermb {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_permbvar:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <64 x i8> @llvm.x86.avx512.permvar.qi.512(<64 x i8> %a1, <64 x i8> %a0)
   ret <64 x i8> %2
@@ -1189,8 +2162,17 @@ define <64 x i8> @stack_fold_permbvar(<64 x i8> %a0, <64 x i8> %a1) {
 declare <64 x i8> @llvm.x86.avx512.permvar.qi.512(<64 x i8>, <64 x i8>) nounwind readonly
 
 define <64 x i8> @stack_fold_permbvar_mask(<64 x i8>* %passthru, <64 x i8> %a0, <64 x i8> %a1, i64 %mask) {
-  ;CHECK-LABEL: stack_fold_permbvar_mask
-  ;CHECK:   vpermb {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_permbvar_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovq %rsi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm2
+; CHECK-NEXT:    vpermb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <64 x i8> @llvm.x86.avx512.permvar.qi.512(<64 x i8> %a1, <64 x i8> %a0)
   %3 = bitcast i64 %mask to <64 x i1>
@@ -1201,8 +2183,15 @@ define <64 x i8> @stack_fold_permbvar_mask(<64 x i8>* %passthru, <64 x i8> %a0,
 }
 
 define <64 x i8> @stack_fold_permbvar_maskz(<64 x i8> %a0, <64 x i8> %a1, i64 %mask) {
-  ;CHECK-LABEL: stack_fold_permbvar_maskz
-  ;CHECK:   vpermb {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_permbvar_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovq %rdi, %k1
+; CHECK-NEXT:    vpermb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <64 x i8> @llvm.x86.avx512.permvar.qi.512(<64 x i8> %a1, <64 x i8> %a0)
   %3 = bitcast i64 %mask to <64 x i1>
@@ -1211,8 +2200,16 @@ define <64 x i8> @stack_fold_permbvar_maskz(<64 x i8> %a0, <64 x i8> %a1, i64 %m
 }
 
 define <16 x i32> @stack_fold_permd(<16 x i32> %a0, <16 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_permd
-  ;CHECK:   vpermd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_permd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1
+; CHECK-NEXT:    vpsubd %zmm1, %zmm0, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x i32> @llvm.x86.avx512.permvar.si.512(<16 x i32> %a1, <16 x i32> %a0)
   ; add forces execution domain
@@ -1222,40 +2219,73 @@ define <16 x i32> @stack_fold_permd(<16 x i32> %a0, <16 x i32> %a1) {
 declare <16 x i32> @llvm.x86.avx512.permvar.si.512(<16 x i32>, <16 x i32>) nounwind readonly
 
 define <64 x i8> @stack_fold_vpermi2b(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2) {
-  ;CHECK-LABEL: stack_fold_vpermi2b
-  ;CHECK:       vpermi2b {{-?[0-9]*}}(%rsp), %zmm1, %zmm0 # 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpermi2b:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermi2b {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <64 x i8> @llvm.x86.avx512.vpermi2var.qi.512(<64 x i8> %x1, <64 x i8> %x0, <64 x i8> %x2)
   ret <64 x i8> %2
 }
 
 define <16 x i32> @stack_fold_vpermi2d(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2) {
-  ;CHECK-LABEL: stack_fold_vpermi2d
-  ;CHECK:       vpermi2d {{-?[0-9]*}}(%rsp), %zmm1, %zmm0 # 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpermi2d:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermi2d {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> %x1, <16 x i32> %x0, <16 x i32> %x2)
   ret <16 x i32> %2
 }
 
 define <8 x i64> @stack_fold_vpermi2q(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2) {
-  ;CHECK-LABEL: stack_fold_vpermi2q
-  ;CHECK:       vpermi2q {{-?[0-9]*}}(%rsp), %zmm1, %zmm0 # 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpermi2q:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermi2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64> %x1, <8 x i64> %x0, <8 x i64> %x2)
   ret <8 x i64> %2
 }
 
 define <32 x i16> @stack_fold_vpermi2w(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2) {
-  ;CHECK-LABEL: stack_fold_vpermi2w
-  ;CHECK:       vpermi2w {{-?[0-9]*}}(%rsp), %zmm1, %zmm0 # 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpermi2w:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermi2w {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <32 x i16> @llvm.x86.avx512.vpermi2var.hi.512(<32 x i16> %x1, <32 x i16> %x0, <32 x i16> %x2)
   ret <32 x i16> %2
 }
 
 define <8 x i64> @stack_fold_permq(<8 x i64> %a0) {
-  ;CHECK-LABEL: stack_fold_permq
-  ;CHECK:   vpermq $235, {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_permq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermq $235, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    # zmm0 = mem[3,2,2,3,7,6,6,7]
+; CHECK-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1
+; CHECK-NEXT:    vpsubq %zmm1, %zmm0, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <8 x i64> %a0, <8 x i64> undef, <8 x i32> <i32 3, i32 2, i32 2, i32 3, i32 7, i32 6, i32 6, i32 7>
   ; add forces execution domain
@@ -1264,8 +2294,19 @@ define <8 x i64> @stack_fold_permq(<8 x i64> %a0) {
 }
 
 define <8 x i64> @stack_fold_permq_mask(<8 x i64>* %passthru, <8 x i64> %a0, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_permq_mask
-  ;CHECK:   vpermq $235, {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_permq_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm0
+; CHECK-NEXT:    vpermq $235, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    # zmm0 {%k1} = mem[3,2,2,3,7,6,6,7]
+; CHECK-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1
+; CHECK-NEXT:    vpsubq %zmm1, %zmm0, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <8 x i64> %a0, <8 x i64> undef, <8 x i32> <i32 3, i32 2, i32 2, i32 3, i32 7, i32 6, i32 6, i32 7>
   %3 = bitcast i8 %mask to <8 x i1>
@@ -1278,8 +2319,16 @@ define <8 x i64> @stack_fold_permq_mask(<8 x i64>* %passthru, <8 x i64> %a0, i8
 }
 
 define <8 x i64> @stack_fold_permq_maskz(<8 x i64>* %passthru, <8 x i64> %a0, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_permq_maskz
-  ;CHECK:   vpermq $235, {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_permq_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vpermq $235, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    # zmm0 {%k1} {z} = mem[3,2,2,3,7,6,6,7]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <8 x i64> %a0, <8 x i64> undef, <8 x i32> <i32 3, i32 2, i32 2, i32 3, i32 7, i32 6, i32 6, i32 7>
   %3 = bitcast i8 %mask to <8 x i1>
@@ -1288,8 +2337,16 @@ define <8 x i64> @stack_fold_permq_maskz(<8 x i64>* %passthru, <8 x i64> %a0, i8
 }
 
 define <8 x i64> @stack_fold_permqvar(<8 x i64> %a0, <8 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_permqvar
-  ;CHECK:   vpermq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_permqvar:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1
+; CHECK-NEXT:    vpsubq %zmm1, %zmm0, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x i64> @llvm.x86.avx512.permvar.di.512(<8 x i64> %a1, <8 x i64> %a0)
   ; add forces execution domain
@@ -1299,8 +2356,18 @@ define <8 x i64> @stack_fold_permqvar(<8 x i64> %a0, <8 x i64> %a1) {
 declare <8 x i64> @llvm.x86.avx512.permvar.di.512(<8 x i64>, <8 x i64>) nounwind readonly
 
 define <8 x i64> @stack_fold_permqvar_mask(<8 x i64>* %passthru, <8 x i64> %a0, <8 x i64> %a1, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_permqvar_mask
-  ;CHECK:   vpermq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_permqvar_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm1
+; CHECK-NEXT:    vpermq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm1 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0
+; CHECK-NEXT:    vpsubq %zmm0, %zmm1, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x i64> @llvm.x86.avx512.permvar.di.512(<8 x i64> %a1, <8 x i64> %a0)
   %3 = bitcast i8 %mask to <8 x i1>
@@ -1313,8 +2380,14 @@ define <8 x i64> @stack_fold_permqvar_mask(<8 x i64>* %passthru, <8 x i64> %a0,
 }
 
 define <64 x i8> @stack_fold_vpermt2b(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2) {
-  ;CHECK-LABEL: stack_fold_vpermt2b
-  ;CHECK:       vpermt2b {{-?[0-9]*}}(%rsp), %zmm1, %zmm0 # 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpermt2b:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermt2b {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <64 x i8> @llvm.x86.avx512.vpermi2var.qi.512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2)
   ret <64 x i8> %2
@@ -1322,8 +2395,14 @@ define <64 x i8> @stack_fold_vpermt2b(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x
 declare <64 x i8> @llvm.x86.avx512.vpermi2var.qi.512(<64 x i8>, <64 x i8>, <64 x i8>)
 
 define <16 x i32> @stack_fold_vpermt2d(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2) {
-  ;CHECK-LABEL: stack_fold_vpermt2d
-  ;CHECK:       vpermt2d {{-?[0-9]*}}(%rsp), %zmm1, %zmm0 # 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpermt2d:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermt2d {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2)
   ret <16 x i32> %2
@@ -1331,8 +2410,14 @@ define <16 x i32> @stack_fold_vpermt2d(<16 x i32> %x0, <16 x i32> %x1, <16 x i32
 declare <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32>, <16 x i32>, <16 x i32>)
 
 define <8 x i64> @stack_fold_vpermt2q(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2) {
-  ;CHECK-LABEL: stack_fold_vpermt2q
-  ;CHECK:       vpermt2q {{-?[0-9]*}}(%rsp), %zmm1, %zmm0 # 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpermt2q:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2)
   ret <8 x i64> %2
@@ -1340,8 +2425,14 @@ define <8 x i64> @stack_fold_vpermt2q(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x
 declare <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64>, <8 x i64>, <8 x i64>)
 
 define <32 x i16> @stack_fold_vpermt2w(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2) {
-  ;CHECK-LABEL: stack_fold_vpermt2w
-  ;CHECK:       vpermt2w {{-?[0-9]*}}(%rsp), %zmm1, %zmm0 # 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpermt2w:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermt2w {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <32 x i16> @llvm.x86.avx512.vpermi2var.hi.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2)
   ret <32 x i16> %2
@@ -1349,8 +2440,14 @@ define <32 x i16> @stack_fold_vpermt2w(<32 x i16> %x0, <32 x i16> %x1, <32 x i16
 declare <32 x i16> @llvm.x86.avx512.vpermi2var.hi.512(<32 x i16>, <32 x i16>, <32 x i16>)
 
 define <32 x i16> @stack_fold_permwvar(<32 x i16> %a0, <32 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_permwvar
-  ;CHECK:   vpermw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_permwvar:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <32 x i16> @llvm.x86.avx512.permvar.hi.512(<32 x i16> %a1, <32 x i16> %a0)
   ret <32 x i16> %2
@@ -1358,8 +2455,17 @@ define <32 x i16> @stack_fold_permwvar(<32 x i16> %a0, <32 x i16> %a1) {
 declare <32 x i16> @llvm.x86.avx512.permvar.hi.512(<32 x i16>, <32 x i16>) nounwind readonly
 
 define <32 x i16> @stack_fold_permwvar_mask(<32 x i16>* %passthru, <32 x i16> %a0, <32 x i16> %a1, i32 %mask) {
-  ;CHECK-LABEL: stack_fold_permwvar_mask
-  ;CHECK:   vpermw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_permwvar_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm2
+; CHECK-NEXT:    vpermw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <32 x i16> @llvm.x86.avx512.permvar.hi.512(<32 x i16> %a1, <32 x i16> %a0)
   %3 = bitcast i32 %mask to <32 x i1>
@@ -1370,8 +2476,15 @@ define <32 x i16> @stack_fold_permwvar_mask(<32 x i16>* %passthru, <32 x i16> %a
 }
 
 define <32 x i16> @stack_fold_permwvar_maskz(<32 x i16> %a0, <32 x i16> %a1, i32 %mask) {
-  ;CHECK-LABEL: stack_fold_permwvar_maskz
-  ;CHECK:   vpermw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_permwvar_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpermw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <32 x i16> @llvm.x86.avx512.permvar.hi.512(<32 x i16> %a1, <32 x i16> %a0)
   %3 = bitcast i32 %mask to <32 x i1>
@@ -1380,9 +2493,45 @@ define <32 x i16> @stack_fold_permwvar_maskz(<32 x i16> %a0, <32 x i16> %a1, i32
 }
 
 define i32 @stack_fold_pextrd(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_pextrd
-  ;CHECK:       vpextrd $1, {{%xmm[0-9][0-9]*}}, {{-?[0-9]*}}(%rsp) {{.*#+}} 4-byte Folded Spill
-  ;CHECK:       movl    {{-?[0-9]*}}(%rsp), %eax {{.*#+}} 4-byte Reload
+; CHECK-LABEL: stack_fold_pextrd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    vpextrd $1, %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   ; add forces execution domain
   %1 = add <4 x i32> %a0, %a1
   %2 = extractelement <4 x i32> %1, i32 1
@@ -1391,65 +2540,262 @@ define i32 @stack_fold_pextrd(<4 x i32> %a0, <4 x i32> %a1) {
 }
 
 define i64 @stack_fold_pextrq(<2 x i64> %a0) {
-  ;CHECK-LABEL: stack_fold_pextrq
-  ;CHECK:       vpextrq $1, {{%xmm[0-9][0-9]*}}, {{-?[0-9]*}}(%rsp) {{.*#+}} 8-byte Folded Spill
-  ;CHECK:       movq    {{-?[0-9]*}}(%rsp), %rax {{.*#+}} 8-byte Reload
+; CHECK-LABEL: stack_fold_pextrq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    vpextrq $1, %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = extractelement <2 x i64> %a0, i32 1
   %2 = tail call <2 x i64> asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   ret i64 %1
 }
 
 define <16 x i8> @stack_fold_pinsrb(<16 x i8> %a0, i8 %a1) {
-  ;CHECK-LABEL: stack_fold_pinsrb
-  ;CHECK:       vpinsrb $1, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_pinsrb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpinsrb $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 4-byte Folded Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = insertelement <16 x i8> %a0, i8 %a1, i32 1
   ret <16 x i8> %2
 }
 
 define <4 x i32> @stack_fold_pinsrd(<4 x i32> %a0, i32 %a1) {
-  ;CHECK-LABEL: stack_fold_pinsrd
-  ;CHECK:       vpinsrd $1, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_pinsrd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpinsrd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 4-byte Folded Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = insertelement <4 x i32> %a0, i32 %a1, i32 1
   ret <4 x i32> %2
 }
 
 define <2 x i64> @stack_fold_pinsrq(<2 x i64> %a0, i64 %a1) {
-  ;CHECK-LABEL: stack_fold_pinsrq
-  ;CHECK:       vpinsrq $1, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_pinsrq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpinsrq $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 8-byte Folded Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = insertelement <2 x i64> %a0, i64 %a1, i32 1
   ret <2 x i64> %2
 }
 
 define <8 x i16> @stack_fold_pinsrw(<8 x i16> %a0, i16 %a1) {
-  ;CHECK-LABEL: stack_fold_pinsrw
-  ;CHECK:       vpinsrw $1, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_pinsrw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpinsrw $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 4-byte Folded Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = insertelement <8 x i16> %a0, i16 %a1, i32 1
   ret <8 x i16> %2
 }
 
 define <16 x i32> @stack_fold_vplzcntd(<16 x i32> %a0) {
-  ;CHECK-LABEL: stack_fold_vplzcntd
-  ;CHECK:       vplzcntd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_vplzcntd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vplzcntd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x i32> @llvm.ctlz.v16i32(<16 x i32> %a0, i1 false)
   ret <16 x i32> %2
 }
 
 define <8 x i64> @stack_fold_vplzcntq(<8 x i64> %a0) {
-  ;CHECK-LABEL: stack_fold_vplzcntq
-  ;CHECK:       vplzcntq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_vplzcntq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vplzcntq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x i64> @llvm.ctlz.v8i64(<8 x i64> %a0, i1 false)
   ret <8 x i64> %2
 }
 
 define <32 x i16> @stack_fold_pmaddubsw_zmm(<64 x i8> %a0, <64 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pmaddubsw_zmm
-  ;CHECK:       vpmaddubsw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaddubsw_zmm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmaddubsw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <32 x i16> @llvm.x86.avx512.pmaddubs.w.512(<64 x i8> %a0, <64 x i8> %a1)
   ret <32 x i16> %2
@@ -1457,8 +2803,17 @@ define <32 x i16> @stack_fold_pmaddubsw_zmm(<64 x i8> %a0, <64 x i8> %a1) {
 declare <32 x i16> @llvm.x86.avx512.pmaddubs.w.512(<64 x i8>, <64 x i8>) nounwind readnone
 
 define <32 x i16> @stack_fold_pmaddubsw_zmm_mask(<32 x i16>* %passthru, <64 x i8> %a0, <64 x i8> %a1, i32 %mask) {
-  ;CHECK-LABEL: stack_fold_pmaddubsw_zmm_mask
-  ;CHECK:       vpmaddubsw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaddubsw_zmm_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm2
+; CHECK-NEXT:    vpmaddubsw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <32 x i16> @llvm.x86.avx512.pmaddubs.w.512(<64 x i8> %a0, <64 x i8> %a1)
   %3 = bitcast i32 %mask to <32 x i1>
@@ -1469,8 +2824,15 @@ define <32 x i16> @stack_fold_pmaddubsw_zmm_mask(<32 x i16>* %passthru, <64 x i8
 }
 
 define <32 x i16> @stack_fold_pmaddubsw_zmm_maskz(<64 x i8> %a0, <64 x i8> %a1, i32 %mask) {
-  ;CHECK-LABEL: stack_fold_pmaddubsw_zmm_maskz
-  ;CHECK:       vpmaddubsw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaddubsw_zmm_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpmaddubsw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <32 x i16> @llvm.x86.avx512.pmaddubs.w.512(<64 x i8> %a0, <64 x i8> %a1)
   %3 = bitcast i32 %mask to <32 x i1>
@@ -1479,8 +2841,14 @@ define <32 x i16> @stack_fold_pmaddubsw_zmm_maskz(<64 x i8> %a0, <64 x i8> %a1,
 }
 
 define <16 x i32> @stack_fold_pmaddwd_zmm(<32 x i16> %a0, <32 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_pmaddwd_zmm
-  ;CHECK:       vpmaddwd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaddwd_zmm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmaddwd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x i32> @llvm.x86.avx512.pmaddw.d.512(<32 x i16> %a0, <32 x i16> %a1)
   ret <16 x i32> %2
@@ -1488,16 +2856,31 @@ define <16 x i32> @stack_fold_pmaddwd_zmm(<32 x i16> %a0, <32 x i16> %a1) {
 declare <16 x i32> @llvm.x86.avx512.pmaddw.d.512(<32 x i16>, <32 x i16>) nounwind readnone
 
 define <16 x i32> @stack_fold_pmaddwd_zmm_commuted(<32 x i16> %a0, <32 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_pmaddwd_zmm_commuted:
-  ;CHECK:       vpmaddwd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaddwd_zmm_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmaddwd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x i32> @llvm.x86.avx512.pmaddw.d.512(<32 x i16> %a1, <32 x i16> %a0)
   ret <16 x i32> %2
 }
 
 define <16 x i32> @stack_fold_pmaddwd_zmm_mask(<16 x i32>* %passthru, <32 x i16> %a0, <32 x i16> %a1, i16 %mask) {
-  ;CHECK-LABEL: stack_fold_pmaddwd_zmm_mask
-  ;CHECK:       vpmaddwd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaddwd_zmm_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm2
+; CHECK-NEXT:    vpmaddwd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x i32> @llvm.x86.avx512.pmaddw.d.512(<32 x i16> %a0, <32 x i16> %a1)
   %3 = bitcast i16 %mask to <16 x i1>
@@ -1508,8 +2891,17 @@ define <16 x i32> @stack_fold_pmaddwd_zmm_mask(<16 x i32>* %passthru, <32 x i16>
 }
 
 define <16 x i32> @stack_fold_pmaddwd_zmm_mask_commuted(<16 x i32>* %passthru, <32 x i16> %a0, <32 x i16> %a1, i16 %mask) {
-  ;CHECK-LABEL: stack_fold_pmaddwd_zmm_mask_commuted:
-  ;CHECK:       vpmaddwd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaddwd_zmm_mask_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm2
+; CHECK-NEXT:    vpmaddwd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x i32> @llvm.x86.avx512.pmaddw.d.512(<32 x i16> %a1, <32 x i16> %a0)
   %3 = bitcast i16 %mask to <16 x i1>
@@ -1520,8 +2912,15 @@ define <16 x i32> @stack_fold_pmaddwd_zmm_mask_commuted(<16 x i32>* %passthru, <
 }
 
 define <16 x i32> @stack_fold_pmaddwd_zmm_maskz(<16 x i32>* %passthru, <32 x i16> %a0, <32 x i16> %a1, i16 %mask) {
-  ;CHECK-LABEL: stack_fold_pmaddwd_zmm_maskz
-  ;CHECK:       vpmaddwd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaddwd_zmm_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vpmaddwd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x i32> @llvm.x86.avx512.pmaddw.d.512(<32 x i16> %a0, <32 x i16> %a1)
   %3 = bitcast i16 %mask to <16 x i1>
@@ -1530,8 +2929,15 @@ define <16 x i32> @stack_fold_pmaddwd_zmm_maskz(<16 x i32>* %passthru, <32 x i16
 }
 
 define <16 x i32> @stack_fold_pmaddwd_zmm_maskz_commuted(<16 x i32>* %passthru, <32 x i16> %a0, <32 x i16> %a1, i16 %mask) {
-  ;CHECK-LABEL: stack_fold_pmaddwd_zmm_maskz_commuted:
-  ;CHECK:       vpmaddwd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaddwd_zmm_maskz_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vpmaddwd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x i32> @llvm.x86.avx512.pmaddw.d.512(<32 x i16> %a1, <32 x i16> %a0)
   %3 = bitcast i16 %mask to <16 x i1>
@@ -1540,8 +2946,14 @@ define <16 x i32> @stack_fold_pmaddwd_zmm_maskz_commuted(<16 x i32>* %passthru,
 }
 
 define <64 x i8> @stack_fold_pmaxsb(<64 x i8> %a0, <64 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pmaxsb
-  ;CHECK:       vpmaxsb {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxsb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmaxsb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp sgt <64 x i8> %a0, %a1
   %3 = select <64 x i1> %2, <64 x i8> %a0, <64 x i8> %a1
@@ -1549,8 +2961,14 @@ define <64 x i8> @stack_fold_pmaxsb(<64 x i8> %a0, <64 x i8> %a1) {
 }
 
 define <64 x i8> @stack_fold_pmaxsb_commuted(<64 x i8> %a0, <64 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pmaxsb_commuted:
-  ;CHECK:       vpmaxsb {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxsb_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmaxsb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp sgt <64 x i8> %a1, %a0
   %3 = select <64 x i1> %2, <64 x i8> %a1, <64 x i8> %a0
@@ -1558,8 +2976,17 @@ define <64 x i8> @stack_fold_pmaxsb_commuted(<64 x i8> %a0, <64 x i8> %a1) {
 }
 
 define <64 x i8> @stack_fold_pmaxsb_mask(<64 x i8> %a0, <64 x i8> %a1, i64 %mask, <64 x i8>* %passthru) {
-  ;CHECK-LABEL: stack_fold_pmaxsb_mask:
-  ;CHECK:       vpmaxsb {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxsb_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovq %rdi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm2
+; CHECK-NEXT:    vpmaxsb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp sgt <64 x i8> %a0, %a1
   %3 = select <64 x i1> %2, <64 x i8> %a0, <64 x i8> %a1
@@ -1571,8 +2998,17 @@ define <64 x i8> @stack_fold_pmaxsb_mask(<64 x i8> %a0, <64 x i8> %a1, i64 %mask
 }
 
 define <64 x i8> @stack_fold_pmaxsb_mask_commuted(<64 x i8> %a0, <64 x i8> %a1, i64 %mask, <64 x i8>* %passthru) {
-  ;CHECK-LABEL: stack_fold_pmaxsb_mask_commuted:
-  ;CHECK:       vpmaxsb {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxsb_mask_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovq %rdi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm2
+; CHECK-NEXT:    vpmaxsb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp sgt <64 x i8> %a1, %a0
   %3 = select <64 x i1> %2, <64 x i8> %a1, <64 x i8> %a0
@@ -1584,8 +3020,15 @@ define <64 x i8> @stack_fold_pmaxsb_mask_commuted(<64 x i8> %a0, <64 x i8> %a1,
 }
 
 define <64 x i8> @stack_fold_pmaxsb_maskz(<64 x i8> %a0, <64 x i8> %a1, i64 %mask) {
-  ;CHECK-LABEL: stack_fold_pmaxsb_maskz:
-  ;CHECK:       vpmaxsb {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxsb_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovq %rdi, %k1
+; CHECK-NEXT:    vpmaxsb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp sgt <64 x i8> %a0, %a1
   %3 = select <64 x i1> %2, <64 x i8> %a0, <64 x i8> %a1
@@ -1595,8 +3038,15 @@ define <64 x i8> @stack_fold_pmaxsb_maskz(<64 x i8> %a0, <64 x i8> %a1, i64 %mas
 }
 
 define <64 x i8> @stack_fold_pmaxsb_maskz_commuted(<64 x i8> %a0, <64 x i8> %a1, i64 %mask) {
-  ;CHECK-LABEL: stack_fold_pmaxsb_maskz_commuted:
-  ;CHECK:       vpmaxsb {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxsb_maskz_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovq %rdi, %k1
+; CHECK-NEXT:    vpmaxsb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp sgt <64 x i8> %a1, %a0
   %3 = select <64 x i1> %2, <64 x i8> %a1, <64 x i8> %a0
@@ -1606,8 +3056,14 @@ define <64 x i8> @stack_fold_pmaxsb_maskz_commuted(<64 x i8> %a0, <64 x i8> %a1,
 }
 
 define <16 x i32> @stack_fold_pmaxsd(<16 x i32> %a0, <16 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_pmaxsd
-  ;CHECK:       vpmaxsd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxsd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmaxsd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp sgt <16 x i32> %a0, %a1
   %3 = select <16 x i1> %2, <16 x i32> %a0, <16 x i32> %a1
@@ -1615,8 +3071,14 @@ define <16 x i32> @stack_fold_pmaxsd(<16 x i32> %a0, <16 x i32> %a1) {
 }
 
 define <16 x i32> @stack_fold_pmaxsd_commuted(<16 x i32> %a0, <16 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_pmaxsd_commuted:
-  ;CHECK:       vpmaxsd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxsd_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmaxsd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp sgt <16 x i32> %a1, %a0
   %3 = select <16 x i1> %2, <16 x i32> %a1, <16 x i32> %a0
@@ -1624,8 +3086,17 @@ define <16 x i32> @stack_fold_pmaxsd_commuted(<16 x i32> %a0, <16 x i32> %a1) {
 }
 
 define <16 x i32> @stack_fold_pmaxsd_mask(<16 x i32> %a0, <16 x i32> %a1, i16 %mask, <16 x i32>* %passthru) {
-  ;CHECK-LABEL: stack_fold_pmaxsd_mask:
-  ;CHECK:       vpmaxsd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxsd_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm2
+; CHECK-NEXT:    vpmaxsd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp sgt <16 x i32> %a0, %a1
   %3 = select <16 x i1> %2, <16 x i32> %a0, <16 x i32> %a1
@@ -1637,8 +3108,17 @@ define <16 x i32> @stack_fold_pmaxsd_mask(<16 x i32> %a0, <16 x i32> %a1, i16 %m
 }
 
 define <16 x i32> @stack_fold_pmaxsd_mask_commuted(<16 x i32> %a0, <16 x i32> %a1, i16 %mask, <16 x i32>* %passthru) {
-  ;CHECK-LABEL: stack_fold_pmaxsd_mask_commuted:
-  ;CHECK:       vpmaxsd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxsd_mask_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm2
+; CHECK-NEXT:    vpmaxsd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp sgt <16 x i32> %a1, %a0
   %3 = select <16 x i1> %2, <16 x i32> %a1, <16 x i32> %a0
@@ -1650,8 +3130,15 @@ define <16 x i32> @stack_fold_pmaxsd_mask_commuted(<16 x i32> %a0, <16 x i32> %a
 }
 
 define <16 x i32> @stack_fold_pmaxsd_maskz(<16 x i32> %a0, <16 x i32> %a1, i16 %mask) {
-  ;CHECK-LABEL: stack_fold_pmaxsd_maskz:
-  ;CHECK:       vpmaxsd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxsd_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpmaxsd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp sgt <16 x i32> %a0, %a1
   %3 = select <16 x i1> %2, <16 x i32> %a0, <16 x i32> %a1
@@ -1661,8 +3148,15 @@ define <16 x i32> @stack_fold_pmaxsd_maskz(<16 x i32> %a0, <16 x i32> %a1, i16 %
 }
 
 define <16 x i32> @stack_fold_pmaxsd_maskz_commuted(<16 x i32> %a0, <16 x i32> %a1, i16 %mask) {
-  ;CHECK-LABEL: stack_fold_pmaxsd_maskz_commuted:
-  ;CHECK:       vpmaxsd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxsd_maskz_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpmaxsd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp sgt <16 x i32> %a1, %a0
   %3 = select <16 x i1> %2, <16 x i32> %a1, <16 x i32> %a0
@@ -1672,8 +3166,14 @@ define <16 x i32> @stack_fold_pmaxsd_maskz_commuted(<16 x i32> %a0, <16 x i32> %
 }
 
 define <8 x i64> @stack_fold_pmaxsq(<8 x i64> %a0, <8 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_pmaxsq
-  ;CHECK:       vpmaxsq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxsq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmaxsq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp sgt <8 x i64> %a0, %a1
   %3 = select <8 x i1> %2, <8 x i64> %a0, <8 x i64> %a1
@@ -1681,8 +3181,14 @@ define <8 x i64> @stack_fold_pmaxsq(<8 x i64> %a0, <8 x i64> %a1) {
 }
 
 define <8 x i64> @stack_fold_pmaxsq_commuted(<8 x i64> %a0, <8 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_pmaxsq_commuted:
-  ;CHECK:       vpmaxsq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxsq_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmaxsq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp sgt <8 x i64> %a1, %a0
   %3 = select <8 x i1> %2, <8 x i64> %a1, <8 x i64> %a0
@@ -1690,8 +3196,17 @@ define <8 x i64> @stack_fold_pmaxsq_commuted(<8 x i64> %a0, <8 x i64> %a1) {
 }
 
 define <8 x i64> @stack_fold_pmaxsq_mask(<8 x i64> %a0, <8 x i64> %a1, i8 %mask, <8 x i64>* %passthru) {
-  ;CHECK-LABEL: stack_fold_pmaxsq_mask:
-  ;CHECK:       vpmaxsq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxsq_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm2
+; CHECK-NEXT:    vpmaxsq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp sgt <8 x i64> %a0, %a1
   %3 = select <8 x i1> %2, <8 x i64> %a0, <8 x i64> %a1
@@ -1703,8 +3218,17 @@ define <8 x i64> @stack_fold_pmaxsq_mask(<8 x i64> %a0, <8 x i64> %a1, i8 %mask,
 }
 
 define <8 x i64> @stack_fold_pmaxsq_mask_commuted(<8 x i64> %a0, <8 x i64> %a1, i8 %mask, <8 x i64>* %passthru) {
-  ;CHECK-LABEL: stack_fold_pmaxsq_mask_commuted:
-  ;CHECK:       vpmaxsq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxsq_mask_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm2
+; CHECK-NEXT:    vpmaxsq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp sgt <8 x i64> %a1, %a0
   %3 = select <8 x i1> %2, <8 x i64> %a1, <8 x i64> %a0
@@ -1716,8 +3240,15 @@ define <8 x i64> @stack_fold_pmaxsq_mask_commuted(<8 x i64> %a0, <8 x i64> %a1,
 }
 
 define <8 x i64> @stack_fold_pmaxsq_maskz(<8 x i64> %a0, <8 x i64> %a1, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_pmaxsq_maskz:
-  ;CHECK:       vpmaxsq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxsq_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpmaxsq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp sgt <8 x i64> %a0, %a1
   %3 = select <8 x i1> %2, <8 x i64> %a0, <8 x i64> %a1
@@ -1727,8 +3258,15 @@ define <8 x i64> @stack_fold_pmaxsq_maskz(<8 x i64> %a0, <8 x i64> %a1, i8 %mask
 }
 
 define <8 x i64> @stack_fold_pmaxsq_maskz_commuted(<8 x i64> %a0, <8 x i64> %a1, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_pmaxsq_maskz_commuted:
-  ;CHECK:       vpmaxsq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxsq_maskz_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpmaxsq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp sgt <8 x i64> %a1, %a0
   %3 = select <8 x i1> %2, <8 x i64> %a1, <8 x i64> %a0
@@ -1738,8 +3276,14 @@ define <8 x i64> @stack_fold_pmaxsq_maskz_commuted(<8 x i64> %a0, <8 x i64> %a1,
 }
 
 define <32 x i16> @stack_fold_pmaxsw(<32 x i16> %a0, <32 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_pmaxsw
-  ;CHECK:       vpmaxsw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxsw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmaxsw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp sgt <32 x i16> %a0, %a1
   %3 = select <32 x i1> %2, <32 x i16> %a0, <32 x i16> %a1
@@ -1747,8 +3291,14 @@ define <32 x i16> @stack_fold_pmaxsw(<32 x i16> %a0, <32 x i16> %a1) {
 }
 
 define <32 x i16> @stack_fold_pmaxsw_commuted(<32 x i16> %a0, <32 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_pmaxsw_commuted:
-  ;CHECK:       vpmaxsw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxsw_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmaxsw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp sgt <32 x i16> %a1, %a0
   %3 = select <32 x i1> %2, <32 x i16> %a1, <32 x i16> %a0
@@ -1756,8 +3306,17 @@ define <32 x i16> @stack_fold_pmaxsw_commuted(<32 x i16> %a0, <32 x i16> %a1) {
 }
 
 define <32 x i16> @stack_fold_pmaxsw_mask(<32 x i16> %a0, <32 x i16> %a1, i32 %mask, <32 x i16>* %passthru) {
-  ;CHECK-LABEL: stack_fold_pmaxsw_mask:
-  ;CHECK:       vpmaxsw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxsw_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm2
+; CHECK-NEXT:    vpmaxsw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp sgt <32 x i16> %a0, %a1
   %3 = select <32 x i1> %2, <32 x i16> %a0, <32 x i16> %a1
@@ -1769,8 +3328,17 @@ define <32 x i16> @stack_fold_pmaxsw_mask(<32 x i16> %a0, <32 x i16> %a1, i32 %m
 }
 
 define <32 x i16> @stack_fold_pmaxsw_mask_commuted(<32 x i16> %a0, <32 x i16> %a1, i32 %mask, <32 x i16>* %passthru) {
-  ;CHECK-LABEL: stack_fold_pmaxsw_mask_commuted:
-  ;CHECK:       vpmaxsw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxsw_mask_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm2
+; CHECK-NEXT:    vpmaxsw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp sgt <32 x i16> %a1, %a0
   %3 = select <32 x i1> %2, <32 x i16> %a1, <32 x i16> %a0
@@ -1782,8 +3350,15 @@ define <32 x i16> @stack_fold_pmaxsw_mask_commuted(<32 x i16> %a0, <32 x i16> %a
 }
 
 define <32 x i16> @stack_fold_pmaxsw_maskz(<32 x i16> %a0, <32 x i16> %a1, i32 %mask) {
-  ;CHECK-LABEL: stack_fold_pmaxsw_maskz:
-  ;CHECK:       vpmaxsw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxsw_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpmaxsw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp sgt <32 x i16> %a0, %a1
   %3 = select <32 x i1> %2, <32 x i16> %a0, <32 x i16> %a1
@@ -1793,8 +3368,15 @@ define <32 x i16> @stack_fold_pmaxsw_maskz(<32 x i16> %a0, <32 x i16> %a1, i32 %
 }
 
 define <32 x i16> @stack_fold_pmaxsw_maskz_commuted(<32 x i16> %a0, <32 x i16> %a1, i32 %mask) {
-  ;CHECK-LABEL: stack_fold_pmaxsw_maskz_commuted:
-  ;CHECK:       vpmaxsw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxsw_maskz_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpmaxsw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp sgt <32 x i16> %a1, %a0
   %3 = select <32 x i1> %2, <32 x i16> %a1, <32 x i16> %a0
@@ -1804,8 +3386,14 @@ define <32 x i16> @stack_fold_pmaxsw_maskz_commuted(<32 x i16> %a0, <32 x i16> %
 }
 
 define <64 x i8> @stack_fold_pmaxub(<64 x i8> %a0, <64 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pmaxub
-  ;CHECK:       vpmaxub {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxub:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmaxub {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp ugt <64 x i8> %a0, %a1
   %3 = select <64 x i1> %2, <64 x i8> %a0, <64 x i8> %a1
@@ -1813,8 +3401,14 @@ define <64 x i8> @stack_fold_pmaxub(<64 x i8> %a0, <64 x i8> %a1) {
 }
 
 define <64 x i8> @stack_fold_pmaxub_commuted(<64 x i8> %a0, <64 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pmaxub_commuted:
-  ;CHECK:       vpmaxub {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxub_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmaxub {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp ugt <64 x i8> %a1, %a0
   %3 = select <64 x i1> %2, <64 x i8> %a1, <64 x i8> %a0
@@ -1822,8 +3416,17 @@ define <64 x i8> @stack_fold_pmaxub_commuted(<64 x i8> %a0, <64 x i8> %a1) {
 }
 
 define <64 x i8> @stack_fold_pmaxub_mask(<64 x i8> %a0, <64 x i8> %a1, i64 %mask, <64 x i8>* %passthru) {
-  ;CHECK-LABEL: stack_fold_pmaxub_mask:
-  ;CHECK:       vpmaxub {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxub_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovq %rdi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm2
+; CHECK-NEXT:    vpmaxub {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp ugt <64 x i8> %a0, %a1
   %3 = select <64 x i1> %2, <64 x i8> %a0, <64 x i8> %a1
@@ -1835,8 +3438,17 @@ define <64 x i8> @stack_fold_pmaxub_mask(<64 x i8> %a0, <64 x i8> %a1, i64 %mask
 }
 
 define <64 x i8> @stack_fold_pmaxub_mask_commuted(<64 x i8> %a0, <64 x i8> %a1, i64 %mask, <64 x i8>* %passthru) {
-  ;CHECK-LABEL: stack_fold_pmaxub_mask_commuted:
-  ;CHECK:       vpmaxub {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxub_mask_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovq %rdi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm2
+; CHECK-NEXT:    vpmaxub {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp ugt <64 x i8> %a1, %a0
   %3 = select <64 x i1> %2, <64 x i8> %a1, <64 x i8> %a0
@@ -1848,8 +3460,15 @@ define <64 x i8> @stack_fold_pmaxub_mask_commuted(<64 x i8> %a0, <64 x i8> %a1,
 }
 
 define <64 x i8> @stack_fold_pmaxub_maskz(<64 x i8> %a0, <64 x i8> %a1, i64 %mask) {
-  ;CHECK-LABEL: stack_fold_pmaxub_maskz:
-  ;CHECK:       vpmaxub {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxub_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovq %rdi, %k1
+; CHECK-NEXT:    vpmaxub {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp ugt <64 x i8> %a0, %a1
   %3 = select <64 x i1> %2, <64 x i8> %a0, <64 x i8> %a1
@@ -1859,8 +3478,15 @@ define <64 x i8> @stack_fold_pmaxub_maskz(<64 x i8> %a0, <64 x i8> %a1, i64 %mas
 }
 
 define <64 x i8> @stack_fold_pmaxub_maskz_commuted(<64 x i8> %a0, <64 x i8> %a1, i64 %mask) {
-  ;CHECK-LABEL: stack_fold_pmaxub_maskz_commuted:
-  ;CHECK:       vpmaxub {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxub_maskz_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovq %rdi, %k1
+; CHECK-NEXT:    vpmaxub {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp ugt <64 x i8> %a1, %a0
   %3 = select <64 x i1> %2, <64 x i8> %a1, <64 x i8> %a0
@@ -1870,8 +3496,14 @@ define <64 x i8> @stack_fold_pmaxub_maskz_commuted(<64 x i8> %a0, <64 x i8> %a1,
 }
 
 define <16 x i32> @stack_fold_pmaxud(<16 x i32> %a0, <16 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_pmaxud
-  ;CHECK:       vpmaxud {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxud:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmaxud {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp ugt <16 x i32> %a0, %a1
   %3 = select <16 x i1> %2, <16 x i32> %a0, <16 x i32> %a1
@@ -1879,8 +3511,14 @@ define <16 x i32> @stack_fold_pmaxud(<16 x i32> %a0, <16 x i32> %a1) {
 }
 
 define <16 x i32> @stack_fold_pmaxud_commuted(<16 x i32> %a0, <16 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_pmaxud_commuted:
-  ;CHECK:       vpmaxud {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxud_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmaxud {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp ugt <16 x i32> %a1, %a0
   %3 = select <16 x i1> %2, <16 x i32> %a1, <16 x i32> %a0
@@ -1888,8 +3526,17 @@ define <16 x i32> @stack_fold_pmaxud_commuted(<16 x i32> %a0, <16 x i32> %a1) {
 }
 
 define <16 x i32> @stack_fold_pmaxud_mask(<16 x i32> %a0, <16 x i32> %a1, i16 %mask, <16 x i32>* %passthru) {
-  ;CHECK-LABEL: stack_fold_pmaxud_mask:
-  ;CHECK:       vpmaxud {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxud_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm2
+; CHECK-NEXT:    vpmaxud {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp ugt <16 x i32> %a0, %a1
   %3 = select <16 x i1> %2, <16 x i32> %a0, <16 x i32> %a1
@@ -1901,8 +3548,17 @@ define <16 x i32> @stack_fold_pmaxud_mask(<16 x i32> %a0, <16 x i32> %a1, i16 %m
 }
 
 define <16 x i32> @stack_fold_pmaxud_mask_commuted(<16 x i32> %a0, <16 x i32> %a1, i16 %mask, <16 x i32>* %passthru) {
-  ;CHECK-LABEL: stack_fold_pmaxud_mask_commuted:
-  ;CHECK:       vpmaxud {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxud_mask_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm2
+; CHECK-NEXT:    vpmaxud {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp ugt <16 x i32> %a1, %a0
   %3 = select <16 x i1> %2, <16 x i32> %a1, <16 x i32> %a0
@@ -1914,8 +3570,15 @@ define <16 x i32> @stack_fold_pmaxud_mask_commuted(<16 x i32> %a0, <16 x i32> %a
 }
 
 define <16 x i32> @stack_fold_pmaxud_maskz(<16 x i32> %a0, <16 x i32> %a1, i16 %mask) {
-  ;CHECK-LABEL: stack_fold_pmaxud_maskz:
-  ;CHECK:       vpmaxud {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxud_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpmaxud {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp ugt <16 x i32> %a0, %a1
   %3 = select <16 x i1> %2, <16 x i32> %a0, <16 x i32> %a1
@@ -1925,8 +3588,15 @@ define <16 x i32> @stack_fold_pmaxud_maskz(<16 x i32> %a0, <16 x i32> %a1, i16 %
 }
 
 define <16 x i32> @stack_fold_pmaxud_maskz_commuted(<16 x i32> %a0, <16 x i32> %a1, i16 %mask) {
-  ;CHECK-LABEL: stack_fold_pmaxud_maskz_commuted:
-  ;CHECK:       vpmaxud {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxud_maskz_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpmaxud {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp ugt <16 x i32> %a1, %a0
   %3 = select <16 x i1> %2, <16 x i32> %a1, <16 x i32> %a0
@@ -1936,8 +3606,14 @@ define <16 x i32> @stack_fold_pmaxud_maskz_commuted(<16 x i32> %a0, <16 x i32> %
 }
 
 define <8 x i64> @stack_fold_pmaxuq(<8 x i64> %a0, <8 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_pmaxuq
-  ;CHECK:       vpmaxuq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxuq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmaxuq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp ugt <8 x i64> %a0, %a1
   %3 = select <8 x i1> %2, <8 x i64> %a0, <8 x i64> %a1
@@ -1945,8 +3621,14 @@ define <8 x i64> @stack_fold_pmaxuq(<8 x i64> %a0, <8 x i64> %a1) {
 }
 
 define <8 x i64> @stack_fold_pmaxuq_commuted(<8 x i64> %a0, <8 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_pmaxuq_commuted:
-  ;CHECK:       vpmaxuq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxuq_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmaxuq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp ugt <8 x i64> %a1, %a0
   %3 = select <8 x i1> %2, <8 x i64> %a1, <8 x i64> %a0
@@ -1954,8 +3636,17 @@ define <8 x i64> @stack_fold_pmaxuq_commuted(<8 x i64> %a0, <8 x i64> %a1) {
 }
 
 define <8 x i64> @stack_fold_pmaxuq_mask(<8 x i64> %a0, <8 x i64> %a1, i8 %mask, <8 x i64>* %passthru) {
-  ;CHECK-LABEL: stack_fold_pmaxuq_mask:
-  ;CHECK:       vpmaxuq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxuq_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm2
+; CHECK-NEXT:    vpmaxuq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp ugt <8 x i64> %a0, %a1
   %3 = select <8 x i1> %2, <8 x i64> %a0, <8 x i64> %a1
@@ -1967,8 +3658,17 @@ define <8 x i64> @stack_fold_pmaxuq_mask(<8 x i64> %a0, <8 x i64> %a1, i8 %mask,
 }
 
 define <8 x i64> @stack_fold_pmaxuq_mask_commuted(<8 x i64> %a0, <8 x i64> %a1, i8 %mask, <8 x i64>* %passthru) {
-  ;CHECK-LABEL: stack_fold_pmaxuq_mask_commuted:
-  ;CHECK:       vpmaxuq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxuq_mask_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm2
+; CHECK-NEXT:    vpmaxuq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp ugt <8 x i64> %a1, %a0
   %3 = select <8 x i1> %2, <8 x i64> %a1, <8 x i64> %a0
@@ -1980,8 +3680,15 @@ define <8 x i64> @stack_fold_pmaxuq_mask_commuted(<8 x i64> %a0, <8 x i64> %a1,
 }
 
 define <8 x i64> @stack_fold_pmaxuq_maskz(<8 x i64> %a0, <8 x i64> %a1, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_pmaxuq_maskz:
-  ;CHECK:       vpmaxuq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxuq_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpmaxuq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp ugt <8 x i64> %a0, %a1
   %3 = select <8 x i1> %2, <8 x i64> %a0, <8 x i64> %a1
@@ -1991,8 +3698,15 @@ define <8 x i64> @stack_fold_pmaxuq_maskz(<8 x i64> %a0, <8 x i64> %a1, i8 %mask
 }
 
 define <8 x i64> @stack_fold_pmaxuq_maskz_commuted(<8 x i64> %a0, <8 x i64> %a1, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_pmaxuq_maskz_commuted:
-  ;CHECK:       vpmaxuq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxuq_maskz_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpmaxuq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp ugt <8 x i64> %a1, %a0
   %3 = select <8 x i1> %2, <8 x i64> %a1, <8 x i64> %a0
@@ -2002,8 +3716,14 @@ define <8 x i64> @stack_fold_pmaxuq_maskz_commuted(<8 x i64> %a0, <8 x i64> %a1,
 }
 
 define <32 x i16> @stack_fold_pmaxuw(<32 x i16> %a0, <32 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_pmaxuw
-  ;CHECK:       vpmaxuw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxuw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmaxuw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp ugt <32 x i16> %a0, %a1
   %3 = select <32 x i1> %2, <32 x i16> %a0, <32 x i16> %a1
@@ -2011,8 +3731,14 @@ define <32 x i16> @stack_fold_pmaxuw(<32 x i16> %a0, <32 x i16> %a1) {
 }
 
 define <32 x i16> @stack_fold_pmaxuw_commuted(<32 x i16> %a0, <32 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_pmaxuw_commuted:
-  ;CHECK:       vpmaxuw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxuw_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmaxuw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp ugt <32 x i16> %a1, %a0
   %3 = select <32 x i1> %2, <32 x i16> %a1, <32 x i16> %a0
@@ -2020,8 +3746,17 @@ define <32 x i16> @stack_fold_pmaxuw_commuted(<32 x i16> %a0, <32 x i16> %a1) {
 }
 
 define <32 x i16> @stack_fold_pmaxuw_mask(<32 x i16> %a0, <32 x i16> %a1, i32 %mask, <32 x i16>* %passthru) {
-  ;CHECK-LABEL: stack_fold_pmaxuw_mask:
-  ;CHECK:       vpmaxuw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxuw_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm2
+; CHECK-NEXT:    vpmaxuw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp ugt <32 x i16> %a0, %a1
   %3 = select <32 x i1> %2, <32 x i16> %a0, <32 x i16> %a1
@@ -2033,8 +3768,17 @@ define <32 x i16> @stack_fold_pmaxuw_mask(<32 x i16> %a0, <32 x i16> %a1, i32 %m
 }
 
 define <32 x i16> @stack_fold_pmaxuw_mask_commuted(<32 x i16> %a0, <32 x i16> %a1, i32 %mask, <32 x i16>* %passthru) {
-  ;CHECK-LABEL: stack_fold_pmaxuw_mask_commuted:
-  ;CHECK:       vpmaxuw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxuw_mask_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm2
+; CHECK-NEXT:    vpmaxuw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp ugt <32 x i16> %a1, %a0
   %3 = select <32 x i1> %2, <32 x i16> %a1, <32 x i16> %a0
@@ -2046,8 +3790,15 @@ define <32 x i16> @stack_fold_pmaxuw_mask_commuted(<32 x i16> %a0, <32 x i16> %a
 }
 
 define <32 x i16> @stack_fold_pmaxuw_maskz(<32 x i16> %a0, <32 x i16> %a1, i32 %mask) {
-  ;CHECK-LABEL: stack_fold_pmaxuw_maskz:
-  ;CHECK:       vpmaxuw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxuw_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpmaxuw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp ugt <32 x i16> %a0, %a1
   %3 = select <32 x i1> %2, <32 x i16> %a0, <32 x i16> %a1
@@ -2057,8 +3808,15 @@ define <32 x i16> @stack_fold_pmaxuw_maskz(<32 x i16> %a0, <32 x i16> %a1, i32 %
 }
 
 define <32 x i16> @stack_fold_pmaxuw_maskz_commuted(<32 x i16> %a0, <32 x i16> %a1, i32 %mask) {
-  ;CHECK-LABEL: stack_fold_pmaxuw_maskz_commuted:
-  ;CHECK:       vpmaxuw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxuw_maskz_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpmaxuw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp ugt <32 x i16> %a1, %a0
   %3 = select <32 x i1> %2, <32 x i16> %a1, <32 x i16> %a0
@@ -2068,8 +3826,14 @@ define <32 x i16> @stack_fold_pmaxuw_maskz_commuted(<32 x i16> %a0, <32 x i16> %
 }
 
 define <64 x i8> @stack_fold_pminsb(<64 x i8> %a0, <64 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pminsb
-  ;CHECK:       vpminsb {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminsb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpminsb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp slt <64 x i8> %a0, %a1
   %3 = select <64 x i1> %2, <64 x i8> %a0, <64 x i8> %a1
@@ -2077,8 +3841,14 @@ define <64 x i8> @stack_fold_pminsb(<64 x i8> %a0, <64 x i8> %a1) {
 }
 
 define <64 x i8> @stack_fold_pminsb_commuted(<64 x i8> %a0, <64 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pminsb_commuted:
-  ;CHECK:       vpminsb {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminsb_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpminsb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp slt <64 x i8> %a1, %a0
   %3 = select <64 x i1> %2, <64 x i8> %a1, <64 x i8> %a0
@@ -2086,8 +3856,17 @@ define <64 x i8> @stack_fold_pminsb_commuted(<64 x i8> %a0, <64 x i8> %a1) {
 }
 
 define <64 x i8> @stack_fold_pminsb_mask(<64 x i8> %a0, <64 x i8> %a1, i64 %mask, <64 x i8>* %passthru) {
-  ;CHECK-LABEL: stack_fold_pminsb_mask:
-  ;CHECK:       vpminsb {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminsb_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovq %rdi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm2
+; CHECK-NEXT:    vpminsb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp slt <64 x i8> %a0, %a1
   %3 = select <64 x i1> %2, <64 x i8> %a0, <64 x i8> %a1
@@ -2099,8 +3878,17 @@ define <64 x i8> @stack_fold_pminsb_mask(<64 x i8> %a0, <64 x i8> %a1, i64 %mask
 }
 
 define <64 x i8> @stack_fold_pminsb_mask_commuted(<64 x i8> %a0, <64 x i8> %a1, i64 %mask, <64 x i8>* %passthru) {
-  ;CHECK-LABEL: stack_fold_pminsb_mask_commuted:
-  ;CHECK:       vpminsb {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminsb_mask_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovq %rdi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm2
+; CHECK-NEXT:    vpminsb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp slt <64 x i8> %a1, %a0
   %3 = select <64 x i1> %2, <64 x i8> %a1, <64 x i8> %a0
@@ -2112,8 +3900,15 @@ define <64 x i8> @stack_fold_pminsb_mask_commuted(<64 x i8> %a0, <64 x i8> %a1,
 }
 
 define <64 x i8> @stack_fold_pminsb_maskz(<64 x i8> %a0, <64 x i8> %a1, i64 %mask) {
-  ;CHECK-LABEL: stack_fold_pminsb_maskz:
-  ;CHECK:       vpminsb {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminsb_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovq %rdi, %k1
+; CHECK-NEXT:    vpminsb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp slt <64 x i8> %a0, %a1
   %3 = select <64 x i1> %2, <64 x i8> %a0, <64 x i8> %a1
@@ -2123,8 +3918,15 @@ define <64 x i8> @stack_fold_pminsb_maskz(<64 x i8> %a0, <64 x i8> %a1, i64 %mas
 }
 
 define <64 x i8> @stack_fold_pminsb_maskz_commuted(<64 x i8> %a0, <64 x i8> %a1, i64 %mask) {
-  ;CHECK-LABEL: stack_fold_pminsb_maskz_commuted:
-  ;CHECK:       vpminsb {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminsb_maskz_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovq %rdi, %k1
+; CHECK-NEXT:    vpminsb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp slt <64 x i8> %a1, %a0
   %3 = select <64 x i1> %2, <64 x i8> %a1, <64 x i8> %a0
@@ -2134,8 +3936,14 @@ define <64 x i8> @stack_fold_pminsb_maskz_commuted(<64 x i8> %a0, <64 x i8> %a1,
 }
 
 define <16 x i32> @stack_fold_pminsd(<16 x i32> %a0, <16 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_pminsd
-  ;CHECK:       vpminsd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminsd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpminsd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp slt <16 x i32> %a0, %a1
   %3 = select <16 x i1> %2, <16 x i32> %a0, <16 x i32> %a1
@@ -2143,8 +3951,14 @@ define <16 x i32> @stack_fold_pminsd(<16 x i32> %a0, <16 x i32> %a1) {
 }
 
 define <16 x i32> @stack_fold_pminsd_commuted(<16 x i32> %a0, <16 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_pminsd_commuted:
-  ;CHECK:       vpminsd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminsd_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpminsd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp slt <16 x i32> %a1, %a0
   %3 = select <16 x i1> %2, <16 x i32> %a1, <16 x i32> %a0
@@ -2152,8 +3966,17 @@ define <16 x i32> @stack_fold_pminsd_commuted(<16 x i32> %a0, <16 x i32> %a1) {
 }
 
 define <16 x i32> @stack_fold_pminsd_mask(<16 x i32> %a0, <16 x i32> %a1, i16 %mask, <16 x i32>* %passthru) {
-  ;CHECK-LABEL: stack_fold_pminsd_mask:
-  ;CHECK:       vpminsd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminsd_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm2
+; CHECK-NEXT:    vpminsd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp slt <16 x i32> %a0, %a1
   %3 = select <16 x i1> %2, <16 x i32> %a0, <16 x i32> %a1
@@ -2165,8 +3988,17 @@ define <16 x i32> @stack_fold_pminsd_mask(<16 x i32> %a0, <16 x i32> %a1, i16 %m
 }
 
 define <16 x i32> @stack_fold_pminsd_mask_commuted(<16 x i32> %a0, <16 x i32> %a1, i16 %mask, <16 x i32>* %passthru) {
-  ;CHECK-LABEL: stack_fold_pminsd_mask_commuted:
-  ;CHECK:       vpminsd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminsd_mask_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm2
+; CHECK-NEXT:    vpminsd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp slt <16 x i32> %a1, %a0
   %3 = select <16 x i1> %2, <16 x i32> %a1, <16 x i32> %a0
@@ -2178,8 +4010,15 @@ define <16 x i32> @stack_fold_pminsd_mask_commuted(<16 x i32> %a0, <16 x i32> %a
 }
 
 define <16 x i32> @stack_fold_pminsd_maskz(<16 x i32> %a0, <16 x i32> %a1, i16 %mask) {
-  ;CHECK-LABEL: stack_fold_pminsd_maskz:
-  ;CHECK:       vpminsd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminsd_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpminsd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp slt <16 x i32> %a0, %a1
   %3 = select <16 x i1> %2, <16 x i32> %a0, <16 x i32> %a1
@@ -2189,8 +4028,15 @@ define <16 x i32> @stack_fold_pminsd_maskz(<16 x i32> %a0, <16 x i32> %a1, i16 %
 }
 
 define <16 x i32> @stack_fold_pminsd_maskz_commuted(<16 x i32> %a0, <16 x i32> %a1, i16 %mask) {
-  ;CHECK-LABEL: stack_fold_pminsd_maskz_commuted:
-  ;CHECK:       vpminsd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminsd_maskz_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpminsd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp slt <16 x i32> %a1, %a0
   %3 = select <16 x i1> %2, <16 x i32> %a1, <16 x i32> %a0
@@ -2200,8 +4046,14 @@ define <16 x i32> @stack_fold_pminsd_maskz_commuted(<16 x i32> %a0, <16 x i32> %
 }
 
 define <8 x i64> @stack_fold_pminsq(<8 x i64> %a0, <8 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_pminsq
-  ;CHECK:       vpminsq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminsq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpminsq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp slt <8 x i64> %a0, %a1
   %3 = select <8 x i1> %2, <8 x i64> %a0, <8 x i64> %a1
@@ -2209,8 +4061,14 @@ define <8 x i64> @stack_fold_pminsq(<8 x i64> %a0, <8 x i64> %a1) {
 }
 
 define <8 x i64> @stack_fold_pminsq_commuted(<8 x i64> %a0, <8 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_pminsq_commuted:
-  ;CHECK:       vpminsq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminsq_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpminsq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp slt <8 x i64> %a1, %a0
   %3 = select <8 x i1> %2, <8 x i64> %a1, <8 x i64> %a0
@@ -2218,8 +4076,17 @@ define <8 x i64> @stack_fold_pminsq_commuted(<8 x i64> %a0, <8 x i64> %a1) {
 }
 
 define <8 x i64> @stack_fold_pminsq_mask(<8 x i64> %a0, <8 x i64> %a1, i8 %mask, <8 x i64>* %passthru) {
-  ;CHECK-LABEL: stack_fold_pminsq_mask:
-  ;CHECK:       vpminsq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminsq_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm2
+; CHECK-NEXT:    vpminsq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp slt <8 x i64> %a0, %a1
   %3 = select <8 x i1> %2, <8 x i64> %a0, <8 x i64> %a1
@@ -2231,8 +4098,17 @@ define <8 x i64> @stack_fold_pminsq_mask(<8 x i64> %a0, <8 x i64> %a1, i8 %mask,
 }
 
 define <8 x i64> @stack_fold_pminsq_mask_commuted(<8 x i64> %a0, <8 x i64> %a1, i8 %mask, <8 x i64>* %passthru) {
-  ;CHECK-LABEL: stack_fold_pminsq_mask_commuted:
-  ;CHECK:       vpminsq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminsq_mask_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm2
+; CHECK-NEXT:    vpminsq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp slt <8 x i64> %a1, %a0
   %3 = select <8 x i1> %2, <8 x i64> %a1, <8 x i64> %a0
@@ -2244,8 +4120,15 @@ define <8 x i64> @stack_fold_pminsq_mask_commuted(<8 x i64> %a0, <8 x i64> %a1,
 }
 
 define <8 x i64> @stack_fold_pminsq_maskz(<8 x i64> %a0, <8 x i64> %a1, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_pminsq_maskz:
-  ;CHECK:       vpminsq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminsq_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpminsq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp slt <8 x i64> %a0, %a1
   %3 = select <8 x i1> %2, <8 x i64> %a0, <8 x i64> %a1
@@ -2255,8 +4138,15 @@ define <8 x i64> @stack_fold_pminsq_maskz(<8 x i64> %a0, <8 x i64> %a1, i8 %mask
 }
 
 define <8 x i64> @stack_fold_pminsq_maskz_commuted(<8 x i64> %a0, <8 x i64> %a1, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_pminsq_maskz_commuted:
-  ;CHECK:       vpminsq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminsq_maskz_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpminsq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp slt <8 x i64> %a1, %a0
   %3 = select <8 x i1> %2, <8 x i64> %a1, <8 x i64> %a0
@@ -2266,8 +4156,14 @@ define <8 x i64> @stack_fold_pminsq_maskz_commuted(<8 x i64> %a0, <8 x i64> %a1,
 }
 
 define <32 x i16> @stack_fold_pminsw(<32 x i16> %a0, <32 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_pminsw
-  ;CHECK:       vpminsw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminsw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpminsw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp slt <32 x i16> %a0, %a1
   %3 = select <32 x i1> %2, <32 x i16> %a0, <32 x i16> %a1
@@ -2275,8 +4171,14 @@ define <32 x i16> @stack_fold_pminsw(<32 x i16> %a0, <32 x i16> %a1) {
 }
 
 define <32 x i16> @stack_fold_pminsw_commuted(<32 x i16> %a0, <32 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_pminsw_commuted:
-  ;CHECK:       vpminsw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminsw_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpminsw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp slt <32 x i16> %a1, %a0
   %3 = select <32 x i1> %2, <32 x i16> %a1, <32 x i16> %a0
@@ -2284,8 +4186,17 @@ define <32 x i16> @stack_fold_pminsw_commuted(<32 x i16> %a0, <32 x i16> %a1) {
 }
 
 define <32 x i16> @stack_fold_pminsw_mask(<32 x i16> %a0, <32 x i16> %a1, i32 %mask, <32 x i16>* %passthru) {
-  ;CHECK-LABEL: stack_fold_pminsw_mask:
-  ;CHECK:       vpminsw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminsw_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm2
+; CHECK-NEXT:    vpminsw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp slt <32 x i16> %a0, %a1
   %3 = select <32 x i1> %2, <32 x i16> %a0, <32 x i16> %a1
@@ -2297,8 +4208,17 @@ define <32 x i16> @stack_fold_pminsw_mask(<32 x i16> %a0, <32 x i16> %a1, i32 %m
 }
 
 define <32 x i16> @stack_fold_pminsw_mask_commuted(<32 x i16> %a0, <32 x i16> %a1, i32 %mask, <32 x i16>* %passthru) {
-  ;CHECK-LABEL: stack_fold_pminsw_mask_commuted:
-  ;CHECK:       vpminsw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminsw_mask_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm2
+; CHECK-NEXT:    vpminsw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp slt <32 x i16> %a1, %a0
   %3 = select <32 x i1> %2, <32 x i16> %a1, <32 x i16> %a0
@@ -2310,8 +4230,15 @@ define <32 x i16> @stack_fold_pminsw_mask_commuted(<32 x i16> %a0, <32 x i16> %a
 }
 
 define <32 x i16> @stack_fold_pminsw_maskz(<32 x i16> %a0, <32 x i16> %a1, i32 %mask) {
-  ;CHECK-LABEL: stack_fold_pminsw_maskz:
-  ;CHECK:       vpminsw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminsw_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpminsw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp slt <32 x i16> %a0, %a1
   %3 = select <32 x i1> %2, <32 x i16> %a0, <32 x i16> %a1
@@ -2321,8 +4248,15 @@ define <32 x i16> @stack_fold_pminsw_maskz(<32 x i16> %a0, <32 x i16> %a1, i32 %
 }
 
 define <32 x i16> @stack_fold_pminsw_maskz_commuted(<32 x i16> %a0, <32 x i16> %a1, i32 %mask) {
-  ;CHECK-LABEL: stack_fold_pminsw_maskz_commuted:
-  ;CHECK:       vpminsw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminsw_maskz_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpminsw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp slt <32 x i16> %a1, %a0
   %3 = select <32 x i1> %2, <32 x i16> %a1, <32 x i16> %a0
@@ -2332,8 +4266,14 @@ define <32 x i16> @stack_fold_pminsw_maskz_commuted(<32 x i16> %a0, <32 x i16> %
 }
 
 define <64 x i8> @stack_fold_pminub(<64 x i8> %a0, <64 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pminub
-  ;CHECK:       vpminub {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminub:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpminub {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp ult <64 x i8> %a0, %a1
   %3 = select <64 x i1> %2, <64 x i8> %a0, <64 x i8> %a1
@@ -2341,8 +4281,14 @@ define <64 x i8> @stack_fold_pminub(<64 x i8> %a0, <64 x i8> %a1) {
 }
 
 define <64 x i8> @stack_fold_pminub_commuted(<64 x i8> %a0, <64 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pminub_commuted:
-  ;CHECK:       vpminub {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminub_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpminub {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp ult <64 x i8> %a1, %a0
   %3 = select <64 x i1> %2, <64 x i8> %a1, <64 x i8> %a0
@@ -2350,8 +4296,17 @@ define <64 x i8> @stack_fold_pminub_commuted(<64 x i8> %a0, <64 x i8> %a1) {
 }
 
 define <64 x i8> @stack_fold_pminub_mask(<64 x i8> %a0, <64 x i8> %a1, i64 %mask, <64 x i8>* %passthru) {
-  ;CHECK-LABEL: stack_fold_pminub_mask:
-  ;CHECK:       vpminub {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminub_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovq %rdi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm2
+; CHECK-NEXT:    vpminub {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp ult <64 x i8> %a0, %a1
   %3 = select <64 x i1> %2, <64 x i8> %a0, <64 x i8> %a1
@@ -2363,8 +4318,17 @@ define <64 x i8> @stack_fold_pminub_mask(<64 x i8> %a0, <64 x i8> %a1, i64 %mask
 }
 
 define <64 x i8> @stack_fold_pminub_mask_commuted(<64 x i8> %a0, <64 x i8> %a1, i64 %mask, <64 x i8>* %passthru) {
-  ;CHECK-LABEL: stack_fold_pminub_mask_commuted:
-  ;CHECK:       vpminub {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminub_mask_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovq %rdi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm2
+; CHECK-NEXT:    vpminub {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp ult <64 x i8> %a1, %a0
   %3 = select <64 x i1> %2, <64 x i8> %a1, <64 x i8> %a0
@@ -2376,8 +4340,15 @@ define <64 x i8> @stack_fold_pminub_mask_commuted(<64 x i8> %a0, <64 x i8> %a1,
 }
 
 define <64 x i8> @stack_fold_pminub_maskz(<64 x i8> %a0, <64 x i8> %a1, i64 %mask) {
-  ;CHECK-LABEL: stack_fold_pminub_maskz:
-  ;CHECK:       vpminub {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminub_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovq %rdi, %k1
+; CHECK-NEXT:    vpminub {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp ult <64 x i8> %a0, %a1
   %3 = select <64 x i1> %2, <64 x i8> %a0, <64 x i8> %a1
@@ -2387,8 +4358,15 @@ define <64 x i8> @stack_fold_pminub_maskz(<64 x i8> %a0, <64 x i8> %a1, i64 %mas
 }
 
 define <64 x i8> @stack_fold_pminub_maskz_commuted(<64 x i8> %a0, <64 x i8> %a1, i64 %mask) {
-  ;CHECK-LABEL: stack_fold_pminub_maskz_commuted:
-  ;CHECK:       vpminub {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminub_maskz_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovq %rdi, %k1
+; CHECK-NEXT:    vpminub {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp ult <64 x i8> %a1, %a0
   %3 = select <64 x i1> %2, <64 x i8> %a1, <64 x i8> %a0
@@ -2398,8 +4376,14 @@ define <64 x i8> @stack_fold_pminub_maskz_commuted(<64 x i8> %a0, <64 x i8> %a1,
 }
 
 define <16 x i32> @stack_fold_pminud(<16 x i32> %a0, <16 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_pminud
-  ;CHECK:       vpminud {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminud:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpminud {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp ult <16 x i32> %a0, %a1
   %3 = select <16 x i1> %2, <16 x i32> %a0, <16 x i32> %a1
@@ -2407,8 +4391,14 @@ define <16 x i32> @stack_fold_pminud(<16 x i32> %a0, <16 x i32> %a1) {
 }
 
 define <16 x i32> @stack_fold_pminud_commuted(<16 x i32> %a0, <16 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_pminud_commuted:
-  ;CHECK:       vpminud {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminud_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpminud {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp ult <16 x i32> %a1, %a0
   %3 = select <16 x i1> %2, <16 x i32> %a1, <16 x i32> %a0
@@ -2416,8 +4406,17 @@ define <16 x i32> @stack_fold_pminud_commuted(<16 x i32> %a0, <16 x i32> %a1) {
 }
 
 define <16 x i32> @stack_fold_pminud_mask(<16 x i32> %a0, <16 x i32> %a1, i16 %mask, <16 x i32>* %passthru) {
-  ;CHECK-LABEL: stack_fold_pminud_mask:
-  ;CHECK:       vpminud {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminud_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm2
+; CHECK-NEXT:    vpminud {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp ult <16 x i32> %a0, %a1
   %3 = select <16 x i1> %2, <16 x i32> %a0, <16 x i32> %a1
@@ -2429,8 +4428,17 @@ define <16 x i32> @stack_fold_pminud_mask(<16 x i32> %a0, <16 x i32> %a1, i16 %m
 }
 
 define <16 x i32> @stack_fold_pminud_mask_commuted(<16 x i32> %a0, <16 x i32> %a1, i16 %mask, <16 x i32>* %passthru) {
-  ;CHECK-LABEL: stack_fold_pminud_mask_commuted:
-  ;CHECK:       vpminud {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminud_mask_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm2
+; CHECK-NEXT:    vpminud {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp ult <16 x i32> %a1, %a0
   %3 = select <16 x i1> %2, <16 x i32> %a1, <16 x i32> %a0
@@ -2442,8 +4450,15 @@ define <16 x i32> @stack_fold_pminud_mask_commuted(<16 x i32> %a0, <16 x i32> %a
 }
 
 define <16 x i32> @stack_fold_pminud_maskz(<16 x i32> %a0, <16 x i32> %a1, i16 %mask) {
-  ;CHECK-LABEL: stack_fold_pminud_maskz:
-  ;CHECK:       vpminud {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminud_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpminud {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp ult <16 x i32> %a0, %a1
   %3 = select <16 x i1> %2, <16 x i32> %a0, <16 x i32> %a1
@@ -2453,8 +4468,15 @@ define <16 x i32> @stack_fold_pminud_maskz(<16 x i32> %a0, <16 x i32> %a1, i16 %
 }
 
 define <16 x i32> @stack_fold_pminud_maskz_commuted(<16 x i32> %a0, <16 x i32> %a1, i16 %mask) {
-  ;CHECK-LABEL: stack_fold_pminud_maskz_commuted:
-  ;CHECK:       vpminud {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminud_maskz_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpminud {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp ult <16 x i32> %a1, %a0
   %3 = select <16 x i1> %2, <16 x i32> %a1, <16 x i32> %a0
@@ -2464,8 +4486,14 @@ define <16 x i32> @stack_fold_pminud_maskz_commuted(<16 x i32> %a0, <16 x i32> %
 }
 
 define <8 x i64> @stack_fold_pminuq(<8 x i64> %a0, <8 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_pminuq
-  ;CHECK:       vpminuq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminuq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpminuq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp ult <8 x i64> %a0, %a1
   %3 = select <8 x i1> %2, <8 x i64> %a0, <8 x i64> %a1
@@ -2473,8 +4501,14 @@ define <8 x i64> @stack_fold_pminuq(<8 x i64> %a0, <8 x i64> %a1) {
 }
 
 define <8 x i64> @stack_fold_pminuq_commuted(<8 x i64> %a0, <8 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_pminuq_commuted:
-  ;CHECK:       vpminuq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminuq_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpminuq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp ult <8 x i64> %a1, %a0
   %3 = select <8 x i1> %2, <8 x i64> %a1, <8 x i64> %a0
@@ -2482,8 +4516,17 @@ define <8 x i64> @stack_fold_pminuq_commuted(<8 x i64> %a0, <8 x i64> %a1) {
 }
 
 define <8 x i64> @stack_fold_pminuq_mask(<8 x i64> %a0, <8 x i64> %a1, i8 %mask, <8 x i64>* %passthru) {
-  ;CHECK-LABEL: stack_fold_pminuq_mask:
-  ;CHECK:       vpminuq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminuq_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm2
+; CHECK-NEXT:    vpminuq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp ult <8 x i64> %a0, %a1
   %3 = select <8 x i1> %2, <8 x i64> %a0, <8 x i64> %a1
@@ -2495,8 +4538,17 @@ define <8 x i64> @stack_fold_pminuq_mask(<8 x i64> %a0, <8 x i64> %a1, i8 %mask,
 }
 
 define <8 x i64> @stack_fold_pminuq_mask_commuted(<8 x i64> %a0, <8 x i64> %a1, i8 %mask, <8 x i64>* %passthru) {
-  ;CHECK-LABEL: stack_fold_pminuq_mask_commuted:
-  ;CHECK:       vpminuq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminuq_mask_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm2
+; CHECK-NEXT:    vpminuq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp ult <8 x i64> %a1, %a0
   %3 = select <8 x i1> %2, <8 x i64> %a1, <8 x i64> %a0
@@ -2508,8 +4560,15 @@ define <8 x i64> @stack_fold_pminuq_mask_commuted(<8 x i64> %a0, <8 x i64> %a1,
 }
 
 define <8 x i64> @stack_fold_pminuq_maskz(<8 x i64> %a0, <8 x i64> %a1, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_pminuq_maskz:
-  ;CHECK:       vpminuq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminuq_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpminuq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp ult <8 x i64> %a0, %a1
   %3 = select <8 x i1> %2, <8 x i64> %a0, <8 x i64> %a1
@@ -2519,8 +4578,15 @@ define <8 x i64> @stack_fold_pminuq_maskz(<8 x i64> %a0, <8 x i64> %a1, i8 %mask
 }
 
 define <8 x i64> @stack_fold_pminuq_maskz_commuted(<8 x i64> %a0, <8 x i64> %a1, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_pminuq_maskz_commuted:
-  ;CHECK:       vpminuq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminuq_maskz_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpminuq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp ult <8 x i64> %a1, %a0
   %3 = select <8 x i1> %2, <8 x i64> %a1, <8 x i64> %a0
@@ -2530,8 +4596,14 @@ define <8 x i64> @stack_fold_pminuq_maskz_commuted(<8 x i64> %a0, <8 x i64> %a1,
 }
 
 define <32 x i16> @stack_fold_pminuw(<32 x i16> %a0, <32 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_pminuw
-  ;CHECK:       vpminuw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminuw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpminuw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp ult <32 x i16> %a0, %a1
   %3 = select <32 x i1> %2, <32 x i16> %a0, <32 x i16> %a1
@@ -2539,8 +4611,14 @@ define <32 x i16> @stack_fold_pminuw(<32 x i16> %a0, <32 x i16> %a1) {
 }
 
 define <32 x i16> @stack_fold_pminuw_commuted(<32 x i16> %a0, <32 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_pminuw_commuted:
-  ;CHECK:       vpminuw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminuw_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpminuw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp ult <32 x i16> %a1, %a0
   %3 = select <32 x i1> %2, <32 x i16> %a1, <32 x i16> %a0
@@ -2548,8 +4626,17 @@ define <32 x i16> @stack_fold_pminuw_commuted(<32 x i16> %a0, <32 x i16> %a1) {
 }
 
 define <32 x i16> @stack_fold_pminuw_mask(<32 x i16> %a0, <32 x i16> %a1, i32 %mask, <32 x i16>* %passthru) {
-  ;CHECK-LABEL: stack_fold_pminuw_mask:
-  ;CHECK:       vpminuw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminuw_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm2
+; CHECK-NEXT:    vpminuw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp ult <32 x i16> %a0, %a1
   %3 = select <32 x i1> %2, <32 x i16> %a0, <32 x i16> %a1
@@ -2561,8 +4648,17 @@ define <32 x i16> @stack_fold_pminuw_mask(<32 x i16> %a0, <32 x i16> %a1, i32 %m
 }
 
 define <32 x i16> @stack_fold_pminuw_mask_commuted(<32 x i16> %a0, <32 x i16> %a1, i32 %mask, <32 x i16>* %passthru) {
-  ;CHECK-LABEL: stack_fold_pminuw_mask_commuted:
-  ;CHECK:       vpminuw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminuw_mask_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm2
+; CHECK-NEXT:    vpminuw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp ult <32 x i16> %a1, %a0
   %3 = select <32 x i1> %2, <32 x i16> %a1, <32 x i16> %a0
@@ -2574,8 +4670,15 @@ define <32 x i16> @stack_fold_pminuw_mask_commuted(<32 x i16> %a0, <32 x i16> %a
 }
 
 define <32 x i16> @stack_fold_pminuw_maskz(<32 x i16> %a0, <32 x i16> %a1, i32 %mask) {
-  ;CHECK-LABEL: stack_fold_pminuw_maskz:
-  ;CHECK:       vpminuw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminuw_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpminuw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp ult <32 x i16> %a0, %a1
   %3 = select <32 x i1> %2, <32 x i16> %a0, <32 x i16> %a1
@@ -2585,8 +4688,15 @@ define <32 x i16> @stack_fold_pminuw_maskz(<32 x i16> %a0, <32 x i16> %a1, i32 %
 }
 
 define <32 x i16> @stack_fold_pminuw_maskz_commuted(<32 x i16> %a0, <32 x i16> %a1, i32 %mask) {
-  ;CHECK-LABEL: stack_fold_pminuw_maskz_commuted:
-  ;CHECK:       vpminuw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminuw_maskz_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpminuw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp ult <32 x i16> %a1, %a0
   %3 = select <32 x i1> %2, <32 x i16> %a1, <32 x i16> %a0
@@ -2596,8 +4706,15 @@ define <32 x i16> @stack_fold_pminuw_maskz_commuted(<32 x i16> %a0, <32 x i16> %
 }
 
 define <16 x i8> @stack_fold_vpmovdb(<16 x i32> %a0) {
-  ;CHECK-LABEL: stack_fold_vpmovdb
-  ;CHECK:       vpmovdb %zmm0, {{-?[0-9]*}}(%rsp) # 16-byte Folded Spill
+; CHECK-LABEL: stack_fold_vpmovdb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmovdb %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Folded Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
   %1 = call <16 x i8> @llvm.x86.avx512.mask.pmov.db.512(<16 x i32> %a0, <16 x i8> undef, i16 -1)
   %2 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   ret <16 x i8> %1
@@ -2605,8 +4722,14 @@ define <16 x i8> @stack_fold_vpmovdb(<16 x i32> %a0) {
 declare <16 x i8> @llvm.x86.avx512.mask.pmov.db.512(<16 x i32>, <16 x i8>, i16)
 
 define <16 x i16> @stack_fold_vpmovdw(<16 x i32> %a0) {
-  ;CHECK-LABEL: stack_fold_vpmovdw
-  ;CHECK:       vpmovdw %zmm0, {{-?[0-9]*}}(%rsp) # 32-byte Folded Spill
+; CHECK-LABEL: stack_fold_vpmovdw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmovdw %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Folded Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; CHECK-NEXT:    retq
   %1 = call <16 x i16> @llvm.x86.avx512.mask.pmov.dw.512(<16 x i32> %a0, <16 x i16> undef, i16 -1)
   %2 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   ret <16 x i16> %1
@@ -2614,8 +4737,17 @@ define <16 x i16> @stack_fold_vpmovdw(<16 x i32> %a0) {
 declare <16 x i16> @llvm.x86.avx512.mask.pmov.dw.512(<16 x i32>, <16 x i16>, i16)
 
 define <2 x i64> @stack_fold_movq_load(<2 x i64> %a0) {
-  ;CHECK-LABEL: stack_fold_movq_load
-  ;CHECK:       vmovq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_movq_load:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = mem[0],zero
+; CHECK-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vpsubq %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <2 x i64> %a0, <2 x i64> zeroinitializer, <2 x i32> <i32 0, i32 2>
   ; add forces execution domain
@@ -2624,8 +4756,14 @@ define <2 x i64> @stack_fold_movq_load(<2 x i64> %a0) {
 }
 
 define <8 x i32> @stack_fold_vpmovqd(<8 x i64> %a0) {
-  ;CHECK-LABEL: stack_fold_vpmovqd
-  ;CHECK:       vpmovqd %zmm0, {{-?[0-9]*}}(%rsp) # 32-byte Folded Spill
+; CHECK-LABEL: stack_fold_vpmovqd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmovqd %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Folded Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; CHECK-NEXT:    retq
   %1 = trunc <8 x i64> %a0 to <8 x i32>
   %2 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   ret <8 x i32> %1
@@ -2633,8 +4771,15 @@ define <8 x i32> @stack_fold_vpmovqd(<8 x i64> %a0) {
 declare <8 x i32> @llvm.x86.avx512.mask.pmov.qd.512(<8 x i64>, <8 x i32>, i8)
 
 define <8 x i16> @stack_fold_vpmovqw(<8 x i64> %a0) {
-  ;CHECK-LABEL: stack_fold_vpmovqw
-  ;CHECK:       vpmovqw %zmm0, {{-?[0-9]*}}(%rsp) # 16-byte Folded Spill
+; CHECK-LABEL: stack_fold_vpmovqw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmovqw %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Folded Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
   %1 = call <8 x i16> @llvm.x86.avx512.mask.pmov.qw.512(<8 x i64> %a0, <8 x i16> undef, i8 -1)
   %2 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   ret <8 x i16> %1
@@ -2642,8 +4787,14 @@ define <8 x i16> @stack_fold_vpmovqw(<8 x i64> %a0) {
 declare <8 x i16> @llvm.x86.avx512.mask.pmov.qw.512(<8 x i64>, <8 x i16>, i8)
 
 define <32 x i8> @stack_fold_vpmovwb(<32 x i16> %a0) {
-  ;CHECK-LABEL: stack_fold_vpmovwb
-  ;CHECK:       vpmovwb %zmm0, {{-?[0-9]*}}(%rsp) # 32-byte Folded Spill
+; CHECK-LABEL: stack_fold_vpmovwb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmovwb %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Folded Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; CHECK-NEXT:    retq
   %1 = trunc <32 x i16> %a0 to <32 x i8>
   %2 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   ret <32 x i8> %1
@@ -2651,8 +4802,15 @@ define <32 x i8> @stack_fold_vpmovwb(<32 x i16> %a0) {
 declare <32 x i8> @llvm.x86.avx512.mask.pmov.wb.512(<32 x i16>, <32 x i8>, i32)
 
 define <16 x i8> @stack_fold_vpmovsdb(<16 x i32> %a0) {
-  ;CHECK-LABEL: stack_fold_vpmovsdb
-  ;CHECK:       vpmovsdb %zmm0, {{-?[0-9]*}}(%rsp) # 16-byte Folded Spill
+; CHECK-LABEL: stack_fold_vpmovsdb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmovsdb %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Folded Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
   %1 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.db.512(<16 x i32> %a0, <16 x i8> undef, i16 -1)
   %2 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   ret <16 x i8> %1
@@ -2660,8 +4818,14 @@ define <16 x i8> @stack_fold_vpmovsdb(<16 x i32> %a0) {
 declare <16 x i8> @llvm.x86.avx512.mask.pmovs.db.512(<16 x i32>, <16 x i8>, i16)
 
 define <16 x i16> @stack_fold_vpmovsdw(<16 x i32> %a0) {
-  ;CHECK-LABEL: stack_fold_vpmovsdw
-  ;CHECK:       vpmovsdw %zmm0, {{-?[0-9]*}}(%rsp) # 32-byte Folded Spill
+; CHECK-LABEL: stack_fold_vpmovsdw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmovsdw %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Folded Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; CHECK-NEXT:    retq
   %1 = call <16 x i16> @llvm.x86.avx512.mask.pmovs.dw.512(<16 x i32> %a0, <16 x i16> undef, i16 -1)
   %2 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   ret <16 x i16> %1
@@ -2669,8 +4833,14 @@ define <16 x i16> @stack_fold_vpmovsdw(<16 x i32> %a0) {
 declare <16 x i16> @llvm.x86.avx512.mask.pmovs.dw.512(<16 x i32>, <16 x i16>, i16)
 
 define <8 x i32> @stack_fold_vpmovsqd(<8 x i64> %a0) {
-  ;CHECK-LABEL: stack_fold_vpmovsqd
-  ;CHECK:       vpmovsqd %zmm0, {{-?[0-9]*}}(%rsp) # 32-byte Folded Spill
+; CHECK-LABEL: stack_fold_vpmovsqd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmovsqd %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Folded Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; CHECK-NEXT:    retq
   %1 = call <8 x i32> @llvm.x86.avx512.mask.pmovs.qd.512(<8 x i64> %a0, <8 x i32> undef, i8 -1)
   %2 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   ret <8 x i32> %1
@@ -2678,8 +4848,15 @@ define <8 x i32> @stack_fold_vpmovsqd(<8 x i64> %a0) {
 declare <8 x i32> @llvm.x86.avx512.mask.pmovs.qd.512(<8 x i64>, <8 x i32>, i8)
 
 define <8 x i16> @stack_fold_vpmovsqw(<8 x i64> %a0) {
-  ;CHECK-LABEL: stack_fold_vpmovsqw
-  ;CHECK:       vpmovsqw %zmm0, {{-?[0-9]*}}(%rsp) # 16-byte Folded Spill
+; CHECK-LABEL: stack_fold_vpmovsqw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmovsqw %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Folded Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
   %1 = call <8 x i16> @llvm.x86.avx512.mask.pmovs.qw.512(<8 x i64> %a0, <8 x i16> undef, i8 -1)
   %2 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   ret <8 x i16> %1
@@ -2687,8 +4864,14 @@ define <8 x i16> @stack_fold_vpmovsqw(<8 x i64> %a0) {
 declare <8 x i16> @llvm.x86.avx512.mask.pmovs.qw.512(<8 x i64>, <8 x i16>, i8)
 
 define <32 x i8> @stack_fold_vpmovswb(<32 x i16> %a0) {
-  ;CHECK-LABEL: stack_fold_vpmovswb
-  ;CHECK:       vpmovswb %zmm0, {{-?[0-9]*}}(%rsp) # 32-byte Folded Spill
+; CHECK-LABEL: stack_fold_vpmovswb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmovswb %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Folded Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; CHECK-NEXT:    retq
   %1 = call <32 x i8> @llvm.x86.avx512.mask.pmovs.wb.512(<32 x i16> %a0, <32 x i8> undef, i32 -1)
   %2 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   ret <32 x i8> %1
@@ -2696,16 +4879,28 @@ define <32 x i8> @stack_fold_vpmovswb(<32 x i16> %a0) {
 declare <32 x i8> @llvm.x86.avx512.mask.pmovs.wb.512(<32 x i16>, <32 x i8>, i32)
 
 define <16 x i32> @stack_fold_pmovsxbd_zmm(<16 x i8> %a0) {
-  ;CHECK-LABEL: stack_fold_pmovsxbd_zmm
-  ;CHECK:       vpmovsxbd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmovsxbd_zmm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmovsxbd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = sext <16 x i8> %a0 to <16 x i32>
   ret <16 x i32> %2
 }
 
 define <8 x i64> @stack_fold_pmovsxbq_zmm(<16 x i8> %a0) {
-  ;CHECK-LABEL: stack_fold_pmovsxbq_zmm
-  ;CHECK:       pmovsxbq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmovsxbq_zmm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmovsxbq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <16 x i8> %a0, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   %3 = sext <8 x i8> %2 to <8 x i64>
@@ -2713,40 +4908,71 @@ define <8 x i64> @stack_fold_pmovsxbq_zmm(<16 x i8> %a0) {
 }
 
 define <32 x i16> @stack_fold_pmovsxbw_zmm(<32 x i8> %a0) {
-  ;CHECK-LABEL: stack_fold_pmovsxbw_zmm
-  ;CHECK:       vpmovsxbw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmovsxbw_zmm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmovsxbw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = sext <32 x i8> %a0 to <32 x i16>
   ret <32 x i16> %2
 }
 
 define <8 x i64> @stack_fold_pmovsxdq_zmm(<8 x i32> %a0) {
-  ;CHECK-LABEL: stack_fold_pmovsxdq_zmm
-  ;CHECK:       vpmovsxdq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmovsxdq_zmm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmovsxdq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = sext <8 x i32> %a0 to <8 x i64>
   ret <8 x i64> %2
 }
 
 define <16 x i32> @stack_fold_pmovsxwd_zmm(<16 x i16> %a0) {
-  ;CHECK-LABEL: stack_fold_pmovsxwd_zmm
-  ;CHECK:       vpmovsxwd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmovsxwd_zmm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmovsxwd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = sext <16 x i16> %a0 to <16 x i32>
   ret <16 x i32> %2
 }
 
 define <8 x i64> @stack_fold_pmovsxwq_zmm(<8 x i16> %a0) {
-  ;CHECK-LABEL: stack_fold_pmovsxwq_zmm
-  ;CHECK:       vpmovsxwq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmovsxwq_zmm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmovsxwq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = sext <8 x i16> %a0 to <8 x i64>
   ret <8 x i64> %2
 }
 
 define <8 x i64> @stack_fold_pmovsxwq_mask_zmm(<8 x i64> %passthru, <8 x i16> %a0, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_pmovsxwq_mask_zmm
-  ;CHECK:       vpmovsxwq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmovsxwq_mask_zmm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpmovsxwq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 {%k1} # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = sext <8 x i16> %a0 to <8 x i64>
   %3 = bitcast i8 %mask to <8 x i1>
@@ -2755,8 +4981,15 @@ define <8 x i64> @stack_fold_pmovsxwq_mask_zmm(<8 x i64> %passthru, <8 x i16> %a
 }
 
 define <8 x i64> @stack_fold_pmovsxwq_maskz_zmm(<8 x i16> %a0, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_pmovsxwq_maskz_zmm
-  ;CHECK:       vpmovsxwq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmovsxwq_maskz_zmm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpmovsxwq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 {%k1} {z} # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = sext <8 x i16> %a0 to <8 x i64>
   %3 = bitcast i8 %mask to <8 x i1>
@@ -2765,8 +4998,15 @@ define <8 x i64> @stack_fold_pmovsxwq_maskz_zmm(<8 x i16> %a0, i8 %mask) {
 }
 
 define <16 x i8> @stack_fold_vpmovusdb(<16 x i32> %a0) {
-  ;CHECK-LABEL: stack_fold_vpmovusdb
-  ;CHECK:       vpmovusdb %zmm0, {{-?[0-9]*}}(%rsp) # 16-byte Folded Spill
+; CHECK-LABEL: stack_fold_vpmovusdb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmovusdb %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Folded Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
   %1 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.db.512(<16 x i32> %a0, <16 x i8> undef, i16 -1)
   %2 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   ret <16 x i8> %1
@@ -2774,8 +5014,14 @@ define <16 x i8> @stack_fold_vpmovusdb(<16 x i32> %a0) {
 declare <16 x i8> @llvm.x86.avx512.mask.pmovus.db.512(<16 x i32>, <16 x i8>, i16)
 
 define <16 x i16> @stack_fold_vpmovusdw(<16 x i32> %a0) {
-  ;CHECK-LABEL: stack_fold_vpmovusdw
-  ;CHECK:       vpmovusdw %zmm0, {{-?[0-9]*}}(%rsp) # 32-byte Folded Spill
+; CHECK-LABEL: stack_fold_vpmovusdw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmovusdw %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Folded Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; CHECK-NEXT:    retq
   %1 = call <16 x i16> @llvm.x86.avx512.mask.pmovus.dw.512(<16 x i32> %a0, <16 x i16> undef, i16 -1)
   %2 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   ret <16 x i16> %1
@@ -2783,8 +5029,14 @@ define <16 x i16> @stack_fold_vpmovusdw(<16 x i32> %a0) {
 declare <16 x i16> @llvm.x86.avx512.mask.pmovus.dw.512(<16 x i32>, <16 x i16>, i16)
 
 define <8 x i32> @stack_fold_vpmovusqd(<8 x i64> %a0) {
-  ;CHECK-LABEL: stack_fold_vpmovusqd
-  ;CHECK:       vpmovusqd %zmm0, {{-?[0-9]*}}(%rsp) # 32-byte Folded Spill
+; CHECK-LABEL: stack_fold_vpmovusqd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmovusqd %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Folded Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; CHECK-NEXT:    retq
   %1 = call <8 x i32> @llvm.x86.avx512.mask.pmovus.qd.512(<8 x i64> %a0, <8 x i32> undef, i8 -1)
   %2 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   ret <8 x i32> %1
@@ -2792,8 +5044,15 @@ define <8 x i32> @stack_fold_vpmovusqd(<8 x i64> %a0) {
 declare <8 x i32> @llvm.x86.avx512.mask.pmovus.qd.512(<8 x i64>, <8 x i32>, i8)
 
 define <8 x i16> @stack_fold_vpmovusqw(<8 x i64> %a0) {
-  ;CHECK-LABEL: stack_fold_vpmovusqw
-  ;CHECK:       vpmovusqw %zmm0, {{-?[0-9]*}}(%rsp) # 16-byte Folded Spill
+; CHECK-LABEL: stack_fold_vpmovusqw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmovusqw %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Folded Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
   %1 = call <8 x i16> @llvm.x86.avx512.mask.pmovus.qw.512(<8 x i64> %a0, <8 x i16> undef, i8 -1)
   %2 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   ret <8 x i16> %1
@@ -2801,8 +5060,14 @@ define <8 x i16> @stack_fold_vpmovusqw(<8 x i64> %a0) {
 declare <8 x i16> @llvm.x86.avx512.mask.pmovus.qw.512(<8 x i64>, <8 x i16>, i8)
 
 define <32 x i8> @stack_fold_vpmovuswb(<32 x i16> %a0) {
-  ;CHECK-LABEL: stack_fold_vpmovuswb
-  ;CHECK:       vpmovuswb %zmm0, {{-?[0-9]*}}(%rsp) # 32-byte Folded Spill
+; CHECK-LABEL: stack_fold_vpmovuswb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmovuswb %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Folded Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; CHECK-NEXT:    retq
   %1 = call <32 x i8> @llvm.x86.avx512.mask.pmovus.wb.512(<32 x i16> %a0, <32 x i8> undef, i32 -1)
   %2 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   ret <32 x i8> %1
@@ -2810,16 +5075,30 @@ define <32 x i8> @stack_fold_vpmovuswb(<32 x i16> %a0) {
 declare <32 x i8> @llvm.x86.avx512.mask.pmovus.wb.512(<32 x i16>, <32 x i8>, i32)
 
 define <16 x i32> @stack_fold_pmovzxbd_zmm(<16 x i8> %a0) {
-  ;CHECK-LABEL: stack_fold_pmovzxbd_zmm
-  ;CHECK:       vpmovzxbd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmovzxbd_zmm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmovzxbd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # zmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero,mem[8],zero,zero,zero,mem[9],zero,zero,zero,mem[10],zero,zero,zero,mem[11],zero,zero,zero,mem[12],zero,zero,zero,mem[13],zero,zero,zero,mem[14],zero,zero,zero,mem[15],zero,zero,zero
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = zext <16 x i8> %a0 to <16 x i32>
   ret <16 x i32> %2
 }
 
 define <8 x i64> @stack_fold_pmovzxbq_zmm(<16 x i8> %a0) {
-  ;CHECK-LABEL: stack_fold_pmovzxbq_zmm
-  ;CHECK:       vpmovzxbq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmovzxbq_zmm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmovzxbq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # zmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero,mem[2],zero,zero,zero,zero,zero,zero,zero,mem[3],zero,zero,zero,zero,zero,zero,zero,mem[4],zero,zero,zero,zero,zero,zero,zero,mem[5],zero,zero,zero,zero,zero,zero,zero,mem[6],zero,zero,zero,zero,zero,zero,zero,mem[7],zero,zero,zero,zero,zero,zero,zero
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <16 x i8> %a0, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   %3 = zext <8 x i8> %2 to <8 x i64>
@@ -2827,40 +5106,76 @@ define <8 x i64> @stack_fold_pmovzxbq_zmm(<16 x i8> %a0) {
 }
 
 define <32 x i16> @stack_fold_pmovzxbw_zmm(<32 x i8> %a0) {
-  ;CHECK-LABEL: stack_fold_pmovzxbw_zmm
-  ;CHECK:       vpmovzxbw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmovzxbw_zmm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmovzxbw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 32-byte Folded Reload
+; CHECK-NEXT:    # zmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero,mem[16],zero,mem[17],zero,mem[18],zero,mem[19],zero,mem[20],zero,mem[21],zero,mem[22],zero,mem[23],zero,mem[24],zero,mem[25],zero,mem[26],zero,mem[27],zero,mem[28],zero,mem[29],zero,mem[30],zero,mem[31],zero
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = zext <32 x i8> %a0 to <32 x i16>
   ret <32 x i16> %2
 }
 
 define <8 x i64> @stack_fold_pmovzxdq_zmm(<8 x i32> %a0) {
-  ;CHECK-LABEL: stack_fold_pmovzxdq_zmm
-  ;CHECK:       vpmovzxdq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmovzxdq_zmm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmovzxdq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 32-byte Folded Reload
+; CHECK-NEXT:    # zmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = zext <8 x i32> %a0 to <8 x i64>
   ret <8 x i64> %2
 }
 
 define <16 x i32> @stack_fold_pmovzxwd_zmm(<16 x i16> %a0) {
-  ;CHECK-LABEL: stack_fold_pmovzxwd_zmm
-  ;CHECK:       vpmovzxwd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmovzxwd_zmm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmovzxwd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 32-byte Folded Reload
+; CHECK-NEXT:    # zmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = zext <16 x i16> %a0 to <16 x i32>
   ret <16 x i32> %2
 }
 
 define <8 x i64> @stack_fold_pmovzxwq_zmm(<8 x i16> %a0) {
-  ;CHECK-LABEL: stack_fold_pmovzxwq_zmm
-  ;CHECK:       vpmovzxwq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmovzxwq_zmm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmovzxwq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # zmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = zext <8 x i16> %a0 to <8 x i64>
   ret <8 x i64> %2
 }
 
 define <8 x i64> @stack_fold_pmovzxwq_mask_zmm(<8 x i64> %passthru, <8 x i16> %a0, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_pmovzxwq_mask_zmm
-  ;CHECK:       vpmovzxwq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmovzxwq_mask_zmm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpmovzxwq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 {%k1} # 16-byte Folded Reload
+; CHECK-NEXT:    # zmm0 {%k1} = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = zext <8 x i16> %a0 to <8 x i64>
   %3 = bitcast i8 %mask to <8 x i1>
@@ -2869,8 +5184,16 @@ define <8 x i64> @stack_fold_pmovzxwq_mask_zmm(<8 x i64> %passthru, <8 x i16> %a
 }
 
 define <8 x i64> @stack_fold_pmovzxwq_maskz_zmm(<8 x i16> %a0, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_pmovzxwq_maskz_zmm
-  ;CHECK:       vpmovzxwq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmovzxwq_maskz_zmm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpmovzxwq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 {%k1} {z} # 16-byte Folded Reload
+; CHECK-NEXT:    # zmm0 {%k1} {z} = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = zext <8 x i16> %a0 to <8 x i64>
   %3 = bitcast i8 %mask to <8 x i1>
@@ -2879,24 +5202,45 @@ define <8 x i64> @stack_fold_pmovzxwq_maskz_zmm(<8 x i16> %a0, i8 %mask) {
 }
 
 define <16 x i32> @stack_fold_pmulld(<16 x i32> %a0, <16 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_pmulld
-  ;CHECK:       vpmulld {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmulld:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmulld {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = mul <16 x i32> %a0, %a1
   ret <16 x i32> %2
 }
 
 define <16 x i32> @stack_fold_pmulld_commuted(<16 x i32> %a0, <16 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_pmulld_commuted:
-  ;CHECK:       vpmulld {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmulld_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmulld {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = mul <16 x i32> %a1, %a0
   ret <16 x i32> %2
 }
 
 define <16 x i32> @stack_fold_pmulld_mask(<16 x i32> %a0, <16 x i32> %a1, <16 x i32>* %a2, i16 %mask) {
-  ;CHECK-LABEL: stack_fold_pmulld_mask
-  ;CHECK:       vpmulld {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmulld_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm1
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm0
+; CHECK-NEXT:    vpmulld {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = mul <16 x i32> %a0, %a1
   %3 = bitcast i16 %mask to <16 x i1>
@@ -2907,8 +5251,17 @@ define <16 x i32> @stack_fold_pmulld_mask(<16 x i32> %a0, <16 x i32> %a1, <16 x
 }
 
 define <16 x i32> @stack_fold_pmulld_mask_commuted(<16 x i32> %a0, <16 x i32> %a1, <16 x i32>* %a2, i16 %mask) {
-  ;CHECK-LABEL: stack_fold_pmulld_mask_commuted:
-  ;CHECK:       vpmulld {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmulld_mask_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm1
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm0
+; CHECK-NEXT:    vpmulld {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = mul <16 x i32> %a1, %a0
   %3 = bitcast i16 %mask to <16 x i1>
@@ -2919,8 +5272,15 @@ define <16 x i32> @stack_fold_pmulld_mask_commuted(<16 x i32> %a0, <16 x i32> %a
 }
 
 define <16 x i32> @stack_fold_pmulld_maskz(<16 x i32> %a0, <16 x i32> %a1, i16 %mask) {
-  ;CHECK-LABEL: stack_fold_pmulld_maskz
-  ;CHECK:       vpmulld {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmulld_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpmulld {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = mul <16 x i32> %a0, %a1
   %3 = bitcast i16 %mask to <16 x i1>
@@ -2929,8 +5289,15 @@ define <16 x i32> @stack_fold_pmulld_maskz(<16 x i32> %a0, <16 x i32> %a1, i16 %
 }
 
 define <16 x i32> @stack_fold_pmulld_maskz_commuted(<16 x i32> %a0, <16 x i32> %a1, i16 %mask) {
-  ;CHECK-LABEL: stack_fold_pmulld_maskz_commuted:
-  ;CHECK:       vpmulld {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmulld_maskz_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpmulld {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = mul <16 x i32> %a1, %a0
   %3 = bitcast i16 %mask to <16 x i1>
@@ -2939,24 +5306,45 @@ define <16 x i32> @stack_fold_pmulld_maskz_commuted(<16 x i32> %a0, <16 x i32> %
 }
 
 define <8 x i64> @stack_fold_pmullq(<8 x i64> %a0, <8 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_pmullq
-  ;CHECK:       vpmullq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmullq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmullq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = mul <8 x i64> %a0, %a1
   ret <8 x i64> %2
 }
 
 define <8 x i64> @stack_fold_pmullq_commuted(<8 x i64> %a0, <8 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_pmullq_commuted:
-  ;CHECK:       vpmullq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmullq_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmullq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = mul <8 x i64> %a1, %a0
   ret <8 x i64> %2
 }
 
 define <8 x i64> @stack_fold_pmullq_mask(<8 x i64> %a0, <8 x i64> %a1, <8 x i64>* %a2, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_pmullq_mask
-  ;CHECK:       vpmullq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmullq_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm1
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm0
+; CHECK-NEXT:    vpmullq {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = mul <8 x i64> %a0, %a1
   %3 = bitcast i8 %mask to <8 x i1>
@@ -2967,8 +5355,17 @@ define <8 x i64> @stack_fold_pmullq_mask(<8 x i64> %a0, <8 x i64> %a1, <8 x i64>
 }
 
 define <8 x i64> @stack_fold_pmullq_mask_commuted(<8 x i64> %a0, <8 x i64> %a1, <8 x i64>* %a2, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_pmullq_mask_commuted:
-  ;CHECK:       vpmullq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmullq_mask_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm1
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm0
+; CHECK-NEXT:    vpmullq {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = mul <8 x i64> %a1, %a0
   %3 = bitcast i8 %mask to <8 x i1>
@@ -2979,8 +5376,15 @@ define <8 x i64> @stack_fold_pmullq_mask_commuted(<8 x i64> %a0, <8 x i64> %a1,
 }
 
 define <8 x i64> @stack_fold_pmullq_maskz(<8 x i64> %a0, <8 x i64> %a1, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_pmullq_maskz
-  ;CHECK:       vpmullq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmullq_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpmullq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = mul <8 x i64> %a0, %a1
   %3 = bitcast i8 %mask to <8 x i1>
@@ -2989,8 +5393,15 @@ define <8 x i64> @stack_fold_pmullq_maskz(<8 x i64> %a0, <8 x i64> %a1, i8 %mask
 }
 
 define <8 x i64> @stack_fold_pmullq_maskz_commuted(<8 x i64> %a0, <8 x i64> %a1, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_pmullq_maskz_commuted:
-  ;CHECK:       vpmullq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmullq_maskz_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpmullq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = mul <8 x i64> %a1, %a0
   %3 = bitcast i8 %mask to <8 x i1>
@@ -2999,24 +5410,45 @@ define <8 x i64> @stack_fold_pmullq_maskz_commuted(<8 x i64> %a0, <8 x i64> %a1,
 }
 
 define <32 x i16> @stack_fold_pmullw(<32 x i16> %a0, <32 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_pmullw
-  ;CHECK:       vpmullw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmullw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmullw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = mul <32 x i16> %a0, %a1
   ret <32 x i16> %2
 }
 
 define <32 x i16> @stack_fold_pmullw_commuted(<32 x i16> %a0, <32 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_pmullw_commuted:
-  ;CHECK:       vpmullw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmullw_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmullw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = mul <32 x i16> %a1, %a0
   ret <32 x i16> %2
 }
 
 define <32 x i16> @stack_fold_pmullw_mask(<32 x i16> %a0, <32 x i16> %a1, <32 x i16>* %a2, i32 %mask) {
-  ;CHECK-LABEL: stack_fold_pmullw_mask
-  ;CHECK:       vpmullw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmullw_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm1
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm0
+; CHECK-NEXT:    vpmullw {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = mul <32 x i16> %a0, %a1
   %3 = bitcast i32 %mask to <32 x i1>
@@ -3027,8 +5459,17 @@ define <32 x i16> @stack_fold_pmullw_mask(<32 x i16> %a0, <32 x i16> %a1, <32 x
 }
 
 define <32 x i16> @stack_fold_pmullw_mask_commuted(<32 x i16> %a0, <32 x i16> %a1, <32 x i16>* %a2, i32 %mask) {
-  ;CHECK-LABEL: stack_fold_pmullw_mask_commuted:
-  ;CHECK:       vpmullw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmullw_mask_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm1
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm0
+; CHECK-NEXT:    vpmullw {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = mul <32 x i16> %a1, %a0
   %3 = bitcast i32 %mask to <32 x i1>
@@ -3039,8 +5480,15 @@ define <32 x i16> @stack_fold_pmullw_mask_commuted(<32 x i16> %a0, <32 x i16> %a
 }
 
 define <32 x i16> @stack_fold_pmullw_maskz(<32 x i16> %a0, <32 x i16> %a1, i32 %mask) {
-  ;CHECK-LABEL: stack_fold_pmullw_maskz
-  ;CHECK:       vpmullw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmullw_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpmullw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = mul <32 x i16> %a0, %a1
   %3 = bitcast i32 %mask to <32 x i1>
@@ -3049,8 +5497,15 @@ define <32 x i16> @stack_fold_pmullw_maskz(<32 x i16> %a0, <32 x i16> %a1, i32 %
 }
 
 define <32 x i16> @stack_fold_pmullw_maskz_commuted(<32 x i16> %a0, <32 x i16> %a1, i32 %mask) {
-  ;CHECK-LABEL: stack_fold_pmullw_maskz_commuted:
-  ;CHECK:       vpmullw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmullw_maskz_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpmullw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = mul <32 x i16> %a1, %a0
   %3 = bitcast i32 %mask to <32 x i1>
@@ -3059,8 +5514,14 @@ define <32 x i16> @stack_fold_pmullw_maskz_commuted(<32 x i16> %a0, <32 x i16> %
 }
 
 define <8 x i64> @stack_fold_pmuldq(<8 x i64> %a0, <8 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_pmuldq
-  ;CHECK:       vpmuldq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmuldq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmuldq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shl <8 x i64> %a0, <i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32>
   %3 = ashr <8 x i64> %2, <i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32>
@@ -3071,8 +5532,14 @@ define <8 x i64> @stack_fold_pmuldq(<8 x i64> %a0, <8 x i64> %a1) {
 }
 
 define <8 x i64> @stack_fold_pmuldq_commuted(<8 x i64> %a0, <8 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_pmuldq_commuted:
-  ;CHECK:       vpmuldq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmuldq_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmuldq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shl <8 x i64> %a0, <i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32>
   %3 = ashr <8 x i64> %2, <i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32>
@@ -3083,8 +5550,17 @@ define <8 x i64> @stack_fold_pmuldq_commuted(<8 x i64> %a0, <8 x i64> %a1) {
 }
 
 define <8 x i64> @stack_fold_pmuldq_mask(<8 x i64> %a0, <8 x i64> %a1, <8 x i64>* %a2, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_pmuldq_mask
-  ;CHECK:       vpmuldq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmuldq_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm1
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm0
+; CHECK-NEXT:    vpmuldq {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shl <8 x i64> %a0, <i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32>
   %3 = ashr <8 x i64> %2, <i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32>
@@ -3099,8 +5575,17 @@ define <8 x i64> @stack_fold_pmuldq_mask(<8 x i64> %a0, <8 x i64> %a1, <8 x i64>
 }
 
 define <8 x i64> @stack_fold_pmuldq_mask_commuted(<8 x i64> %a0, <8 x i64> %a1, <8 x i64>* %a2, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_pmuldq_mask_commuted:
-  ;CHECK:       vpmuldq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmuldq_mask_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm1
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm0
+; CHECK-NEXT:    vpmuldq {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shl <8 x i64> %a0, <i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32>
   %3 = ashr <8 x i64> %2, <i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32>
@@ -3115,8 +5600,15 @@ define <8 x i64> @stack_fold_pmuldq_mask_commuted(<8 x i64> %a0, <8 x i64> %a1,
 }
 
 define <8 x i64> @stack_fold_pmuldq_maskz(<8 x i64> %a0, <8 x i64> %a1, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_pmuldq_maskz
-  ;CHECK:       vpmuldq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmuldq_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpmuldq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shl <8 x i64> %a0, <i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32>
   %3 = ashr <8 x i64> %2, <i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32>
@@ -3129,8 +5621,15 @@ define <8 x i64> @stack_fold_pmuldq_maskz(<8 x i64> %a0, <8 x i64> %a1, i8 %mask
 }
 
 define <8 x i64> @stack_fold_pmuldq_maskz_commuted(<8 x i64> %a0, <8 x i64> %a1, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_pmuldq_maskz_commuted:
-  ;CHECK:       vpmuldq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmuldq_maskz_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpmuldq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shl <8 x i64> %a0, <i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32>
   %3 = ashr <8 x i64> %2, <i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32>
@@ -3146,8 +5645,14 @@ define <8 x i64> @stack_fold_pmuldq_maskz_commuted(<8 x i64> %a0, <8 x i64> %a1,
 
 
 define <8 x i64> @stack_fold_pmuludq(<8 x i64> %a0, <8 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_pmuludq
-  ;CHECK:       vpmuludq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmuludq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmuludq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = and <8 x i64> %a0, <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>
   %3 = and <8 x i64> %a1, <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>
@@ -3156,8 +5661,14 @@ define <8 x i64> @stack_fold_pmuludq(<8 x i64> %a0, <8 x i64> %a1) {
 }
 
 define <8 x i64> @stack_fold_pmuludq_commuted(<8 x i64> %a0, <8 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_pmuludq_commuted:
-  ;CHECK:       vpmuludq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmuludq_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmuludq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = and <8 x i64> %a0, <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>
   %3 = and <8 x i64> %a1, <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>
@@ -3166,8 +5677,17 @@ define <8 x i64> @stack_fold_pmuludq_commuted(<8 x i64> %a0, <8 x i64> %a1) {
 }
 
 define <8 x i64> @stack_fold_pmuludq_mask(<8 x i64> %a0, <8 x i64> %a1, <8 x i64>* %a2, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_pmuludq_mask
-  ;CHECK:       vpmuludq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmuludq_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm1
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm0
+; CHECK-NEXT:    vpmuludq {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = and <8 x i64> %a0, <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>
   %3 = and <8 x i64> %a1, <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>
@@ -3180,8 +5700,17 @@ define <8 x i64> @stack_fold_pmuludq_mask(<8 x i64> %a0, <8 x i64> %a1, <8 x i64
 }
 
 define <8 x i64> @stack_fold_pmuludq_mask_commuted(<8 x i64> %a0, <8 x i64> %a1, <8 x i64>* %a2, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_pmuludq_mask_commuted:
-  ;CHECK:       vpmuludq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmuludq_mask_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm1
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm0
+; CHECK-NEXT:    vpmuludq {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = and <8 x i64> %a0, <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>
   %3 = and <8 x i64> %a1, <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>
@@ -3194,8 +5723,15 @@ define <8 x i64> @stack_fold_pmuludq_mask_commuted(<8 x i64> %a0, <8 x i64> %a1,
 }
 
 define <8 x i64> @stack_fold_pmuludq_maskz(<8 x i64> %a0, <8 x i64> %a1, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_pmuludq_maskz
-  ;CHECK:       vpmuludq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmuludq_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpmuludq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = and <8 x i64> %a0, <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>
   %3 = and <8 x i64> %a1, <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>
@@ -3206,8 +5742,15 @@ define <8 x i64> @stack_fold_pmuludq_maskz(<8 x i64> %a0, <8 x i64> %a1, i8 %mas
 }
 
 define <8 x i64> @stack_fold_pmuludq_maskz_commuted(<8 x i64> %a0, <8 x i64> %a1, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_pmuludq_maskz_commuted:
-  ;CHECK:       vpmuludq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmuludq_maskz_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpmuludq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = and <8 x i64> %a0, <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>
   %3 = and <8 x i64> %a1, <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>
@@ -3218,8 +5761,14 @@ define <8 x i64> @stack_fold_pmuludq_maskz_commuted(<8 x i64> %a0, <8 x i64> %a1
 }
 
 define <16 x i32> @stack_fold_vpopcntd(<16 x i32> %a0) {
-  ;CHECK-LABEL: stack_fold_vpopcntd
-  ;CHECK:       vpopcntd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpopcntd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpopcntd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x i32> @llvm.ctpop.v16i32(<16 x i32> %a0)
   ret <16 x i32> %2
@@ -3227,8 +5776,14 @@ define <16 x i32> @stack_fold_vpopcntd(<16 x i32> %a0) {
 declare <16 x i32> @llvm.ctpop.v16i32(<16 x i32>) nounwind readonly
 
 define <8 x i64> @stack_fold_vpopcntq(<8 x i64> %a0) {
-  ;CHECK-LABEL: stack_fold_vpopcntq
-  ;CHECK:       vpopcntq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpopcntq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpopcntq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x i64> @llvm.ctpop.v8i64(<8 x i64> %a0)
   ret <8 x i64> %2
@@ -3236,24 +5791,45 @@ define <8 x i64> @stack_fold_vpopcntq(<8 x i64> %a0) {
 declare <8 x i64> @llvm.ctpop.v8i64(<8 x i64>) nounwind readnone
 
 define <16 x i32> @stack_fold_pord(<16 x i32> %a0, <16 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_pord
-  ;CHECK:       vorps {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pord:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vorps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = or <16 x i32> %a0, %a1
   ret <16 x i32> %2
 }
 
 define <16 x i32> @stack_fold_pord_commuted(<16 x i32> %a0, <16 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_pord_commuted:
-  ;CHECK:       vorps {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pord_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vorps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = or <16 x i32> %a1, %a0
   ret <16 x i32> %2
 }
 
 define <16 x i32> @stack_fold_pord_mask(<16 x i32> %a0, <16 x i32> %a1, <16 x i32>* %a2, i16 %mask) {
-  ;CHECK-LABEL: stack_fold_pord_mask
-  ;CHECK:       vorps {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pord_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovaps %zmm0, %zmm1
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vmovaps (%rdi), %zmm0
+; CHECK-NEXT:    vorps {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = or <16 x i32> %a0, %a1
   %3 = bitcast i16 %mask to <16 x i1>
@@ -3264,8 +5840,17 @@ define <16 x i32> @stack_fold_pord_mask(<16 x i32> %a0, <16 x i32> %a1, <16 x i3
 }
 
 define <16 x i32> @stack_fold_pord_mask_commuted(<16 x i32> %a0, <16 x i32> %a1, <16 x i32>* %a2, i16 %mask) {
-  ;CHECK-LABEL: stack_fold_pord_mask_commuted:
-  ;CHECK:       vorps {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pord_mask_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovaps %zmm0, %zmm1
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vmovaps (%rdi), %zmm0
+; CHECK-NEXT:    vorps {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = or <16 x i32> %a1, %a0
   %3 = bitcast i16 %mask to <16 x i1>
@@ -3276,8 +5861,15 @@ define <16 x i32> @stack_fold_pord_mask_commuted(<16 x i32> %a0, <16 x i32> %a1,
 }
 
 define <16 x i32> @stack_fold_pord_maskz(<16 x i32> %a0, <16 x i32> %a1, i16 %mask) {
-  ;CHECK-LABEL: stack_fold_pord_maskz
-  ;CHECK:       vorps {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pord_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vorps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = or <16 x i32> %a0, %a1
   %3 = bitcast i16 %mask to <16 x i1>
@@ -3286,8 +5878,15 @@ define <16 x i32> @stack_fold_pord_maskz(<16 x i32> %a0, <16 x i32> %a1, i16 %ma
 }
 
 define <16 x i32> @stack_fold_pord_maskz_commuted(<16 x i32> %a0, <16 x i32> %a1, i16 %mask) {
-  ;CHECK-LABEL: stack_fold_pord_maskz_commuted:
-  ;CHECK:       vorps {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pord_maskz_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vorps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = or <16 x i32> %a1, %a0
   %3 = bitcast i16 %mask to <16 x i1>
@@ -3296,24 +5895,45 @@ define <16 x i32> @stack_fold_pord_maskz_commuted(<16 x i32> %a0, <16 x i32> %a1
 }
 
 define <8 x i64> @stack_fold_porq(<8 x i64> %a0, <8 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_porq
-  ;CHECK:       vorps {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_porq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vorps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = or <8 x i64> %a0, %a1
   ret <8 x i64> %2
 }
 
 define <8 x i64> @stack_fold_porq_commuted(<8 x i64> %a0, <8 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_porq_commuted:
-  ;CHECK:       vorps {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_porq_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vorps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = or <8 x i64> %a1, %a0
   ret <8 x i64> %2
 }
 
 define <8 x i64> @stack_fold_porq_mask(<8 x i64> %a0, <8 x i64> %a1, <8 x i64>* %a2, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_porq_mask
-  ;CHECK:       vorpd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_porq_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovapd %zmm0, %zmm1
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vmovapd (%rdi), %zmm0
+; CHECK-NEXT:    vorpd {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = or <8 x i64> %a0, %a1
   %3 = bitcast i8 %mask to <8 x i1>
@@ -3324,8 +5944,17 @@ define <8 x i64> @stack_fold_porq_mask(<8 x i64> %a0, <8 x i64> %a1, <8 x i64>*
 }
 
 define <8 x i64> @stack_fold_porq_mask_commuted(<8 x i64> %a0, <8 x i64> %a1, <8 x i64>* %a2, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_porq_mask_commuted:
-  ;CHECK:       vorpd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_porq_mask_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovapd %zmm0, %zmm1
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vmovapd (%rdi), %zmm0
+; CHECK-NEXT:    vorpd {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = or <8 x i64> %a1, %a0
   %3 = bitcast i8 %mask to <8 x i1>
@@ -3336,8 +5965,15 @@ define <8 x i64> @stack_fold_porq_mask_commuted(<8 x i64> %a0, <8 x i64> %a1, <8
 }
 
 define <8 x i64> @stack_fold_porq_maskz(<8 x i64> %a0, <8 x i64> %a1, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_porq_maskz
-  ;CHECK:       vorpd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_porq_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vorpd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = or <8 x i64> %a0, %a1
   %3 = bitcast i8 %mask to <8 x i1>
@@ -3346,8 +5982,15 @@ define <8 x i64> @stack_fold_porq_maskz(<8 x i64> %a0, <8 x i64> %a1, i8 %mask)
 }
 
 define <8 x i64> @stack_fold_porq_maskz_commuted(<8 x i64> %a0, <8 x i64> %a1, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_porq_maskz_commuted:
-  ;CHECK:       vorpd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_porq_maskz_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vorpd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = or <8 x i64> %a1, %a0
   %3 = bitcast i8 %mask to <8 x i1>
@@ -3356,17 +5999,43 @@ define <8 x i64> @stack_fold_porq_maskz_commuted(<8 x i64> %a0, <8 x i64> %a1, i
 }
 
 define <8 x i64> @stack_fold_psadbw(<64 x i8> %a0, <64 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_psadbw
-  ;CHECK:       vpsadbw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_psadbw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsadbw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x i64> @llvm.x86.avx512.psad.bw.512(<64 x i8> %a0, <64 x i8> %a1)
   ret <8 x i64> %2
 }
 declare <8 x i64> @llvm.x86.avx512.psad.bw.512(<64 x i8>, <64 x i8>) nounwind readnone
 
+define <8 x i64> @stack_fold_psadbw_commute(<64 x i8> %a0, <64 x i8> %a1) {
+; CHECK-LABEL: stack_fold_psadbw_commute:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsadbw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
+  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
+  %2 = call <8 x i64> @llvm.x86.avx512.psad.bw.512(<64 x i8> %a1, <64 x i8> %a0)
+  ret <8 x i64> %2
+}
+
 define <64 x i8> @stack_fold_pshufb_zmm(<64 x i8> %a0, <64 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pshufb_zmm
-  ;CHECK:       vpshufb {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pshufb_zmm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpshufb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> %a0, <64 x i8> %a1)
   ret <64 x i8> %2
@@ -3374,8 +6043,17 @@ define <64 x i8> @stack_fold_pshufb_zmm(<64 x i8> %a0, <64 x i8> %a1) {
 declare <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8>, <64 x i8>)
 
 define <64 x i8> @stack_fold_pshufb_zmm_mask(<64 x i8>* %passthru, <64 x i8> %a0, <64 x i8> %a1, i64 %mask) {
-  ;CHECK-LABEL: stack_fold_pshufb_zmm_mask
-  ;CHECK:       vpshufb {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pshufb_zmm_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm2
+; CHECK-NEXT:    kmovq %rsi, %k1
+; CHECK-NEXT:    vpshufb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = load <64 x i8>, <64 x i8>* %passthru
   %3 = call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> %a0, <64 x i8> %a1)
@@ -3385,8 +6063,15 @@ define <64 x i8> @stack_fold_pshufb_zmm_mask(<64 x i8>* %passthru, <64 x i8> %a0
 }
 
 define <64 x i8> @stack_fold_pshufb_zmm_maskz(<64 x i8> %a0, <64 x i8> %a1, i64 %mask) {
-  ;CHECK-LABEL: stack_fold_pshufb_zmm_maskz
-  ;CHECK:       vpshufb {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pshufb_zmm_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovq %rdi, %k1
+; CHECK-NEXT:    vpshufb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> %a0, <64 x i8> %a1)
   %3 = bitcast i64 %mask to <64 x i1>
@@ -3395,8 +6080,17 @@ define <64 x i8> @stack_fold_pshufb_zmm_maskz(<64 x i8> %a0, <64 x i8> %a1, i64
 }
 
 define <16 x i32> @stack_fold_pshufd_zmm(<16 x i32> %a0) {
-  ;CHECK-LABEL: stack_fold_pshufd_zmm
-  ;CHECK:       vpshufd $27, {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pshufd_zmm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpshufd $27, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    # zmm0 = mem[3,2,1,0,7,6,5,4,11,10,9,8,15,14,13,12]
+; CHECK-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1
+; CHECK-NEXT:    vpsubd %zmm1, %zmm0, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <16 x i32> %a0, <16 x i32> undef, <16 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 11, i32 10, i32 9, i32 8, i32 15, i32 14, i32 13, i32 12>
   %3 = add <16 x i32> %2, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
@@ -3404,8 +6098,22 @@ define <16 x i32> @stack_fold_pshufd_zmm(<16 x i32> %a0) {
 }
 
 define <16 x i32> @stack_fold_pshufd_zmm_mask(<16 x i32> %passthru, <16 x i32> %a0, i16 %mask) {
-  ;CHECK-LABEL: stack_fold_pshufd_zmm_mask
-  ;CHECK:       vpshufd $27, {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pshufd_zmm_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subq $56, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; CHECK-NEXT:    vpshufd $27, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    # zmm0 {%k1} = mem[3,2,1,0,7,6,5,4,11,10,9,8,15,14,13,12]
+; CHECK-NEXT:    addq $56, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <16 x i32> %a0, <16 x i32> undef, <16 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 11, i32 10, i32 9, i32 8, i32 15, i32 14, i32 13, i32 12>
   %3 = bitcast i16 %mask to <16 x i1>
@@ -3414,8 +6122,16 @@ define <16 x i32> @stack_fold_pshufd_zmm_mask(<16 x i32> %passthru, <16 x i32> %
 }
 
 define <16 x i32> @stack_fold_pshufd_zmm_maskz(<16 x i32> %a0, i16 %mask) {
-  ;CHECK-LABEL: stack_fold_pshufd_zmm_maskz
-  ;CHECK:       vpshufd $27, {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pshufd_zmm_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpshufd $27, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    # zmm0 {%k1} {z} = mem[3,2,1,0,7,6,5,4,11,10,9,8,15,14,13,12]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <16 x i32> %a0, <16 x i32> undef, <16 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 11, i32 10, i32 9, i32 8, i32 15, i32 14, i32 13, i32 12>
   %3 = bitcast i16 %mask to <16 x i1>
@@ -3424,16 +6140,37 @@ define <16 x i32> @stack_fold_pshufd_zmm_maskz(<16 x i32> %a0, i16 %mask) {
 }
 
 define <32 x i16> @stack_fold_pshufhw_zmm(<32 x i16> %a0) {
-  ;CHECK-LABEL: stack_fold_pshufhw_zmm
-  ;CHECK:       vpshufhw $27, {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pshufhw_zmm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpshufhw $27, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    # zmm0 = mem[0,1,2,3,7,6,5,4,8,9,10,11,15,14,13,12,16,17,18,19,23,22,21,20,24,25,26,27,31,30,29,28]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <32 x i16> %a0, <32 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 7, i32 6, i32 5, i32 4, i32 8, i32 9, i32 10, i32 11, i32 15, i32 14, i32 13, i32 12, i32 16, i32 17, i32 18, i32 19, i32 23, i32 22, i32 21, i32 20, i32 24, i32 25, i32 26, i32 27, i32 31, i32 30, i32 29, i32 28>
   ret <32 x i16> %2
 }
 
 define <32 x i16> @stack_fold_pshufhw_zmm_mask(<32 x i16> %passthru, <32 x i16> %a0, i32 %mask) {
-  ;CHECK-LABEL: stack_fold_pshufhw_zmm_mask
-  ;CHECK:       vpshufhw $27, {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pshufhw_zmm_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subq $56, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; CHECK-NEXT:    vpshufhw $27, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    # zmm0 {%k1} = mem[0,1,2,3,7,6,5,4,8,9,10,11,15,14,13,12,16,17,18,19,23,22,21,20,24,25,26,27,31,30,29,28]
+; CHECK-NEXT:    addq $56, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <32 x i16> %a0, <32 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 7, i32 6, i32 5, i32 4, i32 8, i32 9, i32 10, i32 11, i32 15, i32 14, i32 13, i32 12, i32 16, i32 17, i32 18, i32 19, i32 23, i32 22, i32 21, i32 20, i32 24, i32 25, i32 26, i32 27, i32 31, i32 30, i32 29, i32 28>
   %3 = bitcast i32 %mask to <32 x i1>
@@ -3442,8 +6179,16 @@ define <32 x i16> @stack_fold_pshufhw_zmm_mask(<32 x i16> %passthru, <32 x i16>
 }
 
 define <32 x i16> @stack_fold_pshufhw_zmm_maskz(<32 x i16> %a0, i32 %mask) {
-  ;CHECK-LABEL: stack_fold_pshufhw_zmm_maskz
-  ;CHECK:       vpshufhw $27, {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pshufhw_zmm_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpshufhw $27, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    # zmm0 {%k1} {z} = mem[0,1,2,3,7,6,5,4,8,9,10,11,15,14,13,12,16,17,18,19,23,22,21,20,24,25,26,27,31,30,29,28]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <32 x i16> %a0, <32 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 7, i32 6, i32 5, i32 4, i32 8, i32 9, i32 10, i32 11, i32 15, i32 14, i32 13, i32 12, i32 16, i32 17, i32 18, i32 19, i32 23, i32 22, i32 21, i32 20, i32 24, i32 25, i32 26, i32 27, i32 31, i32 30, i32 29, i32 28>
   %3 = bitcast i32 %mask to <32 x i1>
@@ -3452,16 +6197,37 @@ define <32 x i16> @stack_fold_pshufhw_zmm_maskz(<32 x i16> %a0, i32 %mask) {
 }
 
 define <32 x i16> @stack_fold_pshuflw_zmm(<32 x i16> %a0) {
-  ;CHECK-LABEL: stack_fold_pshuflw_zmm
-  ;CHECK:       vpshuflw $27, {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pshuflw_zmm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpshuflw $27, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    # zmm0 = mem[3,2,1,0,4,5,6,7,11,10,9,8,12,13,14,15,19,18,17,16,20,21,22,23,27,26,25,24,28,29,30,31]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <32 x i16> %a0, <32 x i16> undef, <32 x i32> <i32 3, i32 2, i32 1, i32 0, i32 4, i32 5, i32 6, i32 7, i32 11, i32 10, i32 9, i32 8, i32 12, i32 13, i32 14, i32 15, i32 19, i32 18, i32 17, i32 16, i32 20, i32 21, i32 22, i32 23, i32 27, i32 26, i32 25, i32 24, i32 28, i32 29, i32 30, i32 31>
   ret <32 x i16> %2
 }
 
 define <32 x i16> @stack_fold_pshuflw_zmm_mask(<32 x i16> %passthru, <32 x i16> %a0, i32 %mask) {
-  ;CHECK-LABEL: stack_fold_pshuflw_zmm_mask
-  ;CHECK:       vpshuflw $27, {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pshuflw_zmm_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subq $56, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; CHECK-NEXT:    vpshuflw $27, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    # zmm0 {%k1} = mem[3,2,1,0,4,5,6,7,11,10,9,8,12,13,14,15,19,18,17,16,20,21,22,23,27,26,25,24,28,29,30,31]
+; CHECK-NEXT:    addq $56, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <32 x i16> %a0, <32 x i16> undef, <32 x i32> <i32 3, i32 2, i32 1, i32 0, i32 4, i32 5, i32 6, i32 7, i32 11, i32 10, i32 9, i32 8, i32 12, i32 13, i32 14, i32 15, i32 19, i32 18, i32 17, i32 16, i32 20, i32 21, i32 22, i32 23, i32 27, i32 26, i32 25, i32 24, i32 28, i32 29, i32 30, i32 31>
   %3 = bitcast i32 %mask to <32 x i1>
@@ -3470,8 +6236,16 @@ define <32 x i16> @stack_fold_pshuflw_zmm_mask(<32 x i16> %passthru, <32 x i16>
 }
 
 define <32 x i16> @stack_fold_pshuflw_zmm_maskz(<32 x i16> %a0, i32 %mask) {
-  ;CHECK-LABEL: stack_fold_pshuflw_zmm_maskz
-  ;CHECK:       vpshuflw $27, {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pshuflw_zmm_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpshuflw $27, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    # zmm0 {%k1} {z} = mem[3,2,1,0,4,5,6,7,11,10,9,8,12,13,14,15,19,18,17,16,20,21,22,23,27,26,25,24,28,29,30,31]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <32 x i16> %a0, <32 x i16> undef, <32 x i32> <i32 3, i32 2, i32 1, i32 0, i32 4, i32 5, i32 6, i32 7, i32 11, i32 10, i32 9, i32 8, i32 12, i32 13, i32 14, i32 15, i32 19, i32 18, i32 17, i32 16, i32 20, i32 21, i32 22, i32 23, i32 27, i32 26, i32 25, i32 24, i32 28, i32 29, i32 30, i32 31>
   %3 = bitcast i32 %mask to <32 x i1>
@@ -3480,8 +6254,14 @@ define <32 x i16> @stack_fold_pshuflw_zmm_maskz(<32 x i16> %a0, i32 %mask) {
 }
 
 define <16 x i32> @stack_fold_pslld(<16 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_pslld
-  ;CHECK:       vpslld {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pslld:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpslld {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x i32> @llvm.x86.avx512.psll.d.512(<16 x i32> %a0, <4 x i32> %a1)
   ret <16 x i32> %2
@@ -3489,8 +6269,17 @@ define <16 x i32> @stack_fold_pslld(<16 x i32> %a0, <4 x i32> %a1) {
 declare <16 x i32> @llvm.x86.avx512.psll.d.512(<16 x i32>, <4 x i32>) nounwind readnone
 
 define <16 x i32> @stack_fold_pslld_mask(<16 x i32>* %passthru, <16 x i32> %a0, <4 x i32> %a1, i16 %mask) {
-  ;CHECK-LABEL: stack_fold_pslld_mask
-  ;CHECK:       vpslld {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pslld_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm2
+; CHECK-NEXT:    vpslld {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 16-byte Folded Reload
+; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x i32> @llvm.x86.avx512.psll.d.512(<16 x i32> %a0, <4 x i32> %a1)
   %3 = bitcast i16 %mask to <16 x i1>
@@ -3500,8 +6289,15 @@ define <16 x i32> @stack_fold_pslld_mask(<16 x i32>* %passthru, <16 x i32> %a0,
 }
 
 define <16 x i32> @stack_fold_pslld_maskz(<16 x i32> %a0, <4 x i32> %a1, i16 %mask) {
-  ;CHECK-LABEL: stack_fold_pslld_maskz
-  ;CHECK:       vpslld {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pslld_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpslld {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x i32> @llvm.x86.avx512.psll.d.512(<16 x i32> %a0, <4 x i32> %a1)
   %3 = bitcast i16 %mask to <16 x i1>
@@ -3510,8 +6306,14 @@ define <16 x i32> @stack_fold_pslld_maskz(<16 x i32> %a0, <4 x i32> %a1, i16 %ma
 }
 
 define <16 x i32> @stack_fold_pslldi(<16 x i32> %a0) {
-  ;CHECK-LABEL: stack_fold_pslldi
-  ;CHECK:       vpslld $1, {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pslldi:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpslld $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x i32> @llvm.x86.avx512.pslli.d.512(<16 x i32> %a0, i32 1)
   ret <16 x i32> %2
@@ -3519,8 +6321,17 @@ define <16 x i32> @stack_fold_pslldi(<16 x i32> %a0) {
 declare <16 x i32> @llvm.x86.avx512.pslli.d.512(<16 x i32>, i32) nounwind readnone
 
 define <16 x i32> @stack_fold_pslldi_mask(<16 x i32>* %passthru, <16 x i32> %a0, i16 %mask) {
-  ;CHECK-LABEL: stack_fold_pslldi_mask
-  ;CHECK:       vpslld $1, {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pslldi_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm1
+; CHECK-NEXT:    vpslld $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovdqa64 %zmm1, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x i32> @llvm.x86.avx512.pslli.d.512(<16 x i32> %a0, i32 1)
   %3 = bitcast i16 %mask to <16 x i1>
@@ -3530,8 +6341,15 @@ define <16 x i32> @stack_fold_pslldi_mask(<16 x i32>* %passthru, <16 x i32> %a0,
 }
 
 define <16 x i32> @stack_fold_pslldi_maskz(<16 x i32> %a0, i16 %mask) {
-  ;CHECK-LABEL: stack_fold_pslldi_maskz
-  ;CHECK:       vpslld $1, {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pslldi_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpslld $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x i32> @llvm.x86.avx512.pslli.d.512(<16 x i32> %a0, i32 1)
   %3 = bitcast i16 %mask to <16 x i1>
@@ -3540,16 +6358,29 @@ define <16 x i32> @stack_fold_pslldi_maskz(<16 x i32> %a0, i16 %mask) {
 }
 
 define <64 x i8> @stack_fold_pslldq(<64 x i8> %a, <64 x i8> %b) {
-  ;CHECK-LABEL: stack_fold_pslldq
-  ;CHECK:       vpslldq $1, {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pslldq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpslldq $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    # zmm0 = zero,mem[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14],zero,mem[16,17,18,19,20,21,22,23,24,25,26,27,28,29,30],zero,mem[32,33,34,35,36,37,38,39,40,41,42,43,44,45,46],zero,mem[48,49,50,51,52,53,54,55,56,57,58,59,60,61,62]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <64 x i8> %a, <64 x i8> zeroinitializer, <64 x i32> <i32 79, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 95, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 111, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 127, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62>
   ret <64 x i8> %2
 }
 
 define <8 x i64> @stack_fold_psllq(<8 x i64> %a0, <2 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_psllq
-  ;CHECK:       vpsllq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psllq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsllq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x i64> @llvm.x86.avx512.psll.q.512(<8 x i64> %a0, <2 x i64> %a1)
   ret <8 x i64> %2
@@ -3557,8 +6388,14 @@ define <8 x i64> @stack_fold_psllq(<8 x i64> %a0, <2 x i64> %a1) {
 declare <8 x i64> @llvm.x86.avx512.psll.q.512(<8 x i64>, <2 x i64>) nounwind readnone
 
 define <8 x i64> @stack_fold_psllqi(<8 x i64> %a0) {
-  ;CHECK-LABEL: stack_fold_psllqi
-  ;CHECK:       vpsllq $1, {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_psllqi:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsllq $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x i64> @llvm.x86.avx512.pslli.q.512(<8 x i64> %a0, i32 1)
   ret <8 x i64> %2
@@ -3566,8 +6403,14 @@ define <8 x i64> @stack_fold_psllqi(<8 x i64> %a0) {
 declare <8 x i64> @llvm.x86.avx512.pslli.q.512(<8 x i64>, i32) nounwind readnone
 
 define <16 x i32> @stack_fold_psllvd(<16 x i32> %a0, <16 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_psllvd
-  ;CHECK:       vpsllvd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_psllvd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsllvd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x i32> @llvm.x86.avx512.psllv.d.512(<16 x i32> %a0, <16 x i32> %a1)
   ret <16 x i32> %2
@@ -3575,8 +6418,17 @@ define <16 x i32> @stack_fold_psllvd(<16 x i32> %a0, <16 x i32> %a1) {
 declare <16 x i32> @llvm.x86.avx512.psllv.d.512(<16 x i32>, <16 x i32>) nounwind readnone
 
 define <16 x i32> @stack_fold_psllvd_mask(<16 x i32>* %passthru, <16 x i32> %a0, <16 x i32> %a1, i16 %mask) {
-  ;CHECK-LABEL: stack_fold_psllvd_mask
-  ;CHECK:       vpsllvd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_psllvd_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm2
+; CHECK-NEXT:    vpsllvd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x i32> @llvm.x86.avx512.psllv.d.512(<16 x i32> %a0, <16 x i32> %a1)
   %3 = bitcast i16 %mask to <16 x i1>
@@ -3586,8 +6438,15 @@ define <16 x i32> @stack_fold_psllvd_mask(<16 x i32>* %passthru, <16 x i32> %a0,
 }
 
 define <16 x i32> @stack_fold_psllvd_maskz(<16 x i32> %a0, <16 x i32> %a1, i16 %mask) {
-  ;CHECK-LABEL: stack_fold_psllvd_maskz
-  ;CHECK:       vpsllvd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_psllvd_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpsllvd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x i32> @llvm.x86.avx512.psllv.d.512(<16 x i32> %a0, <16 x i32> %a1)
   %3 = bitcast i16 %mask to <16 x i1>
@@ -3596,8 +6455,14 @@ define <16 x i32> @stack_fold_psllvd_maskz(<16 x i32> %a0, <16 x i32> %a1, i16 %
 }
 
 define <8 x i64> @stack_fold_psllvq(<8 x i64> %a0, <8 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_psllvq
-  ;CHECK:       vpsllvq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_psllvq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsllvq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <8 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x i64> @llvm.x86.avx512.psllv.q.512(<8 x i64> %a0, <8 x i64> %a1)
   ret <8 x i64> %2
@@ -3605,8 +6470,14 @@ define <8 x i64> @stack_fold_psllvq(<8 x i64> %a0, <8 x i64> %a1) {
 declare <8 x i64> @llvm.x86.avx512.psllv.q.512(<8 x i64>, <8 x i64>) nounwind readnone
 
 define <32 x i16> @stack_fold_psllvw(<32 x i16> %a0, <32 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_psllvw
-  ;CHECK:       vpsllvw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_psllvw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsllvw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <32 x i16> @llvm.x86.avx512.psllv.w.512(<32 x i16> %a0, <32 x i16> %a1)
   ret <32 x i16> %2
@@ -3614,8 +6485,14 @@ define <32 x i16> @stack_fold_psllvw(<32 x i16> %a0, <32 x i16> %a1) {
 declare <32 x i16> @llvm.x86.avx512.psllv.w.512(<32 x i16>, <32 x i16>) nounwind readnone
 
 define <32 x i16> @stack_fold_psllw(<32 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_psllw
-  ;CHECK:       vpsllw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psllw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsllw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <32 x i16> @llvm.x86.avx512.psll.w.512(<32 x i16> %a0, <8 x i16> %a1)
   ret <32 x i16> %2
@@ -3623,8 +6500,14 @@ define <32 x i16> @stack_fold_psllw(<32 x i16> %a0, <8 x i16> %a1) {
 declare <32 x i16> @llvm.x86.avx512.psll.w.512(<32 x i16>, <8 x i16>) nounwind readnone
 
 define <32 x i16> @stack_fold_psllwi(<32 x i16> %a0) {
-  ;CHECK-LABEL: stack_fold_psllwi
-  ;CHECK:       vpsllw $1, {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_psllwi:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsllw $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <32 x i16> @llvm.x86.avx512.pslli.w.512(<32 x i16> %a0, i32 1)
   ret <32 x i16> %2
@@ -3632,8 +6515,14 @@ define <32 x i16> @stack_fold_psllwi(<32 x i16> %a0) {
 declare <32 x i16> @llvm.x86.avx512.pslli.w.512(<32 x i16>, i32) nounwind readnone
 
 define <16 x i32> @stack_fold_psrad(<16 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_psrad
-  ;CHECK:       vpsrad {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psrad:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsrad {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x i32> @llvm.x86.avx512.psra.d.512(<16 x i32> %a0, <4 x i32> %a1)
   ret <16 x i32> %2
@@ -3641,8 +6530,14 @@ define <16 x i32> @stack_fold_psrad(<16 x i32> %a0, <4 x i32> %a1) {
 declare <16 x i32> @llvm.x86.avx512.psra.d.512(<16 x i32>, <4 x i32>) nounwind readnone
 
 define <16 x i32> @stack_fold_psradi(<16 x i32> %a0) {
-  ;CHECK-LABEL: stack_fold_psradi
-  ;CHECK:       vpsrad $1, {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_psradi:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsrad $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x i32> @llvm.x86.avx512.psrai.d.512(<16 x i32> %a0, i32 1)
   ret <16 x i32> %2
@@ -3650,8 +6545,14 @@ define <16 x i32> @stack_fold_psradi(<16 x i32> %a0) {
 declare <16 x i32> @llvm.x86.avx512.psrai.d.512(<16 x i32>, i32) nounwind readnone
 
 define <8 x i64> @stack_fold_psraq(<8 x i64> %a0, <2 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_psraq
-  ;CHECK:       vpsraq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psraq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsraq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x i64> @llvm.x86.avx512.psra.q.512(<8 x i64> %a0, <2 x i64> %a1)
   ret <8 x i64> %2
@@ -3659,8 +6560,14 @@ define <8 x i64> @stack_fold_psraq(<8 x i64> %a0, <2 x i64> %a1) {
 declare <8 x i64> @llvm.x86.avx512.psra.q.512(<8 x i64>, <2 x i64>) nounwind readnone
 
 define <8 x i64> @stack_fold_psraqi(<8 x i64> %a0) {
-  ;CHECK-LABEL: stack_fold_psraqi
-  ;CHECK:       vpsraq $1, {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_psraqi:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsraq $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x i64> @llvm.x86.avx512.psrai.q.512(<8 x i64> %a0, i32 1)
   ret <8 x i64> %2
@@ -3668,8 +6575,14 @@ define <8 x i64> @stack_fold_psraqi(<8 x i64> %a0) {
 declare <8 x i64> @llvm.x86.avx512.psrai.q.512(<8 x i64>, i32) nounwind readnone
 
 define <16 x i32> @stack_fold_psravd(<16 x i32> %a0, <16 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_psravd
-  ;CHECK:       vpsravd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_psravd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsravd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x i32> @llvm.x86.avx512.psrav.d.512(<16 x i32> %a0, <16 x i32> %a1)
   ret <16 x i32> %2
@@ -3677,8 +6590,14 @@ define <16 x i32> @stack_fold_psravd(<16 x i32> %a0, <16 x i32> %a1) {
 declare <16 x i32> @llvm.x86.avx512.psrav.d.512(<16 x i32>, <16 x i32>) nounwind readnone
 
 define <8 x i64> @stack_fold_psravq(<8 x i64> %a0, <8 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_psravq
-  ;CHECK:       vpsravq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_psravq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsravq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <8 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x i64> @llvm.x86.avx512.psrav.q.512(<8 x i64> %a0, <8 x i64> %a1)
   ret <8 x i64> %2
@@ -3686,8 +6605,14 @@ define <8 x i64> @stack_fold_psravq(<8 x i64> %a0, <8 x i64> %a1) {
 declare <8 x i64> @llvm.x86.avx512.psrav.q.512(<8 x i64>, <8 x i64>) nounwind readnone
 
 define <32 x i16> @stack_fold_psravw(<32 x i16> %a0, <32 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_psravw
-  ;CHECK:       vpsravw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_psravw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsravw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <32 x i16> @llvm.x86.avx512.psrav.w.512(<32 x i16> %a0, <32 x i16> %a1)
   ret <32 x i16> %2
@@ -3695,8 +6620,14 @@ define <32 x i16> @stack_fold_psravw(<32 x i16> %a0, <32 x i16> %a1) {
 declare <32 x i16> @llvm.x86.avx512.psrav.w.512(<32 x i16>, <32 x i16>) nounwind readnone
 
 define <32 x i16> @stack_fold_psraw(<32 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_psraw
-  ;CHECK:       vpsraw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psraw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsraw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <32 x i16> @llvm.x86.avx512.psra.w.512(<32 x i16> %a0, <8 x i16> %a1)
   ret <32 x i16> %2
@@ -3704,8 +6635,14 @@ define <32 x i16> @stack_fold_psraw(<32 x i16> %a0, <8 x i16> %a1) {
 declare <32 x i16> @llvm.x86.avx512.psra.w.512(<32 x i16>, <8 x i16>) nounwind readnone
 
 define <32 x i16> @stack_fold_psrawi(<32 x i16> %a0) {
-  ;CHECK-LABEL: stack_fold_psrawi
-  ;CHECK:       vpsraw $1, {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_psrawi:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsraw $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <32 x i16> @llvm.x86.avx512.psrai.w.512(<32 x i16> %a0, i32 1)
   ret <32 x i16> %2
@@ -3713,8 +6650,14 @@ define <32 x i16> @stack_fold_psrawi(<32 x i16> %a0) {
 declare <32 x i16> @llvm.x86.avx512.psrai.w.512(<32 x i16>, i32) nounwind readnone
 
 define <16 x i32> @stack_fold_psrld(<16 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_psrld
-  ;CHECK:       vpsrld {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psrld:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsrld {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x i32> @llvm.x86.avx512.psrl.d.512(<16 x i32> %a0, <4 x i32> %a1)
   ret <16 x i32> %2
@@ -3722,8 +6665,14 @@ define <16 x i32> @stack_fold_psrld(<16 x i32> %a0, <4 x i32> %a1) {
 declare <16 x i32> @llvm.x86.avx512.psrl.d.512(<16 x i32>, <4 x i32>) nounwind readnone
 
 define <16 x i32> @stack_fold_psrldi(<16 x i32> %a0) {
-  ;CHECK-LABEL: stack_fold_psrldi
-  ;CHECK:       vpsrld $1, {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_psrldi:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsrld $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x i32> @llvm.x86.avx512.psrli.d.512(<16 x i32> %a0, i32 1)
   ret <16 x i32> %2
@@ -3731,16 +6680,29 @@ define <16 x i32> @stack_fold_psrldi(<16 x i32> %a0) {
 declare <16 x i32> @llvm.x86.avx512.psrli.d.512(<16 x i32>, i32) nounwind readnone
 
 define <64 x i8> @stack_fold_psrldq(<64 x i8> %a, <64 x i8> %b) {
-  ;CHECK-LABEL: stack_fold_psrldq
-  ;CHECK:       vpsrldq $2, {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_psrldq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsrldq $2, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    # zmm0 = mem[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero,mem[18,19,20,21,22,23,24,25,26,27,28,29,30,31],zero,zero,mem[34,35,36,37,38,39,40,41,42,43,44,45,46,47],zero,zero,mem[50,51,52,53,54,55,56,57,58,59,60,61,62,63],zero,zero
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <64 x i8> %a, <64 x i8> zeroinitializer, <64 x i32> <i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 64, i32 64, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 64, i32 64, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 64, i32 64, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 64>
   ret <64 x i8> %2
 }
 
 define <8 x i64> @stack_fold_psrlq(<8 x i64> %a0, <2 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_psrlq
-  ;CHECK:       vpsrlq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psrlq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsrlq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x i64> @llvm.x86.avx512.psrl.q.512(<8 x i64> %a0, <2 x i64> %a1)
   ret <8 x i64> %2
@@ -3748,8 +6710,14 @@ define <8 x i64> @stack_fold_psrlq(<8 x i64> %a0, <2 x i64> %a1) {
 declare <8 x i64> @llvm.x86.avx512.psrl.q.512(<8 x i64>, <2 x i64>) nounwind readnone
 
 define <8 x i64> @stack_fold_psrlqi(<8 x i64> %a0) {
-  ;CHECK-LABEL: stack_fold_psrlqi
-  ;CHECK:       vpsrlq $1, {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_psrlqi:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsrlq $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x i64> @llvm.x86.avx512.psrli.q.512(<8 x i64> %a0, i32 1)
   ret <8 x i64> %2
@@ -3757,8 +6725,14 @@ define <8 x i64> @stack_fold_psrlqi(<8 x i64> %a0) {
 declare <8 x i64> @llvm.x86.avx512.psrli.q.512(<8 x i64>, i32) nounwind readnone
 
 define <16 x i32> @stack_fold_psrlvd(<16 x i32> %a0, <16 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_psrlvd
-  ;CHECK:       vpsrlvd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_psrlvd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsrlvd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x i32> @llvm.x86.avx512.psrlv.d.512(<16 x i32> %a0, <16 x i32> %a1)
   ret <16 x i32> %2
@@ -3766,8 +6740,14 @@ define <16 x i32> @stack_fold_psrlvd(<16 x i32> %a0, <16 x i32> %a1) {
 declare <16 x i32> @llvm.x86.avx512.psrlv.d.512(<16 x i32>, <16 x i32>) nounwind readnone
 
 define <8 x i64> @stack_fold_psrlvq(<8 x i64> %a0, <8 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_psrlvq
-  ;CHECK:       vpsrlvq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_psrlvq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsrlvq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <8 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x i64> @llvm.x86.avx512.psrlv.q.512(<8 x i64> %a0, <8 x i64> %a1)
   ret <8 x i64> %2
@@ -3775,8 +6755,14 @@ define <8 x i64> @stack_fold_psrlvq(<8 x i64> %a0, <8 x i64> %a1) {
 declare <8 x i64> @llvm.x86.avx512.psrlv.q.512(<8 x i64>, <8 x i64>) nounwind readnone
 
 define <32 x i16> @stack_fold_psrlvw(<32 x i16> %a0, <32 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_psrlvw
-  ;CHECK:       vpsrlvw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_psrlvw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsrlvw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <32 x i16> @llvm.x86.avx512.psrlv.w.512(<32 x i16> %a0, <32 x i16> %a1)
   ret <32 x i16> %2
@@ -3784,8 +6770,14 @@ define <32 x i16> @stack_fold_psrlvw(<32 x i16> %a0, <32 x i16> %a1) {
 declare <32 x i16> @llvm.x86.avx512.psrlv.w.512(<32 x i16>, <32 x i16>) nounwind readnone
 
 define <32 x i16> @stack_fold_psrlw(<32 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_psrlw
-  ;CHECK:       vpsrlw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psrlw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsrlw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <32 x i16> @llvm.x86.avx512.psrl.w.512(<32 x i16> %a0, <8 x i16> %a1)
   ret <32 x i16> %2
@@ -3793,8 +6785,14 @@ define <32 x i16> @stack_fold_psrlw(<32 x i16> %a0, <8 x i16> %a1) {
 declare <32 x i16> @llvm.x86.avx512.psrl.w.512(<32 x i16>, <8 x i16>) nounwind readnone
 
 define <32 x i16> @stack_fold_psrlwi(<32 x i16> %a0) {
-  ;CHECK-LABEL: stack_fold_psrlwi
-  ;CHECK:       vpsrlw $1, {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_psrlwi:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsrlw $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <32 x i16> @llvm.x86.avx512.psrli.w.512(<32 x i16> %a0, i32 1)
   ret <32 x i16> %2
@@ -3802,80 +6800,157 @@ define <32 x i16> @stack_fold_psrlwi(<32 x i16> %a0) {
 declare <32 x i16> @llvm.x86.avx512.psrli.w.512(<32 x i16>, i32) nounwind readnone
 
 define <64 x i8> @stack_fold_psubb(<64 x i8> %a0, <64 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_psubb
-  ;CHECK:       vpsubb {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_psubb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsubb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = sub <64 x i8> %a0, %a1
   ret <64 x i8> %2
 }
 
 define <16 x i32> @stack_fold_psubd(<16 x i32> %a0, <16 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_psubd
-  ;CHECK:       vpsubd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_psubd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsubd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = sub <16 x i32> %a0, %a1
   ret <16 x i32> %2
 }
 
 define <8 x i64> @stack_fold_psubq(<8 x i64> %a0, <8 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_psubq
-  ;CHECK:       vpsubq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_psubq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsubq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = sub <8 x i64> %a0, %a1
   ret <8 x i64> %2
 }
 
 define <64 x i8> @stack_fold_psubsb(<64 x i8> %a0, <64 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_psubsb
-  ;CHECK:       vpsubsb {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_psubsb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsubsb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <64 x i8> @llvm.ssub.sat.v64i8(<64 x i8> %a0, <64 x i8> %a1)
   ret <64 x i8> %2
 }
 
 define <32 x i16> @stack_fold_psubsw(<32 x i16> %a0, <32 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_psubsw
-  ;CHECK:       vpsubsw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_psubsw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsubsw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <32 x i16> @llvm.ssub.sat.v32i16(<32 x i16> %a0, <32 x i16> %a1)
   ret <32 x i16> %2
 }
 
 define <64 x i8> @stack_fold_psubusb(<64 x i8> %a0, <64 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_psubusb
-  ;CHECK:       vpsubusb {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_psubusb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsubusb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <64 x i8> @llvm.usub.sat.v64i8(<64 x i8> %a0, <64 x i8> %a1)
   ret <64 x i8> %2
 }
 
 define <32 x i16> @stack_fold_psubusw(<32 x i16> %a0, <32 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_psubusw
-  ;CHECK:       vpsubusw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_psubusw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsubusw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <32 x i16> @llvm.usub.sat.v32i16(<32 x i16> %a0, <32 x i16> %a1)
   ret <32 x i16> %2
 }
 
 define <32 x i16> @stack_fold_psubw(<32 x i16> %a0, <32 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_psubw
-  ;CHECK:       vpsubw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_psubw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsubw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = sub <32 x i16> %a0, %a1
   ret <32 x i16> %2
 }
 
 define <8 x i64> @stack_fold_shufi64x2(<8 x i64> %a, <8 x i64> %b) {
-  ;CHECK-LABEL: stack_fold_shufi64x2
-  ;CHECK:   vshufi64x2 $24, {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_shufi64x2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subq $56, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; CHECK-NEXT:    vshufi64x2 $24, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    # zmm0 = zmm0[0,1,4,5],mem[2,3,0,1]
+; CHECK-NEXT:    addq $56, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 0, i32 1, i32 4, i32 5, i32 10, i32 11, i32 8, i32 9>
   ret <8 x i64> %2
 }
 
 define <8 x i64> @stack_fold_shufi64x2_mask(<8 x i64> %a, <8 x i64> %b, i8 %mask, <8 x i64>* %passthru) {
-  ;CHECK-LABEL: stack_fold_shufi64x2_mask
-  ;CHECK:   vshufi64x2 $24, {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[1-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_shufi64x2_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subq $56, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm1
+; CHECK-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; CHECK-NEXT:    vshufi64x2 $24, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm1 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    # zmm1 {%k1} = zmm0[0,1,4,5],mem[2,3,0,1]
+; CHECK-NEXT:    vmovdqa64 %zmm1, %zmm0
+; CHECK-NEXT:    addq $56, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 0, i32 1, i32 4, i32 5, i32 10, i32 11, i32 8, i32 9>
   %3 = bitcast i8 %mask to <8 x i1>
@@ -3886,8 +6961,22 @@ define <8 x i64> @stack_fold_shufi64x2_mask(<8 x i64> %a, <8 x i64> %b, i8 %mask
 }
 
 define <8 x i64> @stack_fold_shufi64x2_maskz(<8 x i64> %a, <8 x i64> %b, i8 %mask, <8 x i64>* %passthru) {
-  ;CHECK-LABEL: stack_fold_shufi64x2_maskz
-  ;CHECK:   vshufi64x2 $24, {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[1-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_shufi64x2_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subq $56, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; CHECK-NEXT:    vshufi64x2 $24, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    # zmm0 {%k1} {z} = zmm0[0,1,4,5],mem[2,3,0,1]
+; CHECK-NEXT:    addq $56, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 0, i32 1, i32 4, i32 5, i32 10, i32 11, i32 8, i32 9>
   %3 = bitcast i8 %mask to <8 x i1>
@@ -3896,8 +6985,24 @@ define <8 x i64> @stack_fold_shufi64x2_maskz(<8 x i64> %a, <8 x i64> %b, i8 %mas
 }
 
 define <16 x i32> @stack_fold_shufi32x4_mask(<16 x i32> %a, <16 x i32> %b, i16 %mask, <16 x i32>* %passthru) {
-  ;CHECK-LABEL: stack_fold_shufi32x4_mask
-  ;CHECK:   vshufi32x4 $20, {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[1-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_shufi32x4_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subq $56, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm1
+; CHECK-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; CHECK-NEXT:    vshufi32x4 $20, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm1 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    # zmm1 {%k1} = zmm0[0,1,2,3,4,5,6,7],mem[4,5,6,7,0,1,2,3]
+; CHECK-NEXT:    vmovdqa64 %zmm1, %zmm0
+; CHECK-NEXT:    addq $56, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <16 x i32> %a, <16 x i32> %b, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 20, i32 21, i32 22, i32 23, i32 16, i32 17, i32 18, i32 19>
   %3 = bitcast i16 %mask to <16 x i1>
@@ -3908,8 +7013,22 @@ define <16 x i32> @stack_fold_shufi32x4_mask(<16 x i32> %a, <16 x i32> %b, i16 %
 }
 
 define <16 x i32> @stack_fold_shufi32x4_maskz(<16 x i32> %a, <16 x i32> %b, i16 %mask) {
-  ;CHECK-LABEL: stack_fold_shufi32x4_maskz
-  ;CHECK:   vshufi32x4 $20, {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[1-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_shufi32x4_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subq $56, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; CHECK-NEXT:    vshufi32x4 $20, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    # zmm0 {%k1} {z} = zmm0[0,1,2,3,4,5,6,7],mem[4,5,6,7,0,1,2,3]
+; CHECK-NEXT:    addq $56, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <16 x i32> %a, <16 x i32> %b, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 20, i32 21, i32 22, i32 23, i32 16, i32 17, i32 18, i32 19>
   %3 = bitcast i16 %mask to <16 x i1>
@@ -3918,8 +7037,14 @@ define <16 x i32> @stack_fold_shufi32x4_maskz(<16 x i32> %a, <16 x i32> %b, i16
 }
 
 define <16 x i32> @stack_fold_ternlogd(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2) {
-  ;CHECK-LABEL: stack_fold_ternlogd
-  ;CHECK:       vpternlogd $33, {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_ternlogd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpternlogd $33, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x i32> @llvm.x86.avx512.pternlog.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i32 33)
   ret <16 x i32> %2
@@ -3927,8 +7052,14 @@ define <16 x i32> @stack_fold_ternlogd(<16 x i32> %x0, <16 x i32> %x1, <16 x i32
 declare <16 x i32> @llvm.x86.avx512.pternlog.d.512(<16 x i32>, <16 x i32>, <16 x i32>, i32)
 
 define <8 x i64> @stack_fold_ternlogq(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2) {
-  ;CHECK-LABEL: stack_fold_ternlogq
-  ;CHECK:       vpternlogq $33, {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_ternlogq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpternlogq $33, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x i64> @llvm.x86.avx512.pternlog.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i32 33)
   ret <8 x i64> %2
@@ -3937,16 +7068,33 @@ define <8 x i64> @stack_fold_ternlogq(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x
 declare <8 x i64> @llvm.x86.avx512.pternlog.q.512(<8 x i64>, <8 x i64>, <8 x i64>, i32)
 
 define <64 x i8> @stack_fold_punpckhbw_zmm(<64 x i8> %a0, <64 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_punpckhbw_zmm
-  ;CHECK:       vpunpckhbw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_punpckhbw_zmm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpunpckhbw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    # zmm0 = zmm0[8],mem[8],zmm0[9],mem[9],zmm0[10],mem[10],zmm0[11],mem[11],zmm0[12],mem[12],zmm0[13],mem[13],zmm0[14],mem[14],zmm0[15],mem[15],zmm0[24],mem[24],zmm0[25],mem[25],zmm0[26],mem[26],zmm0[27],mem[27],zmm0[28],mem[28],zmm0[29],mem[29],zmm0[30],mem[30],zmm0[31],mem[31],zmm0[40],mem[40],zmm0[41],mem[41],zmm0[42],mem[42],zmm0[43],mem[43],zmm0[44],mem[44],zmm0[45],mem[45],zmm0[46],mem[46],zmm0[47],mem[47],zmm0[56],mem[56],zmm0[57],mem[57],zmm0[58],mem[58],zmm0[59],mem[59],zmm0[60],mem[60],zmm0[61],mem[61],zmm0[62],mem[62],zmm0[63],mem[63]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <64 x i8> %a0, <64 x i8> %a1, <64 x i32> <i32 8, i32 72, i32 9, i32 73, i32 10, i32 74, i32 11, i32 75, i32 12, i32 76, i32 13, i32 77, i32 14, i32 78, i32 15, i32 79, i32 24, i32 88, i32 25, i32 89, i32 26, i32 90, i32 27, i32 91, i32 28, i32 92, i32 29, i32 93, i32 30, i32 94, i32 31, i32 95, i32 40, i32 104, i32 41, i32 105, i32 42, i32 106, i32 43, i32 107, i32 44, i32 108, i32 45, i32 109, i32 46, i32 110, i32 47, i32 111, i32 56, i32 120, i32 57, i32 121, i32 58, i32 122, i32 59, i32 123, i32 60, i32 124, i32 61, i32 125, i32 62, i32 126, i32 63, i32 127>
   ret <64 x i8> %2
 }
 
 define <64 x i8> @stack_fold_punpckhbw_mask_zmm(<64 x i8>* %passthru, <64 x i8> %a0, <64 x i8> %a1, i64 %mask) {
-  ;CHECK-LABEL: stack_fold_punpckhbw_mask_zmm
-  ;CHECK:       vpunpckhbw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_punpckhbw_mask_zmm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovq %rsi, %k1
+; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm2
+; CHECK-NEXT:    vpunpckhbw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    # zmm2 {%k1} = zmm0[8],mem[8],zmm0[9],mem[9],zmm0[10],mem[10],zmm0[11],mem[11],zmm0[12],mem[12],zmm0[13],mem[13],zmm0[14],mem[14],zmm0[15],mem[15],zmm0[24],mem[24],zmm0[25],mem[25],zmm0[26],mem[26],zmm0[27],mem[27],zmm0[28],mem[28],zmm0[29],mem[29],zmm0[30],mem[30],zmm0[31],mem[31],zmm0[40],mem[40],zmm0[41],mem[41],zmm0[42],mem[42],zmm0[43],mem[43],zmm0[44],mem[44],zmm0[45],mem[45],zmm0[46],mem[46],zmm0[47],mem[47],zmm0[56],mem[56],zmm0[57],mem[57],zmm0[58],mem[58],zmm0[59],mem[59],zmm0[60],mem[60],zmm0[61],mem[61],zmm0[62],mem[62],zmm0[63],mem[63]
+; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <64 x i8> %a0, <64 x i8> %a1, <64 x i32> <i32 8, i32 72, i32 9, i32 73, i32 10, i32 74, i32 11, i32 75, i32 12, i32 76, i32 13, i32 77, i32 14, i32 78, i32 15, i32 79, i32 24, i32 88, i32 25, i32 89, i32 26, i32 90, i32 27, i32 91, i32 28, i32 92, i32 29, i32 93, i32 30, i32 94, i32 31, i32 95, i32 40, i32 104, i32 41, i32 105, i32 42, i32 106, i32 43, i32 107, i32 44, i32 108, i32 45, i32 109, i32 46, i32 110, i32 47, i32 111, i32 56, i32 120, i32 57, i32 121, i32 58, i32 122, i32 59, i32 123, i32 60, i32 124, i32 61, i32 125, i32 62, i32 126, i32 63, i32 127>
   %3 = bitcast i64 %mask to <64 x i1>
@@ -3957,8 +7105,16 @@ define <64 x i8> @stack_fold_punpckhbw_mask_zmm(<64 x i8>* %passthru, <64 x i8>
 }
 
 define <64 x i8> @stack_fold_punpckhbw_maskz_zmm(<64 x i8> %a0, <64 x i8> %a1, i64 %mask) {
-  ;CHECK-LABEL: stack_fold_punpckhbw_maskz_zmm
-  ;CHECK:       vpunpckhbw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_punpckhbw_maskz_zmm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovq %rdi, %k1
+; CHECK-NEXT:    vpunpckhbw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    # zmm0 {%k1} {z} = zmm0[8],mem[8],zmm0[9],mem[9],zmm0[10],mem[10],zmm0[11],mem[11],zmm0[12],mem[12],zmm0[13],mem[13],zmm0[14],mem[14],zmm0[15],mem[15],zmm0[24],mem[24],zmm0[25],mem[25],zmm0[26],mem[26],zmm0[27],mem[27],zmm0[28],mem[28],zmm0[29],mem[29],zmm0[30],mem[30],zmm0[31],mem[31],zmm0[40],mem[40],zmm0[41],mem[41],zmm0[42],mem[42],zmm0[43],mem[43],zmm0[44],mem[44],zmm0[45],mem[45],zmm0[46],mem[46],zmm0[47],mem[47],zmm0[56],mem[56],zmm0[57],mem[57],zmm0[58],mem[58],zmm0[59],mem[59],zmm0[60],mem[60],zmm0[61],mem[61],zmm0[62],mem[62],zmm0[63],mem[63]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <64 x i8> %a0, <64 x i8> %a1, <64 x i32> <i32 8, i32 72, i32 9, i32 73, i32 10, i32 74, i32 11, i32 75, i32 12, i32 76, i32 13, i32 77, i32 14, i32 78, i32 15, i32 79, i32 24, i32 88, i32 25, i32 89, i32 26, i32 90, i32 27, i32 91, i32 28, i32 92, i32 29, i32 93, i32 30, i32 94, i32 31, i32 95, i32 40, i32 104, i32 41, i32 105, i32 42, i32 106, i32 43, i32 107, i32 44, i32 108, i32 45, i32 109, i32 46, i32 110, i32 47, i32 111, i32 56, i32 120, i32 57, i32 121, i32 58, i32 122, i32 59, i32 123, i32 60, i32 124, i32 61, i32 125, i32 62, i32 126, i32 63, i32 127>
   %3 = bitcast i64 %mask to <64 x i1>
@@ -3967,24 +7123,45 @@ define <64 x i8> @stack_fold_punpckhbw_maskz_zmm(<64 x i8> %a0, <64 x i8> %a1, i
 }
 
 define <16 x i32> @stack_fold_pxord(<16 x i32> %a0, <16 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_pxord
-  ;CHECK:       vxorps {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pxord:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vxorps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = xor <16 x i32> %a0, %a1
   ret <16 x i32> %2
 }
 
 define <16 x i32> @stack_fold_pxord_commuted(<16 x i32> %a0, <16 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_pxord_commuted:
-  ;CHECK:       vxorps {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pxord_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vxorps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = xor <16 x i32> %a1, %a0
   ret <16 x i32> %2
 }
 
 define <16 x i32> @stack_fold_pxord_mask(<16 x i32> %a0, <16 x i32> %a1, <16 x i32>* %a2, i16 %mask) {
-  ;CHECK-LABEL: stack_fold_pxord_mask
-  ;CHECK:       vxorps {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pxord_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovaps %zmm0, %zmm1
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vmovaps (%rdi), %zmm0
+; CHECK-NEXT:    vxorps {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = xor <16 x i32> %a0, %a1
   %3 = bitcast i16 %mask to <16 x i1>
@@ -3995,8 +7172,17 @@ define <16 x i32> @stack_fold_pxord_mask(<16 x i32> %a0, <16 x i32> %a1, <16 x i
 }
 
 define <16 x i32> @stack_fold_pxord_mask_commuted(<16 x i32> %a0, <16 x i32> %a1, <16 x i32>* %a2, i16 %mask) {
-  ;CHECK-LABEL: stack_fold_pxord_mask_commuted:
-  ;CHECK:       vxorps {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pxord_mask_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovaps %zmm0, %zmm1
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vmovaps (%rdi), %zmm0
+; CHECK-NEXT:    vxorps {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = xor <16 x i32> %a1, %a0
   %3 = bitcast i16 %mask to <16 x i1>
@@ -4007,8 +7193,15 @@ define <16 x i32> @stack_fold_pxord_mask_commuted(<16 x i32> %a0, <16 x i32> %a1
 }
 
 define <16 x i32> @stack_fold_pxord_maskz(<16 x i32> %a0, <16 x i32> %a1, i16 %mask) {
-  ;CHECK-LABEL: stack_fold_pxord_maskz
-  ;CHECK:       vxorps {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pxord_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vxorps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = xor <16 x i32> %a0, %a1
   %3 = bitcast i16 %mask to <16 x i1>
@@ -4017,8 +7210,15 @@ define <16 x i32> @stack_fold_pxord_maskz(<16 x i32> %a0, <16 x i32> %a1, i16 %m
 }
 
 define <16 x i32> @stack_fold_pxord_maskz_commuted(<16 x i32> %a0, <16 x i32> %a1, i16 %mask) {
-  ;CHECK-LABEL: stack_fold_pxord_maskz_commuted:
-  ;CHECK:       vxorps {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pxord_maskz_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vxorps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = xor <16 x i32> %a1, %a0
   %3 = bitcast i16 %mask to <16 x i1>
@@ -4027,24 +7227,45 @@ define <16 x i32> @stack_fold_pxord_maskz_commuted(<16 x i32> %a0, <16 x i32> %a
 }
 
 define <8 x i64> @stack_fold_pxorq(<8 x i64> %a0, <8 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_pxorq
-  ;CHECK:       vxorps {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pxorq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vxorps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = xor <8 x i64> %a0, %a1
   ret <8 x i64> %2
 }
 
 define <8 x i64> @stack_fold_pxorq_commuted(<8 x i64> %a0, <8 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_pxorq_commuted:
-  ;CHECK:       vxorps {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pxorq_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vxorps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = xor <8 x i64> %a1, %a0
   ret <8 x i64> %2
 }
 
 define <8 x i64> @stack_fold_pxorq_mask(<8 x i64> %a0, <8 x i64> %a1, <8 x i64>* %a2, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_pxorq_mask
-  ;CHECK:       vxorpd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pxorq_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovapd %zmm0, %zmm1
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vmovapd (%rdi), %zmm0
+; CHECK-NEXT:    vxorpd {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = xor <8 x i64> %a0, %a1
   %3 = bitcast i8 %mask to <8 x i1>
@@ -4055,8 +7276,17 @@ define <8 x i64> @stack_fold_pxorq_mask(<8 x i64> %a0, <8 x i64> %a1, <8 x i64>*
 }
 
 define <8 x i64> @stack_fold_pxorq_mask_commuted(<8 x i64> %a0, <8 x i64> %a1, <8 x i64>* %a2, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_pxorq_mask_commuted:
-  ;CHECK:       vxorpd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pxorq_mask_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    vmovapd %zmm0, %zmm1
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vmovapd (%rdi), %zmm0
+; CHECK-NEXT:    vxorpd {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = xor <8 x i64> %a1, %a0
   %3 = bitcast i8 %mask to <8 x i1>
@@ -4067,8 +7297,15 @@ define <8 x i64> @stack_fold_pxorq_mask_commuted(<8 x i64> %a0, <8 x i64> %a1, <
 }
 
 define <8 x i64> @stack_fold_pxorq_maskz(<8 x i64> %a0, <8 x i64> %a1, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_pxorq_maskz
-  ;CHECK:       vxorpd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pxorq_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vxorpd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = xor <8 x i64> %a0, %a1
   %3 = bitcast i8 %mask to <8 x i1>
@@ -4077,8 +7314,15 @@ define <8 x i64> @stack_fold_pxorq_maskz(<8 x i64> %a0, <8 x i64> %a1, i8 %mask)
 }
 
 define <8 x i64> @stack_fold_pxorq_maskz_commuted(<8 x i64> %a0, <8 x i64> %a1, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_pxorq_maskz_commuted:
-  ;CHECK:       vxorpd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
+; CHECK-LABEL: stack_fold_pxorq_maskz_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vxorpd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = xor <8 x i64> %a1, %a0
   %3 = bitcast i8 %mask to <8 x i1>
diff --git a/llvm/test/CodeGen/X86/stack-folding-int-avx512vl.ll b/llvm/test/CodeGen/X86/stack-folding-int-avx512vl.ll
index 67a02d5b9..e1eaa1a52 100644
--- a/llvm/test/CodeGen/X86/stack-folding-int-avx512vl.ll
+++ b/llvm/test/CodeGen/X86/stack-folding-int-avx512vl.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -O3 -disable-peephole -mtriple=x86_64-unknown-unknown -mattr=+avx512vl,+avx512bw,+avx512dq,+avx512vbmi,+avx512cd,+avx512vpopcntdq < %s | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -9,16 +10,37 @@ target triple = "x86_64-unknown-unknown"
 ; relevant registers and check that the reload is correctly folded into the instruction.
 
 define <8 x i32> @stack_fold_valignd_ymm(<8 x i32> %a, <8 x i32> %b) {
-  ;CHECK-LABEL: stack_fold_valignd_ymm
-  ;CHECK:   valignd $1, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_valignd_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; CHECK-NEXT:    valignd $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    # ymm0 = mem[1,2,3,4,5,6,7],ymm0[0]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <8 x i32> %a, <8 x i32> %b, <8 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
   ret <8 x i32> %2
 }
 
 define <8 x i32> @stack_fold_valignd_ymm_mask(<8 x i32> %a, <8 x i32> %b, <8 x i32>* %passthru, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_valignd_ymm_mask
-  ;CHECK:   valignd $1, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_valignd_ymm_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vmovdqa (%rdi), %ymm1
+; CHECK-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; CHECK-NEXT:    valignd $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 {%k1} # 32-byte Folded Reload
+; CHECK-NEXT:    # ymm1 {%k1} = mem[1,2,3,4,5,6,7],ymm0[0]
+; CHECK-NEXT:    vmovdqa %ymm1, %ymm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <8 x i32> %a, <8 x i32> %b, <8 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
   %3 = bitcast i8 %mask to <8 x i1>
@@ -28,8 +50,18 @@ define <8 x i32> @stack_fold_valignd_ymm_mask(<8 x i32> %a, <8 x i32> %b, <8 x i
 }
 
 define <8 x i32> @stack_fold_valignd_ymm_maskz(<8 x i32> %a, <8 x i32> %b, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_valignd_ymm_maskz
-  ;CHECK:   valignd $1, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_valignd_ymm_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; CHECK-NEXT:    valignd $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 {%k1} {z} # 32-byte Folded Reload
+; CHECK-NEXT:    # ymm0 {%k1} {z} = mem[1,2,3,4,5,6,7],ymm0[0]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <8 x i32> %a, <8 x i32> %b, <8 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
   %3 = bitcast i8 %mask to <8 x i1>
@@ -38,16 +70,31 @@ define <8 x i32> @stack_fold_valignd_ymm_maskz(<8 x i32> %a, <8 x i32> %b, i8 %m
 }
 
 define <4 x i64> @stack_fold_valignq_ymm(<4 x i64> %a, <4 x i64> %b) {
-  ;CHECK-LABEL: stack_fold_valignq_ymm
-  ;CHECK:   valignq $1, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_valignq_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; CHECK-NEXT:    valignq $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    # ymm0 = mem[1,2,3],ymm0[0]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <4 x i64> %a, <4 x i64> %b, <4 x i32> <i32 1, i32 2, i32 3, i32 4>
   ret <4 x i64> %2
 }
 
 define <16 x i8> @stack_fold_pavgb(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pavgb
-  ;CHECK:       vpavgb {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pavgb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpavgb {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = zext <16 x i8> %a0 to <16 x i16>
   %3 = zext <16 x i8> %a1 to <16 x i16>
@@ -59,8 +106,14 @@ define <16 x i8> @stack_fold_pavgb(<16 x i8> %a0, <16 x i8> %a1) {
 }
 
 define <32 x i8> @stack_fold_pavgb_ymm(<32 x i8> %a0, <32 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pavgb_ymm
-  ;CHECK:       vpavgb {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pavgb_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpavgb {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = zext <32 x i8> %a0 to <32 x i16>
   %3 = zext <32 x i8> %a1 to <32 x i16>
@@ -72,8 +125,14 @@ define <32 x i8> @stack_fold_pavgb_ymm(<32 x i8> %a0, <32 x i8> %a1) {
 }
 
 define <8 x i16> @stack_fold_pavgw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_pavgw
-  ;CHECK:       vpavgw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pavgw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpavgw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = zext <8 x i16> %a0 to <8 x i32>
   %3 = zext <8 x i16> %a1 to <8 x i32>
@@ -85,8 +144,14 @@ define <8 x i16> @stack_fold_pavgw(<8 x i16> %a0, <8 x i16> %a1) {
 }
 
 define <16 x i16> @stack_fold_pavgw_ymm(<16 x i16> %a0, <16 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_pavgw_ymm
-  ;CHECK:       vpavgw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pavgw_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpavgw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = zext <16 x i16> %a0 to <16 x i32>
   %3 = zext <16 x i16> %a1 to <16 x i32>
@@ -98,40 +163,72 @@ define <16 x i16> @stack_fold_pavgw_ymm(<16 x i16> %a0, <16 x i16> %a1) {
 }
 
 define <4 x i32> @stack_fold_vpconflictd(<4 x i32> %a0) {
-  ;CHECK-LABEL: stack_fold_vpconflictd
-  ;CHECK:       vpconflictd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpconflictd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpconflictd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <4 x i32> @llvm.x86.avx512.conflict.d.128(<4 x i32> %a0)
   ret <4 x i32> %2
 }
 
 define <8 x i32> @stack_fold_vpconflictd_ymm(<8 x i32> %a0) {
-  ;CHECK-LABEL: stack_fold_vpconflictd_ymm
-  ;CHECK:       vpconflictd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpconflictd_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpconflictd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x i32> @llvm.x86.avx512.conflict.d.256(<8 x i32> %a0)
   ret <8 x i32> %2
 }
 
 define <2 x i64> @stack_fold_vpconflictq(<2 x i64> %a0) {
-  ;CHECK-LABEL: stack_fold_vpconflictq
-  ;CHECK:       vpconflictq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpconflictq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpconflictq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <2 x i64> @llvm.x86.avx512.conflict.q.128(<2 x i64> %a0)
   ret <2 x i64> %2
 }
 
 define <4 x i64> @stack_fold_vpconflictq_ymm(<4 x i64> %a0) {
-  ;CHECK-LABEL: stack_fold_vpconflictq_ymm
-  ;CHECK:       vpconflictq {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpconflictq_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpconflictq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <4 x i64> @llvm.x86.avx512.conflict.q.256(<4 x i64> %a0)
   ret <4 x i64> %2
 }
 
 define <4 x i32> @stack_fold_extracti32x4(<8 x i16> %a0, <8 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_extracti32x4
-  ;CHECK:       vextracti128 $1, {{%ymm[0-9][0-9]*}}, {{-?[0-9]*}}(%rsp) {{.*#+}} 16-byte Folded Spill
+; CHECK-LABEL: stack_fold_extracti32x4:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; CHECK-NEXT:    vextracti128 $1, %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Folded Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
   ; zext forces execution domain
   %1 = zext <8 x i16> %a0 to <8 x i32>
   %2 = shufflevector <8 x i32> %1, <8 x i32> %a1, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
@@ -140,8 +237,16 @@ define <4 x i32> @stack_fold_extracti32x4(<8 x i16> %a0, <8 x i32> %a1) {
 }
 
 define <2 x i64> @stack_fold_extracti64x2(<4 x i32> %a0, <4 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_extracti64x2
-  ;CHECK:       vextracti128 $1, {{%ymm[0-9][0-9]*}}, {{-?[0-9]*}}(%rsp) {{.*#+}} 16-byte Folded Spill
+; CHECK-LABEL: stack_fold_extracti64x2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; CHECK-NEXT:    vextracti128 $1, %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Folded Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
   ; zext forces execution domain
   %1 = zext <4 x i32> %a0 to <4 x i64>
   %2 = shufflevector <4 x i64> %1, <4 x i64> %a1, <2 x i32> <i32 2, i32 3>
@@ -150,8 +255,17 @@ define <2 x i64> @stack_fold_extracti64x2(<4 x i32> %a0, <4 x i64> %a1) {
 }
 
 define <8 x i32> @stack_fold_inserti32x4(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_inserti32x4
-  ;CHECK:       vinserti128 $1, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_inserti32x4:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vinserti128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 16-byte Folded Reload
+; CHECK-NEXT:    vpcmpeqd %ymm1, %ymm1, %ymm1
+; CHECK-NEXT:    vpsubd %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <4 x i32> %a0, <4 x i32> %a1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   ; add forces execution domain
@@ -160,8 +274,17 @@ define <8 x i32> @stack_fold_inserti32x4(<4 x i32> %a0, <4 x i32> %a1) {
 }
 
 define <4 x i64> @stack_fold_inserti64x2(<2 x i64> %a0, <2 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_inserti64x2
-  ;CHECK:       vinserti128 $1, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_inserti64x2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vinserti128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 16-byte Folded Reload
+; CHECK-NEXT:    vpcmpeqd %ymm1, %ymm1, %ymm1
+; CHECK-NEXT:    vpsubq %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <2 x i64> %a0, <2 x i64> %a1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   ; add forces execution domain
@@ -170,8 +293,14 @@ define <4 x i64> @stack_fold_inserti64x2(<2 x i64> %a0, <2 x i64> %a1) {
 }
 
 define <16 x i8> @stack_fold_pabsb(<16 x i8> %a0) {
-  ;CHECK-LABEL: stack_fold_pabsb
-  ;CHECK:       vpabsb {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pabsb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpabsb {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp sgt <16 x i8> %a0, zeroinitializer
   %3 = sub <16 x i8> zeroinitializer, %a0
@@ -180,8 +309,14 @@ define <16 x i8> @stack_fold_pabsb(<16 x i8> %a0) {
 }
 
 define <32 x i8> @stack_fold_pabsb_ymm(<32 x i8> %a0) {
-  ;CHECK-LABEL: stack_fold_pabsb_ymm
-  ;CHECK:       vpabsb {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pabsb_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpabsb {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp sgt <32 x i8> %a0, zeroinitializer
   %3 = sub <32 x i8> zeroinitializer, %a0
@@ -190,8 +325,14 @@ define <32 x i8> @stack_fold_pabsb_ymm(<32 x i8> %a0) {
 }
 
 define <4 x i32> @stack_fold_pabsd(<4 x i32> %a0) {
-  ;CHECK-LABEL: stack_fold_pabsd
-  ;CHECK:       vpabsd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pabsd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpabsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp sgt <4 x i32> %a0, zeroinitializer
   %3 = sub <4 x i32> zeroinitializer, %a0
@@ -200,8 +341,14 @@ define <4 x i32> @stack_fold_pabsd(<4 x i32> %a0) {
 }
 
 define <8 x i32> @stack_fold_pabsd_ymm(<8 x i32> %a0) {
-  ;CHECK-LABEL: stack_fold_pabsd_ymm
-  ;CHECK:       vpabsd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pabsd_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpabsd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp sgt <8 x i32> %a0, zeroinitializer
   %3 = sub <8 x i32> zeroinitializer, %a0
@@ -210,8 +357,14 @@ define <8 x i32> @stack_fold_pabsd_ymm(<8 x i32> %a0) {
 }
 
 define <2 x i64> @stack_fold_pabsq(<2 x i64> %a0) {
-  ;CHECK-LABEL: stack_fold_pabsq
-  ;CHECK:       vpabsq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pabsq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpabsq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp sgt <2 x i64> %a0, zeroinitializer
   %3 = sub <2 x i64> zeroinitializer, %a0
@@ -220,8 +373,14 @@ define <2 x i64> @stack_fold_pabsq(<2 x i64> %a0) {
 }
 
 define <4 x i64> @stack_fold_pabsq_ymm(<4 x i64> %a0) {
-  ;CHECK-LABEL: stack_fold_pabsq_ymm
-  ;CHECK:       vpabsq {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pabsq_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpabsq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp sgt <4 x i64> %a0, zeroinitializer
   %3 = sub <4 x i64> zeroinitializer, %a0
@@ -230,8 +389,14 @@ define <4 x i64> @stack_fold_pabsq_ymm(<4 x i64> %a0) {
 }
 
 define <8 x i16> @stack_fold_pabsw(<8 x i16> %a0) {
-  ;CHECK-LABEL: stack_fold_pabsw
-  ;CHECK:       vpabsw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pabsw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpabsw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp sgt <8 x i16> %a0, zeroinitializer
   %3 = sub <8 x i16> zeroinitializer, %a0
@@ -240,8 +405,14 @@ define <8 x i16> @stack_fold_pabsw(<8 x i16> %a0) {
 }
 
 define <16 x i16> @stack_fold_pabsw_ymm(<16 x i16> %a0) {
-  ;CHECK-LABEL: stack_fold_pabsw_ymm
-  ;CHECK:       vpabsw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pabsw_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpabsw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp sgt <16 x i16> %a0, zeroinitializer
   %3 = sub <16 x i16> zeroinitializer, %a0
@@ -250,8 +421,14 @@ define <16 x i16> @stack_fold_pabsw_ymm(<16 x i16> %a0) {
 }
 
 define <8 x i16> @stack_fold_packssdw(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_packssdw
-  ;CHECK:       vpackssdw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_packssdw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpackssdw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x i16> @llvm.x86.sse2.packssdw.128(<4 x i32> %a0, <4 x i32> %a1)
   ret <8 x i16> %2
@@ -259,8 +436,14 @@ define <8 x i16> @stack_fold_packssdw(<4 x i32> %a0, <4 x i32> %a1) {
 declare <8 x i16> @llvm.x86.sse2.packssdw.128(<4 x i32>, <4 x i32>) nounwind readnone
 
 define <16 x i16> @stack_fold_packssdw_ymm(<8 x i32> %a0, <8 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_packssdw_ymm
-  ;CHECK:       vpackssdw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_packssdw_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpackssdw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x i16> @llvm.x86.avx2.packssdw(<8 x i32> %a0, <8 x i32> %a1)
   ret <16 x i16> %2
@@ -268,8 +451,14 @@ define <16 x i16> @stack_fold_packssdw_ymm(<8 x i32> %a0, <8 x i32> %a1) {
 declare <16 x i16> @llvm.x86.avx2.packssdw(<8 x i32>, <8 x i32>) nounwind readnone
 
 define <16 x i8> @stack_fold_packsswb(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_packsswb
-  ;CHECK:       vpacksswb {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_packsswb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpacksswb {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x i8> @llvm.x86.sse2.packsswb.128(<8 x i16> %a0, <8 x i16> %a1)
   ret <16 x i8> %2
@@ -277,8 +466,14 @@ define <16 x i8> @stack_fold_packsswb(<8 x i16> %a0, <8 x i16> %a1) {
 declare <16 x i8> @llvm.x86.sse2.packsswb.128(<8 x i16>, <8 x i16>) nounwind readnone
 
 define <32 x i8> @stack_fold_packsswb_ymm(<16 x i16> %a0, <16 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_packsswb_ymm
-  ;CHECK:       vpacksswb {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_packsswb_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpacksswb {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <32 x i8> @llvm.x86.avx2.packsswb(<16 x i16> %a0, <16 x i16> %a1)
   ret <32 x i8> %2
@@ -286,8 +481,14 @@ define <32 x i8> @stack_fold_packsswb_ymm(<16 x i16> %a0, <16 x i16> %a1) {
 declare <32 x i8> @llvm.x86.avx2.packsswb(<16 x i16>, <16 x i16>) nounwind readnone
 
 define <8 x i16> @stack_fold_packusdw(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_packusdw
-  ;CHECK:       vpackusdw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_packusdw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpackusdw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x i16> @llvm.x86.sse41.packusdw(<4 x i32> %a0, <4 x i32> %a1)
   ret <8 x i16> %2
@@ -295,8 +496,14 @@ define <8 x i16> @stack_fold_packusdw(<4 x i32> %a0, <4 x i32> %a1) {
 declare <8 x i16> @llvm.x86.sse41.packusdw(<4 x i32>, <4 x i32>) nounwind readnone
 
 define <16 x i16> @stack_fold_packusdw_ymm(<8 x i32> %a0, <8 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_packusdw_ymm
-  ;CHECK:       vpackusdw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_packusdw_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpackusdw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x i16> @llvm.x86.avx2.packusdw(<8 x i32> %a0, <8 x i32> %a1)
   ret <16 x i16> %2
@@ -304,8 +511,14 @@ define <16 x i16> @stack_fold_packusdw_ymm(<8 x i32> %a0, <8 x i32> %a1) {
 declare <16 x i16> @llvm.x86.avx2.packusdw(<8 x i32>, <8 x i32>) nounwind readnone
 
 define <16 x i8> @stack_fold_packuswb(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_packuswb
-  ;CHECK:       vpackuswb {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_packuswb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpackuswb {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x i8> @llvm.x86.sse2.packuswb.128(<8 x i16> %a0, <8 x i16> %a1)
   ret <16 x i8> %2
@@ -313,8 +526,14 @@ define <16 x i8> @stack_fold_packuswb(<8 x i16> %a0, <8 x i16> %a1) {
 declare <16 x i8> @llvm.x86.sse2.packuswb.128(<8 x i16>, <8 x i16>) nounwind readnone
 
 define <32 x i8> @stack_fold_packuswb_ymm(<16 x i16> %a0, <16 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_packuswb_ymm
-  ;CHECK:       vpackuswb {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_packuswb_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpackuswb {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <32 x i8> @llvm.x86.avx2.packuswb(<16 x i16> %a0, <16 x i16> %a1)
   ret <32 x i8> %2
@@ -322,16 +541,31 @@ define <32 x i8> @stack_fold_packuswb_ymm(<16 x i16> %a0, <16 x i16> %a1) {
 declare <32 x i8> @llvm.x86.avx2.packuswb(<16 x i16>, <16 x i16>) nounwind readnone
 
 define <16 x i8> @stack_fold_paddb(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_paddb
-  ;CHECK:       vpaddb {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpaddb {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = add <16 x i8> %a0, %a1
   ret <16 x i8> %2
 }
 
 define <16 x i8> @stack_fold_paddb_mask(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %a2, i16 %mask) {
-  ;CHECK-LABEL: stack_fold_paddb_mask
-  ;CHECK:       vpaddb {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddb_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vmovdqa (%rdi), %xmm2
+; CHECK-NEXT:    vpaddb {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm2 {%k1} # 16-byte Folded Reload
+; CHECK-NEXT:    vmovdqa %xmm2, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = add <16 x i8> %a0, %a1
   %3 = bitcast i16 %mask to <16 x i1>
@@ -342,8 +576,15 @@ define <16 x i8> @stack_fold_paddb_mask(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>*
 }
 
 define <16 x i8> @stack_fold_paddb_maskz(<16 x i8> %a0, <16 x i8> %a1, i16 %mask) {
-  ;CHECK-LABEL: stack_fold_paddb_maskz
-  ;CHECK:       vpaddb {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddb_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpaddb {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 {%k1} {z} # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = add <16 x i8> %a0, %a1
   %3 = bitcast i16 %mask to <16 x i1>
@@ -352,16 +593,31 @@ define <16 x i8> @stack_fold_paddb_maskz(<16 x i8> %a0, <16 x i8> %a1, i16 %mask
 }
 
 define <32 x i8> @stack_fold_paddb_ymm(<32 x i8> %a0, <32 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_paddb_ymm
-  ;CHECK:       vpaddb {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddb_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpaddb {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = add <32 x i8> %a0, %a1
   ret <32 x i8> %2
 }
 
 define <32 x i8> @stack_fold_paddb_mask_ymm(<32 x i8> %a0, <32 x i8> %a1, <32 x i8>* %a2, i32 %mask) {
-  ;CHECK-LABEL: stack_fold_paddb_mask_ymm
-  ;CHECK:       vpaddb {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddb_mask_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vmovdqa (%rdi), %ymm2
+; CHECK-NEXT:    vpaddb {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm2 {%k1} # 32-byte Folded Reload
+; CHECK-NEXT:    vmovdqa %ymm2, %ymm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = add <32 x i8> %a0, %a1
   %3 = bitcast i32 %mask to <32 x i1>
@@ -372,8 +628,15 @@ define <32 x i8> @stack_fold_paddb_mask_ymm(<32 x i8> %a0, <32 x i8> %a1, <32 x
 }
 
 define <32 x i8> @stack_fold_paddb_maskz_ymm(<32 x i8> %a0, <32 x i8> %a1, i32 %mask) {
-  ;CHECK-LABEL: stack_fold_paddb_maskz_ymm
-  ;CHECK:       vpaddb {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddb_maskz_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpaddb {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 {%k1} {z} # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = add <32 x i8> %a0, %a1
   %3 = bitcast i32 %mask to <32 x i1>
@@ -382,40 +645,70 @@ define <32 x i8> @stack_fold_paddb_maskz_ymm(<32 x i8> %a0, <32 x i8> %a1, i32 %
 }
 
 define <4 x i32> @stack_fold_paddd(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_paddd
-  ;CHECK:       vpaddd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpaddd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = add <4 x i32> %a0, %a1
   ret <4 x i32> %2
 }
 
 define <8 x i32> @stack_fold_paddd_ymm(<8 x i32> %a0, <8 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_paddd_ymm
-  ;CHECK:       vpaddd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddd_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpaddd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = add <8 x i32> %a0, %a1
   ret <8 x i32> %2
 }
 
 define <2 x i64> @stack_fold_paddq(<2 x i64> %a0, <2 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_paddq
-  ;CHECK:       vpaddq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpaddq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = add <2 x i64> %a0, %a1
   ret <2 x i64> %2
 }
 
 define <4 x i64> @stack_fold_paddq_ymm(<4 x i64> %a0, <4 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_paddq_ymm
-  ;CHECK:       vpaddq {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddq_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpaddq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = add <4 x i64> %a0, %a1
   ret <4 x i64> %2
 }
 
 define <16 x i8> @stack_fold_paddsb(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_paddsb
-  ;CHECK:       vpaddsb {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddsb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpaddsb {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> %a0, <16 x i8> %a1)
   ret <16 x i8> %2
@@ -423,8 +716,14 @@ define <16 x i8> @stack_fold_paddsb(<16 x i8> %a0, <16 x i8> %a1) {
 declare <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8>, <16 x i8>) nounwind readnone
 
 define <32 x i8> @stack_fold_paddsb_ymm(<32 x i8> %a0, <32 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_paddsb_ymm
-  ;CHECK:       vpaddsb {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddsb_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpaddsb {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <32 x i8> @llvm.sadd.sat.v32i8(<32 x i8> %a0, <32 x i8> %a1)
   ret <32 x i8> %2
@@ -432,8 +731,14 @@ define <32 x i8> @stack_fold_paddsb_ymm(<32 x i8> %a0, <32 x i8> %a1) {
 declare <32 x i8> @llvm.sadd.sat.v32i8(<32 x i8>, <32 x i8>) nounwind readnone
 
 define <8 x i16> @stack_fold_paddsw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_paddsw
-  ;CHECK:       vpaddsw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddsw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpaddsw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> %a0, <8 x i16> %a1)
   ret <8 x i16> %2
@@ -441,8 +746,14 @@ define <8 x i16> @stack_fold_paddsw(<8 x i16> %a0, <8 x i16> %a1) {
 declare <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16>, <8 x i16>) nounwind readnone
 
 define <16 x i16> @stack_fold_paddsw_ymm(<16 x i16> %a0, <16 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_paddsw_ymm
-  ;CHECK:       vpaddsw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddsw_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpaddsw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16> %a0, <16 x i16> %a1)
   ret <16 x i16> %2
@@ -450,8 +761,14 @@ define <16 x i16> @stack_fold_paddsw_ymm(<16 x i16> %a0, <16 x i16> %a1) {
 declare <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16>, <16 x i16>) nounwind readnone
 
 define <16 x i8> @stack_fold_paddusb(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_paddusb
-  ;CHECK:       vpaddusb {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddusb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpaddusb {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> %a0, <16 x i8> %a1)
   ret <16 x i8> %2
@@ -459,8 +776,14 @@ define <16 x i8> @stack_fold_paddusb(<16 x i8> %a0, <16 x i8> %a1) {
 declare <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8>, <16 x i8>) nounwind readnone
 
 define <32 x i8> @stack_fold_paddusb_ymm(<32 x i8> %a0, <32 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_paddusb_ymm
-  ;CHECK:       vpaddusb {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddusb_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpaddusb {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <32 x i8> @llvm.uadd.sat.v32i8(<32 x i8> %a0, <32 x i8> %a1)
   ret <32 x i8> %2
@@ -468,8 +791,14 @@ define <32 x i8> @stack_fold_paddusb_ymm(<32 x i8> %a0, <32 x i8> %a1) {
 declare <32 x i8> @llvm.uadd.sat.v32i8(<32 x i8>, <32 x i8>) nounwind readnone
 
 define <8 x i16> @stack_fold_paddusw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_paddusw
-  ;CHECK:       vpaddusw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddusw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpaddusw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> %a0, <8 x i16> %a1)
   ret <8 x i16> %2
@@ -477,8 +806,14 @@ define <8 x i16> @stack_fold_paddusw(<8 x i16> %a0, <8 x i16> %a1) {
 declare <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16>, <8 x i16>) nounwind readnone
 
 define <16 x i16> @stack_fold_paddusw_ymm(<16 x i16> %a0, <16 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_paddusw_ymm
-  ;CHECK:       vpaddusw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddusw_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpaddusw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x i16> @llvm.uadd.sat.v16i16(<16 x i16> %a0, <16 x i16> %a1)
   ret <16 x i16> %2
@@ -486,32 +821,65 @@ define <16 x i16> @stack_fold_paddusw_ymm(<16 x i16> %a0, <16 x i16> %a1) {
 declare <16 x i16> @llvm.uadd.sat.v16i16(<16 x i16>, <16 x i16>) nounwind readnone
 
 define <8 x i16> @stack_fold_paddw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_paddw
-  ;CHECK:       vpaddw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpaddw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = add <8 x i16> %a0, %a1
   ret <8 x i16> %2
 }
 
 define <16 x i16> @stack_fold_paddw_ymm(<16 x i16> %a0, <16 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_paddw_ymm
-  ;CHECK:       vpaddw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddw_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpaddw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = add <16 x i16> %a0, %a1
   ret <16 x i16> %2
 }
 
 define <32 x i8> @stack_fold_palignr(<32 x i8> %a0, <32 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_palignr
-  ;CHECK:       vpalignr $1, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_palignr:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; CHECK-NEXT:    vpalignr $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    # ymm0 = mem[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],ymm0[0],mem[17,18,19,20,21,22,23,24,25,26,27,28,29,30,31],ymm0[16]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <32 x i8> %a1, <32 x i8> %a0, <32 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 32, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 48>
   ret <32 x i8> %2
 }
 
 define <32 x i8> @stack_fold_palignr_mask(<32 x i8> %a0, <32 x i8> %a1, <32 x i8>* %passthru, i32 %mask) {
-  ;CHECK-LABEL: stack_fold_palignr_mask
-  ;CHECK:       vpalignr $1, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_palignr_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vmovdqa (%rdi), %ymm1
+; CHECK-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; CHECK-NEXT:    vpalignr $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 {%k1} # 32-byte Folded Reload
+; CHECK-NEXT:    # ymm1 {%k1} = mem[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],ymm0[0],mem[17,18,19,20,21,22,23,24,25,26,27,28,29,30,31],ymm0[16]
+; CHECK-NEXT:    vmovdqa %ymm1, %ymm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <32 x i8> %a1, <32 x i8> %a0, <32 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 32, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 48>
   %3 = bitcast i32 %mask to <32 x i1>
@@ -521,8 +889,18 @@ define <32 x i8> @stack_fold_palignr_mask(<32 x i8> %a0, <32 x i8> %a1, <32 x i8
 }
 
 define <32 x i8> @stack_fold_palignr_maskz(<32 x i8> %a0, <32 x i8> %a1, i32 %mask) {
-  ;CHECK-LABEL: stack_fold_palignr_maskz
-  ;CHECK:       vpalignr $1, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_palignr_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; CHECK-NEXT:    vpalignr $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 {%k1} {z} # 32-byte Folded Reload
+; CHECK-NEXT:    # ymm0 {%k1} {z} = mem[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],ymm0[0],mem[17,18,19,20,21,22,23,24,25,26,27,28,29,30,31],ymm0[16]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <32 x i8> %a1, <32 x i8> %a0, <32 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 32, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 48>
   %3 = bitcast i32 %mask to <32 x i1>
@@ -531,8 +909,16 @@ define <32 x i8> @stack_fold_palignr_maskz(<32 x i8> %a0, <32 x i8> %a1, i32 %ma
 }
 
 define i16 @stack_fold_pcmpeqb(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pcmpeqb
-  ;CHECK:       vpcmpeqb {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%k[0-7]}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pcmpeqb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpcmpeqb {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %k0 # 16-byte Folded Reload
+; CHECK-NEXT:    kmovd %k0, %eax
+; CHECK-NEXT:    # kill: def $ax killed $ax killed $eax
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp eq <16 x i8> %a0, %a1
   %3 = bitcast <16 x i1> %2 to i16
@@ -540,8 +926,16 @@ define i16 @stack_fold_pcmpeqb(<16 x i8> %a0, <16 x i8> %a1) {
 }
 
 define i8 @stack_fold_pcmpeqd(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_pcmpeqd
-  ;CHECK:       vpcmpeqd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%k[0-7]}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pcmpeqd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpcmpeqd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %k0 # 16-byte Folded Reload
+; CHECK-NEXT:    kmovd %k0, %eax
+; CHECK-NEXT:    # kill: def $al killed $al killed $eax
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp eq <4 x i32> %a0, %a1
   %3 = shufflevector <4 x i1> %2, <4 x i1> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -550,8 +944,16 @@ define i8 @stack_fold_pcmpeqd(<4 x i32> %a0, <4 x i32> %a1) {
 }
 
 define i8 @stack_fold_pcmpeqq(<2 x i64> %a0, <2 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_pcmpeqq
-  ;CHECK:       vpcmpeqq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%k[0-7]}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pcmpeqq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpcmpeqq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %k0 # 16-byte Folded Reload
+; CHECK-NEXT:    kmovd %k0, %eax
+; CHECK-NEXT:    # kill: def $al killed $al killed $eax
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp eq <2 x i64> %a0, %a1
   %3 = shufflevector <2 x i1> %2, <2 x i1> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 2, i32 3, i32 2, i32 3>
@@ -560,8 +962,16 @@ define i8 @stack_fold_pcmpeqq(<2 x i64> %a0, <2 x i64> %a1) {
 }
 
 define i8 @stack_fold_pcmpeqw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_pcmpeqw
-  ;CHECK:       vpcmpeqw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%k[0-7]}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pcmpeqw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpcmpeqw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %k0 # 16-byte Folded Reload
+; CHECK-NEXT:    kmovd %k0, %eax
+; CHECK-NEXT:    # kill: def $al killed $al killed $eax
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp eq <8 x i16> %a0, %a1
   %3 = bitcast <8 x i1> %2 to i8
@@ -569,8 +979,18 @@ define i8 @stack_fold_pcmpeqw(<8 x i16> %a0, <8 x i16> %a1) {
 }
 
 define <32 x i8> @stack_fold_permbvar(<32 x i8> %a0, <32 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_permbvar
-  ;CHECK:   vpermb {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_permbvar:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; CHECK-NEXT:    vpermb {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    vpcmpeqd %ymm1, %ymm1, %ymm1
+; CHECK-NEXT:    vpsubb %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <32 x i8> @llvm.x86.avx512.permvar.qi.256(<32 x i8> %a1, <32 x i8> %a0)
   ; add forces execution domain
@@ -580,8 +1000,16 @@ define <32 x i8> @stack_fold_permbvar(<32 x i8> %a0, <32 x i8> %a1) {
 declare <32 x i8> @llvm.x86.avx512.permvar.qi.256(<32 x i8>, <32 x i8>) nounwind readonly
 
 define <8 x i32> @stack_fold_permd(<8 x i32> %a0, <8 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_permd
-  ;CHECK:   vpermd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_permd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    vpcmpeqd %ymm1, %ymm1, %ymm1
+; CHECK-NEXT:    vpsubd %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x i32> @llvm.x86.avx2.permd(<8 x i32> %a1, <8 x i32> %a0)
   ; add forces execution domain
@@ -591,72 +1019,129 @@ define <8 x i32> @stack_fold_permd(<8 x i32> %a0, <8 x i32> %a1) {
 declare <8 x i32> @llvm.x86.avx2.permd(<8 x i32>, <8 x i32>) nounwind readonly
 
 define <16 x i8> @stack_fold_vpermi2b(<16 x i8> %x0, <16 x i8> %x1, <16 x i8> %x2) {
-  ;CHECK-LABEL: stack_fold_vpermi2b
-  ;CHECK:       vpermi2b {{-?[0-9]*}}(%rsp), %xmm1, %xmm0 # 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpermi2b:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermi2b {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x i8> @llvm.x86.avx512.vpermi2var.qi.128(<16 x i8> %x1, <16 x i8> %x0, <16 x i8> %x2)
   ret <16 x i8> %2
 }
 
 define <32 x i8> @stack_fold_vpermi2b_ymm(<32 x i8> %x0, <32 x i8> %x1, <32 x i8> %x2) {
-  ;CHECK-LABEL: stack_fold_vpermi2b_ymm
-  ;CHECK:       vpermi2b {{-?[0-9]*}}(%rsp), %ymm1, %ymm0 # 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpermi2b_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermi2b {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <4 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <32 x i8> @llvm.x86.avx512.vpermi2var.qi.256(<32 x i8> %x1, <32 x i8> %x0, <32 x i8> %x2)
   ret <32 x i8> %2
 }
 
 define <4 x i32> @stack_fold_vpermi2d(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2) {
-  ;CHECK-LABEL: stack_fold_vpermi2d
-  ;CHECK:       vpermi2d {{-?[0-9]*}}(%rsp), %xmm1, %xmm0 # 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpermi2d:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermi2d {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <4 x i32> @llvm.x86.avx512.vpermi2var.d.128(<4 x i32> %x1, <4 x i32> %x0, <4 x i32> %x2)
   ret <4 x i32> %2
 }
 
 define <8 x i32> @stack_fold_vpermi2d_ymm(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2) {
-  ;CHECK-LABEL: stack_fold_vpermi2d_ymm
-  ;CHECK:       vpermi2d {{-?[0-9]*}}(%rsp), %ymm1, %ymm0 # 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpermi2d_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermi2d {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <4 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x i32> @llvm.x86.avx512.vpermi2var.d.256(<8 x i32> %x1, <8 x i32> %x0, <8 x i32> %x2)
   ret <8 x i32> %2
 }
 
 define <2 x i64> @stack_fold_vpermi2q(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2) {
-  ;CHECK-LABEL: stack_fold_vpermi2q
-  ;CHECK:       vpermi2q {{-?[0-9]*}}(%rsp), %xmm1, %xmm0 # 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpermi2q:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermi2q {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <2 x i64> @llvm.x86.avx512.vpermi2var.q.128(<2 x i64> %x1, <2 x i64> %x0, <2 x i64> %x2)
   ret <2 x i64> %2
 }
 
 define <4 x i64> @stack_fold_vpermi2q_ymm(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2) {
-  ;CHECK-LABEL: stack_fold_vpermi2q_ymm
-  ;CHECK:       vpermi2q {{-?[0-9]*}}(%rsp), %ymm1, %ymm0 # 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpermi2q_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermi2q {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <4 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <4 x i64> @llvm.x86.avx512.vpermi2var.q.256(<4 x i64> %x1, <4 x i64> %x0, <4 x i64> %x2)
   ret <4 x i64> %2
 }
 
 define <8 x i16> @stack_fold_vpermi2w(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2) {
-  ;CHECK-LABEL: stack_fold_vpermi2w
-  ;CHECK:       vpermi2w {{-?[0-9]*}}(%rsp), %xmm1, %xmm0 # 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpermi2w:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermi2w {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x i16> @llvm.x86.avx512.vpermi2var.hi.128(<8 x i16> %x1, <8 x i16> %x0, <8 x i16> %x2)
   ret <8 x i16> %2
 }
 
 define <16 x i16> @stack_fold_vpermi2w_ymm(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2) {
-  ;CHECK-LABEL: stack_fold_vpermi2w_ymm
-  ;CHECK:       vpermi2w {{-?[0-9]*}}(%rsp), %ymm1, %ymm0 # 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpermi2w_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermi2w {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <4 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x i16> @llvm.x86.avx512.vpermi2var.hi.256(<16 x i16> %x1, <16 x i16> %x0, <16 x i16> %x2)
   ret <16 x i16> %2
 }
 
 define <4 x i64> @stack_fold_permq(<4 x i64> %a0) {
-  ;CHECK-LABEL: stack_fold_permq
-  ;CHECK:   vpermq $235, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_permq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermq $235, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    # ymm0 = mem[3,2,2,3]
+; CHECK-NEXT:    vpcmpeqd %ymm1, %ymm1, %ymm1
+; CHECK-NEXT:    vpsubq %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <4 x i64> %a0, <4 x i64> undef, <4 x i32> <i32 3, i32 2, i32 2, i32 3>
   ; add forces execution domain
@@ -665,8 +1150,18 @@ define <4 x i64> @stack_fold_permq(<4 x i64> %a0) {
 }
 
 define <4 x i64> @stack_fold_permqvar(<4 x i64> %a0, <4 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_permqvar
-  ;CHECK:   vpermq {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_permqvar:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; CHECK-NEXT:    vpermq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    vpcmpeqd %ymm1, %ymm1, %ymm1
+; CHECK-NEXT:    vpsubq %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <4 x i64> @llvm.x86.avx512.permvar.di.256(<4 x i64> %a1, <4 x i64> %a0)
   ; add forces execution domain
@@ -676,8 +1171,14 @@ define <4 x i64> @stack_fold_permqvar(<4 x i64> %a0, <4 x i64> %a1) {
 declare <4 x i64> @llvm.x86.avx512.permvar.di.256(<4 x i64>, <4 x i64>) nounwind readonly
 
 define <16 x i8> @stack_fold_vpermt2b(<16 x i8> %x0, <16 x i8> %x1, <16 x i8> %x2) {
-  ;CHECK-LABEL: stack_fold_vpermt2b
-  ;CHECK:       vpermt2b {{-?[0-9]*}}(%rsp), %xmm1, %xmm0 # 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpermt2b:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermt2b {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x i8> @llvm.x86.avx512.vpermi2var.qi.128(<16 x i8> %x0, <16 x i8> %x1, <16 x i8> %x2)
   ret <16 x i8> %2
@@ -685,8 +1186,14 @@ define <16 x i8> @stack_fold_vpermt2b(<16 x i8> %x0, <16 x i8> %x1, <16 x i8> %x
 declare <16 x i8> @llvm.x86.avx512.vpermi2var.qi.128(<16 x i8>, <16 x i8>, <16 x i8>)
 
 define <32 x i8> @stack_fold_vpermt2b_ymm(<32 x i8> %x0, <32 x i8> %x1, <32 x i8> %x2) {
-  ;CHECK-LABEL: stack_fold_vpermt2b_ymm
-  ;CHECK:       vpermt2b {{-?[0-9]*}}(%rsp), %ymm1, %ymm0 # 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpermt2b_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermt2b {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <4 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <32 x i8> @llvm.x86.avx512.vpermi2var.qi.256(<32 x i8> %x0, <32 x i8> %x1, <32 x i8> %x2)
   ret <32 x i8> %2
@@ -694,8 +1201,14 @@ define <32 x i8> @stack_fold_vpermt2b_ymm(<32 x i8> %x0, <32 x i8> %x1, <32 x i8
 declare <32 x i8> @llvm.x86.avx512.vpermi2var.qi.256(<32 x i8>, <32 x i8>, <32 x i8>)
 
 define <4 x i32> @stack_fold_vpermt2d(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2) {
-  ;CHECK-LABEL: stack_fold_vpermt2d
-  ;CHECK:       vpermt2d {{-?[0-9]*}}(%rsp), %xmm1, %xmm0 # 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpermt2d:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermt2d {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <4 x i32> @llvm.x86.avx512.vpermi2var.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2)
   ret <4 x i32> %2
@@ -703,8 +1216,14 @@ define <4 x i32> @stack_fold_vpermt2d(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x
 declare <4 x i32> @llvm.x86.avx512.vpermi2var.d.128(<4 x i32>, <4 x i32>, <4 x i32>)
 
 define <8 x i32> @stack_fold_vpermt2d_ymm(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2) {
-  ;CHECK-LABEL: stack_fold_vpermt2d_ymm
-  ;CHECK:       vpermt2d {{-?[0-9]*}}(%rsp), %ymm1, %ymm0 # 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpermt2d_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermt2d {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <4 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x i32> @llvm.x86.avx512.vpermi2var.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2)
   ret <8 x i32> %2
@@ -712,8 +1231,14 @@ define <8 x i32> @stack_fold_vpermt2d_ymm(<8 x i32> %x0, <8 x i32> %x1, <8 x i32
 declare <8 x i32> @llvm.x86.avx512.vpermi2var.d.256(<8 x i32>, <8 x i32>, <8 x i32>)
 
 define <2 x i64> @stack_fold_vpermt2q(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2) {
-  ;CHECK-LABEL: stack_fold_vpermt2q
-  ;CHECK:       vpermt2q {{-?[0-9]*}}(%rsp), %xmm1, %xmm0 # 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpermt2q:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <2 x i64> @llvm.x86.avx512.vpermi2var.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2)
   ret <2 x i64> %2
@@ -721,8 +1246,14 @@ define <2 x i64> @stack_fold_vpermt2q(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x
 declare <2 x i64> @llvm.x86.avx512.vpermi2var.q.128(<2 x i64>, <2 x i64>, <2 x i64>)
 
 define <4 x i64> @stack_fold_vpermt2q_ymm(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2) {
-  ;CHECK-LABEL: stack_fold_vpermt2q_ymm
-  ;CHECK:       vpermt2q {{-?[0-9]*}}(%rsp), %ymm1, %ymm0 # 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpermt2q_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <4 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <4 x i64> @llvm.x86.avx512.vpermi2var.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2)
   ret <4 x i64> %2
@@ -730,8 +1261,14 @@ define <4 x i64> @stack_fold_vpermt2q_ymm(<4 x i64> %x0, <4 x i64> %x1, <4 x i64
 declare <4 x i64> @llvm.x86.avx512.vpermi2var.q.256(<4 x i64>, <4 x i64>, <4 x i64>)
 
 define <8 x i16> @stack_fold_vpermt2w(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2) {
-  ;CHECK-LABEL: stack_fold_vpermt2w
-  ;CHECK:       vpermt2w {{-?[0-9]*}}(%rsp), %xmm1, %xmm0 # 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpermt2w:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermt2w {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x i16> @llvm.x86.avx512.vpermi2var.hi.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2)
   ret <8 x i16> %2
@@ -739,8 +1276,14 @@ define <8 x i16> @stack_fold_vpermt2w(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x
 declare <8 x i16> @llvm.x86.avx512.vpermi2var.hi.128(<8 x i16>, <8 x i16>, <8 x i16>)
 
 define <16 x i16> @stack_fold_vpermt2w_ymm(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2) {
-  ;CHECK-LABEL: stack_fold_vpermt2w_ymm
-  ;CHECK:       vpermt2w {{-?[0-9]*}}(%rsp), %ymm1, %ymm0 # 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpermt2w_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermt2w {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <4 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x i16> @llvm.x86.avx512.vpermi2var.hi.256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2)
   ret <16 x i16> %2
@@ -748,8 +1291,18 @@ define <16 x i16> @stack_fold_vpermt2w_ymm(<16 x i16> %x0, <16 x i16> %x1, <16 x
 declare <16 x i16> @llvm.x86.avx512.vpermi2var.hi.256(<16 x i16>, <16 x i16>, <16 x i16>)
 
 define <16 x i16> @stack_fold_permwvar(<16 x i16> %a0, <16 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_permwvar
-  ;CHECK:   vpermw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_permwvar:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; CHECK-NEXT:    vpermw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    vpcmpeqd %ymm1, %ymm1, %ymm1
+; CHECK-NEXT:    vpsubw %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x i16> @llvm.x86.avx512.permvar.hi.256(<16 x i16> %a1, <16 x i16> %a0)
   ; add forces execution domain
@@ -759,40 +1312,70 @@ define <16 x i16> @stack_fold_permwvar(<16 x i16> %a0, <16 x i16> %a1) {
 declare <16 x i16> @llvm.x86.avx512.permvar.hi.256(<16 x i16>, <16 x i16>) nounwind readonly
 
 define <4 x i32> @stack_fold_vplzcntd(<4 x i32> %a0) {
-  ;CHECK-LABEL: stack_fold_vplzcntd
-  ;CHECK:       vplzcntd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vplzcntd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vplzcntd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> %a0, i1 false)
   ret <4 x i32> %2
 }
 
 define <8 x i32> @stack_fold_vplzcntd_ymm(<8 x i32> %a0) {
-  ;CHECK-LABEL: stack_fold_vplzcntd_ymm
-  ;CHECK:       vplzcntd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_vplzcntd_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vplzcntd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x i32> @llvm.ctlz.v8i32(<8 x i32> %a0, i1 false)
   ret <8 x i32> %2
 }
 
 define <2 x i64> @stack_fold_vplzcntq(<2 x i64> %a0) {
-  ;CHECK-LABEL: stack_fold_vplzcntq
-  ;CHECK:       vplzcntq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vplzcntq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vplzcntq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> %a0, i1 false)
   ret <2 x i64> %2
 }
 
 define <4 x i64> @stack_fold_vplzcntq_ymm(<4 x i64> %a0) {
-  ;CHECK-LABEL: stack_fold_vplzcntq_ymm
-  ;CHECK:       vplzcntq {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_vplzcntq_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vplzcntq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <4 x i64> @llvm.ctlz.v4i64(<4 x i64> %a0, i1 false)
   ret <4 x i64> %2
 }
 
 define <8 x i16> @stack_fold_pmaddubsw(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pmaddubsw
-  ;CHECK:       vpmaddubsw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaddubsw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmaddubsw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8> %a0, <16 x i8> %a1)
   ret <8 x i16> %2
@@ -800,8 +1383,17 @@ define <8 x i16> @stack_fold_pmaddubsw(<16 x i8> %a0, <16 x i8> %a1) {
 declare <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8>, <16 x i8>) nounwind readnone
 
 define <8 x i16> @stack_fold_pmaddubsw_mask(<8 x i16>* %passthru, <16 x i8> %a0, <16 x i8> %a1, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_pmaddubsw_mask
-  ;CHECK:       vpmaddubsw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaddubsw_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vmovdqa (%rdi), %xmm2
+; CHECK-NEXT:    vpmaddubsw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm2 {%k1} # 16-byte Folded Reload
+; CHECK-NEXT:    vmovdqa %xmm2, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8> %a0, <16 x i8> %a1)
   %3 = bitcast i8 %mask to <8 x i1>
@@ -812,8 +1404,15 @@ define <8 x i16> @stack_fold_pmaddubsw_mask(<8 x i16>* %passthru, <16 x i8> %a0,
 }
 
 define <8 x i16> @stack_fold_pmaddubsw_maskz(<16 x i8> %a0, <16 x i8> %a1, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_pmaddubsw_maskz
-  ;CHECK:       vpmaddubsw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaddubsw_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpmaddubsw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 {%k1} {z} # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8> %a0, <16 x i8> %a1)
   %3 = bitcast i8 %mask to <8 x i1>
@@ -822,8 +1421,14 @@ define <8 x i16> @stack_fold_pmaddubsw_maskz(<16 x i8> %a0, <16 x i8> %a1, i8 %m
 }
 
 define <16 x i16> @stack_fold_pmaddubsw_ymm(<32 x i8> %a0, <32 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pmaddubsw_ymm
-  ;CHECK:       vpmaddubsw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaddubsw_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmaddubsw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x i16> @llvm.x86.avx2.pmadd.ub.sw(<32 x i8> %a0, <32 x i8> %a1)
   ret <16 x i16> %2
@@ -831,8 +1436,17 @@ define <16 x i16> @stack_fold_pmaddubsw_ymm(<32 x i8> %a0, <32 x i8> %a1) {
 declare <16 x i16> @llvm.x86.avx2.pmadd.ub.sw(<32 x i8>, <32 x i8>) nounwind readnone
 
 define <16 x i16> @stack_fold_pmaddubsw_ymm_mask(<16 x i16>* %passthru, <32 x i8> %a0, <32 x i8> %a1, i16 %mask) {
-  ;CHECK-LABEL: stack_fold_pmaddubsw_ymm_mask
-  ;CHECK:       vpmaddubsw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaddubsw_ymm_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vmovdqa (%rdi), %ymm2
+; CHECK-NEXT:    vpmaddubsw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm2 {%k1} # 32-byte Folded Reload
+; CHECK-NEXT:    vmovdqa %ymm2, %ymm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x i16> @llvm.x86.avx2.pmadd.ub.sw(<32 x i8> %a0, <32 x i8> %a1)
   %3 = bitcast i16 %mask to <16 x i1>
@@ -843,8 +1457,15 @@ define <16 x i16> @stack_fold_pmaddubsw_ymm_mask(<16 x i16>* %passthru, <32 x i8
 }
 
 define <16 x i16> @stack_fold_pmaddubsw_ymm_maskz(<32 x i8> %a0, <32 x i8> %a1, i16 %mask) {
-  ;CHECK-LABEL: stack_fold_pmaddubsw_ymm_maskz
-  ;CHECK:       vpmaddubsw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaddubsw_ymm_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpmaddubsw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 {%k1} {z} # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x i16> @llvm.x86.avx2.pmadd.ub.sw(<32 x i8> %a0, <32 x i8> %a1)
   %3 = bitcast i16 %mask to <16 x i1>
@@ -853,8 +1474,14 @@ define <16 x i16> @stack_fold_pmaddubsw_ymm_maskz(<32 x i8> %a0, <32 x i8> %a1,
 }
 
 define <4 x i32> @stack_fold_pmaddwd(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_pmaddwd
-  ;CHECK:       vpmaddwd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaddwd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmaddwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> %a0, <8 x i16> %a1)
   ret <4 x i32> %2
@@ -862,8 +1489,17 @@ define <4 x i32> @stack_fold_pmaddwd(<8 x i16> %a0, <8 x i16> %a1) {
 declare <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16>, <8 x i16>) nounwind readnone
 
 define <4 x i32> @stack_fold_pmaddwd_mask(<4 x i32>* %passthru, <8 x i16> %a0, <8 x i16> %a1, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_pmaddwd_mask
-  ;CHECK:       vpmaddwd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaddwd_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vmovdqa (%rdi), %xmm2
+; CHECK-NEXT:    vpmaddwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm2 {%k1} # 16-byte Folded Reload
+; CHECK-NEXT:    vmovdqa %xmm2, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> %a0, <8 x i16> %a1)
   %3 = bitcast i8 %mask to <8 x i1>
@@ -875,8 +1511,15 @@ define <4 x i32> @stack_fold_pmaddwd_mask(<4 x i32>* %passthru, <8 x i16> %a0, <
 }
 
 define <4 x i32> @stack_fold_pmaddwd_maskz(<8 x i16> %a0, <8 x i16> %a1, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_pmaddwd_maskz
-  ;CHECK:       vpmaddwd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaddwd_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpmaddwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 {%k1} {z} # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> %a0, <8 x i16> %a1)
   %3 = bitcast i8 %mask to <8 x i1>
@@ -886,8 +1529,14 @@ define <4 x i32> @stack_fold_pmaddwd_maskz(<8 x i16> %a0, <8 x i16> %a1, i8 %mas
 }
 
 define <8 x i32> @stack_fold_pmaddwd_ymm(<16 x i16> %a0, <16 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_pmaddwd_ymm
-  ;CHECK:       vpmaddwd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaddwd_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmaddwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x i32> @llvm.x86.avx2.pmadd.wd(<16 x i16> %a0, <16 x i16> %a1)
   ret <8 x i32> %2
@@ -895,8 +1544,17 @@ define <8 x i32> @stack_fold_pmaddwd_ymm(<16 x i16> %a0, <16 x i16> %a1) {
 declare <8 x i32> @llvm.x86.avx2.pmadd.wd(<16 x i16>, <16 x i16>) nounwind readnone
 
 define <8 x i32> @stack_fold_pmaddwd_ymm_mask(<8 x i32>* %passthru, <16 x i16> %a0, <16 x i16> %a1, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_pmaddwd_ymm_mask
-  ;CHECK:       vpmaddwd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaddwd_ymm_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vmovdqa (%rdi), %ymm2
+; CHECK-NEXT:    vpmaddwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm2 {%k1} # 32-byte Folded Reload
+; CHECK-NEXT:    vmovdqa %ymm2, %ymm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x i32> @llvm.x86.avx2.pmadd.wd(<16 x i16> %a0, <16 x i16> %a1)
   %3 = bitcast i8 %mask to <8 x i1>
@@ -907,8 +1565,15 @@ define <8 x i32> @stack_fold_pmaddwd_ymm_mask(<8 x i32>* %passthru, <16 x i16> %
 }
 
 define <8 x i32> @stack_fold_pmaddwd_ymm_maskz(<16 x i16> %a0, <16 x i16> %a1, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_pmaddwd_ymm_maskz
-  ;CHECK:       vpmaddwd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaddwd_ymm_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpmaddwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 {%k1} {z} # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x i32> @llvm.x86.avx2.pmadd.wd(<16 x i16> %a0, <16 x i16> %a1)
   %3 = bitcast i8 %mask to <8 x i1>
@@ -917,8 +1582,14 @@ define <8 x i32> @stack_fold_pmaddwd_ymm_maskz(<16 x i16> %a0, <16 x i16> %a1, i
 }
 
 define <16 x i8> @stack_fold_pmaxsb(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pmaxsb
-  ;CHECK:       vpmaxsb {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxsb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmaxsb {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp sgt <16 x i8> %a0, %a1
   %3 = select <16 x i1> %2, <16 x i8> %a0, <16 x i8> %a1
@@ -926,8 +1597,14 @@ define <16 x i8> @stack_fold_pmaxsb(<16 x i8> %a0, <16 x i8> %a1) {
 }
 
 define <32 x i8> @stack_fold_pmaxsb_ymm(<32 x i8> %a0, <32 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pmaxsb_ymm
-  ;CHECK:       vpmaxsb {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxsb_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmaxsb {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp sgt <32 x i8> %a0, %a1
   %3 = select <32 x i1> %2, <32 x i8> %a0, <32 x i8> %a1
@@ -935,8 +1612,14 @@ define <32 x i8> @stack_fold_pmaxsb_ymm(<32 x i8> %a0, <32 x i8> %a1) {
 }
 
 define <4 x i32> @stack_fold_pmaxsd(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_pmaxsd
-  ;CHECK:       vpmaxsd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxsd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmaxsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp sgt <4 x i32> %a0, %a1
   %3 = select <4 x i1> %2, <4 x i32> %a0, <4 x i32> %a1
@@ -944,8 +1627,14 @@ define <4 x i32> @stack_fold_pmaxsd(<4 x i32> %a0, <4 x i32> %a1) {
 }
 
 define <8 x i32> @stack_fold_pmaxsd_ymm(<8 x i32> %a0, <8 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_pmaxsd_ymm
-  ;CHECK:       vpmaxsd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxsd_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmaxsd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp sgt <8 x i32> %a0, %a1
   %3 = select <8 x i1> %2, <8 x i32> %a0, <8 x i32> %a1
@@ -953,8 +1642,14 @@ define <8 x i32> @stack_fold_pmaxsd_ymm(<8 x i32> %a0, <8 x i32> %a1) {
 }
 
 define <2 x i64> @stack_fold_pmaxsq(<2 x i64> %a0, <2 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_pmaxsq
-  ;CHECK:       vpmaxsq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxsq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmaxsq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp sgt <2 x i64> %a0, %a1
   %3 = select <2 x i1> %2, <2 x i64> %a0, <2 x i64> %a1
@@ -962,8 +1657,14 @@ define <2 x i64> @stack_fold_pmaxsq(<2 x i64> %a0, <2 x i64> %a1) {
 }
 
 define <4 x i64> @stack_fold_pmaxsq_ymm(<4 x i64> %a0, <4 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_pmaxsq_ymm
-  ;CHECK:       vpmaxsq {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxsq_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmaxsq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp sgt <4 x i64> %a0, %a1
   %3 = select <4 x i1> %2, <4 x i64> %a0, <4 x i64> %a1
@@ -971,8 +1672,14 @@ define <4 x i64> @stack_fold_pmaxsq_ymm(<4 x i64> %a0, <4 x i64> %a1) {
 }
 
 define <8 x i16> @stack_fold_pmaxsw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_pmaxsw
-  ;CHECK:       vpmaxsw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxsw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmaxsw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp sgt <8 x i16> %a0, %a1
   %3 = select <8 x i1> %2, <8 x i16> %a0, <8 x i16> %a1
@@ -980,8 +1687,14 @@ define <8 x i16> @stack_fold_pmaxsw(<8 x i16> %a0, <8 x i16> %a1) {
 }
 
 define <16 x i16> @stack_fold_pmaxsw_ymm(<16 x i16> %a0, <16 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_pmaxsw_ymm
-  ;CHECK:       vpmaxsw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxsw_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmaxsw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp sgt <16 x i16> %a0, %a1
   %3 = select <16 x i1> %2, <16 x i16> %a0, <16 x i16> %a1
@@ -989,8 +1702,14 @@ define <16 x i16> @stack_fold_pmaxsw_ymm(<16 x i16> %a0, <16 x i16> %a1) {
 }
 
 define <16 x i8> @stack_fold_pmaxub(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pmaxub
-  ;CHECK:       vpmaxub {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxub:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmaxub {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp ugt <16 x i8> %a0, %a1
   %3 = select <16 x i1> %2, <16 x i8> %a0, <16 x i8> %a1
@@ -998,8 +1717,14 @@ define <16 x i8> @stack_fold_pmaxub(<16 x i8> %a0, <16 x i8> %a1) {
 }
 
 define <32 x i8> @stack_fold_pmaxub_ymm(<32 x i8> %a0, <32 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pmaxub_ymm
-  ;CHECK:       vpmaxub {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxub_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmaxub {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp ugt <32 x i8> %a0, %a1
   %3 = select <32 x i1> %2, <32 x i8> %a0, <32 x i8> %a1
@@ -1007,8 +1732,14 @@ define <32 x i8> @stack_fold_pmaxub_ymm(<32 x i8> %a0, <32 x i8> %a1) {
 }
 
 define <4 x i32> @stack_fold_pmaxud(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_pmaxud
-  ;CHECK:       vpmaxud {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxud:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmaxud {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp ugt <4 x i32> %a0, %a1
   %3 = select <4 x i1> %2, <4 x i32> %a0, <4 x i32> %a1
@@ -1016,8 +1747,14 @@ define <4 x i32> @stack_fold_pmaxud(<4 x i32> %a0, <4 x i32> %a1) {
 }
 
 define <8 x i32> @stack_fold_pmaxud_ymm(<8 x i32> %a0, <8 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_pmaxud_ymm
-  ;CHECK:       vpmaxud {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxud_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmaxud {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp ugt <8 x i32> %a0, %a1
   %3 = select <8 x i1> %2, <8 x i32> %a0, <8 x i32> %a1
@@ -1025,8 +1762,14 @@ define <8 x i32> @stack_fold_pmaxud_ymm(<8 x i32> %a0, <8 x i32> %a1) {
 }
 
 define <2 x i64> @stack_fold_pmaxuq(<2 x i64> %a0, <2 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_pmaxuq
-  ;CHECK:       vpmaxuq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxuq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmaxuq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp ugt <2 x i64> %a0, %a1
   %3 = select <2 x i1> %2, <2 x i64> %a0, <2 x i64> %a1
@@ -1034,8 +1777,17 @@ define <2 x i64> @stack_fold_pmaxuq(<2 x i64> %a0, <2 x i64> %a1) {
 }
 
 define <2 x i64> @stack_fold_pmaxuq_mask(<2 x i64>* %passthru, <2 x i64> %a0, <2 x i64> %a1, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_pmaxuq_mask
-  ;CHECK:       vpmaxuq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxuq_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovdqa (%rdi), %xmm2
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vpmaxuq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm2 {%k1} # 16-byte Folded Reload
+; CHECK-NEXT:    vmovdqa %xmm2, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = load <2 x i64>, <2 x i64>* %passthru
   %3 = icmp ugt <2 x i64> %a0, %a1
@@ -1047,8 +1799,15 @@ define <2 x i64> @stack_fold_pmaxuq_mask(<2 x i64>* %passthru, <2 x i64> %a0, <2
 }
 
 define <2 x i64> @stack_fold_pmaxuq_maskz(<2 x i64> %a0, <2 x i64> %a1, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_pmaxuq_maskz
-  ;CHECK:       vpmaxuq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxuq_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpmaxuq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 {%k1} {z} # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp ugt <2 x i64> %a0, %a1
   %3 = select <2 x i1> %2, <2 x i64> %a0, <2 x i64> %a1
@@ -1059,8 +1818,14 @@ define <2 x i64> @stack_fold_pmaxuq_maskz(<2 x i64> %a0, <2 x i64> %a1, i8 %mask
 }
 
 define <4 x i64> @stack_fold_pmaxuq_ymm(<4 x i64> %a0, <4 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_pmaxuq_ymm
-  ;CHECK:       vpmaxuq {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxuq_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmaxuq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp ugt <4 x i64> %a0, %a1
   %3 = select <4 x i1> %2, <4 x i64> %a0, <4 x i64> %a1
@@ -1068,8 +1833,17 @@ define <4 x i64> @stack_fold_pmaxuq_ymm(<4 x i64> %a0, <4 x i64> %a1) {
 }
 
 define <4 x i64> @stack_fold_pmaxuq_ymm_mask(<4 x i64>* %passthru, <4 x i64> %a0, <4 x i64> %a1, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_pmaxuq_ymm_mask
-  ;CHECK:       vpmaxuq {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxuq_ymm_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovdqa (%rdi), %ymm2
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vpmaxuq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm2 {%k1} # 32-byte Folded Reload
+; CHECK-NEXT:    vmovdqa %ymm2, %ymm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = load <4 x i64>, <4 x i64>* %passthru
   %3 = icmp ugt <4 x i64> %a0, %a1
@@ -1081,8 +1855,15 @@ define <4 x i64> @stack_fold_pmaxuq_ymm_mask(<4 x i64>* %passthru, <4 x i64> %a0
 }
 
 define <4 x i64> @stack_fold_pmaxuq_ymm_maskz(<4 x i64> %a0, <4 x i64> %a1, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_pmaxuq_ymm_maskz
-  ;CHECK:       vpmaxuq {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxuq_ymm_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpmaxuq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 {%k1} {z} # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp ugt <4 x i64> %a0, %a1
   %3 = select <4 x i1> %2, <4 x i64> %a0, <4 x i64> %a1
@@ -1093,8 +1874,14 @@ define <4 x i64> @stack_fold_pmaxuq_ymm_maskz(<4 x i64> %a0, <4 x i64> %a1, i8 %
 }
 
 define <8 x i16> @stack_fold_pmaxuw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_pmaxuw
-  ;CHECK:       vpmaxuw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxuw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmaxuw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp ugt <8 x i16> %a0, %a1
   %3 = select <8 x i1> %2, <8 x i16> %a0, <8 x i16> %a1
@@ -1102,8 +1889,14 @@ define <8 x i16> @stack_fold_pmaxuw(<8 x i16> %a0, <8 x i16> %a1) {
 }
 
 define <16 x i16> @stack_fold_pmaxuw_ymm(<16 x i16> %a0, <16 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_pmaxuw_ymm
-  ;CHECK:       vpmaxuw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxuw_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmaxuw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp ugt <16 x i16> %a0, %a1
   %3 = select <16 x i1> %2, <16 x i16> %a0, <16 x i16> %a1
@@ -1112,8 +1905,14 @@ define <16 x i16> @stack_fold_pmaxuw_ymm(<16 x i16> %a0, <16 x i16> %a1) {
 declare <16 x i16> @llvm.x86.avx2.pmaxu.w(<16 x i16>, <16 x i16>) nounwind readnone
 
 define <16 x i8> @stack_fold_pminsb(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pminsb
-  ;CHECK:       vpminsb {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminsb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpminsb {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp slt <16 x i8> %a0, %a1
   %3 = select <16 x i1> %2, <16 x i8> %a0, <16 x i8> %a1
@@ -1121,8 +1920,14 @@ define <16 x i8> @stack_fold_pminsb(<16 x i8> %a0, <16 x i8> %a1) {
 }
 
 define <32 x i8> @stack_fold_pminsb_ymm(<32 x i8> %a0, <32 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pminsb_ymm
-  ;CHECK:       vpminsb {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminsb_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpminsb {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp slt <32 x i8> %a0, %a1
   %3 = select <32 x i1> %2, <32 x i8> %a0, <32 x i8> %a1
@@ -1130,8 +1935,14 @@ define <32 x i8> @stack_fold_pminsb_ymm(<32 x i8> %a0, <32 x i8> %a1) {
 }
 
 define <4 x i32> @stack_fold_pminsd(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_pminsd
-  ;CHECK:       vpminsd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminsd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpminsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp slt <4 x i32> %a0, %a1
   %3 = select <4 x i1> %2, <4 x i32> %a0, <4 x i32> %a1
@@ -1139,8 +1950,14 @@ define <4 x i32> @stack_fold_pminsd(<4 x i32> %a0, <4 x i32> %a1) {
 }
 
 define <8 x i32> @stack_fold_pminsd_ymm(<8 x i32> %a0, <8 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_pminsd_ymm
-  ;CHECK:       vpminsd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminsd_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpminsd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp slt <8 x i32> %a0, %a1
   %3 = select <8 x i1> %2, <8 x i32> %a0, <8 x i32> %a1
@@ -1148,8 +1965,14 @@ define <8 x i32> @stack_fold_pminsd_ymm(<8 x i32> %a0, <8 x i32> %a1) {
 }
 
 define <2 x i64> @stack_fold_pminsq(<2 x i64> %a0, <2 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_pminsq
-  ;CHECK:       vpminsq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminsq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpminsq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp slt <2 x i64> %a0, %a1
   %3 = select <2 x i1> %2, <2 x i64> %a0, <2 x i64> %a1
@@ -1157,8 +1980,14 @@ define <2 x i64> @stack_fold_pminsq(<2 x i64> %a0, <2 x i64> %a1) {
 }
 
 define <4 x i64> @stack_fold_pminsq_ymm(<4 x i64> %a0, <4 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_pminsq_ymm
-  ;CHECK:       vpminsq {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminsq_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpminsq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp slt <4 x i64> %a0, %a1
   %3 = select <4 x i1> %2, <4 x i64> %a0, <4 x i64> %a1
@@ -1166,8 +1995,14 @@ define <4 x i64> @stack_fold_pminsq_ymm(<4 x i64> %a0, <4 x i64> %a1) {
 }
 
 define <8 x i16> @stack_fold_pminsw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_pminsw
-  ;CHECK:       vpminsw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminsw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpminsw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp slt <8 x i16> %a0, %a1
   %3 = select <8 x i1> %2, <8 x i16> %a0, <8 x i16> %a1
@@ -1175,8 +2010,14 @@ define <8 x i16> @stack_fold_pminsw(<8 x i16> %a0, <8 x i16> %a1) {
 }
 
 define <16 x i16> @stack_fold_pminsw_ymm(<16 x i16> %a0, <16 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_pminsw_ymm
-  ;CHECK:       vpminsw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminsw_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpminsw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp slt <16 x i16> %a0, %a1
   %3 = select <16 x i1> %2, <16 x i16> %a0, <16 x i16> %a1
@@ -1184,8 +2025,14 @@ define <16 x i16> @stack_fold_pminsw_ymm(<16 x i16> %a0, <16 x i16> %a1) {
 }
 
 define <16 x i8> @stack_fold_pminub(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pminub
-  ;CHECK:       vpminub {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminub:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpminub {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp ult <16 x i8> %a0, %a1
   %3 = select <16 x i1> %2, <16 x i8> %a0, <16 x i8> %a1
@@ -1193,8 +2040,14 @@ define <16 x i8> @stack_fold_pminub(<16 x i8> %a0, <16 x i8> %a1) {
 }
 
 define <32 x i8> @stack_fold_pminub_ymm(<32 x i8> %a0, <32 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pminub_ymm
-  ;CHECK:       vpminub {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminub_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpminub {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp ult <32 x i8> %a0, %a1
   %3 = select <32 x i1> %2, <32 x i8> %a0, <32 x i8> %a1
@@ -1202,8 +2055,14 @@ define <32 x i8> @stack_fold_pminub_ymm(<32 x i8> %a0, <32 x i8> %a1) {
 }
 
 define <4 x i32> @stack_fold_pminud(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_pminud
-  ;CHECK:       vpminud {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminud:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpminud {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp ult <4 x i32> %a0, %a1
   %3 = select <4 x i1> %2, <4 x i32> %a0, <4 x i32> %a1
@@ -1211,8 +2070,14 @@ define <4 x i32> @stack_fold_pminud(<4 x i32> %a0, <4 x i32> %a1) {
 }
 
 define <8 x i32> @stack_fold_pminud_ymm(<8 x i32> %a0, <8 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_pminud_ymm
-  ;CHECK:       vpminud {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminud_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpminud {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp ult <8 x i32> %a0, %a1
   %3 = select <8 x i1> %2, <8 x i32> %a0, <8 x i32> %a1
@@ -1220,8 +2085,14 @@ define <8 x i32> @stack_fold_pminud_ymm(<8 x i32> %a0, <8 x i32> %a1) {
 }
 
 define <2 x i64> @stack_fold_pminuq(<2 x i64> %a0, <2 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_pminuq
-  ;CHECK:       vpminuq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminuq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpminuq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp ult <2 x i64> %a0, %a1
   %3 = select <2 x i1> %2, <2 x i64> %a0, <2 x i64> %a1
@@ -1229,8 +2100,14 @@ define <2 x i64> @stack_fold_pminuq(<2 x i64> %a0, <2 x i64> %a1) {
 }
 
 define <4 x i64> @stack_fold_pminuq_ymm(<4 x i64> %a0, <4 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_pminuq_ymm
-  ;CHECK:       vpminuq {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminuq_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpminuq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp ult <4 x i64> %a0, %a1
   %3 = select <4 x i1> %2, <4 x i64> %a0, <4 x i64> %a1
@@ -1238,8 +2115,14 @@ define <4 x i64> @stack_fold_pminuq_ymm(<4 x i64> %a0, <4 x i64> %a1) {
 }
 
 define <8 x i16> @stack_fold_pminuw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_pminuw
-  ;CHECK:       vpminuw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminuw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpminuw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp ult <8 x i16> %a0, %a1
   %3 = select <8 x i1> %2, <8 x i16> %a0, <8 x i16> %a1
@@ -1247,8 +2130,14 @@ define <8 x i16> @stack_fold_pminuw(<8 x i16> %a0, <8 x i16> %a1) {
 }
 
 define <16 x i16> @stack_fold_pminuw_ymm(<16 x i16> %a0, <16 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_pminuw_ymm
-  ;CHECK:       vpminuw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminuw_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpminuw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = icmp ult <16 x i16> %a0, %a1
   %3 = select <16 x i1> %2, <16 x i16> %a0, <16 x i16> %a1
@@ -1256,8 +2145,15 @@ define <16 x i16> @stack_fold_pminuw_ymm(<16 x i16> %a0, <16 x i16> %a1) {
 }
 
 define <8 x i16> @stack_fold_vpmovdw(<8 x i32> %a0) {
-  ;CHECK-LABEL: stack_fold_vpmovdw
-  ;CHECK:       vpmovdw %ymm0, {{-?[0-9]*}}(%rsp) # 16-byte Folded Spill
+; CHECK-LABEL: stack_fold_vpmovdw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmovdw %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Folded Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
   %1 = call <8 x i16> @llvm.x86.avx512.mask.pmov.dw.256(<8 x i32> %a0, <8 x i16> undef, i8 -1)
   %2 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   ret <8 x i16> %1
@@ -1265,8 +2161,15 @@ define <8 x i16> @stack_fold_vpmovdw(<8 x i32> %a0) {
 declare <8 x i16> @llvm.x86.avx512.mask.pmov.dw.256(<8 x i32>, <8 x i16>, i8)
 
 define <4 x i32> @stack_fold_vpmovqd(<4 x i64> %a0) {
-  ;CHECK-LABEL: stack_fold_vpmovqd
-  ;CHECK:       vpmovqd %ymm0, {{-?[0-9]*}}(%rsp) # 16-byte Folded Spill
+; CHECK-LABEL: stack_fold_vpmovqd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmovqd %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Folded Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
   %1 = trunc <4 x i64> %a0 to <4 x i32>
   %2 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   ret <4 x i32> %1
@@ -1274,8 +2177,15 @@ define <4 x i32> @stack_fold_vpmovqd(<4 x i64> %a0) {
 declare <4 x i32> @llvm.x86.avx512.mask.pmov.qd.256(<4 x i64>, <4 x i32>, i8)
 
 define <16 x i8> @stack_fold_vpmovwb(<16 x i16> %a0) {
-  ;CHECK-LABEL: stack_fold_vpmovwb
-  ;CHECK:       vpmovwb %ymm0, {{-?[0-9]*}}(%rsp) # 16-byte Folded Spill
+; CHECK-LABEL: stack_fold_vpmovwb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmovwb %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Folded Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
   %1 = trunc <16 x i16> %a0 to <16 x i8>
   %2 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   ret <16 x i8> %1
@@ -1283,8 +2193,15 @@ define <16 x i8> @stack_fold_vpmovwb(<16 x i16> %a0) {
 declare <16 x i8> @llvm.x86.avx512.mask.pmov.wb.256(<16 x i16>, <16 x i8>, i16)
 
 define <8 x i16> @stack_fold_vpmovsdw(<8 x i32> %a0) {
-  ;CHECK-LABEL: stack_fold_vpmovsdw
-  ;CHECK:       vpmovsdw %ymm0, {{-?[0-9]*}}(%rsp) # 16-byte Folded Spill
+; CHECK-LABEL: stack_fold_vpmovsdw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmovsdw %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Folded Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
   %1 = call <8 x i16> @llvm.x86.avx512.mask.pmovs.dw.256(<8 x i32> %a0, <8 x i16> undef, i8 -1)
   %2 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   ret <8 x i16> %1
@@ -1292,8 +2209,15 @@ define <8 x i16> @stack_fold_vpmovsdw(<8 x i32> %a0) {
 declare <8 x i16> @llvm.x86.avx512.mask.pmovs.dw.256(<8 x i32>, <8 x i16>, i8)
 
 define <4 x i32> @stack_fold_vpmovsqd(<4 x i64> %a0) {
-  ;CHECK-LABEL: stack_fold_vpmovsqd
-  ;CHECK:       vpmovsqd %ymm0, {{-?[0-9]*}}(%rsp) # 16-byte Folded Spill
+; CHECK-LABEL: stack_fold_vpmovsqd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmovsqd %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Folded Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
   %1 = call <4 x i32> @llvm.x86.avx512.mask.pmovs.qd.256(<4 x i64> %a0, <4 x i32> undef, i8 -1)
   %2 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   ret <4 x i32> %1
@@ -1301,8 +2225,15 @@ define <4 x i32> @stack_fold_vpmovsqd(<4 x i64> %a0) {
 declare <4 x i32> @llvm.x86.avx512.mask.pmovs.qd.256(<4 x i64>, <4 x i32>, i8)
 
 define <16 x i8> @stack_fold_vpmovswb(<16 x i16> %a0) {
-  ;CHECK-LABEL: stack_fold_vpmovswb
-  ;CHECK:       vpmovswb %ymm0, {{-?[0-9]*}}(%rsp) # 16-byte Folded Spill
+; CHECK-LABEL: stack_fold_vpmovswb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmovswb %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Folded Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
   %1 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.wb.256(<16 x i16> %a0, <16 x i8> undef, i16 -1)
   %2 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   ret <16 x i8> %1
@@ -1310,8 +2241,14 @@ define <16 x i8> @stack_fold_vpmovswb(<16 x i16> %a0) {
 declare <16 x i8> @llvm.x86.avx512.mask.pmovs.wb.256(<16 x i16>, <16 x i8>, i16)
 
 define <4 x i32> @stack_fold_pmovsxbd(<16 x i8> %a0) {
-  ;CHECK-LABEL: stack_fold_pmovsxbd
-  ;CHECK:       vpmovsxbd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmovsxbd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmovsxbd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <16 x i8> %a0, <16 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %3 = sext <4 x i8> %2 to <4 x i32>
@@ -1319,8 +2256,14 @@ define <4 x i32> @stack_fold_pmovsxbd(<16 x i8> %a0) {
 }
 
 define <8 x i32> @stack_fold_pmovsxbd_ymm(<16 x i8> %a0) {
-  ;CHECK-LABEL: stack_fold_pmovsxbd_ymm
-  ;CHECK:       vpmovsxbd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmovsxbd_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmovsxbd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <16 x i8> %a0, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   %3 = sext <8 x i8> %2 to <8 x i32>
@@ -1328,8 +2271,14 @@ define <8 x i32> @stack_fold_pmovsxbd_ymm(<16 x i8> %a0) {
 }
 
 define <2 x i64> @stack_fold_pmovsxbq(<16 x i8> %a0) {
-  ;CHECK-LABEL: stack_fold_pmovsxbq
-  ;CHECK:       vpmovsxbq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmovsxbq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmovsxbq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <16 x i8> %a0, <16 x i8> undef, <2 x i32> <i32 0, i32 1>
   %3 = sext <2 x i8> %2 to <2 x i64>
@@ -1337,8 +2286,14 @@ define <2 x i64> @stack_fold_pmovsxbq(<16 x i8> %a0) {
 }
 
 define <4 x i64> @stack_fold_pmovsxbq_ymm(<16 x i8> %a0) {
-  ;CHECK-LABEL: stack_fold_pmovsxbq_ymm
-  ;CHECK:       pmovsxbq {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmovsxbq_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmovsxbq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <16 x i8> %a0, <16 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %3 = sext <4 x i8> %2 to <4 x i64>
@@ -1346,8 +2301,14 @@ define <4 x i64> @stack_fold_pmovsxbq_ymm(<16 x i8> %a0) {
 }
 
 define <8 x i16> @stack_fold_pmovsxbw(<16 x i8> %a0) {
-  ;CHECK-LABEL: stack_fold_pmovsxbw
-  ;CHECK:       vpmovsxbw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmovsxbw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmovsxbw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <16 x i8> %a0, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   %3 = sext <8 x i8> %2 to <8 x i16>
@@ -1355,16 +2316,28 @@ define <8 x i16> @stack_fold_pmovsxbw(<16 x i8> %a0) {
 }
 
 define <16 x i16> @stack_fold_pmovsxbw_ymm(<16 x i8> %a0) {
-  ;CHECK-LABEL: stack_fold_pmovsxbw_ymm
-  ;CHECK:       vpmovsxbw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmovsxbw_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmovsxbw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = sext <16 x i8> %a0 to <16 x i16>
   ret <16 x i16> %2
 }
 
 define <2 x i64> @stack_fold_pmovsxdq(<4 x i32> %a0) {
-  ;CHECK-LABEL: stack_fold_pmovsxdq
-  ;CHECK:       vpmovsxdq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmovsxdq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmovsxdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <4 x i32> %a0, <4 x i32> undef, <2 x i32> <i32 0, i32 1>
   %3 = sext <2 x i32> %2 to <2 x i64>
@@ -1372,16 +2345,28 @@ define <2 x i64> @stack_fold_pmovsxdq(<4 x i32> %a0) {
 }
 
 define <4 x i64> @stack_fold_pmovsxdq_ymm(<4 x i32> %a0) {
-  ;CHECK-LABEL: stack_fold_pmovsxdq_ymm
-  ;CHECK:       vpmovsxdq {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmovsxdq_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmovsxdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = sext <4 x i32> %a0 to <4 x i64>
   ret <4 x i64> %2
 }
 
 define <4 x i32> @stack_fold_pmovsxwd(<8 x i16> %a0) {
-  ;CHECK-LABEL: stack_fold_pmovsxwd
-  ;CHECK:       vpmovsxwd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmovsxwd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmovsxwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <8 x i16> %a0, <8 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %3 = sext <4 x i16> %2 to <4 x i32>
@@ -1389,16 +2374,28 @@ define <4 x i32> @stack_fold_pmovsxwd(<8 x i16> %a0) {
 }
 
 define <8 x i32> @stack_fold_pmovsxwd_ymm(<8 x i16> %a0) {
-  ;CHECK-LABEL: stack_fold_pmovsxwd_ymm
-  ;CHECK:       vpmovsxwd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmovsxwd_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmovsxwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = sext <8 x i16> %a0 to <8 x i32>
   ret <8 x i32> %2
 }
 
 define <2 x i64> @stack_fold_pmovsxwq(<8 x i16> %a0) {
-  ;CHECK-LABEL: stack_fold_pmovsxwq
-  ;CHECK:       vpmovsxwq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmovsxwq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmovsxwq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <8 x i16> %a0, <8 x i16> undef, <2 x i32> <i32 0, i32 1>
   %3 = sext <2 x i16> %2 to <2 x i64>
@@ -1406,8 +2403,14 @@ define <2 x i64> @stack_fold_pmovsxwq(<8 x i16> %a0) {
 }
 
 define <4 x i64> @stack_fold_pmovsxwq_ymm(<8 x i16> %a0) {
-  ;CHECK-LABEL: stack_fold_pmovsxwq_ymm
-  ;CHECK:       vpmovsxwq {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmovsxwq_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmovsxwq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <8 x i16> %a0, <8 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %3 = sext <4 x i16> %2 to <4 x i64>
@@ -1415,8 +2418,15 @@ define <4 x i64> @stack_fold_pmovsxwq_ymm(<8 x i16> %a0) {
 }
 
 define <8 x i16> @stack_fold_vpmovusdw(<8 x i32> %a0) {
-  ;CHECK-LABEL: stack_fold_vpmovusdw
-  ;CHECK:       vpmovusdw %ymm0, {{-?[0-9]*}}(%rsp) # 16-byte Folded Spill
+; CHECK-LABEL: stack_fold_vpmovusdw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmovusdw %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Folded Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
   %1 = call <8 x i16> @llvm.x86.avx512.mask.pmovus.dw.256(<8 x i32> %a0, <8 x i16> undef, i8 -1)
   %2 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   ret <8 x i16> %1
@@ -1424,8 +2434,15 @@ define <8 x i16> @stack_fold_vpmovusdw(<8 x i32> %a0) {
 declare <8 x i16> @llvm.x86.avx512.mask.pmovus.dw.256(<8 x i32>, <8 x i16>, i8)
 
 define <4 x i32> @stack_fold_vpmovusqd(<4 x i64> %a0) {
-  ;CHECK-LABEL: stack_fold_vpmovusqd
-  ;CHECK:       vpmovusqd %ymm0, {{-?[0-9]*}}(%rsp) # 16-byte Folded Spill
+; CHECK-LABEL: stack_fold_vpmovusqd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmovusqd %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Folded Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
   %1 = call <4 x i32> @llvm.x86.avx512.mask.pmovus.qd.256(<4 x i64> %a0, <4 x i32> undef, i8 -1)
   %2 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   ret <4 x i32> %1
@@ -1433,8 +2450,15 @@ define <4 x i32> @stack_fold_vpmovusqd(<4 x i64> %a0) {
 declare <4 x i32> @llvm.x86.avx512.mask.pmovus.qd.256(<4 x i64>, <4 x i32>, i8)
 
 define <16 x i8> @stack_fold_vpmovuswb(<16 x i16> %a0) {
-  ;CHECK-LABEL: stack_fold_vpmovuswb
-  ;CHECK:       vpmovuswb %ymm0, {{-?[0-9]*}}(%rsp) # 16-byte Folded Spill
+; CHECK-LABEL: stack_fold_vpmovuswb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmovuswb %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Folded Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
   %1 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.wb.256(<16 x i16> %a0, <16 x i8> undef, i16 -1)
   %2 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   ret <16 x i8> %1
@@ -1442,8 +2466,15 @@ define <16 x i8> @stack_fold_vpmovuswb(<16 x i16> %a0) {
 declare <16 x i8> @llvm.x86.avx512.mask.pmovus.wb.256(<16 x i16>, <16 x i8>, i16)
 
 define <4 x i32> @stack_fold_pmovzxbd(<16 x i8> %a0) {
-  ;CHECK-LABEL: stack_fold_pmovzxbd
-  ;CHECK:       vpmovzxbd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmovzxbd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmovzxbd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <16 x i8> %a0, <16 x i8> zeroinitializer, <16 x i32> <i32 0, i32 16, i32 17, i32 18, i32 1, i32 19, i32 20, i32 21, i32 2, i32 22, i32 23, i32 24, i32 3, i32 25, i32 26, i32 27>
   %3 = bitcast <16 x i8> %2 to <4 x i32>
@@ -1451,8 +2482,15 @@ define <4 x i32> @stack_fold_pmovzxbd(<16 x i8> %a0) {
 }
 
 define <8 x i32> @stack_fold_pmovzxbd_ymm(<16 x i8> %a0) {
-  ;CHECK-LABEL: stack_fold_pmovzxbd_ymm
-  ;CHECK:       vpmovzxbd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmovzxbd_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmovzxbd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # ymm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <16 x i8> %a0, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   %3 = zext <8 x i8> %2 to <8 x i32>
@@ -1460,8 +2498,15 @@ define <8 x i32> @stack_fold_pmovzxbd_ymm(<16 x i8> %a0) {
 }
 
 define <2 x i64> @stack_fold_pmovzxbq(<16 x i8> %a0) {
-  ;CHECK-LABEL: stack_fold_pmovzxbq
-  ;CHECK:       vpmovzxbq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmovzxbq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmovzxbq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <16 x i8> %a0, <16 x i8> zeroinitializer, <16 x i32> <i32 0, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 1, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28>
   %3 = bitcast <16 x i8> %2 to <2 x i64>
@@ -1469,8 +2514,15 @@ define <2 x i64> @stack_fold_pmovzxbq(<16 x i8> %a0) {
 }
 
 define <4 x i64> @stack_fold_pmovzxbq_ymm(<16 x i8> %a0) {
-  ;CHECK-LABEL: stack_fold_pmovzxbq_ymm
-  ;CHECK:       vpmovzxbq {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmovzxbq_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmovzxbq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # ymm0 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero,mem[2],zero,zero,zero,zero,zero,zero,zero,mem[3],zero,zero,zero,zero,zero,zero,zero
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <16 x i8> %a0, <16 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %3 = zext <4 x i8> %2 to <4 x i64>
@@ -1478,8 +2530,15 @@ define <4 x i64> @stack_fold_pmovzxbq_ymm(<16 x i8> %a0) {
 }
 
 define <8 x i16> @stack_fold_pmovzxbw(<16 x i8> %a0) {
-  ;CHECK-LABEL: stack_fold_pmovzxbw
-  ;CHECK:       vpmovzxbw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmovzxbw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmovzxbw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <16 x i8> %a0, <16 x i8> zeroinitializer, <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23>
   %3 = bitcast <16 x i8> %2 to <8 x i16>
@@ -1487,16 +2546,30 @@ define <8 x i16> @stack_fold_pmovzxbw(<16 x i8> %a0) {
 }
 
 define <16 x i16> @stack_fold_pmovzxbw_ymm(<16 x i8> %a0) {
-  ;CHECK-LABEL: stack_fold_pmovzxbw_ymm
-  ;CHECK:       vpmovzxbw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmovzxbw_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmovzxbw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # ymm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = zext <16 x i8> %a0 to <16 x i16>
   ret <16 x i16> %2
 }
 
 define <2 x i64> @stack_fold_pmovzxdq(<4 x i32> %a0) {
-  ;CHECK-LABEL: stack_fold_pmovzxdq
-  ;CHECK:       vpmovzxdq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmovzxdq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmovzxdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = mem[0],zero,mem[1],zero
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <4 x i32> %a0, <4 x i32> zeroinitializer, <4 x i32> <i32 0, i32 4, i32 1, i32 5>
   %3 = bitcast <4 x i32> %2 to <2 x i64>
@@ -1504,16 +2577,30 @@ define <2 x i64> @stack_fold_pmovzxdq(<4 x i32> %a0) {
 }
 
 define <4 x i64> @stack_fold_pmovzxdq_ymm(<4 x i32> %a0) {
-  ;CHECK-LABEL: stack_fold_pmovzxdq_ymm
-  ;CHECK:       vpmovzxdq {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmovzxdq_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmovzxdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # ymm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = zext <4 x i32> %a0 to <4 x i64>
   ret <4 x i64> %2
 }
 
 define <4 x i32> @stack_fold_pmovzxwd(<8 x i16> %a0) {
-  ;CHECK-LABEL: stack_fold_pmovzxwd
-  ;CHECK:       vpmovzxwd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmovzxwd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmovzxwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <8 x i16> %a0, <8 x i16> zeroinitializer, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11>
   %3 = bitcast <8 x i16> %2 to <4 x i32>
@@ -1521,16 +2608,30 @@ define <4 x i32> @stack_fold_pmovzxwd(<8 x i16> %a0) {
 }
 
 define <8 x i32> @stack_fold_pmovzxwd_ymm(<8 x i16> %a0) {
-  ;CHECK-LABEL: stack_fold_pmovzxwd_ymm
-  ;CHECK:       vpmovzxwd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmovzxwd_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmovzxwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # ymm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = zext <8 x i16> %a0 to <8 x i32>
   ret <8 x i32> %2
 }
 
 define <2 x i64> @stack_fold_pmovzxwq(<8 x i16> %a0) {
-  ;CHECK-LABEL: stack_fold_pmovzxwq
-  ;CHECK:       vpmovzxwq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmovzxwq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmovzxwq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <8 x i16> %a0, <8 x i16> zeroinitializer, <8 x i32> <i32 0, i32 8, i32 9, i32 10, i32 1, i32 11, i32 12, i32 13>
   %3 = bitcast <8 x i16> %2 to <2 x i64>
@@ -1538,8 +2639,15 @@ define <2 x i64> @stack_fold_pmovzxwq(<8 x i16> %a0) {
 }
 
 define <4 x i64> @stack_fold_pmovzxwq_ymm(<8 x i16> %a0) {
-  ;CHECK-LABEL: stack_fold_pmovzxwq_ymm
-  ;CHECK:       vpmovzxwq {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmovzxwq_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmovzxwq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # ymm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <8 x i16> %a0, <8 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %3 = zext <4 x i16> %2 to <4 x i64>
@@ -1547,8 +2655,16 @@ define <4 x i64> @stack_fold_pmovzxwq_ymm(<8 x i16> %a0) {
 }
 
 define <4 x i64> @stack_fold_pmovzxwq_maskz_ymm(<8 x i16> %a0, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_pmovzxwq_maskz_ymm
-  ;CHECK:       vpmovzxwq {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmovzxwq_maskz_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpmovzxwq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 {%k1} {z} # 16-byte Folded Reload
+; CHECK-NEXT:    # ymm0 {%k1} {z} = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <8 x i16> %a0, <8 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %3 = zext <4 x i16> %2 to <4 x i64>
@@ -1559,8 +2675,16 @@ define <4 x i64> @stack_fold_pmovzxwq_maskz_ymm(<8 x i16> %a0, i8 %mask) {
 }
 
 define <4 x i64> @stack_fold_pmovzxwq_mask_ymm(<4 x i64> %passthru, <8 x i16> %a0, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_pmovzxwq_mask_ymm
-  ;CHECK:       vpmovzxwq {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmovzxwq_mask_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpmovzxwq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 {%k1} # 16-byte Folded Reload
+; CHECK-NEXT:    # ymm0 {%k1} = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <8 x i16> %a0, <8 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %3 = zext <4 x i16> %2 to <4 x i64>
@@ -1571,8 +2695,14 @@ define <4 x i64> @stack_fold_pmovzxwq_mask_ymm(<4 x i64> %passthru, <8 x i16> %a
 }
 
 define <2 x i64> @stack_fold_pmuldq(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_pmuldq
-  ;CHECK:       vpmuldq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmuldq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmuldq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = bitcast <4 x i32> %a0 to <2 x i64>
   %3 = bitcast <4 x i32> %a1 to <2 x i64>
@@ -1585,8 +2715,14 @@ define <2 x i64> @stack_fold_pmuldq(<4 x i32> %a0, <4 x i32> %a1) {
 }
 
 define <4 x i64> @stack_fold_pmuldq_ymm(<8 x i32> %a0, <8 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_pmuldq_ymm
-  ;CHECK:       vpmuldq {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmuldq_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmuldq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = bitcast <8 x i32> %a0 to <4 x i64>
   %3 = bitcast <8 x i32> %a1 to <4 x i64>
@@ -1599,8 +2735,14 @@ define <4 x i64> @stack_fold_pmuldq_ymm(<8 x i32> %a0, <8 x i32> %a1) {
 }
 
 define <2 x i64> @stack_fold_pmuludq(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_pmuludq
-  ;CHECK:       vpmuludq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmuludq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmuludq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = bitcast <4 x i32> %a0 to <2 x i64>
   %3 = bitcast <4 x i32> %a1 to <2 x i64>
@@ -1611,8 +2753,14 @@ define <2 x i64> @stack_fold_pmuludq(<4 x i32> %a0, <4 x i32> %a1) {
 }
 
 define <4 x i64> @stack_fold_pmuludq_ymm(<8 x i32> %a0, <8 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_pmuludq_ymm
-  ;CHECK:       vpmuludq {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmuludq_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpmuludq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = bitcast <8 x i32> %a0 to <4 x i64>
   %3 = bitcast <8 x i32> %a1 to <4 x i64>
@@ -1623,8 +2771,19 @@ define <4 x i64> @stack_fold_pmuludq_ymm(<8 x i32> %a0, <8 x i32> %a1) {
 }
 
 define <4 x i64> @stack_fold_pmuludq_ymm_mask(<4 x i64>* %passthru, <8 x i32> %a0, <8 x i32> %a1, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_pmuludq_ymm_mask
-  ;CHECK:       vpmuludq {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmuludq_ymm_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vmovdqa (%rdi), %ymm1
+; CHECK-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; CHECK-NEXT:    vpmuludq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 {%k1} # 32-byte Folded Reload
+; CHECK-NEXT:    vmovdqa %ymm1, %ymm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = bitcast <8 x i32> %a0 to <4 x i64>
   %3 = bitcast <8 x i32> %a1 to <4 x i64>
@@ -1639,8 +2798,17 @@ define <4 x i64> @stack_fold_pmuludq_ymm_mask(<4 x i64>* %passthru, <8 x i32> %a
 }
 
 define <4 x i64> @stack_fold_pmuludq_ymm_maskz(<8 x i32> %a0, <8 x i32> %a1, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_pmuludq_ymm_maskz
-  ;CHECK:       vpmuludq {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmuludq_ymm_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; CHECK-NEXT:    vpmuludq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 {%k1} {z} # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = bitcast <8 x i32> %a0 to <4 x i64>
   %3 = bitcast <8 x i32> %a1 to <4 x i64>
@@ -1654,8 +2822,14 @@ define <4 x i64> @stack_fold_pmuludq_ymm_maskz(<8 x i32> %a0, <8 x i32> %a1, i8
 }
 
 define <4 x i32> @stack_fold_vpopcntd(<4 x i32> %a0) {
-  ;CHECK-LABEL: stack_fold_vpopcntd
-  ;CHECK:       vpopcntd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpopcntd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpopcntd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> %a0)
   ret <4 x i32> %2
@@ -1663,8 +2837,14 @@ define <4 x i32> @stack_fold_vpopcntd(<4 x i32> %a0) {
 declare <4 x i32> @llvm.ctpop.v4i32(<4 x i32>) nounwind readonly
 
 define <8 x i32> @stack_fold_vpopcntd_ymm(<8 x i32> %a0) {
-  ;CHECK-LABEL: stack_fold_vpopcntd_ymm
-  ;CHECK:       vpopcntd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpopcntd_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpopcntd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x i32> @llvm.ctpop.v8i32(<8 x i32> %a0)
   ret <8 x i32> %2
@@ -1672,8 +2852,14 @@ define <8 x i32> @stack_fold_vpopcntd_ymm(<8 x i32> %a0) {
 declare <8 x i32> @llvm.ctpop.v8i32(<8 x i32>) nounwind readonly
 
 define <2 x i64> @stack_fold_vpopcntq(<2 x i64> %a0) {
-  ;CHECK-LABEL: stack_fold_vpopcntq
-  ;CHECK:       vpopcntq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpopcntq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpopcntq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %a0)
   ret <2 x i64> %2
@@ -1681,8 +2867,14 @@ define <2 x i64> @stack_fold_vpopcntq(<2 x i64> %a0) {
 declare <2 x i64> @llvm.ctpop.v2i64(<2 x i64>) nounwind readnone
 
 define <4 x i64> @stack_fold_vpopcntq_ymm(<4 x i64> %a0) {
-  ;CHECK-LABEL: stack_fold_vpopcntq_ymm
-  ;CHECK:       vpopcntq {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpopcntq_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpopcntq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <4 x i64> @llvm.ctpop.v4i64(<4 x i64> %a0)
   ret <4 x i64> %2
@@ -1690,26 +2882,72 @@ define <4 x i64> @stack_fold_vpopcntq_ymm(<4 x i64> %a0) {
 declare <4 x i64> @llvm.ctpop.v4i64(<4 x i64>) nounwind readnone
 
 define <2 x i64> @stack_fold_psadbw(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_psadbw
-  ;CHECK:       vpsadbw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psadbw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsadbw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <2 x i64> @llvm.x86.sse2.psad.bw(<16 x i8> %a0, <16 x i8> %a1)
   ret <2 x i64> %2
 }
 declare <2 x i64> @llvm.x86.sse2.psad.bw(<16 x i8>, <16 x i8>) nounwind readnone
 
+define <2 x i64> @stack_fold_psadbw_commute(<16 x i8> %a0, <16 x i8> %a1) {
+; CHECK-LABEL: stack_fold_psadbw_commute:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsadbw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
+  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
+  %2 = call <2 x i64> @llvm.x86.sse2.psad.bw(<16 x i8> %a1, <16 x i8> %a0)
+  ret <2 x i64> %2
+}
+
 define <4 x i64> @stack_fold_psadbw_ymm(<32 x i8> %a0, <32 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_psadbw_ymm
-  ;CHECK:       vpsadbw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_psadbw_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsadbw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <4 x i64> @llvm.x86.avx2.psad.bw(<32 x i8> %a0, <32 x i8> %a1)
   ret <4 x i64> %2
 }
 declare <4 x i64> @llvm.x86.avx2.psad.bw(<32 x i8>, <32 x i8>) nounwind readnone
 
+define <4 x i64> @stack_fold_psadbw_ymm_commute(<32 x i8> %a0, <32 x i8> %a1) {
+; CHECK-LABEL: stack_fold_psadbw_ymm_commute:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsadbw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
+  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
+  %2 = call <4 x i64> @llvm.x86.avx2.psad.bw(<32 x i8> %a1, <32 x i8> %a0)
+  ret <4 x i64> %2
+}
+
 define <16 x i8> @stack_fold_pshufb(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pshufb
-  ;CHECK:       vpshufb {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pshufb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpshufb {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %a0, <16 x i8> %a1)
   ret <16 x i8> %2
@@ -1717,8 +2955,17 @@ define <16 x i8> @stack_fold_pshufb(<16 x i8> %a0, <16 x i8> %a1) {
 declare <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8>, <16 x i8>) nounwind readnone
 
 define <16 x i8> @stack_fold_pshufb_mask(<16 x i8>* %passthru, <16 x i8> %a0, <16 x i8> %a1, i16 %mask) {
-  ;CHECK-LABEL: stack_fold_pshufb_mask
-  ;CHECK:       vpshufb {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pshufb_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovdqa (%rdi), %xmm2
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vpshufb {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm2 {%k1} # 16-byte Folded Reload
+; CHECK-NEXT:    vmovdqa %xmm2, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = load <16 x i8>, <16 x i8>* %passthru
   %3 = call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %a0, <16 x i8> %a1)
@@ -1728,8 +2975,15 @@ define <16 x i8> @stack_fold_pshufb_mask(<16 x i8>* %passthru, <16 x i8> %a0, <1
 }
 
 define <16 x i8> @stack_fold_pshufb_maskz(<16 x i8> %a0, <16 x i8> %a1, i16 %mask) {
-  ;CHECK-LABEL: stack_fold_pshufb_maskz
-  ;CHECK:       vpshufb {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pshufb_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpshufb {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 {%k1} {z} # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %a0, <16 x i8> %a1)
   %3 = bitcast i16 %mask to <16 x i1>
@@ -1738,8 +2992,14 @@ define <16 x i8> @stack_fold_pshufb_maskz(<16 x i8> %a0, <16 x i8> %a1, i16 %mas
 }
 
 define <32 x i8> @stack_fold_pshufb_ymm(<32 x i8> %a0, <32 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pshufb_ymm
-  ;CHECK:       vpshufb {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pshufb_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpshufb {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> %a0, <32 x i8> %a1)
   ret <32 x i8> %2
@@ -1747,8 +3007,17 @@ define <32 x i8> @stack_fold_pshufb_ymm(<32 x i8> %a0, <32 x i8> %a1) {
 declare <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8>, <32 x i8>)
 
 define <32 x i8> @stack_fold_pshufb_ymm_mask(<32 x i8>* %passthru, <32 x i8> %a0, <32 x i8> %a1, i32 %mask) {
-  ;CHECK-LABEL: stack_fold_pshufb_ymm_mask
-  ;CHECK:       vpshufb {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pshufb_ymm_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovdqa (%rdi), %ymm2
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vpshufb {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm2 {%k1} # 32-byte Folded Reload
+; CHECK-NEXT:    vmovdqa %ymm2, %ymm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = load <32 x i8>, <32 x i8>* %passthru
   %3 = call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> %a0, <32 x i8> %a1)
@@ -1758,8 +3027,15 @@ define <32 x i8> @stack_fold_pshufb_ymm_mask(<32 x i8>* %passthru, <32 x i8> %a0
 }
 
 define <32 x i8> @stack_fold_pshufb_ymm_maskz(<32 x i8> %a0, <32 x i8> %a1, i32 %mask) {
-  ;CHECK-LABEL: stack_fold_pshufb_ymm_maskz
-  ;CHECK:       vpshufb {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pshufb_ymm_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpshufb {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 {%k1} {z} # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> %a0, <32 x i8> %a1)
   %3 = bitcast i32 %mask to <32 x i1>
@@ -1768,8 +3044,17 @@ define <32 x i8> @stack_fold_pshufb_ymm_maskz(<32 x i8> %a0, <32 x i8> %a1, i32
 }
 
 define <4 x i32> @stack_fold_pshufd(<4 x i32> %a0) {
-  ;CHECK-LABEL: stack_fold_pshufd
-  ;CHECK:       vpshufd $27, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pshufd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpshufd $27, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = mem[3,2,1,0]
+; CHECK-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <4 x i32> %a0, <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
   %3 = add <4 x i32> %2, <i32 1, i32 1, i32 1, i32 1>
@@ -1777,8 +3062,18 @@ define <4 x i32> @stack_fold_pshufd(<4 x i32> %a0) {
 }
 
 define <4 x i32> @stack_fold_pshufd_mask(<4 x i32> %passthru, <4 x i32> %a0, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_pshufd_mask
-  ;CHECK:       vpshufd $27, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pshufd_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    vpshufd $27, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 {%k1} # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 {%k1} = mem[3,2,1,0]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <4 x i32> %a0, <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
   %3 = bitcast i8 %mask to <8 x i1>
@@ -1788,8 +3083,16 @@ define <4 x i32> @stack_fold_pshufd_mask(<4 x i32> %passthru, <4 x i32> %a0, i8
 }
 
 define <4 x i32> @stack_fold_pshufd_maskz(<4 x i32> %a0, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_pshufd_maskz
-  ;CHECK:       vpshufd $27, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pshufd_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpshufd $27, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 {%k1} {z} # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 {%k1} {z} = mem[3,2,1,0]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <4 x i32> %a0, <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
   %3 = bitcast i8 %mask to <8 x i1>
@@ -1799,8 +3102,17 @@ define <4 x i32> @stack_fold_pshufd_maskz(<4 x i32> %a0, i8 %mask) {
 }
 
 define <8 x i32> @stack_fold_pshufd_ymm(<8 x i32> %a0) {
-  ;CHECK-LABEL: stack_fold_pshufd_ymm
-  ;CHECK:       vpshufd $27, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pshufd_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpshufd $27, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    # ymm0 = mem[3,2,1,0,7,6,5,4]
+; CHECK-NEXT:    vpcmpeqd %ymm1, %ymm1, %ymm1
+; CHECK-NEXT:    vpsubd %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <8 x i32> %a0, <8 x i32> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
   %3 = add <8 x i32> %2, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
@@ -1808,8 +3120,18 @@ define <8 x i32> @stack_fold_pshufd_ymm(<8 x i32> %a0) {
 }
 
 define <8 x i32> @stack_fold_pshufd_ymm_mask(<8 x i32> %passthru, <8 x i32> %a0, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_pshufd_ymm_mask
-  ;CHECK:       vpshufd $27, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pshufd_ymm_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; CHECK-NEXT:    vpshufd $27, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 {%k1} # 32-byte Folded Reload
+; CHECK-NEXT:    # ymm0 {%k1} = mem[3,2,1,0,7,6,5,4]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <8 x i32> %a0, <8 x i32> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
   %3 = bitcast i8 %mask to <8 x i1>
@@ -1818,8 +3140,16 @@ define <8 x i32> @stack_fold_pshufd_ymm_mask(<8 x i32> %passthru, <8 x i32> %a0,
 }
 
 define <8 x i32> @stack_fold_pshufd_ymm_maskz(<8 x i32> %a0, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_pshufd_ymm_maskz
-  ;CHECK:       vpshufd $27, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pshufd_ymm_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpshufd $27, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 {%k1} {z} # 32-byte Folded Reload
+; CHECK-NEXT:    # ymm0 {%k1} {z} = mem[3,2,1,0,7,6,5,4]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <8 x i32> %a0, <8 x i32> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
   %3 = bitcast i8 %mask to <8 x i1>
@@ -1828,16 +3158,33 @@ define <8 x i32> @stack_fold_pshufd_ymm_maskz(<8 x i32> %a0, i8 %mask) {
 }
 
 define <8 x i16> @stack_fold_pshufhw(<8 x i16> %a0) {
-  ;CHECK-LABEL: stack_fold_pshufhw
-  ;CHECK:       vpshufhw $11, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pshufhw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpshufhw $11, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = mem[0,1,2,3,7,6,4,4]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <8 x i16> %a0, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 7, i32 6, i32 4, i32 4>
   ret <8 x i16> %2
 }
 
 define <8 x i16> @stack_fold_pshufhw_mask(<8 x i16> %passthru, <8 x i16> %a0, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_pshufhw_mask
-  ;CHECK:       vpshufhw $11, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pshufhw_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    vpshufhw $11, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 {%k1} # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 {%k1} = mem[0,1,2,3,7,6,4,4]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <8 x i16> %a0, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 7, i32 6, i32 4, i32 4>
   %3 = bitcast i8 %mask to <8 x i1>
@@ -1846,8 +3193,16 @@ define <8 x i16> @stack_fold_pshufhw_mask(<8 x i16> %passthru, <8 x i16> %a0, i8
 }
 
 define <8 x i16> @stack_fold_pshufhw_maskz(<8 x i16> %a0, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_pshufhw_maskz
-  ;CHECK:       vpshufhw $11, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pshufhw_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpshufhw $11, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 {%k1} {z} # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 {%k1} {z} = mem[0,1,2,3,7,6,4,4]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <8 x i16> %a0, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 7, i32 6, i32 4, i32 4>
   %3 = bitcast i8 %mask to <8 x i1>
@@ -1856,16 +3211,33 @@ define <8 x i16> @stack_fold_pshufhw_maskz(<8 x i16> %a0, i8 %mask) {
 }
 
 define <16 x i16> @stack_fold_pshufhw_ymm(<16 x i16> %a0) {
-  ;CHECK-LABEL: stack_fold_pshufhw_ymm
-  ;CHECK:       vpshufhw $27, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pshufhw_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpshufhw $27, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    # ymm0 = mem[0,1,2,3,7,6,5,4,8,9,10,11,15,14,13,12]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <16 x i16> %a0, <16 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 7, i32 6, i32 5, i32 4, i32 8, i32 9, i32 10, i32 11, i32 15, i32 14, i32 13, i32 12>
   ret <16 x i16> %2
 }
 
 define <16 x i16> @stack_fold_pshufhw_ymm_mask(<16 x i16> %passthru, <16 x i16> %a0, i16 %mask) {
-  ;CHECK-LABEL: stack_fold_pshufhw_ymm_mask
-  ;CHECK:       vpshufhw $27, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pshufhw_ymm_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; CHECK-NEXT:    vpshufhw $27, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 {%k1} # 32-byte Folded Reload
+; CHECK-NEXT:    # ymm0 {%k1} = mem[0,1,2,3,7,6,5,4,8,9,10,11,15,14,13,12]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <16 x i16> %a0, <16 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 7, i32 6, i32 5, i32 4, i32 8, i32 9, i32 10, i32 11, i32 15, i32 14, i32 13, i32 12>
   %3 = bitcast i16 %mask to <16 x i1>
@@ -1874,8 +3246,16 @@ define <16 x i16> @stack_fold_pshufhw_ymm_mask(<16 x i16> %passthru, <16 x i16>
 }
 
 define <16 x i16> @stack_fold_pshufhw_ymm_maskz(<16 x i16> %a0, i16 %mask) {
-  ;CHECK-LABEL: stack_fold_pshufhw_ymm_maskz
-  ;CHECK:       vpshufhw $27, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pshufhw_ymm_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpshufhw $27, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 {%k1} {z} # 32-byte Folded Reload
+; CHECK-NEXT:    # ymm0 {%k1} {z} = mem[0,1,2,3,7,6,5,4,8,9,10,11,15,14,13,12]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <16 x i16> %a0, <16 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 7, i32 6, i32 5, i32 4, i32 8, i32 9, i32 10, i32 11, i32 15, i32 14, i32 13, i32 12>
   %3 = bitcast i16 %mask to <16 x i1>
@@ -1884,16 +3264,33 @@ define <16 x i16> @stack_fold_pshufhw_ymm_maskz(<16 x i16> %a0, i16 %mask) {
 }
 
 define <8 x i16> @stack_fold_pshuflw(<8 x i16> %a0) {
-  ;CHECK-LABEL: stack_fold_pshuflw
-  ;CHECK:       vpshuflw $27, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pshuflw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpshuflw $27, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = mem[3,2,1,0,4,5,6,7]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <8 x i16> %a0, <8 x i16> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 4, i32 5, i32 6, i32 7>
   ret <8 x i16> %2
 }
 
 define <8 x i16> @stack_fold_pshuflw_mask(<8 x i16> %passthru, <8 x i16> %a0, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_pshuflw_mask
-  ;CHECK:       vpshuflw $27, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pshuflw_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    vpshuflw $27, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 {%k1} # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 {%k1} = mem[3,2,1,0,4,5,6,7]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <8 x i16> %a0, <8 x i16> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 4, i32 5, i32 6, i32 7>
   %3 = bitcast i8 %mask to <8 x i1>
@@ -1902,8 +3299,16 @@ define <8 x i16> @stack_fold_pshuflw_mask(<8 x i16> %passthru, <8 x i16> %a0, i8
 }
 
 define <8 x i16> @stack_fold_pshuflw_maskz(<8 x i16> %a0, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_pshuflw_maskz
-  ;CHECK:       vpshuflw $27, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pshuflw_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpshuflw $27, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 {%k1} {z} # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 {%k1} {z} = mem[3,2,1,0,4,5,6,7]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <8 x i16> %a0, <8 x i16> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 4, i32 5, i32 6, i32 7>
   %3 = bitcast i8 %mask to <8 x i1>
@@ -1912,16 +3317,33 @@ define <8 x i16> @stack_fold_pshuflw_maskz(<8 x i16> %a0, i8 %mask) {
 }
 
 define <16 x i16> @stack_fold_pshuflw_ymm(<16 x i16> %a0) {
-  ;CHECK-LABEL: stack_fold_pshuflw_ymm
-  ;CHECK:       vpshuflw $27, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pshuflw_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpshuflw $27, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    # ymm0 = mem[3,2,1,0,4,5,6,7,11,10,9,8,12,13,14,15]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <16 x i16> %a0, <16 x i16> undef, <16 x i32> <i32 3, i32 2, i32 1, i32 0, i32 4, i32 5, i32 6, i32 7, i32 11, i32 10, i32 9, i32 8, i32 12, i32 13, i32 14, i32 15>
   ret <16 x i16> %2
 }
 
 define <16 x i16> @stack_fold_pshuflw_ymm_mask(<16 x i16> %passthru, <16 x i16> %a0, i16 %mask) {
-  ;CHECK-LABEL: stack_fold_pshuflw_ymm_mask
-  ;CHECK:       vpshuflw $27, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pshuflw_ymm_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; CHECK-NEXT:    vpshuflw $27, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 {%k1} # 32-byte Folded Reload
+; CHECK-NEXT:    # ymm0 {%k1} = mem[3,2,1,0,4,5,6,7,11,10,9,8,12,13,14,15]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <16 x i16> %a0, <16 x i16> undef, <16 x i32> <i32 3, i32 2, i32 1, i32 0, i32 4, i32 5, i32 6, i32 7, i32 11, i32 10, i32 9, i32 8, i32 12, i32 13, i32 14, i32 15>
   %3 = bitcast i16 %mask to <16 x i1>
@@ -1930,8 +3352,16 @@ define <16 x i16> @stack_fold_pshuflw_ymm_mask(<16 x i16> %passthru, <16 x i16>
 }
 
 define <16 x i16> @stack_fold_pshuflw_ymm_maskz(<16 x i16> %a0, i16 %mask) {
-  ;CHECK-LABEL: stack_fold_pshuflw_ymm_maskz
-  ;CHECK:       vpshuflw $27, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pshuflw_ymm_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpshuflw $27, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 {%k1} {z} # 32-byte Folded Reload
+; CHECK-NEXT:    # ymm0 {%k1} {z} = mem[3,2,1,0,4,5,6,7,11,10,9,8,12,13,14,15]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <16 x i16> %a0, <16 x i16> undef, <16 x i32> <i32 3, i32 2, i32 1, i32 0, i32 4, i32 5, i32 6, i32 7, i32 11, i32 10, i32 9, i32 8, i32 12, i32 13, i32 14, i32 15>
   %3 = bitcast i16 %mask to <16 x i1>
@@ -1940,8 +3370,14 @@ define <16 x i16> @stack_fold_pshuflw_ymm_maskz(<16 x i16> %a0, i16 %mask) {
 }
 
 define <4 x i32> @stack_fold_pslld(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_pslld
-  ;CHECK:       vpslld {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pslld:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpslld {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <4 x i32> @llvm.x86.sse2.psll.d(<4 x i32> %a0, <4 x i32> %a1)
   ret <4 x i32> %2
@@ -1949,8 +3385,14 @@ define <4 x i32> @stack_fold_pslld(<4 x i32> %a0, <4 x i32> %a1) {
 declare <4 x i32> @llvm.x86.sse2.psll.d(<4 x i32>, <4 x i32>) nounwind readnone
 
 define <8 x i32> @stack_fold_pslld_ymm(<8 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_pslld_ymm
-  ;CHECK:       vpslld {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pslld_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpslld {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x i32> @llvm.x86.avx2.psll.d(<8 x i32> %a0, <4 x i32> %a1)
   ret <8 x i32> %2
@@ -1958,24 +3400,44 @@ define <8 x i32> @stack_fold_pslld_ymm(<8 x i32> %a0, <4 x i32> %a1) {
 declare <8 x i32> @llvm.x86.avx2.psll.d(<8 x i32>, <4 x i32>) nounwind readnone
 
 define <16 x i8> @stack_fold_pslldq(<16 x i8> %a) {
-  ;CHECK-LABEL: stack_fold_pslldq
-  ;CHECK:       vpslldq $12, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pslldq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpslldq $12, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,mem[0,1,2,3]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <16 x i8> zeroinitializer, <16 x i8> %a, <16 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 16, i32 17, i32 18, i32 19>
   ret <16 x i8> %2
 }
 
 define <32 x i8> @stack_fold_pslldq_ymm(<32 x i8> %a) {
-  ;CHECK-LABEL: stack_fold_pslldq_ymm
-  ;CHECK:       vpslldq $15, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_pslldq_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpslldq $15, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    # ymm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,mem[0],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,mem[16]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <32 x i8> zeroinitializer, <32 x i8> %a, <32 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 32, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 48>
   ret <32 x i8> %2
 }
 
 define <2 x i64> @stack_fold_psllq(<2 x i64> %a0, <2 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_psllq
-  ;CHECK:       vpsllq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psllq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsllq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <2 x i64> @llvm.x86.sse2.psll.q(<2 x i64> %a0, <2 x i64> %a1)
   ret <2 x i64> %2
@@ -1983,8 +3445,14 @@ define <2 x i64> @stack_fold_psllq(<2 x i64> %a0, <2 x i64> %a1) {
 declare <2 x i64> @llvm.x86.sse2.psll.q(<2 x i64>, <2 x i64>) nounwind readnone
 
 define <4 x i64> @stack_fold_psllq_ymm(<4 x i64> %a0, <2 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_psllq_ymm
-  ;CHECK:       vpsllq {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psllq_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsllq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <4 x i64> @llvm.x86.avx2.psll.q(<4 x i64> %a0, <2 x i64> %a1)
   ret <4 x i64> %2
@@ -1992,8 +3460,14 @@ define <4 x i64> @stack_fold_psllq_ymm(<4 x i64> %a0, <2 x i64> %a1) {
 declare <4 x i64> @llvm.x86.avx2.psll.q(<4 x i64>, <2 x i64>) nounwind readnone
 
 define <4 x i32> @stack_fold_psllvd(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_psllvd
-  ;CHECK:       vpsllvd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psllvd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsllvd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <4 x i32> @llvm.x86.avx2.psllv.d(<4 x i32> %a0, <4 x i32> %a1)
   ret <4 x i32> %2
@@ -2001,8 +3475,14 @@ define <4 x i32> @stack_fold_psllvd(<4 x i32> %a0, <4 x i32> %a1) {
 declare <4 x i32> @llvm.x86.avx2.psllv.d(<4 x i32>, <4 x i32>) nounwind readnone
 
 define <8 x i32> @stack_fold_psllvd_ymm(<8 x i32> %a0, <8 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_psllvd_ymm
-  ;CHECK:       vpsllvd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_psllvd_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsllvd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x i32> @llvm.x86.avx2.psllv.d.256(<8 x i32> %a0, <8 x i32> %a1)
   ret <8 x i32> %2
@@ -2010,8 +3490,14 @@ define <8 x i32> @stack_fold_psllvd_ymm(<8 x i32> %a0, <8 x i32> %a1) {
 declare <8 x i32> @llvm.x86.avx2.psllv.d.256(<8 x i32>, <8 x i32>) nounwind readnone
 
 define <2 x i64> @stack_fold_psllvq(<2 x i64> %a0, <2 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_psllvq
-  ;CHECK:       vpsllvq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psllvq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsllvq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <2 x i64> @llvm.x86.avx2.psllv.q(<2 x i64> %a0, <2 x i64> %a1)
   ret <2 x i64> %2
@@ -2019,8 +3505,14 @@ define <2 x i64> @stack_fold_psllvq(<2 x i64> %a0, <2 x i64> %a1) {
 declare <2 x i64> @llvm.x86.avx2.psllv.q(<2 x i64>, <2 x i64>) nounwind readnone
 
 define <4 x i64> @stack_fold_psllvq_ymm(<4 x i64> %a0, <4 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_psllvq_ymm
-  ;CHECK:       vpsllvq {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_psllvq_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsllvq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <4 x i64> @llvm.x86.avx2.psllv.q.256(<4 x i64> %a0, <4 x i64> %a1)
   ret <4 x i64> %2
@@ -2028,8 +3520,14 @@ define <4 x i64> @stack_fold_psllvq_ymm(<4 x i64> %a0, <4 x i64> %a1) {
 declare <4 x i64> @llvm.x86.avx2.psllv.q.256(<4 x i64>, <4 x i64>) nounwind readnone
 
 define <8 x i16> @stack_fold_psllvw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_psllvw
-  ;CHECK:       vpsllvw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psllvw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsllvw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x i16> @llvm.x86.avx512.psllv.w.128(<8 x i16> %a0, <8 x i16> %a1)
   ret <8 x i16> %2
@@ -2037,8 +3535,14 @@ define <8 x i16> @stack_fold_psllvw(<8 x i16> %a0, <8 x i16> %a1) {
 declare <8 x i16> @llvm.x86.avx512.psllv.w.128(<8 x i16>, <8 x i16>) nounwind readnone
 
 define <16 x i16> @stack_fold_psllvw_ymm(<16 x i16> %a0, <16 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_psllvw_ymm
-  ;CHECK:       vpsllvw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_psllvw_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsllvw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x i16> @llvm.x86.avx512.psllv.w.256(<16 x i16> %a0, <16 x i16> %a1)
   ret <16 x i16> %2
@@ -2046,8 +3550,14 @@ define <16 x i16> @stack_fold_psllvw_ymm(<16 x i16> %a0, <16 x i16> %a1) {
 declare <16 x i16> @llvm.x86.avx512.psllv.w.256(<16 x i16>, <16 x i16>) nounwind readnone
 
 define <8 x i16> @stack_fold_psllw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_psllw
-  ;CHECK:       vpsllw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psllw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsllw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x i16> @llvm.x86.sse2.psll.w(<8 x i16> %a0, <8 x i16> %a1)
   ret <8 x i16> %2
@@ -2055,8 +3565,14 @@ define <8 x i16> @stack_fold_psllw(<8 x i16> %a0, <8 x i16> %a1) {
 declare <8 x i16> @llvm.x86.sse2.psll.w(<8 x i16>, <8 x i16>) nounwind readnone
 
 define <16 x i16> @stack_fold_psllw_ymm(<16 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_psllw_ymm
-  ;CHECK:       vpsllw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psllw_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsllw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x i16> @llvm.x86.avx2.psll.w(<16 x i16> %a0, <8 x i16> %a1)
   ret <16 x i16> %2
@@ -2064,8 +3580,14 @@ define <16 x i16> @stack_fold_psllw_ymm(<16 x i16> %a0, <8 x i16> %a1) {
 declare <16 x i16> @llvm.x86.avx2.psll.w(<16 x i16>, <8 x i16>) nounwind readnone
 
 define <4 x i32> @stack_fold_psrad(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_psrad
-  ;CHECK:       vpsrad {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psrad:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsrad {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <4 x i32> @llvm.x86.sse2.psra.d(<4 x i32> %a0, <4 x i32> %a1)
   ret <4 x i32> %2
@@ -2073,8 +3595,14 @@ define <4 x i32> @stack_fold_psrad(<4 x i32> %a0, <4 x i32> %a1) {
 declare <4 x i32> @llvm.x86.sse2.psra.d(<4 x i32>, <4 x i32>) nounwind readnone
 
 define <8 x i32> @stack_fold_psrad_ymm(<8 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_psrad_ymm
-  ;CHECK:       vpsrad {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psrad_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsrad {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x i32> @llvm.x86.avx2.psra.d(<8 x i32> %a0, <4 x i32> %a1)
   ret <8 x i32> %2
@@ -2082,8 +3610,14 @@ define <8 x i32> @stack_fold_psrad_ymm(<8 x i32> %a0, <4 x i32> %a1) {
 declare <8 x i32> @llvm.x86.avx2.psra.d(<8 x i32>, <4 x i32>) nounwind readnone
 
 define <2 x i64> @stack_fold_psraq(<2 x i64> %a0, <2 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_psraq
-  ;CHECK:       vpsraq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psraq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsraq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <2 x i64> @llvm.x86.avx512.psra.q.128(<2 x i64> %a0, <2 x i64> %a1)
   ret <2 x i64> %2
@@ -2091,8 +3625,14 @@ define <2 x i64> @stack_fold_psraq(<2 x i64> %a0, <2 x i64> %a1) {
 declare <2 x i64> @llvm.x86.avx512.psra.q.128(<2 x i64>, <2 x i64>) nounwind readnone
 
 define <4 x i64> @stack_fold_psraq_ymm(<4 x i64> %a0, <2 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_psraq_ymm
-  ;CHECK:       vpsraq {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psraq_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsraq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <4 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <4 x i64> @llvm.x86.avx512.psra.q.256(<4 x i64> %a0, <2 x i64> %a1)
   ret <4 x i64> %2
@@ -2100,8 +3640,14 @@ define <4 x i64> @stack_fold_psraq_ymm(<4 x i64> %a0, <2 x i64> %a1) {
 declare <4 x i64> @llvm.x86.avx512.psra.q.256(<4 x i64>, <2 x i64>) nounwind readnone
 
 define <4 x i32> @stack_fold_psravd(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_psravd
-  ;CHECK:       vpsravd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psravd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsravd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <4 x i32> @llvm.x86.avx2.psrav.d(<4 x i32> %a0, <4 x i32> %a1)
   ret <4 x i32> %2
@@ -2109,8 +3655,14 @@ define <4 x i32> @stack_fold_psravd(<4 x i32> %a0, <4 x i32> %a1) {
 declare <4 x i32> @llvm.x86.avx2.psrav.d(<4 x i32>, <4 x i32>) nounwind readnone
 
 define <8 x i32> @stack_fold_psravd_ymm(<8 x i32> %a0, <8 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_psravd_ymm
-  ;CHECK:       vpsravd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_psravd_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsravd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x i32> @llvm.x86.avx2.psrav.d.256(<8 x i32> %a0, <8 x i32> %a1)
   ret <8 x i32> %2
@@ -2118,8 +3670,14 @@ define <8 x i32> @stack_fold_psravd_ymm(<8 x i32> %a0, <8 x i32> %a1) {
 declare <8 x i32> @llvm.x86.avx2.psrav.d.256(<8 x i32>, <8 x i32>) nounwind readnone
 
 define <2 x i64> @stack_fold_psravq(<2 x i64> %a0, <2 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_psravq
-  ;CHECK:       vpsravq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psravq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsravq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <2 x i64> @llvm.x86.avx512.psrav.q.128(<2 x i64> %a0, <2 x i64> %a1)
   ret <2 x i64> %2
@@ -2127,8 +3685,14 @@ define <2 x i64> @stack_fold_psravq(<2 x i64> %a0, <2 x i64> %a1) {
 declare <2 x i64> @llvm.x86.avx512.psrav.q.128(<2 x i64>, <2 x i64>) nounwind readnone
 
 define <4 x i64> @stack_fold_psravq_ymm(<4 x i64> %a0, <4 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_psravq_ymm
-  ;CHECK:       vpsravq {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_psravq_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsravq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <4 x i64> @llvm.x86.avx512.psrav.q.256(<4 x i64> %a0, <4 x i64> %a1)
   ret <4 x i64> %2
@@ -2136,8 +3700,14 @@ define <4 x i64> @stack_fold_psravq_ymm(<4 x i64> %a0, <4 x i64> %a1) {
 declare <4 x i64> @llvm.x86.avx512.psrav.q.256(<4 x i64>, <4 x i64>) nounwind readnone
 
 define <8 x i16> @stack_fold_psravw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_psravw
-  ;CHECK:       vpsravw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psravw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsravw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x i16> @llvm.x86.avx512.psrav.w.128(<8 x i16> %a0, <8 x i16> %a1)
   ret <8 x i16> %2
@@ -2145,8 +3715,14 @@ define <8 x i16> @stack_fold_psravw(<8 x i16> %a0, <8 x i16> %a1) {
 declare <8 x i16> @llvm.x86.avx512.psrav.w.128(<8 x i16>, <8 x i16>) nounwind readnone
 
 define <16 x i16> @stack_fold_psravw_ymm(<16 x i16> %a0, <16 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_psravw_ymm
-  ;CHECK:       vpsravw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_psravw_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsravw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x i16> @llvm.x86.avx512.psrav.w.256(<16 x i16> %a0, <16 x i16> %a1)
   ret <16 x i16> %2
@@ -2154,8 +3730,14 @@ define <16 x i16> @stack_fold_psravw_ymm(<16 x i16> %a0, <16 x i16> %a1) {
 declare <16 x i16> @llvm.x86.avx512.psrav.w.256(<16 x i16>, <16 x i16>) nounwind readnone
 
 define <8 x i16> @stack_fold_psraw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_psraw
-  ;CHECK:       vpsraw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psraw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsraw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x i16> @llvm.x86.sse2.psra.w(<8 x i16> %a0, <8 x i16> %a1)
   ret <8 x i16> %2
@@ -2163,8 +3745,14 @@ define <8 x i16> @stack_fold_psraw(<8 x i16> %a0, <8 x i16> %a1) {
 declare <8 x i16> @llvm.x86.sse2.psra.w(<8 x i16>, <8 x i16>) nounwind readnone
 
 define <16 x i16> @stack_fold_psraw_ymm(<16 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_psraw_ymm
-  ;CHECK:       vpsraw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psraw_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsraw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x i16> @llvm.x86.avx2.psra.w(<16 x i16> %a0, <8 x i16> %a1)
   ret <16 x i16> %2
@@ -2172,8 +3760,14 @@ define <16 x i16> @stack_fold_psraw_ymm(<16 x i16> %a0, <8 x i16> %a1) {
 declare <16 x i16> @llvm.x86.avx2.psra.w(<16 x i16>, <8 x i16>) nounwind readnone
 
 define <4 x i32> @stack_fold_psrld(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_psrld
-  ;CHECK:       vpsrld {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psrld:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsrld {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <4 x i32> @llvm.x86.sse2.psrl.d(<4 x i32> %a0, <4 x i32> %a1)
   ret <4 x i32> %2
@@ -2181,8 +3775,16 @@ define <4 x i32> @stack_fold_psrld(<4 x i32> %a0, <4 x i32> %a1) {
 declare <4 x i32> @llvm.x86.sse2.psrl.d(<4 x i32>, <4 x i32>) nounwind readnone
 
 define <8 x i32> @stack_fold_psrld_ymm(<8 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_psrld_ymm
-  ;CHECK:       vpsrld {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psrld_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; CHECK-NEXT:    vpsrld {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x i32> @llvm.x86.avx2.psrl.d(<8 x i32> %a0, <4 x i32> %a1)
   ret <8 x i32> %2
@@ -2190,24 +3792,44 @@ define <8 x i32> @stack_fold_psrld_ymm(<8 x i32> %a0, <4 x i32> %a1) {
 declare <8 x i32> @llvm.x86.avx2.psrl.d(<8 x i32>, <4 x i32>) nounwind readnone
 
 define <16 x i8> @stack_fold_psrldq(<16 x i8> %a) {
-  ;CHECK-LABEL: stack_fold_psrldq
-  ;CHECK:       vpsrldq $12, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psrldq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsrldq $12, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = mem[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <16 x i8> zeroinitializer, <16 x i8> %a, <16 x i32> <i32 28, i32 29, i32 30, i32 31, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 09, i32 0, i32 0, i32 0, i32 0, i32 0>
   ret <16 x i8> %2
 }
 
 define <32 x i8> @stack_fold_psrldq_ymm(<32 x i8> %a) {
-  ;CHECK-LABEL: stack_fold_psrldq_ymm
-  ;CHECK:       vpsrldq $15, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_psrldq_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsrldq $15, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    # ymm0 = mem[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,mem[31],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <32 x i8> zeroinitializer, <32 x i8> %a, <32 x i32> <i32 47, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 63, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
   ret <32 x i8> %2
 }
 
 define <2 x i64> @stack_fold_psrlq(<2 x i64> %a0, <2 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_psrlq
-  ;CHECK:       vpsrlq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psrlq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsrlq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <2 x i64> @llvm.x86.sse2.psrl.q(<2 x i64> %a0, <2 x i64> %a1)
   ret <2 x i64> %2
@@ -2215,8 +3837,14 @@ define <2 x i64> @stack_fold_psrlq(<2 x i64> %a0, <2 x i64> %a1) {
 declare <2 x i64> @llvm.x86.sse2.psrl.q(<2 x i64>, <2 x i64>) nounwind readnone
 
 define <4 x i64> @stack_fold_psrlq_ymm(<4 x i64> %a0, <2 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_psrlq_ymm
-  ;CHECK:       vpsrlq {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psrlq_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsrlq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <4 x i64> @llvm.x86.avx2.psrl.q(<4 x i64> %a0, <2 x i64> %a1)
   ret <4 x i64> %2
@@ -2224,8 +3852,14 @@ define <4 x i64> @stack_fold_psrlq_ymm(<4 x i64> %a0, <2 x i64> %a1) {
 declare <4 x i64> @llvm.x86.avx2.psrl.q(<4 x i64>, <2 x i64>) nounwind readnone
 
 define <4 x i32> @stack_fold_psrlvd(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_psrlvd
-  ;CHECK:       vpsrlvd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psrlvd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsrlvd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <4 x i32> @llvm.x86.avx2.psrlv.d(<4 x i32> %a0, <4 x i32> %a1)
   ret <4 x i32> %2
@@ -2233,8 +3867,14 @@ define <4 x i32> @stack_fold_psrlvd(<4 x i32> %a0, <4 x i32> %a1) {
 declare <4 x i32> @llvm.x86.avx2.psrlv.d(<4 x i32>, <4 x i32>) nounwind readnone
 
 define <8 x i32> @stack_fold_psrlvd_ymm(<8 x i32> %a0, <8 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_psrlvd_ymm
-  ;CHECK:       vpsrlvd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_psrlvd_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsrlvd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x i32> @llvm.x86.avx2.psrlv.d.256(<8 x i32> %a0, <8 x i32> %a1)
   ret <8 x i32> %2
@@ -2242,8 +3882,14 @@ define <8 x i32> @stack_fold_psrlvd_ymm(<8 x i32> %a0, <8 x i32> %a1) {
 declare <8 x i32> @llvm.x86.avx2.psrlv.d.256(<8 x i32>, <8 x i32>) nounwind readnone
 
 define <2 x i64> @stack_fold_psrlvq(<2 x i64> %a0, <2 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_psrlvq
-  ;CHECK:       vpsrlvq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psrlvq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsrlvq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <2 x i64> @llvm.x86.avx2.psrlv.q(<2 x i64> %a0, <2 x i64> %a1)
   ret <2 x i64> %2
@@ -2251,8 +3897,14 @@ define <2 x i64> @stack_fold_psrlvq(<2 x i64> %a0, <2 x i64> %a1) {
 declare <2 x i64> @llvm.x86.avx2.psrlv.q(<2 x i64>, <2 x i64>) nounwind readnone
 
 define <4 x i64> @stack_fold_psrlvq_ymm(<4 x i64> %a0, <4 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_psrlvq_ymm
-  ;CHECK:       vpsrlvq {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_psrlvq_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsrlvq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <4 x i64> @llvm.x86.avx2.psrlv.q.256(<4 x i64> %a0, <4 x i64> %a1)
   ret <4 x i64> %2
@@ -2260,8 +3912,14 @@ define <4 x i64> @stack_fold_psrlvq_ymm(<4 x i64> %a0, <4 x i64> %a1) {
 declare <4 x i64> @llvm.x86.avx2.psrlv.q.256(<4 x i64>, <4 x i64>) nounwind readnone
 
 define <8 x i16> @stack_fold_psrlvw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_psrlvw
-  ;CHECK:       vpsrlvw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psrlvw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsrlvw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x i16> @llvm.x86.avx512.psrlv.w.128(<8 x i16> %a0, <8 x i16> %a1)
   ret <8 x i16> %2
@@ -2269,8 +3927,14 @@ define <8 x i16> @stack_fold_psrlvw(<8 x i16> %a0, <8 x i16> %a1) {
 declare <8 x i16> @llvm.x86.avx512.psrlv.w.128(<8 x i16>, <8 x i16>) nounwind readnone
 
 define <16 x i16> @stack_fold_psrlvw_ymm(<16 x i16> %a0, <16 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_psrlvw_ymm
-  ;CHECK:       vpsrlvw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_psrlvw_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsrlvw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x i16> @llvm.x86.avx512.psrlv.w.256(<16 x i16> %a0, <16 x i16> %a1)
   ret <16 x i16> %2
@@ -2278,8 +3942,14 @@ define <16 x i16> @stack_fold_psrlvw_ymm(<16 x i16> %a0, <16 x i16> %a1) {
 declare <16 x i16> @llvm.x86.avx512.psrlv.w.256(<16 x i16>, <16 x i16>) nounwind readnone
 
 define <8 x i16> @stack_fold_psrlw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_psrlw
-  ;CHECK:       vpsrlw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psrlw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsrlw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x i16> @llvm.x86.sse2.psrl.w(<8 x i16> %a0, <8 x i16> %a1)
   ret <8 x i16> %2
@@ -2287,8 +3957,14 @@ define <8 x i16> @stack_fold_psrlw(<8 x i16> %a0, <8 x i16> %a1) {
 declare <8 x i16> @llvm.x86.sse2.psrl.w(<8 x i16>, <8 x i16>) nounwind readnone
 
 define <16 x i16> @stack_fold_psrlw_ymm(<16 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_psrlw_ymm
-  ;CHECK:       vpsrlw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psrlw_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsrlw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x i16> @llvm.x86.avx2.psrl.w(<16 x i16> %a0, <8 x i16> %a1)
   ret <16 x i16> %2
@@ -2296,56 +3972,98 @@ define <16 x i16> @stack_fold_psrlw_ymm(<16 x i16> %a0, <8 x i16> %a1) {
 declare <16 x i16> @llvm.x86.avx2.psrl.w(<16 x i16>, <8 x i16>) nounwind readnone
 
 define <16 x i8> @stack_fold_psubb(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_psubb
-  ;CHECK:       vpsubb {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psubb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsubb {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = sub <16 x i8> %a0, %a1
   ret <16 x i8> %2
 }
 
 define <32 x i8> @stack_fold_psubb_ymm(<32 x i8> %a0, <32 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_psubb_ymm
-  ;CHECK:       vpsubb {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_psubb_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsubb {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = sub <32 x i8> %a0, %a1
   ret <32 x i8> %2
 }
 
 define <4 x i32> @stack_fold_psubd(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_psubd
-  ;CHECK:       vpsubd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psubd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsubd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = sub <4 x i32> %a0, %a1
   ret <4 x i32> %2
 }
 
 define <8 x i32> @stack_fold_psubd_ymm(<8 x i32> %a0, <8 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_psubd_ymm
-  ;CHECK:       vpsubd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_psubd_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsubd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = sub <8 x i32> %a0, %a1
   ret <8 x i32> %2
 }
 
 define <2 x i64> @stack_fold_psubq(<2 x i64> %a0, <2 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_psubq
-  ;CHECK:       vpsubq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psubq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsubq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = sub <2 x i64> %a0, %a1
   ret <2 x i64> %2
 }
 
 define <4 x i64> @stack_fold_psubq_ymm(<4 x i64> %a0, <4 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_psubq_ymm
-  ;CHECK:       vpsubq {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_psubq_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsubq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = sub <4 x i64> %a0, %a1
   ret <4 x i64> %2
 }
 
 define <16 x i8> @stack_fold_psubsb(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_psubsb
-  ;CHECK:       vpsubsb {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psubsb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsubsb {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> %a0, <16 x i8> %a1)
   ret <16 x i8> %2
@@ -2353,8 +4071,14 @@ define <16 x i8> @stack_fold_psubsb(<16 x i8> %a0, <16 x i8> %a1) {
 declare <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8>, <16 x i8>) nounwind readnone
 
 define <32 x i8> @stack_fold_psubsb_ymm(<32 x i8> %a0, <32 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_psubsb_ymm
-  ;CHECK:       vpsubsb {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_psubsb_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsubsb {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <32 x i8> @llvm.ssub.sat.v32i8(<32 x i8> %a0, <32 x i8> %a1)
   ret <32 x i8> %2
@@ -2362,8 +4086,14 @@ define <32 x i8> @stack_fold_psubsb_ymm(<32 x i8> %a0, <32 x i8> %a1) {
 declare <32 x i8> @llvm.ssub.sat.v32i8(<32 x i8>, <32 x i8>) nounwind readnone
 
 define <8 x i16> @stack_fold_psubsw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_psubsw
-  ;CHECK:       vpsubsw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psubsw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsubsw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> %a0, <8 x i16> %a1)
   ret <8 x i16> %2
@@ -2371,8 +4101,14 @@ define <8 x i16> @stack_fold_psubsw(<8 x i16> %a0, <8 x i16> %a1) {
 declare <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16>, <8 x i16>) nounwind readnone
 
 define <16 x i16> @stack_fold_psubsw_ymm(<16 x i16> %a0, <16 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_psubsw_ymm
-  ;CHECK:       vpsubsw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_psubsw_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsubsw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16> %a0, <16 x i16> %a1)
   ret <16 x i16> %2
@@ -2380,8 +4116,14 @@ define <16 x i16> @stack_fold_psubsw_ymm(<16 x i16> %a0, <16 x i16> %a1) {
 declare <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16>, <16 x i16>) nounwind readnone
 
 define <16 x i8> @stack_fold_psubusb(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_psubusb
-  ;CHECK:       vpsubusb {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psubusb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsubusb {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> %a0, <16 x i8> %a1)
   ret <16 x i8> %2
@@ -2389,8 +4131,14 @@ define <16 x i8> @stack_fold_psubusb(<16 x i8> %a0, <16 x i8> %a1) {
 declare <16 x i8> @llvm.usub.sat.v16i8(<16 x i8>, <16 x i8>) nounwind readnone
 
 define <32 x i8> @stack_fold_psubusb_ymm(<32 x i8> %a0, <32 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_psubusb_ymm
-  ;CHECK:       vpsubusb {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_psubusb_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsubusb {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <32 x i8> @llvm.usub.sat.v32i8(<32 x i8> %a0, <32 x i8> %a1)
   ret <32 x i8> %2
@@ -2398,8 +4146,14 @@ define <32 x i8> @stack_fold_psubusb_ymm(<32 x i8> %a0, <32 x i8> %a1) {
 declare <32 x i8> @llvm.usub.sat.v32i8(<32 x i8>, <32 x i8>) nounwind readnone
 
 define <8 x i16> @stack_fold_psubusw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_psubusw
-  ;CHECK:       vpsubusw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psubusw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsubusw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> %a0, <8 x i16> %a1)
   ret <8 x i16> %2
@@ -2407,8 +4161,14 @@ define <8 x i16> @stack_fold_psubusw(<8 x i16> %a0, <8 x i16> %a1) {
 declare <8 x i16> @llvm.usub.sat.v8i16(<8 x i16>, <8 x i16>) nounwind readnone
 
 define <16 x i16> @stack_fold_psubusw_ymm(<16 x i16> %a0, <16 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_psubusw_ymm
-  ;CHECK:       vpsubusw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_psubusw_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsubusw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x i16> @llvm.usub.sat.v16i16(<16 x i16> %a0, <16 x i16> %a1)
   ret <16 x i16> %2
@@ -2416,32 +4176,61 @@ define <16 x i16> @stack_fold_psubusw_ymm(<16 x i16> %a0, <16 x i16> %a1) {
 declare <16 x i16> @llvm.usub.sat.v16i16(<16 x i16>, <16 x i16>) nounwind readnone
 
 define <8 x i16> @stack_fold_psubw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_psubw
-  ;CHECK:       vpsubw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psubw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsubw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = sub <8 x i16> %a0, %a1
   ret <8 x i16> %2
 }
 
 define <16 x i16> @stack_fold_psubw_ymm(<16 x i16> %a0, <16 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_psubw_ymm
-  ;CHECK:       vpsubw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_psubw_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsubw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = sub <16 x i16> %a0, %a1
   ret <16 x i16> %2
 }
 
 define <16 x i8> @stack_fold_punpckhbw(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_punpckhbw
-  ;CHECK:       vpunpckhbw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_punpckhbw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpunpckhbw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = xmm0[8],mem[8],xmm0[9],mem[9],xmm0[10],mem[10],xmm0[11],mem[11],xmm0[12],mem[12],xmm0[13],mem[13],xmm0[14],mem[14],xmm0[15],mem[15]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <16 x i8> %a0, <16 x i8> %a1, <16 x i32> <i32 8, i32 24, i32 9, i32 25, i32 10, i32 26, i32 11, i32 27, i32 12, i32 28, i32 13, i32 29, i32 14, i32 30, i32 15, i32 31>
   ret <16 x i8> %2
 }
 
 define <16 x i8> @stack_fold_punpckhbw_mask(<16 x i8>* %passthru, <16 x i8> %a0, <16 x i8> %a1, i16 %mask) {
-  ;CHECK-LABEL: stack_fold_punpckhbw_mask
-  ;CHECK:       vpunpckhbw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_punpckhbw_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vmovdqa (%rdi), %xmm2
+; CHECK-NEXT:    vpunpckhbw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm2 {%k1} # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm2 {%k1} = xmm0[8],mem[8],xmm0[9],mem[9],xmm0[10],mem[10],xmm0[11],mem[11],xmm0[12],mem[12],xmm0[13],mem[13],xmm0[14],mem[14],xmm0[15],mem[15]
+; CHECK-NEXT:    vmovdqa %xmm2, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <16 x i8> %a0, <16 x i8> %a1, <16 x i32> <i32 8, i32 24, i32 9, i32 25, i32 10, i32 26, i32 11, i32 27, i32 12, i32 28, i32 13, i32 29, i32 14, i32 30, i32 15, i32 31>
   %3 = bitcast i16 %mask to <16 x i1>
@@ -2452,8 +4241,16 @@ define <16 x i8> @stack_fold_punpckhbw_mask(<16 x i8>* %passthru, <16 x i8> %a0,
 }
 
 define <16 x i8> @stack_fold_punpckhbw_maskz(<16 x i8> %passthru, <16 x i8> %a0, <16 x i8> %a1, i16 %mask) {
-  ;CHECK-LABEL: stack_fold_punpckhbw_maskz
-  ;CHECK:       vpunpckhbw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_punpckhbw_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpunpckhbw {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} {z} # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 {%k1} {z} = xmm1[8],mem[8],xmm1[9],mem[9],xmm1[10],mem[10],xmm1[11],mem[11],xmm1[12],mem[12],xmm1[13],mem[13],xmm1[14],mem[14],xmm1[15],mem[15]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <16 x i8> %a0, <16 x i8> %a1, <16 x i32> <i32 8, i32 24, i32 9, i32 25, i32 10, i32 26, i32 11, i32 27, i32 12, i32 28, i32 13, i32 29, i32 14, i32 30, i32 15, i32 31>
   %3 = bitcast i16 %mask to <16 x i1>
@@ -2462,16 +4259,33 @@ define <16 x i8> @stack_fold_punpckhbw_maskz(<16 x i8> %passthru, <16 x i8> %a0,
 }
 
 define <32 x i8> @stack_fold_punpckhbw_ymm(<32 x i8> %a0, <32 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_punpckhbw_ymm
-  ;CHECK:       vpunpckhbw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_punpckhbw_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpunpckhbw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    # ymm0 = ymm0[8],mem[8],ymm0[9],mem[9],ymm0[10],mem[10],ymm0[11],mem[11],ymm0[12],mem[12],ymm0[13],mem[13],ymm0[14],mem[14],ymm0[15],mem[15],ymm0[24],mem[24],ymm0[25],mem[25],ymm0[26],mem[26],ymm0[27],mem[27],ymm0[28],mem[28],ymm0[29],mem[29],ymm0[30],mem[30],ymm0[31],mem[31]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <32 x i8> %a0, <32 x i8> %a1, <32 x i32> <i32 8, i32 40, i32 9, i32 41, i32 10, i32 42, i32 11, i32 43, i32 12, i32 44, i32 13, i32 45, i32 14, i32 46, i32 15, i32 47, i32 24, i32 56, i32 25, i32 57, i32 26, i32 58, i32 27, i32 59, i32 28, i32 60, i32 29, i32 61, i32 30, i32 62, i32 31, i32 63>
   ret <32 x i8> %2
 }
 
 define <32 x i8> @stack_fold_punpckhbw_mask_ymm(<32 x i8>* %passthru, <32 x i8> %a0, <32 x i8> %a1, i32 %mask) {
-  ;CHECK-LABEL: stack_fold_punpckhbw_mask_ymm
-  ;CHECK:       vpunpckhbw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_punpckhbw_mask_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vmovdqa (%rdi), %ymm2
+; CHECK-NEXT:    vpunpckhbw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm2 {%k1} # 32-byte Folded Reload
+; CHECK-NEXT:    # ymm2 {%k1} = ymm0[8],mem[8],ymm0[9],mem[9],ymm0[10],mem[10],ymm0[11],mem[11],ymm0[12],mem[12],ymm0[13],mem[13],ymm0[14],mem[14],ymm0[15],mem[15],ymm0[24],mem[24],ymm0[25],mem[25],ymm0[26],mem[26],ymm0[27],mem[27],ymm0[28],mem[28],ymm0[29],mem[29],ymm0[30],mem[30],ymm0[31],mem[31]
+; CHECK-NEXT:    vmovdqa %ymm2, %ymm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <32 x i8> %a0, <32 x i8> %a1, <32 x i32> <i32 8, i32 40, i32 9, i32 41, i32 10, i32 42, i32 11, i32 43, i32 12, i32 44, i32 13, i32 45, i32 14, i32 46, i32 15, i32 47, i32 24, i32 56, i32 25, i32 57, i32 26, i32 58, i32 27, i32 59, i32 28, i32 60, i32 29, i32 61, i32 30, i32 62, i32 31, i32 63>
   %3 = bitcast i32 %mask to <32 x i1>
@@ -2482,8 +4296,16 @@ define <32 x i8> @stack_fold_punpckhbw_mask_ymm(<32 x i8>* %passthru, <32 x i8>
 }
 
 define <32 x i8> @stack_fold_punpckhbw_maskz_ymm(<32 x i8> %a0, <32 x i8> %a1, i32 %mask) {
-  ;CHECK-LABEL: stack_fold_punpckhbw_maskz_ymm
-  ;CHECK:       vpunpckhbw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_punpckhbw_maskz_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpunpckhbw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 {%k1} {z} # 32-byte Folded Reload
+; CHECK-NEXT:    # ymm0 {%k1} {z} = ymm0[8],mem[8],ymm0[9],mem[9],ymm0[10],mem[10],ymm0[11],mem[11],ymm0[12],mem[12],ymm0[13],mem[13],ymm0[14],mem[14],ymm0[15],mem[15],ymm0[24],mem[24],ymm0[25],mem[25],ymm0[26],mem[26],ymm0[27],mem[27],ymm0[28],mem[28],ymm0[29],mem[29],ymm0[30],mem[30],ymm0[31],mem[31]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <32 x i8> %a0, <32 x i8> %a1, <32 x i32> <i32 8, i32 40, i32 9, i32 41, i32 10, i32 42, i32 11, i32 43, i32 12, i32 44, i32 13, i32 45, i32 14, i32 46, i32 15, i32 47, i32 24, i32 56, i32 25, i32 57, i32 26, i32 58, i32 27, i32 59, i32 28, i32 60, i32 29, i32 61, i32 30, i32 62, i32 31, i32 63>
   %3 = bitcast i32 %mask to <32 x i1>
@@ -2492,8 +4314,18 @@ define <32 x i8> @stack_fold_punpckhbw_maskz_ymm(<32 x i8> %a0, <32 x i8> %a1, i
 }
 
 define <4 x i64> @stack_fold_shufi64x2_maskz(<4 x i64> %a, <4 x i64> %b, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_shufi64x2_maskz
-  ;CHECK:   vshufi64x2 $1, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{{%k[1-7]}}} {z} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_shufi64x2_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; CHECK-NEXT:    vshufi64x2 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 {%k1} {z} # 32-byte Folded Reload
+; CHECK-NEXT:    # ymm0 {%k1} {z} = ymm0[2,3],mem[0,1]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <4 x i64> %a, <4 x i64> %b, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
   %3 = bitcast i8 %mask to <8 x i1>
@@ -2503,8 +4335,18 @@ define <4 x i64> @stack_fold_shufi64x2_maskz(<4 x i64> %a, <4 x i64> %b, i8 %mas
 }
 
 define <8 x i32> @stack_fold_shufi32x4_maskz(<8 x i32> %a, <8 x i32> %b, i8 %mask) {
-  ;CHECK-LABEL: stack_fold_shufi32x4_maskz
-  ;CHECK:   vshufi32x4 $1, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{{%k[1-7]}}} {z} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_shufi32x4_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; CHECK-NEXT:    vshufi32x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 {%k1} {z} # 32-byte Folded Reload
+; CHECK-NEXT:    # ymm0 {%k1} {z} = ymm0[4,5,6,7],mem[0,1,2,3]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = shufflevector <8 x i32> %a, <8 x i32> %b, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
   %3 = bitcast i8 %mask to <8 x i1>
diff --git a/llvm/test/CodeGen/X86/stack-folding-int-avx512vnni.ll b/llvm/test/CodeGen/X86/stack-folding-int-avx512vnni.ll
new file mode 100644
index 000000000..34dba4e2a
--- /dev/null
+++ b/llvm/test/CodeGen/X86/stack-folding-int-avx512vnni.ll
@@ -0,0 +1,216 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -O3 -disable-peephole -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw,+avx512dq,+avx512vbmi,+avx512cd,+avx512vpopcntdq,+avx512vnni < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-unknown"
+
+define <16 x i32> @stack_fold_vpdpwssd(<16 x i32> %a0, <16 x i32> %a1, <16 x i32> %a2) {
+; CHECK-LABEL: stack_fold_vpdpwssd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpdpwssd {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
+  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
+  %2 = call <16 x i32> @llvm.x86.avx512.vpdpwssd.512(<16 x i32> %a0, <16 x i32> %a1, <16 x i32> %a2)
+  ret <16 x i32> %2
+}
+
+define <16 x i32> @stack_fold_vpdpwssd_commuted(<16 x i32> %a0, <16 x i32> %a1, <16 x i32> %a2) {
+; CHECK-LABEL: stack_fold_vpdpwssd_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpdpwssd {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
+  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
+  %2 = call <16 x i32> @llvm.x86.avx512.vpdpwssd.512(<16 x i32> %a0, <16 x i32> %a2, <16 x i32> %a1)
+  ret <16 x i32> %2
+}
+
+define <16 x i32> @stack_fold_vpdpwssd_mask(<16 x i32>* %a0, <16 x i32> %a1, <16 x i32> %a2, i16 %mask) {
+; CHECK-LABEL: stack_fold_vpdpwssd_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm2
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vpdpwssd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm0
+; CHECK-NEXT:    retq
+  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
+  %2 = load <16 x i32>, <16 x i32>* %a0
+  %3 = call <16 x i32> @llvm.x86.avx512.vpdpwssd.512(<16 x i32> %2, <16 x i32> %a1, <16 x i32> %a2)
+  %4 = bitcast i16 %mask to <16 x i1>
+  %5 = select <16 x i1> %4, <16 x i32> %3, <16 x i32> %2
+  ret <16 x i32> %5
+}
+
+define <16 x i32> @stack_fold_vpdpwssd_mask_commuted(<16 x i32>* %a0, <16 x i32> %a1, <16 x i32> %a2, i16 %mask) {
+; CHECK-LABEL: stack_fold_vpdpwssd_mask_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm2
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vpdpwssd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm0
+; CHECK-NEXT:    retq
+  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
+  %2 = load <16 x i32>, <16 x i32>* %a0
+  %3 = call <16 x i32> @llvm.x86.avx512.vpdpwssd.512(<16 x i32> %2, <16 x i32> %a2, <16 x i32> %a1)
+  %4 = bitcast i16 %mask to <16 x i1>
+  %5 = select <16 x i1> %4, <16 x i32> %3, <16 x i32> %2
+  ret <16 x i32> %5
+}
+
+define <16 x i32> @stack_fold_vpdpwssd_maskz(<16 x i32> %a0, <16 x i32> %a1, <16 x i32> %a2, i16* %mask) {
+; CHECK-LABEL: stack_fold_vpdpwssd_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovw (%rdi), %k1
+; CHECK-NEXT:    vpdpwssd {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
+  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
+  %2 = call <16 x i32> @llvm.x86.avx512.vpdpwssd.512(<16 x i32> %a0, <16 x i32> %a1, <16 x i32> %a2)
+  %3 = load i16, i16* %mask
+  %4 = bitcast i16 %3 to <16 x i1>
+  %5 = select <16 x i1> %4, <16 x i32> %2, <16 x i32> zeroinitializer
+  ret <16 x i32> %5
+}
+
+define <16 x i32> @stack_fold_vpdpwssd_maskz_commuted(<16 x i32> %a0, <16 x i32> %a1, <16 x i32> %a2, i16* %mask) {
+; CHECK-LABEL: stack_fold_vpdpwssd_maskz_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovw (%rdi), %k1
+; CHECK-NEXT:    vpdpwssd {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
+  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
+  %2 = call <16 x i32> @llvm.x86.avx512.vpdpwssd.512(<16 x i32> %a0, <16 x i32> %a2, <16 x i32> %a1)
+  %3 = load i16, i16* %mask
+  %4 = bitcast i16 %3 to <16 x i1>
+  %5 = select <16 x i1> %4, <16 x i32> %2, <16 x i32> zeroinitializer
+  ret <16 x i32> %5
+}
+
+define <16 x i32> @stack_fold_vpdpwssds(<16 x i32> %a0, <16 x i32> %a1, <16 x i32> %a2) {
+; CHECK-LABEL: stack_fold_vpdpwssds:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpdpwssds {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
+  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
+  %2 = call <16 x i32> @llvm.x86.avx512.vpdpwssds.512(<16 x i32> %a0, <16 x i32> %a1, <16 x i32> %a2)
+  ret <16 x i32> %2
+}
+
+define <16 x i32> @stack_fold_vpdpwssds_commuted(<16 x i32> %a0, <16 x i32> %a1, <16 x i32> %a2) {
+; CHECK-LABEL: stack_fold_vpdpwssds_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpdpwssds {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
+  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
+  %2 = call <16 x i32> @llvm.x86.avx512.vpdpwssds.512(<16 x i32> %a0, <16 x i32> %a2, <16 x i32> %a1)
+  ret <16 x i32> %2
+}
+
+define <16 x i32> @stack_fold_vpdpwssds_mask(<16 x i32>* %a0, <16 x i32> %a1, <16 x i32> %a2, i16 %mask) {
+; CHECK-LABEL: stack_fold_vpdpwssds_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm2
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vpdpwssds {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm0
+; CHECK-NEXT:    retq
+  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
+  %2 = load <16 x i32>, <16 x i32>* %a0
+  %3 = call <16 x i32> @llvm.x86.avx512.vpdpwssds.512(<16 x i32> %2, <16 x i32> %a1, <16 x i32> %a2)
+  %4 = bitcast i16 %mask to <16 x i1>
+  %5 = select <16 x i1> %4, <16 x i32> %3, <16 x i32> %2
+  ret <16 x i32> %5
+}
+
+define <16 x i32> @stack_fold_vpdpwssds_mask_commuted(<16 x i32>* %a0, <16 x i32> %a1, <16 x i32> %a2, i16 %mask) {
+; CHECK-LABEL: stack_fold_vpdpwssds_mask_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm2
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vpdpwssds {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload
+; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm0
+; CHECK-NEXT:    retq
+  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
+  %2 = load <16 x i32>, <16 x i32>* %a0
+  %3 = call <16 x i32> @llvm.x86.avx512.vpdpwssds.512(<16 x i32> %2, <16 x i32> %a2, <16 x i32> %a1)
+  %4 = bitcast i16 %mask to <16 x i1>
+  %5 = select <16 x i1> %4, <16 x i32> %3, <16 x i32> %2
+  ret <16 x i32> %5
+}
+
+define <16 x i32> @stack_fold_vpdpwssds_maskz(<16 x i32> %a0, <16 x i32> %a1, <16 x i32> %a2, i16* %mask) {
+; CHECK-LABEL: stack_fold_vpdpwssds_maskz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovw (%rdi), %k1
+; CHECK-NEXT:    vpdpwssds {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
+  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
+  %2 = call <16 x i32> @llvm.x86.avx512.vpdpwssds.512(<16 x i32> %a0, <16 x i32> %a1, <16 x i32> %a2)
+  %3 = load i16, i16* %mask
+  %4 = bitcast i16 %3 to <16 x i1>
+  %5 = select <16 x i1> %4, <16 x i32> %2, <16 x i32> zeroinitializer
+  ret <16 x i32> %5
+}
+
+define <16 x i32> @stack_fold_vpdpwssds_maskz_commuted(<16 x i32> %a0, <16 x i32> %a1, <16 x i32> %a2, i16* %mask) {
+; CHECK-LABEL: stack_fold_vpdpwssds_maskz_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    kmovw (%rdi), %k1
+; CHECK-NEXT:    vpdpwssds {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} {z} # 64-byte Folded Reload
+; CHECK-NEXT:    retq
+  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
+  %2 = call <16 x i32> @llvm.x86.avx512.vpdpwssds.512(<16 x i32> %a0, <16 x i32> %a2, <16 x i32> %a1)
+  %3 = load i16, i16* %mask
+  %4 = bitcast i16 %3 to <16 x i1>
+  %5 = select <16 x i1> %4, <16 x i32> %2, <16 x i32> zeroinitializer
+  ret <16 x i32> %5
+}
+
+declare <16 x i32> @llvm.x86.avx512.vpdpwssd.512(<16 x i32>, <16 x i32>, <16 x i32>)
+declare <16 x i32> @llvm.x86.avx512.vpdpwssds.512(<16 x i32>, <16 x i32>, <16 x i32>)
diff --git a/llvm/test/CodeGen/X86/stack-folding-int-sse42.ll b/llvm/test/CodeGen/X86/stack-folding-int-sse42.ll
index c498e2bfb..3efaeddfb 100644
--- a/llvm/test/CodeGen/X86/stack-folding-int-sse42.ll
+++ b/llvm/test/CodeGen/X86/stack-folding-int-sse42.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -O3 -disable-peephole -mtriple=x86_64-unknown-unknown -mattr=+sse4.2,+aes,+pclmul < %s | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -9,8 +10,14 @@ target triple = "x86_64-unknown-unknown"
 ; relevant registers and check that the reload is correctly folded into the instruction.
 
 define <2 x i64> @stack_fold_aesdec(<2 x i64> %a0, <2 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_aesdec
-  ;CHECK:       aesdec {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_aesdec:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    aesdec {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x i64> @llvm.x86.aesni.aesdec(<2 x i64> %a0, <2 x i64> %a1)
   ret <2 x i64> %2
@@ -18,8 +25,14 @@ define <2 x i64> @stack_fold_aesdec(<2 x i64> %a0, <2 x i64> %a1) {
 declare <2 x i64> @llvm.x86.aesni.aesdec(<2 x i64>, <2 x i64>) nounwind readnone
 
 define <2 x i64> @stack_fold_aesdeclast(<2 x i64> %a0, <2 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_aesdeclast
-  ;CHECK:       aesdeclast {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_aesdeclast:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    aesdeclast {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x i64> @llvm.x86.aesni.aesdeclast(<2 x i64> %a0, <2 x i64> %a1)
   ret <2 x i64> %2
@@ -27,8 +40,14 @@ define <2 x i64> @stack_fold_aesdeclast(<2 x i64> %a0, <2 x i64> %a1) {
 declare <2 x i64> @llvm.x86.aesni.aesdeclast(<2 x i64>, <2 x i64>) nounwind readnone
 
 define <2 x i64> @stack_fold_aesenc(<2 x i64> %a0, <2 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_aesenc
-  ;CHECK:       aesenc {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_aesenc:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    aesenc {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x i64> @llvm.x86.aesni.aesenc(<2 x i64> %a0, <2 x i64> %a1)
   ret <2 x i64> %2
@@ -36,8 +55,14 @@ define <2 x i64> @stack_fold_aesenc(<2 x i64> %a0, <2 x i64> %a1) {
 declare <2 x i64> @llvm.x86.aesni.aesenc(<2 x i64>, <2 x i64>) nounwind readnone
 
 define <2 x i64> @stack_fold_aesenclast(<2 x i64> %a0, <2 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_aesenclast
-  ;CHECK:       aesenclast {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_aesenclast:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    aesenclast {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x i64> @llvm.x86.aesni.aesenclast(<2 x i64> %a0, <2 x i64> %a1)
   ret <2 x i64> %2
@@ -45,8 +70,14 @@ define <2 x i64> @stack_fold_aesenclast(<2 x i64> %a0, <2 x i64> %a1) {
 declare <2 x i64> @llvm.x86.aesni.aesenclast(<2 x i64>, <2 x i64>) nounwind readnone
 
 define <2 x i64> @stack_fold_aesimc(<2 x i64> %a0) {
-  ;CHECK-LABEL: stack_fold_aesimc
-  ;CHECK:       aesimc {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_aesimc:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    aesimc {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x i64> @llvm.x86.aesni.aesimc(<2 x i64> %a0)
   ret <2 x i64> %2
@@ -54,8 +85,14 @@ define <2 x i64> @stack_fold_aesimc(<2 x i64> %a0) {
 declare <2 x i64> @llvm.x86.aesni.aesimc(<2 x i64>) nounwind readnone
 
 define <2 x i64> @stack_fold_aeskeygenassist(<2 x i64> %a0) {
-  ;CHECK-LABEL: stack_fold_aeskeygenassist
-  ;CHECK:       aeskeygenassist $7, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_aeskeygenassist:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    aeskeygenassist $7, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x i64> @llvm.x86.aesni.aeskeygenassist(<2 x i64> %a0, i8 7)
   ret <2 x i64> %2
@@ -63,8 +100,45 @@ define <2 x i64> @stack_fold_aeskeygenassist(<2 x i64> %a0) {
 declare <2 x i64> @llvm.x86.aesni.aeskeygenassist(<2 x i64>, i8) nounwind readnone
 
 define i32 @stack_fold_crc32_32_8(i32 %a0, i8 %a1) {
-  ;CHECK-LABEL: stack_fold_crc32_32_8
-  ;CHECK:       crc32b {{-?[0-9]*}}(%rsp), %eax {{.*#+}} 1-byte Folded Reload
+; CHECK-LABEL: stack_fold_crc32_32_8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movl %esi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    crc32b {{[-0-9]+}}(%r{{[sb]}}p), %eax # 1-byte Folded Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = call i32 @llvm.x86.sse42.crc32.32.8(i32 %a0, i8 %a1)
   ret i32 %2
@@ -72,8 +146,45 @@ define i32 @stack_fold_crc32_32_8(i32 %a0, i8 %a1) {
 declare i32 @llvm.x86.sse42.crc32.32.8(i32, i8) nounwind
 
 define i32 @stack_fold_crc32_32_16(i32 %a0, i16 %a1) {
-  ;CHECK-LABEL: stack_fold_crc32_32_16
-  ;CHECK:       crc32w {{-?[0-9]*}}(%rsp), %eax {{.*#+}} 2-byte Folded Reload
+; CHECK-LABEL: stack_fold_crc32_32_16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movl %esi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    crc32w {{[-0-9]+}}(%r{{[sb]}}p), %eax # 2-byte Folded Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = call i32 @llvm.x86.sse42.crc32.32.16(i32 %a0, i16 %a1)
   ret i32 %2
@@ -81,8 +192,45 @@ define i32 @stack_fold_crc32_32_16(i32 %a0, i16 %a1) {
 declare i32 @llvm.x86.sse42.crc32.32.16(i32, i16) nounwind
 
 define i32 @stack_fold_crc32_32_32(i32 %a0, i32 %a1) {
-  ;CHECK-LABEL: stack_fold_crc32_32_32
-  ;CHECK:       crc32l {{-?[0-9]*}}(%rsp), %eax {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_crc32_32_32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movl %esi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    crc32l {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Folded Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = call i32 @llvm.x86.sse42.crc32.32.32(i32 %a0, i32 %a1)
   ret i32 %2
@@ -90,8 +238,45 @@ define i32 @stack_fold_crc32_32_32(i32 %a0, i32 %a1) {
 declare i32 @llvm.x86.sse42.crc32.32.32(i32, i32) nounwind
 
 define i64 @stack_fold_crc32_64_64(i64 %a0, i64 %a1) {
-  ;CHECK-LABEL: stack_fold_crc32_64_64
-  ;CHECK:       crc32q {{-?[0-9]*}}(%rsp), %rax {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_crc32_64_64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    crc32q {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Folded Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = call i64 @llvm.x86.sse42.crc32.64.64(i64 %a0, i64 %a1)
   ret i64 %2
@@ -99,8 +284,47 @@ define i64 @stack_fold_crc32_64_64(i64 %a0, i64 %a1) {
 declare i64 @llvm.x86.sse42.crc32.64.64(i64, i64) nounwind
 
 define <4 x i32> @stack_fold_movd_load(i32 %a0) {
-  ;CHECK-LABEL: stack_fold_movd_load
-  ;CHECK:       movd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_movd_load:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = mem[0],zero,zero,zero
+; CHECK-NEXT:    pcmpeqd %xmm1, %xmm1
+; CHECK-NEXT:    psubd %xmm1, %xmm0
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = insertelement <4 x i32> zeroinitializer, i32 %a0, i32 0
   ; add forces execution domain
@@ -109,8 +333,45 @@ define <4 x i32> @stack_fold_movd_load(i32 %a0) {
 }
 
 define i32 @stack_fold_movd_store(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_movd_store
-  ;CHECK:       movd {{%xmm[0-9][0-9]*}}, {{-?[0-9]*}}(%rsp) {{.*#+}} 4-byte Folded Spill
+; CHECK-LABEL: stack_fold_movd_store:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    paddd %xmm1, %xmm0
+; CHECK-NEXT:    movd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   ; add forces execution domain
   %1 = add <4 x i32> %a0, %a1
   %2 = extractelement <4 x i32> %1, i32 0
@@ -119,8 +380,17 @@ define i32 @stack_fold_movd_store(<4 x i32> %a0, <4 x i32> %a1) {
 }
 
 define <2 x i64> @stack_fold_movq_load(<2 x i64> %a0) {
-  ;CHECK-LABEL: stack_fold_movq_load
-  ;CHECK:       movq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_movq_load:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = mem[0],zero
+; CHECK-NEXT:    pcmpeqd %xmm1, %xmm1
+; CHECK-NEXT:    psubq %xmm1, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <2 x i64> %a0, <2 x i64> zeroinitializer, <2 x i32> <i32 0, i32 2>
   ; add forces execution domain
@@ -129,8 +399,45 @@ define <2 x i64> @stack_fold_movq_load(<2 x i64> %a0) {
 }
 
 define i64 @stack_fold_movq_store(<2 x i64> %a0, <2 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_movq_store
-  ;CHECK:       movq {{%xmm[0-9][0-9]*}}, {{-?[0-9]*}}(%rsp) {{.*#+}} 8-byte Folded Spill
+; CHECK-LABEL: stack_fold_movq_store:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    paddq %xmm1, %xmm0
+; CHECK-NEXT:    movq %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   ; add forces execution domain
   %1 = add <2 x i64> %a0, %a1
   %2 = extractelement <2 x i64> %1, i32 0
@@ -139,8 +446,14 @@ define i64 @stack_fold_movq_store(<2 x i64> %a0, <2 x i64> %a1) {
 }
 
 define <8 x i16> @stack_fold_mpsadbw(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_mpsadbw
-  ;CHECK:       mpsadbw $7, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_mpsadbw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    mpsadbw $7, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x i16> @llvm.x86.sse41.mpsadbw(<16 x i8> %a0, <16 x i8> %a1, i8 7)
   ret <8 x i16> %2
@@ -148,8 +461,14 @@ define <8 x i16> @stack_fold_mpsadbw(<16 x i8> %a0, <16 x i8> %a1) {
 declare <8 x i16> @llvm.x86.sse41.mpsadbw(<16 x i8>, <16 x i8>, i8) nounwind readnone
 
 define <16 x i8> @stack_fold_pabsb(<16 x i8> %a0) {
-  ;CHECK-LABEL: stack_fold_pabsb
-  ;CHECK:       pabsb {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pabsb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pabsb {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp sgt <16 x i8> %a0, zeroinitializer
   %3 = sub <16 x i8> zeroinitializer, %a0
@@ -158,8 +477,14 @@ define <16 x i8> @stack_fold_pabsb(<16 x i8> %a0) {
 }
 
 define <4 x i32> @stack_fold_pabsd(<4 x i32> %a0) {
-  ;CHECK-LABEL: stack_fold_pabsd
-  ;CHECK:       pabsd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pabsd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pabsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp sgt <4 x i32> %a0, zeroinitializer
   %3 = sub <4 x i32> zeroinitializer, %a0
@@ -168,8 +493,14 @@ define <4 x i32> @stack_fold_pabsd(<4 x i32> %a0) {
 }
 
 define <8 x i16> @stack_fold_pabsw(<8 x i16> %a0) {
-  ;CHECK-LABEL: stack_fold_pabsw
-  ;CHECK:       pabsw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pabsw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pabsw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp sgt <8 x i16> %a0, zeroinitializer
   %3 = sub <8 x i16> zeroinitializer, %a0
@@ -178,8 +509,14 @@ define <8 x i16> @stack_fold_pabsw(<8 x i16> %a0) {
 }
 
 define <8 x i16> @stack_fold_packssdw(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_packssdw
-  ;CHECK:       packssdw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_packssdw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    packssdw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x i16> @llvm.x86.sse2.packssdw.128(<4 x i32> %a0, <4 x i32> %a1)
   ret <8 x i16> %2
@@ -187,8 +524,14 @@ define <8 x i16> @stack_fold_packssdw(<4 x i32> %a0, <4 x i32> %a1) {
 declare <8 x i16> @llvm.x86.sse2.packssdw.128(<4 x i32>, <4 x i32>) nounwind readnone
 
 define <16 x i8> @stack_fold_packsswb(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_packsswb
-  ;CHECK:       packsswb {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_packsswb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    packsswb {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <16 x i8> @llvm.x86.sse2.packsswb.128(<8 x i16> %a0, <8 x i16> %a1)
   ret <16 x i8> %2
@@ -196,8 +539,14 @@ define <16 x i8> @stack_fold_packsswb(<8 x i16> %a0, <8 x i16> %a1) {
 declare <16 x i8> @llvm.x86.sse2.packsswb.128(<8 x i16>, <8 x i16>) nounwind readnone
 
 define <8 x i16> @stack_fold_packusdw(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_packusdw
-  ;CHECK:       packusdw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_packusdw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    packusdw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x i16> @llvm.x86.sse41.packusdw(<4 x i32> %a0, <4 x i32> %a1)
   ret <8 x i16> %2
@@ -205,8 +554,14 @@ define <8 x i16> @stack_fold_packusdw(<4 x i32> %a0, <4 x i32> %a1) {
 declare <8 x i16> @llvm.x86.sse41.packusdw(<4 x i32>, <4 x i32>) nounwind readnone
 
 define <16 x i8> @stack_fold_packuswb(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_packuswb
-  ;CHECK:       packuswb {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_packuswb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    packuswb {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <16 x i8> @llvm.x86.sse2.packuswb.128(<8 x i16> %a0, <8 x i16> %a1)
   ret <16 x i8> %2
@@ -214,32 +569,56 @@ define <16 x i8> @stack_fold_packuswb(<8 x i16> %a0, <8 x i16> %a1) {
 declare <16 x i8> @llvm.x86.sse2.packuswb.128(<8 x i16>, <8 x i16>) nounwind readnone
 
 define <16 x i8> @stack_fold_paddb(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_paddb
-  ;CHECK:       paddb {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    paddb {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = add <16 x i8> %a0, %a1
   ret <16 x i8> %2
 }
 
 define <4 x i32> @stack_fold_paddd(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_paddd
-  ;CHECK:       paddd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    paddd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = add <4 x i32> %a0, %a1
   ret <4 x i32> %2
 }
 
 define <2 x i64> @stack_fold_paddq(<2 x i64> %a0, <2 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_paddq
-  ;CHECK:       paddq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    paddq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = add <2 x i64> %a0, %a1
   ret <2 x i64> %2
 }
 
 define <16 x i8> @stack_fold_paddsb(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_paddsb
-  ;CHECK:       paddsb {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddsb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    paddsb {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> %a0, <16 x i8> %a1)
   ret <16 x i8> %2
@@ -247,8 +626,14 @@ define <16 x i8> @stack_fold_paddsb(<16 x i8> %a0, <16 x i8> %a1) {
 declare <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8>, <16 x i8>) nounwind readnone
 
 define <8 x i16> @stack_fold_paddsw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_paddsw
-  ;CHECK:       paddsw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddsw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    paddsw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> %a0, <8 x i16> %a1)
   ret <8 x i16> %2
@@ -256,8 +641,14 @@ define <8 x i16> @stack_fold_paddsw(<8 x i16> %a0, <8 x i16> %a1) {
 declare <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16>, <8 x i16>) nounwind readnone
 
 define <16 x i8> @stack_fold_paddusb(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_paddusb
-  ;CHECK:       paddusb {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddusb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    paddusb {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> %a0, <16 x i8> %a1)
   ret <16 x i8> %2
@@ -265,8 +656,14 @@ define <16 x i8> @stack_fold_paddusb(<16 x i8> %a0, <16 x i8> %a1) {
 declare <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8>, <16 x i8>) nounwind readnone
 
 define <8 x i16> @stack_fold_paddusw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_paddusw
-  ;CHECK:       paddusw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddusw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    paddusw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> %a0, <8 x i16> %a1)
   ret <8 x i16> %2
@@ -274,24 +671,45 @@ define <8 x i16> @stack_fold_paddusw(<8 x i16> %a0, <8 x i16> %a1) {
 declare <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16>, <8 x i16>) nounwind readnone
 
 define <8 x i16> @stack_fold_paddw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_paddw
-  ;CHECK:       paddw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    paddw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = add <8 x i16> %a0, %a1
   ret <8 x i16> %2
 }
 
 define <16 x i8> @stack_fold_palignr(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_palignr
-  ;CHECK:       palignr $1, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_palignr:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    palignr $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = mem[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],xmm0[0]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <16 x i8> %a1, <16 x i8> %a0, <16 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16>
   ret <16 x i8> %2
 }
 
 define <16 x i8> @stack_fold_pand(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pand
-  ;CHECK:       pand {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pand:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pand {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    pcmpeqd %xmm1, %xmm1
+; CHECK-NEXT:    psubb %xmm1, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = and <16 x i8> %a0, %a1
   ; add forces execution domain
@@ -300,8 +718,16 @@ define <16 x i8> @stack_fold_pand(<16 x i8> %a0, <16 x i8> %a1) {
 }
 
 define <16 x i8> @stack_fold_pandn(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pandn
-  ;CHECK:       pandn {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pandn:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pandn {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    pcmpeqd %xmm1, %xmm1
+; CHECK-NEXT:    psubb %xmm1, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = xor <16 x i8> %a0, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
   %3 = and <16 x i8> %2, %a1
@@ -311,8 +737,14 @@ define <16 x i8> @stack_fold_pandn(<16 x i8> %a0, <16 x i8> %a1) {
 }
 
 define <16 x i8> @stack_fold_pavgb(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pavgb
-  ;CHECK:       pavgb {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pavgb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pavgb {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = zext <16 x i8> %a0 to <16 x i16>
   %3 = zext <16 x i8> %a1 to <16 x i16>
@@ -324,8 +756,14 @@ define <16 x i8> @stack_fold_pavgb(<16 x i8> %a0, <16 x i8> %a1) {
 }
 
 define <8 x i16> @stack_fold_pavgw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_pavgw
-  ;CHECK:       pavgw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pavgw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pavgw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = zext <8 x i16> %a0 to <8 x i32>
   %3 = zext <8 x i16> %a1 to <8 x i32>
@@ -337,8 +775,16 @@ define <8 x i16> @stack_fold_pavgw(<8 x i16> %a0, <8 x i16> %a1) {
 }
 
 define <16 x i8> @stack_fold_pblendvb(<16 x i8> %a0, <16 x i8> %a1, <16 x i8> %c) {
-  ;CHECK-LABEL: stack_fold_pblendvb
-  ;CHECK:       pblendvb %xmm0, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pblendvb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movdqa %xmm1, %xmm2
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pblendvb %xmm0, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
+; CHECK-NEXT:    movdqa %xmm2, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <16 x i8> @llvm.x86.sse41.pblendvb(<16 x i8> %a1, <16 x i8> %c, <16 x i8> %a0)
   ret <16 x i8> %2
@@ -346,16 +792,29 @@ define <16 x i8> @stack_fold_pblendvb(<16 x i8> %a0, <16 x i8> %a1, <16 x i8> %c
 declare <16 x i8> @llvm.x86.sse41.pblendvb(<16 x i8>, <16 x i8>, <16 x i8>) nounwind readnone
 
 define <8 x i16> @stack_fold_pblendw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_pblendw
-  ;CHECK:       pblendw $7, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pblendw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pblendw $7, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = mem[0,1,2],xmm0[3,4,5,6,7]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <8 x i16> %a0, <8 x i16> %a1, <8 x i32> <i32 8, i32 9, i32 10, i32 3, i32 4, i32 5, i32 6, i32 7>
   ret <8 x i16> %2
 }
 
 define <2 x i64> @stack_fold_pclmulqdq(<2 x i64> %a0, <2 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_pclmulqdq
-  ;CHECK:       pclmulqdq $0, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pclmulqdq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pclmulqdq $0, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x i64> @llvm.x86.pclmulqdq(<2 x i64> %a0, <2 x i64> %a1, i8 0)
   ret <2 x i64> %2
@@ -363,8 +822,14 @@ define <2 x i64> @stack_fold_pclmulqdq(<2 x i64> %a0, <2 x i64> %a1) {
 declare <2 x i64> @llvm.x86.pclmulqdq(<2 x i64>, <2 x i64>, i8) nounwind readnone
 
 define <16 x i8> @stack_fold_pcmpeqb(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pcmpeqb
-  ;CHECK:       pcmpeqb {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pcmpeqb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pcmpeqb {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp eq <16 x i8> %a0, %a1
   %3 = sext <16 x i1> %2 to <16 x i8>
@@ -372,8 +837,14 @@ define <16 x i8> @stack_fold_pcmpeqb(<16 x i8> %a0, <16 x i8> %a1) {
 }
 
 define <4 x i32> @stack_fold_pcmpeqd(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_pcmpeqd
-  ;CHECK:       pcmpeqd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pcmpeqd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pcmpeqd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp eq <4 x i32> %a0, %a1
   %3 = sext <4 x i1> %2 to <4 x i32>
@@ -381,8 +852,14 @@ define <4 x i32> @stack_fold_pcmpeqd(<4 x i32> %a0, <4 x i32> %a1) {
 }
 
 define <2 x i64> @stack_fold_pcmpeqq(<2 x i64> %a0, <2 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_pcmpeqq
-  ;CHECK:       pcmpeqq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pcmpeqq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pcmpeqq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp eq <2 x i64> %a0, %a1
   %3 = sext <2 x i1> %2 to <2 x i64>
@@ -390,8 +867,14 @@ define <2 x i64> @stack_fold_pcmpeqq(<2 x i64> %a0, <2 x i64> %a1) {
 }
 
 define <8 x i16> @stack_fold_pcmpeqw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_pcmpeqw
-  ;CHECK:       pcmpeqw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pcmpeqw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pcmpeqw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp eq <8 x i16> %a0, %a1
   %3 = sext <8 x i1> %2 to <8 x i16>
@@ -399,8 +882,17 @@ define <8 x i16> @stack_fold_pcmpeqw(<8 x i16> %a0, <8 x i16> %a1) {
 }
 
 define i32 @stack_fold_pcmpestri(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pcmpestri
-  ;CHECK:       pcmpestri $7, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pcmpestri:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    movl $7, %eax
+; CHECK-NEXT:    movl $7, %edx
+; CHECK-NEXT:    pcmpestri $7, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    movl %ecx, %eax
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{rax},~{flags}"()
   %2 = call i32 @llvm.x86.sse42.pcmpestri128(<16 x i8> %a0, i32 7, <16 x i8> %a1, i32 7, i8 7)
   ret i32 %2
@@ -408,8 +900,16 @@ define i32 @stack_fold_pcmpestri(<16 x i8> %a0, <16 x i8> %a1) {
 declare i32 @llvm.x86.sse42.pcmpestri128(<16 x i8>, i32, <16 x i8>, i32, i8) nounwind readnone
 
 define <16 x i8> @stack_fold_pcmpestrm(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pcmpestrm
-  ;CHECK:       pcmpestrm $7, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pcmpestrm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    movl $7, %eax
+; CHECK-NEXT:    movl $7, %edx
+; CHECK-NEXT:    pcmpestrm $7, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{rax},~{flags}"()
   %2 = call <16 x i8> @llvm.x86.sse42.pcmpestrm128(<16 x i8> %a0, i32 7, <16 x i8> %a1, i32 7, i8 7)
   ret <16 x i8> %2
@@ -417,8 +917,14 @@ define <16 x i8> @stack_fold_pcmpestrm(<16 x i8> %a0, <16 x i8> %a1) {
 declare <16 x i8> @llvm.x86.sse42.pcmpestrm128(<16 x i8>, i32, <16 x i8>, i32, i8) nounwind readnone
 
 define <16 x i8> @stack_fold_pcmpgtb(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pcmpgtb
-  ;CHECK:       pcmpgtb {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pcmpgtb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pcmpgtb {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp sgt <16 x i8> %a0, %a1
   %3 = sext <16 x i1> %2 to <16 x i8>
@@ -426,8 +932,14 @@ define <16 x i8> @stack_fold_pcmpgtb(<16 x i8> %a0, <16 x i8> %a1) {
 }
 
 define <4 x i32> @stack_fold_pcmpgtd(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_pcmpgtd
-  ;CHECK:       pcmpgtd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pcmpgtd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pcmpgtd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp sgt <4 x i32> %a0, %a1
   %3 = sext <4 x i1> %2 to <4 x i32>
@@ -435,8 +947,14 @@ define <4 x i32> @stack_fold_pcmpgtd(<4 x i32> %a0, <4 x i32> %a1) {
 }
 
 define <2 x i64> @stack_fold_pcmpgtq(<2 x i64> %a0, <2 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_pcmpgtq
-  ;CHECK:       pcmpgtq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pcmpgtq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pcmpgtq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp sgt <2 x i64> %a0, %a1
   %3 = sext <2 x i1> %2 to <2 x i64>
@@ -444,8 +962,14 @@ define <2 x i64> @stack_fold_pcmpgtq(<2 x i64> %a0, <2 x i64> %a1) {
 }
 
 define <8 x i16> @stack_fold_pcmpgtw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_pcmpgtw
-  ;CHECK:       pcmpgtw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pcmpgtw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pcmpgtw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp sgt <8 x i16> %a0, %a1
   %3 = sext <8 x i1> %2 to <8 x i16>
@@ -453,8 +977,15 @@ define <8 x i16> @stack_fold_pcmpgtw(<8 x i16> %a0, <8 x i16> %a1) {
 }
 
 define i32 @stack_fold_pcmpistri(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pcmpistri
-  ;CHECK:       pcmpistri $7, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pcmpistri:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pcmpistri $7, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    movl %ecx, %eax
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call i32 @llvm.x86.sse42.pcmpistri128(<16 x i8> %a0, <16 x i8> %a1, i8 7)
   ret i32 %2
@@ -462,8 +993,14 @@ define i32 @stack_fold_pcmpistri(<16 x i8> %a0, <16 x i8> %a1) {
 declare i32 @llvm.x86.sse42.pcmpistri128(<16 x i8>, <16 x i8>, i8) nounwind readnone
 
 define <16 x i8> @stack_fold_pcmpistrm(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pcmpistrm
-  ;CHECK:       pcmpistrm $7, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pcmpistrm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pcmpistrm $7, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <16 x i8> @llvm.x86.sse42.pcmpistrm128(<16 x i8> %a0, <16 x i8> %a1, i8 7)
   ret <16 x i8> %2
@@ -475,10 +1012,47 @@ declare <16 x i8> @llvm.x86.sse42.pcmpistrm128(<16 x i8>, <16 x i8>, i8) nounwin
 ; We can't naively fold pextrw as it only writes to a 16-bit memory location
 ; even though it can store to a 32-bit register.
 define i16 @stack_fold_pextrw(<8 x i16> %a0) {
-; CHECK-LABEL: stack_fold_pextrw
-; CHECK:       pextrw $1, {{%xmm[0-9][0-9]*}}, %[[GPR32:(e[a-z]+|r[0-9]+d)]]
-; CHECK:       movl %[[GPR32]], {{-?[0-9]*}}(%rsp) {{.*#+}} 4-byte Spill
-; CHECK:       movl {{-?[0-9]*}}(%rsp), %eax {{.*#+}} 4-byte Reload
+; CHECK-LABEL: stack_fold_pextrw:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    pextrw $1, %xmm0, %eax
+; CHECK-NEXT:    addl $2, %eax
+; CHECK-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
+; CHECK-NEXT:    # kill: def $ax killed $ax killed $eax
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
 entry:
 ; add forces execution domain
   %add = add <8 x i16> %a0, <i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8>
@@ -488,9 +1062,45 @@ entry:
 }
 
 define i32 @stack_fold_pextrd(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_pextrd
-  ;CHECK:       pextrd $1, {{%xmm[0-9][0-9]*}}, {{-?[0-9]*}}(%rsp) {{.*#+}} 4-byte Folded Spill
-  ;CHECK:       movl    {{-?[0-9]*}}(%rsp), %eax {{.*#+}} 4-byte Reload
+; CHECK-LABEL: stack_fold_pextrd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    paddd %xmm1, %xmm0
+; CHECK-NEXT:    pextrd $1, %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   ; add forces execution domain
   %1 = add <4 x i32> %a0, %a1
   %2 = extractelement <4 x i32> %1, i32 1
@@ -499,17 +1109,58 @@ define i32 @stack_fold_pextrd(<4 x i32> %a0, <4 x i32> %a1) {
 }
 
 define i64 @stack_fold_pextrq(<2 x i64> %a0) {
-  ;CHECK-LABEL: stack_fold_pextrq
-  ;CHECK:       pextrq $1, {{%xmm[0-9][0-9]*}}, {{-?[0-9]*}}(%rsp) {{.*#+}} 8-byte Folded Spill
-  ;CHECK:       movq    {{-?[0-9]*}}(%rsp), %rax {{.*#+}} 8-byte Reload
+; CHECK-LABEL: stack_fold_pextrq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    pextrq $1, %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = extractelement <2 x i64> %a0, i32 1
   %2 = tail call <2 x i64> asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   ret i64 %1
 }
 
 define <4 x i32> @stack_fold_phaddd(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_phaddd
-  ;CHECK:       phaddd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_phaddd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    phaddd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x i32> @llvm.x86.ssse3.phadd.d.128(<4 x i32> %a0, <4 x i32> %a1)
   ret <4 x i32> %2
@@ -517,8 +1168,14 @@ define <4 x i32> @stack_fold_phaddd(<4 x i32> %a0, <4 x i32> %a1) {
 declare <4 x i32> @llvm.x86.ssse3.phadd.d.128(<4 x i32>, <4 x i32>) nounwind readnone
 
 define <8 x i16> @stack_fold_phaddsw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_phaddsw
-  ;CHECK:       phaddsw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_phaddsw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    phaddsw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x i16> @llvm.x86.ssse3.phadd.sw.128(<8 x i16> %a0, <8 x i16> %a1)
   ret <8 x i16> %2
@@ -526,8 +1183,14 @@ define <8 x i16> @stack_fold_phaddsw(<8 x i16> %a0, <8 x i16> %a1) {
 declare <8 x i16> @llvm.x86.ssse3.phadd.sw.128(<8 x i16>, <8 x i16>) nounwind readnone
 
 define <8 x i16> @stack_fold_phaddw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_phaddw
-  ;CHECK:       phaddw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_phaddw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    phaddw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x i16> @llvm.x86.ssse3.phadd.w.128(<8 x i16> %a0, <8 x i16> %a1)
   ret <8 x i16> %2
@@ -535,8 +1198,14 @@ define <8 x i16> @stack_fold_phaddw(<8 x i16> %a0, <8 x i16> %a1) {
 declare <8 x i16> @llvm.x86.ssse3.phadd.w.128(<8 x i16>, <8 x i16>) nounwind readnone
 
 define <8 x i16> @stack_fold_phminposuw(<8 x i16> %a0) {
-  ;CHECK-LABEL: stack_fold_phminposuw
-  ;CHECK:       phminposuw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_phminposuw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    phminposuw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x i16> @llvm.x86.sse41.phminposuw(<8 x i16> %a0)
   ret <8 x i16> %2
@@ -544,8 +1213,14 @@ define <8 x i16> @stack_fold_phminposuw(<8 x i16> %a0) {
 declare <8 x i16> @llvm.x86.sse41.phminposuw(<8 x i16>) nounwind readnone
 
 define <4 x i32> @stack_fold_phsubd(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_phsubd
-  ;CHECK:       phsubd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_phsubd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    phsubd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x i32> @llvm.x86.ssse3.phsub.d.128(<4 x i32> %a0, <4 x i32> %a1)
   ret <4 x i32> %2
@@ -553,8 +1228,14 @@ define <4 x i32> @stack_fold_phsubd(<4 x i32> %a0, <4 x i32> %a1) {
 declare <4 x i32> @llvm.x86.ssse3.phsub.d.128(<4 x i32>, <4 x i32>) nounwind readnone
 
 define <8 x i16> @stack_fold_phsubsw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_phsubsw
-  ;CHECK:       phsubsw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_phsubsw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    phsubsw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x i16> @llvm.x86.ssse3.phsub.sw.128(<8 x i16> %a0, <8 x i16> %a1)
   ret <8 x i16> %2
@@ -562,8 +1243,14 @@ define <8 x i16> @stack_fold_phsubsw(<8 x i16> %a0, <8 x i16> %a1) {
 declare <8 x i16> @llvm.x86.ssse3.phsub.sw.128(<8 x i16>, <8 x i16>) nounwind readnone
 
 define <8 x i16> @stack_fold_phsubw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_phsubw
-  ;CHECK:       phsubw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_phsubw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    phsubw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x i16> @llvm.x86.ssse3.phsub.w.128(<8 x i16> %a0, <8 x i16> %a1)
   ret <8 x i16> %2
@@ -571,40 +1258,190 @@ define <8 x i16> @stack_fold_phsubw(<8 x i16> %a0, <8 x i16> %a1) {
 declare <8 x i16> @llvm.x86.ssse3.phsub.w.128(<8 x i16>, <8 x i16>) nounwind readnone
 
 define <16 x i8> @stack_fold_pinsrb(<16 x i8> %a0, i8 %a1) {
-  ;CHECK-LABEL: stack_fold_pinsrb
-  ;CHECK:       pinsrb $1, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_pinsrb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pinsrb $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = insertelement <16 x i8> %a0, i8 %a1, i32 1
   ret <16 x i8> %2
 }
 
 define <4 x i32> @stack_fold_pinsrd(<4 x i32> %a0, i32 %a1) {
-  ;CHECK-LABEL: stack_fold_pinsrd
-  ;CHECK:       pinsrd $1, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_pinsrd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pinsrd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = insertelement <4 x i32> %a0, i32 %a1, i32 1
   ret <4 x i32> %2
 }
 
 define <2 x i64> @stack_fold_pinsrq(<2 x i64> %a0, i64 %a1) {
-  ;CHECK-LABEL: stack_fold_pinsrq
-  ;CHECK:       pinsrq $1, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_pinsrq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pinsrq $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Folded Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = insertelement <2 x i64> %a0, i64 %a1, i32 1
   ret <2 x i64> %2
 }
 
 define <8 x i16> @stack_fold_pinsrw(<8 x i16> %a0, i16 %a1) {
-  ;CHECK-LABEL: stack_fold_pinsrw
-  ;CHECK:       pinsrw $1, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_pinsrw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pinsrw $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = insertelement <8 x i16> %a0, i16 %a1, i32 1
   ret <8 x i16> %2
 }
 
 define <8 x i16> @stack_fold_pmaddubsw(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pmaddubsw
-  ;CHECK:       pmaddubsw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaddubsw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pmaddubsw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8> %a0, <16 x i8> %a1)
   ret <8 x i16> %2
@@ -612,8 +1449,14 @@ define <8 x i16> @stack_fold_pmaddubsw(<16 x i8> %a0, <16 x i8> %a1) {
 declare <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8>, <16 x i8>) nounwind readnone
 
 define <4 x i32> @stack_fold_pmaddwd(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_pmaddwd
-  ;CHECK:       pmaddwd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaddwd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pmaddwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> %a0, <8 x i16> %a1)
   ret <4 x i32> %2
@@ -621,8 +1464,14 @@ define <4 x i32> @stack_fold_pmaddwd(<8 x i16> %a0, <8 x i16> %a1) {
 declare <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16>, <8 x i16>) nounwind readnone
 
 define <16 x i8> @stack_fold_pmaxsb(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pmaxsb
-  ;CHECK:       pmaxsb {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxsb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pmaxsb {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp sgt <16 x i8> %a0, %a1
   %3 = select <16 x i1> %2, <16 x i8> %a0, <16 x i8> %a1
@@ -630,8 +1479,14 @@ define <16 x i8> @stack_fold_pmaxsb(<16 x i8> %a0, <16 x i8> %a1) {
 }
 
 define <4 x i32> @stack_fold_pmaxsd(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_pmaxsd
-  ;CHECK:       pmaxsd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxsd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pmaxsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp sgt <4 x i32> %a0, %a1
   %3 = select <4 x i1> %2, <4 x i32> %a0, <4 x i32> %a1
@@ -639,8 +1494,14 @@ define <4 x i32> @stack_fold_pmaxsd(<4 x i32> %a0, <4 x i32> %a1) {
 }
 
 define <8 x i16> @stack_fold_pmaxsw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_pmaxsw
-  ;CHECK:       pmaxsw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxsw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pmaxsw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp sgt <8 x i16> %a0, %a1
   %3 = select <8 x i1> %2, <8 x i16> %a0, <8 x i16> %a1
@@ -648,8 +1509,14 @@ define <8 x i16> @stack_fold_pmaxsw(<8 x i16> %a0, <8 x i16> %a1) {
 }
 
 define <16 x i8> @stack_fold_pmaxub(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pmaxub
-  ;CHECK:       pmaxub {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxub:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pmaxub {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp ugt <16 x i8> %a0, %a1
   %3 = select <16 x i1> %2, <16 x i8> %a0, <16 x i8> %a1
@@ -657,8 +1524,14 @@ define <16 x i8> @stack_fold_pmaxub(<16 x i8> %a0, <16 x i8> %a1) {
 }
 
 define <4 x i32> @stack_fold_pmaxud(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_pmaxud
-  ;CHECK:       pmaxud {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxud:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pmaxud {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp ugt <4 x i32> %a0, %a1
   %3 = select <4 x i1> %2, <4 x i32> %a0, <4 x i32> %a1
@@ -666,8 +1539,14 @@ define <4 x i32> @stack_fold_pmaxud(<4 x i32> %a0, <4 x i32> %a1) {
 }
 
 define <8 x i16> @stack_fold_pmaxuw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_pmaxuw
-  ;CHECK:       pmaxuw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxuw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pmaxuw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp ugt <8 x i16> %a0, %a1
   %3 = select <8 x i1> %2, <8 x i16> %a0, <8 x i16> %a1
@@ -675,8 +1554,14 @@ define <8 x i16> @stack_fold_pmaxuw(<8 x i16> %a0, <8 x i16> %a1) {
 }
 
 define <16 x i8> @stack_fold_pminsb(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pminsb
-  ;CHECK:       pminsb {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminsb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pminsb {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp slt <16 x i8> %a0, %a1
   %3 = select <16 x i1> %2, <16 x i8> %a0, <16 x i8> %a1
@@ -684,8 +1569,14 @@ define <16 x i8> @stack_fold_pminsb(<16 x i8> %a0, <16 x i8> %a1) {
 }
 
 define <4 x i32> @stack_fold_pminsd(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_pminsd
-  ;CHECK:       pminsd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminsd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pminsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp slt <4 x i32> %a0, %a1
   %3 = select <4 x i1> %2, <4 x i32> %a0, <4 x i32> %a1
@@ -693,8 +1584,14 @@ define <4 x i32> @stack_fold_pminsd(<4 x i32> %a0, <4 x i32> %a1) {
 }
 
 define <8 x i16> @stack_fold_pminsw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_pminsw
-  ;CHECK:       pminsw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminsw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pminsw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp slt <8 x i16> %a0, %a1
   %3 = select <8 x i1> %2, <8 x i16> %a0, <8 x i16> %a1
@@ -702,8 +1599,14 @@ define <8 x i16> @stack_fold_pminsw(<8 x i16> %a0, <8 x i16> %a1) {
 }
 
 define <16 x i8> @stack_fold_pminub(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pminub
-  ;CHECK:       pminub {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminub:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pminub {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp ult <16 x i8> %a0, %a1
   %3 = select <16 x i1> %2, <16 x i8> %a0, <16 x i8> %a1
@@ -711,8 +1614,14 @@ define <16 x i8> @stack_fold_pminub(<16 x i8> %a0, <16 x i8> %a1) {
 }
 
 define <4 x i32> @stack_fold_pminud(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_pminud
-  ;CHECK:       pminud {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminud:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pminud {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp ult <4 x i32> %a0, %a1
   %3 = select <4 x i1> %2, <4 x i32> %a0, <4 x i32> %a1
@@ -720,8 +1629,14 @@ define <4 x i32> @stack_fold_pminud(<4 x i32> %a0, <4 x i32> %a1) {
 }
 
 define <8 x i16> @stack_fold_pminuw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_pminuw
-  ;CHECK:       pminuw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminuw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pminuw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = icmp ult <8 x i16> %a0, %a1
   %3 = select <8 x i1> %2, <8 x i16> %a0, <8 x i16> %a1
@@ -729,8 +1644,14 @@ define <8 x i16> @stack_fold_pminuw(<8 x i16> %a0, <8 x i16> %a1) {
 }
 
 define <4 x i32> @stack_fold_pmovsxbd(<16 x i8> %a0) {
-  ;CHECK-LABEL: stack_fold_pmovsxbd
-  ;CHECK:       pmovsxbd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmovsxbd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pmovsxbd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <16 x i8> %a0, <16 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %3 = sext <4 x i8> %2 to <4 x i32>
@@ -738,8 +1659,14 @@ define <4 x i32> @stack_fold_pmovsxbd(<16 x i8> %a0) {
 }
 
 define <2 x i64> @stack_fold_pmovsxbq(<16 x i8> %a0) {
-  ;CHECK-LABEL: stack_fold_pmovsxbq
-  ;CHECK:       pmovsxbq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmovsxbq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pmovsxbq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <16 x i8> %a0, <16 x i8> undef, <2 x i32> <i32 0, i32 1>
   %3 = sext <2 x i8> %2 to <2 x i64>
@@ -747,8 +1674,14 @@ define <2 x i64> @stack_fold_pmovsxbq(<16 x i8> %a0) {
 }
 
 define <8 x i16> @stack_fold_pmovsxbw(<16 x i8> %a0) {
-  ;CHECK-LABEL: stack_fold_pmovsxbw
-  ;CHECK:       pmovsxbw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmovsxbw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pmovsxbw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <16 x i8> %a0, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   %3 = sext <8 x i8> %2 to <8 x i16>
@@ -756,8 +1689,14 @@ define <8 x i16> @stack_fold_pmovsxbw(<16 x i8> %a0) {
 }
 
 define <2 x i64> @stack_fold_pmovsxdq(<4 x i32> %a0) {
-  ;CHECK-LABEL: stack_fold_pmovsxdq
-  ;CHECK:       pmovsxdq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmovsxdq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pmovsxdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <4 x i32> %a0, <4 x i32> undef, <2 x i32> <i32 0, i32 1>
   %3 = sext <2 x i32> %2 to <2 x i64>
@@ -765,8 +1704,14 @@ define <2 x i64> @stack_fold_pmovsxdq(<4 x i32> %a0) {
 }
 
 define <4 x i32> @stack_fold_pmovsxwd(<8 x i16> %a0) {
-  ;CHECK-LABEL: stack_fold_pmovsxwd
-  ;CHECK:       pmovsxwd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmovsxwd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pmovsxwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <8 x i16> %a0, <8 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %3 = sext <4 x i16> %2 to <4 x i32>
@@ -774,8 +1719,14 @@ define <4 x i32> @stack_fold_pmovsxwd(<8 x i16> %a0) {
 }
 
 define <2 x i64> @stack_fold_pmovsxwq(<8 x i16> %a0) {
-  ;CHECK-LABEL: stack_fold_pmovsxwq
-  ;CHECK:       pmovsxwq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmovsxwq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pmovsxwq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <8 x i16> %a0, <8 x i16> undef, <2 x i32> <i32 0, i32 1>
   %3 = sext <2 x i16> %2 to <2 x i64>
@@ -783,8 +1734,15 @@ define <2 x i64> @stack_fold_pmovsxwq(<8 x i16> %a0) {
 }
 
 define <4 x i32> @stack_fold_pmovzxbd(<16 x i8> %a0) {
-  ;CHECK-LABEL: stack_fold_pmovzxbd
-  ;CHECK:       pmovzxbd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmovzxbd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pmovzxbd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <16 x i8> %a0, <16 x i8> zeroinitializer, <16 x i32> <i32 0, i32 16, i32 17, i32 18, i32 1, i32 19, i32 20, i32 21, i32 2, i32 22, i32 23, i32 24, i32 3, i32 25, i32 26, i32 27>
   %3 = bitcast <16 x i8> %2 to <4 x i32>
@@ -792,8 +1750,15 @@ define <4 x i32> @stack_fold_pmovzxbd(<16 x i8> %a0) {
 }
 
 define <2 x i64> @stack_fold_pmovzxbq(<16 x i8> %a0) {
-  ;CHECK-LABEL: stack_fold_pmovzxbq
-  ;CHECK:       pmovzxbq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmovzxbq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pmovzxbq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <16 x i8> %a0, <16 x i8> zeroinitializer, <16 x i32> <i32 0, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 1, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28>
   %3 = bitcast <16 x i8> %2 to <2 x i64>
@@ -801,8 +1766,15 @@ define <2 x i64> @stack_fold_pmovzxbq(<16 x i8> %a0) {
 }
 
 define <8 x i16> @stack_fold_pmovzxbw(<16 x i8> %a0) {
-  ;CHECK-LABEL: stack_fold_pmovzxbw
-  ;CHECK:       pmovzxbw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmovzxbw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pmovzxbw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <16 x i8> %a0, <16 x i8> zeroinitializer, <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23>
   %3 = bitcast <16 x i8> %2 to <8 x i16>
@@ -810,8 +1782,15 @@ define <8 x i16> @stack_fold_pmovzxbw(<16 x i8> %a0) {
 }
 
 define <2 x i64> @stack_fold_pmovzxdq(<4 x i32> %a0) {
-  ;CHECK-LABEL: stack_fold_pmovzxdq
-  ;CHECK:       pmovzxdq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmovzxdq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pmovzxdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = mem[0],zero,mem[1],zero
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <4 x i32> %a0, <4 x i32> zeroinitializer, <4 x i32> <i32 0, i32 4, i32 1, i32 5>
   %3 = bitcast <4 x i32> %2 to <2 x i64>
@@ -819,8 +1798,15 @@ define <2 x i64> @stack_fold_pmovzxdq(<4 x i32> %a0) {
 }
 
 define <4 x i32> @stack_fold_pmovzxwd(<8 x i16> %a0) {
-  ;CHECK-LABEL: stack_fold_pmovzxwd
-  ;CHECK:       pmovzxwd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmovzxwd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pmovzxwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <8 x i16> %a0, <8 x i16> zeroinitializer, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11>
   %3 = bitcast <8 x i16> %2 to <4 x i32>
@@ -828,8 +1814,15 @@ define <4 x i32> @stack_fold_pmovzxwd(<8 x i16> %a0) {
 }
 
 define <2 x i64> @stack_fold_pmovzxwq(<8 x i16> %a0) {
-  ;CHECK-LABEL: stack_fold_pmovzxwq
-  ;CHECK:       pmovzxwq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmovzxwq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pmovzxwq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <8 x i16> %a0, <8 x i16> zeroinitializer, <8 x i32> <i32 0, i32 8, i32 9, i32 10, i32 1, i32 11, i32 12, i32 13>
   %3 = bitcast <8 x i16> %2 to <2 x i64>
@@ -837,8 +1830,14 @@ define <2 x i64> @stack_fold_pmovzxwq(<8 x i16> %a0) {
 }
 
 define <2 x i64> @stack_fold_pmuldq(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_pmuldq
-  ;CHECK:       pmuldq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmuldq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pmuldq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = bitcast <4 x i32> %a0 to <2 x i64>
   %3 = bitcast <4 x i32> %a1 to <2 x i64>
@@ -851,8 +1850,14 @@ define <2 x i64> @stack_fold_pmuldq(<4 x i32> %a0, <4 x i32> %a1) {
 }
 
 define <8 x i16> @stack_fold_pmulhrsw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_pmulhrsw
-  ;CHECK:       pmulhrsw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmulhrsw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pmulhrsw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x i16> @llvm.x86.ssse3.pmul.hr.sw.128(<8 x i16> %a0, <8 x i16> %a1)
   ret <8 x i16> %2
@@ -860,8 +1865,14 @@ define <8 x i16> @stack_fold_pmulhrsw(<8 x i16> %a0, <8 x i16> %a1) {
 declare <8 x i16> @llvm.x86.ssse3.pmul.hr.sw.128(<8 x i16>, <8 x i16>) nounwind readnone
 
 define <8 x i16> @stack_fold_pmulhuw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_pmulhuw
-  ;CHECK:       pmulhuw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmulhuw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pmulhuw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x i16> @llvm.x86.sse2.pmulhu.w(<8 x i16> %a0, <8 x i16> %a1)
   ret <8 x i16> %2
@@ -869,8 +1880,14 @@ define <8 x i16> @stack_fold_pmulhuw(<8 x i16> %a0, <8 x i16> %a1) {
 declare <8 x i16> @llvm.x86.sse2.pmulhu.w(<8 x i16>, <8 x i16>) nounwind readnone
 
 define <8 x i16> @stack_fold_pmulhw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_pmulhw
-  ;CHECK:       pmulhw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmulhw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pmulhw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x i16> @llvm.x86.sse2.pmulh.w(<8 x i16> %a0, <8 x i16> %a1)
   ret <8 x i16> %2
@@ -878,24 +1895,42 @@ define <8 x i16> @stack_fold_pmulhw(<8 x i16> %a0, <8 x i16> %a1) {
 declare <8 x i16> @llvm.x86.sse2.pmulh.w(<8 x i16>, <8 x i16>) nounwind readnone
 
 define <4 x i32> @stack_fold_pmulld(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_pmulld
-  ;CHECK:       pmulld {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmulld:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pmulld {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = mul <4 x i32> %a0, %a1
   ret <4 x i32> %2
 }
 
 define <8 x i16> @stack_fold_pmullw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_pmullw
-  ;CHECK:       pmullw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmullw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pmullw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = mul <8 x i16> %a0, %a1
   ret <8 x i16> %2
 }
 
 define <2 x i64> @stack_fold_pmuludq(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_pmuludq
-  ;CHECK:       pmuludq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmuludq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pmuludq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = bitcast <4 x i32> %a0 to <2 x i64>
   %3 = bitcast <4 x i32> %a1 to <2 x i64>
@@ -906,8 +1941,16 @@ define <2 x i64> @stack_fold_pmuludq(<4 x i32> %a0, <4 x i32> %a1) {
 }
 
 define <16 x i8> @stack_fold_por(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_por
-  ;CHECK:       por {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_por:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    por {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    pcmpeqd %xmm1, %xmm1
+; CHECK-NEXT:    psubb %xmm1, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = or <16 x i8> %a0, %a1
   ; add forces execution domain
@@ -916,8 +1959,14 @@ define <16 x i8> @stack_fold_por(<16 x i8> %a0, <16 x i8> %a1) {
 }
 
 define <2 x i64> @stack_fold_psadbw(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_psadbw
-  ;CHECK:       psadbw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psadbw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    psadbw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x i64> @llvm.x86.sse2.psad.bw(<16 x i8> %a0, <16 x i8> %a1)
   ret <2 x i64> %2
@@ -925,8 +1974,14 @@ define <2 x i64> @stack_fold_psadbw(<16 x i8> %a0, <16 x i8> %a1) {
 declare <2 x i64> @llvm.x86.sse2.psad.bw(<16 x i8>, <16 x i8>) nounwind readnone
 
 define <16 x i8> @stack_fold_pshufb(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pshufb
-  ;CHECK:       pshufb {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pshufb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pshufb {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %a0, <16 x i8> %a1)
   ret <16 x i8> %2
@@ -934,32 +1989,59 @@ define <16 x i8> @stack_fold_pshufb(<16 x i8> %a0, <16 x i8> %a1) {
 declare <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8>, <16 x i8>) nounwind readnone
 
 define <4 x i32> @stack_fold_pshufd(<4 x i32> %a0) {
-  ;CHECK-LABEL: stack_fold_pshufd
-  ;CHECK:       pshufd $27, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pshufd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pshufd $27, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = mem[3,2,1,0]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <4 x i32> %a0, <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
   ret <4 x i32> %2
 }
 
 define <8 x i16> @stack_fold_pshufhw(<8 x i16> %a0) {
-  ;CHECK-LABEL: stack_fold_pshufhw
-  ;CHECK:       pshufhw $11, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pshufhw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pshufhw $11, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = mem[0,1,2,3,7,6,4,4]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <8 x i16> %a0, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 7, i32 6, i32 4, i32 4>
   ret <8 x i16> %2
 }
 
 define <8 x i16> @stack_fold_pshuflw(<8 x i16> %a0) {
-  ;CHECK-LABEL: stack_fold_pshuflw
-  ;CHECK:       pshuflw $27, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pshuflw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pshuflw $27, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = mem[3,2,1,0,4,5,6,7]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <8 x i16> %a0, <8 x i16> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 4, i32 5, i32 6, i32 7>
   ret <8 x i16> %2
 }
 
 define <16 x i8> @stack_fold_psignb(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_psignb
-  ;CHECK:       psignb {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psignb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    psignb {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <16 x i8> @llvm.x86.ssse3.psign.b.128(<16 x i8> %a0, <16 x i8> %a1)
   ret <16 x i8> %2
@@ -967,8 +2049,14 @@ define <16 x i8> @stack_fold_psignb(<16 x i8> %a0, <16 x i8> %a1) {
 declare <16 x i8> @llvm.x86.ssse3.psign.b.128(<16 x i8>, <16 x i8>) nounwind readnone
 
 define <4 x i32> @stack_fold_psignd(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_psignd
-  ;CHECK:       psignd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psignd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    psignd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x i32> @llvm.x86.ssse3.psign.d.128(<4 x i32> %a0, <4 x i32> %a1)
   ret <4 x i32> %2
@@ -976,8 +2064,14 @@ define <4 x i32> @stack_fold_psignd(<4 x i32> %a0, <4 x i32> %a1) {
 declare <4 x i32> @llvm.x86.ssse3.psign.d.128(<4 x i32>, <4 x i32>) nounwind readnone
 
 define <8 x i16> @stack_fold_psignw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_psignw
-  ;CHECK:       psignw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psignw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    psignw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x i16> @llvm.x86.ssse3.psign.w.128(<8 x i16> %a0, <8 x i16> %a1)
   ret <8 x i16> %2
@@ -985,8 +2079,14 @@ define <8 x i16> @stack_fold_psignw(<8 x i16> %a0, <8 x i16> %a1) {
 declare <8 x i16> @llvm.x86.ssse3.psign.w.128(<8 x i16>, <8 x i16>) nounwind readnone
 
 define <4 x i32> @stack_fold_pslld(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_pslld
-  ;CHECK:       pslld {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pslld:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pslld {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x i32> @llvm.x86.sse2.psll.d(<4 x i32> %a0, <4 x i32> %a1)
   ret <4 x i32> %2
@@ -994,8 +2094,14 @@ define <4 x i32> @stack_fold_pslld(<4 x i32> %a0, <4 x i32> %a1) {
 declare <4 x i32> @llvm.x86.sse2.psll.d(<4 x i32>, <4 x i32>) nounwind readnone
 
 define <2 x i64> @stack_fold_psllq(<2 x i64> %a0, <2 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_psllq
-  ;CHECK:       psllq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psllq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    psllq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x i64> @llvm.x86.sse2.psll.q(<2 x i64> %a0, <2 x i64> %a1)
   ret <2 x i64> %2
@@ -1003,8 +2109,14 @@ define <2 x i64> @stack_fold_psllq(<2 x i64> %a0, <2 x i64> %a1) {
 declare <2 x i64> @llvm.x86.sse2.psll.q(<2 x i64>, <2 x i64>) nounwind readnone
 
 define <8 x i16> @stack_fold_psllw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_psllw
-  ;CHECK:       psllw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psllw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    psllw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x i16> @llvm.x86.sse2.psll.w(<8 x i16> %a0, <8 x i16> %a1)
   ret <8 x i16> %2
@@ -1012,8 +2124,14 @@ define <8 x i16> @stack_fold_psllw(<8 x i16> %a0, <8 x i16> %a1) {
 declare <8 x i16> @llvm.x86.sse2.psll.w(<8 x i16>, <8 x i16>) nounwind readnone
 
 define <4 x i32> @stack_fold_psrad(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_psrad
-  ;CHECK:       psrad {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psrad:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    psrad {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x i32> @llvm.x86.sse2.psra.d(<4 x i32> %a0, <4 x i32> %a1)
   ret <4 x i32> %2
@@ -1021,8 +2139,14 @@ define <4 x i32> @stack_fold_psrad(<4 x i32> %a0, <4 x i32> %a1) {
 declare <4 x i32> @llvm.x86.sse2.psra.d(<4 x i32>, <4 x i32>) nounwind readnone
 
 define <8 x i16> @stack_fold_psraw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_psraw
-  ;CHECK:       psraw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psraw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    psraw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x i16> @llvm.x86.sse2.psra.w(<8 x i16> %a0, <8 x i16> %a1)
   ret <8 x i16> %2
@@ -1030,8 +2154,14 @@ define <8 x i16> @stack_fold_psraw(<8 x i16> %a0, <8 x i16> %a1) {
 declare <8 x i16> @llvm.x86.sse2.psra.w(<8 x i16>, <8 x i16>) nounwind readnone
 
 define <4 x i32> @stack_fold_psrld(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_psrld
-  ;CHECK:       psrld {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psrld:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    psrld {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x i32> @llvm.x86.sse2.psrl.d(<4 x i32> %a0, <4 x i32> %a1)
   ret <4 x i32> %2
@@ -1039,8 +2169,14 @@ define <4 x i32> @stack_fold_psrld(<4 x i32> %a0, <4 x i32> %a1) {
 declare <4 x i32> @llvm.x86.sse2.psrl.d(<4 x i32>, <4 x i32>) nounwind readnone
 
 define <2 x i64> @stack_fold_psrlq(<2 x i64> %a0, <2 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_psrlq
-  ;CHECK:       psrlq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psrlq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    psrlq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x i64> @llvm.x86.sse2.psrl.q(<2 x i64> %a0, <2 x i64> %a1)
   ret <2 x i64> %2
@@ -1048,8 +2184,14 @@ define <2 x i64> @stack_fold_psrlq(<2 x i64> %a0, <2 x i64> %a1) {
 declare <2 x i64> @llvm.x86.sse2.psrl.q(<2 x i64>, <2 x i64>) nounwind readnone
 
 define <8 x i16> @stack_fold_psrlw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_psrlw
-  ;CHECK:       psrlw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psrlw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    psrlw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x i16> @llvm.x86.sse2.psrl.w(<8 x i16> %a0, <8 x i16> %a1)
   ret <8 x i16> %2
@@ -1057,32 +2199,56 @@ define <8 x i16> @stack_fold_psrlw(<8 x i16> %a0, <8 x i16> %a1) {
 declare <8 x i16> @llvm.x86.sse2.psrl.w(<8 x i16>, <8 x i16>) nounwind readnone
 
 define <16 x i8> @stack_fold_psubb(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_psubb
-  ;CHECK:       psubb {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psubb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    psubb {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = sub <16 x i8> %a0, %a1
   ret <16 x i8> %2
 }
 
 define <4 x i32> @stack_fold_psubd(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_psubd
-  ;CHECK:       psubd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psubd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    psubd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = sub <4 x i32> %a0, %a1
   ret <4 x i32> %2
 }
 
 define <2 x i64> @stack_fold_psubq(<2 x i64> %a0, <2 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_psubq
-  ;CHECK:       psubq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psubq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    psubq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = sub <2 x i64> %a0, %a1
   ret <2 x i64> %2
 }
 
 define <16 x i8> @stack_fold_psubsb(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_psubsb
-  ;CHECK:       psubsb {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psubsb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    psubsb {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> %a0, <16 x i8> %a1)
   ret <16 x i8> %2
@@ -1090,8 +2256,14 @@ define <16 x i8> @stack_fold_psubsb(<16 x i8> %a0, <16 x i8> %a1) {
 declare <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8>, <16 x i8>) nounwind readnone
 
 define <8 x i16> @stack_fold_psubsw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_psubsw
-  ;CHECK:       psubsw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psubsw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    psubsw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> %a0, <8 x i16> %a1)
   ret <8 x i16> %2
@@ -1099,8 +2271,14 @@ define <8 x i16> @stack_fold_psubsw(<8 x i16> %a0, <8 x i16> %a1) {
 declare <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16>, <8 x i16>) nounwind readnone
 
 define <16 x i8> @stack_fold_psubusb(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_psubusb
-  ;CHECK:       psubusb {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psubusb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    psubusb {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> %a0, <16 x i8> %a1)
   ret <16 x i8> %2
@@ -1108,8 +2286,14 @@ define <16 x i8> @stack_fold_psubusb(<16 x i8> %a0, <16 x i8> %a1) {
 declare <16 x i8> @llvm.usub.sat.v16i8(<16 x i8>, <16 x i8>) nounwind readnone
 
 define <8 x i16> @stack_fold_psubusw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_psubusw
-  ;CHECK:       psubusw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psubusw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    psubusw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> %a0, <8 x i16> %a1)
   ret <8 x i16> %2
@@ -1117,16 +2301,30 @@ define <8 x i16> @stack_fold_psubusw(<8 x i16> %a0, <8 x i16> %a1) {
 declare <8 x i16> @llvm.usub.sat.v8i16(<8 x i16>, <8 x i16>) nounwind readnone
 
 define <8 x i16> @stack_fold_psubw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_psubw
-  ;CHECK:       psubw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_psubw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    psubw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = sub <8 x i16> %a0, %a1
   ret <8 x i16> %2
 }
 
 define i32 @stack_fold_ptest(<2 x i64> %a0, <2 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_ptest
-  ;CHECK:       ptest {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_ptest:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    ptest {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    setb %al
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call i32 @llvm.x86.sse41.ptestc(<2 x i64> %a0, <2 x i64> %a1)
   ret i32 %2
@@ -1134,16 +2332,32 @@ define i32 @stack_fold_ptest(<2 x i64> %a0, <2 x i64> %a1) {
 declare i32 @llvm.x86.sse41.ptestc(<2 x i64>, <2 x i64>) nounwind readnone
 
 define <16 x i8> @stack_fold_punpckhbw(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_punpckhbw
-  ;CHECK:       punpckhbw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_punpckhbw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    punpckhbw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = xmm0[8],mem[8],xmm0[9],mem[9],xmm0[10],mem[10],xmm0[11],mem[11],xmm0[12],mem[12],xmm0[13],mem[13],xmm0[14],mem[14],xmm0[15],mem[15]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <16 x i8> %a0, <16 x i8> %a1, <16 x i32> <i32 8, i32 24, i32 9, i32 25, i32 10, i32 26, i32 11, i32 27, i32 12, i32 28, i32 13, i32 29, i32 14, i32 30, i32 15, i32 31>
   ret <16 x i8> %2
 }
 
 define <4 x i32> @stack_fold_punpckhdq(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_punpckhdq
-  ;CHECK:       punpckhdq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_punpckhdq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    punpckhdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = xmm0[2],mem[2],xmm0[3],mem[3]
+; CHECK-NEXT:    pcmpeqd %xmm1, %xmm1
+; CHECK-NEXT:    psubd %xmm1, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <4 x i32> %a0, <4 x i32> %a1, <4 x i32> <i32 2, i32 6, i32 3, i32 7>
   ; add forces execution domain
@@ -1152,8 +2366,17 @@ define <4 x i32> @stack_fold_punpckhdq(<4 x i32> %a0, <4 x i32> %a1) {
 }
 
 define <2 x i64> @stack_fold_punpckhqdq(<2 x i64> %a0, <2 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_punpckhqdq
-  ;CHECK:       punpckhqdq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_punpckhqdq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    punpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = xmm0[1],mem[1]
+; CHECK-NEXT:    pcmpeqd %xmm1, %xmm1
+; CHECK-NEXT:    psubq %xmm1, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <2 x i64> %a0, <2 x i64> %a1, <2 x i32> <i32 1, i32 3>
   ; add forces execution domain
@@ -1162,24 +2385,47 @@ define <2 x i64> @stack_fold_punpckhqdq(<2 x i64> %a0, <2 x i64> %a1) {
 }
 
 define <8 x i16> @stack_fold_punpckhwd(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_punpckhwd
-  ;CHECK:       punpckhwd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_punpckhwd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <8 x i16> %a0, <8 x i16> %a1, <8 x i32> <i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
   ret <8 x i16> %2
 }
 
 define <16 x i8> @stack_fold_punpcklbw(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_punpcklbw
-  ;CHECK:       punpcklbw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_punpcklbw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    punpcklbw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3],xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <16 x i8> %a0, <16 x i8> %a1, <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23>
   ret <16 x i8> %2
 }
 
 define <4 x i32> @stack_fold_punpckldq(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_punpckldq
-  ;CHECK:       punpckldq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_punpckldq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    punpckldq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
+; CHECK-NEXT:    pcmpeqd %xmm1, %xmm1
+; CHECK-NEXT:    psubd %xmm1, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <4 x i32> %a0, <4 x i32> %a1, <4 x i32> <i32 0, i32 4, i32 1, i32 5>
   ; add forces execution domain
@@ -1188,8 +2434,17 @@ define <4 x i32> @stack_fold_punpckldq(<4 x i32> %a0, <4 x i32> %a1) {
 }
 
 define <2 x i64> @stack_fold_punpcklqdq(<2 x i64> %a0, <2 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_punpcklqdq
-  ;CHECK:       punpcklqdq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_punpcklqdq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    punpcklqdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = xmm0[0],mem[0]
+; CHECK-NEXT:    pcmpeqd %xmm1, %xmm1
+; CHECK-NEXT:    psubq %xmm1, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <2 x i64> %a0, <2 x i64> %a1, <2 x i32> <i32 0, i32 2>
   ; add forces execution domain
@@ -1198,16 +2453,31 @@ define <2 x i64> @stack_fold_punpcklqdq(<2 x i64> %a0, <2 x i64> %a1) {
 }
 
 define <8 x i16> @stack_fold_punpcklwd(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_punpcklwd
-  ;CHECK:       punpcklwd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_punpcklwd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    punpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3]
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <8 x i16> %a0, <8 x i16> %a1, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11>
   ret <8 x i16> %2
 }
 
 define <16 x i8> @stack_fold_pxor(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_pxor
-  ;CHECK:       pxor {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_pxor:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pxor {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    pcmpeqd %xmm1, %xmm1
+; CHECK-NEXT:    psubb %xmm1, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = xor <16 x i8> %a0, %a1
   ; add forces execution domain
diff --git a/llvm/test/CodeGen/X86/stack-folding-lwp.ll b/llvm/test/CodeGen/X86/stack-folding-lwp.ll
index 30b933238..355679d23 100644
--- a/llvm/test/CodeGen/X86/stack-folding-lwp.ll
+++ b/llvm/test/CodeGen/X86/stack-folding-lwp.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -O3 -disable-peephole -mtriple=x86_64-unknown-unknown -mattr=+lwp < %s | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -9,9 +10,48 @@ target triple = "x86_64-unknown-unknown"
 ; relevant registers and check that the reload is correctly folded into the instruction.
 
 define i8 @stack_fold_lwpins_u32(i32 %a0, i32 %a1) {
-; CHECK-LABEL: stack_fold_lwpins_u32
+; CHECK-LABEL: stack_fold_lwpins_u32:
 ; CHECK:       # %bb.0:
-; CHECK:       lwpins $2814, {{-?[0-9]*}}(%rsp), %eax {{.*#+}} 4-byte Folded Reload
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movl %esi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
+; CHECK-NEXT:    lwpins $2814, {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Folded Reload
+; CHECK-NEXT:    # imm = 0xAFE
+; CHECK-NEXT:    setb %al
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call i64 asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = tail call i8 @llvm.x86.lwpins32(i32 %a0, i32 %a1, i32 2814)
   ret i8 %2
@@ -19,9 +59,48 @@ define i8 @stack_fold_lwpins_u32(i32 %a0, i32 %a1) {
 declare i8 @llvm.x86.lwpins32(i32, i32, i32)
 
 define i8 @stack_fold_lwpins_u64(i64 %a0, i32 %a1) {
-; CHECK-LABEL: stack_fold_lwpins_u64
+; CHECK-LABEL: stack_fold_lwpins_u64:
 ; CHECK:       # %bb.0:
-; CHECK:       lwpins $2814, {{-?[0-9]*}}(%rsp), %rax {{.*#+}} 4-byte Folded Reload
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movl %esi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; CHECK-NEXT:    lwpins $2814, {{[-0-9]+}}(%r{{[sb]}}p), %rax # 4-byte Folded Reload
+; CHECK-NEXT:    # imm = 0xAFE
+; CHECK-NEXT:    setb %al
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call i64 asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = tail call i8 @llvm.x86.lwpins64(i64 %a0, i32 %a1, i32 2814)
   ret i8 %2
@@ -29,9 +108,47 @@ define i8 @stack_fold_lwpins_u64(i64 %a0, i32 %a1) {
 declare i8 @llvm.x86.lwpins64(i64, i32, i32)
 
 define void @stack_fold_lwpval_u32(i32 %a0, i32 %a1) {
-; CHECK-LABEL: stack_fold_lwpval_u32
+; CHECK-LABEL: stack_fold_lwpval_u32:
 ; CHECK:       # %bb.0:
-; CHECK:       lwpval $2814, {{-?[0-9]*}}(%rsp), %eax {{.*#+}} 4-byte Folded Reload
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movl %esi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
+; CHECK-NEXT:    lwpval $2814, {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Folded Reload
+; CHECK-NEXT:    # imm = 0xAFE
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call i64 asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   tail call void @llvm.x86.lwpval32(i32 %a0, i32 %a1, i32 2814)
   ret void
@@ -39,9 +156,47 @@ define void @stack_fold_lwpval_u32(i32 %a0, i32 %a1) {
 declare void @llvm.x86.lwpval32(i32, i32, i32)
 
 define void @stack_fold_lwpval_u64(i64 %a0, i32 %a1) {
-; CHECK-LABEL: stack_fold_lwpval_u64
+; CHECK-LABEL: stack_fold_lwpval_u64:
 ; CHECK:       # %bb.0:
-; CHECK:       lwpval $2814, {{-?[0-9]*}}(%rsp), %rax {{.*#+}} 4-byte Folded Reload
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movl %esi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; CHECK-NEXT:    lwpval $2814, {{[-0-9]+}}(%r{{[sb]}}p), %rax # 4-byte Folded Reload
+; CHECK-NEXT:    # imm = 0xAFE
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call i64 asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   tail call void @llvm.x86.lwpval64(i64 %a0, i32 %a1, i32 2814)
   ret void
diff --git a/llvm/test/CodeGen/X86/stack-folding-mmx.ll b/llvm/test/CodeGen/X86/stack-folding-mmx.ll
index 3b1a49567..8b6c2e687 100644
--- a/llvm/test/CodeGen/X86/stack-folding-mmx.ll
+++ b/llvm/test/CodeGen/X86/stack-folding-mmx.ll
@@ -1,8 +1,16 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+mmx,+ssse3 | FileCheck %s
 
 define x86_mmx @stack_fold_cvtpd2pi(<2 x double> %a0) {
-  ;CHECK-LABEL: stack_fold_cvtpd2pi
-  ;CHECK:       cvtpd2pi {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtpd2pi:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    cvtpd2pi {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 16-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call x86_mmx @llvm.x86.sse.cvtpd2pi(<2 x double> %a0) nounwind readnone
   ret x86_mmx %2
@@ -10,8 +18,14 @@ define x86_mmx @stack_fold_cvtpd2pi(<2 x double> %a0) {
 declare x86_mmx @llvm.x86.sse.cvtpd2pi(<2 x double>) nounwind readnone
 
 define <2 x double> @stack_fold_cvtpi2pd(x86_mmx %a0) {
-  ;CHECK-LABEL: stack_fold_cvtpi2pd
-  ;CHECK:       cvtpi2pd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtpi2pd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    cvtpi2pd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm1},~{mm1},~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call <2 x double> @llvm.x86.sse.cvtpi2pd(x86_mmx %a0) nounwind readnone
   ret <2 x double> %2
@@ -19,8 +33,14 @@ define <2 x double> @stack_fold_cvtpi2pd(x86_mmx %a0) {
 declare <2 x double> @llvm.x86.sse.cvtpi2pd(x86_mmx) nounwind readnone
 
 define <4 x float> @stack_fold_cvtpi2ps(<4 x float> %a0, x86_mmx %a1) {
-  ;CHECK-LABEL: stack_fold_cvtpi2ps
-  ;CHECK:       cvtpi2ps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtpi2ps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    cvtpi2ps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm1},~{mm1},~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call <4 x float> @llvm.x86.sse.cvtpi2ps(<4 x float> %a0, x86_mmx %a1) nounwind readnone
   ret <4 x float> %2
@@ -28,8 +48,15 @@ define <4 x float> @stack_fold_cvtpi2ps(<4 x float> %a0, x86_mmx %a1) {
 declare <4 x float> @llvm.x86.sse.cvtpi2ps(<4 x float>, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_cvtps2pi(<4 x float> %a0) {
-  ;CHECK-LABEL: stack_fold_cvtps2pi
-  ;CHECK:       cvtps2pi {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvtps2pi:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    cvtps2pi {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 16-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call x86_mmx @llvm.x86.sse.cvtps2pi(<4 x float> %a0) nounwind readnone
   ret x86_mmx %2
@@ -37,8 +64,15 @@ define x86_mmx @stack_fold_cvtps2pi(<4 x float> %a0) {
 declare x86_mmx @llvm.x86.sse.cvtps2pi(<4 x float>) nounwind readnone
 
 define x86_mmx @stack_fold_cvttpd2pi(<2 x double> %a0) {
-  ;CHECK-LABEL: stack_fold_cvttpd2pi
-  ;CHECK:       cvttpd2pi {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvttpd2pi:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    cvttpd2pi {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 16-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call x86_mmx @llvm.x86.sse.cvttpd2pi(<2 x double> %a0) nounwind readnone
   ret x86_mmx %2
@@ -46,8 +80,15 @@ define x86_mmx @stack_fold_cvttpd2pi(<2 x double> %a0) {
 declare x86_mmx @llvm.x86.sse.cvttpd2pi(<2 x double>) nounwind readnone
 
 define x86_mmx @stack_fold_cvttps2pi(<4 x float> %a0) {
-  ;CHECK-LABEL: stack_fold_cvttps2pi
-  ;CHECK:       cvttps2pi {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_cvttps2pi:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    cvttps2pi {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 16-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call x86_mmx @llvm.x86.sse.cvttps2pi(<4 x float> %a0) nounwind readnone
   ret x86_mmx %2
@@ -55,13 +96,78 @@ define x86_mmx @stack_fold_cvttps2pi(<4 x float> %a0) {
 declare x86_mmx @llvm.x86.sse.cvttps2pi(<4 x float>) nounwind readnone
 
 ; TODO stack_fold_movd_load
-; TODO stack_fold_movd_store
+
+; padd forces execution on mmx
+define i32 @stack_fold_movd_store(x86_mmx %a0) nounwind {
+; CHECK-LABEL: stack_fold_movd_store:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    paddb %mm0, %mm0
+; CHECK-NEXT:    movd %mm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    retq
+  %1 = call x86_mmx @llvm.x86.mmx.padd.b(x86_mmx %a0, x86_mmx %a0)
+  %2 = bitcast x86_mmx %1 to <2 x i32>
+  %3 = extractelement <2 x i32> %2, i32 0
+  %4 = tail call i64 asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
+  ret i32 %3
+}
+
 ; TODO stack_fold_movq_load
-; TODO stack_fold_movq_store
+
+; padd forces execution on mmx
+define i64 @stack_fold_movq_store(x86_mmx %a0) nounwind {
+; CHECK-LABEL: stack_fold_movq_store:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    paddb %mm0, %mm0
+; CHECK-NEXT:    movq %mm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    retq
+  %1 = call x86_mmx @llvm.x86.mmx.padd.b(x86_mmx %a0, x86_mmx %a0)
+  %2 = bitcast x86_mmx %1 to i64
+  %3 = tail call i64 asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
+  ret i64 %2
+}
 
 define x86_mmx @stack_fold_pabsb(x86_mmx %a0) {
-  ;CHECK-LABEL: stack_fold_pabsb
-  ;CHECK:       pabsb {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_pabsb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pabsb {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm1},~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.ssse3.pabs.b(x86_mmx %a0) nounwind readnone
   ret x86_mmx %2
@@ -69,8 +175,15 @@ define x86_mmx @stack_fold_pabsb(x86_mmx %a0) {
 declare x86_mmx @llvm.x86.ssse3.pabs.b(x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_pabsd(x86_mmx %a0) {
-  ;CHECK-LABEL: stack_fold_pabsd
-  ;CHECK:       pabsd {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_pabsd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pabsd {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm1},~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.ssse3.pabs.d(x86_mmx %a0) nounwind readnone
   ret x86_mmx %2
@@ -78,8 +191,15 @@ define x86_mmx @stack_fold_pabsd(x86_mmx %a0) {
 declare x86_mmx @llvm.x86.ssse3.pabs.d(x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_pabsw(x86_mmx %a0) {
-  ;CHECK-LABEL: stack_fold_pabsw
-  ;CHECK:       pabsw {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_pabsw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pabsw {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm1},~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.ssse3.pabs.w(x86_mmx %a0) nounwind readnone
   ret x86_mmx %2
@@ -87,8 +207,15 @@ define x86_mmx @stack_fold_pabsw(x86_mmx %a0) {
 declare x86_mmx @llvm.x86.ssse3.pabs.w(x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_packssdw(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_packssdw
-  ;CHECK:       packssdw {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_packssdw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    packssdw {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.mmx.packssdw(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -96,8 +223,15 @@ define x86_mmx @stack_fold_packssdw(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.mmx.packssdw(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_packsswb(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_packsswb
-  ;CHECK:       packsswb {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_packsswb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    packsswb {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.mmx.packsswb(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -105,8 +239,15 @@ define x86_mmx @stack_fold_packsswb(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.mmx.packsswb(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_packuswb(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_packuswb
-  ;CHECK:       packuswb {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_packuswb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    packuswb {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.mmx.packuswb(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -114,8 +255,15 @@ define x86_mmx @stack_fold_packuswb(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.mmx.packuswb(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_paddb(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_paddb
-  ;CHECK:       paddb {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    paddb {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.mmx.padd.b(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -123,8 +271,15 @@ define x86_mmx @stack_fold_paddb(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.mmx.padd.b(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_paddd(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_paddd
-  ;CHECK:       paddd {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    paddd {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.mmx.padd.d(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -132,8 +287,15 @@ define x86_mmx @stack_fold_paddd(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.mmx.padd.d(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_paddq(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_paddq
-  ;CHECK:       paddq {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    paddq {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.mmx.padd.q(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -141,8 +303,15 @@ define x86_mmx @stack_fold_paddq(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.mmx.padd.q(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_paddsb(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_paddsb
-  ;CHECK:       paddsb {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddsb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    paddsb {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.mmx.padds.b(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -150,8 +319,15 @@ define x86_mmx @stack_fold_paddsb(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.mmx.padds.b(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_paddsw(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_paddsw
-  ;CHECK:       paddsw {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddsw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    paddsw {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.mmx.padds.w(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -159,8 +335,15 @@ define x86_mmx @stack_fold_paddsw(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.mmx.padds.w(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_paddusb(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_paddusb
-  ;CHECK:       paddusb {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddusb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    paddusb {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.mmx.paddus.b(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -168,8 +351,15 @@ define x86_mmx @stack_fold_paddusb(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.mmx.paddus.b(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_paddusw(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_paddusw
-  ;CHECK:       paddusw {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddusw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    paddusw {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.mmx.paddus.w(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -177,8 +367,15 @@ define x86_mmx @stack_fold_paddusw(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.mmx.paddus.w(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_paddw(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_paddw
-  ;CHECK:       paddw {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_paddw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    paddw {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.mmx.padd.w(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -186,8 +383,15 @@ define x86_mmx @stack_fold_paddw(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.mmx.padd.w(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_palignr(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_palignr
-  ;CHECK:       palignr $1, {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_palignr:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    palignr $1, {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.mmx.palignr.b(x86_mmx %a, x86_mmx %b, i8 1) nounwind readnone
   ret x86_mmx %2
@@ -195,8 +399,15 @@ define x86_mmx @stack_fold_palignr(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.mmx.palignr.b(x86_mmx, x86_mmx, i8) nounwind readnone
 
 define x86_mmx @stack_fold_pand(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_pand
-  ;CHECK:       pand {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_pand:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pand {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.mmx.pand(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -204,8 +415,15 @@ define x86_mmx @stack_fold_pand(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.mmx.pand(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_pandn(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_pandn
-  ;CHECK:       pandn {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_pandn:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pandn {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.mmx.pandn(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -213,8 +431,15 @@ define x86_mmx @stack_fold_pandn(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.mmx.pandn(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_pavgb(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_pavgb
-  ;CHECK:       pavgb {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_pavgb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pavgb {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.mmx.pavg.b(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -222,8 +447,15 @@ define x86_mmx @stack_fold_pavgb(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.mmx.pavg.b(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_pavgw(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_pavgw
-  ;CHECK:       pavgw {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_pavgw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pavgw {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.mmx.pavg.w(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -231,8 +463,15 @@ define x86_mmx @stack_fold_pavgw(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.mmx.pavg.w(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_pcmpeqb(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_pcmpeqb
-  ;CHECK:       pcmpeqb {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_pcmpeqb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pcmpeqb {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.mmx.pcmpeq.b(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -240,8 +479,15 @@ define x86_mmx @stack_fold_pcmpeqb(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.mmx.pcmpeq.b(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_pcmpeqd(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_pcmpeqd
-  ;CHECK:       pcmpeqd {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_pcmpeqd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pcmpeqd {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.mmx.pcmpeq.d(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -249,8 +495,15 @@ define x86_mmx @stack_fold_pcmpeqd(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.mmx.pcmpeq.d(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_pcmpeqw(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_pcmpeqw
-  ;CHECK:       pcmpeqw {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_pcmpeqw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pcmpeqw {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.mmx.pcmpeq.w(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -258,8 +511,15 @@ define x86_mmx @stack_fold_pcmpeqw(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.mmx.pcmpeq.w(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_pcmpgtb(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_pcmpgtb
-  ;CHECK:       pcmpgtb {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_pcmpgtb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pcmpgtb {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.mmx.pcmpgt.b(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -267,8 +527,15 @@ define x86_mmx @stack_fold_pcmpgtb(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.mmx.pcmpgt.b(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_pcmpgtd(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_pcmpgtd
-  ;CHECK:       pcmpgtd {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_pcmpgtd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pcmpgtd {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.mmx.pcmpgt.d(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -276,8 +543,15 @@ define x86_mmx @stack_fold_pcmpgtd(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.mmx.pcmpgt.d(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_pcmpgtw(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_pcmpgtw
-  ;CHECK:       pcmpgtw {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_pcmpgtw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pcmpgtw {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.mmx.pcmpgt.w(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -285,8 +559,15 @@ define x86_mmx @stack_fold_pcmpgtw(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.mmx.pcmpgt.w(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_phaddd(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_phaddd
-  ;CHECK:       phaddd {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_phaddd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    phaddd {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.ssse3.phadd.d(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -294,8 +575,15 @@ define x86_mmx @stack_fold_phaddd(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.ssse3.phadd.d(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_phaddsw(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_phaddsw
-  ;CHECK:       phaddsw {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_phaddsw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    phaddsw {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.ssse3.phadd.sw(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -303,8 +591,15 @@ define x86_mmx @stack_fold_phaddsw(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.ssse3.phadd.sw(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_phaddw(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_phaddw
-  ;CHECK:       phaddw {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_phaddw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    phaddw {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.ssse3.phadd.w(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -312,8 +607,15 @@ define x86_mmx @stack_fold_phaddw(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.ssse3.phadd.w(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_phsubd(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_phsubd
-  ;CHECK:       phsubd {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_phsubd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    phsubd {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.ssse3.phsub.d(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -321,8 +623,15 @@ define x86_mmx @stack_fold_phsubd(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.ssse3.phsub.d(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_phsubsw(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_phsubsw
-  ;CHECK:       phsubsw {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_phsubsw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    phsubsw {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.ssse3.phsub.sw(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -330,8 +639,15 @@ define x86_mmx @stack_fold_phsubsw(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.ssse3.phsub.sw(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_phsubw(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_phsubw
-  ;CHECK:       phsubw {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_phsubw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    phsubw {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.ssse3.phsub.w(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -341,8 +657,15 @@ declare x86_mmx @llvm.x86.ssse3.phsub.w(x86_mmx, x86_mmx) nounwind readnone
 ; TODO stack_fold_pinsrw
 
 define x86_mmx @stack_fold_pmaddubsw(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_pmaddubsw
-  ;CHECK:       pmaddubsw {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaddubsw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pmaddubsw {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.ssse3.pmadd.ub.sw(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -350,8 +673,15 @@ define x86_mmx @stack_fold_pmaddubsw(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.ssse3.pmadd.ub.sw(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_pmaddwd(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_pmaddwd
-  ;CHECK:       pmaddwd {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaddwd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pmaddwd {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.mmx.pmadd.wd(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -359,8 +689,15 @@ define x86_mmx @stack_fold_pmaddwd(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.mmx.pmadd.wd(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_pmaxsw(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_pmaxsw
-  ;CHECK:       pmaxsw {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxsw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pmaxsw {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.mmx.pmaxs.w(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -368,8 +705,15 @@ define x86_mmx @stack_fold_pmaxsw(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.mmx.pmaxs.w(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_pmaxub(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_pmaxub
-  ;CHECK:       pmaxub {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmaxub:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pmaxub {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.mmx.pmaxu.b(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -377,8 +721,15 @@ define x86_mmx @stack_fold_pmaxub(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.mmx.pmaxu.b(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_pminsw(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_pminsw
-  ;CHECK:       pminsw {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminsw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pminsw {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.mmx.pmins.w(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -386,8 +737,15 @@ define x86_mmx @stack_fold_pminsw(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.mmx.pmins.w(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_pminub(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_pminub
-  ;CHECK:       pminub {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_pminub:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pminub {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.mmx.pminu.b(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -395,8 +753,15 @@ define x86_mmx @stack_fold_pminub(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.mmx.pminu.b(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_pmulhrsw(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_pmulhrsw
-  ;CHECK:       pmulhrsw {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmulhrsw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pmulhrsw {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.ssse3.pmul.hr.sw(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -404,8 +769,15 @@ define x86_mmx @stack_fold_pmulhrsw(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.ssse3.pmul.hr.sw(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_pmulhuw(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_pmulhuw
-  ;CHECK:       pmulhuw {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmulhuw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pmulhuw {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.mmx.pmulhu.w(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -413,8 +785,15 @@ define x86_mmx @stack_fold_pmulhuw(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.mmx.pmulhu.w(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_pmulhw(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_pmulhw
-  ;CHECK:       pmulhw {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmulhw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pmulhw {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.mmx.pmulh.w(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -422,8 +801,15 @@ define x86_mmx @stack_fold_pmulhw(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.mmx.pmulh.w(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_pmullw(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_pmullw
-  ;CHECK:       pmullw {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmullw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pmullw {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.mmx.pmull.w(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -431,8 +817,15 @@ define x86_mmx @stack_fold_pmullw(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.mmx.pmull.w(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_pmuludq(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_pmuludq
-  ;CHECK:       pmuludq {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_pmuludq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pmuludq {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.mmx.pmulu.dq(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -440,8 +833,15 @@ define x86_mmx @stack_fold_pmuludq(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.mmx.pmulu.dq(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_por(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_por
-  ;CHECK:       por {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_por:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    por {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.mmx.por(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -449,8 +849,15 @@ define x86_mmx @stack_fold_por(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.mmx.por(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_psadbw(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_psadbw
-  ;CHECK:       psadbw {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_psadbw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    psadbw {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.mmx.psad.bw(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -458,8 +865,17 @@ define x86_mmx @stack_fold_psadbw(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.mmx.psad.bw(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_pshufb(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_pshufb
-  ;CHECK:       pshufb {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_pshufb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    movq %mm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Reload
+; CHECK-NEXT:    pshufb {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm1},~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.ssse3.pshuf.b(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -467,8 +883,16 @@ define x86_mmx @stack_fold_pshufb(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.ssse3.pshuf.b(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_pshufw(x86_mmx %a) {
-  ;CHECK-LABEL: stack_fold_pshufw
-  ;CHECK:       pshufw $1, {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_pshufw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pshufw $1, {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    # mm0 = mem[1,0,0,0]
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm1},~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.sse.pshuf.w(x86_mmx %a, i8 1) nounwind readnone
   ret x86_mmx %2
@@ -476,8 +900,15 @@ define x86_mmx @stack_fold_pshufw(x86_mmx %a) {
 declare x86_mmx @llvm.x86.sse.pshuf.w(x86_mmx, i8) nounwind readnone
 
 define x86_mmx @stack_fold_psignb(x86_mmx %a0, x86_mmx %a1) {
-  ;CHECK-LABEL: stack_fold_psignb
-  ;CHECK:       psignb {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_psignb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    psignb {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.ssse3.psign.b(x86_mmx %a0, x86_mmx %a1) nounwind readnone
   ret x86_mmx %2
@@ -485,8 +916,15 @@ define x86_mmx @stack_fold_psignb(x86_mmx %a0, x86_mmx %a1) {
 declare x86_mmx @llvm.x86.ssse3.psign.b(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_psignd(x86_mmx %a0, x86_mmx %a1) {
-  ;CHECK-LABEL: stack_fold_psignd
-  ;CHECK:       psignd {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_psignd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    psignd {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.ssse3.psign.d(x86_mmx %a0, x86_mmx %a1) nounwind readnone
   ret x86_mmx %2
@@ -494,8 +932,15 @@ define x86_mmx @stack_fold_psignd(x86_mmx %a0, x86_mmx %a1) {
 declare x86_mmx @llvm.x86.ssse3.psign.d(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_psignw(x86_mmx %a0, x86_mmx %a1) {
-  ;CHECK-LABEL: stack_fold_psignw
-  ;CHECK:       psignw {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_psignw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    psignw {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.ssse3.psign.w(x86_mmx %a0, x86_mmx %a1) nounwind readnone
   ret x86_mmx %2
@@ -503,8 +948,15 @@ define x86_mmx @stack_fold_psignw(x86_mmx %a0, x86_mmx %a1) {
 declare x86_mmx @llvm.x86.ssse3.psign.w(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_pslld(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_pslld
-  ;CHECK:       pslld {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_pslld:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pslld {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.mmx.psll.d(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -512,8 +964,15 @@ define x86_mmx @stack_fold_pslld(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.mmx.psll.d(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_psllq(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_psllq
-  ;CHECK:       psllq {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_psllq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    psllq {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.mmx.psll.q(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -521,8 +980,15 @@ define x86_mmx @stack_fold_psllq(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.mmx.psll.q(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_psllw(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_psllw
-  ;CHECK:       psllw {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_psllw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    psllw {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.mmx.psll.w(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -530,8 +996,15 @@ define x86_mmx @stack_fold_psllw(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.mmx.psll.w(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_psrad(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_psrad
-  ;CHECK:       psrad {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_psrad:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    psrad {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.mmx.psra.d(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -539,8 +1012,15 @@ define x86_mmx @stack_fold_psrad(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.mmx.psra.d(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_psraw(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_psraw
-  ;CHECK:       psraw {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_psraw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    psraw {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.mmx.psra.w(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -548,8 +1028,15 @@ define x86_mmx @stack_fold_psraw(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.mmx.psra.w(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_psrld(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_psrld
-  ;CHECK:       psrld {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_psrld:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    psrld {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.mmx.psrl.d(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -557,8 +1044,15 @@ define x86_mmx @stack_fold_psrld(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.mmx.psrl.d(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_psrlq(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_psrlq
-  ;CHECK:       psrlq {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_psrlq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    psrlq {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.mmx.psrl.q(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -566,8 +1060,15 @@ define x86_mmx @stack_fold_psrlq(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.mmx.psrl.q(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_psrlw(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_psrlw
-  ;CHECK:       psrlw {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_psrlw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    psrlw {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.mmx.psrl.w(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -575,8 +1076,15 @@ define x86_mmx @stack_fold_psrlw(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.mmx.psrl.w(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_psubb(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_psubb
-  ;CHECK:       psubb {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_psubb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    psubb {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.mmx.psub.b(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -584,8 +1092,15 @@ define x86_mmx @stack_fold_psubb(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.mmx.psub.b(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_psubd(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_psubd
-  ;CHECK:       psubd {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_psubd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    psubd {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.mmx.psub.d(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -593,8 +1108,15 @@ define x86_mmx @stack_fold_psubd(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.mmx.psub.d(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_psubq(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_psubq
-  ;CHECK:       psubq {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_psubq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    psubq {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.mmx.psub.q(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -602,8 +1124,15 @@ define x86_mmx @stack_fold_psubq(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.mmx.psub.q(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_psubsb(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_psubsb
-  ;CHECK:       psubsb {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_psubsb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    psubsb {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.mmx.psubs.b(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -611,8 +1140,15 @@ define x86_mmx @stack_fold_psubsb(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.mmx.psubs.b(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_psubsw(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_psubsw
-  ;CHECK:       psubsw {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_psubsw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    psubsw {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.mmx.psubs.w(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -620,8 +1156,15 @@ define x86_mmx @stack_fold_psubsw(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.mmx.psubs.w(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_psubusb(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_psubusb
-  ;CHECK:       psubusb {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_psubusb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    psubusb {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.mmx.psubus.b(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -629,8 +1172,15 @@ define x86_mmx @stack_fold_psubusb(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.mmx.psubus.b(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_psubusw(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_psubusw
-  ;CHECK:       psubusw {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_psubusw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    psubusw {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.mmx.psubus.w(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -638,8 +1188,15 @@ define x86_mmx @stack_fold_psubusw(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.mmx.psubus.w(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_psubw(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_psubw
-  ;CHECK:       psubw {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_psubw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    psubw {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.mmx.psub.w(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -647,8 +1204,16 @@ define x86_mmx @stack_fold_psubw(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.mmx.psub.w(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_punpckhbw(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_punpckhbw
-  ;CHECK:       punpckhbw {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_punpckhbw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    punpckhbw {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    # mm0 = mm0[4],mem[4],mm0[5],mem[5],mm0[6],mem[6],mm0[7],mem[7]
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.mmx.punpckhbw(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -656,8 +1221,16 @@ define x86_mmx @stack_fold_punpckhbw(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.mmx.punpckhbw(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_punpckhdq(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_punpckhdq
-  ;CHECK:       punpckhdq {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_punpckhdq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    punpckhdq {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    # mm0 = mm0[1],mem[1]
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.mmx.punpckhdq(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -665,8 +1238,16 @@ define x86_mmx @stack_fold_punpckhdq(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.mmx.punpckhdq(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_punpckhwd(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_punpckhwd
-  ;CHECK:       punpckhwd {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_punpckhwd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    # mm0 = mm0[2],mem[2],mm0[3],mem[3]
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.mmx.punpckhwd(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -674,8 +1255,16 @@ define x86_mmx @stack_fold_punpckhwd(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.mmx.punpckhwd(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_punpcklbw(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_punpcklbw
-  ;CHECK:       punpcklbw {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_punpcklbw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    punpcklbw {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    # mm0 = mm0[0],mem[0],mm0[1],mem[1],mm0[2],mem[2],mm0[3],mem[3]
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.mmx.punpcklbw(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -683,8 +1272,16 @@ define x86_mmx @stack_fold_punpcklbw(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.mmx.punpcklbw(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_punpckldq(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_punpckldq
-  ;CHECK:       punpckldq {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_punpckldq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    punpckldq {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    # mm0 = mm0[0],mem[0]
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.mmx.punpckldq(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -692,8 +1289,16 @@ define x86_mmx @stack_fold_punpckldq(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.mmx.punpckldq(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_punpcklwd(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_punpcklwd
-  ;CHECK:       punpcklwd {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_punpcklwd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    punpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    # mm0 = mm0[0],mem[0],mm0[1],mem[1]
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.mmx.punpcklwd(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
@@ -701,8 +1306,15 @@ define x86_mmx @stack_fold_punpcklwd(x86_mmx %a, x86_mmx %b) {
 declare x86_mmx @llvm.x86.mmx.punpcklwd(x86_mmx, x86_mmx) nounwind readnone
 
 define x86_mmx @stack_fold_pxor(x86_mmx %a, x86_mmx %b) {
-  ;CHECK-LABEL: stack_fold_pxor
-  ;CHECK:       pxor {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_pxor:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %mm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    pxor {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
+; CHECK-NEXT:    movq2dq %mm0, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"()
   %2 = call x86_mmx @llvm.x86.mmx.pxor(x86_mmx %a, x86_mmx %b) nounwind readnone
   ret x86_mmx %2
diff --git a/llvm/test/CodeGen/X86/stack-folding-sha.ll b/llvm/test/CodeGen/X86/stack-folding-sha.ll
index 768c8a0f5..43333a39a 100644
--- a/llvm/test/CodeGen/X86/stack-folding-sha.ll
+++ b/llvm/test/CodeGen/X86/stack-folding-sha.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -O3 -disable-peephole -mtriple=x86_64-unknown-unknown -mattr=+sha < %s | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -9,8 +10,16 @@ target triple = "x86_64-unknown-unknown"
 ; relevant registers and check that the reload is correctly folded into the instruction.
 
 define <4 x i32> @stack_fold_sha1msg1(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_sha1msg1
-  ;CHECK:       sha1msg1 {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_sha1msg1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    sha1msg1 {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = tail call <4 x i32> @llvm.x86.sha1msg1(<4 x i32> %a0, <4 x i32> %a1)
   ret <4 x i32> %2
@@ -18,8 +27,16 @@ define <4 x i32> @stack_fold_sha1msg1(<4 x i32> %a0, <4 x i32> %a1) {
 declare <4 x i32> @llvm.x86.sha1msg1(<4 x i32>, <4 x i32>) nounwind readnone
 
 define <4 x i32> @stack_fold_sha1msg2(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_sha1msg2
-  ;CHECK:       sha1msg2 {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_sha1msg2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    sha1msg2 {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = tail call <4 x i32> @llvm.x86.sha1msg2(<4 x i32> %a0, <4 x i32> %a1)
   ret <4 x i32> %2
@@ -27,8 +44,16 @@ define <4 x i32> @stack_fold_sha1msg2(<4 x i32> %a0, <4 x i32> %a1) {
 declare <4 x i32> @llvm.x86.sha1msg2(<4 x i32>, <4 x i32>) nounwind readnone
 
 define <4 x i32> @stack_fold_sha1nexte(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_sha1nexte
-  ;CHECK:       sha1nexte {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_sha1nexte:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    sha1nexte {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = tail call <4 x i32> @llvm.x86.sha1nexte(<4 x i32> %a0, <4 x i32> %a1)
   ret <4 x i32> %2
@@ -36,8 +61,16 @@ define <4 x i32> @stack_fold_sha1nexte(<4 x i32> %a0, <4 x i32> %a1) {
 declare <4 x i32> @llvm.x86.sha1nexte(<4 x i32>, <4 x i32>) nounwind readnone
 
 define <4 x i32> @stack_fold_sha1rnds4(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_sha1rnds4
-  ;CHECK:       sha1rnds4 $3, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_sha1rnds4:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    sha1rnds4 $3, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = tail call <4 x i32> @llvm.x86.sha1rnds4(<4 x i32> %a0, <4 x i32> %a1, i8 3)
   ret <4 x i32> %2
@@ -45,8 +78,16 @@ define <4 x i32> @stack_fold_sha1rnds4(<4 x i32> %a0, <4 x i32> %a1) {
 declare <4 x i32> @llvm.x86.sha1rnds4(<4 x i32>, <4 x i32>, i8) nounwind readnone
 
 define <4 x i32> @stack_fold_sha256msg1(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_sha256msg1
-  ;CHECK:       sha256msg1 {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_sha256msg1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    sha256msg1 {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = tail call <4 x i32> @llvm.x86.sha256msg1(<4 x i32> %a0, <4 x i32> %a1)
   ret <4 x i32> %2
@@ -54,8 +95,16 @@ define <4 x i32> @stack_fold_sha256msg1(<4 x i32> %a0, <4 x i32> %a1) {
 declare <4 x i32> @llvm.x86.sha256msg1(<4 x i32>, <4 x i32>) nounwind readnone
 
 define <4 x i32> @stack_fold_sha256msg2(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_sha256msg2
-  ;CHECK:       sha256msg2 {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_sha256msg2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    sha256msg2 {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = tail call <4 x i32> @llvm.x86.sha256msg2(<4 x i32> %a0, <4 x i32> %a1)
   ret <4 x i32> %2
@@ -63,8 +112,19 @@ define <4 x i32> @stack_fold_sha256msg2(<4 x i32> %a0, <4 x i32> %a1) {
 declare <4 x i32> @llvm.x86.sha256msg2(<4 x i32>, <4 x i32>) nounwind readnone
 
 define <4 x i32> @stack_fold_sha256rnds2(<4 x i32> %a0, <4 x i32> %a1, <4 x i32> %a2) {
-  ;CHECK-LABEL: stack_fold_sha256rnds2
-  ;CHECK:       sha256rnds2 {{%xmm[0-9][0-9]*}}, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_sha256rnds2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    sha256rnds2 %xmm0, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
+; CHECK-NEXT:    movaps %xmm1, %xmm0
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = tail call <4 x i32> @llvm.x86.sha256rnds2(<4 x i32> %a0, <4 x i32> %a1, <4 x i32> %a2)
   ret <4 x i32> %2
diff --git a/llvm/test/CodeGen/X86/stack-folding-tbm.ll b/llvm/test/CodeGen/X86/stack-folding-tbm.ll
index c5fda04e7..c44786658 100644
--- a/llvm/test/CodeGen/X86/stack-folding-tbm.ll
+++ b/llvm/test/CodeGen/X86/stack-folding-tbm.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -O3 -disable-peephole -mtriple=x86_64-unknown-unknown -mattr=+bmi,+tbm < %s | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -9,9 +10,45 @@ target triple = "x86_64-unknown-unknown"
 ; relevant registers and check that the reload is correctly folded into the instruction.
 
 define i32 @stack_fold_bextri_u32(i32 %a0) {
-  ;CHECK-LABEL: stack_fold_bextri_u32
-  ;CHECK:       # %bb.0:
-  ;CHECK:       bextrl $3841, {{-?[0-9]*}}(%rsp), %eax {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_bextri_u32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    bextrl $3841, {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Folded Reload
+; CHECK-NEXT:    # imm = 0xF01
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call i64 asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = tail call i32 @llvm.x86.tbm.bextri.u32(i32 %a0, i32 3841)
   ret i32 %2
@@ -19,9 +56,45 @@ define i32 @stack_fold_bextri_u32(i32 %a0) {
 declare i32 @llvm.x86.tbm.bextri.u32(i32, i32)
 
 define i64 @stack_fold_bextri_u64(i64 %a0) {
-  ;CHECK-LABEL: stack_fold_bextri_u64
-  ;CHECK:       # %bb.0:
-  ;CHECK:       bextrq $3841, {{-?[0-9]*}}(%rsp), %rax {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_bextri_u64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    bextrq $3841, {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Folded Reload
+; CHECK-NEXT:    # imm = 0xF01
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call i64 asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = tail call i64 @llvm.x86.tbm.bextri.u64(i64 %a0, i64 3841)
   ret i64 %2
@@ -29,8 +102,44 @@ define i64 @stack_fold_bextri_u64(i64 %a0) {
 declare i64 @llvm.x86.tbm.bextri.u64(i64, i64)
 
 define i32 @stack_fold_blcfill_u32(i32 %a0) {
-  ;CHECK-LABEL: stack_fold_blcfill_u32
-  ;CHECK:       blcfilll {{-?[0-9]*}}(%rsp), %eax {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_blcfill_u32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    blcfilll {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Folded Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call i64 asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = add i32 %a0, 1
   %3 = and i32 %a0, %2
@@ -38,8 +147,44 @@ define i32 @stack_fold_blcfill_u32(i32 %a0) {
 }
 
 define i64 @stack_fold_blcfill_u64(i64 %a0) {
-  ;CHECK-LABEL: stack_fold_blcfill_u64
-  ;CHECK:       blcfillq {{-?[0-9]*}}(%rsp), %rax {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_blcfill_u64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    blcfillq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Folded Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call i64 asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = add i64 %a0, 1
   %3 = and i64 %a0, %2
@@ -47,8 +192,44 @@ define i64 @stack_fold_blcfill_u64(i64 %a0) {
 }
 
 define i32 @stack_fold_blci_u32(i32 %a0) {
-  ;CHECK-LABEL: stack_fold_blci_u32
-  ;CHECK:       blcil {{-?[0-9]*}}(%rsp), %eax {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_blci_u32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    blcil {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Folded Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call i64 asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = add i32 %a0, 1
   %3 = xor i32 %2, -1
@@ -57,8 +238,44 @@ define i32 @stack_fold_blci_u32(i32 %a0) {
 }
 
 define i64 @stack_fold_blci_u64(i64 %a0) {
-  ;CHECK-LABEL: stack_fold_blci_u64
-  ;CHECK:       blciq {{-?[0-9]*}}(%rsp), %rax {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_blci_u64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    blciq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Folded Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call i64 asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = add i64 %a0, 1
   %3 = xor i64 %2, -1
@@ -67,8 +284,44 @@ define i64 @stack_fold_blci_u64(i64 %a0) {
 }
 
 define i32 @stack_fold_blcic_u32(i32 %a0) {
-  ;CHECK-LABEL: stack_fold_blcic_u32
-  ;CHECK:       blcicl {{-?[0-9]*}}(%rsp), %eax {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_blcic_u32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    blcicl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Folded Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call i64 asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = add i32 %a0, 1
   %3 = xor i32 %a0, -1
@@ -77,8 +330,44 @@ define i32 @stack_fold_blcic_u32(i32 %a0) {
 }
 
 define i64 @stack_fold_blcic_u64(i64 %a0) {
-  ;CHECK-LABEL: stack_fold_blcic_u64
-  ;CHECK:       blcicq {{-?[0-9]*}}(%rsp), %rax {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_blcic_u64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    blcicq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Folded Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call i64 asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = add i64 %a0, 1
   %3 = xor i64 %a0, -1
@@ -87,8 +376,44 @@ define i64 @stack_fold_blcic_u64(i64 %a0) {
 }
 
 define i32 @stack_fold_blcmsk_u32(i32 %a0) {
-  ;CHECK-LABEL: stack_fold_blcmsk_u32
-  ;CHECK:       blcmskl {{-?[0-9]*}}(%rsp), %eax {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_blcmsk_u32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    blcmskl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Folded Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call i64 asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = add i32 %a0, 1
   %3 = xor i32 %a0, %2
@@ -96,8 +421,44 @@ define i32 @stack_fold_blcmsk_u32(i32 %a0) {
 }
 
 define i64 @stack_fold_blcmsk_u64(i64 %a0) {
-  ;CHECK-LABEL: stack_fold_blcmsk_u64
-  ;CHECK:       blcmskq {{-?[0-9]*}}(%rsp), %rax {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_blcmsk_u64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    blcmskq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Folded Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call i64 asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = add i64 %a0, 1
   %3 = xor i64 %a0, %2
@@ -105,8 +466,44 @@ define i64 @stack_fold_blcmsk_u64(i64 %a0) {
 }
 
 define i32 @stack_fold_blcs_u32(i32 %a0) {
-  ;CHECK-LABEL: stack_fold_blcs_u32
-  ;CHECK:       blcsl {{-?[0-9]*}}(%rsp), %eax {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_blcs_u32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    blcsl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Folded Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call i64 asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = add i32 %a0, 1
   %3 = or i32 %a0, %2
@@ -114,8 +511,44 @@ define i32 @stack_fold_blcs_u32(i32 %a0) {
 }
 
 define i64 @stack_fold_blcs_u64(i64 %a0) {
-  ;CHECK-LABEL: stack_fold_blcs_u64
-  ;CHECK:       blcsq {{-?[0-9]*}}(%rsp), %rax {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_blcs_u64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    blcsq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Folded Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call i64 asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = add i64 %a0, 1
   %3 = or i64 %a0, %2
@@ -123,8 +556,44 @@ define i64 @stack_fold_blcs_u64(i64 %a0) {
 }
 
 define i32 @stack_fold_blsfill_u32(i32 %a0) {
-  ;CHECK-LABEL: stack_fold_blsfill_u32
-  ;CHECK:       blsfilll {{-?[0-9]*}}(%rsp), %eax {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_blsfill_u32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    blsfilll {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Folded Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call i64 asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = sub i32 %a0, 1
   %3 = or i32 %a0, %2
@@ -132,8 +601,44 @@ define i32 @stack_fold_blsfill_u32(i32 %a0) {
 }
 
 define i64 @stack_fold_blsfill_u64(i64 %a0) {
-  ;CHECK-LABEL: stack_fold_blsfill_u64
-  ;CHECK:       blsfillq {{-?[0-9]*}}(%rsp), %rax {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_blsfill_u64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    blsfillq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Folded Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call i64 asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = sub i64 %a0, 1
   %3 = or i64 %a0, %2
@@ -141,8 +646,44 @@ define i64 @stack_fold_blsfill_u64(i64 %a0) {
 }
 
 define i32 @stack_fold_blsic_u32(i32 %a0) {
-  ;CHECK-LABEL: stack_fold_blsic_u32
-  ;CHECK:       blsicl {{-?[0-9]*}}(%rsp), %eax {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_blsic_u32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    blsicl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Folded Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call i64 asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = sub i32 %a0, 1
   %3 = xor i32 %a0, -1
@@ -151,8 +692,44 @@ define i32 @stack_fold_blsic_u32(i32 %a0) {
 }
 
 define i64 @stack_fold_blsic_u64(i64 %a0) {
-  ;CHECK-LABEL: stack_fold_blsic_u64
-  ;CHECK:       blsicq {{-?[0-9]*}}(%rsp), %rax {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_blsic_u64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    blsicq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Folded Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call i64 asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = sub i64 %a0, 1
   %3 = xor i64 %a0, -1
@@ -161,8 +738,44 @@ define i64 @stack_fold_blsic_u64(i64 %a0) {
 }
 
 define i32 @stack_fold_t1mskc_u32(i32 %a0) {
-  ;CHECK-LABEL: stack_fold_t1mskc_u32
-  ;CHECK:       t1mskcl {{-?[0-9]*}}(%rsp), %eax {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_t1mskc_u32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    t1mskcl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Folded Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call i64 asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = add i32 %a0, 1
   %3 = xor i32 %a0, -1
@@ -171,8 +784,44 @@ define i32 @stack_fold_t1mskc_u32(i32 %a0) {
 }
 
 define i64 @stack_fold_t1mskc_u64(i64 %a0) {
-  ;CHECK-LABEL: stack_fold_t1mskc_u64
-  ;CHECK:       t1mskcq {{-?[0-9]*}}(%rsp), %rax {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_t1mskc_u64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    t1mskcq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Folded Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call i64 asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = add i64 %a0, 1
   %3 = xor i64 %a0, -1
@@ -181,8 +830,44 @@ define i64 @stack_fold_t1mskc_u64(i64 %a0) {
 }
 
 define i32 @stack_fold_tzmsk_u32(i32 %a0) {
-  ;CHECK-LABEL: stack_fold_tzmsk_u32
-  ;CHECK:       tzmskl {{-?[0-9]*}}(%rsp), %eax {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_tzmsk_u32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    tzmskl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Folded Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call i64 asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = sub i32 %a0, 1
   %3 = xor i32 %a0, -1
@@ -191,8 +876,44 @@ define i32 @stack_fold_tzmsk_u32(i32 %a0) {
 }
 
 define i64 @stack_fold_tzmsk_u64(i64 %a0) {
-  ;CHECK-LABEL: stack_fold_tzmsk_u64
-  ;CHECK:       tzmskq {{-?[0-9]*}}(%rsp), %rax {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_tzmsk_u64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    tzmskq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Folded Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call i64 asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = sub i64 %a0, 1
   %3 = xor i64 %a0, -1
diff --git a/llvm/test/CodeGen/X86/stack-folding-x86_64.ll b/llvm/test/CodeGen/X86/stack-folding-x86_64.ll
index f96880d02..e6fbec1ac 100644
--- a/llvm/test/CodeGen/X86/stack-folding-x86_64.ll
+++ b/llvm/test/CodeGen/X86/stack-folding-x86_64.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -O3 -disable-peephole -mtriple=x86_64-unknown-unknown < %s | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -12,8 +13,44 @@ target triple = "x86_64-unknown-unknown"
 declare i16 @llvm.cttz.i16(i16, i1)
 
 define i32 @stack_fold_bsf_i32(i32 %a0) {
-  ;CHECK-LABEL: stack_fold_bsf_i32
-  ;CHECK:       bsfl {{-?[0-9]*}}(%rsp), %eax {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_bsf_i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    bsfl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Folded Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call i64 asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = call i32 @llvm.cttz.i32(i32 %a0, i1 -1)
   ret i32 %2
@@ -21,8 +58,44 @@ define i32 @stack_fold_bsf_i32(i32 %a0) {
 declare i32 @llvm.cttz.i32(i32, i1)
 
 define i64 @stack_fold_bsf_i64(i64 %a0) {
-  ;CHECK-LABEL: stack_fold_bsf_i64
-  ;CHECK:       bsfq {{-?[0-9]*}}(%rsp), %rax {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_bsf_i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    bsfq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Folded Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call i64 asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = call i64 @llvm.cttz.i64(i64 %a0, i1 -1)
   ret i64 %2
@@ -33,8 +106,45 @@ declare i64 @llvm.cttz.i64(i64, i1)
 declare i16 @llvm.ctlz.i16(i16, i1)
 
 define i32 @stack_fold_bsr_i32(i32 %a0) {
-  ;CHECK-LABEL: stack_fold_bsr_i32
-  ;CHECK:       bsrl {{-?[0-9]*}}(%rsp), %eax {{.*#+}} 4-byte Folded Reload
+; CHECK-LABEL: stack_fold_bsr_i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    bsrl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Folded Reload
+; CHECK-NEXT:    xorl $31, %eax
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call i64 asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = call i32 @llvm.ctlz.i32(i32 %a0, i1 -1)
   ret i32 %2
@@ -42,8 +152,45 @@ define i32 @stack_fold_bsr_i32(i32 %a0) {
 declare i32 @llvm.ctlz.i32(i32, i1)
 
 define i64 @stack_fold_bsr_i64(i64 %a0) {
-  ;CHECK-LABEL: stack_fold_bsr_i64
-  ;CHECK:       bsrq {{-?[0-9]*}}(%rsp), %rax {{.*#+}} 8-byte Folded Reload
+; CHECK-LABEL: stack_fold_bsr_i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    bsrq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Folded Reload
+; CHECK-NEXT:    xorq $63, %rax
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %1 = tail call i64 asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
   %2 = call i64 @llvm.ctlz.i64(i64 %a0, i1 -1)
   ret i64 %2
diff --git a/llvm/test/CodeGen/X86/stack-folding-xop.ll b/llvm/test/CodeGen/X86/stack-folding-xop.ll
index 115d1a9ca..b5b1581f4 100644
--- a/llvm/test/CodeGen/X86/stack-folding-xop.ll
+++ b/llvm/test/CodeGen/X86/stack-folding-xop.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -O3 -disable-peephole -mtriple=x86_64-unknown-unknown -mattr=+avx,+xop < %s | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -9,8 +10,14 @@ target triple = "x86_64-unknown-unknown"
 ; relevant registers and check that the reload is correctly folded into the instruction.
 
 define <2 x double> @stack_fold_vfrczpd(<2 x double> %a0) {
-  ;CHECK-LABEL: stack_fold_vfrczpd
-  ;CHECK:       vfrczpd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vfrczpd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vfrczpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x double> @llvm.x86.xop.vfrcz.pd(<2 x double> %a0)
   ret <2 x double> %2
@@ -18,8 +25,14 @@ define <2 x double> @stack_fold_vfrczpd(<2 x double> %a0) {
 declare <2 x double> @llvm.x86.xop.vfrcz.pd(<2 x double>) nounwind readnone
 
 define <4 x double> @stack_fold_vfrczpd_ymm(<4 x double> %a0) {
-  ;CHECK-LABEL: stack_fold_vfrczpd_ymm
-  ;CHECK:       vfrczpd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_vfrczpd_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vfrczpd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x double> @llvm.x86.xop.vfrcz.pd.256(<4 x double> %a0)
   ret <4 x double> %2
@@ -27,8 +40,14 @@ define <4 x double> @stack_fold_vfrczpd_ymm(<4 x double> %a0) {
 declare <4 x double> @llvm.x86.xop.vfrcz.pd.256(<4 x double>) nounwind readnone
 
 define <4 x float> @stack_fold_vfrczps(<4 x float> %a0) {
-  ;CHECK-LABEL: stack_fold_vfrczps
-  ;CHECK:       vfrczps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vfrczps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vfrczps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x float> @llvm.x86.xop.vfrcz.ps(<4 x float> %a0)
   ret <4 x float> %2
@@ -36,8 +55,14 @@ define <4 x float> @stack_fold_vfrczps(<4 x float> %a0) {
 declare <4 x float> @llvm.x86.xop.vfrcz.ps(<4 x float>) nounwind readnone
 
 define <8 x float> @stack_fold_vfrczps_ymm(<8 x float> %a0) {
-  ;CHECK-LABEL: stack_fold_vfrczps_ymm
-  ;CHECK:       vfrczps {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_vfrczps_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vfrczps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x float> @llvm.x86.xop.vfrcz.ps.256(<8 x float> %a0)
   ret <8 x float> %2
@@ -45,8 +70,14 @@ define <8 x float> @stack_fold_vfrczps_ymm(<8 x float> %a0) {
 declare <8 x float> @llvm.x86.xop.vfrcz.ps.256(<8 x float>) nounwind readnone
 
 define <2 x double> @stack_fold_vfrczsd(<2 x double> %a0) {
-  ;CHECK-LABEL: stack_fold_vfrczsd
-  ;CHECK:       vfrczsd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vfrczsd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vfrczsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x double> @llvm.x86.xop.vfrcz.sd(<2 x double> %a0)
   ret <2 x double> %2
@@ -54,8 +85,14 @@ define <2 x double> @stack_fold_vfrczsd(<2 x double> %a0) {
 declare <2 x double> @llvm.x86.xop.vfrcz.sd(<2 x double>) nounwind readnone
 
 define <4 x float> @stack_fold_vfrczss(<4 x float> %a0) {
-  ;CHECK-LABEL: stack_fold_vfrczss
-  ;CHECK:       vfrczss {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vfrczss:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vfrczss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x float> @llvm.x86.xop.vfrcz.ss(<4 x float> %a0)
   ret <4 x float> %2
@@ -63,15 +100,27 @@ define <4 x float> @stack_fold_vfrczss(<4 x float> %a0) {
 declare <4 x float> @llvm.x86.xop.vfrcz.ss(<4 x float>) nounwind readnone
 
 define <2 x i64> @stack_fold_vpcmov_rm(<2 x i64> %a0, <2 x i64> %a1, <2 x i64> %a2) {
-  ;CHECK-LABEL: stack_fold_vpcmov_rm
-  ;CHECK:       vpcmov {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpcmov_rm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpcmov {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x i64> @llvm.x86.xop.vpcmov(<2 x i64> %a0, <2 x i64> %a1, <2 x i64> %a2)
   ret <2 x i64> %2
 }
 define <2 x i64> @stack_fold_vpcmov_mr(<2 x i64> %a0, <2 x i64> %a1, <2 x i64> %a2) {
-  ;CHECK-LABEL: stack_fold_vpcmov_mr
-  ;CHECK:       vpcmov {{%xmm[0-9][0-9]*}}, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpcmov_mr:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpcmov %xmm1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x i64> @llvm.x86.xop.vpcmov(<2 x i64> %a0, <2 x i64> %a2, <2 x i64> %a1)
   ret <2 x i64> %2
@@ -79,15 +128,27 @@ define <2 x i64> @stack_fold_vpcmov_mr(<2 x i64> %a0, <2 x i64> %a1, <2 x i64> %
 declare <2 x i64> @llvm.x86.xop.vpcmov(<2 x i64>, <2 x i64>, <2 x i64>) nounwind readnone
 
 define <4 x i64> @stack_fold_vpcmov_rm_ymm(<4 x i64> %a0, <4 x i64> %a1, <4 x i64> %a2) {
-  ;CHECK-LABEL: stack_fold_vpcmov_rm_ymm
-  ;CHECK:       vpcmov {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpcmov_rm_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpcmov {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x i64> @llvm.x86.xop.vpcmov.256(<4 x i64> %a0, <4 x i64> %a1, <4 x i64> %a2)
   ret <4 x i64> %2
 }
 define <4 x i64> @stack_fold_vpcmov_mr_ymm(<4 x i64> %a0, <4 x i64> %a1, <4 x i64> %a2) {
-  ;CHECK-LABEL: stack_fold_vpcmov_mr_ymm
-  ;CHECK:       vpcmov {{%ymm[0-9][0-9]*}}, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpcmov_mr_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpcmov %ymm1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x i64> @llvm.x86.xop.vpcmov.256(<4 x i64> %a0, <4 x i64> %a2, <4 x i64> %a1)
   ret <4 x i64> %2
@@ -95,8 +156,14 @@ define <4 x i64> @stack_fold_vpcmov_mr_ymm(<4 x i64> %a0, <4 x i64> %a1, <4 x i6
 declare <4 x i64> @llvm.x86.xop.vpcmov.256(<4 x i64>, <4 x i64>, <4 x i64>) nounwind readnone
 
 define <16 x i8> @stack_fold_vpcomb(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_vpcomb
-  ;CHECK:       vpcomltb {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpcomb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpcomltb {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <16 x i8> @llvm.x86.xop.vpcomb(<16 x i8> %a0, <16 x i8> %a1, i8 0)
   ret <16 x i8> %2
@@ -104,8 +171,14 @@ define <16 x i8> @stack_fold_vpcomb(<16 x i8> %a0, <16 x i8> %a1) {
 declare <16 x i8> @llvm.x86.xop.vpcomb(<16 x i8>, <16 x i8>, i8) nounwind readnone
 
 define <4 x i32> @stack_fold_vpcomd(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_vpcomd
-  ;CHECK:       vpcomltd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpcomd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpcomltd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x i32> @llvm.x86.xop.vpcomd(<4 x i32> %a0, <4 x i32> %a1, i8 0)
   ret <4 x i32> %2
@@ -113,8 +186,14 @@ define <4 x i32> @stack_fold_vpcomd(<4 x i32> %a0, <4 x i32> %a1) {
 declare <4 x i32> @llvm.x86.xop.vpcomd(<4 x i32>, <4 x i32>, i8) nounwind readnone
 
 define <2 x i64> @stack_fold_vpcomq(<2 x i64> %a0, <2 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_vpcomq
-  ;CHECK:       vpcomltq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpcomq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpcomltq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x i64> @llvm.x86.xop.vpcomq(<2 x i64> %a0, <2 x i64> %a1, i8 0)
   ret <2 x i64> %2
@@ -122,8 +201,14 @@ define <2 x i64> @stack_fold_vpcomq(<2 x i64> %a0, <2 x i64> %a1) {
 declare <2 x i64> @llvm.x86.xop.vpcomq(<2 x i64>, <2 x i64>, i8) nounwind readnone
 
 define <16 x i8> @stack_fold_vpcomub(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_vpcomub
-  ;CHECK:       vpcomltub {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpcomub:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpcomltub {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <16 x i8> @llvm.x86.xop.vpcomub(<16 x i8> %a0, <16 x i8> %a1, i8 0)
   ret <16 x i8> %2
@@ -131,8 +216,14 @@ define <16 x i8> @stack_fold_vpcomub(<16 x i8> %a0, <16 x i8> %a1) {
 declare <16 x i8> @llvm.x86.xop.vpcomub(<16 x i8>, <16 x i8>, i8) nounwind readnone
 
 define <4 x i32> @stack_fold_vpcomud(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_vpcomud
-  ;CHECK:       vpcomltud {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpcomud:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpcomltud {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x i32> @llvm.x86.xop.vpcomud(<4 x i32> %a0, <4 x i32> %a1, i8 0)
   ret <4 x i32> %2
@@ -140,8 +231,14 @@ define <4 x i32> @stack_fold_vpcomud(<4 x i32> %a0, <4 x i32> %a1) {
 declare <4 x i32> @llvm.x86.xop.vpcomud(<4 x i32>, <4 x i32>, i8) nounwind readnone
 
 define <2 x i64> @stack_fold_vpcomuq(<2 x i64> %a0, <2 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_vpcomuq
-  ;CHECK:       vpcomltuq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpcomuq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpcomltuq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x i64> @llvm.x86.xop.vpcomuq(<2 x i64> %a0, <2 x i64> %a1, i8 0)
   ret <2 x i64> %2
@@ -149,8 +246,14 @@ define <2 x i64> @stack_fold_vpcomuq(<2 x i64> %a0, <2 x i64> %a1) {
 declare <2 x i64> @llvm.x86.xop.vpcomuq(<2 x i64>, <2 x i64>, i8) nounwind readnone
 
 define <8 x i16> @stack_fold_vpcomuw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_vpcomuw
-  ;CHECK:       vpcomltuw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpcomuw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpcomltuw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x i16> @llvm.x86.xop.vpcomuw(<8 x i16> %a0, <8 x i16> %a1, i8 0)
   ret <8 x i16> %2
@@ -158,8 +261,14 @@ define <8 x i16> @stack_fold_vpcomuw(<8 x i16> %a0, <8 x i16> %a1) {
 declare <8 x i16> @llvm.x86.xop.vpcomuw(<8 x i16>, <8 x i16>, i8) nounwind readnone
 
 define <8 x i16> @stack_fold_vpcomw(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_vpcomw
-  ;CHECK:       vpcomltw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpcomw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpcomltw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x i16> @llvm.x86.xop.vpcomw(<8 x i16> %a0, <8 x i16> %a1, i8 0)
   ret <8 x i16> %2
@@ -167,15 +276,27 @@ define <8 x i16> @stack_fold_vpcomw(<8 x i16> %a0, <8 x i16> %a1) {
 declare <8 x i16> @llvm.x86.xop.vpcomw(<8 x i16>, <8 x i16>, i8) nounwind readnone
 
 define <2 x double> @stack_fold_vpermil2pd_rm(<2 x double> %a0, <2 x double> %a1, <2 x i64> %a2) {
-  ;CHECK-LABEL: stack_fold_vpermil2pd_rm
-  ;CHECK:       vpermil2pd $0, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpermil2pd_rm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermil2pd $0, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x double> @llvm.x86.xop.vpermil2pd(<2 x double> %a0, <2 x double> %a1, <2 x i64> %a2, i8 0)
   ret <2 x double> %2
 }
 define <2 x double> @stack_fold_vpermil2pd_mr(<2 x double> %a0, <2 x i64> %a1, <2 x double> %a2) {
-  ;CHECK-LABEL: stack_fold_vpermil2pd_mr
-  ;CHECK:       vpermil2pd $0, {{%xmm[0-9][0-9]*}}, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpermil2pd_mr:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermil2pd $0, %xmm1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x double> @llvm.x86.xop.vpermil2pd(<2 x double> %a0, <2 x double> %a2, <2 x i64> %a1, i8 0)
   ret <2 x double> %2
@@ -183,15 +304,27 @@ define <2 x double> @stack_fold_vpermil2pd_mr(<2 x double> %a0, <2 x i64> %a1, <
 declare <2 x double> @llvm.x86.xop.vpermil2pd(<2 x double>, <2 x double>, <2 x i64>, i8) nounwind readnone
 
 define <4 x double> @stack_fold_vpermil2pd_rm_ymm(<4 x double> %a0, <4 x double> %a1, <4 x i64> %a2) {
-  ;CHECK-LABEL: stack_fold_vpermil2pd_rm
-  ;CHECK:       vpermil2pd $0, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpermil2pd_rm_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermil2pd $0, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x double> @llvm.x86.xop.vpermil2pd.256(<4 x double> %a0, <4 x double> %a1, <4 x i64> %a2, i8 0)
   ret <4 x double> %2
 }
 define <4 x double> @stack_fold_vpermil2pd_mr_ymm(<4 x double> %a0, <4 x i64> %a1, <4 x double> %a2) {
-  ;CHECK-LABEL: stack_fold_vpermil2pd_mr
-  ;CHECK:       vpermil2pd $0, {{%ymm[0-9][0-9]*}}, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpermil2pd_mr_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermil2pd $0, %ymm1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x double> @llvm.x86.xop.vpermil2pd.256(<4 x double> %a0, <4 x double> %a2, <4 x i64> %a1, i8 0)
   ret <4 x double> %2
@@ -199,15 +332,27 @@ define <4 x double> @stack_fold_vpermil2pd_mr_ymm(<4 x double> %a0, <4 x i64> %a
 declare <4 x double> @llvm.x86.xop.vpermil2pd.256(<4 x double>, <4 x double>, <4 x i64>, i8) nounwind readnone
 
 define <4 x float> @stack_fold_vpermil2ps_rm(<4 x float> %a0, <4 x float> %a1, <4 x i32> %a2) {
-  ;CHECK-LABEL: stack_fold_vpermil2ps_rm
-  ;CHECK:       vpermil2ps $0, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpermil2ps_rm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermil2ps $0, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x float> @llvm.x86.xop.vpermil2ps(<4 x float> %a0, <4 x float> %a1, <4 x i32> %a2, i8 0)
   ret <4 x float> %2
 }
 define <4 x float> @stack_fold_vpermil2ps_mr(<4 x float> %a0, <4 x i32> %a1, <4 x float> %a2) {
-  ;CHECK-LABEL: stack_fold_vpermil2ps_mr
-  ;CHECK:       vpermil2ps $0, {{%xmm[0-9][0-9]*}}, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpermil2ps_mr:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermil2ps $0, %xmm1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x float> @llvm.x86.xop.vpermil2ps(<4 x float> %a0, <4 x float> %a2, <4 x i32> %a1, i8 0)
   ret <4 x float> %2
@@ -215,15 +360,27 @@ define <4 x float> @stack_fold_vpermil2ps_mr(<4 x float> %a0, <4 x i32> %a1, <4
 declare <4 x float> @llvm.x86.xop.vpermil2ps(<4 x float>, <4 x float>, <4 x i32>, i8) nounwind readnone
 
 define <8 x float> @stack_fold_vpermil2ps_rm_ymm(<8 x float> %a0, <8 x float> %a1, <8 x i32> %a2) {
-  ;CHECK-LABEL: stack_fold_vpermil2ps_rm
-  ;CHECK:       vpermil2ps $0, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpermil2ps_rm_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermil2ps $0, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x float> @llvm.x86.xop.vpermil2ps.256(<8 x float> %a0, <8 x float> %a1, <8 x i32> %a2, i8 0)
   ret <8 x float> %2
 }
 define <8 x float> @stack_fold_vpermil2ps_mr_ymm(<8 x float> %a0, <8 x i32> %a1, <8 x float> %a2) {
-  ;CHECK-LABEL: stack_fold_vpermil2ps_mr
-  ;CHECK:       vpermil2ps $0, {{%ymm[0-9][0-9]*}}, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpermil2ps_mr_ymm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpermil2ps $0, %ymm1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x float> @llvm.x86.xop.vpermil2ps.256(<8 x float> %a0, <8 x float> %a2, <8 x i32> %a1, i8 0)
   ret <8 x float> %2
@@ -231,8 +388,14 @@ define <8 x float> @stack_fold_vpermil2ps_mr_ymm(<8 x float> %a0, <8 x i32> %a1,
 declare <8 x float> @llvm.x86.xop.vpermil2ps.256(<8 x float>, <8 x float>, <8 x i32>, i8) nounwind readnone
 
 define <4 x i32> @stack_fold_vphaddbd(<16 x i8> %a0) {
-  ;CHECK-LABEL: stack_fold_vphaddbd
-  ;CHECK:       vphaddbd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vphaddbd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vphaddbd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x i32> @llvm.x86.xop.vphaddbd(<16 x i8> %a0)
   ret <4 x i32> %2
@@ -240,8 +403,14 @@ define <4 x i32> @stack_fold_vphaddbd(<16 x i8> %a0) {
 declare <4 x i32> @llvm.x86.xop.vphaddbd(<16 x i8>) nounwind readnone
 
 define <2 x i64> @stack_fold_vphaddbq(<16 x i8> %a0) {
-  ;CHECK-LABEL: stack_fold_vphaddbq
-  ;CHECK:       vphaddbq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vphaddbq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vphaddbq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x i64> @llvm.x86.xop.vphaddbq(<16 x i8> %a0)
   ret <2 x i64> %2
@@ -249,8 +418,14 @@ define <2 x i64> @stack_fold_vphaddbq(<16 x i8> %a0) {
 declare <2 x i64> @llvm.x86.xop.vphaddbq(<16 x i8>) nounwind readnone
 
 define <8 x i16> @stack_fold_vphaddbw(<16 x i8> %a0) {
-  ;CHECK-LABEL: stack_fold_vphaddbw
-  ;CHECK:       vphaddbw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vphaddbw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vphaddbw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x i16> @llvm.x86.xop.vphaddbw(<16 x i8> %a0)
   ret <8 x i16> %2
@@ -258,8 +433,14 @@ define <8 x i16> @stack_fold_vphaddbw(<16 x i8> %a0) {
 declare <8 x i16> @llvm.x86.xop.vphaddbw(<16 x i8>) nounwind readnone
 
 define <2 x i64> @stack_fold_vphadddq(<4 x i32> %a0) {
-  ;CHECK-LABEL: stack_fold_vphadddq
-  ;CHECK:       vphadddq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vphadddq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vphadddq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x i64> @llvm.x86.xop.vphadddq(<4 x i32> %a0)
   ret <2 x i64> %2
@@ -267,8 +448,14 @@ define <2 x i64> @stack_fold_vphadddq(<4 x i32> %a0) {
 declare <2 x i64> @llvm.x86.xop.vphadddq(<4 x i32>) nounwind readnone
 
 define <4 x i32> @stack_fold_vphaddubd(<16 x i8> %a0) {
-  ;CHECK-LABEL: stack_fold_vphaddubd
-  ;CHECK:       vphaddubd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vphaddubd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vphaddubd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x i32> @llvm.x86.xop.vphaddubd(<16 x i8> %a0)
   ret <4 x i32> %2
@@ -276,8 +463,14 @@ define <4 x i32> @stack_fold_vphaddubd(<16 x i8> %a0) {
 declare <4 x i32> @llvm.x86.xop.vphaddubd(<16 x i8>) nounwind readnone
 
 define <2 x i64> @stack_fold_vphaddubq(<16 x i8> %a0) {
-  ;CHECK-LABEL: stack_fold_vphaddubq
-  ;CHECK:       vphaddubq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vphaddubq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vphaddubq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x i64> @llvm.x86.xop.vphaddubq(<16 x i8> %a0)
   ret <2 x i64> %2
@@ -285,8 +478,14 @@ define <2 x i64> @stack_fold_vphaddubq(<16 x i8> %a0) {
 declare <2 x i64> @llvm.x86.xop.vphaddubq(<16 x i8>) nounwind readnone
 
 define <8 x i16> @stack_fold_vphaddubw(<16 x i8> %a0) {
-  ;CHECK-LABEL: stack_fold_vphaddubw
-  ;CHECK:       vphaddubw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vphaddubw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vphaddubw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x i16> @llvm.x86.xop.vphaddubw(<16 x i8> %a0)
   ret <8 x i16> %2
@@ -294,8 +493,14 @@ define <8 x i16> @stack_fold_vphaddubw(<16 x i8> %a0) {
 declare <8 x i16> @llvm.x86.xop.vphaddubw(<16 x i8>) nounwind readnone
 
 define <2 x i64> @stack_fold_vphaddudq(<4 x i32> %a0) {
-  ;CHECK-LABEL: stack_fold_vphaddudq
-  ;CHECK:       vphaddudq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vphaddudq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vphaddudq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x i64> @llvm.x86.xop.vphaddudq(<4 x i32> %a0)
   ret <2 x i64> %2
@@ -303,8 +508,14 @@ define <2 x i64> @stack_fold_vphaddudq(<4 x i32> %a0) {
 declare <2 x i64> @llvm.x86.xop.vphaddudq(<4 x i32>) nounwind readnone
 
 define <4 x i32> @stack_fold_vphadduwd(<8 x i16> %a0) {
-  ;CHECK-LABEL: stack_fold_vphadduwd
-  ;CHECK:       vphadduwd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vphadduwd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vphadduwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x i32> @llvm.x86.xop.vphadduwd(<8 x i16> %a0)
   ret <4 x i32> %2
@@ -312,8 +523,14 @@ define <4 x i32> @stack_fold_vphadduwd(<8 x i16> %a0) {
 declare <4 x i32> @llvm.x86.xop.vphadduwd(<8 x i16>) nounwind readnone
 
 define <2 x i64> @stack_fold_vphadduwq(<8 x i16> %a0) {
-  ;CHECK-LABEL: stack_fold_vphadduwq
-  ;CHECK:       vphadduwq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vphadduwq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vphadduwq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x i64> @llvm.x86.xop.vphadduwq(<8 x i16> %a0)
   ret <2 x i64> %2
@@ -321,8 +538,14 @@ define <2 x i64> @stack_fold_vphadduwq(<8 x i16> %a0) {
 declare <2 x i64> @llvm.x86.xop.vphadduwq(<8 x i16>) nounwind readnone
 
 define <4 x i32> @stack_fold_vphaddwd(<8 x i16> %a0) {
-  ;CHECK-LABEL: stack_fold_vphaddwd
-  ;CHECK:       vphaddwd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vphaddwd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vphaddwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x i32> @llvm.x86.xop.vphaddwd(<8 x i16> %a0)
   ret <4 x i32> %2
@@ -330,8 +553,14 @@ define <4 x i32> @stack_fold_vphaddwd(<8 x i16> %a0) {
 declare <4 x i32> @llvm.x86.xop.vphaddwd(<8 x i16>) nounwind readnone
 
 define <2 x i64> @stack_fold_vphaddwq(<8 x i16> %a0) {
-  ;CHECK-LABEL: stack_fold_vphaddwq
-  ;CHECK:       vphaddwq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vphaddwq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vphaddwq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x i64> @llvm.x86.xop.vphaddwq(<8 x i16> %a0)
   ret <2 x i64> %2
@@ -339,8 +568,14 @@ define <2 x i64> @stack_fold_vphaddwq(<8 x i16> %a0) {
 declare <2 x i64> @llvm.x86.xop.vphaddwq(<8 x i16>) nounwind readnone
 
 define <8 x i16> @stack_fold_vphsubbw(<16 x i8> %a0) {
-  ;CHECK-LABEL: stack_fold_vphsubbw
-  ;CHECK:       vphsubbw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vphsubbw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vphsubbw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x i16> @llvm.x86.xop.vphsubbw(<16 x i8> %a0)
   ret <8 x i16> %2
@@ -348,8 +583,14 @@ define <8 x i16> @stack_fold_vphsubbw(<16 x i8> %a0) {
 declare <8 x i16> @llvm.x86.xop.vphsubbw(<16 x i8>) nounwind readnone
 
 define <2 x i64> @stack_fold_vphsubdq(<4 x i32> %a0) {
-  ;CHECK-LABEL: stack_fold_vphsubdq
-  ;CHECK:       vphsubdq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vphsubdq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vphsubdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x i64> @llvm.x86.xop.vphsubdq(<4 x i32> %a0)
   ret <2 x i64> %2
@@ -357,8 +598,14 @@ define <2 x i64> @stack_fold_vphsubdq(<4 x i32> %a0) {
 declare <2 x i64> @llvm.x86.xop.vphsubdq(<4 x i32>) nounwind readnone
 
 define <4 x i32> @stack_fold_vphsubwd(<8 x i16> %a0) {
-  ;CHECK-LABEL: stack_fold_vphsubwd
-  ;CHECK:       vphsubwd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vphsubwd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vphsubwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x i32> @llvm.x86.xop.vphsubwd(<8 x i16> %a0)
   ret <4 x i32> %2
@@ -366,8 +613,16 @@ define <4 x i32> @stack_fold_vphsubwd(<8 x i16> %a0) {
 declare <4 x i32> @llvm.x86.xop.vphsubwd(<8 x i16>) nounwind readnone
 
 define <4 x i32> @stack_fold_vpmacsdd(<4 x i32> %a0, <4 x i32> %a1, <4 x i32> %a2) {
-  ;CHECK-LABEL: stack_fold_vpmacsdd
-  ;CHECK:       vpmacsdd {{%xmm[0-9][0-9]*}}, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpmacsdd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    vpmacsdd %xmm1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x i32> @llvm.x86.xop.vpmacsdd(<4 x i32> %a0, <4 x i32> %a1, <4 x i32> %a2)
   ret <4 x i32> %2
@@ -375,8 +630,16 @@ define <4 x i32> @stack_fold_vpmacsdd(<4 x i32> %a0, <4 x i32> %a1, <4 x i32> %a
 declare <4 x i32> @llvm.x86.xop.vpmacsdd(<4 x i32>, <4 x i32>, <4 x i32>) nounwind readnone
 
 define <2 x i64> @stack_fold_vpmacsdqh(<4 x i32> %a0, <4 x i32> %a1, <2 x i64> %a2) {
-  ;CHECK-LABEL: stack_fold_vpmacsdqh
-  ;CHECK:       vpmacsdqh {{%xmm[0-9][0-9]*}}, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpmacsdqh:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    vpmacsdqh %xmm1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x i64> @llvm.x86.xop.vpmacsdqh(<4 x i32> %a0, <4 x i32> %a1, <2 x i64> %a2)
   ret <2 x i64> %2
@@ -384,8 +647,16 @@ define <2 x i64> @stack_fold_vpmacsdqh(<4 x i32> %a0, <4 x i32> %a1, <2 x i64> %
 declare <2 x i64> @llvm.x86.xop.vpmacsdqh(<4 x i32>, <4 x i32>, <2 x i64>) nounwind readnone
 
 define <2 x i64> @stack_fold_vpmacsdql(<4 x i32> %a0, <4 x i32> %a1, <2 x i64> %a2) {
-  ;CHECK-LABEL: stack_fold_vpmacsdql
-  ;CHECK:       vpmacsdql {{%xmm[0-9][0-9]*}}, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpmacsdql:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    vpmacsdql %xmm1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x i64> @llvm.x86.xop.vpmacsdql(<4 x i32> %a0, <4 x i32> %a1, <2 x i64> %a2)
   ret <2 x i64> %2
@@ -393,8 +664,16 @@ define <2 x i64> @stack_fold_vpmacsdql(<4 x i32> %a0, <4 x i32> %a1, <2 x i64> %
 declare <2 x i64> @llvm.x86.xop.vpmacsdql(<4 x i32>, <4 x i32>, <2 x i64>) nounwind readnone
 
 define <4 x i32> @stack_fold_vpmacssdd(<4 x i32> %a0, <4 x i32> %a1, <4 x i32> %a2) {
-  ;CHECK-LABEL: stack_fold_vpmacssdd
-  ;CHECK:       vpmacssdd {{%xmm[0-9][0-9]*}}, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpmacssdd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    vpmacssdd %xmm1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x i32> @llvm.x86.xop.vpmacssdd(<4 x i32> %a0, <4 x i32> %a1, <4 x i32> %a2)
   ret <4 x i32> %2
@@ -402,8 +681,16 @@ define <4 x i32> @stack_fold_vpmacssdd(<4 x i32> %a0, <4 x i32> %a1, <4 x i32> %
 declare <4 x i32> @llvm.x86.xop.vpmacssdd(<4 x i32>, <4 x i32>, <4 x i32>) nounwind readnone
 
 define <2 x i64> @stack_fold_vpmacssdqh(<4 x i32> %a0, <4 x i32> %a1, <2 x i64> %a2) {
-  ;CHECK-LABEL: stack_fold_vpmacssdqh
-  ;CHECK:       vpmacssdqh {{%xmm[0-9][0-9]*}}, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpmacssdqh:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    vpmacssdqh %xmm1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x i64> @llvm.x86.xop.vpmacssdqh(<4 x i32> %a0, <4 x i32> %a1, <2 x i64> %a2)
   ret <2 x i64> %2
@@ -411,8 +698,16 @@ define <2 x i64> @stack_fold_vpmacssdqh(<4 x i32> %a0, <4 x i32> %a1, <2 x i64>
 declare <2 x i64> @llvm.x86.xop.vpmacssdqh(<4 x i32>, <4 x i32>, <2 x i64>) nounwind readnone
 
 define <2 x i64> @stack_fold_vpmacssdql(<4 x i32> %a0, <4 x i32> %a1, <2 x i64> %a2) {
-  ;CHECK-LABEL: stack_fold_vpmacssdql
-  ;CHECK:       vpmacssdql {{%xmm[0-9][0-9]*}}, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpmacssdql:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    vpmacssdql %xmm1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x i64> @llvm.x86.xop.vpmacssdql(<4 x i32> %a0, <4 x i32> %a1, <2 x i64> %a2)
   ret <2 x i64> %2
@@ -420,8 +715,16 @@ define <2 x i64> @stack_fold_vpmacssdql(<4 x i32> %a0, <4 x i32> %a1, <2 x i64>
 declare <2 x i64> @llvm.x86.xop.vpmacssdql(<4 x i32>, <4 x i32>, <2 x i64>) nounwind readnone
 
 define <4 x i32> @stack_fold_vpmacsswd(<8 x i16> %a0, <8 x i16> %a1, <4 x i32> %a2) {
-  ;CHECK-LABEL: stack_fold_vpmacsswd
-  ;CHECK:       vpmacsswd {{%xmm[0-9][0-9]*}}, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpmacsswd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    vpmacsswd %xmm1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x i32> @llvm.x86.xop.vpmacsswd(<8 x i16> %a0, <8 x i16> %a1, <4 x i32> %a2)
   ret <4 x i32> %2
@@ -429,8 +732,16 @@ define <4 x i32> @stack_fold_vpmacsswd(<8 x i16> %a0, <8 x i16> %a1, <4 x i32> %
 declare <4 x i32> @llvm.x86.xop.vpmacsswd(<8 x i16>, <8 x i16>, <4 x i32>) nounwind readnone
 
 define <8 x i16> @stack_fold_vpmacssww(<8 x i16> %a0, <8 x i16> %a1, <8 x i16> %a2) {
-  ;CHECK-LABEL: stack_fold_vpmacssww
-  ;CHECK:       vpmacssww {{%xmm[0-9][0-9]*}}, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpmacssww:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    vpmacssww %xmm1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x i16> @llvm.x86.xop.vpmacssww(<8 x i16> %a0, <8 x i16> %a1, <8 x i16> %a2)
   ret <8 x i16> %2
@@ -438,8 +749,16 @@ define <8 x i16> @stack_fold_vpmacssww(<8 x i16> %a0, <8 x i16> %a1, <8 x i16> %
 declare <8 x i16> @llvm.x86.xop.vpmacssww(<8 x i16>, <8 x i16>, <8 x i16>) nounwind readnone
 
 define <4 x i32> @stack_fold_vpmacswd(<8 x i16> %a0, <8 x i16> %a1, <4 x i32> %a2) {
-  ;CHECK-LABEL: stack_fold_vpmacswd
-  ;CHECK:       vpmacswd {{%xmm[0-9][0-9]*}}, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpmacswd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    vpmacswd %xmm1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x i32> @llvm.x86.xop.vpmacswd(<8 x i16> %a0, <8 x i16> %a1, <4 x i32> %a2)
   ret <4 x i32> %2
@@ -447,8 +766,16 @@ define <4 x i32> @stack_fold_vpmacswd(<8 x i16> %a0, <8 x i16> %a1, <4 x i32> %a
 declare <4 x i32> @llvm.x86.xop.vpmacswd(<8 x i16>, <8 x i16>, <4 x i32>) nounwind readnone
 
 define <8 x i16> @stack_fold_vpmacsww(<8 x i16> %a0, <8 x i16> %a1, <8 x i16> %a2) {
-  ;CHECK-LABEL: stack_fold_vpmacsww
-  ;CHECK:       vpmacsww {{%xmm[0-9][0-9]*}}, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpmacsww:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    vpmacsww %xmm1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x i16> @llvm.x86.xop.vpmacsww(<8 x i16> %a0, <8 x i16> %a1, <8 x i16> %a2)
   ret <8 x i16> %2
@@ -456,8 +783,16 @@ define <8 x i16> @stack_fold_vpmacsww(<8 x i16> %a0, <8 x i16> %a1, <8 x i16> %a
 declare <8 x i16> @llvm.x86.xop.vpmacsww(<8 x i16>, <8 x i16>, <8 x i16>) nounwind readnone
 
 define <4 x i32> @stack_fold_vpmadcsswd(<8 x i16> %a0, <8 x i16> %a1, <4 x i32> %a2) {
-  ;CHECK-LABEL: stack_fold_vpmadcsswd
-  ;CHECK:       vpmadcsswd {{%xmm[0-9][0-9]*}}, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpmadcsswd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    vpmadcsswd %xmm1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x i32> @llvm.x86.xop.vpmadcsswd(<8 x i16> %a0, <8 x i16> %a1, <4 x i32> %a2)
   ret <4 x i32> %2
@@ -465,8 +800,16 @@ define <4 x i32> @stack_fold_vpmadcsswd(<8 x i16> %a0, <8 x i16> %a1, <4 x i32>
 declare <4 x i32> @llvm.x86.xop.vpmadcsswd(<8 x i16>, <8 x i16>, <4 x i32>) nounwind readnone
 
 define <4 x i32> @stack_fold_vpmadcswd(<8 x i16> %a0, <8 x i16> %a1, <4 x i32> %a2) {
-  ;CHECK-LABEL: stack_fold_vpmadcswd
-  ;CHECK:       vpmadcswd {{%xmm[0-9][0-9]*}}, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpmadcswd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    vpmadcswd %xmm1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x i32> @llvm.x86.xop.vpmadcswd(<8 x i16> %a0, <8 x i16> %a1, <4 x i32> %a2)
   ret <4 x i32> %2
@@ -474,15 +817,27 @@ define <4 x i32> @stack_fold_vpmadcswd(<8 x i16> %a0, <8 x i16> %a1, <4 x i32> %
 declare <4 x i32> @llvm.x86.xop.vpmadcswd(<8 x i16>, <8 x i16>, <4 x i32>) nounwind readnone
 
 define <16 x i8> @stack_fold_vpperm_rm(<16 x i8> %a0, <16 x i8> %a1, <16 x i8> %a2) {
-  ;CHECK-LABEL: stack_fold_vpperm_rm
-  ;CHECK:       vpperm {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpperm_rm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpperm {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <16 x i8> @llvm.x86.xop.vpperm(<16 x i8> %a0, <16 x i8> %a1, <16 x i8> %a2)
   ret <16 x i8> %2
 }
 define <16 x i8> @stack_fold_vpperm_mr(<16 x i8> %a0, <16 x i8> %a1, <16 x i8> %a2) {
-  ;CHECK-LABEL: stack_fold_vpperm_mr
-  ;CHECK:       vpperm {{%xmm[0-9][0-9]*}}, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpperm_mr:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpperm %xmm1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <16 x i8> @llvm.x86.xop.vpperm(<16 x i8> %a0, <16 x i8> %a2, <16 x i8> %a1)
   ret <16 x i8> %2
@@ -490,8 +845,14 @@ define <16 x i8> @stack_fold_vpperm_mr(<16 x i8> %a0, <16 x i8> %a1, <16 x i8> %
 declare <16 x i8> @llvm.x86.xop.vpperm(<16 x i8>, <16 x i8>, <16 x i8>) nounwind readnone
 
 define <16 x i8> @stack_fold_vprotb(<16 x i8> %a0) {
-  ;CHECK-LABEL: stack_fold_vprotb
-  ;CHECK:       vprotb $7, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vprotb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vprotb $7, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <16 x i8> @llvm.x86.xop.vprotbi(<16 x i8> %a0, i8 7)
   ret <16 x i8> %2
@@ -499,15 +860,27 @@ define <16 x i8> @stack_fold_vprotb(<16 x i8> %a0) {
 declare <16 x i8> @llvm.x86.xop.vprotbi(<16 x i8>, i8) nounwind readnone
 
 define <16 x i8> @stack_fold_vprotb_rm(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_vprotb_rm
-  ;CHECK:       vprotb {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vprotb_rm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vprotb {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <16 x i8> @llvm.x86.xop.vprotb(<16 x i8> %a0, <16 x i8> %a1)
   ret <16 x i8> %2
 }
 define <16 x i8> @stack_fold_vprotb_mr(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_vprotb_mr
-  ;CHECK:       vprotb {{%xmm[0-9][0-9]*}}, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vprotb_mr:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vprotb %xmm0, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <16 x i8> @llvm.x86.xop.vprotb(<16 x i8> %a1, <16 x i8> %a0)
   ret <16 x i8> %2
@@ -515,8 +888,14 @@ define <16 x i8> @stack_fold_vprotb_mr(<16 x i8> %a0, <16 x i8> %a1) {
 declare <16 x i8> @llvm.x86.xop.vprotb(<16 x i8>, <16 x i8>) nounwind readnone
 
 define <4 x i32> @stack_fold_vprotd(<4 x i32> %a0) {
-  ;CHECK-LABEL: stack_fold_vprotd
-  ;CHECK:       vprotd $7, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vprotd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vprotd $7, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x i32> @llvm.x86.xop.vprotdi(<4 x i32> %a0, i8 7)
   ret <4 x i32> %2
@@ -524,15 +903,27 @@ define <4 x i32> @stack_fold_vprotd(<4 x i32> %a0) {
 declare <4 x i32> @llvm.x86.xop.vprotdi(<4 x i32>, i8) nounwind readnone
 
 define <4 x i32> @stack_fold_vprotd_rm(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_vprotd_rm
-  ;CHECK:       vprotd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vprotd_rm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vprotd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x i32> @llvm.x86.xop.vprotd(<4 x i32> %a0, <4 x i32> %a1)
   ret <4 x i32> %2
 }
 define <4 x i32> @stack_fold_vprotd_mr(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_vprotd_mr
-  ;CHECK:       vprotd {{%xmm[0-9][0-9]*}}, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vprotd_mr:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vprotd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x i32> @llvm.x86.xop.vprotd(<4 x i32> %a1, <4 x i32> %a0)
   ret <4 x i32> %2
@@ -540,8 +931,14 @@ define <4 x i32> @stack_fold_vprotd_mr(<4 x i32> %a0, <4 x i32> %a1) {
 declare <4 x i32> @llvm.x86.xop.vprotd(<4 x i32>, <4 x i32>) nounwind readnone
 
 define <2 x i64> @stack_fold_vprotq(<2 x i64> %a0) {
-  ;CHECK-LABEL: stack_fold_vprotq
-  ;CHECK:       vprotq $7, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vprotq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vprotq $7, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x i64> @llvm.x86.xop.vprotqi(<2 x i64> %a0, i8 7)
   ret <2 x i64> %2
@@ -549,15 +946,27 @@ define <2 x i64> @stack_fold_vprotq(<2 x i64> %a0) {
 declare <2 x i64> @llvm.x86.xop.vprotqi(<2 x i64>, i8) nounwind readnone
 
 define <2 x i64> @stack_fold_vprotq_rm(<2 x i64> %a0, <2 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_vprotq_rm
-  ;CHECK:       vprotq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vprotq_rm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vprotq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x i64> @llvm.x86.xop.vprotq(<2 x i64> %a0, <2 x i64> %a1)
   ret <2 x i64> %2
 }
 define <2 x i64> @stack_fold_vprotq_mr(<2 x i64> %a0, <2 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_vprotq_mr
-  ;CHECK:       vprotq {{%xmm[0-9][0-9]*}}, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vprotq_mr:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vprotq %xmm0, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x i64> @llvm.x86.xop.vprotq(<2 x i64> %a1, <2 x i64> %a0)
   ret <2 x i64> %2
@@ -565,8 +974,14 @@ define <2 x i64> @stack_fold_vprotq_mr(<2 x i64> %a0, <2 x i64> %a1) {
 declare <2 x i64> @llvm.x86.xop.vprotq(<2 x i64>, <2 x i64>) nounwind readnone
 
 define <8 x i16> @stack_fold_vprotw(<8 x i16> %a0) {
-  ;CHECK-LABEL: stack_fold_vprotw
-  ;CHECK:       vprotw $7, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vprotw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vprotw $7, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x i16> @llvm.x86.xop.vprotwi(<8 x i16> %a0, i8 7)
   ret <8 x i16> %2
@@ -574,15 +989,27 @@ define <8 x i16> @stack_fold_vprotw(<8 x i16> %a0) {
 declare <8 x i16> @llvm.x86.xop.vprotwi(<8 x i16>, i8) nounwind readnone
 
 define <8 x i16> @stack_fold_vprotw_rm(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_vprotw_rm
-  ;CHECK:       vprotw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vprotw_rm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vprotw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x i16> @llvm.x86.xop.vprotw(<8 x i16> %a0, <8 x i16> %a1)
   ret <8 x i16> %2
 }
 define <8 x i16> @stack_fold_vprotw_mr(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_vprotw_mr
-  ;CHECK:       vprotw {{%xmm[0-9][0-9]*}}, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vprotw_mr:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vprotw %xmm0, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x i16> @llvm.x86.xop.vprotw(<8 x i16> %a1, <8 x i16> %a0)
   ret <8 x i16> %2
@@ -590,15 +1017,27 @@ define <8 x i16> @stack_fold_vprotw_mr(<8 x i16> %a0, <8 x i16> %a1) {
 declare <8 x i16> @llvm.x86.xop.vprotw(<8 x i16>, <8 x i16>) nounwind readnone
 
 define <16 x i8> @stack_fold_vpshab_rm(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_vpshab_rm
-  ;CHECK:       vpshab {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpshab_rm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpshab {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <16 x i8> @llvm.x86.xop.vpshab(<16 x i8> %a0, <16 x i8> %a1)
   ret <16 x i8> %2
 }
 define <16 x i8> @stack_fold_vpshab_mr(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_vpshab_mr
-  ;CHECK:       vpshab {{%xmm[0-9][0-9]*}}, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpshab_mr:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpshab %xmm0, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <16 x i8> @llvm.x86.xop.vpshab(<16 x i8> %a1, <16 x i8> %a0)
   ret <16 x i8> %2
@@ -606,15 +1045,27 @@ define <16 x i8> @stack_fold_vpshab_mr(<16 x i8> %a0, <16 x i8> %a1) {
 declare <16 x i8> @llvm.x86.xop.vpshab(<16 x i8>, <16 x i8>) nounwind readnone
 
 define <4 x i32> @stack_fold_vpshad_rm(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_vpshad_rm
-  ;CHECK:       vpshad {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpshad_rm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpshad {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x i32> @llvm.x86.xop.vpshad(<4 x i32> %a0, <4 x i32> %a1)
   ret <4 x i32> %2
 }
 define <4 x i32> @stack_fold_vpshad_mr(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_vpshad_mr
-  ;CHECK:       vpshad {{%xmm[0-9][0-9]*}}, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpshad_mr:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpshad %xmm0, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x i32> @llvm.x86.xop.vpshad(<4 x i32> %a1, <4 x i32> %a0)
   ret <4 x i32> %2
@@ -622,15 +1073,27 @@ define <4 x i32> @stack_fold_vpshad_mr(<4 x i32> %a0, <4 x i32> %a1) {
 declare <4 x i32> @llvm.x86.xop.vpshad(<4 x i32>, <4 x i32>) nounwind readnone
 
 define <2 x i64> @stack_fold_vpshaq_rm(<2 x i64> %a0, <2 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_vpshaq_rm
-  ;CHECK:       vpshaq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpshaq_rm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpshaq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x i64> @llvm.x86.xop.vpshaq(<2 x i64> %a0, <2 x i64> %a1)
   ret <2 x i64> %2
 }
 define <2 x i64> @stack_fold_vpshaq_mr(<2 x i64> %a0, <2 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_vpshaq_mr
-  ;CHECK:       vpshaq {{%xmm[0-9][0-9]*}}, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpshaq_mr:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpshaq %xmm0, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x i64> @llvm.x86.xop.vpshaq(<2 x i64> %a1, <2 x i64> %a0)
   ret <2 x i64> %2
@@ -638,15 +1101,27 @@ define <2 x i64> @stack_fold_vpshaq_mr(<2 x i64> %a0, <2 x i64> %a1) {
 declare <2 x i64> @llvm.x86.xop.vpshaq(<2 x i64>, <2 x i64>) nounwind readnone
 
 define <8 x i16> @stack_fold_vpshaw_rm(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_vpshaw_rm
-  ;CHECK:       vpshaw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpshaw_rm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpshaw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x i16> @llvm.x86.xop.vpshaw(<8 x i16> %a0, <8 x i16> %a1)
   ret <8 x i16> %2
 }
 define <8 x i16> @stack_fold_vpshaw_mr(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_vpshaw_mr
-  ;CHECK:       vpshaw {{%xmm[0-9][0-9]*}}, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpshaw_mr:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpshaw %xmm0, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x i16> @llvm.x86.xop.vpshaw(<8 x i16> %a1, <8 x i16> %a0)
   ret <8 x i16> %2
@@ -654,15 +1129,27 @@ define <8 x i16> @stack_fold_vpshaw_mr(<8 x i16> %a0, <8 x i16> %a1) {
 declare <8 x i16> @llvm.x86.xop.vpshaw(<8 x i16>, <8 x i16>) nounwind readnone
 
 define <16 x i8> @stack_fold_vpshlb_rm(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_vpshlb_rm
-  ;CHECK:       vpshlb {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpshlb_rm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpshlb {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <16 x i8> @llvm.x86.xop.vpshlb(<16 x i8> %a0, <16 x i8> %a1)
   ret <16 x i8> %2
 }
 define <16 x i8> @stack_fold_vpshlb_mr(<16 x i8> %a0, <16 x i8> %a1) {
-  ;CHECK-LABEL: stack_fold_vpshlb_mr
-  ;CHECK:       vpshlb {{%xmm[0-9][0-9]*}}, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpshlb_mr:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpshlb %xmm0, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <16 x i8> @llvm.x86.xop.vpshlb(<16 x i8> %a1, <16 x i8> %a0)
   ret <16 x i8> %2
@@ -670,15 +1157,27 @@ define <16 x i8> @stack_fold_vpshlb_mr(<16 x i8> %a0, <16 x i8> %a1) {
 declare <16 x i8> @llvm.x86.xop.vpshlb(<16 x i8>, <16 x i8>) nounwind readnone
 
 define <4 x i32> @stack_fold_vpshld_rm(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_vpshld_rm
-  ;CHECK:       vpshld {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpshld_rm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpshld {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x i32> @llvm.x86.xop.vpshld(<4 x i32> %a0, <4 x i32> %a1)
   ret <4 x i32> %2
 }
 define <4 x i32> @stack_fold_vpshld_mr(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL: stack_fold_vpshld_mr
-  ;CHECK:       vpshld {{%xmm[0-9][0-9]*}}, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpshld_mr:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpshld %xmm0, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x i32> @llvm.x86.xop.vpshld(<4 x i32> %a1, <4 x i32> %a0)
   ret <4 x i32> %2
@@ -686,15 +1185,27 @@ define <4 x i32> @stack_fold_vpshld_mr(<4 x i32> %a0, <4 x i32> %a1) {
 declare <4 x i32> @llvm.x86.xop.vpshld(<4 x i32>, <4 x i32>) nounwind readnone
 
 define <2 x i64> @stack_fold_vpshlq_rm(<2 x i64> %a0, <2 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_vpshlq_rm
-  ;CHECK:       vpshlq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpshlq_rm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpshlq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x i64> @llvm.x86.xop.vpshlq(<2 x i64> %a0, <2 x i64> %a1)
   ret <2 x i64> %2
 }
 define <2 x i64> @stack_fold_vpshlq_mr(<2 x i64> %a0, <2 x i64> %a1) {
-  ;CHECK-LABEL: stack_fold_vpshlq_mr
-  ;CHECK:       vpshlq {{%xmm[0-9][0-9]*}}, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpshlq_mr:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpshlq %xmm0, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <2 x i64> @llvm.x86.xop.vpshlq(<2 x i64> %a1, <2 x i64> %a0)
   ret <2 x i64> %2
@@ -702,15 +1213,27 @@ define <2 x i64> @stack_fold_vpshlq_mr(<2 x i64> %a0, <2 x i64> %a1) {
 declare <2 x i64> @llvm.x86.xop.vpshlq(<2 x i64>, <2 x i64>) nounwind readnone
 
 define <8 x i16> @stack_fold_vpshlw_rm(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_vpshlw_rm
-  ;CHECK:       vpshlw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpshlw_rm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpshlw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x i16> @llvm.x86.xop.vpshlw(<8 x i16> %a0, <8 x i16> %a1)
   ret <8 x i16> %2
 }
 define <8 x i16> @stack_fold_vpshlw_mr(<8 x i16> %a0, <8 x i16> %a1) {
-  ;CHECK-LABEL: stack_fold_vpshlw_mr
-  ;CHECK:       vpshlw {{%xmm[0-9][0-9]*}}, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
+; CHECK-LABEL: stack_fold_vpshlw_mr:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpshlw %xmm0, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x i16> @llvm.x86.xop.vpshlw(<8 x i16> %a1, <8 x i16> %a0)
   ret <8 x i16> %2
diff --git a/llvm/test/CodeGen/X86/stack-protector-2.ll b/llvm/test/CodeGen/X86/stack-protector-2.ll
new file mode 100644
index 000000000..dc86b93cd
--- /dev/null
+++ b/llvm/test/CodeGen/X86/stack-protector-2.ll
@@ -0,0 +1,165 @@
+; RUN: llc -mtriple=x86_64-pc-linux-gnu -start-before=stack-protector -stop-after=stack-protector -o - < %s | FileCheck %s
+; Bugs 42238/43308: Test some additional situations not caught previously.
+
+define void @store_captures() #0 {
+; CHECK-LABEL: @store_captures(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[STACKGUARDSLOT:%.*]] = alloca i8*
+; CHECK-NEXT:    [[STACKGUARD:%.*]] = load volatile i8*, i8* addrspace(257)* inttoptr (i32 40 to i8* addrspace(257)*)
+; CHECK-NEXT:    call void @llvm.stackprotector(i8* [[STACKGUARD]], i8** [[STACKGUARDSLOT]])
+; CHECK-NEXT:    [[RETVAL:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[A:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[J:%.*]] = alloca i32*, align 8
+; CHECK-NEXT:    store i32 0, i32* [[RETVAL]]
+; CHECK-NEXT:    [[LOAD:%.*]] = load i32, i32* [[A]], align 4
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[LOAD]], 1
+; CHECK-NEXT:    store i32 [[ADD]], i32* [[A]], align 4
+; CHECK-NEXT:    store i32* [[A]], i32** [[J]], align 8
+; CHECK-NEXT:    [[STACKGUARD1:%.*]] = load volatile i8*, i8* addrspace(257)* inttoptr (i32 40 to i8* addrspace(257)*)
+; CHECK-NEXT:    [[TMP0:%.*]] = load volatile i8*, i8** [[STACKGUARDSLOT]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i8* [[STACKGUARD1]], [[TMP0]]
+; CHECK-NEXT:    br i1 [[TMP1]], label [[SP_RETURN:%.*]], label [[CALLSTACKCHECKFAILBLK:%.*]], !prof !0
+; CHECK:       SP_return:
+; CHECK-NEXT:    ret void
+; CHECK:       CallStackCheckFailBlk:
+; CHECK-NEXT:    call void @__stack_chk_fail()
+; CHECK-NEXT:    unreachable
+;
+entry:
+  %retval = alloca i32, align 4
+  %a = alloca i32, align 4
+  %j = alloca i32*, align 8
+  store i32 0, i32* %retval
+  %load = load i32, i32* %a, align 4
+  %add = add nsw i32 %load, 1
+  store i32 %add, i32* %a, align 4
+  store i32* %a, i32** %j, align 8
+  ret void
+}
+
+define i32* @non_captures() #0 {
+; load, atomicrmw, and ret do not trigger a stack protector.
+; CHECK-LABEL: @non_captures(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[A:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[LOAD:%.*]] = load i32, i32* [[A]], align 4
+; CHECK-NEXT:    [[ATOM:%.*]] = atomicrmw add i32* [[A]], i32 1 seq_cst
+; CHECK-NEXT:    ret i32* [[A]]
+;
+entry:
+  %a = alloca i32, align 4
+  %load = load i32, i32* %a, align 4
+  %atom = atomicrmw add i32* %a, i32 1 seq_cst
+  ret i32* %a
+}
+
+define void @store_addrspacecast_captures() #0 {
+; CHECK-LABEL: @store_addrspacecast_captures(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[STACKGUARDSLOT:%.*]] = alloca i8*
+; CHECK-NEXT:    [[STACKGUARD:%.*]] = load volatile i8*, i8* addrspace(257)* inttoptr (i32 40 to i8* addrspace(257)*)
+; CHECK-NEXT:    call void @llvm.stackprotector(i8* [[STACKGUARD]], i8** [[STACKGUARDSLOT]])
+; CHECK-NEXT:    [[RETVAL:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[A:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[J:%.*]] = alloca i32 addrspace(1)*, align 8
+; CHECK-NEXT:    store i32 0, i32* [[RETVAL]]
+; CHECK-NEXT:    [[LOAD:%.*]] = load i32, i32* [[A]], align 4
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[LOAD]], 1
+; CHECK-NEXT:    store i32 [[ADD]], i32* [[A]], align 4
+; CHECK-NEXT:    [[A_ADDRSPACECAST:%.*]] = addrspacecast i32* [[A]] to i32 addrspace(1)*
+; CHECK-NEXT:    store i32 addrspace(1)* [[A_ADDRSPACECAST]], i32 addrspace(1)** [[J]], align 8
+; CHECK-NEXT:    [[STACKGUARD1:%.*]] = load volatile i8*, i8* addrspace(257)* inttoptr (i32 40 to i8* addrspace(257)*)
+; CHECK-NEXT:    [[TMP0:%.*]] = load volatile i8*, i8** [[STACKGUARDSLOT]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i8* [[STACKGUARD1]], [[TMP0]]
+; CHECK-NEXT:    br i1 [[TMP1]], label [[SP_RETURN:%.*]], label [[CALLSTACKCHECKFAILBLK:%.*]], !prof !0
+; CHECK:       SP_return:
+; CHECK-NEXT:    ret void
+; CHECK:       CallStackCheckFailBlk:
+; CHECK-NEXT:    call void @__stack_chk_fail()
+; CHECK-NEXT:    unreachable
+;
+entry:
+  %retval = alloca i32, align 4
+  %a = alloca i32, align 4
+  %j = alloca i32 addrspace(1)*, align 8
+  store i32 0, i32* %retval
+  %load = load i32, i32* %a, align 4
+  %add = add nsw i32 %load, 1
+  store i32 %add, i32* %a, align 4
+  %a.addrspacecast = addrspacecast i32* %a to i32 addrspace(1)*
+  store i32 addrspace(1)* %a.addrspacecast, i32 addrspace(1)** %j, align 8
+  ret void
+}
+
+define void @cmpxchg_captures() #0 {
+; CHECK-LABEL: @cmpxchg_captures(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[STACKGUARDSLOT:%.*]] = alloca i8*
+; CHECK-NEXT:    [[STACKGUARD:%.*]] = load volatile i8*, i8* addrspace(257)* inttoptr (i32 40 to i8* addrspace(257)*)
+; CHECK-NEXT:    call void @llvm.stackprotector(i8* [[STACKGUARD]], i8** [[STACKGUARDSLOT]])
+; CHECK-NEXT:    [[RETVAL:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[A:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[J:%.*]] = alloca i32*, align 8
+; CHECK-NEXT:    store i32 0, i32* [[RETVAL]]
+; CHECK-NEXT:    [[LOAD:%.*]] = load i32, i32* [[A]], align 4
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[LOAD]], 1
+; CHECK-NEXT:    store i32 [[ADD]], i32* [[A]], align 4
+; CHECK-NEXT:    [[TMP0:%.*]] = cmpxchg i32** [[J]], i32* null, i32* [[A]] seq_cst monotonic
+; CHECK-NEXT:    [[STACKGUARD1:%.*]] = load volatile i8*, i8* addrspace(257)* inttoptr (i32 40 to i8* addrspace(257)*)
+; CHECK-NEXT:    [[TMP1:%.*]] = load volatile i8*, i8** [[STACKGUARDSLOT]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i8* [[STACKGUARD1]], [[TMP1]]
+; CHECK-NEXT:    br i1 [[TMP2]], label [[SP_RETURN:%.*]], label [[CALLSTACKCHECKFAILBLK:%.*]], !prof !0
+; CHECK:       SP_return:
+; CHECK-NEXT:    ret void
+; CHECK:       CallStackCheckFailBlk:
+; CHECK-NEXT:    call void @__stack_chk_fail()
+; CHECK-NEXT:    unreachable
+;
+entry:
+  %retval = alloca i32, align 4
+  %a = alloca i32, align 4
+  %j = alloca i32*, align 8
+  store i32 0, i32* %retval
+  %load = load i32, i32* %a, align 4
+  %add = add nsw i32 %load, 1
+  store i32 %add, i32* %a, align 4
+
+  cmpxchg i32** %j, i32* null, i32* %a seq_cst monotonic
+  ret void
+}
+
+define void @memset_captures(i64 %c) #0 {
+; CHECK-LABEL: @memset_captures(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[STACKGUARDSLOT:%.*]] = alloca i8*
+; CHECK-NEXT:    [[STACKGUARD:%.*]] = load volatile i8*, i8* addrspace(257)* inttoptr (i32 40 to i8* addrspace(257)*)
+; CHECK-NEXT:    call void @llvm.stackprotector(i8* [[STACKGUARD]], i8** [[STACKGUARDSLOT]])
+; CHECK-NEXT:    [[CADDR:%.*]] = alloca i64, align 8
+; CHECK-NEXT:    store i64 %c, i64* [[CADDR]], align 8
+; CHECK-NEXT:    [[I:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[IPTR:%.*]] = bitcast i32* [[I]] to i8*
+; CHECK-NEXT:    [[COUNT:%.*]] = load i64, i64* [[CADDR]], align 8
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* align 4 [[IPTR]], i8 0, i64 [[COUNT]], i1 false)
+; CHECK-NEXT:    [[STACKGUARD1:%.*]] = load volatile i8*, i8* addrspace(257)* inttoptr (i32 40 to i8* addrspace(257)*)
+; CHECK-NEXT:    [[TMP1:%.*]] = load volatile i8*, i8** [[STACKGUARDSLOT]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i8* [[STACKGUARD1]], [[TMP1]]
+; CHECK-NEXT:    br i1 [[TMP2]], label [[SP_RETURN:%.*]], label [[CALLSTACKCHECKFAILBLK:%.*]], !prof !0
+; CHECK:       SP_return:
+; CHECK-NEXT:    ret void
+; CHECK:       CallStackCheckFailBlk:
+; CHECK-NEXT:    call void @__stack_chk_fail()
+; CHECK-NEXT:    unreachable
+;
+entry:
+  %c.addr = alloca i64, align 8
+  store i64 %c, i64* %c.addr, align 8
+  %i = alloca i32, align 4
+  %i.ptr = bitcast i32* %i to i8*
+  %count = load i64, i64* %c.addr, align 8
+  call void @llvm.memset.p0i8.i64(i8* align 4 %i.ptr, i8 0, i64 %count, i1 false)
+  ret void
+}
+
+declare void @llvm.memset.p0i8.i64(i8* nocapture writeonly, i8, i64, i1 immarg)
+
+attributes #0 = { sspstrong }
diff --git a/llvm/test/CodeGen/X86/stack-protector-strong-macho-win32-xor.ll b/llvm/test/CodeGen/X86/stack-protector-strong-macho-win32-xor.ll
new file mode 100644
index 000000000..f5647c341
--- /dev/null
+++ b/llvm/test/CodeGen/X86/stack-protector-strong-macho-win32-xor.ll
@@ -0,0 +1,57 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=x86_64-pc-windows-macho -O0 < %s -o - | FileCheck %s
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-macho"
+
+; This test checks that on Win32 MachO targets we don't xor the cookie with rbp before checking.
+
+@.str = private unnamed_addr constant [15 x i8] c"Hello World!\0A \00", align 1
+define dso_local i32 @main(i32 %argc, i8** %argv, ...) #0 {
+; CHECK-LABEL: main:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movq %rsp, %rbp
+; CHECK-NEXT:    .cfi_def_cfa_register %rbp
+; CHECK-NEXT:    subq $336, %rsp ## imm = 0x150
+; CHECK-NEXT:    movq ___security_cookie@{{.*}}(%rip), %rax
+; CHECK-NEXT:    movq (%rax), %rax
+; CHECK-NEXT:    movq %rax, -8(%rbp)
+; CHECK-NEXT:    movl %ecx, -276(%rbp)
+; CHECK-NEXT:    movq %rdx, -288(%rbp)
+; CHECK-NEXT:    movslq -276(%rbp), %rax
+; CHECK-NEXT:    movb $1, -272(%rbp,%rax)
+; CHECK-NEXT:    leaq {{.*}}(%rip), %rcx
+; CHECK-NEXT:    callq _printf
+; CHECK-NEXT:    movq -8(%rbp), %rcx
+; CHECK-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) ## 4-byte Spill
+; CHECK-NEXT:    callq ___security_check_cookie
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    addq $336, %rsp ## imm = 0x150
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    retq
+entry:
+  %argc.addr = alloca i32, align 4
+  %argv.addr = alloca i8**, align 8
+  %Buffer = alloca [256 x i8], align 16
+  store i32 %argc, i32* %argc.addr, align 4
+  store i8** %argv, i8*** %argv.addr, align 8
+  %0 = load i32, i32* %argc.addr, align 4
+  %idxprom = sext i32 %0 to i64
+  %arrayidx = getelementptr inbounds [256 x i8], [256 x i8]* %Buffer, i64 0, i64 %idxprom
+  store i8 1, i8* %arrayidx, align 1
+  %call = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([15 x i8], [15 x i8]* @.str, i64 0, i64 0))
+  ret i32 0
+}
+declare dso_local i32 @printf(i8*, ...) #1
+
+attributes #0 = { sspstrong "frame-pointer"="all" "stack-protector-buffer-size"="8"}
+attributes #1 = { "frame-pointer"="all" "stack-protector-buffer-size"="8" }
+
+!llvm.module.flags = !{!0, !1}
+!llvm.ident = !{!2}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 7, !"PIC Level", i32 2}
+!2 = !{!"clang version 10.0.0"}
diff --git a/llvm/test/CodeGen/X86/stack-protector-vreg-to-vreg-copy.ll b/llvm/test/CodeGen/X86/stack-protector-vreg-to-vreg-copy.ll
index 7578e2222..d526ca133 100644
--- a/llvm/test/CodeGen/X86/stack-protector-vreg-to-vreg-copy.ll
+++ b/llvm/test/CodeGen/X86/stack-protector-vreg-to-vreg-copy.ll
@@ -56,6 +56,6 @@ entry:
 
 declare void @f(i32) #1
 
-attributes #0 = { nounwind sspreq "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind sspreq "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind }
diff --git a/llvm/test/CodeGen/X86/stack-protector.ll b/llvm/test/CodeGen/X86/stack-protector.ll
index 874666fa4..1dacd9eb7 100644
--- a/llvm/test/CodeGen/X86/stack-protector.ll
+++ b/llvm/test/CodeGen/X86/stack-protector.ll
@@ -4087,8 +4087,8 @@ define i32 @IgnoreIntrinsicTest() #1 {
   %1 = alloca i32, align 4
   %2 = bitcast i32* %1 to i8*
   call void @llvm.lifetime.start.p0i8(i64 4, i8* nonnull %2)
-  store i32 1, i32* %1, align 4
-  %3 = load i32, i32* %1, align 4
+  store volatile i32 1, i32* %1, align 4
+  %3 = load volatile i32, i32* %1, align 4
   %4 = mul nsw i32 %3, 42
   call void @llvm.lifetime.end.p0i8(i64 4, i8* nonnull %2)
   ret i32 %4
diff --git a/llvm/test/CodeGen/X86/stack-size-section.ll b/llvm/test/CodeGen/X86/stack-size-section.ll
index 7e7ba5b9d..e82962596 100644
--- a/llvm/test/CodeGen/X86/stack-size-section.ll
+++ b/llvm/test/CodeGen/X86/stack-size-section.ll
@@ -46,4 +46,4 @@ define void @dynalloc(i32 %N) #0 {
   ret void
 }
 
-attributes #0 = { "no-frame-pointer-elim"="true" }
+attributes #0 = { "frame-pointer"="all" }
diff --git a/llvm/test/CodeGen/X86/stack_guard_remat.ll b/llvm/test/CodeGen/X86/stack_guard_remat.ll
index cc3cd6b08..91d8265d2 100644
--- a/llvm/test/CodeGen/X86/stack_guard_remat.ll
+++ b/llvm/test/CodeGen/X86/stack_guard_remat.ll
@@ -25,4 +25,4 @@ declare void @foo3(i32*)
 ; Function Attrs: nounwind
 declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture)
 
-attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/X86/stackmap-nops.ll b/llvm/test/CodeGen/X86/stackmap-nops.ll
index 08fee2ecd..e0f40d1d3 100644
--- a/llvm/test/CodeGen/X86/stackmap-nops.ll
+++ b/llvm/test/CodeGen/X86/stackmap-nops.ll
@@ -1,198 +1,304 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=corei7 | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=znver1 | FileCheck --check-prefix=CHECK-AMD %s
+
+; Check that we get a fast sequence of nops on each platform.
 
 define void @nop_test() {
-entry:
 ; CHECK-LABEL: nop_test:
-; CHECK:      nop
-; CHECK:      xchgw %ax, %ax
-; CHECK:      nopl (%rax)
-; CHECK:      nopl 8(%rax)
-; CHECK:      nopl 8(%rax,%rax)
-; CHECK:      nopw 8(%rax,%rax)
-; CHECK:      nopl 512(%rax)
-; CHECK:      nopl 512(%rax,%rax)
-; CHECK:      nopw 512(%rax,%rax)
-; CHECK:      nopw %cs:512(%rax,%rax)
-
-; 11
-; CHECK:      .byte 102
-; CHECK-NEXT: nopw %cs:512(%rax,%rax)
-
-; 12
-; CHECK:      .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: nopw %cs:512(%rax,%rax)
-
-; 13
-; CHECK:      .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: nopw %cs:512(%rax,%rax)
-
-; 14
-; CHECK:      .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: nopw %cs:512(%rax,%rax)
-
-; 15
-; CHECK:      .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: nopw %cs:512(%rax,%rax)
-
-; 16
-; CHECK:      .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: nopw %cs:512(%rax,%rax)
-; CHECK-NEXT: nop
-
-; 17
-; CHECK:      .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: nopw %cs:512(%rax,%rax)
-; CHECK-NEXT: xchgw %ax, %ax
-
-; 18
-; CHECK:      .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: nopw %cs:512(%rax,%rax)
-; CHECK-NEXT: nopl (%rax)
-
-; 19
-; CHECK:      .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: nopw %cs:512(%rax,%rax)
-; CHECK-NEXT: nopl 8(%rax)
-
-; 20
-; CHECK:      .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: nopw %cs:512(%rax,%rax)
-; CHECK-NEXT: nopl 8(%rax,%rax)
-
-; 21
-; CHECK:      .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: nopw %cs:512(%rax,%rax)
-; CHECK-NEXT: nopw 8(%rax,%rax)
-
-; 22
-; CHECK:      .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: nopw %cs:512(%rax,%rax)
-; CHECK-NEXT: nopl 512(%rax)
-
-; 23
-; CHECK:      .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: nopw %cs:512(%rax,%rax)
-; CHECK-NEXT: nopl 512(%rax,%rax)
-
-; 24
-; CHECK:      .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: nopw %cs:512(%rax,%rax)
-; CHECK-NEXT: nopw 512(%rax,%rax)
-
-; 25
-; CHECK:      .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: nopw %cs:512(%rax,%rax)
-; CHECK-NEXT: nopw %cs:512(%rax,%rax)
-
-; 26
-; CHECK:      .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: nopw %cs:512(%rax,%rax)
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: nopw %cs:512(%rax,%rax)
-
-; 27
-; CHECK:      .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: nopw %cs:512(%rax,%rax)
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: nopw %cs:512(%rax,%rax)
-
-; 28
-; CHECK:      .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: nopw %cs:512(%rax,%rax)
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: nopw %cs:512(%rax,%rax)
-
-;29
-; CHECK:      .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: nopw %cs:512(%rax,%rax)
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: nopw %cs:512(%rax,%rax)
-
-; 30
-; CHECK:      .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: nopw %cs:512(%rax,%rax)
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: .byte 102
-; CHECK-NEXT: nopw %cs:512(%rax,%rax)
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movq %rsp, %rbp
+; CHECK-NEXT:    .cfi_def_cfa_register %rbp
+; CHECK-NEXT:  Ltmp0:
+; CHECK-NEXT:  Ltmp1:
+; CHECK-NEXT:    nop
+; CHECK-NEXT:  Ltmp2:
+; CHECK-NEXT:    xchgw %ax, %ax
+; CHECK-NEXT:  Ltmp3:
+; CHECK-NEXT:    nopl (%rax)
+; CHECK-NEXT:  Ltmp4:
+; CHECK-NEXT:    nopl 8(%rax)
+; CHECK-NEXT:  Ltmp5:
+; CHECK-NEXT:    nopl 8(%rax,%rax)
+; CHECK-NEXT:  Ltmp6:
+; CHECK-NEXT:    nopw 8(%rax,%rax)
+; CHECK-NEXT:  Ltmp7:
+; CHECK-NEXT:    nopl 512(%rax)
+; CHECK-NEXT:  Ltmp8:
+; CHECK-NEXT:    nopl 512(%rax,%rax)
+; CHECK-NEXT:  Ltmp9:
+; CHECK-NEXT:    nopw 512(%rax,%rax)
+; CHECK-NEXT:  Ltmp10:
+; CHECK-NEXT:    nopw %cs:512(%rax,%rax)
+; CHECK-NEXT:  Ltmp11:
+; CHECK-NEXT:    nopw %cs:512(%rax,%rax)
+; CHECK-NEXT:    nop
+; CHECK-NEXT:  Ltmp12:
+; CHECK-NEXT:    nopw %cs:512(%rax,%rax)
+; CHECK-NEXT:    xchgw %ax, %ax
+; CHECK-NEXT:  Ltmp13:
+; CHECK-NEXT:    nopw %cs:512(%rax,%rax)
+; CHECK-NEXT:    nopl (%rax)
+; CHECK-NEXT:  Ltmp14:
+; CHECK-NEXT:    nopw %cs:512(%rax,%rax)
+; CHECK-NEXT:    nopl 8(%rax)
+; CHECK-NEXT:  Ltmp15:
+; CHECK-NEXT:    nopw %cs:512(%rax,%rax)
+; CHECK-NEXT:    nopl 8(%rax,%rax)
+; CHECK-NEXT:  Ltmp16:
+; CHECK-NEXT:    nopw %cs:512(%rax,%rax)
+; CHECK-NEXT:    nopw 8(%rax,%rax)
+; CHECK-NEXT:  Ltmp17:
+; CHECK-NEXT:    nopw %cs:512(%rax,%rax)
+; CHECK-NEXT:    nopl 512(%rax)
+; CHECK-NEXT:  Ltmp18:
+; CHECK-NEXT:    nopw %cs:512(%rax,%rax)
+; CHECK-NEXT:    nopl 512(%rax,%rax)
+; CHECK-NEXT:  Ltmp19:
+; CHECK-NEXT:    nopw %cs:512(%rax,%rax)
+; CHECK-NEXT:    nopw 512(%rax,%rax)
+; CHECK-NEXT:  Ltmp20:
+; CHECK-NEXT:    nopw %cs:512(%rax,%rax)
+; CHECK-NEXT:    nopw %cs:512(%rax,%rax)
+; CHECK-NEXT:  Ltmp21:
+; CHECK-NEXT:    nopw %cs:512(%rax,%rax)
+; CHECK-NEXT:    nopw %cs:512(%rax,%rax)
+; CHECK-NEXT:    nop
+; CHECK-NEXT:  Ltmp22:
+; CHECK-NEXT:    nopw %cs:512(%rax,%rax)
+; CHECK-NEXT:    nopw %cs:512(%rax,%rax)
+; CHECK-NEXT:    xchgw %ax, %ax
+; CHECK-NEXT:  Ltmp23:
+; CHECK-NEXT:    nopw %cs:512(%rax,%rax)
+; CHECK-NEXT:    nopw %cs:512(%rax,%rax)
+; CHECK-NEXT:    nopl (%rax)
+; CHECK-NEXT:  Ltmp24:
+; CHECK-NEXT:    nopw %cs:512(%rax,%rax)
+; CHECK-NEXT:    nopw %cs:512(%rax,%rax)
+; CHECK-NEXT:    nopl 8(%rax)
+; CHECK-NEXT:  Ltmp25:
+; CHECK-NEXT:    nopw %cs:512(%rax,%rax)
+; CHECK-NEXT:    nopw %cs:512(%rax,%rax)
+; CHECK-NEXT:    nopl 8(%rax,%rax)
+; CHECK-NEXT:  Ltmp26:
+; CHECK-NEXT:    nopw %cs:512(%rax,%rax)
+; CHECK-NEXT:    nopw %cs:512(%rax,%rax)
+; CHECK-NEXT:    nopw 8(%rax,%rax)
+; CHECK-NEXT:  Ltmp27:
+; CHECK-NEXT:    nopw %cs:512(%rax,%rax)
+; CHECK-NEXT:    nopw %cs:512(%rax,%rax)
+; CHECK-NEXT:    nopl 512(%rax)
+; CHECK-NEXT:  Ltmp28:
+; CHECK-NEXT:    nopw %cs:512(%rax,%rax)
+; CHECK-NEXT:    nopw %cs:512(%rax,%rax)
+; CHECK-NEXT:    nopl 512(%rax,%rax)
+; CHECK-NEXT:  Ltmp29:
+; CHECK-NEXT:    nopw %cs:512(%rax,%rax)
+; CHECK-NEXT:    nopw %cs:512(%rax,%rax)
+; CHECK-NEXT:    nopw 512(%rax,%rax)
+; CHECK-NEXT:  Ltmp30:
+; CHECK-NEXT:    nopw %cs:512(%rax,%rax)
+; CHECK-NEXT:    nopw %cs:512(%rax,%rax)
+; CHECK-NEXT:    nopw %cs:512(%rax,%rax)
+; CHECK-NEXT:  Ltmp31:
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    retq
+;
+; CHECK-AMD-LABEL: nop_test:
+; CHECK-AMD:       ## %bb.0: ## %entry
+; CHECK-AMD-NEXT:    pushq %rbp
+; CHECK-AMD-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-AMD-NEXT:    .cfi_offset %rbp, -16
+; CHECK-AMD-NEXT:    movq %rsp, %rbp
+; CHECK-AMD-NEXT:    .cfi_def_cfa_register %rbp
+; CHECK-AMD-NEXT:  Ltmp0:
+; CHECK-AMD-NEXT:  Ltmp1:
+; CHECK-AMD-NEXT:    nop
+; CHECK-AMD-NEXT:  Ltmp2:
+; CHECK-AMD-NEXT:    xchgw %ax, %ax
+; CHECK-AMD-NEXT:  Ltmp3:
+; CHECK-AMD-NEXT:    nopl (%rax)
+; CHECK-AMD-NEXT:  Ltmp4:
+; CHECK-AMD-NEXT:    nopl 8(%rax)
+; CHECK-AMD-NEXT:  Ltmp5:
+; CHECK-AMD-NEXT:    nopl 8(%rax,%rax)
+; CHECK-AMD-NEXT:  Ltmp6:
+; CHECK-AMD-NEXT:    nopw 8(%rax,%rax)
+; CHECK-AMD-NEXT:  Ltmp7:
+; CHECK-AMD-NEXT:    nopl 512(%rax)
+; CHECK-AMD-NEXT:  Ltmp8:
+; CHECK-AMD-NEXT:    nopl 512(%rax,%rax)
+; CHECK-AMD-NEXT:  Ltmp9:
+; CHECK-AMD-NEXT:    nopw 512(%rax,%rax)
+; CHECK-AMD-NEXT:  Ltmp10:
+; CHECK-AMD-NEXT:    nopw %cs:512(%rax,%rax)
+; CHECK-AMD-NEXT:  Ltmp11:
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    nopw %cs:512(%rax,%rax)
+; CHECK-AMD-NEXT:  Ltmp12:
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    nopw %cs:512(%rax,%rax)
+; CHECK-AMD-NEXT:  Ltmp13:
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    nopw %cs:512(%rax,%rax)
+; CHECK-AMD-NEXT:  Ltmp14:
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    nopw %cs:512(%rax,%rax)
+; CHECK-AMD-NEXT:  Ltmp15:
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    nopw %cs:512(%rax,%rax)
+; CHECK-AMD-NEXT:  Ltmp16:
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    nopw %cs:512(%rax,%rax)
+; CHECK-AMD-NEXT:    nop
+; CHECK-AMD-NEXT:  Ltmp17:
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    nopw %cs:512(%rax,%rax)
+; CHECK-AMD-NEXT:    xchgw %ax, %ax
+; CHECK-AMD-NEXT:  Ltmp18:
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    nopw %cs:512(%rax,%rax)
+; CHECK-AMD-NEXT:    nopl (%rax)
+; CHECK-AMD-NEXT:  Ltmp19:
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    nopw %cs:512(%rax,%rax)
+; CHECK-AMD-NEXT:    nopl 8(%rax)
+; CHECK-AMD-NEXT:  Ltmp20:
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    nopw %cs:512(%rax,%rax)
+; CHECK-AMD-NEXT:    nopl 8(%rax,%rax)
+; CHECK-AMD-NEXT:  Ltmp21:
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    nopw %cs:512(%rax,%rax)
+; CHECK-AMD-NEXT:    nopw 8(%rax,%rax)
+; CHECK-AMD-NEXT:  Ltmp22:
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    nopw %cs:512(%rax,%rax)
+; CHECK-AMD-NEXT:    nopl 512(%rax)
+; CHECK-AMD-NEXT:  Ltmp23:
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    nopw %cs:512(%rax,%rax)
+; CHECK-AMD-NEXT:    nopl 512(%rax,%rax)
+; CHECK-AMD-NEXT:  Ltmp24:
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    nopw %cs:512(%rax,%rax)
+; CHECK-AMD-NEXT:    nopw 512(%rax,%rax)
+; CHECK-AMD-NEXT:  Ltmp25:
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    nopw %cs:512(%rax,%rax)
+; CHECK-AMD-NEXT:    nopw %cs:512(%rax,%rax)
+; CHECK-AMD-NEXT:  Ltmp26:
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    nopw %cs:512(%rax,%rax)
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    nopw %cs:512(%rax,%rax)
+; CHECK-AMD-NEXT:  Ltmp27:
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    nopw %cs:512(%rax,%rax)
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    nopw %cs:512(%rax,%rax)
+; CHECK-AMD-NEXT:  Ltmp28:
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    nopw %cs:512(%rax,%rax)
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    nopw %cs:512(%rax,%rax)
+; CHECK-AMD-NEXT:  Ltmp29:
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    nopw %cs:512(%rax,%rax)
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    nopw %cs:512(%rax,%rax)
+; CHECK-AMD-NEXT:  Ltmp30:
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    nopw %cs:512(%rax,%rax)
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    .byte 102
+; CHECK-AMD-NEXT:    nopw %cs:512(%rax,%rax)
+; CHECK-AMD-NEXT:  Ltmp31:
+; CHECK-AMD-NEXT:    popq %rbp
+; CHECK-AMD-NEXT:    retq
+entry:
   tail call void (i64, i32, ...) @llvm.experimental.stackmap(i64  0, i32  0)
   tail call void (i64, i32, ...) @llvm.experimental.stackmap(i64  1, i32  1)
   tail call void (i64, i32, ...) @llvm.experimental.stackmap(i64  2, i32  2)
diff --git a/llvm/test/CodeGen/X86/statepoint-no-realign-stack.ll b/llvm/test/CodeGen/X86/statepoint-no-realign-stack.ll
new file mode 100644
index 000000000..40e721bfa
--- /dev/null
+++ b/llvm/test/CodeGen/X86/statepoint-no-realign-stack.ll
@@ -0,0 +1,107 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -verify-machineinstrs -mcpu=skylake < %s | FileCheck %s
+
+target datalayout = "e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-linux-gnu"
+
+; Shows a case where we spill a 32 byte value onto a stack which is only
+; 16 byte aligned.  With stack realignment, we can use an aligned spill slot
+; (if we think it's profitable), but without realignment, using a stack
+; slot which is 32 byte aligned or a store which expects 32 byte alignment
+; is incorrect.
+
+declare void @foo()
+define void @can_realign(<8 x i32>* %p) {
+; CHECK-LABEL: can_realign:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movq %rsp, %rbp
+; CHECK-NEXT:    .cfi_def_cfa_register %rbp
+; CHECK-NEXT:    andq $-32, %rsp
+; CHECK-NEXT:    subq $64, %rsp
+; CHECK-NEXT:    vmovaps (%rdi), %ymm0
+; CHECK-NEXT:    vmovaps %ymm0, (%rsp)
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    callq foo
+; CHECK-NEXT:  .Ltmp0:
+; CHECK-NEXT:    movq %rbp, %rsp
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa %rsp, 8
+; CHECK-NEXT:    retq
+  %val = load <8 x i32>, <8 x i32>* %p, align 32
+  call void @foo() ["deopt" (<8 x i32> %val)]
+  ret void
+}
+
+define void @no_realign(<8 x i32>* %p) "no-realign-stack" {
+; CHECK-LABEL: no_realign:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subq $40, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    vmovaps (%rdi), %ymm0
+; CHECK-NEXT:    vmovups %ymm0, (%rsp)
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    callq foo
+; CHECK-NEXT:  .Ltmp1:
+; CHECK-NEXT:    addq $40, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
+  %val = load <8 x i32>, <8 x i32>* %p, align 32
+  call void @foo() ["deopt" (<8 x i32> %val)]
+  ret void
+}
+
+;; Next batch are similiar to the above, but require a reload of the
+;; spilled value as well.
+
+define <4 x i8 addrspace(1)*> @spillfill_can_realign(<4 x i8 addrspace(1)*> %obj) gc "statepoint-example" {
+; CHECK-LABEL: spillfill_can_realign:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movq %rsp, %rbp
+; CHECK-NEXT:    .cfi_def_cfa_register %rbp
+; CHECK-NEXT:    andq $-32, %rsp
+; CHECK-NEXT:    subq $64, %rsp
+; CHECK-NEXT:    vmovaps %ymm0, (%rsp)
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    callq do_safepoint
+; CHECK-NEXT:  .Ltmp2:
+; CHECK-NEXT:    vmovaps (%rsp), %ymm0
+; CHECK-NEXT:    movq %rbp, %rsp
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa %rsp, 8
+; CHECK-NEXT:    retq
+entry:
+  %safepoint_token = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0, <4 x i8 addrspace(1)*> %obj)
+  %obj.relocated = call coldcc <4 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v4p1i8(token %safepoint_token, i32 7, i32 7) ; (%obj, %obj)
+  ret <4 x i8 addrspace(1)*> %obj.relocated
+}
+
+define <4 x i8 addrspace(1)*> @spillfill_no_realign(<4 x i8 addrspace(1)*> %obj) "no-realign-stack" gc "statepoint-example" {
+; CHECK-LABEL: spillfill_no_realign:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    subq $40, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    vmovups %ymm0, (%rsp)
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    callq do_safepoint
+; CHECK-NEXT:  .Ltmp3:
+; CHECK-NEXT:    vmovups (%rsp), %ymm0
+; CHECK-NEXT:    addq $40, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
+entry:
+  %safepoint_token = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0, <4 x i8 addrspace(1)*> %obj)
+  %obj.relocated = call coldcc <4 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v4p1i8(token %safepoint_token, i32 7, i32 7) ; (%obj, %obj)
+  ret <4 x i8 addrspace(1)*> %obj.relocated
+}
+
+declare void @do_safepoint()
+
+declare token @llvm.experimental.gc.statepoint.p0f_isVoidf(i64, i32, void ()*, i32, i32, ...)
+declare i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token, i32, i32)
+declare <4 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v4p1i8(token, i32, i32)
diff --git a/llvm/test/CodeGen/X86/store-global-address.ll b/llvm/test/CodeGen/X86/store-global-address.ll
index 31bb0d59c..98b502bd9 100644
--- a/llvm/test/CodeGen/X86/store-global-address.ll
+++ b/llvm/test/CodeGen/X86/store-global-address.ll
@@ -1,9 +1,14 @@
-; RUN: llc < %s -mtriple=i686-- | grep movl | count 1
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- | FileCheck %s
 
 @dst = global i32 0             ; <i32*> [#uses=1]
 @ptr = global i32* null         ; <i32**> [#uses=1]
 
 define void @test() {
+; CHECK-LABEL: test:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl $dst, ptr
+; CHECK-NEXT:    retl
         store i32* @dst, i32** @ptr
         ret void
 }
diff --git a/llvm/test/CodeGen/X86/store-narrow.ll b/llvm/test/CodeGen/X86/store-narrow.ll
index 9fc166a53..f71f2eefa 100644
--- a/llvm/test/CodeGen/X86/store-narrow.ll
+++ b/llvm/test/CodeGen/X86/store-narrow.ll
@@ -1,10 +1,22 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; rdar://7860110
-; RUN: llc -mtriple=x86_64-apple-darwin10.2 -asm-verbose=false < %s | FileCheck %s -check-prefix=X64
-; RUN: llc -mtriple=i686-apple-darwin10.2 -asm-verbose=false -fixup-byte-word-insts=1 < %s | FileCheck %s -check-prefix=X32 -check-prefix=X32-BWON
-; RUN: llc -mtriple=i686-apple-darwin10.2 -asm-verbose=false -fixup-byte-word-insts=0 < %s | FileCheck %s -check-prefix=X32 -check-prefix=X32-BWOFF
+; RUN: llc -mtriple=x86_64-apple-darwin10.2 < %s | FileCheck %s -check-prefix=X64
+; RUN: llc -mtriple=i686-apple-darwin10.2 -fixup-byte-word-insts=1 < %s | FileCheck %s -check-prefix=X32 -check-prefix=X32-BWON
+; RUN: llc -mtriple=i686-apple-darwin10.2 -fixup-byte-word-insts=0 < %s | FileCheck %s -check-prefix=X32 -check-prefix=X32-BWOFF
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 define void @test1(i32* nocapture %a0, i8 zeroext %a1) nounwind ssp {
+; X64-LABEL: test1:
+; X64:       ## %bb.0: ## %entry
+; X64-NEXT:    movb %sil, (%rdi)
+; X64-NEXT:    retq
+;
+; X32-LABEL: test1:
+; X32:       ## %bb.0: ## %entry
+; X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT:    movb %al, (%ecx)
+; X32-NEXT:    retl
 entry:
   %A = load i32, i32* %a0, align 4
   %B = and i32 %A, -256     ; 0xFFFFFF00
@@ -12,16 +24,20 @@ entry:
   %D = or i32 %C, %B
   store i32 %D, i32* %a0, align 4
   ret void
-
-; X64-LABEL: test1:
-; X64: movb	%sil, (%rdi)
-
-; X32-LABEL: test1:
-; X32: movb	8(%esp), %al
-; X32: movb	%al, (%{{.*}})
 }
 
 define void @test2(i32* nocapture %a0, i8 zeroext %a1) nounwind ssp {
+; X64-LABEL: test2:
+; X64:       ## %bb.0: ## %entry
+; X64-NEXT:    movb %sil, 1(%rdi)
+; X64-NEXT:    retq
+;
+; X32-LABEL: test2:
+; X32:       ## %bb.0: ## %entry
+; X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT:    movb %al, 1(%ecx)
+; X32-NEXT:    retl
 entry:
   %A = load i32, i32* %a0, align 4
   %B = and i32 %A, -65281    ; 0xFFFF00FF
@@ -30,15 +46,27 @@ entry:
   %D = or i32 %B, %CS
   store i32 %D, i32* %a0, align 4
   ret void
-; X64-LABEL: test2:
-; X64: movb	%sil, 1(%rdi)
-
-; X32-LABEL: test2:
-; X32: movb	8(%esp), %[[REG:[abcd]]]l
-; X32: movb	%[[REG]]l, 1(%{{.*}})
 }
 
 define void @test3(i32* nocapture %a0, i16 zeroext %a1) nounwind ssp {
+; X64-LABEL: test3:
+; X64:       ## %bb.0: ## %entry
+; X64-NEXT:    movw %si, (%rdi)
+; X64-NEXT:    retq
+;
+; X32-BWON-LABEL: test3:
+; X32-BWON:       ## %bb.0: ## %entry
+; X32-BWON-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X32-BWON-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-BWON-NEXT:    movw %ax, (%ecx)
+; X32-BWON-NEXT:    retl
+;
+; X32-BWOFF-LABEL: test3:
+; X32-BWOFF:       ## %bb.0: ## %entry
+; X32-BWOFF-NEXT:    movw {{[0-9]+}}(%esp), %ax
+; X32-BWOFF-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-BWOFF-NEXT:    movw %ax, (%ecx)
+; X32-BWOFF-NEXT:    retl
 entry:
   %A = load i32, i32* %a0, align 4
   %B = and i32 %A, -65536    ; 0xFFFF0000
@@ -46,16 +74,27 @@ entry:
   %D = or i32 %B, %C
   store i32 %D, i32* %a0, align 4
   ret void
-; X64-LABEL: test3:
-; X64: movw	%si, (%rdi)
-
-; X32-LABEL: test3:
-; X32-BWON:  movzwl	8(%esp), %eax
-; X32-BWOFF: movw	8(%esp), %ax
-; X32: movw	%ax, (%{{.*}})
 }
 
 define void @test4(i32* nocapture %a0, i16 zeroext %a1) nounwind ssp {
+; X64-LABEL: test4:
+; X64:       ## %bb.0: ## %entry
+; X64-NEXT:    movw %si, 2(%rdi)
+; X64-NEXT:    retq
+;
+; X32-BWON-LABEL: test4:
+; X32-BWON:       ## %bb.0: ## %entry
+; X32-BWON-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X32-BWON-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-BWON-NEXT:    movw %ax, 2(%ecx)
+; X32-BWON-NEXT:    retl
+;
+; X32-BWOFF-LABEL: test4:
+; X32-BWOFF:       ## %bb.0: ## %entry
+; X32-BWOFF-NEXT:    movw {{[0-9]+}}(%esp), %ax
+; X32-BWOFF-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-BWOFF-NEXT:    movw %ax, 2(%ecx)
+; X32-BWOFF-NEXT:    retl
 entry:
   %A = load i32, i32* %a0, align 4
   %B = and i32 %A, 65535    ; 0x0000FFFF
@@ -64,16 +103,27 @@ entry:
   %D = or i32 %B, %CS
   store i32 %D, i32* %a0, align 4
   ret void
-; X64-LABEL: test4:
-; X64: movw	%si, 2(%rdi)
-
-; X32-LABEL: test4:
-; X32-BWON:  movzwl	8(%esp), %e[[REG:[abcd]]]x
-; X32-BWOFF: movw	8(%esp), %[[REG:[abcd]]]x
-; X32: movw	%[[REG]]x, 2(%{{.*}})
 }
 
 define void @test5(i64* nocapture %a0, i16 zeroext %a1) nounwind ssp {
+; X64-LABEL: test5:
+; X64:       ## %bb.0: ## %entry
+; X64-NEXT:    movw %si, 2(%rdi)
+; X64-NEXT:    retq
+;
+; X32-BWON-LABEL: test5:
+; X32-BWON:       ## %bb.0: ## %entry
+; X32-BWON-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X32-BWON-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-BWON-NEXT:    movw %ax, 2(%ecx)
+; X32-BWON-NEXT:    retl
+;
+; X32-BWOFF-LABEL: test5:
+; X32-BWOFF:       ## %bb.0: ## %entry
+; X32-BWOFF-NEXT:    movw {{[0-9]+}}(%esp), %ax
+; X32-BWOFF-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-BWOFF-NEXT:    movw %ax, 2(%ecx)
+; X32-BWOFF-NEXT:    retl
 entry:
   %A = load i64, i64* %a0, align 4
   %B = and i64 %A, -4294901761    ; 0xFFFFFFFF0000FFFF
@@ -82,16 +132,20 @@ entry:
   %D = or i64 %B, %CS
   store i64 %D, i64* %a0, align 4
   ret void
-; X64-LABEL: test5:
-; X64: movw	%si, 2(%rdi)
-
-; X32-LABEL: test5:
-; X32-BWON:  movzwl	8(%esp), %e[[REG:[abcd]]]x
-; X32-BWOFF: movw	8(%esp), %[[REG:[abcd]]]x
-; X32: movw	%[[REG]]x, 2(%{{.*}})
 }
 
 define void @test6(i64* nocapture %a0, i8 zeroext %a1) nounwind ssp {
+; X64-LABEL: test6:
+; X64:       ## %bb.0: ## %entry
+; X64-NEXT:    movb %sil, 5(%rdi)
+; X64-NEXT:    retq
+;
+; X32-LABEL: test6:
+; X32:       ## %bb.0: ## %entry
+; X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT:    movb %al, 5(%ecx)
+; X32-NEXT:    retl
 entry:
   %A = load i64, i64* %a0, align 4
   %B = and i64 %A, -280375465082881    ; 0xFFFF00FFFFFFFFFF
@@ -100,16 +154,23 @@ entry:
   %D = or i64 %B, %CS
   store i64 %D, i64* %a0, align 4
   ret void
-; X64-LABEL: test6:
-; X64: movb	%sil, 5(%rdi)
-
-
-; X32-LABEL: test6:
-; X32: movb	8(%esp), %[[REG:[abcd]l]]
-; X32: movb	%[[REG]], 5(%{{.*}})
 }
 
 define i32 @test7(i64* nocapture %a0, i8 zeroext %a1, i32* %P2) nounwind {
+; X64-LABEL: test7:
+; X64:       ## %bb.0: ## %entry
+; X64-NEXT:    movl (%rdx), %eax
+; X64-NEXT:    movb %sil, 5(%rdi)
+; X64-NEXT:    retq
+;
+; X32-LABEL: test7:
+; X32:       ## %bb.0: ## %entry
+; X32-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl (%eax), %eax
+; X32-NEXT:    movb %cl, 5(%edx)
+; X32-NEXT:    retl
 entry:
   %OtherLoad = load i32 , i32 *%P2
   %A = load i64, i64* %a0, align 4
@@ -119,23 +180,22 @@ entry:
   %D = or i64 %B, %CS
   store i64 %D, i64* %a0, align 4
   ret i32 %OtherLoad
-; X64-LABEL: test7:
-; X64: movb	%sil, 5(%rdi)
-
-
-; X32-LABEL: test7:
-; X32: movb	8(%esp), %[[REG:[abcd]l]]
-; X32: movb	%[[REG]], 5(%{{.*}})
 }
 
 ; PR7833
 
 @g_16 = internal global i32 -1
 
-; X64-LABEL: test8:
-; X64-NEXT: orb  $1, _g_16(%rip)
-; X64-NEXT: ret
 define void @test8() nounwind {
+; X64-LABEL: test8:
+; X64:       ## %bb.0:
+; X64-NEXT:    orb $1, {{.*}}(%rip)
+; X64-NEXT:    retq
+;
+; X32-LABEL: test8:
+; X32:       ## %bb.0:
+; X32-NEXT:    orb $1, _g_16
+; X32-NEXT:    retl
   %tmp = load i32, i32* @g_16
   store i32 0, i32* @g_16
   %or = or i32 %tmp, 1
@@ -143,10 +203,16 @@ define void @test8() nounwind {
   ret void
 }
 
-; X64-LABEL: test9:
-; X64-NEXT: orb $1, _g_16(%rip)
-; X64-NEXT: ret
 define void @test9() nounwind {
+; X64-LABEL: test9:
+; X64:       ## %bb.0:
+; X64-NEXT:    orb $1, {{.*}}(%rip)
+; X64-NEXT:    retq
+;
+; X32-LABEL: test9:
+; X32:       ## %bb.0:
+; X32-NEXT:    orb $1, _g_16
+; X32-NEXT:    retl
   %tmp = load i32, i32* @g_16
   %or = or i32 %tmp, 1
   store i32 %or, i32* @g_16
@@ -154,11 +220,20 @@ define void @test9() nounwind {
 }
 
 ; rdar://8494845 + PR8244
-; X64-LABEL: test10:
-; X64-NEXT: movsbl	(%rdi), %eax
-; X64-NEXT: shrl	$8, %eax
-; X64-NEXT: ret
 define i8 @test10(i8* %P) nounwind ssp {
+; X64-LABEL: test10:
+; X64:       ## %bb.0: ## %entry
+; X64-NEXT:    movsbl (%rdi), %eax
+; X64-NEXT:    shrl $8, %eax
+; X64-NEXT:    ## kill: def $al killed $al killed $eax
+; X64-NEXT:    retq
+;
+; X32-LABEL: test10:
+; X32:       ## %bb.0: ## %entry
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movsbl (%eax), %eax
+; X32-NEXT:    movb %ah, %al
+; X32-NEXT:    retl
 entry:
   %tmp = load i8, i8* %P, align 1
   %conv = sext i8 %tmp to i32
diff --git a/llvm/test/CodeGen/X86/store-zero-and-minus-one.ll b/llvm/test/CodeGen/X86/store-zero-and-minus-one.ll
index 97aa87e9b..12e276e6b 100644
--- a/llvm/test/CodeGen/X86/store-zero-and-minus-one.ll
+++ b/llvm/test/CodeGen/X86/store-zero-and-minus-one.ll
@@ -19,6 +19,23 @@ entry:
 
 }
 
+define void @zero_pgso(i32* %p) !prof !14 {
+; CHECK32-LABEL: zero_pgso:
+; CHECK32:       # %bb.0: # %entry
+; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK32-NEXT:    movl $0, (%eax)
+; CHECK32-NEXT:    retl
+;
+; CHECK64-LABEL: zero_pgso:
+; CHECK64:       # %bb.0: # %entry
+; CHECK64-NEXT:    movl $0, (%rdi)
+; CHECK64-NEXT:    retq
+entry:
+  store i32 0, i32* %p
+  ret void
+
+}
+
 define void @minus_one_optsize(i32* %p) optsize {
 ; CHECK32-LABEL: minus_one_optsize:
 ; CHECK32:       # %bb.0: # %entry
@@ -36,6 +53,22 @@ entry:
 
 }
 
+define void @minus_one_pgso(i32* %p) !prof !14 {
+; CHECK32-LABEL: minus_one_pgso:
+; CHECK32:       # %bb.0: # %entry
+; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK32-NEXT:    movl $-1, (%eax)
+; CHECK32-NEXT:    retl
+;
+; CHECK64-LABEL: minus_one_pgso:
+; CHECK64:       # %bb.0: # %entry
+; CHECK64-NEXT:    movl $-1, (%rdi)
+; CHECK64-NEXT:    retq
+entry:
+  store i32 -1, i32* %p
+  ret void
+
+}
 
 define void @zero_64(i64* %p) minsize {
 ; CHECK32-LABEL: zero_64:
@@ -244,3 +277,20 @@ entry:
   store volatile i16 -1, i16* %p
   ret void
 }
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 1, !"ProfileSummary", !1}
+!1 = !{!2, !3, !4, !5, !6, !7, !8, !9}
+!2 = !{!"ProfileFormat", !"InstrProf"}
+!3 = !{!"TotalCount", i64 10000}
+!4 = !{!"MaxCount", i64 10}
+!5 = !{!"MaxInternalCount", i64 1}
+!6 = !{!"MaxFunctionCount", i64 1000}
+!7 = !{!"NumCounts", i64 3}
+!8 = !{!"NumFunctions", i64 3}
+!9 = !{!"DetailedSummary", !10}
+!10 = !{!11, !12, !13}
+!11 = !{i32 10000, i64 100, i32 1}
+!12 = !{i32 999000, i64 100, i32 1}
+!13 = !{i32 999999, i64 1, i32 2}
+!14 = !{!"function_entry_count", i64 0}
diff --git a/llvm/test/CodeGen/X86/stores-merging.ll b/llvm/test/CodeGen/X86/stores-merging.ll
index 25c93c3e7..6420ac7dc 100644
--- a/llvm/test/CodeGen/X86/stores-merging.ll
+++ b/llvm/test/CodeGen/X86/stores-merging.ll
@@ -26,8 +26,9 @@ define void @redundant_stores_merging() {
 define void @redundant_stores_merging_reverse() {
 ; CHECK-LABEL: redundant_stores_merging_reverse:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    movabsq $1958505086977, %rax # imm = 0x1C800000001
+; CHECK-NEXT:    movabsq $528280977409, %rax # imm = 0x7B00000001
 ; CHECK-NEXT:    movq %rax, e+{{.*}}(%rip)
+; CHECK-NEXT:    movl $456, e+{{.*}}(%rip) # imm = 0x1C8
 ; CHECK-NEXT:    retq
   store i32 123, i32* getelementptr inbounds (%structTy, %structTy* @e, i64 0, i32 2), align 4
   store i32 456, i32* getelementptr inbounds (%structTy, %structTy* @e, i64 0, i32 2), align 4
@@ -219,3 +220,25 @@ define void @extract_vector_store_32_consecutive_bytes(<4 x i64> %v, i8* %ptr) #
   ret void
 }
 
+; https://bugs.llvm.org/show_bug.cgi?id=43446
+define void @pr43446_0(i64 %x) {
+; CHECK-LABEL: pr43446_0:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movb $1, (%rdi)
+; CHECK-NEXT:    retq
+  %a = inttoptr i64 %x to i8*
+  store i8 -2, i8* %a, align 1
+  %b = inttoptr i64 %x to i1*
+  store i1 true, i1* %b, align 1
+  ret void
+}
+define void @pr43446_1(i8* %a) {
+; CHECK-LABEL: pr43446_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movb $1, (%rdi)
+; CHECK-NEXT:    retq
+  store i8 -2, i8* %a, align 1
+  %b = bitcast i8* %a to i1*
+  store i1 true, i1* %b, align 1
+  ret void
+}
diff --git a/llvm/test/CodeGen/X86/storetrunc-fp.ll b/llvm/test/CodeGen/X86/storetrunc-fp.ll
index 0ead0bab4..3e84fc49a 100644
--- a/llvm/test/CodeGen/X86/storetrunc-fp.ll
+++ b/llvm/test/CodeGen/X86/storetrunc-fp.ll
@@ -1,6 +1,15 @@
-; RUN: llc < %s -mtriple=i686-- | not grep flds
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- | FileCheck %s
 
 define void @foo(x86_fp80 %a, x86_fp80 %b, float* %fp) {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    fldt {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fldt {{[0-9]+}}(%esp)
+; CHECK-NEXT:    faddp %st, %st(1)
+; CHECK-NEXT:    fstps (%eax)
+; CHECK-NEXT:    retl
 	%c = fadd x86_fp80 %a, %b
 	%d = fptrunc x86_fp80 %c to float
 	store float %d, float* %fp
diff --git a/llvm/test/CodeGen/X86/sub-of-bias.ll b/llvm/test/CodeGen/X86/sub-of-bias.ll
new file mode 100644
index 000000000..8a51ac1bd
--- /dev/null
+++ b/llvm/test/CodeGen/X86/sub-of-bias.ll
@@ -0,0 +1,229 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=i686-unknown-linux-gnu             < %s | FileCheck %s   --check-prefixes=CHECK,NOBMI,X86,NOBMI-X86
+; RUN: llc -mtriple=i686-unknown-linux-gnu -mattr=+bmi < %s | FileCheck %s   --check-prefixes=CHECK,BMI,X86,BMI-X86
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu             < %s | FileCheck %s --check-prefixes=CHECK,NOBMI,X64,NOBMI-X64
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu -mattr=+bmi < %s | FileCheck %s --check-prefixes=CHECK,BMI,X64,BMI-X64
+
+; Fold
+;   ptr - (ptr & mask)
+; To
+;   ptr & (~mask)
+;
+; This needs to be a backend-level fold because only by now pointers
+; are just registers; in middle-end IR this can only be done via @llvm.ptrmask()
+; intrinsic which is not sufficiently widely-spread yet.
+;
+; https://bugs.llvm.org/show_bug.cgi?id=44448
+
+; The basic positive tests
+
+define i32 @t0_32(i32 %ptr, i32 %mask) nounwind {
+; NOBMI-X86-LABEL: t0_32:
+; NOBMI-X86:       # %bb.0:
+; NOBMI-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; NOBMI-X86-NEXT:    notl %eax
+; NOBMI-X86-NEXT:    andl {{[0-9]+}}(%esp), %eax
+; NOBMI-X86-NEXT:    retl
+;
+; BMI-X86-LABEL: t0_32:
+; BMI-X86:       # %bb.0:
+; BMI-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; BMI-X86-NEXT:    andnl {{[0-9]+}}(%esp), %eax, %eax
+; BMI-X86-NEXT:    retl
+;
+; NOBMI-X64-LABEL: t0_32:
+; NOBMI-X64:       # %bb.0:
+; NOBMI-X64-NEXT:    movl %esi, %eax
+; NOBMI-X64-NEXT:    notl %eax
+; NOBMI-X64-NEXT:    andl %edi, %eax
+; NOBMI-X64-NEXT:    retq
+;
+; BMI-X64-LABEL: t0_32:
+; BMI-X64:       # %bb.0:
+; BMI-X64-NEXT:    andnl %edi, %esi, %eax
+; BMI-X64-NEXT:    retq
+  %bias = and i32 %ptr, %mask
+  %r = sub i32 %ptr, %bias
+  ret i32 %r
+}
+define i64 @t1_64(i64 %ptr, i64 %mask) nounwind {
+; NOBMI-X86-LABEL: t1_64:
+; NOBMI-X86:       # %bb.0:
+; NOBMI-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; NOBMI-X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; NOBMI-X86-NEXT:    notl %eax
+; NOBMI-X86-NEXT:    andl {{[0-9]+}}(%esp), %eax
+; NOBMI-X86-NEXT:    notl %edx
+; NOBMI-X86-NEXT:    andl {{[0-9]+}}(%esp), %edx
+; NOBMI-X86-NEXT:    retl
+;
+; BMI-X86-LABEL: t1_64:
+; BMI-X86:       # %bb.0:
+; BMI-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; BMI-X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; BMI-X86-NEXT:    andnl {{[0-9]+}}(%esp), %eax, %eax
+; BMI-X86-NEXT:    andnl {{[0-9]+}}(%esp), %ecx, %edx
+; BMI-X86-NEXT:    retl
+;
+; NOBMI-X64-LABEL: t1_64:
+; NOBMI-X64:       # %bb.0:
+; NOBMI-X64-NEXT:    movq %rsi, %rax
+; NOBMI-X64-NEXT:    notq %rax
+; NOBMI-X64-NEXT:    andq %rdi, %rax
+; NOBMI-X64-NEXT:    retq
+;
+; BMI-X64-LABEL: t1_64:
+; BMI-X64:       # %bb.0:
+; BMI-X64-NEXT:    andnq %rdi, %rsi, %rax
+; BMI-X64-NEXT:    retq
+  %bias = and i64 %ptr, %mask
+  %r = sub i64 %ptr, %bias
+  ret i64 %r
+}
+
+define i32 @t2_commutative(i32 %ptr, i32 %mask) nounwind {
+; NOBMI-X86-LABEL: t2_commutative:
+; NOBMI-X86:       # %bb.0:
+; NOBMI-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; NOBMI-X86-NEXT:    notl %eax
+; NOBMI-X86-NEXT:    andl {{[0-9]+}}(%esp), %eax
+; NOBMI-X86-NEXT:    retl
+;
+; BMI-X86-LABEL: t2_commutative:
+; BMI-X86:       # %bb.0:
+; BMI-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; BMI-X86-NEXT:    andnl {{[0-9]+}}(%esp), %eax, %eax
+; BMI-X86-NEXT:    retl
+;
+; NOBMI-X64-LABEL: t2_commutative:
+; NOBMI-X64:       # %bb.0:
+; NOBMI-X64-NEXT:    movl %esi, %eax
+; NOBMI-X64-NEXT:    notl %eax
+; NOBMI-X64-NEXT:    andl %edi, %eax
+; NOBMI-X64-NEXT:    retq
+;
+; BMI-X64-LABEL: t2_commutative:
+; BMI-X64:       # %bb.0:
+; BMI-X64-NEXT:    andnl %edi, %esi, %eax
+; BMI-X64-NEXT:    retq
+  %bias = and i32 %mask, %ptr ; swapped
+  %r = sub i32 %ptr, %bias
+  ret i32 %r
+}
+
+; Extra use tests
+
+define i32 @n3_extrause1(i32 %ptr, i32 %mask, i32* %bias_storage) nounwind {
+; X86-LABEL: n3_extrause1:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    andl %eax, %edx
+; X86-NEXT:    movl %edx, (%ecx)
+; X86-NEXT:    subl %edx, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: n3_extrause1:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    andl %edi, %esi
+; X64-NEXT:    movl %esi, (%rdx)
+; X64-NEXT:    subl %esi, %eax
+; X64-NEXT:    retq
+  %bias = and i32 %ptr, %mask ; has extra uses, can't fold
+  store i32 %bias, i32* %bias_storage
+  %r = sub i32 %ptr, %bias
+  ret i32 %r
+}
+
+; Negative tests
+
+define i32 @n4_different_ptrs(i32 %ptr0, i32 %ptr1, i32 %mask) nounwind {
+; X86-LABEL: n4_different_ptrs:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    andl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    subl %ecx, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: n4_different_ptrs:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    andl %edx, %esi
+; X64-NEXT:    subl %esi, %eax
+; X64-NEXT:    retq
+  %bias = and i32 %ptr1, %mask ; not %ptr0
+  %r = sub i32 %ptr0, %bias ; not %ptr1
+  ret i32 %r
+}
+define i32 @n5_different_ptrs_commutative(i32 %ptr0, i32 %ptr1, i32 %mask) nounwind {
+; X86-LABEL: n5_different_ptrs_commutative:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    andl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    subl %ecx, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: n5_different_ptrs_commutative:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    andl %edx, %esi
+; X64-NEXT:    subl %esi, %eax
+; X64-NEXT:    retq
+  %bias = and i32 %mask, %ptr1 ; swapped, not %ptr0
+  %r = sub i32 %ptr0, %bias ; not %ptr1
+  ret i32 %r
+}
+
+define i32 @n6_not_lowbit_mask(i32 %ptr, i32 %mask) nounwind {
+; NOBMI-X86-LABEL: n6_not_lowbit_mask:
+; NOBMI-X86:       # %bb.0:
+; NOBMI-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; NOBMI-X86-NEXT:    notl %eax
+; NOBMI-X86-NEXT:    andl {{[0-9]+}}(%esp), %eax
+; NOBMI-X86-NEXT:    retl
+;
+; BMI-X86-LABEL: n6_not_lowbit_mask:
+; BMI-X86:       # %bb.0:
+; BMI-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; BMI-X86-NEXT:    andnl {{[0-9]+}}(%esp), %eax, %eax
+; BMI-X86-NEXT:    retl
+;
+; NOBMI-X64-LABEL: n6_not_lowbit_mask:
+; NOBMI-X64:       # %bb.0:
+; NOBMI-X64-NEXT:    movl %esi, %eax
+; NOBMI-X64-NEXT:    notl %eax
+; NOBMI-X64-NEXT:    andl %edi, %eax
+; NOBMI-X64-NEXT:    retq
+;
+; BMI-X64-LABEL: n6_not_lowbit_mask:
+; BMI-X64:       # %bb.0:
+; BMI-X64-NEXT:    andnl %edi, %esi, %eax
+; BMI-X64-NEXT:    retq
+  %bias = and i32 %ptr, %mask
+  %r = sub i32 %ptr, %bias
+  ret i32 %r
+}
+
+define i32 @n7_sub_is_not_commutative(i32 %ptr, i32 %mask) nounwind {
+; X86-LABEL: n7_sub_is_not_commutative:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    andl %ecx, %eax
+; X86-NEXT:    subl %ecx, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: n7_sub_is_not_commutative:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %esi, %eax
+; X64-NEXT:    andl %edi, %eax
+; X64-NEXT:    subl %edi, %eax
+; X64-NEXT:    retq
+  %bias = and i32 %ptr, %mask
+  %r = sub i32 %bias, %ptr ; wrong order
+  ret i32 %r
+}
diff --git a/llvm/test/CodeGen/X86/subcarry.ll b/llvm/test/CodeGen/X86/subcarry.ll
index 449391616..f5476cdeb 100644
--- a/llvm/test/CodeGen/X86/subcarry.ll
+++ b/llvm/test/CodeGen/X86/subcarry.ll
@@ -1,6 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-unknown | FileCheck %s
 
+declare { i8, i64 } @llvm.x86.subborrow.64(i8, i64, i64)
+declare { i64, i1 } @llvm.usub.with.overflow.i64(i64, i64)
+
 define i128 @sub128(i128 %a, i128 %b) nounwind {
 ; CHECK-LABEL: sub128:
 ; CHECK:       # %bb.0: # %entry
@@ -87,7 +90,7 @@ entry:
   ret %S %31
 }
 
-define %S @sub(%S* nocapture readonly %this, %S %arg.b) local_unnamed_addr {
+define %S @sub(%S* nocapture readonly %this, %S %arg.b) {
 ; CHECK-LABEL: sub:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    movq %rdi, %rax
@@ -162,3 +165,271 @@ entry:
   %40 = insertvalue %S undef, [4 x i64] %39, 0
   ret %S %40
 }
+
+declare {i64, i1} @llvm.uadd.with.overflow(i64, i64)
+declare {i64, i1} @llvm.usub.with.overflow(i64, i64)
+
+define i64 @sub_from_carry(i64 %x, i64 %y, i64* %valout, i64 %z) {
+; CHECK-LABEL: sub_from_carry:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %rcx, %rax
+; CHECK-NEXT:    negq %rax
+; CHECK-NEXT:    addq %rsi, %rdi
+; CHECK-NEXT:    movq %rdi, (%rdx)
+; CHECK-NEXT:    adcq $0, %rax
+; CHECK-NEXT:    retq
+  %agg = call {i64, i1} @llvm.uadd.with.overflow(i64 %x, i64 %y)
+  %val = extractvalue {i64, i1} %agg, 0
+  %ov = extractvalue {i64, i1} %agg, 1
+  store i64 %val, i64* %valout, align 4
+  %carry = zext i1 %ov to i64
+  %res = sub i64 %carry, %z
+  ret i64 %res
+}
+
+%struct.U320 = type { [5 x i64] }
+
+define i32 @sub_U320_without_i128_or(%struct.U320* nocapture dereferenceable(40) %0, i64 %1, i64 %2, i64 %3, i64 %4, i64 %5) {
+; CHECK-LABEL: sub_U320_without_i128_or:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subq %rsi, (%rdi)
+; CHECK-NEXT:    sbbq %rdx, 8(%rdi)
+; CHECK-NEXT:    sbbq %rcx, 16(%rdi)
+; CHECK-NEXT:    sbbq %r8, 24(%rdi)
+; CHECK-NEXT:    sbbq %r9, 32(%rdi)
+; CHECK-NEXT:    setb %al
+; CHECK-NEXT:    movzbl %al, %eax
+; CHECK-NEXT:    retq
+  %7 = getelementptr inbounds %struct.U320, %struct.U320* %0, i64 0, i32 0, i64 0
+  %8 = load i64, i64* %7, align 8
+  %9 = getelementptr inbounds %struct.U320, %struct.U320* %0, i64 0, i32 0, i64 1
+  %10 = load i64, i64* %9, align 8
+  %11 = getelementptr inbounds %struct.U320, %struct.U320* %0, i64 0, i32 0, i64 2
+  %12 = load i64, i64* %11, align 8
+  %13 = getelementptr inbounds %struct.U320, %struct.U320* %0, i64 0, i32 0, i64 3
+  %14 = load i64, i64* %13, align 8
+  %15 = getelementptr inbounds %struct.U320, %struct.U320* %0, i64 0, i32 0, i64 4
+  %16 = load i64, i64* %15, align 8
+  %17 = sub i64 %8, %1
+  %18 = sub i64 %10, %2
+  %19 = icmp ult i64 %8, %1
+  %20 = zext i1 %19 to i64
+  %21 = sub i64 %18, %20
+  %22 = sub i64 %12, %3
+  %23 = icmp ult i64 %10, %2
+  %24 = icmp ult i64 %18, %20
+  %25 = or i1 %23, %24
+  %26 = zext i1 %25 to i64
+  %27 = sub i64 %22, %26
+  %28 = sub i64 %14, %4
+  %29 = icmp ult i64 %12, %3
+  %30 = icmp ult i64 %22, %26
+  %31 = or i1 %29, %30
+  %32 = zext i1 %31 to i64
+  %33 = sub i64 %28, %32
+  %34 = sub i64 %16, %5
+  %35 = icmp ult i64 %14, %4
+  %36 = icmp ult i64 %28, %32
+  %37 = or i1 %35, %36
+  %38 = zext i1 %37 to i64
+  %39 = sub i64 %34, %38
+  store i64 %17, i64* %7, align 8
+  store i64 %21, i64* %9, align 8
+  store i64 %27, i64* %11, align 8
+  store i64 %33, i64* %13, align 8
+  store i64 %39, i64* %15, align 8
+  %40 = icmp ult i64 %16, %5
+  %41 = icmp ult i64 %34, %38
+  %42 = or i1 %40, %41
+  %43 = zext i1 %42 to i32
+  ret i32 %43
+}
+
+define i32 @sub_U320_usubo(%struct.U320* nocapture dereferenceable(40) %0, i64 %1, i64 %2, i64 %3, i64 %4, i64 %5) {
+; CHECK-LABEL: sub_U320_usubo:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subq %rsi, (%rdi)
+; CHECK-NEXT:    sbbq %rdx, 8(%rdi)
+; CHECK-NEXT:    sbbq %rcx, 16(%rdi)
+; CHECK-NEXT:    sbbq %r8, 24(%rdi)
+; CHECK-NEXT:    sbbq %r9, 32(%rdi)
+; CHECK-NEXT:    setb %al
+; CHECK-NEXT:    movzbl %al, %eax
+; CHECK-NEXT:    retq
+  %7 = getelementptr inbounds %struct.U320, %struct.U320* %0, i64 0, i32 0, i64 0
+  %8 = load i64, i64* %7, align 8
+  %9 = getelementptr inbounds %struct.U320, %struct.U320* %0, i64 0, i32 0, i64 1
+  %10 = load i64, i64* %9, align 8
+  %11 = getelementptr inbounds %struct.U320, %struct.U320* %0, i64 0, i32 0, i64 2
+  %12 = load i64, i64* %11, align 8
+  %13 = getelementptr inbounds %struct.U320, %struct.U320* %0, i64 0, i32 0, i64 3
+  %14 = load i64, i64* %13, align 8
+  %15 = getelementptr inbounds %struct.U320, %struct.U320* %0, i64 0, i32 0, i64 4
+  %16 = load i64, i64* %15, align 8
+  %17 = tail call { i64, i1 } @llvm.usub.with.overflow.i64(i64 %8, i64 %1)
+  %18 = extractvalue { i64, i1 } %17, 1
+  %19 = extractvalue { i64, i1 } %17, 0
+  %20 = zext i1 %18 to i64
+  %21 = tail call { i64, i1 } @llvm.usub.with.overflow.i64(i64 %10, i64 %2)
+  %22 = extractvalue { i64, i1 } %21, 1
+  %23 = extractvalue { i64, i1 } %21, 0
+  %24 = tail call { i64, i1 } @llvm.usub.with.overflow.i64(i64 %23, i64 %20)
+  %25 = extractvalue { i64, i1 } %24, 1
+  %26 = extractvalue { i64, i1 } %24, 0
+  %27 = or i1 %22, %25
+  %28 = zext i1 %27 to i64
+  %29 = tail call { i64, i1 } @llvm.usub.with.overflow.i64(i64 %12, i64 %3)
+  %30 = extractvalue { i64, i1 } %29, 1
+  %31 = extractvalue { i64, i1 } %29, 0
+  %32 = tail call { i64, i1 } @llvm.usub.with.overflow.i64(i64 %31, i64 %28)
+  %33 = extractvalue { i64, i1 } %32, 1
+  %34 = extractvalue { i64, i1 } %32, 0
+  %35 = or i1 %30, %33
+  %36 = zext i1 %35 to i64
+  %37 = tail call { i64, i1 } @llvm.usub.with.overflow.i64(i64 %14, i64 %4)
+  %38 = extractvalue { i64, i1 } %37, 1
+  %39 = extractvalue { i64, i1 } %37, 0
+  %40 = tail call { i64, i1 } @llvm.usub.with.overflow.i64(i64 %39, i64 %36)
+  %41 = extractvalue { i64, i1 } %40, 1
+  %42 = extractvalue { i64, i1 } %40, 0
+  %43 = or i1 %38, %41
+  %44 = zext i1 %43 to i64
+  %45 = tail call { i64, i1 } @llvm.usub.with.overflow.i64(i64 %16, i64 %5)
+  %46 = extractvalue { i64, i1 } %45, 1
+  %47 = extractvalue { i64, i1 } %45, 0
+  %48 = tail call { i64, i1 } @llvm.usub.with.overflow.i64(i64 %47, i64 %44)
+  %49 = extractvalue { i64, i1 } %48, 1
+  %50 = extractvalue { i64, i1 } %48, 0
+  %51 = or i1 %46, %49
+  store i64 %19, i64* %7, align 8
+  store i64 %26, i64* %9, align 8
+  store i64 %34, i64* %11, align 8
+  store i64 %42, i64* %13, align 8
+  store i64 %50, i64* %15, align 8
+  %52 = zext i1 %51 to i32
+  ret i32 %52
+}
+
+%struct.U192 = type { [3 x i64] }
+
+define void @PR39464(%struct.U192* noalias nocapture sret %0, %struct.U192* nocapture readonly dereferenceable(24) %1, %struct.U192* nocapture readonly dereferenceable(24) %2) {
+; CHECK-LABEL: PR39464:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    movq (%rsi), %rcx
+; CHECK-NEXT:    subq (%rdx), %rcx
+; CHECK-NEXT:    movq %rcx, (%rdi)
+; CHECK-NEXT:    movq 8(%rsi), %rcx
+; CHECK-NEXT:    sbbq 8(%rdx), %rcx
+; CHECK-NEXT:    movq %rcx, 8(%rdi)
+; CHECK-NEXT:    movq 16(%rsi), %rcx
+; CHECK-NEXT:    sbbq 16(%rdx), %rcx
+; CHECK-NEXT:    movq %rcx, 16(%rdi)
+; CHECK-NEXT:    retq
+  %4 = getelementptr inbounds %struct.U192, %struct.U192* %1, i64 0, i32 0, i64 0
+  %5 = load i64, i64* %4, align 8
+  %6 = getelementptr inbounds %struct.U192, %struct.U192* %2, i64 0, i32 0, i64 0
+  %7 = load i64, i64* %6, align 8
+  %8 = tail call { i64, i1 } @llvm.usub.with.overflow.i64(i64 %5, i64 %7)
+  %9 = extractvalue { i64, i1 } %8, 1
+  %10 = extractvalue { i64, i1 } %8, 0
+  %11 = zext i1 %9 to i64
+  %12 = getelementptr inbounds %struct.U192, %struct.U192* %0, i64 0, i32 0, i64 0
+  store i64 %10, i64* %12, align 8
+  %13 = getelementptr inbounds %struct.U192, %struct.U192* %1, i64 0, i32 0, i64 1
+  %14 = load i64, i64* %13, align 8
+  %15 = getelementptr inbounds %struct.U192, %struct.U192* %2, i64 0, i32 0, i64 1
+  %16 = load i64, i64* %15, align 8
+  %17 = tail call { i64, i1 } @llvm.usub.with.overflow.i64(i64 %14, i64 %16)
+  %18 = extractvalue { i64, i1 } %17, 1
+  %19 = extractvalue { i64, i1 } %17, 0
+  %20 = tail call { i64, i1 } @llvm.usub.with.overflow.i64(i64 %19, i64 %11)
+  %21 = extractvalue { i64, i1 } %20, 1
+  %22 = extractvalue { i64, i1 } %20, 0
+  %23 = or i1 %18, %21
+  %24 = zext i1 %23 to i64
+  %25 = getelementptr inbounds %struct.U192, %struct.U192* %0, i64 0, i32 0, i64 1
+  store i64 %22, i64* %25, align 8
+  %26 = getelementptr inbounds %struct.U192, %struct.U192* %1, i64 0, i32 0, i64 2
+  %27 = load i64, i64* %26, align 8
+  %28 = getelementptr inbounds %struct.U192, %struct.U192* %2, i64 0, i32 0, i64 2
+  %29 = load i64, i64* %28, align 8
+  %30 = sub i64 %27, %29
+  %31 = sub i64 %30, %24
+  %32 = getelementptr inbounds %struct.U192, %struct.U192* %0, i64 0, i32 0, i64 2
+  store i64 %31, i64* %32, align 8
+  ret void
+}
+
+%uint128 = type { i64, i64 }
+%uint256 = type { %uint128, %uint128 }
+
+; The 256-bit subtraction implementation using two inlined usubo procedures for U128 type { i64, i64 }.
+; This is similar to how LLVM legalize types in CodeGen.
+define void @sub_U256_without_i128_or_recursive(%uint256* sret %0, %uint256* %1, %uint256* %2) nounwind {
+; CHECK-LABEL: sub_U256_without_i128_or_recursive:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    movq (%rsi), %r9
+; CHECK-NEXT:    movq 8(%rsi), %r10
+; CHECK-NEXT:    subq (%rdx), %r9
+; CHECK-NEXT:    sbbq 8(%rdx), %r10
+; CHECK-NEXT:    setb %r8b
+; CHECK-NEXT:    movq 16(%rsi), %rcx
+; CHECK-NEXT:    movq 24(%rsi), %rsi
+; CHECK-NEXT:    xorl %edi, %edi
+; CHECK-NEXT:    subq 16(%rdx), %rcx
+; CHECK-NEXT:    setb %dil
+; CHECK-NEXT:    subq 24(%rdx), %rsi
+; CHECK-NEXT:    movzbl %r8b, %edx
+; CHECK-NEXT:    subq %rdx, %rcx
+; CHECK-NEXT:    sbbq %rdi, %rsi
+; CHECK-NEXT:    movq %r9, (%rax)
+; CHECK-NEXT:    movq %r10, 8(%rax)
+; CHECK-NEXT:    movq %rcx, 16(%rax)
+; CHECK-NEXT:    movq %rsi, 24(%rax)
+; CHECK-NEXT:    retq
+  %4 = getelementptr inbounds %uint256, %uint256* %1, i64 0, i32 0, i32 0
+  %5 = load i64, i64* %4, align 8
+  %6 = getelementptr inbounds %uint256, %uint256* %1, i64 0, i32 0, i32 1
+  %7 = load i64, i64* %6, align 8
+  %8 = getelementptr inbounds %uint256, %uint256* %2, i64 0, i32 0, i32 0
+  %9 = load i64, i64* %8, align 8
+  %10 = getelementptr inbounds %uint256, %uint256* %2, i64 0, i32 0, i32 1
+  %11 = load i64, i64* %10, align 8
+  %12 = sub i64 %5, %9
+  %13 = icmp ult i64 %5, %9
+  %14 = sub i64 %7, %11
+  %15 = icmp ult i64 %7, %11
+  %16 = zext i1 %13 to i64
+  %17 = sub i64 %14, %16
+  %18 = icmp ult i64 %14, %16
+  %19 = or i1 %15, %18
+  %20 = getelementptr inbounds %uint256, %uint256* %1, i64 0, i32 1, i32 0
+  %21 = load i64, i64* %20, align 8
+  %22 = getelementptr inbounds %uint256, %uint256* %1, i64 0, i32 1, i32 1
+  %23 = load i64, i64* %22, align 8
+  %24 = getelementptr inbounds %uint256, %uint256* %2, i64 0, i32 1, i32 0
+  %25 = load i64, i64* %24, align 8
+  %26 = getelementptr inbounds %uint256, %uint256* %2, i64 0, i32 1, i32 1
+  %27 = load i64, i64* %26, align 8
+  %28 = sub i64 %21, %25
+  %29 = icmp ult i64 %21, %25
+  %30 = sub i64 %23, %27
+  %31 = zext i1 %29 to i64
+  %32 = sub i64 %30, %31
+  %33 = zext i1 %19 to i64
+  %34 = sub i64 %28, %33
+  %35 = icmp ult i64 %28, %33
+  %36 = zext i1 %35 to i64
+  %37 = sub i64 %32, %36
+  %38 = getelementptr inbounds %uint256, %uint256* %0, i64 0, i32 0, i32 0
+  store i64 %12, i64* %38, align 8
+  %39 = getelementptr inbounds %uint256, %uint256* %0, i64 0, i32 0, i32 1
+  store i64 %17, i64* %39, align 8
+  %40 = getelementptr inbounds %uint256, %uint256* %0, i64 0, i32 1, i32 0
+  store i64 %34, i64* %40, align 8
+  %41 = getelementptr inbounds %uint256, %uint256* %0, i64 0, i32 1, i32 1
+  store i64 %37, i64* %41, align 8
+  ret void
+}
diff --git a/llvm/test/CodeGen/X86/subreg-to-reg-0.ll b/llvm/test/CodeGen/X86/subreg-to-reg-0.ll
index bd6007629..a99a04bbb 100644
--- a/llvm/test/CodeGen/X86/subreg-to-reg-0.ll
+++ b/llvm/test/CodeGen/X86/subreg-to-reg-0.ll
@@ -1,9 +1,15 @@
-; RUN: llc < %s -mtriple=x86_64-- | grep mov | count 1
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-- | FileCheck %s
 
 ; Do eliminate the zero-extension instruction and rely on
 ; x86-64's implicit zero-extension!
 
 define i64 @foo(i32* %p) nounwind {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl (%rdi), %eax
+; CHECK-NEXT:    incl %eax
+; CHECK-NEXT:    retq
   %t = load i32, i32* %p
   %n = add i32 %t, 1
   %z = zext i32 %n to i64
diff --git a/llvm/test/CodeGen/X86/subreg-to-reg-2.ll b/llvm/test/CodeGen/X86/subreg-to-reg-2.ll
index 6766b013a..3c78947ba 100644
--- a/llvm/test/CodeGen/X86/subreg-to-reg-2.ll
+++ b/llvm/test/CodeGen/X86/subreg-to-reg-2.ll
@@ -1,4 +1,5 @@
-; RUN: llc < %s -mtriple=x86_64-apple-darwin | grep movl
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-apple-darwin | FileCheck %s
 ; rdar://6707985
 
 	%XXOO = type { %"struct.XXC::XXCC", i8*, %"struct.XXC::XXOO::$_71" }
@@ -9,6 +10,13 @@
 	%"struct.XXC::XXOO::$_71" = type { [2 x %XXValue*] }
 
 define internal fastcc %XXValue* @t(i64* %out, %"struct.XXC::ArrayStorage"* %tmp9) nounwind {
+; CHECK-LABEL: t:
+; CHECK:       ## %bb.0: ## %prologue
+; CHECK-NEXT:    movq 22222222, %rax
+; CHECK-NEXT:    movq %rax, (%rdi)
+; CHECK-NEXT:    movl %eax, %eax
+; CHECK-NEXT:    movq 32(%rsi,%rax,8), %rax
+; CHECK-NEXT:    retq
 prologue:
 	%array = load %XXValue*, %XXValue** inttoptr (i64 11111111 to %XXValue**)		; <%XXValue*> [#uses=0]
 	%index = load %XXValue*, %XXValue** inttoptr (i64 22222222 to %XXValue**)		; <%XXValue*> [#uses=1]
diff --git a/llvm/test/CodeGen/X86/subreg-to-reg-4.ll b/llvm/test/CodeGen/X86/subreg-to-reg-4.ll
index 2a9a490c5..f69ff17b6 100644
--- a/llvm/test/CodeGen/X86/subreg-to-reg-4.ll
+++ b/llvm/test/CodeGen/X86/subreg-to-reg-4.ll
@@ -1,17 +1,16 @@
-; RUN: llc < %s -mtriple=x86_64-- > %t
-; RUN: not grep leaq %t
-; RUN: not grep incq %t
-; RUN: not grep decq %t
-; RUN: not grep negq %t
-; RUN: not grep addq %t
-; RUN: not grep subq %t
-; RUN: not grep "movl	%" %t
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-- | FileCheck %s
 
 ; Utilize implicit zero-extension on x86-64 to eliminate explicit
 ; zero-extensions. Shrink 64-bit adds to 32-bit when the high
 ; 32-bits will be zeroed.
 
 define void @bar(i64 %x, i64 %y, i64* %z) nounwind readnone {
+; CHECK-LABEL: bar:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addl %esi, %edi
+; CHECK-NEXT:    movq %rdi, (%rdx)
+; CHECK-NEXT:    retq
 entry:
 	%t0 = add i64 %x, %y
 	%t1 = and i64 %t0, 4294967295
@@ -19,6 +18,12 @@ entry:
 	ret void
 }
 define void @easy(i32 %x, i32 %y, i64* %z) nounwind readnone {
+; CHECK-LABEL: easy:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    # kill: def $edi killed $edi def $rdi
+; CHECK-NEXT:    addl %esi, %edi
+; CHECK-NEXT:    movq %rdi, (%rdx)
+; CHECK-NEXT:    retq
 entry:
 	%t0 = add i32 %x, %y
         %tn = zext i32 %t0 to i64
@@ -27,6 +32,12 @@ entry:
 	ret void
 }
 define void @cola(i64 *%x, i64 %y, i64* %z, i64 %u) nounwind readnone {
+; CHECK-LABEL: cola:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addl (%rdi), %esi
+; CHECK-NEXT:    xorq %rcx, %rsi
+; CHECK-NEXT:    movq %rsi, (%rdx)
+; CHECK-NEXT:    retq
 entry:
         %p = load i64, i64* %x
 	%t0 = add i64 %p, %y
@@ -36,6 +47,12 @@ entry:
 	ret void
 }
 define void @yaks(i64 *%x, i64 %y, i64* %z, i64 %u) nounwind readnone {
+; CHECK-LABEL: yaks:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addl (%rdi), %esi
+; CHECK-NEXT:    xorl %esi, %ecx
+; CHECK-NEXT:    movq %rcx, (%rdx)
+; CHECK-NEXT:    retq
 entry:
         %p = load i64, i64* %x
 	%t0 = add i64 %p, %y
@@ -45,6 +62,12 @@ entry:
 	ret void
 }
 define void @foo(i64 *%x, i64 *%y, i64* %z) nounwind readnone {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movl (%rdi), %eax
+; CHECK-NEXT:    addl (%rsi), %eax
+; CHECK-NEXT:    movq %rax, (%rdx)
+; CHECK-NEXT:    retq
 entry:
         %a = load i64, i64* %x
         %b = load i64, i64* %y
@@ -54,6 +77,12 @@ entry:
 	ret void
 }
 define void @avo(i64 %x, i64* %z, i64 %u) nounwind readnone {
+; CHECK-LABEL: avo:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addl $734847, %edi # imm = 0xB367F
+; CHECK-NEXT:    xorq %rdx, %rdi
+; CHECK-NEXT:    movq %rdi, (%rsi)
+; CHECK-NEXT:    retq
 entry:
 	%t0 = add i64 %x, 734847
 	%t1 = and i64 %t0, 4294967295
@@ -62,6 +91,12 @@ entry:
 	ret void
 }
 define void @phe(i64 %x, i64* %z, i64 %u) nounwind readnone {
+; CHECK-LABEL: phe:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addl $734847, %edi # imm = 0xB367F
+; CHECK-NEXT:    xorl %edi, %edx
+; CHECK-NEXT:    movq %rdx, (%rsi)
+; CHECK-NEXT:    retq
 entry:
 	%t0 = add i64 %x, 734847
         %t1 = xor i64 %t0, %u
@@ -70,6 +105,11 @@ entry:
 	ret void
 }
 define void @oze(i64 %y, i64* %z) nounwind readnone {
+; CHECK-LABEL: oze:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    incl %edi
+; CHECK-NEXT:    movq %rdi, (%rsi)
+; CHECK-NEXT:    retq
 entry:
 	%t0 = add i64 %y, 1
 	%t1 = and i64 %t0, 4294967295
@@ -78,6 +118,11 @@ entry:
 }
 
 define void @sbar(i64 %x, i64 %y, i64* %z) nounwind readnone {
+; CHECK-LABEL: sbar:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    subl %esi, %edi
+; CHECK-NEXT:    movq %rdi, (%rdx)
+; CHECK-NEXT:    retq
 entry:
 	%t0 = sub i64 %x, %y
 	%t1 = and i64 %t0, 4294967295
@@ -85,6 +130,12 @@ entry:
 	ret void
 }
 define void @seasy(i32 %x, i32 %y, i64* %z) nounwind readnone {
+; CHECK-LABEL: seasy:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    # kill: def $edi killed $edi def $rdi
+; CHECK-NEXT:    subl %esi, %edi
+; CHECK-NEXT:    movq %rdi, (%rdx)
+; CHECK-NEXT:    retq
 entry:
 	%t0 = sub i32 %x, %y
         %tn = zext i32 %t0 to i64
@@ -93,6 +144,13 @@ entry:
 	ret void
 }
 define void @scola(i64 *%x, i64 %y, i64* %z, i64 %u) nounwind readnone {
+; CHECK-LABEL: scola:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movl (%rdi), %eax
+; CHECK-NEXT:    subl %esi, %eax
+; CHECK-NEXT:    xorq %rcx, %rax
+; CHECK-NEXT:    movq %rax, (%rdx)
+; CHECK-NEXT:    retq
 entry:
         %p = load i64, i64* %x
 	%t0 = sub i64 %p, %y
@@ -102,6 +160,13 @@ entry:
 	ret void
 }
 define void @syaks(i64 *%x, i64 %y, i64* %z, i64 %u) nounwind readnone {
+; CHECK-LABEL: syaks:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movl (%rdi), %eax
+; CHECK-NEXT:    subl %esi, %eax
+; CHECK-NEXT:    xorl %eax, %ecx
+; CHECK-NEXT:    movq %rcx, (%rdx)
+; CHECK-NEXT:    retq
 entry:
         %p = load i64, i64* %x
 	%t0 = sub i64 %p, %y
@@ -111,6 +176,12 @@ entry:
 	ret void
 }
 define void @sfoo(i64 *%x, i64 *%y, i64* %z) nounwind readnone {
+; CHECK-LABEL: sfoo:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movl (%rdi), %eax
+; CHECK-NEXT:    subl (%rsi), %eax
+; CHECK-NEXT:    movq %rax, (%rdx)
+; CHECK-NEXT:    retq
 entry:
         %a = load i64, i64* %x
         %b = load i64, i64* %y
@@ -120,6 +191,11 @@ entry:
 	ret void
 }
 define void @swya(i64 %y, i64* %z) nounwind readnone {
+; CHECK-LABEL: swya:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    negl %edi
+; CHECK-NEXT:    movq %rdi, (%rsi)
+; CHECK-NEXT:    retq
 entry:
 	%t0 = sub i64 0, %y
 	%t1 = and i64 %t0, 4294967295
@@ -127,6 +203,11 @@ entry:
 	ret void
 }
 define void @soze(i64 %y, i64* %z) nounwind readnone {
+; CHECK-LABEL: soze:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    decl %edi
+; CHECK-NEXT:    movq %rdi, (%rsi)
+; CHECK-NEXT:    retq
 entry:
 	%t0 = sub i64 %y, 1
 	%t1 = and i64 %t0, 4294967295
diff --git a/llvm/test/CodeGen/X86/subvector-broadcast.ll b/llvm/test/CodeGen/X86/subvector-broadcast.ll
index 7ecfac515..430d152e3 100644
--- a/llvm/test/CodeGen/X86/subvector-broadcast.ll
+++ b/llvm/test/CodeGen/X86/subvector-broadcast.ll
@@ -367,25 +367,11 @@ define <32 x i16> @test_broadcast_8i16_32i16(<8 x i16> *%p) nounwind {
 ; X32-AVX-NEXT:    vmovaps %ymm0, %ymm1
 ; X32-AVX-NEXT:    retl
 ;
-; X32-AVX512F-LABEL: test_broadcast_8i16_32i16:
-; X32-AVX512F:       # %bb.0:
-; X32-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = mem[0,1,0,1]
-; X32-AVX512F-NEXT:    vmovdqa %ymm0, %ymm1
-; X32-AVX512F-NEXT:    retl
-;
-; X32-AVX512BW-LABEL: test_broadcast_8i16_32i16:
-; X32-AVX512BW:       # %bb.0:
-; X32-AVX512BW-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; X32-AVX512BW-NEXT:    retl
-;
-; X32-AVX512DQ-LABEL: test_broadcast_8i16_32i16:
-; X32-AVX512DQ:       # %bb.0:
-; X32-AVX512DQ-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX512DQ-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = mem[0,1,0,1]
-; X32-AVX512DQ-NEXT:    vmovdqa %ymm0, %ymm1
-; X32-AVX512DQ-NEXT:    retl
+; X32-AVX512-LABEL: test_broadcast_8i16_32i16:
+; X32-AVX512:       # %bb.0:
+; X32-AVX512-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-AVX512-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; X32-AVX512-NEXT:    retl
 ;
 ; X64-AVX-LABEL: test_broadcast_8i16_32i16:
 ; X64-AVX:       # %bb.0:
@@ -393,22 +379,10 @@ define <32 x i16> @test_broadcast_8i16_32i16(<8 x i16> *%p) nounwind {
 ; X64-AVX-NEXT:    vmovaps %ymm0, %ymm1
 ; X64-AVX-NEXT:    retq
 ;
-; X64-AVX512F-LABEL: test_broadcast_8i16_32i16:
-; X64-AVX512F:       # %bb.0:
-; X64-AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = mem[0,1,0,1]
-; X64-AVX512F-NEXT:    vmovdqa %ymm0, %ymm1
-; X64-AVX512F-NEXT:    retq
-;
-; X64-AVX512BW-LABEL: test_broadcast_8i16_32i16:
-; X64-AVX512BW:       # %bb.0:
-; X64-AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; X64-AVX512BW-NEXT:    retq
-;
-; X64-AVX512DQ-LABEL: test_broadcast_8i16_32i16:
-; X64-AVX512DQ:       # %bb.0:
-; X64-AVX512DQ-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = mem[0,1,0,1]
-; X64-AVX512DQ-NEXT:    vmovdqa %ymm0, %ymm1
-; X64-AVX512DQ-NEXT:    retq
+; X64-AVX512-LABEL: test_broadcast_8i16_32i16:
+; X64-AVX512:       # %bb.0:
+; X64-AVX512-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; X64-AVX512-NEXT:    retq
  %1 = load <8 x i16>, <8 x i16> *%p
  %2 = shufflevector <8 x i16> %1, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
  ret <32 x i16> %2
@@ -422,25 +396,11 @@ define <32 x i16> @test_broadcast_16i16_32i16(<16 x i16> *%p) nounwind {
 ; X32-AVX-NEXT:    vmovaps %ymm0, %ymm1
 ; X32-AVX-NEXT:    retl
 ;
-; X32-AVX512F-LABEL: test_broadcast_16i16_32i16:
-; X32-AVX512F:       # %bb.0:
-; X32-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX512F-NEXT:    vmovaps (%eax), %ymm0
-; X32-AVX512F-NEXT:    vmovaps %ymm0, %ymm1
-; X32-AVX512F-NEXT:    retl
-;
-; X32-AVX512BW-LABEL: test_broadcast_16i16_32i16:
-; X32-AVX512BW:       # %bb.0:
-; X32-AVX512BW-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = mem[0,1,2,3,0,1,2,3]
-; X32-AVX512BW-NEXT:    retl
-;
-; X32-AVX512DQ-LABEL: test_broadcast_16i16_32i16:
-; X32-AVX512DQ:       # %bb.0:
-; X32-AVX512DQ-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX512DQ-NEXT:    vmovaps (%eax), %ymm0
-; X32-AVX512DQ-NEXT:    vmovaps %ymm0, %ymm1
-; X32-AVX512DQ-NEXT:    retl
+; X32-AVX512-LABEL: test_broadcast_16i16_32i16:
+; X32-AVX512:       # %bb.0:
+; X32-AVX512-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-AVX512-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = mem[0,1,2,3,0,1,2,3]
+; X32-AVX512-NEXT:    retl
 ;
 ; X64-AVX-LABEL: test_broadcast_16i16_32i16:
 ; X64-AVX:       # %bb.0:
@@ -448,22 +408,10 @@ define <32 x i16> @test_broadcast_16i16_32i16(<16 x i16> *%p) nounwind {
 ; X64-AVX-NEXT:    vmovaps %ymm0, %ymm1
 ; X64-AVX-NEXT:    retq
 ;
-; X64-AVX512F-LABEL: test_broadcast_16i16_32i16:
-; X64-AVX512F:       # %bb.0:
-; X64-AVX512F-NEXT:    vmovaps (%rdi), %ymm0
-; X64-AVX512F-NEXT:    vmovaps %ymm0, %ymm1
-; X64-AVX512F-NEXT:    retq
-;
-; X64-AVX512BW-LABEL: test_broadcast_16i16_32i16:
-; X64-AVX512BW:       # %bb.0:
-; X64-AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = mem[0,1,2,3,0,1,2,3]
-; X64-AVX512BW-NEXT:    retq
-;
-; X64-AVX512DQ-LABEL: test_broadcast_16i16_32i16:
-; X64-AVX512DQ:       # %bb.0:
-; X64-AVX512DQ-NEXT:    vmovaps (%rdi), %ymm0
-; X64-AVX512DQ-NEXT:    vmovaps %ymm0, %ymm1
-; X64-AVX512DQ-NEXT:    retq
+; X64-AVX512-LABEL: test_broadcast_16i16_32i16:
+; X64-AVX512:       # %bb.0:
+; X64-AVX512-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = mem[0,1,2,3,0,1,2,3]
+; X64-AVX512-NEXT:    retq
  %1 = load <16 x i16>, <16 x i16> *%p
  %2 = shufflevector <16 x i16> %1, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
  ret <32 x i16> %2
@@ -504,25 +452,11 @@ define <64 x i8> @test_broadcast_16i8_64i8(<16 x i8> *%p) nounwind {
 ; X32-AVX-NEXT:    vmovaps %ymm0, %ymm1
 ; X32-AVX-NEXT:    retl
 ;
-; X32-AVX512F-LABEL: test_broadcast_16i8_64i8:
-; X32-AVX512F:       # %bb.0:
-; X32-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = mem[0,1,0,1]
-; X32-AVX512F-NEXT:    vmovdqa %ymm0, %ymm1
-; X32-AVX512F-NEXT:    retl
-;
-; X32-AVX512BW-LABEL: test_broadcast_16i8_64i8:
-; X32-AVX512BW:       # %bb.0:
-; X32-AVX512BW-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; X32-AVX512BW-NEXT:    retl
-;
-; X32-AVX512DQ-LABEL: test_broadcast_16i8_64i8:
-; X32-AVX512DQ:       # %bb.0:
-; X32-AVX512DQ-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX512DQ-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = mem[0,1,0,1]
-; X32-AVX512DQ-NEXT:    vmovdqa %ymm0, %ymm1
-; X32-AVX512DQ-NEXT:    retl
+; X32-AVX512-LABEL: test_broadcast_16i8_64i8:
+; X32-AVX512:       # %bb.0:
+; X32-AVX512-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-AVX512-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; X32-AVX512-NEXT:    retl
 ;
 ; X64-AVX-LABEL: test_broadcast_16i8_64i8:
 ; X64-AVX:       # %bb.0:
@@ -530,22 +464,10 @@ define <64 x i8> @test_broadcast_16i8_64i8(<16 x i8> *%p) nounwind {
 ; X64-AVX-NEXT:    vmovaps %ymm0, %ymm1
 ; X64-AVX-NEXT:    retq
 ;
-; X64-AVX512F-LABEL: test_broadcast_16i8_64i8:
-; X64-AVX512F:       # %bb.0:
-; X64-AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = mem[0,1,0,1]
-; X64-AVX512F-NEXT:    vmovdqa %ymm0, %ymm1
-; X64-AVX512F-NEXT:    retq
-;
-; X64-AVX512BW-LABEL: test_broadcast_16i8_64i8:
-; X64-AVX512BW:       # %bb.0:
-; X64-AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; X64-AVX512BW-NEXT:    retq
-;
-; X64-AVX512DQ-LABEL: test_broadcast_16i8_64i8:
-; X64-AVX512DQ:       # %bb.0:
-; X64-AVX512DQ-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = mem[0,1,0,1]
-; X64-AVX512DQ-NEXT:    vmovdqa %ymm0, %ymm1
-; X64-AVX512DQ-NEXT:    retq
+; X64-AVX512-LABEL: test_broadcast_16i8_64i8:
+; X64-AVX512:       # %bb.0:
+; X64-AVX512-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; X64-AVX512-NEXT:    retq
  %1 = load <16 x i8>, <16 x i8> *%p
  %2 = shufflevector <16 x i8> %1, <16 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
  ret <64 x i8> %2
@@ -559,25 +481,11 @@ define <64 x i8> @test_broadcast_32i8_64i8(<32 x i8> *%p) nounwind {
 ; X32-AVX-NEXT:    vmovaps %ymm0, %ymm1
 ; X32-AVX-NEXT:    retl
 ;
-; X32-AVX512F-LABEL: test_broadcast_32i8_64i8:
-; X32-AVX512F:       # %bb.0:
-; X32-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX512F-NEXT:    vmovaps (%eax), %ymm0
-; X32-AVX512F-NEXT:    vmovaps %ymm0, %ymm1
-; X32-AVX512F-NEXT:    retl
-;
-; X32-AVX512BW-LABEL: test_broadcast_32i8_64i8:
-; X32-AVX512BW:       # %bb.0:
-; X32-AVX512BW-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = mem[0,1,2,3,0,1,2,3]
-; X32-AVX512BW-NEXT:    retl
-;
-; X32-AVX512DQ-LABEL: test_broadcast_32i8_64i8:
-; X32-AVX512DQ:       # %bb.0:
-; X32-AVX512DQ-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX512DQ-NEXT:    vmovaps (%eax), %ymm0
-; X32-AVX512DQ-NEXT:    vmovaps %ymm0, %ymm1
-; X32-AVX512DQ-NEXT:    retl
+; X32-AVX512-LABEL: test_broadcast_32i8_64i8:
+; X32-AVX512:       # %bb.0:
+; X32-AVX512-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-AVX512-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = mem[0,1,2,3,0,1,2,3]
+; X32-AVX512-NEXT:    retl
 ;
 ; X64-AVX-LABEL: test_broadcast_32i8_64i8:
 ; X64-AVX:       # %bb.0:
@@ -585,22 +493,10 @@ define <64 x i8> @test_broadcast_32i8_64i8(<32 x i8> *%p) nounwind {
 ; X64-AVX-NEXT:    vmovaps %ymm0, %ymm1
 ; X64-AVX-NEXT:    retq
 ;
-; X64-AVX512F-LABEL: test_broadcast_32i8_64i8:
-; X64-AVX512F:       # %bb.0:
-; X64-AVX512F-NEXT:    vmovaps (%rdi), %ymm0
-; X64-AVX512F-NEXT:    vmovaps %ymm0, %ymm1
-; X64-AVX512F-NEXT:    retq
-;
-; X64-AVX512BW-LABEL: test_broadcast_32i8_64i8:
-; X64-AVX512BW:       # %bb.0:
-; X64-AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = mem[0,1,2,3,0,1,2,3]
-; X64-AVX512BW-NEXT:    retq
-;
-; X64-AVX512DQ-LABEL: test_broadcast_32i8_64i8:
-; X64-AVX512DQ:       # %bb.0:
-; X64-AVX512DQ-NEXT:    vmovaps (%rdi), %ymm0
-; X64-AVX512DQ-NEXT:    vmovaps %ymm0, %ymm1
-; X64-AVX512DQ-NEXT:    retq
+; X64-AVX512-LABEL: test_broadcast_32i8_64i8:
+; X64-AVX512:       # %bb.0:
+; X64-AVX512-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = mem[0,1,2,3,0,1,2,3]
+; X64-AVX512-NEXT:    retq
  %1 = load <32 x i8>, <32 x i8> *%p
  %2 = shufflevector <32 x i8> %1, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
  ret <64 x i8> %2
@@ -1332,26 +1228,12 @@ define <32 x i16> @reg_broadcast_8i16_32i16(<8 x i16> %a0) nounwind {
 ; X32-AVX-NEXT:    vmovaps %ymm0, %ymm1
 ; X32-AVX-NEXT:    retl
 ;
-; X32-AVX512F-LABEL: reg_broadcast_8i16_32i16:
-; X32-AVX512F:       # %bb.0:
-; X32-AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
-; X32-AVX512F-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
-; X32-AVX512F-NEXT:    vmovaps %ymm0, %ymm1
-; X32-AVX512F-NEXT:    retl
-;
-; X32-AVX512BW-LABEL: reg_broadcast_8i16_32i16:
-; X32-AVX512BW:       # %bb.0:
-; X32-AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
-; X32-AVX512BW-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
-; X32-AVX512BW-NEXT:    vinsertf64x4 $1, %ymm0, %zmm0, %zmm0
-; X32-AVX512BW-NEXT:    retl
-;
-; X32-AVX512DQ-LABEL: reg_broadcast_8i16_32i16:
-; X32-AVX512DQ:       # %bb.0:
-; X32-AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
-; X32-AVX512DQ-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
-; X32-AVX512DQ-NEXT:    vmovaps %ymm0, %ymm1
-; X32-AVX512DQ-NEXT:    retl
+; X32-AVX512-LABEL: reg_broadcast_8i16_32i16:
+; X32-AVX512:       # %bb.0:
+; X32-AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; X32-AVX512-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
+; X32-AVX512-NEXT:    vinsertf64x4 $1, %ymm0, %zmm0, %zmm0
+; X32-AVX512-NEXT:    retl
 ;
 ; X64-AVX-LABEL: reg_broadcast_8i16_32i16:
 ; X64-AVX:       # %bb.0:
@@ -1360,26 +1242,12 @@ define <32 x i16> @reg_broadcast_8i16_32i16(<8 x i16> %a0) nounwind {
 ; X64-AVX-NEXT:    vmovaps %ymm0, %ymm1
 ; X64-AVX-NEXT:    retq
 ;
-; X64-AVX512F-LABEL: reg_broadcast_8i16_32i16:
-; X64-AVX512F:       # %bb.0:
-; X64-AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
-; X64-AVX512F-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
-; X64-AVX512F-NEXT:    vmovaps %ymm0, %ymm1
-; X64-AVX512F-NEXT:    retq
-;
-; X64-AVX512BW-LABEL: reg_broadcast_8i16_32i16:
-; X64-AVX512BW:       # %bb.0:
-; X64-AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
-; X64-AVX512BW-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
-; X64-AVX512BW-NEXT:    vinsertf64x4 $1, %ymm0, %zmm0, %zmm0
-; X64-AVX512BW-NEXT:    retq
-;
-; X64-AVX512DQ-LABEL: reg_broadcast_8i16_32i16:
-; X64-AVX512DQ:       # %bb.0:
-; X64-AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
-; X64-AVX512DQ-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
-; X64-AVX512DQ-NEXT:    vmovaps %ymm0, %ymm1
-; X64-AVX512DQ-NEXT:    retq
+; X64-AVX512-LABEL: reg_broadcast_8i16_32i16:
+; X64-AVX512:       # %bb.0:
+; X64-AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; X64-AVX512-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
+; X64-AVX512-NEXT:    vinsertf64x4 $1, %ymm0, %zmm0, %zmm0
+; X64-AVX512-NEXT:    retq
  %1 = shufflevector <8 x i16> %a0, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
  ret <32 x i16> %1
 }
@@ -1390,42 +1258,22 @@ define <32 x i16> @reg_broadcast_16i16_32i16(<16 x i16> %a0) nounwind {
 ; X32-AVX-NEXT:    vmovaps %ymm0, %ymm1
 ; X32-AVX-NEXT:    retl
 ;
-; X32-AVX512F-LABEL: reg_broadcast_16i16_32i16:
-; X32-AVX512F:       # %bb.0:
-; X32-AVX512F-NEXT:    vmovaps %ymm0, %ymm1
-; X32-AVX512F-NEXT:    retl
-;
-; X32-AVX512BW-LABEL: reg_broadcast_16i16_32i16:
-; X32-AVX512BW:       # %bb.0:
-; X32-AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; X32-AVX512BW-NEXT:    vinsertf64x4 $1, %ymm0, %zmm0, %zmm0
-; X32-AVX512BW-NEXT:    retl
-;
-; X32-AVX512DQ-LABEL: reg_broadcast_16i16_32i16:
-; X32-AVX512DQ:       # %bb.0:
-; X32-AVX512DQ-NEXT:    vmovaps %ymm0, %ymm1
-; X32-AVX512DQ-NEXT:    retl
+; X32-AVX512-LABEL: reg_broadcast_16i16_32i16:
+; X32-AVX512:       # %bb.0:
+; X32-AVX512-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; X32-AVX512-NEXT:    vinsertf64x4 $1, %ymm0, %zmm0, %zmm0
+; X32-AVX512-NEXT:    retl
 ;
 ; X64-AVX-LABEL: reg_broadcast_16i16_32i16:
 ; X64-AVX:       # %bb.0:
 ; X64-AVX-NEXT:    vmovaps %ymm0, %ymm1
 ; X64-AVX-NEXT:    retq
 ;
-; X64-AVX512F-LABEL: reg_broadcast_16i16_32i16:
-; X64-AVX512F:       # %bb.0:
-; X64-AVX512F-NEXT:    vmovaps %ymm0, %ymm1
-; X64-AVX512F-NEXT:    retq
-;
-; X64-AVX512BW-LABEL: reg_broadcast_16i16_32i16:
-; X64-AVX512BW:       # %bb.0:
-; X64-AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; X64-AVX512BW-NEXT:    vinsertf64x4 $1, %ymm0, %zmm0, %zmm0
-; X64-AVX512BW-NEXT:    retq
-;
-; X64-AVX512DQ-LABEL: reg_broadcast_16i16_32i16:
-; X64-AVX512DQ:       # %bb.0:
-; X64-AVX512DQ-NEXT:    vmovaps %ymm0, %ymm1
-; X64-AVX512DQ-NEXT:    retq
+; X64-AVX512-LABEL: reg_broadcast_16i16_32i16:
+; X64-AVX512:       # %bb.0:
+; X64-AVX512-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; X64-AVX512-NEXT:    vinsertf64x4 $1, %ymm0, %zmm0, %zmm0
+; X64-AVX512-NEXT:    retq
  %1 = shufflevector <16 x i16> %a0, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
  ret <32 x i16> %1
 }
@@ -1454,26 +1302,12 @@ define <64 x i8> @reg_broadcast_16i8_64i8(<16 x i8> %a0) nounwind {
 ; X32-AVX-NEXT:    vmovaps %ymm0, %ymm1
 ; X32-AVX-NEXT:    retl
 ;
-; X32-AVX512F-LABEL: reg_broadcast_16i8_64i8:
-; X32-AVX512F:       # %bb.0:
-; X32-AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
-; X32-AVX512F-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
-; X32-AVX512F-NEXT:    vmovaps %ymm0, %ymm1
-; X32-AVX512F-NEXT:    retl
-;
-; X32-AVX512BW-LABEL: reg_broadcast_16i8_64i8:
-; X32-AVX512BW:       # %bb.0:
-; X32-AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
-; X32-AVX512BW-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
-; X32-AVX512BW-NEXT:    vinsertf64x4 $1, %ymm0, %zmm0, %zmm0
-; X32-AVX512BW-NEXT:    retl
-;
-; X32-AVX512DQ-LABEL: reg_broadcast_16i8_64i8:
-; X32-AVX512DQ:       # %bb.0:
-; X32-AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
-; X32-AVX512DQ-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
-; X32-AVX512DQ-NEXT:    vmovaps %ymm0, %ymm1
-; X32-AVX512DQ-NEXT:    retl
+; X32-AVX512-LABEL: reg_broadcast_16i8_64i8:
+; X32-AVX512:       # %bb.0:
+; X32-AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; X32-AVX512-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
+; X32-AVX512-NEXT:    vinsertf64x4 $1, %ymm0, %zmm0, %zmm0
+; X32-AVX512-NEXT:    retl
 ;
 ; X64-AVX-LABEL: reg_broadcast_16i8_64i8:
 ; X64-AVX:       # %bb.0:
@@ -1482,26 +1316,12 @@ define <64 x i8> @reg_broadcast_16i8_64i8(<16 x i8> %a0) nounwind {
 ; X64-AVX-NEXT:    vmovaps %ymm0, %ymm1
 ; X64-AVX-NEXT:    retq
 ;
-; X64-AVX512F-LABEL: reg_broadcast_16i8_64i8:
-; X64-AVX512F:       # %bb.0:
-; X64-AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
-; X64-AVX512F-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
-; X64-AVX512F-NEXT:    vmovaps %ymm0, %ymm1
-; X64-AVX512F-NEXT:    retq
-;
-; X64-AVX512BW-LABEL: reg_broadcast_16i8_64i8:
-; X64-AVX512BW:       # %bb.0:
-; X64-AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
-; X64-AVX512BW-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
-; X64-AVX512BW-NEXT:    vinsertf64x4 $1, %ymm0, %zmm0, %zmm0
-; X64-AVX512BW-NEXT:    retq
-;
-; X64-AVX512DQ-LABEL: reg_broadcast_16i8_64i8:
-; X64-AVX512DQ:       # %bb.0:
-; X64-AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
-; X64-AVX512DQ-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
-; X64-AVX512DQ-NEXT:    vmovaps %ymm0, %ymm1
-; X64-AVX512DQ-NEXT:    retq
+; X64-AVX512-LABEL: reg_broadcast_16i8_64i8:
+; X64-AVX512:       # %bb.0:
+; X64-AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; X64-AVX512-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
+; X64-AVX512-NEXT:    vinsertf64x4 $1, %ymm0, %zmm0, %zmm0
+; X64-AVX512-NEXT:    retq
  %1 = shufflevector <16 x i8> %a0, <16 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
  ret <64 x i8> %1
 }
@@ -1512,42 +1332,22 @@ define <64 x i8> @reg_broadcast_32i8_64i8(<32 x i8> %a0) nounwind {
 ; X32-AVX-NEXT:    vmovaps %ymm0, %ymm1
 ; X32-AVX-NEXT:    retl
 ;
-; X32-AVX512F-LABEL: reg_broadcast_32i8_64i8:
-; X32-AVX512F:       # %bb.0:
-; X32-AVX512F-NEXT:    vmovaps %ymm0, %ymm1
-; X32-AVX512F-NEXT:    retl
-;
-; X32-AVX512BW-LABEL: reg_broadcast_32i8_64i8:
-; X32-AVX512BW:       # %bb.0:
-; X32-AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; X32-AVX512BW-NEXT:    vinsertf64x4 $1, %ymm0, %zmm0, %zmm0
-; X32-AVX512BW-NEXT:    retl
-;
-; X32-AVX512DQ-LABEL: reg_broadcast_32i8_64i8:
-; X32-AVX512DQ:       # %bb.0:
-; X32-AVX512DQ-NEXT:    vmovaps %ymm0, %ymm1
-; X32-AVX512DQ-NEXT:    retl
+; X32-AVX512-LABEL: reg_broadcast_32i8_64i8:
+; X32-AVX512:       # %bb.0:
+; X32-AVX512-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; X32-AVX512-NEXT:    vinsertf64x4 $1, %ymm0, %zmm0, %zmm0
+; X32-AVX512-NEXT:    retl
 ;
 ; X64-AVX-LABEL: reg_broadcast_32i8_64i8:
 ; X64-AVX:       # %bb.0:
 ; X64-AVX-NEXT:    vmovaps %ymm0, %ymm1
 ; X64-AVX-NEXT:    retq
 ;
-; X64-AVX512F-LABEL: reg_broadcast_32i8_64i8:
-; X64-AVX512F:       # %bb.0:
-; X64-AVX512F-NEXT:    vmovaps %ymm0, %ymm1
-; X64-AVX512F-NEXT:    retq
-;
-; X64-AVX512BW-LABEL: reg_broadcast_32i8_64i8:
-; X64-AVX512BW:       # %bb.0:
-; X64-AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; X64-AVX512BW-NEXT:    vinsertf64x4 $1, %ymm0, %zmm0, %zmm0
-; X64-AVX512BW-NEXT:    retq
-;
-; X64-AVX512DQ-LABEL: reg_broadcast_32i8_64i8:
-; X64-AVX512DQ:       # %bb.0:
-; X64-AVX512DQ-NEXT:    vmovaps %ymm0, %ymm1
-; X64-AVX512DQ-NEXT:    retq
+; X64-AVX512-LABEL: reg_broadcast_32i8_64i8:
+; X64-AVX512:       # %bb.0:
+; X64-AVX512-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; X64-AVX512-NEXT:    vinsertf64x4 $1, %ymm0, %zmm0, %zmm0
+; X64-AVX512-NEXT:    retq
  %1 = shufflevector <32 x i8> %a0, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
  ret <64 x i8> %1
 }
@@ -1662,13 +1462,13 @@ define <8 x float> @broadcast_v8f32_v2f32_u1uu0uEu(<2 x float>* %vp, <8 x float>
 ; X32:       # %bb.0:
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-NEXT:    vbroadcastsd (%eax), %ymm1
-; X32-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
+; X32-NEXT:    vshufpd {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[2],ymm0[3]
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: broadcast_v8f32_v2f32_u1uu0uEu:
 ; X64:       # %bb.0:
 ; X64-NEXT:    vbroadcastsd (%rdi), %ymm1
-; X64-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
+; X64-NEXT:    vshufpd {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[2],ymm0[3]
 ; X64-NEXT:    retq
   %vec = load <2 x float>, <2 x float>* %vp
   %shuf = shufflevector <2 x float> %vec, <2 x float> undef, <8 x i32> <i32 undef, i32 1, i32 undef, i32 undef, i32 0, i32 2, i32 3, i32 undef>
diff --git a/llvm/test/CodeGen/X86/swifterror.ll b/llvm/test/CodeGen/X86/swifterror.ll
index c37322952..1388c61c1 100644
--- a/llvm/test/CodeGen/X86/swifterror.ll
+++ b/llvm/test/CodeGen/X86/swifterror.ll
@@ -533,7 +533,7 @@ define swiftcc void @swifterror_reg_clobber(%swift_error** nocapture %err) {
 }
 
 ; CHECK-APPLE-LABEL: params_in_reg
-; Save callee save registers to store clobbered arugments.
+; Save callee save registers to store clobbered arguments.
 ; CHECK-APPLE:  pushq   %rbp
 ; CHECK-APPLE:  pushq   %r15
 ; CHECK-APPLE:  pushq   %r14
diff --git a/llvm/test/CodeGen/X86/switch-bt.ll b/llvm/test/CodeGen/X86/switch-bt.ll
index 958d053c2..965cdbf17 100644
--- a/llvm/test/CodeGen/X86/switch-bt.ll
+++ b/llvm/test/CodeGen/X86/switch-bt.ll
@@ -155,3 +155,42 @@ sw.epilog:
 ; CHECK: cmpl $60
 ; CHECK: jne
 }
+
+
+; Omit the range check when the default case is unreachable, see PR43129.
+declare void @g(i32)
+define void @test5(i32 %x) {
+
+; CHECK-LABEL: test5
+; CHECK-NOT: cmp
+
+; 73 = 2^0 + 2^3 + 2^6
+; CHECK:      movl $73
+; CHECK-NEXT: btl
+; CHECK-NEXT: jb
+
+; 146 = 2^1 + 2^4 + 2^7
+; CHECK-NEXT: movl $146
+; CHECK-NEXT: btl
+; CHECK-NEXT: jae
+
+
+entry:
+  switch i32 %x, label %return [
+    i32 0, label %bb0
+    i32 3, label %bb0
+    i32 6, label %bb0
+
+    i32 1, label %bb1
+    i32 4, label %bb1
+    i32 7, label %bb1
+
+    i32 2, label %bb2
+    i32 5, label %bb2
+    i32 8, label %bb2
+  ]
+bb0: tail call void @g(i32 0) br label %return
+bb1: tail call void @g(i32 1) br label %return
+bb2: tail call void @g(i32 2) br label %return
+return: unreachable
+}
diff --git a/llvm/test/CodeGen/X86/switch-density.ll b/llvm/test/CodeGen/X86/switch-density.ll
index 52216fb4d..8055a2b87 100644
--- a/llvm/test/CodeGen/X86/switch-density.ll
+++ b/llvm/test/CodeGen/X86/switch-density.ll
@@ -79,3 +79,72 @@ return: ret void
 ; CHECK: ja
 ; CHECK: jmpq *.LJTI
 }
+
+define void @dense_optsize(i32 %x) optsize {
+entry:
+  switch i32 %x, label %return [
+    i32 12, label %bb0
+    i32 4,  label %bb1
+    i32 16, label %bb1
+    i32 20, label %bb2
+    i32 8,  label %bb3
+  ]
+bb0: tail call void @g(i32 0) br label %return
+bb1: tail call void @g(i32 1) br label %return
+bb2: tail call void @g(i32 1) br label %return
+bb3: tail call void @g(i32 2) br label %return
+return: ret void
+
+; Lowered as branches.
+; CHECK-LABEL: dense_optsize
+; CHECK: cmpl $11
+; CHECK: cmpl $20
+; CHECK: cmpl $16
+; CHECK: cmpl $12
+; CHECK: cmpl $4
+; CHECK: cmpl $8
+; CHECK: retq
+}
+
+define void @dense_pgso(i32 %x) !prof !14 {
+entry:
+  switch i32 %x, label %return [
+    i32 12, label %bb0
+    i32 4,  label %bb1
+    i32 16, label %bb1
+    i32 20, label %bb2
+    i32 8,  label %bb3
+  ]
+bb0: tail call void @g(i32 0) br label %return
+bb1: tail call void @g(i32 1) br label %return
+bb2: tail call void @g(i32 1) br label %return
+bb3: tail call void @g(i32 2) br label %return
+return: ret void
+
+; Lowered as branches.
+; CHECK-LABEL: dense_pgso
+; CHECK: cmpl $11
+; CHECK: cmpl $20
+; CHECK: cmpl $16
+; CHECK: cmpl $12
+; CHECK: cmpl $4
+; CHECK: cmpl $8
+; CHECK: retq
+}
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 1, !"ProfileSummary", !1}
+!1 = !{!2, !3, !4, !5, !6, !7, !8, !9}
+!2 = !{!"ProfileFormat", !"InstrProf"}
+!3 = !{!"TotalCount", i64 10000}
+!4 = !{!"MaxCount", i64 10}
+!5 = !{!"MaxInternalCount", i64 1}
+!6 = !{!"MaxFunctionCount", i64 1000}
+!7 = !{!"NumCounts", i64 3}
+!8 = !{!"NumFunctions", i64 3}
+!9 = !{!"DetailedSummary", !10}
+!10 = !{!11, !12, !13}
+!11 = !{i32 10000, i64 100, i32 1}
+!12 = !{i32 999000, i64 100, i32 1}
+!13 = !{i32 999999, i64 1, i32 2}
+!14 = !{!"function_entry_count", i64 0}
diff --git a/llvm/test/CodeGen/X86/switch-zextload.ll b/llvm/test/CodeGen/X86/switch-zextload.ll
index 68cfade04..1142ef3b1 100644
--- a/llvm/test/CodeGen/X86/switch-zextload.ll
+++ b/llvm/test/CodeGen/X86/switch-zextload.ll
@@ -1,4 +1,5 @@
-; RUN: llc < %s | grep mov | count 1
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s | FileCheck %s
 
 ; Do zextload, instead of a load and a separate zext.
 
@@ -8,6 +9,17 @@ target triple = "i386-apple-darwin9.6"
 	%struct.node_t = type { i8, i8, i8, i8, i32, i32, %struct.node_t**, %struct.node_t*, %struct.move_s }
 
 define fastcc void @set_proof_and_disproof_numbers(%struct.node_t* nocapture %node) nounwind {
+; CHECK-LABEL: set_proof_and_disproof_numbers:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    movzbl 0, %eax
+; CHECK-NEXT:    cmpl $3, %eax
+; CHECK-NEXT:    ja LBB0_3
+; CHECK-NEXT:  ## %bb.1: ## %entry
+; CHECK-NEXT:    jmpl *LJTI0_0(,%eax,4)
+; CHECK-NEXT:  LBB0_3: ## %return
+; CHECK-NEXT:    retl
+; CHECK-NEXT:  LBB0_2: ## %bb31
+; CHECK-NEXT:    ud2
 entry:
 	%0 = load i8, i8* null, align 1		; <i8> [#uses=1]
 	switch i8 %0, label %return [
diff --git a/llvm/test/CodeGen/X86/tail-call-deref.ll b/llvm/test/CodeGen/X86/tail-call-deref.ll
new file mode 100644
index 000000000..5df7cf4da
--- /dev/null
+++ b/llvm/test/CodeGen/X86/tail-call-deref.ll
@@ -0,0 +1,36 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=x86_64-linux < %s | FileCheck %s
+
+declare i8* @foo()
+
+define dereferenceable(8) i8* @test1() nounwind {
+; CHECK-LABEL: test1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    jmp foo # TAILCALL
+  %ret = tail call i8* @foo()
+  ret i8* %ret
+}
+
+define i8* @test2() nounwind {
+; CHECK-LABEL: test2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    jmp foo # TAILCALL
+  %ret = tail call dereferenceable(8)  i8* @foo()
+  ret i8* %ret
+}
+
+define dereferenceable_or_null(8) i8* @test3() nounwind {
+; CHECK-LABEL: test3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    jmp foo # TAILCALL
+  %ret = tail call i8* @foo()
+  ret i8* %ret
+}
+
+define i8* @test4() nounwind {
+; CHECK-LABEL: test4:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    jmp foo # TAILCALL
+  %ret = tail call dereferenceable_or_null(8) i8* @foo()
+  ret i8* %ret
+}
diff --git a/llvm/test/CodeGen/X86/tail-dup-merge-loop-headers.ll b/llvm/test/CodeGen/X86/tail-dup-merge-loop-headers.ll
index c11ffaba1..ceca19108 100644
--- a/llvm/test/CodeGen/X86/tail-dup-merge-loop-headers.ll
+++ b/llvm/test/CodeGen/X86/tail-dup-merge-loop-headers.ll
@@ -5,22 +5,23 @@
 define void @tail_dup_merge_loops(i32 %a, i8* %b, i8* %c) local_unnamed_addr #0 {
 ; CHECK-LABEL: tail_dup_merge_loops:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    testl %edi, %edi
-; CHECK-NEXT:    jne .LBB0_2
-; CHECK-NEXT:    jmp .LBB0_5
+; CHECK-NEXT:    jmp .LBB0_1
+; CHECK-NEXT:    .p2align 4, 0x90
 ; CHECK-NEXT:  .LBB0_3: # %inner_loop_exit
-; CHECK-NEXT:    # in Loop: Header=BB0_2 Depth=1
+; CHECK-NEXT:    # in Loop: Header=BB0_1 Depth=1
 ; CHECK-NEXT:    incq %rsi
+; CHECK-NEXT:  .LBB0_1: # %outer_loop_top
+; CHECK-NEXT:    # =>This Loop Header: Depth=1
+; CHECK-NEXT:    # Child Loop BB0_4 Depth 2
 ; CHECK-NEXT:    testl %edi, %edi
 ; CHECK-NEXT:    je .LBB0_5
 ; CHECK-NEXT:    .p2align 4, 0x90
-; CHECK-NEXT:  .LBB0_2: # %inner_loop_top
-; CHECK-NEXT:    # =>This Loop Header: Depth=1
-; CHECK-NEXT:    # Child Loop BB0_4 Depth 2
+; CHECK-NEXT:  # %bb.2: # %inner_loop_top
+; CHECK-NEXT:    # in Loop: Header=BB0_1 Depth=1
 ; CHECK-NEXT:    cmpb $0, (%rsi)
 ; CHECK-NEXT:    js .LBB0_3
 ; CHECK-NEXT:  .LBB0_4: # %inner_loop_latch
-; CHECK-NEXT:    # Parent Loop BB0_2 Depth=1
+; CHECK-NEXT:    # Parent Loop BB0_1 Depth=1
 ; CHECK-NEXT:    # => This Inner Loop Header: Depth=2
 ; CHECK-NEXT:    addq $2, %rsi
 ; CHECK-NEXT:    cmpb $0, (%rsi)
@@ -97,7 +98,7 @@ define i32 @loop_shared_header(i8* %exe, i32 %exesz, i32 %headsize, i32 %min, i3
 ; CHECK-NEXT:    movl $1, %ebx
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB1_26
+; CHECK-NEXT:    jne .LBB1_27
 ; CHECK-NEXT:  # %bb.1: # %if.end19
 ; CHECK-NEXT:    movl %esi, %r13d
 ; CHECK-NEXT:    movq %rdi, %r12
@@ -108,101 +109,106 @@ define i32 @loop_shared_header(i8* %exe, i32 %exesz, i32 %headsize, i32 %min, i3
 ; CHECK-NEXT:    movq %r15, %rdi
 ; CHECK-NEXT:    callq cli_calloc
 ; CHECK-NEXT:    testl %r13d, %r13d
-; CHECK-NEXT:    je .LBB1_25
+; CHECK-NEXT:    je .LBB1_26
 ; CHECK-NEXT:  # %bb.2: # %if.end19
 ; CHECK-NEXT:    testl %ebp, %ebp
-; CHECK-NEXT:    je .LBB1_25
+; CHECK-NEXT:    je .LBB1_26
 ; CHECK-NEXT:  # %bb.3: # %if.end19
 ; CHECK-NEXT:    movq %rax, %rbx
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB1_25
+; CHECK-NEXT:    jne .LBB1_26
 ; CHECK-NEXT:  # %bb.4: # %if.end19
 ; CHECK-NEXT:    cmpq %r12, %rbx
-; CHECK-NEXT:    jb .LBB1_25
+; CHECK-NEXT:    jb .LBB1_26
 ; CHECK-NEXT:  # %bb.5: # %if.end50
 ; CHECK-NEXT:    movq %rbx, %rdi
 ; CHECK-NEXT:    movq %r15, %rdx
 ; CHECK-NEXT:    callq memcpy
 ; CHECK-NEXT:    cmpl $4, %r14d
-; CHECK-NEXT:    jb .LBB1_28
+; CHECK-NEXT:    jb .LBB1_29
 ; CHECK-NEXT:  # %bb.6: # %shared_preheader
 ; CHECK-NEXT:    movb $32, %dl
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    # implicit-def: $rcx
+; CHECK-NEXT:    jmp .LBB1_9
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB1_7: # %merge_predecessor_split
+; CHECK-NEXT:    # in Loop: Header=BB1_9 Depth=1
+; CHECK-NEXT:    movb $32, %dl
+; CHECK-NEXT:    xorl %esi, %esi
+; CHECK-NEXT:  .LBB1_8: # %outer_loop_latch
+; CHECK-NEXT:    # in Loop: Header=BB1_9 Depth=1
+; CHECK-NEXT:    movzwl %si, %esi
+; CHECK-NEXT:    decl %esi
+; CHECK-NEXT:    movzwl %si, %esi
+; CHECK-NEXT:    leaq 1(%rcx,%rsi), %rcx
+; CHECK-NEXT:  .LBB1_9: # %outer_loop_header
+; CHECK-NEXT:    # =>This Loop Header: Depth=1
+; CHECK-NEXT:    # Child Loop BB1_10 Depth 2
 ; CHECK-NEXT:    testl %ebp, %ebp
-; CHECK-NEXT:    je .LBB1_18
+; CHECK-NEXT:    je .LBB1_19
 ; CHECK-NEXT:    .p2align 4, 0x90
-; CHECK-NEXT:  .LBB1_9: # %shared_loop_header
-; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:  .LBB1_10: # %shared_loop_header
+; CHECK-NEXT:    # Parent Loop BB1_9 Depth=1
+; CHECK-NEXT:    # => This Inner Loop Header: Depth=2
 ; CHECK-NEXT:    testq %rbx, %rbx
-; CHECK-NEXT:    jne .LBB1_27
-; CHECK-NEXT:  # %bb.10: # %inner_loop_body
-; CHECK-NEXT:    # in Loop: Header=BB1_9 Depth=1
+; CHECK-NEXT:    jne .LBB1_28
+; CHECK-NEXT:  # %bb.11: # %inner_loop_body
+; CHECK-NEXT:    # in Loop: Header=BB1_10 Depth=2
 ; CHECK-NEXT:    testl %eax, %eax
-; CHECK-NEXT:    jns .LBB1_9
-; CHECK-NEXT:  # %bb.11: # %if.end96.i
+; CHECK-NEXT:    jns .LBB1_10
+; CHECK-NEXT:  # %bb.12: # %if.end96.i
 ; CHECK-NEXT:    # in Loop: Header=BB1_9 Depth=1
 ; CHECK-NEXT:    cmpl $3, %ebp
-; CHECK-NEXT:    jae .LBB1_22
-; CHECK-NEXT:  # %bb.12: # %if.end287.i
+; CHECK-NEXT:    jae .LBB1_23
+; CHECK-NEXT:  # %bb.13: # %if.end287.i
 ; CHECK-NEXT:    # in Loop: Header=BB1_9 Depth=1
 ; CHECK-NEXT:    xorl %esi, %esi
 ; CHECK-NEXT:    cmpl $1, %ebp
 ; CHECK-NEXT:    setne %dl
 ; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB1_16
-; CHECK-NEXT:  # %bb.13: # %if.end308.i
+; CHECK-NEXT:    jne .LBB1_17
+; CHECK-NEXT:  # %bb.14: # %if.end308.i
 ; CHECK-NEXT:    # in Loop: Header=BB1_9 Depth=1
 ; CHECK-NEXT:    testb %al, %al
 ; CHECK-NEXT:    je .LBB1_7
-; CHECK-NEXT:  # %bb.14: # %if.end335.i
+; CHECK-NEXT:  # %bb.15: # %if.end335.i
 ; CHECK-NEXT:    # in Loop: Header=BB1_9 Depth=1
 ; CHECK-NEXT:    xorl %edx, %edx
 ; CHECK-NEXT:    testb %dl, %dl
 ; CHECK-NEXT:    movl $0, %esi
 ; CHECK-NEXT:    jne .LBB1_8
-; CHECK-NEXT:  # %bb.15: # %merge_other
+; CHECK-NEXT:  # %bb.16: # %merge_other
 ; CHECK-NEXT:    # in Loop: Header=BB1_9 Depth=1
 ; CHECK-NEXT:    xorl %esi, %esi
-; CHECK-NEXT:    jmp .LBB1_17
-; CHECK-NEXT:  .LBB1_16: # in Loop: Header=BB1_9 Depth=1
+; CHECK-NEXT:    jmp .LBB1_18
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB1_17: # in Loop: Header=BB1_9 Depth=1
 ; CHECK-NEXT:    movb %dl, %sil
 ; CHECK-NEXT:    addl $3, %esi
-; CHECK-NEXT:  .LBB1_17: # %outer_loop_latch
+; CHECK-NEXT:  .LBB1_18: # %outer_loop_latch
 ; CHECK-NEXT:    # in Loop: Header=BB1_9 Depth=1
 ; CHECK-NEXT:    # implicit-def: $dl
 ; CHECK-NEXT:    jmp .LBB1_8
-; CHECK-NEXT:  .LBB1_7: # %merge_predecessor_split
-; CHECK-NEXT:    # in Loop: Header=BB1_9 Depth=1
-; CHECK-NEXT:    movb $32, %dl
-; CHECK-NEXT:    xorl %esi, %esi
-; CHECK-NEXT:  .LBB1_8: # %outer_loop_latch
-; CHECK-NEXT:    # in Loop: Header=BB1_9 Depth=1
-; CHECK-NEXT:    movzwl %si, %esi
-; CHECK-NEXT:    decl %esi
-; CHECK-NEXT:    movzwl %si, %esi
-; CHECK-NEXT:    leaq 1(%rcx,%rsi), %rcx
-; CHECK-NEXT:    testl %ebp, %ebp
-; CHECK-NEXT:    jne .LBB1_9
-; CHECK-NEXT:  .LBB1_18: # %while.cond.us1412.i
+; CHECK-NEXT:  .LBB1_26:
+; CHECK-NEXT:    movl $1, %ebx
+; CHECK-NEXT:    jmp .LBB1_27
+; CHECK-NEXT:  .LBB1_19: # %while.cond.us1412.i
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    testb %al, %al
 ; CHECK-NEXT:    movl $1, %ebx
-; CHECK-NEXT:    jne .LBB1_20
-; CHECK-NEXT:  # %bb.19: # %while.cond.us1412.i
+; CHECK-NEXT:    jne .LBB1_21
+; CHECK-NEXT:  # %bb.20: # %while.cond.us1412.i
 ; CHECK-NEXT:    decb %dl
-; CHECK-NEXT:    jne .LBB1_26
-; CHECK-NEXT:  .LBB1_20: # %if.end41.us1436.i
-; CHECK-NEXT:  .LBB1_25:
-; CHECK-NEXT:    movl $1, %ebx
-; CHECK-NEXT:    jmp .LBB1_26
-; CHECK-NEXT:  .LBB1_22: # %if.then99.i
+; CHECK-NEXT:    jne .LBB1_27
+; CHECK-NEXT:  .LBB1_21: # %if.end41.us1436.i
+; CHECK-NEXT:  .LBB1_23: # %if.then99.i
 ; CHECK-NEXT:    xorl %ebx, %ebx
 ; CHECK-NEXT:    movl $.str.6, %edi
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    callq cli_dbgmsg
-; CHECK-NEXT:  .LBB1_26: # %cleanup
+; CHECK-NEXT:  .LBB1_27: # %cleanup
 ; CHECK-NEXT:    movl %ebx, %eax
 ; CHECK-NEXT:    addq $8, %rsp
 ; CHECK-NEXT:    popq %rbx
@@ -212,8 +218,8 @@ define i32 @loop_shared_header(i8* %exe, i32 %exesz, i32 %headsize, i32 %min, i3
 ; CHECK-NEXT:    popq %r15
 ; CHECK-NEXT:    popq %rbp
 ; CHECK-NEXT:    retq
-; CHECK-NEXT:  .LBB1_27: # %wunpsect.exit.thread.loopexit389
-; CHECK-NEXT:  .LBB1_28: # %wunpsect.exit.thread.loopexit391
+; CHECK-NEXT:  .LBB1_28: # %wunpsect.exit.thread.loopexit389
+; CHECK-NEXT:  .LBB1_29: # %wunpsect.exit.thread.loopexit391
 entry:
   %0 = load i32, i32* undef, align 4
   %mul = shl nsw i32 %0, 2
diff --git a/llvm/test/CodeGen/X86/tail-dup-repeat.ll b/llvm/test/CodeGen/X86/tail-dup-repeat.ll
index bfa1ee611..9a06c9a72 100644
--- a/llvm/test/CodeGen/X86/tail-dup-repeat.ll
+++ b/llvm/test/CodeGen/X86/tail-dup-repeat.ll
@@ -10,28 +10,33 @@
 define void @repeated_tail_dup(i1 %a1, i1 %a2, i32* %a4, i32* %a5, i8* %a6, i32 %a7) #0 align 2 {
 ; CHECK-LABEL: repeated_tail_dup:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    testb $1, %dil
-; CHECK-NEXT:    je .LBB0_3
+; CHECK-NEXT:    jmp .LBB0_1
 ; CHECK-NEXT:    .p2align 4, 0x90
 ; CHECK-NEXT:  .LBB0_2: # %land.lhs.true
+; CHECK-NEXT:    # in Loop: Header=BB0_1 Depth=1
 ; CHECK-NEXT:    movl $10, (%rdx)
 ; CHECK-NEXT:  .LBB0_6: # %dup2
+; CHECK-NEXT:    # in Loop: Header=BB0_1 Depth=1
 ; CHECK-NEXT:    movl $2, (%rcx)
 ; CHECK-NEXT:    testl %r9d, %r9d
 ; CHECK-NEXT:    jne .LBB0_8
 ; CHECK-NEXT:  .LBB0_1: # %for.cond
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    testb $1, %dil
 ; CHECK-NEXT:    jne .LBB0_2
-; CHECK-NEXT:  .LBB0_3: # %if.end56
+; CHECK-NEXT:  # %bb.3: # %if.end56
+; CHECK-NEXT:    # in Loop: Header=BB0_1 Depth=1
 ; CHECK-NEXT:    testb $1, %sil
 ; CHECK-NEXT:    je .LBB0_5
 ; CHECK-NEXT:  # %bb.4: # %if.then64
+; CHECK-NEXT:    # in Loop: Header=BB0_1 Depth=1
 ; CHECK-NEXT:    movb $1, (%r8)
 ; CHECK-NEXT:    testl %r9d, %r9d
 ; CHECK-NEXT:    je .LBB0_1
 ; CHECK-NEXT:    jmp .LBB0_8
 ; CHECK-NEXT:    .p2align 4, 0x90
 ; CHECK-NEXT:  .LBB0_5: # %if.end70
+; CHECK-NEXT:    # in Loop: Header=BB0_1 Depth=1
 ; CHECK-NEXT:    movl $12, (%rdx)
 ; CHECK-NEXT:    jmp .LBB0_6
 ; CHECK-NEXT:  .LBB0_8: # %for.end
diff --git a/llvm/test/CodeGen/X86/tail-merge-wineh.ll b/llvm/test/CodeGen/X86/tail-merge-wineh.ll
index 69c2fda69..aef6a0e69 100644
--- a/llvm/test/CodeGen/X86/tail-merge-wineh.ll
+++ b/llvm/test/CodeGen/X86/tail-merge-wineh.ll
@@ -103,5 +103,5 @@ declare x86_stdcallcc void @_CxxThrowException(i8*, %eh.ThrowInfo*)
 
 declare i32 @__CxxFrameHandler3(...)
 
-attributes #0 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { noreturn }
diff --git a/llvm/test/CodeGen/X86/tail-opts.ll b/llvm/test/CodeGen/X86/tail-opts.ll
index 63189af6a..32ad04a65 100644
--- a/llvm/test/CodeGen/X86/tail-opts.ll
+++ b/llvm/test/CodeGen/X86/tail-opts.ll
@@ -242,75 +242,68 @@ define fastcc void @c_expand_expr_stmt(%union.tree_node* %expr) nounwind {
 ; CHECK-NEXT:    pushq %rbx
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB3_17
+; CHECK-NEXT:    jne .LBB3_9
 ; CHECK-NEXT:  # %bb.1: # %entry
 ; CHECK-NEXT:    movb 0, %bl
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB3_16
+; CHECK-NEXT:    jne .LBB3_8
 ; CHECK-NEXT:  # %bb.2: # %bb.i
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    je .LBB3_16
+; CHECK-NEXT:    je .LBB3_8
 ; CHECK-NEXT:  # %bb.3: # %lvalue_p.exit
 ; CHECK-NEXT:    movq 0, %rax
 ; CHECK-NEXT:    movzbl (%rax), %ecx
 ; CHECK-NEXT:    testl %ecx, %ecx
-; CHECK-NEXT:    je .LBB3_12
+; CHECK-NEXT:    je .LBB3_10
 ; CHECK-NEXT:  # %bb.4: # %lvalue_p.exit
 ; CHECK-NEXT:    cmpl $2, %ecx
-; CHECK-NEXT:    jne .LBB3_5
-; CHECK-NEXT:  # %bb.6: # %bb.i1
+; CHECK-NEXT:    jne .LBB3_15
+; CHECK-NEXT:  # %bb.5: # %bb.i1
 ; CHECK-NEXT:    movq 32(%rax), %rax
 ; CHECK-NEXT:    movzbl 16(%rax), %ecx
 ; CHECK-NEXT:    testl %ecx, %ecx
-; CHECK-NEXT:    je .LBB3_10
-; CHECK-NEXT:  # %bb.7: # %bb.i1
+; CHECK-NEXT:    je .LBB3_13
+; CHECK-NEXT:  # %bb.6: # %bb.i1
 ; CHECK-NEXT:    cmpl $2, %ecx
-; CHECK-NEXT:    jne .LBB3_8
-; CHECK-NEXT:  # %bb.9: # %bb.i.i
+; CHECK-NEXT:    jne .LBB3_15
+; CHECK-NEXT:  # %bb.7: # %bb.i.i
 ; CHECK-NEXT:    xorl %edi, %edi
 ; CHECK-NEXT:    callq lvalue_p
 ; CHECK-NEXT:    testl %eax, %eax
 ; CHECK-NEXT:    setne %al
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    je .LBB3_15
-; CHECK-NEXT:    jmp .LBB3_17
-; CHECK-NEXT:  .LBB3_16: # %bb1
+; CHECK-NEXT:    jmp .LBB3_16
+; CHECK-NEXT:  .LBB3_8: # %bb1
 ; CHECK-NEXT:    cmpb $23, %bl
-; CHECK-NEXT:  .LBB3_17: # %bb3
-; CHECK-NEXT:  .LBB3_12: # %bb2.i3
+; CHECK-NEXT:  .LBB3_9: # %bb3
+; CHECK-NEXT:  .LBB3_15:
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:  .LBB3_16: # %lvalue_p.exit4
+; CHECK-NEXT:    testb %al, %al
+; CHECK-NEXT:    jne .LBB3_9
+; CHECK-NEXT:  # %bb.17: # %lvalue_p.exit4
+; CHECK-NEXT:    testb %bl, %bl
+; CHECK-NEXT:  .LBB3_10: # %bb2.i3
 ; CHECK-NEXT:    movq 8(%rax), %rax
 ; CHECK-NEXT:    movb 16(%rax), %cl
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    cmpb $23, %cl
-; CHECK-NEXT:    je .LBB3_14
-; CHECK-NEXT:  # %bb.13: # %bb2.i3
+; CHECK-NEXT:    je .LBB3_16
+; CHECK-NEXT:  # %bb.11: # %bb2.i3
 ; CHECK-NEXT:    cmpb $16, %cl
-; CHECK-NEXT:    je .LBB3_14
-; CHECK-NEXT:    jmp .LBB3_17
-; CHECK-NEXT:  .LBB3_5:
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    je .LBB3_15
-; CHECK-NEXT:    jmp .LBB3_17
-; CHECK-NEXT:  .LBB3_10: # %bb2.i.i2
+; CHECK-NEXT:    je .LBB3_16
+; CHECK-NEXT:    jmp .LBB3_9
+; CHECK-NEXT:  .LBB3_13: # %bb2.i.i2
 ; CHECK-NEXT:    movq 8(%rax), %rax
 ; CHECK-NEXT:    movb 16(%rax), %cl
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    cmpb $16, %cl
-; CHECK-NEXT:    je .LBB3_14
-; CHECK-NEXT:  # %bb.11: # %bb2.i.i2
+; CHECK-NEXT:    je .LBB3_16
+; CHECK-NEXT:  # %bb.14: # %bb2.i.i2
 ; CHECK-NEXT:    cmpb $23, %cl
-; CHECK-NEXT:    je .LBB3_14
-; CHECK-NEXT:    jmp .LBB3_17
-; CHECK-NEXT:  .LBB3_8:
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:  .LBB3_14: # %lvalue_p.exit4
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB3_17
-; CHECK-NEXT:  .LBB3_15: # %lvalue_p.exit4
-; CHECK-NEXT:    testb %bl, %bl
+; CHECK-NEXT:    je .LBB3_16
+; CHECK-NEXT:    jmp .LBB3_9
 entry:
   %tmp4 = load i8, i8* null, align 8                  ; <i8> [#uses=3]
   switch i8 %tmp4, label %bb3 [
@@ -480,6 +473,47 @@ return:
   ret void
 }
 
+define void @one_pgso(i32 %v) nounwind !prof !14 {
+; CHECK-LABEL: one_pgso:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    testl %edi, %edi
+; CHECK-NEXT:    je .LBB6_3
+; CHECK-NEXT:  # %bb.1: # %bby
+; CHECK-NEXT:    cmpl $16, %edi
+; CHECK-NEXT:    je .LBB6_4
+; CHECK-NEXT:  # %bb.2: # %bb7
+; CHECK-NEXT:    jmp tail_call_me # TAILCALL
+; CHECK-NEXT:  .LBB6_3: # %bbx
+; CHECK-NEXT:    cmpl $128, %edi
+; CHECK-NEXT:    jne tail_call_me # TAILCALL
+; CHECK-NEXT:  .LBB6_4: # %return
+; CHECK-NEXT:    retq
+entry:
+  %0 = icmp eq i32 %v, 0
+  br i1 %0, label %bbx, label %bby
+
+bby:
+  switch i32 %v, label %bb7 [
+    i32 16, label %return
+  ]
+
+bb7:
+  tail call void @tail_call_me()
+  ret void
+
+bbx:
+  switch i32 %v, label %bb12 [
+    i32 128, label %return
+  ]
+
+bb12:
+  tail call void @tail_call_me()
+  ret void
+
+return:
+  ret void
+}
+
 ; two - Same as one, but with two instructions in the common
 ; tail instead of one. This is too much to be merged, given
 ; the optsize attribute.
@@ -491,10 +525,51 @@ define void @two() nounwind optsize {
 ; CHECK-NEXT:    testb %al, %al
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    je .LBB6_1
+; CHECK-NEXT:    je .LBB7_1
+; CHECK-NEXT:  # %bb.2: # %return
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  .LBB7_1: # %bb7
+; CHECK-NEXT:    movl $0, {{.*}}(%rip)
+; CHECK-NEXT:    movl $1, {{.*}}(%rip)
+entry:
+  %0 = icmp eq i32 undef, 0
+  br i1 %0, label %bbx, label %bby
+
+bby:
+  switch i32 undef, label %bb7 [
+    i32 16, label %return
+  ]
+
+bb7:
+  store volatile i32 0, i32* @XYZ
+  store volatile i32 1, i32* @XYZ
+  unreachable
+
+bbx:
+  switch i32 undef, label %bb12 [
+    i32 128, label %return
+  ]
+
+bb12:
+  store volatile i32 0, i32* @XYZ
+  store volatile i32 1, i32* @XYZ
+  unreachable
+
+return:
+  ret void
+}
+
+define void @two_pgso() nounwind !prof !14 {
+; CHECK-LABEL: two_pgso:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    testb %al, %al
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    testb %al, %al
+; CHECK-NEXT:    je .LBB8_1
 ; CHECK-NEXT:  # %bb.2: # %return
 ; CHECK-NEXT:    retq
-; CHECK-NEXT:  .LBB6_1: # %bb7
+; CHECK-NEXT:  .LBB8_1: # %bb7
 ; CHECK-NEXT:    movl $0, {{.*}}(%rip)
 ; CHECK-NEXT:    movl $1, {{.*}}(%rip)
 entry:
@@ -534,10 +609,10 @@ define void @two_minsize() nounwind minsize {
 ; CHECK-NEXT:    testb %al, %al
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    je .LBB7_1
+; CHECK-NEXT:    je .LBB9_1
 ; CHECK-NEXT:  # %bb.2: # %return
 ; CHECK-NEXT:    retq
-; CHECK-NEXT:  .LBB7_1: # %bb7
+; CHECK-NEXT:  .LBB9_1: # %bb7
 ; CHECK-NEXT:    movl $0, {{.*}}(%rip)
 ; CHECK-NEXT:    movl $1, {{.*}}(%rip)
 entry:
@@ -575,20 +650,20 @@ define void @two_nosize(i32 %x, i32 %y, i32 %z) nounwind {
 ; CHECK-LABEL: two_nosize:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    testl %edi, %edi
-; CHECK-NEXT:    je .LBB8_3
+; CHECK-NEXT:    je .LBB10_3
 ; CHECK-NEXT:  # %bb.1: # %bby
 ; CHECK-NEXT:    testl %esi, %esi
-; CHECK-NEXT:    je .LBB8_4
+; CHECK-NEXT:    je .LBB10_4
 ; CHECK-NEXT:  # %bb.2: # %bb7
 ; CHECK-NEXT:    movl $0, {{.*}}(%rip)
 ; CHECK-NEXT:    jmp tail_call_me # TAILCALL
-; CHECK-NEXT:  .LBB8_3: # %bbx
+; CHECK-NEXT:  .LBB10_3: # %bbx
 ; CHECK-NEXT:    cmpl $-1, %edx
-; CHECK-NEXT:    je .LBB8_4
+; CHECK-NEXT:    je .LBB10_4
 ; CHECK-NEXT:  # %bb.5: # %bb12
 ; CHECK-NEXT:    movl $0, {{.*}}(%rip)
 ; CHECK-NEXT:    jmp tail_call_me # TAILCALL
-; CHECK-NEXT:  .LBB8_4: # %return
+; CHECK-NEXT:  .LBB10_4: # %return
 ; CHECK-NEXT:    retq
 entry:
   %0 = icmp eq i32 %x, 0
@@ -628,11 +703,11 @@ define i64 @TESTE(i64 %parami, i64 %paraml) nounwind readnone {
 ; CHECK-NEXT:    movl $1, %eax
 ; CHECK-NEXT:    cmovgq %rdi, %rax
 ; CHECK-NEXT:    testq %rsi, %rsi
-; CHECK-NEXT:    jle .LBB9_2
+; CHECK-NEXT:    jle .LBB11_2
 ; CHECK-NEXT:  # %bb.1: # %bb.nph
 ; CHECK-NEXT:    imulq %rdi, %rsi
 ; CHECK-NEXT:    movq %rsi, %rax
-; CHECK-NEXT:  .LBB9_2: # %for.end
+; CHECK-NEXT:  .LBB11_2: # %for.end
 ; CHECK-NEXT:    retq
 entry:
   %cmp = icmp slt i64 %parami, 1                  ; <i1> [#uses=1]
@@ -661,24 +736,24 @@ define void @merge_aborts() {
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    callq qux
 ; CHECK-NEXT:    testb $1, %al
-; CHECK-NEXT:    je .LBB10_5
+; CHECK-NEXT:    je .LBB12_5
 ; CHECK-NEXT:  # %bb.1: # %cont1
 ; CHECK-NEXT:    callq qux
 ; CHECK-NEXT:    testb $1, %al
-; CHECK-NEXT:    je .LBB10_5
+; CHECK-NEXT:    je .LBB12_5
 ; CHECK-NEXT:  # %bb.2: # %cont2
 ; CHECK-NEXT:    callq qux
 ; CHECK-NEXT:    testb $1, %al
-; CHECK-NEXT:    je .LBB10_5
+; CHECK-NEXT:    je .LBB12_5
 ; CHECK-NEXT:  # %bb.3: # %cont3
 ; CHECK-NEXT:    callq qux
 ; CHECK-NEXT:    testb $1, %al
-; CHECK-NEXT:    je .LBB10_5
+; CHECK-NEXT:    je .LBB12_5
 ; CHECK-NEXT:  # %bb.4: # %cont4
 ; CHECK-NEXT:    popq %rax
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK-NEXT:    retq
-; CHECK-NEXT:  .LBB10_5: # %abort1
+; CHECK-NEXT:  .LBB12_5: # %abort1
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    callq abort
 entry:
@@ -721,27 +796,27 @@ define void @merge_alternating_aborts() {
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    callq qux
 ; CHECK-NEXT:    testb $1, %al
-; CHECK-NEXT:    je .LBB11_5
+; CHECK-NEXT:    je .LBB13_5
 ; CHECK-NEXT:  # %bb.1: # %cont1
 ; CHECK-NEXT:    callq qux
 ; CHECK-NEXT:    testb $1, %al
-; CHECK-NEXT:    je .LBB11_6
+; CHECK-NEXT:    je .LBB13_6
 ; CHECK-NEXT:  # %bb.2: # %cont2
 ; CHECK-NEXT:    callq qux
 ; CHECK-NEXT:    testb $1, %al
-; CHECK-NEXT:    je .LBB11_5
+; CHECK-NEXT:    je .LBB13_5
 ; CHECK-NEXT:  # %bb.3: # %cont3
 ; CHECK-NEXT:    callq qux
 ; CHECK-NEXT:    testb $1, %al
-; CHECK-NEXT:    je .LBB11_6
+; CHECK-NEXT:    je .LBB13_6
 ; CHECK-NEXT:  # %bb.4: # %cont4
 ; CHECK-NEXT:    popq %rax
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK-NEXT:    retq
-; CHECK-NEXT:  .LBB11_5: # %abort1
+; CHECK-NEXT:  .LBB13_5: # %abort1
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    callq abort
-; CHECK-NEXT:  .LBB11_6: # %abort2
+; CHECK-NEXT:  .LBB13_6: # %abort2
 ; CHECK-NEXT:    callq alt_abort
 entry:
   %c1 = call i1 @qux()
@@ -770,3 +845,20 @@ abort4:
 cont4:
   ret void
 }
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 1, !"ProfileSummary", !1}
+!1 = !{!2, !3, !4, !5, !6, !7, !8, !9}
+!2 = !{!"ProfileFormat", !"InstrProf"}
+!3 = !{!"TotalCount", i64 10000}
+!4 = !{!"MaxCount", i64 10}
+!5 = !{!"MaxInternalCount", i64 1}
+!6 = !{!"MaxFunctionCount", i64 1000}
+!7 = !{!"NumCounts", i64 3}
+!8 = !{!"NumFunctions", i64 3}
+!9 = !{!"DetailedSummary", !10}
+!10 = !{!11, !12, !13}
+!11 = !{i32 10000, i64 100, i32 1}
+!12 = !{i32 999000, i64 100, i32 1}
+!13 = !{i32 999999, i64 1, i32 2}
+!14 = !{!"function_entry_count", i64 0}
diff --git a/llvm/test/CodeGen/X86/tailcall-assume.ll b/llvm/test/CodeGen/X86/tailcall-assume.ll
new file mode 100644
index 000000000..3baac9c41
--- /dev/null
+++ b/llvm/test/CodeGen/X86/tailcall-assume.ll
@@ -0,0 +1,15 @@
+; RUN: llc -mtriple=x86_64-linux < %s | FileCheck %s
+
+; Intrinsic call to @llvm.assume should not prevent tail call optimization.
+; CHECK-LABEL: foo:
+; CHECK:       jmp bar # TAILCALL
+define i8* @foo() {
+  %1 = tail call i8* @bar()
+  %2 = icmp ne i8* %1, null
+  tail call void @llvm.assume(i1 %2)
+  ret i8* %1
+}
+
+declare i8* @bar()
+declare void @llvm.assume(i1)
+
diff --git a/llvm/test/CodeGen/X86/tailcall-tailcc.ll b/llvm/test/CodeGen/X86/tailcall-tailcc.ll
new file mode 100644
index 000000000..5a427034a
--- /dev/null
+++ b/llvm/test/CodeGen/X86/tailcall-tailcc.ll
@@ -0,0 +1,155 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s -check-prefix=X64
+; RUN: llc < %s -mtriple=i686-unknown-unknown   | FileCheck %s -check-prefix=X32
+
+; With -tailcallopt, CodeGen guarantees a tail call optimization
+; for all of these.
+
+declare tailcc i32 @tailcallee(i32 %a1, i32 %a2, i32 %a3, i32 %a4)
+
+define tailcc i32 @tailcaller(i32 %in1, i32 %in2) nounwind {
+; X64-LABEL: tailcaller:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl %edi, %edx
+; X64-NEXT:    movl %esi, %ecx
+; X64-NEXT:    popq %rax
+; X64-NEXT:    jmp tailcallee # TAILCALL
+;
+; X32-LABEL: tailcaller:
+; X32:       # %bb.0: # %entry
+; X32-NEXT:    subl $16, %esp
+; X32-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X32-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X32-NEXT:    addl $8, %esp
+; X32-NEXT:    jmp tailcallee # TAILCALL
+entry:
+  %tmp11 = tail call tailcc i32 @tailcallee(i32 %in1, i32 %in2, i32 %in1, i32 %in2)
+  ret i32 %tmp11
+}
+
+declare tailcc i8* @alias_callee()
+
+define tailcc noalias i8* @noalias_caller() nounwind {
+; X64-LABEL: noalias_caller:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    popq %rax
+; X64-NEXT:    jmp alias_callee # TAILCALL
+;
+; X32-LABEL: noalias_caller:
+; X32:       # %bb.0:
+; X32-NEXT:    jmp alias_callee # TAILCALL
+  %p = tail call tailcc i8* @alias_callee()
+  ret i8* %p
+}
+
+declare tailcc noalias i8* @noalias_callee()
+
+define tailcc i8* @alias_caller() nounwind {
+; X64-LABEL: alias_caller:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    popq %rax
+; X64-NEXT:    jmp noalias_callee # TAILCALL
+;
+; X32-LABEL: alias_caller:
+; X32:       # %bb.0:
+; X32-NEXT:    jmp noalias_callee # TAILCALL
+  %p = tail call tailcc noalias i8* @noalias_callee()
+  ret i8* %p
+}
+
+declare tailcc i32 @i32_callee()
+
+define tailcc i32 @ret_undef() nounwind {
+; X64-LABEL: ret_undef:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    popq %rax
+; X64-NEXT:    jmp i32_callee # TAILCALL
+;
+; X32-LABEL: ret_undef:
+; X32:       # %bb.0:
+; X32-NEXT:    jmp i32_callee # TAILCALL
+  %p = tail call tailcc i32 @i32_callee()
+  ret i32 undef
+}
+
+declare tailcc void @does_not_return()
+
+define tailcc i32 @noret() nounwind {
+; X64-LABEL: noret:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    popq %rax
+; X64-NEXT:    jmp does_not_return # TAILCALL
+;
+; X32-LABEL: noret:
+; X32:       # %bb.0:
+; X32-NEXT:    jmp does_not_return # TAILCALL
+  tail call tailcc void @does_not_return()
+  unreachable
+}
+
+define tailcc void @void_test(i32, i32, i32, i32) {
+; X64-LABEL: void_test:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    .cfi_def_cfa_offset 16
+; X64-NEXT:    popq %rax
+; X64-NEXT:    .cfi_def_cfa_offset 8
+; X64-NEXT:    jmp void_test # TAILCALL
+;
+; X32-LABEL: void_test:
+; X32:       # %bb.0: # %entry
+; X32-NEXT:    pushl %esi
+; X32-NEXT:    .cfi_def_cfa_offset 8
+; X32-NEXT:    subl $8, %esp
+; X32-NEXT:    .cfi_def_cfa_offset 16
+; X32-NEXT:    .cfi_offset %esi, -8
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X32-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X32-NEXT:    addl $8, %esp
+; X32-NEXT:    .cfi_def_cfa_offset 8
+; X32-NEXT:    popl %esi
+; X32-NEXT:    .cfi_def_cfa_offset 4
+; X32-NEXT:    jmp void_test # TAILCALL
+  entry:
+   tail call tailcc void @void_test( i32 %0, i32 %1, i32 %2, i32 %3)
+   ret void
+}
+
+define tailcc i1 @i1test(i32, i32, i32, i32) {
+; X64-LABEL: i1test:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    .cfi_def_cfa_offset 16
+; X64-NEXT:    popq %rax
+; X64-NEXT:    .cfi_def_cfa_offset 8
+; X64-NEXT:    jmp i1test # TAILCALL
+;
+; X32-LABEL: i1test:
+; X32:       # %bb.0: # %entry
+; X32-NEXT:    pushl %esi
+; X32-NEXT:    .cfi_def_cfa_offset 8
+; X32-NEXT:    subl $8, %esp
+; X32-NEXT:    .cfi_def_cfa_offset 16
+; X32-NEXT:    .cfi_offset %esi, -8
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X32-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X32-NEXT:    addl $8, %esp
+; X32-NEXT:    .cfi_def_cfa_offset 8
+; X32-NEXT:    popl %esi
+; X32-NEXT:    .cfi_def_cfa_offset 4
+; X32-NEXT:    jmp i1test # TAILCALL
+  entry:
+  %4 = tail call tailcc i1 @i1test( i32 %0, i32 %1, i32 %2, i32 %3)
+  ret i1 %4
+}
diff --git a/llvm/test/CodeGen/X86/tailcall.ll b/llvm/test/CodeGen/X86/tailcall.ll
index 1f49aee90..a710e9ae0 100644
--- a/llvm/test/CodeGen/X86/tailcall.ll
+++ b/llvm/test/CodeGen/X86/tailcall.ll
@@ -1,4 +1,5 @@
-; RUN: llc < %s -mtriple=i686-- -tailcallopt | grep TAILCALL | count 7
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- -tailcallopt | FileCheck %s
 
 ; With -tailcallopt, CodeGen guarantees a tail call optimization
 ; for all of these.
@@ -6,6 +7,15 @@
 declare fastcc i32 @tailcallee(i32 %a1, i32 %a2, i32 %a3, i32 %a4)
 
 define fastcc i32 @tailcaller(i32 %in1, i32 %in2) nounwind {
+; CHECK-LABEL: tailcaller:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    subl $16, %esp
+; CHECK-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    addl $8, %esp
+; CHECK-NEXT:    jmp tailcallee # TAILCALL
 entry:
   %tmp11 = tail call fastcc i32 @tailcallee(i32 %in1, i32 %in2, i32 %in1, i32 %in2)
   ret i32 %tmp11
@@ -14,6 +24,9 @@ entry:
 declare fastcc i8* @alias_callee()
 
 define fastcc noalias i8* @noalias_caller() nounwind {
+; CHECK-LABEL: noalias_caller:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    jmp alias_callee # TAILCALL
   %p = tail call fastcc i8* @alias_callee()
   ret i8* %p
 }
@@ -21,6 +34,9 @@ define fastcc noalias i8* @noalias_caller() nounwind {
 declare fastcc noalias i8* @noalias_callee()
 
 define fastcc i8* @alias_caller() nounwind {
+; CHECK-LABEL: alias_caller:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    jmp noalias_callee # TAILCALL
   %p = tail call fastcc noalias i8* @noalias_callee()
   ret i8* %p
 }
@@ -28,6 +44,9 @@ define fastcc i8* @alias_caller() nounwind {
 declare fastcc i32 @i32_callee()
 
 define fastcc i32 @ret_undef() nounwind {
+; CHECK-LABEL: ret_undef:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    jmp i32_callee # TAILCALL
   %p = tail call fastcc i32 @i32_callee()
   ret i32 undef
 }
@@ -35,17 +54,52 @@ define fastcc i32 @ret_undef() nounwind {
 declare fastcc void @does_not_return()
 
 define fastcc i32 @noret() nounwind {
+; CHECK-LABEL: noret:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    jmp does_not_return # TAILCALL
   tail call fastcc void @does_not_return()
   unreachable
 }
 
 define fastcc void @void_test(i32, i32, i32, i32) {
+; CHECK-LABEL: void_test:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushl %esi
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    subl $8, %esp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset %esi, -8
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; CHECK-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    addl $8, %esp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    popl %esi
+; CHECK-NEXT:    .cfi_def_cfa_offset 4
+; CHECK-NEXT:    jmp void_test # TAILCALL
   entry:
    tail call fastcc void @void_test( i32 %0, i32 %1, i32 %2, i32 %3)
-   ret void 
+   ret void
 }
 
 define fastcc i1 @i1test(i32, i32, i32, i32) {
+; CHECK-LABEL: i1test:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushl %esi
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    subl $8, %esp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset %esi, -8
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; CHECK-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    addl $8, %esp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    popl %esi
+; CHECK-NEXT:    .cfi_def_cfa_offset 4
+; CHECK-NEXT:    jmp i1test # TAILCALL
   entry:
   %4 = tail call fastcc i1 @i1test( i32 %0, i32 %1, i32 %2, i32 %3)
   ret i1 %4
diff --git a/llvm/test/CodeGen/X86/tailcc-calleesave.ll b/llvm/test/CodeGen/X86/tailcc-calleesave.ll
new file mode 100644
index 000000000..09685fb17
--- /dev/null
+++ b/llvm/test/CodeGen/X86/tailcc-calleesave.ll
@@ -0,0 +1,19 @@
+; RUN: llc -mcpu=core < %s | FileCheck %s
+
+target triple = "i686-apple-darwin"
+
+declare tailcc void @foo(i32, i32, i32, i32, i32, i32)
+declare i32* @bar(i32*)
+
+define tailcc void @hoge(i32 %b) nounwind {
+; Do not overwrite pushed callee-save registers
+; CHECK: pushl
+; CHECK: subl $[[SIZE:[0-9]+]], %esp
+; CHECK-NOT: [[SIZE]](%esp)
+  %a = alloca i32
+  store i32 0, i32* %a
+  %d = tail call i32* @bar(i32* %a) nounwind
+  store i32 %b, i32* %d
+  tail call tailcc void @foo(i32 1, i32 2, i32 3, i32 4, i32 5, i32 6) nounwind
+  ret void
+}
diff --git a/llvm/test/CodeGen/X86/tailcc-disable-tail-calls.ll b/llvm/test/CodeGen/X86/tailcc-disable-tail-calls.ll
new file mode 100644
index 000000000..3199b8c34
--- /dev/null
+++ b/llvm/test/CodeGen/X86/tailcc-disable-tail-calls.ll
@@ -0,0 +1,40 @@
+; RUN: llc < %s -mtriple=x86_64-- | FileCheck %s --check-prefix=NO-OPTION
+; RUN: llc < %s -mtriple=x86_64-- -disable-tail-calls | FileCheck %s --check-prefix=DISABLE-TRUE
+; RUN: llc < %s -mtriple=x86_64-- -disable-tail-calls=false | FileCheck %s --check-prefix=DISABLE-FALSE
+
+; Check that command line option "-disable-tail-calls" overrides function
+; attribute "disable-tail-calls".
+
+; NO-OPTION-LABEL: {{\_?}}func_attr
+; NO-OPTION: callq {{\_?}}callee
+
+; DISABLE-FALSE-LABEL: {{\_?}}func_attr
+; DISABLE-FALSE: jmp {{\_?}}callee
+
+; DISABLE-TRUE-LABEL: {{\_?}}func_attr
+; DISABLE-TRUE: callq {{\_?}}callee
+
+define tailcc i32 @func_attr(i32 %a) #0 {
+entry:
+  %call = tail call tailcc i32 @callee(i32 %a)
+  ret i32 %call
+}
+
+; NO-OPTION-LABEL: {{\_?}}func_noattr
+; NO-OPTION: jmp {{\_?}}callee
+
+; DISABLE-FALSE-LABEL: {{\_?}}func_noattr
+; DISABLE-FALSE: jmp {{\_?}}callee
+
+; DISABLE-TRUE-LABEL: {{\_?}}func_noattr
+; DISABLE-TRUE: callq {{\_?}}callee
+
+define tailcc i32 @func_noattr(i32 %a) {
+entry:
+  %call = tail call tailcc i32 @callee(i32 %a)
+  ret i32 %call
+}
+
+declare tailcc i32 @callee(i32)
+
+attributes #0 = { "disable-tail-calls"="true" }
diff --git a/llvm/test/CodeGen/X86/tailcc-fastcc.ll b/llvm/test/CodeGen/X86/tailcc-fastcc.ll
new file mode 100644
index 000000000..03369855d
--- /dev/null
+++ b/llvm/test/CodeGen/X86/tailcc-fastcc.ll
@@ -0,0 +1,49 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -tailcallopt < %s -mtriple=x86_64-unknown-unknown | FileCheck %s -check-prefix=X64
+; RUN: llc -tailcallopt < %s -mtriple=i686-unknown-unknown   | FileCheck %s -check-prefix=X32
+
+; llc -tailcallopt should not enable tail calls from fastcc to tailcc or vice versa
+
+declare tailcc i32 @tailcallee1(i32 %a1, i32 %a2, i32 %a3, i32 %a4)
+
+define fastcc i32 @tailcaller1(i32 %in1, i32 %in2) nounwind {
+; X64-LABEL: tailcaller1:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl %edi, %edx
+; X64-NEXT:    movl %esi, %ecx
+; X64-NEXT:    callq tailcallee1
+; X64-NEXT:    retq $8
+;
+; X32-LABEL: tailcaller1:
+; X32:       # %bb.0: # %entry
+; X32-NEXT:    pushl %edx
+; X32-NEXT:    pushl %ecx
+; X32-NEXT:    calll tailcallee1
+; X32-NEXT:    retl
+entry:
+  %tmp11 = tail call tailcc i32 @tailcallee1(i32 %in1, i32 %in2, i32 %in1, i32 %in2)
+  ret i32 %tmp11
+}
+
+declare fastcc i32 @tailcallee2(i32 %a1, i32 %a2, i32 %a3, i32 %a4)
+
+define tailcc i32 @tailcaller2(i32 %in1, i32 %in2) nounwind {
+; X64-LABEL: tailcaller2:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl %edi, %edx
+; X64-NEXT:    movl %esi, %ecx
+; X64-NEXT:    callq tailcallee2
+; X64-NEXT:    retq $8
+;
+; X32-LABEL: tailcaller2:
+; X32:       # %bb.0: # %entry
+; X32-NEXT:    pushl %edx
+; X32-NEXT:    pushl %ecx
+; X32-NEXT:    calll tailcallee2
+; X32-NEXT:    retl
+entry:
+  %tmp11 = tail call fastcc i32 @tailcallee2(i32 %in1, i32 %in2, i32 %in1, i32 %in2)
+  ret i32 %tmp11
+}
diff --git a/llvm/test/CodeGen/X86/tailcc-fastisel.ll b/llvm/test/CodeGen/X86/tailcc-fastisel.ll
new file mode 100644
index 000000000..e6d75faf4
--- /dev/null
+++ b/llvm/test/CodeGen/X86/tailcc-fastisel.ll
@@ -0,0 +1,18 @@
+; RUN: llc < %s -mtriple=x86_64-apple-darwin -fast-isel -fast-isel-abort=1 | FileCheck %s
+
+%0 = type { i64, i32, i8* }
+
+define tailcc i8* @"visit_array_aux<`Reference>"(%0 %arg, i32 %arg1) nounwind {
+fail:                                             ; preds = %entry
+  %tmp20 = tail call tailcc i8* @"visit_array_aux<`Reference>"(%0 %arg, i32 undef) ; <i8*> [#uses=1]
+; CHECK: jmp "_visit_array_aux<`Reference>" ## TAILCALL
+  ret i8* %tmp20
+}
+
+define i32 @foo() nounwind {
+entry:
+ %0 = tail call i32 (...) @bar() nounwind       ; <i32> [#uses=1]
+ ret i32 %0
+}
+
+declare i32 @bar(...) nounwind
diff --git a/llvm/test/CodeGen/X86/tailcc-largecode.ll b/llvm/test/CodeGen/X86/tailcc-largecode.ll
new file mode 100644
index 000000000..a3b5c3007
--- /dev/null
+++ b/llvm/test/CodeGen/X86/tailcc-largecode.ll
@@ -0,0 +1,71 @@
+; RUN: llc < %s -mtriple=x86_64-linux-gnu -code-model=large -enable-misched=false | FileCheck %s
+
+declare tailcc i32 @callee(i32 %arg)
+define tailcc i32 @directcall(i32 %arg) {
+entry:
+; This is the large code model, so &callee may not fit into the jmp
+; instruction.  Instead, stick it into a register.
+;  CHECK: movabsq $callee, [[REGISTER:%r[a-z0-9]+]]
+;  CHECK: jmpq    *[[REGISTER]]  # TAILCALL
+  %res = tail call tailcc i32 @callee(i32 %arg)
+  ret i32 %res
+}
+
+; Check that the register used for an indirect tail call doesn't
+; clobber any of the arguments.
+define tailcc i32 @indirect_manyargs(i32(i32,i32,i32,i32,i32,i32,i32)* %target) {
+; Adjust the stack to enter the function.  (The amount of the
+; adjustment may change in the future, in which case the location of
+; the stack argument and the return adjustment will change too.)
+;  CHECK: pushq
+; Put the call target into R11, which won't be clobbered while restoring
+; callee-saved registers and won't be used for passing arguments.
+;  CHECK: movq %rdi, %rax
+; Pass the stack argument.
+;  CHECK: movl $7, 16(%rsp)
+; Pass the register arguments, in the right registers.
+;  CHECK: movl $1, %edi
+;  CHECK: movl $2, %esi
+;  CHECK: movl $3, %edx
+;  CHECK: movl $4, %ecx
+;  CHECK: movl $5, %r8d
+;  CHECK: movl $6, %r9d
+; Adjust the stack to "return".
+;  CHECK: popq
+; And tail-call to the target.
+;  CHECK: jmpq *%rax  # TAILCALL
+  %res = tail call tailcc i32 %target(i32 1, i32 2, i32 3, i32 4, i32 5,
+                                      i32 6, i32 7)
+  ret i32 %res
+}
+
+; Check that the register used for a direct tail call doesn't clobber
+; any of the arguments.
+declare tailcc i32 @manyargs_callee(i32,i32,i32,i32,i32,i32,i32)
+define tailcc i32 @direct_manyargs() {
+; Adjust the stack to enter the function.  (The amount of the
+; adjustment may change in the future, in which case the location of
+; the stack argument and the return adjustment will change too.)
+;  CHECK: pushq
+; Pass the stack argument.
+;  CHECK: movl $7, 16(%rsp)
+; This is the large code model, so &manyargs_callee may not fit into
+; the jmp instruction.  Put it into a register which won't be clobbered
+; while restoring callee-saved registers and won't be used for passing
+; arguments.
+;  CHECK: movabsq $manyargs_callee, %rax
+; Pass the register arguments, in the right registers.
+;  CHECK: movl $1, %edi
+;  CHECK: movl $2, %esi
+;  CHECK: movl $3, %edx
+;  CHECK: movl $4, %ecx
+;  CHECK: movl $5, %r8d
+;  CHECK: movl $6, %r9d
+; Adjust the stack to "return".
+;  CHECK: popq
+; And tail-call to the target.
+;  CHECK: jmpq *%rax  # TAILCALL
+  %res = tail call tailcc i32 @manyargs_callee(i32 1, i32 2, i32 3, i32 4,
+                                               i32 5, i32 6, i32 7)
+  ret i32 %res
+}
diff --git a/llvm/test/CodeGen/X86/tailcc-stackalign.ll b/llvm/test/CodeGen/X86/tailcc-stackalign.ll
new file mode 100644
index 000000000..36333a9a2
--- /dev/null
+++ b/llvm/test/CodeGen/X86/tailcc-stackalign.ll
@@ -0,0 +1,23 @@
+; RUN: llc < %s  -mtriple=i686-unknown-linux -no-x86-call-frame-opt | FileCheck %s
+; Linux has 8 byte alignment so the params cause stack size 20, 
+; ensure that a normal tailcc call has matching stack size
+
+
+define tailcc i32 @tailcallee(i32 %a1, i32 %a2, i32 %a3, i32 %a4) {
+       ret i32 %a3
+}
+
+define tailcc i32 @tailcaller(i32 %in1, i32 %in2, i32 %in3, i32 %in4) {
+       %tmp11 = tail call tailcc i32 @tailcallee(i32 %in1, i32 %in2,
+                                                 i32 %in1, i32 %in2)
+       ret i32 %tmp11
+}
+
+define i32 @main(i32 %argc, i8** %argv) {
+ %tmp1 = call tailcc i32 @tailcaller( i32 1, i32 2, i32 3, i32 4 )
+ ; expect match subl [stacksize] here
+ ret i32 0
+}
+
+; CHECK: calll tailcaller
+; CHECK-NEXT: subl $12
diff --git a/llvm/test/CodeGen/X86/tailcc-structret.ll b/llvm/test/CodeGen/X86/tailcc-structret.ll
new file mode 100644
index 000000000..2d83d4a3c
--- /dev/null
+++ b/llvm/test/CodeGen/X86/tailcc-structret.ll
@@ -0,0 +1,7 @@
+; RUN: llc < %s -mtriple=i686-unknown-linux | FileCheck %s
+define tailcc { { i8*, i8* }*, i8*} @init({ { i8*, i8* }*, i8*}, i32) {
+entry:
+      %2 = tail call tailcc { { i8*, i8* }*, i8* } @init({ { i8*, i8*}*, i8*} %0, i32 %1)
+      ret { { i8*, i8* }*, i8*} %2
+; CHECK: jmp init
+}
diff --git a/llvm/test/CodeGen/X86/tailccbyval.ll b/llvm/test/CodeGen/X86/tailccbyval.ll
new file mode 100644
index 000000000..dbde868e5
--- /dev/null
+++ b/llvm/test/CodeGen/X86/tailccbyval.ll
@@ -0,0 +1,21 @@
+; RUN: llc < %s -mtriple=i686-unknown-linux | FileCheck %s
+%struct.s = type {i32, i32, i32, i32, i32, i32, i32, i32,
+                  i32, i32, i32, i32, i32, i32, i32, i32,
+                  i32, i32, i32, i32, i32, i32, i32, i32 }
+
+define  tailcc i32 @tailcallee(%struct.s* byval %a) nounwind {
+entry:
+        %tmp2 = getelementptr %struct.s, %struct.s* %a, i32 0, i32 0
+        %tmp3 = load i32, i32* %tmp2
+        ret i32 %tmp3
+; CHECK: tailcallee
+; CHECK: movl 4(%esp), %eax
+}
+
+define  tailcc i32 @tailcaller(%struct.s* byval %a) nounwind {
+entry:
+        %tmp4 = tail call tailcc i32 @tailcallee(%struct.s* byval %a )
+        ret i32 %tmp4
+; CHECK: tailcaller
+; CHECK: jmp tailcallee
+}
diff --git a/llvm/test/CodeGen/X86/tailccbyval64.ll b/llvm/test/CodeGen/X86/tailccbyval64.ll
new file mode 100644
index 000000000..47d20ea97
--- /dev/null
+++ b/llvm/test/CodeGen/X86/tailccbyval64.ll
@@ -0,0 +1,42 @@
+; RUN: llc < %s -mcpu=generic -mtriple=x86_64-linux | FileCheck %s
+
+; FIXME: Win64 does not support byval.
+
+; Expect the entry point.
+; CHECK-LABEL: tailcaller:
+
+; Expect 2 rep;movs because of tail call byval lowering.
+; CHECK: rep;
+; CHECK: rep;
+
+; A sequence of copyto/copyfrom virtual registers is used to deal with byval
+; lowering appearing after moving arguments to registers. The following two
+; checks verify that the register allocator changes those sequences to direct
+; moves to argument register where it can (for registers that are not used in 
+; byval lowering - not rsi, not rdi, not rcx).
+; Expect argument 4 to be moved directly to register edx.
+; CHECK: movl $7, %edx
+
+; Expect argument 6 to be moved directly to register r8.
+; CHECK: movl $17, %r8d
+
+; Expect not call but jmp to @tailcallee.
+; CHECK: jmp tailcallee
+
+; Expect the trailer.
+; CHECK: .size tailcaller
+
+%struct.s = type { i64, i64, i64, i64, i64, i64, i64, i64,
+                   i64, i64, i64, i64, i64, i64, i64, i64,
+                   i64, i64, i64, i64, i64, i64, i64, i64 }
+
+declare  tailcc i64 @tailcallee(%struct.s* byval %a, i64 %val, i64 %val2, i64 %val3, i64 %val4, i64 %val5)
+
+
+define  tailcc i64 @tailcaller(i64 %b, %struct.s* byval %a) {
+entry:
+        %tmp2 = getelementptr %struct.s, %struct.s* %a, i32 0, i32 1
+        %tmp3 = load i64, i64* %tmp2, align 8
+        %tmp4 = tail call tailcc i64 @tailcallee(%struct.s* byval %a , i64 %tmp3, i64 %b, i64 7, i64 13, i64 17)
+        ret i64 %tmp4
+}
diff --git a/llvm/test/CodeGen/X86/tailccfp.ll b/llvm/test/CodeGen/X86/tailccfp.ll
new file mode 100644
index 000000000..32814e93f
--- /dev/null
+++ b/llvm/test/CodeGen/X86/tailccfp.ll
@@ -0,0 +1,6 @@
+; RUN: llc < %s -mtriple=i686-- | FileCheck %s
+define tailcc i32 @bar(i32 %X, i32(double, i32) *%FP) {
+     %Y = tail call tailcc i32 %FP(double 0.0, i32 %X)
+     ret i32 %Y
+; CHECK: jmpl
+}
diff --git a/llvm/test/CodeGen/X86/tailccfp2.ll b/llvm/test/CodeGen/X86/tailccfp2.ll
new file mode 100644
index 000000000..f8b29b386
--- /dev/null
+++ b/llvm/test/CodeGen/X86/tailccfp2.ll
@@ -0,0 +1,27 @@
+; RUN: llc < %s -mtriple=i686-- | FileCheck %s
+
+declare i32 @putchar(i32)
+
+define tailcc i32 @checktail(i32 %x, i32* %f, i32 %g) nounwind {
+; CHECK-LABEL: checktail:
+        %tmp1 = icmp sgt i32 %x, 0
+        br i1 %tmp1, label %if-then, label %if-else
+
+if-then:
+        %fun_ptr = bitcast i32* %f to i32(i32, i32*, i32)* 
+        %arg1    = add i32 %x, -1
+        call i32 @putchar(i32 90)       
+; CHECK: jmpl *%e{{.*}}
+        %res = tail call tailcc i32 %fun_ptr( i32 %arg1, i32 * %f, i32 %g)
+        ret i32 %res
+
+if-else:
+        ret i32  %x
+}
+
+
+define i32 @main() nounwind { 
+ %f   = bitcast i32 (i32, i32*, i32)* @checktail to i32*
+ %res = tail call tailcc i32 @checktail( i32 10, i32* %f,i32 10)
+ ret i32 %res
+}
diff --git a/llvm/test/CodeGen/X86/tailccpic1.ll b/llvm/test/CodeGen/X86/tailccpic1.ll
new file mode 100644
index 000000000..de8f2219b
--- /dev/null
+++ b/llvm/test/CodeGen/X86/tailccpic1.ll
@@ -0,0 +1,16 @@
+; RUN: llc < %s  -mtriple=i686-pc-linux-gnu -relocation-model=pic | FileCheck %s
+
+; This test uses guaranteed TCO so these will be tail calls, despite the early
+; binding issues.
+
+define protected tailcc i32 @tailcallee(i32 %a1, i32 %a2, i32 %a3, i32 %a4) {
+entry:
+	ret i32 %a3
+}
+
+define tailcc i32 @tailcaller(i32 %in1, i32 %in2) {
+entry:
+	%tmp11 = tail call tailcc i32 @tailcallee( i32 %in1, i32 %in2, i32 %in1, i32 %in2 )		; <i32> [#uses=1]
+	ret i32 %tmp11
+; CHECK: jmp tailcallee
+}
diff --git a/llvm/test/CodeGen/X86/tailccpic2.ll b/llvm/test/CodeGen/X86/tailccpic2.ll
new file mode 100644
index 000000000..314cd8f2f
--- /dev/null
+++ b/llvm/test/CodeGen/X86/tailccpic2.ll
@@ -0,0 +1,15 @@
+; RUN: llc < %s  -mtriple=i686-pc-linux-gnu -relocation-model=pic | FileCheck %s
+
+define tailcc i32 @tailcallee(i32 %a1, i32 %a2, i32 %a3, i32 %a4) {
+entry:
+	ret i32 %a3
+}
+
+define tailcc i32 @tailcaller(i32 %in1, i32 %in2) {
+entry:
+	%tmp11 = tail call tailcc i32 @tailcallee( i32 %in1, i32 %in2, i32 %in1, i32 %in2 )		; <i32> [#uses=1]
+	ret i32 %tmp11
+; CHECK: movl tailcallee@GOT
+; CHECK: jmpl
+}
+
diff --git a/llvm/test/CodeGen/X86/tailccstack64.ll b/llvm/test/CodeGen/X86/tailccstack64.ll
new file mode 100644
index 000000000..bd0f4a739
--- /dev/null
+++ b/llvm/test/CodeGen/X86/tailccstack64.ll
@@ -0,0 +1,28 @@
+; RUN: llc < %s -mcpu=generic -mtriple=x86_64-linux -post-RA-scheduler=true | FileCheck %s
+; RUN: llc < %s -mcpu=generic -mtriple=x86_64-win32 -post-RA-scheduler=true | FileCheck %s
+
+; FIXME: Redundant unused stack allocation could be eliminated.
+; CHECK: subq  ${{24|72|80}}, %rsp
+
+; Check that lowered arguments on the stack do not overwrite each other.
+; Add %in1 %p1 to a different temporary register (%eax).
+; CHECK: movl  [[A1:32|144]](%rsp), [[R1:%e..]]
+; Move param %in1 to temp register (%r10d).
+; CHECK: movl  [[A2:40|152]](%rsp), [[R2:%[a-z0-9]+]]
+; Add %in1 %p1 to a different temporary register (%eax).
+; CHECK: addl {{%edi|%ecx}}, [[R1]]
+; Move param %in2 to stack.
+; CHECK-DAG: movl  [[R2]], [[A1]](%rsp)
+; Move result of addition to stack.
+; CHECK-DAG: movl  [[R1]], [[A2]](%rsp)
+; Eventually, do a TAILCALL
+; CHECK: TAILCALL
+
+declare tailcc i32 @tailcallee(i32 %p1, i32 %p2, i32 %p3, i32 %p4, i32 %p5, i32 %p6, i32 %a, i32 %b) nounwind
+
+define tailcc i32 @tailcaller(i32 %p1, i32 %p2, i32 %p3, i32 %p4, i32 %p5, i32 %p6, i32 %in1, i32 %in2) nounwind {
+entry:
+        %tmp = add i32 %in1, %p1
+        %retval = tail call tailcc i32 @tailcallee(i32 %p1, i32 %p2, i32 %p3, i32 %p4, i32 %p5, i32 %p6, i32 %in2,i32 %tmp)
+        ret i32 %retval
+}
diff --git a/llvm/test/CodeGen/X86/taildup-heapallocsite.ll b/llvm/test/CodeGen/X86/taildup-heapallocsite.ll
new file mode 100644
index 000000000..281fc9efb
--- /dev/null
+++ b/llvm/test/CodeGen/X86/taildup-heapallocsite.ll
@@ -0,0 +1,90 @@
+; RUN: llc < %s -tail-dup-placement-threshold=4 | FileCheck %s
+
+; Based on test case from PR43695:
+; __declspec(allocator) void *alloc(unsigned int size);
+; void f2();
+; void f1(unsigned int *size_ptr) {
+;     void *hg = alloc(size_ptr ? *size_ptr : 1UL);
+;     f2();
+; }
+
+; In this case, block placement duplicates the heap allocation site.
+
+; ModuleID = 't.cpp'
+source_filename = "t.cpp"
+target datalayout = "e-m:w-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc19.22.27905"
+
+define dso_local void @taildupit(i32* readonly %size_ptr) !dbg !8 {
+entry:
+  call void @llvm.dbg.value(metadata i32* %size_ptr, metadata !14, metadata !DIExpression()), !dbg !17
+  %tobool = icmp eq i32* %size_ptr, null, !dbg !18
+  br i1 %tobool, label %cond.end, label %cond.true, !dbg !18
+
+cond.true:                                        ; preds = %entry
+  %0 = load i32, i32* %size_ptr, align 4, !dbg !18, !tbaa !19
+  br label %cond.end, !dbg !18
+
+cond.end:                                         ; preds = %entry, %cond.true
+  %cond = phi i32 [ %0, %cond.true ], [ 1, %entry ], !dbg !18
+  %call = tail call i8* @alloc(i32 %cond), !dbg !18, !heapallocsite !2
+  call void @llvm.dbg.value(metadata i8* %call, metadata !15, metadata !DIExpression()), !dbg !17
+  tail call void @f2(), !dbg !23
+  ret void, !dbg !24
+}
+
+; CHECK-LABEL: taildupit: # @taildupit
+; CHECK: testq
+; CHECK: je
+; CHECK: callq alloc
+; CHECK-NEXT: [[L1:.Ltmp[0-9]+]]
+; CHECK: jmp f2 # TAILCALL
+; CHECK: callq alloc
+; CHECK-NEXT: [[L3:.Ltmp[0-9]+]]
+; CHECK: jmp f2 # TAILCALL
+
+; CHECK-LABEL: .short 4423                    # Record kind: S_GPROC32_ID
+; CHECK:       .short 4446                    # Record kind: S_HEAPALLOCSITE
+; CHECK-NEXT:  .secrel32 [[L0:.Ltmp[0-9]+]]
+; CHECK-NEXT:  .secidx [[L0]]
+; CHECK-NEXT:  .short [[L1]]-[[L0]]
+; CHECK-NEXT:  .long 3
+; CHECK:       .short 4446                    # Record kind: S_HEAPALLOCSITE
+; CHECK-NEXT:  .secrel32 [[L2:.Ltmp[0-9]+]]
+; CHECK-NEXT:  .secidx [[L2]]
+; CHECK-NEXT:  .short [[L3]]-[[L2]]
+; CHECK-NEXT:  .long 3
+
+declare dso_local i8* @alloc(i32)
+
+declare dso_local void @f2()
+
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5, !6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 10.0.0 (git@github.com:llvm/llvm-project.git 0650355c09ab8e6605ae37b818270a7a7c8ce2c7)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "t.cpp", directory: "C:\\src\\llvm-project\\build", checksumkind: CSK_MD5, checksum: "b227901e92d848fa564190b0762d757c")
+!2 = !{}
+!3 = !{i32 2, !"CodeView", i32 1}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 2}
+!6 = !{i32 7, !"PIC Level", i32 2}
+!8 = distinct !DISubprogram(name: "f1", linkageName: "?f1@@YAXPEAI@Z", scope: !1, file: !1, line: 5, type: !9, scopeLine: 5, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !13)
+!9 = !DISubroutineType(types: !10)
+!10 = !{null, !11}
+!11 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !12, size: 64)
+!12 = !DIBasicType(name: "unsigned int", size: 32, encoding: DW_ATE_unsigned)
+!13 = !{!14, !15}
+!14 = !DILocalVariable(name: "size_ptr", arg: 1, scope: !8, file: !1, line: 5, type: !11)
+!15 = !DILocalVariable(name: "hg", scope: !8, file: !1, line: 6, type: !16)
+!16 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: null, size: 64)
+!17 = !DILocation(line: 0, scope: !8)
+!18 = !DILocation(line: 6, scope: !8)
+!19 = !{!20, !20, i64 0}
+!20 = !{!"int", !21, i64 0}
+!21 = !{!"omnipotent char", !22, i64 0}
+!22 = !{!"Simple C++ TBAA"}
+!23 = !DILocation(line: 7, scope: !8)
+!24 = !DILocation(line: 8, scope: !8)
diff --git a/llvm/test/CodeGen/X86/test-shrink-bug.ll b/llvm/test/CodeGen/X86/test-shrink-bug.ll
index e7b51f120..c5cf32732 100644
--- a/llvm/test/CodeGen/X86/test-shrink-bug.ll
+++ b/llvm/test/CodeGen/X86/test-shrink-bug.ll
@@ -69,11 +69,9 @@ define void @fail(i16 %a, <2 x i8> %b) {
 ; CHECK-X64-NEXT:    testl $263, %edi # imm = 0x107
 ; CHECK-X64-NEXT:    je .LBB1_3
 ; CHECK-X64-NEXT:  # %bb.1:
-; CHECK-X64-NEXT:    pand {{.*}}(%rip), %xmm0
-; CHECK-X64-NEXT:    pcmpeqd {{.*}}(%rip), %xmm0
-; CHECK-X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,0,3,2]
-; CHECK-X64-NEXT:    pand %xmm0, %xmm1
-; CHECK-X64-NEXT:    pextrw $4, %xmm1, %eax
+; CHECK-X64-NEXT:    pcmpeqb {{.*}}(%rip), %xmm0
+; CHECK-X64-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; CHECK-X64-NEXT:    pextrw $1, %xmm0, %eax
 ; CHECK-X64-NEXT:    testb $1, %al
 ; CHECK-X64-NEXT:    jne .LBB1_3
 ; CHECK-X64-NEXT:  # %bb.2: # %no
diff --git a/llvm/test/CodeGen/X86/test-vs-bittest.ll b/llvm/test/CodeGen/X86/test-vs-bittest.ll
index fa43ecb35..38ff7a40a 100644
--- a/llvm/test/CodeGen/X86/test-vs-bittest.ll
+++ b/llvm/test/CodeGen/X86/test-vs-bittest.ll
@@ -49,6 +49,30 @@ no:
   ret void
 }
 
+define void @test64_pgso(i64 inreg %x) !prof !14 {
+; CHECK-LABEL: test64_pgso:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    btl $11, %edi
+; CHECK-NEXT:    jb .LBB2_2
+; CHECK-NEXT:  # %bb.1: # %yes
+; CHECK-NEXT:    callq bar
+; CHECK-NEXT:  .LBB2_2: # %no
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
+  %t = and i64 %x, 2048
+  %s = icmp eq i64 %t, 0
+  br i1 %s, label %yes, label %no
+
+yes:
+  call void @bar()
+  ret void
+no:
+  ret void
+}
+
 ; This test is identical to test64 above with only the destination of the br
 ; reversed. This somehow causes the two functions to get slightly different
 ; initial IR. One has an extra invert of the setcc. This previous caused one
@@ -60,10 +84,10 @@ define void @test64_2(i64 inreg %x) {
 ; CHECK-NEXT:    pushq %rax
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    testl $2048, %edi # imm = 0x800
-; CHECK-NEXT:    je .LBB2_2
+; CHECK-NEXT:    je .LBB3_2
 ; CHECK-NEXT:  # %bb.1: # %yes
 ; CHECK-NEXT:    callq bar
-; CHECK-NEXT:  .LBB2_2: # %no
+; CHECK-NEXT:  .LBB3_2: # %no
 ; CHECK-NEXT:    popq %rax
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK-NEXT:    retq
@@ -84,10 +108,34 @@ define void @test64_optsize_2(i64 inreg %x) optsize {
 ; CHECK-NEXT:    pushq %rax
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    btl $11, %edi
-; CHECK-NEXT:    jae .LBB3_2
+; CHECK-NEXT:    jae .LBB4_2
 ; CHECK-NEXT:  # %bb.1: # %yes
 ; CHECK-NEXT:    callq bar
-; CHECK-NEXT:  .LBB3_2: # %no
+; CHECK-NEXT:  .LBB4_2: # %no
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
+  %t = and i64 %x, 2048
+  %s = icmp eq i64 %t, 0
+  br i1 %s, label %no, label %yes
+
+yes:
+  call void @bar()
+  ret void
+no:
+  ret void
+}
+
+define void @test64_pgso_2(i64 inreg %x) !prof !14 {
+; CHECK-LABEL: test64_pgso_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    btl $11, %edi
+; CHECK-NEXT:    jae .LBB5_2
+; CHECK-NEXT:  # %bb.1: # %yes
+; CHECK-NEXT:    callq bar
+; CHECK-NEXT:  .LBB5_2: # %no
 ; CHECK-NEXT:    popq %rax
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK-NEXT:    retq
@@ -108,10 +156,10 @@ define void @test64_3(i64 inreg %x) {
 ; CHECK-NEXT:    pushq %rax
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    btq $32, %rdi
-; CHECK-NEXT:    jb .LBB4_2
+; CHECK-NEXT:    jb .LBB6_2
 ; CHECK-NEXT:  # %bb.1: # %yes
 ; CHECK-NEXT:    callq bar
-; CHECK-NEXT:  .LBB4_2: # %no
+; CHECK-NEXT:  .LBB6_2: # %no
 ; CHECK-NEXT:    popq %rax
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK-NEXT:    retq
@@ -132,10 +180,34 @@ define void @test64_optsize_3(i64 inreg %x) optsize {
 ; CHECK-NEXT:    pushq %rax
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    btq $32, %rdi
-; CHECK-NEXT:    jb .LBB5_2
+; CHECK-NEXT:    jb .LBB7_2
 ; CHECK-NEXT:  # %bb.1: # %yes
 ; CHECK-NEXT:    callq bar
-; CHECK-NEXT:  .LBB5_2: # %no
+; CHECK-NEXT:  .LBB7_2: # %no
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
+  %t = and i64 %x, 4294967296
+  %s = icmp eq i64 %t, 0
+  br i1 %s, label %yes, label %no
+
+yes:
+  call void @bar()
+  ret void
+no:
+  ret void
+}
+
+define void @test64_pgso_3(i64 inreg %x) !prof !14 {
+; CHECK-LABEL: test64_pgso_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    btq $32, %rdi
+; CHECK-NEXT:    jb .LBB8_2
+; CHECK-NEXT:  # %bb.1: # %yes
+; CHECK-NEXT:    callq bar
+; CHECK-NEXT:  .LBB8_2: # %no
 ; CHECK-NEXT:    popq %rax
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK-NEXT:    retq
@@ -156,10 +228,10 @@ define void @test64_4(i64 inreg %x) {
 ; CHECK-NEXT:    pushq %rax
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    btq $32, %rdi
-; CHECK-NEXT:    jae .LBB6_2
+; CHECK-NEXT:    jae .LBB9_2
 ; CHECK-NEXT:  # %bb.1: # %yes
 ; CHECK-NEXT:    callq bar
-; CHECK-NEXT:  .LBB6_2: # %no
+; CHECK-NEXT:  .LBB9_2: # %no
 ; CHECK-NEXT:    popq %rax
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK-NEXT:    retq
@@ -180,10 +252,34 @@ define void @test64_optsize_4(i64 inreg %x) optsize {
 ; CHECK-NEXT:    pushq %rax
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    btq $32, %rdi
-; CHECK-NEXT:    jae .LBB7_2
+; CHECK-NEXT:    jae .LBB10_2
 ; CHECK-NEXT:  # %bb.1: # %yes
 ; CHECK-NEXT:    callq bar
-; CHECK-NEXT:  .LBB7_2: # %no
+; CHECK-NEXT:  .LBB10_2: # %no
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
+  %t = and i64 %x, 4294967296
+  %s = icmp eq i64 %t, 0
+  br i1 %s, label %no, label %yes
+
+yes:
+  call void @bar()
+  ret void
+no:
+  ret void
+}
+
+define void @test64_pgso_4(i64 inreg %x) !prof !14 {
+; CHECK-LABEL: test64_pgso_4:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    btq $32, %rdi
+; CHECK-NEXT:    jae .LBB11_2
+; CHECK-NEXT:  # %bb.1: # %yes
+; CHECK-NEXT:    callq bar
+; CHECK-NEXT:  .LBB11_2: # %no
 ; CHECK-NEXT:    popq %rax
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK-NEXT:    retq
@@ -204,10 +300,10 @@ define void @test32(i32 inreg %x) {
 ; CHECK-NEXT:    pushq %rax
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    testl $2048, %edi # imm = 0x800
-; CHECK-NEXT:    jne .LBB8_2
+; CHECK-NEXT:    jne .LBB12_2
 ; CHECK-NEXT:  # %bb.1: # %yes
 ; CHECK-NEXT:    callq bar
-; CHECK-NEXT:  .LBB8_2: # %no
+; CHECK-NEXT:  .LBB12_2: # %no
 ; CHECK-NEXT:    popq %rax
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK-NEXT:    retq
@@ -228,10 +324,10 @@ define void @test32_optsize(i32 inreg %x) optsize {
 ; CHECK-NEXT:    pushq %rax
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    btl $11, %edi
-; CHECK-NEXT:    jb .LBB9_2
+; CHECK-NEXT:    jb .LBB13_2
 ; CHECK-NEXT:  # %bb.1: # %yes
 ; CHECK-NEXT:    callq bar
-; CHECK-NEXT:  .LBB9_2: # %no
+; CHECK-NEXT:  .LBB13_2: # %no
 ; CHECK-NEXT:    popq %rax
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK-NEXT:    retq
@@ -252,10 +348,10 @@ define void @test32_2(i32 inreg %x) {
 ; CHECK-NEXT:    pushq %rax
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    testl $2048, %edi # imm = 0x800
-; CHECK-NEXT:    je .LBB10_2
+; CHECK-NEXT:    je .LBB14_2
 ; CHECK-NEXT:  # %bb.1: # %yes
 ; CHECK-NEXT:    callq bar
-; CHECK-NEXT:  .LBB10_2: # %no
+; CHECK-NEXT:  .LBB14_2: # %no
 ; CHECK-NEXT:    popq %rax
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK-NEXT:    retq
@@ -276,10 +372,34 @@ define void @test32_optsize_2(i32 inreg %x) optsize {
 ; CHECK-NEXT:    pushq %rax
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    btl $11, %edi
-; CHECK-NEXT:    jae .LBB11_2
+; CHECK-NEXT:    jae .LBB15_2
 ; CHECK-NEXT:  # %bb.1: # %yes
 ; CHECK-NEXT:    callq bar
-; CHECK-NEXT:  .LBB11_2: # %no
+; CHECK-NEXT:  .LBB15_2: # %no
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
+  %t = and i32 %x, 2048
+  %s = icmp eq i32 %t, 0
+  br i1 %s, label %no, label %yes
+
+yes:
+  call void @bar()
+  ret void
+no:
+  ret void
+}
+
+define void @test32_pgso_2(i32 inreg %x) !prof !14 {
+; CHECK-LABEL: test32_pgso_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    btl $11, %edi
+; CHECK-NEXT:    jae .LBB16_2
+; CHECK-NEXT:  # %bb.1: # %yes
+; CHECK-NEXT:    callq bar
+; CHECK-NEXT:  .LBB16_2: # %no
 ; CHECK-NEXT:    popq %rax
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK-NEXT:    retq
@@ -300,10 +420,10 @@ define void @test16(i16 inreg %x) {
 ; CHECK-NEXT:    pushq %rax
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    testl $2048, %edi # imm = 0x800
-; CHECK-NEXT:    jne .LBB12_2
+; CHECK-NEXT:    jne .LBB17_2
 ; CHECK-NEXT:  # %bb.1: # %yes
 ; CHECK-NEXT:    callq bar
-; CHECK-NEXT:  .LBB12_2: # %no
+; CHECK-NEXT:  .LBB17_2: # %no
 ; CHECK-NEXT:    popq %rax
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK-NEXT:    retq
@@ -324,10 +444,34 @@ define void @test16_optsize(i16 inreg %x) optsize {
 ; CHECK-NEXT:    pushq %rax
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    btl $11, %edi
-; CHECK-NEXT:    jb .LBB13_2
+; CHECK-NEXT:    jb .LBB18_2
 ; CHECK-NEXT:  # %bb.1: # %yes
 ; CHECK-NEXT:    callq bar
-; CHECK-NEXT:  .LBB13_2: # %no
+; CHECK-NEXT:  .LBB18_2: # %no
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
+  %t = and i16 %x, 2048
+  %s = icmp eq i16 %t, 0
+  br i1 %s, label %yes, label %no
+
+yes:
+  call void @bar()
+  ret void
+no:
+  ret void
+}
+
+define void @test16_pgso(i16 inreg %x) !prof !14 {
+; CHECK-LABEL: test16_pgso:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    btl $11, %edi
+; CHECK-NEXT:    jb .LBB19_2
+; CHECK-NEXT:  # %bb.1: # %yes
+; CHECK-NEXT:    callq bar
+; CHECK-NEXT:  .LBB19_2: # %no
 ; CHECK-NEXT:    popq %rax
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK-NEXT:    retq
@@ -348,10 +492,10 @@ define void @test16_2(i16 inreg %x) {
 ; CHECK-NEXT:    pushq %rax
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    testl $2048, %edi # imm = 0x800
-; CHECK-NEXT:    je .LBB14_2
+; CHECK-NEXT:    je .LBB20_2
 ; CHECK-NEXT:  # %bb.1: # %yes
 ; CHECK-NEXT:    callq bar
-; CHECK-NEXT:  .LBB14_2: # %no
+; CHECK-NEXT:  .LBB20_2: # %no
 ; CHECK-NEXT:    popq %rax
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK-NEXT:    retq
@@ -372,10 +516,34 @@ define void @test16_optsize_2(i16 inreg %x) optsize {
 ; CHECK-NEXT:    pushq %rax
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    btl $11, %edi
-; CHECK-NEXT:    jae .LBB15_2
+; CHECK-NEXT:    jae .LBB21_2
 ; CHECK-NEXT:  # %bb.1: # %yes
 ; CHECK-NEXT:    callq bar
-; CHECK-NEXT:  .LBB15_2: # %no
+; CHECK-NEXT:  .LBB21_2: # %no
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
+  %t = and i16 %x, 2048
+  %s = icmp eq i16 %t, 0
+  br i1 %s, label %no, label %yes
+
+yes:
+  call void @bar()
+  ret void
+no:
+  ret void
+}
+
+define void @test16_pgso_2(i16 inreg %x) !prof !14 {
+; CHECK-LABEL: test16_pgso_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    btl $11, %edi
+; CHECK-NEXT:    jae .LBB22_2
+; CHECK-NEXT:  # %bb.1: # %yes
+; CHECK-NEXT:    callq bar
+; CHECK-NEXT:  .LBB22_2: # %no
 ; CHECK-NEXT:    popq %rax
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK-NEXT:    retq
@@ -390,4 +558,142 @@ no:
   ret void
 }
 
+define i64 @is_upper_bit_clear_i64(i64 %x) {
+; CHECK-LABEL: is_upper_bit_clear_i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    btq $37, %rdi
+; CHECK-NEXT:    setae %al
+; CHECK-NEXT:    retq
+  %sh = lshr i64 %x, 37
+  %m = and i64 %sh, 1
+  %r = xor i64 %m, 1
+  ret i64 %r
+}
+
+define i64 @is_lower_bit_clear_i64(i64 %x) {
+; CHECK-LABEL: is_lower_bit_clear_i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    testl $134217728, %edi # imm = 0x8000000
+; CHECK-NEXT:    sete %al
+; CHECK-NEXT:    retq
+  %sh = lshr i64 %x, 27
+  %m = and i64 %sh, 1
+  %r = xor i64 %m, 1
+  ret i64 %r
+}
+
+define i32 @is_bit_clear_i32(i32 %x) {
+; CHECK-LABEL: is_bit_clear_i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    testl $134217728, %edi # imm = 0x8000000
+; CHECK-NEXT:    sete %al
+; CHECK-NEXT:    retq
+  %sh = lshr i32 %x, 27
+  %n = xor i32 %sh, -1
+  %r = and i32 %n, 1
+  ret i32 %r
+}
+
+define i16 @is_bit_clear_i16(i16 %x) {
+; CHECK-LABEL: is_bit_clear_i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    testb $-128, %dil
+; CHECK-NEXT:    sete %al
+; CHECK-NEXT:    # kill: def $ax killed $ax killed $eax
+; CHECK-NEXT:    retq
+  %sh = lshr i16 %x, 7
+  %m = and i16 %sh, 1
+  %r = xor i16 %m, 1
+  ret i16 %r
+}
+
+define i8 @is_bit_clear_i8(i8 %x) {
+; CHECK-LABEL: is_bit_clear_i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    testb $8, %dil
+; CHECK-NEXT:    sete %al
+; CHECK-NEXT:    retq
+  %sh = lshr i8 %x, 3
+  %m = and i8 %sh, 1
+  %r = xor i8 %m, 1
+  ret i8 %r
+}
+
+; TODO: We could use bt/test on the 64-bit value.
+
+define i8 @overshift(i64 %x) {
+; CHECK-LABEL: overshift:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    shrq $42, %rax
+; CHECK-NEXT:    notb %al
+; CHECK-NEXT:    andb $1, %al
+; CHECK-NEXT:    # kill: def $al killed $al killed $rax
+; CHECK-NEXT:    retq
+  %a = lshr i64 %x, 42
+  %t = trunc i64 %a to i8
+  %n = xor i8 %t, -1
+  %r = and i8 %n, 1
+  ret i8 %r
+}
+
+define i32 @setcc_is_bit_clear(i32 %x) {
+; CHECK-LABEL: setcc_is_bit_clear:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    testl $1024, %edi # imm = 0x400
+; CHECK-NEXT:    sete %al
+; CHECK-NEXT:    retq
+  %a1 = and i32 %x, 1024
+  %b1 = icmp eq i32 %a1, 0
+  %r = zext i1 %b1 to i32
+  ret i32 %r
+}
+
+define i32 @is_bit_set(i32 %x) {
+; CHECK-LABEL: is_bit_set:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    shrl $10, %eax
+; CHECK-NEXT:    andl $1, %eax
+; CHECK-NEXT:    retq
+  %sh = lshr i32 %x, 10
+  %m = and i32 %sh, 1
+  ret i32 %m
+}
+
+define i32 @setcc_is_bit_set(i32 %x) {
+; CHECK-LABEL: setcc_is_bit_set:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    shrl $10, %eax
+; CHECK-NEXT:    andl $1, %eax
+; CHECK-NEXT:    retq
+  %a1 = and i32 %x, 1024
+  %b1 = icmp ne i32 %a1, 0
+  %r = zext i1 %b1 to i32
+  ret i32 %r
+}
+
 declare void @bar()
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 1, !"ProfileSummary", !1}
+!1 = !{!2, !3, !4, !5, !6, !7, !8, !9}
+!2 = !{!"ProfileFormat", !"InstrProf"}
+!3 = !{!"TotalCount", i64 10000}
+!4 = !{!"MaxCount", i64 10}
+!5 = !{!"MaxInternalCount", i64 1}
+!6 = !{!"MaxFunctionCount", i64 1000}
+!7 = !{!"NumCounts", i64 3}
+!8 = !{!"NumFunctions", i64 3}
+!9 = !{!"DetailedSummary", !10}
+!10 = !{!11, !12, !13}
+!11 = !{i32 10000, i64 100, i32 1}
+!12 = !{i32 999000, i64 100, i32 1}
+!13 = !{i32 999999, i64 1, i32 2}
+!14 = !{!"function_entry_count", i64 0}
diff --git a/llvm/test/CodeGen/X86/throws-cfi-fp.ll b/llvm/test/CodeGen/X86/throws-cfi-fp.ll
index 3ec97d754..16ebda9ac 100644
--- a/llvm/test/CodeGen/X86/throws-cfi-fp.ll
+++ b/llvm/test/CodeGen/X86/throws-cfi-fp.ll
@@ -82,5 +82,5 @@ declare void @_ZSt9terminatev()
 ; Function Attrs: nounwind
 declare i32 @puts(i8* nocapture readonly)
 
-attributes #0 = { "no-frame-pointer-elim"="true" }
+attributes #0 = { "frame-pointer"="all" }
 
diff --git a/llvm/test/CodeGen/X86/tls-addr-non-leaf-function.ll b/llvm/test/CodeGen/X86/tls-addr-non-leaf-function.ll
index 6ebced191..ad8d56173 100644
--- a/llvm/test/CodeGen/X86/tls-addr-non-leaf-function.ll
+++ b/llvm/test/CodeGen/X86/tls-addr-non-leaf-function.ll
@@ -4,7 +4,7 @@
 ; This test runs twice with different options regarding the frame pointer:
 ; first the elimination is disabled, then it is enabled. The disabled case is
 ; the "control group".
-; The function 'foo' below is marked with the "no-frame-pointer-elim-non-leaf"
+; The function 'foo' below is marked with the "frame-pointer"="non-leaf"
 ; attribute which dictates that the frame pointer should not be eliminated
 ; unless the function is a leaf (i.e. it doesn't call any other function).
 ; Now, 'foo' is not a leaf function, because it performs a TLS access which on
@@ -31,7 +31,7 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @x = thread_local global i32 0
-define i32 @foo() "no-frame-pointer-elim-non-leaf" {
+define i32 @foo() "frame-pointer"="non-leaf" {
   %a = load i32, i32* @x, align 4
   ret i32 %a
 }
diff --git a/llvm/test/CodeGen/X86/tls-shrink-wrapping.ll b/llvm/test/CodeGen/X86/tls-shrink-wrapping.ll
index 216bb95f7..f6431d1ea 100644
--- a/llvm/test/CodeGen/X86/tls-shrink-wrapping.ll
+++ b/llvm/test/CodeGen/X86/tls-shrink-wrapping.ll
@@ -51,6 +51,6 @@ if.end:                                           ; preds = %if.then, %entry
 
 declare void @f(...) #1
 
-attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind }
diff --git a/llvm/test/CodeGen/X86/trap.ll b/llvm/test/CodeGen/X86/trap.ll
index ca33f9e6b..452be48a6 100644
--- a/llvm/test/CodeGen/X86/trap.ll
+++ b/llvm/test/CodeGen/X86/trap.ll
@@ -1,13 +1,19 @@
 ; RUN: llc < %s -mtriple=i686-apple-darwin8 -mcpu=yonah | FileCheck %s -check-prefix=DARWIN
 ; RUN: llc < %s -mtriple=i686-unknown-linux -mcpu=yonah | FileCheck %s -check-prefix=LINUX
 ; RUN: llc < %s -mtriple=x86_64-scei-ps4 | FileCheck %s -check-prefix=PS4
+; RUN: llc < %s -mtriple=x86_64-windows-msvc | FileCheck %s -check-prefix=WIN64
 
 ; DARWIN-LABEL: test0:
 ; DARWIN: ud2
 ; LINUX-LABEL: test0:
 ; LINUX: ud2
+; FIXME: PS4 probably doesn't want two ud2s.
 ; PS4-LABEL: test0:
 ; PS4: ud2
+; PS4: ud2
+; WIN64-LABEL: test0:
+; WIN64: ud2
+; WIN64-NOT: ud2
 define i32 @test0() noreturn nounwind  {
 entry:
 	tail call void @llvm.trap( )
@@ -20,6 +26,9 @@ entry:
 ; LINUX: int3
 ; PS4-LABEL: test1:
 ; PS4: int     $65
+; WIN64-LABEL: test1:
+; WIN64: int3
+; WIN64-NOT: ud2
 define i32 @test1() noreturn nounwind  {
 entry:
 	tail call void @llvm.debugtrap( )
diff --git a/llvm/test/CodeGen/X86/trunc-and.ll b/llvm/test/CodeGen/X86/trunc-and.ll
index 28a55c257..09fe6413d 100644
--- a/llvm/test/CodeGen/X86/trunc-and.ll
+++ b/llvm/test/CodeGen/X86/trunc-and.ll
@@ -24,3 +24,27 @@ define i16 @PR40793(<8 x i16> %t1) {
 
 declare <2 x double> @llvm.fabs.v2f64(<2 x double>)
 
+; This would infinite loop by trying to truncate and any_extend.
+
+%struct.anon = type { [9 x i8], [3 x i8] }
+
+@b = common local_unnamed_addr global %struct.anon zeroinitializer, align 4
+
+define i32 @d() {
+; CHECK-LABEL: d:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movzbl b+{{.*}}(%rip), %ecx
+; CHECK-NEXT:    andl $7, %ecx
+; CHECK-NEXT:    movl $d, %eax
+; CHECK-NEXT:    addl %ecx, %eax
+; CHECK-NEXT:    # kill: def $eax killed $eax killed $rax
+; CHECK-NEXT:    retq
+  %bf.load = load i72, i72* bitcast (%struct.anon* @b to i72*), align 4
+  %bf.lshr = lshr i72 %bf.load, 64
+  %t0 = trunc i72 %bf.lshr to i64
+  %bf.cast = and i64 %t0, 7
+  %add.ptr = getelementptr i8, i8* bitcast (i32 ()* @d to i8*), i64 %bf.cast
+  %t1 = ptrtoint i8* %add.ptr to i64
+  %t2 = trunc i64 %t1 to i32
+  ret i32 %t2
+}
diff --git a/llvm/test/CodeGen/X86/trunc-ext-ld-st.ll b/llvm/test/CodeGen/X86/trunc-ext-ld-st.ll
index c799441c1..f807444a3 100644
--- a/llvm/test/CodeGen/X86/trunc-ext-ld-st.ll
+++ b/llvm/test/CodeGen/X86/trunc-ext-ld-st.ll
@@ -8,23 +8,16 @@ define void @load_2_i8(<2 x i8>* %A)  {
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movzwl (%rdi), %eax
 ; SSE2-NEXT:    movd %eax, %xmm0
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,1,3]
-; SSE2-NEXT:    paddq {{.*}}(%rip), %xmm0
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
-; SSE2-NEXT:    packuswb %xmm0, %xmm0
-; SSE2-NEXT:    packuswb %xmm0, %xmm0
-; SSE2-NEXT:    packuswb %xmm0, %xmm0
+; SSE2-NEXT:    paddb {{.*}}(%rip), %xmm0
 ; SSE2-NEXT:    movd %xmm0, %eax
 ; SSE2-NEXT:    movw %ax, (%rdi)
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: load_2_i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pmovzxbq {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero
-; SSE41-NEXT:    paddq {{.*}}(%rip), %xmm0
-; SSE41-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; SSE41-NEXT:    movzwl (%rdi), %eax
+; SSE41-NEXT:    movd %eax, %xmm0
+; SSE41-NEXT:    paddb {{.*}}(%rip), %xmm0
 ; SSE41-NEXT:    pextrw $0, %xmm0, (%rdi)
 ; SSE41-NEXT:    retq
    %T = load <2 x i8>, <2 x i8>* %A
@@ -35,25 +28,12 @@ define void @load_2_i8(<2 x i8>* %A)  {
 
 ; Read 32-bits
 define void @load_2_i16(<2 x i16>* %A)  {
-; SSE2-LABEL: load_2_i16:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,0,3]
-; SSE2-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,5,6,7]
-; SSE2-NEXT:    paddq {{.*}}(%rip), %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
-; SSE2-NEXT:    movd %xmm0, (%rdi)
-; SSE2-NEXT:    retq
-;
-; SSE41-LABEL: load_2_i16:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero
-; SSE41-NEXT:    paddq {{.*}}(%rip), %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; SSE41-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
-; SSE41-NEXT:    movd %xmm0, (%rdi)
-; SSE41-NEXT:    retq
+; CHECK-LABEL: load_2_i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; CHECK-NEXT:    paddw {{.*}}(%rip), %xmm0
+; CHECK-NEXT:    movd %xmm0, (%rdi)
+; CHECK-NEXT:    retq
    %T = load <2 x i16>, <2 x i16>* %A
    %G = add <2 x i16> %T, <i16 9, i16 7>
    store <2 x i16> %G, <2 x i16>* %A
@@ -61,22 +41,12 @@ define void @load_2_i16(<2 x i16>* %A)  {
 }
 
 define void @load_2_i32(<2 x i32>* %A)  {
-; SSE2-LABEL: load_2_i32:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,1,3]
-; SSE2-NEXT:    paddd {{.*}}(%rip), %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; SSE2-NEXT:    movq %xmm0, (%rdi)
-; SSE2-NEXT:    retq
-;
-; SSE41-LABEL: load_2_i32:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm0 = mem[0],zero,mem[1],zero
-; SSE41-NEXT:    paddd {{.*}}(%rip), %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; SSE41-NEXT:    movq %xmm0, (%rdi)
-; SSE41-NEXT:    retq
+; CHECK-LABEL: load_2_i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    paddd {{.*}}(%rip), %xmm0
+; CHECK-NEXT:    movq %xmm0, (%rdi)
+; CHECK-NEXT:    retq
    %T = load <2 x i32>, <2 x i32>* %A
    %G = add <2 x i32> %T, <i32 9, i32 7>
    store <2 x i32> %G, <2 x i32>* %A
@@ -84,25 +54,12 @@ define void @load_2_i32(<2 x i32>* %A)  {
 }
 
 define void @load_4_i8(<4 x i8>* %A)  {
-; SSE2-LABEL: load_4_i8:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
-; SSE2-NEXT:    paddd {{.*}}(%rip), %xmm0
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
-; SSE2-NEXT:    packuswb %xmm0, %xmm0
-; SSE2-NEXT:    packuswb %xmm0, %xmm0
-; SSE2-NEXT:    movd %xmm0, (%rdi)
-; SSE2-NEXT:    retq
-;
-; SSE41-LABEL: load_4_i8:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
-; SSE41-NEXT:    paddd {{.*}}(%rip), %xmm0
-; SSE41-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
-; SSE41-NEXT:    movd %xmm0, (%rdi)
-; SSE41-NEXT:    retq
+; CHECK-LABEL: load_4_i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; CHECK-NEXT:    paddb {{.*}}(%rip), %xmm0
+; CHECK-NEXT:    movd %xmm0, (%rdi)
+; CHECK-NEXT:    retq
    %T = load <4 x i8>, <4 x i8>* %A
    %G = add <4 x i8> %T, <i8 1, i8 4, i8 9, i8 7>
    store <4 x i8> %G, <4 x i8>* %A
@@ -110,24 +67,12 @@ define void @load_4_i8(<4 x i8>* %A)  {
 }
 
 define void @load_4_i16(<4 x i16>* %A)  {
-; SSE2-LABEL: load_4_i16:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
-; SSE2-NEXT:    paddw {{.*}}(%rip), %xmm0
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
-; SSE2-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,6,6,7]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; SSE2-NEXT:    movq %xmm0, (%rdi)
-; SSE2-NEXT:    retq
-;
-; SSE41-LABEL: load_4_i16:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
-; SSE41-NEXT:    paddw {{.*}}(%rip), %xmm0
-; SSE41-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
-; SSE41-NEXT:    movq %xmm0, (%rdi)
-; SSE41-NEXT:    retq
+; CHECK-LABEL: load_4_i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    paddw {{.*}}(%rip), %xmm0
+; CHECK-NEXT:    movq %xmm0, (%rdi)
+; CHECK-NEXT:    retq
    %T = load <4 x i16>, <4 x i16>* %A
    %G = add <4 x i16> %T, <i16 1, i16 4, i16 9, i16 7>
    store <4 x i16> %G, <4 x i16>* %A
diff --git a/llvm/test/CodeGen/X86/trunc-subvector.ll b/llvm/test/CodeGen/X86/trunc-subvector.ll
index d52fe0f8f..05b2b6608 100644
--- a/llvm/test/CodeGen/X86/trunc-subvector.ll
+++ b/llvm/test/CodeGen/X86/trunc-subvector.ll
@@ -40,25 +40,14 @@ define <4 x i32> @test2(<8 x i32> %v) {
 define <2 x i32> @test3(<8 x i32> %v) {
 ; SSE2-LABEL: test3:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa %xmm1, %xmm0
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT:    movaps %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
-; AVX2-LABEL: test3:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; AVX2-NEXT:    vpmovsxdq %xmm0, %xmm0
-; AVX2-NEXT:    vzeroupper
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: test3:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpmovsxdq %ymm0, %zmm0
-; AVX512-NEXT:    vextracti32x4 $2, %zmm0, %xmm0
-; AVX512-NEXT:    vzeroupper
-; AVX512-NEXT:    retq
+; AVX-LABEL: test3:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX-NEXT:    vzeroupper
+; AVX-NEXT:    retq
   %x = sext <8 x i32> %v to <8 x i64>
   %s = shufflevector <8 x i64> %x, <8 x i64> undef, <2 x i32> <i32 4, i32 5>
   %t = trunc <2 x i64> %s to <2 x i32>
@@ -68,23 +57,13 @@ define <2 x i32> @test3(<8 x i32> %v) {
 define <2 x i32> @test4(<8 x i32> %v) {
 ; SSE2-LABEL: test4:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; SSE2-NEXT:    retq
 ;
-; AVX2-LABEL: test4:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpmovsxdq %xmm0, %xmm0
-; AVX2-NEXT:    vzeroupper
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: test4:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpmovsxdq %ymm0, %zmm0
-; AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
-; AVX512-NEXT:    vzeroupper
-; AVX512-NEXT:    retq
+; AVX-LABEL: test4:
+; AVX:       # %bb.0:
+; AVX-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX-NEXT:    vzeroupper
+; AVX-NEXT:    retq
   %x = sext <8 x i32> %v to <8 x i64>
   %s = shufflevector <8 x i64> %x, <8 x i64> undef, <2 x i32> <i32 0, i32 1>
   %t = trunc <2 x i64> %s to <2 x i32>
@@ -94,32 +73,31 @@ define <2 x i32> @test4(<8 x i32> %v) {
 define <2 x i32> @test5(<8 x i32> %v) {
 ; SSE2-LABEL: test5:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm2
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[2,3],xmm1[0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,3,2,2]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; SSE2-NEXT:    retq
 ;
 ; AVX2-LABEL: test5:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpmovsxdq %xmm0, %ymm1
+; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; AVX2-NEXT:    vpmovsxdq %xmm0, %xmm0
-; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm1
-; AVX2-NEXT:    vpalignr {{.*#+}} xmm0 = xmm1[8,9,10,11,12,13,14,15],xmm0[0,1,2,3,4,5,6,7]
+; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [6,6,6,6]
+; AVX2-NEXT:    vpermd %ymm0, %ymm1, %ymm0
+; AVX2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: test5:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpmovsxdq %ymm0, %zmm0
-; AVX512-NEXT:    vextracti32x4 $2, %zmm0, %xmm1
-; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; AVX512-NEXT:    vpalignr {{.*#+}} xmm0 = xmm0[8,9,10,11,12,13,14,15],xmm1[0,1,2,3,4,5,6,7]
+; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512-NEXT:    vpextrq $1, %xmm1, %rax
+; AVX512-NEXT:    vextracti32x4 $2, %zmm0, %xmm0
+; AVX512-NEXT:    vmovq %xmm0, %rcx
+; AVX512-NEXT:    vmovd %eax, %xmm0
+; AVX512-NEXT:    vpinsrd $1, %ecx, %xmm0, %xmm0
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
   %x = sext <8 x i32> %v to <8 x i64>
@@ -165,23 +143,13 @@ define <2 x i32> @test8(<8 x i32> %v) {
 ; SSE2-LABEL: test8:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movaps %xmm1, %xmm0
-; SSE2-NEXT:    xorps %xmm1, %xmm1
-; SSE2-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; SSE2-NEXT:    retq
 ;
-; AVX2-LABEL: test8:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
-; AVX2-NEXT:    vzeroupper
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: test8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpmovzxdq {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero
-; AVX512-NEXT:    vextracti32x4 $2, %zmm0, %xmm0
-; AVX512-NEXT:    vzeroupper
-; AVX512-NEXT:    retq
+; AVX-LABEL: test8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX-NEXT:    vzeroupper
+; AVX-NEXT:    retq
   %x = zext <8 x i32> %v to <8 x i64>
   %s = shufflevector <8 x i64> %x, <8 x i64> undef, <2 x i32> <i32 4, i32 5>
   %t = trunc <2 x i64> %s to <2 x i32>
@@ -191,22 +159,13 @@ define <2 x i32> @test8(<8 x i32> %v) {
 define <2 x i32> @test9(<8 x i32> %v) {
 ; SSE2-LABEL: test9:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    xorps %xmm1, %xmm1
-; SSE2-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; SSE2-NEXT:    retq
 ;
-; AVX2-LABEL: test9:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
-; AVX2-NEXT:    vzeroupper
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: test9:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpmovzxdq {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero
-; AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
-; AVX512-NEXT:    vzeroupper
-; AVX512-NEXT:    retq
+; AVX-LABEL: test9:
+; AVX:       # %bb.0:
+; AVX-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX-NEXT:    vzeroupper
+; AVX-NEXT:    retq
   %x = zext <8 x i32> %v to <8 x i64>
   %s = shufflevector <8 x i64> %x, <8 x i64> undef, <2 x i32> <i32 0, i32 1>
   %t = trunc <2 x i64> %s to <2 x i32>
@@ -216,28 +175,31 @@ define <2 x i32> @test9(<8 x i32> %v) {
 define <2 x i32> @test10(<8 x i32> %v) {
 ; SSE2-LABEL: test10:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    xorps %xmm2, %xmm2
-; SSE2-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
-; SSE2-NEXT:    unpckhps {{.*#+}} xmm0 = xmm0[2],xmm2[2],xmm0[3],xmm2[3]
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[2,3],xmm1[0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,3,2,2]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; SSE2-NEXT:    retq
 ;
 ; AVX2-LABEL: test10:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
-; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm1
-; AVX2-NEXT:    vpalignr {{.*#+}} xmm0 = xmm1[8,9,10,11,12,13,14,15],xmm0[0,1,2,3,4,5,6,7]
+; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [6,6,6,6]
+; AVX2-NEXT:    vpermd %ymm0, %ymm1, %ymm0
+; AVX2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: test10:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpmovzxdq {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero
-; AVX512-NEXT:    vextracti32x4 $2, %zmm0, %xmm1
-; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; AVX512-NEXT:    vpalignr {{.*#+}} xmm0 = xmm0[8,9,10,11,12,13,14,15],xmm1[0,1,2,3,4,5,6,7]
+; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512-NEXT:    vpextrq $1, %xmm1, %rax
+; AVX512-NEXT:    vextracti32x4 $2, %zmm0, %xmm0
+; AVX512-NEXT:    vmovq %xmm0, %rcx
+; AVX512-NEXT:    vmovd %eax, %xmm0
+; AVX512-NEXT:    vpinsrd $1, %ecx, %xmm0, %xmm0
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
   %x = zext <8 x i32> %v to <8 x i64>
diff --git a/llvm/test/CodeGen/X86/twoaddr-coalesce-3.ll b/llvm/test/CodeGen/X86/twoaddr-coalesce-3.ll
index b53ff00df..002ad63d3 100644
--- a/llvm/test/CodeGen/X86/twoaddr-coalesce-3.ll
+++ b/llvm/test/CodeGen/X86/twoaddr-coalesce-3.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -mtriple=x86_64-- -relocation-model=pic | FileCheck %s
+; RUN: llc < %s -mtriple=i686-- -relocation-model=pic | FileCheck %s
 ; This test is to ensure the TwoAddrInstruction pass chooses the proper operands to
 ; merge and generates fewer mov insns.
 
diff --git a/llvm/test/CodeGen/X86/twoaddr-coalesce.ll b/llvm/test/CodeGen/X86/twoaddr-coalesce.ll
index 81af91811..0ab3d6ba2 100644
--- a/llvm/test/CodeGen/X86/twoaddr-coalesce.ll
+++ b/llvm/test/CodeGen/X86/twoaddr-coalesce.ll
@@ -1,9 +1,33 @@
-; RUN: llc < %s -mtriple=i686-- | grep mov | count 2
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- | FileCheck %s
 ; rdar://6523745
 
 @"\01LC" = internal constant [4 x i8] c"%d\0A\00"		; <[4 x i8]*> [#uses=1]
 
 define i32 @foo() nounwind {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0: # %bb1.thread
+; CHECK-NEXT:    pushl %ebx
+; CHECK-NEXT:    xorl %ebx, %ebx
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB0_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movl %ebx, %eax
+; CHECK-NEXT:    shrb $7, %al
+; CHECK-NEXT:    addb %bl, %al
+; CHECK-NEXT:    sarb %al
+; CHECK-NEXT:    movsbl %al, %eax
+; CHECK-NEXT:    pushl %eax
+; CHECK-NEXT:    pushl $LC
+; CHECK-NEXT:    calll printf
+; CHECK-NEXT:    addl $8, %esp
+; CHECK-NEXT:    incl %ebx
+; CHECK-NEXT:    cmpl $258, %ebx # imm = 0x102
+; CHECK-NEXT:    jne .LBB0_1
+; CHECK-NEXT:  # %bb.2: # %bb2
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    popl %ebx
+; CHECK-NEXT:    retl
 bb1.thread:
 	br label %bb1
 
diff --git a/llvm/test/CodeGen/X86/typeid-alias.ll b/llvm/test/CodeGen/X86/typeid-alias.ll
new file mode 100644
index 000000000..885ab703b
--- /dev/null
+++ b/llvm/test/CodeGen/X86/typeid-alias.ll
@@ -0,0 +1,18 @@
+; Test that we don't crash in the case where the type info object is an
+; alias pointing to a non-zero offset within a global.
+
+; RUN: llc < %s | FileCheck %s
+
+target triple = "x86_64-unknown-linux-gnu"
+
+%struct.exception = type { i8, i8, i32, i8*, i8*, i32, i8* }
+@g = global %struct.exception zeroinitializer
+@a = alias %struct.exception, %struct.exception* inttoptr (i64 add (i64 ptrtoint (%struct.exception* @g to i64), i64 1297036692682702848) to %struct.exception*)
+
+define i32 @typeid() {
+  ; CHECK: movl $1, %eax
+  %eh_typeid = tail call i32 @llvm.eh.typeid.for( i8* getelementptr (%struct.exception, %struct.exception* @a, i32 0, i32 0))
+  ret i32 %eh_typeid
+}
+
+declare i32 @llvm.eh.typeid.for(i8*)
diff --git a/llvm/test/CodeGen/X86/uadd_sat.ll b/llvm/test/CodeGen/X86/uadd_sat.ll
index 5ee53c017..3d3a1f534 100644
--- a/llvm/test/CodeGen/X86/uadd_sat.ll
+++ b/llvm/test/CodeGen/X86/uadd_sat.ll
@@ -2,10 +2,12 @@
 ; RUN: llc < %s -mtriple=i686 -mattr=cmov | FileCheck %s --check-prefixes=CHECK,X86
 ; RUN: llc < %s -mtriple=x86_64-linux | FileCheck %s --check-prefixes=CHECK,X64
 
-declare  i4  @llvm.uadd.sat.i4   (i4,  i4)
-declare  i32 @llvm.uadd.sat.i32  (i32, i32)
-declare  i64 @llvm.uadd.sat.i64  (i64, i64)
-declare  <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32>, <4 x i32>)
+declare i4 @llvm.uadd.sat.i4(i4, i4)
+declare i8 @llvm.uadd.sat.i8(i8, i8)
+declare i16 @llvm.uadd.sat.i16(i16, i16)
+declare i32 @llvm.uadd.sat.i32(i32, i32)
+declare i64 @llvm.uadd.sat.i64(i64, i64)
+declare <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32>, <4 x i32>)
 
 define i32 @func(i32 %x, i32 %y) nounwind {
 ; X86-LABEL: func:
@@ -22,8 +24,8 @@ define i32 @func(i32 %x, i32 %y) nounwind {
 ; X64-NEXT:    movl $-1, %eax
 ; X64-NEXT:    cmovael %edi, %eax
 ; X64-NEXT:    retq
-  %tmp = call i32 @llvm.uadd.sat.i32(i32 %x, i32 %y);
-  ret i32 %tmp;
+  %tmp = call i32 @llvm.uadd.sat.i32(i32 %x, i32 %y)
+  ret i32 %tmp
 }
 
 define i64 @func2(i64 %x, i64 %y) nounwind {
@@ -44,38 +46,77 @@ define i64 @func2(i64 %x, i64 %y) nounwind {
 ; X64-NEXT:    movq $-1, %rax
 ; X64-NEXT:    cmovaeq %rdi, %rax
 ; X64-NEXT:    retq
-  %tmp = call i64 @llvm.uadd.sat.i64(i64 %x, i64 %y);
-  ret i64 %tmp;
+  %tmp = call i64 @llvm.uadd.sat.i64(i64 %x, i64 %y)
+  ret i64 %tmp
 }
 
-define i4 @func3(i4 %x, i4 %y) nounwind {
-; X86-LABEL: func3:
+define zeroext i16 @func16(i16 zeroext %x, i16 zeroext %y) nounwind {
+; X86-LABEL: func16:
+; X86:       # %bb.0:
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    addw {{[0-9]+}}(%esp), %cx
+; X86-NEXT:    movl $65535, %eax # imm = 0xFFFF
+; X86-NEXT:    cmovael %ecx, %eax
+; X86-NEXT:    # kill: def $ax killed $ax killed $eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: func16:
+; X64:       # %bb.0:
+; X64-NEXT:    addw %si, %di
+; X64-NEXT:    movl $65535, %eax # imm = 0xFFFF
+; X64-NEXT:    cmovael %edi, %eax
+; X64-NEXT:    # kill: def $ax killed $ax killed $eax
+; X64-NEXT:    retq
+  %tmp = call i16 @llvm.uadd.sat.i16(i16 %x, i16 %y)
+  ret i16 %tmp
+}
+
+define zeroext i8 @func8(i8 zeroext %x, i8 zeroext %y) nounwind {
+; X86-LABEL: func8:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
-; X86-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X86-NEXT:    shlb $4, %cl
-; X86-NEXT:    shlb $4, %al
-; X86-NEXT:    addb %cl, %al
+; X86-NEXT:    addb {{[0-9]+}}(%esp), %al
 ; X86-NEXT:    movzbl %al, %ecx
 ; X86-NEXT:    movl $255, %eax
 ; X86-NEXT:    cmovael %ecx, %eax
-; X86-NEXT:    shrb $4, %al
 ; X86-NEXT:    # kill: def $al killed $al killed $eax
 ; X86-NEXT:    retl
 ;
-; X64-LABEL: func3:
+; X64-LABEL: func8:
 ; X64:       # %bb.0:
-; X64-NEXT:    shlb $4, %sil
-; X64-NEXT:    shlb $4, %dil
 ; X64-NEXT:    addb %sil, %dil
 ; X64-NEXT:    movzbl %dil, %ecx
 ; X64-NEXT:    movl $255, %eax
 ; X64-NEXT:    cmovael %ecx, %eax
-; X64-NEXT:    shrb $4, %al
 ; X64-NEXT:    # kill: def $al killed $al killed $eax
 ; X64-NEXT:    retq
-  %tmp = call i4 @llvm.uadd.sat.i4(i4 %x, i4 %y);
-  ret i4 %tmp;
+  %tmp = call i8 @llvm.uadd.sat.i8(i8 %x, i8 %y)
+  ret i8 %tmp
+}
+
+define zeroext i4 @func3(i4 zeroext %x, i4 zeroext %y) nounwind {
+; X86-LABEL: func3:
+; X86:       # %bb.0:
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    addb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    movzbl %al, %ecx
+; X86-NEXT:    cmpb $15, %al
+; X86-NEXT:    movl $15, %eax
+; X86-NEXT:    cmovbl %ecx, %eax
+; X86-NEXT:    movzbl %al, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: func3:
+; X64:       # %bb.0:
+; X64-NEXT:    addb %sil, %dil
+; X64-NEXT:    movzbl %dil, %eax
+; X64-NEXT:    cmpb $15, %al
+; X64-NEXT:    movl $15, %ecx
+; X64-NEXT:    cmovbl %eax, %ecx
+; X64-NEXT:    movzbl %cl, %eax
+; X64-NEXT:    retq
+  %tmp = call i4 @llvm.uadd.sat.i4(i4 %x, i4 %y)
+  ret i4 %tmp
 }
 
 define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
@@ -116,6 +157,6 @@ define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; X64-NEXT:    pcmpgtd %xmm2, %xmm0
 ; X64-NEXT:    por %xmm1, %xmm0
 ; X64-NEXT:    retq
-  %tmp = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> %x, <4 x i32> %y);
-  ret <4 x i32> %tmp;
+  %tmp = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> %x, <4 x i32> %y)
+  ret <4 x i32> %tmp
 }
diff --git a/llvm/test/CodeGen/X86/uadd_sat_plus.ll b/llvm/test/CodeGen/X86/uadd_sat_plus.ll
new file mode 100644
index 000000000..ac1020337
--- /dev/null
+++ b/llvm/test/CodeGen/X86/uadd_sat_plus.ll
@@ -0,0 +1,138 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686 -mattr=cmov | FileCheck %s --check-prefixes=CHECK,X86
+; RUN: llc < %s -mtriple=x86_64-linux | FileCheck %s --check-prefixes=CHECK,X64
+
+declare i4 @llvm.uadd.sat.i4(i4, i4)
+declare i8 @llvm.uadd.sat.i8(i8, i8)
+declare i16 @llvm.uadd.sat.i16(i16, i16)
+declare i32 @llvm.uadd.sat.i32(i32, i32)
+declare i64 @llvm.uadd.sat.i64(i64, i64)
+
+define i32 @func32(i32 %x, i32 %y, i32 %z) nounwind {
+; X86-LABEL: func32:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    imull {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    addl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl $-1, %eax
+; X86-NEXT:    cmovael %ecx, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: func32:
+; X64:       # %bb.0:
+; X64-NEXT:    imull %edx, %esi
+; X64-NEXT:    addl %edi, %esi
+; X64-NEXT:    movl $-1, %eax
+; X64-NEXT:    cmovael %esi, %eax
+; X64-NEXT:    retq
+  %a = mul i32 %y, %z
+  %tmp = call i32 @llvm.uadd.sat.i32(i32 %x, i32 %a)
+  ret i32 %tmp
+}
+
+define i64 @func64(i64 %x, i64 %y, i64 %z) nounwind {
+; X86-LABEL: func64:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    addl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    adcl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl $-1, %ecx
+; X86-NEXT:    cmovbl %ecx, %edx
+; X86-NEXT:    cmovbl %ecx, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: func64:
+; X64:       # %bb.0:
+; X64-NEXT:    addq %rdx, %rdi
+; X64-NEXT:    movq $-1, %rax
+; X64-NEXT:    cmovaeq %rdi, %rax
+; X64-NEXT:    retq
+  %a = mul i64 %y, %z
+  %tmp = call i64 @llvm.uadd.sat.i64(i64 %x, i64 %z)
+  ret i64 %tmp
+}
+
+define zeroext i16 @func16(i16 zeroext %x, i16 zeroext %y, i16 zeroext %z) nounwind {
+; X86-LABEL: func16:
+; X86:       # %bb.0:
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    imulw {{[0-9]+}}(%esp), %cx
+; X86-NEXT:    addw {{[0-9]+}}(%esp), %cx
+; X86-NEXT:    movl $65535, %eax # imm = 0xFFFF
+; X86-NEXT:    cmovael %ecx, %eax
+; X86-NEXT:    # kill: def $ax killed $ax killed $eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: func16:
+; X64:       # %bb.0:
+; X64-NEXT:    imull %edx, %esi
+; X64-NEXT:    addw %di, %si
+; X64-NEXT:    movl $65535, %eax # imm = 0xFFFF
+; X64-NEXT:    cmovael %esi, %eax
+; X64-NEXT:    # kill: def $ax killed $ax killed $eax
+; X64-NEXT:    retq
+  %a = mul i16 %y, %z
+  %tmp = call i16 @llvm.uadd.sat.i16(i16 %x, i16 %a)
+  ret i16 %tmp
+}
+
+define zeroext i8 @func8(i8 zeroext %x, i8 zeroext %y, i8 zeroext %z) nounwind {
+; X86-LABEL: func8:
+; X86:       # %bb.0:
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    mulb {{[0-9]+}}(%esp)
+; X86-NEXT:    addb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    movzbl %al, %ecx
+; X86-NEXT:    movl $255, %eax
+; X86-NEXT:    cmovael %ecx, %eax
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: func8:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %esi, %eax
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    mulb %dl
+; X64-NEXT:    addb %dil, %al
+; X64-NEXT:    movzbl %al, %ecx
+; X64-NEXT:    movl $255, %eax
+; X64-NEXT:    cmovael %ecx, %eax
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    retq
+  %a = mul i8 %y, %z
+  %tmp = call i8 @llvm.uadd.sat.i8(i8 %x, i8 %a)
+  ret i8 %tmp
+}
+
+define zeroext i4 @func4(i4 zeroext %x, i4 zeroext %y, i4 zeroext %z) nounwind {
+; X86-LABEL: func4:
+; X86:       # %bb.0:
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    mulb {{[0-9]+}}(%esp)
+; X86-NEXT:    andb $15, %al
+; X86-NEXT:    addb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    movzbl %al, %ecx
+; X86-NEXT:    cmpb $15, %al
+; X86-NEXT:    movl $15, %eax
+; X86-NEXT:    cmovbl %ecx, %eax
+; X86-NEXT:    movzbl %al, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: func4:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %esi, %eax
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    mulb %dl
+; X64-NEXT:    andb $15, %al
+; X64-NEXT:    addb %dil, %al
+; X64-NEXT:    movzbl %al, %eax
+; X64-NEXT:    cmpb $15, %al
+; X64-NEXT:    movl $15, %ecx
+; X64-NEXT:    cmovbl %eax, %ecx
+; X64-NEXT:    movzbl %cl, %eax
+; X64-NEXT:    retq
+  %a = mul i4 %y, %z
+  %tmp = call i4 @llvm.uadd.sat.i4(i4 %x, i4 %a)
+  ret i4 %tmp
+}
diff --git a/llvm/test/CodeGen/X86/uadd_sat_vec.ll b/llvm/test/CodeGen/X86/uadd_sat_vec.ll
index 08ac7dc56..b398c44b4 100644
--- a/llvm/test/CodeGen/X86/uadd_sat_vec.ll
+++ b/llvm/test/CodeGen/X86/uadd_sat_vec.ll
@@ -210,30 +210,13 @@ define void @v8i8(<8 x i8>* %px, <8 x i8>* %py, <8 x i8>* %pz) nounwind {
 ; SSE-NEXT:    movq %xmm1, (%rdx)
 ; SSE-NEXT:    retq
 ;
-; AVX1-LABEL: v8i8:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
-; AVX1-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
-; AVX1-NEXT:    vpaddusb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vmovq %xmm0, (%rdx)
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: v8i8:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
-; AVX2-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
-; AVX2-NEXT:    vpaddusb %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vmovq %xmm0, (%rdx)
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: v8i8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
-; AVX512-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
-; AVX512-NEXT:    vpaddusb %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX512-NEXT:    vpmovwb %xmm0, (%rdx)
-; AVX512-NEXT:    retq
+; AVX-LABEL: v8i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
+; AVX-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
+; AVX-NEXT:    vpaddusb %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vmovq %xmm0, (%rdx)
+; AVX-NEXT:    retq
   %x = load <8 x i8>, <8 x i8>* %px
   %y = load <8 x i8>, <8 x i8>* %py
   %z = call <8 x i8> @llvm.uadd.sat.v8i8(<8 x i8> %x, <8 x i8> %y)
@@ -250,30 +233,13 @@ define void @v4i8(<4 x i8>* %px, <4 x i8>* %py, <4 x i8>* %pz) nounwind {
 ; SSE-NEXT:    movd %xmm1, (%rdx)
 ; SSE-NEXT:    retq
 ;
-; AVX1-LABEL: v4i8:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX1-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX1-NEXT:    vpaddusb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vmovd %xmm0, (%rdx)
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: v4i8:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX2-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX2-NEXT:    vpaddusb %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vmovd %xmm0, (%rdx)
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: v4i8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX512-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX512-NEXT:    vpaddusb %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; AVX512-NEXT:    vpmovdb %xmm0, (%rdx)
-; AVX512-NEXT:    retq
+; AVX-LABEL: v4i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX-NEXT:    vpaddusb %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vmovd %xmm0, (%rdx)
+; AVX-NEXT:    retq
   %x = load <4 x i8>, <4 x i8>* %px
   %y = load <4 x i8>, <4 x i8>* %py
   %z = call <4 x i8> @llvm.uadd.sat.v4i8(<4 x i8> %x, <4 x i8> %y)
@@ -314,36 +280,15 @@ define void @v2i8(<2 x i8>* %px, <2 x i8>* %py, <2 x i8>* %pz) nounwind {
 ; SSE41-NEXT:    pextrw $0, %xmm1, (%rdx)
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: v2i8:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    movzwl (%rdi), %eax
-; AVX1-NEXT:    vmovd %eax, %xmm0
-; AVX1-NEXT:    movzwl (%rsi), %eax
-; AVX1-NEXT:    vmovd %eax, %xmm1
-; AVX1-NEXT:    vpaddusb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpextrw $0, %xmm0, (%rdx)
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: v2i8:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    movzwl (%rdi), %eax
-; AVX2-NEXT:    vmovd %eax, %xmm0
-; AVX2-NEXT:    movzwl (%rsi), %eax
-; AVX2-NEXT:    vmovd %eax, %xmm1
-; AVX2-NEXT:    vpaddusb %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpextrw $0, %xmm0, (%rdx)
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: v2i8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    movzwl (%rdi), %eax
-; AVX512-NEXT:    vmovd %eax, %xmm0
-; AVX512-NEXT:    movzwl (%rsi), %eax
-; AVX512-NEXT:    vmovd %eax, %xmm1
-; AVX512-NEXT:    vpaddusb %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT:    vpmovqb %xmm0, (%rdx)
-; AVX512-NEXT:    retq
+; AVX-LABEL: v2i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    movzwl (%rdi), %eax
+; AVX-NEXT:    vmovd %eax, %xmm0
+; AVX-NEXT:    movzwl (%rsi), %eax
+; AVX-NEXT:    vmovd %eax, %xmm1
+; AVX-NEXT:    vpaddusb %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpextrw $0, %xmm0, (%rdx)
+; AVX-NEXT:    retq
   %x = load <2 x i8>, <2 x i8>* %px
   %y = load <2 x i8>, <2 x i8>* %py
   %z = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> %x, <2 x i8> %y)
@@ -360,30 +305,13 @@ define void @v4i16(<4 x i16>* %px, <4 x i16>* %py, <4 x i16>* %pz) nounwind {
 ; SSE-NEXT:    movq %xmm1, (%rdx)
 ; SSE-NEXT:    retq
 ;
-; AVX1-LABEL: v4i16:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
-; AVX1-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
-; AVX1-NEXT:    vpaddusw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vmovq %xmm0, (%rdx)
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: v4i16:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
-; AVX2-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
-; AVX2-NEXT:    vpaddusw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vmovq %xmm0, (%rdx)
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: v4i16:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
-; AVX512-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
-; AVX512-NEXT:    vpaddusw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; AVX512-NEXT:    vpmovdw %xmm0, (%rdx)
-; AVX512-NEXT:    retq
+; AVX-LABEL: v4i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
+; AVX-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
+; AVX-NEXT:    vpaddusw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vmovq %xmm0, (%rdx)
+; AVX-NEXT:    retq
   %x = load <4 x i16>, <4 x i16>* %px
   %y = load <4 x i16>, <4 x i16>* %py
   %z = call <4 x i16> @llvm.uadd.sat.v4i16(<4 x i16> %x, <4 x i16> %y)
@@ -400,30 +328,13 @@ define void @v2i16(<2 x i16>* %px, <2 x i16>* %py, <2 x i16>* %pz) nounwind {
 ; SSE-NEXT:    movd %xmm1, (%rdx)
 ; SSE-NEXT:    retq
 ;
-; AVX1-LABEL: v2i16:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX1-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX1-NEXT:    vpaddusw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vmovd %xmm0, (%rdx)
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: v2i16:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX2-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX2-NEXT:    vpaddusw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vmovd %xmm0, (%rdx)
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: v2i16:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX512-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX512-NEXT:    vpaddusw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; AVX512-NEXT:    vpmovqw %xmm0, (%rdx)
-; AVX512-NEXT:    retq
+; AVX-LABEL: v2i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX-NEXT:    vpaddusw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vmovd %xmm0, (%rdx)
+; AVX-NEXT:    retq
   %x = load <2 x i16>, <2 x i16>* %px
   %y = load <2 x i16>, <2 x i16>* %py
   %z = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> %x, <2 x i16> %y)
@@ -631,97 +542,54 @@ define <16 x i1> @v16i1(<16 x i1> %x, <16 x i1> %y) nounwind {
 define <2 x i32> @v2i32(<2 x i32> %x, <2 x i32> %y) nounwind {
 ; SSE2-LABEL: v2i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    psllq $32, %xmm0
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [9223372039002259456,9223372039002259456]
-; SSE2-NEXT:    psllq $32, %xmm1
-; SSE2-NEXT:    paddq %xmm0, %xmm1
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648]
+; SSE2-NEXT:    paddd %xmm0, %xmm1
 ; SSE2-NEXT:    pxor %xmm2, %xmm0
 ; SSE2-NEXT:    pxor %xmm1, %xmm2
-; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm3
 ; SSE2-NEXT:    pcmpgtd %xmm2, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
 ; SSE2-NEXT:    por %xmm1, %xmm0
-; SSE2-NEXT:    por %xmm3, %xmm0
-; SSE2-NEXT:    psrlq $32, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: v2i32:
 ; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    psllq $32, %xmm0
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [9223372039002259456,9223372039002259456]
-; SSSE3-NEXT:    psllq $32, %xmm1
-; SSSE3-NEXT:    paddq %xmm0, %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648]
+; SSSE3-NEXT:    paddd %xmm0, %xmm1
 ; SSSE3-NEXT:    pxor %xmm2, %xmm0
 ; SSSE3-NEXT:    pxor %xmm1, %xmm2
-; SSSE3-NEXT:    movdqa %xmm0, %xmm3
-; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm3
 ; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm2, %xmm3
 ; SSSE3-NEXT:    por %xmm1, %xmm0
-; SSSE3-NEXT:    por %xmm3, %xmm0
-; SSSE3-NEXT:    psrlq $32, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: v2i32:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    psllq $32, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [9223372039002259456,9223372039002259456]
-; SSE41-NEXT:    psllq $32, %xmm1
-; SSE41-NEXT:    paddq %xmm0, %xmm1
-; SSE41-NEXT:    movdqa %xmm0, %xmm3
-; SSE41-NEXT:    pxor %xmm2, %xmm3
-; SSE41-NEXT:    pxor %xmm1, %xmm2
-; SSE41-NEXT:    movdqa %xmm3, %xmm0
-; SSE41-NEXT:    pcmpgtd %xmm2, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm3, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE41-NEXT:    pand %xmm4, %xmm2
-; SSE41-NEXT:    por %xmm2, %xmm0
 ; SSE41-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
-; SSE41-NEXT:    psrlq $32, %xmm1
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pxor %xmm1, %xmm2
+; SSE41-NEXT:    pminud %xmm2, %xmm0
+; SSE41-NEXT:    paddd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: v2i32:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpsllq $32, %xmm0, %xmm0
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
-; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm3
-; AVX1-NEXT:    vpsllq $32, %xmm1, %xmm1
-; AVX1-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm1
-; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm3, %xmm1
-; AVX1-NEXT:    vpor %xmm0, %xmm1, %xmm0
-; AVX1-NEXT:    vpsrlq $32, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm2
+; AVX1-NEXT:    vpminud %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: v2i32:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpsllq $32, %xmm0, %xmm0
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
-; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm3
-; AVX2-NEXT:    vpsllq $32, %xmm1, %xmm1
-; AVX2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm1
-; AVX2-NEXT:    vpcmpgtq %xmm1, %xmm3, %xmm1
-; AVX2-NEXT:    vpor %xmm0, %xmm1, %xmm0
-; AVX2-NEXT:    vpsrlq $32, %xmm0, %xmm0
+; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm2
+; AVX2-NEXT:    vpminud %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: v2i32:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsllq $32, %xmm0, %xmm0
-; AVX512-NEXT:    vpsllq $32, %xmm1, %xmm1
 ; AVX512-NEXT:    vmovdqa %xmm1, %xmm2
 ; AVX512-NEXT:    vpternlogq $15, %xmm1, %xmm1, %xmm2
-; AVX512-NEXT:    vpminuq %xmm2, %xmm0, %xmm0
-; AVX512-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrlq $32, %xmm0, %xmm0
+; AVX512-NEXT:    vpminud %xmm2, %xmm0, %xmm0
+; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    retq
   %z = call <2 x i32> @llvm.uadd.sat.v2i32(<2 x i32> %x, <2 x i32> %y)
   ret <2 x i32> %z
@@ -1028,12 +896,11 @@ define <2 x i64> @v2i64(<2 x i64> %x, <2 x i64> %y) nounwind {
 ; SSE41-NEXT:    pxor %xmm2, %xmm3
 ; SSE41-NEXT:    pxor %xmm1, %xmm2
 ; SSE41-NEXT:    movdqa %xmm3, %xmm0
-; SSE41-NEXT:    pcmpgtd %xmm2, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm3, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE41-NEXT:    pand %xmm4, %xmm2
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm2, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm2, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pand %xmm2, %xmm0
+; SSE41-NEXT:    por %xmm3, %xmm0
 ; SSE41-NEXT:    pcmpeqd %xmm2, %xmm2
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
 ; SSE41-NEXT:    movapd %xmm1, %xmm0
@@ -1140,11 +1007,10 @@ define <4 x i64> @v4i64(<4 x i64> %x, <4 x i64> %y) nounwind {
 ; SSE41-NEXT:    movdqa %xmm2, %xmm6
 ; SSE41-NEXT:    pxor %xmm4, %xmm6
 ; SSE41-NEXT:    movdqa %xmm5, %xmm0
-; SSE41-NEXT:    pcmpgtd %xmm6, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm5, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm6[1,1,3,3]
-; SSE41-NEXT:    pand %xmm7, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm6, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm6, %xmm5
+; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
+; SSE41-NEXT:    pand %xmm6, %xmm0
 ; SSE41-NEXT:    por %xmm5, %xmm0
 ; SSE41-NEXT:    pcmpeqd %xmm5, %xmm5
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm5, %xmm2
@@ -1152,12 +1018,11 @@ define <4 x i64> @v4i64(<4 x i64> %x, <4 x i64> %y) nounwind {
 ; SSE41-NEXT:    pxor %xmm4, %xmm1
 ; SSE41-NEXT:    pxor %xmm3, %xmm4
 ; SSE41-NEXT:    movdqa %xmm1, %xmm6
-; SSE41-NEXT:    pcmpgtd %xmm4, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm1, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3]
-; SSE41-NEXT:    pand %xmm7, %xmm0
-; SSE41-NEXT:    por %xmm6, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm4, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm4, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2]
+; SSE41-NEXT:    pand %xmm6, %xmm0
+; SSE41-NEXT:    por %xmm1, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm5, %xmm3
 ; SSE41-NEXT:    movapd %xmm2, %xmm0
 ; SSE41-NEXT:    movapd %xmm3, %xmm1
@@ -1327,58 +1192,54 @@ define <8 x i64> @v8i64(<8 x i64> %x, <8 x i64> %y) nounwind {
 ; SSE41-NEXT:    movdqa %xmm7, %xmm8
 ; SSE41-NEXT:    movdqa %xmm6, %xmm10
 ; SSE41-NEXT:    movdqa %xmm5, %xmm11
-; SSE41-NEXT:    movdqa %xmm4, %xmm12
+; SSE41-NEXT:    movdqa %xmm4, %xmm5
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm7 = [9223372039002259456,9223372039002259456]
-; SSE41-NEXT:    paddq %xmm0, %xmm12
+; SSE41-NEXT:    paddq %xmm0, %xmm5
 ; SSE41-NEXT:    movdqa %xmm0, %xmm6
 ; SSE41-NEXT:    pxor %xmm7, %xmm6
-; SSE41-NEXT:    movdqa %xmm12, %xmm4
+; SSE41-NEXT:    movdqa %xmm5, %xmm4
 ; SSE41-NEXT:    pxor %xmm7, %xmm4
 ; SSE41-NEXT:    movdqa %xmm6, %xmm0
-; SSE41-NEXT:    pcmpgtd %xmm4, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm6, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm4
-; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm4, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm4, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm6, %xmm0
 ; SSE41-NEXT:    pcmpeqd %xmm9, %xmm9
-; SSE41-NEXT:    blendvpd %xmm0, %xmm9, %xmm12
+; SSE41-NEXT:    blendvpd %xmm0, %xmm9, %xmm5
 ; SSE41-NEXT:    paddq %xmm1, %xmm11
 ; SSE41-NEXT:    pxor %xmm7, %xmm1
 ; SSE41-NEXT:    movdqa %xmm11, %xmm0
 ; SSE41-NEXT:    pxor %xmm7, %xmm0
 ; SSE41-NEXT:    movdqa %xmm1, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm1, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm1, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm9, %xmm11
 ; SSE41-NEXT:    paddq %xmm2, %xmm10
 ; SSE41-NEXT:    pxor %xmm7, %xmm2
 ; SSE41-NEXT:    movdqa %xmm10, %xmm0
 ; SSE41-NEXT:    pxor %xmm7, %xmm0
 ; SSE41-NEXT:    movdqa %xmm2, %xmm1
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm2, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm4, %xmm0
-; SSE41-NEXT:    por %xmm1, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,0,2,2]
+; SSE41-NEXT:    pand %xmm1, %xmm0
+; SSE41-NEXT:    por %xmm2, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm9, %xmm10
 ; SSE41-NEXT:    paddq %xmm3, %xmm8
 ; SSE41-NEXT:    pxor %xmm7, %xmm3
 ; SSE41-NEXT:    pxor %xmm8, %xmm7
 ; SSE41-NEXT:    movdqa %xmm3, %xmm1
-; SSE41-NEXT:    pcmpgtd %xmm7, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm3, %xmm7
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[1,1,3,3]
-; SSE41-NEXT:    pand %xmm2, %xmm0
-; SSE41-NEXT:    por %xmm1, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm7, %xmm1
+; SSE41-NEXT:    pcmpgtd %xmm7, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pand %xmm1, %xmm0
+; SSE41-NEXT:    por %xmm3, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm9, %xmm8
-; SSE41-NEXT:    movapd %xmm12, %xmm0
+; SSE41-NEXT:    movapd %xmm5, %xmm0
 ; SSE41-NEXT:    movapd %xmm11, %xmm1
 ; SSE41-NEXT:    movapd %xmm10, %xmm2
 ; SSE41-NEXT:    movapd %xmm8, %xmm3
@@ -1391,7 +1252,7 @@ define <8 x i64> @v8i64(<8 x i64> %x, <8 x i64> %y) nounwind {
 ; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm9
 ; AVX1-NEXT:    vxorps %xmm7, %xmm7, %xmm7
 ; AVX1-NEXT:    vcmptrueps %ymm7, %ymm7, %ymm10
-; AVX1-NEXT:    vxorps %ymm10, %ymm2, %ymm8
+; AVX1-NEXT:    vxorps %ymm2, %ymm10, %ymm8
 ; AVX1-NEXT:    vextractf128 $1, %ymm8, %xmm6
 ; AVX1-NEXT:    vpxor %xmm4, %xmm6, %xmm7
 ; AVX1-NEXT:    vpcmpgtq %xmm9, %xmm7, %xmm7
@@ -1406,7 +1267,7 @@ define <8 x i64> @v8i64(<8 x i64> %x, <8 x i64> %y) nounwind {
 ; AVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
 ; AVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm8
-; AVX1-NEXT:    vxorps %ymm10, %ymm3, %ymm6
+; AVX1-NEXT:    vxorps %ymm3, %ymm10, %ymm6
 ; AVX1-NEXT:    vextractf128 $1, %ymm6, %xmm7
 ; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm5
 ; AVX1-NEXT:    vpcmpgtq %xmm8, %xmm5, %xmm5
diff --git a/llvm/test/CodeGen/X86/udiv_fix.ll b/llvm/test/CodeGen/X86/udiv_fix.ll
new file mode 100644
index 000000000..7540b394b
--- /dev/null
+++ b/llvm/test/CodeGen/X86/udiv_fix.ll
@@ -0,0 +1,344 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-linux | FileCheck %s --check-prefix=X64
+; RUN: llc < %s -mtriple=i686 -mattr=cmov | FileCheck %s --check-prefix=X86
+
+declare  i4  @llvm.udiv.fix.i4   (i4,  i4,  i32)
+declare  i15 @llvm.udiv.fix.i15  (i15, i15, i32)
+declare  i16 @llvm.udiv.fix.i16  (i16, i16, i32)
+declare  i18 @llvm.udiv.fix.i18  (i18, i18, i32)
+declare  i64 @llvm.udiv.fix.i64  (i64, i64, i32)
+declare  <4 x i32> @llvm.udiv.fix.v4i32(<4 x i32>, <4 x i32>, i32)
+
+define i16 @func(i16 %x, i16 %y) nounwind {
+; X64-LABEL: func:
+; X64:       # %bb.0:
+; X64-NEXT:    movzwl %si, %ecx
+; X64-NEXT:    movzwl %di, %eax
+; X64-NEXT:    shll $7, %eax
+; X64-NEXT:    xorl %edx, %edx
+; X64-NEXT:    divl %ecx
+; X64-NEXT:    # kill: def $ax killed $ax killed $eax
+; X64-NEXT:    retq
+;
+; X86-LABEL: func:
+; X86:       # %bb.0:
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    shll $7, %eax
+; X86-NEXT:    xorl %edx, %edx
+; X86-NEXT:    divl %ecx
+; X86-NEXT:    # kill: def $ax killed $ax killed $eax
+; X86-NEXT:    retl
+  %tmp = call i16 @llvm.udiv.fix.i16(i16 %x, i16 %y, i32 7)
+  ret i16 %tmp
+}
+
+define i16 @func2(i8 %x, i8 %y) nounwind {
+; X64-LABEL: func2:
+; X64:       # %bb.0:
+; X64-NEXT:    movsbl %dil, %eax
+; X64-NEXT:    andl $32767, %eax # imm = 0x7FFF
+; X64-NEXT:    movsbl %sil, %ecx
+; X64-NEXT:    andl $32767, %ecx # imm = 0x7FFF
+; X64-NEXT:    shll $14, %eax
+; X64-NEXT:    xorl %edx, %edx
+; X64-NEXT:    divl %ecx
+; X64-NEXT:    addl %eax, %eax
+; X64-NEXT:    cwtl
+; X64-NEXT:    shrl %eax
+; X64-NEXT:    # kill: def $ax killed $ax killed $eax
+; X64-NEXT:    retq
+;
+; X86-LABEL: func2:
+; X86:       # %bb.0:
+; X86-NEXT:    movsbl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    andl $32767, %ecx # imm = 0x7FFF
+; X86-NEXT:    movsbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    andl $32767, %eax # imm = 0x7FFF
+; X86-NEXT:    shll $14, %eax
+; X86-NEXT:    xorl %edx, %edx
+; X86-NEXT:    divl %ecx
+; X86-NEXT:    addl %eax, %eax
+; X86-NEXT:    cwtl
+; X86-NEXT:    shrl %eax
+; X86-NEXT:    # kill: def $ax killed $ax killed $eax
+; X86-NEXT:    retl
+  %x2 = sext i8 %x to i15
+  %y2 = sext i8 %y to i15
+  %tmp = call i15 @llvm.udiv.fix.i15(i15 %x2, i15 %y2, i32 14)
+  %tmp2 = sext i15 %tmp to i16
+  ret i16 %tmp2
+}
+
+define i16 @func3(i15 %x, i8 %y) nounwind {
+; X64-LABEL: func3:
+; X64:       # %bb.0:
+; X64-NEXT:    # kill: def $edi killed $edi def $rdi
+; X64-NEXT:    leal (%rdi,%rdi), %eax
+; X64-NEXT:    movzbl %sil, %ecx
+; X64-NEXT:    shll $4, %ecx
+; X64-NEXT:    # kill: def $ax killed $ax killed $eax
+; X64-NEXT:    xorl %edx, %edx
+; X64-NEXT:    divw %cx
+; X64-NEXT:    # kill: def $ax killed $ax def $eax
+; X64-NEXT:    addl %eax, %eax
+; X64-NEXT:    cwtl
+; X64-NEXT:    shrl %eax
+; X64-NEXT:    # kill: def $ax killed $ax killed $eax
+; X64-NEXT:    retq
+;
+; X86-LABEL: func3:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    addl %eax, %eax
+; X86-NEXT:    movzbl %cl, %ecx
+; X86-NEXT:    shll $4, %ecx
+; X86-NEXT:    # kill: def $ax killed $ax killed $eax
+; X86-NEXT:    xorl %edx, %edx
+; X86-NEXT:    divw %cx
+; X86-NEXT:    # kill: def $ax killed $ax def $eax
+; X86-NEXT:    addl %eax, %eax
+; X86-NEXT:    cwtl
+; X86-NEXT:    shrl %eax
+; X86-NEXT:    # kill: def $ax killed $ax killed $eax
+; X86-NEXT:    retl
+  %y2 = sext i8 %y to i15
+  %y3 = shl i15 %y2, 7
+  %tmp = call i15 @llvm.udiv.fix.i15(i15 %x, i15 %y3, i32 4)
+  %tmp2 = sext i15 %tmp to i16
+  ret i16 %tmp2
+}
+
+define i4 @func4(i4 %x, i4 %y) nounwind {
+; X64-LABEL: func4:
+; X64:       # %bb.0:
+; X64-NEXT:    andb $15, %sil
+; X64-NEXT:    andb $15, %dil
+; X64-NEXT:    shlb $2, %dil
+; X64-NEXT:    movzbl %dil, %eax
+; X64-NEXT:    divb %sil
+; X64-NEXT:    retq
+;
+; X86-LABEL: func4:
+; X86:       # %bb.0:
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-NEXT:    andb $15, %cl
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    andb $15, %al
+; X86-NEXT:    shlb $2, %al
+; X86-NEXT:    movzbl %al, %eax
+; X86-NEXT:    divb %cl
+; X86-NEXT:    retl
+  %tmp = call i4 @llvm.udiv.fix.i4(i4 %x, i4 %y, i32 2)
+  ret i4 %tmp
+}
+
+define i64 @func5(i64 %x, i64 %y) nounwind {
+; X64-LABEL: func5:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movq %rsi, %rdx
+; X64-NEXT:    movq %rdi, %rsi
+; X64-NEXT:    shlq $31, %rdi
+; X64-NEXT:    shrq $33, %rsi
+; X64-NEXT:    xorl %ecx, %ecx
+; X64-NEXT:    callq __udivti3
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+;
+; X86-LABEL: func5:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    movl %esp, %ebp
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    andl $-8, %esp
+; X86-NEXT:    subl $24, %esp
+; X86-NEXT:    movl 8(%ebp), %eax
+; X86-NEXT:    movl 12(%ebp), %ecx
+; X86-NEXT:    movl %ecx, %edx
+; X86-NEXT:    shrl %edx
+; X86-NEXT:    shldl $31, %eax, %ecx
+; X86-NEXT:    shll $31, %eax
+; X86-NEXT:    movl %esp, %esi
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl 20(%ebp)
+; X86-NEXT:    pushl 16(%ebp)
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl %edx
+; X86-NEXT:    pushl %ecx
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    calll __udivti3
+; X86-NEXT:    addl $32, %esp
+; X86-NEXT:    movl (%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    leal -4(%ebp), %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl
+  %tmp = call i64 @llvm.udiv.fix.i64(i64 %x, i64 %y, i32 31)
+  ret i64 %tmp
+}
+
+define i18 @func6(i16 %x, i16 %y) nounwind {
+; X64-LABEL: func6:
+; X64:       # %bb.0:
+; X64-NEXT:    movswl %di, %eax
+; X64-NEXT:    andl $262143, %eax # imm = 0x3FFFF
+; X64-NEXT:    movswl %si, %ecx
+; X64-NEXT:    andl $262143, %ecx # imm = 0x3FFFF
+; X64-NEXT:    shll $7, %eax
+; X64-NEXT:    xorl %edx, %edx
+; X64-NEXT:    divl %ecx
+; X64-NEXT:    retq
+;
+; X86-LABEL: func6:
+; X86:       # %bb.0:
+; X86-NEXT:    movswl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    andl $262143, %ecx # imm = 0x3FFFF
+; X86-NEXT:    movswl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    andl $262143, %eax # imm = 0x3FFFF
+; X86-NEXT:    shll $7, %eax
+; X86-NEXT:    xorl %edx, %edx
+; X86-NEXT:    divl %ecx
+; X86-NEXT:    retl
+  %x2 = sext i16 %x to i18
+  %y2 = sext i16 %y to i18
+  %tmp = call i18 @llvm.udiv.fix.i18(i18 %x2, i18 %y2, i32 7)
+  ret i18 %tmp
+}
+
+define i16 @func7(i16 %x, i16 %y) nounwind {
+; X64-LABEL: func7:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    shll $16, %eax
+; X64-NEXT:    movzwl %si, %ecx
+; X64-NEXT:    xorl %edx, %edx
+; X64-NEXT:    divl %ecx
+; X64-NEXT:    # kill: def $ax killed $ax killed $eax
+; X64-NEXT:    retq
+;
+; X86-LABEL: func7:
+; X86:       # %bb.0:
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    shll $16, %eax
+; X86-NEXT:    xorl %edx, %edx
+; X86-NEXT:    divl %ecx
+; X86-NEXT:    # kill: def $ax killed $ax killed $eax
+; X86-NEXT:    retl
+  %tmp = call i16 @llvm.udiv.fix.i16(i16 %x, i16 %y, i32 16)
+  ret i16 %tmp
+}
+
+define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
+; X64-LABEL: vec:
+; X64:       # %bb.0:
+; X64-NEXT:    pxor %xmm2, %xmm2
+; X64-NEXT:    movdqa %xmm1, %xmm4
+; X64-NEXT:    punpckhdq {{.*#+}} xmm4 = xmm4[2],xmm2[2],xmm4[3],xmm2[3]
+; X64-NEXT:    movq %xmm4, %rcx
+; X64-NEXT:    movdqa %xmm0, %xmm5
+; X64-NEXT:    punpckhdq {{.*#+}} xmm5 = xmm5[2],xmm2[2],xmm5[3],xmm2[3]
+; X64-NEXT:    psllq $31, %xmm5
+; X64-NEXT:    movq %xmm5, %rax
+; X64-NEXT:    xorl %edx, %edx
+; X64-NEXT:    divq %rcx
+; X64-NEXT:    movq %rax, %xmm3
+; X64-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[2,3,0,1]
+; X64-NEXT:    movq %xmm4, %rcx
+; X64-NEXT:    pshufd {{.*#+}} xmm4 = xmm5[2,3,0,1]
+; X64-NEXT:    movq %xmm4, %rax
+; X64-NEXT:    xorl %edx, %edx
+; X64-NEXT:    divq %rcx
+; X64-NEXT:    movq %rax, %xmm4
+; X64-NEXT:    punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm4[0]
+; X64-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; X64-NEXT:    movq %xmm1, %rcx
+; X64-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; X64-NEXT:    psllq $31, %xmm0
+; X64-NEXT:    movq %xmm0, %rax
+; X64-NEXT:    xorl %edx, %edx
+; X64-NEXT:    divq %rcx
+; X64-NEXT:    movq %rax, %xmm2
+; X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; X64-NEXT:    movq %xmm1, %rcx
+; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; X64-NEXT:    movq %xmm0, %rax
+; X64-NEXT:    xorl %edx, %edx
+; X64-NEXT:    divq %rcx
+; X64-NEXT:    movq %rax, %xmm0
+; X64-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm0[0]
+; X64-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,2],xmm3[0,2]
+; X64-NEXT:    movaps %xmm2, %xmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: vec:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl %eax, %ecx
+; X86-NEXT:    shrl %ecx
+; X86-NEXT:    shll $31, %eax
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl %ecx
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    calll __udivdi3
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    movl %eax, (%esp) # 4-byte Spill
+; X86-NEXT:    movl %ebx, %eax
+; X86-NEXT:    shrl %eax
+; X86-NEXT:    shll $31, %ebx
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    calll __udivdi3
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    movl %eax, %ebx
+; X86-NEXT:    movl %ebp, %eax
+; X86-NEXT:    shrl %eax
+; X86-NEXT:    shll $31, %ebp
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    calll __udivdi3
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    movl %eax, %ebp
+; X86-NEXT:    movl %edi, %eax
+; X86-NEXT:    shrl %eax
+; X86-NEXT:    shll $31, %edi
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    calll __udivdi3
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    movl %eax, 12(%esi)
+; X86-NEXT:    movl %ebp, 8(%esi)
+; X86-NEXT:    movl %ebx, 4(%esi)
+; X86-NEXT:    movl (%esp), %eax # 4-byte Reload
+; X86-NEXT:    movl %eax, (%esi)
+; X86-NEXT:    movl %esi, %eax
+; X86-NEXT:    addl $4, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl $4
+  %tmp = call <4 x i32> @llvm.udiv.fix.v4i32(<4 x i32> %x, <4 x i32> %y, i32 31)
+  ret <4 x i32> %tmp
+}
diff --git a/llvm/test/CodeGen/X86/uint64-to-float.ll b/llvm/test/CodeGen/X86/uint64-to-float.ll
index 662361491..fdab44392 100644
--- a/llvm/test/CodeGen/X86/uint64-to-float.ll
+++ b/llvm/test/CodeGen/X86/uint64-to-float.ll
@@ -13,11 +13,10 @@ define float @test(i64 %a) nounwind {
 ; X86-NEXT:    movl %esp, %ebp
 ; X86-NEXT:    andl $-8, %esp
 ; X86-NEXT:    subl $16, %esp
+; X86-NEXT:    movl 12(%ebp), %eax
 ; X86-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; X86-NEXT:    movlps %xmm0, {{[0-9]+}}(%esp)
-; X86-NEXT:    xorl %eax, %eax
-; X86-NEXT:    cmpl $0, 12(%ebp)
-; X86-NEXT:    setns %al
+; X86-NEXT:    shrl $31, %eax
 ; X86-NEXT:    fildll {{[0-9]+}}(%esp)
 ; X86-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
 ; X86-NEXT:    fstps {{[0-9]+}}(%esp)
diff --git a/llvm/test/CodeGen/X86/umul-with-carry.ll b/llvm/test/CodeGen/X86/umul-with-carry.ll
index 5f76dd390..621a484fe 100644
--- a/llvm/test/CodeGen/X86/umul-with-carry.ll
+++ b/llvm/test/CodeGen/X86/umul-with-carry.ll
@@ -1,5 +1,5 @@
-; RUN: llc < %s -mtriple=i386-- | grep "jc" | count 1
-; XFAIL: *
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i386-- | FileCheck %s
 
 ; FIXME: umul-with-overflow not supported yet.
 
@@ -7,6 +7,24 @@
 @no = internal constant [4 x i8] c"no\0A\00"
 
 define i1 @func(i32 %v1, i32 %v2) nounwind {
+; CHECK-LABEL: func:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    mull {{[0-9]+}}(%esp)
+; CHECK-NEXT:    jno .LBB0_1
+; CHECK-NEXT:  # %bb.2: # %carry
+; CHECK-NEXT:    pushl $no
+; CHECK-NEXT:    calll printf
+; CHECK-NEXT:    addl $4, %esp
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    retl
+; CHECK-NEXT:  .LBB0_1: # %normal
+; CHECK-NEXT:    pushl %eax
+; CHECK-NEXT:    pushl $ok
+; CHECK-NEXT:    calll printf
+; CHECK-NEXT:    addl $8, %esp
+; CHECK-NEXT:    movb $1, %al
+; CHECK-NEXT:    retl
 entry:
   %t = call {i32, i1} @llvm.umul.with.overflow.i32(i32 %v1, i32 %v2)
   %sum = extractvalue {i32, i1} %t, 0
diff --git a/llvm/test/CodeGen/X86/umul_fix.ll b/llvm/test/CodeGen/X86/umul_fix.ll
index 6f7e22245..8481fe4ac 100644
--- a/llvm/test/CodeGen/X86/umul_fix.ll
+++ b/llvm/test/CodeGen/X86/umul_fix.ll
@@ -25,8 +25,8 @@ define i32 @func(i32 %x, i32 %y) nounwind {
 ; X86-NEXT:    mull {{[0-9]+}}(%esp)
 ; X86-NEXT:    shrdl $2, %edx, %eax
 ; X86-NEXT:    retl
-  %tmp = call i32 @llvm.umul.fix.i32(i32 %x, i32 %y, i32 2);
-  ret i32 %tmp;
+  %tmp = call i32 @llvm.umul.fix.i32(i32 %x, i32 %y, i32 2)
+  ret i32 %tmp
 }
 
 define i64 @func2(i64 %x, i64 %y) nounwind {
@@ -60,16 +60,17 @@ define i64 @func2(i64 %x, i64 %y) nounwind {
 ; X86-NEXT:    addl %ebp, %eax
 ; X86-NEXT:    adcl %edi, %edx
 ; X86-NEXT:    imull {{[0-9]+}}(%esp), %esi
-; X86-NEXT:    addl %esi, %edx
-; X86-NEXT:    shldl $30, %eax, %edx
+; X86-NEXT:    addl %edx, %esi
+; X86-NEXT:    shldl $30, %eax, %esi
 ; X86-NEXT:    shldl $30, %ecx, %eax
+; X86-NEXT:    movl %esi, %edx
 ; X86-NEXT:    popl %esi
 ; X86-NEXT:    popl %edi
 ; X86-NEXT:    popl %ebx
 ; X86-NEXT:    popl %ebp
 ; X86-NEXT:    retl
-  %tmp = call i64 @llvm.umul.fix.i64(i64 %x, i64 %y, i32 2);
-  ret i64 %tmp;
+  %tmp = call i64 @llvm.umul.fix.i64(i64 %x, i64 %y, i32 2)
+  ret i64 %tmp
 }
 
 define i4 @func3(i4 %x, i4 %y) nounwind {
@@ -95,8 +96,8 @@ define i4 @func3(i4 %x, i4 %y) nounwind {
 ; X86-NEXT:    shrb $2, %al
 ; X86-NEXT:    # kill: def $al killed $al killed $eax
 ; X86-NEXT:    retl
-  %tmp = call i4 @llvm.umul.fix.i4(i4 %x, i4 %y, i32 2);
-  ret i4 %tmp;
+  %tmp = call i4 @llvm.umul.fix.i4(i4 %x, i4 %y, i32 2)
+  ret i4 %tmp
 }
 
 define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
@@ -152,8 +153,8 @@ define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; X86-NEXT:    popl %ebx
 ; X86-NEXT:    popl %ebp
 ; X86-NEXT:    retl $4
-  %tmp = call <4 x i32> @llvm.umul.fix.v4i32(<4 x i32> %x, <4 x i32> %y, i32 2);
-  ret <4 x i32> %tmp;
+  %tmp = call <4 x i32> @llvm.umul.fix.v4i32(<4 x i32> %x, <4 x i32> %y, i32 2)
+  ret <4 x i32> %tmp
 }
 
 ; These result in regular integer multiplication
@@ -169,8 +170,8 @@ define i32 @func4(i32 %x, i32 %y) nounwind {
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    imull {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    retl
-  %tmp = call i32 @llvm.umul.fix.i32(i32 %x, i32 %y, i32 0);
-  ret i32 %tmp;
+  %tmp = call i32 @llvm.umul.fix.i32(i32 %x, i32 %y, i32 0)
+  ret i32 %tmp
 }
 
 define i64 @func5(i64 %x, i64 %y) nounwind {
@@ -193,8 +194,8 @@ define i64 @func5(i64 %x, i64 %y) nounwind {
 ; X86-NEXT:    addl %esi, %edx
 ; X86-NEXT:    popl %esi
 ; X86-NEXT:    retl
-  %tmp = call i64 @llvm.umul.fix.i64(i64 %x, i64 %y, i32 0);
-  ret i64 %tmp;
+  %tmp = call i64 @llvm.umul.fix.i64(i64 %x, i64 %y, i32 0)
+  ret i64 %tmp
 }
 
 define i4 @func6(i4 %x, i4 %y) nounwind {
@@ -215,8 +216,8 @@ define i4 @func6(i4 %x, i4 %y) nounwind {
 ; X86-NEXT:    andb $15, %cl
 ; X86-NEXT:    mulb %cl
 ; X86-NEXT:    retl
-  %tmp = call i4 @llvm.umul.fix.i4(i4 %x, i4 %y, i32 0);
-  ret i4 %tmp;
+  %tmp = call i4 @llvm.umul.fix.i4(i4 %x, i4 %y, i32 0)
+  ret i4 %tmp
 }
 
 define <4 x i32> @vec2(<4 x i32> %x, <4 x i32> %y) nounwind {
@@ -251,8 +252,8 @@ define <4 x i32> @vec2(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; X86-NEXT:    popl %esi
 ; X86-NEXT:    popl %edi
 ; X86-NEXT:    retl $4
-  %tmp = call <4 x i32> @llvm.umul.fix.v4i32(<4 x i32> %x, <4 x i32> %y, i32 0);
-  ret <4 x i32> %tmp;
+  %tmp = call <4 x i32> @llvm.umul.fix.v4i32(<4 x i32> %x, <4 x i32> %y, i32 0)
+  ret <4 x i32> %tmp
 }
 
 define i64 @func7(i64 %x, i64 %y) nounwind {
@@ -291,8 +292,8 @@ define i64 @func7(i64 %x, i64 %y) nounwind {
 ; X86-NEXT:    popl %ebx
 ; X86-NEXT:    popl %ebp
 ; X86-NEXT:    retl
-  %tmp = call i64 @llvm.umul.fix.i64(i64 %x, i64 %y, i32 32);
-  ret i64 %tmp;
+  %tmp = call i64 @llvm.umul.fix.i64(i64 %x, i64 %y, i32 32)
+  ret i64 %tmp
 }
 
 define i64 @func8(i64 %x, i64 %y) nounwind {
@@ -318,30 +319,29 @@ define i64 @func8(i64 %x, i64 %y) nounwind {
 ; X86-NEXT:    movl %eax, %ebx
 ; X86-NEXT:    movl %ecx, %eax
 ; X86-NEXT:    mull %esi
-; X86-NEXT:    movl %edx, %ecx
-; X86-NEXT:    addl %ebx, %ecx
+; X86-NEXT:    addl %edx, %ebx
 ; X86-NEXT:    adcl $0, %edi
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    mull %ebp
-; X86-NEXT:    movl %edx, %ebx
+; X86-NEXT:    movl %edx, %ecx
 ; X86-NEXT:    movl %eax, %ebp
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    mull %esi
-; X86-NEXT:    addl %ecx, %eax
+; X86-NEXT:    addl %ebx, %eax
 ; X86-NEXT:    adcl %edi, %edx
-; X86-NEXT:    adcl $0, %ebx
+; X86-NEXT:    adcl $0, %ecx
 ; X86-NEXT:    addl %ebp, %edx
-; X86-NEXT:    adcl $0, %ebx
-; X86-NEXT:    shldl $1, %edx, %ebx
+; X86-NEXT:    adcl $0, %ecx
+; X86-NEXT:    shldl $1, %edx, %ecx
 ; X86-NEXT:    shrdl $31, %edx, %eax
-; X86-NEXT:    movl %ebx, %edx
+; X86-NEXT:    movl %ecx, %edx
 ; X86-NEXT:    popl %esi
 ; X86-NEXT:    popl %edi
 ; X86-NEXT:    popl %ebx
 ; X86-NEXT:    popl %ebp
 ; X86-NEXT:    retl
-  %tmp = call i64 @llvm.umul.fix.i64(i64 %x, i64 %y, i32 63);
-  ret i64 %tmp;
+  %tmp = call i64 @llvm.umul.fix.i64(i64 %x, i64 %y, i32 63)
+  ret i64 %tmp
 }
 
 define i64 @func9(i64 %x, i64 %y) nounwind {
@@ -388,6 +388,6 @@ define i64 @func9(i64 %x, i64 %y) nounwind {
 ; X86-NEXT:    popl %ebx
 ; X86-NEXT:    popl %ebp
 ; X86-NEXT:    retl
-  %tmp = call i64 @llvm.umul.fix.i64(i64 %x, i64 %y, i32 64);
-  ret i64 %tmp;
+  %tmp = call i64 @llvm.umul.fix.i64(i64 %x, i64 %y, i32 64)
+  ret i64 %tmp
 }
diff --git a/llvm/test/CodeGen/X86/umul_fix_sat.ll b/llvm/test/CodeGen/X86/umul_fix_sat.ll
new file mode 100644
index 000000000..18b769f32
--- /dev/null
+++ b/llvm/test/CodeGen/X86/umul_fix_sat.ll
@@ -0,0 +1,542 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-linux | FileCheck %s --check-prefix=X64
+; RUN: llc < %s -mtriple=i686 -mattr=cmov | FileCheck %s --check-prefix=X86
+
+declare  i4  @llvm.umul.fix.sat.i4   (i4,  i4, i32)
+declare  i32 @llvm.umul.fix.sat.i32  (i32, i32, i32)
+declare  i64 @llvm.umul.fix.sat.i64  (i64, i64, i32)
+declare  <4 x i32> @llvm.umul.fix.sat.v4i32(<4 x i32>, <4 x i32>, i32)
+
+define i32 @func(i32 %x, i32 %y) nounwind {
+; X64-LABEL: func:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %esi, %eax
+; X64-NEXT:    movl %edi, %ecx
+; X64-NEXT:    imulq %rax, %rcx
+; X64-NEXT:    movq %rcx, %rax
+; X64-NEXT:    shrq $32, %rax
+; X64-NEXT:    shrdl $2, %eax, %ecx
+; X64-NEXT:    cmpl $3, %eax
+; X64-NEXT:    movl $-1, %eax
+; X64-NEXT:    cmovbel %ecx, %eax
+; X64-NEXT:    retq
+;
+; X86-LABEL: func:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    mull {{[0-9]+}}(%esp)
+; X86-NEXT:    shrdl $2, %edx, %eax
+; X86-NEXT:    cmpl $3, %edx
+; X86-NEXT:    movl $-1, %ecx
+; X86-NEXT:    cmoval %ecx, %eax
+; X86-NEXT:    retl
+  %tmp = call i32 @llvm.umul.fix.sat.i32(i32 %x, i32 %y, i32 2)
+  ret i32 %tmp
+}
+
+define i64 @func2(i64 %x, i64 %y) nounwind {
+; X64-LABEL: func2:
+; X64:       # %bb.0:
+; X64-NEXT:    movq %rdi, %rax
+; X64-NEXT:    mulq %rsi
+; X64-NEXT:    shrdq $2, %rdx, %rax
+; X64-NEXT:    cmpq $3, %rdx
+; X64-NEXT:    movq $-1, %rcx
+; X64-NEXT:    cmovaq %rcx, %rax
+; X64-NEXT:    retq
+;
+; X86-LABEL: func2:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    mull %esi
+; X86-NEXT:    movl %edx, %edi
+; X86-NEXT:    movl %eax, %ebx
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    mull {{[0-9]+}}(%esp)
+; X86-NEXT:    movl %eax, %ecx
+; X86-NEXT:    movl %edx, %ebp
+; X86-NEXT:    addl %ebx, %ebp
+; X86-NEXT:    adcl $0, %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    mull %esi
+; X86-NEXT:    movl %edx, %ebx
+; X86-NEXT:    movl %eax, %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    mull {{[0-9]+}}(%esp)
+; X86-NEXT:    addl %ebp, %eax
+; X86-NEXT:    adcl %edi, %edx
+; X86-NEXT:    adcl $0, %ebx
+; X86-NEXT:    addl %esi, %edx
+; X86-NEXT:    adcl $0, %ebx
+; X86-NEXT:    shrdl $2, %eax, %ecx
+; X86-NEXT:    shrdl $2, %edx, %eax
+; X86-NEXT:    shrl $2, %edx
+; X86-NEXT:    orl %ebx, %edx
+; X86-NEXT:    movl $-1, %edx
+; X86-NEXT:    cmovnel %edx, %ecx
+; X86-NEXT:    cmovel %eax, %edx
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl
+  %tmp = call i64 @llvm.umul.fix.sat.i64(i64 %x, i64 %y, i32 2)
+  ret i64 %tmp
+}
+
+define i4 @func3(i4 %x, i4 %y) nounwind {
+; X64-LABEL: func3:
+; X64:       # %bb.0:
+; X64-NEXT:    andl $15, %esi
+; X64-NEXT:    shlb $4, %dil
+; X64-NEXT:    movzbl %dil, %eax
+; X64-NEXT:    imull %esi, %eax
+; X64-NEXT:    movl %eax, %ecx
+; X64-NEXT:    shrb $2, %cl
+; X64-NEXT:    shrl $8, %eax
+; X64-NEXT:    movl %eax, %edx
+; X64-NEXT:    shlb $6, %dl
+; X64-NEXT:    orb %cl, %dl
+; X64-NEXT:    movzbl %dl, %ecx
+; X64-NEXT:    cmpb $3, %al
+; X64-NEXT:    movl $255, %eax
+; X64-NEXT:    cmovbel %ecx, %eax
+; X64-NEXT:    shrb $4, %al
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    retq
+;
+; X86-LABEL: func3:
+; X86:       # %bb.0:
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    andb $15, %al
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-NEXT:    movzbl %al, %edx
+; X86-NEXT:    shlb $4, %cl
+; X86-NEXT:    movzbl %cl, %eax
+; X86-NEXT:    imull %edx, %eax
+; X86-NEXT:    movb %ah, %cl
+; X86-NEXT:    shlb $6, %cl
+; X86-NEXT:    shrb $2, %al
+; X86-NEXT:    orb %cl, %al
+; X86-NEXT:    movzbl %al, %ecx
+; X86-NEXT:    cmpb $3, %ah
+; X86-NEXT:    movl $255, %eax
+; X86-NEXT:    cmovbel %ecx, %eax
+; X86-NEXT:    shrb $4, %al
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    retl
+  %tmp = call i4 @llvm.umul.fix.sat.i4(i4 %x, i4 %y, i32 2)
+  ret i4 %tmp
+}
+
+define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
+; X64-LABEL: vec:
+; X64:       # %bb.0:
+; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,1,2,3]
+; X64-NEXT:    movd %xmm2, %eax
+; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,1,2,3]
+; X64-NEXT:    movd %xmm2, %ecx
+; X64-NEXT:    imulq %rax, %rcx
+; X64-NEXT:    movq %rcx, %rax
+; X64-NEXT:    shrq $32, %rax
+; X64-NEXT:    shrdl $2, %eax, %ecx
+; X64-NEXT:    cmpl $3, %eax
+; X64-NEXT:    movl $-1, %eax
+; X64-NEXT:    cmoval %eax, %ecx
+; X64-NEXT:    movd %ecx, %xmm2
+; X64-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,0,1]
+; X64-NEXT:    movd %xmm3, %ecx
+; X64-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,0,1]
+; X64-NEXT:    movd %xmm3, %edx
+; X64-NEXT:    imulq %rcx, %rdx
+; X64-NEXT:    movq %rdx, %rcx
+; X64-NEXT:    shrq $32, %rcx
+; X64-NEXT:    shrdl $2, %ecx, %edx
+; X64-NEXT:    cmpl $3, %ecx
+; X64-NEXT:    cmoval %eax, %edx
+; X64-NEXT:    movd %edx, %xmm3
+; X64-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; X64-NEXT:    movd %xmm1, %ecx
+; X64-NEXT:    movd %xmm0, %edx
+; X64-NEXT:    imulq %rcx, %rdx
+; X64-NEXT:    movq %rdx, %rcx
+; X64-NEXT:    shrq $32, %rcx
+; X64-NEXT:    shrdl $2, %ecx, %edx
+; X64-NEXT:    cmpl $3, %ecx
+; X64-NEXT:    cmoval %eax, %edx
+; X64-NEXT:    movd %edx, %xmm2
+; X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; X64-NEXT:    movd %xmm1, %ecx
+; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; X64-NEXT:    movd %xmm0, %edx
+; X64-NEXT:    imulq %rcx, %rdx
+; X64-NEXT:    movq %rdx, %rcx
+; X64-NEXT:    shrq $32, %rcx
+; X64-NEXT:    shrdl $2, %ecx, %edx
+; X64-NEXT:    cmpl $3, %ecx
+; X64-NEXT:    cmoval %eax, %edx
+; X64-NEXT:    movd %edx, %xmm0
+; X64-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
+; X64-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; X64-NEXT:    movdqa %xmm2, %xmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: vec:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    mull {{[0-9]+}}(%esp)
+; X86-NEXT:    movl %eax, %esi
+; X86-NEXT:    shrdl $2, %edx, %esi
+; X86-NEXT:    cmpl $3, %edx
+; X86-NEXT:    movl $-1, %ecx
+; X86-NEXT:    cmoval %ecx, %esi
+; X86-NEXT:    movl %ebp, %eax
+; X86-NEXT:    mull {{[0-9]+}}(%esp)
+; X86-NEXT:    movl %eax, %ebp
+; X86-NEXT:    shrdl $2, %edx, %ebp
+; X86-NEXT:    cmpl $3, %edx
+; X86-NEXT:    cmoval %ecx, %ebp
+; X86-NEXT:    movl %ebx, %eax
+; X86-NEXT:    mull {{[0-9]+}}(%esp)
+; X86-NEXT:    movl %eax, %ebx
+; X86-NEXT:    shrdl $2, %edx, %ebx
+; X86-NEXT:    cmpl $3, %edx
+; X86-NEXT:    cmoval %ecx, %ebx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    mull {{[0-9]+}}(%esp)
+; X86-NEXT:    shrdl $2, %edx, %eax
+; X86-NEXT:    cmpl $3, %edx
+; X86-NEXT:    cmoval %ecx, %eax
+; X86-NEXT:    movl %eax, 12(%edi)
+; X86-NEXT:    movl %ebx, 8(%edi)
+; X86-NEXT:    movl %ebp, 4(%edi)
+; X86-NEXT:    movl %esi, (%edi)
+; X86-NEXT:    movl %edi, %eax
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl $4
+  %tmp = call <4 x i32> @llvm.umul.fix.sat.v4i32(<4 x i32> %x, <4 x i32> %y, i32 2)
+  ret <4 x i32> %tmp
+}
+
+; These result in regular integer multiplication
+define i32 @func4(i32 %x, i32 %y) nounwind {
+; X64-LABEL: func4:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    mull %esi
+; X64-NEXT:    movl $-1, %ecx
+; X64-NEXT:    cmovol %ecx, %eax
+; X64-NEXT:    retq
+;
+; X86-LABEL: func4:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    mull {{[0-9]+}}(%esp)
+; X86-NEXT:    movl $-1, %ecx
+; X86-NEXT:    cmovol %ecx, %eax
+; X86-NEXT:    retl
+  %tmp = call i32 @llvm.umul.fix.sat.i32(i32 %x, i32 %y, i32 0)
+  ret i32 %tmp
+}
+
+define i64 @func5(i64 %x, i64 %y) {
+; X64-LABEL: func5:
+; X64:       # %bb.0:
+; X64-NEXT:    movq %rdi, %rax
+; X64-NEXT:    mulq %rsi
+; X64-NEXT:    movq $-1, %rcx
+; X64-NEXT:    cmovoq %rcx, %rax
+; X64-NEXT:    retq
+;
+; X86-LABEL: func5:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    .cfi_def_cfa_offset 8
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    .cfi_def_cfa_offset 12
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    .cfi_def_cfa_offset 16
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    .cfi_def_cfa_offset 20
+; X86-NEXT:    .cfi_offset %esi, -20
+; X86-NEXT:    .cfi_offset %edi, -16
+; X86-NEXT:    .cfi_offset %ebx, -12
+; X86-NEXT:    .cfi_offset %ebp, -8
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    testl %esi, %esi
+; X86-NEXT:    setne %dl
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    setne %cl
+; X86-NEXT:    andb %dl, %cl
+; X86-NEXT:    mull {{[0-9]+}}(%esp)
+; X86-NEXT:    movl %eax, %edi
+; X86-NEXT:    seto %bl
+; X86-NEXT:    movl %esi, %eax
+; X86-NEXT:    mull %ebp
+; X86-NEXT:    movl %eax, %esi
+; X86-NEXT:    seto %ch
+; X86-NEXT:    orb %bl, %ch
+; X86-NEXT:    addl %edi, %esi
+; X86-NEXT:    movl %ebp, %eax
+; X86-NEXT:    mull {{[0-9]+}}(%esp)
+; X86-NEXT:    addl %esi, %edx
+; X86-NEXT:    setb %bl
+; X86-NEXT:    orb %ch, %bl
+; X86-NEXT:    orb %cl, %bl
+; X86-NEXT:    movl $-1, %ecx
+; X86-NEXT:    cmovnel %ecx, %eax
+; X86-NEXT:    cmovnel %ecx, %edx
+; X86-NEXT:    popl %esi
+; X86-NEXT:    .cfi_def_cfa_offset 16
+; X86-NEXT:    popl %edi
+; X86-NEXT:    .cfi_def_cfa_offset 12
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    .cfi_def_cfa_offset 8
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    .cfi_def_cfa_offset 4
+; X86-NEXT:    retl
+  %tmp = call i64 @llvm.umul.fix.sat.i64(i64 %x, i64 %y, i32 0)
+  ret i64 %tmp
+}
+
+define i4 @func6(i4 %x, i4 %y) nounwind {
+; X64-LABEL: func6:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    andb $15, %sil
+; X64-NEXT:    shlb $4, %al
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    mulb %sil
+; X64-NEXT:    movzbl %al, %ecx
+; X64-NEXT:    movl $255, %eax
+; X64-NEXT:    cmovnol %ecx, %eax
+; X64-NEXT:    shrb $4, %al
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    retq
+;
+; X86-LABEL: func6:
+; X86:       # %bb.0:
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-NEXT:    andb $15, %cl
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    shlb $4, %al
+; X86-NEXT:    mulb %cl
+; X86-NEXT:    movzbl %al, %ecx
+; X86-NEXT:    movl $255, %eax
+; X86-NEXT:    cmovnol %ecx, %eax
+; X86-NEXT:    shrb $4, %al
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    retl
+  %tmp = call i4 @llvm.umul.fix.sat.i4(i4 %x, i4 %y, i32 0)
+  ret i4 %tmp
+}
+
+define <4 x i32> @vec2(<4 x i32> %x, <4 x i32> %y) nounwind {
+; X64-LABEL: vec2:
+; X64:       # %bb.0:
+; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,1,2,3]
+; X64-NEXT:    movd %xmm2, %eax
+; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,1,2,3]
+; X64-NEXT:    movd %xmm2, %ecx
+; X64-NEXT:    mull %ecx
+; X64-NEXT:    movl $-1, %ecx
+; X64-NEXT:    cmovol %ecx, %eax
+; X64-NEXT:    movd %eax, %xmm2
+; X64-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,0,1]
+; X64-NEXT:    movd %xmm3, %eax
+; X64-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,0,1]
+; X64-NEXT:    movd %xmm3, %edx
+; X64-NEXT:    mull %edx
+; X64-NEXT:    cmovol %ecx, %eax
+; X64-NEXT:    movd %eax, %xmm3
+; X64-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; X64-NEXT:    movd %xmm0, %eax
+; X64-NEXT:    movd %xmm1, %edx
+; X64-NEXT:    mull %edx
+; X64-NEXT:    cmovol %ecx, %eax
+; X64-NEXT:    movd %eax, %xmm2
+; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; X64-NEXT:    movd %xmm0, %eax
+; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X64-NEXT:    movd %xmm0, %edx
+; X64-NEXT:    mull %edx
+; X64-NEXT:    cmovol %ecx, %eax
+; X64-NEXT:    movd %eax, %xmm0
+; X64-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
+; X64-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; X64-NEXT:    movdqa %xmm2, %xmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: vec2:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    mull {{[0-9]+}}(%esp)
+; X86-NEXT:    movl %eax, %ebp
+; X86-NEXT:    movl $-1, %esi
+; X86-NEXT:    cmovol %esi, %ebp
+; X86-NEXT:    movl %ebx, %eax
+; X86-NEXT:    mull {{[0-9]+}}(%esp)
+; X86-NEXT:    movl %eax, %ebx
+; X86-NEXT:    cmovol %esi, %ebx
+; X86-NEXT:    movl %edi, %eax
+; X86-NEXT:    mull {{[0-9]+}}(%esp)
+; X86-NEXT:    movl %eax, %edi
+; X86-NEXT:    cmovol %esi, %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    mull {{[0-9]+}}(%esp)
+; X86-NEXT:    cmovol %esi, %eax
+; X86-NEXT:    movl %eax, 12(%ecx)
+; X86-NEXT:    movl %edi, 8(%ecx)
+; X86-NEXT:    movl %ebx, 4(%ecx)
+; X86-NEXT:    movl %ebp, (%ecx)
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl $4
+  %tmp = call <4 x i32> @llvm.umul.fix.sat.v4i32(<4 x i32> %x, <4 x i32> %y, i32 0)
+  ret <4 x i32> %tmp
+}
+
+define i64 @func7(i64 %x, i64 %y) nounwind {
+; X64-LABEL: func7:
+; X64:       # %bb.0:
+; X64-NEXT:    movq %rdi, %rax
+; X64-NEXT:    mulq %rsi
+; X64-NEXT:    shrdq $32, %rdx, %rax
+; X64-NEXT:    movl $4294967295, %ecx # imm = 0xFFFFFFFF
+; X64-NEXT:    cmpq %rcx, %rdx
+; X64-NEXT:    movq $-1, %rcx
+; X64-NEXT:    cmovaq %rcx, %rax
+; X64-NEXT:    retq
+;
+; X86-LABEL: func7:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    mull %ebp
+; X86-NEXT:    movl %edx, %edi
+; X86-NEXT:    movl %eax, %ebx
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    mull %esi
+; X86-NEXT:    movl %edx, %ecx
+; X86-NEXT:    addl %ebx, %ecx
+; X86-NEXT:    adcl $0, %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    mull %ebp
+; X86-NEXT:    movl %edx, %ebx
+; X86-NEXT:    movl %eax, %ebp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    mull %esi
+; X86-NEXT:    addl %ecx, %eax
+; X86-NEXT:    adcl %edi, %edx
+; X86-NEXT:    adcl $0, %ebx
+; X86-NEXT:    addl %ebp, %edx
+; X86-NEXT:    adcl $0, %ebx
+; X86-NEXT:    xorl %ecx, %ecx
+; X86-NEXT:    cmpl $1, %ebx
+; X86-NEXT:    sbbl %ecx, %ecx
+; X86-NEXT:    notl %ecx
+; X86-NEXT:    orl %ecx, %eax
+; X86-NEXT:    orl %ecx, %edx
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl
+  %tmp = call i64 @llvm.umul.fix.sat.i64(i64 %x, i64 %y, i32 32)
+  ret i64 %tmp
+}
+
+define i64 @func8(i64 %x, i64 %y) nounwind {
+; X64-LABEL: func8:
+; X64:       # %bb.0:
+; X64-NEXT:    movq %rdi, %rax
+; X64-NEXT:    mulq %rsi
+; X64-NEXT:    shrdq $63, %rdx, %rax
+; X64-NEXT:    movabsq $9223372036854775807, %rcx # imm = 0x7FFFFFFFFFFFFFFF
+; X64-NEXT:    cmpq %rcx, %rdx
+; X64-NEXT:    movq $-1, %rcx
+; X64-NEXT:    cmovaq %rcx, %rax
+; X64-NEXT:    retq
+;
+; X86-LABEL: func8:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    mull %esi
+; X86-NEXT:    movl %edx, %edi
+; X86-NEXT:    movl %eax, %ebx
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    mull {{[0-9]+}}(%esp)
+; X86-NEXT:    movl %edx, %ebp
+; X86-NEXT:    addl %ebx, %ebp
+; X86-NEXT:    adcl $0, %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    mull %esi
+; X86-NEXT:    movl %edx, %ecx
+; X86-NEXT:    movl %eax, %ebx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    mull {{[0-9]+}}(%esp)
+; X86-NEXT:    addl %ebp, %eax
+; X86-NEXT:    adcl %edi, %edx
+; X86-NEXT:    adcl $0, %ecx
+; X86-NEXT:    addl %ebx, %edx
+; X86-NEXT:    adcl $0, %ecx
+; X86-NEXT:    shrdl $31, %edx, %eax
+; X86-NEXT:    movl %edx, %esi
+; X86-NEXT:    shrl $31, %esi
+; X86-NEXT:    xorl %edi, %edi
+; X86-NEXT:    cmpl $1, %esi
+; X86-NEXT:    sbbl %edi, %edi
+; X86-NEXT:    notl %edi
+; X86-NEXT:    orl %edi, %eax
+; X86-NEXT:    shldl $1, %edx, %ecx
+; X86-NEXT:    orl %edi, %ecx
+; X86-NEXT:    movl %ecx, %edx
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl
+  %tmp = call i64 @llvm.umul.fix.sat.i64(i64 %x, i64 %y, i32 63)
+  ret i64 %tmp
+}
diff --git a/llvm/test/CodeGen/X86/umulo-128-legalisation-lowering.ll b/llvm/test/CodeGen/X86/umulo-128-legalisation-lowering.ll
index 7961fbab9..7ce16bbc3 100644
--- a/llvm/test/CodeGen/X86/umulo-128-legalisation-lowering.ll
+++ b/llvm/test/CodeGen/X86/umulo-128-legalisation-lowering.ll
@@ -39,8 +39,8 @@ define { i128, i8 } @muloti_test(i128 %l, i128 %r) unnamed_addr #0 {
 ; X86-NEXT:    .cfi_def_cfa_offset 16
 ; X86-NEXT:    pushl %esi
 ; X86-NEXT:    .cfi_def_cfa_offset 20
-; X86-NEXT:    subl $28, %esp
-; X86-NEXT:    .cfi_def_cfa_offset 48
+; X86-NEXT:    subl $24, %esp
+; X86-NEXT:    .cfi_def_cfa_offset 44
 ; X86-NEXT:    .cfi_offset %esi, -20
 ; X86-NEXT:    .cfi_offset %edi, -16
 ; X86-NEXT:    .cfi_offset %ebx, -12
@@ -64,70 +64,57 @@ define { i128, i8 } @muloti_test(i128 %l, i128 %r) unnamed_addr #0 {
 ; X86-NEXT:    movl %edx, %ecx
 ; X86-NEXT:    addl %esi, %ecx
 ; X86-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    mull %ebp
-; X86-NEXT:    movl %eax, %edi
+; X86-NEXT:    mull %esi
+; X86-NEXT:    movl %esi, %ebx
+; X86-NEXT:    movl %eax, %esi
 ; X86-NEXT:    seto {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebx
-; X86-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    mull %ebx
-; X86-NEXT:    movl %eax, %esi
+; X86-NEXT:    mull %edi
+; X86-NEXT:    movl %eax, %ebp
 ; X86-NEXT:    seto {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X86-NEXT:    addl %edi, %esi
-; X86-NEXT:    movl %ebx, %eax
-; X86-NEXT:    mull %ebp
-; X86-NEXT:    movl %ebp, %ebx
+; X86-NEXT:    addl %esi, %ebp
+; X86-NEXT:    movl %edi, %eax
+; X86-NEXT:    mull %ebx
+; X86-NEXT:    movl %ebx, %esi
+; X86-NEXT:    movl %eax, %ebx
 ; X86-NEXT:    movl %edx, %edi
-; X86-NEXT:    addl %esi, %edi
+; X86-NEXT:    addl %ebp, %edi
 ; X86-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X86-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
 ; X86-NEXT:    adcl %ecx, %edi
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebp
-; X86-NEXT:    movl %ebp, %eax
-; X86-NEXT:    movl %ebx, %ecx
-; X86-NEXT:    mull %ebx
-; X86-NEXT:    movl %edx, %ebx
-; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    mull %ecx
+; X86-NEXT:    movl %esi, %ecx
+; X86-NEXT:    mull %esi
 ; X86-NEXT:    movl %edx, %esi
-; X86-NEXT:    movl %eax, %ecx
-; X86-NEXT:    addl %ebx, %ecx
-; X86-NEXT:    adcl $0, %esi
-; X86-NEXT:    movl %ebp, %eax
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebp
-; X86-NEXT:    mull %ebp
-; X86-NEXT:    movl %edx, %ebx
-; X86-NEXT:    addl %ecx, %eax
 ; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    adcl %esi, %ebx
-; X86-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    mull %ebp
-; X86-NEXT:    movl %edx, %ecx
-; X86-NEXT:    movl %eax, %esi
-; X86-NEXT:    addl %ebx, %esi
-; X86-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
-; X86-NEXT:    adcl %eax, %ecx
-; X86-NEXT:    xorl %edx, %edx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    mull %edx
+; X86-NEXT:    mull %ecx
 ; X86-NEXT:    movl %edx, %ebp
-; X86-NEXT:    movl %eax, %ebx
+; X86-NEXT:    movl %eax, %ecx
+; X86-NEXT:    addl %esi, %ecx
+; X86-NEXT:    adcl $0, %ebp
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    xorl %edx, %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-NEXT:    mull %edx
-; X86-NEXT:    addl %ebx, %eax
-; X86-NEXT:    adcl %ebp, %edx
+; X86-NEXT:    movl %edx, %esi
+; X86-NEXT:    addl %ecx, %eax
+; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    adcl %ebp, %esi
+; X86-NEXT:    setb %cl
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-NEXT:    movl %ebp, %eax
+; X86-NEXT:    mull {{[0-9]+}}(%esp)
 ; X86-NEXT:    addl %esi, %eax
+; X86-NEXT:    movzbl %cl, %ecx
 ; X86-NEXT:    adcl %ecx, %edx
-; X86-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-NEXT:    addl %ebx, %eax
 ; X86-NEXT:    adcl %edi, %edx
 ; X86-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X86-NEXT:    cmpl $0, {{[0-9]+}}(%esp)
+; X86-NEXT:    testl %ebp, %ebp
 ; X86-NEXT:    setne %cl
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-NEXT:    testl %esi, %esi
@@ -167,7 +154,7 @@ define { i128, i8 } @muloti_test(i128 %l, i128 %r) unnamed_addr #0 {
 ; X86-NEXT:    andb $1, %al
 ; X86-NEXT:    movb %al, 16(%ecx)
 ; X86-NEXT:    movl %ecx, %eax
-; X86-NEXT:    addl $28, %esp
+; X86-NEXT:    addl $24, %esp
 ; X86-NEXT:    .cfi_def_cfa_offset 20
 ; X86-NEXT:    popl %esi
 ; X86-NEXT:    .cfi_def_cfa_offset 16
diff --git a/llvm/test/CodeGen/X86/undef-label.ll b/llvm/test/CodeGen/X86/undef-label.ll
index b4be383d5..56e0ca907 100644
--- a/llvm/test/CodeGen/X86/undef-label.ll
+++ b/llvm/test/CodeGen/X86/undef-label.ll
@@ -11,7 +11,7 @@ define void @xyz() {
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    movl $g, %eax
 ; CHECK-NEXT:    movq %rax, %xmm0
-; CHECK-NEXT:    xorps %xmm1, %xmm1
+; CHECK-NEXT:    xorpd %xmm1, %xmm1
 ; CHECK-NEXT:    ucomisd %xmm1, %xmm0
 ; CHECK-NEXT:    jne .LBB0_1
 ; CHECK-NEXT:    jnp .LBB0_2
diff --git a/llvm/test/CodeGen/X86/unfold-masked-merge-vector-variablemask.ll b/llvm/test/CodeGen/X86/unfold-masked-merge-vector-variablemask.ll
index 8923c393d..51b704ec0 100644
--- a/llvm/test/CodeGen/X86/unfold-masked-merge-vector-variablemask.ll
+++ b/llvm/test/CodeGen/X86/unfold-masked-merge-vector-variablemask.ll
@@ -68,17 +68,13 @@ define <2 x i8> @out_v2i8(<2 x i8> %x, <2 x i8> %y, <2 x i8> %mask) nounwind {
 ; CHECK-SSE2-LABEL: out_v2i8:
 ; CHECK-SSE2:       # %bb.0:
 ; CHECK-SSE2-NEXT:    andps %xmm2, %xmm0
-; CHECK-SSE2-NEXT:    xorps {{.*}}(%rip), %xmm2
-; CHECK-SSE2-NEXT:    andps %xmm1, %xmm2
+; CHECK-SSE2-NEXT:    andnps %xmm1, %xmm2
 ; CHECK-SSE2-NEXT:    orps %xmm2, %xmm0
 ; CHECK-SSE2-NEXT:    retq
 ;
 ; CHECK-XOP-LABEL: out_v2i8:
 ; CHECK-XOP:       # %bb.0:
-; CHECK-XOP-NEXT:    vandps %xmm2, %xmm0, %xmm0
-; CHECK-XOP-NEXT:    vxorps {{.*}}(%rip), %xmm2, %xmm2
-; CHECK-XOP-NEXT:    vandps %xmm2, %xmm1, %xmm1
-; CHECK-XOP-NEXT:    vorps %xmm1, %xmm0, %xmm0
+; CHECK-XOP-NEXT:    vpcmov %xmm2, %xmm1, %xmm0, %xmm0
 ; CHECK-XOP-NEXT:    retq
   %mx = and <2 x i8> %x, %mask
   %notmask = xor <2 x i8> %mask, <i8 -1, i8 -1>
@@ -174,17 +170,13 @@ define <4 x i8> @out_v4i8(<4 x i8> %x, <4 x i8> %y, <4 x i8> %mask) nounwind {
 ; CHECK-SSE2-LABEL: out_v4i8:
 ; CHECK-SSE2:       # %bb.0:
 ; CHECK-SSE2-NEXT:    andps %xmm2, %xmm0
-; CHECK-SSE2-NEXT:    xorps {{.*}}(%rip), %xmm2
-; CHECK-SSE2-NEXT:    andps %xmm1, %xmm2
+; CHECK-SSE2-NEXT:    andnps %xmm1, %xmm2
 ; CHECK-SSE2-NEXT:    orps %xmm2, %xmm0
 ; CHECK-SSE2-NEXT:    retq
 ;
 ; CHECK-XOP-LABEL: out_v4i8:
 ; CHECK-XOP:       # %bb.0:
-; CHECK-XOP-NEXT:    vandps %xmm2, %xmm0, %xmm0
-; CHECK-XOP-NEXT:    vxorps {{.*}}(%rip), %xmm2, %xmm2
-; CHECK-XOP-NEXT:    vandps %xmm2, %xmm1, %xmm1
-; CHECK-XOP-NEXT:    vorps %xmm1, %xmm0, %xmm0
+; CHECK-XOP-NEXT:    vpcmov %xmm2, %xmm1, %xmm0, %xmm0
 ; CHECK-XOP-NEXT:    retq
   %mx = and <4 x i8> %x, %mask
   %notmask = xor <4 x i8> %mask, <i8 -1, i8 -1, i8 -1, i8 -1>
@@ -247,17 +239,13 @@ define <4 x i8> @out_v4i8_undef(<4 x i8> %x, <4 x i8> %y, <4 x i8> %mask) nounwi
 ; CHECK-SSE2-LABEL: out_v4i8_undef:
 ; CHECK-SSE2:       # %bb.0:
 ; CHECK-SSE2-NEXT:    andps %xmm2, %xmm0
-; CHECK-SSE2-NEXT:    xorps {{.*}}(%rip), %xmm2
-; CHECK-SSE2-NEXT:    andps %xmm1, %xmm2
+; CHECK-SSE2-NEXT:    andnps %xmm1, %xmm2
 ; CHECK-SSE2-NEXT:    orps %xmm2, %xmm0
 ; CHECK-SSE2-NEXT:    retq
 ;
 ; CHECK-XOP-LABEL: out_v4i8_undef:
 ; CHECK-XOP:       # %bb.0:
-; CHECK-XOP-NEXT:    vandps %xmm2, %xmm0, %xmm0
-; CHECK-XOP-NEXT:    vxorps {{.*}}(%rip), %xmm2, %xmm2
-; CHECK-XOP-NEXT:    vandps %xmm2, %xmm1, %xmm1
-; CHECK-XOP-NEXT:    vorps %xmm1, %xmm0, %xmm0
+; CHECK-XOP-NEXT:    vpcmov %xmm2, %xmm1, %xmm0, %xmm0
 ; CHECK-XOP-NEXT:    retq
   %mx = and <4 x i8> %x, %mask
   %notmask = xor <4 x i8> %mask, <i8 -1, i8 -1, i8 undef, i8 -1>
@@ -300,17 +288,13 @@ define <2 x i16> @out_v2i16(<2 x i16> %x, <2 x i16> %y, <2 x i16> %mask) nounwin
 ; CHECK-SSE2-LABEL: out_v2i16:
 ; CHECK-SSE2:       # %bb.0:
 ; CHECK-SSE2-NEXT:    andps %xmm2, %xmm0
-; CHECK-SSE2-NEXT:    xorps {{.*}}(%rip), %xmm2
-; CHECK-SSE2-NEXT:    andps %xmm1, %xmm2
+; CHECK-SSE2-NEXT:    andnps %xmm1, %xmm2
 ; CHECK-SSE2-NEXT:    orps %xmm2, %xmm0
 ; CHECK-SSE2-NEXT:    retq
 ;
 ; CHECK-XOP-LABEL: out_v2i16:
 ; CHECK-XOP:       # %bb.0:
-; CHECK-XOP-NEXT:    vandps %xmm2, %xmm0, %xmm0
-; CHECK-XOP-NEXT:    vxorps {{.*}}(%rip), %xmm2, %xmm2
-; CHECK-XOP-NEXT:    vandps %xmm2, %xmm1, %xmm1
-; CHECK-XOP-NEXT:    vorps %xmm1, %xmm0, %xmm0
+; CHECK-XOP-NEXT:    vpcmov %xmm2, %xmm1, %xmm0, %xmm0
 ; CHECK-XOP-NEXT:    retq
   %mx = and <2 x i16> %x, %mask
   %notmask = xor <2 x i16> %mask, <i16 -1, i16 -1>
@@ -483,17 +467,13 @@ define <8 x i8> @out_v8i8(<8 x i8> %x, <8 x i8> %y, <8 x i8> %mask) nounwind {
 ; CHECK-SSE2-LABEL: out_v8i8:
 ; CHECK-SSE2:       # %bb.0:
 ; CHECK-SSE2-NEXT:    andps %xmm2, %xmm0
-; CHECK-SSE2-NEXT:    xorps {{.*}}(%rip), %xmm2
-; CHECK-SSE2-NEXT:    andps %xmm1, %xmm2
+; CHECK-SSE2-NEXT:    andnps %xmm1, %xmm2
 ; CHECK-SSE2-NEXT:    orps %xmm2, %xmm0
 ; CHECK-SSE2-NEXT:    retq
 ;
 ; CHECK-XOP-LABEL: out_v8i8:
 ; CHECK-XOP:       # %bb.0:
-; CHECK-XOP-NEXT:    vandps %xmm2, %xmm0, %xmm0
-; CHECK-XOP-NEXT:    vxorps {{.*}}(%rip), %xmm2, %xmm2
-; CHECK-XOP-NEXT:    vandps %xmm2, %xmm1, %xmm1
-; CHECK-XOP-NEXT:    vorps %xmm1, %xmm0, %xmm0
+; CHECK-XOP-NEXT:    vpcmov %xmm2, %xmm1, %xmm0, %xmm0
 ; CHECK-XOP-NEXT:    retq
   %mx = and <8 x i8> %x, %mask
   %notmask = xor <8 x i8> %mask, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
@@ -568,17 +548,13 @@ define <4 x i16> @out_v4i16(<4 x i16> %x, <4 x i16> %y, <4 x i16> %mask) nounwin
 ; CHECK-SSE2-LABEL: out_v4i16:
 ; CHECK-SSE2:       # %bb.0:
 ; CHECK-SSE2-NEXT:    andps %xmm2, %xmm0
-; CHECK-SSE2-NEXT:    xorps {{.*}}(%rip), %xmm2
-; CHECK-SSE2-NEXT:    andps %xmm1, %xmm2
+; CHECK-SSE2-NEXT:    andnps %xmm1, %xmm2
 ; CHECK-SSE2-NEXT:    orps %xmm2, %xmm0
 ; CHECK-SSE2-NEXT:    retq
 ;
 ; CHECK-XOP-LABEL: out_v4i16:
 ; CHECK-XOP:       # %bb.0:
-; CHECK-XOP-NEXT:    vandps %xmm2, %xmm0, %xmm0
-; CHECK-XOP-NEXT:    vxorps {{.*}}(%rip), %xmm2, %xmm2
-; CHECK-XOP-NEXT:    vandps %xmm2, %xmm1, %xmm1
-; CHECK-XOP-NEXT:    vorps %xmm1, %xmm0, %xmm0
+; CHECK-XOP-NEXT:    vpcmov %xmm2, %xmm1, %xmm0, %xmm0
 ; CHECK-XOP-NEXT:    retq
   %mx = and <4 x i16> %x, %mask
   %notmask = xor <4 x i16> %mask, <i16 -1, i16 -1, i16 -1, i16 -1>
@@ -641,17 +617,13 @@ define <4 x i16> @out_v4i16_undef(<4 x i16> %x, <4 x i16> %y, <4 x i16> %mask) n
 ; CHECK-SSE2-LABEL: out_v4i16_undef:
 ; CHECK-SSE2:       # %bb.0:
 ; CHECK-SSE2-NEXT:    andps %xmm2, %xmm0
-; CHECK-SSE2-NEXT:    xorps {{.*}}(%rip), %xmm2
-; CHECK-SSE2-NEXT:    andps %xmm1, %xmm2
+; CHECK-SSE2-NEXT:    andnps %xmm1, %xmm2
 ; CHECK-SSE2-NEXT:    orps %xmm2, %xmm0
 ; CHECK-SSE2-NEXT:    retq
 ;
 ; CHECK-XOP-LABEL: out_v4i16_undef:
 ; CHECK-XOP:       # %bb.0:
-; CHECK-XOP-NEXT:    vandps %xmm2, %xmm0, %xmm0
-; CHECK-XOP-NEXT:    vxorps {{.*}}(%rip), %xmm2, %xmm2
-; CHECK-XOP-NEXT:    vandps %xmm2, %xmm1, %xmm1
-; CHECK-XOP-NEXT:    vorps %xmm1, %xmm0, %xmm0
+; CHECK-XOP-NEXT:    vpcmov %xmm2, %xmm1, %xmm0, %xmm0
 ; CHECK-XOP-NEXT:    retq
   %mx = and <4 x i16> %x, %mask
   %notmask = xor <4 x i16> %mask, <i16 -1, i16 -1, i16 undef, i16 -1>
@@ -692,17 +664,13 @@ define <2 x i32> @out_v2i32(<2 x i32> %x, <2 x i32> %y, <2 x i32> %mask) nounwin
 ; CHECK-SSE2-LABEL: out_v2i32:
 ; CHECK-SSE2:       # %bb.0:
 ; CHECK-SSE2-NEXT:    andps %xmm2, %xmm0
-; CHECK-SSE2-NEXT:    xorps {{.*}}(%rip), %xmm2
-; CHECK-SSE2-NEXT:    andps %xmm1, %xmm2
+; CHECK-SSE2-NEXT:    andnps %xmm1, %xmm2
 ; CHECK-SSE2-NEXT:    orps %xmm2, %xmm0
 ; CHECK-SSE2-NEXT:    retq
 ;
 ; CHECK-XOP-LABEL: out_v2i32:
 ; CHECK-XOP:       # %bb.0:
-; CHECK-XOP-NEXT:    vandps %xmm2, %xmm0, %xmm0
-; CHECK-XOP-NEXT:    vxorps {{.*}}(%rip), %xmm2, %xmm2
-; CHECK-XOP-NEXT:    vandps %xmm2, %xmm1, %xmm1
-; CHECK-XOP-NEXT:    vorps %xmm1, %xmm0, %xmm0
+; CHECK-XOP-NEXT:    vpcmov %xmm2, %xmm1, %xmm0, %xmm0
 ; CHECK-XOP-NEXT:    retq
   %mx = and <2 x i32> %x, %mask
   %notmask = xor <2 x i32> %mask, <i32 -1, i32 -1>
diff --git a/llvm/test/CodeGen/X86/unreachable-trap.ll b/llvm/test/CodeGen/X86/unreachable-trap.ll
index 8de0510ed..ee1a11c76 100644
--- a/llvm/test/CodeGen/X86/unreachable-trap.ll
+++ b/llvm/test/CodeGen/X86/unreachable-trap.ll
@@ -1,10 +1,13 @@
-; RUN: llc -o - %s -mtriple=x86_64-windows-msvc | FileCheck %s --check-prefixes=CHECK,TRAP_AFTER_NORETURN
+; RUN: llc -o - %s -mtriple=x86_64-linux-gnu | FileCheck %s --check-prefixes=CHECK,NORMAL
+; RUN: llc -o - %s -mtriple=x86_64-windows-msvc | FileCheck %s --check-prefixes=CHECK,NORMAL
+; RUN: llc -o - %s -mtriple=x86_64-scei-ps4 | FileCheck %s --check-prefixes=CHECK,TRAP_AFTER_NORETURN
 ; RUN: llc -o - %s -mtriple=x86_64-apple-darwin | FileCheck %s --check-prefixes=CHECK,NO_TRAP_AFTER_NORETURN
 
 ; CHECK-LABEL: call_exit:
 ; CHECK: callq {{_?}}exit
 ; TRAP_AFTER_NORETURN: ud2
 ; NO_TRAP_AFTER_NORETURN-NOT: ud2
+; NORMAL-NOT: ud2
 define i32 @call_exit() noreturn nounwind {
   tail call void @exit(i32 0)
   unreachable
@@ -14,13 +17,17 @@ define i32 @call_exit() noreturn nounwind {
 ; CHECK: ud2
 ; TRAP_AFTER_NORETURN: ud2
 ; NO_TRAP_AFTER_NORETURN-NOT: ud2
+; NORMAL-NOT: ud2
 define i32 @trap() noreturn nounwind {
   tail call void @llvm.trap()
   unreachable
 }
 
 ; CHECK-LABEL: unreachable:
-; CHECK: ud2
+; TRAP_AFTER_NORETURN: ud2
+; NO_TRAP_AFTER_NORETURN: ud2
+; NORMAL-NOT: ud2
+; NORMAL: # -- End function
 define i32 @unreachable() noreturn nounwind {
   unreachable
 }
diff --git a/llvm/test/CodeGen/X86/unused_stackslots.ll b/llvm/test/CodeGen/X86/unused_stackslots.ll
index dca01275c..67653f1e3 100644
--- a/llvm/test/CodeGen/X86/unused_stackslots.ll
+++ b/llvm/test/CodeGen/X86/unused_stackslots.ll
@@ -239,8 +239,8 @@ declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture readonly, i
 ; Function Attrs: argmemonly nounwind
 declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture) #1
 
-attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { argmemonly nounwind }
-attributes #2 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #3 = { nounwind }
 
diff --git a/llvm/test/CodeGen/X86/urem-lkk.ll b/llvm/test/CodeGen/X86/urem-lkk.ll
new file mode 100644
index 000000000..84b0d0d68
--- /dev/null
+++ b/llvm/test/CodeGen/X86/urem-lkk.ll
@@ -0,0 +1,108 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s --check-prefix=CHECK
+
+define i32 @fold_urem_positive_odd(i32 %x) {
+; CHECK-LABEL: fold_urem_positive_odd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    movl %edi, %ecx
+; CHECK-NEXT:    imulq $1491936009, %rcx, %rcx # imm = 0x58ED2309
+; CHECK-NEXT:    shrq $32, %rcx
+; CHECK-NEXT:    movl %edi, %edx
+; CHECK-NEXT:    subl %ecx, %edx
+; CHECK-NEXT:    shrl %edx
+; CHECK-NEXT:    addl %ecx, %edx
+; CHECK-NEXT:    shrl $6, %edx
+; CHECK-NEXT:    imull $95, %edx, %ecx
+; CHECK-NEXT:    subl %ecx, %eax
+; CHECK-NEXT:    retq
+  %1 = urem i32 %x, 95
+  ret i32 %1
+}
+
+
+define i32 @fold_urem_positive_even(i32 %x) {
+; CHECK-LABEL: fold_urem_positive_even:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    movl %edi, %ecx
+; CHECK-NEXT:    movl $4149100483, %edx # imm = 0xF74E3FC3
+; CHECK-NEXT:    imulq %rcx, %rdx
+; CHECK-NEXT:    shrq $42, %rdx
+; CHECK-NEXT:    imull $1060, %edx, %ecx # imm = 0x424
+; CHECK-NEXT:    subl %ecx, %eax
+; CHECK-NEXT:    retq
+  %1 = urem i32 %x, 1060
+  ret i32 %1
+}
+
+
+; Don't fold if we can combine urem with udiv.
+define i32 @combine_urem_udiv(i32 %x) {
+; CHECK-LABEL: combine_urem_udiv:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    # kill: def $edi killed $edi def $rdi
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    imulq $1491936009, %rax, %rax # imm = 0x58ED2309
+; CHECK-NEXT:    shrq $32, %rax
+; CHECK-NEXT:    movl %edi, %ecx
+; CHECK-NEXT:    subl %eax, %ecx
+; CHECK-NEXT:    shrl %ecx
+; CHECK-NEXT:    addl %eax, %ecx
+; CHECK-NEXT:    shrl $6, %ecx
+; CHECK-NEXT:    imull $95, %ecx, %eax
+; CHECK-NEXT:    subl %eax, %edi
+; CHECK-NEXT:    leal (%rdi,%rcx), %eax
+; CHECK-NEXT:    retq
+  %1 = urem i32 %x, 95
+  %2 = udiv i32 %x, 95
+  %3 = add i32 %1, %2
+  ret i32 %3
+}
+
+; Don't fold for divisors that are a power of two.
+define i32 @dont_fold_urem_power_of_two(i32 %x) {
+; CHECK-LABEL: dont_fold_urem_power_of_two:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    andl $63, %eax
+; CHECK-NEXT:    retq
+  %1 = urem i32 %x, 64
+  ret i32 %1
+}
+
+; Don't fold if the divisor is one.
+define i32 @dont_fold_urem_one(i32 %x) {
+; CHECK-LABEL: dont_fold_urem_one:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    retq
+  %1 = urem i32 %x, 1
+  ret i32 %1
+}
+
+; Don't fold if the divisor is 2^32.
+define i32 @dont_fold_urem_i32_umax(i32 %x) {
+; CHECK-LABEL: dont_fold_urem_i32_umax:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    retq
+  %1 = urem i32 %x, 4294967296
+  ret i32 %1
+}
+
+; Don't fold i64 urem
+define i64 @dont_fold_urem_i64(i64 %x) {
+; CHECK-LABEL: dont_fold_urem_i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    shrq %rax
+; CHECK-NEXT:    movabsq $6023426636313322977, %rcx # imm = 0x5397829CBC14E5E1
+; CHECK-NEXT:    mulq %rcx
+; CHECK-NEXT:    shrq $4, %rdx
+; CHECK-NEXT:    imulq $98, %rdx, %rax
+; CHECK-NEXT:    subq %rax, %rdi
+; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    retq
+  %1 = urem i64 %x, 98
+  ret i64 %1
+}
diff --git a/llvm/test/CodeGen/X86/urem-power-of-two.ll b/llvm/test/CodeGen/X86/urem-power-of-two.ll
index c60305e36..ba9b552df 100644
--- a/llvm/test/CodeGen/X86/urem-power-of-two.ll
+++ b/llvm/test/CodeGen/X86/urem-power-of-two.ll
@@ -83,7 +83,6 @@ define i8 @and_pow_2(i8 %x, i8 %y) {
 ; X86-NEXT:    movb {{[0-9]+}}(%esp), %cl
 ; X86-NEXT:    andb $4, %cl
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    # kill: def $eax killed $eax def $ax
 ; X86-NEXT:    divb %cl
 ; X86-NEXT:    movzbl %ah, %eax
 ; X86-NEXT:    # kill: def $al killed $al killed $eax
@@ -93,7 +92,6 @@ define i8 @and_pow_2(i8 %x, i8 %y) {
 ; X64:       # %bb.0:
 ; X64-NEXT:    andb $4, %sil
 ; X64-NEXT:    movzbl %dil, %eax
-; X64-NEXT:    # kill: def $eax killed $eax def $ax
 ; X64-NEXT:    divb %sil
 ; X64-NEXT:    movzbl %ah, %eax
 ; X64-NEXT:    # kill: def $al killed $al killed $eax
diff --git a/llvm/test/CodeGen/X86/urem-seteq-nonzero.ll b/llvm/test/CodeGen/X86/urem-seteq-nonzero.ll
new file mode 100644
index 000000000..d52d47e46
--- /dev/null
+++ b/llvm/test/CodeGen/X86/urem-seteq-nonzero.ll
@@ -0,0 +1,324 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=i686-unknown-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,X86
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,X64
+
+define i1 @t32_3_1(i32 %X) nounwind {
+; X86-LABEL: t32_3_1:
+; X86:       # %bb.0:
+; X86-NEXT:    imull $-1431655765, {{[0-9]+}}(%esp), %eax # imm = 0xAAAAAAAB
+; X86-NEXT:    addl $1431655765, %eax # imm = 0x55555555
+; X86-NEXT:    cmpl $1431655765, %eax # imm = 0x55555555
+; X86-NEXT:    setb %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: t32_3_1:
+; X64:       # %bb.0:
+; X64-NEXT:    imull $-1431655765, %edi, %eax # imm = 0xAAAAAAAB
+; X64-NEXT:    addl $1431655765, %eax # imm = 0x55555555
+; X64-NEXT:    cmpl $1431655765, %eax # imm = 0x55555555
+; X64-NEXT:    setb %al
+; X64-NEXT:    retq
+  %urem = urem i32 %X, 3
+  %cmp = icmp eq i32 %urem, 1
+  ret i1 %cmp
+}
+
+define i1 @t32_3_2(i32 %X) nounwind {
+; X86-LABEL: t32_3_2:
+; X86:       # %bb.0:
+; X86-NEXT:    imull $-1431655765, {{[0-9]+}}(%esp), %eax # imm = 0xAAAAAAAB
+; X86-NEXT:    addl $-1431655766, %eax # imm = 0xAAAAAAAA
+; X86-NEXT:    cmpl $1431655765, %eax # imm = 0x55555555
+; X86-NEXT:    setb %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: t32_3_2:
+; X64:       # %bb.0:
+; X64-NEXT:    imull $-1431655765, %edi, %eax # imm = 0xAAAAAAAB
+; X64-NEXT:    addl $-1431655766, %eax # imm = 0xAAAAAAAA
+; X64-NEXT:    cmpl $1431655765, %eax # imm = 0x55555555
+; X64-NEXT:    setb %al
+; X64-NEXT:    retq
+  %urem = urem i32 %X, 3
+  %cmp = icmp eq i32 %urem, 2
+  ret i1 %cmp
+}
+
+
+define i1 @t32_5_1(i32 %X) nounwind {
+; X86-LABEL: t32_5_1:
+; X86:       # %bb.0:
+; X86-NEXT:    imull $-858993459, {{[0-9]+}}(%esp), %eax # imm = 0xCCCCCCCD
+; X86-NEXT:    addl $858993459, %eax # imm = 0x33333333
+; X86-NEXT:    cmpl $858993459, %eax # imm = 0x33333333
+; X86-NEXT:    setb %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: t32_5_1:
+; X64:       # %bb.0:
+; X64-NEXT:    imull $-858993459, %edi, %eax # imm = 0xCCCCCCCD
+; X64-NEXT:    addl $858993459, %eax # imm = 0x33333333
+; X64-NEXT:    cmpl $858993459, %eax # imm = 0x33333333
+; X64-NEXT:    setb %al
+; X64-NEXT:    retq
+  %urem = urem i32 %X, 5
+  %cmp = icmp eq i32 %urem, 1
+  ret i1 %cmp
+}
+
+define i1 @t32_5_2(i32 %X) nounwind {
+; X86-LABEL: t32_5_2:
+; X86:       # %bb.0:
+; X86-NEXT:    imull $-858993459, {{[0-9]+}}(%esp), %eax # imm = 0xCCCCCCCD
+; X86-NEXT:    addl $1717986918, %eax # imm = 0x66666666
+; X86-NEXT:    cmpl $858993459, %eax # imm = 0x33333333
+; X86-NEXT:    setb %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: t32_5_2:
+; X64:       # %bb.0:
+; X64-NEXT:    imull $-858993459, %edi, %eax # imm = 0xCCCCCCCD
+; X64-NEXT:    addl $1717986918, %eax # imm = 0x66666666
+; X64-NEXT:    cmpl $858993459, %eax # imm = 0x33333333
+; X64-NEXT:    setb %al
+; X64-NEXT:    retq
+  %urem = urem i32 %X, 5
+  %cmp = icmp eq i32 %urem, 2
+  ret i1 %cmp
+}
+
+define i1 @t32_5_3(i32 %X) nounwind {
+; X86-LABEL: t32_5_3:
+; X86:       # %bb.0:
+; X86-NEXT:    imull $-858993459, {{[0-9]+}}(%esp), %eax # imm = 0xCCCCCCCD
+; X86-NEXT:    addl $-1717986919, %eax # imm = 0x99999999
+; X86-NEXT:    cmpl $858993459, %eax # imm = 0x33333333
+; X86-NEXT:    setb %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: t32_5_3:
+; X64:       # %bb.0:
+; X64-NEXT:    imull $-858993459, %edi, %eax # imm = 0xCCCCCCCD
+; X64-NEXT:    addl $-1717986919, %eax # imm = 0x99999999
+; X64-NEXT:    cmpl $858993459, %eax # imm = 0x33333333
+; X64-NEXT:    setb %al
+; X64-NEXT:    retq
+  %urem = urem i32 %X, 5
+  %cmp = icmp eq i32 %urem, 3
+  ret i1 %cmp
+}
+
+define i1 @t32_5_4(i32 %X) nounwind {
+; X86-LABEL: t32_5_4:
+; X86:       # %bb.0:
+; X86-NEXT:    imull $-858993459, {{[0-9]+}}(%esp), %eax # imm = 0xCCCCCCCD
+; X86-NEXT:    addl $-858993460, %eax # imm = 0xCCCCCCCC
+; X86-NEXT:    cmpl $858993459, %eax # imm = 0x33333333
+; X86-NEXT:    setb %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: t32_5_4:
+; X64:       # %bb.0:
+; X64-NEXT:    imull $-858993459, %edi, %eax # imm = 0xCCCCCCCD
+; X64-NEXT:    addl $-858993460, %eax # imm = 0xCCCCCCCC
+; X64-NEXT:    cmpl $858993459, %eax # imm = 0x33333333
+; X64-NEXT:    setb %al
+; X64-NEXT:    retq
+  %urem = urem i32 %X, 5
+  %cmp = icmp eq i32 %urem, 4
+  ret i1 %cmp
+}
+
+
+define i1 @t32_6_1(i32 %X) nounwind {
+; X86-LABEL: t32_6_1:
+; X86:       # %bb.0:
+; X86-NEXT:    imull $-1431655765, {{[0-9]+}}(%esp), %eax # imm = 0xAAAAAAAB
+; X86-NEXT:    addl $1431655765, %eax # imm = 0x55555555
+; X86-NEXT:    rorl %eax
+; X86-NEXT:    cmpl $715827883, %eax # imm = 0x2AAAAAAB
+; X86-NEXT:    setb %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: t32_6_1:
+; X64:       # %bb.0:
+; X64-NEXT:    imull $-1431655765, %edi, %eax # imm = 0xAAAAAAAB
+; X64-NEXT:    addl $1431655765, %eax # imm = 0x55555555
+; X64-NEXT:    rorl %eax
+; X64-NEXT:    cmpl $715827883, %eax # imm = 0x2AAAAAAB
+; X64-NEXT:    setb %al
+; X64-NEXT:    retq
+  %urem = urem i32 %X, 6
+  %cmp = icmp eq i32 %urem, 1
+  ret i1 %cmp
+}
+
+define i1 @t32_6_2(i32 %X) nounwind {
+; X86-LABEL: t32_6_2:
+; X86:       # %bb.0:
+; X86-NEXT:    imull $-1431655765, {{[0-9]+}}(%esp), %eax # imm = 0xAAAAAAAB
+; X86-NEXT:    addl $-1431655766, %eax # imm = 0xAAAAAAAA
+; X86-NEXT:    rorl %eax
+; X86-NEXT:    cmpl $715827883, %eax # imm = 0x2AAAAAAB
+; X86-NEXT:    setb %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: t32_6_2:
+; X64:       # %bb.0:
+; X64-NEXT:    imull $-1431655765, %edi, %eax # imm = 0xAAAAAAAB
+; X64-NEXT:    addl $-1431655766, %eax # imm = 0xAAAAAAAA
+; X64-NEXT:    rorl %eax
+; X64-NEXT:    cmpl $715827883, %eax # imm = 0x2AAAAAAB
+; X64-NEXT:    setb %al
+; X64-NEXT:    retq
+  %urem = urem i32 %X, 6
+  %cmp = icmp eq i32 %urem, 2
+  ret i1 %cmp
+}
+
+define i1 @t32_6_3(i32 %X) nounwind {
+; X86-LABEL: t32_6_3:
+; X86:       # %bb.0:
+; X86-NEXT:    imull $-1431655765, {{[0-9]+}}(%esp), %eax # imm = 0xAAAAAAAB
+; X86-NEXT:    decl %eax
+; X86-NEXT:    rorl %eax
+; X86-NEXT:    cmpl $715827883, %eax # imm = 0x2AAAAAAB
+; X86-NEXT:    setb %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: t32_6_3:
+; X64:       # %bb.0:
+; X64-NEXT:    imull $-1431655765, %edi, %eax # imm = 0xAAAAAAAB
+; X64-NEXT:    decl %eax
+; X64-NEXT:    rorl %eax
+; X64-NEXT:    cmpl $715827883, %eax # imm = 0x2AAAAAAB
+; X64-NEXT:    setb %al
+; X64-NEXT:    retq
+  %urem = urem i32 %X, 6
+  %cmp = icmp eq i32 %urem, 3
+  ret i1 %cmp
+}
+
+define i1 @t32_6_4(i32 %X) nounwind {
+; X86-LABEL: t32_6_4:
+; X86:       # %bb.0:
+; X86-NEXT:    imull $-1431655765, {{[0-9]+}}(%esp), %eax # imm = 0xAAAAAAAB
+; X86-NEXT:    addl $1431655764, %eax # imm = 0x55555554
+; X86-NEXT:    rorl %eax
+; X86-NEXT:    cmpl $715827882, %eax # imm = 0x2AAAAAAA
+; X86-NEXT:    setb %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: t32_6_4:
+; X64:       # %bb.0:
+; X64-NEXT:    imull $-1431655765, %edi, %eax # imm = 0xAAAAAAAB
+; X64-NEXT:    addl $1431655764, %eax # imm = 0x55555554
+; X64-NEXT:    rorl %eax
+; X64-NEXT:    cmpl $715827882, %eax # imm = 0x2AAAAAAA
+; X64-NEXT:    setb %al
+; X64-NEXT:    retq
+  %urem = urem i32 %X, 6
+  %cmp = icmp eq i32 %urem, 4
+  ret i1 %cmp
+}
+
+define i1 @t32_6_5(i32 %X) nounwind {
+; X86-LABEL: t32_6_5:
+; X86:       # %bb.0:
+; X86-NEXT:    imull $-1431655765, {{[0-9]+}}(%esp), %eax # imm = 0xAAAAAAAB
+; X86-NEXT:    addl $-1431655767, %eax # imm = 0xAAAAAAA9
+; X86-NEXT:    rorl %eax
+; X86-NEXT:    cmpl $715827882, %eax # imm = 0x2AAAAAAA
+; X86-NEXT:    setb %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: t32_6_5:
+; X64:       # %bb.0:
+; X64-NEXT:    imull $-1431655765, %edi, %eax # imm = 0xAAAAAAAB
+; X64-NEXT:    addl $-1431655767, %eax # imm = 0xAAAAAAA9
+; X64-NEXT:    rorl %eax
+; X64-NEXT:    cmpl $715827882, %eax # imm = 0x2AAAAAAA
+; X64-NEXT:    setb %al
+; X64-NEXT:    retq
+  %urem = urem i32 %X, 6
+  %cmp = icmp eq i32 %urem, 5
+  ret i1 %cmp
+}
+
+;-------------------------------------------------------------------------------
+; Other widths.
+
+define i1 @t16_3_2(i16 %X) nounwind {
+; X86-LABEL: t16_3_2:
+; X86:       # %bb.0:
+; X86-NEXT:    imull $-21845, {{[0-9]+}}(%esp), %eax # imm = 0xAAAB
+; X86-NEXT:    addl $-21846, %eax # imm = 0xAAAA
+; X86-NEXT:    movzwl %ax, %eax
+; X86-NEXT:    cmpl $21845, %eax # imm = 0x5555
+; X86-NEXT:    setb %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: t16_3_2:
+; X64:       # %bb.0:
+; X64-NEXT:    imull $-21845, %edi, %eax # imm = 0xAAAB
+; X64-NEXT:    addl $-21846, %eax # imm = 0xAAAA
+; X64-NEXT:    movzwl %ax, %eax
+; X64-NEXT:    cmpl $21845, %eax # imm = 0x5555
+; X64-NEXT:    setb %al
+; X64-NEXT:    retq
+  %urem = urem i16 %X, 3
+  %cmp = icmp eq i16 %urem, 2
+  ret i1 %cmp
+}
+
+define i1 @t8_3_2(i8 %X) nounwind {
+; X86-LABEL: t8_3_2:
+; X86:       # %bb.0:
+; X86-NEXT:    imull $-85, {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    addb $-86, %al
+; X86-NEXT:    cmpb $85, %al
+; X86-NEXT:    setb %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: t8_3_2:
+; X64:       # %bb.0:
+; X64-NEXT:    imull $-85, %edi, %eax
+; X64-NEXT:    addb $-86, %al
+; X64-NEXT:    cmpb $85, %al
+; X64-NEXT:    setb %al
+; X64-NEXT:    retq
+  %urem = urem i8 %X, 3
+  %cmp = icmp eq i8 %urem, 2
+  ret i1 %cmp
+}
+
+define i1 @t64_3_2(i64 %X) nounwind {
+; X86-LABEL: t64_3_2:
+; X86:       # %bb.0:
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    pushl $0
+; X86-NEXT:    pushl $3
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    calll __umoddi3
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    xorl $2, %eax
+; X86-NEXT:    orl %edx, %eax
+; X86-NEXT:    sete %al
+; X86-NEXT:    addl $12, %esp
+; X86-NEXT:    retl
+;
+; X64-LABEL: t64_3_2:
+; X64:       # %bb.0:
+; X64-NEXT:    movabsq $-6148914691236517205, %rax # imm = 0xAAAAAAAAAAAAAAAB
+; X64-NEXT:    imulq %rdi, %rax
+; X64-NEXT:    movabsq $-6148914691236517206, %rcx # imm = 0xAAAAAAAAAAAAAAAA
+; X64-NEXT:    addq %rax, %rcx
+; X64-NEXT:    movabsq $6148914691236517205, %rax # imm = 0x5555555555555555
+; X64-NEXT:    cmpq %rax, %rcx
+; X64-NEXT:    setb %al
+; X64-NEXT:    retq
+  %urem = urem i64 %X, 3
+  %cmp = icmp eq i64 %urem, 2
+  ret i1 %cmp
+}
diff --git a/llvm/test/CodeGen/X86/urem-seteq-vec-nonsplat.ll b/llvm/test/CodeGen/X86/urem-seteq-vec-nonsplat.ll
index 905f45b69..089acd843 100644
--- a/llvm/test/CodeGen/X86/urem-seteq-vec-nonsplat.ll
+++ b/llvm/test/CodeGen/X86/urem-seteq-vec-nonsplat.ll
@@ -5,38 +5,34 @@
 ; RUN: llc -mtriple=x86_64-unknown-linux-gnu -mattr=+avx2 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-AVX,CHECK-AVX2
 ; RUN: llc -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512f,+avx512vl < %s | FileCheck %s --check-prefixes=CHECK,CHECK-AVX,CHECK-AVX512VL
 
-; At the moment, BuildUREMEqFold does not handle nonsplat vectors.
-
 ; Odd+Even divisors
 define <4 x i32> @test_urem_odd_even(<4 x i32> %X) nounwind {
 ; CHECK-SSE2-LABEL: test_urem_odd_even:
 ; CHECK-SSE2:       # %bb.0:
-; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm1
-; CHECK-SSE2-NEXT:    psrld $1, %xmm1
-; CHECK-SSE2-NEXT:    movdqa %xmm1, %xmm2
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1],xmm0[3,3]
-; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [3435973837,2454267027,1374389535,1374389535]
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
-; CHECK-SSE2-NEXT:    pmuludq %xmm2, %xmm4
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,3,2,3]
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,0],xmm0[0,0]
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[2,0],xmm0[2,3]
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [3435973837,2454267027,1374389535,1374389535]
+; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm2
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
+; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm3
+; CHECK-SSE2-NEXT:    psrld $1, %xmm3
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm0[3,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
 ; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm1
 ; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
-; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
-; CHECK-SSE2-NEXT:    movdqa %xmm1, %xmm2
-; CHECK-SSE2-NEXT:    psrld $5, %xmm2
-; CHECK-SSE2-NEXT:    movdqa %xmm1, %xmm3
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm1
+; CHECK-SSE2-NEXT:    psrld $5, %xmm1
+; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm3
 ; CHECK-SSE2-NEXT:    psrld $2, %xmm3
-; CHECK-SSE2-NEXT:    psrld $3, %xmm1
-; CHECK-SSE2-NEXT:    movsd {{.*#+}} xmm1 = xmm3[0],xmm1[1]
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm2[3,3]
-; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [5,14,25,100]
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,1,3,3]
+; CHECK-SSE2-NEXT:    psrld $3, %xmm2
+; CHECK-SSE2-NEXT:    movsd {{.*#+}} xmm2 = xmm3[0],xmm2[1]
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm1[3,3]
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [5,14,25,100]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,3,3]
 ; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm4
 ; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[0,2,2,3]
-; CHECK-SSE2-NEXT:    pmuludq %xmm2, %xmm1
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
 ; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
 ; CHECK-SSE2-NEXT:    psubd %xmm1, %xmm0
 ; CHECK-SSE2-NEXT:    pxor %xmm1, %xmm1
@@ -48,14 +44,13 @@ define <4 x i32> @test_urem_odd_even(<4 x i32> %X) nounwind {
 ; CHECK-SSE41:       # %bb.0:
 ; CHECK-SSE41-NEXT:    movdqa %xmm0, %xmm1
 ; CHECK-SSE41-NEXT:    psrld $1, %xmm1
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3],xmm0[4,5,6,7]
-; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [3435973837,2454267027,1374389535,1374389535]
-; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
-; CHECK-SSE41-NEXT:    pmuludq %xmm2, %xmm4
-; CHECK-SSE41-NEXT:    pmuludq %xmm3, %xmm1
-; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm4[2,3],xmm1[4,5],xmm4[6,7]
+; CHECK-SSE41-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[3,3]
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [3435973837,2454267027,1374389535,1374389535]
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
+; CHECK-SSE41-NEXT:    pmuludq %xmm1, %xmm3
+; CHECK-SSE41-NEXT:    pmuludq %xmm0, %xmm2
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,3,3]
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5],xmm3[6,7]
 ; CHECK-SSE41-NEXT:    movdqa %xmm1, %xmm2
 ; CHECK-SSE41-NEXT:    psrld $5, %xmm2
 ; CHECK-SSE41-NEXT:    movdqa %xmm1, %xmm3
@@ -74,14 +69,13 @@ define <4 x i32> @test_urem_odd_even(<4 x i32> %X) nounwind {
 ; CHECK-AVX1-LABEL: test_urem_odd_even:
 ; CHECK-AVX1:       # %bb.0:
 ; CHECK-AVX1-NEXT:    vpsrld $1, %xmm0, %xmm1
-; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3],xmm0[4,5,6,7]
-; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; CHECK-AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [3435973837,2454267027,1374389535,1374389535]
-; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
-; CHECK-AVX1-NEXT:    vpmuludq %xmm4, %xmm2, %xmm2
+; CHECK-AVX1-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[3,3]
+; CHECK-AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [3435973837,2454267027,1374389535,1374389535]
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
 ; CHECK-AVX1-NEXT:    vpmuludq %xmm3, %xmm1, %xmm1
-; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
+; CHECK-AVX1-NEXT:    vpmuludq %xmm2, %xmm0, %xmm2
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
 ; CHECK-AVX1-NEXT:    vpsrld $5, %xmm1, %xmm2
 ; CHECK-AVX1-NEXT:    vpsrld $2, %xmm1, %xmm3
 ; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
@@ -115,18 +109,9 @@ define <4 x i32> @test_urem_odd_even(<4 x i32> %X) nounwind {
 ;
 ; CHECK-AVX512VL-LABEL: test_urem_odd_even:
 ; CHECK-AVX512VL:       # %bb.0:
-; CHECK-AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm1 = [3435973837,2454267027,1374389535,1374389535]
-; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; CHECK-AVX512VL-NEXT:    vpsrlvd {{.*}}(%rip), %xmm0, %xmm3
-; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
-; CHECK-AVX512VL-NEXT:    vpmuludq %xmm2, %xmm4, %xmm2
-; CHECK-AVX512VL-NEXT:    vpmuludq %xmm1, %xmm3, %xmm1
-; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; CHECK-AVX512VL-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; CHECK-AVX512VL-NEXT:    vpsrlvd {{.*}}(%rip), %xmm1, %xmm1
-; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
-; CHECK-AVX512VL-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
-; CHECK-AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vprorvd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
 ; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
 ; CHECK-AVX512VL-NEXT:    vpsrld $31, %xmm0, %xmm0
 ; CHECK-AVX512VL-NEXT:    retq
@@ -142,97 +127,33 @@ define <4 x i32> @test_urem_odd_even(<4 x i32> %X) nounwind {
 define <4 x i32> @test_urem_odd_allones_eq(<4 x i32> %X) nounwind {
 ; CHECK-SSE2-LABEL: test_urem_odd_allones_eq:
 ; CHECK-SSE2:       # %bb.0:
-; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = <3435973837,u,2147483649,u>
-; CHECK-SSE2-NEXT:    pmuludq %xmm0, %xmm1
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
-; CHECK-SSE2-NEXT:    pmuludq {{.*}}(%rip), %xmm2
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
-; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
-; CHECK-SSE2-NEXT:    movdqa %xmm1, %xmm2
-; CHECK-SSE2-NEXT:    psrld $2, %xmm2
-; CHECK-SSE2-NEXT:    psrld $31, %xmm1
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[2,0],xmm2[3,0]
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,1],xmm1[0,2]
-; CHECK-SSE2-NEXT:    pmuludq {{.*}}(%rip), %xmm2
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
-; CHECK-SSE2-NEXT:    pmuludq {{.*}}(%rip), %xmm3
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,2,2,3]
-; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
-; CHECK-SSE2-NEXT:    psubd %xmm1, %xmm0
-; CHECK-SSE2-NEXT:    pxor %xmm1, %xmm1
-; CHECK-SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
-; CHECK-SSE2-NEXT:    psrld $31, %xmm0
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq {{.*}}(%rip), %xmm0
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; CHECK-SSE2-NEXT:    pmuludq {{.*}}(%rip), %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; CHECK-SSE2-NEXT:    pxor {{.*}}(%rip), %xmm0
+; CHECK-SSE2-NEXT:    pcmpgtd {{.*}}(%rip), %xmm0
+; CHECK-SSE2-NEXT:    pandn {{.*}}(%rip), %xmm0
 ; CHECK-SSE2-NEXT:    retq
 ;
 ; CHECK-SSE41-LABEL: test_urem_odd_allones_eq:
 ; CHECK-SSE41:       # %bb.0:
-; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
-; CHECK-SSE41-NEXT:    pmuludq {{.*}}(%rip), %xmm1
-; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm2 = <3435973837,u,2147483649,u>
-; CHECK-SSE41-NEXT:    pmuludq %xmm0, %xmm2
-; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
-; CHECK-SSE41-NEXT:    movdqa %xmm2, %xmm1
-; CHECK-SSE41-NEXT:    psrld $31, %xmm1
-; CHECK-SSE41-NEXT:    psrld $2, %xmm2
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm1[4,5],xmm2[6,7]
-; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm2
-; CHECK-SSE41-NEXT:    psubd %xmm2, %xmm0
-; CHECK-SSE41-NEXT:    pxor %xmm1, %xmm1
+; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm0
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [858993459,858993459,1,858993459]
+; CHECK-SSE41-NEXT:    pminud %xmm0, %xmm1
 ; CHECK-SSE41-NEXT:    pcmpeqd %xmm1, %xmm0
 ; CHECK-SSE41-NEXT:    psrld $31, %xmm0
 ; CHECK-SSE41-NEXT:    retq
 ;
-; CHECK-AVX1-LABEL: test_urem_odd_allones_eq:
-; CHECK-AVX1:       # %bb.0:
-; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
-; CHECK-AVX1-NEXT:    vpmuludq {{.*}}(%rip), %xmm1, %xmm1
-; CHECK-AVX1-NEXT:    vpmuludq {{.*}}(%rip), %xmm0, %xmm2
-; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
-; CHECK-AVX1-NEXT:    vpsrld $31, %xmm1, %xmm2
-; CHECK-AVX1-NEXT:    vpsrld $2, %xmm1, %xmm1
-; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm2[4,5],xmm1[6,7]
-; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
-; CHECK-AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
-; CHECK-AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; CHECK-AVX1-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
-; CHECK-AVX1-NEXT:    vpsrld $31, %xmm0, %xmm0
-; CHECK-AVX1-NEXT:    retq
-;
-; CHECK-AVX2-LABEL: test_urem_odd_allones_eq:
-; CHECK-AVX2:       # %bb.0:
-; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
-; CHECK-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [3435973837,3435973837,3435973837,3435973837]
-; CHECK-AVX2-NEXT:    vpmuludq %xmm2, %xmm1, %xmm1
-; CHECK-AVX2-NEXT:    vpmuludq {{.*}}(%rip), %xmm0, %xmm2
-; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2],xmm1[3]
-; CHECK-AVX2-NEXT:    vpsrlvd {{.*}}(%rip), %xmm1, %xmm1
-; CHECK-AVX2-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
-; CHECK-AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
-; CHECK-AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; CHECK-AVX2-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
-; CHECK-AVX2-NEXT:    vpsrld $31, %xmm0, %xmm0
-; CHECK-AVX2-NEXT:    retq
-;
-; CHECK-AVX512VL-LABEL: test_urem_odd_allones_eq:
-; CHECK-AVX512VL:       # %bb.0:
-; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
-; CHECK-AVX512VL-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [3435973837,3435973837,3435973837,3435973837]
-; CHECK-AVX512VL-NEXT:    vpmuludq %xmm2, %xmm1, %xmm1
-; CHECK-AVX512VL-NEXT:    vpmuludq {{.*}}(%rip), %xmm0, %xmm2
-; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; CHECK-AVX512VL-NEXT:    vpblendd {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2],xmm1[3]
-; CHECK-AVX512VL-NEXT:    vpsrlvd {{.*}}(%rip), %xmm1, %xmm1
-; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
-; CHECK-AVX512VL-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
-; CHECK-AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
-; CHECK-AVX512VL-NEXT:    vpsrld $31, %xmm0, %xmm0
-; CHECK-AVX512VL-NEXT:    retq
+; CHECK-AVX-LABEL: test_urem_odd_allones_eq:
+; CHECK-AVX:       # %bb.0:
+; CHECK-AVX-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
+; CHECK-AVX-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX-NEXT:    retq
   %urem = urem <4 x i32> %X, <i32 5, i32 5, i32 4294967295, i32 5>
   %cmp = icmp eq <4 x i32> %urem, <i32 0, i32 0, i32 0, i32 0>
   %ret = zext <4 x i1> %cmp to <4 x i32>
@@ -241,98 +162,33 @@ define <4 x i32> @test_urem_odd_allones_eq(<4 x i32> %X) nounwind {
 define <4 x i32> @test_urem_odd_allones_ne(<4 x i32> %X) nounwind {
 ; CHECK-SSE2-LABEL: test_urem_odd_allones_ne:
 ; CHECK-SSE2:       # %bb.0:
-; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = <3435973837,u,2147483649,u>
-; CHECK-SSE2-NEXT:    pmuludq %xmm0, %xmm1
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
-; CHECK-SSE2-NEXT:    pmuludq {{.*}}(%rip), %xmm2
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
-; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
-; CHECK-SSE2-NEXT:    movdqa %xmm1, %xmm2
-; CHECK-SSE2-NEXT:    psrld $2, %xmm2
-; CHECK-SSE2-NEXT:    psrld $31, %xmm1
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[2,0],xmm2[3,0]
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,1],xmm1[0,2]
-; CHECK-SSE2-NEXT:    pmuludq {{.*}}(%rip), %xmm2
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
-; CHECK-SSE2-NEXT:    pmuludq {{.*}}(%rip), %xmm3
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,2,2,3]
-; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
-; CHECK-SSE2-NEXT:    psubd %xmm1, %xmm0
-; CHECK-SSE2-NEXT:    pxor %xmm1, %xmm1
-; CHECK-SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
-; CHECK-SSE2-NEXT:    pandn {{.*}}(%rip), %xmm0
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq {{.*}}(%rip), %xmm0
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; CHECK-SSE2-NEXT:    pmuludq {{.*}}(%rip), %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; CHECK-SSE2-NEXT:    pxor {{.*}}(%rip), %xmm0
+; CHECK-SSE2-NEXT:    pcmpgtd {{.*}}(%rip), %xmm0
+; CHECK-SSE2-NEXT:    psrld $31, %xmm0
 ; CHECK-SSE2-NEXT:    retq
 ;
 ; CHECK-SSE41-LABEL: test_urem_odd_allones_ne:
 ; CHECK-SSE41:       # %bb.0:
-; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
-; CHECK-SSE41-NEXT:    pmuludq {{.*}}(%rip), %xmm1
-; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm2 = <3435973837,u,2147483649,u>
-; CHECK-SSE41-NEXT:    pmuludq %xmm0, %xmm2
-; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
-; CHECK-SSE41-NEXT:    movdqa %xmm2, %xmm1
-; CHECK-SSE41-NEXT:    psrld $31, %xmm1
-; CHECK-SSE41-NEXT:    psrld $2, %xmm2
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm1[4,5],xmm2[6,7]
-; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm2
-; CHECK-SSE41-NEXT:    psubd %xmm2, %xmm0
-; CHECK-SSE41-NEXT:    pxor %xmm1, %xmm1
+; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm0
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [858993460,858993460,2,858993460]
+; CHECK-SSE41-NEXT:    pmaxud %xmm0, %xmm1
 ; CHECK-SSE41-NEXT:    pcmpeqd %xmm1, %xmm0
-; CHECK-SSE41-NEXT:    pandn {{.*}}(%rip), %xmm0
+; CHECK-SSE41-NEXT:    psrld $31, %xmm0
 ; CHECK-SSE41-NEXT:    retq
 ;
-; CHECK-AVX1-LABEL: test_urem_odd_allones_ne:
-; CHECK-AVX1:       # %bb.0:
-; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
-; CHECK-AVX1-NEXT:    vpmuludq {{.*}}(%rip), %xmm1, %xmm1
-; CHECK-AVX1-NEXT:    vpmuludq {{.*}}(%rip), %xmm0, %xmm2
-; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
-; CHECK-AVX1-NEXT:    vpsrld $31, %xmm1, %xmm2
-; CHECK-AVX1-NEXT:    vpsrld $2, %xmm1, %xmm1
-; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm2[4,5],xmm1[6,7]
-; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
-; CHECK-AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
-; CHECK-AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; CHECK-AVX1-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
-; CHECK-AVX1-NEXT:    vpandn {{.*}}(%rip), %xmm0, %xmm0
-; CHECK-AVX1-NEXT:    retq
-;
-; CHECK-AVX2-LABEL: test_urem_odd_allones_ne:
-; CHECK-AVX2:       # %bb.0:
-; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
-; CHECK-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [3435973837,3435973837,3435973837,3435973837]
-; CHECK-AVX2-NEXT:    vpmuludq %xmm2, %xmm1, %xmm1
-; CHECK-AVX2-NEXT:    vpmuludq {{.*}}(%rip), %xmm0, %xmm2
-; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2],xmm1[3]
-; CHECK-AVX2-NEXT:    vpsrlvd {{.*}}(%rip), %xmm1, %xmm1
-; CHECK-AVX2-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
-; CHECK-AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
-; CHECK-AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; CHECK-AVX2-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
-; CHECK-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [1,1,1,1]
-; CHECK-AVX2-NEXT:    vpandn %xmm1, %xmm0, %xmm0
-; CHECK-AVX2-NEXT:    retq
-;
-; CHECK-AVX512VL-LABEL: test_urem_odd_allones_ne:
-; CHECK-AVX512VL:       # %bb.0:
-; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
-; CHECK-AVX512VL-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [3435973837,3435973837,3435973837,3435973837]
-; CHECK-AVX512VL-NEXT:    vpmuludq %xmm2, %xmm1, %xmm1
-; CHECK-AVX512VL-NEXT:    vpmuludq {{.*}}(%rip), %xmm0, %xmm2
-; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; CHECK-AVX512VL-NEXT:    vpblendd {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2],xmm1[3]
-; CHECK-AVX512VL-NEXT:    vpsrlvd {{.*}}(%rip), %xmm1, %xmm1
-; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
-; CHECK-AVX512VL-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
-; CHECK-AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
-; CHECK-AVX512VL-NEXT:    vpandnd {{.*}}(%rip){1to4}, %xmm0, %xmm0
-; CHECK-AVX512VL-NEXT:    retq
+; CHECK-AVX-LABEL: test_urem_odd_allones_ne:
+; CHECK-AVX:       # %bb.0:
+; CHECK-AVX-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX-NEXT:    vpmaxud {{.*}}(%rip), %xmm0, %xmm1
+; CHECK-AVX-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX-NEXT:    retq
   %urem = urem <4 x i32> %X, <i32 5, i32 5, i32 4294967295, i32 5>
   %cmp = icmp ne <4 x i32> %urem, <i32 0, i32 0, i32 0, i32 0>
   %ret = zext <4 x i1> %cmp to <4 x i32>
@@ -430,17 +286,9 @@ define <4 x i32> @test_urem_even_allones_eq(<4 x i32> %X) nounwind {
 ;
 ; CHECK-AVX512VL-LABEL: test_urem_even_allones_eq:
 ; CHECK-AVX512VL:       # %bb.0:
-; CHECK-AVX512VL-NEXT:    vpsrlvd {{.*}}(%rip), %xmm0, %xmm1
-; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; CHECK-AVX512VL-NEXT:    vpbroadcastd {{.*#+}} xmm3 = [2454267027,2454267027,2454267027,2454267027]
-; CHECK-AVX512VL-NEXT:    vpmuludq %xmm3, %xmm2, %xmm2
-; CHECK-AVX512VL-NEXT:    vpmuludq {{.*}}(%rip), %xmm1, %xmm1
-; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; CHECK-AVX512VL-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; CHECK-AVX512VL-NEXT:    vpsrlvd {{.*}}(%rip), %xmm1, %xmm1
-; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
-; CHECK-AVX512VL-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
-; CHECK-AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vprorvd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
 ; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
 ; CHECK-AVX512VL-NEXT:    vpsrld $31, %xmm0, %xmm0
 ; CHECK-AVX512VL-NEXT:    retq
@@ -540,19 +388,11 @@ define <4 x i32> @test_urem_even_allones_ne(<4 x i32> %X) nounwind {
 ;
 ; CHECK-AVX512VL-LABEL: test_urem_even_allones_ne:
 ; CHECK-AVX512VL:       # %bb.0:
-; CHECK-AVX512VL-NEXT:    vpsrlvd {{.*}}(%rip), %xmm0, %xmm1
-; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; CHECK-AVX512VL-NEXT:    vpbroadcastd {{.*#+}} xmm3 = [2454267027,2454267027,2454267027,2454267027]
-; CHECK-AVX512VL-NEXT:    vpmuludq %xmm3, %xmm2, %xmm2
-; CHECK-AVX512VL-NEXT:    vpmuludq {{.*}}(%rip), %xmm1, %xmm1
-; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; CHECK-AVX512VL-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; CHECK-AVX512VL-NEXT:    vpsrlvd {{.*}}(%rip), %xmm1, %xmm1
-; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
-; CHECK-AVX512VL-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
-; CHECK-AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vprorvd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpmaxud {{.*}}(%rip), %xmm0, %xmm1
 ; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
-; CHECK-AVX512VL-NEXT:    vpandnd {{.*}}(%rip){1to4}, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpsrld $31, %xmm0, %xmm0
 ; CHECK-AVX512VL-NEXT:    retq
   %urem = urem <4 x i32> %X, <i32 14, i32 14, i32 4294967295, i32 14>
   %cmp = icmp ne <4 x i32> %urem, <i32 0, i32 0, i32 0, i32 0>
@@ -564,32 +404,30 @@ define <4 x i32> @test_urem_even_allones_ne(<4 x i32> %X) nounwind {
 define <4 x i32> @test_urem_odd_even_allones_eq(<4 x i32> %X) nounwind {
 ; CHECK-SSE2-LABEL: test_urem_odd_even_allones_eq:
 ; CHECK-SSE2:       # %bb.0:
-; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm1
-; CHECK-SSE2-NEXT:    psrld $1, %xmm1
-; CHECK-SSE2-NEXT:    movdqa %xmm1, %xmm2
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1],xmm0[3,3]
-; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [3435973837,2454267027,2147483649,1374389535]
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
-; CHECK-SSE2-NEXT:    pmuludq %xmm2, %xmm4
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,3,2,3]
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,0],xmm0[0,0]
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[2,0],xmm0[2,3]
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [3435973837,2454267027,2147483649,1374389535]
+; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm2
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
+; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm3
+; CHECK-SSE2-NEXT:    psrld $1, %xmm3
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm0[3,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
 ; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm1
 ; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
-; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
-; CHECK-SSE2-NEXT:    movdqa %xmm1, %xmm2
-; CHECK-SSE2-NEXT:    psrld $5, %xmm2
-; CHECK-SSE2-NEXT:    movdqa %xmm1, %xmm3
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm1
+; CHECK-SSE2-NEXT:    psrld $5, %xmm1
+; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm3
 ; CHECK-SSE2-NEXT:    psrld $2, %xmm3
-; CHECK-SSE2-NEXT:    psrld $31, %xmm1
-; CHECK-SSE2-NEXT:    movsd {{.*#+}} xmm1 = xmm3[0],xmm1[1]
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm2[3,3]
-; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [5,14,4294967295,100]
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,1,3,3]
+; CHECK-SSE2-NEXT:    psrld $31, %xmm2
+; CHECK-SSE2-NEXT:    movsd {{.*#+}} xmm2 = xmm3[0],xmm2[1]
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm1[3,3]
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [5,14,4294967295,100]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,3,3]
 ; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm4
 ; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[0,2,2,3]
-; CHECK-SSE2-NEXT:    pmuludq %xmm2, %xmm1
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
 ; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
 ; CHECK-SSE2-NEXT:    psubd %xmm1, %xmm0
 ; CHECK-SSE2-NEXT:    pxor %xmm1, %xmm1
@@ -601,14 +439,13 @@ define <4 x i32> @test_urem_odd_even_allones_eq(<4 x i32> %X) nounwind {
 ; CHECK-SSE41:       # %bb.0:
 ; CHECK-SSE41-NEXT:    movdqa %xmm0, %xmm1
 ; CHECK-SSE41-NEXT:    psrld $1, %xmm1
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3],xmm0[4,5,6,7]
-; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [3435973837,2454267027,2147483649,1374389535]
-; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
-; CHECK-SSE41-NEXT:    pmuludq %xmm2, %xmm4
-; CHECK-SSE41-NEXT:    pmuludq %xmm3, %xmm1
-; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm4[2,3],xmm1[4,5],xmm4[6,7]
+; CHECK-SSE41-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[3,3]
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [3435973837,2454267027,2147483649,1374389535]
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
+; CHECK-SSE41-NEXT:    pmuludq %xmm1, %xmm3
+; CHECK-SSE41-NEXT:    pmuludq %xmm0, %xmm2
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,3,3]
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5],xmm3[6,7]
 ; CHECK-SSE41-NEXT:    movdqa %xmm1, %xmm2
 ; CHECK-SSE41-NEXT:    psrld $5, %xmm2
 ; CHECK-SSE41-NEXT:    movdqa %xmm1, %xmm3
@@ -627,14 +464,13 @@ define <4 x i32> @test_urem_odd_even_allones_eq(<4 x i32> %X) nounwind {
 ; CHECK-AVX1-LABEL: test_urem_odd_even_allones_eq:
 ; CHECK-AVX1:       # %bb.0:
 ; CHECK-AVX1-NEXT:    vpsrld $1, %xmm0, %xmm1
-; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3],xmm0[4,5,6,7]
-; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; CHECK-AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [3435973837,2454267027,2147483649,1374389535]
-; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
-; CHECK-AVX1-NEXT:    vpmuludq %xmm4, %xmm2, %xmm2
+; CHECK-AVX1-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[3,3]
+; CHECK-AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [3435973837,2454267027,2147483649,1374389535]
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
 ; CHECK-AVX1-NEXT:    vpmuludq %xmm3, %xmm1, %xmm1
-; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
+; CHECK-AVX1-NEXT:    vpmuludq %xmm2, %xmm0, %xmm2
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
 ; CHECK-AVX1-NEXT:    vpsrld $5, %xmm1, %xmm2
 ; CHECK-AVX1-NEXT:    vpsrld $2, %xmm1, %xmm3
 ; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
@@ -668,18 +504,9 @@ define <4 x i32> @test_urem_odd_even_allones_eq(<4 x i32> %X) nounwind {
 ;
 ; CHECK-AVX512VL-LABEL: test_urem_odd_even_allones_eq:
 ; CHECK-AVX512VL:       # %bb.0:
-; CHECK-AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm1 = [3435973837,2454267027,2147483649,1374389535]
-; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; CHECK-AVX512VL-NEXT:    vpsrlvd {{.*}}(%rip), %xmm0, %xmm3
-; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
-; CHECK-AVX512VL-NEXT:    vpmuludq %xmm2, %xmm4, %xmm2
-; CHECK-AVX512VL-NEXT:    vpmuludq %xmm1, %xmm3, %xmm1
-; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; CHECK-AVX512VL-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; CHECK-AVX512VL-NEXT:    vpsrlvd {{.*}}(%rip), %xmm1, %xmm1
-; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
-; CHECK-AVX512VL-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
-; CHECK-AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vprorvd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
 ; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
 ; CHECK-AVX512VL-NEXT:    vpsrld $31, %xmm0, %xmm0
 ; CHECK-AVX512VL-NEXT:    retq
@@ -691,32 +518,30 @@ define <4 x i32> @test_urem_odd_even_allones_eq(<4 x i32> %X) nounwind {
 define <4 x i32> @test_urem_odd_even_allones_ne(<4 x i32> %X) nounwind {
 ; CHECK-SSE2-LABEL: test_urem_odd_even_allones_ne:
 ; CHECK-SSE2:       # %bb.0:
-; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm1
-; CHECK-SSE2-NEXT:    psrld $1, %xmm1
-; CHECK-SSE2-NEXT:    movdqa %xmm1, %xmm2
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1],xmm0[3,3]
-; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [3435973837,2454267027,2147483649,1374389535]
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
-; CHECK-SSE2-NEXT:    pmuludq %xmm2, %xmm4
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,3,2,3]
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,0],xmm0[0,0]
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[2,0],xmm0[2,3]
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [3435973837,2454267027,2147483649,1374389535]
+; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm2
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
+; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm3
+; CHECK-SSE2-NEXT:    psrld $1, %xmm3
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm0[3,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
 ; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm1
 ; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
-; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
-; CHECK-SSE2-NEXT:    movdqa %xmm1, %xmm2
-; CHECK-SSE2-NEXT:    psrld $5, %xmm2
-; CHECK-SSE2-NEXT:    movdqa %xmm1, %xmm3
-; CHECK-SSE2-NEXT:    psrld $2, %xmm3
-; CHECK-SSE2-NEXT:    psrld $31, %xmm1
-; CHECK-SSE2-NEXT:    movsd {{.*#+}} xmm1 = xmm3[0],xmm1[1]
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm2[3,3]
-; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [5,14,4294967295,100]
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,1,3,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm1
+; CHECK-SSE2-NEXT:    psrld $5, %xmm1
+; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm3
+; CHECK-SSE2-NEXT:    psrld $2, %xmm3
+; CHECK-SSE2-NEXT:    psrld $31, %xmm2
+; CHECK-SSE2-NEXT:    movsd {{.*#+}} xmm2 = xmm3[0],xmm2[1]
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm1[3,3]
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [5,14,4294967295,100]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,3,3]
 ; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm4
 ; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[0,2,2,3]
-; CHECK-SSE2-NEXT:    pmuludq %xmm2, %xmm1
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
 ; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
 ; CHECK-SSE2-NEXT:    psubd %xmm1, %xmm0
 ; CHECK-SSE2-NEXT:    pxor %xmm1, %xmm1
@@ -728,14 +553,13 @@ define <4 x i32> @test_urem_odd_even_allones_ne(<4 x i32> %X) nounwind {
 ; CHECK-SSE41:       # %bb.0:
 ; CHECK-SSE41-NEXT:    movdqa %xmm0, %xmm1
 ; CHECK-SSE41-NEXT:    psrld $1, %xmm1
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3],xmm0[4,5,6,7]
-; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [3435973837,2454267027,2147483649,1374389535]
-; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
-; CHECK-SSE41-NEXT:    pmuludq %xmm2, %xmm4
-; CHECK-SSE41-NEXT:    pmuludq %xmm3, %xmm1
-; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm4[2,3],xmm1[4,5],xmm4[6,7]
+; CHECK-SSE41-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[3,3]
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [3435973837,2454267027,2147483649,1374389535]
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
+; CHECK-SSE41-NEXT:    pmuludq %xmm1, %xmm3
+; CHECK-SSE41-NEXT:    pmuludq %xmm0, %xmm2
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,3,3]
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5],xmm3[6,7]
 ; CHECK-SSE41-NEXT:    movdqa %xmm1, %xmm2
 ; CHECK-SSE41-NEXT:    psrld $5, %xmm2
 ; CHECK-SSE41-NEXT:    movdqa %xmm1, %xmm3
@@ -754,14 +578,13 @@ define <4 x i32> @test_urem_odd_even_allones_ne(<4 x i32> %X) nounwind {
 ; CHECK-AVX1-LABEL: test_urem_odd_even_allones_ne:
 ; CHECK-AVX1:       # %bb.0:
 ; CHECK-AVX1-NEXT:    vpsrld $1, %xmm0, %xmm1
-; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3],xmm0[4,5,6,7]
-; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; CHECK-AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [3435973837,2454267027,2147483649,1374389535]
-; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
-; CHECK-AVX1-NEXT:    vpmuludq %xmm4, %xmm2, %xmm2
+; CHECK-AVX1-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[3,3]
+; CHECK-AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [3435973837,2454267027,2147483649,1374389535]
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
 ; CHECK-AVX1-NEXT:    vpmuludq %xmm3, %xmm1, %xmm1
-; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
+; CHECK-AVX1-NEXT:    vpmuludq %xmm2, %xmm0, %xmm2
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
 ; CHECK-AVX1-NEXT:    vpsrld $5, %xmm1, %xmm2
 ; CHECK-AVX1-NEXT:    vpsrld $2, %xmm1, %xmm3
 ; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
@@ -796,20 +619,11 @@ define <4 x i32> @test_urem_odd_even_allones_ne(<4 x i32> %X) nounwind {
 ;
 ; CHECK-AVX512VL-LABEL: test_urem_odd_even_allones_ne:
 ; CHECK-AVX512VL:       # %bb.0:
-; CHECK-AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm1 = [3435973837,2454267027,2147483649,1374389535]
-; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; CHECK-AVX512VL-NEXT:    vpsrlvd {{.*}}(%rip), %xmm0, %xmm3
-; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
-; CHECK-AVX512VL-NEXT:    vpmuludq %xmm2, %xmm4, %xmm2
-; CHECK-AVX512VL-NEXT:    vpmuludq %xmm1, %xmm3, %xmm1
-; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; CHECK-AVX512VL-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; CHECK-AVX512VL-NEXT:    vpsrlvd {{.*}}(%rip), %xmm1, %xmm1
-; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
-; CHECK-AVX512VL-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
-; CHECK-AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vprorvd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpmaxud {{.*}}(%rip), %xmm0, %xmm1
 ; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
-; CHECK-AVX512VL-NEXT:    vpandnd {{.*}}(%rip){1to4}, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpsrld $31, %xmm0, %xmm0
 ; CHECK-AVX512VL-NEXT:    retq
   %urem = urem <4 x i32> %X, <i32 5, i32 14, i32 4294967295, i32 100>
   %cmp = icmp ne <4 x i32> %urem, <i32 0, i32 0, i32 0, i32 0>
@@ -897,16 +711,9 @@ define <4 x i32> @test_urem_odd_poweroftwo(<4 x i32> %X) nounwind {
 ;
 ; CHECK-AVX512VL-LABEL: test_urem_odd_poweroftwo:
 ; CHECK-AVX512VL:       # %bb.0:
-; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
-; CHECK-AVX512VL-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [3435973837,3435973837,3435973837,3435973837]
-; CHECK-AVX512VL-NEXT:    vpmuludq %xmm2, %xmm1, %xmm1
-; CHECK-AVX512VL-NEXT:    vpmuludq {{.*}}(%rip), %xmm0, %xmm2
-; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; CHECK-AVX512VL-NEXT:    vpblendd {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2],xmm1[3]
-; CHECK-AVX512VL-NEXT:    vpsrlvd {{.*}}(%rip), %xmm1, %xmm1
-; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
-; CHECK-AVX512VL-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
-; CHECK-AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vprorvd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
 ; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
 ; CHECK-AVX512VL-NEXT:    vpsrld $31, %xmm0, %xmm0
 ; CHECK-AVX512VL-NEXT:    retq
@@ -1003,17 +810,9 @@ define <4 x i32> @test_urem_even_poweroftwo(<4 x i32> %X) nounwind {
 ;
 ; CHECK-AVX512VL-LABEL: test_urem_even_poweroftwo:
 ; CHECK-AVX512VL:       # %bb.0:
-; CHECK-AVX512VL-NEXT:    vpsrlvd {{.*}}(%rip), %xmm0, %xmm1
-; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; CHECK-AVX512VL-NEXT:    vpbroadcastd {{.*#+}} xmm3 = [2454267027,2454267027,2454267027,2454267027]
-; CHECK-AVX512VL-NEXT:    vpmuludq %xmm3, %xmm2, %xmm2
-; CHECK-AVX512VL-NEXT:    vpmuludq {{.*}}(%rip), %xmm1, %xmm1
-; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; CHECK-AVX512VL-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; CHECK-AVX512VL-NEXT:    vpsrlvd {{.*}}(%rip), %xmm1, %xmm1
-; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
-; CHECK-AVX512VL-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
-; CHECK-AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vprorvd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
 ; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
 ; CHECK-AVX512VL-NEXT:    vpsrld $31, %xmm0, %xmm0
 ; CHECK-AVX512VL-NEXT:    retq
@@ -1027,31 +826,29 @@ define <4 x i32> @test_urem_even_poweroftwo(<4 x i32> %X) nounwind {
 define <4 x i32> @test_urem_odd_even_poweroftwo(<4 x i32> %X) nounwind {
 ; CHECK-SSE2-LABEL: test_urem_odd_even_poweroftwo:
 ; CHECK-SSE2:       # %bb.0:
-; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm1
-; CHECK-SSE2-NEXT:    psrld $1, %xmm1
-; CHECK-SSE2-NEXT:    movdqa %xmm1, %xmm2
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1],xmm0[3,3]
-; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [3435973837,2454267027,268435456,1374389535]
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
-; CHECK-SSE2-NEXT:    pmuludq %xmm2, %xmm4
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,3,2,3]
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,0],xmm0[0,0]
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[2,0],xmm0[2,3]
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [3435973837,2454267027,268435456,1374389535]
+; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm2
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
+; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm3
+; CHECK-SSE2-NEXT:    psrld $1, %xmm3
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm0[3,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
 ; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm1
 ; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
-; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
-; CHECK-SSE2-NEXT:    movdqa %xmm1, %xmm2
-; CHECK-SSE2-NEXT:    psrld $5, %xmm2
-; CHECK-SSE2-NEXT:    movdqa %xmm1, %xmm3
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm1
+; CHECK-SSE2-NEXT:    psrld $5, %xmm1
+; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm3
 ; CHECK-SSE2-NEXT:    psrld $2, %xmm3
-; CHECK-SSE2-NEXT:    movsd {{.*#+}} xmm1 = xmm3[0],xmm1[1]
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm2[3,3]
-; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [5,14,16,100]
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,1,3,3]
+; CHECK-SSE2-NEXT:    movsd {{.*#+}} xmm2 = xmm3[0],xmm2[1]
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm1[3,3]
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [5,14,16,100]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,3,3]
 ; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm4
 ; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[0,2,2,3]
-; CHECK-SSE2-NEXT:    pmuludq %xmm2, %xmm1
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
 ; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
 ; CHECK-SSE2-NEXT:    psubd %xmm1, %xmm0
 ; CHECK-SSE2-NEXT:    pxor %xmm1, %xmm1
@@ -1063,14 +860,13 @@ define <4 x i32> @test_urem_odd_even_poweroftwo(<4 x i32> %X) nounwind {
 ; CHECK-SSE41:       # %bb.0:
 ; CHECK-SSE41-NEXT:    movdqa %xmm0, %xmm1
 ; CHECK-SSE41-NEXT:    psrld $1, %xmm1
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3],xmm0[4,5,6,7]
-; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [3435973837,2454267027,268435456,1374389535]
-; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
-; CHECK-SSE41-NEXT:    pmuludq %xmm2, %xmm4
-; CHECK-SSE41-NEXT:    pmuludq %xmm3, %xmm1
-; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm4[2,3],xmm1[4,5],xmm4[6,7]
+; CHECK-SSE41-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[3,3]
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [3435973837,2454267027,268435456,1374389535]
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
+; CHECK-SSE41-NEXT:    pmuludq %xmm1, %xmm3
+; CHECK-SSE41-NEXT:    pmuludq %xmm0, %xmm2
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,3,3]
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5],xmm3[6,7]
 ; CHECK-SSE41-NEXT:    movdqa %xmm1, %xmm2
 ; CHECK-SSE41-NEXT:    psrld $5, %xmm2
 ; CHECK-SSE41-NEXT:    movdqa %xmm1, %xmm3
@@ -1088,14 +884,13 @@ define <4 x i32> @test_urem_odd_even_poweroftwo(<4 x i32> %X) nounwind {
 ; CHECK-AVX1-LABEL: test_urem_odd_even_poweroftwo:
 ; CHECK-AVX1:       # %bb.0:
 ; CHECK-AVX1-NEXT:    vpsrld $1, %xmm0, %xmm1
-; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3],xmm0[4,5,6,7]
-; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; CHECK-AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [3435973837,2454267027,268435456,1374389535]
-; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
-; CHECK-AVX1-NEXT:    vpmuludq %xmm4, %xmm2, %xmm2
+; CHECK-AVX1-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[3,3]
+; CHECK-AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [3435973837,2454267027,268435456,1374389535]
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
 ; CHECK-AVX1-NEXT:    vpmuludq %xmm3, %xmm1, %xmm1
-; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
+; CHECK-AVX1-NEXT:    vpmuludq %xmm2, %xmm0, %xmm2
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
 ; CHECK-AVX1-NEXT:    vpsrld $5, %xmm1, %xmm2
 ; CHECK-AVX1-NEXT:    vpsrld $2, %xmm1, %xmm3
 ; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
@@ -1128,18 +923,9 @@ define <4 x i32> @test_urem_odd_even_poweroftwo(<4 x i32> %X) nounwind {
 ;
 ; CHECK-AVX512VL-LABEL: test_urem_odd_even_poweroftwo:
 ; CHECK-AVX512VL:       # %bb.0:
-; CHECK-AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm1 = [3435973837,2454267027,268435456,1374389535]
-; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; CHECK-AVX512VL-NEXT:    vpsrlvd {{.*}}(%rip), %xmm0, %xmm3
-; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
-; CHECK-AVX512VL-NEXT:    vpmuludq %xmm2, %xmm4, %xmm2
-; CHECK-AVX512VL-NEXT:    vpmuludq %xmm1, %xmm3, %xmm1
-; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; CHECK-AVX512VL-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; CHECK-AVX512VL-NEXT:    vpsrlvd {{.*}}(%rip), %xmm1, %xmm1
-; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
-; CHECK-AVX512VL-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
-; CHECK-AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vprorvd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
 ; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
 ; CHECK-AVX512VL-NEXT:    vpsrld $31, %xmm0, %xmm0
 ; CHECK-AVX512VL-NEXT:    retq
@@ -1155,100 +941,48 @@ define <4 x i32> @test_urem_odd_even_poweroftwo(<4 x i32> %X) nounwind {
 define <4 x i32> @test_urem_odd_one(<4 x i32> %X) nounwind {
 ; CHECK-SSE2-LABEL: test_urem_odd_one:
 ; CHECK-SSE2:       # %bb.0:
-; CHECK-SSE2-NEXT:    movl $-858993459, %eax # imm = 0xCCCCCCCD
-; CHECK-SSE2-NEXT:    movd %eax, %xmm1
-; CHECK-SSE2-NEXT:    pmuludq %xmm0, %xmm1
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [3435973837,3435973837,3435973837,3435973837]
 ; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
-; CHECK-SSE2-NEXT:    pmuludq {{.*}}(%rip), %xmm2
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
-; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
-; CHECK-SSE2-NEXT:    psrld $2, %xmm1
-; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm2
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[2,0],xmm1[3,0]
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0,2]
-; CHECK-SSE2-NEXT:    pmuludq {{.*}}(%rip), %xmm1
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; CHECK-SSE2-NEXT:    pmuludq {{.*}}(%rip), %xmm3
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,2,2,3]
-; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
-; CHECK-SSE2-NEXT:    psubd %xmm1, %xmm0
-; CHECK-SSE2-NEXT:    pxor %xmm1, %xmm1
-; CHECK-SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
-; CHECK-SSE2-NEXT:    psrld $31, %xmm0
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm0
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; CHECK-SSE2-NEXT:    pxor {{.*}}(%rip), %xmm0
+; CHECK-SSE2-NEXT:    pcmpgtd {{.*}}(%rip), %xmm0
+; CHECK-SSE2-NEXT:    pandn {{.*}}(%rip), %xmm0
 ; CHECK-SSE2-NEXT:    retq
 ;
 ; CHECK-SSE41-LABEL: test_urem_odd_one:
 ; CHECK-SSE41:       # %bb.0:
-; CHECK-SSE41-NEXT:    movl $-858993459, %eax # imm = 0xCCCCCCCD
-; CHECK-SSE41-NEXT:    movd %eax, %xmm1
-; CHECK-SSE41-NEXT:    pmuludq %xmm0, %xmm1
-; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
-; CHECK-SSE41-NEXT:    pmuludq {{.*}}(%rip), %xmm2
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
-; CHECK-SSE41-NEXT:    psrld $2, %xmm2
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm0[4,5],xmm2[6,7]
-; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm2
-; CHECK-SSE41-NEXT:    psubd %xmm2, %xmm0
-; CHECK-SSE41-NEXT:    pxor %xmm1, %xmm1
+; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm0
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [858993459,858993459,4294967295,858993459]
+; CHECK-SSE41-NEXT:    pminud %xmm0, %xmm1
 ; CHECK-SSE41-NEXT:    pcmpeqd %xmm1, %xmm0
 ; CHECK-SSE41-NEXT:    psrld $31, %xmm0
 ; CHECK-SSE41-NEXT:    retq
 ;
 ; CHECK-AVX1-LABEL: test_urem_odd_one:
 ; CHECK-AVX1:       # %bb.0:
-; CHECK-AVX1-NEXT:    movl $-858993459, %eax # imm = 0xCCCCCCCD
-; CHECK-AVX1-NEXT:    vmovd %eax, %xmm1
-; CHECK-AVX1-NEXT:    vpmuludq %xmm1, %xmm0, %xmm1
-; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
-; CHECK-AVX1-NEXT:    vpmuludq {{.*}}(%rip), %xmm2, %xmm2
-; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
-; CHECK-AVX1-NEXT:    vpsrld $2, %xmm1, %xmm1
-; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm0[4,5],xmm1[6,7]
-; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
-; CHECK-AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
-; CHECK-AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
 ; CHECK-AVX1-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
 ; CHECK-AVX1-NEXT:    vpsrld $31, %xmm0, %xmm0
 ; CHECK-AVX1-NEXT:    retq
 ;
 ; CHECK-AVX2-LABEL: test_urem_odd_one:
 ; CHECK-AVX2:       # %bb.0:
-; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
-; CHECK-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [3435973837,3435973837,3435973837,3435973837]
-; CHECK-AVX2-NEXT:    vpmuludq %xmm2, %xmm1, %xmm1
-; CHECK-AVX2-NEXT:    movl $-858993459, %eax # imm = 0xCCCCCCCD
-; CHECK-AVX2-NEXT:    vmovd %eax, %xmm2
-; CHECK-AVX2-NEXT:    vpmuludq %xmm2, %xmm0, %xmm2
-; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2],xmm1[3]
-; CHECK-AVX2-NEXT:    vpsrld $2, %xmm1, %xmm1
-; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0,1],xmm0[2],xmm1[3]
-; CHECK-AVX2-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
-; CHECK-AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
-; CHECK-AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [3435973837,3435973837,3435973837,3435973837]
+; CHECK-AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
 ; CHECK-AVX2-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
 ; CHECK-AVX2-NEXT:    vpsrld $31, %xmm0, %xmm0
 ; CHECK-AVX2-NEXT:    retq
 ;
 ; CHECK-AVX512VL-LABEL: test_urem_odd_one:
 ; CHECK-AVX512VL:       # %bb.0:
-; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
-; CHECK-AVX512VL-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [3435973837,3435973837,3435973837,3435973837]
-; CHECK-AVX512VL-NEXT:    vpmuludq %xmm2, %xmm1, %xmm1
-; CHECK-AVX512VL-NEXT:    movl $-858993459, %eax # imm = 0xCCCCCCCD
-; CHECK-AVX512VL-NEXT:    vmovd %eax, %xmm2
-; CHECK-AVX512VL-NEXT:    vpmuludq %xmm2, %xmm0, %xmm2
-; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; CHECK-AVX512VL-NEXT:    vpblendd {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2],xmm1[3]
-; CHECK-AVX512VL-NEXT:    vpsrld $2, %xmm1, %xmm1
-; CHECK-AVX512VL-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0,1],xmm0[2],xmm1[3]
-; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
-; CHECK-AVX512VL-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
-; CHECK-AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip){1to4}, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
 ; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
 ; CHECK-AVX512VL-NEXT:    vpsrld $31, %xmm0, %xmm0
 ; CHECK-AVX512VL-NEXT:    retq
@@ -1354,20 +1088,9 @@ define <4 x i32> @test_urem_even_one(<4 x i32> %X) nounwind {
 ;
 ; CHECK-AVX512VL-LABEL: test_urem_even_one:
 ; CHECK-AVX512VL:       # %bb.0:
-; CHECK-AVX512VL-NEXT:    vpsrlvd {{.*}}(%rip), %xmm0, %xmm1
-; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; CHECK-AVX512VL-NEXT:    vpbroadcastd {{.*#+}} xmm3 = [2454267027,2454267027,2454267027,2454267027]
-; CHECK-AVX512VL-NEXT:    vpmuludq %xmm3, %xmm2, %xmm2
-; CHECK-AVX512VL-NEXT:    movl $-1840700269, %eax # imm = 0x92492493
-; CHECK-AVX512VL-NEXT:    vmovd %eax, %xmm3
-; CHECK-AVX512VL-NEXT:    vpmuludq %xmm3, %xmm1, %xmm1
-; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; CHECK-AVX512VL-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; CHECK-AVX512VL-NEXT:    vpsrld $2, %xmm1, %xmm1
-; CHECK-AVX512VL-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0,1],xmm0[2],xmm1[3]
-; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
-; CHECK-AVX512VL-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
-; CHECK-AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip){1to4}, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vprord $1, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
 ; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
 ; CHECK-AVX512VL-NEXT:    vpsrld $31, %xmm0, %xmm0
 ; CHECK-AVX512VL-NEXT:    retq
@@ -1381,33 +1104,333 @@ define <4 x i32> @test_urem_even_one(<4 x i32> %X) nounwind {
 define <4 x i32> @test_urem_odd_even_one(<4 x i32> %X) nounwind {
 ; CHECK-SSE2-LABEL: test_urem_odd_even_one:
 ; CHECK-SSE2:       # %bb.0:
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [3435973837,2454267027,0,1374389535]
+; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm2
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
+; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm3
+; CHECK-SSE2-NEXT:    psrld $1, %xmm3
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm0[3,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm1
+; CHECK-SSE2-NEXT:    psrld $5, %xmm1
+; CHECK-SSE2-NEXT:    psrld $2, %xmm2
+; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm3
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm1[3,3]
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [5,14,1,100]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm4
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[0,2,2,3]
+; CHECK-SSE2-NEXT:    movaps %xmm0, %xmm4
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[2,0],xmm2[3,0]
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,1],xmm4[0,2]
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
+; CHECK-SSE2-NEXT:    psubd %xmm1, %xmm0
+; CHECK-SSE2-NEXT:    pxor %xmm1, %xmm1
+; CHECK-SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-SSE2-NEXT:    psrld $31, %xmm0
+; CHECK-SSE2-NEXT:    retq
+;
+; CHECK-SSE41-LABEL: test_urem_odd_even_one:
+; CHECK-SSE41:       # %bb.0:
+; CHECK-SSE41-NEXT:    movdqa %xmm0, %xmm1
+; CHECK-SSE41-NEXT:    psrld $1, %xmm1
+; CHECK-SSE41-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[3,3]
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [3435973837,2454267027,0,1374389535]
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
+; CHECK-SSE41-NEXT:    pmuludq %xmm1, %xmm3
+; CHECK-SSE41-NEXT:    pmuludq %xmm0, %xmm2
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,3,3]
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5],xmm3[6,7]
+; CHECK-SSE41-NEXT:    movdqa %xmm1, %xmm2
+; CHECK-SSE41-NEXT:    psrld $5, %xmm2
+; CHECK-SSE41-NEXT:    psrld $2, %xmm1
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm2[4,5,6,7]
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm0[4,5],xmm1[6,7]
+; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm1
+; CHECK-SSE41-NEXT:    psubd %xmm1, %xmm0
+; CHECK-SSE41-NEXT:    pxor %xmm1, %xmm1
+; CHECK-SSE41-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-SSE41-NEXT:    psrld $31, %xmm0
+; CHECK-SSE41-NEXT:    retq
+;
+; CHECK-AVX1-LABEL: test_urem_odd_even_one:
+; CHECK-AVX1:       # %bb.0:
+; CHECK-AVX1-NEXT:    vpsrld $1, %xmm0, %xmm1
+; CHECK-AVX1-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[3,3]
+; CHECK-AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [3435973837,2454267027,0,1374389535]
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpmuludq %xmm3, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpmuludq %xmm2, %xmm0, %xmm2
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
+; CHECK-AVX1-NEXT:    vpsrld $5, %xmm1, %xmm2
+; CHECK-AVX1-NEXT:    vpsrld $2, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm2[4,5,6,7]
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm0[4,5],xmm1[6,7]
+; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    retq
+;
+; CHECK-AVX2-LABEL: test_urem_odd_even_one:
+; CHECK-AVX2:       # %bb.0:
+; CHECK-AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [3435973837,2454267027,0,1374389535]
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpsrlvd {{.*}}(%rip), %xmm0, %xmm3
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpmuludq %xmm2, %xmm4, %xmm2
+; CHECK-AVX2-NEXT:    vpmuludq %xmm1, %xmm3, %xmm1
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
+; CHECK-AVX2-NEXT:    vpsrlvd {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0,1],xmm0[2],xmm1[3]
+; CHECK-AVX2-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    retq
+;
+; CHECK-AVX512VL-LABEL: test_urem_odd_even_one:
+; CHECK-AVX512VL:       # %bb.0:
+; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vprorvd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
+; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    retq
+  %urem = urem <4 x i32> %X, <i32 5, i32 14, i32 1, i32 100>
+  %cmp = icmp eq <4 x i32> %urem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+;------------------------------------------------------------------------------;
+
+; One INT_MIN divisor in odd divisor
+define <4 x i32> @test_urem_odd_INT_MIN(<4 x i32> %X) nounwind {
+; CHECK-SSE2-LABEL: test_urem_odd_INT_MIN:
+; CHECK-SSE2:       # %bb.0:
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = <3435973837,u,2,u>
+; CHECK-SSE2-NEXT:    pmuludq %xmm0, %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq {{.*}}(%rip), %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; CHECK-SSE2-NEXT:    movdqa %xmm1, %xmm2
+; CHECK-SSE2-NEXT:    psrld $2, %xmm2
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[2,0],xmm2[3,0]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,1],xmm1[0,2]
+; CHECK-SSE2-NEXT:    pmuludq {{.*}}(%rip), %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    pmuludq {{.*}}(%rip), %xmm3
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; CHECK-SSE2-NEXT:    psubd %xmm1, %xmm0
+; CHECK-SSE2-NEXT:    pxor %xmm1, %xmm1
+; CHECK-SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-SSE2-NEXT:    psrld $31, %xmm0
+; CHECK-SSE2-NEXT:    retq
+;
+; CHECK-SSE41-LABEL: test_urem_odd_INT_MIN:
+; CHECK-SSE41:       # %bb.0:
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; CHECK-SSE41-NEXT:    pmuludq {{.*}}(%rip), %xmm1
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm2 = <3435973837,u,2,u>
+; CHECK-SSE41-NEXT:    pmuludq %xmm0, %xmm2
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
+; CHECK-SSE41-NEXT:    psrld $2, %xmm1
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm2[4,5],xmm1[6,7]
+; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm1
+; CHECK-SSE41-NEXT:    psubd %xmm1, %xmm0
+; CHECK-SSE41-NEXT:    pxor %xmm1, %xmm1
+; CHECK-SSE41-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-SSE41-NEXT:    psrld $31, %xmm0
+; CHECK-SSE41-NEXT:    retq
+;
+; CHECK-AVX1-LABEL: test_urem_odd_INT_MIN:
+; CHECK-AVX1:       # %bb.0:
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpmuludq {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpmuludq {{.*}}(%rip), %xmm0, %xmm2
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
+; CHECK-AVX1-NEXT:    vpsrld $2, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm2[4,5],xmm1[6,7]
+; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    retq
+;
+; CHECK-AVX2-LABEL: test_urem_odd_INT_MIN:
+; CHECK-AVX2:       # %bb.0:
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [3435973837,3435973837,3435973837,3435973837]
+; CHECK-AVX2-NEXT:    vpmuludq %xmm2, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpmuludq {{.*}}(%rip), %xmm0, %xmm2
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2],xmm1[3]
+; CHECK-AVX2-NEXT:    vpsrlvd {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    retq
+;
+; CHECK-AVX512VL-LABEL: test_urem_odd_INT_MIN:
+; CHECK-AVX512VL:       # %bb.0:
+; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vprorvd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
+; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    retq
+  %urem = urem <4 x i32> %X, <i32 5, i32 5, i32 2147483648, i32 5>
+  %cmp = icmp eq <4 x i32> %urem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; One INT_MIN divisor in even divisor
+define <4 x i32> @test_urem_even_INT_MIN(<4 x i32> %X) nounwind {
+; CHECK-SSE2-LABEL: test_urem_even_INT_MIN:
+; CHECK-SSE2:       # %bb.0:
 ; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; CHECK-SSE2-NEXT:    psrld $1, %xmm1
+; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm2
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[2,0],xmm1[3,0]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0,2]
+; CHECK-SSE2-NEXT:    pmuludq {{.*}}(%rip), %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
+; CHECK-SSE2-NEXT:    pmuludq {{.*}}(%rip), %xmm3
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,3,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
 ; CHECK-SSE2-NEXT:    movdqa %xmm1, %xmm2
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1],xmm0[3,3]
-; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [3435973837,2454267027,0,1374389535]
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
-; CHECK-SSE2-NEXT:    pmuludq %xmm2, %xmm4
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,3,2,3]
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,0],xmm0[0,0]
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[2,0],xmm0[2,3]
+; CHECK-SSE2-NEXT:    psrld $2, %xmm2
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[2,0],xmm2[3,0]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,1],xmm1[0,2]
+; CHECK-SSE2-NEXT:    pmuludq {{.*}}(%rip), %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    pmuludq {{.*}}(%rip), %xmm3
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; CHECK-SSE2-NEXT:    psubd %xmm1, %xmm0
+; CHECK-SSE2-NEXT:    pxor %xmm1, %xmm1
+; CHECK-SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-SSE2-NEXT:    psrld $31, %xmm0
+; CHECK-SSE2-NEXT:    retq
+;
+; CHECK-SSE41-LABEL: test_urem_even_INT_MIN:
+; CHECK-SSE41:       # %bb.0:
+; CHECK-SSE41-NEXT:    movdqa %xmm0, %xmm1
+; CHECK-SSE41-NEXT:    psrld $1, %xmm1
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm0[4,5],xmm1[6,7]
+; CHECK-SSE41-NEXT:    pmuludq {{.*}}(%rip), %xmm1
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-SSE41-NEXT:    pmuludq {{.*}}(%rip), %xmm2
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
+; CHECK-SSE41-NEXT:    psrld $2, %xmm2
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm1[4,5],xmm2[6,7]
+; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm2
+; CHECK-SSE41-NEXT:    psubd %xmm2, %xmm0
+; CHECK-SSE41-NEXT:    pxor %xmm1, %xmm1
+; CHECK-SSE41-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-SSE41-NEXT:    psrld $31, %xmm0
+; CHECK-SSE41-NEXT:    retq
+;
+; CHECK-AVX1-LABEL: test_urem_even_INT_MIN:
+; CHECK-AVX1:       # %bb.0:
+; CHECK-AVX1-NEXT:    vpsrld $1, %xmm0, %xmm1
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm1[0,1,2,3],xmm0[4,5],xmm1[6,7]
+; CHECK-AVX1-NEXT:    vpmuludq {{.*}}(%rip), %xmm2, %xmm2
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpmuludq {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
+; CHECK-AVX1-NEXT:    vpsrld $2, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm2[4,5],xmm1[6,7]
+; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    retq
+;
+; CHECK-AVX2-LABEL: test_urem_even_INT_MIN:
+; CHECK-AVX2:       # %bb.0:
+; CHECK-AVX2-NEXT:    vpsrlvd {{.*}}(%rip), %xmm0, %xmm1
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm3 = [2454267027,2454267027,2454267027,2454267027]
+; CHECK-AVX2-NEXT:    vpmuludq %xmm3, %xmm2, %xmm2
+; CHECK-AVX2-NEXT:    vpmuludq {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
+; CHECK-AVX2-NEXT:    vpsrlvd {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    retq
+;
+; CHECK-AVX512VL-LABEL: test_urem_even_INT_MIN:
+; CHECK-AVX512VL:       # %bb.0:
+; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vprorvd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
+; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    retq
+  %urem = urem <4 x i32> %X, <i32 14, i32 14, i32 2147483648, i32 14>
+  %cmp = icmp eq <4 x i32> %urem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; One INT_MIN divisor in odd+even divisor
+define <4 x i32> @test_urem_odd_even_INT_MIN(<4 x i32> %X) nounwind {
+; CHECK-SSE2-LABEL: test_urem_odd_even_INT_MIN:
+; CHECK-SSE2:       # %bb.0:
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [3435973837,2454267027,2,1374389535]
+; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm2
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
+; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm3
+; CHECK-SSE2-NEXT:    psrld $1, %xmm3
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm0[3,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
 ; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm1
 ; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
-; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
-; CHECK-SSE2-NEXT:    movdqa %xmm1, %xmm2
-; CHECK-SSE2-NEXT:    psrld $5, %xmm2
-; CHECK-SSE2-NEXT:    psrld $2, %xmm1
-; CHECK-SSE2-NEXT:    movdqa %xmm1, %xmm3
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm2[3,3]
-; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [5,14,1,100]
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[1,1,3,3]
-; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm5
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm5[0,2,2,3]
-; CHECK-SSE2-NEXT:    movaps %xmm0, %xmm5
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm5 = xmm5[2,0],xmm2[3,0]
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,1],xmm5[0,2]
-; CHECK-SSE2-NEXT:    pmuludq %xmm4, %xmm1
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm1
+; CHECK-SSE2-NEXT:    psrld $5, %xmm1
+; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm3
+; CHECK-SSE2-NEXT:    psrld $2, %xmm3
+; CHECK-SSE2-NEXT:    movsd {{.*#+}} xmm2 = xmm3[0],xmm2[1]
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm1[3,3]
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [5,14,2147483648,100]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm4
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[0,2,2,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
 ; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
 ; CHECK-SSE2-NEXT:    psubd %xmm1, %xmm0
 ; CHECK-SSE2-NEXT:    pxor %xmm1, %xmm1
@@ -1415,45 +1438,46 @@ define <4 x i32> @test_urem_odd_even_one(<4 x i32> %X) nounwind {
 ; CHECK-SSE2-NEXT:    psrld $31, %xmm0
 ; CHECK-SSE2-NEXT:    retq
 ;
-; CHECK-SSE41-LABEL: test_urem_odd_even_one:
+; CHECK-SSE41-LABEL: test_urem_odd_even_INT_MIN:
 ; CHECK-SSE41:       # %bb.0:
 ; CHECK-SSE41-NEXT:    movdqa %xmm0, %xmm1
 ; CHECK-SSE41-NEXT:    psrld $1, %xmm1
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3],xmm0[4,5,6,7]
-; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [3435973837,2454267027,0,1374389535]
-; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
-; CHECK-SSE41-NEXT:    pmuludq %xmm2, %xmm4
-; CHECK-SSE41-NEXT:    pmuludq %xmm3, %xmm1
-; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm4[2,3],xmm1[4,5],xmm4[6,7]
+; CHECK-SSE41-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[3,3]
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [3435973837,2454267027,2,1374389535]
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
+; CHECK-SSE41-NEXT:    pmuludq %xmm1, %xmm3
+; CHECK-SSE41-NEXT:    pmuludq %xmm0, %xmm2
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,3,3]
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5],xmm3[6,7]
 ; CHECK-SSE41-NEXT:    movdqa %xmm1, %xmm2
 ; CHECK-SSE41-NEXT:    psrld $5, %xmm2
-; CHECK-SSE41-NEXT:    psrld $2, %xmm1
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm2[4,5,6,7]
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm0[4,5],xmm1[6,7]
-; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm1
-; CHECK-SSE41-NEXT:    psubd %xmm1, %xmm0
+; CHECK-SSE41-NEXT:    movdqa %xmm1, %xmm3
+; CHECK-SSE41-NEXT:    psrld $2, %xmm3
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm1[4,5,6,7]
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm3 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
+; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm3
+; CHECK-SSE41-NEXT:    psubd %xmm3, %xmm0
 ; CHECK-SSE41-NEXT:    pxor %xmm1, %xmm1
 ; CHECK-SSE41-NEXT:    pcmpeqd %xmm1, %xmm0
 ; CHECK-SSE41-NEXT:    psrld $31, %xmm0
 ; CHECK-SSE41-NEXT:    retq
 ;
-; CHECK-AVX1-LABEL: test_urem_odd_even_one:
+; CHECK-AVX1-LABEL: test_urem_odd_even_INT_MIN:
 ; CHECK-AVX1:       # %bb.0:
 ; CHECK-AVX1-NEXT:    vpsrld $1, %xmm0, %xmm1
-; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3],xmm0[4,5,6,7]
-; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; CHECK-AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [3435973837,2454267027,0,1374389535]
-; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
-; CHECK-AVX1-NEXT:    vpmuludq %xmm4, %xmm2, %xmm2
+; CHECK-AVX1-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[3,3]
+; CHECK-AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [3435973837,2454267027,2,1374389535]
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
 ; CHECK-AVX1-NEXT:    vpmuludq %xmm3, %xmm1, %xmm1
-; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
+; CHECK-AVX1-NEXT:    vpmuludq %xmm2, %xmm0, %xmm2
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
 ; CHECK-AVX1-NEXT:    vpsrld $5, %xmm1, %xmm2
-; CHECK-AVX1-NEXT:    vpsrld $2, %xmm1, %xmm1
-; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm2[4,5,6,7]
-; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm0[4,5],xmm1[6,7]
+; CHECK-AVX1-NEXT:    vpsrld $2, %xmm1, %xmm3
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm3[0,1,2,3],xmm1[4,5,6,7]
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
 ; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
 ; CHECK-AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
 ; CHECK-AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
@@ -1461,9 +1485,9 @@ define <4 x i32> @test_urem_odd_even_one(<4 x i32> %X) nounwind {
 ; CHECK-AVX1-NEXT:    vpsrld $31, %xmm0, %xmm0
 ; CHECK-AVX1-NEXT:    retq
 ;
-; CHECK-AVX2-LABEL: test_urem_odd_even_one:
+; CHECK-AVX2-LABEL: test_urem_odd_even_INT_MIN:
 ; CHECK-AVX2:       # %bb.0:
-; CHECK-AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [3435973837,2454267027,0,1374389535]
+; CHECK-AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [3435973837,2454267027,2,1374389535]
 ; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
 ; CHECK-AVX2-NEXT:    vpsrlvd {{.*}}(%rip), %xmm0, %xmm3
 ; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
@@ -1472,7 +1496,6 @@ define <4 x i32> @test_urem_odd_even_one(<4 x i32> %X) nounwind {
 ; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
 ; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
 ; CHECK-AVX2-NEXT:    vpsrlvd {{.*}}(%rip), %xmm1, %xmm1
-; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0,1],xmm0[2],xmm1[3]
 ; CHECK-AVX2-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
 ; CHECK-AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
 ; CHECK-AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
@@ -1480,25 +1503,15 @@ define <4 x i32> @test_urem_odd_even_one(<4 x i32> %X) nounwind {
 ; CHECK-AVX2-NEXT:    vpsrld $31, %xmm0, %xmm0
 ; CHECK-AVX2-NEXT:    retq
 ;
-; CHECK-AVX512VL-LABEL: test_urem_odd_even_one:
+; CHECK-AVX512VL-LABEL: test_urem_odd_even_INT_MIN:
 ; CHECK-AVX512VL:       # %bb.0:
-; CHECK-AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm1 = [3435973837,2454267027,0,1374389535]
-; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; CHECK-AVX512VL-NEXT:    vpsrlvd {{.*}}(%rip), %xmm0, %xmm3
-; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
-; CHECK-AVX512VL-NEXT:    vpmuludq %xmm2, %xmm4, %xmm2
-; CHECK-AVX512VL-NEXT:    vpmuludq %xmm1, %xmm3, %xmm1
-; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; CHECK-AVX512VL-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; CHECK-AVX512VL-NEXT:    vpsrlvd {{.*}}(%rip), %xmm1, %xmm1
-; CHECK-AVX512VL-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0,1],xmm0[2],xmm1[3]
-; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
-; CHECK-AVX512VL-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
-; CHECK-AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vprorvd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
 ; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
 ; CHECK-AVX512VL-NEXT:    vpsrld $31, %xmm0, %xmm0
 ; CHECK-AVX512VL-NEXT:    retq
-  %urem = urem <4 x i32> %X, <i32 5, i32 14, i32 1, i32 100>
+  %urem = urem <4 x i32> %X, <i32 5, i32 14, i32 2147483648, i32 100>
   %cmp = icmp eq <4 x i32> %urem, <i32 0, i32 0, i32 0, i32 0>
   %ret = zext <4 x i1> %cmp to <4 x i32>
   ret <4 x i32> %ret
@@ -1601,17 +1614,9 @@ define <4 x i32> @test_urem_odd_allones_and_poweroftwo(<4 x i32> %X) nounwind {
 ;
 ; CHECK-AVX512VL-LABEL: test_urem_odd_allones_and_poweroftwo:
 ; CHECK-AVX512VL:       # %bb.0:
-; CHECK-AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm1 = [3435973837,2147483649,268435456,3435973837]
-; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
-; CHECK-AVX512VL-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2
-; CHECK-AVX512VL-NEXT:    vpmuludq %xmm1, %xmm0, %xmm1
-; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; CHECK-AVX512VL-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; CHECK-AVX512VL-NEXT:    vpsrlvd {{.*}}(%rip), %xmm1, %xmm1
-; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
-; CHECK-AVX512VL-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
-; CHECK-AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vprorvd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
 ; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
 ; CHECK-AVX512VL-NEXT:    vpsrld $31, %xmm0, %xmm0
 ; CHECK-AVX512VL-NEXT:    retq
@@ -1726,18 +1731,9 @@ define <4 x i32> @test_urem_even_allones_and_poweroftwo(<4 x i32> %X) nounwind {
 ;
 ; CHECK-AVX512VL-LABEL: test_urem_even_allones_and_poweroftwo:
 ; CHECK-AVX512VL:       # %bb.0:
-; CHECK-AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm1 = [2454267027,2147483649,268435456,2454267027]
-; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; CHECK-AVX512VL-NEXT:    vpsrlvd {{.*}}(%rip), %xmm0, %xmm3
-; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
-; CHECK-AVX512VL-NEXT:    vpmuludq %xmm2, %xmm4, %xmm2
-; CHECK-AVX512VL-NEXT:    vpmuludq %xmm1, %xmm3, %xmm1
-; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; CHECK-AVX512VL-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; CHECK-AVX512VL-NEXT:    vpsrlvd {{.*}}(%rip), %xmm1, %xmm1
-; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
-; CHECK-AVX512VL-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
-; CHECK-AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vprorvd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
 ; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
 ; CHECK-AVX512VL-NEXT:    vpsrld $31, %xmm0, %xmm0
 ; CHECK-AVX512VL-NEXT:    retq
@@ -1845,17 +1841,9 @@ define <4 x i32> @test_urem_odd_even_allones_and_poweroftwo(<4 x i32> %X) nounwi
 ;
 ; CHECK-AVX512VL-LABEL: test_urem_odd_even_allones_and_poweroftwo:
 ; CHECK-AVX512VL:       # %bb.0:
-; CHECK-AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm1 = [3435973837,2147483649,268435456,1374389535]
-; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
-; CHECK-AVX512VL-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2
-; CHECK-AVX512VL-NEXT:    vpmuludq %xmm1, %xmm0, %xmm1
-; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; CHECK-AVX512VL-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; CHECK-AVX512VL-NEXT:    vpsrlvd {{.*}}(%rip), %xmm1, %xmm1
-; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
-; CHECK-AVX512VL-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
-; CHECK-AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vprorvd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
 ; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
 ; CHECK-AVX512VL-NEXT:    vpsrld $31, %xmm0, %xmm0
 ; CHECK-AVX512VL-NEXT:    retq
@@ -1871,113 +1859,35 @@ define <4 x i32> @test_urem_odd_even_allones_and_poweroftwo(<4 x i32> %X) nounwi
 define <4 x i32> @test_urem_odd_allones_and_one(<4 x i32> %X) nounwind {
 ; CHECK-SSE2-LABEL: test_urem_odd_allones_and_one:
 ; CHECK-SSE2:       # %bb.0:
-; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [3435973837,2147483649,0,3435973837]
-; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm2
-; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [3435973837,4294967295,0,3435973837]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm0
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
-; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm3
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,3,2,3]
-; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm1
-; CHECK-SSE2-NEXT:    psrld $2, %xmm1
-; CHECK-SSE2-NEXT:    psrld $31, %xmm2
-; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm3
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm1[3,3]
-; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [5,4294967295,1,5]
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[1,1,3,3]
-; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm5
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm5[0,2,2,3]
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,0],xmm1[0,0]
-; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm5
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm5 = xmm5[2,0],xmm1[3,0]
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[2,0],xmm5[0,2]
-; CHECK-SSE2-NEXT:    pmuludq %xmm4, %xmm2
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
-; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
-; CHECK-SSE2-NEXT:    psubd %xmm1, %xmm0
-; CHECK-SSE2-NEXT:    pxor %xmm1, %xmm1
-; CHECK-SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
-; CHECK-SSE2-NEXT:    psrld $31, %xmm0
+; CHECK-SSE2-NEXT:    pmuludq %xmm2, %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; CHECK-SSE2-NEXT:    pxor {{.*}}(%rip), %xmm0
+; CHECK-SSE2-NEXT:    pcmpgtd {{.*}}(%rip), %xmm0
+; CHECK-SSE2-NEXT:    pandn {{.*}}(%rip), %xmm0
 ; CHECK-SSE2-NEXT:    retq
 ;
 ; CHECK-SSE41-LABEL: test_urem_odd_allones_and_one:
 ; CHECK-SSE41:       # %bb.0:
-; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [3435973837,2147483649,0,3435973837]
-; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
-; CHECK-SSE41-NEXT:    pmuludq %xmm2, %xmm3
-; CHECK-SSE41-NEXT:    pmuludq %xmm0, %xmm1
-; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5],xmm3[6,7]
-; CHECK-SSE41-NEXT:    movdqa %xmm1, %xmm2
-; CHECK-SSE41-NEXT:    psrld $31, %xmm2
-; CHECK-SSE41-NEXT:    psrld $2, %xmm1
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5,6,7]
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm0[4,5],xmm1[6,7]
-; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm1
-; CHECK-SSE41-NEXT:    psubd %xmm1, %xmm0
-; CHECK-SSE41-NEXT:    pxor %xmm1, %xmm1
+; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm0
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [858993459,1,4294967295,858993459]
+; CHECK-SSE41-NEXT:    pminud %xmm0, %xmm1
 ; CHECK-SSE41-NEXT:    pcmpeqd %xmm1, %xmm0
 ; CHECK-SSE41-NEXT:    psrld $31, %xmm0
 ; CHECK-SSE41-NEXT:    retq
 ;
-; CHECK-AVX1-LABEL: test_urem_odd_allones_and_one:
-; CHECK-AVX1:       # %bb.0:
-; CHECK-AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [3435973837,2147483649,0,3435973837]
-; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
-; CHECK-AVX1-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2
-; CHECK-AVX1-NEXT:    vpmuludq %xmm1, %xmm0, %xmm1
-; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
-; CHECK-AVX1-NEXT:    vpsrld $31, %xmm1, %xmm2
-; CHECK-AVX1-NEXT:    vpsrld $2, %xmm1, %xmm1
-; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5,6,7]
-; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm0[4,5],xmm1[6,7]
-; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
-; CHECK-AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
-; CHECK-AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; CHECK-AVX1-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
-; CHECK-AVX1-NEXT:    vpsrld $31, %xmm0, %xmm0
-; CHECK-AVX1-NEXT:    retq
-;
-; CHECK-AVX2-LABEL: test_urem_odd_allones_and_one:
-; CHECK-AVX2:       # %bb.0:
-; CHECK-AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [3435973837,2147483649,0,3435973837]
-; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
-; CHECK-AVX2-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2
-; CHECK-AVX2-NEXT:    vpmuludq %xmm1, %xmm0, %xmm1
-; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; CHECK-AVX2-NEXT:    vpsrlvd {{.*}}(%rip), %xmm1, %xmm1
-; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0,1],xmm0[2],xmm1[3]
-; CHECK-AVX2-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
-; CHECK-AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
-; CHECK-AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; CHECK-AVX2-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
-; CHECK-AVX2-NEXT:    vpsrld $31, %xmm0, %xmm0
-; CHECK-AVX2-NEXT:    retq
-;
-; CHECK-AVX512VL-LABEL: test_urem_odd_allones_and_one:
-; CHECK-AVX512VL:       # %bb.0:
-; CHECK-AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm1 = [3435973837,2147483649,0,3435973837]
-; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
-; CHECK-AVX512VL-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2
-; CHECK-AVX512VL-NEXT:    vpmuludq %xmm1, %xmm0, %xmm1
-; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; CHECK-AVX512VL-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; CHECK-AVX512VL-NEXT:    vpsrlvd {{.*}}(%rip), %xmm1, %xmm1
-; CHECK-AVX512VL-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0,1],xmm0[2],xmm1[3]
-; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
-; CHECK-AVX512VL-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
-; CHECK-AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
-; CHECK-AVX512VL-NEXT:    vpsrld $31, %xmm0, %xmm0
-; CHECK-AVX512VL-NEXT:    retq
+; CHECK-AVX-LABEL: test_urem_odd_allones_and_one:
+; CHECK-AVX:       # %bb.0:
+; CHECK-AVX-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
+; CHECK-AVX-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX-NEXT:    retq
   %urem = urem <4 x i32> %X, <i32 5, i32 4294967295, i32 1, i32 5>
   %cmp = icmp eq <4 x i32> %urem, <i32 0, i32 0, i32 0, i32 0>
   %ret = zext <4 x i1> %cmp to <4 x i32>
@@ -2004,20 +1914,18 @@ define <4 x i32> @test_urem_even_allones_and_one(<4 x i32> %X) nounwind {
 ; CHECK-SSE2-NEXT:    movdqa %xmm1, %xmm2
 ; CHECK-SSE2-NEXT:    psrld $2, %xmm2
 ; CHECK-SSE2-NEXT:    psrld $31, %xmm1
-; CHECK-SSE2-NEXT:    movdqa %xmm1, %xmm3
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm2[3,3]
-; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [14,4294967295,1,14]
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[1,1,3,3]
-; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm5
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm5[0,2,2,3]
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,0],xmm2[0,0]
-; CHECK-SSE2-NEXT:    movaps %xmm0, %xmm5
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm5 = xmm5[2,0],xmm2[3,0]
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[2,0],xmm5[0,2]
-; CHECK-SSE2-NEXT:    pmuludq %xmm4, %xmm1
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
-; CHECK-SSE2-NEXT:    psubd %xmm1, %xmm0
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm2[3,3]
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [14,4294967295,1,14]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm4
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[0,2,2,3]
+; CHECK-SSE2-NEXT:    movaps %xmm0, %xmm4
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[2,0],xmm2[3,0]
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,1],xmm4[0,2]
+; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; CHECK-SSE2-NEXT:    psubd %xmm2, %xmm0
 ; CHECK-SSE2-NEXT:    pxor %xmm1, %xmm1
 ; CHECK-SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
 ; CHECK-SSE2-NEXT:    psrld $31, %xmm0
@@ -2090,19 +1998,9 @@ define <4 x i32> @test_urem_even_allones_and_one(<4 x i32> %X) nounwind {
 ;
 ; CHECK-AVX512VL-LABEL: test_urem_even_allones_and_one:
 ; CHECK-AVX512VL:       # %bb.0:
-; CHECK-AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm1 = [2454267027,2147483649,0,2454267027]
-; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; CHECK-AVX512VL-NEXT:    vpsrlvd {{.*}}(%rip), %xmm0, %xmm3
-; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
-; CHECK-AVX512VL-NEXT:    vpmuludq %xmm2, %xmm4, %xmm2
-; CHECK-AVX512VL-NEXT:    vpmuludq %xmm1, %xmm3, %xmm1
-; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; CHECK-AVX512VL-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; CHECK-AVX512VL-NEXT:    vpsrlvd {{.*}}(%rip), %xmm1, %xmm1
-; CHECK-AVX512VL-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0,1],xmm0[2],xmm1[3]
-; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
-; CHECK-AVX512VL-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
-; CHECK-AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vprorvd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
 ; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
 ; CHECK-AVX512VL-NEXT:    vpsrld $31, %xmm0, %xmm0
 ; CHECK-AVX512VL-NEXT:    retq
@@ -2130,15 +2028,15 @@ define <4 x i32> @test_urem_odd_even_allones_and_one(<4 x i32> %X) nounwind {
 ; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm3
 ; CHECK-SSE2-NEXT:    psrld $31, %xmm3
 ; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm1[3,3]
-; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [5,4294967295,1,100]
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[1,1,3,3]
-; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm5
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm5[0,2,2,3]
-; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm5
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm5 = xmm5[2,0],xmm1[3,0]
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [5,4294967295,1,100]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm4
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[0,2,2,3]
 ; CHECK-SSE2-NEXT:    psrld $2, %xmm2
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,3],xmm5[0,2]
-; CHECK-SSE2-NEXT:    pmuludq %xmm4, %xmm2
+; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm4
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[2,0],xmm2[3,0]
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,1],xmm4[0,2]
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
 ; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
 ; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
 ; CHECK-SSE2-NEXT:    psubd %xmm1, %xmm0
@@ -2213,18 +2111,9 @@ define <4 x i32> @test_urem_odd_even_allones_and_one(<4 x i32> %X) nounwind {
 ;
 ; CHECK-AVX512VL-LABEL: test_urem_odd_even_allones_and_one:
 ; CHECK-AVX512VL:       # %bb.0:
-; CHECK-AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm1 = [3435973837,2147483649,0,1374389535]
-; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
-; CHECK-AVX512VL-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2
-; CHECK-AVX512VL-NEXT:    vpmuludq %xmm1, %xmm0, %xmm1
-; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; CHECK-AVX512VL-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; CHECK-AVX512VL-NEXT:    vpsrlvd {{.*}}(%rip), %xmm1, %xmm1
-; CHECK-AVX512VL-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0,1],xmm0[2],xmm1[3]
-; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
-; CHECK-AVX512VL-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
-; CHECK-AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vprorvd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
 ; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
 ; CHECK-AVX512VL-NEXT:    vpsrld $31, %xmm0, %xmm0
 ; CHECK-AVX512VL-NEXT:    retq
@@ -2249,22 +2138,19 @@ define <4 x i32> @test_urem_odd_poweroftwo_and_one(<4 x i32> %X) nounwind {
 ; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm3
 ; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,3,2,3]
 ; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm1
-; CHECK-SSE2-NEXT:    psrld $2, %xmm1
-; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm3
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm1[3,3]
-; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [5,16,1,5]
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[1,1,3,3]
-; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm5
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm5[0,2,2,3]
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,0],xmm1[0,0]
-; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm5
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm5 = xmm5[2,0],xmm1[3,0]
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[2,0],xmm5[0,2]
-; CHECK-SSE2-NEXT:    pmuludq %xmm4, %xmm2
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
-; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
-; CHECK-SSE2-NEXT:    psubd %xmm1, %xmm0
+; CHECK-SSE2-NEXT:    psrld $2, %xmm2
+; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm1
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[2,0],xmm2[3,0]
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm2[3,3]
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,1],xmm1[0,2]
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [5,16,1,5]
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; CHECK-SSE2-NEXT:    psubd %xmm2, %xmm0
 ; CHECK-SSE2-NEXT:    pxor %xmm1, %xmm1
 ; CHECK-SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
 ; CHECK-SSE2-NEXT:    psrld $31, %xmm0
@@ -2328,18 +2214,9 @@ define <4 x i32> @test_urem_odd_poweroftwo_and_one(<4 x i32> %X) nounwind {
 ;
 ; CHECK-AVX512VL-LABEL: test_urem_odd_poweroftwo_and_one:
 ; CHECK-AVX512VL:       # %bb.0:
-; CHECK-AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm1 = [3435973837,268435456,0,3435973837]
-; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
-; CHECK-AVX512VL-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2
-; CHECK-AVX512VL-NEXT:    vpmuludq %xmm1, %xmm0, %xmm1
-; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; CHECK-AVX512VL-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; CHECK-AVX512VL-NEXT:    vpsrlvd {{.*}}(%rip), %xmm1, %xmm1
-; CHECK-AVX512VL-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0,1],xmm0[2],xmm1[3]
-; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
-; CHECK-AVX512VL-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
-; CHECK-AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vprorvd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
 ; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
 ; CHECK-AVX512VL-NEXT:    vpsrld $31, %xmm0, %xmm0
 ; CHECK-AVX512VL-NEXT:    retq
@@ -2366,21 +2243,18 @@ define <4 x i32> @test_urem_even_poweroftwo_and_one(<4 x i32> %X) nounwind {
 ; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm1
 ; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
 ; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
-; CHECK-SSE2-NEXT:    movdqa %xmm1, %xmm2
-; CHECK-SSE2-NEXT:    psrld $2, %xmm2
-; CHECK-SSE2-NEXT:    movdqa %xmm1, %xmm3
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm2[3,3]
-; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [14,16,1,14]
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[1,1,3,3]
-; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm5
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm5[0,2,2,3]
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,0],xmm2[0,0]
-; CHECK-SSE2-NEXT:    movaps %xmm0, %xmm5
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm5 = xmm5[2,0],xmm2[3,0]
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[2,0],xmm5[0,2]
-; CHECK-SSE2-NEXT:    pmuludq %xmm4, %xmm1
+; CHECK-SSE2-NEXT:    psrld $2, %xmm1
+; CHECK-SSE2-NEXT:    movaps %xmm0, %xmm2
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[2,0],xmm1[3,0]
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[1,1],xmm1[3,3]
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0,2]
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [14,16,1,14]
+; CHECK-SSE2-NEXT:    pmuludq %xmm2, %xmm1
 ; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm4, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
 ; CHECK-SSE2-NEXT:    psubd %xmm1, %xmm0
 ; CHECK-SSE2-NEXT:    pxor %xmm1, %xmm1
 ; CHECK-SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
@@ -2451,19 +2325,9 @@ define <4 x i32> @test_urem_even_poweroftwo_and_one(<4 x i32> %X) nounwind {
 ;
 ; CHECK-AVX512VL-LABEL: test_urem_even_poweroftwo_and_one:
 ; CHECK-AVX512VL:       # %bb.0:
-; CHECK-AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm1 = [2454267027,268435456,0,2454267027]
-; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; CHECK-AVX512VL-NEXT:    vpsrlvd {{.*}}(%rip), %xmm0, %xmm3
-; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
-; CHECK-AVX512VL-NEXT:    vpmuludq %xmm2, %xmm4, %xmm2
-; CHECK-AVX512VL-NEXT:    vpmuludq %xmm1, %xmm3, %xmm1
-; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; CHECK-AVX512VL-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; CHECK-AVX512VL-NEXT:    vpsrlvd {{.*}}(%rip), %xmm1, %xmm1
-; CHECK-AVX512VL-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0,1],xmm0[2],xmm1[3]
-; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
-; CHECK-AVX512VL-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
-; CHECK-AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vprorvd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
 ; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
 ; CHECK-AVX512VL-NEXT:    vpsrld $31, %xmm0, %xmm0
 ; CHECK-AVX512VL-NEXT:    retq
@@ -2487,20 +2351,19 @@ define <4 x i32> @test_urem_odd_even_poweroftwo_and_one(<4 x i32> %X) nounwind {
 ; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,3,2,3]
 ; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
 ; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm1
-; CHECK-SSE2-NEXT:    psrld $5, %xmm1
-; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm3
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm1[3,3]
+; CHECK-SSE2-NEXT:    psrld $2, %xmm1
+; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm4
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[2,0],xmm1[3,0]
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,1],xmm4[0,2]
 ; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [5,16,1,100]
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[1,1,3,3]
-; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm5
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm5[0,2,2,3]
-; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm5
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm5 = xmm5[2,0],xmm1[3,0]
-; CHECK-SSE2-NEXT:    psrld $2, %xmm2
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,3],xmm5[0,2]
-; CHECK-SSE2-NEXT:    pmuludq %xmm4, %xmm2
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
-; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
+; CHECK-SSE2-NEXT:    pmuludq %xmm4, %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; CHECK-SSE2-NEXT:    psrld $5, %xmm2
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm2[3,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
 ; CHECK-SSE2-NEXT:    psubd %xmm1, %xmm0
 ; CHECK-SSE2-NEXT:    pxor %xmm1, %xmm1
 ; CHECK-SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
@@ -2570,18 +2433,9 @@ define <4 x i32> @test_urem_odd_even_poweroftwo_and_one(<4 x i32> %X) nounwind {
 ;
 ; CHECK-AVX512VL-LABEL: test_urem_odd_even_poweroftwo_and_one:
 ; CHECK-AVX512VL:       # %bb.0:
-; CHECK-AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm1 = [3435973837,268435456,0,1374389535]
-; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
-; CHECK-AVX512VL-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2
-; CHECK-AVX512VL-NEXT:    vpmuludq %xmm1, %xmm0, %xmm1
-; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; CHECK-AVX512VL-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; CHECK-AVX512VL-NEXT:    vpsrlvd {{.*}}(%rip), %xmm1, %xmm1
-; CHECK-AVX512VL-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0,1],xmm0[2],xmm1[3]
-; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
-; CHECK-AVX512VL-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
-; CHECK-AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vprorvd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
 ; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
 ; CHECK-AVX512VL-NEXT:    vpsrld $31, %xmm0, %xmm0
 ; CHECK-AVX512VL-NEXT:    retq
@@ -2599,27 +2453,25 @@ define <4 x i32> @test_urem_odd_allones_and_poweroftwo_and_one(<4 x i32> %X) nou
 ; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [3435973837,2147483649,268435456,0]
 ; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm2
 ; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,3,2,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
 ; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
-; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm4
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,3,2,3]
-; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
-; CHECK-SSE2-NEXT:    movdqa %xmm3, %xmm1
-; CHECK-SSE2-NEXT:    psrld $31, %xmm1
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[3,3]
-; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [5,4294967295,16,1]
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[1,1,3,3]
-; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm5
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm5[0,2,2,3]
-; CHECK-SSE2-NEXT:    psrld $2, %xmm3
-; CHECK-SSE2-NEXT:    movaps %xmm0, %xmm5
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm5 = xmm5[3,0],xmm2[3,3]
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,3],xmm5[2,0]
-; CHECK-SSE2-NEXT:    pmuludq %xmm4, %xmm3
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,2,2,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm3
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,3,2,3]
 ; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; CHECK-SSE2-NEXT:    psubd %xmm2, %xmm0
+; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm1
+; CHECK-SSE2-NEXT:    psrld $2, %xmm1
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,3],xmm2[2,1]
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [5,4294967295,16,1]
+; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; CHECK-SSE2-NEXT:    psrld $31, %xmm2
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1],xmm0[3,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm2, %xmm3
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; CHECK-SSE2-NEXT:    psubd %xmm1, %xmm0
 ; CHECK-SSE2-NEXT:    pxor %xmm1, %xmm1
 ; CHECK-SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
 ; CHECK-SSE2-NEXT:    psrld $31, %xmm0
@@ -2688,18 +2540,9 @@ define <4 x i32> @test_urem_odd_allones_and_poweroftwo_and_one(<4 x i32> %X) nou
 ;
 ; CHECK-AVX512VL-LABEL: test_urem_odd_allones_and_poweroftwo_and_one:
 ; CHECK-AVX512VL:       # %bb.0:
-; CHECK-AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm1 = [3435973837,2147483649,268435456,0]
-; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
-; CHECK-AVX512VL-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2
-; CHECK-AVX512VL-NEXT:    vpmuludq %xmm1, %xmm0, %xmm1
-; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; CHECK-AVX512VL-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; CHECK-AVX512VL-NEXT:    vpsrlvd {{.*}}(%rip), %xmm1, %xmm1
-; CHECK-AVX512VL-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0,1,2],xmm0[3]
-; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
-; CHECK-AVX512VL-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
-; CHECK-AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vprorvd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
 ; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
 ; CHECK-AVX512VL-NEXT:    vpsrld $31, %xmm0, %xmm0
 ; CHECK-AVX512VL-NEXT:    retq
@@ -2725,20 +2568,18 @@ define <4 x i32> @test_urem_even_allones_and_poweroftwo_and_one(<4 x i32> %X) no
 ; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,3,2,3]
 ; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
 ; CHECK-SSE2-NEXT:    movdqa %xmm1, %xmm2
-; CHECK-SSE2-NEXT:    psrld $31, %xmm2
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1],xmm0[3,3]
+; CHECK-SSE2-NEXT:    psrld $2, %xmm2
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,3],xmm1[2,1]
 ; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [14,4294967295,16,1]
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[1,1,3,3]
-; CHECK-SSE2-NEXT:    pmuludq %xmm2, %xmm5
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm5[0,2,2,3]
-; CHECK-SSE2-NEXT:    psrld $2, %xmm1
-; CHECK-SSE2-NEXT:    movaps %xmm0, %xmm5
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm5 = xmm5[3,0],xmm4[3,3]
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,3],xmm5[2,0]
-; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm1
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
-; CHECK-SSE2-NEXT:    psubd %xmm1, %xmm0
+; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    psrld $31, %xmm1
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[3,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm3
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; CHECK-SSE2-NEXT:    psubd %xmm2, %xmm0
 ; CHECK-SSE2-NEXT:    pxor %xmm1, %xmm1
 ; CHECK-SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
 ; CHECK-SSE2-NEXT:    psrld $31, %xmm0
@@ -2815,21 +2656,9 @@ define <4 x i32> @test_urem_even_allones_and_poweroftwo_and_one(<4 x i32> %X) no
 ;
 ; CHECK-AVX512VL-LABEL: test_urem_even_allones_and_poweroftwo_and_one:
 ; CHECK-AVX512VL:       # %bb.0:
-; CHECK-AVX512VL-NEXT:    movl $1, %eax
-; CHECK-AVX512VL-NEXT:    vmovd %eax, %xmm1
-; CHECK-AVX512VL-NEXT:    vpsrlvd %xmm1, %xmm0, %xmm1
-; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; CHECK-AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm3 = [2454267027,2147483649,268435456,0]
-; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
-; CHECK-AVX512VL-NEXT:    vpmuludq %xmm4, %xmm2, %xmm2
-; CHECK-AVX512VL-NEXT:    vpmuludq %xmm3, %xmm1, %xmm1
-; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; CHECK-AVX512VL-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; CHECK-AVX512VL-NEXT:    vpsrlvd {{.*}}(%rip), %xmm1, %xmm1
-; CHECK-AVX512VL-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0,1,2],xmm0[3]
-; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
-; CHECK-AVX512VL-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
-; CHECK-AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vprorvd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
 ; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
 ; CHECK-AVX512VL-NEXT:    vpsrld $31, %xmm0, %xmm0
 ; CHECK-AVX512VL-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/urem-seteq-vec-nonzero.ll b/llvm/test/CodeGen/X86/urem-seteq-vec-nonzero.ll
new file mode 100644
index 000000000..b3814a1b3
--- /dev/null
+++ b/llvm/test/CodeGen/X86/urem-seteq-vec-nonzero.ll
@@ -0,0 +1,369 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu -mattr=+sse2 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-SSE,CHECK-SSE2
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu -mattr=+sse4.1 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-SSE,CHECK-SSE41
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu -mattr=+avx < %s | FileCheck %s --check-prefixes=CHECK,CHECK-AVX,CHECK-AVX1
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu -mattr=+avx2 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-AVX,CHECK-AVX2
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512f,+avx512vl < %s | FileCheck %s --check-prefixes=CHECK,CHECK-AVX,CHECK-AVX512VL
+
+define <4 x i1> @t32_3(<4 x i32> %X) nounwind {
+; CHECK-SSE2-LABEL: t32_3:
+; CHECK-SSE2:       # %bb.0:
+; CHECK-SSE2-NEXT:    psubd {{.*}}(%rip), %xmm0
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [2863311531,2863311531,2863311531,2863311531]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm0
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[0,2,2,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1]
+; CHECK-SSE2-NEXT:    pxor {{.*}}(%rip), %xmm3
+; CHECK-SSE2-NEXT:    pcmpgtd {{.*}}(%rip), %xmm3
+; CHECK-SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
+; CHECK-SSE2-NEXT:    pxor %xmm3, %xmm0
+; CHECK-SSE2-NEXT:    retq
+;
+; CHECK-SSE41-LABEL: t32_3:
+; CHECK-SSE41:       # %bb.0:
+; CHECK-SSE41-NEXT:    psubd {{.*}}(%rip), %xmm0
+; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm0
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [1431655765,1431655764,1431655764,1431655764]
+; CHECK-SSE41-NEXT:    pminud %xmm0, %xmm1
+; CHECK-SSE41-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-SSE41-NEXT:    retq
+;
+; CHECK-AVX1-LABEL: t32_3:
+; CHECK-AVX1:       # %bb.0:
+; CHECK-AVX1-NEXT:    vpsubd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
+; CHECK-AVX1-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    retq
+;
+; CHECK-AVX2-LABEL: t32_3:
+; CHECK-AVX2:       # %bb.0:
+; CHECK-AVX2-NEXT:    vpsubd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [2863311531,2863311531,2863311531,2863311531]
+; CHECK-AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
+; CHECK-AVX2-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    retq
+;
+; CHECK-AVX512VL-LABEL: t32_3:
+; CHECK-AVX512VL:       # %bb.0:
+; CHECK-AVX512VL-NEXT:    vpsubd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip){1to4}, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
+; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    retq
+  %urem = urem <4 x i32> %X, <i32 3, i32 3, i32 3, i32 3>
+  %cmp = icmp eq <4 x i32> %urem, <i32 0, i32 1, i32 2, i32 2>
+  ret <4 x i1> %cmp
+}
+
+define <4 x i1> @t32_5(<4 x i32> %X) nounwind {
+; CHECK-SSE2-LABEL: t32_5:
+; CHECK-SSE2:       # %bb.0:
+; CHECK-SSE2-NEXT:    psubd {{.*}}(%rip), %xmm0
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [3435973837,3435973837,3435973837,3435973837]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm0
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[0,2,2,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1]
+; CHECK-SSE2-NEXT:    pxor {{.*}}(%rip), %xmm3
+; CHECK-SSE2-NEXT:    pcmpgtd {{.*}}(%rip), %xmm3
+; CHECK-SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
+; CHECK-SSE2-NEXT:    pxor %xmm3, %xmm0
+; CHECK-SSE2-NEXT:    retq
+;
+; CHECK-SSE41-LABEL: t32_5:
+; CHECK-SSE41:       # %bb.0:
+; CHECK-SSE41-NEXT:    psubd {{.*}}(%rip), %xmm0
+; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm0
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [858993458,858993458,858993458,858993458]
+; CHECK-SSE41-NEXT:    pminud %xmm0, %xmm1
+; CHECK-SSE41-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-SSE41-NEXT:    retq
+;
+; CHECK-AVX1-LABEL: t32_5:
+; CHECK-AVX1:       # %bb.0:
+; CHECK-AVX1-NEXT:    vpsubd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
+; CHECK-AVX1-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    retq
+;
+; CHECK-AVX2-LABEL: t32_5:
+; CHECK-AVX2:       # %bb.0:
+; CHECK-AVX2-NEXT:    vpsubd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [3435973837,3435973837,3435973837,3435973837]
+; CHECK-AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [858993458,858993458,858993458,858993458]
+; CHECK-AVX2-NEXT:    vpminud %xmm1, %xmm0, %xmm1
+; CHECK-AVX2-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    retq
+;
+; CHECK-AVX512VL-LABEL: t32_5:
+; CHECK-AVX512VL:       # %bb.0:
+; CHECK-AVX512VL-NEXT:    vpsubd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip){1to4}, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpminud {{.*}}(%rip){1to4}, %xmm0, %xmm1
+; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    retq
+  %urem = urem <4 x i32> %X, <i32 5, i32 5, i32 5, i32 5>
+  %cmp = icmp eq <4 x i32> %urem, <i32 1, i32 2, i32 3, i32 4>
+  ret <4 x i1> %cmp
+}
+
+define <4 x i1> @t32_6_part0(<4 x i32> %X) nounwind {
+; CHECK-SSE2-LABEL: t32_6_part0:
+; CHECK-SSE2:       # %bb.0:
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [2863311531,2863311531,2863311531,2863311531]
+; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm2
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm3
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,3,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; CHECK-SSE2-NEXT:    psrld $2, %xmm2
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [6,6,6,6]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm3
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; CHECK-SSE2-NEXT:    psubd %xmm2, %xmm0
+; CHECK-SSE2-NEXT:    pcmpeqd {{.*}}(%rip), %xmm0
+; CHECK-SSE2-NEXT:    retq
+;
+; CHECK-SSE41-LABEL: t32_6_part0:
+; CHECK-SSE41:       # %bb.0:
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [2863311531,2863311531,2863311531,2863311531]
+; CHECK-SSE41-NEXT:    pmuludq %xmm2, %xmm1
+; CHECK-SSE41-NEXT:    pmuludq %xmm0, %xmm2
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
+; CHECK-SSE41-NEXT:    psrld $2, %xmm2
+; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm2
+; CHECK-SSE41-NEXT:    psubd %xmm2, %xmm0
+; CHECK-SSE41-NEXT:    pcmpeqd {{.*}}(%rip), %xmm0
+; CHECK-SSE41-NEXT:    retq
+;
+; CHECK-AVX1-LABEL: t32_6_part0:
+; CHECK-AVX1:       # %bb.0:
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; CHECK-AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [2863311531,2863311531,2863311531,2863311531]
+; CHECK-AVX1-NEXT:    vpmuludq %xmm2, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpmuludq %xmm2, %xmm0, %xmm2
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
+; CHECK-AVX1-NEXT:    vpsrld $2, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpcmpeqd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    retq
+;
+; CHECK-AVX2-LABEL: t32_6_part0:
+; CHECK-AVX2:       # %bb.0:
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [2863311531,2863311531,2863311531,2863311531]
+; CHECK-AVX2-NEXT:    vpmuludq %xmm2, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpmuludq %xmm2, %xmm0, %xmm2
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2],xmm1[3]
+; CHECK-AVX2-NEXT:    vpsrld $2, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [6,6,6,6]
+; CHECK-AVX2-NEXT:    vpmulld %xmm2, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpcmpeqd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    retq
+;
+; CHECK-AVX512VL-LABEL: t32_6_part0:
+; CHECK-AVX512VL:       # %bb.0:
+; CHECK-AVX512VL-NEXT:    vpsubd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip){1to4}, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vprord $1, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpminud {{.*}}(%rip){1to4}, %xmm0, %xmm1
+; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    retq
+  %urem = urem <4 x i32> %X, <i32 6, i32 6, i32 6, i32 6>
+  %cmp = icmp eq <4 x i32> %urem, <i32 0, i32 1, i32 2, i32 3>
+  ret <4 x i1> %cmp
+}
+
+define <4 x i1> @t32_6_part1(<4 x i32> %X) nounwind {
+; CHECK-SSE2-LABEL: t32_6_part1:
+; CHECK-SSE2:       # %bb.0:
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [2863311531,2863311531,2863311531,2863311531]
+; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm2
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm3
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,3,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; CHECK-SSE2-NEXT:    psrld $2, %xmm2
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [6,6,6,6]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm3
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; CHECK-SSE2-NEXT:    psubd %xmm2, %xmm0
+; CHECK-SSE2-NEXT:    pcmpeqd {{.*}}(%rip), %xmm0
+; CHECK-SSE2-NEXT:    retq
+;
+; CHECK-SSE41-LABEL: t32_6_part1:
+; CHECK-SSE41:       # %bb.0:
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [2863311531,2863311531,2863311531,2863311531]
+; CHECK-SSE41-NEXT:    pmuludq %xmm2, %xmm1
+; CHECK-SSE41-NEXT:    pmuludq %xmm0, %xmm2
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
+; CHECK-SSE41-NEXT:    psrld $2, %xmm2
+; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm2
+; CHECK-SSE41-NEXT:    psubd %xmm2, %xmm0
+; CHECK-SSE41-NEXT:    pcmpeqd {{.*}}(%rip), %xmm0
+; CHECK-SSE41-NEXT:    retq
+;
+; CHECK-AVX1-LABEL: t32_6_part1:
+; CHECK-AVX1:       # %bb.0:
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; CHECK-AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [2863311531,2863311531,2863311531,2863311531]
+; CHECK-AVX1-NEXT:    vpmuludq %xmm2, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpmuludq %xmm2, %xmm0, %xmm2
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
+; CHECK-AVX1-NEXT:    vpsrld $2, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpcmpeqd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    retq
+;
+; CHECK-AVX2-LABEL: t32_6_part1:
+; CHECK-AVX2:       # %bb.0:
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [2863311531,2863311531,2863311531,2863311531]
+; CHECK-AVX2-NEXT:    vpmuludq %xmm2, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpmuludq %xmm2, %xmm0, %xmm2
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2],xmm1[3]
+; CHECK-AVX2-NEXT:    vpsrld $2, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [6,6,6,6]
+; CHECK-AVX2-NEXT:    vpmulld %xmm2, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpcmpeqd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    retq
+;
+; CHECK-AVX512VL-LABEL: t32_6_part1:
+; CHECK-AVX512VL:       # %bb.0:
+; CHECK-AVX512VL-NEXT:    vpsubd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip){1to4}, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vprord $1, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
+; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    retq
+  %urem = urem <4 x i32> %X, <i32 6, i32 6, i32 6, i32 6>
+  %cmp = icmp eq <4 x i32> %urem, <i32 4, i32 5, i32 0, i32 0>
+  ret <4 x i1> %cmp
+}
+
+define <4 x i1> @t32_tautological(<4 x i32> %X) nounwind {
+; CHECK-SSE2-LABEL: t32_tautological:
+; CHECK-SSE2:       # %bb.0:
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [0,0,2147483648,2863311531]
+; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm2
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm3
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,3,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,1,2,2]
+; CHECK-SSE2-NEXT:    psrld $1, %xmm2
+; CHECK-SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; CHECK-SSE2-NEXT:    movsd {{.*#+}} xmm1 = xmm0[0],xmm1[1]
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [1,1,2,3]
+; CHECK-SSE2-NEXT:    movapd %xmm1, %xmm3
+; CHECK-SSE2-NEXT:    pmuludq %xmm2, %xmm3
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; CHECK-SSE2-NEXT:    psubd %xmm3, %xmm0
+; CHECK-SSE2-NEXT:    pcmpeqd {{.*}}(%rip), %xmm0
+; CHECK-SSE2-NEXT:    retq
+;
+; CHECK-SSE41-LABEL: t32_tautological:
+; CHECK-SSE41:       # %bb.0:
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [0,0,2147483648,2863311531]
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; CHECK-SSE41-NEXT:    pmuludq %xmm2, %xmm3
+; CHECK-SSE41-NEXT:    pmuludq %xmm0, %xmm1
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5],xmm3[6,7]
+; CHECK-SSE41-NEXT:    movdqa %xmm1, %xmm2
+; CHECK-SSE41-NEXT:    psrld $1, %xmm2
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm1[0,1,2,3,4,5],xmm2[6,7]
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm0[0,1,2,3],xmm2[4,5,6,7]
+; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm2
+; CHECK-SSE41-NEXT:    psubd %xmm2, %xmm0
+; CHECK-SSE41-NEXT:    pcmpeqd {{.*}}(%rip), %xmm0
+; CHECK-SSE41-NEXT:    retq
+;
+; CHECK-AVX1-LABEL: t32_tautological:
+; CHECK-AVX1:       # %bb.0:
+; CHECK-AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [0,0,2147483648,2863311531]
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2
+; CHECK-AVX1-NEXT:    vpmuludq %xmm1, %xmm0, %xmm1
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
+; CHECK-AVX1-NEXT:    vpsrld $1, %xmm1, %xmm2
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5],xmm2[6,7]
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0,1,2,3],xmm1[4,5,6,7]
+; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpcmpeqd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    retq
+;
+; CHECK-AVX2-LABEL: t32_tautological:
+; CHECK-AVX2:       # %bb.0:
+; CHECK-AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [0,0,2147483648,2863311531]
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2
+; CHECK-AVX2-NEXT:    vpmuludq %xmm1, %xmm0, %xmm1
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
+; CHECK-AVX2-NEXT:    vpsrlvd {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3]
+; CHECK-AVX2-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpcmpeqd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    retq
+;
+; CHECK-AVX512VL-LABEL: t32_tautological:
+; CHECK-AVX512VL:       # %bb.0:
+; CHECK-AVX512VL-NEXT:    vpsubd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip){1to4}, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
+; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX512VL-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2],xmm0[3]
+; CHECK-AVX512VL-NEXT:    retq
+  %urem = urem <4 x i32> %X, <i32 1, i32 1, i32 2, i32 3>
+  %cmp = icmp eq <4 x i32> %urem, <i32 0, i32 1, i32 2, i32 2>
+  ret <4 x i1> %cmp
+}
diff --git a/llvm/test/CodeGen/X86/urem-seteq-vec-splat.ll b/llvm/test/CodeGen/X86/urem-seteq-vec-splat.ll
index face44a7c..409f0bca2 100644
--- a/llvm/test/CodeGen/X86/urem-seteq-vec-splat.ll
+++ b/llvm/test/CodeGen/X86/urem-seteq-vec-splat.ll
@@ -150,6 +150,144 @@ define <4 x i32> @test_urem_even_100(<4 x i32> %X) nounwind {
   ret <4 x i32> %ret
 }
 
+; Negative divisors should be negated, and thus this is still splat vectors.
+
+; Odd divisor
+define <4 x i32> @test_urem_odd_neg25(<4 x i32> %X) nounwind {
+; CHECK-SSE2-LABEL: test_urem_odd_neg25:
+; CHECK-SSE2:       # %bb.0:
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [3264175145,1030792151,1030792151,3264175145]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm0
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,2,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm2, %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; CHECK-SSE2-NEXT:    pxor {{.*}}(%rip), %xmm0
+; CHECK-SSE2-NEXT:    pcmpgtd {{.*}}(%rip), %xmm0
+; CHECK-SSE2-NEXT:    pandn {{.*}}(%rip), %xmm0
+; CHECK-SSE2-NEXT:    retq
+;
+; CHECK-SSE41-LABEL: test_urem_odd_neg25:
+; CHECK-SSE41:       # %bb.0:
+; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm0
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [171798691,1,1,171798691]
+; CHECK-SSE41-NEXT:    pminud %xmm0, %xmm1
+; CHECK-SSE41-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-SSE41-NEXT:    psrld $31, %xmm0
+; CHECK-SSE41-NEXT:    retq
+;
+; CHECK-AVX-LABEL: test_urem_odd_neg25:
+; CHECK-AVX:       # %bb.0:
+; CHECK-AVX-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
+; CHECK-AVX-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX-NEXT:    retq
+  %urem = urem <4 x i32> %X, <i32 25, i32 -25, i32 -25, i32 25>
+  %cmp = icmp eq <4 x i32> %urem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Even divisors
+define <4 x i32> @test_urem_even_neg100(<4 x i32> %X) nounwind {
+; CHECK-SSE2-LABEL: test_urem_even_neg100:
+; CHECK-SSE2:       # %bb.0:
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq {{.*}}(%rip), %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
+; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm2
+; CHECK-SSE2-NEXT:    psrld $2, %xmm2
+; CHECK-SSE2-NEXT:    pmuludq {{.*}}(%rip), %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm1
+; CHECK-SSE2-NEXT:    psrld $27, %xmm1
+; CHECK-SSE2-NEXT:    pmuludq {{.*}}(%rip), %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; CHECK-SSE2-NEXT:    psrld $5, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq {{.*}}(%rip), %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; CHECK-SSE2-NEXT:    psubd %xmm1, %xmm0
+; CHECK-SSE2-NEXT:    pxor %xmm1, %xmm1
+; CHECK-SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-SSE2-NEXT:    psrld $31, %xmm0
+; CHECK-SSE2-NEXT:    retq
+;
+; CHECK-SSE41-LABEL: test_urem_even_neg100:
+; CHECK-SSE41:       # %bb.0:
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; CHECK-SSE41-NEXT:    pmuludq {{.*}}(%rip), %xmm1
+; CHECK-SSE41-NEXT:    movdqa %xmm0, %xmm2
+; CHECK-SSE41-NEXT:    psrld $2, %xmm2
+; CHECK-SSE41-NEXT:    pmuludq {{.*}}(%rip), %xmm2
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
+; CHECK-SSE41-NEXT:    movdqa %xmm2, %xmm1
+; CHECK-SSE41-NEXT:    psrld $5, %xmm1
+; CHECK-SSE41-NEXT:    psrld $27, %xmm2
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
+; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm2
+; CHECK-SSE41-NEXT:    psubd %xmm2, %xmm0
+; CHECK-SSE41-NEXT:    pxor %xmm1, %xmm1
+; CHECK-SSE41-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-SSE41-NEXT:    psrld $31, %xmm0
+; CHECK-SSE41-NEXT:    retq
+;
+; CHECK-AVX1-LABEL: test_urem_even_neg100:
+; CHECK-AVX1:       # %bb.0:
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpmuludq {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpsrld $2, %xmm0, %xmm2
+; CHECK-AVX1-NEXT:    vpmuludq {{.*}}(%rip), %xmm2, %xmm2
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
+; CHECK-AVX1-NEXT:    vpsrld $5, %xmm1, %xmm2
+; CHECK-AVX1-NEXT:    vpsrld $27, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
+; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    retq
+;
+; CHECK-AVX2-LABEL: test_urem_even_neg100:
+; CHECK-AVX2:       # %bb.0:
+; CHECK-AVX2-NEXT:    vpsrlvd {{.*}}(%rip), %xmm0, %xmm1
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm3 = [1374389535,1374389535,1374389535,1374389535]
+; CHECK-AVX2-NEXT:    vpmuludq %xmm3, %xmm2, %xmm2
+; CHECK-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm3 = [536870925,536870925,536870925,536870925]
+; CHECK-AVX2-NEXT:    vpmuludq %xmm3, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
+; CHECK-AVX2-NEXT:    vpsrlvd {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    retq
+;
+; CHECK-AVX512VL-LABEL: test_urem_even_neg100:
+; CHECK-AVX512VL:       # %bb.0:
+; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vprord $2, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
+; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    retq
+  %urem = urem <4 x i32> %X, <i32 -100, i32 100, i32 -100, i32 100>
+  %cmp = icmp eq <4 x i32> %urem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
 ;------------------------------------------------------------------------------;
 ; Comparison constant has undef elements.
 ;------------------------------------------------------------------------------;
@@ -348,6 +486,47 @@ define <4 x i32> @test_urem_even_undef1(<4 x i32> %X) nounwind {
 ; Negative tests
 ;------------------------------------------------------------------------------;
 
+define <4 x i32> @test_urem_one_eq(<4 x i32> %X) nounwind {
+; CHECK-SSE-LABEL: test_urem_one_eq:
+; CHECK-SSE:       # %bb.0:
+; CHECK-SSE-NEXT:    movaps {{.*#+}} xmm0 = [1,1,1,1]
+; CHECK-SSE-NEXT:    retq
+;
+; CHECK-AVX1-LABEL: test_urem_one_eq:
+; CHECK-AVX1:       # %bb.0:
+; CHECK-AVX1-NEXT:    vmovaps {{.*#+}} xmm0 = [1,1,1,1]
+; CHECK-AVX1-NEXT:    retq
+;
+; CHECK-AVX2-LABEL: test_urem_one_eq:
+; CHECK-AVX2:       # %bb.0:
+; CHECK-AVX2-NEXT:    vbroadcastss {{.*#+}} xmm0 = [1,1,1,1]
+; CHECK-AVX2-NEXT:    retq
+;
+; CHECK-AVX512VL-LABEL: test_urem_one_eq:
+; CHECK-AVX512VL:       # %bb.0:
+; CHECK-AVX512VL-NEXT:    vbroadcastss {{.*#+}} xmm0 = [1,1,1,1]
+; CHECK-AVX512VL-NEXT:    retq
+  %urem = urem <4 x i32> %X, <i32 1, i32 1, i32 1, i32 1>
+  %cmp = icmp eq <4 x i32> %urem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+define <4 x i32> @test_urem_one_ne(<4 x i32> %X) nounwind {
+; CHECK-SSE-LABEL: test_urem_one_ne:
+; CHECK-SSE:       # %bb.0:
+; CHECK-SSE-NEXT:    xorps %xmm0, %xmm0
+; CHECK-SSE-NEXT:    retq
+;
+; CHECK-AVX-LABEL: test_urem_one_ne:
+; CHECK-AVX:       # %bb.0:
+; CHECK-AVX-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; CHECK-AVX-NEXT:    retq
+  %urem = urem <4 x i32> %X, <i32 1, i32 1, i32 1, i32 1>
+  %cmp = icmp ne <4 x i32> %urem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
 ; We can lower remainder of division by powers of two much better elsewhere.
 define <4 x i32> @test_urem_pow2(<4 x i32> %X) nounwind {
 ; CHECK-SSE-LABEL: test_urem_pow2:
@@ -388,6 +567,46 @@ define <4 x i32> @test_urem_pow2(<4 x i32> %X) nounwind {
   ret <4 x i32> %ret
 }
 
+; We could lower remainder of division by INT_MIN much better elsewhere.
+define <4 x i32> @test_urem_int_min(<4 x i32> %X) nounwind {
+; CHECK-SSE-LABEL: test_urem_int_min:
+; CHECK-SSE:       # %bb.0:
+; CHECK-SSE-NEXT:    pand {{.*}}(%rip), %xmm0
+; CHECK-SSE-NEXT:    pxor %xmm1, %xmm1
+; CHECK-SSE-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-SSE-NEXT:    psrld $31, %xmm0
+; CHECK-SSE-NEXT:    retq
+;
+; CHECK-AVX1-LABEL: test_urem_int_min:
+; CHECK-AVX1:       # %bb.0:
+; CHECK-AVX1-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    retq
+;
+; CHECK-AVX2-LABEL: test_urem_int_min:
+; CHECK-AVX2:       # %bb.0:
+; CHECK-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [2147483647,2147483647,2147483647,2147483647]
+; CHECK-AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    retq
+;
+; CHECK-AVX512VL-LABEL: test_urem_int_min:
+; CHECK-AVX512VL:       # %bb.0:
+; CHECK-AVX512VL-NEXT:    vpandd {{.*}}(%rip){1to4}, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpsrld $31, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    retq
+  %urem = urem <4 x i32> %X, <i32 2147483648, i32 2147483648, i32 2147483648, i32 2147483648>
+  %cmp = icmp eq <4 x i32> %urem, <i32 0, i32 0, i32 0, i32 0>
+  %ret = zext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
 ; We could lower remainder of division by all-ones much better elsewhere.
 define <4 x i32> @test_urem_allones(<4 x i32> %X) nounwind {
 ; CHECK-SSE2-LABEL: test_urem_allones:
@@ -442,45 +661,3 @@ define <4 x i32> @test_urem_allones(<4 x i32> %X) nounwind {
   %ret = zext <4 x i1> %cmp to <4 x i32>
   ret <4 x i32> %ret
 }
-
-; If all divisors are ones, this is constant-folded.
-define <4 x i32> @test_urem_one_eq(<4 x i32> %X) nounwind {
-; CHECK-SSE-LABEL: test_urem_one_eq:
-; CHECK-SSE:       # %bb.0:
-; CHECK-SSE-NEXT:    movaps {{.*#+}} xmm0 = [1,1,1,1]
-; CHECK-SSE-NEXT:    retq
-;
-; CHECK-AVX1-LABEL: test_urem_one_eq:
-; CHECK-AVX1:       # %bb.0:
-; CHECK-AVX1-NEXT:    vmovaps {{.*#+}} xmm0 = [1,1,1,1]
-; CHECK-AVX1-NEXT:    retq
-;
-; CHECK-AVX2-LABEL: test_urem_one_eq:
-; CHECK-AVX2:       # %bb.0:
-; CHECK-AVX2-NEXT:    vbroadcastss {{.*#+}} xmm0 = [1,1,1,1]
-; CHECK-AVX2-NEXT:    retq
-;
-; CHECK-AVX512VL-LABEL: test_urem_one_eq:
-; CHECK-AVX512VL:       # %bb.0:
-; CHECK-AVX512VL-NEXT:    vbroadcastss {{.*#+}} xmm0 = [1,1,1,1]
-; CHECK-AVX512VL-NEXT:    retq
-  %urem = urem <4 x i32> %X, <i32 1, i32 1, i32 1, i32 1>
-  %cmp = icmp eq <4 x i32> %urem, <i32 0, i32 0, i32 0, i32 0>
-  %ret = zext <4 x i1> %cmp to <4 x i32>
-  ret <4 x i32> %ret
-}
-define <4 x i32> @test_urem_one_ne(<4 x i32> %X) nounwind {
-; CHECK-SSE-LABEL: test_urem_one_ne:
-; CHECK-SSE:       # %bb.0:
-; CHECK-SSE-NEXT:    xorps %xmm0, %xmm0
-; CHECK-SSE-NEXT:    retq
-;
-; CHECK-AVX-LABEL: test_urem_one_ne:
-; CHECK-AVX:       # %bb.0:
-; CHECK-AVX-NEXT:    vxorps %xmm0, %xmm0, %xmm0
-; CHECK-AVX-NEXT:    retq
-  %urem = urem <4 x i32> %X, <i32 1, i32 1, i32 1, i32 1>
-  %cmp = icmp ne <4 x i32> %urem, <i32 0, i32 0, i32 0, i32 0>
-  %ret = zext <4 x i1> %cmp to <4 x i32>
-  ret <4 x i32> %ret
-}
diff --git a/llvm/test/CodeGen/X86/urem-seteq-vec-tautological.ll b/llvm/test/CodeGen/X86/urem-seteq-vec-tautological.ll
new file mode 100644
index 000000000..f361a399a
--- /dev/null
+++ b/llvm/test/CodeGen/X86/urem-seteq-vec-tautological.ll
@@ -0,0 +1,275 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu -mattr=+sse2 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-SSE,CHECK-SSE2
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu -mattr=+sse4.1 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-SSE,CHECK-SSE41
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu -mattr=+avx < %s | FileCheck %s --check-prefixes=CHECK,CHECK-AVX,CHECK-AVX1
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu -mattr=+avx2 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-AVX,CHECK-AVX2
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512f,+avx512vl < %s | FileCheck %s --check-prefixes=CHECK,CHECK-AVX,CHECK-AVX512VL
+
+define <4 x i1> @t0_all_tautological(<4 x i32> %X) nounwind {
+; CHECK-SSE-LABEL: t0_all_tautological:
+; CHECK-SSE:       # %bb.0:
+; CHECK-SSE-NEXT:    pand {{.*}}(%rip), %xmm0
+; CHECK-SSE-NEXT:    pcmpeqd {{.*}}(%rip), %xmm0
+; CHECK-SSE-NEXT:    retq
+;
+; CHECK-AVX-LABEL: t0_all_tautological:
+; CHECK-AVX:       # %bb.0:
+; CHECK-AVX-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX-NEXT:    vpcmpeqd {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX-NEXT:    retq
+  %urem = urem <4 x i32> %X, <i32 1, i32 1, i32 2, i32 2>
+  %cmp = icmp eq <4 x i32> %urem, <i32 0, i32 1, i32 2, i32 3>
+  ret <4 x i1> %cmp
+}
+
+define <4 x i1> @t1_all_odd_eq(<4 x i32> %X) nounwind {
+; CHECK-SSE2-LABEL: t1_all_odd_eq:
+; CHECK-SSE2:       # %bb.0:
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [2863311531,2863311531,2863311531,2863311531]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm0
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; CHECK-SSE2-NEXT:    pxor {{.*}}(%rip), %xmm0
+; CHECK-SSE2-NEXT:    pcmpgtd {{.*}}(%rip), %xmm0
+; CHECK-SSE2-NEXT:    pandn {{.*}}(%rip), %xmm0
+; CHECK-SSE2-NEXT:    retq
+;
+; CHECK-SSE41-LABEL: t1_all_odd_eq:
+; CHECK-SSE41:       # %bb.0:
+; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm0
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [1431655765,4294967295,4294967295,4294967295]
+; CHECK-SSE41-NEXT:    pminud %xmm0, %xmm1
+; CHECK-SSE41-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-SSE41-NEXT:    pxor %xmm1, %xmm1
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
+; CHECK-SSE41-NEXT:    retq
+;
+; CHECK-AVX1-LABEL: t1_all_odd_eq:
+; CHECK-AVX1:       # %bb.0:
+; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
+; CHECK-AVX1-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
+; CHECK-AVX1-NEXT:    retq
+;
+; CHECK-AVX2-LABEL: t1_all_odd_eq:
+; CHECK-AVX2:       # %bb.0:
+; CHECK-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [2863311531,2863311531,2863311531,2863311531]
+; CHECK-AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
+; CHECK-AVX2-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
+; CHECK-AVX2-NEXT:    retq
+;
+; CHECK-AVX512VL-LABEL: t1_all_odd_eq:
+; CHECK-AVX512VL:       # %bb.0:
+; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip){1to4}, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
+; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX512VL-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
+; CHECK-AVX512VL-NEXT:    retq
+  %urem = urem <4 x i32> %X, <i32 3, i32 1, i32 1, i32 9>
+  %cmp = icmp eq <4 x i32> %urem, <i32 0, i32 42, i32 0, i32 42>
+  ret <4 x i1> %cmp
+}
+
+define <4 x i1> @t1_all_odd_ne(<4 x i32> %X) nounwind {
+; CHECK-SSE2-LABEL: t1_all_odd_ne:
+; CHECK-SSE2:       # %bb.0:
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [2863311531,2863311531,2863311531,2863311531]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm0
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; CHECK-SSE2-NEXT:    pxor {{.*}}(%rip), %xmm0
+; CHECK-SSE2-NEXT:    pcmpgtd {{.*}}(%rip), %xmm0
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; CHECK-SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; CHECK-SSE2-NEXT:    retq
+;
+; CHECK-SSE41-LABEL: t1_all_odd_ne:
+; CHECK-SSE41:       # %bb.0:
+; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm0
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [1431655765,4294967295,4294967295,4294967295]
+; CHECK-SSE41-NEXT:    pminud %xmm0, %xmm1
+; CHECK-SSE41-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
+; CHECK-SSE41-NEXT:    pxor %xmm1, %xmm0
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
+; CHECK-SSE41-NEXT:    retq
+;
+; CHECK-AVX1-LABEL: t1_all_odd_ne:
+; CHECK-AVX1:       # %bb.0:
+; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
+; CHECK-AVX1-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
+; CHECK-AVX1-NEXT:    retq
+;
+; CHECK-AVX2-LABEL: t1_all_odd_ne:
+; CHECK-AVX2:       # %bb.0:
+; CHECK-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [2863311531,2863311531,2863311531,2863311531]
+; CHECK-AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
+; CHECK-AVX2-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
+; CHECK-AVX2-NEXT:    retq
+;
+; CHECK-AVX512VL-LABEL: t1_all_odd_ne:
+; CHECK-AVX512VL:       # %bb.0:
+; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip){1to4}, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm1
+; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
+; CHECK-AVX512VL-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
+; CHECK-AVX512VL-NEXT:    retq
+  %urem = urem <4 x i32> %X, <i32 3, i32 1, i32 1, i32 9>
+  %cmp = icmp ne <4 x i32> %urem, <i32 0, i32 42, i32 0, i32 42>
+  ret <4 x i1> %cmp
+}
+
+define <8 x i1> @t2_narrow(<8 x i16> %X) nounwind {
+; CHECK-SSE2-LABEL: t2_narrow:
+; CHECK-SSE2:       # %bb.0:
+; CHECK-SSE2-NEXT:    pmullw {{.*}}(%rip), %xmm0
+; CHECK-SSE2-NEXT:    psubusw {{.*}}(%rip), %xmm0
+; CHECK-SSE2-NEXT:    pxor %xmm1, %xmm1
+; CHECK-SSE2-NEXT:    pcmpeqw %xmm1, %xmm0
+; CHECK-SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
+; CHECK-SSE2-NEXT:    retq
+;
+; CHECK-SSE41-LABEL: t2_narrow:
+; CHECK-SSE41:       # %bb.0:
+; CHECK-SSE41-NEXT:    pmullw {{.*}}(%rip), %xmm0
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [21845,65535,65535,65535,21845,65535,65535,65535]
+; CHECK-SSE41-NEXT:    pminuw %xmm0, %xmm1
+; CHECK-SSE41-NEXT:    pcmpeqw %xmm1, %xmm0
+; CHECK-SSE41-NEXT:    pxor %xmm1, %xmm1
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
+; CHECK-SSE41-NEXT:    retq
+;
+; CHECK-AVX1-LABEL: t2_narrow:
+; CHECK-AVX1:       # %bb.0:
+; CHECK-AVX1-NEXT:    vpmullw {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpminuw {{.*}}(%rip), %xmm0, %xmm1
+; CHECK-AVX1-NEXT:    vpcmpeqw %xmm1, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
+; CHECK-AVX1-NEXT:    retq
+;
+; CHECK-AVX2-LABEL: t2_narrow:
+; CHECK-AVX2:       # %bb.0:
+; CHECK-AVX2-NEXT:    vpmullw {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpminuw {{.*}}(%rip), %xmm0, %xmm1
+; CHECK-AVX2-NEXT:    vpcmpeqw %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
+; CHECK-AVX2-NEXT:    retq
+;
+; CHECK-AVX512VL-LABEL: t2_narrow:
+; CHECK-AVX512VL:       # %bb.0:
+; CHECK-AVX512VL-NEXT:    vpmullw {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpminuw {{.*}}(%rip), %xmm0, %xmm1
+; CHECK-AVX512VL-NEXT:    vpcmpeqw %xmm1, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX512VL-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
+; CHECK-AVX512VL-NEXT:    retq
+  %urem = urem <8 x i16> %X, <i16 3, i16 1, i16 1, i16 9, i16 3, i16 1, i16 1, i16 9>
+  %cmp = icmp eq <8 x i16> %urem, <i16 0, i16 0, i16 42, i16 42, i16 0, i16 0, i16 42, i16 42>
+  ret <8 x i1> %cmp
+}
+
+define <2 x i1> @t3_wide(<2 x i64> %X) nounwind {
+; CHECK-SSE-LABEL: t3_wide:
+; CHECK-SSE:       # %bb.0:
+; CHECK-SSE-NEXT:    movdqa {{.*#+}} xmm1 = [12297829382473034411,12297829382473034411]
+; CHECK-SSE-NEXT:    movdqa %xmm0, %xmm2
+; CHECK-SSE-NEXT:    pmuludq %xmm1, %xmm2
+; CHECK-SSE-NEXT:    movdqa %xmm0, %xmm3
+; CHECK-SSE-NEXT:    psrlq $32, %xmm3
+; CHECK-SSE-NEXT:    pmuludq %xmm1, %xmm3
+; CHECK-SSE-NEXT:    pmuludq {{.*}}(%rip), %xmm0
+; CHECK-SSE-NEXT:    paddq %xmm3, %xmm0
+; CHECK-SSE-NEXT:    psllq $32, %xmm0
+; CHECK-SSE-NEXT:    paddq %xmm2, %xmm0
+; CHECK-SSE-NEXT:    pxor {{.*}}(%rip), %xmm0
+; CHECK-SSE-NEXT:    movdqa {{.*#+}} xmm1 = [15372286730238776661,9223372034707292159]
+; CHECK-SSE-NEXT:    movdqa %xmm0, %xmm2
+; CHECK-SSE-NEXT:    pcmpgtd %xmm1, %xmm2
+; CHECK-SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
+; CHECK-SSE-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; CHECK-SSE-NEXT:    pand %xmm3, %xmm0
+; CHECK-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,3,3]
+; CHECK-SSE-NEXT:    por %xmm0, %xmm1
+; CHECK-SSE-NEXT:    pcmpeqd %xmm0, %xmm0
+; CHECK-SSE-NEXT:    pxor %xmm1, %xmm0
+; CHECK-SSE-NEXT:    movq {{.*#+}} xmm0 = xmm0[0],zero
+; CHECK-SSE-NEXT:    retq
+;
+; CHECK-AVX1-LABEL: t3_wide:
+; CHECK-AVX1:       # %bb.0:
+; CHECK-AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [12297829382473034411,12297829382473034411]
+; CHECK-AVX1-NEXT:    vpmuludq %xmm1, %xmm0, %xmm2
+; CHECK-AVX1-NEXT:    vpsrlq $32, %xmm0, %xmm3
+; CHECK-AVX1-NEXT:    vpmuludq %xmm1, %xmm3, %xmm1
+; CHECK-AVX1-NEXT:    vpmuludq {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpsllq $32, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
+; CHECK-AVX1-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpcmpgtq {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; CHECK-AVX1-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
+; CHECK-AVX1-NEXT:    retq
+;
+; CHECK-AVX2-LABEL: t3_wide:
+; CHECK-AVX2:       # %bb.0:
+; CHECK-AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [12297829382473034411,12297829382473034411]
+; CHECK-AVX2-NEXT:    vpmuludq %xmm1, %xmm0, %xmm2
+; CHECK-AVX2-NEXT:    vpsrlq $32, %xmm0, %xmm3
+; CHECK-AVX2-NEXT:    vpmuludq %xmm1, %xmm3, %xmm1
+; CHECK-AVX2-NEXT:    vpmuludq {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpsllq $32, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
+; CHECK-AVX2-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpcmpgtq {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
+; CHECK-AVX2-NEXT:    retq
+;
+; CHECK-AVX512VL-LABEL: t3_wide:
+; CHECK-AVX512VL:       # %bb.0:
+; CHECK-AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm1 = [12297829382473034411,12297829382473034411]
+; CHECK-AVX512VL-NEXT:    vpmuludq %xmm1, %xmm0, %xmm2
+; CHECK-AVX512VL-NEXT:    vpsrlq $32, %xmm0, %xmm3
+; CHECK-AVX512VL-NEXT:    vpmuludq %xmm1, %xmm3, %xmm1
+; CHECK-AVX512VL-NEXT:    vpmuludq {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpsllq $32, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
+; CHECK-AVX512VL-NEXT:    vpminuq {{.*}}(%rip), %xmm0, %xmm1
+; CHECK-AVX512VL-NEXT:    vpcmpeqq %xmm1, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
+; CHECK-AVX512VL-NEXT:    retq
+  %urem = urem <2 x i64> %X, <i64 3, i64 1>
+  %cmp = icmp eq <2 x i64> %urem, <i64 0, i64 42>
+  ret <2 x i1> %cmp
+}
diff --git a/llvm/test/CodeGen/X86/urem-seteq.ll b/llvm/test/CodeGen/X86/urem-seteq.ll
index 26b9e85fe..1a16cf99b 100644
--- a/llvm/test/CodeGen/X86/urem-seteq.ll
+++ b/llvm/test/CodeGen/X86/urem-seteq.ll
@@ -6,10 +6,6 @@
 ; Odd divisors
 ;------------------------------------------------------------------------------;
 
-; This tests the BuildREMEqFold optimization with UREM, i32, odd divisor, SETEQ.
-; The corresponding pseudocode is:
-; Q <- [N * multInv(5, 2^32)] <=> [N * 0xCCCCCCCD] <=> [N * (-858993459)]
-; res <- [Q <= (2^32 - 1) / 5] <=> [Q <= 858993459] <=> [Q < 858993460]
 define i32 @test_urem_odd(i32 %X) nounwind {
 ; X86-LABEL: test_urem_odd:
 ; X86:       # %bb.0:
@@ -104,12 +100,6 @@ define i32 @test_urem_odd_bit31(i32 %X) nounwind {
 ; Even divisors
 ;------------------------------------------------------------------------------;
 
-; This tests the BuildREMEqFold optimization with UREM, i16, even divisor, SETNE.
-; In this case, D <=> 14 <=> 7 * 2^1, so D0 = 7 and K = 1.
-; The corresponding pseudocode is:
-; Q <- [N * multInv(D0, 2^16)] <=> [N * multInv(7, 2^16)] <=> [N * 28087]
-; Q <- [Q >>rot K] <=> [Q >>rot 1]
-; res <- ![Q <= (2^16 - 1) / 7] <=> ![Q <= 9362] <=> [Q > 9362]
 define i16 @test_urem_even(i16 %X) nounwind {
 ; X86-LABEL: test_urem_even:
 ; X86:       # %bb.0:
@@ -239,11 +229,57 @@ define i32 @test_urem_odd_setne(i32 %X) nounwind {
   ret i32 %ret
 }
 
+; The fold is only valid for positive divisors, negative-ones should be negated.
+define i32 @test_urem_negative_odd(i32 %X) nounwind {
+; X86-LABEL: test_urem_negative_odd:
+; X86:       # %bb.0:
+; X86-NEXT:    imull $858993459, {{[0-9]+}}(%esp), %ecx # imm = 0x33333333
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    cmpl $1, %ecx
+; X86-NEXT:    seta %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: test_urem_negative_odd:
+; X64:       # %bb.0:
+; X64-NEXT:    imull $858993459, %edi, %ecx # imm = 0x33333333
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpl $1, %ecx
+; X64-NEXT:    seta %al
+; X64-NEXT:    retq
+  %urem = urem i32 %X, -5
+  %cmp = icmp ne i32 %urem, 0
+  %ret = zext i1 %cmp to i32
+  ret i32 %ret
+}
+define i32 @test_urem_negative_even(i32 %X) nounwind {
+; X86-LABEL: test_urem_negative_even:
+; X86:       # %bb.0:
+; X86-NEXT:    imull $-920350135, {{[0-9]+}}(%esp), %ecx # imm = 0xC9249249
+; X86-NEXT:    rorl %ecx
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    cmpl $1, %ecx
+; X86-NEXT:    seta %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: test_urem_negative_even:
+; X64:       # %bb.0:
+; X64-NEXT:    imull $-920350135, %edi, %ecx # imm = 0xC9249249
+; X64-NEXT:    rorl %ecx
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpl $1, %ecx
+; X64-NEXT:    seta %al
+; X64-NEXT:    retq
+  %urem = urem i32 %X, -14
+  %cmp = icmp ne i32 %urem, 0
+  %ret = zext i1 %cmp to i32
+  ret i32 %ret
+}
+
 ;------------------------------------------------------------------------------;
 ; Negative tests
 ;------------------------------------------------------------------------------;
 
-; The fold is invalid if divisor is 1.
+; We can lower remainder of division by one much better elsewhere.
 define i32 @test_urem_one(i32 %X) nounwind {
 ; CHECK-LABEL: test_urem_one:
 ; CHECK:       # %bb.0:
@@ -255,46 +291,67 @@ define i32 @test_urem_one(i32 %X) nounwind {
   ret i32 %ret
 }
 
-; We can lower remainder of division by all-ones much better elsewhere.
-define i32 @test_urem_allones(i32 %X) nounwind {
-; X86-LABEL: test_urem_allones:
+; We can lower remainder of division by powers of two much better elsewhere.
+define i32 @test_urem_pow2(i32 %X) nounwind {
+; X86-LABEL: test_urem_pow2:
 ; X86:       # %bb.0:
-; X86-NEXT:    xorl %ecx, %ecx
-; X86-NEXT:    subl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    xorl %eax, %eax
-; X86-NEXT:    cmpl $2, %ecx
-; X86-NEXT:    setb %al
+; X86-NEXT:    testb $15, {{[0-9]+}}(%esp)
+; X86-NEXT:    sete %al
 ; X86-NEXT:    retl
 ;
-; X64-LABEL: test_urem_allones:
+; X64-LABEL: test_urem_pow2:
 ; X64:       # %bb.0:
-; X64-NEXT:    negl %edi
 ; X64-NEXT:    xorl %eax, %eax
-; X64-NEXT:    cmpl $2, %edi
-; X64-NEXT:    setb %al
+; X64-NEXT:    testb $15, %dil
+; X64-NEXT:    sete %al
 ; X64-NEXT:    retq
-  %urem = urem i32 %X, 4294967295
+  %urem = urem i32 %X, 16
   %cmp = icmp eq i32 %urem, 0
   %ret = zext i1 %cmp to i32
   ret i32 %ret
 }
 
-; We can lower remainder of division by powers of two much better elsewhere.
-define i32 @test_urem_pow2(i32 %X) nounwind {
-; X86-LABEL: test_urem_pow2:
+; The fold is only valid for positive divisors, and we can't negate INT_MIN.
+define i32 @test_urem_int_min(i32 %X) nounwind {
+; X86-LABEL: test_urem_int_min:
 ; X86:       # %bb.0:
 ; X86-NEXT:    xorl %eax, %eax
-; X86-NEXT:    testb $15, {{[0-9]+}}(%esp)
+; X86-NEXT:    testl $2147483647, {{[0-9]+}}(%esp) # imm = 0x7FFFFFFF
 ; X86-NEXT:    sete %al
 ; X86-NEXT:    retl
 ;
-; X64-LABEL: test_urem_pow2:
+; X64-LABEL: test_urem_int_min:
 ; X64:       # %bb.0:
 ; X64-NEXT:    xorl %eax, %eax
-; X64-NEXT:    testb $15, %dil
+; X64-NEXT:    testl $2147483647, %edi # imm = 0x7FFFFFFF
 ; X64-NEXT:    sete %al
 ; X64-NEXT:    retq
-  %urem = urem i32 %X, 16
+  %urem = urem i32 %X, 2147483648
+  %cmp = icmp eq i32 %urem, 0
+  %ret = zext i1 %cmp to i32
+  ret i32 %ret
+}
+
+; We can lower remainder of division by all-ones much better elsewhere.
+define i32 @test_urem_allones(i32 %X) nounwind {
+; X86-LABEL: test_urem_allones:
+; X86:       # %bb.0:
+; X86-NEXT:    xorl %ecx, %ecx
+; X86-NEXT:    subl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    cmpl $2, %ecx
+; X86-NEXT:    setb %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: test_urem_allones:
+; X64:       # %bb.0:
+; X64-NEXT:    negl %edi
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpl $2, %edi
+; X64-NEXT:    setb %al
+; X64-NEXT:    retq
+  %urem = urem i32 %X, 4294967295
   %cmp = icmp eq i32 %urem, 0
   %ret = zext i1 %cmp to i32
   ret i32 %ret
diff --git a/llvm/test/CodeGen/X86/urem-vector-lkk.ll b/llvm/test/CodeGen/X86/urem-vector-lkk.ll
new file mode 100644
index 000000000..65eb3557f
--- /dev/null
+++ b/llvm/test/CodeGen/X86/urem-vector-lkk.ll
@@ -0,0 +1,378 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx  | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX1
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
+
+define <4 x i16> @fold_urem_vec_1(<4 x i16> %x) {
+; SSE-LABEL: fold_urem_vec_1:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pextrw $1, %xmm0, %eax
+; SSE-NEXT:    movl %eax, %ecx
+; SSE-NEXT:    shrl $2, %ecx
+; SSE-NEXT:    imull $16913, %ecx, %ecx # imm = 0x4211
+; SSE-NEXT:    shrl $19, %ecx
+; SSE-NEXT:    imull $124, %ecx, %ecx
+; SSE-NEXT:    subl %ecx, %eax
+; SSE-NEXT:    movd %xmm0, %ecx
+; SSE-NEXT:    movzwl %cx, %edx
+; SSE-NEXT:    imull $44151, %edx, %edx # imm = 0xAC77
+; SSE-NEXT:    shrl $22, %edx
+; SSE-NEXT:    imull $95, %edx, %edx
+; SSE-NEXT:    subl %edx, %ecx
+; SSE-NEXT:    movd %ecx, %xmm1
+; SSE-NEXT:    pinsrw $1, %eax, %xmm1
+; SSE-NEXT:    pextrw $2, %xmm0, %eax
+; SSE-NEXT:    movl %eax, %ecx
+; SSE-NEXT:    shrl %ecx
+; SSE-NEXT:    imull $2675, %ecx, %ecx # imm = 0xA73
+; SSE-NEXT:    shrl $17, %ecx
+; SSE-NEXT:    imull $98, %ecx, %ecx
+; SSE-NEXT:    subl %ecx, %eax
+; SSE-NEXT:    pinsrw $2, %eax, %xmm1
+; SSE-NEXT:    pextrw $3, %xmm0, %eax
+; SSE-NEXT:    imull $1373, %eax, %ecx # imm = 0x55D
+; SSE-NEXT:    shrl $16, %ecx
+; SSE-NEXT:    movl %eax, %edx
+; SSE-NEXT:    subl %ecx, %edx
+; SSE-NEXT:    movzwl %dx, %edx
+; SSE-NEXT:    shrl %edx
+; SSE-NEXT:    addl %ecx, %edx
+; SSE-NEXT:    shrl $9, %edx
+; SSE-NEXT:    imull $1003, %edx, %ecx # imm = 0x3EB
+; SSE-NEXT:    subl %ecx, %eax
+; SSE-NEXT:    pinsrw $3, %eax, %xmm1
+; SSE-NEXT:    movdqa %xmm1, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: fold_urem_vec_1:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpextrw $1, %xmm0, %eax
+; AVX-NEXT:    movl %eax, %ecx
+; AVX-NEXT:    shrl $2, %ecx
+; AVX-NEXT:    imull $16913, %ecx, %ecx # imm = 0x4211
+; AVX-NEXT:    shrl $19, %ecx
+; AVX-NEXT:    imull $124, %ecx, %ecx
+; AVX-NEXT:    subl %ecx, %eax
+; AVX-NEXT:    vmovd %xmm0, %ecx
+; AVX-NEXT:    movzwl %cx, %edx
+; AVX-NEXT:    imull $44151, %edx, %edx # imm = 0xAC77
+; AVX-NEXT:    shrl $22, %edx
+; AVX-NEXT:    imull $95, %edx, %edx
+; AVX-NEXT:    subl %edx, %ecx
+; AVX-NEXT:    vmovd %ecx, %xmm1
+; AVX-NEXT:    vpinsrw $1, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrw $2, %xmm0, %eax
+; AVX-NEXT:    movl %eax, %ecx
+; AVX-NEXT:    shrl %ecx
+; AVX-NEXT:    imull $2675, %ecx, %ecx # imm = 0xA73
+; AVX-NEXT:    shrl $17, %ecx
+; AVX-NEXT:    imull $98, %ecx, %ecx
+; AVX-NEXT:    subl %ecx, %eax
+; AVX-NEXT:    vpinsrw $2, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrw $3, %xmm0, %eax
+; AVX-NEXT:    imull $1373, %eax, %ecx # imm = 0x55D
+; AVX-NEXT:    shrl $16, %ecx
+; AVX-NEXT:    movl %eax, %edx
+; AVX-NEXT:    subl %ecx, %edx
+; AVX-NEXT:    movzwl %dx, %edx
+; AVX-NEXT:    shrl %edx
+; AVX-NEXT:    addl %ecx, %edx
+; AVX-NEXT:    shrl $9, %edx
+; AVX-NEXT:    imull $1003, %edx, %ecx # imm = 0x3EB
+; AVX-NEXT:    subl %ecx, %eax
+; AVX-NEXT:    vpinsrw $3, %eax, %xmm1, %xmm0
+; AVX-NEXT:    retq
+  %1 = urem <4 x i16> %x, <i16 95, i16 124, i16 98, i16 1003>
+  ret <4 x i16> %1
+}
+
+define <4 x i16> @fold_urem_vec_2(<4 x i16> %x) {
+; SSE-LABEL: fold_urem_vec_2:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqa {{.*#+}} xmm1 = [44151,44151,44151,44151,44151,44151,44151,44151]
+; SSE-NEXT:    pmulhuw %xmm0, %xmm1
+; SSE-NEXT:    psrlw $6, %xmm1
+; SSE-NEXT:    pmullw {{.*}}(%rip), %xmm1
+; SSE-NEXT:    psubw %xmm1, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: fold_urem_vec_2:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpmulhuw {{.*}}(%rip), %xmm0, %xmm1
+; AVX-NEXT:    vpsrlw $6, %xmm1, %xmm1
+; AVX-NEXT:    vpmullw {{.*}}(%rip), %xmm1, %xmm1
+; AVX-NEXT:    vpsubw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %1 = urem <4 x i16> %x, <i16 95, i16 95, i16 95, i16 95>
+  ret <4 x i16> %1
+}
+
+
+; Don't fold if we can combine urem with udiv.
+define <4 x i16> @combine_urem_udiv(<4 x i16> %x) {
+; SSE-LABEL: combine_urem_udiv:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqa {{.*#+}} xmm1 = [44151,44151,44151,44151,44151,44151,44151,44151]
+; SSE-NEXT:    pmulhuw %xmm0, %xmm1
+; SSE-NEXT:    psrlw $6, %xmm1
+; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [95,95,95,95,95,95,95,95]
+; SSE-NEXT:    pmullw %xmm1, %xmm2
+; SSE-NEXT:    psubw %xmm2, %xmm0
+; SSE-NEXT:    paddw %xmm1, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: combine_urem_udiv:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpmulhuw {{.*}}(%rip), %xmm0, %xmm1
+; AVX-NEXT:    vpsrlw $6, %xmm1, %xmm1
+; AVX-NEXT:    vpmullw {{.*}}(%rip), %xmm1, %xmm2
+; AVX-NEXT:    vpsubw %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %1 = urem <4 x i16> %x, <i16 95, i16 95, i16 95, i16 95>
+  %2 = udiv <4 x i16> %x, <i16 95, i16 95, i16 95, i16 95>
+  %3 = add <4 x i16> %1, %2
+  ret <4 x i16> %3
+}
+
+; Don't fold for divisors that are a power of two.
+define <4 x i16> @dont_fold_urem_power_of_two(<4 x i16> %x) {
+; SSE-LABEL: dont_fold_urem_power_of_two:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pextrw $3, %xmm0, %eax
+; SSE-NEXT:    imull $44151, %eax, %ecx # imm = 0xAC77
+; SSE-NEXT:    shrl $22, %ecx
+; SSE-NEXT:    imull $95, %ecx, %ecx
+; SSE-NEXT:    subl %ecx, %eax
+; SSE-NEXT:    pextrw $1, %xmm0, %ecx
+; SSE-NEXT:    andl $31, %ecx
+; SSE-NEXT:    movd %xmm0, %edx
+; SSE-NEXT:    andl $63, %edx
+; SSE-NEXT:    movd %edx, %xmm1
+; SSE-NEXT:    pinsrw $1, %ecx, %xmm1
+; SSE-NEXT:    pextrw $2, %xmm0, %ecx
+; SSE-NEXT:    andl $7, %ecx
+; SSE-NEXT:    pinsrw $2, %ecx, %xmm1
+; SSE-NEXT:    pinsrw $3, %eax, %xmm1
+; SSE-NEXT:    movdqa %xmm1, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: dont_fold_urem_power_of_two:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpextrw $3, %xmm0, %eax
+; AVX-NEXT:    imull $44151, %eax, %ecx # imm = 0xAC77
+; AVX-NEXT:    shrl $22, %ecx
+; AVX-NEXT:    imull $95, %ecx, %ecx
+; AVX-NEXT:    subl %ecx, %eax
+; AVX-NEXT:    vpextrw $1, %xmm0, %ecx
+; AVX-NEXT:    andl $31, %ecx
+; AVX-NEXT:    vmovd %xmm0, %edx
+; AVX-NEXT:    andl $63, %edx
+; AVX-NEXT:    vmovd %edx, %xmm1
+; AVX-NEXT:    vpinsrw $1, %ecx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrw $2, %xmm0, %ecx
+; AVX-NEXT:    andl $7, %ecx
+; AVX-NEXT:    vpinsrw $2, %ecx, %xmm1, %xmm0
+; AVX-NEXT:    vpinsrw $3, %eax, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %1 = urem <4 x i16> %x, <i16 64, i16 32, i16 8, i16 95>
+  ret <4 x i16> %1
+}
+
+; Don't fold if the divisor is one.
+define <4 x i16> @dont_fold_urem_one(<4 x i16> %x) {
+; SSE-LABEL: dont_fold_urem_one:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pextrw $2, %xmm0, %eax
+; SSE-NEXT:    imull $25645, %eax, %ecx # imm = 0x642D
+; SSE-NEXT:    shrl $16, %ecx
+; SSE-NEXT:    movl %eax, %edx
+; SSE-NEXT:    subl %ecx, %edx
+; SSE-NEXT:    movzwl %dx, %edx
+; SSE-NEXT:    shrl %edx
+; SSE-NEXT:    addl %ecx, %edx
+; SSE-NEXT:    shrl $4, %edx
+; SSE-NEXT:    leal (%rdx,%rdx,2), %ecx
+; SSE-NEXT:    shll $3, %ecx
+; SSE-NEXT:    subl %ecx, %edx
+; SSE-NEXT:    addl %eax, %edx
+; SSE-NEXT:    pextrw $1, %xmm0, %eax
+; SSE-NEXT:    imull $51307, %eax, %ecx # imm = 0xC86B
+; SSE-NEXT:    shrl $25, %ecx
+; SSE-NEXT:    imull $654, %ecx, %ecx # imm = 0x28E
+; SSE-NEXT:    subl %ecx, %eax
+; SSE-NEXT:    pxor %xmm1, %xmm1
+; SSE-NEXT:    pinsrw $1, %eax, %xmm1
+; SSE-NEXT:    pinsrw $2, %edx, %xmm1
+; SSE-NEXT:    pextrw $3, %xmm0, %eax
+; SSE-NEXT:    imull $12375, %eax, %ecx # imm = 0x3057
+; SSE-NEXT:    shrl $26, %ecx
+; SSE-NEXT:    imull $5423, %ecx, %ecx # imm = 0x152F
+; SSE-NEXT:    subl %ecx, %eax
+; SSE-NEXT:    pinsrw $3, %eax, %xmm1
+; SSE-NEXT:    movdqa %xmm1, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: dont_fold_urem_one:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpextrw $2, %xmm0, %eax
+; AVX-NEXT:    imull $25645, %eax, %ecx # imm = 0x642D
+; AVX-NEXT:    shrl $16, %ecx
+; AVX-NEXT:    movl %eax, %edx
+; AVX-NEXT:    subl %ecx, %edx
+; AVX-NEXT:    movzwl %dx, %edx
+; AVX-NEXT:    shrl %edx
+; AVX-NEXT:    addl %ecx, %edx
+; AVX-NEXT:    shrl $4, %edx
+; AVX-NEXT:    leal (%rdx,%rdx,2), %ecx
+; AVX-NEXT:    shll $3, %ecx
+; AVX-NEXT:    subl %ecx, %edx
+; AVX-NEXT:    addl %eax, %edx
+; AVX-NEXT:    vpextrw $1, %xmm0, %eax
+; AVX-NEXT:    imull $51307, %eax, %ecx # imm = 0xC86B
+; AVX-NEXT:    shrl $25, %ecx
+; AVX-NEXT:    imull $654, %ecx, %ecx # imm = 0x28E
+; AVX-NEXT:    subl %ecx, %eax
+; AVX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vpinsrw $1, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpinsrw $2, %edx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrw $3, %xmm0, %eax
+; AVX-NEXT:    imull $12375, %eax, %ecx # imm = 0x3057
+; AVX-NEXT:    shrl $26, %ecx
+; AVX-NEXT:    imull $5423, %ecx, %ecx # imm = 0x152F
+; AVX-NEXT:    subl %ecx, %eax
+; AVX-NEXT:    vpinsrw $3, %eax, %xmm1, %xmm0
+; AVX-NEXT:    retq
+  %1 = urem <4 x i16> %x, <i16 1, i16 654, i16 23, i16 5423>
+  ret <4 x i16> %1
+}
+
+; Don't fold if the divisor is 2^16.
+define <4 x i16> @dont_fold_urem_i16_smax(<4 x i16> %x) {
+; CHECK-LABEL: dont_fold_urem_i16_smax:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    retq
+  %1 = urem <4 x i16> %x, <i16 1, i16 65536, i16 23, i16 5423>
+  ret <4 x i16> %1
+}
+
+; Don't fold i64 urem.
+define <4 x i64> @dont_fold_urem_i64(<4 x i64> %x) {
+; SSE-LABEL: dont_fold_urem_i64:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movq %xmm1, %rcx
+; SSE-NEXT:    movabsq $7218291159277650633, %rdx # imm = 0x642C8590B21642C9
+; SSE-NEXT:    movq %rcx, %rax
+; SSE-NEXT:    mulq %rdx
+; SSE-NEXT:    movq %rcx, %rax
+; SSE-NEXT:    subq %rdx, %rax
+; SSE-NEXT:    shrq %rax
+; SSE-NEXT:    addq %rdx, %rax
+; SSE-NEXT:    shrq $4, %rax
+; SSE-NEXT:    leaq (%rax,%rax,2), %rdx
+; SSE-NEXT:    shlq $3, %rdx
+; SSE-NEXT:    subq %rdx, %rax
+; SSE-NEXT:    addq %rcx, %rax
+; SSE-NEXT:    movq %rax, %xmm2
+; SSE-NEXT:    pextrq $1, %xmm1, %rcx
+; SSE-NEXT:    movabsq $-4513890722074972339, %rdx # imm = 0xC15B704DCBCA2F4D
+; SSE-NEXT:    movq %rcx, %rax
+; SSE-NEXT:    mulq %rdx
+; SSE-NEXT:    shrq $12, %rdx
+; SSE-NEXT:    imulq $5423, %rdx, %rax # imm = 0x152F
+; SSE-NEXT:    subq %rax, %rcx
+; SSE-NEXT:    movq %rcx, %xmm1
+; SSE-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm1[0]
+; SSE-NEXT:    pextrq $1, %xmm0, %rcx
+; SSE-NEXT:    movq %rcx, %rax
+; SSE-NEXT:    shrq %rax
+; SSE-NEXT:    movabsq $7220743857598845893, %rdx # imm = 0x64353C48064353C5
+; SSE-NEXT:    mulq %rdx
+; SSE-NEXT:    shrq $7, %rdx
+; SSE-NEXT:    imulq $654, %rdx, %rax # imm = 0x28E
+; SSE-NEXT:    subq %rax, %rcx
+; SSE-NEXT:    movq %rcx, %xmm0
+; SSE-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6,7]
+; SSE-NEXT:    movdqa %xmm2, %xmm1
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: dont_fold_urem_i64:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vmovq %xmm1, %rcx
+; AVX1-NEXT:    movabsq $7218291159277650633, %rdx # imm = 0x642C8590B21642C9
+; AVX1-NEXT:    movq %rcx, %rax
+; AVX1-NEXT:    mulq %rdx
+; AVX1-NEXT:    movq %rcx, %rax
+; AVX1-NEXT:    subq %rdx, %rax
+; AVX1-NEXT:    shrq %rax
+; AVX1-NEXT:    addq %rdx, %rax
+; AVX1-NEXT:    shrq $4, %rax
+; AVX1-NEXT:    leaq (%rax,%rax,2), %rdx
+; AVX1-NEXT:    shlq $3, %rdx
+; AVX1-NEXT:    subq %rdx, %rax
+; AVX1-NEXT:    addq %rcx, %rax
+; AVX1-NEXT:    vmovq %rax, %xmm2
+; AVX1-NEXT:    vpextrq $1, %xmm1, %rcx
+; AVX1-NEXT:    movabsq $-4513890722074972339, %rdx # imm = 0xC15B704DCBCA2F4D
+; AVX1-NEXT:    movq %rcx, %rax
+; AVX1-NEXT:    mulq %rdx
+; AVX1-NEXT:    shrq $12, %rdx
+; AVX1-NEXT:    imulq $5423, %rdx, %rax # imm = 0x152F
+; AVX1-NEXT:    subq %rax, %rcx
+; AVX1-NEXT:    vmovq %rcx, %xmm1
+; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX1-NEXT:    vpextrq $1, %xmm0, %rcx
+; AVX1-NEXT:    movq %rcx, %rax
+; AVX1-NEXT:    shrq %rax
+; AVX1-NEXT:    movabsq $7220743857598845893, %rdx # imm = 0x64353C48064353C5
+; AVX1-NEXT:    mulq %rdx
+; AVX1-NEXT:    shrq $7, %rdx
+; AVX1-NEXT:    imulq $654, %rdx, %rax # imm = 0x28E
+; AVX1-NEXT:    subq %rax, %rcx
+; AVX1-NEXT:    vmovq %rcx, %xmm0
+; AVX1-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6,7]
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: dont_fold_urem_i64:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vmovq %xmm1, %rcx
+; AVX2-NEXT:    movabsq $7218291159277650633, %rdx # imm = 0x642C8590B21642C9
+; AVX2-NEXT:    movq %rcx, %rax
+; AVX2-NEXT:    mulq %rdx
+; AVX2-NEXT:    movq %rcx, %rax
+; AVX2-NEXT:    subq %rdx, %rax
+; AVX2-NEXT:    shrq %rax
+; AVX2-NEXT:    addq %rdx, %rax
+; AVX2-NEXT:    shrq $4, %rax
+; AVX2-NEXT:    leaq (%rax,%rax,2), %rdx
+; AVX2-NEXT:    shlq $3, %rdx
+; AVX2-NEXT:    subq %rdx, %rax
+; AVX2-NEXT:    addq %rcx, %rax
+; AVX2-NEXT:    vmovq %rax, %xmm2
+; AVX2-NEXT:    vpextrq $1, %xmm1, %rcx
+; AVX2-NEXT:    movabsq $-4513890722074972339, %rdx # imm = 0xC15B704DCBCA2F4D
+; AVX2-NEXT:    movq %rcx, %rax
+; AVX2-NEXT:    mulq %rdx
+; AVX2-NEXT:    shrq $12, %rdx
+; AVX2-NEXT:    imulq $5423, %rdx, %rax # imm = 0x152F
+; AVX2-NEXT:    subq %rax, %rcx
+; AVX2-NEXT:    vmovq %rcx, %xmm1
+; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX2-NEXT:    vpextrq $1, %xmm0, %rcx
+; AVX2-NEXT:    movq %rcx, %rax
+; AVX2-NEXT:    shrq %rax
+; AVX2-NEXT:    movabsq $7220743857598845893, %rdx # imm = 0x64353C48064353C5
+; AVX2-NEXT:    mulq %rdx
+; AVX2-NEXT:    shrq $7, %rdx
+; AVX2-NEXT:    imulq $654, %rdx, %rax # imm = 0x28E
+; AVX2-NEXT:    subq %rax, %rcx
+; AVX2-NEXT:    vmovq %rcx, %xmm0
+; AVX2-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6,7]
+; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %1 = urem <4 x i64> %x, <i64 1, i64 654, i64 23, i64 5423>
+  ret <4 x i64> %1
+}
\ No newline at end of file
diff --git a/llvm/test/CodeGen/X86/use-cr-result-of-dom-icmp-st.ll b/llvm/test/CodeGen/X86/use-cr-result-of-dom-icmp-st.ll
new file mode 100644
index 000000000..baae1ab53
--- /dev/null
+++ b/llvm/test/CodeGen/X86/use-cr-result-of-dom-icmp-st.ll
@@ -0,0 +1,615 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=x86_64-unknown-unknown -O3 -cgp-icmp-eq2icmp-st -verify-machineinstrs < %s | FileCheck %s
+
+; Test cases are generated from:
+; long long NAME(PARAM a, PARAM b) {
+;   if (LHS > RHS)
+;     return b;
+;   if (LHS < RHS)
+;     return a;\
+;   return a * b;
+; }
+; Please note funtion name is defined as <PARAM>_<LHS>_<RHS>. Take ll_a_op_b__1
+; for example. ll is PARAM, a_op_b (i.e., a << b) is LHS, _1 (i.e., -1) is RHS.
+
+target datalayout = "e-m:e-i64:64-n32:64"
+
+define i64 @ll_a_op_b__2(i64 %a, i64 %b) {
+; CHECK-LABEL: ll_a_op_b__2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movq %rsi, %rax
+; CHECK-NEXT:    movq %rdi, %rdx
+; CHECK-NEXT:    movl %eax, %ecx
+; CHECK-NEXT:    shlq %cl, %rdx
+; CHECK-NEXT:    cmpq $-2, %rdx
+; CHECK-NEXT:    jg .LBB0_2
+; CHECK-NEXT:  # %bb.1: # %if.end
+; CHECK-NEXT:    movl $1, %ecx
+; CHECK-NEXT:    cmovlq %rcx, %rax
+; CHECK-NEXT:    imulq %rdi, %rax
+; CHECK-NEXT:  .LBB0_2: # %return
+; CHECK-NEXT:    retq
+entry:
+  %shl = shl i64 %a, %b
+  %cmp = icmp sgt i64 %shl, -2
+  br i1 %cmp, label %return, label %if.end
+
+if.end:                                           ; preds = %entry
+  %cmp2 = icmp eq i64 %shl, -2
+  %mul = select i1 %cmp2, i64 %b, i64 1
+  %spec.select = mul nsw i64 %mul, %a
+  ret i64 %spec.select
+
+return:                                           ; preds = %entry
+  ret i64 %b
+}
+
+define i64 @ll_a_op_b__1(i64 %a, i64 %b) {
+; CHECK-LABEL: ll_a_op_b__1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movq %rsi, %rax
+; CHECK-NEXT:    movq %rdi, %rdx
+; CHECK-NEXT:    movl %eax, %ecx
+; CHECK-NEXT:    shlq %cl, %rdx
+; CHECK-NEXT:    testq %rdx, %rdx
+; CHECK-NEXT:    js .LBB1_1
+; CHECK-NEXT:  # %bb.2: # %return
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  .LBB1_1: # %if.end
+; CHECK-NEXT:    cmpq $-1, %rdx
+; CHECK-NEXT:    movl $1, %ecx
+; CHECK-NEXT:    cmovlq %rcx, %rax
+; CHECK-NEXT:    imulq %rdi, %rax
+; CHECK-NEXT:    retq
+entry:
+  %shl = shl i64 %a, %b
+  %cmp = icmp sgt i64 %shl, -1
+  br i1 %cmp, label %return, label %if.end
+
+if.end:                                           ; preds = %entry
+  %cmp2 = icmp eq i64 %shl, -1
+  %mul = select i1 %cmp2, i64 %b, i64 1
+  %spec.select = mul nsw i64 %mul, %a
+  ret i64 %spec.select
+
+return:                                           ; preds = %entry
+  ret i64 %b
+}
+
+define i64 @ll_a_op_b_0(i64 %a, i64 %b) {
+; CHECK-LABEL: ll_a_op_b_0:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movq %rsi, %rax
+; CHECK-NEXT:    movq %rdi, %rdx
+; CHECK-NEXT:    movl %eax, %ecx
+; CHECK-NEXT:    shlq %cl, %rdx
+; CHECK-NEXT:    testq %rdx, %rdx
+; CHECK-NEXT:    jle .LBB2_1
+; CHECK-NEXT:  # %bb.2: # %return
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  .LBB2_1: # %if.end
+; CHECK-NEXT:    movl $1, %ecx
+; CHECK-NEXT:    cmovsq %rcx, %rax
+; CHECK-NEXT:    imulq %rdi, %rax
+; CHECK-NEXT:    retq
+entry:
+  %shl = shl i64 %a, %b
+  %cmp = icmp sgt i64 %shl, 0
+  br i1 %cmp, label %return, label %if.end
+
+if.end:                                           ; preds = %entry
+  %cmp2 = icmp eq i64 %shl, 0
+  %mul = select i1 %cmp2, i64 %b, i64 1
+  %spec.select = mul nsw i64 %mul, %a
+  ret i64 %spec.select
+
+return:                                           ; preds = %entry
+  ret i64 %b
+}
+
+define i64 @ll_a_op_b_1(i64 %a, i64 %b) {
+; CHECK-LABEL: ll_a_op_b_1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movq %rsi, %rax
+; CHECK-NEXT:    movq %rdi, %rdx
+; CHECK-NEXT:    movl %eax, %ecx
+; CHECK-NEXT:    shlq %cl, %rdx
+; CHECK-NEXT:    cmpq $1, %rdx
+; CHECK-NEXT:    jg .LBB3_2
+; CHECK-NEXT:  # %bb.1: # %if.end
+; CHECK-NEXT:    testq %rdx, %rdx
+; CHECK-NEXT:    movl $1, %ecx
+; CHECK-NEXT:    cmovleq %rcx, %rax
+; CHECK-NEXT:    imulq %rdi, %rax
+; CHECK-NEXT:  .LBB3_2: # %return
+; CHECK-NEXT:    retq
+entry:
+  %shl = shl i64 %a, %b
+  %cmp = icmp sgt i64 %shl, 1
+  br i1 %cmp, label %return, label %if.end
+
+if.end:                                           ; preds = %entry
+  %cmp2 = icmp eq i64 %shl, 1
+  %mul = select i1 %cmp2, i64 %b, i64 1
+  %spec.select = mul nsw i64 %mul, %a
+  ret i64 %spec.select
+
+return:                                           ; preds = %entry
+  ret i64 %b
+}
+
+define i64 @ll_a_op_b_2(i64 %a, i64 %b) {
+; CHECK-LABEL: ll_a_op_b_2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movq %rsi, %rax
+; CHECK-NEXT:    movq %rdi, %rdx
+; CHECK-NEXT:    movl %eax, %ecx
+; CHECK-NEXT:    shlq %cl, %rdx
+; CHECK-NEXT:    cmpq $2, %rdx
+; CHECK-NEXT:    jg .LBB4_2
+; CHECK-NEXT:  # %bb.1: # %if.end
+; CHECK-NEXT:    movl $1, %ecx
+; CHECK-NEXT:    cmovlq %rcx, %rax
+; CHECK-NEXT:    imulq %rdi, %rax
+; CHECK-NEXT:  .LBB4_2: # %return
+; CHECK-NEXT:    retq
+entry:
+  %shl = shl i64 %a, %b
+  %cmp = icmp sgt i64 %shl, 2
+  br i1 %cmp, label %return, label %if.end
+
+if.end:                                           ; preds = %entry
+  %cmp2 = icmp eq i64 %shl, 2
+  %mul = select i1 %cmp2, i64 %b, i64 1
+  %spec.select = mul nsw i64 %mul, %a
+  ret i64 %spec.select
+
+return:                                           ; preds = %entry
+  ret i64 %b
+}
+
+define i64 @ll_a__2(i64 %a, i64 %b) {
+; CHECK-LABEL: ll_a__2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movq %rsi, %rax
+; CHECK-NEXT:    cmpq $-2, %rdi
+; CHECK-NEXT:    jg .LBB5_2
+; CHECK-NEXT:  # %bb.1: # %if.end
+; CHECK-NEXT:    movl $1, %ecx
+; CHECK-NEXT:    cmovlq %rcx, %rax
+; CHECK-NEXT:    imulq %rdi, %rax
+; CHECK-NEXT:  .LBB5_2: # %return
+; CHECK-NEXT:    retq
+entry:
+  %cmp = icmp sgt i64 %a, -2
+  br i1 %cmp, label %return, label %if.end
+
+if.end:                                           ; preds = %entry
+  %cmp1 = icmp eq i64 %a, -2
+  %mul = select i1 %cmp1, i64 %b, i64 1
+  %spec.select = mul nsw i64 %mul, %a
+  ret i64 %spec.select
+
+return:                                           ; preds = %entry
+  ret i64 %b
+}
+
+define i64 @ll_a__1(i64 %a, i64 %b) {
+; CHECK-LABEL: ll_a__1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movq %rsi, %rax
+; CHECK-NEXT:    testq %rdi, %rdi
+; CHECK-NEXT:    js .LBB6_1
+; CHECK-NEXT:  # %bb.2: # %return
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  .LBB6_1: # %if.end
+; CHECK-NEXT:    cmpq $-1, %rdi
+; CHECK-NEXT:    movl $1, %ecx
+; CHECK-NEXT:    cmovlq %rcx, %rax
+; CHECK-NEXT:    imulq %rdi, %rax
+; CHECK-NEXT:    retq
+entry:
+  %cmp = icmp sgt i64 %a, -1
+  br i1 %cmp, label %return, label %if.end
+
+if.end:                                           ; preds = %entry
+  %cmp1 = icmp eq i64 %a, -1
+  %mul = select i1 %cmp1, i64 %b, i64 1
+  %spec.select = mul nsw i64 %mul, %a
+  ret i64 %spec.select
+
+return:                                           ; preds = %entry
+  ret i64 %b
+}
+
+define i64 @ll_a_0(i64 %a, i64 %b) {
+; CHECK-LABEL: ll_a_0:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movq %rsi, %rax
+; CHECK-NEXT:    testq %rdi, %rdi
+; CHECK-NEXT:    jle .LBB7_1
+; CHECK-NEXT:  # %bb.2: # %return
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  .LBB7_1: # %if.end
+; CHECK-NEXT:    movl $1, %ecx
+; CHECK-NEXT:    cmovsq %rcx, %rax
+; CHECK-NEXT:    imulq %rdi, %rax
+; CHECK-NEXT:    retq
+entry:
+  %cmp = icmp sgt i64 %a, 0
+  br i1 %cmp, label %return, label %if.end
+
+if.end:                                           ; preds = %entry
+  %cmp1 = icmp eq i64 %a, 0
+  %mul = select i1 %cmp1, i64 %b, i64 1
+  %spec.select = mul nsw i64 %mul, %a
+  ret i64 %spec.select
+
+return:                                           ; preds = %entry
+  ret i64 %b
+}
+
+define i64 @ll_a_1(i64 %a, i64 %b) {
+; CHECK-LABEL: ll_a_1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movq %rsi, %rax
+; CHECK-NEXT:    cmpq $1, %rdi
+; CHECK-NEXT:    jg .LBB8_2
+; CHECK-NEXT:  # %bb.1: # %if.end
+; CHECK-NEXT:    testq %rdi, %rdi
+; CHECK-NEXT:    movl $1, %ecx
+; CHECK-NEXT:    cmovleq %rcx, %rax
+; CHECK-NEXT:    imulq %rdi, %rax
+; CHECK-NEXT:  .LBB8_2: # %return
+; CHECK-NEXT:    retq
+entry:
+  %cmp = icmp sgt i64 %a, 1
+  br i1 %cmp, label %return, label %if.end
+
+if.end:                                           ; preds = %entry
+  %cmp1 = icmp eq i64 %a, 1
+  %mul = select i1 %cmp1, i64 %b, i64 1
+  %spec.select = mul nsw i64 %mul, %a
+  ret i64 %spec.select
+
+return:                                           ; preds = %entry
+  ret i64 %b
+}
+
+define i64 @ll_a_2(i64 %a, i64 %b) {
+; CHECK-LABEL: ll_a_2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movq %rsi, %rax
+; CHECK-NEXT:    cmpq $2, %rdi
+; CHECK-NEXT:    jg .LBB9_2
+; CHECK-NEXT:  # %bb.1: # %if.end
+; CHECK-NEXT:    movl $1, %ecx
+; CHECK-NEXT:    cmovlq %rcx, %rax
+; CHECK-NEXT:    imulq %rdi, %rax
+; CHECK-NEXT:  .LBB9_2: # %return
+; CHECK-NEXT:    retq
+entry:
+  %cmp = icmp sgt i64 %a, 2
+  br i1 %cmp, label %return, label %if.end
+
+if.end:                                           ; preds = %entry
+  %cmp1 = icmp eq i64 %a, 2
+  %mul = select i1 %cmp1, i64 %b, i64 1
+  %spec.select = mul nsw i64 %mul, %a
+  ret i64 %spec.select
+
+return:                                           ; preds = %entry
+  ret i64 %b
+}
+
+define i64 @i_a_op_b__2(i32 signext %a, i32 signext %b) {
+; CHECK-LABEL: i_a_op_b__2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movl %esi, %ecx
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    shll %cl, %eax
+; CHECK-NEXT:    cmpl $-2, %eax
+; CHECK-NEXT:    jg .LBB10_2
+; CHECK-NEXT:  # %bb.1: # %if.end
+; CHECK-NEXT:    movl $1, %eax
+; CHECK-NEXT:    cmovll %eax, %ecx
+; CHECK-NEXT:    imull %edi, %ecx
+; CHECK-NEXT:  .LBB10_2: # %return
+; CHECK-NEXT:    movslq %ecx, %rax
+; CHECK-NEXT:    retq
+entry:
+  %shl = shl i32 %a, %b
+  %cmp = icmp sgt i32 %shl, -2
+  br i1 %cmp, label %return, label %if.end
+
+if.end:                                           ; preds = %entry
+  %cmp2 = icmp eq i32 %shl, -2
+  %mul = select i1 %cmp2, i32 %b, i32 1
+  %spec.select = mul nsw i32 %mul, %a
+  br label %return
+
+return:                                           ; preds = %if.end, %entry
+  %retval.0.in = phi i32 [ %b, %entry ], [ %spec.select, %if.end ]
+  %retval.0 = sext i32 %retval.0.in to i64
+  ret i64 %retval.0
+}
+
+define i64 @i_a_op_b__1(i32 signext %a, i32 signext %b) {
+; CHECK-LABEL: i_a_op_b__1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movl %esi, %ecx
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    shll %cl, %eax
+; CHECK-NEXT:    testl %eax, %eax
+; CHECK-NEXT:    js .LBB11_1
+; CHECK-NEXT:  # %bb.2: # %return
+; CHECK-NEXT:    movslq %ecx, %rax
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  .LBB11_1: # %if.end
+; CHECK-NEXT:    cmpl $-1, %eax
+; CHECK-NEXT:    movl $1, %eax
+; CHECK-NEXT:    cmovll %eax, %ecx
+; CHECK-NEXT:    imull %edi, %ecx
+; CHECK-NEXT:    movslq %ecx, %rax
+; CHECK-NEXT:    retq
+entry:
+  %shl = shl i32 %a, %b
+  %cmp = icmp sgt i32 %shl, -1
+  br i1 %cmp, label %return, label %if.end
+
+if.end:                                           ; preds = %entry
+  %cmp2 = icmp eq i32 %shl, -1
+  %mul = select i1 %cmp2, i32 %b, i32 1
+  %spec.select = mul nsw i32 %mul, %a
+  br label %return
+
+return:                                           ; preds = %if.end, %entry
+  %retval.0.in = phi i32 [ %b, %entry ], [ %spec.select, %if.end ]
+  %retval.0 = sext i32 %retval.0.in to i64
+  ret i64 %retval.0
+}
+
+define i64 @i_a_op_b_0(i32 signext %a, i32 signext %b) {
+; CHECK-LABEL: i_a_op_b_0:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movl %esi, %ecx
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    shll %cl, %eax
+; CHECK-NEXT:    testl %eax, %eax
+; CHECK-NEXT:    jle .LBB12_1
+; CHECK-NEXT:  # %bb.2: # %return
+; CHECK-NEXT:    movslq %ecx, %rax
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  .LBB12_1: # %if.end
+; CHECK-NEXT:    movl $1, %eax
+; CHECK-NEXT:    cmovsl %eax, %ecx
+; CHECK-NEXT:    imull %edi, %ecx
+; CHECK-NEXT:    movslq %ecx, %rax
+; CHECK-NEXT:    retq
+entry:
+  %shl = shl i32 %a, %b
+  %cmp = icmp sgt i32 %shl, 0
+  br i1 %cmp, label %return, label %if.end
+
+if.end:                                           ; preds = %entry
+  %cmp2 = icmp eq i32 %shl, 0
+  %mul = select i1 %cmp2, i32 %b, i32 1
+  %spec.select = mul nsw i32 %mul, %a
+  br label %return
+
+return:                                           ; preds = %if.end, %entry
+  %retval.0.in = phi i32 [ %b, %entry ], [ %spec.select, %if.end ]
+  %retval.0 = sext i32 %retval.0.in to i64
+  ret i64 %retval.0
+}
+
+define i64 @i_a_op_b_1(i32 signext %a, i32 signext %b) {
+; CHECK-LABEL: i_a_op_b_1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movl %esi, %ecx
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    shll %cl, %eax
+; CHECK-NEXT:    cmpl $1, %eax
+; CHECK-NEXT:    jg .LBB13_2
+; CHECK-NEXT:  # %bb.1: # %if.end
+; CHECK-NEXT:    testl %eax, %eax
+; CHECK-NEXT:    movl $1, %eax
+; CHECK-NEXT:    cmovlel %eax, %ecx
+; CHECK-NEXT:    imull %edi, %ecx
+; CHECK-NEXT:  .LBB13_2: # %return
+; CHECK-NEXT:    movslq %ecx, %rax
+; CHECK-NEXT:    retq
+entry:
+  %shl = shl i32 %a, %b
+  %cmp = icmp sgt i32 %shl, 1
+  br i1 %cmp, label %return, label %if.end
+
+if.end:                                           ; preds = %entry
+  %cmp2 = icmp eq i32 %shl, 1
+  %mul = select i1 %cmp2, i32 %b, i32 1
+  %spec.select = mul nsw i32 %mul, %a
+  br label %return
+
+return:                                           ; preds = %if.end, %entry
+  %retval.0.in = phi i32 [ %b, %entry ], [ %spec.select, %if.end ]
+  %retval.0 = sext i32 %retval.0.in to i64
+  ret i64 %retval.0
+}
+
+define i64 @i_a_op_b_2(i32 signext %a, i32 signext %b) {
+; CHECK-LABEL: i_a_op_b_2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movl %esi, %ecx
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    shll %cl, %eax
+; CHECK-NEXT:    cmpl $2, %eax
+; CHECK-NEXT:    jg .LBB14_2
+; CHECK-NEXT:  # %bb.1: # %if.end
+; CHECK-NEXT:    movl $1, %eax
+; CHECK-NEXT:    cmovll %eax, %ecx
+; CHECK-NEXT:    imull %edi, %ecx
+; CHECK-NEXT:  .LBB14_2: # %return
+; CHECK-NEXT:    movslq %ecx, %rax
+; CHECK-NEXT:    retq
+entry:
+  %shl = shl i32 %a, %b
+  %cmp = icmp sgt i32 %shl, 2
+  br i1 %cmp, label %return, label %if.end
+
+if.end:                                           ; preds = %entry
+  %cmp2 = icmp eq i32 %shl, 2
+  %mul = select i1 %cmp2, i32 %b, i32 1
+  %spec.select = mul nsw i32 %mul, %a
+  br label %return
+
+return:                                           ; preds = %if.end, %entry
+  %retval.0.in = phi i32 [ %b, %entry ], [ %spec.select, %if.end ]
+  %retval.0 = sext i32 %retval.0.in to i64
+  ret i64 %retval.0
+}
+
+define i64 @i_a__2(i32 signext %a, i32 signext %b) {
+; CHECK-LABEL: i_a__2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cmpl $-2, %edi
+; CHECK-NEXT:    jg .LBB15_2
+; CHECK-NEXT:  # %bb.1: # %if.end
+; CHECK-NEXT:    movl $1, %eax
+; CHECK-NEXT:    cmovll %eax, %esi
+; CHECK-NEXT:    imull %edi, %esi
+; CHECK-NEXT:  .LBB15_2: # %return
+; CHECK-NEXT:    movslq %esi, %rax
+; CHECK-NEXT:    retq
+entry:
+  %cmp = icmp sgt i32 %a, -2
+  br i1 %cmp, label %return, label %if.end
+
+if.end:                                           ; preds = %entry
+  %cmp1 = icmp eq i32 %a, -2
+  %mul = select i1 %cmp1, i32 %b, i32 1
+  %spec.select = mul nsw i32 %mul, %a
+  br label %return
+
+return:                                           ; preds = %if.end, %entry
+  %retval.0.in = phi i32 [ %b, %entry ], [ %spec.select, %if.end ]
+  %retval.0 = sext i32 %retval.0.in to i64
+  ret i64 %retval.0
+}
+
+define i64 @i_a__1(i32 signext %a, i32 signext %b) {
+; CHECK-LABEL: i_a__1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    testl %edi, %edi
+; CHECK-NEXT:    js .LBB16_1
+; CHECK-NEXT:  # %bb.2: # %return
+; CHECK-NEXT:    movslq %esi, %rax
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  .LBB16_1: # %if.end
+; CHECK-NEXT:    cmpl $-1, %edi
+; CHECK-NEXT:    movl $1, %eax
+; CHECK-NEXT:    cmovll %eax, %esi
+; CHECK-NEXT:    imull %edi, %esi
+; CHECK-NEXT:    movslq %esi, %rax
+; CHECK-NEXT:    retq
+entry:
+  %cmp = icmp sgt i32 %a, -1
+  br i1 %cmp, label %return, label %if.end
+
+if.end:                                           ; preds = %entry
+  %cmp1 = icmp eq i32 %a, -1
+  %mul = select i1 %cmp1, i32 %b, i32 1
+  %spec.select = mul nsw i32 %mul, %a
+  br label %return
+
+return:                                           ; preds = %if.end, %entry
+  %retval.0.in = phi i32 [ %b, %entry ], [ %spec.select, %if.end ]
+  %retval.0 = sext i32 %retval.0.in to i64
+  ret i64 %retval.0
+}
+
+define i64 @i_a_0(i32 signext %a, i32 signext %b) {
+; CHECK-LABEL: i_a_0:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    testl %edi, %edi
+; CHECK-NEXT:    jle .LBB17_1
+; CHECK-NEXT:  # %bb.2: # %return
+; CHECK-NEXT:    movslq %esi, %rax
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  .LBB17_1: # %if.end
+; CHECK-NEXT:    movl $1, %eax
+; CHECK-NEXT:    cmovsl %eax, %esi
+; CHECK-NEXT:    imull %edi, %esi
+; CHECK-NEXT:    movslq %esi, %rax
+; CHECK-NEXT:    retq
+entry:
+  %cmp = icmp sgt i32 %a, 0
+  br i1 %cmp, label %return, label %if.end
+
+if.end:                                           ; preds = %entry
+  %cmp1 = icmp eq i32 %a, 0
+  %mul = select i1 %cmp1, i32 %b, i32 1
+  %spec.select = mul nsw i32 %mul, %a
+  br label %return
+
+return:                                           ; preds = %if.end, %entry
+  %retval.0.in = phi i32 [ %b, %entry ], [ %spec.select, %if.end ]
+  %retval.0 = sext i32 %retval.0.in to i64
+  ret i64 %retval.0
+}
+
+define i64 @i_a_1(i32 signext %a, i32 signext %b) {
+; CHECK-LABEL: i_a_1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cmpl $1, %edi
+; CHECK-NEXT:    jg .LBB18_2
+; CHECK-NEXT:  # %bb.1: # %if.end
+; CHECK-NEXT:    testl %edi, %edi
+; CHECK-NEXT:    movl $1, %eax
+; CHECK-NEXT:    cmovlel %eax, %esi
+; CHECK-NEXT:    imull %edi, %esi
+; CHECK-NEXT:  .LBB18_2: # %return
+; CHECK-NEXT:    movslq %esi, %rax
+; CHECK-NEXT:    retq
+entry:
+  %cmp = icmp sgt i32 %a, 1
+  br i1 %cmp, label %return, label %if.end
+
+if.end:                                           ; preds = %entry
+  %cmp1 = icmp eq i32 %a, 1
+  %mul = select i1 %cmp1, i32 %b, i32 1
+  %spec.select = mul nsw i32 %mul, %a
+  br label %return
+
+return:                                           ; preds = %if.end, %entry
+  %retval.0.in = phi i32 [ %b, %entry ], [ %spec.select, %if.end ]
+  %retval.0 = sext i32 %retval.0.in to i64
+  ret i64 %retval.0
+}
+
+define i64 @i_a_2(i32 signext %a, i32 signext %b) {
+; CHECK-LABEL: i_a_2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cmpl $2, %edi
+; CHECK-NEXT:    jg .LBB19_2
+; CHECK-NEXT:  # %bb.1: # %if.end
+; CHECK-NEXT:    movl $1, %eax
+; CHECK-NEXT:    cmovll %eax, %esi
+; CHECK-NEXT:    imull %edi, %esi
+; CHECK-NEXT:  .LBB19_2: # %return
+; CHECK-NEXT:    movslq %esi, %rax
+; CHECK-NEXT:    retq
+entry:
+  %cmp = icmp sgt i32 %a, 2
+  br i1 %cmp, label %return, label %if.end
+
+if.end:                                           ; preds = %entry
+  %cmp1 = icmp eq i32 %a, 2
+  %mul = select i1 %cmp1, i32 %b, i32 1
+  %spec.select = mul nsw i32 %mul, %a
+  br label %return
+
+return:                                           ; preds = %if.end, %entry
+  %retval.0.in = phi i32 [ %b, %entry ], [ %spec.select, %if.end ]
+  %retval.0 = sext i32 %retval.0.in to i64
+  ret i64 %retval.0
+}
diff --git a/llvm/test/CodeGen/X86/usub_sat.ll b/llvm/test/CodeGen/X86/usub_sat.ll
index a0ca559f8..f39aee388 100644
--- a/llvm/test/CodeGen/X86/usub_sat.ll
+++ b/llvm/test/CodeGen/X86/usub_sat.ll
@@ -2,10 +2,12 @@
 ; RUN: llc < %s -mtriple=i686 -mattr=cmov | FileCheck %s --check-prefixes=CHECK,X86
 ; RUN: llc < %s -mtriple=x86_64-linux | FileCheck %s --check-prefixes=CHECK,X64
 
-declare  i4  @llvm.usub.sat.i4   (i4,  i4)
-declare  i32 @llvm.usub.sat.i32  (i32, i32)
-declare  i64 @llvm.usub.sat.i64  (i64, i64)
-declare  <4 x i32> @llvm.usub.sat.v4i32(<4 x i32>, <4 x i32>)
+declare i4 @llvm.usub.sat.i4(i4, i4)
+declare i8 @llvm.usub.sat.i8(i8, i8)
+declare i16 @llvm.usub.sat.i16(i16, i16)
+declare i32 @llvm.usub.sat.i32(i32, i32)
+declare i64 @llvm.usub.sat.i64(i64, i64)
+declare <4 x i32> @llvm.usub.sat.v4i32(<4 x i32>, <4 x i32>)
 
 define i32 @func(i32 %x, i32 %y) nounwind {
 ; X86-LABEL: func:
@@ -22,8 +24,8 @@ define i32 @func(i32 %x, i32 %y) nounwind {
 ; X64-NEXT:    subl %esi, %edi
 ; X64-NEXT:    cmovael %edi, %eax
 ; X64-NEXT:    retq
-  %tmp = call i32 @llvm.usub.sat.i32(i32 %x, i32 %y);
-  ret i32 %tmp;
+  %tmp = call i32 @llvm.usub.sat.i32(i32 %x, i32 %y)
+  ret i32 %tmp
 }
 
 define i64 @func2(i64 %x, i64 %y) nounwind {
@@ -44,38 +46,81 @@ define i64 @func2(i64 %x, i64 %y) nounwind {
 ; X64-NEXT:    subq %rsi, %rdi
 ; X64-NEXT:    cmovaeq %rdi, %rax
 ; X64-NEXT:    retq
-  %tmp = call i64 @llvm.usub.sat.i64(i64 %x, i64 %y);
-  ret i64 %tmp;
+  %tmp = call i64 @llvm.usub.sat.i64(i64 %x, i64 %y)
+  ret i64 %tmp
 }
 
-define i4 @func3(i4 %x, i4 %y) nounwind {
-; X86-LABEL: func3:
+define zeroext i16 @func16(i16 zeroext %x, i16 zeroext %y) nounwind {
+; X86-LABEL: func16:
+; X86:       # %bb.0:
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    xorl %ecx, %ecx
+; X86-NEXT:    subw {{[0-9]+}}(%esp), %ax
+; X86-NEXT:    cmovbl %ecx, %eax
+; X86-NEXT:    # kill: def $ax killed $ax killed $eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: func16:
+; X64:       # %bb.0:
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    subw %si, %di
+; X64-NEXT:    cmovael %edi, %eax
+; X64-NEXT:    # kill: def $ax killed $ax killed $eax
+; X64-NEXT:    retq
+  %tmp = call i16 @llvm.usub.sat.i16(i16 %x, i16 %y)
+  ret i16 %tmp
+}
+
+define zeroext i8 @func8(i8 zeroext %x, i8 zeroext %y) nounwind {
+; X86-LABEL: func8:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
-; X86-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X86-NEXT:    shlb $4, %cl
-; X86-NEXT:    shlb $4, %al
-; X86-NEXT:    xorl %edx, %edx
-; X86-NEXT:    subb %cl, %al
+; X86-NEXT:    xorl %ecx, %ecx
+; X86-NEXT:    subb {{[0-9]+}}(%esp), %al
 ; X86-NEXT:    movzbl %al, %eax
-; X86-NEXT:    cmovbl %edx, %eax
-; X86-NEXT:    shrb $4, %al
+; X86-NEXT:    cmovbl %ecx, %eax
 ; X86-NEXT:    # kill: def $al killed $al killed $eax
 ; X86-NEXT:    retl
 ;
-; X64-LABEL: func3:
+; X64-LABEL: func8:
 ; X64:       # %bb.0:
-; X64-NEXT:    shlb $4, %sil
-; X64-NEXT:    shlb $4, %dil
 ; X64-NEXT:    xorl %ecx, %ecx
 ; X64-NEXT:    subb %sil, %dil
 ; X64-NEXT:    movzbl %dil, %eax
 ; X64-NEXT:    cmovbl %ecx, %eax
-; X64-NEXT:    shrb $4, %al
 ; X64-NEXT:    # kill: def $al killed $al killed $eax
 ; X64-NEXT:    retq
-  %tmp = call i4 @llvm.usub.sat.i4(i4 %x, i4 %y);
-  ret i4 %tmp;
+  %tmp = call i8 @llvm.usub.sat.i8(i8 %x, i8 %y)
+  ret i8 %tmp
+}
+
+define zeroext i4 @func3(i4 zeroext %x, i4 zeroext %y) nounwind {
+; X86-LABEL: func3:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-NEXT:    movzbl %cl, %edx
+; X86-NEXT:    movzbl %al, %ebx
+; X86-NEXT:    cmpb %al, %cl
+; X86-NEXT:    cmoval %edx, %ebx
+; X86-NEXT:    subb %al, %bl
+; X86-NEXT:    movzbl %bl, %eax
+; X86-NEXT:    andl $15, %eax
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    retl
+;
+; X64-LABEL: func3:
+; X64:       # %bb.0:
+; X64-NEXT:    cmpb %sil, %dil
+; X64-NEXT:    movl %esi, %eax
+; X64-NEXT:    cmoval %edi, %eax
+; X64-NEXT:    subb %sil, %al
+; X64-NEXT:    movzbl %al, %eax
+; X64-NEXT:    andl $15, %eax
+; X64-NEXT:    retq
+  %tmp = call i4 @llvm.usub.sat.i4(i4 %x, i4 %y)
+  ret i4 %tmp
 }
 
 define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
@@ -117,6 +162,6 @@ define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; X64-NEXT:    psubd %xmm1, %xmm0
 ; X64-NEXT:    pand %xmm2, %xmm0
 ; X64-NEXT:    retq
-  %tmp = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> %x, <4 x i32> %y);
-  ret <4 x i32> %tmp;
+  %tmp = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> %x, <4 x i32> %y)
+  ret <4 x i32> %tmp
 }
diff --git a/llvm/test/CodeGen/X86/usub_sat_plus.ll b/llvm/test/CodeGen/X86/usub_sat_plus.ll
new file mode 100644
index 000000000..f56e0962e
--- /dev/null
+++ b/llvm/test/CodeGen/X86/usub_sat_plus.ll
@@ -0,0 +1,149 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686 -mattr=cmov | FileCheck %s --check-prefixes=CHECK,X86
+; RUN: llc < %s -mtriple=x86_64-linux | FileCheck %s --check-prefixes=CHECK,X64
+
+declare i4 @llvm.usub.sat.i4(i4, i4)
+declare i8 @llvm.usub.sat.i8(i8, i8)
+declare i16 @llvm.usub.sat.i16(i16, i16)
+declare i32 @llvm.usub.sat.i32(i32, i32)
+declare i64 @llvm.usub.sat.i64(i64, i64)
+
+define i32 @func32(i32 %x, i32 %y, i32 %z) nounwind {
+; X86-LABEL: func32:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    imull {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    xorl %edx, %edx
+; X86-NEXT:    subl %ecx, %eax
+; X86-NEXT:    cmovbl %edx, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: func32:
+; X64:       # %bb.0:
+; X64-NEXT:    imull %edx, %esi
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    subl %esi, %edi
+; X64-NEXT:    cmovael %edi, %eax
+; X64-NEXT:    retq
+  %a = mul i32 %y, %z
+  %tmp = call i32 @llvm.usub.sat.i32(i32 %x, i32 %a)
+  ret i32 %tmp
+}
+
+define i64 @func64(i64 %x, i64 %y, i64 %z) nounwind {
+; X86-LABEL: func64:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    xorl %ecx, %ecx
+; X86-NEXT:    subl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    sbbl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    cmovbl %ecx, %edx
+; X86-NEXT:    cmovbl %ecx, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: func64:
+; X64:       # %bb.0:
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    subq %rdx, %rdi
+; X64-NEXT:    cmovaeq %rdi, %rax
+; X64-NEXT:    retq
+  %a = mul i64 %y, %z
+  %tmp = call i64 @llvm.usub.sat.i64(i64 %x, i64 %z)
+  ret i64 %tmp
+}
+
+define zeroext i16 @func16(i16 zeroext %x, i16 zeroext %y, i16 zeroext %z) nounwind {
+; X86-LABEL: func16:
+; X86:       # %bb.0:
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    imulw {{[0-9]+}}(%esp), %cx
+; X86-NEXT:    xorl %edx, %edx
+; X86-NEXT:    subw %cx, %ax
+; X86-NEXT:    cmovbl %edx, %eax
+; X86-NEXT:    # kill: def $ax killed $ax killed $eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: func16:
+; X64:       # %bb.0:
+; X64-NEXT:    imull %edx, %esi
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    subw %si, %di
+; X64-NEXT:    cmovael %edi, %eax
+; X64-NEXT:    # kill: def $ax killed $ax killed $eax
+; X64-NEXT:    retq
+  %a = mul i16 %y, %z
+  %tmp = call i16 @llvm.usub.sat.i16(i16 %x, i16 %a)
+  ret i16 %tmp
+}
+
+define zeroext i8 @func8(i8 zeroext %x, i8 zeroext %y, i8 zeroext %z) nounwind {
+; X86-LABEL: func8:
+; X86:       # %bb.0:
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    mulb {{[0-9]+}}(%esp)
+; X86-NEXT:    xorl %edx, %edx
+; X86-NEXT:    subb %al, %cl
+; X86-NEXT:    movzbl %cl, %eax
+; X86-NEXT:    cmovbl %edx, %eax
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: func8:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %esi, %eax
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    mulb %dl
+; X64-NEXT:    xorl %ecx, %ecx
+; X64-NEXT:    subb %al, %dil
+; X64-NEXT:    movzbl %dil, %eax
+; X64-NEXT:    cmovbl %ecx, %eax
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    retq
+  %a = mul i8 %y, %z
+  %tmp = call i8 @llvm.usub.sat.i8(i8 %x, i8 %a)
+  ret i8 %tmp
+}
+
+define zeroext i4 @func4(i4 zeroext %x, i4 zeroext %y, i4 zeroext %z) nounwind {
+; X86-LABEL: func4:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    mulb {{[0-9]+}}(%esp)
+; X86-NEXT:    movl %eax, %edx
+; X86-NEXT:    andb $15, %dl
+; X86-NEXT:    movzbl %dl, %esi
+; X86-NEXT:    movzbl %cl, %ebx
+; X86-NEXT:    cmpb %dl, %cl
+; X86-NEXT:    cmovbel %esi, %ebx
+; X86-NEXT:    subb %al, %bl
+; X86-NEXT:    movzbl %bl, %eax
+; X86-NEXT:    andl $15, %eax
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    retl
+;
+; X64-LABEL: func4:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %esi, %eax
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    mulb %dl
+; X64-NEXT:    movl %eax, %ecx
+; X64-NEXT:    andb $15, %cl
+; X64-NEXT:    movzbl %cl, %ecx
+; X64-NEXT:    cmpb %cl, %dil
+; X64-NEXT:    cmoval %edi, %ecx
+; X64-NEXT:    subb %al, %cl
+; X64-NEXT:    movzbl %cl, %eax
+; X64-NEXT:    andl $15, %eax
+; X64-NEXT:    retq
+  %a = mul i4 %y, %z
+  %tmp = call i4 @llvm.usub.sat.i4(i4 %x, i4 %a)
+  ret i4 %tmp
+}
diff --git a/llvm/test/CodeGen/X86/usub_sat_vec.ll b/llvm/test/CodeGen/X86/usub_sat_vec.ll
index b7b46aa7e..d56f9150a 100644
--- a/llvm/test/CodeGen/X86/usub_sat_vec.ll
+++ b/llvm/test/CodeGen/X86/usub_sat_vec.ll
@@ -210,30 +210,13 @@ define void @v8i8(<8 x i8>* %px, <8 x i8>* %py, <8 x i8>* %pz) nounwind {
 ; SSE-NEXT:    movq %xmm0, (%rdx)
 ; SSE-NEXT:    retq
 ;
-; AVX1-LABEL: v8i8:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
-; AVX1-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
-; AVX1-NEXT:    vpsubusb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vmovq %xmm0, (%rdx)
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: v8i8:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
-; AVX2-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
-; AVX2-NEXT:    vpsubusb %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vmovq %xmm0, (%rdx)
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: v8i8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
-; AVX512-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
-; AVX512-NEXT:    vpsubusb %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX512-NEXT:    vpmovwb %xmm0, (%rdx)
-; AVX512-NEXT:    retq
+; AVX-LABEL: v8i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
+; AVX-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
+; AVX-NEXT:    vpsubusb %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vmovq %xmm0, (%rdx)
+; AVX-NEXT:    retq
   %x = load <8 x i8>, <8 x i8>* %px
   %y = load <8 x i8>, <8 x i8>* %py
   %z = call <8 x i8> @llvm.usub.sat.v8i8(<8 x i8> %x, <8 x i8> %y)
@@ -250,30 +233,13 @@ define void @v4i8(<4 x i8>* %px, <4 x i8>* %py, <4 x i8>* %pz) nounwind {
 ; SSE-NEXT:    movd %xmm0, (%rdx)
 ; SSE-NEXT:    retq
 ;
-; AVX1-LABEL: v4i8:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX1-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX1-NEXT:    vpsubusb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vmovd %xmm0, (%rdx)
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: v4i8:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX2-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX2-NEXT:    vpsubusb %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vmovd %xmm0, (%rdx)
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: v4i8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX512-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX512-NEXT:    vpsubusb %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; AVX512-NEXT:    vpmovdb %xmm0, (%rdx)
-; AVX512-NEXT:    retq
+; AVX-LABEL: v4i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX-NEXT:    vpsubusb %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vmovd %xmm0, (%rdx)
+; AVX-NEXT:    retq
   %x = load <4 x i8>, <4 x i8>* %px
   %y = load <4 x i8>, <4 x i8>* %py
   %z = call <4 x i8> @llvm.usub.sat.v4i8(<4 x i8> %x, <4 x i8> %y)
@@ -314,36 +280,15 @@ define void @v2i8(<2 x i8>* %px, <2 x i8>* %py, <2 x i8>* %pz) nounwind {
 ; SSE41-NEXT:    pextrw $0, %xmm0, (%rdx)
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: v2i8:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    movzwl (%rdi), %eax
-; AVX1-NEXT:    vmovd %eax, %xmm0
-; AVX1-NEXT:    movzwl (%rsi), %eax
-; AVX1-NEXT:    vmovd %eax, %xmm1
-; AVX1-NEXT:    vpsubusb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpextrw $0, %xmm0, (%rdx)
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: v2i8:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    movzwl (%rdi), %eax
-; AVX2-NEXT:    vmovd %eax, %xmm0
-; AVX2-NEXT:    movzwl (%rsi), %eax
-; AVX2-NEXT:    vmovd %eax, %xmm1
-; AVX2-NEXT:    vpsubusb %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpextrw $0, %xmm0, (%rdx)
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: v2i8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    movzwl (%rdi), %eax
-; AVX512-NEXT:    vmovd %eax, %xmm0
-; AVX512-NEXT:    movzwl (%rsi), %eax
-; AVX512-NEXT:    vmovd %eax, %xmm1
-; AVX512-NEXT:    vpsubusb %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT:    vpmovqb %xmm0, (%rdx)
-; AVX512-NEXT:    retq
+; AVX-LABEL: v2i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    movzwl (%rdi), %eax
+; AVX-NEXT:    vmovd %eax, %xmm0
+; AVX-NEXT:    movzwl (%rsi), %eax
+; AVX-NEXT:    vmovd %eax, %xmm1
+; AVX-NEXT:    vpsubusb %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpextrw $0, %xmm0, (%rdx)
+; AVX-NEXT:    retq
   %x = load <2 x i8>, <2 x i8>* %px
   %y = load <2 x i8>, <2 x i8>* %py
   %z = call <2 x i8> @llvm.usub.sat.v2i8(<2 x i8> %x, <2 x i8> %y)
@@ -360,30 +305,13 @@ define void @v4i16(<4 x i16>* %px, <4 x i16>* %py, <4 x i16>* %pz) nounwind {
 ; SSE-NEXT:    movq %xmm0, (%rdx)
 ; SSE-NEXT:    retq
 ;
-; AVX1-LABEL: v4i16:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
-; AVX1-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
-; AVX1-NEXT:    vpsubusw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vmovq %xmm0, (%rdx)
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: v4i16:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
-; AVX2-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
-; AVX2-NEXT:    vpsubusw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vmovq %xmm0, (%rdx)
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: v4i16:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
-; AVX512-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
-; AVX512-NEXT:    vpsubusw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; AVX512-NEXT:    vpmovdw %xmm0, (%rdx)
-; AVX512-NEXT:    retq
+; AVX-LABEL: v4i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
+; AVX-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
+; AVX-NEXT:    vpsubusw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vmovq %xmm0, (%rdx)
+; AVX-NEXT:    retq
   %x = load <4 x i16>, <4 x i16>* %px
   %y = load <4 x i16>, <4 x i16>* %py
   %z = call <4 x i16> @llvm.usub.sat.v4i16(<4 x i16> %x, <4 x i16> %y)
@@ -400,30 +328,13 @@ define void @v2i16(<2 x i16>* %px, <2 x i16>* %py, <2 x i16>* %pz) nounwind {
 ; SSE-NEXT:    movd %xmm0, (%rdx)
 ; SSE-NEXT:    retq
 ;
-; AVX1-LABEL: v2i16:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX1-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX1-NEXT:    vpsubusw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vmovd %xmm0, (%rdx)
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: v2i16:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX2-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX2-NEXT:    vpsubusw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vmovd %xmm0, (%rdx)
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: v2i16:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX512-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX512-NEXT:    vpsubusw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; AVX512-NEXT:    vpmovqw %xmm0, (%rdx)
-; AVX512-NEXT:    retq
+; AVX-LABEL: v2i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX-NEXT:    vpsubusw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vmovd %xmm0, (%rdx)
+; AVX-NEXT:    retq
   %x = load <2 x i16>, <2 x i16>* %px
   %y = load <2 x i16>, <2 x i16>* %py
   %z = call <2 x i16> @llvm.usub.sat.v2i16(<2 x i16> %x, <2 x i16> %y)
@@ -631,99 +542,37 @@ define <16 x i1> @v16i1(<16 x i1> %x, <16 x i1> %y) nounwind {
 define <2 x i32> @v2i32(<2 x i32> %x, <2 x i32> %y) nounwind {
 ; SSE2-LABEL: v2i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    psllq $32, %xmm1
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [9223372039002259456,9223372039002259456]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648]
 ; SSE2-NEXT:    movdqa %xmm1, %xmm3
 ; SSE2-NEXT:    pxor %xmm2, %xmm3
-; SSE2-NEXT:    psllq $32, %xmm0
 ; SSE2-NEXT:    pxor %xmm0, %xmm2
-; SSE2-NEXT:    movdqa %xmm2, %xmm4
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm4
 ; SSE2-NEXT:    pcmpgtd %xmm3, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
-; SSE2-NEXT:    pand %xmm4, %xmm3
-; SSE2-NEXT:    por %xmm2, %xmm3
-; SSE2-NEXT:    psubq %xmm1, %xmm0
-; SSE2-NEXT:    pand %xmm3, %xmm0
-; SSE2-NEXT:    psrlq $32, %xmm0
+; SSE2-NEXT:    psubd %xmm1, %xmm0
+; SSE2-NEXT:    pand %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: v2i32:
 ; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    psllq $32, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [9223372039002259456,9223372039002259456]
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648]
 ; SSSE3-NEXT:    movdqa %xmm1, %xmm3
 ; SSSE3-NEXT:    pxor %xmm2, %xmm3
-; SSSE3-NEXT:    psllq $32, %xmm0
 ; SSSE3-NEXT:    pxor %xmm0, %xmm2
-; SSSE3-NEXT:    movdqa %xmm2, %xmm4
-; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm4
 ; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm4, %xmm3
-; SSSE3-NEXT:    por %xmm2, %xmm3
-; SSSE3-NEXT:    psubq %xmm1, %xmm0
-; SSSE3-NEXT:    pand %xmm3, %xmm0
-; SSSE3-NEXT:    psrlq $32, %xmm0
+; SSSE3-NEXT:    psubd %xmm1, %xmm0
+; SSSE3-NEXT:    pand %xmm2, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: v2i32:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    psllq $32, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [9223372039002259456,9223372039002259456]
-; SSE41-NEXT:    movdqa %xmm1, %xmm3
-; SSE41-NEXT:    pxor %xmm0, %xmm3
-; SSE41-NEXT:    psllq $32, %xmm2
-; SSE41-NEXT:    pxor %xmm2, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
-; SSE41-NEXT:    psubq %xmm1, %xmm2
-; SSE41-NEXT:    pxor %xmm1, %xmm1
-; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
-; SSE41-NEXT:    psrlq $32, %xmm1
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pmaxud %xmm1, %xmm0
+; SSE41-NEXT:    psubd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: v2i32:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpsllq $32, %xmm1, %xmm1
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
-; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm3
-; AVX1-NEXT:    vpsllq $32, %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm2
-; AVX1-NEXT:    vpcmpgtq %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpsubq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpand %xmm0, %xmm2, %xmm0
-; AVX1-NEXT:    vpsrlq $32, %xmm0, %xmm0
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: v2i32:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpsllq $32, %xmm1, %xmm1
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
-; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm3
-; AVX2-NEXT:    vpsllq $32, %xmm0, %xmm0
-; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm2
-; AVX2-NEXT:    vpcmpgtq %xmm3, %xmm2, %xmm2
-; AVX2-NEXT:    vpsubq %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpand %xmm0, %xmm2, %xmm0
-; AVX2-NEXT:    vpsrlq $32, %xmm0, %xmm0
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: v2i32:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsllq $32, %xmm1, %xmm1
-; AVX512-NEXT:    vpsllq $32, %xmm0, %xmm0
-; AVX512-NEXT:    vpmaxuq %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpsubq %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrlq $32, %xmm0, %xmm0
-; AVX512-NEXT:    retq
+; AVX-LABEL: v2i32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
   %z = call <2 x i32> @llvm.usub.sat.v2i32(<2 x i32> %x, <2 x i32> %y)
   ret <2 x i32> %z
 }
@@ -981,17 +830,16 @@ define <2 x i64> @v2i64(<2 x i64> %x, <2 x i64> %y) nounwind {
 ; SSE41-LABEL: v2i64:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [9223372039002259456,9223372039002259456]
-; SSE41-NEXT:    movdqa %xmm1, %xmm3
-; SSE41-NEXT:    pxor %xmm0, %xmm3
-; SSE41-NEXT:    pxor %xmm2, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [9223372039002259456,9223372039002259456]
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pxor %xmm3, %xmm0
+; SSE41-NEXT:    pxor %xmm2, %xmm3
+; SSE41-NEXT:    movdqa %xmm3, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm3, %xmm0
 ; SSE41-NEXT:    psubq %xmm1, %xmm2
 ; SSE41-NEXT:    pxor %xmm1, %xmm1
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
@@ -1099,12 +947,11 @@ define <4 x i64> @v4i64(<4 x i64> %x, <4 x i64> %y) nounwind {
 ; SSE41-NEXT:    pxor %xmm6, %xmm2
 ; SSE41-NEXT:    pxor %xmm6, %xmm5
 ; SSE41-NEXT:    movdqa %xmm5, %xmm0
-; SSE41-NEXT:    pcmpgtd %xmm2, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm2, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm5[1,1,3,3]
-; SSE41-NEXT:    pand %xmm7, %xmm2
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm2, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm2, %xmm5
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm5[0,0,2,2]
+; SSE41-NEXT:    pand %xmm2, %xmm0
+; SSE41-NEXT:    por %xmm5, %xmm0
 ; SSE41-NEXT:    pxor %xmm2, %xmm2
 ; SSE41-NEXT:    pxor %xmm5, %xmm5
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm5
@@ -1112,12 +959,11 @@ define <4 x i64> @v4i64(<4 x i64> %x, <4 x i64> %y) nounwind {
 ; SSE41-NEXT:    pxor %xmm6, %xmm0
 ; SSE41-NEXT:    pxor %xmm1, %xmm6
 ; SSE41-NEXT:    movdqa %xmm6, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3]
-; SSE41-NEXT:    pand %xmm7, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm6, %xmm0
 ; SSE41-NEXT:    psubq %xmm3, %xmm1
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
 ; SSE41-NEXT:    movapd %xmm5, %xmm0
@@ -1285,37 +1131,34 @@ define <8 x i64> @v8i64(<8 x i64> %x, <8 x i64> %y) nounwind {
 ; SSE41-NEXT:    pxor %xmm10, %xmm4
 ; SSE41-NEXT:    pxor %xmm10, %xmm8
 ; SSE41-NEXT:    movdqa %xmm8, %xmm0
-; SSE41-NEXT:    pcmpgtd %xmm4, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm11 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm4, %xmm8
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm8[1,1,3,3]
-; SSE41-NEXT:    pand %xmm11, %xmm4
-; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm4, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm4, %xmm8
+; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm8[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm8, %xmm0
 ; SSE41-NEXT:    pxor %xmm8, %xmm8
-; SSE41-NEXT:    pxor %xmm11, %xmm11
+; SSE41-NEXT:    xorpd %xmm11, %xmm11
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm9, %xmm11
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    movdqa %xmm1, %xmm4
 ; SSE41-NEXT:    psubq %xmm5, %xmm1
 ; SSE41-NEXT:    pxor %xmm10, %xmm5
-; SSE41-NEXT:    pxor %xmm10, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm4
+; SSE41-NEXT:    pxor %xmm10, %xmm4
+; SSE41-NEXT:    movdqa %xmm4, %xmm9
+; SSE41-NEXT:    pcmpeqd %xmm5, %xmm9
 ; SSE41-NEXT:    pcmpgtd %xmm5, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm9 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm5, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2]
 ; SSE41-NEXT:    pand %xmm9, %xmm0
 ; SSE41-NEXT:    por %xmm4, %xmm0
 ; SSE41-NEXT:    pxor %xmm5, %xmm5
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm5
-; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    movdqa %xmm2, %xmm1
 ; SSE41-NEXT:    psubq %xmm6, %xmm2
 ; SSE41-NEXT:    pxor %xmm10, %xmm6
-; SSE41-NEXT:    pxor %xmm10, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    pxor %xmm10, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm6, %xmm4
 ; SSE41-NEXT:    pcmpgtd %xmm6, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm6, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2]
 ; SSE41-NEXT:    pand %xmm4, %xmm0
 ; SSE41-NEXT:    por %xmm1, %xmm0
 ; SSE41-NEXT:    pxor %xmm6, %xmm6
@@ -1324,12 +1167,11 @@ define <8 x i64> @v8i64(<8 x i64> %x, <8 x i64> %y) nounwind {
 ; SSE41-NEXT:    pxor %xmm10, %xmm0
 ; SSE41-NEXT:    pxor %xmm3, %xmm10
 ; SSE41-NEXT:    movdqa %xmm10, %xmm1
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm10
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm10[1,1,3,3]
-; SSE41-NEXT:    pand %xmm2, %xmm0
-; SSE41-NEXT:    por %xmm1, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm10
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm10[0,0,2,2]
+; SSE41-NEXT:    pand %xmm1, %xmm0
+; SSE41-NEXT:    por %xmm10, %xmm0
 ; SSE41-NEXT:    psubq %xmm7, %xmm3
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm8
 ; SSE41-NEXT:    movapd %xmm11, %xmm0
diff --git a/llvm/test/CodeGen/X86/uwtables.ll b/llvm/test/CodeGen/X86/uwtables.ll
index e41fdcc45..c9da19a7c 100644
--- a/llvm/test/CodeGen/X86/uwtables.ll
+++ b/llvm/test/CodeGen/X86/uwtables.ll
@@ -39,5 +39,5 @@ declare i32 @__gxx_personality_v0(...)
 declare void @__cxa_call_unexpected(i8*) local_unnamed_addr
 
 
-attributes #0 = { noreturn nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noreturn nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
diff --git a/llvm/test/CodeGen/X86/v8i1-masks.ll b/llvm/test/CodeGen/X86/v8i1-masks.ll
index de97281d6..82ff35772 100644
--- a/llvm/test/CodeGen/X86/v8i1-masks.ll
+++ b/llvm/test/CodeGen/X86/v8i1-masks.ll
@@ -163,3 +163,1296 @@ define <8 x i32> @and_mask_constant(<8 x i32> %v0, <8 x i32> %v1) {
   %r = zext <8 x i1> %mand to <8 x i32>
   ret <8 x i32> %r
 }
+
+define <8 x i32> @two_ands(<8 x float> %x) local_unnamed_addr #0 {
+; X32-LABEL: two_ands:
+; X32:       ## %bb.0: ## %entry
+; X32-NEXT:    vmovaps {{.*#+}} ymm1 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]
+; X32-NEXT:    vcmpleps %ymm0, %ymm1, %ymm1
+; X32-NEXT:    vcmpltps LCPI3_1, %ymm0, %ymm0
+; X32-NEXT:    vandps %ymm0, %ymm1, %ymm0
+; X32-NEXT:    retl
+;
+; X64-LABEL: two_ands:
+; X64:       ## %bb.0: ## %entry
+; X64-NEXT:    vmovaps {{.*#+}} ymm1 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]
+; X64-NEXT:    vcmpleps %ymm0, %ymm1, %ymm1
+; X64-NEXT:    vcmpltps {{.*}}(%rip), %ymm0, %ymm0
+; X64-NEXT:    vandps %ymm0, %ymm1, %ymm0
+; X64-NEXT:    retq
+;
+; X32-AVX2-LABEL: two_ands:
+; X32-AVX2:       ## %bb.0: ## %entry
+; X32-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm1 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]
+; X32-AVX2-NEXT:    vcmpleps %ymm0, %ymm1, %ymm1
+; X32-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; X32-AVX2-NEXT:    vcmpltps %ymm2, %ymm0, %ymm0
+; X32-AVX2-NEXT:    vandps %ymm0, %ymm1, %ymm0
+; X32-AVX2-NEXT:    retl
+;
+; X64-AVX2-LABEL: two_ands:
+; X64-AVX2:       ## %bb.0: ## %entry
+; X64-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm1 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]
+; X64-AVX2-NEXT:    vcmpleps %ymm0, %ymm1, %ymm1
+; X64-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; X64-AVX2-NEXT:    vcmpltps %ymm2, %ymm0, %ymm0
+; X64-AVX2-NEXT:    vandps %ymm0, %ymm1, %ymm0
+; X64-AVX2-NEXT:    retq
+entry:
+  %cmp = fcmp oge <8 x float> %x, <float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01>
+  %cmp1 = fcmp olt <8 x float> %x, <float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>
+  %and4 = and <8 x i1> %cmp, %cmp1
+  %and = sext <8 x i1> %and4 to <8 x i32>
+  ret <8 x i32> %and
+}
+
+define <8 x i32> @three_ands(<8 x float> %x) {
+; X32-LABEL: three_ands:
+; X32:       ## %bb.0: ## %entry
+; X32-NEXT:    vmovaps {{.*#+}} ymm1 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]
+; X32-NEXT:    vcmpleps %ymm0, %ymm1, %ymm1
+; X32-NEXT:    vcmpltps LCPI4_1, %ymm0, %ymm2
+; X32-NEXT:    vandps %ymm2, %ymm1, %ymm1
+; X32-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; X32-NEXT:    vpackssdw %xmm2, %xmm1, %xmm1
+; X32-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; X32-NEXT:    vcmpneqps %ymm2, %ymm0, %ymm0
+; X32-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; X32-NEXT:    vpackssdw %xmm2, %xmm0, %xmm0
+; X32-NEXT:    vpand %xmm0, %xmm1, %xmm0
+; X32-NEXT:    vpmovsxwd %xmm0, %xmm1
+; X32-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; X32-NEXT:    vpmovsxwd %xmm0, %xmm0
+; X32-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; X32-NEXT:    retl
+;
+; X64-LABEL: three_ands:
+; X64:       ## %bb.0: ## %entry
+; X64-NEXT:    vmovaps {{.*#+}} ymm1 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]
+; X64-NEXT:    vcmpleps %ymm0, %ymm1, %ymm1
+; X64-NEXT:    vcmpltps {{.*}}(%rip), %ymm0, %ymm2
+; X64-NEXT:    vandps %ymm2, %ymm1, %ymm1
+; X64-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; X64-NEXT:    vpackssdw %xmm2, %xmm1, %xmm1
+; X64-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; X64-NEXT:    vcmpneqps %ymm2, %ymm0, %ymm0
+; X64-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; X64-NEXT:    vpackssdw %xmm2, %xmm0, %xmm0
+; X64-NEXT:    vpand %xmm0, %xmm1, %xmm0
+; X64-NEXT:    vpmovsxwd %xmm0, %xmm1
+; X64-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; X64-NEXT:    vpmovsxwd %xmm0, %xmm0
+; X64-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; X64-NEXT:    retq
+;
+; X32-AVX2-LABEL: three_ands:
+; X32-AVX2:       ## %bb.0: ## %entry
+; X32-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm1 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]
+; X32-AVX2-NEXT:    vcmpleps %ymm0, %ymm1, %ymm1
+; X32-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; X32-AVX2-NEXT:    vcmpltps %ymm2, %ymm0, %ymm2
+; X32-AVX2-NEXT:    vandps %ymm2, %ymm1, %ymm1
+; X32-AVX2-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; X32-AVX2-NEXT:    vpackssdw %xmm2, %xmm1, %xmm1
+; X32-AVX2-NEXT:    vxorps %xmm2, %xmm2, %xmm2
+; X32-AVX2-NEXT:    vcmpneqps %ymm2, %ymm0, %ymm0
+; X32-AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; X32-AVX2-NEXT:    vpackssdw %xmm2, %xmm0, %xmm0
+; X32-AVX2-NEXT:    vpand %xmm0, %xmm1, %xmm0
+; X32-AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
+; X32-AVX2-NEXT:    retl
+;
+; X64-AVX2-LABEL: three_ands:
+; X64-AVX2:       ## %bb.0: ## %entry
+; X64-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm1 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]
+; X64-AVX2-NEXT:    vcmpleps %ymm0, %ymm1, %ymm1
+; X64-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; X64-AVX2-NEXT:    vcmpltps %ymm2, %ymm0, %ymm2
+; X64-AVX2-NEXT:    vandps %ymm2, %ymm1, %ymm1
+; X64-AVX2-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; X64-AVX2-NEXT:    vpackssdw %xmm2, %xmm1, %xmm1
+; X64-AVX2-NEXT:    vxorps %xmm2, %xmm2, %xmm2
+; X64-AVX2-NEXT:    vcmpneqps %ymm2, %ymm0, %ymm0
+; X64-AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; X64-AVX2-NEXT:    vpackssdw %xmm2, %xmm0, %xmm0
+; X64-AVX2-NEXT:    vpand %xmm0, %xmm1, %xmm0
+; X64-AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
+; X64-AVX2-NEXT:    retq
+entry:
+  %cmp = fcmp oge <8 x float> %x, <float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01>
+  %cmp1 = fcmp olt <8 x float> %x, <float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>
+  %and8 = and <8 x i1> %cmp, %cmp1
+  %cmp3 = fcmp une <8 x float> %x, zeroinitializer
+  %and59 = and <8 x i1> %and8, %cmp3
+  %and5 = sext <8 x i1> %and59 to <8 x i32>
+  ret <8 x i32> %and5
+}
+
+define <8 x i32> @four_ands(<8 x float> %x) {
+; X32-LABEL: four_ands:
+; X32:       ## %bb.0: ## %entry
+; X32-NEXT:    vmovaps {{.*#+}} ymm1 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]
+; X32-NEXT:    vcmpleps %ymm0, %ymm1, %ymm1
+; X32-NEXT:    vcmpltps LCPI5_1, %ymm0, %ymm2
+; X32-NEXT:    vandps %ymm2, %ymm1, %ymm1
+; X32-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; X32-NEXT:    vpackssdw %xmm2, %xmm1, %xmm1
+; X32-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; X32-NEXT:    vcmpneqps %ymm2, %ymm0, %ymm2
+; X32-NEXT:    vextractf128 $1, %ymm2, %xmm3
+; X32-NEXT:    vpackssdw %xmm3, %xmm2, %xmm2
+; X32-NEXT:    vcmpneqps LCPI5_2, %ymm0, %ymm0
+; X32-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; X32-NEXT:    vpackssdw %xmm3, %xmm0, %xmm0
+; X32-NEXT:    vpand %xmm0, %xmm2, %xmm0
+; X32-NEXT:    vpand %xmm0, %xmm1, %xmm0
+; X32-NEXT:    vpmovsxwd %xmm0, %xmm1
+; X32-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; X32-NEXT:    vpmovsxwd %xmm0, %xmm0
+; X32-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; X32-NEXT:    retl
+;
+; X64-LABEL: four_ands:
+; X64:       ## %bb.0: ## %entry
+; X64-NEXT:    vmovaps {{.*#+}} ymm1 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]
+; X64-NEXT:    vcmpleps %ymm0, %ymm1, %ymm1
+; X64-NEXT:    vcmpltps {{.*}}(%rip), %ymm0, %ymm2
+; X64-NEXT:    vandps %ymm2, %ymm1, %ymm1
+; X64-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; X64-NEXT:    vpackssdw %xmm2, %xmm1, %xmm1
+; X64-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; X64-NEXT:    vcmpneqps %ymm2, %ymm0, %ymm2
+; X64-NEXT:    vextractf128 $1, %ymm2, %xmm3
+; X64-NEXT:    vpackssdw %xmm3, %xmm2, %xmm2
+; X64-NEXT:    vcmpneqps {{.*}}(%rip), %ymm0, %ymm0
+; X64-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; X64-NEXT:    vpackssdw %xmm3, %xmm0, %xmm0
+; X64-NEXT:    vpand %xmm0, %xmm2, %xmm0
+; X64-NEXT:    vpand %xmm0, %xmm1, %xmm0
+; X64-NEXT:    vpmovsxwd %xmm0, %xmm1
+; X64-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; X64-NEXT:    vpmovsxwd %xmm0, %xmm0
+; X64-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; X64-NEXT:    retq
+;
+; X32-AVX2-LABEL: four_ands:
+; X32-AVX2:       ## %bb.0: ## %entry
+; X32-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm1 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]
+; X32-AVX2-NEXT:    vcmpleps %ymm0, %ymm1, %ymm1
+; X32-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; X32-AVX2-NEXT:    vcmpltps %ymm2, %ymm0, %ymm2
+; X32-AVX2-NEXT:    vandps %ymm2, %ymm1, %ymm1
+; X32-AVX2-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; X32-AVX2-NEXT:    vpackssdw %xmm2, %xmm1, %xmm1
+; X32-AVX2-NEXT:    vxorps %xmm2, %xmm2, %xmm2
+; X32-AVX2-NEXT:    vcmpneqps %ymm2, %ymm0, %ymm2
+; X32-AVX2-NEXT:    vextractf128 $1, %ymm2, %xmm3
+; X32-AVX2-NEXT:    vpackssdw %xmm3, %xmm2, %xmm2
+; X32-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm3 = [1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1]
+; X32-AVX2-NEXT:    vcmpneqps %ymm3, %ymm0, %ymm0
+; X32-AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; X32-AVX2-NEXT:    vpackssdw %xmm3, %xmm0, %xmm0
+; X32-AVX2-NEXT:    vpand %xmm0, %xmm2, %xmm0
+; X32-AVX2-NEXT:    vpand %xmm0, %xmm1, %xmm0
+; X32-AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
+; X32-AVX2-NEXT:    retl
+;
+; X64-AVX2-LABEL: four_ands:
+; X64-AVX2:       ## %bb.0: ## %entry
+; X64-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm1 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]
+; X64-AVX2-NEXT:    vcmpleps %ymm0, %ymm1, %ymm1
+; X64-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; X64-AVX2-NEXT:    vcmpltps %ymm2, %ymm0, %ymm2
+; X64-AVX2-NEXT:    vandps %ymm2, %ymm1, %ymm1
+; X64-AVX2-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; X64-AVX2-NEXT:    vpackssdw %xmm2, %xmm1, %xmm1
+; X64-AVX2-NEXT:    vxorps %xmm2, %xmm2, %xmm2
+; X64-AVX2-NEXT:    vcmpneqps %ymm2, %ymm0, %ymm2
+; X64-AVX2-NEXT:    vextractf128 $1, %ymm2, %xmm3
+; X64-AVX2-NEXT:    vpackssdw %xmm3, %xmm2, %xmm2
+; X64-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm3 = [1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1]
+; X64-AVX2-NEXT:    vcmpneqps %ymm3, %ymm0, %ymm0
+; X64-AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; X64-AVX2-NEXT:    vpackssdw %xmm3, %xmm0, %xmm0
+; X64-AVX2-NEXT:    vpand %xmm0, %xmm2, %xmm0
+; X64-AVX2-NEXT:    vpand %xmm0, %xmm1, %xmm0
+; X64-AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
+; X64-AVX2-NEXT:    retq
+entry:
+  %cmp = fcmp oge <8 x float> %x, <float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01>
+  %cmp1 = fcmp olt <8 x float> %x, <float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>
+  %and12 = and <8 x i1> %cmp, %cmp1
+  %cmp3 = fcmp une <8 x float> %x, zeroinitializer
+  %and513 = and <8 x i1> %and12, %cmp3
+  %cmp6 = fcmp une <8 x float> %x, <float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000>
+  %and814 = and <8 x i1> %and513, %cmp6
+  %and8 = sext <8 x i1> %and814 to <8 x i32>
+  ret <8 x i32> %and8
+}
+
+define <8 x i32> @five_ands(<8 x float> %x) {
+; X32-LABEL: five_ands:
+; X32:       ## %bb.0: ## %entry
+; X32-NEXT:    vmovaps {{.*#+}} ymm1 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]
+; X32-NEXT:    vcmpleps %ymm0, %ymm1, %ymm1
+; X32-NEXT:    vcmpltps LCPI6_1, %ymm0, %ymm2
+; X32-NEXT:    vandps %ymm2, %ymm1, %ymm1
+; X32-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; X32-NEXT:    vpackssdw %xmm2, %xmm1, %xmm1
+; X32-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; X32-NEXT:    vcmpneqps %ymm2, %ymm0, %ymm2
+; X32-NEXT:    vextractf128 $1, %ymm2, %xmm3
+; X32-NEXT:    vpackssdw %xmm3, %xmm2, %xmm2
+; X32-NEXT:    vcmpneqps LCPI6_2, %ymm0, %ymm3
+; X32-NEXT:    vextractf128 $1, %ymm3, %xmm4
+; X32-NEXT:    vpackssdw %xmm4, %xmm3, %xmm3
+; X32-NEXT:    vpand %xmm3, %xmm2, %xmm2
+; X32-NEXT:    vcmpneqps LCPI6_3, %ymm0, %ymm0
+; X32-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; X32-NEXT:    vpackssdw %xmm3, %xmm0, %xmm0
+; X32-NEXT:    vpand %xmm0, %xmm2, %xmm0
+; X32-NEXT:    vpand %xmm0, %xmm1, %xmm0
+; X32-NEXT:    vpmovsxwd %xmm0, %xmm1
+; X32-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; X32-NEXT:    vpmovsxwd %xmm0, %xmm0
+; X32-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; X32-NEXT:    retl
+;
+; X64-LABEL: five_ands:
+; X64:       ## %bb.0: ## %entry
+; X64-NEXT:    vmovaps {{.*#+}} ymm1 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]
+; X64-NEXT:    vcmpleps %ymm0, %ymm1, %ymm1
+; X64-NEXT:    vcmpltps {{.*}}(%rip), %ymm0, %ymm2
+; X64-NEXT:    vandps %ymm2, %ymm1, %ymm1
+; X64-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; X64-NEXT:    vpackssdw %xmm2, %xmm1, %xmm1
+; X64-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; X64-NEXT:    vcmpneqps %ymm2, %ymm0, %ymm2
+; X64-NEXT:    vextractf128 $1, %ymm2, %xmm3
+; X64-NEXT:    vpackssdw %xmm3, %xmm2, %xmm2
+; X64-NEXT:    vcmpneqps {{.*}}(%rip), %ymm0, %ymm3
+; X64-NEXT:    vextractf128 $1, %ymm3, %xmm4
+; X64-NEXT:    vpackssdw %xmm4, %xmm3, %xmm3
+; X64-NEXT:    vpand %xmm3, %xmm2, %xmm2
+; X64-NEXT:    vcmpneqps {{.*}}(%rip), %ymm0, %ymm0
+; X64-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; X64-NEXT:    vpackssdw %xmm3, %xmm0, %xmm0
+; X64-NEXT:    vpand %xmm0, %xmm2, %xmm0
+; X64-NEXT:    vpand %xmm0, %xmm1, %xmm0
+; X64-NEXT:    vpmovsxwd %xmm0, %xmm1
+; X64-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; X64-NEXT:    vpmovsxwd %xmm0, %xmm0
+; X64-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; X64-NEXT:    retq
+;
+; X32-AVX2-LABEL: five_ands:
+; X32-AVX2:       ## %bb.0: ## %entry
+; X32-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm1 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]
+; X32-AVX2-NEXT:    vcmpleps %ymm0, %ymm1, %ymm1
+; X32-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; X32-AVX2-NEXT:    vcmpltps %ymm2, %ymm0, %ymm2
+; X32-AVX2-NEXT:    vandps %ymm2, %ymm1, %ymm1
+; X32-AVX2-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; X32-AVX2-NEXT:    vpackssdw %xmm2, %xmm1, %xmm1
+; X32-AVX2-NEXT:    vxorps %xmm2, %xmm2, %xmm2
+; X32-AVX2-NEXT:    vcmpneqps %ymm2, %ymm0, %ymm2
+; X32-AVX2-NEXT:    vextractf128 $1, %ymm2, %xmm3
+; X32-AVX2-NEXT:    vpackssdw %xmm3, %xmm2, %xmm2
+; X32-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm3 = [1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1]
+; X32-AVX2-NEXT:    vcmpneqps %ymm3, %ymm0, %ymm3
+; X32-AVX2-NEXT:    vextractf128 $1, %ymm3, %xmm4
+; X32-AVX2-NEXT:    vpackssdw %xmm4, %xmm3, %xmm3
+; X32-AVX2-NEXT:    vpand %xmm3, %xmm2, %xmm2
+; X32-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm3 = [2.00000003E-1,2.00000003E-1,2.00000003E-1,2.00000003E-1,2.00000003E-1,2.00000003E-1,2.00000003E-1,2.00000003E-1]
+; X32-AVX2-NEXT:    vcmpneqps %ymm3, %ymm0, %ymm0
+; X32-AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; X32-AVX2-NEXT:    vpackssdw %xmm3, %xmm0, %xmm0
+; X32-AVX2-NEXT:    vpand %xmm0, %xmm2, %xmm0
+; X32-AVX2-NEXT:    vpand %xmm0, %xmm1, %xmm0
+; X32-AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
+; X32-AVX2-NEXT:    retl
+;
+; X64-AVX2-LABEL: five_ands:
+; X64-AVX2:       ## %bb.0: ## %entry
+; X64-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm1 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]
+; X64-AVX2-NEXT:    vcmpleps %ymm0, %ymm1, %ymm1
+; X64-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; X64-AVX2-NEXT:    vcmpltps %ymm2, %ymm0, %ymm2
+; X64-AVX2-NEXT:    vandps %ymm2, %ymm1, %ymm1
+; X64-AVX2-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; X64-AVX2-NEXT:    vpackssdw %xmm2, %xmm1, %xmm1
+; X64-AVX2-NEXT:    vxorps %xmm2, %xmm2, %xmm2
+; X64-AVX2-NEXT:    vcmpneqps %ymm2, %ymm0, %ymm2
+; X64-AVX2-NEXT:    vextractf128 $1, %ymm2, %xmm3
+; X64-AVX2-NEXT:    vpackssdw %xmm3, %xmm2, %xmm2
+; X64-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm3 = [1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1]
+; X64-AVX2-NEXT:    vcmpneqps %ymm3, %ymm0, %ymm3
+; X64-AVX2-NEXT:    vextractf128 $1, %ymm3, %xmm4
+; X64-AVX2-NEXT:    vpackssdw %xmm4, %xmm3, %xmm3
+; X64-AVX2-NEXT:    vpand %xmm3, %xmm2, %xmm2
+; X64-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm3 = [2.00000003E-1,2.00000003E-1,2.00000003E-1,2.00000003E-1,2.00000003E-1,2.00000003E-1,2.00000003E-1,2.00000003E-1]
+; X64-AVX2-NEXT:    vcmpneqps %ymm3, %ymm0, %ymm0
+; X64-AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; X64-AVX2-NEXT:    vpackssdw %xmm3, %xmm0, %xmm0
+; X64-AVX2-NEXT:    vpand %xmm0, %xmm2, %xmm0
+; X64-AVX2-NEXT:    vpand %xmm0, %xmm1, %xmm0
+; X64-AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
+; X64-AVX2-NEXT:    retq
+entry:
+  %cmp = fcmp oge <8 x float> %x, <float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01>
+  %cmp1 = fcmp olt <8 x float> %x, <float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>
+  %and16 = and <8 x i1> %cmp, %cmp1
+  %cmp3 = fcmp une <8 x float> %x, zeroinitializer
+  %and517 = and <8 x i1> %and16, %cmp3
+  %cmp6 = fcmp une <8 x float> %x, <float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000>
+  %and818 = and <8 x i1> %and517, %cmp6
+  %cmp9 = fcmp une <8 x float> %x, <float 0x3FC99999A0000000, float 0x3FC99999A0000000, float 0x3FC99999A0000000, float 0x3FC99999A0000000, float 0x3FC99999A0000000, float 0x3FC99999A0000000, float 0x3FC99999A0000000, float 0x3FC99999A0000000>
+  %and1119 = and <8 x i1> %and818, %cmp9
+  %and11 = sext <8 x i1> %and1119 to <8 x i32>
+  ret <8 x i32> %and11
+}
+
+define <8 x i32> @two_or(<8 x float> %x) {
+; X32-LABEL: two_or:
+; X32:       ## %bb.0: ## %entry
+; X32-NEXT:    vmovaps {{.*#+}} ymm1 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]
+; X32-NEXT:    vcmpleps %ymm0, %ymm1, %ymm1
+; X32-NEXT:    vcmpltps LCPI7_1, %ymm0, %ymm0
+; X32-NEXT:    vorps %ymm0, %ymm1, %ymm0
+; X32-NEXT:    retl
+;
+; X64-LABEL: two_or:
+; X64:       ## %bb.0: ## %entry
+; X64-NEXT:    vmovaps {{.*#+}} ymm1 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]
+; X64-NEXT:    vcmpleps %ymm0, %ymm1, %ymm1
+; X64-NEXT:    vcmpltps {{.*}}(%rip), %ymm0, %ymm0
+; X64-NEXT:    vorps %ymm0, %ymm1, %ymm0
+; X64-NEXT:    retq
+;
+; X32-AVX2-LABEL: two_or:
+; X32-AVX2:       ## %bb.0: ## %entry
+; X32-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm1 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]
+; X32-AVX2-NEXT:    vcmpleps %ymm0, %ymm1, %ymm1
+; X32-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; X32-AVX2-NEXT:    vcmpltps %ymm2, %ymm0, %ymm0
+; X32-AVX2-NEXT:    vorps %ymm0, %ymm1, %ymm0
+; X32-AVX2-NEXT:    retl
+;
+; X64-AVX2-LABEL: two_or:
+; X64-AVX2:       ## %bb.0: ## %entry
+; X64-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm1 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]
+; X64-AVX2-NEXT:    vcmpleps %ymm0, %ymm1, %ymm1
+; X64-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; X64-AVX2-NEXT:    vcmpltps %ymm2, %ymm0, %ymm0
+; X64-AVX2-NEXT:    vorps %ymm0, %ymm1, %ymm0
+; X64-AVX2-NEXT:    retq
+entry:
+  %cmp = fcmp oge <8 x float> %x, <float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01>
+  %cmp1 = fcmp olt <8 x float> %x, <float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>
+  %or4 = or <8 x i1> %cmp, %cmp1
+  %or = sext <8 x i1> %or4 to <8 x i32>
+  ret <8 x i32> %or
+}
+
+define <8 x i32> @three_or(<8 x float> %x) {
+; X32-LABEL: three_or:
+; X32:       ## %bb.0: ## %entry
+; X32-NEXT:    vmovaps {{.*#+}} ymm1 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]
+; X32-NEXT:    vcmpleps %ymm0, %ymm1, %ymm1
+; X32-NEXT:    vcmpltps LCPI8_1, %ymm0, %ymm2
+; X32-NEXT:    vorps %ymm2, %ymm1, %ymm1
+; X32-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; X32-NEXT:    vpackssdw %xmm2, %xmm1, %xmm1
+; X32-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; X32-NEXT:    vcmpneqps %ymm2, %ymm0, %ymm0
+; X32-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; X32-NEXT:    vpackssdw %xmm2, %xmm0, %xmm0
+; X32-NEXT:    vpor %xmm0, %xmm1, %xmm0
+; X32-NEXT:    vpmovsxwd %xmm0, %xmm1
+; X32-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; X32-NEXT:    vpmovsxwd %xmm0, %xmm0
+; X32-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; X32-NEXT:    retl
+;
+; X64-LABEL: three_or:
+; X64:       ## %bb.0: ## %entry
+; X64-NEXT:    vmovaps {{.*#+}} ymm1 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]
+; X64-NEXT:    vcmpleps %ymm0, %ymm1, %ymm1
+; X64-NEXT:    vcmpltps {{.*}}(%rip), %ymm0, %ymm2
+; X64-NEXT:    vorps %ymm2, %ymm1, %ymm1
+; X64-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; X64-NEXT:    vpackssdw %xmm2, %xmm1, %xmm1
+; X64-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; X64-NEXT:    vcmpneqps %ymm2, %ymm0, %ymm0
+; X64-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; X64-NEXT:    vpackssdw %xmm2, %xmm0, %xmm0
+; X64-NEXT:    vpor %xmm0, %xmm1, %xmm0
+; X64-NEXT:    vpmovsxwd %xmm0, %xmm1
+; X64-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; X64-NEXT:    vpmovsxwd %xmm0, %xmm0
+; X64-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; X64-NEXT:    retq
+;
+; X32-AVX2-LABEL: three_or:
+; X32-AVX2:       ## %bb.0: ## %entry
+; X32-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm1 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]
+; X32-AVX2-NEXT:    vcmpleps %ymm0, %ymm1, %ymm1
+; X32-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; X32-AVX2-NEXT:    vcmpltps %ymm2, %ymm0, %ymm2
+; X32-AVX2-NEXT:    vorps %ymm2, %ymm1, %ymm1
+; X32-AVX2-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; X32-AVX2-NEXT:    vpackssdw %xmm2, %xmm1, %xmm1
+; X32-AVX2-NEXT:    vxorps %xmm2, %xmm2, %xmm2
+; X32-AVX2-NEXT:    vcmpneqps %ymm2, %ymm0, %ymm0
+; X32-AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; X32-AVX2-NEXT:    vpackssdw %xmm2, %xmm0, %xmm0
+; X32-AVX2-NEXT:    vpor %xmm0, %xmm1, %xmm0
+; X32-AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
+; X32-AVX2-NEXT:    retl
+;
+; X64-AVX2-LABEL: three_or:
+; X64-AVX2:       ## %bb.0: ## %entry
+; X64-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm1 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]
+; X64-AVX2-NEXT:    vcmpleps %ymm0, %ymm1, %ymm1
+; X64-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; X64-AVX2-NEXT:    vcmpltps %ymm2, %ymm0, %ymm2
+; X64-AVX2-NEXT:    vorps %ymm2, %ymm1, %ymm1
+; X64-AVX2-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; X64-AVX2-NEXT:    vpackssdw %xmm2, %xmm1, %xmm1
+; X64-AVX2-NEXT:    vxorps %xmm2, %xmm2, %xmm2
+; X64-AVX2-NEXT:    vcmpneqps %ymm2, %ymm0, %ymm0
+; X64-AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; X64-AVX2-NEXT:    vpackssdw %xmm2, %xmm0, %xmm0
+; X64-AVX2-NEXT:    vpor %xmm0, %xmm1, %xmm0
+; X64-AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
+; X64-AVX2-NEXT:    retq
+entry:
+  %cmp = fcmp oge <8 x float> %x, <float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01>
+  %cmp1 = fcmp olt <8 x float> %x, <float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>
+  %or8 = or <8 x i1> %cmp, %cmp1
+  %cmp3 = fcmp une <8 x float> %x, zeroinitializer
+  %or59 = or <8 x i1> %or8, %cmp3
+  %or5 = sext <8 x i1> %or59 to <8 x i32>
+  ret <8 x i32> %or5
+}
+; Function Attrs: norecurse nounwind readnone ssp uwtable
+define <8 x i32> @four_or(<8 x float> %x) {
+; X32-LABEL: four_or:
+; X32:       ## %bb.0: ## %entry
+; X32-NEXT:    vmovaps {{.*#+}} ymm1 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]
+; X32-NEXT:    vcmpleps %ymm0, %ymm1, %ymm1
+; X32-NEXT:    vcmpltps LCPI9_1, %ymm0, %ymm2
+; X32-NEXT:    vorps %ymm2, %ymm1, %ymm1
+; X32-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; X32-NEXT:    vpackssdw %xmm2, %xmm1, %xmm1
+; X32-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; X32-NEXT:    vcmpneqps %ymm2, %ymm0, %ymm2
+; X32-NEXT:    vextractf128 $1, %ymm2, %xmm3
+; X32-NEXT:    vpackssdw %xmm3, %xmm2, %xmm2
+; X32-NEXT:    vcmpneqps LCPI9_2, %ymm0, %ymm0
+; X32-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; X32-NEXT:    vpackssdw %xmm3, %xmm0, %xmm0
+; X32-NEXT:    vpor %xmm0, %xmm2, %xmm0
+; X32-NEXT:    vpor %xmm0, %xmm1, %xmm0
+; X32-NEXT:    vpmovsxwd %xmm0, %xmm1
+; X32-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; X32-NEXT:    vpmovsxwd %xmm0, %xmm0
+; X32-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; X32-NEXT:    retl
+;
+; X64-LABEL: four_or:
+; X64:       ## %bb.0: ## %entry
+; X64-NEXT:    vmovaps {{.*#+}} ymm1 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]
+; X64-NEXT:    vcmpleps %ymm0, %ymm1, %ymm1
+; X64-NEXT:    vcmpltps {{.*}}(%rip), %ymm0, %ymm2
+; X64-NEXT:    vorps %ymm2, %ymm1, %ymm1
+; X64-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; X64-NEXT:    vpackssdw %xmm2, %xmm1, %xmm1
+; X64-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; X64-NEXT:    vcmpneqps %ymm2, %ymm0, %ymm2
+; X64-NEXT:    vextractf128 $1, %ymm2, %xmm3
+; X64-NEXT:    vpackssdw %xmm3, %xmm2, %xmm2
+; X64-NEXT:    vcmpneqps {{.*}}(%rip), %ymm0, %ymm0
+; X64-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; X64-NEXT:    vpackssdw %xmm3, %xmm0, %xmm0
+; X64-NEXT:    vpor %xmm0, %xmm2, %xmm0
+; X64-NEXT:    vpor %xmm0, %xmm1, %xmm0
+; X64-NEXT:    vpmovsxwd %xmm0, %xmm1
+; X64-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; X64-NEXT:    vpmovsxwd %xmm0, %xmm0
+; X64-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; X64-NEXT:    retq
+;
+; X32-AVX2-LABEL: four_or:
+; X32-AVX2:       ## %bb.0: ## %entry
+; X32-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm1 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]
+; X32-AVX2-NEXT:    vcmpleps %ymm0, %ymm1, %ymm1
+; X32-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; X32-AVX2-NEXT:    vcmpltps %ymm2, %ymm0, %ymm2
+; X32-AVX2-NEXT:    vorps %ymm2, %ymm1, %ymm1
+; X32-AVX2-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; X32-AVX2-NEXT:    vpackssdw %xmm2, %xmm1, %xmm1
+; X32-AVX2-NEXT:    vxorps %xmm2, %xmm2, %xmm2
+; X32-AVX2-NEXT:    vcmpneqps %ymm2, %ymm0, %ymm2
+; X32-AVX2-NEXT:    vextractf128 $1, %ymm2, %xmm3
+; X32-AVX2-NEXT:    vpackssdw %xmm3, %xmm2, %xmm2
+; X32-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm3 = [1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1]
+; X32-AVX2-NEXT:    vcmpneqps %ymm3, %ymm0, %ymm0
+; X32-AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; X32-AVX2-NEXT:    vpackssdw %xmm3, %xmm0, %xmm0
+; X32-AVX2-NEXT:    vpor %xmm0, %xmm2, %xmm0
+; X32-AVX2-NEXT:    vpor %xmm0, %xmm1, %xmm0
+; X32-AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
+; X32-AVX2-NEXT:    retl
+;
+; X64-AVX2-LABEL: four_or:
+; X64-AVX2:       ## %bb.0: ## %entry
+; X64-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm1 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]
+; X64-AVX2-NEXT:    vcmpleps %ymm0, %ymm1, %ymm1
+; X64-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; X64-AVX2-NEXT:    vcmpltps %ymm2, %ymm0, %ymm2
+; X64-AVX2-NEXT:    vorps %ymm2, %ymm1, %ymm1
+; X64-AVX2-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; X64-AVX2-NEXT:    vpackssdw %xmm2, %xmm1, %xmm1
+; X64-AVX2-NEXT:    vxorps %xmm2, %xmm2, %xmm2
+; X64-AVX2-NEXT:    vcmpneqps %ymm2, %ymm0, %ymm2
+; X64-AVX2-NEXT:    vextractf128 $1, %ymm2, %xmm3
+; X64-AVX2-NEXT:    vpackssdw %xmm3, %xmm2, %xmm2
+; X64-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm3 = [1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1]
+; X64-AVX2-NEXT:    vcmpneqps %ymm3, %ymm0, %ymm0
+; X64-AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; X64-AVX2-NEXT:    vpackssdw %xmm3, %xmm0, %xmm0
+; X64-AVX2-NEXT:    vpor %xmm0, %xmm2, %xmm0
+; X64-AVX2-NEXT:    vpor %xmm0, %xmm1, %xmm0
+; X64-AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
+; X64-AVX2-NEXT:    retq
+entry:
+  %cmp = fcmp oge <8 x float> %x, <float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01>
+  %cmp1 = fcmp olt <8 x float> %x, <float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>
+  %or12 = or <8 x i1> %cmp, %cmp1
+  %cmp3 = fcmp une <8 x float> %x, zeroinitializer
+  %or513 = or <8 x i1> %or12, %cmp3
+  %cmp6 = fcmp une <8 x float> %x, <float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000>
+  %or814 = or <8 x i1> %or513, %cmp6
+  %or8 = sext <8 x i1> %or814 to <8 x i32>
+  ret <8 x i32> %or8
+}
+; Function Attrs: norecurse nounwind readnone ssp uwtable
+define <8 x i32> @five_or(<8 x float> %x) {
+; X32-LABEL: five_or:
+; X32:       ## %bb.0: ## %entry
+; X32-NEXT:    vmovaps {{.*#+}} ymm1 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]
+; X32-NEXT:    vcmpleps %ymm0, %ymm1, %ymm1
+; X32-NEXT:    vcmpltps LCPI10_1, %ymm0, %ymm2
+; X32-NEXT:    vorps %ymm2, %ymm1, %ymm1
+; X32-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; X32-NEXT:    vpackssdw %xmm2, %xmm1, %xmm1
+; X32-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; X32-NEXT:    vcmpneqps %ymm2, %ymm0, %ymm2
+; X32-NEXT:    vextractf128 $1, %ymm2, %xmm3
+; X32-NEXT:    vpackssdw %xmm3, %xmm2, %xmm2
+; X32-NEXT:    vcmpneqps LCPI10_2, %ymm0, %ymm3
+; X32-NEXT:    vextractf128 $1, %ymm3, %xmm4
+; X32-NEXT:    vpackssdw %xmm4, %xmm3, %xmm3
+; X32-NEXT:    vpor %xmm3, %xmm2, %xmm2
+; X32-NEXT:    vcmpneqps LCPI10_3, %ymm0, %ymm0
+; X32-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; X32-NEXT:    vpackssdw %xmm3, %xmm0, %xmm0
+; X32-NEXT:    vpor %xmm0, %xmm2, %xmm0
+; X32-NEXT:    vpor %xmm0, %xmm1, %xmm0
+; X32-NEXT:    vpmovsxwd %xmm0, %xmm1
+; X32-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; X32-NEXT:    vpmovsxwd %xmm0, %xmm0
+; X32-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; X32-NEXT:    retl
+;
+; X64-LABEL: five_or:
+; X64:       ## %bb.0: ## %entry
+; X64-NEXT:    vmovaps {{.*#+}} ymm1 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]
+; X64-NEXT:    vcmpleps %ymm0, %ymm1, %ymm1
+; X64-NEXT:    vcmpltps {{.*}}(%rip), %ymm0, %ymm2
+; X64-NEXT:    vorps %ymm2, %ymm1, %ymm1
+; X64-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; X64-NEXT:    vpackssdw %xmm2, %xmm1, %xmm1
+; X64-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; X64-NEXT:    vcmpneqps %ymm2, %ymm0, %ymm2
+; X64-NEXT:    vextractf128 $1, %ymm2, %xmm3
+; X64-NEXT:    vpackssdw %xmm3, %xmm2, %xmm2
+; X64-NEXT:    vcmpneqps {{.*}}(%rip), %ymm0, %ymm3
+; X64-NEXT:    vextractf128 $1, %ymm3, %xmm4
+; X64-NEXT:    vpackssdw %xmm4, %xmm3, %xmm3
+; X64-NEXT:    vpor %xmm3, %xmm2, %xmm2
+; X64-NEXT:    vcmpneqps {{.*}}(%rip), %ymm0, %ymm0
+; X64-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; X64-NEXT:    vpackssdw %xmm3, %xmm0, %xmm0
+; X64-NEXT:    vpor %xmm0, %xmm2, %xmm0
+; X64-NEXT:    vpor %xmm0, %xmm1, %xmm0
+; X64-NEXT:    vpmovsxwd %xmm0, %xmm1
+; X64-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; X64-NEXT:    vpmovsxwd %xmm0, %xmm0
+; X64-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; X64-NEXT:    retq
+;
+; X32-AVX2-LABEL: five_or:
+; X32-AVX2:       ## %bb.0: ## %entry
+; X32-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm1 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]
+; X32-AVX2-NEXT:    vcmpleps %ymm0, %ymm1, %ymm1
+; X32-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; X32-AVX2-NEXT:    vcmpltps %ymm2, %ymm0, %ymm2
+; X32-AVX2-NEXT:    vorps %ymm2, %ymm1, %ymm1
+; X32-AVX2-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; X32-AVX2-NEXT:    vpackssdw %xmm2, %xmm1, %xmm1
+; X32-AVX2-NEXT:    vxorps %xmm2, %xmm2, %xmm2
+; X32-AVX2-NEXT:    vcmpneqps %ymm2, %ymm0, %ymm2
+; X32-AVX2-NEXT:    vextractf128 $1, %ymm2, %xmm3
+; X32-AVX2-NEXT:    vpackssdw %xmm3, %xmm2, %xmm2
+; X32-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm3 = [1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1]
+; X32-AVX2-NEXT:    vcmpneqps %ymm3, %ymm0, %ymm3
+; X32-AVX2-NEXT:    vextractf128 $1, %ymm3, %xmm4
+; X32-AVX2-NEXT:    vpackssdw %xmm4, %xmm3, %xmm3
+; X32-AVX2-NEXT:    vpor %xmm3, %xmm2, %xmm2
+; X32-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm3 = [2.00000003E-1,2.00000003E-1,2.00000003E-1,2.00000003E-1,2.00000003E-1,2.00000003E-1,2.00000003E-1,2.00000003E-1]
+; X32-AVX2-NEXT:    vcmpneqps %ymm3, %ymm0, %ymm0
+; X32-AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; X32-AVX2-NEXT:    vpackssdw %xmm3, %xmm0, %xmm0
+; X32-AVX2-NEXT:    vpor %xmm0, %xmm2, %xmm0
+; X32-AVX2-NEXT:    vpor %xmm0, %xmm1, %xmm0
+; X32-AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
+; X32-AVX2-NEXT:    retl
+;
+; X64-AVX2-LABEL: five_or:
+; X64-AVX2:       ## %bb.0: ## %entry
+; X64-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm1 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]
+; X64-AVX2-NEXT:    vcmpleps %ymm0, %ymm1, %ymm1
+; X64-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; X64-AVX2-NEXT:    vcmpltps %ymm2, %ymm0, %ymm2
+; X64-AVX2-NEXT:    vorps %ymm2, %ymm1, %ymm1
+; X64-AVX2-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; X64-AVX2-NEXT:    vpackssdw %xmm2, %xmm1, %xmm1
+; X64-AVX2-NEXT:    vxorps %xmm2, %xmm2, %xmm2
+; X64-AVX2-NEXT:    vcmpneqps %ymm2, %ymm0, %ymm2
+; X64-AVX2-NEXT:    vextractf128 $1, %ymm2, %xmm3
+; X64-AVX2-NEXT:    vpackssdw %xmm3, %xmm2, %xmm2
+; X64-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm3 = [1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1]
+; X64-AVX2-NEXT:    vcmpneqps %ymm3, %ymm0, %ymm3
+; X64-AVX2-NEXT:    vextractf128 $1, %ymm3, %xmm4
+; X64-AVX2-NEXT:    vpackssdw %xmm4, %xmm3, %xmm3
+; X64-AVX2-NEXT:    vpor %xmm3, %xmm2, %xmm2
+; X64-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm3 = [2.00000003E-1,2.00000003E-1,2.00000003E-1,2.00000003E-1,2.00000003E-1,2.00000003E-1,2.00000003E-1,2.00000003E-1]
+; X64-AVX2-NEXT:    vcmpneqps %ymm3, %ymm0, %ymm0
+; X64-AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; X64-AVX2-NEXT:    vpackssdw %xmm3, %xmm0, %xmm0
+; X64-AVX2-NEXT:    vpor %xmm0, %xmm2, %xmm0
+; X64-AVX2-NEXT:    vpor %xmm0, %xmm1, %xmm0
+; X64-AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
+; X64-AVX2-NEXT:    retq
+entry:
+  %cmp = fcmp oge <8 x float> %x, <float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01>
+  %cmp1 = fcmp olt <8 x float> %x, <float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>
+  %or16 = or <8 x i1> %cmp, %cmp1
+  %cmp3 = fcmp une <8 x float> %x, zeroinitializer
+  %or517 = or <8 x i1> %or16, %cmp3
+  %cmp6 = fcmp une <8 x float> %x, <float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000>
+  %or818 = or <8 x i1> %or517, %cmp6
+  %cmp9 = fcmp une <8 x float> %x, <float 0x3FC99999A0000000, float 0x3FC99999A0000000, float 0x3FC99999A0000000, float 0x3FC99999A0000000, float 0x3FC99999A0000000, float 0x3FC99999A0000000, float 0x3FC99999A0000000, float 0x3FC99999A0000000>
+  %or1119 = or <8 x i1> %or818, %cmp9
+  %or11 = sext <8 x i1> %or1119 to <8 x i32>
+  ret <8 x i32> %or11
+}
+
+define <8 x i32> @three_or_and(<8 x float> %x) {
+; X32-LABEL: three_or_and:
+; X32:       ## %bb.0: ## %entry
+; X32-NEXT:    vmovaps {{.*#+}} ymm1 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]
+; X32-NEXT:    vcmpleps %ymm0, %ymm1, %ymm1
+; X32-NEXT:    vcmpltps LCPI11_1, %ymm0, %ymm2
+; X32-NEXT:    vxorps %xmm3, %xmm3, %xmm3
+; X32-NEXT:    vcmpneqps %ymm3, %ymm0, %ymm0
+; X32-NEXT:    vandps %ymm0, %ymm2, %ymm0
+; X32-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; X32-NEXT:    retl
+;
+; X64-LABEL: three_or_and:
+; X64:       ## %bb.0: ## %entry
+; X64-NEXT:    vmovaps {{.*#+}} ymm1 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]
+; X64-NEXT:    vcmpleps %ymm0, %ymm1, %ymm1
+; X64-NEXT:    vcmpltps {{.*}}(%rip), %ymm0, %ymm2
+; X64-NEXT:    vxorps %xmm3, %xmm3, %xmm3
+; X64-NEXT:    vcmpneqps %ymm3, %ymm0, %ymm0
+; X64-NEXT:    vandps %ymm0, %ymm2, %ymm0
+; X64-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; X64-NEXT:    retq
+;
+; X32-AVX2-LABEL: three_or_and:
+; X32-AVX2:       ## %bb.0: ## %entry
+; X32-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm1 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]
+; X32-AVX2-NEXT:    vcmpleps %ymm0, %ymm1, %ymm1
+; X32-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; X32-AVX2-NEXT:    vcmpltps %ymm2, %ymm0, %ymm2
+; X32-AVX2-NEXT:    vxorps %xmm3, %xmm3, %xmm3
+; X32-AVX2-NEXT:    vcmpneqps %ymm3, %ymm0, %ymm0
+; X32-AVX2-NEXT:    vandps %ymm0, %ymm2, %ymm0
+; X32-AVX2-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; X32-AVX2-NEXT:    retl
+;
+; X64-AVX2-LABEL: three_or_and:
+; X64-AVX2:       ## %bb.0: ## %entry
+; X64-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm1 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]
+; X64-AVX2-NEXT:    vcmpleps %ymm0, %ymm1, %ymm1
+; X64-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; X64-AVX2-NEXT:    vcmpltps %ymm2, %ymm0, %ymm2
+; X64-AVX2-NEXT:    vxorps %xmm3, %xmm3, %xmm3
+; X64-AVX2-NEXT:    vcmpneqps %ymm3, %ymm0, %ymm0
+; X64-AVX2-NEXT:    vandps %ymm0, %ymm2, %ymm0
+; X64-AVX2-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; X64-AVX2-NEXT:    retq
+entry:
+  %cmp = fcmp oge <8 x float> %x, <float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01>
+  %cmp1 = fcmp olt <8 x float> %x, <float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>
+  %cmp3 = fcmp une <8 x float> %x, zeroinitializer
+  %and7 = and <8 x i1> %cmp1, %cmp3
+  %or8 = or <8 x i1> %and7, %cmp
+  %or = sext <8 x i1> %or8 to <8 x i32>
+  ret <8 x i32> %or
+}
+
+define <8 x i32> @four_or_and(<8 x float> %x) {
+; X32-LABEL: four_or_and:
+; X32:       ## %bb.0: ## %entry
+; X32-NEXT:    vmovaps {{.*#+}} ymm1 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]
+; X32-NEXT:    vcmpleps %ymm0, %ymm1, %ymm1
+; X32-NEXT:    vcmpltps LCPI12_1, %ymm0, %ymm2
+; X32-NEXT:    vandps %ymm2, %ymm1, %ymm1
+; X32-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; X32-NEXT:    vpackssdw %xmm2, %xmm1, %xmm1
+; X32-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; X32-NEXT:    vcmpneqps %ymm2, %ymm0, %ymm2
+; X32-NEXT:    vcmpneqps LCPI12_2, %ymm0, %ymm0
+; X32-NEXT:    vandps %ymm0, %ymm2, %ymm0
+; X32-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; X32-NEXT:    vpackssdw %xmm2, %xmm0, %xmm0
+; X32-NEXT:    vpor %xmm0, %xmm1, %xmm0
+; X32-NEXT:    vpmovsxwd %xmm0, %xmm1
+; X32-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; X32-NEXT:    vpmovsxwd %xmm0, %xmm0
+; X32-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; X32-NEXT:    retl
+;
+; X64-LABEL: four_or_and:
+; X64:       ## %bb.0: ## %entry
+; X64-NEXT:    vmovaps {{.*#+}} ymm1 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]
+; X64-NEXT:    vcmpleps %ymm0, %ymm1, %ymm1
+; X64-NEXT:    vcmpltps {{.*}}(%rip), %ymm0, %ymm2
+; X64-NEXT:    vandps %ymm2, %ymm1, %ymm1
+; X64-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; X64-NEXT:    vpackssdw %xmm2, %xmm1, %xmm1
+; X64-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; X64-NEXT:    vcmpneqps %ymm2, %ymm0, %ymm2
+; X64-NEXT:    vcmpneqps {{.*}}(%rip), %ymm0, %ymm0
+; X64-NEXT:    vandps %ymm0, %ymm2, %ymm0
+; X64-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; X64-NEXT:    vpackssdw %xmm2, %xmm0, %xmm0
+; X64-NEXT:    vpor %xmm0, %xmm1, %xmm0
+; X64-NEXT:    vpmovsxwd %xmm0, %xmm1
+; X64-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; X64-NEXT:    vpmovsxwd %xmm0, %xmm0
+; X64-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; X64-NEXT:    retq
+;
+; X32-AVX2-LABEL: four_or_and:
+; X32-AVX2:       ## %bb.0: ## %entry
+; X32-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm1 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]
+; X32-AVX2-NEXT:    vcmpleps %ymm0, %ymm1, %ymm1
+; X32-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; X32-AVX2-NEXT:    vcmpltps %ymm2, %ymm0, %ymm2
+; X32-AVX2-NEXT:    vandps %ymm2, %ymm1, %ymm1
+; X32-AVX2-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; X32-AVX2-NEXT:    vpackssdw %xmm2, %xmm1, %xmm1
+; X32-AVX2-NEXT:    vxorps %xmm2, %xmm2, %xmm2
+; X32-AVX2-NEXT:    vcmpneqps %ymm2, %ymm0, %ymm2
+; X32-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm3 = [1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1]
+; X32-AVX2-NEXT:    vcmpneqps %ymm3, %ymm0, %ymm0
+; X32-AVX2-NEXT:    vandps %ymm0, %ymm2, %ymm0
+; X32-AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; X32-AVX2-NEXT:    vpackssdw %xmm2, %xmm0, %xmm0
+; X32-AVX2-NEXT:    vpor %xmm0, %xmm1, %xmm0
+; X32-AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
+; X32-AVX2-NEXT:    retl
+;
+; X64-AVX2-LABEL: four_or_and:
+; X64-AVX2:       ## %bb.0: ## %entry
+; X64-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm1 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]
+; X64-AVX2-NEXT:    vcmpleps %ymm0, %ymm1, %ymm1
+; X64-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; X64-AVX2-NEXT:    vcmpltps %ymm2, %ymm0, %ymm2
+; X64-AVX2-NEXT:    vandps %ymm2, %ymm1, %ymm1
+; X64-AVX2-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; X64-AVX2-NEXT:    vpackssdw %xmm2, %xmm1, %xmm1
+; X64-AVX2-NEXT:    vxorps %xmm2, %xmm2, %xmm2
+; X64-AVX2-NEXT:    vcmpneqps %ymm2, %ymm0, %ymm2
+; X64-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm3 = [1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1]
+; X64-AVX2-NEXT:    vcmpneqps %ymm3, %ymm0, %ymm0
+; X64-AVX2-NEXT:    vandps %ymm0, %ymm2, %ymm0
+; X64-AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; X64-AVX2-NEXT:    vpackssdw %xmm2, %xmm0, %xmm0
+; X64-AVX2-NEXT:    vpor %xmm0, %xmm1, %xmm0
+; X64-AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
+; X64-AVX2-NEXT:    retq
+entry:
+  %cmp = fcmp oge <8 x float> %x, <float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01>
+  %cmp1 = fcmp olt <8 x float> %x, <float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>
+  %and11 = and <8 x i1> %cmp, %cmp1
+  %cmp3 = fcmp une <8 x float> %x, zeroinitializer
+  %cmp5 = fcmp une <8 x float> %x, <float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000>
+  %and712 = and <8 x i1> %cmp3, %cmp5
+  %or13 = or <8 x i1> %and11, %and712
+  %or = sext <8 x i1> %or13 to <8 x i32>
+  ret <8 x i32> %or
+}
+
+define <8 x i32> @five_or_and(<8 x float> %x) {
+; X32-LABEL: five_or_and:
+; X32:       ## %bb.0: ## %entry
+; X32-NEXT:    vmovaps {{.*#+}} ymm1 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]
+; X32-NEXT:    vcmpleps %ymm0, %ymm1, %ymm1
+; X32-NEXT:    vcmpltps LCPI13_1, %ymm0, %ymm2
+; X32-NEXT:    vxorps %xmm3, %xmm3, %xmm3
+; X32-NEXT:    vcmpneqps %ymm3, %ymm0, %ymm3
+; X32-NEXT:    vandps %ymm3, %ymm2, %ymm2
+; X32-NEXT:    vorps %ymm1, %ymm2, %ymm1
+; X32-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; X32-NEXT:    vpackssdw %xmm2, %xmm1, %xmm1
+; X32-NEXT:    vcmpneqps LCPI13_2, %ymm0, %ymm2
+; X32-NEXT:    vcmpneqps LCPI13_3, %ymm0, %ymm0
+; X32-NEXT:    vandps %ymm0, %ymm2, %ymm0
+; X32-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; X32-NEXT:    vpackssdw %xmm2, %xmm0, %xmm0
+; X32-NEXT:    vpor %xmm0, %xmm1, %xmm0
+; X32-NEXT:    vpmovsxwd %xmm0, %xmm1
+; X32-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; X32-NEXT:    vpmovsxwd %xmm0, %xmm0
+; X32-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; X32-NEXT:    retl
+;
+; X64-LABEL: five_or_and:
+; X64:       ## %bb.0: ## %entry
+; X64-NEXT:    vmovaps {{.*#+}} ymm1 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]
+; X64-NEXT:    vcmpleps %ymm0, %ymm1, %ymm1
+; X64-NEXT:    vcmpltps {{.*}}(%rip), %ymm0, %ymm2
+; X64-NEXT:    vxorps %xmm3, %xmm3, %xmm3
+; X64-NEXT:    vcmpneqps %ymm3, %ymm0, %ymm3
+; X64-NEXT:    vandps %ymm3, %ymm2, %ymm2
+; X64-NEXT:    vorps %ymm1, %ymm2, %ymm1
+; X64-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; X64-NEXT:    vpackssdw %xmm2, %xmm1, %xmm1
+; X64-NEXT:    vcmpneqps {{.*}}(%rip), %ymm0, %ymm2
+; X64-NEXT:    vcmpneqps {{.*}}(%rip), %ymm0, %ymm0
+; X64-NEXT:    vandps %ymm0, %ymm2, %ymm0
+; X64-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; X64-NEXT:    vpackssdw %xmm2, %xmm0, %xmm0
+; X64-NEXT:    vpor %xmm0, %xmm1, %xmm0
+; X64-NEXT:    vpmovsxwd %xmm0, %xmm1
+; X64-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; X64-NEXT:    vpmovsxwd %xmm0, %xmm0
+; X64-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; X64-NEXT:    retq
+;
+; X32-AVX2-LABEL: five_or_and:
+; X32-AVX2:       ## %bb.0: ## %entry
+; X32-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm1 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]
+; X32-AVX2-NEXT:    vcmpleps %ymm0, %ymm1, %ymm1
+; X32-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; X32-AVX2-NEXT:    vcmpltps %ymm2, %ymm0, %ymm2
+; X32-AVX2-NEXT:    vxorps %xmm3, %xmm3, %xmm3
+; X32-AVX2-NEXT:    vcmpneqps %ymm3, %ymm0, %ymm3
+; X32-AVX2-NEXT:    vandps %ymm3, %ymm2, %ymm2
+; X32-AVX2-NEXT:    vorps %ymm1, %ymm2, %ymm1
+; X32-AVX2-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; X32-AVX2-NEXT:    vpackssdw %xmm2, %xmm1, %xmm1
+; X32-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm2 = [1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1]
+; X32-AVX2-NEXT:    vcmpneqps %ymm2, %ymm0, %ymm2
+; X32-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm3 = [2.00000003E-1,2.00000003E-1,2.00000003E-1,2.00000003E-1,2.00000003E-1,2.00000003E-1,2.00000003E-1,2.00000003E-1]
+; X32-AVX2-NEXT:    vcmpneqps %ymm3, %ymm0, %ymm0
+; X32-AVX2-NEXT:    vandps %ymm0, %ymm2, %ymm0
+; X32-AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; X32-AVX2-NEXT:    vpackssdw %xmm2, %xmm0, %xmm0
+; X32-AVX2-NEXT:    vpor %xmm0, %xmm1, %xmm0
+; X32-AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
+; X32-AVX2-NEXT:    retl
+;
+; X64-AVX2-LABEL: five_or_and:
+; X64-AVX2:       ## %bb.0: ## %entry
+; X64-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm1 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]
+; X64-AVX2-NEXT:    vcmpleps %ymm0, %ymm1, %ymm1
+; X64-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; X64-AVX2-NEXT:    vcmpltps %ymm2, %ymm0, %ymm2
+; X64-AVX2-NEXT:    vxorps %xmm3, %xmm3, %xmm3
+; X64-AVX2-NEXT:    vcmpneqps %ymm3, %ymm0, %ymm3
+; X64-AVX2-NEXT:    vandps %ymm3, %ymm2, %ymm2
+; X64-AVX2-NEXT:    vorps %ymm1, %ymm2, %ymm1
+; X64-AVX2-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; X64-AVX2-NEXT:    vpackssdw %xmm2, %xmm1, %xmm1
+; X64-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm2 = [1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1]
+; X64-AVX2-NEXT:    vcmpneqps %ymm2, %ymm0, %ymm2
+; X64-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm3 = [2.00000003E-1,2.00000003E-1,2.00000003E-1,2.00000003E-1,2.00000003E-1,2.00000003E-1,2.00000003E-1,2.00000003E-1]
+; X64-AVX2-NEXT:    vcmpneqps %ymm3, %ymm0, %ymm0
+; X64-AVX2-NEXT:    vandps %ymm0, %ymm2, %ymm0
+; X64-AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; X64-AVX2-NEXT:    vpackssdw %xmm2, %xmm0, %xmm0
+; X64-AVX2-NEXT:    vpor %xmm0, %xmm1, %xmm0
+; X64-AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
+; X64-AVX2-NEXT:    retq
+entry:
+  %cmp = fcmp oge <8 x float> %x, <float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01>
+  %cmp1 = fcmp olt <8 x float> %x, <float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>
+  %cmp3 = fcmp une <8 x float> %x, zeroinitializer
+  %and15 = and <8 x i1> %cmp1, %cmp3
+  %or16 = or <8 x i1> %and15, %cmp
+  %cmp5 = fcmp une <8 x float> %x, <float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000>
+  %cmp7 = fcmp une <8 x float> %x, <float 0x3FC99999A0000000, float 0x3FC99999A0000000, float 0x3FC99999A0000000, float 0x3FC99999A0000000, float 0x3FC99999A0000000, float 0x3FC99999A0000000, float 0x3FC99999A0000000, float 0x3FC99999A0000000>
+  %and917 = and <8 x i1> %cmp5, %cmp7
+  %or1018 = or <8 x i1> %or16, %and917
+  %or10 = sext <8 x i1> %or1018 to <8 x i32>
+  ret <8 x i32> %or10
+}
+
+define <8 x i32> @four_or_and_xor(<8 x float> %x) {
+; X32-LABEL: four_or_and_xor:
+; X32:       ## %bb.0: ## %entry
+; X32-NEXT:    vmovaps {{.*#+}} ymm1 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]
+; X32-NEXT:    vcmpleps %ymm0, %ymm1, %ymm1
+; X32-NEXT:    vcmpltps LCPI14_1, %ymm0, %ymm2
+; X32-NEXT:    vxorps %ymm2, %ymm1, %ymm1
+; X32-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; X32-NEXT:    vpackssdw %xmm2, %xmm1, %xmm1
+; X32-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; X32-NEXT:    vcmpneqps %ymm2, %ymm0, %ymm2
+; X32-NEXT:    vcmpneqps LCPI14_2, %ymm0, %ymm0
+; X32-NEXT:    vandps %ymm0, %ymm2, %ymm0
+; X32-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; X32-NEXT:    vpackssdw %xmm2, %xmm0, %xmm0
+; X32-NEXT:    vpor %xmm0, %xmm1, %xmm0
+; X32-NEXT:    vpmovsxwd %xmm0, %xmm1
+; X32-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; X32-NEXT:    vpmovsxwd %xmm0, %xmm0
+; X32-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; X32-NEXT:    retl
+;
+; X64-LABEL: four_or_and_xor:
+; X64:       ## %bb.0: ## %entry
+; X64-NEXT:    vmovaps {{.*#+}} ymm1 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]
+; X64-NEXT:    vcmpleps %ymm0, %ymm1, %ymm1
+; X64-NEXT:    vcmpltps {{.*}}(%rip), %ymm0, %ymm2
+; X64-NEXT:    vxorps %ymm2, %ymm1, %ymm1
+; X64-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; X64-NEXT:    vpackssdw %xmm2, %xmm1, %xmm1
+; X64-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; X64-NEXT:    vcmpneqps %ymm2, %ymm0, %ymm2
+; X64-NEXT:    vcmpneqps {{.*}}(%rip), %ymm0, %ymm0
+; X64-NEXT:    vandps %ymm0, %ymm2, %ymm0
+; X64-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; X64-NEXT:    vpackssdw %xmm2, %xmm0, %xmm0
+; X64-NEXT:    vpor %xmm0, %xmm1, %xmm0
+; X64-NEXT:    vpmovsxwd %xmm0, %xmm1
+; X64-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; X64-NEXT:    vpmovsxwd %xmm0, %xmm0
+; X64-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; X64-NEXT:    retq
+;
+; X32-AVX2-LABEL: four_or_and_xor:
+; X32-AVX2:       ## %bb.0: ## %entry
+; X32-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm1 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]
+; X32-AVX2-NEXT:    vcmpleps %ymm0, %ymm1, %ymm1
+; X32-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; X32-AVX2-NEXT:    vcmpltps %ymm2, %ymm0, %ymm2
+; X32-AVX2-NEXT:    vxorps %ymm2, %ymm1, %ymm1
+; X32-AVX2-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; X32-AVX2-NEXT:    vpackssdw %xmm2, %xmm1, %xmm1
+; X32-AVX2-NEXT:    vxorps %xmm2, %xmm2, %xmm2
+; X32-AVX2-NEXT:    vcmpneqps %ymm2, %ymm0, %ymm2
+; X32-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm3 = [1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1]
+; X32-AVX2-NEXT:    vcmpneqps %ymm3, %ymm0, %ymm0
+; X32-AVX2-NEXT:    vandps %ymm0, %ymm2, %ymm0
+; X32-AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; X32-AVX2-NEXT:    vpackssdw %xmm2, %xmm0, %xmm0
+; X32-AVX2-NEXT:    vpor %xmm0, %xmm1, %xmm0
+; X32-AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
+; X32-AVX2-NEXT:    retl
+;
+; X64-AVX2-LABEL: four_or_and_xor:
+; X64-AVX2:       ## %bb.0: ## %entry
+; X64-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm1 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]
+; X64-AVX2-NEXT:    vcmpleps %ymm0, %ymm1, %ymm1
+; X64-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; X64-AVX2-NEXT:    vcmpltps %ymm2, %ymm0, %ymm2
+; X64-AVX2-NEXT:    vxorps %ymm2, %ymm1, %ymm1
+; X64-AVX2-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; X64-AVX2-NEXT:    vpackssdw %xmm2, %xmm1, %xmm1
+; X64-AVX2-NEXT:    vxorps %xmm2, %xmm2, %xmm2
+; X64-AVX2-NEXT:    vcmpneqps %ymm2, %ymm0, %ymm2
+; X64-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm3 = [1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1]
+; X64-AVX2-NEXT:    vcmpneqps %ymm3, %ymm0, %ymm0
+; X64-AVX2-NEXT:    vandps %ymm0, %ymm2, %ymm0
+; X64-AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; X64-AVX2-NEXT:    vpackssdw %xmm2, %xmm0, %xmm0
+; X64-AVX2-NEXT:    vpor %xmm0, %xmm1, %xmm0
+; X64-AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
+; X64-AVX2-NEXT:    retq
+entry:
+  %cmp = fcmp oge <8 x float> %x, <float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01>
+  %cmp1 = fcmp olt <8 x float> %x, <float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>
+  %xor10 = xor <8 x i1> %cmp, %cmp1
+  %cmp3 = fcmp une <8 x float> %x, zeroinitializer
+  %cmp5 = fcmp une <8 x float> %x, <float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000>
+  %and11 = and <8 x i1> %cmp3, %cmp5
+  %or12 = or <8 x i1> %xor10, %and11
+  %or = sext <8 x i1> %or12 to <8 x i32>
+  ret <8 x i32> %or
+}
+; Function Attrs: norecurse nounwind readnone ssp uwtable
+define <8 x i32> @five_or_and_xor(<8 x float> %x) {
+; X32-LABEL: five_or_and_xor:
+; X32:       ## %bb.0: ## %entry
+; X32-NEXT:    vmovaps {{.*#+}} ymm1 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]
+; X32-NEXT:    vcmpleps %ymm0, %ymm1, %ymm1
+; X32-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; X32-NEXT:    vpackssdw %xmm2, %xmm1, %xmm1
+; X32-NEXT:    vcmpltps LCPI15_1, %ymm0, %ymm2
+; X32-NEXT:    vxorps %xmm3, %xmm3, %xmm3
+; X32-NEXT:    vcmpneqps %ymm3, %ymm0, %ymm3
+; X32-NEXT:    vxorps %ymm3, %ymm2, %ymm2
+; X32-NEXT:    vextractf128 $1, %ymm2, %xmm3
+; X32-NEXT:    vpackssdw %xmm3, %xmm2, %xmm2
+; X32-NEXT:    vcmpneqps LCPI15_2, %ymm0, %ymm3
+; X32-NEXT:    vcmpneqps LCPI15_3, %ymm0, %ymm0
+; X32-NEXT:    vandps %ymm0, %ymm3, %ymm0
+; X32-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; X32-NEXT:    vpackssdw %xmm3, %xmm0, %xmm0
+; X32-NEXT:    vpxor %xmm0, %xmm2, %xmm0
+; X32-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; X32-NEXT:    vpmovsxwd %xmm0, %xmm1
+; X32-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; X32-NEXT:    vpmovsxwd %xmm0, %xmm0
+; X32-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; X32-NEXT:    retl
+;
+; X64-LABEL: five_or_and_xor:
+; X64:       ## %bb.0: ## %entry
+; X64-NEXT:    vmovaps {{.*#+}} ymm1 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]
+; X64-NEXT:    vcmpleps %ymm0, %ymm1, %ymm1
+; X64-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; X64-NEXT:    vpackssdw %xmm2, %xmm1, %xmm1
+; X64-NEXT:    vcmpltps {{.*}}(%rip), %ymm0, %ymm2
+; X64-NEXT:    vxorps %xmm3, %xmm3, %xmm3
+; X64-NEXT:    vcmpneqps %ymm3, %ymm0, %ymm3
+; X64-NEXT:    vxorps %ymm3, %ymm2, %ymm2
+; X64-NEXT:    vextractf128 $1, %ymm2, %xmm3
+; X64-NEXT:    vpackssdw %xmm3, %xmm2, %xmm2
+; X64-NEXT:    vcmpneqps {{.*}}(%rip), %ymm0, %ymm3
+; X64-NEXT:    vcmpneqps {{.*}}(%rip), %ymm0, %ymm0
+; X64-NEXT:    vandps %ymm0, %ymm3, %ymm0
+; X64-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; X64-NEXT:    vpackssdw %xmm3, %xmm0, %xmm0
+; X64-NEXT:    vpxor %xmm0, %xmm2, %xmm0
+; X64-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; X64-NEXT:    vpmovsxwd %xmm0, %xmm1
+; X64-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; X64-NEXT:    vpmovsxwd %xmm0, %xmm0
+; X64-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; X64-NEXT:    retq
+;
+; X32-AVX2-LABEL: five_or_and_xor:
+; X32-AVX2:       ## %bb.0: ## %entry
+; X32-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm1 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]
+; X32-AVX2-NEXT:    vcmpleps %ymm0, %ymm1, %ymm1
+; X32-AVX2-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; X32-AVX2-NEXT:    vpackssdw %xmm2, %xmm1, %xmm1
+; X32-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; X32-AVX2-NEXT:    vcmpltps %ymm2, %ymm0, %ymm2
+; X32-AVX2-NEXT:    vxorps %xmm3, %xmm3, %xmm3
+; X32-AVX2-NEXT:    vcmpneqps %ymm3, %ymm0, %ymm3
+; X32-AVX2-NEXT:    vxorps %ymm3, %ymm2, %ymm2
+; X32-AVX2-NEXT:    vextractf128 $1, %ymm2, %xmm3
+; X32-AVX2-NEXT:    vpackssdw %xmm3, %xmm2, %xmm2
+; X32-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm3 = [1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1]
+; X32-AVX2-NEXT:    vcmpneqps %ymm3, %ymm0, %ymm3
+; X32-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm4 = [2.00000003E-1,2.00000003E-1,2.00000003E-1,2.00000003E-1,2.00000003E-1,2.00000003E-1,2.00000003E-1,2.00000003E-1]
+; X32-AVX2-NEXT:    vcmpneqps %ymm4, %ymm0, %ymm0
+; X32-AVX2-NEXT:    vandps %ymm0, %ymm3, %ymm0
+; X32-AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; X32-AVX2-NEXT:    vpackssdw %xmm3, %xmm0, %xmm0
+; X32-AVX2-NEXT:    vpxor %xmm0, %xmm2, %xmm0
+; X32-AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; X32-AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
+; X32-AVX2-NEXT:    retl
+;
+; X64-AVX2-LABEL: five_or_and_xor:
+; X64-AVX2:       ## %bb.0: ## %entry
+; X64-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm1 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]
+; X64-AVX2-NEXT:    vcmpleps %ymm0, %ymm1, %ymm1
+; X64-AVX2-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; X64-AVX2-NEXT:    vpackssdw %xmm2, %xmm1, %xmm1
+; X64-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; X64-AVX2-NEXT:    vcmpltps %ymm2, %ymm0, %ymm2
+; X64-AVX2-NEXT:    vxorps %xmm3, %xmm3, %xmm3
+; X64-AVX2-NEXT:    vcmpneqps %ymm3, %ymm0, %ymm3
+; X64-AVX2-NEXT:    vxorps %ymm3, %ymm2, %ymm2
+; X64-AVX2-NEXT:    vextractf128 $1, %ymm2, %xmm3
+; X64-AVX2-NEXT:    vpackssdw %xmm3, %xmm2, %xmm2
+; X64-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm3 = [1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1]
+; X64-AVX2-NEXT:    vcmpneqps %ymm3, %ymm0, %ymm3
+; X64-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm4 = [2.00000003E-1,2.00000003E-1,2.00000003E-1,2.00000003E-1,2.00000003E-1,2.00000003E-1,2.00000003E-1,2.00000003E-1]
+; X64-AVX2-NEXT:    vcmpneqps %ymm4, %ymm0, %ymm0
+; X64-AVX2-NEXT:    vandps %ymm0, %ymm3, %ymm0
+; X64-AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; X64-AVX2-NEXT:    vpackssdw %xmm3, %xmm0, %xmm0
+; X64-AVX2-NEXT:    vpxor %xmm0, %xmm2, %xmm0
+; X64-AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
+; X64-AVX2-NEXT:    retq
+entry:
+  %cmp = fcmp oge <8 x float> %x, <float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01>
+  %cmp1 = fcmp olt <8 x float> %x, <float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>
+  %cmp3 = fcmp une <8 x float> %x, zeroinitializer
+  %xor14 = xor <8 x i1> %cmp1, %cmp3
+  %cmp5 = fcmp une <8 x float> %x, <float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000>
+  %cmp7 = fcmp une <8 x float> %x, <float 0x3FC99999A0000000, float 0x3FC99999A0000000, float 0x3FC99999A0000000, float 0x3FC99999A0000000, float 0x3FC99999A0000000, float 0x3FC99999A0000000, float 0x3FC99999A0000000, float 0x3FC99999A0000000>
+  %and15 = and <8 x i1> %cmp5, %cmp7
+  %xor916 = xor <8 x i1> %xor14, %and15
+  %or17 = or <8 x i1> %xor916, %cmp
+  %or = sext <8 x i1> %or17 to <8 x i32>
+  ret <8 x i32> %or
+}
+define <8 x i32> @six_or_and_xor(<8 x float> %x) {
+; X32-LABEL: six_or_and_xor:
+; X32:       ## %bb.0: ## %entry
+; X32-NEXT:    vmovaps {{.*#+}} ymm1 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]
+; X32-NEXT:    vcmpleps %ymm0, %ymm1, %ymm1
+; X32-NEXT:    vcmpltps LCPI16_1, %ymm0, %ymm2
+; X32-NEXT:    vxorps %xmm3, %xmm3, %xmm3
+; X32-NEXT:    vcmpneqps %ymm3, %ymm0, %ymm3
+; X32-NEXT:    vandps %ymm3, %ymm2, %ymm2
+; X32-NEXT:    vextractf128 $1, %ymm2, %xmm3
+; X32-NEXT:    vpackssdw %xmm3, %xmm2, %xmm2
+; X32-NEXT:    vcmpneqps LCPI16_2, %ymm0, %ymm3
+; X32-NEXT:    vextractf128 $1, %ymm3, %xmm4
+; X32-NEXT:    vpackssdw %xmm4, %xmm3, %xmm3
+; X32-NEXT:    vpand %xmm3, %xmm2, %xmm2
+; X32-NEXT:    vcmpneqps LCPI16_3, %ymm0, %ymm3
+; X32-NEXT:    vxorps %ymm1, %ymm3, %ymm1
+; X32-NEXT:    vextractf128 $1, %ymm1, %xmm3
+; X32-NEXT:    vpackssdw %xmm3, %xmm1, %xmm1
+; X32-NEXT:    vpxor %xmm2, %xmm1, %xmm1
+; X32-NEXT:    vcmpneqps LCPI16_4, %ymm0, %ymm0
+; X32-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; X32-NEXT:    vpackssdw %xmm2, %xmm0, %xmm0
+; X32-NEXT:    vpor %xmm0, %xmm1, %xmm0
+; X32-NEXT:    vpmovsxwd %xmm0, %xmm1
+; X32-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; X32-NEXT:    vpmovsxwd %xmm0, %xmm0
+; X32-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; X32-NEXT:    retl
+;
+; X64-LABEL: six_or_and_xor:
+; X64:       ## %bb.0: ## %entry
+; X64-NEXT:    vmovaps {{.*#+}} ymm1 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]
+; X64-NEXT:    vcmpleps %ymm0, %ymm1, %ymm1
+; X64-NEXT:    vcmpltps {{.*}}(%rip), %ymm0, %ymm2
+; X64-NEXT:    vxorps %xmm3, %xmm3, %xmm3
+; X64-NEXT:    vcmpneqps %ymm3, %ymm0, %ymm3
+; X64-NEXT:    vandps %ymm3, %ymm2, %ymm2
+; X64-NEXT:    vextractf128 $1, %ymm2, %xmm3
+; X64-NEXT:    vpackssdw %xmm3, %xmm2, %xmm2
+; X64-NEXT:    vcmpneqps {{.*}}(%rip), %ymm0, %ymm3
+; X64-NEXT:    vextractf128 $1, %ymm3, %xmm4
+; X64-NEXT:    vpackssdw %xmm4, %xmm3, %xmm3
+; X64-NEXT:    vpand %xmm3, %xmm2, %xmm2
+; X64-NEXT:    vcmpneqps {{.*}}(%rip), %ymm0, %ymm3
+; X64-NEXT:    vxorps %ymm1, %ymm3, %ymm1
+; X64-NEXT:    vextractf128 $1, %ymm1, %xmm3
+; X64-NEXT:    vpackssdw %xmm3, %xmm1, %xmm1
+; X64-NEXT:    vpxor %xmm2, %xmm1, %xmm1
+; X64-NEXT:    vcmpneqps {{.*}}(%rip), %ymm0, %ymm0
+; X64-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; X64-NEXT:    vpackssdw %xmm2, %xmm0, %xmm0
+; X64-NEXT:    vpor %xmm0, %xmm1, %xmm0
+; X64-NEXT:    vpmovsxwd %xmm0, %xmm1
+; X64-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; X64-NEXT:    vpmovsxwd %xmm0, %xmm0
+; X64-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; X64-NEXT:    retq
+;
+; X32-AVX2-LABEL: six_or_and_xor:
+; X32-AVX2:       ## %bb.0: ## %entry
+; X32-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm1 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]
+; X32-AVX2-NEXT:    vcmpleps %ymm0, %ymm1, %ymm1
+; X32-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; X32-AVX2-NEXT:    vcmpltps %ymm2, %ymm0, %ymm2
+; X32-AVX2-NEXT:    vxorps %xmm3, %xmm3, %xmm3
+; X32-AVX2-NEXT:    vcmpneqps %ymm3, %ymm0, %ymm3
+; X32-AVX2-NEXT:    vandps %ymm3, %ymm2, %ymm2
+; X32-AVX2-NEXT:    vextractf128 $1, %ymm2, %xmm3
+; X32-AVX2-NEXT:    vpackssdw %xmm3, %xmm2, %xmm2
+; X32-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm3 = [1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1]
+; X32-AVX2-NEXT:    vcmpneqps %ymm3, %ymm0, %ymm3
+; X32-AVX2-NEXT:    vextractf128 $1, %ymm3, %xmm4
+; X32-AVX2-NEXT:    vpackssdw %xmm4, %xmm3, %xmm3
+; X32-AVX2-NEXT:    vpand %xmm3, %xmm2, %xmm2
+; X32-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm3 = [2.00000003E-1,2.00000003E-1,2.00000003E-1,2.00000003E-1,2.00000003E-1,2.00000003E-1,2.00000003E-1,2.00000003E-1]
+; X32-AVX2-NEXT:    vcmpneqps %ymm3, %ymm0, %ymm3
+; X32-AVX2-NEXT:    vxorps %ymm1, %ymm3, %ymm1
+; X32-AVX2-NEXT:    vextractf128 $1, %ymm1, %xmm3
+; X32-AVX2-NEXT:    vpackssdw %xmm3, %xmm1, %xmm1
+; X32-AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm1
+; X32-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm2 = [4.00000006E-1,4.00000006E-1,4.00000006E-1,4.00000006E-1,4.00000006E-1,4.00000006E-1,4.00000006E-1,4.00000006E-1]
+; X32-AVX2-NEXT:    vcmpneqps %ymm2, %ymm0, %ymm0
+; X32-AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; X32-AVX2-NEXT:    vpackssdw %xmm2, %xmm0, %xmm0
+; X32-AVX2-NEXT:    vpor %xmm0, %xmm1, %xmm0
+; X32-AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
+; X32-AVX2-NEXT:    retl
+;
+; X64-AVX2-LABEL: six_or_and_xor:
+; X64-AVX2:       ## %bb.0: ## %entry
+; X64-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm1 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]
+; X64-AVX2-NEXT:    vcmpleps %ymm0, %ymm1, %ymm1
+; X64-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; X64-AVX2-NEXT:    vcmpltps %ymm2, %ymm0, %ymm2
+; X64-AVX2-NEXT:    vxorps %xmm3, %xmm3, %xmm3
+; X64-AVX2-NEXT:    vcmpneqps %ymm3, %ymm0, %ymm3
+; X64-AVX2-NEXT:    vandps %ymm3, %ymm2, %ymm2
+; X64-AVX2-NEXT:    vextractf128 $1, %ymm2, %xmm3
+; X64-AVX2-NEXT:    vpackssdw %xmm3, %xmm2, %xmm2
+; X64-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm3 = [1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1,1.00000001E-1]
+; X64-AVX2-NEXT:    vcmpneqps %ymm3, %ymm0, %ymm3
+; X64-AVX2-NEXT:    vextractf128 $1, %ymm3, %xmm4
+; X64-AVX2-NEXT:    vpackssdw %xmm4, %xmm3, %xmm3
+; X64-AVX2-NEXT:    vpand %xmm3, %xmm2, %xmm2
+; X64-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm3 = [2.00000003E-1,2.00000003E-1,2.00000003E-1,2.00000003E-1,2.00000003E-1,2.00000003E-1,2.00000003E-1,2.00000003E-1]
+; X64-AVX2-NEXT:    vcmpneqps %ymm3, %ymm0, %ymm3
+; X64-AVX2-NEXT:    vxorps %ymm1, %ymm3, %ymm1
+; X64-AVX2-NEXT:    vextractf128 $1, %ymm1, %xmm3
+; X64-AVX2-NEXT:    vpackssdw %xmm3, %xmm1, %xmm1
+; X64-AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm1
+; X64-AVX2-NEXT:    vbroadcastss {{.*#+}} ymm2 = [4.00000006E-1,4.00000006E-1,4.00000006E-1,4.00000006E-1,4.00000006E-1,4.00000006E-1,4.00000006E-1,4.00000006E-1]
+; X64-AVX2-NEXT:    vcmpneqps %ymm2, %ymm0, %ymm0
+; X64-AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; X64-AVX2-NEXT:    vpackssdw %xmm2, %xmm0, %xmm0
+; X64-AVX2-NEXT:    vpor %xmm0, %xmm1, %xmm0
+; X64-AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
+; X64-AVX2-NEXT:    retq
+entry:
+  %cmp = fcmp oge <8 x float> %x, <float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01>
+  %cmp1 = fcmp olt <8 x float> %x, <float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>
+  %cmp3 = fcmp une <8 x float> %x, zeroinitializer
+  %and18 = and <8 x i1> %cmp1, %cmp3
+  %cmp5 = fcmp une <8 x float> %x, <float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000>
+  %and719 = and <8 x i1> %and18, %cmp5
+  %cmp8 = fcmp une <8 x float> %x, <float 0x3FC99999A0000000, float 0x3FC99999A0000000, float 0x3FC99999A0000000, float 0x3FC99999A0000000, float 0x3FC99999A0000000, float 0x3FC99999A0000000, float 0x3FC99999A0000000, float 0x3FC99999A0000000>
+  %xor20 = xor <8 x i1> %cmp8, %cmp
+  %xor1021 = xor <8 x i1> %xor20, %and719
+  %cmp11 = fcmp une <8 x float> %x, <float 0x3FD99999A0000000, float 0x3FD99999A0000000, float 0x3FD99999A0000000, float 0x3FD99999A0000000, float 0x3FD99999A0000000, float 0x3FD99999A0000000, float 0x3FD99999A0000000, float 0x3FD99999A0000000>
+  %or22 = or <8 x i1> %xor1021, %cmp11
+  %or = sext <8 x i1> %or22 to <8 x i32>
+  ret <8 x i32> %or
+}
diff --git a/llvm/test/CodeGen/X86/var-permute-512.ll b/llvm/test/CodeGen/X86/var-permute-512.ll
index 3f9f96b00..a39a1ee15 100644
--- a/llvm/test/CodeGen/X86/var-permute-512.ll
+++ b/llvm/test/CodeGen/X86/var-permute-512.ll
@@ -98,174 +98,176 @@ define <32 x i16> @var_shuffle_v32i16(<32 x i16> %v, <32 x i16> %indices) nounwi
 ; NOBW-NEXT:    movq %rsp, %rbp
 ; NOBW-NEXT:    andq $-64, %rsp
 ; NOBW-NEXT:    subq $2112, %rsp # imm = 0x840
+; NOBW-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
+; NOBW-NEXT:    vextracti128 $1, %ymm1, %xmm3
 ; NOBW-NEXT:    vextracti128 $1, %ymm2, %xmm4
 ; NOBW-NEXT:    vmovd %xmm4, %eax
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vmovaps %ymm0, (%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    andl $31, %eax
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    andl $31, %eax
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm0, (%rsp)
-; NOBW-NEXT:    movzwl 1472(%rsp,%rax,2), %eax
+; NOBW-NEXT:    movzwl 1536(%rsp,%rax,2), %eax
 ; NOBW-NEXT:    vmovd %eax, %xmm0
 ; NOBW-NEXT:    vpextrw $1, %xmm4, %eax
 ; NOBW-NEXT:    andl $31, %eax
-; NOBW-NEXT:    vpinsrw $1, 1408(%rsp,%rax,2), %xmm0, %xmm0
+; NOBW-NEXT:    vpinsrw $1, 1600(%rsp,%rax,2), %xmm0, %xmm0
 ; NOBW-NEXT:    vpextrw $2, %xmm4, %eax
 ; NOBW-NEXT:    andl $31, %eax
-; NOBW-NEXT:    vpinsrw $2, 1344(%rsp,%rax,2), %xmm0, %xmm0
+; NOBW-NEXT:    vpinsrw $2, 1664(%rsp,%rax,2), %xmm0, %xmm0
 ; NOBW-NEXT:    vpextrw $3, %xmm4, %eax
 ; NOBW-NEXT:    andl $31, %eax
-; NOBW-NEXT:    vpinsrw $3, 1280(%rsp,%rax,2), %xmm0, %xmm0
+; NOBW-NEXT:    vpinsrw $3, 1728(%rsp,%rax,2), %xmm0, %xmm0
 ; NOBW-NEXT:    vpextrw $4, %xmm4, %eax
 ; NOBW-NEXT:    andl $31, %eax
-; NOBW-NEXT:    vpinsrw $4, 1216(%rsp,%rax,2), %xmm0, %xmm0
+; NOBW-NEXT:    vpinsrw $4, 1792(%rsp,%rax,2), %xmm0, %xmm0
 ; NOBW-NEXT:    vpextrw $5, %xmm4, %eax
 ; NOBW-NEXT:    andl $31, %eax
-; NOBW-NEXT:    vpinsrw $5, 1152(%rsp,%rax,2), %xmm0, %xmm0
+; NOBW-NEXT:    vpinsrw $5, 1856(%rsp,%rax,2), %xmm0, %xmm0
 ; NOBW-NEXT:    vpextrw $6, %xmm4, %eax
 ; NOBW-NEXT:    andl $31, %eax
-; NOBW-NEXT:    vpinsrw $6, 1088(%rsp,%rax,2), %xmm0, %xmm0
+; NOBW-NEXT:    vpinsrw $6, 1920(%rsp,%rax,2), %xmm0, %xmm0
 ; NOBW-NEXT:    vpextrw $7, %xmm4, %eax
 ; NOBW-NEXT:    andl $31, %eax
-; NOBW-NEXT:    vpinsrw $7, 1024(%rsp,%rax,2), %xmm0, %xmm0
+; NOBW-NEXT:    vpinsrw $7, 1984(%rsp,%rax,2), %xmm0, %xmm0
 ; NOBW-NEXT:    vmovd %xmm2, %eax
 ; NOBW-NEXT:    andl $31, %eax
-; NOBW-NEXT:    movzwl 1984(%rsp,%rax,2), %eax
-; NOBW-NEXT:    vmovd %eax, %xmm1
-; NOBW-NEXT:    vpextrw $1, %xmm2, %eax
-; NOBW-NEXT:    andl $31, %eax
-; NOBW-NEXT:    vpinsrw $1, 1920(%rsp,%rax,2), %xmm1, %xmm1
-; NOBW-NEXT:    vpextrw $2, %xmm2, %eax
-; NOBW-NEXT:    andl $31, %eax
-; NOBW-NEXT:    vpinsrw $2, 1856(%rsp,%rax,2), %xmm1, %xmm1
-; NOBW-NEXT:    vpextrw $3, %xmm2, %eax
-; NOBW-NEXT:    andl $31, %eax
-; NOBW-NEXT:    vpinsrw $3, 1792(%rsp,%rax,2), %xmm1, %xmm1
-; NOBW-NEXT:    vpextrw $4, %xmm2, %eax
-; NOBW-NEXT:    andl $31, %eax
-; NOBW-NEXT:    vpinsrw $4, 1728(%rsp,%rax,2), %xmm1, %xmm1
-; NOBW-NEXT:    vpextrw $5, %xmm2, %eax
-; NOBW-NEXT:    andl $31, %eax
-; NOBW-NEXT:    vpinsrw $5, 1664(%rsp,%rax,2), %xmm1, %xmm1
-; NOBW-NEXT:    vpextrw $6, %xmm2, %eax
-; NOBW-NEXT:    andl $31, %eax
-; NOBW-NEXT:    vpinsrw $6, 1600(%rsp,%rax,2), %xmm1, %xmm1
-; NOBW-NEXT:    vpextrw $7, %xmm2, %eax
-; NOBW-NEXT:    vextracti128 $1, %ymm3, %xmm2
-; NOBW-NEXT:    andl $31, %eax
-; NOBW-NEXT:    vpinsrw $7, 1536(%rsp,%rax,2), %xmm1, %xmm1
-; NOBW-NEXT:    vmovd %xmm2, %eax
-; NOBW-NEXT:    andl $31, %eax
-; NOBW-NEXT:    movzwl 448(%rsp,%rax,2), %eax
+; NOBW-NEXT:    movzwl 1024(%rsp,%rax,2), %eax
 ; NOBW-NEXT:    vmovd %eax, %xmm4
 ; NOBW-NEXT:    vpextrw $1, %xmm2, %eax
 ; NOBW-NEXT:    andl $31, %eax
-; NOBW-NEXT:    vpinsrw $1, 384(%rsp,%rax,2), %xmm4, %xmm4
+; NOBW-NEXT:    vpinsrw $1, 1088(%rsp,%rax,2), %xmm4, %xmm4
 ; NOBW-NEXT:    vpextrw $2, %xmm2, %eax
 ; NOBW-NEXT:    andl $31, %eax
-; NOBW-NEXT:    vpinsrw $2, 320(%rsp,%rax,2), %xmm4, %xmm4
+; NOBW-NEXT:    vpinsrw $2, 1152(%rsp,%rax,2), %xmm4, %xmm4
 ; NOBW-NEXT:    vpextrw $3, %xmm2, %eax
 ; NOBW-NEXT:    andl $31, %eax
-; NOBW-NEXT:    vpinsrw $3, 256(%rsp,%rax,2), %xmm4, %xmm4
+; NOBW-NEXT:    vpinsrw $3, 1216(%rsp,%rax,2), %xmm4, %xmm4
 ; NOBW-NEXT:    vpextrw $4, %xmm2, %eax
 ; NOBW-NEXT:    andl $31, %eax
-; NOBW-NEXT:    vpinsrw $4, 192(%rsp,%rax,2), %xmm4, %xmm4
+; NOBW-NEXT:    vpinsrw $4, 1280(%rsp,%rax,2), %xmm4, %xmm4
 ; NOBW-NEXT:    vpextrw $5, %xmm2, %eax
 ; NOBW-NEXT:    andl $31, %eax
-; NOBW-NEXT:    vpinsrw $5, 128(%rsp,%rax,2), %xmm4, %xmm4
+; NOBW-NEXT:    vpinsrw $5, 1344(%rsp,%rax,2), %xmm4, %xmm4
 ; NOBW-NEXT:    vpextrw $6, %xmm2, %eax
 ; NOBW-NEXT:    andl $31, %eax
-; NOBW-NEXT:    vpinsrw $6, 64(%rsp,%rax,2), %xmm4, %xmm4
+; NOBW-NEXT:    vpinsrw $6, 1408(%rsp,%rax,2), %xmm4, %xmm4
 ; NOBW-NEXT:    vpextrw $7, %xmm2, %eax
 ; NOBW-NEXT:    andl $31, %eax
-; NOBW-NEXT:    vpinsrw $7, (%rsp,%rax,2), %xmm4, %xmm2
+; NOBW-NEXT:    vpinsrw $7, 1472(%rsp,%rax,2), %xmm4, %xmm2
 ; NOBW-NEXT:    vmovd %xmm3, %eax
 ; NOBW-NEXT:    andl $31, %eax
-; NOBW-NEXT:    movzwl 960(%rsp,%rax,2), %eax
+; NOBW-NEXT:    movzwl 512(%rsp,%rax,2), %eax
 ; NOBW-NEXT:    vmovd %eax, %xmm4
 ; NOBW-NEXT:    vpextrw $1, %xmm3, %eax
 ; NOBW-NEXT:    andl $31, %eax
-; NOBW-NEXT:    vpinsrw $1, 896(%rsp,%rax,2), %xmm4, %xmm4
+; NOBW-NEXT:    vpinsrw $1, 576(%rsp,%rax,2), %xmm4, %xmm4
 ; NOBW-NEXT:    vpextrw $2, %xmm3, %eax
 ; NOBW-NEXT:    andl $31, %eax
-; NOBW-NEXT:    vpinsrw $2, 832(%rsp,%rax,2), %xmm4, %xmm4
+; NOBW-NEXT:    vpinsrw $2, 640(%rsp,%rax,2), %xmm4, %xmm4
 ; NOBW-NEXT:    vpextrw $3, %xmm3, %eax
 ; NOBW-NEXT:    andl $31, %eax
-; NOBW-NEXT:    vpinsrw $3, 768(%rsp,%rax,2), %xmm4, %xmm4
+; NOBW-NEXT:    vpinsrw $3, 704(%rsp,%rax,2), %xmm4, %xmm4
 ; NOBW-NEXT:    vpextrw $4, %xmm3, %eax
 ; NOBW-NEXT:    andl $31, %eax
-; NOBW-NEXT:    vpinsrw $4, 704(%rsp,%rax,2), %xmm4, %xmm4
+; NOBW-NEXT:    vpinsrw $4, 768(%rsp,%rax,2), %xmm4, %xmm4
 ; NOBW-NEXT:    vpextrw $5, %xmm3, %eax
 ; NOBW-NEXT:    andl $31, %eax
-; NOBW-NEXT:    vpinsrw $5, 640(%rsp,%rax,2), %xmm4, %xmm4
+; NOBW-NEXT:    vpinsrw $5, 832(%rsp,%rax,2), %xmm4, %xmm4
 ; NOBW-NEXT:    vpextrw $6, %xmm3, %eax
 ; NOBW-NEXT:    andl $31, %eax
-; NOBW-NEXT:    vpinsrw $6, 576(%rsp,%rax,2), %xmm4, %xmm4
+; NOBW-NEXT:    vpinsrw $6, 896(%rsp,%rax,2), %xmm4, %xmm4
 ; NOBW-NEXT:    vpextrw $7, %xmm3, %eax
 ; NOBW-NEXT:    andl $31, %eax
-; NOBW-NEXT:    vpinsrw $7, 512(%rsp,%rax,2), %xmm4, %xmm3
-; NOBW-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
-; NOBW-NEXT:    vinserti128 $1, %xmm2, %ymm3, %ymm1
+; NOBW-NEXT:    vpinsrw $7, 960(%rsp,%rax,2), %xmm4, %xmm3
+; NOBW-NEXT:    vmovd %xmm1, %eax
+; NOBW-NEXT:    andl $31, %eax
+; NOBW-NEXT:    movzwl (%rsp,%rax,2), %eax
+; NOBW-NEXT:    vmovd %eax, %xmm4
+; NOBW-NEXT:    vpextrw $1, %xmm1, %eax
+; NOBW-NEXT:    andl $31, %eax
+; NOBW-NEXT:    vpinsrw $1, 64(%rsp,%rax,2), %xmm4, %xmm4
+; NOBW-NEXT:    vpextrw $2, %xmm1, %eax
+; NOBW-NEXT:    andl $31, %eax
+; NOBW-NEXT:    vpinsrw $2, 128(%rsp,%rax,2), %xmm4, %xmm4
+; NOBW-NEXT:    vpextrw $3, %xmm1, %eax
+; NOBW-NEXT:    andl $31, %eax
+; NOBW-NEXT:    vpinsrw $3, 192(%rsp,%rax,2), %xmm4, %xmm4
+; NOBW-NEXT:    vpextrw $4, %xmm1, %eax
+; NOBW-NEXT:    andl $31, %eax
+; NOBW-NEXT:    vpinsrw $4, 256(%rsp,%rax,2), %xmm4, %xmm4
+; NOBW-NEXT:    vpextrw $5, %xmm1, %eax
+; NOBW-NEXT:    andl $31, %eax
+; NOBW-NEXT:    vpinsrw $5, 320(%rsp,%rax,2), %xmm4, %xmm4
+; NOBW-NEXT:    vpextrw $6, %xmm1, %eax
+; NOBW-NEXT:    andl $31, %eax
+; NOBW-NEXT:    vpinsrw $6, 384(%rsp,%rax,2), %xmm4, %xmm4
+; NOBW-NEXT:    vpextrw $7, %xmm1, %eax
+; NOBW-NEXT:    andl $31, %eax
+; NOBW-NEXT:    vpinsrw $7, 448(%rsp,%rax,2), %xmm4, %xmm1
+; NOBW-NEXT:    vinserti128 $1, %xmm0, %ymm2, %ymm0
+; NOBW-NEXT:    vinserti128 $1, %xmm3, %ymm1, %ymm1
+; NOBW-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; NOBW-NEXT:    movq %rbp, %rsp
 ; NOBW-NEXT:    popq %rbp
 ; NOBW-NEXT:    retq
@@ -380,334 +382,336 @@ define <64 x i8> @var_shuffle_v64i8(<64 x i8> %v, <64 x i8> %indices) nounwind {
 ; NOBW-NEXT:    movq %rsp, %rbp
 ; NOBW-NEXT:    andq $-64, %rsp
 ; NOBW-NEXT:    subq $4160, %rsp # imm = 0x1040
+; NOBW-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
+; NOBW-NEXT:    vextracti128 $1, %ymm1, %xmm3
 ; NOBW-NEXT:    vextracti128 $1, %ymm2, %xmm4
 ; NOBW-NEXT:    vpextrb $0, %xmm4, %eax
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vmovaps %ymm0, (%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    andl $63, %eax
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    andl $63, %eax
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; NOBW-NEXT:    vextractf64x4 $1, %zmm0, {{[0-9]+}}(%rsp)
 ; NOBW-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
-; NOBW-NEXT:    vmovaps %ymm0, (%rsp)
-; NOBW-NEXT:    movzbl 3008(%rsp,%rax), %eax
+; NOBW-NEXT:    movzbl 3072(%rsp,%rax), %eax
 ; NOBW-NEXT:    vmovd %eax, %xmm0
 ; NOBW-NEXT:    vpextrb $1, %xmm4, %eax
 ; NOBW-NEXT:    andl $63, %eax
-; NOBW-NEXT:    vpinsrb $1, 2944(%rsp,%rax), %xmm0, %xmm0
+; NOBW-NEXT:    vpinsrb $1, 3136(%rsp,%rax), %xmm0, %xmm0
 ; NOBW-NEXT:    vpextrb $2, %xmm4, %eax
 ; NOBW-NEXT:    andl $63, %eax
-; NOBW-NEXT:    vpinsrb $2, 2880(%rsp,%rax), %xmm0, %xmm0
+; NOBW-NEXT:    vpinsrb $2, 3200(%rsp,%rax), %xmm0, %xmm0
 ; NOBW-NEXT:    vpextrb $3, %xmm4, %eax
 ; NOBW-NEXT:    andl $63, %eax
-; NOBW-NEXT:    vpinsrb $3, 2816(%rsp,%rax), %xmm0, %xmm0
+; NOBW-NEXT:    vpinsrb $3, 3264(%rsp,%rax), %xmm0, %xmm0
 ; NOBW-NEXT:    vpextrb $4, %xmm4, %eax
 ; NOBW-NEXT:    andl $63, %eax
-; NOBW-NEXT:    vpinsrb $4, 2752(%rsp,%rax), %xmm0, %xmm0
+; NOBW-NEXT:    vpinsrb $4, 3328(%rsp,%rax), %xmm0, %xmm0
 ; NOBW-NEXT:    vpextrb $5, %xmm4, %eax
 ; NOBW-NEXT:    andl $63, %eax
-; NOBW-NEXT:    vpinsrb $5, 2688(%rsp,%rax), %xmm0, %xmm0
+; NOBW-NEXT:    vpinsrb $5, 3392(%rsp,%rax), %xmm0, %xmm0
 ; NOBW-NEXT:    vpextrb $6, %xmm4, %eax
 ; NOBW-NEXT:    andl $63, %eax
-; NOBW-NEXT:    vpinsrb $6, 2624(%rsp,%rax), %xmm0, %xmm0
+; NOBW-NEXT:    vpinsrb $6, 3456(%rsp,%rax), %xmm0, %xmm0
 ; NOBW-NEXT:    vpextrb $7, %xmm4, %eax
 ; NOBW-NEXT:    andl $63, %eax
-; NOBW-NEXT:    vpinsrb $7, 2560(%rsp,%rax), %xmm0, %xmm0
+; NOBW-NEXT:    vpinsrb $7, 3520(%rsp,%rax), %xmm0, %xmm0
 ; NOBW-NEXT:    vpextrb $8, %xmm4, %eax
 ; NOBW-NEXT:    andl $63, %eax
-; NOBW-NEXT:    vpinsrb $8, 2496(%rsp,%rax), %xmm0, %xmm0
+; NOBW-NEXT:    vpinsrb $8, 3584(%rsp,%rax), %xmm0, %xmm0
 ; NOBW-NEXT:    vpextrb $9, %xmm4, %eax
 ; NOBW-NEXT:    andl $63, %eax
-; NOBW-NEXT:    vpinsrb $9, 2432(%rsp,%rax), %xmm0, %xmm0
+; NOBW-NEXT:    vpinsrb $9, 3648(%rsp,%rax), %xmm0, %xmm0
 ; NOBW-NEXT:    vpextrb $10, %xmm4, %eax
 ; NOBW-NEXT:    andl $63, %eax
-; NOBW-NEXT:    vpinsrb $10, 2368(%rsp,%rax), %xmm0, %xmm0
+; NOBW-NEXT:    vpinsrb $10, 3712(%rsp,%rax), %xmm0, %xmm0
 ; NOBW-NEXT:    vpextrb $11, %xmm4, %eax
 ; NOBW-NEXT:    andl $63, %eax
-; NOBW-NEXT:    vpinsrb $11, 2304(%rsp,%rax), %xmm0, %xmm0
+; NOBW-NEXT:    vpinsrb $11, 3776(%rsp,%rax), %xmm0, %xmm0
 ; NOBW-NEXT:    vpextrb $12, %xmm4, %eax
 ; NOBW-NEXT:    andl $63, %eax
-; NOBW-NEXT:    vpinsrb $12, 2240(%rsp,%rax), %xmm0, %xmm0
+; NOBW-NEXT:    vpinsrb $12, 3840(%rsp,%rax), %xmm0, %xmm0
 ; NOBW-NEXT:    vpextrb $13, %xmm4, %eax
 ; NOBW-NEXT:    andl $63, %eax
-; NOBW-NEXT:    vpinsrb $13, 2176(%rsp,%rax), %xmm0, %xmm0
+; NOBW-NEXT:    vpinsrb $13, 3904(%rsp,%rax), %xmm0, %xmm0
 ; NOBW-NEXT:    vpextrb $14, %xmm4, %eax
 ; NOBW-NEXT:    andl $63, %eax
-; NOBW-NEXT:    vpinsrb $14, 2112(%rsp,%rax), %xmm0, %xmm0
+; NOBW-NEXT:    vpinsrb $14, 3968(%rsp,%rax), %xmm0, %xmm0
 ; NOBW-NEXT:    vpextrb $15, %xmm4, %eax
 ; NOBW-NEXT:    andl $63, %eax
-; NOBW-NEXT:    vpinsrb $15, 2048(%rsp,%rax), %xmm0, %xmm0
-; NOBW-NEXT:    vpextrb $0, %xmm2, %eax
-; NOBW-NEXT:    andl $63, %eax
-; NOBW-NEXT:    movzbl 4032(%rsp,%rax), %eax
-; NOBW-NEXT:    vmovd %eax, %xmm1
-; NOBW-NEXT:    vpextrb $1, %xmm2, %eax
-; NOBW-NEXT:    andl $63, %eax
-; NOBW-NEXT:    vpinsrb $1, 3968(%rsp,%rax), %xmm1, %xmm1
-; NOBW-NEXT:    vpextrb $2, %xmm2, %eax
-; NOBW-NEXT:    andl $63, %eax
-; NOBW-NEXT:    vpinsrb $2, 3904(%rsp,%rax), %xmm1, %xmm1
-; NOBW-NEXT:    vpextrb $3, %xmm2, %eax
-; NOBW-NEXT:    andl $63, %eax
-; NOBW-NEXT:    vpinsrb $3, 3840(%rsp,%rax), %xmm1, %xmm1
-; NOBW-NEXT:    vpextrb $4, %xmm2, %eax
-; NOBW-NEXT:    andl $63, %eax
-; NOBW-NEXT:    vpinsrb $4, 3776(%rsp,%rax), %xmm1, %xmm1
-; NOBW-NEXT:    vpextrb $5, %xmm2, %eax
-; NOBW-NEXT:    andl $63, %eax
-; NOBW-NEXT:    vpinsrb $5, 3712(%rsp,%rax), %xmm1, %xmm1
-; NOBW-NEXT:    vpextrb $6, %xmm2, %eax
-; NOBW-NEXT:    andl $63, %eax
-; NOBW-NEXT:    vpinsrb $6, 3648(%rsp,%rax), %xmm1, %xmm1
-; NOBW-NEXT:    vpextrb $7, %xmm2, %eax
-; NOBW-NEXT:    andl $63, %eax
-; NOBW-NEXT:    vpinsrb $7, 3584(%rsp,%rax), %xmm1, %xmm1
-; NOBW-NEXT:    vpextrb $8, %xmm2, %eax
-; NOBW-NEXT:    andl $63, %eax
-; NOBW-NEXT:    vpinsrb $8, 3520(%rsp,%rax), %xmm1, %xmm1
-; NOBW-NEXT:    vpextrb $9, %xmm2, %eax
-; NOBW-NEXT:    andl $63, %eax
-; NOBW-NEXT:    vpinsrb $9, 3456(%rsp,%rax), %xmm1, %xmm1
-; NOBW-NEXT:    vpextrb $10, %xmm2, %eax
-; NOBW-NEXT:    andl $63, %eax
-; NOBW-NEXT:    vpinsrb $10, 3392(%rsp,%rax), %xmm1, %xmm1
-; NOBW-NEXT:    vpextrb $11, %xmm2, %eax
-; NOBW-NEXT:    andl $63, %eax
-; NOBW-NEXT:    vpinsrb $11, 3328(%rsp,%rax), %xmm1, %xmm1
-; NOBW-NEXT:    vpextrb $12, %xmm2, %eax
-; NOBW-NEXT:    andl $63, %eax
-; NOBW-NEXT:    vpinsrb $12, 3264(%rsp,%rax), %xmm1, %xmm1
-; NOBW-NEXT:    vpextrb $13, %xmm2, %eax
-; NOBW-NEXT:    andl $63, %eax
-; NOBW-NEXT:    vpinsrb $13, 3200(%rsp,%rax), %xmm1, %xmm1
-; NOBW-NEXT:    vpextrb $14, %xmm2, %eax
-; NOBW-NEXT:    andl $63, %eax
-; NOBW-NEXT:    vpinsrb $14, 3136(%rsp,%rax), %xmm1, %xmm1
-; NOBW-NEXT:    vpextrb $15, %xmm2, %eax
-; NOBW-NEXT:    vextracti128 $1, %ymm3, %xmm2
-; NOBW-NEXT:    andl $63, %eax
-; NOBW-NEXT:    vpinsrb $15, 3072(%rsp,%rax), %xmm1, %xmm1
+; NOBW-NEXT:    vpinsrb $15, 4032(%rsp,%rax), %xmm0, %xmm0
 ; NOBW-NEXT:    vpextrb $0, %xmm2, %eax
 ; NOBW-NEXT:    andl $63, %eax
-; NOBW-NEXT:    movzbl 960(%rsp,%rax), %eax
+; NOBW-NEXT:    movzbl 2048(%rsp,%rax), %eax
 ; NOBW-NEXT:    vmovd %eax, %xmm4
 ; NOBW-NEXT:    vpextrb $1, %xmm2, %eax
 ; NOBW-NEXT:    andl $63, %eax
-; NOBW-NEXT:    vpinsrb $1, 896(%rsp,%rax), %xmm4, %xmm4
+; NOBW-NEXT:    vpinsrb $1, 2112(%rsp,%rax), %xmm4, %xmm4
 ; NOBW-NEXT:    vpextrb $2, %xmm2, %eax
 ; NOBW-NEXT:    andl $63, %eax
-; NOBW-NEXT:    vpinsrb $2, 832(%rsp,%rax), %xmm4, %xmm4
+; NOBW-NEXT:    vpinsrb $2, 2176(%rsp,%rax), %xmm4, %xmm4
 ; NOBW-NEXT:    vpextrb $3, %xmm2, %eax
 ; NOBW-NEXT:    andl $63, %eax
-; NOBW-NEXT:    vpinsrb $3, 768(%rsp,%rax), %xmm4, %xmm4
+; NOBW-NEXT:    vpinsrb $3, 2240(%rsp,%rax), %xmm4, %xmm4
 ; NOBW-NEXT:    vpextrb $4, %xmm2, %eax
 ; NOBW-NEXT:    andl $63, %eax
-; NOBW-NEXT:    vpinsrb $4, 704(%rsp,%rax), %xmm4, %xmm4
+; NOBW-NEXT:    vpinsrb $4, 2304(%rsp,%rax), %xmm4, %xmm4
 ; NOBW-NEXT:    vpextrb $5, %xmm2, %eax
 ; NOBW-NEXT:    andl $63, %eax
-; NOBW-NEXT:    vpinsrb $5, 640(%rsp,%rax), %xmm4, %xmm4
+; NOBW-NEXT:    vpinsrb $5, 2368(%rsp,%rax), %xmm4, %xmm4
 ; NOBW-NEXT:    vpextrb $6, %xmm2, %eax
 ; NOBW-NEXT:    andl $63, %eax
-; NOBW-NEXT:    vpinsrb $6, 576(%rsp,%rax), %xmm4, %xmm4
+; NOBW-NEXT:    vpinsrb $6, 2432(%rsp,%rax), %xmm4, %xmm4
 ; NOBW-NEXT:    vpextrb $7, %xmm2, %eax
 ; NOBW-NEXT:    andl $63, %eax
-; NOBW-NEXT:    vpinsrb $7, 512(%rsp,%rax), %xmm4, %xmm4
+; NOBW-NEXT:    vpinsrb $7, 2496(%rsp,%rax), %xmm4, %xmm4
 ; NOBW-NEXT:    vpextrb $8, %xmm2, %eax
 ; NOBW-NEXT:    andl $63, %eax
-; NOBW-NEXT:    vpinsrb $8, 448(%rsp,%rax), %xmm4, %xmm4
+; NOBW-NEXT:    vpinsrb $8, 2560(%rsp,%rax), %xmm4, %xmm4
 ; NOBW-NEXT:    vpextrb $9, %xmm2, %eax
 ; NOBW-NEXT:    andl $63, %eax
-; NOBW-NEXT:    vpinsrb $9, 384(%rsp,%rax), %xmm4, %xmm4
+; NOBW-NEXT:    vpinsrb $9, 2624(%rsp,%rax), %xmm4, %xmm4
 ; NOBW-NEXT:    vpextrb $10, %xmm2, %eax
 ; NOBW-NEXT:    andl $63, %eax
-; NOBW-NEXT:    vpinsrb $10, 320(%rsp,%rax), %xmm4, %xmm4
+; NOBW-NEXT:    vpinsrb $10, 2688(%rsp,%rax), %xmm4, %xmm4
 ; NOBW-NEXT:    vpextrb $11, %xmm2, %eax
 ; NOBW-NEXT:    andl $63, %eax
-; NOBW-NEXT:    vpinsrb $11, 256(%rsp,%rax), %xmm4, %xmm4
+; NOBW-NEXT:    vpinsrb $11, 2752(%rsp,%rax), %xmm4, %xmm4
 ; NOBW-NEXT:    vpextrb $12, %xmm2, %eax
 ; NOBW-NEXT:    andl $63, %eax
-; NOBW-NEXT:    vpinsrb $12, 192(%rsp,%rax), %xmm4, %xmm4
+; NOBW-NEXT:    vpinsrb $12, 2816(%rsp,%rax), %xmm4, %xmm4
 ; NOBW-NEXT:    vpextrb $13, %xmm2, %eax
 ; NOBW-NEXT:    andl $63, %eax
-; NOBW-NEXT:    vpinsrb $13, 128(%rsp,%rax), %xmm4, %xmm4
+; NOBW-NEXT:    vpinsrb $13, 2880(%rsp,%rax), %xmm4, %xmm4
 ; NOBW-NEXT:    vpextrb $14, %xmm2, %eax
 ; NOBW-NEXT:    andl $63, %eax
-; NOBW-NEXT:    vpinsrb $14, 64(%rsp,%rax), %xmm4, %xmm4
+; NOBW-NEXT:    vpinsrb $14, 2944(%rsp,%rax), %xmm4, %xmm4
 ; NOBW-NEXT:    vpextrb $15, %xmm2, %eax
 ; NOBW-NEXT:    andl $63, %eax
-; NOBW-NEXT:    vpinsrb $15, (%rsp,%rax), %xmm4, %xmm2
+; NOBW-NEXT:    vpinsrb $15, 3008(%rsp,%rax), %xmm4, %xmm2
 ; NOBW-NEXT:    vpextrb $0, %xmm3, %eax
 ; NOBW-NEXT:    andl $63, %eax
-; NOBW-NEXT:    movzbl 1984(%rsp,%rax), %eax
+; NOBW-NEXT:    movzbl 1024(%rsp,%rax), %eax
 ; NOBW-NEXT:    vmovd %eax, %xmm4
 ; NOBW-NEXT:    vpextrb $1, %xmm3, %eax
 ; NOBW-NEXT:    andl $63, %eax
-; NOBW-NEXT:    vpinsrb $1, 1920(%rsp,%rax), %xmm4, %xmm4
+; NOBW-NEXT:    vpinsrb $1, 1088(%rsp,%rax), %xmm4, %xmm4
 ; NOBW-NEXT:    vpextrb $2, %xmm3, %eax
 ; NOBW-NEXT:    andl $63, %eax
-; NOBW-NEXT:    vpinsrb $2, 1856(%rsp,%rax), %xmm4, %xmm4
+; NOBW-NEXT:    vpinsrb $2, 1152(%rsp,%rax), %xmm4, %xmm4
 ; NOBW-NEXT:    vpextrb $3, %xmm3, %eax
 ; NOBW-NEXT:    andl $63, %eax
-; NOBW-NEXT:    vpinsrb $3, 1792(%rsp,%rax), %xmm4, %xmm4
+; NOBW-NEXT:    vpinsrb $3, 1216(%rsp,%rax), %xmm4, %xmm4
 ; NOBW-NEXT:    vpextrb $4, %xmm3, %eax
 ; NOBW-NEXT:    andl $63, %eax
-; NOBW-NEXT:    vpinsrb $4, 1728(%rsp,%rax), %xmm4, %xmm4
+; NOBW-NEXT:    vpinsrb $4, 1280(%rsp,%rax), %xmm4, %xmm4
 ; NOBW-NEXT:    vpextrb $5, %xmm3, %eax
 ; NOBW-NEXT:    andl $63, %eax
-; NOBW-NEXT:    vpinsrb $5, 1664(%rsp,%rax), %xmm4, %xmm4
+; NOBW-NEXT:    vpinsrb $5, 1344(%rsp,%rax), %xmm4, %xmm4
 ; NOBW-NEXT:    vpextrb $6, %xmm3, %eax
 ; NOBW-NEXT:    andl $63, %eax
-; NOBW-NEXT:    vpinsrb $6, 1600(%rsp,%rax), %xmm4, %xmm4
+; NOBW-NEXT:    vpinsrb $6, 1408(%rsp,%rax), %xmm4, %xmm4
 ; NOBW-NEXT:    vpextrb $7, %xmm3, %eax
 ; NOBW-NEXT:    andl $63, %eax
-; NOBW-NEXT:    vpinsrb $7, 1536(%rsp,%rax), %xmm4, %xmm4
+; NOBW-NEXT:    vpinsrb $7, 1472(%rsp,%rax), %xmm4, %xmm4
 ; NOBW-NEXT:    vpextrb $8, %xmm3, %eax
 ; NOBW-NEXT:    andl $63, %eax
-; NOBW-NEXT:    vpinsrb $8, 1472(%rsp,%rax), %xmm4, %xmm4
+; NOBW-NEXT:    vpinsrb $8, 1536(%rsp,%rax), %xmm4, %xmm4
 ; NOBW-NEXT:    vpextrb $9, %xmm3, %eax
 ; NOBW-NEXT:    andl $63, %eax
-; NOBW-NEXT:    vpinsrb $9, 1408(%rsp,%rax), %xmm4, %xmm4
+; NOBW-NEXT:    vpinsrb $9, 1600(%rsp,%rax), %xmm4, %xmm4
 ; NOBW-NEXT:    vpextrb $10, %xmm3, %eax
 ; NOBW-NEXT:    andl $63, %eax
-; NOBW-NEXT:    vpinsrb $10, 1344(%rsp,%rax), %xmm4, %xmm4
+; NOBW-NEXT:    vpinsrb $10, 1664(%rsp,%rax), %xmm4, %xmm4
 ; NOBW-NEXT:    vpextrb $11, %xmm3, %eax
 ; NOBW-NEXT:    andl $63, %eax
-; NOBW-NEXT:    vpinsrb $11, 1280(%rsp,%rax), %xmm4, %xmm4
+; NOBW-NEXT:    vpinsrb $11, 1728(%rsp,%rax), %xmm4, %xmm4
 ; NOBW-NEXT:    vpextrb $12, %xmm3, %eax
 ; NOBW-NEXT:    andl $63, %eax
-; NOBW-NEXT:    vpinsrb $12, 1216(%rsp,%rax), %xmm4, %xmm4
+; NOBW-NEXT:    vpinsrb $12, 1792(%rsp,%rax), %xmm4, %xmm4
 ; NOBW-NEXT:    vpextrb $13, %xmm3, %eax
 ; NOBW-NEXT:    andl $63, %eax
-; NOBW-NEXT:    vpinsrb $13, 1152(%rsp,%rax), %xmm4, %xmm4
+; NOBW-NEXT:    vpinsrb $13, 1856(%rsp,%rax), %xmm4, %xmm4
 ; NOBW-NEXT:    vpextrb $14, %xmm3, %eax
 ; NOBW-NEXT:    andl $63, %eax
-; NOBW-NEXT:    vpinsrb $14, 1088(%rsp,%rax), %xmm4, %xmm4
+; NOBW-NEXT:    vpinsrb $14, 1920(%rsp,%rax), %xmm4, %xmm4
 ; NOBW-NEXT:    vpextrb $15, %xmm3, %eax
 ; NOBW-NEXT:    andl $63, %eax
-; NOBW-NEXT:    vpinsrb $15, 1024(%rsp,%rax), %xmm4, %xmm3
-; NOBW-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
-; NOBW-NEXT:    vinserti128 $1, %xmm2, %ymm3, %ymm1
+; NOBW-NEXT:    vpinsrb $15, 1984(%rsp,%rax), %xmm4, %xmm3
+; NOBW-NEXT:    vpextrb $0, %xmm1, %eax
+; NOBW-NEXT:    andl $63, %eax
+; NOBW-NEXT:    movzbl (%rsp,%rax), %eax
+; NOBW-NEXT:    vmovd %eax, %xmm4
+; NOBW-NEXT:    vpextrb $1, %xmm1, %eax
+; NOBW-NEXT:    andl $63, %eax
+; NOBW-NEXT:    vpinsrb $1, 64(%rsp,%rax), %xmm4, %xmm4
+; NOBW-NEXT:    vpextrb $2, %xmm1, %eax
+; NOBW-NEXT:    andl $63, %eax
+; NOBW-NEXT:    vpinsrb $2, 128(%rsp,%rax), %xmm4, %xmm4
+; NOBW-NEXT:    vpextrb $3, %xmm1, %eax
+; NOBW-NEXT:    andl $63, %eax
+; NOBW-NEXT:    vpinsrb $3, 192(%rsp,%rax), %xmm4, %xmm4
+; NOBW-NEXT:    vpextrb $4, %xmm1, %eax
+; NOBW-NEXT:    andl $63, %eax
+; NOBW-NEXT:    vpinsrb $4, 256(%rsp,%rax), %xmm4, %xmm4
+; NOBW-NEXT:    vpextrb $5, %xmm1, %eax
+; NOBW-NEXT:    andl $63, %eax
+; NOBW-NEXT:    vpinsrb $5, 320(%rsp,%rax), %xmm4, %xmm4
+; NOBW-NEXT:    vpextrb $6, %xmm1, %eax
+; NOBW-NEXT:    andl $63, %eax
+; NOBW-NEXT:    vpinsrb $6, 384(%rsp,%rax), %xmm4, %xmm4
+; NOBW-NEXT:    vpextrb $7, %xmm1, %eax
+; NOBW-NEXT:    andl $63, %eax
+; NOBW-NEXT:    vpinsrb $7, 448(%rsp,%rax), %xmm4, %xmm4
+; NOBW-NEXT:    vpextrb $8, %xmm1, %eax
+; NOBW-NEXT:    andl $63, %eax
+; NOBW-NEXT:    vpinsrb $8, 512(%rsp,%rax), %xmm4, %xmm4
+; NOBW-NEXT:    vpextrb $9, %xmm1, %eax
+; NOBW-NEXT:    andl $63, %eax
+; NOBW-NEXT:    vpinsrb $9, 576(%rsp,%rax), %xmm4, %xmm4
+; NOBW-NEXT:    vpextrb $10, %xmm1, %eax
+; NOBW-NEXT:    andl $63, %eax
+; NOBW-NEXT:    vpinsrb $10, 640(%rsp,%rax), %xmm4, %xmm4
+; NOBW-NEXT:    vpextrb $11, %xmm1, %eax
+; NOBW-NEXT:    andl $63, %eax
+; NOBW-NEXT:    vpinsrb $11, 704(%rsp,%rax), %xmm4, %xmm4
+; NOBW-NEXT:    vpextrb $12, %xmm1, %eax
+; NOBW-NEXT:    andl $63, %eax
+; NOBW-NEXT:    vpinsrb $12, 768(%rsp,%rax), %xmm4, %xmm4
+; NOBW-NEXT:    vpextrb $13, %xmm1, %eax
+; NOBW-NEXT:    andl $63, %eax
+; NOBW-NEXT:    vpinsrb $13, 832(%rsp,%rax), %xmm4, %xmm4
+; NOBW-NEXT:    vpextrb $14, %xmm1, %eax
+; NOBW-NEXT:    andl $63, %eax
+; NOBW-NEXT:    vpinsrb $14, 896(%rsp,%rax), %xmm4, %xmm4
+; NOBW-NEXT:    vpextrb $15, %xmm1, %eax
+; NOBW-NEXT:    andl $63, %eax
+; NOBW-NEXT:    vpinsrb $15, 960(%rsp,%rax), %xmm4, %xmm1
+; NOBW-NEXT:    vinserti128 $1, %xmm0, %ymm2, %ymm0
+; NOBW-NEXT:    vinserti128 $1, %xmm3, %ymm1, %ymm1
+; NOBW-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; NOBW-NEXT:    movq %rbp, %rsp
 ; NOBW-NEXT:    popq %rbp
 ; NOBW-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/variable-sized-darwin-bzero.ll b/llvm/test/CodeGen/X86/variable-sized-darwin-bzero.ll
index 55c5dd39d..e39647547 100644
--- a/llvm/test/CodeGen/X86/variable-sized-darwin-bzero.ll
+++ b/llvm/test/CodeGen/X86/variable-sized-darwin-bzero.ll
@@ -1,6 +1,18 @@
-; RUN: llc < %s -mtriple=i686-apple-darwin10 | grep __bzero
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-apple-darwin10 | FileCheck %s
 
 define void @foo(i8* %p, i64 %n) {
+; CHECK-LABEL: foo:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    subl $12, %esp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movl %eax, (%esp)
+; CHECK-NEXT:    calll ___bzero
+; CHECK-NEXT:    addl $12, %esp
+; CHECK-NEXT:    retl
   call void @llvm.memset.p0i8.i64(i8* align 4 %p, i8 0, i64 %n, i1 false)
   ret void
 }
diff --git a/llvm/test/CodeGen/X86/vec-copysign-avx512.ll b/llvm/test/CodeGen/X86/vec-copysign-avx512.ll
index b08b15ce0..2443327db 100644
--- a/llvm/test/CodeGen/X86/vec-copysign-avx512.ll
+++ b/llvm/test/CodeGen/X86/vec-copysign-avx512.ll
@@ -3,55 +3,28 @@
 ; RUN: llc < %s -mtriple=x86_64-apple-macosx10.10.0 -mattr=+avx512vl,+avx512dq | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512VLDQ
 
 define <4 x float> @v4f32(<4 x float> %a, <4 x float> %b) nounwind {
-; AVX512VL-LABEL: v4f32:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpandd {{.*}}(%rip){1to4}, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpandd {{.*}}(%rip){1to4}, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512VL-NEXT:    retq
-;
-; AVX512VLDQ-LABEL: v4f32:
-; AVX512VLDQ:       ## %bb.0:
-; AVX512VLDQ-NEXT:    vandps {{.*}}(%rip){1to4}, %xmm1, %xmm1
-; AVX512VLDQ-NEXT:    vandps {{.*}}(%rip){1to4}, %xmm0, %xmm0
-; AVX512VLDQ-NEXT:    vorps %xmm1, %xmm0, %xmm0
-; AVX512VLDQ-NEXT:    retq
+; CHECK-LABEL: v4f32:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vpternlogd $228, {{.*}}(%rip){1to4}, %xmm1, %xmm0
+; CHECK-NEXT:    retq
   %tmp = tail call <4 x float> @llvm.copysign.v4f32( <4 x float> %a, <4 x float> %b )
   ret <4 x float> %tmp
 }
 
 define <8 x float> @v8f32(<8 x float> %a, <8 x float> %b) nounwind {
-; AVX512VL-LABEL: v8f32:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpandd {{.*}}(%rip){1to8}, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpandd {{.*}}(%rip){1to8}, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpor %ymm1, %ymm0, %ymm0
-; AVX512VL-NEXT:    retq
-;
-; AVX512VLDQ-LABEL: v8f32:
-; AVX512VLDQ:       ## %bb.0:
-; AVX512VLDQ-NEXT:    vandps {{.*}}(%rip){1to8}, %ymm1, %ymm1
-; AVX512VLDQ-NEXT:    vandps {{.*}}(%rip){1to8}, %ymm0, %ymm0
-; AVX512VLDQ-NEXT:    vorps %ymm1, %ymm0, %ymm0
-; AVX512VLDQ-NEXT:    retq
+; CHECK-LABEL: v8f32:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vpternlogd $228, {{.*}}(%rip){1to8}, %ymm1, %ymm0
+; CHECK-NEXT:    retq
   %tmp = tail call <8 x float> @llvm.copysign.v8f32( <8 x float> %a, <8 x float> %b )
   ret <8 x float> %tmp
 }
 
 define <16 x float> @v16f32(<16 x float> %a, <16 x float> %b) nounwind {
-; AVX512VL-LABEL: v16f32:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpandd {{.*}}(%rip){1to16}, %zmm1, %zmm1
-; AVX512VL-NEXT:    vpandd {{.*}}(%rip){1to16}, %zmm0, %zmm0
-; AVX512VL-NEXT:    vpord %zmm1, %zmm0, %zmm0
-; AVX512VL-NEXT:    retq
-;
-; AVX512VLDQ-LABEL: v16f32:
-; AVX512VLDQ:       ## %bb.0:
-; AVX512VLDQ-NEXT:    vandps {{.*}}(%rip){1to16}, %zmm1, %zmm1
-; AVX512VLDQ-NEXT:    vandps {{.*}}(%rip){1to16}, %zmm0, %zmm0
-; AVX512VLDQ-NEXT:    vorps %zmm1, %zmm0, %zmm0
-; AVX512VLDQ-NEXT:    retq
+; CHECK-LABEL: v16f32:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vpternlogd $228, {{.*}}(%rip){1to16}, %zmm1, %zmm0
+; CHECK-NEXT:    retq
   %tmp = tail call <16 x float> @llvm.copysign.v16f32( <16 x float> %a, <16 x float> %b )
   ret <16 x float> %tmp
 }
@@ -59,46 +32,26 @@ define <16 x float> @v16f32(<16 x float> %a, <16 x float> %b) nounwind {
 define <2 x double> @v2f64(<2 x double> %a, <2 x double> %b) nounwind {
 ; CHECK-LABEL: v2f64:
 ; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vandps {{.*}}(%rip), %xmm1, %xmm1
-; CHECK-NEXT:    vandps {{.*}}(%rip), %xmm0, %xmm0
-; CHECK-NEXT:    vorps %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    vpternlogq $228, {{.*}}(%rip), %xmm1, %xmm0
 ; CHECK-NEXT:    retq
   %tmp = tail call <2 x double> @llvm.copysign.v2f64( <2 x double> %a, <2 x double> %b )
   ret <2 x double> %tmp
 }
 
 define <4 x double> @v4f64(<4 x double> %a, <4 x double> %b) nounwind {
-; AVX512VL-LABEL: v4f64:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpandq {{.*}}(%rip){1to4}, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpandq {{.*}}(%rip){1to4}, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpor %ymm1, %ymm0, %ymm0
-; AVX512VL-NEXT:    retq
-;
-; AVX512VLDQ-LABEL: v4f64:
-; AVX512VLDQ:       ## %bb.0:
-; AVX512VLDQ-NEXT:    vandpd {{.*}}(%rip){1to4}, %ymm1, %ymm1
-; AVX512VLDQ-NEXT:    vandpd {{.*}}(%rip){1to4}, %ymm0, %ymm0
-; AVX512VLDQ-NEXT:    vorpd %ymm1, %ymm0, %ymm0
-; AVX512VLDQ-NEXT:    retq
+; CHECK-LABEL: v4f64:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vpternlogq $228, {{.*}}(%rip){1to4}, %ymm1, %ymm0
+; CHECK-NEXT:    retq
   %tmp = tail call <4 x double> @llvm.copysign.v4f64( <4 x double> %a, <4 x double> %b )
   ret <4 x double> %tmp
 }
 
 define <8 x double> @v8f64(<8 x double> %a, <8 x double> %b) nounwind {
-; AVX512VL-LABEL: v8f64:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpandq {{.*}}(%rip){1to8}, %zmm1, %zmm1
-; AVX512VL-NEXT:    vpandq {{.*}}(%rip){1to8}, %zmm0, %zmm0
-; AVX512VL-NEXT:    vporq %zmm1, %zmm0, %zmm0
-; AVX512VL-NEXT:    retq
-;
-; AVX512VLDQ-LABEL: v8f64:
-; AVX512VLDQ:       ## %bb.0:
-; AVX512VLDQ-NEXT:    vandpd {{.*}}(%rip){1to8}, %zmm1, %zmm1
-; AVX512VLDQ-NEXT:    vandpd {{.*}}(%rip){1to8}, %zmm0, %zmm0
-; AVX512VLDQ-NEXT:    vorpd %zmm1, %zmm0, %zmm0
-; AVX512VLDQ-NEXT:    retq
+; CHECK-LABEL: v8f64:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vpternlogq $228, {{.*}}(%rip){1to8}, %zmm1, %zmm0
+; CHECK-NEXT:    retq
   %tmp = tail call <8 x double> @llvm.copysign.v8f64( <8 x double> %a, <8 x double> %b )
   ret <8 x double> %tmp
 }
diff --git a/llvm/test/CodeGen/X86/vec-strict-128.ll b/llvm/test/CodeGen/X86/vec-strict-128.ll
new file mode 100644
index 000000000..5aa0802ad
--- /dev/null
+++ b/llvm/test/CodeGen/X86/vec-strict-128.ll
@@ -0,0 +1,438 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse2 -O3 | FileCheck %s --check-prefixes=SSE,SSE-X86
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 -O3 | FileCheck %s --check-prefixes=SSE,SSE-X64
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+fma -O3 | FileCheck %s --check-prefixes=AVX
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+fma -O3 | FileCheck %s --check-prefixes=AVX
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx512f -mattr=+avx512vl -O3 | FileCheck %s --check-prefixes=AVX
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f -mattr=+avx512vl -O3 | FileCheck %s --check-prefixes=AVX
+
+declare <2 x double> @llvm.experimental.constrained.fadd.v2f64(<2 x double>, <2 x double>, metadata, metadata)
+declare <4 x float> @llvm.experimental.constrained.fadd.v4f32(<4 x float>, <4 x float>, metadata, metadata)
+declare <2 x double> @llvm.experimental.constrained.fsub.v2f64(<2 x double>, <2 x double>, metadata, metadata)
+declare <4 x float> @llvm.experimental.constrained.fsub.v4f32(<4 x float>, <4 x float>, metadata, metadata)
+declare <2 x double> @llvm.experimental.constrained.fmul.v2f64(<2 x double>, <2 x double>, metadata, metadata)
+declare <4 x float> @llvm.experimental.constrained.fmul.v4f32(<4 x float>, <4 x float>, metadata, metadata)
+declare <2 x double> @llvm.experimental.constrained.fdiv.v2f64(<2 x double>, <2 x double>, metadata, metadata)
+declare <4 x float> @llvm.experimental.constrained.fdiv.v4f32(<4 x float>, <4 x float>, metadata, metadata)
+declare <2 x double> @llvm.experimental.constrained.sqrt.v2f64(<2 x double>, metadata, metadata)
+declare <4 x float> @llvm.experimental.constrained.sqrt.v4f32(<4 x float>, metadata, metadata)
+declare float @llvm.experimental.constrained.fptrunc.f32.f64(double, metadata, metadata)
+declare <2 x float> @llvm.experimental.constrained.fptrunc.v2f32.v2f64(<2 x double>, metadata, metadata)
+declare double @llvm.experimental.constrained.fpext.f64.f32(float, metadata)
+declare <2 x double> @llvm.experimental.constrained.fpext.v2f64.v2f32(<2 x float>, metadata)
+declare <2 x double> @llvm.experimental.constrained.fma.v2f64(<2 x double>, <2 x double>, <2 x double>, metadata, metadata)
+declare <4 x float> @llvm.experimental.constrained.fma.v4f32(<4 x float>, <4 x float>, <4 x float>, metadata, metadata)
+
+define <2 x double> @f1(<2 x double> %a, <2 x double> %b) #0 {
+; SSE-LABEL: f1:
+; SSE:       # %bb.0:
+; SSE-NEXT:    addpd %xmm1, %xmm0
+; SSE-NEXT:    ret{{[l|q]}}
+;
+; AVX-LABEL: f1:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    ret{{[l|q]}}
+  %ret = call <2 x double> @llvm.experimental.constrained.fadd.v2f64(<2 x double> %a, <2 x double> %b,
+                                                                     metadata !"round.dynamic",
+                                                                     metadata !"fpexcept.strict") #0
+  ret <2 x double> %ret
+}
+
+define <4 x float> @f2(<4 x float> %a, <4 x float> %b) #0 {
+; SSE-LABEL: f2:
+; SSE:       # %bb.0:
+; SSE-NEXT:    addps %xmm1, %xmm0
+; SSE-NEXT:    ret{{[l|q]}}
+;
+; AVX-LABEL: f2:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vaddps %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    ret{{[l|q]}}
+  %ret = call <4 x float> @llvm.experimental.constrained.fadd.v4f32(<4 x float> %a, <4 x float> %b,
+                                                                    metadata !"round.dynamic",
+                                                                    metadata !"fpexcept.strict") #0
+  ret <4 x float> %ret
+}
+
+define <2 x double> @f3(<2 x double> %a, <2 x double> %b) #0 {
+; SSE-LABEL: f3:
+; SSE:       # %bb.0:
+; SSE-NEXT:    subpd %xmm1, %xmm0
+; SSE-NEXT:    ret{{[l|q]}}
+;
+; AVX-LABEL: f3:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vsubpd %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    ret{{[l|q]}}
+  %ret = call <2 x double> @llvm.experimental.constrained.fsub.v2f64(<2 x double> %a, <2 x double> %b,
+                                                                     metadata !"round.dynamic",
+                                                                     metadata !"fpexcept.strict") #0
+  ret <2 x double> %ret
+}
+
+define <4 x float> @f4(<4 x float> %a, <4 x float> %b) #0 {
+; SSE-LABEL: f4:
+; SSE:       # %bb.0:
+; SSE-NEXT:    subps %xmm1, %xmm0
+; SSE-NEXT:    ret{{[l|q]}}
+;
+; AVX-LABEL: f4:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vsubps %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    ret{{[l|q]}}
+  %ret = call <4 x float> @llvm.experimental.constrained.fsub.v4f32(<4 x float> %a, <4 x float> %b,
+                                                                    metadata !"round.dynamic",
+                                                                    metadata !"fpexcept.strict") #0
+  ret <4 x float> %ret
+}
+
+define <2 x double> @f5(<2 x double> %a, <2 x double> %b) #0 {
+; SSE-LABEL: f5:
+; SSE:       # %bb.0:
+; SSE-NEXT:    mulpd %xmm1, %xmm0
+; SSE-NEXT:    ret{{[l|q]}}
+;
+; AVX-LABEL: f5:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmulpd %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    ret{{[l|q]}}
+  %ret = call <2 x double> @llvm.experimental.constrained.fmul.v2f64(<2 x double> %a, <2 x double> %b,
+                                                                     metadata !"round.dynamic",
+                                                                     metadata !"fpexcept.strict") #0
+  ret <2 x double> %ret
+}
+
+define <4 x float> @f6(<4 x float> %a, <4 x float> %b) #0 {
+; SSE-LABEL: f6:
+; SSE:       # %bb.0:
+; SSE-NEXT:    mulps %xmm1, %xmm0
+; SSE-NEXT:    ret{{[l|q]}}
+;
+; AVX-LABEL: f6:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmulps %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    ret{{[l|q]}}
+  %ret = call <4 x float> @llvm.experimental.constrained.fmul.v4f32(<4 x float> %a, <4 x float> %b,
+                                                                    metadata !"round.dynamic",
+                                                                    metadata !"fpexcept.strict") #0
+  ret <4 x float> %ret
+}
+
+define <2 x double> @f7(<2 x double> %a, <2 x double> %b) #0 {
+; SSE-LABEL: f7:
+; SSE:       # %bb.0:
+; SSE-NEXT:    divpd %xmm1, %xmm0
+; SSE-NEXT:    ret{{[l|q]}}
+;
+; AVX-LABEL: f7:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vdivpd %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    ret{{[l|q]}}
+  %ret = call <2 x double> @llvm.experimental.constrained.fdiv.v2f64(<2 x double> %a, <2 x double> %b,
+                                                                     metadata !"round.dynamic",
+                                                                     metadata !"fpexcept.strict") #0
+  ret <2 x double> %ret
+}
+
+define <4 x float> @f8(<4 x float> %a, <4 x float> %b) #0 {
+; SSE-LABEL: f8:
+; SSE:       # %bb.0:
+; SSE-NEXT:    divps %xmm1, %xmm0
+; SSE-NEXT:    ret{{[l|q]}}
+;
+; AVX-LABEL: f8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vdivps %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    ret{{[l|q]}}
+  %ret = call <4 x float> @llvm.experimental.constrained.fdiv.v4f32(<4 x float> %a, <4 x float> %b,
+                                                                    metadata !"round.dynamic",
+                                                                    metadata !"fpexcept.strict") #0
+  ret <4 x float> %ret
+}
+
+define <2 x double> @f9(<2 x double> %a) #0 {
+; SSE-LABEL: f9:
+; SSE:       # %bb.0:
+; SSE-NEXT:    sqrtpd %xmm0, %xmm0
+; SSE-NEXT:    ret{{[l|q]}}
+;
+; AVX-LABEL: f9:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vsqrtpd %xmm0, %xmm0
+; AVX-NEXT:    ret{{[l|q]}}
+  %sqrt = call <2 x double> @llvm.experimental.constrained.sqrt.v2f64(
+                              <2 x double> %a,
+                              metadata !"round.dynamic",
+                              metadata !"fpexcept.strict") #0
+  ret <2 x double> %sqrt
+}
+
+define <4 x float> @f10(<4 x float> %a) #0 {
+; SSE-LABEL: f10:
+; SSE:       # %bb.0:
+; SSE-NEXT:    sqrtps %xmm0, %xmm0
+; SSE-NEXT:    ret{{[l|q]}}
+;
+; AVX-LABEL: f10:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vsqrtps %xmm0, %xmm0
+; AVX-NEXT:    ret{{[l|q]}}
+  %sqrt = call <4 x float> @llvm.experimental.constrained.sqrt.v4f32(
+                              <4 x float> %a,
+                              metadata !"round.dynamic",
+                              metadata !"fpexcept.strict") #0
+  ret <4 x float > %sqrt
+}
+
+define <4 x float> @f11(<2 x double> %a0, <4 x float> %a1) #0 {
+; SSE-LABEL: f11:
+; SSE:       # %bb.0:
+; SSE-NEXT:    cvtsd2ss %xmm0, %xmm1
+; SSE-NEXT:    movaps %xmm1, %xmm0
+; SSE-NEXT:    ret{{[l|q]}}
+;
+; AVX-LABEL: f11:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcvtsd2ss %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    ret{{[l|q]}}
+  %ext = extractelement <2 x double> %a0, i32 0
+  %cvt = call float @llvm.experimental.constrained.fptrunc.f32.f64(double %ext,
+                                                                   metadata !"round.dynamic",
+                                                                   metadata !"fpexcept.strict")
+  %res = insertelement <4 x float> %a1, float %cvt, i32 0
+  ret <4 x float> %res
+}
+
+define <2 x double> @f12(<2 x double> %a0, <4 x float> %a1) #0 {
+; SSE-LABEL: f12:
+; SSE:       # %bb.0:
+; SSE-NEXT:    cvtss2sd %xmm1, %xmm0
+; SSE-NEXT:    ret{{[l|q]}}
+;
+; AVX-LABEL: f12:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcvtss2sd %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    ret{{[l|q]}}
+  %ext = extractelement <4 x float> %a1, i32 0
+  %cvt = call double @llvm.experimental.constrained.fpext.f64.f32(float %ext,
+                                                                  metadata !"fpexcept.strict") #0
+  %res = insertelement <2 x double> %a0, double %cvt, i32 0
+  ret <2 x double> %res
+}
+
+define <4 x float> @f13(<4 x float> %a, <4 x float> %b, <4 x float> %c) #0 {
+; SSE-X86-LABEL: f13:
+; SSE-X86:       # %bb.0:
+; SSE-X86-NEXT:    subl $108, %esp
+; SSE-X86-NEXT:    .cfi_def_cfa_offset 112
+; SSE-X86-NEXT:    movups %xmm2, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; SSE-X86-NEXT:    movups %xmm1, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; SSE-X86-NEXT:    movups %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; SSE-X86-NEXT:    movss %xmm2, {{[0-9]+}}(%esp)
+; SSE-X86-NEXT:    movss %xmm1, {{[0-9]+}}(%esp)
+; SSE-X86-NEXT:    movss %xmm0, (%esp)
+; SSE-X86-NEXT:    calll fmaf
+; SSE-X86-NEXT:    fstpt {{[-0-9]+}}(%e{{[sb]}}p) # 10-byte Folded Spill
+; SSE-X86-NEXT:    movups {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
+; SSE-X86-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
+; SSE-X86-NEXT:    movss %xmm0, {{[0-9]+}}(%esp)
+; SSE-X86-NEXT:    movups {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
+; SSE-X86-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
+; SSE-X86-NEXT:    movss %xmm0, {{[0-9]+}}(%esp)
+; SSE-X86-NEXT:    movups {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
+; SSE-X86-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
+; SSE-X86-NEXT:    movss %xmm0, (%esp)
+; SSE-X86-NEXT:    calll fmaf
+; SSE-X86-NEXT:    fstpt {{[-0-9]+}}(%e{{[sb]}}p) # 10-byte Folded Spill
+; SSE-X86-NEXT:    movups {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
+; SSE-X86-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE-X86-NEXT:    movss %xmm0, {{[0-9]+}}(%esp)
+; SSE-X86-NEXT:    movups {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
+; SSE-X86-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE-X86-NEXT:    movss %xmm0, {{[0-9]+}}(%esp)
+; SSE-X86-NEXT:    movups {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
+; SSE-X86-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE-X86-NEXT:    movss %xmm0, (%esp)
+; SSE-X86-NEXT:    calll fmaf
+; SSE-X86-NEXT:    movups {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
+; SSE-X86-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE-X86-NEXT:    movss %xmm0, {{[0-9]+}}(%esp)
+; SSE-X86-NEXT:    movups {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
+; SSE-X86-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE-X86-NEXT:    movss %xmm0, {{[0-9]+}}(%esp)
+; SSE-X86-NEXT:    movups {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
+; SSE-X86-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE-X86-NEXT:    movss %xmm0, (%esp)
+; SSE-X86-NEXT:    fstps {{[0-9]+}}(%esp)
+; SSE-X86-NEXT:    fldt {{[-0-9]+}}(%e{{[sb]}}p) # 10-byte Folded Reload
+; SSE-X86-NEXT:    fstps {{[0-9]+}}(%esp)
+; SSE-X86-NEXT:    fldt {{[-0-9]+}}(%e{{[sb]}}p) # 10-byte Folded Reload
+; SSE-X86-NEXT:    fstps {{[0-9]+}}(%esp)
+; SSE-X86-NEXT:    calll fmaf
+; SSE-X86-NEXT:    fstps {{[0-9]+}}(%esp)
+; SSE-X86-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE-X86-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE-X86-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE-X86-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE-X86-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE-X86-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; SSE-X86-NEXT:    movlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE-X86-NEXT:    addl $108, %esp
+; SSE-X86-NEXT:    .cfi_def_cfa_offset 4
+; SSE-X86-NEXT:    retl
+;
+; SSE-X64-LABEL: f13:
+; SSE-X64:       # %bb.0:
+; SSE-X64-NEXT:    subq $88, %rsp
+; SSE-X64-NEXT:    .cfi_def_cfa_offset 96
+; SSE-X64-NEXT:    movaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE-X64-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE-X64-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE-X64-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE-X64-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; SSE-X64-NEXT:    shufps {{.*#+}} xmm2 = xmm2[3,1,2,3]
+; SSE-X64-NEXT:    callq fmaf
+; SSE-X64-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
+; SSE-X64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; SSE-X64-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
+; SSE-X64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; SSE-X64-NEXT:    movhlps {{.*#+}} xmm1 = xmm1[1,1]
+; SSE-X64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
+; SSE-X64-NEXT:    movhlps {{.*#+}} xmm2 = xmm2[1,1]
+; SSE-X64-NEXT:    callq fmaf
+; SSE-X64-NEXT:    unpcklps (%rsp), %xmm0 # 16-byte Folded Reload
+; SSE-X64-NEXT:    # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
+; SSE-X64-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
+; SSE-X64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; SSE-X64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; SSE-X64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
+; SSE-X64-NEXT:    callq fmaf
+; SSE-X64-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE-X64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; SSE-X64-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE-X64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; SSE-X64-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; SSE-X64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
+; SSE-X64-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,2,3]
+; SSE-X64-NEXT:    callq fmaf
+; SSE-X64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; SSE-X64-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE-X64-NEXT:    unpcklpd (%rsp), %xmm1 # 16-byte Folded Reload
+; SSE-X64-NEXT:    # xmm1 = xmm1[0],mem[0]
+; SSE-X64-NEXT:    movaps %xmm1, %xmm0
+; SSE-X64-NEXT:    addq $88, %rsp
+; SSE-X64-NEXT:    .cfi_def_cfa_offset 8
+; SSE-X64-NEXT:    retq
+;
+; AVX-LABEL: f13:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vfmadd213ps {{.*#+}} xmm0 = (xmm1 * xmm0) + xmm2
+; AVX-NEXT:    ret{{[l|q]}}
+  %res = call <4 x float> @llvm.experimental.constrained.fma.v4f32(<4 x float> %a, <4 x float> %b, <4 x float> %c,
+                                                                   metadata !"round.dynamic",
+                                                                   metadata !"fpexcept.strict") #0
+  ret <4 x float> %res
+}
+
+define <2 x double> @f14(<2 x double> %a, <2 x double> %b, <2 x double> %c) #0 {
+; SSE-X86-LABEL: f14:
+; SSE-X86:       # %bb.0:
+; SSE-X86-NEXT:    pushl %ebp
+; SSE-X86-NEXT:    .cfi_def_cfa_offset 8
+; SSE-X86-NEXT:    .cfi_offset %ebp, -8
+; SSE-X86-NEXT:    movl %esp, %ebp
+; SSE-X86-NEXT:    .cfi_def_cfa_register %ebp
+; SSE-X86-NEXT:    andl $-16, %esp
+; SSE-X86-NEXT:    subl $112, %esp
+; SSE-X86-NEXT:    movaps %xmm2, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; SSE-X86-NEXT:    movaps %xmm1, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; SSE-X86-NEXT:    movaps %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; SSE-X86-NEXT:    movlps %xmm2, {{[0-9]+}}(%esp)
+; SSE-X86-NEXT:    movlps %xmm1, {{[0-9]+}}(%esp)
+; SSE-X86-NEXT:    movlps %xmm0, (%esp)
+; SSE-X86-NEXT:    calll fma
+; SSE-X86-NEXT:    movaps {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
+; SSE-X86-NEXT:    movhps %xmm0, {{[0-9]+}}(%esp)
+; SSE-X86-NEXT:    movaps {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
+; SSE-X86-NEXT:    movhps %xmm0, {{[0-9]+}}(%esp)
+; SSE-X86-NEXT:    movaps {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
+; SSE-X86-NEXT:    movhps %xmm0, (%esp)
+; SSE-X86-NEXT:    fstpl {{[0-9]+}}(%esp)
+; SSE-X86-NEXT:    calll fma
+; SSE-X86-NEXT:    fstpl {{[0-9]+}}(%esp)
+; SSE-X86-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE-X86-NEXT:    movhps {{.*#+}} xmm0 = xmm0[0,1],mem[0,1]
+; SSE-X86-NEXT:    movl %ebp, %esp
+; SSE-X86-NEXT:    popl %ebp
+; SSE-X86-NEXT:    .cfi_def_cfa %esp, 4
+; SSE-X86-NEXT:    retl
+;
+; SSE-X64-LABEL: f14:
+; SSE-X64:       # %bb.0:
+; SSE-X64-NEXT:    subq $72, %rsp
+; SSE-X64-NEXT:    .cfi_def_cfa_offset 80
+; SSE-X64-NEXT:    movaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE-X64-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE-X64-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
+; SSE-X64-NEXT:    callq fma
+; SSE-X64-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE-X64-NEXT:    movaps (%rsp), %xmm0 # 16-byte Reload
+; SSE-X64-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
+; SSE-X64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; SSE-X64-NEXT:    movhlps {{.*#+}} xmm1 = xmm1[1,1]
+; SSE-X64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
+; SSE-X64-NEXT:    movhlps {{.*#+}} xmm2 = xmm2[1,1]
+; SSE-X64-NEXT:    callq fma
+; SSE-X64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; SSE-X64-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSE-X64-NEXT:    movaps %xmm1, %xmm0
+; SSE-X64-NEXT:    addq $72, %rsp
+; SSE-X64-NEXT:    .cfi_def_cfa_offset 8
+; SSE-X64-NEXT:    retq
+;
+; AVX-LABEL: f14:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vfmadd213pd {{.*#+}} xmm0 = (xmm1 * xmm0) + xmm2
+; AVX-NEXT:    ret{{[l|q]}}
+  %res = call <2 x double> @llvm.experimental.constrained.fma.v2f64(<2 x double> %a, <2 x double> %b, <2 x double> %c,
+                                                                    metadata !"round.dynamic",
+                                                                    metadata !"fpexcept.strict") #0
+  ret <2 x double> %res
+}
+
+define <2 x double> @f15(<2 x float> %a) #0 {
+; SSE-LABEL: f15:
+; SSE:       # %bb.0:
+; SSE-NEXT:    cvtps2pd %xmm0, %xmm0
+; SSE-NEXT:    ret{{[l|q]}}
+;
+; AVX-LABEL: f15:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcvtps2pd %xmm0, %xmm0
+; AVX-NEXT:    ret{{[l|q]}}
+  %ret = call <2 x double> @llvm.experimental.constrained.fpext.v2f64.v2f32(
+                                <2 x float> %a,
+                                metadata !"fpexcept.strict") #0
+  ret <2 x double> %ret
+}
+
+define <2 x float> @f16(<2 x double> %a) #0 {
+; SSE-LABEL: f16:
+; SSE:       # %bb.0:
+; SSE-NEXT:    cvtpd2ps %xmm0, %xmm0
+; SSE-NEXT:    ret{{[l|q]}}
+;
+; AVX-LABEL: f16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcvtpd2ps %xmm0, %xmm0
+; AVX-NEXT:    ret{{[l|q]}}
+  %ret = call <2 x float> @llvm.experimental.constrained.fptrunc.v2f32.v2f64(
+                                <2 x double> %a,
+                                metadata !"round.dynamic",
+                                metadata !"fpexcept.strict") #0
+  ret <2 x float> %ret
+}
+
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/X86/vec-strict-256.ll b/llvm/test/CodeGen/X86/vec-strict-256.ll
new file mode 100644
index 000000000..97980a4f0
--- /dev/null
+++ b/llvm/test/CodeGen/X86/vec-strict-256.ll
@@ -0,0 +1,298 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+fma -O3 | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+fma -O3 | FileCheck %s
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx512f -mattr=+avx512vl -O3 | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f -mattr=+avx512vl -O3 | FileCheck %s
+
+declare <4 x double> @llvm.experimental.constrained.fadd.v4f64(<4 x double>, <4 x double>, metadata, metadata)
+declare <8 x float> @llvm.experimental.constrained.fadd.v8f32(<8 x float>, <8 x float>, metadata, metadata)
+declare <4 x double> @llvm.experimental.constrained.fsub.v4f64(<4 x double>, <4 x double>, metadata, metadata)
+declare <8 x float> @llvm.experimental.constrained.fsub.v8f32(<8 x float>, <8 x float>, metadata, metadata)
+declare <4 x double> @llvm.experimental.constrained.fmul.v4f64(<4 x double>, <4 x double>, metadata, metadata)
+declare <8 x float> @llvm.experimental.constrained.fmul.v8f32(<8 x float>, <8 x float>, metadata, metadata)
+declare <4 x double> @llvm.experimental.constrained.fdiv.v4f64(<4 x double>, <4 x double>, metadata, metadata)
+declare <8 x float> @llvm.experimental.constrained.fdiv.v8f32(<8 x float>, <8 x float>, metadata, metadata)
+declare <4 x double> @llvm.experimental.constrained.sqrt.v4f64(<4 x double>, metadata, metadata)
+declare <8 x float> @llvm.experimental.constrained.sqrt.v8f32(<8 x float>, metadata, metadata)
+declare <4 x double> @llvm.experimental.constrained.fpext.v4f64.v4f32(<4 x float>, metadata)
+declare <4 x float> @llvm.experimental.constrained.fptrunc.v4f32.v4f64(<4 x double>, metadata, metadata)
+declare <4 x double> @llvm.experimental.constrained.fma.v4f64(<4 x double>, <4 x double>, <4 x double>, metadata, metadata)
+declare <8 x float> @llvm.experimental.constrained.fma.v8f32(<8 x float>, <8 x float>, <8 x float>, metadata, metadata)
+declare <8 x float> @llvm.experimental.constrained.ceil.v8f32(<8 x float>, metadata)
+declare <4 x double>  @llvm.experimental.constrained.ceil.v4f64(<4 x double>, metadata)
+declare <8 x float> @llvm.experimental.constrained.floor.v8f32(<8 x float>, metadata)
+declare <4 x double> @llvm.experimental.constrained.floor.v4f64(<4 x double>, metadata)
+declare <8 x float> @llvm.experimental.constrained.trunc.v8f32(<8 x float>, metadata)
+declare <4 x double> @llvm.experimental.constrained.trunc.v4f64(<4 x double>, metadata)
+declare <8 x float> @llvm.experimental.constrained.rint.v8f32(<8 x float>, metadata, metadata)
+declare <4 x double> @llvm.experimental.constrained.rint.v4f64(<4 x double>, metadata, metadata)
+declare <8 x float> @llvm.experimental.constrained.nearbyint.v8f32(<8 x float>, metadata, metadata)
+declare <4 x double> @llvm.experimental.constrained.nearbyint.v4f64(<4 x double>, metadata, metadata)
+
+define <4 x double> @f1(<4 x double> %a, <4 x double> %b) #0 {
+; CHECK-LABEL: f1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %ret = call <4 x double> @llvm.experimental.constrained.fadd.v4f64(<4 x double> %a, <4 x double> %b,
+                                                                     metadata !"round.dynamic",
+                                                                     metadata !"fpexcept.strict") #0
+  ret <4 x double> %ret
+}
+
+define <8 x float> @f2(<8 x float> %a, <8 x float> %b) #0 {
+; CHECK-LABEL: f2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vaddps %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %ret = call <8 x float> @llvm.experimental.constrained.fadd.v8f32(<8 x float> %a, <8 x float> %b,
+                                                                    metadata !"round.dynamic",
+                                                                    metadata !"fpexcept.strict") #0
+  ret <8 x float> %ret
+}
+
+define <4 x double> @f3(<4 x double> %a, <4 x double> %b) #0 {
+; CHECK-LABEL: f3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsubpd %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %ret = call <4 x double> @llvm.experimental.constrained.fsub.v4f64(<4 x double> %a, <4 x double> %b,
+                                                                     metadata !"round.dynamic",
+                                                                     metadata !"fpexcept.strict") #0
+  ret <4 x double> %ret
+}
+
+define <8 x float> @f4(<8 x float> %a, <8 x float> %b) #0 {
+; CHECK-LABEL: f4:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsubps %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %ret = call <8 x float> @llvm.experimental.constrained.fsub.v8f32(<8 x float> %a, <8 x float> %b,
+                                                                    metadata !"round.dynamic",
+                                                                    metadata !"fpexcept.strict") #0
+  ret <8 x float> %ret
+}
+
+define <4 x double> @f5(<4 x double> %a, <4 x double> %b) #0 {
+; CHECK-LABEL: f5:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmulpd %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %ret = call <4 x double> @llvm.experimental.constrained.fmul.v4f64(<4 x double> %a, <4 x double> %b,
+                                                                     metadata !"round.dynamic",
+                                                                     metadata !"fpexcept.strict") #0
+  ret <4 x double> %ret
+}
+
+define <8 x float> @f6(<8 x float> %a, <8 x float> %b) #0 {
+; CHECK-LABEL: f6:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmulps %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %ret = call <8 x float> @llvm.experimental.constrained.fmul.v8f32(<8 x float> %a, <8 x float> %b,
+                                                                    metadata !"round.dynamic",
+                                                                    metadata !"fpexcept.strict") #0
+  ret <8 x float> %ret
+}
+
+define <4 x double> @f7(<4 x double> %a, <4 x double> %b) #0 {
+; CHECK-LABEL: f7:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vdivpd %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %ret = call <4 x double> @llvm.experimental.constrained.fdiv.v4f64(<4 x double> %a, <4 x double> %b,
+                                                                     metadata !"round.dynamic",
+                                                                     metadata !"fpexcept.strict") #0
+  ret <4 x double> %ret
+}
+
+define <8 x float> @f8(<8 x float> %a, <8 x float> %b) #0 {
+; CHECK-LABEL: f8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vdivps %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %ret = call <8 x float> @llvm.experimental.constrained.fdiv.v8f32(<8 x float> %a, <8 x float> %b,
+                                                                    metadata !"round.dynamic",
+                                                                    metadata !"fpexcept.strict") #0
+  ret <8 x float> %ret
+}
+
+define <4 x double> @f9(<4 x double> %a) #0 {
+; CHECK-LABEL: f9:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsqrtpd %ymm0, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %ret = call <4 x double> @llvm.experimental.constrained.sqrt.v4f64(
+                              <4 x double> %a,
+                              metadata !"round.dynamic",
+                              metadata !"fpexcept.strict") #0
+  ret <4 x double> %ret
+}
+
+
+define <8 x float> @f10(<8 x float> %a) #0 {
+; CHECK-LABEL: f10:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsqrtps %ymm0, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %ret = call <8 x float> @llvm.experimental.constrained.sqrt.v8f32(
+                              <8 x float> %a,
+                              metadata !"round.dynamic",
+                              metadata !"fpexcept.strict") #0
+  ret <8 x float > %ret
+}
+
+define <4 x double> @f11(<4 x float> %a) #0 {
+; CHECK-LABEL: f11:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vcvtps2pd %xmm0, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %ret = call <4 x double> @llvm.experimental.constrained.fpext.v4f64.v4f32(
+                                <4 x float> %a,
+                                metadata !"fpexcept.strict") #0
+  ret <4 x double> %ret
+}
+
+define <4 x float> @f12(<4 x double> %a) #0 {
+; CHECK-LABEL: f12:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vcvtpd2ps %ymm0, %xmm0
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    ret{{[l|q]}}
+  %ret = call <4 x float> @llvm.experimental.constrained.fptrunc.v4f32.v4f64(
+                                <4 x double> %a,
+                                metadata !"round.dynamic",
+                                metadata !"fpexcept.strict") #0
+  ret <4 x float> %ret
+}
+
+define <8 x float> @f13(<8 x float> %a, <8 x float> %b, <8 x float> %c) #0 {
+; CHECK-LABEL: f13:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vfmadd213ps {{.*#+}} ymm0 = (ymm1 * ymm0) + ymm2
+; CHECK-NEXT:    ret{{[l|q]}}
+  %res = call <8 x float> @llvm.experimental.constrained.fma.v8f32(<8 x float> %a, <8 x float> %b, <8 x float> %c,
+                                                                   metadata !"round.dynamic",
+                                                                   metadata !"fpexcept.strict") #0
+  ret <8 x float> %res
+}
+
+define <4 x double> @f14(<4 x double> %a, <4 x double> %b, <4 x double> %c) #0 {
+; CHECK-LABEL: f14:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vfmadd213pd {{.*#+}} ymm0 = (ymm1 * ymm0) + ymm2
+; CHECK-NEXT:    ret{{[l|q]}}
+  %res = call <4 x double> @llvm.experimental.constrained.fma.v4f64(<4 x double> %a, <4 x double> %b, <4 x double> %c,
+                                                                    metadata !"round.dynamic",
+                                                                    metadata !"fpexcept.strict") #0
+  ret <4 x double> %res
+}
+
+define <8 x float> @fceilv8f32(<8 x float> %f) #0 {
+; CHECK-LABEL: fceilv8f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vroundps $10, %ymm0, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %res = call <8 x float> @llvm.experimental.constrained.ceil.v8f32(
+                          <8 x float> %f, metadata !"fpexcept.strict")
+  ret <8 x float> %res
+}
+
+define <4 x double> @fceilv4f64(<4 x double> %f) #0 {
+; CHECK-LABEL: fceilv4f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vroundpd $10, %ymm0, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %res = call <4 x double> @llvm.experimental.constrained.ceil.v4f64(
+                        <4 x double> %f, metadata !"fpexcept.strict")
+  ret <4 x double> %res
+}
+
+define <8 x float> @ffloorv8f32(<8 x float> %f) #0 {
+; CHECK-LABEL: ffloorv8f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vroundps $9, %ymm0, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %res = call <8 x float> @llvm.experimental.constrained.floor.v8f32(
+                          <8 x float> %f, metadata !"fpexcept.strict")
+  ret <8 x float> %res
+}
+
+define <4 x double> @ffloorv4f64(<4 x double> %f) #0 {
+; CHECK-LABEL: ffloorv4f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vroundpd $9, %ymm0, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %res = call <4 x double> @llvm.experimental.constrained.floor.v4f64(
+                        <4 x double> %f, metadata !"fpexcept.strict")
+  ret <4 x double> %res
+}
+
+
+define <8 x float> @ftruncv8f32(<8 x float> %f) #0 {
+; CHECK-LABEL: ftruncv8f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vroundps $11, %ymm0, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %res = call <8 x float> @llvm.experimental.constrained.trunc.v8f32(
+                          <8 x float> %f, metadata !"fpexcept.strict")
+  ret <8 x float> %res
+}
+
+define <4 x double> @ftruncv4f64(<4 x double> %f) #0 {
+; CHECK-LABEL: ftruncv4f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vroundpd $11, %ymm0, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %res = call <4 x double> @llvm.experimental.constrained.trunc.v4f64(
+                        <4 x double> %f, metadata !"fpexcept.strict")
+  ret <4 x double> %res
+}
+
+
+define <8 x float> @frintv8f32(<8 x float> %f) #0 {
+; CHECK-LABEL: frintv8f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vroundps $4, %ymm0, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %res = call <8 x float> @llvm.experimental.constrained.rint.v8f32(
+                          <8 x float> %f,
+                          metadata !"round.dynamic", metadata !"fpexcept.strict")
+  ret <8 x float> %res
+}
+
+define <4 x double> @frintv4f64(<4 x double> %f) #0 {
+; CHECK-LABEL: frintv4f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vroundpd $4, %ymm0, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %res = call <4 x double> @llvm.experimental.constrained.rint.v4f64(
+                        <4 x double> %f,
+                        metadata !"round.dynamic", metadata !"fpexcept.strict")
+  ret <4 x double> %res
+}
+
+
+define <8 x float> @fnearbyintv8f32(<8 x float> %f) #0 {
+; CHECK-LABEL: fnearbyintv8f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vroundps $12, %ymm0, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %res = call <8 x float> @llvm.experimental.constrained.nearbyint.v8f32(
+                          <8 x float> %f,
+                          metadata !"round.dynamic", metadata !"fpexcept.strict")
+  ret <8 x float> %res
+}
+
+define <4 x double> @fnearbyintv4f64(<4 x double> %f) #0 {
+; CHECK-LABEL: fnearbyintv4f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vroundpd $12, %ymm0, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %res = call <4 x double> @llvm.experimental.constrained.nearbyint.v4f64(
+                        <4 x double> %f,
+                        metadata !"round.dynamic", metadata !"fpexcept.strict")
+  ret <4 x double> %res
+}
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/X86/vec-strict-512.ll b/llvm/test/CodeGen/X86/vec-strict-512.ll
new file mode 100644
index 000000000..b2a2c7efb
--- /dev/null
+++ b/llvm/test/CodeGen/X86/vec-strict-512.ll
@@ -0,0 +1,283 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx512f -mattr=+avx512vl -O3 | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f -mattr=+avx512vl -O3 | FileCheck %s
+
+declare <8 x double> @llvm.experimental.constrained.fadd.v8f64(<8 x double>, <8 x double>, metadata, metadata)
+declare <16 x float> @llvm.experimental.constrained.fadd.v16f32(<16 x float>, <16 x float>, metadata, metadata)
+declare <8 x double> @llvm.experimental.constrained.fsub.v8f64(<8 x double>, <8 x double>, metadata, metadata)
+declare <16 x float> @llvm.experimental.constrained.fsub.v16f32(<16 x float>, <16 x float>, metadata, metadata)
+declare <8 x double> @llvm.experimental.constrained.fmul.v8f64(<8 x double>, <8 x double>, metadata, metadata)
+declare <16 x float> @llvm.experimental.constrained.fmul.v16f32(<16 x float>, <16 x float>, metadata, metadata)
+declare <8 x double> @llvm.experimental.constrained.fdiv.v8f64(<8 x double>, <8 x double>, metadata, metadata)
+declare <16 x float> @llvm.experimental.constrained.fdiv.v16f32(<16 x float>, <16 x float>, metadata, metadata)
+declare <8 x double> @llvm.experimental.constrained.sqrt.v8f64(<8 x double>, metadata, metadata)
+declare <16 x float> @llvm.experimental.constrained.sqrt.v16f32(<16 x float>, metadata, metadata)
+declare <8 x double> @llvm.experimental.constrained.fpext.v8f64.v8f32(<8 x float>, metadata)
+declare <8 x float> @llvm.experimental.constrained.fptrunc.v8f32.v8f64(<8 x double>, metadata, metadata)
+declare <8 x double> @llvm.experimental.constrained.fma.v8f64(<8 x double>, <8 x double>, <8 x double>, metadata, metadata)
+declare <16 x float> @llvm.experimental.constrained.fma.v16f32(<16 x float>, <16 x float>, <16 x float>, metadata, metadata)
+declare <16 x float> @llvm.experimental.constrained.ceil.v16f32(<16 x float>, metadata)
+declare <8 x double>  @llvm.experimental.constrained.ceil.v8f64(<8 x double>, metadata)
+declare <16 x float> @llvm.experimental.constrained.floor.v16f32(<16 x float>, metadata)
+declare <8 x double> @llvm.experimental.constrained.floor.v8f64(<8 x double>, metadata)
+declare <16 x float> @llvm.experimental.constrained.trunc.v16f32(<16 x float>, metadata)
+declare <8 x double> @llvm.experimental.constrained.trunc.v8f64(<8 x double>, metadata)
+declare <16 x float> @llvm.experimental.constrained.rint.v16f32(<16 x float>, metadata, metadata)
+declare <8 x double> @llvm.experimental.constrained.rint.v8f64(<8 x double>, metadata, metadata)
+declare <16 x float> @llvm.experimental.constrained.nearbyint.v16f32(<16 x float>, metadata, metadata)
+declare <8 x double> @llvm.experimental.constrained.nearbyint.v8f64(<8 x double>, metadata, metadata)
+
+
+define <8 x double> @f1(<8 x double> %a, <8 x double> %b) #0 {
+; CHECK-LABEL: f1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vaddpd %zmm1, %zmm0, %zmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %ret = call <8 x double> @llvm.experimental.constrained.fadd.v8f64(<8 x double> %a, <8 x double> %b,
+                                                                     metadata !"round.dynamic",
+                                                                     metadata !"fpexcept.strict") #0
+  ret <8 x double> %ret
+}
+
+define <16 x float> @f2(<16 x float> %a, <16 x float> %b) #0 {
+; CHECK-LABEL: f2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vaddps %zmm1, %zmm0, %zmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %ret = call <16 x float> @llvm.experimental.constrained.fadd.v16f32(<16 x float> %a, <16 x float> %b,
+                                                                      metadata !"round.dynamic",
+                                                                      metadata !"fpexcept.strict") #0
+  ret <16 x float> %ret
+}
+
+define <8 x double> @f3(<8 x double> %a, <8 x double> %b) #0 {
+; CHECK-LABEL: f3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsubpd %zmm1, %zmm0, %zmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %ret = call <8 x double> @llvm.experimental.constrained.fsub.v8f64(<8 x double> %a, <8 x double> %b,
+                                                                     metadata !"round.dynamic",
+                                                                     metadata !"fpexcept.strict") #0
+  ret <8 x double> %ret
+}
+
+define <16 x float> @f4(<16 x float> %a, <16 x float> %b) #0 {
+; CHECK-LABEL: f4:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsubps %zmm1, %zmm0, %zmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %ret = call <16 x float> @llvm.experimental.constrained.fsub.v16f32(<16 x float> %a, <16 x float> %b,
+                                                                      metadata !"round.dynamic",
+                                                                      metadata !"fpexcept.strict") #0
+  ret <16 x float> %ret
+}
+
+define <8 x double> @f5(<8 x double> %a, <8 x double> %b) #0 {
+; CHECK-LABEL: f5:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmulpd %zmm1, %zmm0, %zmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %ret = call <8 x double> @llvm.experimental.constrained.fmul.v8f64(<8 x double> %a, <8 x double> %b,
+                                                                     metadata !"round.dynamic",
+                                                                     metadata !"fpexcept.strict") #0
+  ret <8 x double> %ret
+}
+
+define <16 x float> @f6(<16 x float> %a, <16 x float> %b) #0 {
+; CHECK-LABEL: f6:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmulps %zmm1, %zmm0, %zmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %ret = call <16 x float> @llvm.experimental.constrained.fmul.v16f32(<16 x float> %a, <16 x float> %b,
+                                                                      metadata !"round.dynamic",
+                                                                      metadata !"fpexcept.strict") #0
+  ret <16 x float> %ret
+}
+
+define <8 x double> @f7(<8 x double> %a, <8 x double> %b) #0 {
+; CHECK-LABEL: f7:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vdivpd %zmm1, %zmm0, %zmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %ret = call <8 x double> @llvm.experimental.constrained.fdiv.v8f64(<8 x double> %a, <8 x double> %b,
+                                                                     metadata !"round.dynamic",
+                                                                     metadata !"fpexcept.strict") #0
+  ret <8 x double> %ret
+}
+
+define <16 x float> @f8(<16 x float> %a, <16 x float> %b) #0 {
+; CHECK-LABEL: f8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vdivps %zmm1, %zmm0, %zmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %ret = call <16 x float> @llvm.experimental.constrained.fdiv.v16f32(<16 x float> %a, <16 x float> %b,
+                                                                      metadata !"round.dynamic",
+                                                                      metadata !"fpexcept.strict") #0
+  ret <16 x float> %ret
+}
+
+define <8 x double> @f9(<8 x double> %a) #0 {
+; CHECK-LABEL: f9:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsqrtpd %zmm0, %zmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %ret = call <8 x double> @llvm.experimental.constrained.sqrt.v8f64(
+                              <8 x double> %a,
+                              metadata !"round.dynamic",
+                              metadata !"fpexcept.strict") #0
+  ret <8 x double> %ret
+}
+
+
+define <16 x float> @f10(<16 x float> %a) #0 {
+; CHECK-LABEL: f10:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsqrtps %zmm0, %zmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %ret = call <16 x float> @llvm.experimental.constrained.sqrt.v16f32(
+                              <16 x float> %a,
+                              metadata !"round.dynamic",
+                              metadata !"fpexcept.strict") #0
+  ret <16 x float > %ret
+}
+
+define <8 x double> @f11(<8 x float> %a) #0 {
+; CHECK-LABEL: f11:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vcvtps2pd %ymm0, %zmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %ret = call <8 x double> @llvm.experimental.constrained.fpext.v8f64.v8f32(
+                                <8 x float> %a,
+                                metadata !"fpexcept.strict") #0
+  ret <8 x double> %ret
+}
+
+define <8 x float> @f12(<8 x double> %a) #0 {
+; CHECK-LABEL: f12:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vcvtpd2ps %zmm0, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %ret = call <8 x float> @llvm.experimental.constrained.fptrunc.v8f32.v8f64(
+                                <8 x double> %a,
+                                metadata !"round.dynamic",
+                                metadata !"fpexcept.strict") #0
+  ret <8 x float> %ret
+}
+
+define <16 x float> @f13(<16 x float> %a, <16 x float> %b, <16 x float> %c) #0 {
+; CHECK-LABEL: f13:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vfmadd213ps {{.*#+}} zmm0 = (zmm1 * zmm0) + zmm2
+; CHECK-NEXT:    ret{{[l|q]}}
+  %res = call <16 x float> @llvm.experimental.constrained.fma.v16f32(<16 x float> %a, <16 x float> %b, <16 x float> %c,
+                                                                     metadata !"round.dynamic",
+                                                                     metadata !"fpexcept.strict") #0
+  ret <16 x float> %res
+}
+
+define <8 x double> @f14(<8 x double> %a, <8 x double> %b, <8 x double> %c) #0 {
+; CHECK-LABEL: f14:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vfmadd213pd {{.*#+}} zmm0 = (zmm1 * zmm0) + zmm2
+; CHECK-NEXT:    ret{{[l|q]}}
+  %res = call <8 x double> @llvm.experimental.constrained.fma.v8f64(<8 x double> %a, <8 x double> %b, <8 x double> %c,
+                                                                    metadata !"round.dynamic",
+                                                                    metadata !"fpexcept.strict") #0
+  ret <8 x double> %res
+}
+
+define <16 x float> @strict_vector_fceil_v16f32(<16 x float> %f) #0 {
+; CHECK-LABEL: strict_vector_fceil_v16f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vrndscaleps $10, %zmm0, %zmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %res = call <16 x float> @llvm.experimental.constrained.ceil.v16f32(<16 x float> %f, metadata !"fpexcept.strict")
+  ret <16 x float> %res
+}
+
+define <8 x double> @strict_vector_fceil_v8f64(<8 x double> %f) #0 {
+; CHECK-LABEL: strict_vector_fceil_v8f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vrndscalepd $10, %zmm0, %zmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %res = call <8 x double> @llvm.experimental.constrained.ceil.v8f64(<8 x double> %f, metadata !"fpexcept.strict")
+  ret <8 x double> %res
+}
+
+define <16 x float> @strict_vector_ffloor_v16f32(<16 x float> %f) #0 {
+; CHECK-LABEL: strict_vector_ffloor_v16f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vrndscaleps $9, %zmm0, %zmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %res = call <16 x float> @llvm.experimental.constrained.floor.v16f32(<16 x float> %f, metadata !"fpexcept.strict")
+  ret <16 x float> %res
+}
+
+define <8 x double> @strict_vector_ffloor_v8f64(<8 x double> %f) #0 {
+; CHECK-LABEL: strict_vector_ffloor_v8f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vrndscalepd $9, %zmm0, %zmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %res = call <8 x double> @llvm.experimental.constrained.floor.v8f64(<8 x double> %f, metadata !"fpexcept.strict")
+  ret <8 x double> %res
+}
+
+define <16 x float> @strict_vector_ftrunc_v16f32(<16 x float> %f) #0 {
+; CHECK-LABEL: strict_vector_ftrunc_v16f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vrndscaleps $11, %zmm0, %zmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %res = call <16 x float> @llvm.experimental.constrained.trunc.v16f32(<16 x float> %f, metadata !"fpexcept.strict")
+  ret <16 x float> %res
+}
+
+define <8 x double> @strict_vector_ftrunc_v8f64(<8 x double> %f) #0 {
+; CHECK-LABEL: strict_vector_ftrunc_v8f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vrndscalepd $11, %zmm0, %zmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %res = call <8 x double> @llvm.experimental.constrained.trunc.v8f64(<8 x double> %f, metadata !"fpexcept.strict")
+  ret <8 x double> %res
+}
+
+define <16 x float> @strict_vector_frint_v16f32(<16 x float> %f) #0 {
+; CHECK-LABEL: strict_vector_frint_v16f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vrndscaleps $4, %zmm0, %zmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %res = call <16 x float> @llvm.experimental.constrained.rint.v16f32(<16 x float> %f,
+                             metadata !"round.dynamic", metadata !"fpexcept.strict")
+  ret <16 x float> %res
+}
+
+define <8 x double> @strict_vector_frint_v8f64(<8 x double> %f) #0 {
+; CHECK-LABEL: strict_vector_frint_v8f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vrndscalepd $4, %zmm0, %zmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %res = call <8 x double> @llvm.experimental.constrained.rint.v8f64(<8 x double> %f,
+                            metadata !"round.dynamic", metadata !"fpexcept.strict")
+  ret <8 x double> %res
+}
+
+define <16 x float> @strict_vector_fnearbyint_v16f32(<16 x float> %f) #0 {
+; CHECK-LABEL: strict_vector_fnearbyint_v16f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vrndscaleps $12, %zmm0, %zmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %res = call <16 x float> @llvm.experimental.constrained.nearbyint.v16f32(<16 x float> %f,
+                             metadata !"round.dynamic", metadata !"fpexcept.strict")
+  ret <16 x float> %res
+}
+
+define <8 x double> @strict_vector_fnearbyint_v8f64(<8 x double> %f) #0 {
+; CHECK-LABEL: strict_vector_fnearbyint_v8f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vrndscalepd $12, %zmm0, %zmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %res = call <8 x double> @llvm.experimental.constrained.nearbyint.v8f64(<8 x double> %f,
+                             metadata !"round.dynamic", metadata !"fpexcept.strict")
+  ret <8 x double> %res
+}
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/X86/vec-strict-cmp-128.ll b/llvm/test/CodeGen/X86/vec-strict-cmp-128.ll
new file mode 100644
index 000000000..3b56e0114
--- /dev/null
+++ b/llvm/test/CodeGen/X86/vec-strict-cmp-128.ll
@@ -0,0 +1,4462 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse2 -O3 | FileCheck %s --check-prefixes=CHECK,SSE,SSE-32
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 -O3 | FileCheck %s --check-prefixes=CHECK,SSE,SSE-64
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx -O3 | FileCheck %s --check-prefixes=CHECK,AVX,AVX-32
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx -O3 | FileCheck %s --check-prefixes=CHECK,AVX,AVX-64
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx512f -mattr=+avx512vl -O3 | FileCheck %s --check-prefixes=CHECK,AVX512-32
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f -mattr=+avx512vl -O3 | FileCheck %s --check-prefixes=CHECK,AVX512-64
+
+define <4 x i32> @test_v4f32_oeq_q(<4 x i32> %a, <4 x i32> %b, <4 x float> %f1, <4 x float> %f2) #0 {
+; SSE-32-LABEL: test_v4f32_oeq_q:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    andl $-16, %esp
+; SSE-32-NEXT:    subl $16, %esp
+; SSE-32-NEXT:    cmpeqps 8(%ebp), %xmm2
+; SSE-32-NEXT:    andps %xmm2, %xmm0
+; SSE-32-NEXT:    andnps %xmm1, %xmm2
+; SSE-32-NEXT:    orps %xmm2, %xmm0
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_v4f32_oeq_q:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    cmpeqps %xmm3, %xmm2
+; SSE-64-NEXT:    andps %xmm2, %xmm0
+; SSE-64-NEXT:    andnps %xmm1, %xmm2
+; SSE-64-NEXT:    orps %xmm2, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_v4f32_oeq_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-16, %esp
+; AVX-32-NEXT:    subl $16, %esp
+; AVX-32-NEXT:    vcmpeqps 8(%ebp), %xmm2, %xmm2
+; AVX-32-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v4f32_oeq_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpeqps %xmm3, %xmm2, %xmm2
+; AVX-64-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v4f32_oeq_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-16, %esp
+; AVX512-32-NEXT:    subl $16, %esp
+; AVX512-32-NEXT:    vcmpeqps 8(%ebp), %xmm2, %k1
+; AVX512-32-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v4f32_oeq_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpeqps %xmm3, %xmm2, %k1
+; AVX512-64-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %f1, <4 x float> %f2, metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <4 x i1> %cond, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_v4f32_ogt_q(<4 x i32> %a, <4 x i32> %b, <4 x float> %f1, <4 x float> %f2) #0 {
+; SSE-32-LABEL: test_v4f32_ogt_q:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    andl $-16, %esp
+; SSE-32-NEXT:    subl $16, %esp
+; SSE-32-NEXT:    movaps 8(%ebp), %xmm3
+; SSE-32-NEXT:    movaps %xmm3, %xmm4
+; SSE-32-NEXT:    shufps {{.*#+}} xmm4 = xmm4[3,1],xmm3[2,3]
+; SSE-32-NEXT:    movaps %xmm2, %xmm5
+; SSE-32-NEXT:    shufps {{.*#+}} xmm5 = xmm5[3,1],xmm2[2,3]
+; SSE-32-NEXT:    xorl %eax, %eax
+; SSE-32-NEXT:    ucomiss %xmm4, %xmm5
+; SSE-32-NEXT:    movl $-1, %ecx
+; SSE-32-NEXT:    movl $0, %edx
+; SSE-32-NEXT:    cmoval %ecx, %edx
+; SSE-32-NEXT:    movd %edx, %xmm4
+; SSE-32-NEXT:    movaps %xmm3, %xmm5
+; SSE-32-NEXT:    unpckhpd {{.*#+}} xmm5 = xmm5[1],xmm3[1]
+; SSE-32-NEXT:    movaps %xmm2, %xmm6
+; SSE-32-NEXT:    unpckhpd {{.*#+}} xmm6 = xmm6[1],xmm2[1]
+; SSE-32-NEXT:    ucomiss %xmm5, %xmm6
+; SSE-32-NEXT:    movl $0, %edx
+; SSE-32-NEXT:    cmoval %ecx, %edx
+; SSE-32-NEXT:    movd %edx, %xmm5
+; SSE-32-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
+; SSE-32-NEXT:    ucomiss %xmm3, %xmm2
+; SSE-32-NEXT:    movl $0, %edx
+; SSE-32-NEXT:    cmoval %ecx, %edx
+; SSE-32-NEXT:    movd %edx, %xmm4
+; SSE-32-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,2,3]
+; SSE-32-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,2,3]
+; SSE-32-NEXT:    ucomiss %xmm3, %xmm2
+; SSE-32-NEXT:    cmoval %ecx, %eax
+; SSE-32-NEXT:    movd %eax, %xmm2
+; SSE-32-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
+; SSE-32-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm5[0]
+; SSE-32-NEXT:    pand %xmm4, %xmm0
+; SSE-32-NEXT:    pandn %xmm1, %xmm4
+; SSE-32-NEXT:    por %xmm4, %xmm0
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_v4f32_ogt_q:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movaps %xmm3, %xmm4
+; SSE-64-NEXT:    shufps {{.*#+}} xmm4 = xmm4[3,1],xmm3[2,3]
+; SSE-64-NEXT:    movaps %xmm2, %xmm5
+; SSE-64-NEXT:    shufps {{.*#+}} xmm5 = xmm5[3,1],xmm2[2,3]
+; SSE-64-NEXT:    xorl %eax, %eax
+; SSE-64-NEXT:    ucomiss %xmm4, %xmm5
+; SSE-64-NEXT:    movl $-1, %ecx
+; SSE-64-NEXT:    movl $0, %edx
+; SSE-64-NEXT:    cmoval %ecx, %edx
+; SSE-64-NEXT:    movd %edx, %xmm4
+; SSE-64-NEXT:    movaps %xmm3, %xmm5
+; SSE-64-NEXT:    unpckhpd {{.*#+}} xmm5 = xmm5[1],xmm3[1]
+; SSE-64-NEXT:    movaps %xmm2, %xmm6
+; SSE-64-NEXT:    unpckhpd {{.*#+}} xmm6 = xmm6[1],xmm2[1]
+; SSE-64-NEXT:    ucomiss %xmm5, %xmm6
+; SSE-64-NEXT:    movl $0, %edx
+; SSE-64-NEXT:    cmoval %ecx, %edx
+; SSE-64-NEXT:    movd %edx, %xmm5
+; SSE-64-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
+; SSE-64-NEXT:    ucomiss %xmm3, %xmm2
+; SSE-64-NEXT:    movl $0, %edx
+; SSE-64-NEXT:    cmoval %ecx, %edx
+; SSE-64-NEXT:    movd %edx, %xmm4
+; SSE-64-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,2,3]
+; SSE-64-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,2,3]
+; SSE-64-NEXT:    ucomiss %xmm3, %xmm2
+; SSE-64-NEXT:    cmoval %ecx, %eax
+; SSE-64-NEXT:    movd %eax, %xmm2
+; SSE-64-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
+; SSE-64-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm5[0]
+; SSE-64-NEXT:    pand %xmm4, %xmm0
+; SSE-64-NEXT:    pandn %xmm1, %xmm4
+; SSE-64-NEXT:    por %xmm4, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_v4f32_ogt_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-16, %esp
+; AVX-32-NEXT:    subl $16, %esp
+; AVX-32-NEXT:    vmovaps 8(%ebp), %xmm3
+; AVX-32-NEXT:    vcmplt_oqps %xmm2, %xmm3, %xmm2
+; AVX-32-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v4f32_ogt_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmplt_oqps %xmm2, %xmm3, %xmm2
+; AVX-64-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v4f32_ogt_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-16, %esp
+; AVX512-32-NEXT:    subl $16, %esp
+; AVX512-32-NEXT:    vcmpgt_oqps 8(%ebp), %xmm2, %k1
+; AVX512-32-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v4f32_ogt_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmplt_oqps %xmm2, %xmm3, %k1
+; AVX512-64-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %f1, <4 x float> %f2, metadata !"ogt",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <4 x i1> %cond, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_v4f32_oge_q(<4 x i32> %a, <4 x i32> %b, <4 x float> %f1, <4 x float> %f2) #0 {
+; SSE-32-LABEL: test_v4f32_oge_q:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    andl $-16, %esp
+; SSE-32-NEXT:    subl $16, %esp
+; SSE-32-NEXT:    movaps 8(%ebp), %xmm3
+; SSE-32-NEXT:    movaps %xmm3, %xmm4
+; SSE-32-NEXT:    shufps {{.*#+}} xmm4 = xmm4[3,1],xmm3[2,3]
+; SSE-32-NEXT:    movaps %xmm2, %xmm5
+; SSE-32-NEXT:    shufps {{.*#+}} xmm5 = xmm5[3,1],xmm2[2,3]
+; SSE-32-NEXT:    xorl %eax, %eax
+; SSE-32-NEXT:    ucomiss %xmm4, %xmm5
+; SSE-32-NEXT:    movl $-1, %ecx
+; SSE-32-NEXT:    movl $0, %edx
+; SSE-32-NEXT:    cmovael %ecx, %edx
+; SSE-32-NEXT:    movd %edx, %xmm4
+; SSE-32-NEXT:    movaps %xmm3, %xmm5
+; SSE-32-NEXT:    unpckhpd {{.*#+}} xmm5 = xmm5[1],xmm3[1]
+; SSE-32-NEXT:    movaps %xmm2, %xmm6
+; SSE-32-NEXT:    unpckhpd {{.*#+}} xmm6 = xmm6[1],xmm2[1]
+; SSE-32-NEXT:    ucomiss %xmm5, %xmm6
+; SSE-32-NEXT:    movl $0, %edx
+; SSE-32-NEXT:    cmovael %ecx, %edx
+; SSE-32-NEXT:    movd %edx, %xmm5
+; SSE-32-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
+; SSE-32-NEXT:    ucomiss %xmm3, %xmm2
+; SSE-32-NEXT:    movl $0, %edx
+; SSE-32-NEXT:    cmovael %ecx, %edx
+; SSE-32-NEXT:    movd %edx, %xmm4
+; SSE-32-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,2,3]
+; SSE-32-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,2,3]
+; SSE-32-NEXT:    ucomiss %xmm3, %xmm2
+; SSE-32-NEXT:    cmovael %ecx, %eax
+; SSE-32-NEXT:    movd %eax, %xmm2
+; SSE-32-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
+; SSE-32-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm5[0]
+; SSE-32-NEXT:    pand %xmm4, %xmm0
+; SSE-32-NEXT:    pandn %xmm1, %xmm4
+; SSE-32-NEXT:    por %xmm4, %xmm0
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_v4f32_oge_q:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movaps %xmm3, %xmm4
+; SSE-64-NEXT:    shufps {{.*#+}} xmm4 = xmm4[3,1],xmm3[2,3]
+; SSE-64-NEXT:    movaps %xmm2, %xmm5
+; SSE-64-NEXT:    shufps {{.*#+}} xmm5 = xmm5[3,1],xmm2[2,3]
+; SSE-64-NEXT:    xorl %eax, %eax
+; SSE-64-NEXT:    ucomiss %xmm4, %xmm5
+; SSE-64-NEXT:    movl $-1, %ecx
+; SSE-64-NEXT:    movl $0, %edx
+; SSE-64-NEXT:    cmovael %ecx, %edx
+; SSE-64-NEXT:    movd %edx, %xmm4
+; SSE-64-NEXT:    movaps %xmm3, %xmm5
+; SSE-64-NEXT:    unpckhpd {{.*#+}} xmm5 = xmm5[1],xmm3[1]
+; SSE-64-NEXT:    movaps %xmm2, %xmm6
+; SSE-64-NEXT:    unpckhpd {{.*#+}} xmm6 = xmm6[1],xmm2[1]
+; SSE-64-NEXT:    ucomiss %xmm5, %xmm6
+; SSE-64-NEXT:    movl $0, %edx
+; SSE-64-NEXT:    cmovael %ecx, %edx
+; SSE-64-NEXT:    movd %edx, %xmm5
+; SSE-64-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
+; SSE-64-NEXT:    ucomiss %xmm3, %xmm2
+; SSE-64-NEXT:    movl $0, %edx
+; SSE-64-NEXT:    cmovael %ecx, %edx
+; SSE-64-NEXT:    movd %edx, %xmm4
+; SSE-64-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,2,3]
+; SSE-64-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,2,3]
+; SSE-64-NEXT:    ucomiss %xmm3, %xmm2
+; SSE-64-NEXT:    cmovael %ecx, %eax
+; SSE-64-NEXT:    movd %eax, %xmm2
+; SSE-64-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
+; SSE-64-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm5[0]
+; SSE-64-NEXT:    pand %xmm4, %xmm0
+; SSE-64-NEXT:    pandn %xmm1, %xmm4
+; SSE-64-NEXT:    por %xmm4, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_v4f32_oge_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-16, %esp
+; AVX-32-NEXT:    subl $16, %esp
+; AVX-32-NEXT:    vmovaps 8(%ebp), %xmm3
+; AVX-32-NEXT:    vcmple_oqps %xmm2, %xmm3, %xmm2
+; AVX-32-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v4f32_oge_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmple_oqps %xmm2, %xmm3, %xmm2
+; AVX-64-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v4f32_oge_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-16, %esp
+; AVX512-32-NEXT:    subl $16, %esp
+; AVX512-32-NEXT:    vcmpge_oqps 8(%ebp), %xmm2, %k1
+; AVX512-32-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v4f32_oge_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmple_oqps %xmm2, %xmm3, %k1
+; AVX512-64-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %f1, <4 x float> %f2, metadata !"oge",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <4 x i1> %cond, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_v4f32_olt_q(<4 x i32> %a, <4 x i32> %b, <4 x float> %f1, <4 x float> %f2) #0 {
+; SSE-32-LABEL: test_v4f32_olt_q:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    andl $-16, %esp
+; SSE-32-NEXT:    subl $16, %esp
+; SSE-32-NEXT:    movaps 8(%ebp), %xmm3
+; SSE-32-NEXT:    movaps %xmm2, %xmm4
+; SSE-32-NEXT:    shufps {{.*#+}} xmm4 = xmm4[3,1],xmm2[2,3]
+; SSE-32-NEXT:    movaps %xmm3, %xmm5
+; SSE-32-NEXT:    shufps {{.*#+}} xmm5 = xmm5[3,1],xmm3[2,3]
+; SSE-32-NEXT:    xorl %eax, %eax
+; SSE-32-NEXT:    ucomiss %xmm4, %xmm5
+; SSE-32-NEXT:    movl $-1, %ecx
+; SSE-32-NEXT:    movl $0, %edx
+; SSE-32-NEXT:    cmoval %ecx, %edx
+; SSE-32-NEXT:    movd %edx, %xmm4
+; SSE-32-NEXT:    movaps %xmm2, %xmm5
+; SSE-32-NEXT:    unpckhpd {{.*#+}} xmm5 = xmm5[1],xmm2[1]
+; SSE-32-NEXT:    movaps %xmm3, %xmm6
+; SSE-32-NEXT:    unpckhpd {{.*#+}} xmm6 = xmm6[1],xmm3[1]
+; SSE-32-NEXT:    ucomiss %xmm5, %xmm6
+; SSE-32-NEXT:    movl $0, %edx
+; SSE-32-NEXT:    cmoval %ecx, %edx
+; SSE-32-NEXT:    movd %edx, %xmm5
+; SSE-32-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
+; SSE-32-NEXT:    ucomiss %xmm2, %xmm3
+; SSE-32-NEXT:    movl $0, %edx
+; SSE-32-NEXT:    cmoval %ecx, %edx
+; SSE-32-NEXT:    movd %edx, %xmm4
+; SSE-32-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,2,3]
+; SSE-32-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,2,3]
+; SSE-32-NEXT:    ucomiss %xmm2, %xmm3
+; SSE-32-NEXT:    cmoval %ecx, %eax
+; SSE-32-NEXT:    movd %eax, %xmm2
+; SSE-32-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
+; SSE-32-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm5[0]
+; SSE-32-NEXT:    pand %xmm4, %xmm0
+; SSE-32-NEXT:    pandn %xmm1, %xmm4
+; SSE-32-NEXT:    por %xmm4, %xmm0
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_v4f32_olt_q:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movaps %xmm2, %xmm4
+; SSE-64-NEXT:    shufps {{.*#+}} xmm4 = xmm4[3,1],xmm2[2,3]
+; SSE-64-NEXT:    movaps %xmm3, %xmm5
+; SSE-64-NEXT:    shufps {{.*#+}} xmm5 = xmm5[3,1],xmm3[2,3]
+; SSE-64-NEXT:    xorl %eax, %eax
+; SSE-64-NEXT:    ucomiss %xmm4, %xmm5
+; SSE-64-NEXT:    movl $-1, %ecx
+; SSE-64-NEXT:    movl $0, %edx
+; SSE-64-NEXT:    cmoval %ecx, %edx
+; SSE-64-NEXT:    movd %edx, %xmm4
+; SSE-64-NEXT:    movaps %xmm2, %xmm5
+; SSE-64-NEXT:    unpckhpd {{.*#+}} xmm5 = xmm5[1],xmm2[1]
+; SSE-64-NEXT:    movaps %xmm3, %xmm6
+; SSE-64-NEXT:    unpckhpd {{.*#+}} xmm6 = xmm6[1],xmm3[1]
+; SSE-64-NEXT:    ucomiss %xmm5, %xmm6
+; SSE-64-NEXT:    movl $0, %edx
+; SSE-64-NEXT:    cmoval %ecx, %edx
+; SSE-64-NEXT:    movd %edx, %xmm5
+; SSE-64-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
+; SSE-64-NEXT:    ucomiss %xmm2, %xmm3
+; SSE-64-NEXT:    movl $0, %edx
+; SSE-64-NEXT:    cmoval %ecx, %edx
+; SSE-64-NEXT:    movd %edx, %xmm4
+; SSE-64-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,2,3]
+; SSE-64-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,2,3]
+; SSE-64-NEXT:    ucomiss %xmm2, %xmm3
+; SSE-64-NEXT:    cmoval %ecx, %eax
+; SSE-64-NEXT:    movd %eax, %xmm2
+; SSE-64-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
+; SSE-64-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm5[0]
+; SSE-64-NEXT:    pand %xmm4, %xmm0
+; SSE-64-NEXT:    pandn %xmm1, %xmm4
+; SSE-64-NEXT:    por %xmm4, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_v4f32_olt_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-16, %esp
+; AVX-32-NEXT:    subl $16, %esp
+; AVX-32-NEXT:    vcmplt_oqps 8(%ebp), %xmm2, %xmm2
+; AVX-32-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v4f32_olt_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmplt_oqps %xmm3, %xmm2, %xmm2
+; AVX-64-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v4f32_olt_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-16, %esp
+; AVX512-32-NEXT:    subl $16, %esp
+; AVX512-32-NEXT:    vcmplt_oqps 8(%ebp), %xmm2, %k1
+; AVX512-32-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v4f32_olt_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmplt_oqps %xmm3, %xmm2, %k1
+; AVX512-64-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %f1, <4 x float> %f2, metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <4 x i1> %cond, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_v4f32_ole_q(<4 x i32> %a, <4 x i32> %b, <4 x float> %f1, <4 x float> %f2) #0 {
+; SSE-32-LABEL: test_v4f32_ole_q:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    andl $-16, %esp
+; SSE-32-NEXT:    subl $16, %esp
+; SSE-32-NEXT:    movaps 8(%ebp), %xmm3
+; SSE-32-NEXT:    movaps %xmm2, %xmm4
+; SSE-32-NEXT:    shufps {{.*#+}} xmm4 = xmm4[3,1],xmm2[2,3]
+; SSE-32-NEXT:    movaps %xmm3, %xmm5
+; SSE-32-NEXT:    shufps {{.*#+}} xmm5 = xmm5[3,1],xmm3[2,3]
+; SSE-32-NEXT:    xorl %eax, %eax
+; SSE-32-NEXT:    ucomiss %xmm4, %xmm5
+; SSE-32-NEXT:    movl $-1, %ecx
+; SSE-32-NEXT:    movl $0, %edx
+; SSE-32-NEXT:    cmovael %ecx, %edx
+; SSE-32-NEXT:    movd %edx, %xmm4
+; SSE-32-NEXT:    movaps %xmm2, %xmm5
+; SSE-32-NEXT:    unpckhpd {{.*#+}} xmm5 = xmm5[1],xmm2[1]
+; SSE-32-NEXT:    movaps %xmm3, %xmm6
+; SSE-32-NEXT:    unpckhpd {{.*#+}} xmm6 = xmm6[1],xmm3[1]
+; SSE-32-NEXT:    ucomiss %xmm5, %xmm6
+; SSE-32-NEXT:    movl $0, %edx
+; SSE-32-NEXT:    cmovael %ecx, %edx
+; SSE-32-NEXT:    movd %edx, %xmm5
+; SSE-32-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
+; SSE-32-NEXT:    ucomiss %xmm2, %xmm3
+; SSE-32-NEXT:    movl $0, %edx
+; SSE-32-NEXT:    cmovael %ecx, %edx
+; SSE-32-NEXT:    movd %edx, %xmm4
+; SSE-32-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,2,3]
+; SSE-32-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,2,3]
+; SSE-32-NEXT:    ucomiss %xmm2, %xmm3
+; SSE-32-NEXT:    cmovael %ecx, %eax
+; SSE-32-NEXT:    movd %eax, %xmm2
+; SSE-32-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
+; SSE-32-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm5[0]
+; SSE-32-NEXT:    pand %xmm4, %xmm0
+; SSE-32-NEXT:    pandn %xmm1, %xmm4
+; SSE-32-NEXT:    por %xmm4, %xmm0
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_v4f32_ole_q:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movaps %xmm2, %xmm4
+; SSE-64-NEXT:    shufps {{.*#+}} xmm4 = xmm4[3,1],xmm2[2,3]
+; SSE-64-NEXT:    movaps %xmm3, %xmm5
+; SSE-64-NEXT:    shufps {{.*#+}} xmm5 = xmm5[3,1],xmm3[2,3]
+; SSE-64-NEXT:    xorl %eax, %eax
+; SSE-64-NEXT:    ucomiss %xmm4, %xmm5
+; SSE-64-NEXT:    movl $-1, %ecx
+; SSE-64-NEXT:    movl $0, %edx
+; SSE-64-NEXT:    cmovael %ecx, %edx
+; SSE-64-NEXT:    movd %edx, %xmm4
+; SSE-64-NEXT:    movaps %xmm2, %xmm5
+; SSE-64-NEXT:    unpckhpd {{.*#+}} xmm5 = xmm5[1],xmm2[1]
+; SSE-64-NEXT:    movaps %xmm3, %xmm6
+; SSE-64-NEXT:    unpckhpd {{.*#+}} xmm6 = xmm6[1],xmm3[1]
+; SSE-64-NEXT:    ucomiss %xmm5, %xmm6
+; SSE-64-NEXT:    movl $0, %edx
+; SSE-64-NEXT:    cmovael %ecx, %edx
+; SSE-64-NEXT:    movd %edx, %xmm5
+; SSE-64-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
+; SSE-64-NEXT:    ucomiss %xmm2, %xmm3
+; SSE-64-NEXT:    movl $0, %edx
+; SSE-64-NEXT:    cmovael %ecx, %edx
+; SSE-64-NEXT:    movd %edx, %xmm4
+; SSE-64-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,2,3]
+; SSE-64-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,2,3]
+; SSE-64-NEXT:    ucomiss %xmm2, %xmm3
+; SSE-64-NEXT:    cmovael %ecx, %eax
+; SSE-64-NEXT:    movd %eax, %xmm2
+; SSE-64-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
+; SSE-64-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm5[0]
+; SSE-64-NEXT:    pand %xmm4, %xmm0
+; SSE-64-NEXT:    pandn %xmm1, %xmm4
+; SSE-64-NEXT:    por %xmm4, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_v4f32_ole_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-16, %esp
+; AVX-32-NEXT:    subl $16, %esp
+; AVX-32-NEXT:    vcmple_oqps 8(%ebp), %xmm2, %xmm2
+; AVX-32-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v4f32_ole_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmple_oqps %xmm3, %xmm2, %xmm2
+; AVX-64-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v4f32_ole_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-16, %esp
+; AVX512-32-NEXT:    subl $16, %esp
+; AVX512-32-NEXT:    vcmple_oqps 8(%ebp), %xmm2, %k1
+; AVX512-32-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v4f32_ole_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmple_oqps %xmm3, %xmm2, %k1
+; AVX512-64-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %f1, <4 x float> %f2, metadata !"ole",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <4 x i1> %cond, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_v4f32_one_q(<4 x i32> %a, <4 x i32> %b, <4 x float> %f1, <4 x float> %f2) #0 {
+; SSE-32-LABEL: test_v4f32_one_q:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    andl $-16, %esp
+; SSE-32-NEXT:    subl $16, %esp
+; SSE-32-NEXT:    movaps 8(%ebp), %xmm3
+; SSE-32-NEXT:    movaps %xmm2, %xmm4
+; SSE-32-NEXT:    cmpneqps %xmm3, %xmm4
+; SSE-32-NEXT:    cmpordps %xmm3, %xmm2
+; SSE-32-NEXT:    andps %xmm4, %xmm2
+; SSE-32-NEXT:    andps %xmm2, %xmm0
+; SSE-32-NEXT:    andnps %xmm1, %xmm2
+; SSE-32-NEXT:    orps %xmm2, %xmm0
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_v4f32_one_q:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movaps %xmm2, %xmm4
+; SSE-64-NEXT:    cmpneqps %xmm3, %xmm4
+; SSE-64-NEXT:    cmpordps %xmm3, %xmm2
+; SSE-64-NEXT:    andps %xmm4, %xmm2
+; SSE-64-NEXT:    andps %xmm2, %xmm0
+; SSE-64-NEXT:    andnps %xmm1, %xmm2
+; SSE-64-NEXT:    orps %xmm2, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_v4f32_one_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-16, %esp
+; AVX-32-NEXT:    subl $16, %esp
+; AVX-32-NEXT:    vcmpneq_oqps 8(%ebp), %xmm2, %xmm2
+; AVX-32-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v4f32_one_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpneq_oqps %xmm3, %xmm2, %xmm2
+; AVX-64-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v4f32_one_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-16, %esp
+; AVX512-32-NEXT:    subl $16, %esp
+; AVX512-32-NEXT:    vcmpneq_oqps 8(%ebp), %xmm2, %k1
+; AVX512-32-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v4f32_one_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpneq_oqps %xmm3, %xmm2, %k1
+; AVX512-64-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %f1, <4 x float> %f2, metadata !"one",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <4 x i1> %cond, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_v4f32_ord_q(<4 x i32> %a, <4 x i32> %b, <4 x float> %f1, <4 x float> %f2) #0 {
+; SSE-32-LABEL: test_v4f32_ord_q:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    andl $-16, %esp
+; SSE-32-NEXT:    subl $16, %esp
+; SSE-32-NEXT:    cmpordps 8(%ebp), %xmm2
+; SSE-32-NEXT:    andps %xmm2, %xmm0
+; SSE-32-NEXT:    andnps %xmm1, %xmm2
+; SSE-32-NEXT:    orps %xmm2, %xmm0
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_v4f32_ord_q:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    cmpordps %xmm3, %xmm2
+; SSE-64-NEXT:    andps %xmm2, %xmm0
+; SSE-64-NEXT:    andnps %xmm1, %xmm2
+; SSE-64-NEXT:    orps %xmm2, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_v4f32_ord_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-16, %esp
+; AVX-32-NEXT:    subl $16, %esp
+; AVX-32-NEXT:    vcmpordps 8(%ebp), %xmm2, %xmm2
+; AVX-32-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v4f32_ord_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpordps %xmm3, %xmm2, %xmm2
+; AVX-64-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v4f32_ord_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-16, %esp
+; AVX512-32-NEXT:    subl $16, %esp
+; AVX512-32-NEXT:    vcmpordps 8(%ebp), %xmm2, %k1
+; AVX512-32-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v4f32_ord_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpordps %xmm3, %xmm2, %k1
+; AVX512-64-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %f1, <4 x float> %f2, metadata !"ord",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <4 x i1> %cond, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_v4f32_ueq_q(<4 x i32> %a, <4 x i32> %b, <4 x float> %f1, <4 x float> %f2) #0 {
+; SSE-32-LABEL: test_v4f32_ueq_q:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    andl $-16, %esp
+; SSE-32-NEXT:    subl $16, %esp
+; SSE-32-NEXT:    movaps 8(%ebp), %xmm3
+; SSE-32-NEXT:    movaps %xmm2, %xmm4
+; SSE-32-NEXT:    cmpeqps %xmm3, %xmm4
+; SSE-32-NEXT:    cmpunordps %xmm3, %xmm2
+; SSE-32-NEXT:    orps %xmm4, %xmm2
+; SSE-32-NEXT:    andps %xmm2, %xmm0
+; SSE-32-NEXT:    andnps %xmm1, %xmm2
+; SSE-32-NEXT:    orps %xmm2, %xmm0
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_v4f32_ueq_q:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movaps %xmm2, %xmm4
+; SSE-64-NEXT:    cmpeqps %xmm3, %xmm4
+; SSE-64-NEXT:    cmpunordps %xmm3, %xmm2
+; SSE-64-NEXT:    orps %xmm4, %xmm2
+; SSE-64-NEXT:    andps %xmm2, %xmm0
+; SSE-64-NEXT:    andnps %xmm1, %xmm2
+; SSE-64-NEXT:    orps %xmm2, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_v4f32_ueq_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-16, %esp
+; AVX-32-NEXT:    subl $16, %esp
+; AVX-32-NEXT:    vcmpeq_uqps 8(%ebp), %xmm2, %xmm2
+; AVX-32-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v4f32_ueq_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpeq_uqps %xmm3, %xmm2, %xmm2
+; AVX-64-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v4f32_ueq_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-16, %esp
+; AVX512-32-NEXT:    subl $16, %esp
+; AVX512-32-NEXT:    vcmpeq_uqps 8(%ebp), %xmm2, %k1
+; AVX512-32-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v4f32_ueq_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpeq_uqps %xmm3, %xmm2, %k1
+; AVX512-64-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %f1, <4 x float> %f2, metadata !"ueq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <4 x i1> %cond, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_v4f32_ugt_q(<4 x i32> %a, <4 x i32> %b, <4 x float> %f1, <4 x float> %f2) #0 {
+; SSE-32-LABEL: test_v4f32_ugt_q:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    andl $-16, %esp
+; SSE-32-NEXT:    subl $16, %esp
+; SSE-32-NEXT:    movaps 8(%ebp), %xmm3
+; SSE-32-NEXT:    movaps %xmm2, %xmm4
+; SSE-32-NEXT:    shufps {{.*#+}} xmm4 = xmm4[3,1],xmm2[2,3]
+; SSE-32-NEXT:    movaps %xmm3, %xmm5
+; SSE-32-NEXT:    shufps {{.*#+}} xmm5 = xmm5[3,1],xmm3[2,3]
+; SSE-32-NEXT:    xorl %eax, %eax
+; SSE-32-NEXT:    ucomiss %xmm4, %xmm5
+; SSE-32-NEXT:    movl $-1, %ecx
+; SSE-32-NEXT:    movl $0, %edx
+; SSE-32-NEXT:    cmovbl %ecx, %edx
+; SSE-32-NEXT:    movd %edx, %xmm4
+; SSE-32-NEXT:    movaps %xmm2, %xmm5
+; SSE-32-NEXT:    unpckhpd {{.*#+}} xmm5 = xmm5[1],xmm2[1]
+; SSE-32-NEXT:    movaps %xmm3, %xmm6
+; SSE-32-NEXT:    unpckhpd {{.*#+}} xmm6 = xmm6[1],xmm3[1]
+; SSE-32-NEXT:    ucomiss %xmm5, %xmm6
+; SSE-32-NEXT:    movl $0, %edx
+; SSE-32-NEXT:    cmovbl %ecx, %edx
+; SSE-32-NEXT:    movd %edx, %xmm5
+; SSE-32-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
+; SSE-32-NEXT:    ucomiss %xmm2, %xmm3
+; SSE-32-NEXT:    movl $0, %edx
+; SSE-32-NEXT:    cmovbl %ecx, %edx
+; SSE-32-NEXT:    movd %edx, %xmm4
+; SSE-32-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,2,3]
+; SSE-32-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,2,3]
+; SSE-32-NEXT:    ucomiss %xmm2, %xmm3
+; SSE-32-NEXT:    cmovbl %ecx, %eax
+; SSE-32-NEXT:    movd %eax, %xmm2
+; SSE-32-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
+; SSE-32-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm5[0]
+; SSE-32-NEXT:    pand %xmm4, %xmm0
+; SSE-32-NEXT:    pandn %xmm1, %xmm4
+; SSE-32-NEXT:    por %xmm4, %xmm0
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_v4f32_ugt_q:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movaps %xmm2, %xmm4
+; SSE-64-NEXT:    shufps {{.*#+}} xmm4 = xmm4[3,1],xmm2[2,3]
+; SSE-64-NEXT:    movaps %xmm3, %xmm5
+; SSE-64-NEXT:    shufps {{.*#+}} xmm5 = xmm5[3,1],xmm3[2,3]
+; SSE-64-NEXT:    xorl %eax, %eax
+; SSE-64-NEXT:    ucomiss %xmm4, %xmm5
+; SSE-64-NEXT:    movl $-1, %ecx
+; SSE-64-NEXT:    movl $0, %edx
+; SSE-64-NEXT:    cmovbl %ecx, %edx
+; SSE-64-NEXT:    movd %edx, %xmm4
+; SSE-64-NEXT:    movaps %xmm2, %xmm5
+; SSE-64-NEXT:    unpckhpd {{.*#+}} xmm5 = xmm5[1],xmm2[1]
+; SSE-64-NEXT:    movaps %xmm3, %xmm6
+; SSE-64-NEXT:    unpckhpd {{.*#+}} xmm6 = xmm6[1],xmm3[1]
+; SSE-64-NEXT:    ucomiss %xmm5, %xmm6
+; SSE-64-NEXT:    movl $0, %edx
+; SSE-64-NEXT:    cmovbl %ecx, %edx
+; SSE-64-NEXT:    movd %edx, %xmm5
+; SSE-64-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
+; SSE-64-NEXT:    ucomiss %xmm2, %xmm3
+; SSE-64-NEXT:    movl $0, %edx
+; SSE-64-NEXT:    cmovbl %ecx, %edx
+; SSE-64-NEXT:    movd %edx, %xmm4
+; SSE-64-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,2,3]
+; SSE-64-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,2,3]
+; SSE-64-NEXT:    ucomiss %xmm2, %xmm3
+; SSE-64-NEXT:    cmovbl %ecx, %eax
+; SSE-64-NEXT:    movd %eax, %xmm2
+; SSE-64-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
+; SSE-64-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm5[0]
+; SSE-64-NEXT:    pand %xmm4, %xmm0
+; SSE-64-NEXT:    pandn %xmm1, %xmm4
+; SSE-64-NEXT:    por %xmm4, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_v4f32_ugt_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-16, %esp
+; AVX-32-NEXT:    subl $16, %esp
+; AVX-32-NEXT:    vcmpnle_uqps 8(%ebp), %xmm2, %xmm2
+; AVX-32-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v4f32_ugt_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpnle_uqps %xmm3, %xmm2, %xmm2
+; AVX-64-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v4f32_ugt_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-16, %esp
+; AVX512-32-NEXT:    subl $16, %esp
+; AVX512-32-NEXT:    vcmpnle_uqps 8(%ebp), %xmm2, %k1
+; AVX512-32-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v4f32_ugt_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpnle_uqps %xmm3, %xmm2, %k1
+; AVX512-64-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %f1, <4 x float> %f2, metadata !"ugt",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <4 x i1> %cond, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_v4f32_uge_q(<4 x i32> %a, <4 x i32> %b, <4 x float> %f1, <4 x float> %f2) #0 {
+; SSE-32-LABEL: test_v4f32_uge_q:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    andl $-16, %esp
+; SSE-32-NEXT:    subl $16, %esp
+; SSE-32-NEXT:    movaps 8(%ebp), %xmm3
+; SSE-32-NEXT:    movaps %xmm2, %xmm4
+; SSE-32-NEXT:    shufps {{.*#+}} xmm4 = xmm4[3,1],xmm2[2,3]
+; SSE-32-NEXT:    movaps %xmm3, %xmm5
+; SSE-32-NEXT:    shufps {{.*#+}} xmm5 = xmm5[3,1],xmm3[2,3]
+; SSE-32-NEXT:    xorl %eax, %eax
+; SSE-32-NEXT:    ucomiss %xmm4, %xmm5
+; SSE-32-NEXT:    movl $-1, %ecx
+; SSE-32-NEXT:    movl $0, %edx
+; SSE-32-NEXT:    cmovbel %ecx, %edx
+; SSE-32-NEXT:    movd %edx, %xmm4
+; SSE-32-NEXT:    movaps %xmm2, %xmm5
+; SSE-32-NEXT:    unpckhpd {{.*#+}} xmm5 = xmm5[1],xmm2[1]
+; SSE-32-NEXT:    movaps %xmm3, %xmm6
+; SSE-32-NEXT:    unpckhpd {{.*#+}} xmm6 = xmm6[1],xmm3[1]
+; SSE-32-NEXT:    ucomiss %xmm5, %xmm6
+; SSE-32-NEXT:    movl $0, %edx
+; SSE-32-NEXT:    cmovbel %ecx, %edx
+; SSE-32-NEXT:    movd %edx, %xmm5
+; SSE-32-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
+; SSE-32-NEXT:    ucomiss %xmm2, %xmm3
+; SSE-32-NEXT:    movl $0, %edx
+; SSE-32-NEXT:    cmovbel %ecx, %edx
+; SSE-32-NEXT:    movd %edx, %xmm4
+; SSE-32-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,2,3]
+; SSE-32-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,2,3]
+; SSE-32-NEXT:    ucomiss %xmm2, %xmm3
+; SSE-32-NEXT:    cmovbel %ecx, %eax
+; SSE-32-NEXT:    movd %eax, %xmm2
+; SSE-32-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
+; SSE-32-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm5[0]
+; SSE-32-NEXT:    pand %xmm4, %xmm0
+; SSE-32-NEXT:    pandn %xmm1, %xmm4
+; SSE-32-NEXT:    por %xmm4, %xmm0
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_v4f32_uge_q:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movaps %xmm2, %xmm4
+; SSE-64-NEXT:    shufps {{.*#+}} xmm4 = xmm4[3,1],xmm2[2,3]
+; SSE-64-NEXT:    movaps %xmm3, %xmm5
+; SSE-64-NEXT:    shufps {{.*#+}} xmm5 = xmm5[3,1],xmm3[2,3]
+; SSE-64-NEXT:    xorl %eax, %eax
+; SSE-64-NEXT:    ucomiss %xmm4, %xmm5
+; SSE-64-NEXT:    movl $-1, %ecx
+; SSE-64-NEXT:    movl $0, %edx
+; SSE-64-NEXT:    cmovbel %ecx, %edx
+; SSE-64-NEXT:    movd %edx, %xmm4
+; SSE-64-NEXT:    movaps %xmm2, %xmm5
+; SSE-64-NEXT:    unpckhpd {{.*#+}} xmm5 = xmm5[1],xmm2[1]
+; SSE-64-NEXT:    movaps %xmm3, %xmm6
+; SSE-64-NEXT:    unpckhpd {{.*#+}} xmm6 = xmm6[1],xmm3[1]
+; SSE-64-NEXT:    ucomiss %xmm5, %xmm6
+; SSE-64-NEXT:    movl $0, %edx
+; SSE-64-NEXT:    cmovbel %ecx, %edx
+; SSE-64-NEXT:    movd %edx, %xmm5
+; SSE-64-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
+; SSE-64-NEXT:    ucomiss %xmm2, %xmm3
+; SSE-64-NEXT:    movl $0, %edx
+; SSE-64-NEXT:    cmovbel %ecx, %edx
+; SSE-64-NEXT:    movd %edx, %xmm4
+; SSE-64-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,2,3]
+; SSE-64-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,2,3]
+; SSE-64-NEXT:    ucomiss %xmm2, %xmm3
+; SSE-64-NEXT:    cmovbel %ecx, %eax
+; SSE-64-NEXT:    movd %eax, %xmm2
+; SSE-64-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
+; SSE-64-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm5[0]
+; SSE-64-NEXT:    pand %xmm4, %xmm0
+; SSE-64-NEXT:    pandn %xmm1, %xmm4
+; SSE-64-NEXT:    por %xmm4, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_v4f32_uge_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-16, %esp
+; AVX-32-NEXT:    subl $16, %esp
+; AVX-32-NEXT:    vcmpnlt_uqps 8(%ebp), %xmm2, %xmm2
+; AVX-32-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v4f32_uge_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpnlt_uqps %xmm3, %xmm2, %xmm2
+; AVX-64-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v4f32_uge_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-16, %esp
+; AVX512-32-NEXT:    subl $16, %esp
+; AVX512-32-NEXT:    vcmpnlt_uqps 8(%ebp), %xmm2, %k1
+; AVX512-32-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v4f32_uge_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpnlt_uqps %xmm3, %xmm2, %k1
+; AVX512-64-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %f1, <4 x float> %f2, metadata !"uge",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <4 x i1> %cond, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_v4f32_ult_q(<4 x i32> %a, <4 x i32> %b, <4 x float> %f1, <4 x float> %f2) #0 {
+; SSE-32-LABEL: test_v4f32_ult_q:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    andl $-16, %esp
+; SSE-32-NEXT:    subl $16, %esp
+; SSE-32-NEXT:    movaps 8(%ebp), %xmm3
+; SSE-32-NEXT:    movaps %xmm3, %xmm4
+; SSE-32-NEXT:    shufps {{.*#+}} xmm4 = xmm4[3,1],xmm3[2,3]
+; SSE-32-NEXT:    movaps %xmm2, %xmm5
+; SSE-32-NEXT:    shufps {{.*#+}} xmm5 = xmm5[3,1],xmm2[2,3]
+; SSE-32-NEXT:    xorl %eax, %eax
+; SSE-32-NEXT:    ucomiss %xmm4, %xmm5
+; SSE-32-NEXT:    movl $-1, %ecx
+; SSE-32-NEXT:    movl $0, %edx
+; SSE-32-NEXT:    cmovbl %ecx, %edx
+; SSE-32-NEXT:    movd %edx, %xmm4
+; SSE-32-NEXT:    movaps %xmm3, %xmm5
+; SSE-32-NEXT:    unpckhpd {{.*#+}} xmm5 = xmm5[1],xmm3[1]
+; SSE-32-NEXT:    movaps %xmm2, %xmm6
+; SSE-32-NEXT:    unpckhpd {{.*#+}} xmm6 = xmm6[1],xmm2[1]
+; SSE-32-NEXT:    ucomiss %xmm5, %xmm6
+; SSE-32-NEXT:    movl $0, %edx
+; SSE-32-NEXT:    cmovbl %ecx, %edx
+; SSE-32-NEXT:    movd %edx, %xmm5
+; SSE-32-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
+; SSE-32-NEXT:    ucomiss %xmm3, %xmm2
+; SSE-32-NEXT:    movl $0, %edx
+; SSE-32-NEXT:    cmovbl %ecx, %edx
+; SSE-32-NEXT:    movd %edx, %xmm4
+; SSE-32-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,2,3]
+; SSE-32-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,2,3]
+; SSE-32-NEXT:    ucomiss %xmm3, %xmm2
+; SSE-32-NEXT:    cmovbl %ecx, %eax
+; SSE-32-NEXT:    movd %eax, %xmm2
+; SSE-32-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
+; SSE-32-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm5[0]
+; SSE-32-NEXT:    pand %xmm4, %xmm0
+; SSE-32-NEXT:    pandn %xmm1, %xmm4
+; SSE-32-NEXT:    por %xmm4, %xmm0
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_v4f32_ult_q:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movaps %xmm3, %xmm4
+; SSE-64-NEXT:    shufps {{.*#+}} xmm4 = xmm4[3,1],xmm3[2,3]
+; SSE-64-NEXT:    movaps %xmm2, %xmm5
+; SSE-64-NEXT:    shufps {{.*#+}} xmm5 = xmm5[3,1],xmm2[2,3]
+; SSE-64-NEXT:    xorl %eax, %eax
+; SSE-64-NEXT:    ucomiss %xmm4, %xmm5
+; SSE-64-NEXT:    movl $-1, %ecx
+; SSE-64-NEXT:    movl $0, %edx
+; SSE-64-NEXT:    cmovbl %ecx, %edx
+; SSE-64-NEXT:    movd %edx, %xmm4
+; SSE-64-NEXT:    movaps %xmm3, %xmm5
+; SSE-64-NEXT:    unpckhpd {{.*#+}} xmm5 = xmm5[1],xmm3[1]
+; SSE-64-NEXT:    movaps %xmm2, %xmm6
+; SSE-64-NEXT:    unpckhpd {{.*#+}} xmm6 = xmm6[1],xmm2[1]
+; SSE-64-NEXT:    ucomiss %xmm5, %xmm6
+; SSE-64-NEXT:    movl $0, %edx
+; SSE-64-NEXT:    cmovbl %ecx, %edx
+; SSE-64-NEXT:    movd %edx, %xmm5
+; SSE-64-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
+; SSE-64-NEXT:    ucomiss %xmm3, %xmm2
+; SSE-64-NEXT:    movl $0, %edx
+; SSE-64-NEXT:    cmovbl %ecx, %edx
+; SSE-64-NEXT:    movd %edx, %xmm4
+; SSE-64-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,2,3]
+; SSE-64-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,2,3]
+; SSE-64-NEXT:    ucomiss %xmm3, %xmm2
+; SSE-64-NEXT:    cmovbl %ecx, %eax
+; SSE-64-NEXT:    movd %eax, %xmm2
+; SSE-64-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
+; SSE-64-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm5[0]
+; SSE-64-NEXT:    pand %xmm4, %xmm0
+; SSE-64-NEXT:    pandn %xmm1, %xmm4
+; SSE-64-NEXT:    por %xmm4, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_v4f32_ult_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-16, %esp
+; AVX-32-NEXT:    subl $16, %esp
+; AVX-32-NEXT:    vmovaps 8(%ebp), %xmm3
+; AVX-32-NEXT:    vcmpnle_uqps %xmm2, %xmm3, %xmm2
+; AVX-32-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v4f32_ult_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpnle_uqps %xmm2, %xmm3, %xmm2
+; AVX-64-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v4f32_ult_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-16, %esp
+; AVX512-32-NEXT:    subl $16, %esp
+; AVX512-32-NEXT:    vcmpnge_uqps 8(%ebp), %xmm2, %k1
+; AVX512-32-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v4f32_ult_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpnle_uqps %xmm2, %xmm3, %k1
+; AVX512-64-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %f1, <4 x float> %f2, metadata !"ult",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <4 x i1> %cond, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_v4f32_ule_q(<4 x i32> %a, <4 x i32> %b, <4 x float> %f1, <4 x float> %f2) #0 {
+; SSE-32-LABEL: test_v4f32_ule_q:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    andl $-16, %esp
+; SSE-32-NEXT:    subl $16, %esp
+; SSE-32-NEXT:    movaps 8(%ebp), %xmm3
+; SSE-32-NEXT:    movaps %xmm3, %xmm4
+; SSE-32-NEXT:    shufps {{.*#+}} xmm4 = xmm4[3,1],xmm3[2,3]
+; SSE-32-NEXT:    movaps %xmm2, %xmm5
+; SSE-32-NEXT:    shufps {{.*#+}} xmm5 = xmm5[3,1],xmm2[2,3]
+; SSE-32-NEXT:    xorl %eax, %eax
+; SSE-32-NEXT:    ucomiss %xmm4, %xmm5
+; SSE-32-NEXT:    movl $-1, %ecx
+; SSE-32-NEXT:    movl $0, %edx
+; SSE-32-NEXT:    cmovbel %ecx, %edx
+; SSE-32-NEXT:    movd %edx, %xmm4
+; SSE-32-NEXT:    movaps %xmm3, %xmm5
+; SSE-32-NEXT:    unpckhpd {{.*#+}} xmm5 = xmm5[1],xmm3[1]
+; SSE-32-NEXT:    movaps %xmm2, %xmm6
+; SSE-32-NEXT:    unpckhpd {{.*#+}} xmm6 = xmm6[1],xmm2[1]
+; SSE-32-NEXT:    ucomiss %xmm5, %xmm6
+; SSE-32-NEXT:    movl $0, %edx
+; SSE-32-NEXT:    cmovbel %ecx, %edx
+; SSE-32-NEXT:    movd %edx, %xmm5
+; SSE-32-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
+; SSE-32-NEXT:    ucomiss %xmm3, %xmm2
+; SSE-32-NEXT:    movl $0, %edx
+; SSE-32-NEXT:    cmovbel %ecx, %edx
+; SSE-32-NEXT:    movd %edx, %xmm4
+; SSE-32-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,2,3]
+; SSE-32-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,2,3]
+; SSE-32-NEXT:    ucomiss %xmm3, %xmm2
+; SSE-32-NEXT:    cmovbel %ecx, %eax
+; SSE-32-NEXT:    movd %eax, %xmm2
+; SSE-32-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
+; SSE-32-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm5[0]
+; SSE-32-NEXT:    pand %xmm4, %xmm0
+; SSE-32-NEXT:    pandn %xmm1, %xmm4
+; SSE-32-NEXT:    por %xmm4, %xmm0
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_v4f32_ule_q:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movaps %xmm3, %xmm4
+; SSE-64-NEXT:    shufps {{.*#+}} xmm4 = xmm4[3,1],xmm3[2,3]
+; SSE-64-NEXT:    movaps %xmm2, %xmm5
+; SSE-64-NEXT:    shufps {{.*#+}} xmm5 = xmm5[3,1],xmm2[2,3]
+; SSE-64-NEXT:    xorl %eax, %eax
+; SSE-64-NEXT:    ucomiss %xmm4, %xmm5
+; SSE-64-NEXT:    movl $-1, %ecx
+; SSE-64-NEXT:    movl $0, %edx
+; SSE-64-NEXT:    cmovbel %ecx, %edx
+; SSE-64-NEXT:    movd %edx, %xmm4
+; SSE-64-NEXT:    movaps %xmm3, %xmm5
+; SSE-64-NEXT:    unpckhpd {{.*#+}} xmm5 = xmm5[1],xmm3[1]
+; SSE-64-NEXT:    movaps %xmm2, %xmm6
+; SSE-64-NEXT:    unpckhpd {{.*#+}} xmm6 = xmm6[1],xmm2[1]
+; SSE-64-NEXT:    ucomiss %xmm5, %xmm6
+; SSE-64-NEXT:    movl $0, %edx
+; SSE-64-NEXT:    cmovbel %ecx, %edx
+; SSE-64-NEXT:    movd %edx, %xmm5
+; SSE-64-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
+; SSE-64-NEXT:    ucomiss %xmm3, %xmm2
+; SSE-64-NEXT:    movl $0, %edx
+; SSE-64-NEXT:    cmovbel %ecx, %edx
+; SSE-64-NEXT:    movd %edx, %xmm4
+; SSE-64-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,2,3]
+; SSE-64-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,2,3]
+; SSE-64-NEXT:    ucomiss %xmm3, %xmm2
+; SSE-64-NEXT:    cmovbel %ecx, %eax
+; SSE-64-NEXT:    movd %eax, %xmm2
+; SSE-64-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
+; SSE-64-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm5[0]
+; SSE-64-NEXT:    pand %xmm4, %xmm0
+; SSE-64-NEXT:    pandn %xmm1, %xmm4
+; SSE-64-NEXT:    por %xmm4, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_v4f32_ule_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-16, %esp
+; AVX-32-NEXT:    subl $16, %esp
+; AVX-32-NEXT:    vmovaps 8(%ebp), %xmm3
+; AVX-32-NEXT:    vcmpnlt_uqps %xmm2, %xmm3, %xmm2
+; AVX-32-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v4f32_ule_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpnlt_uqps %xmm2, %xmm3, %xmm2
+; AVX-64-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v4f32_ule_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-16, %esp
+; AVX512-32-NEXT:    subl $16, %esp
+; AVX512-32-NEXT:    vcmpngt_uqps 8(%ebp), %xmm2, %k1
+; AVX512-32-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v4f32_ule_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpnlt_uqps %xmm2, %xmm3, %k1
+; AVX512-64-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %f1, <4 x float> %f2, metadata !"ule",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <4 x i1> %cond, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_v4f32_une_q(<4 x i32> %a, <4 x i32> %b, <4 x float> %f1, <4 x float> %f2) #0 {
+; SSE-32-LABEL: test_v4f32_une_q:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    andl $-16, %esp
+; SSE-32-NEXT:    subl $16, %esp
+; SSE-32-NEXT:    cmpneqps 8(%ebp), %xmm2
+; SSE-32-NEXT:    andps %xmm2, %xmm0
+; SSE-32-NEXT:    andnps %xmm1, %xmm2
+; SSE-32-NEXT:    orps %xmm2, %xmm0
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_v4f32_une_q:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    cmpneqps %xmm3, %xmm2
+; SSE-64-NEXT:    andps %xmm2, %xmm0
+; SSE-64-NEXT:    andnps %xmm1, %xmm2
+; SSE-64-NEXT:    orps %xmm2, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_v4f32_une_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-16, %esp
+; AVX-32-NEXT:    subl $16, %esp
+; AVX-32-NEXT:    vcmpneqps 8(%ebp), %xmm2, %xmm2
+; AVX-32-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v4f32_une_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpneqps %xmm3, %xmm2, %xmm2
+; AVX-64-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v4f32_une_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-16, %esp
+; AVX512-32-NEXT:    subl $16, %esp
+; AVX512-32-NEXT:    vcmpneqps 8(%ebp), %xmm2, %k1
+; AVX512-32-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v4f32_une_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpneqps %xmm3, %xmm2, %k1
+; AVX512-64-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %f1, <4 x float> %f2, metadata !"une",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <4 x i1> %cond, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_v4f32_uno_q(<4 x i32> %a, <4 x i32> %b, <4 x float> %f1, <4 x float> %f2) #0 {
+; SSE-32-LABEL: test_v4f32_uno_q:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    andl $-16, %esp
+; SSE-32-NEXT:    subl $16, %esp
+; SSE-32-NEXT:    cmpunordps 8(%ebp), %xmm2
+; SSE-32-NEXT:    andps %xmm2, %xmm0
+; SSE-32-NEXT:    andnps %xmm1, %xmm2
+; SSE-32-NEXT:    orps %xmm2, %xmm0
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_v4f32_uno_q:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    cmpunordps %xmm3, %xmm2
+; SSE-64-NEXT:    andps %xmm2, %xmm0
+; SSE-64-NEXT:    andnps %xmm1, %xmm2
+; SSE-64-NEXT:    orps %xmm2, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_v4f32_uno_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-16, %esp
+; AVX-32-NEXT:    subl $16, %esp
+; AVX-32-NEXT:    vcmpunordps 8(%ebp), %xmm2, %xmm2
+; AVX-32-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v4f32_uno_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpunordps %xmm3, %xmm2, %xmm2
+; AVX-64-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v4f32_uno_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-16, %esp
+; AVX512-32-NEXT:    subl $16, %esp
+; AVX512-32-NEXT:    vcmpunordps 8(%ebp), %xmm2, %k1
+; AVX512-32-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v4f32_uno_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpunordps %xmm3, %xmm2, %k1
+; AVX512-64-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(
+                                               <4 x float> %f1, <4 x float> %f2, metadata !"uno",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <4 x i1> %cond, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %res
+}
+
+define <2 x i64> @test_v2f64_oeq_q(<2 x i64> %a, <2 x i64> %b, <2 x double> %f1, <2 x double> %f2) #0 {
+; SSE-32-LABEL: test_v2f64_oeq_q:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    andl $-16, %esp
+; SSE-32-NEXT:    subl $16, %esp
+; SSE-32-NEXT:    cmpeqpd 8(%ebp), %xmm2
+; SSE-32-NEXT:    andpd %xmm2, %xmm0
+; SSE-32-NEXT:    andnpd %xmm1, %xmm2
+; SSE-32-NEXT:    orpd %xmm2, %xmm0
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_v2f64_oeq_q:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    cmpeqpd %xmm3, %xmm2
+; SSE-64-NEXT:    andpd %xmm2, %xmm0
+; SSE-64-NEXT:    andnpd %xmm1, %xmm2
+; SSE-64-NEXT:    orpd %xmm2, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_v2f64_oeq_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-16, %esp
+; AVX-32-NEXT:    subl $16, %esp
+; AVX-32-NEXT:    vcmpeqpd 8(%ebp), %xmm2, %xmm2
+; AVX-32-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v2f64_oeq_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpeqpd %xmm3, %xmm2, %xmm2
+; AVX-64-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v2f64_oeq_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-16, %esp
+; AVX512-32-NEXT:    subl $16, %esp
+; AVX512-32-NEXT:    vcmpeqpd 8(%ebp), %xmm2, %k1
+; AVX512-32-NEXT:    vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v2f64_oeq_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpeqpd %xmm3, %xmm2, %k1
+; AVX512-64-NEXT:    vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <2 x i1> @llvm.experimental.constrained.fcmp.v2f64(
+                                               <2 x double> %f1, <2 x double> %f2, metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <2 x i1> %cond, <2 x i64> %a, <2 x i64> %b
+  ret <2 x i64> %res
+}
+
+define <2 x i64> @test_v2f64_ogt_q(<2 x i64> %a, <2 x i64> %b, <2 x double> %f1, <2 x double> %f2) #0 {
+; SSE-32-LABEL: test_v2f64_ogt_q:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    andl $-16, %esp
+; SSE-32-NEXT:    subl $16, %esp
+; SSE-32-NEXT:    movapd 8(%ebp), %xmm3
+; SSE-32-NEXT:    xorl %eax, %eax
+; SSE-32-NEXT:    ucomisd %xmm3, %xmm2
+; SSE-32-NEXT:    movl $-1, %ecx
+; SSE-32-NEXT:    movl $0, %edx
+; SSE-32-NEXT:    cmoval %ecx, %edx
+; SSE-32-NEXT:    movd %edx, %xmm4
+; SSE-32-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[0,0,1,1]
+; SSE-32-NEXT:    unpckhpd {{.*#+}} xmm3 = xmm3[1,1]
+; SSE-32-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1,1]
+; SSE-32-NEXT:    ucomisd %xmm3, %xmm2
+; SSE-32-NEXT:    cmoval %ecx, %eax
+; SSE-32-NEXT:    movd %eax, %xmm2
+; SSE-32-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,0,1,1]
+; SSE-32-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm2[0]
+; SSE-32-NEXT:    pand %xmm4, %xmm0
+; SSE-32-NEXT:    pandn %xmm1, %xmm4
+; SSE-32-NEXT:    por %xmm4, %xmm0
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_v2f64_ogt_q:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    xorl %eax, %eax
+; SSE-64-NEXT:    ucomisd %xmm3, %xmm2
+; SSE-64-NEXT:    movq $-1, %rcx
+; SSE-64-NEXT:    movl $0, %edx
+; SSE-64-NEXT:    cmovaq %rcx, %rdx
+; SSE-64-NEXT:    movq %rdx, %xmm4
+; SSE-64-NEXT:    unpckhpd {{.*#+}} xmm3 = xmm3[1,1]
+; SSE-64-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1,1]
+; SSE-64-NEXT:    ucomisd %xmm3, %xmm2
+; SSE-64-NEXT:    cmovaq %rcx, %rax
+; SSE-64-NEXT:    movq %rax, %xmm2
+; SSE-64-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm2[0]
+; SSE-64-NEXT:    pand %xmm4, %xmm0
+; SSE-64-NEXT:    pandn %xmm1, %xmm4
+; SSE-64-NEXT:    por %xmm4, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_v2f64_ogt_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-16, %esp
+; AVX-32-NEXT:    subl $16, %esp
+; AVX-32-NEXT:    vmovapd 8(%ebp), %xmm3
+; AVX-32-NEXT:    vcmplt_oqpd %xmm2, %xmm3, %xmm2
+; AVX-32-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v2f64_ogt_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmplt_oqpd %xmm2, %xmm3, %xmm2
+; AVX-64-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v2f64_ogt_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-16, %esp
+; AVX512-32-NEXT:    subl $16, %esp
+; AVX512-32-NEXT:    vcmpgt_oqpd 8(%ebp), %xmm2, %k1
+; AVX512-32-NEXT:    vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v2f64_ogt_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmplt_oqpd %xmm2, %xmm3, %k1
+; AVX512-64-NEXT:    vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <2 x i1> @llvm.experimental.constrained.fcmp.v2f64(
+                                               <2 x double> %f1, <2 x double> %f2, metadata !"ogt",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <2 x i1> %cond, <2 x i64> %a, <2 x i64> %b
+  ret <2 x i64> %res
+}
+
+define <2 x i64> @test_v2f64_oge_q(<2 x i64> %a, <2 x i64> %b, <2 x double> %f1, <2 x double> %f2) #0 {
+; SSE-32-LABEL: test_v2f64_oge_q:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    andl $-16, %esp
+; SSE-32-NEXT:    subl $16, %esp
+; SSE-32-NEXT:    movapd 8(%ebp), %xmm3
+; SSE-32-NEXT:    xorl %eax, %eax
+; SSE-32-NEXT:    ucomisd %xmm3, %xmm2
+; SSE-32-NEXT:    movl $-1, %ecx
+; SSE-32-NEXT:    movl $0, %edx
+; SSE-32-NEXT:    cmovael %ecx, %edx
+; SSE-32-NEXT:    movd %edx, %xmm4
+; SSE-32-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[0,0,1,1]
+; SSE-32-NEXT:    unpckhpd {{.*#+}} xmm3 = xmm3[1,1]
+; SSE-32-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1,1]
+; SSE-32-NEXT:    ucomisd %xmm3, %xmm2
+; SSE-32-NEXT:    cmovael %ecx, %eax
+; SSE-32-NEXT:    movd %eax, %xmm2
+; SSE-32-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,0,1,1]
+; SSE-32-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm2[0]
+; SSE-32-NEXT:    pand %xmm4, %xmm0
+; SSE-32-NEXT:    pandn %xmm1, %xmm4
+; SSE-32-NEXT:    por %xmm4, %xmm0
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_v2f64_oge_q:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    xorl %eax, %eax
+; SSE-64-NEXT:    ucomisd %xmm3, %xmm2
+; SSE-64-NEXT:    movq $-1, %rcx
+; SSE-64-NEXT:    movl $0, %edx
+; SSE-64-NEXT:    cmovaeq %rcx, %rdx
+; SSE-64-NEXT:    movq %rdx, %xmm4
+; SSE-64-NEXT:    unpckhpd {{.*#+}} xmm3 = xmm3[1,1]
+; SSE-64-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1,1]
+; SSE-64-NEXT:    ucomisd %xmm3, %xmm2
+; SSE-64-NEXT:    cmovaeq %rcx, %rax
+; SSE-64-NEXT:    movq %rax, %xmm2
+; SSE-64-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm2[0]
+; SSE-64-NEXT:    pand %xmm4, %xmm0
+; SSE-64-NEXT:    pandn %xmm1, %xmm4
+; SSE-64-NEXT:    por %xmm4, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_v2f64_oge_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-16, %esp
+; AVX-32-NEXT:    subl $16, %esp
+; AVX-32-NEXT:    vmovapd 8(%ebp), %xmm3
+; AVX-32-NEXT:    vcmple_oqpd %xmm2, %xmm3, %xmm2
+; AVX-32-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v2f64_oge_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmple_oqpd %xmm2, %xmm3, %xmm2
+; AVX-64-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v2f64_oge_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-16, %esp
+; AVX512-32-NEXT:    subl $16, %esp
+; AVX512-32-NEXT:    vcmpge_oqpd 8(%ebp), %xmm2, %k1
+; AVX512-32-NEXT:    vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v2f64_oge_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmple_oqpd %xmm2, %xmm3, %k1
+; AVX512-64-NEXT:    vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <2 x i1> @llvm.experimental.constrained.fcmp.v2f64(
+                                               <2 x double> %f1, <2 x double> %f2, metadata !"oge",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <2 x i1> %cond, <2 x i64> %a, <2 x i64> %b
+  ret <2 x i64> %res
+}
+
+define <2 x i64> @test_v2f64_olt_q(<2 x i64> %a, <2 x i64> %b, <2 x double> %f1, <2 x double> %f2) #0 {
+; SSE-32-LABEL: test_v2f64_olt_q:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    andl $-16, %esp
+; SSE-32-NEXT:    subl $16, %esp
+; SSE-32-NEXT:    movapd 8(%ebp), %xmm3
+; SSE-32-NEXT:    xorl %eax, %eax
+; SSE-32-NEXT:    ucomisd %xmm2, %xmm3
+; SSE-32-NEXT:    movl $-1, %ecx
+; SSE-32-NEXT:    movl $0, %edx
+; SSE-32-NEXT:    cmoval %ecx, %edx
+; SSE-32-NEXT:    movd %edx, %xmm4
+; SSE-32-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[0,0,1,1]
+; SSE-32-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1,1]
+; SSE-32-NEXT:    unpckhpd {{.*#+}} xmm3 = xmm3[1,1]
+; SSE-32-NEXT:    ucomisd %xmm2, %xmm3
+; SSE-32-NEXT:    cmoval %ecx, %eax
+; SSE-32-NEXT:    movd %eax, %xmm2
+; SSE-32-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,0,1,1]
+; SSE-32-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm2[0]
+; SSE-32-NEXT:    pand %xmm4, %xmm0
+; SSE-32-NEXT:    pandn %xmm1, %xmm4
+; SSE-32-NEXT:    por %xmm4, %xmm0
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_v2f64_olt_q:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    xorl %eax, %eax
+; SSE-64-NEXT:    ucomisd %xmm2, %xmm3
+; SSE-64-NEXT:    movq $-1, %rcx
+; SSE-64-NEXT:    movl $0, %edx
+; SSE-64-NEXT:    cmovaq %rcx, %rdx
+; SSE-64-NEXT:    movq %rdx, %xmm4
+; SSE-64-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1,1]
+; SSE-64-NEXT:    unpckhpd {{.*#+}} xmm3 = xmm3[1,1]
+; SSE-64-NEXT:    ucomisd %xmm2, %xmm3
+; SSE-64-NEXT:    cmovaq %rcx, %rax
+; SSE-64-NEXT:    movq %rax, %xmm2
+; SSE-64-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm2[0]
+; SSE-64-NEXT:    pand %xmm4, %xmm0
+; SSE-64-NEXT:    pandn %xmm1, %xmm4
+; SSE-64-NEXT:    por %xmm4, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_v2f64_olt_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-16, %esp
+; AVX-32-NEXT:    subl $16, %esp
+; AVX-32-NEXT:    vcmplt_oqpd 8(%ebp), %xmm2, %xmm2
+; AVX-32-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v2f64_olt_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmplt_oqpd %xmm3, %xmm2, %xmm2
+; AVX-64-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v2f64_olt_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-16, %esp
+; AVX512-32-NEXT:    subl $16, %esp
+; AVX512-32-NEXT:    vcmplt_oqpd 8(%ebp), %xmm2, %k1
+; AVX512-32-NEXT:    vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v2f64_olt_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmplt_oqpd %xmm3, %xmm2, %k1
+; AVX512-64-NEXT:    vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <2 x i1> @llvm.experimental.constrained.fcmp.v2f64(
+                                               <2 x double> %f1, <2 x double> %f2, metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <2 x i1> %cond, <2 x i64> %a, <2 x i64> %b
+  ret <2 x i64> %res
+}
+
+define <2 x i64> @test_v2f64_ole_q(<2 x i64> %a, <2 x i64> %b, <2 x double> %f1, <2 x double> %f2) #0 {
+; SSE-32-LABEL: test_v2f64_ole_q:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    andl $-16, %esp
+; SSE-32-NEXT:    subl $16, %esp
+; SSE-32-NEXT:    movapd 8(%ebp), %xmm3
+; SSE-32-NEXT:    xorl %eax, %eax
+; SSE-32-NEXT:    ucomisd %xmm2, %xmm3
+; SSE-32-NEXT:    movl $-1, %ecx
+; SSE-32-NEXT:    movl $0, %edx
+; SSE-32-NEXT:    cmovael %ecx, %edx
+; SSE-32-NEXT:    movd %edx, %xmm4
+; SSE-32-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[0,0,1,1]
+; SSE-32-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1,1]
+; SSE-32-NEXT:    unpckhpd {{.*#+}} xmm3 = xmm3[1,1]
+; SSE-32-NEXT:    ucomisd %xmm2, %xmm3
+; SSE-32-NEXT:    cmovael %ecx, %eax
+; SSE-32-NEXT:    movd %eax, %xmm2
+; SSE-32-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,0,1,1]
+; SSE-32-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm2[0]
+; SSE-32-NEXT:    pand %xmm4, %xmm0
+; SSE-32-NEXT:    pandn %xmm1, %xmm4
+; SSE-32-NEXT:    por %xmm4, %xmm0
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_v2f64_ole_q:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    xorl %eax, %eax
+; SSE-64-NEXT:    ucomisd %xmm2, %xmm3
+; SSE-64-NEXT:    movq $-1, %rcx
+; SSE-64-NEXT:    movl $0, %edx
+; SSE-64-NEXT:    cmovaeq %rcx, %rdx
+; SSE-64-NEXT:    movq %rdx, %xmm4
+; SSE-64-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1,1]
+; SSE-64-NEXT:    unpckhpd {{.*#+}} xmm3 = xmm3[1,1]
+; SSE-64-NEXT:    ucomisd %xmm2, %xmm3
+; SSE-64-NEXT:    cmovaeq %rcx, %rax
+; SSE-64-NEXT:    movq %rax, %xmm2
+; SSE-64-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm2[0]
+; SSE-64-NEXT:    pand %xmm4, %xmm0
+; SSE-64-NEXT:    pandn %xmm1, %xmm4
+; SSE-64-NEXT:    por %xmm4, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_v2f64_ole_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-16, %esp
+; AVX-32-NEXT:    subl $16, %esp
+; AVX-32-NEXT:    vcmple_oqpd 8(%ebp), %xmm2, %xmm2
+; AVX-32-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v2f64_ole_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmple_oqpd %xmm3, %xmm2, %xmm2
+; AVX-64-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v2f64_ole_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-16, %esp
+; AVX512-32-NEXT:    subl $16, %esp
+; AVX512-32-NEXT:    vcmple_oqpd 8(%ebp), %xmm2, %k1
+; AVX512-32-NEXT:    vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v2f64_ole_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmple_oqpd %xmm3, %xmm2, %k1
+; AVX512-64-NEXT:    vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <2 x i1> @llvm.experimental.constrained.fcmp.v2f64(
+                                               <2 x double> %f1, <2 x double> %f2, metadata !"ole",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <2 x i1> %cond, <2 x i64> %a, <2 x i64> %b
+  ret <2 x i64> %res
+}
+
+define <2 x i64> @test_v2f64_one_q(<2 x i64> %a, <2 x i64> %b, <2 x double> %f1, <2 x double> %f2) #0 {
+; SSE-32-LABEL: test_v2f64_one_q:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    andl $-16, %esp
+; SSE-32-NEXT:    subl $16, %esp
+; SSE-32-NEXT:    movapd 8(%ebp), %xmm3
+; SSE-32-NEXT:    movapd %xmm2, %xmm4
+; SSE-32-NEXT:    cmpneqpd %xmm3, %xmm4
+; SSE-32-NEXT:    cmpordpd %xmm3, %xmm2
+; SSE-32-NEXT:    andpd %xmm4, %xmm2
+; SSE-32-NEXT:    andpd %xmm2, %xmm0
+; SSE-32-NEXT:    andnpd %xmm1, %xmm2
+; SSE-32-NEXT:    orpd %xmm2, %xmm0
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_v2f64_one_q:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movapd %xmm2, %xmm4
+; SSE-64-NEXT:    cmpneqpd %xmm3, %xmm4
+; SSE-64-NEXT:    cmpordpd %xmm3, %xmm2
+; SSE-64-NEXT:    andpd %xmm4, %xmm2
+; SSE-64-NEXT:    andpd %xmm2, %xmm0
+; SSE-64-NEXT:    andnpd %xmm1, %xmm2
+; SSE-64-NEXT:    orpd %xmm2, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_v2f64_one_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-16, %esp
+; AVX-32-NEXT:    subl $16, %esp
+; AVX-32-NEXT:    vcmpneq_oqpd 8(%ebp), %xmm2, %xmm2
+; AVX-32-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v2f64_one_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpneq_oqpd %xmm3, %xmm2, %xmm2
+; AVX-64-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v2f64_one_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-16, %esp
+; AVX512-32-NEXT:    subl $16, %esp
+; AVX512-32-NEXT:    vcmpneq_oqpd 8(%ebp), %xmm2, %k1
+; AVX512-32-NEXT:    vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v2f64_one_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpneq_oqpd %xmm3, %xmm2, %k1
+; AVX512-64-NEXT:    vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <2 x i1> @llvm.experimental.constrained.fcmp.v2f64(
+                                               <2 x double> %f1, <2 x double> %f2, metadata !"one",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <2 x i1> %cond, <2 x i64> %a, <2 x i64> %b
+  ret <2 x i64> %res
+}
+
+define <2 x i64> @test_v2f64_ord_q(<2 x i64> %a, <2 x i64> %b, <2 x double> %f1, <2 x double> %f2) #0 {
+; SSE-32-LABEL: test_v2f64_ord_q:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    andl $-16, %esp
+; SSE-32-NEXT:    subl $16, %esp
+; SSE-32-NEXT:    cmpordpd 8(%ebp), %xmm2
+; SSE-32-NEXT:    andpd %xmm2, %xmm0
+; SSE-32-NEXT:    andnpd %xmm1, %xmm2
+; SSE-32-NEXT:    orpd %xmm2, %xmm0
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_v2f64_ord_q:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    cmpordpd %xmm3, %xmm2
+; SSE-64-NEXT:    andpd %xmm2, %xmm0
+; SSE-64-NEXT:    andnpd %xmm1, %xmm2
+; SSE-64-NEXT:    orpd %xmm2, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_v2f64_ord_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-16, %esp
+; AVX-32-NEXT:    subl $16, %esp
+; AVX-32-NEXT:    vcmpordpd 8(%ebp), %xmm2, %xmm2
+; AVX-32-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v2f64_ord_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpordpd %xmm3, %xmm2, %xmm2
+; AVX-64-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v2f64_ord_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-16, %esp
+; AVX512-32-NEXT:    subl $16, %esp
+; AVX512-32-NEXT:    vcmpordpd 8(%ebp), %xmm2, %k1
+; AVX512-32-NEXT:    vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v2f64_ord_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpordpd %xmm3, %xmm2, %k1
+; AVX512-64-NEXT:    vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <2 x i1> @llvm.experimental.constrained.fcmp.v2f64(
+                                               <2 x double> %f1, <2 x double> %f2, metadata !"ord",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <2 x i1> %cond, <2 x i64> %a, <2 x i64> %b
+  ret <2 x i64> %res
+}
+
+define <2 x i64> @test_v2f64_ueq_q(<2 x i64> %a, <2 x i64> %b, <2 x double> %f1, <2 x double> %f2) #0 {
+; SSE-32-LABEL: test_v2f64_ueq_q:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    andl $-16, %esp
+; SSE-32-NEXT:    subl $16, %esp
+; SSE-32-NEXT:    movapd 8(%ebp), %xmm3
+; SSE-32-NEXT:    movapd %xmm2, %xmm4
+; SSE-32-NEXT:    cmpeqpd %xmm3, %xmm4
+; SSE-32-NEXT:    cmpunordpd %xmm3, %xmm2
+; SSE-32-NEXT:    orpd %xmm4, %xmm2
+; SSE-32-NEXT:    andpd %xmm2, %xmm0
+; SSE-32-NEXT:    andnpd %xmm1, %xmm2
+; SSE-32-NEXT:    orpd %xmm2, %xmm0
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_v2f64_ueq_q:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movapd %xmm2, %xmm4
+; SSE-64-NEXT:    cmpeqpd %xmm3, %xmm4
+; SSE-64-NEXT:    cmpunordpd %xmm3, %xmm2
+; SSE-64-NEXT:    orpd %xmm4, %xmm2
+; SSE-64-NEXT:    andpd %xmm2, %xmm0
+; SSE-64-NEXT:    andnpd %xmm1, %xmm2
+; SSE-64-NEXT:    orpd %xmm2, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_v2f64_ueq_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-16, %esp
+; AVX-32-NEXT:    subl $16, %esp
+; AVX-32-NEXT:    vcmpeq_uqpd 8(%ebp), %xmm2, %xmm2
+; AVX-32-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v2f64_ueq_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpeq_uqpd %xmm3, %xmm2, %xmm2
+; AVX-64-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v2f64_ueq_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-16, %esp
+; AVX512-32-NEXT:    subl $16, %esp
+; AVX512-32-NEXT:    vcmpeq_uqpd 8(%ebp), %xmm2, %k1
+; AVX512-32-NEXT:    vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v2f64_ueq_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpeq_uqpd %xmm3, %xmm2, %k1
+; AVX512-64-NEXT:    vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <2 x i1> @llvm.experimental.constrained.fcmp.v2f64(
+                                               <2 x double> %f1, <2 x double> %f2, metadata !"ueq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <2 x i1> %cond, <2 x i64> %a, <2 x i64> %b
+  ret <2 x i64> %res
+}
+
+define <2 x i64> @test_v2f64_ugt_q(<2 x i64> %a, <2 x i64> %b, <2 x double> %f1, <2 x double> %f2) #0 {
+; SSE-32-LABEL: test_v2f64_ugt_q:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    andl $-16, %esp
+; SSE-32-NEXT:    subl $16, %esp
+; SSE-32-NEXT:    movapd 8(%ebp), %xmm3
+; SSE-32-NEXT:    xorl %eax, %eax
+; SSE-32-NEXT:    ucomisd %xmm2, %xmm3
+; SSE-32-NEXT:    movl $-1, %ecx
+; SSE-32-NEXT:    movl $0, %edx
+; SSE-32-NEXT:    cmovbl %ecx, %edx
+; SSE-32-NEXT:    movd %edx, %xmm4
+; SSE-32-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[0,0,1,1]
+; SSE-32-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1,1]
+; SSE-32-NEXT:    unpckhpd {{.*#+}} xmm3 = xmm3[1,1]
+; SSE-32-NEXT:    ucomisd %xmm2, %xmm3
+; SSE-32-NEXT:    cmovbl %ecx, %eax
+; SSE-32-NEXT:    movd %eax, %xmm2
+; SSE-32-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,0,1,1]
+; SSE-32-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm2[0]
+; SSE-32-NEXT:    pand %xmm4, %xmm0
+; SSE-32-NEXT:    pandn %xmm1, %xmm4
+; SSE-32-NEXT:    por %xmm4, %xmm0
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_v2f64_ugt_q:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    xorl %eax, %eax
+; SSE-64-NEXT:    ucomisd %xmm2, %xmm3
+; SSE-64-NEXT:    movq $-1, %rcx
+; SSE-64-NEXT:    movl $0, %edx
+; SSE-64-NEXT:    cmovbq %rcx, %rdx
+; SSE-64-NEXT:    movq %rdx, %xmm4
+; SSE-64-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1,1]
+; SSE-64-NEXT:    unpckhpd {{.*#+}} xmm3 = xmm3[1,1]
+; SSE-64-NEXT:    ucomisd %xmm2, %xmm3
+; SSE-64-NEXT:    cmovbq %rcx, %rax
+; SSE-64-NEXT:    movq %rax, %xmm2
+; SSE-64-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm2[0]
+; SSE-64-NEXT:    pand %xmm4, %xmm0
+; SSE-64-NEXT:    pandn %xmm1, %xmm4
+; SSE-64-NEXT:    por %xmm4, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_v2f64_ugt_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-16, %esp
+; AVX-32-NEXT:    subl $16, %esp
+; AVX-32-NEXT:    vcmpnle_uqpd 8(%ebp), %xmm2, %xmm2
+; AVX-32-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v2f64_ugt_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpnle_uqpd %xmm3, %xmm2, %xmm2
+; AVX-64-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v2f64_ugt_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-16, %esp
+; AVX512-32-NEXT:    subl $16, %esp
+; AVX512-32-NEXT:    vcmpnle_uqpd 8(%ebp), %xmm2, %k1
+; AVX512-32-NEXT:    vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v2f64_ugt_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpnle_uqpd %xmm3, %xmm2, %k1
+; AVX512-64-NEXT:    vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <2 x i1> @llvm.experimental.constrained.fcmp.v2f64(
+                                               <2 x double> %f1, <2 x double> %f2, metadata !"ugt",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <2 x i1> %cond, <2 x i64> %a, <2 x i64> %b
+  ret <2 x i64> %res
+}
+
+define <2 x i64> @test_v2f64_uge_q(<2 x i64> %a, <2 x i64> %b, <2 x double> %f1, <2 x double> %f2) #0 {
+; SSE-32-LABEL: test_v2f64_uge_q:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    andl $-16, %esp
+; SSE-32-NEXT:    subl $16, %esp
+; SSE-32-NEXT:    movapd 8(%ebp), %xmm3
+; SSE-32-NEXT:    xorl %eax, %eax
+; SSE-32-NEXT:    ucomisd %xmm2, %xmm3
+; SSE-32-NEXT:    movl $-1, %ecx
+; SSE-32-NEXT:    movl $0, %edx
+; SSE-32-NEXT:    cmovbel %ecx, %edx
+; SSE-32-NEXT:    movd %edx, %xmm4
+; SSE-32-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[0,0,1,1]
+; SSE-32-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1,1]
+; SSE-32-NEXT:    unpckhpd {{.*#+}} xmm3 = xmm3[1,1]
+; SSE-32-NEXT:    ucomisd %xmm2, %xmm3
+; SSE-32-NEXT:    cmovbel %ecx, %eax
+; SSE-32-NEXT:    movd %eax, %xmm2
+; SSE-32-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,0,1,1]
+; SSE-32-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm2[0]
+; SSE-32-NEXT:    pand %xmm4, %xmm0
+; SSE-32-NEXT:    pandn %xmm1, %xmm4
+; SSE-32-NEXT:    por %xmm4, %xmm0
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_v2f64_uge_q:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    xorl %eax, %eax
+; SSE-64-NEXT:    ucomisd %xmm2, %xmm3
+; SSE-64-NEXT:    movq $-1, %rcx
+; SSE-64-NEXT:    movl $0, %edx
+; SSE-64-NEXT:    cmovbeq %rcx, %rdx
+; SSE-64-NEXT:    movq %rdx, %xmm4
+; SSE-64-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1,1]
+; SSE-64-NEXT:    unpckhpd {{.*#+}} xmm3 = xmm3[1,1]
+; SSE-64-NEXT:    ucomisd %xmm2, %xmm3
+; SSE-64-NEXT:    cmovbeq %rcx, %rax
+; SSE-64-NEXT:    movq %rax, %xmm2
+; SSE-64-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm2[0]
+; SSE-64-NEXT:    pand %xmm4, %xmm0
+; SSE-64-NEXT:    pandn %xmm1, %xmm4
+; SSE-64-NEXT:    por %xmm4, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_v2f64_uge_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-16, %esp
+; AVX-32-NEXT:    subl $16, %esp
+; AVX-32-NEXT:    vcmpnlt_uqpd 8(%ebp), %xmm2, %xmm2
+; AVX-32-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v2f64_uge_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpnlt_uqpd %xmm3, %xmm2, %xmm2
+; AVX-64-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v2f64_uge_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-16, %esp
+; AVX512-32-NEXT:    subl $16, %esp
+; AVX512-32-NEXT:    vcmpnlt_uqpd 8(%ebp), %xmm2, %k1
+; AVX512-32-NEXT:    vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v2f64_uge_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpnlt_uqpd %xmm3, %xmm2, %k1
+; AVX512-64-NEXT:    vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <2 x i1> @llvm.experimental.constrained.fcmp.v2f64(
+                                               <2 x double> %f1, <2 x double> %f2, metadata !"uge",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <2 x i1> %cond, <2 x i64> %a, <2 x i64> %b
+  ret <2 x i64> %res
+}
+
+define <2 x i64> @test_v2f64_ult_q(<2 x i64> %a, <2 x i64> %b, <2 x double> %f1, <2 x double> %f2) #0 {
+; SSE-32-LABEL: test_v2f64_ult_q:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    andl $-16, %esp
+; SSE-32-NEXT:    subl $16, %esp
+; SSE-32-NEXT:    movapd 8(%ebp), %xmm3
+; SSE-32-NEXT:    xorl %eax, %eax
+; SSE-32-NEXT:    ucomisd %xmm3, %xmm2
+; SSE-32-NEXT:    movl $-1, %ecx
+; SSE-32-NEXT:    movl $0, %edx
+; SSE-32-NEXT:    cmovbl %ecx, %edx
+; SSE-32-NEXT:    movd %edx, %xmm4
+; SSE-32-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[0,0,1,1]
+; SSE-32-NEXT:    unpckhpd {{.*#+}} xmm3 = xmm3[1,1]
+; SSE-32-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1,1]
+; SSE-32-NEXT:    ucomisd %xmm3, %xmm2
+; SSE-32-NEXT:    cmovbl %ecx, %eax
+; SSE-32-NEXT:    movd %eax, %xmm2
+; SSE-32-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,0,1,1]
+; SSE-32-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm2[0]
+; SSE-32-NEXT:    pand %xmm4, %xmm0
+; SSE-32-NEXT:    pandn %xmm1, %xmm4
+; SSE-32-NEXT:    por %xmm4, %xmm0
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_v2f64_ult_q:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    xorl %eax, %eax
+; SSE-64-NEXT:    ucomisd %xmm3, %xmm2
+; SSE-64-NEXT:    movq $-1, %rcx
+; SSE-64-NEXT:    movl $0, %edx
+; SSE-64-NEXT:    cmovbq %rcx, %rdx
+; SSE-64-NEXT:    movq %rdx, %xmm4
+; SSE-64-NEXT:    unpckhpd {{.*#+}} xmm3 = xmm3[1,1]
+; SSE-64-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1,1]
+; SSE-64-NEXT:    ucomisd %xmm3, %xmm2
+; SSE-64-NEXT:    cmovbq %rcx, %rax
+; SSE-64-NEXT:    movq %rax, %xmm2
+; SSE-64-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm2[0]
+; SSE-64-NEXT:    pand %xmm4, %xmm0
+; SSE-64-NEXT:    pandn %xmm1, %xmm4
+; SSE-64-NEXT:    por %xmm4, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_v2f64_ult_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-16, %esp
+; AVX-32-NEXT:    subl $16, %esp
+; AVX-32-NEXT:    vmovapd 8(%ebp), %xmm3
+; AVX-32-NEXT:    vcmpnle_uqpd %xmm2, %xmm3, %xmm2
+; AVX-32-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v2f64_ult_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpnle_uqpd %xmm2, %xmm3, %xmm2
+; AVX-64-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v2f64_ult_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-16, %esp
+; AVX512-32-NEXT:    subl $16, %esp
+; AVX512-32-NEXT:    vcmpnge_uqpd 8(%ebp), %xmm2, %k1
+; AVX512-32-NEXT:    vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v2f64_ult_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpnle_uqpd %xmm2, %xmm3, %k1
+; AVX512-64-NEXT:    vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <2 x i1> @llvm.experimental.constrained.fcmp.v2f64(
+                                               <2 x double> %f1, <2 x double> %f2, metadata !"ult",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <2 x i1> %cond, <2 x i64> %a, <2 x i64> %b
+  ret <2 x i64> %res
+}
+
+define <2 x i64> @test_v2f64_ule_q(<2 x i64> %a, <2 x i64> %b, <2 x double> %f1, <2 x double> %f2) #0 {
+; SSE-32-LABEL: test_v2f64_ule_q:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    andl $-16, %esp
+; SSE-32-NEXT:    subl $16, %esp
+; SSE-32-NEXT:    movapd 8(%ebp), %xmm3
+; SSE-32-NEXT:    xorl %eax, %eax
+; SSE-32-NEXT:    ucomisd %xmm3, %xmm2
+; SSE-32-NEXT:    movl $-1, %ecx
+; SSE-32-NEXT:    movl $0, %edx
+; SSE-32-NEXT:    cmovbel %ecx, %edx
+; SSE-32-NEXT:    movd %edx, %xmm4
+; SSE-32-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[0,0,1,1]
+; SSE-32-NEXT:    unpckhpd {{.*#+}} xmm3 = xmm3[1,1]
+; SSE-32-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1,1]
+; SSE-32-NEXT:    ucomisd %xmm3, %xmm2
+; SSE-32-NEXT:    cmovbel %ecx, %eax
+; SSE-32-NEXT:    movd %eax, %xmm2
+; SSE-32-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,0,1,1]
+; SSE-32-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm2[0]
+; SSE-32-NEXT:    pand %xmm4, %xmm0
+; SSE-32-NEXT:    pandn %xmm1, %xmm4
+; SSE-32-NEXT:    por %xmm4, %xmm0
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_v2f64_ule_q:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    xorl %eax, %eax
+; SSE-64-NEXT:    ucomisd %xmm3, %xmm2
+; SSE-64-NEXT:    movq $-1, %rcx
+; SSE-64-NEXT:    movl $0, %edx
+; SSE-64-NEXT:    cmovbeq %rcx, %rdx
+; SSE-64-NEXT:    movq %rdx, %xmm4
+; SSE-64-NEXT:    unpckhpd {{.*#+}} xmm3 = xmm3[1,1]
+; SSE-64-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1,1]
+; SSE-64-NEXT:    ucomisd %xmm3, %xmm2
+; SSE-64-NEXT:    cmovbeq %rcx, %rax
+; SSE-64-NEXT:    movq %rax, %xmm2
+; SSE-64-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm2[0]
+; SSE-64-NEXT:    pand %xmm4, %xmm0
+; SSE-64-NEXT:    pandn %xmm1, %xmm4
+; SSE-64-NEXT:    por %xmm4, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_v2f64_ule_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-16, %esp
+; AVX-32-NEXT:    subl $16, %esp
+; AVX-32-NEXT:    vmovapd 8(%ebp), %xmm3
+; AVX-32-NEXT:    vcmpnlt_uqpd %xmm2, %xmm3, %xmm2
+; AVX-32-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v2f64_ule_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpnlt_uqpd %xmm2, %xmm3, %xmm2
+; AVX-64-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v2f64_ule_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-16, %esp
+; AVX512-32-NEXT:    subl $16, %esp
+; AVX512-32-NEXT:    vcmpngt_uqpd 8(%ebp), %xmm2, %k1
+; AVX512-32-NEXT:    vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v2f64_ule_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpnlt_uqpd %xmm2, %xmm3, %k1
+; AVX512-64-NEXT:    vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <2 x i1> @llvm.experimental.constrained.fcmp.v2f64(
+                                               <2 x double> %f1, <2 x double> %f2, metadata !"ule",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <2 x i1> %cond, <2 x i64> %a, <2 x i64> %b
+  ret <2 x i64> %res
+}
+
+define <2 x i64> @test_v2f64_une_q(<2 x i64> %a, <2 x i64> %b, <2 x double> %f1, <2 x double> %f2) #0 {
+; SSE-32-LABEL: test_v2f64_une_q:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    andl $-16, %esp
+; SSE-32-NEXT:    subl $16, %esp
+; SSE-32-NEXT:    cmpneqpd 8(%ebp), %xmm2
+; SSE-32-NEXT:    andpd %xmm2, %xmm0
+; SSE-32-NEXT:    andnpd %xmm1, %xmm2
+; SSE-32-NEXT:    orpd %xmm2, %xmm0
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_v2f64_une_q:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    cmpneqpd %xmm3, %xmm2
+; SSE-64-NEXT:    andpd %xmm2, %xmm0
+; SSE-64-NEXT:    andnpd %xmm1, %xmm2
+; SSE-64-NEXT:    orpd %xmm2, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_v2f64_une_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-16, %esp
+; AVX-32-NEXT:    subl $16, %esp
+; AVX-32-NEXT:    vcmpneqpd 8(%ebp), %xmm2, %xmm2
+; AVX-32-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v2f64_une_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpneqpd %xmm3, %xmm2, %xmm2
+; AVX-64-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v2f64_une_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-16, %esp
+; AVX512-32-NEXT:    subl $16, %esp
+; AVX512-32-NEXT:    vcmpneqpd 8(%ebp), %xmm2, %k1
+; AVX512-32-NEXT:    vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v2f64_une_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpneqpd %xmm3, %xmm2, %k1
+; AVX512-64-NEXT:    vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <2 x i1> @llvm.experimental.constrained.fcmp.v2f64(
+                                               <2 x double> %f1, <2 x double> %f2, metadata !"une",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <2 x i1> %cond, <2 x i64> %a, <2 x i64> %b
+  ret <2 x i64> %res
+}
+
+define <2 x i64> @test_v2f64_uno_q(<2 x i64> %a, <2 x i64> %b, <2 x double> %f1, <2 x double> %f2) #0 {
+; SSE-32-LABEL: test_v2f64_uno_q:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    andl $-16, %esp
+; SSE-32-NEXT:    subl $16, %esp
+; SSE-32-NEXT:    cmpunordpd 8(%ebp), %xmm2
+; SSE-32-NEXT:    andpd %xmm2, %xmm0
+; SSE-32-NEXT:    andnpd %xmm1, %xmm2
+; SSE-32-NEXT:    orpd %xmm2, %xmm0
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_v2f64_uno_q:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    cmpunordpd %xmm3, %xmm2
+; SSE-64-NEXT:    andpd %xmm2, %xmm0
+; SSE-64-NEXT:    andnpd %xmm1, %xmm2
+; SSE-64-NEXT:    orpd %xmm2, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_v2f64_uno_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-16, %esp
+; AVX-32-NEXT:    subl $16, %esp
+; AVX-32-NEXT:    vcmpunordpd 8(%ebp), %xmm2, %xmm2
+; AVX-32-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v2f64_uno_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpunordpd %xmm3, %xmm2, %xmm2
+; AVX-64-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v2f64_uno_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-16, %esp
+; AVX512-32-NEXT:    subl $16, %esp
+; AVX512-32-NEXT:    vcmpunordpd 8(%ebp), %xmm2, %k1
+; AVX512-32-NEXT:    vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v2f64_uno_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpunordpd %xmm3, %xmm2, %k1
+; AVX512-64-NEXT:    vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <2 x i1> @llvm.experimental.constrained.fcmp.v2f64(
+                                               <2 x double> %f1, <2 x double> %f2, metadata !"uno",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <2 x i1> %cond, <2 x i64> %a, <2 x i64> %b
+  ret <2 x i64> %res
+}
+
+define <4 x i32> @test_v4f32_oeq_s(<4 x i32> %a, <4 x i32> %b, <4 x float> %f1, <4 x float> %f2) #0 {
+; SSE-32-LABEL: test_v4f32_oeq_s:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    andl $-16, %esp
+; SSE-32-NEXT:    subl $16, %esp
+; SSE-32-NEXT:    movaps 8(%ebp), %xmm3
+; SSE-32-NEXT:    movaps %xmm2, %xmm4
+; SSE-32-NEXT:    cmpltps %xmm3, %xmm4
+; SSE-32-NEXT:    cmpeqps %xmm3, %xmm2
+; SSE-32-NEXT:    andps %xmm2, %xmm0
+; SSE-32-NEXT:    andnps %xmm1, %xmm2
+; SSE-32-NEXT:    orps %xmm2, %xmm0
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_v4f32_oeq_s:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movaps %xmm2, %xmm4
+; SSE-64-NEXT:    cmpltps %xmm3, %xmm4
+; SSE-64-NEXT:    cmpeqps %xmm3, %xmm2
+; SSE-64-NEXT:    andps %xmm2, %xmm0
+; SSE-64-NEXT:    andnps %xmm1, %xmm2
+; SSE-64-NEXT:    orps %xmm2, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_v4f32_oeq_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-16, %esp
+; AVX-32-NEXT:    subl $16, %esp
+; AVX-32-NEXT:    vcmpeq_osps 8(%ebp), %xmm2, %xmm2
+; AVX-32-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v4f32_oeq_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpeq_osps %xmm3, %xmm2, %xmm2
+; AVX-64-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v4f32_oeq_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-16, %esp
+; AVX512-32-NEXT:    subl $16, %esp
+; AVX512-32-NEXT:    vcmpeq_osps 8(%ebp), %xmm2, %k1
+; AVX512-32-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v4f32_oeq_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpeq_osps %xmm3, %xmm2, %k1
+; AVX512-64-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <4 x i1> @llvm.experimental.constrained.fcmps.v4f32(
+                                               <4 x float> %f1, <4 x float> %f2, metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <4 x i1> %cond, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_v4f32_ogt_s(<4 x i32> %a, <4 x i32> %b, <4 x float> %f1, <4 x float> %f2) #0 {
+; SSE-32-LABEL: test_v4f32_ogt_s:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    andl $-16, %esp
+; SSE-32-NEXT:    subl $16, %esp
+; SSE-32-NEXT:    movaps 8(%ebp), %xmm3
+; SSE-32-NEXT:    cmpltps %xmm2, %xmm3
+; SSE-32-NEXT:    andps %xmm3, %xmm0
+; SSE-32-NEXT:    andnps %xmm1, %xmm3
+; SSE-32-NEXT:    orps %xmm3, %xmm0
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_v4f32_ogt_s:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    cmpltps %xmm2, %xmm3
+; SSE-64-NEXT:    andps %xmm3, %xmm0
+; SSE-64-NEXT:    andnps %xmm1, %xmm3
+; SSE-64-NEXT:    orps %xmm3, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_v4f32_ogt_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-16, %esp
+; AVX-32-NEXT:    subl $16, %esp
+; AVX-32-NEXT:    vmovaps 8(%ebp), %xmm3
+; AVX-32-NEXT:    vcmpltps %xmm2, %xmm3, %xmm2
+; AVX-32-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v4f32_ogt_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpltps %xmm2, %xmm3, %xmm2
+; AVX-64-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v4f32_ogt_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-16, %esp
+; AVX512-32-NEXT:    subl $16, %esp
+; AVX512-32-NEXT:    vcmpgtps 8(%ebp), %xmm2, %k1
+; AVX512-32-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v4f32_ogt_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpltps %xmm2, %xmm3, %k1
+; AVX512-64-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <4 x i1> @llvm.experimental.constrained.fcmps.v4f32(
+                                               <4 x float> %f1, <4 x float> %f2, metadata !"ogt",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <4 x i1> %cond, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_v4f32_oge_s(<4 x i32> %a, <4 x i32> %b, <4 x float> %f1, <4 x float> %f2) #0 {
+; SSE-32-LABEL: test_v4f32_oge_s:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    andl $-16, %esp
+; SSE-32-NEXT:    subl $16, %esp
+; SSE-32-NEXT:    movaps 8(%ebp), %xmm3
+; SSE-32-NEXT:    cmpleps %xmm2, %xmm3
+; SSE-32-NEXT:    andps %xmm3, %xmm0
+; SSE-32-NEXT:    andnps %xmm1, %xmm3
+; SSE-32-NEXT:    orps %xmm3, %xmm0
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_v4f32_oge_s:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    cmpleps %xmm2, %xmm3
+; SSE-64-NEXT:    andps %xmm3, %xmm0
+; SSE-64-NEXT:    andnps %xmm1, %xmm3
+; SSE-64-NEXT:    orps %xmm3, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_v4f32_oge_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-16, %esp
+; AVX-32-NEXT:    subl $16, %esp
+; AVX-32-NEXT:    vmovaps 8(%ebp), %xmm3
+; AVX-32-NEXT:    vcmpleps %xmm2, %xmm3, %xmm2
+; AVX-32-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v4f32_oge_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpleps %xmm2, %xmm3, %xmm2
+; AVX-64-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v4f32_oge_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-16, %esp
+; AVX512-32-NEXT:    subl $16, %esp
+; AVX512-32-NEXT:    vcmpgeps 8(%ebp), %xmm2, %k1
+; AVX512-32-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v4f32_oge_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpleps %xmm2, %xmm3, %k1
+; AVX512-64-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <4 x i1> @llvm.experimental.constrained.fcmps.v4f32(
+                                               <4 x float> %f1, <4 x float> %f2, metadata !"oge",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <4 x i1> %cond, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_v4f32_olt_s(<4 x i32> %a, <4 x i32> %b, <4 x float> %f1, <4 x float> %f2) #0 {
+; SSE-32-LABEL: test_v4f32_olt_s:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    andl $-16, %esp
+; SSE-32-NEXT:    subl $16, %esp
+; SSE-32-NEXT:    cmpltps 8(%ebp), %xmm2
+; SSE-32-NEXT:    andps %xmm2, %xmm0
+; SSE-32-NEXT:    andnps %xmm1, %xmm2
+; SSE-32-NEXT:    orps %xmm2, %xmm0
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_v4f32_olt_s:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    cmpltps %xmm3, %xmm2
+; SSE-64-NEXT:    andps %xmm2, %xmm0
+; SSE-64-NEXT:    andnps %xmm1, %xmm2
+; SSE-64-NEXT:    orps %xmm2, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_v4f32_olt_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-16, %esp
+; AVX-32-NEXT:    subl $16, %esp
+; AVX-32-NEXT:    vcmpltps 8(%ebp), %xmm2, %xmm2
+; AVX-32-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v4f32_olt_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpltps %xmm3, %xmm2, %xmm2
+; AVX-64-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v4f32_olt_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-16, %esp
+; AVX512-32-NEXT:    subl $16, %esp
+; AVX512-32-NEXT:    vcmpltps 8(%ebp), %xmm2, %k1
+; AVX512-32-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v4f32_olt_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpltps %xmm3, %xmm2, %k1
+; AVX512-64-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <4 x i1> @llvm.experimental.constrained.fcmps.v4f32(
+                                               <4 x float> %f1, <4 x float> %f2, metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <4 x i1> %cond, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_v4f32_ole_s(<4 x i32> %a, <4 x i32> %b, <4 x float> %f1, <4 x float> %f2) #0 {
+; SSE-32-LABEL: test_v4f32_ole_s:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    andl $-16, %esp
+; SSE-32-NEXT:    subl $16, %esp
+; SSE-32-NEXT:    cmpleps 8(%ebp), %xmm2
+; SSE-32-NEXT:    andps %xmm2, %xmm0
+; SSE-32-NEXT:    andnps %xmm1, %xmm2
+; SSE-32-NEXT:    orps %xmm2, %xmm0
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_v4f32_ole_s:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    cmpleps %xmm3, %xmm2
+; SSE-64-NEXT:    andps %xmm2, %xmm0
+; SSE-64-NEXT:    andnps %xmm1, %xmm2
+; SSE-64-NEXT:    orps %xmm2, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_v4f32_ole_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-16, %esp
+; AVX-32-NEXT:    subl $16, %esp
+; AVX-32-NEXT:    vcmpleps 8(%ebp), %xmm2, %xmm2
+; AVX-32-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v4f32_ole_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpleps %xmm3, %xmm2, %xmm2
+; AVX-64-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v4f32_ole_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-16, %esp
+; AVX512-32-NEXT:    subl $16, %esp
+; AVX512-32-NEXT:    vcmpleps 8(%ebp), %xmm2, %k1
+; AVX512-32-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v4f32_ole_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpleps %xmm3, %xmm2, %k1
+; AVX512-64-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <4 x i1> @llvm.experimental.constrained.fcmps.v4f32(
+                                               <4 x float> %f1, <4 x float> %f2, metadata !"ole",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <4 x i1> %cond, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_v4f32_one_s(<4 x i32> %a, <4 x i32> %b, <4 x float> %f1, <4 x float> %f2) #0 {
+; SSE-32-LABEL: test_v4f32_one_s:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    andl $-16, %esp
+; SSE-32-NEXT:    subl $16, %esp
+; SSE-32-NEXT:    movaps 8(%ebp), %xmm3
+; SSE-32-NEXT:    movaps %xmm2, %xmm4
+; SSE-32-NEXT:    cmpltps %xmm3, %xmm4
+; SSE-32-NEXT:    movaps %xmm2, %xmm4
+; SSE-32-NEXT:    cmpneqps %xmm3, %xmm4
+; SSE-32-NEXT:    cmpordps %xmm3, %xmm2
+; SSE-32-NEXT:    andps %xmm4, %xmm2
+; SSE-32-NEXT:    andps %xmm2, %xmm0
+; SSE-32-NEXT:    andnps %xmm1, %xmm2
+; SSE-32-NEXT:    orps %xmm2, %xmm0
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_v4f32_one_s:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movaps %xmm2, %xmm4
+; SSE-64-NEXT:    cmpltps %xmm3, %xmm4
+; SSE-64-NEXT:    movaps %xmm2, %xmm4
+; SSE-64-NEXT:    cmpneqps %xmm3, %xmm4
+; SSE-64-NEXT:    cmpordps %xmm3, %xmm2
+; SSE-64-NEXT:    andps %xmm4, %xmm2
+; SSE-64-NEXT:    andps %xmm2, %xmm0
+; SSE-64-NEXT:    andnps %xmm1, %xmm2
+; SSE-64-NEXT:    orps %xmm2, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_v4f32_one_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-16, %esp
+; AVX-32-NEXT:    subl $16, %esp
+; AVX-32-NEXT:    vcmpneq_osps 8(%ebp), %xmm2, %xmm2
+; AVX-32-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v4f32_one_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpneq_osps %xmm3, %xmm2, %xmm2
+; AVX-64-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v4f32_one_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-16, %esp
+; AVX512-32-NEXT:    subl $16, %esp
+; AVX512-32-NEXT:    vcmpneq_osps 8(%ebp), %xmm2, %k1
+; AVX512-32-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v4f32_one_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpneq_osps %xmm3, %xmm2, %k1
+; AVX512-64-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <4 x i1> @llvm.experimental.constrained.fcmps.v4f32(
+                                               <4 x float> %f1, <4 x float> %f2, metadata !"one",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <4 x i1> %cond, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_v4f32_ord_s(<4 x i32> %a, <4 x i32> %b, <4 x float> %f1, <4 x float> %f2) #0 {
+; SSE-32-LABEL: test_v4f32_ord_s:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    andl $-16, %esp
+; SSE-32-NEXT:    subl $16, %esp
+; SSE-32-NEXT:    movaps 8(%ebp), %xmm3
+; SSE-32-NEXT:    movaps %xmm2, %xmm4
+; SSE-32-NEXT:    cmpltps %xmm3, %xmm4
+; SSE-32-NEXT:    cmpordps %xmm3, %xmm2
+; SSE-32-NEXT:    andps %xmm2, %xmm0
+; SSE-32-NEXT:    andnps %xmm1, %xmm2
+; SSE-32-NEXT:    orps %xmm2, %xmm0
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_v4f32_ord_s:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movaps %xmm2, %xmm4
+; SSE-64-NEXT:    cmpltps %xmm3, %xmm4
+; SSE-64-NEXT:    cmpordps %xmm3, %xmm2
+; SSE-64-NEXT:    andps %xmm2, %xmm0
+; SSE-64-NEXT:    andnps %xmm1, %xmm2
+; SSE-64-NEXT:    orps %xmm2, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_v4f32_ord_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-16, %esp
+; AVX-32-NEXT:    subl $16, %esp
+; AVX-32-NEXT:    vcmpord_sps 8(%ebp), %xmm2, %xmm2
+; AVX-32-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v4f32_ord_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpord_sps %xmm3, %xmm2, %xmm2
+; AVX-64-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v4f32_ord_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-16, %esp
+; AVX512-32-NEXT:    subl $16, %esp
+; AVX512-32-NEXT:    vcmpord_sps 8(%ebp), %xmm2, %k1
+; AVX512-32-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v4f32_ord_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpord_sps %xmm3, %xmm2, %k1
+; AVX512-64-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <4 x i1> @llvm.experimental.constrained.fcmps.v4f32(
+                                               <4 x float> %f1, <4 x float> %f2, metadata !"ord",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <4 x i1> %cond, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_v4f32_ueq_s(<4 x i32> %a, <4 x i32> %b, <4 x float> %f1, <4 x float> %f2) #0 {
+; SSE-32-LABEL: test_v4f32_ueq_s:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    andl $-16, %esp
+; SSE-32-NEXT:    subl $16, %esp
+; SSE-32-NEXT:    movaps 8(%ebp), %xmm3
+; SSE-32-NEXT:    movaps %xmm2, %xmm4
+; SSE-32-NEXT:    cmpltps %xmm3, %xmm4
+; SSE-32-NEXT:    movaps %xmm2, %xmm4
+; SSE-32-NEXT:    cmpeqps %xmm3, %xmm4
+; SSE-32-NEXT:    cmpunordps %xmm3, %xmm2
+; SSE-32-NEXT:    orps %xmm4, %xmm2
+; SSE-32-NEXT:    andps %xmm2, %xmm0
+; SSE-32-NEXT:    andnps %xmm1, %xmm2
+; SSE-32-NEXT:    orps %xmm2, %xmm0
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_v4f32_ueq_s:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movaps %xmm2, %xmm4
+; SSE-64-NEXT:    cmpltps %xmm3, %xmm4
+; SSE-64-NEXT:    movaps %xmm2, %xmm4
+; SSE-64-NEXT:    cmpeqps %xmm3, %xmm4
+; SSE-64-NEXT:    cmpunordps %xmm3, %xmm2
+; SSE-64-NEXT:    orps %xmm4, %xmm2
+; SSE-64-NEXT:    andps %xmm2, %xmm0
+; SSE-64-NEXT:    andnps %xmm1, %xmm2
+; SSE-64-NEXT:    orps %xmm2, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_v4f32_ueq_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-16, %esp
+; AVX-32-NEXT:    subl $16, %esp
+; AVX-32-NEXT:    vcmpeq_usps 8(%ebp), %xmm2, %xmm2
+; AVX-32-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v4f32_ueq_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpeq_usps %xmm3, %xmm2, %xmm2
+; AVX-64-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v4f32_ueq_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-16, %esp
+; AVX512-32-NEXT:    subl $16, %esp
+; AVX512-32-NEXT:    vcmpeq_usps 8(%ebp), %xmm2, %k1
+; AVX512-32-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v4f32_ueq_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpeq_usps %xmm3, %xmm2, %k1
+; AVX512-64-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <4 x i1> @llvm.experimental.constrained.fcmps.v4f32(
+                                               <4 x float> %f1, <4 x float> %f2, metadata !"ueq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <4 x i1> %cond, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_v4f32_ugt_s(<4 x i32> %a, <4 x i32> %b, <4 x float> %f1, <4 x float> %f2) #0 {
+; SSE-32-LABEL: test_v4f32_ugt_s:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    andl $-16, %esp
+; SSE-32-NEXT:    subl $16, %esp
+; SSE-32-NEXT:    cmpnleps 8(%ebp), %xmm2
+; SSE-32-NEXT:    andps %xmm2, %xmm0
+; SSE-32-NEXT:    andnps %xmm1, %xmm2
+; SSE-32-NEXT:    orps %xmm2, %xmm0
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_v4f32_ugt_s:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    cmpnleps %xmm3, %xmm2
+; SSE-64-NEXT:    andps %xmm2, %xmm0
+; SSE-64-NEXT:    andnps %xmm1, %xmm2
+; SSE-64-NEXT:    orps %xmm2, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_v4f32_ugt_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-16, %esp
+; AVX-32-NEXT:    subl $16, %esp
+; AVX-32-NEXT:    vcmpnleps 8(%ebp), %xmm2, %xmm2
+; AVX-32-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v4f32_ugt_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpnleps %xmm3, %xmm2, %xmm2
+; AVX-64-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v4f32_ugt_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-16, %esp
+; AVX512-32-NEXT:    subl $16, %esp
+; AVX512-32-NEXT:    vcmpnleps 8(%ebp), %xmm2, %k1
+; AVX512-32-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v4f32_ugt_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpnleps %xmm3, %xmm2, %k1
+; AVX512-64-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <4 x i1> @llvm.experimental.constrained.fcmps.v4f32(
+                                               <4 x float> %f1, <4 x float> %f2, metadata !"ugt",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <4 x i1> %cond, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_v4f32_uge_s(<4 x i32> %a, <4 x i32> %b, <4 x float> %f1, <4 x float> %f2) #0 {
+; SSE-32-LABEL: test_v4f32_uge_s:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    andl $-16, %esp
+; SSE-32-NEXT:    subl $16, %esp
+; SSE-32-NEXT:    cmpnltps 8(%ebp), %xmm2
+; SSE-32-NEXT:    andps %xmm2, %xmm0
+; SSE-32-NEXT:    andnps %xmm1, %xmm2
+; SSE-32-NEXT:    orps %xmm2, %xmm0
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_v4f32_uge_s:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    cmpnltps %xmm3, %xmm2
+; SSE-64-NEXT:    andps %xmm2, %xmm0
+; SSE-64-NEXT:    andnps %xmm1, %xmm2
+; SSE-64-NEXT:    orps %xmm2, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_v4f32_uge_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-16, %esp
+; AVX-32-NEXT:    subl $16, %esp
+; AVX-32-NEXT:    vcmpnltps 8(%ebp), %xmm2, %xmm2
+; AVX-32-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v4f32_uge_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpnltps %xmm3, %xmm2, %xmm2
+; AVX-64-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v4f32_uge_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-16, %esp
+; AVX512-32-NEXT:    subl $16, %esp
+; AVX512-32-NEXT:    vcmpnltps 8(%ebp), %xmm2, %k1
+; AVX512-32-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v4f32_uge_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpnltps %xmm3, %xmm2, %k1
+; AVX512-64-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <4 x i1> @llvm.experimental.constrained.fcmps.v4f32(
+                                               <4 x float> %f1, <4 x float> %f2, metadata !"uge",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <4 x i1> %cond, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_v4f32_ult_s(<4 x i32> %a, <4 x i32> %b, <4 x float> %f1, <4 x float> %f2) #0 {
+; SSE-32-LABEL: test_v4f32_ult_s:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    andl $-16, %esp
+; SSE-32-NEXT:    subl $16, %esp
+; SSE-32-NEXT:    movaps 8(%ebp), %xmm3
+; SSE-32-NEXT:    cmpnleps %xmm2, %xmm3
+; SSE-32-NEXT:    andps %xmm3, %xmm0
+; SSE-32-NEXT:    andnps %xmm1, %xmm3
+; SSE-32-NEXT:    orps %xmm3, %xmm0
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_v4f32_ult_s:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    cmpnleps %xmm2, %xmm3
+; SSE-64-NEXT:    andps %xmm3, %xmm0
+; SSE-64-NEXT:    andnps %xmm1, %xmm3
+; SSE-64-NEXT:    orps %xmm3, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_v4f32_ult_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-16, %esp
+; AVX-32-NEXT:    subl $16, %esp
+; AVX-32-NEXT:    vmovaps 8(%ebp), %xmm3
+; AVX-32-NEXT:    vcmpnleps %xmm2, %xmm3, %xmm2
+; AVX-32-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v4f32_ult_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpnleps %xmm2, %xmm3, %xmm2
+; AVX-64-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v4f32_ult_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-16, %esp
+; AVX512-32-NEXT:    subl $16, %esp
+; AVX512-32-NEXT:    vcmpngeps 8(%ebp), %xmm2, %k1
+; AVX512-32-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v4f32_ult_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpnleps %xmm2, %xmm3, %k1
+; AVX512-64-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <4 x i1> @llvm.experimental.constrained.fcmps.v4f32(
+                                               <4 x float> %f1, <4 x float> %f2, metadata !"ult",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <4 x i1> %cond, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_v4f32_ule_s(<4 x i32> %a, <4 x i32> %b, <4 x float> %f1, <4 x float> %f2) #0 {
+; SSE-32-LABEL: test_v4f32_ule_s:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    andl $-16, %esp
+; SSE-32-NEXT:    subl $16, %esp
+; SSE-32-NEXT:    movaps 8(%ebp), %xmm3
+; SSE-32-NEXT:    cmpnltps %xmm2, %xmm3
+; SSE-32-NEXT:    andps %xmm3, %xmm0
+; SSE-32-NEXT:    andnps %xmm1, %xmm3
+; SSE-32-NEXT:    orps %xmm3, %xmm0
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_v4f32_ule_s:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    cmpnltps %xmm2, %xmm3
+; SSE-64-NEXT:    andps %xmm3, %xmm0
+; SSE-64-NEXT:    andnps %xmm1, %xmm3
+; SSE-64-NEXT:    orps %xmm3, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_v4f32_ule_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-16, %esp
+; AVX-32-NEXT:    subl $16, %esp
+; AVX-32-NEXT:    vmovaps 8(%ebp), %xmm3
+; AVX-32-NEXT:    vcmpnltps %xmm2, %xmm3, %xmm2
+; AVX-32-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v4f32_ule_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpnltps %xmm2, %xmm3, %xmm2
+; AVX-64-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v4f32_ule_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-16, %esp
+; AVX512-32-NEXT:    subl $16, %esp
+; AVX512-32-NEXT:    vcmpngtps 8(%ebp), %xmm2, %k1
+; AVX512-32-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v4f32_ule_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpnltps %xmm2, %xmm3, %k1
+; AVX512-64-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <4 x i1> @llvm.experimental.constrained.fcmps.v4f32(
+                                               <4 x float> %f1, <4 x float> %f2, metadata !"ule",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <4 x i1> %cond, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_v4f32_une_s(<4 x i32> %a, <4 x i32> %b, <4 x float> %f1, <4 x float> %f2) #0 {
+; SSE-32-LABEL: test_v4f32_une_s:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    andl $-16, %esp
+; SSE-32-NEXT:    subl $16, %esp
+; SSE-32-NEXT:    movaps 8(%ebp), %xmm3
+; SSE-32-NEXT:    movaps %xmm2, %xmm4
+; SSE-32-NEXT:    cmpltps %xmm3, %xmm4
+; SSE-32-NEXT:    cmpneqps %xmm3, %xmm2
+; SSE-32-NEXT:    andps %xmm2, %xmm0
+; SSE-32-NEXT:    andnps %xmm1, %xmm2
+; SSE-32-NEXT:    orps %xmm2, %xmm0
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_v4f32_une_s:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movaps %xmm2, %xmm4
+; SSE-64-NEXT:    cmpltps %xmm3, %xmm4
+; SSE-64-NEXT:    cmpneqps %xmm3, %xmm2
+; SSE-64-NEXT:    andps %xmm2, %xmm0
+; SSE-64-NEXT:    andnps %xmm1, %xmm2
+; SSE-64-NEXT:    orps %xmm2, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_v4f32_une_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-16, %esp
+; AVX-32-NEXT:    subl $16, %esp
+; AVX-32-NEXT:    vcmpneq_usps 8(%ebp), %xmm2, %xmm2
+; AVX-32-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v4f32_une_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpneq_usps %xmm3, %xmm2, %xmm2
+; AVX-64-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v4f32_une_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-16, %esp
+; AVX512-32-NEXT:    subl $16, %esp
+; AVX512-32-NEXT:    vcmpneq_usps 8(%ebp), %xmm2, %k1
+; AVX512-32-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v4f32_une_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpneq_usps %xmm3, %xmm2, %k1
+; AVX512-64-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <4 x i1> @llvm.experimental.constrained.fcmps.v4f32(
+                                               <4 x float> %f1, <4 x float> %f2, metadata !"une",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <4 x i1> %cond, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_v4f32_uno_s(<4 x i32> %a, <4 x i32> %b, <4 x float> %f1, <4 x float> %f2) #0 {
+; SSE-32-LABEL: test_v4f32_uno_s:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    andl $-16, %esp
+; SSE-32-NEXT:    subl $16, %esp
+; SSE-32-NEXT:    movaps 8(%ebp), %xmm3
+; SSE-32-NEXT:    movaps %xmm2, %xmm4
+; SSE-32-NEXT:    cmpltps %xmm3, %xmm4
+; SSE-32-NEXT:    cmpunordps %xmm3, %xmm2
+; SSE-32-NEXT:    andps %xmm2, %xmm0
+; SSE-32-NEXT:    andnps %xmm1, %xmm2
+; SSE-32-NEXT:    orps %xmm2, %xmm0
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_v4f32_uno_s:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movaps %xmm2, %xmm4
+; SSE-64-NEXT:    cmpltps %xmm3, %xmm4
+; SSE-64-NEXT:    cmpunordps %xmm3, %xmm2
+; SSE-64-NEXT:    andps %xmm2, %xmm0
+; SSE-64-NEXT:    andnps %xmm1, %xmm2
+; SSE-64-NEXT:    orps %xmm2, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_v4f32_uno_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-16, %esp
+; AVX-32-NEXT:    subl $16, %esp
+; AVX-32-NEXT:    vcmpunord_sps 8(%ebp), %xmm2, %xmm2
+; AVX-32-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v4f32_uno_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpunord_sps %xmm3, %xmm2, %xmm2
+; AVX-64-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v4f32_uno_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-16, %esp
+; AVX512-32-NEXT:    subl $16, %esp
+; AVX512-32-NEXT:    vcmpunord_sps 8(%ebp), %xmm2, %k1
+; AVX512-32-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v4f32_uno_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpunord_sps %xmm3, %xmm2, %k1
+; AVX512-64-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <4 x i1> @llvm.experimental.constrained.fcmps.v4f32(
+                                               <4 x float> %f1, <4 x float> %f2, metadata !"uno",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <4 x i1> %cond, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %res
+}
+
+define <2 x i64> @test_v2f64_oeq_s(<2 x i64> %a, <2 x i64> %b, <2 x double> %f1, <2 x double> %f2) #0 {
+; SSE-32-LABEL: test_v2f64_oeq_s:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    andl $-16, %esp
+; SSE-32-NEXT:    subl $16, %esp
+; SSE-32-NEXT:    movapd 8(%ebp), %xmm3
+; SSE-32-NEXT:    movapd %xmm2, %xmm4
+; SSE-32-NEXT:    cmpltpd %xmm3, %xmm4
+; SSE-32-NEXT:    cmpeqpd %xmm3, %xmm2
+; SSE-32-NEXT:    andpd %xmm2, %xmm0
+; SSE-32-NEXT:    andnpd %xmm1, %xmm2
+; SSE-32-NEXT:    orpd %xmm2, %xmm0
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_v2f64_oeq_s:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movapd %xmm2, %xmm4
+; SSE-64-NEXT:    cmpltpd %xmm3, %xmm4
+; SSE-64-NEXT:    cmpeqpd %xmm3, %xmm2
+; SSE-64-NEXT:    andpd %xmm2, %xmm0
+; SSE-64-NEXT:    andnpd %xmm1, %xmm2
+; SSE-64-NEXT:    orpd %xmm2, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_v2f64_oeq_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-16, %esp
+; AVX-32-NEXT:    subl $16, %esp
+; AVX-32-NEXT:    vcmpeq_ospd 8(%ebp), %xmm2, %xmm2
+; AVX-32-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v2f64_oeq_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpeq_ospd %xmm3, %xmm2, %xmm2
+; AVX-64-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v2f64_oeq_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-16, %esp
+; AVX512-32-NEXT:    subl $16, %esp
+; AVX512-32-NEXT:    vcmpeq_ospd 8(%ebp), %xmm2, %k1
+; AVX512-32-NEXT:    vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v2f64_oeq_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpeq_ospd %xmm3, %xmm2, %k1
+; AVX512-64-NEXT:    vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <2 x i1> @llvm.experimental.constrained.fcmps.v2f64(
+                                               <2 x double> %f1, <2 x double> %f2, metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <2 x i1> %cond, <2 x i64> %a, <2 x i64> %b
+  ret <2 x i64> %res
+}
+
+define <2 x i64> @test_v2f64_ogt_s(<2 x i64> %a, <2 x i64> %b, <2 x double> %f1, <2 x double> %f2) #0 {
+; SSE-32-LABEL: test_v2f64_ogt_s:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    andl $-16, %esp
+; SSE-32-NEXT:    subl $16, %esp
+; SSE-32-NEXT:    movapd 8(%ebp), %xmm3
+; SSE-32-NEXT:    cmpltpd %xmm2, %xmm3
+; SSE-32-NEXT:    andpd %xmm3, %xmm0
+; SSE-32-NEXT:    andnpd %xmm1, %xmm3
+; SSE-32-NEXT:    orpd %xmm3, %xmm0
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_v2f64_ogt_s:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    cmpltpd %xmm2, %xmm3
+; SSE-64-NEXT:    andpd %xmm3, %xmm0
+; SSE-64-NEXT:    andnpd %xmm1, %xmm3
+; SSE-64-NEXT:    orpd %xmm3, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_v2f64_ogt_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-16, %esp
+; AVX-32-NEXT:    subl $16, %esp
+; AVX-32-NEXT:    vmovapd 8(%ebp), %xmm3
+; AVX-32-NEXT:    vcmpltpd %xmm2, %xmm3, %xmm2
+; AVX-32-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v2f64_ogt_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpltpd %xmm2, %xmm3, %xmm2
+; AVX-64-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v2f64_ogt_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-16, %esp
+; AVX512-32-NEXT:    subl $16, %esp
+; AVX512-32-NEXT:    vcmpgtpd 8(%ebp), %xmm2, %k1
+; AVX512-32-NEXT:    vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v2f64_ogt_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpltpd %xmm2, %xmm3, %k1
+; AVX512-64-NEXT:    vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <2 x i1> @llvm.experimental.constrained.fcmps.v2f64(
+                                               <2 x double> %f1, <2 x double> %f2, metadata !"ogt",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <2 x i1> %cond, <2 x i64> %a, <2 x i64> %b
+  ret <2 x i64> %res
+}
+
+define <2 x i64> @test_v2f64_oge_s(<2 x i64> %a, <2 x i64> %b, <2 x double> %f1, <2 x double> %f2) #0 {
+; SSE-32-LABEL: test_v2f64_oge_s:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    andl $-16, %esp
+; SSE-32-NEXT:    subl $16, %esp
+; SSE-32-NEXT:    movapd 8(%ebp), %xmm3
+; SSE-32-NEXT:    cmplepd %xmm2, %xmm3
+; SSE-32-NEXT:    andpd %xmm3, %xmm0
+; SSE-32-NEXT:    andnpd %xmm1, %xmm3
+; SSE-32-NEXT:    orpd %xmm3, %xmm0
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_v2f64_oge_s:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    cmplepd %xmm2, %xmm3
+; SSE-64-NEXT:    andpd %xmm3, %xmm0
+; SSE-64-NEXT:    andnpd %xmm1, %xmm3
+; SSE-64-NEXT:    orpd %xmm3, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_v2f64_oge_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-16, %esp
+; AVX-32-NEXT:    subl $16, %esp
+; AVX-32-NEXT:    vmovapd 8(%ebp), %xmm3
+; AVX-32-NEXT:    vcmplepd %xmm2, %xmm3, %xmm2
+; AVX-32-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v2f64_oge_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmplepd %xmm2, %xmm3, %xmm2
+; AVX-64-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v2f64_oge_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-16, %esp
+; AVX512-32-NEXT:    subl $16, %esp
+; AVX512-32-NEXT:    vcmpgepd 8(%ebp), %xmm2, %k1
+; AVX512-32-NEXT:    vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v2f64_oge_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmplepd %xmm2, %xmm3, %k1
+; AVX512-64-NEXT:    vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <2 x i1> @llvm.experimental.constrained.fcmps.v2f64(
+                                               <2 x double> %f1, <2 x double> %f2, metadata !"oge",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <2 x i1> %cond, <2 x i64> %a, <2 x i64> %b
+  ret <2 x i64> %res
+}
+
+define <2 x i64> @test_v2f64_olt_s(<2 x i64> %a, <2 x i64> %b, <2 x double> %f1, <2 x double> %f2) #0 {
+; SSE-32-LABEL: test_v2f64_olt_s:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    andl $-16, %esp
+; SSE-32-NEXT:    subl $16, %esp
+; SSE-32-NEXT:    cmpltpd 8(%ebp), %xmm2
+; SSE-32-NEXT:    andpd %xmm2, %xmm0
+; SSE-32-NEXT:    andnpd %xmm1, %xmm2
+; SSE-32-NEXT:    orpd %xmm2, %xmm0
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_v2f64_olt_s:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    cmpltpd %xmm3, %xmm2
+; SSE-64-NEXT:    andpd %xmm2, %xmm0
+; SSE-64-NEXT:    andnpd %xmm1, %xmm2
+; SSE-64-NEXT:    orpd %xmm2, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_v2f64_olt_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-16, %esp
+; AVX-32-NEXT:    subl $16, %esp
+; AVX-32-NEXT:    vcmpltpd 8(%ebp), %xmm2, %xmm2
+; AVX-32-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v2f64_olt_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpltpd %xmm3, %xmm2, %xmm2
+; AVX-64-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v2f64_olt_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-16, %esp
+; AVX512-32-NEXT:    subl $16, %esp
+; AVX512-32-NEXT:    vcmpltpd 8(%ebp), %xmm2, %k1
+; AVX512-32-NEXT:    vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v2f64_olt_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpltpd %xmm3, %xmm2, %k1
+; AVX512-64-NEXT:    vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <2 x i1> @llvm.experimental.constrained.fcmps.v2f64(
+                                               <2 x double> %f1, <2 x double> %f2, metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <2 x i1> %cond, <2 x i64> %a, <2 x i64> %b
+  ret <2 x i64> %res
+}
+
+define <2 x i64> @test_v2f64_ole_s(<2 x i64> %a, <2 x i64> %b, <2 x double> %f1, <2 x double> %f2) #0 {
+; SSE-32-LABEL: test_v2f64_ole_s:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    andl $-16, %esp
+; SSE-32-NEXT:    subl $16, %esp
+; SSE-32-NEXT:    cmplepd 8(%ebp), %xmm2
+; SSE-32-NEXT:    andpd %xmm2, %xmm0
+; SSE-32-NEXT:    andnpd %xmm1, %xmm2
+; SSE-32-NEXT:    orpd %xmm2, %xmm0
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_v2f64_ole_s:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    cmplepd %xmm3, %xmm2
+; SSE-64-NEXT:    andpd %xmm2, %xmm0
+; SSE-64-NEXT:    andnpd %xmm1, %xmm2
+; SSE-64-NEXT:    orpd %xmm2, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_v2f64_ole_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-16, %esp
+; AVX-32-NEXT:    subl $16, %esp
+; AVX-32-NEXT:    vcmplepd 8(%ebp), %xmm2, %xmm2
+; AVX-32-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v2f64_ole_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmplepd %xmm3, %xmm2, %xmm2
+; AVX-64-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v2f64_ole_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-16, %esp
+; AVX512-32-NEXT:    subl $16, %esp
+; AVX512-32-NEXT:    vcmplepd 8(%ebp), %xmm2, %k1
+; AVX512-32-NEXT:    vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v2f64_ole_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmplepd %xmm3, %xmm2, %k1
+; AVX512-64-NEXT:    vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <2 x i1> @llvm.experimental.constrained.fcmps.v2f64(
+                                               <2 x double> %f1, <2 x double> %f2, metadata !"ole",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <2 x i1> %cond, <2 x i64> %a, <2 x i64> %b
+  ret <2 x i64> %res
+}
+
+define <2 x i64> @test_v2f64_one_s(<2 x i64> %a, <2 x i64> %b, <2 x double> %f1, <2 x double> %f2) #0 {
+; SSE-32-LABEL: test_v2f64_one_s:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    andl $-16, %esp
+; SSE-32-NEXT:    subl $16, %esp
+; SSE-32-NEXT:    movapd 8(%ebp), %xmm3
+; SSE-32-NEXT:    movapd %xmm2, %xmm4
+; SSE-32-NEXT:    cmpltpd %xmm3, %xmm4
+; SSE-32-NEXT:    movapd %xmm2, %xmm4
+; SSE-32-NEXT:    cmpneqpd %xmm3, %xmm4
+; SSE-32-NEXT:    cmpordpd %xmm3, %xmm2
+; SSE-32-NEXT:    andpd %xmm4, %xmm2
+; SSE-32-NEXT:    andpd %xmm2, %xmm0
+; SSE-32-NEXT:    andnpd %xmm1, %xmm2
+; SSE-32-NEXT:    orpd %xmm2, %xmm0
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_v2f64_one_s:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movapd %xmm2, %xmm4
+; SSE-64-NEXT:    cmpltpd %xmm3, %xmm4
+; SSE-64-NEXT:    movapd %xmm2, %xmm4
+; SSE-64-NEXT:    cmpneqpd %xmm3, %xmm4
+; SSE-64-NEXT:    cmpordpd %xmm3, %xmm2
+; SSE-64-NEXT:    andpd %xmm4, %xmm2
+; SSE-64-NEXT:    andpd %xmm2, %xmm0
+; SSE-64-NEXT:    andnpd %xmm1, %xmm2
+; SSE-64-NEXT:    orpd %xmm2, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_v2f64_one_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-16, %esp
+; AVX-32-NEXT:    subl $16, %esp
+; AVX-32-NEXT:    vcmpneq_ospd 8(%ebp), %xmm2, %xmm2
+; AVX-32-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v2f64_one_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpneq_ospd %xmm3, %xmm2, %xmm2
+; AVX-64-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v2f64_one_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-16, %esp
+; AVX512-32-NEXT:    subl $16, %esp
+; AVX512-32-NEXT:    vcmpneq_ospd 8(%ebp), %xmm2, %k1
+; AVX512-32-NEXT:    vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v2f64_one_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpneq_ospd %xmm3, %xmm2, %k1
+; AVX512-64-NEXT:    vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <2 x i1> @llvm.experimental.constrained.fcmps.v2f64(
+                                               <2 x double> %f1, <2 x double> %f2, metadata !"one",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <2 x i1> %cond, <2 x i64> %a, <2 x i64> %b
+  ret <2 x i64> %res
+}
+
+define <2 x i64> @test_v2f64_ord_s(<2 x i64> %a, <2 x i64> %b, <2 x double> %f1, <2 x double> %f2) #0 {
+; SSE-32-LABEL: test_v2f64_ord_s:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    andl $-16, %esp
+; SSE-32-NEXT:    subl $16, %esp
+; SSE-32-NEXT:    movapd 8(%ebp), %xmm3
+; SSE-32-NEXT:    movapd %xmm2, %xmm4
+; SSE-32-NEXT:    cmpltpd %xmm3, %xmm4
+; SSE-32-NEXT:    cmpordpd %xmm3, %xmm2
+; SSE-32-NEXT:    andpd %xmm2, %xmm0
+; SSE-32-NEXT:    andnpd %xmm1, %xmm2
+; SSE-32-NEXT:    orpd %xmm2, %xmm0
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_v2f64_ord_s:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movapd %xmm2, %xmm4
+; SSE-64-NEXT:    cmpltpd %xmm3, %xmm4
+; SSE-64-NEXT:    cmpordpd %xmm3, %xmm2
+; SSE-64-NEXT:    andpd %xmm2, %xmm0
+; SSE-64-NEXT:    andnpd %xmm1, %xmm2
+; SSE-64-NEXT:    orpd %xmm2, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_v2f64_ord_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-16, %esp
+; AVX-32-NEXT:    subl $16, %esp
+; AVX-32-NEXT:    vcmpord_spd 8(%ebp), %xmm2, %xmm2
+; AVX-32-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v2f64_ord_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpord_spd %xmm3, %xmm2, %xmm2
+; AVX-64-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v2f64_ord_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-16, %esp
+; AVX512-32-NEXT:    subl $16, %esp
+; AVX512-32-NEXT:    vcmpord_spd 8(%ebp), %xmm2, %k1
+; AVX512-32-NEXT:    vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v2f64_ord_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpord_spd %xmm3, %xmm2, %k1
+; AVX512-64-NEXT:    vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <2 x i1> @llvm.experimental.constrained.fcmps.v2f64(
+                                               <2 x double> %f1, <2 x double> %f2, metadata !"ord",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <2 x i1> %cond, <2 x i64> %a, <2 x i64> %b
+  ret <2 x i64> %res
+}
+
+define <2 x i64> @test_v2f64_ueq_s(<2 x i64> %a, <2 x i64> %b, <2 x double> %f1, <2 x double> %f2) #0 {
+; SSE-32-LABEL: test_v2f64_ueq_s:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    andl $-16, %esp
+; SSE-32-NEXT:    subl $16, %esp
+; SSE-32-NEXT:    movapd 8(%ebp), %xmm3
+; SSE-32-NEXT:    movapd %xmm2, %xmm4
+; SSE-32-NEXT:    cmpltpd %xmm3, %xmm4
+; SSE-32-NEXT:    movapd %xmm2, %xmm4
+; SSE-32-NEXT:    cmpeqpd %xmm3, %xmm4
+; SSE-32-NEXT:    cmpunordpd %xmm3, %xmm2
+; SSE-32-NEXT:    orpd %xmm4, %xmm2
+; SSE-32-NEXT:    andpd %xmm2, %xmm0
+; SSE-32-NEXT:    andnpd %xmm1, %xmm2
+; SSE-32-NEXT:    orpd %xmm2, %xmm0
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_v2f64_ueq_s:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movapd %xmm2, %xmm4
+; SSE-64-NEXT:    cmpltpd %xmm3, %xmm4
+; SSE-64-NEXT:    movapd %xmm2, %xmm4
+; SSE-64-NEXT:    cmpeqpd %xmm3, %xmm4
+; SSE-64-NEXT:    cmpunordpd %xmm3, %xmm2
+; SSE-64-NEXT:    orpd %xmm4, %xmm2
+; SSE-64-NEXT:    andpd %xmm2, %xmm0
+; SSE-64-NEXT:    andnpd %xmm1, %xmm2
+; SSE-64-NEXT:    orpd %xmm2, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_v2f64_ueq_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-16, %esp
+; AVX-32-NEXT:    subl $16, %esp
+; AVX-32-NEXT:    vcmpeq_uspd 8(%ebp), %xmm2, %xmm2
+; AVX-32-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v2f64_ueq_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpeq_uspd %xmm3, %xmm2, %xmm2
+; AVX-64-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v2f64_ueq_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-16, %esp
+; AVX512-32-NEXT:    subl $16, %esp
+; AVX512-32-NEXT:    vcmpeq_uspd 8(%ebp), %xmm2, %k1
+; AVX512-32-NEXT:    vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v2f64_ueq_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpeq_uspd %xmm3, %xmm2, %k1
+; AVX512-64-NEXT:    vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <2 x i1> @llvm.experimental.constrained.fcmps.v2f64(
+                                               <2 x double> %f1, <2 x double> %f2, metadata !"ueq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <2 x i1> %cond, <2 x i64> %a, <2 x i64> %b
+  ret <2 x i64> %res
+}
+
+define <2 x i64> @test_v2f64_ugt_s(<2 x i64> %a, <2 x i64> %b, <2 x double> %f1, <2 x double> %f2) #0 {
+; SSE-32-LABEL: test_v2f64_ugt_s:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    andl $-16, %esp
+; SSE-32-NEXT:    subl $16, %esp
+; SSE-32-NEXT:    cmpnlepd 8(%ebp), %xmm2
+; SSE-32-NEXT:    andpd %xmm2, %xmm0
+; SSE-32-NEXT:    andnpd %xmm1, %xmm2
+; SSE-32-NEXT:    orpd %xmm2, %xmm0
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_v2f64_ugt_s:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    cmpnlepd %xmm3, %xmm2
+; SSE-64-NEXT:    andpd %xmm2, %xmm0
+; SSE-64-NEXT:    andnpd %xmm1, %xmm2
+; SSE-64-NEXT:    orpd %xmm2, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_v2f64_ugt_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-16, %esp
+; AVX-32-NEXT:    subl $16, %esp
+; AVX-32-NEXT:    vcmpnlepd 8(%ebp), %xmm2, %xmm2
+; AVX-32-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v2f64_ugt_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpnlepd %xmm3, %xmm2, %xmm2
+; AVX-64-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v2f64_ugt_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-16, %esp
+; AVX512-32-NEXT:    subl $16, %esp
+; AVX512-32-NEXT:    vcmpnlepd 8(%ebp), %xmm2, %k1
+; AVX512-32-NEXT:    vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v2f64_ugt_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpnlepd %xmm3, %xmm2, %k1
+; AVX512-64-NEXT:    vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <2 x i1> @llvm.experimental.constrained.fcmps.v2f64(
+                                               <2 x double> %f1, <2 x double> %f2, metadata !"ugt",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <2 x i1> %cond, <2 x i64> %a, <2 x i64> %b
+  ret <2 x i64> %res
+}
+
+define <2 x i64> @test_v2f64_uge_s(<2 x i64> %a, <2 x i64> %b, <2 x double> %f1, <2 x double> %f2) #0 {
+; SSE-32-LABEL: test_v2f64_uge_s:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    andl $-16, %esp
+; SSE-32-NEXT:    subl $16, %esp
+; SSE-32-NEXT:    cmpnltpd 8(%ebp), %xmm2
+; SSE-32-NEXT:    andpd %xmm2, %xmm0
+; SSE-32-NEXT:    andnpd %xmm1, %xmm2
+; SSE-32-NEXT:    orpd %xmm2, %xmm0
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_v2f64_uge_s:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    cmpnltpd %xmm3, %xmm2
+; SSE-64-NEXT:    andpd %xmm2, %xmm0
+; SSE-64-NEXT:    andnpd %xmm1, %xmm2
+; SSE-64-NEXT:    orpd %xmm2, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_v2f64_uge_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-16, %esp
+; AVX-32-NEXT:    subl $16, %esp
+; AVX-32-NEXT:    vcmpnltpd 8(%ebp), %xmm2, %xmm2
+; AVX-32-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v2f64_uge_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpnltpd %xmm3, %xmm2, %xmm2
+; AVX-64-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v2f64_uge_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-16, %esp
+; AVX512-32-NEXT:    subl $16, %esp
+; AVX512-32-NEXT:    vcmpnltpd 8(%ebp), %xmm2, %k1
+; AVX512-32-NEXT:    vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v2f64_uge_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpnltpd %xmm3, %xmm2, %k1
+; AVX512-64-NEXT:    vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <2 x i1> @llvm.experimental.constrained.fcmps.v2f64(
+                                               <2 x double> %f1, <2 x double> %f2, metadata !"uge",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <2 x i1> %cond, <2 x i64> %a, <2 x i64> %b
+  ret <2 x i64> %res
+}
+
+define <2 x i64> @test_v2f64_ult_s(<2 x i64> %a, <2 x i64> %b, <2 x double> %f1, <2 x double> %f2) #0 {
+; SSE-32-LABEL: test_v2f64_ult_s:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    andl $-16, %esp
+; SSE-32-NEXT:    subl $16, %esp
+; SSE-32-NEXT:    movapd 8(%ebp), %xmm3
+; SSE-32-NEXT:    cmpnlepd %xmm2, %xmm3
+; SSE-32-NEXT:    andpd %xmm3, %xmm0
+; SSE-32-NEXT:    andnpd %xmm1, %xmm3
+; SSE-32-NEXT:    orpd %xmm3, %xmm0
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_v2f64_ult_s:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    cmpnlepd %xmm2, %xmm3
+; SSE-64-NEXT:    andpd %xmm3, %xmm0
+; SSE-64-NEXT:    andnpd %xmm1, %xmm3
+; SSE-64-NEXT:    orpd %xmm3, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_v2f64_ult_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-16, %esp
+; AVX-32-NEXT:    subl $16, %esp
+; AVX-32-NEXT:    vmovapd 8(%ebp), %xmm3
+; AVX-32-NEXT:    vcmpnlepd %xmm2, %xmm3, %xmm2
+; AVX-32-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v2f64_ult_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpnlepd %xmm2, %xmm3, %xmm2
+; AVX-64-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v2f64_ult_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-16, %esp
+; AVX512-32-NEXT:    subl $16, %esp
+; AVX512-32-NEXT:    vcmpngepd 8(%ebp), %xmm2, %k1
+; AVX512-32-NEXT:    vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v2f64_ult_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpnlepd %xmm2, %xmm3, %k1
+; AVX512-64-NEXT:    vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <2 x i1> @llvm.experimental.constrained.fcmps.v2f64(
+                                               <2 x double> %f1, <2 x double> %f2, metadata !"ult",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <2 x i1> %cond, <2 x i64> %a, <2 x i64> %b
+  ret <2 x i64> %res
+}
+
+define <2 x i64> @test_v2f64_ule_s(<2 x i64> %a, <2 x i64> %b, <2 x double> %f1, <2 x double> %f2) #0 {
+; SSE-32-LABEL: test_v2f64_ule_s:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    andl $-16, %esp
+; SSE-32-NEXT:    subl $16, %esp
+; SSE-32-NEXT:    movapd 8(%ebp), %xmm3
+; SSE-32-NEXT:    cmpnltpd %xmm2, %xmm3
+; SSE-32-NEXT:    andpd %xmm3, %xmm0
+; SSE-32-NEXT:    andnpd %xmm1, %xmm3
+; SSE-32-NEXT:    orpd %xmm3, %xmm0
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_v2f64_ule_s:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    cmpnltpd %xmm2, %xmm3
+; SSE-64-NEXT:    andpd %xmm3, %xmm0
+; SSE-64-NEXT:    andnpd %xmm1, %xmm3
+; SSE-64-NEXT:    orpd %xmm3, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_v2f64_ule_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-16, %esp
+; AVX-32-NEXT:    subl $16, %esp
+; AVX-32-NEXT:    vmovapd 8(%ebp), %xmm3
+; AVX-32-NEXT:    vcmpnltpd %xmm2, %xmm3, %xmm2
+; AVX-32-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v2f64_ule_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpnltpd %xmm2, %xmm3, %xmm2
+; AVX-64-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v2f64_ule_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-16, %esp
+; AVX512-32-NEXT:    subl $16, %esp
+; AVX512-32-NEXT:    vcmpngtpd 8(%ebp), %xmm2, %k1
+; AVX512-32-NEXT:    vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v2f64_ule_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpnltpd %xmm2, %xmm3, %k1
+; AVX512-64-NEXT:    vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <2 x i1> @llvm.experimental.constrained.fcmps.v2f64(
+                                               <2 x double> %f1, <2 x double> %f2, metadata !"ule",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <2 x i1> %cond, <2 x i64> %a, <2 x i64> %b
+  ret <2 x i64> %res
+}
+
+define <2 x i64> @test_v2f64_une_s(<2 x i64> %a, <2 x i64> %b, <2 x double> %f1, <2 x double> %f2) #0 {
+; SSE-32-LABEL: test_v2f64_une_s:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    andl $-16, %esp
+; SSE-32-NEXT:    subl $16, %esp
+; SSE-32-NEXT:    movapd 8(%ebp), %xmm3
+; SSE-32-NEXT:    movapd %xmm2, %xmm4
+; SSE-32-NEXT:    cmpltpd %xmm3, %xmm4
+; SSE-32-NEXT:    cmpneqpd %xmm3, %xmm2
+; SSE-32-NEXT:    andpd %xmm2, %xmm0
+; SSE-32-NEXT:    andnpd %xmm1, %xmm2
+; SSE-32-NEXT:    orpd %xmm2, %xmm0
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_v2f64_une_s:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movapd %xmm2, %xmm4
+; SSE-64-NEXT:    cmpltpd %xmm3, %xmm4
+; SSE-64-NEXT:    cmpneqpd %xmm3, %xmm2
+; SSE-64-NEXT:    andpd %xmm2, %xmm0
+; SSE-64-NEXT:    andnpd %xmm1, %xmm2
+; SSE-64-NEXT:    orpd %xmm2, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_v2f64_une_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-16, %esp
+; AVX-32-NEXT:    subl $16, %esp
+; AVX-32-NEXT:    vcmpneq_uspd 8(%ebp), %xmm2, %xmm2
+; AVX-32-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v2f64_une_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpneq_uspd %xmm3, %xmm2, %xmm2
+; AVX-64-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v2f64_une_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-16, %esp
+; AVX512-32-NEXT:    subl $16, %esp
+; AVX512-32-NEXT:    vcmpneq_uspd 8(%ebp), %xmm2, %k1
+; AVX512-32-NEXT:    vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v2f64_une_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpneq_uspd %xmm3, %xmm2, %k1
+; AVX512-64-NEXT:    vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <2 x i1> @llvm.experimental.constrained.fcmps.v2f64(
+                                               <2 x double> %f1, <2 x double> %f2, metadata !"une",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <2 x i1> %cond, <2 x i64> %a, <2 x i64> %b
+  ret <2 x i64> %res
+}
+
+define <2 x i64> @test_v2f64_uno_s(<2 x i64> %a, <2 x i64> %b, <2 x double> %f1, <2 x double> %f2) #0 {
+; SSE-32-LABEL: test_v2f64_uno_s:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    andl $-16, %esp
+; SSE-32-NEXT:    subl $16, %esp
+; SSE-32-NEXT:    movapd 8(%ebp), %xmm3
+; SSE-32-NEXT:    movapd %xmm2, %xmm4
+; SSE-32-NEXT:    cmpltpd %xmm3, %xmm4
+; SSE-32-NEXT:    cmpunordpd %xmm3, %xmm2
+; SSE-32-NEXT:    andpd %xmm2, %xmm0
+; SSE-32-NEXT:    andnpd %xmm1, %xmm2
+; SSE-32-NEXT:    orpd %xmm2, %xmm0
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_v2f64_uno_s:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movapd %xmm2, %xmm4
+; SSE-64-NEXT:    cmpltpd %xmm3, %xmm4
+; SSE-64-NEXT:    cmpunordpd %xmm3, %xmm2
+; SSE-64-NEXT:    andpd %xmm2, %xmm0
+; SSE-64-NEXT:    andnpd %xmm1, %xmm2
+; SSE-64-NEXT:    orpd %xmm2, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_v2f64_uno_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-16, %esp
+; AVX-32-NEXT:    subl $16, %esp
+; AVX-32-NEXT:    vcmpunord_spd 8(%ebp), %xmm2, %xmm2
+; AVX-32-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v2f64_uno_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpunord_spd %xmm3, %xmm2, %xmm2
+; AVX-64-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v2f64_uno_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-16, %esp
+; AVX512-32-NEXT:    subl $16, %esp
+; AVX512-32-NEXT:    vcmpunord_spd 8(%ebp), %xmm2, %k1
+; AVX512-32-NEXT:    vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v2f64_uno_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpunord_spd %xmm3, %xmm2, %k1
+; AVX512-64-NEXT:    vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <2 x i1> @llvm.experimental.constrained.fcmps.v2f64(
+                                               <2 x double> %f1, <2 x double> %f2, metadata !"uno",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <2 x i1> %cond, <2 x i64> %a, <2 x i64> %b
+  ret <2 x i64> %res
+}
+
+attributes #0 = { strictfp nounwind }
+
+declare <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(<4 x float>, <4 x float>, metadata, metadata)
+declare <2 x i1> @llvm.experimental.constrained.fcmp.v2f64(<2 x double>, <2 x double>, metadata, metadata)
+declare <4 x i1> @llvm.experimental.constrained.fcmps.v4f32(<4 x float>, <4 x float>, metadata, metadata)
+declare <2 x i1> @llvm.experimental.constrained.fcmps.v2f64(<2 x double>, <2 x double>, metadata, metadata)
diff --git a/llvm/test/CodeGen/X86/vec-strict-cmp-256.ll b/llvm/test/CodeGen/X86/vec-strict-cmp-256.ll
new file mode 100644
index 000000000..583c2b484
--- /dev/null
+++ b/llvm/test/CodeGen/X86/vec-strict-cmp-256.ll
@@ -0,0 +1,2436 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx -O3 | FileCheck %s --check-prefixes=CHECK,AVX,AVX-32
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx -O3 | FileCheck %s --check-prefixes=CHECK,AVX,AVX-64
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx512f -mattr=+avx512vl -O3 | FileCheck %s --check-prefixes=CHECK,AVX512-32
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f -mattr=+avx512vl -O3 | FileCheck %s --check-prefixes=CHECK,AVX512-64
+
+define <8 x i32> @test_v8f32_oeq_q(<8 x i32> %a, <8 x i32> %b, <8 x float> %f1, <8 x float> %f2) #0 {
+; AVX-32-LABEL: test_v8f32_oeq_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-32, %esp
+; AVX-32-NEXT:    subl $32, %esp
+; AVX-32-NEXT:    vcmpeqps 8(%ebp), %ymm2, %ymm2
+; AVX-32-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v8f32_oeq_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpeqps %ymm3, %ymm2, %ymm2
+; AVX-64-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v8f32_oeq_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-32, %esp
+; AVX512-32-NEXT:    subl $32, %esp
+; AVX512-32-NEXT:    vcmpeqps 8(%ebp), %ymm2, %k1
+; AVX512-32-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v8f32_oeq_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpeqps %ymm3, %ymm2, %k1
+; AVX512-64-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <8 x i1> @llvm.experimental.constrained.fcmp.v8f32(
+                                               <8 x float> %f1, <8 x float> %f2, metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <8 x i1> %cond, <8 x i32> %a, <8 x i32> %b
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_v8f32_ogt_q(<8 x i32> %a, <8 x i32> %b, <8 x float> %f1, <8 x float> %f2) #0 {
+; AVX-32-LABEL: test_v8f32_ogt_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-32, %esp
+; AVX-32-NEXT:    subl $32, %esp
+; AVX-32-NEXT:    vmovaps 8(%ebp), %ymm3
+; AVX-32-NEXT:    vcmplt_oqps %ymm2, %ymm3, %ymm2
+; AVX-32-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v8f32_ogt_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmplt_oqps %ymm2, %ymm3, %ymm2
+; AVX-64-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v8f32_ogt_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-32, %esp
+; AVX512-32-NEXT:    subl $32, %esp
+; AVX512-32-NEXT:    vcmpgt_oqps 8(%ebp), %ymm2, %k1
+; AVX512-32-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v8f32_ogt_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmplt_oqps %ymm2, %ymm3, %k1
+; AVX512-64-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <8 x i1> @llvm.experimental.constrained.fcmp.v8f32(
+                                               <8 x float> %f1, <8 x float> %f2, metadata !"ogt",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <8 x i1> %cond, <8 x i32> %a, <8 x i32> %b
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_v8f32_oge_q(<8 x i32> %a, <8 x i32> %b, <8 x float> %f1, <8 x float> %f2) #0 {
+; AVX-32-LABEL: test_v8f32_oge_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-32, %esp
+; AVX-32-NEXT:    subl $32, %esp
+; AVX-32-NEXT:    vmovaps 8(%ebp), %ymm3
+; AVX-32-NEXT:    vcmple_oqps %ymm2, %ymm3, %ymm2
+; AVX-32-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v8f32_oge_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmple_oqps %ymm2, %ymm3, %ymm2
+; AVX-64-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v8f32_oge_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-32, %esp
+; AVX512-32-NEXT:    subl $32, %esp
+; AVX512-32-NEXT:    vcmpge_oqps 8(%ebp), %ymm2, %k1
+; AVX512-32-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v8f32_oge_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmple_oqps %ymm2, %ymm3, %k1
+; AVX512-64-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <8 x i1> @llvm.experimental.constrained.fcmp.v8f32(
+                                               <8 x float> %f1, <8 x float> %f2, metadata !"oge",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <8 x i1> %cond, <8 x i32> %a, <8 x i32> %b
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_v8f32_olt_q(<8 x i32> %a, <8 x i32> %b, <8 x float> %f1, <8 x float> %f2) #0 {
+; AVX-32-LABEL: test_v8f32_olt_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-32, %esp
+; AVX-32-NEXT:    subl $32, %esp
+; AVX-32-NEXT:    vcmplt_oqps 8(%ebp), %ymm2, %ymm2
+; AVX-32-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v8f32_olt_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmplt_oqps %ymm3, %ymm2, %ymm2
+; AVX-64-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v8f32_olt_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-32, %esp
+; AVX512-32-NEXT:    subl $32, %esp
+; AVX512-32-NEXT:    vcmplt_oqps 8(%ebp), %ymm2, %k1
+; AVX512-32-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v8f32_olt_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmplt_oqps %ymm3, %ymm2, %k1
+; AVX512-64-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <8 x i1> @llvm.experimental.constrained.fcmp.v8f32(
+                                               <8 x float> %f1, <8 x float> %f2, metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <8 x i1> %cond, <8 x i32> %a, <8 x i32> %b
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_v8f32_ole_q(<8 x i32> %a, <8 x i32> %b, <8 x float> %f1, <8 x float> %f2) #0 {
+; AVX-32-LABEL: test_v8f32_ole_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-32, %esp
+; AVX-32-NEXT:    subl $32, %esp
+; AVX-32-NEXT:    vcmple_oqps 8(%ebp), %ymm2, %ymm2
+; AVX-32-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v8f32_ole_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmple_oqps %ymm3, %ymm2, %ymm2
+; AVX-64-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v8f32_ole_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-32, %esp
+; AVX512-32-NEXT:    subl $32, %esp
+; AVX512-32-NEXT:    vcmple_oqps 8(%ebp), %ymm2, %k1
+; AVX512-32-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v8f32_ole_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmple_oqps %ymm3, %ymm2, %k1
+; AVX512-64-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <8 x i1> @llvm.experimental.constrained.fcmp.v8f32(
+                                               <8 x float> %f1, <8 x float> %f2, metadata !"ole",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <8 x i1> %cond, <8 x i32> %a, <8 x i32> %b
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_v8f32_one_q(<8 x i32> %a, <8 x i32> %b, <8 x float> %f1, <8 x float> %f2) #0 {
+; AVX-32-LABEL: test_v8f32_one_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-32, %esp
+; AVX-32-NEXT:    subl $32, %esp
+; AVX-32-NEXT:    vcmpneq_oqps 8(%ebp), %ymm2, %ymm2
+; AVX-32-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v8f32_one_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpneq_oqps %ymm3, %ymm2, %ymm2
+; AVX-64-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v8f32_one_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-32, %esp
+; AVX512-32-NEXT:    subl $32, %esp
+; AVX512-32-NEXT:    vcmpneq_oqps 8(%ebp), %ymm2, %k1
+; AVX512-32-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v8f32_one_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpneq_oqps %ymm3, %ymm2, %k1
+; AVX512-64-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <8 x i1> @llvm.experimental.constrained.fcmp.v8f32(
+                                               <8 x float> %f1, <8 x float> %f2, metadata !"one",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <8 x i1> %cond, <8 x i32> %a, <8 x i32> %b
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_v8f32_ord_q(<8 x i32> %a, <8 x i32> %b, <8 x float> %f1, <8 x float> %f2) #0 {
+; AVX-32-LABEL: test_v8f32_ord_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-32, %esp
+; AVX-32-NEXT:    subl $32, %esp
+; AVX-32-NEXT:    vcmpordps 8(%ebp), %ymm2, %ymm2
+; AVX-32-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v8f32_ord_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpordps %ymm3, %ymm2, %ymm2
+; AVX-64-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v8f32_ord_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-32, %esp
+; AVX512-32-NEXT:    subl $32, %esp
+; AVX512-32-NEXT:    vcmpordps 8(%ebp), %ymm2, %k1
+; AVX512-32-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v8f32_ord_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpordps %ymm3, %ymm2, %k1
+; AVX512-64-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <8 x i1> @llvm.experimental.constrained.fcmp.v8f32(
+                                               <8 x float> %f1, <8 x float> %f2, metadata !"ord",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <8 x i1> %cond, <8 x i32> %a, <8 x i32> %b
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_v8f32_ueq_q(<8 x i32> %a, <8 x i32> %b, <8 x float> %f1, <8 x float> %f2) #0 {
+; AVX-32-LABEL: test_v8f32_ueq_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-32, %esp
+; AVX-32-NEXT:    subl $32, %esp
+; AVX-32-NEXT:    vcmpeq_uqps 8(%ebp), %ymm2, %ymm2
+; AVX-32-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v8f32_ueq_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpeq_uqps %ymm3, %ymm2, %ymm2
+; AVX-64-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v8f32_ueq_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-32, %esp
+; AVX512-32-NEXT:    subl $32, %esp
+; AVX512-32-NEXT:    vcmpeq_uqps 8(%ebp), %ymm2, %k1
+; AVX512-32-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v8f32_ueq_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpeq_uqps %ymm3, %ymm2, %k1
+; AVX512-64-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <8 x i1> @llvm.experimental.constrained.fcmp.v8f32(
+                                               <8 x float> %f1, <8 x float> %f2, metadata !"ueq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <8 x i1> %cond, <8 x i32> %a, <8 x i32> %b
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_v8f32_ugt_q(<8 x i32> %a, <8 x i32> %b, <8 x float> %f1, <8 x float> %f2) #0 {
+; AVX-32-LABEL: test_v8f32_ugt_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-32, %esp
+; AVX-32-NEXT:    subl $32, %esp
+; AVX-32-NEXT:    vcmpnle_uqps 8(%ebp), %ymm2, %ymm2
+; AVX-32-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v8f32_ugt_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpnle_uqps %ymm3, %ymm2, %ymm2
+; AVX-64-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v8f32_ugt_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-32, %esp
+; AVX512-32-NEXT:    subl $32, %esp
+; AVX512-32-NEXT:    vcmpnle_uqps 8(%ebp), %ymm2, %k1
+; AVX512-32-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v8f32_ugt_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpnle_uqps %ymm3, %ymm2, %k1
+; AVX512-64-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <8 x i1> @llvm.experimental.constrained.fcmp.v8f32(
+                                               <8 x float> %f1, <8 x float> %f2, metadata !"ugt",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <8 x i1> %cond, <8 x i32> %a, <8 x i32> %b
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_v8f32_uge_q(<8 x i32> %a, <8 x i32> %b, <8 x float> %f1, <8 x float> %f2) #0 {
+; AVX-32-LABEL: test_v8f32_uge_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-32, %esp
+; AVX-32-NEXT:    subl $32, %esp
+; AVX-32-NEXT:    vcmpnlt_uqps 8(%ebp), %ymm2, %ymm2
+; AVX-32-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v8f32_uge_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpnlt_uqps %ymm3, %ymm2, %ymm2
+; AVX-64-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v8f32_uge_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-32, %esp
+; AVX512-32-NEXT:    subl $32, %esp
+; AVX512-32-NEXT:    vcmpnlt_uqps 8(%ebp), %ymm2, %k1
+; AVX512-32-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v8f32_uge_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpnlt_uqps %ymm3, %ymm2, %k1
+; AVX512-64-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <8 x i1> @llvm.experimental.constrained.fcmp.v8f32(
+                                               <8 x float> %f1, <8 x float> %f2, metadata !"uge",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <8 x i1> %cond, <8 x i32> %a, <8 x i32> %b
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_v8f32_ult_q(<8 x i32> %a, <8 x i32> %b, <8 x float> %f1, <8 x float> %f2) #0 {
+; AVX-32-LABEL: test_v8f32_ult_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-32, %esp
+; AVX-32-NEXT:    subl $32, %esp
+; AVX-32-NEXT:    vmovaps 8(%ebp), %ymm3
+; AVX-32-NEXT:    vcmpnle_uqps %ymm2, %ymm3, %ymm2
+; AVX-32-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v8f32_ult_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpnle_uqps %ymm2, %ymm3, %ymm2
+; AVX-64-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v8f32_ult_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-32, %esp
+; AVX512-32-NEXT:    subl $32, %esp
+; AVX512-32-NEXT:    vcmpnge_uqps 8(%ebp), %ymm2, %k1
+; AVX512-32-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v8f32_ult_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpnle_uqps %ymm2, %ymm3, %k1
+; AVX512-64-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <8 x i1> @llvm.experimental.constrained.fcmp.v8f32(
+                                               <8 x float> %f1, <8 x float> %f2, metadata !"ult",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <8 x i1> %cond, <8 x i32> %a, <8 x i32> %b
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_v8f32_ule_q(<8 x i32> %a, <8 x i32> %b, <8 x float> %f1, <8 x float> %f2) #0 {
+; AVX-32-LABEL: test_v8f32_ule_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-32, %esp
+; AVX-32-NEXT:    subl $32, %esp
+; AVX-32-NEXT:    vmovaps 8(%ebp), %ymm3
+; AVX-32-NEXT:    vcmpnlt_uqps %ymm2, %ymm3, %ymm2
+; AVX-32-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v8f32_ule_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpnlt_uqps %ymm2, %ymm3, %ymm2
+; AVX-64-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v8f32_ule_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-32, %esp
+; AVX512-32-NEXT:    subl $32, %esp
+; AVX512-32-NEXT:    vcmpngt_uqps 8(%ebp), %ymm2, %k1
+; AVX512-32-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v8f32_ule_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpnlt_uqps %ymm2, %ymm3, %k1
+; AVX512-64-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <8 x i1> @llvm.experimental.constrained.fcmp.v8f32(
+                                               <8 x float> %f1, <8 x float> %f2, metadata !"ule",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <8 x i1> %cond, <8 x i32> %a, <8 x i32> %b
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_v8f32_une_q(<8 x i32> %a, <8 x i32> %b, <8 x float> %f1, <8 x float> %f2) #0 {
+; AVX-32-LABEL: test_v8f32_une_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-32, %esp
+; AVX-32-NEXT:    subl $32, %esp
+; AVX-32-NEXT:    vcmpneqps 8(%ebp), %ymm2, %ymm2
+; AVX-32-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v8f32_une_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpneqps %ymm3, %ymm2, %ymm2
+; AVX-64-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v8f32_une_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-32, %esp
+; AVX512-32-NEXT:    subl $32, %esp
+; AVX512-32-NEXT:    vcmpneqps 8(%ebp), %ymm2, %k1
+; AVX512-32-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v8f32_une_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpneqps %ymm3, %ymm2, %k1
+; AVX512-64-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <8 x i1> @llvm.experimental.constrained.fcmp.v8f32(
+                                               <8 x float> %f1, <8 x float> %f2, metadata !"une",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <8 x i1> %cond, <8 x i32> %a, <8 x i32> %b
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_v8f32_uno_q(<8 x i32> %a, <8 x i32> %b, <8 x float> %f1, <8 x float> %f2) #0 {
+; AVX-32-LABEL: test_v8f32_uno_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-32, %esp
+; AVX-32-NEXT:    subl $32, %esp
+; AVX-32-NEXT:    vcmpunordps 8(%ebp), %ymm2, %ymm2
+; AVX-32-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v8f32_uno_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpunordps %ymm3, %ymm2, %ymm2
+; AVX-64-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v8f32_uno_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-32, %esp
+; AVX512-32-NEXT:    subl $32, %esp
+; AVX512-32-NEXT:    vcmpunordps 8(%ebp), %ymm2, %k1
+; AVX512-32-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v8f32_uno_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpunordps %ymm3, %ymm2, %k1
+; AVX512-64-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <8 x i1> @llvm.experimental.constrained.fcmp.v8f32(
+                                               <8 x float> %f1, <8 x float> %f2, metadata !"uno",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <8 x i1> %cond, <8 x i32> %a, <8 x i32> %b
+  ret <8 x i32> %res
+}
+
+define <4 x i64> @test_v4f64_oeq_q(<4 x i64> %a, <4 x i64> %b, <4 x double> %f1, <4 x double> %f2) #0 {
+; AVX-32-LABEL: test_v4f64_oeq_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-32, %esp
+; AVX-32-NEXT:    subl $32, %esp
+; AVX-32-NEXT:    vcmpeqpd 8(%ebp), %ymm2, %ymm2
+; AVX-32-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v4f64_oeq_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpeqpd %ymm3, %ymm2, %ymm2
+; AVX-64-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v4f64_oeq_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-32, %esp
+; AVX512-32-NEXT:    subl $32, %esp
+; AVX512-32-NEXT:    vcmpeqpd 8(%ebp), %ymm2, %k1
+; AVX512-32-NEXT:    vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v4f64_oeq_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpeqpd %ymm3, %ymm2, %k1
+; AVX512-64-NEXT:    vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f64(
+                                               <4 x double> %f1, <4 x double> %f2, metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <4 x i1> %cond, <4 x i64> %a, <4 x i64> %b
+  ret <4 x i64> %res
+}
+
+define <4 x i64> @test_v4f64_ogt_q(<4 x i64> %a, <4 x i64> %b, <4 x double> %f1, <4 x double> %f2) #0 {
+; AVX-32-LABEL: test_v4f64_ogt_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-32, %esp
+; AVX-32-NEXT:    subl $32, %esp
+; AVX-32-NEXT:    vmovapd 8(%ebp), %ymm3
+; AVX-32-NEXT:    vcmplt_oqpd %ymm2, %ymm3, %ymm2
+; AVX-32-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v4f64_ogt_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmplt_oqpd %ymm2, %ymm3, %ymm2
+; AVX-64-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v4f64_ogt_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-32, %esp
+; AVX512-32-NEXT:    subl $32, %esp
+; AVX512-32-NEXT:    vcmpgt_oqpd 8(%ebp), %ymm2, %k1
+; AVX512-32-NEXT:    vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v4f64_ogt_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmplt_oqpd %ymm2, %ymm3, %k1
+; AVX512-64-NEXT:    vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f64(
+                                               <4 x double> %f1, <4 x double> %f2, metadata !"ogt",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <4 x i1> %cond, <4 x i64> %a, <4 x i64> %b
+  ret <4 x i64> %res
+}
+
+define <4 x i64> @test_v4f64_oge_q(<4 x i64> %a, <4 x i64> %b, <4 x double> %f1, <4 x double> %f2) #0 {
+; AVX-32-LABEL: test_v4f64_oge_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-32, %esp
+; AVX-32-NEXT:    subl $32, %esp
+; AVX-32-NEXT:    vmovapd 8(%ebp), %ymm3
+; AVX-32-NEXT:    vcmple_oqpd %ymm2, %ymm3, %ymm2
+; AVX-32-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v4f64_oge_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmple_oqpd %ymm2, %ymm3, %ymm2
+; AVX-64-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v4f64_oge_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-32, %esp
+; AVX512-32-NEXT:    subl $32, %esp
+; AVX512-32-NEXT:    vcmpge_oqpd 8(%ebp), %ymm2, %k1
+; AVX512-32-NEXT:    vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v4f64_oge_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmple_oqpd %ymm2, %ymm3, %k1
+; AVX512-64-NEXT:    vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f64(
+                                               <4 x double> %f1, <4 x double> %f2, metadata !"oge",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <4 x i1> %cond, <4 x i64> %a, <4 x i64> %b
+  ret <4 x i64> %res
+}
+
+define <4 x i64> @test_v4f64_olt_q(<4 x i64> %a, <4 x i64> %b, <4 x double> %f1, <4 x double> %f2) #0 {
+; AVX-32-LABEL: test_v4f64_olt_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-32, %esp
+; AVX-32-NEXT:    subl $32, %esp
+; AVX-32-NEXT:    vcmplt_oqpd 8(%ebp), %ymm2, %ymm2
+; AVX-32-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v4f64_olt_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmplt_oqpd %ymm3, %ymm2, %ymm2
+; AVX-64-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v4f64_olt_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-32, %esp
+; AVX512-32-NEXT:    subl $32, %esp
+; AVX512-32-NEXT:    vcmplt_oqpd 8(%ebp), %ymm2, %k1
+; AVX512-32-NEXT:    vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v4f64_olt_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmplt_oqpd %ymm3, %ymm2, %k1
+; AVX512-64-NEXT:    vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f64(
+                                               <4 x double> %f1, <4 x double> %f2, metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <4 x i1> %cond, <4 x i64> %a, <4 x i64> %b
+  ret <4 x i64> %res
+}
+
+define <4 x i64> @test_v4f64_ole_q(<4 x i64> %a, <4 x i64> %b, <4 x double> %f1, <4 x double> %f2) #0 {
+; AVX-32-LABEL: test_v4f64_ole_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-32, %esp
+; AVX-32-NEXT:    subl $32, %esp
+; AVX-32-NEXT:    vcmple_oqpd 8(%ebp), %ymm2, %ymm2
+; AVX-32-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v4f64_ole_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmple_oqpd %ymm3, %ymm2, %ymm2
+; AVX-64-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v4f64_ole_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-32, %esp
+; AVX512-32-NEXT:    subl $32, %esp
+; AVX512-32-NEXT:    vcmple_oqpd 8(%ebp), %ymm2, %k1
+; AVX512-32-NEXT:    vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v4f64_ole_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmple_oqpd %ymm3, %ymm2, %k1
+; AVX512-64-NEXT:    vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f64(
+                                               <4 x double> %f1, <4 x double> %f2, metadata !"ole",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <4 x i1> %cond, <4 x i64> %a, <4 x i64> %b
+  ret <4 x i64> %res
+}
+
+define <4 x i64> @test_v4f64_one_q(<4 x i64> %a, <4 x i64> %b, <4 x double> %f1, <4 x double> %f2) #0 {
+; AVX-32-LABEL: test_v4f64_one_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-32, %esp
+; AVX-32-NEXT:    subl $32, %esp
+; AVX-32-NEXT:    vcmpneq_oqpd 8(%ebp), %ymm2, %ymm2
+; AVX-32-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v4f64_one_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpneq_oqpd %ymm3, %ymm2, %ymm2
+; AVX-64-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v4f64_one_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-32, %esp
+; AVX512-32-NEXT:    subl $32, %esp
+; AVX512-32-NEXT:    vcmpneq_oqpd 8(%ebp), %ymm2, %k1
+; AVX512-32-NEXT:    vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v4f64_one_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpneq_oqpd %ymm3, %ymm2, %k1
+; AVX512-64-NEXT:    vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f64(
+                                               <4 x double> %f1, <4 x double> %f2, metadata !"one",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <4 x i1> %cond, <4 x i64> %a, <4 x i64> %b
+  ret <4 x i64> %res
+}
+
+define <4 x i64> @test_v4f64_ord_q(<4 x i64> %a, <4 x i64> %b, <4 x double> %f1, <4 x double> %f2) #0 {
+; AVX-32-LABEL: test_v4f64_ord_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-32, %esp
+; AVX-32-NEXT:    subl $32, %esp
+; AVX-32-NEXT:    vcmpordpd 8(%ebp), %ymm2, %ymm2
+; AVX-32-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v4f64_ord_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpordpd %ymm3, %ymm2, %ymm2
+; AVX-64-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v4f64_ord_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-32, %esp
+; AVX512-32-NEXT:    subl $32, %esp
+; AVX512-32-NEXT:    vcmpordpd 8(%ebp), %ymm2, %k1
+; AVX512-32-NEXT:    vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v4f64_ord_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpordpd %ymm3, %ymm2, %k1
+; AVX512-64-NEXT:    vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f64(
+                                               <4 x double> %f1, <4 x double> %f2, metadata !"ord",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <4 x i1> %cond, <4 x i64> %a, <4 x i64> %b
+  ret <4 x i64> %res
+}
+
+define <4 x i64> @test_v4f64_ueq_q(<4 x i64> %a, <4 x i64> %b, <4 x double> %f1, <4 x double> %f2) #0 {
+; AVX-32-LABEL: test_v4f64_ueq_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-32, %esp
+; AVX-32-NEXT:    subl $32, %esp
+; AVX-32-NEXT:    vcmpeq_uqpd 8(%ebp), %ymm2, %ymm2
+; AVX-32-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v4f64_ueq_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpeq_uqpd %ymm3, %ymm2, %ymm2
+; AVX-64-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v4f64_ueq_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-32, %esp
+; AVX512-32-NEXT:    subl $32, %esp
+; AVX512-32-NEXT:    vcmpeq_uqpd 8(%ebp), %ymm2, %k1
+; AVX512-32-NEXT:    vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v4f64_ueq_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpeq_uqpd %ymm3, %ymm2, %k1
+; AVX512-64-NEXT:    vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f64(
+                                               <4 x double> %f1, <4 x double> %f2, metadata !"ueq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <4 x i1> %cond, <4 x i64> %a, <4 x i64> %b
+  ret <4 x i64> %res
+}
+
+define <4 x i64> @test_v4f64_ugt_q(<4 x i64> %a, <4 x i64> %b, <4 x double> %f1, <4 x double> %f2) #0 {
+; AVX-32-LABEL: test_v4f64_ugt_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-32, %esp
+; AVX-32-NEXT:    subl $32, %esp
+; AVX-32-NEXT:    vcmpnle_uqpd 8(%ebp), %ymm2, %ymm2
+; AVX-32-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v4f64_ugt_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpnle_uqpd %ymm3, %ymm2, %ymm2
+; AVX-64-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v4f64_ugt_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-32, %esp
+; AVX512-32-NEXT:    subl $32, %esp
+; AVX512-32-NEXT:    vcmpnle_uqpd 8(%ebp), %ymm2, %k1
+; AVX512-32-NEXT:    vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v4f64_ugt_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpnle_uqpd %ymm3, %ymm2, %k1
+; AVX512-64-NEXT:    vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f64(
+                                               <4 x double> %f1, <4 x double> %f2, metadata !"ugt",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <4 x i1> %cond, <4 x i64> %a, <4 x i64> %b
+  ret <4 x i64> %res
+}
+
+define <4 x i64> @test_v4f64_uge_q(<4 x i64> %a, <4 x i64> %b, <4 x double> %f1, <4 x double> %f2) #0 {
+; AVX-32-LABEL: test_v4f64_uge_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-32, %esp
+; AVX-32-NEXT:    subl $32, %esp
+; AVX-32-NEXT:    vcmpnlt_uqpd 8(%ebp), %ymm2, %ymm2
+; AVX-32-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v4f64_uge_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpnlt_uqpd %ymm3, %ymm2, %ymm2
+; AVX-64-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v4f64_uge_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-32, %esp
+; AVX512-32-NEXT:    subl $32, %esp
+; AVX512-32-NEXT:    vcmpnlt_uqpd 8(%ebp), %ymm2, %k1
+; AVX512-32-NEXT:    vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v4f64_uge_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpnlt_uqpd %ymm3, %ymm2, %k1
+; AVX512-64-NEXT:    vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f64(
+                                               <4 x double> %f1, <4 x double> %f2, metadata !"uge",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <4 x i1> %cond, <4 x i64> %a, <4 x i64> %b
+  ret <4 x i64> %res
+}
+
+define <4 x i64> @test_v4f64_ult_q(<4 x i64> %a, <4 x i64> %b, <4 x double> %f1, <4 x double> %f2) #0 {
+; AVX-32-LABEL: test_v4f64_ult_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-32, %esp
+; AVX-32-NEXT:    subl $32, %esp
+; AVX-32-NEXT:    vmovapd 8(%ebp), %ymm3
+; AVX-32-NEXT:    vcmpnle_uqpd %ymm2, %ymm3, %ymm2
+; AVX-32-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v4f64_ult_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpnle_uqpd %ymm2, %ymm3, %ymm2
+; AVX-64-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v4f64_ult_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-32, %esp
+; AVX512-32-NEXT:    subl $32, %esp
+; AVX512-32-NEXT:    vcmpnge_uqpd 8(%ebp), %ymm2, %k1
+; AVX512-32-NEXT:    vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v4f64_ult_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpnle_uqpd %ymm2, %ymm3, %k1
+; AVX512-64-NEXT:    vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f64(
+                                               <4 x double> %f1, <4 x double> %f2, metadata !"ult",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <4 x i1> %cond, <4 x i64> %a, <4 x i64> %b
+  ret <4 x i64> %res
+}
+
+define <4 x i64> @test_v4f64_ule_q(<4 x i64> %a, <4 x i64> %b, <4 x double> %f1, <4 x double> %f2) #0 {
+; AVX-32-LABEL: test_v4f64_ule_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-32, %esp
+; AVX-32-NEXT:    subl $32, %esp
+; AVX-32-NEXT:    vmovapd 8(%ebp), %ymm3
+; AVX-32-NEXT:    vcmpnlt_uqpd %ymm2, %ymm3, %ymm2
+; AVX-32-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v4f64_ule_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpnlt_uqpd %ymm2, %ymm3, %ymm2
+; AVX-64-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v4f64_ule_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-32, %esp
+; AVX512-32-NEXT:    subl $32, %esp
+; AVX512-32-NEXT:    vcmpngt_uqpd 8(%ebp), %ymm2, %k1
+; AVX512-32-NEXT:    vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v4f64_ule_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpnlt_uqpd %ymm2, %ymm3, %k1
+; AVX512-64-NEXT:    vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f64(
+                                               <4 x double> %f1, <4 x double> %f2, metadata !"ule",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <4 x i1> %cond, <4 x i64> %a, <4 x i64> %b
+  ret <4 x i64> %res
+}
+
+define <4 x i64> @test_v4f64_une_q(<4 x i64> %a, <4 x i64> %b, <4 x double> %f1, <4 x double> %f2) #0 {
+; AVX-32-LABEL: test_v4f64_une_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-32, %esp
+; AVX-32-NEXT:    subl $32, %esp
+; AVX-32-NEXT:    vcmpneqpd 8(%ebp), %ymm2, %ymm2
+; AVX-32-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v4f64_une_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpneqpd %ymm3, %ymm2, %ymm2
+; AVX-64-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v4f64_une_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-32, %esp
+; AVX512-32-NEXT:    subl $32, %esp
+; AVX512-32-NEXT:    vcmpneqpd 8(%ebp), %ymm2, %k1
+; AVX512-32-NEXT:    vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v4f64_une_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpneqpd %ymm3, %ymm2, %k1
+; AVX512-64-NEXT:    vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f64(
+                                               <4 x double> %f1, <4 x double> %f2, metadata !"une",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <4 x i1> %cond, <4 x i64> %a, <4 x i64> %b
+  ret <4 x i64> %res
+}
+
+define <4 x i64> @test_v4f64_uno_q(<4 x i64> %a, <4 x i64> %b, <4 x double> %f1, <4 x double> %f2) #0 {
+; AVX-32-LABEL: test_v4f64_uno_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-32, %esp
+; AVX-32-NEXT:    subl $32, %esp
+; AVX-32-NEXT:    vcmpunordpd 8(%ebp), %ymm2, %ymm2
+; AVX-32-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v4f64_uno_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpunordpd %ymm3, %ymm2, %ymm2
+; AVX-64-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v4f64_uno_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-32, %esp
+; AVX512-32-NEXT:    subl $32, %esp
+; AVX512-32-NEXT:    vcmpunordpd 8(%ebp), %ymm2, %k1
+; AVX512-32-NEXT:    vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v4f64_uno_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpunordpd %ymm3, %ymm2, %k1
+; AVX512-64-NEXT:    vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f64(
+                                               <4 x double> %f1, <4 x double> %f2, metadata !"uno",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <4 x i1> %cond, <4 x i64> %a, <4 x i64> %b
+  ret <4 x i64> %res
+}
+
+define <8 x i32> @test_v8f32_oeq_s(<8 x i32> %a, <8 x i32> %b, <8 x float> %f1, <8 x float> %f2) #0 {
+; AVX-32-LABEL: test_v8f32_oeq_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-32, %esp
+; AVX-32-NEXT:    subl $32, %esp
+; AVX-32-NEXT:    vcmpeq_osps 8(%ebp), %ymm2, %ymm2
+; AVX-32-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v8f32_oeq_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpeq_osps %ymm3, %ymm2, %ymm2
+; AVX-64-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v8f32_oeq_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-32, %esp
+; AVX512-32-NEXT:    subl $32, %esp
+; AVX512-32-NEXT:    vcmpeq_osps 8(%ebp), %ymm2, %k1
+; AVX512-32-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v8f32_oeq_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpeq_osps %ymm3, %ymm2, %k1
+; AVX512-64-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <8 x i1> @llvm.experimental.constrained.fcmps.v8f32(
+                                               <8 x float> %f1, <8 x float> %f2, metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <8 x i1> %cond, <8 x i32> %a, <8 x i32> %b
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_v8f32_ogt_s(<8 x i32> %a, <8 x i32> %b, <8 x float> %f1, <8 x float> %f2) #0 {
+; AVX-32-LABEL: test_v8f32_ogt_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-32, %esp
+; AVX-32-NEXT:    subl $32, %esp
+; AVX-32-NEXT:    vmovaps 8(%ebp), %ymm3
+; AVX-32-NEXT:    vcmpltps %ymm2, %ymm3, %ymm2
+; AVX-32-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v8f32_ogt_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpltps %ymm2, %ymm3, %ymm2
+; AVX-64-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v8f32_ogt_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-32, %esp
+; AVX512-32-NEXT:    subl $32, %esp
+; AVX512-32-NEXT:    vcmpgtps 8(%ebp), %ymm2, %k1
+; AVX512-32-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v8f32_ogt_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpltps %ymm2, %ymm3, %k1
+; AVX512-64-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <8 x i1> @llvm.experimental.constrained.fcmps.v8f32(
+                                               <8 x float> %f1, <8 x float> %f2, metadata !"ogt",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <8 x i1> %cond, <8 x i32> %a, <8 x i32> %b
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_v8f32_oge_s(<8 x i32> %a, <8 x i32> %b, <8 x float> %f1, <8 x float> %f2) #0 {
+; AVX-32-LABEL: test_v8f32_oge_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-32, %esp
+; AVX-32-NEXT:    subl $32, %esp
+; AVX-32-NEXT:    vmovaps 8(%ebp), %ymm3
+; AVX-32-NEXT:    vcmpleps %ymm2, %ymm3, %ymm2
+; AVX-32-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v8f32_oge_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpleps %ymm2, %ymm3, %ymm2
+; AVX-64-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v8f32_oge_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-32, %esp
+; AVX512-32-NEXT:    subl $32, %esp
+; AVX512-32-NEXT:    vcmpgeps 8(%ebp), %ymm2, %k1
+; AVX512-32-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v8f32_oge_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpleps %ymm2, %ymm3, %k1
+; AVX512-64-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <8 x i1> @llvm.experimental.constrained.fcmps.v8f32(
+                                               <8 x float> %f1, <8 x float> %f2, metadata !"oge",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <8 x i1> %cond, <8 x i32> %a, <8 x i32> %b
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_v8f32_olt_s(<8 x i32> %a, <8 x i32> %b, <8 x float> %f1, <8 x float> %f2) #0 {
+; AVX-32-LABEL: test_v8f32_olt_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-32, %esp
+; AVX-32-NEXT:    subl $32, %esp
+; AVX-32-NEXT:    vcmpltps 8(%ebp), %ymm2, %ymm2
+; AVX-32-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v8f32_olt_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpltps %ymm3, %ymm2, %ymm2
+; AVX-64-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v8f32_olt_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-32, %esp
+; AVX512-32-NEXT:    subl $32, %esp
+; AVX512-32-NEXT:    vcmpltps 8(%ebp), %ymm2, %k1
+; AVX512-32-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v8f32_olt_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpltps %ymm3, %ymm2, %k1
+; AVX512-64-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <8 x i1> @llvm.experimental.constrained.fcmps.v8f32(
+                                               <8 x float> %f1, <8 x float> %f2, metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <8 x i1> %cond, <8 x i32> %a, <8 x i32> %b
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_v8f32_ole_s(<8 x i32> %a, <8 x i32> %b, <8 x float> %f1, <8 x float> %f2) #0 {
+; AVX-32-LABEL: test_v8f32_ole_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-32, %esp
+; AVX-32-NEXT:    subl $32, %esp
+; AVX-32-NEXT:    vcmpleps 8(%ebp), %ymm2, %ymm2
+; AVX-32-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v8f32_ole_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpleps %ymm3, %ymm2, %ymm2
+; AVX-64-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v8f32_ole_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-32, %esp
+; AVX512-32-NEXT:    subl $32, %esp
+; AVX512-32-NEXT:    vcmpleps 8(%ebp), %ymm2, %k1
+; AVX512-32-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v8f32_ole_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpleps %ymm3, %ymm2, %k1
+; AVX512-64-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <8 x i1> @llvm.experimental.constrained.fcmps.v8f32(
+                                               <8 x float> %f1, <8 x float> %f2, metadata !"ole",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <8 x i1> %cond, <8 x i32> %a, <8 x i32> %b
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_v8f32_one_s(<8 x i32> %a, <8 x i32> %b, <8 x float> %f1, <8 x float> %f2) #0 {
+; AVX-32-LABEL: test_v8f32_one_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-32, %esp
+; AVX-32-NEXT:    subl $32, %esp
+; AVX-32-NEXT:    vcmpneq_osps 8(%ebp), %ymm2, %ymm2
+; AVX-32-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v8f32_one_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpneq_osps %ymm3, %ymm2, %ymm2
+; AVX-64-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v8f32_one_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-32, %esp
+; AVX512-32-NEXT:    subl $32, %esp
+; AVX512-32-NEXT:    vcmpneq_osps 8(%ebp), %ymm2, %k1
+; AVX512-32-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v8f32_one_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpneq_osps %ymm3, %ymm2, %k1
+; AVX512-64-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <8 x i1> @llvm.experimental.constrained.fcmps.v8f32(
+                                               <8 x float> %f1, <8 x float> %f2, metadata !"one",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <8 x i1> %cond, <8 x i32> %a, <8 x i32> %b
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_v8f32_ord_s(<8 x i32> %a, <8 x i32> %b, <8 x float> %f1, <8 x float> %f2) #0 {
+; AVX-32-LABEL: test_v8f32_ord_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-32, %esp
+; AVX-32-NEXT:    subl $32, %esp
+; AVX-32-NEXT:    vcmpord_sps 8(%ebp), %ymm2, %ymm2
+; AVX-32-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v8f32_ord_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpord_sps %ymm3, %ymm2, %ymm2
+; AVX-64-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v8f32_ord_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-32, %esp
+; AVX512-32-NEXT:    subl $32, %esp
+; AVX512-32-NEXT:    vcmpord_sps 8(%ebp), %ymm2, %k1
+; AVX512-32-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v8f32_ord_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpord_sps %ymm3, %ymm2, %k1
+; AVX512-64-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <8 x i1> @llvm.experimental.constrained.fcmps.v8f32(
+                                               <8 x float> %f1, <8 x float> %f2, metadata !"ord",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <8 x i1> %cond, <8 x i32> %a, <8 x i32> %b
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_v8f32_ueq_s(<8 x i32> %a, <8 x i32> %b, <8 x float> %f1, <8 x float> %f2) #0 {
+; AVX-32-LABEL: test_v8f32_ueq_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-32, %esp
+; AVX-32-NEXT:    subl $32, %esp
+; AVX-32-NEXT:    vcmpeq_usps 8(%ebp), %ymm2, %ymm2
+; AVX-32-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v8f32_ueq_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpeq_usps %ymm3, %ymm2, %ymm2
+; AVX-64-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v8f32_ueq_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-32, %esp
+; AVX512-32-NEXT:    subl $32, %esp
+; AVX512-32-NEXT:    vcmpeq_usps 8(%ebp), %ymm2, %k1
+; AVX512-32-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v8f32_ueq_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpeq_usps %ymm3, %ymm2, %k1
+; AVX512-64-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <8 x i1> @llvm.experimental.constrained.fcmps.v8f32(
+                                               <8 x float> %f1, <8 x float> %f2, metadata !"ueq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <8 x i1> %cond, <8 x i32> %a, <8 x i32> %b
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_v8f32_ugt_s(<8 x i32> %a, <8 x i32> %b, <8 x float> %f1, <8 x float> %f2) #0 {
+; AVX-32-LABEL: test_v8f32_ugt_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-32, %esp
+; AVX-32-NEXT:    subl $32, %esp
+; AVX-32-NEXT:    vcmpnleps 8(%ebp), %ymm2, %ymm2
+; AVX-32-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v8f32_ugt_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpnleps %ymm3, %ymm2, %ymm2
+; AVX-64-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v8f32_ugt_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-32, %esp
+; AVX512-32-NEXT:    subl $32, %esp
+; AVX512-32-NEXT:    vcmpnleps 8(%ebp), %ymm2, %k1
+; AVX512-32-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v8f32_ugt_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpnleps %ymm3, %ymm2, %k1
+; AVX512-64-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <8 x i1> @llvm.experimental.constrained.fcmps.v8f32(
+                                               <8 x float> %f1, <8 x float> %f2, metadata !"ugt",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <8 x i1> %cond, <8 x i32> %a, <8 x i32> %b
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_v8f32_uge_s(<8 x i32> %a, <8 x i32> %b, <8 x float> %f1, <8 x float> %f2) #0 {
+; AVX-32-LABEL: test_v8f32_uge_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-32, %esp
+; AVX-32-NEXT:    subl $32, %esp
+; AVX-32-NEXT:    vcmpnltps 8(%ebp), %ymm2, %ymm2
+; AVX-32-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v8f32_uge_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpnltps %ymm3, %ymm2, %ymm2
+; AVX-64-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v8f32_uge_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-32, %esp
+; AVX512-32-NEXT:    subl $32, %esp
+; AVX512-32-NEXT:    vcmpnltps 8(%ebp), %ymm2, %k1
+; AVX512-32-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v8f32_uge_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpnltps %ymm3, %ymm2, %k1
+; AVX512-64-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <8 x i1> @llvm.experimental.constrained.fcmps.v8f32(
+                                               <8 x float> %f1, <8 x float> %f2, metadata !"uge",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <8 x i1> %cond, <8 x i32> %a, <8 x i32> %b
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_v8f32_ult_s(<8 x i32> %a, <8 x i32> %b, <8 x float> %f1, <8 x float> %f2) #0 {
+; AVX-32-LABEL: test_v8f32_ult_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-32, %esp
+; AVX-32-NEXT:    subl $32, %esp
+; AVX-32-NEXT:    vmovaps 8(%ebp), %ymm3
+; AVX-32-NEXT:    vcmpnleps %ymm2, %ymm3, %ymm2
+; AVX-32-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v8f32_ult_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpnleps %ymm2, %ymm3, %ymm2
+; AVX-64-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v8f32_ult_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-32, %esp
+; AVX512-32-NEXT:    subl $32, %esp
+; AVX512-32-NEXT:    vcmpngeps 8(%ebp), %ymm2, %k1
+; AVX512-32-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v8f32_ult_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpnleps %ymm2, %ymm3, %k1
+; AVX512-64-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <8 x i1> @llvm.experimental.constrained.fcmps.v8f32(
+                                               <8 x float> %f1, <8 x float> %f2, metadata !"ult",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <8 x i1> %cond, <8 x i32> %a, <8 x i32> %b
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_v8f32_ule_s(<8 x i32> %a, <8 x i32> %b, <8 x float> %f1, <8 x float> %f2) #0 {
+; AVX-32-LABEL: test_v8f32_ule_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-32, %esp
+; AVX-32-NEXT:    subl $32, %esp
+; AVX-32-NEXT:    vmovaps 8(%ebp), %ymm3
+; AVX-32-NEXT:    vcmpnltps %ymm2, %ymm3, %ymm2
+; AVX-32-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v8f32_ule_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpnltps %ymm2, %ymm3, %ymm2
+; AVX-64-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v8f32_ule_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-32, %esp
+; AVX512-32-NEXT:    subl $32, %esp
+; AVX512-32-NEXT:    vcmpngtps 8(%ebp), %ymm2, %k1
+; AVX512-32-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v8f32_ule_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpnltps %ymm2, %ymm3, %k1
+; AVX512-64-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <8 x i1> @llvm.experimental.constrained.fcmps.v8f32(
+                                               <8 x float> %f1, <8 x float> %f2, metadata !"ule",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <8 x i1> %cond, <8 x i32> %a, <8 x i32> %b
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_v8f32_une_s(<8 x i32> %a, <8 x i32> %b, <8 x float> %f1, <8 x float> %f2) #0 {
+; AVX-32-LABEL: test_v8f32_une_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-32, %esp
+; AVX-32-NEXT:    subl $32, %esp
+; AVX-32-NEXT:    vcmpneq_usps 8(%ebp), %ymm2, %ymm2
+; AVX-32-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v8f32_une_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpneq_usps %ymm3, %ymm2, %ymm2
+; AVX-64-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v8f32_une_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-32, %esp
+; AVX512-32-NEXT:    subl $32, %esp
+; AVX512-32-NEXT:    vcmpneq_usps 8(%ebp), %ymm2, %k1
+; AVX512-32-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v8f32_une_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpneq_usps %ymm3, %ymm2, %k1
+; AVX512-64-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <8 x i1> @llvm.experimental.constrained.fcmps.v8f32(
+                                               <8 x float> %f1, <8 x float> %f2, metadata !"une",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <8 x i1> %cond, <8 x i32> %a, <8 x i32> %b
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_v8f32_uno_s(<8 x i32> %a, <8 x i32> %b, <8 x float> %f1, <8 x float> %f2) #0 {
+; AVX-32-LABEL: test_v8f32_uno_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-32, %esp
+; AVX-32-NEXT:    subl $32, %esp
+; AVX-32-NEXT:    vcmpunord_sps 8(%ebp), %ymm2, %ymm2
+; AVX-32-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v8f32_uno_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpunord_sps %ymm3, %ymm2, %ymm2
+; AVX-64-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v8f32_uno_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-32, %esp
+; AVX512-32-NEXT:    subl $32, %esp
+; AVX512-32-NEXT:    vcmpunord_sps 8(%ebp), %ymm2, %k1
+; AVX512-32-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v8f32_uno_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpunord_sps %ymm3, %ymm2, %k1
+; AVX512-64-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <8 x i1> @llvm.experimental.constrained.fcmps.v8f32(
+                                               <8 x float> %f1, <8 x float> %f2, metadata !"uno",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <8 x i1> %cond, <8 x i32> %a, <8 x i32> %b
+  ret <8 x i32> %res
+}
+
+define <4 x i64> @test_v4f64_oeq_s(<4 x i64> %a, <4 x i64> %b, <4 x double> %f1, <4 x double> %f2) #0 {
+; AVX-32-LABEL: test_v4f64_oeq_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-32, %esp
+; AVX-32-NEXT:    subl $32, %esp
+; AVX-32-NEXT:    vcmpeq_ospd 8(%ebp), %ymm2, %ymm2
+; AVX-32-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v4f64_oeq_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpeq_ospd %ymm3, %ymm2, %ymm2
+; AVX-64-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v4f64_oeq_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-32, %esp
+; AVX512-32-NEXT:    subl $32, %esp
+; AVX512-32-NEXT:    vcmpeq_ospd 8(%ebp), %ymm2, %k1
+; AVX512-32-NEXT:    vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v4f64_oeq_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpeq_ospd %ymm3, %ymm2, %k1
+; AVX512-64-NEXT:    vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <4 x i1> @llvm.experimental.constrained.fcmps.v4f64(
+                                               <4 x double> %f1, <4 x double> %f2, metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <4 x i1> %cond, <4 x i64> %a, <4 x i64> %b
+  ret <4 x i64> %res
+}
+
+define <4 x i64> @test_v4f64_ogt_s(<4 x i64> %a, <4 x i64> %b, <4 x double> %f1, <4 x double> %f2) #0 {
+; AVX-32-LABEL: test_v4f64_ogt_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-32, %esp
+; AVX-32-NEXT:    subl $32, %esp
+; AVX-32-NEXT:    vmovapd 8(%ebp), %ymm3
+; AVX-32-NEXT:    vcmpltpd %ymm2, %ymm3, %ymm2
+; AVX-32-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v4f64_ogt_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpltpd %ymm2, %ymm3, %ymm2
+; AVX-64-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v4f64_ogt_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-32, %esp
+; AVX512-32-NEXT:    subl $32, %esp
+; AVX512-32-NEXT:    vcmpgtpd 8(%ebp), %ymm2, %k1
+; AVX512-32-NEXT:    vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v4f64_ogt_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpltpd %ymm2, %ymm3, %k1
+; AVX512-64-NEXT:    vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <4 x i1> @llvm.experimental.constrained.fcmps.v4f64(
+                                               <4 x double> %f1, <4 x double> %f2, metadata !"ogt",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <4 x i1> %cond, <4 x i64> %a, <4 x i64> %b
+  ret <4 x i64> %res
+}
+
+define <4 x i64> @test_v4f64_oge_s(<4 x i64> %a, <4 x i64> %b, <4 x double> %f1, <4 x double> %f2) #0 {
+; AVX-32-LABEL: test_v4f64_oge_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-32, %esp
+; AVX-32-NEXT:    subl $32, %esp
+; AVX-32-NEXT:    vmovapd 8(%ebp), %ymm3
+; AVX-32-NEXT:    vcmplepd %ymm2, %ymm3, %ymm2
+; AVX-32-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v4f64_oge_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmplepd %ymm2, %ymm3, %ymm2
+; AVX-64-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v4f64_oge_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-32, %esp
+; AVX512-32-NEXT:    subl $32, %esp
+; AVX512-32-NEXT:    vcmpgepd 8(%ebp), %ymm2, %k1
+; AVX512-32-NEXT:    vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v4f64_oge_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmplepd %ymm2, %ymm3, %k1
+; AVX512-64-NEXT:    vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <4 x i1> @llvm.experimental.constrained.fcmps.v4f64(
+                                               <4 x double> %f1, <4 x double> %f2, metadata !"oge",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <4 x i1> %cond, <4 x i64> %a, <4 x i64> %b
+  ret <4 x i64> %res
+}
+
+define <4 x i64> @test_v4f64_olt_s(<4 x i64> %a, <4 x i64> %b, <4 x double> %f1, <4 x double> %f2) #0 {
+; AVX-32-LABEL: test_v4f64_olt_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-32, %esp
+; AVX-32-NEXT:    subl $32, %esp
+; AVX-32-NEXT:    vcmpltpd 8(%ebp), %ymm2, %ymm2
+; AVX-32-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v4f64_olt_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpltpd %ymm3, %ymm2, %ymm2
+; AVX-64-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v4f64_olt_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-32, %esp
+; AVX512-32-NEXT:    subl $32, %esp
+; AVX512-32-NEXT:    vcmpltpd 8(%ebp), %ymm2, %k1
+; AVX512-32-NEXT:    vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v4f64_olt_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpltpd %ymm3, %ymm2, %k1
+; AVX512-64-NEXT:    vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <4 x i1> @llvm.experimental.constrained.fcmps.v4f64(
+                                               <4 x double> %f1, <4 x double> %f2, metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <4 x i1> %cond, <4 x i64> %a, <4 x i64> %b
+  ret <4 x i64> %res
+}
+
+define <4 x i64> @test_v4f64_ole_s(<4 x i64> %a, <4 x i64> %b, <4 x double> %f1, <4 x double> %f2) #0 {
+; AVX-32-LABEL: test_v4f64_ole_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-32, %esp
+; AVX-32-NEXT:    subl $32, %esp
+; AVX-32-NEXT:    vcmplepd 8(%ebp), %ymm2, %ymm2
+; AVX-32-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v4f64_ole_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmplepd %ymm3, %ymm2, %ymm2
+; AVX-64-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v4f64_ole_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-32, %esp
+; AVX512-32-NEXT:    subl $32, %esp
+; AVX512-32-NEXT:    vcmplepd 8(%ebp), %ymm2, %k1
+; AVX512-32-NEXT:    vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v4f64_ole_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmplepd %ymm3, %ymm2, %k1
+; AVX512-64-NEXT:    vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <4 x i1> @llvm.experimental.constrained.fcmps.v4f64(
+                                               <4 x double> %f1, <4 x double> %f2, metadata !"ole",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <4 x i1> %cond, <4 x i64> %a, <4 x i64> %b
+  ret <4 x i64> %res
+}
+
+define <4 x i64> @test_v4f64_one_s(<4 x i64> %a, <4 x i64> %b, <4 x double> %f1, <4 x double> %f2) #0 {
+; AVX-32-LABEL: test_v4f64_one_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-32, %esp
+; AVX-32-NEXT:    subl $32, %esp
+; AVX-32-NEXT:    vcmpneq_ospd 8(%ebp), %ymm2, %ymm2
+; AVX-32-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v4f64_one_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpneq_ospd %ymm3, %ymm2, %ymm2
+; AVX-64-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v4f64_one_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-32, %esp
+; AVX512-32-NEXT:    subl $32, %esp
+; AVX512-32-NEXT:    vcmpneq_ospd 8(%ebp), %ymm2, %k1
+; AVX512-32-NEXT:    vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v4f64_one_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpneq_ospd %ymm3, %ymm2, %k1
+; AVX512-64-NEXT:    vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <4 x i1> @llvm.experimental.constrained.fcmps.v4f64(
+                                               <4 x double> %f1, <4 x double> %f2, metadata !"one",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <4 x i1> %cond, <4 x i64> %a, <4 x i64> %b
+  ret <4 x i64> %res
+}
+
+define <4 x i64> @test_v4f64_ord_s(<4 x i64> %a, <4 x i64> %b, <4 x double> %f1, <4 x double> %f2) #0 {
+; AVX-32-LABEL: test_v4f64_ord_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-32, %esp
+; AVX-32-NEXT:    subl $32, %esp
+; AVX-32-NEXT:    vcmpord_spd 8(%ebp), %ymm2, %ymm2
+; AVX-32-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v4f64_ord_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpord_spd %ymm3, %ymm2, %ymm2
+; AVX-64-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v4f64_ord_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-32, %esp
+; AVX512-32-NEXT:    subl $32, %esp
+; AVX512-32-NEXT:    vcmpord_spd 8(%ebp), %ymm2, %k1
+; AVX512-32-NEXT:    vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v4f64_ord_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpord_spd %ymm3, %ymm2, %k1
+; AVX512-64-NEXT:    vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <4 x i1> @llvm.experimental.constrained.fcmps.v4f64(
+                                               <4 x double> %f1, <4 x double> %f2, metadata !"ord",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <4 x i1> %cond, <4 x i64> %a, <4 x i64> %b
+  ret <4 x i64> %res
+}
+
+define <4 x i64> @test_v4f64_ueq_s(<4 x i64> %a, <4 x i64> %b, <4 x double> %f1, <4 x double> %f2) #0 {
+; AVX-32-LABEL: test_v4f64_ueq_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-32, %esp
+; AVX-32-NEXT:    subl $32, %esp
+; AVX-32-NEXT:    vcmpeq_uspd 8(%ebp), %ymm2, %ymm2
+; AVX-32-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v4f64_ueq_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpeq_uspd %ymm3, %ymm2, %ymm2
+; AVX-64-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v4f64_ueq_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-32, %esp
+; AVX512-32-NEXT:    subl $32, %esp
+; AVX512-32-NEXT:    vcmpeq_uspd 8(%ebp), %ymm2, %k1
+; AVX512-32-NEXT:    vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v4f64_ueq_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpeq_uspd %ymm3, %ymm2, %k1
+; AVX512-64-NEXT:    vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <4 x i1> @llvm.experimental.constrained.fcmps.v4f64(
+                                               <4 x double> %f1, <4 x double> %f2, metadata !"ueq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <4 x i1> %cond, <4 x i64> %a, <4 x i64> %b
+  ret <4 x i64> %res
+}
+
+define <4 x i64> @test_v4f64_ugt_s(<4 x i64> %a, <4 x i64> %b, <4 x double> %f1, <4 x double> %f2) #0 {
+; AVX-32-LABEL: test_v4f64_ugt_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-32, %esp
+; AVX-32-NEXT:    subl $32, %esp
+; AVX-32-NEXT:    vcmpnlepd 8(%ebp), %ymm2, %ymm2
+; AVX-32-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v4f64_ugt_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpnlepd %ymm3, %ymm2, %ymm2
+; AVX-64-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v4f64_ugt_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-32, %esp
+; AVX512-32-NEXT:    subl $32, %esp
+; AVX512-32-NEXT:    vcmpnlepd 8(%ebp), %ymm2, %k1
+; AVX512-32-NEXT:    vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v4f64_ugt_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpnlepd %ymm3, %ymm2, %k1
+; AVX512-64-NEXT:    vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <4 x i1> @llvm.experimental.constrained.fcmps.v4f64(
+                                               <4 x double> %f1, <4 x double> %f2, metadata !"ugt",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <4 x i1> %cond, <4 x i64> %a, <4 x i64> %b
+  ret <4 x i64> %res
+}
+
+define <4 x i64> @test_v4f64_uge_s(<4 x i64> %a, <4 x i64> %b, <4 x double> %f1, <4 x double> %f2) #0 {
+; AVX-32-LABEL: test_v4f64_uge_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-32, %esp
+; AVX-32-NEXT:    subl $32, %esp
+; AVX-32-NEXT:    vcmpnltpd 8(%ebp), %ymm2, %ymm2
+; AVX-32-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v4f64_uge_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpnltpd %ymm3, %ymm2, %ymm2
+; AVX-64-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v4f64_uge_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-32, %esp
+; AVX512-32-NEXT:    subl $32, %esp
+; AVX512-32-NEXT:    vcmpnltpd 8(%ebp), %ymm2, %k1
+; AVX512-32-NEXT:    vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v4f64_uge_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpnltpd %ymm3, %ymm2, %k1
+; AVX512-64-NEXT:    vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <4 x i1> @llvm.experimental.constrained.fcmps.v4f64(
+                                               <4 x double> %f1, <4 x double> %f2, metadata !"uge",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <4 x i1> %cond, <4 x i64> %a, <4 x i64> %b
+  ret <4 x i64> %res
+}
+
+define <4 x i64> @test_v4f64_ult_s(<4 x i64> %a, <4 x i64> %b, <4 x double> %f1, <4 x double> %f2) #0 {
+; AVX-32-LABEL: test_v4f64_ult_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-32, %esp
+; AVX-32-NEXT:    subl $32, %esp
+; AVX-32-NEXT:    vmovapd 8(%ebp), %ymm3
+; AVX-32-NEXT:    vcmpnlepd %ymm2, %ymm3, %ymm2
+; AVX-32-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v4f64_ult_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpnlepd %ymm2, %ymm3, %ymm2
+; AVX-64-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v4f64_ult_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-32, %esp
+; AVX512-32-NEXT:    subl $32, %esp
+; AVX512-32-NEXT:    vcmpngepd 8(%ebp), %ymm2, %k1
+; AVX512-32-NEXT:    vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v4f64_ult_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpnlepd %ymm2, %ymm3, %k1
+; AVX512-64-NEXT:    vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <4 x i1> @llvm.experimental.constrained.fcmps.v4f64(
+                                               <4 x double> %f1, <4 x double> %f2, metadata !"ult",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <4 x i1> %cond, <4 x i64> %a, <4 x i64> %b
+  ret <4 x i64> %res
+}
+
+define <4 x i64> @test_v4f64_ule_s(<4 x i64> %a, <4 x i64> %b, <4 x double> %f1, <4 x double> %f2) #0 {
+; AVX-32-LABEL: test_v4f64_ule_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-32, %esp
+; AVX-32-NEXT:    subl $32, %esp
+; AVX-32-NEXT:    vmovapd 8(%ebp), %ymm3
+; AVX-32-NEXT:    vcmpnltpd %ymm2, %ymm3, %ymm2
+; AVX-32-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v4f64_ule_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpnltpd %ymm2, %ymm3, %ymm2
+; AVX-64-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v4f64_ule_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-32, %esp
+; AVX512-32-NEXT:    subl $32, %esp
+; AVX512-32-NEXT:    vcmpngtpd 8(%ebp), %ymm2, %k1
+; AVX512-32-NEXT:    vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v4f64_ule_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpnltpd %ymm2, %ymm3, %k1
+; AVX512-64-NEXT:    vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <4 x i1> @llvm.experimental.constrained.fcmps.v4f64(
+                                               <4 x double> %f1, <4 x double> %f2, metadata !"ule",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <4 x i1> %cond, <4 x i64> %a, <4 x i64> %b
+  ret <4 x i64> %res
+}
+
+define <4 x i64> @test_v4f64_une_s(<4 x i64> %a, <4 x i64> %b, <4 x double> %f1, <4 x double> %f2) #0 {
+; AVX-32-LABEL: test_v4f64_une_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-32, %esp
+; AVX-32-NEXT:    subl $32, %esp
+; AVX-32-NEXT:    vcmpneq_uspd 8(%ebp), %ymm2, %ymm2
+; AVX-32-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v4f64_une_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpneq_uspd %ymm3, %ymm2, %ymm2
+; AVX-64-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v4f64_une_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-32, %esp
+; AVX512-32-NEXT:    subl $32, %esp
+; AVX512-32-NEXT:    vcmpneq_uspd 8(%ebp), %ymm2, %k1
+; AVX512-32-NEXT:    vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v4f64_une_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpneq_uspd %ymm3, %ymm2, %k1
+; AVX512-64-NEXT:    vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <4 x i1> @llvm.experimental.constrained.fcmps.v4f64(
+                                               <4 x double> %f1, <4 x double> %f2, metadata !"une",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <4 x i1> %cond, <4 x i64> %a, <4 x i64> %b
+  ret <4 x i64> %res
+}
+
+define <4 x i64> @test_v4f64_uno_s(<4 x i64> %a, <4 x i64> %b, <4 x double> %f1, <4 x double> %f2) #0 {
+; AVX-32-LABEL: test_v4f64_uno_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-32, %esp
+; AVX-32-NEXT:    subl $32, %esp
+; AVX-32-NEXT:    vcmpunord_spd 8(%ebp), %ymm2, %ymm2
+; AVX-32-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v4f64_uno_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcmpunord_spd %ymm3, %ymm2, %ymm2
+; AVX-64-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v4f64_uno_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-32, %esp
+; AVX512-32-NEXT:    subl $32, %esp
+; AVX512-32-NEXT:    vcmpunord_spd 8(%ebp), %ymm2, %k1
+; AVX512-32-NEXT:    vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v4f64_uno_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpunord_spd %ymm3, %ymm2, %k1
+; AVX512-64-NEXT:    vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <4 x i1> @llvm.experimental.constrained.fcmps.v4f64(
+                                               <4 x double> %f1, <4 x double> %f2, metadata !"uno",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <4 x i1> %cond, <4 x i64> %a, <4 x i64> %b
+  ret <4 x i64> %res
+}
+
+attributes #0 = { strictfp nounwind }
+
+declare <8 x i1> @llvm.experimental.constrained.fcmp.v8f32(<8 x float>, <8 x float>, metadata, metadata)
+declare <4 x i1> @llvm.experimental.constrained.fcmp.v4f64(<4 x double>, <4 x double>, metadata, metadata)
+declare <8 x i1> @llvm.experimental.constrained.fcmps.v8f32(<8 x float>, <8 x float>, metadata, metadata)
+declare <4 x i1> @llvm.experimental.constrained.fcmps.v4f64(<4 x double>, <4 x double>, metadata, metadata)
diff --git a/llvm/test/CodeGen/X86/vec-strict-cmp-512.ll b/llvm/test/CodeGen/X86/vec-strict-cmp-512.ll
new file mode 100644
index 000000000..25b652573
--- /dev/null
+++ b/llvm/test/CodeGen/X86/vec-strict-cmp-512.ll
@@ -0,0 +1,1410 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx512f -O3 | FileCheck %s --check-prefixes=CHECK,AVX512-32
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f -O3 | FileCheck %s --check-prefixes=CHECK,AVX512-64
+
+define <16 x i32> @test_v16f32_oeq_q(<16 x i32> %a, <16 x i32> %b, <16 x float> %f1, <16 x float> %f2) #0 {
+; AVX512-32-LABEL: test_v16f32_oeq_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-64, %esp
+; AVX512-32-NEXT:    subl $64, %esp
+; AVX512-32-NEXT:    vcmpeqps 8(%ebp), %zmm2, %k1
+; AVX512-32-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v16f32_oeq_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpeqps %zmm3, %zmm2, %k1
+; AVX512-64-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <16 x i1> @llvm.experimental.constrained.fcmp.v16f32(
+                                               <16 x float> %f1, <16 x float> %f2, metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <16 x i1> %cond, <16 x i32> %a, <16 x i32> %b
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_v16f32_ogt_q(<16 x i32> %a, <16 x i32> %b, <16 x float> %f1, <16 x float> %f2) #0 {
+; AVX512-32-LABEL: test_v16f32_ogt_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-64, %esp
+; AVX512-32-NEXT:    subl $64, %esp
+; AVX512-32-NEXT:    vcmpgt_oqps 8(%ebp), %zmm2, %k1
+; AVX512-32-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v16f32_ogt_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmplt_oqps %zmm2, %zmm3, %k1
+; AVX512-64-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <16 x i1> @llvm.experimental.constrained.fcmp.v16f32(
+                                               <16 x float> %f1, <16 x float> %f2, metadata !"ogt",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <16 x i1> %cond, <16 x i32> %a, <16 x i32> %b
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_v16f32_oge_q(<16 x i32> %a, <16 x i32> %b, <16 x float> %f1, <16 x float> %f2) #0 {
+; AVX512-32-LABEL: test_v16f32_oge_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-64, %esp
+; AVX512-32-NEXT:    subl $64, %esp
+; AVX512-32-NEXT:    vcmpge_oqps 8(%ebp), %zmm2, %k1
+; AVX512-32-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v16f32_oge_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmple_oqps %zmm2, %zmm3, %k1
+; AVX512-64-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <16 x i1> @llvm.experimental.constrained.fcmp.v16f32(
+                                               <16 x float> %f1, <16 x float> %f2, metadata !"oge",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <16 x i1> %cond, <16 x i32> %a, <16 x i32> %b
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_v16f32_olt_q(<16 x i32> %a, <16 x i32> %b, <16 x float> %f1, <16 x float> %f2) #0 {
+; AVX512-32-LABEL: test_v16f32_olt_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-64, %esp
+; AVX512-32-NEXT:    subl $64, %esp
+; AVX512-32-NEXT:    vcmplt_oqps 8(%ebp), %zmm2, %k1
+; AVX512-32-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v16f32_olt_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmplt_oqps %zmm3, %zmm2, %k1
+; AVX512-64-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <16 x i1> @llvm.experimental.constrained.fcmp.v16f32(
+                                               <16 x float> %f1, <16 x float> %f2, metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <16 x i1> %cond, <16 x i32> %a, <16 x i32> %b
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_v16f32_ole_q(<16 x i32> %a, <16 x i32> %b, <16 x float> %f1, <16 x float> %f2) #0 {
+; AVX512-32-LABEL: test_v16f32_ole_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-64, %esp
+; AVX512-32-NEXT:    subl $64, %esp
+; AVX512-32-NEXT:    vcmple_oqps 8(%ebp), %zmm2, %k1
+; AVX512-32-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v16f32_ole_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmple_oqps %zmm3, %zmm2, %k1
+; AVX512-64-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <16 x i1> @llvm.experimental.constrained.fcmp.v16f32(
+                                               <16 x float> %f1, <16 x float> %f2, metadata !"ole",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <16 x i1> %cond, <16 x i32> %a, <16 x i32> %b
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_v16f32_one_q(<16 x i32> %a, <16 x i32> %b, <16 x float> %f1, <16 x float> %f2) #0 {
+; AVX512-32-LABEL: test_v16f32_one_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-64, %esp
+; AVX512-32-NEXT:    subl $64, %esp
+; AVX512-32-NEXT:    vcmpneq_oqps 8(%ebp), %zmm2, %k1
+; AVX512-32-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v16f32_one_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpneq_oqps %zmm3, %zmm2, %k1
+; AVX512-64-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <16 x i1> @llvm.experimental.constrained.fcmp.v16f32(
+                                               <16 x float> %f1, <16 x float> %f2, metadata !"one",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <16 x i1> %cond, <16 x i32> %a, <16 x i32> %b
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_v16f32_ord_q(<16 x i32> %a, <16 x i32> %b, <16 x float> %f1, <16 x float> %f2) #0 {
+; AVX512-32-LABEL: test_v16f32_ord_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-64, %esp
+; AVX512-32-NEXT:    subl $64, %esp
+; AVX512-32-NEXT:    vcmpordps 8(%ebp), %zmm2, %k1
+; AVX512-32-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v16f32_ord_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpordps %zmm3, %zmm2, %k1
+; AVX512-64-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <16 x i1> @llvm.experimental.constrained.fcmp.v16f32(
+                                               <16 x float> %f1, <16 x float> %f2, metadata !"ord",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <16 x i1> %cond, <16 x i32> %a, <16 x i32> %b
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_v16f32_ueq_q(<16 x i32> %a, <16 x i32> %b, <16 x float> %f1, <16 x float> %f2) #0 {
+; AVX512-32-LABEL: test_v16f32_ueq_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-64, %esp
+; AVX512-32-NEXT:    subl $64, %esp
+; AVX512-32-NEXT:    vcmpeq_uqps 8(%ebp), %zmm2, %k1
+; AVX512-32-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v16f32_ueq_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpeq_uqps %zmm3, %zmm2, %k1
+; AVX512-64-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <16 x i1> @llvm.experimental.constrained.fcmp.v16f32(
+                                               <16 x float> %f1, <16 x float> %f2, metadata !"ueq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <16 x i1> %cond, <16 x i32> %a, <16 x i32> %b
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_v16f32_ugt_q(<16 x i32> %a, <16 x i32> %b, <16 x float> %f1, <16 x float> %f2) #0 {
+; AVX512-32-LABEL: test_v16f32_ugt_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-64, %esp
+; AVX512-32-NEXT:    subl $64, %esp
+; AVX512-32-NEXT:    vcmpnle_uqps 8(%ebp), %zmm2, %k1
+; AVX512-32-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v16f32_ugt_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpnle_uqps %zmm3, %zmm2, %k1
+; AVX512-64-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <16 x i1> @llvm.experimental.constrained.fcmp.v16f32(
+                                               <16 x float> %f1, <16 x float> %f2, metadata !"ugt",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <16 x i1> %cond, <16 x i32> %a, <16 x i32> %b
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_v16f32_uge_q(<16 x i32> %a, <16 x i32> %b, <16 x float> %f1, <16 x float> %f2) #0 {
+; AVX512-32-LABEL: test_v16f32_uge_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-64, %esp
+; AVX512-32-NEXT:    subl $64, %esp
+; AVX512-32-NEXT:    vcmpnlt_uqps 8(%ebp), %zmm2, %k1
+; AVX512-32-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v16f32_uge_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpnlt_uqps %zmm3, %zmm2, %k1
+; AVX512-64-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <16 x i1> @llvm.experimental.constrained.fcmp.v16f32(
+                                               <16 x float> %f1, <16 x float> %f2, metadata !"uge",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <16 x i1> %cond, <16 x i32> %a, <16 x i32> %b
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_v16f32_ult_q(<16 x i32> %a, <16 x i32> %b, <16 x float> %f1, <16 x float> %f2) #0 {
+; AVX512-32-LABEL: test_v16f32_ult_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-64, %esp
+; AVX512-32-NEXT:    subl $64, %esp
+; AVX512-32-NEXT:    vcmpnge_uqps 8(%ebp), %zmm2, %k1
+; AVX512-32-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v16f32_ult_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpnle_uqps %zmm2, %zmm3, %k1
+; AVX512-64-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <16 x i1> @llvm.experimental.constrained.fcmp.v16f32(
+                                               <16 x float> %f1, <16 x float> %f2, metadata !"ult",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <16 x i1> %cond, <16 x i32> %a, <16 x i32> %b
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_v16f32_ule_q(<16 x i32> %a, <16 x i32> %b, <16 x float> %f1, <16 x float> %f2) #0 {
+; AVX512-32-LABEL: test_v16f32_ule_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-64, %esp
+; AVX512-32-NEXT:    subl $64, %esp
+; AVX512-32-NEXT:    vcmpngt_uqps 8(%ebp), %zmm2, %k1
+; AVX512-32-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v16f32_ule_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpnlt_uqps %zmm2, %zmm3, %k1
+; AVX512-64-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <16 x i1> @llvm.experimental.constrained.fcmp.v16f32(
+                                               <16 x float> %f1, <16 x float> %f2, metadata !"ule",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <16 x i1> %cond, <16 x i32> %a, <16 x i32> %b
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_v16f32_une_q(<16 x i32> %a, <16 x i32> %b, <16 x float> %f1, <16 x float> %f2) #0 {
+; AVX512-32-LABEL: test_v16f32_une_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-64, %esp
+; AVX512-32-NEXT:    subl $64, %esp
+; AVX512-32-NEXT:    vcmpneqps 8(%ebp), %zmm2, %k1
+; AVX512-32-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v16f32_une_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpneqps %zmm3, %zmm2, %k1
+; AVX512-64-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <16 x i1> @llvm.experimental.constrained.fcmp.v16f32(
+                                               <16 x float> %f1, <16 x float> %f2, metadata !"une",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <16 x i1> %cond, <16 x i32> %a, <16 x i32> %b
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_v16f32_uno_q(<16 x i32> %a, <16 x i32> %b, <16 x float> %f1, <16 x float> %f2) #0 {
+; AVX512-32-LABEL: test_v16f32_uno_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-64, %esp
+; AVX512-32-NEXT:    subl $64, %esp
+; AVX512-32-NEXT:    vcmpunordps 8(%ebp), %zmm2, %k1
+; AVX512-32-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v16f32_uno_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpunordps %zmm3, %zmm2, %k1
+; AVX512-64-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <16 x i1> @llvm.experimental.constrained.fcmp.v16f32(
+                                               <16 x float> %f1, <16 x float> %f2, metadata !"uno",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <16 x i1> %cond, <16 x i32> %a, <16 x i32> %b
+  ret <16 x i32> %res
+}
+
+define <8 x i64> @test_v8f64_oeq_q(<8 x i64> %a, <8 x i64> %b, <8 x double> %f1, <8 x double> %f2) #0 {
+; AVX512-32-LABEL: test_v8f64_oeq_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-64, %esp
+; AVX512-32-NEXT:    subl $64, %esp
+; AVX512-32-NEXT:    vcmpeqpd 8(%ebp), %zmm2, %k1
+; AVX512-32-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v8f64_oeq_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpeqpd %zmm3, %zmm2, %k1
+; AVX512-64-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <8 x i1> @llvm.experimental.constrained.fcmp.v8f64(
+                                               <8 x double> %f1, <8 x double> %f2, metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <8 x i1> %cond, <8 x i64> %a, <8 x i64> %b
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_v8f64_ogt_q(<8 x i64> %a, <8 x i64> %b, <8 x double> %f1, <8 x double> %f2) #0 {
+; AVX512-32-LABEL: test_v8f64_ogt_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-64, %esp
+; AVX512-32-NEXT:    subl $64, %esp
+; AVX512-32-NEXT:    vcmpgt_oqpd 8(%ebp), %zmm2, %k1
+; AVX512-32-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v8f64_ogt_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmplt_oqpd %zmm2, %zmm3, %k1
+; AVX512-64-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <8 x i1> @llvm.experimental.constrained.fcmp.v8f64(
+                                               <8 x double> %f1, <8 x double> %f2, metadata !"ogt",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <8 x i1> %cond, <8 x i64> %a, <8 x i64> %b
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_v8f64_oge_q(<8 x i64> %a, <8 x i64> %b, <8 x double> %f1, <8 x double> %f2) #0 {
+; AVX512-32-LABEL: test_v8f64_oge_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-64, %esp
+; AVX512-32-NEXT:    subl $64, %esp
+; AVX512-32-NEXT:    vcmpge_oqpd 8(%ebp), %zmm2, %k1
+; AVX512-32-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v8f64_oge_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmple_oqpd %zmm2, %zmm3, %k1
+; AVX512-64-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <8 x i1> @llvm.experimental.constrained.fcmp.v8f64(
+                                               <8 x double> %f1, <8 x double> %f2, metadata !"oge",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <8 x i1> %cond, <8 x i64> %a, <8 x i64> %b
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_v8f64_olt_q(<8 x i64> %a, <8 x i64> %b, <8 x double> %f1, <8 x double> %f2) #0 {
+; AVX512-32-LABEL: test_v8f64_olt_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-64, %esp
+; AVX512-32-NEXT:    subl $64, %esp
+; AVX512-32-NEXT:    vcmplt_oqpd 8(%ebp), %zmm2, %k1
+; AVX512-32-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v8f64_olt_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmplt_oqpd %zmm3, %zmm2, %k1
+; AVX512-64-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <8 x i1> @llvm.experimental.constrained.fcmp.v8f64(
+                                               <8 x double> %f1, <8 x double> %f2, metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <8 x i1> %cond, <8 x i64> %a, <8 x i64> %b
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_v8f64_ole_q(<8 x i64> %a, <8 x i64> %b, <8 x double> %f1, <8 x double> %f2) #0 {
+; AVX512-32-LABEL: test_v8f64_ole_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-64, %esp
+; AVX512-32-NEXT:    subl $64, %esp
+; AVX512-32-NEXT:    vcmple_oqpd 8(%ebp), %zmm2, %k1
+; AVX512-32-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v8f64_ole_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmple_oqpd %zmm3, %zmm2, %k1
+; AVX512-64-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <8 x i1> @llvm.experimental.constrained.fcmp.v8f64(
+                                               <8 x double> %f1, <8 x double> %f2, metadata !"ole",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <8 x i1> %cond, <8 x i64> %a, <8 x i64> %b
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_v8f64_one_q(<8 x i64> %a, <8 x i64> %b, <8 x double> %f1, <8 x double> %f2) #0 {
+; AVX512-32-LABEL: test_v8f64_one_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-64, %esp
+; AVX512-32-NEXT:    subl $64, %esp
+; AVX512-32-NEXT:    vcmpneq_oqpd 8(%ebp), %zmm2, %k1
+; AVX512-32-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v8f64_one_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpneq_oqpd %zmm3, %zmm2, %k1
+; AVX512-64-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <8 x i1> @llvm.experimental.constrained.fcmp.v8f64(
+                                               <8 x double> %f1, <8 x double> %f2, metadata !"one",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <8 x i1> %cond, <8 x i64> %a, <8 x i64> %b
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_v8f64_ord_q(<8 x i64> %a, <8 x i64> %b, <8 x double> %f1, <8 x double> %f2) #0 {
+; AVX512-32-LABEL: test_v8f64_ord_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-64, %esp
+; AVX512-32-NEXT:    subl $64, %esp
+; AVX512-32-NEXT:    vcmpordpd 8(%ebp), %zmm2, %k1
+; AVX512-32-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v8f64_ord_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpordpd %zmm3, %zmm2, %k1
+; AVX512-64-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <8 x i1> @llvm.experimental.constrained.fcmp.v8f64(
+                                               <8 x double> %f1, <8 x double> %f2, metadata !"ord",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <8 x i1> %cond, <8 x i64> %a, <8 x i64> %b
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_v8f64_ueq_q(<8 x i64> %a, <8 x i64> %b, <8 x double> %f1, <8 x double> %f2) #0 {
+; AVX512-32-LABEL: test_v8f64_ueq_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-64, %esp
+; AVX512-32-NEXT:    subl $64, %esp
+; AVX512-32-NEXT:    vcmpeq_uqpd 8(%ebp), %zmm2, %k1
+; AVX512-32-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v8f64_ueq_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpeq_uqpd %zmm3, %zmm2, %k1
+; AVX512-64-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <8 x i1> @llvm.experimental.constrained.fcmp.v8f64(
+                                               <8 x double> %f1, <8 x double> %f2, metadata !"ueq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <8 x i1> %cond, <8 x i64> %a, <8 x i64> %b
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_v8f64_ugt_q(<8 x i64> %a, <8 x i64> %b, <8 x double> %f1, <8 x double> %f2) #0 {
+; AVX512-32-LABEL: test_v8f64_ugt_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-64, %esp
+; AVX512-32-NEXT:    subl $64, %esp
+; AVX512-32-NEXT:    vcmpnle_uqpd 8(%ebp), %zmm2, %k1
+; AVX512-32-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v8f64_ugt_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpnle_uqpd %zmm3, %zmm2, %k1
+; AVX512-64-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <8 x i1> @llvm.experimental.constrained.fcmp.v8f64(
+                                               <8 x double> %f1, <8 x double> %f2, metadata !"ugt",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <8 x i1> %cond, <8 x i64> %a, <8 x i64> %b
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_v8f64_uge_q(<8 x i64> %a, <8 x i64> %b, <8 x double> %f1, <8 x double> %f2) #0 {
+; AVX512-32-LABEL: test_v8f64_uge_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-64, %esp
+; AVX512-32-NEXT:    subl $64, %esp
+; AVX512-32-NEXT:    vcmpnlt_uqpd 8(%ebp), %zmm2, %k1
+; AVX512-32-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v8f64_uge_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpnlt_uqpd %zmm3, %zmm2, %k1
+; AVX512-64-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <8 x i1> @llvm.experimental.constrained.fcmp.v8f64(
+                                               <8 x double> %f1, <8 x double> %f2, metadata !"uge",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <8 x i1> %cond, <8 x i64> %a, <8 x i64> %b
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_v8f64_ult_q(<8 x i64> %a, <8 x i64> %b, <8 x double> %f1, <8 x double> %f2) #0 {
+; AVX512-32-LABEL: test_v8f64_ult_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-64, %esp
+; AVX512-32-NEXT:    subl $64, %esp
+; AVX512-32-NEXT:    vcmpnge_uqpd 8(%ebp), %zmm2, %k1
+; AVX512-32-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v8f64_ult_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpnle_uqpd %zmm2, %zmm3, %k1
+; AVX512-64-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <8 x i1> @llvm.experimental.constrained.fcmp.v8f64(
+                                               <8 x double> %f1, <8 x double> %f2, metadata !"ult",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <8 x i1> %cond, <8 x i64> %a, <8 x i64> %b
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_v8f64_ule_q(<8 x i64> %a, <8 x i64> %b, <8 x double> %f1, <8 x double> %f2) #0 {
+; AVX512-32-LABEL: test_v8f64_ule_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-64, %esp
+; AVX512-32-NEXT:    subl $64, %esp
+; AVX512-32-NEXT:    vcmpngt_uqpd 8(%ebp), %zmm2, %k1
+; AVX512-32-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v8f64_ule_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpnlt_uqpd %zmm2, %zmm3, %k1
+; AVX512-64-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <8 x i1> @llvm.experimental.constrained.fcmp.v8f64(
+                                               <8 x double> %f1, <8 x double> %f2, metadata !"ule",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <8 x i1> %cond, <8 x i64> %a, <8 x i64> %b
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_v8f64_une_q(<8 x i64> %a, <8 x i64> %b, <8 x double> %f1, <8 x double> %f2) #0 {
+; AVX512-32-LABEL: test_v8f64_une_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-64, %esp
+; AVX512-32-NEXT:    subl $64, %esp
+; AVX512-32-NEXT:    vcmpneqpd 8(%ebp), %zmm2, %k1
+; AVX512-32-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v8f64_une_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpneqpd %zmm3, %zmm2, %k1
+; AVX512-64-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <8 x i1> @llvm.experimental.constrained.fcmp.v8f64(
+                                               <8 x double> %f1, <8 x double> %f2, metadata !"une",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <8 x i1> %cond, <8 x i64> %a, <8 x i64> %b
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_v8f64_uno_q(<8 x i64> %a, <8 x i64> %b, <8 x double> %f1, <8 x double> %f2) #0 {
+; AVX512-32-LABEL: test_v8f64_uno_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-64, %esp
+; AVX512-32-NEXT:    subl $64, %esp
+; AVX512-32-NEXT:    vcmpunordpd 8(%ebp), %zmm2, %k1
+; AVX512-32-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v8f64_uno_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpunordpd %zmm3, %zmm2, %k1
+; AVX512-64-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <8 x i1> @llvm.experimental.constrained.fcmp.v8f64(
+                                               <8 x double> %f1, <8 x double> %f2, metadata !"uno",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <8 x i1> %cond, <8 x i64> %a, <8 x i64> %b
+  ret <8 x i64> %res
+}
+
+define <16 x i32> @test_v16f32_oeq_s(<16 x i32> %a, <16 x i32> %b, <16 x float> %f1, <16 x float> %f2) #0 {
+; AVX512-32-LABEL: test_v16f32_oeq_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-64, %esp
+; AVX512-32-NEXT:    subl $64, %esp
+; AVX512-32-NEXT:    vcmpeq_osps 8(%ebp), %zmm2, %k1
+; AVX512-32-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v16f32_oeq_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpeq_osps %zmm3, %zmm2, %k1
+; AVX512-64-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <16 x i1> @llvm.experimental.constrained.fcmps.v16f32(
+                                               <16 x float> %f1, <16 x float> %f2, metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <16 x i1> %cond, <16 x i32> %a, <16 x i32> %b
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_v16f32_ogt_s(<16 x i32> %a, <16 x i32> %b, <16 x float> %f1, <16 x float> %f2) #0 {
+; AVX512-32-LABEL: test_v16f32_ogt_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-64, %esp
+; AVX512-32-NEXT:    subl $64, %esp
+; AVX512-32-NEXT:    vcmpgtps 8(%ebp), %zmm2, %k1
+; AVX512-32-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v16f32_ogt_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpltps %zmm2, %zmm3, %k1
+; AVX512-64-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <16 x i1> @llvm.experimental.constrained.fcmps.v16f32(
+                                               <16 x float> %f1, <16 x float> %f2, metadata !"ogt",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <16 x i1> %cond, <16 x i32> %a, <16 x i32> %b
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_v16f32_oge_s(<16 x i32> %a, <16 x i32> %b, <16 x float> %f1, <16 x float> %f2) #0 {
+; AVX512-32-LABEL: test_v16f32_oge_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-64, %esp
+; AVX512-32-NEXT:    subl $64, %esp
+; AVX512-32-NEXT:    vcmpgeps 8(%ebp), %zmm2, %k1
+; AVX512-32-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v16f32_oge_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpleps %zmm2, %zmm3, %k1
+; AVX512-64-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <16 x i1> @llvm.experimental.constrained.fcmps.v16f32(
+                                               <16 x float> %f1, <16 x float> %f2, metadata !"oge",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <16 x i1> %cond, <16 x i32> %a, <16 x i32> %b
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_v16f32_olt_s(<16 x i32> %a, <16 x i32> %b, <16 x float> %f1, <16 x float> %f2) #0 {
+; AVX512-32-LABEL: test_v16f32_olt_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-64, %esp
+; AVX512-32-NEXT:    subl $64, %esp
+; AVX512-32-NEXT:    vcmpltps 8(%ebp), %zmm2, %k1
+; AVX512-32-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v16f32_olt_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpltps %zmm3, %zmm2, %k1
+; AVX512-64-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <16 x i1> @llvm.experimental.constrained.fcmps.v16f32(
+                                               <16 x float> %f1, <16 x float> %f2, metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <16 x i1> %cond, <16 x i32> %a, <16 x i32> %b
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_v16f32_ole_s(<16 x i32> %a, <16 x i32> %b, <16 x float> %f1, <16 x float> %f2) #0 {
+; AVX512-32-LABEL: test_v16f32_ole_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-64, %esp
+; AVX512-32-NEXT:    subl $64, %esp
+; AVX512-32-NEXT:    vcmpleps 8(%ebp), %zmm2, %k1
+; AVX512-32-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v16f32_ole_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpleps %zmm3, %zmm2, %k1
+; AVX512-64-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <16 x i1> @llvm.experimental.constrained.fcmps.v16f32(
+                                               <16 x float> %f1, <16 x float> %f2, metadata !"ole",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <16 x i1> %cond, <16 x i32> %a, <16 x i32> %b
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_v16f32_one_s(<16 x i32> %a, <16 x i32> %b, <16 x float> %f1, <16 x float> %f2) #0 {
+; AVX512-32-LABEL: test_v16f32_one_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-64, %esp
+; AVX512-32-NEXT:    subl $64, %esp
+; AVX512-32-NEXT:    vcmpneq_osps 8(%ebp), %zmm2, %k1
+; AVX512-32-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v16f32_one_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpneq_osps %zmm3, %zmm2, %k1
+; AVX512-64-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <16 x i1> @llvm.experimental.constrained.fcmps.v16f32(
+                                               <16 x float> %f1, <16 x float> %f2, metadata !"one",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <16 x i1> %cond, <16 x i32> %a, <16 x i32> %b
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_v16f32_ord_s(<16 x i32> %a, <16 x i32> %b, <16 x float> %f1, <16 x float> %f2) #0 {
+; AVX512-32-LABEL: test_v16f32_ord_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-64, %esp
+; AVX512-32-NEXT:    subl $64, %esp
+; AVX512-32-NEXT:    vcmpord_sps 8(%ebp), %zmm2, %k1
+; AVX512-32-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v16f32_ord_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpord_sps %zmm3, %zmm2, %k1
+; AVX512-64-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <16 x i1> @llvm.experimental.constrained.fcmps.v16f32(
+                                               <16 x float> %f1, <16 x float> %f2, metadata !"ord",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <16 x i1> %cond, <16 x i32> %a, <16 x i32> %b
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_v16f32_ueq_s(<16 x i32> %a, <16 x i32> %b, <16 x float> %f1, <16 x float> %f2) #0 {
+; AVX512-32-LABEL: test_v16f32_ueq_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-64, %esp
+; AVX512-32-NEXT:    subl $64, %esp
+; AVX512-32-NEXT:    vcmpeq_usps 8(%ebp), %zmm2, %k1
+; AVX512-32-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v16f32_ueq_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpeq_usps %zmm3, %zmm2, %k1
+; AVX512-64-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <16 x i1> @llvm.experimental.constrained.fcmps.v16f32(
+                                               <16 x float> %f1, <16 x float> %f2, metadata !"ueq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <16 x i1> %cond, <16 x i32> %a, <16 x i32> %b
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_v16f32_ugt_s(<16 x i32> %a, <16 x i32> %b, <16 x float> %f1, <16 x float> %f2) #0 {
+; AVX512-32-LABEL: test_v16f32_ugt_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-64, %esp
+; AVX512-32-NEXT:    subl $64, %esp
+; AVX512-32-NEXT:    vcmpnleps 8(%ebp), %zmm2, %k1
+; AVX512-32-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v16f32_ugt_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpnleps %zmm3, %zmm2, %k1
+; AVX512-64-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <16 x i1> @llvm.experimental.constrained.fcmps.v16f32(
+                                               <16 x float> %f1, <16 x float> %f2, metadata !"ugt",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <16 x i1> %cond, <16 x i32> %a, <16 x i32> %b
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_v16f32_uge_s(<16 x i32> %a, <16 x i32> %b, <16 x float> %f1, <16 x float> %f2) #0 {
+; AVX512-32-LABEL: test_v16f32_uge_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-64, %esp
+; AVX512-32-NEXT:    subl $64, %esp
+; AVX512-32-NEXT:    vcmpnltps 8(%ebp), %zmm2, %k1
+; AVX512-32-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v16f32_uge_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpnltps %zmm3, %zmm2, %k1
+; AVX512-64-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <16 x i1> @llvm.experimental.constrained.fcmps.v16f32(
+                                               <16 x float> %f1, <16 x float> %f2, metadata !"uge",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <16 x i1> %cond, <16 x i32> %a, <16 x i32> %b
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_v16f32_ult_s(<16 x i32> %a, <16 x i32> %b, <16 x float> %f1, <16 x float> %f2) #0 {
+; AVX512-32-LABEL: test_v16f32_ult_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-64, %esp
+; AVX512-32-NEXT:    subl $64, %esp
+; AVX512-32-NEXT:    vcmpngeps 8(%ebp), %zmm2, %k1
+; AVX512-32-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v16f32_ult_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpnleps %zmm2, %zmm3, %k1
+; AVX512-64-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <16 x i1> @llvm.experimental.constrained.fcmps.v16f32(
+                                               <16 x float> %f1, <16 x float> %f2, metadata !"ult",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <16 x i1> %cond, <16 x i32> %a, <16 x i32> %b
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_v16f32_ule_s(<16 x i32> %a, <16 x i32> %b, <16 x float> %f1, <16 x float> %f2) #0 {
+; AVX512-32-LABEL: test_v16f32_ule_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-64, %esp
+; AVX512-32-NEXT:    subl $64, %esp
+; AVX512-32-NEXT:    vcmpngtps 8(%ebp), %zmm2, %k1
+; AVX512-32-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v16f32_ule_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpnltps %zmm2, %zmm3, %k1
+; AVX512-64-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <16 x i1> @llvm.experimental.constrained.fcmps.v16f32(
+                                               <16 x float> %f1, <16 x float> %f2, metadata !"ule",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <16 x i1> %cond, <16 x i32> %a, <16 x i32> %b
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_v16f32_une_s(<16 x i32> %a, <16 x i32> %b, <16 x float> %f1, <16 x float> %f2) #0 {
+; AVX512-32-LABEL: test_v16f32_une_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-64, %esp
+; AVX512-32-NEXT:    subl $64, %esp
+; AVX512-32-NEXT:    vcmpneq_usps 8(%ebp), %zmm2, %k1
+; AVX512-32-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v16f32_une_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpneq_usps %zmm3, %zmm2, %k1
+; AVX512-64-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <16 x i1> @llvm.experimental.constrained.fcmps.v16f32(
+                                               <16 x float> %f1, <16 x float> %f2, metadata !"une",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <16 x i1> %cond, <16 x i32> %a, <16 x i32> %b
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_v16f32_uno_s(<16 x i32> %a, <16 x i32> %b, <16 x float> %f1, <16 x float> %f2) #0 {
+; AVX512-32-LABEL: test_v16f32_uno_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-64, %esp
+; AVX512-32-NEXT:    subl $64, %esp
+; AVX512-32-NEXT:    vcmpunord_sps 8(%ebp), %zmm2, %k1
+; AVX512-32-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v16f32_uno_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpunord_sps %zmm3, %zmm2, %k1
+; AVX512-64-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <16 x i1> @llvm.experimental.constrained.fcmps.v16f32(
+                                               <16 x float> %f1, <16 x float> %f2, metadata !"uno",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <16 x i1> %cond, <16 x i32> %a, <16 x i32> %b
+  ret <16 x i32> %res
+}
+
+define <8 x i64> @test_v8f64_oeq_s(<8 x i64> %a, <8 x i64> %b, <8 x double> %f1, <8 x double> %f2) #0 {
+; AVX512-32-LABEL: test_v8f64_oeq_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-64, %esp
+; AVX512-32-NEXT:    subl $64, %esp
+; AVX512-32-NEXT:    vcmpeq_ospd 8(%ebp), %zmm2, %k1
+; AVX512-32-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v8f64_oeq_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpeq_ospd %zmm3, %zmm2, %k1
+; AVX512-64-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <8 x i1> @llvm.experimental.constrained.fcmps.v8f64(
+                                               <8 x double> %f1, <8 x double> %f2, metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <8 x i1> %cond, <8 x i64> %a, <8 x i64> %b
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_v8f64_ogt_s(<8 x i64> %a, <8 x i64> %b, <8 x double> %f1, <8 x double> %f2) #0 {
+; AVX512-32-LABEL: test_v8f64_ogt_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-64, %esp
+; AVX512-32-NEXT:    subl $64, %esp
+; AVX512-32-NEXT:    vcmpgtpd 8(%ebp), %zmm2, %k1
+; AVX512-32-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v8f64_ogt_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpltpd %zmm2, %zmm3, %k1
+; AVX512-64-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <8 x i1> @llvm.experimental.constrained.fcmps.v8f64(
+                                               <8 x double> %f1, <8 x double> %f2, metadata !"ogt",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <8 x i1> %cond, <8 x i64> %a, <8 x i64> %b
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_v8f64_oge_s(<8 x i64> %a, <8 x i64> %b, <8 x double> %f1, <8 x double> %f2) #0 {
+; AVX512-32-LABEL: test_v8f64_oge_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-64, %esp
+; AVX512-32-NEXT:    subl $64, %esp
+; AVX512-32-NEXT:    vcmpgepd 8(%ebp), %zmm2, %k1
+; AVX512-32-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v8f64_oge_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmplepd %zmm2, %zmm3, %k1
+; AVX512-64-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <8 x i1> @llvm.experimental.constrained.fcmps.v8f64(
+                                               <8 x double> %f1, <8 x double> %f2, metadata !"oge",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <8 x i1> %cond, <8 x i64> %a, <8 x i64> %b
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_v8f64_olt_s(<8 x i64> %a, <8 x i64> %b, <8 x double> %f1, <8 x double> %f2) #0 {
+; AVX512-32-LABEL: test_v8f64_olt_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-64, %esp
+; AVX512-32-NEXT:    subl $64, %esp
+; AVX512-32-NEXT:    vcmpltpd 8(%ebp), %zmm2, %k1
+; AVX512-32-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v8f64_olt_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpltpd %zmm3, %zmm2, %k1
+; AVX512-64-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <8 x i1> @llvm.experimental.constrained.fcmps.v8f64(
+                                               <8 x double> %f1, <8 x double> %f2, metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <8 x i1> %cond, <8 x i64> %a, <8 x i64> %b
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_v8f64_ole_s(<8 x i64> %a, <8 x i64> %b, <8 x double> %f1, <8 x double> %f2) #0 {
+; AVX512-32-LABEL: test_v8f64_ole_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-64, %esp
+; AVX512-32-NEXT:    subl $64, %esp
+; AVX512-32-NEXT:    vcmplepd 8(%ebp), %zmm2, %k1
+; AVX512-32-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v8f64_ole_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmplepd %zmm3, %zmm2, %k1
+; AVX512-64-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <8 x i1> @llvm.experimental.constrained.fcmps.v8f64(
+                                               <8 x double> %f1, <8 x double> %f2, metadata !"ole",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <8 x i1> %cond, <8 x i64> %a, <8 x i64> %b
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_v8f64_one_s(<8 x i64> %a, <8 x i64> %b, <8 x double> %f1, <8 x double> %f2) #0 {
+; AVX512-32-LABEL: test_v8f64_one_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-64, %esp
+; AVX512-32-NEXT:    subl $64, %esp
+; AVX512-32-NEXT:    vcmpneq_ospd 8(%ebp), %zmm2, %k1
+; AVX512-32-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v8f64_one_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpneq_ospd %zmm3, %zmm2, %k1
+; AVX512-64-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <8 x i1> @llvm.experimental.constrained.fcmps.v8f64(
+                                               <8 x double> %f1, <8 x double> %f2, metadata !"one",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <8 x i1> %cond, <8 x i64> %a, <8 x i64> %b
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_v8f64_ord_s(<8 x i64> %a, <8 x i64> %b, <8 x double> %f1, <8 x double> %f2) #0 {
+; AVX512-32-LABEL: test_v8f64_ord_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-64, %esp
+; AVX512-32-NEXT:    subl $64, %esp
+; AVX512-32-NEXT:    vcmpord_spd 8(%ebp), %zmm2, %k1
+; AVX512-32-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v8f64_ord_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpord_spd %zmm3, %zmm2, %k1
+; AVX512-64-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <8 x i1> @llvm.experimental.constrained.fcmps.v8f64(
+                                               <8 x double> %f1, <8 x double> %f2, metadata !"ord",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <8 x i1> %cond, <8 x i64> %a, <8 x i64> %b
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_v8f64_ueq_s(<8 x i64> %a, <8 x i64> %b, <8 x double> %f1, <8 x double> %f2) #0 {
+; AVX512-32-LABEL: test_v8f64_ueq_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-64, %esp
+; AVX512-32-NEXT:    subl $64, %esp
+; AVX512-32-NEXT:    vcmpeq_uspd 8(%ebp), %zmm2, %k1
+; AVX512-32-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v8f64_ueq_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpeq_uspd %zmm3, %zmm2, %k1
+; AVX512-64-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <8 x i1> @llvm.experimental.constrained.fcmps.v8f64(
+                                               <8 x double> %f1, <8 x double> %f2, metadata !"ueq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <8 x i1> %cond, <8 x i64> %a, <8 x i64> %b
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_v8f64_ugt_s(<8 x i64> %a, <8 x i64> %b, <8 x double> %f1, <8 x double> %f2) #0 {
+; AVX512-32-LABEL: test_v8f64_ugt_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-64, %esp
+; AVX512-32-NEXT:    subl $64, %esp
+; AVX512-32-NEXT:    vcmpnlepd 8(%ebp), %zmm2, %k1
+; AVX512-32-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v8f64_ugt_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpnlepd %zmm3, %zmm2, %k1
+; AVX512-64-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <8 x i1> @llvm.experimental.constrained.fcmps.v8f64(
+                                               <8 x double> %f1, <8 x double> %f2, metadata !"ugt",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <8 x i1> %cond, <8 x i64> %a, <8 x i64> %b
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_v8f64_uge_s(<8 x i64> %a, <8 x i64> %b, <8 x double> %f1, <8 x double> %f2) #0 {
+; AVX512-32-LABEL: test_v8f64_uge_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-64, %esp
+; AVX512-32-NEXT:    subl $64, %esp
+; AVX512-32-NEXT:    vcmpnltpd 8(%ebp), %zmm2, %k1
+; AVX512-32-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v8f64_uge_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpnltpd %zmm3, %zmm2, %k1
+; AVX512-64-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <8 x i1> @llvm.experimental.constrained.fcmps.v8f64(
+                                               <8 x double> %f1, <8 x double> %f2, metadata !"uge",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <8 x i1> %cond, <8 x i64> %a, <8 x i64> %b
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_v8f64_ult_s(<8 x i64> %a, <8 x i64> %b, <8 x double> %f1, <8 x double> %f2) #0 {
+; AVX512-32-LABEL: test_v8f64_ult_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-64, %esp
+; AVX512-32-NEXT:    subl $64, %esp
+; AVX512-32-NEXT:    vcmpngepd 8(%ebp), %zmm2, %k1
+; AVX512-32-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v8f64_ult_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpnlepd %zmm2, %zmm3, %k1
+; AVX512-64-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <8 x i1> @llvm.experimental.constrained.fcmps.v8f64(
+                                               <8 x double> %f1, <8 x double> %f2, metadata !"ult",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <8 x i1> %cond, <8 x i64> %a, <8 x i64> %b
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_v8f64_ule_s(<8 x i64> %a, <8 x i64> %b, <8 x double> %f1, <8 x double> %f2) #0 {
+; AVX512-32-LABEL: test_v8f64_ule_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-64, %esp
+; AVX512-32-NEXT:    subl $64, %esp
+; AVX512-32-NEXT:    vcmpngtpd 8(%ebp), %zmm2, %k1
+; AVX512-32-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v8f64_ule_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpnltpd %zmm2, %zmm3, %k1
+; AVX512-64-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <8 x i1> @llvm.experimental.constrained.fcmps.v8f64(
+                                               <8 x double> %f1, <8 x double> %f2, metadata !"ule",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <8 x i1> %cond, <8 x i64> %a, <8 x i64> %b
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_v8f64_une_s(<8 x i64> %a, <8 x i64> %b, <8 x double> %f1, <8 x double> %f2) #0 {
+; AVX512-32-LABEL: test_v8f64_une_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-64, %esp
+; AVX512-32-NEXT:    subl $64, %esp
+; AVX512-32-NEXT:    vcmpneq_uspd 8(%ebp), %zmm2, %k1
+; AVX512-32-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v8f64_une_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpneq_uspd %zmm3, %zmm2, %k1
+; AVX512-64-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <8 x i1> @llvm.experimental.constrained.fcmps.v8f64(
+                                               <8 x double> %f1, <8 x double> %f2, metadata !"une",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <8 x i1> %cond, <8 x i64> %a, <8 x i64> %b
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_v8f64_uno_s(<8 x i64> %a, <8 x i64> %b, <8 x double> %f1, <8 x double> %f2) #0 {
+; AVX512-32-LABEL: test_v8f64_uno_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-64, %esp
+; AVX512-32-NEXT:    subl $64, %esp
+; AVX512-32-NEXT:    vcmpunord_spd 8(%ebp), %zmm2, %k1
+; AVX512-32-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v8f64_uno_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vcmpunord_spd %zmm3, %zmm2, %k1
+; AVX512-64-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <8 x i1> @llvm.experimental.constrained.fcmps.v8f64(
+                                               <8 x double> %f1, <8 x double> %f2, metadata !"uno",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <8 x i1> %cond, <8 x i64> %a, <8 x i64> %b
+  ret <8 x i64> %res
+}
+
+attributes #0 = { strictfp nounwind }
+
+declare <16 x i1> @llvm.experimental.constrained.fcmp.v16f32(<16 x float>, <16 x float>, metadata, metadata)
+declare <8 x i1> @llvm.experimental.constrained.fcmp.v8f64(<8 x double>, <8 x double>, metadata, metadata)
+declare <16 x i1> @llvm.experimental.constrained.fcmps.v16f32(<16 x float>, <16 x float>, metadata, metadata)
+declare <8 x i1> @llvm.experimental.constrained.fcmps.v8f64(<8 x double>, <8 x double>, metadata, metadata)
diff --git a/llvm/test/CodeGen/X86/vec-strict-cmp-sub128.ll b/llvm/test/CodeGen/X86/vec-strict-cmp-sub128.ll
new file mode 100644
index 000000000..6dbac0e8f
--- /dev/null
+++ b/llvm/test/CodeGen/X86/vec-strict-cmp-sub128.ll
@@ -0,0 +1,308 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse2 -O3 | FileCheck %s --check-prefixes=CHECK,SSE,SSE-32
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 -O3 | FileCheck %s --check-prefixes=CHECK,SSE,SSE-64
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx -O3 | FileCheck %s --check-prefixes=CHECK,AVX,AVX-32
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx -O3 | FileCheck %s --check-prefixes=CHECK,AVX,AVX-64
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx512f -mattr=+avx512vl -O3 | FileCheck %s --check-prefixes=CHECK,AVX512-32
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f -mattr=+avx512vl -O3 | FileCheck %s --check-prefixes=CHECK,AVX512-64
+
+define <2 x i32> @test_v2f32_ogt_s(<2 x i32> %a, <2 x i32> %b, <2 x float> %f1, <2 x float> %f2) #0 {
+; SSE-32-LABEL: test_v2f32_ogt_s:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    andl $-16, %esp
+; SSE-32-NEXT:    subl $16, %esp
+; SSE-32-NEXT:    movaps 8(%ebp), %xmm3
+; SSE-32-NEXT:    xorl %eax, %eax
+; SSE-32-NEXT:    comiss %xmm3, %xmm2
+; SSE-32-NEXT:    movl $-1, %ecx
+; SSE-32-NEXT:    movl $0, %edx
+; SSE-32-NEXT:    cmoval %ecx, %edx
+; SSE-32-NEXT:    movd %edx, %xmm4
+; SSE-32-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,2,3]
+; SSE-32-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,2,3]
+; SSE-32-NEXT:    comiss %xmm3, %xmm2
+; SSE-32-NEXT:    cmoval %ecx, %eax
+; SSE-32-NEXT:    movd %eax, %xmm2
+; SSE-32-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
+; SSE-32-NEXT:    pand %xmm4, %xmm0
+; SSE-32-NEXT:    pandn %xmm1, %xmm4
+; SSE-32-NEXT:    por %xmm4, %xmm0
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_v2f32_ogt_s:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    xorl %eax, %eax
+; SSE-64-NEXT:    comiss %xmm3, %xmm2
+; SSE-64-NEXT:    movl $-1, %ecx
+; SSE-64-NEXT:    movl $0, %edx
+; SSE-64-NEXT:    cmoval %ecx, %edx
+; SSE-64-NEXT:    movd %edx, %xmm4
+; SSE-64-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,2,3]
+; SSE-64-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,2,3]
+; SSE-64-NEXT:    comiss %xmm3, %xmm2
+; SSE-64-NEXT:    cmoval %ecx, %eax
+; SSE-64-NEXT:    movd %eax, %xmm2
+; SSE-64-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
+; SSE-64-NEXT:    pand %xmm4, %xmm0
+; SSE-64-NEXT:    pandn %xmm1, %xmm4
+; SSE-64-NEXT:    por %xmm4, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_v2f32_ogt_s:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-16, %esp
+; AVX-32-NEXT:    subl $16, %esp
+; AVX-32-NEXT:    vmovaps 8(%ebp), %xmm3
+; AVX-32-NEXT:    vmovshdup {{.*#+}} xmm4 = xmm3[1,1,3,3]
+; AVX-32-NEXT:    vmovshdup {{.*#+}} xmm5 = xmm2[1,1,3,3]
+; AVX-32-NEXT:    xorl %eax, %eax
+; AVX-32-NEXT:    vcomiss %xmm4, %xmm5
+; AVX-32-NEXT:    movl $-1, %ecx
+; AVX-32-NEXT:    movl $0, %edx
+; AVX-32-NEXT:    cmoval %ecx, %edx
+; AVX-32-NEXT:    vcomiss %xmm3, %xmm2
+; AVX-32-NEXT:    cmoval %ecx, %eax
+; AVX-32-NEXT:    vmovd %eax, %xmm2
+; AVX-32-NEXT:    vpinsrd $1, %edx, %xmm2, %xmm2
+; AVX-32-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v2f32_ogt_s:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vmovshdup {{.*#+}} xmm4 = xmm3[1,1,3,3]
+; AVX-64-NEXT:    vmovshdup {{.*#+}} xmm5 = xmm2[1,1,3,3]
+; AVX-64-NEXT:    xorl %eax, %eax
+; AVX-64-NEXT:    vcomiss %xmm4, %xmm5
+; AVX-64-NEXT:    movl $-1, %ecx
+; AVX-64-NEXT:    movl $0, %edx
+; AVX-64-NEXT:    cmoval %ecx, %edx
+; AVX-64-NEXT:    vcomiss %xmm3, %xmm2
+; AVX-64-NEXT:    cmoval %ecx, %eax
+; AVX-64-NEXT:    vmovd %eax, %xmm2
+; AVX-64-NEXT:    vpinsrd $1, %edx, %xmm2, %xmm2
+; AVX-64-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v2f32_ogt_s:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-16, %esp
+; AVX512-32-NEXT:    subl $16, %esp
+; AVX512-32-NEXT:    vmovaps 8(%ebp), %xmm3
+; AVX512-32-NEXT:    movw $-3, %ax
+; AVX512-32-NEXT:    kmovw %eax, %k0
+; AVX512-32-NEXT:    vcomiss %xmm3, %xmm2
+; AVX512-32-NEXT:    seta %al
+; AVX512-32-NEXT:    andl $1, %eax
+; AVX512-32-NEXT:    kmovw %eax, %k1
+; AVX512-32-NEXT:    kandw %k0, %k1, %k0
+; AVX512-32-NEXT:    vmovshdup {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; AVX512-32-NEXT:    vmovshdup {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; AVX512-32-NEXT:    vcomiss %xmm3, %xmm2
+; AVX512-32-NEXT:    seta %al
+; AVX512-32-NEXT:    kmovw %eax, %k1
+; AVX512-32-NEXT:    kshiftlw $15, %k1, %k1
+; AVX512-32-NEXT:    kshiftrw $14, %k1, %k1
+; AVX512-32-NEXT:    korw %k1, %k0, %k1
+; AVX512-32-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v2f32_ogt_s:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    movw $-3, %ax
+; AVX512-64-NEXT:    kmovw %eax, %k0
+; AVX512-64-NEXT:    vcomiss %xmm3, %xmm2
+; AVX512-64-NEXT:    seta %al
+; AVX512-64-NEXT:    andl $1, %eax
+; AVX512-64-NEXT:    kmovw %eax, %k1
+; AVX512-64-NEXT:    kandw %k0, %k1, %k0
+; AVX512-64-NEXT:    vmovshdup {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; AVX512-64-NEXT:    vmovshdup {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; AVX512-64-NEXT:    vcomiss %xmm3, %xmm2
+; AVX512-64-NEXT:    seta %al
+; AVX512-64-NEXT:    kmovw %eax, %k1
+; AVX512-64-NEXT:    kshiftlw $15, %k1, %k1
+; AVX512-64-NEXT:    kshiftrw $14, %k1, %k1
+; AVX512-64-NEXT:    korw %k1, %k0, %k1
+; AVX512-64-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <2 x i1> @llvm.experimental.constrained.fcmps.v2f32(
+                                               <2 x float> %f1, <2 x float> %f2, metadata !"ogt",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <2 x i1> %cond, <2 x i32> %a, <2 x i32> %b
+  ret <2 x i32> %res
+}
+
+define <2 x i32> @test_v2f32_oeq_q(<2 x i32> %a, <2 x i32> %b, <2 x float> %f1, <2 x float> %f2) #0 {
+; SSE-32-LABEL: test_v2f32_oeq_q:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    andl $-16, %esp
+; SSE-32-NEXT:    subl $16, %esp
+; SSE-32-NEXT:    movaps 8(%ebp), %xmm3
+; SSE-32-NEXT:    xorl %eax, %eax
+; SSE-32-NEXT:    ucomiss %xmm3, %xmm2
+; SSE-32-NEXT:    movl $-1, %ecx
+; SSE-32-NEXT:    movl $-1, %edx
+; SSE-32-NEXT:    cmovnel %eax, %edx
+; SSE-32-NEXT:    cmovpl %eax, %edx
+; SSE-32-NEXT:    movd %edx, %xmm4
+; SSE-32-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,2,3]
+; SSE-32-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,2,3]
+; SSE-32-NEXT:    ucomiss %xmm3, %xmm2
+; SSE-32-NEXT:    cmovnel %eax, %ecx
+; SSE-32-NEXT:    cmovpl %eax, %ecx
+; SSE-32-NEXT:    movd %ecx, %xmm2
+; SSE-32-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
+; SSE-32-NEXT:    pand %xmm4, %xmm0
+; SSE-32-NEXT:    pandn %xmm1, %xmm4
+; SSE-32-NEXT:    por %xmm4, %xmm0
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_v2f32_oeq_q:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    xorl %eax, %eax
+; SSE-64-NEXT:    ucomiss %xmm3, %xmm2
+; SSE-64-NEXT:    movl $-1, %ecx
+; SSE-64-NEXT:    movl $-1, %edx
+; SSE-64-NEXT:    cmovnel %eax, %edx
+; SSE-64-NEXT:    cmovpl %eax, %edx
+; SSE-64-NEXT:    movd %edx, %xmm4
+; SSE-64-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,2,3]
+; SSE-64-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,2,3]
+; SSE-64-NEXT:    ucomiss %xmm3, %xmm2
+; SSE-64-NEXT:    cmovnel %eax, %ecx
+; SSE-64-NEXT:    cmovpl %eax, %ecx
+; SSE-64-NEXT:    movd %ecx, %xmm2
+; SSE-64-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
+; SSE-64-NEXT:    pand %xmm4, %xmm0
+; SSE-64-NEXT:    pandn %xmm1, %xmm4
+; SSE-64-NEXT:    por %xmm4, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_v2f32_oeq_q:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    andl $-16, %esp
+; AVX-32-NEXT:    subl $16, %esp
+; AVX-32-NEXT:    vmovaps 8(%ebp), %xmm3
+; AVX-32-NEXT:    vmovshdup {{.*#+}} xmm4 = xmm3[1,1,3,3]
+; AVX-32-NEXT:    vmovshdup {{.*#+}} xmm5 = xmm2[1,1,3,3]
+; AVX-32-NEXT:    xorl %eax, %eax
+; AVX-32-NEXT:    vucomiss %xmm4, %xmm5
+; AVX-32-NEXT:    movl $-1, %ecx
+; AVX-32-NEXT:    movl $-1, %edx
+; AVX-32-NEXT:    cmovnel %eax, %edx
+; AVX-32-NEXT:    cmovpl %eax, %edx
+; AVX-32-NEXT:    vucomiss %xmm3, %xmm2
+; AVX-32-NEXT:    cmovnel %eax, %ecx
+; AVX-32-NEXT:    cmovpl %eax, %ecx
+; AVX-32-NEXT:    vmovd %ecx, %xmm2
+; AVX-32-NEXT:    vpinsrd $1, %edx, %xmm2, %xmm2
+; AVX-32-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_v2f32_oeq_q:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vmovshdup {{.*#+}} xmm4 = xmm3[1,1,3,3]
+; AVX-64-NEXT:    vmovshdup {{.*#+}} xmm5 = xmm2[1,1,3,3]
+; AVX-64-NEXT:    xorl %eax, %eax
+; AVX-64-NEXT:    vucomiss %xmm4, %xmm5
+; AVX-64-NEXT:    movl $-1, %ecx
+; AVX-64-NEXT:    movl $-1, %edx
+; AVX-64-NEXT:    cmovnel %eax, %edx
+; AVX-64-NEXT:    cmovpl %eax, %edx
+; AVX-64-NEXT:    vucomiss %xmm3, %xmm2
+; AVX-64-NEXT:    cmovnel %eax, %ecx
+; AVX-64-NEXT:    cmovpl %eax, %ecx
+; AVX-64-NEXT:    vmovd %ecx, %xmm2
+; AVX-64-NEXT:    vpinsrd $1, %edx, %xmm2, %xmm2
+; AVX-64-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-64-NEXT:    retq
+;
+; AVX512-32-LABEL: test_v2f32_oeq_q:
+; AVX512-32:       # %bb.0:
+; AVX512-32-NEXT:    pushl %ebp
+; AVX512-32-NEXT:    movl %esp, %ebp
+; AVX512-32-NEXT:    andl $-16, %esp
+; AVX512-32-NEXT:    subl $16, %esp
+; AVX512-32-NEXT:    vmovaps 8(%ebp), %xmm3
+; AVX512-32-NEXT:    vmovshdup {{.*#+}} xmm4 = xmm3[1,1,3,3]
+; AVX512-32-NEXT:    vmovshdup {{.*#+}} xmm5 = xmm2[1,1,3,3]
+; AVX512-32-NEXT:    vucomiss %xmm4, %xmm5
+; AVX512-32-NEXT:    setnp %al
+; AVX512-32-NEXT:    sete %cl
+; AVX512-32-NEXT:    testb %al, %cl
+; AVX512-32-NEXT:    setne %al
+; AVX512-32-NEXT:    kmovw %eax, %k0
+; AVX512-32-NEXT:    kshiftlw $15, %k0, %k0
+; AVX512-32-NEXT:    kshiftrw $14, %k0, %k0
+; AVX512-32-NEXT:    vucomiss %xmm3, %xmm2
+; AVX512-32-NEXT:    setnp %al
+; AVX512-32-NEXT:    sete %cl
+; AVX512-32-NEXT:    testb %al, %cl
+; AVX512-32-NEXT:    setne %al
+; AVX512-32-NEXT:    andl $1, %eax
+; AVX512-32-NEXT:    kmovw %eax, %k1
+; AVX512-32-NEXT:    movw $-3, %ax
+; AVX512-32-NEXT:    kmovw %eax, %k2
+; AVX512-32-NEXT:    kandw %k2, %k1, %k1
+; AVX512-32-NEXT:    korw %k0, %k1, %k1
+; AVX512-32-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-32-NEXT:    movl %ebp, %esp
+; AVX512-32-NEXT:    popl %ebp
+; AVX512-32-NEXT:    retl
+;
+; AVX512-64-LABEL: test_v2f32_oeq_q:
+; AVX512-64:       # %bb.0:
+; AVX512-64-NEXT:    vmovshdup {{.*#+}} xmm4 = xmm3[1,1,3,3]
+; AVX512-64-NEXT:    vmovshdup {{.*#+}} xmm5 = xmm2[1,1,3,3]
+; AVX512-64-NEXT:    vucomiss %xmm4, %xmm5
+; AVX512-64-NEXT:    setnp %al
+; AVX512-64-NEXT:    sete %cl
+; AVX512-64-NEXT:    testb %al, %cl
+; AVX512-64-NEXT:    setne %al
+; AVX512-64-NEXT:    kmovw %eax, %k0
+; AVX512-64-NEXT:    kshiftlw $15, %k0, %k0
+; AVX512-64-NEXT:    kshiftrw $14, %k0, %k0
+; AVX512-64-NEXT:    vucomiss %xmm3, %xmm2
+; AVX512-64-NEXT:    setnp %al
+; AVX512-64-NEXT:    sete %cl
+; AVX512-64-NEXT:    testb %al, %cl
+; AVX512-64-NEXT:    setne %al
+; AVX512-64-NEXT:    andl $1, %eax
+; AVX512-64-NEXT:    kmovw %eax, %k1
+; AVX512-64-NEXT:    movw $-3, %ax
+; AVX512-64-NEXT:    kmovw %eax, %k2
+; AVX512-64-NEXT:    kandw %k2, %k1, %k1
+; AVX512-64-NEXT:    korw %k0, %k1, %k1
+; AVX512-64-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512-64-NEXT:    retq
+  %cond = call <2 x i1> @llvm.experimental.constrained.fcmp.v2f32(
+                                               <2 x float> %f1, <2 x float> %f2, metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select <2 x i1> %cond, <2 x i32> %a, <2 x i32> %b
+  ret <2 x i32> %res
+}
+
+attributes #0 = { strictfp nounwind }
+
+declare <2 x i1> @llvm.experimental.constrained.fcmp.v2f32(<2 x float>, <2 x float>, metadata, metadata)
+declare <2 x i1> @llvm.experimental.constrained.fcmps.v2f32(<2 x float>, <2 x float>, metadata, metadata)
diff --git a/llvm/test/CodeGen/X86/vec-strict-fptoint-128.ll b/llvm/test/CodeGen/X86/vec-strict-fptoint-128.ll
new file mode 100644
index 000000000..fb1ec511f
--- /dev/null
+++ b/llvm/test/CodeGen/X86/vec-strict-fptoint-128.ll
@@ -0,0 +1,2813 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse2 -O3 | FileCheck %s --check-prefixes=CHECK,SSE-32
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 -O3 | FileCheck %s --check-prefixes=CHECK,SSE-64
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx -O3 | FileCheck %s --check-prefixes=CHECK,AVX,AVX-32
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx -O3 | FileCheck %s --check-prefixes=CHECK,AVX,AVX-64
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx512f -O3 | FileCheck %s --check-prefixes=CHECK,AVX512F,AVX512F-32
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f -O3 | FileCheck %s --check-prefixes=CHECK,AVX512F,AVX512F-64
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx512vl -O3 | FileCheck %s --check-prefixes=CHECK,AVX512VL,AVX512VL-32
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vl -O3 | FileCheck %s --check-prefixes=CHECK,AVX512VL,AVX512VL-64
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=avx512dq -O3 | FileCheck %s --check-prefixes=CHECK,AVX512DQ,AVX512DQ-32
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512dq -O3 | FileCheck %s --check-prefixes=CHECK,AVX512DQ,AVX512DQ-64
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=avx512dq,avx512vl -O3 | FileCheck %s --check-prefixes=CHECK,AVX512VLDQ,AVX512VLDQ-32
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512dq,avx512vl -O3 | FileCheck %s --check-prefixes=CHECK,AVX512VLDQ,AVX512VLDQ-64
+
+declare <2 x i64> @llvm.experimental.constrained.fptosi.v2i64.v2f64(<2 x double>, metadata)
+declare <2 x i64> @llvm.experimental.constrained.fptoui.v2i64.v2f64(<2 x double>, metadata)
+declare <2 x i64> @llvm.experimental.constrained.fptosi.v2i64.v2f32(<2 x float>, metadata)
+declare <2 x i64> @llvm.experimental.constrained.fptoui.v2i64.v2f32(<2 x float>, metadata)
+declare <2 x i32> @llvm.experimental.constrained.fptosi.v2i32.v2f64(<2 x double>, metadata)
+declare <2 x i32> @llvm.experimental.constrained.fptoui.v2i32.v2f64(<2 x double>, metadata)
+declare <2 x i32> @llvm.experimental.constrained.fptosi.v2i32.v2f32(<2 x float>, metadata)
+declare <2 x i32> @llvm.experimental.constrained.fptoui.v2i32.v2f32(<2 x float>, metadata)
+declare <2 x i16> @llvm.experimental.constrained.fptosi.v2i16.v2f64(<2 x double>, metadata)
+declare <2 x i16> @llvm.experimental.constrained.fptoui.v2i16.v2f64(<2 x double>, metadata)
+declare <2 x i16> @llvm.experimental.constrained.fptosi.v2i16.v2f32(<2 x float>, metadata)
+declare <2 x i16> @llvm.experimental.constrained.fptoui.v2i16.v2f32(<2 x float>, metadata)
+declare <2 x i8> @llvm.experimental.constrained.fptosi.v2i8.v2f64(<2 x double>, metadata)
+declare <2 x i8> @llvm.experimental.constrained.fptoui.v2i8.v2f64(<2 x double>, metadata)
+declare <2 x i8> @llvm.experimental.constrained.fptosi.v2i8.v2f32(<2 x float>, metadata)
+declare <2 x i8> @llvm.experimental.constrained.fptoui.v2i8.v2f32(<2 x float>, metadata)
+declare <2 x i1> @llvm.experimental.constrained.fptosi.v2i1.v2f64(<2 x double>, metadata)
+declare <2 x i1> @llvm.experimental.constrained.fptoui.v2i1.v2f64(<2 x double>, metadata)
+declare <2 x i1> @llvm.experimental.constrained.fptosi.v2i1.v2f32(<2 x float>, metadata)
+declare <2 x i1> @llvm.experimental.constrained.fptoui.v2i1.v2f32(<2 x float>, metadata)
+declare <4 x i32> @llvm.experimental.constrained.fptosi.v4i32.v4f32(<4 x float>, metadata)
+declare <4 x i32> @llvm.experimental.constrained.fptoui.v4i32.v4f32(<4 x float>, metadata)
+declare <4 x i16> @llvm.experimental.constrained.fptosi.v4i16.v4f32(<4 x float>, metadata)
+declare <4 x i16> @llvm.experimental.constrained.fptoui.v4i16.v4f32(<4 x float>, metadata)
+declare <4 x i8> @llvm.experimental.constrained.fptosi.v4i8.v4f32(<4 x float>, metadata)
+declare <4 x i8> @llvm.experimental.constrained.fptoui.v4i8.v4f32(<4 x float>, metadata)
+declare <4 x i1> @llvm.experimental.constrained.fptosi.v4i1.v4f32(<4 x float>, metadata)
+declare <4 x i1> @llvm.experimental.constrained.fptoui.v4i1.v4f32(<4 x float>, metadata)
+
+define <2 x i64> @strict_vector_fptosi_v2f64_to_v2i64(<2 x double> %a) #0 {
+; SSE-32-LABEL: strict_vector_fptosi_v2f64_to_v2i64:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    .cfi_def_cfa_offset 8
+; SSE-32-NEXT:    .cfi_offset %ebp, -8
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    .cfi_def_cfa_register %ebp
+; SSE-32-NEXT:    andl $-8, %esp
+; SSE-32-NEXT:    subl $24, %esp
+; SSE-32-NEXT:    movhps %xmm0, {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    movlps %xmm0, {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fldl {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fnstcw {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; SSE-32-NEXT:    orl $3072, %eax # imm = 0xC00
+; SSE-32-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fldcw {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fistpll {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fldcw {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fldl {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fnstcw (%esp)
+; SSE-32-NEXT:    movzwl (%esp), %eax
+; SSE-32-NEXT:    orl $3072, %eax # imm = 0xC00
+; SSE-32-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fldcw {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fistpll {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fldcw (%esp)
+; SSE-32-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
+; SSE-32-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE-32-NEXT:    movlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    .cfi_def_cfa %esp, 4
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: strict_vector_fptosi_v2f64_to_v2i64:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    cvttsd2si %xmm0, %rax
+; SSE-64-NEXT:    movq %rax, %xmm1
+; SSE-64-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1,1]
+; SSE-64-NEXT:    cvttsd2si %xmm0, %rax
+; SSE-64-NEXT:    movq %rax, %xmm0
+; SSE-64-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSE-64-NEXT:    movdqa %xmm1, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: strict_vector_fptosi_v2f64_to_v2i64:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    .cfi_def_cfa_offset 8
+; AVX-32-NEXT:    .cfi_offset %ebp, -8
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    .cfi_def_cfa_register %ebp
+; AVX-32-NEXT:    andl $-8, %esp
+; AVX-32-NEXT:    subl $16, %esp
+; AVX-32-NEXT:    vmovlps %xmm0, {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    vmovhps %xmm0, (%esp)
+; AVX-32-NEXT:    fldl {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    fldl (%esp)
+; AVX-32-NEXT:    fisttpll (%esp)
+; AVX-32-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vpinsrd $1, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrd $2, (%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrd $3, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    .cfi_def_cfa %esp, 4
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: strict_vector_fptosi_v2f64_to_v2i64:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcvttsd2si %xmm0, %rax
+; AVX-64-NEXT:    vmovq %rax, %xmm1
+; AVX-64-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
+; AVX-64-NEXT:    vcvttsd2si %xmm0, %rax
+; AVX-64-NEXT:    vmovq %rax, %xmm0
+; AVX-64-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX-64-NEXT:    retq
+;
+; AVX512F-32-LABEL: strict_vector_fptosi_v2f64_to_v2i64:
+; AVX512F-32:       # %bb.0:
+; AVX512F-32-NEXT:    pushl %ebp
+; AVX512F-32-NEXT:    .cfi_def_cfa_offset 8
+; AVX512F-32-NEXT:    .cfi_offset %ebp, -8
+; AVX512F-32-NEXT:    movl %esp, %ebp
+; AVX512F-32-NEXT:    .cfi_def_cfa_register %ebp
+; AVX512F-32-NEXT:    andl $-8, %esp
+; AVX512F-32-NEXT:    subl $16, %esp
+; AVX512F-32-NEXT:    vmovlps %xmm0, {{[0-9]+}}(%esp)
+; AVX512F-32-NEXT:    vmovhps %xmm0, (%esp)
+; AVX512F-32-NEXT:    fldl {{[0-9]+}}(%esp)
+; AVX512F-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX512F-32-NEXT:    fldl (%esp)
+; AVX512F-32-NEXT:    fisttpll (%esp)
+; AVX512F-32-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX512F-32-NEXT:    vpinsrd $1, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512F-32-NEXT:    vpinsrd $2, (%esp), %xmm0, %xmm0
+; AVX512F-32-NEXT:    vpinsrd $3, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512F-32-NEXT:    movl %ebp, %esp
+; AVX512F-32-NEXT:    popl %ebp
+; AVX512F-32-NEXT:    .cfi_def_cfa %esp, 4
+; AVX512F-32-NEXT:    retl
+;
+; AVX512F-64-LABEL: strict_vector_fptosi_v2f64_to_v2i64:
+; AVX512F-64:       # %bb.0:
+; AVX512F-64-NEXT:    vcvttsd2si %xmm0, %rax
+; AVX512F-64-NEXT:    vmovq %rax, %xmm1
+; AVX512F-64-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
+; AVX512F-64-NEXT:    vcvttsd2si %xmm0, %rax
+; AVX512F-64-NEXT:    vmovq %rax, %xmm0
+; AVX512F-64-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512F-64-NEXT:    retq
+;
+; AVX512VL-32-LABEL: strict_vector_fptosi_v2f64_to_v2i64:
+; AVX512VL-32:       # %bb.0:
+; AVX512VL-32-NEXT:    pushl %ebp
+; AVX512VL-32-NEXT:    .cfi_def_cfa_offset 8
+; AVX512VL-32-NEXT:    .cfi_offset %ebp, -8
+; AVX512VL-32-NEXT:    movl %esp, %ebp
+; AVX512VL-32-NEXT:    .cfi_def_cfa_register %ebp
+; AVX512VL-32-NEXT:    andl $-8, %esp
+; AVX512VL-32-NEXT:    subl $16, %esp
+; AVX512VL-32-NEXT:    vmovlps %xmm0, {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    vmovhps %xmm0, (%esp)
+; AVX512VL-32-NEXT:    fldl {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    fldl (%esp)
+; AVX512VL-32-NEXT:    fisttpll (%esp)
+; AVX512VL-32-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX512VL-32-NEXT:    vpinsrd $1, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512VL-32-NEXT:    vpinsrd $2, (%esp), %xmm0, %xmm0
+; AVX512VL-32-NEXT:    vpinsrd $3, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512VL-32-NEXT:    movl %ebp, %esp
+; AVX512VL-32-NEXT:    popl %ebp
+; AVX512VL-32-NEXT:    .cfi_def_cfa %esp, 4
+; AVX512VL-32-NEXT:    retl
+;
+; AVX512VL-64-LABEL: strict_vector_fptosi_v2f64_to_v2i64:
+; AVX512VL-64:       # %bb.0:
+; AVX512VL-64-NEXT:    vcvttsd2si %xmm0, %rax
+; AVX512VL-64-NEXT:    vmovq %rax, %xmm1
+; AVX512VL-64-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
+; AVX512VL-64-NEXT:    vcvttsd2si %xmm0, %rax
+; AVX512VL-64-NEXT:    vmovq %rax, %xmm0
+; AVX512VL-64-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512VL-64-NEXT:    retq
+;
+; AVX512DQ-LABEL: strict_vector_fptosi_v2f64_to_v2i64:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vmovaps %xmm0, %xmm0
+; AVX512DQ-NEXT:    vcvttpd2qq %zmm0, %zmm0
+; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+;
+; AVX512VLDQ-LABEL: strict_vector_fptosi_v2f64_to_v2i64:
+; AVX512VLDQ:       # %bb.0:
+; AVX512VLDQ-NEXT:    vcvttpd2qq %xmm0, %xmm0
+; AVX512VLDQ-NEXT:    ret{{[l|q]}}
+  %ret = call <2 x i64> @llvm.experimental.constrained.fptosi.v2i64.v2f64(<2 x double> %a,
+                                              metadata !"fpexcept.strict")
+  ret <2 x i64> %ret
+}
+
+define <2 x i64> @strict_vector_fptoui_v2f64_to_v2i64(<2 x double> %a) #0 {
+; SSE-32-LABEL: strict_vector_fptoui_v2f64_to_v2i64:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    .cfi_def_cfa_offset 8
+; SSE-32-NEXT:    .cfi_offset %ebp, -8
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    .cfi_def_cfa_register %ebp
+; SSE-32-NEXT:    andl $-8, %esp
+; SSE-32-NEXT:    subl $24, %esp
+; SSE-32-NEXT:    movsd {{.*#+}} xmm2 = mem[0],zero
+; SSE-32-NEXT:    comisd %xmm2, %xmm0
+; SSE-32-NEXT:    xorpd %xmm1, %xmm1
+; SSE-32-NEXT:    xorpd %xmm3, %xmm3
+; SSE-32-NEXT:    jb .LBB1_2
+; SSE-32-NEXT:  # %bb.1:
+; SSE-32-NEXT:    movapd %xmm2, %xmm3
+; SSE-32-NEXT:  .LBB1_2:
+; SSE-32-NEXT:    movapd %xmm0, %xmm4
+; SSE-32-NEXT:    subsd %xmm3, %xmm4
+; SSE-32-NEXT:    movsd %xmm4, {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    setae %al
+; SSE-32-NEXT:    fldl {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fnstcw {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
+; SSE-32-NEXT:    orl $3072, %ecx # imm = 0xC00
+; SSE-32-NEXT:    movw %cx, {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fldcw {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fistpll {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fldcw {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1,1]
+; SSE-32-NEXT:    comisd %xmm2, %xmm0
+; SSE-32-NEXT:    jb .LBB1_4
+; SSE-32-NEXT:  # %bb.3:
+; SSE-32-NEXT:    movapd %xmm2, %xmm1
+; SSE-32-NEXT:  .LBB1_4:
+; SSE-32-NEXT:    subsd %xmm1, %xmm0
+; SSE-32-NEXT:    movsd %xmm0, {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    setae %cl
+; SSE-32-NEXT:    fldl {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fnstcw (%esp)
+; SSE-32-NEXT:    movzwl (%esp), %edx
+; SSE-32-NEXT:    orl $3072, %edx # imm = 0xC00
+; SSE-32-NEXT:    movw %dx, {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fldcw {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fistpll {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fldcw (%esp)
+; SSE-32-NEXT:    movzbl %al, %eax
+; SSE-32-NEXT:    shll $31, %eax
+; SSE-32-NEXT:    xorl {{[0-9]+}}(%esp), %eax
+; SSE-32-NEXT:    movd %eax, %xmm1
+; SSE-32-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE-32-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE-32-NEXT:    movzbl %cl, %eax
+; SSE-32-NEXT:    shll $31, %eax
+; SSE-32-NEXT:    xorl {{[0-9]+}}(%esp), %eax
+; SSE-32-NEXT:    movd %eax, %xmm1
+; SSE-32-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE-32-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSE-32-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    .cfi_def_cfa %esp, 4
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: strict_vector_fptoui_v2f64_to_v2i64:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
+; SSE-64-NEXT:    comisd %xmm3, %xmm0
+; SSE-64-NEXT:    xorpd %xmm2, %xmm2
+; SSE-64-NEXT:    xorpd %xmm1, %xmm1
+; SSE-64-NEXT:    jb .LBB1_2
+; SSE-64-NEXT:  # %bb.1:
+; SSE-64-NEXT:    movapd %xmm3, %xmm1
+; SSE-64-NEXT:  .LBB1_2:
+; SSE-64-NEXT:    movapd %xmm0, %xmm4
+; SSE-64-NEXT:    subsd %xmm1, %xmm4
+; SSE-64-NEXT:    cvttsd2si %xmm4, %rax
+; SSE-64-NEXT:    setae %cl
+; SSE-64-NEXT:    movzbl %cl, %ecx
+; SSE-64-NEXT:    shlq $63, %rcx
+; SSE-64-NEXT:    xorq %rax, %rcx
+; SSE-64-NEXT:    movq %rcx, %xmm1
+; SSE-64-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1,1]
+; SSE-64-NEXT:    comisd %xmm3, %xmm0
+; SSE-64-NEXT:    jb .LBB1_4
+; SSE-64-NEXT:  # %bb.3:
+; SSE-64-NEXT:    movapd %xmm3, %xmm2
+; SSE-64-NEXT:  .LBB1_4:
+; SSE-64-NEXT:    subsd %xmm2, %xmm0
+; SSE-64-NEXT:    cvttsd2si %xmm0, %rax
+; SSE-64-NEXT:    setae %cl
+; SSE-64-NEXT:    movzbl %cl, %ecx
+; SSE-64-NEXT:    shlq $63, %rcx
+; SSE-64-NEXT:    xorq %rax, %rcx
+; SSE-64-NEXT:    movq %rcx, %xmm0
+; SSE-64-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSE-64-NEXT:    movdqa %xmm1, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: strict_vector_fptoui_v2f64_to_v2i64:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    .cfi_def_cfa_offset 8
+; AVX-32-NEXT:    .cfi_offset %ebp, -8
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    .cfi_def_cfa_register %ebp
+; AVX-32-NEXT:    andl $-8, %esp
+; AVX-32-NEXT:    subl $16, %esp
+; AVX-32-NEXT:    vpermilpd {{.*#+}} xmm3 = xmm0[1,0]
+; AVX-32-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
+; AVX-32-NEXT:    vcomisd %xmm1, %xmm3
+; AVX-32-NEXT:    vxorpd %xmm2, %xmm2, %xmm2
+; AVX-32-NEXT:    vxorpd %xmm4, %xmm4, %xmm4
+; AVX-32-NEXT:    jb .LBB1_2
+; AVX-32-NEXT:  # %bb.1:
+; AVX-32-NEXT:    vmovapd %xmm1, %xmm4
+; AVX-32-NEXT:  .LBB1_2:
+; AVX-32-NEXT:    vsubsd %xmm4, %xmm3, %xmm3
+; AVX-32-NEXT:    vmovsd %xmm3, (%esp)
+; AVX-32-NEXT:    fldl (%esp)
+; AVX-32-NEXT:    fisttpll (%esp)
+; AVX-32-NEXT:    setae %al
+; AVX-32-NEXT:    movzbl %al, %eax
+; AVX-32-NEXT:    shll $31, %eax
+; AVX-32-NEXT:    xorl {{[0-9]+}}(%esp), %eax
+; AVX-32-NEXT:    vcomisd %xmm1, %xmm0
+; AVX-32-NEXT:    jb .LBB1_4
+; AVX-32-NEXT:  # %bb.3:
+; AVX-32-NEXT:    vmovapd %xmm1, %xmm2
+; AVX-32-NEXT:  .LBB1_4:
+; AVX-32-NEXT:    vsubsd %xmm2, %xmm0, %xmm0
+; AVX-32-NEXT:    vmovsd %xmm0, {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    fldl {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    setae %cl
+; AVX-32-NEXT:    movzbl %cl, %ecx
+; AVX-32-NEXT:    shll $31, %ecx
+; AVX-32-NEXT:    xorl {{[0-9]+}}(%esp), %ecx
+; AVX-32-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vpinsrd $1, %ecx, %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrd $2, (%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrd $3, %eax, %xmm0, %xmm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    .cfi_def_cfa %esp, 4
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: strict_vector_fptoui_v2f64_to_v2i64:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
+; AVX-64-NEXT:    vcomisd %xmm1, %xmm0
+; AVX-64-NEXT:    vxorpd %xmm2, %xmm2, %xmm2
+; AVX-64-NEXT:    vxorpd %xmm3, %xmm3, %xmm3
+; AVX-64-NEXT:    jb .LBB1_2
+; AVX-64-NEXT:  # %bb.1:
+; AVX-64-NEXT:    vmovapd %xmm1, %xmm3
+; AVX-64-NEXT:  .LBB1_2:
+; AVX-64-NEXT:    vsubsd %xmm3, %xmm0, %xmm3
+; AVX-64-NEXT:    vcvttsd2si %xmm3, %rax
+; AVX-64-NEXT:    setae %cl
+; AVX-64-NEXT:    movzbl %cl, %ecx
+; AVX-64-NEXT:    shlq $63, %rcx
+; AVX-64-NEXT:    xorq %rax, %rcx
+; AVX-64-NEXT:    vmovq %rcx, %xmm3
+; AVX-64-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
+; AVX-64-NEXT:    vcomisd %xmm1, %xmm0
+; AVX-64-NEXT:    jb .LBB1_4
+; AVX-64-NEXT:  # %bb.3:
+; AVX-64-NEXT:    vmovapd %xmm1, %xmm2
+; AVX-64-NEXT:  .LBB1_4:
+; AVX-64-NEXT:    vsubsd %xmm2, %xmm0, %xmm0
+; AVX-64-NEXT:    vcvttsd2si %xmm0, %rax
+; AVX-64-NEXT:    setae %cl
+; AVX-64-NEXT:    movzbl %cl, %ecx
+; AVX-64-NEXT:    shlq $63, %rcx
+; AVX-64-NEXT:    xorq %rax, %rcx
+; AVX-64-NEXT:    vmovq %rcx, %xmm0
+; AVX-64-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm3[0],xmm0[0]
+; AVX-64-NEXT:    retq
+;
+; AVX512F-32-LABEL: strict_vector_fptoui_v2f64_to_v2i64:
+; AVX512F-32:       # %bb.0:
+; AVX512F-32-NEXT:    pushl %ebp
+; AVX512F-32-NEXT:    .cfi_def_cfa_offset 8
+; AVX512F-32-NEXT:    .cfi_offset %ebp, -8
+; AVX512F-32-NEXT:    movl %esp, %ebp
+; AVX512F-32-NEXT:    .cfi_def_cfa_register %ebp
+; AVX512F-32-NEXT:    andl $-8, %esp
+; AVX512F-32-NEXT:    subl $16, %esp
+; AVX512F-32-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
+; AVX512F-32-NEXT:    vmovsd {{.*#+}} xmm2 = mem[0],zero
+; AVX512F-32-NEXT:    xorl %eax, %eax
+; AVX512F-32-NEXT:    vcomisd %xmm2, %xmm1
+; AVX512F-32-NEXT:    setb %cl
+; AVX512F-32-NEXT:    kmovw %ecx, %k1
+; AVX512F-32-NEXT:    vxorpd %xmm3, %xmm3, %xmm3
+; AVX512F-32-NEXT:    vmovapd %xmm2, %xmm4
+; AVX512F-32-NEXT:    vmovsd %xmm3, %xmm4, %xmm4 {%k1}
+; AVX512F-32-NEXT:    vsubsd %xmm4, %xmm1, %xmm1
+; AVX512F-32-NEXT:    vmovsd %xmm1, (%esp)
+; AVX512F-32-NEXT:    fldl (%esp)
+; AVX512F-32-NEXT:    fisttpll (%esp)
+; AVX512F-32-NEXT:    setae %al
+; AVX512F-32-NEXT:    shll $31, %eax
+; AVX512F-32-NEXT:    xorl {{[0-9]+}}(%esp), %eax
+; AVX512F-32-NEXT:    xorl %ecx, %ecx
+; AVX512F-32-NEXT:    vcomisd %xmm2, %xmm0
+; AVX512F-32-NEXT:    setb %dl
+; AVX512F-32-NEXT:    kmovw %edx, %k1
+; AVX512F-32-NEXT:    vmovsd %xmm3, %xmm2, %xmm2 {%k1}
+; AVX512F-32-NEXT:    vsubsd %xmm2, %xmm0, %xmm0
+; AVX512F-32-NEXT:    vmovsd %xmm0, {{[0-9]+}}(%esp)
+; AVX512F-32-NEXT:    fldl {{[0-9]+}}(%esp)
+; AVX512F-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX512F-32-NEXT:    setae %cl
+; AVX512F-32-NEXT:    shll $31, %ecx
+; AVX512F-32-NEXT:    xorl {{[0-9]+}}(%esp), %ecx
+; AVX512F-32-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX512F-32-NEXT:    vpinsrd $1, %ecx, %xmm0, %xmm0
+; AVX512F-32-NEXT:    vpinsrd $2, (%esp), %xmm0, %xmm0
+; AVX512F-32-NEXT:    vpinsrd $3, %eax, %xmm0, %xmm0
+; AVX512F-32-NEXT:    movl %ebp, %esp
+; AVX512F-32-NEXT:    popl %ebp
+; AVX512F-32-NEXT:    .cfi_def_cfa %esp, 4
+; AVX512F-32-NEXT:    retl
+;
+; AVX512F-64-LABEL: strict_vector_fptoui_v2f64_to_v2i64:
+; AVX512F-64:       # %bb.0:
+; AVX512F-64-NEXT:    vcvttsd2usi %xmm0, %rax
+; AVX512F-64-NEXT:    vmovq %rax, %xmm1
+; AVX512F-64-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
+; AVX512F-64-NEXT:    vcvttsd2usi %xmm0, %rax
+; AVX512F-64-NEXT:    vmovq %rax, %xmm0
+; AVX512F-64-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512F-64-NEXT:    retq
+;
+; AVX512VL-32-LABEL: strict_vector_fptoui_v2f64_to_v2i64:
+; AVX512VL-32:       # %bb.0:
+; AVX512VL-32-NEXT:    pushl %ebp
+; AVX512VL-32-NEXT:    .cfi_def_cfa_offset 8
+; AVX512VL-32-NEXT:    .cfi_offset %ebp, -8
+; AVX512VL-32-NEXT:    movl %esp, %ebp
+; AVX512VL-32-NEXT:    .cfi_def_cfa_register %ebp
+; AVX512VL-32-NEXT:    andl $-8, %esp
+; AVX512VL-32-NEXT:    subl $16, %esp
+; AVX512VL-32-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
+; AVX512VL-32-NEXT:    vmovsd {{.*#+}} xmm2 = mem[0],zero
+; AVX512VL-32-NEXT:    xorl %eax, %eax
+; AVX512VL-32-NEXT:    vcomisd %xmm2, %xmm1
+; AVX512VL-32-NEXT:    setb %cl
+; AVX512VL-32-NEXT:    kmovw %ecx, %k1
+; AVX512VL-32-NEXT:    vxorpd %xmm3, %xmm3, %xmm3
+; AVX512VL-32-NEXT:    vmovapd %xmm2, %xmm4
+; AVX512VL-32-NEXT:    vmovsd %xmm3, %xmm4, %xmm4 {%k1}
+; AVX512VL-32-NEXT:    vsubsd %xmm4, %xmm1, %xmm1
+; AVX512VL-32-NEXT:    vmovsd %xmm1, (%esp)
+; AVX512VL-32-NEXT:    fldl (%esp)
+; AVX512VL-32-NEXT:    fisttpll (%esp)
+; AVX512VL-32-NEXT:    setae %al
+; AVX512VL-32-NEXT:    shll $31, %eax
+; AVX512VL-32-NEXT:    xorl {{[0-9]+}}(%esp), %eax
+; AVX512VL-32-NEXT:    xorl %ecx, %ecx
+; AVX512VL-32-NEXT:    vcomisd %xmm2, %xmm0
+; AVX512VL-32-NEXT:    setb %dl
+; AVX512VL-32-NEXT:    kmovw %edx, %k1
+; AVX512VL-32-NEXT:    vmovsd %xmm3, %xmm2, %xmm2 {%k1}
+; AVX512VL-32-NEXT:    vsubsd %xmm2, %xmm0, %xmm0
+; AVX512VL-32-NEXT:    vmovsd %xmm0, {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    fldl {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    setae %cl
+; AVX512VL-32-NEXT:    shll $31, %ecx
+; AVX512VL-32-NEXT:    xorl {{[0-9]+}}(%esp), %ecx
+; AVX512VL-32-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX512VL-32-NEXT:    vpinsrd $1, %ecx, %xmm0, %xmm0
+; AVX512VL-32-NEXT:    vpinsrd $2, (%esp), %xmm0, %xmm0
+; AVX512VL-32-NEXT:    vpinsrd $3, %eax, %xmm0, %xmm0
+; AVX512VL-32-NEXT:    movl %ebp, %esp
+; AVX512VL-32-NEXT:    popl %ebp
+; AVX512VL-32-NEXT:    .cfi_def_cfa %esp, 4
+; AVX512VL-32-NEXT:    retl
+;
+; AVX512VL-64-LABEL: strict_vector_fptoui_v2f64_to_v2i64:
+; AVX512VL-64:       # %bb.0:
+; AVX512VL-64-NEXT:    vcvttsd2usi %xmm0, %rax
+; AVX512VL-64-NEXT:    vmovq %rax, %xmm1
+; AVX512VL-64-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
+; AVX512VL-64-NEXT:    vcvttsd2usi %xmm0, %rax
+; AVX512VL-64-NEXT:    vmovq %rax, %xmm0
+; AVX512VL-64-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512VL-64-NEXT:    retq
+;
+; AVX512DQ-LABEL: strict_vector_fptoui_v2f64_to_v2i64:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vmovaps %xmm0, %xmm0
+; AVX512DQ-NEXT:    vcvttpd2uqq %zmm0, %zmm0
+; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+;
+; AVX512VLDQ-LABEL: strict_vector_fptoui_v2f64_to_v2i64:
+; AVX512VLDQ:       # %bb.0:
+; AVX512VLDQ-NEXT:    vcvttpd2uqq %xmm0, %xmm0
+; AVX512VLDQ-NEXT:    ret{{[l|q]}}
+  %ret = call <2 x i64> @llvm.experimental.constrained.fptoui.v2i64.v2f64(<2 x double> %a,
+                                              metadata !"fpexcept.strict")
+  ret <2 x i64> %ret
+}
+
+define <2 x i64> @strict_vector_fptosi_v2f32_to_v2i64(<2 x float> %a) #0 {
+; SSE-32-LABEL: strict_vector_fptosi_v2f32_to_v2i64:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    .cfi_def_cfa_offset 8
+; SSE-32-NEXT:    .cfi_offset %ebp, -8
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    .cfi_def_cfa_register %ebp
+; SSE-32-NEXT:    andl $-8, %esp
+; SSE-32-NEXT:    subl $24, %esp
+; SSE-32-NEXT:    movss %xmm0, {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE-32-NEXT:    movss %xmm0, {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    flds {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fnstcw {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; SSE-32-NEXT:    orl $3072, %eax # imm = 0xC00
+; SSE-32-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fldcw {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fistpll {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fldcw {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    flds {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fnstcw (%esp)
+; SSE-32-NEXT:    movzwl (%esp), %eax
+; SSE-32-NEXT:    orl $3072, %eax # imm = 0xC00
+; SSE-32-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fldcw {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fistpll {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fldcw (%esp)
+; SSE-32-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE-32-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
+; SSE-32-NEXT:    movlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    .cfi_def_cfa %esp, 4
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: strict_vector_fptosi_v2f32_to_v2i64:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    cvttss2si %xmm0, %rax
+; SSE-64-NEXT:    movq %rax, %xmm1
+; SSE-64-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE-64-NEXT:    cvttss2si %xmm0, %rax
+; SSE-64-NEXT:    movq %rax, %xmm0
+; SSE-64-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSE-64-NEXT:    movdqa %xmm1, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: strict_vector_fptosi_v2f32_to_v2i64:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    .cfi_def_cfa_offset 8
+; AVX-32-NEXT:    .cfi_offset %ebp, -8
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    .cfi_def_cfa_register %ebp
+; AVX-32-NEXT:    andl $-8, %esp
+; AVX-32-NEXT:    subl $16, %esp
+; AVX-32-NEXT:    vmovss %xmm0, (%esp)
+; AVX-32-NEXT:    vextractps $1, %xmm0, {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    flds (%esp)
+; AVX-32-NEXT:    fisttpll (%esp)
+; AVX-32-NEXT:    flds {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vpinsrd $1, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrd $2, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrd $3, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    .cfi_def_cfa %esp, 4
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: strict_vector_fptosi_v2f32_to_v2i64:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcvttss2si %xmm0, %rax
+; AVX-64-NEXT:    vmovq %rax, %xmm1
+; AVX-64-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; AVX-64-NEXT:    vcvttss2si %xmm0, %rax
+; AVX-64-NEXT:    vmovq %rax, %xmm0
+; AVX-64-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX-64-NEXT:    retq
+;
+; AVX512F-32-LABEL: strict_vector_fptosi_v2f32_to_v2i64:
+; AVX512F-32:       # %bb.0:
+; AVX512F-32-NEXT:    pushl %ebp
+; AVX512F-32-NEXT:    .cfi_def_cfa_offset 8
+; AVX512F-32-NEXT:    .cfi_offset %ebp, -8
+; AVX512F-32-NEXT:    movl %esp, %ebp
+; AVX512F-32-NEXT:    .cfi_def_cfa_register %ebp
+; AVX512F-32-NEXT:    andl $-8, %esp
+; AVX512F-32-NEXT:    subl $16, %esp
+; AVX512F-32-NEXT:    vmovd %xmm0, (%esp)
+; AVX512F-32-NEXT:    vextractps $1, %xmm0, {{[0-9]+}}(%esp)
+; AVX512F-32-NEXT:    flds (%esp)
+; AVX512F-32-NEXT:    fisttpll (%esp)
+; AVX512F-32-NEXT:    flds {{[0-9]+}}(%esp)
+; AVX512F-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX512F-32-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX512F-32-NEXT:    vpinsrd $1, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512F-32-NEXT:    vpinsrd $2, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512F-32-NEXT:    vpinsrd $3, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512F-32-NEXT:    movl %ebp, %esp
+; AVX512F-32-NEXT:    popl %ebp
+; AVX512F-32-NEXT:    .cfi_def_cfa %esp, 4
+; AVX512F-32-NEXT:    retl
+;
+; AVX512F-64-LABEL: strict_vector_fptosi_v2f32_to_v2i64:
+; AVX512F-64:       # %bb.0:
+; AVX512F-64-NEXT:    vcvttss2si %xmm0, %rax
+; AVX512F-64-NEXT:    vmovq %rax, %xmm1
+; AVX512F-64-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; AVX512F-64-NEXT:    vcvttss2si %xmm0, %rax
+; AVX512F-64-NEXT:    vmovq %rax, %xmm0
+; AVX512F-64-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512F-64-NEXT:    retq
+;
+; AVX512VL-32-LABEL: strict_vector_fptosi_v2f32_to_v2i64:
+; AVX512VL-32:       # %bb.0:
+; AVX512VL-32-NEXT:    pushl %ebp
+; AVX512VL-32-NEXT:    .cfi_def_cfa_offset 8
+; AVX512VL-32-NEXT:    .cfi_offset %ebp, -8
+; AVX512VL-32-NEXT:    movl %esp, %ebp
+; AVX512VL-32-NEXT:    .cfi_def_cfa_register %ebp
+; AVX512VL-32-NEXT:    andl $-8, %esp
+; AVX512VL-32-NEXT:    subl $16, %esp
+; AVX512VL-32-NEXT:    vmovd %xmm0, (%esp)
+; AVX512VL-32-NEXT:    vextractps $1, %xmm0, {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    flds (%esp)
+; AVX512VL-32-NEXT:    fisttpll (%esp)
+; AVX512VL-32-NEXT:    flds {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX512VL-32-NEXT:    vpinsrd $1, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512VL-32-NEXT:    vpinsrd $2, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512VL-32-NEXT:    vpinsrd $3, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512VL-32-NEXT:    movl %ebp, %esp
+; AVX512VL-32-NEXT:    popl %ebp
+; AVX512VL-32-NEXT:    .cfi_def_cfa %esp, 4
+; AVX512VL-32-NEXT:    retl
+;
+; AVX512VL-64-LABEL: strict_vector_fptosi_v2f32_to_v2i64:
+; AVX512VL-64:       # %bb.0:
+; AVX512VL-64-NEXT:    vcvttss2si %xmm0, %rax
+; AVX512VL-64-NEXT:    vmovq %rax, %xmm1
+; AVX512VL-64-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; AVX512VL-64-NEXT:    vcvttss2si %xmm0, %rax
+; AVX512VL-64-NEXT:    vmovq %rax, %xmm0
+; AVX512VL-64-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512VL-64-NEXT:    retq
+;
+; AVX512DQ-32-LABEL: strict_vector_fptosi_v2f32_to_v2i64:
+; AVX512DQ-32:       # %bb.0:
+; AVX512DQ-32-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; AVX512DQ-32-NEXT:    vblendps {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3]
+; AVX512DQ-32-NEXT:    vcvttps2qq %ymm1, %zmm1
+; AVX512DQ-32-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[1],zero,zero,zero
+; AVX512DQ-32-NEXT:    vcvttps2qq %ymm0, %zmm0
+; AVX512DQ-32-NEXT:    vmovd %xmm0, %eax
+; AVX512DQ-32-NEXT:    vpinsrd $2, %eax, %xmm1, %xmm1
+; AVX512DQ-32-NEXT:    vpextrd $1, %xmm0, %eax
+; AVX512DQ-32-NEXT:    vpinsrd $3, %eax, %xmm1, %xmm0
+; AVX512DQ-32-NEXT:    vzeroupper
+; AVX512DQ-32-NEXT:    retl
+;
+; AVX512DQ-64-LABEL: strict_vector_fptosi_v2f32_to_v2i64:
+; AVX512DQ-64:       # %bb.0:
+; AVX512DQ-64-NEXT:    vcvttss2si %xmm0, %rax
+; AVX512DQ-64-NEXT:    vmovq %rax, %xmm1
+; AVX512DQ-64-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; AVX512DQ-64-NEXT:    vcvttss2si %xmm0, %rax
+; AVX512DQ-64-NEXT:    vmovq %rax, %xmm0
+; AVX512DQ-64-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512DQ-64-NEXT:    retq
+;
+; AVX512VLDQ-LABEL: strict_vector_fptosi_v2f32_to_v2i64:
+; AVX512VLDQ:       # %bb.0:
+; AVX512VLDQ-NEXT:    vcvttps2qq %xmm0, %xmm0
+; AVX512VLDQ-NEXT:    ret{{[l|q]}}
+  %ret = call <2 x i64> @llvm.experimental.constrained.fptosi.v2i64.v2f32(<2 x float> %a,
+                                              metadata !"fpexcept.strict")
+  ret <2 x i64> %ret
+}
+
+define <2 x i64> @strict_vector_fptoui_v2f32_to_v2i64(<2 x float> %a) #0 {
+; SSE-32-LABEL: strict_vector_fptoui_v2f32_to_v2i64:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    .cfi_def_cfa_offset 8
+; SSE-32-NEXT:    .cfi_offset %ebp, -8
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    .cfi_def_cfa_register %ebp
+; SSE-32-NEXT:    andl $-8, %esp
+; SSE-32-NEXT:    subl $24, %esp
+; SSE-32-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE-32-NEXT:    comiss %xmm2, %xmm0
+; SSE-32-NEXT:    xorps %xmm1, %xmm1
+; SSE-32-NEXT:    xorps %xmm3, %xmm3
+; SSE-32-NEXT:    jb .LBB3_2
+; SSE-32-NEXT:  # %bb.1:
+; SSE-32-NEXT:    movaps %xmm2, %xmm3
+; SSE-32-NEXT:  .LBB3_2:
+; SSE-32-NEXT:    movaps %xmm0, %xmm4
+; SSE-32-NEXT:    subss %xmm3, %xmm4
+; SSE-32-NEXT:    movss %xmm4, {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    setae %al
+; SSE-32-NEXT:    flds {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fnstcw {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
+; SSE-32-NEXT:    orl $3072, %ecx # imm = 0xC00
+; SSE-32-NEXT:    movw %cx, {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fldcw {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fistpll {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fldcw {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE-32-NEXT:    comiss %xmm2, %xmm0
+; SSE-32-NEXT:    jb .LBB3_4
+; SSE-32-NEXT:  # %bb.3:
+; SSE-32-NEXT:    movaps %xmm2, %xmm1
+; SSE-32-NEXT:  .LBB3_4:
+; SSE-32-NEXT:    subss %xmm1, %xmm0
+; SSE-32-NEXT:    movss %xmm0, {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    setae %cl
+; SSE-32-NEXT:    flds {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fnstcw (%esp)
+; SSE-32-NEXT:    movzwl (%esp), %edx
+; SSE-32-NEXT:    orl $3072, %edx # imm = 0xC00
+; SSE-32-NEXT:    movw %dx, {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fldcw {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fistpll {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fldcw (%esp)
+; SSE-32-NEXT:    movzbl %al, %eax
+; SSE-32-NEXT:    shll $31, %eax
+; SSE-32-NEXT:    xorl {{[0-9]+}}(%esp), %eax
+; SSE-32-NEXT:    movd %eax, %xmm1
+; SSE-32-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE-32-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE-32-NEXT:    movzbl %cl, %eax
+; SSE-32-NEXT:    shll $31, %eax
+; SSE-32-NEXT:    xorl {{[0-9]+}}(%esp), %eax
+; SSE-32-NEXT:    movd %eax, %xmm1
+; SSE-32-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE-32-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSE-32-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    .cfi_def_cfa %esp, 4
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: strict_vector_fptoui_v2f32_to_v2i64:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movss {{.*#+}} xmm3 = mem[0],zero,zero,zero
+; SSE-64-NEXT:    comiss %xmm3, %xmm0
+; SSE-64-NEXT:    xorps %xmm2, %xmm2
+; SSE-64-NEXT:    xorps %xmm1, %xmm1
+; SSE-64-NEXT:    jb .LBB3_2
+; SSE-64-NEXT:  # %bb.1:
+; SSE-64-NEXT:    movaps %xmm3, %xmm1
+; SSE-64-NEXT:  .LBB3_2:
+; SSE-64-NEXT:    movaps %xmm0, %xmm4
+; SSE-64-NEXT:    subss %xmm1, %xmm4
+; SSE-64-NEXT:    cvttss2si %xmm4, %rax
+; SSE-64-NEXT:    setae %cl
+; SSE-64-NEXT:    movzbl %cl, %ecx
+; SSE-64-NEXT:    shlq $63, %rcx
+; SSE-64-NEXT:    xorq %rax, %rcx
+; SSE-64-NEXT:    movq %rcx, %xmm1
+; SSE-64-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE-64-NEXT:    comiss %xmm3, %xmm0
+; SSE-64-NEXT:    jb .LBB3_4
+; SSE-64-NEXT:  # %bb.3:
+; SSE-64-NEXT:    movaps %xmm3, %xmm2
+; SSE-64-NEXT:  .LBB3_4:
+; SSE-64-NEXT:    subss %xmm2, %xmm0
+; SSE-64-NEXT:    cvttss2si %xmm0, %rax
+; SSE-64-NEXT:    setae %cl
+; SSE-64-NEXT:    movzbl %cl, %ecx
+; SSE-64-NEXT:    shlq $63, %rcx
+; SSE-64-NEXT:    xorq %rax, %rcx
+; SSE-64-NEXT:    movq %rcx, %xmm0
+; SSE-64-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSE-64-NEXT:    movdqa %xmm1, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: strict_vector_fptoui_v2f32_to_v2i64:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    .cfi_def_cfa_offset 8
+; AVX-32-NEXT:    .cfi_offset %ebp, -8
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    .cfi_def_cfa_register %ebp
+; AVX-32-NEXT:    andl $-8, %esp
+; AVX-32-NEXT:    subl $16, %esp
+; AVX-32-NEXT:    vmovshdup {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; AVX-32-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vcomiss %xmm1, %xmm3
+; AVX-32-NEXT:    vxorps %xmm2, %xmm2, %xmm2
+; AVX-32-NEXT:    vxorps %xmm4, %xmm4, %xmm4
+; AVX-32-NEXT:    jb .LBB3_2
+; AVX-32-NEXT:  # %bb.1:
+; AVX-32-NEXT:    vmovaps %xmm1, %xmm4
+; AVX-32-NEXT:  .LBB3_2:
+; AVX-32-NEXT:    vsubss %xmm4, %xmm3, %xmm3
+; AVX-32-NEXT:    vmovss %xmm3, {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    flds {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    setae %al
+; AVX-32-NEXT:    movzbl %al, %eax
+; AVX-32-NEXT:    shll $31, %eax
+; AVX-32-NEXT:    xorl {{[0-9]+}}(%esp), %eax
+; AVX-32-NEXT:    vcomiss %xmm1, %xmm0
+; AVX-32-NEXT:    jb .LBB3_4
+; AVX-32-NEXT:  # %bb.3:
+; AVX-32-NEXT:    vmovaps %xmm1, %xmm2
+; AVX-32-NEXT:  .LBB3_4:
+; AVX-32-NEXT:    vsubss %xmm2, %xmm0, %xmm0
+; AVX-32-NEXT:    vmovss %xmm0, (%esp)
+; AVX-32-NEXT:    flds (%esp)
+; AVX-32-NEXT:    fisttpll (%esp)
+; AVX-32-NEXT:    setae %cl
+; AVX-32-NEXT:    movzbl %cl, %ecx
+; AVX-32-NEXT:    shll $31, %ecx
+; AVX-32-NEXT:    xorl {{[0-9]+}}(%esp), %ecx
+; AVX-32-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vpinsrd $1, %ecx, %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrd $2, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrd $3, %eax, %xmm0, %xmm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    .cfi_def_cfa %esp, 4
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: strict_vector_fptoui_v2f32_to_v2i64:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX-64-NEXT:    vcomiss %xmm1, %xmm0
+; AVX-64-NEXT:    vxorps %xmm2, %xmm2, %xmm2
+; AVX-64-NEXT:    vxorps %xmm3, %xmm3, %xmm3
+; AVX-64-NEXT:    jb .LBB3_2
+; AVX-64-NEXT:  # %bb.1:
+; AVX-64-NEXT:    vmovaps %xmm1, %xmm3
+; AVX-64-NEXT:  .LBB3_2:
+; AVX-64-NEXT:    vsubss %xmm3, %xmm0, %xmm3
+; AVX-64-NEXT:    vcvttss2si %xmm3, %rax
+; AVX-64-NEXT:    setae %cl
+; AVX-64-NEXT:    movzbl %cl, %ecx
+; AVX-64-NEXT:    shlq $63, %rcx
+; AVX-64-NEXT:    xorq %rax, %rcx
+; AVX-64-NEXT:    vmovq %rcx, %xmm3
+; AVX-64-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; AVX-64-NEXT:    vcomiss %xmm1, %xmm0
+; AVX-64-NEXT:    jb .LBB3_4
+; AVX-64-NEXT:  # %bb.3:
+; AVX-64-NEXT:    vmovaps %xmm1, %xmm2
+; AVX-64-NEXT:  .LBB3_4:
+; AVX-64-NEXT:    vsubss %xmm2, %xmm0, %xmm0
+; AVX-64-NEXT:    vcvttss2si %xmm0, %rax
+; AVX-64-NEXT:    setae %cl
+; AVX-64-NEXT:    movzbl %cl, %ecx
+; AVX-64-NEXT:    shlq $63, %rcx
+; AVX-64-NEXT:    xorq %rax, %rcx
+; AVX-64-NEXT:    vmovq %rcx, %xmm0
+; AVX-64-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm3[0],xmm0[0]
+; AVX-64-NEXT:    retq
+;
+; AVX512F-32-LABEL: strict_vector_fptoui_v2f32_to_v2i64:
+; AVX512F-32:       # %bb.0:
+; AVX512F-32-NEXT:    pushl %ebp
+; AVX512F-32-NEXT:    .cfi_def_cfa_offset 8
+; AVX512F-32-NEXT:    .cfi_offset %ebp, -8
+; AVX512F-32-NEXT:    movl %esp, %ebp
+; AVX512F-32-NEXT:    .cfi_def_cfa_register %ebp
+; AVX512F-32-NEXT:    andl $-8, %esp
+; AVX512F-32-NEXT:    subl $16, %esp
+; AVX512F-32-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX512F-32-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; AVX512F-32-NEXT:    xorl %eax, %eax
+; AVX512F-32-NEXT:    vcomiss %xmm2, %xmm1
+; AVX512F-32-NEXT:    setb %cl
+; AVX512F-32-NEXT:    kmovw %ecx, %k1
+; AVX512F-32-NEXT:    vxorps %xmm3, %xmm3, %xmm3
+; AVX512F-32-NEXT:    vmovaps %xmm2, %xmm4
+; AVX512F-32-NEXT:    vmovss %xmm3, %xmm4, %xmm4 {%k1}
+; AVX512F-32-NEXT:    vsubss %xmm4, %xmm1, %xmm1
+; AVX512F-32-NEXT:    vmovss %xmm1, {{[0-9]+}}(%esp)
+; AVX512F-32-NEXT:    flds {{[0-9]+}}(%esp)
+; AVX512F-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX512F-32-NEXT:    setae %al
+; AVX512F-32-NEXT:    shll $31, %eax
+; AVX512F-32-NEXT:    xorl {{[0-9]+}}(%esp), %eax
+; AVX512F-32-NEXT:    xorl %ecx, %ecx
+; AVX512F-32-NEXT:    vcomiss %xmm2, %xmm0
+; AVX512F-32-NEXT:    setb %dl
+; AVX512F-32-NEXT:    kmovw %edx, %k1
+; AVX512F-32-NEXT:    vmovss %xmm3, %xmm2, %xmm2 {%k1}
+; AVX512F-32-NEXT:    vsubss %xmm2, %xmm0, %xmm0
+; AVX512F-32-NEXT:    vmovss %xmm0, (%esp)
+; AVX512F-32-NEXT:    flds (%esp)
+; AVX512F-32-NEXT:    fisttpll (%esp)
+; AVX512F-32-NEXT:    setae %cl
+; AVX512F-32-NEXT:    shll $31, %ecx
+; AVX512F-32-NEXT:    xorl {{[0-9]+}}(%esp), %ecx
+; AVX512F-32-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX512F-32-NEXT:    vpinsrd $1, %ecx, %xmm0, %xmm0
+; AVX512F-32-NEXT:    vpinsrd $2, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512F-32-NEXT:    vpinsrd $3, %eax, %xmm0, %xmm0
+; AVX512F-32-NEXT:    movl %ebp, %esp
+; AVX512F-32-NEXT:    popl %ebp
+; AVX512F-32-NEXT:    .cfi_def_cfa %esp, 4
+; AVX512F-32-NEXT:    retl
+;
+; AVX512F-64-LABEL: strict_vector_fptoui_v2f32_to_v2i64:
+; AVX512F-64:       # %bb.0:
+; AVX512F-64-NEXT:    vcvttss2usi %xmm0, %rax
+; AVX512F-64-NEXT:    vmovq %rax, %xmm1
+; AVX512F-64-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; AVX512F-64-NEXT:    vcvttss2usi %xmm0, %rax
+; AVX512F-64-NEXT:    vmovq %rax, %xmm0
+; AVX512F-64-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512F-64-NEXT:    retq
+;
+; AVX512VL-32-LABEL: strict_vector_fptoui_v2f32_to_v2i64:
+; AVX512VL-32:       # %bb.0:
+; AVX512VL-32-NEXT:    pushl %ebp
+; AVX512VL-32-NEXT:    .cfi_def_cfa_offset 8
+; AVX512VL-32-NEXT:    .cfi_offset %ebp, -8
+; AVX512VL-32-NEXT:    movl %esp, %ebp
+; AVX512VL-32-NEXT:    .cfi_def_cfa_register %ebp
+; AVX512VL-32-NEXT:    andl $-8, %esp
+; AVX512VL-32-NEXT:    subl $16, %esp
+; AVX512VL-32-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX512VL-32-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; AVX512VL-32-NEXT:    xorl %eax, %eax
+; AVX512VL-32-NEXT:    vcomiss %xmm2, %xmm1
+; AVX512VL-32-NEXT:    setb %cl
+; AVX512VL-32-NEXT:    kmovw %ecx, %k1
+; AVX512VL-32-NEXT:    vxorps %xmm3, %xmm3, %xmm3
+; AVX512VL-32-NEXT:    vmovaps %xmm2, %xmm4
+; AVX512VL-32-NEXT:    vmovss %xmm3, %xmm4, %xmm4 {%k1}
+; AVX512VL-32-NEXT:    vsubss %xmm4, %xmm1, %xmm1
+; AVX512VL-32-NEXT:    vmovss %xmm1, {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    flds {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    setae %al
+; AVX512VL-32-NEXT:    shll $31, %eax
+; AVX512VL-32-NEXT:    xorl {{[0-9]+}}(%esp), %eax
+; AVX512VL-32-NEXT:    xorl %ecx, %ecx
+; AVX512VL-32-NEXT:    vcomiss %xmm2, %xmm0
+; AVX512VL-32-NEXT:    setb %dl
+; AVX512VL-32-NEXT:    kmovw %edx, %k1
+; AVX512VL-32-NEXT:    vmovss %xmm3, %xmm2, %xmm2 {%k1}
+; AVX512VL-32-NEXT:    vsubss %xmm2, %xmm0, %xmm0
+; AVX512VL-32-NEXT:    vmovss %xmm0, (%esp)
+; AVX512VL-32-NEXT:    flds (%esp)
+; AVX512VL-32-NEXT:    fisttpll (%esp)
+; AVX512VL-32-NEXT:    setae %cl
+; AVX512VL-32-NEXT:    shll $31, %ecx
+; AVX512VL-32-NEXT:    xorl {{[0-9]+}}(%esp), %ecx
+; AVX512VL-32-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX512VL-32-NEXT:    vpinsrd $1, %ecx, %xmm0, %xmm0
+; AVX512VL-32-NEXT:    vpinsrd $2, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512VL-32-NEXT:    vpinsrd $3, %eax, %xmm0, %xmm0
+; AVX512VL-32-NEXT:    movl %ebp, %esp
+; AVX512VL-32-NEXT:    popl %ebp
+; AVX512VL-32-NEXT:    .cfi_def_cfa %esp, 4
+; AVX512VL-32-NEXT:    retl
+;
+; AVX512VL-64-LABEL: strict_vector_fptoui_v2f32_to_v2i64:
+; AVX512VL-64:       # %bb.0:
+; AVX512VL-64-NEXT:    vcvttss2usi %xmm0, %rax
+; AVX512VL-64-NEXT:    vmovq %rax, %xmm1
+; AVX512VL-64-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; AVX512VL-64-NEXT:    vcvttss2usi %xmm0, %rax
+; AVX512VL-64-NEXT:    vmovq %rax, %xmm0
+; AVX512VL-64-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512VL-64-NEXT:    retq
+;
+; AVX512DQ-32-LABEL: strict_vector_fptoui_v2f32_to_v2i64:
+; AVX512DQ-32:       # %bb.0:
+; AVX512DQ-32-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; AVX512DQ-32-NEXT:    vblendps {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3]
+; AVX512DQ-32-NEXT:    vcvttps2uqq %ymm1, %zmm1
+; AVX512DQ-32-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[1],zero,zero,zero
+; AVX512DQ-32-NEXT:    vcvttps2uqq %ymm0, %zmm0
+; AVX512DQ-32-NEXT:    vmovd %xmm0, %eax
+; AVX512DQ-32-NEXT:    vpinsrd $2, %eax, %xmm1, %xmm1
+; AVX512DQ-32-NEXT:    vpextrd $1, %xmm0, %eax
+; AVX512DQ-32-NEXT:    vpinsrd $3, %eax, %xmm1, %xmm0
+; AVX512DQ-32-NEXT:    vzeroupper
+; AVX512DQ-32-NEXT:    retl
+;
+; AVX512DQ-64-LABEL: strict_vector_fptoui_v2f32_to_v2i64:
+; AVX512DQ-64:       # %bb.0:
+; AVX512DQ-64-NEXT:    vcvttss2usi %xmm0, %rax
+; AVX512DQ-64-NEXT:    vmovq %rax, %xmm1
+; AVX512DQ-64-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; AVX512DQ-64-NEXT:    vcvttss2usi %xmm0, %rax
+; AVX512DQ-64-NEXT:    vmovq %rax, %xmm0
+; AVX512DQ-64-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512DQ-64-NEXT:    retq
+;
+; AVX512VLDQ-LABEL: strict_vector_fptoui_v2f32_to_v2i64:
+; AVX512VLDQ:       # %bb.0:
+; AVX512VLDQ-NEXT:    vcvttps2uqq %xmm0, %xmm0
+; AVX512VLDQ-NEXT:    ret{{[l|q]}}
+  %ret = call <2 x i64> @llvm.experimental.constrained.fptoui.v2i64.v2f32(<2 x float> %a,
+                                              metadata !"fpexcept.strict")
+  ret <2 x i64> %ret
+}
+
+define <2 x i32> @strict_vector_fptosi_v2f64_to_v2i32(<2 x double> %a) #0 {
+; SSE-32-LABEL: strict_vector_fptosi_v2f64_to_v2i32:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    cvttpd2dq %xmm0, %xmm0
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: strict_vector_fptosi_v2f64_to_v2i32:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    cvttpd2dq %xmm0, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-LABEL: strict_vector_fptosi_v2f64_to_v2i32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcvttpd2dq %xmm0, %xmm0
+; AVX-NEXT:    ret{{[l|q]}}
+;
+; AVX512F-LABEL: strict_vector_fptosi_v2f64_to_v2i32:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vcvttpd2dq %xmm0, %xmm0
+; AVX512F-NEXT:    ret{{[l|q]}}
+;
+; AVX512VL-LABEL: strict_vector_fptosi_v2f64_to_v2i32:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vcvttpd2dq %xmm0, %xmm0
+; AVX512VL-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQ-LABEL: strict_vector_fptosi_v2f64_to_v2i32:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vcvttpd2dq %xmm0, %xmm0
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+;
+; AVX512VLDQ-LABEL: strict_vector_fptosi_v2f64_to_v2i32:
+; AVX512VLDQ:       # %bb.0:
+; AVX512VLDQ-NEXT:    vcvttpd2dq %xmm0, %xmm0
+; AVX512VLDQ-NEXT:    ret{{[l|q]}}
+  %ret = call <2 x i32> @llvm.experimental.constrained.fptosi.v2i32.v2f64(<2 x double> %a,
+                                              metadata !"fpexcept.strict")
+  ret <2 x i32> %ret
+}
+
+define <2 x i32> @strict_vector_fptoui_v2f64_to_v2i32(<2 x double> %a) #0 {
+; SSE-32-LABEL: strict_vector_fptoui_v2f64_to_v2i32:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
+; SSE-32-NEXT:    comisd %xmm3, %xmm0
+; SSE-32-NEXT:    xorpd %xmm2, %xmm2
+; SSE-32-NEXT:    xorpd %xmm1, %xmm1
+; SSE-32-NEXT:    jb .LBB5_2
+; SSE-32-NEXT:  # %bb.1:
+; SSE-32-NEXT:    movapd %xmm3, %xmm1
+; SSE-32-NEXT:  .LBB5_2:
+; SSE-32-NEXT:    setae %al
+; SSE-32-NEXT:    movzbl %al, %eax
+; SSE-32-NEXT:    shll $31, %eax
+; SSE-32-NEXT:    movapd %xmm0, %xmm4
+; SSE-32-NEXT:    subsd %xmm1, %xmm4
+; SSE-32-NEXT:    cvttsd2si %xmm4, %ecx
+; SSE-32-NEXT:    xorl %eax, %ecx
+; SSE-32-NEXT:    movd %ecx, %xmm1
+; SSE-32-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1,1]
+; SSE-32-NEXT:    comisd %xmm3, %xmm0
+; SSE-32-NEXT:    jb .LBB5_4
+; SSE-32-NEXT:  # %bb.3:
+; SSE-32-NEXT:    movapd %xmm3, %xmm2
+; SSE-32-NEXT:  .LBB5_4:
+; SSE-32-NEXT:    setae %al
+; SSE-32-NEXT:    movzbl %al, %eax
+; SSE-32-NEXT:    shll $31, %eax
+; SSE-32-NEXT:    subsd %xmm2, %xmm0
+; SSE-32-NEXT:    cvttsd2si %xmm0, %ecx
+; SSE-32-NEXT:    xorl %eax, %ecx
+; SSE-32-NEXT:    movd %ecx, %xmm0
+; SSE-32-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE-32-NEXT:    movdqa %xmm1, %xmm0
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: strict_vector_fptoui_v2f64_to_v2i32:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    cvttsd2si %xmm0, %rax
+; SSE-64-NEXT:    movd %eax, %xmm1
+; SSE-64-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1,1]
+; SSE-64-NEXT:    cvttsd2si %xmm0, %rax
+; SSE-64-NEXT:    movd %eax, %xmm0
+; SSE-64-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE-64-NEXT:    movdqa %xmm1, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: strict_vector_fptoui_v2f64_to_v2i32:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    .cfi_def_cfa_offset 8
+; AVX-32-NEXT:    .cfi_offset %ebp, -8
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    .cfi_def_cfa_register %ebp
+; AVX-32-NEXT:    andl $-8, %esp
+; AVX-32-NEXT:    subl $16, %esp
+; AVX-32-NEXT:    vmovlps %xmm0, {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    vmovhps %xmm0, (%esp)
+; AVX-32-NEXT:    fldl {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    fldl (%esp)
+; AVX-32-NEXT:    fisttpll (%esp)
+; AVX-32-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vpinsrd $1, (%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    .cfi_def_cfa %esp, 4
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: strict_vector_fptoui_v2f64_to_v2i32:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
+; AVX-64-NEXT:    vcvttsd2si %xmm1, %rax
+; AVX-64-NEXT:    vcvttsd2si %xmm0, %rcx
+; AVX-64-NEXT:    vmovd %ecx, %xmm0
+; AVX-64-NEXT:    vpinsrd $1, %eax, %xmm0, %xmm0
+; AVX-64-NEXT:    retq
+;
+; AVX512F-LABEL: strict_vector_fptoui_v2f64_to_v2i32:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vmovaps %xmm0, %xmm0
+; AVX512F-NEXT:    vcvttpd2udq %zmm0, %ymm0
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    ret{{[l|q]}}
+;
+; AVX512VL-LABEL: strict_vector_fptoui_v2f64_to_v2i32:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vcvttpd2udq %xmm0, %xmm0
+; AVX512VL-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQ-LABEL: strict_vector_fptoui_v2f64_to_v2i32:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vmovaps %xmm0, %xmm0
+; AVX512DQ-NEXT:    vcvttpd2udq %zmm0, %ymm0
+; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+;
+; AVX512VLDQ-LABEL: strict_vector_fptoui_v2f64_to_v2i32:
+; AVX512VLDQ:       # %bb.0:
+; AVX512VLDQ-NEXT:    vcvttpd2udq %xmm0, %xmm0
+; AVX512VLDQ-NEXT:    ret{{[l|q]}}
+  %ret = call <2 x i32> @llvm.experimental.constrained.fptoui.v2i32.v2f64(<2 x double> %a,
+                                              metadata !"fpexcept.strict")
+  ret <2 x i32> %ret
+}
+
+define <2 x i32> @strict_vector_fptosi_v2f32_to_v2i32(<2 x float> %a) #0 {
+; SSE-32-LABEL: strict_vector_fptosi_v2f32_to_v2i32:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    movq {{.*#+}} xmm0 = xmm0[0],zero
+; SSE-32-NEXT:    cvttps2dq %xmm0, %xmm0
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: strict_vector_fptosi_v2f32_to_v2i32:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movq {{.*#+}} xmm0 = xmm0[0],zero
+; SSE-64-NEXT:    cvttps2dq %xmm0, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-LABEL: strict_vector_fptosi_v2f32_to_v2i32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
+; AVX-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX-NEXT:    ret{{[l|q]}}
+;
+; AVX512F-LABEL: strict_vector_fptosi_v2f32_to_v2i32:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
+; AVX512F-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX512F-NEXT:    ret{{[l|q]}}
+;
+; AVX512VL-LABEL: strict_vector_fptosi_v2f32_to_v2i32:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
+; AVX512VL-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX512VL-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQ-LABEL: strict_vector_fptosi_v2f32_to_v2i32:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
+; AVX512DQ-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+;
+; AVX512VLDQ-LABEL: strict_vector_fptosi_v2f32_to_v2i32:
+; AVX512VLDQ:       # %bb.0:
+; AVX512VLDQ-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
+; AVX512VLDQ-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX512VLDQ-NEXT:    ret{{[l|q]}}
+  %ret = call <2 x i32> @llvm.experimental.constrained.fptosi.v2i32.v2f32(<2 x float> %a,
+                                              metadata !"fpexcept.strict")
+  ret <2 x i32> %ret
+}
+
+define <2 x i32> @strict_vector_fptoui_v2f32_to_v2i32(<2 x float> %a) #0 {
+; SSE-32-LABEL: strict_vector_fptoui_v2f32_to_v2i32:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    movss {{.*#+}} xmm3 = mem[0],zero,zero,zero
+; SSE-32-NEXT:    comiss %xmm3, %xmm0
+; SSE-32-NEXT:    xorps %xmm2, %xmm2
+; SSE-32-NEXT:    xorps %xmm1, %xmm1
+; SSE-32-NEXT:    jb .LBB7_2
+; SSE-32-NEXT:  # %bb.1:
+; SSE-32-NEXT:    movaps %xmm3, %xmm1
+; SSE-32-NEXT:  .LBB7_2:
+; SSE-32-NEXT:    setae %al
+; SSE-32-NEXT:    movzbl %al, %eax
+; SSE-32-NEXT:    shll $31, %eax
+; SSE-32-NEXT:    movaps %xmm0, %xmm4
+; SSE-32-NEXT:    subss %xmm1, %xmm4
+; SSE-32-NEXT:    cvttss2si %xmm4, %ecx
+; SSE-32-NEXT:    xorl %eax, %ecx
+; SSE-32-NEXT:    movd %ecx, %xmm1
+; SSE-32-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE-32-NEXT:    comiss %xmm3, %xmm0
+; SSE-32-NEXT:    jb .LBB7_4
+; SSE-32-NEXT:  # %bb.3:
+; SSE-32-NEXT:    movaps %xmm3, %xmm2
+; SSE-32-NEXT:  .LBB7_4:
+; SSE-32-NEXT:    setae %al
+; SSE-32-NEXT:    movzbl %al, %eax
+; SSE-32-NEXT:    shll $31, %eax
+; SSE-32-NEXT:    subss %xmm2, %xmm0
+; SSE-32-NEXT:    cvttss2si %xmm0, %ecx
+; SSE-32-NEXT:    xorl %eax, %ecx
+; SSE-32-NEXT:    movd %ecx, %xmm0
+; SSE-32-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE-32-NEXT:    movdqa %xmm1, %xmm0
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: strict_vector_fptoui_v2f32_to_v2i32:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    cvttss2si %xmm0, %rax
+; SSE-64-NEXT:    movd %eax, %xmm1
+; SSE-64-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE-64-NEXT:    cvttss2si %xmm0, %rax
+; SSE-64-NEXT:    movd %eax, %xmm0
+; SSE-64-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE-64-NEXT:    movdqa %xmm1, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: strict_vector_fptoui_v2f32_to_v2i32:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    .cfi_def_cfa_offset 8
+; AVX-32-NEXT:    .cfi_offset %ebp, -8
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    .cfi_def_cfa_register %ebp
+; AVX-32-NEXT:    andl $-8, %esp
+; AVX-32-NEXT:    subl $16, %esp
+; AVX-32-NEXT:    vmovss %xmm0, (%esp)
+; AVX-32-NEXT:    vextractps $1, %xmm0, {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    flds (%esp)
+; AVX-32-NEXT:    fisttpll (%esp)
+; AVX-32-NEXT:    flds {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vpinsrd $1, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    .cfi_def_cfa %esp, 4
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: strict_vector_fptoui_v2f32_to_v2i32:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX-64-NEXT:    vcvttss2si %xmm1, %rax
+; AVX-64-NEXT:    vcvttss2si %xmm0, %rcx
+; AVX-64-NEXT:    vmovd %ecx, %xmm0
+; AVX-64-NEXT:    vpinsrd $1, %eax, %xmm0, %xmm0
+; AVX-64-NEXT:    retq
+;
+; AVX512F-LABEL: strict_vector_fptoui_v2f32_to_v2i32:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
+; AVX512F-NEXT:    vcvttps2udq %zmm0, %zmm0
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    ret{{[l|q]}}
+;
+; AVX512VL-LABEL: strict_vector_fptoui_v2f32_to_v2i32:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
+; AVX512VL-NEXT:    vcvttps2udq %xmm0, %xmm0
+; AVX512VL-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQ-LABEL: strict_vector_fptoui_v2f32_to_v2i32:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
+; AVX512DQ-NEXT:    vcvttps2udq %zmm0, %zmm0
+; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+;
+; AVX512VLDQ-LABEL: strict_vector_fptoui_v2f32_to_v2i32:
+; AVX512VLDQ:       # %bb.0:
+; AVX512VLDQ-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
+; AVX512VLDQ-NEXT:    vcvttps2udq %xmm0, %xmm0
+; AVX512VLDQ-NEXT:    ret{{[l|q]}}
+  %ret = call <2 x i32> @llvm.experimental.constrained.fptoui.v2i32.v2f32(<2 x float> %a,
+                                              metadata !"fpexcept.strict")
+  ret <2 x i32> %ret
+}
+
+define <2 x i16> @strict_vector_fptosi_v2f64_to_v2i16(<2 x double> %a) #0 {
+; SSE-32-LABEL: strict_vector_fptosi_v2f64_to_v2i16:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    cvttpd2dq %xmm0, %xmm0
+; SSE-32-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: strict_vector_fptosi_v2f64_to_v2i16:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    cvttpd2dq %xmm0, %xmm0
+; SSE-64-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; SSE-64-NEXT:    retq
+;
+; AVX-LABEL: strict_vector_fptosi_v2f64_to_v2i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcvttpd2dq %xmm0, %xmm0
+; AVX-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; AVX-NEXT:    ret{{[l|q]}}
+;
+; AVX512F-LABEL: strict_vector_fptosi_v2f64_to_v2i16:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vcvttpd2dq %xmm0, %xmm0
+; AVX512F-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; AVX512F-NEXT:    ret{{[l|q]}}
+;
+; AVX512VL-LABEL: strict_vector_fptosi_v2f64_to_v2i16:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vcvttpd2dq %xmm0, %xmm0
+; AVX512VL-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; AVX512VL-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQ-LABEL: strict_vector_fptosi_v2f64_to_v2i16:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vcvttpd2dq %xmm0, %xmm0
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+;
+; AVX512VLDQ-LABEL: strict_vector_fptosi_v2f64_to_v2i16:
+; AVX512VLDQ:       # %bb.0:
+; AVX512VLDQ-NEXT:    vcvttpd2dq %xmm0, %xmm0
+; AVX512VLDQ-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; AVX512VLDQ-NEXT:    ret{{[l|q]}}
+  %ret = call <2 x i16> @llvm.experimental.constrained.fptosi.v2i16.v2f64(<2 x double> %a,
+                                              metadata !"fpexcept.strict")
+  ret <2 x i16> %ret
+}
+
+define <2 x i16> @strict_vector_fptoui_v2f64_to_v2i16(<2 x double> %a) #0 {
+; SSE-32-LABEL: strict_vector_fptoui_v2f64_to_v2i16:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    cvttpd2dq %xmm0, %xmm0
+; SSE-32-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: strict_vector_fptoui_v2f64_to_v2i16:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    cvttpd2dq %xmm0, %xmm0
+; SSE-64-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; SSE-64-NEXT:    retq
+;
+; AVX-LABEL: strict_vector_fptoui_v2f64_to_v2i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcvttpd2dq %xmm0, %xmm0
+; AVX-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; AVX-NEXT:    ret{{[l|q]}}
+;
+; AVX512F-LABEL: strict_vector_fptoui_v2f64_to_v2i16:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vcvttpd2dq %xmm0, %xmm0
+; AVX512F-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; AVX512F-NEXT:    ret{{[l|q]}}
+;
+; AVX512VL-LABEL: strict_vector_fptoui_v2f64_to_v2i16:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vcvttpd2dq %xmm0, %xmm0
+; AVX512VL-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; AVX512VL-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQ-LABEL: strict_vector_fptoui_v2f64_to_v2i16:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vcvttpd2dq %xmm0, %xmm0
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+;
+; AVX512VLDQ-LABEL: strict_vector_fptoui_v2f64_to_v2i16:
+; AVX512VLDQ:       # %bb.0:
+; AVX512VLDQ-NEXT:    vcvttpd2dq %xmm0, %xmm0
+; AVX512VLDQ-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; AVX512VLDQ-NEXT:    ret{{[l|q]}}
+  %ret = call <2 x i16> @llvm.experimental.constrained.fptoui.v2i16.v2f64(<2 x double> %a,
+                                              metadata !"fpexcept.strict")
+  ret <2 x i16> %ret
+}
+
+define <2 x i16> @strict_vector_fptosi_v2f32_to_v2i16(<2 x float> %a) #0 {
+; SSE-32-LABEL: strict_vector_fptosi_v2f32_to_v2i16:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    movq {{.*#+}} xmm0 = xmm0[0],zero
+; SSE-32-NEXT:    cvttps2dq %xmm0, %xmm0
+; SSE-32-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: strict_vector_fptosi_v2f32_to_v2i16:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movq {{.*#+}} xmm0 = xmm0[0],zero
+; SSE-64-NEXT:    cvttps2dq %xmm0, %xmm0
+; SSE-64-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; SSE-64-NEXT:    retq
+;
+; AVX-LABEL: strict_vector_fptosi_v2f32_to_v2i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
+; AVX-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; AVX-NEXT:    ret{{[l|q]}}
+;
+; AVX512F-LABEL: strict_vector_fptosi_v2f32_to_v2i16:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
+; AVX512F-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX512F-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; AVX512F-NEXT:    ret{{[l|q]}}
+;
+; AVX512VL-LABEL: strict_vector_fptosi_v2f32_to_v2i16:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
+; AVX512VL-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX512VL-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; AVX512VL-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQ-LABEL: strict_vector_fptosi_v2f32_to_v2i16:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
+; AVX512DQ-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+;
+; AVX512VLDQ-LABEL: strict_vector_fptosi_v2f32_to_v2i16:
+; AVX512VLDQ:       # %bb.0:
+; AVX512VLDQ-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
+; AVX512VLDQ-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX512VLDQ-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; AVX512VLDQ-NEXT:    ret{{[l|q]}}
+  %ret = call <2 x i16> @llvm.experimental.constrained.fptosi.v2i16.v2f32(<2 x float> %a,
+                                              metadata !"fpexcept.strict")
+  ret <2 x i16> %ret
+}
+
+define <2 x i16> @strict_vector_fptoui_v2f32_to_v2i16(<2 x float> %a) #0 {
+; SSE-32-LABEL: strict_vector_fptoui_v2f32_to_v2i16:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    movq {{.*#+}} xmm0 = xmm0[0],zero
+; SSE-32-NEXT:    cvttps2dq %xmm0, %xmm0
+; SSE-32-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: strict_vector_fptoui_v2f32_to_v2i16:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movq {{.*#+}} xmm0 = xmm0[0],zero
+; SSE-64-NEXT:    cvttps2dq %xmm0, %xmm0
+; SSE-64-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; SSE-64-NEXT:    retq
+;
+; AVX-LABEL: strict_vector_fptoui_v2f32_to_v2i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
+; AVX-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; AVX-NEXT:    ret{{[l|q]}}
+;
+; AVX512F-LABEL: strict_vector_fptoui_v2f32_to_v2i16:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
+; AVX512F-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX512F-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; AVX512F-NEXT:    ret{{[l|q]}}
+;
+; AVX512VL-LABEL: strict_vector_fptoui_v2f32_to_v2i16:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
+; AVX512VL-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX512VL-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; AVX512VL-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQ-LABEL: strict_vector_fptoui_v2f32_to_v2i16:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
+; AVX512DQ-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+;
+; AVX512VLDQ-LABEL: strict_vector_fptoui_v2f32_to_v2i16:
+; AVX512VLDQ:       # %bb.0:
+; AVX512VLDQ-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
+; AVX512VLDQ-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX512VLDQ-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; AVX512VLDQ-NEXT:    ret{{[l|q]}}
+  %ret = call <2 x i16> @llvm.experimental.constrained.fptoui.v2i16.v2f32(<2 x float> %a,
+                                              metadata !"fpexcept.strict")
+  ret <2 x i16> %ret
+}
+
+define <2 x i8> @strict_vector_fptosi_v2f64_to_v2i8(<2 x double> %a) #0 {
+; SSE-32-LABEL: strict_vector_fptosi_v2f64_to_v2i8:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    cvttpd2dq %xmm0, %xmm0
+; SSE-32-NEXT:    andpd {{\.LCPI.*}}, %xmm0
+; SSE-32-NEXT:    packuswb %xmm0, %xmm0
+; SSE-32-NEXT:    packuswb %xmm0, %xmm0
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: strict_vector_fptosi_v2f64_to_v2i8:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    cvttpd2dq %xmm0, %xmm0
+; SSE-64-NEXT:    andpd {{.*}}(%rip), %xmm0
+; SSE-64-NEXT:    packuswb %xmm0, %xmm0
+; SSE-64-NEXT:    packuswb %xmm0, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-LABEL: strict_vector_fptosi_v2f64_to_v2i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcvttpd2dq %xmm0, %xmm0
+; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX-NEXT:    ret{{[l|q]}}
+;
+; AVX512F-LABEL: strict_vector_fptosi_v2f64_to_v2i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vcvttpd2dq %xmm0, %xmm0
+; AVX512F-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512F-NEXT:    ret{{[l|q]}}
+;
+; AVX512VL-LABEL: strict_vector_fptosi_v2f64_to_v2i8:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vcvttpd2dq %xmm0, %xmm0
+; AVX512VL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512VL-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQ-LABEL: strict_vector_fptosi_v2f64_to_v2i8:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vcvttpd2dq %xmm0, %xmm0
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+;
+; AVX512VLDQ-LABEL: strict_vector_fptosi_v2f64_to_v2i8:
+; AVX512VLDQ:       # %bb.0:
+; AVX512VLDQ-NEXT:    vcvttpd2dq %xmm0, %xmm0
+; AVX512VLDQ-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512VLDQ-NEXT:    ret{{[l|q]}}
+  %ret = call <2 x i8> @llvm.experimental.constrained.fptosi.v2i8.v2f64(<2 x double> %a,
+                                              metadata !"fpexcept.strict")
+  ret <2 x i8> %ret
+}
+
+define <2 x i8> @strict_vector_fptoui_v2f64_to_v2i8(<2 x double> %a) #0 {
+; SSE-32-LABEL: strict_vector_fptoui_v2f64_to_v2i8:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    cvttpd2dq %xmm0, %xmm0
+; SSE-32-NEXT:    andpd {{\.LCPI.*}}, %xmm0
+; SSE-32-NEXT:    packuswb %xmm0, %xmm0
+; SSE-32-NEXT:    packuswb %xmm0, %xmm0
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: strict_vector_fptoui_v2f64_to_v2i8:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    cvttpd2dq %xmm0, %xmm0
+; SSE-64-NEXT:    andpd {{.*}}(%rip), %xmm0
+; SSE-64-NEXT:    packuswb %xmm0, %xmm0
+; SSE-64-NEXT:    packuswb %xmm0, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-LABEL: strict_vector_fptoui_v2f64_to_v2i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcvttpd2dq %xmm0, %xmm0
+; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX-NEXT:    ret{{[l|q]}}
+;
+; AVX512F-LABEL: strict_vector_fptoui_v2f64_to_v2i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vcvttpd2dq %xmm0, %xmm0
+; AVX512F-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512F-NEXT:    ret{{[l|q]}}
+;
+; AVX512VL-LABEL: strict_vector_fptoui_v2f64_to_v2i8:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vcvttpd2dq %xmm0, %xmm0
+; AVX512VL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512VL-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQ-LABEL: strict_vector_fptoui_v2f64_to_v2i8:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vcvttpd2dq %xmm0, %xmm0
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+;
+; AVX512VLDQ-LABEL: strict_vector_fptoui_v2f64_to_v2i8:
+; AVX512VLDQ:       # %bb.0:
+; AVX512VLDQ-NEXT:    vcvttpd2dq %xmm0, %xmm0
+; AVX512VLDQ-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512VLDQ-NEXT:    ret{{[l|q]}}
+  %ret = call <2 x i8> @llvm.experimental.constrained.fptoui.v2i8.v2f64(<2 x double> %a,
+                                              metadata !"fpexcept.strict")
+  ret <2 x i8> %ret
+}
+
+define <2 x i8> @strict_vector_fptosi_v2f32_to_v2i8(<2 x float> %a) #0 {
+; SSE-32-LABEL: strict_vector_fptosi_v2f32_to_v2i8:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    movq {{.*#+}} xmm0 = xmm0[0],zero
+; SSE-32-NEXT:    cvttps2dq %xmm0, %xmm0
+; SSE-32-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; SSE-32-NEXT:    packuswb %xmm0, %xmm0
+; SSE-32-NEXT:    packuswb %xmm0, %xmm0
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: strict_vector_fptosi_v2f32_to_v2i8:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movq {{.*#+}} xmm0 = xmm0[0],zero
+; SSE-64-NEXT:    cvttps2dq %xmm0, %xmm0
+; SSE-64-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE-64-NEXT:    packuswb %xmm0, %xmm0
+; SSE-64-NEXT:    packuswb %xmm0, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-LABEL: strict_vector_fptosi_v2f32_to_v2i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
+; AVX-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX-NEXT:    ret{{[l|q]}}
+;
+; AVX512F-LABEL: strict_vector_fptosi_v2f32_to_v2i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
+; AVX512F-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX512F-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512F-NEXT:    ret{{[l|q]}}
+;
+; AVX512VL-LABEL: strict_vector_fptosi_v2f32_to_v2i8:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
+; AVX512VL-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX512VL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512VL-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQ-LABEL: strict_vector_fptosi_v2f32_to_v2i8:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
+; AVX512DQ-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+;
+; AVX512VLDQ-LABEL: strict_vector_fptosi_v2f32_to_v2i8:
+; AVX512VLDQ:       # %bb.0:
+; AVX512VLDQ-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
+; AVX512VLDQ-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX512VLDQ-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512VLDQ-NEXT:    ret{{[l|q]}}
+  %ret = call <2 x i8> @llvm.experimental.constrained.fptosi.v2i8.v2f32(<2 x float> %a,
+                                              metadata !"fpexcept.strict")
+  ret <2 x i8> %ret
+}
+
+define <2 x i8> @strict_vector_fptoui_v2f32_to_v2i8(<2 x float> %a) #0 {
+; SSE-32-LABEL: strict_vector_fptoui_v2f32_to_v2i8:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    movq {{.*#+}} xmm0 = xmm0[0],zero
+; SSE-32-NEXT:    cvttps2dq %xmm0, %xmm0
+; SSE-32-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; SSE-32-NEXT:    packuswb %xmm0, %xmm0
+; SSE-32-NEXT:    packuswb %xmm0, %xmm0
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: strict_vector_fptoui_v2f32_to_v2i8:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movq {{.*#+}} xmm0 = xmm0[0],zero
+; SSE-64-NEXT:    cvttps2dq %xmm0, %xmm0
+; SSE-64-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE-64-NEXT:    packuswb %xmm0, %xmm0
+; SSE-64-NEXT:    packuswb %xmm0, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-LABEL: strict_vector_fptoui_v2f32_to_v2i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
+; AVX-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX-NEXT:    ret{{[l|q]}}
+;
+; AVX512F-LABEL: strict_vector_fptoui_v2f32_to_v2i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
+; AVX512F-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX512F-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512F-NEXT:    ret{{[l|q]}}
+;
+; AVX512VL-LABEL: strict_vector_fptoui_v2f32_to_v2i8:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
+; AVX512VL-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX512VL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512VL-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQ-LABEL: strict_vector_fptoui_v2f32_to_v2i8:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
+; AVX512DQ-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+;
+; AVX512VLDQ-LABEL: strict_vector_fptoui_v2f32_to_v2i8:
+; AVX512VLDQ:       # %bb.0:
+; AVX512VLDQ-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
+; AVX512VLDQ-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX512VLDQ-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512VLDQ-NEXT:    ret{{[l|q]}}
+  %ret = call <2 x i8> @llvm.experimental.constrained.fptoui.v2i8.v2f32(<2 x float> %a,
+                                              metadata !"fpexcept.strict")
+  ret <2 x i8> %ret
+}
+
+define <2 x i1> @strict_vector_fptosi_v2f64_to_v2i1(<2 x double> %a) #0 {
+; SSE-32-LABEL: strict_vector_fptosi_v2f64_to_v2i1:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    .cfi_def_cfa_offset 8
+; SSE-32-NEXT:    .cfi_offset %ebp, -8
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    .cfi_def_cfa_register %ebp
+; SSE-32-NEXT:    andl $-8, %esp
+; SSE-32-NEXT:    subl $24, %esp
+; SSE-32-NEXT:    movhps %xmm0, {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    movlps %xmm0, {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fldl {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fnstcw {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; SSE-32-NEXT:    orl $3072, %eax # imm = 0xC00
+; SSE-32-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fldcw {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fistpll {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fldcw {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fldl {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fnstcw (%esp)
+; SSE-32-NEXT:    movzwl (%esp), %eax
+; SSE-32-NEXT:    orl $3072, %eax # imm = 0xC00
+; SSE-32-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fldcw {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fistpll {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fldcw (%esp)
+; SSE-32-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
+; SSE-32-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE-32-NEXT:    movlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    .cfi_def_cfa %esp, 4
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: strict_vector_fptosi_v2f64_to_v2i1:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    cvttsd2si %xmm0, %rax
+; SSE-64-NEXT:    movq %rax, %xmm1
+; SSE-64-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1,1]
+; SSE-64-NEXT:    cvttsd2si %xmm0, %rax
+; SSE-64-NEXT:    movq %rax, %xmm0
+; SSE-64-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSE-64-NEXT:    movdqa %xmm1, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: strict_vector_fptosi_v2f64_to_v2i1:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    .cfi_def_cfa_offset 8
+; AVX-32-NEXT:    .cfi_offset %ebp, -8
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    .cfi_def_cfa_register %ebp
+; AVX-32-NEXT:    andl $-8, %esp
+; AVX-32-NEXT:    subl $16, %esp
+; AVX-32-NEXT:    vmovlps %xmm0, {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    vmovhps %xmm0, (%esp)
+; AVX-32-NEXT:    fldl {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    fldl (%esp)
+; AVX-32-NEXT:    fisttpll (%esp)
+; AVX-32-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vpinsrd $1, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrd $2, (%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrd $3, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    .cfi_def_cfa %esp, 4
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: strict_vector_fptosi_v2f64_to_v2i1:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcvttsd2si %xmm0, %rax
+; AVX-64-NEXT:    vmovq %rax, %xmm1
+; AVX-64-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
+; AVX-64-NEXT:    vcvttsd2si %xmm0, %rax
+; AVX-64-NEXT:    vmovq %rax, %xmm0
+; AVX-64-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX-64-NEXT:    retq
+;
+; AVX512F-LABEL: strict_vector_fptosi_v2f64_to_v2i1:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vcvttpd2dq %xmm0, %xmm0
+; AVX512F-NEXT:    vpslld $31, %xmm0, %xmm0
+; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k1
+; AVX512F-NEXT:    vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    ret{{[l|q]}}
+;
+; AVX512VL-LABEL: strict_vector_fptosi_v2f64_to_v2i1:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vcvttpd2dq %xmm0, %xmm0
+; AVX512VL-NEXT:    vpslld $31, %xmm0, %xmm0
+; AVX512VL-NEXT:    vptestmd %xmm0, %xmm0, %k1
+; AVX512VL-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX512VL-NEXT:    vmovdqa64 %xmm0, %xmm0 {%k1} {z}
+; AVX512VL-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQ-LABEL: strict_vector_fptosi_v2f64_to_v2i1:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vcvttpd2dq %xmm0, %xmm0
+; AVX512DQ-NEXT:    vpslld $31, %xmm0, %xmm0
+; AVX512DQ-NEXT:    vpmovd2m %zmm0, %k0
+; AVX512DQ-NEXT:    vpmovm2q %k0, %zmm0
+; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+;
+; AVX512VLDQ-LABEL: strict_vector_fptosi_v2f64_to_v2i1:
+; AVX512VLDQ:       # %bb.0:
+; AVX512VLDQ-NEXT:    vcvttpd2dq %xmm0, %xmm0
+; AVX512VLDQ-NEXT:    vpslld $31, %xmm0, %xmm0
+; AVX512VLDQ-NEXT:    vpmovd2m %xmm0, %k0
+; AVX512VLDQ-NEXT:    vpmovm2q %k0, %xmm0
+; AVX512VLDQ-NEXT:    ret{{[l|q]}}
+  %ret = call <2 x i1> @llvm.experimental.constrained.fptosi.v2i1.v2f64(<2 x double> %a,
+                                              metadata !"fpexcept.strict")
+  ret <2 x i1> %ret
+}
+
+define <2 x i1> @strict_vector_fptoui_v2f64_to_v2i1(<2 x double> %a) #0 {
+; SSE-32-LABEL: strict_vector_fptoui_v2f64_to_v2i1:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    .cfi_def_cfa_offset 8
+; SSE-32-NEXT:    .cfi_offset %ebp, -8
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    .cfi_def_cfa_register %ebp
+; SSE-32-NEXT:    andl $-8, %esp
+; SSE-32-NEXT:    subl $24, %esp
+; SSE-32-NEXT:    movsd {{.*#+}} xmm2 = mem[0],zero
+; SSE-32-NEXT:    comisd %xmm2, %xmm0
+; SSE-32-NEXT:    xorpd %xmm1, %xmm1
+; SSE-32-NEXT:    xorpd %xmm3, %xmm3
+; SSE-32-NEXT:    jb .LBB17_2
+; SSE-32-NEXT:  # %bb.1:
+; SSE-32-NEXT:    movapd %xmm2, %xmm3
+; SSE-32-NEXT:  .LBB17_2:
+; SSE-32-NEXT:    movapd %xmm0, %xmm4
+; SSE-32-NEXT:    subsd %xmm3, %xmm4
+; SSE-32-NEXT:    movsd %xmm4, {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    setae %al
+; SSE-32-NEXT:    fldl {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fnstcw {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
+; SSE-32-NEXT:    orl $3072, %ecx # imm = 0xC00
+; SSE-32-NEXT:    movw %cx, {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fldcw {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fistpll {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fldcw {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1,1]
+; SSE-32-NEXT:    comisd %xmm2, %xmm0
+; SSE-32-NEXT:    jb .LBB17_4
+; SSE-32-NEXT:  # %bb.3:
+; SSE-32-NEXT:    movapd %xmm2, %xmm1
+; SSE-32-NEXT:  .LBB17_4:
+; SSE-32-NEXT:    subsd %xmm1, %xmm0
+; SSE-32-NEXT:    movsd %xmm0, {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    setae %cl
+; SSE-32-NEXT:    fldl {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fnstcw (%esp)
+; SSE-32-NEXT:    movzwl (%esp), %edx
+; SSE-32-NEXT:    orl $3072, %edx # imm = 0xC00
+; SSE-32-NEXT:    movw %dx, {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fldcw {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fistpll {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fldcw (%esp)
+; SSE-32-NEXT:    movzbl %al, %eax
+; SSE-32-NEXT:    shll $31, %eax
+; SSE-32-NEXT:    xorl {{[0-9]+}}(%esp), %eax
+; SSE-32-NEXT:    movd %eax, %xmm1
+; SSE-32-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE-32-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE-32-NEXT:    movzbl %cl, %eax
+; SSE-32-NEXT:    shll $31, %eax
+; SSE-32-NEXT:    xorl {{[0-9]+}}(%esp), %eax
+; SSE-32-NEXT:    movd %eax, %xmm1
+; SSE-32-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE-32-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSE-32-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    .cfi_def_cfa %esp, 4
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: strict_vector_fptoui_v2f64_to_v2i1:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
+; SSE-64-NEXT:    comisd %xmm3, %xmm0
+; SSE-64-NEXT:    xorpd %xmm2, %xmm2
+; SSE-64-NEXT:    xorpd %xmm1, %xmm1
+; SSE-64-NEXT:    jb .LBB17_2
+; SSE-64-NEXT:  # %bb.1:
+; SSE-64-NEXT:    movapd %xmm3, %xmm1
+; SSE-64-NEXT:  .LBB17_2:
+; SSE-64-NEXT:    movapd %xmm0, %xmm4
+; SSE-64-NEXT:    subsd %xmm1, %xmm4
+; SSE-64-NEXT:    cvttsd2si %xmm4, %rax
+; SSE-64-NEXT:    setae %cl
+; SSE-64-NEXT:    movzbl %cl, %ecx
+; SSE-64-NEXT:    shlq $63, %rcx
+; SSE-64-NEXT:    xorq %rax, %rcx
+; SSE-64-NEXT:    movq %rcx, %xmm1
+; SSE-64-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1,1]
+; SSE-64-NEXT:    comisd %xmm3, %xmm0
+; SSE-64-NEXT:    jb .LBB17_4
+; SSE-64-NEXT:  # %bb.3:
+; SSE-64-NEXT:    movapd %xmm3, %xmm2
+; SSE-64-NEXT:  .LBB17_4:
+; SSE-64-NEXT:    subsd %xmm2, %xmm0
+; SSE-64-NEXT:    cvttsd2si %xmm0, %rax
+; SSE-64-NEXT:    setae %cl
+; SSE-64-NEXT:    movzbl %cl, %ecx
+; SSE-64-NEXT:    shlq $63, %rcx
+; SSE-64-NEXT:    xorq %rax, %rcx
+; SSE-64-NEXT:    movq %rcx, %xmm0
+; SSE-64-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSE-64-NEXT:    movdqa %xmm1, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: strict_vector_fptoui_v2f64_to_v2i1:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    .cfi_def_cfa_offset 8
+; AVX-32-NEXT:    .cfi_offset %ebp, -8
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    .cfi_def_cfa_register %ebp
+; AVX-32-NEXT:    andl $-8, %esp
+; AVX-32-NEXT:    subl $16, %esp
+; AVX-32-NEXT:    vpermilpd {{.*#+}} xmm3 = xmm0[1,0]
+; AVX-32-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
+; AVX-32-NEXT:    vcomisd %xmm1, %xmm3
+; AVX-32-NEXT:    vxorpd %xmm2, %xmm2, %xmm2
+; AVX-32-NEXT:    vxorpd %xmm4, %xmm4, %xmm4
+; AVX-32-NEXT:    jb .LBB17_2
+; AVX-32-NEXT:  # %bb.1:
+; AVX-32-NEXT:    vmovapd %xmm1, %xmm4
+; AVX-32-NEXT:  .LBB17_2:
+; AVX-32-NEXT:    vsubsd %xmm4, %xmm3, %xmm3
+; AVX-32-NEXT:    vmovsd %xmm3, (%esp)
+; AVX-32-NEXT:    fldl (%esp)
+; AVX-32-NEXT:    fisttpll (%esp)
+; AVX-32-NEXT:    setae %al
+; AVX-32-NEXT:    movzbl %al, %eax
+; AVX-32-NEXT:    shll $31, %eax
+; AVX-32-NEXT:    xorl {{[0-9]+}}(%esp), %eax
+; AVX-32-NEXT:    vcomisd %xmm1, %xmm0
+; AVX-32-NEXT:    jb .LBB17_4
+; AVX-32-NEXT:  # %bb.3:
+; AVX-32-NEXT:    vmovapd %xmm1, %xmm2
+; AVX-32-NEXT:  .LBB17_4:
+; AVX-32-NEXT:    vsubsd %xmm2, %xmm0, %xmm0
+; AVX-32-NEXT:    vmovsd %xmm0, {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    fldl {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    setae %cl
+; AVX-32-NEXT:    movzbl %cl, %ecx
+; AVX-32-NEXT:    shll $31, %ecx
+; AVX-32-NEXT:    xorl {{[0-9]+}}(%esp), %ecx
+; AVX-32-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vpinsrd $1, %ecx, %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrd $2, (%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrd $3, %eax, %xmm0, %xmm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    .cfi_def_cfa %esp, 4
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: strict_vector_fptoui_v2f64_to_v2i1:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
+; AVX-64-NEXT:    vcomisd %xmm1, %xmm0
+; AVX-64-NEXT:    vxorpd %xmm2, %xmm2, %xmm2
+; AVX-64-NEXT:    vxorpd %xmm3, %xmm3, %xmm3
+; AVX-64-NEXT:    jb .LBB17_2
+; AVX-64-NEXT:  # %bb.1:
+; AVX-64-NEXT:    vmovapd %xmm1, %xmm3
+; AVX-64-NEXT:  .LBB17_2:
+; AVX-64-NEXT:    vsubsd %xmm3, %xmm0, %xmm3
+; AVX-64-NEXT:    vcvttsd2si %xmm3, %rax
+; AVX-64-NEXT:    setae %cl
+; AVX-64-NEXT:    movzbl %cl, %ecx
+; AVX-64-NEXT:    shlq $63, %rcx
+; AVX-64-NEXT:    xorq %rax, %rcx
+; AVX-64-NEXT:    vmovq %rcx, %xmm3
+; AVX-64-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
+; AVX-64-NEXT:    vcomisd %xmm1, %xmm0
+; AVX-64-NEXT:    jb .LBB17_4
+; AVX-64-NEXT:  # %bb.3:
+; AVX-64-NEXT:    vmovapd %xmm1, %xmm2
+; AVX-64-NEXT:  .LBB17_4:
+; AVX-64-NEXT:    vsubsd %xmm2, %xmm0, %xmm0
+; AVX-64-NEXT:    vcvttsd2si %xmm0, %rax
+; AVX-64-NEXT:    setae %cl
+; AVX-64-NEXT:    movzbl %cl, %ecx
+; AVX-64-NEXT:    shlq $63, %rcx
+; AVX-64-NEXT:    xorq %rax, %rcx
+; AVX-64-NEXT:    vmovq %rcx, %xmm0
+; AVX-64-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm3[0],xmm0[0]
+; AVX-64-NEXT:    retq
+;
+; AVX512F-LABEL: strict_vector_fptoui_v2f64_to_v2i1:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vmovaps %xmm0, %xmm0
+; AVX512F-NEXT:    vcvttpd2udq %zmm0, %ymm0
+; AVX512F-NEXT:    vpslld $31, %ymm0, %ymm0
+; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k1
+; AVX512F-NEXT:    vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    ret{{[l|q]}}
+;
+; AVX512VL-LABEL: strict_vector_fptoui_v2f64_to_v2i1:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vcvttpd2udq %xmm0, %xmm0
+; AVX512VL-NEXT:    vpslld $31, %xmm0, %xmm0
+; AVX512VL-NEXT:    vptestmd %xmm0, %xmm0, %k1
+; AVX512VL-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX512VL-NEXT:    vmovdqa64 %xmm0, %xmm0 {%k1} {z}
+; AVX512VL-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQ-LABEL: strict_vector_fptoui_v2f64_to_v2i1:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vmovaps %xmm0, %xmm0
+; AVX512DQ-NEXT:    vcvttpd2udq %zmm0, %ymm0
+; AVX512DQ-NEXT:    vpslld $31, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpmovd2m %zmm0, %k0
+; AVX512DQ-NEXT:    vpmovm2q %k0, %zmm0
+; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+;
+; AVX512VLDQ-LABEL: strict_vector_fptoui_v2f64_to_v2i1:
+; AVX512VLDQ:       # %bb.0:
+; AVX512VLDQ-NEXT:    vcvttpd2udq %xmm0, %xmm0
+; AVX512VLDQ-NEXT:    vpslld $31, %xmm0, %xmm0
+; AVX512VLDQ-NEXT:    vpmovd2m %xmm0, %k0
+; AVX512VLDQ-NEXT:    vpmovm2q %k0, %xmm0
+; AVX512VLDQ-NEXT:    ret{{[l|q]}}
+  %ret = call <2 x i1> @llvm.experimental.constrained.fptoui.v2i1.v2f64(<2 x double> %a,
+                                              metadata !"fpexcept.strict")
+  ret <2 x i1> %ret
+}
+
+define <2 x i1> @strict_vector_fptosi_v2f32_to_v2i1(<2 x float> %a) #0 {
+; SSE-32-LABEL: strict_vector_fptosi_v2f32_to_v2i1:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    .cfi_def_cfa_offset 8
+; SSE-32-NEXT:    .cfi_offset %ebp, -8
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    .cfi_def_cfa_register %ebp
+; SSE-32-NEXT:    andl $-8, %esp
+; SSE-32-NEXT:    subl $24, %esp
+; SSE-32-NEXT:    movss %xmm0, {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE-32-NEXT:    movss %xmm0, {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    flds {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fnstcw {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; SSE-32-NEXT:    orl $3072, %eax # imm = 0xC00
+; SSE-32-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fldcw {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fistpll {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fldcw {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    flds {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fnstcw (%esp)
+; SSE-32-NEXT:    movzwl (%esp), %eax
+; SSE-32-NEXT:    orl $3072, %eax # imm = 0xC00
+; SSE-32-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fldcw {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fistpll {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fldcw (%esp)
+; SSE-32-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE-32-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
+; SSE-32-NEXT:    movlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    .cfi_def_cfa %esp, 4
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: strict_vector_fptosi_v2f32_to_v2i1:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    cvttss2si %xmm0, %rax
+; SSE-64-NEXT:    movq %rax, %xmm1
+; SSE-64-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE-64-NEXT:    cvttss2si %xmm0, %rax
+; SSE-64-NEXT:    movq %rax, %xmm0
+; SSE-64-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSE-64-NEXT:    movdqa %xmm1, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: strict_vector_fptosi_v2f32_to_v2i1:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    .cfi_def_cfa_offset 8
+; AVX-32-NEXT:    .cfi_offset %ebp, -8
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    .cfi_def_cfa_register %ebp
+; AVX-32-NEXT:    andl $-8, %esp
+; AVX-32-NEXT:    subl $16, %esp
+; AVX-32-NEXT:    vmovss %xmm0, (%esp)
+; AVX-32-NEXT:    vextractps $1, %xmm0, {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    flds (%esp)
+; AVX-32-NEXT:    fisttpll (%esp)
+; AVX-32-NEXT:    flds {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vpinsrd $1, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrd $2, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrd $3, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    .cfi_def_cfa %esp, 4
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: strict_vector_fptosi_v2f32_to_v2i1:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcvttss2si %xmm0, %rax
+; AVX-64-NEXT:    vmovq %rax, %xmm1
+; AVX-64-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; AVX-64-NEXT:    vcvttss2si %xmm0, %rax
+; AVX-64-NEXT:    vmovq %rax, %xmm0
+; AVX-64-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX-64-NEXT:    retq
+;
+; AVX512F-LABEL: strict_vector_fptosi_v2f32_to_v2i1:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vcvttss2si %xmm0, %eax
+; AVX512F-NEXT:    andl $1, %eax
+; AVX512F-NEXT:    kmovw %eax, %k0
+; AVX512F-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; AVX512F-NEXT:    vcvttss2si %xmm0, %eax
+; AVX512F-NEXT:    kmovw %eax, %k1
+; AVX512F-NEXT:    kshiftlw $1, %k1, %k1
+; AVX512F-NEXT:    korw %k1, %k0, %k1
+; AVX512F-NEXT:    vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    ret{{[l|q]}}
+;
+; AVX512VL-LABEL: strict_vector_fptosi_v2f32_to_v2i1:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vcvttss2si %xmm0, %eax
+; AVX512VL-NEXT:    andl $1, %eax
+; AVX512VL-NEXT:    kmovw %eax, %k0
+; AVX512VL-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; AVX512VL-NEXT:    vcvttss2si %xmm0, %eax
+; AVX512VL-NEXT:    kmovw %eax, %k1
+; AVX512VL-NEXT:    kshiftlw $1, %k1, %k1
+; AVX512VL-NEXT:    korw %k1, %k0, %k1
+; AVX512VL-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX512VL-NEXT:    vmovdqa64 %xmm0, %xmm0 {%k1} {z}
+; AVX512VL-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQ-LABEL: strict_vector_fptosi_v2f32_to_v2i1:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX512DQ-NEXT:    vcvttss2si %xmm1, %eax
+; AVX512DQ-NEXT:    kmovw %eax, %k0
+; AVX512DQ-NEXT:    kshiftlb $1, %k0, %k0
+; AVX512DQ-NEXT:    vcvttss2si %xmm0, %eax
+; AVX512DQ-NEXT:    kmovw %eax, %k1
+; AVX512DQ-NEXT:    kshiftlb $7, %k1, %k1
+; AVX512DQ-NEXT:    kshiftrb $7, %k1, %k1
+; AVX512DQ-NEXT:    korw %k0, %k1, %k0
+; AVX512DQ-NEXT:    vpmovm2q %k0, %zmm0
+; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+;
+; AVX512VLDQ-LABEL: strict_vector_fptosi_v2f32_to_v2i1:
+; AVX512VLDQ:       # %bb.0:
+; AVX512VLDQ-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX512VLDQ-NEXT:    vcvttss2si %xmm1, %eax
+; AVX512VLDQ-NEXT:    kmovw %eax, %k0
+; AVX512VLDQ-NEXT:    kshiftlb $1, %k0, %k0
+; AVX512VLDQ-NEXT:    vcvttss2si %xmm0, %eax
+; AVX512VLDQ-NEXT:    kmovw %eax, %k1
+; AVX512VLDQ-NEXT:    kshiftlb $7, %k1, %k1
+; AVX512VLDQ-NEXT:    kshiftrb $7, %k1, %k1
+; AVX512VLDQ-NEXT:    korw %k0, %k1, %k0
+; AVX512VLDQ-NEXT:    vpmovm2q %k0, %xmm0
+; AVX512VLDQ-NEXT:    ret{{[l|q]}}
+  %ret = call <2 x i1> @llvm.experimental.constrained.fptosi.v2i1.v2f32(<2 x float> %a,
+                                              metadata !"fpexcept.strict")
+  ret <2 x i1> %ret
+}
+
+define <2 x i1> @strict_vector_fptoui_v2f32_to_v2i1(<2 x float> %a) #0 {
+; SSE-32-LABEL: strict_vector_fptoui_v2f32_to_v2i1:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    .cfi_def_cfa_offset 8
+; SSE-32-NEXT:    .cfi_offset %ebp, -8
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    .cfi_def_cfa_register %ebp
+; SSE-32-NEXT:    andl $-8, %esp
+; SSE-32-NEXT:    subl $24, %esp
+; SSE-32-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE-32-NEXT:    comiss %xmm2, %xmm0
+; SSE-32-NEXT:    xorps %xmm1, %xmm1
+; SSE-32-NEXT:    xorps %xmm3, %xmm3
+; SSE-32-NEXT:    jb .LBB19_2
+; SSE-32-NEXT:  # %bb.1:
+; SSE-32-NEXT:    movaps %xmm2, %xmm3
+; SSE-32-NEXT:  .LBB19_2:
+; SSE-32-NEXT:    movaps %xmm0, %xmm4
+; SSE-32-NEXT:    subss %xmm3, %xmm4
+; SSE-32-NEXT:    movss %xmm4, {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    setae %al
+; SSE-32-NEXT:    flds {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fnstcw {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
+; SSE-32-NEXT:    orl $3072, %ecx # imm = 0xC00
+; SSE-32-NEXT:    movw %cx, {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fldcw {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fistpll {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fldcw {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE-32-NEXT:    comiss %xmm2, %xmm0
+; SSE-32-NEXT:    jb .LBB19_4
+; SSE-32-NEXT:  # %bb.3:
+; SSE-32-NEXT:    movaps %xmm2, %xmm1
+; SSE-32-NEXT:  .LBB19_4:
+; SSE-32-NEXT:    subss %xmm1, %xmm0
+; SSE-32-NEXT:    movss %xmm0, {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    setae %cl
+; SSE-32-NEXT:    flds {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fnstcw (%esp)
+; SSE-32-NEXT:    movzwl (%esp), %edx
+; SSE-32-NEXT:    orl $3072, %edx # imm = 0xC00
+; SSE-32-NEXT:    movw %dx, {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fldcw {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fistpll {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fldcw (%esp)
+; SSE-32-NEXT:    movzbl %al, %eax
+; SSE-32-NEXT:    shll $31, %eax
+; SSE-32-NEXT:    xorl {{[0-9]+}}(%esp), %eax
+; SSE-32-NEXT:    movd %eax, %xmm1
+; SSE-32-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE-32-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE-32-NEXT:    movzbl %cl, %eax
+; SSE-32-NEXT:    shll $31, %eax
+; SSE-32-NEXT:    xorl {{[0-9]+}}(%esp), %eax
+; SSE-32-NEXT:    movd %eax, %xmm1
+; SSE-32-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE-32-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSE-32-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    .cfi_def_cfa %esp, 4
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: strict_vector_fptoui_v2f32_to_v2i1:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movss {{.*#+}} xmm3 = mem[0],zero,zero,zero
+; SSE-64-NEXT:    comiss %xmm3, %xmm0
+; SSE-64-NEXT:    xorps %xmm2, %xmm2
+; SSE-64-NEXT:    xorps %xmm1, %xmm1
+; SSE-64-NEXT:    jb .LBB19_2
+; SSE-64-NEXT:  # %bb.1:
+; SSE-64-NEXT:    movaps %xmm3, %xmm1
+; SSE-64-NEXT:  .LBB19_2:
+; SSE-64-NEXT:    movaps %xmm0, %xmm4
+; SSE-64-NEXT:    subss %xmm1, %xmm4
+; SSE-64-NEXT:    cvttss2si %xmm4, %rax
+; SSE-64-NEXT:    setae %cl
+; SSE-64-NEXT:    movzbl %cl, %ecx
+; SSE-64-NEXT:    shlq $63, %rcx
+; SSE-64-NEXT:    xorq %rax, %rcx
+; SSE-64-NEXT:    movq %rcx, %xmm1
+; SSE-64-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE-64-NEXT:    comiss %xmm3, %xmm0
+; SSE-64-NEXT:    jb .LBB19_4
+; SSE-64-NEXT:  # %bb.3:
+; SSE-64-NEXT:    movaps %xmm3, %xmm2
+; SSE-64-NEXT:  .LBB19_4:
+; SSE-64-NEXT:    subss %xmm2, %xmm0
+; SSE-64-NEXT:    cvttss2si %xmm0, %rax
+; SSE-64-NEXT:    setae %cl
+; SSE-64-NEXT:    movzbl %cl, %ecx
+; SSE-64-NEXT:    shlq $63, %rcx
+; SSE-64-NEXT:    xorq %rax, %rcx
+; SSE-64-NEXT:    movq %rcx, %xmm0
+; SSE-64-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSE-64-NEXT:    movdqa %xmm1, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: strict_vector_fptoui_v2f32_to_v2i1:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    .cfi_def_cfa_offset 8
+; AVX-32-NEXT:    .cfi_offset %ebp, -8
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    .cfi_def_cfa_register %ebp
+; AVX-32-NEXT:    andl $-8, %esp
+; AVX-32-NEXT:    subl $16, %esp
+; AVX-32-NEXT:    vmovshdup {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; AVX-32-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vcomiss %xmm1, %xmm3
+; AVX-32-NEXT:    vxorps %xmm2, %xmm2, %xmm2
+; AVX-32-NEXT:    vxorps %xmm4, %xmm4, %xmm4
+; AVX-32-NEXT:    jb .LBB19_2
+; AVX-32-NEXT:  # %bb.1:
+; AVX-32-NEXT:    vmovaps %xmm1, %xmm4
+; AVX-32-NEXT:  .LBB19_2:
+; AVX-32-NEXT:    vsubss %xmm4, %xmm3, %xmm3
+; AVX-32-NEXT:    vmovss %xmm3, {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    flds {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    setae %al
+; AVX-32-NEXT:    movzbl %al, %eax
+; AVX-32-NEXT:    shll $31, %eax
+; AVX-32-NEXT:    xorl {{[0-9]+}}(%esp), %eax
+; AVX-32-NEXT:    vcomiss %xmm1, %xmm0
+; AVX-32-NEXT:    jb .LBB19_4
+; AVX-32-NEXT:  # %bb.3:
+; AVX-32-NEXT:    vmovaps %xmm1, %xmm2
+; AVX-32-NEXT:  .LBB19_4:
+; AVX-32-NEXT:    vsubss %xmm2, %xmm0, %xmm0
+; AVX-32-NEXT:    vmovss %xmm0, (%esp)
+; AVX-32-NEXT:    flds (%esp)
+; AVX-32-NEXT:    fisttpll (%esp)
+; AVX-32-NEXT:    setae %cl
+; AVX-32-NEXT:    movzbl %cl, %ecx
+; AVX-32-NEXT:    shll $31, %ecx
+; AVX-32-NEXT:    xorl {{[0-9]+}}(%esp), %ecx
+; AVX-32-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vpinsrd $1, %ecx, %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrd $2, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrd $3, %eax, %xmm0, %xmm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    .cfi_def_cfa %esp, 4
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: strict_vector_fptoui_v2f32_to_v2i1:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX-64-NEXT:    vcomiss %xmm1, %xmm0
+; AVX-64-NEXT:    vxorps %xmm2, %xmm2, %xmm2
+; AVX-64-NEXT:    vxorps %xmm3, %xmm3, %xmm3
+; AVX-64-NEXT:    jb .LBB19_2
+; AVX-64-NEXT:  # %bb.1:
+; AVX-64-NEXT:    vmovaps %xmm1, %xmm3
+; AVX-64-NEXT:  .LBB19_2:
+; AVX-64-NEXT:    vsubss %xmm3, %xmm0, %xmm3
+; AVX-64-NEXT:    vcvttss2si %xmm3, %rax
+; AVX-64-NEXT:    setae %cl
+; AVX-64-NEXT:    movzbl %cl, %ecx
+; AVX-64-NEXT:    shlq $63, %rcx
+; AVX-64-NEXT:    xorq %rax, %rcx
+; AVX-64-NEXT:    vmovq %rcx, %xmm3
+; AVX-64-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; AVX-64-NEXT:    vcomiss %xmm1, %xmm0
+; AVX-64-NEXT:    jb .LBB19_4
+; AVX-64-NEXT:  # %bb.3:
+; AVX-64-NEXT:    vmovaps %xmm1, %xmm2
+; AVX-64-NEXT:  .LBB19_4:
+; AVX-64-NEXT:    vsubss %xmm2, %xmm0, %xmm0
+; AVX-64-NEXT:    vcvttss2si %xmm0, %rax
+; AVX-64-NEXT:    setae %cl
+; AVX-64-NEXT:    movzbl %cl, %ecx
+; AVX-64-NEXT:    shlq $63, %rcx
+; AVX-64-NEXT:    xorq %rax, %rcx
+; AVX-64-NEXT:    vmovq %rcx, %xmm0
+; AVX-64-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm3[0],xmm0[0]
+; AVX-64-NEXT:    retq
+;
+; AVX512F-LABEL: strict_vector_fptoui_v2f32_to_v2i1:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vcvttss2si %xmm0, %eax
+; AVX512F-NEXT:    andl $1, %eax
+; AVX512F-NEXT:    kmovw %eax, %k0
+; AVX512F-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; AVX512F-NEXT:    vcvttss2si %xmm0, %eax
+; AVX512F-NEXT:    kmovw %eax, %k1
+; AVX512F-NEXT:    kshiftlw $1, %k1, %k1
+; AVX512F-NEXT:    korw %k1, %k0, %k1
+; AVX512F-NEXT:    vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    ret{{[l|q]}}
+;
+; AVX512VL-LABEL: strict_vector_fptoui_v2f32_to_v2i1:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vcvttss2si %xmm0, %eax
+; AVX512VL-NEXT:    andl $1, %eax
+; AVX512VL-NEXT:    kmovw %eax, %k0
+; AVX512VL-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; AVX512VL-NEXT:    vcvttss2si %xmm0, %eax
+; AVX512VL-NEXT:    kmovw %eax, %k1
+; AVX512VL-NEXT:    kshiftlw $1, %k1, %k1
+; AVX512VL-NEXT:    korw %k1, %k0, %k1
+; AVX512VL-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX512VL-NEXT:    vmovdqa64 %xmm0, %xmm0 {%k1} {z}
+; AVX512VL-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQ-LABEL: strict_vector_fptoui_v2f32_to_v2i1:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX512DQ-NEXT:    vcvttss2si %xmm1, %eax
+; AVX512DQ-NEXT:    kmovw %eax, %k0
+; AVX512DQ-NEXT:    kshiftlb $1, %k0, %k0
+; AVX512DQ-NEXT:    vcvttss2si %xmm0, %eax
+; AVX512DQ-NEXT:    kmovw %eax, %k1
+; AVX512DQ-NEXT:    kshiftlb $7, %k1, %k1
+; AVX512DQ-NEXT:    kshiftrb $7, %k1, %k1
+; AVX512DQ-NEXT:    korw %k0, %k1, %k0
+; AVX512DQ-NEXT:    vpmovm2q %k0, %zmm0
+; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+;
+; AVX512VLDQ-LABEL: strict_vector_fptoui_v2f32_to_v2i1:
+; AVX512VLDQ:       # %bb.0:
+; AVX512VLDQ-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX512VLDQ-NEXT:    vcvttss2si %xmm1, %eax
+; AVX512VLDQ-NEXT:    kmovw %eax, %k0
+; AVX512VLDQ-NEXT:    kshiftlb $1, %k0, %k0
+; AVX512VLDQ-NEXT:    vcvttss2si %xmm0, %eax
+; AVX512VLDQ-NEXT:    kmovw %eax, %k1
+; AVX512VLDQ-NEXT:    kshiftlb $7, %k1, %k1
+; AVX512VLDQ-NEXT:    kshiftrb $7, %k1, %k1
+; AVX512VLDQ-NEXT:    korw %k0, %k1, %k0
+; AVX512VLDQ-NEXT:    vpmovm2q %k0, %xmm0
+; AVX512VLDQ-NEXT:    ret{{[l|q]}}
+  %ret = call <2 x i1> @llvm.experimental.constrained.fptoui.v2i1.v2f32(<2 x float> %a,
+                                              metadata !"fpexcept.strict")
+  ret <2 x i1> %ret
+}
+
+define <4 x i32> @strict_vector_fptosi_v4f32_to_v4i32(<4 x float> %a) #0 {
+; SSE-32-LABEL: strict_vector_fptosi_v4f32_to_v4i32:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    cvttps2dq %xmm0, %xmm0
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: strict_vector_fptosi_v4f32_to_v4i32:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    cvttps2dq %xmm0, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-LABEL: strict_vector_fptosi_v4f32_to_v4i32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX-NEXT:    ret{{[l|q]}}
+;
+; AVX512F-LABEL: strict_vector_fptosi_v4f32_to_v4i32:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX512F-NEXT:    ret{{[l|q]}}
+;
+; AVX512VL-LABEL: strict_vector_fptosi_v4f32_to_v4i32:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX512VL-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQ-LABEL: strict_vector_fptosi_v4f32_to_v4i32:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+;
+; AVX512VLDQ-LABEL: strict_vector_fptosi_v4f32_to_v4i32:
+; AVX512VLDQ:       # %bb.0:
+; AVX512VLDQ-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX512VLDQ-NEXT:    ret{{[l|q]}}
+  %ret = call <4 x i32> @llvm.experimental.constrained.fptosi.v4i32.v4f32(<4 x float> %a,
+                                              metadata !"fpexcept.strict")
+  ret <4 x i32> %ret
+}
+
+define <4 x i32> @strict_vector_fptoui_v4f32_to_v4i32(<4 x float> %a) #0 {
+; SSE-32-LABEL: strict_vector_fptoui_v4f32_to_v4i32:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    movaps {{.*#+}} xmm2 = [2.14748365E+9,2.14748365E+9,2.14748365E+9,2.14748365E+9]
+; SSE-32-NEXT:    movaps %xmm0, %xmm3
+; SSE-32-NEXT:    cmpltps %xmm2, %xmm3
+; SSE-32-NEXT:    movaps %xmm3, %xmm1
+; SSE-32-NEXT:    andnps {{\.LCPI.*}}, %xmm1
+; SSE-32-NEXT:    andnps %xmm2, %xmm3
+; SSE-32-NEXT:    subps %xmm3, %xmm0
+; SSE-32-NEXT:    cvttps2dq %xmm0, %xmm0
+; SSE-32-NEXT:    xorps %xmm0, %xmm1
+; SSE-32-NEXT:    movaps %xmm1, %xmm0
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: strict_vector_fptoui_v4f32_to_v4i32:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movaps {{.*#+}} xmm2 = [2.14748365E+9,2.14748365E+9,2.14748365E+9,2.14748365E+9]
+; SSE-64-NEXT:    movaps %xmm0, %xmm3
+; SSE-64-NEXT:    cmpltps %xmm2, %xmm3
+; SSE-64-NEXT:    movaps %xmm3, %xmm1
+; SSE-64-NEXT:    andnps {{.*}}(%rip), %xmm1
+; SSE-64-NEXT:    andnps %xmm2, %xmm3
+; SSE-64-NEXT:    subps %xmm3, %xmm0
+; SSE-64-NEXT:    cvttps2dq %xmm0, %xmm0
+; SSE-64-NEXT:    xorps %xmm0, %xmm1
+; SSE-64-NEXT:    movaps %xmm1, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-LABEL: strict_vector_fptoui_v4f32_to_v4i32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovaps {{.*#+}} xmm1 = [2.14748365E+9,2.14748365E+9,2.14748365E+9,2.14748365E+9]
+; AVX-NEXT:    vcmpltps %xmm1, %xmm0, %xmm2
+; AVX-NEXT:    vxorps %xmm3, %xmm3, %xmm3
+; AVX-NEXT:    vmovaps {{.*#+}} xmm4 = [2147483648,2147483648,2147483648,2147483648]
+; AVX-NEXT:    vblendvps %xmm2, %xmm3, %xmm4, %xmm4
+; AVX-NEXT:    vblendvps %xmm2, %xmm3, %xmm1, %xmm1
+; AVX-NEXT:    vsubps %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX-NEXT:    vxorps %xmm4, %xmm0, %xmm0
+; AVX-NEXT:    ret{{[l|q]}}
+;
+; AVX512F-LABEL: strict_vector_fptoui_v4f32_to_v4i32:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vmovaps %xmm0, %xmm0
+; AVX512F-NEXT:    vcvttps2udq %zmm0, %zmm0
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    ret{{[l|q]}}
+;
+; AVX512VL-LABEL: strict_vector_fptoui_v4f32_to_v4i32:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vcvttps2udq %xmm0, %xmm0
+; AVX512VL-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQ-LABEL: strict_vector_fptoui_v4f32_to_v4i32:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vmovaps %xmm0, %xmm0
+; AVX512DQ-NEXT:    vcvttps2udq %zmm0, %zmm0
+; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+;
+; AVX512VLDQ-LABEL: strict_vector_fptoui_v4f32_to_v4i32:
+; AVX512VLDQ:       # %bb.0:
+; AVX512VLDQ-NEXT:    vcvttps2udq %xmm0, %xmm0
+; AVX512VLDQ-NEXT:    ret{{[l|q]}}
+  %ret = call <4 x i32> @llvm.experimental.constrained.fptoui.v4i32.v4f32(<4 x float> %a,
+                                              metadata !"fpexcept.strict")
+  ret <4 x i32> %ret
+}
+
+define <4 x i8> @strict_vector_fptosi_v4f32_to_v4i8(<4 x float> %a) #0 {
+; SSE-32-LABEL: strict_vector_fptosi_v4f32_to_v4i8:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    cvttps2dq %xmm0, %xmm0
+; SSE-32-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; SSE-32-NEXT:    packuswb %xmm0, %xmm0
+; SSE-32-NEXT:    packuswb %xmm0, %xmm0
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: strict_vector_fptosi_v4f32_to_v4i8:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    cvttps2dq %xmm0, %xmm0
+; SSE-64-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE-64-NEXT:    packuswb %xmm0, %xmm0
+; SSE-64-NEXT:    packuswb %xmm0, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-LABEL: strict_vector_fptosi_v4f32_to_v4i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX-NEXT:    ret{{[l|q]}}
+;
+; AVX512F-LABEL: strict_vector_fptosi_v4f32_to_v4i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX512F-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512F-NEXT:    ret{{[l|q]}}
+;
+; AVX512VL-LABEL: strict_vector_fptosi_v4f32_to_v4i8:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX512VL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512VL-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQ-LABEL: strict_vector_fptosi_v4f32_to_v4i8:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+;
+; AVX512VLDQ-LABEL: strict_vector_fptosi_v4f32_to_v4i8:
+; AVX512VLDQ:       # %bb.0:
+; AVX512VLDQ-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX512VLDQ-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512VLDQ-NEXT:    ret{{[l|q]}}
+  %ret = call <4 x i8> @llvm.experimental.constrained.fptosi.v4i8.v4f32(<4 x float> %a,
+                                              metadata !"fpexcept.strict")
+  ret <4 x i8> %ret
+}
+
+define <4 x i8> @strict_vector_fptoui_v4f32_to_v4i8(<4 x float> %a) #0 {
+; SSE-32-LABEL: strict_vector_fptoui_v4f32_to_v4i8:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    cvttps2dq %xmm0, %xmm0
+; SSE-32-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; SSE-32-NEXT:    packuswb %xmm0, %xmm0
+; SSE-32-NEXT:    packuswb %xmm0, %xmm0
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: strict_vector_fptoui_v4f32_to_v4i8:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    cvttps2dq %xmm0, %xmm0
+; SSE-64-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE-64-NEXT:    packuswb %xmm0, %xmm0
+; SSE-64-NEXT:    packuswb %xmm0, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-LABEL: strict_vector_fptoui_v4f32_to_v4i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX-NEXT:    ret{{[l|q]}}
+;
+; AVX512F-LABEL: strict_vector_fptoui_v4f32_to_v4i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX512F-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512F-NEXT:    ret{{[l|q]}}
+;
+; AVX512VL-LABEL: strict_vector_fptoui_v4f32_to_v4i8:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX512VL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512VL-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQ-LABEL: strict_vector_fptoui_v4f32_to_v4i8:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+;
+; AVX512VLDQ-LABEL: strict_vector_fptoui_v4f32_to_v4i8:
+; AVX512VLDQ:       # %bb.0:
+; AVX512VLDQ-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX512VLDQ-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512VLDQ-NEXT:    ret{{[l|q]}}
+  %ret = call <4 x i8> @llvm.experimental.constrained.fptoui.v4i8.v4f32(<4 x float> %a,
+                                              metadata !"fpexcept.strict")
+  ret <4 x i8> %ret
+}
+
+define <4 x i1> @strict_vector_fptosi_v4f32_to_v4i1(<4 x float> %a) #0 {
+; SSE-32-LABEL: strict_vector_fptosi_v4f32_to_v4i1:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    cvttps2dq %xmm0, %xmm0
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: strict_vector_fptosi_v4f32_to_v4i1:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    cvttps2dq %xmm0, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-LABEL: strict_vector_fptosi_v4f32_to_v4i1:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX-NEXT:    ret{{[l|q]}}
+;
+; AVX512F-LABEL: strict_vector_fptosi_v4f32_to_v4i1:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k1
+; AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    ret{{[l|q]}}
+;
+; AVX512VL-LABEL: strict_vector_fptosi_v4f32_to_v4i1:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX512VL-NEXT:    vptestmd %xmm0, %xmm0, %k1
+; AVX512VL-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX512VL-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
+; AVX512VL-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQ-LABEL: strict_vector_fptosi_v4f32_to_v4i1:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX512DQ-NEXT:    vpmovd2m %zmm0, %k0
+; AVX512DQ-NEXT:    vpmovm2d %k0, %zmm0
+; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+;
+; AVX512VLDQ-LABEL: strict_vector_fptosi_v4f32_to_v4i1:
+; AVX512VLDQ:       # %bb.0:
+; AVX512VLDQ-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX512VLDQ-NEXT:    vpmovd2m %xmm0, %k0
+; AVX512VLDQ-NEXT:    vpmovm2d %k0, %xmm0
+; AVX512VLDQ-NEXT:    ret{{[l|q]}}
+  %ret = call <4 x i1> @llvm.experimental.constrained.fptosi.v4i1.v4f32(<4 x float> %a,
+                                              metadata !"fpexcept.strict")
+  ret <4 x i1> %ret
+}
+
+define <4 x i1> @strict_vector_fptoui_v4f32_to_v4i1(<4 x float> %a) #0 {
+; SSE-32-LABEL: strict_vector_fptoui_v4f32_to_v4i1:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    cvttps2dq %xmm0, %xmm0
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: strict_vector_fptoui_v4f32_to_v4i1:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    cvttps2dq %xmm0, %xmm0
+; SSE-64-NEXT:    retq
+;
+; AVX-LABEL: strict_vector_fptoui_v4f32_to_v4i1:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX-NEXT:    ret{{[l|q]}}
+;
+; AVX512F-LABEL: strict_vector_fptoui_v4f32_to_v4i1:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX512F-NEXT:    vpslld $31, %xmm0, %xmm0
+; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k1
+; AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    ret{{[l|q]}}
+;
+; AVX512VL-LABEL: strict_vector_fptoui_v4f32_to_v4i1:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX512VL-NEXT:    vpslld $31, %xmm0, %xmm0
+; AVX512VL-NEXT:    vptestmd %xmm0, %xmm0, %k1
+; AVX512VL-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX512VL-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
+; AVX512VL-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQ-LABEL: strict_vector_fptoui_v4f32_to_v4i1:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX512DQ-NEXT:    vpslld $31, %xmm0, %xmm0
+; AVX512DQ-NEXT:    vpmovd2m %zmm0, %k0
+; AVX512DQ-NEXT:    vpmovm2d %k0, %zmm0
+; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+;
+; AVX512VLDQ-LABEL: strict_vector_fptoui_v4f32_to_v4i1:
+; AVX512VLDQ:       # %bb.0:
+; AVX512VLDQ-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX512VLDQ-NEXT:    vpslld $31, %xmm0, %xmm0
+; AVX512VLDQ-NEXT:    vpmovd2m %xmm0, %k0
+; AVX512VLDQ-NEXT:    vpmovm2d %k0, %xmm0
+; AVX512VLDQ-NEXT:    ret{{[l|q]}}
+  %ret = call <4 x i1> @llvm.experimental.constrained.fptoui.v4i1.v4f32(<4 x float> %a,
+                                              metadata !"fpexcept.strict")
+  ret <4 x i1> %ret
+}
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/X86/vec-strict-fptoint-256.ll b/llvm/test/CodeGen/X86/vec-strict-fptoint-256.ll
new file mode 100644
index 000000000..6b7f083a2
--- /dev/null
+++ b/llvm/test/CodeGen/X86/vec-strict-fptoint-256.ll
@@ -0,0 +1,1761 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx -O3 | FileCheck %s --check-prefixes=CHECK,AVX-32
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx -O3 | FileCheck %s --check-prefixes=CHECK,AVX-64
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx512f -O3 | FileCheck %s --check-prefixes=CHECK,AVX512F,AVX512F-32
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f -O3 | FileCheck %s --check-prefixes=CHECK,AVX512F,AVX512F-64
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx512vl -O3 | FileCheck %s --check-prefixes=CHECK,AVX512VL,AVX512VL-32
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vl -O3 | FileCheck %s --check-prefixes=CHECK,AVX512VL,AVX512VL-64
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=avx512dq -O3 | FileCheck %s --check-prefixes=CHECK,AVX512DQ,AVX512DQ-32
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512dq -O3 | FileCheck %s --check-prefixes=CHECK,AVX512DQ,AVX512DQ-64
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=avx512dq,avx512vl -O3 | FileCheck %s --check-prefixes=CHECK,AVX512DQVL,AVX512DQVL-32
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512dq,avx512vl -O3 | FileCheck %s --check-prefixes=CHECK,AVX512DQVL,AVX512DQVL-64
+
+
+declare <4 x i64> @llvm.experimental.constrained.fptosi.v4i64.v4f64(<4 x double>, metadata)
+declare <4 x i64> @llvm.experimental.constrained.fptoui.v4i64.v4f64(<4 x double>, metadata)
+declare <4 x i64> @llvm.experimental.constrained.fptosi.v4i64.v4f32(<4 x float>, metadata)
+declare <4 x i64> @llvm.experimental.constrained.fptoui.v4i64.v4f32(<4 x float>, metadata)
+declare <4 x i32> @llvm.experimental.constrained.fptosi.v4i32.v4f64(<4 x double>, metadata)
+declare <4 x i32> @llvm.experimental.constrained.fptoui.v4i32.v4f64(<4 x double>, metadata)
+declare <4 x i16> @llvm.experimental.constrained.fptosi.v4i16.v4f64(<4 x double>, metadata)
+declare <4 x i16> @llvm.experimental.constrained.fptoui.v4i16.v4f64(<4 x double>, metadata)
+declare <4 x i8> @llvm.experimental.constrained.fptosi.v4i8.v4f64(<4 x double>, metadata)
+declare <4 x i8> @llvm.experimental.constrained.fptoui.v4i8.v4f64(<4 x double>, metadata)
+declare <4 x i8> @llvm.experimental.constrained.fptosi.v4i8.v4f32(<4 x float>, metadata)
+declare <4 x i8> @llvm.experimental.constrained.fptoui.v4i8.v4f32(<4 x float>, metadata)
+declare <4 x i1> @llvm.experimental.constrained.fptosi.v4i1.v4f64(<4 x double>, metadata)
+declare <4 x i1> @llvm.experimental.constrained.fptoui.v4i1.v4f64(<4 x double>, metadata)
+declare <8 x i32> @llvm.experimental.constrained.fptosi.v8i32.v8f32(<8 x float>, metadata)
+declare <8 x i32> @llvm.experimental.constrained.fptoui.v8i32.v8f32(<8 x float>, metadata)
+declare <8 x i16> @llvm.experimental.constrained.fptosi.v8i16.v8f32(<8 x float>, metadata)
+declare <8 x i16> @llvm.experimental.constrained.fptoui.v8i16.v8f32(<8 x float>, metadata)
+declare <8 x i8> @llvm.experimental.constrained.fptosi.v8i8.v8f32(<8 x float>, metadata)
+declare <8 x i8> @llvm.experimental.constrained.fptoui.v8i8.v8f32(<8 x float>, metadata)
+declare <8 x i1> @llvm.experimental.constrained.fptosi.v8i1.v8f32(<8 x float>, metadata)
+declare <8 x i1> @llvm.experimental.constrained.fptoui.v8i1.v8f32(<8 x float>, metadata)
+
+define <4 x i64> @strict_vector_fptosi_v4f64_to_v4i64(<4 x double> %a) #0 {
+; AVX-32-LABEL: strict_vector_fptosi_v4f64_to_v4i64:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    .cfi_def_cfa_offset 8
+; AVX-32-NEXT:    .cfi_offset %ebp, -8
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    .cfi_def_cfa_register %ebp
+; AVX-32-NEXT:    andl $-8, %esp
+; AVX-32-NEXT:    subl $32, %esp
+; AVX-32-NEXT:    vmovlps %xmm0, {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    vmovhps %xmm0, {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX-32-NEXT:    vmovlps %xmm0, {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    vmovhps %xmm0, (%esp)
+; AVX-32-NEXT:    fldl {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    fldl {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    fldl {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    fldl (%esp)
+; AVX-32-NEXT:    fisttpll (%esp)
+; AVX-32-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vpinsrd $1, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrd $2, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrd $3, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vpinsrd $1, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX-32-NEXT:    vpinsrd $2, (%esp), %xmm1, %xmm1
+; AVX-32-NEXT:    vpinsrd $3, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX-32-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    .cfi_def_cfa %esp, 4
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: strict_vector_fptosi_v4f64_to_v4i64:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX-64-NEXT:    vcvttsd2si %xmm1, %rax
+; AVX-64-NEXT:    vmovq %rax, %xmm2
+; AVX-64-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm1[1,0]
+; AVX-64-NEXT:    vcvttsd2si %xmm1, %rax
+; AVX-64-NEXT:    vmovq %rax, %xmm1
+; AVX-64-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX-64-NEXT:    vcvttsd2si %xmm0, %rax
+; AVX-64-NEXT:    vmovq %rax, %xmm2
+; AVX-64-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
+; AVX-64-NEXT:    vcvttsd2si %xmm0, %rax
+; AVX-64-NEXT:    vmovq %rax, %xmm0
+; AVX-64-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm2[0],xmm0[0]
+; AVX-64-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX-64-NEXT:    retq
+;
+; AVX512F-32-LABEL: strict_vector_fptosi_v4f64_to_v4i64:
+; AVX512F-32:       # %bb.0:
+; AVX512F-32-NEXT:    pushl %ebp
+; AVX512F-32-NEXT:    .cfi_def_cfa_offset 8
+; AVX512F-32-NEXT:    .cfi_offset %ebp, -8
+; AVX512F-32-NEXT:    movl %esp, %ebp
+; AVX512F-32-NEXT:    .cfi_def_cfa_register %ebp
+; AVX512F-32-NEXT:    andl $-8, %esp
+; AVX512F-32-NEXT:    subl $32, %esp
+; AVX512F-32-NEXT:    vmovlps %xmm0, {{[0-9]+}}(%esp)
+; AVX512F-32-NEXT:    vmovhps %xmm0, {{[0-9]+}}(%esp)
+; AVX512F-32-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX512F-32-NEXT:    vmovlps %xmm0, {{[0-9]+}}(%esp)
+; AVX512F-32-NEXT:    vmovhps %xmm0, (%esp)
+; AVX512F-32-NEXT:    fldl {{[0-9]+}}(%esp)
+; AVX512F-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX512F-32-NEXT:    fldl {{[0-9]+}}(%esp)
+; AVX512F-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX512F-32-NEXT:    fldl {{[0-9]+}}(%esp)
+; AVX512F-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX512F-32-NEXT:    fldl (%esp)
+; AVX512F-32-NEXT:    fisttpll (%esp)
+; AVX512F-32-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX512F-32-NEXT:    vpinsrd $1, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512F-32-NEXT:    vpinsrd $2, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512F-32-NEXT:    vpinsrd $3, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512F-32-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX512F-32-NEXT:    vpinsrd $1, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512F-32-NEXT:    vpinsrd $2, (%esp), %xmm1, %xmm1
+; AVX512F-32-NEXT:    vpinsrd $3, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512F-32-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512F-32-NEXT:    movl %ebp, %esp
+; AVX512F-32-NEXT:    popl %ebp
+; AVX512F-32-NEXT:    .cfi_def_cfa %esp, 4
+; AVX512F-32-NEXT:    retl
+;
+; AVX512F-64-LABEL: strict_vector_fptosi_v4f64_to_v4i64:
+; AVX512F-64:       # %bb.0:
+; AVX512F-64-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX512F-64-NEXT:    vcvttsd2si %xmm1, %rax
+; AVX512F-64-NEXT:    vmovq %rax, %xmm2
+; AVX512F-64-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm1[1,0]
+; AVX512F-64-NEXT:    vcvttsd2si %xmm1, %rax
+; AVX512F-64-NEXT:    vmovq %rax, %xmm1
+; AVX512F-64-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX512F-64-NEXT:    vcvttsd2si %xmm0, %rax
+; AVX512F-64-NEXT:    vmovq %rax, %xmm2
+; AVX512F-64-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
+; AVX512F-64-NEXT:    vcvttsd2si %xmm0, %rax
+; AVX512F-64-NEXT:    vmovq %rax, %xmm0
+; AVX512F-64-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm2[0],xmm0[0]
+; AVX512F-64-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512F-64-NEXT:    retq
+;
+; AVX512VL-32-LABEL: strict_vector_fptosi_v4f64_to_v4i64:
+; AVX512VL-32:       # %bb.0:
+; AVX512VL-32-NEXT:    pushl %ebp
+; AVX512VL-32-NEXT:    .cfi_def_cfa_offset 8
+; AVX512VL-32-NEXT:    .cfi_offset %ebp, -8
+; AVX512VL-32-NEXT:    movl %esp, %ebp
+; AVX512VL-32-NEXT:    .cfi_def_cfa_register %ebp
+; AVX512VL-32-NEXT:    andl $-8, %esp
+; AVX512VL-32-NEXT:    subl $32, %esp
+; AVX512VL-32-NEXT:    vmovlps %xmm0, {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    vmovhps %xmm0, {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX512VL-32-NEXT:    vmovlps %xmm0, {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    vmovhps %xmm0, (%esp)
+; AVX512VL-32-NEXT:    fldl {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    fldl {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    fldl {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    fldl (%esp)
+; AVX512VL-32-NEXT:    fisttpll (%esp)
+; AVX512VL-32-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX512VL-32-NEXT:    vpinsrd $1, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512VL-32-NEXT:    vpinsrd $2, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512VL-32-NEXT:    vpinsrd $3, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512VL-32-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX512VL-32-NEXT:    vpinsrd $1, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512VL-32-NEXT:    vpinsrd $2, (%esp), %xmm1, %xmm1
+; AVX512VL-32-NEXT:    vpinsrd $3, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512VL-32-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512VL-32-NEXT:    movl %ebp, %esp
+; AVX512VL-32-NEXT:    popl %ebp
+; AVX512VL-32-NEXT:    .cfi_def_cfa %esp, 4
+; AVX512VL-32-NEXT:    retl
+;
+; AVX512VL-64-LABEL: strict_vector_fptosi_v4f64_to_v4i64:
+; AVX512VL-64:       # %bb.0:
+; AVX512VL-64-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX512VL-64-NEXT:    vcvttsd2si %xmm1, %rax
+; AVX512VL-64-NEXT:    vmovq %rax, %xmm2
+; AVX512VL-64-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm1[1,0]
+; AVX512VL-64-NEXT:    vcvttsd2si %xmm1, %rax
+; AVX512VL-64-NEXT:    vmovq %rax, %xmm1
+; AVX512VL-64-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX512VL-64-NEXT:    vcvttsd2si %xmm0, %rax
+; AVX512VL-64-NEXT:    vmovq %rax, %xmm2
+; AVX512VL-64-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
+; AVX512VL-64-NEXT:    vcvttsd2si %xmm0, %rax
+; AVX512VL-64-NEXT:    vmovq %rax, %xmm0
+; AVX512VL-64-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm2[0],xmm0[0]
+; AVX512VL-64-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512VL-64-NEXT:    retq
+;
+; AVX512DQ-LABEL: strict_vector_fptosi_v4f64_to_v4i64:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vmovaps %ymm0, %ymm0
+; AVX512DQ-NEXT:    vcvttpd2qq %zmm0, %zmm0
+; AVX512DQ-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQVL-LABEL: strict_vector_fptosi_v4f64_to_v4i64:
+; AVX512DQVL:       # %bb.0:
+; AVX512DQVL-NEXT:    vcvttpd2qq %ymm0, %ymm0
+; AVX512DQVL-NEXT:    ret{{[l|q]}}
+  %ret = call <4 x i64> @llvm.experimental.constrained.fptosi.v4i64.v4f64(<4 x double> %a,
+                                              metadata !"fpexcept.strict")
+  ret <4 x i64> %ret
+}
+
+define <4 x i64> @strict_vector_fptoui_v4f64_to_v4i64(<4 x double> %a) #0 {
+; AVX-32-LABEL: strict_vector_fptoui_v4f64_to_v4i64:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    .cfi_def_cfa_offset 8
+; AVX-32-NEXT:    .cfi_offset %ebp, -8
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    .cfi_def_cfa_register %ebp
+; AVX-32-NEXT:    andl $-8, %esp
+; AVX-32-NEXT:    subl $32, %esp
+; AVX-32-NEXT:    vpermilpd {{.*#+}} xmm3 = xmm0[1,0]
+; AVX-32-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
+; AVX-32-NEXT:    vcomisd %xmm1, %xmm3
+; AVX-32-NEXT:    vxorpd %xmm2, %xmm2, %xmm2
+; AVX-32-NEXT:    vxorpd %xmm4, %xmm4, %xmm4
+; AVX-32-NEXT:    jb .LBB1_2
+; AVX-32-NEXT:  # %bb.1:
+; AVX-32-NEXT:    vmovapd %xmm1, %xmm4
+; AVX-32-NEXT:  .LBB1_2:
+; AVX-32-NEXT:    vsubsd %xmm4, %xmm3, %xmm3
+; AVX-32-NEXT:    vmovsd %xmm3, {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    fldl {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    setae %al
+; AVX-32-NEXT:    movzbl %al, %eax
+; AVX-32-NEXT:    shll $31, %eax
+; AVX-32-NEXT:    xorl {{[0-9]+}}(%esp), %eax
+; AVX-32-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; AVX-32-NEXT:    vpermilpd {{.*#+}} xmm4 = xmm3[1,0]
+; AVX-32-NEXT:    vcomisd %xmm1, %xmm4
+; AVX-32-NEXT:    vxorpd %xmm5, %xmm5, %xmm5
+; AVX-32-NEXT:    jb .LBB1_4
+; AVX-32-NEXT:  # %bb.3:
+; AVX-32-NEXT:    vmovapd %xmm1, %xmm5
+; AVX-32-NEXT:  .LBB1_4:
+; AVX-32-NEXT:    vsubsd %xmm5, %xmm4, %xmm4
+; AVX-32-NEXT:    vmovsd %xmm4, (%esp)
+; AVX-32-NEXT:    fldl (%esp)
+; AVX-32-NEXT:    fisttpll (%esp)
+; AVX-32-NEXT:    setae %cl
+; AVX-32-NEXT:    movzbl %cl, %ecx
+; AVX-32-NEXT:    shll $31, %ecx
+; AVX-32-NEXT:    xorl {{[0-9]+}}(%esp), %ecx
+; AVX-32-NEXT:    vcomisd %xmm1, %xmm3
+; AVX-32-NEXT:    vxorpd %xmm4, %xmm4, %xmm4
+; AVX-32-NEXT:    jb .LBB1_6
+; AVX-32-NEXT:  # %bb.5:
+; AVX-32-NEXT:    vmovapd %xmm1, %xmm4
+; AVX-32-NEXT:  .LBB1_6:
+; AVX-32-NEXT:    vsubsd %xmm4, %xmm3, %xmm3
+; AVX-32-NEXT:    vmovsd %xmm3, {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    fldl {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    setae %dl
+; AVX-32-NEXT:    movzbl %dl, %edx
+; AVX-32-NEXT:    shll $31, %edx
+; AVX-32-NEXT:    xorl {{[0-9]+}}(%esp), %edx
+; AVX-32-NEXT:    vcomisd %xmm1, %xmm0
+; AVX-32-NEXT:    jb .LBB1_8
+; AVX-32-NEXT:  # %bb.7:
+; AVX-32-NEXT:    vmovapd %xmm1, %xmm2
+; AVX-32-NEXT:  .LBB1_8:
+; AVX-32-NEXT:    vsubsd %xmm2, %xmm0, %xmm0
+; AVX-32-NEXT:    vmovsd %xmm0, {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    fldl {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vpinsrd $1, %edx, %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrd $2, (%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrd $3, %ecx, %xmm0, %xmm0
+; AVX-32-NEXT:    setae %cl
+; AVX-32-NEXT:    movzbl %cl, %ecx
+; AVX-32-NEXT:    shll $31, %ecx
+; AVX-32-NEXT:    xorl {{[0-9]+}}(%esp), %ecx
+; AVX-32-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vpinsrd $1, %ecx, %xmm1, %xmm1
+; AVX-32-NEXT:    vpinsrd $2, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX-32-NEXT:    vpinsrd $3, %eax, %xmm1, %xmm1
+; AVX-32-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    .cfi_def_cfa %esp, 4
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: strict_vector_fptoui_v4f64_to_v4i64:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; AVX-64-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
+; AVX-64-NEXT:    vcomisd %xmm1, %xmm3
+; AVX-64-NEXT:    vxorpd %xmm2, %xmm2, %xmm2
+; AVX-64-NEXT:    vxorpd %xmm4, %xmm4, %xmm4
+; AVX-64-NEXT:    jb .LBB1_2
+; AVX-64-NEXT:  # %bb.1:
+; AVX-64-NEXT:    vmovapd %xmm1, %xmm4
+; AVX-64-NEXT:  .LBB1_2:
+; AVX-64-NEXT:    vsubsd %xmm4, %xmm3, %xmm4
+; AVX-64-NEXT:    vcvttsd2si %xmm4, %rcx
+; AVX-64-NEXT:    setae %al
+; AVX-64-NEXT:    movzbl %al, %eax
+; AVX-64-NEXT:    shlq $63, %rax
+; AVX-64-NEXT:    xorq %rcx, %rax
+; AVX-64-NEXT:    vpermilpd {{.*#+}} xmm4 = xmm3[1,0]
+; AVX-64-NEXT:    vcomisd %xmm1, %xmm4
+; AVX-64-NEXT:    vxorpd %xmm5, %xmm5, %xmm5
+; AVX-64-NEXT:    jb .LBB1_4
+; AVX-64-NEXT:  # %bb.3:
+; AVX-64-NEXT:    vmovapd %xmm1, %xmm5
+; AVX-64-NEXT:  .LBB1_4:
+; AVX-64-NEXT:    vmovq %rax, %xmm3
+; AVX-64-NEXT:    vsubsd %xmm5, %xmm4, %xmm4
+; AVX-64-NEXT:    vcvttsd2si %xmm4, %rax
+; AVX-64-NEXT:    setae %cl
+; AVX-64-NEXT:    movzbl %cl, %ecx
+; AVX-64-NEXT:    shlq $63, %rcx
+; AVX-64-NEXT:    xorq %rax, %rcx
+; AVX-64-NEXT:    vmovq %rcx, %xmm4
+; AVX-64-NEXT:    vcomisd %xmm1, %xmm0
+; AVX-64-NEXT:    vxorpd %xmm5, %xmm5, %xmm5
+; AVX-64-NEXT:    jb .LBB1_6
+; AVX-64-NEXT:  # %bb.5:
+; AVX-64-NEXT:    vmovapd %xmm1, %xmm5
+; AVX-64-NEXT:  .LBB1_6:
+; AVX-64-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm4[0]
+; AVX-64-NEXT:    vsubsd %xmm5, %xmm0, %xmm4
+; AVX-64-NEXT:    vcvttsd2si %xmm4, %rax
+; AVX-64-NEXT:    setae %cl
+; AVX-64-NEXT:    movzbl %cl, %ecx
+; AVX-64-NEXT:    shlq $63, %rcx
+; AVX-64-NEXT:    xorq %rax, %rcx
+; AVX-64-NEXT:    vmovq %rcx, %xmm4
+; AVX-64-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
+; AVX-64-NEXT:    vcomisd %xmm1, %xmm0
+; AVX-64-NEXT:    jb .LBB1_8
+; AVX-64-NEXT:  # %bb.7:
+; AVX-64-NEXT:    vmovapd %xmm1, %xmm2
+; AVX-64-NEXT:  .LBB1_8:
+; AVX-64-NEXT:    vsubsd %xmm2, %xmm0, %xmm0
+; AVX-64-NEXT:    vcvttsd2si %xmm0, %rax
+; AVX-64-NEXT:    setae %cl
+; AVX-64-NEXT:    movzbl %cl, %ecx
+; AVX-64-NEXT:    shlq $63, %rcx
+; AVX-64-NEXT:    xorq %rax, %rcx
+; AVX-64-NEXT:    vmovq %rcx, %xmm0
+; AVX-64-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm4[0],xmm0[0]
+; AVX-64-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
+; AVX-64-NEXT:    retq
+;
+; AVX512F-32-LABEL: strict_vector_fptoui_v4f64_to_v4i64:
+; AVX512F-32:       # %bb.0:
+; AVX512F-32-NEXT:    pushl %ebp
+; AVX512F-32-NEXT:    .cfi_def_cfa_offset 8
+; AVX512F-32-NEXT:    .cfi_offset %ebp, -8
+; AVX512F-32-NEXT:    movl %esp, %ebp
+; AVX512F-32-NEXT:    .cfi_def_cfa_register %ebp
+; AVX512F-32-NEXT:    pushl %ebx
+; AVX512F-32-NEXT:    pushl %esi
+; AVX512F-32-NEXT:    andl $-8, %esp
+; AVX512F-32-NEXT:    subl $32, %esp
+; AVX512F-32-NEXT:    .cfi_offset %esi, -16
+; AVX512F-32-NEXT:    .cfi_offset %ebx, -12
+; AVX512F-32-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
+; AVX512F-32-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
+; AVX512F-32-NEXT:    vcomisd %xmm1, %xmm2
+; AVX512F-32-NEXT:    setb %cl
+; AVX512F-32-NEXT:    kmovw %ecx, %k1
+; AVX512F-32-NEXT:    vxorpd %xmm3, %xmm3, %xmm3
+; AVX512F-32-NEXT:    vmovapd %xmm1, %xmm4
+; AVX512F-32-NEXT:    vmovsd %xmm3, %xmm4, %xmm4 {%k1}
+; AVX512F-32-NEXT:    vsubsd %xmm4, %xmm2, %xmm2
+; AVX512F-32-NEXT:    vmovsd %xmm2, {{[0-9]+}}(%esp)
+; AVX512F-32-NEXT:    fldl {{[0-9]+}}(%esp)
+; AVX512F-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX512F-32-NEXT:    movl $0, %eax
+; AVX512F-32-NEXT:    setae %al
+; AVX512F-32-NEXT:    shll $31, %eax
+; AVX512F-32-NEXT:    xorl {{[0-9]+}}(%esp), %eax
+; AVX512F-32-NEXT:    movl %eax, %esi
+; AVX512F-32-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX512F-32-NEXT:    vpermilpd {{.*#+}} xmm4 = xmm2[1,0]
+; AVX512F-32-NEXT:    xorl %ecx, %ecx
+; AVX512F-32-NEXT:    vcomisd %xmm1, %xmm4
+; AVX512F-32-NEXT:    setb %dl
+; AVX512F-32-NEXT:    kmovw %edx, %k1
+; AVX512F-32-NEXT:    vmovapd %xmm1, %xmm5
+; AVX512F-32-NEXT:    vmovsd %xmm3, %xmm5, %xmm5 {%k1}
+; AVX512F-32-NEXT:    vsubsd %xmm5, %xmm4, %xmm4
+; AVX512F-32-NEXT:    vmovsd %xmm4, (%esp)
+; AVX512F-32-NEXT:    fldl (%esp)
+; AVX512F-32-NEXT:    fisttpll (%esp)
+; AVX512F-32-NEXT:    setae %cl
+; AVX512F-32-NEXT:    shll $31, %ecx
+; AVX512F-32-NEXT:    xorl {{[0-9]+}}(%esp), %ecx
+; AVX512F-32-NEXT:    xorl %edx, %edx
+; AVX512F-32-NEXT:    vcomisd %xmm1, %xmm2
+; AVX512F-32-NEXT:    setb %bl
+; AVX512F-32-NEXT:    kmovw %ebx, %k1
+; AVX512F-32-NEXT:    vmovapd %xmm1, %xmm4
+; AVX512F-32-NEXT:    vmovsd %xmm3, %xmm4, %xmm4 {%k1}
+; AVX512F-32-NEXT:    vsubsd %xmm4, %xmm2, %xmm2
+; AVX512F-32-NEXT:    vmovsd %xmm2, {{[0-9]+}}(%esp)
+; AVX512F-32-NEXT:    fldl {{[0-9]+}}(%esp)
+; AVX512F-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX512F-32-NEXT:    setae %dl
+; AVX512F-32-NEXT:    shll $31, %edx
+; AVX512F-32-NEXT:    xorl {{[0-9]+}}(%esp), %edx
+; AVX512F-32-NEXT:    xorl %ebx, %ebx
+; AVX512F-32-NEXT:    vcomisd %xmm1, %xmm0
+; AVX512F-32-NEXT:    setb %al
+; AVX512F-32-NEXT:    kmovw %eax, %k1
+; AVX512F-32-NEXT:    vmovsd %xmm3, %xmm1, %xmm1 {%k1}
+; AVX512F-32-NEXT:    vsubsd %xmm1, %xmm0, %xmm0
+; AVX512F-32-NEXT:    vmovsd %xmm0, {{[0-9]+}}(%esp)
+; AVX512F-32-NEXT:    fldl {{[0-9]+}}(%esp)
+; AVX512F-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX512F-32-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX512F-32-NEXT:    vpinsrd $1, %edx, %xmm0, %xmm0
+; AVX512F-32-NEXT:    vpinsrd $2, (%esp), %xmm0, %xmm0
+; AVX512F-32-NEXT:    vpinsrd $3, %ecx, %xmm0, %xmm0
+; AVX512F-32-NEXT:    setae %bl
+; AVX512F-32-NEXT:    shll $31, %ebx
+; AVX512F-32-NEXT:    xorl {{[0-9]+}}(%esp), %ebx
+; AVX512F-32-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX512F-32-NEXT:    vpinsrd $1, %ebx, %xmm1, %xmm1
+; AVX512F-32-NEXT:    vpinsrd $2, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512F-32-NEXT:    vpinsrd $3, %esi, %xmm1, %xmm1
+; AVX512F-32-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX512F-32-NEXT:    leal -8(%ebp), %esp
+; AVX512F-32-NEXT:    popl %esi
+; AVX512F-32-NEXT:    popl %ebx
+; AVX512F-32-NEXT:    popl %ebp
+; AVX512F-32-NEXT:    .cfi_def_cfa %esp, 4
+; AVX512F-32-NEXT:    retl
+;
+; AVX512F-64-LABEL: strict_vector_fptoui_v4f64_to_v4i64:
+; AVX512F-64:       # %bb.0:
+; AVX512F-64-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX512F-64-NEXT:    vcvttsd2usi %xmm1, %rax
+; AVX512F-64-NEXT:    vmovq %rax, %xmm2
+; AVX512F-64-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm1[1,0]
+; AVX512F-64-NEXT:    vcvttsd2usi %xmm1, %rax
+; AVX512F-64-NEXT:    vmovq %rax, %xmm1
+; AVX512F-64-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX512F-64-NEXT:    vcvttsd2usi %xmm0, %rax
+; AVX512F-64-NEXT:    vmovq %rax, %xmm2
+; AVX512F-64-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
+; AVX512F-64-NEXT:    vcvttsd2usi %xmm0, %rax
+; AVX512F-64-NEXT:    vmovq %rax, %xmm0
+; AVX512F-64-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm2[0],xmm0[0]
+; AVX512F-64-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512F-64-NEXT:    retq
+;
+; AVX512VL-32-LABEL: strict_vector_fptoui_v4f64_to_v4i64:
+; AVX512VL-32:       # %bb.0:
+; AVX512VL-32-NEXT:    pushl %ebp
+; AVX512VL-32-NEXT:    .cfi_def_cfa_offset 8
+; AVX512VL-32-NEXT:    .cfi_offset %ebp, -8
+; AVX512VL-32-NEXT:    movl %esp, %ebp
+; AVX512VL-32-NEXT:    .cfi_def_cfa_register %ebp
+; AVX512VL-32-NEXT:    pushl %ebx
+; AVX512VL-32-NEXT:    pushl %esi
+; AVX512VL-32-NEXT:    andl $-8, %esp
+; AVX512VL-32-NEXT:    subl $32, %esp
+; AVX512VL-32-NEXT:    .cfi_offset %esi, -16
+; AVX512VL-32-NEXT:    .cfi_offset %ebx, -12
+; AVX512VL-32-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
+; AVX512VL-32-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
+; AVX512VL-32-NEXT:    vcomisd %xmm1, %xmm2
+; AVX512VL-32-NEXT:    setb %cl
+; AVX512VL-32-NEXT:    kmovw %ecx, %k1
+; AVX512VL-32-NEXT:    vxorpd %xmm3, %xmm3, %xmm3
+; AVX512VL-32-NEXT:    vmovapd %xmm1, %xmm4
+; AVX512VL-32-NEXT:    vmovsd %xmm3, %xmm4, %xmm4 {%k1}
+; AVX512VL-32-NEXT:    vsubsd %xmm4, %xmm2, %xmm2
+; AVX512VL-32-NEXT:    vmovsd %xmm2, {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    fldl {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    movl $0, %eax
+; AVX512VL-32-NEXT:    setae %al
+; AVX512VL-32-NEXT:    shll $31, %eax
+; AVX512VL-32-NEXT:    xorl {{[0-9]+}}(%esp), %eax
+; AVX512VL-32-NEXT:    movl %eax, %esi
+; AVX512VL-32-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX512VL-32-NEXT:    vpermilpd {{.*#+}} xmm4 = xmm2[1,0]
+; AVX512VL-32-NEXT:    xorl %ecx, %ecx
+; AVX512VL-32-NEXT:    vcomisd %xmm1, %xmm4
+; AVX512VL-32-NEXT:    setb %dl
+; AVX512VL-32-NEXT:    kmovw %edx, %k1
+; AVX512VL-32-NEXT:    vmovapd %xmm1, %xmm5
+; AVX512VL-32-NEXT:    vmovsd %xmm3, %xmm5, %xmm5 {%k1}
+; AVX512VL-32-NEXT:    vsubsd %xmm5, %xmm4, %xmm4
+; AVX512VL-32-NEXT:    vmovsd %xmm4, (%esp)
+; AVX512VL-32-NEXT:    fldl (%esp)
+; AVX512VL-32-NEXT:    fisttpll (%esp)
+; AVX512VL-32-NEXT:    setae %cl
+; AVX512VL-32-NEXT:    shll $31, %ecx
+; AVX512VL-32-NEXT:    xorl {{[0-9]+}}(%esp), %ecx
+; AVX512VL-32-NEXT:    xorl %edx, %edx
+; AVX512VL-32-NEXT:    vcomisd %xmm1, %xmm2
+; AVX512VL-32-NEXT:    setb %bl
+; AVX512VL-32-NEXT:    kmovw %ebx, %k1
+; AVX512VL-32-NEXT:    vmovapd %xmm1, %xmm4
+; AVX512VL-32-NEXT:    vmovsd %xmm3, %xmm4, %xmm4 {%k1}
+; AVX512VL-32-NEXT:    vsubsd %xmm4, %xmm2, %xmm2
+; AVX512VL-32-NEXT:    vmovsd %xmm2, {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    fldl {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    setae %dl
+; AVX512VL-32-NEXT:    shll $31, %edx
+; AVX512VL-32-NEXT:    xorl {{[0-9]+}}(%esp), %edx
+; AVX512VL-32-NEXT:    xorl %ebx, %ebx
+; AVX512VL-32-NEXT:    vcomisd %xmm1, %xmm0
+; AVX512VL-32-NEXT:    setb %al
+; AVX512VL-32-NEXT:    kmovw %eax, %k1
+; AVX512VL-32-NEXT:    vmovsd %xmm3, %xmm1, %xmm1 {%k1}
+; AVX512VL-32-NEXT:    vsubsd %xmm1, %xmm0, %xmm0
+; AVX512VL-32-NEXT:    vmovsd %xmm0, {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    fldl {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX512VL-32-NEXT:    vpinsrd $1, %edx, %xmm0, %xmm0
+; AVX512VL-32-NEXT:    vpinsrd $2, (%esp), %xmm0, %xmm0
+; AVX512VL-32-NEXT:    vpinsrd $3, %ecx, %xmm0, %xmm0
+; AVX512VL-32-NEXT:    setae %bl
+; AVX512VL-32-NEXT:    shll $31, %ebx
+; AVX512VL-32-NEXT:    xorl {{[0-9]+}}(%esp), %ebx
+; AVX512VL-32-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX512VL-32-NEXT:    vpinsrd $1, %ebx, %xmm1, %xmm1
+; AVX512VL-32-NEXT:    vpinsrd $2, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512VL-32-NEXT:    vpinsrd $3, %esi, %xmm1, %xmm1
+; AVX512VL-32-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX512VL-32-NEXT:    leal -8(%ebp), %esp
+; AVX512VL-32-NEXT:    popl %esi
+; AVX512VL-32-NEXT:    popl %ebx
+; AVX512VL-32-NEXT:    popl %ebp
+; AVX512VL-32-NEXT:    .cfi_def_cfa %esp, 4
+; AVX512VL-32-NEXT:    retl
+;
+; AVX512VL-64-LABEL: strict_vector_fptoui_v4f64_to_v4i64:
+; AVX512VL-64:       # %bb.0:
+; AVX512VL-64-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX512VL-64-NEXT:    vcvttsd2usi %xmm1, %rax
+; AVX512VL-64-NEXT:    vmovq %rax, %xmm2
+; AVX512VL-64-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm1[1,0]
+; AVX512VL-64-NEXT:    vcvttsd2usi %xmm1, %rax
+; AVX512VL-64-NEXT:    vmovq %rax, %xmm1
+; AVX512VL-64-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX512VL-64-NEXT:    vcvttsd2usi %xmm0, %rax
+; AVX512VL-64-NEXT:    vmovq %rax, %xmm2
+; AVX512VL-64-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
+; AVX512VL-64-NEXT:    vcvttsd2usi %xmm0, %rax
+; AVX512VL-64-NEXT:    vmovq %rax, %xmm0
+; AVX512VL-64-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm2[0],xmm0[0]
+; AVX512VL-64-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512VL-64-NEXT:    retq
+;
+; AVX512DQ-LABEL: strict_vector_fptoui_v4f64_to_v4i64:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vmovaps %ymm0, %ymm0
+; AVX512DQ-NEXT:    vcvttpd2uqq %zmm0, %zmm0
+; AVX512DQ-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQVL-LABEL: strict_vector_fptoui_v4f64_to_v4i64:
+; AVX512DQVL:       # %bb.0:
+; AVX512DQVL-NEXT:    vcvttpd2uqq %ymm0, %ymm0
+; AVX512DQVL-NEXT:    ret{{[l|q]}}
+  %ret = call <4 x i64> @llvm.experimental.constrained.fptoui.v4i64.v4f64(<4 x double> %a,
+                                              metadata !"fpexcept.strict")
+  ret <4 x i64> %ret
+}
+
+define <4 x i64> @strict_vector_fptosi_v4f32_to_v4i64(<4 x float> %a) #0 {
+; AVX-32-LABEL: strict_vector_fptosi_v4f32_to_v4i64:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    .cfi_def_cfa_offset 8
+; AVX-32-NEXT:    .cfi_offset %ebp, -8
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    .cfi_def_cfa_register %ebp
+; AVX-32-NEXT:    andl $-8, %esp
+; AVX-32-NEXT:    subl $32, %esp
+; AVX-32-NEXT:    vmovss %xmm0, {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    vextractps $1, %xmm0, {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    vextractps $2, %xmm0, {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    vextractps $3, %xmm0, (%esp)
+; AVX-32-NEXT:    flds {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    flds {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    flds {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    flds (%esp)
+; AVX-32-NEXT:    fisttpll (%esp)
+; AVX-32-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vpinsrd $1, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrd $2, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrd $3, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vpinsrd $1, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX-32-NEXT:    vpinsrd $2, (%esp), %xmm1, %xmm1
+; AVX-32-NEXT:    vpinsrd $3, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX-32-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    .cfi_def_cfa %esp, 4
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: strict_vector_fptosi_v4f32_to_v4i64:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,1,2,3]
+; AVX-64-NEXT:    vcvttss2si %xmm1, %rax
+; AVX-64-NEXT:    vmovq %rax, %xmm1
+; AVX-64-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
+; AVX-64-NEXT:    vcvttss2si %xmm2, %rax
+; AVX-64-NEXT:    vmovq %rax, %xmm2
+; AVX-64-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX-64-NEXT:    vcvttss2si %xmm0, %rax
+; AVX-64-NEXT:    vmovq %rax, %xmm2
+; AVX-64-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; AVX-64-NEXT:    vcvttss2si %xmm0, %rax
+; AVX-64-NEXT:    vmovq %rax, %xmm0
+; AVX-64-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm2[0],xmm0[0]
+; AVX-64-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX-64-NEXT:    retq
+;
+; AVX512F-32-LABEL: strict_vector_fptosi_v4f32_to_v4i64:
+; AVX512F-32:       # %bb.0:
+; AVX512F-32-NEXT:    pushl %ebp
+; AVX512F-32-NEXT:    .cfi_def_cfa_offset 8
+; AVX512F-32-NEXT:    .cfi_offset %ebp, -8
+; AVX512F-32-NEXT:    movl %esp, %ebp
+; AVX512F-32-NEXT:    .cfi_def_cfa_register %ebp
+; AVX512F-32-NEXT:    andl $-8, %esp
+; AVX512F-32-NEXT:    subl $32, %esp
+; AVX512F-32-NEXT:    vmovd %xmm0, {{[0-9]+}}(%esp)
+; AVX512F-32-NEXT:    vextractps $1, %xmm0, {{[0-9]+}}(%esp)
+; AVX512F-32-NEXT:    vextractps $2, %xmm0, {{[0-9]+}}(%esp)
+; AVX512F-32-NEXT:    vextractps $3, %xmm0, (%esp)
+; AVX512F-32-NEXT:    flds {{[0-9]+}}(%esp)
+; AVX512F-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX512F-32-NEXT:    flds {{[0-9]+}}(%esp)
+; AVX512F-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX512F-32-NEXT:    flds {{[0-9]+}}(%esp)
+; AVX512F-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX512F-32-NEXT:    flds (%esp)
+; AVX512F-32-NEXT:    fisttpll (%esp)
+; AVX512F-32-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX512F-32-NEXT:    vpinsrd $1, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512F-32-NEXT:    vpinsrd $2, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512F-32-NEXT:    vpinsrd $3, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512F-32-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX512F-32-NEXT:    vpinsrd $1, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512F-32-NEXT:    vpinsrd $2, (%esp), %xmm1, %xmm1
+; AVX512F-32-NEXT:    vpinsrd $3, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512F-32-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512F-32-NEXT:    movl %ebp, %esp
+; AVX512F-32-NEXT:    popl %ebp
+; AVX512F-32-NEXT:    .cfi_def_cfa %esp, 4
+; AVX512F-32-NEXT:    retl
+;
+; AVX512F-64-LABEL: strict_vector_fptosi_v4f32_to_v4i64:
+; AVX512F-64:       # %bb.0:
+; AVX512F-64-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,1,2,3]
+; AVX512F-64-NEXT:    vcvttss2si %xmm1, %rax
+; AVX512F-64-NEXT:    vmovq %rax, %xmm1
+; AVX512F-64-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
+; AVX512F-64-NEXT:    vcvttss2si %xmm2, %rax
+; AVX512F-64-NEXT:    vmovq %rax, %xmm2
+; AVX512F-64-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX512F-64-NEXT:    vcvttss2si %xmm0, %rax
+; AVX512F-64-NEXT:    vmovq %rax, %xmm2
+; AVX512F-64-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; AVX512F-64-NEXT:    vcvttss2si %xmm0, %rax
+; AVX512F-64-NEXT:    vmovq %rax, %xmm0
+; AVX512F-64-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm2[0],xmm0[0]
+; AVX512F-64-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512F-64-NEXT:    retq
+;
+; AVX512VL-32-LABEL: strict_vector_fptosi_v4f32_to_v4i64:
+; AVX512VL-32:       # %bb.0:
+; AVX512VL-32-NEXT:    pushl %ebp
+; AVX512VL-32-NEXT:    .cfi_def_cfa_offset 8
+; AVX512VL-32-NEXT:    .cfi_offset %ebp, -8
+; AVX512VL-32-NEXT:    movl %esp, %ebp
+; AVX512VL-32-NEXT:    .cfi_def_cfa_register %ebp
+; AVX512VL-32-NEXT:    andl $-8, %esp
+; AVX512VL-32-NEXT:    subl $32, %esp
+; AVX512VL-32-NEXT:    vmovd %xmm0, {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    vextractps $1, %xmm0, {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    vextractps $2, %xmm0, {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    vextractps $3, %xmm0, (%esp)
+; AVX512VL-32-NEXT:    flds {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    flds {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    flds {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    flds (%esp)
+; AVX512VL-32-NEXT:    fisttpll (%esp)
+; AVX512VL-32-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX512VL-32-NEXT:    vpinsrd $1, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512VL-32-NEXT:    vpinsrd $2, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512VL-32-NEXT:    vpinsrd $3, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512VL-32-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX512VL-32-NEXT:    vpinsrd $1, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512VL-32-NEXT:    vpinsrd $2, (%esp), %xmm1, %xmm1
+; AVX512VL-32-NEXT:    vpinsrd $3, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512VL-32-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512VL-32-NEXT:    movl %ebp, %esp
+; AVX512VL-32-NEXT:    popl %ebp
+; AVX512VL-32-NEXT:    .cfi_def_cfa %esp, 4
+; AVX512VL-32-NEXT:    retl
+;
+; AVX512VL-64-LABEL: strict_vector_fptosi_v4f32_to_v4i64:
+; AVX512VL-64:       # %bb.0:
+; AVX512VL-64-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,1,2,3]
+; AVX512VL-64-NEXT:    vcvttss2si %xmm1, %rax
+; AVX512VL-64-NEXT:    vmovq %rax, %xmm1
+; AVX512VL-64-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
+; AVX512VL-64-NEXT:    vcvttss2si %xmm2, %rax
+; AVX512VL-64-NEXT:    vmovq %rax, %xmm2
+; AVX512VL-64-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX512VL-64-NEXT:    vcvttss2si %xmm0, %rax
+; AVX512VL-64-NEXT:    vmovq %rax, %xmm2
+; AVX512VL-64-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; AVX512VL-64-NEXT:    vcvttss2si %xmm0, %rax
+; AVX512VL-64-NEXT:    vmovq %rax, %xmm0
+; AVX512VL-64-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm2[0],xmm0[0]
+; AVX512VL-64-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512VL-64-NEXT:    retq
+;
+; AVX512DQ-LABEL: strict_vector_fptosi_v4f32_to_v4i64:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vmovaps %xmm0, %xmm0
+; AVX512DQ-NEXT:    vcvttps2qq %ymm0, %zmm0
+; AVX512DQ-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQVL-LABEL: strict_vector_fptosi_v4f32_to_v4i64:
+; AVX512DQVL:       # %bb.0:
+; AVX512DQVL-NEXT:    vcvttps2qq %xmm0, %ymm0
+; AVX512DQVL-NEXT:    ret{{[l|q]}}
+  %ret = call <4 x i64> @llvm.experimental.constrained.fptosi.v4i64.v4f32(<4 x float> %a,
+                                              metadata !"fpexcept.strict")
+  ret <4 x i64> %ret
+}
+
+define <4 x i64> @strict_vector_fptoui_v4f32_to_v4i64(<4 x float> %a) #0 {
+; AVX-32-LABEL: strict_vector_fptoui_v4f32_to_v4i64:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    .cfi_def_cfa_offset 8
+; AVX-32-NEXT:    .cfi_offset %ebp, -8
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    .cfi_def_cfa_register %ebp
+; AVX-32-NEXT:    andl $-8, %esp
+; AVX-32-NEXT:    subl $32, %esp
+; AVX-32-NEXT:    vmovshdup {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; AVX-32-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vcomiss %xmm1, %xmm3
+; AVX-32-NEXT:    vxorps %xmm2, %xmm2, %xmm2
+; AVX-32-NEXT:    vxorps %xmm4, %xmm4, %xmm4
+; AVX-32-NEXT:    jb .LBB3_2
+; AVX-32-NEXT:  # %bb.1:
+; AVX-32-NEXT:    vmovaps %xmm1, %xmm4
+; AVX-32-NEXT:  .LBB3_2:
+; AVX-32-NEXT:    vsubss %xmm4, %xmm3, %xmm3
+; AVX-32-NEXT:    vmovss %xmm3, {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    flds {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    setae %al
+; AVX-32-NEXT:    movzbl %al, %eax
+; AVX-32-NEXT:    shll $31, %eax
+; AVX-32-NEXT:    xorl {{[0-9]+}}(%esp), %eax
+; AVX-32-NEXT:    vpermilps {{.*#+}} xmm3 = xmm0[3,1,2,3]
+; AVX-32-NEXT:    vcomiss %xmm1, %xmm3
+; AVX-32-NEXT:    vxorps %xmm4, %xmm4, %xmm4
+; AVX-32-NEXT:    jb .LBB3_4
+; AVX-32-NEXT:  # %bb.3:
+; AVX-32-NEXT:    vmovaps %xmm1, %xmm4
+; AVX-32-NEXT:  .LBB3_4:
+; AVX-32-NEXT:    vsubss %xmm4, %xmm3, %xmm3
+; AVX-32-NEXT:    vmovss %xmm3, (%esp)
+; AVX-32-NEXT:    flds (%esp)
+; AVX-32-NEXT:    fisttpll (%esp)
+; AVX-32-NEXT:    setae %cl
+; AVX-32-NEXT:    movzbl %cl, %ecx
+; AVX-32-NEXT:    shll $31, %ecx
+; AVX-32-NEXT:    xorl {{[0-9]+}}(%esp), %ecx
+; AVX-32-NEXT:    vpermilpd {{.*#+}} xmm3 = xmm0[1,0]
+; AVX-32-NEXT:    vcomiss %xmm1, %xmm3
+; AVX-32-NEXT:    vxorps %xmm4, %xmm4, %xmm4
+; AVX-32-NEXT:    jb .LBB3_6
+; AVX-32-NEXT:  # %bb.5:
+; AVX-32-NEXT:    vmovaps %xmm1, %xmm4
+; AVX-32-NEXT:  .LBB3_6:
+; AVX-32-NEXT:    vsubss %xmm4, %xmm3, %xmm3
+; AVX-32-NEXT:    vmovss %xmm3, {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    flds {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    setae %dl
+; AVX-32-NEXT:    movzbl %dl, %edx
+; AVX-32-NEXT:    shll $31, %edx
+; AVX-32-NEXT:    xorl {{[0-9]+}}(%esp), %edx
+; AVX-32-NEXT:    vcomiss %xmm1, %xmm0
+; AVX-32-NEXT:    jb .LBB3_8
+; AVX-32-NEXT:  # %bb.7:
+; AVX-32-NEXT:    vmovaps %xmm1, %xmm2
+; AVX-32-NEXT:  .LBB3_8:
+; AVX-32-NEXT:    vsubss %xmm2, %xmm0, %xmm0
+; AVX-32-NEXT:    vmovss %xmm0, {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    flds {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vpinsrd $1, %edx, %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrd $2, (%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrd $3, %ecx, %xmm0, %xmm0
+; AVX-32-NEXT:    setae %cl
+; AVX-32-NEXT:    movzbl %cl, %ecx
+; AVX-32-NEXT:    shll $31, %ecx
+; AVX-32-NEXT:    xorl {{[0-9]+}}(%esp), %ecx
+; AVX-32-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vpinsrd $1, %ecx, %xmm1, %xmm1
+; AVX-32-NEXT:    vpinsrd $2, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX-32-NEXT:    vpinsrd $3, %eax, %xmm1, %xmm1
+; AVX-32-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    .cfi_def_cfa %esp, 4
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: strict_vector_fptoui_v4f32_to_v4i64:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vpermilps {{.*#+}} xmm3 = xmm0[3,1,2,3]
+; AVX-64-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX-64-NEXT:    vcomiss %xmm1, %xmm3
+; AVX-64-NEXT:    vxorps %xmm2, %xmm2, %xmm2
+; AVX-64-NEXT:    vxorps %xmm4, %xmm4, %xmm4
+; AVX-64-NEXT:    jb .LBB3_2
+; AVX-64-NEXT:  # %bb.1:
+; AVX-64-NEXT:    vmovaps %xmm1, %xmm4
+; AVX-64-NEXT:  .LBB3_2:
+; AVX-64-NEXT:    vsubss %xmm4, %xmm3, %xmm3
+; AVX-64-NEXT:    vcvttss2si %xmm3, %rcx
+; AVX-64-NEXT:    setae %al
+; AVX-64-NEXT:    movzbl %al, %eax
+; AVX-64-NEXT:    shlq $63, %rax
+; AVX-64-NEXT:    xorq %rcx, %rax
+; AVX-64-NEXT:    vpermilpd {{.*#+}} xmm4 = xmm0[1,0]
+; AVX-64-NEXT:    vcomiss %xmm1, %xmm4
+; AVX-64-NEXT:    vxorps %xmm5, %xmm5, %xmm5
+; AVX-64-NEXT:    jb .LBB3_4
+; AVX-64-NEXT:  # %bb.3:
+; AVX-64-NEXT:    vmovaps %xmm1, %xmm5
+; AVX-64-NEXT:  .LBB3_4:
+; AVX-64-NEXT:    vmovq %rax, %xmm3
+; AVX-64-NEXT:    vsubss %xmm5, %xmm4, %xmm4
+; AVX-64-NEXT:    vcvttss2si %xmm4, %rax
+; AVX-64-NEXT:    setae %cl
+; AVX-64-NEXT:    movzbl %cl, %ecx
+; AVX-64-NEXT:    shlq $63, %rcx
+; AVX-64-NEXT:    xorq %rax, %rcx
+; AVX-64-NEXT:    vmovq %rcx, %xmm4
+; AVX-64-NEXT:    vcomiss %xmm1, %xmm0
+; AVX-64-NEXT:    vxorps %xmm5, %xmm5, %xmm5
+; AVX-64-NEXT:    jb .LBB3_6
+; AVX-64-NEXT:  # %bb.5:
+; AVX-64-NEXT:    vmovaps %xmm1, %xmm5
+; AVX-64-NEXT:  .LBB3_6:
+; AVX-64-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm4[0],xmm3[0]
+; AVX-64-NEXT:    vsubss %xmm5, %xmm0, %xmm4
+; AVX-64-NEXT:    vcvttss2si %xmm4, %rax
+; AVX-64-NEXT:    setae %cl
+; AVX-64-NEXT:    movzbl %cl, %ecx
+; AVX-64-NEXT:    shlq $63, %rcx
+; AVX-64-NEXT:    xorq %rax, %rcx
+; AVX-64-NEXT:    vmovq %rcx, %xmm4
+; AVX-64-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; AVX-64-NEXT:    vcomiss %xmm1, %xmm0
+; AVX-64-NEXT:    jb .LBB3_8
+; AVX-64-NEXT:  # %bb.7:
+; AVX-64-NEXT:    vmovaps %xmm1, %xmm2
+; AVX-64-NEXT:  .LBB3_8:
+; AVX-64-NEXT:    vsubss %xmm2, %xmm0, %xmm0
+; AVX-64-NEXT:    vcvttss2si %xmm0, %rax
+; AVX-64-NEXT:    setae %cl
+; AVX-64-NEXT:    movzbl %cl, %ecx
+; AVX-64-NEXT:    shlq $63, %rcx
+; AVX-64-NEXT:    xorq %rax, %rcx
+; AVX-64-NEXT:    vmovq %rcx, %xmm0
+; AVX-64-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm4[0],xmm0[0]
+; AVX-64-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
+; AVX-64-NEXT:    retq
+;
+; AVX512F-32-LABEL: strict_vector_fptoui_v4f32_to_v4i64:
+; AVX512F-32:       # %bb.0:
+; AVX512F-32-NEXT:    pushl %ebp
+; AVX512F-32-NEXT:    .cfi_def_cfa_offset 8
+; AVX512F-32-NEXT:    .cfi_offset %ebp, -8
+; AVX512F-32-NEXT:    movl %esp, %ebp
+; AVX512F-32-NEXT:    .cfi_def_cfa_register %ebp
+; AVX512F-32-NEXT:    pushl %ebx
+; AVX512F-32-NEXT:    pushl %esi
+; AVX512F-32-NEXT:    andl $-8, %esp
+; AVX512F-32-NEXT:    subl $32, %esp
+; AVX512F-32-NEXT:    .cfi_offset %esi, -16
+; AVX512F-32-NEXT:    .cfi_offset %ebx, -12
+; AVX512F-32-NEXT:    vmovshdup {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; AVX512F-32-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX512F-32-NEXT:    vcomiss %xmm1, %xmm2
+; AVX512F-32-NEXT:    setb %cl
+; AVX512F-32-NEXT:    kmovw %ecx, %k1
+; AVX512F-32-NEXT:    vxorps %xmm3, %xmm3, %xmm3
+; AVX512F-32-NEXT:    vmovaps %xmm1, %xmm4
+; AVX512F-32-NEXT:    vmovss %xmm3, %xmm4, %xmm4 {%k1}
+; AVX512F-32-NEXT:    vsubss %xmm4, %xmm2, %xmm2
+; AVX512F-32-NEXT:    vmovss %xmm2, {{[0-9]+}}(%esp)
+; AVX512F-32-NEXT:    flds {{[0-9]+}}(%esp)
+; AVX512F-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX512F-32-NEXT:    movl $0, %eax
+; AVX512F-32-NEXT:    setae %al
+; AVX512F-32-NEXT:    shll $31, %eax
+; AVX512F-32-NEXT:    xorl {{[0-9]+}}(%esp), %eax
+; AVX512F-32-NEXT:    movl %eax, %esi
+; AVX512F-32-NEXT:    vpermilps {{.*#+}} xmm2 = xmm0[3,1,2,3]
+; AVX512F-32-NEXT:    xorl %ecx, %ecx
+; AVX512F-32-NEXT:    vcomiss %xmm1, %xmm2
+; AVX512F-32-NEXT:    setb %dl
+; AVX512F-32-NEXT:    kmovw %edx, %k1
+; AVX512F-32-NEXT:    vmovaps %xmm1, %xmm4
+; AVX512F-32-NEXT:    vmovss %xmm3, %xmm4, %xmm4 {%k1}
+; AVX512F-32-NEXT:    vsubss %xmm4, %xmm2, %xmm2
+; AVX512F-32-NEXT:    vmovss %xmm2, (%esp)
+; AVX512F-32-NEXT:    flds (%esp)
+; AVX512F-32-NEXT:    fisttpll (%esp)
+; AVX512F-32-NEXT:    setae %cl
+; AVX512F-32-NEXT:    shll $31, %ecx
+; AVX512F-32-NEXT:    xorl {{[0-9]+}}(%esp), %ecx
+; AVX512F-32-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
+; AVX512F-32-NEXT:    xorl %edx, %edx
+; AVX512F-32-NEXT:    vcomiss %xmm1, %xmm2
+; AVX512F-32-NEXT:    setb %bl
+; AVX512F-32-NEXT:    kmovw %ebx, %k1
+; AVX512F-32-NEXT:    vmovaps %xmm1, %xmm4
+; AVX512F-32-NEXT:    vmovss %xmm3, %xmm4, %xmm4 {%k1}
+; AVX512F-32-NEXT:    vsubss %xmm4, %xmm2, %xmm2
+; AVX512F-32-NEXT:    vmovss %xmm2, {{[0-9]+}}(%esp)
+; AVX512F-32-NEXT:    flds {{[0-9]+}}(%esp)
+; AVX512F-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX512F-32-NEXT:    setae %dl
+; AVX512F-32-NEXT:    shll $31, %edx
+; AVX512F-32-NEXT:    xorl {{[0-9]+}}(%esp), %edx
+; AVX512F-32-NEXT:    xorl %ebx, %ebx
+; AVX512F-32-NEXT:    vcomiss %xmm1, %xmm0
+; AVX512F-32-NEXT:    setb %al
+; AVX512F-32-NEXT:    kmovw %eax, %k1
+; AVX512F-32-NEXT:    vmovss %xmm3, %xmm1, %xmm1 {%k1}
+; AVX512F-32-NEXT:    vsubss %xmm1, %xmm0, %xmm0
+; AVX512F-32-NEXT:    vmovss %xmm0, {{[0-9]+}}(%esp)
+; AVX512F-32-NEXT:    flds {{[0-9]+}}(%esp)
+; AVX512F-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX512F-32-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX512F-32-NEXT:    vpinsrd $1, %edx, %xmm0, %xmm0
+; AVX512F-32-NEXT:    vpinsrd $2, (%esp), %xmm0, %xmm0
+; AVX512F-32-NEXT:    vpinsrd $3, %ecx, %xmm0, %xmm0
+; AVX512F-32-NEXT:    setae %bl
+; AVX512F-32-NEXT:    shll $31, %ebx
+; AVX512F-32-NEXT:    xorl {{[0-9]+}}(%esp), %ebx
+; AVX512F-32-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX512F-32-NEXT:    vpinsrd $1, %ebx, %xmm1, %xmm1
+; AVX512F-32-NEXT:    vpinsrd $2, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512F-32-NEXT:    vpinsrd $3, %esi, %xmm1, %xmm1
+; AVX512F-32-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX512F-32-NEXT:    leal -8(%ebp), %esp
+; AVX512F-32-NEXT:    popl %esi
+; AVX512F-32-NEXT:    popl %ebx
+; AVX512F-32-NEXT:    popl %ebp
+; AVX512F-32-NEXT:    .cfi_def_cfa %esp, 4
+; AVX512F-32-NEXT:    retl
+;
+; AVX512F-64-LABEL: strict_vector_fptoui_v4f32_to_v4i64:
+; AVX512F-64:       # %bb.0:
+; AVX512F-64-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,1,2,3]
+; AVX512F-64-NEXT:    vcvttss2usi %xmm1, %rax
+; AVX512F-64-NEXT:    vmovq %rax, %xmm1
+; AVX512F-64-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
+; AVX512F-64-NEXT:    vcvttss2usi %xmm2, %rax
+; AVX512F-64-NEXT:    vmovq %rax, %xmm2
+; AVX512F-64-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX512F-64-NEXT:    vcvttss2usi %xmm0, %rax
+; AVX512F-64-NEXT:    vmovq %rax, %xmm2
+; AVX512F-64-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; AVX512F-64-NEXT:    vcvttss2usi %xmm0, %rax
+; AVX512F-64-NEXT:    vmovq %rax, %xmm0
+; AVX512F-64-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm2[0],xmm0[0]
+; AVX512F-64-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512F-64-NEXT:    retq
+;
+; AVX512VL-32-LABEL: strict_vector_fptoui_v4f32_to_v4i64:
+; AVX512VL-32:       # %bb.0:
+; AVX512VL-32-NEXT:    pushl %ebp
+; AVX512VL-32-NEXT:    .cfi_def_cfa_offset 8
+; AVX512VL-32-NEXT:    .cfi_offset %ebp, -8
+; AVX512VL-32-NEXT:    movl %esp, %ebp
+; AVX512VL-32-NEXT:    .cfi_def_cfa_register %ebp
+; AVX512VL-32-NEXT:    pushl %ebx
+; AVX512VL-32-NEXT:    pushl %esi
+; AVX512VL-32-NEXT:    andl $-8, %esp
+; AVX512VL-32-NEXT:    subl $32, %esp
+; AVX512VL-32-NEXT:    .cfi_offset %esi, -16
+; AVX512VL-32-NEXT:    .cfi_offset %ebx, -12
+; AVX512VL-32-NEXT:    vmovshdup {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; AVX512VL-32-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX512VL-32-NEXT:    vcomiss %xmm1, %xmm2
+; AVX512VL-32-NEXT:    setb %cl
+; AVX512VL-32-NEXT:    kmovw %ecx, %k1
+; AVX512VL-32-NEXT:    vxorps %xmm3, %xmm3, %xmm3
+; AVX512VL-32-NEXT:    vmovaps %xmm1, %xmm4
+; AVX512VL-32-NEXT:    vmovss %xmm3, %xmm4, %xmm4 {%k1}
+; AVX512VL-32-NEXT:    vsubss %xmm4, %xmm2, %xmm2
+; AVX512VL-32-NEXT:    vmovss %xmm2, {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    flds {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    movl $0, %eax
+; AVX512VL-32-NEXT:    setae %al
+; AVX512VL-32-NEXT:    shll $31, %eax
+; AVX512VL-32-NEXT:    xorl {{[0-9]+}}(%esp), %eax
+; AVX512VL-32-NEXT:    movl %eax, %esi
+; AVX512VL-32-NEXT:    vpermilps {{.*#+}} xmm2 = xmm0[3,1,2,3]
+; AVX512VL-32-NEXT:    xorl %ecx, %ecx
+; AVX512VL-32-NEXT:    vcomiss %xmm1, %xmm2
+; AVX512VL-32-NEXT:    setb %dl
+; AVX512VL-32-NEXT:    kmovw %edx, %k1
+; AVX512VL-32-NEXT:    vmovaps %xmm1, %xmm4
+; AVX512VL-32-NEXT:    vmovss %xmm3, %xmm4, %xmm4 {%k1}
+; AVX512VL-32-NEXT:    vsubss %xmm4, %xmm2, %xmm2
+; AVX512VL-32-NEXT:    vmovss %xmm2, (%esp)
+; AVX512VL-32-NEXT:    flds (%esp)
+; AVX512VL-32-NEXT:    fisttpll (%esp)
+; AVX512VL-32-NEXT:    setae %cl
+; AVX512VL-32-NEXT:    shll $31, %ecx
+; AVX512VL-32-NEXT:    xorl {{[0-9]+}}(%esp), %ecx
+; AVX512VL-32-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
+; AVX512VL-32-NEXT:    xorl %edx, %edx
+; AVX512VL-32-NEXT:    vcomiss %xmm1, %xmm2
+; AVX512VL-32-NEXT:    setb %bl
+; AVX512VL-32-NEXT:    kmovw %ebx, %k1
+; AVX512VL-32-NEXT:    vmovaps %xmm1, %xmm4
+; AVX512VL-32-NEXT:    vmovss %xmm3, %xmm4, %xmm4 {%k1}
+; AVX512VL-32-NEXT:    vsubss %xmm4, %xmm2, %xmm2
+; AVX512VL-32-NEXT:    vmovss %xmm2, {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    flds {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    setae %dl
+; AVX512VL-32-NEXT:    shll $31, %edx
+; AVX512VL-32-NEXT:    xorl {{[0-9]+}}(%esp), %edx
+; AVX512VL-32-NEXT:    xorl %ebx, %ebx
+; AVX512VL-32-NEXT:    vcomiss %xmm1, %xmm0
+; AVX512VL-32-NEXT:    setb %al
+; AVX512VL-32-NEXT:    kmovw %eax, %k1
+; AVX512VL-32-NEXT:    vmovss %xmm3, %xmm1, %xmm1 {%k1}
+; AVX512VL-32-NEXT:    vsubss %xmm1, %xmm0, %xmm0
+; AVX512VL-32-NEXT:    vmovss %xmm0, {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    flds {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX512VL-32-NEXT:    vpinsrd $1, %edx, %xmm0, %xmm0
+; AVX512VL-32-NEXT:    vpinsrd $2, (%esp), %xmm0, %xmm0
+; AVX512VL-32-NEXT:    vpinsrd $3, %ecx, %xmm0, %xmm0
+; AVX512VL-32-NEXT:    setae %bl
+; AVX512VL-32-NEXT:    shll $31, %ebx
+; AVX512VL-32-NEXT:    xorl {{[0-9]+}}(%esp), %ebx
+; AVX512VL-32-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX512VL-32-NEXT:    vpinsrd $1, %ebx, %xmm1, %xmm1
+; AVX512VL-32-NEXT:    vpinsrd $2, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512VL-32-NEXT:    vpinsrd $3, %esi, %xmm1, %xmm1
+; AVX512VL-32-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX512VL-32-NEXT:    leal -8(%ebp), %esp
+; AVX512VL-32-NEXT:    popl %esi
+; AVX512VL-32-NEXT:    popl %ebx
+; AVX512VL-32-NEXT:    popl %ebp
+; AVX512VL-32-NEXT:    .cfi_def_cfa %esp, 4
+; AVX512VL-32-NEXT:    retl
+;
+; AVX512VL-64-LABEL: strict_vector_fptoui_v4f32_to_v4i64:
+; AVX512VL-64:       # %bb.0:
+; AVX512VL-64-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,1,2,3]
+; AVX512VL-64-NEXT:    vcvttss2usi %xmm1, %rax
+; AVX512VL-64-NEXT:    vmovq %rax, %xmm1
+; AVX512VL-64-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
+; AVX512VL-64-NEXT:    vcvttss2usi %xmm2, %rax
+; AVX512VL-64-NEXT:    vmovq %rax, %xmm2
+; AVX512VL-64-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX512VL-64-NEXT:    vcvttss2usi %xmm0, %rax
+; AVX512VL-64-NEXT:    vmovq %rax, %xmm2
+; AVX512VL-64-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; AVX512VL-64-NEXT:    vcvttss2usi %xmm0, %rax
+; AVX512VL-64-NEXT:    vmovq %rax, %xmm0
+; AVX512VL-64-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm2[0],xmm0[0]
+; AVX512VL-64-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512VL-64-NEXT:    retq
+;
+; AVX512DQ-LABEL: strict_vector_fptoui_v4f32_to_v4i64:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vmovaps %xmm0, %xmm0
+; AVX512DQ-NEXT:    vcvttps2uqq %ymm0, %zmm0
+; AVX512DQ-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQVL-LABEL: strict_vector_fptoui_v4f32_to_v4i64:
+; AVX512DQVL:       # %bb.0:
+; AVX512DQVL-NEXT:    vcvttps2uqq %xmm0, %ymm0
+; AVX512DQVL-NEXT:    ret{{[l|q]}}
+  %ret = call <4 x i64> @llvm.experimental.constrained.fptoui.v4i64.v4f32(<4 x float> %a,
+                                              metadata !"fpexcept.strict")
+  ret <4 x i64> %ret
+}
+
+define <4 x i32> @strict_vector_fptosi_v4f64_to_v4i32(<4 x double> %a) #0 {
+; CHECK-LABEL: strict_vector_fptosi_v4f64_to_v4i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vcvttpd2dq %ymm0, %xmm0
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    ret{{[l|q]}}
+  %ret = call <4 x i32> @llvm.experimental.constrained.fptosi.v4i32.v4f64(<4 x double> %a,
+                                              metadata !"fpexcept.strict")
+  ret <4 x i32> %ret
+}
+
+define <4 x i32> @strict_vector_fptoui_v4f64_to_v4i32(<4 x double> %a) #0 {
+; AVX-32-LABEL: strict_vector_fptoui_v4f64_to_v4i32:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vmovapd {{.*#+}} ymm1 = [2.147483648E+9,2.147483648E+9,2.147483648E+9,2.147483648E+9]
+; AVX-32-NEXT:    vcmpltpd %ymm1, %ymm0, %ymm2
+; AVX-32-NEXT:    vextractf128 $1, %ymm2, %xmm3
+; AVX-32-NEXT:    vshufps {{.*#+}} xmm3 = xmm2[0,2],xmm3[0,2]
+; AVX-32-NEXT:    vxorps %xmm4, %xmm4, %xmm4
+; AVX-32-NEXT:    vmovaps {{.*#+}} xmm5 = [2147483648,2147483648,2147483648,2147483648]
+; AVX-32-NEXT:    vblendvps %xmm3, %xmm4, %xmm5, %xmm3
+; AVX-32-NEXT:    vxorps %xmm4, %xmm4, %xmm4
+; AVX-32-NEXT:    vblendvpd %ymm2, %ymm4, %ymm1, %ymm1
+; AVX-32-NEXT:    vsubpd %ymm1, %ymm0, %ymm0
+; AVX-32-NEXT:    vcvttpd2dq %ymm0, %xmm0
+; AVX-32-NEXT:    vxorpd %xmm3, %xmm0, %xmm0
+; AVX-32-NEXT:    vzeroupper
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: strict_vector_fptoui_v4f64_to_v4i32:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vmovapd {{.*#+}} ymm1 = [2.147483648E+9,2.147483648E+9,2.147483648E+9,2.147483648E+9]
+; AVX-64-NEXT:    vcmpltpd %ymm1, %ymm0, %ymm2
+; AVX-64-NEXT:    vextractf128 $1, %ymm2, %xmm3
+; AVX-64-NEXT:    vshufps {{.*#+}} xmm3 = xmm2[0,2],xmm3[0,2]
+; AVX-64-NEXT:    vxorps %xmm4, %xmm4, %xmm4
+; AVX-64-NEXT:    vmovaps {{.*#+}} xmm5 = [2147483648,2147483648,2147483648,2147483648]
+; AVX-64-NEXT:    vblendvps %xmm3, %xmm4, %xmm5, %xmm3
+; AVX-64-NEXT:    vxorps %xmm4, %xmm4, %xmm4
+; AVX-64-NEXT:    vblendvpd %ymm2, %ymm4, %ymm1, %ymm1
+; AVX-64-NEXT:    vsubpd %ymm1, %ymm0, %ymm0
+; AVX-64-NEXT:    vcvttpd2dq %ymm0, %xmm0
+; AVX-64-NEXT:    vxorpd %xmm3, %xmm0, %xmm0
+; AVX-64-NEXT:    vzeroupper
+; AVX-64-NEXT:    retq
+;
+; AVX512F-LABEL: strict_vector_fptoui_v4f64_to_v4i32:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vmovaps %ymm0, %ymm0
+; AVX512F-NEXT:    vcvttpd2udq %zmm0, %ymm0
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    ret{{[l|q]}}
+;
+; AVX512VL-LABEL: strict_vector_fptoui_v4f64_to_v4i32:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vcvttpd2udq %ymm0, %xmm0
+; AVX512VL-NEXT:    vzeroupper
+; AVX512VL-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQ-LABEL: strict_vector_fptoui_v4f64_to_v4i32:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vmovaps %ymm0, %ymm0
+; AVX512DQ-NEXT:    vcvttpd2udq %zmm0, %ymm0
+; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQVL-LABEL: strict_vector_fptoui_v4f64_to_v4i32:
+; AVX512DQVL:       # %bb.0:
+; AVX512DQVL-NEXT:    vcvttpd2udq %ymm0, %xmm0
+; AVX512DQVL-NEXT:    vzeroupper
+; AVX512DQVL-NEXT:    ret{{[l|q]}}
+  %ret = call <4 x i32> @llvm.experimental.constrained.fptoui.v4i32.v4f64(<4 x double> %a,
+                                              metadata !"fpexcept.strict")
+  ret <4 x i32> %ret
+}
+
+define <4 x i16> @strict_vector_fptosi_v4f64_to_v4i16(<4 x double> %a) #0 {
+; CHECK-LABEL: strict_vector_fptosi_v4f64_to_v4i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vcvttpd2dq %ymm0, %xmm0
+; CHECK-NEXT:    vpackssdw %xmm0, %xmm0, %xmm0
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    ret{{[l|q]}}
+  %ret = call <4 x i16> @llvm.experimental.constrained.fptosi.v4i16.v4f64(<4 x double> %a,
+                                              metadata !"fpexcept.strict")
+  ret <4 x i16> %ret
+}
+
+define <4 x i16> @strict_vector_fptoui_v4f64_to_v4i16(<4 x double> %a) #0 {
+; CHECK-LABEL: strict_vector_fptoui_v4f64_to_v4i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vcvttpd2dq %ymm0, %xmm0
+; CHECK-NEXT:    vpackusdw %xmm0, %xmm0, %xmm0
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    ret{{[l|q]}}
+  %ret = call <4 x i16> @llvm.experimental.constrained.fptoui.v4i16.v4f64(<4 x double> %a,
+                                              metadata !"fpexcept.strict")
+  ret <4 x i16> %ret
+}
+
+define <4 x i8> @strict_vector_fptosi_v4f64_to_v4i8(<4 x double> %a) #0 {
+; CHECK-LABEL: strict_vector_fptosi_v4f64_to_v4i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vcvttpd2dq %ymm0, %xmm0
+; CHECK-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    ret{{[l|q]}}
+  %ret = call <4 x i8> @llvm.experimental.constrained.fptosi.v4i8.v4f64(<4 x double> %a,
+                                              metadata !"fpexcept.strict")
+  ret <4 x i8> %ret
+}
+
+define <4 x i8> @strict_vector_fptoui_v4f64_to_v4i8(<4 x double> %a) #0 {
+; CHECK-LABEL: strict_vector_fptoui_v4f64_to_v4i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vcvttpd2dq %ymm0, %xmm0
+; CHECK-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    ret{{[l|q]}}
+  %ret = call <4 x i8> @llvm.experimental.constrained.fptoui.v4i8.v4f64(<4 x double> %a,
+                                              metadata !"fpexcept.strict")
+  ret <4 x i8> %ret
+}
+
+define <4 x i1> @strict_vector_fptosi_v4f64_to_v4i1(<4 x double> %a) #0 {
+; AVX-32-LABEL: strict_vector_fptosi_v4f64_to_v4i1:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vcvttpd2dq %ymm0, %xmm0
+; AVX-32-NEXT:    vzeroupper
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: strict_vector_fptosi_v4f64_to_v4i1:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcvttpd2dq %ymm0, %xmm0
+; AVX-64-NEXT:    vzeroupper
+; AVX-64-NEXT:    retq
+;
+; AVX512F-LABEL: strict_vector_fptosi_v4f64_to_v4i1:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vcvttpd2dq %ymm0, %xmm0
+; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k1
+; AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    ret{{[l|q]}}
+;
+; AVX512VL-LABEL: strict_vector_fptosi_v4f64_to_v4i1:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vcvttpd2dq %ymm0, %xmm0
+; AVX512VL-NEXT:    vptestmd %xmm0, %xmm0, %k1
+; AVX512VL-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX512VL-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
+; AVX512VL-NEXT:    vzeroupper
+; AVX512VL-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQ-LABEL: strict_vector_fptosi_v4f64_to_v4i1:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vcvttpd2dq %ymm0, %xmm0
+; AVX512DQ-NEXT:    vpmovd2m %zmm0, %k0
+; AVX512DQ-NEXT:    vpmovm2d %k0, %zmm0
+; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQVL-LABEL: strict_vector_fptosi_v4f64_to_v4i1:
+; AVX512DQVL:       # %bb.0:
+; AVX512DQVL-NEXT:    vcvttpd2dq %ymm0, %xmm0
+; AVX512DQVL-NEXT:    vpmovd2m %xmm0, %k0
+; AVX512DQVL-NEXT:    vpmovm2d %k0, %xmm0
+; AVX512DQVL-NEXT:    vzeroupper
+; AVX512DQVL-NEXT:    ret{{[l|q]}}
+  %ret = call <4 x i1> @llvm.experimental.constrained.fptosi.v4i1.v4f64(<4 x double> %a,
+                                              metadata !"fpexcept.strict")
+  ret <4 x i1> %ret
+}
+
+define <4 x i1> @strict_vector_fptoui_v4f64_to_v4i1(<4 x double> %a) #0 {
+; AVX-32-LABEL: strict_vector_fptoui_v4f64_to_v4i1:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vcvttpd2dq %ymm0, %xmm0
+; AVX-32-NEXT:    vzeroupper
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: strict_vector_fptoui_v4f64_to_v4i1:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcvttpd2dq %ymm0, %xmm0
+; AVX-64-NEXT:    vzeroupper
+; AVX-64-NEXT:    retq
+;
+; AVX512F-LABEL: strict_vector_fptoui_v4f64_to_v4i1:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vcvttpd2dq %ymm0, %xmm0
+; AVX512F-NEXT:    vpslld $31, %xmm0, %xmm0
+; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k1
+; AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    ret{{[l|q]}}
+;
+; AVX512VL-LABEL: strict_vector_fptoui_v4f64_to_v4i1:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vcvttpd2dq %ymm0, %xmm0
+; AVX512VL-NEXT:    vpslld $31, %xmm0, %xmm0
+; AVX512VL-NEXT:    vptestmd %xmm0, %xmm0, %k1
+; AVX512VL-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX512VL-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
+; AVX512VL-NEXT:    vzeroupper
+; AVX512VL-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQ-LABEL: strict_vector_fptoui_v4f64_to_v4i1:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vcvttpd2dq %ymm0, %xmm0
+; AVX512DQ-NEXT:    vpslld $31, %xmm0, %xmm0
+; AVX512DQ-NEXT:    vpmovd2m %zmm0, %k0
+; AVX512DQ-NEXT:    vpmovm2d %k0, %zmm0
+; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQVL-LABEL: strict_vector_fptoui_v4f64_to_v4i1:
+; AVX512DQVL:       # %bb.0:
+; AVX512DQVL-NEXT:    vcvttpd2dq %ymm0, %xmm0
+; AVX512DQVL-NEXT:    vpslld $31, %xmm0, %xmm0
+; AVX512DQVL-NEXT:    vpmovd2m %xmm0, %k0
+; AVX512DQVL-NEXT:    vpmovm2d %k0, %xmm0
+; AVX512DQVL-NEXT:    vzeroupper
+; AVX512DQVL-NEXT:    ret{{[l|q]}}
+  %ret = call <4 x i1> @llvm.experimental.constrained.fptoui.v4i1.v4f64(<4 x double> %a,
+                                              metadata !"fpexcept.strict")
+  ret <4 x i1> %ret
+}
+
+define <8 x i32> @strict_vector_fptosi_v8f32_to_v8i32(<8 x float> %a) #0 {
+; CHECK-LABEL: strict_vector_fptosi_v8f32_to_v8i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vcvttps2dq %ymm0, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %ret = call <8 x i32> @llvm.experimental.constrained.fptosi.v8i32.v8f32(<8 x float> %a,
+                                              metadata !"fpexcept.strict")
+  ret <8 x i32> %ret
+}
+
+define <8 x i32> @strict_vector_fptoui_v8f32_to_v8i32(<8 x float> %a) #0 {
+; AVX-32-LABEL: strict_vector_fptoui_v8f32_to_v8i32:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vmovaps {{.*#+}} ymm1 = [2.14748365E+9,2.14748365E+9,2.14748365E+9,2.14748365E+9,2.14748365E+9,2.14748365E+9,2.14748365E+9,2.14748365E+9]
+; AVX-32-NEXT:    vcmpltps %ymm1, %ymm0, %ymm2
+; AVX-32-NEXT:    vxorps %xmm3, %xmm3, %xmm3
+; AVX-32-NEXT:    vmovaps {{.*#+}} ymm4 = [2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648]
+; AVX-32-NEXT:    vblendvps %ymm2, %ymm3, %ymm4, %ymm4
+; AVX-32-NEXT:    vblendvps %ymm2, %ymm3, %ymm1, %ymm1
+; AVX-32-NEXT:    vsubps %ymm1, %ymm0, %ymm0
+; AVX-32-NEXT:    vcvttps2dq %ymm0, %ymm0
+; AVX-32-NEXT:    vxorps %ymm4, %ymm0, %ymm0
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: strict_vector_fptoui_v8f32_to_v8i32:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vmovaps {{.*#+}} ymm1 = [2.14748365E+9,2.14748365E+9,2.14748365E+9,2.14748365E+9,2.14748365E+9,2.14748365E+9,2.14748365E+9,2.14748365E+9]
+; AVX-64-NEXT:    vcmpltps %ymm1, %ymm0, %ymm2
+; AVX-64-NEXT:    vxorps %xmm3, %xmm3, %xmm3
+; AVX-64-NEXT:    vmovaps {{.*#+}} ymm4 = [2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648]
+; AVX-64-NEXT:    vblendvps %ymm2, %ymm3, %ymm4, %ymm4
+; AVX-64-NEXT:    vblendvps %ymm2, %ymm3, %ymm1, %ymm1
+; AVX-64-NEXT:    vsubps %ymm1, %ymm0, %ymm0
+; AVX-64-NEXT:    vcvttps2dq %ymm0, %ymm0
+; AVX-64-NEXT:    vxorps %ymm4, %ymm0, %ymm0
+; AVX-64-NEXT:    retq
+;
+; AVX512F-LABEL: strict_vector_fptoui_v8f32_to_v8i32:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vmovaps %ymm0, %ymm0
+; AVX512F-NEXT:    vcvttps2udq %zmm0, %zmm0
+; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512F-NEXT:    ret{{[l|q]}}
+;
+; AVX512VL-LABEL: strict_vector_fptoui_v8f32_to_v8i32:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vcvttps2udq %ymm0, %ymm0
+; AVX512VL-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQ-LABEL: strict_vector_fptoui_v8f32_to_v8i32:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vmovaps %ymm0, %ymm0
+; AVX512DQ-NEXT:    vcvttps2udq %zmm0, %zmm0
+; AVX512DQ-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQVL-LABEL: strict_vector_fptoui_v8f32_to_v8i32:
+; AVX512DQVL:       # %bb.0:
+; AVX512DQVL-NEXT:    vcvttps2udq %ymm0, %ymm0
+; AVX512DQVL-NEXT:    ret{{[l|q]}}
+  %ret = call <8 x i32> @llvm.experimental.constrained.fptoui.v8i32.v8f32(<8 x float> %a,
+                                              metadata !"fpexcept.strict")
+  ret <8 x i32> %ret
+}
+
+define <8 x i16> @strict_vector_fptosi_v8f32_to_v8i16(<8 x float> %a) #0 {
+; AVX-32-LABEL: strict_vector_fptosi_v8f32_to_v8i16:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vcvttps2dq %ymm0, %ymm0
+; AVX-32-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX-32-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
+; AVX-32-NEXT:    vzeroupper
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: strict_vector_fptosi_v8f32_to_v8i16:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcvttps2dq %ymm0, %ymm0
+; AVX-64-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX-64-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
+; AVX-64-NEXT:    vzeroupper
+; AVX-64-NEXT:    retq
+;
+; AVX512F-LABEL: strict_vector_fptosi_v8f32_to_v8i16:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vcvttps2dq %ymm0, %ymm0
+; AVX512F-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    ret{{[l|q]}}
+;
+; AVX512VL-LABEL: strict_vector_fptosi_v8f32_to_v8i16:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vcvttps2dq %ymm0, %ymm0
+; AVX512VL-NEXT:    vpmovdw %ymm0, %xmm0
+; AVX512VL-NEXT:    vzeroupper
+; AVX512VL-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQ-LABEL: strict_vector_fptosi_v8f32_to_v8i16:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vcvttps2dq %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQVL-LABEL: strict_vector_fptosi_v8f32_to_v8i16:
+; AVX512DQVL:       # %bb.0:
+; AVX512DQVL-NEXT:    vcvttps2dq %ymm0, %ymm0
+; AVX512DQVL-NEXT:    vpmovdw %ymm0, %xmm0
+; AVX512DQVL-NEXT:    vzeroupper
+; AVX512DQVL-NEXT:    ret{{[l|q]}}
+  %ret = call <8 x i16> @llvm.experimental.constrained.fptosi.v8i16.v8f32(<8 x float> %a,
+                                              metadata !"fpexcept.strict")
+  ret <8 x i16> %ret
+}
+
+define <8 x i16> @strict_vector_fptoui_v8f32_to_v8i16(<8 x float> %a) #0 {
+; AVX-32-LABEL: strict_vector_fptoui_v8f32_to_v8i16:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vcvttps2dq %ymm0, %ymm0
+; AVX-32-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX-32-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX-32-NEXT:    vzeroupper
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: strict_vector_fptoui_v8f32_to_v8i16:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcvttps2dq %ymm0, %ymm0
+; AVX-64-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX-64-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX-64-NEXT:    vzeroupper
+; AVX-64-NEXT:    retq
+;
+; AVX512F-LABEL: strict_vector_fptoui_v8f32_to_v8i16:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vcvttps2dq %ymm0, %ymm0
+; AVX512F-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    ret{{[l|q]}}
+;
+; AVX512VL-LABEL: strict_vector_fptoui_v8f32_to_v8i16:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vcvttps2dq %ymm0, %ymm0
+; AVX512VL-NEXT:    vpmovdw %ymm0, %xmm0
+; AVX512VL-NEXT:    vzeroupper
+; AVX512VL-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQ-LABEL: strict_vector_fptoui_v8f32_to_v8i16:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vcvttps2dq %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQVL-LABEL: strict_vector_fptoui_v8f32_to_v8i16:
+; AVX512DQVL:       # %bb.0:
+; AVX512DQVL-NEXT:    vcvttps2dq %ymm0, %ymm0
+; AVX512DQVL-NEXT:    vpmovdw %ymm0, %xmm0
+; AVX512DQVL-NEXT:    vzeroupper
+; AVX512DQVL-NEXT:    ret{{[l|q]}}
+  %ret = call <8 x i16> @llvm.experimental.constrained.fptoui.v8i16.v8f32(<8 x float> %a,
+                                              metadata !"fpexcept.strict")
+  ret <8 x i16> %ret
+}
+
+define <8 x i8> @strict_vector_fptosi_v8f32_to_v8i8(<8 x float> %a) #0 {
+; AVX-32-LABEL: strict_vector_fptosi_v8f32_to_v8i8:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vcvttps2dq %ymm0, %ymm0
+; AVX-32-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX-32-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
+; AVX-32-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
+; AVX-32-NEXT:    vzeroupper
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: strict_vector_fptosi_v8f32_to_v8i8:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcvttps2dq %ymm0, %ymm0
+; AVX-64-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX-64-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
+; AVX-64-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
+; AVX-64-NEXT:    vzeroupper
+; AVX-64-NEXT:    retq
+;
+; AVX512F-LABEL: strict_vector_fptosi_v8f32_to_v8i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vcvttps2dq %ymm0, %ymm0
+; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    ret{{[l|q]}}
+;
+; AVX512VL-LABEL: strict_vector_fptosi_v8f32_to_v8i8:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vcvttps2dq %ymm0, %ymm0
+; AVX512VL-NEXT:    vpmovdb %ymm0, %xmm0
+; AVX512VL-NEXT:    vzeroupper
+; AVX512VL-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQ-LABEL: strict_vector_fptosi_v8f32_to_v8i8:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vcvttps2dq %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQVL-LABEL: strict_vector_fptosi_v8f32_to_v8i8:
+; AVX512DQVL:       # %bb.0:
+; AVX512DQVL-NEXT:    vcvttps2dq %ymm0, %ymm0
+; AVX512DQVL-NEXT:    vpmovdb %ymm0, %xmm0
+; AVX512DQVL-NEXT:    vzeroupper
+; AVX512DQVL-NEXT:    ret{{[l|q]}}
+  %ret = call <8 x i8> @llvm.experimental.constrained.fptosi.v8i8.v8f32(<8 x float> %a,
+                                              metadata !"fpexcept.strict")
+  ret <8 x i8> %ret
+}
+
+define <8 x i8> @strict_vector_fptoui_v8f32_to_v8i8(<8 x float> %a) #0 {
+; AVX-32-LABEL: strict_vector_fptoui_v8f32_to_v8i8:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vcvttps2dq %ymm0, %ymm0
+; AVX-32-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX-32-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
+; AVX-32-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
+; AVX-32-NEXT:    vzeroupper
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: strict_vector_fptoui_v8f32_to_v8i8:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcvttps2dq %ymm0, %ymm0
+; AVX-64-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX-64-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
+; AVX-64-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
+; AVX-64-NEXT:    vzeroupper
+; AVX-64-NEXT:    retq
+;
+; AVX512F-LABEL: strict_vector_fptoui_v8f32_to_v8i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vcvttps2dq %ymm0, %ymm0
+; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    ret{{[l|q]}}
+;
+; AVX512VL-LABEL: strict_vector_fptoui_v8f32_to_v8i8:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vcvttps2dq %ymm0, %ymm0
+; AVX512VL-NEXT:    vpmovdb %ymm0, %xmm0
+; AVX512VL-NEXT:    vzeroupper
+; AVX512VL-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQ-LABEL: strict_vector_fptoui_v8f32_to_v8i8:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vcvttps2dq %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQVL-LABEL: strict_vector_fptoui_v8f32_to_v8i8:
+; AVX512DQVL:       # %bb.0:
+; AVX512DQVL-NEXT:    vcvttps2dq %ymm0, %ymm0
+; AVX512DQVL-NEXT:    vpmovdb %ymm0, %xmm0
+; AVX512DQVL-NEXT:    vzeroupper
+; AVX512DQVL-NEXT:    ret{{[l|q]}}
+  %ret = call <8 x i8> @llvm.experimental.constrained.fptoui.v8i8.v8f32(<8 x float> %a,
+                                              metadata !"fpexcept.strict")
+  ret <8 x i8> %ret
+}
+
+define <8 x i1> @strict_vector_fptosi_v8f32_to_v8i1(<8 x float> %a) #0 {
+; AVX-32-LABEL: strict_vector_fptosi_v8f32_to_v8i1:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vcvttps2dq %ymm0, %ymm0
+; AVX-32-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX-32-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
+; AVX-32-NEXT:    vzeroupper
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: strict_vector_fptosi_v8f32_to_v8i1:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcvttps2dq %ymm0, %ymm0
+; AVX-64-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX-64-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
+; AVX-64-NEXT:    vzeroupper
+; AVX-64-NEXT:    retq
+;
+; AVX512F-LABEL: strict_vector_fptosi_v8f32_to_v8i1:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vcvttps2dq %ymm0, %ymm0
+; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k1
+; AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    ret{{[l|q]}}
+;
+; AVX512VL-LABEL: strict_vector_fptosi_v8f32_to_v8i1:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vcvttps2dq %ymm0, %ymm0
+; AVX512VL-NEXT:    vptestmd %ymm0, %ymm0, %k1
+; AVX512VL-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
+; AVX512VL-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
+; AVX512VL-NEXT:    vpmovdw %ymm0, %xmm0
+; AVX512VL-NEXT:    vzeroupper
+; AVX512VL-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQ-LABEL: strict_vector_fptosi_v8f32_to_v8i1:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vcvttps2dq %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpmovd2m %zmm0, %k0
+; AVX512DQ-NEXT:    vpmovm2d %k0, %zmm0
+; AVX512DQ-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQVL-LABEL: strict_vector_fptosi_v8f32_to_v8i1:
+; AVX512DQVL:       # %bb.0:
+; AVX512DQVL-NEXT:    vcvttps2dq %ymm0, %ymm0
+; AVX512DQVL-NEXT:    vpmovd2m %ymm0, %k0
+; AVX512DQVL-NEXT:    vpmovm2d %k0, %ymm0
+; AVX512DQVL-NEXT:    vpmovdw %ymm0, %xmm0
+; AVX512DQVL-NEXT:    vzeroupper
+; AVX512DQVL-NEXT:    ret{{[l|q]}}
+  %ret = call <8 x i1> @llvm.experimental.constrained.fptosi.v8i1.v8f32(<8 x float> %a,
+                                              metadata !"fpexcept.strict")
+  ret <8 x i1> %ret
+}
+
+define <8 x i1> @strict_vector_fptoui_v8f32_to_v8i1(<8 x float> %a) #0 {
+; AVX-32-LABEL: strict_vector_fptoui_v8f32_to_v8i1:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vcvttps2dq %ymm0, %ymm0
+; AVX-32-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX-32-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX-32-NEXT:    vzeroupper
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: strict_vector_fptoui_v8f32_to_v8i1:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vcvttps2dq %ymm0, %ymm0
+; AVX-64-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX-64-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX-64-NEXT:    vzeroupper
+; AVX-64-NEXT:    retq
+;
+; AVX512F-LABEL: strict_vector_fptoui_v8f32_to_v8i1:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vcvttps2dq %ymm0, %ymm0
+; AVX512F-NEXT:    vpslld $31, %ymm0, %ymm0
+; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k1
+; AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    ret{{[l|q]}}
+;
+; AVX512VL-LABEL: strict_vector_fptoui_v8f32_to_v8i1:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vcvttps2dq %ymm0, %ymm0
+; AVX512VL-NEXT:    vpslld $31, %ymm0, %ymm0
+; AVX512VL-NEXT:    vptestmd %ymm0, %ymm0, %k1
+; AVX512VL-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
+; AVX512VL-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
+; AVX512VL-NEXT:    vpmovdw %ymm0, %xmm0
+; AVX512VL-NEXT:    vzeroupper
+; AVX512VL-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQ-LABEL: strict_vector_fptoui_v8f32_to_v8i1:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vcvttps2dq %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpslld $31, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpmovd2m %zmm0, %k0
+; AVX512DQ-NEXT:    vpmovm2d %k0, %zmm0
+; AVX512DQ-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQVL-LABEL: strict_vector_fptoui_v8f32_to_v8i1:
+; AVX512DQVL:       # %bb.0:
+; AVX512DQVL-NEXT:    vcvttps2dq %ymm0, %ymm0
+; AVX512DQVL-NEXT:    vpslld $31, %ymm0, %ymm0
+; AVX512DQVL-NEXT:    vpmovd2m %ymm0, %k0
+; AVX512DQVL-NEXT:    vpmovm2d %k0, %ymm0
+; AVX512DQVL-NEXT:    vpmovdw %ymm0, %xmm0
+; AVX512DQVL-NEXT:    vzeroupper
+; AVX512DQVL-NEXT:    ret{{[l|q]}}
+  %ret = call <8 x i1> @llvm.experimental.constrained.fptoui.v8i1.v8f32(<8 x float> %a,
+                                              metadata !"fpexcept.strict")
+  ret <8 x i1> %ret
+}
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/X86/vec-strict-fptoint-512.ll b/llvm/test/CodeGen/X86/vec-strict-fptoint-512.ll
new file mode 100644
index 000000000..3cc9edb83
--- /dev/null
+++ b/llvm/test/CodeGen/X86/vec-strict-fptoint-512.ll
@@ -0,0 +1,908 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx512f,avx512vl -O3 | FileCheck %s --check-prefixes=CHECK,AVX512VL,AVX512VL-32
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,avx512vl -O3 | FileCheck %s --check-prefixes=CHECK,AVX512VL,AVX512VL-64
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=avx512f,avx512dq -O3 | FileCheck %s --check-prefixes=CHECK,AVX512DQ,AVX512DQ-32
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512f,avx512dq -O3 | FileCheck %s --check-prefixes=CHECK,AVX512DQ,AVX512DQ-64
+
+declare <8 x i64> @llvm.experimental.constrained.fptosi.v8i64.v8f64(<8 x double>, metadata)
+declare <8 x i64> @llvm.experimental.constrained.fptoui.v8i64.v8f64(<8 x double>, metadata)
+declare <8 x i64> @llvm.experimental.constrained.fptosi.v8i64.v8f32(<8 x float>, metadata)
+declare <8 x i64> @llvm.experimental.constrained.fptoui.v8i64.v8f32(<8 x float>, metadata)
+declare <8 x i32> @llvm.experimental.constrained.fptosi.v8i32.v8f64(<8 x double>, metadata)
+declare <8 x i32> @llvm.experimental.constrained.fptoui.v8i32.v8f64(<8 x double>, metadata)
+declare <8 x i16> @llvm.experimental.constrained.fptosi.v8i16.v8f64(<8 x double>, metadata)
+declare <8 x i16> @llvm.experimental.constrained.fptoui.v8i16.v8f64(<8 x double>, metadata)
+declare <8 x i8> @llvm.experimental.constrained.fptosi.v8i8.v8f64(<8 x double>, metadata)
+declare <8 x i8> @llvm.experimental.constrained.fptoui.v8i8.v8f64(<8 x double>, metadata)
+declare <8 x i1> @llvm.experimental.constrained.fptosi.v8i1.v8f64(<8 x double>, metadata)
+declare <8 x i1> @llvm.experimental.constrained.fptoui.v8i1.v8f64(<8 x double>, metadata)
+declare <8 x i1> @llvm.experimental.constrained.fptosi.v8i1.v8f32(<8 x float>, metadata)
+declare <8 x i1> @llvm.experimental.constrained.fptoui.v8i1.v8f32(<8 x float>, metadata)
+
+declare <16 x i32> @llvm.experimental.constrained.fptosi.v16i32.v16f32(<16 x float>, metadata)
+declare <16 x i32> @llvm.experimental.constrained.fptoui.v16i32.v16f32(<16 x float>, metadata)
+declare <16 x i16> @llvm.experimental.constrained.fptosi.v16i16.v16f32(<16 x float>, metadata)
+declare <16 x i16> @llvm.experimental.constrained.fptoui.v16i16.v16f32(<16 x float>, metadata)
+declare <16 x i8> @llvm.experimental.constrained.fptosi.v16i8.v16f32(<16 x float>, metadata)
+declare <16 x i8> @llvm.experimental.constrained.fptoui.v16i8.v16f32(<16 x float>, metadata)
+declare <16 x i1> @llvm.experimental.constrained.fptosi.v16i1.v16f32(<16 x float>, metadata)
+declare <16 x i1> @llvm.experimental.constrained.fptoui.v16i1.v16f32(<16 x float>, metadata)
+
+define <8 x i64> @strict_vector_fptosi_v8f64_to_v8i64(<8 x double> %a) #0 {
+; AVX512VL-32-LABEL: strict_vector_fptosi_v8f64_to_v8i64:
+; AVX512VL-32:       # %bb.0:
+; AVX512VL-32-NEXT:    pushl %ebp
+; AVX512VL-32-NEXT:    .cfi_def_cfa_offset 8
+; AVX512VL-32-NEXT:    .cfi_offset %ebp, -8
+; AVX512VL-32-NEXT:    movl %esp, %ebp
+; AVX512VL-32-NEXT:    .cfi_def_cfa_register %ebp
+; AVX512VL-32-NEXT:    andl $-8, %esp
+; AVX512VL-32-NEXT:    subl $64, %esp
+; AVX512VL-32-NEXT:    vextractf32x4 $2, %zmm0, %xmm1
+; AVX512VL-32-NEXT:    vmovlps %xmm1, {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    vmovhps %xmm1, {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    vextractf32x4 $3, %zmm0, %xmm1
+; AVX512VL-32-NEXT:    vmovlps %xmm1, {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    vmovhps %xmm1, (%esp)
+; AVX512VL-32-NEXT:    vmovlps %xmm0, {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    vmovhps %xmm0, {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX512VL-32-NEXT:    vmovlps %xmm0, {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    vmovhps %xmm0, {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    fldl {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    fldl {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    fldl {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    fldl (%esp)
+; AVX512VL-32-NEXT:    fisttpll (%esp)
+; AVX512VL-32-NEXT:    fldl {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    fldl {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    fldl {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    fldl {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX512VL-32-NEXT:    vpinsrd $1, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512VL-32-NEXT:    vpinsrd $2, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512VL-32-NEXT:    vpinsrd $3, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512VL-32-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX512VL-32-NEXT:    vpinsrd $1, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512VL-32-NEXT:    vpinsrd $2, (%esp), %xmm1, %xmm1
+; AVX512VL-32-NEXT:    vpinsrd $3, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512VL-32-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512VL-32-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX512VL-32-NEXT:    vpinsrd $1, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512VL-32-NEXT:    vpinsrd $2, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512VL-32-NEXT:    vpinsrd $3, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512VL-32-NEXT:    vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; AVX512VL-32-NEXT:    vpinsrd $1, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX512VL-32-NEXT:    vpinsrd $2, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX512VL-32-NEXT:    vpinsrd $3, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX512VL-32-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512VL-32-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512VL-32-NEXT:    movl %ebp, %esp
+; AVX512VL-32-NEXT:    popl %ebp
+; AVX512VL-32-NEXT:    .cfi_def_cfa %esp, 4
+; AVX512VL-32-NEXT:    retl
+;
+; AVX512VL-64-LABEL: strict_vector_fptosi_v8f64_to_v8i64:
+; AVX512VL-64:       # %bb.0:
+; AVX512VL-64-NEXT:    vextractf32x4 $3, %zmm0, %xmm1
+; AVX512VL-64-NEXT:    vcvttsd2si %xmm1, %rax
+; AVX512VL-64-NEXT:    vmovq %rax, %xmm2
+; AVX512VL-64-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm1[1,0]
+; AVX512VL-64-NEXT:    vcvttsd2si %xmm1, %rax
+; AVX512VL-64-NEXT:    vmovq %rax, %xmm1
+; AVX512VL-64-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX512VL-64-NEXT:    vextractf32x4 $2, %zmm0, %xmm2
+; AVX512VL-64-NEXT:    vcvttsd2si %xmm2, %rax
+; AVX512VL-64-NEXT:    vmovq %rax, %xmm3
+; AVX512VL-64-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm2[1,0]
+; AVX512VL-64-NEXT:    vcvttsd2si %xmm2, %rax
+; AVX512VL-64-NEXT:    vmovq %rax, %xmm2
+; AVX512VL-64-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
+; AVX512VL-64-NEXT:    vinserti128 $1, %xmm1, %ymm2, %ymm1
+; AVX512VL-64-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX512VL-64-NEXT:    vcvttsd2si %xmm2, %rax
+; AVX512VL-64-NEXT:    vmovq %rax, %xmm3
+; AVX512VL-64-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm2[1,0]
+; AVX512VL-64-NEXT:    vcvttsd2si %xmm2, %rax
+; AVX512VL-64-NEXT:    vmovq %rax, %xmm2
+; AVX512VL-64-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
+; AVX512VL-64-NEXT:    vcvttsd2si %xmm0, %rax
+; AVX512VL-64-NEXT:    vmovq %rax, %xmm3
+; AVX512VL-64-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
+; AVX512VL-64-NEXT:    vcvttsd2si %xmm0, %rax
+; AVX512VL-64-NEXT:    vmovq %rax, %xmm0
+; AVX512VL-64-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm3[0],xmm0[0]
+; AVX512VL-64-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX512VL-64-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512VL-64-NEXT:    retq
+;
+; AVX512DQ-LABEL: strict_vector_fptosi_v8f64_to_v8i64:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vcvttpd2qq %zmm0, %zmm0
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+  %ret = call <8 x i64> @llvm.experimental.constrained.fptosi.v8i64.v8f64(<8 x double> %a,
+                                              metadata !"fpexcept.strict")
+  ret <8 x i64> %ret
+}
+
+define <8 x i64> @strict_vector_fptoui_v8f64_to_v8i64(<8 x double> %a) #0 {
+; AVX512VL-32-LABEL: strict_vector_fptoui_v8f64_to_v8i64:
+; AVX512VL-32:       # %bb.0:
+; AVX512VL-32-NEXT:    pushl %ebp
+; AVX512VL-32-NEXT:    .cfi_def_cfa_offset 8
+; AVX512VL-32-NEXT:    .cfi_offset %ebp, -8
+; AVX512VL-32-NEXT:    movl %esp, %ebp
+; AVX512VL-32-NEXT:    .cfi_def_cfa_register %ebp
+; AVX512VL-32-NEXT:    pushl %ebx
+; AVX512VL-32-NEXT:    pushl %edi
+; AVX512VL-32-NEXT:    pushl %esi
+; AVX512VL-32-NEXT:    andl $-8, %esp
+; AVX512VL-32-NEXT:    subl $80, %esp
+; AVX512VL-32-NEXT:    .cfi_offset %esi, -20
+; AVX512VL-32-NEXT:    .cfi_offset %edi, -16
+; AVX512VL-32-NEXT:    .cfi_offset %ebx, -12
+; AVX512VL-32-NEXT:    vpermilpd {{.*#+}} xmm3 = xmm0[1,0]
+; AVX512VL-32-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
+; AVX512VL-32-NEXT:    vcomisd %xmm1, %xmm3
+; AVX512VL-32-NEXT:    setb %al
+; AVX512VL-32-NEXT:    kmovw %eax, %k1
+; AVX512VL-32-NEXT:    vxorpd %xmm2, %xmm2, %xmm2
+; AVX512VL-32-NEXT:    vmovapd %xmm1, %xmm4
+; AVX512VL-32-NEXT:    vmovsd %xmm2, %xmm4, %xmm4 {%k1}
+; AVX512VL-32-NEXT:    vsubsd %xmm4, %xmm3, %xmm3
+; AVX512VL-32-NEXT:    vmovsd %xmm3, {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    fldl {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    movl $0, %eax
+; AVX512VL-32-NEXT:    setae %al
+; AVX512VL-32-NEXT:    shll $31, %eax
+; AVX512VL-32-NEXT:    xorl {{[0-9]+}}(%esp), %eax
+; AVX512VL-32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; AVX512VL-32-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; AVX512VL-32-NEXT:    vpermilpd {{.*#+}} xmm4 = xmm3[1,0]
+; AVX512VL-32-NEXT:    vcomisd %xmm1, %xmm4
+; AVX512VL-32-NEXT:    setb %al
+; AVX512VL-32-NEXT:    kmovw %eax, %k1
+; AVX512VL-32-NEXT:    vmovapd %xmm1, %xmm5
+; AVX512VL-32-NEXT:    vmovsd %xmm2, %xmm5, %xmm5 {%k1}
+; AVX512VL-32-NEXT:    vsubsd %xmm5, %xmm4, %xmm4
+; AVX512VL-32-NEXT:    vmovsd %xmm4, {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    fldl {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    movl $0, %eax
+; AVX512VL-32-NEXT:    setae %al
+; AVX512VL-32-NEXT:    shll $31, %eax
+; AVX512VL-32-NEXT:    xorl {{[0-9]+}}(%esp), %eax
+; AVX512VL-32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; AVX512VL-32-NEXT:    vcomisd %xmm1, %xmm3
+; AVX512VL-32-NEXT:    setb %al
+; AVX512VL-32-NEXT:    kmovw %eax, %k1
+; AVX512VL-32-NEXT:    vmovapd %xmm1, %xmm4
+; AVX512VL-32-NEXT:    vmovsd %xmm2, %xmm4, %xmm4 {%k1}
+; AVX512VL-32-NEXT:    vsubsd %xmm4, %xmm3, %xmm3
+; AVX512VL-32-NEXT:    vmovsd %xmm3, {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    fldl {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    movl $0, %eax
+; AVX512VL-32-NEXT:    setae %al
+; AVX512VL-32-NEXT:    shll $31, %eax
+; AVX512VL-32-NEXT:    xorl {{[0-9]+}}(%esp), %eax
+; AVX512VL-32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; AVX512VL-32-NEXT:    vextractf32x4 $2, %zmm0, %xmm3
+; AVX512VL-32-NEXT:    vpermilpd {{.*#+}} xmm4 = xmm3[1,0]
+; AVX512VL-32-NEXT:    vcomisd %xmm1, %xmm4
+; AVX512VL-32-NEXT:    setb %al
+; AVX512VL-32-NEXT:    kmovw %eax, %k1
+; AVX512VL-32-NEXT:    vmovapd %xmm1, %xmm5
+; AVX512VL-32-NEXT:    vmovsd %xmm2, %xmm5, %xmm5 {%k1}
+; AVX512VL-32-NEXT:    vsubsd %xmm5, %xmm4, %xmm4
+; AVX512VL-32-NEXT:    vmovsd %xmm4, {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    fldl {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    movl $0, %eax
+; AVX512VL-32-NEXT:    setae %al
+; AVX512VL-32-NEXT:    shll $31, %eax
+; AVX512VL-32-NEXT:    xorl {{[0-9]+}}(%esp), %eax
+; AVX512VL-32-NEXT:    movl %eax, %edi
+; AVX512VL-32-NEXT:    vcomisd %xmm1, %xmm3
+; AVX512VL-32-NEXT:    setb %al
+; AVX512VL-32-NEXT:    kmovw %eax, %k1
+; AVX512VL-32-NEXT:    vmovapd %xmm1, %xmm4
+; AVX512VL-32-NEXT:    vmovsd %xmm2, %xmm4, %xmm4 {%k1}
+; AVX512VL-32-NEXT:    vsubsd %xmm4, %xmm3, %xmm3
+; AVX512VL-32-NEXT:    vmovsd %xmm3, {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    fldl {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    movl $0, %eax
+; AVX512VL-32-NEXT:    setae %al
+; AVX512VL-32-NEXT:    shll $31, %eax
+; AVX512VL-32-NEXT:    xorl {{[0-9]+}}(%esp), %eax
+; AVX512VL-32-NEXT:    movl %eax, %esi
+; AVX512VL-32-NEXT:    vextractf32x4 $3, %zmm0, %xmm3
+; AVX512VL-32-NEXT:    vpermilpd {{.*#+}} xmm4 = xmm3[1,0]
+; AVX512VL-32-NEXT:    xorl %edx, %edx
+; AVX512VL-32-NEXT:    vcomisd %xmm1, %xmm4
+; AVX512VL-32-NEXT:    setb %al
+; AVX512VL-32-NEXT:    kmovw %eax, %k1
+; AVX512VL-32-NEXT:    vmovapd %xmm1, %xmm5
+; AVX512VL-32-NEXT:    vmovsd %xmm2, %xmm5, %xmm5 {%k1}
+; AVX512VL-32-NEXT:    vsubsd %xmm5, %xmm4, %xmm4
+; AVX512VL-32-NEXT:    vmovsd %xmm4, (%esp)
+; AVX512VL-32-NEXT:    fldl (%esp)
+; AVX512VL-32-NEXT:    fisttpll (%esp)
+; AVX512VL-32-NEXT:    setae %dl
+; AVX512VL-32-NEXT:    shll $31, %edx
+; AVX512VL-32-NEXT:    xorl {{[0-9]+}}(%esp), %edx
+; AVX512VL-32-NEXT:    xorl %eax, %eax
+; AVX512VL-32-NEXT:    vcomisd %xmm1, %xmm3
+; AVX512VL-32-NEXT:    setb %cl
+; AVX512VL-32-NEXT:    kmovw %ecx, %k1
+; AVX512VL-32-NEXT:    vmovapd %xmm1, %xmm4
+; AVX512VL-32-NEXT:    vmovsd %xmm2, %xmm4, %xmm4 {%k1}
+; AVX512VL-32-NEXT:    vsubsd %xmm4, %xmm3, %xmm3
+; AVX512VL-32-NEXT:    vmovsd %xmm3, {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    fldl {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    setae %al
+; AVX512VL-32-NEXT:    shll $31, %eax
+; AVX512VL-32-NEXT:    xorl {{[0-9]+}}(%esp), %eax
+; AVX512VL-32-NEXT:    xorl %ecx, %ecx
+; AVX512VL-32-NEXT:    vcomisd %xmm1, %xmm0
+; AVX512VL-32-NEXT:    setb %bl
+; AVX512VL-32-NEXT:    kmovw %ebx, %k1
+; AVX512VL-32-NEXT:    vmovsd %xmm2, %xmm1, %xmm1 {%k1}
+; AVX512VL-32-NEXT:    vsubsd %xmm1, %xmm0, %xmm0
+; AVX512VL-32-NEXT:    vmovsd %xmm0, {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    fldl {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX512VL-32-NEXT:    vpinsrd $1, %eax, %xmm0, %xmm0
+; AVX512VL-32-NEXT:    vpinsrd $2, (%esp), %xmm0, %xmm0
+; AVX512VL-32-NEXT:    vpinsrd $3, %edx, %xmm0, %xmm0
+; AVX512VL-32-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX512VL-32-NEXT:    vpinsrd $1, %esi, %xmm1, %xmm1
+; AVX512VL-32-NEXT:    vpinsrd $2, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512VL-32-NEXT:    vpinsrd $3, %edi, %xmm1, %xmm1
+; AVX512VL-32-NEXT:    vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; AVX512VL-32-NEXT:    vpinsrd $1, {{[-0-9]+}}(%e{{[sb]}}p), %xmm2, %xmm2 # 4-byte Folded Reload
+; AVX512VL-32-NEXT:    vpinsrd $2, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX512VL-32-NEXT:    vpinsrd $3, {{[-0-9]+}}(%e{{[sb]}}p), %xmm2, %xmm2 # 4-byte Folded Reload
+; AVX512VL-32-NEXT:    setae %cl
+; AVX512VL-32-NEXT:    shll $31, %ecx
+; AVX512VL-32-NEXT:    xorl {{[0-9]+}}(%esp), %ecx
+; AVX512VL-32-NEXT:    vmovd {{.*#+}} xmm3 = mem[0],zero,zero,zero
+; AVX512VL-32-NEXT:    vpinsrd $1, %ecx, %xmm3, %xmm3
+; AVX512VL-32-NEXT:    vpinsrd $2, {{[0-9]+}}(%esp), %xmm3, %xmm3
+; AVX512VL-32-NEXT:    vpinsrd $3, {{[-0-9]+}}(%e{{[sb]}}p), %xmm3, %xmm3 # 4-byte Folded Reload
+; AVX512VL-32-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX512VL-32-NEXT:    vinserti128 $1, %xmm2, %ymm3, %ymm1
+; AVX512VL-32-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512VL-32-NEXT:    leal -12(%ebp), %esp
+; AVX512VL-32-NEXT:    popl %esi
+; AVX512VL-32-NEXT:    popl %edi
+; AVX512VL-32-NEXT:    popl %ebx
+; AVX512VL-32-NEXT:    popl %ebp
+; AVX512VL-32-NEXT:    .cfi_def_cfa %esp, 4
+; AVX512VL-32-NEXT:    retl
+;
+; AVX512VL-64-LABEL: strict_vector_fptoui_v8f64_to_v8i64:
+; AVX512VL-64:       # %bb.0:
+; AVX512VL-64-NEXT:    vextractf32x4 $3, %zmm0, %xmm1
+; AVX512VL-64-NEXT:    vcvttsd2usi %xmm1, %rax
+; AVX512VL-64-NEXT:    vmovq %rax, %xmm2
+; AVX512VL-64-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm1[1,0]
+; AVX512VL-64-NEXT:    vcvttsd2usi %xmm1, %rax
+; AVX512VL-64-NEXT:    vmovq %rax, %xmm1
+; AVX512VL-64-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX512VL-64-NEXT:    vextractf32x4 $2, %zmm0, %xmm2
+; AVX512VL-64-NEXT:    vcvttsd2usi %xmm2, %rax
+; AVX512VL-64-NEXT:    vmovq %rax, %xmm3
+; AVX512VL-64-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm2[1,0]
+; AVX512VL-64-NEXT:    vcvttsd2usi %xmm2, %rax
+; AVX512VL-64-NEXT:    vmovq %rax, %xmm2
+; AVX512VL-64-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
+; AVX512VL-64-NEXT:    vinserti128 $1, %xmm1, %ymm2, %ymm1
+; AVX512VL-64-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX512VL-64-NEXT:    vcvttsd2usi %xmm2, %rax
+; AVX512VL-64-NEXT:    vmovq %rax, %xmm3
+; AVX512VL-64-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm2[1,0]
+; AVX512VL-64-NEXT:    vcvttsd2usi %xmm2, %rax
+; AVX512VL-64-NEXT:    vmovq %rax, %xmm2
+; AVX512VL-64-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
+; AVX512VL-64-NEXT:    vcvttsd2usi %xmm0, %rax
+; AVX512VL-64-NEXT:    vmovq %rax, %xmm3
+; AVX512VL-64-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
+; AVX512VL-64-NEXT:    vcvttsd2usi %xmm0, %rax
+; AVX512VL-64-NEXT:    vmovq %rax, %xmm0
+; AVX512VL-64-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm3[0],xmm0[0]
+; AVX512VL-64-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX512VL-64-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512VL-64-NEXT:    retq
+;
+; AVX512DQ-LABEL: strict_vector_fptoui_v8f64_to_v8i64:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vcvttpd2uqq %zmm0, %zmm0
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+  %ret = call <8 x i64> @llvm.experimental.constrained.fptoui.v8i64.v8f64(<8 x double> %a,
+                                              metadata !"fpexcept.strict")
+  ret <8 x i64> %ret
+}
+
+define <8 x i64> @strict_vector_fptosi_v8f32_to_v8i64(<8 x float> %a) #0 {
+; AVX512VL-32-LABEL: strict_vector_fptosi_v8f32_to_v8i64:
+; AVX512VL-32:       # %bb.0:
+; AVX512VL-32-NEXT:    pushl %ebp
+; AVX512VL-32-NEXT:    .cfi_def_cfa_offset 8
+; AVX512VL-32-NEXT:    .cfi_offset %ebp, -8
+; AVX512VL-32-NEXT:    movl %esp, %ebp
+; AVX512VL-32-NEXT:    .cfi_def_cfa_register %ebp
+; AVX512VL-32-NEXT:    andl $-8, %esp
+; AVX512VL-32-NEXT:    subl $64, %esp
+; AVX512VL-32-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512VL-32-NEXT:    vmovd %xmm1, {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    vextractps $1, %xmm1, {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    vextractps $2, %xmm1, {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    vextractps $3, %xmm1, (%esp)
+; AVX512VL-32-NEXT:    vmovd %xmm0, {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    vextractps $1, %xmm0, {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    vextractps $2, %xmm0, {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    vextractps $3, %xmm0, {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    flds {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    flds {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    flds {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    flds (%esp)
+; AVX512VL-32-NEXT:    fisttpll (%esp)
+; AVX512VL-32-NEXT:    flds {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    flds {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    flds {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    flds {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX512VL-32-NEXT:    vpinsrd $1, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512VL-32-NEXT:    vpinsrd $2, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512VL-32-NEXT:    vpinsrd $3, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512VL-32-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX512VL-32-NEXT:    vpinsrd $1, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512VL-32-NEXT:    vpinsrd $2, (%esp), %xmm1, %xmm1
+; AVX512VL-32-NEXT:    vpinsrd $3, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512VL-32-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512VL-32-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX512VL-32-NEXT:    vpinsrd $1, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512VL-32-NEXT:    vpinsrd $2, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512VL-32-NEXT:    vpinsrd $3, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512VL-32-NEXT:    vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; AVX512VL-32-NEXT:    vpinsrd $1, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX512VL-32-NEXT:    vpinsrd $2, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX512VL-32-NEXT:    vpinsrd $3, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX512VL-32-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512VL-32-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512VL-32-NEXT:    movl %ebp, %esp
+; AVX512VL-32-NEXT:    popl %ebp
+; AVX512VL-32-NEXT:    .cfi_def_cfa %esp, 4
+; AVX512VL-32-NEXT:    retl
+;
+; AVX512VL-64-LABEL: strict_vector_fptosi_v8f32_to_v8i64:
+; AVX512VL-64:       # %bb.0:
+; AVX512VL-64-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX512VL-64-NEXT:    vpermilps {{.*#+}} xmm2 = xmm1[3,1,2,3]
+; AVX512VL-64-NEXT:    vcvttss2si %xmm2, %rax
+; AVX512VL-64-NEXT:    vmovq %rax, %xmm2
+; AVX512VL-64-NEXT:    vpermilpd {{.*#+}} xmm3 = xmm1[1,0]
+; AVX512VL-64-NEXT:    vcvttss2si %xmm3, %rax
+; AVX512VL-64-NEXT:    vmovq %rax, %xmm3
+; AVX512VL-64-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
+; AVX512VL-64-NEXT:    vcvttss2si %xmm1, %rax
+; AVX512VL-64-NEXT:    vmovq %rax, %xmm3
+; AVX512VL-64-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; AVX512VL-64-NEXT:    vcvttss2si %xmm1, %rax
+; AVX512VL-64-NEXT:    vmovq %rax, %xmm1
+; AVX512VL-64-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm3[0],xmm1[0]
+; AVX512VL-64-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512VL-64-NEXT:    vpermilps {{.*#+}} xmm2 = xmm0[3,1,2,3]
+; AVX512VL-64-NEXT:    vcvttss2si %xmm2, %rax
+; AVX512VL-64-NEXT:    vmovq %rax, %xmm2
+; AVX512VL-64-NEXT:    vpermilpd {{.*#+}} xmm3 = xmm0[1,0]
+; AVX512VL-64-NEXT:    vcvttss2si %xmm3, %rax
+; AVX512VL-64-NEXT:    vmovq %rax, %xmm3
+; AVX512VL-64-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
+; AVX512VL-64-NEXT:    vcvttss2si %xmm0, %rax
+; AVX512VL-64-NEXT:    vmovq %rax, %xmm3
+; AVX512VL-64-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; AVX512VL-64-NEXT:    vcvttss2si %xmm0, %rax
+; AVX512VL-64-NEXT:    vmovq %rax, %xmm0
+; AVX512VL-64-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm3[0],xmm0[0]
+; AVX512VL-64-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX512VL-64-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512VL-64-NEXT:    retq
+;
+; AVX512DQ-LABEL: strict_vector_fptosi_v8f32_to_v8i64:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vcvttps2qq %ymm0, %zmm0
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+  %ret = call <8 x i64> @llvm.experimental.constrained.fptosi.v8i64.v8f32(<8 x float> %a,
+                                              metadata !"fpexcept.strict")
+  ret <8 x i64> %ret
+}
+
+define <8 x i64> @strict_vector_fptoui_v8f32_to_v8i64(<8 x float> %a) #0 {
+; AVX512VL-32-LABEL: strict_vector_fptoui_v8f32_to_v8i64:
+; AVX512VL-32:       # %bb.0:
+; AVX512VL-32-NEXT:    pushl %ebp
+; AVX512VL-32-NEXT:    .cfi_def_cfa_offset 8
+; AVX512VL-32-NEXT:    .cfi_offset %ebp, -8
+; AVX512VL-32-NEXT:    movl %esp, %ebp
+; AVX512VL-32-NEXT:    .cfi_def_cfa_register %ebp
+; AVX512VL-32-NEXT:    pushl %ebx
+; AVX512VL-32-NEXT:    pushl %edi
+; AVX512VL-32-NEXT:    pushl %esi
+; AVX512VL-32-NEXT:    andl $-8, %esp
+; AVX512VL-32-NEXT:    subl $80, %esp
+; AVX512VL-32-NEXT:    .cfi_offset %esi, -20
+; AVX512VL-32-NEXT:    .cfi_offset %edi, -16
+; AVX512VL-32-NEXT:    .cfi_offset %ebx, -12
+; AVX512VL-32-NEXT:    vmovshdup {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; AVX512VL-32-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX512VL-32-NEXT:    vcomiss %xmm1, %xmm3
+; AVX512VL-32-NEXT:    setb %al
+; AVX512VL-32-NEXT:    kmovw %eax, %k1
+; AVX512VL-32-NEXT:    vxorps %xmm2, %xmm2, %xmm2
+; AVX512VL-32-NEXT:    vmovaps %xmm1, %xmm4
+; AVX512VL-32-NEXT:    vmovss %xmm2, %xmm4, %xmm4 {%k1}
+; AVX512VL-32-NEXT:    vsubss %xmm4, %xmm3, %xmm3
+; AVX512VL-32-NEXT:    vmovss %xmm3, {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    flds {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    movl $0, %eax
+; AVX512VL-32-NEXT:    setae %al
+; AVX512VL-32-NEXT:    shll $31, %eax
+; AVX512VL-32-NEXT:    xorl {{[0-9]+}}(%esp), %eax
+; AVX512VL-32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; AVX512VL-32-NEXT:    vpermilps {{.*#+}} xmm3 = xmm0[3,1,2,3]
+; AVX512VL-32-NEXT:    vcomiss %xmm1, %xmm3
+; AVX512VL-32-NEXT:    setb %al
+; AVX512VL-32-NEXT:    kmovw %eax, %k1
+; AVX512VL-32-NEXT:    vmovaps %xmm1, %xmm4
+; AVX512VL-32-NEXT:    vmovss %xmm2, %xmm4, %xmm4 {%k1}
+; AVX512VL-32-NEXT:    vsubss %xmm4, %xmm3, %xmm3
+; AVX512VL-32-NEXT:    vmovss %xmm3, {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    flds {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    movl $0, %eax
+; AVX512VL-32-NEXT:    setae %al
+; AVX512VL-32-NEXT:    shll $31, %eax
+; AVX512VL-32-NEXT:    xorl {{[0-9]+}}(%esp), %eax
+; AVX512VL-32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; AVX512VL-32-NEXT:    vpermilpd {{.*#+}} xmm3 = xmm0[1,0]
+; AVX512VL-32-NEXT:    vcomiss %xmm1, %xmm3
+; AVX512VL-32-NEXT:    setb %al
+; AVX512VL-32-NEXT:    kmovw %eax, %k1
+; AVX512VL-32-NEXT:    vmovaps %xmm1, %xmm4
+; AVX512VL-32-NEXT:    vmovss %xmm2, %xmm4, %xmm4 {%k1}
+; AVX512VL-32-NEXT:    vsubss %xmm4, %xmm3, %xmm3
+; AVX512VL-32-NEXT:    vmovss %xmm3, {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    flds {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    movl $0, %eax
+; AVX512VL-32-NEXT:    setae %al
+; AVX512VL-32-NEXT:    shll $31, %eax
+; AVX512VL-32-NEXT:    xorl {{[0-9]+}}(%esp), %eax
+; AVX512VL-32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; AVX512VL-32-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; AVX512VL-32-NEXT:    vmovshdup {{.*#+}} xmm4 = xmm3[1,1,3,3]
+; AVX512VL-32-NEXT:    vcomiss %xmm1, %xmm4
+; AVX512VL-32-NEXT:    setb %al
+; AVX512VL-32-NEXT:    kmovw %eax, %k1
+; AVX512VL-32-NEXT:    vmovaps %xmm1, %xmm5
+; AVX512VL-32-NEXT:    vmovss %xmm2, %xmm5, %xmm5 {%k1}
+; AVX512VL-32-NEXT:    vsubss %xmm5, %xmm4, %xmm4
+; AVX512VL-32-NEXT:    vmovss %xmm4, {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    flds {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    movl $0, %eax
+; AVX512VL-32-NEXT:    setae %al
+; AVX512VL-32-NEXT:    shll $31, %eax
+; AVX512VL-32-NEXT:    xorl {{[0-9]+}}(%esp), %eax
+; AVX512VL-32-NEXT:    movl %eax, %edi
+; AVX512VL-32-NEXT:    vcomiss %xmm1, %xmm3
+; AVX512VL-32-NEXT:    setb %al
+; AVX512VL-32-NEXT:    kmovw %eax, %k1
+; AVX512VL-32-NEXT:    vmovaps %xmm1, %xmm4
+; AVX512VL-32-NEXT:    vmovss %xmm2, %xmm4, %xmm4 {%k1}
+; AVX512VL-32-NEXT:    vsubss %xmm4, %xmm3, %xmm4
+; AVX512VL-32-NEXT:    vmovss %xmm4, {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    flds {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    movl $0, %eax
+; AVX512VL-32-NEXT:    setae %al
+; AVX512VL-32-NEXT:    shll $31, %eax
+; AVX512VL-32-NEXT:    xorl {{[0-9]+}}(%esp), %eax
+; AVX512VL-32-NEXT:    movl %eax, %esi
+; AVX512VL-32-NEXT:    vpermilps {{.*#+}} xmm4 = xmm3[3,1,2,3]
+; AVX512VL-32-NEXT:    xorl %edx, %edx
+; AVX512VL-32-NEXT:    vcomiss %xmm1, %xmm4
+; AVX512VL-32-NEXT:    setb %al
+; AVX512VL-32-NEXT:    kmovw %eax, %k1
+; AVX512VL-32-NEXT:    vmovaps %xmm1, %xmm5
+; AVX512VL-32-NEXT:    vmovss %xmm2, %xmm5, %xmm5 {%k1}
+; AVX512VL-32-NEXT:    vsubss %xmm5, %xmm4, %xmm4
+; AVX512VL-32-NEXT:    vmovss %xmm4, (%esp)
+; AVX512VL-32-NEXT:    flds (%esp)
+; AVX512VL-32-NEXT:    fisttpll (%esp)
+; AVX512VL-32-NEXT:    setae %dl
+; AVX512VL-32-NEXT:    shll $31, %edx
+; AVX512VL-32-NEXT:    xorl {{[0-9]+}}(%esp), %edx
+; AVX512VL-32-NEXT:    vpermilpd {{.*#+}} xmm3 = xmm3[1,0]
+; AVX512VL-32-NEXT:    xorl %eax, %eax
+; AVX512VL-32-NEXT:    vcomiss %xmm1, %xmm3
+; AVX512VL-32-NEXT:    setb %cl
+; AVX512VL-32-NEXT:    kmovw %ecx, %k1
+; AVX512VL-32-NEXT:    vmovaps %xmm1, %xmm4
+; AVX512VL-32-NEXT:    vmovss %xmm2, %xmm4, %xmm4 {%k1}
+; AVX512VL-32-NEXT:    vsubss %xmm4, %xmm3, %xmm3
+; AVX512VL-32-NEXT:    vmovss %xmm3, {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    flds {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    setae %al
+; AVX512VL-32-NEXT:    shll $31, %eax
+; AVX512VL-32-NEXT:    xorl {{[0-9]+}}(%esp), %eax
+; AVX512VL-32-NEXT:    xorl %ecx, %ecx
+; AVX512VL-32-NEXT:    vcomiss %xmm1, %xmm0
+; AVX512VL-32-NEXT:    setb %bl
+; AVX512VL-32-NEXT:    kmovw %ebx, %k1
+; AVX512VL-32-NEXT:    vmovss %xmm2, %xmm1, %xmm1 {%k1}
+; AVX512VL-32-NEXT:    vsubss %xmm1, %xmm0, %xmm0
+; AVX512VL-32-NEXT:    vmovss %xmm0, {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    flds {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    fisttpll {{[0-9]+}}(%esp)
+; AVX512VL-32-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX512VL-32-NEXT:    vpinsrd $1, %eax, %xmm0, %xmm0
+; AVX512VL-32-NEXT:    vpinsrd $2, (%esp), %xmm0, %xmm0
+; AVX512VL-32-NEXT:    vpinsrd $3, %edx, %xmm0, %xmm0
+; AVX512VL-32-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX512VL-32-NEXT:    vpinsrd $1, %esi, %xmm1, %xmm1
+; AVX512VL-32-NEXT:    vpinsrd $2, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512VL-32-NEXT:    vpinsrd $3, %edi, %xmm1, %xmm1
+; AVX512VL-32-NEXT:    vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; AVX512VL-32-NEXT:    vpinsrd $1, {{[-0-9]+}}(%e{{[sb]}}p), %xmm2, %xmm2 # 4-byte Folded Reload
+; AVX512VL-32-NEXT:    vpinsrd $2, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX512VL-32-NEXT:    vpinsrd $3, {{[-0-9]+}}(%e{{[sb]}}p), %xmm2, %xmm2 # 4-byte Folded Reload
+; AVX512VL-32-NEXT:    setae %cl
+; AVX512VL-32-NEXT:    shll $31, %ecx
+; AVX512VL-32-NEXT:    xorl {{[0-9]+}}(%esp), %ecx
+; AVX512VL-32-NEXT:    vmovd {{.*#+}} xmm3 = mem[0],zero,zero,zero
+; AVX512VL-32-NEXT:    vpinsrd $1, %ecx, %xmm3, %xmm3
+; AVX512VL-32-NEXT:    vpinsrd $2, {{[0-9]+}}(%esp), %xmm3, %xmm3
+; AVX512VL-32-NEXT:    vpinsrd $3, {{[-0-9]+}}(%e{{[sb]}}p), %xmm3, %xmm3 # 4-byte Folded Reload
+; AVX512VL-32-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX512VL-32-NEXT:    vinserti128 $1, %xmm2, %ymm3, %ymm1
+; AVX512VL-32-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512VL-32-NEXT:    leal -12(%ebp), %esp
+; AVX512VL-32-NEXT:    popl %esi
+; AVX512VL-32-NEXT:    popl %edi
+; AVX512VL-32-NEXT:    popl %ebx
+; AVX512VL-32-NEXT:    popl %ebp
+; AVX512VL-32-NEXT:    .cfi_def_cfa %esp, 4
+; AVX512VL-32-NEXT:    retl
+;
+; AVX512VL-64-LABEL: strict_vector_fptoui_v8f32_to_v8i64:
+; AVX512VL-64:       # %bb.0:
+; AVX512VL-64-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX512VL-64-NEXT:    vpermilps {{.*#+}} xmm2 = xmm1[3,1,2,3]
+; AVX512VL-64-NEXT:    vcvttss2usi %xmm2, %rax
+; AVX512VL-64-NEXT:    vmovq %rax, %xmm2
+; AVX512VL-64-NEXT:    vpermilpd {{.*#+}} xmm3 = xmm1[1,0]
+; AVX512VL-64-NEXT:    vcvttss2usi %xmm3, %rax
+; AVX512VL-64-NEXT:    vmovq %rax, %xmm3
+; AVX512VL-64-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
+; AVX512VL-64-NEXT:    vcvttss2usi %xmm1, %rax
+; AVX512VL-64-NEXT:    vmovq %rax, %xmm3
+; AVX512VL-64-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; AVX512VL-64-NEXT:    vcvttss2usi %xmm1, %rax
+; AVX512VL-64-NEXT:    vmovq %rax, %xmm1
+; AVX512VL-64-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm3[0],xmm1[0]
+; AVX512VL-64-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512VL-64-NEXT:    vpermilps {{.*#+}} xmm2 = xmm0[3,1,2,3]
+; AVX512VL-64-NEXT:    vcvttss2usi %xmm2, %rax
+; AVX512VL-64-NEXT:    vmovq %rax, %xmm2
+; AVX512VL-64-NEXT:    vpermilpd {{.*#+}} xmm3 = xmm0[1,0]
+; AVX512VL-64-NEXT:    vcvttss2usi %xmm3, %rax
+; AVX512VL-64-NEXT:    vmovq %rax, %xmm3
+; AVX512VL-64-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
+; AVX512VL-64-NEXT:    vcvttss2usi %xmm0, %rax
+; AVX512VL-64-NEXT:    vmovq %rax, %xmm3
+; AVX512VL-64-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; AVX512VL-64-NEXT:    vcvttss2usi %xmm0, %rax
+; AVX512VL-64-NEXT:    vmovq %rax, %xmm0
+; AVX512VL-64-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm3[0],xmm0[0]
+; AVX512VL-64-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX512VL-64-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512VL-64-NEXT:    retq
+;
+; AVX512DQ-LABEL: strict_vector_fptoui_v8f32_to_v8i64:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vcvttps2uqq %ymm0, %zmm0
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+  %ret = call <8 x i64> @llvm.experimental.constrained.fptoui.v8i64.v8f32(<8 x float> %a,
+                                              metadata !"fpexcept.strict")
+  ret <8 x i64> %ret
+}
+
+define <8 x i32> @strict_vector_fptosi_v8f64_to_v8i32(<8 x double> %a) #0 {
+; CHECK-LABEL: strict_vector_fptosi_v8f64_to_v8i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vcvttpd2dq %zmm0, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %ret = call <8 x i32> @llvm.experimental.constrained.fptosi.v8i32.v8f64(<8 x double> %a,
+                                              metadata !"fpexcept.strict")
+  ret <8 x i32> %ret
+}
+
+define <8 x i32> @strict_vector_fptoui_v8f64_to_v8i32(<8 x double> %a) #0 {
+; CHECK-LABEL: strict_vector_fptoui_v8f64_to_v8i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vcvttpd2udq %zmm0, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %ret = call <8 x i32> @llvm.experimental.constrained.fptoui.v8i32.v8f64(<8 x double> %a,
+                                              metadata !"fpexcept.strict")
+  ret <8 x i32> %ret
+}
+
+define <8 x i16> @strict_vector_fptosi_v8f64_to_v8i16(<8 x double> %a) #0 {
+; AVX512VL-LABEL: strict_vector_fptosi_v8f64_to_v8i16:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vcvttpd2dq %zmm0, %ymm0
+; AVX512VL-NEXT:    vpmovdw %ymm0, %xmm0
+; AVX512VL-NEXT:    vzeroupper
+; AVX512VL-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQ-LABEL: strict_vector_fptosi_v8f64_to_v8i16:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vcvttpd2dq %zmm0, %ymm0
+; AVX512DQ-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+  %ret = call <8 x i16> @llvm.experimental.constrained.fptosi.v8i16.v8f64(<8 x double> %a,
+                                              metadata !"fpexcept.strict")
+  ret <8 x i16> %ret
+}
+
+define <8 x i16> @strict_vector_fptoui_v8f64_to_v8i16(<8 x double> %a) #0 {
+; AVX512VL-LABEL: strict_vector_fptoui_v8f64_to_v8i16:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vcvttpd2dq %zmm0, %ymm0
+; AVX512VL-NEXT:    vpmovdw %ymm0, %xmm0
+; AVX512VL-NEXT:    vzeroupper
+; AVX512VL-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQ-LABEL: strict_vector_fptoui_v8f64_to_v8i16:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vcvttpd2dq %zmm0, %ymm0
+; AVX512DQ-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+  %ret = call <8 x i16> @llvm.experimental.constrained.fptoui.v8i16.v8f64(<8 x double> %a,
+                                              metadata !"fpexcept.strict")
+  ret <8 x i16> %ret
+}
+
+define <8 x i8> @strict_vector_fptosi_v8f64_to_v8i8(<8 x double> %a) #0 {
+; AVX512VL-LABEL: strict_vector_fptosi_v8f64_to_v8i8:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vcvttpd2dq %zmm0, %ymm0
+; AVX512VL-NEXT:    vpmovdb %ymm0, %xmm0
+; AVX512VL-NEXT:    vzeroupper
+; AVX512VL-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQ-LABEL: strict_vector_fptosi_v8f64_to_v8i8:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vcvttpd2dq %zmm0, %ymm0
+; AVX512DQ-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+  %ret = call <8 x i8> @llvm.experimental.constrained.fptosi.v8i8.v8f64(<8 x double> %a,
+                                              metadata !"fpexcept.strict")
+  ret <8 x i8> %ret
+}
+
+define <8 x i8> @strict_vector_fptoui_v8f64_to_v8i8(<8 x double> %a) #0 {
+; AVX512VL-LABEL: strict_vector_fptoui_v8f64_to_v8i8:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vcvttpd2dq %zmm0, %ymm0
+; AVX512VL-NEXT:    vpmovdb %ymm0, %xmm0
+; AVX512VL-NEXT:    vzeroupper
+; AVX512VL-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQ-LABEL: strict_vector_fptoui_v8f64_to_v8i8:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vcvttpd2dq %zmm0, %ymm0
+; AVX512DQ-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+  %ret = call <8 x i8> @llvm.experimental.constrained.fptoui.v8i8.v8f64(<8 x double> %a,
+                                              metadata !"fpexcept.strict")
+  ret <8 x i8> %ret
+}
+
+define <8 x i1> @strict_vector_fptosi_v8f64_to_v8i1(<8 x double> %a) #0 {
+; AVX512VL-LABEL: strict_vector_fptosi_v8f64_to_v8i1:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vcvttpd2dq %zmm0, %ymm0
+; AVX512VL-NEXT:    vptestmd %ymm0, %ymm0, %k1
+; AVX512VL-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
+; AVX512VL-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
+; AVX512VL-NEXT:    vpmovdw %ymm0, %xmm0
+; AVX512VL-NEXT:    vzeroupper
+; AVX512VL-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQ-LABEL: strict_vector_fptosi_v8f64_to_v8i1:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vcvttpd2dq %zmm0, %ymm0
+; AVX512DQ-NEXT:    vpmovd2m %zmm0, %k0
+; AVX512DQ-NEXT:    vpmovm2d %k0, %zmm0
+; AVX512DQ-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+  %ret = call <8 x i1> @llvm.experimental.constrained.fptosi.v8i1.v8f64(<8 x double> %a,
+                                              metadata !"fpexcept.strict")
+  ret <8 x i1> %ret
+}
+
+define <8 x i1> @strict_vector_fptoui_v8f64_to_v8i1(<8 x double> %a) #0 {
+; AVX512VL-LABEL: strict_vector_fptoui_v8f64_to_v8i1:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vcvttpd2dq %zmm0, %ymm0
+; AVX512VL-NEXT:    vpslld $31, %ymm0, %ymm0
+; AVX512VL-NEXT:    vptestmd %ymm0, %ymm0, %k1
+; AVX512VL-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
+; AVX512VL-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
+; AVX512VL-NEXT:    vpmovdw %ymm0, %xmm0
+; AVX512VL-NEXT:    vzeroupper
+; AVX512VL-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQ-LABEL: strict_vector_fptoui_v8f64_to_v8i1:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vcvttpd2dq %zmm0, %ymm0
+; AVX512DQ-NEXT:    vpslld $31, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpmovd2m %zmm0, %k0
+; AVX512DQ-NEXT:    vpmovm2d %k0, %zmm0
+; AVX512DQ-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+  %ret = call <8 x i1> @llvm.experimental.constrained.fptoui.v8i1.v8f64(<8 x double> %a,
+                                              metadata !"fpexcept.strict")
+  ret <8 x i1> %ret
+}
+
+define <16 x i32> @strict_vector_fptosi_v16f32_to_v16i32(<16 x float> %a) #0 {
+; CHECK-LABEL: strict_vector_fptosi_v16f32_to_v16i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vcvttps2dq %zmm0, %zmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %ret = call <16 x i32> @llvm.experimental.constrained.fptosi.v16i32.v16f32(<16 x float> %a,
+                                              metadata !"fpexcept.strict")
+  ret <16 x i32> %ret
+}
+
+define <16 x i32> @strict_vector_fptoui_v16f32_to_v16i32(<16 x float> %a) #0 {
+; CHECK-LABEL: strict_vector_fptoui_v16f32_to_v16i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vcvttps2udq %zmm0, %zmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %ret = call <16 x i32> @llvm.experimental.constrained.fptoui.v16i32.v16f32(<16 x float> %a,
+                                              metadata !"fpexcept.strict")
+  ret <16 x i32> %ret
+}
+
+define <16 x i16> @strict_vector_fptosi_v16f32_to_v16i16(<16 x float> %a) #0 {
+; CHECK-LABEL: strict_vector_fptosi_v16f32_to_v16i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vcvttps2dq %zmm0, %zmm0
+; CHECK-NEXT:    vpmovdw %zmm0, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %ret = call <16 x i16> @llvm.experimental.constrained.fptosi.v16i16.v16f32(<16 x float> %a,
+                                              metadata !"fpexcept.strict")
+  ret <16 x i16> %ret
+}
+
+define <16 x i16> @strict_vector_fptoui_v16f32_to_v16i16(<16 x float> %a) #0 {
+; CHECK-LABEL: strict_vector_fptoui_v16f32_to_v16i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vcvttps2dq %zmm0, %zmm0
+; CHECK-NEXT:    vpmovdw %zmm0, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %ret = call <16 x i16> @llvm.experimental.constrained.fptoui.v16i16.v16f32(<16 x float> %a,
+                                              metadata !"fpexcept.strict")
+  ret <16 x i16> %ret
+}
+
+define <16 x i8> @strict_vector_fptosi_v16f32_to_v16i8(<16 x float> %a) #0 {
+; CHECK-LABEL: strict_vector_fptosi_v16f32_to_v16i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vcvttps2dq %zmm0, %zmm0
+; CHECK-NEXT:    vpmovdb %zmm0, %xmm0
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    ret{{[l|q]}}
+  %ret = call <16 x i8> @llvm.experimental.constrained.fptosi.v16i8.v16f32(<16 x float> %a,
+                                              metadata !"fpexcept.strict")
+  ret <16 x i8> %ret
+}
+
+define <16 x i8> @strict_vector_fptoui_v16f32_to_v16i8(<16 x float> %a) #0 {
+; CHECK-LABEL: strict_vector_fptoui_v16f32_to_v16i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vcvttps2dq %zmm0, %zmm0
+; CHECK-NEXT:    vpmovdb %zmm0, %xmm0
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    ret{{[l|q]}}
+  %ret = call <16 x i8> @llvm.experimental.constrained.fptoui.v16i8.v16f32(<16 x float> %a,
+                                              metadata !"fpexcept.strict")
+  ret <16 x i8> %ret
+}
+
+define <16 x i1> @strict_vector_fptosi_v16f32_to_v16i1(<16 x float> %a) #0 {
+; AVX512VL-LABEL: strict_vector_fptosi_v16f32_to_v16i1:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vcvttps2dq %zmm0, %zmm0
+; AVX512VL-NEXT:    vptestmd %zmm0, %zmm0, %k1
+; AVX512VL-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512VL-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512VL-NEXT:    vzeroupper
+; AVX512VL-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQ-LABEL: strict_vector_fptosi_v16f32_to_v16i1:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vcvttps2dq %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpmovd2m %zmm0, %k0
+; AVX512DQ-NEXT:    vpmovm2d %k0, %zmm0
+; AVX512DQ-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+  %ret = call <16 x i1> @llvm.experimental.constrained.fptosi.v16i1.v16f32(<16 x float> %a,
+                                              metadata !"fpexcept.strict")
+  ret <16 x i1> %ret
+}
+
+define <16 x i1> @strict_vector_fptoui_v16f32_to_v16i1(<16 x float> %a) #0 {
+; AVX512VL-LABEL: strict_vector_fptoui_v16f32_to_v16i1:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vcvttps2dq %zmm0, %zmm0
+; AVX512VL-NEXT:    vpslld $31, %zmm0, %zmm0
+; AVX512VL-NEXT:    vptestmd %zmm0, %zmm0, %k1
+; AVX512VL-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512VL-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512VL-NEXT:    vzeroupper
+; AVX512VL-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQ-LABEL: strict_vector_fptoui_v16f32_to_v16i1:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vcvttps2dq %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpslld $31, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpmovd2m %zmm0, %k0
+; AVX512DQ-NEXT:    vpmovm2d %k0, %zmm0
+; AVX512DQ-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+  %ret = call <16 x i1> @llvm.experimental.constrained.fptoui.v16i1.v16f32(<16 x float> %a,
+                                              metadata !"fpexcept.strict")
+  ret <16 x i1> %ret
+}
+
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/X86/vec-strict-inttofp-128.ll b/llvm/test/CodeGen/X86/vec-strict-inttofp-128.ll
new file mode 100644
index 000000000..f3c79c2cb
--- /dev/null
+++ b/llvm/test/CodeGen/X86/vec-strict-inttofp-128.ll
@@ -0,0 +1,1377 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse2 -O3 | FileCheck %s --check-prefixes=SSE,SSE-32
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 -O3 | FileCheck %s --check-prefixes=SSE,SSE-64
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse2 -O3 | FileCheck %s --check-prefixes=SSE41,SSE41-32
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 -O3 | FileCheck %s --check-prefixes=SSE41,SSE41-64
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx -O3 | FileCheck %s --check-prefixes=AVX,AVX1,AVX-32,AVX1-32
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx -O3 | FileCheck %s --check-prefixes=AVX,AVX1,AVX-64,AVX1-64
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx512f -O3 | FileCheck %s --check-prefixes=AVX,AVX512F,AVX-32,AVX512F-32
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f -O3 | FileCheck %s --check-prefixes=AVX,AVX512F,AVX-64,AVX512F-64
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=avx512vl -O3 | FileCheck %s --check-prefixes=AVX,AVX512VL,AVX-32,AVX512VL-32
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512vl -O3 | FileCheck %s --check-prefixes=AVX,AVX512VL,AVX-64,AVX512VL-64
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=avx512f,avx512dq -O3 | FileCheck %s --check-prefixes=AVX,AVX512DQ,AVX512DQ-32
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512f,avx512dq -O3 | FileCheck %s --check-prefixes=AVX,AVX512DQ,AVX512DQ-64
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=avx512dq,avx512vl -O3 | FileCheck %s --check-prefixes=AVX,AVX512DQVL,AVX512DQVL-32
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512dq,avx512vl -O3 | FileCheck %s --check-prefixes=AVX,AVX512DQVL,AVX512DQVL-64
+
+declare <2 x float> @llvm.experimental.constrained.sitofp.v2f32.v2i32(<2 x i32>, metadata, metadata)
+declare <2 x float> @llvm.experimental.constrained.uitofp.v2f32.v2i32(<2 x i32>, metadata, metadata)
+declare <2 x float> @llvm.experimental.constrained.sitofp.v2f32.v2i64(<2 x i64>, metadata, metadata)
+declare <2 x float> @llvm.experimental.constrained.uitofp.v2f32.v2i64(<2 x i64>, metadata, metadata)
+declare <4 x float> @llvm.experimental.constrained.sitofp.v4f32.v4i1(<4 x i1>, metadata, metadata)
+declare <4 x float> @llvm.experimental.constrained.uitofp.v4f32.v4i1(<4 x i1>, metadata, metadata)
+declare <4 x float> @llvm.experimental.constrained.sitofp.v4f32.v4i8(<4 x i8>, metadata, metadata)
+declare <4 x float> @llvm.experimental.constrained.uitofp.v4f32.v4i8(<4 x i8>, metadata, metadata)
+declare <4 x float> @llvm.experimental.constrained.sitofp.v4f32.v4i16(<4 x i16>, metadata, metadata)
+declare <4 x float> @llvm.experimental.constrained.uitofp.v4f32.v4i16(<4 x i16>, metadata, metadata)
+declare <4 x float> @llvm.experimental.constrained.sitofp.v4f32.v4i32(<4 x i32>, metadata, metadata)
+declare <4 x float> @llvm.experimental.constrained.uitofp.v4f32.v4i32(<4 x i32>, metadata, metadata)
+declare <2 x double> @llvm.experimental.constrained.sitofp.v2f64.v2i1(<2 x i1>, metadata, metadata)
+declare <2 x double> @llvm.experimental.constrained.uitofp.v2f64.v2i1(<2 x i1>, metadata, metadata)
+declare <2 x double> @llvm.experimental.constrained.sitofp.v2f64.v2i8(<2 x i8>, metadata, metadata)
+declare <2 x double> @llvm.experimental.constrained.uitofp.v2f64.v2i8(<2 x i8>, metadata, metadata)
+declare <2 x double> @llvm.experimental.constrained.sitofp.v2f64.v2i16(<2 x i16>, metadata, metadata)
+declare <2 x double> @llvm.experimental.constrained.uitofp.v2f64.v2i16(<2 x i16>, metadata, metadata)
+declare <2 x double> @llvm.experimental.constrained.sitofp.v2f64.v2i32(<2 x i32>, metadata, metadata)
+declare <2 x double> @llvm.experimental.constrained.uitofp.v2f64.v2i32(<2 x i32>, metadata, metadata)
+declare <2 x double> @llvm.experimental.constrained.sitofp.v2f64.v2i64(<2 x i64>, metadata, metadata)
+declare <2 x double> @llvm.experimental.constrained.uitofp.v2f64.v2i64(<2 x i64>, metadata, metadata)
+
+define <2 x float> @sitofp_v2i32_v2f32(<2 x i32> %x) #0 {
+; SSE-LABEL: sitofp_v2i32_v2f32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movq {{.*#+}} xmm0 = xmm0[0],zero
+; SSE-NEXT:    cvtdq2ps %xmm0, %xmm0
+; SSE-NEXT:    ret{{[l|q]}}
+;
+; SSE41-LABEL: sitofp_v2i32_v2f32:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movq {{.*#+}} xmm0 = xmm0[0],zero
+; SSE41-NEXT:    cvtdq2ps %xmm0, %xmm0
+; SSE41-NEXT:    ret{{[l|q]}}
+;
+; AVX-LABEL: sitofp_v2i32_v2f32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
+; AVX-NEXT:    vcvtdq2ps %xmm0, %xmm0
+; AVX-NEXT:    ret{{[l|q]}}
+ %result = call <2 x float> @llvm.experimental.constrained.sitofp.v2f32.v2i32(<2 x i32> %x,
+                                                              metadata !"round.dynamic",
+                                                              metadata !"fpexcept.strict") #0
+  ret <2 x float> %result
+}
+
+define <2 x float> @uitofp_v2i32_v2f32(<2 x i32> %x) #0 {
+; SSE-LABEL: uitofp_v2i32_v2f32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    xorpd %xmm1, %xmm1
+; SSE-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE-NEXT:    movapd {{.*#+}} xmm1 = [4.503599627370496E+15,4.503599627370496E+15]
+; SSE-NEXT:    orpd %xmm1, %xmm0
+; SSE-NEXT:    subpd %xmm1, %xmm0
+; SSE-NEXT:    cvtpd2ps %xmm0, %xmm0
+; SSE-NEXT:    ret{{[l|q]}}
+;
+; SSE41-LABEL: uitofp_v2i32_v2f32:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    xorpd %xmm1, %xmm1
+; SSE41-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE41-NEXT:    movapd {{.*#+}} xmm1 = [4.503599627370496E+15,4.503599627370496E+15]
+; SSE41-NEXT:    orpd %xmm1, %xmm0
+; SSE41-NEXT:    subpd %xmm1, %xmm0
+; SSE41-NEXT:    cvtpd2ps %xmm0, %xmm0
+; SSE41-NEXT:    ret{{[l|q]}}
+;
+; AVX1-LABEL: uitofp_v2i32_v2f32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [4.503599627370496E+15,4.503599627370496E+15]
+; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vsubpd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vcvtpd2ps %xmm0, %xmm0
+; AVX1-NEXT:    ret{{[l|q]}}
+;
+; AVX512F-LABEL: uitofp_v2i32_v2f32:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
+; AVX512F-NEXT:    vcvtudq2ps %zmm0, %zmm0
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    ret{{[l|q]}}
+;
+; AVX512VL-LABEL: uitofp_v2i32_v2f32:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
+; AVX512VL-NEXT:    vcvtudq2ps %xmm0, %xmm0
+; AVX512VL-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQ-LABEL: uitofp_v2i32_v2f32:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
+; AVX512DQ-NEXT:    vcvtudq2ps %zmm0, %zmm0
+; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQVL-LABEL: uitofp_v2i32_v2f32:
+; AVX512DQVL:       # %bb.0:
+; AVX512DQVL-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
+; AVX512DQVL-NEXT:    vcvtudq2ps %xmm0, %xmm0
+; AVX512DQVL-NEXT:    ret{{[l|q]}}
+ %result = call <2 x float> @llvm.experimental.constrained.uitofp.v2f32.v2i32(<2 x i32> %x,
+                                                              metadata !"round.dynamic",
+                                                              metadata !"fpexcept.strict") #0
+  ret <2 x float> %result
+}
+
+define <2 x float> @sitofp_v2i64_v2f32(<2 x i64> %x) #0 {
+; SSE-32-LABEL: sitofp_v2i64_v2f32:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    .cfi_def_cfa_offset 8
+; SSE-32-NEXT:    .cfi_offset %ebp, -8
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    .cfi_def_cfa_register %ebp
+; SSE-32-NEXT:    andl $-8, %esp
+; SSE-32-NEXT:    subl $24, %esp
+; SSE-32-NEXT:    movq %xmm0, {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE-32-NEXT:    movq %xmm0, {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fildll {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fstps (%esp)
+; SSE-32-NEXT:    fildll {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fstps {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE-32-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE-32-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    .cfi_def_cfa %esp, 4
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: sitofp_v2i64_v2f32:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movq %xmm0, %rax
+; SSE-64-NEXT:    cvtsi2ss %rax, %xmm1
+; SSE-64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE-64-NEXT:    movq %xmm0, %rax
+; SSE-64-NEXT:    xorps %xmm0, %xmm0
+; SSE-64-NEXT:    cvtsi2ss %rax, %xmm0
+; SSE-64-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE-64-NEXT:    movaps %xmm1, %xmm0
+; SSE-64-NEXT:    retq
+;
+; SSE41-32-LABEL: sitofp_v2i64_v2f32:
+; SSE41-32:       # %bb.0:
+; SSE41-32-NEXT:    pushl %ebp
+; SSE41-32-NEXT:    .cfi_def_cfa_offset 8
+; SSE41-32-NEXT:    .cfi_offset %ebp, -8
+; SSE41-32-NEXT:    movl %esp, %ebp
+; SSE41-32-NEXT:    .cfi_def_cfa_register %ebp
+; SSE41-32-NEXT:    andl $-8, %esp
+; SSE41-32-NEXT:    subl $24, %esp
+; SSE41-32-NEXT:    movq %xmm0, {{[0-9]+}}(%esp)
+; SSE41-32-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE41-32-NEXT:    movq %xmm0, {{[0-9]+}}(%esp)
+; SSE41-32-NEXT:    fildll {{[0-9]+}}(%esp)
+; SSE41-32-NEXT:    fstps (%esp)
+; SSE41-32-NEXT:    fildll {{[0-9]+}}(%esp)
+; SSE41-32-NEXT:    fstps {{[0-9]+}}(%esp)
+; SSE41-32-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE41-32-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE41-32-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE41-32-NEXT:    movl %ebp, %esp
+; SSE41-32-NEXT:    popl %ebp
+; SSE41-32-NEXT:    .cfi_def_cfa %esp, 4
+; SSE41-32-NEXT:    retl
+;
+; SSE41-64-LABEL: sitofp_v2i64_v2f32:
+; SSE41-64:       # %bb.0:
+; SSE41-64-NEXT:    movq %xmm0, %rax
+; SSE41-64-NEXT:    cvtsi2ss %rax, %xmm1
+; SSE41-64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE41-64-NEXT:    movq %xmm0, %rax
+; SSE41-64-NEXT:    xorps %xmm0, %xmm0
+; SSE41-64-NEXT:    cvtsi2ss %rax, %xmm0
+; SSE41-64-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE41-64-NEXT:    movaps %xmm1, %xmm0
+; SSE41-64-NEXT:    retq
+;
+; AVX-32-LABEL: sitofp_v2i64_v2f32:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    .cfi_def_cfa_offset 8
+; AVX-32-NEXT:    .cfi_offset %ebp, -8
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    .cfi_def_cfa_register %ebp
+; AVX-32-NEXT:    andl $-8, %esp
+; AVX-32-NEXT:    subl $24, %esp
+; AVX-32-NEXT:    vmovlps %xmm0, {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX-32-NEXT:    vmovlps %xmm0, {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    fildll {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    fstps (%esp)
+; AVX-32-NEXT:    fildll {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    fstps {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[2,3]
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    .cfi_def_cfa %esp, 4
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: sitofp_v2i64_v2f32:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX-64-NEXT:    vcvtsi2ss %rax, %xmm1, %xmm1
+; AVX-64-NEXT:    vmovq %xmm0, %rax
+; AVX-64-NEXT:    vcvtsi2ss %rax, %xmm2, %xmm0
+; AVX-64-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[2,3]
+; AVX-64-NEXT:    retq
+;
+; AVX512DQ-32-LABEL: sitofp_v2i64_v2f32:
+; AVX512DQ-32:       # %bb.0:
+; AVX512DQ-32-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512DQ-32-NEXT:    vcvtqq2ps %zmm0, %ymm1
+; AVX512DQ-32-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX512DQ-32-NEXT:    vcvtqq2ps %zmm0, %ymm0
+; AVX512DQ-32-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],zero,zero
+; AVX512DQ-32-NEXT:    vzeroupper
+; AVX512DQ-32-NEXT:    retl
+;
+; AVX512DQ-64-LABEL: sitofp_v2i64_v2f32:
+; AVX512DQ-64:       # %bb.0:
+; AVX512DQ-64-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX512DQ-64-NEXT:    vcvtsi2ss %rax, %xmm1, %xmm1
+; AVX512DQ-64-NEXT:    vmovq %xmm0, %rax
+; AVX512DQ-64-NEXT:    vcvtsi2ss %rax, %xmm2, %xmm0
+; AVX512DQ-64-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[2,3]
+; AVX512DQ-64-NEXT:    retq
+;
+; AVX512DQVL-LABEL: sitofp_v2i64_v2f32:
+; AVX512DQVL:       # %bb.0:
+; AVX512DQVL-NEXT:    vcvtqq2ps %xmm0, %xmm0
+; AVX512DQVL-NEXT:    ret{{[l|q]}}
+ %result = call <2 x float> @llvm.experimental.constrained.sitofp.v2f32.v2i64(<2 x i64> %x,
+                                                              metadata !"round.dynamic",
+                                                              metadata !"fpexcept.strict") #0
+  ret <2 x float> %result
+}
+
+define <2 x float> @uitofp_v2i64_v2f32(<2 x i64> %x) #0 {
+; SSE-32-LABEL: uitofp_v2i64_v2f32:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    .cfi_def_cfa_offset 8
+; SSE-32-NEXT:    .cfi_offset %ebp, -8
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    .cfi_def_cfa_register %ebp
+; SSE-32-NEXT:    andl $-8, %esp
+; SSE-32-NEXT:    subl $24, %esp
+; SSE-32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE-32-NEXT:    movq %xmm1, {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    movq %xmm0, {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,1,2,3]
+; SSE-32-NEXT:    movd %xmm1, %eax
+; SSE-32-NEXT:    shrl $31, %eax
+; SSE-32-NEXT:    fildll {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; SSE-32-NEXT:    fstps (%esp)
+; SSE-32-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE-32-NEXT:    movd %xmm0, %eax
+; SSE-32-NEXT:    shrl $31, %eax
+; SSE-32-NEXT:    fildll {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; SSE-32-NEXT:    fstps {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE-32-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE-32-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    .cfi_def_cfa %esp, 4
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: uitofp_v2i64_v2f32:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movdqa %xmm0, %xmm1
+; SSE-64-NEXT:    movq %xmm0, %rax
+; SSE-64-NEXT:    movq %rax, %rcx
+; SSE-64-NEXT:    shrq %rcx
+; SSE-64-NEXT:    movl %eax, %edx
+; SSE-64-NEXT:    andl $1, %edx
+; SSE-64-NEXT:    orq %rcx, %rdx
+; SSE-64-NEXT:    testq %rax, %rax
+; SSE-64-NEXT:    cmovnsq %rax, %rdx
+; SSE-64-NEXT:    xorps %xmm0, %xmm0
+; SSE-64-NEXT:    cvtsi2ss %rdx, %xmm0
+; SSE-64-NEXT:    jns .LBB3_2
+; SSE-64-NEXT:  # %bb.1:
+; SSE-64-NEXT:    addss %xmm0, %xmm0
+; SSE-64-NEXT:  .LBB3_2:
+; SSE-64-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE-64-NEXT:    movq %xmm1, %rax
+; SSE-64-NEXT:    movq %rax, %rcx
+; SSE-64-NEXT:    shrq %rcx
+; SSE-64-NEXT:    movl %eax, %edx
+; SSE-64-NEXT:    andl $1, %edx
+; SSE-64-NEXT:    orq %rcx, %rdx
+; SSE-64-NEXT:    testq %rax, %rax
+; SSE-64-NEXT:    cmovnsq %rax, %rdx
+; SSE-64-NEXT:    xorps %xmm1, %xmm1
+; SSE-64-NEXT:    cvtsi2ss %rdx, %xmm1
+; SSE-64-NEXT:    jns .LBB3_4
+; SSE-64-NEXT:  # %bb.3:
+; SSE-64-NEXT:    addss %xmm1, %xmm1
+; SSE-64-NEXT:  .LBB3_4:
+; SSE-64-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE-64-NEXT:    retq
+;
+; SSE41-32-LABEL: uitofp_v2i64_v2f32:
+; SSE41-32:       # %bb.0:
+; SSE41-32-NEXT:    pushl %ebp
+; SSE41-32-NEXT:    .cfi_def_cfa_offset 8
+; SSE41-32-NEXT:    .cfi_offset %ebp, -8
+; SSE41-32-NEXT:    movl %esp, %ebp
+; SSE41-32-NEXT:    .cfi_def_cfa_register %ebp
+; SSE41-32-NEXT:    andl $-8, %esp
+; SSE41-32-NEXT:    subl $24, %esp
+; SSE41-32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE41-32-NEXT:    movq %xmm1, {{[0-9]+}}(%esp)
+; SSE41-32-NEXT:    movq %xmm0, {{[0-9]+}}(%esp)
+; SSE41-32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,1,2,3]
+; SSE41-32-NEXT:    movd %xmm1, %eax
+; SSE41-32-NEXT:    shrl $31, %eax
+; SSE41-32-NEXT:    fildll {{[0-9]+}}(%esp)
+; SSE41-32-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; SSE41-32-NEXT:    fstps (%esp)
+; SSE41-32-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE41-32-NEXT:    movd %xmm0, %eax
+; SSE41-32-NEXT:    shrl $31, %eax
+; SSE41-32-NEXT:    fildll {{[0-9]+}}(%esp)
+; SSE41-32-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; SSE41-32-NEXT:    fstps {{[0-9]+}}(%esp)
+; SSE41-32-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE41-32-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE41-32-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE41-32-NEXT:    movl %ebp, %esp
+; SSE41-32-NEXT:    popl %ebp
+; SSE41-32-NEXT:    .cfi_def_cfa %esp, 4
+; SSE41-32-NEXT:    retl
+;
+; SSE41-64-LABEL: uitofp_v2i64_v2f32:
+; SSE41-64:       # %bb.0:
+; SSE41-64-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-64-NEXT:    movq %xmm0, %rax
+; SSE41-64-NEXT:    movq %rax, %rcx
+; SSE41-64-NEXT:    shrq %rcx
+; SSE41-64-NEXT:    movl %eax, %edx
+; SSE41-64-NEXT:    andl $1, %edx
+; SSE41-64-NEXT:    orq %rcx, %rdx
+; SSE41-64-NEXT:    testq %rax, %rax
+; SSE41-64-NEXT:    cmovnsq %rax, %rdx
+; SSE41-64-NEXT:    xorps %xmm0, %xmm0
+; SSE41-64-NEXT:    cvtsi2ss %rdx, %xmm0
+; SSE41-64-NEXT:    jns .LBB3_2
+; SSE41-64-NEXT:  # %bb.1:
+; SSE41-64-NEXT:    addss %xmm0, %xmm0
+; SSE41-64-NEXT:  .LBB3_2:
+; SSE41-64-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE41-64-NEXT:    movq %xmm1, %rax
+; SSE41-64-NEXT:    movq %rax, %rcx
+; SSE41-64-NEXT:    shrq %rcx
+; SSE41-64-NEXT:    movl %eax, %edx
+; SSE41-64-NEXT:    andl $1, %edx
+; SSE41-64-NEXT:    orq %rcx, %rdx
+; SSE41-64-NEXT:    testq %rax, %rax
+; SSE41-64-NEXT:    cmovnsq %rax, %rdx
+; SSE41-64-NEXT:    xorps %xmm1, %xmm1
+; SSE41-64-NEXT:    cvtsi2ss %rdx, %xmm1
+; SSE41-64-NEXT:    jns .LBB3_4
+; SSE41-64-NEXT:  # %bb.3:
+; SSE41-64-NEXT:    addss %xmm1, %xmm1
+; SSE41-64-NEXT:  .LBB3_4:
+; SSE41-64-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE41-64-NEXT:    retq
+;
+; AVX-32-LABEL: uitofp_v2i64_v2f32:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    .cfi_def_cfa_offset 8
+; AVX-32-NEXT:    .cfi_offset %ebp, -8
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    .cfi_def_cfa_register %ebp
+; AVX-32-NEXT:    andl $-8, %esp
+; AVX-32-NEXT:    subl $24, %esp
+; AVX-32-NEXT:    vmovlps %xmm0, {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-32-NEXT:    vmovlps %xmm1, {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    vextractps $1, %xmm0, %eax
+; AVX-32-NEXT:    shrl $31, %eax
+; AVX-32-NEXT:    fildll {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; AVX-32-NEXT:    fstps {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    vextractps $3, %xmm0, %eax
+; AVX-32-NEXT:    shrl $31, %eax
+; AVX-32-NEXT:    fildll {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; AVX-32-NEXT:    fstps (%esp)
+; AVX-32-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[2,3]
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    .cfi_def_cfa %esp, 4
+; AVX-32-NEXT:    retl
+;
+; AVX1-64-LABEL: uitofp_v2i64_v2f32:
+; AVX1-64:       # %bb.0:
+; AVX1-64-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm1
+; AVX1-64-NEXT:    vpsrlq $1, %xmm0, %xmm2
+; AVX1-64-NEXT:    vpor %xmm1, %xmm2, %xmm1
+; AVX1-64-NEXT:    vblendvpd %xmm0, %xmm1, %xmm0, %xmm1
+; AVX1-64-NEXT:    vpextrq $1, %xmm1, %rax
+; AVX1-64-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm2
+; AVX1-64-NEXT:    vmovq %xmm1, %rax
+; AVX1-64-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm1
+; AVX1-64-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0],xmm2[0],zero,zero
+; AVX1-64-NEXT:    vaddps %xmm1, %xmm1, %xmm2
+; AVX1-64-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX1-64-NEXT:    vpcmpgtq %xmm0, %xmm3, %xmm0
+; AVX1-64-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
+; AVX1-64-NEXT:    vblendvps %xmm0, %xmm2, %xmm1, %xmm0
+; AVX1-64-NEXT:    retq
+;
+; AVX512F-64-LABEL: uitofp_v2i64_v2f32:
+; AVX512F-64:       # %bb.0:
+; AVX512F-64-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX512F-64-NEXT:    vcvtusi2ss %rax, %xmm1, %xmm1
+; AVX512F-64-NEXT:    vmovq %xmm0, %rax
+; AVX512F-64-NEXT:    vcvtusi2ss %rax, %xmm2, %xmm0
+; AVX512F-64-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[2,3]
+; AVX512F-64-NEXT:    retq
+;
+; AVX512VL-64-LABEL: uitofp_v2i64_v2f32:
+; AVX512VL-64:       # %bb.0:
+; AVX512VL-64-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX512VL-64-NEXT:    vcvtusi2ss %rax, %xmm1, %xmm1
+; AVX512VL-64-NEXT:    vmovq %xmm0, %rax
+; AVX512VL-64-NEXT:    vcvtusi2ss %rax, %xmm2, %xmm0
+; AVX512VL-64-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[2,3]
+; AVX512VL-64-NEXT:    retq
+;
+; AVX512DQ-32-LABEL: uitofp_v2i64_v2f32:
+; AVX512DQ-32:       # %bb.0:
+; AVX512DQ-32-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512DQ-32-NEXT:    vcvtuqq2ps %zmm0, %ymm1
+; AVX512DQ-32-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX512DQ-32-NEXT:    vcvtuqq2ps %zmm0, %ymm0
+; AVX512DQ-32-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],zero,zero
+; AVX512DQ-32-NEXT:    vzeroupper
+; AVX512DQ-32-NEXT:    retl
+;
+; AVX512DQ-64-LABEL: uitofp_v2i64_v2f32:
+; AVX512DQ-64:       # %bb.0:
+; AVX512DQ-64-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX512DQ-64-NEXT:    vcvtusi2ss %rax, %xmm1, %xmm1
+; AVX512DQ-64-NEXT:    vmovq %xmm0, %rax
+; AVX512DQ-64-NEXT:    vcvtusi2ss %rax, %xmm2, %xmm0
+; AVX512DQ-64-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[2,3]
+; AVX512DQ-64-NEXT:    retq
+;
+; AVX512DQVL-LABEL: uitofp_v2i64_v2f32:
+; AVX512DQVL:       # %bb.0:
+; AVX512DQVL-NEXT:    vcvtuqq2ps %xmm0, %xmm0
+; AVX512DQVL-NEXT:    ret{{[l|q]}}
+ %result = call <2 x float> @llvm.experimental.constrained.uitofp.v2f32.v2i64(<2 x i64> %x,
+                                                              metadata !"round.dynamic",
+                                                              metadata !"fpexcept.strict") #0
+  ret <2 x float> %result
+}
+
+define <4 x float> @sitofp_v4i1_v4f32(<4 x i1> %x) #0 {
+; SSE-LABEL: sitofp_v4i1_v4f32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pslld $31, %xmm0
+; SSE-NEXT:    psrad $31, %xmm0
+; SSE-NEXT:    cvtdq2ps %xmm0, %xmm0
+; SSE-NEXT:    ret{{[l|q]}}
+;
+; SSE41-LABEL: sitofp_v4i1_v4f32:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pslld $31, %xmm0
+; SSE41-NEXT:    psrad $31, %xmm0
+; SSE41-NEXT:    cvtdq2ps %xmm0, %xmm0
+; SSE41-NEXT:    ret{{[l|q]}}
+;
+; AVX-LABEL: sitofp_v4i1_v4f32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpslld $31, %xmm0, %xmm0
+; AVX-NEXT:    vpsrad $31, %xmm0, %xmm0
+; AVX-NEXT:    vcvtdq2ps %xmm0, %xmm0
+; AVX-NEXT:    ret{{[l|q]}}
+ %result = call <4 x float> @llvm.experimental.constrained.sitofp.v4f32.v4i1(<4 x i1> %x,
+                                                              metadata !"round.dynamic",
+                                                              metadata !"fpexcept.strict") #0
+  ret <4 x float> %result
+}
+
+define <4 x float> @uitofp_v4i1_v4f32(<4 x i1> %x) #0 {
+; SSE-32-LABEL: uitofp_v4i1_v4f32:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    andps {{\.LCPI.*}}, %xmm0
+; SSE-32-NEXT:    cvtdq2ps %xmm0, %xmm0
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: uitofp_v4i1_v4f32:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    andps {{.*}}(%rip), %xmm0
+; SSE-64-NEXT:    cvtdq2ps %xmm0, %xmm0
+; SSE-64-NEXT:    retq
+;
+; SSE41-32-LABEL: uitofp_v4i1_v4f32:
+; SSE41-32:       # %bb.0:
+; SSE41-32-NEXT:    andps {{\.LCPI.*}}, %xmm0
+; SSE41-32-NEXT:    cvtdq2ps %xmm0, %xmm0
+; SSE41-32-NEXT:    retl
+;
+; SSE41-64-LABEL: uitofp_v4i1_v4f32:
+; SSE41-64:       # %bb.0:
+; SSE41-64-NEXT:    andps {{.*}}(%rip), %xmm0
+; SSE41-64-NEXT:    cvtdq2ps %xmm0, %xmm0
+; SSE41-64-NEXT:    retq
+;
+; AVX1-32-LABEL: uitofp_v4i1_v4f32:
+; AVX1-32:       # %bb.0:
+; AVX1-32-NEXT:    vandps {{\.LCPI.*}}, %xmm0, %xmm0
+; AVX1-32-NEXT:    vcvtdq2ps %xmm0, %xmm0
+; AVX1-32-NEXT:    retl
+;
+; AVX1-64-LABEL: uitofp_v4i1_v4f32:
+; AVX1-64:       # %bb.0:
+; AVX1-64-NEXT:    vandps {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-64-NEXT:    vcvtdq2ps %xmm0, %xmm0
+; AVX1-64-NEXT:    retq
+;
+; AVX512F-LABEL: uitofp_v4i1_v4f32:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vbroadcastss {{.*#+}} xmm1 = [1,1,1,1]
+; AVX512F-NEXT:    vandps %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    vcvtdq2ps %xmm0, %xmm0
+; AVX512F-NEXT:    ret{{[l|q]}}
+;
+; AVX512VL-32-LABEL: uitofp_v4i1_v4f32:
+; AVX512VL-32:       # %bb.0:
+; AVX512VL-32-NEXT:    vpandd {{\.LCPI.*}}{1to4}, %xmm0, %xmm0
+; AVX512VL-32-NEXT:    vcvtdq2ps %xmm0, %xmm0
+; AVX512VL-32-NEXT:    retl
+;
+; AVX512VL-64-LABEL: uitofp_v4i1_v4f32:
+; AVX512VL-64:       # %bb.0:
+; AVX512VL-64-NEXT:    vpandd {{.*}}(%rip){1to4}, %xmm0, %xmm0
+; AVX512VL-64-NEXT:    vcvtdq2ps %xmm0, %xmm0
+; AVX512VL-64-NEXT:    retq
+;
+; AVX512DQ-LABEL: uitofp_v4i1_v4f32:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vbroadcastss {{.*#+}} xmm1 = [1,1,1,1]
+; AVX512DQ-NEXT:    vandps %xmm1, %xmm0, %xmm0
+; AVX512DQ-NEXT:    vcvtdq2ps %xmm0, %xmm0
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQVL-32-LABEL: uitofp_v4i1_v4f32:
+; AVX512DQVL-32:       # %bb.0:
+; AVX512DQVL-32-NEXT:    vandps {{\.LCPI.*}}{1to4}, %xmm0, %xmm0
+; AVX512DQVL-32-NEXT:    vcvtdq2ps %xmm0, %xmm0
+; AVX512DQVL-32-NEXT:    retl
+;
+; AVX512DQVL-64-LABEL: uitofp_v4i1_v4f32:
+; AVX512DQVL-64:       # %bb.0:
+; AVX512DQVL-64-NEXT:    vandps {{.*}}(%rip){1to4}, %xmm0, %xmm0
+; AVX512DQVL-64-NEXT:    vcvtdq2ps %xmm0, %xmm0
+; AVX512DQVL-64-NEXT:    retq
+ %result = call <4 x float> @llvm.experimental.constrained.uitofp.v4f32.v4i1(<4 x i1> %x,
+                                                              metadata !"round.dynamic",
+                                                              metadata !"fpexcept.strict") #0
+  ret <4 x float> %result
+}
+
+define <4 x float> @sitofp_v4i8_v4f32(<4 x i8> %x) #0 {
+; SSE-LABEL: sitofp_v4i8_v4f32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSE-NEXT:    psrad $24, %xmm0
+; SSE-NEXT:    cvtdq2ps %xmm0, %xmm0
+; SSE-NEXT:    ret{{[l|q]}}
+;
+; SSE41-LABEL: sitofp_v4i8_v4f32:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE41-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSE41-NEXT:    psrad $24, %xmm0
+; SSE41-NEXT:    cvtdq2ps %xmm0, %xmm0
+; SSE41-NEXT:    ret{{[l|q]}}
+;
+; AVX-LABEL: sitofp_v4i8_v4f32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpmovsxbd %xmm0, %xmm0
+; AVX-NEXT:    vcvtdq2ps %xmm0, %xmm0
+; AVX-NEXT:    ret{{[l|q]}}
+ %result = call <4 x float> @llvm.experimental.constrained.sitofp.v4f32.v4i8(<4 x i8> %x,
+                                                              metadata !"round.dynamic",
+                                                              metadata !"fpexcept.strict") #0
+  ret <4 x float> %result
+}
+
+define <4 x float> @uitofp_v4i8_v4f32(<4 x i8> %x) #0 {
+; SSE-LABEL: uitofp_v4i8_v4f32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pxor %xmm1, %xmm1
+; SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSE-NEXT:    cvtdq2ps %xmm0, %xmm0
+; SSE-NEXT:    ret{{[l|q]}}
+;
+; SSE41-LABEL: uitofp_v4i8_v4f32:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pxor %xmm1, %xmm1
+; SSE41-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSE41-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSE41-NEXT:    cvtdq2ps %xmm0, %xmm0
+; SSE41-NEXT:    ret{{[l|q]}}
+;
+; AVX-LABEL: uitofp_v4i8_v4f32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; AVX-NEXT:    vcvtdq2ps %xmm0, %xmm0
+; AVX-NEXT:    ret{{[l|q]}}
+ %result = call <4 x float> @llvm.experimental.constrained.uitofp.v4f32.v4i8(<4 x i8> %x,
+                                                              metadata !"round.dynamic",
+                                                              metadata !"fpexcept.strict") #0
+  ret <4 x float> %result
+}
+
+define <4 x float> @sitofp_v4i16_v4f32(<4 x i16> %x) #0 {
+; SSE-LABEL: sitofp_v4i16_v4f32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSE-NEXT:    psrad $16, %xmm0
+; SSE-NEXT:    cvtdq2ps %xmm0, %xmm0
+; SSE-NEXT:    ret{{[l|q]}}
+;
+; SSE41-LABEL: sitofp_v4i16_v4f32:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSE41-NEXT:    psrad $16, %xmm0
+; SSE41-NEXT:    cvtdq2ps %xmm0, %xmm0
+; SSE41-NEXT:    ret{{[l|q]}}
+;
+; AVX-LABEL: sitofp_v4i16_v4f32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpmovsxwd %xmm0, %xmm0
+; AVX-NEXT:    vcvtdq2ps %xmm0, %xmm0
+; AVX-NEXT:    ret{{[l|q]}}
+ %result = call <4 x float> @llvm.experimental.constrained.sitofp.v4f32.v4i16(<4 x i16> %x,
+                                                              metadata !"round.dynamic",
+                                                              metadata !"fpexcept.strict") #0
+  ret <4 x float> %result
+}
+
+define <4 x float> @uitofp_v4i16_v4f32(<4 x i16> %x) #0 {
+; SSE-LABEL: uitofp_v4i16_v4f32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pxor %xmm1, %xmm1
+; SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSE-NEXT:    cvtdq2ps %xmm0, %xmm0
+; SSE-NEXT:    ret{{[l|q]}}
+;
+; SSE41-LABEL: uitofp_v4i16_v4f32:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pxor %xmm1, %xmm1
+; SSE41-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSE41-NEXT:    cvtdq2ps %xmm0, %xmm0
+; SSE41-NEXT:    ret{{[l|q]}}
+;
+; AVX-LABEL: uitofp_v4i16_v4f32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX-NEXT:    vcvtdq2ps %xmm0, %xmm0
+; AVX-NEXT:    ret{{[l|q]}}
+ %result = call <4 x float> @llvm.experimental.constrained.uitofp.v4f32.v4i16(<4 x i16> %x,
+                                                              metadata !"round.dynamic",
+                                                              metadata !"fpexcept.strict") #0
+  ret <4 x float> %result
+}
+
+define <4 x float> @sitofp_v4i32_v4f32(<4 x i32> %x) #0 {
+; SSE-LABEL: sitofp_v4i32_v4f32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    cvtdq2ps %xmm0, %xmm0
+; SSE-NEXT:    ret{{[l|q]}}
+;
+; SSE41-LABEL: sitofp_v4i32_v4f32:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    cvtdq2ps %xmm0, %xmm0
+; SSE41-NEXT:    ret{{[l|q]}}
+;
+; AVX-LABEL: sitofp_v4i32_v4f32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcvtdq2ps %xmm0, %xmm0
+; AVX-NEXT:    ret{{[l|q]}}
+ %result = call <4 x float> @llvm.experimental.constrained.sitofp.v4f32.v4i32(<4 x i32> %x,
+                                                              metadata !"round.dynamic",
+                                                              metadata !"fpexcept.strict") #0
+  ret <4 x float> %result
+}
+
+define <4 x float> @uitofp_v4i32_v4f32(<4 x i32> %x) #0 {
+; SSE-32-LABEL: uitofp_v4i32_v4f32:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    movdqa {{.*#+}} xmm1 = [65535,65535,65535,65535]
+; SSE-32-NEXT:    pand %xmm0, %xmm1
+; SSE-32-NEXT:    por {{\.LCPI.*}}, %xmm1
+; SSE-32-NEXT:    psrld $16, %xmm0
+; SSE-32-NEXT:    por {{\.LCPI.*}}, %xmm0
+; SSE-32-NEXT:    subps {{\.LCPI.*}}, %xmm0
+; SSE-32-NEXT:    addps %xmm1, %xmm0
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: uitofp_v4i32_v4f32:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movdqa {{.*#+}} xmm1 = [65535,65535,65535,65535]
+; SSE-64-NEXT:    pand %xmm0, %xmm1
+; SSE-64-NEXT:    por {{.*}}(%rip), %xmm1
+; SSE-64-NEXT:    psrld $16, %xmm0
+; SSE-64-NEXT:    por {{.*}}(%rip), %xmm0
+; SSE-64-NEXT:    subps {{.*}}(%rip), %xmm0
+; SSE-64-NEXT:    addps %xmm1, %xmm0
+; SSE-64-NEXT:    retq
+;
+; SSE41-32-LABEL: uitofp_v4i32_v4f32:
+; SSE41-32:       # %bb.0:
+; SSE41-32-NEXT:    movdqa {{.*#+}} xmm1 = [65535,65535,65535,65535]
+; SSE41-32-NEXT:    pand %xmm0, %xmm1
+; SSE41-32-NEXT:    por {{\.LCPI.*}}, %xmm1
+; SSE41-32-NEXT:    psrld $16, %xmm0
+; SSE41-32-NEXT:    por {{\.LCPI.*}}, %xmm0
+; SSE41-32-NEXT:    subps {{\.LCPI.*}}, %xmm0
+; SSE41-32-NEXT:    addps %xmm1, %xmm0
+; SSE41-32-NEXT:    retl
+;
+; SSE41-64-LABEL: uitofp_v4i32_v4f32:
+; SSE41-64:       # %bb.0:
+; SSE41-64-NEXT:    movdqa {{.*#+}} xmm1 = [65535,65535,65535,65535]
+; SSE41-64-NEXT:    pand %xmm0, %xmm1
+; SSE41-64-NEXT:    por {{.*}}(%rip), %xmm1
+; SSE41-64-NEXT:    psrld $16, %xmm0
+; SSE41-64-NEXT:    por {{.*}}(%rip), %xmm0
+; SSE41-64-NEXT:    subps {{.*}}(%rip), %xmm0
+; SSE41-64-NEXT:    addps %xmm1, %xmm0
+; SSE41-64-NEXT:    retq
+;
+; AVX1-32-LABEL: uitofp_v4i32_v4f32:
+; AVX1-32:       # %bb.0:
+; AVX1-32-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],mem[1],xmm0[2],mem[3],xmm0[4],mem[5],xmm0[6],mem[7]
+; AVX1-32-NEXT:    vpsrld $16, %xmm0, %xmm0
+; AVX1-32-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],mem[1],xmm0[2],mem[3],xmm0[4],mem[5],xmm0[6],mem[7]
+; AVX1-32-NEXT:    vsubps {{\.LCPI.*}}, %xmm0, %xmm0
+; AVX1-32-NEXT:    vaddps %xmm0, %xmm1, %xmm0
+; AVX1-32-NEXT:    retl
+;
+; AVX1-64-LABEL: uitofp_v4i32_v4f32:
+; AVX1-64:       # %bb.0:
+; AVX1-64-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],mem[1],xmm0[2],mem[3],xmm0[4],mem[5],xmm0[6],mem[7]
+; AVX1-64-NEXT:    vpsrld $16, %xmm0, %xmm0
+; AVX1-64-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],mem[1],xmm0[2],mem[3],xmm0[4],mem[5],xmm0[6],mem[7]
+; AVX1-64-NEXT:    vsubps {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-64-NEXT:    vaddps %xmm0, %xmm1, %xmm0
+; AVX1-64-NEXT:    retq
+;
+; AVX512F-LABEL: uitofp_v4i32_v4f32:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vmovaps %xmm0, %xmm0
+; AVX512F-NEXT:    vcvtudq2ps %zmm0, %zmm0
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    ret{{[l|q]}}
+;
+; AVX512VL-LABEL: uitofp_v4i32_v4f32:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vcvtudq2ps %xmm0, %xmm0
+; AVX512VL-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQ-LABEL: uitofp_v4i32_v4f32:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vmovaps %xmm0, %xmm0
+; AVX512DQ-NEXT:    vcvtudq2ps %zmm0, %zmm0
+; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQVL-LABEL: uitofp_v4i32_v4f32:
+; AVX512DQVL:       # %bb.0:
+; AVX512DQVL-NEXT:    vcvtudq2ps %xmm0, %xmm0
+; AVX512DQVL-NEXT:    ret{{[l|q]}}
+ %result = call <4 x float> @llvm.experimental.constrained.uitofp.v4f32.v4i32(<4 x i32> %x,
+                                                              metadata !"round.dynamic",
+                                                              metadata !"fpexcept.strict") #0
+  ret <4 x float> %result
+}
+
+define <2 x double> @sitofp_v2i1_v2f64(<2 x i1> %x) #0 {
+; SSE-LABEL: sitofp_v2i1_v2f64:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSE-NEXT:    pslld $31, %xmm0
+; SSE-NEXT:    psrad $31, %xmm0
+; SSE-NEXT:    cvtdq2pd %xmm0, %xmm0
+; SSE-NEXT:    ret{{[l|q]}}
+;
+; SSE41-LABEL: sitofp_v2i1_v2f64:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSE41-NEXT:    pslld $31, %xmm0
+; SSE41-NEXT:    psrad $31, %xmm0
+; SSE41-NEXT:    cvtdq2pd %xmm0, %xmm0
+; SSE41-NEXT:    ret{{[l|q]}}
+;
+; AVX-LABEL: sitofp_v2i1_v2f64:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX-NEXT:    vpslld $31, %xmm0, %xmm0
+; AVX-NEXT:    vpsrad $31, %xmm0, %xmm0
+; AVX-NEXT:    vcvtdq2pd %xmm0, %xmm0
+; AVX-NEXT:    ret{{[l|q]}}
+ %result = call <2 x double> @llvm.experimental.constrained.sitofp.v2f64.v2i1(<2 x i1> %x,
+                                                              metadata !"round.dynamic",
+                                                              metadata !"fpexcept.strict") #0
+  ret <2 x double> %result
+}
+
+define <2 x double> @uitofp_v2i1_v2f64(<2 x i1> %x) #0 {
+; SSE-32-LABEL: uitofp_v2i1_v2f64:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSE-32-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; SSE-32-NEXT:    cvtdq2pd %xmm0, %xmm0
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: uitofp_v2i1_v2f64:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSE-64-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE-64-NEXT:    cvtdq2pd %xmm0, %xmm0
+; SSE-64-NEXT:    retq
+;
+; SSE41-32-LABEL: uitofp_v2i1_v2f64:
+; SSE41-32:       # %bb.0:
+; SSE41-32-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSE41-32-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; SSE41-32-NEXT:    cvtdq2pd %xmm0, %xmm0
+; SSE41-32-NEXT:    retl
+;
+; SSE41-64-LABEL: uitofp_v2i1_v2f64:
+; SSE41-64:       # %bb.0:
+; SSE41-64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSE41-64-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE41-64-NEXT:    cvtdq2pd %xmm0, %xmm0
+; SSE41-64-NEXT:    retq
+;
+; AVX1-32-LABEL: uitofp_v2i1_v2f64:
+; AVX1-32:       # %bb.0:
+; AVX1-32-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX1-32-NEXT:    vandps {{\.LCPI.*}}, %xmm0, %xmm0
+; AVX1-32-NEXT:    vcvtdq2pd %xmm0, %xmm0
+; AVX1-32-NEXT:    retl
+;
+; AVX1-64-LABEL: uitofp_v2i1_v2f64:
+; AVX1-64:       # %bb.0:
+; AVX1-64-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX1-64-NEXT:    vandps {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-64-NEXT:    vcvtdq2pd %xmm0, %xmm0
+; AVX1-64-NEXT:    retq
+;
+; AVX512F-LABEL: uitofp_v2i1_v2f64:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX512F-NEXT:    vbroadcastss {{.*#+}} xmm1 = [1,1,1,1]
+; AVX512F-NEXT:    vandps %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    vcvtdq2pd %xmm0, %xmm0
+; AVX512F-NEXT:    ret{{[l|q]}}
+;
+; AVX512VL-32-LABEL: uitofp_v2i1_v2f64:
+; AVX512VL-32:       # %bb.0:
+; AVX512VL-32-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX512VL-32-NEXT:    vpandd {{\.LCPI.*}}{1to4}, %xmm0, %xmm0
+; AVX512VL-32-NEXT:    vcvtdq2pd %xmm0, %xmm0
+; AVX512VL-32-NEXT:    retl
+;
+; AVX512VL-64-LABEL: uitofp_v2i1_v2f64:
+; AVX512VL-64:       # %bb.0:
+; AVX512VL-64-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX512VL-64-NEXT:    vpandd {{.*}}(%rip){1to4}, %xmm0, %xmm0
+; AVX512VL-64-NEXT:    vcvtdq2pd %xmm0, %xmm0
+; AVX512VL-64-NEXT:    retq
+;
+; AVX512DQ-LABEL: uitofp_v2i1_v2f64:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX512DQ-NEXT:    vbroadcastss {{.*#+}} xmm1 = [1,1,1,1]
+; AVX512DQ-NEXT:    vandps %xmm1, %xmm0, %xmm0
+; AVX512DQ-NEXT:    vcvtdq2pd %xmm0, %xmm0
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQVL-32-LABEL: uitofp_v2i1_v2f64:
+; AVX512DQVL-32:       # %bb.0:
+; AVX512DQVL-32-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX512DQVL-32-NEXT:    vandps {{\.LCPI.*}}{1to4}, %xmm0, %xmm0
+; AVX512DQVL-32-NEXT:    vcvtdq2pd %xmm0, %xmm0
+; AVX512DQVL-32-NEXT:    retl
+;
+; AVX512DQVL-64-LABEL: uitofp_v2i1_v2f64:
+; AVX512DQVL-64:       # %bb.0:
+; AVX512DQVL-64-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX512DQVL-64-NEXT:    vandps {{.*}}(%rip){1to4}, %xmm0, %xmm0
+; AVX512DQVL-64-NEXT:    vcvtdq2pd %xmm0, %xmm0
+; AVX512DQVL-64-NEXT:    retq
+ %result = call <2 x double> @llvm.experimental.constrained.uitofp.v2f64.v2i1(<2 x i1> %x,
+                                                              metadata !"round.dynamic",
+                                                              metadata !"fpexcept.strict") #0
+  ret <2 x double> %result
+}
+
+define <2 x double> @sitofp_v2i8_v2f64(<2 x i8> %x) #0 {
+; SSE-LABEL: sitofp_v2i8_v2f64:
+; SSE:       # %bb.0:
+; SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSE-NEXT:    psrad $24, %xmm0
+; SSE-NEXT:    cvtdq2pd %xmm0, %xmm0
+; SSE-NEXT:    ret{{[l|q]}}
+;
+; SSE41-LABEL: sitofp_v2i8_v2f64:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE41-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSE41-NEXT:    psrad $24, %xmm0
+; SSE41-NEXT:    cvtdq2pd %xmm0, %xmm0
+; SSE41-NEXT:    ret{{[l|q]}}
+;
+; AVX-LABEL: sitofp_v2i8_v2f64:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpmovsxbd %xmm0, %xmm0
+; AVX-NEXT:    vcvtdq2pd %xmm0, %xmm0
+; AVX-NEXT:    ret{{[l|q]}}
+ %result = call <2 x double> @llvm.experimental.constrained.sitofp.v2f64.v2i8(<2 x i8> %x,
+                                                              metadata !"round.dynamic",
+                                                              metadata !"fpexcept.strict") #0
+  ret <2 x double> %result
+}
+
+define <2 x double> @uitofp_v2i8_v2f64(<2 x i8> %x) #0 {
+; SSE-LABEL: uitofp_v2i8_v2f64:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pxor %xmm1, %xmm1
+; SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSE-NEXT:    cvtdq2pd %xmm0, %xmm0
+; SSE-NEXT:    ret{{[l|q]}}
+;
+; SSE41-LABEL: uitofp_v2i8_v2f64:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pxor %xmm1, %xmm1
+; SSE41-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSE41-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSE41-NEXT:    cvtdq2pd %xmm0, %xmm0
+; SSE41-NEXT:    ret{{[l|q]}}
+;
+; AVX-LABEL: uitofp_v2i8_v2f64:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; AVX-NEXT:    vcvtdq2pd %xmm0, %xmm0
+; AVX-NEXT:    ret{{[l|q]}}
+ %result = call <2 x double> @llvm.experimental.constrained.uitofp.v2f64.v2i8(<2 x i8> %x,
+                                                              metadata !"round.dynamic",
+                                                              metadata !"fpexcept.strict") #0
+  ret <2 x double> %result
+}
+
+define <2 x double> @sitofp_v2i16_v2f64(<2 x i16> %x) #0 {
+; SSE-LABEL: sitofp_v2i16_v2f64:
+; SSE:       # %bb.0:
+; SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSE-NEXT:    psrad $16, %xmm0
+; SSE-NEXT:    cvtdq2pd %xmm0, %xmm0
+; SSE-NEXT:    ret{{[l|q]}}
+;
+; SSE41-LABEL: sitofp_v2i16_v2f64:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSE41-NEXT:    psrad $16, %xmm0
+; SSE41-NEXT:    cvtdq2pd %xmm0, %xmm0
+; SSE41-NEXT:    ret{{[l|q]}}
+;
+; AVX-LABEL: sitofp_v2i16_v2f64:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpmovsxwd %xmm0, %xmm0
+; AVX-NEXT:    vcvtdq2pd %xmm0, %xmm0
+; AVX-NEXT:    ret{{[l|q]}}
+ %result = call <2 x double> @llvm.experimental.constrained.sitofp.v2f64.v2i16(<2 x i16> %x,
+                                                              metadata !"round.dynamic",
+                                                              metadata !"fpexcept.strict") #0
+  ret <2 x double> %result
+}
+
+define <2 x double> @uitofp_v2i16_v2f64(<2 x i16> %x) #0 {
+; SSE-LABEL: uitofp_v2i16_v2f64:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pxor %xmm1, %xmm1
+; SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSE-NEXT:    cvtdq2pd %xmm0, %xmm0
+; SSE-NEXT:    ret{{[l|q]}}
+;
+; SSE41-LABEL: uitofp_v2i16_v2f64:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pxor %xmm1, %xmm1
+; SSE41-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSE41-NEXT:    cvtdq2pd %xmm0, %xmm0
+; SSE41-NEXT:    ret{{[l|q]}}
+;
+; AVX-LABEL: uitofp_v2i16_v2f64:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX-NEXT:    vcvtdq2pd %xmm0, %xmm0
+; AVX-NEXT:    ret{{[l|q]}}
+ %result = call <2 x double> @llvm.experimental.constrained.uitofp.v2f64.v2i16(<2 x i16> %x,
+                                                              metadata !"round.dynamic",
+                                                              metadata !"fpexcept.strict") #0
+  ret <2 x double> %result
+}
+
+define <2 x double> @sitofp_v2i32_v2f64(<2 x i32> %x) #0 {
+; SSE-LABEL: sitofp_v2i32_v2f64:
+; SSE:       # %bb.0:
+; SSE-NEXT:    cvtdq2pd %xmm0, %xmm0
+; SSE-NEXT:    ret{{[l|q]}}
+;
+; SSE41-LABEL: sitofp_v2i32_v2f64:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    cvtdq2pd %xmm0, %xmm0
+; SSE41-NEXT:    ret{{[l|q]}}
+;
+; AVX-LABEL: sitofp_v2i32_v2f64:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcvtdq2pd %xmm0, %xmm0
+; AVX-NEXT:    ret{{[l|q]}}
+ %result = call <2 x double> @llvm.experimental.constrained.sitofp.v2f64.v2i32(<2 x i32> %x,
+                                                              metadata !"round.dynamic",
+                                                              metadata !"fpexcept.strict") #0
+  ret <2 x double> %result
+}
+
+define <2 x double> @uitofp_v2i32_v2f64(<2 x i32> %x) #0 {
+; SSE-LABEL: uitofp_v2i32_v2f64:
+; SSE:       # %bb.0:
+; SSE-NEXT:    xorpd %xmm1, %xmm1
+; SSE-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE-NEXT:    movapd {{.*#+}} xmm1 = [4.503599627370496E+15,4.503599627370496E+15]
+; SSE-NEXT:    orpd %xmm1, %xmm0
+; SSE-NEXT:    subpd %xmm1, %xmm0
+; SSE-NEXT:    ret{{[l|q]}}
+;
+; SSE41-LABEL: uitofp_v2i32_v2f64:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    xorpd %xmm1, %xmm1
+; SSE41-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE41-NEXT:    movapd {{.*#+}} xmm1 = [4.503599627370496E+15,4.503599627370496E+15]
+; SSE41-NEXT:    orpd %xmm1, %xmm0
+; SSE41-NEXT:    subpd %xmm1, %xmm0
+; SSE41-NEXT:    ret{{[l|q]}}
+;
+; AVX1-LABEL: uitofp_v2i32_v2f64:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [4.503599627370496E+15,4.503599627370496E+15]
+; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vsubpd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    ret{{[l|q]}}
+;
+; AVX512F-LABEL: uitofp_v2i32_v2f64:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
+; AVX512F-NEXT:    vcvtudq2pd %ymm0, %zmm0
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    ret{{[l|q]}}
+;
+; AVX512VL-LABEL: uitofp_v2i32_v2f64:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vcvtudq2pd %xmm0, %xmm0
+; AVX512VL-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQ-LABEL: uitofp_v2i32_v2f64:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
+; AVX512DQ-NEXT:    vcvtudq2pd %ymm0, %zmm0
+; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQVL-LABEL: uitofp_v2i32_v2f64:
+; AVX512DQVL:       # %bb.0:
+; AVX512DQVL-NEXT:    vcvtudq2pd %xmm0, %xmm0
+; AVX512DQVL-NEXT:    ret{{[l|q]}}
+ %result = call <2 x double> @llvm.experimental.constrained.uitofp.v2f64.v2i32(<2 x i32> %x,
+                                                              metadata !"round.dynamic",
+                                                              metadata !"fpexcept.strict") #0
+  ret <2 x double> %result
+}
+
+define <2 x double> @sitofp_v2i64_v2f64(<2 x i64> %x) #0 {
+; SSE-32-LABEL: sitofp_v2i64_v2f64:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    pushl %ebp
+; SSE-32-NEXT:    .cfi_def_cfa_offset 8
+; SSE-32-NEXT:    .cfi_offset %ebp, -8
+; SSE-32-NEXT:    movl %esp, %ebp
+; SSE-32-NEXT:    .cfi_def_cfa_register %ebp
+; SSE-32-NEXT:    andl $-8, %esp
+; SSE-32-NEXT:    subl $32, %esp
+; SSE-32-NEXT:    movq %xmm0, {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE-32-NEXT:    movq %xmm0, {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fildll {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fstpl {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fildll {{[0-9]+}}(%esp)
+; SSE-32-NEXT:    fstpl (%esp)
+; SSE-32-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE-32-NEXT:    movhps {{.*#+}} xmm0 = xmm0[0,1],mem[0,1]
+; SSE-32-NEXT:    movl %ebp, %esp
+; SSE-32-NEXT:    popl %ebp
+; SSE-32-NEXT:    .cfi_def_cfa %esp, 4
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: sitofp_v2i64_v2f64:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movq %xmm0, %rax
+; SSE-64-NEXT:    cvtsi2sd %rax, %xmm1
+; SSE-64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE-64-NEXT:    movq %xmm0, %rax
+; SSE-64-NEXT:    xorps %xmm0, %xmm0
+; SSE-64-NEXT:    cvtsi2sd %rax, %xmm0
+; SSE-64-NEXT:    unpcklpd {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSE-64-NEXT:    movapd %xmm1, %xmm0
+; SSE-64-NEXT:    retq
+;
+; SSE41-32-LABEL: sitofp_v2i64_v2f64:
+; SSE41-32:       # %bb.0:
+; SSE41-32-NEXT:    pushl %ebp
+; SSE41-32-NEXT:    .cfi_def_cfa_offset 8
+; SSE41-32-NEXT:    .cfi_offset %ebp, -8
+; SSE41-32-NEXT:    movl %esp, %ebp
+; SSE41-32-NEXT:    .cfi_def_cfa_register %ebp
+; SSE41-32-NEXT:    andl $-8, %esp
+; SSE41-32-NEXT:    subl $32, %esp
+; SSE41-32-NEXT:    movq %xmm0, {{[0-9]+}}(%esp)
+; SSE41-32-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE41-32-NEXT:    movq %xmm0, {{[0-9]+}}(%esp)
+; SSE41-32-NEXT:    fildll {{[0-9]+}}(%esp)
+; SSE41-32-NEXT:    fstpl {{[0-9]+}}(%esp)
+; SSE41-32-NEXT:    fildll {{[0-9]+}}(%esp)
+; SSE41-32-NEXT:    fstpl (%esp)
+; SSE41-32-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE41-32-NEXT:    movhps {{.*#+}} xmm0 = xmm0[0,1],mem[0,1]
+; SSE41-32-NEXT:    movl %ebp, %esp
+; SSE41-32-NEXT:    popl %ebp
+; SSE41-32-NEXT:    .cfi_def_cfa %esp, 4
+; SSE41-32-NEXT:    retl
+;
+; SSE41-64-LABEL: sitofp_v2i64_v2f64:
+; SSE41-64:       # %bb.0:
+; SSE41-64-NEXT:    movq %xmm0, %rax
+; SSE41-64-NEXT:    cvtsi2sd %rax, %xmm1
+; SSE41-64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE41-64-NEXT:    movq %xmm0, %rax
+; SSE41-64-NEXT:    xorps %xmm0, %xmm0
+; SSE41-64-NEXT:    cvtsi2sd %rax, %xmm0
+; SSE41-64-NEXT:    unpcklpd {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSE41-64-NEXT:    movapd %xmm1, %xmm0
+; SSE41-64-NEXT:    retq
+;
+; AVX-32-LABEL: sitofp_v2i64_v2f64:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    .cfi_def_cfa_offset 8
+; AVX-32-NEXT:    .cfi_offset %ebp, -8
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    .cfi_def_cfa_register %ebp
+; AVX-32-NEXT:    andl $-8, %esp
+; AVX-32-NEXT:    subl $32, %esp
+; AVX-32-NEXT:    vmovlps %xmm0, {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX-32-NEXT:    vmovlps %xmm0, {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    fildll {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    fstpl {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    fildll {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    fstpl (%esp)
+; AVX-32-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-32-NEXT:    vmovhps {{.*#+}} xmm0 = xmm0[0,1],mem[0,1]
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    .cfi_def_cfa %esp, 4
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: sitofp_v2i64_v2f64:
+; AVX-64:       # %bb.0:
+; AVX-64-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX-64-NEXT:    vcvtsi2sd %rax, %xmm1, %xmm1
+; AVX-64-NEXT:    vmovq %xmm0, %rax
+; AVX-64-NEXT:    vcvtsi2sd %rax, %xmm2, %xmm0
+; AVX-64-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX-64-NEXT:    retq
+;
+; AVX512DQ-LABEL: sitofp_v2i64_v2f64:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vmovaps %xmm0, %xmm0
+; AVX512DQ-NEXT:    vcvtqq2pd %zmm0, %zmm0
+; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQVL-LABEL: sitofp_v2i64_v2f64:
+; AVX512DQVL:       # %bb.0:
+; AVX512DQVL-NEXT:    vcvtqq2pd %xmm0, %xmm0
+; AVX512DQVL-NEXT:    ret{{[l|q]}}
+ %result = call <2 x double> @llvm.experimental.constrained.sitofp.v2f64.v2i64(<2 x i64> %x,
+                                                              metadata !"round.dynamic",
+                                                              metadata !"fpexcept.strict") #0
+  ret <2 x double> %result
+}
+
+define <2 x double> @uitofp_v2i64_v2f64(<2 x i64> %x) #0 {
+; SSE-32-LABEL: uitofp_v2i64_v2f64:
+; SSE-32:       # %bb.0:
+; SSE-32-NEXT:    movdqa {{.*#+}} xmm1 = [4294967295,0,4294967295,0]
+; SSE-32-NEXT:    pand %xmm0, %xmm1
+; SSE-32-NEXT:    por {{\.LCPI.*}}, %xmm1
+; SSE-32-NEXT:    psrlq $32, %xmm0
+; SSE-32-NEXT:    por {{\.LCPI.*}}, %xmm0
+; SSE-32-NEXT:    subpd {{\.LCPI.*}}, %xmm0
+; SSE-32-NEXT:    addpd %xmm1, %xmm0
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: uitofp_v2i64_v2f64:
+; SSE-64:       # %bb.0:
+; SSE-64-NEXT:    movdqa {{.*#+}} xmm1 = [4294967295,4294967295]
+; SSE-64-NEXT:    pand %xmm0, %xmm1
+; SSE-64-NEXT:    por {{.*}}(%rip), %xmm1
+; SSE-64-NEXT:    psrlq $32, %xmm0
+; SSE-64-NEXT:    por {{.*}}(%rip), %xmm0
+; SSE-64-NEXT:    subpd {{.*}}(%rip), %xmm0
+; SSE-64-NEXT:    addpd %xmm1, %xmm0
+; SSE-64-NEXT:    retq
+;
+; SSE41-32-LABEL: uitofp_v2i64_v2f64:
+; SSE41-32:       # %bb.0:
+; SSE41-32-NEXT:    movdqa {{.*#+}} xmm1 = [4294967295,0,4294967295,0]
+; SSE41-32-NEXT:    pand %xmm0, %xmm1
+; SSE41-32-NEXT:    por {{\.LCPI.*}}, %xmm1
+; SSE41-32-NEXT:    psrlq $32, %xmm0
+; SSE41-32-NEXT:    por {{\.LCPI.*}}, %xmm0
+; SSE41-32-NEXT:    subpd {{\.LCPI.*}}, %xmm0
+; SSE41-32-NEXT:    addpd %xmm1, %xmm0
+; SSE41-32-NEXT:    retl
+;
+; SSE41-64-LABEL: uitofp_v2i64_v2f64:
+; SSE41-64:       # %bb.0:
+; SSE41-64-NEXT:    movdqa {{.*#+}} xmm1 = [4294967295,4294967295]
+; SSE41-64-NEXT:    pand %xmm0, %xmm1
+; SSE41-64-NEXT:    por {{.*}}(%rip), %xmm1
+; SSE41-64-NEXT:    psrlq $32, %xmm0
+; SSE41-64-NEXT:    por {{.*}}(%rip), %xmm0
+; SSE41-64-NEXT:    subpd {{.*}}(%rip), %xmm0
+; SSE41-64-NEXT:    addpd %xmm1, %xmm0
+; SSE41-64-NEXT:    retq
+;
+; AVX1-32-LABEL: uitofp_v2i64_v2f64:
+; AVX1-32:       # %bb.0:
+; AVX1-32-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX1-32-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
+; AVX1-32-NEXT:    vpor {{\.LCPI.*}}, %xmm1, %xmm1
+; AVX1-32-NEXT:    vpsrlq $32, %xmm0, %xmm0
+; AVX1-32-NEXT:    vpor {{\.LCPI.*}}, %xmm0, %xmm0
+; AVX1-32-NEXT:    vsubpd {{\.LCPI.*}}, %xmm0, %xmm0
+; AVX1-32-NEXT:    vaddpd %xmm0, %xmm1, %xmm0
+; AVX1-32-NEXT:    retl
+;
+; AVX1-64-LABEL: uitofp_v2i64_v2f64:
+; AVX1-64:       # %bb.0:
+; AVX1-64-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX1-64-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
+; AVX1-64-NEXT:    vpor {{.*}}(%rip), %xmm1, %xmm1
+; AVX1-64-NEXT:    vpsrlq $32, %xmm0, %xmm0
+; AVX1-64-NEXT:    vpor {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-64-NEXT:    vsubpd {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-64-NEXT:    vaddpd %xmm0, %xmm1, %xmm0
+; AVX1-64-NEXT:    retq
+;
+; AVX512F-32-LABEL: uitofp_v2i64_v2f64:
+; AVX512F-32:       # %bb.0:
+; AVX512F-32-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-32-NEXT:    vpblendd {{.*#+}} xmm1 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
+; AVX512F-32-NEXT:    vpor {{\.LCPI.*}}, %xmm1, %xmm1
+; AVX512F-32-NEXT:    vpsrlq $32, %xmm0, %xmm0
+; AVX512F-32-NEXT:    vpor {{\.LCPI.*}}, %xmm0, %xmm0
+; AVX512F-32-NEXT:    vsubpd {{\.LCPI.*}}, %xmm0, %xmm0
+; AVX512F-32-NEXT:    vaddpd %xmm0, %xmm1, %xmm0
+; AVX512F-32-NEXT:    retl
+;
+; AVX512F-64-LABEL: uitofp_v2i64_v2f64:
+; AVX512F-64:       # %bb.0:
+; AVX512F-64-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-64-NEXT:    vpblendd {{.*#+}} xmm1 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
+; AVX512F-64-NEXT:    vpor {{.*}}(%rip), %xmm1, %xmm1
+; AVX512F-64-NEXT:    vpsrlq $32, %xmm0, %xmm0
+; AVX512F-64-NEXT:    vpor {{.*}}(%rip), %xmm0, %xmm0
+; AVX512F-64-NEXT:    vsubpd {{.*}}(%rip), %xmm0, %xmm0
+; AVX512F-64-NEXT:    vaddpd %xmm0, %xmm1, %xmm0
+; AVX512F-64-NEXT:    retq
+;
+; AVX512VL-32-LABEL: uitofp_v2i64_v2f64:
+; AVX512VL-32:       # %bb.0:
+; AVX512VL-32-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm1
+; AVX512VL-32-NEXT:    vpor {{\.LCPI.*}}, %xmm1, %xmm1
+; AVX512VL-32-NEXT:    vpsrlq $32, %xmm0, %xmm0
+; AVX512VL-32-NEXT:    vpor {{\.LCPI.*}}, %xmm0, %xmm0
+; AVX512VL-32-NEXT:    vsubpd {{\.LCPI.*}}, %xmm0, %xmm0
+; AVX512VL-32-NEXT:    vaddpd %xmm0, %xmm1, %xmm0
+; AVX512VL-32-NEXT:    retl
+;
+; AVX512VL-64-LABEL: uitofp_v2i64_v2f64:
+; AVX512VL-64:       # %bb.0:
+; AVX512VL-64-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm1
+; AVX512VL-64-NEXT:    vpor {{.*}}(%rip), %xmm1, %xmm1
+; AVX512VL-64-NEXT:    vpsrlq $32, %xmm0, %xmm0
+; AVX512VL-64-NEXT:    vpor {{.*}}(%rip), %xmm0, %xmm0
+; AVX512VL-64-NEXT:    vsubpd {{.*}}(%rip), %xmm0, %xmm0
+; AVX512VL-64-NEXT:    vaddpd %xmm0, %xmm1, %xmm0
+; AVX512VL-64-NEXT:    retq
+;
+; AVX512DQ-LABEL: uitofp_v2i64_v2f64:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vmovaps %xmm0, %xmm0
+; AVX512DQ-NEXT:    vcvtuqq2pd %zmm0, %zmm0
+; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQVL-LABEL: uitofp_v2i64_v2f64:
+; AVX512DQVL:       # %bb.0:
+; AVX512DQVL-NEXT:    vcvtuqq2pd %xmm0, %xmm0
+; AVX512DQVL-NEXT:    ret{{[l|q]}}
+ %result = call <2 x double> @llvm.experimental.constrained.uitofp.v2f64.v2i64(<2 x i64> %x,
+                                                              metadata !"round.dynamic",
+                                                              metadata !"fpexcept.strict") #0
+  ret <2 x double> %result
+}
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/X86/vec-strict-inttofp-256.ll b/llvm/test/CodeGen/X86/vec-strict-inttofp-256.ll
new file mode 100644
index 000000000..60f0c3430
--- /dev/null
+++ b/llvm/test/CodeGen/X86/vec-strict-inttofp-256.ll
@@ -0,0 +1,1147 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx -O3 | FileCheck %s --check-prefixes=CHECK,AVX1,AVX-32,AVX1-32
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx -O3 | FileCheck %s --check-prefixes=CHECK,AVX1,AVX-64,AVX1-64
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx2 -O3 | FileCheck %s --check-prefixes=CHECK,AVX2,AVX-32,AVX2-32
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 -O3 | FileCheck %s --check-prefixes=CHECK,AVX2,AVX-64,AVX2-64
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx512f -O3 | FileCheck %s --check-prefixes=CHECK,AVX512F,AVX-32,AVX512F-32
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f -O3 | FileCheck %s --check-prefixes=CHECK,AVX512F,AVX-64,AVX512F-64
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=avx512vl -O3 | FileCheck %s --check-prefixes=CHECK,AVX512VL,AVX-32,AVX512VL-32
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512vl -O3 | FileCheck %s --check-prefixes=CHECK,AVX512VL,AVX-64,AVX512VL-64
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=avx512dq -O3 | FileCheck %s --check-prefixes=CHECK,AVX512DQ,AVX512DQ-32
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512dq -O3 | FileCheck %s --check-prefixes=CHECK,AVX512DQ,AVX512DQ-64
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=avx512dq,avx512vl -O3 | FileCheck %s --check-prefixes=CHECK,AVX512DQVL,AVX512DQVL-32
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512dq,avx512vl -O3 | FileCheck %s --check-prefixes=CHECK,AVX512DQVL,AVX512DQVL-64
+
+declare <8 x float> @llvm.experimental.constrained.sitofp.v8f32.v8i1(<8 x i1>, metadata, metadata)
+declare <8 x float> @llvm.experimental.constrained.uitofp.v8f32.v8i1(<8 x i1>, metadata, metadata)
+declare <8 x float> @llvm.experimental.constrained.sitofp.v8f32.v8i8(<8 x i8>, metadata, metadata)
+declare <8 x float> @llvm.experimental.constrained.uitofp.v8f32.v8i8(<8 x i8>, metadata, metadata)
+declare <8 x float> @llvm.experimental.constrained.sitofp.v8f32.v8i16(<8 x i16>, metadata, metadata)
+declare <8 x float> @llvm.experimental.constrained.uitofp.v8f32.v8i16(<8 x i16>, metadata, metadata)
+declare <8 x float> @llvm.experimental.constrained.sitofp.v8f32.v8i32(<8 x i32>, metadata, metadata)
+declare <8 x float> @llvm.experimental.constrained.uitofp.v8f32.v8i32(<8 x i32>, metadata, metadata)
+declare <4 x double> @llvm.experimental.constrained.sitofp.v4f64.v4i1(<4 x i1>, metadata, metadata)
+declare <4 x double> @llvm.experimental.constrained.uitofp.v4f64.v4i1(<4 x i1>, metadata, metadata)
+declare <4 x double> @llvm.experimental.constrained.sitofp.v4f64.v4i8(<4 x i8>, metadata, metadata)
+declare <4 x double> @llvm.experimental.constrained.uitofp.v4f64.v4i8(<4 x i8>, metadata, metadata)
+declare <4 x double> @llvm.experimental.constrained.sitofp.v4f64.v4i16(<4 x i16>, metadata, metadata)
+declare <4 x double> @llvm.experimental.constrained.uitofp.v4f64.v4i16(<4 x i16>, metadata, metadata)
+declare <4 x double> @llvm.experimental.constrained.sitofp.v4f64.v4i32(<4 x i32>, metadata, metadata)
+declare <4 x double> @llvm.experimental.constrained.uitofp.v4f64.v4i32(<4 x i32>, metadata, metadata)
+declare <4 x double> @llvm.experimental.constrained.sitofp.v4f64.v4i64(<4 x i64>, metadata, metadata)
+declare <4 x double> @llvm.experimental.constrained.uitofp.v4f64.v4i64(<4 x i64>, metadata, metadata)
+declare <4 x float> @llvm.experimental.constrained.sitofp.v4f32.v4i64(<4 x i64>, metadata, metadata)
+declare <4 x float> @llvm.experimental.constrained.uitofp.v4f32.v4i64(<4 x i64>, metadata, metadata)
+
+define <8 x float> @sitofp_v8i1_v8f32(<8 x i1> %x) #0 {
+; AVX1-LABEL: sitofp_v8i1_v8f32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm0[4,4,5,5,6,6,7,7]
+; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX1-NEXT:    vpslld $31, %xmm0, %xmm0
+; AVX1-NEXT:    vpsrad $31, %xmm0, %xmm0
+; AVX1-NEXT:    vpslld $31, %xmm1, %xmm1
+; AVX1-NEXT:    vpsrad $31, %xmm1, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX1-NEXT:    ret{{[l|q]}}
+;
+; AVX2-LABEL: sitofp_v8i1_v8f32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2-NEXT:    vpslld $31, %ymm0, %ymm0
+; AVX2-NEXT:    vpsrad $31, %ymm0, %ymm0
+; AVX2-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX2-NEXT:    ret{{[l|q]}}
+;
+; AVX512F-LABEL: sitofp_v8i1_v8f32:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512F-NEXT:    vpslld $31, %ymm0, %ymm0
+; AVX512F-NEXT:    vpsrad $31, %ymm0, %ymm0
+; AVX512F-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX512F-NEXT:    ret{{[l|q]}}
+;
+; AVX512VL-LABEL: sitofp_v8i1_v8f32:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512VL-NEXT:    vpslld $31, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpsrad $31, %ymm0, %ymm0
+; AVX512VL-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX512VL-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQ-LABEL: sitofp_v8i1_v8f32:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512DQ-NEXT:    vpslld $31, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpsrad $31, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQVL-LABEL: sitofp_v8i1_v8f32:
+; AVX512DQVL:       # %bb.0:
+; AVX512DQVL-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512DQVL-NEXT:    vpslld $31, %ymm0, %ymm0
+; AVX512DQVL-NEXT:    vpsrad $31, %ymm0, %ymm0
+; AVX512DQVL-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX512DQVL-NEXT:    ret{{[l|q]}}
+ %result = call <8 x float> @llvm.experimental.constrained.sitofp.v8f32.v8i1(<8 x i1> %x,
+                                                              metadata !"round.dynamic",
+                                                              metadata !"fpexcept.strict") #0
+  ret <8 x float> %result
+}
+
+define <8 x float> @uitofp_v8i1_v8f32(<8 x i1> %x) #0 {
+; AVX1-32-LABEL: uitofp_v8i1_v8f32:
+; AVX1-32:       # %bb.0:
+; AVX1-32-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; AVX1-32-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX1-32-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; AVX1-32-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX1-32-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-32-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX1-32-NEXT:    retl
+;
+; AVX1-64-LABEL: uitofp_v8i1_v8f32:
+; AVX1-64:       # %bb.0:
+; AVX1-64-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-64-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX1-64-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; AVX1-64-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX1-64-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-64-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX1-64-NEXT:    retq
+;
+; AVX2-32-LABEL: uitofp_v8i1_v8f32:
+; AVX2-32:       # %bb.0:
+; AVX2-32-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; AVX2-32-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2-32-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX2-32-NEXT:    retl
+;
+; AVX2-64-LABEL: uitofp_v8i1_v8f32:
+; AVX2-64:       # %bb.0:
+; AVX2-64-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX2-64-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2-64-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX2-64-NEXT:    retq
+;
+; AVX512F-32-LABEL: uitofp_v8i1_v8f32:
+; AVX512F-32:       # %bb.0:
+; AVX512F-32-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; AVX512F-32-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512F-32-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX512F-32-NEXT:    retl
+;
+; AVX512F-64-LABEL: uitofp_v8i1_v8f32:
+; AVX512F-64:       # %bb.0:
+; AVX512F-64-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX512F-64-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512F-64-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX512F-64-NEXT:    retq
+;
+; AVX512VL-32-LABEL: uitofp_v8i1_v8f32:
+; AVX512VL-32:       # %bb.0:
+; AVX512VL-32-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; AVX512VL-32-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512VL-32-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX512VL-32-NEXT:    retl
+;
+; AVX512VL-64-LABEL: uitofp_v8i1_v8f32:
+; AVX512VL-64:       # %bb.0:
+; AVX512VL-64-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX512VL-64-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512VL-64-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX512VL-64-NEXT:    retq
+;
+; AVX512DQ-32-LABEL: uitofp_v8i1_v8f32:
+; AVX512DQ-32:       # %bb.0:
+; AVX512DQ-32-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; AVX512DQ-32-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512DQ-32-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX512DQ-32-NEXT:    retl
+;
+; AVX512DQ-64-LABEL: uitofp_v8i1_v8f32:
+; AVX512DQ-64:       # %bb.0:
+; AVX512DQ-64-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX512DQ-64-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512DQ-64-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX512DQ-64-NEXT:    retq
+;
+; AVX512DQVL-32-LABEL: uitofp_v8i1_v8f32:
+; AVX512DQVL-32:       # %bb.0:
+; AVX512DQVL-32-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; AVX512DQVL-32-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512DQVL-32-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX512DQVL-32-NEXT:    retl
+;
+; AVX512DQVL-64-LABEL: uitofp_v8i1_v8f32:
+; AVX512DQVL-64:       # %bb.0:
+; AVX512DQVL-64-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX512DQVL-64-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512DQVL-64-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX512DQVL-64-NEXT:    retq
+ %result = call <8 x float> @llvm.experimental.constrained.uitofp.v8f32.v8i1(<8 x i1> %x,
+                                                              metadata !"round.dynamic",
+                                                              metadata !"fpexcept.strict") #0
+  ret <8 x float> %result
+}
+
+define <8 x float> @sitofp_v8i8_v8f32(<8 x i8> %x) #0 {
+; AVX1-LABEL: sitofp_v8i8_v8f32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm1
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX1-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX1-NEXT:    ret{{[l|q]}}
+;
+; AVX2-LABEL: sitofp_v8i8_v8f32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpmovsxbd %xmm0, %ymm0
+; AVX2-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX2-NEXT:    ret{{[l|q]}}
+;
+; AVX512F-LABEL: sitofp_v8i8_v8f32:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpmovsxbd %xmm0, %ymm0
+; AVX512F-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX512F-NEXT:    ret{{[l|q]}}
+;
+; AVX512VL-LABEL: sitofp_v8i8_v8f32:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpmovsxbd %xmm0, %ymm0
+; AVX512VL-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX512VL-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQ-LABEL: sitofp_v8i8_v8f32:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vpmovsxbd %xmm0, %ymm0
+; AVX512DQ-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQVL-LABEL: sitofp_v8i8_v8f32:
+; AVX512DQVL:       # %bb.0:
+; AVX512DQVL-NEXT:    vpmovsxbd %xmm0, %ymm0
+; AVX512DQVL-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX512DQVL-NEXT:    ret{{[l|q]}}
+ %result = call <8 x float> @llvm.experimental.constrained.sitofp.v8f32.v8i8(<8 x i8> %x,
+                                                              metadata !"round.dynamic",
+                                                              metadata !"fpexcept.strict") #0
+  ret <8 x float> %result
+}
+
+define <8 x float> @uitofp_v8i8_v8f32(<8 x i8> %x) #0 {
+; AVX1-LABEL: uitofp_v8i8_v8f32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX1-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX1-NEXT:    ret{{[l|q]}}
+;
+; AVX2-LABEL: uitofp_v8i8_v8f32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
+; AVX2-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX2-NEXT:    ret{{[l|q]}}
+;
+; AVX512F-LABEL: uitofp_v8i8_v8f32:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
+; AVX512F-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX512F-NEXT:    ret{{[l|q]}}
+;
+; AVX512VL-LABEL: uitofp_v8i8_v8f32:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
+; AVX512VL-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX512VL-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQ-LABEL: uitofp_v8i8_v8f32:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
+; AVX512DQ-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQVL-LABEL: uitofp_v8i8_v8f32:
+; AVX512DQVL:       # %bb.0:
+; AVX512DQVL-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
+; AVX512DQVL-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX512DQVL-NEXT:    ret{{[l|q]}}
+ %result = call <8 x float> @llvm.experimental.constrained.uitofp.v8f32.v8i8(<8 x i8> %x,
+                                                              metadata !"round.dynamic",
+                                                              metadata !"fpexcept.strict") #0
+  ret <8 x float> %result
+}
+
+define <8 x float> @sitofp_v8i16_v8f32(<8 x i16> %x) #0 {
+; AVX1-LABEL: sitofp_v8i16_v8f32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm1
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX1-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX1-NEXT:    ret{{[l|q]}}
+;
+; AVX2-LABEL: sitofp_v8i16_v8f32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
+; AVX2-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX2-NEXT:    ret{{[l|q]}}
+;
+; AVX512F-LABEL: sitofp_v8i16_v8f32:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpmovsxwd %xmm0, %ymm0
+; AVX512F-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX512F-NEXT:    ret{{[l|q]}}
+;
+; AVX512VL-LABEL: sitofp_v8i16_v8f32:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpmovsxwd %xmm0, %ymm0
+; AVX512VL-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX512VL-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQ-LABEL: sitofp_v8i16_v8f32:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vpmovsxwd %xmm0, %ymm0
+; AVX512DQ-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQVL-LABEL: sitofp_v8i16_v8f32:
+; AVX512DQVL:       # %bb.0:
+; AVX512DQVL-NEXT:    vpmovsxwd %xmm0, %ymm0
+; AVX512DQVL-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX512DQVL-NEXT:    ret{{[l|q]}}
+ %result = call <8 x float> @llvm.experimental.constrained.sitofp.v8f32.v8i16(<8 x i16> %x,
+                                                              metadata !"round.dynamic",
+                                                              metadata !"fpexcept.strict") #0
+  ret <8 x float> %result
+}
+
+define <8 x float> @uitofp_v8i16_v8f32(<8 x i16> %x) #0 {
+; AVX1-LABEL: uitofp_v8i16_v8f32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX1-NEXT:    ret{{[l|q]}}
+;
+; AVX2-LABEL: uitofp_v8i16_v8f32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX2-NEXT:    ret{{[l|q]}}
+;
+; AVX512F-LABEL: uitofp_v8i16_v8f32:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512F-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX512F-NEXT:    ret{{[l|q]}}
+;
+; AVX512VL-LABEL: uitofp_v8i16_v8f32:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512VL-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX512VL-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQ-LABEL: uitofp_v8i16_v8f32:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512DQ-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQVL-LABEL: uitofp_v8i16_v8f32:
+; AVX512DQVL:       # %bb.0:
+; AVX512DQVL-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512DQVL-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX512DQVL-NEXT:    ret{{[l|q]}}
+ %result = call <8 x float> @llvm.experimental.constrained.uitofp.v8f32.v8i16(<8 x i16> %x,
+                                                              metadata !"round.dynamic",
+                                                              metadata !"fpexcept.strict") #0
+  ret <8 x float> %result
+}
+
+define <8 x float> @sitofp_v8i32_v8f32(<8 x i32> %x) #0 {
+; CHECK-LABEL: sitofp_v8i32_v8f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
+ %result = call <8 x float> @llvm.experimental.constrained.sitofp.v8f32.v8i32(<8 x i32> %x,
+                                                              metadata !"round.dynamic",
+                                                              metadata !"fpexcept.strict") #0
+  ret <8 x float> %result
+}
+
+define <8 x float> @uitofp_v8i32_v8f32(<8 x i32> %x) #0 {
+; AVX1-32-LABEL: uitofp_v8i32_v8f32:
+; AVX1-32:       # %bb.0:
+; AVX1-32-NEXT:    vpsrld $16, %xmm0, %xmm1
+; AVX1-32-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX1-32-NEXT:    vpsrld $16, %xmm2, %xmm2
+; AVX1-32-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; AVX1-32-NEXT:    vcvtdq2ps %ymm1, %ymm1
+; AVX1-32-NEXT:    vmulps {{\.LCPI.*}}, %ymm1, %ymm1
+; AVX1-32-NEXT:    vandps {{\.LCPI.*}}, %ymm0, %ymm0
+; AVX1-32-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX1-32-NEXT:    vaddps %ymm0, %ymm1, %ymm0
+; AVX1-32-NEXT:    retl
+;
+; AVX1-64-LABEL: uitofp_v8i32_v8f32:
+; AVX1-64:       # %bb.0:
+; AVX1-64-NEXT:    vpsrld $16, %xmm0, %xmm1
+; AVX1-64-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX1-64-NEXT:    vpsrld $16, %xmm2, %xmm2
+; AVX1-64-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; AVX1-64-NEXT:    vcvtdq2ps %ymm1, %ymm1
+; AVX1-64-NEXT:    vmulps {{.*}}(%rip), %ymm1, %ymm1
+; AVX1-64-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
+; AVX1-64-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX1-64-NEXT:    vaddps %ymm0, %ymm1, %ymm0
+; AVX1-64-NEXT:    retq
+;
+; AVX2-LABEL: uitofp_v8i32_v8f32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [1258291200,1258291200,1258291200,1258291200,1258291200,1258291200,1258291200,1258291200]
+; AVX2-NEXT:    vpblendw {{.*#+}} ymm1 = ymm0[0],ymm1[1],ymm0[2],ymm1[3],ymm0[4],ymm1[5],ymm0[6],ymm1[7],ymm0[8],ymm1[9],ymm0[10],ymm1[11],ymm0[12],ymm1[13],ymm0[14],ymm1[15]
+; AVX2-NEXT:    vpsrld $16, %ymm0, %ymm0
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm2 = [1392508928,1392508928,1392508928,1392508928,1392508928,1392508928,1392508928,1392508928]
+; AVX2-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm2[1],ymm0[2],ymm2[3],ymm0[4],ymm2[5],ymm0[6],ymm2[7],ymm0[8],ymm2[9],ymm0[10],ymm2[11],ymm0[12],ymm2[13],ymm0[14],ymm2[15]
+; AVX2-NEXT:    vbroadcastss {{.*#+}} ymm2 = [5.49764202E+11,5.49764202E+11,5.49764202E+11,5.49764202E+11,5.49764202E+11,5.49764202E+11,5.49764202E+11,5.49764202E+11]
+; AVX2-NEXT:    vsubps %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vaddps %ymm0, %ymm1, %ymm0
+; AVX2-NEXT:    ret{{[l|q]}}
+;
+; AVX512F-LABEL: uitofp_v8i32_v8f32:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vmovaps %ymm0, %ymm0
+; AVX512F-NEXT:    vcvtudq2ps %zmm0, %zmm0
+; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512F-NEXT:    ret{{[l|q]}}
+;
+; AVX512VL-LABEL: uitofp_v8i32_v8f32:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vcvtudq2ps %ymm0, %ymm0
+; AVX512VL-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQ-LABEL: uitofp_v8i32_v8f32:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vmovaps %ymm0, %ymm0
+; AVX512DQ-NEXT:    vcvtudq2ps %zmm0, %zmm0
+; AVX512DQ-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQVL-LABEL: uitofp_v8i32_v8f32:
+; AVX512DQVL:       # %bb.0:
+; AVX512DQVL-NEXT:    vcvtudq2ps %ymm0, %ymm0
+; AVX512DQVL-NEXT:    ret{{[l|q]}}
+ %result = call <8 x float> @llvm.experimental.constrained.uitofp.v8f32.v8i32(<8 x i32> %x,
+                                                              metadata !"round.dynamic",
+                                                              metadata !"fpexcept.strict") #0
+  ret <8 x float> %result
+}
+
+define <4 x double> @sitofp_v4i1_v4f64(<4 x i1> %x) #0 {
+; CHECK-LABEL: sitofp_v4i1_v4f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpslld $31, %xmm0, %xmm0
+; CHECK-NEXT:    vpsrad $31, %xmm0, %xmm0
+; CHECK-NEXT:    vcvtdq2pd %xmm0, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
+ %result = call <4 x double> @llvm.experimental.constrained.sitofp.v4f64.v4i1(<4 x i1> %x,
+                                                              metadata !"round.dynamic",
+                                                              metadata !"fpexcept.strict") #0
+  ret <4 x double> %result
+}
+
+define <4 x double> @uitofp_v4i1_v4f64(<4 x i1> %x) #0 {
+; AVX1-32-LABEL: uitofp_v4i1_v4f64:
+; AVX1-32:       # %bb.0:
+; AVX1-32-NEXT:    vandps {{\.LCPI.*}}, %xmm0, %xmm0
+; AVX1-32-NEXT:    vcvtdq2pd %xmm0, %ymm0
+; AVX1-32-NEXT:    retl
+;
+; AVX1-64-LABEL: uitofp_v4i1_v4f64:
+; AVX1-64:       # %bb.0:
+; AVX1-64-NEXT:    vandps {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-64-NEXT:    vcvtdq2pd %xmm0, %ymm0
+; AVX1-64-NEXT:    retq
+;
+; AVX2-LABEL: uitofp_v4i1_v4f64:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vbroadcastss {{.*#+}} xmm1 = [1,1,1,1]
+; AVX2-NEXT:    vandps %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vcvtdq2pd %xmm0, %ymm0
+; AVX2-NEXT:    ret{{[l|q]}}
+;
+; AVX512F-LABEL: uitofp_v4i1_v4f64:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vbroadcastss {{.*#+}} xmm1 = [1,1,1,1]
+; AVX512F-NEXT:    vandps %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    vcvtdq2pd %xmm0, %ymm0
+; AVX512F-NEXT:    ret{{[l|q]}}
+;
+; AVX512VL-32-LABEL: uitofp_v4i1_v4f64:
+; AVX512VL-32:       # %bb.0:
+; AVX512VL-32-NEXT:    vpandd {{\.LCPI.*}}{1to4}, %xmm0, %xmm0
+; AVX512VL-32-NEXT:    vcvtdq2pd %xmm0, %ymm0
+; AVX512VL-32-NEXT:    retl
+;
+; AVX512VL-64-LABEL: uitofp_v4i1_v4f64:
+; AVX512VL-64:       # %bb.0:
+; AVX512VL-64-NEXT:    vpandd {{.*}}(%rip){1to4}, %xmm0, %xmm0
+; AVX512VL-64-NEXT:    vcvtdq2pd %xmm0, %ymm0
+; AVX512VL-64-NEXT:    retq
+;
+; AVX512DQ-LABEL: uitofp_v4i1_v4f64:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vbroadcastss {{.*#+}} xmm1 = [1,1,1,1]
+; AVX512DQ-NEXT:    vandps %xmm1, %xmm0, %xmm0
+; AVX512DQ-NEXT:    vcvtdq2pd %xmm0, %ymm0
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQVL-32-LABEL: uitofp_v4i1_v4f64:
+; AVX512DQVL-32:       # %bb.0:
+; AVX512DQVL-32-NEXT:    vandps {{\.LCPI.*}}{1to4}, %xmm0, %xmm0
+; AVX512DQVL-32-NEXT:    vcvtdq2pd %xmm0, %ymm0
+; AVX512DQVL-32-NEXT:    retl
+;
+; AVX512DQVL-64-LABEL: uitofp_v4i1_v4f64:
+; AVX512DQVL-64:       # %bb.0:
+; AVX512DQVL-64-NEXT:    vandps {{.*}}(%rip){1to4}, %xmm0, %xmm0
+; AVX512DQVL-64-NEXT:    vcvtdq2pd %xmm0, %ymm0
+; AVX512DQVL-64-NEXT:    retq
+ %result = call <4 x double> @llvm.experimental.constrained.uitofp.v4f64.v4i1(<4 x i1> %x,
+                                                              metadata !"round.dynamic",
+                                                              metadata !"fpexcept.strict") #0
+  ret <4 x double> %result
+}
+
+define <4 x double> @sitofp_v4i8_v4f64(<4 x i8> %x) #0 {
+; CHECK-LABEL: sitofp_v4i8_v4f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmovsxbd %xmm0, %xmm0
+; CHECK-NEXT:    vcvtdq2pd %xmm0, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
+ %result = call <4 x double> @llvm.experimental.constrained.sitofp.v4f64.v4i8(<4 x i8> %x,
+                                                              metadata !"round.dynamic",
+                                                              metadata !"fpexcept.strict") #0
+  ret <4 x double> %result
+}
+
+define <4 x double> @uitofp_v4i8_v4f64(<4 x i8> %x) #0 {
+; CHECK-LABEL: uitofp_v4i8_v4f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; CHECK-NEXT:    vcvtdq2pd %xmm0, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
+ %result = call <4 x double> @llvm.experimental.constrained.uitofp.v4f64.v4i8(<4 x i8> %x,
+                                                              metadata !"round.dynamic",
+                                                              metadata !"fpexcept.strict") #0
+  ret <4 x double> %result
+}
+
+define <4 x double> @sitofp_v4i16_v4f64(<4 x i16> %x) #0 {
+; CHECK-LABEL: sitofp_v4i16_v4f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmovsxwd %xmm0, %xmm0
+; CHECK-NEXT:    vcvtdq2pd %xmm0, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
+ %result = call <4 x double> @llvm.experimental.constrained.sitofp.v4f64.v4i16(<4 x i16> %x,
+                                                              metadata !"round.dynamic",
+                                                              metadata !"fpexcept.strict") #0
+  ret <4 x double> %result
+}
+
+define <4 x double> @uitofp_v4i16_v4f64(<4 x i16> %x) #0 {
+; CHECK-LABEL: uitofp_v4i16_v4f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; CHECK-NEXT:    vcvtdq2pd %xmm0, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
+ %result = call <4 x double> @llvm.experimental.constrained.uitofp.v4f64.v4i16(<4 x i16> %x,
+                                                              metadata !"round.dynamic",
+                                                              metadata !"fpexcept.strict") #0
+  ret <4 x double> %result
+}
+
+define <4 x double> @sitofp_v4i32_v4f64(<4 x i32> %x) #0 {
+; CHECK-LABEL: sitofp_v4i32_v4f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vcvtdq2pd %xmm0, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
+ %result = call <4 x double> @llvm.experimental.constrained.sitofp.v4f64.v4i32(<4 x i32> %x,
+                                                              metadata !"round.dynamic",
+                                                              metadata !"fpexcept.strict") #0
+  ret <4 x double> %result
+}
+
+define <4 x double> @uitofp_v4i32_v4f64(<4 x i32> %x) #0 {
+; AVX1-LABEL: uitofp_v4i32_v4f64:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm1 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    vbroadcastsd {{.*#+}} ymm1 = [4.503599627370496E+15,4.503599627370496E+15,4.503599627370496E+15,4.503599627370496E+15]
+; AVX1-NEXT:    vorpd %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vsubpd %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    ret{{[l|q]}}
+;
+; AVX2-LABEL: uitofp_v4i32_v4f64:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [4.503599627370496E+15,4.503599627370496E+15,4.503599627370496E+15,4.503599627370496E+15]
+; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vsubpd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    ret{{[l|q]}}
+;
+; AVX512F-LABEL: uitofp_v4i32_v4f64:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vmovaps %xmm0, %xmm0
+; AVX512F-NEXT:    vcvtudq2pd %ymm0, %zmm0
+; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512F-NEXT:    ret{{[l|q]}}
+;
+; AVX512VL-LABEL: uitofp_v4i32_v4f64:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vcvtudq2pd %xmm0, %ymm0
+; AVX512VL-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQ-LABEL: uitofp_v4i32_v4f64:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vmovaps %xmm0, %xmm0
+; AVX512DQ-NEXT:    vcvtudq2pd %ymm0, %zmm0
+; AVX512DQ-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQVL-LABEL: uitofp_v4i32_v4f64:
+; AVX512DQVL:       # %bb.0:
+; AVX512DQVL-NEXT:    vcvtudq2pd %xmm0, %ymm0
+; AVX512DQVL-NEXT:    ret{{[l|q]}}
+ %result = call <4 x double> @llvm.experimental.constrained.uitofp.v4f64.v4i32(<4 x i32> %x,
+                                                              metadata !"round.dynamic",
+                                                              metadata !"fpexcept.strict") #0
+  ret <4 x double> %result
+}
+
+define <4 x double> @sitofp_v4i64_v4f64(<4 x i64> %x) #0 {
+; AVX-32-LABEL: sitofp_v4i64_v4f64:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    .cfi_def_cfa_offset 8
+; AVX-32-NEXT:    .cfi_offset %ebp, -8
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    .cfi_def_cfa_register %ebp
+; AVX-32-NEXT:    andl $-8, %esp
+; AVX-32-NEXT:    subl $64, %esp
+; AVX-32-NEXT:    vmovlps %xmm0, {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-32-NEXT:    vmovlps %xmm1, {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX-32-NEXT:    vmovlps %xmm0, {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX-32-NEXT:    vmovlps %xmm0, {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    fildll {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    fstpl {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    fildll {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    fstpl {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-32-NEXT:    vmovhps {{.*#+}} xmm0 = xmm0[0,1],mem[0,1]
+; AVX-32-NEXT:    fildll {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    fstpl {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    fildll {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    fstpl (%esp)
+; AVX-32-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
+; AVX-32-NEXT:    vmovhps {{.*#+}} xmm1 = xmm1[0,1],mem[0,1]
+; AVX-32-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    .cfi_def_cfa %esp, 4
+; AVX-32-NEXT:    retl
+;
+; AVX1-64-LABEL: sitofp_v4i64_v4f64:
+; AVX1-64:       # %bb.0:
+; AVX1-64-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-64-NEXT:    vpextrq $1, %xmm1, %rax
+; AVX1-64-NEXT:    vcvtsi2sd %rax, %xmm2, %xmm2
+; AVX1-64-NEXT:    vmovq %xmm1, %rax
+; AVX1-64-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm1
+; AVX1-64-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; AVX1-64-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX1-64-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm2
+; AVX1-64-NEXT:    vmovq %xmm0, %rax
+; AVX1-64-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm0
+; AVX1-64-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; AVX1-64-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-64-NEXT:    retq
+;
+; AVX2-64-LABEL: sitofp_v4i64_v4f64:
+; AVX2-64:       # %bb.0:
+; AVX2-64-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-64-NEXT:    vpextrq $1, %xmm1, %rax
+; AVX2-64-NEXT:    vcvtsi2sd %rax, %xmm2, %xmm2
+; AVX2-64-NEXT:    vmovq %xmm1, %rax
+; AVX2-64-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm1
+; AVX2-64-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; AVX2-64-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX2-64-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm2
+; AVX2-64-NEXT:    vmovq %xmm0, %rax
+; AVX2-64-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm0
+; AVX2-64-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; AVX2-64-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-64-NEXT:    retq
+;
+; AVX512F-64-LABEL: sitofp_v4i64_v4f64:
+; AVX512F-64:       # %bb.0:
+; AVX512F-64-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-64-NEXT:    vpextrq $1, %xmm1, %rax
+; AVX512F-64-NEXT:    vcvtsi2sd %rax, %xmm2, %xmm2
+; AVX512F-64-NEXT:    vmovq %xmm1, %rax
+; AVX512F-64-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm1
+; AVX512F-64-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; AVX512F-64-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX512F-64-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm2
+; AVX512F-64-NEXT:    vmovq %xmm0, %rax
+; AVX512F-64-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm0
+; AVX512F-64-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; AVX512F-64-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX512F-64-NEXT:    retq
+;
+; AVX512VL-64-LABEL: sitofp_v4i64_v4f64:
+; AVX512VL-64:       # %bb.0:
+; AVX512VL-64-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512VL-64-NEXT:    vpextrq $1, %xmm1, %rax
+; AVX512VL-64-NEXT:    vcvtsi2sd %rax, %xmm2, %xmm2
+; AVX512VL-64-NEXT:    vmovq %xmm1, %rax
+; AVX512VL-64-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm1
+; AVX512VL-64-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; AVX512VL-64-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX512VL-64-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm2
+; AVX512VL-64-NEXT:    vmovq %xmm0, %rax
+; AVX512VL-64-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm0
+; AVX512VL-64-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; AVX512VL-64-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX512VL-64-NEXT:    retq
+;
+; AVX512DQ-LABEL: sitofp_v4i64_v4f64:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vmovaps %ymm0, %ymm0
+; AVX512DQ-NEXT:    vcvtqq2pd %zmm0, %zmm0
+; AVX512DQ-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQVL-LABEL: sitofp_v4i64_v4f64:
+; AVX512DQVL:       # %bb.0:
+; AVX512DQVL-NEXT:    vcvtqq2pd %ymm0, %ymm0
+; AVX512DQVL-NEXT:    ret{{[l|q]}}
+ %result = call <4 x double> @llvm.experimental.constrained.sitofp.v4f64.v4i64(<4 x i64> %x,
+                                                              metadata !"round.dynamic",
+                                                              metadata !"fpexcept.strict") #0
+  ret <4 x double> %result
+}
+
+define <4 x double> @uitofp_v4i64_v4f64(<4 x i64> %x) #0 {
+; AVX1-32-LABEL: uitofp_v4i64_v4f64:
+; AVX1-32:       # %bb.0:
+; AVX1-32-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; AVX1-32-NEXT:    vblendps {{.*#+}} ymm1 = ymm0[0],ymm1[1],ymm0[2],ymm1[3],ymm0[4],ymm1[5],ymm0[6],ymm1[7]
+; AVX1-32-NEXT:    vorps {{\.LCPI.*}}, %ymm1, %ymm1
+; AVX1-32-NEXT:    vpsrlq $32, %xmm0, %xmm2
+; AVX1-32-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-32-NEXT:    vpsrlq $32, %xmm0, %xmm0
+; AVX1-32-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
+; AVX1-32-NEXT:    vorpd {{\.LCPI.*}}, %ymm0, %ymm0
+; AVX1-32-NEXT:    vsubpd {{\.LCPI.*}}, %ymm0, %ymm0
+; AVX1-32-NEXT:    vaddpd %ymm0, %ymm1, %ymm0
+; AVX1-32-NEXT:    retl
+;
+; AVX1-64-LABEL: uitofp_v4i64_v4f64:
+; AVX1-64:       # %bb.0:
+; AVX1-64-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; AVX1-64-NEXT:    vblendps {{.*#+}} ymm1 = ymm0[0],ymm1[1],ymm0[2],ymm1[3],ymm0[4],ymm1[5],ymm0[6],ymm1[7]
+; AVX1-64-NEXT:    vorps {{.*}}(%rip), %ymm1, %ymm1
+; AVX1-64-NEXT:    vpsrlq $32, %xmm0, %xmm2
+; AVX1-64-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-64-NEXT:    vpsrlq $32, %xmm0, %xmm0
+; AVX1-64-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
+; AVX1-64-NEXT:    vorpd {{.*}}(%rip), %ymm0, %ymm0
+; AVX1-64-NEXT:    vsubpd {{.*}}(%rip), %ymm0, %ymm0
+; AVX1-64-NEXT:    vaddpd %ymm0, %ymm1, %ymm0
+; AVX1-64-NEXT:    retq
+;
+; AVX2-32-LABEL: uitofp_v4i64_v4f64:
+; AVX2-32:       # %bb.0:
+; AVX2-32-NEXT:    vpsrlq $32, %ymm0, %ymm1
+; AVX2-32-NEXT:    vpor {{\.LCPI.*}}, %ymm1, %ymm1
+; AVX2-32-NEXT:    vbroadcastsd {{.*#+}} ymm2 = [1.9342813118337666E+25,1.9342813118337666E+25,1.9342813118337666E+25,1.9342813118337666E+25]
+; AVX2-32-NEXT:    vsubpd %ymm2, %ymm1, %ymm1
+; AVX2-32-NEXT:    vxorpd %xmm2, %xmm2, %xmm2
+; AVX2-32-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm2[1],ymm0[2],ymm2[3],ymm0[4],ymm2[5],ymm0[6],ymm2[7]
+; AVX2-32-NEXT:    vpor {{\.LCPI.*}}, %ymm0, %ymm0
+; AVX2-32-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
+; AVX2-32-NEXT:    retl
+;
+; AVX2-64-LABEL: uitofp_v4i64_v4f64:
+; AVX2-64:       # %bb.0:
+; AVX2-64-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX2-64-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0],ymm1[1],ymm0[2],ymm1[3],ymm0[4],ymm1[5],ymm0[6],ymm1[7]
+; AVX2-64-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200]
+; AVX2-64-NEXT:    vpor %ymm2, %ymm1, %ymm1
+; AVX2-64-NEXT:    vpsrlq $32, %ymm0, %ymm0
+; AVX2-64-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [4985484787499139072,4985484787499139072,4985484787499139072,4985484787499139072]
+; AVX2-64-NEXT:    vpor %ymm2, %ymm0, %ymm0
+; AVX2-64-NEXT:    vbroadcastsd {{.*#+}} ymm2 = [1.9342813118337666E+25,1.9342813118337666E+25,1.9342813118337666E+25,1.9342813118337666E+25]
+; AVX2-64-NEXT:    vsubpd %ymm2, %ymm0, %ymm0
+; AVX2-64-NEXT:    vaddpd %ymm0, %ymm1, %ymm0
+; AVX2-64-NEXT:    retq
+;
+; AVX512F-32-LABEL: uitofp_v4i64_v4f64:
+; AVX512F-32:       # %bb.0:
+; AVX512F-32-NEXT:    vpsrlq $32, %ymm0, %ymm1
+; AVX512F-32-NEXT:    vpor {{\.LCPI.*}}, %ymm1, %ymm1
+; AVX512F-32-NEXT:    vbroadcastsd {{.*#+}} ymm2 = [1.9342813118337666E+25,1.9342813118337666E+25,1.9342813118337666E+25,1.9342813118337666E+25]
+; AVX512F-32-NEXT:    vsubpd %ymm2, %ymm1, %ymm1
+; AVX512F-32-NEXT:    vxorpd %xmm2, %xmm2, %xmm2
+; AVX512F-32-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm2[1],ymm0[2],ymm2[3],ymm0[4],ymm2[5],ymm0[6],ymm2[7]
+; AVX512F-32-NEXT:    vpor {{\.LCPI.*}}, %ymm0, %ymm0
+; AVX512F-32-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
+; AVX512F-32-NEXT:    retl
+;
+; AVX512F-64-LABEL: uitofp_v4i64_v4f64:
+; AVX512F-64:       # %bb.0:
+; AVX512F-64-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-64-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0],ymm1[1],ymm0[2],ymm1[3],ymm0[4],ymm1[5],ymm0[6],ymm1[7]
+; AVX512F-64-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200]
+; AVX512F-64-NEXT:    vpor %ymm2, %ymm1, %ymm1
+; AVX512F-64-NEXT:    vpsrlq $32, %ymm0, %ymm0
+; AVX512F-64-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [4985484787499139072,4985484787499139072,4985484787499139072,4985484787499139072]
+; AVX512F-64-NEXT:    vpor %ymm2, %ymm0, %ymm0
+; AVX512F-64-NEXT:    vbroadcastsd {{.*#+}} ymm2 = [1.9342813118337666E+25,1.9342813118337666E+25,1.9342813118337666E+25,1.9342813118337666E+25]
+; AVX512F-64-NEXT:    vsubpd %ymm2, %ymm0, %ymm0
+; AVX512F-64-NEXT:    vaddpd %ymm0, %ymm1, %ymm0
+; AVX512F-64-NEXT:    retq
+;
+; AVX512VL-32-LABEL: uitofp_v4i64_v4f64:
+; AVX512VL-32:       # %bb.0:
+; AVX512VL-32-NEXT:    vpand {{\.LCPI.*}}, %ymm0, %ymm1
+; AVX512VL-32-NEXT:    vpor {{\.LCPI.*}}, %ymm1, %ymm1
+; AVX512VL-32-NEXT:    vpsrlq $32, %ymm0, %ymm0
+; AVX512VL-32-NEXT:    vpor {{\.LCPI.*}}, %ymm0, %ymm0
+; AVX512VL-32-NEXT:    vsubpd {{\.LCPI.*}}{1to4}, %ymm0, %ymm0
+; AVX512VL-32-NEXT:    vaddpd %ymm0, %ymm1, %ymm0
+; AVX512VL-32-NEXT:    retl
+;
+; AVX512VL-64-LABEL: uitofp_v4i64_v4f64:
+; AVX512VL-64:       # %bb.0:
+; AVX512VL-64-NEXT:    vpandq {{.*}}(%rip){1to4}, %ymm0, %ymm1
+; AVX512VL-64-NEXT:    vporq {{.*}}(%rip){1to4}, %ymm1, %ymm1
+; AVX512VL-64-NEXT:    vpsrlq $32, %ymm0, %ymm0
+; AVX512VL-64-NEXT:    vporq {{.*}}(%rip){1to4}, %ymm0, %ymm0
+; AVX512VL-64-NEXT:    vsubpd {{.*}}(%rip){1to4}, %ymm0, %ymm0
+; AVX512VL-64-NEXT:    vaddpd %ymm0, %ymm1, %ymm0
+; AVX512VL-64-NEXT:    retq
+;
+; AVX512DQ-LABEL: uitofp_v4i64_v4f64:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vmovaps %ymm0, %ymm0
+; AVX512DQ-NEXT:    vcvtuqq2pd %zmm0, %zmm0
+; AVX512DQ-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQVL-LABEL: uitofp_v4i64_v4f64:
+; AVX512DQVL:       # %bb.0:
+; AVX512DQVL-NEXT:    vcvtuqq2pd %ymm0, %ymm0
+; AVX512DQVL-NEXT:    ret{{[l|q]}}
+ %result = call <4 x double> @llvm.experimental.constrained.uitofp.v4f64.v4i64(<4 x i64> %x,
+                                                              metadata !"round.dynamic",
+                                                              metadata !"fpexcept.strict") #0
+  ret <4 x double> %result
+}
+
+define <4 x float> @sitofp_v4i64_v4f32(<4 x i64> %x) #0 {
+; AVX-32-LABEL: sitofp_v4i64_v4f32:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    .cfi_def_cfa_offset 8
+; AVX-32-NEXT:    .cfi_offset %ebp, -8
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    .cfi_def_cfa_register %ebp
+; AVX-32-NEXT:    andl $-8, %esp
+; AVX-32-NEXT:    subl $48, %esp
+; AVX-32-NEXT:    vmovlps %xmm0, {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-32-NEXT:    vmovlps %xmm1, {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX-32-NEXT:    vmovlps %xmm0, {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX-32-NEXT:    vmovlps %xmm0, {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    fildll {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    fstps {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    fildll {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    fstps {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    fildll {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    fstps {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    fildll {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    fstps (%esp)
+; AVX-32-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[2,3]
+; AVX-32-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1],mem[0],xmm0[3]
+; AVX-32-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],mem[0]
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    .cfi_def_cfa %esp, 4
+; AVX-32-NEXT:    vzeroupper
+; AVX-32-NEXT:    retl
+;
+; AVX1-64-LABEL: sitofp_v4i64_v4f32:
+; AVX1-64:       # %bb.0:
+; AVX1-64-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX1-64-NEXT:    vcvtsi2ss %rax, %xmm1, %xmm1
+; AVX1-64-NEXT:    vmovq %xmm0, %rax
+; AVX1-64-NEXT:    vcvtsi2ss %rax, %xmm2, %xmm2
+; AVX1-64-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[2,3]
+; AVX1-64-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-64-NEXT:    vmovq %xmm0, %rax
+; AVX1-64-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm2
+; AVX1-64-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0],xmm1[3]
+; AVX1-64-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX1-64-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm0
+; AVX1-64-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
+; AVX1-64-NEXT:    vzeroupper
+; AVX1-64-NEXT:    retq
+;
+; AVX2-64-LABEL: sitofp_v4i64_v4f32:
+; AVX2-64:       # %bb.0:
+; AVX2-64-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX2-64-NEXT:    vcvtsi2ss %rax, %xmm1, %xmm1
+; AVX2-64-NEXT:    vmovq %xmm0, %rax
+; AVX2-64-NEXT:    vcvtsi2ss %rax, %xmm2, %xmm2
+; AVX2-64-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[2,3]
+; AVX2-64-NEXT:    vextracti128 $1, %ymm0, %xmm0
+; AVX2-64-NEXT:    vmovq %xmm0, %rax
+; AVX2-64-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm2
+; AVX2-64-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0],xmm1[3]
+; AVX2-64-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX2-64-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm0
+; AVX2-64-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
+; AVX2-64-NEXT:    vzeroupper
+; AVX2-64-NEXT:    retq
+;
+; AVX512F-64-LABEL: sitofp_v4i64_v4f32:
+; AVX512F-64:       # %bb.0:
+; AVX512F-64-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX512F-64-NEXT:    vcvtsi2ss %rax, %xmm1, %xmm1
+; AVX512F-64-NEXT:    vmovq %xmm0, %rax
+; AVX512F-64-NEXT:    vcvtsi2ss %rax, %xmm2, %xmm2
+; AVX512F-64-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[2,3]
+; AVX512F-64-NEXT:    vextracti128 $1, %ymm0, %xmm0
+; AVX512F-64-NEXT:    vmovq %xmm0, %rax
+; AVX512F-64-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm2
+; AVX512F-64-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0],xmm1[3]
+; AVX512F-64-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX512F-64-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm0
+; AVX512F-64-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
+; AVX512F-64-NEXT:    vzeroupper
+; AVX512F-64-NEXT:    retq
+;
+; AVX512VL-64-LABEL: sitofp_v4i64_v4f32:
+; AVX512VL-64:       # %bb.0:
+; AVX512VL-64-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX512VL-64-NEXT:    vcvtsi2ss %rax, %xmm1, %xmm1
+; AVX512VL-64-NEXT:    vmovq %xmm0, %rax
+; AVX512VL-64-NEXT:    vcvtsi2ss %rax, %xmm2, %xmm2
+; AVX512VL-64-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[2,3]
+; AVX512VL-64-NEXT:    vextracti128 $1, %ymm0, %xmm0
+; AVX512VL-64-NEXT:    vmovq %xmm0, %rax
+; AVX512VL-64-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm2
+; AVX512VL-64-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0],xmm1[3]
+; AVX512VL-64-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX512VL-64-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm0
+; AVX512VL-64-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
+; AVX512VL-64-NEXT:    vzeroupper
+; AVX512VL-64-NEXT:    retq
+;
+; AVX512DQ-LABEL: sitofp_v4i64_v4f32:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vmovaps %ymm0, %ymm0
+; AVX512DQ-NEXT:    vcvtqq2ps %zmm0, %ymm0
+; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQVL-LABEL: sitofp_v4i64_v4f32:
+; AVX512DQVL:       # %bb.0:
+; AVX512DQVL-NEXT:    vcvtqq2ps %ymm0, %xmm0
+; AVX512DQVL-NEXT:    vzeroupper
+; AVX512DQVL-NEXT:    ret{{[l|q]}}
+ %result = call <4 x float> @llvm.experimental.constrained.sitofp.v4f32.v4i64(<4 x i64> %x,
+                                                              metadata !"round.dynamic",
+                                                              metadata !"fpexcept.strict") #0
+  ret <4 x float> %result
+}
+
+define <4 x float> @uitofp_v4i64_v4f32(<4 x i64> %x) #0 {
+; AVX-32-LABEL: uitofp_v4i64_v4f32:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    pushl %ebp
+; AVX-32-NEXT:    .cfi_def_cfa_offset 8
+; AVX-32-NEXT:    .cfi_offset %ebp, -8
+; AVX-32-NEXT:    movl %esp, %ebp
+; AVX-32-NEXT:    .cfi_def_cfa_register %ebp
+; AVX-32-NEXT:    andl $-8, %esp
+; AVX-32-NEXT:    subl $48, %esp
+; AVX-32-NEXT:    vmovlps %xmm0, {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-32-NEXT:    vmovlps %xmm1, {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX-32-NEXT:    vmovlps %xmm1, {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    vpermilps {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; AVX-32-NEXT:    vmovlps %xmm2, {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    vextractps $1, %xmm0, %eax
+; AVX-32-NEXT:    shrl $31, %eax
+; AVX-32-NEXT:    fildll {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; AVX-32-NEXT:    fstps (%esp)
+; AVX-32-NEXT:    vextractps $3, %xmm0, %eax
+; AVX-32-NEXT:    shrl $31, %eax
+; AVX-32-NEXT:    fildll {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; AVX-32-NEXT:    fstps {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    vextractps $1, %xmm1, %eax
+; AVX-32-NEXT:    shrl $31, %eax
+; AVX-32-NEXT:    fildll {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; AVX-32-NEXT:    fstps {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    vextractps $3, %xmm1, %eax
+; AVX-32-NEXT:    shrl $31, %eax
+; AVX-32-NEXT:    fildll {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; AVX-32-NEXT:    fstps {{[0-9]+}}(%esp)
+; AVX-32-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[2,3]
+; AVX-32-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1],mem[0],xmm0[3]
+; AVX-32-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],mem[0]
+; AVX-32-NEXT:    movl %ebp, %esp
+; AVX-32-NEXT:    popl %ebp
+; AVX-32-NEXT:    .cfi_def_cfa %esp, 4
+; AVX-32-NEXT:    vzeroupper
+; AVX-32-NEXT:    retl
+;
+; AVX1-64-LABEL: uitofp_v4i64_v4f32:
+; AVX1-64:       # %bb.0:
+; AVX1-64-NEXT:    vpsrlq $1, %xmm0, %xmm1
+; AVX1-64-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX1-64-NEXT:    vpsrlq $1, %xmm2, %xmm3
+; AVX1-64-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
+; AVX1-64-NEXT:    vandpd {{.*}}(%rip), %ymm0, %ymm3
+; AVX1-64-NEXT:    vorpd %ymm3, %ymm1, %ymm1
+; AVX1-64-NEXT:    vblendvpd %xmm0, %xmm1, %xmm0, %xmm3
+; AVX1-64-NEXT:    vpextrq $1, %xmm3, %rax
+; AVX1-64-NEXT:    vcvtsi2ss %rax, %xmm4, %xmm4
+; AVX1-64-NEXT:    vmovq %xmm3, %rax
+; AVX1-64-NEXT:    vcvtsi2ss %rax, %xmm5, %xmm3
+; AVX1-64-NEXT:    vinsertps {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[2,3]
+; AVX1-64-NEXT:    vextractf128 $1, %ymm1, %xmm1
+; AVX1-64-NEXT:    vblendvpd %xmm2, %xmm1, %xmm2, %xmm1
+; AVX1-64-NEXT:    vmovq %xmm1, %rax
+; AVX1-64-NEXT:    vcvtsi2ss %rax, %xmm5, %xmm4
+; AVX1-64-NEXT:    vinsertps {{.*#+}} xmm3 = xmm3[0,1],xmm4[0],xmm3[3]
+; AVX1-64-NEXT:    vpextrq $1, %xmm1, %rax
+; AVX1-64-NEXT:    vcvtsi2ss %rax, %xmm5, %xmm1
+; AVX1-64-NEXT:    vinsertps {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[0]
+; AVX1-64-NEXT:    vaddps %xmm1, %xmm1, %xmm3
+; AVX1-64-NEXT:    vxorps %xmm4, %xmm4, %xmm4
+; AVX1-64-NEXT:    vpcmpgtq %xmm2, %xmm4, %xmm2
+; AVX1-64-NEXT:    vpackssdw %xmm2, %xmm0, %xmm0
+; AVX1-64-NEXT:    vblendvps %xmm0, %xmm3, %xmm1, %xmm0
+; AVX1-64-NEXT:    vzeroupper
+; AVX1-64-NEXT:    retq
+;
+; AVX2-64-LABEL: uitofp_v4i64_v4f32:
+; AVX2-64:       # %bb.0:
+; AVX2-64-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX2-64-NEXT:    vpcmpgtq %ymm0, %ymm1, %ymm1
+; AVX2-64-NEXT:    vextracti128 $1, %ymm1, %xmm2
+; AVX2-64-NEXT:    vpackssdw %xmm2, %xmm1, %xmm1
+; AVX2-64-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [1,1,1,1]
+; AVX2-64-NEXT:    vpand %ymm2, %ymm0, %ymm2
+; AVX2-64-NEXT:    vpsrlq $1, %ymm0, %ymm3
+; AVX2-64-NEXT:    vpor %ymm2, %ymm3, %ymm2
+; AVX2-64-NEXT:    vblendvpd %ymm0, %ymm2, %ymm0, %ymm0
+; AVX2-64-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX2-64-NEXT:    vcvtsi2ss %rax, %xmm4, %xmm2
+; AVX2-64-NEXT:    vmovq %xmm0, %rax
+; AVX2-64-NEXT:    vcvtsi2ss %rax, %xmm4, %xmm3
+; AVX2-64-NEXT:    vinsertps {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[2,3]
+; AVX2-64-NEXT:    vextracti128 $1, %ymm0, %xmm0
+; AVX2-64-NEXT:    vmovq %xmm0, %rax
+; AVX2-64-NEXT:    vcvtsi2ss %rax, %xmm4, %xmm3
+; AVX2-64-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1],xmm3[0],xmm2[3]
+; AVX2-64-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX2-64-NEXT:    vcvtsi2ss %rax, %xmm4, %xmm0
+; AVX2-64-NEXT:    vinsertps {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[0]
+; AVX2-64-NEXT:    vaddps %xmm0, %xmm0, %xmm2
+; AVX2-64-NEXT:    vblendvps %xmm1, %xmm2, %xmm0, %xmm0
+; AVX2-64-NEXT:    vzeroupper
+; AVX2-64-NEXT:    retq
+;
+; AVX512F-64-LABEL: uitofp_v4i64_v4f32:
+; AVX512F-64:       # %bb.0:
+; AVX512F-64-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX512F-64-NEXT:    vcvtusi2ss %rax, %xmm1, %xmm1
+; AVX512F-64-NEXT:    vmovq %xmm0, %rax
+; AVX512F-64-NEXT:    vcvtusi2ss %rax, %xmm2, %xmm2
+; AVX512F-64-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[2,3]
+; AVX512F-64-NEXT:    vextracti128 $1, %ymm0, %xmm0
+; AVX512F-64-NEXT:    vmovq %xmm0, %rax
+; AVX512F-64-NEXT:    vcvtusi2ss %rax, %xmm3, %xmm2
+; AVX512F-64-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0],xmm1[3]
+; AVX512F-64-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX512F-64-NEXT:    vcvtusi2ss %rax, %xmm3, %xmm0
+; AVX512F-64-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
+; AVX512F-64-NEXT:    vzeroupper
+; AVX512F-64-NEXT:    retq
+;
+; AVX512VL-64-LABEL: uitofp_v4i64_v4f32:
+; AVX512VL-64:       # %bb.0:
+; AVX512VL-64-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX512VL-64-NEXT:    vcvtusi2ss %rax, %xmm1, %xmm1
+; AVX512VL-64-NEXT:    vmovq %xmm0, %rax
+; AVX512VL-64-NEXT:    vcvtusi2ss %rax, %xmm2, %xmm2
+; AVX512VL-64-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[2,3]
+; AVX512VL-64-NEXT:    vextracti128 $1, %ymm0, %xmm0
+; AVX512VL-64-NEXT:    vmovq %xmm0, %rax
+; AVX512VL-64-NEXT:    vcvtusi2ss %rax, %xmm3, %xmm2
+; AVX512VL-64-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0],xmm1[3]
+; AVX512VL-64-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX512VL-64-NEXT:    vcvtusi2ss %rax, %xmm3, %xmm0
+; AVX512VL-64-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
+; AVX512VL-64-NEXT:    vzeroupper
+; AVX512VL-64-NEXT:    retq
+;
+; AVX512DQ-LABEL: uitofp_v4i64_v4f32:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vmovaps %ymm0, %ymm0
+; AVX512DQ-NEXT:    vcvtuqq2ps %zmm0, %ymm0
+; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQVL-LABEL: uitofp_v4i64_v4f32:
+; AVX512DQVL:       # %bb.0:
+; AVX512DQVL-NEXT:    vcvtuqq2ps %ymm0, %xmm0
+; AVX512DQVL-NEXT:    vzeroupper
+; AVX512DQVL-NEXT:    ret{{[l|q]}}
+ %result = call <4 x float> @llvm.experimental.constrained.uitofp.v4f32.v4i64(<4 x i64> %x,
+                                                              metadata !"round.dynamic",
+                                                              metadata !"fpexcept.strict") #0
+  ret <4 x float> %result
+}
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/X86/vec-strict-inttofp-512.ll b/llvm/test/CodeGen/X86/vec-strict-inttofp-512.ll
new file mode 100644
index 000000000..b61c401a6
--- /dev/null
+++ b/llvm/test/CodeGen/X86/vec-strict-inttofp-512.ll
@@ -0,0 +1,606 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx512f -O3 | FileCheck %s --check-prefixes=CHECK,NODQ-32
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f -O3 | FileCheck %s --check-prefixes=CHECK,NODQ-64
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=avx512f,avx512dq -O3 | FileCheck %s --check-prefixes=CHECK,DQ,DQ-32
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512f,avx512dq -O3 | FileCheck %s --check-prefixes=CHECK,DQ,DQ-64
+
+declare <16 x float> @llvm.experimental.constrained.sitofp.v16f32.v16i1(<16 x i1>, metadata, metadata)
+declare <16 x float> @llvm.experimental.constrained.uitofp.v16f32.v16i1(<16 x i1>, metadata, metadata)
+declare <16 x float> @llvm.experimental.constrained.sitofp.v16f32.v16i8(<16 x i8>, metadata, metadata)
+declare <16 x float> @llvm.experimental.constrained.uitofp.v16f32.v16i8(<16 x i8>, metadata, metadata)
+declare <16 x float> @llvm.experimental.constrained.sitofp.v16f32.v16i16(<16 x i16>, metadata, metadata)
+declare <16 x float> @llvm.experimental.constrained.uitofp.v16f32.v16i16(<16 x i16>, metadata, metadata)
+declare <16 x float> @llvm.experimental.constrained.sitofp.v16f32.v16i32(<16 x i32>, metadata, metadata)
+declare <16 x float> @llvm.experimental.constrained.uitofp.v16f32.v16i32(<16 x i32>, metadata, metadata)
+declare <8 x double> @llvm.experimental.constrained.sitofp.v8f64.v8i1(<8 x i1>, metadata, metadata)
+declare <8 x double> @llvm.experimental.constrained.uitofp.v8f64.v8i1(<8 x i1>, metadata, metadata)
+declare <8 x double> @llvm.experimental.constrained.sitofp.v8f64.v8i8(<8 x i8>, metadata, metadata)
+declare <8 x double> @llvm.experimental.constrained.uitofp.v8f64.v8i8(<8 x i8>, metadata, metadata)
+declare <8 x double> @llvm.experimental.constrained.sitofp.v8f64.v8i16(<8 x i16>, metadata, metadata)
+declare <8 x double> @llvm.experimental.constrained.uitofp.v8f64.v8i16(<8 x i16>, metadata, metadata)
+declare <8 x double> @llvm.experimental.constrained.sitofp.v8f64.v8i32(<8 x i32>, metadata, metadata)
+declare <8 x double> @llvm.experimental.constrained.uitofp.v8f64.v8i32(<8 x i32>, metadata, metadata)
+declare <8 x double> @llvm.experimental.constrained.sitofp.v8f64.v8i64(<8 x i64>, metadata, metadata)
+declare <8 x double> @llvm.experimental.constrained.uitofp.v8f64.v8i64(<8 x i64>, metadata, metadata)
+declare <8 x float> @llvm.experimental.constrained.sitofp.v8f32.v8i64(<8 x i64>, metadata, metadata)
+declare <8 x float> @llvm.experimental.constrained.uitofp.v8f32.v8i64(<8 x i64>, metadata, metadata)
+
+define <16 x float> @sitofp_v16i1_v16f32(<16 x i1> %x) #0 {
+; CHECK-LABEL: sitofp_v16i1_v16f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; CHECK-NEXT:    vpslld $31, %zmm0, %zmm0
+; CHECK-NEXT:    vpsrad $31, %zmm0, %zmm0
+; CHECK-NEXT:    vcvtdq2ps %zmm0, %zmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+ %result = call <16 x float> @llvm.experimental.constrained.sitofp.v16f32.v16i1(<16 x i1> %x,
+                                                              metadata !"round.dynamic",
+                                                              metadata !"fpexcept.strict") #0
+  ret <16 x float> %result
+}
+
+define <16 x float> @uitofp_v16i1_v16f32(<16 x i1> %x) #0 {
+; NODQ-32-LABEL: uitofp_v16i1_v16f32:
+; NODQ-32:       # %bb.0:
+; NODQ-32-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; NODQ-32-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; NODQ-32-NEXT:    vcvtdq2ps %zmm0, %zmm0
+; NODQ-32-NEXT:    retl
+;
+; NODQ-64-LABEL: uitofp_v16i1_v16f32:
+; NODQ-64:       # %bb.0:
+; NODQ-64-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; NODQ-64-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; NODQ-64-NEXT:    vcvtdq2ps %zmm0, %zmm0
+; NODQ-64-NEXT:    retq
+;
+; DQ-32-LABEL: uitofp_v16i1_v16f32:
+; DQ-32:       # %bb.0:
+; DQ-32-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; DQ-32-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; DQ-32-NEXT:    vcvtdq2ps %zmm0, %zmm0
+; DQ-32-NEXT:    retl
+;
+; DQ-64-LABEL: uitofp_v16i1_v16f32:
+; DQ-64:       # %bb.0:
+; DQ-64-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; DQ-64-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; DQ-64-NEXT:    vcvtdq2ps %zmm0, %zmm0
+; DQ-64-NEXT:    retq
+ %result = call <16 x float> @llvm.experimental.constrained.uitofp.v16f32.v16i1(<16 x i1> %x,
+                                                              metadata !"round.dynamic",
+                                                              metadata !"fpexcept.strict") #0
+  ret <16 x float> %result
+}
+
+define <16 x float> @sitofp_v16i8_v16f32(<16 x i8> %x) #0 {
+; CHECK-LABEL: sitofp_v16i8_v16f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmovsxbd %xmm0, %zmm0
+; CHECK-NEXT:    vcvtdq2ps %zmm0, %zmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+ %result = call <16 x float> @llvm.experimental.constrained.sitofp.v16f32.v16i8(<16 x i8> %x,
+                                                              metadata !"round.dynamic",
+                                                              metadata !"fpexcept.strict") #0
+  ret <16 x float> %result
+}
+
+define <16 x float> @uitofp_v16i8_v16f32(<16 x i8> %x) #0 {
+; CHECK-LABEL: uitofp_v16i8_v16f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; CHECK-NEXT:    vcvtdq2ps %zmm0, %zmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+ %result = call <16 x float> @llvm.experimental.constrained.uitofp.v16f32.v16i8(<16 x i8> %x,
+                                                              metadata !"round.dynamic",
+                                                              metadata !"fpexcept.strict") #0
+  ret <16 x float> %result
+}
+
+define <16 x float> @sitofp_v16i16_v16f32(<16 x i16> %x) #0 {
+; CHECK-LABEL: sitofp_v16i16_v16f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmovsxwd %ymm0, %zmm0
+; CHECK-NEXT:    vcvtdq2ps %zmm0, %zmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+ %result = call <16 x float> @llvm.experimental.constrained.sitofp.v16f32.v16i16(<16 x i16> %x,
+                                                              metadata !"round.dynamic",
+                                                              metadata !"fpexcept.strict") #0
+  ret <16 x float> %result
+}
+
+define <16 x float> @uitofp_v16i16_v16f32(<16 x i16> %x) #0 {
+; CHECK-LABEL: uitofp_v16i16_v16f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
+; CHECK-NEXT:    vcvtdq2ps %zmm0, %zmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+ %result = call <16 x float> @llvm.experimental.constrained.uitofp.v16f32.v16i16(<16 x i16> %x,
+                                                              metadata !"round.dynamic",
+                                                              metadata !"fpexcept.strict") #0
+  ret <16 x float> %result
+}
+
+define <16 x float> @sitofp_v16i32_v16f32(<16 x i32> %x) #0 {
+; CHECK-LABEL: sitofp_v16i32_v16f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vcvtdq2ps %zmm0, %zmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+ %result = call <16 x float> @llvm.experimental.constrained.sitofp.v16f32.v16i32(<16 x i32> %x,
+                                                              metadata !"round.dynamic",
+                                                              metadata !"fpexcept.strict") #0
+  ret <16 x float> %result
+}
+
+define <16 x float> @uitofp_v16i32_v16f32(<16 x i32> %x) #0 {
+; CHECK-LABEL: uitofp_v16i32_v16f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vcvtudq2ps %zmm0, %zmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+ %result = call <16 x float> @llvm.experimental.constrained.uitofp.v16f32.v16i32(<16 x i32> %x,
+                                                              metadata !"round.dynamic",
+                                                              metadata !"fpexcept.strict") #0
+  ret <16 x float> %result
+}
+
+define <8 x double> @sitofp_v8i1_v8f64(<8 x i1> %x) #0 {
+; CHECK-LABEL: sitofp_v8i1_v8f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; CHECK-NEXT:    vpslld $31, %ymm0, %ymm0
+; CHECK-NEXT:    vpsrad $31, %ymm0, %ymm0
+; CHECK-NEXT:    vcvtdq2pd %ymm0, %zmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+ %result = call <8 x double> @llvm.experimental.constrained.sitofp.v8f64.v8i1(<8 x i1> %x,
+                                                              metadata !"round.dynamic",
+                                                              metadata !"fpexcept.strict") #0
+  ret <8 x double> %result
+}
+
+define <8 x double> @uitofp_v8i1_v8f64(<8 x i1> %x) #0 {
+; NODQ-32-LABEL: uitofp_v8i1_v8f64:
+; NODQ-32:       # %bb.0:
+; NODQ-32-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; NODQ-32-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; NODQ-32-NEXT:    vcvtdq2pd %ymm0, %zmm0
+; NODQ-32-NEXT:    retl
+;
+; NODQ-64-LABEL: uitofp_v8i1_v8f64:
+; NODQ-64:       # %bb.0:
+; NODQ-64-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; NODQ-64-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; NODQ-64-NEXT:    vcvtdq2pd %ymm0, %zmm0
+; NODQ-64-NEXT:    retq
+;
+; DQ-32-LABEL: uitofp_v8i1_v8f64:
+; DQ-32:       # %bb.0:
+; DQ-32-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; DQ-32-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; DQ-32-NEXT:    vcvtdq2pd %ymm0, %zmm0
+; DQ-32-NEXT:    retl
+;
+; DQ-64-LABEL: uitofp_v8i1_v8f64:
+; DQ-64:       # %bb.0:
+; DQ-64-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; DQ-64-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; DQ-64-NEXT:    vcvtdq2pd %ymm0, %zmm0
+; DQ-64-NEXT:    retq
+ %result = call <8 x double> @llvm.experimental.constrained.uitofp.v8f64.v8i1(<8 x i1> %x,
+                                                              metadata !"round.dynamic",
+                                                              metadata !"fpexcept.strict") #0
+  ret <8 x double> %result
+}
+
+define <8 x double> @sitofp_v8i8_v8f64(<8 x i8> %x) #0 {
+; CHECK-LABEL: sitofp_v8i8_v8f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmovsxbd %xmm0, %ymm0
+; CHECK-NEXT:    vcvtdq2pd %ymm0, %zmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+ %result = call <8 x double> @llvm.experimental.constrained.sitofp.v8f64.v8i8(<8 x i8> %x,
+                                                              metadata !"round.dynamic",
+                                                              metadata !"fpexcept.strict") #0
+  ret <8 x double> %result
+}
+
+define <8 x double> @uitofp_v8i8_v8f64(<8 x i8> %x) #0 {
+; CHECK-LABEL: uitofp_v8i8_v8f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
+; CHECK-NEXT:    vcvtdq2pd %ymm0, %zmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+ %result = call <8 x double> @llvm.experimental.constrained.uitofp.v8f64.v8i8(<8 x i8> %x,
+                                                              metadata !"round.dynamic",
+                                                              metadata !"fpexcept.strict") #0
+  ret <8 x double> %result
+}
+
+define <8 x double> @sitofp_v8i16_v8f64(<8 x i16> %x) #0 {
+; CHECK-LABEL: sitofp_v8i16_v8f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmovsxwd %xmm0, %ymm0
+; CHECK-NEXT:    vcvtdq2pd %ymm0, %zmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+ %result = call <8 x double> @llvm.experimental.constrained.sitofp.v8f64.v8i16(<8 x i16> %x,
+                                                              metadata !"round.dynamic",
+                                                              metadata !"fpexcept.strict") #0
+  ret <8 x double> %result
+}
+
+define <8 x double> @uitofp_v8i16_v8f64(<8 x i16> %x) #0 {
+; CHECK-LABEL: uitofp_v8i16_v8f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; CHECK-NEXT:    vcvtdq2pd %ymm0, %zmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+ %result = call <8 x double> @llvm.experimental.constrained.uitofp.v8f64.v8i16(<8 x i16> %x,
+                                                              metadata !"round.dynamic",
+                                                              metadata !"fpexcept.strict") #0
+  ret <8 x double> %result
+}
+
+define <8 x double> @sitofp_v8i32_v8f64(<8 x i32> %x) #0 {
+; CHECK-LABEL: sitofp_v8i32_v8f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vcvtdq2pd %ymm0, %zmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+ %result = call <8 x double> @llvm.experimental.constrained.sitofp.v8f64.v8i32(<8 x i32> %x,
+                                                              metadata !"round.dynamic",
+                                                              metadata !"fpexcept.strict") #0
+  ret <8 x double> %result
+}
+
+define <8 x double> @uitofp_v8i32_v8f64(<8 x i32> %x) #0 {
+; CHECK-LABEL: uitofp_v8i32_v8f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vcvtudq2pd %ymm0, %zmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+ %result = call <8 x double> @llvm.experimental.constrained.uitofp.v8f64.v8i32(<8 x i32> %x,
+                                                              metadata !"round.dynamic",
+                                                              metadata !"fpexcept.strict") #0
+  ret <8 x double> %result
+}
+
+define <8 x double> @sitofp_v8i64_v8f64(<8 x i64> %x) #0 {
+; NODQ-32-LABEL: sitofp_v8i64_v8f64:
+; NODQ-32:       # %bb.0:
+; NODQ-32-NEXT:    pushl %ebp
+; NODQ-32-NEXT:    .cfi_def_cfa_offset 8
+; NODQ-32-NEXT:    .cfi_offset %ebp, -8
+; NODQ-32-NEXT:    movl %esp, %ebp
+; NODQ-32-NEXT:    .cfi_def_cfa_register %ebp
+; NODQ-32-NEXT:    andl $-8, %esp
+; NODQ-32-NEXT:    subl $128, %esp
+; NODQ-32-NEXT:    vextractf32x4 $2, %zmm0, %xmm1
+; NODQ-32-NEXT:    vmovlps %xmm1, {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; NODQ-32-NEXT:    vmovlps %xmm1, {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    vextractf32x4 $3, %zmm0, %xmm1
+; NODQ-32-NEXT:    vmovlps %xmm1, {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; NODQ-32-NEXT:    vmovlps %xmm1, {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    vmovlps %xmm0, {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; NODQ-32-NEXT:    vmovlps %xmm1, {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; NODQ-32-NEXT:    vmovlps %xmm0, {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; NODQ-32-NEXT:    vmovlps %xmm0, {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    fildll {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    fstpl {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    fildll {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    fstpl {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; NODQ-32-NEXT:    vmovhps {{.*#+}} xmm0 = xmm0[0,1],mem[0,1]
+; NODQ-32-NEXT:    fildll {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    fstpl {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    fildll {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    fstpl (%esp)
+; NODQ-32-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
+; NODQ-32-NEXT:    vmovhps {{.*#+}} xmm1 = xmm1[0,1],mem[0,1]
+; NODQ-32-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; NODQ-32-NEXT:    fildll {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    fstpl {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    fildll {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    fstpl {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
+; NODQ-32-NEXT:    vmovhps {{.*#+}} xmm1 = xmm1[0,1],mem[0,1]
+; NODQ-32-NEXT:    fildll {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    fstpl {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    fildll {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    fstpl {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    vmovsd {{.*#+}} xmm2 = mem[0],zero
+; NODQ-32-NEXT:    vmovhps {{.*#+}} xmm2 = xmm2[0,1],mem[0,1]
+; NODQ-32-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; NODQ-32-NEXT:    vinsertf64x4 $1, %ymm0, %zmm1, %zmm0
+; NODQ-32-NEXT:    movl %ebp, %esp
+; NODQ-32-NEXT:    popl %ebp
+; NODQ-32-NEXT:    .cfi_def_cfa %esp, 4
+; NODQ-32-NEXT:    retl
+;
+; NODQ-64-LABEL: sitofp_v8i64_v8f64:
+; NODQ-64:       # %bb.0:
+; NODQ-64-NEXT:    vextracti32x4 $3, %zmm0, %xmm1
+; NODQ-64-NEXT:    vpextrq $1, %xmm1, %rax
+; NODQ-64-NEXT:    vcvtsi2sd %rax, %xmm2, %xmm2
+; NODQ-64-NEXT:    vmovq %xmm1, %rax
+; NODQ-64-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm1
+; NODQ-64-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; NODQ-64-NEXT:    vextracti32x4 $2, %zmm0, %xmm2
+; NODQ-64-NEXT:    vpextrq $1, %xmm2, %rax
+; NODQ-64-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm3
+; NODQ-64-NEXT:    vmovq %xmm2, %rax
+; NODQ-64-NEXT:    vcvtsi2sd %rax, %xmm4, %xmm2
+; NODQ-64-NEXT:    vunpcklpd {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; NODQ-64-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
+; NODQ-64-NEXT:    vextracti128 $1, %ymm0, %xmm2
+; NODQ-64-NEXT:    vpextrq $1, %xmm2, %rax
+; NODQ-64-NEXT:    vcvtsi2sd %rax, %xmm4, %xmm3
+; NODQ-64-NEXT:    vmovq %xmm2, %rax
+; NODQ-64-NEXT:    vcvtsi2sd %rax, %xmm4, %xmm2
+; NODQ-64-NEXT:    vunpcklpd {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; NODQ-64-NEXT:    vpextrq $1, %xmm0, %rax
+; NODQ-64-NEXT:    vcvtsi2sd %rax, %xmm4, %xmm3
+; NODQ-64-NEXT:    vmovq %xmm0, %rax
+; NODQ-64-NEXT:    vcvtsi2sd %rax, %xmm4, %xmm0
+; NODQ-64-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm3[0]
+; NODQ-64-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; NODQ-64-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
+; NODQ-64-NEXT:    retq
+;
+; DQ-LABEL: sitofp_v8i64_v8f64:
+; DQ:       # %bb.0:
+; DQ-NEXT:    vcvtqq2pd %zmm0, %zmm0
+; DQ-NEXT:    ret{{[l|q]}}
+ %result = call <8 x double> @llvm.experimental.constrained.sitofp.v8f64.v8i64(<8 x i64> %x,
+                                                              metadata !"round.dynamic",
+                                                              metadata !"fpexcept.strict") #0
+  ret <8 x double> %result
+}
+
+define <8 x double> @uitofp_v8i64_v8f64(<8 x i64> %x) #0 {
+; NODQ-32-LABEL: uitofp_v8i64_v8f64:
+; NODQ-32:       # %bb.0:
+; NODQ-32-NEXT:    vpandq {{\.LCPI.*}}, %zmm0, %zmm1
+; NODQ-32-NEXT:    vporq {{\.LCPI.*}}, %zmm1, %zmm1
+; NODQ-32-NEXT:    vpsrlq $32, %zmm0, %zmm0
+; NODQ-32-NEXT:    vporq {{\.LCPI.*}}, %zmm0, %zmm0
+; NODQ-32-NEXT:    vsubpd {{\.LCPI.*}}{1to8}, %zmm0, %zmm0
+; NODQ-32-NEXT:    vaddpd %zmm0, %zmm1, %zmm0
+; NODQ-32-NEXT:    retl
+;
+; NODQ-64-LABEL: uitofp_v8i64_v8f64:
+; NODQ-64:       # %bb.0:
+; NODQ-64-NEXT:    vpandq {{.*}}(%rip){1to8}, %zmm0, %zmm1
+; NODQ-64-NEXT:    vporq {{.*}}(%rip){1to8}, %zmm1, %zmm1
+; NODQ-64-NEXT:    vpsrlq $32, %zmm0, %zmm0
+; NODQ-64-NEXT:    vporq {{.*}}(%rip){1to8}, %zmm0, %zmm0
+; NODQ-64-NEXT:    vsubpd {{.*}}(%rip){1to8}, %zmm0, %zmm0
+; NODQ-64-NEXT:    vaddpd %zmm0, %zmm1, %zmm0
+; NODQ-64-NEXT:    retq
+;
+; DQ-LABEL: uitofp_v8i64_v8f64:
+; DQ:       # %bb.0:
+; DQ-NEXT:    vcvtuqq2pd %zmm0, %zmm0
+; DQ-NEXT:    ret{{[l|q]}}
+ %result = call <8 x double> @llvm.experimental.constrained.uitofp.v8f64.v8i64(<8 x i64> %x,
+                                                              metadata !"round.dynamic",
+                                                              metadata !"fpexcept.strict") #0
+  ret <8 x double> %result
+}
+
+define <8 x float> @sitofp_v8i64_v8f32(<8 x i64> %x) #0 {
+; NODQ-32-LABEL: sitofp_v8i64_v8f32:
+; NODQ-32:       # %bb.0:
+; NODQ-32-NEXT:    pushl %ebp
+; NODQ-32-NEXT:    .cfi_def_cfa_offset 8
+; NODQ-32-NEXT:    .cfi_offset %ebp, -8
+; NODQ-32-NEXT:    movl %esp, %ebp
+; NODQ-32-NEXT:    .cfi_def_cfa_register %ebp
+; NODQ-32-NEXT:    andl $-8, %esp
+; NODQ-32-NEXT:    subl $96, %esp
+; NODQ-32-NEXT:    vmovlps %xmm0, {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; NODQ-32-NEXT:    vmovlps %xmm1, {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; NODQ-32-NEXT:    vmovlps %xmm1, {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; NODQ-32-NEXT:    vmovlps %xmm1, {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    vextractf32x4 $2, %zmm0, %xmm1
+; NODQ-32-NEXT:    vmovlps %xmm1, {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; NODQ-32-NEXT:    vmovlps %xmm1, {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    vextractf32x4 $3, %zmm0, %xmm0
+; NODQ-32-NEXT:    vmovlps %xmm0, {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; NODQ-32-NEXT:    vmovlps %xmm0, {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    fildll {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    fstps {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    fildll {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    fstps {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    fildll {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    fstps {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    fildll {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    fstps {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; NODQ-32-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[2,3]
+; NODQ-32-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1],mem[0],xmm0[3]
+; NODQ-32-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],mem[0]
+; NODQ-32-NEXT:    fildll {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    fstps {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    fildll {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    fstps {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    fildll {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    fstps {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    fildll {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    fstps (%esp)
+; NODQ-32-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; NODQ-32-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[2,3]
+; NODQ-32-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],mem[0],xmm1[3]
+; NODQ-32-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],mem[0]
+; NODQ-32-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; NODQ-32-NEXT:    movl %ebp, %esp
+; NODQ-32-NEXT:    popl %ebp
+; NODQ-32-NEXT:    .cfi_def_cfa %esp, 4
+; NODQ-32-NEXT:    retl
+;
+; NODQ-64-LABEL: sitofp_v8i64_v8f32:
+; NODQ-64:       # %bb.0:
+; NODQ-64-NEXT:    vextracti32x4 $2, %zmm0, %xmm1
+; NODQ-64-NEXT:    vpextrq $1, %xmm1, %rax
+; NODQ-64-NEXT:    vcvtsi2ss %rax, %xmm2, %xmm2
+; NODQ-64-NEXT:    vmovq %xmm1, %rax
+; NODQ-64-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm1
+; NODQ-64-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[2,3]
+; NODQ-64-NEXT:    vextracti32x4 $3, %zmm0, %xmm2
+; NODQ-64-NEXT:    vmovq %xmm2, %rax
+; NODQ-64-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm3
+; NODQ-64-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm3[0],xmm1[3]
+; NODQ-64-NEXT:    vpextrq $1, %xmm2, %rax
+; NODQ-64-NEXT:    vcvtsi2ss %rax, %xmm4, %xmm2
+; NODQ-64-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm2[0]
+; NODQ-64-NEXT:    vpextrq $1, %xmm0, %rax
+; NODQ-64-NEXT:    vcvtsi2ss %rax, %xmm4, %xmm2
+; NODQ-64-NEXT:    vmovq %xmm0, %rax
+; NODQ-64-NEXT:    vcvtsi2ss %rax, %xmm4, %xmm3
+; NODQ-64-NEXT:    vinsertps {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[2,3]
+; NODQ-64-NEXT:    vextracti128 $1, %ymm0, %xmm0
+; NODQ-64-NEXT:    vmovq %xmm0, %rax
+; NODQ-64-NEXT:    vcvtsi2ss %rax, %xmm4, %xmm3
+; NODQ-64-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1],xmm3[0],xmm2[3]
+; NODQ-64-NEXT:    vpextrq $1, %xmm0, %rax
+; NODQ-64-NEXT:    vcvtsi2ss %rax, %xmm4, %xmm0
+; NODQ-64-NEXT:    vinsertps {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[0]
+; NODQ-64-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; NODQ-64-NEXT:    retq
+;
+; DQ-LABEL: sitofp_v8i64_v8f32:
+; DQ:       # %bb.0:
+; DQ-NEXT:    vcvtqq2ps %zmm0, %ymm0
+; DQ-NEXT:    ret{{[l|q]}}
+ %result = call <8 x float> @llvm.experimental.constrained.sitofp.v8f32.v8i64(<8 x i64> %x,
+                                                              metadata !"round.dynamic",
+                                                              metadata !"fpexcept.strict") #0
+  ret <8 x float> %result
+}
+
+define <8 x float> @uitofp_v8i64_v8f32(<8 x i64> %x) #0 {
+; NODQ-32-LABEL: uitofp_v8i64_v8f32:
+; NODQ-32:       # %bb.0:
+; NODQ-32-NEXT:    pushl %ebp
+; NODQ-32-NEXT:    .cfi_def_cfa_offset 8
+; NODQ-32-NEXT:    .cfi_offset %ebp, -8
+; NODQ-32-NEXT:    movl %esp, %ebp
+; NODQ-32-NEXT:    .cfi_def_cfa_register %ebp
+; NODQ-32-NEXT:    andl $-8, %esp
+; NODQ-32-NEXT:    subl $96, %esp
+; NODQ-32-NEXT:    vmovlps %xmm0, {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; NODQ-32-NEXT:    vmovlps %xmm1, {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; NODQ-32-NEXT:    vmovlps %xmm3, {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    vpermilps {{.*#+}} xmm1 = xmm3[2,3,0,1]
+; NODQ-32-NEXT:    vmovlps %xmm1, {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    vextractf32x4 $2, %zmm0, %xmm2
+; NODQ-32-NEXT:    vmovlps %xmm2, {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    vpermilps {{.*#+}} xmm1 = xmm2[2,3,0,1]
+; NODQ-32-NEXT:    vmovlps %xmm1, {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    vextractf32x4 $3, %zmm0, %xmm1
+; NODQ-32-NEXT:    vmovlps %xmm1, {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    vpermilps {{.*#+}} xmm4 = xmm1[2,3,0,1]
+; NODQ-32-NEXT:    vmovlps %xmm4, {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    vextractps $1, %xmm0, %eax
+; NODQ-32-NEXT:    shrl $31, %eax
+; NODQ-32-NEXT:    fildll {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; NODQ-32-NEXT:    fstps (%esp)
+; NODQ-32-NEXT:    vextractps $3, %xmm0, %eax
+; NODQ-32-NEXT:    shrl $31, %eax
+; NODQ-32-NEXT:    fildll {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; NODQ-32-NEXT:    fstps {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    vextractps $1, %xmm3, %eax
+; NODQ-32-NEXT:    shrl $31, %eax
+; NODQ-32-NEXT:    fildll {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; NODQ-32-NEXT:    fstps {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    vextractps $3, %xmm3, %eax
+; NODQ-32-NEXT:    shrl $31, %eax
+; NODQ-32-NEXT:    fildll {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; NODQ-32-NEXT:    fstps {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    vextractps $1, %xmm2, %eax
+; NODQ-32-NEXT:    shrl $31, %eax
+; NODQ-32-NEXT:    fildll {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; NODQ-32-NEXT:    fstps {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    vextractps $3, %xmm2, %eax
+; NODQ-32-NEXT:    shrl $31, %eax
+; NODQ-32-NEXT:    fildll {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; NODQ-32-NEXT:    fstps {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    vextractps $1, %xmm1, %eax
+; NODQ-32-NEXT:    shrl $31, %eax
+; NODQ-32-NEXT:    fildll {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; NODQ-32-NEXT:    fstps {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    vextractps $3, %xmm1, %eax
+; NODQ-32-NEXT:    shrl $31, %eax
+; NODQ-32-NEXT:    fildll {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; NODQ-32-NEXT:    fstps {{[0-9]+}}(%esp)
+; NODQ-32-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; NODQ-32-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[2,3]
+; NODQ-32-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1],mem[0],xmm0[3]
+; NODQ-32-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],mem[0]
+; NODQ-32-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; NODQ-32-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[2,3]
+; NODQ-32-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],mem[0],xmm1[3]
+; NODQ-32-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],mem[0]
+; NODQ-32-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; NODQ-32-NEXT:    movl %ebp, %esp
+; NODQ-32-NEXT:    popl %ebp
+; NODQ-32-NEXT:    .cfi_def_cfa %esp, 4
+; NODQ-32-NEXT:    retl
+;
+; NODQ-64-LABEL: uitofp_v8i64_v8f32:
+; NODQ-64:       # %bb.0:
+; NODQ-64-NEXT:    vextracti32x4 $2, %zmm0, %xmm1
+; NODQ-64-NEXT:    vpextrq $1, %xmm1, %rax
+; NODQ-64-NEXT:    vcvtusi2ss %rax, %xmm2, %xmm2
+; NODQ-64-NEXT:    vmovq %xmm1, %rax
+; NODQ-64-NEXT:    vcvtusi2ss %rax, %xmm3, %xmm1
+; NODQ-64-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[2,3]
+; NODQ-64-NEXT:    vextracti32x4 $3, %zmm0, %xmm2
+; NODQ-64-NEXT:    vmovq %xmm2, %rax
+; NODQ-64-NEXT:    vcvtusi2ss %rax, %xmm3, %xmm3
+; NODQ-64-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm3[0],xmm1[3]
+; NODQ-64-NEXT:    vpextrq $1, %xmm2, %rax
+; NODQ-64-NEXT:    vcvtusi2ss %rax, %xmm4, %xmm2
+; NODQ-64-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm2[0]
+; NODQ-64-NEXT:    vpextrq $1, %xmm0, %rax
+; NODQ-64-NEXT:    vcvtusi2ss %rax, %xmm4, %xmm2
+; NODQ-64-NEXT:    vmovq %xmm0, %rax
+; NODQ-64-NEXT:    vcvtusi2ss %rax, %xmm4, %xmm3
+; NODQ-64-NEXT:    vinsertps {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[2,3]
+; NODQ-64-NEXT:    vextracti128 $1, %ymm0, %xmm0
+; NODQ-64-NEXT:    vmovq %xmm0, %rax
+; NODQ-64-NEXT:    vcvtusi2ss %rax, %xmm4, %xmm3
+; NODQ-64-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1],xmm3[0],xmm2[3]
+; NODQ-64-NEXT:    vpextrq $1, %xmm0, %rax
+; NODQ-64-NEXT:    vcvtusi2ss %rax, %xmm4, %xmm0
+; NODQ-64-NEXT:    vinsertps {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[0]
+; NODQ-64-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; NODQ-64-NEXT:    retq
+;
+; DQ-LABEL: uitofp_v8i64_v8f32:
+; DQ:       # %bb.0:
+; DQ-NEXT:    vcvtuqq2ps %zmm0, %ymm0
+; DQ-NEXT:    ret{{[l|q]}}
+ %result = call <8 x float> @llvm.experimental.constrained.uitofp.v8f32.v8i64(<8 x i64> %x,
+                                                              metadata !"round.dynamic",
+                                                              metadata !"fpexcept.strict") #0
+  ret <8 x float> %result
+}
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/X86/vec-strict-round-128.ll b/llvm/test/CodeGen/X86/vec-strict-round-128.ll
new file mode 100644
index 000000000..3f8b4f84a
--- /dev/null
+++ b/llvm/test/CodeGen/X86/vec-strict-round-128.ll
@@ -0,0 +1,174 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse4.1 -O3 | FileCheck %s --check-prefixes=SSE41
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 -O3 | FileCheck %s --check-prefixes=SSE41
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx -O3 | FileCheck %s --check-prefixes=AVX
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx -O3 | FileCheck %s --check-prefixes=AVX
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx512f -O3 | FileCheck %s --check-prefixes=AVX
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f -O3 | FileCheck %s --check-prefixes=AVX
+
+declare <4 x float> @llvm.experimental.constrained.ceil.v4f32(<4 x float>, metadata)
+declare <2 x double>  @llvm.experimental.constrained.ceil.v2f64(<2 x double>, metadata)
+declare <4 x float> @llvm.experimental.constrained.floor.v4f32(<4 x float>, metadata)
+declare <2 x double> @llvm.experimental.constrained.floor.v2f64(<2 x double>, metadata)
+declare <4 x float> @llvm.experimental.constrained.trunc.v4f32(<4 x float>, metadata)
+declare <2 x double> @llvm.experimental.constrained.trunc.v2f64(<2 x double>, metadata)
+declare <4 x float> @llvm.experimental.constrained.rint.v4f32(<4 x float>, metadata, metadata)
+declare <2 x double> @llvm.experimental.constrained.rint.v2f64(<2 x double>, metadata, metadata)
+declare <4 x float> @llvm.experimental.constrained.nearbyint.v4f32(<4 x float>, metadata, metadata)
+declare <2 x double> @llvm.experimental.constrained.nearbyint.v2f64(<2 x double>, metadata, metadata)
+
+define <4 x float> @fceilv4f32(<4 x float> %f) #0 {
+; SSE41-LABEL: fceilv4f32:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    roundps $10, %xmm0, %xmm0
+; SSE41-NEXT:    ret{{[l|q]}}
+;
+; AVX-LABEL: fceilv4f32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vroundps $10, %xmm0, %xmm0
+; AVX-NEXT:    ret{{[l|q]}}
+  %res = call <4 x float> @llvm.experimental.constrained.ceil.v4f32(
+                          <4 x float> %f, metadata !"fpexcept.strict")
+  ret <4 x float> %res
+}
+
+define <2 x double> @fceilv2f64(<2 x double> %f) #0 {
+; SSE41-LABEL: fceilv2f64:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    roundpd $10, %xmm0, %xmm0
+; SSE41-NEXT:    ret{{[l|q]}}
+;
+; AVX-LABEL: fceilv2f64:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vroundpd $10, %xmm0, %xmm0
+; AVX-NEXT:    ret{{[l|q]}}
+  %res = call <2 x double> @llvm.experimental.constrained.ceil.v2f64(
+                        <2 x double> %f, metadata !"fpexcept.strict")
+  ret <2 x double> %res
+}
+
+define <4 x float> @ffloorv4f32(<4 x float> %f) #0 {
+; SSE41-LABEL: ffloorv4f32:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    roundps $9, %xmm0, %xmm0
+; SSE41-NEXT:    ret{{[l|q]}}
+;
+; AVX-LABEL: ffloorv4f32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vroundps $9, %xmm0, %xmm0
+; AVX-NEXT:    ret{{[l|q]}}
+  %res = call <4 x float> @llvm.experimental.constrained.floor.v4f32(
+                          <4 x float> %f, metadata !"fpexcept.strict")
+  ret <4 x float> %res
+}
+
+define <2 x double> @ffloorv2f64(<2 x double> %f) #0 {
+; SSE41-LABEL: ffloorv2f64:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    roundpd $9, %xmm0, %xmm0
+; SSE41-NEXT:    ret{{[l|q]}}
+;
+; AVX-LABEL: ffloorv2f64:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vroundpd $9, %xmm0, %xmm0
+; AVX-NEXT:    ret{{[l|q]}}
+  %res = call <2 x double> @llvm.experimental.constrained.floor.v2f64(
+                        <2 x double> %f, metadata !"fpexcept.strict")
+  ret <2 x double> %res
+}
+
+define <4 x float> @ftruncv4f32(<4 x float> %f) #0 {
+; SSE41-LABEL: ftruncv4f32:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    roundps $11, %xmm0, %xmm0
+; SSE41-NEXT:    ret{{[l|q]}}
+;
+; AVX-LABEL: ftruncv4f32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vroundps $11, %xmm0, %xmm0
+; AVX-NEXT:    ret{{[l|q]}}
+  %res = call <4 x float> @llvm.experimental.constrained.trunc.v4f32(
+                          <4 x float> %f, metadata !"fpexcept.strict")
+  ret <4 x float> %res
+}
+
+define <2 x double> @ftruncv2f64(<2 x double> %f) #0 {
+; SSE41-LABEL: ftruncv2f64:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    roundpd $11, %xmm0, %xmm0
+; SSE41-NEXT:    ret{{[l|q]}}
+;
+; AVX-LABEL: ftruncv2f64:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vroundpd $11, %xmm0, %xmm0
+; AVX-NEXT:    ret{{[l|q]}}
+  %res = call <2 x double> @llvm.experimental.constrained.trunc.v2f64(
+                        <2 x double> %f, metadata !"fpexcept.strict")
+  ret <2 x double> %res
+}
+
+define <4 x float> @frintv4f32(<4 x float> %f) #0 {
+; SSE41-LABEL: frintv4f32:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    roundps $4, %xmm0, %xmm0
+; SSE41-NEXT:    ret{{[l|q]}}
+;
+; AVX-LABEL: frintv4f32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vroundps $4, %xmm0, %xmm0
+; AVX-NEXT:    ret{{[l|q]}}
+  %res = call <4 x float> @llvm.experimental.constrained.rint.v4f32(
+                          <4 x float> %f,
+                          metadata !"round.dynamic", metadata !"fpexcept.strict")
+  ret <4 x float> %res
+}
+
+define <2 x double> @frintv2f64(<2 x double> %f) #0 {
+; SSE41-LABEL: frintv2f64:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    roundpd $4, %xmm0, %xmm0
+; SSE41-NEXT:    ret{{[l|q]}}
+;
+; AVX-LABEL: frintv2f64:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vroundpd $4, %xmm0, %xmm0
+; AVX-NEXT:    ret{{[l|q]}}
+  %res = call <2 x double> @llvm.experimental.constrained.rint.v2f64(
+                        <2 x double> %f,
+                        metadata !"round.dynamic", metadata !"fpexcept.strict")
+  ret <2 x double> %res
+}
+
+define <4 x float> @fnearbyintv4f32(<4 x float> %f) #0 {
+; SSE41-LABEL: fnearbyintv4f32:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    roundps $12, %xmm0, %xmm0
+; SSE41-NEXT:    ret{{[l|q]}}
+;
+; AVX-LABEL: fnearbyintv4f32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vroundps $12, %xmm0, %xmm0
+; AVX-NEXT:    ret{{[l|q]}}
+  %res = call <4 x float> @llvm.experimental.constrained.nearbyint.v4f32(
+                          <4 x float> %f,
+                          metadata !"round.dynamic", metadata !"fpexcept.strict")
+  ret <4 x float> %res
+}
+
+define <2 x double> @fnearbyintv2f64(<2 x double> %f) #0 {
+; SSE41-LABEL: fnearbyintv2f64:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    roundpd $12, %xmm0, %xmm0
+; SSE41-NEXT:    ret{{[l|q]}}
+;
+; AVX-LABEL: fnearbyintv2f64:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vroundpd $12, %xmm0, %xmm0
+; AVX-NEXT:    ret{{[l|q]}}
+  %res = call <2 x double> @llvm.experimental.constrained.nearbyint.v2f64(
+                        <2 x double> %f,
+                        metadata !"round.dynamic", metadata !"fpexcept.strict")
+  ret <2 x double> %res
+}
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/X86/vec_align.ll b/llvm/test/CodeGen/X86/vec_align.ll
index 558d768ac..5942db099 100644
--- a/llvm/test/CodeGen/X86/vec_align.ll
+++ b/llvm/test/CodeGen/X86/vec_align.ll
@@ -1,4 +1,5 @@
-; RUN: llc < %s -mcpu=yonah -relocation-model=static | grep movaps | count 2
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mcpu=yonah -relocation-model=static | FileCheck %s
 
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
 target triple = "i686-apple-darwin8"
@@ -8,6 +9,10 @@ target triple = "i686-apple-darwin8"
 @G = external global { float,float,float,float}, align 16
 
 define %f4 @test1(float %W, float %X, float %Y, float %Z) nounwind {
+; CHECK-LABEL: test1:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    movaps {{[0-9]+}}(%esp), %xmm0
+; CHECK-NEXT:    retl
         %tmp = insertelement %f4 undef, float %W, i32 0
         %tmp2 = insertelement %f4 %tmp, float %X, i32 1
         %tmp4 = insertelement %f4 %tmp2, float %Y, i32 2
@@ -16,11 +21,15 @@ define %f4 @test1(float %W, float %X, float %Y, float %Z) nounwind {
 }
 
 define %f4 @test2() nounwind {
+; CHECK-LABEL: test2:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    movaps _G, %xmm0
+; CHECK-NEXT:    retl
 	%Wp = getelementptr { float,float,float,float}, { float,float,float,float}* @G, i32 0, i32 0
 	%Xp = getelementptr { float,float,float,float}, { float,float,float,float}* @G, i32 0, i32 1
 	%Yp = getelementptr { float,float,float,float}, { float,float,float,float}* @G, i32 0, i32 2
 	%Zp = getelementptr { float,float,float,float}, { float,float,float,float}* @G, i32 0, i32 3
-	
+
 	%W = load float, float* %Wp
 	%X = load float, float* %Xp
 	%Y = load float, float* %Yp
diff --git a/llvm/test/CodeGen/X86/vec_align_i256.ll b/llvm/test/CodeGen/X86/vec_align_i256.ll
index 078bcb154..999e88c64 100644
--- a/llvm/test/CodeGen/X86/vec_align_i256.ll
+++ b/llvm/test/CodeGen/X86/vec_align_i256.ll
@@ -1,14 +1,16 @@
-; RUN: llc < %s -mcpu=corei7-avx | FileCheck %s 
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mcpu=corei7-avx | FileCheck %s
 
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
 target triple = "i686-apple-darwin8"
 
 ; Make sure that we are not generating a movaps because the vector is aligned to 1.
-;CHECK: @foo
-;CHECK: xor
-;CHECK-NEXT: vmovups
-;CHECK-NEXT: ret
 define void @foo() {
+; CHECK-LABEL: foo:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; CHECK-NEXT:    vmovups %xmm0, (%eax)
+; CHECK-NEXT:    retl
   store <16 x i16> zeroinitializer, <16 x i16>* undef, align 1
   ret void
 }
diff --git a/llvm/test/CodeGen/X86/vec_anyext.ll b/llvm/test/CodeGen/X86/vec_anyext.ll
index b42fbb55b..75564b596 100644
--- a/llvm/test/CodeGen/X86/vec_anyext.ll
+++ b/llvm/test/CodeGen/X86/vec_anyext.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64--
 ; PR 9267
 
diff --git a/llvm/test/CodeGen/X86/vec_call.ll b/llvm/test/CodeGen/X86/vec_call.ll
index 8c2d8ce66..e0f95ec31 100644
--- a/llvm/test/CodeGen/X86/vec_call.ll
+++ b/llvm/test/CodeGen/X86/vec_call.ll
@@ -1,10 +1,28 @@
-; RUN: llc < %s -mcpu=generic -mattr=+sse2 -mtriple=i686-apple-darwin8 | \
-; RUN:   grep "subl.*60"
-; RUN: llc < %s -mcpu=generic -mattr=+sse2 -mtriple=i686-apple-darwin8 | \
-; RUN:   grep "movaps.*32"
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mcpu=generic -mattr=+sse2 -mtriple=i686-apple-darwin8 | FileCheck %s
 
 
 define void @test() {
+; CHECK-LABEL: test:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    subl $60, %esp
+; CHECK-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-NEXT:    movaps {{.*#+}} xmm0 = [0,1,3,9]
+; CHECK-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movl $7, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movl $6, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movl $5, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movl $4, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movl $3, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movl $2, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movl $1, (%esp)
+; CHECK-NEXT:    movaps {{.*#+}} xmm0 = [4,3,2,1]
+; CHECK-NEXT:    movaps {{.*#+}} xmm1 = [8,7,6,5]
+; CHECK-NEXT:    movaps {{.*#+}} xmm2 = [6,4,2,0]
+; CHECK-NEXT:    movaps {{.*#+}} xmm3 = [8,4,2,1]
+; CHECK-NEXT:    calll _xx
+; CHECK-NEXT:    addl $60, %esp
+; CHECK-NEXT:    retl
         tail call void @xx( i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, <2 x i64> bitcast (<4 x i32> < i32 4, i32 3, i32 2, i32 1 > to <2 x i64>), <2 x i64> bitcast (<4 x i32> < i32 8, i32 7, i32 6, i32 5 > to <2 x i64>), <2 x i64> bitcast (<4 x i32> < i32 6, i32 4, i32 2, i32 0 > to <2 x i64>), <2 x i64> bitcast (<4 x i32> < i32 8, i32 4, i32 2, i32 1 > to <2 x i64>), <2 x i64> bitcast (<4 x i32> < i32 0, i32 1, i32 3, i32 9 > to <2 x i64>) )
         ret void
 }
diff --git a/llvm/test/CodeGen/X86/vec_cast2.ll b/llvm/test/CodeGen/X86/vec_cast2.ll
index 8e1ffe683..e43216dcd 100644
--- a/llvm/test/CodeGen/X86/vec_cast2.ll
+++ b/llvm/test/CodeGen/X86/vec_cast2.ll
@@ -1,28 +1,15 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=i386-apple-darwin10 -mattr=+avx | FileCheck %s
-; RUN: llc < %s -mtriple=i386-apple-darwin10 -mattr=+avx -x86-experimental-vector-widening-legalization | FileCheck %s --check-prefix=CHECK-WIDE
 
 define <8 x float> @cvt_v8i8_v8f32(<8 x i8> %src) {
 ; CHECK-LABEL: cvt_v8i8_v8f32:
 ; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm0[4,4,5,5,6,6,7,7]
-; CHECK-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; CHECK-NEXT:    vpslld $24, %xmm0, %xmm0
-; CHECK-NEXT:    vpsrad $24, %xmm0, %xmm0
-; CHECK-NEXT:    vpslld $24, %xmm1, %xmm1
-; CHECK-NEXT:    vpsrad $24, %xmm1, %xmm1
-; CHECK-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; CHECK-NEXT:    vpmovsxbd %xmm0, %xmm1
+; CHECK-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; CHECK-NEXT:    vpmovsxbd %xmm0, %xmm0
+; CHECK-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; CHECK-NEXT:    vcvtdq2ps %ymm0, %ymm0
 ; CHECK-NEXT:    retl
-;
-; CHECK-WIDE-LABEL: cvt_v8i8_v8f32:
-; CHECK-WIDE:       ## %bb.0:
-; CHECK-WIDE-NEXT:    vpmovsxbd %xmm0, %xmm1
-; CHECK-WIDE-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; CHECK-WIDE-NEXT:    vpmovsxbd %xmm0, %xmm0
-; CHECK-WIDE-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; CHECK-WIDE-NEXT:    vcvtdq2ps %ymm0, %ymm0
-; CHECK-WIDE-NEXT:    retl
   %res = sitofp <8 x i8> %src to <8 x float>
   ret <8 x float> %res
 }
@@ -36,15 +23,6 @@ define <8 x float> @cvt_v8i16_v8f32(<8 x i16> %src) {
 ; CHECK-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; CHECK-NEXT:    vcvtdq2ps %ymm0, %ymm0
 ; CHECK-NEXT:    retl
-;
-; CHECK-WIDE-LABEL: cvt_v8i16_v8f32:
-; CHECK-WIDE:       ## %bb.0:
-; CHECK-WIDE-NEXT:    vpmovsxwd %xmm0, %xmm1
-; CHECK-WIDE-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; CHECK-WIDE-NEXT:    vpmovsxwd %xmm0, %xmm0
-; CHECK-WIDE-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; CHECK-WIDE-NEXT:    vcvtdq2ps %ymm0, %ymm0
-; CHECK-WIDE-NEXT:    retl
   %res = sitofp <8 x i16> %src to <8 x float>
   ret <8 x float> %res
 }
@@ -52,16 +30,9 @@ define <8 x float> @cvt_v8i16_v8f32(<8 x i16> %src) {
 define <4 x float> @cvt_v4i8_v4f32(<4 x i8> %src) {
 ; CHECK-LABEL: cvt_v4i8_v4f32:
 ; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vpslld $24, %xmm0, %xmm0
-; CHECK-NEXT:    vpsrad $24, %xmm0, %xmm0
+; CHECK-NEXT:    vpmovsxbd %xmm0, %xmm0
 ; CHECK-NEXT:    vcvtdq2ps %xmm0, %xmm0
 ; CHECK-NEXT:    retl
-;
-; CHECK-WIDE-LABEL: cvt_v4i8_v4f32:
-; CHECK-WIDE:       ## %bb.0:
-; CHECK-WIDE-NEXT:    vpmovsxbd %xmm0, %xmm0
-; CHECK-WIDE-NEXT:    vcvtdq2ps %xmm0, %xmm0
-; CHECK-WIDE-NEXT:    retl
   %res = sitofp <4 x i8> %src to <4 x float>
   ret <4 x float> %res
 }
@@ -69,16 +40,9 @@ define <4 x float> @cvt_v4i8_v4f32(<4 x i8> %src) {
 define <4 x float> @cvt_v4i16_v4f32(<4 x i16> %src) {
 ; CHECK-LABEL: cvt_v4i16_v4f32:
 ; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vpslld $16, %xmm0, %xmm0
-; CHECK-NEXT:    vpsrad $16, %xmm0, %xmm0
+; CHECK-NEXT:    vpmovsxwd %xmm0, %xmm0
 ; CHECK-NEXT:    vcvtdq2ps %xmm0, %xmm0
 ; CHECK-NEXT:    retl
-;
-; CHECK-WIDE-LABEL: cvt_v4i16_v4f32:
-; CHECK-WIDE:       ## %bb.0:
-; CHECK-WIDE-NEXT:    vpmovsxwd %xmm0, %xmm0
-; CHECK-WIDE-NEXT:    vcvtdq2ps %xmm0, %xmm0
-; CHECK-WIDE-NEXT:    retl
   %res = sitofp <4 x i16> %src to <4 x float>
   ret <4 x float> %res
 }
@@ -86,22 +50,12 @@ define <4 x float> @cvt_v4i16_v4f32(<4 x i16> %src) {
 define <8 x float> @cvt_v8u8_v8f32(<8 x i8> %src) {
 ; CHECK-LABEL: cvt_v8u8_v8f32:
 ; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vpand LCPI4_0, %xmm0, %xmm0
-; CHECK-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; CHECK-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
-; CHECK-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; CHECK-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; CHECK-NEXT:    vpmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; CHECK-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; CHECK-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; CHECK-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; CHECK-NEXT:    vcvtdq2ps %ymm0, %ymm0
 ; CHECK-NEXT:    retl
-;
-; CHECK-WIDE-LABEL: cvt_v8u8_v8f32:
-; CHECK-WIDE:       ## %bb.0:
-; CHECK-WIDE-NEXT:    vpmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; CHECK-WIDE-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; CHECK-WIDE-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; CHECK-WIDE-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; CHECK-WIDE-NEXT:    vcvtdq2ps %ymm0, %ymm0
-; CHECK-WIDE-NEXT:    retl
   %res = uitofp <8 x i8> %src to <8 x float>
   ret <8 x float> %res
 }
@@ -115,15 +69,6 @@ define <8 x float> @cvt_v8u16_v8f32(<8 x i16> %src) {
 ; CHECK-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; CHECK-NEXT:    vcvtdq2ps %ymm0, %ymm0
 ; CHECK-NEXT:    retl
-;
-; CHECK-WIDE-LABEL: cvt_v8u16_v8f32:
-; CHECK-WIDE:       ## %bb.0:
-; CHECK-WIDE-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; CHECK-WIDE-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
-; CHECK-WIDE-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; CHECK-WIDE-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; CHECK-WIDE-NEXT:    vcvtdq2ps %ymm0, %ymm0
-; CHECK-WIDE-NEXT:    retl
   %res = uitofp <8 x i16> %src to <8 x float>
   ret <8 x float> %res
 }
@@ -131,15 +76,9 @@ define <8 x float> @cvt_v8u16_v8f32(<8 x i16> %src) {
 define <4 x float> @cvt_v4u8_v4f32(<4 x i8> %src) {
 ; CHECK-LABEL: cvt_v4u8_v4f32:
 ; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vandps LCPI6_0, %xmm0, %xmm0
+; CHECK-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; CHECK-NEXT:    vcvtdq2ps %xmm0, %xmm0
 ; CHECK-NEXT:    retl
-;
-; CHECK-WIDE-LABEL: cvt_v4u8_v4f32:
-; CHECK-WIDE:       ## %bb.0:
-; CHECK-WIDE-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; CHECK-WIDE-NEXT:    vcvtdq2ps %xmm0, %xmm0
-; CHECK-WIDE-NEXT:    retl
   %res = uitofp <4 x i8> %src to <4 x float>
   ret <4 x float> %res
 }
@@ -147,16 +86,9 @@ define <4 x float> @cvt_v4u8_v4f32(<4 x i8> %src) {
 define <4 x float> @cvt_v4u16_v4f32(<4 x i16> %src) {
 ; CHECK-LABEL: cvt_v4u16_v4f32:
 ; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; CHECK-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7]
+; CHECK-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; CHECK-NEXT:    vcvtdq2ps %xmm0, %xmm0
 ; CHECK-NEXT:    retl
-;
-; CHECK-WIDE-LABEL: cvt_v4u16_v4f32:
-; CHECK-WIDE:       ## %bb.0:
-; CHECK-WIDE-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; CHECK-WIDE-NEXT:    vcvtdq2ps %xmm0, %xmm0
-; CHECK-WIDE-NEXT:    retl
   %res = uitofp <4 x i16> %src to <4 x float>
   ret <4 x float> %res
 }
@@ -167,17 +99,9 @@ define <8 x i8> @cvt_v8f32_v8i8(<8 x float> %src) {
 ; CHECK-NEXT:    vcvttps2dq %ymm0, %ymm0
 ; CHECK-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; CHECK-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retl
-;
-; CHECK-WIDE-LABEL: cvt_v8f32_v8i8:
-; CHECK-WIDE:       ## %bb.0:
-; CHECK-WIDE-NEXT:    vcvttps2dq %ymm0, %ymm0
-; CHECK-WIDE-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; CHECK-WIDE-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
-; CHECK-WIDE-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
-; CHECK-WIDE-NEXT:    vzeroupper
-; CHECK-WIDE-NEXT:    retl
   %res = fptosi <8 x float> %src to <8 x i8>
   ret <8 x i8> %res
 }
@@ -190,14 +114,6 @@ define <8 x i16> @cvt_v8f32_v8i16(<8 x float> %src) {
 ; CHECK-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retl
-;
-; CHECK-WIDE-LABEL: cvt_v8f32_v8i16:
-; CHECK-WIDE:       ## %bb.0:
-; CHECK-WIDE-NEXT:    vcvttps2dq %ymm0, %ymm0
-; CHECK-WIDE-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; CHECK-WIDE-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
-; CHECK-WIDE-NEXT:    vzeroupper
-; CHECK-WIDE-NEXT:    retl
   %res = fptosi <8 x float> %src to <8 x i16>
   ret <8 x i16> %res
 }
@@ -206,13 +122,8 @@ define <4 x i8> @cvt_v4f32_v4i8(<4 x float> %src) {
 ; CHECK-LABEL: cvt_v4f32_v4i8:
 ; CHECK:       ## %bb.0:
 ; CHECK-NEXT:    vcvttps2dq %xmm0, %xmm0
+; CHECK-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
 ; CHECK-NEXT:    retl
-;
-; CHECK-WIDE-LABEL: cvt_v4f32_v4i8:
-; CHECK-WIDE:       ## %bb.0:
-; CHECK-WIDE-NEXT:    vcvttps2dq %xmm0, %xmm0
-; CHECK-WIDE-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
-; CHECK-WIDE-NEXT:    retl
   %res = fptosi <4 x float> %src to <4 x i8>
   ret <4 x i8> %res
 }
@@ -221,13 +132,8 @@ define <4 x i16> @cvt_v4f32_v4i16(<4 x float> %src) {
 ; CHECK-LABEL: cvt_v4f32_v4i16:
 ; CHECK:       ## %bb.0:
 ; CHECK-NEXT:    vcvttps2dq %xmm0, %xmm0
+; CHECK-NEXT:    vpackssdw %xmm0, %xmm0, %xmm0
 ; CHECK-NEXT:    retl
-;
-; CHECK-WIDE-LABEL: cvt_v4f32_v4i16:
-; CHECK-WIDE:       ## %bb.0:
-; CHECK-WIDE-NEXT:    vcvttps2dq %xmm0, %xmm0
-; CHECK-WIDE-NEXT:    vpackssdw %xmm0, %xmm0, %xmm0
-; CHECK-WIDE-NEXT:    retl
   %res = fptosi <4 x float> %src to <4 x i16>
   ret <4 x i16> %res
 }
@@ -237,18 +143,10 @@ define <8 x i8> @cvt_v8f32_v8u8(<8 x float> %src) {
 ; CHECK:       ## %bb.0:
 ; CHECK-NEXT:    vcvttps2dq %ymm0, %ymm0
 ; CHECK-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; CHECK-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retl
-;
-; CHECK-WIDE-LABEL: cvt_v8f32_v8u8:
-; CHECK-WIDE:       ## %bb.0:
-; CHECK-WIDE-NEXT:    vcvttps2dq %ymm0, %ymm0
-; CHECK-WIDE-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; CHECK-WIDE-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
-; CHECK-WIDE-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
-; CHECK-WIDE-NEXT:    vzeroupper
-; CHECK-WIDE-NEXT:    retl
   %res = fptoui <8 x float> %src to <8 x i8>
   ret <8 x i8> %res
 }
@@ -261,14 +159,6 @@ define <8 x i16> @cvt_v8f32_v8u16(<8 x float> %src) {
 ; CHECK-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retl
-;
-; CHECK-WIDE-LABEL: cvt_v8f32_v8u16:
-; CHECK-WIDE:       ## %bb.0:
-; CHECK-WIDE-NEXT:    vcvttps2dq %ymm0, %ymm0
-; CHECK-WIDE-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; CHECK-WIDE-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
-; CHECK-WIDE-NEXT:    vzeroupper
-; CHECK-WIDE-NEXT:    retl
   %res = fptoui <8 x float> %src to <8 x i16>
   ret <8 x i16> %res
 }
@@ -277,13 +167,8 @@ define <4 x i8> @cvt_v4f32_v4u8(<4 x float> %src) {
 ; CHECK-LABEL: cvt_v4f32_v4u8:
 ; CHECK:       ## %bb.0:
 ; CHECK-NEXT:    vcvttps2dq %xmm0, %xmm0
+; CHECK-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
 ; CHECK-NEXT:    retl
-;
-; CHECK-WIDE-LABEL: cvt_v4f32_v4u8:
-; CHECK-WIDE:       ## %bb.0:
-; CHECK-WIDE-NEXT:    vcvttps2dq %xmm0, %xmm0
-; CHECK-WIDE-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
-; CHECK-WIDE-NEXT:    retl
   %res = fptoui <4 x float> %src to <4 x i8>
   ret <4 x i8> %res
 }
@@ -292,13 +177,8 @@ define <4 x i16> @cvt_v4f32_v4u16(<4 x float> %src) {
 ; CHECK-LABEL: cvt_v4f32_v4u16:
 ; CHECK:       ## %bb.0:
 ; CHECK-NEXT:    vcvttps2dq %xmm0, %xmm0
+; CHECK-NEXT:    vpackusdw %xmm0, %xmm0, %xmm0
 ; CHECK-NEXT:    retl
-;
-; CHECK-WIDE-LABEL: cvt_v4f32_v4u16:
-; CHECK-WIDE:       ## %bb.0:
-; CHECK-WIDE-NEXT:    vcvttps2dq %xmm0, %xmm0
-; CHECK-WIDE-NEXT:    vpackusdw %xmm0, %xmm0, %xmm0
-; CHECK-WIDE-NEXT:    retl
   %res = fptoui <4 x float> %src to <4 x i16>
   ret <4 x i16> %res
 }
diff --git a/llvm/test/CodeGen/X86/vec_cast3.ll b/llvm/test/CodeGen/X86/vec_cast3.ll
index f89cf47c2..82b8c00c0 100644
--- a/llvm/test/CodeGen/X86/vec_cast3.ll
+++ b/llvm/test/CodeGen/X86/vec_cast3.ll
@@ -1,21 +1,12 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=i386-apple-darwin10 -mattr=+avx | FileCheck %s
-; RUN: llc < %s -mtriple=i386-apple-darwin10 -mattr=+avx -x86-experimental-vector-widening-legalization | FileCheck %s --check-prefix=CHECK-WIDE
 
 define <2 x float> @cvt_v2i8_v2f32(<2 x i8> %src) {
 ; CHECK-LABEL: cvt_v2i8_v2f32:
 ; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vpsllq $56, %xmm0, %xmm0
-; CHECK-NEXT:    vpsrad $24, %xmm0, %xmm0
-; CHECK-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
+; CHECK-NEXT:    vpmovsxbd %xmm0, %xmm0
 ; CHECK-NEXT:    vcvtdq2ps %xmm0, %xmm0
 ; CHECK-NEXT:    retl
-;
-; CHECK-WIDE-LABEL: cvt_v2i8_v2f32:
-; CHECK-WIDE:       ## %bb.0:
-; CHECK-WIDE-NEXT:    vpmovsxbd %xmm0, %xmm0
-; CHECK-WIDE-NEXT:    vcvtdq2ps %xmm0, %xmm0
-; CHECK-WIDE-NEXT:    retl
   %res = sitofp <2 x i8> %src to <2 x float>
   ret <2 x float> %res
 }
@@ -23,17 +14,9 @@ define <2 x float> @cvt_v2i8_v2f32(<2 x i8> %src) {
 define <2 x float> @cvt_v2i16_v2f32(<2 x i16> %src) {
 ; CHECK-LABEL: cvt_v2i16_v2f32:
 ; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vpsllq $48, %xmm0, %xmm0
-; CHECK-NEXT:    vpsrad $16, %xmm0, %xmm0
-; CHECK-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
+; CHECK-NEXT:    vpmovsxwd %xmm0, %xmm0
 ; CHECK-NEXT:    vcvtdq2ps %xmm0, %xmm0
 ; CHECK-NEXT:    retl
-;
-; CHECK-WIDE-LABEL: cvt_v2i16_v2f32:
-; CHECK-WIDE:       ## %bb.0:
-; CHECK-WIDE-NEXT:    vpmovsxwd %xmm0, %xmm0
-; CHECK-WIDE-NEXT:    vcvtdq2ps %xmm0, %xmm0
-; CHECK-WIDE-NEXT:    retl
   %res = sitofp <2 x i16> %src to <2 x float>
   ret <2 x float> %res
 }
@@ -41,14 +24,8 @@ define <2 x float> @cvt_v2i16_v2f32(<2 x i16> %src) {
 define <2 x float> @cvt_v2i32_v2f32(<2 x i32> %src) {
 ; CHECK-LABEL: cvt_v2i32_v2f32:
 ; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; CHECK-NEXT:    vcvtdq2ps %xmm0, %xmm0
 ; CHECK-NEXT:    retl
-;
-; CHECK-WIDE-LABEL: cvt_v2i32_v2f32:
-; CHECK-WIDE:       ## %bb.0:
-; CHECK-WIDE-NEXT:    vcvtdq2ps %xmm0, %xmm0
-; CHECK-WIDE-NEXT:    retl
   %res = sitofp <2 x i32> %src to <2 x float>
   ret <2 x float> %res
 }
@@ -56,15 +33,9 @@ define <2 x float> @cvt_v2i32_v2f32(<2 x i32> %src) {
 define <2 x float> @cvt_v2u8_v2f32(<2 x i8> %src) {
 ; CHECK-LABEL: cvt_v2u8_v2f32:
 ; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[u,u,u,u,u,u,u,u]
+; CHECK-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; CHECK-NEXT:    vcvtdq2ps %xmm0, %xmm0
 ; CHECK-NEXT:    retl
-;
-; CHECK-WIDE-LABEL: cvt_v2u8_v2f32:
-; CHECK-WIDE:       ## %bb.0:
-; CHECK-WIDE-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; CHECK-WIDE-NEXT:    vcvtdq2ps %xmm0, %xmm0
-; CHECK-WIDE-NEXT:    retl
   %res = uitofp <2 x i8> %src to <2 x float>
   ret <2 x float> %res
 }
@@ -72,15 +43,9 @@ define <2 x float> @cvt_v2u8_v2f32(<2 x i8> %src) {
 define <2 x float> @cvt_v2u16_v2f32(<2 x i16> %src) {
 ; CHECK-LABEL: cvt_v2u16_v2f32:
 ; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1],zero,zero,xmm0[8,9],zero,zero,xmm0[8,9],zero,zero,xmm0[10,11],zero,zero
+; CHECK-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; CHECK-NEXT:    vcvtdq2ps %xmm0, %xmm0
 ; CHECK-NEXT:    retl
-;
-; CHECK-WIDE-LABEL: cvt_v2u16_v2f32:
-; CHECK-WIDE:       ## %bb.0:
-; CHECK-WIDE-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; CHECK-WIDE-NEXT:    vcvtdq2ps %xmm0, %xmm0
-; CHECK-WIDE-NEXT:    retl
   %res = uitofp <2 x i16> %src to <2 x float>
   ret <2 x float> %res
 }
@@ -88,22 +53,12 @@ define <2 x float> @cvt_v2u16_v2f32(<2 x i16> %src) {
 define <2 x float> @cvt_v2u32_v2f32(<2 x i32> %src) {
 ; CHECK-LABEL: cvt_v2u32_v2f32:
 ; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; CHECK-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
-; CHECK-NEXT:    vmovaps {{.*#+}} xmm1 = [4.503599627370496E+15,4.503599627370496E+15]
-; CHECK-NEXT:    vorps %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
+; CHECK-NEXT:    vmovdqa {{.*#+}} xmm1 = [4.503599627370496E+15,4.503599627370496E+15]
+; CHECK-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; CHECK-NEXT:    vsubpd %xmm1, %xmm0, %xmm0
 ; CHECK-NEXT:    vcvtpd2ps %xmm0, %xmm0
 ; CHECK-NEXT:    retl
-;
-; CHECK-WIDE-LABEL: cvt_v2u32_v2f32:
-; CHECK-WIDE:       ## %bb.0:
-; CHECK-WIDE-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
-; CHECK-WIDE-NEXT:    vmovdqa {{.*#+}} xmm1 = [4.503599627370496E+15,4.503599627370496E+15]
-; CHECK-WIDE-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; CHECK-WIDE-NEXT:    vsubpd %xmm1, %xmm0, %xmm0
-; CHECK-WIDE-NEXT:    vcvtpd2ps %xmm0, %xmm0
-; CHECK-WIDE-NEXT:    retl
   %res = uitofp <2 x i32> %src to <2 x float>
   ret <2 x float> %res
 }
@@ -112,14 +67,8 @@ define <2 x i8> @cvt_v2f32_v2i8(<2 x float> %src) {
 ; CHECK-LABEL: cvt_v2f32_v2i8:
 ; CHECK:       ## %bb.0:
 ; CHECK-NEXT:    vcvttps2dq %xmm0, %xmm0
-; CHECK-NEXT:    vpmovsxdq %xmm0, %xmm0
+; CHECK-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
 ; CHECK-NEXT:    retl
-;
-; CHECK-WIDE-LABEL: cvt_v2f32_v2i8:
-; CHECK-WIDE:       ## %bb.0:
-; CHECK-WIDE-NEXT:    vcvttps2dq %xmm0, %xmm0
-; CHECK-WIDE-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; CHECK-WIDE-NEXT:    retl
   %res = fptosi <2 x float> %src to <2 x i8>
   ret <2 x i8> %res
 }
@@ -128,14 +77,8 @@ define <2 x i16> @cvt_v2f32_v2i16(<2 x float> %src) {
 ; CHECK-LABEL: cvt_v2f32_v2i16:
 ; CHECK:       ## %bb.0:
 ; CHECK-NEXT:    vcvttps2dq %xmm0, %xmm0
-; CHECK-NEXT:    vpmovsxdq %xmm0, %xmm0
+; CHECK-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
 ; CHECK-NEXT:    retl
-;
-; CHECK-WIDE-LABEL: cvt_v2f32_v2i16:
-; CHECK-WIDE:       ## %bb.0:
-; CHECK-WIDE-NEXT:    vcvttps2dq %xmm0, %xmm0
-; CHECK-WIDE-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
-; CHECK-WIDE-NEXT:    retl
   %res = fptosi <2 x float> %src to <2 x i16>
   ret <2 x i16> %res
 }
@@ -144,13 +87,7 @@ define <2 x i32> @cvt_v2f32_v2i32(<2 x float> %src) {
 ; CHECK-LABEL: cvt_v2f32_v2i32:
 ; CHECK:       ## %bb.0:
 ; CHECK-NEXT:    vcvttps2dq %xmm0, %xmm0
-; CHECK-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
 ; CHECK-NEXT:    retl
-;
-; CHECK-WIDE-LABEL: cvt_v2f32_v2i32:
-; CHECK-WIDE:       ## %bb.0:
-; CHECK-WIDE-NEXT:    vcvttps2dq %xmm0, %xmm0
-; CHECK-WIDE-NEXT:    retl
   %res = fptosi <2 x float> %src to <2 x i32>
   ret <2 x i32> %res
 }
@@ -159,14 +96,8 @@ define <2 x i8> @cvt_v2f32_v2u8(<2 x float> %src) {
 ; CHECK-LABEL: cvt_v2f32_v2u8:
 ; CHECK:       ## %bb.0:
 ; CHECK-NEXT:    vcvttps2dq %xmm0, %xmm0
-; CHECK-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
+; CHECK-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
 ; CHECK-NEXT:    retl
-;
-; CHECK-WIDE-LABEL: cvt_v2f32_v2u8:
-; CHECK-WIDE:       ## %bb.0:
-; CHECK-WIDE-NEXT:    vcvttps2dq %xmm0, %xmm0
-; CHECK-WIDE-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; CHECK-WIDE-NEXT:    retl
   %res = fptoui <2 x float> %src to <2 x i8>
   ret <2 x i8> %res
 }
@@ -175,14 +106,8 @@ define <2 x i16> @cvt_v2f32_v2u16(<2 x float> %src) {
 ; CHECK-LABEL: cvt_v2f32_v2u16:
 ; CHECK:       ## %bb.0:
 ; CHECK-NEXT:    vcvttps2dq %xmm0, %xmm0
-; CHECK-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
+; CHECK-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
 ; CHECK-NEXT:    retl
-;
-; CHECK-WIDE-LABEL: cvt_v2f32_v2u16:
-; CHECK-WIDE:       ## %bb.0:
-; CHECK-WIDE-NEXT:    vcvttps2dq %xmm0, %xmm0
-; CHECK-WIDE-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
-; CHECK-WIDE-NEXT:    retl
   %res = fptoui <2 x float> %src to <2 x i16>
   ret <2 x i16> %res
 }
@@ -190,51 +115,14 @@ define <2 x i16> @cvt_v2f32_v2u16(<2 x float> %src) {
 define <2 x i32> @cvt_v2f32_v2u32(<2 x float> %src) {
 ; CHECK-LABEL: cvt_v2f32_v2u32:
 ; CHECK:       ## %bb.0:
-; CHECK-NEXT:    subl $36, %esp
-; CHECK-NEXT:    .cfi_def_cfa_offset 40
-; CHECK-NEXT:    vmovshdup {{.*#+}} xmm2 = xmm0[1,1,3,3]
-; CHECK-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; CHECK-NEXT:    vucomiss %xmm1, %xmm2
-; CHECK-NEXT:    jb LBB11_2
-; CHECK-NEXT:  ## %bb.1:
-; CHECK-NEXT:    vsubss %xmm1, %xmm2, %xmm2
-; CHECK-NEXT:  LBB11_2:
-; CHECK-NEXT:    vmovss %xmm2, (%esp)
-; CHECK-NEXT:    flds (%esp)
-; CHECK-NEXT:    fisttpll (%esp)
-; CHECK-NEXT:    setae %al
-; CHECK-NEXT:    movzbl %al, %eax
-; CHECK-NEXT:    shll $31, %eax
-; CHECK-NEXT:    xorl {{[0-9]+}}(%esp), %eax
-; CHECK-NEXT:    vucomiss %xmm1, %xmm0
-; CHECK-NEXT:    jb LBB11_4
-; CHECK-NEXT:  ## %bb.3:
-; CHECK-NEXT:    vsubss %xmm1, %xmm0, %xmm0
-; CHECK-NEXT:  LBB11_4:
-; CHECK-NEXT:    vmovss %xmm0, {{[0-9]+}}(%esp)
-; CHECK-NEXT:    flds {{[0-9]+}}(%esp)
-; CHECK-NEXT:    fisttpll {{[0-9]+}}(%esp)
-; CHECK-NEXT:    setae %cl
-; CHECK-NEXT:    movzbl %cl, %ecx
-; CHECK-NEXT:    shll $31, %ecx
-; CHECK-NEXT:    xorl {{[0-9]+}}(%esp), %ecx
-; CHECK-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; CHECK-NEXT:    vpinsrd $1, %ecx, %xmm0, %xmm0
-; CHECK-NEXT:    vpinsrd $2, (%esp), %xmm0, %xmm0
-; CHECK-NEXT:    vpinsrd $3, %eax, %xmm0, %xmm0
-; CHECK-NEXT:    addl $36, %esp
+; CHECK-NEXT:    vmovaps {{.*#+}} xmm1 = [2.14748365E+9,2.14748365E+9,2.14748365E+9,2.14748365E+9]
+; CHECK-NEXT:    vcmpltps %xmm1, %xmm0, %xmm2
+; CHECK-NEXT:    vsubps %xmm1, %xmm0, %xmm1
+; CHECK-NEXT:    vcvttps2dq %xmm1, %xmm1
+; CHECK-NEXT:    vxorps LCPI11_1, %xmm1, %xmm1
+; CHECK-NEXT:    vcvttps2dq %xmm0, %xmm0
+; CHECK-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
 ; CHECK-NEXT:    retl
-;
-; CHECK-WIDE-LABEL: cvt_v2f32_v2u32:
-; CHECK-WIDE:       ## %bb.0:
-; CHECK-WIDE-NEXT:    vmovaps {{.*#+}} xmm1 = [2.14748365E+9,2.14748365E+9,2.14748365E+9,2.14748365E+9]
-; CHECK-WIDE-NEXT:    vcmpltps %xmm1, %xmm0, %xmm2
-; CHECK-WIDE-NEXT:    vsubps %xmm1, %xmm0, %xmm1
-; CHECK-WIDE-NEXT:    vcvttps2dq %xmm1, %xmm1
-; CHECK-WIDE-NEXT:    vxorps LCPI11_1, %xmm1, %xmm1
-; CHECK-WIDE-NEXT:    vcvttps2dq %xmm0, %xmm0
-; CHECK-WIDE-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
-; CHECK-WIDE-NEXT:    retl
   %res = fptoui <2 x float> %src to <2 x i32>
   ret <2 x i32> %res
 }
@@ -247,14 +135,6 @@ define <32 x i8> @PR40146(<4 x i64> %x) {
 ; CHECK-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
 ; CHECK-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; CHECK-NEXT:    retl
-;
-; CHECK-WIDE-LABEL: PR40146:
-; CHECK-WIDE:       ## %bb.0:
-; CHECK-WIDE-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; CHECK-WIDE-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; CHECK-WIDE-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
-; CHECK-WIDE-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; CHECK-WIDE-NEXT:    retl
   %perm = shufflevector <4 x i64> %x, <4 x i64> undef, <4 x i32> <i32 0, i32 undef, i32 1, i32 undef>
   %t1 = bitcast <4 x i64> %perm to <32 x i8>
   %t2 = shufflevector <32 x i8> %t1, <32 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <32 x i32> <i32 0, i32 32, i32 1, i32 32, i32 2, i32 32, i32 3, i32 32, i32 4, i32 32, i32 5, i32 32, i32 6, i32 32, i32 7, i32 32, i32 16, i32 48, i32 17, i32 48, i32 18, i32 48, i32 19, i32 48, i32 20, i32 48, i32 21, i32 48, i32 22, i32 48, i32 23, i32 48>
diff --git a/llvm/test/CodeGen/X86/vec_ctbits.ll b/llvm/test/CodeGen/X86/vec_ctbits.ll
index 26330f940..e0ea1a298 100644
--- a/llvm/test/CodeGen/X86/vec_ctbits.ll
+++ b/llvm/test/CodeGen/X86/vec_ctbits.ll
@@ -110,9 +110,8 @@ declare <2 x i32> @llvm.ctpop.v2i32(<2 x i32>)
 define <2 x i32> @promtz(<2 x i32> %a) nounwind {
 ; CHECK-LABEL: promtz:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    por {{.*}}(%rip), %xmm0
 ; CHECK-NEXT:    pcmpeqd %xmm1, %xmm1
-; CHECK-NEXT:    paddq %xmm0, %xmm1
+; CHECK-NEXT:    paddd %xmm0, %xmm1
 ; CHECK-NEXT:    pandn %xmm1, %xmm0
 ; CHECK-NEXT:    movdqa %xmm0, %xmm1
 ; CHECK-NEXT:    psrlw $1, %xmm1
@@ -129,7 +128,12 @@ define <2 x i32> @promtz(<2 x i32> %a) nounwind {
 ; CHECK-NEXT:    paddb %xmm0, %xmm1
 ; CHECK-NEXT:    pand {{.*}}(%rip), %xmm1
 ; CHECK-NEXT:    pxor %xmm0, %xmm0
+; CHECK-NEXT:    movdqa %xmm1, %xmm2
+; CHECK-NEXT:    punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm0[2],xmm2[3],xmm0[3]
+; CHECK-NEXT:    psadbw %xmm0, %xmm2
+; CHECK-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
 ; CHECK-NEXT:    psadbw %xmm0, %xmm1
+; CHECK-NEXT:    packuswb %xmm2, %xmm1
 ; CHECK-NEXT:    movdqa %xmm1, %xmm0
 ; CHECK-NEXT:    retq
   %c = call <2 x i32> @llvm.cttz.v2i32(<2 x i32> %a, i1 false)
@@ -139,44 +143,44 @@ define <2 x i32> @promtz(<2 x i32> %a) nounwind {
 define <2 x i32> @promlz(<2 x i32> %a) nounwind {
 ; CHECK-LABEL: promlz:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pand {{.*}}(%rip), %xmm0
-; CHECK-NEXT:    pxor %xmm1, %xmm1
-; CHECK-NEXT:    movdqa %xmm0, %xmm2
-; CHECK-NEXT:    psrlq $1, %xmm2
-; CHECK-NEXT:    por %xmm0, %xmm2
-; CHECK-NEXT:    movdqa %xmm2, %xmm0
-; CHECK-NEXT:    psrlq $2, %xmm0
-; CHECK-NEXT:    por %xmm2, %xmm0
-; CHECK-NEXT:    movdqa %xmm0, %xmm2
-; CHECK-NEXT:    psrlq $4, %xmm2
-; CHECK-NEXT:    por %xmm0, %xmm2
-; CHECK-NEXT:    movdqa %xmm2, %xmm0
-; CHECK-NEXT:    psrlq $8, %xmm0
-; CHECK-NEXT:    por %xmm2, %xmm0
-; CHECK-NEXT:    movdqa %xmm0, %xmm2
-; CHECK-NEXT:    psrlq $16, %xmm2
-; CHECK-NEXT:    por %xmm0, %xmm2
-; CHECK-NEXT:    movdqa %xmm2, %xmm0
-; CHECK-NEXT:    psrlq $32, %xmm0
-; CHECK-NEXT:    por %xmm2, %xmm0
+; CHECK-NEXT:    movdqa %xmm0, %xmm1
+; CHECK-NEXT:    psrld $1, %xmm1
+; CHECK-NEXT:    por %xmm0, %xmm1
+; CHECK-NEXT:    movdqa %xmm1, %xmm0
+; CHECK-NEXT:    psrld $2, %xmm0
+; CHECK-NEXT:    por %xmm1, %xmm0
+; CHECK-NEXT:    movdqa %xmm0, %xmm1
+; CHECK-NEXT:    psrld $4, %xmm1
+; CHECK-NEXT:    por %xmm0, %xmm1
+; CHECK-NEXT:    movdqa %xmm1, %xmm0
+; CHECK-NEXT:    psrld $8, %xmm0
+; CHECK-NEXT:    por %xmm1, %xmm0
+; CHECK-NEXT:    movdqa %xmm0, %xmm1
+; CHECK-NEXT:    psrld $16, %xmm1
+; CHECK-NEXT:    por %xmm0, %xmm1
 ; CHECK-NEXT:    pcmpeqd %xmm2, %xmm2
-; CHECK-NEXT:    pxor %xmm0, %xmm2
+; CHECK-NEXT:    pxor %xmm1, %xmm2
 ; CHECK-NEXT:    movdqa %xmm2, %xmm0
 ; CHECK-NEXT:    psrlw $1, %xmm0
 ; CHECK-NEXT:    pand {{.*}}(%rip), %xmm0
 ; CHECK-NEXT:    psubb %xmm0, %xmm2
 ; CHECK-NEXT:    movdqa {{.*#+}} xmm0 = [51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51]
-; CHECK-NEXT:    movdqa %xmm2, %xmm3
-; CHECK-NEXT:    pand %xmm0, %xmm3
+; CHECK-NEXT:    movdqa %xmm2, %xmm1
+; CHECK-NEXT:    pand %xmm0, %xmm1
 ; CHECK-NEXT:    psrlw $2, %xmm2
 ; CHECK-NEXT:    pand %xmm0, %xmm2
-; CHECK-NEXT:    paddb %xmm3, %xmm2
+; CHECK-NEXT:    paddb %xmm1, %xmm2
 ; CHECK-NEXT:    movdqa %xmm2, %xmm0
 ; CHECK-NEXT:    psrlw $4, %xmm0
 ; CHECK-NEXT:    paddb %xmm2, %xmm0
 ; CHECK-NEXT:    pand {{.*}}(%rip), %xmm0
+; CHECK-NEXT:    pxor %xmm1, %xmm1
+; CHECK-NEXT:    movdqa %xmm0, %xmm2
+; CHECK-NEXT:    punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; CHECK-NEXT:    psadbw %xmm1, %xmm2
+; CHECK-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; CHECK-NEXT:    psadbw %xmm1, %xmm0
-; CHECK-NEXT:    psubq {{.*}}(%rip), %xmm0
+; CHECK-NEXT:    packuswb %xmm2, %xmm0
 ; CHECK-NEXT:    retq
   %c = call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> %a, i1 false)
   ret <2 x i32> %c
@@ -186,23 +190,27 @@ define <2 x i32> @promlz(<2 x i32> %a) nounwind {
 define <2 x i32> @prompop(<2 x i32> %a) nounwind {
 ; CHECK-LABEL: prompop:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pand {{.*}}(%rip), %xmm0
-; CHECK-NEXT:    pxor %xmm2, %xmm2
 ; CHECK-NEXT:    movdqa %xmm0, %xmm1
 ; CHECK-NEXT:    psrlw $1, %xmm1
 ; CHECK-NEXT:    pand {{.*}}(%rip), %xmm1
 ; CHECK-NEXT:    psubb %xmm1, %xmm0
 ; CHECK-NEXT:    movdqa {{.*#+}} xmm1 = [51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51]
-; CHECK-NEXT:    movdqa %xmm0, %xmm3
-; CHECK-NEXT:    pand %xmm1, %xmm3
+; CHECK-NEXT:    movdqa %xmm0, %xmm2
+; CHECK-NEXT:    pand %xmm1, %xmm2
 ; CHECK-NEXT:    psrlw $2, %xmm0
 ; CHECK-NEXT:    pand %xmm1, %xmm0
-; CHECK-NEXT:    paddb %xmm3, %xmm0
+; CHECK-NEXT:    paddb %xmm2, %xmm0
 ; CHECK-NEXT:    movdqa %xmm0, %xmm1
 ; CHECK-NEXT:    psrlw $4, %xmm1
 ; CHECK-NEXT:    paddb %xmm0, %xmm1
 ; CHECK-NEXT:    pand {{.*}}(%rip), %xmm1
-; CHECK-NEXT:    psadbw %xmm2, %xmm1
+; CHECK-NEXT:    pxor %xmm0, %xmm0
+; CHECK-NEXT:    movdqa %xmm1, %xmm2
+; CHECK-NEXT:    punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm0[2],xmm2[3],xmm0[3]
+; CHECK-NEXT:    psadbw %xmm0, %xmm2
+; CHECK-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; CHECK-NEXT:    psadbw %xmm0, %xmm1
+; CHECK-NEXT:    packuswb %xmm2, %xmm1
 ; CHECK-NEXT:    movdqa %xmm1, %xmm0
 ; CHECK-NEXT:    retq
   %c = call <2 x i32> @llvm.ctpop.v2i32(<2 x i32> %a)
diff --git a/llvm/test/CodeGen/X86/vec_extract-mmx.ll b/llvm/test/CodeGen/X86/vec_extract-mmx.ll
index 5d68857f6..5ce632d0a 100644
--- a/llvm/test/CodeGen/X86/vec_extract-mmx.ll
+++ b/llvm/test/CodeGen/X86/vec_extract-mmx.ll
@@ -113,23 +113,15 @@ define i32 @test3(x86_mmx %a) nounwind {
 define i32 @test4(x86_mmx %a) nounwind {
 ; X32-LABEL: test4:
 ; X32:       # %bb.0:
-; X32-NEXT:    pushl %ebp
-; X32-NEXT:    movl %esp, %ebp
-; X32-NEXT:    andl $-8, %esp
-; X32-NEXT:    subl $8, %esp
-; X32-NEXT:    movq %mm0, (%esp)
-; X32-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; X32-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,0,1]
+; X32-NEXT:    movq2dq %mm0, %xmm0
+; X32-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
 ; X32-NEXT:    movd %xmm0, %eax
-; X32-NEXT:    movl %ebp, %esp
-; X32-NEXT:    popl %ebp
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: test4:
 ; X64:       # %bb.0:
-; X64-NEXT:    movq %mm0, -{{[0-9]+}}(%rsp)
-; X64-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,0,1]
+; X64-NEXT:    movq2dq %mm0, %xmm0
+; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
 ; X64-NEXT:    movd %xmm0, %eax
 ; X64-NEXT:    retq
   %tmp0 = bitcast x86_mmx %a to <2 x i32>
diff --git a/llvm/test/CodeGen/X86/vec_extract.ll b/llvm/test/CodeGen/X86/vec_extract.ll
index 3fb669dd4..9b347c121 100644
--- a/llvm/test/CodeGen/X86/vec_extract.ll
+++ b/llvm/test/CodeGen/X86/vec_extract.ll
@@ -57,13 +57,15 @@ define void @test3(float* %R, <4 x float>* %P1) nounwind {
 ; X32:       # %bb.0: # %entry
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X32-NEXT:    movaps (%ecx), %xmm0
+; X32-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
 ; X32-NEXT:    movss %xmm0, (%eax)
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: test3:
 ; X64:       # %bb.0: # %entry
-; X64-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X64-NEXT:    movaps (%rsi), %xmm0
+; X64-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
 ; X64-NEXT:    movss %xmm0, (%rdi)
 ; X64-NEXT:    retq
 entry:
@@ -108,15 +110,11 @@ define <4 x i32> @ossfuzz15662(<4 x i32*>* %in) {
 ; X32-LABEL: ossfuzz15662:
 ; X32:       # %bb.0:
 ; X32-NEXT:    xorps %xmm0, %xmm0
-; X32-NEXT:    movaps %xmm0, (%eax)
-; X32-NEXT:    xorps %xmm0, %xmm0
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: ossfuzz15662:
 ; X64:       # %bb.0:
 ; X64-NEXT:    xorps %xmm0, %xmm0
-; X64-NEXT:    movaps %xmm0, (%rax)
-; X64-NEXT:    xorps %xmm0, %xmm0
 ; X64-NEXT:    retq
    %C10 = icmp ule i1 false, false
    %C3 = icmp ule i1 true, undef
diff --git a/llvm/test/CodeGen/X86/vec_fp_to_int.ll b/llvm/test/CodeGen/X86/vec_fp_to_int.ll
index 071d46860..bf2ea5e06 100644
--- a/llvm/test/CodeGen/X86/vec_fp_to_int.ll
+++ b/llvm/test/CodeGen/X86/vec_fp_to_int.ll
@@ -21,7 +21,7 @@ define <2 x i64> @fptosi_2f64_to_2i64(<2 x double> %a) {
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    cvttsd2si %xmm0, %rax
 ; SSE-NEXT:    movq %rax, %xmm1
-; SSE-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
+; SSE-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1,1]
 ; SSE-NEXT:    cvttsd2si %xmm0, %rax
 ; SSE-NEXT:    movq %rax, %xmm0
 ; SSE-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
@@ -93,13 +93,11 @@ define <2 x i32> @fptosi_2f64_to_2i32(<2 x double> %a) {
 ; SSE-LABEL: fptosi_2f64_to_2i32:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    cvttpd2dq %xmm0, %xmm0
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,1,3]
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: fptosi_2f64_to_2i32:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vcvttpd2dq %xmm0, %xmm0
-; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
 ; AVX-NEXT:    retq
   %cvt = fptosi <2 x double> %a to <2 x i32>
   ret <2 x i32> %cvt
@@ -127,13 +125,13 @@ define <4 x i64> @fptosi_4f64_to_4i64(<4 x double> %a) {
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    cvttsd2si %xmm0, %rax
 ; SSE-NEXT:    movq %rax, %xmm2
-; SSE-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
+; SSE-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1,1]
 ; SSE-NEXT:    cvttsd2si %xmm0, %rax
 ; SSE-NEXT:    movq %rax, %xmm0
 ; SSE-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm0[0]
 ; SSE-NEXT:    cvttsd2si %xmm1, %rax
 ; SSE-NEXT:    movq %rax, %xmm3
-; SSE-NEXT:    movhlps {{.*#+}} xmm1 = xmm1[1,1]
+; SSE-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1,1]
 ; SSE-NEXT:    cvttsd2si %xmm1, %rax
 ; SSE-NEXT:    movq %rax, %xmm0
 ; SSE-NEXT:    punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm0[0]
@@ -336,53 +334,45 @@ define <2 x i64> @fptoui_2f64_to_2i64(<2 x double> %a) {
 define <4 x i32> @fptoui_2f64_to_4i32(<2 x double> %a) {
 ; SSE-LABEL: fptoui_2f64_to_4i32:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    movsd {{.*#+}} xmm2 = mem[0],zero
-; SSE-NEXT:    movapd %xmm0, %xmm1
-; SSE-NEXT:    subsd %xmm2, %xmm1
-; SSE-NEXT:    cvttsd2si %xmm1, %rax
-; SSE-NEXT:    movabsq $-9223372036854775808, %rcx # imm = 0x8000000000000000
-; SSE-NEXT:    xorq %rcx, %rax
-; SSE-NEXT:    cvttsd2si %xmm0, %rdx
-; SSE-NEXT:    ucomisd %xmm2, %xmm0
-; SSE-NEXT:    cmovaeq %rax, %rdx
-; SSE-NEXT:    movq %rdx, %xmm1
+; SSE-NEXT:    cvttsd2si %xmm0, %rax
 ; SSE-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1,1]
-; SSE-NEXT:    movapd %xmm0, %xmm3
-; SSE-NEXT:    subsd %xmm2, %xmm3
-; SSE-NEXT:    cvttsd2si %xmm3, %rax
-; SSE-NEXT:    xorq %rcx, %rax
 ; SSE-NEXT:    cvttsd2si %xmm0, %rcx
-; SSE-NEXT:    ucomisd %xmm2, %xmm0
-; SSE-NEXT:    cmovaeq %rax, %rcx
-; SSE-NEXT:    movq %rcx, %xmm0
-; SSE-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
-; SSE-NEXT:    pxor %xmm0, %xmm0
-; SSE-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,2],xmm0[2,3]
-; SSE-NEXT:    movaps %xmm1, %xmm0
+; SSE-NEXT:    movd %eax, %xmm0
+; SSE-NEXT:    movd %ecx, %xmm1
+; SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE-NEXT:    movq {{.*#+}} xmm0 = xmm0[0],zero
 ; SSE-NEXT:    retq
 ;
-; VEX-LABEL: fptoui_2f64_to_4i32:
-; VEX:       # %bb.0:
-; VEX-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
-; VEX-NEXT:    vsubsd %xmm1, %xmm0, %xmm2
-; VEX-NEXT:    vcvttsd2si %xmm2, %rax
-; VEX-NEXT:    movabsq $-9223372036854775808, %rcx # imm = 0x8000000000000000
-; VEX-NEXT:    xorq %rcx, %rax
-; VEX-NEXT:    vcvttsd2si %xmm0, %rdx
-; VEX-NEXT:    vucomisd %xmm1, %xmm0
-; VEX-NEXT:    cmovaeq %rax, %rdx
-; VEX-NEXT:    vmovq %rdx, %xmm2
-; VEX-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
-; VEX-NEXT:    vsubsd %xmm1, %xmm0, %xmm3
-; VEX-NEXT:    vcvttsd2si %xmm3, %rax
-; VEX-NEXT:    xorq %rcx, %rax
-; VEX-NEXT:    vcvttsd2si %xmm0, %rcx
-; VEX-NEXT:    vucomisd %xmm1, %xmm0
-; VEX-NEXT:    cmovaeq %rax, %rcx
-; VEX-NEXT:    vmovq %rcx, %xmm0
-; VEX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm2[0],xmm0[0]
-; VEX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,2],zero,zero
-; VEX-NEXT:    retq
+; AVX1-LABEL: fptoui_2f64_to_4i32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; AVX1-NEXT:    vmovapd {{.*#+}} ymm1 = [2.147483648E+9,2.147483648E+9,2.147483648E+9,2.147483648E+9]
+; AVX1-NEXT:    vcmpltpd %ymm1, %ymm0, %ymm2
+; AVX1-NEXT:    vpackssdw %xmm0, %xmm2, %xmm2
+; AVX1-NEXT:    vcvttpd2dq %ymm0, %xmm3
+; AVX1-NEXT:    vsubpd %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vcvttpd2dq %ymm0, %xmm0
+; AVX1-NEXT:    vxorpd {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vblendvps %xmm2, %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: fptoui_2f64_to_4i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm1 = [2.147483648E+9,2.147483648E+9,2.147483648E+9,2.147483648E+9]
+; AVX2-NEXT:    vcmpltpd %ymm1, %ymm0, %ymm2
+; AVX2-NEXT:    vpackssdw %xmm0, %xmm2, %xmm2
+; AVX2-NEXT:    vsubpd %ymm1, %ymm0, %ymm1
+; AVX2-NEXT:    vcvttpd2dq %ymm1, %xmm1
+; AVX2-NEXT:    vbroadcastss {{.*#+}} xmm3 = [2147483648,2147483648,2147483648,2147483648]
+; AVX2-NEXT:    vxorpd %xmm3, %xmm1, %xmm1
+; AVX2-NEXT:    vcvttpd2dq %ymm0, %xmm0
+; AVX2-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX2-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: fptoui_2f64_to_4i32:
 ; AVX512F:       # %bb.0:
@@ -417,51 +407,45 @@ define <4 x i32> @fptoui_2f64_to_4i32(<2 x double> %a) {
 define <4 x i32> @fptoui_2f64_to_2i32(<2 x double> %a) {
 ; SSE-LABEL: fptoui_2f64_to_2i32:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
-; SSE-NEXT:    movapd %xmm0, %xmm2
-; SSE-NEXT:    subsd %xmm1, %xmm2
-; SSE-NEXT:    cvttsd2si %xmm2, %rax
-; SSE-NEXT:    movabsq $-9223372036854775808, %rcx # imm = 0x8000000000000000
-; SSE-NEXT:    xorq %rcx, %rax
-; SSE-NEXT:    cvttsd2si %xmm0, %rdx
-; SSE-NEXT:    ucomisd %xmm1, %xmm0
-; SSE-NEXT:    cmovaeq %rax, %rdx
-; SSE-NEXT:    movq %rdx, %xmm2
+; SSE-NEXT:    cvttsd2si %xmm0, %rax
+; SSE-NEXT:    movd %eax, %xmm1
 ; SSE-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1,1]
-; SSE-NEXT:    movapd %xmm0, %xmm3
-; SSE-NEXT:    subsd %xmm1, %xmm3
-; SSE-NEXT:    cvttsd2si %xmm3, %rax
-; SSE-NEXT:    xorq %rcx, %rax
-; SSE-NEXT:    cvttsd2si %xmm0, %rcx
-; SSE-NEXT:    ucomisd %xmm1, %xmm0
-; SSE-NEXT:    cmovaeq %rax, %rcx
-; SSE-NEXT:    movq %rcx, %xmm0
-; SSE-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm0[0]
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
+; SSE-NEXT:    cvttsd2si %xmm0, %rax
+; SSE-NEXT:    movd %eax, %xmm0
+; SSE-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE-NEXT:    movdqa %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
-; VEX-LABEL: fptoui_2f64_to_2i32:
-; VEX:       # %bb.0:
-; VEX-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
-; VEX-NEXT:    vsubsd %xmm1, %xmm0, %xmm2
-; VEX-NEXT:    vcvttsd2si %xmm2, %rax
-; VEX-NEXT:    movabsq $-9223372036854775808, %rcx # imm = 0x8000000000000000
-; VEX-NEXT:    xorq %rcx, %rax
-; VEX-NEXT:    vcvttsd2si %xmm0, %rdx
-; VEX-NEXT:    vucomisd %xmm1, %xmm0
-; VEX-NEXT:    cmovaeq %rax, %rdx
-; VEX-NEXT:    vmovq %rdx, %xmm2
-; VEX-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
-; VEX-NEXT:    vsubsd %xmm1, %xmm0, %xmm3
-; VEX-NEXT:    vcvttsd2si %xmm3, %rax
-; VEX-NEXT:    xorq %rcx, %rax
-; VEX-NEXT:    vcvttsd2si %xmm0, %rcx
-; VEX-NEXT:    vucomisd %xmm1, %xmm0
-; VEX-NEXT:    cmovaeq %rax, %rcx
-; VEX-NEXT:    vmovq %rcx, %xmm0
-; VEX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm2[0],xmm0[0]
-; VEX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; VEX-NEXT:    retq
+; AVX1-LABEL: fptoui_2f64_to_2i32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; AVX1-NEXT:    vmovapd {{.*#+}} ymm1 = [2.147483648E+9,2.147483648E+9,2.147483648E+9,2.147483648E+9]
+; AVX1-NEXT:    vcmpltpd %ymm1, %ymm0, %ymm2
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm3
+; AVX1-NEXT:    vpackssdw %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vcvttpd2dq %ymm0, %xmm3
+; AVX1-NEXT:    vsubpd %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vcvttpd2dq %ymm0, %xmm0
+; AVX1-NEXT:    vxorpd {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vblendvps %xmm2, %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: fptoui_2f64_to_2i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm1 = [2.147483648E+9,2.147483648E+9,2.147483648E+9,2.147483648E+9]
+; AVX2-NEXT:    vcmpltpd %ymm1, %ymm0, %ymm2
+; AVX2-NEXT:    vextractf128 $1, %ymm2, %xmm3
+; AVX2-NEXT:    vpackssdw %xmm3, %xmm2, %xmm2
+; AVX2-NEXT:    vsubpd %ymm1, %ymm0, %ymm1
+; AVX2-NEXT:    vcvttpd2dq %ymm1, %xmm1
+; AVX2-NEXT:    vbroadcastss {{.*#+}} xmm3 = [2147483648,2147483648,2147483648,2147483648]
+; AVX2-NEXT:    vxorpd %xmm3, %xmm1, %xmm1
+; AVX2-NEXT:    vcvttpd2dq %ymm0, %xmm0
+; AVX2-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: fptoui_2f64_to_2i32:
 ; AVX512F:       # %bb.0:
@@ -496,29 +480,13 @@ define <4 x i32> @fptoui_2f64_to_2i32(<2 x double> %a) {
 define <4 x i32> @fptoui_4f64_to_2i32(<2 x double> %a) {
 ; SSE-LABEL: fptoui_4f64_to_2i32:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    movsd {{.*#+}} xmm2 = mem[0],zero
-; SSE-NEXT:    movapd %xmm0, %xmm1
-; SSE-NEXT:    subsd %xmm2, %xmm1
-; SSE-NEXT:    cvttsd2si %xmm1, %rax
-; SSE-NEXT:    movabsq $-9223372036854775808, %rcx # imm = 0x8000000000000000
-; SSE-NEXT:    xorq %rcx, %rax
-; SSE-NEXT:    cvttsd2si %xmm0, %rdx
-; SSE-NEXT:    ucomisd %xmm2, %xmm0
-; SSE-NEXT:    cmovaeq %rax, %rdx
-; SSE-NEXT:    movq %rdx, %xmm1
+; SSE-NEXT:    cvttsd2si %xmm0, %rax
+; SSE-NEXT:    movd %eax, %xmm1
 ; SSE-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1,1]
-; SSE-NEXT:    movapd %xmm0, %xmm3
-; SSE-NEXT:    subsd %xmm2, %xmm3
-; SSE-NEXT:    cvttsd2si %xmm3, %rax
-; SSE-NEXT:    xorq %rcx, %rax
-; SSE-NEXT:    cvttsd2si %xmm0, %rcx
-; SSE-NEXT:    ucomisd %xmm2, %xmm0
-; SSE-NEXT:    cmovaeq %rax, %rcx
-; SSE-NEXT:    movq %rcx, %xmm0
-; SSE-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
-; SSE-NEXT:    pxor %xmm0, %xmm0
-; SSE-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,2],xmm0[2,3]
-; SSE-NEXT:    movaps %xmm1, %xmm0
+; SSE-NEXT:    cvttsd2si %xmm0, %rax
+; SSE-NEXT:    movd %eax, %xmm0
+; SSE-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE-NEXT:    movq {{.*#+}} xmm0 = xmm1[0],zero
 ; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: fptoui_4f64_to_2i32:
@@ -764,46 +732,20 @@ define <4 x i64> @fptoui_4f64_to_4i64(<4 x double> %a) {
 define <4 x i32> @fptoui_4f64_to_4i32(<4 x double> %a) {
 ; SSE-LABEL: fptoui_4f64_to_4i32:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    movsd {{.*#+}} xmm2 = mem[0],zero
-; SSE-NEXT:    movapd %xmm1, %xmm3
-; SSE-NEXT:    subsd %xmm2, %xmm3
-; SSE-NEXT:    cvttsd2si %xmm3, %rcx
-; SSE-NEXT:    movabsq $-9223372036854775808, %rax # imm = 0x8000000000000000
-; SSE-NEXT:    xorq %rax, %rcx
-; SSE-NEXT:    cvttsd2si %xmm1, %rdx
-; SSE-NEXT:    ucomisd %xmm2, %xmm1
-; SSE-NEXT:    cmovaeq %rcx, %rdx
-; SSE-NEXT:    movq %rdx, %xmm3
+; SSE-NEXT:    cvttsd2si %xmm1, %rax
+; SSE-NEXT:    movd %eax, %xmm2
 ; SSE-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1,1]
-; SSE-NEXT:    movapd %xmm1, %xmm4
-; SSE-NEXT:    subsd %xmm2, %xmm4
-; SSE-NEXT:    cvttsd2si %xmm4, %rcx
-; SSE-NEXT:    xorq %rax, %rcx
-; SSE-NEXT:    cvttsd2si %xmm1, %rdx
-; SSE-NEXT:    ucomisd %xmm2, %xmm1
-; SSE-NEXT:    cmovaeq %rcx, %rdx
-; SSE-NEXT:    movq %rdx, %xmm1
-; SSE-NEXT:    punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm1[0]
-; SSE-NEXT:    movapd %xmm0, %xmm1
-; SSE-NEXT:    subsd %xmm2, %xmm1
-; SSE-NEXT:    cvttsd2si %xmm1, %rcx
-; SSE-NEXT:    xorq %rax, %rcx
-; SSE-NEXT:    cvttsd2si %xmm0, %rdx
-; SSE-NEXT:    ucomisd %xmm2, %xmm0
-; SSE-NEXT:    cmovaeq %rcx, %rdx
-; SSE-NEXT:    movq %rdx, %xmm1
+; SSE-NEXT:    cvttsd2si %xmm1, %rax
+; SSE-NEXT:    movd %eax, %xmm1
+; SSE-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSE-NEXT:    cvttsd2si %xmm0, %rax
+; SSE-NEXT:    movd %eax, %xmm1
 ; SSE-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1,1]
-; SSE-NEXT:    movapd %xmm0, %xmm4
-; SSE-NEXT:    subsd %xmm2, %xmm4
-; SSE-NEXT:    cvttsd2si %xmm4, %rcx
-; SSE-NEXT:    xorq %rax, %rcx
 ; SSE-NEXT:    cvttsd2si %xmm0, %rax
-; SSE-NEXT:    ucomisd %xmm2, %xmm0
-; SSE-NEXT:    cmovaeq %rcx, %rax
-; SSE-NEXT:    movq %rax, %xmm0
-; SSE-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
-; SSE-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,2],xmm3[0,2]
-; SSE-NEXT:    movaps %xmm1, %xmm0
+; SSE-NEXT:    movd %eax, %xmm0
+; SSE-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; SSE-NEXT:    movdqa %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: fptoui_4f64_to_4i32:
@@ -874,13 +816,11 @@ define <2 x i32> @fptosi_2f32_to_2i32(<2 x float> %a) {
 ; SSE-LABEL: fptosi_2f32_to_2i32:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    cvttps2dq %xmm0, %xmm0
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,1,3]
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: fptosi_2f32_to_2i32:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vcvttps2dq %xmm0, %xmm0
-; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
 ; AVX-NEXT:    retq
   %cvt = fptosi <2 x float> %a to <2 x i32>
   ret <2 x i32> %cvt
@@ -1259,77 +1199,66 @@ define <4 x i64> @fptosi_8f32_to_4i64(<8 x float> %a) {
 define <2 x i32> @fptoui_2f32_to_2i32(<2 x float> %a) {
 ; SSE-LABEL: fptoui_2f32_to_2i32:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE-NEXT:    movaps {{.*#+}} xmm2 = [2.14748365E+9,2.14748365E+9,2.14748365E+9,2.14748365E+9]
 ; SSE-NEXT:    movaps %xmm0, %xmm1
-; SSE-NEXT:    subss %xmm2, %xmm1
-; SSE-NEXT:    cvttss2si %xmm1, %rax
-; SSE-NEXT:    movabsq $-9223372036854775808, %rcx # imm = 0x8000000000000000
-; SSE-NEXT:    xorq %rcx, %rax
-; SSE-NEXT:    cvttss2si %xmm0, %rdx
-; SSE-NEXT:    ucomiss %xmm2, %xmm0
-; SSE-NEXT:    cmovaeq %rax, %rdx
-; SSE-NEXT:    movq %rdx, %xmm1
-; SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; SSE-NEXT:    movaps %xmm0, %xmm3
-; SSE-NEXT:    subss %xmm2, %xmm3
-; SSE-NEXT:    cvttss2si %xmm3, %rax
-; SSE-NEXT:    xorq %rcx, %rax
-; SSE-NEXT:    cvttss2si %xmm0, %rcx
-; SSE-NEXT:    ucomiss %xmm2, %xmm0
-; SSE-NEXT:    cmovaeq %rax, %rcx
-; SSE-NEXT:    movq %rcx, %xmm0
-; SSE-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
-; SSE-NEXT:    movdqa %xmm1, %xmm0
+; SSE-NEXT:    cmpltps %xmm2, %xmm1
+; SSE-NEXT:    cvttps2dq %xmm0, %xmm3
+; SSE-NEXT:    subps %xmm2, %xmm0
+; SSE-NEXT:    cvttps2dq %xmm0, %xmm0
+; SSE-NEXT:    xorps {{.*}}(%rip), %xmm0
+; SSE-NEXT:    andps %xmm1, %xmm3
+; SSE-NEXT:    andnps %xmm0, %xmm1
+; SSE-NEXT:    orps %xmm3, %xmm1
+; SSE-NEXT:    movaps %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
-; VEX-LABEL: fptoui_2f32_to_2i32:
-; VEX:       # %bb.0:
-; VEX-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; VEX-NEXT:    vsubss %xmm1, %xmm0, %xmm2
-; VEX-NEXT:    vcvttss2si %xmm2, %rax
-; VEX-NEXT:    movabsq $-9223372036854775808, %rcx # imm = 0x8000000000000000
-; VEX-NEXT:    xorq %rcx, %rax
-; VEX-NEXT:    vcvttss2si %xmm0, %rdx
-; VEX-NEXT:    vucomiss %xmm1, %xmm0
-; VEX-NEXT:    cmovaeq %rax, %rdx
-; VEX-NEXT:    vmovq %rdx, %xmm2
-; VEX-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; VEX-NEXT:    vsubss %xmm1, %xmm0, %xmm3
-; VEX-NEXT:    vcvttss2si %xmm3, %rax
-; VEX-NEXT:    xorq %rcx, %rax
-; VEX-NEXT:    vcvttss2si %xmm0, %rcx
-; VEX-NEXT:    vucomiss %xmm1, %xmm0
-; VEX-NEXT:    cmovaeq %rax, %rcx
-; VEX-NEXT:    vmovq %rcx, %xmm0
-; VEX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm2[0],xmm0[0]
-; VEX-NEXT:    retq
+; AVX1-LABEL: fptoui_2f32_to_2i32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vmovaps {{.*#+}} xmm1 = [2.14748365E+9,2.14748365E+9,2.14748365E+9,2.14748365E+9]
+; AVX1-NEXT:    vcmpltps %xmm1, %xmm0, %xmm2
+; AVX1-NEXT:    vsubps %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vcvttps2dq %xmm1, %xmm1
+; AVX1-NEXT:    vxorps {{.*}}(%rip), %xmm1, %xmm1
+; AVX1-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX1-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: fptoui_2f32_to_2i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vbroadcastss {{.*#+}} xmm1 = [2.14748365E+9,2.14748365E+9,2.14748365E+9,2.14748365E+9]
+; AVX2-NEXT:    vcmpltps %xmm1, %xmm0, %xmm2
+; AVX2-NEXT:    vsubps %xmm1, %xmm0, %xmm1
+; AVX2-NEXT:    vcvttps2dq %xmm1, %xmm1
+; AVX2-NEXT:    vbroadcastss {{.*#+}} xmm3 = [2147483648,2147483648,2147483648,2147483648]
+; AVX2-NEXT:    vxorps %xmm3, %xmm1, %xmm1
+; AVX2-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX2-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: fptoui_2f32_to_2i32:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; AVX512F-NEXT:    vcvttps2udq %zmm0, %zmm0
-; AVX512F-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: fptoui_2f32_to_2i32:
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vcvttps2udq %xmm0, %xmm0
-; AVX512VL-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512DQ-LABEL: fptoui_2f32_to_2i32:
 ; AVX512DQ:       # %bb.0:
 ; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; AVX512DQ-NEXT:    vcvttps2udq %zmm0, %zmm0
-; AVX512DQ-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
+; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512VLDQ-LABEL: fptoui_2f32_to_2i32:
 ; AVX512VLDQ:       # %bb.0:
 ; AVX512VLDQ-NEXT:    vcvttps2udq %xmm0, %xmm0
-; AVX512VLDQ-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
 ; AVX512VLDQ-NEXT:    retq
   %cvt = fptoui <2 x float> %a to <2 x i32>
   ret <2 x i32> %cvt
@@ -2225,7 +2154,8 @@ define <4 x i32> @fptosi_2f16_to_4i32(<2 x half> %a) nounwind {
 ; SSE-LABEL: fptosi_2f16_to_4i32:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pushq %rax
-; SSE-NEXT:    movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; SSE-NEXT:    movss %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; SSE-NEXT:    movaps %xmm1, %xmm0
 ; SSE-NEXT:    callq __gnu_f2h_ieee
 ; SSE-NEXT:    movzwl %ax, %edi
 ; SSE-NEXT:    callq __gnu_h2f_ieee
@@ -2235,20 +2165,20 @@ define <4 x i32> @fptosi_2f16_to_4i32(<2 x half> %a) nounwind {
 ; SSE-NEXT:    callq __gnu_f2h_ieee
 ; SSE-NEXT:    movzwl %ax, %edi
 ; SSE-NEXT:    callq __gnu_h2f_ieee
-; SSE-NEXT:    cvttss2si %xmm0, %rax
-; SSE-NEXT:    movq %rax, %xmm1
-; SSE-NEXT:    cvttss2si (%rsp), %rax # 4-byte Folded Reload
-; SSE-NEXT:    movq %rax, %xmm0
-; SSE-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSE-NEXT:    pxor %xmm1, %xmm1
-; SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[2,3]
+; SSE-NEXT:    cvttss2si %xmm0, %eax
+; SSE-NEXT:    cvttss2si (%rsp), %ecx # 4-byte Folded Reload
+; SSE-NEXT:    movd %ecx, %xmm0
+; SSE-NEXT:    movd %eax, %xmm1
+; SSE-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE-NEXT:    movq {{.*#+}} xmm0 = xmm1[0],zero
 ; SSE-NEXT:    popq %rax
 ; SSE-NEXT:    retq
 ;
 ; VEX-LABEL: fptosi_2f16_to_4i32:
 ; VEX:       # %bb.0:
 ; VEX-NEXT:    pushq %rax
-; VEX-NEXT:    vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; VEX-NEXT:    vmovss %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; VEX-NEXT:    vmovaps %xmm1, %xmm0
 ; VEX-NEXT:    callq __gnu_f2h_ieee
 ; VEX-NEXT:    movzwl %ax, %edi
 ; VEX-NEXT:    callq __gnu_h2f_ieee
@@ -2258,27 +2188,27 @@ define <4 x i32> @fptosi_2f16_to_4i32(<2 x half> %a) nounwind {
 ; VEX-NEXT:    callq __gnu_f2h_ieee
 ; VEX-NEXT:    movzwl %ax, %edi
 ; VEX-NEXT:    callq __gnu_h2f_ieee
-; VEX-NEXT:    vcvttss2si %xmm0, %rax
-; VEX-NEXT:    vmovq %rax, %xmm0
-; VEX-NEXT:    vcvttss2si (%rsp), %rax # 4-byte Folded Reload
-; VEX-NEXT:    vmovq %rax, %xmm1
-; VEX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; VEX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,2],zero,zero
+; VEX-NEXT:    vcvttss2si %xmm0, %eax
+; VEX-NEXT:    vcvttss2si (%rsp), %ecx # 4-byte Folded Reload
+; VEX-NEXT:    vmovd %ecx, %xmm0
+; VEX-NEXT:    vmovd %eax, %xmm1
+; VEX-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; VEX-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; VEX-NEXT:    popq %rax
 ; VEX-NEXT:    retq
 ;
 ; AVX512-LABEL: fptosi_2f16_to_4i32:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; AVX512-NEXT:    vcvtph2ps %xmm0, %xmm0
 ; AVX512-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
 ; AVX512-NEXT:    vcvtph2ps %xmm1, %xmm1
-; AVX512-NEXT:    vcvttss2si %xmm1, %rax
-; AVX512-NEXT:    vmovq %rax, %xmm1
-; AVX512-NEXT:    vcvttss2si %xmm0, %rax
-; AVX512-NEXT:    vmovq %rax, %xmm0
-; AVX512-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; AVX512-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,2],zero,zero
+; AVX512-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
+; AVX512-NEXT:    vcvtph2ps %xmm0, %xmm0
+; AVX512-NEXT:    vcvttss2si %xmm0, %eax
+; AVX512-NEXT:    vcvttss2si %xmm1, %ecx
+; AVX512-NEXT:    vmovd %ecx, %xmm0
+; AVX512-NEXT:    vmovd %eax, %xmm1
+; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; AVX512-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; AVX512-NEXT:    retq
   %cvt = fptosi <2 x half> %a to <2 x i32>
   %ext = shufflevector <2 x i32> %cvt, <2 x i32> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -2295,32 +2225,31 @@ define <4 x i32> @fptosi_2f80_to_4i32(<2 x x86_fp80> %a) nounwind {
 ; SSE-NEXT:    orl $3072, %eax # imm = 0xC00
 ; SSE-NEXT:    movw %ax, -{{[0-9]+}}(%rsp)
 ; SSE-NEXT:    fldcw -{{[0-9]+}}(%rsp)
-; SSE-NEXT:    fistpll -{{[0-9]+}}(%rsp)
+; SSE-NEXT:    fistpl -{{[0-9]+}}(%rsp)
 ; SSE-NEXT:    fldcw -{{[0-9]+}}(%rsp)
 ; SSE-NEXT:    fnstcw -{{[0-9]+}}(%rsp)
 ; SSE-NEXT:    movzwl -{{[0-9]+}}(%rsp), %eax
 ; SSE-NEXT:    orl $3072, %eax # imm = 0xC00
 ; SSE-NEXT:    movw %ax, -{{[0-9]+}}(%rsp)
 ; SSE-NEXT:    fldcw -{{[0-9]+}}(%rsp)
-; SSE-NEXT:    fistpll -{{[0-9]+}}(%rsp)
+; SSE-NEXT:    fistpl -{{[0-9]+}}(%rsp)
 ; SSE-NEXT:    fldcw -{{[0-9]+}}(%rsp)
-; SSE-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
-; SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; SSE-NEXT:    movlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSE-NEXT:    xorps %xmm1, %xmm1
-; SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[2,3]
+; SSE-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE-NEXT:    movq {{.*#+}} xmm0 = xmm1[0],zero
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: fptosi_2f80_to_4i32:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    fldt {{[0-9]+}}(%rsp)
 ; AVX-NEXT:    fldt {{[0-9]+}}(%rsp)
-; AVX-NEXT:    fisttpll -{{[0-9]+}}(%rsp)
-; AVX-NEXT:    fisttpll -{{[0-9]+}}(%rsp)
-; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; AVX-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,2],zero,zero
+; AVX-NEXT:    fisttpl -{{[0-9]+}}(%rsp)
+; AVX-NEXT:    fisttpl -{{[0-9]+}}(%rsp)
+; AVX-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; AVX-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; AVX-NEXT:    retq
   %cvt = fptosi <2 x x86_fp80> %a to <2 x i32>
   %ext = shufflevector <2 x i32> %cvt, <2 x i32> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -2330,51 +2259,36 @@ define <4 x i32> @fptosi_2f80_to_4i32(<2 x x86_fp80> %a) nounwind {
 define <4 x i32> @fptosi_2f128_to_4i32(<2 x fp128> %a) nounwind {
 ; SSE-LABEL: fptosi_2f128_to_4i32:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pushq %r14
 ; SSE-NEXT:    pushq %rbx
-; SSE-NEXT:    subq $24, %rsp
-; SSE-NEXT:    movq %rsi, %r14
-; SSE-NEXT:    movq %rdi, %rbx
-; SSE-NEXT:    movq %rdx, %rdi
-; SSE-NEXT:    movq %rcx, %rsi
-; SSE-NEXT:    callq __fixtfdi
-; SSE-NEXT:    movq %rax, %xmm0
-; SSE-NEXT:    movdqa %xmm0, (%rsp) # 16-byte Spill
-; SSE-NEXT:    movq %rbx, %rdi
-; SSE-NEXT:    movq %r14, %rsi
-; SSE-NEXT:    callq __fixtfdi
-; SSE-NEXT:    movq %rax, %xmm0
-; SSE-NEXT:    punpcklqdq (%rsp), %xmm0 # 16-byte Folded Reload
-; SSE-NEXT:    # xmm0 = xmm0[0],mem[0]
-; SSE-NEXT:    xorps %xmm1, %xmm1
-; SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[2,3]
-; SSE-NEXT:    addq $24, %rsp
+; SSE-NEXT:    subq $16, %rsp
+; SSE-NEXT:    movaps %xmm1, (%rsp) # 16-byte Spill
+; SSE-NEXT:    callq __fixtfsi
+; SSE-NEXT:    movl %eax, %ebx
+; SSE-NEXT:    movaps (%rsp), %xmm0 # 16-byte Reload
+; SSE-NEXT:    callq __fixtfsi
+; SSE-NEXT:    movd %eax, %xmm0
+; SSE-NEXT:    movd %ebx, %xmm1
+; SSE-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE-NEXT:    movq {{.*#+}} xmm0 = xmm1[0],zero
+; SSE-NEXT:    addq $16, %rsp
 ; SSE-NEXT:    popq %rbx
-; SSE-NEXT:    popq %r14
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: fptosi_2f128_to_4i32:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    pushq %r14
 ; AVX-NEXT:    pushq %rbx
-; AVX-NEXT:    subq $24, %rsp
-; AVX-NEXT:    movq %rsi, %r14
-; AVX-NEXT:    movq %rdi, %rbx
-; AVX-NEXT:    movq %rdx, %rdi
-; AVX-NEXT:    movq %rcx, %rsi
-; AVX-NEXT:    callq __fixtfdi
-; AVX-NEXT:    vmovq %rax, %xmm0
-; AVX-NEXT:    vmovdqa %xmm0, (%rsp) # 16-byte Spill
-; AVX-NEXT:    movq %rbx, %rdi
-; AVX-NEXT:    movq %r14, %rsi
-; AVX-NEXT:    callq __fixtfdi
-; AVX-NEXT:    vmovq %rax, %xmm0
-; AVX-NEXT:    vpunpcklqdq (%rsp), %xmm0, %xmm0 # 16-byte Folded Reload
-; AVX-NEXT:    # xmm0 = xmm0[0],mem[0]
-; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,2],zero,zero
-; AVX-NEXT:    addq $24, %rsp
+; AVX-NEXT:    subq $16, %rsp
+; AVX-NEXT:    vmovaps %xmm1, (%rsp) # 16-byte Spill
+; AVX-NEXT:    callq __fixtfsi
+; AVX-NEXT:    movl %eax, %ebx
+; AVX-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
+; AVX-NEXT:    callq __fixtfsi
+; AVX-NEXT:    vmovd %eax, %xmm0
+; AVX-NEXT:    vmovd %ebx, %xmm1
+; AVX-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; AVX-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
+; AVX-NEXT:    addq $16, %rsp
 ; AVX-NEXT:    popq %rbx
-; AVX-NEXT:    popq %r14
 ; AVX-NEXT:    retq
   %cvt = fptosi <2 x fp128> %a to <2 x i32>
   %ext = shufflevector <2 x i32> %cvt, <2 x i32> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -2385,41 +2299,16 @@ define <2 x i8> @fptosi_2f32_to_2i8(<2 x float> %a) {
 ; SSE-LABEL: fptosi_2f32_to_2i8:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    cvttps2dq %xmm0, %xmm0
-; SSE-NEXT:    pxor %xmm1, %xmm1
-; SSE-NEXT:    pcmpgtd %xmm0, %xmm1
-; SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE-NEXT:    packuswb %xmm0, %xmm0
+; SSE-NEXT:    packuswb %xmm0, %xmm0
 ; SSE-NEXT:    retq
 ;
-; VEX-LABEL: fptosi_2f32_to_2i8:
-; VEX:       # %bb.0:
-; VEX-NEXT:    vcvttps2dq %xmm0, %xmm0
-; VEX-NEXT:    vpmovsxdq %xmm0, %xmm0
-; VEX-NEXT:    retq
-;
-; AVX512F-LABEL: fptosi_2f32_to_2i8:
-; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vcvttps2dq %xmm0, %xmm0
-; AVX512F-NEXT:    vpmovsxdq %xmm0, %xmm0
-; AVX512F-NEXT:    retq
-;
-; AVX512VL-LABEL: fptosi_2f32_to_2i8:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vcvttps2dq %xmm0, %xmm0
-; AVX512VL-NEXT:    vpmovsxdq %xmm0, %xmm0
-; AVX512VL-NEXT:    retq
-;
-; AVX512DQ-LABEL: fptosi_2f32_to_2i8:
-; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
-; AVX512DQ-NEXT:    vcvttps2qq %ymm0, %zmm0
-; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
-; AVX512DQ-NEXT:    vzeroupper
-; AVX512DQ-NEXT:    retq
-;
-; AVX512VLDQ-LABEL: fptosi_2f32_to_2i8:
-; AVX512VLDQ:       # %bb.0:
-; AVX512VLDQ-NEXT:    vcvttps2qq %xmm0, %xmm0
-; AVX512VLDQ-NEXT:    retq
+; AVX-LABEL: fptosi_2f32_to_2i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX-NEXT:    retq
   %cvt = fptosi <2 x float> %a to <2 x i8>
   ret <2 x i8> %cvt
 }
@@ -2428,41 +2317,14 @@ define <2 x i16> @fptosi_2f32_to_2i16(<2 x float> %a) {
 ; SSE-LABEL: fptosi_2f32_to_2i16:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    cvttps2dq %xmm0, %xmm0
-; SSE-NEXT:    pxor %xmm1, %xmm1
-; SSE-NEXT:    pcmpgtd %xmm0, %xmm1
-; SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
 ; SSE-NEXT:    retq
 ;
-; VEX-LABEL: fptosi_2f32_to_2i16:
-; VEX:       # %bb.0:
-; VEX-NEXT:    vcvttps2dq %xmm0, %xmm0
-; VEX-NEXT:    vpmovsxdq %xmm0, %xmm0
-; VEX-NEXT:    retq
-;
-; AVX512F-LABEL: fptosi_2f32_to_2i16:
-; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vcvttps2dq %xmm0, %xmm0
-; AVX512F-NEXT:    vpmovsxdq %xmm0, %xmm0
-; AVX512F-NEXT:    retq
-;
-; AVX512VL-LABEL: fptosi_2f32_to_2i16:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vcvttps2dq %xmm0, %xmm0
-; AVX512VL-NEXT:    vpmovsxdq %xmm0, %xmm0
-; AVX512VL-NEXT:    retq
-;
-; AVX512DQ-LABEL: fptosi_2f32_to_2i16:
-; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
-; AVX512DQ-NEXT:    vcvttps2qq %ymm0, %zmm0
-; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
-; AVX512DQ-NEXT:    vzeroupper
-; AVX512DQ-NEXT:    retq
-;
-; AVX512VLDQ-LABEL: fptosi_2f32_to_2i16:
-; AVX512VLDQ:       # %bb.0:
-; AVX512VLDQ-NEXT:    vcvttps2qq %xmm0, %xmm0
-; AVX512VLDQ-NEXT:    retq
+; AVX-LABEL: fptosi_2f32_to_2i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; AVX-NEXT:    retq
   %cvt = fptosi <2 x float> %a to <2 x i16>
   ret <2 x i16> %cvt
 }
@@ -2471,40 +2333,16 @@ define <2 x i8> @fptoui_2f32_to_2i8(<2 x float> %a) {
 ; SSE-LABEL: fptoui_2f32_to_2i8:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    cvttps2dq %xmm0, %xmm0
-; SSE-NEXT:    xorps %xmm1, %xmm1
-; SSE-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE-NEXT:    packuswb %xmm0, %xmm0
+; SSE-NEXT:    packuswb %xmm0, %xmm0
 ; SSE-NEXT:    retq
 ;
-; VEX-LABEL: fptoui_2f32_to_2i8:
-; VEX:       # %bb.0:
-; VEX-NEXT:    vcvttps2dq %xmm0, %xmm0
-; VEX-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
-; VEX-NEXT:    retq
-;
-; AVX512F-LABEL: fptoui_2f32_to_2i8:
-; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vcvttps2dq %xmm0, %xmm0
-; AVX512F-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
-; AVX512F-NEXT:    retq
-;
-; AVX512VL-LABEL: fptoui_2f32_to_2i8:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vcvttps2dq %xmm0, %xmm0
-; AVX512VL-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
-; AVX512VL-NEXT:    retq
-;
-; AVX512DQ-LABEL: fptoui_2f32_to_2i8:
-; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
-; AVX512DQ-NEXT:    vcvttps2uqq %ymm0, %zmm0
-; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
-; AVX512DQ-NEXT:    vzeroupper
-; AVX512DQ-NEXT:    retq
-;
-; AVX512VLDQ-LABEL: fptoui_2f32_to_2i8:
-; AVX512VLDQ:       # %bb.0:
-; AVX512VLDQ-NEXT:    vcvttps2uqq %xmm0, %xmm0
-; AVX512VLDQ-NEXT:    retq
+; AVX-LABEL: fptoui_2f32_to_2i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX-NEXT:    retq
   %cvt = fptoui <2 x float> %a to <2 x i8>
   ret <2 x i8> %cvt
 }
@@ -2513,40 +2351,14 @@ define <2 x i16> @fptoui_2f32_to_2i16(<2 x float> %a) {
 ; SSE-LABEL: fptoui_2f32_to_2i16:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    cvttps2dq %xmm0, %xmm0
-; SSE-NEXT:    xorps %xmm1, %xmm1
-; SSE-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
 ; SSE-NEXT:    retq
 ;
-; VEX-LABEL: fptoui_2f32_to_2i16:
-; VEX:       # %bb.0:
-; VEX-NEXT:    vcvttps2dq %xmm0, %xmm0
-; VEX-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
-; VEX-NEXT:    retq
-;
-; AVX512F-LABEL: fptoui_2f32_to_2i16:
-; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vcvttps2dq %xmm0, %xmm0
-; AVX512F-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
-; AVX512F-NEXT:    retq
-;
-; AVX512VL-LABEL: fptoui_2f32_to_2i16:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vcvttps2dq %xmm0, %xmm0
-; AVX512VL-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
-; AVX512VL-NEXT:    retq
-;
-; AVX512DQ-LABEL: fptoui_2f32_to_2i16:
-; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
-; AVX512DQ-NEXT:    vcvttps2uqq %ymm0, %zmm0
-; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
-; AVX512DQ-NEXT:    vzeroupper
-; AVX512DQ-NEXT:    retq
-;
-; AVX512VLDQ-LABEL: fptoui_2f32_to_2i16:
-; AVX512VLDQ:       # %bb.0:
-; AVX512VLDQ-NEXT:    vcvttps2uqq %xmm0, %xmm0
-; AVX512VLDQ-NEXT:    retq
+; AVX-LABEL: fptoui_2f32_to_2i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; AVX-NEXT:    retq
   %cvt = fptoui <2 x float> %a to <2 x i16>
   ret <2 x i16> %cvt
 }
@@ -2555,41 +2367,16 @@ define <2 x i8> @fptosi_2f64_to_2i8(<2 x double> %a) {
 ; SSE-LABEL: fptosi_2f64_to_2i8:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    cvttpd2dq %xmm0, %xmm0
-; SSE-NEXT:    pxor %xmm1, %xmm1
-; SSE-NEXT:    pcmpgtd %xmm0, %xmm1
-; SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE-NEXT:    andpd {{.*}}(%rip), %xmm0
+; SSE-NEXT:    packuswb %xmm0, %xmm0
+; SSE-NEXT:    packuswb %xmm0, %xmm0
 ; SSE-NEXT:    retq
 ;
-; VEX-LABEL: fptosi_2f64_to_2i8:
-; VEX:       # %bb.0:
-; VEX-NEXT:    vcvttpd2dq %xmm0, %xmm0
-; VEX-NEXT:    vpmovsxdq %xmm0, %xmm0
-; VEX-NEXT:    retq
-;
-; AVX512F-LABEL: fptosi_2f64_to_2i8:
-; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vcvttpd2dq %xmm0, %xmm0
-; AVX512F-NEXT:    vpmovsxdq %xmm0, %xmm0
-; AVX512F-NEXT:    retq
-;
-; AVX512VL-LABEL: fptosi_2f64_to_2i8:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vcvttpd2dq %xmm0, %xmm0
-; AVX512VL-NEXT:    vpmovsxdq %xmm0, %xmm0
-; AVX512VL-NEXT:    retq
-;
-; AVX512DQ-LABEL: fptosi_2f64_to_2i8:
-; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; AVX512DQ-NEXT:    vcvttpd2qq %zmm0, %zmm0
-; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
-; AVX512DQ-NEXT:    vzeroupper
-; AVX512DQ-NEXT:    retq
-;
-; AVX512VLDQ-LABEL: fptosi_2f64_to_2i8:
-; AVX512VLDQ:       # %bb.0:
-; AVX512VLDQ-NEXT:    vcvttpd2qq %xmm0, %xmm0
-; AVX512VLDQ-NEXT:    retq
+; AVX-LABEL: fptosi_2f64_to_2i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcvttpd2dq %xmm0, %xmm0
+; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX-NEXT:    retq
   %cvt = fptosi <2 x double> %a to <2 x i8>
   ret <2 x i8> %cvt
 }
@@ -2598,41 +2385,14 @@ define <2 x i16> @fptosi_2f64_to_2i16(<2 x double> %a) {
 ; SSE-LABEL: fptosi_2f64_to_2i16:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    cvttpd2dq %xmm0, %xmm0
-; SSE-NEXT:    pxor %xmm1, %xmm1
-; SSE-NEXT:    pcmpgtd %xmm0, %xmm1
-; SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
 ; SSE-NEXT:    retq
 ;
-; VEX-LABEL: fptosi_2f64_to_2i16:
-; VEX:       # %bb.0:
-; VEX-NEXT:    vcvttpd2dq %xmm0, %xmm0
-; VEX-NEXT:    vpmovsxdq %xmm0, %xmm0
-; VEX-NEXT:    retq
-;
-; AVX512F-LABEL: fptosi_2f64_to_2i16:
-; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vcvttpd2dq %xmm0, %xmm0
-; AVX512F-NEXT:    vpmovsxdq %xmm0, %xmm0
-; AVX512F-NEXT:    retq
-;
-; AVX512VL-LABEL: fptosi_2f64_to_2i16:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vcvttpd2dq %xmm0, %xmm0
-; AVX512VL-NEXT:    vpmovsxdq %xmm0, %xmm0
-; AVX512VL-NEXT:    retq
-;
-; AVX512DQ-LABEL: fptosi_2f64_to_2i16:
-; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; AVX512DQ-NEXT:    vcvttpd2qq %zmm0, %zmm0
-; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
-; AVX512DQ-NEXT:    vzeroupper
-; AVX512DQ-NEXT:    retq
-;
-; AVX512VLDQ-LABEL: fptosi_2f64_to_2i16:
-; AVX512VLDQ:       # %bb.0:
-; AVX512VLDQ-NEXT:    vcvttpd2qq %xmm0, %xmm0
-; AVX512VLDQ-NEXT:    retq
+; AVX-LABEL: fptosi_2f64_to_2i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcvttpd2dq %xmm0, %xmm0
+; AVX-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; AVX-NEXT:    retq
   %cvt = fptosi <2 x double> %a to <2 x i16>
   ret <2 x i16> %cvt
 }
@@ -2641,40 +2401,16 @@ define <2 x i8> @fptoui_2f64_to_2i8(<2 x double> %a) {
 ; SSE-LABEL: fptoui_2f64_to_2i8:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    cvttpd2dq %xmm0, %xmm0
-; SSE-NEXT:    xorpd %xmm1, %xmm1
-; SSE-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE-NEXT:    andpd {{.*}}(%rip), %xmm0
+; SSE-NEXT:    packuswb %xmm0, %xmm0
+; SSE-NEXT:    packuswb %xmm0, %xmm0
 ; SSE-NEXT:    retq
 ;
-; VEX-LABEL: fptoui_2f64_to_2i8:
-; VEX:       # %bb.0:
-; VEX-NEXT:    vcvttpd2dq %xmm0, %xmm0
-; VEX-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
-; VEX-NEXT:    retq
-;
-; AVX512F-LABEL: fptoui_2f64_to_2i8:
-; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vcvttpd2dq %xmm0, %xmm0
-; AVX512F-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
-; AVX512F-NEXT:    retq
-;
-; AVX512VL-LABEL: fptoui_2f64_to_2i8:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vcvttpd2dq %xmm0, %xmm0
-; AVX512VL-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
-; AVX512VL-NEXT:    retq
-;
-; AVX512DQ-LABEL: fptoui_2f64_to_2i8:
-; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; AVX512DQ-NEXT:    vcvttpd2uqq %zmm0, %zmm0
-; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
-; AVX512DQ-NEXT:    vzeroupper
-; AVX512DQ-NEXT:    retq
-;
-; AVX512VLDQ-LABEL: fptoui_2f64_to_2i8:
-; AVX512VLDQ:       # %bb.0:
-; AVX512VLDQ-NEXT:    vcvttpd2uqq %xmm0, %xmm0
-; AVX512VLDQ-NEXT:    retq
+; AVX-LABEL: fptoui_2f64_to_2i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcvttpd2dq %xmm0, %xmm0
+; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX-NEXT:    retq
   %cvt = fptoui <2 x double> %a to <2 x i8>
   ret <2 x i8> %cvt
 }
@@ -2683,40 +2419,14 @@ define <2 x i16> @fptoui_2f64_to_2i16(<2 x double> %a) {
 ; SSE-LABEL: fptoui_2f64_to_2i16:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    cvttpd2dq %xmm0, %xmm0
-; SSE-NEXT:    xorpd %xmm1, %xmm1
-; SSE-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
 ; SSE-NEXT:    retq
 ;
-; VEX-LABEL: fptoui_2f64_to_2i16:
-; VEX:       # %bb.0:
-; VEX-NEXT:    vcvttpd2dq %xmm0, %xmm0
-; VEX-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
-; VEX-NEXT:    retq
-;
-; AVX512F-LABEL: fptoui_2f64_to_2i16:
-; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vcvttpd2dq %xmm0, %xmm0
-; AVX512F-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
-; AVX512F-NEXT:    retq
-;
-; AVX512VL-LABEL: fptoui_2f64_to_2i16:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vcvttpd2dq %xmm0, %xmm0
-; AVX512VL-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
-; AVX512VL-NEXT:    retq
-;
-; AVX512DQ-LABEL: fptoui_2f64_to_2i16:
-; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; AVX512DQ-NEXT:    vcvttpd2uqq %zmm0, %zmm0
-; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
-; AVX512DQ-NEXT:    vzeroupper
-; AVX512DQ-NEXT:    retq
-;
-; AVX512VLDQ-LABEL: fptoui_2f64_to_2i16:
-; AVX512VLDQ:       # %bb.0:
-; AVX512VLDQ-NEXT:    vcvttpd2uqq %xmm0, %xmm0
-; AVX512VLDQ-NEXT:    retq
+; AVX-LABEL: fptoui_2f64_to_2i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcvttpd2dq %xmm0, %xmm0
+; AVX-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; AVX-NEXT:    retq
   %cvt = fptoui <2 x double> %a to <2 x i16>
   ret <2 x i16> %cvt
 }
@@ -2895,10 +2605,10 @@ define <16 x i8> @fptoui_16f32_to_16i8(<16 x float> %a) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vcvttps2dq %ymm1, %ymm1
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT:    vpackusdw %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpackssdw %xmm2, %xmm1, %xmm1
 ; AVX1-NEXT:    vcvttps2dq %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX1-NEXT:    vpackusdw %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpackssdw %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -2907,10 +2617,10 @@ define <16 x i8> @fptoui_16f32_to_16i8(<16 x float> %a) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vcvttps2dq %ymm1, %ymm1
 ; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX2-NEXT:    vpackusdw %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpackssdw %xmm2, %xmm1, %xmm1
 ; AVX2-NEXT:    vcvttps2dq %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm2
-; AVX2-NEXT:    vpackusdw %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpackssdw %xmm2, %xmm0, %xmm0
 ; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/vec_insert-5.ll b/llvm/test/CodeGen/X86/vec_insert-5.ll
index 3f8bc4adb..cd9d2692f 100644
--- a/llvm/test/CodeGen/X86/vec_insert-5.ll
+++ b/llvm/test/CodeGen/X86/vec_insert-5.ll
@@ -19,8 +19,7 @@ define void  @t1(i32 %a, x86_mmx* %P) nounwind {
 ; X64:       # %bb.0:
 ; X64-NEXT:    shll $12, %edi
 ; X64-NEXT:    movd %edi, %xmm0
-; X64-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6,7]
-; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,0,1,1]
 ; X64-NEXT:    movq %xmm0, (%rsi)
 ; X64-NEXT:    retq
  %tmp12 = shl i32 %a, 12
diff --git a/llvm/test/CodeGen/X86/vec_insert-7.ll b/llvm/test/CodeGen/X86/vec_insert-7.ll
index 94204ee4c..52d6e7ca7 100644
--- a/llvm/test/CodeGen/X86/vec_insert-7.ll
+++ b/llvm/test/CodeGen/X86/vec_insert-7.ll
@@ -8,11 +8,12 @@
 define x86_mmx @mmx_movzl(x86_mmx %x) nounwind {
 ; X32-LABEL: mmx_movzl:
 ; X32:       ## %bb.0:
-; X32-NEXT:    subl $12, %esp
+; X32-NEXT:    movq2dq %mm0, %xmm0
 ; X32-NEXT:    movl $32, %eax
-; X32-NEXT:    movd %eax, %xmm0
-; X32-NEXT:    movdq2q %xmm0, %mm0
-; X32-NEXT:    addl $12, %esp
+; X32-NEXT:    pinsrd $0, %eax, %xmm0
+; X32-NEXT:    pxor %xmm1, %xmm1
+; X32-NEXT:    pblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3],xmm0[4,5,6,7]
+; X32-NEXT:    movdq2q %xmm1, %mm0
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: mmx_movzl:
diff --git a/llvm/test/CodeGen/X86/vec_insert-mmx.ll b/llvm/test/CodeGen/X86/vec_insert-mmx.ll
index 07fdbcae4..1c190f333 100644
--- a/llvm/test/CodeGen/X86/vec_insert-mmx.ll
+++ b/llvm/test/CodeGen/X86/vec_insert-mmx.ll
@@ -61,13 +61,7 @@ define void @t3() {
 ; X32-NEXT:    movl L_g0$non_lazy_ptr, %eax
 ; X32-NEXT:    movl L_g1$non_lazy_ptr, %ecx
 ; X32-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X32-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
-; X32-NEXT:    movzwl (%eax), %eax
-; X32-NEXT:    movd %eax, %xmm1
-; X32-NEXT:    movss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
-; X32-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
-; X32-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,6,6,7]
-; X32-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X32-NEXT:    pinsrw $0, (%eax), %xmm0
 ; X32-NEXT:    movq %xmm0, (%ecx)
 ; X32-NEXT:    retl
 ;
@@ -75,10 +69,8 @@ define void @t3() {
 ; X64:       ## %bb.0:
 ; X64-NEXT:    movq _g0@{{.*}}(%rip), %rax
 ; X64-NEXT:    movq _g1@{{.*}}(%rip), %rcx
-; X64-NEXT:    pmovzxwd {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
-; X64-NEXT:    movzwl (%rax), %eax
-; X64-NEXT:    pinsrd $0, %eax, %xmm0
-; X64-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
+; X64-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; X64-NEXT:    pinsrw $0, (%rax), %xmm0
 ; X64-NEXT:    movq %xmm0, (%rcx)
 ; X64-NEXT:    retq
   load i16, i16* @g0
diff --git a/llvm/test/CodeGen/X86/vec_int_to_fp.ll b/llvm/test/CodeGen/X86/vec_int_to_fp.ll
index ceacb6c7b..c310e722b 100644
--- a/llvm/test/CodeGen/X86/vec_int_to_fp.ll
+++ b/llvm/test/CodeGen/X86/vec_int_to_fp.ll
@@ -18,6 +18,78 @@
 ; Signed Integer to Double
 ;
 
+define <2 x float> @sitofp_2i32_to_2f32(<2 x i32> %a) {
+; SSE-LABEL: sitofp_2i32_to_2f32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    cvtdq2ps %xmm0, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: sitofp_2i32_to_2f32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcvtdq2ps %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %cvt = sitofp <2 x i32> %a to <2 x float>
+  ret <2 x float> %cvt
+}
+
+define <2 x float> @uitofp_2i32_to_2f32(<2 x i32> %a) {
+; SSE2-LABEL: uitofp_2i32_to_2f32:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    xorpd %xmm1, %xmm1
+; SSE2-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT:    movapd {{.*#+}} xmm1 = [4.503599627370496E+15,4.503599627370496E+15]
+; SSE2-NEXT:    orpd %xmm1, %xmm0
+; SSE2-NEXT:    subpd %xmm1, %xmm0
+; SSE2-NEXT:    cvtpd2ps %xmm0, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: uitofp_2i32_to_2f32:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
+; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [4.503599627370496E+15,4.503599627370496E+15]
+; SSE41-NEXT:    por %xmm1, %xmm0
+; SSE41-NEXT:    subpd %xmm1, %xmm0
+; SSE41-NEXT:    cvtpd2ps %xmm0, %xmm0
+; SSE41-NEXT:    retq
+;
+; VEX-LABEL: uitofp_2i32_to_2f32:
+; VEX:       # %bb.0:
+; VEX-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
+; VEX-NEXT:    vmovdqa {{.*#+}} xmm1 = [4.503599627370496E+15,4.503599627370496E+15]
+; VEX-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; VEX-NEXT:    vsubpd %xmm1, %xmm0, %xmm0
+; VEX-NEXT:    vcvtpd2ps %xmm0, %xmm0
+; VEX-NEXT:    retq
+;
+; AVX512F-LABEL: uitofp_2i32_to_2f32:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512F-NEXT:    vcvtudq2ps %zmm0, %zmm0
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: uitofp_2i32_to_2f32:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vcvtudq2ps %xmm0, %xmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512DQ-LABEL: uitofp_2i32_to_2f32:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512DQ-NEXT:    vcvtudq2ps %zmm0, %zmm0
+; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    retq
+;
+; AVX512VLDQ-LABEL: uitofp_2i32_to_2f32:
+; AVX512VLDQ:       # %bb.0:
+; AVX512VLDQ-NEXT:    vcvtudq2ps %xmm0, %xmm0
+; AVX512VLDQ-NEXT:    retq
+  %cvt = uitofp <2 x i32> %a to <2 x float>
+  ret <2 x float> %cvt
+}
+
 define <2 x double> @sitofp_2i64_to_2f64(<2 x i64> %a) {
 ; SSE2-LABEL: sitofp_2i64_to_2f64:
 ; SSE2:       # %bb.0:
@@ -27,8 +99,8 @@ define <2 x double> @sitofp_2i64_to_2f64(<2 x i64> %a) {
 ; SSE2-NEXT:    movq %xmm0, %rax
 ; SSE2-NEXT:    xorps %xmm0, %xmm0
 ; SSE2-NEXT:    cvtsi2sd %rax, %xmm0
-; SSE2-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0]
-; SSE2-NEXT:    movaps %xmm1, %xmm0
+; SSE2-NEXT:    unpcklpd {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSE2-NEXT:    movapd %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: sitofp_2i64_to_2f64:
@@ -38,7 +110,7 @@ define <2 x double> @sitofp_2i64_to_2f64(<2 x i64> %a) {
 ; SSE41-NEXT:    movq %xmm0, %rax
 ; SSE41-NEXT:    xorps %xmm0, %xmm0
 ; SSE41-NEXT:    cvtsi2sd %rax, %xmm0
-; SSE41-NEXT:    movlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE41-NEXT:    unpcklpd {{.*#+}} xmm0 = xmm0[0],xmm1[0]
 ; SSE41-NEXT:    retq
 ;
 ; VEX-LABEL: sitofp_2i64_to_2f64:
@@ -47,7 +119,7 @@ define <2 x double> @sitofp_2i64_to_2f64(<2 x i64> %a) {
 ; VEX-NEXT:    vcvtsi2sd %rax, %xmm1, %xmm1
 ; VEX-NEXT:    vmovq %xmm0, %rax
 ; VEX-NEXT:    vcvtsi2sd %rax, %xmm2, %xmm0
-; VEX-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; VEX-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm1[0]
 ; VEX-NEXT:    retq
 ;
 ; AVX512F-LABEL: sitofp_2i64_to_2f64:
@@ -56,7 +128,7 @@ define <2 x double> @sitofp_2i64_to_2f64(<2 x i64> %a) {
 ; AVX512F-NEXT:    vcvtsi2sd %rax, %xmm1, %xmm1
 ; AVX512F-NEXT:    vmovq %xmm0, %rax
 ; AVX512F-NEXT:    vcvtsi2sd %rax, %xmm2, %xmm0
-; AVX512F-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX512F-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm1[0]
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: sitofp_2i64_to_2f64:
@@ -65,7 +137,7 @@ define <2 x double> @sitofp_2i64_to_2f64(<2 x i64> %a) {
 ; AVX512VL-NEXT:    vcvtsi2sd %rax, %xmm1, %xmm1
 ; AVX512VL-NEXT:    vmovq %xmm0, %rax
 ; AVX512VL-NEXT:    vcvtsi2sd %rax, %xmm2, %xmm0
-; AVX512VL-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX512VL-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm1[0]
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512DQ-LABEL: sitofp_2i64_to_2f64:
@@ -237,16 +309,16 @@ define <4 x double> @sitofp_4i64_to_4f64(<4 x i64> %a) {
 ; SSE2-NEXT:    movq %xmm0, %rax
 ; SSE2-NEXT:    xorps %xmm0, %xmm0
 ; SSE2-NEXT:    cvtsi2sd %rax, %xmm0
-; SSE2-NEXT:    movlhps {{.*#+}} xmm2 = xmm2[0],xmm0[0]
+; SSE2-NEXT:    unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm0[0]
 ; SSE2-NEXT:    movq %xmm1, %rax
 ; SSE2-NEXT:    cvtsi2sd %rax, %xmm3
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
 ; SSE2-NEXT:    movq %xmm0, %rax
 ; SSE2-NEXT:    xorps %xmm0, %xmm0
 ; SSE2-NEXT:    cvtsi2sd %rax, %xmm0
-; SSE2-NEXT:    movlhps {{.*#+}} xmm3 = xmm3[0],xmm0[0]
-; SSE2-NEXT:    movaps %xmm2, %xmm0
-; SSE2-NEXT:    movaps %xmm3, %xmm1
+; SSE2-NEXT:    unpcklpd {{.*#+}} xmm3 = xmm3[0],xmm0[0]
+; SSE2-NEXT:    movapd %xmm2, %xmm0
+; SSE2-NEXT:    movapd %xmm3, %xmm1
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: sitofp_4i64_to_4f64:
@@ -256,14 +328,14 @@ define <4 x double> @sitofp_4i64_to_4f64(<4 x i64> %a) {
 ; SSE41-NEXT:    movq %xmm0, %rax
 ; SSE41-NEXT:    xorps %xmm0, %xmm0
 ; SSE41-NEXT:    cvtsi2sd %rax, %xmm0
-; SSE41-NEXT:    movlhps {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SSE41-NEXT:    unpcklpd {{.*#+}} xmm0 = xmm0[0],xmm2[0]
 ; SSE41-NEXT:    pextrq $1, %xmm1, %rax
 ; SSE41-NEXT:    xorps %xmm2, %xmm2
 ; SSE41-NEXT:    cvtsi2sd %rax, %xmm2
 ; SSE41-NEXT:    movq %xmm1, %rax
 ; SSE41-NEXT:    xorps %xmm1, %xmm1
 ; SSE41-NEXT:    cvtsi2sd %rax, %xmm1
-; SSE41-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; SSE41-NEXT:    unpcklpd {{.*#+}} xmm1 = xmm1[0],xmm2[0]
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: sitofp_4i64_to_4f64:
@@ -273,12 +345,12 @@ define <4 x double> @sitofp_4i64_to_4f64(<4 x i64> %a) {
 ; AVX1-NEXT:    vcvtsi2sd %rax, %xmm2, %xmm2
 ; AVX1-NEXT:    vmovq %xmm1, %rax
 ; AVX1-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm1
-; AVX1-NEXT:    vmovlhps {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; AVX1-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm1[0],xmm2[0]
 ; AVX1-NEXT:    vpextrq $1, %xmm0, %rax
 ; AVX1-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm2
 ; AVX1-NEXT:    vmovq %xmm0, %rax
 ; AVX1-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm0
-; AVX1-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; AVX1-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm2[0]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
@@ -289,12 +361,12 @@ define <4 x double> @sitofp_4i64_to_4f64(<4 x i64> %a) {
 ; AVX2-NEXT:    vcvtsi2sd %rax, %xmm2, %xmm2
 ; AVX2-NEXT:    vmovq %xmm1, %rax
 ; AVX2-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm1
-; AVX2-NEXT:    vmovlhps {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; AVX2-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm1[0],xmm2[0]
 ; AVX2-NEXT:    vpextrq $1, %xmm0, %rax
 ; AVX2-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm2
 ; AVX2-NEXT:    vmovq %xmm0, %rax
 ; AVX2-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm0
-; AVX2-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; AVX2-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm2[0]
 ; AVX2-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX2-NEXT:    retq
 ;
@@ -305,12 +377,12 @@ define <4 x double> @sitofp_4i64_to_4f64(<4 x i64> %a) {
 ; AVX512F-NEXT:    vcvtsi2sd %rax, %xmm2, %xmm2
 ; AVX512F-NEXT:    vmovq %xmm1, %rax
 ; AVX512F-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm1
-; AVX512F-NEXT:    vmovlhps {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; AVX512F-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm1[0],xmm2[0]
 ; AVX512F-NEXT:    vpextrq $1, %xmm0, %rax
 ; AVX512F-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm2
 ; AVX512F-NEXT:    vmovq %xmm0, %rax
 ; AVX512F-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm0
-; AVX512F-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; AVX512F-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm2[0]
 ; AVX512F-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX512F-NEXT:    retq
 ;
@@ -321,12 +393,12 @@ define <4 x double> @sitofp_4i64_to_4f64(<4 x i64> %a) {
 ; AVX512VL-NEXT:    vcvtsi2sd %rax, %xmm2, %xmm2
 ; AVX512VL-NEXT:    vmovq %xmm1, %rax
 ; AVX512VL-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm1
-; AVX512VL-NEXT:    vmovlhps {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; AVX512VL-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm1[0],xmm2[0]
 ; AVX512VL-NEXT:    vpextrq $1, %xmm0, %rax
 ; AVX512VL-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm2
 ; AVX512VL-NEXT:    vmovq %xmm0, %rax
 ; AVX512VL-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm0
-; AVX512VL-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; AVX512VL-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm2[0]
 ; AVX512VL-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
@@ -579,35 +651,27 @@ define <2 x double> @uitofp_2i64_to_2f64(<2 x i64> %a) {
 define <2 x double> @uitofp_2i32_to_2f64(<4 x i32> %a) {
 ; SSE2-LABEL: uitofp_2i32_to_2f64:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [65535,0,65535,0,0,0,0,0]
-; SSE2-NEXT:    pand %xmm0, %xmm1
-; SSE2-NEXT:    cvtdq2pd %xmm1, %xmm1
-; SSE2-NEXT:    psrld $16, %xmm0
-; SSE2-NEXT:    cvtdq2pd %xmm0, %xmm0
-; SSE2-NEXT:    mulpd {{.*}}(%rip), %xmm0
-; SSE2-NEXT:    addpd %xmm1, %xmm0
+; SSE2-NEXT:    xorpd %xmm1, %xmm1
+; SSE2-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT:    movapd {{.*#+}} xmm1 = [4.503599627370496E+15,4.503599627370496E+15]
+; SSE2-NEXT:    orpd %xmm1, %xmm0
+; SSE2-NEXT:    subpd %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: uitofp_2i32_to_2f64:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pxor %xmm1, %xmm1
-; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4,5,6,7]
-; SSE41-NEXT:    cvtdq2pd %xmm1, %xmm1
-; SSE41-NEXT:    psrld $16, %xmm0
-; SSE41-NEXT:    cvtdq2pd %xmm0, %xmm0
-; SSE41-NEXT:    mulpd {{.*}}(%rip), %xmm0
-; SSE41-NEXT:    addpd %xmm1, %xmm0
+; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
+; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [4.503599627370496E+15,4.503599627370496E+15]
+; SSE41-NEXT:    por %xmm1, %xmm0
+; SSE41-NEXT:    subpd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; VEX-LABEL: uitofp_2i32_to_2f64:
 ; VEX:       # %bb.0:
-; VEX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; VEX-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4,5,6,7]
-; VEX-NEXT:    vcvtdq2pd %xmm1, %xmm1
-; VEX-NEXT:    vpsrld $16, %xmm0, %xmm0
-; VEX-NEXT:    vcvtdq2pd %xmm0, %xmm0
-; VEX-NEXT:    vmulpd {{.*}}(%rip), %xmm0, %xmm0
-; VEX-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
+; VEX-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
+; VEX-NEXT:    vmovdqa {{.*#+}} xmm1 = [4.503599627370496E+15,4.503599627370496E+15]
+; VEX-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; VEX-NEXT:    vsubpd %xmm1, %xmm0, %xmm0
 ; VEX-NEXT:    retq
 ;
 ; AVX512F-LABEL: uitofp_2i32_to_2f64:
@@ -643,36 +707,37 @@ define <2 x double> @uitofp_2i32_to_2f64(<4 x i32> %a) {
 define <2 x double> @uitofp_4i32_to_2f64(<4 x i32> %a) {
 ; SSE2-LABEL: uitofp_4i32_to_2f64:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [65535,0,65535,0,0,0,0,0]
-; SSE2-NEXT:    pand %xmm0, %xmm1
-; SSE2-NEXT:    cvtdq2pd %xmm1, %xmm1
-; SSE2-NEXT:    psrld $16, %xmm0
-; SSE2-NEXT:    cvtdq2pd %xmm0, %xmm0
-; SSE2-NEXT:    mulpd {{.*}}(%rip), %xmm0
-; SSE2-NEXT:    addpd %xmm1, %xmm0
+; SSE2-NEXT:    xorpd %xmm1, %xmm1
+; SSE2-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT:    movapd {{.*#+}} xmm1 = [4.503599627370496E+15,4.503599627370496E+15]
+; SSE2-NEXT:    orpd %xmm1, %xmm0
+; SSE2-NEXT:    subpd %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: uitofp_4i32_to_2f64:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pxor %xmm1, %xmm1
-; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4,5,6,7]
-; SSE41-NEXT:    cvtdq2pd %xmm1, %xmm1
-; SSE41-NEXT:    psrld $16, %xmm0
-; SSE41-NEXT:    cvtdq2pd %xmm0, %xmm0
-; SSE41-NEXT:    mulpd {{.*}}(%rip), %xmm0
-; SSE41-NEXT:    addpd %xmm1, %xmm0
+; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
+; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [4.503599627370496E+15,4.503599627370496E+15]
+; SSE41-NEXT:    por %xmm1, %xmm0
+; SSE41-NEXT:    subpd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
-; VEX-LABEL: uitofp_4i32_to_2f64:
-; VEX:       # %bb.0:
-; VEX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; VEX-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7]
-; VEX-NEXT:    vpsrld $16, %xmm0, %xmm0
-; VEX-NEXT:    vcvtdq2pd %xmm1, %xmm1
-; VEX-NEXT:    vcvtdq2pd %xmm0, %xmm0
-; VEX-NEXT:    vmulpd {{.*}}(%rip), %xmm0, %xmm0
-; VEX-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
-; VEX-NEXT:    retq
+; AVX1-LABEL: uitofp_4i32_to_2f64:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
+; AVX1-NEXT:    vmovddup {{.*#+}} xmm1 = [4.503599627370496E+15,4.503599627370496E+15]
+; AVX1-NEXT:    # xmm1 = mem[0,0]
+; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vsubpd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: uitofp_4i32_to_2f64:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} xmm1 = [4.503599627370496E+15,4.503599627370496E+15]
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vsubpd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: uitofp_4i32_to_2f64:
 ; AVX512F:       # %bb.0:
@@ -931,67 +996,47 @@ define <4 x double> @uitofp_4i64_to_4f64(<4 x i64> %a) {
 define <4 x double> @uitofp_4i32_to_4f64(<4 x i32> %a) {
 ; SSE2-LABEL: uitofp_4i32_to_4f64:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    psrld $16, %xmm1
-; SSE2-NEXT:    cvtdq2pd %xmm1, %xmm1
-; SSE2-NEXT:    movapd {{.*#+}} xmm2 = [6.5536E+4,6.5536E+4]
-; SSE2-NEXT:    mulpd %xmm2, %xmm1
-; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [65535,0,65535,0,0,0,0,0]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[2,3,0,1]
-; SSE2-NEXT:    pand %xmm3, %xmm0
-; SSE2-NEXT:    cvtdq2pd %xmm0, %xmm0
-; SSE2-NEXT:    addpd %xmm1, %xmm0
-; SSE2-NEXT:    movdqa %xmm4, %xmm1
-; SSE2-NEXT:    psrld $16, %xmm1
-; SSE2-NEXT:    cvtdq2pd %xmm1, %xmm5
-; SSE2-NEXT:    mulpd %xmm2, %xmm5
-; SSE2-NEXT:    pand %xmm3, %xmm4
-; SSE2-NEXT:    cvtdq2pd %xmm4, %xmm1
-; SSE2-NEXT:    addpd %xmm5, %xmm1
+; SSE2-NEXT:    movapd %xmm0, %xmm1
+; SSE2-NEXT:    xorpd %xmm2, %xmm2
+; SSE2-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; SSE2-NEXT:    movapd {{.*#+}} xmm3 = [4.503599627370496E+15,4.503599627370496E+15]
+; SSE2-NEXT:    orpd %xmm3, %xmm0
+; SSE2-NEXT:    subpd %xmm3, %xmm0
+; SSE2-NEXT:    unpckhps {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; SSE2-NEXT:    orpd %xmm3, %xmm1
+; SSE2-NEXT:    subpd %xmm3, %xmm1
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: uitofp_4i32_to_4f64:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    psrld $16, %xmm1
-; SSE41-NEXT:    cvtdq2pd %xmm1, %xmm1
-; SSE41-NEXT:    movapd {{.*#+}} xmm2 = [6.5536E+4,6.5536E+4]
-; SSE41-NEXT:    mulpd %xmm2, %xmm1
-; SSE41-NEXT:    pxor %xmm3, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[2,3,0,1]
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0],xmm3[1],xmm0[2],xmm3[3],xmm0[4,5,6,7]
-; SSE41-NEXT:    cvtdq2pd %xmm0, %xmm0
-; SSE41-NEXT:    addpd %xmm1, %xmm0
-; SSE41-NEXT:    movdqa %xmm4, %xmm1
-; SSE41-NEXT:    psrld $16, %xmm1
-; SSE41-NEXT:    cvtdq2pd %xmm1, %xmm5
-; SSE41-NEXT:    mulpd %xmm2, %xmm5
-; SSE41-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0],xmm3[1],xmm4[2],xmm3[3],xmm4[4,5,6,7]
-; SSE41-NEXT:    cvtdq2pd %xmm4, %xmm1
-; SSE41-NEXT:    addpd %xmm5, %xmm1
+; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero
+; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [4.503599627370496E+15,4.503599627370496E+15]
+; SSE41-NEXT:    por %xmm3, %xmm2
+; SSE41-NEXT:    subpd %xmm3, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero
+; SSE41-NEXT:    por %xmm3, %xmm1
+; SSE41-NEXT:    subpd %xmm3, %xmm1
+; SSE41-NEXT:    movapd %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: uitofp_4i32_to_4f64:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7]
-; AVX1-NEXT:    vcvtdq2pd %xmm1, %ymm1
-; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm0
-; AVX1-NEXT:    vcvtdq2pd %xmm0, %ymm0
-; AVX1-NEXT:    vmulpd {{.*}}(%rip), %ymm0, %ymm0
-; AVX1-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm1 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    vbroadcastsd {{.*#+}} ymm1 = [4.503599627370496E+15,4.503599627370496E+15,4.503599627370496E+15,4.503599627370496E+15]
+; AVX1-NEXT:    vorpd %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vsubpd %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: uitofp_4i32_to_4f64:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX2-NEXT:    vcvtdq2pd %xmm1, %ymm1
-; AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm2 = [6.5536E+4,6.5536E+4,6.5536E+4,6.5536E+4]
-; AVX2-NEXT:    vmulpd %ymm2, %ymm1, %ymm1
-; AVX2-NEXT:    vxorpd %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3],xmm0[4],xmm2[5],xmm0[6],xmm2[7]
-; AVX2-NEXT:    vcvtdq2pd %xmm0, %ymm0
-; AVX2-NEXT:    vaddpd %ymm0, %ymm1, %ymm0
+; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [4.503599627370496E+15,4.503599627370496E+15,4.503599627370496E+15,4.503599627370496E+15]
+; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vsubpd %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: uitofp_4i32_to_4f64:
@@ -1204,7 +1249,7 @@ define <4 x float> @sitofp_2i64_to_4f32(<2 x i64> %a) {
 ; AVX512VL-NEXT:    vcvtsi2ss %rax, %xmm1, %xmm1
 ; AVX512VL-NEXT:    vmovq %xmm0, %rax
 ; AVX512VL-NEXT:    vcvtsi2ss %rax, %xmm2, %xmm0
-; AVX512VL-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX512VL-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; AVX512VL-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; AVX512VL-NEXT:    retq
 ;
@@ -1235,7 +1280,7 @@ define <4 x float> @sitofp_2i64_to_4f32_zero(<2 x i64> %a) {
 ; SSE2-NEXT:    movq %xmm0, %rax
 ; SSE2-NEXT:    xorps %xmm0, %xmm0
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm0
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; SSE2-NEXT:    movq {{.*#+}} xmm0 = xmm0[0],zero
 ; SSE2-NEXT:    retq
 ;
@@ -1274,7 +1319,7 @@ define <4 x float> @sitofp_2i64_to_4f32_zero(<2 x i64> %a) {
 ; AVX512VL-NEXT:    vcvtsi2ss %rax, %xmm1, %xmm1
 ; AVX512VL-NEXT:    vpextrq $1, %xmm0, %rax
 ; AVX512VL-NEXT:    vcvtsi2ss %rax, %xmm2, %xmm0
-; AVX512VL-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; AVX512VL-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
 ; AVX512VL-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; AVX512VL-NEXT:    retq
 ;
@@ -1304,7 +1349,7 @@ define <4 x float> @sitofp_4i64_to_4f32_undef(<2 x i64> %a) {
 ; SSE2-NEXT:    movq %xmm0, %rax
 ; SSE2-NEXT:    xorps %xmm0, %xmm0
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm0
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE2-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
 ; SSE2-NEXT:    movq {{.*#+}} xmm0 = xmm1[0],zero
 ; SSE2-NEXT:    retq
 ;
@@ -1342,7 +1387,7 @@ define <4 x float> @sitofp_4i64_to_4f32_undef(<2 x i64> %a) {
 ; AVX512VL-NEXT:    vcvtsi2ss %rax, %xmm1, %xmm1
 ; AVX512VL-NEXT:    vmovq %xmm0, %rax
 ; AVX512VL-NEXT:    vcvtsi2ss %rax, %xmm2, %xmm0
-; AVX512VL-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX512VL-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; AVX512VL-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; AVX512VL-NEXT:    retq
 ;
@@ -1708,14 +1753,13 @@ define <8 x float> @sitofp_8i16_to_8f32(<8 x i16> %a) {
 define <8 x float> @sitofp_8i8_to_8f32(<16 x i8> %a) {
 ; SSE2-LABEL: sitofp_8i8_to_8f32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; SSE2-NEXT:    psrad $24, %xmm1
-; SSE2-NEXT:    cvtdq2ps %xmm1, %xmm2
-; SSE2-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
 ; SSE2-NEXT:    psrad $24, %xmm0
-; SSE2-NEXT:    cvtdq2ps %xmm0, %xmm1
-; SSE2-NEXT:    movaps %xmm2, %xmm0
+; SSE2-NEXT:    cvtdq2ps %xmm0, %xmm0
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    psrad $24, %xmm1
+; SSE2-NEXT:    cvtdq2ps %xmm1, %xmm1
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: sitofp_8i8_to_8f32:
@@ -1756,14 +1800,13 @@ define <8 x float> @sitofp_8i8_to_8f32(<16 x i8> %a) {
 define <8 x float> @sitofp_16i8_to_8f32(<16 x i8> %a) {
 ; SSE2-LABEL: sitofp_16i8_to_8f32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; SSE2-NEXT:    psrad $24, %xmm1
-; SSE2-NEXT:    cvtdq2ps %xmm1, %xmm2
-; SSE2-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
 ; SSE2-NEXT:    psrad $24, %xmm0
-; SSE2-NEXT:    cvtdq2ps %xmm0, %xmm1
-; SSE2-NEXT:    movaps %xmm2, %xmm0
+; SSE2-NEXT:    cvtdq2ps %xmm0, %xmm0
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    psrad $24, %xmm1
+; SSE2-NEXT:    cvtdq2ps %xmm1, %xmm1
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: sitofp_16i8_to_8f32:
@@ -1812,12 +1855,12 @@ define <4 x float> @uitofp_2i64_to_4f32(<2 x i64> %a) {
 ; SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; SSE2-NEXT:    movq %xmm0, %rax
 ; SSE2-NEXT:    testq %rax, %rax
-; SSE2-NEXT:    js .LBB39_1
+; SSE2-NEXT:    js .LBB41_1
 ; SSE2-NEXT:  # %bb.2:
 ; SSE2-NEXT:    xorps %xmm0, %xmm0
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm0
-; SSE2-NEXT:    jmp .LBB39_3
-; SSE2-NEXT:  .LBB39_1:
+; SSE2-NEXT:    jmp .LBB41_3
+; SSE2-NEXT:  .LBB41_1:
 ; SSE2-NEXT:    movq %rax, %rcx
 ; SSE2-NEXT:    shrq %rcx
 ; SSE2-NEXT:    andl $1, %eax
@@ -1825,17 +1868,17 @@ define <4 x float> @uitofp_2i64_to_4f32(<2 x i64> %a) {
 ; SSE2-NEXT:    xorps %xmm0, %xmm0
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm0
 ; SSE2-NEXT:    addss %xmm0, %xmm0
-; SSE2-NEXT:  .LBB39_3:
+; SSE2-NEXT:  .LBB41_3:
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
 ; SSE2-NEXT:    movq %xmm1, %rax
 ; SSE2-NEXT:    testq %rax, %rax
-; SSE2-NEXT:    js .LBB39_4
+; SSE2-NEXT:    js .LBB41_4
 ; SSE2-NEXT:  # %bb.5:
 ; SSE2-NEXT:    xorps %xmm1, %xmm1
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm1
 ; SSE2-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; SSE2-NEXT:    retq
-; SSE2-NEXT:  .LBB39_4:
+; SSE2-NEXT:  .LBB41_4:
 ; SSE2-NEXT:    movq %rax, %rcx
 ; SSE2-NEXT:    shrq %rcx
 ; SSE2-NEXT:    andl $1, %eax
@@ -1848,70 +1891,45 @@ define <4 x float> @uitofp_2i64_to_4f32(<2 x i64> %a) {
 ;
 ; SSE41-LABEL: uitofp_2i64_to_4f32:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pextrq $1, %xmm0, %rax
-; SSE41-NEXT:    testq %rax, %rax
-; SSE41-NEXT:    js .LBB39_1
-; SSE41-NEXT:  # %bb.2:
-; SSE41-NEXT:    cvtsi2ss %rax, %xmm1
-; SSE41-NEXT:    jmp .LBB39_3
-; SSE41-NEXT:  .LBB39_1:
-; SSE41-NEXT:    movq %rax, %rcx
-; SSE41-NEXT:    shrq %rcx
-; SSE41-NEXT:    andl $1, %eax
-; SSE41-NEXT:    orq %rcx, %rax
-; SSE41-NEXT:    cvtsi2ss %rax, %xmm1
-; SSE41-NEXT:    addss %xmm1, %xmm1
-; SSE41-NEXT:  .LBB39_3:
-; SSE41-NEXT:    movq %xmm0, %rax
-; SSE41-NEXT:    testq %rax, %rax
-; SSE41-NEXT:    js .LBB39_4
-; SSE41-NEXT:  # %bb.5:
-; SSE41-NEXT:    xorps %xmm0, %xmm0
-; SSE41-NEXT:    cvtsi2ss %rax, %xmm0
-; SSE41-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[2,3]
-; SSE41-NEXT:    retq
-; SSE41-NEXT:  .LBB39_4:
-; SSE41-NEXT:    movq %rax, %rcx
-; SSE41-NEXT:    shrq %rcx
-; SSE41-NEXT:    andl $1, %eax
-; SSE41-NEXT:    orq %rcx, %rax
+; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [1,1]
+; SSE41-NEXT:    pand %xmm0, %xmm1
+; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    pxor %xmm3, %xmm3
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSE41-NEXT:    movdqa %xmm0, %xmm4
+; SSE41-NEXT:    psrlq $1, %xmm4
+; SSE41-NEXT:    por %xmm1, %xmm4
+; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm2
+; SSE41-NEXT:    pextrq $1, %xmm2, %rax
 ; SSE41-NEXT:    xorps %xmm0, %xmm0
 ; SSE41-NEXT:    cvtsi2ss %rax, %xmm0
-; SSE41-NEXT:    addss %xmm0, %xmm0
-; SSE41-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[2,3]
+; SSE41-NEXT:    movq %xmm2, %rax
+; SSE41-NEXT:    xorps %xmm1, %xmm1
+; SSE41-NEXT:    cvtsi2ss %rax, %xmm1
+; SSE41-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0],xmm0[0],zero,zero
+; SSE41-NEXT:    movaps %xmm1, %xmm2
+; SSE41-NEXT:    addps %xmm1, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,3,2,3]
+; SSE41-NEXT:    blendvps %xmm0, %xmm2, %xmm1
+; SSE41-NEXT:    movaps %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; VEX-LABEL: uitofp_2i64_to_4f32:
 ; VEX:       # %bb.0:
-; VEX-NEXT:    vpextrq $1, %xmm0, %rax
-; VEX-NEXT:    testq %rax, %rax
-; VEX-NEXT:    js .LBB39_1
-; VEX-NEXT:  # %bb.2:
-; VEX-NEXT:    vcvtsi2ss %rax, %xmm1, %xmm1
-; VEX-NEXT:    jmp .LBB39_3
-; VEX-NEXT:  .LBB39_1:
-; VEX-NEXT:    movq %rax, %rcx
-; VEX-NEXT:    shrq %rcx
-; VEX-NEXT:    andl $1, %eax
-; VEX-NEXT:    orq %rcx, %rax
-; VEX-NEXT:    vcvtsi2ss %rax, %xmm1, %xmm1
-; VEX-NEXT:    vaddss %xmm1, %xmm1, %xmm1
-; VEX-NEXT:  .LBB39_3:
-; VEX-NEXT:    vmovq %xmm0, %rax
-; VEX-NEXT:    testq %rax, %rax
-; VEX-NEXT:    js .LBB39_4
-; VEX-NEXT:  # %bb.5:
-; VEX-NEXT:    vcvtsi2ss %rax, %xmm2, %xmm0
-; VEX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],zero,zero
-; VEX-NEXT:    retq
-; VEX-NEXT:  .LBB39_4:
-; VEX-NEXT:    movq %rax, %rcx
-; VEX-NEXT:    shrq %rcx
-; VEX-NEXT:    andl $1, %eax
-; VEX-NEXT:    orq %rcx, %rax
-; VEX-NEXT:    vcvtsi2ss %rax, %xmm2, %xmm0
-; VEX-NEXT:    vaddss %xmm0, %xmm0, %xmm0
-; VEX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],zero,zero
+; VEX-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm1
+; VEX-NEXT:    vpsrlq $1, %xmm0, %xmm2
+; VEX-NEXT:    vpor %xmm1, %xmm2, %xmm1
+; VEX-NEXT:    vblendvpd %xmm0, %xmm1, %xmm0, %xmm1
+; VEX-NEXT:    vpextrq $1, %xmm1, %rax
+; VEX-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm2
+; VEX-NEXT:    vmovq %xmm1, %rax
+; VEX-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm1
+; VEX-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0],xmm2[0],zero,zero
+; VEX-NEXT:    vaddps %xmm1, %xmm1, %xmm2
+; VEX-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; VEX-NEXT:    vpcmpgtq %xmm0, %xmm3, %xmm0
+; VEX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
+; VEX-NEXT:    vblendvps %xmm0, %xmm2, %xmm1, %xmm0
 ; VEX-NEXT:    retq
 ;
 ; AVX512F-LABEL: uitofp_2i64_to_4f32:
@@ -1929,7 +1947,7 @@ define <4 x float> @uitofp_2i64_to_4f32(<2 x i64> %a) {
 ; AVX512VL-NEXT:    vcvtusi2ss %rax, %xmm1, %xmm1
 ; AVX512VL-NEXT:    vmovq %xmm0, %rax
 ; AVX512VL-NEXT:    vcvtusi2ss %rax, %xmm2, %xmm0
-; AVX512VL-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX512VL-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; AVX512VL-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; AVX512VL-NEXT:    retq
 ;
@@ -1956,12 +1974,12 @@ define <4 x float> @uitofp_2i64_to_2f32(<2 x i64> %a) {
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; SSE2-NEXT:    movq %xmm1, %rax
 ; SSE2-NEXT:    testq %rax, %rax
-; SSE2-NEXT:    js .LBB40_1
+; SSE2-NEXT:    js .LBB42_1
 ; SSE2-NEXT:  # %bb.2:
 ; SSE2-NEXT:    xorps %xmm1, %xmm1
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm1
-; SSE2-NEXT:    jmp .LBB40_3
-; SSE2-NEXT:  .LBB40_1:
+; SSE2-NEXT:    jmp .LBB42_3
+; SSE2-NEXT:  .LBB42_1:
 ; SSE2-NEXT:    movq %rax, %rcx
 ; SSE2-NEXT:    shrq %rcx
 ; SSE2-NEXT:    andl $1, %eax
@@ -1969,15 +1987,15 @@ define <4 x float> @uitofp_2i64_to_2f32(<2 x i64> %a) {
 ; SSE2-NEXT:    xorps %xmm1, %xmm1
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm1
 ; SSE2-NEXT:    addss %xmm1, %xmm1
-; SSE2-NEXT:  .LBB40_3:
+; SSE2-NEXT:  .LBB42_3:
 ; SSE2-NEXT:    movq %xmm0, %rax
 ; SSE2-NEXT:    testq %rax, %rax
-; SSE2-NEXT:    js .LBB40_4
+; SSE2-NEXT:    js .LBB42_4
 ; SSE2-NEXT:  # %bb.5:
 ; SSE2-NEXT:    xorps %xmm0, %xmm0
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm0
-; SSE2-NEXT:    jmp .LBB40_6
-; SSE2-NEXT:  .LBB40_4:
+; SSE2-NEXT:    jmp .LBB42_6
+; SSE2-NEXT:  .LBB42_4:
 ; SSE2-NEXT:    movq %rax, %rcx
 ; SSE2-NEXT:    shrq %rcx
 ; SSE2-NEXT:    andl $1, %eax
@@ -1985,80 +2003,53 @@ define <4 x float> @uitofp_2i64_to_2f32(<2 x i64> %a) {
 ; SSE2-NEXT:    xorps %xmm0, %xmm0
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm0
 ; SSE2-NEXT:    addss %xmm0, %xmm0
-; SSE2-NEXT:  .LBB40_6:
+; SSE2-NEXT:  .LBB42_6:
 ; SSE2-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; SSE2-NEXT:    movq {{.*#+}} xmm0 = xmm0[0],zero
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: uitofp_2i64_to_2f32:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    movq %xmm0, %rax
-; SSE41-NEXT:    testq %rax, %rax
-; SSE41-NEXT:    js .LBB40_1
-; SSE41-NEXT:  # %bb.2:
-; SSE41-NEXT:    xorps %xmm0, %xmm0
-; SSE41-NEXT:    cvtsi2ss %rax, %xmm0
-; SSE41-NEXT:    jmp .LBB40_3
-; SSE41-NEXT:  .LBB40_1:
-; SSE41-NEXT:    movq %rax, %rcx
-; SSE41-NEXT:    shrq %rcx
-; SSE41-NEXT:    andl $1, %eax
-; SSE41-NEXT:    orq %rcx, %rax
+; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [1,1]
+; SSE41-NEXT:    pand %xmm0, %xmm1
+; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    pxor %xmm3, %xmm3
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSE41-NEXT:    movdqa %xmm0, %xmm4
+; SSE41-NEXT:    psrlq $1, %xmm4
+; SSE41-NEXT:    por %xmm1, %xmm4
+; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm2
+; SSE41-NEXT:    pextrq $1, %xmm2, %rax
 ; SSE41-NEXT:    xorps %xmm0, %xmm0
 ; SSE41-NEXT:    cvtsi2ss %rax, %xmm0
-; SSE41-NEXT:    addss %xmm0, %xmm0
-; SSE41-NEXT:  .LBB40_3:
-; SSE41-NEXT:    pextrq $1, %xmm1, %rax
-; SSE41-NEXT:    testq %rax, %rax
-; SSE41-NEXT:    js .LBB40_4
-; SSE41-NEXT:  # %bb.5:
-; SSE41-NEXT:    xorps %xmm1, %xmm1
-; SSE41-NEXT:    cvtsi2ss %rax, %xmm1
-; SSE41-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],zero,zero
-; SSE41-NEXT:    retq
-; SSE41-NEXT:  .LBB40_4:
-; SSE41-NEXT:    movq %rax, %rcx
-; SSE41-NEXT:    shrq %rcx
-; SSE41-NEXT:    andl $1, %eax
-; SSE41-NEXT:    orq %rcx, %rax
+; SSE41-NEXT:    movq %xmm2, %rax
 ; SSE41-NEXT:    xorps %xmm1, %xmm1
 ; SSE41-NEXT:    cvtsi2ss %rax, %xmm1
-; SSE41-NEXT:    addss %xmm1, %xmm1
-; SSE41-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],zero,zero
+; SSE41-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0],xmm0[0],zero,zero
+; SSE41-NEXT:    movaps %xmm1, %xmm2
+; SSE41-NEXT:    addps %xmm1, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,3,2,3]
+; SSE41-NEXT:    blendvps %xmm0, %xmm2, %xmm1
+; SSE41-NEXT:    movq {{.*#+}} xmm0 = xmm1[0],zero
 ; SSE41-NEXT:    retq
 ;
 ; VEX-LABEL: uitofp_2i64_to_2f32:
 ; VEX:       # %bb.0:
-; VEX-NEXT:    vmovq %xmm0, %rax
-; VEX-NEXT:    testq %rax, %rax
-; VEX-NEXT:    js .LBB40_1
-; VEX-NEXT:  # %bb.2:
-; VEX-NEXT:    vcvtsi2ss %rax, %xmm1, %xmm1
-; VEX-NEXT:    jmp .LBB40_3
-; VEX-NEXT:  .LBB40_1:
-; VEX-NEXT:    movq %rax, %rcx
-; VEX-NEXT:    shrq %rcx
-; VEX-NEXT:    andl $1, %eax
-; VEX-NEXT:    orq %rcx, %rax
-; VEX-NEXT:    vcvtsi2ss %rax, %xmm1, %xmm1
-; VEX-NEXT:    vaddss %xmm1, %xmm1, %xmm1
-; VEX-NEXT:  .LBB40_3:
-; VEX-NEXT:    vpextrq $1, %xmm0, %rax
-; VEX-NEXT:    testq %rax, %rax
-; VEX-NEXT:    js .LBB40_4
-; VEX-NEXT:  # %bb.5:
-; VEX-NEXT:    vcvtsi2ss %rax, %xmm2, %xmm0
-; VEX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],zero,zero
-; VEX-NEXT:    retq
-; VEX-NEXT:  .LBB40_4:
-; VEX-NEXT:    movq %rax, %rcx
-; VEX-NEXT:    shrq %rcx
-; VEX-NEXT:    andl $1, %eax
-; VEX-NEXT:    orq %rcx, %rax
-; VEX-NEXT:    vcvtsi2ss %rax, %xmm2, %xmm0
-; VEX-NEXT:    vaddss %xmm0, %xmm0, %xmm0
-; VEX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],zero,zero
+; VEX-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm1
+; VEX-NEXT:    vpsrlq $1, %xmm0, %xmm2
+; VEX-NEXT:    vpor %xmm1, %xmm2, %xmm1
+; VEX-NEXT:    vblendvpd %xmm0, %xmm1, %xmm0, %xmm1
+; VEX-NEXT:    vpextrq $1, %xmm1, %rax
+; VEX-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm2
+; VEX-NEXT:    vmovq %xmm1, %rax
+; VEX-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm1
+; VEX-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0],xmm2[0],zero,zero
+; VEX-NEXT:    vaddps %xmm1, %xmm1, %xmm2
+; VEX-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; VEX-NEXT:    vpcmpgtq %xmm0, %xmm3, %xmm0
+; VEX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
+; VEX-NEXT:    vblendvps %xmm0, %xmm2, %xmm1, %xmm0
+; VEX-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; VEX-NEXT:    retq
 ;
 ; AVX512F-LABEL: uitofp_2i64_to_2f32:
@@ -2076,7 +2067,7 @@ define <4 x float> @uitofp_2i64_to_2f32(<2 x i64> %a) {
 ; AVX512VL-NEXT:    vcvtusi2ss %rax, %xmm1, %xmm1
 ; AVX512VL-NEXT:    vpextrq $1, %xmm0, %rax
 ; AVX512VL-NEXT:    vcvtusi2ss %rax, %xmm2, %xmm0
-; AVX512VL-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; AVX512VL-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
 ; AVX512VL-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; AVX512VL-NEXT:    retq
 ;
@@ -2102,27 +2093,27 @@ define <4 x float> @uitofp_4i64_to_4f32_undef(<2 x i64> %a) {
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movq %xmm0, %rax
 ; SSE2-NEXT:    testq %rax, %rax
-; SSE2-NEXT:    js .LBB41_1
+; SSE2-NEXT:    js .LBB43_1
 ; SSE2-NEXT:  # %bb.2:
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm1
-; SSE2-NEXT:    jmp .LBB41_3
-; SSE2-NEXT:  .LBB41_1:
+; SSE2-NEXT:    jmp .LBB43_3
+; SSE2-NEXT:  .LBB43_1:
 ; SSE2-NEXT:    movq %rax, %rcx
 ; SSE2-NEXT:    shrq %rcx
 ; SSE2-NEXT:    andl $1, %eax
 ; SSE2-NEXT:    orq %rcx, %rax
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm1
 ; SSE2-NEXT:    addss %xmm1, %xmm1
-; SSE2-NEXT:  .LBB41_3:
+; SSE2-NEXT:  .LBB43_3:
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
 ; SSE2-NEXT:    movq %xmm0, %rax
 ; SSE2-NEXT:    testq %rax, %rax
-; SSE2-NEXT:    js .LBB41_4
+; SSE2-NEXT:    js .LBB43_4
 ; SSE2-NEXT:  # %bb.5:
 ; SSE2-NEXT:    xorps %xmm0, %xmm0
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm0
-; SSE2-NEXT:    jmp .LBB41_6
-; SSE2-NEXT:  .LBB41_4:
+; SSE2-NEXT:    jmp .LBB43_6
+; SSE2-NEXT:  .LBB43_4:
 ; SSE2-NEXT:    movq %rax, %rcx
 ; SSE2-NEXT:    shrq %rcx
 ; SSE2-NEXT:    andl $1, %eax
@@ -2130,78 +2121,87 @@ define <4 x float> @uitofp_4i64_to_4f32_undef(<2 x i64> %a) {
 ; SSE2-NEXT:    xorps %xmm0, %xmm0
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm0
 ; SSE2-NEXT:    addss %xmm0, %xmm0
-; SSE2-NEXT:  .LBB41_6:
+; SSE2-NEXT:  .LBB43_6:
 ; SSE2-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
 ; SSE2-NEXT:    movq {{.*#+}} xmm0 = xmm1[0],zero
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: uitofp_4i64_to_4f32_undef:
 ; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [1,1]
+; SSE41-NEXT:    pand %xmm0, %xmm1
+; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    psrlq $1, %xmm2
+; SSE41-NEXT:    por %xmm1, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,3,2,3]
+; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm0
 ; SSE41-NEXT:    pextrq $1, %xmm0, %rax
-; SSE41-NEXT:    testq %rax, %rax
-; SSE41-NEXT:    js .LBB41_1
-; SSE41-NEXT:  # %bb.2:
-; SSE41-NEXT:    cvtsi2ss %rax, %xmm1
-; SSE41-NEXT:    jmp .LBB41_3
-; SSE41-NEXT:  .LBB41_1:
-; SSE41-NEXT:    movq %rax, %rcx
-; SSE41-NEXT:    shrq %rcx
-; SSE41-NEXT:    andl $1, %eax
-; SSE41-NEXT:    orq %rcx, %rax
-; SSE41-NEXT:    cvtsi2ss %rax, %xmm1
-; SSE41-NEXT:    addss %xmm1, %xmm1
-; SSE41-NEXT:  .LBB41_3:
+; SSE41-NEXT:    cvtsi2ss %rax, %xmm3
 ; SSE41-NEXT:    movq %xmm0, %rax
-; SSE41-NEXT:    testq %rax, %rax
-; SSE41-NEXT:    js .LBB41_4
-; SSE41-NEXT:  # %bb.5:
-; SSE41-NEXT:    xorps %xmm0, %xmm0
-; SSE41-NEXT:    cvtsi2ss %rax, %xmm0
-; SSE41-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],zero,zero
-; SSE41-NEXT:    retq
-; SSE41-NEXT:  .LBB41_4:
-; SSE41-NEXT:    movq %rax, %rcx
-; SSE41-NEXT:    shrq %rcx
-; SSE41-NEXT:    andl $1, %eax
-; SSE41-NEXT:    orq %rcx, %rax
-; SSE41-NEXT:    xorps %xmm0, %xmm0
-; SSE41-NEXT:    cvtsi2ss %rax, %xmm0
-; SSE41-NEXT:    addss %xmm0, %xmm0
-; SSE41-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],zero,zero
+; SSE41-NEXT:    xorps %xmm2, %xmm2
+; SSE41-NEXT:    cvtsi2ss %rax, %xmm2
+; SSE41-NEXT:    insertps {{.*#+}} xmm2 = xmm2[0],xmm3[0],zero,zero
+; SSE41-NEXT:    movaps %xmm2, %xmm3
+; SSE41-NEXT:    addps %xmm2, %xmm3
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm3, %xmm2
+; SSE41-NEXT:    movaps %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
-; VEX-LABEL: uitofp_4i64_to_4f32_undef:
-; VEX:       # %bb.0:
-; VEX-NEXT:    vpextrq $1, %xmm0, %rax
-; VEX-NEXT:    testq %rax, %rax
-; VEX-NEXT:    js .LBB41_1
-; VEX-NEXT:  # %bb.2:
-; VEX-NEXT:    vcvtsi2ss %rax, %xmm1, %xmm1
-; VEX-NEXT:    jmp .LBB41_3
-; VEX-NEXT:  .LBB41_1:
-; VEX-NEXT:    movq %rax, %rcx
-; VEX-NEXT:    shrq %rcx
-; VEX-NEXT:    andl $1, %eax
-; VEX-NEXT:    orq %rcx, %rax
-; VEX-NEXT:    vcvtsi2ss %rax, %xmm1, %xmm1
-; VEX-NEXT:    vaddss %xmm1, %xmm1, %xmm1
-; VEX-NEXT:  .LBB41_3:
-; VEX-NEXT:    vmovq %xmm0, %rax
-; VEX-NEXT:    testq %rax, %rax
-; VEX-NEXT:    js .LBB41_4
-; VEX-NEXT:  # %bb.5:
-; VEX-NEXT:    vcvtsi2ss %rax, %xmm2, %xmm0
-; VEX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],zero,zero
-; VEX-NEXT:    retq
-; VEX-NEXT:  .LBB41_4:
-; VEX-NEXT:    movq %rax, %rcx
-; VEX-NEXT:    shrq %rcx
-; VEX-NEXT:    andl $1, %eax
-; VEX-NEXT:    orq %rcx, %rax
-; VEX-NEXT:    vcvtsi2ss %rax, %xmm2, %xmm0
-; VEX-NEXT:    vaddss %xmm0, %xmm0, %xmm0
-; VEX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],zero,zero
-; VEX-NEXT:    retq
+; AVX1-LABEL: uitofp_4i64_to_4f32_undef:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; AVX1-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm1
+; AVX1-NEXT:    vpsrlq $1, %xmm0, %xmm2
+; AVX1-NEXT:    vorps %ymm1, %ymm2, %ymm1
+; AVX1-NEXT:    vblendvpd %xmm0, %xmm1, %xmm0, %xmm2
+; AVX1-NEXT:    vpextrq $1, %xmm2, %rax
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm3
+; AVX1-NEXT:    vmovq %xmm2, %rax
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm4, %xmm2
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[2,3]
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm1
+; AVX1-NEXT:    vcvtdq2ps %xmm1, %xmm3
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1],xmm3[0],xmm2[3]
+; AVX1-NEXT:    vpextrq $1, %xmm1, %rax
+; AVX1-NEXT:    vcvtsi2ss %eax, %xmm4, %xmm1
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[0]
+; AVX1-NEXT:    vaddps %xmm1, %xmm1, %xmm2
+; AVX1-NEXT:    vxorps %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm3, %xmm3
+; AVX1-NEXT:    vpackssdw %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vblendvps %xmm0, %xmm2, %xmm1, %xmm0
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: uitofp_4i64_to_4f32_undef:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [1,1,1,1]
+; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm1
+; AVX2-NEXT:    vpsrlq $1, %ymm0, %ymm2
+; AVX2-NEXT:    vpor %ymm1, %ymm2, %ymm1
+; AVX2-NEXT:    vblendvpd %ymm0, %ymm1, %ymm0, %ymm1
+; AVX2-NEXT:    vpextrq $1, %xmm1, %rax
+; AVX2-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm2
+; AVX2-NEXT:    vmovq %xmm1, %rax
+; AVX2-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm3
+; AVX2-NEXT:    vinsertps {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[2,3]
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm1
+; AVX2-NEXT:    vmovq %xmm1, %rax
+; AVX2-NEXT:    vcvtsi2ss %rax, %xmm4, %xmm3
+; AVX2-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1],xmm3[0],xmm2[3]
+; AVX2-NEXT:    vpextrq $1, %xmm1, %rax
+; AVX2-NEXT:    vcvtsi2ss %rax, %xmm4, %xmm1
+; AVX2-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[0]
+; AVX2-NEXT:    vaddps %xmm1, %xmm1, %xmm2
+; AVX2-NEXT:    vxorps %xmm3, %xmm3, %xmm3
+; AVX2-NEXT:    vpcmpgtq %ymm0, %ymm3, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; AVX2-NEXT:    vpackssdw %xmm3, %xmm0, %xmm0
+; AVX2-NEXT:    vblendvps %xmm0, %xmm2, %xmm1, %xmm0
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: uitofp_4i64_to_4f32_undef:
 ; AVX512F:       # %bb.0:
@@ -2218,7 +2218,7 @@ define <4 x float> @uitofp_4i64_to_4f32_undef(<2 x i64> %a) {
 ; AVX512VL-NEXT:    vcvtusi2ss %rax, %xmm1, %xmm1
 ; AVX512VL-NEXT:    vmovq %xmm0, %rax
 ; AVX512VL-NEXT:    vcvtusi2ss %rax, %xmm2, %xmm0
-; AVX512VL-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX512VL-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; AVX512VL-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; AVX512VL-NEXT:    retq
 ;
@@ -2249,7 +2249,7 @@ define <4 x float> @uitofp_4i32_to_4f32(<4 x i32> %a) {
 ; SSE2-NEXT:    por {{.*}}(%rip), %xmm1
 ; SSE2-NEXT:    psrld $16, %xmm0
 ; SSE2-NEXT:    por {{.*}}(%rip), %xmm0
-; SSE2-NEXT:    addps {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    subps {{.*}}(%rip), %xmm0
 ; SSE2-NEXT:    addps %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
@@ -2259,7 +2259,7 @@ define <4 x float> @uitofp_4i32_to_4f32(<4 x i32> %a) {
 ; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7]
 ; SSE41-NEXT:    psrld $16, %xmm0
 ; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0],mem[1],xmm0[2],mem[3],xmm0[4],mem[5],xmm0[6],mem[7]
-; SSE41-NEXT:    addps {{.*}}(%rip), %xmm0
+; SSE41-NEXT:    subps {{.*}}(%rip), %xmm0
 ; SSE41-NEXT:    addps %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
@@ -2268,7 +2268,7 @@ define <4 x float> @uitofp_4i32_to_4f32(<4 x i32> %a) {
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],mem[1],xmm0[2],mem[3],xmm0[4],mem[5],xmm0[6],mem[7]
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm0
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],mem[1],xmm0[2],mem[3],xmm0[4],mem[5],xmm0[6],mem[7]
-; AVX1-NEXT:    vaddps {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vsubps {{.*}}(%rip), %xmm0, %xmm0
 ; AVX1-NEXT:    vaddps %xmm0, %xmm1, %xmm0
 ; AVX1-NEXT:    retq
 ;
@@ -2279,8 +2279,8 @@ define <4 x float> @uitofp_4i32_to_4f32(<4 x i32> %a) {
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm0
 ; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [1392508928,1392508928,1392508928,1392508928]
 ; AVX2-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3],xmm0[4],xmm2[5],xmm0[6],xmm2[7]
-; AVX2-NEXT:    vbroadcastss {{.*#+}} xmm2 = [-5.49764202E+11,-5.49764202E+11,-5.49764202E+11,-5.49764202E+11]
-; AVX2-NEXT:    vaddps %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vbroadcastss {{.*#+}} xmm2 = [5.49764202E+11,5.49764202E+11,5.49764202E+11,5.49764202E+11]
+; AVX2-NEXT:    vsubps %xmm2, %xmm0, %xmm0
 ; AVX2-NEXT:    vaddps %xmm0, %xmm1, %xmm0
 ; AVX2-NEXT:    retq
 ;
@@ -2458,41 +2458,41 @@ define <4 x float> @uitofp_4i64_to_4f32(<4 x i64> %a) {
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movq %xmm1, %rax
 ; SSE2-NEXT:    testq %rax, %rax
-; SSE2-NEXT:    js .LBB47_1
+; SSE2-NEXT:    js .LBB49_1
 ; SSE2-NEXT:  # %bb.2:
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm2
-; SSE2-NEXT:    jmp .LBB47_3
-; SSE2-NEXT:  .LBB47_1:
+; SSE2-NEXT:    jmp .LBB49_3
+; SSE2-NEXT:  .LBB49_1:
 ; SSE2-NEXT:    movq %rax, %rcx
 ; SSE2-NEXT:    shrq %rcx
 ; SSE2-NEXT:    andl $1, %eax
 ; SSE2-NEXT:    orq %rcx, %rax
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm2
 ; SSE2-NEXT:    addss %xmm2, %xmm2
-; SSE2-NEXT:  .LBB47_3:
+; SSE2-NEXT:  .LBB49_3:
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
 ; SSE2-NEXT:    movq %xmm1, %rax
 ; SSE2-NEXT:    testq %rax, %rax
-; SSE2-NEXT:    js .LBB47_4
+; SSE2-NEXT:    js .LBB49_4
 ; SSE2-NEXT:  # %bb.5:
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm3
-; SSE2-NEXT:    jmp .LBB47_6
-; SSE2-NEXT:  .LBB47_4:
+; SSE2-NEXT:    jmp .LBB49_6
+; SSE2-NEXT:  .LBB49_4:
 ; SSE2-NEXT:    movq %rax, %rcx
 ; SSE2-NEXT:    shrq %rcx
 ; SSE2-NEXT:    andl $1, %eax
 ; SSE2-NEXT:    orq %rcx, %rax
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm3
 ; SSE2-NEXT:    addss %xmm3, %xmm3
-; SSE2-NEXT:  .LBB47_6:
+; SSE2-NEXT:  .LBB49_6:
 ; SSE2-NEXT:    movq %xmm0, %rax
 ; SSE2-NEXT:    testq %rax, %rax
-; SSE2-NEXT:    js .LBB47_7
+; SSE2-NEXT:    js .LBB49_7
 ; SSE2-NEXT:  # %bb.8:
 ; SSE2-NEXT:    xorps %xmm1, %xmm1
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm1
-; SSE2-NEXT:    jmp .LBB47_9
-; SSE2-NEXT:  .LBB47_7:
+; SSE2-NEXT:    jmp .LBB49_9
+; SSE2-NEXT:  .LBB49_7:
 ; SSE2-NEXT:    movq %rax, %rcx
 ; SSE2-NEXT:    shrq %rcx
 ; SSE2-NEXT:    andl $1, %eax
@@ -2500,17 +2500,17 @@ define <4 x float> @uitofp_4i64_to_4f32(<4 x i64> %a) {
 ; SSE2-NEXT:    xorps %xmm1, %xmm1
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm1
 ; SSE2-NEXT:    addss %xmm1, %xmm1
-; SSE2-NEXT:  .LBB47_9:
+; SSE2-NEXT:  .LBB49_9:
 ; SSE2-NEXT:    unpcklps {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
 ; SSE2-NEXT:    movq %xmm0, %rax
 ; SSE2-NEXT:    testq %rax, %rax
-; SSE2-NEXT:    js .LBB47_10
+; SSE2-NEXT:    js .LBB49_10
 ; SSE2-NEXT:  # %bb.11:
 ; SSE2-NEXT:    xorps %xmm0, %xmm0
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm0
-; SSE2-NEXT:    jmp .LBB47_12
-; SSE2-NEXT:  .LBB47_10:
+; SSE2-NEXT:    jmp .LBB49_12
+; SSE2-NEXT:  .LBB49_10:
 ; SSE2-NEXT:    movq %rax, %rcx
 ; SSE2-NEXT:    shrq %rcx
 ; SSE2-NEXT:    andl $1, %eax
@@ -2518,7 +2518,7 @@ define <4 x float> @uitofp_4i64_to_4f32(<4 x i64> %a) {
 ; SSE2-NEXT:    xorps %xmm0, %xmm0
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm0
 ; SSE2-NEXT:    addss %xmm0, %xmm0
-; SSE2-NEXT:  .LBB47_12:
+; SSE2-NEXT:  .LBB49_12:
 ; SSE2-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
 ; SSE2-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm2[0]
 ; SSE2-NEXT:    movaps %xmm1, %xmm0
@@ -2526,202 +2526,100 @@ define <4 x float> @uitofp_4i64_to_4f32(<4 x i64> %a) {
 ;
 ; SSE41-LABEL: uitofp_4i64_to_4f32:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pextrq $1, %xmm0, %rax
-; SSE41-NEXT:    testq %rax, %rax
-; SSE41-NEXT:    js .LBB47_1
-; SSE41-NEXT:  # %bb.2:
-; SSE41-NEXT:    cvtsi2ss %rax, %xmm2
-; SSE41-NEXT:    jmp .LBB47_3
-; SSE41-NEXT:  .LBB47_1:
-; SSE41-NEXT:    movq %rax, %rcx
-; SSE41-NEXT:    shrq %rcx
-; SSE41-NEXT:    andl $1, %eax
-; SSE41-NEXT:    orq %rcx, %rax
-; SSE41-NEXT:    cvtsi2ss %rax, %xmm2
-; SSE41-NEXT:    addss %xmm2, %xmm2
-; SSE41-NEXT:  .LBB47_3:
-; SSE41-NEXT:    movq %xmm0, %rax
-; SSE41-NEXT:    testq %rax, %rax
-; SSE41-NEXT:    js .LBB47_4
-; SSE41-NEXT:  # %bb.5:
+; SSE41-NEXT:    movdqa %xmm1, %xmm2
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [1,1]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    movdqa %xmm1, %xmm3
+; SSE41-NEXT:    psrlq $1, %xmm3
+; SSE41-NEXT:    por %xmm0, %xmm3
+; SSE41-NEXT:    movdqa %xmm1, %xmm5
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm5
+; SSE41-NEXT:    pextrq $1, %xmm5, %rax
 ; SSE41-NEXT:    xorps %xmm0, %xmm0
 ; SSE41-NEXT:    cvtsi2ss %rax, %xmm0
-; SSE41-NEXT:    jmp .LBB47_6
-; SSE41-NEXT:  .LBB47_4:
-; SSE41-NEXT:    movq %rax, %rcx
-; SSE41-NEXT:    shrq %rcx
-; SSE41-NEXT:    andl $1, %eax
-; SSE41-NEXT:    orq %rcx, %rax
+; SSE41-NEXT:    movq %xmm5, %rax
+; SSE41-NEXT:    xorps %xmm3, %xmm3
+; SSE41-NEXT:    cvtsi2ss %rax, %xmm3
+; SSE41-NEXT:    insertps {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[2,3]
+; SSE41-NEXT:    pand %xmm2, %xmm4
+; SSE41-NEXT:    movdqa %xmm2, %xmm5
+; SSE41-NEXT:    psrlq $1, %xmm5
+; SSE41-NEXT:    por %xmm4, %xmm5
+; SSE41-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,3],xmm2[1,3]
+; SSE41-NEXT:    movaps %xmm2, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm5, %xmm2
+; SSE41-NEXT:    movq %xmm2, %rax
 ; SSE41-NEXT:    xorps %xmm0, %xmm0
 ; SSE41-NEXT:    cvtsi2ss %rax, %xmm0
-; SSE41-NEXT:    addss %xmm0, %xmm0
-; SSE41-NEXT:  .LBB47_6:
-; SSE41-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[2,3]
-; SSE41-NEXT:    movq %xmm1, %rax
-; SSE41-NEXT:    testq %rax, %rax
-; SSE41-NEXT:    js .LBB47_7
-; SSE41-NEXT:  # %bb.8:
-; SSE41-NEXT:    xorps %xmm2, %xmm2
-; SSE41-NEXT:    cvtsi2ss %rax, %xmm2
-; SSE41-NEXT:    jmp .LBB47_9
-; SSE41-NEXT:  .LBB47_7:
-; SSE41-NEXT:    movq %rax, %rcx
-; SSE41-NEXT:    shrq %rcx
-; SSE41-NEXT:    andl $1, %eax
-; SSE41-NEXT:    orq %rcx, %rax
-; SSE41-NEXT:    xorps %xmm2, %xmm2
-; SSE41-NEXT:    cvtsi2ss %rax, %xmm2
-; SSE41-NEXT:    addss %xmm2, %xmm2
-; SSE41-NEXT:  .LBB47_9:
-; SSE41-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0,1],xmm2[0],xmm0[3]
-; SSE41-NEXT:    pextrq $1, %xmm1, %rax
-; SSE41-NEXT:    testq %rax, %rax
-; SSE41-NEXT:    js .LBB47_10
-; SSE41-NEXT:  # %bb.11:
-; SSE41-NEXT:    xorps %xmm1, %xmm1
-; SSE41-NEXT:    cvtsi2ss %rax, %xmm1
-; SSE41-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[0]
-; SSE41-NEXT:    retq
-; SSE41-NEXT:  .LBB47_10:
-; SSE41-NEXT:    movq %rax, %rcx
-; SSE41-NEXT:    shrq %rcx
-; SSE41-NEXT:    andl $1, %eax
-; SSE41-NEXT:    orq %rcx, %rax
-; SSE41-NEXT:    xorps %xmm1, %xmm1
-; SSE41-NEXT:    cvtsi2ss %rax, %xmm1
-; SSE41-NEXT:    addss %xmm1, %xmm1
-; SSE41-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[0]
+; SSE41-NEXT:    insertps {{.*#+}} xmm3 = xmm3[0,1],xmm0[0],xmm3[3]
+; SSE41-NEXT:    pextrq $1, %xmm2, %rax
+; SSE41-NEXT:    xorps %xmm0, %xmm0
+; SSE41-NEXT:    cvtsi2ss %rax, %xmm0
+; SSE41-NEXT:    insertps {{.*#+}} xmm3 = xmm3[0,1,2],xmm0[0]
+; SSE41-NEXT:    movaps %xmm3, %xmm2
+; SSE41-NEXT:    addps %xmm3, %xmm2
+; SSE41-NEXT:    movaps %xmm1, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm2, %xmm3
+; SSE41-NEXT:    movaps %xmm3, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: uitofp_4i64_to_4f32:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX1-NEXT:    testq %rax, %rax
-; AVX1-NEXT:    js .LBB47_1
-; AVX1-NEXT:  # %bb.2:
-; AVX1-NEXT:    vcvtsi2ss %rax, %xmm1, %xmm1
-; AVX1-NEXT:    jmp .LBB47_3
-; AVX1-NEXT:  .LBB47_1:
-; AVX1-NEXT:    movq %rax, %rcx
-; AVX1-NEXT:    shrq %rcx
-; AVX1-NEXT:    andl $1, %eax
-; AVX1-NEXT:    orq %rcx, %rax
-; AVX1-NEXT:    vcvtsi2ss %rax, %xmm1, %xmm1
-; AVX1-NEXT:    vaddss %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:  .LBB47_3:
-; AVX1-NEXT:    vmovq %xmm0, %rax
-; AVX1-NEXT:    testq %rax, %rax
-; AVX1-NEXT:    js .LBB47_4
-; AVX1-NEXT:  # %bb.5:
-; AVX1-NEXT:    vcvtsi2ss %rax, %xmm2, %xmm2
-; AVX1-NEXT:    jmp .LBB47_6
-; AVX1-NEXT:  .LBB47_4:
-; AVX1-NEXT:    movq %rax, %rcx
-; AVX1-NEXT:    shrq %rcx
-; AVX1-NEXT:    andl $1, %eax
-; AVX1-NEXT:    orq %rcx, %rax
-; AVX1-NEXT:    vcvtsi2ss %rax, %xmm2, %xmm2
-; AVX1-NEXT:    vaddss %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:  .LBB47_6:
-; AVX1-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[2,3]
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT:    vmovq %xmm0, %rax
-; AVX1-NEXT:    testq %rax, %rax
-; AVX1-NEXT:    js .LBB47_7
-; AVX1-NEXT:  # %bb.8:
-; AVX1-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm2
-; AVX1-NEXT:    jmp .LBB47_9
-; AVX1-NEXT:  .LBB47_7:
-; AVX1-NEXT:    movq %rax, %rcx
-; AVX1-NEXT:    shrq %rcx
-; AVX1-NEXT:    andl $1, %eax
-; AVX1-NEXT:    orq %rcx, %rax
-; AVX1-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm2
-; AVX1-NEXT:    vaddss %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:  .LBB47_9:
-; AVX1-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0],xmm1[3]
-; AVX1-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX1-NEXT:    testq %rax, %rax
-; AVX1-NEXT:    js .LBB47_10
-; AVX1-NEXT:  # %bb.11:
-; AVX1-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm0
-; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
-; AVX1-NEXT:    vzeroupper
-; AVX1-NEXT:    retq
-; AVX1-NEXT:  .LBB47_10:
-; AVX1-NEXT:    movq %rax, %rcx
-; AVX1-NEXT:    shrq %rcx
-; AVX1-NEXT:    andl $1, %eax
-; AVX1-NEXT:    orq %rcx, %rax
-; AVX1-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm0
-; AVX1-NEXT:    vaddss %xmm0, %xmm0, %xmm0
-; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
+; AVX1-NEXT:    vpsrlq $1, %xmm0, %xmm1
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT:    vpsrlq $1, %xmm2, %xmm3
+; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
+; AVX1-NEXT:    vandpd {{.*}}(%rip), %ymm0, %ymm3
+; AVX1-NEXT:    vorpd %ymm3, %ymm1, %ymm1
+; AVX1-NEXT:    vblendvpd %xmm0, %xmm1, %xmm0, %xmm3
+; AVX1-NEXT:    vpextrq $1, %xmm3, %rax
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm4, %xmm4
+; AVX1-NEXT:    vmovq %xmm3, %rax
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm5, %xmm3
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[2,3]
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm1
+; AVX1-NEXT:    vblendvpd %xmm2, %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vmovq %xmm1, %rax
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm5, %xmm4
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm3 = xmm3[0,1],xmm4[0],xmm3[3]
+; AVX1-NEXT:    vpextrq $1, %xmm1, %rax
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm5, %xmm1
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[0]
+; AVX1-NEXT:    vaddps %xmm1, %xmm1, %xmm3
+; AVX1-NEXT:    vxorps %xmm4, %xmm4, %xmm4
+; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm4, %xmm2
+; AVX1-NEXT:    vpackssdw %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vblendvps %xmm0, %xmm3, %xmm1, %xmm0
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: uitofp_4i64_to_4f32:
 ; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT:    vpcmpgtq %ymm0, %ymm1, %ymm1
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
+; AVX2-NEXT:    vpackssdw %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [1,1,1,1]
+; AVX2-NEXT:    vpand %ymm2, %ymm0, %ymm2
+; AVX2-NEXT:    vpsrlq $1, %ymm0, %ymm3
+; AVX2-NEXT:    vpor %ymm2, %ymm3, %ymm2
+; AVX2-NEXT:    vblendvpd %ymm0, %ymm2, %ymm0, %ymm0
 ; AVX2-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX2-NEXT:    testq %rax, %rax
-; AVX2-NEXT:    js .LBB47_1
-; AVX2-NEXT:  # %bb.2:
-; AVX2-NEXT:    vcvtsi2ss %rax, %xmm1, %xmm1
-; AVX2-NEXT:    jmp .LBB47_3
-; AVX2-NEXT:  .LBB47_1:
-; AVX2-NEXT:    movq %rax, %rcx
-; AVX2-NEXT:    shrq %rcx
-; AVX2-NEXT:    andl $1, %eax
-; AVX2-NEXT:    orq %rcx, %rax
-; AVX2-NEXT:    vcvtsi2ss %rax, %xmm1, %xmm1
-; AVX2-NEXT:    vaddss %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:  .LBB47_3:
+; AVX2-NEXT:    vcvtsi2ss %rax, %xmm4, %xmm2
 ; AVX2-NEXT:    vmovq %xmm0, %rax
-; AVX2-NEXT:    testq %rax, %rax
-; AVX2-NEXT:    js .LBB47_4
-; AVX2-NEXT:  # %bb.5:
-; AVX2-NEXT:    vcvtsi2ss %rax, %xmm2, %xmm2
-; AVX2-NEXT:    jmp .LBB47_6
-; AVX2-NEXT:  .LBB47_4:
-; AVX2-NEXT:    movq %rax, %rcx
-; AVX2-NEXT:    shrq %rcx
-; AVX2-NEXT:    andl $1, %eax
-; AVX2-NEXT:    orq %rcx, %rax
-; AVX2-NEXT:    vcvtsi2ss %rax, %xmm2, %xmm2
-; AVX2-NEXT:    vaddss %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:  .LBB47_6:
-; AVX2-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[2,3]
+; AVX2-NEXT:    vcvtsi2ss %rax, %xmm4, %xmm3
+; AVX2-NEXT:    vinsertps {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[2,3]
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
 ; AVX2-NEXT:    vmovq %xmm0, %rax
-; AVX2-NEXT:    testq %rax, %rax
-; AVX2-NEXT:    js .LBB47_7
-; AVX2-NEXT:  # %bb.8:
-; AVX2-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm2
-; AVX2-NEXT:    jmp .LBB47_9
-; AVX2-NEXT:  .LBB47_7:
-; AVX2-NEXT:    movq %rax, %rcx
-; AVX2-NEXT:    shrq %rcx
-; AVX2-NEXT:    andl $1, %eax
-; AVX2-NEXT:    orq %rcx, %rax
-; AVX2-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm2
-; AVX2-NEXT:    vaddss %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:  .LBB47_9:
-; AVX2-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0],xmm1[3]
+; AVX2-NEXT:    vcvtsi2ss %rax, %xmm4, %xmm3
+; AVX2-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1],xmm3[0],xmm2[3]
 ; AVX2-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX2-NEXT:    testq %rax, %rax
-; AVX2-NEXT:    js .LBB47_10
-; AVX2-NEXT:  # %bb.11:
-; AVX2-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm0
-; AVX2-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
-; AVX2-NEXT:    vzeroupper
-; AVX2-NEXT:    retq
-; AVX2-NEXT:  .LBB47_10:
-; AVX2-NEXT:    movq %rax, %rcx
-; AVX2-NEXT:    shrq %rcx
-; AVX2-NEXT:    andl $1, %eax
-; AVX2-NEXT:    orq %rcx, %rax
-; AVX2-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm0
-; AVX2-NEXT:    vaddss %xmm0, %xmm0, %xmm0
-; AVX2-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
+; AVX2-NEXT:    vcvtsi2ss %rax, %xmm4, %xmm0
+; AVX2-NEXT:    vinsertps {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[0]
+; AVX2-NEXT:    vaddps %xmm0, %xmm0, %xmm2
+; AVX2-NEXT:    vblendvps %xmm1, %xmm2, %xmm0, %xmm0
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
@@ -2787,14 +2685,14 @@ define <8 x float> @uitofp_8i32_to_8f32(<8 x i32> %a) {
 ; SSE2-NEXT:    psrld $16, %xmm0
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [1392508928,1392508928,1392508928,1392508928]
 ; SSE2-NEXT:    por %xmm5, %xmm0
-; SSE2-NEXT:    movaps {{.*#+}} xmm6 = [-5.49764202E+11,-5.49764202E+11,-5.49764202E+11,-5.49764202E+11]
-; SSE2-NEXT:    addps %xmm6, %xmm0
+; SSE2-NEXT:    movaps {{.*#+}} xmm6 = [5.49764202E+11,5.49764202E+11,5.49764202E+11,5.49764202E+11]
+; SSE2-NEXT:    subps %xmm6, %xmm0
 ; SSE2-NEXT:    addps %xmm3, %xmm0
 ; SSE2-NEXT:    pand %xmm1, %xmm2
 ; SSE2-NEXT:    por %xmm4, %xmm2
 ; SSE2-NEXT:    psrld $16, %xmm1
 ; SSE2-NEXT:    por %xmm5, %xmm1
-; SSE2-NEXT:    addps %xmm6, %xmm1
+; SSE2-NEXT:    subps %xmm6, %xmm1
 ; SSE2-NEXT:    addps %xmm2, %xmm1
 ; SSE2-NEXT:    retq
 ;
@@ -2806,13 +2704,13 @@ define <8 x float> @uitofp_8i32_to_8f32(<8 x i32> %a) {
 ; SSE41-NEXT:    psrld $16, %xmm0
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [1392508928,1392508928,1392508928,1392508928]
 ; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0],xmm4[1],xmm0[2],xmm4[3],xmm0[4],xmm4[5],xmm0[6],xmm4[7]
-; SSE41-NEXT:    movaps {{.*#+}} xmm5 = [-5.49764202E+11,-5.49764202E+11,-5.49764202E+11,-5.49764202E+11]
-; SSE41-NEXT:    addps %xmm5, %xmm0
+; SSE41-NEXT:    movaps {{.*#+}} xmm5 = [5.49764202E+11,5.49764202E+11,5.49764202E+11,5.49764202E+11]
+; SSE41-NEXT:    subps %xmm5, %xmm0
 ; SSE41-NEXT:    addps %xmm3, %xmm0
 ; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
 ; SSE41-NEXT:    psrld $16, %xmm1
 ; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0],xmm4[1],xmm1[2],xmm4[3],xmm1[4],xmm4[5],xmm1[6],xmm4[7]
-; SSE41-NEXT:    addps %xmm5, %xmm1
+; SSE41-NEXT:    subps %xmm5, %xmm1
 ; SSE41-NEXT:    addps %xmm2, %xmm1
 ; SSE41-NEXT:    retq
 ;
@@ -2836,8 +2734,8 @@ define <8 x float> @uitofp_8i32_to_8f32(<8 x i32> %a) {
 ; AVX2-NEXT:    vpsrld $16, %ymm0, %ymm0
 ; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm2 = [1392508928,1392508928,1392508928,1392508928,1392508928,1392508928,1392508928,1392508928]
 ; AVX2-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm2[1],ymm0[2],ymm2[3],ymm0[4],ymm2[5],ymm0[6],ymm2[7],ymm0[8],ymm2[9],ymm0[10],ymm2[11],ymm0[12],ymm2[13],ymm0[14],ymm2[15]
-; AVX2-NEXT:    vbroadcastss {{.*#+}} ymm2 = [-5.49764202E+11,-5.49764202E+11,-5.49764202E+11,-5.49764202E+11,-5.49764202E+11,-5.49764202E+11,-5.49764202E+11,-5.49764202E+11]
-; AVX2-NEXT:    vaddps %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vbroadcastss {{.*#+}} ymm2 = [5.49764202E+11,5.49764202E+11,5.49764202E+11,5.49764202E+11,5.49764202E+11,5.49764202E+11,5.49764202E+11,5.49764202E+11]
+; AVX2-NEXT:    vsubps %ymm2, %ymm0, %ymm0
 ; AVX2-NEXT:    vaddps %ymm0, %ymm1, %ymm0
 ; AVX2-NEXT:    retq
 ;
@@ -3025,7 +2923,7 @@ define <2 x double> @sitofp_load_2i64_to_2f64(<2 x i64> *%a) {
 ; SSE2-NEXT:    movq %xmm1, %rax
 ; SSE2-NEXT:    xorps %xmm1, %xmm1
 ; SSE2-NEXT:    cvtsi2sd %rax, %xmm1
-; SSE2-NEXT:    movlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE2-NEXT:    unpcklpd {{.*#+}} xmm0 = xmm0[0],xmm1[0]
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: sitofp_load_2i64_to_2f64:
@@ -3036,7 +2934,7 @@ define <2 x double> @sitofp_load_2i64_to_2f64(<2 x i64> *%a) {
 ; SSE41-NEXT:    movq %xmm0, %rax
 ; SSE41-NEXT:    xorps %xmm0, %xmm0
 ; SSE41-NEXT:    cvtsi2sd %rax, %xmm0
-; SSE41-NEXT:    movlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE41-NEXT:    unpcklpd {{.*#+}} xmm0 = xmm0[0],xmm1[0]
 ; SSE41-NEXT:    retq
 ;
 ; VEX-LABEL: sitofp_load_2i64_to_2f64:
@@ -3046,7 +2944,7 @@ define <2 x double> @sitofp_load_2i64_to_2f64(<2 x i64> *%a) {
 ; VEX-NEXT:    vcvtsi2sd %rax, %xmm1, %xmm1
 ; VEX-NEXT:    vmovq %xmm0, %rax
 ; VEX-NEXT:    vcvtsi2sd %rax, %xmm2, %xmm0
-; VEX-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; VEX-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm1[0]
 ; VEX-NEXT:    retq
 ;
 ; AVX512F-LABEL: sitofp_load_2i64_to_2f64:
@@ -3056,7 +2954,7 @@ define <2 x double> @sitofp_load_2i64_to_2f64(<2 x i64> *%a) {
 ; AVX512F-NEXT:    vcvtsi2sd %rax, %xmm1, %xmm1
 ; AVX512F-NEXT:    vmovq %xmm0, %rax
 ; AVX512F-NEXT:    vcvtsi2sd %rax, %xmm2, %xmm0
-; AVX512F-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX512F-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm1[0]
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: sitofp_load_2i64_to_2f64:
@@ -3066,7 +2964,7 @@ define <2 x double> @sitofp_load_2i64_to_2f64(<2 x i64> *%a) {
 ; AVX512VL-NEXT:    vcvtsi2sd %rax, %xmm1, %xmm1
 ; AVX512VL-NEXT:    vmovq %xmm0, %rax
 ; AVX512VL-NEXT:    vcvtsi2sd %rax, %xmm2, %xmm0
-; AVX512VL-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX512VL-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm1[0]
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512DQ-LABEL: sitofp_load_2i64_to_2f64:
@@ -3164,15 +3062,15 @@ define <2 x double> @sitofp_load_2i16_to_2f64(<2 x i16> *%a) {
 ;
 ; SSE41-LABEL: sitofp_load_2i16_to_2f64:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pmovsxwq (%rdi), %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSE41-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE41-NEXT:    pmovsxwd %xmm0, %xmm0
 ; SSE41-NEXT:    cvtdq2pd %xmm0, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: sitofp_load_2i16_to_2f64:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpmovsxwq (%rdi), %xmm0
-; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-NEXT:    vpmovsxwd %xmm0, %xmm0
 ; AVX-NEXT:    vcvtdq2pd %xmm0, %xmm0
 ; AVX-NEXT:    retq
   %ld = load <2 x i16>, <2 x i16> *%a
@@ -3193,15 +3091,17 @@ define <2 x double> @sitofp_load_2i8_to_2f64(<2 x i8> *%a) {
 ;
 ; SSE41-LABEL: sitofp_load_2i8_to_2f64:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pmovsxbq (%rdi), %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSE41-NEXT:    movzwl (%rdi), %eax
+; SSE41-NEXT:    movd %eax, %xmm0
+; SSE41-NEXT:    pmovsxbd %xmm0, %xmm0
 ; SSE41-NEXT:    cvtdq2pd %xmm0, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: sitofp_load_2i8_to_2f64:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpmovsxbq (%rdi), %xmm0
-; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX-NEXT:    movzwl (%rdi), %eax
+; AVX-NEXT:    vmovd %eax, %xmm0
+; AVX-NEXT:    vpmovsxbd %xmm0, %xmm0
 ; AVX-NEXT:    vcvtdq2pd %xmm0, %xmm0
 ; AVX-NEXT:    retq
   %ld = load <2 x i8>, <2 x i8> *%a
@@ -3220,7 +3120,7 @@ define <4 x double> @sitofp_load_4i64_to_4f64(<4 x i64> *%a) {
 ; SSE2-NEXT:    movq %xmm1, %rax
 ; SSE2-NEXT:    xorps %xmm1, %xmm1
 ; SSE2-NEXT:    cvtsi2sd %rax, %xmm1
-; SSE2-NEXT:    movlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE2-NEXT:    unpcklpd {{.*#+}} xmm0 = xmm0[0],xmm1[0]
 ; SSE2-NEXT:    movq %xmm2, %rax
 ; SSE2-NEXT:    xorps %xmm1, %xmm1
 ; SSE2-NEXT:    cvtsi2sd %rax, %xmm1
@@ -3228,7 +3128,7 @@ define <4 x double> @sitofp_load_4i64_to_4f64(<4 x i64> *%a) {
 ; SSE2-NEXT:    movq %xmm2, %rax
 ; SSE2-NEXT:    xorps %xmm2, %xmm2
 ; SSE2-NEXT:    cvtsi2sd %rax, %xmm2
-; SSE2-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; SSE2-NEXT:    unpcklpd {{.*#+}} xmm1 = xmm1[0],xmm2[0]
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: sitofp_load_4i64_to_4f64:
@@ -3240,64 +3140,64 @@ define <4 x double> @sitofp_load_4i64_to_4f64(<4 x i64> *%a) {
 ; SSE41-NEXT:    movq %xmm0, %rax
 ; SSE41-NEXT:    xorps %xmm0, %xmm0
 ; SSE41-NEXT:    cvtsi2sd %rax, %xmm0
-; SSE41-NEXT:    movlhps {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SSE41-NEXT:    unpcklpd {{.*#+}} xmm0 = xmm0[0],xmm2[0]
 ; SSE41-NEXT:    pextrq $1, %xmm1, %rax
 ; SSE41-NEXT:    xorps %xmm2, %xmm2
 ; SSE41-NEXT:    cvtsi2sd %rax, %xmm2
 ; SSE41-NEXT:    movq %xmm1, %rax
 ; SSE41-NEXT:    xorps %xmm1, %xmm1
 ; SSE41-NEXT:    cvtsi2sd %rax, %xmm1
-; SSE41-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; SSE41-NEXT:    unpcklpd {{.*#+}} xmm1 = xmm1[0],xmm2[0]
 ; SSE41-NEXT:    retq
 ;
 ; VEX-LABEL: sitofp_load_4i64_to_4f64:
 ; VEX:       # %bb.0:
-; VEX-NEXT:    vmovdqa (%rdi), %xmm0
+; VEX-NEXT:    vmovapd (%rdi), %xmm0
 ; VEX-NEXT:    vmovdqa 16(%rdi), %xmm1
 ; VEX-NEXT:    vpextrq $1, %xmm1, %rax
 ; VEX-NEXT:    vcvtsi2sd %rax, %xmm2, %xmm2
 ; VEX-NEXT:    vmovq %xmm1, %rax
 ; VEX-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm1
-; VEX-NEXT:    vmovlhps {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; VEX-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm1[0],xmm2[0]
 ; VEX-NEXT:    vpextrq $1, %xmm0, %rax
 ; VEX-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm2
 ; VEX-NEXT:    vmovq %xmm0, %rax
 ; VEX-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm0
-; VEX-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; VEX-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm2[0]
 ; VEX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; VEX-NEXT:    retq
 ;
 ; AVX512F-LABEL: sitofp_load_4i64_to_4f64:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512F-NEXT:    vmovapd (%rdi), %xmm0
 ; AVX512F-NEXT:    vmovdqa 16(%rdi), %xmm1
 ; AVX512F-NEXT:    vpextrq $1, %xmm1, %rax
 ; AVX512F-NEXT:    vcvtsi2sd %rax, %xmm2, %xmm2
 ; AVX512F-NEXT:    vmovq %xmm1, %rax
 ; AVX512F-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm1
-; AVX512F-NEXT:    vmovlhps {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; AVX512F-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm1[0],xmm2[0]
 ; AVX512F-NEXT:    vpextrq $1, %xmm0, %rax
 ; AVX512F-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm2
 ; AVX512F-NEXT:    vmovq %xmm0, %rax
 ; AVX512F-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm0
-; AVX512F-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; AVX512F-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm2[0]
 ; AVX512F-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: sitofp_load_4i64_to_4f64:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512VL-NEXT:    vmovapd (%rdi), %xmm0
 ; AVX512VL-NEXT:    vmovdqa 16(%rdi), %xmm1
 ; AVX512VL-NEXT:    vpextrq $1, %xmm1, %rax
 ; AVX512VL-NEXT:    vcvtsi2sd %rax, %xmm2, %xmm2
 ; AVX512VL-NEXT:    vmovq %xmm1, %rax
 ; AVX512VL-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm1
-; AVX512VL-NEXT:    vmovlhps {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; AVX512VL-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm1[0],xmm2[0]
 ; AVX512VL-NEXT:    vpextrq $1, %xmm0, %rax
 ; AVX512VL-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm2
 ; AVX512VL-NEXT:    vmovq %xmm0, %rax
 ; AVX512VL-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm0
-; AVX512VL-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; AVX512VL-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm2[0]
 ; AVX512VL-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
@@ -3490,38 +3390,28 @@ define <2 x double> @uitofp_load_2i64_to_2f64(<2 x i64> *%a) {
 define <2 x double> @uitofp_load_2i32_to_2f64(<2 x i32> *%a) {
 ; SSE2-LABEL: uitofp_load_2i32_to_2f64:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [65535,0,65535,0,0,0,0,0]
-; SSE2-NEXT:    pand %xmm0, %xmm1
-; SSE2-NEXT:    cvtdq2pd %xmm1, %xmm1
-; SSE2-NEXT:    psrld $16, %xmm0
-; SSE2-NEXT:    cvtdq2pd %xmm0, %xmm0
-; SSE2-NEXT:    mulpd {{.*}}(%rip), %xmm0
-; SSE2-NEXT:    addpd %xmm1, %xmm0
+; SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE2-NEXT:    xorpd %xmm1, %xmm1
+; SSE2-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT:    movapd {{.*#+}} xmm1 = [4.503599627370496E+15,4.503599627370496E+15]
+; SSE2-NEXT:    orpd %xmm1, %xmm0
+; SSE2-NEXT:    subpd %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: uitofp_load_2i32_to_2f64:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; SSE41-NEXT:    pxor %xmm1, %xmm1
-; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4,5,6,7]
-; SSE41-NEXT:    cvtdq2pd %xmm1, %xmm1
-; SSE41-NEXT:    psrld $16, %xmm0
-; SSE41-NEXT:    cvtdq2pd %xmm0, %xmm0
-; SSE41-NEXT:    mulpd {{.*}}(%rip), %xmm0
-; SSE41-NEXT:    addpd %xmm1, %xmm0
+; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm0 = mem[0],zero,mem[1],zero
+; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [4.503599627370496E+15,4.503599627370496E+15]
+; SSE41-NEXT:    por %xmm1, %xmm0
+; SSE41-NEXT:    subpd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; VEX-LABEL: uitofp_load_2i32_to_2f64:
 ; VEX:       # %bb.0:
-; VEX-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
-; VEX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; VEX-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4,5,6,7]
-; VEX-NEXT:    vcvtdq2pd %xmm1, %xmm1
-; VEX-NEXT:    vpsrld $16, %xmm0, %xmm0
-; VEX-NEXT:    vcvtdq2pd %xmm0, %xmm0
-; VEX-NEXT:    vmulpd {{.*}}(%rip), %xmm0, %xmm0
-; VEX-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
+; VEX-NEXT:    vpmovzxdq {{.*#+}} xmm0 = mem[0],zero,mem[1],zero
+; VEX-NEXT:    vmovdqa {{.*#+}} xmm1 = [4.503599627370496E+15,4.503599627370496E+15]
+; VEX-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; VEX-NEXT:    vsubpd %xmm1, %xmm0, %xmm0
 ; VEX-NEXT:    retq
 ;
 ; AVX512F-LABEL: uitofp_load_2i32_to_2f64:
@@ -3557,43 +3447,43 @@ define <2 x double> @uitofp_load_2i32_to_2f64(<2 x i32> *%a) {
 define <2 x double> @uitofp_load_4i32_to_2f64_2(<4 x i32>* %x) {
 ; SSE2-LABEL: uitofp_load_4i32_to_2f64_2:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa (%rdi), %xmm0
-; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [65535,0,65535,0,0,0,0,0]
-; SSE2-NEXT:    pand %xmm0, %xmm1
-; SSE2-NEXT:    cvtdq2pd %xmm1, %xmm1
-; SSE2-NEXT:    psrld $16, %xmm0
-; SSE2-NEXT:    cvtdq2pd %xmm0, %xmm0
-; SSE2-NEXT:    mulpd {{.*}}(%rip), %xmm0
-; SSE2-NEXT:    addpd %xmm1, %xmm0
+; SSE2-NEXT:    movapd (%rdi), %xmm0
+; SSE2-NEXT:    xorpd %xmm1, %xmm1
+; SSE2-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT:    movapd {{.*#+}} xmm1 = [4.503599627370496E+15,4.503599627370496E+15]
+; SSE2-NEXT:    orpd %xmm1, %xmm0
+; SSE2-NEXT:    subpd %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: uitofp_load_4i32_to_2f64_2:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa (%rdi), %xmm0
-; SSE41-NEXT:    pxor %xmm1, %xmm1
-; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4,5,6,7]
-; SSE41-NEXT:    cvtdq2pd %xmm1, %xmm1
-; SSE41-NEXT:    psrld $16, %xmm0
-; SSE41-NEXT:    cvtdq2pd %xmm0, %xmm0
-; SSE41-NEXT:    mulpd {{.*}}(%rip), %xmm0
-; SSE41-NEXT:    addpd %xmm1, %xmm0
+; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm0 = mem[0],zero,mem[1],zero
+; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [4.503599627370496E+15,4.503599627370496E+15]
+; SSE41-NEXT:    por %xmm1, %xmm0
+; SSE41-NEXT:    subpd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
-; VEX-LABEL: uitofp_load_4i32_to_2f64_2:
-; VEX:       # %bb.0:
-; VEX-NEXT:    vmovdqa (%rdi), %xmm0
-; VEX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; VEX-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7]
-; VEX-NEXT:    vpsrld $16, %xmm0, %xmm0
-; VEX-NEXT:    vcvtdq2pd %xmm1, %xmm1
-; VEX-NEXT:    vcvtdq2pd %xmm0, %xmm0
-; VEX-NEXT:    vmulpd {{.*}}(%rip), %xmm0, %xmm0
-; VEX-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
-; VEX-NEXT:    retq
+; AVX1-LABEL: uitofp_load_4i32_to_2f64_2:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm0 = mem[0],zero,mem[1],zero
+; AVX1-NEXT:    vmovddup {{.*#+}} xmm1 = [4.503599627370496E+15,4.503599627370496E+15]
+; AVX1-NEXT:    # xmm1 = mem[0,0]
+; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vsubpd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: uitofp_load_4i32_to_2f64_2:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} xmm1 = [4.503599627370496E+15,4.503599627370496E+15]
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vsubpd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: uitofp_load_4i32_to_2f64_2:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX512F-NEXT:    vmovaps (%rdi), %xmm0
 ; AVX512F-NEXT:    vcvtudq2pd %ymm0, %zmm0
 ; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
 ; AVX512F-NEXT:    vzeroupper
@@ -3606,7 +3496,7 @@ define <2 x double> @uitofp_load_4i32_to_2f64_2(<4 x i32>* %x) {
 ;
 ; AVX512DQ-LABEL: uitofp_load_4i32_to_2f64_2:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX512DQ-NEXT:    vmovaps (%rdi), %xmm0
 ; AVX512DQ-NEXT:    vcvtudq2pd %ymm0, %zmm0
 ; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
 ; AVX512DQ-NEXT:    vzeroupper
@@ -3625,39 +3515,39 @@ define <2 x double> @uitofp_load_4i32_to_2f64_2(<4 x i32>* %x) {
 define <2 x double> @uitofp_volatile_load_4i32_to_2f64_2(<4 x i32>* %x) {
 ; SSE2-LABEL: uitofp_volatile_load_4i32_to_2f64_2:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa (%rdi), %xmm0
-; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [65535,0,65535,0,0,0,0,0]
-; SSE2-NEXT:    pand %xmm0, %xmm1
-; SSE2-NEXT:    cvtdq2pd %xmm1, %xmm1
-; SSE2-NEXT:    psrld $16, %xmm0
-; SSE2-NEXT:    cvtdq2pd %xmm0, %xmm0
-; SSE2-NEXT:    mulpd {{.*}}(%rip), %xmm0
-; SSE2-NEXT:    addpd %xmm1, %xmm0
+; SSE2-NEXT:    movapd (%rdi), %xmm0
+; SSE2-NEXT:    xorpd %xmm1, %xmm1
+; SSE2-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT:    movapd {{.*#+}} xmm1 = [4.503599627370496E+15,4.503599627370496E+15]
+; SSE2-NEXT:    orpd %xmm1, %xmm0
+; SSE2-NEXT:    subpd %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: uitofp_volatile_load_4i32_to_2f64_2:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa (%rdi), %xmm0
-; SSE41-NEXT:    pxor %xmm1, %xmm1
-; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4,5,6,7]
-; SSE41-NEXT:    cvtdq2pd %xmm1, %xmm1
-; SSE41-NEXT:    psrld $16, %xmm0
-; SSE41-NEXT:    cvtdq2pd %xmm0, %xmm0
-; SSE41-NEXT:    mulpd {{.*}}(%rip), %xmm0
-; SSE41-NEXT:    addpd %xmm1, %xmm0
+; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm0 = mem[0],zero,mem[1],zero
+; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [4.503599627370496E+15,4.503599627370496E+15]
+; SSE41-NEXT:    por %xmm1, %xmm0
+; SSE41-NEXT:    subpd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
-; VEX-LABEL: uitofp_volatile_load_4i32_to_2f64_2:
-; VEX:       # %bb.0:
-; VEX-NEXT:    vmovdqa (%rdi), %xmm0
-; VEX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; VEX-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7]
-; VEX-NEXT:    vpsrld $16, %xmm0, %xmm0
-; VEX-NEXT:    vcvtdq2pd %xmm1, %xmm1
-; VEX-NEXT:    vcvtdq2pd %xmm0, %xmm0
-; VEX-NEXT:    vmulpd {{.*}}(%rip), %xmm0, %xmm0
-; VEX-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
-; VEX-NEXT:    retq
+; AVX1-LABEL: uitofp_volatile_load_4i32_to_2f64_2:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm0 = mem[0],zero,mem[1],zero
+; AVX1-NEXT:    vmovddup {{.*#+}} xmm1 = [4.503599627370496E+15,4.503599627370496E+15]
+; AVX1-NEXT:    # xmm1 = mem[0,0]
+; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vsubpd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: uitofp_volatile_load_4i32_to_2f64_2:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} xmm1 = [4.503599627370496E+15,4.503599627370496E+15]
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vsubpd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: uitofp_volatile_load_4i32_to_2f64_2:
 ; AVX512F:       # %bb.0:
@@ -3732,15 +3622,17 @@ define <2 x double> @uitofp_load_2i8_to_2f64(<2 x i8> *%a) {
 ;
 ; SSE41-LABEL: uitofp_load_2i8_to_2f64:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pmovzxbq {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSE41-NEXT:    movzwl (%rdi), %eax
+; SSE41-NEXT:    movd %eax, %xmm0
+; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SSE41-NEXT:    cvtdq2pd %xmm0, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: uitofp_load_2i8_to_2f64:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpmovzxbq {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero
-; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX-NEXT:    movzwl (%rdi), %eax
+; AVX-NEXT:    vmovd %eax, %xmm0
+; AVX-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; AVX-NEXT:    vcvtdq2pd %xmm0, %xmm0
 ; AVX-NEXT:    retq
   %ld = load <2 x i8>, <2 x i8> *%a
@@ -3870,71 +3762,49 @@ define <4 x double> @uitofp_load_4i64_to_4f64(<4 x i64> *%a) {
 define <4 x double> @uitofp_load_4i32_to_4f64(<4 x i32> *%a) {
 ; SSE2-LABEL: uitofp_load_4i32_to_4f64:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa (%rdi), %xmm0
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    psrld $16, %xmm1
-; SSE2-NEXT:    cvtdq2pd %xmm1, %xmm1
-; SSE2-NEXT:    movapd {{.*#+}} xmm2 = [6.5536E+4,6.5536E+4]
-; SSE2-NEXT:    mulpd %xmm2, %xmm1
-; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [65535,0,65535,0,0,0,0,0]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[2,3,0,1]
-; SSE2-NEXT:    pand %xmm3, %xmm0
-; SSE2-NEXT:    cvtdq2pd %xmm0, %xmm0
-; SSE2-NEXT:    addpd %xmm1, %xmm0
-; SSE2-NEXT:    movdqa %xmm4, %xmm1
-; SSE2-NEXT:    psrld $16, %xmm1
-; SSE2-NEXT:    cvtdq2pd %xmm1, %xmm5
-; SSE2-NEXT:    mulpd %xmm2, %xmm5
-; SSE2-NEXT:    pand %xmm3, %xmm4
-; SSE2-NEXT:    cvtdq2pd %xmm4, %xmm1
-; SSE2-NEXT:    addpd %xmm5, %xmm1
+; SSE2-NEXT:    movapd (%rdi), %xmm1
+; SSE2-NEXT:    xorpd %xmm2, %xmm2
+; SSE2-NEXT:    movapd %xmm1, %xmm0
+; SSE2-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; SSE2-NEXT:    movapd {{.*#+}} xmm3 = [4.503599627370496E+15,4.503599627370496E+15]
+; SSE2-NEXT:    orpd %xmm3, %xmm0
+; SSE2-NEXT:    subpd %xmm3, %xmm0
+; SSE2-NEXT:    unpckhps {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; SSE2-NEXT:    orpd %xmm3, %xmm1
+; SSE2-NEXT:    subpd %xmm3, %xmm1
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: uitofp_load_4i32_to_4f64:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa (%rdi), %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    psrld $16, %xmm1
-; SSE41-NEXT:    cvtdq2pd %xmm1, %xmm1
-; SSE41-NEXT:    movapd {{.*#+}} xmm2 = [6.5536E+4,6.5536E+4]
-; SSE41-NEXT:    mulpd %xmm2, %xmm1
-; SSE41-NEXT:    pxor %xmm3, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[2,3,0,1]
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0],xmm3[1],xmm0[2],xmm3[3],xmm0[4,5,6,7]
-; SSE41-NEXT:    cvtdq2pd %xmm0, %xmm0
-; SSE41-NEXT:    addpd %xmm1, %xmm0
-; SSE41-NEXT:    movdqa %xmm4, %xmm1
-; SSE41-NEXT:    psrld $16, %xmm1
-; SSE41-NEXT:    cvtdq2pd %xmm1, %xmm5
-; SSE41-NEXT:    mulpd %xmm2, %xmm5
-; SSE41-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0],xmm3[1],xmm4[2],xmm3[3],xmm4[4,5,6,7]
-; SSE41-NEXT:    cvtdq2pd %xmm4, %xmm1
-; SSE41-NEXT:    addpd %xmm5, %xmm1
+; SSE41-NEXT:    movdqa (%rdi), %xmm1
+; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero
+; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [4.503599627370496E+15,4.503599627370496E+15]
+; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    subpd %xmm2, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
+; SSE41-NEXT:    por %xmm2, %xmm1
+; SSE41-NEXT:    subpd %xmm2, %xmm1
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: uitofp_load_4i32_to_4f64:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vmovdqa (%rdi), %xmm0
 ; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7]
-; AVX1-NEXT:    vcvtdq2pd %xmm1, %ymm1
-; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm0
-; AVX1-NEXT:    vcvtdq2pd %xmm0, %ymm0
-; AVX1-NEXT:    vmulpd {{.*}}(%rip), %ymm0, %ymm0
-; AVX1-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm1 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    vbroadcastsd {{.*#+}} ymm1 = [4.503599627370496E+15,4.503599627370496E+15,4.503599627370496E+15,4.503599627370496E+15]
+; AVX1-NEXT:    vorpd %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vsubpd %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: uitofp_load_4i32_to_4f64:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX2-NEXT:    vcvtdq2pd %xmm1, %ymm1
-; AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm2 = [6.5536E+4,6.5536E+4,6.5536E+4,6.5536E+4]
-; AVX2-NEXT:    vmulpd %ymm2, %ymm1, %ymm1
-; AVX2-NEXT:    vxorpd %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3],xmm0[4],xmm2[5],xmm0[6],xmm2[7]
-; AVX2-NEXT:    vcvtdq2pd %xmm0, %ymm0
-; AVX2-NEXT:    vaddpd %ymm0, %ymm1, %ymm0
+; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [4.503599627370496E+15,4.503599627370496E+15,4.503599627370496E+15,4.503599627370496E+15]
+; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vsubpd %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: uitofp_load_4i32_to_4f64:
@@ -4286,7 +4156,7 @@ define <8 x float> @sitofp_load_8i64_to_8f32(<8 x i64> *%a) {
 ;
 ; VEX-LABEL: sitofp_load_8i64_to_8f32:
 ; VEX:       # %bb.0:
-; VEX-NEXT:    vmovdqa (%rdi), %xmm0
+; VEX-NEXT:    vmovaps (%rdi), %xmm0
 ; VEX-NEXT:    vmovdqa 16(%rdi), %xmm1
 ; VEX-NEXT:    vmovdqa 32(%rdi), %xmm2
 ; VEX-NEXT:    vmovdqa 48(%rdi), %xmm3
@@ -4317,7 +4187,7 @@ define <8 x float> @sitofp_load_8i64_to_8f32(<8 x i64> *%a) {
 ;
 ; AVX512F-LABEL: sitofp_load_8i64_to_8f32:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512F-NEXT:    vmovaps (%rdi), %xmm0
 ; AVX512F-NEXT:    vmovdqa 16(%rdi), %xmm1
 ; AVX512F-NEXT:    vmovdqa 32(%rdi), %xmm2
 ; AVX512F-NEXT:    vmovdqa 48(%rdi), %xmm3
@@ -4348,7 +4218,7 @@ define <8 x float> @sitofp_load_8i64_to_8f32(<8 x i64> *%a) {
 ;
 ; AVX512VL-LABEL: sitofp_load_8i64_to_8f32:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512VL-NEXT:    vmovaps (%rdi), %xmm0
 ; AVX512VL-NEXT:    vmovdqa 16(%rdi), %xmm1
 ; AVX512VL-NEXT:    vmovdqa 32(%rdi), %xmm2
 ; AVX512VL-NEXT:    vmovdqa 48(%rdi), %xmm3
@@ -4454,8 +4324,8 @@ define <8 x float> @sitofp_load_8i16_to_8f32(<8 x i16> *%a) {
 define <8 x float> @sitofp_load_8i8_to_8f32(<8 x i8> *%a) {
 ; SSE2-LABEL: sitofp_load_8i8_to_8f32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
 ; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
 ; SSE2-NEXT:    psrad $24, %xmm0
 ; SSE2-NEXT:    cvtdq2ps %xmm0, %xmm0
@@ -4507,41 +4377,41 @@ define <4 x float> @uitofp_load_4i64_to_4f32(<4 x i64> *%a) {
 ; SSE2-NEXT:    movdqa 16(%rdi), %xmm0
 ; SSE2-NEXT:    movq %xmm0, %rax
 ; SSE2-NEXT:    testq %rax, %rax
-; SSE2-NEXT:    js .LBB81_1
+; SSE2-NEXT:    js .LBB83_1
 ; SSE2-NEXT:  # %bb.2:
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm1
-; SSE2-NEXT:    jmp .LBB81_3
-; SSE2-NEXT:  .LBB81_1:
+; SSE2-NEXT:    jmp .LBB83_3
+; SSE2-NEXT:  .LBB83_1:
 ; SSE2-NEXT:    movq %rax, %rcx
 ; SSE2-NEXT:    shrq %rcx
 ; SSE2-NEXT:    andl $1, %eax
 ; SSE2-NEXT:    orq %rcx, %rax
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm1
 ; SSE2-NEXT:    addss %xmm1, %xmm1
-; SSE2-NEXT:  .LBB81_3:
+; SSE2-NEXT:  .LBB83_3:
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
 ; SSE2-NEXT:    movq %xmm0, %rax
 ; SSE2-NEXT:    testq %rax, %rax
-; SSE2-NEXT:    js .LBB81_4
+; SSE2-NEXT:    js .LBB83_4
 ; SSE2-NEXT:  # %bb.5:
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm3
-; SSE2-NEXT:    jmp .LBB81_6
-; SSE2-NEXT:  .LBB81_4:
+; SSE2-NEXT:    jmp .LBB83_6
+; SSE2-NEXT:  .LBB83_4:
 ; SSE2-NEXT:    movq %rax, %rcx
 ; SSE2-NEXT:    shrq %rcx
 ; SSE2-NEXT:    andl $1, %eax
 ; SSE2-NEXT:    orq %rcx, %rax
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm3
 ; SSE2-NEXT:    addss %xmm3, %xmm3
-; SSE2-NEXT:  .LBB81_6:
+; SSE2-NEXT:  .LBB83_6:
 ; SSE2-NEXT:    movq %xmm2, %rax
 ; SSE2-NEXT:    testq %rax, %rax
-; SSE2-NEXT:    js .LBB81_7
+; SSE2-NEXT:    js .LBB83_7
 ; SSE2-NEXT:  # %bb.8:
 ; SSE2-NEXT:    xorps %xmm0, %xmm0
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm0
-; SSE2-NEXT:    jmp .LBB81_9
-; SSE2-NEXT:  .LBB81_7:
+; SSE2-NEXT:    jmp .LBB83_9
+; SSE2-NEXT:  .LBB83_7:
 ; SSE2-NEXT:    movq %rax, %rcx
 ; SSE2-NEXT:    shrq %rcx
 ; SSE2-NEXT:    andl $1, %eax
@@ -4549,17 +4419,17 @@ define <4 x float> @uitofp_load_4i64_to_4f32(<4 x i64> *%a) {
 ; SSE2-NEXT:    xorps %xmm0, %xmm0
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm0
 ; SSE2-NEXT:    addss %xmm0, %xmm0
-; SSE2-NEXT:  .LBB81_9:
+; SSE2-NEXT:  .LBB83_9:
 ; SSE2-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
 ; SSE2-NEXT:    movq %xmm2, %rax
 ; SSE2-NEXT:    testq %rax, %rax
-; SSE2-NEXT:    js .LBB81_10
+; SSE2-NEXT:    js .LBB83_10
 ; SSE2-NEXT:  # %bb.11:
 ; SSE2-NEXT:    xorps %xmm2, %xmm2
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm2
-; SSE2-NEXT:    jmp .LBB81_12
-; SSE2-NEXT:  .LBB81_10:
+; SSE2-NEXT:    jmp .LBB83_12
+; SSE2-NEXT:  .LBB83_10:
 ; SSE2-NEXT:    movq %rax, %rcx
 ; SSE2-NEXT:    shrq %rcx
 ; SSE2-NEXT:    andl $1, %eax
@@ -4567,146 +4437,113 @@ define <4 x float> @uitofp_load_4i64_to_4f32(<4 x i64> *%a) {
 ; SSE2-NEXT:    xorps %xmm2, %xmm2
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm2
 ; SSE2-NEXT:    addss %xmm2, %xmm2
-; SSE2-NEXT:  .LBB81_12:
+; SSE2-NEXT:  .LBB83_12:
 ; SSE2-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
 ; SSE2-NEXT:    movlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: uitofp_load_4i64_to_4f32:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa (%rdi), %xmm0
-; SSE41-NEXT:    movdqa 16(%rdi), %xmm1
-; SSE41-NEXT:    pextrq $1, %xmm0, %rax
-; SSE41-NEXT:    testq %rax, %rax
-; SSE41-NEXT:    js .LBB81_1
-; SSE41-NEXT:  # %bb.2:
-; SSE41-NEXT:    cvtsi2ss %rax, %xmm2
-; SSE41-NEXT:    jmp .LBB81_3
-; SSE41-NEXT:  .LBB81_1:
-; SSE41-NEXT:    movq %rax, %rcx
-; SSE41-NEXT:    shrq %rcx
-; SSE41-NEXT:    andl $1, %eax
-; SSE41-NEXT:    orq %rcx, %rax
-; SSE41-NEXT:    cvtsi2ss %rax, %xmm2
-; SSE41-NEXT:    addss %xmm2, %xmm2
-; SSE41-NEXT:  .LBB81_3:
-; SSE41-NEXT:    movq %xmm0, %rax
-; SSE41-NEXT:    testq %rax, %rax
-; SSE41-NEXT:    js .LBB81_4
-; SSE41-NEXT:  # %bb.5:
+; SSE41-NEXT:    movdqa (%rdi), %xmm1
+; SSE41-NEXT:    movdqa 16(%rdi), %xmm2
+; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [1,1]
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    movdqa %xmm1, %xmm3
+; SSE41-NEXT:    psrlq $1, %xmm3
+; SSE41-NEXT:    por %xmm0, %xmm3
+; SSE41-NEXT:    movdqa %xmm1, %xmm5
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm5
+; SSE41-NEXT:    pextrq $1, %xmm5, %rax
+; SSE41-NEXT:    xorps %xmm0, %xmm0
+; SSE41-NEXT:    cvtsi2ss %rax, %xmm0
+; SSE41-NEXT:    movq %xmm5, %rax
+; SSE41-NEXT:    xorps %xmm3, %xmm3
+; SSE41-NEXT:    cvtsi2ss %rax, %xmm3
+; SSE41-NEXT:    insertps {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[2,3]
+; SSE41-NEXT:    pand %xmm2, %xmm4
+; SSE41-NEXT:    movdqa %xmm2, %xmm5
+; SSE41-NEXT:    psrlq $1, %xmm5
+; SSE41-NEXT:    por %xmm4, %xmm5
+; SSE41-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,3],xmm2[1,3]
+; SSE41-NEXT:    movaps %xmm2, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm5, %xmm2
+; SSE41-NEXT:    movq %xmm2, %rax
 ; SSE41-NEXT:    xorps %xmm0, %xmm0
 ; SSE41-NEXT:    cvtsi2ss %rax, %xmm0
-; SSE41-NEXT:    jmp .LBB81_6
-; SSE41-NEXT:  .LBB81_4:
-; SSE41-NEXT:    movq %rax, %rcx
-; SSE41-NEXT:    shrq %rcx
-; SSE41-NEXT:    andl $1, %eax
-; SSE41-NEXT:    orq %rcx, %rax
+; SSE41-NEXT:    insertps {{.*#+}} xmm3 = xmm3[0,1],xmm0[0],xmm3[3]
+; SSE41-NEXT:    pextrq $1, %xmm2, %rax
 ; SSE41-NEXT:    xorps %xmm0, %xmm0
 ; SSE41-NEXT:    cvtsi2ss %rax, %xmm0
-; SSE41-NEXT:    addss %xmm0, %xmm0
-; SSE41-NEXT:  .LBB81_6:
-; SSE41-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[2,3]
-; SSE41-NEXT:    movq %xmm1, %rax
-; SSE41-NEXT:    testq %rax, %rax
-; SSE41-NEXT:    js .LBB81_7
-; SSE41-NEXT:  # %bb.8:
-; SSE41-NEXT:    xorps %xmm2, %xmm2
-; SSE41-NEXT:    cvtsi2ss %rax, %xmm2
-; SSE41-NEXT:    jmp .LBB81_9
-; SSE41-NEXT:  .LBB81_7:
-; SSE41-NEXT:    movq %rax, %rcx
-; SSE41-NEXT:    shrq %rcx
-; SSE41-NEXT:    andl $1, %eax
-; SSE41-NEXT:    orq %rcx, %rax
-; SSE41-NEXT:    xorps %xmm2, %xmm2
-; SSE41-NEXT:    cvtsi2ss %rax, %xmm2
-; SSE41-NEXT:    addss %xmm2, %xmm2
-; SSE41-NEXT:  .LBB81_9:
-; SSE41-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0,1],xmm2[0],xmm0[3]
-; SSE41-NEXT:    pextrq $1, %xmm1, %rax
-; SSE41-NEXT:    testq %rax, %rax
-; SSE41-NEXT:    js .LBB81_10
-; SSE41-NEXT:  # %bb.11:
-; SSE41-NEXT:    xorps %xmm1, %xmm1
-; SSE41-NEXT:    cvtsi2ss %rax, %xmm1
-; SSE41-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[0]
-; SSE41-NEXT:    retq
-; SSE41-NEXT:  .LBB81_10:
-; SSE41-NEXT:    movq %rax, %rcx
-; SSE41-NEXT:    shrq %rcx
-; SSE41-NEXT:    andl $1, %eax
-; SSE41-NEXT:    orq %rcx, %rax
-; SSE41-NEXT:    xorps %xmm1, %xmm1
-; SSE41-NEXT:    cvtsi2ss %rax, %xmm1
-; SSE41-NEXT:    addss %xmm1, %xmm1
-; SSE41-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[0]
+; SSE41-NEXT:    insertps {{.*#+}} xmm3 = xmm3[0,1,2],xmm0[0]
+; SSE41-NEXT:    movaps %xmm3, %xmm2
+; SSE41-NEXT:    addps %xmm3, %xmm2
+; SSE41-NEXT:    movaps %xmm1, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm2, %xmm3
+; SSE41-NEXT:    movaps %xmm3, %xmm0
 ; SSE41-NEXT:    retq
 ;
-; VEX-LABEL: uitofp_load_4i64_to_4f32:
-; VEX:       # %bb.0:
-; VEX-NEXT:    vmovdqa (%rdi), %xmm2
-; VEX-NEXT:    vmovdqa 16(%rdi), %xmm0
-; VEX-NEXT:    vpextrq $1, %xmm2, %rax
-; VEX-NEXT:    testq %rax, %rax
-; VEX-NEXT:    js .LBB81_1
-; VEX-NEXT:  # %bb.2:
-; VEX-NEXT:    vcvtsi2ss %rax, %xmm1, %xmm1
-; VEX-NEXT:    jmp .LBB81_3
-; VEX-NEXT:  .LBB81_1:
-; VEX-NEXT:    movq %rax, %rcx
-; VEX-NEXT:    shrq %rcx
-; VEX-NEXT:    andl $1, %eax
-; VEX-NEXT:    orq %rcx, %rax
-; VEX-NEXT:    vcvtsi2ss %rax, %xmm1, %xmm1
-; VEX-NEXT:    vaddss %xmm1, %xmm1, %xmm1
-; VEX-NEXT:  .LBB81_3:
-; VEX-NEXT:    vmovq %xmm2, %rax
-; VEX-NEXT:    testq %rax, %rax
-; VEX-NEXT:    js .LBB81_4
-; VEX-NEXT:  # %bb.5:
-; VEX-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm2
-; VEX-NEXT:    jmp .LBB81_6
-; VEX-NEXT:  .LBB81_4:
-; VEX-NEXT:    movq %rax, %rcx
-; VEX-NEXT:    shrq %rcx
-; VEX-NEXT:    andl $1, %eax
-; VEX-NEXT:    orq %rcx, %rax
-; VEX-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm2
-; VEX-NEXT:    vaddss %xmm2, %xmm2, %xmm2
-; VEX-NEXT:  .LBB81_6:
-; VEX-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[2,3]
-; VEX-NEXT:    vmovq %xmm0, %rax
-; VEX-NEXT:    testq %rax, %rax
-; VEX-NEXT:    js .LBB81_7
-; VEX-NEXT:  # %bb.8:
-; VEX-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm2
-; VEX-NEXT:    jmp .LBB81_9
-; VEX-NEXT:  .LBB81_7:
-; VEX-NEXT:    movq %rax, %rcx
-; VEX-NEXT:    shrq %rcx
-; VEX-NEXT:    andl $1, %eax
-; VEX-NEXT:    orq %rcx, %rax
-; VEX-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm2
-; VEX-NEXT:    vaddss %xmm2, %xmm2, %xmm2
-; VEX-NEXT:  .LBB81_9:
-; VEX-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0],xmm1[3]
-; VEX-NEXT:    vpextrq $1, %xmm0, %rax
-; VEX-NEXT:    testq %rax, %rax
-; VEX-NEXT:    js .LBB81_10
-; VEX-NEXT:  # %bb.11:
-; VEX-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm0
-; VEX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
-; VEX-NEXT:    retq
-; VEX-NEXT:  .LBB81_10:
-; VEX-NEXT:    movq %rax, %rcx
-; VEX-NEXT:    shrq %rcx
-; VEX-NEXT:    andl $1, %eax
-; VEX-NEXT:    orq %rcx, %rax
-; VEX-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm0
-; VEX-NEXT:    vaddss %xmm0, %xmm0, %xmm0
-; VEX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
-; VEX-NEXT:    retq
+; AVX1-LABEL: uitofp_load_4i64_to_4f32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vmovapd (%rdi), %ymm0
+; AVX1-NEXT:    vmovdqa (%rdi), %xmm1
+; AVX1-NEXT:    vmovdqa 16(%rdi), %xmm2
+; AVX1-NEXT:    vpsrlq $1, %xmm1, %xmm3
+; AVX1-NEXT:    vpsrlq $1, %xmm2, %xmm4
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
+; AVX1-NEXT:    vandpd {{.*}}(%rip), %ymm0, %ymm0
+; AVX1-NEXT:    vorpd %ymm0, %ymm3, %ymm0
+; AVX1-NEXT:    vblendvpd %xmm1, %xmm0, %xmm1, %xmm3
+; AVX1-NEXT:    vpextrq $1, %xmm3, %rax
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm5, %xmm4
+; AVX1-NEXT:    vmovq %xmm3, %rax
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm5, %xmm3
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[2,3]
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vblendvpd %xmm2, %xmm0, %xmm2, %xmm0
+; AVX1-NEXT:    vmovq %xmm0, %rax
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm5, %xmm4
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm3 = xmm3[0,1],xmm4[0],xmm3[3]
+; AVX1-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm5, %xmm0
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm3[0,1,2],xmm0[0]
+; AVX1-NEXT:    vaddps %xmm0, %xmm0, %xmm3
+; AVX1-NEXT:    vxorps %xmm4, %xmm4, %xmm4
+; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm4, %xmm2
+; AVX1-NEXT:    vpackssdw %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vblendvps %xmm1, %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: uitofp_load_4i64_to_4f32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT:    vpcmpgtq %ymm0, %ymm1, %ymm1
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
+; AVX2-NEXT:    vpackssdw %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [1,1,1,1]
+; AVX2-NEXT:    vpand %ymm2, %ymm0, %ymm2
+; AVX2-NEXT:    vpsrlq $1, %ymm0, %ymm3
+; AVX2-NEXT:    vpor %ymm2, %ymm3, %ymm2
+; AVX2-NEXT:    vblendvpd %ymm0, %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX2-NEXT:    vcvtsi2ss %rax, %xmm4, %xmm2
+; AVX2-NEXT:    vmovq %xmm0, %rax
+; AVX2-NEXT:    vcvtsi2ss %rax, %xmm4, %xmm3
+; AVX2-NEXT:    vinsertps {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[2,3]
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
+; AVX2-NEXT:    vmovq %xmm0, %rax
+; AVX2-NEXT:    vcvtsi2ss %rax, %xmm4, %xmm3
+; AVX2-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1],xmm3[0],xmm2[3]
+; AVX2-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX2-NEXT:    vcvtsi2ss %rax, %xmm4, %xmm0
+; AVX2-NEXT:    vinsertps {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[0]
+; AVX2-NEXT:    vaddps %xmm0, %xmm0, %xmm2
+; AVX2-NEXT:    vblendvps %xmm1, %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: uitofp_load_4i64_to_4f32:
 ; AVX512F:       # %bb.0:
@@ -4768,7 +4605,7 @@ define <4 x float> @uitofp_load_4i32_to_4f32(<4 x i32> *%a) {
 ; SSE2-NEXT:    por {{.*}}(%rip), %xmm1
 ; SSE2-NEXT:    psrld $16, %xmm0
 ; SSE2-NEXT:    por {{.*}}(%rip), %xmm0
-; SSE2-NEXT:    addps {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    subps {{.*}}(%rip), %xmm0
 ; SSE2-NEXT:    addps %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
@@ -4779,7 +4616,7 @@ define <4 x float> @uitofp_load_4i32_to_4f32(<4 x i32> *%a) {
 ; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7]
 ; SSE41-NEXT:    psrld $16, %xmm0
 ; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0],mem[1],xmm0[2],mem[3],xmm0[4],mem[5],xmm0[6],mem[7]
-; SSE41-NEXT:    addps {{.*}}(%rip), %xmm0
+; SSE41-NEXT:    subps {{.*}}(%rip), %xmm0
 ; SSE41-NEXT:    addps %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
@@ -4789,7 +4626,7 @@ define <4 x float> @uitofp_load_4i32_to_4f32(<4 x i32> *%a) {
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],mem[1],xmm0[2],mem[3],xmm0[4],mem[5],xmm0[6],mem[7]
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm0
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],mem[1],xmm0[2],mem[3],xmm0[4],mem[5],xmm0[6],mem[7]
-; AVX1-NEXT:    vaddps {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vsubps {{.*}}(%rip), %xmm0, %xmm0
 ; AVX1-NEXT:    vaddps %xmm0, %xmm1, %xmm0
 ; AVX1-NEXT:    retq
 ;
@@ -4801,8 +4638,8 @@ define <4 x float> @uitofp_load_4i32_to_4f32(<4 x i32> *%a) {
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm0
 ; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [1392508928,1392508928,1392508928,1392508928]
 ; AVX2-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3],xmm0[4],xmm2[5],xmm0[6],xmm2[7]
-; AVX2-NEXT:    vbroadcastss {{.*#+}} xmm2 = [-5.49764202E+11,-5.49764202E+11,-5.49764202E+11,-5.49764202E+11]
-; AVX2-NEXT:    vaddps %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vbroadcastss {{.*#+}} xmm2 = [5.49764202E+11,5.49764202E+11,5.49764202E+11,5.49764202E+11]
+; AVX2-NEXT:    vsubps %xmm2, %xmm0, %xmm0
 ; AVX2-NEXT:    vaddps %xmm0, %xmm1, %xmm0
 ; AVX2-NEXT:    retq
 ;
@@ -4896,41 +4733,41 @@ define <8 x float> @uitofp_load_8i64_to_8f32(<8 x i64> *%a) {
 ; SSE2-NEXT:    movdqa 48(%rdi), %xmm1
 ; SSE2-NEXT:    movq %xmm0, %rax
 ; SSE2-NEXT:    testq %rax, %rax
-; SSE2-NEXT:    js .LBB85_1
+; SSE2-NEXT:    js .LBB87_1
 ; SSE2-NEXT:  # %bb.2:
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm3
-; SSE2-NEXT:    jmp .LBB85_3
-; SSE2-NEXT:  .LBB85_1:
+; SSE2-NEXT:    jmp .LBB87_3
+; SSE2-NEXT:  .LBB87_1:
 ; SSE2-NEXT:    movq %rax, %rcx
 ; SSE2-NEXT:    shrq %rcx
 ; SSE2-NEXT:    andl $1, %eax
 ; SSE2-NEXT:    orq %rcx, %rax
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm3
 ; SSE2-NEXT:    addss %xmm3, %xmm3
-; SSE2-NEXT:  .LBB85_3:
+; SSE2-NEXT:  .LBB87_3:
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
 ; SSE2-NEXT:    movq %xmm0, %rax
 ; SSE2-NEXT:    testq %rax, %rax
-; SSE2-NEXT:    js .LBB85_4
+; SSE2-NEXT:    js .LBB87_4
 ; SSE2-NEXT:  # %bb.5:
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm4
-; SSE2-NEXT:    jmp .LBB85_6
-; SSE2-NEXT:  .LBB85_4:
+; SSE2-NEXT:    jmp .LBB87_6
+; SSE2-NEXT:  .LBB87_4:
 ; SSE2-NEXT:    movq %rax, %rcx
 ; SSE2-NEXT:    shrq %rcx
 ; SSE2-NEXT:    andl $1, %eax
 ; SSE2-NEXT:    orq %rcx, %rax
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm4
 ; SSE2-NEXT:    addss %xmm4, %xmm4
-; SSE2-NEXT:  .LBB85_6:
+; SSE2-NEXT:  .LBB87_6:
 ; SSE2-NEXT:    movq %xmm5, %rax
 ; SSE2-NEXT:    testq %rax, %rax
-; SSE2-NEXT:    js .LBB85_7
+; SSE2-NEXT:    js .LBB87_7
 ; SSE2-NEXT:  # %bb.8:
 ; SSE2-NEXT:    xorps %xmm0, %xmm0
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm0
-; SSE2-NEXT:    jmp .LBB85_9
-; SSE2-NEXT:  .LBB85_7:
+; SSE2-NEXT:    jmp .LBB87_9
+; SSE2-NEXT:  .LBB87_7:
 ; SSE2-NEXT:    movq %rax, %rcx
 ; SSE2-NEXT:    shrq %rcx
 ; SSE2-NEXT:    andl $1, %eax
@@ -4938,30 +4775,30 @@ define <8 x float> @uitofp_load_8i64_to_8f32(<8 x i64> *%a) {
 ; SSE2-NEXT:    xorps %xmm0, %xmm0
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm0
 ; SSE2-NEXT:    addss %xmm0, %xmm0
-; SSE2-NEXT:  .LBB85_9:
+; SSE2-NEXT:  .LBB87_9:
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[2,3,0,1]
 ; SSE2-NEXT:    movq %xmm5, %rax
 ; SSE2-NEXT:    testq %rax, %rax
-; SSE2-NEXT:    js .LBB85_10
+; SSE2-NEXT:    js .LBB87_10
 ; SSE2-NEXT:  # %bb.11:
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm6
-; SSE2-NEXT:    jmp .LBB85_12
-; SSE2-NEXT:  .LBB85_10:
+; SSE2-NEXT:    jmp .LBB87_12
+; SSE2-NEXT:  .LBB87_10:
 ; SSE2-NEXT:    movq %rax, %rcx
 ; SSE2-NEXT:    shrq %rcx
 ; SSE2-NEXT:    andl $1, %eax
 ; SSE2-NEXT:    orq %rcx, %rax
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm6
 ; SSE2-NEXT:    addss %xmm6, %xmm6
-; SSE2-NEXT:  .LBB85_12:
+; SSE2-NEXT:  .LBB87_12:
 ; SSE2-NEXT:    movq %xmm1, %rax
 ; SSE2-NEXT:    testq %rax, %rax
-; SSE2-NEXT:    js .LBB85_13
+; SSE2-NEXT:    js .LBB87_13
 ; SSE2-NEXT:  # %bb.14:
 ; SSE2-NEXT:    xorps %xmm5, %xmm5
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm5
-; SSE2-NEXT:    jmp .LBB85_15
-; SSE2-NEXT:  .LBB85_13:
+; SSE2-NEXT:    jmp .LBB87_15
+; SSE2-NEXT:  .LBB87_13:
 ; SSE2-NEXT:    movq %rax, %rcx
 ; SSE2-NEXT:    shrq %rcx
 ; SSE2-NEXT:    andl $1, %eax
@@ -4969,32 +4806,32 @@ define <8 x float> @uitofp_load_8i64_to_8f32(<8 x i64> *%a) {
 ; SSE2-NEXT:    xorps %xmm5, %xmm5
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm5
 ; SSE2-NEXT:    addss %xmm5, %xmm5
-; SSE2-NEXT:  .LBB85_15:
+; SSE2-NEXT:  .LBB87_15:
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
 ; SSE2-NEXT:    movq %xmm1, %rax
 ; SSE2-NEXT:    testq %rax, %rax
-; SSE2-NEXT:    js .LBB85_16
+; SSE2-NEXT:    js .LBB87_16
 ; SSE2-NEXT:  # %bb.17:
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm7
-; SSE2-NEXT:    jmp .LBB85_18
-; SSE2-NEXT:  .LBB85_16:
+; SSE2-NEXT:    jmp .LBB87_18
+; SSE2-NEXT:  .LBB87_16:
 ; SSE2-NEXT:    movq %rax, %rcx
 ; SSE2-NEXT:    shrq %rcx
 ; SSE2-NEXT:    andl $1, %eax
 ; SSE2-NEXT:    orq %rcx, %rax
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm7
 ; SSE2-NEXT:    addss %xmm7, %xmm7
-; SSE2-NEXT:  .LBB85_18:
+; SSE2-NEXT:  .LBB87_18:
 ; SSE2-NEXT:    unpcklps {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
 ; SSE2-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm6[0],xmm0[1],xmm6[1]
 ; SSE2-NEXT:    movq %xmm2, %rax
 ; SSE2-NEXT:    testq %rax, %rax
-; SSE2-NEXT:    js .LBB85_19
+; SSE2-NEXT:    js .LBB87_19
 ; SSE2-NEXT:  # %bb.20:
 ; SSE2-NEXT:    xorps %xmm1, %xmm1
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm1
-; SSE2-NEXT:    jmp .LBB85_21
-; SSE2-NEXT:  .LBB85_19:
+; SSE2-NEXT:    jmp .LBB87_21
+; SSE2-NEXT:  .LBB87_19:
 ; SSE2-NEXT:    movq %rax, %rcx
 ; SSE2-NEXT:    shrq %rcx
 ; SSE2-NEXT:    andl $1, %eax
@@ -5002,18 +4839,18 @@ define <8 x float> @uitofp_load_8i64_to_8f32(<8 x i64> *%a) {
 ; SSE2-NEXT:    xorps %xmm1, %xmm1
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm1
 ; SSE2-NEXT:    addss %xmm1, %xmm1
-; SSE2-NEXT:  .LBB85_21:
+; SSE2-NEXT:  .LBB87_21:
 ; SSE2-NEXT:    movlhps {{.*#+}} xmm0 = xmm0[0],xmm3[0]
 ; SSE2-NEXT:    unpcklps {{.*#+}} xmm5 = xmm5[0],xmm7[0],xmm5[1],xmm7[1]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
 ; SSE2-NEXT:    movq %xmm2, %rax
 ; SSE2-NEXT:    testq %rax, %rax
-; SSE2-NEXT:    js .LBB85_22
+; SSE2-NEXT:    js .LBB87_22
 ; SSE2-NEXT:  # %bb.23:
 ; SSE2-NEXT:    xorps %xmm2, %xmm2
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm2
-; SSE2-NEXT:    jmp .LBB85_24
-; SSE2-NEXT:  .LBB85_22:
+; SSE2-NEXT:    jmp .LBB87_24
+; SSE2-NEXT:  .LBB87_22:
 ; SSE2-NEXT:    movq %rax, %rcx
 ; SSE2-NEXT:    shrq %rcx
 ; SSE2-NEXT:    andl $1, %eax
@@ -5021,277 +4858,201 @@ define <8 x float> @uitofp_load_8i64_to_8f32(<8 x i64> *%a) {
 ; SSE2-NEXT:    xorps %xmm2, %xmm2
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm2
 ; SSE2-NEXT:    addss %xmm2, %xmm2
-; SSE2-NEXT:  .LBB85_24:
+; SSE2-NEXT:  .LBB87_24:
 ; SSE2-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
 ; SSE2-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm5[0]
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: uitofp_load_8i64_to_8f32:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa (%rdi), %xmm0
-; SSE41-NEXT:    movdqa 16(%rdi), %xmm4
-; SSE41-NEXT:    movdqa 32(%rdi), %xmm1
+; SSE41-NEXT:    movdqa (%rdi), %xmm4
+; SSE41-NEXT:    movdqa 16(%rdi), %xmm5
+; SSE41-NEXT:    movdqa 32(%rdi), %xmm6
 ; SSE41-NEXT:    movdqa 48(%rdi), %xmm2
-; SSE41-NEXT:    pextrq $1, %xmm0, %rax
-; SSE41-NEXT:    testq %rax, %rax
-; SSE41-NEXT:    js .LBB85_1
-; SSE41-NEXT:  # %bb.2:
-; SSE41-NEXT:    cvtsi2ss %rax, %xmm3
-; SSE41-NEXT:    jmp .LBB85_3
-; SSE41-NEXT:  .LBB85_1:
-; SSE41-NEXT:    movq %rax, %rcx
-; SSE41-NEXT:    shrq %rcx
-; SSE41-NEXT:    andl $1, %eax
-; SSE41-NEXT:    orq %rcx, %rax
+; SSE41-NEXT:    movdqa {{.*#+}} xmm7 = [1,1]
+; SSE41-NEXT:    movdqa %xmm4, %xmm0
+; SSE41-NEXT:    pand %xmm7, %xmm0
+; SSE41-NEXT:    movdqa %xmm4, %xmm1
+; SSE41-NEXT:    psrlq $1, %xmm1
+; SSE41-NEXT:    por %xmm0, %xmm1
+; SSE41-NEXT:    movdqa %xmm4, %xmm3
+; SSE41-NEXT:    movdqa %xmm4, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm3
+; SSE41-NEXT:    pextrq $1, %xmm3, %rax
+; SSE41-NEXT:    xorps %xmm0, %xmm0
+; SSE41-NEXT:    cvtsi2ss %rax, %xmm0
+; SSE41-NEXT:    movq %xmm3, %rax
+; SSE41-NEXT:    xorps %xmm3, %xmm3
 ; SSE41-NEXT:    cvtsi2ss %rax, %xmm3
-; SSE41-NEXT:    addss %xmm3, %xmm3
-; SSE41-NEXT:  .LBB85_3:
-; SSE41-NEXT:    movq %xmm0, %rax
-; SSE41-NEXT:    testq %rax, %rax
-; SSE41-NEXT:    js .LBB85_4
-; SSE41-NEXT:  # %bb.5:
+; SSE41-NEXT:    insertps {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[2,3]
+; SSE41-NEXT:    movdqa %xmm5, %xmm0
+; SSE41-NEXT:    pand %xmm7, %xmm0
+; SSE41-NEXT:    movdqa %xmm5, %xmm1
+; SSE41-NEXT:    psrlq $1, %xmm1
+; SSE41-NEXT:    por %xmm0, %xmm1
+; SSE41-NEXT:    shufps {{.*#+}} xmm4 = xmm4[1,3],xmm5[1,3]
+; SSE41-NEXT:    movaps %xmm5, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm5
+; SSE41-NEXT:    movq %xmm5, %rax
 ; SSE41-NEXT:    xorps %xmm0, %xmm0
 ; SSE41-NEXT:    cvtsi2ss %rax, %xmm0
-; SSE41-NEXT:    jmp .LBB85_6
-; SSE41-NEXT:  .LBB85_4:
-; SSE41-NEXT:    movq %rax, %rcx
-; SSE41-NEXT:    shrq %rcx
-; SSE41-NEXT:    andl $1, %eax
-; SSE41-NEXT:    orq %rcx, %rax
+; SSE41-NEXT:    insertps {{.*#+}} xmm3 = xmm3[0,1],xmm0[0],xmm3[3]
+; SSE41-NEXT:    pextrq $1, %xmm5, %rax
 ; SSE41-NEXT:    xorps %xmm0, %xmm0
 ; SSE41-NEXT:    cvtsi2ss %rax, %xmm0
-; SSE41-NEXT:    addss %xmm0, %xmm0
-; SSE41-NEXT:  .LBB85_6:
-; SSE41-NEXT:    movq %xmm4, %rax
-; SSE41-NEXT:    testq %rax, %rax
-; SSE41-NEXT:    js .LBB85_7
-; SSE41-NEXT:  # %bb.8:
-; SSE41-NEXT:    cvtsi2ss %rax, %xmm5
-; SSE41-NEXT:    jmp .LBB85_9
-; SSE41-NEXT:  .LBB85_7:
-; SSE41-NEXT:    movq %rax, %rcx
-; SSE41-NEXT:    shrq %rcx
-; SSE41-NEXT:    andl $1, %eax
-; SSE41-NEXT:    orq %rcx, %rax
-; SSE41-NEXT:    cvtsi2ss %rax, %xmm5
-; SSE41-NEXT:    addss %xmm5, %xmm5
-; SSE41-NEXT:  .LBB85_9:
+; SSE41-NEXT:    insertps {{.*#+}} xmm3 = xmm3[0,1,2],xmm0[0]
+; SSE41-NEXT:    movaps %xmm3, %xmm1
+; SSE41-NEXT:    addps %xmm3, %xmm1
+; SSE41-NEXT:    movaps %xmm4, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm1, %xmm3
+; SSE41-NEXT:    movdqa %xmm6, %xmm0
+; SSE41-NEXT:    pand %xmm7, %xmm0
+; SSE41-NEXT:    movdqa %xmm6, %xmm1
+; SSE41-NEXT:    psrlq $1, %xmm1
+; SSE41-NEXT:    por %xmm0, %xmm1
+; SSE41-NEXT:    movdqa %xmm6, %xmm4
+; SSE41-NEXT:    movdqa %xmm6, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm4
 ; SSE41-NEXT:    pextrq $1, %xmm4, %rax
-; SSE41-NEXT:    testq %rax, %rax
-; SSE41-NEXT:    js .LBB85_10
-; SSE41-NEXT:  # %bb.11:
-; SSE41-NEXT:    xorps %xmm4, %xmm4
-; SSE41-NEXT:    cvtsi2ss %rax, %xmm4
-; SSE41-NEXT:    jmp .LBB85_12
-; SSE41-NEXT:  .LBB85_10:
-; SSE41-NEXT:    movq %rax, %rcx
-; SSE41-NEXT:    shrq %rcx
-; SSE41-NEXT:    andl $1, %eax
-; SSE41-NEXT:    orq %rcx, %rax
-; SSE41-NEXT:    xorps %xmm4, %xmm4
-; SSE41-NEXT:    cvtsi2ss %rax, %xmm4
-; SSE41-NEXT:    addss %xmm4, %xmm4
-; SSE41-NEXT:  .LBB85_12:
-; SSE41-NEXT:    pextrq $1, %xmm1, %rax
-; SSE41-NEXT:    testq %rax, %rax
-; SSE41-NEXT:    js .LBB85_13
-; SSE41-NEXT:  # %bb.14:
-; SSE41-NEXT:    cvtsi2ss %rax, %xmm6
-; SSE41-NEXT:    jmp .LBB85_15
-; SSE41-NEXT:  .LBB85_13:
-; SSE41-NEXT:    movq %rax, %rcx
-; SSE41-NEXT:    shrq %rcx
-; SSE41-NEXT:    andl $1, %eax
-; SSE41-NEXT:    orq %rcx, %rax
-; SSE41-NEXT:    cvtsi2ss %rax, %xmm6
-; SSE41-NEXT:    addss %xmm6, %xmm6
-; SSE41-NEXT:  .LBB85_15:
-; SSE41-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[2,3]
-; SSE41-NEXT:    movq %xmm1, %rax
-; SSE41-NEXT:    testq %rax, %rax
-; SSE41-NEXT:    js .LBB85_16
-; SSE41-NEXT:  # %bb.17:
-; SSE41-NEXT:    xorps %xmm1, %xmm1
-; SSE41-NEXT:    cvtsi2ss %rax, %xmm1
-; SSE41-NEXT:    jmp .LBB85_18
-; SSE41-NEXT:  .LBB85_16:
-; SSE41-NEXT:    movq %rax, %rcx
-; SSE41-NEXT:    shrq %rcx
-; SSE41-NEXT:    andl $1, %eax
-; SSE41-NEXT:    orq %rcx, %rax
+; SSE41-NEXT:    xorps %xmm0, %xmm0
+; SSE41-NEXT:    cvtsi2ss %rax, %xmm0
+; SSE41-NEXT:    movq %xmm4, %rax
 ; SSE41-NEXT:    xorps %xmm1, %xmm1
 ; SSE41-NEXT:    cvtsi2ss %rax, %xmm1
-; SSE41-NEXT:    addss %xmm1, %xmm1
-; SSE41-NEXT:  .LBB85_18:
-; SSE41-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0],xmm6[0],xmm1[2,3]
-; SSE41-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0,1],xmm5[0],xmm0[3]
+; SSE41-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[2,3]
+; SSE41-NEXT:    pand %xmm2, %xmm7
+; SSE41-NEXT:    movdqa %xmm2, %xmm4
+; SSE41-NEXT:    psrlq $1, %xmm4
+; SSE41-NEXT:    por %xmm7, %xmm4
+; SSE41-NEXT:    shufps {{.*#+}} xmm6 = xmm6[1,3],xmm2[1,3]
+; SSE41-NEXT:    movaps %xmm2, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm2
 ; SSE41-NEXT:    movq %xmm2, %rax
-; SSE41-NEXT:    testq %rax, %rax
-; SSE41-NEXT:    js .LBB85_19
-; SSE41-NEXT:  # %bb.20:
-; SSE41-NEXT:    xorps %xmm3, %xmm3
-; SSE41-NEXT:    cvtsi2ss %rax, %xmm3
-; SSE41-NEXT:    jmp .LBB85_21
-; SSE41-NEXT:  .LBB85_19:
-; SSE41-NEXT:    movq %rax, %rcx
-; SSE41-NEXT:    shrq %rcx
-; SSE41-NEXT:    andl $1, %eax
-; SSE41-NEXT:    orq %rcx, %rax
-; SSE41-NEXT:    xorps %xmm3, %xmm3
-; SSE41-NEXT:    cvtsi2ss %rax, %xmm3
-; SSE41-NEXT:    addss %xmm3, %xmm3
-; SSE41-NEXT:  .LBB85_21:
-; SSE41-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0,1],xmm3[0],xmm1[3]
-; SSE41-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm4[0]
+; SSE41-NEXT:    xorps %xmm0, %xmm0
+; SSE41-NEXT:    cvtsi2ss %rax, %xmm0
+; SSE41-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0,1],xmm0[0],xmm1[3]
 ; SSE41-NEXT:    pextrq $1, %xmm2, %rax
-; SSE41-NEXT:    testq %rax, %rax
-; SSE41-NEXT:    js .LBB85_22
-; SSE41-NEXT:  # %bb.23:
-; SSE41-NEXT:    xorps %xmm2, %xmm2
-; SSE41-NEXT:    cvtsi2ss %rax, %xmm2
-; SSE41-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm2[0]
-; SSE41-NEXT:    retq
-; SSE41-NEXT:  .LBB85_22:
-; SSE41-NEXT:    movq %rax, %rcx
-; SSE41-NEXT:    shrq %rcx
-; SSE41-NEXT:    andl $1, %eax
-; SSE41-NEXT:    orq %rcx, %rax
-; SSE41-NEXT:    xorps %xmm2, %xmm2
-; SSE41-NEXT:    cvtsi2ss %rax, %xmm2
-; SSE41-NEXT:    addss %xmm2, %xmm2
-; SSE41-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm2[0]
+; SSE41-NEXT:    xorps %xmm0, %xmm0
+; SSE41-NEXT:    cvtsi2ss %rax, %xmm0
+; SSE41-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm0[0]
+; SSE41-NEXT:    movaps %xmm1, %xmm2
+; SSE41-NEXT:    addps %xmm1, %xmm2
+; SSE41-NEXT:    movaps %xmm6, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm2, %xmm1
+; SSE41-NEXT:    movaps %xmm3, %xmm0
 ; SSE41-NEXT:    retq
 ;
-; VEX-LABEL: uitofp_load_8i64_to_8f32:
-; VEX:       # %bb.0:
-; VEX-NEXT:    vmovdqa (%rdi), %xmm1
-; VEX-NEXT:    vmovdqa 16(%rdi), %xmm0
-; VEX-NEXT:    vmovdqa 32(%rdi), %xmm4
-; VEX-NEXT:    vmovdqa 48(%rdi), %xmm3
-; VEX-NEXT:    vpextrq $1, %xmm4, %rax
-; VEX-NEXT:    testq %rax, %rax
-; VEX-NEXT:    js .LBB85_1
-; VEX-NEXT:  # %bb.2:
-; VEX-NEXT:    vcvtsi2ss %rax, %xmm2, %xmm2
-; VEX-NEXT:    jmp .LBB85_3
-; VEX-NEXT:  .LBB85_1:
-; VEX-NEXT:    movq %rax, %rcx
-; VEX-NEXT:    shrq %rcx
-; VEX-NEXT:    andl $1, %eax
-; VEX-NEXT:    orq %rcx, %rax
-; VEX-NEXT:    vcvtsi2ss %rax, %xmm2, %xmm2
-; VEX-NEXT:    vaddss %xmm2, %xmm2, %xmm2
-; VEX-NEXT:  .LBB85_3:
-; VEX-NEXT:    vmovq %xmm4, %rax
-; VEX-NEXT:    testq %rax, %rax
-; VEX-NEXT:    js .LBB85_4
-; VEX-NEXT:  # %bb.5:
-; VEX-NEXT:    vcvtsi2ss %rax, %xmm5, %xmm5
-; VEX-NEXT:    jmp .LBB85_6
-; VEX-NEXT:  .LBB85_4:
-; VEX-NEXT:    movq %rax, %rcx
-; VEX-NEXT:    shrq %rcx
-; VEX-NEXT:    andl $1, %eax
-; VEX-NEXT:    orq %rcx, %rax
-; VEX-NEXT:    vcvtsi2ss %rax, %xmm5, %xmm4
-; VEX-NEXT:    vaddss %xmm4, %xmm4, %xmm5
-; VEX-NEXT:  .LBB85_6:
-; VEX-NEXT:    vmovq %xmm3, %rax
-; VEX-NEXT:    testq %rax, %rax
-; VEX-NEXT:    js .LBB85_7
-; VEX-NEXT:  # %bb.8:
-; VEX-NEXT:    vcvtsi2ss %rax, %xmm6, %xmm4
-; VEX-NEXT:    jmp .LBB85_9
-; VEX-NEXT:  .LBB85_7:
-; VEX-NEXT:    movq %rax, %rcx
-; VEX-NEXT:    shrq %rcx
-; VEX-NEXT:    andl $1, %eax
-; VEX-NEXT:    orq %rcx, %rax
-; VEX-NEXT:    vcvtsi2ss %rax, %xmm6, %xmm4
-; VEX-NEXT:    vaddss %xmm4, %xmm4, %xmm4
-; VEX-NEXT:  .LBB85_9:
-; VEX-NEXT:    vpextrq $1, %xmm3, %rax
-; VEX-NEXT:    testq %rax, %rax
-; VEX-NEXT:    js .LBB85_10
-; VEX-NEXT:  # %bb.11:
-; VEX-NEXT:    vcvtsi2ss %rax, %xmm6, %xmm3
-; VEX-NEXT:    jmp .LBB85_12
-; VEX-NEXT:  .LBB85_10:
-; VEX-NEXT:    movq %rax, %rcx
-; VEX-NEXT:    shrq %rcx
-; VEX-NEXT:    andl $1, %eax
-; VEX-NEXT:    orq %rcx, %rax
-; VEX-NEXT:    vcvtsi2ss %rax, %xmm6, %xmm3
-; VEX-NEXT:    vaddss %xmm3, %xmm3, %xmm3
-; VEX-NEXT:  .LBB85_12:
-; VEX-NEXT:    vpextrq $1, %xmm1, %rax
-; VEX-NEXT:    testq %rax, %rax
-; VEX-NEXT:    js .LBB85_13
-; VEX-NEXT:  # %bb.14:
-; VEX-NEXT:    vcvtsi2ss %rax, %xmm6, %xmm6
-; VEX-NEXT:    jmp .LBB85_15
-; VEX-NEXT:  .LBB85_13:
-; VEX-NEXT:    movq %rax, %rcx
-; VEX-NEXT:    shrq %rcx
-; VEX-NEXT:    andl $1, %eax
-; VEX-NEXT:    orq %rcx, %rax
-; VEX-NEXT:    vcvtsi2ss %rax, %xmm6, %xmm6
-; VEX-NEXT:    vaddss %xmm6, %xmm6, %xmm6
-; VEX-NEXT:  .LBB85_15:
-; VEX-NEXT:    vinsertps {{.*#+}} xmm2 = xmm5[0],xmm2[0],xmm5[2,3]
-; VEX-NEXT:    vmovq %xmm1, %rax
-; VEX-NEXT:    testq %rax, %rax
-; VEX-NEXT:    js .LBB85_16
-; VEX-NEXT:  # %bb.17:
-; VEX-NEXT:    vcvtsi2ss %rax, %xmm7, %xmm1
-; VEX-NEXT:    jmp .LBB85_18
-; VEX-NEXT:  .LBB85_16:
-; VEX-NEXT:    movq %rax, %rcx
-; VEX-NEXT:    shrq %rcx
-; VEX-NEXT:    andl $1, %eax
-; VEX-NEXT:    orq %rcx, %rax
-; VEX-NEXT:    vcvtsi2ss %rax, %xmm7, %xmm1
-; VEX-NEXT:    vaddss %xmm1, %xmm1, %xmm1
-; VEX-NEXT:  .LBB85_18:
-; VEX-NEXT:    vinsertps {{.*#+}} xmm5 = xmm1[0],xmm6[0],xmm1[2,3]
-; VEX-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0,1],xmm4[0],xmm2[3]
-; VEX-NEXT:    vmovq %xmm0, %rax
-; VEX-NEXT:    testq %rax, %rax
-; VEX-NEXT:    js .LBB85_19
-; VEX-NEXT:  # %bb.20:
-; VEX-NEXT:    vcvtsi2ss %rax, %xmm7, %xmm2
-; VEX-NEXT:    jmp .LBB85_21
-; VEX-NEXT:  .LBB85_19:
-; VEX-NEXT:    movq %rax, %rcx
-; VEX-NEXT:    shrq %rcx
-; VEX-NEXT:    andl $1, %eax
-; VEX-NEXT:    orq %rcx, %rax
-; VEX-NEXT:    vcvtsi2ss %rax, %xmm7, %xmm2
-; VEX-NEXT:    vaddss %xmm2, %xmm2, %xmm2
-; VEX-NEXT:  .LBB85_21:
-; VEX-NEXT:    vinsertps {{.*#+}} xmm2 = xmm5[0,1],xmm2[0],xmm5[3]
-; VEX-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm3[0]
-; VEX-NEXT:    vpextrq $1, %xmm0, %rax
-; VEX-NEXT:    testq %rax, %rax
-; VEX-NEXT:    js .LBB85_22
-; VEX-NEXT:  # %bb.23:
-; VEX-NEXT:    vcvtsi2ss %rax, %xmm7, %xmm0
-; VEX-NEXT:    jmp .LBB85_24
-; VEX-NEXT:  .LBB85_22:
-; VEX-NEXT:    movq %rax, %rcx
-; VEX-NEXT:    shrq %rcx
-; VEX-NEXT:    andl $1, %eax
-; VEX-NEXT:    orq %rcx, %rax
-; VEX-NEXT:    vcvtsi2ss %rax, %xmm7, %xmm0
-; VEX-NEXT:    vaddss %xmm0, %xmm0, %xmm0
-; VEX-NEXT:  .LBB85_24:
-; VEX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[0]
-; VEX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; VEX-NEXT:    retq
+; AVX1-LABEL: uitofp_load_8i64_to_8f32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vmovapd {{.*#+}} ymm2 = [1,1,1,1]
+; AVX1-NEXT:    vandpd 32(%rdi), %ymm2, %ymm3
+; AVX1-NEXT:    vmovaps (%rdi), %xmm0
+; AVX1-NEXT:    vmovdqa 16(%rdi), %xmm1
+; AVX1-NEXT:    vmovdqa 32(%rdi), %xmm4
+; AVX1-NEXT:    vmovdqa 48(%rdi), %xmm5
+; AVX1-NEXT:    vpsrlq $1, %xmm4, %xmm6
+; AVX1-NEXT:    vpsrlq $1, %xmm5, %xmm7
+; AVX1-NEXT:    vinsertf128 $1, %xmm7, %ymm6, %ymm6
+; AVX1-NEXT:    vorpd %ymm3, %ymm6, %ymm3
+; AVX1-NEXT:    vblendvpd %xmm4, %xmm3, %xmm4, %xmm6
+; AVX1-NEXT:    vpextrq $1, %xmm6, %rax
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm8, %xmm7
+; AVX1-NEXT:    vmovq %xmm6, %rax
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm8, %xmm6
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[2,3]
+; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm3
+; AVX1-NEXT:    vblendvpd %xmm5, %xmm3, %xmm5, %xmm3
+; AVX1-NEXT:    vmovq %xmm3, %rax
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm8, %xmm7
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm6 = xmm6[0,1],xmm7[0],xmm6[3]
+; AVX1-NEXT:    vpextrq $1, %xmm3, %rax
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm8, %xmm3
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm3 = xmm6[0,1,2],xmm3[0]
+; AVX1-NEXT:    vaddps %xmm3, %xmm3, %xmm6
+; AVX1-NEXT:    vxorps %xmm7, %xmm7, %xmm7
+; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm7, %xmm5
+; AVX1-NEXT:    vpackssdw %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vblendvps %xmm4, %xmm6, %xmm3, %xmm3
+; AVX1-NEXT:    vandpd (%rdi), %ymm2, %ymm2
+; AVX1-NEXT:    vpsrlq $1, %xmm0, %xmm4
+; AVX1-NEXT:    vpsrlq $1, %xmm1, %xmm5
+; AVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm4, %ymm4
+; AVX1-NEXT:    vorpd %ymm2, %ymm4, %ymm2
+; AVX1-NEXT:    vblendvpd %xmm0, %xmm2, %xmm0, %xmm4
+; AVX1-NEXT:    vpextrq $1, %xmm4, %rax
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm8, %xmm5
+; AVX1-NEXT:    vmovq %xmm4, %rax
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm8, %xmm4
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[2,3]
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm2
+; AVX1-NEXT:    vblendvpd %xmm1, %xmm2, %xmm1, %xmm2
+; AVX1-NEXT:    vmovq %xmm2, %rax
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm8, %xmm5
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm4 = xmm4[0,1],xmm5[0],xmm4[3]
+; AVX1-NEXT:    vpextrq $1, %xmm2, %rax
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm8, %xmm2
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm2 = xmm4[0,1,2],xmm2[0]
+; AVX1-NEXT:    vaddps %xmm2, %xmm2, %xmm4
+; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm7, %xmm1
+; AVX1-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vblendvps %xmm0, %xmm4, %xmm2, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: uitofp_load_8i64_to_8f32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vmovaps (%rdi), %ymm0
+; AVX2-NEXT:    vmovdqa 32(%rdi), %ymm1
+; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpcmpgtq %ymm1, %ymm2, %ymm3
+; AVX2-NEXT:    vextracti128 $1, %ymm3, %xmm4
+; AVX2-NEXT:    vpackssdw %xmm4, %xmm3, %xmm3
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [1,1,1,1]
+; AVX2-NEXT:    vpand %ymm4, %ymm1, %ymm5
+; AVX2-NEXT:    vpsrlq $1, %ymm1, %ymm6
+; AVX2-NEXT:    vpor %ymm5, %ymm6, %ymm5
+; AVX2-NEXT:    vblendvpd %ymm1, %ymm5, %ymm1, %ymm1
+; AVX2-NEXT:    vpextrq $1, %xmm1, %rax
+; AVX2-NEXT:    vcvtsi2ss %rax, %xmm7, %xmm5
+; AVX2-NEXT:    vmovq %xmm1, %rax
+; AVX2-NEXT:    vcvtsi2ss %rax, %xmm7, %xmm6
+; AVX2-NEXT:    vinsertps {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[2,3]
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm1
+; AVX2-NEXT:    vmovq %xmm1, %rax
+; AVX2-NEXT:    vcvtsi2ss %rax, %xmm7, %xmm6
+; AVX2-NEXT:    vinsertps {{.*#+}} xmm5 = xmm5[0,1],xmm6[0],xmm5[3]
+; AVX2-NEXT:    vpextrq $1, %xmm1, %rax
+; AVX2-NEXT:    vcvtsi2ss %rax, %xmm7, %xmm1
+; AVX2-NEXT:    vinsertps {{.*#+}} xmm1 = xmm5[0,1,2],xmm1[0]
+; AVX2-NEXT:    vaddps %xmm1, %xmm1, %xmm5
+; AVX2-NEXT:    vblendvps %xmm3, %xmm5, %xmm1, %xmm1
+; AVX2-NEXT:    vpcmpgtq %ymm0, %ymm2, %ymm2
+; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm3
+; AVX2-NEXT:    vpackssdw %xmm3, %xmm2, %xmm2
+; AVX2-NEXT:    vpand %ymm4, %ymm0, %ymm3
+; AVX2-NEXT:    vpsrlq $1, %ymm0, %ymm4
+; AVX2-NEXT:    vpor %ymm3, %ymm4, %ymm3
+; AVX2-NEXT:    vblendvpd %ymm0, %ymm3, %ymm0, %ymm0
+; AVX2-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX2-NEXT:    vcvtsi2ss %rax, %xmm7, %xmm3
+; AVX2-NEXT:    vmovq %xmm0, %rax
+; AVX2-NEXT:    vcvtsi2ss %rax, %xmm7, %xmm4
+; AVX2-NEXT:    vinsertps {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[2,3]
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
+; AVX2-NEXT:    vmovq %xmm0, %rax
+; AVX2-NEXT:    vcvtsi2ss %rax, %xmm7, %xmm4
+; AVX2-NEXT:    vinsertps {{.*#+}} xmm3 = xmm3[0,1],xmm4[0],xmm3[3]
+; AVX2-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX2-NEXT:    vcvtsi2ss %rax, %xmm7, %xmm0
+; AVX2-NEXT:    vinsertps {{.*#+}} xmm0 = xmm3[0,1,2],xmm0[0]
+; AVX2-NEXT:    vaddps %xmm0, %xmm0, %xmm3
+; AVX2-NEXT:    vblendvps %xmm2, %xmm3, %xmm0, %xmm0
+; AVX2-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: uitofp_load_8i64_to_8f32:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512F-NEXT:    vmovaps (%rdi), %xmm0
 ; AVX512F-NEXT:    vmovdqa 16(%rdi), %xmm1
 ; AVX512F-NEXT:    vmovdqa 32(%rdi), %xmm2
 ; AVX512F-NEXT:    vmovdqa 48(%rdi), %xmm3
@@ -5322,7 +5083,7 @@ define <8 x float> @uitofp_load_8i64_to_8f32(<8 x i64> *%a) {
 ;
 ; AVX512VL-LABEL: uitofp_load_8i64_to_8f32:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512VL-NEXT:    vmovaps (%rdi), %xmm0
 ; AVX512VL-NEXT:    vmovdqa 16(%rdi), %xmm1
 ; AVX512VL-NEXT:    vmovdqa 32(%rdi), %xmm2
 ; AVX512VL-NEXT:    vmovdqa 48(%rdi), %xmm3
@@ -5378,14 +5139,14 @@ define <8 x float> @uitofp_load_8i32_to_8f32(<8 x i32> *%a) {
 ; SSE2-NEXT:    psrld $16, %xmm0
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [1392508928,1392508928,1392508928,1392508928]
 ; SSE2-NEXT:    por %xmm5, %xmm0
-; SSE2-NEXT:    movaps {{.*#+}} xmm6 = [-5.49764202E+11,-5.49764202E+11,-5.49764202E+11,-5.49764202E+11]
-; SSE2-NEXT:    addps %xmm6, %xmm0
+; SSE2-NEXT:    movaps {{.*#+}} xmm6 = [5.49764202E+11,5.49764202E+11,5.49764202E+11,5.49764202E+11]
+; SSE2-NEXT:    subps %xmm6, %xmm0
 ; SSE2-NEXT:    addps %xmm3, %xmm0
 ; SSE2-NEXT:    pand %xmm1, %xmm2
 ; SSE2-NEXT:    por %xmm4, %xmm2
 ; SSE2-NEXT:    psrld $16, %xmm1
 ; SSE2-NEXT:    por %xmm5, %xmm1
-; SSE2-NEXT:    addps %xmm6, %xmm1
+; SSE2-NEXT:    subps %xmm6, %xmm1
 ; SSE2-NEXT:    addps %xmm2, %xmm1
 ; SSE2-NEXT:    retq
 ;
@@ -5399,13 +5160,13 @@ define <8 x float> @uitofp_load_8i32_to_8f32(<8 x i32> *%a) {
 ; SSE41-NEXT:    psrld $16, %xmm0
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [1392508928,1392508928,1392508928,1392508928]
 ; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0],xmm4[1],xmm0[2],xmm4[3],xmm0[4],xmm4[5],xmm0[6],xmm4[7]
-; SSE41-NEXT:    movaps {{.*#+}} xmm5 = [-5.49764202E+11,-5.49764202E+11,-5.49764202E+11,-5.49764202E+11]
-; SSE41-NEXT:    addps %xmm5, %xmm0
+; SSE41-NEXT:    movaps {{.*#+}} xmm5 = [5.49764202E+11,5.49764202E+11,5.49764202E+11,5.49764202E+11]
+; SSE41-NEXT:    subps %xmm5, %xmm0
 ; SSE41-NEXT:    addps %xmm3, %xmm0
 ; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
 ; SSE41-NEXT:    psrld $16, %xmm1
 ; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0],xmm4[1],xmm1[2],xmm4[3],xmm1[4],xmm4[5],xmm1[6],xmm4[7]
-; SSE41-NEXT:    addps %xmm5, %xmm1
+; SSE41-NEXT:    subps %xmm5, %xmm1
 ; SSE41-NEXT:    addps %xmm2, %xmm1
 ; SSE41-NEXT:    retq
 ;
@@ -5432,8 +5193,8 @@ define <8 x float> @uitofp_load_8i32_to_8f32(<8 x i32> *%a) {
 ; AVX2-NEXT:    vpsrld $16, %ymm0, %ymm0
 ; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm2 = [1392508928,1392508928,1392508928,1392508928,1392508928,1392508928,1392508928,1392508928]
 ; AVX2-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm2[1],ymm0[2],ymm2[3],ymm0[4],ymm2[5],ymm0[6],ymm2[7],ymm0[8],ymm2[9],ymm0[10],ymm2[11],ymm0[12],ymm2[13],ymm0[14],ymm2[15]
-; AVX2-NEXT:    vbroadcastss {{.*#+}} ymm2 = [-5.49764202E+11,-5.49764202E+11,-5.49764202E+11,-5.49764202E+11,-5.49764202E+11,-5.49764202E+11,-5.49764202E+11,-5.49764202E+11]
-; AVX2-NEXT:    vaddps %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vbroadcastss {{.*#+}} ymm2 = [5.49764202E+11,5.49764202E+11,5.49764202E+11,5.49764202E+11,5.49764202E+11,5.49764202E+11,5.49764202E+11,5.49764202E+11]
+; AVX2-NEXT:    vsubps %ymm2, %ymm0, %ymm0
 ; AVX2-NEXT:    vaddps %ymm0, %ymm1, %ymm0
 ; AVX2-NEXT:    retq
 ;
@@ -5577,14 +5338,12 @@ define void @aggregate_sitofp_8i16_to_8f32(%Arguments* nocapture readonly %a0) {
 ; SSE41-LABEL: aggregate_sitofp_8i16_to_8f32:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movq 24(%rdi), %rax
-; SSE41-NEXT:    movdqu 8(%rdi), %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; SSE41-NEXT:    pmovsxwd %xmm1, %xmm1
+; SSE41-NEXT:    pmovsxwd 16(%rdi), %xmm0
+; SSE41-NEXT:    pmovsxwd 8(%rdi), %xmm1
 ; SSE41-NEXT:    cvtdq2ps %xmm1, %xmm1
-; SSE41-NEXT:    pmovsxwd %xmm0, %xmm0
 ; SSE41-NEXT:    cvtdq2ps %xmm0, %xmm0
-; SSE41-NEXT:    movaps %xmm0, (%rax)
-; SSE41-NEXT:    movaps %xmm1, 16(%rax)
+; SSE41-NEXT:    movaps %xmm0, 16(%rax)
+; SSE41-NEXT:    movaps %xmm1, (%rax)
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: aggregate_sitofp_8i16_to_8f32:
diff --git a/llvm/test/CodeGen/X86/vec_minmax_sint.ll b/llvm/test/CodeGen/X86/vec_minmax_sint.ll
index a6afe94f0..143025d71 100644
--- a/llvm/test/CodeGen/X86/vec_minmax_sint.ll
+++ b/llvm/test/CodeGen/X86/vec_minmax_sint.ll
@@ -34,17 +34,16 @@ define <2 x i64> @max_gt_v2i64(<2 x i64> %a, <2 x i64> %b) {
 ; SSE41-LABEL: max_gt_v2i64:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483648,2147483648]
-; SSE41-NEXT:    movdqa %xmm1, %xmm3
-; SSE41-NEXT:    pxor %xmm0, %xmm3
-; SSE41-NEXT:    pxor %xmm2, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [2147483648,2147483648]
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pxor %xmm3, %xmm0
+; SSE41-NEXT:    pxor %xmm2, %xmm3
+; SSE41-NEXT:    movdqa %xmm3, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm3, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
 ; SSE41-NEXT:    movapd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
@@ -126,23 +125,21 @@ define <4 x i64> @max_gt_v4i64(<4 x i64> %a, <4 x i64> %b) {
 ; SSE41-NEXT:    movdqa %xmm0, %xmm7
 ; SSE41-NEXT:    pxor %xmm5, %xmm7
 ; SSE41-NEXT:    movdqa %xmm7, %xmm0
-; SSE41-NEXT:    pcmpgtd %xmm6, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm8 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm6, %xmm7
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm7[1,1,3,3]
-; SSE41-NEXT:    pand %xmm8, %xmm6
-; SSE41-NEXT:    por %xmm6, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm6, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm6, %xmm7
+; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm7[0,0,2,2]
+; SSE41-NEXT:    pand %xmm6, %xmm0
+; SSE41-NEXT:    por %xmm7, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm2
 ; SSE41-NEXT:    movdqa %xmm3, %xmm0
 ; SSE41-NEXT:    pxor %xmm5, %xmm0
 ; SSE41-NEXT:    pxor %xmm1, %xmm5
 ; SSE41-NEXT:    movdqa %xmm5, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[1,1,3,3]
-; SSE41-NEXT:    pand %xmm6, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm5
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm5, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm3
 ; SSE41-NEXT:    movapd %xmm2, %xmm0
 ; SSE41-NEXT:    movapd %xmm3, %xmm1
@@ -424,17 +421,16 @@ define <2 x i64> @max_ge_v2i64(<2 x i64> %a, <2 x i64> %b) {
 ; SSE41-LABEL: max_ge_v2i64:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483648,2147483648]
-; SSE41-NEXT:    movdqa %xmm1, %xmm3
-; SSE41-NEXT:    pxor %xmm0, %xmm3
-; SSE41-NEXT:    pxor %xmm2, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [2147483648,2147483648]
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pxor %xmm3, %xmm0
+; SSE41-NEXT:    pxor %xmm2, %xmm3
+; SSE41-NEXT:    movdqa %xmm3, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm3, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
 ; SSE41-NEXT:    movapd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
@@ -516,23 +512,21 @@ define <4 x i64> @max_ge_v4i64(<4 x i64> %a, <4 x i64> %b) {
 ; SSE41-NEXT:    movdqa %xmm0, %xmm7
 ; SSE41-NEXT:    pxor %xmm5, %xmm7
 ; SSE41-NEXT:    movdqa %xmm7, %xmm0
-; SSE41-NEXT:    pcmpgtd %xmm6, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm8 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm6, %xmm7
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm7[1,1,3,3]
-; SSE41-NEXT:    pand %xmm8, %xmm6
-; SSE41-NEXT:    por %xmm6, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm6, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm6, %xmm7
+; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm7[0,0,2,2]
+; SSE41-NEXT:    pand %xmm6, %xmm0
+; SSE41-NEXT:    por %xmm7, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm2
 ; SSE41-NEXT:    movdqa %xmm3, %xmm0
 ; SSE41-NEXT:    pxor %xmm5, %xmm0
 ; SSE41-NEXT:    pxor %xmm1, %xmm5
 ; SSE41-NEXT:    movdqa %xmm5, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[1,1,3,3]
-; SSE41-NEXT:    pand %xmm6, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm5
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm5, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm3
 ; SSE41-NEXT:    movapd %xmm2, %xmm0
 ; SSE41-NEXT:    movapd %xmm3, %xmm1
@@ -814,17 +808,15 @@ define <2 x i64> @min_lt_v2i64(<2 x i64> %a, <2 x i64> %b) {
 ; SSE41-LABEL: min_lt_v2i64:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483648,2147483648]
-; SSE41-NEXT:    movdqa %xmm2, %xmm3
-; SSE41-NEXT:    pxor %xmm0, %xmm3
-; SSE41-NEXT:    pxor %xmm1, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [2147483648,2147483648]
+; SSE41-NEXT:    pxor %xmm3, %xmm0
+; SSE41-NEXT:    pxor %xmm1, %xmm3
+; SSE41-NEXT:    movdqa %xmm3, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm3, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
 ; SSE41-NEXT:    movapd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
@@ -906,23 +898,21 @@ define <4 x i64> @min_lt_v4i64(<4 x i64> %a, <4 x i64> %b) {
 ; SSE41-NEXT:    movdqa %xmm2, %xmm6
 ; SSE41-NEXT:    pxor %xmm5, %xmm6
 ; SSE41-NEXT:    movdqa %xmm6, %xmm7
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm7
-; SSE41-NEXT:    pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3]
-; SSE41-NEXT:    pand %xmm8, %xmm0
-; SSE41-NEXT:    por %xmm7, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm7
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
+; SSE41-NEXT:    pand %xmm7, %xmm0
+; SSE41-NEXT:    por %xmm6, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm2
 ; SSE41-NEXT:    movdqa %xmm1, %xmm0
 ; SSE41-NEXT:    pxor %xmm5, %xmm0
 ; SSE41-NEXT:    pxor %xmm3, %xmm5
 ; SSE41-NEXT:    movdqa %xmm5, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[1,1,3,3]
-; SSE41-NEXT:    pand %xmm6, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm5
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm5, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm3
 ; SSE41-NEXT:    movapd %xmm2, %xmm0
 ; SSE41-NEXT:    movapd %xmm3, %xmm1
@@ -1199,17 +1189,15 @@ define <2 x i64> @min_le_v2i64(<2 x i64> %a, <2 x i64> %b) {
 ; SSE41-LABEL: min_le_v2i64:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483648,2147483648]
-; SSE41-NEXT:    movdqa %xmm2, %xmm3
-; SSE41-NEXT:    pxor %xmm0, %xmm3
-; SSE41-NEXT:    pxor %xmm1, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [2147483648,2147483648]
+; SSE41-NEXT:    pxor %xmm3, %xmm0
+; SSE41-NEXT:    pxor %xmm1, %xmm3
+; SSE41-NEXT:    movdqa %xmm3, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm3, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
 ; SSE41-NEXT:    movapd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
@@ -1291,23 +1279,21 @@ define <4 x i64> @min_le_v4i64(<4 x i64> %a, <4 x i64> %b) {
 ; SSE41-NEXT:    movdqa %xmm2, %xmm6
 ; SSE41-NEXT:    pxor %xmm5, %xmm6
 ; SSE41-NEXT:    movdqa %xmm6, %xmm7
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm7
-; SSE41-NEXT:    pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3]
-; SSE41-NEXT:    pand %xmm8, %xmm0
-; SSE41-NEXT:    por %xmm7, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm7
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
+; SSE41-NEXT:    pand %xmm7, %xmm0
+; SSE41-NEXT:    por %xmm6, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm2
 ; SSE41-NEXT:    movdqa %xmm1, %xmm0
 ; SSE41-NEXT:    pxor %xmm5, %xmm0
 ; SSE41-NEXT:    pxor %xmm3, %xmm5
 ; SSE41-NEXT:    movdqa %xmm5, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[1,1,3,3]
-; SSE41-NEXT:    pand %xmm6, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm5
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm5, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm3
 ; SSE41-NEXT:    movapd %xmm2, %xmm0
 ; SSE41-NEXT:    movapd %xmm3, %xmm1
diff --git a/llvm/test/CodeGen/X86/vec_minmax_uint.ll b/llvm/test/CodeGen/X86/vec_minmax_uint.ll
index 4f8477a73..beb690600 100644
--- a/llvm/test/CodeGen/X86/vec_minmax_uint.ll
+++ b/llvm/test/CodeGen/X86/vec_minmax_uint.ll
@@ -34,17 +34,16 @@ define <2 x i64> @max_gt_v2i64(<2 x i64> %a, <2 x i64> %b) {
 ; SSE41-LABEL: max_gt_v2i64:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [9223372039002259456,9223372039002259456]
-; SSE41-NEXT:    movdqa %xmm1, %xmm3
-; SSE41-NEXT:    pxor %xmm0, %xmm3
-; SSE41-NEXT:    pxor %xmm2, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [9223372039002259456,9223372039002259456]
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pxor %xmm3, %xmm0
+; SSE41-NEXT:    pxor %xmm2, %xmm3
+; SSE41-NEXT:    movdqa %xmm3, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm3, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
 ; SSE41-NEXT:    movapd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
@@ -136,23 +135,21 @@ define <4 x i64> @max_gt_v4i64(<4 x i64> %a, <4 x i64> %b) {
 ; SSE41-NEXT:    movdqa %xmm0, %xmm7
 ; SSE41-NEXT:    pxor %xmm5, %xmm7
 ; SSE41-NEXT:    movdqa %xmm7, %xmm0
-; SSE41-NEXT:    pcmpgtd %xmm6, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm8 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm6, %xmm7
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm7[1,1,3,3]
-; SSE41-NEXT:    pand %xmm8, %xmm6
-; SSE41-NEXT:    por %xmm6, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm6, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm6, %xmm7
+; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm7[0,0,2,2]
+; SSE41-NEXT:    pand %xmm6, %xmm0
+; SSE41-NEXT:    por %xmm7, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm2
 ; SSE41-NEXT:    movdqa %xmm3, %xmm0
 ; SSE41-NEXT:    pxor %xmm5, %xmm0
 ; SSE41-NEXT:    pxor %xmm1, %xmm5
 ; SSE41-NEXT:    movdqa %xmm5, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[1,1,3,3]
-; SSE41-NEXT:    pand %xmm6, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm5
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm5, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm3
 ; SSE41-NEXT:    movapd %xmm2, %xmm0
 ; SSE41-NEXT:    movapd %xmm3, %xmm1
@@ -452,17 +449,16 @@ define <2 x i64> @max_ge_v2i64(<2 x i64> %a, <2 x i64> %b) {
 ; SSE41-LABEL: max_ge_v2i64:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [9223372039002259456,9223372039002259456]
-; SSE41-NEXT:    movdqa %xmm1, %xmm3
-; SSE41-NEXT:    pxor %xmm0, %xmm3
-; SSE41-NEXT:    pxor %xmm2, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [9223372039002259456,9223372039002259456]
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pxor %xmm3, %xmm0
+; SSE41-NEXT:    pxor %xmm2, %xmm3
+; SSE41-NEXT:    movdqa %xmm3, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm3, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
 ; SSE41-NEXT:    movapd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
@@ -554,23 +550,21 @@ define <4 x i64> @max_ge_v4i64(<4 x i64> %a, <4 x i64> %b) {
 ; SSE41-NEXT:    movdqa %xmm0, %xmm7
 ; SSE41-NEXT:    pxor %xmm5, %xmm7
 ; SSE41-NEXT:    movdqa %xmm7, %xmm0
-; SSE41-NEXT:    pcmpgtd %xmm6, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm8 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm6, %xmm7
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm7[1,1,3,3]
-; SSE41-NEXT:    pand %xmm8, %xmm6
-; SSE41-NEXT:    por %xmm6, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm6, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm6, %xmm7
+; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm7[0,0,2,2]
+; SSE41-NEXT:    pand %xmm6, %xmm0
+; SSE41-NEXT:    por %xmm7, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm2
 ; SSE41-NEXT:    movdqa %xmm3, %xmm0
 ; SSE41-NEXT:    pxor %xmm5, %xmm0
 ; SSE41-NEXT:    pxor %xmm1, %xmm5
 ; SSE41-NEXT:    movdqa %xmm5, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[1,1,3,3]
-; SSE41-NEXT:    pand %xmm6, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm5
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm5, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm3
 ; SSE41-NEXT:    movapd %xmm2, %xmm0
 ; SSE41-NEXT:    movapd %xmm3, %xmm1
@@ -870,17 +864,15 @@ define <2 x i64> @min_lt_v2i64(<2 x i64> %a, <2 x i64> %b) {
 ; SSE41-LABEL: min_lt_v2i64:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [9223372039002259456,9223372039002259456]
-; SSE41-NEXT:    movdqa %xmm2, %xmm3
-; SSE41-NEXT:    pxor %xmm0, %xmm3
-; SSE41-NEXT:    pxor %xmm1, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [9223372039002259456,9223372039002259456]
+; SSE41-NEXT:    pxor %xmm3, %xmm0
+; SSE41-NEXT:    pxor %xmm1, %xmm3
+; SSE41-NEXT:    movdqa %xmm3, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm3, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
 ; SSE41-NEXT:    movapd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
@@ -971,23 +963,21 @@ define <4 x i64> @min_lt_v4i64(<4 x i64> %a, <4 x i64> %b) {
 ; SSE41-NEXT:    movdqa %xmm2, %xmm6
 ; SSE41-NEXT:    pxor %xmm5, %xmm6
 ; SSE41-NEXT:    movdqa %xmm6, %xmm7
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm7
-; SSE41-NEXT:    pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3]
-; SSE41-NEXT:    pand %xmm8, %xmm0
-; SSE41-NEXT:    por %xmm7, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm7
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
+; SSE41-NEXT:    pand %xmm7, %xmm0
+; SSE41-NEXT:    por %xmm6, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm2
 ; SSE41-NEXT:    movdqa %xmm1, %xmm0
 ; SSE41-NEXT:    pxor %xmm5, %xmm0
 ; SSE41-NEXT:    pxor %xmm3, %xmm5
 ; SSE41-NEXT:    movdqa %xmm5, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[1,1,3,3]
-; SSE41-NEXT:    pand %xmm6, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm5
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm5, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm3
 ; SSE41-NEXT:    movapd %xmm2, %xmm0
 ; SSE41-NEXT:    movapd %xmm3, %xmm1
@@ -1287,17 +1277,15 @@ define <2 x i64> @min_le_v2i64(<2 x i64> %a, <2 x i64> %b) {
 ; SSE41-LABEL: min_le_v2i64:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [9223372039002259456,9223372039002259456]
-; SSE41-NEXT:    movdqa %xmm2, %xmm3
-; SSE41-NEXT:    pxor %xmm0, %xmm3
-; SSE41-NEXT:    pxor %xmm1, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [9223372039002259456,9223372039002259456]
+; SSE41-NEXT:    pxor %xmm3, %xmm0
+; SSE41-NEXT:    pxor %xmm1, %xmm3
+; SSE41-NEXT:    movdqa %xmm3, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm3, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
 ; SSE41-NEXT:    movapd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
@@ -1388,23 +1376,21 @@ define <4 x i64> @min_le_v4i64(<4 x i64> %a, <4 x i64> %b) {
 ; SSE41-NEXT:    movdqa %xmm2, %xmm6
 ; SSE41-NEXT:    pxor %xmm5, %xmm6
 ; SSE41-NEXT:    movdqa %xmm6, %xmm7
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm7
-; SSE41-NEXT:    pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3]
-; SSE41-NEXT:    pand %xmm8, %xmm0
-; SSE41-NEXT:    por %xmm7, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm7
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
+; SSE41-NEXT:    pand %xmm7, %xmm0
+; SSE41-NEXT:    por %xmm6, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm2
 ; SSE41-NEXT:    movdqa %xmm1, %xmm0
 ; SSE41-NEXT:    pxor %xmm5, %xmm0
 ; SSE41-NEXT:    pxor %xmm3, %xmm5
 ; SSE41-NEXT:    movdqa %xmm5, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[1,1,3,3]
-; SSE41-NEXT:    pand %xmm6, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm5
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm5, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm3
 ; SSE41-NEXT:    movapd %xmm2, %xmm0
 ; SSE41-NEXT:    movapd %xmm3, %xmm1
diff --git a/llvm/test/CodeGen/X86/vec_round.ll b/llvm/test/CodeGen/X86/vec_round.ll
index 9258f9ee5..6bc38400a 100644
--- a/llvm/test/CodeGen/X86/vec_round.ll
+++ b/llvm/test/CodeGen/X86/vec_round.ll
@@ -1,14 +1,22 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mcpu=nehalem -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 declare void @use(<2 x double>)
 
-; CHECK-LABEL: @test
-; CHECK: callq round
-
 ; Function Attrs: nounwind uwtable
 define void @test() {
+; CHECK-LABEL: test:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    callq round
+; CHECK-NEXT:    movddup {{.*#+}} xmm0 = xmm0[0,0]
+; CHECK-NEXT:    callq use
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
 entry:
   %tmp = call <2 x double> @llvm.round.v2f64(<2 x double> undef)
   call void @use(<2 x double> %tmp)
diff --git a/llvm/test/CodeGen/X86/vec_saddo.ll b/llvm/test/CodeGen/X86/vec_saddo.ll
index dbe1b7286..785439883 100644
--- a/llvm/test/CodeGen/X86/vec_saddo.ll
+++ b/llvm/test/CodeGen/X86/vec_saddo.ll
@@ -49,134 +49,46 @@ define <1 x i32> @saddo_v1i32(<1 x i32> %a0, <1 x i32> %a1, <1 x i32>* %p2) noun
 }
 
 define <2 x i32> @saddo_v2i32(<2 x i32> %a0, <2 x i32> %a1, <2 x i32>* %p2) nounwind {
-; SSE2-LABEL: saddo_v2i32:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    psllq $32, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,3,2,3]
-; SSE2-NEXT:    psrad $31, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; SSE2-NEXT:    psllq $32, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,3,2,3]
-; SSE2-NEXT:    psrad $31, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; SSE2-NEXT:    paddq %xmm2, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm0
-; SSE2-NEXT:    psllq $32, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,3,2,3]
-; SSE2-NEXT:    psrad $31, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,0,3,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE2-NEXT:    pxor %xmm3, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; SSE2-NEXT:    movq %xmm1, (%rdi)
-; SSE2-NEXT:    retq
-;
-; SSSE3-LABEL: saddo_v2i32:
-; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    psllq $32, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,3,2,3]
-; SSSE3-NEXT:    psrad $31, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; SSSE3-NEXT:    psllq $32, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,3,2,3]
-; SSSE3-NEXT:    psrad $31, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; SSSE3-NEXT:    paddq %xmm2, %xmm1
-; SSSE3-NEXT:    movdqa %xmm1, %xmm0
-; SSSE3-NEXT:    psllq $32, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,3,2,3]
-; SSSE3-NEXT:    psrad $31, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm2, %xmm3
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSSE3-NEXT:    pxor %xmm3, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; SSSE3-NEXT:    movq %xmm1, (%rdi)
-; SSSE3-NEXT:    retq
-;
-; SSE41-LABEL: saddo_v2i32:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa %xmm1, %xmm2
-; SSE41-NEXT:    psllq $32, %xmm2
-; SSE41-NEXT:    psrad $31, %xmm2
-; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
-; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    psllq $32, %xmm1
-; SSE41-NEXT:    psrad $31, %xmm1
-; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
-; SSE41-NEXT:    paddq %xmm2, %xmm1
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    psllq $32, %xmm0
-; SSE41-NEXT:    psrad $31, %xmm0
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3],xmm1[4,5],xmm0[6,7]
-; SSE41-NEXT:    pcmpeqq %xmm1, %xmm0
-; SSE41-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE41-NEXT:    pxor %xmm2, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; SSE41-NEXT:    movq %xmm1, (%rdi)
-; SSE41-NEXT:    retq
+; SSE-LABEL: saddo_v2i32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pxor %xmm2, %xmm2
+; SSE-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSE-NEXT:    paddd %xmm0, %xmm1
+; SSE-NEXT:    pcmpgtd %xmm1, %xmm0
+; SSE-NEXT:    pxor %xmm2, %xmm0
+; SSE-NEXT:    movq %xmm1, (%rdi)
+; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: saddo_v2i32:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpsllq $32, %xmm1, %xmm2
-; AVX1-NEXT:    vpsrad $31, %xmm2, %xmm2
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
-; AVX1-NEXT:    vpsllq $32, %xmm0, %xmm2
-; AVX1-NEXT:    vpsrad $31, %xmm2, %xmm2
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
-; AVX1-NEXT:    vpaddq %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpsllq $32, %xmm1, %xmm0
-; AVX1-NEXT:    vpsrad $31, %xmm0, %xmm0
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3],xmm1[4,5],xmm0[6,7]
-; AVX1-NEXT:    vpcmpeqq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm2
+; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor %xmm0, %xmm2, %xmm0
 ; AVX1-NEXT:    vmovq %xmm1, (%rdi)
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: saddo_v2i32:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpsllq $32, %xmm1, %xmm2
-; AVX2-NEXT:    vpsrad $31, %xmm2, %xmm2
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; AVX2-NEXT:    vpsllq $32, %xmm0, %xmm2
-; AVX2-NEXT:    vpsrad $31, %xmm2, %xmm2
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
-; AVX2-NEXT:    vpaddq %xmm1, %xmm0, %xmm1
-; AVX2-NEXT:    vpsllq $32, %xmm1, %xmm0
-; AVX2-NEXT:    vpsrad $31, %xmm0, %xmm0
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2],xmm0[3]
-; AVX2-NEXT:    vpcmpeqq %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm2
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm1
+; AVX2-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor %xmm0, %xmm2, %xmm0
 ; AVX2-NEXT:    vmovq %xmm1, (%rdi)
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: saddo_v2i32:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsllq $32, %xmm1, %xmm1
-; AVX512-NEXT:    vpsraq $32, %xmm1, %xmm1
-; AVX512-NEXT:    vpsllq $32, %xmm0, %xmm0
-; AVX512-NEXT:    vpsraq $32, %xmm0, %xmm0
-; AVX512-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpsllq $32, %xmm0, %xmm1
-; AVX512-NEXT:    vpsraq $32, %xmm1, %xmm1
-; AVX512-NEXT:    vpmovqd %xmm0, (%rdi)
-; AVX512-NEXT:    vpcmpeqq %xmm0, %xmm1, %xmm0
-; AVX512-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
+; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512-NEXT:    vpcmpgtd %xmm1, %xmm2, %k0
+; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm1
+; AVX512-NEXT:    vpcmpgtd %xmm1, %xmm0, %k1
+; AVX512-NEXT:    kxorw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
+; AVX512-NEXT:    vmovq %xmm1, (%rdi)
 ; AVX512-NEXT:    retq
   %t = call {<2 x i32>, <2 x i1>} @llvm.sadd.with.overflow.v2i32(<2 x i32> %a0, <2 x i32> %a1)
   %val = extractvalue {<2 x i32>, <2 x i1>} %t, 0
@@ -190,82 +102,45 @@ define <3 x i32> @saddo_v3i32(<3 x i32> %a0, <3 x i32> %a1, <3 x i32>* %p2) noun
 ; SSE2-LABEL: saddo_v3i32:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm4
-; SSE2-NEXT:    pxor %xmm4, %xmm3
-; SSE2-NEXT:    pxor %xmm5, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm5
-; SSE2-NEXT:    pxor %xmm4, %xmm5
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm3
-; SSE2-NEXT:    paddd %xmm1, %xmm0
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm2
-; SSE2-NEXT:    pxor %xmm4, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm2
-; SSE2-NEXT:    pandn %xmm3, %xmm2
-; SSE2-NEXT:    movq %xmm0, (%rdi)
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; SSE2-NEXT:    movd %xmm0, 8(%rdi)
-; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSE2-NEXT:    paddd %xmm0, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
+; SSE2-NEXT:    pxor %xmm2, %xmm0
+; SSE2-NEXT:    movq %xmm1, (%rdi)
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE2-NEXT:    movd %xmm1, 8(%rdi)
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: saddo_v3i32:
 ; SSSE3:       # %bb.0:
 ; SSSE3-NEXT:    pxor %xmm2, %xmm2
-; SSSE3-NEXT:    pxor %xmm3, %xmm3
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm3
-; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm4
-; SSSE3-NEXT:    pxor %xmm4, %xmm3
-; SSSE3-NEXT:    pxor %xmm5, %xmm5
-; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm5
-; SSSE3-NEXT:    pxor %xmm4, %xmm5
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm3
-; SSSE3-NEXT:    paddd %xmm1, %xmm0
-; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm2
-; SSSE3-NEXT:    pxor %xmm4, %xmm2
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm2
-; SSSE3-NEXT:    pandn %xmm3, %xmm2
-; SSSE3-NEXT:    movq %xmm0, (%rdi)
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; SSSE3-NEXT:    movd %xmm0, 8(%rdi)
-; SSSE3-NEXT:    movdqa %xmm2, %xmm0
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSSE3-NEXT:    paddd %xmm0, %xmm1
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
+; SSSE3-NEXT:    pxor %xmm2, %xmm0
+; SSSE3-NEXT:    movq %xmm1, (%rdi)
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSSE3-NEXT:    movd %xmm1, 8(%rdi)
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: saddo_v3i32:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pxor %xmm2, %xmm2
-; SSE41-NEXT:    pxor %xmm3, %xmm3
-; SSE41-NEXT:    pcmpgtd %xmm1, %xmm3
-; SSE41-NEXT:    pcmpeqd %xmm4, %xmm4
-; SSE41-NEXT:    pxor %xmm4, %xmm3
-; SSE41-NEXT:    pxor %xmm5, %xmm5
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm5
-; SSE41-NEXT:    pxor %xmm4, %xmm5
-; SSE41-NEXT:    pcmpeqd %xmm5, %xmm3
-; SSE41-NEXT:    paddd %xmm1, %xmm0
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm2
-; SSE41-NEXT:    pxor %xmm4, %xmm2
-; SSE41-NEXT:    pcmpeqd %xmm5, %xmm2
-; SSE41-NEXT:    pandn %xmm3, %xmm2
-; SSE41-NEXT:    pextrd $2, %xmm0, 8(%rdi)
-; SSE41-NEXT:    movq %xmm0, (%rdi)
-; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSE41-NEXT:    paddd %xmm0, %xmm1
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
+; SSE41-NEXT:    pxor %xmm2, %xmm0
+; SSE41-NEXT:    pextrd $2, %xmm1, 8(%rdi)
+; SSE41-NEXT:    movq %xmm1, (%rdi)
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: saddo_v3i32:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm3
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm2, %xmm5
-; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm5, %xmm3
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm2
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm0
-; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm5, %xmm0
-; AVX1-NEXT:    vpandn %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor %xmm0, %xmm2, %xmm0
 ; AVX1-NEXT:    vpextrd $2, %xmm1, 8(%rdi)
 ; AVX1-NEXT:    vmovq %xmm1, (%rdi)
 ; AVX1-NEXT:    retq
@@ -273,17 +148,10 @@ define <3 x i32> @saddo_v3i32(<3 x i32> %a0, <3 x i32> %a1, <3 x i32>* %p2) noun
 ; AVX2-LABEL: saddo_v3i32:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm3
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpxor %xmm4, %xmm3, %xmm3
-; AVX2-NEXT:    vpcmpgtd %xmm0, %xmm2, %xmm5
-; AVX2-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm5, %xmm3
+; AVX2-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm2
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm1
-; AVX2-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm0
-; AVX2-NEXT:    vpxor %xmm4, %xmm0, %xmm0
-; AVX2-NEXT:    vpcmpeqd %xmm0, %xmm5, %xmm0
-; AVX2-NEXT:    vpandn %xmm3, %xmm0, %xmm0
+; AVX2-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor %xmm0, %xmm2, %xmm0
 ; AVX2-NEXT:    vpextrd $2, %xmm1, 8(%rdi)
 ; AVX2-NEXT:    vmovq %xmm1, (%rdi)
 ; AVX2-NEXT:    retq
@@ -291,13 +159,10 @@ define <3 x i32> @saddo_v3i32(<3 x i32> %a0, <3 x i32> %a1, <3 x i32>* %p2) noun
 ; AVX512-LABEL: saddo_v3i32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpcmpnltd %xmm2, %xmm1, %k0
-; AVX512-NEXT:    vpcmpnltd %xmm2, %xmm0, %k1
-; AVX512-NEXT:    kxorw %k0, %k1, %k0
+; AVX512-NEXT:    vpcmpgtd %xmm1, %xmm2, %k0
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm1
-; AVX512-NEXT:    vpcmpnltd %xmm2, %xmm1, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
-; AVX512-NEXT:    kandnw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpgtd %xmm1, %xmm0, %k1
+; AVX512-NEXT:    kxorw %k1, %k0, %k1
 ; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
 ; AVX512-NEXT:    vpextrd $2, %xmm1, 8(%rdi)
@@ -315,67 +180,40 @@ define <4 x i32> @saddo_v4i32(<4 x i32> %a0, <4 x i32> %a1, <4 x i32>* %p2) noun
 ; SSE-LABEL: saddo_v4i32:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pxor %xmm2, %xmm2
-; SSE-NEXT:    pxor %xmm3, %xmm3
-; SSE-NEXT:    pcmpgtd %xmm1, %xmm3
-; SSE-NEXT:    pcmpeqd %xmm4, %xmm4
-; SSE-NEXT:    pxor %xmm4, %xmm3
-; SSE-NEXT:    pxor %xmm5, %xmm5
-; SSE-NEXT:    pcmpgtd %xmm0, %xmm5
-; SSE-NEXT:    pxor %xmm4, %xmm5
-; SSE-NEXT:    pcmpeqd %xmm5, %xmm3
-; SSE-NEXT:    paddd %xmm1, %xmm0
-; SSE-NEXT:    pcmpgtd %xmm0, %xmm2
-; SSE-NEXT:    pxor %xmm4, %xmm2
-; SSE-NEXT:    pcmpeqd %xmm5, %xmm2
-; SSE-NEXT:    pandn %xmm3, %xmm2
-; SSE-NEXT:    movdqa %xmm0, (%rdi)
-; SSE-NEXT:    movdqa %xmm2, %xmm0
+; SSE-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSE-NEXT:    paddd %xmm0, %xmm1
+; SSE-NEXT:    pcmpgtd %xmm1, %xmm0
+; SSE-NEXT:    pxor %xmm2, %xmm0
+; SSE-NEXT:    movdqa %xmm1, (%rdi)
 ; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: saddo_v4i32:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm3
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm2, %xmm5
-; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm5, %xmm3
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm2
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm0
-; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm5, %xmm0
-; AVX1-NEXT:    vpandn %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor %xmm0, %xmm2, %xmm0
 ; AVX1-NEXT:    vmovdqa %xmm1, (%rdi)
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: saddo_v4i32:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm3
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpxor %xmm4, %xmm3, %xmm3
-; AVX2-NEXT:    vpcmpgtd %xmm0, %xmm2, %xmm5
-; AVX2-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm5, %xmm3
+; AVX2-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm2
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm1
-; AVX2-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm0
-; AVX2-NEXT:    vpxor %xmm4, %xmm0, %xmm0
-; AVX2-NEXT:    vpcmpeqd %xmm0, %xmm5, %xmm0
-; AVX2-NEXT:    vpandn %xmm3, %xmm0, %xmm0
+; AVX2-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor %xmm0, %xmm2, %xmm0
 ; AVX2-NEXT:    vmovdqa %xmm1, (%rdi)
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: saddo_v4i32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpcmpnltd %xmm2, %xmm1, %k0
-; AVX512-NEXT:    vpcmpnltd %xmm2, %xmm0, %k1
-; AVX512-NEXT:    kxorw %k0, %k1, %k0
+; AVX512-NEXT:    vpcmpgtd %xmm1, %xmm2, %k0
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm1
-; AVX512-NEXT:    vpcmpnltd %xmm2, %xmm1, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
-; AVX512-NEXT:    kandnw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpgtd %xmm1, %xmm0, %k1
+; AVX512-NEXT:    kxorw %k1, %k0, %k1
 ; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
 ; AVX512-NEXT:    vmovdqa %xmm1, (%rdi)
@@ -392,164 +230,118 @@ define <6 x i32> @saddo_v6i32(<6 x i32> %a0, <6 x i32> %a1, <6 x i32>* %p2) noun
 ; SSE2-LABEL: saddo_v6i32:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movq %rdi, %rax
+; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
+; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm1[0]
 ; SSE2-NEXT:    movd %r8d, %xmm0
 ; SSE2-NEXT:    movd %ecx, %xmm1
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
 ; SSE2-NEXT:    movd %edx, %xmm0
-; SSE2-NEXT:    movd %esi, %xmm4
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1]
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm1[0]
+; SSE2-NEXT:    movd %esi, %xmm3
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1]
+; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm1[0]
 ; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; SSE2-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; SSE2-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSE2-NEXT:    movd %r9d, %xmm3
-; SSE2-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
-; SSE2-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; SSE2-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; SSE2-NEXT:    movd %r9d, %xmm0
+; SSE2-NEXT:    movd {{.*#+}} xmm4 = mem[0],zero,zero,zero
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
 ; SSE2-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pxor %xmm6, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm6
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm5
-; SSE2-NEXT:    pxor %xmm5, %xmm6
-; SSE2-NEXT:    pxor %xmm7, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm4, %xmm7
-; SSE2-NEXT:    pxor %xmm5, %xmm7
-; SSE2-NEXT:    pcmpeqd %xmm7, %xmm6
-; SSE2-NEXT:    paddd %xmm4, %xmm0
-; SSE2-NEXT:    pxor %xmm4, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE2-NEXT:    pxor %xmm5, %xmm4
-; SSE2-NEXT:    pcmpeqd %xmm7, %xmm4
-; SSE2-NEXT:    pandn %xmm6, %xmm4
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    paddd %xmm2, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm3
+; SSE2-NEXT:    pxor %xmm5, %xmm5
 ; SSE2-NEXT:    pxor %xmm6, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm6
-; SSE2-NEXT:    pxor %xmm5, %xmm6
-; SSE2-NEXT:    pxor %xmm7, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm7
-; SSE2-NEXT:    pxor %xmm5, %xmm7
-; SSE2-NEXT:    pcmpeqd %xmm7, %xmm6
-; SSE2-NEXT:    paddd %xmm3, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
-; SSE2-NEXT:    pxor %xmm5, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm7, %xmm2
-; SSE2-NEXT:    pandn %xmm6, %xmm2
-; SSE2-NEXT:    movq %xmm1, 16(%rcx)
-; SSE2-NEXT:    movdqa %xmm0, (%rcx)
-; SSE2-NEXT:    movq %xmm2, 16(%rdi)
-; SSE2-NEXT:    movdqa %xmm4, (%rdi)
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm6
+; SSE2-NEXT:    pxor %xmm3, %xmm6
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    paddd %xmm1, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm0
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm5
+; SSE2-NEXT:    pxor %xmm0, %xmm5
+; SSE2-NEXT:    movq %xmm2, 16(%rcx)
+; SSE2-NEXT:    movdqa %xmm4, (%rcx)
+; SSE2-NEXT:    movq %xmm5, 16(%rdi)
+; SSE2-NEXT:    movdqa %xmm6, (%rdi)
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: saddo_v6i32:
 ; SSSE3:       # %bb.0:
 ; SSSE3-NEXT:    movq %rdi, %rax
+; SSSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSSE3-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSSE3-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
+; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm1[0]
 ; SSSE3-NEXT:    movd %r8d, %xmm0
 ; SSSE3-NEXT:    movd %ecx, %xmm1
 ; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
 ; SSSE3-NEXT:    movd %edx, %xmm0
-; SSSE3-NEXT:    movd %esi, %xmm4
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1]
-; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm1[0]
+; SSSE3-NEXT:    movd %esi, %xmm3
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1]
+; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm1[0]
 ; SSSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; SSSE3-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; SSSE3-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; SSSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSSE3-NEXT:    movd %r9d, %xmm3
-; SSSE3-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
-; SSSE3-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; SSSE3-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; SSSE3-NEXT:    movd %r9d, %xmm0
+; SSSE3-NEXT:    movd {{.*#+}} xmm4 = mem[0],zero,zero,zero
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
 ; SSSE3-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
-; SSSE3-NEXT:    pxor %xmm2, %xmm2
-; SSSE3-NEXT:    pxor %xmm6, %xmm6
-; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm6
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm5
-; SSSE3-NEXT:    pxor %xmm5, %xmm6
-; SSSE3-NEXT:    pxor %xmm7, %xmm7
-; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm7
-; SSSE3-NEXT:    pxor %xmm5, %xmm7
-; SSSE3-NEXT:    pcmpeqd %xmm7, %xmm6
-; SSSE3-NEXT:    paddd %xmm4, %xmm0
-; SSSE3-NEXT:    pxor %xmm4, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSSE3-NEXT:    pxor %xmm5, %xmm4
-; SSSE3-NEXT:    pcmpeqd %xmm7, %xmm4
-; SSSE3-NEXT:    pandn %xmm6, %xmm4
+; SSSE3-NEXT:    movdqa %xmm3, %xmm4
+; SSSE3-NEXT:    paddd %xmm2, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm3
+; SSSE3-NEXT:    pxor %xmm5, %xmm5
 ; SSSE3-NEXT:    pxor %xmm6, %xmm6
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm6
-; SSSE3-NEXT:    pxor %xmm5, %xmm6
-; SSSE3-NEXT:    pxor %xmm7, %xmm7
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm7
-; SSSE3-NEXT:    pxor %xmm5, %xmm7
-; SSSE3-NEXT:    pcmpeqd %xmm7, %xmm6
-; SSSE3-NEXT:    paddd %xmm3, %xmm1
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
-; SSSE3-NEXT:    pxor %xmm5, %xmm2
-; SSSE3-NEXT:    pcmpeqd %xmm7, %xmm2
-; SSSE3-NEXT:    pandn %xmm6, %xmm2
-; SSSE3-NEXT:    movq %xmm1, 16(%rcx)
-; SSSE3-NEXT:    movdqa %xmm0, (%rcx)
-; SSSE3-NEXT:    movq %xmm2, 16(%rdi)
-; SSSE3-NEXT:    movdqa %xmm4, (%rdi)
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm6
+; SSSE3-NEXT:    pxor %xmm3, %xmm6
+; SSSE3-NEXT:    movdqa %xmm0, %xmm2
+; SSSE3-NEXT:    paddd %xmm1, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm0
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm5
+; SSSE3-NEXT:    pxor %xmm0, %xmm5
+; SSSE3-NEXT:    movq %xmm2, 16(%rcx)
+; SSSE3-NEXT:    movdqa %xmm4, (%rcx)
+; SSSE3-NEXT:    movq %xmm5, 16(%rdi)
+; SSSE3-NEXT:    movdqa %xmm6, (%rdi)
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: saddo_v6i32:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movq %rdi, %rax
-; SSE41-NEXT:    movd %esi, %xmm4
-; SSE41-NEXT:    pinsrd $1, %edx, %xmm4
-; SSE41-NEXT:    pinsrd $2, %ecx, %xmm4
-; SSE41-NEXT:    pinsrd $3, %r8d, %xmm4
-; SSE41-NEXT:    movd %r9d, %xmm2
-; SSE41-NEXT:    pinsrd $1, {{[0-9]+}}(%rsp), %xmm2
-; SSE41-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE41-NEXT:    movd %esi, %xmm1
+; SSE41-NEXT:    pinsrd $1, %edx, %xmm1
+; SSE41-NEXT:    pinsrd $2, %ecx, %xmm1
+; SSE41-NEXT:    pinsrd $3, %r8d, %xmm1
+; SSE41-NEXT:    movd %r9d, %xmm0
 ; SSE41-NEXT:    pinsrd $1, {{[0-9]+}}(%rsp), %xmm0
-; SSE41-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; SSE41-NEXT:    pinsrd $1, {{[0-9]+}}(%rsp), %xmm1
-; SSE41-NEXT:    pinsrd $2, {{[0-9]+}}(%rsp), %xmm1
-; SSE41-NEXT:    pinsrd $3, {{[0-9]+}}(%rsp), %xmm1
+; SSE41-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE41-NEXT:    pinsrd $1, {{[0-9]+}}(%rsp), %xmm2
+; SSE41-NEXT:    movd {{.*#+}} xmm3 = mem[0],zero,zero,zero
+; SSE41-NEXT:    pinsrd $1, {{[0-9]+}}(%rsp), %xmm3
+; SSE41-NEXT:    pinsrd $2, {{[0-9]+}}(%rsp), %xmm3
+; SSE41-NEXT:    pinsrd $3, {{[0-9]+}}(%rsp), %xmm3
 ; SSE41-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
-; SSE41-NEXT:    pxor %xmm3, %xmm3
-; SSE41-NEXT:    pxor %xmm6, %xmm6
-; SSE41-NEXT:    pcmpgtd %xmm1, %xmm6
-; SSE41-NEXT:    pcmpeqd %xmm5, %xmm5
-; SSE41-NEXT:    pxor %xmm5, %xmm6
-; SSE41-NEXT:    pxor %xmm7, %xmm7
-; SSE41-NEXT:    pcmpgtd %xmm4, %xmm7
-; SSE41-NEXT:    pxor %xmm5, %xmm7
-; SSE41-NEXT:    pcmpeqd %xmm7, %xmm6
-; SSE41-NEXT:    paddd %xmm4, %xmm1
-; SSE41-NEXT:    pxor %xmm4, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm1, %xmm4
-; SSE41-NEXT:    pxor %xmm5, %xmm4
-; SSE41-NEXT:    pcmpeqd %xmm7, %xmm4
-; SSE41-NEXT:    pandn %xmm6, %xmm4
+; SSE41-NEXT:    movdqa %xmm1, %xmm4
+; SSE41-NEXT:    paddd %xmm3, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm4, %xmm1
+; SSE41-NEXT:    pxor %xmm5, %xmm5
 ; SSE41-NEXT:    pxor %xmm6, %xmm6
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
-; SSE41-NEXT:    pxor %xmm5, %xmm6
-; SSE41-NEXT:    pxor %xmm7, %xmm7
-; SSE41-NEXT:    pcmpgtd %xmm2, %xmm7
-; SSE41-NEXT:    pxor %xmm5, %xmm7
-; SSE41-NEXT:    pcmpeqd %xmm7, %xmm6
-; SSE41-NEXT:    paddd %xmm2, %xmm0
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSE41-NEXT:    pxor %xmm5, %xmm3
-; SSE41-NEXT:    pcmpeqd %xmm7, %xmm3
-; SSE41-NEXT:    pandn %xmm6, %xmm3
-; SSE41-NEXT:    movq %xmm0, 16(%rcx)
-; SSE41-NEXT:    movdqa %xmm1, (%rcx)
-; SSE41-NEXT:    movq %xmm3, 16(%rdi)
-; SSE41-NEXT:    movdqa %xmm4, (%rdi)
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm6
+; SSE41-NEXT:    pxor %xmm1, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm2, %xmm5
+; SSE41-NEXT:    paddd %xmm0, %xmm2
+; SSE41-NEXT:    pcmpgtd %xmm2, %xmm0
+; SSE41-NEXT:    pxor %xmm5, %xmm0
+; SSE41-NEXT:    movq %xmm2, 16(%rcx)
+; SSE41-NEXT:    movdqa %xmm4, (%rcx)
+; SSE41-NEXT:    movq %xmm0, 16(%rdi)
+; SSE41-NEXT:    movdqa %xmm6, (%rdi)
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: saddo_v6i32:
@@ -557,28 +349,15 @@ define <6 x i32> @saddo_v6i32(<6 x i32> %a0, <6 x i32> %a1, <6 x i32>* %p2) noun
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
 ; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
 ; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm3, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
-; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm6
-; AVX1-NEXT:    vpcmpgtd %xmm6, %xmm3, %xmm7
-; AVX1-NEXT:    vpxor %xmm5, %xmm7, %xmm7
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm7, %xmm8
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm3, %xmm4
-; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm9
-; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm3, %xmm4
-; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm9, %xmm4, %xmm9
-; AVX1-NEXT:    vinsertf128 $1, %xmm8, %ymm9, %ymm8
-; AVX1-NEXT:    vpaddd %xmm2, %xmm6, %xmm2
-; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm3, %xmm6
-; AVX1-NEXT:    vpxor %xmm5, %xmm6, %xmm6
-; AVX1-NEXT:    vpcmpeqd %xmm6, %xmm7, %xmm6
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm3, %xmm3
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
+; AVX1-NEXT:    vpaddd %xmm2, %xmm4, %xmm2
+; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm4, %xmm4
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm3, %xmm0
-; AVX1-NEXT:    vpxor %xmm5, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm4, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm0, %ymm0
-; AVX1-NEXT:    vandnps %ymm8, %ymm0, %ymm0
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
+; AVX1-NEXT:    vxorps %ymm0, %ymm3, %ymm0
 ; AVX1-NEXT:    vmovq %xmm2, 16(%rdi)
 ; AVX1-NEXT:    vmovdqa %xmm1, (%rdi)
 ; AVX1-NEXT:    retq
@@ -586,17 +365,10 @@ define <6 x i32> @saddo_v6i32(<6 x i32> %a0, <6 x i32> %a1, <6 x i32>* %p2) noun
 ; AVX2-LABEL: saddo_v6i32:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm2, %ymm3
-; AVX2-NEXT:    vpcmpeqd %ymm4, %ymm4, %ymm4
-; AVX2-NEXT:    vpxor %ymm4, %ymm3, %ymm3
-; AVX2-NEXT:    vpcmpgtd %ymm0, %ymm2, %ymm5
-; AVX2-NEXT:    vpxor %ymm4, %ymm5, %ymm5
-; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm5, %ymm3
+; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm2, %ymm2
 ; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm1
-; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm2, %ymm0
-; AVX2-NEXT:    vpxor %ymm4, %ymm0, %ymm0
-; AVX2-NEXT:    vpcmpeqd %ymm0, %ymm5, %ymm0
-; AVX2-NEXT:    vpandn %ymm3, %ymm0, %ymm0
+; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %ymm0, %ymm2, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
 ; AVX2-NEXT:    vmovq %xmm2, 16(%rdi)
 ; AVX2-NEXT:    vmovdqa %xmm1, (%rdi)
@@ -605,13 +377,10 @@ define <6 x i32> @saddo_v6i32(<6 x i32> %a0, <6 x i32> %a1, <6 x i32>* %p2) noun
 ; AVX512-LABEL: saddo_v6i32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpcmpnltd %ymm2, %ymm1, %k0
-; AVX512-NEXT:    vpcmpnltd %ymm2, %ymm0, %k1
-; AVX512-NEXT:    kxorw %k0, %k1, %k0
+; AVX512-NEXT:    vpcmpgtd %ymm1, %ymm2, %k0
 ; AVX512-NEXT:    vpaddd %ymm1, %ymm0, %ymm1
-; AVX512-NEXT:    vpcmpnltd %ymm2, %ymm1, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
-; AVX512-NEXT:    kandnw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpgtd %ymm1, %ymm0, %k1
+; AVX512-NEXT:    kxorw %k1, %k0, %k1
 ; AVX512-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
 ; AVX512-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
 ; AVX512-NEXT:    vextracti128 $1, %ymm1, %xmm2
@@ -629,37 +398,18 @@ define <6 x i32> @saddo_v6i32(<6 x i32> %a0, <6 x i32> %a1, <6 x i32>* %p2) noun
 define <8 x i32> @saddo_v8i32(<8 x i32> %a0, <8 x i32> %a1, <8 x i32>* %p2) nounwind {
 ; SSE-LABEL: saddo_v8i32:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    movdqa %xmm1, %xmm4
-; SSE-NEXT:    pxor %xmm1, %xmm1
-; SSE-NEXT:    pxor %xmm6, %xmm6
-; SSE-NEXT:    pcmpgtd %xmm2, %xmm6
-; SSE-NEXT:    pcmpeqd %xmm5, %xmm5
-; SSE-NEXT:    pxor %xmm5, %xmm6
-; SSE-NEXT:    pxor %xmm7, %xmm7
-; SSE-NEXT:    pcmpgtd %xmm0, %xmm7
-; SSE-NEXT:    pxor %xmm5, %xmm7
-; SSE-NEXT:    pcmpeqd %xmm7, %xmm6
-; SSE-NEXT:    paddd %xmm2, %xmm0
-; SSE-NEXT:    pxor %xmm2, %xmm2
-; SSE-NEXT:    pcmpgtd %xmm0, %xmm2
-; SSE-NEXT:    pxor %xmm5, %xmm2
-; SSE-NEXT:    pcmpeqd %xmm7, %xmm2
-; SSE-NEXT:    pandn %xmm6, %xmm2
-; SSE-NEXT:    pxor %xmm6, %xmm6
-; SSE-NEXT:    pcmpgtd %xmm3, %xmm6
-; SSE-NEXT:    pxor %xmm5, %xmm6
-; SSE-NEXT:    pxor %xmm7, %xmm7
-; SSE-NEXT:    pcmpgtd %xmm4, %xmm7
-; SSE-NEXT:    pxor %xmm5, %xmm7
-; SSE-NEXT:    pcmpeqd %xmm7, %xmm6
-; SSE-NEXT:    paddd %xmm3, %xmm4
-; SSE-NEXT:    pcmpgtd %xmm4, %xmm1
-; SSE-NEXT:    pxor %xmm5, %xmm1
-; SSE-NEXT:    pcmpeqd %xmm7, %xmm1
-; SSE-NEXT:    pandn %xmm6, %xmm1
-; SSE-NEXT:    movdqa %xmm4, 16(%rdi)
-; SSE-NEXT:    movdqa %xmm0, (%rdi)
-; SSE-NEXT:    movdqa %xmm2, %xmm0
+; SSE-NEXT:    pxor %xmm4, %xmm4
+; SSE-NEXT:    pxor %xmm5, %xmm5
+; SSE-NEXT:    pcmpgtd %xmm2, %xmm5
+; SSE-NEXT:    paddd %xmm0, %xmm2
+; SSE-NEXT:    pcmpgtd %xmm2, %xmm0
+; SSE-NEXT:    pxor %xmm5, %xmm0
+; SSE-NEXT:    pcmpgtd %xmm3, %xmm4
+; SSE-NEXT:    paddd %xmm1, %xmm3
+; SSE-NEXT:    pcmpgtd %xmm3, %xmm1
+; SSE-NEXT:    pxor %xmm4, %xmm1
+; SSE-NEXT:    movdqa %xmm3, 16(%rdi)
+; SSE-NEXT:    movdqa %xmm2, (%rdi)
 ; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: saddo_v8i32:
@@ -667,28 +417,15 @@ define <8 x i32> @saddo_v8i32(<8 x i32> %a0, <8 x i32> %a1, <8 x i32>* %p2) noun
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
 ; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
 ; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm3, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
-; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm6
-; AVX1-NEXT:    vpcmpgtd %xmm6, %xmm3, %xmm7
-; AVX1-NEXT:    vpxor %xmm5, %xmm7, %xmm7
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm7, %xmm8
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm3, %xmm4
-; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm9
-; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm3, %xmm4
-; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm9, %xmm4, %xmm9
-; AVX1-NEXT:    vinsertf128 $1, %xmm8, %ymm9, %ymm8
-; AVX1-NEXT:    vpaddd %xmm2, %xmm6, %xmm2
-; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm3, %xmm6
-; AVX1-NEXT:    vpxor %xmm5, %xmm6, %xmm6
-; AVX1-NEXT:    vpcmpeqd %xmm6, %xmm7, %xmm6
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm3, %xmm3
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
+; AVX1-NEXT:    vpaddd %xmm2, %xmm4, %xmm2
+; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm4, %xmm4
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm3, %xmm0
-; AVX1-NEXT:    vpxor %xmm5, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm4, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm0, %ymm0
-; AVX1-NEXT:    vandnps %ymm8, %ymm0, %ymm0
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
+; AVX1-NEXT:    vxorps %ymm0, %ymm3, %ymm0
 ; AVX1-NEXT:    vmovdqa %xmm2, 16(%rdi)
 ; AVX1-NEXT:    vmovdqa %xmm1, (%rdi)
 ; AVX1-NEXT:    retq
@@ -696,30 +433,20 @@ define <8 x i32> @saddo_v8i32(<8 x i32> %a0, <8 x i32> %a1, <8 x i32>* %p2) noun
 ; AVX2-LABEL: saddo_v8i32:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm2, %ymm3
-; AVX2-NEXT:    vpcmpeqd %ymm4, %ymm4, %ymm4
-; AVX2-NEXT:    vpxor %ymm4, %ymm3, %ymm3
-; AVX2-NEXT:    vpcmpgtd %ymm0, %ymm2, %ymm5
-; AVX2-NEXT:    vpxor %ymm4, %ymm5, %ymm5
-; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm5, %ymm3
+; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm2, %ymm2
 ; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm1
-; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm2, %ymm0
-; AVX2-NEXT:    vpxor %ymm4, %ymm0, %ymm0
-; AVX2-NEXT:    vpcmpeqd %ymm0, %ymm5, %ymm0
-; AVX2-NEXT:    vpandn %ymm3, %ymm0, %ymm0
+; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %ymm0, %ymm2, %ymm0
 ; AVX2-NEXT:    vmovdqa %ymm1, (%rdi)
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: saddo_v8i32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpcmpnltd %ymm2, %ymm1, %k0
-; AVX512-NEXT:    vpcmpnltd %ymm2, %ymm0, %k1
-; AVX512-NEXT:    kxorw %k0, %k1, %k0
+; AVX512-NEXT:    vpcmpgtd %ymm1, %ymm2, %k0
 ; AVX512-NEXT:    vpaddd %ymm1, %ymm0, %ymm1
-; AVX512-NEXT:    vpcmpnltd %ymm2, %ymm1, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
-; AVX512-NEXT:    kandnw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpgtd %ymm1, %ymm0, %k1
+; AVX512-NEXT:    kxorw %k1, %k0, %k1
 ; AVX512-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
 ; AVX512-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
 ; AVX512-NEXT:    vmovdqa %ymm1, (%rdi)
@@ -735,132 +462,70 @@ define <8 x i32> @saddo_v8i32(<8 x i32> %a0, <8 x i32> %a1, <8 x i32>* %p2) noun
 define <16 x i32> @saddo_v16i32(<16 x i32> %a0, <16 x i32> %a1, <16 x i32>* %p2) nounwind {
 ; SSE-LABEL: saddo_v16i32:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    movdqa %xmm3, %xmm8
-; SSE-NEXT:    pxor %xmm3, %xmm3
-; SSE-NEXT:    pxor %xmm11, %xmm11
-; SSE-NEXT:    pcmpgtd %xmm4, %xmm11
-; SSE-NEXT:    pcmpeqd %xmm10, %xmm10
-; SSE-NEXT:    pxor %xmm10, %xmm11
-; SSE-NEXT:    pxor %xmm12, %xmm12
-; SSE-NEXT:    pcmpgtd %xmm0, %xmm12
-; SSE-NEXT:    pxor %xmm10, %xmm12
-; SSE-NEXT:    pcmpeqd %xmm12, %xmm11
-; SSE-NEXT:    paddd %xmm4, %xmm0
+; SSE-NEXT:    pxor %xmm8, %xmm8
 ; SSE-NEXT:    pxor %xmm9, %xmm9
-; SSE-NEXT:    pcmpgtd %xmm0, %xmm9
-; SSE-NEXT:    pxor %xmm10, %xmm9
-; SSE-NEXT:    pcmpeqd %xmm12, %xmm9
-; SSE-NEXT:    pandn %xmm11, %xmm9
-; SSE-NEXT:    pxor %xmm12, %xmm12
-; SSE-NEXT:    pcmpgtd %xmm5, %xmm12
-; SSE-NEXT:    pxor %xmm10, %xmm12
-; SSE-NEXT:    pxor %xmm4, %xmm4
-; SSE-NEXT:    pcmpgtd %xmm1, %xmm4
-; SSE-NEXT:    pxor %xmm10, %xmm4
-; SSE-NEXT:    pcmpeqd %xmm4, %xmm12
-; SSE-NEXT:    paddd %xmm5, %xmm1
-; SSE-NEXT:    pxor %xmm11, %xmm11
-; SSE-NEXT:    pcmpgtd %xmm1, %xmm11
-; SSE-NEXT:    pxor %xmm10, %xmm11
-; SSE-NEXT:    pcmpeqd %xmm4, %xmm11
-; SSE-NEXT:    pandn %xmm12, %xmm11
-; SSE-NEXT:    pxor %xmm4, %xmm4
-; SSE-NEXT:    pcmpgtd %xmm6, %xmm4
-; SSE-NEXT:    pxor %xmm10, %xmm4
-; SSE-NEXT:    pxor %xmm5, %xmm5
-; SSE-NEXT:    pcmpgtd %xmm2, %xmm5
-; SSE-NEXT:    pxor %xmm10, %xmm5
-; SSE-NEXT:    pcmpeqd %xmm5, %xmm4
-; SSE-NEXT:    paddd %xmm6, %xmm2
-; SSE-NEXT:    pxor %xmm6, %xmm6
-; SSE-NEXT:    pcmpgtd %xmm2, %xmm6
-; SSE-NEXT:    pxor %xmm10, %xmm6
-; SSE-NEXT:    pcmpeqd %xmm5, %xmm6
-; SSE-NEXT:    pandn %xmm4, %xmm6
-; SSE-NEXT:    pxor %xmm4, %xmm4
-; SSE-NEXT:    pcmpgtd %xmm7, %xmm4
-; SSE-NEXT:    pxor %xmm10, %xmm4
-; SSE-NEXT:    pxor %xmm5, %xmm5
-; SSE-NEXT:    pcmpgtd %xmm8, %xmm5
-; SSE-NEXT:    pxor %xmm10, %xmm5
-; SSE-NEXT:    pcmpeqd %xmm5, %xmm4
-; SSE-NEXT:    paddd %xmm7, %xmm8
-; SSE-NEXT:    pcmpgtd %xmm8, %xmm3
-; SSE-NEXT:    pxor %xmm10, %xmm3
-; SSE-NEXT:    pcmpeqd %xmm5, %xmm3
-; SSE-NEXT:    pandn %xmm4, %xmm3
-; SSE-NEXT:    movdqa %xmm8, 48(%rdi)
-; SSE-NEXT:    movdqa %xmm2, 32(%rdi)
-; SSE-NEXT:    movdqa %xmm1, 16(%rdi)
-; SSE-NEXT:    movdqa %xmm0, (%rdi)
-; SSE-NEXT:    movdqa %xmm9, %xmm0
-; SSE-NEXT:    movdqa %xmm11, %xmm1
-; SSE-NEXT:    movdqa %xmm6, %xmm2
+; SSE-NEXT:    pcmpgtd %xmm4, %xmm9
+; SSE-NEXT:    paddd %xmm0, %xmm4
+; SSE-NEXT:    pcmpgtd %xmm4, %xmm0
+; SSE-NEXT:    pxor %xmm9, %xmm0
+; SSE-NEXT:    pxor %xmm9, %xmm9
+; SSE-NEXT:    pcmpgtd %xmm5, %xmm9
+; SSE-NEXT:    paddd %xmm1, %xmm5
+; SSE-NEXT:    pcmpgtd %xmm5, %xmm1
+; SSE-NEXT:    pxor %xmm9, %xmm1
+; SSE-NEXT:    pxor %xmm9, %xmm9
+; SSE-NEXT:    pcmpgtd %xmm6, %xmm9
+; SSE-NEXT:    paddd %xmm2, %xmm6
+; SSE-NEXT:    pcmpgtd %xmm6, %xmm2
+; SSE-NEXT:    pxor %xmm9, %xmm2
+; SSE-NEXT:    pcmpgtd %xmm7, %xmm8
+; SSE-NEXT:    paddd %xmm3, %xmm7
+; SSE-NEXT:    pcmpgtd %xmm7, %xmm3
+; SSE-NEXT:    pxor %xmm8, %xmm3
+; SSE-NEXT:    movdqa %xmm7, 48(%rdi)
+; SSE-NEXT:    movdqa %xmm6, 32(%rdi)
+; SSE-NEXT:    movdqa %xmm5, 16(%rdi)
+; SSE-NEXT:    movdqa %xmm4, (%rdi)
 ; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: saddo_v16i32:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm9
+; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm4
 ; AVX1-NEXT:    vpxor %xmm5, %xmm5, %xmm5
-; AVX1-NEXT:    vpcmpgtd %xmm9, %xmm5, %xmm7
-; AVX1-NEXT:    vpcmpeqd %xmm6, %xmm6, %xmm6
-; AVX1-NEXT:    vpxor %xmm6, %xmm7, %xmm8
+; AVX1-NEXT:    vpcmpgtd %xmm4, %xmm5, %xmm6
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm7
-; AVX1-NEXT:    vpcmpgtd %xmm7, %xmm5, %xmm4
-; AVX1-NEXT:    vpxor %xmm6, %xmm4, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm8, %xmm4, %xmm8
-; AVX1-NEXT:    vpaddd %xmm9, %xmm7, %xmm9
-; AVX1-NEXT:    vpcmpgtd %xmm9, %xmm5, %xmm7
-; AVX1-NEXT:    vpxor %xmm6, %xmm7, %xmm7
-; AVX1-NEXT:    vpcmpeqd %xmm7, %xmm4, %xmm4
-; AVX1-NEXT:    vpandn %xmm8, %xmm4, %xmm8
+; AVX1-NEXT:    vpaddd %xmm4, %xmm7, %xmm8
+; AVX1-NEXT:    vpcmpgtd %xmm8, %xmm7, %xmm7
+; AVX1-NEXT:    vpxor %xmm7, %xmm6, %xmm6
 ; AVX1-NEXT:    vpcmpgtd %xmm3, %xmm5, %xmm7
-; AVX1-NEXT:    vpxor %xmm6, %xmm7, %xmm7
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm5, %xmm4
-; AVX1-NEXT:    vpxor %xmm6, %xmm4, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm7, %xmm4, %xmm7
-; AVX1-NEXT:    vpaddd %xmm3, %xmm1, %xmm10
-; AVX1-NEXT:    vpcmpgtd %xmm10, %xmm5, %xmm1
-; AVX1-NEXT:    vpxor %xmm6, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm4, %xmm1
-; AVX1-NEXT:    vpandn %xmm7, %xmm1, %xmm1
-; AVX1-NEXT:    vpackssdw %xmm8, %xmm1, %xmm8
-; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm4
-; AVX1-NEXT:    vpcmpgtd %xmm4, %xmm5, %xmm7
-; AVX1-NEXT:    vpxor %xmm6, %xmm7, %xmm7
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm5, %xmm3
-; AVX1-NEXT:    vpxor %xmm6, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpeqd %xmm7, %xmm3, %xmm7
-; AVX1-NEXT:    vpaddd %xmm4, %xmm1, %xmm4
-; AVX1-NEXT:    vpcmpgtd %xmm4, %xmm5, %xmm1
-; AVX1-NEXT:    vpxor %xmm6, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm3, %xmm1
-; AVX1-NEXT:    vpandn %xmm7, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm5, %xmm3
-; AVX1-NEXT:    vpxor %xmm6, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm5, %xmm7
-; AVX1-NEXT:    vpxor %xmm6, %xmm7, %xmm7
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm7, %xmm3
+; AVX1-NEXT:    vpaddd %xmm3, %xmm1, %xmm3
+; AVX1-NEXT:    vpcmpgtd %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vpxor %xmm1, %xmm7, %xmm1
+; AVX1-NEXT:    vpackssdw %xmm6, %xmm1, %xmm1
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm6
+; AVX1-NEXT:    vpcmpgtd %xmm6, %xmm5, %xmm7
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
+; AVX1-NEXT:    vpaddd %xmm6, %xmm4, %xmm6
+; AVX1-NEXT:    vpcmpgtd %xmm6, %xmm4, %xmm4
+; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm4
+; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm5, %xmm5
 ; AVX1-NEXT:    vpaddd %xmm2, %xmm0, %xmm2
-; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm5, %xmm0
-; AVX1-NEXT:    vpxor %xmm6, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm7, %xmm0
-; AVX1-NEXT:    vpandn %xmm3, %xmm0, %xmm0
-; AVX1-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpacksswb %xmm8, %xmm0, %xmm1
+; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor %xmm0, %xmm5, %xmm0
+; AVX1-NEXT:    vpackssdw %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vpacksswb %xmm1, %xmm0, %xmm1
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,2,3]
-; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
-; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[2,3,0,1]
-; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm1[1,1,2,3]
+; AVX1-NEXT:    vpmovsxbd %xmm4, %xmm4
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm1[2,3,0,1]
+; AVX1-NEXT:    vpmovsxbd %xmm4, %xmm4
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,3,0,1]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm3, %ymm1
-; AVX1-NEXT:    vmovdqa %xmm9, 48(%rdi)
-; AVX1-NEXT:    vmovdqa %xmm10, 32(%rdi)
-; AVX1-NEXT:    vmovdqa %xmm4, 16(%rdi)
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm4, %ymm1
+; AVX1-NEXT:    vmovdqa %xmm8, 48(%rdi)
+; AVX1-NEXT:    vmovdqa %xmm3, 32(%rdi)
+; AVX1-NEXT:    vmovdqa %xmm6, 16(%rdi)
 ; AVX1-NEXT:    vmovdqa %xmm2, (%rdi)
 ; AVX1-NEXT:    retq
 ;
@@ -868,28 +533,15 @@ define <16 x i32> @saddo_v16i32(<16 x i32> %a0, <16 x i32> %a1, <16 x i32>* %p2)
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
 ; AVX2-NEXT:    vpcmpgtd %ymm3, %ymm4, %ymm5
-; AVX2-NEXT:    vpcmpeqd %ymm6, %ymm6, %ymm6
-; AVX2-NEXT:    vpxor %ymm6, %ymm5, %ymm5
-; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm4, %ymm7
-; AVX2-NEXT:    vpxor %ymm6, %ymm7, %ymm7
-; AVX2-NEXT:    vpcmpeqd %ymm5, %ymm7, %ymm5
 ; AVX2-NEXT:    vpaddd %ymm3, %ymm1, %ymm3
-; AVX2-NEXT:    vpcmpgtd %ymm3, %ymm4, %ymm1
-; AVX2-NEXT:    vpxor %ymm6, %ymm1, %ymm1
-; AVX2-NEXT:    vpcmpeqd %ymm1, %ymm7, %ymm1
-; AVX2-NEXT:    vpandn %ymm5, %ymm1, %ymm1
+; AVX2-NEXT:    vpcmpgtd %ymm3, %ymm1, %ymm1
+; AVX2-NEXT:    vpxor %ymm1, %ymm5, %ymm1
 ; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm5
 ; AVX2-NEXT:    vpackssdw %xmm5, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpgtd %ymm2, %ymm4, %ymm5
-; AVX2-NEXT:    vpxor %ymm6, %ymm5, %ymm5
-; AVX2-NEXT:    vpcmpgtd %ymm0, %ymm4, %ymm7
-; AVX2-NEXT:    vpxor %ymm6, %ymm7, %ymm7
-; AVX2-NEXT:    vpcmpeqd %ymm5, %ymm7, %ymm5
+; AVX2-NEXT:    vpcmpgtd %ymm2, %ymm4, %ymm4
 ; AVX2-NEXT:    vpaddd %ymm2, %ymm0, %ymm2
-; AVX2-NEXT:    vpcmpgtd %ymm2, %ymm4, %ymm0
-; AVX2-NEXT:    vpxor %ymm6, %ymm0, %ymm0
-; AVX2-NEXT:    vpcmpeqd %ymm0, %ymm7, %ymm0
-; AVX2-NEXT:    vpandn %ymm5, %ymm0, %ymm0
+; AVX2-NEXT:    vpcmpgtd %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %ymm0, %ymm4, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm4
 ; AVX2-NEXT:    vpackssdw %xmm4, %xmm0, %xmm0
 ; AVX2-NEXT:    vpacksswb %xmm1, %xmm0, %xmm1
@@ -903,13 +555,10 @@ define <16 x i32> @saddo_v16i32(<16 x i32> %a0, <16 x i32> %a1, <16 x i32>* %p2)
 ; AVX512-LABEL: saddo_v16i32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpcmpnltd %zmm2, %zmm1, %k0
-; AVX512-NEXT:    vpcmpnltd %zmm2, %zmm0, %k1
-; AVX512-NEXT:    kxorw %k0, %k1, %k0
+; AVX512-NEXT:    vpcmpgtd %zmm1, %zmm2, %k0
 ; AVX512-NEXT:    vpaddd %zmm1, %zmm0, %zmm1
-; AVX512-NEXT:    vpcmpnltd %zmm2, %zmm1, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
-; AVX512-NEXT:    kandnw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpgtd %zmm1, %zmm0, %k1
+; AVX512-NEXT:    kxorw %k1, %k0, %k1
 ; AVX512-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
 ; AVX512-NEXT:    vmovdqa64 %zmm1, (%rdi)
 ; AVX512-NEXT:    retq
@@ -1155,187 +804,62 @@ define <8 x i32> @saddo_v8i16(<8 x i16> %a0, <8 x i16> %a1, <8 x i16>* %p2) noun
 }
 
 define <2 x i32> @saddo_v2i64(<2 x i64> %a0, <2 x i64> %a1, <2 x i64>* %p2) nounwind {
-; SSE2-LABEL: saddo_v2i64:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
-; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    paddq %xmm1, %xmm0
-; SSE2-NEXT:    pxor %xmm2, %xmm1
-; SSE2-NEXT:    movdqa %xmm2, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pand %xmm5, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm4
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pxor %xmm1, %xmm4
-; SSE2-NEXT:    pxor %xmm2, %xmm3
-; SSE2-NEXT:    movdqa %xmm2, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm5
-; SSE2-NEXT:    pxor %xmm1, %xmm5
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,0,3,2]
-; SSE2-NEXT:    pand %xmm4, %xmm3
-; SSE2-NEXT:    movdqa %xmm0, (%rdi)
-; SSE2-NEXT:    pxor %xmm2, %xmm0
-; SSE2-NEXT:    movdqa %xmm2, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm0, %xmm2
-; SSE2-NEXT:    pxor %xmm1, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,0,3,2]
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    pandn %xmm3, %xmm0
-; SSE2-NEXT:    retq
-;
-; SSSE3-LABEL: saddo_v2i64:
-; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
-; SSSE3-NEXT:    movdqa %xmm0, %xmm3
-; SSSE3-NEXT:    paddq %xmm1, %xmm0
-; SSSE3-NEXT:    pxor %xmm2, %xmm1
-; SSSE3-NEXT:    movdqa %xmm2, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm5, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    por %xmm1, %xmm4
-; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSSE3-NEXT:    pxor %xmm1, %xmm4
-; SSSE3-NEXT:    pxor %xmm2, %xmm3
-; SSSE3-NEXT:    movdqa %xmm2, %xmm5
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm5
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm6, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSSE3-NEXT:    por %xmm3, %xmm5
-; SSSE3-NEXT:    pxor %xmm1, %xmm5
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm4, %xmm3
-; SSSE3-NEXT:    movdqa %xmm0, (%rdi)
-; SSSE3-NEXT:    pxor %xmm2, %xmm0
-; SSSE3-NEXT:    movdqa %xmm2, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm6, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    por %xmm0, %xmm2
-; SSSE3-NEXT:    pxor %xmm1, %xmm2
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm2, %xmm0
-; SSSE3-NEXT:    pandn %xmm3, %xmm0
-; SSSE3-NEXT:    retq
-;
-; SSE41-LABEL: saddo_v2i64:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
-; SSE41-NEXT:    movdqa %xmm0, %xmm3
-; SSE41-NEXT:    paddq %xmm1, %xmm0
-; SSE41-NEXT:    pxor %xmm2, %xmm1
-; SSE41-NEXT:    movdqa %xmm2, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm1, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm2, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE41-NEXT:    por %xmm1, %xmm4
-; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE41-NEXT:    pxor %xmm1, %xmm4
-; SSE41-NEXT:    pxor %xmm2, %xmm3
-; SSE41-NEXT:    movdqa %xmm2, %xmm5
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm2, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pand %xmm6, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE41-NEXT:    por %xmm3, %xmm5
-; SSE41-NEXT:    pxor %xmm1, %xmm5
-; SSE41-NEXT:    pcmpeqq %xmm5, %xmm4
-; SSE41-NEXT:    movdqa %xmm0, (%rdi)
-; SSE41-NEXT:    pxor %xmm2, %xmm0
-; SSE41-NEXT:    movdqa %xmm2, %xmm3
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm2, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm6, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
-; SSE41-NEXT:    pxor %xmm1, %xmm0
-; SSE41-NEXT:    pcmpeqq %xmm5, %xmm0
-; SSE41-NEXT:    pandn %xmm4, %xmm0
-; SSE41-NEXT:    retq
+; SSE-LABEL: saddo_v2i64:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
+; SSE-NEXT:    movdqa %xmm0, %xmm3
+; SSE-NEXT:    pxor %xmm2, %xmm3
+; SSE-NEXT:    paddq %xmm1, %xmm0
+; SSE-NEXT:    pxor %xmm0, %xmm2
+; SSE-NEXT:    movdqa %xmm3, %xmm4
+; SSE-NEXT:    pcmpgtd %xmm2, %xmm4
+; SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE-NEXT:    pcmpeqd %xmm3, %xmm2
+; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE-NEXT:    pand %xmm5, %xmm2
+; SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
+; SSE-NEXT:    por %xmm2, %xmm3
+; SSE-NEXT:    pxor %xmm2, %xmm2
+; SSE-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,3,3]
+; SSE-NEXT:    pxor %xmm3, %xmm1
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; SSE-NEXT:    movdqa %xmm0, (%rdi)
+; SSE-NEXT:    movdqa %xmm1, %xmm0
+; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: saddo_v2i64:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm2, %xmm3
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm5
-; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX1-NEXT:    vpcmpeqq %xmm3, %xmm5, %xmm3
+; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm2, %xmm2
 ; AVX1-NEXT:    vpaddq %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm2, %xmm0
-; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpeqq %xmm0, %xmm5, %xmm0
-; AVX1-NEXT:    vpandn %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor %xmm0, %xmm2, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; AVX1-NEXT:    vmovdqa %xmm1, (%rdi)
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: saddo_v2i64:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtq %xmm1, %xmm2, %xmm3
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpxor %xmm4, %xmm3, %xmm3
-; AVX2-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm5
-; AVX2-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX2-NEXT:    vpcmpeqq %xmm3, %xmm5, %xmm3
+; AVX2-NEXT:    vpcmpgtq %xmm1, %xmm2, %xmm2
 ; AVX2-NEXT:    vpaddq %xmm1, %xmm0, %xmm1
-; AVX2-NEXT:    vpcmpgtq %xmm1, %xmm2, %xmm0
-; AVX2-NEXT:    vpxor %xmm4, %xmm0, %xmm0
-; AVX2-NEXT:    vpcmpeqq %xmm0, %xmm5, %xmm0
-; AVX2-NEXT:    vpandn %xmm3, %xmm0, %xmm0
+; AVX2-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor %xmm0, %xmm2, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; AVX2-NEXT:    vmovdqa %xmm1, (%rdi)
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: saddo_v2i64:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpcmpnltq %xmm2, %xmm1, %k0
-; AVX512-NEXT:    vpcmpnltq %xmm2, %xmm0, %k1
-; AVX512-NEXT:    kxorw %k0, %k1, %k0
-; AVX512-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpcmpnltq %xmm2, %xmm0, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
-; AVX512-NEXT:    kandnw %k1, %k0, %k1
-; AVX512-NEXT:    vmovdqa %xmm0, (%rdi)
+; AVX512-NEXT:    vpcmpgtq %xmm1, %xmm2, %k0
+; AVX512-NEXT:    vpaddq %xmm1, %xmm0, %xmm1
+; AVX512-NEXT:    vpcmpgtq %xmm1, %xmm0, %k1
+; AVX512-NEXT:    kxorw %k1, %k0, %k1
 ; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX512-NEXT:    vmovdqa64 %xmm0, %xmm0 {%k1} {z}
+; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
+; AVX512-NEXT:    vmovdqa %xmm1, (%rdi)
 ; AVX512-NEXT:    retq
   %t = call {<2 x i64>, <2 x i1>} @llvm.sadd.with.overflow.v2i64(<2 x i64> %a0, <2 x i64> %a1)
   %val = extractvalue {<2 x i64>, <2 x i1>} %t, 0
@@ -1622,44 +1146,44 @@ define <2 x i32> @saddo_v2i128(<2 x i128> %a0, <2 x i128> %a1, <2 x i128>* %p2)
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pushq %rbp
 ; SSE2-NEXT:    pushq %rbx
-; SSE2-NEXT:    movq {{[0-9]+}}(%rsp), %r10
 ; SSE2-NEXT:    movq {{[0-9]+}}(%rsp), %r11
+; SSE2-NEXT:    movq {{[0-9]+}}(%rsp), %r10
+; SSE2-NEXT:    testq %r9, %r9
+; SSE2-NEXT:    setns %al
+; SSE2-NEXT:    testq %rsi, %rsi
+; SSE2-NEXT:    setns %bl
+; SSE2-NEXT:    cmpb %al, %bl
+; SSE2-NEXT:    sete %bpl
+; SSE2-NEXT:    addq %r8, %rdi
+; SSE2-NEXT:    adcq %r9, %rsi
+; SSE2-NEXT:    setns %al
+; SSE2-NEXT:    cmpb %al, %bl
+; SSE2-NEXT:    setne %al
+; SSE2-NEXT:    andb %bpl, %al
 ; SSE2-NEXT:    addq {{[0-9]+}}(%rsp), %rdx
-; SSE2-NEXT:    movq %rcx, %rax
-; SSE2-NEXT:    adcq %r11, %rax
+; SSE2-NEXT:    movq %rcx, %rbp
+; SSE2-NEXT:    adcq %r10, %rbp
 ; SSE2-NEXT:    setns %bl
 ; SSE2-NEXT:    testq %rcx, %rcx
 ; SSE2-NEXT:    setns %cl
 ; SSE2-NEXT:    cmpb %bl, %cl
-; SSE2-NEXT:    setne %bpl
-; SSE2-NEXT:    testq %r11, %r11
+; SSE2-NEXT:    setne %r8b
+; SSE2-NEXT:    testq %r10, %r10
 ; SSE2-NEXT:    setns %bl
 ; SSE2-NEXT:    cmpb %bl, %cl
 ; SSE2-NEXT:    sete %cl
-; SSE2-NEXT:    andb %bpl, %cl
-; SSE2-NEXT:    movzbl %cl, %ebp
-; SSE2-NEXT:    testq %r9, %r9
-; SSE2-NEXT:    setns %bl
-; SSE2-NEXT:    testq %rsi, %rsi
-; SSE2-NEXT:    setns %cl
-; SSE2-NEXT:    cmpb %bl, %cl
-; SSE2-NEXT:    sete %r11b
-; SSE2-NEXT:    addq %r8, %rdi
-; SSE2-NEXT:    adcq %r9, %rsi
-; SSE2-NEXT:    setns %bl
-; SSE2-NEXT:    cmpb %bl, %cl
-; SSE2-NEXT:    setne %cl
-; SSE2-NEXT:    andb %r11b, %cl
+; SSE2-NEXT:    andb %r8b, %cl
 ; SSE2-NEXT:    movzbl %cl, %ecx
-; SSE2-NEXT:    movd %ecx, %xmm0
-; SSE2-NEXT:    pinsrw $4, %ebp, %xmm0
-; SSE2-NEXT:    movq %rdx, 16(%r10)
-; SSE2-NEXT:    movq %rdi, (%r10)
-; SSE2-NEXT:    movq %rax, 24(%r10)
-; SSE2-NEXT:    movq %rsi, 8(%r10)
-; SSE2-NEXT:    psllq $63, %xmm0
-; SSE2-NEXT:    psrad $31, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    negl %ecx
+; SSE2-NEXT:    movd %ecx, %xmm1
+; SSE2-NEXT:    movzbl %al, %eax
+; SSE2-NEXT:    negl %eax
+; SSE2-NEXT:    movd %eax, %xmm0
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT:    movq %rdx, 16(%r11)
+; SSE2-NEXT:    movq %rdi, (%r11)
+; SSE2-NEXT:    movq %rbp, 24(%r11)
+; SSE2-NEXT:    movq %rsi, 8(%r11)
 ; SSE2-NEXT:    popq %rbx
 ; SSE2-NEXT:    popq %rbp
 ; SSE2-NEXT:    retq
@@ -1668,44 +1192,44 @@ define <2 x i32> @saddo_v2i128(<2 x i128> %a0, <2 x i128> %a1, <2 x i128>* %p2)
 ; SSSE3:       # %bb.0:
 ; SSSE3-NEXT:    pushq %rbp
 ; SSSE3-NEXT:    pushq %rbx
-; SSSE3-NEXT:    movq {{[0-9]+}}(%rsp), %r10
 ; SSSE3-NEXT:    movq {{[0-9]+}}(%rsp), %r11
+; SSSE3-NEXT:    movq {{[0-9]+}}(%rsp), %r10
+; SSSE3-NEXT:    testq %r9, %r9
+; SSSE3-NEXT:    setns %al
+; SSSE3-NEXT:    testq %rsi, %rsi
+; SSSE3-NEXT:    setns %bl
+; SSSE3-NEXT:    cmpb %al, %bl
+; SSSE3-NEXT:    sete %bpl
+; SSSE3-NEXT:    addq %r8, %rdi
+; SSSE3-NEXT:    adcq %r9, %rsi
+; SSSE3-NEXT:    setns %al
+; SSSE3-NEXT:    cmpb %al, %bl
+; SSSE3-NEXT:    setne %al
+; SSSE3-NEXT:    andb %bpl, %al
 ; SSSE3-NEXT:    addq {{[0-9]+}}(%rsp), %rdx
-; SSSE3-NEXT:    movq %rcx, %rax
-; SSSE3-NEXT:    adcq %r11, %rax
+; SSSE3-NEXT:    movq %rcx, %rbp
+; SSSE3-NEXT:    adcq %r10, %rbp
 ; SSSE3-NEXT:    setns %bl
 ; SSSE3-NEXT:    testq %rcx, %rcx
 ; SSSE3-NEXT:    setns %cl
 ; SSSE3-NEXT:    cmpb %bl, %cl
-; SSSE3-NEXT:    setne %bpl
-; SSSE3-NEXT:    testq %r11, %r11
+; SSSE3-NEXT:    setne %r8b
+; SSSE3-NEXT:    testq %r10, %r10
 ; SSSE3-NEXT:    setns %bl
 ; SSSE3-NEXT:    cmpb %bl, %cl
 ; SSSE3-NEXT:    sete %cl
-; SSSE3-NEXT:    andb %bpl, %cl
-; SSSE3-NEXT:    movzbl %cl, %ebp
-; SSSE3-NEXT:    testq %r9, %r9
-; SSSE3-NEXT:    setns %bl
-; SSSE3-NEXT:    testq %rsi, %rsi
-; SSSE3-NEXT:    setns %cl
-; SSSE3-NEXT:    cmpb %bl, %cl
-; SSSE3-NEXT:    sete %r11b
-; SSSE3-NEXT:    addq %r8, %rdi
-; SSSE3-NEXT:    adcq %r9, %rsi
-; SSSE3-NEXT:    setns %bl
-; SSSE3-NEXT:    cmpb %bl, %cl
-; SSSE3-NEXT:    setne %cl
-; SSSE3-NEXT:    andb %r11b, %cl
+; SSSE3-NEXT:    andb %r8b, %cl
 ; SSSE3-NEXT:    movzbl %cl, %ecx
-; SSSE3-NEXT:    movd %ecx, %xmm0
-; SSSE3-NEXT:    pinsrw $4, %ebp, %xmm0
-; SSSE3-NEXT:    movq %rdx, 16(%r10)
-; SSSE3-NEXT:    movq %rdi, (%r10)
-; SSSE3-NEXT:    movq %rax, 24(%r10)
-; SSSE3-NEXT:    movq %rsi, 8(%r10)
-; SSSE3-NEXT:    psllq $63, %xmm0
-; SSSE3-NEXT:    psrad $31, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    negl %ecx
+; SSSE3-NEXT:    movd %ecx, %xmm1
+; SSSE3-NEXT:    movzbl %al, %eax
+; SSSE3-NEXT:    negl %eax
+; SSSE3-NEXT:    movd %eax, %xmm0
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSSE3-NEXT:    movq %rdx, 16(%r11)
+; SSSE3-NEXT:    movq %rdi, (%r11)
+; SSSE3-NEXT:    movq %rbp, 24(%r11)
+; SSSE3-NEXT:    movq %rsi, 8(%r11)
 ; SSSE3-NEXT:    popq %rbx
 ; SSSE3-NEXT:    popq %rbp
 ; SSSE3-NEXT:    retq
@@ -1714,44 +1238,43 @@ define <2 x i32> @saddo_v2i128(<2 x i128> %a0, <2 x i128> %a1, <2 x i128>* %p2)
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pushq %rbp
 ; SSE41-NEXT:    pushq %rbx
-; SSE41-NEXT:    movq {{[0-9]+}}(%rsp), %r10
 ; SSE41-NEXT:    movq {{[0-9]+}}(%rsp), %r11
+; SSE41-NEXT:    movq {{[0-9]+}}(%rsp), %r10
+; SSE41-NEXT:    testq %r9, %r9
+; SSE41-NEXT:    setns %al
+; SSE41-NEXT:    testq %rsi, %rsi
+; SSE41-NEXT:    setns %bl
+; SSE41-NEXT:    cmpb %al, %bl
+; SSE41-NEXT:    sete %bpl
+; SSE41-NEXT:    addq %r8, %rdi
+; SSE41-NEXT:    adcq %r9, %rsi
+; SSE41-NEXT:    setns %al
+; SSE41-NEXT:    cmpb %al, %bl
+; SSE41-NEXT:    setne %al
+; SSE41-NEXT:    andb %bpl, %al
 ; SSE41-NEXT:    addq {{[0-9]+}}(%rsp), %rdx
-; SSE41-NEXT:    movq %rcx, %rax
-; SSE41-NEXT:    adcq %r11, %rax
+; SSE41-NEXT:    movq %rcx, %rbp
+; SSE41-NEXT:    adcq %r10, %rbp
 ; SSE41-NEXT:    setns %bl
 ; SSE41-NEXT:    testq %rcx, %rcx
 ; SSE41-NEXT:    setns %cl
 ; SSE41-NEXT:    cmpb %bl, %cl
-; SSE41-NEXT:    setne %bpl
-; SSE41-NEXT:    testq %r11, %r11
+; SSE41-NEXT:    setne %r8b
+; SSE41-NEXT:    testq %r10, %r10
 ; SSE41-NEXT:    setns %bl
 ; SSE41-NEXT:    cmpb %bl, %cl
 ; SSE41-NEXT:    sete %cl
-; SSE41-NEXT:    andb %bpl, %cl
-; SSE41-NEXT:    movzbl %cl, %ebp
-; SSE41-NEXT:    testq %r9, %r9
-; SSE41-NEXT:    setns %bl
-; SSE41-NEXT:    testq %rsi, %rsi
-; SSE41-NEXT:    setns %cl
-; SSE41-NEXT:    cmpb %bl, %cl
-; SSE41-NEXT:    sete %r11b
-; SSE41-NEXT:    addq %r8, %rdi
-; SSE41-NEXT:    adcq %r9, %rsi
-; SSE41-NEXT:    setns %bl
-; SSE41-NEXT:    cmpb %bl, %cl
-; SSE41-NEXT:    setne %cl
-; SSE41-NEXT:    andb %r11b, %cl
+; SSE41-NEXT:    andb %r8b, %cl
 ; SSE41-NEXT:    movzbl %cl, %ecx
-; SSE41-NEXT:    movd %ecx, %xmm0
-; SSE41-NEXT:    pinsrb $8, %ebp, %xmm0
-; SSE41-NEXT:    movq %rdx, 16(%r10)
-; SSE41-NEXT:    movq %rdi, (%r10)
-; SSE41-NEXT:    movq %rax, 24(%r10)
-; SSE41-NEXT:    movq %rsi, 8(%r10)
-; SSE41-NEXT:    psllq $63, %xmm0
-; SSE41-NEXT:    psrad $31, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE41-NEXT:    negl %ecx
+; SSE41-NEXT:    movzbl %al, %eax
+; SSE41-NEXT:    negl %eax
+; SSE41-NEXT:    movd %eax, %xmm0
+; SSE41-NEXT:    pinsrd $1, %ecx, %xmm0
+; SSE41-NEXT:    movq %rdx, 16(%r11)
+; SSE41-NEXT:    movq %rdi, (%r11)
+; SSE41-NEXT:    movq %rbp, 24(%r11)
+; SSE41-NEXT:    movq %rsi, 8(%r11)
 ; SSE41-NEXT:    popq %rbx
 ; SSE41-NEXT:    popq %rbp
 ; SSE41-NEXT:    retq
@@ -1760,44 +1283,43 @@ define <2 x i32> @saddo_v2i128(<2 x i128> %a0, <2 x i128> %a1, <2 x i128>* %p2)
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    pushq %rbp
 ; AVX1-NEXT:    pushq %rbx
-; AVX1-NEXT:    movq {{[0-9]+}}(%rsp), %r10
 ; AVX1-NEXT:    movq {{[0-9]+}}(%rsp), %r11
+; AVX1-NEXT:    movq {{[0-9]+}}(%rsp), %r10
+; AVX1-NEXT:    testq %r9, %r9
+; AVX1-NEXT:    setns %al
+; AVX1-NEXT:    testq %rsi, %rsi
+; AVX1-NEXT:    setns %bl
+; AVX1-NEXT:    cmpb %al, %bl
+; AVX1-NEXT:    sete %bpl
+; AVX1-NEXT:    addq %r8, %rdi
+; AVX1-NEXT:    adcq %r9, %rsi
+; AVX1-NEXT:    setns %al
+; AVX1-NEXT:    cmpb %al, %bl
+; AVX1-NEXT:    setne %al
+; AVX1-NEXT:    andb %bpl, %al
 ; AVX1-NEXT:    addq {{[0-9]+}}(%rsp), %rdx
-; AVX1-NEXT:    movq %rcx, %rax
-; AVX1-NEXT:    adcq %r11, %rax
+; AVX1-NEXT:    movq %rcx, %rbp
+; AVX1-NEXT:    adcq %r10, %rbp
 ; AVX1-NEXT:    setns %bl
 ; AVX1-NEXT:    testq %rcx, %rcx
 ; AVX1-NEXT:    setns %cl
 ; AVX1-NEXT:    cmpb %bl, %cl
-; AVX1-NEXT:    setne %bpl
-; AVX1-NEXT:    testq %r11, %r11
+; AVX1-NEXT:    setne %r8b
+; AVX1-NEXT:    testq %r10, %r10
 ; AVX1-NEXT:    setns %bl
 ; AVX1-NEXT:    cmpb %bl, %cl
 ; AVX1-NEXT:    sete %cl
-; AVX1-NEXT:    andb %bpl, %cl
-; AVX1-NEXT:    movzbl %cl, %ebp
-; AVX1-NEXT:    testq %r9, %r9
-; AVX1-NEXT:    setns %bl
-; AVX1-NEXT:    testq %rsi, %rsi
-; AVX1-NEXT:    setns %cl
-; AVX1-NEXT:    cmpb %bl, %cl
-; AVX1-NEXT:    sete %r11b
-; AVX1-NEXT:    addq %r8, %rdi
-; AVX1-NEXT:    adcq %r9, %rsi
-; AVX1-NEXT:    setns %bl
-; AVX1-NEXT:    cmpb %bl, %cl
-; AVX1-NEXT:    setne %cl
-; AVX1-NEXT:    andb %r11b, %cl
+; AVX1-NEXT:    andb %r8b, %cl
 ; AVX1-NEXT:    movzbl %cl, %ecx
-; AVX1-NEXT:    vmovd %ecx, %xmm0
-; AVX1-NEXT:    vpinsrb $8, %ebp, %xmm0, %xmm0
-; AVX1-NEXT:    movq %rdx, 16(%r10)
-; AVX1-NEXT:    movq %rdi, (%r10)
-; AVX1-NEXT:    movq %rax, 24(%r10)
-; AVX1-NEXT:    movq %rsi, 8(%r10)
-; AVX1-NEXT:    vpsllq $63, %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    negl %ecx
+; AVX1-NEXT:    movzbl %al, %eax
+; AVX1-NEXT:    negl %eax
+; AVX1-NEXT:    vmovd %eax, %xmm0
+; AVX1-NEXT:    vpinsrd $1, %ecx, %xmm0, %xmm0
+; AVX1-NEXT:    movq %rdx, 16(%r11)
+; AVX1-NEXT:    movq %rdi, (%r11)
+; AVX1-NEXT:    movq %rbp, 24(%r11)
+; AVX1-NEXT:    movq %rsi, 8(%r11)
 ; AVX1-NEXT:    popq %rbx
 ; AVX1-NEXT:    popq %rbp
 ; AVX1-NEXT:    retq
@@ -1806,92 +1328,91 @@ define <2 x i32> @saddo_v2i128(<2 x i128> %a0, <2 x i128> %a1, <2 x i128>* %p2)
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    pushq %rbp
 ; AVX2-NEXT:    pushq %rbx
-; AVX2-NEXT:    movq {{[0-9]+}}(%rsp), %r10
 ; AVX2-NEXT:    movq {{[0-9]+}}(%rsp), %r11
+; AVX2-NEXT:    movq {{[0-9]+}}(%rsp), %r10
+; AVX2-NEXT:    testq %r9, %r9
+; AVX2-NEXT:    setns %al
+; AVX2-NEXT:    testq %rsi, %rsi
+; AVX2-NEXT:    setns %bl
+; AVX2-NEXT:    cmpb %al, %bl
+; AVX2-NEXT:    sete %bpl
+; AVX2-NEXT:    addq %r8, %rdi
+; AVX2-NEXT:    adcq %r9, %rsi
+; AVX2-NEXT:    setns %al
+; AVX2-NEXT:    cmpb %al, %bl
+; AVX2-NEXT:    setne %al
+; AVX2-NEXT:    andb %bpl, %al
 ; AVX2-NEXT:    addq {{[0-9]+}}(%rsp), %rdx
-; AVX2-NEXT:    movq %rcx, %rax
-; AVX2-NEXT:    adcq %r11, %rax
+; AVX2-NEXT:    movq %rcx, %rbp
+; AVX2-NEXT:    adcq %r10, %rbp
 ; AVX2-NEXT:    setns %bl
 ; AVX2-NEXT:    testq %rcx, %rcx
 ; AVX2-NEXT:    setns %cl
 ; AVX2-NEXT:    cmpb %bl, %cl
-; AVX2-NEXT:    setne %bpl
-; AVX2-NEXT:    testq %r11, %r11
+; AVX2-NEXT:    setne %r8b
+; AVX2-NEXT:    testq %r10, %r10
 ; AVX2-NEXT:    setns %bl
 ; AVX2-NEXT:    cmpb %bl, %cl
 ; AVX2-NEXT:    sete %cl
-; AVX2-NEXT:    andb %bpl, %cl
-; AVX2-NEXT:    movzbl %cl, %ebp
-; AVX2-NEXT:    testq %r9, %r9
-; AVX2-NEXT:    setns %bl
-; AVX2-NEXT:    testq %rsi, %rsi
-; AVX2-NEXT:    setns %cl
-; AVX2-NEXT:    cmpb %bl, %cl
-; AVX2-NEXT:    sete %r11b
-; AVX2-NEXT:    addq %r8, %rdi
-; AVX2-NEXT:    adcq %r9, %rsi
-; AVX2-NEXT:    setns %bl
-; AVX2-NEXT:    cmpb %bl, %cl
-; AVX2-NEXT:    setne %cl
-; AVX2-NEXT:    andb %r11b, %cl
+; AVX2-NEXT:    andb %r8b, %cl
 ; AVX2-NEXT:    movzbl %cl, %ecx
-; AVX2-NEXT:    vmovd %ecx, %xmm0
-; AVX2-NEXT:    vpinsrb $8, %ebp, %xmm0, %xmm0
-; AVX2-NEXT:    movq %rdx, 16(%r10)
-; AVX2-NEXT:    movq %rdi, (%r10)
-; AVX2-NEXT:    movq %rax, 24(%r10)
-; AVX2-NEXT:    movq %rsi, 8(%r10)
-; AVX2-NEXT:    vpsllq $63, %xmm0, %xmm0
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm0
+; AVX2-NEXT:    negl %ecx
+; AVX2-NEXT:    movzbl %al, %eax
+; AVX2-NEXT:    negl %eax
+; AVX2-NEXT:    vmovd %eax, %xmm0
+; AVX2-NEXT:    vpinsrd $1, %ecx, %xmm0, %xmm0
+; AVX2-NEXT:    movq %rdx, 16(%r11)
+; AVX2-NEXT:    movq %rdi, (%r11)
+; AVX2-NEXT:    movq %rbp, 24(%r11)
+; AVX2-NEXT:    movq %rsi, 8(%r11)
 ; AVX2-NEXT:    popq %rbx
 ; AVX2-NEXT:    popq %rbp
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: saddo_v2i128:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    pushq %r14
+; AVX512-NEXT:    pushq %rbp
 ; AVX512-NEXT:    pushq %rbx
-; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %r10
 ; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %r11
-; AVX512-NEXT:    addq {{[0-9]+}}(%rsp), %rdx
-; AVX512-NEXT:    movq %rcx, %r14
-; AVX512-NEXT:    adcq %r11, %r14
-; AVX512-NEXT:    setns %bl
-; AVX512-NEXT:    testq %rcx, %rcx
-; AVX512-NEXT:    setns %cl
-; AVX512-NEXT:    cmpb %bl, %cl
-; AVX512-NEXT:    setne %bl
-; AVX512-NEXT:    testq %r11, %r11
-; AVX512-NEXT:    setns %al
-; AVX512-NEXT:    cmpb %al, %cl
-; AVX512-NEXT:    sete %al
-; AVX512-NEXT:    andb %bl, %al
-; AVX512-NEXT:    kmovd %eax, %k0
-; AVX512-NEXT:    kshiftlw $1, %k0, %k0
+; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %r10
 ; AVX512-NEXT:    testq %r9, %r9
 ; AVX512-NEXT:    setns %al
 ; AVX512-NEXT:    testq %rsi, %rsi
-; AVX512-NEXT:    setns %cl
-; AVX512-NEXT:    cmpb %al, %cl
-; AVX512-NEXT:    sete %al
+; AVX512-NEXT:    setns %bl
+; AVX512-NEXT:    cmpb %al, %bl
+; AVX512-NEXT:    sete %bpl
 ; AVX512-NEXT:    addq %r8, %rdi
 ; AVX512-NEXT:    adcq %r9, %rsi
+; AVX512-NEXT:    setns %al
+; AVX512-NEXT:    cmpb %al, %bl
+; AVX512-NEXT:    setne %al
+; AVX512-NEXT:    andb %bpl, %al
+; AVX512-NEXT:    addq {{[0-9]+}}(%rsp), %rdx
+; AVX512-NEXT:    movq %rcx, %rbp
+; AVX512-NEXT:    adcq %r10, %rbp
 ; AVX512-NEXT:    setns %bl
+; AVX512-NEXT:    testq %rcx, %rcx
+; AVX512-NEXT:    setns %cl
 ; AVX512-NEXT:    cmpb %bl, %cl
-; AVX512-NEXT:    setne %cl
-; AVX512-NEXT:    andb %al, %cl
-; AVX512-NEXT:    andl $1, %ecx
-; AVX512-NEXT:    kmovw %ecx, %k1
+; AVX512-NEXT:    setne %r8b
+; AVX512-NEXT:    testq %r10, %r10
+; AVX512-NEXT:    setns %bl
+; AVX512-NEXT:    cmpb %bl, %cl
+; AVX512-NEXT:    sete %cl
+; AVX512-NEXT:    andb %r8b, %cl
+; AVX512-NEXT:    kmovd %ecx, %k0
+; AVX512-NEXT:    kshiftlw $1, %k0, %k0
+; AVX512-NEXT:    andl $1, %eax
+; AVX512-NEXT:    kmovw %eax, %k1
 ; AVX512-NEXT:    korw %k0, %k1, %k1
-; AVX512-NEXT:    movq %rdx, 16(%r10)
-; AVX512-NEXT:    movq %rdi, (%r10)
-; AVX512-NEXT:    movq %r14, 24(%r10)
-; AVX512-NEXT:    movq %rsi, 8(%r10)
 ; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX512-NEXT:    vmovdqa64 %xmm0, %xmm0 {%k1} {z}
+; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
+; AVX512-NEXT:    movq %rdx, 16(%r11)
+; AVX512-NEXT:    movq %rdi, (%r11)
+; AVX512-NEXT:    movq %rbp, 24(%r11)
+; AVX512-NEXT:    movq %rsi, 8(%r11)
 ; AVX512-NEXT:    popq %rbx
-; AVX512-NEXT:    popq %r14
+; AVX512-NEXT:    popq %rbp
 ; AVX512-NEXT:    retq
   %t = call {<2 x i128>, <2 x i1>} @llvm.sadd.with.overflow.v2i128(<2 x i128> %a0, <2 x i128> %a1)
   %val = extractvalue {<2 x i128>, <2 x i1>} %t, 0
diff --git a/llvm/test/CodeGen/X86/vec_setcc.ll b/llvm/test/CodeGen/X86/vec_setcc.ll
index e9494d845..127222b3c 100644
--- a/llvm/test/CodeGen/X86/vec_setcc.ll
+++ b/llvm/test/CodeGen/X86/vec_setcc.ll
@@ -197,3 +197,48 @@ define <2 x i64> @test_setcc_constfold_vi64(<2 x i64> %l, <2 x i64> %r) {
   %res = or <2 x i64> %mask1, %mask2
   ret <2 x i64> %res
 }
+
+; This asserted in type legalization for v3i1 setcc after v3i16 was made
+; a simple value type.
+define <3 x i1> @test_setcc_v3i1_v3i16(<3 x i16>* %a) nounwind {
+; SSE2-LABEL: test_setcc_v3i1_v3i16:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    pcmpeqw %xmm0, %xmm1
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; SSE2-NEXT:    movdqa %xmm1, -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    movb -{{[0-9]+}}(%rsp), %al
+; SSE2-NEXT:    movb -{{[0-9]+}}(%rsp), %dl
+; SSE2-NEXT:    movb -{{[0-9]+}}(%rsp), %cl
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: test_setcc_v3i1_v3i16:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; SSE41-NEXT:    pxor %xmm1, %xmm1
+; SSE41-NEXT:    pcmpeqw %xmm0, %xmm1
+; SSE41-NEXT:    pextrb $0, %xmm1, %eax
+; SSE41-NEXT:    pextrb $2, %xmm1, %edx
+; SSE41-NEXT:    pextrb $4, %xmm1, %ecx
+; SSE41-NEXT:    # kill: def $al killed $al killed $eax
+; SSE41-NEXT:    # kill: def $dl killed $dl killed $edx
+; SSE41-NEXT:    # kill: def $cl killed $cl killed $ecx
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: test_setcc_v3i1_v3i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
+; AVX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vpcmpeqw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX-NEXT:    vpextrb $2, %xmm0, %edx
+; AVX-NEXT:    vpextrb $4, %xmm0, %ecx
+; AVX-NEXT:    # kill: def $al killed $al killed $eax
+; AVX-NEXT:    # kill: def $dl killed $dl killed $edx
+; AVX-NEXT:    # kill: def $cl killed $cl killed $ecx
+; AVX-NEXT:    retq
+  %b = load <3 x i16>, <3 x i16>* %a
+  %cmp = icmp eq <3 x i16> %b, <i16 0, i16 0, i16 0>
+  ret <3 x i1> %cmp
+}
diff --git a/llvm/test/CodeGen/X86/vec_shift6.ll b/llvm/test/CodeGen/X86/vec_shift6.ll
index 1415d8549..214a4c859 100644
--- a/llvm/test/CodeGen/X86/vec_shift6.ll
+++ b/llvm/test/CodeGen/X86/vec_shift6.ll
@@ -162,13 +162,23 @@ define <32 x i16> @test7(<32 x i16> %a) {
 ; SSE-NEXT:    pmullw %xmm4, %xmm3
 ; SSE-NEXT:    retq
 ;
-; AVX-LABEL: test7:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [2,2,4,8,128,1,512,2048,2,2,4,8,128,1,512,2048]
-; AVX-NEXT:    # ymm2 = mem[0,1,0,1]
-; AVX-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
-; AVX-NEXT:    vpmullw %ymm2, %ymm1, %ymm1
-; AVX-NEXT:    retq
+; AVX2-LABEL: test7:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [2,2,4,8,128,1,512,2048,2,2,4,8,128,1,512,2048]
+; AVX2-NEXT:    # ymm2 = mem[0,1,0,1]
+; AVX2-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpmullw %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: test7:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
+; AVX512-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [2,2,4,8,128,1,512,2048,2,2,4,8,128,1,512,2048]
+; AVX512-NEXT:    # ymm2 = mem[0,1,0,1]
+; AVX512-NEXT:    vpmullw %ymm2, %ymm1, %ymm1
+; AVX512-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
+; AVX512-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512-NEXT:    retq
   %shl = shl <32 x i16> %a, <i16 1, i16 1, i16 2, i16 3, i16 7, i16 0, i16 9, i16 11, i16 1, i16 1, i16 2, i16 3, i16 7, i16 0, i16 9, i16 11, i16 1, i16 1, i16 2, i16 3, i16 7, i16 0, i16 9, i16 11, i16 1, i16 1, i16 2, i16 3, i16 7, i16 0, i16 9, i16 11>
   ret <32 x i16> %shl
 }
diff --git a/llvm/test/CodeGen/X86/vec_shuf-insert.ll b/llvm/test/CodeGen/X86/vec_shuf-insert.ll
index 2e1a1d613..555feab32 100644
--- a/llvm/test/CodeGen/X86/vec_shuf-insert.ll
+++ b/llvm/test/CodeGen/X86/vec_shuf-insert.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-unknown-linux -mcpu=corei7-avx | FileCheck %s
 
 ; These tests check that an insert_subvector which replaces one of the halves
@@ -7,23 +8,25 @@
 declare <8 x float> @llvm.x86.avx.vinsertf128.ps.256(<8 x float>, <4 x float>, i8)
 
 define <8 x float> @lower_half(<4 x float> %v1, <4 x float> %v2, <4 x float> %v3) {
+; CHECK-LABEL: lower_half:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    # kill: def $xmm2 killed $xmm2 def $ymm2
+; CHECK-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm0
+; CHECK-NEXT:    retq
   %1 = shufflevector <4 x float> %v1, <4 x float> %v2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   %2 = tail call <8 x float> @llvm.x86.avx.vinsertf128.ps.256(<8 x float> %1, <4 x float> %v3, i8 0)
   ret <8 x float> %2
 
-; CHECK-LABEL: lower_half
-; CHECK-NOT: vinsertf128
-; CHECK: vinsertf128 $1, %xmm1, %ymm2, %ymm0
-; CHECK-NEXT: ret
 }
 
 define <8 x float> @upper_half(<4 x float> %v1, <4 x float> %v2, <4 x float> %v3) {
+; CHECK-LABEL: upper_half:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; CHECK-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; CHECK-NEXT:    retq
   %1 = shufflevector <4 x float> %v1, <4 x float> %v2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   %2 = tail call <8 x float> @llvm.x86.avx.vinsertf128.ps.256(<8 x float> %1, <4 x float> %v3, i8 1)
   ret <8 x float> %2
 
-; CHECK-LABEL: upper_half
-; CHECK-NOT: vinsertf128
-; CHECK: vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; CHECK-NEXT: ret
 }
diff --git a/llvm/test/CodeGen/X86/vec_smulo.ll b/llvm/test/CodeGen/X86/vec_smulo.ll
index 3f53f9f22..b2639ca33 100644
--- a/llvm/test/CodeGen/X86/vec_smulo.ll
+++ b/llvm/test/CodeGen/X86/vec_smulo.ll
@@ -51,238 +51,123 @@ define <1 x i32> @smulo_v1i32(<1 x i32> %a0, <1 x i32> %a1, <1 x i32>* %p2) noun
 define <2 x i32> @smulo_v2i32(<2 x i32> %a0, <2 x i32> %a1, <2 x i32>* %p2) nounwind {
 ; SSE2-LABEL: smulo_v2i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    psllq $32, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,3,2,3]
-; SSE2-NEXT:    psrad $31, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,3,0,1]
-; SSE2-NEXT:    movq %xmm1, %r8
-; SSE2-NEXT:    psllq $32, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,3,2,3]
-; SSE2-NEXT:    psrad $31, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
-; SSE2-NEXT:    movq %xmm0, %rcx
-; SSE2-NEXT:    movq %xmm2, %rdx
-; SSE2-NEXT:    movq %xmm1, %rsi
-; SSE2-NEXT:    xorl %eax, %eax
-; SSE2-NEXT:    imulq %rdx, %rsi
-; SSE2-NEXT:    movq $-1, %r9
-; SSE2-NEXT:    movl $0, %edx
-; SSE2-NEXT:    cmovoq %r9, %rdx
-; SSE2-NEXT:    movq %rsi, %xmm1
-; SSE2-NEXT:    imulq %r8, %rcx
-; SSE2-NEXT:    movq %rcx, %xmm0
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
-; SSE2-NEXT:    movdqa %xmm1, %xmm0
-; SSE2-NEXT:    psllq $32, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,3,2,3]
-; SSE2-NEXT:    psrad $31, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,0,3,2]
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT:    pxor %xmm0, %xmm2
-; SSE2-NEXT:    movq %rdx, %xmm0
-; SSE2-NEXT:    cmovoq %r9, %rax
-; SSE2-NEXT:    movq %rax, %xmm3
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
-; SSE2-NEXT:    por %xmm2, %xmm0
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
+; SSE2-NEXT:    pand %xmm0, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSE2-NEXT:    pand %xmm1, %xmm2
+; SSE2-NEXT:    paddd %xmm3, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pmuludq %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,3,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pmuludq %xmm3, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,3,2,3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
+; SSE2-NEXT:    psubd %xmm2, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; SSE2-NEXT:    movq %xmm1, (%rdi)
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT:    movq %xmm0, (%rdi)
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psrad $31, %xmm1
+; SSE2-NEXT:    pcmpeqd %xmm4, %xmm1
+; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: smulo_v2i32:
 ; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    psllq $32, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,3,2,3]
-; SSSE3-NEXT:    psrad $31, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,3,0,1]
-; SSSE3-NEXT:    movq %xmm1, %r8
-; SSSE3-NEXT:    psllq $32, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,3,2,3]
-; SSSE3-NEXT:    psrad $31, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
-; SSSE3-NEXT:    movq %xmm0, %rcx
-; SSSE3-NEXT:    movq %xmm2, %rdx
-; SSSE3-NEXT:    movq %xmm1, %rsi
-; SSSE3-NEXT:    xorl %eax, %eax
-; SSSE3-NEXT:    imulq %rdx, %rsi
-; SSSE3-NEXT:    movq $-1, %r9
-; SSSE3-NEXT:    movl $0, %edx
-; SSSE3-NEXT:    cmovoq %r9, %rdx
-; SSSE3-NEXT:    movq %rsi, %xmm1
-; SSSE3-NEXT:    imulq %r8, %rcx
-; SSSE3-NEXT:    movq %rcx, %xmm0
-; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
-; SSSE3-NEXT:    movdqa %xmm1, %xmm0
-; SSSE3-NEXT:    psllq $32, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,3,2,3]
-; SSSE3-NEXT:    psrad $31, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm2, %xmm0
-; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSSE3-NEXT:    pxor %xmm0, %xmm2
-; SSSE3-NEXT:    movq %rdx, %xmm0
-; SSSE3-NEXT:    cmovoq %r9, %rax
-; SSSE3-NEXT:    movq %rax, %xmm3
-; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    pxor %xmm2, %xmm2
+; SSSE3-NEXT:    pxor %xmm3, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm3
+; SSSE3-NEXT:    pand %xmm0, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSSE3-NEXT:    pand %xmm1, %xmm2
+; SSSE3-NEXT:    paddd %xmm3, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pmuludq %xmm1, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,3,2,3]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    pmuludq %xmm3, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,3,2,3]
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
+; SSSE3-NEXT:    psubd %xmm2, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; SSSE3-NEXT:    movq %xmm1, (%rdi)
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSSE3-NEXT:    movq %xmm0, (%rdi)
+; SSSE3-NEXT:    movdqa %xmm0, %xmm1
+; SSSE3-NEXT:    psrad $31, %xmm1
+; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm1
+; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm0
+; SSSE3-NEXT:    pxor %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: smulo_v2i32:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa %xmm1, %xmm2
-; SSE41-NEXT:    psllq $32, %xmm2
-; SSE41-NEXT:    psrad $31, %xmm2
-; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
-; SSE41-NEXT:    movq %xmm2, %r8
-; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    psllq $32, %xmm1
-; SSE41-NEXT:    psrad $31, %xmm1
-; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
-; SSE41-NEXT:    movq %xmm1, %rcx
-; SSE41-NEXT:    pextrq $1, %xmm2, %rdx
-; SSE41-NEXT:    pextrq $1, %xmm1, %rsi
-; SSE41-NEXT:    xorl %eax, %eax
-; SSE41-NEXT:    imulq %rdx, %rsi
-; SSE41-NEXT:    movq $-1, %r9
-; SSE41-NEXT:    movl $0, %edx
-; SSE41-NEXT:    cmovoq %r9, %rdx
-; SSE41-NEXT:    movq %rsi, %xmm0
-; SSE41-NEXT:    imulq %r8, %rcx
-; SSE41-NEXT:    movq %rcx, %xmm1
-; SSE41-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    psllq $32, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; SSE41-NEXT:    pmuldq %xmm2, %xmm3
+; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    pmuldq %xmm1, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3],xmm2[4,5],xmm3[6,7]
+; SSE41-NEXT:    pmulld %xmm1, %xmm0
+; SSE41-NEXT:    movq %xmm0, (%rdi)
 ; SSE41-NEXT:    psrad $31, %xmm0
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3],xmm1[4,5],xmm0[6,7]
-; SSE41-NEXT:    pcmpeqq %xmm1, %xmm0
-; SSE41-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE41-NEXT:    pxor %xmm0, %xmm2
-; SSE41-NEXT:    movq %rdx, %xmm3
-; SSE41-NEXT:    cmovoq %r9, %rax
-; SSE41-NEXT:    movq %rax, %xmm0
-; SSE41-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
-; SSE41-NEXT:    por %xmm2, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; SSE41-NEXT:    movq %xmm1, (%rdi)
+; SSE41-NEXT:    pcmpeqd %xmm2, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
+; SSE41-NEXT:    pxor %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: smulo_v2i32:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpsllq $32, %xmm1, %xmm2
-; AVX1-NEXT:    vpsrad $31, %xmm2, %xmm2
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
-; AVX1-NEXT:    vmovq %xmm1, %r8
-; AVX1-NEXT:    vpsllq $32, %xmm0, %xmm2
-; AVX1-NEXT:    vpsrad $31, %xmm2, %xmm2
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
-; AVX1-NEXT:    vmovq %xmm0, %rcx
-; AVX1-NEXT:    vpextrq $1, %xmm1, %rdx
-; AVX1-NEXT:    vpextrq $1, %xmm0, %rsi
-; AVX1-NEXT:    xorl %eax, %eax
-; AVX1-NEXT:    imulq %rdx, %rsi
-; AVX1-NEXT:    movq $-1, %r9
-; AVX1-NEXT:    movl $0, %edx
-; AVX1-NEXT:    cmovoq %r9, %rdx
-; AVX1-NEXT:    vmovq %rsi, %xmm0
-; AVX1-NEXT:    imulq %r8, %rcx
-; AVX1-NEXT:    vmovq %rcx, %xmm1
-; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
-; AVX1-NEXT:    vpsllq $32, %xmm1, %xmm0
-; AVX1-NEXT:    vpsrad $31, %xmm0, %xmm0
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3],xmm1[4,5],xmm0[6,7]
-; AVX1-NEXT:    vpcmpeqq %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; AVX1-NEXT:    vpmuldq %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vpmuldq %xmm1, %xmm0, %xmm3
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
+; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vpsrad $31, %xmm1, %xmm0
+; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm2, %xmm0
 ; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
 ; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vmovq %rdx, %xmm2
-; AVX1-NEXT:    cmovoq %r9, %rax
-; AVX1-NEXT:    vmovq %rax, %xmm3
-; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
-; AVX1-NEXT:    vpor %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
 ; AVX1-NEXT:    vmovq %xmm1, (%rdi)
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: smulo_v2i32:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpsllq $32, %xmm1, %xmm2
-; AVX2-NEXT:    vpsrad $31, %xmm2, %xmm2
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; AVX2-NEXT:    vmovq %xmm1, %r8
-; AVX2-NEXT:    vpsllq $32, %xmm0, %xmm2
-; AVX2-NEXT:    vpsrad $31, %xmm2, %xmm2
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
-; AVX2-NEXT:    vmovq %xmm0, %rcx
-; AVX2-NEXT:    vpextrq $1, %xmm1, %rdx
-; AVX2-NEXT:    vpextrq $1, %xmm0, %rsi
-; AVX2-NEXT:    xorl %eax, %eax
-; AVX2-NEXT:    imulq %rdx, %rsi
-; AVX2-NEXT:    movq $-1, %r9
-; AVX2-NEXT:    movl $0, %edx
-; AVX2-NEXT:    cmovoq %r9, %rdx
-; AVX2-NEXT:    vmovq %rsi, %xmm0
-; AVX2-NEXT:    imulq %r8, %rcx
-; AVX2-NEXT:    vmovq %rcx, %xmm1
-; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
-; AVX2-NEXT:    vpsllq $32, %xmm1, %xmm0
-; AVX2-NEXT:    vpsrad $31, %xmm0, %xmm0
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2],xmm0[3]
-; AVX2-NEXT:    vpcmpeqq %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; AVX2-NEXT:    vpmuldq %xmm2, %xmm3, %xmm2
+; AVX2-NEXT:    vpmuldq %xmm1, %xmm0, %xmm3
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; AVX2-NEXT:    vpblendd {{.*#+}} xmm2 = xmm3[0],xmm2[1],xmm3[2],xmm2[3]
+; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
+; AVX2-NEXT:    vpsrad $31, %xmm1, %xmm0
+; AVX2-NEXT:    vpcmpeqd %xmm0, %xmm2, %xmm0
 ; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
 ; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm0
-; AVX2-NEXT:    vmovq %rdx, %xmm2
-; AVX2-NEXT:    cmovoq %r9, %rax
-; AVX2-NEXT:    vmovq %rax, %xmm3
-; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
-; AVX2-NEXT:    vpor %xmm2, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
 ; AVX2-NEXT:    vmovq %xmm1, (%rdi)
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: smulo_v2i32:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsllq $32, %xmm1, %xmm1
-; AVX512-NEXT:    vpsraq $32, %xmm1, %xmm1
-; AVX512-NEXT:    vmovq %xmm1, %rax
-; AVX512-NEXT:    vpsllq $32, %xmm0, %xmm0
-; AVX512-NEXT:    vpsraq $32, %xmm0, %xmm0
-; AVX512-NEXT:    vmovq %xmm0, %rcx
-; AVX512-NEXT:    vpextrq $1, %xmm1, %rdx
-; AVX512-NEXT:    vpextrq $1, %xmm0, %rsi
-; AVX512-NEXT:    imulq %rdx, %rsi
-; AVX512-NEXT:    seto %dl
-; AVX512-NEXT:    vmovq %rsi, %xmm0
-; AVX512-NEXT:    imulq %rax, %rcx
-; AVX512-NEXT:    vmovq %rcx, %xmm1
-; AVX512-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX512-NEXT:    vpsllq $32, %xmm0, %xmm1
-; AVX512-NEXT:    vpsraq $32, %xmm1, %xmm1
-; AVX512-NEXT:    vpcmpneqq %xmm0, %xmm1, %k0
-; AVX512-NEXT:    kmovd %edx, %k1
-; AVX512-NEXT:    kshiftlw $1, %k1, %k1
-; AVX512-NEXT:    seto %al
-; AVX512-NEXT:    andl $1, %eax
-; AVX512-NEXT:    kmovw %eax, %k2
-; AVX512-NEXT:    korw %k1, %k2, %k1
-; AVX512-NEXT:    korw %k1, %k0, %k1
-; AVX512-NEXT:    vpmovqd %xmm0, (%rdi)
+; AVX512-NEXT:    vpmuldq %xmm1, %xmm0, %xmm2
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
+; AVX512-NEXT:    vpmuldq %xmm3, %xmm4, %xmm3
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm4 = [1,5,3,7]
+; AVX512-NEXT:    vpermi2d %xmm3, %xmm2, %xmm4
+; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
+; AVX512-NEXT:    vpsrad $31, %xmm1, %xmm0
+; AVX512-NEXT:    vpcmpneqd %xmm0, %xmm4, %k1
 ; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX512-NEXT:    vmovdqa64 %xmm0, %xmm0 {%k1} {z}
+; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
+; AVX512-NEXT:    vmovq %xmm1, (%rdi)
 ; AVX512-NEXT:    retq
   %t = call {<2 x i32>, <2 x i1>} @llvm.smul.with.overflow.v2i32(<2 x i32> %a0, <2 x i32> %a1)
   %val = extractvalue {<2 x i32>, <2 x i1>} %t, 0
@@ -310,17 +195,17 @@ define <3 x i32> @smulo_v3i32(<3 x i32> %a0, <3 x i32> %a1, <3 x i32>* %p2) noun
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,3,2,3]
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
 ; SSE2-NEXT:    psubd %xmm2, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,2,2,3]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; SSE2-NEXT:    movq %xmm0, (%rdi)
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSE2-NEXT:    movq %xmm2, (%rdi)
 ; SSE2-NEXT:    psrad $31, %xmm2
 ; SSE2-NEXT:    pcmpeqd %xmm4, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE2-NEXT:    pxor %xmm2, %xmm0
-; SSE2-NEXT:    movd %xmm1, 8(%rdi)
+; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
+; SSE2-NEXT:    pxor %xmm2, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; SSE2-NEXT:    movd %xmm0, 8(%rdi)
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: smulo_v3i32:
@@ -340,17 +225,17 @@ define <3 x i32> @smulo_v3i32(<3 x i32> %a0, <3 x i32> %a1, <3 x i32>* %p2) noun
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,3,2,3]
 ; SSSE3-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
 ; SSSE3-NEXT:    psubd %xmm2, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,2,2,3]
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; SSSE3-NEXT:    movq %xmm0, (%rdi)
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; SSSE3-NEXT:    movdqa %xmm0, %xmm2
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSSE3-NEXT:    movq %xmm2, (%rdi)
 ; SSSE3-NEXT:    psrad $31, %xmm2
 ; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm2
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSSE3-NEXT:    pxor %xmm2, %xmm0
-; SSSE3-NEXT:    movd %xmm1, 8(%rdi)
+; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm1
+; SSSE3-NEXT:    pxor %xmm2, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; SSSE3-NEXT:    movd %xmm0, 8(%rdi)
+; SSSE3-NEXT:    movdqa %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: smulo_v3i32:
@@ -562,188 +447,190 @@ define <6 x i32> @smulo_v6i32(<6 x i32> %a0, <6 x i32> %a1, <6 x i32>* %p2) noun
 ; SSE2-LABEL: smulo_v6i32:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movq %rdi, %rax
-; SSE2-NEXT:    movd %r8d, %xmm9
+; SSE2-NEXT:    movd %r8d, %xmm8
 ; SSE2-NEXT:    movd %ecx, %xmm0
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm9[0],xmm0[1],xmm9[1]
-; SSE2-NEXT:    movd %edx, %xmm6
-; SSE2-NEXT:    movd %esi, %xmm5
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm0[0]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm8[0],xmm0[1],xmm8[1]
+; SSE2-NEXT:    movd %edx, %xmm3
+; SSE2-NEXT:    movd %esi, %xmm6
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm6 = xmm6[0],xmm3[0],xmm6[1],xmm3[1]
+; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm6 = xmm6[0],xmm0[0]
 ; SSE2-NEXT:    movd {{.*#+}} xmm10 = mem[0],zero,zero,zero
 ; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm10[0],xmm0[1],xmm10[1]
-; SSE2-NEXT:    movd {{.*#+}} xmm7 = mem[0],zero,zero,zero
-; SSE2-NEXT:    movd {{.*#+}} xmm3 = mem[0],zero,zero,zero
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm7[0],xmm3[1],xmm7[1]
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm0[0]
-; SSE2-NEXT:    movd %r9d, %xmm12
-; SSE2-NEXT:    movd {{.*#+}} xmm11 = mem[0],zero,zero,zero
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm12 = xmm12[0],xmm11[0],xmm12[1],xmm11[1]
-; SSE2-NEXT:    movd {{.*#+}} xmm8 = mem[0],zero,zero,zero
+; SSE2-NEXT:    movd {{.*#+}} xmm12 = mem[0],zero,zero,zero
+; SSE2-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm12[0],xmm1[1],xmm12[1]
+; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSE2-NEXT:    movd %r9d, %xmm13
 ; SSE2-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm8[0],xmm2[1],xmm8[1]
+; SSE2-NEXT:    movdqa %xmm13, %xmm11
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm11 = xmm11[0],xmm2[0],xmm11[1],xmm2[1]
+; SSE2-NEXT:    movd {{.*#+}} xmm9 = mem[0],zero,zero,zero
+; SSE2-NEXT:    movd {{.*#+}} xmm7 = mem[0],zero,zero,zero
+; SSE2-NEXT:    pmuludq %xmm7, %xmm13
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm7 = xmm7[0],xmm9[0],xmm7[1],xmm9[1]
 ; SSE2-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
 ; SSE2-NEXT:    pxor %xmm4, %xmm4
+; SSE2-NEXT:    pxor %xmm5, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm5
+; SSE2-NEXT:    pand %xmm6, %xmm5
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE2-NEXT:    pand %xmm5, %xmm0
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm5, %xmm1
-; SSE2-NEXT:    pand %xmm3, %xmm1
-; SSE2-NEXT:    paddd %xmm0, %xmm1
-; SSE2-NEXT:    pmuludq %xmm5, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[1,3,2,3]
-; SSE2-NEXT:    shufps {{.*#+}} xmm7 = xmm7[0,0],xmm10[0,0]
-; SSE2-NEXT:    shufps {{.*#+}} xmm6 = xmm6[0,0],xmm9[0,0]
-; SSE2-NEXT:    pmuludq %xmm7, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[1,3,2,3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1]
-; SSE2-NEXT:    psubd %xmm1, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm3[0,2,2,3]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,2,2,3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm7 = xmm7[0],xmm0[0],xmm7[1],xmm0[1]
-; SSE2-NEXT:    movdqa %xmm7, (%rcx)
-; SSE2-NEXT:    psrad $31, %xmm7
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm7
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE2-NEXT:    pxor %xmm0, %xmm7
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm1
-; SSE2-NEXT:    pand %xmm12, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm12, %xmm4
-; SSE2-NEXT:    pand %xmm2, %xmm4
-; SSE2-NEXT:    paddd %xmm1, %xmm4
-; SSE2-NEXT:    pmuludq %xmm12, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,3,2,3]
-; SSE2-NEXT:    pmuludq %xmm8, %xmm11
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm11[1,3,2,3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
-; SSE2-NEXT:    psubd %xmm4, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm6, %xmm0
+; SSE2-NEXT:    pand %xmm1, %xmm0
+; SSE2-NEXT:    paddd %xmm5, %xmm0
+; SSE2-NEXT:    pmuludq %xmm6, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[1,3,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm12 = xmm12[0,0],xmm10[0,0]
+; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,0],xmm8[0,0]
+; SSE2-NEXT:    pmuludq %xmm12, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[1,3,2,3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
+; SSE2-NEXT:    psubd %xmm0, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[0,2,2,3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT:    movdqa %xmm0, (%rcx)
+; SSE2-NEXT:    psrad $31, %xmm0
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm0
+; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
+; SSE2-NEXT:    pxor %xmm1, %xmm0
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm7, %xmm3
+; SSE2-NEXT:    pand %xmm11, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm11, %xmm4
+; SSE2-NEXT:    pand %xmm7, %xmm4
+; SSE2-NEXT:    paddd %xmm3, %xmm4
+; SSE2-NEXT:    pmuludq %xmm9, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,3,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm13[1,3,2,3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]
+; SSE2-NEXT:    psubd %xmm4, %xmm5
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm11[0,2,2,3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; SSE2-NEXT:    movq %xmm2, 16(%rcx)
-; SSE2-NEXT:    psrad $31, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm2
-; SSE2-NEXT:    pxor %xmm0, %xmm2
-; SSE2-NEXT:    movq %xmm2, 16(%rdi)
-; SSE2-NEXT:    movdqa %xmm7, (%rdi)
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm13[0,2,2,3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; SSE2-NEXT:    movq %xmm3, 16(%rcx)
+; SSE2-NEXT:    psrad $31, %xmm3
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm3
+; SSE2-NEXT:    pxor %xmm1, %xmm3
+; SSE2-NEXT:    movq %xmm3, 16(%rdi)
+; SSE2-NEXT:    movdqa %xmm0, (%rdi)
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: smulo_v6i32:
 ; SSSE3:       # %bb.0:
 ; SSSE3-NEXT:    movq %rdi, %rax
-; SSSE3-NEXT:    movd %r8d, %xmm9
+; SSSE3-NEXT:    movd %r8d, %xmm8
 ; SSSE3-NEXT:    movd %ecx, %xmm0
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm9[0],xmm0[1],xmm9[1]
-; SSSE3-NEXT:    movd %edx, %xmm6
-; SSSE3-NEXT:    movd %esi, %xmm5
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
-; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm0[0]
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm8[0],xmm0[1],xmm8[1]
+; SSSE3-NEXT:    movd %edx, %xmm3
+; SSSE3-NEXT:    movd %esi, %xmm6
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm6 = xmm6[0],xmm3[0],xmm6[1],xmm3[1]
+; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm6 = xmm6[0],xmm0[0]
 ; SSSE3-NEXT:    movd {{.*#+}} xmm10 = mem[0],zero,zero,zero
 ; SSSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm10[0],xmm0[1],xmm10[1]
-; SSSE3-NEXT:    movd {{.*#+}} xmm7 = mem[0],zero,zero,zero
-; SSSE3-NEXT:    movd {{.*#+}} xmm3 = mem[0],zero,zero,zero
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm7[0],xmm3[1],xmm7[1]
-; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm0[0]
-; SSSE3-NEXT:    movd %r9d, %xmm12
-; SSSE3-NEXT:    movd {{.*#+}} xmm11 = mem[0],zero,zero,zero
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm12 = xmm12[0],xmm11[0],xmm12[1],xmm11[1]
-; SSSE3-NEXT:    movd {{.*#+}} xmm8 = mem[0],zero,zero,zero
+; SSSE3-NEXT:    movd {{.*#+}} xmm12 = mem[0],zero,zero,zero
+; SSSE3-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm12[0],xmm1[1],xmm12[1]
+; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSSE3-NEXT:    movd %r9d, %xmm13
 ; SSSE3-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm8[0],xmm2[1],xmm8[1]
+; SSSE3-NEXT:    movdqa %xmm13, %xmm11
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm11 = xmm11[0],xmm2[0],xmm11[1],xmm2[1]
+; SSSE3-NEXT:    movd {{.*#+}} xmm9 = mem[0],zero,zero,zero
+; SSSE3-NEXT:    movd {{.*#+}} xmm7 = mem[0],zero,zero,zero
+; SSSE3-NEXT:    pmuludq %xmm7, %xmm13
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm7 = xmm7[0],xmm9[0],xmm7[1],xmm9[1]
 ; SSSE3-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
 ; SSSE3-NEXT:    pxor %xmm4, %xmm4
+; SSSE3-NEXT:    pxor %xmm5, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm5
+; SSSE3-NEXT:    pand %xmm6, %xmm5
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSSE3-NEXT:    pand %xmm5, %xmm0
-; SSSE3-NEXT:    pxor %xmm1, %xmm1
-; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm1
-; SSSE3-NEXT:    pand %xmm3, %xmm1
-; SSSE3-NEXT:    paddd %xmm0, %xmm1
-; SSSE3-NEXT:    pmuludq %xmm5, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[1,3,2,3]
-; SSSE3-NEXT:    shufps {{.*#+}} xmm7 = xmm7[0,0],xmm10[0,0]
-; SSSE3-NEXT:    shufps {{.*#+}} xmm6 = xmm6[0,0],xmm9[0,0]
-; SSSE3-NEXT:    pmuludq %xmm7, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[1,3,2,3]
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1]
-; SSSE3-NEXT:    psubd %xmm1, %xmm5
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm3[0,2,2,3]
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,2,2,3]
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm7 = xmm7[0],xmm0[0],xmm7[1],xmm0[1]
-; SSSE3-NEXT:    movdqa %xmm7, (%rcx)
-; SSSE3-NEXT:    psrad $31, %xmm7
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm7
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSSE3-NEXT:    pxor %xmm0, %xmm7
-; SSSE3-NEXT:    pxor %xmm1, %xmm1
-; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm1
-; SSSE3-NEXT:    pand %xmm12, %xmm1
-; SSSE3-NEXT:    pcmpgtd %xmm12, %xmm4
-; SSSE3-NEXT:    pand %xmm2, %xmm4
-; SSSE3-NEXT:    paddd %xmm1, %xmm4
-; SSSE3-NEXT:    pmuludq %xmm12, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,3,2,3]
-; SSSE3-NEXT:    pmuludq %xmm8, %xmm11
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm11[1,3,2,3]
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
-; SSSE3-NEXT:    psubd %xmm4, %xmm1
+; SSSE3-NEXT:    pcmpgtd %xmm6, %xmm0
+; SSSE3-NEXT:    pand %xmm1, %xmm0
+; SSSE3-NEXT:    paddd %xmm5, %xmm0
+; SSSE3-NEXT:    pmuludq %xmm6, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[1,3,2,3]
+; SSSE3-NEXT:    shufps {{.*#+}} xmm12 = xmm12[0,0],xmm10[0,0]
+; SSSE3-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,0],xmm8[0,0]
+; SSSE3-NEXT:    pmuludq %xmm12, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[1,3,2,3]
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
+; SSSE3-NEXT:    psubd %xmm0, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[0,2,2,3]
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSSE3-NEXT:    movdqa %xmm0, (%rcx)
+; SSSE3-NEXT:    psrad $31, %xmm0
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm0
+; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm1
+; SSSE3-NEXT:    pxor %xmm1, %xmm0
+; SSSE3-NEXT:    pxor %xmm3, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm7, %xmm3
+; SSSE3-NEXT:    pand %xmm11, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm11, %xmm4
+; SSSE3-NEXT:    pand %xmm7, %xmm4
+; SSSE3-NEXT:    paddd %xmm3, %xmm4
+; SSSE3-NEXT:    pmuludq %xmm9, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,3,2,3]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm13[1,3,2,3]
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]
+; SSSE3-NEXT:    psubd %xmm4, %xmm5
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm11[0,2,2,3]
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; SSSE3-NEXT:    movq %xmm2, 16(%rcx)
-; SSSE3-NEXT:    psrad $31, %xmm2
-; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm2
-; SSSE3-NEXT:    pxor %xmm0, %xmm2
-; SSSE3-NEXT:    movq %xmm2, 16(%rdi)
-; SSSE3-NEXT:    movdqa %xmm7, (%rdi)
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm13[0,2,2,3]
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; SSSE3-NEXT:    movq %xmm3, 16(%rcx)
+; SSSE3-NEXT:    psrad $31, %xmm3
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm3
+; SSSE3-NEXT:    pxor %xmm1, %xmm3
+; SSSE3-NEXT:    movq %xmm3, 16(%rdi)
+; SSSE3-NEXT:    movdqa %xmm0, (%rdi)
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: smulo_v6i32:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movq %rdi, %rax
-; SSE41-NEXT:    movd %esi, %xmm3
-; SSE41-NEXT:    pinsrd $1, %edx, %xmm3
-; SSE41-NEXT:    pinsrd $2, %ecx, %xmm3
-; SSE41-NEXT:    pinsrd $3, %r8d, %xmm3
-; SSE41-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE41-NEXT:    pinsrd $1, {{[0-9]+}}(%rsp), %xmm0
-; SSE41-NEXT:    movd %r9d, %xmm1
+; SSE41-NEXT:    movd %esi, %xmm2
+; SSE41-NEXT:    pinsrd $1, %edx, %xmm2
+; SSE41-NEXT:    pinsrd $2, %ecx, %xmm2
+; SSE41-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; SSE41-NEXT:    pinsrd $1, {{[0-9]+}}(%rsp), %xmm1
-; SSE41-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; SSE41-NEXT:    pinsrd $1, {{[0-9]+}}(%rsp), %xmm2
-; SSE41-NEXT:    pinsrd $2, {{[0-9]+}}(%rsp), %xmm2
-; SSE41-NEXT:    pinsrd $3, {{[0-9]+}}(%rsp), %xmm2
+; SSE41-NEXT:    pinsrd $2, {{[0-9]+}}(%rsp), %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pmuldq %xmm2, %xmm1
+; SSE41-NEXT:    pinsrd $3, %r8d, %xmm2
+; SSE41-NEXT:    movd {{.*#+}} xmm3 = mem[0],zero,zero,zero
+; SSE41-NEXT:    movd %r9d, %xmm4
+; SSE41-NEXT:    movdqa %xmm4, %xmm5
+; SSE41-NEXT:    pmuldq %xmm3, %xmm4
+; SSE41-NEXT:    pinsrd $1, {{[0-9]+}}(%rsp), %xmm3
+; SSE41-NEXT:    pinsrd $1, {{[0-9]+}}(%rsp), %xmm5
+; SSE41-NEXT:    pmulld %xmm3, %xmm5
+; SSE41-NEXT:    pinsrd $3, {{[0-9]+}}(%rsp), %xmm0
 ; SSE41-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[1,1,3,3]
-; SSE41-NEXT:    pmuldq %xmm4, %xmm5
-; SSE41-NEXT:    movdqa %xmm3, %xmm4
-; SSE41-NEXT:    pmuldq %xmm2, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE41-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1],xmm5[2,3],xmm4[4,5],xmm5[6,7]
-; SSE41-NEXT:    pmulld %xmm3, %xmm2
-; SSE41-NEXT:    movdqa %xmm2, (%rcx)
-; SSE41-NEXT:    psrad $31, %xmm2
-; SSE41-NEXT:    pcmpeqd %xmm4, %xmm2
-; SSE41-NEXT:    pcmpeqd %xmm3, %xmm3
-; SSE41-NEXT:    pxor %xmm3, %xmm2
-; SSE41-NEXT:    movd {{.*#+}} xmm4 = mem[0],zero,zero,zero
-; SSE41-NEXT:    movd {{.*#+}} xmm5 = mem[0],zero,zero,zero
-; SSE41-NEXT:    pmuldq %xmm4, %xmm5
-; SSE41-NEXT:    movdqa %xmm1, %xmm4
-; SSE41-NEXT:    pmuldq %xmm0, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE41-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1],xmm5[2,3],xmm4[4,5],xmm5[6,7]
-; SSE41-NEXT:    pmulld %xmm0, %xmm1
-; SSE41-NEXT:    movq %xmm1, 16(%rcx)
-; SSE41-NEXT:    psrad $31, %xmm1
-; SSE41-NEXT:    pcmpeqd %xmm4, %xmm1
-; SSE41-NEXT:    pxor %xmm3, %xmm1
-; SSE41-NEXT:    movq %xmm1, 16(%rdi)
-; SSE41-NEXT:    movdqa %xmm2, (%rdi)
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm0[1,1,3,3]
+; SSE41-NEXT:    pmuldq %xmm3, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm6[2,3],xmm1[4,5],xmm6[6,7]
+; SSE41-NEXT:    pmulld %xmm2, %xmm0
+; SSE41-NEXT:    movdqa %xmm0, (%rcx)
+; SSE41-NEXT:    psrad $31, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm1, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
+; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE41-NEXT:    movd {{.*#+}} xmm3 = mem[0],zero,zero,zero
+; SSE41-NEXT:    pmuldq %xmm2, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3]
+; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3],xmm2[4,5],xmm3[6,7]
+; SSE41-NEXT:    movq %xmm5, 16(%rcx)
+; SSE41-NEXT:    psrad $31, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm2, %xmm5
+; SSE41-NEXT:    pxor %xmm1, %xmm5
+; SSE41-NEXT:    movq %xmm5, 16(%rdi)
+; SSE41-NEXT:    movdqa %xmm0, (%rdi)
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: smulo_v6i32:
@@ -1735,14 +1622,15 @@ define <2 x i32> @smulo_v2i64(<2 x i64> %a0, <2 x i64> %a1, <2 x i64>* %p2) noun
 ; SSE2-NEXT:    movq $-1, %r9
 ; SSE2-NEXT:    movl $0, %edx
 ; SSE2-NEXT:    cmovoq %r9, %rdx
-; SSE2-NEXT:    movq %rdx, %xmm0
+; SSE2-NEXT:    movq %rsi, %xmm1
 ; SSE2-NEXT:    imulq %r8, %rcx
+; SSE2-NEXT:    movq %rcx, %xmm0
+; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSE2-NEXT:    movq %rdx, %xmm0
 ; SSE2-NEXT:    cmovoq %r9, %rax
-; SSE2-NEXT:    movq %rax, %xmm1
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSE2-NEXT:    movq %rsi, %xmm1
-; SSE2-NEXT:    movq %rcx, %xmm2
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; SSE2-NEXT:    movq %rax, %xmm2
+; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; SSE2-NEXT:    movdqa %xmm1, (%rdi)
 ; SSE2-NEXT:    retq
 ;
@@ -1759,14 +1647,15 @@ define <2 x i32> @smulo_v2i64(<2 x i64> %a0, <2 x i64> %a1, <2 x i64>* %p2) noun
 ; SSSE3-NEXT:    movq $-1, %r9
 ; SSSE3-NEXT:    movl $0, %edx
 ; SSSE3-NEXT:    cmovoq %r9, %rdx
-; SSSE3-NEXT:    movq %rdx, %xmm0
+; SSSE3-NEXT:    movq %rsi, %xmm1
 ; SSSE3-NEXT:    imulq %r8, %rcx
+; SSSE3-NEXT:    movq %rcx, %xmm0
+; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSSE3-NEXT:    movq %rdx, %xmm0
 ; SSSE3-NEXT:    cmovoq %r9, %rax
-; SSSE3-NEXT:    movq %rax, %xmm1
-; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSSE3-NEXT:    movq %rsi, %xmm1
-; SSSE3-NEXT:    movq %rcx, %xmm2
-; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; SSSE3-NEXT:    movq %rax, %xmm2
+; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; SSSE3-NEXT:    movdqa %xmm1, (%rdi)
 ; SSSE3-NEXT:    retq
 ;
@@ -1781,15 +1670,16 @@ define <2 x i32> @smulo_v2i64(<2 x i64> %a0, <2 x i64> %a1, <2 x i64>* %p2) noun
 ; SSE41-NEXT:    movq $-1, %r9
 ; SSE41-NEXT:    movl $0, %edx
 ; SSE41-NEXT:    cmovoq %r9, %rdx
-; SSE41-NEXT:    movq %rdx, %xmm1
+; SSE41-NEXT:    movq %rsi, %xmm0
 ; SSE41-NEXT:    imulq %r8, %rcx
+; SSE41-NEXT:    movq %rcx, %xmm1
+; SSE41-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSE41-NEXT:    movq %rdx, %xmm0
 ; SSE41-NEXT:    cmovoq %r9, %rax
-; SSE41-NEXT:    movq %rax, %xmm0
-; SSE41-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSE41-NEXT:    movq %rsi, %xmm1
-; SSE41-NEXT:    movq %rcx, %xmm2
-; SSE41-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm1[0]
-; SSE41-NEXT:    movdqa %xmm2, (%rdi)
+; SSE41-NEXT:    movq %rax, %xmm2
+; SSE41-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm0[0]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
+; SSE41-NEXT:    movdqa %xmm1, (%rdi)
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: smulo_v2i64:
@@ -1803,14 +1693,15 @@ define <2 x i32> @smulo_v2i64(<2 x i64> %a0, <2 x i64> %a1, <2 x i64>* %p2) noun
 ; AVX1-NEXT:    movq $-1, %r9
 ; AVX1-NEXT:    movl $0, %edx
 ; AVX1-NEXT:    cmovoq %r9, %rdx
-; AVX1-NEXT:    vmovq %rdx, %xmm0
+; AVX1-NEXT:    vmovq %rsi, %xmm0
 ; AVX1-NEXT:    imulq %r8, %rcx
+; AVX1-NEXT:    vmovq %rcx, %xmm1
+; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; AVX1-NEXT:    vmovq %rdx, %xmm0
 ; AVX1-NEXT:    cmovoq %r9, %rax
-; AVX1-NEXT:    vmovq %rax, %xmm1
-; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX1-NEXT:    vmovq %rsi, %xmm1
-; AVX1-NEXT:    vmovq %rcx, %xmm2
-; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX1-NEXT:    vmovq %rax, %xmm2
+; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm2[0],xmm0[0]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; AVX1-NEXT:    vmovdqa %xmm1, (%rdi)
 ; AVX1-NEXT:    retq
 ;
@@ -1825,14 +1716,15 @@ define <2 x i32> @smulo_v2i64(<2 x i64> %a0, <2 x i64> %a1, <2 x i64>* %p2) noun
 ; AVX2-NEXT:    movq $-1, %r9
 ; AVX2-NEXT:    movl $0, %edx
 ; AVX2-NEXT:    cmovoq %r9, %rdx
-; AVX2-NEXT:    vmovq %rdx, %xmm0
+; AVX2-NEXT:    vmovq %rsi, %xmm0
 ; AVX2-NEXT:    imulq %r8, %rcx
+; AVX2-NEXT:    vmovq %rcx, %xmm1
+; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; AVX2-NEXT:    vmovq %rdx, %xmm0
 ; AVX2-NEXT:    cmovoq %r9, %rax
-; AVX2-NEXT:    vmovq %rax, %xmm1
-; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX2-NEXT:    vmovq %rsi, %xmm1
-; AVX2-NEXT:    vmovq %rcx, %xmm2
-; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX2-NEXT:    vmovq %rax, %xmm2
+; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm2[0],xmm0[0]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; AVX2-NEXT:    vmovdqa %xmm1, (%rdi)
 ; AVX2-NEXT:    retq
 ;
@@ -1844,19 +1736,22 @@ define <2 x i32> @smulo_v2i64(<2 x i64> %a0, <2 x i64> %a1, <2 x i64>* %p2) noun
 ; AVX512-NEXT:    vpextrq $1, %xmm0, %rsi
 ; AVX512-NEXT:    imulq %rdx, %rsi
 ; AVX512-NEXT:    seto %dl
-; AVX512-NEXT:    kmovd %edx, %k0
-; AVX512-NEXT:    kshiftlw $1, %k0, %k0
-; AVX512-NEXT:    imulq %rax, %rcx
-; AVX512-NEXT:    seto %al
-; AVX512-NEXT:    andl $1, %eax
-; AVX512-NEXT:    kmovw %eax, %k1
-; AVX512-NEXT:    korw %k0, %k1, %k1
 ; AVX512-NEXT:    vmovq %rsi, %xmm0
+; AVX512-NEXT:    imulq %rax, %rcx
 ; AVX512-NEXT:    vmovq %rcx, %xmm1
-; AVX512-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX512-NEXT:    vmovdqa %xmm0, (%rdi)
+; AVX512-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; AVX512-NEXT:    seto %al
+; AVX512-NEXT:    movw $-3, %cx
+; AVX512-NEXT:    kmovd %ecx, %k0
+; AVX512-NEXT:    kmovd %eax, %k1
+; AVX512-NEXT:    kandw %k0, %k1, %k0
+; AVX512-NEXT:    kmovd %edx, %k1
+; AVX512-NEXT:    kshiftlw $15, %k1, %k1
+; AVX512-NEXT:    kshiftrw $14, %k1, %k1
+; AVX512-NEXT:    korw %k1, %k0, %k1
 ; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX512-NEXT:    vmovdqa64 %xmm0, %xmm0 {%k1} {z}
+; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
+; AVX512-NEXT:    vmovdqa %xmm1, (%rdi)
 ; AVX512-NEXT:    retq
   %t = call {<2 x i64>, <2 x i1>} @llvm.smul.with.overflow.v2i64(<2 x i64> %a0, <2 x i64> %a1)
   %val = extractvalue {<2 x i64>, <2 x i1>} %t, 0
@@ -1873,52 +1768,52 @@ define <4 x i32> @smulo_v4i24(<4 x i24> %a0, <4 x i24> %a1, <4 x i24>* %p2) noun
 ; SSE2-NEXT:    psrad $8, %xmm0
 ; SSE2-NEXT:    pslld $8, %xmm1
 ; SSE2-NEXT:    psrad $8, %xmm1
-; SSE2-NEXT:    pxor %xmm4, %xmm4
+; SSE2-NEXT:    pxor %xmm3, %xmm3
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
 ; SSE2-NEXT:    pand %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE2-NEXT:    pand %xmm1, %xmm4
-; SSE2-NEXT:    paddd %xmm2, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSE2-NEXT:    pand %xmm1, %xmm3
+; SSE2-NEXT:    paddd %xmm2, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
 ; SSE2-NEXT:    pmuludq %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,3,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[1,3,2,3]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pmuludq %xmm5, %xmm2
+; SSE2-NEXT:    pmuludq %xmm4, %xmm2
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,3,2,3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
-; SSE2-NEXT:    psubd %xmm4, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[0,2,2,3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1]
+; SSE2-NEXT:    psubd %xmm3, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[0,2,2,3]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
-; SSE2-NEXT:    movdqa %xmm4, %xmm1
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; SSE2-NEXT:    movdqa %xmm3, %xmm1
 ; SSE2-NEXT:    pslld $8, %xmm1
 ; SSE2-NEXT:    psrad $8, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[3,1,2,3]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[2,3,0,1]
-; SSE2-NEXT:    psrad $31, %xmm4
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm4
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm3
-; SSE2-NEXT:    pxor %xmm3, %xmm4
-; SSE2-NEXT:    pxor %xmm3, %xmm1
-; SSE2-NEXT:    por %xmm4, %xmm1
+; SSE2-NEXT:    pcmpeqd %xmm3, %xmm1
+; SSE2-NEXT:    psrad $31, %xmm3
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm3
+; SSE2-NEXT:    pcmpeqd %xmm4, %xmm4
+; SSE2-NEXT:    pxor %xmm4, %xmm3
+; SSE2-NEXT:    pxor %xmm4, %xmm1
+; SSE2-NEXT:    por %xmm3, %xmm1
 ; SSE2-NEXT:    movd %xmm0, %eax
 ; SSE2-NEXT:    movw %ax, (%rdi)
-; SSE2-NEXT:    movd %xmm2, %ecx
-; SSE2-NEXT:    movw %cx, 3(%rdi)
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE2-NEXT:    movd %xmm0, %ecx
+; SSE2-NEXT:    movw %cx, 6(%rdi)
+; SSE2-NEXT:    movd %xmm2, %edx
+; SSE2-NEXT:    movw %dx, 3(%rdi)
 ; SSE2-NEXT:    shrl $16, %eax
 ; SSE2-NEXT:    movb %al, 2(%rdi)
-; SSE2-NEXT:    shrl $16, %ecx
-; SSE2-NEXT:    movb %cl, 5(%rdi)
-; SSE2-NEXT:    movd %xmm5, %eax
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
+; SSE2-NEXT:    movd %xmm0, %eax
 ; SSE2-NEXT:    movw %ax, 9(%rdi)
-; SSE2-NEXT:    movd %xmm6, %ecx
-; SSE2-NEXT:    movw %cx, 6(%rdi)
-; SSE2-NEXT:    shrl $16, %eax
-; SSE2-NEXT:    movb %al, 11(%rdi)
 ; SSE2-NEXT:    shrl $16, %ecx
 ; SSE2-NEXT:    movb %cl, 8(%rdi)
+; SSE2-NEXT:    shrl $16, %edx
+; SSE2-NEXT:    movb %dl, 5(%rdi)
+; SSE2-NEXT:    shrl $16, %eax
+; SSE2-NEXT:    movb %al, 11(%rdi)
 ; SSE2-NEXT:    movdqa %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
@@ -1928,52 +1823,52 @@ define <4 x i32> @smulo_v4i24(<4 x i24> %a0, <4 x i24> %a1, <4 x i24>* %p2) noun
 ; SSSE3-NEXT:    psrad $8, %xmm0
 ; SSSE3-NEXT:    pslld $8, %xmm1
 ; SSSE3-NEXT:    psrad $8, %xmm1
-; SSSE3-NEXT:    pxor %xmm4, %xmm4
+; SSSE3-NEXT:    pxor %xmm3, %xmm3
 ; SSSE3-NEXT:    pxor %xmm2, %xmm2
 ; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
 ; SSSE3-NEXT:    pand %xmm0, %xmm2
-; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSSE3-NEXT:    pand %xmm1, %xmm4
-; SSSE3-NEXT:    paddd %xmm2, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSSE3-NEXT:    pand %xmm1, %xmm3
+; SSSE3-NEXT:    paddd %xmm2, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
 ; SSSE3-NEXT:    pmuludq %xmm1, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,3,2,3]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[1,3,2,3]
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    pmuludq %xmm5, %xmm2
+; SSSE3-NEXT:    pmuludq %xmm4, %xmm2
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,3,2,3]
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
-; SSSE3-NEXT:    psubd %xmm4, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[0,2,2,3]
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1]
+; SSSE3-NEXT:    psubd %xmm3, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[0,2,2,3]
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
-; SSSE3-NEXT:    movdqa %xmm4, %xmm1
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; SSSE3-NEXT:    movdqa %xmm3, %xmm1
 ; SSSE3-NEXT:    pslld $8, %xmm1
 ; SSSE3-NEXT:    psrad $8, %xmm1
-; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[3,1,2,3]
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[2,3,0,1]
-; SSSE3-NEXT:    psrad $31, %xmm4
-; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm4
-; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm3
-; SSSE3-NEXT:    pxor %xmm3, %xmm4
-; SSSE3-NEXT:    pxor %xmm3, %xmm1
-; SSSE3-NEXT:    por %xmm4, %xmm1
+; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm1
+; SSSE3-NEXT:    psrad $31, %xmm3
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm3
+; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm4
+; SSSE3-NEXT:    pxor %xmm4, %xmm3
+; SSSE3-NEXT:    pxor %xmm4, %xmm1
+; SSSE3-NEXT:    por %xmm3, %xmm1
 ; SSSE3-NEXT:    movd %xmm0, %eax
 ; SSSE3-NEXT:    movw %ax, (%rdi)
-; SSSE3-NEXT:    movd %xmm2, %ecx
-; SSSE3-NEXT:    movw %cx, 3(%rdi)
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSSE3-NEXT:    movd %xmm0, %ecx
+; SSSE3-NEXT:    movw %cx, 6(%rdi)
+; SSSE3-NEXT:    movd %xmm2, %edx
+; SSSE3-NEXT:    movw %dx, 3(%rdi)
 ; SSSE3-NEXT:    shrl $16, %eax
 ; SSSE3-NEXT:    movb %al, 2(%rdi)
-; SSSE3-NEXT:    shrl $16, %ecx
-; SSSE3-NEXT:    movb %cl, 5(%rdi)
-; SSSE3-NEXT:    movd %xmm5, %eax
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
+; SSSE3-NEXT:    movd %xmm0, %eax
 ; SSSE3-NEXT:    movw %ax, 9(%rdi)
-; SSSE3-NEXT:    movd %xmm6, %ecx
-; SSSE3-NEXT:    movw %cx, 6(%rdi)
-; SSSE3-NEXT:    shrl $16, %eax
-; SSSE3-NEXT:    movb %al, 11(%rdi)
 ; SSSE3-NEXT:    shrl $16, %ecx
 ; SSSE3-NEXT:    movb %cl, 8(%rdi)
+; SSSE3-NEXT:    shrl $16, %edx
+; SSSE3-NEXT:    movb %dl, 5(%rdi)
+; SSSE3-NEXT:    shrl $16, %eax
+; SSSE3-NEXT:    movb %al, 11(%rdi)
 ; SSSE3-NEXT:    movdqa %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
@@ -2353,8 +2248,10 @@ define <4 x i32> @smulo_v4i1(<4 x i1> %a0, <4 x i1> %a1, <4 x i1>* %p2) nounwind
 ; AVX512-NEXT:    setne %cl
 ; AVX512-NEXT:    orb %al, %cl
 ; AVX512-NEXT:    setne %al
+; AVX512-NEXT:    kmovd %eax, %k1
+; AVX512-NEXT:    movw $-3, %ax
 ; AVX512-NEXT:    kmovd %eax, %k0
-; AVX512-NEXT:    kshiftrw $1, %k0, %k1
+; AVX512-NEXT:    kandw %k0, %k1, %k1
 ; AVX512-NEXT:    movl %edx, %eax
 ; AVX512-NEXT:    imulb %sil
 ; AVX512-NEXT:    movl %eax, %edx
@@ -2367,11 +2264,12 @@ define <4 x i32> @smulo_v4i1(<4 x i1> %a0, <4 x i1> %a1, <4 x i1>* %p2) nounwind
 ; AVX512-NEXT:    orb %al, %cl
 ; AVX512-NEXT:    setne %al
 ; AVX512-NEXT:    kmovd %eax, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
-; AVX512-NEXT:    kshiftlw $15, %k1, %k1
-; AVX512-NEXT:    kshiftrw $14, %k1, %k1
-; AVX512-NEXT:    kxorw %k1, %k0, %k0
-; AVX512-NEXT:    kshiftrw $2, %k0, %k1
+; AVX512-NEXT:    kshiftlw $15, %k2, %k2
+; AVX512-NEXT:    kshiftrw $14, %k2, %k2
+; AVX512-NEXT:    korw %k2, %k1, %k2
+; AVX512-NEXT:    movw $-5, %ax
+; AVX512-NEXT:    kmovd %eax, %k1
+; AVX512-NEXT:    kandw %k1, %k2, %k2
 ; AVX512-NEXT:    movl %r11d, %eax
 ; AVX512-NEXT:    imulb %bl
 ; AVX512-NEXT:    movl %eax, %esi
@@ -2383,13 +2281,11 @@ define <4 x i32> @smulo_v4i1(<4 x i1> %a0, <4 x i1> %a1, <4 x i1>* %p2) nounwind
 ; AVX512-NEXT:    setne %cl
 ; AVX512-NEXT:    orb %al, %cl
 ; AVX512-NEXT:    setne %al
-; AVX512-NEXT:    kmovd %eax, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
-; AVX512-NEXT:    kshiftlw $15, %k1, %k1
-; AVX512-NEXT:    kshiftrw $13, %k1, %k1
-; AVX512-NEXT:    kxorw %k1, %k0, %k0
-; AVX512-NEXT:    kshiftlw $13, %k0, %k0
-; AVX512-NEXT:    kshiftrw $13, %k0, %k0
+; AVX512-NEXT:    kmovd %eax, %k3
+; AVX512-NEXT:    kshiftlw $2, %k3, %k3
+; AVX512-NEXT:    korw %k3, %k2, %k2
+; AVX512-NEXT:    kshiftlw $13, %k2, %k2
+; AVX512-NEXT:    kshiftrw $13, %k2, %k2
 ; AVX512-NEXT:    movl %r10d, %eax
 ; AVX512-NEXT:    imulb %r9b
 ; AVX512-NEXT:    # kill: def $al killed $al def $eax
@@ -2401,30 +2297,29 @@ define <4 x i32> @smulo_v4i1(<4 x i1> %a0, <4 x i1> %a1, <4 x i1>* %p2) nounwind
 ; AVX512-NEXT:    setne %bl
 ; AVX512-NEXT:    orb %cl, %bl
 ; AVX512-NEXT:    setne %cl
-; AVX512-NEXT:    kmovd %ecx, %k1
-; AVX512-NEXT:    kshiftlw $3, %k1, %k1
-; AVX512-NEXT:    korw %k1, %k0, %k1
+; AVX512-NEXT:    kmovd %ecx, %k3
+; AVX512-NEXT:    kshiftlw $3, %k3, %k3
+; AVX512-NEXT:    korw %k3, %k2, %k2
 ; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
-; AVX512-NEXT:    kmovd %r8d, %k0
-; AVX512-NEXT:    kshiftrw $1, %k0, %k1
+; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k2} {z}
+; AVX512-NEXT:    kmovd %r8d, %k2
+; AVX512-NEXT:    kandw %k0, %k2, %k0
 ; AVX512-NEXT:    kmovd %edx, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
-; AVX512-NEXT:    kshiftlw $15, %k1, %k1
-; AVX512-NEXT:    kshiftrw $14, %k1, %k1
-; AVX512-NEXT:    kxorw %k1, %k0, %k0
-; AVX512-NEXT:    kshiftrw $2, %k0, %k1
-; AVX512-NEXT:    kmovd %esi, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
+; AVX512-NEXT:    kshiftlw $15, %k2, %k2
+; AVX512-NEXT:    kshiftrw $14, %k2, %k2
+; AVX512-NEXT:    korw %k2, %k0, %k0
+; AVX512-NEXT:    kandw %k1, %k0, %k0
+; AVX512-NEXT:    kmovd %esi, %k1
 ; AVX512-NEXT:    kshiftlw $15, %k1, %k1
 ; AVX512-NEXT:    kshiftrw $13, %k1, %k1
-; AVX512-NEXT:    kxorw %k1, %k0, %k0
-; AVX512-NEXT:    kshiftrw $3, %k0, %k1
-; AVX512-NEXT:    kmovd %eax, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
+; AVX512-NEXT:    korw %k1, %k0, %k0
+; AVX512-NEXT:    movw $-9, %cx
+; AVX512-NEXT:    kmovd %ecx, %k1
+; AVX512-NEXT:    kandw %k1, %k0, %k0
+; AVX512-NEXT:    kmovd %eax, %k1
 ; AVX512-NEXT:    kshiftlw $15, %k1, %k1
 ; AVX512-NEXT:    kshiftrw $12, %k1, %k1
-; AVX512-NEXT:    kxorw %k1, %k0, %k0
+; AVX512-NEXT:    korw %k1, %k0, %k0
 ; AVX512-NEXT:    kmovd %k0, %eax
 ; AVX512-NEXT:    movb %al, (%rdi)
 ; AVX512-NEXT:    popq %rbx
@@ -2467,20 +2362,17 @@ define <2 x i32> @smulo_v2i128(<2 x i128> %a0, <2 x i128> %a1, <2 x i128>* %p2)
 ; SSE2-NEXT:    movq %r12, %rcx
 ; SSE2-NEXT:    callq __muloti4
 ; SSE2-NEXT:    xorl %ecx, %ecx
-; SSE2-NEXT:    cmpq $0, {{[0-9]+}}(%rsp)
-; SSE2-NEXT:    setne %cl
-; SSE2-NEXT:    xorl %esi, %esi
-; SSE2-NEXT:    cmpq $0, {{[0-9]+}}(%rsp)
-; SSE2-NEXT:    setne %sil
-; SSE2-NEXT:    movd %esi, %xmm0
-; SSE2-NEXT:    pinsrw $4, %ecx, %xmm0
+; SSE2-NEXT:    cmpq {{[0-9]+}}(%rsp), %rcx
+; SSE2-NEXT:    sbbl %esi, %esi
+; SSE2-NEXT:    movd %esi, %xmm1
+; SSE2-NEXT:    cmpq {{[0-9]+}}(%rsp), %rcx
+; SSE2-NEXT:    sbbl %ecx, %ecx
+; SSE2-NEXT:    movd %ecx, %xmm0
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; SSE2-NEXT:    movq %rdx, 24(%r15)
 ; SSE2-NEXT:    movq %rax, 16(%r15)
 ; SSE2-NEXT:    movq %rbp, 8(%r15)
 ; SSE2-NEXT:    movq %r13, (%r15)
-; SSE2-NEXT:    psllq $63, %xmm0
-; SSE2-NEXT:    psrad $31, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
 ; SSE2-NEXT:    addq $24, %rsp
 ; SSE2-NEXT:    popq %rbx
 ; SSE2-NEXT:    popq %r12
@@ -2519,20 +2411,17 @@ define <2 x i32> @smulo_v2i128(<2 x i128> %a0, <2 x i128> %a1, <2 x i128>* %p2)
 ; SSSE3-NEXT:    movq %r12, %rcx
 ; SSSE3-NEXT:    callq __muloti4
 ; SSSE3-NEXT:    xorl %ecx, %ecx
-; SSSE3-NEXT:    cmpq $0, {{[0-9]+}}(%rsp)
-; SSSE3-NEXT:    setne %cl
-; SSSE3-NEXT:    xorl %esi, %esi
-; SSSE3-NEXT:    cmpq $0, {{[0-9]+}}(%rsp)
-; SSSE3-NEXT:    setne %sil
-; SSSE3-NEXT:    movd %esi, %xmm0
-; SSSE3-NEXT:    pinsrw $4, %ecx, %xmm0
+; SSSE3-NEXT:    cmpq {{[0-9]+}}(%rsp), %rcx
+; SSSE3-NEXT:    sbbl %esi, %esi
+; SSSE3-NEXT:    movd %esi, %xmm1
+; SSSE3-NEXT:    cmpq {{[0-9]+}}(%rsp), %rcx
+; SSSE3-NEXT:    sbbl %ecx, %ecx
+; SSSE3-NEXT:    movd %ecx, %xmm0
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; SSSE3-NEXT:    movq %rdx, 24(%r15)
 ; SSSE3-NEXT:    movq %rax, 16(%r15)
 ; SSSE3-NEXT:    movq %rbp, 8(%r15)
 ; SSSE3-NEXT:    movq %r13, (%r15)
-; SSSE3-NEXT:    psllq $63, %xmm0
-; SSSE3-NEXT:    psrad $31, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
 ; SSSE3-NEXT:    addq $24, %rsp
 ; SSSE3-NEXT:    popq %rbx
 ; SSSE3-NEXT:    popq %r12
@@ -2571,20 +2460,16 @@ define <2 x i32> @smulo_v2i128(<2 x i128> %a0, <2 x i128> %a1, <2 x i128>* %p2)
 ; SSE41-NEXT:    movq %r12, %rcx
 ; SSE41-NEXT:    callq __muloti4
 ; SSE41-NEXT:    xorl %ecx, %ecx
-; SSE41-NEXT:    cmpq $0, {{[0-9]+}}(%rsp)
-; SSE41-NEXT:    setne %cl
-; SSE41-NEXT:    xorl %esi, %esi
-; SSE41-NEXT:    cmpq $0, {{[0-9]+}}(%rsp)
-; SSE41-NEXT:    setne %sil
-; SSE41-NEXT:    movd %esi, %xmm0
-; SSE41-NEXT:    pinsrb $8, %ecx, %xmm0
+; SSE41-NEXT:    cmpq {{[0-9]+}}(%rsp), %rcx
+; SSE41-NEXT:    sbbl %esi, %esi
+; SSE41-NEXT:    cmpq {{[0-9]+}}(%rsp), %rcx
+; SSE41-NEXT:    sbbl %ecx, %ecx
+; SSE41-NEXT:    movd %ecx, %xmm0
+; SSE41-NEXT:    pinsrd $1, %esi, %xmm0
 ; SSE41-NEXT:    movq %rdx, 24(%r15)
 ; SSE41-NEXT:    movq %rax, 16(%r15)
 ; SSE41-NEXT:    movq %rbp, 8(%r15)
 ; SSE41-NEXT:    movq %r13, (%r15)
-; SSE41-NEXT:    psllq $63, %xmm0
-; SSE41-NEXT:    psrad $31, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
 ; SSE41-NEXT:    addq $24, %rsp
 ; SSE41-NEXT:    popq %rbx
 ; SSE41-NEXT:    popq %r12
@@ -2623,20 +2508,16 @@ define <2 x i32> @smulo_v2i128(<2 x i128> %a0, <2 x i128> %a1, <2 x i128>* %p2)
 ; AVX1-NEXT:    movq %r12, %rcx
 ; AVX1-NEXT:    callq __muloti4
 ; AVX1-NEXT:    xorl %ecx, %ecx
-; AVX1-NEXT:    cmpq $0, {{[0-9]+}}(%rsp)
-; AVX1-NEXT:    setne %cl
-; AVX1-NEXT:    xorl %esi, %esi
-; AVX1-NEXT:    cmpq $0, {{[0-9]+}}(%rsp)
-; AVX1-NEXT:    setne %sil
-; AVX1-NEXT:    vmovd %esi, %xmm0
-; AVX1-NEXT:    vpinsrb $8, %ecx, %xmm0, %xmm0
+; AVX1-NEXT:    cmpq {{[0-9]+}}(%rsp), %rcx
+; AVX1-NEXT:    sbbl %esi, %esi
+; AVX1-NEXT:    cmpq {{[0-9]+}}(%rsp), %rcx
+; AVX1-NEXT:    sbbl %ecx, %ecx
+; AVX1-NEXT:    vmovd %ecx, %xmm0
+; AVX1-NEXT:    vpinsrd $1, %esi, %xmm0, %xmm0
 ; AVX1-NEXT:    movq %rdx, 24(%r15)
 ; AVX1-NEXT:    movq %rax, 16(%r15)
 ; AVX1-NEXT:    movq %rbp, 8(%r15)
 ; AVX1-NEXT:    movq %r13, (%r15)
-; AVX1-NEXT:    vpsllq $63, %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm0
 ; AVX1-NEXT:    addq $24, %rsp
 ; AVX1-NEXT:    popq %rbx
 ; AVX1-NEXT:    popq %r12
@@ -2675,20 +2556,16 @@ define <2 x i32> @smulo_v2i128(<2 x i128> %a0, <2 x i128> %a1, <2 x i128>* %p2)
 ; AVX2-NEXT:    movq %r12, %rcx
 ; AVX2-NEXT:    callq __muloti4
 ; AVX2-NEXT:    xorl %ecx, %ecx
-; AVX2-NEXT:    cmpq $0, {{[0-9]+}}(%rsp)
-; AVX2-NEXT:    setne %cl
-; AVX2-NEXT:    xorl %esi, %esi
-; AVX2-NEXT:    cmpq $0, {{[0-9]+}}(%rsp)
-; AVX2-NEXT:    setne %sil
-; AVX2-NEXT:    vmovd %esi, %xmm0
-; AVX2-NEXT:    vpinsrb $8, %ecx, %xmm0, %xmm0
+; AVX2-NEXT:    cmpq {{[0-9]+}}(%rsp), %rcx
+; AVX2-NEXT:    sbbl %esi, %esi
+; AVX2-NEXT:    cmpq {{[0-9]+}}(%rsp), %rcx
+; AVX2-NEXT:    sbbl %ecx, %ecx
+; AVX2-NEXT:    vmovd %ecx, %xmm0
+; AVX2-NEXT:    vpinsrd $1, %esi, %xmm0, %xmm0
 ; AVX2-NEXT:    movq %rdx, 24(%r15)
 ; AVX2-NEXT:    movq %rax, 16(%r15)
 ; AVX2-NEXT:    movq %rbp, 8(%r15)
 ; AVX2-NEXT:    movq %r13, (%r15)
-; AVX2-NEXT:    vpsllq $63, %xmm0, %xmm0
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm0
 ; AVX2-NEXT:    addq $24, %rsp
 ; AVX2-NEXT:    popq %rbx
 ; AVX2-NEXT:    popq %r12
@@ -2708,39 +2585,39 @@ define <2 x i32> @smulo_v2i128(<2 x i128> %a0, <2 x i128> %a1, <2 x i128>* %p2)
 ; AVX512-NEXT:    pushq %rbx
 ; AVX512-NEXT:    subq $24, %rsp
 ; AVX512-NEXT:    movq %r8, %rax
-; AVX512-NEXT:    movq %rcx, %r14
+; AVX512-NEXT:    movq %rcx, %r15
 ; AVX512-NEXT:    movq %rdx, %rbx
-; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %r15
 ; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %r12
+; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %r13
 ; AVX512-NEXT:    movq $0, {{[0-9]+}}(%rsp)
 ; AVX512-NEXT:    leaq {{[0-9]+}}(%rsp), %r8
 ; AVX512-NEXT:    movq %rax, %rdx
 ; AVX512-NEXT:    movq %r9, %rcx
 ; AVX512-NEXT:    callq __muloti4
-; AVX512-NEXT:    movq %rax, %r13
+; AVX512-NEXT:    movq %rax, %r14
 ; AVX512-NEXT:    movq %rdx, %rbp
 ; AVX512-NEXT:    movq $0, {{[0-9]+}}(%rsp)
 ; AVX512-NEXT:    leaq {{[0-9]+}}(%rsp), %r8
 ; AVX512-NEXT:    movq %rbx, %rdi
-; AVX512-NEXT:    movq %r14, %rsi
+; AVX512-NEXT:    movq %r15, %rsi
 ; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %rdx
-; AVX512-NEXT:    movq %r12, %rcx
+; AVX512-NEXT:    movq %r13, %rcx
 ; AVX512-NEXT:    callq __muloti4
 ; AVX512-NEXT:    cmpq $0, {{[0-9]+}}(%rsp)
 ; AVX512-NEXT:    setne %cl
 ; AVX512-NEXT:    kmovd %ecx, %k0
-; AVX512-NEXT:    kshiftlw $1, %k0, %k0
 ; AVX512-NEXT:    cmpq $0, {{[0-9]+}}(%rsp)
 ; AVX512-NEXT:    setne %cl
+; AVX512-NEXT:    kshiftlw $1, %k0, %k0
 ; AVX512-NEXT:    andl $1, %ecx
 ; AVX512-NEXT:    kmovw %ecx, %k1
 ; AVX512-NEXT:    korw %k0, %k1, %k1
-; AVX512-NEXT:    movq %rdx, 24(%r15)
-; AVX512-NEXT:    movq %rax, 16(%r15)
-; AVX512-NEXT:    movq %rbp, 8(%r15)
-; AVX512-NEXT:    movq %r13, (%r15)
 ; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX512-NEXT:    vmovdqa64 %xmm0, %xmm0 {%k1} {z}
+; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
+; AVX512-NEXT:    movq %rdx, 24(%r12)
+; AVX512-NEXT:    movq %rax, 16(%r12)
+; AVX512-NEXT:    movq %rbp, 8(%r12)
+; AVX512-NEXT:    movq %r14, (%r12)
 ; AVX512-NEXT:    addq $24, %rsp
 ; AVX512-NEXT:    popq %rbx
 ; AVX512-NEXT:    popq %r12
diff --git a/llvm/test/CodeGen/X86/vec_split.ll b/llvm/test/CodeGen/X86/vec_split.ll
index 9ce9b5b15..559b19a86 100644
--- a/llvm/test/CodeGen/X86/vec_split.ll
+++ b/llvm/test/CodeGen/X86/vec_split.ll
@@ -1,19 +1,28 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=x86_64-- -mattr=sse4.1 < %s | FileCheck %s -check-prefix=SSE4
 ; RUN: llc -mtriple=x86_64-- -mattr=avx < %s | FileCheck %s -check-prefix=AVX1
 ; RUN: llc -mtriple=x86_64-- -mattr=avx2 < %s | FileCheck %s -check-prefix=AVX2
 
 define <16 x i16> @split16(<16 x i16> %a, <16 x i16> %b, <16 x i8> %__mask) {
 ; SSE4-LABEL: split16:
-; SSE4: pminuw
-; SSE4: pminuw
-; SSE4: ret
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pminuw %xmm2, %xmm0
+; SSE4-NEXT:    pminuw %xmm3, %xmm1
+; SSE4-NEXT:    retq
+;
 ; AVX1-LABEL: split16:
-; AVX1: vpminuw
-; AVX1: vpminuw
-; AVX1: ret
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; AVX1-NEXT:    vpminuw %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vpminuw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
 ; AVX2-LABEL: split16:
-; AVX2: vpminuw
-; AVX2: ret
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpminuw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
   %1 = icmp ult <16 x i16> %a, %b
   %2 = select <16 x i1> %1, <16 x i16> %a, <16 x i16> %b
   ret <16 x i16> %2
@@ -21,21 +30,32 @@ define <16 x i16> @split16(<16 x i16> %a, <16 x i16> %b, <16 x i8> %__mask) {
 
 define <32 x i16> @split32(<32 x i16> %a, <32 x i16> %b, <32 x i8> %__mask) {
 ; SSE4-LABEL: split32:
-; SSE4: pminuw
-; SSE4: pminuw
-; SSE4: pminuw
-; SSE4: pminuw
-; SSE4: ret
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pminuw %xmm4, %xmm0
+; SSE4-NEXT:    pminuw %xmm5, %xmm1
+; SSE4-NEXT:    pminuw %xmm6, %xmm2
+; SSE4-NEXT:    pminuw %xmm7, %xmm3
+; SSE4-NEXT:    retq
+;
 ; AVX1-LABEL: split32:
-; AVX1: vpminuw
-; AVX1: vpminuw
-; AVX1: vpminuw
-; AVX1: vpminuw
-; AVX1: ret
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm4
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm5
+; AVX1-NEXT:    vpminuw %xmm4, %xmm5, %xmm4
+; AVX1-NEXT:    vpminuw %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
+; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm4
+; AVX1-NEXT:    vpminuw %xmm2, %xmm4, %xmm2
+; AVX1-NEXT:    vpminuw %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; AVX1-NEXT:    retq
+;
 ; AVX2-LABEL: split32:
-; AVX2: vpminuw
-; AVX2: vpminuw
-; AVX2: ret
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpminuw %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpminuw %ymm3, %ymm1, %ymm1
+; AVX2-NEXT:    retq
   %1 = icmp ult <32 x i16> %a, %b
   %2 = select <32 x i1> %1, <32 x i16> %a, <32 x i16> %b
   ret <32 x i16> %2
@@ -44,29 +64,40 @@ define <32 x i16> @split32(<32 x i16> %a, <32 x i16> %b, <32 x i8> %__mask) {
 ; PR19492
 define i128 @split128(<2 x i128> %a, <2 x i128> %b) {
 ; SSE4-LABEL: split128:
-; SSE4: addq
-; SSE4: adcq
-; SSE4: addq
-; SSE4: adcq
-; SSE4: addq
-; SSE4: adcq
-; SSE4: ret
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    movq %rdx, %rax
+; SSE4-NEXT:    addq %r8, %rdi
+; SSE4-NEXT:    adcq %r9, %rsi
+; SSE4-NEXT:    addq {{[0-9]+}}(%rsp), %rax
+; SSE4-NEXT:    adcq {{[0-9]+}}(%rsp), %rcx
+; SSE4-NEXT:    addq %rdi, %rax
+; SSE4-NEXT:    adcq %rsi, %rcx
+; SSE4-NEXT:    movq %rcx, %rdx
+; SSE4-NEXT:    retq
+;
 ; AVX1-LABEL: split128:
-; AVX1: addq
-; AVX1: adcq
-; AVX1: addq
-; AVX1: adcq
-; AVX1: addq
-; AVX1: adcq
-; AVX1: ret
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    movq %rdx, %rax
+; AVX1-NEXT:    addq %r8, %rdi
+; AVX1-NEXT:    adcq %r9, %rsi
+; AVX1-NEXT:    addq {{[0-9]+}}(%rsp), %rax
+; AVX1-NEXT:    adcq {{[0-9]+}}(%rsp), %rcx
+; AVX1-NEXT:    addq %rdi, %rax
+; AVX1-NEXT:    adcq %rsi, %rcx
+; AVX1-NEXT:    movq %rcx, %rdx
+; AVX1-NEXT:    retq
+;
 ; AVX2-LABEL: split128:
-; AVX2: addq
-; AVX2: adcq
-; AVX2: addq
-; AVX2: adcq
-; AVX2: addq
-; AVX2: adcq
-; AVX2: ret
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    movq %rdx, %rax
+; AVX2-NEXT:    addq %r8, %rdi
+; AVX2-NEXT:    adcq %r9, %rsi
+; AVX2-NEXT:    addq {{[0-9]+}}(%rsp), %rax
+; AVX2-NEXT:    adcq {{[0-9]+}}(%rsp), %rcx
+; AVX2-NEXT:    addq %rdi, %rax
+; AVX2-NEXT:    adcq %rsi, %rcx
+; AVX2-NEXT:    movq %rcx, %rdx
+; AVX2-NEXT:    retq
   %add = add nsw <2 x i128> %a, %b
   %rdx.shuf = shufflevector <2 x i128> %add, <2 x i128> undef, <2 x i32> <i32 undef, i32 0>
   %bin.rdx = add <2 x i128> %add, %rdx.shuf
diff --git a/llvm/test/CodeGen/X86/vec_ss_load_fold.ll b/llvm/test/CodeGen/X86/vec_ss_load_fold.ll
index 1f5503067..259c19c68 100644
--- a/llvm/test/CodeGen/X86/vec_ss_load_fold.ll
+++ b/llvm/test/CodeGen/X86/vec_ss_load_fold.ll
@@ -64,8 +64,8 @@ define i16 @test1(float %f) nounwind {
 ; X32_AVX512-NEXT:    vmulss LCPI0_1, %xmm0, %xmm0
 ; X32_AVX512-NEXT:    vxorps %xmm1, %xmm1, %xmm1
 ; X32_AVX512-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
-; X32_AVX512-NEXT:    vxorps %xmm1, %xmm1, %xmm1
 ; X32_AVX512-NEXT:    vminss LCPI0_2, %xmm0, %xmm0
+; X32_AVX512-NEXT:    vxorps %xmm1, %xmm1, %xmm1
 ; X32_AVX512-NEXT:    vmaxss %xmm1, %xmm0, %xmm0
 ; X32_AVX512-NEXT:    vcvttss2si %xmm0, %eax
 ; X32_AVX512-NEXT:    ## kill: def $ax killed $ax killed $eax
@@ -77,8 +77,8 @@ define i16 @test1(float %f) nounwind {
 ; X64_AVX512-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm0
 ; X64_AVX512-NEXT:    vxorps %xmm1, %xmm1, %xmm1
 ; X64_AVX512-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
-; X64_AVX512-NEXT:    vxorps %xmm1, %xmm1, %xmm1
 ; X64_AVX512-NEXT:    vminss {{.*}}(%rip), %xmm0, %xmm0
+; X64_AVX512-NEXT:    vxorps %xmm1, %xmm1, %xmm1
 ; X64_AVX512-NEXT:    vmaxss %xmm1, %xmm0, %xmm0
 ; X64_AVX512-NEXT:    vcvttss2si %xmm0, %eax
 ; X64_AVX512-NEXT:    ## kill: def $ax killed $ax killed $eax
diff --git a/llvm/test/CodeGen/X86/vec_ssubo.ll b/llvm/test/CodeGen/X86/vec_ssubo.ll
index 515dc5c5a..48834a7de 100644
--- a/llvm/test/CodeGen/X86/vec_ssubo.ll
+++ b/llvm/test/CodeGen/X86/vec_ssubo.ll
@@ -49,134 +49,47 @@ define <1 x i32> @ssubo_v1i32(<1 x i32> %a0, <1 x i32> %a1, <1 x i32>* %p2) noun
 }
 
 define <2 x i32> @ssubo_v2i32(<2 x i32> %a0, <2 x i32> %a1, <2 x i32>* %p2) nounwind {
-; SSE2-LABEL: ssubo_v2i32:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    psllq $32, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,3,2,3]
-; SSE2-NEXT:    psrad $31, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; SSE2-NEXT:    psllq $32, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,3,2,3]
-; SSE2-NEXT:    psrad $31, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; SSE2-NEXT:    psubq %xmm2, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm0
-; SSE2-NEXT:    psllq $32, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,3,2,3]
-; SSE2-NEXT:    psrad $31, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,0,3,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE2-NEXT:    pxor %xmm3, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; SSE2-NEXT:    movq %xmm1, (%rdi)
-; SSE2-NEXT:    retq
-;
-; SSSE3-LABEL: ssubo_v2i32:
-; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    psllq $32, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,3,2,3]
-; SSSE3-NEXT:    psrad $31, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; SSSE3-NEXT:    psllq $32, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,3,2,3]
-; SSSE3-NEXT:    psrad $31, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; SSSE3-NEXT:    psubq %xmm2, %xmm1
-; SSSE3-NEXT:    movdqa %xmm1, %xmm0
-; SSSE3-NEXT:    psllq $32, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,3,2,3]
-; SSSE3-NEXT:    psrad $31, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm2, %xmm3
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSSE3-NEXT:    pxor %xmm3, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; SSSE3-NEXT:    movq %xmm1, (%rdi)
-; SSSE3-NEXT:    retq
-;
-; SSE41-LABEL: ssubo_v2i32:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa %xmm1, %xmm2
-; SSE41-NEXT:    psllq $32, %xmm2
-; SSE41-NEXT:    psrad $31, %xmm2
-; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
-; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    psllq $32, %xmm1
-; SSE41-NEXT:    psrad $31, %xmm1
-; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
-; SSE41-NEXT:    psubq %xmm2, %xmm1
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    psllq $32, %xmm0
-; SSE41-NEXT:    psrad $31, %xmm0
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3],xmm1[4,5],xmm0[6,7]
-; SSE41-NEXT:    pcmpeqq %xmm1, %xmm0
-; SSE41-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE41-NEXT:    pxor %xmm2, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; SSE41-NEXT:    movq %xmm1, (%rdi)
-; SSE41-NEXT:    retq
+; SSE-LABEL: ssubo_v2i32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pxor %xmm2, %xmm2
+; SSE-NEXT:    movdqa %xmm0, %xmm3
+; SSE-NEXT:    psubd %xmm1, %xmm3
+; SSE-NEXT:    pcmpgtd %xmm2, %xmm1
+; SSE-NEXT:    pcmpgtd %xmm3, %xmm0
+; SSE-NEXT:    pxor %xmm1, %xmm0
+; SSE-NEXT:    movq %xmm3, (%rdi)
+; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: ssubo_v2i32:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpsllq $32, %xmm1, %xmm2
-; AVX1-NEXT:    vpsrad $31, %xmm2, %xmm2
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
-; AVX1-NEXT:    vpsllq $32, %xmm0, %xmm2
-; AVX1-NEXT:    vpsrad $31, %xmm2, %xmm2
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
-; AVX1-NEXT:    vpsubq %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpsllq $32, %xmm1, %xmm0
-; AVX1-NEXT:    vpsrad $31, %xmm0, %xmm0
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3],xmm1[4,5],xmm0[6,7]
-; AVX1-NEXT:    vpcmpeqq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm1, %xmm2
+; AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor %xmm0, %xmm2, %xmm0
 ; AVX1-NEXT:    vmovq %xmm1, (%rdi)
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: ssubo_v2i32:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpsllq $32, %xmm1, %xmm2
-; AVX2-NEXT:    vpsrad $31, %xmm2, %xmm2
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; AVX2-NEXT:    vpsllq $32, %xmm0, %xmm2
-; AVX2-NEXT:    vpsrad $31, %xmm2, %xmm2
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
-; AVX2-NEXT:    vpsubq %xmm1, %xmm0, %xmm1
-; AVX2-NEXT:    vpsllq $32, %xmm1, %xmm0
-; AVX2-NEXT:    vpsrad $31, %xmm0, %xmm0
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2],xmm0[3]
-; AVX2-NEXT:    vpcmpeqq %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpcmpgtd %xmm2, %xmm1, %xmm2
+; AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm1
+; AVX2-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor %xmm0, %xmm2, %xmm0
 ; AVX2-NEXT:    vmovq %xmm1, (%rdi)
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: ssubo_v2i32:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsllq $32, %xmm1, %xmm1
-; AVX512-NEXT:    vpsraq $32, %xmm1, %xmm1
-; AVX512-NEXT:    vpsllq $32, %xmm0, %xmm0
-; AVX512-NEXT:    vpsraq $32, %xmm0, %xmm0
-; AVX512-NEXT:    vpsubq %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpsllq $32, %xmm0, %xmm1
-; AVX512-NEXT:    vpsraq $32, %xmm1, %xmm1
-; AVX512-NEXT:    vpmovqd %xmm0, (%rdi)
-; AVX512-NEXT:    vpcmpeqq %xmm0, %xmm1, %xmm0
-; AVX512-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
+; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512-NEXT:    vpcmpgtd %xmm2, %xmm1, %k0
+; AVX512-NEXT:    vpsubd %xmm1, %xmm0, %xmm1
+; AVX512-NEXT:    vpcmpgtd %xmm1, %xmm0, %k1
+; AVX512-NEXT:    kxorw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
+; AVX512-NEXT:    vmovq %xmm1, (%rdi)
 ; AVX512-NEXT:    retq
   %t = call {<2 x i32>, <2 x i1>} @llvm.ssub.with.overflow.v2i32(<2 x i32> %a0, <2 x i32> %a1)
   %val = extractvalue {<2 x i32>, <2 x i1>} %t, 0
@@ -189,87 +102,49 @@ define <2 x i32> @ssubo_v2i32(<2 x i32> %a0, <2 x i32> %a1, <2 x i32>* %p2) noun
 define <3 x i32> @ssubo_v3i32(<3 x i32> %a0, <3 x i32> %a1, <3 x i32>* %p2) nounwind {
 ; SSE2-LABEL: ssubo_v3i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pxor %xmm3, %xmm3
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm4
-; SSE2-NEXT:    pxor %xmm4, %xmm2
-; SSE2-NEXT:    pxor %xmm5, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm5
-; SSE2-NEXT:    pxor %xmm4, %xmm5
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm2
-; SSE2-NEXT:    psubd %xmm1, %xmm0
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSE2-NEXT:    pxor %xmm4, %xmm3
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm3
-; SSE2-NEXT:    pxor %xmm4, %xmm3
-; SSE2-NEXT:    pandn %xmm3, %xmm2
-; SSE2-NEXT:    movq %xmm0, (%rdi)
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; SSE2-NEXT:    movd %xmm0, 8(%rdi)
-; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm3
+; SSE2-NEXT:    psubd %xmm1, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm0
+; SSE2-NEXT:    movq %xmm3, (%rdi)
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[2,3,0,1]
+; SSE2-NEXT:    movd %xmm1, 8(%rdi)
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: ssubo_v3i32:
 ; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    pxor %xmm3, %xmm3
 ; SSSE3-NEXT:    pxor %xmm2, %xmm2
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
-; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm4
-; SSSE3-NEXT:    pxor %xmm4, %xmm2
-; SSSE3-NEXT:    pxor %xmm5, %xmm5
-; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm5
-; SSSE3-NEXT:    pxor %xmm4, %xmm5
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm2
-; SSSE3-NEXT:    psubd %xmm1, %xmm0
-; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSSE3-NEXT:    pxor %xmm4, %xmm3
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm3
-; SSSE3-NEXT:    pxor %xmm4, %xmm3
-; SSSE3-NEXT:    pandn %xmm3, %xmm2
-; SSSE3-NEXT:    movq %xmm0, (%rdi)
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; SSSE3-NEXT:    movd %xmm0, 8(%rdi)
-; SSSE3-NEXT:    movdqa %xmm2, %xmm0
+; SSSE3-NEXT:    movdqa %xmm0, %xmm3
+; SSSE3-NEXT:    psubd %xmm1, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm1
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
+; SSSE3-NEXT:    pxor %xmm1, %xmm0
+; SSSE3-NEXT:    movq %xmm3, (%rdi)
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[2,3,0,1]
+; SSSE3-NEXT:    movd %xmm1, 8(%rdi)
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ssubo_v3i32:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pxor %xmm3, %xmm3
 ; SSE41-NEXT:    pxor %xmm2, %xmm2
-; SSE41-NEXT:    pcmpgtd %xmm1, %xmm2
-; SSE41-NEXT:    pcmpeqd %xmm4, %xmm4
-; SSE41-NEXT:    pxor %xmm4, %xmm2
-; SSE41-NEXT:    pxor %xmm5, %xmm5
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm5
-; SSE41-NEXT:    pxor %xmm4, %xmm5
-; SSE41-NEXT:    pcmpeqd %xmm5, %xmm2
-; SSE41-NEXT:    psubd %xmm1, %xmm0
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSE41-NEXT:    pxor %xmm4, %xmm3
-; SSE41-NEXT:    pcmpeqd %xmm5, %xmm3
-; SSE41-NEXT:    pxor %xmm4, %xmm3
-; SSE41-NEXT:    pandn %xmm3, %xmm2
-; SSE41-NEXT:    pextrd $2, %xmm0, 8(%rdi)
-; SSE41-NEXT:    movq %xmm0, (%rdi)
-; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    movdqa %xmm0, %xmm3
+; SSE41-NEXT:    psubd %xmm1, %xmm3
+; SSE41-NEXT:    pcmpgtd %xmm2, %xmm1
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
+; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    pextrd $2, %xmm3, 8(%rdi)
+; SSE41-NEXT:    movq %xmm3, (%rdi)
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ssubo_v3i32:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm3
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm2, %xmm5
-; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm5, %xmm3
+; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm1, %xmm2
 ; AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm0
-; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm5, %xmm0
-; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm0
-; AVX1-NEXT:    vpandn %xmm0, %xmm3, %xmm0
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor %xmm0, %xmm2, %xmm0
 ; AVX1-NEXT:    vpextrd $2, %xmm1, 8(%rdi)
 ; AVX1-NEXT:    vmovq %xmm1, (%rdi)
 ; AVX1-NEXT:    retq
@@ -277,18 +152,10 @@ define <3 x i32> @ssubo_v3i32(<3 x i32> %a0, <3 x i32> %a1, <3 x i32>* %p2) noun
 ; AVX2-LABEL: ssubo_v3i32:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm3
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpxor %xmm4, %xmm3, %xmm3
-; AVX2-NEXT:    vpcmpgtd %xmm0, %xmm2, %xmm5
-; AVX2-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm5, %xmm3
+; AVX2-NEXT:    vpcmpgtd %xmm2, %xmm1, %xmm2
 ; AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm1
-; AVX2-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm0
-; AVX2-NEXT:    vpxor %xmm4, %xmm0, %xmm0
-; AVX2-NEXT:    vpcmpeqd %xmm0, %xmm5, %xmm0
-; AVX2-NEXT:    vpxor %xmm4, %xmm0, %xmm0
-; AVX2-NEXT:    vpandn %xmm0, %xmm3, %xmm0
+; AVX2-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor %xmm0, %xmm2, %xmm0
 ; AVX2-NEXT:    vpextrd $2, %xmm1, 8(%rdi)
 ; AVX2-NEXT:    vmovq %xmm1, (%rdi)
 ; AVX2-NEXT:    retq
@@ -296,13 +163,10 @@ define <3 x i32> @ssubo_v3i32(<3 x i32> %a0, <3 x i32> %a1, <3 x i32>* %p2) noun
 ; AVX512-LABEL: ssubo_v3i32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpcmpnltd %xmm2, %xmm1, %k0
-; AVX512-NEXT:    vpcmpnltd %xmm2, %xmm0, %k1
-; AVX512-NEXT:    kxorw %k0, %k1, %k0
+; AVX512-NEXT:    vpcmpgtd %xmm2, %xmm1, %k0
 ; AVX512-NEXT:    vpsubd %xmm1, %xmm0, %xmm1
-; AVX512-NEXT:    vpcmpnltd %xmm2, %xmm1, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
-; AVX512-NEXT:    kandw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpgtd %xmm1, %xmm0, %k1
+; AVX512-NEXT:    kxorw %k1, %k0, %k1
 ; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
 ; AVX512-NEXT:    vpextrd $2, %xmm1, 8(%rdi)
@@ -319,71 +183,42 @@ define <3 x i32> @ssubo_v3i32(<3 x i32> %a0, <3 x i32> %a1, <3 x i32>* %p2) noun
 define <4 x i32> @ssubo_v4i32(<4 x i32> %a0, <4 x i32> %a1, <4 x i32>* %p2) nounwind {
 ; SSE-LABEL: ssubo_v4i32:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pxor %xmm3, %xmm3
 ; SSE-NEXT:    pxor %xmm2, %xmm2
-; SSE-NEXT:    pcmpgtd %xmm1, %xmm2
-; SSE-NEXT:    pcmpeqd %xmm4, %xmm4
-; SSE-NEXT:    pxor %xmm4, %xmm2
-; SSE-NEXT:    pxor %xmm5, %xmm5
-; SSE-NEXT:    pcmpgtd %xmm0, %xmm5
-; SSE-NEXT:    pxor %xmm4, %xmm5
-; SSE-NEXT:    pcmpeqd %xmm5, %xmm2
-; SSE-NEXT:    psubd %xmm1, %xmm0
-; SSE-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSE-NEXT:    pxor %xmm4, %xmm3
-; SSE-NEXT:    pcmpeqd %xmm5, %xmm3
-; SSE-NEXT:    pxor %xmm4, %xmm3
-; SSE-NEXT:    pandn %xmm3, %xmm2
-; SSE-NEXT:    movdqa %xmm0, (%rdi)
-; SSE-NEXT:    movdqa %xmm2, %xmm0
+; SSE-NEXT:    movdqa %xmm0, %xmm3
+; SSE-NEXT:    psubd %xmm1, %xmm3
+; SSE-NEXT:    pcmpgtd %xmm2, %xmm1
+; SSE-NEXT:    pcmpgtd %xmm3, %xmm0
+; SSE-NEXT:    pxor %xmm1, %xmm0
+; SSE-NEXT:    movdqa %xmm3, (%rdi)
 ; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: ssubo_v4i32:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm3
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm2, %xmm5
-; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm5, %xmm3
+; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm1, %xmm2
 ; AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm0
-; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm5, %xmm0
-; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm0
-; AVX1-NEXT:    vpandn %xmm0, %xmm3, %xmm0
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor %xmm0, %xmm2, %xmm0
 ; AVX1-NEXT:    vmovdqa %xmm1, (%rdi)
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: ssubo_v4i32:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm3
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpxor %xmm4, %xmm3, %xmm3
-; AVX2-NEXT:    vpcmpgtd %xmm0, %xmm2, %xmm5
-; AVX2-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm5, %xmm3
+; AVX2-NEXT:    vpcmpgtd %xmm2, %xmm1, %xmm2
 ; AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm1
-; AVX2-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm0
-; AVX2-NEXT:    vpxor %xmm4, %xmm0, %xmm0
-; AVX2-NEXT:    vpcmpeqd %xmm0, %xmm5, %xmm0
-; AVX2-NEXT:    vpxor %xmm4, %xmm0, %xmm0
-; AVX2-NEXT:    vpandn %xmm0, %xmm3, %xmm0
+; AVX2-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor %xmm0, %xmm2, %xmm0
 ; AVX2-NEXT:    vmovdqa %xmm1, (%rdi)
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: ssubo_v4i32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpcmpnltd %xmm2, %xmm1, %k0
-; AVX512-NEXT:    vpcmpnltd %xmm2, %xmm0, %k1
-; AVX512-NEXT:    kxorw %k0, %k1, %k0
+; AVX512-NEXT:    vpcmpgtd %xmm2, %xmm1, %k0
 ; AVX512-NEXT:    vpsubd %xmm1, %xmm0, %xmm1
-; AVX512-NEXT:    vpcmpnltd %xmm2, %xmm1, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
-; AVX512-NEXT:    kandw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpgtd %xmm1, %xmm0, %k1
+; AVX512-NEXT:    kxorw %k1, %k0, %k1
 ; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
 ; AVX512-NEXT:    vmovdqa %xmm1, (%rdi)
@@ -400,201 +235,132 @@ define <6 x i32> @ssubo_v6i32(<6 x i32> %a0, <6 x i32> %a1, <6 x i32>* %p2) noun
 ; SSE2-LABEL: ssubo_v6i32:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movq %rdi, %rax
-; SSE2-NEXT:    movd %r8d, %xmm0
-; SSE2-NEXT:    movd %ecx, %xmm1
+; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; SSE2-NEXT:    movd %edx, %xmm2
-; SSE2-NEXT:    movd %esi, %xmm0
+; SSE2-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
 ; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE2-NEXT:    movd %r8d, %xmm1
+; SSE2-NEXT:    movd %ecx, %xmm2
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSE2-NEXT:    movd %edx, %xmm1
+; SSE2-NEXT:    movd %esi, %xmm3
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm2[0]
 ; SSE2-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; SSE2-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; SSE2-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; SSE2-NEXT:    movd {{.*#+}} xmm6 = mem[0],zero,zero,zero
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm6 = xmm6[0],xmm1[0],xmm6[1],xmm1[1]
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm6 = xmm6[0],xmm2[0]
 ; SSE2-NEXT:    movd %r9d, %xmm1
-; SSE2-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
-; SSE2-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
 ; SSE2-NEXT:    movd {{.*#+}} xmm4 = mem[0],zero,zero,zero
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
 ; SSE2-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
-; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpgtd %xmm6, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm5
-; SSE2-NEXT:    pxor %xmm5, %xmm2
-; SSE2-NEXT:    pxor %xmm7, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm7
-; SSE2-NEXT:    pxor %xmm5, %xmm7
-; SSE2-NEXT:    pcmpeqd %xmm7, %xmm2
-; SSE2-NEXT:    psubd %xmm6, %xmm0
-; SSE2-NEXT:    pxor %xmm6, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm6
-; SSE2-NEXT:    pxor %xmm5, %xmm6
-; SSE2-NEXT:    pcmpeqd %xmm7, %xmm6
-; SSE2-NEXT:    pxor %xmm5, %xmm6
-; SSE2-NEXT:    pandn %xmm6, %xmm2
-; SSE2-NEXT:    pxor %xmm6, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm4, %xmm6
-; SSE2-NEXT:    pxor %xmm5, %xmm6
-; SSE2-NEXT:    pxor %xmm7, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm7
-; SSE2-NEXT:    pxor %xmm5, %xmm7
-; SSE2-NEXT:    pcmpeqd %xmm7, %xmm6
-; SSE2-NEXT:    psubd %xmm4, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
-; SSE2-NEXT:    pxor %xmm5, %xmm3
-; SSE2-NEXT:    pcmpeqd %xmm7, %xmm3
-; SSE2-NEXT:    pxor %xmm5, %xmm3
-; SSE2-NEXT:    pandn %xmm3, %xmm6
-; SSE2-NEXT:    movq %xmm1, 16(%rcx)
-; SSE2-NEXT:    movdqa %xmm0, (%rcx)
-; SSE2-NEXT:    movq %xmm6, 16(%rdi)
-; SSE2-NEXT:    movdqa %xmm2, (%rdi)
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    psubd %xmm0, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm3
+; SSE2-NEXT:    pxor %xmm5, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm5, %xmm0
+; SSE2-NEXT:    pxor %xmm3, %xmm0
+; SSE2-NEXT:    movdqa %xmm1, %xmm3
+; SSE2-NEXT:    psubd %xmm2, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm5, %xmm2
+; SSE2-NEXT:    pxor %xmm1, %xmm2
+; SSE2-NEXT:    movq %xmm3, 16(%rcx)
+; SSE2-NEXT:    movdqa %xmm4, (%rcx)
+; SSE2-NEXT:    movq %xmm2, 16(%rdi)
+; SSE2-NEXT:    movdqa %xmm0, (%rdi)
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: ssubo_v6i32:
 ; SSSE3:       # %bb.0:
 ; SSSE3-NEXT:    movq %rdi, %rax
-; SSSE3-NEXT:    movd %r8d, %xmm0
-; SSSE3-NEXT:    movd %ecx, %xmm1
+; SSSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSSE3-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; SSSE3-NEXT:    movd %edx, %xmm2
-; SSSE3-NEXT:    movd %esi, %xmm0
+; SSSE3-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
 ; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSSE3-NEXT:    movd %r8d, %xmm1
+; SSSE3-NEXT:    movd %ecx, %xmm2
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSSE3-NEXT:    movd %edx, %xmm1
+; SSSE3-NEXT:    movd %esi, %xmm3
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm2[0]
 ; SSSE3-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; SSSE3-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
 ; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; SSSE3-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; SSSE3-NEXT:    movd {{.*#+}} xmm6 = mem[0],zero,zero,zero
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm6 = xmm6[0],xmm1[0],xmm6[1],xmm1[1]
-; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm6 = xmm6[0],xmm2[0]
 ; SSSE3-NEXT:    movd %r9d, %xmm1
-; SSSE3-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
-; SSSE3-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
 ; SSSE3-NEXT:    movd {{.*#+}} xmm4 = mem[0],zero,zero,zero
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
 ; SSSE3-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
-; SSSE3-NEXT:    pxor %xmm3, %xmm3
-; SSSE3-NEXT:    pxor %xmm2, %xmm2
-; SSSE3-NEXT:    pcmpgtd %xmm6, %xmm2
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm5
-; SSSE3-NEXT:    pxor %xmm5, %xmm2
-; SSSE3-NEXT:    pxor %xmm7, %xmm7
-; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm7
-; SSSE3-NEXT:    pxor %xmm5, %xmm7
-; SSSE3-NEXT:    pcmpeqd %xmm7, %xmm2
-; SSSE3-NEXT:    psubd %xmm6, %xmm0
-; SSSE3-NEXT:    pxor %xmm6, %xmm6
-; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm6
-; SSSE3-NEXT:    pxor %xmm5, %xmm6
-; SSSE3-NEXT:    pcmpeqd %xmm7, %xmm6
-; SSSE3-NEXT:    pxor %xmm5, %xmm6
-; SSSE3-NEXT:    pandn %xmm6, %xmm2
-; SSSE3-NEXT:    pxor %xmm6, %xmm6
-; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm6
-; SSSE3-NEXT:    pxor %xmm5, %xmm6
-; SSSE3-NEXT:    pxor %xmm7, %xmm7
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm7
-; SSSE3-NEXT:    pxor %xmm5, %xmm7
-; SSSE3-NEXT:    pcmpeqd %xmm7, %xmm6
-; SSSE3-NEXT:    psubd %xmm4, %xmm1
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm3
-; SSSE3-NEXT:    pxor %xmm5, %xmm3
-; SSSE3-NEXT:    pcmpeqd %xmm7, %xmm3
-; SSSE3-NEXT:    pxor %xmm5, %xmm3
-; SSSE3-NEXT:    pandn %xmm3, %xmm6
-; SSSE3-NEXT:    movq %xmm1, 16(%rcx)
-; SSSE3-NEXT:    movdqa %xmm0, (%rcx)
-; SSSE3-NEXT:    movq %xmm6, 16(%rdi)
-; SSSE3-NEXT:    movdqa %xmm2, (%rdi)
+; SSSE3-NEXT:    movdqa %xmm3, %xmm4
+; SSSE3-NEXT:    psubd %xmm0, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm3
+; SSSE3-NEXT:    pxor %xmm5, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm0
+; SSSE3-NEXT:    pxor %xmm3, %xmm0
+; SSSE3-NEXT:    movdqa %xmm1, %xmm3
+; SSSE3-NEXT:    psubd %xmm2, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm1
+; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm2
+; SSSE3-NEXT:    pxor %xmm1, %xmm2
+; SSSE3-NEXT:    movq %xmm3, 16(%rcx)
+; SSSE3-NEXT:    movdqa %xmm4, (%rcx)
+; SSSE3-NEXT:    movq %xmm2, 16(%rdi)
+; SSSE3-NEXT:    movdqa %xmm0, (%rdi)
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ssubo_v6i32:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movq %rdi, %rax
-; SSE41-NEXT:    movd %esi, %xmm0
-; SSE41-NEXT:    pinsrd $1, %edx, %xmm0
-; SSE41-NEXT:    pinsrd $2, %ecx, %xmm0
-; SSE41-NEXT:    pinsrd $3, %r8d, %xmm0
-; SSE41-NEXT:    movd %r9d, %xmm1
-; SSE41-NEXT:    pinsrd $1, {{[0-9]+}}(%rsp), %xmm1
+; SSE41-NEXT:    movd %esi, %xmm1
+; SSE41-NEXT:    pinsrd $1, %edx, %xmm1
+; SSE41-NEXT:    pinsrd $2, %ecx, %xmm1
+; SSE41-NEXT:    pinsrd $3, %r8d, %xmm1
+; SSE41-NEXT:    movd %r9d, %xmm0
+; SSE41-NEXT:    pinsrd $1, {{[0-9]+}}(%rsp), %xmm0
+; SSE41-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE41-NEXT:    pinsrd $1, {{[0-9]+}}(%rsp), %xmm2
 ; SSE41-NEXT:    movd {{.*#+}} xmm3 = mem[0],zero,zero,zero
 ; SSE41-NEXT:    pinsrd $1, {{[0-9]+}}(%rsp), %xmm3
-; SSE41-NEXT:    movd {{.*#+}} xmm6 = mem[0],zero,zero,zero
-; SSE41-NEXT:    pinsrd $1, {{[0-9]+}}(%rsp), %xmm6
-; SSE41-NEXT:    pinsrd $2, {{[0-9]+}}(%rsp), %xmm6
-; SSE41-NEXT:    pinsrd $3, {{[0-9]+}}(%rsp), %xmm6
+; SSE41-NEXT:    pinsrd $2, {{[0-9]+}}(%rsp), %xmm3
+; SSE41-NEXT:    pinsrd $3, {{[0-9]+}}(%rsp), %xmm3
 ; SSE41-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
-; SSE41-NEXT:    pxor %xmm4, %xmm4
-; SSE41-NEXT:    pxor %xmm2, %xmm2
-; SSE41-NEXT:    pcmpgtd %xmm6, %xmm2
-; SSE41-NEXT:    pcmpeqd %xmm5, %xmm5
-; SSE41-NEXT:    pxor %xmm5, %xmm2
-; SSE41-NEXT:    pxor %xmm7, %xmm7
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm7
-; SSE41-NEXT:    pxor %xmm5, %xmm7
-; SSE41-NEXT:    pcmpeqd %xmm7, %xmm2
-; SSE41-NEXT:    psubd %xmm6, %xmm0
-; SSE41-NEXT:    pxor %xmm6, %xmm6
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
-; SSE41-NEXT:    pxor %xmm5, %xmm6
-; SSE41-NEXT:    pcmpeqd %xmm7, %xmm6
-; SSE41-NEXT:    pxor %xmm5, %xmm6
-; SSE41-NEXT:    pandn %xmm6, %xmm2
-; SSE41-NEXT:    pxor %xmm6, %xmm6
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm6
-; SSE41-NEXT:    pxor %xmm5, %xmm6
-; SSE41-NEXT:    pxor %xmm7, %xmm7
-; SSE41-NEXT:    pcmpgtd %xmm1, %xmm7
-; SSE41-NEXT:    pxor %xmm5, %xmm7
-; SSE41-NEXT:    pcmpeqd %xmm7, %xmm6
-; SSE41-NEXT:    psubd %xmm3, %xmm1
-; SSE41-NEXT:    pcmpgtd %xmm1, %xmm4
-; SSE41-NEXT:    pxor %xmm5, %xmm4
-; SSE41-NEXT:    pcmpeqd %xmm7, %xmm4
-; SSE41-NEXT:    pxor %xmm5, %xmm4
-; SSE41-NEXT:    pandn %xmm4, %xmm6
+; SSE41-NEXT:    movdqa %xmm1, %xmm4
+; SSE41-NEXT:    psubd %xmm3, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm4, %xmm1
+; SSE41-NEXT:    pxor %xmm5, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm5, %xmm3
+; SSE41-NEXT:    pxor %xmm1, %xmm3
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    psubd %xmm2, %xmm1
+; SSE41-NEXT:    pcmpgtd %xmm5, %xmm2
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
+; SSE41-NEXT:    pxor %xmm2, %xmm0
 ; SSE41-NEXT:    movq %xmm1, 16(%rcx)
-; SSE41-NEXT:    movdqa %xmm0, (%rcx)
-; SSE41-NEXT:    movq %xmm6, 16(%rdi)
-; SSE41-NEXT:    movdqa %xmm2, (%rdi)
+; SSE41-NEXT:    movdqa %xmm4, (%rcx)
+; SSE41-NEXT:    movq %xmm0, 16(%rdi)
+; SSE41-NEXT:    movdqa %xmm3, (%rdi)
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ssubo_v6i32:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
 ; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm3, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
-; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm6
-; AVX1-NEXT:    vpcmpgtd %xmm6, %xmm3, %xmm7
-; AVX1-NEXT:    vpxor %xmm5, %xmm7, %xmm7
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm7, %xmm8
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm3, %xmm4
-; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm9
-; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm3, %xmm4
-; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm9, %xmm4, %xmm9
-; AVX1-NEXT:    vinsertf128 $1, %xmm8, %ymm9, %ymm8
-; AVX1-NEXT:    vpsubd %xmm2, %xmm6, %xmm2
-; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm3, %xmm6
-; AVX1-NEXT:    vpxor %xmm5, %xmm6, %xmm6
-; AVX1-NEXT:    vpcmpeqd %xmm6, %xmm7, %xmm6
-; AVX1-NEXT:    vpxor %xmm5, %xmm6, %xmm6
+; AVX1-NEXT:    vpcmpgtd %xmm3, %xmm2, %xmm4
+; AVX1-NEXT:    vpcmpgtd %xmm3, %xmm1, %xmm3
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
+; AVX1-NEXT:    vpsubd %xmm2, %xmm4, %xmm2
+; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm4, %xmm4
 ; AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm3, %xmm0
-; AVX1-NEXT:    vpxor %xmm5, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm4, %xmm0
-; AVX1-NEXT:    vpxor %xmm5, %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm0, %ymm0
-; AVX1-NEXT:    vandnps %ymm0, %ymm8, %ymm0
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
+; AVX1-NEXT:    vxorps %ymm0, %ymm3, %ymm0
 ; AVX1-NEXT:    vmovq %xmm2, 16(%rdi)
 ; AVX1-NEXT:    vmovdqa %xmm1, (%rdi)
 ; AVX1-NEXT:    retq
@@ -602,18 +368,10 @@ define <6 x i32> @ssubo_v6i32(<6 x i32> %a0, <6 x i32> %a1, <6 x i32>* %p2) noun
 ; AVX2-LABEL: ssubo_v6i32:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm2, %ymm3
-; AVX2-NEXT:    vpcmpeqd %ymm4, %ymm4, %ymm4
-; AVX2-NEXT:    vpxor %ymm4, %ymm3, %ymm3
-; AVX2-NEXT:    vpcmpgtd %ymm0, %ymm2, %ymm5
-; AVX2-NEXT:    vpxor %ymm4, %ymm5, %ymm5
-; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm5, %ymm3
+; AVX2-NEXT:    vpcmpgtd %ymm2, %ymm1, %ymm2
 ; AVX2-NEXT:    vpsubd %ymm1, %ymm0, %ymm1
-; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm2, %ymm0
-; AVX2-NEXT:    vpxor %ymm4, %ymm0, %ymm0
-; AVX2-NEXT:    vpcmpeqd %ymm0, %ymm5, %ymm0
-; AVX2-NEXT:    vpxor %ymm4, %ymm0, %ymm0
-; AVX2-NEXT:    vpandn %ymm0, %ymm3, %ymm0
+; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %ymm0, %ymm2, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
 ; AVX2-NEXT:    vmovq %xmm2, 16(%rdi)
 ; AVX2-NEXT:    vmovdqa %xmm1, (%rdi)
@@ -622,13 +380,10 @@ define <6 x i32> @ssubo_v6i32(<6 x i32> %a0, <6 x i32> %a1, <6 x i32>* %p2) noun
 ; AVX512-LABEL: ssubo_v6i32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpcmpnltd %ymm2, %ymm1, %k0
-; AVX512-NEXT:    vpcmpnltd %ymm2, %ymm0, %k1
-; AVX512-NEXT:    kxorw %k0, %k1, %k0
+; AVX512-NEXT:    vpcmpgtd %ymm2, %ymm1, %k0
 ; AVX512-NEXT:    vpsubd %ymm1, %ymm0, %ymm1
-; AVX512-NEXT:    vpcmpnltd %ymm2, %ymm1, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
-; AVX512-NEXT:    kandw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpgtd %ymm1, %ymm0, %k1
+; AVX512-NEXT:    kxorw %k1, %k0, %k1
 ; AVX512-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
 ; AVX512-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
 ; AVX512-NEXT:    vextracti128 $1, %ymm1, %xmm2
@@ -646,70 +401,35 @@ define <6 x i32> @ssubo_v6i32(<6 x i32> %a0, <6 x i32> %a1, <6 x i32>* %p2) noun
 define <8 x i32> @ssubo_v8i32(<8 x i32> %a0, <8 x i32> %a1, <8 x i32>* %p2) nounwind {
 ; SSE-LABEL: ssubo_v8i32:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pxor %xmm5, %xmm5
 ; SSE-NEXT:    pxor %xmm4, %xmm4
-; SSE-NEXT:    pcmpgtd %xmm2, %xmm4
-; SSE-NEXT:    pcmpeqd %xmm6, %xmm6
-; SSE-NEXT:    pxor %xmm6, %xmm4
-; SSE-NEXT:    pxor %xmm7, %xmm7
-; SSE-NEXT:    pcmpgtd %xmm0, %xmm7
-; SSE-NEXT:    pxor %xmm6, %xmm7
-; SSE-NEXT:    pcmpeqd %xmm7, %xmm4
-; SSE-NEXT:    psubd %xmm2, %xmm0
-; SSE-NEXT:    pxor %xmm2, %xmm2
-; SSE-NEXT:    pcmpgtd %xmm0, %xmm2
-; SSE-NEXT:    pxor %xmm6, %xmm2
-; SSE-NEXT:    pcmpeqd %xmm7, %xmm2
-; SSE-NEXT:    pxor %xmm6, %xmm2
-; SSE-NEXT:    pandn %xmm2, %xmm4
-; SSE-NEXT:    pxor %xmm2, %xmm2
-; SSE-NEXT:    pcmpgtd %xmm3, %xmm2
-; SSE-NEXT:    pxor %xmm6, %xmm2
-; SSE-NEXT:    pxor %xmm7, %xmm7
-; SSE-NEXT:    pcmpgtd %xmm1, %xmm7
-; SSE-NEXT:    pxor %xmm6, %xmm7
-; SSE-NEXT:    pcmpeqd %xmm7, %xmm2
-; SSE-NEXT:    psubd %xmm3, %xmm1
-; SSE-NEXT:    pcmpgtd %xmm1, %xmm5
-; SSE-NEXT:    pxor %xmm6, %xmm5
-; SSE-NEXT:    pcmpeqd %xmm7, %xmm5
-; SSE-NEXT:    pxor %xmm6, %xmm5
-; SSE-NEXT:    pandn %xmm5, %xmm2
-; SSE-NEXT:    movdqa %xmm1, 16(%rdi)
-; SSE-NEXT:    movdqa %xmm0, (%rdi)
-; SSE-NEXT:    movdqa %xmm4, %xmm0
-; SSE-NEXT:    movdqa %xmm2, %xmm1
+; SSE-NEXT:    movdqa %xmm0, %xmm5
+; SSE-NEXT:    psubd %xmm2, %xmm5
+; SSE-NEXT:    pcmpgtd %xmm4, %xmm2
+; SSE-NEXT:    pcmpgtd %xmm5, %xmm0
+; SSE-NEXT:    pxor %xmm2, %xmm0
+; SSE-NEXT:    movdqa %xmm1, %xmm2
+; SSE-NEXT:    psubd %xmm3, %xmm2
+; SSE-NEXT:    pcmpgtd %xmm4, %xmm3
+; SSE-NEXT:    pcmpgtd %xmm2, %xmm1
+; SSE-NEXT:    pxor %xmm3, %xmm1
+; SSE-NEXT:    movdqa %xmm2, 16(%rdi)
+; SSE-NEXT:    movdqa %xmm5, (%rdi)
 ; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: ssubo_v8i32:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
 ; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm3, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
-; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm6
-; AVX1-NEXT:    vpcmpgtd %xmm6, %xmm3, %xmm7
-; AVX1-NEXT:    vpxor %xmm5, %xmm7, %xmm7
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm7, %xmm8
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm3, %xmm4
-; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm9
-; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm3, %xmm4
-; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm9, %xmm4, %xmm9
-; AVX1-NEXT:    vinsertf128 $1, %xmm8, %ymm9, %ymm8
-; AVX1-NEXT:    vpsubd %xmm2, %xmm6, %xmm2
-; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm3, %xmm6
-; AVX1-NEXT:    vpxor %xmm5, %xmm6, %xmm6
-; AVX1-NEXT:    vpcmpeqd %xmm6, %xmm7, %xmm6
-; AVX1-NEXT:    vpxor %xmm5, %xmm6, %xmm6
+; AVX1-NEXT:    vpcmpgtd %xmm3, %xmm2, %xmm4
+; AVX1-NEXT:    vpcmpgtd %xmm3, %xmm1, %xmm3
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
+; AVX1-NEXT:    vpsubd %xmm2, %xmm4, %xmm2
+; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm4, %xmm4
 ; AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm3, %xmm0
-; AVX1-NEXT:    vpxor %xmm5, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm4, %xmm0
-; AVX1-NEXT:    vpxor %xmm5, %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm0, %ymm0
-; AVX1-NEXT:    vandnps %ymm0, %ymm8, %ymm0
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
+; AVX1-NEXT:    vxorps %ymm0, %ymm3, %ymm0
 ; AVX1-NEXT:    vmovdqa %xmm2, 16(%rdi)
 ; AVX1-NEXT:    vmovdqa %xmm1, (%rdi)
 ; AVX1-NEXT:    retq
@@ -717,31 +437,20 @@ define <8 x i32> @ssubo_v8i32(<8 x i32> %a0, <8 x i32> %a1, <8 x i32>* %p2) noun
 ; AVX2-LABEL: ssubo_v8i32:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm2, %ymm3
-; AVX2-NEXT:    vpcmpeqd %ymm4, %ymm4, %ymm4
-; AVX2-NEXT:    vpxor %ymm4, %ymm3, %ymm3
-; AVX2-NEXT:    vpcmpgtd %ymm0, %ymm2, %ymm5
-; AVX2-NEXT:    vpxor %ymm4, %ymm5, %ymm5
-; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm5, %ymm3
+; AVX2-NEXT:    vpcmpgtd %ymm2, %ymm1, %ymm2
 ; AVX2-NEXT:    vpsubd %ymm1, %ymm0, %ymm1
-; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm2, %ymm0
-; AVX2-NEXT:    vpxor %ymm4, %ymm0, %ymm0
-; AVX2-NEXT:    vpcmpeqd %ymm0, %ymm5, %ymm0
-; AVX2-NEXT:    vpxor %ymm4, %ymm0, %ymm0
-; AVX2-NEXT:    vpandn %ymm0, %ymm3, %ymm0
+; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %ymm0, %ymm2, %ymm0
 ; AVX2-NEXT:    vmovdqa %ymm1, (%rdi)
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: ssubo_v8i32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpcmpnltd %ymm2, %ymm1, %k0
-; AVX512-NEXT:    vpcmpnltd %ymm2, %ymm0, %k1
-; AVX512-NEXT:    kxorw %k0, %k1, %k0
+; AVX512-NEXT:    vpcmpgtd %ymm2, %ymm1, %k0
 ; AVX512-NEXT:    vpsubd %ymm1, %ymm0, %ymm1
-; AVX512-NEXT:    vpcmpnltd %ymm2, %ymm1, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
-; AVX512-NEXT:    kandw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpgtd %ymm1, %ymm0, %k1
+; AVX512-NEXT:    kxorw %k1, %k0, %k1
 ; AVX512-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
 ; AVX512-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
 ; AVX512-NEXT:    vmovdqa %ymm1, (%rdi)
@@ -757,128 +466,59 @@ define <8 x i32> @ssubo_v8i32(<8 x i32> %a0, <8 x i32> %a1, <8 x i32>* %p2) noun
 define <16 x i32> @ssubo_v16i32(<16 x i32> %a0, <16 x i32> %a1, <16 x i32>* %p2) nounwind {
 ; SSE-LABEL: ssubo_v16i32:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pxor %xmm10, %xmm10
-; SSE-NEXT:    pxor %xmm8, %xmm8
-; SSE-NEXT:    pcmpgtd %xmm4, %xmm8
-; SSE-NEXT:    pcmpeqd %xmm11, %xmm11
-; SSE-NEXT:    pxor %xmm11, %xmm8
-; SSE-NEXT:    pxor %xmm9, %xmm9
-; SSE-NEXT:    pcmpgtd %xmm0, %xmm9
-; SSE-NEXT:    pxor %xmm11, %xmm9
-; SSE-NEXT:    pcmpeqd %xmm9, %xmm8
-; SSE-NEXT:    psubd %xmm4, %xmm0
-; SSE-NEXT:    pxor %xmm4, %xmm4
-; SSE-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE-NEXT:    pxor %xmm11, %xmm4
-; SSE-NEXT:    pcmpeqd %xmm9, %xmm4
-; SSE-NEXT:    pxor %xmm11, %xmm4
-; SSE-NEXT:    pandn %xmm4, %xmm8
 ; SSE-NEXT:    pxor %xmm9, %xmm9
-; SSE-NEXT:    pcmpgtd %xmm5, %xmm9
-; SSE-NEXT:    pxor %xmm11, %xmm9
-; SSE-NEXT:    pxor %xmm4, %xmm4
-; SSE-NEXT:    pcmpgtd %xmm1, %xmm4
-; SSE-NEXT:    pxor %xmm11, %xmm4
-; SSE-NEXT:    pcmpeqd %xmm4, %xmm9
-; SSE-NEXT:    psubd %xmm5, %xmm1
-; SSE-NEXT:    pxor %xmm5, %xmm5
-; SSE-NEXT:    pcmpgtd %xmm1, %xmm5
-; SSE-NEXT:    pxor %xmm11, %xmm5
-; SSE-NEXT:    pcmpeqd %xmm4, %xmm5
-; SSE-NEXT:    pxor %xmm11, %xmm5
-; SSE-NEXT:    pandn %xmm5, %xmm9
-; SSE-NEXT:    pxor %xmm4, %xmm4
-; SSE-NEXT:    pcmpgtd %xmm6, %xmm4
-; SSE-NEXT:    pxor %xmm11, %xmm4
-; SSE-NEXT:    pxor %xmm5, %xmm5
-; SSE-NEXT:    pcmpgtd %xmm2, %xmm5
-; SSE-NEXT:    pxor %xmm11, %xmm5
-; SSE-NEXT:    pcmpeqd %xmm5, %xmm4
-; SSE-NEXT:    psubd %xmm6, %xmm2
-; SSE-NEXT:    pxor %xmm6, %xmm6
-; SSE-NEXT:    pcmpgtd %xmm2, %xmm6
-; SSE-NEXT:    pxor %xmm11, %xmm6
-; SSE-NEXT:    pcmpeqd %xmm5, %xmm6
-; SSE-NEXT:    pxor %xmm11, %xmm6
-; SSE-NEXT:    pandn %xmm6, %xmm4
-; SSE-NEXT:    pxor %xmm5, %xmm5
-; SSE-NEXT:    pcmpgtd %xmm7, %xmm5
-; SSE-NEXT:    pxor %xmm11, %xmm5
-; SSE-NEXT:    pxor %xmm6, %xmm6
-; SSE-NEXT:    pcmpgtd %xmm3, %xmm6
-; SSE-NEXT:    pxor %xmm11, %xmm6
-; SSE-NEXT:    pcmpeqd %xmm6, %xmm5
-; SSE-NEXT:    psubd %xmm7, %xmm3
-; SSE-NEXT:    pcmpgtd %xmm3, %xmm10
-; SSE-NEXT:    pxor %xmm11, %xmm10
-; SSE-NEXT:    pcmpeqd %xmm6, %xmm10
-; SSE-NEXT:    pxor %xmm11, %xmm10
-; SSE-NEXT:    pandn %xmm10, %xmm5
-; SSE-NEXT:    movdqa %xmm3, 48(%rdi)
-; SSE-NEXT:    movdqa %xmm2, 32(%rdi)
-; SSE-NEXT:    movdqa %xmm1, 16(%rdi)
-; SSE-NEXT:    movdqa %xmm0, (%rdi)
-; SSE-NEXT:    movdqa %xmm8, %xmm0
-; SSE-NEXT:    movdqa %xmm9, %xmm1
-; SSE-NEXT:    movdqa %xmm4, %xmm2
-; SSE-NEXT:    movdqa %xmm5, %xmm3
+; SSE-NEXT:    movdqa %xmm0, %xmm8
+; SSE-NEXT:    psubd %xmm4, %xmm8
+; SSE-NEXT:    pcmpgtd %xmm9, %xmm4
+; SSE-NEXT:    pcmpgtd %xmm8, %xmm0
+; SSE-NEXT:    pxor %xmm4, %xmm0
+; SSE-NEXT:    movdqa %xmm1, %xmm4
+; SSE-NEXT:    psubd %xmm5, %xmm4
+; SSE-NEXT:    pcmpgtd %xmm9, %xmm5
+; SSE-NEXT:    pcmpgtd %xmm4, %xmm1
+; SSE-NEXT:    pxor %xmm5, %xmm1
+; SSE-NEXT:    movdqa %xmm2, %xmm5
+; SSE-NEXT:    psubd %xmm6, %xmm5
+; SSE-NEXT:    pcmpgtd %xmm9, %xmm6
+; SSE-NEXT:    pcmpgtd %xmm5, %xmm2
+; SSE-NEXT:    pxor %xmm6, %xmm2
+; SSE-NEXT:    movdqa %xmm3, %xmm6
+; SSE-NEXT:    psubd %xmm7, %xmm6
+; SSE-NEXT:    pcmpgtd %xmm9, %xmm7
+; SSE-NEXT:    pcmpgtd %xmm6, %xmm3
+; SSE-NEXT:    pxor %xmm7, %xmm3
+; SSE-NEXT:    movdqa %xmm6, 48(%rdi)
+; SSE-NEXT:    movdqa %xmm5, 32(%rdi)
+; SSE-NEXT:    movdqa %xmm4, 16(%rdi)
+; SSE-NEXT:    movdqa %xmm8, (%rdi)
 ; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: ssubo_v16i32:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm4
-; AVX1-NEXT:    vpxor %xmm9, %xmm9, %xmm9
-; AVX1-NEXT:    vpcmpgtd %xmm4, %xmm9, %xmm7
-; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
-; AVX1-NEXT:    vpxor %xmm5, %xmm7, %xmm8
+; AVX1-NEXT:    vpxor %xmm5, %xmm5, %xmm5
+; AVX1-NEXT:    vpcmpgtd %xmm5, %xmm4, %xmm6
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm7
-; AVX1-NEXT:    vpcmpgtd %xmm7, %xmm9, %xmm6
-; AVX1-NEXT:    vpxor %xmm5, %xmm6, %xmm6
-; AVX1-NEXT:    vpcmpeqd %xmm8, %xmm6, %xmm8
-; AVX1-NEXT:    vpsubd %xmm4, %xmm7, %xmm10
-; AVX1-NEXT:    vpcmpgtd %xmm10, %xmm9, %xmm7
-; AVX1-NEXT:    vpxor %xmm5, %xmm7, %xmm7
-; AVX1-NEXT:    vpcmpeqd %xmm7, %xmm6, %xmm6
-; AVX1-NEXT:    vpxor %xmm5, %xmm6, %xmm6
-; AVX1-NEXT:    vpandn %xmm6, %xmm8, %xmm6
-; AVX1-NEXT:    vpcmpgtd %xmm3, %xmm9, %xmm7
-; AVX1-NEXT:    vpxor %xmm5, %xmm7, %xmm7
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm9, %xmm4
-; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm7, %xmm4, %xmm7
+; AVX1-NEXT:    vpsubd %xmm4, %xmm7, %xmm8
+; AVX1-NEXT:    vpcmpgtd %xmm8, %xmm7, %xmm7
+; AVX1-NEXT:    vpxor %xmm7, %xmm6, %xmm6
+; AVX1-NEXT:    vpcmpgtd %xmm5, %xmm3, %xmm7
 ; AVX1-NEXT:    vpsubd %xmm3, %xmm1, %xmm3
-; AVX1-NEXT:    vpcmpgtd %xmm3, %xmm9, %xmm1
-; AVX1-NEXT:    vpxor %xmm5, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm4, %xmm1
-; AVX1-NEXT:    vpxor %xmm5, %xmm1, %xmm1
-; AVX1-NEXT:    vpandn %xmm1, %xmm7, %xmm1
-; AVX1-NEXT:    vpackssdw %xmm6, %xmm1, %xmm8
-; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm4
-; AVX1-NEXT:    vpcmpgtd %xmm4, %xmm9, %xmm6
-; AVX1-NEXT:    vpxor %xmm5, %xmm6, %xmm6
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm7
-; AVX1-NEXT:    vpcmpgtd %xmm7, %xmm9, %xmm1
-; AVX1-NEXT:    vpxor %xmm5, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm6, %xmm1, %xmm6
-; AVX1-NEXT:    vpsubd %xmm4, %xmm7, %xmm7
-; AVX1-NEXT:    vpcmpgtd %xmm7, %xmm9, %xmm4
-; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm1, %xmm1
-; AVX1-NEXT:    vpxor %xmm5, %xmm1, %xmm1
-; AVX1-NEXT:    vpandn %xmm1, %xmm6, %xmm1
-; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm9, %xmm4
-; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm9, %xmm6
-; AVX1-NEXT:    vpxor %xmm5, %xmm6, %xmm6
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm6, %xmm4
+; AVX1-NEXT:    vpcmpgtd %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vpxor %xmm1, %xmm7, %xmm1
+; AVX1-NEXT:    vpackssdw %xmm6, %xmm1, %xmm1
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm6
+; AVX1-NEXT:    vpcmpgtd %xmm5, %xmm6, %xmm7
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
+; AVX1-NEXT:    vpsubd %xmm6, %xmm4, %xmm6
+; AVX1-NEXT:    vpcmpgtd %xmm6, %xmm4, %xmm4
+; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm4
+; AVX1-NEXT:    vpcmpgtd %xmm5, %xmm2, %xmm5
 ; AVX1-NEXT:    vpsubd %xmm2, %xmm0, %xmm2
-; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm9, %xmm0
-; AVX1-NEXT:    vpxor %xmm5, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm6, %xmm0
-; AVX1-NEXT:    vpxor %xmm5, %xmm0, %xmm0
-; AVX1-NEXT:    vpandn %xmm0, %xmm4, %xmm0
-; AVX1-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpacksswb %xmm8, %xmm0, %xmm1
+; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor %xmm0, %xmm5, %xmm0
+; AVX1-NEXT:    vpackssdw %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vpacksswb %xmm1, %xmm0, %xmm1
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm1[1,1,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm4, %xmm4
@@ -888,40 +528,25 @@ define <16 x i32> @ssubo_v16i32(<16 x i32> %a0, <16 x i32> %a1, <16 x i32>* %p2)
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,3,0,1]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm4, %ymm1
-; AVX1-NEXT:    vmovdqa %xmm10, 48(%rdi)
+; AVX1-NEXT:    vmovdqa %xmm8, 48(%rdi)
 ; AVX1-NEXT:    vmovdqa %xmm3, 32(%rdi)
-; AVX1-NEXT:    vmovdqa %xmm7, 16(%rdi)
+; AVX1-NEXT:    vmovdqa %xmm6, 16(%rdi)
 ; AVX1-NEXT:    vmovdqa %xmm2, (%rdi)
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: ssubo_v16i32:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpcmpgtd %ymm3, %ymm4, %ymm5
-; AVX2-NEXT:    vpcmpeqd %ymm6, %ymm6, %ymm6
-; AVX2-NEXT:    vpxor %ymm6, %ymm5, %ymm5
-; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm4, %ymm7
-; AVX2-NEXT:    vpxor %ymm6, %ymm7, %ymm7
-; AVX2-NEXT:    vpcmpeqd %ymm5, %ymm7, %ymm5
+; AVX2-NEXT:    vpcmpgtd %ymm4, %ymm3, %ymm5
 ; AVX2-NEXT:    vpsubd %ymm3, %ymm1, %ymm3
-; AVX2-NEXT:    vpcmpgtd %ymm3, %ymm4, %ymm1
-; AVX2-NEXT:    vpxor %ymm6, %ymm1, %ymm1
-; AVX2-NEXT:    vpcmpeqd %ymm1, %ymm7, %ymm1
-; AVX2-NEXT:    vpxor %ymm6, %ymm1, %ymm1
-; AVX2-NEXT:    vpandn %ymm1, %ymm5, %ymm1
+; AVX2-NEXT:    vpcmpgtd %ymm3, %ymm1, %ymm1
+; AVX2-NEXT:    vpxor %ymm1, %ymm5, %ymm1
 ; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm5
 ; AVX2-NEXT:    vpackssdw %xmm5, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpgtd %ymm2, %ymm4, %ymm5
-; AVX2-NEXT:    vpxor %ymm6, %ymm5, %ymm5
-; AVX2-NEXT:    vpcmpgtd %ymm0, %ymm4, %ymm7
-; AVX2-NEXT:    vpxor %ymm6, %ymm7, %ymm7
-; AVX2-NEXT:    vpcmpeqd %ymm5, %ymm7, %ymm5
+; AVX2-NEXT:    vpcmpgtd %ymm4, %ymm2, %ymm4
 ; AVX2-NEXT:    vpsubd %ymm2, %ymm0, %ymm2
-; AVX2-NEXT:    vpcmpgtd %ymm2, %ymm4, %ymm0
-; AVX2-NEXT:    vpxor %ymm6, %ymm0, %ymm0
-; AVX2-NEXT:    vpcmpeqd %ymm0, %ymm7, %ymm0
-; AVX2-NEXT:    vpxor %ymm6, %ymm0, %ymm0
-; AVX2-NEXT:    vpandn %ymm0, %ymm5, %ymm0
+; AVX2-NEXT:    vpcmpgtd %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %ymm0, %ymm4, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm4
 ; AVX2-NEXT:    vpackssdw %xmm4, %xmm0, %xmm0
 ; AVX2-NEXT:    vpacksswb %xmm1, %xmm0, %xmm1
@@ -935,13 +560,10 @@ define <16 x i32> @ssubo_v16i32(<16 x i32> %a0, <16 x i32> %a1, <16 x i32>* %p2)
 ; AVX512-LABEL: ssubo_v16i32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpcmpnltd %zmm2, %zmm1, %k0
-; AVX512-NEXT:    vpcmpnltd %zmm2, %zmm0, %k1
-; AVX512-NEXT:    kxorw %k0, %k1, %k0
+; AVX512-NEXT:    vpcmpgtd %zmm2, %zmm1, %k0
 ; AVX512-NEXT:    vpsubd %zmm1, %zmm0, %zmm1
-; AVX512-NEXT:    vpcmpnltd %zmm2, %zmm1, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
-; AVX512-NEXT:    kandw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpgtd %zmm1, %zmm0, %k1
+; AVX512-NEXT:    kxorw %k1, %k0, %k1
 ; AVX512-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
 ; AVX512-NEXT:    vmovdqa64 %zmm1, (%rdi)
 ; AVX512-NEXT:    retq
@@ -1187,195 +809,67 @@ define <8 x i32> @ssubo_v8i16(<8 x i16> %a0, <8 x i16> %a1, <8 x i16>* %p2) noun
 }
 
 define <2 x i32> @ssubo_v2i64(<2 x i64> %a0, <2 x i64> %a1, <2 x i64>* %p2) nounwind {
-; SSE2-LABEL: ssubo_v2i64:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
-; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    psubq %xmm1, %xmm0
-; SSE2-NEXT:    pxor %xmm2, %xmm1
-; SSE2-NEXT:    movdqa %xmm2, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pand %xmm5, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm4
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm5
-; SSE2-NEXT:    pxor %xmm5, %xmm4
-; SSE2-NEXT:    pxor %xmm2, %xmm3
-; SSE2-NEXT:    movdqa %xmm2, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm6
-; SSE2-NEXT:    pxor %xmm5, %xmm6
-; SSE2-NEXT:    pcmpeqd %xmm6, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,0,3,2]
-; SSE2-NEXT:    pand %xmm4, %xmm1
-; SSE2-NEXT:    movdqa %xmm0, (%rdi)
-; SSE2-NEXT:    pxor %xmm2, %xmm0
-; SSE2-NEXT:    movdqa %xmm2, %xmm3
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pand %xmm4, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,1,3,3]
-; SSE2-NEXT:    por %xmm0, %xmm2
-; SSE2-NEXT:    pxor %xmm5, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm6, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,0,3,2]
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    pxor %xmm5, %xmm0
-; SSE2-NEXT:    pandn %xmm0, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm0
-; SSE2-NEXT:    retq
-;
-; SSSE3-LABEL: ssubo_v2i64:
-; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
-; SSSE3-NEXT:    movdqa %xmm0, %xmm3
-; SSSE3-NEXT:    psubq %xmm1, %xmm0
-; SSSE3-NEXT:    pxor %xmm2, %xmm1
-; SSSE3-NEXT:    movdqa %xmm2, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm5, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    por %xmm1, %xmm4
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm5
-; SSSE3-NEXT:    pxor %xmm5, %xmm4
-; SSSE3-NEXT:    pxor %xmm2, %xmm3
-; SSSE3-NEXT:    movdqa %xmm2, %xmm1
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm6, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    por %xmm3, %xmm6
-; SSSE3-NEXT:    pxor %xmm5, %xmm6
-; SSSE3-NEXT:    pcmpeqd %xmm6, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm4, %xmm1
-; SSSE3-NEXT:    movdqa %xmm0, (%rdi)
-; SSSE3-NEXT:    pxor %xmm2, %xmm0
-; SSSE3-NEXT:    movdqa %xmm2, %xmm3
-; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm4, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm0, %xmm2
-; SSSE3-NEXT:    pxor %xmm5, %xmm2
-; SSSE3-NEXT:    pcmpeqd %xmm6, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm2, %xmm0
-; SSSE3-NEXT:    pxor %xmm5, %xmm0
-; SSSE3-NEXT:    pandn %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa %xmm1, %xmm0
-; SSSE3-NEXT:    retq
-;
-; SSE41-LABEL: ssubo_v2i64:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
-; SSE41-NEXT:    movdqa %xmm0, %xmm3
-; SSE41-NEXT:    psubq %xmm1, %xmm0
-; SSE41-NEXT:    pxor %xmm2, %xmm1
-; SSE41-NEXT:    movdqa %xmm2, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm1, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm2, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,1,3,3]
-; SSE41-NEXT:    por %xmm6, %xmm1
-; SSE41-NEXT:    pcmpeqd %xmm4, %xmm4
-; SSE41-NEXT:    pxor %xmm4, %xmm1
-; SSE41-NEXT:    pxor %xmm2, %xmm3
-; SSE41-NEXT:    movdqa %xmm2, %xmm5
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm2, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pand %xmm6, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE41-NEXT:    por %xmm3, %xmm5
-; SSE41-NEXT:    pxor %xmm4, %xmm5
-; SSE41-NEXT:    pcmpeqq %xmm5, %xmm1
-; SSE41-NEXT:    movdqa %xmm0, (%rdi)
-; SSE41-NEXT:    pxor %xmm2, %xmm0
-; SSE41-NEXT:    movdqa %xmm2, %xmm3
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm2, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm6, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm0, %xmm2
-; SSE41-NEXT:    pxor %xmm4, %xmm2
-; SSE41-NEXT:    pcmpeqq %xmm5, %xmm2
-; SSE41-NEXT:    pxor %xmm4, %xmm2
-; SSE41-NEXT:    pandn %xmm2, %xmm1
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    retq
+; SSE-LABEL: ssubo_v2i64:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
+; SSE-NEXT:    movdqa %xmm0, %xmm3
+; SSE-NEXT:    pxor %xmm2, %xmm3
+; SSE-NEXT:    psubq %xmm1, %xmm0
+; SSE-NEXT:    movdqa %xmm0, (%rdi)
+; SSE-NEXT:    pxor %xmm2, %xmm0
+; SSE-NEXT:    movdqa %xmm3, %xmm4
+; SSE-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE-NEXT:    pcmpeqd %xmm3, %xmm0
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE-NEXT:    pand %xmm5, %xmm0
+; SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
+; SSE-NEXT:    por %xmm0, %xmm3
+; SSE-NEXT:    pxor %xmm2, %xmm1
+; SSE-NEXT:    movdqa %xmm1, %xmm0
+; SSE-NEXT:    pcmpgtd %xmm2, %xmm0
+; SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[0,0,2,2]
+; SSE-NEXT:    pcmpeqd %xmm2, %xmm1
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE-NEXT:    pand %xmm4, %xmm1
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE-NEXT:    por %xmm1, %xmm0
+; SSE-NEXT:    pxor %xmm3, %xmm0
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: ssubo_v2i64:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm2, %xmm3
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm5
-; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX1-NEXT:    vpcmpeqq %xmm3, %xmm5, %xmm3
+; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm1, %xmm2
 ; AVX1-NEXT:    vpsubq %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm2, %xmm0
-; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpeqq %xmm0, %xmm5, %xmm0
-; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm0
-; AVX1-NEXT:    vpandn %xmm0, %xmm3, %xmm0
+; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor %xmm0, %xmm2, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; AVX1-NEXT:    vmovdqa %xmm1, (%rdi)
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: ssubo_v2i64:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtq %xmm1, %xmm2, %xmm3
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpxor %xmm4, %xmm3, %xmm3
-; AVX2-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm5
-; AVX2-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX2-NEXT:    vpcmpeqq %xmm3, %xmm5, %xmm3
+; AVX2-NEXT:    vpcmpgtq %xmm2, %xmm1, %xmm2
 ; AVX2-NEXT:    vpsubq %xmm1, %xmm0, %xmm1
-; AVX2-NEXT:    vpcmpgtq %xmm1, %xmm2, %xmm0
-; AVX2-NEXT:    vpxor %xmm4, %xmm0, %xmm0
-; AVX2-NEXT:    vpcmpeqq %xmm0, %xmm5, %xmm0
-; AVX2-NEXT:    vpxor %xmm4, %xmm0, %xmm0
-; AVX2-NEXT:    vpandn %xmm0, %xmm3, %xmm0
+; AVX2-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor %xmm0, %xmm2, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; AVX2-NEXT:    vmovdqa %xmm1, (%rdi)
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: ssubo_v2i64:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpcmpnltq %xmm2, %xmm1, %k0
-; AVX512-NEXT:    vpcmpnltq %xmm2, %xmm0, %k1
-; AVX512-NEXT:    kxorw %k0, %k1, %k0
-; AVX512-NEXT:    vpsubq %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpcmpnltq %xmm2, %xmm0, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
-; AVX512-NEXT:    kandw %k1, %k0, %k1
-; AVX512-NEXT:    vmovdqa %xmm0, (%rdi)
+; AVX512-NEXT:    vpcmpgtq %xmm2, %xmm1, %k0
+; AVX512-NEXT:    vpsubq %xmm1, %xmm0, %xmm1
+; AVX512-NEXT:    vpcmpgtq %xmm1, %xmm0, %k1
+; AVX512-NEXT:    kxorw %k1, %k0, %k1
 ; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX512-NEXT:    vmovdqa64 %xmm0, %xmm0 {%k1} {z}
+; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
+; AVX512-NEXT:    vmovdqa %xmm1, (%rdi)
 ; AVX512-NEXT:    retq
   %t = call {<2 x i64>, <2 x i1>} @llvm.ssub.with.overflow.v2i64(<2 x i64> %a0, <2 x i64> %a1)
   %val = extractvalue {<2 x i64>, <2 x i1>} %t, 0
@@ -1661,44 +1155,44 @@ define <2 x i32> @ssubo_v2i128(<2 x i128> %a0, <2 x i128> %a1, <2 x i128>* %p2)
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pushq %rbp
 ; SSE2-NEXT:    pushq %rbx
-; SSE2-NEXT:    movq {{[0-9]+}}(%rsp), %r10
 ; SSE2-NEXT:    movq {{[0-9]+}}(%rsp), %r11
-; SSE2-NEXT:    subq {{[0-9]+}}(%rsp), %rdx
-; SSE2-NEXT:    movq %rcx, %rax
-; SSE2-NEXT:    sbbq %r11, %rax
+; SSE2-NEXT:    movq {{[0-9]+}}(%rsp), %r10
+; SSE2-NEXT:    testq %r9, %r9
+; SSE2-NEXT:    setns %al
+; SSE2-NEXT:    testq %rsi, %rsi
 ; SSE2-NEXT:    setns %bl
-; SSE2-NEXT:    testq %rcx, %rcx
-; SSE2-NEXT:    setns %cl
-; SSE2-NEXT:    cmpb %bl, %cl
+; SSE2-NEXT:    cmpb %al, %bl
 ; SSE2-NEXT:    setne %bpl
-; SSE2-NEXT:    testq %r11, %r11
-; SSE2-NEXT:    setns %bl
-; SSE2-NEXT:    cmpb %bl, %cl
-; SSE2-NEXT:    setne %cl
-; SSE2-NEXT:    andb %bpl, %cl
-; SSE2-NEXT:    movzbl %cl, %ebp
-; SSE2-NEXT:    testq %r9, %r9
+; SSE2-NEXT:    subq %r8, %rdi
+; SSE2-NEXT:    sbbq %r9, %rsi
+; SSE2-NEXT:    setns %al
+; SSE2-NEXT:    cmpb %al, %bl
+; SSE2-NEXT:    setne %al
+; SSE2-NEXT:    andb %bpl, %al
+; SSE2-NEXT:    subq {{[0-9]+}}(%rsp), %rdx
+; SSE2-NEXT:    movq %rcx, %rbp
+; SSE2-NEXT:    sbbq %r10, %rbp
 ; SSE2-NEXT:    setns %bl
-; SSE2-NEXT:    testq %rsi, %rsi
+; SSE2-NEXT:    testq %rcx, %rcx
 ; SSE2-NEXT:    setns %cl
 ; SSE2-NEXT:    cmpb %bl, %cl
-; SSE2-NEXT:    setne %r11b
-; SSE2-NEXT:    subq %r8, %rdi
-; SSE2-NEXT:    sbbq %r9, %rsi
+; SSE2-NEXT:    setne %r8b
+; SSE2-NEXT:    testq %r10, %r10
 ; SSE2-NEXT:    setns %bl
 ; SSE2-NEXT:    cmpb %bl, %cl
 ; SSE2-NEXT:    setne %cl
-; SSE2-NEXT:    andb %r11b, %cl
+; SSE2-NEXT:    andb %r8b, %cl
 ; SSE2-NEXT:    movzbl %cl, %ecx
-; SSE2-NEXT:    movd %ecx, %xmm0
-; SSE2-NEXT:    pinsrw $4, %ebp, %xmm0
-; SSE2-NEXT:    movq %rdx, 16(%r10)
-; SSE2-NEXT:    movq %rdi, (%r10)
-; SSE2-NEXT:    movq %rax, 24(%r10)
-; SSE2-NEXT:    movq %rsi, 8(%r10)
-; SSE2-NEXT:    psllq $63, %xmm0
-; SSE2-NEXT:    psrad $31, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    negl %ecx
+; SSE2-NEXT:    movd %ecx, %xmm1
+; SSE2-NEXT:    movzbl %al, %eax
+; SSE2-NEXT:    negl %eax
+; SSE2-NEXT:    movd %eax, %xmm0
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT:    movq %rdx, 16(%r11)
+; SSE2-NEXT:    movq %rdi, (%r11)
+; SSE2-NEXT:    movq %rbp, 24(%r11)
+; SSE2-NEXT:    movq %rsi, 8(%r11)
 ; SSE2-NEXT:    popq %rbx
 ; SSE2-NEXT:    popq %rbp
 ; SSE2-NEXT:    retq
@@ -1707,44 +1201,44 @@ define <2 x i32> @ssubo_v2i128(<2 x i128> %a0, <2 x i128> %a1, <2 x i128>* %p2)
 ; SSSE3:       # %bb.0:
 ; SSSE3-NEXT:    pushq %rbp
 ; SSSE3-NEXT:    pushq %rbx
-; SSSE3-NEXT:    movq {{[0-9]+}}(%rsp), %r10
 ; SSSE3-NEXT:    movq {{[0-9]+}}(%rsp), %r11
-; SSSE3-NEXT:    subq {{[0-9]+}}(%rsp), %rdx
-; SSSE3-NEXT:    movq %rcx, %rax
-; SSSE3-NEXT:    sbbq %r11, %rax
+; SSSE3-NEXT:    movq {{[0-9]+}}(%rsp), %r10
+; SSSE3-NEXT:    testq %r9, %r9
+; SSSE3-NEXT:    setns %al
+; SSSE3-NEXT:    testq %rsi, %rsi
 ; SSSE3-NEXT:    setns %bl
-; SSSE3-NEXT:    testq %rcx, %rcx
-; SSSE3-NEXT:    setns %cl
-; SSSE3-NEXT:    cmpb %bl, %cl
+; SSSE3-NEXT:    cmpb %al, %bl
 ; SSSE3-NEXT:    setne %bpl
-; SSSE3-NEXT:    testq %r11, %r11
-; SSSE3-NEXT:    setns %bl
-; SSSE3-NEXT:    cmpb %bl, %cl
-; SSSE3-NEXT:    setne %cl
-; SSSE3-NEXT:    andb %bpl, %cl
-; SSSE3-NEXT:    movzbl %cl, %ebp
-; SSSE3-NEXT:    testq %r9, %r9
+; SSSE3-NEXT:    subq %r8, %rdi
+; SSSE3-NEXT:    sbbq %r9, %rsi
+; SSSE3-NEXT:    setns %al
+; SSSE3-NEXT:    cmpb %al, %bl
+; SSSE3-NEXT:    setne %al
+; SSSE3-NEXT:    andb %bpl, %al
+; SSSE3-NEXT:    subq {{[0-9]+}}(%rsp), %rdx
+; SSSE3-NEXT:    movq %rcx, %rbp
+; SSSE3-NEXT:    sbbq %r10, %rbp
 ; SSSE3-NEXT:    setns %bl
-; SSSE3-NEXT:    testq %rsi, %rsi
+; SSSE3-NEXT:    testq %rcx, %rcx
 ; SSSE3-NEXT:    setns %cl
 ; SSSE3-NEXT:    cmpb %bl, %cl
-; SSSE3-NEXT:    setne %r11b
-; SSSE3-NEXT:    subq %r8, %rdi
-; SSSE3-NEXT:    sbbq %r9, %rsi
+; SSSE3-NEXT:    setne %r8b
+; SSSE3-NEXT:    testq %r10, %r10
 ; SSSE3-NEXT:    setns %bl
 ; SSSE3-NEXT:    cmpb %bl, %cl
 ; SSSE3-NEXT:    setne %cl
-; SSSE3-NEXT:    andb %r11b, %cl
+; SSSE3-NEXT:    andb %r8b, %cl
 ; SSSE3-NEXT:    movzbl %cl, %ecx
-; SSSE3-NEXT:    movd %ecx, %xmm0
-; SSSE3-NEXT:    pinsrw $4, %ebp, %xmm0
-; SSSE3-NEXT:    movq %rdx, 16(%r10)
-; SSSE3-NEXT:    movq %rdi, (%r10)
-; SSSE3-NEXT:    movq %rax, 24(%r10)
-; SSSE3-NEXT:    movq %rsi, 8(%r10)
-; SSSE3-NEXT:    psllq $63, %xmm0
-; SSSE3-NEXT:    psrad $31, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    negl %ecx
+; SSSE3-NEXT:    movd %ecx, %xmm1
+; SSSE3-NEXT:    movzbl %al, %eax
+; SSSE3-NEXT:    negl %eax
+; SSSE3-NEXT:    movd %eax, %xmm0
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSSE3-NEXT:    movq %rdx, 16(%r11)
+; SSSE3-NEXT:    movq %rdi, (%r11)
+; SSSE3-NEXT:    movq %rbp, 24(%r11)
+; SSSE3-NEXT:    movq %rsi, 8(%r11)
 ; SSSE3-NEXT:    popq %rbx
 ; SSSE3-NEXT:    popq %rbp
 ; SSSE3-NEXT:    retq
@@ -1753,44 +1247,43 @@ define <2 x i32> @ssubo_v2i128(<2 x i128> %a0, <2 x i128> %a1, <2 x i128>* %p2)
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pushq %rbp
 ; SSE41-NEXT:    pushq %rbx
-; SSE41-NEXT:    movq {{[0-9]+}}(%rsp), %r10
 ; SSE41-NEXT:    movq {{[0-9]+}}(%rsp), %r11
-; SSE41-NEXT:    subq {{[0-9]+}}(%rsp), %rdx
-; SSE41-NEXT:    movq %rcx, %rax
-; SSE41-NEXT:    sbbq %r11, %rax
+; SSE41-NEXT:    movq {{[0-9]+}}(%rsp), %r10
+; SSE41-NEXT:    testq %r9, %r9
+; SSE41-NEXT:    setns %al
+; SSE41-NEXT:    testq %rsi, %rsi
 ; SSE41-NEXT:    setns %bl
-; SSE41-NEXT:    testq %rcx, %rcx
-; SSE41-NEXT:    setns %cl
-; SSE41-NEXT:    cmpb %bl, %cl
+; SSE41-NEXT:    cmpb %al, %bl
 ; SSE41-NEXT:    setne %bpl
-; SSE41-NEXT:    testq %r11, %r11
-; SSE41-NEXT:    setns %bl
-; SSE41-NEXT:    cmpb %bl, %cl
-; SSE41-NEXT:    setne %cl
-; SSE41-NEXT:    andb %bpl, %cl
-; SSE41-NEXT:    movzbl %cl, %ebp
-; SSE41-NEXT:    testq %r9, %r9
+; SSE41-NEXT:    subq %r8, %rdi
+; SSE41-NEXT:    sbbq %r9, %rsi
+; SSE41-NEXT:    setns %al
+; SSE41-NEXT:    cmpb %al, %bl
+; SSE41-NEXT:    setne %al
+; SSE41-NEXT:    andb %bpl, %al
+; SSE41-NEXT:    subq {{[0-9]+}}(%rsp), %rdx
+; SSE41-NEXT:    movq %rcx, %rbp
+; SSE41-NEXT:    sbbq %r10, %rbp
 ; SSE41-NEXT:    setns %bl
-; SSE41-NEXT:    testq %rsi, %rsi
+; SSE41-NEXT:    testq %rcx, %rcx
 ; SSE41-NEXT:    setns %cl
 ; SSE41-NEXT:    cmpb %bl, %cl
-; SSE41-NEXT:    setne %r11b
-; SSE41-NEXT:    subq %r8, %rdi
-; SSE41-NEXT:    sbbq %r9, %rsi
+; SSE41-NEXT:    setne %r8b
+; SSE41-NEXT:    testq %r10, %r10
 ; SSE41-NEXT:    setns %bl
 ; SSE41-NEXT:    cmpb %bl, %cl
 ; SSE41-NEXT:    setne %cl
-; SSE41-NEXT:    andb %r11b, %cl
+; SSE41-NEXT:    andb %r8b, %cl
 ; SSE41-NEXT:    movzbl %cl, %ecx
-; SSE41-NEXT:    movd %ecx, %xmm0
-; SSE41-NEXT:    pinsrb $8, %ebp, %xmm0
-; SSE41-NEXT:    movq %rdx, 16(%r10)
-; SSE41-NEXT:    movq %rdi, (%r10)
-; SSE41-NEXT:    movq %rax, 24(%r10)
-; SSE41-NEXT:    movq %rsi, 8(%r10)
-; SSE41-NEXT:    psllq $63, %xmm0
-; SSE41-NEXT:    psrad $31, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE41-NEXT:    negl %ecx
+; SSE41-NEXT:    movzbl %al, %eax
+; SSE41-NEXT:    negl %eax
+; SSE41-NEXT:    movd %eax, %xmm0
+; SSE41-NEXT:    pinsrd $1, %ecx, %xmm0
+; SSE41-NEXT:    movq %rdx, 16(%r11)
+; SSE41-NEXT:    movq %rdi, (%r11)
+; SSE41-NEXT:    movq %rbp, 24(%r11)
+; SSE41-NEXT:    movq %rsi, 8(%r11)
 ; SSE41-NEXT:    popq %rbx
 ; SSE41-NEXT:    popq %rbp
 ; SSE41-NEXT:    retq
@@ -1799,44 +1292,43 @@ define <2 x i32> @ssubo_v2i128(<2 x i128> %a0, <2 x i128> %a1, <2 x i128>* %p2)
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    pushq %rbp
 ; AVX1-NEXT:    pushq %rbx
-; AVX1-NEXT:    movq {{[0-9]+}}(%rsp), %r10
 ; AVX1-NEXT:    movq {{[0-9]+}}(%rsp), %r11
-; AVX1-NEXT:    subq {{[0-9]+}}(%rsp), %rdx
-; AVX1-NEXT:    movq %rcx, %rax
-; AVX1-NEXT:    sbbq %r11, %rax
+; AVX1-NEXT:    movq {{[0-9]+}}(%rsp), %r10
+; AVX1-NEXT:    testq %r9, %r9
+; AVX1-NEXT:    setns %al
+; AVX1-NEXT:    testq %rsi, %rsi
 ; AVX1-NEXT:    setns %bl
-; AVX1-NEXT:    testq %rcx, %rcx
-; AVX1-NEXT:    setns %cl
-; AVX1-NEXT:    cmpb %bl, %cl
+; AVX1-NEXT:    cmpb %al, %bl
 ; AVX1-NEXT:    setne %bpl
-; AVX1-NEXT:    testq %r11, %r11
-; AVX1-NEXT:    setns %bl
-; AVX1-NEXT:    cmpb %bl, %cl
-; AVX1-NEXT:    setne %cl
-; AVX1-NEXT:    andb %bpl, %cl
-; AVX1-NEXT:    movzbl %cl, %ebp
-; AVX1-NEXT:    testq %r9, %r9
+; AVX1-NEXT:    subq %r8, %rdi
+; AVX1-NEXT:    sbbq %r9, %rsi
+; AVX1-NEXT:    setns %al
+; AVX1-NEXT:    cmpb %al, %bl
+; AVX1-NEXT:    setne %al
+; AVX1-NEXT:    andb %bpl, %al
+; AVX1-NEXT:    subq {{[0-9]+}}(%rsp), %rdx
+; AVX1-NEXT:    movq %rcx, %rbp
+; AVX1-NEXT:    sbbq %r10, %rbp
 ; AVX1-NEXT:    setns %bl
-; AVX1-NEXT:    testq %rsi, %rsi
+; AVX1-NEXT:    testq %rcx, %rcx
 ; AVX1-NEXT:    setns %cl
 ; AVX1-NEXT:    cmpb %bl, %cl
-; AVX1-NEXT:    setne %r11b
-; AVX1-NEXT:    subq %r8, %rdi
-; AVX1-NEXT:    sbbq %r9, %rsi
+; AVX1-NEXT:    setne %r8b
+; AVX1-NEXT:    testq %r10, %r10
 ; AVX1-NEXT:    setns %bl
 ; AVX1-NEXT:    cmpb %bl, %cl
 ; AVX1-NEXT:    setne %cl
-; AVX1-NEXT:    andb %r11b, %cl
+; AVX1-NEXT:    andb %r8b, %cl
 ; AVX1-NEXT:    movzbl %cl, %ecx
-; AVX1-NEXT:    vmovd %ecx, %xmm0
-; AVX1-NEXT:    vpinsrb $8, %ebp, %xmm0, %xmm0
-; AVX1-NEXT:    movq %rdx, 16(%r10)
-; AVX1-NEXT:    movq %rdi, (%r10)
-; AVX1-NEXT:    movq %rax, 24(%r10)
-; AVX1-NEXT:    movq %rsi, 8(%r10)
-; AVX1-NEXT:    vpsllq $63, %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    negl %ecx
+; AVX1-NEXT:    movzbl %al, %eax
+; AVX1-NEXT:    negl %eax
+; AVX1-NEXT:    vmovd %eax, %xmm0
+; AVX1-NEXT:    vpinsrd $1, %ecx, %xmm0, %xmm0
+; AVX1-NEXT:    movq %rdx, 16(%r11)
+; AVX1-NEXT:    movq %rdi, (%r11)
+; AVX1-NEXT:    movq %rbp, 24(%r11)
+; AVX1-NEXT:    movq %rsi, 8(%r11)
 ; AVX1-NEXT:    popq %rbx
 ; AVX1-NEXT:    popq %rbp
 ; AVX1-NEXT:    retq
@@ -1845,92 +1337,91 @@ define <2 x i32> @ssubo_v2i128(<2 x i128> %a0, <2 x i128> %a1, <2 x i128>* %p2)
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    pushq %rbp
 ; AVX2-NEXT:    pushq %rbx
-; AVX2-NEXT:    movq {{[0-9]+}}(%rsp), %r10
 ; AVX2-NEXT:    movq {{[0-9]+}}(%rsp), %r11
-; AVX2-NEXT:    subq {{[0-9]+}}(%rsp), %rdx
-; AVX2-NEXT:    movq %rcx, %rax
-; AVX2-NEXT:    sbbq %r11, %rax
+; AVX2-NEXT:    movq {{[0-9]+}}(%rsp), %r10
+; AVX2-NEXT:    testq %r9, %r9
+; AVX2-NEXT:    setns %al
+; AVX2-NEXT:    testq %rsi, %rsi
 ; AVX2-NEXT:    setns %bl
-; AVX2-NEXT:    testq %rcx, %rcx
-; AVX2-NEXT:    setns %cl
-; AVX2-NEXT:    cmpb %bl, %cl
+; AVX2-NEXT:    cmpb %al, %bl
 ; AVX2-NEXT:    setne %bpl
-; AVX2-NEXT:    testq %r11, %r11
-; AVX2-NEXT:    setns %bl
-; AVX2-NEXT:    cmpb %bl, %cl
-; AVX2-NEXT:    setne %cl
-; AVX2-NEXT:    andb %bpl, %cl
-; AVX2-NEXT:    movzbl %cl, %ebp
-; AVX2-NEXT:    testq %r9, %r9
+; AVX2-NEXT:    subq %r8, %rdi
+; AVX2-NEXT:    sbbq %r9, %rsi
+; AVX2-NEXT:    setns %al
+; AVX2-NEXT:    cmpb %al, %bl
+; AVX2-NEXT:    setne %al
+; AVX2-NEXT:    andb %bpl, %al
+; AVX2-NEXT:    subq {{[0-9]+}}(%rsp), %rdx
+; AVX2-NEXT:    movq %rcx, %rbp
+; AVX2-NEXT:    sbbq %r10, %rbp
 ; AVX2-NEXT:    setns %bl
-; AVX2-NEXT:    testq %rsi, %rsi
+; AVX2-NEXT:    testq %rcx, %rcx
 ; AVX2-NEXT:    setns %cl
 ; AVX2-NEXT:    cmpb %bl, %cl
-; AVX2-NEXT:    setne %r11b
-; AVX2-NEXT:    subq %r8, %rdi
-; AVX2-NEXT:    sbbq %r9, %rsi
+; AVX2-NEXT:    setne %r8b
+; AVX2-NEXT:    testq %r10, %r10
 ; AVX2-NEXT:    setns %bl
 ; AVX2-NEXT:    cmpb %bl, %cl
 ; AVX2-NEXT:    setne %cl
-; AVX2-NEXT:    andb %r11b, %cl
+; AVX2-NEXT:    andb %r8b, %cl
 ; AVX2-NEXT:    movzbl %cl, %ecx
-; AVX2-NEXT:    vmovd %ecx, %xmm0
-; AVX2-NEXT:    vpinsrb $8, %ebp, %xmm0, %xmm0
-; AVX2-NEXT:    movq %rdx, 16(%r10)
-; AVX2-NEXT:    movq %rdi, (%r10)
-; AVX2-NEXT:    movq %rax, 24(%r10)
-; AVX2-NEXT:    movq %rsi, 8(%r10)
-; AVX2-NEXT:    vpsllq $63, %xmm0, %xmm0
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm0
+; AVX2-NEXT:    negl %ecx
+; AVX2-NEXT:    movzbl %al, %eax
+; AVX2-NEXT:    negl %eax
+; AVX2-NEXT:    vmovd %eax, %xmm0
+; AVX2-NEXT:    vpinsrd $1, %ecx, %xmm0, %xmm0
+; AVX2-NEXT:    movq %rdx, 16(%r11)
+; AVX2-NEXT:    movq %rdi, (%r11)
+; AVX2-NEXT:    movq %rbp, 24(%r11)
+; AVX2-NEXT:    movq %rsi, 8(%r11)
 ; AVX2-NEXT:    popq %rbx
 ; AVX2-NEXT:    popq %rbp
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: ssubo_v2i128:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    pushq %r14
+; AVX512-NEXT:    pushq %rbp
 ; AVX512-NEXT:    pushq %rbx
-; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %r10
 ; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %r11
-; AVX512-NEXT:    subq {{[0-9]+}}(%rsp), %rdx
-; AVX512-NEXT:    movq %rcx, %r14
-; AVX512-NEXT:    sbbq %r11, %r14
-; AVX512-NEXT:    setns %bl
-; AVX512-NEXT:    testq %rcx, %rcx
-; AVX512-NEXT:    setns %cl
-; AVX512-NEXT:    cmpb %bl, %cl
-; AVX512-NEXT:    setne %bl
-; AVX512-NEXT:    testq %r11, %r11
-; AVX512-NEXT:    setns %al
-; AVX512-NEXT:    cmpb %al, %cl
-; AVX512-NEXT:    setne %al
-; AVX512-NEXT:    andb %bl, %al
-; AVX512-NEXT:    kmovd %eax, %k0
-; AVX512-NEXT:    kshiftlw $1, %k0, %k0
+; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %r10
 ; AVX512-NEXT:    testq %r9, %r9
 ; AVX512-NEXT:    setns %al
 ; AVX512-NEXT:    testq %rsi, %rsi
-; AVX512-NEXT:    setns %cl
-; AVX512-NEXT:    cmpb %al, %cl
-; AVX512-NEXT:    setne %al
+; AVX512-NEXT:    setns %bl
+; AVX512-NEXT:    cmpb %al, %bl
+; AVX512-NEXT:    setne %bpl
 ; AVX512-NEXT:    subq %r8, %rdi
 ; AVX512-NEXT:    sbbq %r9, %rsi
+; AVX512-NEXT:    setns %al
+; AVX512-NEXT:    cmpb %al, %bl
+; AVX512-NEXT:    setne %al
+; AVX512-NEXT:    andb %bpl, %al
+; AVX512-NEXT:    subq {{[0-9]+}}(%rsp), %rdx
+; AVX512-NEXT:    movq %rcx, %rbp
+; AVX512-NEXT:    sbbq %r10, %rbp
+; AVX512-NEXT:    setns %bl
+; AVX512-NEXT:    testq %rcx, %rcx
+; AVX512-NEXT:    setns %cl
+; AVX512-NEXT:    cmpb %bl, %cl
+; AVX512-NEXT:    setne %r8b
+; AVX512-NEXT:    testq %r10, %r10
 ; AVX512-NEXT:    setns %bl
 ; AVX512-NEXT:    cmpb %bl, %cl
 ; AVX512-NEXT:    setne %cl
-; AVX512-NEXT:    andb %al, %cl
-; AVX512-NEXT:    andl $1, %ecx
-; AVX512-NEXT:    kmovw %ecx, %k1
+; AVX512-NEXT:    andb %r8b, %cl
+; AVX512-NEXT:    kmovd %ecx, %k0
+; AVX512-NEXT:    kshiftlw $1, %k0, %k0
+; AVX512-NEXT:    andl $1, %eax
+; AVX512-NEXT:    kmovw %eax, %k1
 ; AVX512-NEXT:    korw %k0, %k1, %k1
-; AVX512-NEXT:    movq %rdx, 16(%r10)
-; AVX512-NEXT:    movq %rdi, (%r10)
-; AVX512-NEXT:    movq %r14, 24(%r10)
-; AVX512-NEXT:    movq %rsi, 8(%r10)
 ; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX512-NEXT:    vmovdqa64 %xmm0, %xmm0 {%k1} {z}
+; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
+; AVX512-NEXT:    movq %rdx, 16(%r11)
+; AVX512-NEXT:    movq %rdi, (%r11)
+; AVX512-NEXT:    movq %rbp, 24(%r11)
+; AVX512-NEXT:    movq %rsi, 8(%r11)
 ; AVX512-NEXT:    popq %rbx
-; AVX512-NEXT:    popq %r14
+; AVX512-NEXT:    popq %rbp
 ; AVX512-NEXT:    retq
   %t = call {<2 x i128>, <2 x i1>} @llvm.ssub.with.overflow.v2i128(<2 x i128> %a0, <2 x i128> %a1)
   %val = extractvalue {<2 x i128>, <2 x i1>} %t, 0
diff --git a/llvm/test/CodeGen/X86/vec_trunc_sext.ll b/llvm/test/CodeGen/X86/vec_trunc_sext.ll
index 1e2de8e20..539be51d7 100644
--- a/llvm/test/CodeGen/X86/vec_trunc_sext.ll
+++ b/llvm/test/CodeGen/X86/vec_trunc_sext.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=-sse4.1 | FileCheck %s --check-prefix=NO_SSE_41
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=SSE_41
 
diff --git a/llvm/test/CodeGen/X86/vec_uaddo.ll b/llvm/test/CodeGen/X86/vec_uaddo.ll
index 4e9cd2efb..2b38aa243 100644
--- a/llvm/test/CodeGen/X86/vec_uaddo.ll
+++ b/llvm/test/CodeGen/X86/vec_uaddo.ll
@@ -47,91 +47,61 @@ define <1 x i32> @uaddo_v1i32(<1 x i32> %a0, <1 x i32> %a1, <1 x i32>* %p2) noun
 define <2 x i32> @uaddo_v2i32(<2 x i32> %a0, <2 x i32> %a1, <2 x i32>* %p2) nounwind {
 ; SSE2-LABEL: uaddo_v2i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [4294967295,0,4294967295,0]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    paddq %xmm1, %xmm0
-; SSE2-NEXT:    pand %xmm0, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,0,3,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pxor %xmm3, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; SSE2-NEXT:    movq %xmm0, (%rdi)
-; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648]
+; SSE2-NEXT:    paddd %xmm0, %xmm1
+; SSE2-NEXT:    pxor %xmm2, %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm0
+; SSE2-NEXT:    movq %xmm1, (%rdi)
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: uaddo_v2i32:
 ; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [4294967295,0,4294967295,0]
-; SSSE3-NEXT:    pand %xmm2, %xmm1
-; SSSE3-NEXT:    pand %xmm2, %xmm0
-; SSSE3-NEXT:    paddq %xmm1, %xmm0
-; SSSE3-NEXT:    pand %xmm0, %xmm2
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm2, %xmm3
-; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSSE3-NEXT:    pxor %xmm3, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; SSSE3-NEXT:    movq %xmm0, (%rdi)
-; SSSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648]
+; SSSE3-NEXT:    paddd %xmm0, %xmm1
+; SSSE3-NEXT:    pxor %xmm2, %xmm0
+; SSSE3-NEXT:    pxor %xmm1, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm0
+; SSSE3-NEXT:    movq %xmm1, (%rdi)
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: uaddo_v2i32:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pxor %xmm2, %xmm2
-; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
-; SSE41-NEXT:    paddq %xmm1, %xmm0
-; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
-; SSE41-NEXT:    pcmpeqq %xmm0, %xmm2
-; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE41-NEXT:    pxor %xmm2, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; SSE41-NEXT:    movq %xmm0, (%rdi)
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    paddd %xmm0, %xmm1
+; SSE41-NEXT:    pmaxud %xmm1, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm1, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm2, %xmm2
+; SSE41-NEXT:    pxor %xmm2, %xmm0
+; SSE41-NEXT:    movq %xmm1, (%rdi)
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: uaddo_v2i32:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
-; AVX1-NEXT:    vpaddq %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
-; AVX1-NEXT:    vpcmpeqq %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vpmaxud %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm1, %xmm0
 ; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
 ; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
 ; AVX1-NEXT:    vmovq %xmm1, (%rdi)
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: uaddo_v2i32:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
-; AVX2-NEXT:    vpaddq %xmm1, %xmm0, %xmm1
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; AVX2-NEXT:    vpcmpeqq %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm1
+; AVX2-NEXT:    vpmaxud %xmm0, %xmm1, %xmm0
+; AVX2-NEXT:    vpcmpeqd %xmm0, %xmm1, %xmm0
 ; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
 ; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
 ; AVX2-NEXT:    vmovq %xmm1, (%rdi)
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: uaddo_v2i32:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; AVX512-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
-; AVX512-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpblendd {{.*#+}} xmm1 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
-; AVX512-NEXT:    vpmovqd %xmm0, (%rdi)
-; AVX512-NEXT:    vpcmpeqq %xmm0, %xmm1, %xmm0
-; AVX512-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
+; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm1
+; AVX512-NEXT:    vpcmpltud %xmm0, %xmm1, %k1
+; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
+; AVX512-NEXT:    vmovq %xmm1, (%rdi)
 ; AVX512-NEXT:    retq
   %t = call {<2 x i32>, <2 x i1>} @llvm.uadd.with.overflow.v2i32(<2 x i32> %a0, <2 x i32> %a1)
   %val = extractvalue {<2 x i32>, <2 x i1>} %t, 0
@@ -922,12 +892,12 @@ define <2 x i32> @uaddo_v2i64(<2 x i64> %a0, <2 x i64> %a1, <2 x i64>* %p2) noun
 ; SSE-NEXT:    pxor %xmm1, %xmm2
 ; SSE-NEXT:    movdqa %xmm0, %xmm3
 ; SSE-NEXT:    pcmpgtd %xmm2, %xmm3
-; SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
 ; SSE-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE-NEXT:    pand %xmm4, %xmm2
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE-NEXT:    por %xmm2, %xmm0
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3]
+; SSE-NEXT:    pand %xmm3, %xmm0
+; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,1,3,3]
+; SSE-NEXT:    por %xmm0, %xmm2
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
 ; SSE-NEXT:    movdqa %xmm1, (%rdi)
 ; SSE-NEXT:    retq
 ;
@@ -938,6 +908,7 @@ define <2 x i32> @uaddo_v2i64(<2 x i64> %a0, <2 x i64> %a1, <2 x i64>* %p2) noun
 ; AVX1-NEXT:    vpaddq %xmm1, %xmm0, %xmm1
 ; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm0
 ; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm3, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; AVX1-NEXT:    vmovdqa %xmm1, (%rdi)
 ; AVX1-NEXT:    retq
 ;
@@ -948,6 +919,7 @@ define <2 x i32> @uaddo_v2i64(<2 x i64> %a0, <2 x i64> %a1, <2 x i64>* %p2) noun
 ; AVX2-NEXT:    vpaddq %xmm1, %xmm0, %xmm1
 ; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm0
 ; AVX2-NEXT:    vpcmpgtq %xmm0, %xmm3, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; AVX2-NEXT:    vmovdqa %xmm1, (%rdi)
 ; AVX2-NEXT:    retq
 ;
@@ -955,9 +927,9 @@ define <2 x i32> @uaddo_v2i64(<2 x i64> %a0, <2 x i64> %a1, <2 x i64>* %p2) noun
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpaddq %xmm1, %xmm0, %xmm1
 ; AVX512-NEXT:    vpcmpltuq %xmm0, %xmm1, %k1
-; AVX512-NEXT:    vmovdqa %xmm1, (%rdi)
 ; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX512-NEXT:    vmovdqa64 %xmm0, %xmm0 {%k1} {z}
+; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
+; AVX512-NEXT:    vmovdqa %xmm1, (%rdi)
 ; AVX512-NEXT:    retq
   %t = call {<2 x i64>, <2 x i1>} @llvm.uadd.with.overflow.v2i64(<2 x i64> %a0, <2 x i64> %a1)
   %val = extractvalue {<2 x i64>, <2 x i1>} %t, 0
@@ -1225,21 +1197,17 @@ define <2 x i32> @uaddo_v2i128(<2 x i128> %a0, <2 x i128> %a1, <2 x i128>* %p2)
 ; SSE2-NEXT:    movq {{[0-9]+}}(%rsp), %r10
 ; SSE2-NEXT:    addq {{[0-9]+}}(%rsp), %rdx
 ; SSE2-NEXT:    adcq {{[0-9]+}}(%rsp), %rcx
-; SSE2-NEXT:    setb %al
-; SSE2-NEXT:    movzbl %al, %r11d
+; SSE2-NEXT:    sbbl %eax, %eax
 ; SSE2-NEXT:    addq %r8, %rdi
 ; SSE2-NEXT:    adcq %r9, %rsi
-; SSE2-NEXT:    setb %al
-; SSE2-NEXT:    movzbl %al, %eax
+; SSE2-NEXT:    movd %eax, %xmm1
+; SSE2-NEXT:    sbbl %eax, %eax
 ; SSE2-NEXT:    movd %eax, %xmm0
-; SSE2-NEXT:    pinsrw $4, %r11d, %xmm0
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; SSE2-NEXT:    movq %rdx, 16(%r10)
 ; SSE2-NEXT:    movq %rdi, (%r10)
 ; SSE2-NEXT:    movq %rcx, 24(%r10)
 ; SSE2-NEXT:    movq %rsi, 8(%r10)
-; SSE2-NEXT:    psllq $63, %xmm0
-; SSE2-NEXT:    psrad $31, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: uaddo_v2i128:
@@ -1247,21 +1215,17 @@ define <2 x i32> @uaddo_v2i128(<2 x i128> %a0, <2 x i128> %a1, <2 x i128>* %p2)
 ; SSSE3-NEXT:    movq {{[0-9]+}}(%rsp), %r10
 ; SSSE3-NEXT:    addq {{[0-9]+}}(%rsp), %rdx
 ; SSSE3-NEXT:    adcq {{[0-9]+}}(%rsp), %rcx
-; SSSE3-NEXT:    setb %al
-; SSSE3-NEXT:    movzbl %al, %r11d
+; SSSE3-NEXT:    sbbl %eax, %eax
 ; SSSE3-NEXT:    addq %r8, %rdi
 ; SSSE3-NEXT:    adcq %r9, %rsi
-; SSSE3-NEXT:    setb %al
-; SSSE3-NEXT:    movzbl %al, %eax
+; SSSE3-NEXT:    movd %eax, %xmm1
+; SSSE3-NEXT:    sbbl %eax, %eax
 ; SSSE3-NEXT:    movd %eax, %xmm0
-; SSSE3-NEXT:    pinsrw $4, %r11d, %xmm0
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; SSSE3-NEXT:    movq %rdx, 16(%r10)
 ; SSSE3-NEXT:    movq %rdi, (%r10)
 ; SSSE3-NEXT:    movq %rcx, 24(%r10)
 ; SSSE3-NEXT:    movq %rsi, 8(%r10)
-; SSSE3-NEXT:    psllq $63, %xmm0
-; SSSE3-NEXT:    psrad $31, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: uaddo_v2i128:
@@ -1269,21 +1233,16 @@ define <2 x i32> @uaddo_v2i128(<2 x i128> %a0, <2 x i128> %a1, <2 x i128>* %p2)
 ; SSE41-NEXT:    movq {{[0-9]+}}(%rsp), %r10
 ; SSE41-NEXT:    addq {{[0-9]+}}(%rsp), %rdx
 ; SSE41-NEXT:    adcq {{[0-9]+}}(%rsp), %rcx
-; SSE41-NEXT:    setb %al
-; SSE41-NEXT:    movzbl %al, %r11d
+; SSE41-NEXT:    sbbl %r11d, %r11d
 ; SSE41-NEXT:    addq %r8, %rdi
 ; SSE41-NEXT:    adcq %r9, %rsi
-; SSE41-NEXT:    setb %al
-; SSE41-NEXT:    movzbl %al, %eax
+; SSE41-NEXT:    sbbl %eax, %eax
 ; SSE41-NEXT:    movd %eax, %xmm0
-; SSE41-NEXT:    pinsrb $8, %r11d, %xmm0
+; SSE41-NEXT:    pinsrd $1, %r11d, %xmm0
 ; SSE41-NEXT:    movq %rdx, 16(%r10)
 ; SSE41-NEXT:    movq %rdi, (%r10)
 ; SSE41-NEXT:    movq %rcx, 24(%r10)
 ; SSE41-NEXT:    movq %rsi, 8(%r10)
-; SSE41-NEXT:    psllq $63, %xmm0
-; SSE41-NEXT:    psrad $31, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: uaddo_v2i128:
@@ -1291,21 +1250,16 @@ define <2 x i32> @uaddo_v2i128(<2 x i128> %a0, <2 x i128> %a1, <2 x i128>* %p2)
 ; AVX1-NEXT:    movq {{[0-9]+}}(%rsp), %r10
 ; AVX1-NEXT:    addq {{[0-9]+}}(%rsp), %rdx
 ; AVX1-NEXT:    adcq {{[0-9]+}}(%rsp), %rcx
-; AVX1-NEXT:    setb %al
-; AVX1-NEXT:    movzbl %al, %r11d
+; AVX1-NEXT:    sbbl %r11d, %r11d
 ; AVX1-NEXT:    addq %r8, %rdi
 ; AVX1-NEXT:    adcq %r9, %rsi
-; AVX1-NEXT:    setb %al
-; AVX1-NEXT:    movzbl %al, %eax
+; AVX1-NEXT:    sbbl %eax, %eax
 ; AVX1-NEXT:    vmovd %eax, %xmm0
-; AVX1-NEXT:    vpinsrb $8, %r11d, %xmm0, %xmm0
+; AVX1-NEXT:    vpinsrd $1, %r11d, %xmm0, %xmm0
 ; AVX1-NEXT:    movq %rdx, 16(%r10)
 ; AVX1-NEXT:    movq %rdi, (%r10)
 ; AVX1-NEXT:    movq %rcx, 24(%r10)
 ; AVX1-NEXT:    movq %rsi, 8(%r10)
-; AVX1-NEXT:    vpsllq $63, %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: uaddo_v2i128:
@@ -1313,43 +1267,38 @@ define <2 x i32> @uaddo_v2i128(<2 x i128> %a0, <2 x i128> %a1, <2 x i128>* %p2)
 ; AVX2-NEXT:    movq {{[0-9]+}}(%rsp), %r10
 ; AVX2-NEXT:    addq {{[0-9]+}}(%rsp), %rdx
 ; AVX2-NEXT:    adcq {{[0-9]+}}(%rsp), %rcx
-; AVX2-NEXT:    setb %al
-; AVX2-NEXT:    movzbl %al, %r11d
+; AVX2-NEXT:    sbbl %r11d, %r11d
 ; AVX2-NEXT:    addq %r8, %rdi
 ; AVX2-NEXT:    adcq %r9, %rsi
-; AVX2-NEXT:    setb %al
-; AVX2-NEXT:    movzbl %al, %eax
+; AVX2-NEXT:    sbbl %eax, %eax
 ; AVX2-NEXT:    vmovd %eax, %xmm0
-; AVX2-NEXT:    vpinsrb $8, %r11d, %xmm0, %xmm0
+; AVX2-NEXT:    vpinsrd $1, %r11d, %xmm0, %xmm0
 ; AVX2-NEXT:    movq %rdx, 16(%r10)
 ; AVX2-NEXT:    movq %rdi, (%r10)
 ; AVX2-NEXT:    movq %rcx, 24(%r10)
 ; AVX2-NEXT:    movq %rsi, 8(%r10)
-; AVX2-NEXT:    vpsllq $63, %xmm0, %xmm0
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: uaddo_v2i128:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %r10
+; AVX512-NEXT:    addq %r8, %rdi
+; AVX512-NEXT:    adcq %r9, %rsi
+; AVX512-NEXT:    setb %r8b
 ; AVX512-NEXT:    addq {{[0-9]+}}(%rsp), %rdx
 ; AVX512-NEXT:    adcq {{[0-9]+}}(%rsp), %rcx
 ; AVX512-NEXT:    setb %al
 ; AVX512-NEXT:    kmovd %eax, %k0
 ; AVX512-NEXT:    kshiftlw $1, %k0, %k0
-; AVX512-NEXT:    addq %r8, %rdi
-; AVX512-NEXT:    adcq %r9, %rsi
-; AVX512-NEXT:    setb %al
-; AVX512-NEXT:    andl $1, %eax
-; AVX512-NEXT:    kmovw %eax, %k1
+; AVX512-NEXT:    andl $1, %r8d
+; AVX512-NEXT:    kmovw %r8d, %k1
 ; AVX512-NEXT:    korw %k0, %k1, %k1
+; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
 ; AVX512-NEXT:    movq %rdx, 16(%r10)
 ; AVX512-NEXT:    movq %rdi, (%r10)
 ; AVX512-NEXT:    movq %rcx, 24(%r10)
 ; AVX512-NEXT:    movq %rsi, 8(%r10)
-; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX512-NEXT:    vmovdqa64 %xmm0, %xmm0 {%k1} {z}
 ; AVX512-NEXT:    retq
   %t = call {<2 x i128>, <2 x i1>} @llvm.uadd.with.overflow.v2i128(<2 x i128> %a0, <2 x i128> %a1)
   %val = extractvalue {<2 x i128>, <2 x i1>} %t, 0
diff --git a/llvm/test/CodeGen/X86/vec_uint_to_fp-fastmath.ll b/llvm/test/CodeGen/X86/vec_uint_to_fp-fastmath.ll
index 5f4489c5e..cccd1d739 100644
--- a/llvm/test/CodeGen/X86/vec_uint_to_fp-fastmath.ll
+++ b/llvm/test/CodeGen/X86/vec_uint_to_fp-fastmath.ll
@@ -11,72 +11,90 @@
 ; RUN: llc < %s -mtriple=x86_64 -enable-unsafe-fp-math -mattr=+avx512vl \
 ; RUN:   | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512VL
 
+; Check that the constant used in the vectors are the right ones.
 ; SSE2: [[MASKCSTADDR:.LCPI[0-9_]+]]:
 ; SSE2-NEXT: .long 65535 # 0xffff
 ; SSE2-NEXT: .long 65535 # 0xffff
 ; SSE2-NEXT: .long 65535 # 0xffff
 ; SSE2-NEXT: .long 65535 # 0xffff
 
-; CST: [[FPMASKCSTADDR:.LCPI[0-9_]+]]:
-; CST-NEXT: .long 1199570944 # float 65536
-; CST-NEXT: .long 1199570944 # float 65536
-; CST-NEXT: .long 1199570944 # float 65536
-; CST-NEXT: .long 1199570944 # float 65536
+; CST: [[LOWCSTADDR:.LCPI[0-9_]+]]:
+; CST-NEXT: .long 1258291200 # 0x4b000000
+; CST-NEXT: .long 1258291200 # 0x4b000000
+; CST-NEXT: .long 1258291200 # 0x4b000000
+; CST-NEXT: .long 1258291200 # 0x4b000000
 
-; AVX2: [[FPMASKCSTADDR:.LCPI[0-9_]+]]:
-; AVX2-NEXT: .long 1199570944 # float 65536
+; CST: [[HIGHCSTADDR:.LCPI[0-9_]+]]:
+; CST-NEXT: .long 1392508928 # 0x53000000
+; CST-NEXT: .long 1392508928 # 0x53000000
+; CST-NEXT: .long 1392508928 # 0x53000000
+; CST-NEXT: .long 1392508928 # 0x53000000
+
+; CST: [[MAGICCSTADDR:.LCPI[0-9_]+]]:
+; CST-NEXT: .long 1392509056 # float 5.49764202E+11
+; CST-NEXT: .long 1392509056 # float 5.49764202E+11
+; CST-NEXT: .long 1392509056 # float 5.49764202E+11
+; CST-NEXT: .long 1392509056 # float 5.49764202E+11
+
+; AVX2: [[LOWCSTADDR:.LCPI[0-9_]+]]:
+; AVX2-NEXT: .long 1258291200 # 0x4b000000
+
+; AVX2: [[HIGHCSTADDR:.LCPI[0-9_]+]]:
+; AVX2-NEXT: .long 1392508928 # 0x53000000
+
+; AVX2: [[MAGICCSTADDR:.LCPI[0-9_]+]]:
+; AVX2-NEXT: .long 1392509056 # float 5.49764202E+11
 
 define <4 x float> @test_uitofp_v4i32_to_v4f32(<4 x i32> %arg) {
 ; SSE2-LABEL: test_uitofp_v4i32_to_v4f32:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    movaps {{.*#+}} xmm1 = [65535,65535,65535,65535]
-; SSE2-NEXT:    andps %xmm0, %xmm1
-; SSE2-NEXT:    cvtdq2ps %xmm1, %xmm1
-; SSE2-NEXT:    psrld $16, %xmm0
-; SSE2-NEXT:    cvtdq2ps %xmm0, %xmm0
-; SSE2-NEXT:    mulps [[FPMASKCSTADDR]](%rip), %xmm0
-; SSE2-NEXT:    addps %xmm1, %xmm0
-; SSE2-NEXT:    retq
+; SSE2: movdqa [[MASKCSTADDR]](%rip), [[MASK:%xmm[0-9]+]]
+; SSE2-NEXT: pand %xmm0, [[MASK]]
+; After this instruction, MASK will have the value of the low parts
+; of the vector.
+; SSE2-NEXT: por [[LOWCSTADDR]](%rip), [[MASK]]
+; SSE2-NEXT: psrld $16, %xmm0
+; SSE2-NEXT: por [[HIGHCSTADDR]](%rip), %xmm0
+; SSE2-NEXT: subps [[MAGICCSTADDR]](%rip), %xmm0
+; SSE2-NEXT: addps [[MASK]], %xmm0
+; SSE2-NEXT: retq
 ;
+; Currently we commute the arguments of the first blend, but this could be
+; improved to match the lowering of the second blend.
 ; SSE41-LABEL: test_uitofp_v4i32_to_v4f32:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pxor %xmm1, %xmm1
-; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7]
-; SSE41-NEXT:    cvtdq2ps %xmm1, %xmm1
-; SSE41-NEXT:    psrld $16, %xmm0
-; SSE41-NEXT:    cvtdq2ps %xmm0, %xmm0
-; SSE41-NEXT:    mulps [[FPMASKCSTADDR]](%rip), %xmm0
-; SSE41-NEXT:    addps %xmm1, %xmm0
-; SSE41-NEXT:    retq
+; SSE41: movdqa [[LOWCSTADDR]](%rip), [[LOWVEC:%xmm[0-9]+]]
+; SSE41-NEXT: pblendw $85, %xmm0, [[LOWVEC]]
+; SSE41-NEXT: psrld $16, %xmm0
+; SSE41-NEXT: pblendw $170, [[HIGHCSTADDR]](%rip), %xmm0
+; SSE41-NEXT: subps [[MAGICCSTADDR]](%rip), %xmm0
+; SSE41-NEXT: addps [[LOWVEC]], %xmm0
+; SSE41-NEXT: retq
 ;
 ; AVX-LABEL: test_uitofp_v4i32_to_v4f32:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7]
-; AVX-NEXT:    vcvtdq2ps %xmm1, %xmm1
-; AVX-NEXT:    vpsrld $16, %xmm0, %xmm0
-; AVX-NEXT:    vcvtdq2ps %xmm0, %xmm0
-; AVX-NEXT:    vmulps [[FPMASKCSTADDR]](%rip), %xmm0, %xmm0
-; AVX-NEXT:    vaddps %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    retq
+; AVX: vpblendw $170, [[LOWCSTADDR]](%rip), %xmm0, [[LOWVEC:%xmm[0-9]+]]
+; AVX-NEXT: vpsrld $16, %xmm0, [[SHIFTVEC:%xmm[0-9]+]]
+; AVX-NEXT: vpblendw $170, [[HIGHCSTADDR]](%rip), [[SHIFTVEC]], [[HIGHVEC:%xmm[0-9]+]]
+; AVX-NEXT: vsubps [[MAGICCSTADDR]](%rip), [[HIGHVEC]], [[TMP:%xmm[0-9]+]]
+; AVX-NEXT: vaddps [[TMP]], [[LOWVEC]], %xmm0
+; AVX-NEXT: retq
 ;
+; The lowering for AVX2 is a bit messy, because we select broadcast
+; instructions, instead of folding the constant loads.
 ; AVX2-LABEL: test_uitofp_v4i32_to_v4f32:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX2-NEXT:    vcvtdq2ps %xmm1, %xmm1
-; AVX2-NEXT:    vbroadcastss [[FPMASKCSTADDR]](%rip), %xmm2
-; AVX2-NEXT:    vmulps %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vxorps %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3],xmm0[4],xmm2[5],xmm0[6],xmm2[7]
-; AVX2-NEXT:    vcvtdq2ps %xmm0, %xmm0
-; AVX2-NEXT:    vaddps %xmm0, %xmm1, %xmm0
-; AVX2-NEXT:    retq
+; AVX2: vpbroadcastd [[LOWCSTADDR]](%rip), [[LOWCST:%xmm[0-9]+]]
+; AVX2-NEXT: vpblendw $170, [[LOWCST]], %xmm0, [[LOWVEC:%xmm[0-9]+]]
+; AVX2-NEXT: vpsrld $16, %xmm0, [[SHIFTVEC:%xmm[0-9]+]]
+; AVX2-NEXT: vpbroadcastd [[HIGHCSTADDR]](%rip), [[HIGHCST:%xmm[0-9]+]]
+; AVX2-NEXT: vpblendw $170, [[HIGHCST]], [[SHIFTVEC]], [[HIGHVEC:%xmm[0-9]+]]
+; AVX2-NEXT: vbroadcastss [[MAGICCSTADDR]](%rip), [[MAGICCST:%xmm[0-9]+]]
+; AVX2-NEXT: vsubps [[MAGICCST]], [[HIGHVEC]], [[TMP:%xmm[0-9]+]]
+; AVX2-NEXT: vaddps [[TMP]], [[LOWVEC]], %xmm0
+; AVX2-NEXT: retq
 ;
 ; AVX512F-LABEL: test_uitofp_v4i32_to_v4f32:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    # kill
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; AVX512F-NEXT:    vcvtudq2ps %zmm0, %zmm0
-; AVX512F-NEXT:    # kill
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
@@ -88,86 +106,76 @@ define <4 x float> @test_uitofp_v4i32_to_v4f32(<4 x i32> %arg) {
   ret <4 x float> %tmp
 }
 
-; AVX: [[FPMASKCSTADDR_v8:.LCPI[0-9_]+]]:
-; AVX-NEXT: .long 1199570944 # float 65536
-; AVX-NEXT: .long 1199570944 # float 65536
-; AVX-NEXT: .long 1199570944 # float 65536
-; AVX-NEXT: .long 1199570944 # float 65536
+; Match the AVX2 constants used in the next function
+; AVX2: [[LOWCSTADDR:.LCPI[0-9_]+]]:
+; AVX2-NEXT: .long 1258291200 # 0x4b000000
 
-; AVX: [[MASKCSTADDR_v8:.LCPI[0-9_]+]]:
-; AVX-NEXT: .long 65535 # 0xffff
-; AVX-NEXT: .long 65535 # 0xffff
-; AVX-NEXT: .long 65535 # 0xffff
-; AVX-NEXT: .long 65535 # 0xffff
+; AVX2: [[HIGHCSTADDR:.LCPI[0-9_]+]]:
+; AVX2-NEXT: .long 1392508928 # 0x53000000
 
-; AVX2: [[FPMASKCSTADDR_v8:.LCPI[0-9_]+]]:
-; AVX2-NEXT: .long 1199570944 # float 65536
+; AVX2: [[MAGICCSTADDR:.LCPI[0-9_]+]]:
+; AVX2-NEXT: .long 1392509056 # float 5.49764202E+11
 
 define <8 x float> @test_uitofp_v8i32_to_v8f32(<8 x i32> %arg) {
+; Legalization will break the thing is 2 x <4 x i32> on anthing prior AVX.
+; The constant used for in the vector instruction are shared between the
+; two sequences of instructions.
+;
 ; SSE2-LABEL: test_uitofp_v8i32_to_v8f32:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    psrld $16, %xmm2
-; SSE2-NEXT:    cvtdq2ps %xmm2, %xmm2
-; SSE2-NEXT:    movaps {{.*#+}} xmm3 = [6.5536E+4,6.5536E+4,6.5536E+4,6.5536E+4]
-; SSE2-NEXT:    mulps %xmm3, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [65535,65535,65535,65535]
-; SSE2-NEXT:    pand %xmm4, %xmm0
-; SSE2-NEXT:    cvtdq2ps %xmm0, %xmm0
-; SSE2-NEXT:    addps %xmm2, %xmm0
-; SSE2-NEXT:    movdqa %xmm1, %xmm2
-; SSE2-NEXT:    psrld $16, %xmm2
-; SSE2-NEXT:    cvtdq2ps %xmm2, %xmm2
-; SSE2-NEXT:    mulps %xmm3, %xmm2
-; SSE2-NEXT:    pand %xmm4, %xmm1
-; SSE2-NEXT:    cvtdq2ps %xmm1, %xmm1
-; SSE2-NEXT:    addps %xmm2, %xmm1
-; SSE2-NEXT:    retq
+; SSE2: movdqa {{.*#+}} [[MASK:xmm[0-9]+]] = [65535,65535,65535,65535]
+; SSE2-NEXT: movdqa %xmm0, [[VECLOW:%xmm[0-9]+]]
+; SSE2-NEXT: pand %[[MASK]], [[VECLOW]]
+; SSE2-NEXT: movdqa {{.*#+}} [[LOWCST:xmm[0-9]+]] = [1258291200,1258291200,1258291200,1258291200]
+; SSE2-NEXT: por %[[LOWCST]], [[VECLOW]]
+; SSE2-NEXT: psrld $16, %xmm0
+; SSE2-NEXT: movdqa {{.*#+}} [[HIGHCST:xmm[0-9]+]] = [1392508928,1392508928,1392508928,1392508928]
+; SSE2-NEXT: por %[[HIGHCST]], %xmm0
+; SSE2-NEXT: movaps {{.*#+}} [[MAGICCST:xmm[0-9]+]] = [5.49764202E+11,5.49764202E+11,5.49764202E+11,5.49764202E+11]
+; SSE2-NEXT: subps %[[MAGICCST]], %xmm0
+; SSE2-NEXT: addps [[VECLOW]], %xmm0
+; MASK is the low vector of the second part after this point.
+; SSE2-NEXT: pand %xmm1, %[[MASK]]
+; SSE2-NEXT: por %[[LOWCST]], %[[MASK]]
+; SSE2-NEXT: psrld $16, %xmm1
+; SSE2-NEXT: por %[[HIGHCST]], %xmm1
+; SSE2-NEXT: subps %[[MAGICCST]], %xmm1
+; SSE2-NEXT: addps %[[MASK]], %xmm1
+; SSE2-NEXT: retq
 ;
 ; SSE41-LABEL: test_uitofp_v8i32_to_v8f32:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    psrld $16, %xmm2
-; SSE41-NEXT:    cvtdq2ps %xmm2, %xmm2
-; SSE41-NEXT:    movaps {{.*#+}} xmm3 = [6.5536E+4,6.5536E+4,6.5536E+4,6.5536E+4]
-; SSE41-NEXT:    mulps %xmm3, %xmm2
-; SSE41-NEXT:    pxor %xmm4, %xmm4
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0],xmm4[1],xmm0[2],xmm4[3],xmm0[4],xmm4[5],xmm0[6],xmm4[7]
-; SSE41-NEXT:    cvtdq2ps %xmm0, %xmm0
-; SSE41-NEXT:    addps %xmm2, %xmm0
-; SSE41-NEXT:    movdqa %xmm1, %xmm2
-; SSE41-NEXT:    psrld $16, %xmm2
-; SSE41-NEXT:    cvtdq2ps %xmm2, %xmm2
-; SSE41-NEXT:    mulps %xmm3, %xmm2
-; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0],xmm4[1],xmm1[2],xmm4[3],xmm1[4],xmm4[5],xmm1[6],xmm4[7]
-; SSE41-NEXT:    cvtdq2ps %xmm1, %xmm1
-; SSE41-NEXT:    addps %xmm2, %xmm1
-; SSE41-NEXT:    retq
+; SSE41: movdqa {{.*#+}} [[LOWCST:xmm[0-9]+]] = [1258291200,1258291200,1258291200,1258291200]
+; SSE41-NEXT: movdqa %xmm0, [[VECLOW:%xmm[0-9]+]]
+; SSE41-NEXT: pblendw $170, %[[LOWCST]], [[VECLOW]]
+; SSE41-NEXT: psrld $16, %xmm0
+; SSE41-NEXT: movdqa {{.*#+}} [[HIGHCST:xmm[0-9]+]] = [1392508928,1392508928,1392508928,1392508928]
+; SSE41-NEXT: pblendw $170, %[[HIGHCST]], %xmm0
+; SSE41-NEXT: movaps {{.*#+}} [[MAGICCST:xmm[0-9]+]] = [5.49764202E+11,5.49764202E+11,5.49764202E+11,5.49764202E+11]
+; SSE41-NEXT: subps %[[MAGICCST]], %xmm0
+; SSE41-NEXT: addps [[VECLOW]], %xmm0
+; LOWCST is the low vector of the second part after this point.
+; The operands of the blend are inverted because we reuse xmm1
+; in the next shift.
+; SSE41-NEXT: pblendw $85, %xmm1, %[[LOWCST]]
+; SSE41-NEXT: psrld $16, %xmm1
+; SSE41-NEXT: pblendw $170, %[[HIGHCST]], %xmm1
+; SSE41-NEXT: subps %[[MAGICCST]], %xmm1
+; SSE41-NEXT: addps %[[LOWCST]], %xmm1
+; SSE41-NEXT: retq
 ;
-; AVX-LABEL: test_uitofp_v8i32_to_v8f32:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX-NEXT:    vpsrld $16, %xmm2, %xmm2
-; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
-; AVX-NEXT:    vcvtdq2ps %ymm1, %ymm1
-; AVX-NEXT:    vmulps [[FPMASKCSTADDR_v8]](%rip), %ymm1, %ymm1
-; AVX-NEXT:    vandps [[MASKCSTADDR_v8]](%rip), %ymm0, %ymm0
-; AVX-NEXT:    vcvtdq2ps %ymm0, %ymm0
-; AVX-NEXT:    vaddps %ymm0, %ymm1, %ymm0
-; AVX-NEXT:    retq
+; Test that we are not lowering uinttofp to scalars
+; AVX-NOT: cvtsd2ss
+; AVX: retq
 ;
 ; AVX2-LABEL: test_uitofp_v8i32_to_v8f32:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpsrld $16, %ymm0, %ymm1
-; AVX2-NEXT:    vcvtdq2ps %ymm1, %ymm1
-; AVX2-NEXT:    vbroadcastss [[FPMASKCSTADDR_v8]](%rip), %ymm2
-; AVX2-NEXT:    vmulps %ymm2, %ymm1, %ymm1
-; AVX2-NEXT:    vxorps %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm2[1],ymm0[2],ymm2[3],ymm0[4],ymm2[5],ymm0[6],ymm2[7],ymm0[8],ymm2[9],ymm0[10],ymm2[11],ymm0[12],ymm2[13],ymm0[14],ymm2[15]
-; AVX2-NEXT:    vcvtdq2ps %ymm0, %ymm0
-; AVX2-NEXT:    vaddps %ymm0, %ymm1, %ymm0
-; AVX2-NEXT:    retq
+; AVX2: vpbroadcastd [[LOWCSTADDR]](%rip), [[LOWCST:%ymm[0-9]+]]
+; AVX2-NEXT: vpblendw $170, [[LOWCST]], %ymm0, [[LOWVEC:%ymm[0-9]+]]
+; AVX2-NEXT: vpsrld $16, %ymm0, [[SHIFTVEC:%ymm[0-9]+]]
+; AVX2-NEXT: vpbroadcastd [[HIGHCSTADDR]](%rip), [[HIGHCST:%ymm[0-9]+]]
+; AVX2-NEXT: vpblendw $170, [[HIGHCST]], [[SHIFTVEC]], [[HIGHVEC:%ymm[0-9]+]]
+; AVX2-NEXT: vbroadcastss [[MAGICCSTADDR]](%rip), [[MAGICCST:%ymm[0-9]+]]
+; AVX2-NEXT: vsubps [[MAGICCST]], [[HIGHVEC]], [[TMP:%ymm[0-9]+]]
+; AVX2-NEXT: vaddps [[TMP]], [[LOWVEC]], %ymm0
+; AVX2-NEXT: retq
 ;
 ; AVX512F-LABEL: test_uitofp_v8i32_to_v8f32:
 ; AVX512F:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/vec_uint_to_fp.ll b/llvm/test/CodeGen/X86/vec_uint_to_fp.ll
index c7871d86c..4dd5223f0 100644
--- a/llvm/test/CodeGen/X86/vec_uint_to_fp.ll
+++ b/llvm/test/CodeGen/X86/vec_uint_to_fp.ll
@@ -23,10 +23,10 @@
 ; CST-NEXT: .long	1392508928              ## 0x53000000
 
 ; CST: [[MAGICCSTADDR:LCPI0_[0-9]+]]:
-; CST-NEXT: .long	3539992704              ## float -5.49764202E+11
-; CST-NEXT: .long	3539992704              ## float -5.49764202E+11
-; CST-NEXT: .long	3539992704              ## float -5.49764202E+11
-; CST-NEXT: .long	3539992704              ## float -5.49764202E+11
+; CST-NEXT: .long	1392509056              ## float 5.49764202E+11
+; CST-NEXT: .long	1392509056              ## float 5.49764202E+11
+; CST-NEXT: .long	1392509056              ## float 5.49764202E+11
+; CST-NEXT: .long	1392509056              ## float 5.49764202E+11
 
 ; AVX2: [[LOWCSTADDR:LCPI0_[0-9]+]]:
 ; AVX2-NEXT: .long	1258291200              ## 0x4b000000
@@ -35,7 +35,7 @@
 ; AVX2-NEXT: .long	1392508928              ## 0x53000000
 
 ; AVX2: [[MAGICCSTADDR:LCPI0_[0-9]+]]:
-; AVX2-NEXT: .long	3539992704              ## float -5.49764202E+11
+; AVX2-NEXT: .long	1392509056              ## float 5.49764202E+11
 
 define <4 x float> @test1(<4 x i32> %A) nounwind {
 ; CHECK-LABEL: test1:
@@ -47,7 +47,7 @@ define <4 x float> @test1(<4 x i32> %A) nounwind {
 ; SSE-NEXT: por [[LOWCSTADDR]](%rip), [[MASK]]
 ; SSE-NEXT: psrld $16, %xmm0
 ; SSE-NEXT: por [[HIGHCSTADDR]](%rip), %xmm0
-; SSE-NEXT: addps [[MAGICCSTADDR]](%rip), %xmm0
+; SSE-NEXT: subps [[MAGICCSTADDR]](%rip), %xmm0
 ; SSE-NEXT: addps [[MASK]], %xmm0
 ; SSE-NEXT: retq
 ;
@@ -57,14 +57,14 @@ define <4 x float> @test1(<4 x i32> %A) nounwind {
 ; SSE41-NEXT: pblendw $85, %xmm0, [[LOWVEC]]
 ; SSE41-NEXT: psrld $16, %xmm0
 ; SSE41-NEXT: pblendw $170, [[HIGHCSTADDR]](%rip), %xmm0
-; SSE41-NEXT: addps [[MAGICCSTADDR]](%rip), %xmm0
+; SSE41-NEXT: subps [[MAGICCSTADDR]](%rip), %xmm0
 ; SSE41-NEXT: addps [[LOWVEC]], %xmm0
 ; SSE41-NEXT: retq
 ;
 ; AVX: vpblendw $170, [[LOWCSTADDR]](%rip), %xmm0, [[LOWVEC:%xmm[0-9]+]]
 ; AVX-NEXT: vpsrld $16, %xmm0, [[SHIFTVEC:%xmm[0-9]+]]
 ; AVX-NEXT: vpblendw $170, [[HIGHCSTADDR]](%rip), [[SHIFTVEC]], [[HIGHVEC:%xmm[0-9]+]]
-; AVX-NEXT: vaddps [[MAGICCSTADDR]](%rip), [[HIGHVEC]], [[TMP:%xmm[0-9]+]]
+; AVX-NEXT: vsubps [[MAGICCSTADDR]](%rip), [[HIGHVEC]], [[TMP:%xmm[0-9]+]]
 ; AVX-NEXT: vaddps [[TMP]], [[LOWVEC]], %xmm0
 ; AVX-NEXT: retq
 ;
@@ -76,7 +76,7 @@ define <4 x float> @test1(<4 x i32> %A) nounwind {
 ; AVX2-NEXT: vpbroadcastd [[HIGHCSTADDR]](%rip), [[HIGHCST:%xmm[0-9]+]]
 ; AVX2-NEXT: vpblendw $170, [[HIGHCST]], [[SHIFTVEC]], [[HIGHVEC:%xmm[0-9]+]]
 ; AVX2-NEXT: vbroadcastss [[MAGICCSTADDR]](%rip), [[MAGICCST:%xmm[0-9]+]]
-; AVX2-NEXT: vaddps [[MAGICCST]], [[HIGHVEC]], [[TMP:%xmm[0-9]+]]
+; AVX2-NEXT: vsubps [[MAGICCST]], [[HIGHVEC]], [[TMP:%xmm[0-9]+]]
 ; AVX2-NEXT: vaddps [[TMP]], [[LOWVEC]], %xmm0
 ; AVX2-NEXT: retq
   %C = uitofp <4 x i32> %A to <4 x float>
@@ -91,7 +91,7 @@ define <4 x float> @test1(<4 x i32> %A) nounwind {
 ; AVX2-NEXT: .long	1392508928              ## 0x53000000
 
 ; AVX2: [[MAGICCSTADDR:LCPI1_[0-9]+]]:
-; AVX2-NEXT: .long	3539992704              ## float -5.49764202E+11
+; AVX2-NEXT: .long	1392509056              ## float 5.49764202E+11
 
 define <8 x float> @test2(<8 x i32> %A) nounwind {
 ; CHECK-LABEL: test2:
@@ -107,15 +107,15 @@ define <8 x float> @test2(<8 x i32> %A) nounwind {
 ; SSE-NEXT: psrld $16, %xmm0
 ; SSE-NEXT: movdqa {{.*#+}} [[HIGHCST:xmm[0-9]+]] = [1392508928,1392508928,1392508928,1392508928]
 ; SSE-NEXT: por %[[HIGHCST]], %xmm0
-; SSE-NEXT: movaps {{.*#+}} [[MAGICCST:xmm[0-9]+]] = [-5.49764202E+11,-5.49764202E+11,-5.49764202E+11,-5.49764202E+11]
-; SSE-NEXT: addps %[[MAGICCST]], %xmm0
+; SSE-NEXT: movaps {{.*#+}} [[MAGICCST:xmm[0-9]+]] = [5.49764202E+11,5.49764202E+11,5.49764202E+11,5.49764202E+11]
+; SSE-NEXT: subps %[[MAGICCST]], %xmm0
 ; SSE-NEXT: addps [[VECLOW]], %xmm0
 ; MASK is the low vector of the second part after this point.
 ; SSE-NEXT: pand %xmm1, %[[MASK]]
 ; SSE-NEXT: por %[[LOWCST]], %[[MASK]]
 ; SSE-NEXT: psrld $16, %xmm1
 ; SSE-NEXT: por %[[HIGHCST]], %xmm1
-; SSE-NEXT: addps %[[MAGICCST]], %xmm1
+; SSE-NEXT: subps %[[MAGICCST]], %xmm1
 ; SSE-NEXT: addps %[[MASK]], %xmm1
 ; SSE-NEXT: retq
 ;
@@ -125,8 +125,8 @@ define <8 x float> @test2(<8 x i32> %A) nounwind {
 ; SSE41-NEXT: psrld $16, %xmm0
 ; SSE41-NEXT: movdqa {{.*#+}} [[HIGHCST:xmm[0-9]+]] = [1392508928,1392508928,1392508928,1392508928]
 ; SSE41-NEXT: pblendw $170, %[[HIGHCST]], %xmm0
-; SSE41-NEXT: movaps {{.*#+}} [[MAGICCST:xmm[0-9]+]] = [-5.49764202E+11,-5.49764202E+11,-5.49764202E+11,-5.49764202E+11]
-; SSE41-NEXT: addps %[[MAGICCST]], %xmm0
+; SSE41-NEXT: movaps {{.*#+}} [[MAGICCST:xmm[0-9]+]] = [5.49764202E+11,5.49764202E+11,5.49764202E+11,5.49764202E+11]
+; SSE41-NEXT: subps %[[MAGICCST]], %xmm0
 ; SSE41-NEXT: addps [[VECLOW]], %xmm0
 ; LOWCST is the low vector of the second part after this point.
 ; The operands of the blend are inverted because we reuse xmm1
@@ -134,7 +134,7 @@ define <8 x float> @test2(<8 x i32> %A) nounwind {
 ; SSE41-NEXT: pblendw $85, %xmm1, %[[LOWCST]]
 ; SSE41-NEXT: psrld $16, %xmm1
 ; SSE41-NEXT: pblendw $170, %[[HIGHCST]], %xmm1
-; SSE41-NEXT: addps %[[MAGICCST]], %xmm1
+; SSE41-NEXT: subps %[[MAGICCST]], %xmm1
 ; SSE41-NEXT: addps %[[LOWCST]], %xmm1
 ; SSE41-NEXT: retq
 ;
@@ -148,7 +148,7 @@ define <8 x float> @test2(<8 x i32> %A) nounwind {
 ; AVX2-NEXT: vpbroadcastd [[HIGHCSTADDR]](%rip), [[HIGHCST:%ymm[0-9]+]]
 ; AVX2-NEXT: vpblendw $170, [[HIGHCST]], [[SHIFTVEC]], [[HIGHVEC:%ymm[0-9]+]]
 ; AVX2-NEXT: vbroadcastss [[MAGICCSTADDR]](%rip), [[MAGICCST:%ymm[0-9]+]]
-; AVX2-NEXT: vaddps [[MAGICCST]], [[HIGHVEC]], [[TMP:%ymm[0-9]+]]
+; AVX2-NEXT: vsubps [[MAGICCST]], [[HIGHVEC]], [[TMP:%ymm[0-9]+]]
 ; AVX2-NEXT: vaddps [[TMP]], [[LOWVEC]], %ymm0
 ; AVX2-NEXT: retq
   %C = uitofp <8 x i32> %A to <8 x float>
@@ -159,8 +159,8 @@ define <4 x double> @test3(<4 x i32> %arg) {
 ; CHECK-LABEL: test3:
 ; This test used to crash because we were custom lowering it as if it was
 ; a conversion between <4 x i32> and <4 x float>.
-; AVX: vcvtdq2pd
-; AVX2: vcvtdq2pd
+; AVX: vsubpd
+; AVX2: vsubpd
 ; CHECK: retq
   %tmp = uitofp <4 x i32> %arg to <4 x double>
   ret <4 x double> %tmp
diff --git a/llvm/test/CodeGen/X86/vec_umulo.ll b/llvm/test/CodeGen/X86/vec_umulo.ll
index 0bcaacc21..7db20d176 100644
--- a/llvm/test/CodeGen/X86/vec_umulo.ll
+++ b/llvm/test/CodeGen/X86/vec_umulo.ll
@@ -57,205 +57,105 @@ define <1 x i32> @umulo_v1i32(<1 x i32> %a0, <1 x i32> %a1, <1 x i32>* %p2) noun
 define <2 x i32> @umulo_v2i32(<2 x i32> %a0, <2 x i32> %a1, <2 x i32>* %p2) nounwind {
 ; SSE2-LABEL: umulo_v2i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [4294967295,0,4294967295,0]
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,0,1]
-; SSE2-NEXT:    movq %xmm3, %r8
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
-; SSE2-NEXT:    movq %xmm2, %r10
-; SSE2-NEXT:    movq %xmm0, %rax
-; SSE2-NEXT:    movq %xmm1, %rdx
-; SSE2-NEXT:    xorl %esi, %esi
-; SSE2-NEXT:    mulq %rdx
-; SSE2-NEXT:    movq $-1, %r9
-; SSE2-NEXT:    movl $0, %ecx
-; SSE2-NEXT:    cmovoq %r9, %rcx
-; SSE2-NEXT:    movq %rax, %xmm0
-; SSE2-NEXT:    movq %r8, %rax
-; SSE2-NEXT:    mulq %r10
-; SSE2-NEXT:    movq %rax, %xmm1
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,2,2,3]
-; SSE2-NEXT:    psrlq $32, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pmuludq %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,3,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pmuludq %xmm2, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,3,2,3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,0,3,2]
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT:    pxor %xmm0, %xmm2
-; SSE2-NEXT:    movq %rcx, %xmm0
-; SSE2-NEXT:    cmovoq %r9, %rsi
-; SSE2-NEXT:    movq %rsi, %xmm3
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
-; SSE2-NEXT:    por %xmm2, %xmm0
-; SSE2-NEXT:    movq %xmm1, (%rdi)
+; SSE2-NEXT:    pcmpeqd %xmm3, %xmm2
+; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
+; SSE2-NEXT:    pxor %xmm2, %xmm1
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
+; SSE2-NEXT:    movq %xmm0, (%rdi)
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: umulo_v2i32:
 ; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [4294967295,0,4294967295,0]
-; SSSE3-NEXT:    pand %xmm2, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,0,1]
-; SSSE3-NEXT:    movq %xmm3, %r8
-; SSSE3-NEXT:    pand %xmm2, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
-; SSSE3-NEXT:    movq %xmm2, %r10
-; SSSE3-NEXT:    movq %xmm0, %rax
-; SSSE3-NEXT:    movq %xmm1, %rdx
-; SSSE3-NEXT:    xorl %esi, %esi
-; SSSE3-NEXT:    mulq %rdx
-; SSSE3-NEXT:    movq $-1, %r9
-; SSSE3-NEXT:    movl $0, %ecx
-; SSSE3-NEXT:    cmovoq %r9, %rcx
-; SSSE3-NEXT:    movq %rax, %xmm0
-; SSSE3-NEXT:    movq %r8, %rax
-; SSSE3-NEXT:    mulq %r10
-; SSSE3-NEXT:    movq %rax, %xmm1
-; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,2,2,3]
-; SSSE3-NEXT:    psrlq $32, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pmuludq %xmm1, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,3,2,3]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    pmuludq %xmm2, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,3,2,3]
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
 ; SSSE3-NEXT:    pxor %xmm2, %xmm2
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm2, %xmm0
-; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSSE3-NEXT:    pxor %xmm0, %xmm2
-; SSSE3-NEXT:    movq %rcx, %xmm0
-; SSSE3-NEXT:    cmovoq %r9, %rsi
-; SSSE3-NEXT:    movq %rsi, %xmm3
-; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
-; SSSE3-NEXT:    por %xmm2, %xmm0
-; SSSE3-NEXT:    movq %xmm1, (%rdi)
+; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm2
+; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm1
+; SSSE3-NEXT:    pxor %xmm2, %xmm1
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
+; SSSE3-NEXT:    movq %xmm0, (%rdi)
+; SSSE3-NEXT:    movdqa %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: umulo_v2i32:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pxor %xmm2, %xmm2
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
-; SSE41-NEXT:    movq %xmm0, %r8
-; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
-; SSE41-NEXT:    movq %xmm1, %rcx
-; SSE41-NEXT:    pextrq $1, %xmm0, %rax
-; SSE41-NEXT:    pextrq $1, %xmm1, %rdx
-; SSE41-NEXT:    xorl %esi, %esi
-; SSE41-NEXT:    mulq %rdx
-; SSE41-NEXT:    movq %rax, %r9
-; SSE41-NEXT:    movq $-1, %r10
-; SSE41-NEXT:    movl $0, %eax
-; SSE41-NEXT:    cmovoq %r10, %rax
-; SSE41-NEXT:    movq %rax, %xmm0
-; SSE41-NEXT:    movq %r8, %rax
-; SSE41-NEXT:    mulq %rcx
-; SSE41-NEXT:    cmovoq %r10, %rsi
-; SSE41-NEXT:    movq %rsi, %xmm1
-; SSE41-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
-; SSE41-NEXT:    movq %r9, %xmm0
-; SSE41-NEXT:    movq %rax, %xmm3
-; SSE41-NEXT:    punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm0[0]
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,2,2,3]
-; SSE41-NEXT:    psrlq $32, %xmm3
-; SSE41-NEXT:    pcmpeqq %xmm2, %xmm3
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE41-NEXT:    pxor %xmm3, %xmm0
-; SSE41-NEXT:    por %xmm1, %xmm0
-; SSE41-NEXT:    movq %xmm4, (%rdi)
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; SSE41-NEXT:    pmuludq %xmm2, %xmm3
+; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    pmuludq %xmm1, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3],xmm2[4,5],xmm3[6,7]
+; SSE41-NEXT:    pxor %xmm3, %xmm3
+; SSE41-NEXT:    pcmpeqd %xmm2, %xmm3
+; SSE41-NEXT:    pcmpeqd %xmm2, %xmm2
+; SSE41-NEXT:    pxor %xmm3, %xmm2
+; SSE41-NEXT:    pmulld %xmm1, %xmm0
+; SSE41-NEXT:    movq %xmm0, (%rdi)
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: umulo_v2i32:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
-; AVX1-NEXT:    vmovq %xmm0, %r8
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
-; AVX1-NEXT:    vmovq %xmm1, %rcx
-; AVX1-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX1-NEXT:    vpextrq $1, %xmm1, %rdx
-; AVX1-NEXT:    xorl %esi, %esi
-; AVX1-NEXT:    mulq %rdx
-; AVX1-NEXT:    movq %rax, %r9
-; AVX1-NEXT:    movq $-1, %r10
-; AVX1-NEXT:    movl $0, %eax
-; AVX1-NEXT:    cmovoq %r10, %rax
-; AVX1-NEXT:    vmovq %rax, %xmm0
-; AVX1-NEXT:    movq %r8, %rax
-; AVX1-NEXT:    mulq %rcx
-; AVX1-NEXT:    cmovoq %r10, %rsi
-; AVX1-NEXT:    vmovq %rsi, %xmm1
-; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX1-NEXT:    vmovq %r9, %xmm1
-; AVX1-NEXT:    vmovq %rax, %xmm3
-; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm3[0],xmm1[0]
-; AVX1-NEXT:    vpsrlq $32, %xmm1, %xmm3
-; AVX1-NEXT:    vpcmpeqq %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; AVX1-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vpmuludq %xmm1, %xmm0, %xmm3
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
+; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
 ; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpor %xmm0, %xmm2, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; AVX1-NEXT:    vmovq %xmm1, (%rdi)
+; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vmovq %xmm0, (%rdi)
+; AVX1-NEXT:    vmovdqa %xmm2, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: umulo_v2i32:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
-; AVX2-NEXT:    vmovq %xmm0, %r8
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; AVX2-NEXT:    vmovq %xmm1, %rcx
-; AVX2-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX2-NEXT:    vpextrq $1, %xmm1, %rdx
-; AVX2-NEXT:    xorl %esi, %esi
-; AVX2-NEXT:    mulq %rdx
-; AVX2-NEXT:    movq %rax, %r9
-; AVX2-NEXT:    movq $-1, %r10
-; AVX2-NEXT:    movl $0, %eax
-; AVX2-NEXT:    cmovoq %r10, %rax
-; AVX2-NEXT:    vmovq %rax, %xmm0
-; AVX2-NEXT:    movq %r8, %rax
-; AVX2-NEXT:    mulq %rcx
-; AVX2-NEXT:    cmovoq %r10, %rsi
-; AVX2-NEXT:    vmovq %rsi, %xmm1
-; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX2-NEXT:    vmovq %r9, %xmm1
-; AVX2-NEXT:    vmovq %rax, %xmm3
-; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm3[0],xmm1[0]
-; AVX2-NEXT:    vpsrlq $32, %xmm1, %xmm3
-; AVX2-NEXT:    vpcmpeqq %xmm2, %xmm3, %xmm2
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; AVX2-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2
+; AVX2-NEXT:    vpmuludq %xmm1, %xmm0, %xmm3
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; AVX2-NEXT:    vpblendd {{.*#+}} xmm2 = xmm3[0],xmm2[1],xmm3[2],xmm2[3]
+; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
 ; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
 ; AVX2-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX2-NEXT:    vpor %xmm0, %xmm2, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; AVX2-NEXT:    vmovq %xmm1, (%rdi)
+; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vmovq %xmm0, (%rdi)
+; AVX2-NEXT:    vmovdqa %xmm2, %xmm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: umulo_v2i32:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
-; AVX512-NEXT:    vmovq %xmm0, %rcx
-; AVX512-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; AVX512-NEXT:    vmovq %xmm1, %rsi
-; AVX512-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX512-NEXT:    vpextrq $1, %xmm1, %rdx
-; AVX512-NEXT:    mulq %rdx
-; AVX512-NEXT:    seto %r8b
-; AVX512-NEXT:    vmovq %rax, %xmm0
-; AVX512-NEXT:    movq %rcx, %rax
-; AVX512-NEXT:    mulq %rsi
-; AVX512-NEXT:    vmovq %rax, %xmm1
-; AVX512-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX512-NEXT:    vpsrlq $32, %xmm0, %xmm1
-; AVX512-NEXT:    vptestmq %xmm1, %xmm1, %k0
-; AVX512-NEXT:    kmovd %r8d, %k1
-; AVX512-NEXT:    kshiftlw $1, %k1, %k1
-; AVX512-NEXT:    seto %al
-; AVX512-NEXT:    andl $1, %eax
-; AVX512-NEXT:    kmovw %eax, %k2
-; AVX512-NEXT:    korw %k1, %k2, %k1
-; AVX512-NEXT:    korw %k1, %k0, %k1
-; AVX512-NEXT:    vpmovqd %xmm0, (%rdi)
+; AVX512-NEXT:    vpmuludq %xmm1, %xmm0, %xmm2
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
+; AVX512-NEXT:    vpmuludq %xmm3, %xmm4, %xmm3
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm4 = [1,5,3,7]
+; AVX512-NEXT:    vpermi2d %xmm3, %xmm2, %xmm4
+; AVX512-NEXT:    vptestmd %xmm4, %xmm4, %k1
+; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm1
 ; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX512-NEXT:    vmovdqa64 %xmm0, %xmm0 {%k1} {z}
+; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
+; AVX512-NEXT:    vmovq %xmm1, (%rdi)
 ; AVX512-NEXT:    retq
   %t = call {<2 x i32>, <2 x i1>} @llvm.umul.with.overflow.v2i32(<2 x i32> %a0, <2 x i32> %a1)
   %val = extractvalue {<2 x i32>, <2 x i1>} %t, 0
@@ -279,12 +179,10 @@ define <3 x i32> @umulo_v3i32(<3 x i32> %a0, <3 x i32> %a1, <3 x i32>* %p2) noun
 ; SSE2-NEXT:    pcmpeqd %xmm3, %xmm2
 ; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
 ; SSE2-NEXT:    pxor %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[0,2,2,3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
+; SSE2-NEXT:    movd %xmm2, 8(%rdi)
 ; SSE2-NEXT:    movq %xmm0, (%rdi)
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; SSE2-NEXT:    movd %xmm0, 8(%rdi)
 ; SSE2-NEXT:    movdqa %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
@@ -301,12 +199,10 @@ define <3 x i32> @umulo_v3i32(<3 x i32> %a0, <3 x i32> %a1, <3 x i32>* %p2) noun
 ; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm2
 ; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm1
 ; SSSE3-NEXT:    pxor %xmm2, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[0,2,2,3]
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
+; SSSE3-NEXT:    movd %xmm2, 8(%rdi)
 ; SSSE3-NEXT:    movq %xmm0, (%rdi)
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; SSSE3-NEXT:    movd %xmm0, 8(%rdi)
 ; SSSE3-NEXT:    movdqa %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
@@ -607,45 +503,45 @@ define <6 x i32> @umulo_v6i32(<6 x i32> %a0, <6 x i32> %a1, <6 x i32>* %p2) noun
 ; SSE41-LABEL: umulo_v6i32:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movq %rdi, %rax
-; SSE41-NEXT:    movd %esi, %xmm0
-; SSE41-NEXT:    pinsrd $1, %edx, %xmm0
-; SSE41-NEXT:    pinsrd $2, %ecx, %xmm0
-; SSE41-NEXT:    pinsrd $3, %r8d, %xmm0
+; SSE41-NEXT:    movd %esi, %xmm2
+; SSE41-NEXT:    pinsrd $1, %edx, %xmm2
+; SSE41-NEXT:    pinsrd $2, %ecx, %xmm2
 ; SSE41-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; SSE41-NEXT:    pinsrd $1, {{[0-9]+}}(%rsp), %xmm1
-; SSE41-NEXT:    movd %r9d, %xmm2
-; SSE41-NEXT:    pinsrd $1, {{[0-9]+}}(%rsp), %xmm2
+; SSE41-NEXT:    pinsrd $2, {{[0-9]+}}(%rsp), %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pmuludq %xmm2, %xmm1
+; SSE41-NEXT:    pinsrd $3, %r8d, %xmm2
 ; SSE41-NEXT:    movd {{.*#+}} xmm3 = mem[0],zero,zero,zero
+; SSE41-NEXT:    movd %r9d, %xmm4
+; SSE41-NEXT:    movdqa %xmm4, %xmm5
+; SSE41-NEXT:    pmuludq %xmm3, %xmm4
 ; SSE41-NEXT:    pinsrd $1, {{[0-9]+}}(%rsp), %xmm3
-; SSE41-NEXT:    pinsrd $2, {{[0-9]+}}(%rsp), %xmm3
-; SSE41-NEXT:    pinsrd $3, {{[0-9]+}}(%rsp), %xmm3
+; SSE41-NEXT:    pinsrd $1, {{[0-9]+}}(%rsp), %xmm5
+; SSE41-NEXT:    pmulld %xmm3, %xmm5
+; SSE41-NEXT:    pinsrd $3, {{[0-9]+}}(%rsp), %xmm0
 ; SSE41-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pmuludq %xmm4, %xmm5
-; SSE41-NEXT:    movdqa %xmm0, %xmm4
-; SSE41-NEXT:    pmuludq %xmm3, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE41-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1],xmm5[2,3],xmm4[4,5],xmm5[6,7]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm0[1,1,3,3]
+; SSE41-NEXT:    pmuludq %xmm3, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm6[2,3],xmm1[4,5],xmm6[6,7]
 ; SSE41-NEXT:    pxor %xmm8, %xmm8
-; SSE41-NEXT:    pcmpeqd %xmm8, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm8, %xmm1
 ; SSE41-NEXT:    pcmpeqd %xmm6, %xmm6
-; SSE41-NEXT:    pxor %xmm6, %xmm4
+; SSE41-NEXT:    pxor %xmm6, %xmm1
 ; SSE41-NEXT:    movd {{.*#+}} xmm7 = mem[0],zero,zero,zero
-; SSE41-NEXT:    movd {{.*#+}} xmm5 = mem[0],zero,zero,zero
-; SSE41-NEXT:    pmuludq %xmm7, %xmm5
-; SSE41-NEXT:    movdqa %xmm2, %xmm7
-; SSE41-NEXT:    pmuludq %xmm1, %xmm7
-; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE41-NEXT:    pblendw {{.*#+}} xmm7 = xmm7[0,1],xmm5[2,3],xmm7[4,5],xmm5[6,7]
-; SSE41-NEXT:    pcmpeqd %xmm8, %xmm7
-; SSE41-NEXT:    pxor %xmm6, %xmm7
-; SSE41-NEXT:    pmulld %xmm0, %xmm3
-; SSE41-NEXT:    pmulld %xmm1, %xmm2
-; SSE41-NEXT:    movq %xmm2, 16(%rcx)
-; SSE41-NEXT:    movdqa %xmm3, (%rcx)
-; SSE41-NEXT:    movq %xmm7, 16(%rdi)
-; SSE41-NEXT:    movdqa %xmm4, (%rdi)
+; SSE41-NEXT:    movd {{.*#+}} xmm3 = mem[0],zero,zero,zero
+; SSE41-NEXT:    pmuludq %xmm7, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE41-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1],xmm3[2,3],xmm4[4,5],xmm3[6,7]
+; SSE41-NEXT:    pcmpeqd %xmm8, %xmm4
+; SSE41-NEXT:    pxor %xmm6, %xmm4
+; SSE41-NEXT:    pmulld %xmm2, %xmm0
+; SSE41-NEXT:    movq %xmm5, 16(%rcx)
+; SSE41-NEXT:    movdqa %xmm0, (%rcx)
+; SSE41-NEXT:    movq %xmm4, 16(%rdi)
+; SSE41-NEXT:    movdqa %xmm1, (%rdi)
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: umulo_v6i32:
@@ -659,7 +555,7 @@ define <6 x i32> @umulo_v6i32(<6 x i32> %a0, <6 x i32> %a1, <6 x i32>* %p2) noun
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm5[0,1],xmm2[2,3],xmm5[4,5],xmm2[6,7]
 ; AVX1-NEXT:    vpxor %xmm8, %xmm8, %xmm8
-; AVX1-NEXT:    vpcmpeqd %xmm8, %xmm2, %xmm2
+; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm8, %xmm2
 ; AVX1-NEXT:    vpcmpeqd %xmm6, %xmm6, %xmm6
 ; AVX1-NEXT:    vpxor %xmm6, %xmm2, %xmm2
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm1[1,1,3,3]
@@ -668,7 +564,7 @@ define <6 x i32> @umulo_v6i32(<6 x i32> %a0, <6 x i32> %a1, <6 x i32>* %p2) noun
 ; AVX1-NEXT:    vpmuludq %xmm1, %xmm0, %xmm7
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm5 = xmm7[0,1],xmm5[2,3],xmm7[4,5],xmm5[6,7]
-; AVX1-NEXT:    vpcmpeqd %xmm8, %xmm5, %xmm5
+; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm8, %xmm5
 ; AVX1-NEXT:    vpxor %xmm6, %xmm5, %xmm5
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm5, %ymm2
 ; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
@@ -831,7 +727,7 @@ define <8 x i32> @umulo_v8i32(<8 x i32> %a0, <8 x i32> %a1, <8 x i32>* %p2) noun
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm5[0,1],xmm2[2,3],xmm5[4,5],xmm2[6,7]
 ; AVX1-NEXT:    vpxor %xmm8, %xmm8, %xmm8
-; AVX1-NEXT:    vpcmpeqd %xmm8, %xmm2, %xmm2
+; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm8, %xmm2
 ; AVX1-NEXT:    vpcmpeqd %xmm6, %xmm6, %xmm6
 ; AVX1-NEXT:    vpxor %xmm6, %xmm2, %xmm2
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm1[1,1,3,3]
@@ -840,7 +736,7 @@ define <8 x i32> @umulo_v8i32(<8 x i32> %a0, <8 x i32> %a1, <8 x i32>* %p2) noun
 ; AVX1-NEXT:    vpmuludq %xmm1, %xmm0, %xmm7
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm5 = xmm7[0,1],xmm5[2,3],xmm7[4,5],xmm5[6,7]
-; AVX1-NEXT:    vpcmpeqd %xmm8, %xmm5, %xmm5
+; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm8, %xmm5
 ; AVX1-NEXT:    vpxor %xmm6, %xmm5, %xmm5
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm5, %ymm2
 ; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
@@ -1079,17 +975,17 @@ define <16 x i32> @umulo_v16i32(<16 x i32> %a0, <16 x i32> %a1, <16 x i32>* %p2)
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm7 = xmm7[0,1],xmm6[2,3],xmm7[4,5],xmm6[6,7]
 ; AVX1-NEXT:    vpxor %xmm8, %xmm8, %xmm8
-; AVX1-NEXT:    vpcmpeqd %xmm8, %xmm7, %xmm7
+; AVX1-NEXT:    vpcmpeqd %xmm7, %xmm8, %xmm7
 ; AVX1-NEXT:    vpcmpeqd %xmm9, %xmm9, %xmm9
-; AVX1-NEXT:    vpxor %xmm9, %xmm7, %xmm7
+; AVX1-NEXT:    vpxor %xmm7, %xmm9, %xmm7
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm3[1,1,3,3]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm1[1,1,3,3]
 ; AVX1-NEXT:    vpmuludq %xmm6, %xmm4, %xmm4
 ; AVX1-NEXT:    vpmuludq %xmm3, %xmm1, %xmm6
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm4 = xmm6[0,1],xmm4[2,3],xmm6[4,5],xmm4[6,7]
-; AVX1-NEXT:    vpcmpeqd %xmm8, %xmm4, %xmm4
-; AVX1-NEXT:    vpxor %xmm9, %xmm4, %xmm4
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm8, %xmm4
+; AVX1-NEXT:    vpxor %xmm4, %xmm9, %xmm4
 ; AVX1-NEXT:    vpackssdw %xmm7, %xmm4, %xmm11
 ; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm6
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm6[1,1,3,3]
@@ -1099,16 +995,16 @@ define <16 x i32> @umulo_v16i32(<16 x i32> %a0, <16 x i32> %a1, <16 x i32>* %p2)
 ; AVX1-NEXT:    vpmuludq %xmm6, %xmm4, %xmm7
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm5 = xmm7[0,1],xmm5[2,3],xmm7[4,5],xmm5[6,7]
-; AVX1-NEXT:    vpcmpeqd %xmm8, %xmm5, %xmm5
-; AVX1-NEXT:    vpxor %xmm9, %xmm5, %xmm13
+; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm8, %xmm5
+; AVX1-NEXT:    vpxor %xmm5, %xmm9, %xmm13
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm2[1,1,3,3]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm0[1,1,3,3]
 ; AVX1-NEXT:    vpmuludq %xmm7, %xmm5, %xmm5
 ; AVX1-NEXT:    vpmuludq %xmm2, %xmm0, %xmm7
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm5 = xmm7[0,1],xmm5[2,3],xmm7[4,5],xmm5[6,7]
-; AVX1-NEXT:    vpcmpeqd %xmm8, %xmm5, %xmm5
-; AVX1-NEXT:    vpxor %xmm9, %xmm5, %xmm5
+; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm8, %xmm5
+; AVX1-NEXT:    vpxor %xmm5, %xmm9, %xmm5
 ; AVX1-NEXT:    vpackssdw %xmm13, %xmm5, %xmm5
 ; AVX1-NEXT:    vpacksswb %xmm11, %xmm5, %xmm5
 ; AVX1-NEXT:    vpmulld %xmm2, %xmm0, %xmm2
@@ -1509,124 +1405,124 @@ define <2 x i32> @umulo_v2i64(<2 x i64> %a0, <2 x i64> %a1, <2 x i64>* %p2) noun
 ; SSE2-LABEL: umulo_v2i64:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
-; SSE2-NEXT:    movq %xmm2, %r9
+; SSE2-NEXT:    movq %xmm2, %r8
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
-; SSE2-NEXT:    movq %xmm2, %rsi
+; SSE2-NEXT:    movq %xmm2, %r10
 ; SSE2-NEXT:    movq %xmm0, %rax
 ; SSE2-NEXT:    movq %xmm1, %rdx
 ; SSE2-NEXT:    xorl %ecx, %ecx
 ; SSE2-NEXT:    mulq %rdx
-; SSE2-NEXT:    movq %rax, %r8
-; SSE2-NEXT:    movq $-1, %r10
-; SSE2-NEXT:    movl $0, %eax
-; SSE2-NEXT:    cmovoq %r10, %rax
+; SSE2-NEXT:    movq $-1, %r9
+; SSE2-NEXT:    movl $0, %esi
+; SSE2-NEXT:    cmovoq %r9, %rsi
+; SSE2-NEXT:    movq %rax, %xmm1
+; SSE2-NEXT:    movq %r8, %rax
+; SSE2-NEXT:    mulq %r10
 ; SSE2-NEXT:    movq %rax, %xmm0
-; SSE2-NEXT:    movq %r9, %rax
-; SSE2-NEXT:    mulq %rsi
-; SSE2-NEXT:    cmovoq %r10, %rcx
-; SSE2-NEXT:    movq %rcx, %xmm1
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSE2-NEXT:    movq %r8, %xmm1
-; SSE2-NEXT:    movq %rax, %xmm2
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSE2-NEXT:    movq %rsi, %xmm0
+; SSE2-NEXT:    cmovoq %r9, %rcx
+; SSE2-NEXT:    movq %rcx, %xmm2
+; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; SSE2-NEXT:    movdqa %xmm1, (%rdi)
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: umulo_v2i64:
 ; SSSE3:       # %bb.0:
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
-; SSSE3-NEXT:    movq %xmm2, %r9
+; SSSE3-NEXT:    movq %xmm2, %r8
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
-; SSSE3-NEXT:    movq %xmm2, %rsi
+; SSSE3-NEXT:    movq %xmm2, %r10
 ; SSSE3-NEXT:    movq %xmm0, %rax
 ; SSSE3-NEXT:    movq %xmm1, %rdx
 ; SSSE3-NEXT:    xorl %ecx, %ecx
 ; SSSE3-NEXT:    mulq %rdx
-; SSSE3-NEXT:    movq %rax, %r8
-; SSSE3-NEXT:    movq $-1, %r10
-; SSSE3-NEXT:    movl $0, %eax
-; SSSE3-NEXT:    cmovoq %r10, %rax
+; SSSE3-NEXT:    movq $-1, %r9
+; SSSE3-NEXT:    movl $0, %esi
+; SSSE3-NEXT:    cmovoq %r9, %rsi
+; SSSE3-NEXT:    movq %rax, %xmm1
+; SSSE3-NEXT:    movq %r8, %rax
+; SSSE3-NEXT:    mulq %r10
 ; SSSE3-NEXT:    movq %rax, %xmm0
-; SSSE3-NEXT:    movq %r9, %rax
-; SSSE3-NEXT:    mulq %rsi
-; SSSE3-NEXT:    cmovoq %r10, %rcx
-; SSSE3-NEXT:    movq %rcx, %xmm1
-; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSSE3-NEXT:    movq %r8, %xmm1
-; SSSE3-NEXT:    movq %rax, %xmm2
-; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSSE3-NEXT:    movq %rsi, %xmm0
+; SSSE3-NEXT:    cmovoq %r9, %rcx
+; SSSE3-NEXT:    movq %rcx, %xmm2
+; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; SSSE3-NEXT:    movdqa %xmm1, (%rdi)
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: umulo_v2i64:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movq %xmm0, %rcx
-; SSE41-NEXT:    movq %xmm1, %r9
+; SSE41-NEXT:    movq %xmm0, %r10
+; SSE41-NEXT:    movq %xmm1, %r8
 ; SSE41-NEXT:    pextrq $1, %xmm0, %rax
 ; SSE41-NEXT:    pextrq $1, %xmm1, %rdx
 ; SSE41-NEXT:    xorl %esi, %esi
 ; SSE41-NEXT:    mulq %rdx
-; SSE41-NEXT:    movq %rax, %r8
-; SSE41-NEXT:    movq $-1, %r10
-; SSE41-NEXT:    movl $0, %eax
-; SSE41-NEXT:    cmovoq %r10, %rax
+; SSE41-NEXT:    movq $-1, %r9
+; SSE41-NEXT:    movl $0, %ecx
+; SSE41-NEXT:    cmovoq %r9, %rcx
+; SSE41-NEXT:    movq %rax, %xmm0
+; SSE41-NEXT:    movq %r10, %rax
+; SSE41-NEXT:    mulq %r8
 ; SSE41-NEXT:    movq %rax, %xmm1
-; SSE41-NEXT:    movq %rcx, %rax
-; SSE41-NEXT:    mulq %r9
-; SSE41-NEXT:    cmovoq %r10, %rsi
-; SSE41-NEXT:    movq %rsi, %xmm0
-; SSE41-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSE41-NEXT:    movq %r8, %xmm1
-; SSE41-NEXT:    movq %rax, %xmm2
-; SSE41-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm1[0]
-; SSE41-NEXT:    movdqa %xmm2, (%rdi)
+; SSE41-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSE41-NEXT:    movq %rcx, %xmm0
+; SSE41-NEXT:    cmovoq %r9, %rsi
+; SSE41-NEXT:    movq %rsi, %xmm2
+; SSE41-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm0[0]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
+; SSE41-NEXT:    movdqa %xmm1, (%rdi)
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: umulo_v2i64:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovq %xmm0, %rcx
-; AVX1-NEXT:    vmovq %xmm1, %r9
+; AVX1-NEXT:    vmovq %xmm0, %r10
+; AVX1-NEXT:    vmovq %xmm1, %r8
 ; AVX1-NEXT:    vpextrq $1, %xmm0, %rax
 ; AVX1-NEXT:    vpextrq $1, %xmm1, %rdx
 ; AVX1-NEXT:    xorl %esi, %esi
 ; AVX1-NEXT:    mulq %rdx
-; AVX1-NEXT:    movq %rax, %r8
-; AVX1-NEXT:    movq $-1, %r10
-; AVX1-NEXT:    movl $0, %eax
-; AVX1-NEXT:    cmovoq %r10, %rax
+; AVX1-NEXT:    movq $-1, %r9
+; AVX1-NEXT:    movl $0, %ecx
+; AVX1-NEXT:    cmovoq %r9, %rcx
 ; AVX1-NEXT:    vmovq %rax, %xmm0
-; AVX1-NEXT:    movq %rcx, %rax
-; AVX1-NEXT:    mulq %r9
-; AVX1-NEXT:    cmovoq %r10, %rsi
-; AVX1-NEXT:    vmovq %rsi, %xmm1
-; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX1-NEXT:    vmovq %r8, %xmm1
-; AVX1-NEXT:    vmovq %rax, %xmm2
-; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX1-NEXT:    movq %r10, %rax
+; AVX1-NEXT:    mulq %r8
+; AVX1-NEXT:    vmovq %rax, %xmm1
+; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; AVX1-NEXT:    vmovq %rcx, %xmm0
+; AVX1-NEXT:    cmovoq %r9, %rsi
+; AVX1-NEXT:    vmovq %rsi, %xmm2
+; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm2[0],xmm0[0]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; AVX1-NEXT:    vmovdqa %xmm1, (%rdi)
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: umulo_v2i64:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vmovq %xmm0, %rcx
-; AVX2-NEXT:    vmovq %xmm1, %r9
+; AVX2-NEXT:    vmovq %xmm0, %r10
+; AVX2-NEXT:    vmovq %xmm1, %r8
 ; AVX2-NEXT:    vpextrq $1, %xmm0, %rax
 ; AVX2-NEXT:    vpextrq $1, %xmm1, %rdx
 ; AVX2-NEXT:    xorl %esi, %esi
 ; AVX2-NEXT:    mulq %rdx
-; AVX2-NEXT:    movq %rax, %r8
-; AVX2-NEXT:    movq $-1, %r10
-; AVX2-NEXT:    movl $0, %eax
-; AVX2-NEXT:    cmovoq %r10, %rax
+; AVX2-NEXT:    movq $-1, %r9
+; AVX2-NEXT:    movl $0, %ecx
+; AVX2-NEXT:    cmovoq %r9, %rcx
 ; AVX2-NEXT:    vmovq %rax, %xmm0
-; AVX2-NEXT:    movq %rcx, %rax
-; AVX2-NEXT:    mulq %r9
-; AVX2-NEXT:    cmovoq %r10, %rsi
-; AVX2-NEXT:    vmovq %rsi, %xmm1
-; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX2-NEXT:    vmovq %r8, %xmm1
-; AVX2-NEXT:    vmovq %rax, %xmm2
-; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX2-NEXT:    movq %r10, %rax
+; AVX2-NEXT:    mulq %r8
+; AVX2-NEXT:    vmovq %rax, %xmm1
+; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; AVX2-NEXT:    vmovq %rcx, %xmm0
+; AVX2-NEXT:    cmovoq %r9, %rsi
+; AVX2-NEXT:    vmovq %rsi, %xmm2
+; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm2[0],xmm0[0]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; AVX2-NEXT:    vmovdqa %xmm1, (%rdi)
 ; AVX2-NEXT:    retq
 ;
@@ -1637,22 +1533,24 @@ define <2 x i32> @umulo_v2i64(<2 x i64> %a0, <2 x i64> %a1, <2 x i64>* %p2) noun
 ; AVX512-NEXT:    vpextrq $1, %xmm0, %rax
 ; AVX512-NEXT:    vpextrq $1, %xmm1, %rdx
 ; AVX512-NEXT:    mulq %rdx
-; AVX512-NEXT:    movq %rax, %r8
-; AVX512-NEXT:    seto %al
-; AVX512-NEXT:    kmovd %eax, %k0
-; AVX512-NEXT:    kshiftlw $1, %k0, %k0
+; AVX512-NEXT:    seto %r8b
+; AVX512-NEXT:    vmovq %rax, %xmm0
 ; AVX512-NEXT:    movq %rcx, %rax
 ; AVX512-NEXT:    mulq %rsi
-; AVX512-NEXT:    seto %cl
-; AVX512-NEXT:    andl $1, %ecx
-; AVX512-NEXT:    kmovw %ecx, %k1
-; AVX512-NEXT:    korw %k0, %k1, %k1
-; AVX512-NEXT:    vmovq %r8, %xmm0
 ; AVX512-NEXT:    vmovq %rax, %xmm1
-; AVX512-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX512-NEXT:    vmovdqa %xmm0, (%rdi)
+; AVX512-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; AVX512-NEXT:    seto %al
+; AVX512-NEXT:    movw $-3, %cx
+; AVX512-NEXT:    kmovd %ecx, %k0
+; AVX512-NEXT:    kmovd %eax, %k1
+; AVX512-NEXT:    kandw %k0, %k1, %k0
+; AVX512-NEXT:    kmovd %r8d, %k1
+; AVX512-NEXT:    kshiftlw $15, %k1, %k1
+; AVX512-NEXT:    kshiftrw $14, %k1, %k1
+; AVX512-NEXT:    korw %k1, %k0, %k1
 ; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX512-NEXT:    vmovdqa64 %xmm0, %xmm0 {%k1} {z}
+; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
+; AVX512-NEXT:    vmovdqa %xmm1, (%rdi)
 ; AVX512-NEXT:    retq
   %t = call {<2 x i64>, <2 x i1>} @llvm.umul.with.overflow.v2i64(<2 x i64> %a0, <2 x i64> %a1)
   %val = extractvalue {<2 x i64>, <2 x i1>} %t, 0
@@ -1665,90 +1563,90 @@ define <2 x i32> @umulo_v2i64(<2 x i64> %a0, <2 x i64> %a1, <2 x i64>* %p2) noun
 define <4 x i32> @umulo_v4i24(<4 x i24> %a0, <4 x i24> %a1, <4 x i24>* %p2) nounwind {
 ; SSE2-LABEL: umulo_v4i24:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,0,255,255,255,0,255,255,255,0,255,255,255,0]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pmuludq %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,3,2,3]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pmuludq %xmm4, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,3,2,3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,0,255,255,255,0,255,255,255,0,255,255,255,0]
+; SSE2-NEXT:    pand %xmm0, %xmm1
+; SSE2-NEXT:    pand %xmm0, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3]
+; SSE2-NEXT:    pmuludq %xmm1, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,3,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pmuludq %xmm0, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,3,2,3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1]
 ; SSE2-NEXT:    pxor %xmm4, %xmm4
 ; SSE2-NEXT:    pcmpeqd %xmm4, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,2,2,3]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[0,2,2,3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[3,1,2,3]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[2,3,0,1]
-; SSE2-NEXT:    psrld $24, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm4
-; SSE2-NEXT:    pxor %xmm4, %xmm3
-; SSE2-NEXT:    pxor %xmm4, %xmm1
-; SSE2-NEXT:    por %xmm3, %xmm1
-; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm5
+; SSE2-NEXT:    pxor %xmm5, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[0,2,2,3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm6[0],xmm0[1],xmm6[1]
+; SSE2-NEXT:    psrld $24, %xmm0
+; SSE2-NEXT:    pcmpeqd %xmm4, %xmm0
+; SSE2-NEXT:    pxor %xmm5, %xmm0
+; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    movd %xmm2, %eax
 ; SSE2-NEXT:    movw %ax, (%rdi)
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
 ; SSE2-NEXT:    movd %xmm2, %ecx
-; SSE2-NEXT:    movw %cx, 3(%rdi)
+; SSE2-NEXT:    movw %cx, 6(%rdi)
+; SSE2-NEXT:    movd %xmm1, %edx
+; SSE2-NEXT:    movw %dx, 3(%rdi)
 ; SSE2-NEXT:    shrl $16, %eax
 ; SSE2-NEXT:    movb %al, 2(%rdi)
-; SSE2-NEXT:    shrl $16, %ecx
-; SSE2-NEXT:    movb %cl, 5(%rdi)
-; SSE2-NEXT:    movd %xmm5, %eax
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
+; SSE2-NEXT:    movd %xmm1, %eax
 ; SSE2-NEXT:    movw %ax, 9(%rdi)
-; SSE2-NEXT:    movd %xmm6, %ecx
-; SSE2-NEXT:    movw %cx, 6(%rdi)
-; SSE2-NEXT:    shrl $16, %eax
-; SSE2-NEXT:    movb %al, 11(%rdi)
 ; SSE2-NEXT:    shrl $16, %ecx
 ; SSE2-NEXT:    movb %cl, 8(%rdi)
-; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    shrl $16, %edx
+; SSE2-NEXT:    movb %dl, 5(%rdi)
+; SSE2-NEXT:    shrl $16, %eax
+; SSE2-NEXT:    movb %al, 11(%rdi)
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: umulo_v4i24:
 ; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,0,255,255,255,0,255,255,255,0,255,255,255,0]
-; SSSE3-NEXT:    pand %xmm2, %xmm1
-; SSSE3-NEXT:    pand %xmm2, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    pmuludq %xmm1, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,3,2,3]
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    pmuludq %xmm4, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,3,2,3]
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; SSSE3-NEXT:    movdqa %xmm0, %xmm2
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,0,255,255,255,0,255,255,255,0,255,255,255,0]
+; SSSE3-NEXT:    pand %xmm0, %xmm1
+; SSSE3-NEXT:    pand %xmm0, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    pmuludq %xmm1, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,3,2,3]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    pmuludq %xmm0, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,3,2,3]
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1]
 ; SSSE3-NEXT:    pxor %xmm4, %xmm4
 ; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,2,2,3]
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[0,2,2,3]
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1]
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[3,1,2,3]
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[2,3,0,1]
-; SSSE3-NEXT:    psrld $24, %xmm1
-; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm1
-; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm4
-; SSSE3-NEXT:    pxor %xmm4, %xmm3
-; SSSE3-NEXT:    pxor %xmm4, %xmm1
-; SSSE3-NEXT:    por %xmm3, %xmm1
-; SSSE3-NEXT:    movd %xmm0, %eax
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm5
+; SSSE3-NEXT:    pxor %xmm5, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[0,2,2,3]
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm6[0],xmm0[1],xmm6[1]
+; SSSE3-NEXT:    psrld $24, %xmm0
+; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm0
+; SSSE3-NEXT:    pxor %xmm5, %xmm0
+; SSSE3-NEXT:    por %xmm3, %xmm0
+; SSSE3-NEXT:    movd %xmm2, %eax
 ; SSSE3-NEXT:    movw %ax, (%rdi)
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
 ; SSSE3-NEXT:    movd %xmm2, %ecx
-; SSSE3-NEXT:    movw %cx, 3(%rdi)
+; SSSE3-NEXT:    movw %cx, 6(%rdi)
+; SSSE3-NEXT:    movd %xmm1, %edx
+; SSSE3-NEXT:    movw %dx, 3(%rdi)
 ; SSSE3-NEXT:    shrl $16, %eax
 ; SSSE3-NEXT:    movb %al, 2(%rdi)
-; SSSE3-NEXT:    shrl $16, %ecx
-; SSSE3-NEXT:    movb %cl, 5(%rdi)
-; SSSE3-NEXT:    movd %xmm5, %eax
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
+; SSSE3-NEXT:    movd %xmm1, %eax
 ; SSSE3-NEXT:    movw %ax, 9(%rdi)
-; SSSE3-NEXT:    movd %xmm6, %ecx
-; SSSE3-NEXT:    movw %cx, 6(%rdi)
-; SSSE3-NEXT:    shrl $16, %eax
-; SSSE3-NEXT:    movb %al, 11(%rdi)
 ; SSSE3-NEXT:    shrl $16, %ecx
 ; SSSE3-NEXT:    movb %cl, 8(%rdi)
-; SSSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSSE3-NEXT:    shrl $16, %edx
+; SSSE3-NEXT:    movb %dl, 5(%rdi)
+; SSSE3-NEXT:    shrl $16, %eax
+; SSSE3-NEXT:    movb %al, 11(%rdi)
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: umulo_v4i24:
@@ -2083,8 +1981,10 @@ define <4 x i32> @umulo_v4i1(<4 x i1> %a0, <4 x i1> %a1, <4 x i1>* %p2) nounwind
 ; AVX512-NEXT:    setne %cl
 ; AVX512-NEXT:    orb %al, %cl
 ; AVX512-NEXT:    setne %al
+; AVX512-NEXT:    kmovd %eax, %k1
+; AVX512-NEXT:    movw $-3, %ax
 ; AVX512-NEXT:    kmovd %eax, %k0
-; AVX512-NEXT:    kshiftrw $1, %k0, %k1
+; AVX512-NEXT:    kandw %k0, %k1, %k1
 ; AVX512-NEXT:    movl %edx, %eax
 ; AVX512-NEXT:    mulb %sil
 ; AVX512-NEXT:    movl %eax, %edx
@@ -2094,11 +1994,12 @@ define <4 x i32> @umulo_v4i1(<4 x i1> %a0, <4 x i1> %a1, <4 x i1>* %p2) nounwind
 ; AVX512-NEXT:    orb %al, %cl
 ; AVX512-NEXT:    setne %al
 ; AVX512-NEXT:    kmovd %eax, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
-; AVX512-NEXT:    kshiftlw $15, %k1, %k1
-; AVX512-NEXT:    kshiftrw $14, %k1, %k1
-; AVX512-NEXT:    kxorw %k1, %k0, %k0
-; AVX512-NEXT:    kshiftrw $2, %k0, %k1
+; AVX512-NEXT:    kshiftlw $15, %k2, %k2
+; AVX512-NEXT:    kshiftrw $14, %k2, %k2
+; AVX512-NEXT:    korw %k2, %k1, %k2
+; AVX512-NEXT:    movw $-5, %ax
+; AVX512-NEXT:    kmovd %eax, %k1
+; AVX512-NEXT:    kandw %k1, %k2, %k2
 ; AVX512-NEXT:    movl %r11d, %eax
 ; AVX512-NEXT:    mulb %bl
 ; AVX512-NEXT:    movl %eax, %esi
@@ -2107,13 +2008,11 @@ define <4 x i32> @umulo_v4i1(<4 x i1> %a0, <4 x i1> %a1, <4 x i1>* %p2) nounwind
 ; AVX512-NEXT:    setne %cl
 ; AVX512-NEXT:    orb %al, %cl
 ; AVX512-NEXT:    setne %al
-; AVX512-NEXT:    kmovd %eax, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
-; AVX512-NEXT:    kshiftlw $15, %k1, %k1
-; AVX512-NEXT:    kshiftrw $13, %k1, %k1
-; AVX512-NEXT:    kxorw %k1, %k0, %k0
-; AVX512-NEXT:    kshiftlw $13, %k0, %k0
-; AVX512-NEXT:    kshiftrw $13, %k0, %k0
+; AVX512-NEXT:    kmovd %eax, %k3
+; AVX512-NEXT:    kshiftlw $2, %k3, %k3
+; AVX512-NEXT:    korw %k3, %k2, %k2
+; AVX512-NEXT:    kshiftlw $13, %k2, %k2
+; AVX512-NEXT:    kshiftrw $13, %k2, %k2
 ; AVX512-NEXT:    movl %r9d, %eax
 ; AVX512-NEXT:    mulb %r10b
 ; AVX512-NEXT:    # kill: def $al killed $al def $eax
@@ -2122,30 +2021,29 @@ define <4 x i32> @umulo_v4i1(<4 x i1> %a0, <4 x i1> %a1, <4 x i1>* %p2) nounwind
 ; AVX512-NEXT:    setne %bl
 ; AVX512-NEXT:    orb %cl, %bl
 ; AVX512-NEXT:    setne %cl
-; AVX512-NEXT:    kmovd %ecx, %k1
-; AVX512-NEXT:    kshiftlw $3, %k1, %k1
-; AVX512-NEXT:    korw %k1, %k0, %k1
+; AVX512-NEXT:    kmovd %ecx, %k3
+; AVX512-NEXT:    kshiftlw $3, %k3, %k3
+; AVX512-NEXT:    korw %k3, %k2, %k2
 ; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
-; AVX512-NEXT:    kmovd %r8d, %k0
-; AVX512-NEXT:    kshiftrw $1, %k0, %k1
+; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k2} {z}
+; AVX512-NEXT:    kmovd %r8d, %k2
+; AVX512-NEXT:    kandw %k0, %k2, %k0
 ; AVX512-NEXT:    kmovd %edx, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
-; AVX512-NEXT:    kshiftlw $15, %k1, %k1
-; AVX512-NEXT:    kshiftrw $14, %k1, %k1
-; AVX512-NEXT:    kxorw %k1, %k0, %k0
-; AVX512-NEXT:    kshiftrw $2, %k0, %k1
-; AVX512-NEXT:    kmovd %esi, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
+; AVX512-NEXT:    kshiftlw $15, %k2, %k2
+; AVX512-NEXT:    kshiftrw $14, %k2, %k2
+; AVX512-NEXT:    korw %k2, %k0, %k0
+; AVX512-NEXT:    kandw %k1, %k0, %k0
+; AVX512-NEXT:    kmovd %esi, %k1
 ; AVX512-NEXT:    kshiftlw $15, %k1, %k1
 ; AVX512-NEXT:    kshiftrw $13, %k1, %k1
-; AVX512-NEXT:    kxorw %k1, %k0, %k0
-; AVX512-NEXT:    kshiftrw $3, %k0, %k1
-; AVX512-NEXT:    kmovd %eax, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
+; AVX512-NEXT:    korw %k1, %k0, %k0
+; AVX512-NEXT:    movw $-9, %cx
+; AVX512-NEXT:    kmovd %ecx, %k1
+; AVX512-NEXT:    kandw %k1, %k0, %k0
+; AVX512-NEXT:    kmovd %eax, %k1
 ; AVX512-NEXT:    kshiftlw $15, %k1, %k1
 ; AVX512-NEXT:    kshiftrw $12, %k1, %k1
-; AVX512-NEXT:    kxorw %k1, %k0, %k0
+; AVX512-NEXT:    korw %k1, %k0, %k0
 ; AVX512-NEXT:    kmovd %k0, %eax
 ; AVX512-NEXT:    movb %al, (%rdi)
 ; AVX512-NEXT:    popq %rbx
@@ -2167,66 +2065,67 @@ define <2 x i32> @umulo_v2i128(<2 x i128> %a0, <2 x i128> %a1, <2 x i128>* %p2)
 ; SSE2-NEXT:    pushq %r13
 ; SSE2-NEXT:    pushq %r12
 ; SSE2-NEXT:    pushq %rbx
-; SSE2-NEXT:    movq %rcx, %rax
-; SSE2-NEXT:    movq %rdx, %r12
-; SSE2-NEXT:    movq %rdi, %r11
+; SSE2-NEXT:    movq %r9, %r10
+; SSE2-NEXT:    movq %rcx, %r12
+; SSE2-NEXT:    movq %rdx, %r11
+; SSE2-NEXT:    movq %rsi, %rax
 ; SSE2-NEXT:    movq {{[0-9]+}}(%rsp), %r14
 ; SSE2-NEXT:    movq {{[0-9]+}}(%rsp), %r15
-; SSE2-NEXT:    movq {{[0-9]+}}(%rsp), %r10
+; SSE2-NEXT:    movq {{[0-9]+}}(%rsp), %r9
 ; SSE2-NEXT:    testq %r10, %r10
-; SSE2-NEXT:    setne %dl
-; SSE2-NEXT:    testq %rcx, %rcx
-; SSE2-NEXT:    setne %r13b
-; SSE2-NEXT:    andb %dl, %r13b
-; SSE2-NEXT:    mulq %r15
-; SSE2-NEXT:    movq %rax, %rdi
-; SSE2-NEXT:    seto %bpl
-; SSE2-NEXT:    movq %r10, %rax
-; SSE2-NEXT:    mulq %r12
-; SSE2-NEXT:    movq %rax, %rbx
-; SSE2-NEXT:    seto %cl
-; SSE2-NEXT:    orb %bpl, %cl
-; SSE2-NEXT:    addq %rdi, %rbx
-; SSE2-NEXT:    movq %r12, %rax
-; SSE2-NEXT:    mulq %r15
-; SSE2-NEXT:    movq %rax, %r10
-; SSE2-NEXT:    movq %rdx, %r15
-; SSE2-NEXT:    addq %rbx, %r15
-; SSE2-NEXT:    setb %al
-; SSE2-NEXT:    orb %cl, %al
-; SSE2-NEXT:    orb %r13b, %al
-; SSE2-NEXT:    movzbl %al, %ebp
-; SSE2-NEXT:    testq %r9, %r9
-; SSE2-NEXT:    setne %al
+; SSE2-NEXT:    setne %cl
 ; SSE2-NEXT:    testq %rsi, %rsi
 ; SSE2-NEXT:    setne %r13b
-; SSE2-NEXT:    andb %al, %r13b
-; SSE2-NEXT:    movq %rsi, %rax
+; SSE2-NEXT:    andb %cl, %r13b
 ; SSE2-NEXT:    mulq %r8
 ; SSE2-NEXT:    movq %rax, %rsi
-; SSE2-NEXT:    seto %r12b
-; SSE2-NEXT:    movq %r9, %rax
-; SSE2-NEXT:    mulq %r11
-; SSE2-NEXT:    movq %rax, %rdi
+; SSE2-NEXT:    seto %bpl
+; SSE2-NEXT:    movq %r10, %rax
+; SSE2-NEXT:    mulq %rdi
+; SSE2-NEXT:    movq %rax, %rcx
 ; SSE2-NEXT:    seto %bl
-; SSE2-NEXT:    orb %r12b, %bl
-; SSE2-NEXT:    addq %rsi, %rdi
-; SSE2-NEXT:    movq %r11, %rax
+; SSE2-NEXT:    orb %bpl, %bl
+; SSE2-NEXT:    addq %rsi, %rcx
+; SSE2-NEXT:    movq %rdi, %rax
 ; SSE2-NEXT:    mulq %r8
-; SSE2-NEXT:    addq %rdi, %rdx
+; SSE2-NEXT:    movq %rax, %rdi
+; SSE2-NEXT:    movq %rdx, %rsi
+; SSE2-NEXT:    addq %rcx, %rsi
 ; SSE2-NEXT:    setb %cl
 ; SSE2-NEXT:    orb %bl, %cl
 ; SSE2-NEXT:    orb %r13b, %cl
+; SSE2-NEXT:    testq %r9, %r9
+; SSE2-NEXT:    setne %al
+; SSE2-NEXT:    testq %r12, %r12
+; SSE2-NEXT:    setne %r8b
+; SSE2-NEXT:    andb %al, %r8b
+; SSE2-NEXT:    movq %r12, %rax
+; SSE2-NEXT:    mulq %r15
+; SSE2-NEXT:    movq %rax, %rbp
+; SSE2-NEXT:    seto %r10b
+; SSE2-NEXT:    movq %r9, %rax
+; SSE2-NEXT:    mulq %r11
+; SSE2-NEXT:    movq %rax, %rbx
+; SSE2-NEXT:    seto %r9b
+; SSE2-NEXT:    orb %r10b, %r9b
+; SSE2-NEXT:    addq %rbp, %rbx
+; SSE2-NEXT:    movq %r11, %rax
+; SSE2-NEXT:    mulq %r15
+; SSE2-NEXT:    addq %rbx, %rdx
+; SSE2-NEXT:    setb %bl
+; SSE2-NEXT:    orb %r9b, %bl
+; SSE2-NEXT:    orb %r8b, %bl
+; SSE2-NEXT:    movzbl %bl, %ebp
+; SSE2-NEXT:    negl %ebp
+; SSE2-NEXT:    movd %ebp, %xmm1
 ; SSE2-NEXT:    movzbl %cl, %ecx
+; SSE2-NEXT:    negl %ecx
 ; SSE2-NEXT:    movd %ecx, %xmm0
-; SSE2-NEXT:    pinsrw $4, %ebp, %xmm0
-; SSE2-NEXT:    movq %r10, 16(%r14)
-; SSE2-NEXT:    movq %rax, (%r14)
-; SSE2-NEXT:    movq %r15, 24(%r14)
-; SSE2-NEXT:    movq %rdx, 8(%r14)
-; SSE2-NEXT:    psllq $63, %xmm0
-; SSE2-NEXT:    psrad $31, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT:    movq %rax, 16(%r14)
+; SSE2-NEXT:    movq %rdi, (%r14)
+; SSE2-NEXT:    movq %rdx, 24(%r14)
+; SSE2-NEXT:    movq %rsi, 8(%r14)
 ; SSE2-NEXT:    popq %rbx
 ; SSE2-NEXT:    popq %r12
 ; SSE2-NEXT:    popq %r13
@@ -2243,66 +2142,67 @@ define <2 x i32> @umulo_v2i128(<2 x i128> %a0, <2 x i128> %a1, <2 x i128>* %p2)
 ; SSSE3-NEXT:    pushq %r13
 ; SSSE3-NEXT:    pushq %r12
 ; SSSE3-NEXT:    pushq %rbx
-; SSSE3-NEXT:    movq %rcx, %rax
-; SSSE3-NEXT:    movq %rdx, %r12
-; SSSE3-NEXT:    movq %rdi, %r11
+; SSSE3-NEXT:    movq %r9, %r10
+; SSSE3-NEXT:    movq %rcx, %r12
+; SSSE3-NEXT:    movq %rdx, %r11
+; SSSE3-NEXT:    movq %rsi, %rax
 ; SSSE3-NEXT:    movq {{[0-9]+}}(%rsp), %r14
 ; SSSE3-NEXT:    movq {{[0-9]+}}(%rsp), %r15
-; SSSE3-NEXT:    movq {{[0-9]+}}(%rsp), %r10
+; SSSE3-NEXT:    movq {{[0-9]+}}(%rsp), %r9
 ; SSSE3-NEXT:    testq %r10, %r10
-; SSSE3-NEXT:    setne %dl
-; SSSE3-NEXT:    testq %rcx, %rcx
-; SSSE3-NEXT:    setne %r13b
-; SSSE3-NEXT:    andb %dl, %r13b
-; SSSE3-NEXT:    mulq %r15
-; SSSE3-NEXT:    movq %rax, %rdi
-; SSSE3-NEXT:    seto %bpl
-; SSSE3-NEXT:    movq %r10, %rax
-; SSSE3-NEXT:    mulq %r12
-; SSSE3-NEXT:    movq %rax, %rbx
-; SSSE3-NEXT:    seto %cl
-; SSSE3-NEXT:    orb %bpl, %cl
-; SSSE3-NEXT:    addq %rdi, %rbx
-; SSSE3-NEXT:    movq %r12, %rax
-; SSSE3-NEXT:    mulq %r15
-; SSSE3-NEXT:    movq %rax, %r10
-; SSSE3-NEXT:    movq %rdx, %r15
-; SSSE3-NEXT:    addq %rbx, %r15
-; SSSE3-NEXT:    setb %al
-; SSSE3-NEXT:    orb %cl, %al
-; SSSE3-NEXT:    orb %r13b, %al
-; SSSE3-NEXT:    movzbl %al, %ebp
-; SSSE3-NEXT:    testq %r9, %r9
-; SSSE3-NEXT:    setne %al
+; SSSE3-NEXT:    setne %cl
 ; SSSE3-NEXT:    testq %rsi, %rsi
 ; SSSE3-NEXT:    setne %r13b
-; SSSE3-NEXT:    andb %al, %r13b
-; SSSE3-NEXT:    movq %rsi, %rax
+; SSSE3-NEXT:    andb %cl, %r13b
 ; SSSE3-NEXT:    mulq %r8
 ; SSSE3-NEXT:    movq %rax, %rsi
-; SSSE3-NEXT:    seto %r12b
-; SSSE3-NEXT:    movq %r9, %rax
-; SSSE3-NEXT:    mulq %r11
-; SSSE3-NEXT:    movq %rax, %rdi
+; SSSE3-NEXT:    seto %bpl
+; SSSE3-NEXT:    movq %r10, %rax
+; SSSE3-NEXT:    mulq %rdi
+; SSSE3-NEXT:    movq %rax, %rcx
 ; SSSE3-NEXT:    seto %bl
-; SSSE3-NEXT:    orb %r12b, %bl
-; SSSE3-NEXT:    addq %rsi, %rdi
-; SSSE3-NEXT:    movq %r11, %rax
+; SSSE3-NEXT:    orb %bpl, %bl
+; SSSE3-NEXT:    addq %rsi, %rcx
+; SSSE3-NEXT:    movq %rdi, %rax
 ; SSSE3-NEXT:    mulq %r8
-; SSSE3-NEXT:    addq %rdi, %rdx
+; SSSE3-NEXT:    movq %rax, %rdi
+; SSSE3-NEXT:    movq %rdx, %rsi
+; SSSE3-NEXT:    addq %rcx, %rsi
 ; SSSE3-NEXT:    setb %cl
 ; SSSE3-NEXT:    orb %bl, %cl
 ; SSSE3-NEXT:    orb %r13b, %cl
+; SSSE3-NEXT:    testq %r9, %r9
+; SSSE3-NEXT:    setne %al
+; SSSE3-NEXT:    testq %r12, %r12
+; SSSE3-NEXT:    setne %r8b
+; SSSE3-NEXT:    andb %al, %r8b
+; SSSE3-NEXT:    movq %r12, %rax
+; SSSE3-NEXT:    mulq %r15
+; SSSE3-NEXT:    movq %rax, %rbp
+; SSSE3-NEXT:    seto %r10b
+; SSSE3-NEXT:    movq %r9, %rax
+; SSSE3-NEXT:    mulq %r11
+; SSSE3-NEXT:    movq %rax, %rbx
+; SSSE3-NEXT:    seto %r9b
+; SSSE3-NEXT:    orb %r10b, %r9b
+; SSSE3-NEXT:    addq %rbp, %rbx
+; SSSE3-NEXT:    movq %r11, %rax
+; SSSE3-NEXT:    mulq %r15
+; SSSE3-NEXT:    addq %rbx, %rdx
+; SSSE3-NEXT:    setb %bl
+; SSSE3-NEXT:    orb %r9b, %bl
+; SSSE3-NEXT:    orb %r8b, %bl
+; SSSE3-NEXT:    movzbl %bl, %ebp
+; SSSE3-NEXT:    negl %ebp
+; SSSE3-NEXT:    movd %ebp, %xmm1
 ; SSSE3-NEXT:    movzbl %cl, %ecx
+; SSSE3-NEXT:    negl %ecx
 ; SSSE3-NEXT:    movd %ecx, %xmm0
-; SSSE3-NEXT:    pinsrw $4, %ebp, %xmm0
-; SSSE3-NEXT:    movq %r10, 16(%r14)
-; SSSE3-NEXT:    movq %rax, (%r14)
-; SSSE3-NEXT:    movq %r15, 24(%r14)
-; SSSE3-NEXT:    movq %rdx, 8(%r14)
-; SSSE3-NEXT:    psllq $63, %xmm0
-; SSSE3-NEXT:    psrad $31, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSSE3-NEXT:    movq %rax, 16(%r14)
+; SSSE3-NEXT:    movq %rdi, (%r14)
+; SSSE3-NEXT:    movq %rdx, 24(%r14)
+; SSSE3-NEXT:    movq %rsi, 8(%r14)
 ; SSSE3-NEXT:    popq %rbx
 ; SSSE3-NEXT:    popq %r12
 ; SSSE3-NEXT:    popq %r13
@@ -2319,66 +2219,66 @@ define <2 x i32> @umulo_v2i128(<2 x i128> %a0, <2 x i128> %a1, <2 x i128>* %p2)
 ; SSE41-NEXT:    pushq %r13
 ; SSE41-NEXT:    pushq %r12
 ; SSE41-NEXT:    pushq %rbx
-; SSE41-NEXT:    movq %rcx, %rax
-; SSE41-NEXT:    movq %rdx, %r12
-; SSE41-NEXT:    movq %rdi, %r11
+; SSE41-NEXT:    movq %r9, %r10
+; SSE41-NEXT:    movq %rcx, %r12
+; SSE41-NEXT:    movq %rdx, %r11
+; SSE41-NEXT:    movq %rsi, %rax
 ; SSE41-NEXT:    movq {{[0-9]+}}(%rsp), %r14
 ; SSE41-NEXT:    movq {{[0-9]+}}(%rsp), %r15
-; SSE41-NEXT:    movq {{[0-9]+}}(%rsp), %r10
+; SSE41-NEXT:    movq {{[0-9]+}}(%rsp), %r9
 ; SSE41-NEXT:    testq %r10, %r10
-; SSE41-NEXT:    setne %dl
-; SSE41-NEXT:    testq %rcx, %rcx
-; SSE41-NEXT:    setne %r13b
-; SSE41-NEXT:    andb %dl, %r13b
-; SSE41-NEXT:    mulq %r15
-; SSE41-NEXT:    movq %rax, %rdi
-; SSE41-NEXT:    seto %bpl
-; SSE41-NEXT:    movq %r10, %rax
-; SSE41-NEXT:    mulq %r12
-; SSE41-NEXT:    movq %rax, %rbx
-; SSE41-NEXT:    seto %cl
-; SSE41-NEXT:    orb %bpl, %cl
-; SSE41-NEXT:    addq %rdi, %rbx
-; SSE41-NEXT:    movq %r12, %rax
-; SSE41-NEXT:    mulq %r15
-; SSE41-NEXT:    movq %rax, %r10
-; SSE41-NEXT:    movq %rdx, %r15
-; SSE41-NEXT:    addq %rbx, %r15
-; SSE41-NEXT:    setb %al
-; SSE41-NEXT:    orb %cl, %al
-; SSE41-NEXT:    orb %r13b, %al
-; SSE41-NEXT:    movzbl %al, %ebp
-; SSE41-NEXT:    testq %r9, %r9
-; SSE41-NEXT:    setne %al
+; SSE41-NEXT:    setne %cl
 ; SSE41-NEXT:    testq %rsi, %rsi
 ; SSE41-NEXT:    setne %r13b
-; SSE41-NEXT:    andb %al, %r13b
-; SSE41-NEXT:    movq %rsi, %rax
+; SSE41-NEXT:    andb %cl, %r13b
 ; SSE41-NEXT:    mulq %r8
 ; SSE41-NEXT:    movq %rax, %rsi
-; SSE41-NEXT:    seto %r12b
-; SSE41-NEXT:    movq %r9, %rax
-; SSE41-NEXT:    mulq %r11
-; SSE41-NEXT:    movq %rax, %rdi
+; SSE41-NEXT:    seto %bpl
+; SSE41-NEXT:    movq %r10, %rax
+; SSE41-NEXT:    mulq %rdi
+; SSE41-NEXT:    movq %rax, %rcx
 ; SSE41-NEXT:    seto %bl
-; SSE41-NEXT:    orb %r12b, %bl
-; SSE41-NEXT:    addq %rsi, %rdi
-; SSE41-NEXT:    movq %r11, %rax
+; SSE41-NEXT:    orb %bpl, %bl
+; SSE41-NEXT:    addq %rsi, %rcx
+; SSE41-NEXT:    movq %rdi, %rax
 ; SSE41-NEXT:    mulq %r8
-; SSE41-NEXT:    addq %rdi, %rdx
+; SSE41-NEXT:    movq %rax, %rdi
+; SSE41-NEXT:    movq %rdx, %rsi
+; SSE41-NEXT:    addq %rcx, %rsi
 ; SSE41-NEXT:    setb %cl
 ; SSE41-NEXT:    orb %bl, %cl
 ; SSE41-NEXT:    orb %r13b, %cl
+; SSE41-NEXT:    testq %r9, %r9
+; SSE41-NEXT:    setne %al
+; SSE41-NEXT:    testq %r12, %r12
+; SSE41-NEXT:    setne %r8b
+; SSE41-NEXT:    andb %al, %r8b
+; SSE41-NEXT:    movq %r12, %rax
+; SSE41-NEXT:    mulq %r15
+; SSE41-NEXT:    movq %rax, %rbp
+; SSE41-NEXT:    seto %r10b
+; SSE41-NEXT:    movq %r9, %rax
+; SSE41-NEXT:    mulq %r11
+; SSE41-NEXT:    movq %rax, %rbx
+; SSE41-NEXT:    seto %r9b
+; SSE41-NEXT:    orb %r10b, %r9b
+; SSE41-NEXT:    addq %rbp, %rbx
+; SSE41-NEXT:    movq %r11, %rax
+; SSE41-NEXT:    mulq %r15
+; SSE41-NEXT:    addq %rbx, %rdx
+; SSE41-NEXT:    setb %bl
+; SSE41-NEXT:    orb %r9b, %bl
+; SSE41-NEXT:    orb %r8b, %bl
+; SSE41-NEXT:    movzbl %bl, %ebp
+; SSE41-NEXT:    negl %ebp
 ; SSE41-NEXT:    movzbl %cl, %ecx
+; SSE41-NEXT:    negl %ecx
 ; SSE41-NEXT:    movd %ecx, %xmm0
-; SSE41-NEXT:    pinsrb $8, %ebp, %xmm0
-; SSE41-NEXT:    movq %r10, 16(%r14)
-; SSE41-NEXT:    movq %rax, (%r14)
-; SSE41-NEXT:    movq %r15, 24(%r14)
-; SSE41-NEXT:    movq %rdx, 8(%r14)
-; SSE41-NEXT:    psllq $63, %xmm0
-; SSE41-NEXT:    psrad $31, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE41-NEXT:    pinsrd $1, %ebp, %xmm0
+; SSE41-NEXT:    movq %rax, 16(%r14)
+; SSE41-NEXT:    movq %rdi, (%r14)
+; SSE41-NEXT:    movq %rdx, 24(%r14)
+; SSE41-NEXT:    movq %rsi, 8(%r14)
 ; SSE41-NEXT:    popq %rbx
 ; SSE41-NEXT:    popq %r12
 ; SSE41-NEXT:    popq %r13
@@ -2395,66 +2295,66 @@ define <2 x i32> @umulo_v2i128(<2 x i128> %a0, <2 x i128> %a1, <2 x i128>* %p2)
 ; AVX1-NEXT:    pushq %r13
 ; AVX1-NEXT:    pushq %r12
 ; AVX1-NEXT:    pushq %rbx
-; AVX1-NEXT:    movq %rcx, %rax
-; AVX1-NEXT:    movq %rdx, %r12
-; AVX1-NEXT:    movq %rdi, %r11
+; AVX1-NEXT:    movq %r9, %r10
+; AVX1-NEXT:    movq %rcx, %r12
+; AVX1-NEXT:    movq %rdx, %r11
+; AVX1-NEXT:    movq %rsi, %rax
 ; AVX1-NEXT:    movq {{[0-9]+}}(%rsp), %r14
 ; AVX1-NEXT:    movq {{[0-9]+}}(%rsp), %r15
-; AVX1-NEXT:    movq {{[0-9]+}}(%rsp), %r10
+; AVX1-NEXT:    movq {{[0-9]+}}(%rsp), %r9
 ; AVX1-NEXT:    testq %r10, %r10
-; AVX1-NEXT:    setne %dl
-; AVX1-NEXT:    testq %rcx, %rcx
+; AVX1-NEXT:    setne %cl
+; AVX1-NEXT:    testq %rsi, %rsi
 ; AVX1-NEXT:    setne %r13b
-; AVX1-NEXT:    andb %dl, %r13b
-; AVX1-NEXT:    mulq %r15
-; AVX1-NEXT:    movq %rax, %rdi
+; AVX1-NEXT:    andb %cl, %r13b
+; AVX1-NEXT:    mulq %r8
+; AVX1-NEXT:    movq %rax, %rsi
 ; AVX1-NEXT:    seto %bpl
 ; AVX1-NEXT:    movq %r10, %rax
-; AVX1-NEXT:    mulq %r12
-; AVX1-NEXT:    movq %rax, %rbx
-; AVX1-NEXT:    seto %cl
-; AVX1-NEXT:    orb %bpl, %cl
-; AVX1-NEXT:    addq %rdi, %rbx
-; AVX1-NEXT:    movq %r12, %rax
-; AVX1-NEXT:    mulq %r15
-; AVX1-NEXT:    movq %rax, %r10
-; AVX1-NEXT:    movq %rdx, %r15
-; AVX1-NEXT:    addq %rbx, %r15
-; AVX1-NEXT:    setb %al
-; AVX1-NEXT:    orb %cl, %al
-; AVX1-NEXT:    orb %r13b, %al
-; AVX1-NEXT:    movzbl %al, %ebp
+; AVX1-NEXT:    mulq %rdi
+; AVX1-NEXT:    movq %rax, %rcx
+; AVX1-NEXT:    seto %bl
+; AVX1-NEXT:    orb %bpl, %bl
+; AVX1-NEXT:    addq %rsi, %rcx
+; AVX1-NEXT:    movq %rdi, %rax
+; AVX1-NEXT:    mulq %r8
+; AVX1-NEXT:    movq %rax, %rdi
+; AVX1-NEXT:    movq %rdx, %rsi
+; AVX1-NEXT:    addq %rcx, %rsi
+; AVX1-NEXT:    setb %cl
+; AVX1-NEXT:    orb %bl, %cl
+; AVX1-NEXT:    orb %r13b, %cl
 ; AVX1-NEXT:    testq %r9, %r9
 ; AVX1-NEXT:    setne %al
-; AVX1-NEXT:    testq %rsi, %rsi
-; AVX1-NEXT:    setne %r13b
-; AVX1-NEXT:    andb %al, %r13b
-; AVX1-NEXT:    movq %rsi, %rax
-; AVX1-NEXT:    mulq %r8
-; AVX1-NEXT:    movq %rax, %rsi
-; AVX1-NEXT:    seto %r12b
+; AVX1-NEXT:    testq %r12, %r12
+; AVX1-NEXT:    setne %r8b
+; AVX1-NEXT:    andb %al, %r8b
+; AVX1-NEXT:    movq %r12, %rax
+; AVX1-NEXT:    mulq %r15
+; AVX1-NEXT:    movq %rax, %rbp
+; AVX1-NEXT:    seto %r10b
 ; AVX1-NEXT:    movq %r9, %rax
 ; AVX1-NEXT:    mulq %r11
-; AVX1-NEXT:    movq %rax, %rdi
-; AVX1-NEXT:    seto %cl
-; AVX1-NEXT:    orb %r12b, %cl
-; AVX1-NEXT:    addq %rsi, %rdi
+; AVX1-NEXT:    movq %rax, %rbx
+; AVX1-NEXT:    seto %r9b
+; AVX1-NEXT:    orb %r10b, %r9b
+; AVX1-NEXT:    addq %rbp, %rbx
 ; AVX1-NEXT:    movq %r11, %rax
-; AVX1-NEXT:    mulq %r8
-; AVX1-NEXT:    addq %rdi, %rdx
+; AVX1-NEXT:    mulq %r15
+; AVX1-NEXT:    addq %rbx, %rdx
 ; AVX1-NEXT:    setb %bl
-; AVX1-NEXT:    orb %cl, %bl
-; AVX1-NEXT:    orb %r13b, %bl
-; AVX1-NEXT:    movzbl %bl, %ecx
+; AVX1-NEXT:    orb %r9b, %bl
+; AVX1-NEXT:    orb %r8b, %bl
+; AVX1-NEXT:    movzbl %bl, %ebp
+; AVX1-NEXT:    negl %ebp
+; AVX1-NEXT:    movzbl %cl, %ecx
+; AVX1-NEXT:    negl %ecx
 ; AVX1-NEXT:    vmovd %ecx, %xmm0
-; AVX1-NEXT:    vpinsrb $8, %ebp, %xmm0, %xmm0
-; AVX1-NEXT:    movq %r10, 16(%r14)
-; AVX1-NEXT:    movq %rax, (%r14)
-; AVX1-NEXT:    movq %r15, 24(%r14)
-; AVX1-NEXT:    movq %rdx, 8(%r14)
-; AVX1-NEXT:    vpsllq $63, %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vpinsrd $1, %ebp, %xmm0, %xmm0
+; AVX1-NEXT:    movq %rax, 16(%r14)
+; AVX1-NEXT:    movq %rdi, (%r14)
+; AVX1-NEXT:    movq %rdx, 24(%r14)
+; AVX1-NEXT:    movq %rsi, 8(%r14)
 ; AVX1-NEXT:    popq %rbx
 ; AVX1-NEXT:    popq %r12
 ; AVX1-NEXT:    popq %r13
@@ -2471,66 +2371,66 @@ define <2 x i32> @umulo_v2i128(<2 x i128> %a0, <2 x i128> %a1, <2 x i128>* %p2)
 ; AVX2-NEXT:    pushq %r13
 ; AVX2-NEXT:    pushq %r12
 ; AVX2-NEXT:    pushq %rbx
-; AVX2-NEXT:    movq %rcx, %rax
-; AVX2-NEXT:    movq %rdx, %r12
-; AVX2-NEXT:    movq %rdi, %r11
+; AVX2-NEXT:    movq %r9, %r10
+; AVX2-NEXT:    movq %rcx, %r12
+; AVX2-NEXT:    movq %rdx, %r11
+; AVX2-NEXT:    movq %rsi, %rax
 ; AVX2-NEXT:    movq {{[0-9]+}}(%rsp), %r14
 ; AVX2-NEXT:    movq {{[0-9]+}}(%rsp), %r15
-; AVX2-NEXT:    movq {{[0-9]+}}(%rsp), %r10
+; AVX2-NEXT:    movq {{[0-9]+}}(%rsp), %r9
 ; AVX2-NEXT:    testq %r10, %r10
-; AVX2-NEXT:    setne %dl
-; AVX2-NEXT:    testq %rcx, %rcx
+; AVX2-NEXT:    setne %cl
+; AVX2-NEXT:    testq %rsi, %rsi
 ; AVX2-NEXT:    setne %r13b
-; AVX2-NEXT:    andb %dl, %r13b
-; AVX2-NEXT:    mulq %r15
-; AVX2-NEXT:    movq %rax, %rdi
+; AVX2-NEXT:    andb %cl, %r13b
+; AVX2-NEXT:    mulq %r8
+; AVX2-NEXT:    movq %rax, %rsi
 ; AVX2-NEXT:    seto %bpl
 ; AVX2-NEXT:    movq %r10, %rax
-; AVX2-NEXT:    mulq %r12
-; AVX2-NEXT:    movq %rax, %rbx
-; AVX2-NEXT:    seto %cl
-; AVX2-NEXT:    orb %bpl, %cl
-; AVX2-NEXT:    addq %rdi, %rbx
-; AVX2-NEXT:    movq %r12, %rax
-; AVX2-NEXT:    mulq %r15
-; AVX2-NEXT:    movq %rax, %r10
-; AVX2-NEXT:    movq %rdx, %r15
-; AVX2-NEXT:    addq %rbx, %r15
-; AVX2-NEXT:    setb %al
-; AVX2-NEXT:    orb %cl, %al
-; AVX2-NEXT:    orb %r13b, %al
-; AVX2-NEXT:    movzbl %al, %ebp
+; AVX2-NEXT:    mulq %rdi
+; AVX2-NEXT:    movq %rax, %rcx
+; AVX2-NEXT:    seto %bl
+; AVX2-NEXT:    orb %bpl, %bl
+; AVX2-NEXT:    addq %rsi, %rcx
+; AVX2-NEXT:    movq %rdi, %rax
+; AVX2-NEXT:    mulq %r8
+; AVX2-NEXT:    movq %rax, %rdi
+; AVX2-NEXT:    movq %rdx, %rsi
+; AVX2-NEXT:    addq %rcx, %rsi
+; AVX2-NEXT:    setb %cl
+; AVX2-NEXT:    orb %bl, %cl
+; AVX2-NEXT:    orb %r13b, %cl
 ; AVX2-NEXT:    testq %r9, %r9
 ; AVX2-NEXT:    setne %al
-; AVX2-NEXT:    testq %rsi, %rsi
-; AVX2-NEXT:    setne %r13b
-; AVX2-NEXT:    andb %al, %r13b
-; AVX2-NEXT:    movq %rsi, %rax
-; AVX2-NEXT:    mulq %r8
-; AVX2-NEXT:    movq %rax, %rsi
-; AVX2-NEXT:    seto %r12b
+; AVX2-NEXT:    testq %r12, %r12
+; AVX2-NEXT:    setne %r8b
+; AVX2-NEXT:    andb %al, %r8b
+; AVX2-NEXT:    movq %r12, %rax
+; AVX2-NEXT:    mulq %r15
+; AVX2-NEXT:    movq %rax, %rbp
+; AVX2-NEXT:    seto %r10b
 ; AVX2-NEXT:    movq %r9, %rax
 ; AVX2-NEXT:    mulq %r11
-; AVX2-NEXT:    movq %rax, %rdi
-; AVX2-NEXT:    seto %cl
-; AVX2-NEXT:    orb %r12b, %cl
-; AVX2-NEXT:    addq %rsi, %rdi
+; AVX2-NEXT:    movq %rax, %rbx
+; AVX2-NEXT:    seto %r9b
+; AVX2-NEXT:    orb %r10b, %r9b
+; AVX2-NEXT:    addq %rbp, %rbx
 ; AVX2-NEXT:    movq %r11, %rax
-; AVX2-NEXT:    mulq %r8
-; AVX2-NEXT:    addq %rdi, %rdx
+; AVX2-NEXT:    mulq %r15
+; AVX2-NEXT:    addq %rbx, %rdx
 ; AVX2-NEXT:    setb %bl
-; AVX2-NEXT:    orb %cl, %bl
-; AVX2-NEXT:    orb %r13b, %bl
-; AVX2-NEXT:    movzbl %bl, %ecx
+; AVX2-NEXT:    orb %r9b, %bl
+; AVX2-NEXT:    orb %r8b, %bl
+; AVX2-NEXT:    movzbl %bl, %ebp
+; AVX2-NEXT:    negl %ebp
+; AVX2-NEXT:    movzbl %cl, %ecx
+; AVX2-NEXT:    negl %ecx
 ; AVX2-NEXT:    vmovd %ecx, %xmm0
-; AVX2-NEXT:    vpinsrb $8, %ebp, %xmm0, %xmm0
-; AVX2-NEXT:    movq %r10, 16(%r14)
-; AVX2-NEXT:    movq %rax, (%r14)
-; AVX2-NEXT:    movq %r15, 24(%r14)
-; AVX2-NEXT:    movq %rdx, 8(%r14)
-; AVX2-NEXT:    vpsllq $63, %xmm0, %xmm0
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm0
+; AVX2-NEXT:    vpinsrd $1, %ebp, %xmm0, %xmm0
+; AVX2-NEXT:    movq %rax, 16(%r14)
+; AVX2-NEXT:    movq %rdi, (%r14)
+; AVX2-NEXT:    movq %rdx, 24(%r14)
+; AVX2-NEXT:    movq %rsi, 8(%r14)
 ; AVX2-NEXT:    popq %rbx
 ; AVX2-NEXT:    popq %r12
 ; AVX2-NEXT:    popq %r13
@@ -2547,66 +2447,68 @@ define <2 x i32> @umulo_v2i128(<2 x i128> %a0, <2 x i128> %a1, <2 x i128>* %p2)
 ; AVX512-NEXT:    pushq %r13
 ; AVX512-NEXT:    pushq %r12
 ; AVX512-NEXT:    pushq %rbx
-; AVX512-NEXT:    movq %rcx, %rax
-; AVX512-NEXT:    movq %rdx, %r12
-; AVX512-NEXT:    movq %rdi, %r11
+; AVX512-NEXT:    movq %r9, %r10
+; AVX512-NEXT:    movq %rcx, %r9
+; AVX512-NEXT:    movq %rdx, %r11
+; AVX512-NEXT:    movq %rsi, %rax
+; AVX512-NEXT:    movq %rdi, %rsi
 ; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %r14
 ; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %r15
-; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %r10
+; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %r12
 ; AVX512-NEXT:    testq %r10, %r10
 ; AVX512-NEXT:    setne %dl
-; AVX512-NEXT:    testq %rcx, %rcx
-; AVX512-NEXT:    setne %r13b
-; AVX512-NEXT:    andb %dl, %r13b
-; AVX512-NEXT:    mulq %r15
-; AVX512-NEXT:    movq %rax, %rdi
+; AVX512-NEXT:    testq %rax, %rax
+; AVX512-NEXT:    setne %bl
+; AVX512-NEXT:    andb %dl, %bl
+; AVX512-NEXT:    mulq %r8
+; AVX512-NEXT:    movq %rax, %r13
 ; AVX512-NEXT:    seto %bpl
 ; AVX512-NEXT:    movq %r10, %rax
-; AVX512-NEXT:    mulq %r12
-; AVX512-NEXT:    movq %rax, %rbx
+; AVX512-NEXT:    mulq %rdi
+; AVX512-NEXT:    movq %rax, %rdi
 ; AVX512-NEXT:    seto %cl
 ; AVX512-NEXT:    orb %bpl, %cl
-; AVX512-NEXT:    addq %rdi, %rbx
-; AVX512-NEXT:    movq %r12, %rax
-; AVX512-NEXT:    mulq %r15
-; AVX512-NEXT:    movq %rax, %r10
-; AVX512-NEXT:    movq %rdx, %r15
-; AVX512-NEXT:    addq %rbx, %r15
-; AVX512-NEXT:    setb %al
-; AVX512-NEXT:    orb %cl, %al
-; AVX512-NEXT:    orb %r13b, %al
-; AVX512-NEXT:    kmovd %eax, %k0
-; AVX512-NEXT:    kshiftlw $1, %k0, %k0
-; AVX512-NEXT:    testq %r9, %r9
-; AVX512-NEXT:    setne %al
-; AVX512-NEXT:    testq %rsi, %rsi
-; AVX512-NEXT:    setne %cl
-; AVX512-NEXT:    andb %al, %cl
+; AVX512-NEXT:    addq %r13, %rdi
 ; AVX512-NEXT:    movq %rsi, %rax
 ; AVX512-NEXT:    mulq %r8
-; AVX512-NEXT:    movq %rax, %rsi
-; AVX512-NEXT:    seto %bpl
+; AVX512-NEXT:    movq %rax, %r8
+; AVX512-NEXT:    movq %rdx, %r10
+; AVX512-NEXT:    addq %rdi, %r10
+; AVX512-NEXT:    setb %sil
+; AVX512-NEXT:    orb %cl, %sil
+; AVX512-NEXT:    orb %bl, %sil
+; AVX512-NEXT:    testq %r12, %r12
+; AVX512-NEXT:    setne %al
+; AVX512-NEXT:    testq %r9, %r9
+; AVX512-NEXT:    setne %bpl
+; AVX512-NEXT:    andb %al, %bpl
 ; AVX512-NEXT:    movq %r9, %rax
-; AVX512-NEXT:    mulq %r11
+; AVX512-NEXT:    mulq %r15
 ; AVX512-NEXT:    movq %rax, %rdi
-; AVX512-NEXT:    seto %bl
-; AVX512-NEXT:    orb %bpl, %bl
-; AVX512-NEXT:    addq %rsi, %rdi
+; AVX512-NEXT:    seto %r9b
+; AVX512-NEXT:    movq %r12, %rax
+; AVX512-NEXT:    mulq %r11
+; AVX512-NEXT:    movq %rax, %rbx
+; AVX512-NEXT:    seto %cl
+; AVX512-NEXT:    orb %r9b, %cl
+; AVX512-NEXT:    addq %rdi, %rbx
 ; AVX512-NEXT:    movq %r11, %rax
-; AVX512-NEXT:    mulq %r8
-; AVX512-NEXT:    addq %rdi, %rdx
-; AVX512-NEXT:    setb %sil
-; AVX512-NEXT:    orb %bl, %sil
-; AVX512-NEXT:    orb %cl, %sil
+; AVX512-NEXT:    mulq %r15
+; AVX512-NEXT:    addq %rbx, %rdx
+; AVX512-NEXT:    setb %dil
+; AVX512-NEXT:    orb %cl, %dil
+; AVX512-NEXT:    orb %bpl, %dil
+; AVX512-NEXT:    kmovd %edi, %k0
+; AVX512-NEXT:    kshiftlw $1, %k0, %k0
 ; AVX512-NEXT:    andl $1, %esi
 ; AVX512-NEXT:    kmovw %esi, %k1
 ; AVX512-NEXT:    korw %k0, %k1, %k1
-; AVX512-NEXT:    movq %r10, 16(%r14)
-; AVX512-NEXT:    movq %rax, (%r14)
-; AVX512-NEXT:    movq %r15, 24(%r14)
-; AVX512-NEXT:    movq %rdx, 8(%r14)
 ; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX512-NEXT:    vmovdqa64 %xmm0, %xmm0 {%k1} {z}
+; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
+; AVX512-NEXT:    movq %rax, 16(%r14)
+; AVX512-NEXT:    movq %r8, (%r14)
+; AVX512-NEXT:    movq %rdx, 24(%r14)
+; AVX512-NEXT:    movq %r10, 8(%r14)
 ; AVX512-NEXT:    popq %rbx
 ; AVX512-NEXT:    popq %r12
 ; AVX512-NEXT:    popq %r13
diff --git a/llvm/test/CodeGen/X86/vec_usubo.ll b/llvm/test/CodeGen/X86/vec_usubo.ll
index c5a7b19cf..136449776 100644
--- a/llvm/test/CodeGen/X86/vec_usubo.ll
+++ b/llvm/test/CodeGen/X86/vec_usubo.ll
@@ -47,91 +47,66 @@ define <1 x i32> @usubo_v1i32(<1 x i32> %a0, <1 x i32> %a1, <1 x i32>* %p2) noun
 define <2 x i32> @usubo_v2i32(<2 x i32> %a0, <2 x i32> %a1, <2 x i32>* %p2) nounwind {
 ; SSE2-LABEL: usubo_v2i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [4294967295,0,4294967295,0]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    psubq %xmm1, %xmm0
-; SSE2-NEXT:    pand %xmm0, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,0,3,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pxor %xmm3, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm0, %xmm3
+; SSE2-NEXT:    pxor %xmm2, %xmm3
+; SSE2-NEXT:    psubd %xmm1, %xmm0
+; SSE2-NEXT:    pxor %xmm0, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm2
 ; SSE2-NEXT:    movq %xmm0, (%rdi)
-; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    movdqa %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: usubo_v2i32:
 ; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [4294967295,0,4294967295,0]
-; SSSE3-NEXT:    pand %xmm2, %xmm1
-; SSSE3-NEXT:    pand %xmm2, %xmm0
-; SSSE3-NEXT:    psubq %xmm1, %xmm0
-; SSSE3-NEXT:    pand %xmm0, %xmm2
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm2, %xmm3
-; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSSE3-NEXT:    pxor %xmm3, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648]
+; SSSE3-NEXT:    movdqa %xmm0, %xmm3
+; SSSE3-NEXT:    pxor %xmm2, %xmm3
+; SSSE3-NEXT:    psubd %xmm1, %xmm0
+; SSSE3-NEXT:    pxor %xmm0, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm2
 ; SSSE3-NEXT:    movq %xmm0, (%rdi)
-; SSSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSSE3-NEXT:    movdqa %xmm2, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: usubo_v2i32:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pxor %xmm2, %xmm2
-; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
-; SSE41-NEXT:    psubq %xmm1, %xmm0
-; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
-; SSE41-NEXT:    pcmpeqq %xmm0, %xmm2
+; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    psubd %xmm1, %xmm2
+; SSE41-NEXT:    pminud %xmm2, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm2, %xmm0
 ; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE41-NEXT:    pxor %xmm2, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; SSE41-NEXT:    movq %xmm0, (%rdi)
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    movq %xmm2, (%rdi)
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: usubo_v2i32:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
-; AVX1-NEXT:    vpsubq %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
-; AVX1-NEXT:    vpcmpeqq %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vpminud %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm1, %xmm0
 ; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
 ; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
 ; AVX1-NEXT:    vmovq %xmm1, (%rdi)
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: usubo_v2i32:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
-; AVX2-NEXT:    vpsubq %xmm1, %xmm0, %xmm1
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; AVX2-NEXT:    vpcmpeqq %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm1
+; AVX2-NEXT:    vpminud %xmm0, %xmm1, %xmm0
+; AVX2-NEXT:    vpcmpeqd %xmm0, %xmm1, %xmm0
 ; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
 ; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
 ; AVX2-NEXT:    vmovq %xmm1, (%rdi)
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: usubo_v2i32:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; AVX512-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
-; AVX512-NEXT:    vpsubq %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpblendd {{.*#+}} xmm1 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
-; AVX512-NEXT:    vpmovqd %xmm0, (%rdi)
-; AVX512-NEXT:    vpcmpeqq %xmm0, %xmm1, %xmm0
-; AVX512-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
+; AVX512-NEXT:    vpsubd %xmm1, %xmm0, %xmm1
+; AVX512-NEXT:    vpcmpnleud %xmm0, %xmm1, %k1
+; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
+; AVX512-NEXT:    vmovq %xmm1, (%rdi)
 ; AVX512-NEXT:    retq
   %t = call {<2 x i32>, <2 x i1>} @llvm.usub.with.overflow.v2i32(<2 x i32> %a0, <2 x i32> %a1)
   %val = extractvalue {<2 x i32>, <2 x i1>} %t, 0
@@ -963,12 +938,12 @@ define <2 x i32> @usubo_v2i64(<2 x i64> %a0, <2 x i64> %a1, <2 x i64>* %p2) noun
 ; SSE-NEXT:    pxor %xmm0, %xmm2
 ; SSE-NEXT:    movdqa %xmm2, %xmm1
 ; SSE-NEXT:    pcmpgtd %xmm3, %xmm1
-; SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[0,0,2,2]
 ; SSE-NEXT:    pcmpeqd %xmm3, %xmm2
 ; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE-NEXT:    pand %xmm4, %xmm2
+; SSE-NEXT:    pand %xmm1, %xmm2
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
 ; SSE-NEXT:    por %xmm2, %xmm1
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
 ; SSE-NEXT:    movdqa %xmm0, (%rdi)
 ; SSE-NEXT:    movdqa %xmm1, %xmm0
 ; SSE-NEXT:    retq
@@ -980,6 +955,7 @@ define <2 x i32> @usubo_v2i64(<2 x i64> %a0, <2 x i64> %a1, <2 x i64>* %p2) noun
 ; AVX1-NEXT:    vpsubq %xmm1, %xmm0, %xmm1
 ; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm0
 ; AVX1-NEXT:    vpcmpgtq %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; AVX1-NEXT:    vmovdqa %xmm1, (%rdi)
 ; AVX1-NEXT:    retq
 ;
@@ -990,6 +966,7 @@ define <2 x i32> @usubo_v2i64(<2 x i64> %a0, <2 x i64> %a1, <2 x i64>* %p2) noun
 ; AVX2-NEXT:    vpsubq %xmm1, %xmm0, %xmm1
 ; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm0
 ; AVX2-NEXT:    vpcmpgtq %xmm3, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; AVX2-NEXT:    vmovdqa %xmm1, (%rdi)
 ; AVX2-NEXT:    retq
 ;
@@ -997,9 +974,9 @@ define <2 x i32> @usubo_v2i64(<2 x i64> %a0, <2 x i64> %a1, <2 x i64>* %p2) noun
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpsubq %xmm1, %xmm0, %xmm1
 ; AVX512-NEXT:    vpcmpnleuq %xmm0, %xmm1, %k1
-; AVX512-NEXT:    vmovdqa %xmm1, (%rdi)
 ; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX512-NEXT:    vmovdqa64 %xmm0, %xmm0 {%k1} {z}
+; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
+; AVX512-NEXT:    vmovdqa %xmm1, (%rdi)
 ; AVX512-NEXT:    retq
   %t = call {<2 x i64>, <2 x i1>} @llvm.usub.with.overflow.v2i64(<2 x i64> %a0, <2 x i64> %a1)
   %val = extractvalue {<2 x i64>, <2 x i1>} %t, 0
@@ -1267,21 +1244,17 @@ define <2 x i32> @usubo_v2i128(<2 x i128> %a0, <2 x i128> %a1, <2 x i128>* %p2)
 ; SSE2-NEXT:    movq {{[0-9]+}}(%rsp), %r10
 ; SSE2-NEXT:    subq {{[0-9]+}}(%rsp), %rdx
 ; SSE2-NEXT:    sbbq {{[0-9]+}}(%rsp), %rcx
-; SSE2-NEXT:    setb %al
-; SSE2-NEXT:    movzbl %al, %r11d
+; SSE2-NEXT:    sbbl %eax, %eax
 ; SSE2-NEXT:    subq %r8, %rdi
 ; SSE2-NEXT:    sbbq %r9, %rsi
-; SSE2-NEXT:    setb %al
-; SSE2-NEXT:    movzbl %al, %eax
+; SSE2-NEXT:    movd %eax, %xmm1
+; SSE2-NEXT:    sbbl %eax, %eax
 ; SSE2-NEXT:    movd %eax, %xmm0
-; SSE2-NEXT:    pinsrw $4, %r11d, %xmm0
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; SSE2-NEXT:    movq %rdx, 16(%r10)
 ; SSE2-NEXT:    movq %rdi, (%r10)
 ; SSE2-NEXT:    movq %rcx, 24(%r10)
 ; SSE2-NEXT:    movq %rsi, 8(%r10)
-; SSE2-NEXT:    psllq $63, %xmm0
-; SSE2-NEXT:    psrad $31, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: usubo_v2i128:
@@ -1289,21 +1262,17 @@ define <2 x i32> @usubo_v2i128(<2 x i128> %a0, <2 x i128> %a1, <2 x i128>* %p2)
 ; SSSE3-NEXT:    movq {{[0-9]+}}(%rsp), %r10
 ; SSSE3-NEXT:    subq {{[0-9]+}}(%rsp), %rdx
 ; SSSE3-NEXT:    sbbq {{[0-9]+}}(%rsp), %rcx
-; SSSE3-NEXT:    setb %al
-; SSSE3-NEXT:    movzbl %al, %r11d
+; SSSE3-NEXT:    sbbl %eax, %eax
 ; SSSE3-NEXT:    subq %r8, %rdi
 ; SSSE3-NEXT:    sbbq %r9, %rsi
-; SSSE3-NEXT:    setb %al
-; SSSE3-NEXT:    movzbl %al, %eax
+; SSSE3-NEXT:    movd %eax, %xmm1
+; SSSE3-NEXT:    sbbl %eax, %eax
 ; SSSE3-NEXT:    movd %eax, %xmm0
-; SSSE3-NEXT:    pinsrw $4, %r11d, %xmm0
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; SSSE3-NEXT:    movq %rdx, 16(%r10)
 ; SSSE3-NEXT:    movq %rdi, (%r10)
 ; SSSE3-NEXT:    movq %rcx, 24(%r10)
 ; SSSE3-NEXT:    movq %rsi, 8(%r10)
-; SSSE3-NEXT:    psllq $63, %xmm0
-; SSSE3-NEXT:    psrad $31, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: usubo_v2i128:
@@ -1311,21 +1280,16 @@ define <2 x i32> @usubo_v2i128(<2 x i128> %a0, <2 x i128> %a1, <2 x i128>* %p2)
 ; SSE41-NEXT:    movq {{[0-9]+}}(%rsp), %r10
 ; SSE41-NEXT:    subq {{[0-9]+}}(%rsp), %rdx
 ; SSE41-NEXT:    sbbq {{[0-9]+}}(%rsp), %rcx
-; SSE41-NEXT:    setb %al
-; SSE41-NEXT:    movzbl %al, %r11d
+; SSE41-NEXT:    sbbl %r11d, %r11d
 ; SSE41-NEXT:    subq %r8, %rdi
 ; SSE41-NEXT:    sbbq %r9, %rsi
-; SSE41-NEXT:    setb %al
-; SSE41-NEXT:    movzbl %al, %eax
+; SSE41-NEXT:    sbbl %eax, %eax
 ; SSE41-NEXT:    movd %eax, %xmm0
-; SSE41-NEXT:    pinsrb $8, %r11d, %xmm0
+; SSE41-NEXT:    pinsrd $1, %r11d, %xmm0
 ; SSE41-NEXT:    movq %rdx, 16(%r10)
 ; SSE41-NEXT:    movq %rdi, (%r10)
 ; SSE41-NEXT:    movq %rcx, 24(%r10)
 ; SSE41-NEXT:    movq %rsi, 8(%r10)
-; SSE41-NEXT:    psllq $63, %xmm0
-; SSE41-NEXT:    psrad $31, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: usubo_v2i128:
@@ -1333,21 +1297,16 @@ define <2 x i32> @usubo_v2i128(<2 x i128> %a0, <2 x i128> %a1, <2 x i128>* %p2)
 ; AVX1-NEXT:    movq {{[0-9]+}}(%rsp), %r10
 ; AVX1-NEXT:    subq {{[0-9]+}}(%rsp), %rdx
 ; AVX1-NEXT:    sbbq {{[0-9]+}}(%rsp), %rcx
-; AVX1-NEXT:    setb %al
-; AVX1-NEXT:    movzbl %al, %r11d
+; AVX1-NEXT:    sbbl %r11d, %r11d
 ; AVX1-NEXT:    subq %r8, %rdi
 ; AVX1-NEXT:    sbbq %r9, %rsi
-; AVX1-NEXT:    setb %al
-; AVX1-NEXT:    movzbl %al, %eax
+; AVX1-NEXT:    sbbl %eax, %eax
 ; AVX1-NEXT:    vmovd %eax, %xmm0
-; AVX1-NEXT:    vpinsrb $8, %r11d, %xmm0, %xmm0
+; AVX1-NEXT:    vpinsrd $1, %r11d, %xmm0, %xmm0
 ; AVX1-NEXT:    movq %rdx, 16(%r10)
 ; AVX1-NEXT:    movq %rdi, (%r10)
 ; AVX1-NEXT:    movq %rcx, 24(%r10)
 ; AVX1-NEXT:    movq %rsi, 8(%r10)
-; AVX1-NEXT:    vpsllq $63, %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: usubo_v2i128:
@@ -1355,43 +1314,38 @@ define <2 x i32> @usubo_v2i128(<2 x i128> %a0, <2 x i128> %a1, <2 x i128>* %p2)
 ; AVX2-NEXT:    movq {{[0-9]+}}(%rsp), %r10
 ; AVX2-NEXT:    subq {{[0-9]+}}(%rsp), %rdx
 ; AVX2-NEXT:    sbbq {{[0-9]+}}(%rsp), %rcx
-; AVX2-NEXT:    setb %al
-; AVX2-NEXT:    movzbl %al, %r11d
+; AVX2-NEXT:    sbbl %r11d, %r11d
 ; AVX2-NEXT:    subq %r8, %rdi
 ; AVX2-NEXT:    sbbq %r9, %rsi
-; AVX2-NEXT:    setb %al
-; AVX2-NEXT:    movzbl %al, %eax
+; AVX2-NEXT:    sbbl %eax, %eax
 ; AVX2-NEXT:    vmovd %eax, %xmm0
-; AVX2-NEXT:    vpinsrb $8, %r11d, %xmm0, %xmm0
+; AVX2-NEXT:    vpinsrd $1, %r11d, %xmm0, %xmm0
 ; AVX2-NEXT:    movq %rdx, 16(%r10)
 ; AVX2-NEXT:    movq %rdi, (%r10)
 ; AVX2-NEXT:    movq %rcx, 24(%r10)
 ; AVX2-NEXT:    movq %rsi, 8(%r10)
-; AVX2-NEXT:    vpsllq $63, %xmm0, %xmm0
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: usubo_v2i128:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %r10
+; AVX512-NEXT:    subq %r8, %rdi
+; AVX512-NEXT:    sbbq %r9, %rsi
+; AVX512-NEXT:    setb %r8b
 ; AVX512-NEXT:    subq {{[0-9]+}}(%rsp), %rdx
 ; AVX512-NEXT:    sbbq {{[0-9]+}}(%rsp), %rcx
 ; AVX512-NEXT:    setb %al
 ; AVX512-NEXT:    kmovd %eax, %k0
 ; AVX512-NEXT:    kshiftlw $1, %k0, %k0
-; AVX512-NEXT:    subq %r8, %rdi
-; AVX512-NEXT:    sbbq %r9, %rsi
-; AVX512-NEXT:    setb %al
-; AVX512-NEXT:    andl $1, %eax
-; AVX512-NEXT:    kmovw %eax, %k1
+; AVX512-NEXT:    andl $1, %r8d
+; AVX512-NEXT:    kmovw %r8d, %k1
 ; AVX512-NEXT:    korw %k0, %k1, %k1
+; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
 ; AVX512-NEXT:    movq %rdx, 16(%r10)
 ; AVX512-NEXT:    movq %rdi, (%r10)
 ; AVX512-NEXT:    movq %rcx, 24(%r10)
 ; AVX512-NEXT:    movq %rsi, 8(%r10)
-; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX512-NEXT:    vmovdqa64 %xmm0, %xmm0 {%k1} {z}
 ; AVX512-NEXT:    retq
   %t = call {<2 x i128>, <2 x i1>} @llvm.usub.with.overflow.v2i128(<2 x i128> %a0, <2 x i128> %a1)
   %val = extractvalue {<2 x i128>, <2 x i1>} %t, 0
diff --git a/llvm/test/CodeGen/X86/vec_zero-2.ll b/llvm/test/CodeGen/X86/vec_zero-2.ll
index dce924cc6..bbb3ea645 100644
--- a/llvm/test/CodeGen/X86/vec_zero-2.ll
+++ b/llvm/test/CodeGen/X86/vec_zero-2.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=i686-- -mattr=+sse2
 
 define i32 @t() {
diff --git a/llvm/test/CodeGen/X86/vector-bitreverse.ll b/llvm/test/CodeGen/X86/vector-bitreverse.ll
index e152785b0..954d7a2c5 100644
--- a/llvm/test/CodeGen/X86/vector-bitreverse.ll
+++ b/llvm/test/CodeGen/X86/vector-bitreverse.ll
@@ -1323,21 +1323,23 @@ define <64 x i8> @test_bitreverse_v64i8(<64 x i8> %a) nounwind {
 ;
 ; AVX512F-LABEL: test_bitreverse_v64i8:
 ; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512F-NEXT:    vpand %ymm2, %ymm0, %ymm3
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm4 = [0,128,64,192,32,160,96,224,16,144,80,208,48,176,112,240,0,128,64,192,32,160,96,224,16,144,80,208,48,176,112,240]
-; AVX512F-NEXT:    vpshufb %ymm3, %ymm4, %ymm3
-; AVX512F-NEXT:    vpsrlw $4, %ymm0, %ymm0
-; AVX512F-NEXT:    vpand %ymm2, %ymm0, %ymm0
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm5 = [0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15,0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15]
-; AVX512F-NEXT:    vpshufb %ymm0, %ymm5, %ymm0
-; AVX512F-NEXT:    vpor %ymm0, %ymm3, %ymm0
 ; AVX512F-NEXT:    vpand %ymm2, %ymm1, %ymm3
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm4 = [0,128,64,192,32,160,96,224,16,144,80,208,48,176,112,240,0,128,64,192,32,160,96,224,16,144,80,208,48,176,112,240]
 ; AVX512F-NEXT:    vpshufb %ymm3, %ymm4, %ymm3
 ; AVX512F-NEXT:    vpsrlw $4, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm5 = [0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15,0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15]
 ; AVX512F-NEXT:    vpshufb %ymm1, %ymm5, %ymm1
 ; AVX512F-NEXT:    vpor %ymm1, %ymm3, %ymm1
+; AVX512F-NEXT:    vpand %ymm2, %ymm0, %ymm3
+; AVX512F-NEXT:    vpshufb %ymm3, %ymm4, %ymm3
+; AVX512F-NEXT:    vpsrlw $4, %ymm0, %ymm0
+; AVX512F-NEXT:    vpand %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vpshufb %ymm0, %ymm5, %ymm0
+; AVX512F-NEXT:    vpor %ymm0, %ymm3, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: test_bitreverse_v64i8:
@@ -1613,24 +1615,26 @@ define <32 x i16> @test_bitreverse_v32i16(<32 x i16> %a) nounwind {
 ;
 ; AVX512F-LABEL: test_bitreverse_v32i16:
 ; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm2 = [1,0,3,2,5,4,7,6,9,8,11,10,13,12,15,14,1,0,3,2,5,4,7,6,9,8,11,10,13,12,15,14]
-; AVX512F-NEXT:    vpshufb %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vpshufb %ymm2, %ymm1, %ymm1
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512F-NEXT:    vpand %ymm3, %ymm0, %ymm4
+; AVX512F-NEXT:    vpand %ymm3, %ymm1, %ymm4
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm5 = [0,128,64,192,32,160,96,224,16,144,80,208,48,176,112,240,0,128,64,192,32,160,96,224,16,144,80,208,48,176,112,240]
 ; AVX512F-NEXT:    vpshufb %ymm4, %ymm5, %ymm4
-; AVX512F-NEXT:    vpsrlw $4, %ymm0, %ymm0
-; AVX512F-NEXT:    vpand %ymm3, %ymm0, %ymm0
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm6 = [0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15,0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15]
-; AVX512F-NEXT:    vpshufb %ymm0, %ymm6, %ymm0
-; AVX512F-NEXT:    vpor %ymm0, %ymm4, %ymm0
-; AVX512F-NEXT:    vpshufb %ymm2, %ymm1, %ymm1
-; AVX512F-NEXT:    vpand %ymm3, %ymm1, %ymm2
-; AVX512F-NEXT:    vpshufb %ymm2, %ymm5, %ymm2
 ; AVX512F-NEXT:    vpsrlw $4, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpand %ymm3, %ymm1, %ymm1
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm6 = [0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15,0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15]
 ; AVX512F-NEXT:    vpshufb %ymm1, %ymm6, %ymm1
-; AVX512F-NEXT:    vpor %ymm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vpor %ymm1, %ymm4, %ymm1
+; AVX512F-NEXT:    vpshufb %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vpand %ymm3, %ymm0, %ymm2
+; AVX512F-NEXT:    vpshufb %ymm2, %ymm5, %ymm2
+; AVX512F-NEXT:    vpsrlw $4, %ymm0, %ymm0
+; AVX512F-NEXT:    vpand %ymm3, %ymm0, %ymm0
+; AVX512F-NEXT:    vpshufb %ymm0, %ymm6, %ymm0
+; AVX512F-NEXT:    vpor %ymm0, %ymm2, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: test_bitreverse_v32i16:
@@ -1907,30 +1911,26 @@ define <16 x i32> @test_bitreverse_v16i32(<16 x i32> %a) nounwind {
 ;
 ; AVX512F-LABEL: test_bitreverse_v16i32:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpsrld $24, %zmm0, %zmm1
-; AVX512F-NEXT:    vpsrld $8, %zmm0, %zmm2
-; AVX512F-NEXT:    vpandd {{.*}}(%rip){1to16}, %zmm2, %zmm2
-; AVX512F-NEXT:    vpord %zmm1, %zmm2, %zmm1
-; AVX512F-NEXT:    vpslld $24, %zmm0, %zmm2
-; AVX512F-NEXT:    vpslld $8, %zmm0, %zmm0
-; AVX512F-NEXT:    vpandd {{.*}}(%rip){1to16}, %zmm0, %zmm0
-; AVX512F-NEXT:    vpord %zmm1, %zmm0, %zmm0
-; AVX512F-NEXT:    vpord %zmm0, %zmm2, %zmm0
-; AVX512F-NEXT:    vpandd {{.*}}(%rip){1to16}, %zmm0, %zmm1
-; AVX512F-NEXT:    vpslld $4, %zmm1, %zmm1
-; AVX512F-NEXT:    vpandd {{.*}}(%rip){1to16}, %zmm0, %zmm0
-; AVX512F-NEXT:    vpsrld $4, %zmm0, %zmm0
-; AVX512F-NEXT:    vpord %zmm1, %zmm0, %zmm0
-; AVX512F-NEXT:    vpandd {{.*}}(%rip){1to16}, %zmm0, %zmm1
-; AVX512F-NEXT:    vpslld $2, %zmm1, %zmm1
-; AVX512F-NEXT:    vpandd {{.*}}(%rip){1to16}, %zmm0, %zmm0
-; AVX512F-NEXT:    vpsrld $2, %zmm0, %zmm0
-; AVX512F-NEXT:    vpord %zmm1, %zmm0, %zmm0
-; AVX512F-NEXT:    vpandd {{.*}}(%rip){1to16}, %zmm0, %zmm1
-; AVX512F-NEXT:    vpslld $1, %zmm1, %zmm1
-; AVX512F-NEXT:    vpandd {{.*}}(%rip){1to16}, %zmm0, %zmm0
-; AVX512F-NEXT:    vpsrld $1, %zmm0, %zmm0
-; AVX512F-NEXT:    vpord %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm2 = [3,2,1,0,7,6,5,4,11,10,9,8,15,14,13,12,3,2,1,0,7,6,5,4,11,10,9,8,15,14,13,12]
+; AVX512F-NEXT:    vpshufb %ymm2, %ymm1, %ymm1
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX512F-NEXT:    vpand %ymm3, %ymm1, %ymm4
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm5 = [0,128,64,192,32,160,96,224,16,144,80,208,48,176,112,240,0,128,64,192,32,160,96,224,16,144,80,208,48,176,112,240]
+; AVX512F-NEXT:    vpshufb %ymm4, %ymm5, %ymm4
+; AVX512F-NEXT:    vpsrlw $4, %ymm1, %ymm1
+; AVX512F-NEXT:    vpand %ymm3, %ymm1, %ymm1
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm6 = [0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15,0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15]
+; AVX512F-NEXT:    vpshufb %ymm1, %ymm6, %ymm1
+; AVX512F-NEXT:    vpor %ymm1, %ymm4, %ymm1
+; AVX512F-NEXT:    vpshufb %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vpand %ymm3, %ymm0, %ymm2
+; AVX512F-NEXT:    vpshufb %ymm2, %ymm5, %ymm2
+; AVX512F-NEXT:    vpsrlw $4, %ymm0, %ymm0
+; AVX512F-NEXT:    vpand %ymm3, %ymm0, %ymm0
+; AVX512F-NEXT:    vpshufb %ymm0, %ymm6, %ymm0
+; AVX512F-NEXT:    vpor %ymm0, %ymm2, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: test_bitreverse_v16i32:
@@ -2215,42 +2215,26 @@ define <8 x i64> @test_bitreverse_v8i64(<8 x i64> %a) nounwind {
 ;
 ; AVX512F-LABEL: test_bitreverse_v8i64:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpsrlq $56, %zmm0, %zmm1
-; AVX512F-NEXT:    vpsrlq $40, %zmm0, %zmm2
-; AVX512F-NEXT:    vpandq {{.*}}(%rip){1to8}, %zmm2, %zmm2
-; AVX512F-NEXT:    vporq %zmm1, %zmm2, %zmm1
-; AVX512F-NEXT:    vpsrlq $24, %zmm0, %zmm2
-; AVX512F-NEXT:    vpandq {{.*}}(%rip){1to8}, %zmm2, %zmm2
-; AVX512F-NEXT:    vporq %zmm1, %zmm2, %zmm1
-; AVX512F-NEXT:    vpsrlq $8, %zmm0, %zmm2
-; AVX512F-NEXT:    vpandq {{.*}}(%rip){1to8}, %zmm2, %zmm2
-; AVX512F-NEXT:    vporq %zmm1, %zmm2, %zmm1
-; AVX512F-NEXT:    vpsllq $8, %zmm0, %zmm2
-; AVX512F-NEXT:    vpandq {{.*}}(%rip){1to8}, %zmm2, %zmm2
-; AVX512F-NEXT:    vpsllq $24, %zmm0, %zmm3
-; AVX512F-NEXT:    vpandq {{.*}}(%rip){1to8}, %zmm3, %zmm3
-; AVX512F-NEXT:    vporq %zmm2, %zmm3, %zmm2
-; AVX512F-NEXT:    vpsllq $56, %zmm0, %zmm3
-; AVX512F-NEXT:    vpsllq $40, %zmm0, %zmm0
-; AVX512F-NEXT:    vpandq {{.*}}(%rip){1to8}, %zmm0, %zmm0
-; AVX512F-NEXT:    vporq %zmm2, %zmm0, %zmm0
-; AVX512F-NEXT:    vporq %zmm1, %zmm0, %zmm0
-; AVX512F-NEXT:    vporq %zmm0, %zmm3, %zmm0
-; AVX512F-NEXT:    vpandq {{.*}}(%rip){1to8}, %zmm0, %zmm1
-; AVX512F-NEXT:    vpsllq $4, %zmm1, %zmm1
-; AVX512F-NEXT:    vpandq {{.*}}(%rip){1to8}, %zmm0, %zmm0
-; AVX512F-NEXT:    vpsrlq $4, %zmm0, %zmm0
-; AVX512F-NEXT:    vporq %zmm1, %zmm0, %zmm0
-; AVX512F-NEXT:    vpandq {{.*}}(%rip){1to8}, %zmm0, %zmm1
-; AVX512F-NEXT:    vpsllq $2, %zmm1, %zmm1
-; AVX512F-NEXT:    vpandq {{.*}}(%rip){1to8}, %zmm0, %zmm0
-; AVX512F-NEXT:    vpsrlq $2, %zmm0, %zmm0
-; AVX512F-NEXT:    vporq %zmm1, %zmm0, %zmm0
-; AVX512F-NEXT:    vpandq {{.*}}(%rip){1to8}, %zmm0, %zmm1
-; AVX512F-NEXT:    vpsllq $1, %zmm1, %zmm1
-; AVX512F-NEXT:    vpandq {{.*}}(%rip){1to8}, %zmm0, %zmm0
-; AVX512F-NEXT:    vpsrlq $1, %zmm0, %zmm0
-; AVX512F-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm2 = [7,6,5,4,3,2,1,0,15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0,15,14,13,12,11,10,9,8]
+; AVX512F-NEXT:    vpshufb %ymm2, %ymm1, %ymm1
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX512F-NEXT:    vpand %ymm3, %ymm1, %ymm4
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm5 = [0,128,64,192,32,160,96,224,16,144,80,208,48,176,112,240,0,128,64,192,32,160,96,224,16,144,80,208,48,176,112,240]
+; AVX512F-NEXT:    vpshufb %ymm4, %ymm5, %ymm4
+; AVX512F-NEXT:    vpsrlw $4, %ymm1, %ymm1
+; AVX512F-NEXT:    vpand %ymm3, %ymm1, %ymm1
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm6 = [0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15,0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15]
+; AVX512F-NEXT:    vpshufb %ymm1, %ymm6, %ymm1
+; AVX512F-NEXT:    vpor %ymm1, %ymm4, %ymm1
+; AVX512F-NEXT:    vpshufb %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vpand %ymm3, %ymm0, %ymm2
+; AVX512F-NEXT:    vpshufb %ymm2, %ymm5, %ymm2
+; AVX512F-NEXT:    vpsrlw $4, %ymm0, %ymm0
+; AVX512F-NEXT:    vpand %ymm3, %ymm0, %ymm0
+; AVX512F-NEXT:    vpshufb %ymm0, %ymm6, %ymm0
+; AVX512F-NEXT:    vpor %ymm0, %ymm2, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: test_bitreverse_v8i64:
diff --git a/llvm/test/CodeGen/X86/vector-blend.ll b/llvm/test/CodeGen/X86/vector-blend.ll
index c6bcd299d..ce97fec08 100644
--- a/llvm/test/CodeGen/X86/vector-blend.ll
+++ b/llvm/test/CodeGen/X86/vector-blend.ll
@@ -64,24 +64,30 @@ entry:
 define <4 x i8> @vsel_4xi8(<4 x i8> %v1, <4 x i8> %v2) {
 ; SSE2-LABEL: vsel_4xi8:
 ; SSE2:       # %bb.0: # %entry
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[2,0],xmm0[3,0]
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm1[0,2]
+; SSE2-NEXT:    movaps {{.*#+}} xmm2 = [255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; SSE2-NEXT:    andps %xmm2, %xmm0
+; SSE2-NEXT:    andnps %xmm1, %xmm2
+; SSE2-NEXT:    orps %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: vsel_4xi8:
 ; SSSE3:       # %bb.0: # %entry
-; SSSE3-NEXT:    shufps {{.*#+}} xmm1 = xmm1[2,0],xmm0[3,0]
-; SSSE3-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm1[0,2]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSSE3-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,2,5,6,u,u,u,u,u,u,u,u,u,u,u,u]
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: vsel_4xi8:
 ; SSE41:       # %bb.0: # %entry
-; SSE41-NEXT:    blendps {{.*#+}} xmm0 = xmm0[0,1],xmm1[2],xmm0[3]
+; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    movaps {{.*#+}} xmm0 = <255,255,0,255,u,u,u,u,u,u,u,u,u,u,u,u>
+; SSE41-NEXT:    pblendvb %xmm0, %xmm2, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: vsel_4xi8:
 ; AVX:       # %bb.0: # %entry
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1],xmm1[2],xmm0[3]
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm2 = <255,255,0,255,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX-NEXT:    vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    retq
 entry:
   %vsel = select <4 x i1> <i1 true, i1 true, i1 false, i1 true>, <4 x i8> %v1, <4 x i8> %v2
@@ -91,26 +97,28 @@ entry:
 define <4 x i16> @vsel_4xi16(<4 x i16> %v1, <4 x i16> %v2) {
 ; SSE2-LABEL: vsel_4xi16:
 ; SSE2:       # %bb.0: # %entry
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,0],xmm0[0,0]
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[2,0],xmm0[2,3]
-; SSE2-NEXT:    movaps %xmm1, %xmm0
+; SSE2-NEXT:    movaps {{.*#+}} xmm2 = [65535,0,65535,65535,65535,65535,65535,65535]
+; SSE2-NEXT:    andps %xmm2, %xmm0
+; SSE2-NEXT:    andnps %xmm1, %xmm2
+; SSE2-NEXT:    orps %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: vsel_4xi16:
 ; SSSE3:       # %bb.0: # %entry
-; SSSE3-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,0],xmm0[0,0]
-; SSSE3-NEXT:    shufps {{.*#+}} xmm1 = xmm1[2,0],xmm0[2,3]
-; SSSE3-NEXT:    movaps %xmm1, %xmm0
+; SSSE3-NEXT:    movaps {{.*#+}} xmm2 = [65535,0,65535,65535,65535,65535,65535,65535]
+; SSSE3-NEXT:    andps %xmm2, %xmm0
+; SSSE3-NEXT:    andnps %xmm1, %xmm2
+; SSSE3-NEXT:    orps %xmm2, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: vsel_4xi16:
 ; SSE41:       # %bb.0: # %entry
-; SSE41-NEXT:    blendps {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3]
+; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3,4,5,6,7]
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: vsel_4xi16:
 ; AVX:       # %bb.0: # %entry
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3]
+; AVX-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3,4,5,6,7]
 ; AVX-NEXT:    retq
 entry:
   %vsel = select <4 x i1> <i1 true, i1 false, i1 true, i1 true>, <4 x i16> %v1, <4 x i16> %v2
@@ -914,37 +922,19 @@ entry:
 }
 
 define <4 x i32> @blend_neg_logic_v4i32_2(<4 x i32> %v, <4 x i32> %c) {
-; SSE2-LABEL: blend_neg_logic_v4i32_2:
-; SSE2:       # %bb.0: # %entry
-; SSE2-NEXT:    psrad $31, %xmm1
-; SSE2-NEXT:    pxor %xmm1, %xmm0
-; SSE2-NEXT:    psubd %xmm0, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm0
-; SSE2-NEXT:    retq
-;
-; SSSE3-LABEL: blend_neg_logic_v4i32_2:
-; SSSE3:       # %bb.0: # %entry
-; SSSE3-NEXT:    psrad $31, %xmm1
-; SSSE3-NEXT:    pxor %xmm1, %xmm0
-; SSSE3-NEXT:    psubd %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa %xmm1, %xmm0
-; SSSE3-NEXT:    retq
-;
-; SSE41-LABEL: blend_neg_logic_v4i32_2:
-; SSE41:       # %bb.0: # %entry
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    pxor %xmm3, %xmm3
-; SSE41-NEXT:    psubd %xmm0, %xmm3
-; SSE41-NEXT:    movaps %xmm1, %xmm0
-; SSE41-NEXT:    blendvps %xmm0, %xmm2, %xmm3
-; SSE41-NEXT:    movaps %xmm3, %xmm0
-; SSE41-NEXT:    retq
+; SSE-LABEL: blend_neg_logic_v4i32_2:
+; SSE:       # %bb.0: # %entry
+; SSE-NEXT:    psrad $31, %xmm1
+; SSE-NEXT:    pxor %xmm1, %xmm0
+; SSE-NEXT:    psubd %xmm0, %xmm1
+; SSE-NEXT:    movdqa %xmm1, %xmm0
+; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: blend_neg_logic_v4i32_2:
 ; AVX:       # %bb.0: # %entry
-; AVX-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX-NEXT:    vpsubd %xmm0, %xmm2, %xmm2
-; AVX-NEXT:    vblendvps %xmm1, %xmm0, %xmm2, %xmm0
+; AVX-NEXT:    vpsrad $31, %xmm1, %xmm1
+; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpsubd %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    retq
 entry:
   %0 = ashr <4 x i32> %c, <i32 31, i32 31, i32 31, i32 31>
diff --git a/llvm/test/CodeGen/X86/vector-compare-all_of.ll b/llvm/test/CodeGen/X86/vector-compare-all_of.ll
index c5a86ca70..02c9f4f17 100644
--- a/llvm/test/CodeGen/X86/vector-compare-all_of.ll
+++ b/llvm/test/CodeGen/X86/vector-compare-all_of.ll
@@ -937,7 +937,6 @@ define i1 @bool_reduction_v2f64(<2 x double> %x, <2 x double> %y) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcmpltpd %xmm0, %xmm1, %k0
 ; AVX512-NEXT:    kmovd %k0, %eax
-; AVX512-NEXT:    andb $3, %al
 ; AVX512-NEXT:    cmpb $3, %al
 ; AVX512-NEXT:    sete %al
 ; AVX512-NEXT:    retq
@@ -969,7 +968,6 @@ define i1 @bool_reduction_v4f32(<4 x float> %x, <4 x float> %y) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcmpeqps %xmm1, %xmm0, %k0
 ; AVX512-NEXT:    kmovd %k0, %eax
-; AVX512-NEXT:    andb $15, %al
 ; AVX512-NEXT:    cmpb $15, %al
 ; AVX512-NEXT:    sete %al
 ; AVX512-NEXT:    retq
@@ -1006,7 +1004,6 @@ define i1 @bool_reduction_v4f64(<4 x double> %x, <4 x double> %y) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcmplepd %ymm0, %ymm1, %k0
 ; AVX512-NEXT:    kmovd %k0, %eax
-; AVX512-NEXT:    andb $15, %al
 ; AVX512-NEXT:    cmpb $15, %al
 ; AVX512-NEXT:    sete %al
 ; AVX512-NEXT:    vzeroupper
@@ -1087,7 +1084,6 @@ define i1 @bool_reduction_v2i64(<2 x i64> %x, <2 x i64> %y) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpcmpnleuq %xmm1, %xmm0, %k0
 ; AVX512-NEXT:    kmovd %k0, %eax
-; AVX512-NEXT:    andb $3, %al
 ; AVX512-NEXT:    cmpb $3, %al
 ; AVX512-NEXT:    sete %al
 ; AVX512-NEXT:    retq
@@ -1121,7 +1117,6 @@ define i1 @bool_reduction_v4i32(<4 x i32> %x, <4 x i32> %y) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpcmpneqd %xmm1, %xmm0, %k0
 ; AVX512-NEXT:    kmovd %k0, %eax
-; AVX512-NEXT:    andb $15, %al
 ; AVX512-NEXT:    cmpb $15, %al
 ; AVX512-NEXT:    sete %al
 ; AVX512-NEXT:    retq
@@ -1244,7 +1239,6 @@ define i1 @bool_reduction_v4i64(<4 x i64> %x, <4 x i64> %y) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpcmpgtq %ymm0, %ymm1, %k0
 ; AVX512-NEXT:    kmovd %k0, %eax
-; AVX512-NEXT:    andb $15, %al
 ; AVX512-NEXT:    cmpb $15, %al
 ; AVX512-NEXT:    sete %al
 ; AVX512-NEXT:    vzeroupper
diff --git a/llvm/test/CodeGen/X86/vector-compare-any_of.ll b/llvm/test/CodeGen/X86/vector-compare-any_of.ll
index a64d031fe..40c63a717 100644
--- a/llvm/test/CodeGen/X86/vector-compare-any_of.ll
+++ b/llvm/test/CodeGen/X86/vector-compare-any_of.ll
@@ -857,7 +857,7 @@ define i1 @bool_reduction_v2f64(<2 x double> %x, <2 x double> %y) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcmpltpd %xmm0, %xmm1, %k0
 ; AVX512-NEXT:    kmovd %k0, %eax
-; AVX512-NEXT:    testb $3, %al
+; AVX512-NEXT:    testb %al, %al
 ; AVX512-NEXT:    setne %al
 ; AVX512-NEXT:    retq
   %a = fcmp ogt <2 x double> %x, %y
@@ -888,7 +888,7 @@ define i1 @bool_reduction_v4f32(<4 x float> %x, <4 x float> %y) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcmpeqps %xmm1, %xmm0, %k0
 ; AVX512-NEXT:    kmovd %k0, %eax
-; AVX512-NEXT:    testb $15, %al
+; AVX512-NEXT:    testb %al, %al
 ; AVX512-NEXT:    setne %al
 ; AVX512-NEXT:    retq
   %a = fcmp oeq <4 x float> %x, %y
@@ -924,7 +924,7 @@ define i1 @bool_reduction_v4f64(<4 x double> %x, <4 x double> %y) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcmplepd %ymm0, %ymm1, %k0
 ; AVX512-NEXT:    kmovd %k0, %eax
-; AVX512-NEXT:    testb $15, %al
+; AVX512-NEXT:    testb %al, %al
 ; AVX512-NEXT:    setne %al
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
@@ -1004,7 +1004,7 @@ define i1 @bool_reduction_v2i64(<2 x i64> %x, <2 x i64> %y) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpcmpnleuq %xmm1, %xmm0, %k0
 ; AVX512-NEXT:    kmovd %k0, %eax
-; AVX512-NEXT:    testb $3, %al
+; AVX512-NEXT:    testb %al, %al
 ; AVX512-NEXT:    setne %al
 ; AVX512-NEXT:    retq
   %a = icmp ugt <2 x i64> %x, %y
@@ -1035,7 +1035,7 @@ define i1 @bool_reduction_v4i32(<4 x i32> %x, <4 x i32> %y) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpcmpneqd %xmm1, %xmm0, %k0
 ; AVX512-NEXT:    kmovd %k0, %eax
-; AVX512-NEXT:    testb $15, %al
+; AVX512-NEXT:    testb %al, %al
 ; AVX512-NEXT:    setne %al
 ; AVX512-NEXT:    retq
   %a = icmp ne <4 x i32> %x, %y
@@ -1157,7 +1157,7 @@ define i1 @bool_reduction_v4i64(<4 x i64> %x, <4 x i64> %y) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpcmpgtq %ymm0, %ymm1, %k0
 ; AVX512-NEXT:    kmovd %k0, %eax
-; AVX512-NEXT:    testb $15, %al
+; AVX512-NEXT:    testb %al, %al
 ; AVX512-NEXT:    setne %al
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/vector-compare-results.ll b/llvm/test/CodeGen/X86/vector-compare-results.ll
index a01249d64..47a6ffb0a 100644
--- a/llvm/test/CodeGen/X86/vector-compare-results.ll
+++ b/llvm/test/CodeGen/X86/vector-compare-results.ll
@@ -732,22 +732,26 @@ define <32 x i1> @test_cmp_v32i16(<32 x i16> %a0, <32 x i16> %a1) nounwind {
 ;
 ; AVX512F-LABEL: test_cmp_v32i16:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpcmpgtw %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
+; AVX512F-NEXT:    vpcmpgtw %ymm2, %ymm3, %ymm2
+; AVX512F-NEXT:    vpcmpgtw %ymm1, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
 ; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
-; AVX512F-NEXT:    vpcmpgtw %ymm3, %ymm1, %ymm1
-; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
 ; AVX512F-NEXT:    vpmovdb %zmm1, %xmm1
 ; AVX512F-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512DQ-LABEL: test_cmp_v32i16:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vpcmpgtw %ymm2, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
+; AVX512DQ-NEXT:    vpcmpgtw %ymm2, %ymm3, %ymm2
+; AVX512DQ-NEXT:    vpcmpgtw %ymm1, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
 ; AVX512DQ-NEXT:    vpmovdb %zmm0, %xmm0
-; AVX512DQ-NEXT:    vpcmpgtw %ymm3, %ymm1, %ymm1
-; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
+; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
 ; AVX512DQ-NEXT:    vpmovdb %zmm1, %xmm1
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    retq
@@ -823,25 +827,49 @@ define <64 x i1> @test_cmp_v64i8(<64 x i8> %a0, <64 x i8> %a1) nounwind {
 ;
 ; AVX512F-LABEL: test_cmp_v64i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpcmpgtb %ymm2, %ymm0, %ymm0
-; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm4
-; AVX512F-NEXT:    vpcmpgtb %ymm3, %ymm1, %ymm2
-; AVX512F-NEXT:    vextracti128 $1, %ymm2, %xmm3
-; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
-; AVX512F-NEXT:    vmovdqa %xmm4, %xmm1
-; AVX512F-NEXT:    # kill: def $xmm2 killed $xmm2 killed $ymm2
+; AVX512F-NEXT:    movq %rdi, %rax
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
+; AVX512F-NEXT:    vpcmpgtb %ymm2, %ymm3, %ymm2
+; AVX512F-NEXT:    vpcmpgtb %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vpmovsxbd %xmm0, %zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm0
+; AVX512F-NEXT:    vpmovsxbd %xmm0, %zmm0
+; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k1
+; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm0
+; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k2
+; AVX512F-NEXT:    vextracti128 $1, %ymm2, %xmm0
+; AVX512F-NEXT:    vpmovsxbd %xmm0, %zmm0
+; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k3
+; AVX512F-NEXT:    kmovw %k3, 6(%rdi)
+; AVX512F-NEXT:    kmovw %k2, 4(%rdi)
+; AVX512F-NEXT:    kmovw %k1, 2(%rdi)
+; AVX512F-NEXT:    kmovw %k0, (%rdi)
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512DQ-LABEL: test_cmp_v64i8:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vpcmpgtb %ymm2, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vextracti128 $1, %ymm0, %xmm4
-; AVX512DQ-NEXT:    vpcmpgtb %ymm3, %ymm1, %ymm2
-; AVX512DQ-NEXT:    vextracti128 $1, %ymm2, %xmm3
-; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
-; AVX512DQ-NEXT:    vmovdqa %xmm4, %xmm1
-; AVX512DQ-NEXT:    # kill: def $xmm2 killed $xmm2 killed $ymm2
+; AVX512DQ-NEXT:    movq %rdi, %rax
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
+; AVX512DQ-NEXT:    vpcmpgtb %ymm2, %ymm3, %ymm2
+; AVX512DQ-NEXT:    vpcmpgtb %ymm1, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpmovsxbd %xmm0, %zmm1
+; AVX512DQ-NEXT:    vpmovd2m %zmm1, %k0
+; AVX512DQ-NEXT:    vextracti128 $1, %ymm0, %xmm0
+; AVX512DQ-NEXT:    vpmovsxbd %xmm0, %zmm0
+; AVX512DQ-NEXT:    vpmovd2m %zmm0, %k1
+; AVX512DQ-NEXT:    vpmovsxbd %xmm2, %zmm0
+; AVX512DQ-NEXT:    vpmovd2m %zmm0, %k2
+; AVX512DQ-NEXT:    vextracti128 $1, %ymm2, %xmm0
+; AVX512DQ-NEXT:    vpmovsxbd %xmm0, %zmm0
+; AVX512DQ-NEXT:    vpmovd2m %zmm0, %k3
+; AVX512DQ-NEXT:    kmovw %k3, 6(%rdi)
+; AVX512DQ-NEXT:    kmovw %k2, 4(%rdi)
+; AVX512DQ-NEXT:    kmovw %k1, 2(%rdi)
+; AVX512DQ-NEXT:    kmovw %k0, (%rdi)
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
 ;
@@ -1447,35 +1475,45 @@ define <64 x i1> @test_cmp_v64i16(<64 x i16> %a0, <64 x i16> %a1) nounwind {
 ;
 ; AVX512F-LABEL: test_cmp_v64i16:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpcmpgtw %ymm7, %ymm3, %ymm3
-; AVX512F-NEXT:    vpcmpgtw %ymm6, %ymm2, %ymm2
-; AVX512F-NEXT:    vpcmpgtw %ymm5, %ymm1, %ymm1
+; AVX512F-NEXT:    movq %rdi, %rax
 ; AVX512F-NEXT:    vpcmpgtw %ymm4, %ymm0, %ymm0
-; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
-; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
-; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
-; AVX512F-NEXT:    vpmovdb %zmm1, %xmm1
-; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
-; AVX512F-NEXT:    vpmovdb %zmm2, %xmm2
-; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm3 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
-; AVX512F-NEXT:    vpmovdb %zmm3, %xmm3
+; AVX512F-NEXT:    vpmovsxwd %ymm0, %zmm0
+; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k0
+; AVX512F-NEXT:    vpcmpgtw %ymm5, %ymm1, %ymm0
+; AVX512F-NEXT:    vpmovsxwd %ymm0, %zmm0
+; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k1
+; AVX512F-NEXT:    vpcmpgtw %ymm6, %ymm2, %ymm0
+; AVX512F-NEXT:    vpmovsxwd %ymm0, %zmm0
+; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k2
+; AVX512F-NEXT:    vpcmpgtw %ymm7, %ymm3, %ymm0
+; AVX512F-NEXT:    vpmovsxwd %ymm0, %zmm0
+; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k3
+; AVX512F-NEXT:    kmovw %k3, 6(%rdi)
+; AVX512F-NEXT:    kmovw %k2, 4(%rdi)
+; AVX512F-NEXT:    kmovw %k1, 2(%rdi)
+; AVX512F-NEXT:    kmovw %k0, (%rdi)
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512DQ-LABEL: test_cmp_v64i16:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vpcmpgtw %ymm7, %ymm3, %ymm3
-; AVX512DQ-NEXT:    vpcmpgtw %ymm6, %ymm2, %ymm2
-; AVX512DQ-NEXT:    vpcmpgtw %ymm5, %ymm1, %ymm1
+; AVX512DQ-NEXT:    movq %rdi, %rax
 ; AVX512DQ-NEXT:    vpcmpgtw %ymm4, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
-; AVX512DQ-NEXT:    vpmovdb %zmm0, %xmm0
-; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
-; AVX512DQ-NEXT:    vpmovdb %zmm1, %xmm1
-; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
-; AVX512DQ-NEXT:    vpmovdb %zmm2, %xmm2
-; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} zmm3 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
-; AVX512DQ-NEXT:    vpmovdb %zmm3, %xmm3
+; AVX512DQ-NEXT:    vpmovsxwd %ymm0, %zmm0
+; AVX512DQ-NEXT:    vpmovd2m %zmm0, %k0
+; AVX512DQ-NEXT:    vpcmpgtw %ymm5, %ymm1, %ymm0
+; AVX512DQ-NEXT:    vpmovsxwd %ymm0, %zmm0
+; AVX512DQ-NEXT:    vpmovd2m %zmm0, %k1
+; AVX512DQ-NEXT:    vpcmpgtw %ymm6, %ymm2, %ymm0
+; AVX512DQ-NEXT:    vpmovsxwd %ymm0, %zmm0
+; AVX512DQ-NEXT:    vpmovd2m %zmm0, %k2
+; AVX512DQ-NEXT:    vpcmpgtw %ymm7, %ymm3, %ymm0
+; AVX512DQ-NEXT:    vpmovsxwd %ymm0, %zmm0
+; AVX512DQ-NEXT:    vpmovd2m %zmm0, %k3
+; AVX512DQ-NEXT:    kmovw %k3, 6(%rdi)
+; AVX512DQ-NEXT:    kmovw %k2, 4(%rdi)
+; AVX512DQ-NEXT:    kmovw %k1, 2(%rdi)
+; AVX512DQ-NEXT:    kmovw %k0, (%rdi)
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
 ;
@@ -1668,10 +1706,12 @@ define <128 x i1> @test_cmp_v128i8(<128 x i8> %a0, <128 x i8> %a1) nounwind {
 ;
 ; AVX512BW-LABEL: test_cmp_v128i8:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpcmpgtb %zmm3, %zmm1, %k0
-; AVX512BW-NEXT:    vpcmpgtb %zmm2, %zmm0, %k1
-; AVX512BW-NEXT:    vpmovm2b %k1, %zmm0
-; AVX512BW-NEXT:    vpmovm2b %k0, %zmm1
+; AVX512BW-NEXT:    movq %rdi, %rax
+; AVX512BW-NEXT:    vpcmpgtb %zmm2, %zmm0, %k0
+; AVX512BW-NEXT:    vpcmpgtb %zmm3, %zmm1, %k1
+; AVX512BW-NEXT:    kmovq %k1, 8(%rdi)
+; AVX512BW-NEXT:    kmovq %k0, (%rdi)
+; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
   %1 = icmp sgt <128 x i8> %a0, %a1
   ret <128 x i1> %1
diff --git a/llvm/test/CodeGen/X86/vector-constrained-fp-intrinsics-flags.ll b/llvm/test/CodeGen/X86/vector-constrained-fp-intrinsics-flags.ll
new file mode 100644
index 000000000..b1ef02c85
--- /dev/null
+++ b/llvm/test/CodeGen/X86/vector-constrained-fp-intrinsics-flags.ll
@@ -0,0 +1,59 @@
+; RUN: llc -O3 -mtriple=x86_64-pc-linux -stop-after=finalize-isel < %s | FileCheck %s
+
+define <1 x float> @constrained_vector_fadd_v1f32() #0 {
+; CHECK-LABEL: name: constrained_vector_fadd_v1f32
+; CHECK: [[MOVSSrm_alt:%[0-9]+]]:fr32 = MOVSSrm_alt $rip, 1, $noreg, %const.0, $noreg :: (load 4 from constant-pool)
+; CHECK: [[ADDSSrm:%[0-9]+]]:fr32 = ADDSSrm [[MOVSSrm_alt]], $rip, 1, $noreg, %const.1, $noreg, implicit $mxcsr :: (load 4 from constant-pool)
+; CHECK: $xmm0 = COPY [[ADDSSrm]]
+; CHECK: RET 0, $xmm0
+entry:
+  %add = call <1 x float> @llvm.experimental.constrained.fadd.v1f32(<1 x float> <float 0x7FF0000000000000>, <1 x float> <float 1.0>, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret <1 x float> %add
+}
+
+define <3 x float> @constrained_vector_fadd_v3f32() #0 {
+; CHECK-LABEL: name: constrained_vector_fadd_v3f32
+; CHECK: [[FsFLD0SS:%[0-9]+]]:fr32 = FsFLD0SS
+; CHECK: [[MOVSSrm_alt:%[0-9]+]]:fr32 = MOVSSrm_alt $rip, 1, $noreg, %const.0, $noreg :: (load 4 from constant-pool)
+; CHECK: [[ADDSSrr:%[0-9]+]]:fr32 = ADDSSrr [[MOVSSrm_alt]], killed [[FsFLD0SS]], implicit $mxcsr
+; CHECK: [[ADDSSrm:%[0-9]+]]:fr32 = ADDSSrm [[MOVSSrm_alt]], $rip, 1, $noreg, %const.1, $noreg, implicit $mxcsr :: (load 4 from constant-pool)
+; CHECK: [[ADDSSrm1:%[0-9]+]]:fr32 = ADDSSrm [[MOVSSrm_alt]], $rip, 1, $noreg, %const.2, $noreg, implicit $mxcsr :: (load 4 from constant-pool)
+; CHECK: [[COPY:%[0-9]+]]:vr128 = COPY [[ADDSSrm1]]
+; CHECK: [[COPY1:%[0-9]+]]:vr128 = COPY [[ADDSSrm]]
+; CHECK: [[UNPCKLPSrr:%[0-9]+]]:vr128 = UNPCKLPSrr [[COPY1]], killed [[COPY]]
+; CHECK: [[COPY2:%[0-9]+]]:vr128 = COPY [[ADDSSrr]]
+; CHECK: [[UNPCKLPDrr:%[0-9]+]]:vr128 = UNPCKLPDrr [[UNPCKLPSrr]], killed [[COPY2]]
+; CHECK: $xmm0 = COPY [[UNPCKLPDrr]]
+; CHECK: RET 0, $xmm0
+entry:
+  %add = call <3 x float> @llvm.experimental.constrained.fadd.v3f32(
+           <3 x float> <float 0xFFFFFFFFE0000000, float 0xFFFFFFFFE0000000,
+                        float 0xFFFFFFFFE0000000>,
+           <3 x float> <float 2.0, float 1.0, float 0.0>,
+           metadata !"round.dynamic",
+           metadata !"fpexcept.strict") #0
+  ret <3 x float> %add
+}
+
+define <4 x double> @constrained_vector_fadd_v4f64() #0 {
+; CHECK-LABEL: name: constrained_vector_fadd_v4f64
+; CHECK: [[MOVAPDrm:%[0-9]+]]:vr128 = MOVAPDrm $rip, 1, $noreg, %const.0, $noreg :: (load 16 from constant-pool)
+; CHECK: [[ADDPDrm:%[0-9]+]]:vr128 = ADDPDrm [[MOVAPDrm]], $rip, 1, $noreg, %const.1, $noreg, implicit $mxcsr :: (load 16 from constant-pool)
+; CHECK: [[ADDPDrm1:%[0-9]+]]:vr128 = ADDPDrm [[MOVAPDrm]], $rip, 1, $noreg, %const.2, $noreg, implicit $mxcsr :: (load 16 from constant-pool)
+; CHECK: $xmm0 = COPY [[ADDPDrm1]]
+; CHECK: $xmm1 = COPY [[ADDPDrm]]
+; CHECK: RET 0, $xmm0, $xmm1
+entry:
+  %add = call <4 x double> @llvm.experimental.constrained.fadd.v4f64(
+           <4 x double> <double 0x7FEFFFFFFFFFFFFF, double 0x7FEFFFFFFFFFFFFF,
+                         double 0x7FEFFFFFFFFFFFFF, double 0x7FEFFFFFFFFFFFFF>,
+           <4 x double> <double 1.000000e+00, double 1.000000e-01,
+                         double 2.000000e+00, double 2.000000e-01>,
+           metadata !"round.dynamic",
+           metadata !"fpexcept.strict") #0
+  ret <4 x double> %add
+}
+
+declare <1 x float> @llvm.experimental.constrained.fadd.v1f32(<1 x float>, <1 x float>, metadata, metadata)
+declare <3 x float> @llvm.experimental.constrained.fadd.v3f32(<3 x float>, <3 x float>, metadata, metadata)
+declare <4 x double> @llvm.experimental.constrained.fadd.v4f64(<4 x double>, <4 x double>, metadata, metadata)
diff --git a/llvm/test/CodeGen/X86/vector-constrained-fp-intrinsics-fma.ll b/llvm/test/CodeGen/X86/vector-constrained-fp-intrinsics-fma.ll
index e35e76d2f..5f39f33dd 100644
--- a/llvm/test/CodeGen/X86/vector-constrained-fp-intrinsics-fma.ll
+++ b/llvm/test/CodeGen/X86/vector-constrained-fp-intrinsics-fma.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -O3 -mtriple=x86_64-pc-linux -mattr=+fma < %s | FileCheck %s
 
-define <1 x float> @constrained_vector_fma_v1f32() {
+define <1 x float> @constrained_vector_fma_v1f32() #0 {
 ; CHECK-LABEL: constrained_vector_fma_v1f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
@@ -14,11 +14,11 @@ entry:
            <1 x float> <float 2.5>,
            <1 x float> <float 4.5>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #0
   ret <1 x float> %fma
 }
 
-define <2 x double> @constrained_vector_fma_v2f64() {
+define <2 x double> @constrained_vector_fma_v2f64() #0 {
 ; CHECK-LABEL: constrained_vector_fma_v2f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vmovapd {{.*#+}} xmm1 = [1.5E+0,5.0E-1]
@@ -31,11 +31,11 @@ entry:
            <2 x double> <double 3.5, double 2.5>,
            <2 x double> <double 5.5, double 4.5>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #0
   ret <2 x double> %fma
 }
 
-define <3 x float> @constrained_vector_fma_v3f32() {
+define <3 x float> @constrained_vector_fma_v3f32() #0 {
 ; CHECK-LABEL: constrained_vector_fma_v3f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
@@ -56,11 +56,11 @@ entry:
            <3 x float> <float 5.5, float 4.5, float 3.5>,
            <3 x float> <float 8.5, float 7.5, float 6.5>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #0
   ret <3 x float> %fma
 }
 
-define <3 x double> @constrained_vector_fma_v3f64() {
+define <3 x double> @constrained_vector_fma_v3f64() #0 {
 ; CHECK-LABEL: constrained_vector_fma_v3f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
@@ -77,11 +77,11 @@ entry:
            <3 x double> <double 5.5, double 4.5, double 3.5>,
            <3 x double> <double 8.5, double 7.5, double 6.5>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #0
   ret <3 x double> %fma
 }
 
-define <4 x double> @constrained_vector_fma_v4f64() {
+define <4 x double> @constrained_vector_fma_v4f64() #0 {
 ; CHECK-LABEL: constrained_vector_fma_v4f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vmovapd {{.*#+}} ymm1 = [3.5E+0,2.5E+0,1.5E+0,5.0E-1]
@@ -94,11 +94,11 @@ entry:
            <4 x double> <double 7.5, double 6.5, double 5.5, double 4.5>,
            <4 x double> <double 11.5, double 10.5, double 9.5, double 8.5>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #0
   ret <4 x double> %fma
 }
 
-define <4 x float> @constrained_vector_fma_v4f32() {
+define <4 x float> @constrained_vector_fma_v4f32() #0 {
 ; CHECK-LABEL: constrained_vector_fma_v4f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vmovaps {{.*#+}} xmm1 = [3.5E+0,2.5E+0,1.5E+0,5.0E-1]
@@ -111,11 +111,11 @@ entry:
            <4 x float> <float 7.5, float 6.5, float 5.5, float 4.5>,
            <4 x float> <float 11.5, float 10.5, float 9.5, float 8.5>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #0
   ret <4 x float> %fma
 }
 
-define <8 x float> @constrained_vector_fma_v8f32() {
+define <8 x float> @constrained_vector_fma_v8f32() #0 {
 ; CHECK-LABEL: constrained_vector_fma_v8f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vmovaps {{.*#+}} ymm1 = [3.5E+0,2.5E+0,1.5E+0,5.0E-1,7.5E+0,6.5E+0,5.5E+0,4.5E+0]
@@ -131,10 +131,12 @@ entry:
            <8 x float> <float 11.5, float 10.5, float 9.5, float 8.5,
                         float 15.5, float 14.5, float 13.5, float 12.5>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #0
   ret <8 x float> %fma
 }
 
+attributes #0 = { strictfp }
+
 ; Single width declarations
 declare <2 x double> @llvm.experimental.constrained.fma.v2f64(<2 x double>, <2 x double>, <2 x double>, metadata, metadata)
 declare <4 x float> @llvm.experimental.constrained.fma.v4f32(<4 x float>, <4 x float>, <4 x float>, metadata, metadata)
diff --git a/llvm/test/CodeGen/X86/vector-constrained-fp-intrinsics.ll b/llvm/test/CodeGen/X86/vector-constrained-fp-intrinsics.ll
index ec564d135..cd8a06f71 100644
--- a/llvm/test/CodeGen/X86/vector-constrained-fp-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/vector-constrained-fp-intrinsics.ll
@@ -1,8 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -O3 -mtriple=x86_64-pc-linux < %s | FileCheck %s
-; RUN: llc -O3 -mtriple=x86_64-pc-linux -mattr=+avx < %s | FileCheck --check-prefix=AVX %s
+; RUN: llc -O3 -mtriple=x86_64-pc-linux -mattr=+avx < %s | FileCheck %s --check-prefix=AVX --check-prefix=AVX1
+; RUN: llc -O3 -mtriple=x86_64-pc-linux -mattr=+avx512f < %s | FileCheck %s --check-prefix=AVX --check-prefix=AVX512 --check-prefix=AVX512F
+; RUN: llc -O3 -mtriple=x86_64-pc-linux -mattr=+avx512dq < %s | FileCheck %s --check-prefix=AVX --check-prefix=AVX512 --check-prefix=AVX512DQ
 
-define <1 x float> @constrained_vector_fdiv_v1f32() {
+define <1 x float> @constrained_vector_fdiv_v1f32() #0 {
 ; CHECK-LABEL: constrained_vector_fdiv_v1f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
@@ -19,11 +21,11 @@ entry:
            <1 x float> <float 1.000000e+00>,
            <1 x float> <float 1.000000e+01>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #0
   ret <1 x float> %div
 }
 
-define <2 x double> @constrained_vector_fdiv_v2f64() {
+define <2 x double> @constrained_vector_fdiv_v2f64() #0 {
 ; CHECK-LABEL: constrained_vector_fdiv_v2f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    movapd {{.*#+}} xmm0 = [1.0E+0,2.0E+0]
@@ -40,11 +42,11 @@ entry:
            <2 x double> <double 1.000000e+00, double 2.000000e+00>,
            <2 x double> <double 1.000000e+01, double 1.000000e+01>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #0
   ret <2 x double> %div
 }
 
-define <3 x float> @constrained_vector_fdiv_v3f32() {
+define <3 x float> @constrained_vector_fdiv_v3f32() #0 {
 ; CHECK-LABEL: constrained_vector_fdiv_v3f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
@@ -75,11 +77,11 @@ entry:
            <3 x float> <float 1.000000e+00, float 2.000000e+00, float 3.000000e+00>,
            <3 x float> <float 1.000000e+01, float 1.000000e+01, float 1.000000e+01>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #0
   ret <3 x float> %div
 }
 
-define <3 x double> @constrained_vector_fdiv_v3f64() {
+define <3 x double> @constrained_vector_fdiv_v3f64() #0 {
 ; CHECK-LABEL: constrained_vector_fdiv_v3f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    movapd {{.*#+}} xmm0 = [1.0E+0,2.0E+0]
@@ -105,25 +107,32 @@ entry:
            <3 x double> <double 1.000000e+00, double 2.000000e+00, double 3.000000e+00>,
            <3 x double> <double 1.000000e+01, double 1.000000e+01, double 1.000000e+01>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #0
   ret <3 x double> %div
 }
 
-define <4 x double> @constrained_vector_fdiv_v4f64() {
+define <4 x double> @constrained_vector_fdiv_v4f64() #0 {
 ; CHECK-LABEL: constrained_vector_fdiv_v4f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    movapd {{.*#+}} xmm2 = [1.0E+1,1.0E+1]
-; CHECK-NEXT:    movapd {{.*#+}} xmm0 = [1.0E+0,2.0E+0]
-; CHECK-NEXT:    divpd %xmm2, %xmm0
 ; CHECK-NEXT:    movapd {{.*#+}} xmm1 = [3.0E+0,4.0E+0]
 ; CHECK-NEXT:    divpd %xmm2, %xmm1
+; CHECK-NEXT:    movapd {{.*#+}} xmm0 = [1.0E+0,2.0E+0]
+; CHECK-NEXT:    divpd %xmm2, %xmm0
 ; CHECK-NEXT:    retq
 ;
-; AVX-LABEL: constrained_vector_fdiv_v4f64:
-; AVX:       # %bb.0: # %entry
-; AVX-NEXT:    vmovapd {{.*#+}} ymm0 = [1.0E+0,2.0E+0,3.0E+0,4.0E+0]
-; AVX-NEXT:    vdivpd {{.*}}(%rip), %ymm0, %ymm0
-; AVX-NEXT:    retq
+; AVX1-LABEL: constrained_vector_fdiv_v4f64:
+; AVX1:       # %bb.0: # %entry
+; AVX1-NEXT:    vmovapd {{.*#+}} ymm0 = [1.0E+0,2.0E+0,3.0E+0,4.0E+0]
+; AVX1-NEXT:    vdivpd {{.*}}(%rip), %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX512-LABEL: constrained_vector_fdiv_v4f64:
+; AVX512:       # %bb.0: # %entry
+; AVX512-NEXT:    vbroadcastsd {{.*#+}} ymm0 = [1.0E+1,1.0E+1,1.0E+1,1.0E+1]
+; AVX512-NEXT:    vmovapd {{.*#+}} ymm1 = [1.0E+0,2.0E+0,3.0E+0,4.0E+0]
+; AVX512-NEXT:    vdivpd %ymm0, %ymm1, %ymm0
+; AVX512-NEXT:    retq
 entry:
   %div = call <4 x double> @llvm.experimental.constrained.fdiv.v4f64(
            <4 x double> <double 1.000000e+00, double 2.000000e+00,
@@ -131,11 +140,11 @@ entry:
            <4 x double> <double 1.000000e+01, double 1.000000e+01,
                          double 1.000000e+01, double 1.000000e+01>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #0
   ret <4 x double> %div
 }
 
-define <1 x float> @constrained_vector_frem_v1f32() {
+define <1 x float> @constrained_vector_frem_v1f32() #0 {
 ; CHECK-LABEL: constrained_vector_frem_v1f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    pushq %rax
@@ -162,11 +171,11 @@ entry:
            <1 x float> <float 1.000000e+00>,
            <1 x float> <float 1.000000e+01>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #0
   ret <1 x float> %rem
 }
 
-define <2 x double> @constrained_vector_frem_v2f64() {
+define <2 x double> @constrained_vector_frem_v2f64() #0 {
 ; CHECK-LABEL: constrained_vector_frem_v2f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    subq $24, %rsp
@@ -205,11 +214,11 @@ entry:
            <2 x double> <double 1.000000e+00, double 2.000000e+00>,
            <2 x double> <double 1.000000e+01, double 1.000000e+01>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #0
   ret <2 x double> %rem
 }
 
-define <3 x float> @constrained_vector_frem_v3f32() {
+define <3 x float> @constrained_vector_frem_v3f32() #0 {
 ; CHECK-LABEL: constrained_vector_frem_v3f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    subq $40, %rsp
@@ -261,11 +270,11 @@ entry:
            <3 x float> <float 1.000000e+00, float 2.000000e+00, float 3.000000e+00>,
            <3 x float> <float 1.000000e+01, float 1.000000e+01, float 1.000000e+01>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #0
   ret <3 x float> %rem
 }
 
-define <3 x double> @constrained_vector_frem_v3f64() {
+define <3 x double> @constrained_vector_frem_v3f64() #0 {
 ; CHECK-LABEL: constrained_vector_frem_v3f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    subq $24, %rsp
@@ -283,9 +292,9 @@ define <3 x double> @constrained_vector_frem_v3f64() {
 ; CHECK-NEXT:    callq fmod
 ; CHECK-NEXT:    movsd %xmm0, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    fldl {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Reload
+; CHECK-NEXT:    movsd (%rsp), %xmm0 # 8-byte Reload
 ; CHECK-NEXT:    # xmm0 = mem[0],zero
-; CHECK-NEXT:    movsd (%rsp), %xmm1 # 8-byte Reload
+; CHECK-NEXT:    movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 8-byte Reload
 ; CHECK-NEXT:    # xmm1 = mem[0],zero
 ; CHECK-NEXT:    addq $24, %rsp
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
@@ -319,11 +328,11 @@ entry:
            <3 x double> <double 1.000000e+00, double 2.000000e+00, double 3.000000e+00>,
            <3 x double> <double 1.000000e+01, double 1.000000e+01, double 1.000000e+01>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #0
   ret <3 x double> %rem
 }
 
-define <4 x double> @constrained_vector_frem_v4f64() {
+define <4 x double> @constrained_vector_frem_v4f64() #0 {
 ; CHECK-LABEL: constrained_vector_frem_v4f64:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    subq $40, %rsp
@@ -386,11 +395,11 @@ define <4 x double> @constrained_vector_frem_v4f64() {
            <4 x double> <double 1.000000e+01, double 1.000000e+01,
                          double 1.000000e+01, double 1.000000e+01>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #0
   ret <4 x double> %rem
 }
 
-define <1 x float> @constrained_vector_fmul_v1f32() {
+define <1 x float> @constrained_vector_fmul_v1f32() #0 {
 ; CHECK-LABEL: constrained_vector_fmul_v1f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
@@ -407,11 +416,11 @@ entry:
            <1 x float> <float 0x7FF0000000000000>,
            <1 x float> <float 2.000000e+00>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #0
   ret <1 x float> %mul
 }
 
-define <2 x double> @constrained_vector_fmul_v2f64() {
+define <2 x double> @constrained_vector_fmul_v2f64() #0 {
 ; CHECK-LABEL: constrained_vector_fmul_v2f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    movapd {{.*#+}} xmm0 = [1.7976931348623157E+308,1.7976931348623157E+308]
@@ -428,11 +437,11 @@ entry:
            <2 x double> <double 0x7FEFFFFFFFFFFFFF, double 0x7FEFFFFFFFFFFFFF>,
            <2 x double> <double 2.000000e+00, double 3.000000e+00>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #0
   ret <2 x double> %mul
 }
 
-define <3 x float> @constrained_vector_fmul_v3f32() {
+define <3 x float> @constrained_vector_fmul_v3f32() #0 {
 ; CHECK-LABEL: constrained_vector_fmul_v3f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
@@ -460,11 +469,11 @@ entry:
                         float 0x7FF0000000000000>,
            <3 x float> <float 1.000000e+00, float 1.000000e+01, float 1.000000e+02>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #0
   ret <3 x float> %mul
 }
 
-define <3 x double> @constrained_vector_fmul_v3f64() {
+define <3 x double> @constrained_vector_fmul_v3f64() #0 {
 ; CHECK-LABEL: constrained_vector_fmul_v3f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    movapd {{.*#+}} xmm0 = [1.7976931348623157E+308,1.7976931348623157E+308]
@@ -491,24 +500,30 @@ entry:
                          double 0x7FEFFFFFFFFFFFFF>,
            <3 x double> <double 1.000000e+00, double 1.000000e+01, double 1.000000e+02>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #0
   ret <3 x double> %mul
 }
 
-define <4 x double> @constrained_vector_fmul_v4f64() {
+define <4 x double> @constrained_vector_fmul_v4f64() #0 {
 ; CHECK-LABEL: constrained_vector_fmul_v4f64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    movapd {{.*#+}} xmm1 = [1.7976931348623157E+308,1.7976931348623157E+308]
-; CHECK-NEXT:    movapd {{.*#+}} xmm0 = [2.0E+0,3.0E+0]
-; CHECK-NEXT:    mulpd %xmm1, %xmm0
-; CHECK-NEXT:    mulpd {{.*}}(%rip), %xmm1
+; CHECK-NEXT:    movapd {{.*#+}} xmm0 = [1.7976931348623157E+308,1.7976931348623157E+308]
+; CHECK-NEXT:    movapd {{.*#+}} xmm1 = [4.0E+0,5.0E+0]
+; CHECK-NEXT:    mulpd %xmm0, %xmm1
+; CHECK-NEXT:    mulpd {{.*}}(%rip), %xmm0
 ; CHECK-NEXT:    retq
 ;
-; AVX-LABEL: constrained_vector_fmul_v4f64:
-; AVX:       # %bb.0: # %entry
-; AVX-NEXT:    vmovapd {{.*#+}} ymm0 = [1.7976931348623157E+308,1.7976931348623157E+308,1.7976931348623157E+308,1.7976931348623157E+308]
-; AVX-NEXT:    vmulpd {{.*}}(%rip), %ymm0, %ymm0
-; AVX-NEXT:    retq
+; AVX1-LABEL: constrained_vector_fmul_v4f64:
+; AVX1:       # %bb.0: # %entry
+; AVX1-NEXT:    vmovapd {{.*#+}} ymm0 = [1.7976931348623157E+308,1.7976931348623157E+308,1.7976931348623157E+308,1.7976931348623157E+308]
+; AVX1-NEXT:    vmulpd {{.*}}(%rip), %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX512-LABEL: constrained_vector_fmul_v4f64:
+; AVX512:       # %bb.0: # %entry
+; AVX512-NEXT:    vbroadcastsd {{.*#+}} ymm0 = [1.7976931348623157E+308,1.7976931348623157E+308,1.7976931348623157E+308,1.7976931348623157E+308]
+; AVX512-NEXT:    vmulpd {{.*}}(%rip), %ymm0, %ymm0
+; AVX512-NEXT:    retq
 entry:
   %mul = call <4 x double> @llvm.experimental.constrained.fmul.v4f64(
            <4 x double> <double 0x7FEFFFFFFFFFFFFF, double 0x7FEFFFFFFFFFFFFF,
@@ -516,11 +531,11 @@ entry:
            <4 x double> <double 2.000000e+00, double 3.000000e+00,
                          double 4.000000e+00, double 5.000000e+00>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #0
   ret <4 x double> %mul
 }
 
-define <1 x float> @constrained_vector_fadd_v1f32() {
+define <1 x float> @constrained_vector_fadd_v1f32() #0 {
 ; CHECK-LABEL: constrained_vector_fadd_v1f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
@@ -537,11 +552,11 @@ entry:
            <1 x float> <float 0x7FF0000000000000>,
            <1 x float> <float 1.0>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #0
   ret <1 x float> %add
 }
 
-define <2 x double> @constrained_vector_fadd_v2f64() {
+define <2 x double> @constrained_vector_fadd_v2f64() #0 {
 ; CHECK-LABEL: constrained_vector_fadd_v2f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    movapd {{.*#+}} xmm0 = [1.7976931348623157E+308,1.7976931348623157E+308]
@@ -558,11 +573,11 @@ entry:
            <2 x double> <double 0x7FEFFFFFFFFFFFFF, double 0x7FEFFFFFFFFFFFFF>,
            <2 x double> <double 1.000000e+00, double 1.000000e-01>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #0
   ret <2 x double> %add
 }
 
-define <3 x float> @constrained_vector_fadd_v3f32() {
+define <3 x float> @constrained_vector_fadd_v3f32() #0 {
 ; CHECK-LABEL: constrained_vector_fadd_v3f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    xorps %xmm1, %xmm1
@@ -591,11 +606,11 @@ entry:
                         float 0xFFFFFFFFE0000000>,
            <3 x float> <float 2.0, float 1.0, float 0.0>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #0
   ret <3 x float> %add
 }
 
-define <3 x double> @constrained_vector_fadd_v3f64() {
+define <3 x double> @constrained_vector_fadd_v3f64() #0 {
 ; CHECK-LABEL: constrained_vector_fadd_v3f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    movapd {{.*#+}} xmm0 = [1.7976931348623157E+308,1.7976931348623157E+308]
@@ -622,24 +637,30 @@ entry:
                          double 0x7FEFFFFFFFFFFFFF>,
            <3 x double> <double 2.0, double 1.0, double 0.0>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #0
   ret <3 x double> %add
 }
 
-define <4 x double> @constrained_vector_fadd_v4f64() {
+define <4 x double> @constrained_vector_fadd_v4f64() #0 {
 ; CHECK-LABEL: constrained_vector_fadd_v4f64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    movapd {{.*#+}} xmm1 = [1.7976931348623157E+308,1.7976931348623157E+308]
-; CHECK-NEXT:    movapd {{.*#+}} xmm0 = [1.0E+0,1.0000000000000001E-1]
-; CHECK-NEXT:    addpd %xmm1, %xmm0
-; CHECK-NEXT:    addpd {{.*}}(%rip), %xmm1
+; CHECK-NEXT:    movapd {{.*#+}} xmm0 = [1.7976931348623157E+308,1.7976931348623157E+308]
+; CHECK-NEXT:    movapd {{.*#+}} xmm1 = [2.0E+0,2.0000000000000001E-1]
+; CHECK-NEXT:    addpd %xmm0, %xmm1
+; CHECK-NEXT:    addpd {{.*}}(%rip), %xmm0
 ; CHECK-NEXT:    retq
 ;
-; AVX-LABEL: constrained_vector_fadd_v4f64:
-; AVX:       # %bb.0: # %entry
-; AVX-NEXT:    vmovapd {{.*#+}} ymm0 = [1.7976931348623157E+308,1.7976931348623157E+308,1.7976931348623157E+308,1.7976931348623157E+308]
-; AVX-NEXT:    vaddpd {{.*}}(%rip), %ymm0, %ymm0
-; AVX-NEXT:    retq
+; AVX1-LABEL: constrained_vector_fadd_v4f64:
+; AVX1:       # %bb.0: # %entry
+; AVX1-NEXT:    vmovapd {{.*#+}} ymm0 = [1.7976931348623157E+308,1.7976931348623157E+308,1.7976931348623157E+308,1.7976931348623157E+308]
+; AVX1-NEXT:    vaddpd {{.*}}(%rip), %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX512-LABEL: constrained_vector_fadd_v4f64:
+; AVX512:       # %bb.0: # %entry
+; AVX512-NEXT:    vbroadcastsd {{.*#+}} ymm0 = [1.7976931348623157E+308,1.7976931348623157E+308,1.7976931348623157E+308,1.7976931348623157E+308]
+; AVX512-NEXT:    vaddpd {{.*}}(%rip), %ymm0, %ymm0
+; AVX512-NEXT:    retq
 entry:
   %add = call <4 x double> @llvm.experimental.constrained.fadd.v4f64(
            <4 x double> <double 0x7FEFFFFFFFFFFFFF, double 0x7FEFFFFFFFFFFFFF,
@@ -647,11 +668,11 @@ entry:
            <4 x double> <double 1.000000e+00, double 1.000000e-01,
                          double 2.000000e+00, double 2.000000e-01>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #0
   ret <4 x double> %add
 }
 
-define <1 x float> @constrained_vector_fsub_v1f32() {
+define <1 x float> @constrained_vector_fsub_v1f32() #0 {
 ; CHECK-LABEL: constrained_vector_fsub_v1f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
@@ -668,11 +689,11 @@ entry:
            <1 x float> <float 0x7FF0000000000000>,
            <1 x float> <float 1.000000e+00>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #0
   ret <1 x float> %sub
 }
 
-define <2 x double> @constrained_vector_fsub_v2f64() {
+define <2 x double> @constrained_vector_fsub_v2f64() #0 {
 ; CHECK-LABEL: constrained_vector_fsub_v2f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    movapd {{.*#+}} xmm0 = [-1.7976931348623157E+308,-1.7976931348623157E+308]
@@ -689,11 +710,11 @@ entry:
            <2 x double> <double 0xFFEFFFFFFFFFFFFF, double 0xFFEFFFFFFFFFFFFF>,
            <2 x double> <double 1.000000e+00, double 1.000000e-01>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #0
   ret <2 x double> %sub
 }
 
-define <3 x float> @constrained_vector_fsub_v3f32() {
+define <3 x float> @constrained_vector_fsub_v3f32() #0 {
 ; CHECK-LABEL: constrained_vector_fsub_v3f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    xorps %xmm0, %xmm0
@@ -723,11 +744,11 @@ entry:
                         float 0xFFFFFFFFE0000000>,
            <3 x float> <float 2.0, float 1.0, float 0.0>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #0
   ret <3 x float> %sub
 }
 
-define <3 x double> @constrained_vector_fsub_v3f64() {
+define <3 x double> @constrained_vector_fsub_v3f64() #0 {
 ; CHECK-LABEL: constrained_vector_fsub_v3f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    xorpd %xmm0, %xmm0
@@ -756,24 +777,30 @@ entry:
                          double 0xFFEFFFFFFFFFFFFF>,
            <3 x double> <double 2.0, double 1.0, double 0.0>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #0
   ret <3 x double> %sub
 }
 
-define <4 x double> @constrained_vector_fsub_v4f64() {
+define <4 x double> @constrained_vector_fsub_v4f64() #0 {
 ; CHECK-LABEL: constrained_vector_fsub_v4f64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    movapd {{.*#+}} xmm1 = [-1.7976931348623157E+308,-1.7976931348623157E+308]
-; CHECK-NEXT:    movapd %xmm1, %xmm0
-; CHECK-NEXT:    subpd {{.*}}(%rip), %xmm0
+; CHECK-NEXT:    movapd {{.*#+}} xmm0 = [-1.7976931348623157E+308,-1.7976931348623157E+308]
+; CHECK-NEXT:    movapd %xmm0, %xmm1
 ; CHECK-NEXT:    subpd {{.*}}(%rip), %xmm1
+; CHECK-NEXT:    subpd {{.*}}(%rip), %xmm0
 ; CHECK-NEXT:    retq
 ;
-; AVX-LABEL: constrained_vector_fsub_v4f64:
-; AVX:       # %bb.0: # %entry
-; AVX-NEXT:    vmovapd {{.*#+}} ymm0 = [-1.7976931348623157E+308,-1.7976931348623157E+308,-1.7976931348623157E+308,-1.7976931348623157E+308]
-; AVX-NEXT:    vsubpd {{.*}}(%rip), %ymm0, %ymm0
-; AVX-NEXT:    retq
+; AVX1-LABEL: constrained_vector_fsub_v4f64:
+; AVX1:       # %bb.0: # %entry
+; AVX1-NEXT:    vmovapd {{.*#+}} ymm0 = [-1.7976931348623157E+308,-1.7976931348623157E+308,-1.7976931348623157E+308,-1.7976931348623157E+308]
+; AVX1-NEXT:    vsubpd {{.*}}(%rip), %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX512-LABEL: constrained_vector_fsub_v4f64:
+; AVX512:       # %bb.0: # %entry
+; AVX512-NEXT:    vbroadcastsd {{.*#+}} ymm0 = [-1.7976931348623157E+308,-1.7976931348623157E+308,-1.7976931348623157E+308,-1.7976931348623157E+308]
+; AVX512-NEXT:    vsubpd {{.*}}(%rip), %ymm0, %ymm0
+; AVX512-NEXT:    retq
 entry:
   %sub = call <4 x double> @llvm.experimental.constrained.fsub.v4f64(
            <4 x double> <double 0xFFEFFFFFFFFFFFFF, double 0xFFEFFFFFFFFFFFFF,
@@ -781,11 +808,11 @@ entry:
            <4 x double> <double 1.000000e+00, double 1.000000e-01,
                          double 2.000000e+00, double 2.000000e-01>,
            metadata !"round.dynamic",
-           metadata !"fpexcept.strict")
+           metadata !"fpexcept.strict") #0
   ret <4 x double> %sub
 }
 
-define <1 x float> @constrained_vector_sqrt_v1f32() {
+define <1 x float> @constrained_vector_sqrt_v1f32() #0 {
 ; CHECK-LABEL: constrained_vector_sqrt_v1f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
@@ -801,11 +828,11 @@ entry:
   %sqrt = call <1 x float> @llvm.experimental.constrained.sqrt.v1f32(
                               <1 x float> <float 42.0>,
                               metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #0
   ret <1 x float> %sqrt
 }
 
-define <2 x double> @constrained_vector_sqrt_v2f64() {
+define <2 x double> @constrained_vector_sqrt_v2f64() #0 {
 ; CHECK-LABEL: constrained_vector_sqrt_v2f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    sqrtpd {{.*}}(%rip), %xmm0
@@ -819,11 +846,11 @@ entry:
   %sqrt = call <2 x double> @llvm.experimental.constrained.sqrt.v2f64(
                               <2 x double> <double 42.0, double 42.1>,
                               metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #0
   ret <2 x double> %sqrt
 }
 
-define <3 x float> @constrained_vector_sqrt_v3f32() {
+define <3 x float> @constrained_vector_sqrt_v3f32() #0 {
 ; CHECK-LABEL: constrained_vector_sqrt_v3f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
@@ -851,11 +878,11 @@ entry:
   %sqrt = call <3 x float> @llvm.experimental.constrained.sqrt.v3f32(
                               <3 x float> <float 42.0, float 43.0, float 44.0>,
                               metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #0
   ret <3 x float> %sqrt
 }
 
-define <3 x double> @constrained_vector_sqrt_v3f64() {
+define <3 x double> @constrained_vector_sqrt_v3f64() #0 {
 ; CHECK-LABEL: constrained_vector_sqrt_v3f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
@@ -878,15 +905,15 @@ entry:
   %sqrt = call <3 x double> @llvm.experimental.constrained.sqrt.v3f64(
                           <3 x double> <double 42.0, double 42.1, double 42.2>,
                           metadata !"round.dynamic",
-                          metadata !"fpexcept.strict")
+                          metadata !"fpexcept.strict") #0
   ret <3 x double> %sqrt
 }
 
-define <4 x double> @constrained_vector_sqrt_v4f64() {
+define <4 x double> @constrained_vector_sqrt_v4f64() #0 {
 ; CHECK-LABEL: constrained_vector_sqrt_v4f64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    sqrtpd {{.*}}(%rip), %xmm0
 ; CHECK-NEXT:    sqrtpd {{.*}}(%rip), %xmm1
+; CHECK-NEXT:    sqrtpd {{.*}}(%rip), %xmm0
 ; CHECK-NEXT:    retq
 ;
 ; AVX-LABEL: constrained_vector_sqrt_v4f64:
@@ -898,11 +925,11 @@ define <4 x double> @constrained_vector_sqrt_v4f64() {
                               <4 x double> <double 42.0, double 42.1,
                                             double 42.2, double 42.3>,
                               metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #0
   ret <4 x double> %sqrt
 }
 
-define <1 x float> @constrained_vector_pow_v1f32() {
+define <1 x float> @constrained_vector_pow_v1f32() #0 {
 ; CHECK-LABEL: constrained_vector_pow_v1f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    pushq %rax
@@ -929,11 +956,11 @@ entry:
                              <1 x float> <float 42.0>,
                              <1 x float> <float 3.0>,
                              metadata !"round.dynamic",
-                             metadata !"fpexcept.strict")
+                             metadata !"fpexcept.strict") #0
   ret <1 x float> %pow
 }
 
-define <2 x double> @constrained_vector_pow_v2f64() {
+define <2 x double> @constrained_vector_pow_v2f64() #0 {
 ; CHECK-LABEL: constrained_vector_pow_v2f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    subq $24, %rsp
@@ -972,11 +999,11 @@ entry:
                              <2 x double> <double 42.1, double 42.2>,
                              <2 x double> <double 3.0, double 3.0>,
                              metadata !"round.dynamic",
-                             metadata !"fpexcept.strict")
+                             metadata !"fpexcept.strict") #0
   ret <2 x double> %pow
 }
 
-define <3 x float> @constrained_vector_pow_v3f32() {
+define <3 x float> @constrained_vector_pow_v3f32() #0 {
 ; CHECK-LABEL: constrained_vector_pow_v3f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    subq $40, %rsp
@@ -1028,11 +1055,11 @@ entry:
                              <3 x float> <float 42.0, float 43.0, float 44.0>,
                              <3 x float> <float 3.0, float 3.0, float 3.0>,
                              metadata !"round.dynamic",
-                             metadata !"fpexcept.strict")
+                             metadata !"fpexcept.strict") #0
   ret <3 x float> %pow
 }
 
-define <3 x double> @constrained_vector_pow_v3f64() {
+define <3 x double> @constrained_vector_pow_v3f64() #0 {
 ; CHECK-LABEL: constrained_vector_pow_v3f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    subq $24, %rsp
@@ -1050,9 +1077,9 @@ define <3 x double> @constrained_vector_pow_v3f64() {
 ; CHECK-NEXT:    callq pow
 ; CHECK-NEXT:    movsd %xmm0, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    fldl {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Reload
+; CHECK-NEXT:    movsd (%rsp), %xmm0 # 8-byte Reload
 ; CHECK-NEXT:    # xmm0 = mem[0],zero
-; CHECK-NEXT:    movsd (%rsp), %xmm1 # 8-byte Reload
+; CHECK-NEXT:    movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 8-byte Reload
 ; CHECK-NEXT:    # xmm1 = mem[0],zero
 ; CHECK-NEXT:    addq $24, %rsp
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
@@ -1086,11 +1113,11 @@ entry:
                           <3 x double> <double 42.0, double 42.1, double 42.2>,
                           <3 x double> <double 3.0, double 3.0, double 3.0>,
                           metadata !"round.dynamic",
-                          metadata !"fpexcept.strict")
+                          metadata !"fpexcept.strict") #0
   ret <3 x double> %pow
 }
 
-define <4 x double> @constrained_vector_pow_v4f64() {
+define <4 x double> @constrained_vector_pow_v4f64() #0 {
 ; CHECK-LABEL: constrained_vector_pow_v4f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    subq $40, %rsp
@@ -1154,11 +1181,11 @@ entry:
                              <4 x double> <double 3.0, double 3.0,
                                            double 3.0, double 3.0>,
                              metadata !"round.dynamic",
-                             metadata !"fpexcept.strict")
+                             metadata !"fpexcept.strict") #0
   ret <4 x double> %pow
 }
 
-define <1 x float> @constrained_vector_powi_v1f32() {
+define <1 x float> @constrained_vector_powi_v1f32() #0 {
 ; CHECK-LABEL: constrained_vector_powi_v1f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    pushq %rax
@@ -1185,11 +1212,11 @@ entry:
                               <1 x float> <float 42.0>,
                               i32 3,
                               metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #0
   ret <1 x float> %powi
 }
 
-define <2 x double> @constrained_vector_powi_v2f64() {
+define <2 x double> @constrained_vector_powi_v2f64() #0 {
 ; CHECK-LABEL: constrained_vector_powi_v2f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    subq $24, %rsp
@@ -1228,11 +1255,11 @@ entry:
                               <2 x double> <double 42.1, double 42.2>,
                               i32 3,
                               metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #0
   ret <2 x double> %powi
 }
 
-define <3 x float> @constrained_vector_powi_v3f32() {
+define <3 x float> @constrained_vector_powi_v3f32() #0 {
 ; CHECK-LABEL: constrained_vector_powi_v3f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    subq $40, %rsp
@@ -1284,11 +1311,11 @@ entry:
                               <3 x float> <float 42.0, float 43.0, float 44.0>,
                               i32 3,
                               metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #0
   ret <3 x float> %powi
 }
 
-define <3 x double> @constrained_vector_powi_v3f64() {
+define <3 x double> @constrained_vector_powi_v3f64() #0 {
 ; CHECK-LABEL: constrained_vector_powi_v3f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    subq $24, %rsp
@@ -1306,9 +1333,9 @@ define <3 x double> @constrained_vector_powi_v3f64() {
 ; CHECK-NEXT:    callq __powidf2
 ; CHECK-NEXT:    movsd %xmm0, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    fldl {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Reload
+; CHECK-NEXT:    movsd (%rsp), %xmm0 # 8-byte Reload
 ; CHECK-NEXT:    # xmm0 = mem[0],zero
-; CHECK-NEXT:    movsd (%rsp), %xmm1 # 8-byte Reload
+; CHECK-NEXT:    movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 8-byte Reload
 ; CHECK-NEXT:    # xmm1 = mem[0],zero
 ; CHECK-NEXT:    addq $24, %rsp
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
@@ -1342,11 +1369,11 @@ entry:
                           <3 x double> <double 42.0, double 42.1, double 42.2>,
                           i32 3,
                           metadata !"round.dynamic",
-                          metadata !"fpexcept.strict")
+                          metadata !"fpexcept.strict") #0
   ret <3 x double> %powi
 }
 
-define <4 x double> @constrained_vector_powi_v4f64() {
+define <4 x double> @constrained_vector_powi_v4f64() #0 {
 ; CHECK-LABEL: constrained_vector_powi_v4f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    subq $40, %rsp
@@ -1409,11 +1436,11 @@ entry:
                                             double 42.3, double 42.4>,
                               i32 3,
                               metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #0
   ret <4 x double> %powi
 }
 
-define <1 x float> @constrained_vector_sin_v1f32() {
+define <1 x float> @constrained_vector_sin_v1f32() #0 {
 ; CHECK-LABEL: constrained_vector_sin_v1f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    pushq %rax
@@ -1437,11 +1464,11 @@ entry:
   %sin = call <1 x float> @llvm.experimental.constrained.sin.v1f32(
                              <1 x float> <float 42.0>,
                              metadata !"round.dynamic",
-                             metadata !"fpexcept.strict")
+                             metadata !"fpexcept.strict") #0
   ret <1 x float> %sin
 }
 
-define <2 x double> @constrained_vector_sin_v2f64() {
+define <2 x double> @constrained_vector_sin_v2f64() #0 {
 ; CHECK-LABEL: constrained_vector_sin_v2f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    subq $24, %rsp
@@ -1475,11 +1502,11 @@ entry:
   %sin = call <2 x double> @llvm.experimental.constrained.sin.v2f64(
                              <2 x double> <double 42.0, double 42.1>,
                              metadata !"round.dynamic",
-                             metadata !"fpexcept.strict")
+                             metadata !"fpexcept.strict") #0
   ret <2 x double> %sin
 }
 
-define <3 x float> @constrained_vector_sin_v3f32() {
+define <3 x float> @constrained_vector_sin_v3f32() #0 {
 ; CHECK-LABEL: constrained_vector_sin_v3f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    subq $40, %rsp
@@ -1524,11 +1551,11 @@ entry:
   %sin = call <3 x float> @llvm.experimental.constrained.sin.v3f32(
                               <3 x float> <float 42.0, float 43.0, float 44.0>,
                               metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #0
   ret <3 x float> %sin
 }
 
-define <3 x double> @constrained_vector_sin_v3f64() {
+define <3 x double> @constrained_vector_sin_v3f64() #0 {
 ; CHECK-LABEL: constrained_vector_sin_v3f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    subq $24, %rsp
@@ -1543,9 +1570,9 @@ define <3 x double> @constrained_vector_sin_v3f64() {
 ; CHECK-NEXT:    callq sin
 ; CHECK-NEXT:    movsd %xmm0, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    fldl {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Reload
+; CHECK-NEXT:    movsd (%rsp), %xmm0 # 8-byte Reload
 ; CHECK-NEXT:    # xmm0 = mem[0],zero
-; CHECK-NEXT:    movsd (%rsp), %xmm1 # 8-byte Reload
+; CHECK-NEXT:    movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 8-byte Reload
 ; CHECK-NEXT:    # xmm1 = mem[0],zero
 ; CHECK-NEXT:    addq $24, %rsp
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
@@ -1575,11 +1602,11 @@ entry:
   %sin = call <3 x double> @llvm.experimental.constrained.sin.v3f64(
                           <3 x double> <double 42.0, double 42.1, double 42.2>,
                           metadata !"round.dynamic",
-                          metadata !"fpexcept.strict")
+                          metadata !"fpexcept.strict") #0
   ret <3 x double> %sin
 }
 
-define <4 x double> @constrained_vector_sin_v4f64() {
+define <4 x double> @constrained_vector_sin_v4f64() #0 {
 ; CHECK-LABEL: constrained_vector_sin_v4f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    subq $40, %rsp
@@ -1633,11 +1660,11 @@ entry:
                              <4 x double> <double 42.0, double 42.1,
                                            double 42.2, double 42.3>,
                              metadata !"round.dynamic",
-                             metadata !"fpexcept.strict")
+                             metadata !"fpexcept.strict") #0
   ret <4 x double> %sin
 }
 
-define <1 x float> @constrained_vector_cos_v1f32() {
+define <1 x float> @constrained_vector_cos_v1f32() #0 {
 ; CHECK-LABEL: constrained_vector_cos_v1f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    pushq %rax
@@ -1661,11 +1688,11 @@ entry:
   %cos = call <1 x float> @llvm.experimental.constrained.cos.v1f32(
                              <1 x float> <float 42.0>,
                              metadata !"round.dynamic",
-                             metadata !"fpexcept.strict")
+                             metadata !"fpexcept.strict") #0
   ret <1 x float> %cos
 }
 
-define <2 x double> @constrained_vector_cos_v2f64() {
+define <2 x double> @constrained_vector_cos_v2f64() #0 {
 ; CHECK-LABEL: constrained_vector_cos_v2f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    subq $24, %rsp
@@ -1699,11 +1726,11 @@ entry:
   %cos = call <2 x double> @llvm.experimental.constrained.cos.v2f64(
                              <2 x double> <double 42.0, double 42.1>,
                              metadata !"round.dynamic",
-                             metadata !"fpexcept.strict")
+                             metadata !"fpexcept.strict") #0
   ret <2 x double> %cos
 }
 
-define <3 x float> @constrained_vector_cos_v3f32() {
+define <3 x float> @constrained_vector_cos_v3f32() #0 {
 ; CHECK-LABEL: constrained_vector_cos_v3f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    subq $40, %rsp
@@ -1748,11 +1775,11 @@ entry:
   %cos = call <3 x float> @llvm.experimental.constrained.cos.v3f32(
                               <3 x float> <float 42.0, float 43.0, float 44.0>,
                               metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #0
   ret <3 x float> %cos
 }
 
-define <3 x double> @constrained_vector_cos_v3f64() {
+define <3 x double> @constrained_vector_cos_v3f64() #0 {
 ; CHECK-LABEL: constrained_vector_cos_v3f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    subq $24, %rsp
@@ -1767,9 +1794,9 @@ define <3 x double> @constrained_vector_cos_v3f64() {
 ; CHECK-NEXT:    callq cos
 ; CHECK-NEXT:    movsd %xmm0, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    fldl {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Reload
+; CHECK-NEXT:    movsd (%rsp), %xmm0 # 8-byte Reload
 ; CHECK-NEXT:    # xmm0 = mem[0],zero
-; CHECK-NEXT:    movsd (%rsp), %xmm1 # 8-byte Reload
+; CHECK-NEXT:    movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 8-byte Reload
 ; CHECK-NEXT:    # xmm1 = mem[0],zero
 ; CHECK-NEXT:    addq $24, %rsp
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
@@ -1799,11 +1826,11 @@ entry:
   %cos = call <3 x double> @llvm.experimental.constrained.cos.v3f64(
                           <3 x double> <double 42.0, double 42.1, double 42.2>,
                           metadata !"round.dynamic",
-                          metadata !"fpexcept.strict")
+                          metadata !"fpexcept.strict") #0
   ret <3 x double> %cos
 }
 
-define <4 x double> @constrained_vector_cos_v4f64() {
+define <4 x double> @constrained_vector_cos_v4f64() #0 {
 ; CHECK-LABEL: constrained_vector_cos_v4f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    subq $40, %rsp
@@ -1857,11 +1884,11 @@ entry:
                              <4 x double> <double 42.0, double 42.1,
                                            double 42.2, double 42.3>,
                              metadata !"round.dynamic",
-                             metadata !"fpexcept.strict")
+                             metadata !"fpexcept.strict") #0
   ret <4 x double> %cos
 }
 
-define <1 x float> @constrained_vector_exp_v1f32() {
+define <1 x float> @constrained_vector_exp_v1f32() #0 {
 ; CHECK-LABEL: constrained_vector_exp_v1f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    pushq %rax
@@ -1885,11 +1912,11 @@ entry:
   %exp = call <1 x float> @llvm.experimental.constrained.exp.v1f32(
                              <1 x float> <float 42.0>,
                              metadata !"round.dynamic",
-                             metadata !"fpexcept.strict")
+                             metadata !"fpexcept.strict") #0
   ret <1 x float> %exp
 }
 
-define <2 x double> @constrained_vector_exp_v2f64() {
+define <2 x double> @constrained_vector_exp_v2f64() #0 {
 ; CHECK-LABEL: constrained_vector_exp_v2f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    subq $24, %rsp
@@ -1923,11 +1950,11 @@ entry:
   %exp = call <2 x double> @llvm.experimental.constrained.exp.v2f64(
                              <2 x double> <double 42.0, double 42.1>,
                              metadata !"round.dynamic",
-                             metadata !"fpexcept.strict")
+                             metadata !"fpexcept.strict") #0
   ret <2 x double> %exp
 }
 
-define <3 x float> @constrained_vector_exp_v3f32() {
+define <3 x float> @constrained_vector_exp_v3f32() #0 {
 ; CHECK-LABEL: constrained_vector_exp_v3f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    subq $40, %rsp
@@ -1972,11 +1999,11 @@ entry:
   %exp = call <3 x float> @llvm.experimental.constrained.exp.v3f32(
                               <3 x float> <float 42.0, float 43.0, float 44.0>,
                               metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #0
   ret <3 x float> %exp
 }
 
-define <3 x double> @constrained_vector_exp_v3f64() {
+define <3 x double> @constrained_vector_exp_v3f64() #0 {
 ; CHECK-LABEL: constrained_vector_exp_v3f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    subq $24, %rsp
@@ -1991,9 +2018,9 @@ define <3 x double> @constrained_vector_exp_v3f64() {
 ; CHECK-NEXT:    callq exp
 ; CHECK-NEXT:    movsd %xmm0, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    fldl {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Reload
+; CHECK-NEXT:    movsd (%rsp), %xmm0 # 8-byte Reload
 ; CHECK-NEXT:    # xmm0 = mem[0],zero
-; CHECK-NEXT:    movsd (%rsp), %xmm1 # 8-byte Reload
+; CHECK-NEXT:    movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 8-byte Reload
 ; CHECK-NEXT:    # xmm1 = mem[0],zero
 ; CHECK-NEXT:    addq $24, %rsp
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
@@ -2023,11 +2050,11 @@ entry:
   %exp = call <3 x double> @llvm.experimental.constrained.exp.v3f64(
                           <3 x double> <double 42.0, double 42.1, double 42.2>,
                           metadata !"round.dynamic",
-                          metadata !"fpexcept.strict")
+                          metadata !"fpexcept.strict") #0
   ret <3 x double> %exp
 }
 
-define <4 x double> @constrained_vector_exp_v4f64() {
+define <4 x double> @constrained_vector_exp_v4f64() #0 {
 ; CHECK-LABEL: constrained_vector_exp_v4f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    subq $40, %rsp
@@ -2081,11 +2108,11 @@ entry:
                              <4 x double> <double 42.0, double 42.1,
                                            double 42.2, double 42.3>,
                              metadata !"round.dynamic",
-                             metadata !"fpexcept.strict")
+                             metadata !"fpexcept.strict") #0
   ret <4 x double> %exp
 }
 
-define <1 x float> @constrained_vector_exp2_v1f32() {
+define <1 x float> @constrained_vector_exp2_v1f32() #0 {
 ; CHECK-LABEL: constrained_vector_exp2_v1f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    pushq %rax
@@ -2109,11 +2136,11 @@ entry:
   %exp2 = call <1 x float> @llvm.experimental.constrained.exp2.v1f32(
                              <1 x float> <float 42.0>,
                              metadata !"round.dynamic",
-                             metadata !"fpexcept.strict")
+                             metadata !"fpexcept.strict") #0
   ret <1 x float> %exp2
 }
 
-define <2 x double> @constrained_vector_exp2_v2f64() {
+define <2 x double> @constrained_vector_exp2_v2f64() #0 {
 ; CHECK-LABEL: constrained_vector_exp2_v2f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    subq $24, %rsp
@@ -2147,11 +2174,11 @@ entry:
   %exp2 = call <2 x double> @llvm.experimental.constrained.exp2.v2f64(
                               <2 x double> <double 42.1, double 42.0>,
                               metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #0
   ret <2 x double> %exp2
 }
 
-define <3 x float> @constrained_vector_exp2_v3f32() {
+define <3 x float> @constrained_vector_exp2_v3f32() #0 {
 ; CHECK-LABEL: constrained_vector_exp2_v3f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    subq $40, %rsp
@@ -2196,11 +2223,11 @@ entry:
   %exp2 = call <3 x float> @llvm.experimental.constrained.exp2.v3f32(
                               <3 x float> <float 42.0, float 43.0, float 44.0>,
                               metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #0
   ret <3 x float> %exp2
 }
 
-define <3 x double> @constrained_vector_exp2_v3f64() {
+define <3 x double> @constrained_vector_exp2_v3f64() #0 {
 ; CHECK-LABEL: constrained_vector_exp2_v3f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    subq $24, %rsp
@@ -2215,9 +2242,9 @@ define <3 x double> @constrained_vector_exp2_v3f64() {
 ; CHECK-NEXT:    callq exp2
 ; CHECK-NEXT:    movsd %xmm0, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    fldl {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Reload
+; CHECK-NEXT:    movsd (%rsp), %xmm0 # 8-byte Reload
 ; CHECK-NEXT:    # xmm0 = mem[0],zero
-; CHECK-NEXT:    movsd (%rsp), %xmm1 # 8-byte Reload
+; CHECK-NEXT:    movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 8-byte Reload
 ; CHECK-NEXT:    # xmm1 = mem[0],zero
 ; CHECK-NEXT:    addq $24, %rsp
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
@@ -2247,11 +2274,11 @@ entry:
   %exp2 = call <3 x double> @llvm.experimental.constrained.exp2.v3f64(
                           <3 x double> <double 42.0, double 42.1, double 42.2>,
                           metadata !"round.dynamic",
-                          metadata !"fpexcept.strict")
+                          metadata !"fpexcept.strict") #0
   ret <3 x double> %exp2
 }
 
-define <4 x double> @constrained_vector_exp2_v4f64() {
+define <4 x double> @constrained_vector_exp2_v4f64() #0 {
 ; CHECK-LABEL: constrained_vector_exp2_v4f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    subq $40, %rsp
@@ -2305,11 +2332,11 @@ entry:
                               <4 x double> <double 42.1, double 42.2,
                                             double 42.3, double 42.4>,
                               metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #0
   ret <4 x double> %exp2
 }
 
-define <1 x float> @constrained_vector_log_v1f32() {
+define <1 x float> @constrained_vector_log_v1f32() #0 {
 ; CHECK-LABEL: constrained_vector_log_v1f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    pushq %rax
@@ -2333,11 +2360,11 @@ entry:
   %log = call <1 x float> @llvm.experimental.constrained.log.v1f32(
                              <1 x float> <float 42.0>,
                              metadata !"round.dynamic",
-                             metadata !"fpexcept.strict")
+                             metadata !"fpexcept.strict") #0
   ret <1 x float> %log
 }
 
-define <2 x double> @constrained_vector_log_v2f64() {
+define <2 x double> @constrained_vector_log_v2f64() #0 {
 ; CHECK-LABEL: constrained_vector_log_v2f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    subq $24, %rsp
@@ -2371,11 +2398,11 @@ entry:
   %log = call <2 x double> @llvm.experimental.constrained.log.v2f64(
                              <2 x double> <double 42.0, double 42.1>,
                              metadata !"round.dynamic",
-                             metadata !"fpexcept.strict")
+                             metadata !"fpexcept.strict") #0
   ret <2 x double> %log
 }
 
-define <3 x float> @constrained_vector_log_v3f32() {
+define <3 x float> @constrained_vector_log_v3f32() #0 {
 ; CHECK-LABEL: constrained_vector_log_v3f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    subq $40, %rsp
@@ -2420,11 +2447,11 @@ entry:
   %log = call <3 x float> @llvm.experimental.constrained.log.v3f32(
                               <3 x float> <float 42.0, float 43.0, float 44.0>,
                               metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #0
   ret <3 x float> %log
 }
 
-define <3 x double> @constrained_vector_log_v3f64() {
+define <3 x double> @constrained_vector_log_v3f64() #0 {
 ; CHECK-LABEL: constrained_vector_log_v3f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    subq $24, %rsp
@@ -2439,9 +2466,9 @@ define <3 x double> @constrained_vector_log_v3f64() {
 ; CHECK-NEXT:    callq log
 ; CHECK-NEXT:    movsd %xmm0, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    fldl {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Reload
+; CHECK-NEXT:    movsd (%rsp), %xmm0 # 8-byte Reload
 ; CHECK-NEXT:    # xmm0 = mem[0],zero
-; CHECK-NEXT:    movsd (%rsp), %xmm1 # 8-byte Reload
+; CHECK-NEXT:    movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 8-byte Reload
 ; CHECK-NEXT:    # xmm1 = mem[0],zero
 ; CHECK-NEXT:    addq $24, %rsp
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
@@ -2471,11 +2498,11 @@ entry:
   %log = call <3 x double> @llvm.experimental.constrained.log.v3f64(
                           <3 x double> <double 42.0, double 42.1, double 42.2>,
                           metadata !"round.dynamic",
-                          metadata !"fpexcept.strict")
+                          metadata !"fpexcept.strict") #0
   ret <3 x double> %log
 }
 
-define <4 x double> @constrained_vector_log_v4f64() {
+define <4 x double> @constrained_vector_log_v4f64() #0 {
 ; CHECK-LABEL: constrained_vector_log_v4f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    subq $40, %rsp
@@ -2529,11 +2556,11 @@ entry:
                              <4 x double> <double 42.0, double 42.1,
                                            double 42.2, double 42.3>,
                              metadata !"round.dynamic",
-                             metadata !"fpexcept.strict")
+                             metadata !"fpexcept.strict") #0
   ret <4 x double> %log
 }
 
-define <1 x float> @constrained_vector_log10_v1f32() {
+define <1 x float> @constrained_vector_log10_v1f32() #0 {
 ; CHECK-LABEL: constrained_vector_log10_v1f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    pushq %rax
@@ -2557,11 +2584,11 @@ entry:
   %log10 = call <1 x float> @llvm.experimental.constrained.log10.v1f32(
                              <1 x float> <float 42.0>,
                              metadata !"round.dynamic",
-                             metadata !"fpexcept.strict")
+                             metadata !"fpexcept.strict") #0
   ret <1 x float> %log10
 }
 
-define <2 x double> @constrained_vector_log10_v2f64() {
+define <2 x double> @constrained_vector_log10_v2f64() #0 {
 ; CHECK-LABEL: constrained_vector_log10_v2f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    subq $24, %rsp
@@ -2595,11 +2622,11 @@ entry:
   %log10 = call <2 x double> @llvm.experimental.constrained.log10.v2f64(
                                <2 x double> <double 42.0, double 42.1>,
                                metadata !"round.dynamic",
-                               metadata !"fpexcept.strict")
+                               metadata !"fpexcept.strict") #0
   ret <2 x double> %log10
 }
 
-define <3 x float> @constrained_vector_log10_v3f32() {
+define <3 x float> @constrained_vector_log10_v3f32() #0 {
 ; CHECK-LABEL: constrained_vector_log10_v3f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    subq $40, %rsp
@@ -2644,11 +2671,11 @@ entry:
   %log10 = call <3 x float> @llvm.experimental.constrained.log10.v3f32(
                               <3 x float> <float 42.0, float 43.0, float 44.0>,
                               metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #0
   ret <3 x float> %log10
 }
 
-define <3 x double> @constrained_vector_log10_v3f64() {
+define <3 x double> @constrained_vector_log10_v3f64() #0 {
 ; CHECK-LABEL: constrained_vector_log10_v3f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    subq $24, %rsp
@@ -2663,9 +2690,9 @@ define <3 x double> @constrained_vector_log10_v3f64() {
 ; CHECK-NEXT:    callq log10
 ; CHECK-NEXT:    movsd %xmm0, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    fldl {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Reload
+; CHECK-NEXT:    movsd (%rsp), %xmm0 # 8-byte Reload
 ; CHECK-NEXT:    # xmm0 = mem[0],zero
-; CHECK-NEXT:    movsd (%rsp), %xmm1 # 8-byte Reload
+; CHECK-NEXT:    movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 8-byte Reload
 ; CHECK-NEXT:    # xmm1 = mem[0],zero
 ; CHECK-NEXT:    addq $24, %rsp
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
@@ -2695,11 +2722,11 @@ entry:
   %log10 = call <3 x double> @llvm.experimental.constrained.log10.v3f64(
                           <3 x double> <double 42.0, double 42.1, double 42.2>,
                           metadata !"round.dynamic",
-                          metadata !"fpexcept.strict")
+                          metadata !"fpexcept.strict") #0
   ret <3 x double> %log10
 }
 
-define <4 x double> @constrained_vector_log10_v4f64() {
+define <4 x double> @constrained_vector_log10_v4f64() #0 {
 ; CHECK-LABEL: constrained_vector_log10_v4f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    subq $40, %rsp
@@ -2753,11 +2780,11 @@ entry:
                                <4 x double> <double 42.0, double 42.1,
                                              double 42.2, double 42.3>,
                                metadata !"round.dynamic",
-                               metadata !"fpexcept.strict")
+                               metadata !"fpexcept.strict") #0
   ret <4 x double> %log10
 }
 
-define <1 x float> @constrained_vector_log2_v1f32() {
+define <1 x float> @constrained_vector_log2_v1f32() #0 {
 ; CHECK-LABEL: constrained_vector_log2_v1f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    pushq %rax
@@ -2781,11 +2808,11 @@ entry:
   %log2 = call <1 x float> @llvm.experimental.constrained.log2.v1f32(
                              <1 x float> <float 42.0>,
                              metadata !"round.dynamic",
-                             metadata !"fpexcept.strict")
+                             metadata !"fpexcept.strict") #0
   ret <1 x float> %log2
 }
 
-define <2 x double> @constrained_vector_log2_v2f64() {
+define <2 x double> @constrained_vector_log2_v2f64() #0 {
 ; CHECK-LABEL: constrained_vector_log2_v2f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    subq $24, %rsp
@@ -2819,11 +2846,11 @@ entry:
   %log2 = call <2 x double> @llvm.experimental.constrained.log2.v2f64(
                               <2 x double> <double 42.0, double 42.1>,
                               metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #0
   ret <2 x double> %log2
 }
 
-define <3 x float> @constrained_vector_log2_v3f32() {
+define <3 x float> @constrained_vector_log2_v3f32() #0 {
 ; CHECK-LABEL: constrained_vector_log2_v3f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    subq $40, %rsp
@@ -2868,11 +2895,11 @@ entry:
   %log2 = call <3 x float> @llvm.experimental.constrained.log2.v3f32(
                               <3 x float> <float 42.0, float 43.0, float 44.0>,
                               metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #0
   ret <3 x float> %log2
 }
 
-define <3 x double> @constrained_vector_log2_v3f64() {
+define <3 x double> @constrained_vector_log2_v3f64() #0 {
 ; CHECK-LABEL: constrained_vector_log2_v3f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    subq $24, %rsp
@@ -2887,9 +2914,9 @@ define <3 x double> @constrained_vector_log2_v3f64() {
 ; CHECK-NEXT:    callq log2
 ; CHECK-NEXT:    movsd %xmm0, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    fldl {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Reload
+; CHECK-NEXT:    movsd (%rsp), %xmm0 # 8-byte Reload
 ; CHECK-NEXT:    # xmm0 = mem[0],zero
-; CHECK-NEXT:    movsd (%rsp), %xmm1 # 8-byte Reload
+; CHECK-NEXT:    movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 8-byte Reload
 ; CHECK-NEXT:    # xmm1 = mem[0],zero
 ; CHECK-NEXT:    addq $24, %rsp
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
@@ -2919,11 +2946,11 @@ entry:
   %log2 = call <3 x double> @llvm.experimental.constrained.log2.v3f64(
                           <3 x double> <double 42.0, double 42.1, double 42.2>,
                           metadata !"round.dynamic",
-                          metadata !"fpexcept.strict")
+                          metadata !"fpexcept.strict") #0
   ret <3 x double> %log2
 }
 
-define <4 x double> @constrained_vector_log2_v4f64() {
+define <4 x double> @constrained_vector_log2_v4f64() #0 {
 ; CHECK-LABEL: constrained_vector_log2_v4f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    subq $40, %rsp
@@ -2977,11 +3004,11 @@ entry:
                               <4 x double> <double 42.0, double 42.1,
                                             double 42.2, double 42.3>,
                               metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #0
   ret <4 x double> %log2
 }
 
-define <1 x float> @constrained_vector_rint_v1f32() {
+define <1 x float> @constrained_vector_rint_v1f32() #0 {
 ; CHECK-LABEL: constrained_vector_rint_v1f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    pushq %rax
@@ -3001,11 +3028,11 @@ entry:
   %rint = call <1 x float> @llvm.experimental.constrained.rint.v1f32(
                              <1 x float> <float 42.0>,
                              metadata !"round.dynamic",
-                             metadata !"fpexcept.strict")
+                             metadata !"fpexcept.strict") #0
   ret <1 x float> %rint
 }
 
-define <2 x double> @constrained_vector_rint_v2f64() {
+define <2 x double> @constrained_vector_rint_v2f64() #0 {
 ; CHECK-LABEL: constrained_vector_rint_v2f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    subq $24, %rsp
@@ -3029,11 +3056,11 @@ entry:
   %rint = call <2 x double> @llvm.experimental.constrained.rint.v2f64(
                         <2 x double> <double 42.1, double 42.0>,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret <2 x double> %rint
 }
 
-define <3 x float> @constrained_vector_rint_v3f32() {
+define <3 x float> @constrained_vector_rint_v3f32() #0 {
 ; CHECK-LABEL: constrained_vector_rint_v3f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    subq $40, %rsp
@@ -3070,11 +3097,11 @@ define <3 x float> @constrained_vector_rint_v3f32() {
   %rint = call <3 x float> @llvm.experimental.constrained.rint.v3f32(
                               <3 x float> <float 42.0, float 43.0, float 44.0>,
                               metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #0
   ret <3 x float> %rint
 }
 
-define <3 x double> @constrained_vector_rint_v3f64() {
+define <3 x double> @constrained_vector_rint_v3f64() #0 {
 ; CHECK-LABEL: constrained_vector_rint_v3f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    subq $24, %rsp
@@ -3089,9 +3116,9 @@ define <3 x double> @constrained_vector_rint_v3f64() {
 ; CHECK-NEXT:    callq rint
 ; CHECK-NEXT:    movsd %xmm0, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    fldl {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Reload
+; CHECK-NEXT:    movsd (%rsp), %xmm0 # 8-byte Reload
 ; CHECK-NEXT:    # xmm0 = mem[0],zero
-; CHECK-NEXT:    movsd (%rsp), %xmm1 # 8-byte Reload
+; CHECK-NEXT:    movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 8-byte Reload
 ; CHECK-NEXT:    # xmm1 = mem[0],zero
 ; CHECK-NEXT:    addq $24, %rsp
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
@@ -3108,11 +3135,11 @@ entry:
   %rint = call <3 x double> @llvm.experimental.constrained.rint.v3f64(
                           <3 x double> <double 42.0, double 42.1, double 42.2>,
                           metadata !"round.dynamic",
-                          metadata !"fpexcept.strict")
+                          metadata !"fpexcept.strict") #0
   ret <3 x double> %rint
 }
 
-define <4 x double> @constrained_vector_rint_v4f64() {
+define <4 x double> @constrained_vector_rint_v4f64() #0 {
 ; CHECK-LABEL: constrained_vector_rint_v4f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    subq $40, %rsp
@@ -3147,11 +3174,11 @@ entry:
                         <4 x double> <double 42.1, double 42.2,
                                       double 42.3, double 42.4>,
                         metadata !"round.dynamic",
-                        metadata !"fpexcept.strict")
+                        metadata !"fpexcept.strict") #0
   ret <4 x double> %rint
 }
 
-define <1 x float> @constrained_vector_nearbyint_v1f32() {
+define <1 x float> @constrained_vector_nearbyint_v1f32() #0 {
 ; CHECK-LABEL: constrained_vector_nearbyint_v1f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    pushq %rax
@@ -3171,11 +3198,11 @@ entry:
   %nearby = call <1 x float> @llvm.experimental.constrained.nearbyint.v1f32(
                                <1 x float> <float 42.0>,
                                metadata !"round.dynamic",
-                               metadata !"fpexcept.strict")
+                               metadata !"fpexcept.strict") #0
   ret <1 x float> %nearby
 }
 
-define <2 x double> @constrained_vector_nearbyint_v2f64() {
+define <2 x double> @constrained_vector_nearbyint_v2f64() #0 {
 ; CHECK-LABEL: constrained_vector_nearbyint_v2f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    subq $24, %rsp
@@ -3199,11 +3226,11 @@ entry:
   %nearby = call <2 x double> @llvm.experimental.constrained.nearbyint.v2f64(
                                 <2 x double> <double 42.1, double 42.0>,
                                 metadata !"round.dynamic",
-                                metadata !"fpexcept.strict")
+                                metadata !"fpexcept.strict") #0
   ret <2 x double> %nearby
 }
 
-define <3 x float> @constrained_vector_nearbyint_v3f32() {
+define <3 x float> @constrained_vector_nearbyint_v3f32() #0 {
 ; CHECK-LABEL: constrained_vector_nearbyint_v3f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    subq $40, %rsp
@@ -3240,11 +3267,11 @@ entry:
   %nearby = call <3 x float> @llvm.experimental.constrained.nearbyint.v3f32(
                               <3 x float> <float 42.0, float 43.0, float 44.0>,
                               metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #0
   ret <3 x float> %nearby
 }
 
-define <3 x double> @constrained_vector_nearby_v3f64() {
+define <3 x double> @constrained_vector_nearby_v3f64() #0 {
 ; CHECK-LABEL: constrained_vector_nearby_v3f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    subq $24, %rsp
@@ -3259,9 +3286,9 @@ define <3 x double> @constrained_vector_nearby_v3f64() {
 ; CHECK-NEXT:    callq nearbyint
 ; CHECK-NEXT:    movsd %xmm0, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    fldl {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Reload
+; CHECK-NEXT:    movsd (%rsp), %xmm0 # 8-byte Reload
 ; CHECK-NEXT:    # xmm0 = mem[0],zero
-; CHECK-NEXT:    movsd (%rsp), %xmm1 # 8-byte Reload
+; CHECK-NEXT:    movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 8-byte Reload
 ; CHECK-NEXT:    # xmm1 = mem[0],zero
 ; CHECK-NEXT:    addq $24, %rsp
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
@@ -3278,11 +3305,11 @@ entry:
   %nearby = call <3 x double> @llvm.experimental.constrained.nearbyint.v3f64(
                           <3 x double> <double 42.0, double 42.1, double 42.2>,
                           metadata !"round.dynamic",
-                          metadata !"fpexcept.strict")
+                          metadata !"fpexcept.strict") #0
   ret <3 x double> %nearby
 }
 
-define <4 x double> @constrained_vector_nearbyint_v4f64() {
+define <4 x double> @constrained_vector_nearbyint_v4f64() #0 {
 ; CHECK-LABEL: constrained_vector_nearbyint_v4f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    subq $40, %rsp
@@ -3317,11 +3344,11 @@ entry:
                                 <4 x double> <double 42.1, double 42.2,
                                               double 42.3, double 42.4>,
                                 metadata !"round.dynamic",
-                                metadata !"fpexcept.strict")
+                                metadata !"fpexcept.strict") #0
   ret <4 x double> %nearby
 }
 
-define <1 x float> @constrained_vector_maxnum_v1f32() {
+define <1 x float> @constrained_vector_maxnum_v1f32() #0 {
 ; CHECK-LABEL: constrained_vector_maxnum_v1f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    pushq %rax
@@ -3346,12 +3373,11 @@ define <1 x float> @constrained_vector_maxnum_v1f32() {
 entry:
   %max = call <1 x float> @llvm.experimental.constrained.maxnum.v1f32(
                                <1 x float> <float 42.0>, <1 x float> <float 41.0>,
-                               metadata !"round.dynamic",
-                               metadata !"fpexcept.strict")
+                               metadata !"fpexcept.strict") #0
   ret <1 x float> %max
 }
 
-define <2 x double> @constrained_vector_maxnum_v2f64() {
+define <2 x double> @constrained_vector_maxnum_v2f64() #0 {
 ; CHECK-LABEL: constrained_vector_maxnum_v2f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    subq $24, %rsp
@@ -3389,12 +3415,11 @@ entry:
   %max = call <2 x double> @llvm.experimental.constrained.maxnum.v2f64(
                                 <2 x double> <double 43.0, double 42.0>,
                                 <2 x double> <double 41.0, double 40.0>,
-                                metadata !"round.dynamic",
-                                metadata !"fpexcept.strict")
+                                metadata !"fpexcept.strict") #0
   ret <2 x double> %max
 }
 
-define <3 x float> @constrained_vector_maxnum_v3f32() {
+define <3 x float> @constrained_vector_maxnum_v3f32() #0 {
 ; CHECK-LABEL: constrained_vector_maxnum_v3f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    subq $40, %rsp
@@ -3445,12 +3470,11 @@ entry:
   %max = call <3 x float> @llvm.experimental.constrained.maxnum.v3f32(
                               <3 x float> <float 43.0, float 44.0, float 45.0>,
                               <3 x float> <float 41.0, float 42.0, float 43.0>,
-                              metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #0
   ret <3 x float> %max
 }
 
-define <3 x double> @constrained_vector_max_v3f64() {
+define <3 x double> @constrained_vector_max_v3f64() #0 {
 ; CHECK-LABEL: constrained_vector_max_v3f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    subq $24, %rsp
@@ -3468,9 +3492,9 @@ define <3 x double> @constrained_vector_max_v3f64() {
 ; CHECK-NEXT:    callq fmax
 ; CHECK-NEXT:    movsd %xmm0, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    fldl {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Reload
+; CHECK-NEXT:    movsd (%rsp), %xmm0 # 8-byte Reload
 ; CHECK-NEXT:    # xmm0 = mem[0],zero
-; CHECK-NEXT:    movsd (%rsp), %xmm1 # 8-byte Reload
+; CHECK-NEXT:    movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 8-byte Reload
 ; CHECK-NEXT:    # xmm1 = mem[0],zero
 ; CHECK-NEXT:    addq $24, %rsp
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
@@ -3503,12 +3527,11 @@ entry:
   %max = call <3 x double> @llvm.experimental.constrained.maxnum.v3f64(
                           <3 x double> <double 43.0, double 44.0, double 45.0>,
                           <3 x double> <double 40.0, double 41.0, double 42.0>,
-                          metadata !"round.dynamic",
-                          metadata !"fpexcept.strict")
+                          metadata !"fpexcept.strict") #0
   ret <3 x double> %max
 }
 
-define <4 x double> @constrained_vector_maxnum_v4f64() {
+define <4 x double> @constrained_vector_maxnum_v4f64() #0 {
 ; CHECK-LABEL: constrained_vector_maxnum_v4f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    subq $40, %rsp
@@ -3571,12 +3594,11 @@ entry:
                                               double 46.0, double 47.0>,
                                 <4 x double> <double 40.0, double 41.0,
                                               double 42.0, double 43.0>,
-                                metadata !"round.dynamic",
-                                metadata !"fpexcept.strict")
+                                metadata !"fpexcept.strict") #0
   ret <4 x double> %max
 }
 
-define <1 x float> @constrained_vector_minnum_v1f32() {
+define <1 x float> @constrained_vector_minnum_v1f32() #0 {
 ; CHECK-LABEL: constrained_vector_minnum_v1f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    pushq %rax
@@ -3601,12 +3623,11 @@ define <1 x float> @constrained_vector_minnum_v1f32() {
  entry:
   %min = call <1 x float> @llvm.experimental.constrained.minnum.v1f32(
                                <1 x float> <float 42.0>, <1 x float> <float 41.0>,
-                               metadata !"round.dynamic",
-                               metadata !"fpexcept.strict")
+                               metadata !"fpexcept.strict") #0
   ret <1 x float> %min
 }
 
-define <2 x double> @constrained_vector_minnum_v2f64() {
+define <2 x double> @constrained_vector_minnum_v2f64() #0 {
 ; CHECK-LABEL: constrained_vector_minnum_v2f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    subq $24, %rsp
@@ -3644,12 +3665,11 @@ entry:
   %min = call <2 x double> @llvm.experimental.constrained.minnum.v2f64(
                                 <2 x double> <double 43.0, double 42.0>,
                                 <2 x double> <double 41.0, double 40.0>,
-                                metadata !"round.dynamic",
-                                metadata !"fpexcept.strict")
+                                metadata !"fpexcept.strict") #0
   ret <2 x double> %min
 }
 
-define <3 x float> @constrained_vector_minnum_v3f32() {
+define <3 x float> @constrained_vector_minnum_v3f32() #0 {
 ; CHECK-LABEL: constrained_vector_minnum_v3f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    subq $40, %rsp
@@ -3700,12 +3720,11 @@ entry:
   %min = call <3 x float> @llvm.experimental.constrained.minnum.v3f32(
                               <3 x float> <float 43.0, float 44.0, float 45.0>,
                               <3 x float> <float 41.0, float 42.0, float 43.0>,
-                              metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
+                              metadata !"fpexcept.strict") #0
   ret <3 x float> %min
 }
 
-define <3 x double> @constrained_vector_min_v3f64() {
+define <3 x double> @constrained_vector_min_v3f64() #0 {
 ; CHECK-LABEL: constrained_vector_min_v3f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    subq $24, %rsp
@@ -3723,9 +3742,9 @@ define <3 x double> @constrained_vector_min_v3f64() {
 ; CHECK-NEXT:    callq fmin
 ; CHECK-NEXT:    movsd %xmm0, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    fldl {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Reload
+; CHECK-NEXT:    movsd (%rsp), %xmm0 # 8-byte Reload
 ; CHECK-NEXT:    # xmm0 = mem[0],zero
-; CHECK-NEXT:    movsd (%rsp), %xmm1 # 8-byte Reload
+; CHECK-NEXT:    movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 8-byte Reload
 ; CHECK-NEXT:    # xmm1 = mem[0],zero
 ; CHECK-NEXT:    addq $24, %rsp
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
@@ -3758,12 +3777,11 @@ entry:
  %min = call <3 x double> @llvm.experimental.constrained.minnum.v3f64(
                           <3 x double> <double 43.0, double 44.0, double 45.0>,
                           <3 x double> <double 40.0, double 41.0, double 42.0>,
-                          metadata !"round.dynamic",
-                          metadata !"fpexcept.strict")
+                          metadata !"fpexcept.strict") #0
   ret <3 x double> %min
 }
 
-define <4 x double> @constrained_vector_minnum_v4f64() {
+define <4 x double> @constrained_vector_minnum_v4f64() #0 {
 ; CHECK-LABEL: constrained_vector_minnum_v4f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    subq $40, %rsp
@@ -3826,783 +3844,3741 @@ entry:
                                               double 46.0, double 47.0>,
                                 <4 x double> <double 40.0, double 41.0,
                                               double 42.0, double 43.0>,
-                                metadata !"round.dynamic",
-                                metadata !"fpexcept.strict")
+                                metadata !"fpexcept.strict") #0
   ret <4 x double> %min
 }
 
-define <1 x float> @constrained_vector_fptrunc_v1f64() {
-; CHECK-LABEL: constrained_vector_fptrunc_v1f64:
+define <1 x i32> @constrained_vector_fptosi_v1i32_v1f32() #0 {
+; CHECK-LABEL: constrained_vector_fptosi_v1i32_v1f32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; CHECK-NEXT:    cvtsd2ss %xmm0, %xmm0
+; CHECK-NEXT:    cvttss2si {{.*}}(%rip), %eax
 ; CHECK-NEXT:    retq
 ;
-; AVX-LABEL: constrained_vector_fptrunc_v1f64:
+; AVX-LABEL: constrained_vector_fptosi_v1i32_v1f32:
 ; AVX:       # %bb.0: # %entry
-; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; AVX-NEXT:    vcvtsd2ss %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vcvttss2si {{.*}}(%rip), %eax
 ; AVX-NEXT:    retq
 entry:
-  %result = call <1 x float> @llvm.experimental.constrained.fptrunc.v1f32.v1f64(
-                                <1 x double><double 42.1>,
-                                metadata !"round.dynamic",
-                                metadata !"fpexcept.strict")
-  ret <1 x float> %result
+  %result = call <1 x i32> @llvm.experimental.constrained.fptosi.v1i32.v1f32(
+                               <1 x float><float 42.0>,
+                               metadata !"fpexcept.strict") #0
+  ret <1 x i32> %result
 }
 
-define <2 x float> @constrained_vector_fptrunc_v2f64() {
-; CHECK-LABEL: constrained_vector_fptrunc_v2f64:
+define <2 x i32> @constrained_vector_fptosi_v2i32_v2f32() #0 {
+; CHECK-LABEL: constrained_vector_fptosi_v2i32_v2f32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; CHECK-NEXT:    cvtsd2ss %xmm0, %xmm1
-; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; CHECK-NEXT:    cvtsd2ss %xmm0, %xmm0
-; CHECK-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; CHECK-NEXT:    cvttps2dq {{.*}}(%rip), %xmm0
 ; CHECK-NEXT:    retq
 ;
-; AVX-LABEL: constrained_vector_fptrunc_v2f64:
+; AVX-LABEL: constrained_vector_fptosi_v2i32_v2f32:
 ; AVX:       # %bb.0: # %entry
-; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; AVX-NEXT:    vcvtsd2ss %xmm0, %xmm0, %xmm0
-; AVX-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
-; AVX-NEXT:    vcvtsd2ss %xmm1, %xmm1, %xmm1
-; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3]
+; AVX-NEXT:    vcvttps2dq {{.*}}(%rip), %xmm0
 ; AVX-NEXT:    retq
 entry:
-  %result = call <2 x float> @llvm.experimental.constrained.fptrunc.v2f32.v2f64(
-                                <2 x double><double 42.1, double 42.2>,
-                                metadata !"round.dynamic",
-                                metadata !"fpexcept.strict")
-  ret <2 x float> %result
+  %result = call <2 x i32> @llvm.experimental.constrained.fptosi.v2i32.v2f32(
+                                <2 x float><float 42.0, float 43.0>,
+                                metadata !"fpexcept.strict") #0
+  ret <2 x i32> %result
 }
 
-define <3 x float> @constrained_vector_fptrunc_v3f64() {
-; CHECK-LABEL: constrained_vector_fptrunc_v3f64:
+define <3 x i32> @constrained_vector_fptosi_v3i32_v3f32() #0 {
+; CHECK-LABEL: constrained_vector_fptosi_v3i32_v3f32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; CHECK-NEXT:    cvtsd2ss %xmm0, %xmm1
-; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; CHECK-NEXT:    cvtsd2ss %xmm0, %xmm0
-; CHECK-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; CHECK-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
-; CHECK-NEXT:    cvtsd2ss %xmm1, %xmm1
-; CHECK-NEXT:    movlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; CHECK-NEXT:    cvttss2si {{.*}}(%rip), %eax
+; CHECK-NEXT:    movd %eax, %xmm1
+; CHECK-NEXT:    cvttss2si {{.*}}(%rip), %eax
+; CHECK-NEXT:    movd %eax, %xmm0
+; CHECK-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; CHECK-NEXT:    cvttss2si {{.*}}(%rip), %eax
+; CHECK-NEXT:    movd %eax, %xmm1
+; CHECK-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
 ; CHECK-NEXT:    retq
 ;
-; AVX-LABEL: constrained_vector_fptrunc_v3f64:
+; AVX-LABEL: constrained_vector_fptosi_v3i32_v3f32:
 ; AVX:       # %bb.0: # %entry
-; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; AVX-NEXT:    vcvtsd2ss %xmm0, %xmm0, %xmm0
-; AVX-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
-; AVX-NEXT:    vcvtsd2ss %xmm1, %xmm1, %xmm1
-; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3]
-; AVX-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
-; AVX-NEXT:    vcvtsd2ss %xmm1, %xmm1, %xmm1
-; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1],xmm1[0],xmm0[3]
+; AVX-NEXT:    vcvttss2si {{.*}}(%rip), %eax
+; AVX-NEXT:    vmovd %eax, %xmm0
+; AVX-NEXT:    vcvttss2si {{.*}}(%rip), %eax
+; AVX-NEXT:    vpinsrd $1, %eax, %xmm0, %xmm0
+; AVX-NEXT:    vcvttss2si {{.*}}(%rip), %eax
+; AVX-NEXT:    vpinsrd $2, %eax, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 entry:
-  %result = call <3 x float> @llvm.experimental.constrained.fptrunc.v3f32.v3f64(
-                                <3 x double><double 42.1, double 42.2,
-                                             double 42.3>,
-                                metadata !"round.dynamic",
-                                metadata !"fpexcept.strict")
-  ret <3 x float> %result
+  %result = call <3 x i32> @llvm.experimental.constrained.fptosi.v3i32.v3f32(
+                                <3 x float><float 42.0, float 43.0,
+                                            float 44.0>,
+                                metadata !"fpexcept.strict") #0
+  ret <3 x i32> %result
 }
 
-define <4 x float> @constrained_vector_fptrunc_v4f64() {
-; CHECK-LABEL: constrained_vector_fptrunc_v4f64:
+define <4 x i32> @constrained_vector_fptosi_v4i32_v4f32() #0 {
+; CHECK-LABEL: constrained_vector_fptosi_v4i32_v4f32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; CHECK-NEXT:    cvtsd2ss %xmm0, %xmm0
-; CHECK-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
-; CHECK-NEXT:    cvtsd2ss %xmm1, %xmm1
-; CHECK-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; CHECK-NEXT:    cvtsd2ss %xmm0, %xmm2
-; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; CHECK-NEXT:    cvtsd2ss %xmm0, %xmm0
-; CHECK-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; CHECK-NEXT:    movlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; CHECK-NEXT:    cvttps2dq {{.*}}(%rip), %xmm0
 ; CHECK-NEXT:    retq
 ;
-; AVX-LABEL: constrained_vector_fptrunc_v4f64:
+; AVX-LABEL: constrained_vector_fptosi_v4i32_v4f32:
 ; AVX:       # %bb.0: # %entry
-; AVX-NEXT:    vcvtpd2psy {{.*}}(%rip), %xmm0
+; AVX-NEXT:    vcvttps2dq {{.*}}(%rip), %xmm0
 ; AVX-NEXT:    retq
 entry:
-  %result = call <4 x float> @llvm.experimental.constrained.fptrunc.v4f32.v4f64(
-                                <4 x double><double 42.1, double 42.2,
-                                             double 42.3, double 42.4>,
-                                metadata !"round.dynamic",
-                                metadata !"fpexcept.strict")
-  ret <4 x float> %result
+  %result = call <4 x i32> @llvm.experimental.constrained.fptosi.v4i32.v4f32(
+                                <4 x float><float 42.0, float 43.0,
+                                            float 44.0, float 45.0>,
+                                metadata !"fpexcept.strict") #0
+  ret <4 x i32> %result
 }
 
-define <1 x double> @constrained_vector_fpext_v1f32() {
-; CHECK-LABEL: constrained_vector_fpext_v1f32:
+define <1 x i64> @constrained_vector_fptosi_v1i64_v1f32() #0 {
+; CHECK-LABEL: constrained_vector_fptosi_v1i64_v1f32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; CHECK-NEXT:    cvtss2sd %xmm0, %xmm0
+; CHECK-NEXT:    cvttss2si {{.*}}(%rip), %rax
 ; CHECK-NEXT:    retq
 ;
-; AVX-LABEL: constrained_vector_fpext_v1f32:
+; AVX-LABEL: constrained_vector_fptosi_v1i64_v1f32:
 ; AVX:       # %bb.0: # %entry
-; AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vcvttss2si {{.*}}(%rip), %rax
 ; AVX-NEXT:    retq
 entry:
-  %result = call <1 x double> @llvm.experimental.constrained.fpext.v1f64.v1f32(
-                                <1 x float><float 42.0>,
-                                metadata !"fpexcept.strict")
-  ret <1 x double> %result
+  %result = call <1 x i64> @llvm.experimental.constrained.fptosi.v1i64.v1f32(
+                               <1 x float><float 42.0>,
+                               metadata !"fpexcept.strict") #0
+  ret <1 x i64> %result
 }
 
-define <2 x double> @constrained_vector_fpext_v2f32() {
-; CHECK-LABEL: constrained_vector_fpext_v2f32:
+define <2 x i64> @constrained_vector_fptosi_v2i64_v2f32() #0 {
+; CHECK-LABEL: constrained_vector_fptosi_v2i64_v2f32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; CHECK-NEXT:    cvtss2sd %xmm0, %xmm1
-; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; CHECK-NEXT:    cvtss2sd %xmm0, %xmm0
-; CHECK-NEXT:    movlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; CHECK-NEXT:    cvttss2si {{.*}}(%rip), %rax
+; CHECK-NEXT:    movq %rax, %xmm1
+; CHECK-NEXT:    cvttss2si {{.*}}(%rip), %rax
+; CHECK-NEXT:    movq %rax, %xmm0
+; CHECK-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
 ; CHECK-NEXT:    retq
 ;
-; AVX-LABEL: constrained_vector_fpext_v2f32:
+; AVX-LABEL: constrained_vector_fptosi_v2i64_v2f32:
 ; AVX:       # %bb.0: # %entry
-; AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
-; AVX-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX-NEXT:    vcvttss2si {{.*}}(%rip), %rax
+; AVX-NEXT:    vmovq %rax, %xmm0
+; AVX-NEXT:    vcvttss2si {{.*}}(%rip), %rax
+; AVX-NEXT:    vmovq %rax, %xmm1
+; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
 ; AVX-NEXT:    retq
 entry:
-  %result = call <2 x double> @llvm.experimental.constrained.fpext.v2f64.v2f32(
+  %result = call <2 x i64> @llvm.experimental.constrained.fptosi.v2i64.v2f32(
                                 <2 x float><float 42.0, float 43.0>,
-                                metadata !"fpexcept.strict")
-  ret <2 x double> %result
+                                metadata !"fpexcept.strict") #0
+  ret <2 x i64> %result
 }
 
-define <3 x double> @constrained_vector_fpext_v3f32() {
-; CHECK-LABEL: constrained_vector_fpext_v3f32:
+define <3 x i64> @constrained_vector_fptosi_v3i64_v3f32() #0 {
+; CHECK-LABEL: constrained_vector_fptosi_v3i64_v3f32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; CHECK-NEXT:    cvtss2sd %xmm0, %xmm0
-; CHECK-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; CHECK-NEXT:    cvtss2sd %xmm1, %xmm1
-; CHECK-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; CHECK-NEXT:    cvtss2sd %xmm2, %xmm2
-; CHECK-NEXT:    movsd %xmm2, -{{[0-9]+}}(%rsp)
-; CHECK-NEXT:    fldl -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    cvttss2si {{.*}}(%rip), %rcx
+; CHECK-NEXT:    cvttss2si {{.*}}(%rip), %rdx
+; CHECK-NEXT:    cvttss2si {{.*}}(%rip), %rax
 ; CHECK-NEXT:    retq
 ;
-; AVX-LABEL: constrained_vector_fpext_v3f32:
-; AVX:       # %bb.0: # %entry
-; AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
-; AVX-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
-; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX-NEXT:    retq
+; AVX1-LABEL: constrained_vector_fptosi_v3i64_v3f32:
+; AVX1:       # %bb.0: # %entry
+; AVX1-NEXT:    vcvttss2si {{.*}}(%rip), %rax
+; AVX1-NEXT:    vmovq %rax, %xmm0
+; AVX1-NEXT:    vcvttss2si {{.*}}(%rip), %rax
+; AVX1-NEXT:    vmovq %rax, %xmm1
+; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX1-NEXT:    vcvttss2si {{.*}}(%rip), %rax
+; AVX1-NEXT:    vmovq %rax, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX512-LABEL: constrained_vector_fptosi_v3i64_v3f32:
+; AVX512:       # %bb.0: # %entry
+; AVX512-NEXT:    vcvttss2si {{.*}}(%rip), %rax
+; AVX512-NEXT:    vmovq %rax, %xmm0
+; AVX512-NEXT:    vcvttss2si {{.*}}(%rip), %rax
+; AVX512-NEXT:    vmovq %rax, %xmm1
+; AVX512-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512-NEXT:    vcvttss2si {{.*}}(%rip), %rax
+; AVX512-NEXT:    vmovq %rax, %xmm1
+; AVX512-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512-NEXT:    retq
 entry:
-  %result = call <3 x double> @llvm.experimental.constrained.fpext.v3f64.v3f32(
+  %result = call <3 x i64> @llvm.experimental.constrained.fptosi.v3i64.v3f32(
                                 <3 x float><float 42.0, float 43.0,
                                             float 44.0>,
-                                metadata !"fpexcept.strict")
-  ret <3 x double> %result
+                                metadata !"fpexcept.strict") #0
+  ret <3 x i64> %result
 }
 
-define <4 x double> @constrained_vector_fpext_v4f32() {
-; CHECK-LABEL: constrained_vector_fpext_v4f32:
+define <4 x i64> @constrained_vector_fptosi_v4i64_v4f32() #0 {
+; CHECK-LABEL: constrained_vector_fptosi_v4i64_v4f32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; CHECK-NEXT:    cvtss2sd %xmm0, %xmm1
-; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; CHECK-NEXT:    cvtss2sd %xmm0, %xmm0
-; CHECK-NEXT:    movlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; CHECK-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; CHECK-NEXT:    cvtss2sd %xmm1, %xmm2
-; CHECK-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; CHECK-NEXT:    cvtss2sd %xmm1, %xmm1
-; CHECK-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; CHECK-NEXT:    cvttss2si {{.*}}(%rip), %rax
+; CHECK-NEXT:    movq %rax, %xmm1
+; CHECK-NEXT:    cvttss2si {{.*}}(%rip), %rax
+; CHECK-NEXT:    movq %rax, %xmm0
+; CHECK-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; CHECK-NEXT:    cvttss2si {{.*}}(%rip), %rax
+; CHECK-NEXT:    movq %rax, %xmm2
+; CHECK-NEXT:    cvttss2si {{.*}}(%rip), %rax
+; CHECK-NEXT:    movq %rax, %xmm1
+; CHECK-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
 ; CHECK-NEXT:    retq
 ;
-; AVX-LABEL: constrained_vector_fpext_v4f32:
-; AVX:       # %bb.0: # %entry
-; AVX-NEXT:    vcvtps2pd {{.*}}(%rip), %ymm0
-; AVX-NEXT:    retq
+; AVX1-LABEL: constrained_vector_fptosi_v4i64_v4f32:
+; AVX1:       # %bb.0: # %entry
+; AVX1-NEXT:    vcvttss2si {{.*}}(%rip), %rax
+; AVX1-NEXT:    vmovq %rax, %xmm0
+; AVX1-NEXT:    vcvttss2si {{.*}}(%rip), %rax
+; AVX1-NEXT:    vmovq %rax, %xmm1
+; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX1-NEXT:    vcvttss2si {{.*}}(%rip), %rax
+; AVX1-NEXT:    vmovq %rax, %xmm1
+; AVX1-NEXT:    vcvttss2si {{.*}}(%rip), %rax
+; AVX1-NEXT:    vmovq %rax, %xmm2
+; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX512F-LABEL: constrained_vector_fptosi_v4i64_v4f32:
+; AVX512F:       # %bb.0: # %entry
+; AVX512F-NEXT:    vcvttss2si {{.*}}(%rip), %rax
+; AVX512F-NEXT:    vmovq %rax, %xmm0
+; AVX512F-NEXT:    vcvttss2si {{.*}}(%rip), %rax
+; AVX512F-NEXT:    vmovq %rax, %xmm1
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512F-NEXT:    vcvttss2si {{.*}}(%rip), %rax
+; AVX512F-NEXT:    vmovq %rax, %xmm1
+; AVX512F-NEXT:    vcvttss2si {{.*}}(%rip), %rax
+; AVX512F-NEXT:    vmovq %rax, %xmm2
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX512F-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX512F-NEXT:    retq
+;
+; AVX512DQ-LABEL: constrained_vector_fptosi_v4i64_v4f32:
+; AVX512DQ:       # %bb.0: # %entry
+; AVX512DQ-NEXT:    vmovaps {{.*#+}} xmm0 = [4.2E+1,4.3E+1,4.4E+1,4.5E+1]
+; AVX512DQ-NEXT:    vcvttps2qq %ymm0, %zmm0
+; AVX512DQ-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512DQ-NEXT:    retq
 entry:
-  %result = call <4 x double> @llvm.experimental.constrained.fpext.v4f64.v4f32(
+  %result = call <4 x i64> @llvm.experimental.constrained.fptosi.v4i64.v4f32(
                                 <4 x float><float 42.0, float 43.0,
                                             float 44.0, float 45.0>,
-                                metadata !"fpexcept.strict")
-  ret <4 x double> %result
+                                metadata !"fpexcept.strict") #0
+  ret <4 x i64> %result
 }
 
-define <1 x float> @constrained_vector_ceil_v1f32() {
-; CHECK-LABEL: constrained_vector_ceil_v1f32:
+define <1 x i32> @constrained_vector_fptosi_v1i32_v1f64() #0 {
+; CHECK-LABEL: constrained_vector_fptosi_v1i32_v1f64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    pushq %rax
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; CHECK-NEXT:    callq ceilf
-; CHECK-NEXT:    popq %rax
-; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    cvttsd2si {{.*}}(%rip), %eax
 ; CHECK-NEXT:    retq
 ;
-; AVX-LABEL: constrained_vector_ceil_v1f32:
+; AVX-LABEL: constrained_vector_fptosi_v1i32_v1f64:
 ; AVX:       # %bb.0: # %entry
-; AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX-NEXT:    vroundss $10, %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vcvttsd2si {{.*}}(%rip), %eax
 ; AVX-NEXT:    retq
 entry:
-  %ceil = call <1 x float> @llvm.experimental.constrained.ceil.v1f32(
-                               <1 x float> <float 1.5>,
-                               metadata !"round.dynamic",
-                               metadata !"fpexcept.strict")
-  ret <1 x float> %ceil
+  %result = call <1 x i32> @llvm.experimental.constrained.fptosi.v1i32.v1f64(
+                               <1 x double><double 42.1>,
+                               metadata !"fpexcept.strict") #0
+  ret <1 x i32> %result
 }
 
-define <2 x double> @constrained_vector_ceil_v2f64() {
-; CHECK-LABEL: constrained_vector_ceil_v2f64:
+
+define <2 x i32> @constrained_vector_fptosi_v2i32_v2f64() #0 {
+; CHECK-LABEL: constrained_vector_fptosi_v2i32_v2f64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    subq $24, %rsp
-; CHECK-NEXT:    .cfi_def_cfa_offset 32
-; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; CHECK-NEXT:    callq ceil
-; CHECK-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
-; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; CHECK-NEXT:    callq ceil
-; CHECK-NEXT:    unpcklpd (%rsp), %xmm0 # 16-byte Folded Reload
-; CHECK-NEXT:    # xmm0 = xmm0[0],mem[0]
-; CHECK-NEXT:    addq $24, %rsp
-; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    cvttpd2dq {{.*}}(%rip), %xmm0
 ; CHECK-NEXT:    retq
 ;
-; AVX-LABEL: constrained_vector_ceil_v2f64:
+; AVX-LABEL: constrained_vector_fptosi_v2i32_v2f64:
 ; AVX:       # %bb.0: # %entry
-; AVX-NEXT:    vroundpd $10, {{.*}}(%rip), %xmm0
+; AVX-NEXT:    vcvttpd2dqx {{.*}}(%rip), %xmm0
 ; AVX-NEXT:    retq
 entry:
-  %ceil = call <2 x double> @llvm.experimental.constrained.ceil.v2f64(
-                                <2 x double> <double 1.1, double 1.9>,
-                                metadata !"round.dynamic",
-                                metadata !"fpexcept.strict")
-  ret <2 x double> %ceil
+  %result = call <2 x i32> @llvm.experimental.constrained.fptosi.v2i32.v2f64(
+                                <2 x double><double 42.1, double 42.2>,
+                                metadata !"fpexcept.strict") #0
+  ret <2 x i32> %result
 }
 
-define <3 x float> @constrained_vector_ceil_v3f32() {
-; CHECK-LABEL: constrained_vector_ceil_v3f32:
+define <3 x i32> @constrained_vector_fptosi_v3i32_v3f64() #0 {
+; CHECK-LABEL: constrained_vector_fptosi_v3i32_v3f64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    subq $40, %rsp
-; CHECK-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; CHECK-NEXT:    callq ceilf
-; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; CHECK-NEXT:    callq ceilf
-; CHECK-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
-; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; CHECK-NEXT:    callq ceilf
-; CHECK-NEXT:    movaps (%rsp), %xmm1 # 16-byte Reload
-; CHECK-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; CHECK-NEXT:    unpcklpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
-; CHECK-NEXT:    # xmm1 = xmm1[0],mem[0]
-; CHECK-NEXT:    movaps %xmm1, %xmm0
-; CHECK-NEXT:    addq $40, %rsp
-; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    cvttsd2si {{.*}}(%rip), %eax
+; CHECK-NEXT:    movd %eax, %xmm1
+; CHECK-NEXT:    cvttsd2si {{.*}}(%rip), %eax
+; CHECK-NEXT:    movd %eax, %xmm0
+; CHECK-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; CHECK-NEXT:    cvttsd2si {{.*}}(%rip), %eax
+; CHECK-NEXT:    movd %eax, %xmm1
+; CHECK-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
 ; CHECK-NEXT:    retq
 ;
-; AVX-LABEL: constrained_vector_ceil_v3f32:
+; AVX-LABEL: constrained_vector_fptosi_v3i32_v3f64:
 ; AVX:       # %bb.0: # %entry
-; AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX-NEXT:    vroundss $10, %xmm0, %xmm0, %xmm0
-; AVX-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX-NEXT:    vroundss $10, %xmm1, %xmm1, %xmm1
-; AVX-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; AVX-NEXT:    vroundss $10, %xmm2, %xmm2, %xmm2
-; AVX-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[2,3]
-; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3]
+; AVX-NEXT:    vcvttsd2si {{.*}}(%rip), %eax
+; AVX-NEXT:    vmovd %eax, %xmm0
+; AVX-NEXT:    vcvttsd2si {{.*}}(%rip), %eax
+; AVX-NEXT:    vpinsrd $1, %eax, %xmm0, %xmm0
+; AVX-NEXT:    vcvttsd2si {{.*}}(%rip), %eax
+; AVX-NEXT:    vpinsrd $2, %eax, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 entry:
-  %ceil = call <3 x float> @llvm.experimental.constrained.ceil.v3f32(
-                              <3 x float> <float 1.5, float 2.5, float 3.5>,
-                              metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
-  ret <3 x float> %ceil
+  %result = call <3 x i32> @llvm.experimental.constrained.fptosi.v3i32.v3f64(
+                                <3 x double><double 42.1, double 42.2,
+                                             double 42.3>,
+                                metadata !"fpexcept.strict") #0
+  ret <3 x i32> %result
 }
 
-define <3 x double> @constrained_vector_ceil_v3f64() {
-; CHECK-LABEL: constrained_vector_ceil_v3f64:
+define <4 x i32> @constrained_vector_fptosi_v4i32_v4f64() #0 {
+; CHECK-LABEL: constrained_vector_fptosi_v4i32_v4f64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    subq $24, %rsp
-; CHECK-NEXT:    .cfi_def_cfa_offset 32
-; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; CHECK-NEXT:    callq ceil
-; CHECK-NEXT:    movsd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; CHECK-NEXT:    callq ceil
-; CHECK-NEXT:    movsd %xmm0, (%rsp) # 8-byte Spill
-; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; CHECK-NEXT:    callq ceil
-; CHECK-NEXT:    movsd %xmm0, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    fldl {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Reload
-; CHECK-NEXT:    # xmm0 = mem[0],zero
-; CHECK-NEXT:    movsd (%rsp), %xmm1 # 8-byte Reload
-; CHECK-NEXT:    # xmm1 = mem[0],zero
-; CHECK-NEXT:    addq $24, %rsp
-; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    cvttpd2dq {{.*}}(%rip), %xmm1
+; CHECK-NEXT:    cvttpd2dq {{.*}}(%rip), %xmm0
+; CHECK-NEXT:    unpcklpd {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; CHECK-NEXT:    retq
+;
+; AVX-LABEL: constrained_vector_fptosi_v4i32_v4f64:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    vcvttpd2dqy {{.*}}(%rip), %xmm0
+; AVX-NEXT:    retq
+entry:
+  %result = call <4 x i32> @llvm.experimental.constrained.fptosi.v4i32.v4f64(
+                                <4 x double><double 42.1, double 42.2,
+                                             double 42.3, double 42.4>,
+                                metadata !"fpexcept.strict") #0
+  ret <4 x i32> %result
+}
+
+define <1 x i64> @constrained_vector_fptosi_v1i64_v1f64() #0 {
+; CHECK-LABEL: constrained_vector_fptosi_v1i64_v1f64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cvttsd2si {{.*}}(%rip), %rax
+; CHECK-NEXT:    retq
+;
+; AVX-LABEL: constrained_vector_fptosi_v1i64_v1f64:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    vcvttsd2si {{.*}}(%rip), %rax
+; AVX-NEXT:    retq
+entry:
+  %result = call <1 x i64> @llvm.experimental.constrained.fptosi.v1i64.v1f64(
+                               <1 x double><double 42.1>,
+                               metadata !"fpexcept.strict") #0
+  ret <1 x i64> %result
+}
+
+define <2 x i64> @constrained_vector_fptosi_v2i64_v2f64() #0 {
+; CHECK-LABEL: constrained_vector_fptosi_v2i64_v2f64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cvttsd2si {{.*}}(%rip), %rax
+; CHECK-NEXT:    movq %rax, %xmm1
+; CHECK-NEXT:    cvttsd2si {{.*}}(%rip), %rax
+; CHECK-NEXT:    movq %rax, %xmm0
+; CHECK-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; CHECK-NEXT:    retq
+;
+; AVX1-LABEL: constrained_vector_fptosi_v2i64_v2f64:
+; AVX1:       # %bb.0: # %entry
+; AVX1-NEXT:    vcvttsd2si {{.*}}(%rip), %rax
+; AVX1-NEXT:    vmovq %rax, %xmm0
+; AVX1-NEXT:    vcvttsd2si {{.*}}(%rip), %rax
+; AVX1-NEXT:    vmovq %rax, %xmm1
+; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX1-NEXT:    retq
+;
+; AVX512F-LABEL: constrained_vector_fptosi_v2i64_v2f64:
+; AVX512F:       # %bb.0: # %entry
+; AVX512F-NEXT:    vcvttsd2si {{.*}}(%rip), %rax
+; AVX512F-NEXT:    vmovq %rax, %xmm0
+; AVX512F-NEXT:    vcvttsd2si {{.*}}(%rip), %rax
+; AVX512F-NEXT:    vmovq %rax, %xmm1
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512F-NEXT:    retq
+;
+; AVX512DQ-LABEL: constrained_vector_fptosi_v2i64_v2f64:
+; AVX512DQ:       # %bb.0: # %entry
+; AVX512DQ-NEXT:    vmovaps {{.*#+}} xmm0 = [4.2100000000000001E+1,4.2200000000000003E+1]
+; AVX512DQ-NEXT:    vcvttpd2qq %zmm0, %zmm0
+; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    retq
+entry:
+  %result = call <2 x i64> @llvm.experimental.constrained.fptosi.v2i64.v2f64(
+                                <2 x double><double 42.1, double 42.2>,
+                                metadata !"fpexcept.strict") #0
+  ret <2 x i64> %result
+}
+
+define <3 x i64> @constrained_vector_fptosi_v3i64_v3f64() #0 {
+; CHECK-LABEL: constrained_vector_fptosi_v3i64_v3f64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cvttsd2si {{.*}}(%rip), %rcx
+; CHECK-NEXT:    cvttsd2si {{.*}}(%rip), %rdx
+; CHECK-NEXT:    cvttsd2si {{.*}}(%rip), %rax
+; CHECK-NEXT:    retq
+;
+; AVX1-LABEL: constrained_vector_fptosi_v3i64_v3f64:
+; AVX1:       # %bb.0: # %entry
+; AVX1-NEXT:    vcvttsd2si {{.*}}(%rip), %rax
+; AVX1-NEXT:    vmovq %rax, %xmm0
+; AVX1-NEXT:    vcvttsd2si {{.*}}(%rip), %rax
+; AVX1-NEXT:    vmovq %rax, %xmm1
+; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX1-NEXT:    vcvttsd2si {{.*}}(%rip), %rax
+; AVX1-NEXT:    vmovq %rax, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX512-LABEL: constrained_vector_fptosi_v3i64_v3f64:
+; AVX512:       # %bb.0: # %entry
+; AVX512-NEXT:    vcvttsd2si {{.*}}(%rip), %rax
+; AVX512-NEXT:    vmovq %rax, %xmm0
+; AVX512-NEXT:    vcvttsd2si {{.*}}(%rip), %rax
+; AVX512-NEXT:    vmovq %rax, %xmm1
+; AVX512-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512-NEXT:    vcvttsd2si {{.*}}(%rip), %rax
+; AVX512-NEXT:    vmovq %rax, %xmm1
+; AVX512-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512-NEXT:    retq
+entry:
+  %result = call <3 x i64> @llvm.experimental.constrained.fptosi.v3i64.v3f64(
+                                <3 x double><double 42.1, double 42.2,
+                                             double 42.3>,
+                                metadata !"fpexcept.strict") #0
+  ret <3 x i64> %result
+}
+
+define <4 x i64> @constrained_vector_fptosi_v4i64_v4f64() #0 {
+; CHECK-LABEL: constrained_vector_fptosi_v4i64_v4f64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cvttsd2si {{.*}}(%rip), %rax
+; CHECK-NEXT:    movq %rax, %xmm1
+; CHECK-NEXT:    cvttsd2si {{.*}}(%rip), %rax
+; CHECK-NEXT:    movq %rax, %xmm0
+; CHECK-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; CHECK-NEXT:    cvttsd2si {{.*}}(%rip), %rax
+; CHECK-NEXT:    movq %rax, %xmm2
+; CHECK-NEXT:    cvttsd2si {{.*}}(%rip), %rax
+; CHECK-NEXT:    movq %rax, %xmm1
+; CHECK-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; CHECK-NEXT:    retq
+;
+; AVX1-LABEL: constrained_vector_fptosi_v4i64_v4f64:
+; AVX1:       # %bb.0: # %entry
+; AVX1-NEXT:    vcvttsd2si {{.*}}(%rip), %rax
+; AVX1-NEXT:    vmovq %rax, %xmm0
+; AVX1-NEXT:    vcvttsd2si {{.*}}(%rip), %rax
+; AVX1-NEXT:    vmovq %rax, %xmm1
+; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX1-NEXT:    vcvttsd2si {{.*}}(%rip), %rax
+; AVX1-NEXT:    vmovq %rax, %xmm1
+; AVX1-NEXT:    vcvttsd2si {{.*}}(%rip), %rax
+; AVX1-NEXT:    vmovq %rax, %xmm2
+; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX512F-LABEL: constrained_vector_fptosi_v4i64_v4f64:
+; AVX512F:       # %bb.0: # %entry
+; AVX512F-NEXT:    vcvttsd2si {{.*}}(%rip), %rax
+; AVX512F-NEXT:    vmovq %rax, %xmm0
+; AVX512F-NEXT:    vcvttsd2si {{.*}}(%rip), %rax
+; AVX512F-NEXT:    vmovq %rax, %xmm1
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512F-NEXT:    vcvttsd2si {{.*}}(%rip), %rax
+; AVX512F-NEXT:    vmovq %rax, %xmm1
+; AVX512F-NEXT:    vcvttsd2si {{.*}}(%rip), %rax
+; AVX512F-NEXT:    vmovq %rax, %xmm2
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX512F-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX512F-NEXT:    retq
+;
+; AVX512DQ-LABEL: constrained_vector_fptosi_v4i64_v4f64:
+; AVX512DQ:       # %bb.0: # %entry
+; AVX512DQ-NEXT:    vmovaps {{.*#+}} ymm0 = [4.2100000000000001E+1,4.2200000000000003E+1,4.2299999999999997E+1,4.2399999999999999E+1]
+; AVX512DQ-NEXT:    vcvttpd2qq %zmm0, %zmm0
+; AVX512DQ-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512DQ-NEXT:    retq
+entry:
+  %result = call <4 x i64> @llvm.experimental.constrained.fptosi.v4i64.v4f64(
+                                <4 x double><double 42.1, double 42.2,
+                                             double 42.3, double 42.4>,
+                                metadata !"fpexcept.strict") #0
+  ret <4 x i64> %result
+}
+
+define <1 x i32> @constrained_vector_fptoui_v1i32_v1f32() #0 {
+; CHECK-LABEL: constrained_vector_fptoui_v1i32_v1f32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cvttss2si {{.*}}(%rip), %rax
+; CHECK-NEXT:    # kill: def $eax killed $eax killed $rax
+; CHECK-NEXT:    retq
+;
+; AVX1-LABEL: constrained_vector_fptoui_v1i32_v1f32:
+; AVX1:       # %bb.0: # %entry
+; AVX1-NEXT:    vcvttss2si {{.*}}(%rip), %rax
+; AVX1-NEXT:    # kill: def $eax killed $eax killed $rax
+; AVX1-NEXT:    retq
+;
+; AVX512-LABEL: constrained_vector_fptoui_v1i32_v1f32:
+; AVX512:       # %bb.0: # %entry
+; AVX512-NEXT:    vcvttss2usi {{.*}}(%rip), %eax
+; AVX512-NEXT:    retq
+entry:
+  %result = call <1 x i32> @llvm.experimental.constrained.fptoui.v1i32.v1f32(
+                               <1 x float><float 42.0>,
+                               metadata !"fpexcept.strict") #0
+  ret <1 x i32> %result
+}
+
+define <2 x i32> @constrained_vector_fptoui_v2i32_v2f32() #0 {
+; CHECK-LABEL: constrained_vector_fptoui_v2i32_v2f32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cvttss2si {{.*}}(%rip), %rax
+; CHECK-NEXT:    movd %eax, %xmm1
+; CHECK-NEXT:    cvttss2si {{.*}}(%rip), %rax
+; CHECK-NEXT:    movd %eax, %xmm0
+; CHECK-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; CHECK-NEXT:    retq
+;
+; AVX1-LABEL: constrained_vector_fptoui_v2i32_v2f32:
+; AVX1:       # %bb.0: # %entry
+; AVX1-NEXT:    vcvttss2si {{.*}}(%rip), %rax
+; AVX1-NEXT:    vcvttss2si {{.*}}(%rip), %rcx
+; AVX1-NEXT:    vmovd %ecx, %xmm0
+; AVX1-NEXT:    vpinsrd $1, %eax, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX512-LABEL: constrained_vector_fptoui_v2i32_v2f32:
+; AVX512:       # %bb.0: # %entry
+; AVX512-NEXT:    vmovaps {{.*#+}} xmm0 = [4.2E+1,4.3E+1,0.0E+0,0.0E+0]
+; AVX512-NEXT:    vcvttps2udq %zmm0, %zmm0
+; AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512-NEXT:    vzeroupper
+; AVX512-NEXT:    retq
+entry:
+  %result = call <2 x i32> @llvm.experimental.constrained.fptoui.v2i32.v2f32(
+                                <2 x float><float 42.0, float 43.0>,
+                                metadata !"fpexcept.strict") #0
+  ret <2 x i32> %result
+}
+
+define <3 x i32> @constrained_vector_fptoui_v3i32_v3f32() #0 {
+; CHECK-LABEL: constrained_vector_fptoui_v3i32_v3f32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cvttss2si {{.*}}(%rip), %rax
+; CHECK-NEXT:    movd %eax, %xmm1
+; CHECK-NEXT:    cvttss2si {{.*}}(%rip), %rax
+; CHECK-NEXT:    movd %eax, %xmm0
+; CHECK-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; CHECK-NEXT:    cvttss2si {{.*}}(%rip), %rax
+; CHECK-NEXT:    movd %eax, %xmm1
+; CHECK-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; CHECK-NEXT:    retq
+;
+; AVX1-LABEL: constrained_vector_fptoui_v3i32_v3f32:
+; AVX1:       # %bb.0: # %entry
+; AVX1-NEXT:    vcvttss2si {{.*}}(%rip), %rax
+; AVX1-NEXT:    vcvttss2si {{.*}}(%rip), %rcx
+; AVX1-NEXT:    vmovd %ecx, %xmm0
+; AVX1-NEXT:    vpinsrd $1, %eax, %xmm0, %xmm0
+; AVX1-NEXT:    vcvttss2si {{.*}}(%rip), %rax
+; AVX1-NEXT:    vpinsrd $2, %eax, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX512-LABEL: constrained_vector_fptoui_v3i32_v3f32:
+; AVX512:       # %bb.0: # %entry
+; AVX512-NEXT:    vcvttss2usi {{.*}}(%rip), %eax
+; AVX512-NEXT:    vmovd %eax, %xmm0
+; AVX512-NEXT:    vcvttss2usi {{.*}}(%rip), %eax
+; AVX512-NEXT:    vpinsrd $1, %eax, %xmm0, %xmm0
+; AVX512-NEXT:    vcvttss2usi {{.*}}(%rip), %eax
+; AVX512-NEXT:    vpinsrd $2, %eax, %xmm0, %xmm0
+; AVX512-NEXT:    retq
+entry:
+  %result = call <3 x i32> @llvm.experimental.constrained.fptoui.v3i32.v3f32(
+                                <3 x float><float 42.0, float 43.0,
+                                            float 44.0>,
+                                metadata !"fpexcept.strict") #0
+  ret <3 x i32> %result
+}
+
+define <4 x i32> @constrained_vector_fptoui_v4i32_v4f32() #0 {
+; CHECK-LABEL: constrained_vector_fptoui_v4i32_v4f32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movaps {{.*#+}} xmm0 = [2.14748365E+9,2.14748365E+9,2.14748365E+9,2.14748365E+9]
+; CHECK-NEXT:    movaps {{.*#+}} xmm1 = [4.2E+1,4.3E+1,4.4E+1,4.5E+1]
+; CHECK-NEXT:    movaps %xmm1, %xmm2
+; CHECK-NEXT:    cmpltps %xmm0, %xmm2
+; CHECK-NEXT:    movaps %xmm2, %xmm3
+; CHECK-NEXT:    andnps {{.*}}(%rip), %xmm3
+; CHECK-NEXT:    andnps %xmm0, %xmm2
+; CHECK-NEXT:    subps %xmm2, %xmm1
+; CHECK-NEXT:    cvttps2dq %xmm1, %xmm0
+; CHECK-NEXT:    xorps %xmm3, %xmm0
+; CHECK-NEXT:    retq
+;
+; AVX1-LABEL: constrained_vector_fptoui_v4i32_v4f32:
+; AVX1:       # %bb.0: # %entry
+; AVX1-NEXT:    vmovaps {{.*#+}} xmm0 = [2.14748365E+9,2.14748365E+9,2.14748365E+9,2.14748365E+9]
+; AVX1-NEXT:    vmovaps {{.*#+}} xmm1 = [4.2E+1,4.3E+1,4.4E+1,4.5E+1]
+; AVX1-NEXT:    vcmpltps %xmm0, %xmm1, %xmm2
+; AVX1-NEXT:    vxorps %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    vmovaps {{.*#+}} xmm4 = [2147483648,2147483648,2147483648,2147483648]
+; AVX1-NEXT:    vblendvps %xmm2, %xmm3, %xmm4, %xmm4
+; AVX1-NEXT:    vblendvps %xmm2, %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vsubps %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX1-NEXT:    vxorps %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX512-LABEL: constrained_vector_fptoui_v4i32_v4f32:
+; AVX512:       # %bb.0: # %entry
+; AVX512-NEXT:    vmovaps {{.*#+}} xmm0 = [4.2E+1,4.3E+1,4.4E+1,4.5E+1]
+; AVX512-NEXT:    vcvttps2udq %zmm0, %zmm0
+; AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512-NEXT:    vzeroupper
+; AVX512-NEXT:    retq
+entry:
+  %result = call <4 x i32> @llvm.experimental.constrained.fptoui.v4i32.v4f32(
+                                <4 x float><float 42.0, float 43.0,
+                                            float 44.0, float 45.0>,
+                                metadata !"fpexcept.strict") #0
+  ret <4 x i32> %result
+}
+
+define <1 x i64> @constrained_vector_fptoui_v1i64_v1f32() #0 {
+; CHECK-LABEL: constrained_vector_fptoui_v1i64_v1f32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; CHECK-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; CHECK-NEXT:    comiss %xmm0, %xmm2
+; CHECK-NEXT:    xorps %xmm1, %xmm1
+; CHECK-NEXT:    ja .LBB115_2
+; CHECK-NEXT:  # %bb.1: # %entry
+; CHECK-NEXT:    movaps %xmm2, %xmm1
+; CHECK-NEXT:  .LBB115_2: # %entry
+; CHECK-NEXT:    subss %xmm1, %xmm0
+; CHECK-NEXT:    cvttss2si %xmm0, %rcx
+; CHECK-NEXT:    setbe %al
+; CHECK-NEXT:    movzbl %al, %eax
+; CHECK-NEXT:    shlq $63, %rax
+; CHECK-NEXT:    xorq %rcx, %rax
+; CHECK-NEXT:    retq
+;
+; AVX1-LABEL: constrained_vector_fptoui_v1i64_v1f32:
+; AVX1:       # %bb.0: # %entry
+; AVX1-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX1-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX1-NEXT:    vcomiss %xmm0, %xmm1
+; AVX1-NEXT:    vxorps %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    ja .LBB115_2
+; AVX1-NEXT:  # %bb.1: # %entry
+; AVX1-NEXT:    vmovaps %xmm1, %xmm2
+; AVX1-NEXT:  .LBB115_2: # %entry
+; AVX1-NEXT:    vsubss %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vcvttss2si %xmm0, %rcx
+; AVX1-NEXT:    setbe %al
+; AVX1-NEXT:    movzbl %al, %eax
+; AVX1-NEXT:    shlq $63, %rax
+; AVX1-NEXT:    xorq %rcx, %rax
+; AVX1-NEXT:    retq
+;
+; AVX512-LABEL: constrained_vector_fptoui_v1i64_v1f32:
+; AVX512:       # %bb.0: # %entry
+; AVX512-NEXT:    vcvttss2usi {{.*}}(%rip), %rax
+; AVX512-NEXT:    retq
+entry:
+  %result = call <1 x i64> @llvm.experimental.constrained.fptoui.v1i64.v1f32(
+                               <1 x float><float 42.0>,
+                               metadata !"fpexcept.strict") #0
+  ret <1 x i64> %result
+}
+
+define <2 x i64> @constrained_vector_fptoui_v2i64_v2f32() #0 {
+; CHECK-LABEL: constrained_vector_fptoui_v2i64_v2f32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; CHECK-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; CHECK-NEXT:    comiss %xmm2, %xmm1
+; CHECK-NEXT:    xorps %xmm0, %xmm0
+; CHECK-NEXT:    xorps %xmm3, %xmm3
+; CHECK-NEXT:    ja .LBB116_2
+; CHECK-NEXT:  # %bb.1: # %entry
+; CHECK-NEXT:    movaps %xmm1, %xmm3
+; CHECK-NEXT:  .LBB116_2: # %entry
+; CHECK-NEXT:    subss %xmm3, %xmm2
+; CHECK-NEXT:    cvttss2si %xmm2, %rax
+; CHECK-NEXT:    setbe %cl
+; CHECK-NEXT:    movzbl %cl, %ecx
+; CHECK-NEXT:    shlq $63, %rcx
+; CHECK-NEXT:    xorq %rax, %rcx
+; CHECK-NEXT:    movq %rcx, %xmm2
+; CHECK-NEXT:    movss {{.*#+}} xmm3 = mem[0],zero,zero,zero
+; CHECK-NEXT:    comiss %xmm3, %xmm1
+; CHECK-NEXT:    ja .LBB116_4
+; CHECK-NEXT:  # %bb.3: # %entry
+; CHECK-NEXT:    movaps %xmm1, %xmm0
+; CHECK-NEXT:  .LBB116_4: # %entry
+; CHECK-NEXT:    subss %xmm0, %xmm3
+; CHECK-NEXT:    cvttss2si %xmm3, %rax
+; CHECK-NEXT:    setbe %cl
+; CHECK-NEXT:    movzbl %cl, %ecx
+; CHECK-NEXT:    shlq $63, %rcx
+; CHECK-NEXT:    xorq %rax, %rcx
+; CHECK-NEXT:    movq %rcx, %xmm0
+; CHECK-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; CHECK-NEXT:    retq
+;
+; AVX1-LABEL: constrained_vector_fptoui_v2i64_v2f32:
+; AVX1:       # %bb.0: # %entry
+; AVX1-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; AVX1-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX1-NEXT:    vcomiss %xmm2, %xmm0
+; AVX1-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vxorps %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    ja .LBB116_2
+; AVX1-NEXT:  # %bb.1: # %entry
+; AVX1-NEXT:    vmovaps %xmm0, %xmm3
+; AVX1-NEXT:  .LBB116_2: # %entry
+; AVX1-NEXT:    vsubss %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vcvttss2si %xmm2, %rax
+; AVX1-NEXT:    setbe %cl
+; AVX1-NEXT:    movzbl %cl, %ecx
+; AVX1-NEXT:    shlq $63, %rcx
+; AVX1-NEXT:    xorq %rax, %rcx
+; AVX1-NEXT:    vmovq %rcx, %xmm2
+; AVX1-NEXT:    vmovss {{.*#+}} xmm3 = mem[0],zero,zero,zero
+; AVX1-NEXT:    vcomiss %xmm3, %xmm0
+; AVX1-NEXT:    ja .LBB116_4
+; AVX1-NEXT:  # %bb.3: # %entry
+; AVX1-NEXT:    vmovaps %xmm0, %xmm1
+; AVX1-NEXT:  .LBB116_4: # %entry
+; AVX1-NEXT:    vsubss %xmm1, %xmm3, %xmm0
+; AVX1-NEXT:    vcvttss2si %xmm0, %rax
+; AVX1-NEXT:    setbe %cl
+; AVX1-NEXT:    movzbl %cl, %ecx
+; AVX1-NEXT:    shlq $63, %rcx
+; AVX1-NEXT:    xorq %rax, %rcx
+; AVX1-NEXT:    vmovq %rcx, %xmm0
+; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; AVX1-NEXT:    retq
+;
+; AVX512-LABEL: constrained_vector_fptoui_v2i64_v2f32:
+; AVX512:       # %bb.0: # %entry
+; AVX512-NEXT:    vcvttss2usi {{.*}}(%rip), %rax
+; AVX512-NEXT:    vmovq %rax, %xmm0
+; AVX512-NEXT:    vcvttss2usi {{.*}}(%rip), %rax
+; AVX512-NEXT:    vmovq %rax, %xmm1
+; AVX512-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512-NEXT:    retq
+entry:
+  %result = call <2 x i64> @llvm.experimental.constrained.fptoui.v2i64.v2f32(
+                                <2 x float><float 42.0, float 43.0>,
+                                metadata !"fpexcept.strict") #0
+  ret <2 x i64> %result
+}
+
+define <3 x i64> @constrained_vector_fptoui_v3i64_v3f32() #0 {
+; CHECK-LABEL: constrained_vector_fptoui_v3i64_v3f32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; CHECK-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; CHECK-NEXT:    comiss %xmm2, %xmm1
+; CHECK-NEXT:    xorps %xmm0, %xmm0
+; CHECK-NEXT:    xorps %xmm3, %xmm3
+; CHECK-NEXT:    ja .LBB117_2
+; CHECK-NEXT:  # %bb.1: # %entry
+; CHECK-NEXT:    movaps %xmm1, %xmm3
+; CHECK-NEXT:  .LBB117_2: # %entry
+; CHECK-NEXT:    subss %xmm3, %xmm2
+; CHECK-NEXT:    cvttss2si %xmm2, %rcx
+; CHECK-NEXT:    setbe %al
+; CHECK-NEXT:    movzbl %al, %eax
+; CHECK-NEXT:    shlq $63, %rax
+; CHECK-NEXT:    xorq %rcx, %rax
+; CHECK-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; CHECK-NEXT:    comiss %xmm2, %xmm1
+; CHECK-NEXT:    xorps %xmm3, %xmm3
+; CHECK-NEXT:    ja .LBB117_4
+; CHECK-NEXT:  # %bb.3: # %entry
+; CHECK-NEXT:    movaps %xmm1, %xmm3
+; CHECK-NEXT:  .LBB117_4: # %entry
+; CHECK-NEXT:    subss %xmm3, %xmm2
+; CHECK-NEXT:    cvttss2si %xmm2, %rcx
+; CHECK-NEXT:    setbe %dl
+; CHECK-NEXT:    movzbl %dl, %edx
+; CHECK-NEXT:    shlq $63, %rdx
+; CHECK-NEXT:    xorq %rcx, %rdx
+; CHECK-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; CHECK-NEXT:    comiss %xmm2, %xmm1
+; CHECK-NEXT:    ja .LBB117_6
+; CHECK-NEXT:  # %bb.5: # %entry
+; CHECK-NEXT:    movaps %xmm1, %xmm0
+; CHECK-NEXT:  .LBB117_6: # %entry
+; CHECK-NEXT:    subss %xmm0, %xmm2
+; CHECK-NEXT:    cvttss2si %xmm2, %rsi
+; CHECK-NEXT:    setbe %cl
+; CHECK-NEXT:    movzbl %cl, %ecx
+; CHECK-NEXT:    shlq $63, %rcx
+; CHECK-NEXT:    xorq %rsi, %rcx
+; CHECK-NEXT:    retq
+;
+; AVX1-LABEL: constrained_vector_fptoui_v3i64_v3f32:
+; AVX1:       # %bb.0: # %entry
+; AVX1-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; AVX1-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX1-NEXT:    vcomiss %xmm2, %xmm0
+; AVX1-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vxorps %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    ja .LBB117_2
+; AVX1-NEXT:  # %bb.1: # %entry
+; AVX1-NEXT:    vmovaps %xmm0, %xmm3
+; AVX1-NEXT:  .LBB117_2: # %entry
+; AVX1-NEXT:    vsubss %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vcvttss2si %xmm2, %rax
+; AVX1-NEXT:    setbe %cl
+; AVX1-NEXT:    movzbl %cl, %ecx
+; AVX1-NEXT:    shlq $63, %rcx
+; AVX1-NEXT:    xorq %rax, %rcx
+; AVX1-NEXT:    vmovq %rcx, %xmm2
+; AVX1-NEXT:    vmovss {{.*#+}} xmm3 = mem[0],zero,zero,zero
+; AVX1-NEXT:    vcomiss %xmm3, %xmm0
+; AVX1-NEXT:    vxorps %xmm4, %xmm4, %xmm4
+; AVX1-NEXT:    ja .LBB117_4
+; AVX1-NEXT:  # %bb.3: # %entry
+; AVX1-NEXT:    vmovaps %xmm0, %xmm4
+; AVX1-NEXT:  .LBB117_4: # %entry
+; AVX1-NEXT:    vsubss %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vcvttss2si %xmm3, %rax
+; AVX1-NEXT:    setbe %cl
+; AVX1-NEXT:    movzbl %cl, %ecx
+; AVX1-NEXT:    shlq $63, %rcx
+; AVX1-NEXT:    xorq %rax, %rcx
+; AVX1-NEXT:    vmovq %rcx, %xmm3
+; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
+; AVX1-NEXT:    vmovss {{.*#+}} xmm3 = mem[0],zero,zero,zero
+; AVX1-NEXT:    vcomiss %xmm3, %xmm0
+; AVX1-NEXT:    ja .LBB117_6
+; AVX1-NEXT:  # %bb.5: # %entry
+; AVX1-NEXT:    vmovaps %xmm0, %xmm1
+; AVX1-NEXT:  .LBB117_6: # %entry
+; AVX1-NEXT:    vsubss %xmm1, %xmm3, %xmm0
+; AVX1-NEXT:    vcvttss2si %xmm0, %rax
+; AVX1-NEXT:    setbe %cl
+; AVX1-NEXT:    movzbl %cl, %ecx
+; AVX1-NEXT:    shlq $63, %rcx
+; AVX1-NEXT:    xorq %rax, %rcx
+; AVX1-NEXT:    vmovq %rcx, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX512-LABEL: constrained_vector_fptoui_v3i64_v3f32:
+; AVX512:       # %bb.0: # %entry
+; AVX512-NEXT:    vcvttss2usi {{.*}}(%rip), %rax
+; AVX512-NEXT:    vmovq %rax, %xmm0
+; AVX512-NEXT:    vcvttss2usi {{.*}}(%rip), %rax
+; AVX512-NEXT:    vmovq %rax, %xmm1
+; AVX512-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512-NEXT:    vcvttss2usi {{.*}}(%rip), %rax
+; AVX512-NEXT:    vmovq %rax, %xmm1
+; AVX512-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512-NEXT:    retq
+entry:
+  %result = call <3 x i64> @llvm.experimental.constrained.fptoui.v3i64.v3f32(
+                                <3 x float><float 42.0, float 43.0,
+                                            float 44.0>,
+                                metadata !"fpexcept.strict") #0
+  ret <3 x i64> %result
+}
+
+define <4 x i64> @constrained_vector_fptoui_v4i64_v4f32() #0 {
+; CHECK-LABEL: constrained_vector_fptoui_v4i64_v4f32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; CHECK-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; CHECK-NEXT:    comiss %xmm0, %xmm2
+; CHECK-NEXT:    xorps %xmm1, %xmm1
+; CHECK-NEXT:    xorps %xmm3, %xmm3
+; CHECK-NEXT:    ja .LBB118_2
+; CHECK-NEXT:  # %bb.1: # %entry
+; CHECK-NEXT:    movaps %xmm2, %xmm3
+; CHECK-NEXT:  .LBB118_2: # %entry
+; CHECK-NEXT:    subss %xmm3, %xmm0
+; CHECK-NEXT:    cvttss2si %xmm0, %rcx
+; CHECK-NEXT:    setbe %al
+; CHECK-NEXT:    movzbl %al, %eax
+; CHECK-NEXT:    shlq $63, %rax
+; CHECK-NEXT:    xorq %rcx, %rax
+; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; CHECK-NEXT:    comiss %xmm0, %xmm2
+; CHECK-NEXT:    xorps %xmm4, %xmm4
+; CHECK-NEXT:    ja .LBB118_4
+; CHECK-NEXT:  # %bb.3: # %entry
+; CHECK-NEXT:    movaps %xmm2, %xmm4
+; CHECK-NEXT:  .LBB118_4: # %entry
+; CHECK-NEXT:    movq %rax, %xmm3
+; CHECK-NEXT:    subss %xmm4, %xmm0
+; CHECK-NEXT:    cvttss2si %xmm0, %rax
+; CHECK-NEXT:    setbe %cl
+; CHECK-NEXT:    movzbl %cl, %ecx
+; CHECK-NEXT:    shlq $63, %rcx
+; CHECK-NEXT:    xorq %rax, %rcx
+; CHECK-NEXT:    movq %rcx, %xmm0
+; CHECK-NEXT:    movss {{.*#+}} xmm4 = mem[0],zero,zero,zero
+; CHECK-NEXT:    comiss %xmm4, %xmm2
+; CHECK-NEXT:    xorps %xmm5, %xmm5
+; CHECK-NEXT:    ja .LBB118_6
+; CHECK-NEXT:  # %bb.5: # %entry
+; CHECK-NEXT:    movaps %xmm2, %xmm5
+; CHECK-NEXT:  .LBB118_6: # %entry
+; CHECK-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
+; CHECK-NEXT:    subss %xmm5, %xmm4
+; CHECK-NEXT:    cvttss2si %xmm4, %rax
+; CHECK-NEXT:    setbe %cl
+; CHECK-NEXT:    movzbl %cl, %ecx
+; CHECK-NEXT:    shlq $63, %rcx
+; CHECK-NEXT:    xorq %rax, %rcx
+; CHECK-NEXT:    movq %rcx, %xmm3
+; CHECK-NEXT:    movss {{.*#+}} xmm4 = mem[0],zero,zero,zero
+; CHECK-NEXT:    comiss %xmm4, %xmm2
+; CHECK-NEXT:    ja .LBB118_8
+; CHECK-NEXT:  # %bb.7: # %entry
+; CHECK-NEXT:    movaps %xmm2, %xmm1
+; CHECK-NEXT:  .LBB118_8: # %entry
+; CHECK-NEXT:    subss %xmm1, %xmm4
+; CHECK-NEXT:    cvttss2si %xmm4, %rax
+; CHECK-NEXT:    setbe %cl
+; CHECK-NEXT:    movzbl %cl, %ecx
+; CHECK-NEXT:    shlq $63, %rcx
+; CHECK-NEXT:    xorq %rax, %rcx
+; CHECK-NEXT:    movq %rcx, %xmm1
+; CHECK-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; CHECK-NEXT:    retq
+;
+; AVX1-LABEL: constrained_vector_fptoui_v4i64_v4f32:
+; AVX1:       # %bb.0: # %entry
+; AVX1-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; AVX1-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX1-NEXT:    vcomiss %xmm2, %xmm0
+; AVX1-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vxorps %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    ja .LBB118_2
+; AVX1-NEXT:  # %bb.1: # %entry
+; AVX1-NEXT:    vmovaps %xmm0, %xmm3
+; AVX1-NEXT:  .LBB118_2: # %entry
+; AVX1-NEXT:    vsubss %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vcvttss2si %xmm2, %rcx
+; AVX1-NEXT:    setbe %al
+; AVX1-NEXT:    movzbl %al, %eax
+; AVX1-NEXT:    shlq $63, %rax
+; AVX1-NEXT:    xorq %rcx, %rax
+; AVX1-NEXT:    vmovss {{.*#+}} xmm3 = mem[0],zero,zero,zero
+; AVX1-NEXT:    vcomiss %xmm3, %xmm0
+; AVX1-NEXT:    vxorps %xmm4, %xmm4, %xmm4
+; AVX1-NEXT:    ja .LBB118_4
+; AVX1-NEXT:  # %bb.3: # %entry
+; AVX1-NEXT:    vmovaps %xmm0, %xmm4
+; AVX1-NEXT:  .LBB118_4: # %entry
+; AVX1-NEXT:    vmovq %rax, %xmm2
+; AVX1-NEXT:    vsubss %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vcvttss2si %xmm3, %rax
+; AVX1-NEXT:    setbe %cl
+; AVX1-NEXT:    movzbl %cl, %ecx
+; AVX1-NEXT:    shlq $63, %rcx
+; AVX1-NEXT:    xorq %rax, %rcx
+; AVX1-NEXT:    vmovq %rcx, %xmm3
+; AVX1-NEXT:    vmovss {{.*#+}} xmm4 = mem[0],zero,zero,zero
+; AVX1-NEXT:    vcomiss %xmm4, %xmm0
+; AVX1-NEXT:    vxorps %xmm5, %xmm5, %xmm5
+; AVX1-NEXT:    ja .LBB118_6
+; AVX1-NEXT:  # %bb.5: # %entry
+; AVX1-NEXT:    vmovaps %xmm0, %xmm5
+; AVX1-NEXT:  .LBB118_6: # %entry
+; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
+; AVX1-NEXT:    vsubss %xmm5, %xmm4, %xmm3
+; AVX1-NEXT:    vcvttss2si %xmm3, %rax
+; AVX1-NEXT:    setbe %cl
+; AVX1-NEXT:    movzbl %cl, %ecx
+; AVX1-NEXT:    shlq $63, %rcx
+; AVX1-NEXT:    xorq %rax, %rcx
+; AVX1-NEXT:    vmovq %rcx, %xmm3
+; AVX1-NEXT:    vmovss {{.*#+}} xmm4 = mem[0],zero,zero,zero
+; AVX1-NEXT:    vcomiss %xmm4, %xmm0
+; AVX1-NEXT:    ja .LBB118_8
+; AVX1-NEXT:  # %bb.7: # %entry
+; AVX1-NEXT:    vmovaps %xmm0, %xmm1
+; AVX1-NEXT:  .LBB118_8: # %entry
+; AVX1-NEXT:    vsubss %xmm1, %xmm4, %xmm0
+; AVX1-NEXT:    vcvttss2si %xmm0, %rax
+; AVX1-NEXT:    setbe %cl
+; AVX1-NEXT:    movzbl %cl, %ecx
+; AVX1-NEXT:    shlq $63, %rcx
+; AVX1-NEXT:    xorq %rax, %rcx
+; AVX1-NEXT:    vmovq %rcx, %xmm0
+; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX512F-LABEL: constrained_vector_fptoui_v4i64_v4f32:
+; AVX512F:       # %bb.0: # %entry
+; AVX512F-NEXT:    vcvttss2usi {{.*}}(%rip), %rax
+; AVX512F-NEXT:    vmovq %rax, %xmm0
+; AVX512F-NEXT:    vcvttss2usi {{.*}}(%rip), %rax
+; AVX512F-NEXT:    vmovq %rax, %xmm1
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512F-NEXT:    vcvttss2usi {{.*}}(%rip), %rax
+; AVX512F-NEXT:    vmovq %rax, %xmm1
+; AVX512F-NEXT:    vcvttss2usi {{.*}}(%rip), %rax
+; AVX512F-NEXT:    vmovq %rax, %xmm2
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX512F-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX512F-NEXT:    retq
+;
+; AVX512DQ-LABEL: constrained_vector_fptoui_v4i64_v4f32:
+; AVX512DQ:       # %bb.0: # %entry
+; AVX512DQ-NEXT:    vmovaps {{.*#+}} xmm0 = [4.2E+1,4.3E+1,4.4E+1,4.5E+1]
+; AVX512DQ-NEXT:    vcvttps2uqq %ymm0, %zmm0
+; AVX512DQ-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512DQ-NEXT:    retq
+entry:
+  %result = call <4 x i64> @llvm.experimental.constrained.fptoui.v4i64.v4f32(
+                                <4 x float><float 42.0, float 43.0,
+                                            float 44.0, float 45.0>,
+                                metadata !"fpexcept.strict") #0
+  ret <4 x i64> %result
+}
+
+define <1 x i32> @constrained_vector_fptoui_v1i32_v1f64() #0 {
+; CHECK-LABEL: constrained_vector_fptoui_v1i32_v1f64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cvttsd2si {{.*}}(%rip), %rax
+; CHECK-NEXT:    # kill: def $eax killed $eax killed $rax
+; CHECK-NEXT:    retq
+;
+; AVX1-LABEL: constrained_vector_fptoui_v1i32_v1f64:
+; AVX1:       # %bb.0: # %entry
+; AVX1-NEXT:    vcvttsd2si {{.*}}(%rip), %rax
+; AVX1-NEXT:    # kill: def $eax killed $eax killed $rax
+; AVX1-NEXT:    retq
+;
+; AVX512-LABEL: constrained_vector_fptoui_v1i32_v1f64:
+; AVX512:       # %bb.0: # %entry
+; AVX512-NEXT:    vcvttsd2usi {{.*}}(%rip), %eax
+; AVX512-NEXT:    retq
+entry:
+  %result = call <1 x i32> @llvm.experimental.constrained.fptoui.v1i32.v1f64(
+                               <1 x double><double 42.1>,
+                               metadata !"fpexcept.strict") #0
+  ret <1 x i32> %result
+}
+
+define <2 x i32> @constrained_vector_fptoui_v2i32_v2f64() #0 {
+; CHECK-LABEL: constrained_vector_fptoui_v2i32_v2f64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cvttsd2si {{.*}}(%rip), %rax
+; CHECK-NEXT:    movd %eax, %xmm1
+; CHECK-NEXT:    cvttsd2si {{.*}}(%rip), %rax
+; CHECK-NEXT:    movd %eax, %xmm0
+; CHECK-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; CHECK-NEXT:    retq
+;
+; AVX1-LABEL: constrained_vector_fptoui_v2i32_v2f64:
+; AVX1:       # %bb.0: # %entry
+; AVX1-NEXT:    vcvttsd2si {{.*}}(%rip), %rax
+; AVX1-NEXT:    vcvttsd2si {{.*}}(%rip), %rcx
+; AVX1-NEXT:    vmovd %ecx, %xmm0
+; AVX1-NEXT:    vpinsrd $1, %eax, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX512-LABEL: constrained_vector_fptoui_v2i32_v2f64:
+; AVX512:       # %bb.0: # %entry
+; AVX512-NEXT:    vmovaps {{.*#+}} ymm0 = [4.2100000000000001E+1,4.2200000000000003E+1,0.0E+0,0.0E+0]
+; AVX512-NEXT:    vcvttpd2udq %zmm0, %ymm0
+; AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512-NEXT:    vzeroupper
+; AVX512-NEXT:    retq
+entry:
+  %result = call <2 x i32> @llvm.experimental.constrained.fptoui.v2i32.v2f64(
+                                <2 x double><double 42.1, double 42.2>,
+                                metadata !"fpexcept.strict") #0
+  ret <2 x i32> %result
+}
+
+define <3 x i32> @constrained_vector_fptoui_v3i32_v3f64() #0 {
+; CHECK-LABEL: constrained_vector_fptoui_v3i32_v3f64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cvttsd2si {{.*}}(%rip), %rax
+; CHECK-NEXT:    movd %eax, %xmm1
+; CHECK-NEXT:    cvttsd2si {{.*}}(%rip), %rax
+; CHECK-NEXT:    movd %eax, %xmm0
+; CHECK-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; CHECK-NEXT:    cvttsd2si {{.*}}(%rip), %rax
+; CHECK-NEXT:    movd %eax, %xmm1
+; CHECK-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; CHECK-NEXT:    retq
+;
+; AVX1-LABEL: constrained_vector_fptoui_v3i32_v3f64:
+; AVX1:       # %bb.0: # %entry
+; AVX1-NEXT:    vcvttsd2si {{.*}}(%rip), %rax
+; AVX1-NEXT:    vcvttsd2si {{.*}}(%rip), %rcx
+; AVX1-NEXT:    vmovd %ecx, %xmm0
+; AVX1-NEXT:    vpinsrd $1, %eax, %xmm0, %xmm0
+; AVX1-NEXT:    vcvttsd2si {{.*}}(%rip), %rax
+; AVX1-NEXT:    vpinsrd $2, %eax, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX512-LABEL: constrained_vector_fptoui_v3i32_v3f64:
+; AVX512:       # %bb.0: # %entry
+; AVX512-NEXT:    vcvttsd2usi {{.*}}(%rip), %eax
+; AVX512-NEXT:    vmovd %eax, %xmm0
+; AVX512-NEXT:    vcvttsd2usi {{.*}}(%rip), %eax
+; AVX512-NEXT:    vpinsrd $1, %eax, %xmm0, %xmm0
+; AVX512-NEXT:    vcvttsd2usi {{.*}}(%rip), %eax
+; AVX512-NEXT:    vpinsrd $2, %eax, %xmm0, %xmm0
+; AVX512-NEXT:    retq
+entry:
+  %result = call <3 x i32> @llvm.experimental.constrained.fptoui.v3i32.v3f64(
+                                <3 x double><double 42.1, double 42.2,
+                                             double 42.3>,
+                                metadata !"fpexcept.strict") #0
+  ret <3 x i32> %result
+}
+
+define <4 x i32> @constrained_vector_fptoui_v4i32_v4f64() #0 {
+; CHECK-LABEL: constrained_vector_fptoui_v4i32_v4f64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cvttsd2si {{.*}}(%rip), %rax
+; CHECK-NEXT:    movd %eax, %xmm0
+; CHECK-NEXT:    cvttsd2si {{.*}}(%rip), %rax
+; CHECK-NEXT:    movd %eax, %xmm1
+; CHECK-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; CHECK-NEXT:    cvttsd2si {{.*}}(%rip), %rax
+; CHECK-NEXT:    movd %eax, %xmm2
+; CHECK-NEXT:    cvttsd2si {{.*}}(%rip), %rax
+; CHECK-NEXT:    movd %eax, %xmm0
+; CHECK-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; CHECK-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; CHECK-NEXT:    retq
+;
+; AVX1-LABEL: constrained_vector_fptoui_v4i32_v4f64:
+; AVX1:       # %bb.0: # %entry
+; AVX1-NEXT:    vmovapd {{.*#+}} ymm0 = [2.147483648E+9,2.147483648E+9,2.147483648E+9,2.147483648E+9]
+; AVX1-NEXT:    vmovapd {{.*#+}} ymm1 = [4.2100000000000001E+1,4.2200000000000003E+1,4.2299999999999997E+1,4.2399999999999999E+1]
+; AVX1-NEXT:    vcmpltpd %ymm0, %ymm1, %ymm2
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm3
+; AVX1-NEXT:    vshufps {{.*#+}} xmm3 = xmm2[0,2],xmm3[0,2]
+; AVX1-NEXT:    vxorps %xmm4, %xmm4, %xmm4
+; AVX1-NEXT:    vmovaps {{.*#+}} xmm5 = [2147483648,2147483648,2147483648,2147483648]
+; AVX1-NEXT:    vblendvps %xmm3, %xmm4, %xmm5, %xmm3
+; AVX1-NEXT:    vxorps %xmm4, %xmm4, %xmm4
+; AVX1-NEXT:    vblendvpd %ymm2, %ymm4, %ymm0, %ymm0
+; AVX1-NEXT:    vsubpd %ymm0, %ymm1, %ymm0
+; AVX1-NEXT:    vcvttpd2dq %ymm0, %xmm0
+; AVX1-NEXT:    vxorpd %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX512-LABEL: constrained_vector_fptoui_v4i32_v4f64:
+; AVX512:       # %bb.0: # %entry
+; AVX512-NEXT:    vmovaps {{.*#+}} ymm0 = [4.2100000000000001E+1,4.2200000000000003E+1,4.2299999999999997E+1,4.2399999999999999E+1]
+; AVX512-NEXT:    vcvttpd2udq %zmm0, %ymm0
+; AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512-NEXT:    vzeroupper
+; AVX512-NEXT:    retq
+entry:
+  %result = call <4 x i32> @llvm.experimental.constrained.fptoui.v4i32.v4f64(
+                                <4 x double><double 42.1, double 42.2,
+                                             double 42.3, double 42.4>,
+                                metadata !"fpexcept.strict") #0
+  ret <4 x i32> %result
+}
+
+define <1 x i64> @constrained_vector_fptoui_v1i64_v1f64() #0 {
+; CHECK-LABEL: constrained_vector_fptoui_v1i64_v1f64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    movsd {{.*#+}} xmm2 = mem[0],zero
+; CHECK-NEXT:    comisd %xmm0, %xmm2
+; CHECK-NEXT:    xorpd %xmm1, %xmm1
+; CHECK-NEXT:    ja .LBB123_2
+; CHECK-NEXT:  # %bb.1: # %entry
+; CHECK-NEXT:    movapd %xmm2, %xmm1
+; CHECK-NEXT:  .LBB123_2: # %entry
+; CHECK-NEXT:    subsd %xmm1, %xmm0
+; CHECK-NEXT:    cvttsd2si %xmm0, %rcx
+; CHECK-NEXT:    setbe %al
+; CHECK-NEXT:    movzbl %al, %eax
+; CHECK-NEXT:    shlq $63, %rax
+; CHECK-NEXT:    xorq %rcx, %rax
+; CHECK-NEXT:    retq
+;
+; AVX1-LABEL: constrained_vector_fptoui_v1i64_v1f64:
+; AVX1:       # %bb.0: # %entry
+; AVX1-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX1-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
+; AVX1-NEXT:    vcomisd %xmm0, %xmm1
+; AVX1-NEXT:    vxorpd %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    ja .LBB123_2
+; AVX1-NEXT:  # %bb.1: # %entry
+; AVX1-NEXT:    vmovapd %xmm1, %xmm2
+; AVX1-NEXT:  .LBB123_2: # %entry
+; AVX1-NEXT:    vsubsd %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vcvttsd2si %xmm0, %rcx
+; AVX1-NEXT:    setbe %al
+; AVX1-NEXT:    movzbl %al, %eax
+; AVX1-NEXT:    shlq $63, %rax
+; AVX1-NEXT:    xorq %rcx, %rax
+; AVX1-NEXT:    retq
+;
+; AVX512-LABEL: constrained_vector_fptoui_v1i64_v1f64:
+; AVX512:       # %bb.0: # %entry
+; AVX512-NEXT:    vcvttsd2usi {{.*}}(%rip), %rax
+; AVX512-NEXT:    retq
+entry:
+  %result = call <1 x i64> @llvm.experimental.constrained.fptoui.v1i64.v1f64(
+                               <1 x double><double 42.1>,
+                               metadata !"fpexcept.strict") #0
+  ret <1 x i64> %result
+}
+
+define <2 x i64> @constrained_vector_fptoui_v2i64_v2f64() #0 {
+; CHECK-LABEL: constrained_vector_fptoui_v2i64_v2f64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movsd {{.*#+}} xmm2 = mem[0],zero
+; CHECK-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
+; CHECK-NEXT:    comisd %xmm2, %xmm1
+; CHECK-NEXT:    xorpd %xmm0, %xmm0
+; CHECK-NEXT:    xorpd %xmm3, %xmm3
+; CHECK-NEXT:    ja .LBB124_2
+; CHECK-NEXT:  # %bb.1: # %entry
+; CHECK-NEXT:    movapd %xmm1, %xmm3
+; CHECK-NEXT:  .LBB124_2: # %entry
+; CHECK-NEXT:    subsd %xmm3, %xmm2
+; CHECK-NEXT:    cvttsd2si %xmm2, %rax
+; CHECK-NEXT:    setbe %cl
+; CHECK-NEXT:    movzbl %cl, %ecx
+; CHECK-NEXT:    shlq $63, %rcx
+; CHECK-NEXT:    xorq %rax, %rcx
+; CHECK-NEXT:    movq %rcx, %xmm2
+; CHECK-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
+; CHECK-NEXT:    comisd %xmm3, %xmm1
+; CHECK-NEXT:    ja .LBB124_4
+; CHECK-NEXT:  # %bb.3: # %entry
+; CHECK-NEXT:    movapd %xmm1, %xmm0
+; CHECK-NEXT:  .LBB124_4: # %entry
+; CHECK-NEXT:    subsd %xmm0, %xmm3
+; CHECK-NEXT:    cvttsd2si %xmm3, %rax
+; CHECK-NEXT:    setbe %cl
+; CHECK-NEXT:    movzbl %cl, %ecx
+; CHECK-NEXT:    shlq $63, %rcx
+; CHECK-NEXT:    xorq %rax, %rcx
+; CHECK-NEXT:    movq %rcx, %xmm0
+; CHECK-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; CHECK-NEXT:    retq
+;
+; AVX1-LABEL: constrained_vector_fptoui_v2i64_v2f64:
+; AVX1:       # %bb.0: # %entry
+; AVX1-NEXT:    vmovsd {{.*#+}} xmm2 = mem[0],zero
+; AVX1-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX1-NEXT:    vcomisd %xmm2, %xmm0
+; AVX1-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vxorpd %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    ja .LBB124_2
+; AVX1-NEXT:  # %bb.1: # %entry
+; AVX1-NEXT:    vmovapd %xmm0, %xmm3
+; AVX1-NEXT:  .LBB124_2: # %entry
+; AVX1-NEXT:    vsubsd %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vcvttsd2si %xmm2, %rax
+; AVX1-NEXT:    setbe %cl
+; AVX1-NEXT:    movzbl %cl, %ecx
+; AVX1-NEXT:    shlq $63, %rcx
+; AVX1-NEXT:    xorq %rax, %rcx
+; AVX1-NEXT:    vmovq %rcx, %xmm2
+; AVX1-NEXT:    vmovsd {{.*#+}} xmm3 = mem[0],zero
+; AVX1-NEXT:    vcomisd %xmm3, %xmm0
+; AVX1-NEXT:    ja .LBB124_4
+; AVX1-NEXT:  # %bb.3: # %entry
+; AVX1-NEXT:    vmovapd %xmm0, %xmm1
+; AVX1-NEXT:  .LBB124_4: # %entry
+; AVX1-NEXT:    vsubsd %xmm1, %xmm3, %xmm0
+; AVX1-NEXT:    vcvttsd2si %xmm0, %rax
+; AVX1-NEXT:    setbe %cl
+; AVX1-NEXT:    movzbl %cl, %ecx
+; AVX1-NEXT:    shlq $63, %rcx
+; AVX1-NEXT:    xorq %rax, %rcx
+; AVX1-NEXT:    vmovq %rcx, %xmm0
+; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; AVX1-NEXT:    retq
+;
+; AVX512F-LABEL: constrained_vector_fptoui_v2i64_v2f64:
+; AVX512F:       # %bb.0: # %entry
+; AVX512F-NEXT:    vcvttsd2usi {{.*}}(%rip), %rax
+; AVX512F-NEXT:    vmovq %rax, %xmm0
+; AVX512F-NEXT:    vcvttsd2usi {{.*}}(%rip), %rax
+; AVX512F-NEXT:    vmovq %rax, %xmm1
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512F-NEXT:    retq
+;
+; AVX512DQ-LABEL: constrained_vector_fptoui_v2i64_v2f64:
+; AVX512DQ:       # %bb.0: # %entry
+; AVX512DQ-NEXT:    vmovaps {{.*#+}} xmm0 = [4.2100000000000001E+1,4.2200000000000003E+1]
+; AVX512DQ-NEXT:    vcvttpd2uqq %zmm0, %zmm0
+; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    retq
+entry:
+  %result = call <2 x i64> @llvm.experimental.constrained.fptoui.v2i64.v2f64(
+                                <2 x double><double 42.1, double 42.2>,
+                                metadata !"fpexcept.strict") #0
+  ret <2 x i64> %result
+}
+
+define <3 x i64> @constrained_vector_fptoui_v3i64_v3f64() #0 {
+; CHECK-LABEL: constrained_vector_fptoui_v3i64_v3f64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movsd {{.*#+}} xmm2 = mem[0],zero
+; CHECK-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
+; CHECK-NEXT:    comisd %xmm2, %xmm1
+; CHECK-NEXT:    xorpd %xmm0, %xmm0
+; CHECK-NEXT:    xorpd %xmm3, %xmm3
+; CHECK-NEXT:    ja .LBB125_2
+; CHECK-NEXT:  # %bb.1: # %entry
+; CHECK-NEXT:    movapd %xmm1, %xmm3
+; CHECK-NEXT:  .LBB125_2: # %entry
+; CHECK-NEXT:    subsd %xmm3, %xmm2
+; CHECK-NEXT:    cvttsd2si %xmm2, %rcx
+; CHECK-NEXT:    setbe %al
+; CHECK-NEXT:    movzbl %al, %eax
+; CHECK-NEXT:    shlq $63, %rax
+; CHECK-NEXT:    xorq %rcx, %rax
+; CHECK-NEXT:    movsd {{.*#+}} xmm2 = mem[0],zero
+; CHECK-NEXT:    comisd %xmm2, %xmm1
+; CHECK-NEXT:    xorpd %xmm3, %xmm3
+; CHECK-NEXT:    ja .LBB125_4
+; CHECK-NEXT:  # %bb.3: # %entry
+; CHECK-NEXT:    movapd %xmm1, %xmm3
+; CHECK-NEXT:  .LBB125_4: # %entry
+; CHECK-NEXT:    subsd %xmm3, %xmm2
+; CHECK-NEXT:    cvttsd2si %xmm2, %rcx
+; CHECK-NEXT:    setbe %dl
+; CHECK-NEXT:    movzbl %dl, %edx
+; CHECK-NEXT:    shlq $63, %rdx
+; CHECK-NEXT:    xorq %rcx, %rdx
+; CHECK-NEXT:    movsd {{.*#+}} xmm2 = mem[0],zero
+; CHECK-NEXT:    comisd %xmm2, %xmm1
+; CHECK-NEXT:    ja .LBB125_6
+; CHECK-NEXT:  # %bb.5: # %entry
+; CHECK-NEXT:    movapd %xmm1, %xmm0
+; CHECK-NEXT:  .LBB125_6: # %entry
+; CHECK-NEXT:    subsd %xmm0, %xmm2
+; CHECK-NEXT:    cvttsd2si %xmm2, %rsi
+; CHECK-NEXT:    setbe %cl
+; CHECK-NEXT:    movzbl %cl, %ecx
+; CHECK-NEXT:    shlq $63, %rcx
+; CHECK-NEXT:    xorq %rsi, %rcx
+; CHECK-NEXT:    retq
+;
+; AVX1-LABEL: constrained_vector_fptoui_v3i64_v3f64:
+; AVX1:       # %bb.0: # %entry
+; AVX1-NEXT:    vmovsd {{.*#+}} xmm2 = mem[0],zero
+; AVX1-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX1-NEXT:    vcomisd %xmm2, %xmm0
+; AVX1-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vxorpd %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    ja .LBB125_2
+; AVX1-NEXT:  # %bb.1: # %entry
+; AVX1-NEXT:    vmovapd %xmm0, %xmm3
+; AVX1-NEXT:  .LBB125_2: # %entry
+; AVX1-NEXT:    vsubsd %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vcvttsd2si %xmm2, %rax
+; AVX1-NEXT:    setbe %cl
+; AVX1-NEXT:    movzbl %cl, %ecx
+; AVX1-NEXT:    shlq $63, %rcx
+; AVX1-NEXT:    xorq %rax, %rcx
+; AVX1-NEXT:    vmovq %rcx, %xmm2
+; AVX1-NEXT:    vmovsd {{.*#+}} xmm3 = mem[0],zero
+; AVX1-NEXT:    vcomisd %xmm3, %xmm0
+; AVX1-NEXT:    vxorpd %xmm4, %xmm4, %xmm4
+; AVX1-NEXT:    ja .LBB125_4
+; AVX1-NEXT:  # %bb.3: # %entry
+; AVX1-NEXT:    vmovapd %xmm0, %xmm4
+; AVX1-NEXT:  .LBB125_4: # %entry
+; AVX1-NEXT:    vsubsd %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vcvttsd2si %xmm3, %rax
+; AVX1-NEXT:    setbe %cl
+; AVX1-NEXT:    movzbl %cl, %ecx
+; AVX1-NEXT:    shlq $63, %rcx
+; AVX1-NEXT:    xorq %rax, %rcx
+; AVX1-NEXT:    vmovq %rcx, %xmm3
+; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
+; AVX1-NEXT:    vmovsd {{.*#+}} xmm3 = mem[0],zero
+; AVX1-NEXT:    vcomisd %xmm3, %xmm0
+; AVX1-NEXT:    ja .LBB125_6
+; AVX1-NEXT:  # %bb.5: # %entry
+; AVX1-NEXT:    vmovapd %xmm0, %xmm1
+; AVX1-NEXT:  .LBB125_6: # %entry
+; AVX1-NEXT:    vsubsd %xmm1, %xmm3, %xmm0
+; AVX1-NEXT:    vcvttsd2si %xmm0, %rax
+; AVX1-NEXT:    setbe %cl
+; AVX1-NEXT:    movzbl %cl, %ecx
+; AVX1-NEXT:    shlq $63, %rcx
+; AVX1-NEXT:    xorq %rax, %rcx
+; AVX1-NEXT:    vmovq %rcx, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX512-LABEL: constrained_vector_fptoui_v3i64_v3f64:
+; AVX512:       # %bb.0: # %entry
+; AVX512-NEXT:    vcvttsd2usi {{.*}}(%rip), %rax
+; AVX512-NEXT:    vmovq %rax, %xmm0
+; AVX512-NEXT:    vcvttsd2usi {{.*}}(%rip), %rax
+; AVX512-NEXT:    vmovq %rax, %xmm1
+; AVX512-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512-NEXT:    vcvttsd2usi {{.*}}(%rip), %rax
+; AVX512-NEXT:    vmovq %rax, %xmm1
+; AVX512-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512-NEXT:    retq
+entry:
+  %result = call <3 x i64> @llvm.experimental.constrained.fptoui.v3i64.v3f64(
+                                <3 x double><double 42.1, double 42.2,
+                                             double 42.3>,
+                                metadata !"fpexcept.strict") #0
+  ret <3 x i64> %result
+}
+
+define <4 x i64> @constrained_vector_fptoui_v4i64_v4f64() #0 {
+; CHECK-LABEL: constrained_vector_fptoui_v4i64_v4f64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    movsd {{.*#+}} xmm2 = mem[0],zero
+; CHECK-NEXT:    comisd %xmm0, %xmm2
+; CHECK-NEXT:    xorpd %xmm1, %xmm1
+; CHECK-NEXT:    xorpd %xmm3, %xmm3
+; CHECK-NEXT:    ja .LBB126_2
+; CHECK-NEXT:  # %bb.1: # %entry
+; CHECK-NEXT:    movapd %xmm2, %xmm3
+; CHECK-NEXT:  .LBB126_2: # %entry
+; CHECK-NEXT:    subsd %xmm3, %xmm0
+; CHECK-NEXT:    cvttsd2si %xmm0, %rcx
+; CHECK-NEXT:    setbe %al
+; CHECK-NEXT:    movzbl %al, %eax
+; CHECK-NEXT:    shlq $63, %rax
+; CHECK-NEXT:    xorq %rcx, %rax
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    comisd %xmm0, %xmm2
+; CHECK-NEXT:    xorpd %xmm4, %xmm4
+; CHECK-NEXT:    ja .LBB126_4
+; CHECK-NEXT:  # %bb.3: # %entry
+; CHECK-NEXT:    movapd %xmm2, %xmm4
+; CHECK-NEXT:  .LBB126_4: # %entry
+; CHECK-NEXT:    movq %rax, %xmm3
+; CHECK-NEXT:    subsd %xmm4, %xmm0
+; CHECK-NEXT:    cvttsd2si %xmm0, %rax
+; CHECK-NEXT:    setbe %cl
+; CHECK-NEXT:    movzbl %cl, %ecx
+; CHECK-NEXT:    shlq $63, %rcx
+; CHECK-NEXT:    xorq %rax, %rcx
+; CHECK-NEXT:    movq %rcx, %xmm0
+; CHECK-NEXT:    movsd {{.*#+}} xmm4 = mem[0],zero
+; CHECK-NEXT:    comisd %xmm4, %xmm2
+; CHECK-NEXT:    xorpd %xmm5, %xmm5
+; CHECK-NEXT:    ja .LBB126_6
+; CHECK-NEXT:  # %bb.5: # %entry
+; CHECK-NEXT:    movapd %xmm2, %xmm5
+; CHECK-NEXT:  .LBB126_6: # %entry
+; CHECK-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
+; CHECK-NEXT:    subsd %xmm5, %xmm4
+; CHECK-NEXT:    cvttsd2si %xmm4, %rax
+; CHECK-NEXT:    setbe %cl
+; CHECK-NEXT:    movzbl %cl, %ecx
+; CHECK-NEXT:    shlq $63, %rcx
+; CHECK-NEXT:    xorq %rax, %rcx
+; CHECK-NEXT:    movq %rcx, %xmm3
+; CHECK-NEXT:    movsd {{.*#+}} xmm4 = mem[0],zero
+; CHECK-NEXT:    comisd %xmm4, %xmm2
+; CHECK-NEXT:    ja .LBB126_8
+; CHECK-NEXT:  # %bb.7: # %entry
+; CHECK-NEXT:    movapd %xmm2, %xmm1
+; CHECK-NEXT:  .LBB126_8: # %entry
+; CHECK-NEXT:    subsd %xmm1, %xmm4
+; CHECK-NEXT:    cvttsd2si %xmm4, %rax
+; CHECK-NEXT:    setbe %cl
+; CHECK-NEXT:    movzbl %cl, %ecx
+; CHECK-NEXT:    shlq $63, %rcx
+; CHECK-NEXT:    xorq %rax, %rcx
+; CHECK-NEXT:    movq %rcx, %xmm1
+; CHECK-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; CHECK-NEXT:    retq
+;
+; AVX1-LABEL: constrained_vector_fptoui_v4i64_v4f64:
+; AVX1:       # %bb.0: # %entry
+; AVX1-NEXT:    vmovsd {{.*#+}} xmm2 = mem[0],zero
+; AVX1-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX1-NEXT:    vcomisd %xmm2, %xmm0
+; AVX1-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vxorpd %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    ja .LBB126_2
+; AVX1-NEXT:  # %bb.1: # %entry
+; AVX1-NEXT:    vmovapd %xmm0, %xmm3
+; AVX1-NEXT:  .LBB126_2: # %entry
+; AVX1-NEXT:    vsubsd %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vcvttsd2si %xmm2, %rcx
+; AVX1-NEXT:    setbe %al
+; AVX1-NEXT:    movzbl %al, %eax
+; AVX1-NEXT:    shlq $63, %rax
+; AVX1-NEXT:    xorq %rcx, %rax
+; AVX1-NEXT:    vmovsd {{.*#+}} xmm3 = mem[0],zero
+; AVX1-NEXT:    vcomisd %xmm3, %xmm0
+; AVX1-NEXT:    vxorpd %xmm4, %xmm4, %xmm4
+; AVX1-NEXT:    ja .LBB126_4
+; AVX1-NEXT:  # %bb.3: # %entry
+; AVX1-NEXT:    vmovapd %xmm0, %xmm4
+; AVX1-NEXT:  .LBB126_4: # %entry
+; AVX1-NEXT:    vmovq %rax, %xmm2
+; AVX1-NEXT:    vsubsd %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vcvttsd2si %xmm3, %rax
+; AVX1-NEXT:    setbe %cl
+; AVX1-NEXT:    movzbl %cl, %ecx
+; AVX1-NEXT:    shlq $63, %rcx
+; AVX1-NEXT:    xorq %rax, %rcx
+; AVX1-NEXT:    vmovq %rcx, %xmm3
+; AVX1-NEXT:    vmovsd {{.*#+}} xmm4 = mem[0],zero
+; AVX1-NEXT:    vcomisd %xmm4, %xmm0
+; AVX1-NEXT:    vxorpd %xmm5, %xmm5, %xmm5
+; AVX1-NEXT:    ja .LBB126_6
+; AVX1-NEXT:  # %bb.5: # %entry
+; AVX1-NEXT:    vmovapd %xmm0, %xmm5
+; AVX1-NEXT:  .LBB126_6: # %entry
+; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
+; AVX1-NEXT:    vsubsd %xmm5, %xmm4, %xmm3
+; AVX1-NEXT:    vcvttsd2si %xmm3, %rax
+; AVX1-NEXT:    setbe %cl
+; AVX1-NEXT:    movzbl %cl, %ecx
+; AVX1-NEXT:    shlq $63, %rcx
+; AVX1-NEXT:    xorq %rax, %rcx
+; AVX1-NEXT:    vmovq %rcx, %xmm3
+; AVX1-NEXT:    vmovsd {{.*#+}} xmm4 = mem[0],zero
+; AVX1-NEXT:    vcomisd %xmm4, %xmm0
+; AVX1-NEXT:    ja .LBB126_8
+; AVX1-NEXT:  # %bb.7: # %entry
+; AVX1-NEXT:    vmovapd %xmm0, %xmm1
+; AVX1-NEXT:  .LBB126_8: # %entry
+; AVX1-NEXT:    vsubsd %xmm1, %xmm4, %xmm0
+; AVX1-NEXT:    vcvttsd2si %xmm0, %rax
+; AVX1-NEXT:    setbe %cl
+; AVX1-NEXT:    movzbl %cl, %ecx
+; AVX1-NEXT:    shlq $63, %rcx
+; AVX1-NEXT:    xorq %rax, %rcx
+; AVX1-NEXT:    vmovq %rcx, %xmm0
+; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX512F-LABEL: constrained_vector_fptoui_v4i64_v4f64:
+; AVX512F:       # %bb.0: # %entry
+; AVX512F-NEXT:    vcvttsd2usi {{.*}}(%rip), %rax
+; AVX512F-NEXT:    vmovq %rax, %xmm0
+; AVX512F-NEXT:    vcvttsd2usi {{.*}}(%rip), %rax
+; AVX512F-NEXT:    vmovq %rax, %xmm1
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512F-NEXT:    vcvttsd2usi {{.*}}(%rip), %rax
+; AVX512F-NEXT:    vmovq %rax, %xmm1
+; AVX512F-NEXT:    vcvttsd2usi {{.*}}(%rip), %rax
+; AVX512F-NEXT:    vmovq %rax, %xmm2
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX512F-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX512F-NEXT:    retq
+;
+; AVX512DQ-LABEL: constrained_vector_fptoui_v4i64_v4f64:
+; AVX512DQ:       # %bb.0: # %entry
+; AVX512DQ-NEXT:    vmovaps {{.*#+}} ymm0 = [4.2100000000000001E+1,4.2200000000000003E+1,4.2299999999999997E+1,4.2399999999999999E+1]
+; AVX512DQ-NEXT:    vcvttpd2uqq %zmm0, %zmm0
+; AVX512DQ-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512DQ-NEXT:    retq
+entry:
+  %result = call <4 x i64> @llvm.experimental.constrained.fptoui.v4i64.v4f64(
+                                <4 x double><double 42.1, double 42.2,
+                                             double 42.3, double 42.4>,
+                                metadata !"fpexcept.strict") #0
+  ret <4 x i64> %result
+}
+
+
+define <1 x float> @constrained_vector_fptrunc_v1f64() #0 {
+; CHECK-LABEL: constrained_vector_fptrunc_v1f64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    cvtsd2ss %xmm0, %xmm0
+; CHECK-NEXT:    retq
+;
+; AVX-LABEL: constrained_vector_fptrunc_v1f64:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-NEXT:    vcvtsd2ss %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    retq
+entry:
+  %result = call <1 x float> @llvm.experimental.constrained.fptrunc.v1f32.v1f64(
+                                <1 x double><double 42.1>,
+                                metadata !"round.dynamic",
+                                metadata !"fpexcept.strict") #0
+  ret <1 x float> %result
+}
+
+define <2 x float> @constrained_vector_fptrunc_v2f64() #0 {
+; CHECK-LABEL: constrained_vector_fptrunc_v2f64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cvtpd2ps {{.*}}(%rip), %xmm0
+; CHECK-NEXT:    retq
+;
+; AVX-LABEL: constrained_vector_fptrunc_v2f64:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    vcvtpd2psx {{.*}}(%rip), %xmm0
+; AVX-NEXT:    retq
+entry:
+  %result = call <2 x float> @llvm.experimental.constrained.fptrunc.v2f32.v2f64(
+                                <2 x double><double 42.1, double 42.2>,
+                                metadata !"round.dynamic",
+                                metadata !"fpexcept.strict") #0
+  ret <2 x float> %result
+}
+
+define <3 x float> @constrained_vector_fptrunc_v3f64() #0 {
+; CHECK-LABEL: constrained_vector_fptrunc_v3f64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    cvtsd2ss %xmm0, %xmm1
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    cvtsd2ss %xmm0, %xmm0
+; CHECK-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; CHECK-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
+; CHECK-NEXT:    cvtsd2ss %xmm1, %xmm1
+; CHECK-NEXT:    movlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; CHECK-NEXT:    retq
+;
+; AVX-LABEL: constrained_vector_fptrunc_v3f64:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-NEXT:    vcvtsd2ss %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
+; AVX-NEXT:    vcvtsd2ss %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
+; AVX-NEXT:    vcvtsd2ss %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1],xmm1[0],xmm0[3]
+; AVX-NEXT:    retq
+entry:
+  %result = call <3 x float> @llvm.experimental.constrained.fptrunc.v3f32.v3f64(
+                                <3 x double><double 42.1, double 42.2,
+                                             double 42.3>,
+                                metadata !"round.dynamic",
+                                metadata !"fpexcept.strict") #0
+  ret <3 x float> %result
+}
+
+define <4 x float> @constrained_vector_fptrunc_v4f64() #0 {
+; CHECK-LABEL: constrained_vector_fptrunc_v4f64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cvtpd2ps {{.*}}(%rip), %xmm1
+; CHECK-NEXT:    cvtpd2ps {{.*}}(%rip), %xmm0
+; CHECK-NEXT:    unpcklpd {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; CHECK-NEXT:    retq
+;
+; AVX-LABEL: constrained_vector_fptrunc_v4f64:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    vcvtpd2psy {{.*}}(%rip), %xmm0
+; AVX-NEXT:    retq
+entry:
+  %result = call <4 x float> @llvm.experimental.constrained.fptrunc.v4f32.v4f64(
+                                <4 x double><double 42.1, double 42.2,
+                                             double 42.3, double 42.4>,
+                                metadata !"round.dynamic",
+                                metadata !"fpexcept.strict") #0
+  ret <4 x float> %result
+}
+
+define <1 x double> @constrained_vector_fpext_v1f32() #0 {
+; CHECK-LABEL: constrained_vector_fpext_v1f32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; CHECK-NEXT:    cvtss2sd %xmm0, %xmm0
+; CHECK-NEXT:    retq
+;
+; AVX-LABEL: constrained_vector_fpext_v1f32:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    retq
+entry:
+  %result = call <1 x double> @llvm.experimental.constrained.fpext.v1f64.v1f32(
+                                <1 x float><float 42.0>,
+                                metadata !"fpexcept.strict") #0
+  ret <1 x double> %result
+}
+
+define <2 x double> @constrained_vector_fpext_v2f32() #0 {
+; CHECK-LABEL: constrained_vector_fpext_v2f32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cvtps2pd {{.*}}(%rip), %xmm0
+; CHECK-NEXT:    retq
+;
+; AVX-LABEL: constrained_vector_fpext_v2f32:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    vcvtps2pd {{.*}}(%rip), %xmm0
+; AVX-NEXT:    retq
+entry:
+  %result = call <2 x double> @llvm.experimental.constrained.fpext.v2f64.v2f32(
+                                <2 x float><float 42.0, float 43.0>,
+                                metadata !"fpexcept.strict") #0
+  ret <2 x double> %result
+}
+
+define <3 x double> @constrained_vector_fpext_v3f32() #0 {
+; CHECK-LABEL: constrained_vector_fpext_v3f32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; CHECK-NEXT:    cvtss2sd %xmm0, %xmm1
+; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; CHECK-NEXT:    cvtss2sd %xmm0, %xmm0
+; CHECK-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; CHECK-NEXT:    cvtss2sd %xmm2, %xmm2
+; CHECK-NEXT:    movsd %xmm2, -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    fldl -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    retq
+;
+; AVX-LABEL: constrained_vector_fpext_v3f32:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX-NEXT:    retq
+entry:
+  %result = call <3 x double> @llvm.experimental.constrained.fpext.v3f64.v3f32(
+                                <3 x float><float 42.0, float 43.0,
+                                            float 44.0>,
+                                metadata !"fpexcept.strict") #0
+  ret <3 x double> %result
+}
+
+define <4 x double> @constrained_vector_fpext_v4f32() #0 {
+; CHECK-LABEL: constrained_vector_fpext_v4f32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cvtps2pd {{.*}}(%rip), %xmm1
+; CHECK-NEXT:    cvtps2pd {{.*}}(%rip), %xmm0
+; CHECK-NEXT:    retq
+;
+; AVX-LABEL: constrained_vector_fpext_v4f32:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    vcvtps2pd {{.*}}(%rip), %ymm0
+; AVX-NEXT:    retq
+entry:
+  %result = call <4 x double> @llvm.experimental.constrained.fpext.v4f64.v4f32(
+                                <4 x float><float 42.0, float 43.0,
+                                            float 44.0, float 45.0>,
+                                metadata !"fpexcept.strict") #0
+  ret <4 x double> %result
+}
+
+define <1 x float> @constrained_vector_ceil_v1f32() #0 {
+; CHECK-LABEL: constrained_vector_ceil_v1f32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; CHECK-NEXT:    callq ceilf
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
+;
+; AVX-LABEL: constrained_vector_ceil_v1f32:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-NEXT:    vroundss $10, %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    retq
+entry:
+  %ceil = call <1 x float> @llvm.experimental.constrained.ceil.v1f32(
+                               <1 x float> <float 1.5>,
+                               metadata !"fpexcept.strict") #0
+  ret <1 x float> %ceil
+}
+
+define <2 x double> @constrained_vector_ceil_v2f64() #0 {
+; CHECK-LABEL: constrained_vector_ceil_v2f64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    subq $24, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    callq ceil
+; CHECK-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    callq ceil
+; CHECK-NEXT:    unpcklpd (%rsp), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = xmm0[0],mem[0]
+; CHECK-NEXT:    addq $24, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
+;
+; AVX-LABEL: constrained_vector_ceil_v2f64:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    vroundpd $10, {{.*}}(%rip), %xmm0
+; AVX-NEXT:    retq
+entry:
+  %ceil = call <2 x double> @llvm.experimental.constrained.ceil.v2f64(
+                                <2 x double> <double 1.1, double 1.9>,
+                                metadata !"fpexcept.strict") #0
+  ret <2 x double> %ceil
+}
+
+define <3 x float> @constrained_vector_ceil_v3f32() #0 {
+; CHECK-LABEL: constrained_vector_ceil_v3f32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    subq $40, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; CHECK-NEXT:    callq ceilf
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; CHECK-NEXT:    callq ceilf
+; CHECK-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
+; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; CHECK-NEXT:    callq ceilf
+; CHECK-NEXT:    movaps (%rsp), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; CHECK-NEXT:    unpcklpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm1 = xmm1[0],mem[0]
+; CHECK-NEXT:    movaps %xmm1, %xmm0
+; CHECK-NEXT:    addq $40, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
+;
+; AVX-LABEL: constrained_vector_ceil_v3f32:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-NEXT:    vroundss $10, %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX-NEXT:    vroundss $10, %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; AVX-NEXT:    vroundss $10, %xmm2, %xmm2, %xmm2
+; AVX-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[2,3]
+; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3]
+; AVX-NEXT:    retq
+entry:
+  %ceil = call <3 x float> @llvm.experimental.constrained.ceil.v3f32(
+                              <3 x float> <float 1.5, float 2.5, float 3.5>,
+                              metadata !"fpexcept.strict") #0
+  ret <3 x float> %ceil
+}
+
+define <3 x double> @constrained_vector_ceil_v3f64() #0 {
+; CHECK-LABEL: constrained_vector_ceil_v3f64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    subq $24, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    callq ceil
+; CHECK-NEXT:    movsd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    callq ceil
+; CHECK-NEXT:    movsd %xmm0, (%rsp) # 8-byte Spill
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    callq ceil
+; CHECK-NEXT:    movsd %xmm0, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    fldl {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movsd (%rsp), %xmm0 # 8-byte Reload
+; CHECK-NEXT:    # xmm0 = mem[0],zero
+; CHECK-NEXT:    movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 8-byte Reload
+; CHECK-NEXT:    # xmm1 = mem[0],zero
+; CHECK-NEXT:    addq $24, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK-NEXT:    retq
 ;
 ; AVX-LABEL: constrained_vector_ceil_v3f64:
 ; AVX:       # %bb.0: # %entry
 ; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; AVX-NEXT:    vroundsd $10, %xmm0, %xmm0, %xmm0
-; AVX-NEXT:    vroundpd $10, {{.*}}(%rip), %xmm1
+; AVX-NEXT:    vroundsd $10, %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vroundpd $10, {{.*}}(%rip), %xmm1
+; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX-NEXT:    retq
+entry:
+  %ceil = call <3 x double> @llvm.experimental.constrained.ceil.v3f64(
+                          <3 x double> <double 1.1, double 1.9, double 1.5>,
+                          metadata !"fpexcept.strict") #0
+  ret <3 x double> %ceil
+}
+
+define <1 x float> @constrained_vector_floor_v1f32() #0 {
+; CHECK-LABEL: constrained_vector_floor_v1f32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; CHECK-NEXT:    callq floorf
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
+;
+; AVX-LABEL: constrained_vector_floor_v1f32:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-NEXT:    vroundss $9, %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    retq
+entry:
+  %floor = call <1 x float> @llvm.experimental.constrained.floor.v1f32(
+                               <1 x float> <float 1.5>,
+                               metadata !"fpexcept.strict") #0
+  ret <1 x float> %floor
+}
+
+
+define <2 x double> @constrained_vector_floor_v2f64() #0 {
+; CHECK-LABEL: constrained_vector_floor_v2f64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    subq $24, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    callq floor
+; CHECK-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    callq floor
+; CHECK-NEXT:    unpcklpd (%rsp), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = xmm0[0],mem[0]
+; CHECK-NEXT:    addq $24, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
+;
+; AVX-LABEL: constrained_vector_floor_v2f64:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    vroundpd $9, {{.*}}(%rip), %xmm0
+; AVX-NEXT:    retq
+entry:
+  %floor = call <2 x double> @llvm.experimental.constrained.floor.v2f64(
+                                <2 x double> <double 1.1, double 1.9>,
+                                metadata !"fpexcept.strict") #0
+  ret <2 x double> %floor
+}
+
+define <3 x float> @constrained_vector_floor_v3f32() #0 {
+; CHECK-LABEL: constrained_vector_floor_v3f32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    subq $40, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; CHECK-NEXT:    callq floorf
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; CHECK-NEXT:    callq floorf
+; CHECK-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
+; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; CHECK-NEXT:    callq floorf
+; CHECK-NEXT:    movaps (%rsp), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; CHECK-NEXT:    unpcklpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm1 = xmm1[0],mem[0]
+; CHECK-NEXT:    movaps %xmm1, %xmm0
+; CHECK-NEXT:    addq $40, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
+;
+; AVX-LABEL: constrained_vector_floor_v3f32:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-NEXT:    vroundss $9, %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX-NEXT:    vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; AVX-NEXT:    vroundss $9, %xmm2, %xmm2, %xmm2
+; AVX-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[2,3]
+; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3]
+; AVX-NEXT:    retq
+entry:
+  %floor = call <3 x float> @llvm.experimental.constrained.floor.v3f32(
+                              <3 x float> <float 1.5, float 2.5, float 3.5>,
+                              metadata !"fpexcept.strict") #0
+  ret <3 x float> %floor
+}
+
+define <3 x double> @constrained_vector_floor_v3f64() #0 {
+; CHECK-LABEL: constrained_vector_floor_v3f64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    subq $24, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    callq floor
+; CHECK-NEXT:    movsd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    callq floor
+; CHECK-NEXT:    movsd %xmm0, (%rsp) # 8-byte Spill
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    callq floor
+; CHECK-NEXT:    movsd %xmm0, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    fldl {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movsd (%rsp), %xmm0 # 8-byte Reload
+; CHECK-NEXT:    # xmm0 = mem[0],zero
+; CHECK-NEXT:    movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 8-byte Reload
+; CHECK-NEXT:    # xmm1 = mem[0],zero
+; CHECK-NEXT:    addq $24, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
+;
+; AVX-LABEL: constrained_vector_floor_v3f64:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-NEXT:    vroundsd $9, %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vroundpd $9, {{.*}}(%rip), %xmm1
+; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX-NEXT:    retq
+entry:
+  %floor = call <3 x double> @llvm.experimental.constrained.floor.v3f64(
+                          <3 x double> <double 1.1, double 1.9, double 1.5>,
+                          metadata !"fpexcept.strict") #0
+  ret <3 x double> %floor
+}
+
+define <1 x float> @constrained_vector_round_v1f32() #0 {
+; CHECK-LABEL: constrained_vector_round_v1f32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; CHECK-NEXT:    callq roundf
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
+;
+; AVX-LABEL: constrained_vector_round_v1f32:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    pushq %rax
+; AVX-NEXT:    .cfi_def_cfa_offset 16
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-NEXT:    callq roundf
+; AVX-NEXT:    popq %rax
+; AVX-NEXT:    .cfi_def_cfa_offset 8
+; AVX-NEXT:    retq
+entry:
+  %round = call <1 x float> @llvm.experimental.constrained.round.v1f32(
+                               <1 x float> <float 1.5>,
+                               metadata !"fpexcept.strict") #0
+  ret <1 x float> %round
+}
+
+define <2 x double> @constrained_vector_round_v2f64() #0 {
+; CHECK-LABEL: constrained_vector_round_v2f64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    subq $24, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    callq round
+; CHECK-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    callq round
+; CHECK-NEXT:    unpcklpd (%rsp), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = xmm0[0],mem[0]
+; CHECK-NEXT:    addq $24, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
+;
+; AVX-LABEL: constrained_vector_round_v2f64:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    subq $24, %rsp
+; AVX-NEXT:    .cfi_def_cfa_offset 32
+; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-NEXT:    callq round
+; AVX-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
+; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-NEXT:    callq round
+; AVX-NEXT:    vunpcklpd (%rsp), %xmm0, %xmm0 # 16-byte Folded Reload
+; AVX-NEXT:    # xmm0 = xmm0[0],mem[0]
+; AVX-NEXT:    addq $24, %rsp
+; AVX-NEXT:    .cfi_def_cfa_offset 8
+; AVX-NEXT:    retq
+entry:
+  %round = call <2 x double> @llvm.experimental.constrained.round.v2f64(
+                                <2 x double> <double 1.1, double 1.9>,
+                                metadata !"fpexcept.strict") #0
+  ret <2 x double> %round
+}
+
+define <3 x float> @constrained_vector_round_v3f32() #0 {
+; CHECK-LABEL: constrained_vector_round_v3f32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    subq $40, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; CHECK-NEXT:    callq roundf
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; CHECK-NEXT:    callq roundf
+; CHECK-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
+; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; CHECK-NEXT:    callq roundf
+; CHECK-NEXT:    movaps (%rsp), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; CHECK-NEXT:    unpcklpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm1 = xmm1[0],mem[0]
+; CHECK-NEXT:    movaps %xmm1, %xmm0
+; CHECK-NEXT:    addq $40, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
+;
+; AVX-LABEL: constrained_vector_round_v3f32:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    subq $40, %rsp
+; AVX-NEXT:    .cfi_def_cfa_offset 48
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-NEXT:    callq roundf
+; AVX-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-NEXT:    callq roundf
+; AVX-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-NEXT:    callq roundf
+; AVX-NEXT:    vmovaps (%rsp), %xmm1 # 16-byte Reload
+; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3]
+; AVX-NEXT:    vinsertps $32, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; AVX-NEXT:    # xmm0 = xmm0[0,1],mem[0],xmm0[3]
+; AVX-NEXT:    addq $40, %rsp
+; AVX-NEXT:    .cfi_def_cfa_offset 8
+; AVX-NEXT:    retq
+entry:
+  %round = call <3 x float> @llvm.experimental.constrained.round.v3f32(
+                              <3 x float> <float 1.5, float 2.5, float 3.5>,
+                              metadata !"fpexcept.strict") #0
+  ret <3 x float> %round
+}
+
+
+define <3 x double> @constrained_vector_round_v3f64() #0 {
+; CHECK-LABEL: constrained_vector_round_v3f64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    subq $24, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    callq round
+; CHECK-NEXT:    movsd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    callq round
+; CHECK-NEXT:    movsd %xmm0, (%rsp) # 8-byte Spill
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    callq round
+; CHECK-NEXT:    movsd %xmm0, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    fldl {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movsd (%rsp), %xmm0 # 8-byte Reload
+; CHECK-NEXT:    # xmm0 = mem[0],zero
+; CHECK-NEXT:    movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 8-byte Reload
+; CHECK-NEXT:    # xmm1 = mem[0],zero
+; CHECK-NEXT:    addq $24, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
+;
+; AVX-LABEL: constrained_vector_round_v3f64:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    subq $56, %rsp
+; AVX-NEXT:    .cfi_def_cfa_offset 64
+; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-NEXT:    callq round
+; AVX-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
+; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-NEXT:    callq round
+; AVX-NEXT:    vunpcklpd (%rsp), %xmm0, %xmm0 # 16-byte Folded Reload
+; AVX-NEXT:    # xmm0 = xmm0[0],mem[0]
+; AVX-NEXT:    vmovups %ymm0, (%rsp) # 32-byte Spill
+; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-NEXT:    vzeroupper
+; AVX-NEXT:    callq round
+; AVX-NEXT:    vmovups (%rsp), %ymm1 # 32-byte Reload
+; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX-NEXT:    addq $56, %rsp
+; AVX-NEXT:    .cfi_def_cfa_offset 8
+; AVX-NEXT:    retq
+entry:
+  %round = call <3 x double> @llvm.experimental.constrained.round.v3f64(
+                          <3 x double> <double 1.1, double 1.9, double 1.5>,
+                          metadata !"fpexcept.strict") #0
+  ret <3 x double> %round
+}
+
+define <1 x float> @constrained_vector_trunc_v1f32() #0 {
+; CHECK-LABEL: constrained_vector_trunc_v1f32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; CHECK-NEXT:    callq truncf
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
+;
+; AVX-LABEL: constrained_vector_trunc_v1f32:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-NEXT:    vroundss $11, %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    retq
+entry:
+  %trunc = call <1 x float> @llvm.experimental.constrained.trunc.v1f32(
+                               <1 x float> <float 1.5>,
+                               metadata !"fpexcept.strict") #0
+  ret <1 x float> %trunc
+}
+
+define <2 x double> @constrained_vector_trunc_v2f64() #0 {
+; CHECK-LABEL: constrained_vector_trunc_v2f64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    subq $24, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    callq trunc
+; CHECK-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    callq trunc
+; CHECK-NEXT:    unpcklpd (%rsp), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = xmm0[0],mem[0]
+; CHECK-NEXT:    addq $24, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
+;
+; AVX-LABEL: constrained_vector_trunc_v2f64:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    vroundpd $11, {{.*}}(%rip), %xmm0
+; AVX-NEXT:    retq
+entry:
+  %trunc = call <2 x double> @llvm.experimental.constrained.trunc.v2f64(
+                                <2 x double> <double 1.1, double 1.9>,
+                                metadata !"fpexcept.strict") #0
+  ret <2 x double> %trunc
+}
+
+define <3 x float> @constrained_vector_trunc_v3f32() #0 {
+; CHECK-LABEL: constrained_vector_trunc_v3f32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    subq $40, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; CHECK-NEXT:    callq truncf
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; CHECK-NEXT:    callq truncf
+; CHECK-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
+; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; CHECK-NEXT:    callq truncf
+; CHECK-NEXT:    movaps (%rsp), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; CHECK-NEXT:    unpcklpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm1 = xmm1[0],mem[0]
+; CHECK-NEXT:    movaps %xmm1, %xmm0
+; CHECK-NEXT:    addq $40, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
+;
+; AVX-LABEL: constrained_vector_trunc_v3f32:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-NEXT:    vroundss $11, %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX-NEXT:    vroundss $11, %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; AVX-NEXT:    vroundss $11, %xmm2, %xmm2, %xmm2
+; AVX-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[2,3]
+; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3]
+; AVX-NEXT:    retq
+entry:
+  %trunc = call <3 x float> @llvm.experimental.constrained.trunc.v3f32(
+                              <3 x float> <float 1.5, float 2.5, float 3.5>,
+                              metadata !"fpexcept.strict") #0
+  ret <3 x float> %trunc
+}
+
+define <3 x double> @constrained_vector_trunc_v3f64() #0 {
+; CHECK-LABEL: constrained_vector_trunc_v3f64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    subq $24, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    callq trunc
+; CHECK-NEXT:    movsd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    callq trunc
+; CHECK-NEXT:    movsd %xmm0, (%rsp) # 8-byte Spill
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    callq trunc
+; CHECK-NEXT:    movsd %xmm0, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    fldl {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movsd (%rsp), %xmm0 # 8-byte Reload
+; CHECK-NEXT:    # xmm0 = mem[0],zero
+; CHECK-NEXT:    movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 8-byte Reload
+; CHECK-NEXT:    # xmm1 = mem[0],zero
+; CHECK-NEXT:    addq $24, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
+;
+; AVX-LABEL: constrained_vector_trunc_v3f64:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-NEXT:    vroundsd $11, %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vroundpd $11, {{.*}}(%rip), %xmm1
 ; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX-NEXT:    retq
 entry:
-  %ceil = call <3 x double> @llvm.experimental.constrained.ceil.v3f64(
-                          <3 x double> <double 1.1, double 1.9, double 1.5>,
-                          metadata !"round.dynamic",
-                          metadata !"fpexcept.strict")
-  ret <3 x double> %ceil
+  %trunc = call <3 x double> @llvm.experimental.constrained.trunc.v3f64(
+                          <3 x double> <double 1.1, double 1.9, double 1.5>,
+                          metadata !"fpexcept.strict") #0
+  ret <3 x double> %trunc
+}
+
+define <1 x double> @constrained_vector_sitofp_v1f64_v1i32(<1 x i32> %x) #0 {
+; CHECK-LABEL: constrained_vector_sitofp_v1f64_v1i32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cvtsi2sd %edi, %xmm0
+; CHECK-NEXT:    retq
+;
+; AVX-LABEL: constrained_vector_sitofp_v1f64_v1i32:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    vcvtsi2sd %edi, %xmm0, %xmm0
+; AVX-NEXT:    retq
+entry:
+  %result = call <1 x double>
+           @llvm.experimental.constrained.sitofp.v1f64.v1i32(<1 x i32> %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret <1 x double> %result
+}
+
+define <1 x float> @constrained_vector_sitofp_v1f32_v1i32(<1 x i32> %x) #0 {
+; CHECK-LABEL: constrained_vector_sitofp_v1f32_v1i32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cvtsi2ss %edi, %xmm0
+; CHECK-NEXT:    retq
+;
+; AVX-LABEL: constrained_vector_sitofp_v1f32_v1i32:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    vcvtsi2ss %edi, %xmm0, %xmm0
+; AVX-NEXT:    retq
+entry:
+  %result = call <1 x float>
+           @llvm.experimental.constrained.sitofp.v1f32.v1i32(<1 x i32> %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret <1 x float> %result
+}
+
+define <1 x double> @constrained_vector_sitofp_v1f64_v1i64(<1 x i64> %x) #0 {
+; CHECK-LABEL: constrained_vector_sitofp_v1f64_v1i64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cvtsi2sd %rdi, %xmm0
+; CHECK-NEXT:    retq
+;
+; AVX-LABEL: constrained_vector_sitofp_v1f64_v1i64:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    vcvtsi2sd %rdi, %xmm0, %xmm0
+; AVX-NEXT:    retq
+entry:
+  %result = call <1 x double>
+           @llvm.experimental.constrained.sitofp.v1f64.v1i64(<1 x i64> %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret <1 x double> %result
+}
+
+define <1 x float> @constrained_vector_sitofp_v1f32_v1i64(<1 x i64> %x) #0 {
+; CHECK-LABEL: constrained_vector_sitofp_v1f32_v1i64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cvtsi2ss %rdi, %xmm0
+; CHECK-NEXT:    retq
+;
+; AVX-LABEL: constrained_vector_sitofp_v1f32_v1i64:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    vcvtsi2ss %rdi, %xmm0, %xmm0
+; AVX-NEXT:    retq
+entry:
+  %result = call <1 x float>
+           @llvm.experimental.constrained.sitofp.v1f32.v1i64(<1 x i64> %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret <1 x float> %result
+}
+
+define <2 x double> @constrained_vector_sitofp_v2f64_v2i32(<2 x i32> %x) #0 {
+; CHECK-LABEL: constrained_vector_sitofp_v2f64_v2i32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cvtdq2pd %xmm0, %xmm0
+; CHECK-NEXT:    retq
+;
+; AVX-LABEL: constrained_vector_sitofp_v2f64_v2i32:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    vcvtdq2pd %xmm0, %xmm0
+; AVX-NEXT:    retq
+entry:
+  %result = call <2 x double>
+           @llvm.experimental.constrained.sitofp.v2f64.v2i32(<2 x i32> %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret <2 x double> %result
+}
+
+define <2 x float> @constrained_vector_sitofp_v2f32_v2i32(<2 x i32> %x) #0 {
+; CHECK-LABEL: constrained_vector_sitofp_v2f32_v2i32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movq {{.*#+}} xmm0 = xmm0[0],zero
+; CHECK-NEXT:    cvtdq2ps %xmm0, %xmm0
+; CHECK-NEXT:    retq
+;
+; AVX-LABEL: constrained_vector_sitofp_v2f32_v2i32:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
+; AVX-NEXT:    vcvtdq2ps %xmm0, %xmm0
+; AVX-NEXT:    retq
+entry:
+  %result = call <2 x float>
+           @llvm.experimental.constrained.sitofp.v2f32.v2i32(<2 x i32> %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret <2 x float> %result
+}
+
+define <2 x double> @constrained_vector_sitofp_v2f64_v2i64(<2 x i64> %x) #0 {
+; CHECK-LABEL: constrained_vector_sitofp_v2f64_v2i64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movq %xmm0, %rax
+; CHECK-NEXT:    cvtsi2sd %rax, %xmm1
+; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; CHECK-NEXT:    movq %xmm0, %rax
+; CHECK-NEXT:    xorps %xmm0, %xmm0
+; CHECK-NEXT:    cvtsi2sd %rax, %xmm0
+; CHECK-NEXT:    unpcklpd {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; CHECK-NEXT:    movapd %xmm1, %xmm0
+; CHECK-NEXT:    retq
+;
+; AVX1-LABEL: constrained_vector_sitofp_v2f64_v2i64:
+; AVX1:       # %bb.0: # %entry
+; AVX1-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX1-NEXT:    vcvtsi2sd %rax, %xmm1, %xmm1
+; AVX1-NEXT:    vmovq %xmm0, %rax
+; AVX1-NEXT:    vcvtsi2sd %rax, %xmm2, %xmm0
+; AVX1-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX1-NEXT:    retq
+;
+; AVX512F-LABEL: constrained_vector_sitofp_v2f64_v2i64:
+; AVX512F:       # %bb.0: # %entry
+; AVX512F-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX512F-NEXT:    vcvtsi2sd %rax, %xmm1, %xmm1
+; AVX512F-NEXT:    vmovq %xmm0, %rax
+; AVX512F-NEXT:    vcvtsi2sd %rax, %xmm2, %xmm0
+; AVX512F-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX512F-NEXT:    retq
+;
+; AVX512DQ-LABEL: constrained_vector_sitofp_v2f64_v2i64:
+; AVX512DQ:       # %bb.0: # %entry
+; AVX512DQ-NEXT:    vmovaps %xmm0, %xmm0
+; AVX512DQ-NEXT:    vcvtqq2pd %zmm0, %zmm0
+; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    retq
+entry:
+  %result = call <2 x double>
+           @llvm.experimental.constrained.sitofp.v2f64.v2i64(<2 x i64> %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret <2 x double> %result
 }
 
-define <1 x float> @constrained_vector_floor_v1f32() {
-; CHECK-LABEL: constrained_vector_floor_v1f32:
+define <2 x float> @constrained_vector_sitofp_v2f32_v2i64(<2 x i64> %x) #0 {
+; CHECK-LABEL: constrained_vector_sitofp_v2f32_v2i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    pushq %rax
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; CHECK-NEXT:    callq floorf
-; CHECK-NEXT:    popq %rax
-; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    movq %xmm0, %rax
+; CHECK-NEXT:    cvtsi2ss %rax, %xmm1
+; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; CHECK-NEXT:    movq %xmm0, %rax
+; CHECK-NEXT:    xorps %xmm0, %xmm0
+; CHECK-NEXT:    cvtsi2ss %rax, %xmm0
+; CHECK-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; CHECK-NEXT:    movaps %xmm1, %xmm0
 ; CHECK-NEXT:    retq
 ;
-; AVX-LABEL: constrained_vector_floor_v1f32:
+; AVX-LABEL: constrained_vector_sitofp_v2f32_v2i64:
 ; AVX:       # %bb.0: # %entry
-; AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX-NEXT:    vroundss $9, %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX-NEXT:    vcvtsi2ss %rax, %xmm1, %xmm1
+; AVX-NEXT:    vmovq %xmm0, %rax
+; AVX-NEXT:    vcvtsi2ss %rax, %xmm2, %xmm0
+; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[2,3]
 ; AVX-NEXT:    retq
 entry:
-  %floor = call <1 x float> @llvm.experimental.constrained.floor.v1f32(
-                               <1 x float> <float 1.5>,
-                               metadata !"round.dynamic",
-                               metadata !"fpexcept.strict")
-  ret <1 x float> %floor
+  %result = call <2 x float>
+           @llvm.experimental.constrained.sitofp.v2f32.v2i64(<2 x i64> %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret <2 x float> %result
 }
 
-
-define <2 x double> @constrained_vector_floor_v2f64() {
-; CHECK-LABEL: constrained_vector_floor_v2f64:
+define <3 x double> @constrained_vector_sitofp_v3f64_v3i32(<3 x i32> %x) #0 {
+; CHECK-LABEL: constrained_vector_sitofp_v3f64_v3i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    subq $24, %rsp
-; CHECK-NEXT:    .cfi_def_cfa_offset 32
-; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; CHECK-NEXT:    callq floor
-; CHECK-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
-; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; CHECK-NEXT:    callq floor
-; CHECK-NEXT:    unpcklpd (%rsp), %xmm0 # 16-byte Folded Reload
-; CHECK-NEXT:    # xmm0 = xmm0[0],mem[0]
-; CHECK-NEXT:    addq $24, %rsp
-; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    movd %xmm0, %eax
+; CHECK-NEXT:    cvtsi2sd %eax, %xmm2
+; CHECK-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; CHECK-NEXT:    movd %xmm1, %eax
+; CHECK-NEXT:    xorps %xmm1, %xmm1
+; CHECK-NEXT:    cvtsi2sd %eax, %xmm1
+; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; CHECK-NEXT:    movd %xmm0, %eax
+; CHECK-NEXT:    xorps %xmm0, %xmm0
+; CHECK-NEXT:    cvtsi2sd %eax, %xmm0
+; CHECK-NEXT:    movsd %xmm0, -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    fldl -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movapd %xmm2, %xmm0
 ; CHECK-NEXT:    retq
 ;
-; AVX-LABEL: constrained_vector_floor_v2f64:
+; AVX-LABEL: constrained_vector_sitofp_v3f64_v3i32:
 ; AVX:       # %bb.0: # %entry
-; AVX-NEXT:    vroundpd $9, {{.*}}(%rip), %xmm0
+; AVX-NEXT:    vextractps $1, %xmm0, %eax
+; AVX-NEXT:    vcvtsi2sd %eax, %xmm1, %xmm1
+; AVX-NEXT:    vmovd %xmm0, %eax
+; AVX-NEXT:    vcvtsi2sd %eax, %xmm2, %xmm2
+; AVX-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX-NEXT:    vpextrd $2, %xmm0, %eax
+; AVX-NEXT:    vcvtsi2sd %eax, %xmm3, %xmm0
+; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX-NEXT:    retq
 entry:
-  %floor = call <2 x double> @llvm.experimental.constrained.floor.v2f64(
-                                <2 x double> <double 1.1, double 1.9>,
-                                metadata !"round.dynamic",
-                                metadata !"fpexcept.strict")
-  ret <2 x double> %floor
+  %result = call <3 x double>
+           @llvm.experimental.constrained.sitofp.v3f64.v3i32(<3 x i32> %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret <3 x double> %result
 }
 
-define <3 x float> @constrained_vector_floor_v3f32() {
-; CHECK-LABEL: constrained_vector_floor_v3f32:
+define <3 x float> @constrained_vector_sitofp_v3f32_v3i32(<3 x i32> %x) #0 {
+; CHECK-LABEL: constrained_vector_sitofp_v3f32_v3i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    subq $40, %rsp
-; CHECK-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; CHECK-NEXT:    callq floorf
-; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; CHECK-NEXT:    callq floorf
-; CHECK-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
-; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; CHECK-NEXT:    callq floorf
-; CHECK-NEXT:    movaps (%rsp), %xmm1 # 16-byte Reload
-; CHECK-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; CHECK-NEXT:    unpcklpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
-; CHECK-NEXT:    # xmm1 = xmm1[0],mem[0]
+; CHECK-NEXT:    movd %xmm0, %eax
+; CHECK-NEXT:    cvtsi2ss %eax, %xmm1
+; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
+; CHECK-NEXT:    movd %xmm2, %eax
+; CHECK-NEXT:    xorps %xmm2, %xmm2
+; CHECK-NEXT:    cvtsi2ss %eax, %xmm2
+; CHECK-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; CHECK-NEXT:    movd %xmm0, %eax
+; CHECK-NEXT:    xorps %xmm0, %xmm0
+; CHECK-NEXT:    cvtsi2ss %eax, %xmm0
+; CHECK-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0]
 ; CHECK-NEXT:    movaps %xmm1, %xmm0
-; CHECK-NEXT:    addq $40, %rsp
-; CHECK-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK-NEXT:    retq
 ;
-; AVX-LABEL: constrained_vector_floor_v3f32:
+; AVX-LABEL: constrained_vector_sitofp_v3f32_v3i32:
 ; AVX:       # %bb.0: # %entry
-; AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX-NEXT:    vroundss $9, %xmm0, %xmm0, %xmm0
-; AVX-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX-NEXT:    vroundss $9, %xmm1, %xmm1, %xmm1
-; AVX-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; AVX-NEXT:    vroundss $9, %xmm2, %xmm2, %xmm2
-; AVX-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[2,3]
+; AVX-NEXT:    vextractps $1, %xmm0, %eax
+; AVX-NEXT:    vcvtsi2ss %eax, %xmm1, %xmm1
+; AVX-NEXT:    vmovd %xmm0, %eax
+; AVX-NEXT:    vcvtsi2ss %eax, %xmm2, %xmm2
+; AVX-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[2,3]
+; AVX-NEXT:    vpextrd $2, %xmm0, %eax
+; AVX-NEXT:    vcvtsi2ss %eax, %xmm3, %xmm0
 ; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3]
 ; AVX-NEXT:    retq
 entry:
-  %floor = call <3 x float> @llvm.experimental.constrained.floor.v3f32(
-                              <3 x float> <float 1.5, float 2.5, float 3.5>,
-                              metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
-  ret <3 x float> %floor
+  %result = call <3 x float>
+           @llvm.experimental.constrained.sitofp.v3f32.v3i32(<3 x i32> %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret <3 x float> %result
 }
 
-define <3 x double> @constrained_vector_floor_v3f64() {
-; CHECK-LABEL: constrained_vector_floor_v3f64:
+define <3 x double> @constrained_vector_sitofp_v3f64_v3i64(<3 x i64> %x) #0 {
+; CHECK-LABEL: constrained_vector_sitofp_v3f64_v3i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    subq $24, %rsp
-; CHECK-NEXT:    .cfi_def_cfa_offset 32
-; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; CHECK-NEXT:    callq floor
-; CHECK-NEXT:    movsd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; CHECK-NEXT:    callq floor
-; CHECK-NEXT:    movsd %xmm0, (%rsp) # 8-byte Spill
-; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; CHECK-NEXT:    callq floor
-; CHECK-NEXT:    movsd %xmm0, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    fldl {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Reload
-; CHECK-NEXT:    # xmm0 = mem[0],zero
-; CHECK-NEXT:    movsd (%rsp), %xmm1 # 8-byte Reload
-; CHECK-NEXT:    # xmm1 = mem[0],zero
-; CHECK-NEXT:    addq $24, %rsp
-; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    cvtsi2sd %rsi, %xmm1
+; CHECK-NEXT:    cvtsi2sd %rdi, %xmm0
+; CHECK-NEXT:    cvtsi2sd %rdx, %xmm2
+; CHECK-NEXT:    movsd %xmm2, -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    fldl -{{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    retq
 ;
-; AVX-LABEL: constrained_vector_floor_v3f64:
+; AVX1-LABEL: constrained_vector_sitofp_v3f64_v3i64:
+; AVX1:       # %bb.0: # %entry
+; AVX1-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX1-NEXT:    vcvtsi2sd %rax, %xmm1, %xmm1
+; AVX1-NEXT:    vmovq %xmm0, %rax
+; AVX1-NEXT:    vcvtsi2sd %rax, %xmm2, %xmm2
+; AVX1-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vmovq %xmm0, %rax
+; AVX1-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX512-LABEL: constrained_vector_sitofp_v3f64_v3i64:
+; AVX512:       # %bb.0: # %entry
+; AVX512-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX512-NEXT:    vcvtsi2sd %rax, %xmm1, %xmm1
+; AVX512-NEXT:    vmovq %xmm0, %rax
+; AVX512-NEXT:    vcvtsi2sd %rax, %xmm2, %xmm2
+; AVX512-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm0
+; AVX512-NEXT:    vmovq %xmm0, %rax
+; AVX512-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm0
+; AVX512-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX512-NEXT:    retq
+entry:
+  %result = call <3 x double>
+           @llvm.experimental.constrained.sitofp.v3f64.v3i64(<3 x i64> %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret <3 x double> %result
+}
+
+define <3 x float> @constrained_vector_sitofp_v3f32_v3i64(<3 x i64> %x) #0 {
+; CHECK-LABEL: constrained_vector_sitofp_v3f32_v3i64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cvtsi2ss %rsi, %xmm1
+; CHECK-NEXT:    cvtsi2ss %rdi, %xmm0
+; CHECK-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; CHECK-NEXT:    xorps %xmm1, %xmm1
+; CHECK-NEXT:    cvtsi2ss %rdx, %xmm1
+; CHECK-NEXT:    movlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; CHECK-NEXT:    retq
+;
+; AVX1-LABEL: constrained_vector_sitofp_v3f32_v3i64:
+; AVX1:       # %bb.0: # %entry
+; AVX1-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm1, %xmm1
+; AVX1-NEXT:    vmovq %xmm0, %rax
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm2, %xmm2
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[2,3]
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vmovq %xmm0, %rax
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm0
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3]
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX512-LABEL: constrained_vector_sitofp_v3f32_v3i64:
+; AVX512:       # %bb.0: # %entry
+; AVX512-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX512-NEXT:    vcvtsi2ss %rax, %xmm1, %xmm1
+; AVX512-NEXT:    vmovq %xmm0, %rax
+; AVX512-NEXT:    vcvtsi2ss %rax, %xmm2, %xmm2
+; AVX512-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[2,3]
+; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm0
+; AVX512-NEXT:    vmovq %xmm0, %rax
+; AVX512-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm0
+; AVX512-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3]
+; AVX512-NEXT:    vzeroupper
+; AVX512-NEXT:    retq
+entry:
+  %result = call <3 x float>
+           @llvm.experimental.constrained.sitofp.v3f32.v3i64(<3 x i64> %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret <3 x float> %result
+}
+
+define <4 x double> @constrained_vector_sitofp_v4f64_v4i32(<4 x i32> %x) #0 {
+; CHECK-LABEL: constrained_vector_sitofp_v4f64_v4i32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cvtdq2pd %xmm0, %xmm2
+; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; CHECK-NEXT:    cvtdq2pd %xmm0, %xmm1
+; CHECK-NEXT:    movaps %xmm2, %xmm0
+; CHECK-NEXT:    retq
+;
+; AVX-LABEL: constrained_vector_sitofp_v4f64_v4i32:
 ; AVX:       # %bb.0: # %entry
-; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; AVX-NEXT:    vroundsd $9, %xmm0, %xmm0, %xmm0
-; AVX-NEXT:    vroundpd $9, {{.*}}(%rip), %xmm1
-; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX-NEXT:    vcvtdq2pd %xmm0, %ymm0
 ; AVX-NEXT:    retq
 entry:
-  %floor = call <3 x double> @llvm.experimental.constrained.floor.v3f64(
-                          <3 x double> <double 1.1, double 1.9, double 1.5>,
-                          metadata !"round.dynamic",
-                          metadata !"fpexcept.strict")
-  ret <3 x double> %floor
+  %result = call <4 x double>
+           @llvm.experimental.constrained.sitofp.v4f64.v4i32(<4 x i32> %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret <4 x double> %result
 }
 
-define <1 x float> @constrained_vector_round_v1f32() {
-; CHECK-LABEL: constrained_vector_round_v1f32:
+define <4 x float> @constrained_vector_sitofp_v4f32_v4i32(<4 x i32> %x) #0 {
+; CHECK-LABEL: constrained_vector_sitofp_v4f32_v4i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    pushq %rax
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; CHECK-NEXT:    callq roundf
-; CHECK-NEXT:    popq %rax
-; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    cvtdq2ps %xmm0, %xmm0
+; CHECK-NEXT:    retq
+;
+; AVX-LABEL: constrained_vector_sitofp_v4f32_v4i32:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    vcvtdq2ps %xmm0, %xmm0
+; AVX-NEXT:    retq
+entry:
+  %result = call <4 x float>
+           @llvm.experimental.constrained.sitofp.v4f32.v4i32(<4 x i32> %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret <4 x float> %result
+}
+
+define <4 x double> @constrained_vector_sitofp_v4f64_v4i64(<4 x i64> %x) #0 {
+; CHECK-LABEL: constrained_vector_sitofp_v4f64_v4i64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movq %xmm0, %rax
+; CHECK-NEXT:    cvtsi2sd %rax, %xmm2
+; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; CHECK-NEXT:    movq %xmm0, %rax
+; CHECK-NEXT:    xorps %xmm0, %xmm0
+; CHECK-NEXT:    cvtsi2sd %rax, %xmm0
+; CHECK-NEXT:    unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm0[0]
+; CHECK-NEXT:    movq %xmm1, %rax
+; CHECK-NEXT:    cvtsi2sd %rax, %xmm3
+; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; CHECK-NEXT:    movq %xmm0, %rax
+; CHECK-NEXT:    xorps %xmm0, %xmm0
+; CHECK-NEXT:    cvtsi2sd %rax, %xmm0
+; CHECK-NEXT:    unpcklpd {{.*#+}} xmm3 = xmm3[0],xmm0[0]
+; CHECK-NEXT:    movapd %xmm2, %xmm0
+; CHECK-NEXT:    movapd %xmm3, %xmm1
+; CHECK-NEXT:    retq
+;
+; AVX1-LABEL: constrained_vector_sitofp_v4f64_v4i64:
+; AVX1:       # %bb.0: # %entry
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vpextrq $1, %xmm1, %rax
+; AVX1-NEXT:    vcvtsi2sd %rax, %xmm2, %xmm2
+; AVX1-NEXT:    vmovq %xmm1, %rax
+; AVX1-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm1
+; AVX1-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; AVX1-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX1-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm2
+; AVX1-NEXT:    vmovq %xmm0, %rax
+; AVX1-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm0
+; AVX1-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX512F-LABEL: constrained_vector_sitofp_v4f64_v4i64:
+; AVX512F:       # %bb.0: # %entry
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-NEXT:    vpextrq $1, %xmm1, %rax
+; AVX512F-NEXT:    vcvtsi2sd %rax, %xmm2, %xmm2
+; AVX512F-NEXT:    vmovq %xmm1, %rax
+; AVX512F-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm1
+; AVX512F-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; AVX512F-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX512F-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm2
+; AVX512F-NEXT:    vmovq %xmm0, %rax
+; AVX512F-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm0
+; AVX512F-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; AVX512F-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX512F-NEXT:    retq
+;
+; AVX512DQ-LABEL: constrained_vector_sitofp_v4f64_v4i64:
+; AVX512DQ:       # %bb.0: # %entry
+; AVX512DQ-NEXT:    vmovaps %ymm0, %ymm0
+; AVX512DQ-NEXT:    vcvtqq2pd %zmm0, %zmm0
+; AVX512DQ-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512DQ-NEXT:    retq
+entry:
+  %result = call <4 x double>
+           @llvm.experimental.constrained.sitofp.v4f64.v4i64(<4 x i64> %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret <4 x double> %result
+}
+
+define <4 x float> @constrained_vector_sitofp_v4f32_v4i64(<4 x i64> %x) #0 {
+; CHECK-LABEL: constrained_vector_sitofp_v4f32_v4i64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movq %xmm1, %rax
+; CHECK-NEXT:    cvtsi2ss %rax, %xmm2
+; CHECK-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; CHECK-NEXT:    movq %xmm1, %rax
+; CHECK-NEXT:    xorps %xmm1, %xmm1
+; CHECK-NEXT:    cvtsi2ss %rax, %xmm1
+; CHECK-NEXT:    unpcklps {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; CHECK-NEXT:    movq %xmm0, %rax
+; CHECK-NEXT:    xorps %xmm1, %xmm1
+; CHECK-NEXT:    cvtsi2ss %rax, %xmm1
+; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; CHECK-NEXT:    movq %xmm0, %rax
+; CHECK-NEXT:    xorps %xmm0, %xmm0
+; CHECK-NEXT:    cvtsi2ss %rax, %xmm0
+; CHECK-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; CHECK-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; CHECK-NEXT:    movaps %xmm1, %xmm0
+; CHECK-NEXT:    retq
+;
+; AVX1-LABEL: constrained_vector_sitofp_v4f32_v4i64:
+; AVX1:       # %bb.0: # %entry
+; AVX1-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm1, %xmm1
+; AVX1-NEXT:    vmovq %xmm0, %rax
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm2, %xmm2
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[2,3]
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vmovq %xmm0, %rax
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm2
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0],xmm1[3]
+; AVX1-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm0
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX512F-LABEL: constrained_vector_sitofp_v4f32_v4i64:
+; AVX512F:       # %bb.0: # %entry
+; AVX512F-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX512F-NEXT:    vcvtsi2ss %rax, %xmm1, %xmm1
+; AVX512F-NEXT:    vmovq %xmm0, %rax
+; AVX512F-NEXT:    vcvtsi2ss %rax, %xmm2, %xmm2
+; AVX512F-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[2,3]
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm0
+; AVX512F-NEXT:    vmovq %xmm0, %rax
+; AVX512F-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm2
+; AVX512F-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0],xmm1[3]
+; AVX512F-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX512F-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm0
+; AVX512F-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512DQ-LABEL: constrained_vector_sitofp_v4f32_v4i64:
+; AVX512DQ:       # %bb.0: # %entry
+; AVX512DQ-NEXT:    vmovaps %ymm0, %ymm0
+; AVX512DQ-NEXT:    vcvtqq2ps %zmm0, %ymm0
+; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    retq
+entry:
+  %result = call <4 x float>
+           @llvm.experimental.constrained.sitofp.v4f32.v4i64(<4 x i64> %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret <4 x float> %result
+}
+
+define <1 x double> @constrained_vector_uitofp_v1f64_v1i32(<1 x i32> %x) #0 {
+; CHECK-LABEL: constrained_vector_uitofp_v1f64_v1i32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    cvtsi2sd %rax, %xmm0
+; CHECK-NEXT:    retq
+;
+; AVX1-LABEL: constrained_vector_uitofp_v1f64_v1i32:
+; AVX1:       # %bb.0: # %entry
+; AVX1-NEXT:    movl %edi, %eax
+; AVX1-NEXT:    vcvtsi2sd %rax, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX512-LABEL: constrained_vector_uitofp_v1f64_v1i32:
+; AVX512:       # %bb.0: # %entry
+; AVX512-NEXT:    vcvtusi2sd %edi, %xmm0, %xmm0
+; AVX512-NEXT:    retq
+entry:
+  %result = call <1 x double>
+           @llvm.experimental.constrained.uitofp.v1f64.v1i32(<1 x i32> %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret <1 x double> %result
+}
+
+define <1 x float> @constrained_vector_uitofp_v1f32_v1i32(<1 x i32> %x) #0 {
+; CHECK-LABEL: constrained_vector_uitofp_v1f32_v1i32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    cvtsi2ss %rax, %xmm0
+; CHECK-NEXT:    retq
+;
+; AVX1-LABEL: constrained_vector_uitofp_v1f32_v1i32:
+; AVX1:       # %bb.0: # %entry
+; AVX1-NEXT:    movl %edi, %eax
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX512-LABEL: constrained_vector_uitofp_v1f32_v1i32:
+; AVX512:       # %bb.0: # %entry
+; AVX512-NEXT:    vcvtusi2ss %edi, %xmm0, %xmm0
+; AVX512-NEXT:    retq
+entry:
+  %result = call <1 x float>
+           @llvm.experimental.constrained.uitofp.v1f32.v1i32(<1 x i32> %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret <1 x float> %result
+}
+
+define <1 x double> @constrained_vector_uitofp_v1f64_v1i64(<1 x i64> %x) #0 {
+; CHECK-LABEL: constrained_vector_uitofp_v1f64_v1i64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movq %rdi, %xmm1
+; CHECK-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]
+; CHECK-NEXT:    subpd {{.*}}(%rip), %xmm1
+; CHECK-NEXT:    movapd %xmm1, %xmm0
+; CHECK-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1]
+; CHECK-NEXT:    addpd %xmm1, %xmm0
+; CHECK-NEXT:    retq
+;
+; AVX1-LABEL: constrained_vector_uitofp_v1f64_v1i64:
+; AVX1:       # %bb.0: # %entry
+; AVX1-NEXT:    vmovq %rdi, %xmm0
+; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
+; AVX1-NEXT:    vsubpd {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
+; AVX1-NEXT:    vaddpd %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX512-LABEL: constrained_vector_uitofp_v1f64_v1i64:
+; AVX512:       # %bb.0: # %entry
+; AVX512-NEXT:    vcvtusi2sd %rdi, %xmm0, %xmm0
+; AVX512-NEXT:    retq
+entry:
+  %result = call <1 x double>
+           @llvm.experimental.constrained.uitofp.v1f64.v1i64(<1 x i64> %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret <1 x double> %result
+}
+
+define <1 x float> @constrained_vector_uitofp_v1f32_v1i64(<1 x i64> %x) #0 {
+; CHECK-LABEL: constrained_vector_uitofp_v1f32_v1i64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    shrq %rax
+; CHECK-NEXT:    movl %edi, %ecx
+; CHECK-NEXT:    andl $1, %ecx
+; CHECK-NEXT:    orq %rax, %rcx
+; CHECK-NEXT:    testq %rdi, %rdi
+; CHECK-NEXT:    cmovnsq %rdi, %rcx
+; CHECK-NEXT:    cvtsi2ss %rcx, %xmm0
+; CHECK-NEXT:    jns .LBB170_2
+; CHECK-NEXT:  # %bb.1:
+; CHECK-NEXT:    addss %xmm0, %xmm0
+; CHECK-NEXT:  .LBB170_2: # %entry
+; CHECK-NEXT:    retq
+;
+; AVX1-LABEL: constrained_vector_uitofp_v1f32_v1i64:
+; AVX1:       # %bb.0: # %entry
+; AVX1-NEXT:    movq %rdi, %rax
+; AVX1-NEXT:    shrq %rax
+; AVX1-NEXT:    movl %edi, %ecx
+; AVX1-NEXT:    andl $1, %ecx
+; AVX1-NEXT:    orq %rax, %rcx
+; AVX1-NEXT:    testq %rdi, %rdi
+; AVX1-NEXT:    cmovnsq %rdi, %rcx
+; AVX1-NEXT:    vcvtsi2ss %rcx, %xmm0, %xmm0
+; AVX1-NEXT:    jns .LBB170_2
+; AVX1-NEXT:  # %bb.1:
+; AVX1-NEXT:    vaddss %xmm0, %xmm0, %xmm0
+; AVX1-NEXT:  .LBB170_2: # %entry
+; AVX1-NEXT:    retq
+;
+; AVX512-LABEL: constrained_vector_uitofp_v1f32_v1i64:
+; AVX512:       # %bb.0: # %entry
+; AVX512-NEXT:    vcvtusi2ss %rdi, %xmm0, %xmm0
+; AVX512-NEXT:    retq
+entry:
+  %result = call <1 x float>
+           @llvm.experimental.constrained.uitofp.v1f32.v1i64(<1 x i64> %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret <1 x float> %result
+}
+
+define <2 x double> @constrained_vector_uitofp_v2f64_v2i32(<2 x i32> %x) #0 {
+; CHECK-LABEL: constrained_vector_uitofp_v2f64_v2i32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xorpd %xmm1, %xmm1
+; CHECK-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; CHECK-NEXT:    movapd {{.*#+}} xmm1 = [4.503599627370496E+15,4.503599627370496E+15]
+; CHECK-NEXT:    orpd %xmm1, %xmm0
+; CHECK-NEXT:    subpd %xmm1, %xmm0
+; CHECK-NEXT:    retq
+;
+; AVX1-LABEL: constrained_vector_uitofp_v2f64_v2i32:
+; AVX1:       # %bb.0: # %entry
+; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [4.503599627370496E+15,4.503599627370496E+15]
+; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vsubpd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX512-LABEL: constrained_vector_uitofp_v2f64_v2i32:
+; AVX512:       # %bb.0: # %entry
+; AVX512-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
+; AVX512-NEXT:    vcvtudq2pd %ymm0, %zmm0
+; AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512-NEXT:    vzeroupper
+; AVX512-NEXT:    retq
+entry:
+  %result = call <2 x double>
+           @llvm.experimental.constrained.uitofp.v2f64.v2i32(<2 x i32> %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret <2 x double> %result
+}
+
+define <2 x float> @constrained_vector_uitofp_v2f32_v2i32(<2 x i32> %x) #0 {
+; CHECK-LABEL: constrained_vector_uitofp_v2f32_v2i32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xorpd %xmm1, %xmm1
+; CHECK-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; CHECK-NEXT:    movapd {{.*#+}} xmm1 = [4.503599627370496E+15,4.503599627370496E+15]
+; CHECK-NEXT:    orpd %xmm1, %xmm0
+; CHECK-NEXT:    subpd %xmm1, %xmm0
+; CHECK-NEXT:    cvtpd2ps %xmm0, %xmm0
+; CHECK-NEXT:    retq
+;
+; AVX1-LABEL: constrained_vector_uitofp_v2f32_v2i32:
+; AVX1:       # %bb.0: # %entry
+; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [4.503599627370496E+15,4.503599627370496E+15]
+; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vsubpd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vcvtpd2ps %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX512-LABEL: constrained_vector_uitofp_v2f32_v2i32:
+; AVX512:       # %bb.0: # %entry
+; AVX512-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
+; AVX512-NEXT:    vcvtudq2ps %zmm0, %zmm0
+; AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512-NEXT:    vzeroupper
+; AVX512-NEXT:    retq
+entry:
+  %result = call <2 x float>
+           @llvm.experimental.constrained.uitofp.v2f32.v2i32(<2 x i32> %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret <2 x float> %result
+}
+
+define <2 x double> @constrained_vector_uitofp_v2f64_v2i64(<2 x i64> %x) #0 {
+; CHECK-LABEL: constrained_vector_uitofp_v2f64_v2i64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movdqa {{.*#+}} xmm1 = [4294967295,4294967295]
+; CHECK-NEXT:    pand %xmm0, %xmm1
+; CHECK-NEXT:    por {{.*}}(%rip), %xmm1
+; CHECK-NEXT:    psrlq $32, %xmm0
+; CHECK-NEXT:    por {{.*}}(%rip), %xmm0
+; CHECK-NEXT:    subpd {{.*}}(%rip), %xmm0
+; CHECK-NEXT:    addpd %xmm1, %xmm0
+; CHECK-NEXT:    retq
+;
+; AVX1-LABEL: constrained_vector_uitofp_v2f64_v2i64:
+; AVX1:       # %bb.0: # %entry
+; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
+; AVX1-NEXT:    vpor {{.*}}(%rip), %xmm1, %xmm1
+; AVX1-NEXT:    vpsrlq $32, %xmm0, %xmm0
+; AVX1-NEXT:    vpor {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vsubpd {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vaddpd %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX512F-LABEL: constrained_vector_uitofp_v2f64_v2i64:
+; AVX512F:       # %bb.0: # %entry
+; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vpblendd {{.*#+}} xmm1 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
+; AVX512F-NEXT:    vpor {{.*}}(%rip), %xmm1, %xmm1
+; AVX512F-NEXT:    vpsrlq $32, %xmm0, %xmm0
+; AVX512F-NEXT:    vpor {{.*}}(%rip), %xmm0, %xmm0
+; AVX512F-NEXT:    vsubpd {{.*}}(%rip), %xmm0, %xmm0
+; AVX512F-NEXT:    vaddpd %xmm0, %xmm1, %xmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512DQ-LABEL: constrained_vector_uitofp_v2f64_v2i64:
+; AVX512DQ:       # %bb.0: # %entry
+; AVX512DQ-NEXT:    vmovaps %xmm0, %xmm0
+; AVX512DQ-NEXT:    vcvtuqq2pd %zmm0, %zmm0
+; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    retq
+entry:
+  %result = call <2 x double>
+           @llvm.experimental.constrained.uitofp.v2f64.v2i64(<2 x i64> %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret <2 x double> %result
+}
+
+define <2 x float> @constrained_vector_uitofp_v2f32_v2i64(<2 x i64> %x) #0 {
+; CHECK-LABEL: constrained_vector_uitofp_v2f32_v2i64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movdqa %xmm0, %xmm1
+; CHECK-NEXT:    movq %xmm0, %rax
+; CHECK-NEXT:    movq %rax, %rcx
+; CHECK-NEXT:    shrq %rcx
+; CHECK-NEXT:    movl %eax, %edx
+; CHECK-NEXT:    andl $1, %edx
+; CHECK-NEXT:    orq %rcx, %rdx
+; CHECK-NEXT:    testq %rax, %rax
+; CHECK-NEXT:    cmovnsq %rax, %rdx
+; CHECK-NEXT:    xorps %xmm0, %xmm0
+; CHECK-NEXT:    cvtsi2ss %rdx, %xmm0
+; CHECK-NEXT:    jns .LBB174_2
+; CHECK-NEXT:  # %bb.1:
+; CHECK-NEXT:    addss %xmm0, %xmm0
+; CHECK-NEXT:  .LBB174_2: # %entry
+; CHECK-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; CHECK-NEXT:    movq %xmm1, %rax
+; CHECK-NEXT:    movq %rax, %rcx
+; CHECK-NEXT:    shrq %rcx
+; CHECK-NEXT:    movl %eax, %edx
+; CHECK-NEXT:    andl $1, %edx
+; CHECK-NEXT:    orq %rcx, %rdx
+; CHECK-NEXT:    testq %rax, %rax
+; CHECK-NEXT:    cmovnsq %rax, %rdx
+; CHECK-NEXT:    xorps %xmm1, %xmm1
+; CHECK-NEXT:    cvtsi2ss %rdx, %xmm1
+; CHECK-NEXT:    jns .LBB174_4
+; CHECK-NEXT:  # %bb.3:
+; CHECK-NEXT:    addss %xmm1, %xmm1
+; CHECK-NEXT:  .LBB174_4: # %entry
+; CHECK-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; CHECK-NEXT:    retq
+;
+; AVX1-LABEL: constrained_vector_uitofp_v2f32_v2i64:
+; AVX1:       # %bb.0: # %entry
+; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm1
+; AVX1-NEXT:    vpsrlq $1, %xmm0, %xmm2
+; AVX1-NEXT:    vpor %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vblendvpd %xmm0, %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vpextrq $1, %xmm1, %rax
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm2
+; AVX1-NEXT:    vmovq %xmm1, %rax
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm1
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0],xmm2[0],zero,zero
+; AVX1-NEXT:    vaddps %xmm1, %xmm1, %xmm2
+; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm3, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
+; AVX1-NEXT:    vblendvps %xmm0, %xmm2, %xmm1, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX512-LABEL: constrained_vector_uitofp_v2f32_v2i64:
+; AVX512:       # %bb.0: # %entry
+; AVX512-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX512-NEXT:    vcvtusi2ss %rax, %xmm1, %xmm1
+; AVX512-NEXT:    vmovq %xmm0, %rax
+; AVX512-NEXT:    vcvtusi2ss %rax, %xmm2, %xmm0
+; AVX512-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[2,3]
+; AVX512-NEXT:    retq
+entry:
+  %result = call <2 x float>
+           @llvm.experimental.constrained.uitofp.v2f32.v2i64(<2 x i64> %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret <2 x float> %result
+}
+
+define <3 x double> @constrained_vector_uitofp_v3f64_v3i32(<3 x i32> %x) #0 {
+; CHECK-LABEL: constrained_vector_uitofp_v3f64_v3i32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movd %xmm0, %eax
+; CHECK-NEXT:    cvtsi2sd %rax, %xmm2
+; CHECK-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; CHECK-NEXT:    movd %xmm1, %eax
+; CHECK-NEXT:    xorps %xmm1, %xmm1
+; CHECK-NEXT:    cvtsi2sd %rax, %xmm1
+; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; CHECK-NEXT:    movd %xmm0, %eax
+; CHECK-NEXT:    xorps %xmm0, %xmm0
+; CHECK-NEXT:    cvtsi2sd %rax, %xmm0
+; CHECK-NEXT:    movsd %xmm0, -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    fldl -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movapd %xmm2, %xmm0
+; CHECK-NEXT:    retq
+;
+; AVX1-LABEL: constrained_vector_uitofp_v3f64_v3i32:
+; AVX1:       # %bb.0: # %entry
+; AVX1-NEXT:    vextractps $1, %xmm0, %eax
+; AVX1-NEXT:    vcvtsi2sd %rax, %xmm1, %xmm1
+; AVX1-NEXT:    vmovd %xmm0, %eax
+; AVX1-NEXT:    vcvtsi2sd %rax, %xmm2, %xmm2
+; AVX1-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX1-NEXT:    vpextrd $2, %xmm0, %eax
+; AVX1-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX512-LABEL: constrained_vector_uitofp_v3f64_v3i32:
+; AVX512:       # %bb.0: # %entry
+; AVX512-NEXT:    vextractps $1, %xmm0, %eax
+; AVX512-NEXT:    vcvtusi2sd %eax, %xmm1, %xmm1
+; AVX512-NEXT:    vmovd %xmm0, %eax
+; AVX512-NEXT:    vcvtusi2sd %eax, %xmm2, %xmm2
+; AVX512-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX512-NEXT:    vpextrd $2, %xmm0, %eax
+; AVX512-NEXT:    vcvtusi2sd %eax, %xmm3, %xmm0
+; AVX512-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX512-NEXT:    retq
+entry:
+  %result = call <3 x double>
+           @llvm.experimental.constrained.uitofp.v3f64.v3i32(<3 x i32> %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret <3 x double> %result
+}
+
+define <3 x float> @constrained_vector_uitofp_v3f32_v3i32(<3 x i32> %x) #0 {
+; CHECK-LABEL: constrained_vector_uitofp_v3f32_v3i32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movd %xmm0, %eax
+; CHECK-NEXT:    cvtsi2ss %rax, %xmm1
+; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
+; CHECK-NEXT:    movd %xmm2, %eax
+; CHECK-NEXT:    xorps %xmm2, %xmm2
+; CHECK-NEXT:    cvtsi2ss %rax, %xmm2
+; CHECK-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; CHECK-NEXT:    movd %xmm0, %eax
+; CHECK-NEXT:    xorps %xmm0, %xmm0
+; CHECK-NEXT:    cvtsi2ss %rax, %xmm0
+; CHECK-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; CHECK-NEXT:    movaps %xmm1, %xmm0
 ; CHECK-NEXT:    retq
 ;
-; AVX-LABEL: constrained_vector_round_v1f32:
-; AVX:       # %bb.0: # %entry
-; AVX-NEXT:    pushq %rax
-; AVX-NEXT:    .cfi_def_cfa_offset 16
-; AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX-NEXT:    callq roundf
-; AVX-NEXT:    popq %rax
-; AVX-NEXT:    .cfi_def_cfa_offset 8
-; AVX-NEXT:    retq
+; AVX1-LABEL: constrained_vector_uitofp_v3f32_v3i32:
+; AVX1:       # %bb.0: # %entry
+; AVX1-NEXT:    vextractps $1, %xmm0, %eax
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm1, %xmm1
+; AVX1-NEXT:    vmovd %xmm0, %eax
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm2, %xmm2
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[2,3]
+; AVX1-NEXT:    vpextrd $2, %xmm0, %eax
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm0
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3]
+; AVX1-NEXT:    retq
+;
+; AVX512-LABEL: constrained_vector_uitofp_v3f32_v3i32:
+; AVX512:       # %bb.0: # %entry
+; AVX512-NEXT:    vextractps $1, %xmm0, %eax
+; AVX512-NEXT:    vcvtusi2ss %eax, %xmm1, %xmm1
+; AVX512-NEXT:    vmovd %xmm0, %eax
+; AVX512-NEXT:    vcvtusi2ss %eax, %xmm2, %xmm2
+; AVX512-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[2,3]
+; AVX512-NEXT:    vpextrd $2, %xmm0, %eax
+; AVX512-NEXT:    vcvtusi2ss %eax, %xmm3, %xmm0
+; AVX512-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3]
+; AVX512-NEXT:    retq
 entry:
-  %round = call <1 x float> @llvm.experimental.constrained.round.v1f32(
-                               <1 x float> <float 1.5>,
-                               metadata !"round.dynamic",
-                               metadata !"fpexcept.strict")
-  ret <1 x float> %round
+  %result = call <3 x float>
+           @llvm.experimental.constrained.uitofp.v3f32.v3i32(<3 x i32> %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret <3 x float> %result
 }
 
-define <2 x double> @constrained_vector_round_v2f64() {
-; CHECK-LABEL: constrained_vector_round_v2f64:
+define <3 x double> @constrained_vector_uitofp_v3f64_v3i64(<3 x i64> %x) #0 {
+; CHECK-LABEL: constrained_vector_uitofp_v3f64_v3i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    subq $24, %rsp
-; CHECK-NEXT:    .cfi_def_cfa_offset 32
-; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; CHECK-NEXT:    callq round
-; CHECK-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
-; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; CHECK-NEXT:    callq round
-; CHECK-NEXT:    unpcklpd (%rsp), %xmm0 # 16-byte Folded Reload
-; CHECK-NEXT:    # xmm0 = xmm0[0],mem[0]
-; CHECK-NEXT:    addq $24, %rsp
-; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    movq %rdi, %xmm1
+; CHECK-NEXT:    movdqa {{.*#+}} xmm2 = [1127219200,1160773632,0,0]
+; CHECK-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; CHECK-NEXT:    movapd {{.*#+}} xmm3 = [4.503599627370496E+15,1.9342813113834067E+25]
+; CHECK-NEXT:    subpd %xmm3, %xmm1
+; CHECK-NEXT:    movapd %xmm1, %xmm0
+; CHECK-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1]
+; CHECK-NEXT:    addpd %xmm1, %xmm0
+; CHECK-NEXT:    movq %rsi, %xmm4
+; CHECK-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
+; CHECK-NEXT:    subpd %xmm3, %xmm4
+; CHECK-NEXT:    movapd %xmm4, %xmm1
+; CHECK-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm4[1]
+; CHECK-NEXT:    addpd %xmm4, %xmm1
+; CHECK-NEXT:    movq %rdx, %xmm4
+; CHECK-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
+; CHECK-NEXT:    subpd %xmm3, %xmm4
+; CHECK-NEXT:    movapd %xmm4, %xmm2
+; CHECK-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm4[1]
+; CHECK-NEXT:    addpd %xmm4, %xmm2
+; CHECK-NEXT:    movlpd %xmm2, -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    fldl -{{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    retq
 ;
-; AVX-LABEL: constrained_vector_round_v2f64:
-; AVX:       # %bb.0: # %entry
-; AVX-NEXT:    subq $24, %rsp
-; AVX-NEXT:    .cfi_def_cfa_offset 32
-; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; AVX-NEXT:    callq round
-; AVX-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
-; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; AVX-NEXT:    callq round
-; AVX-NEXT:    vunpcklpd (%rsp), %xmm0, %xmm0 # 16-byte Folded Reload
-; AVX-NEXT:    # xmm0 = xmm0[0],mem[0]
-; AVX-NEXT:    addq $24, %rsp
-; AVX-NEXT:    .cfi_def_cfa_offset 8
-; AVX-NEXT:    retq
+; AVX1-LABEL: constrained_vector_uitofp_v3f64_v3i64:
+; AVX1:       # %bb.0: # %entry
+; AVX1-NEXT:    vmovapd {{.*#+}} xmm1 = [1127219200,1160773632,0,0]
+; AVX1-NEXT:    vunpcklps {{.*#+}} xmm2 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX1-NEXT:    vmovapd {{.*#+}} xmm3 = [4.503599627370496E+15,1.9342813113834067E+25]
+; AVX1-NEXT:    vsubpd %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpermilpd {{.*#+}} xmm4 = xmm2[1,0]
+; AVX1-NEXT:    vaddpd %xmm2, %xmm4, %xmm2
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm4 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vunpcklps {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
+; AVX1-NEXT:    vsubpd %xmm3, %xmm4, %xmm4
+; AVX1-NEXT:    vpermilpd {{.*#+}} xmm5 = xmm4[1,0]
+; AVX1-NEXT:    vaddpd %xmm4, %xmm5, %xmm4
+; AVX1-NEXT:    vunpcklpd {{.*#+}} xmm2 = xmm2[0],xmm4[0]
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX1-NEXT:    vsubpd %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
+; AVX1-NEXT:    vaddpd %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX512-LABEL: constrained_vector_uitofp_v3f64_v3i64:
+; AVX512:       # %bb.0: # %entry
+; AVX512-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX512-NEXT:    vcvtusi2sd %rax, %xmm1, %xmm1
+; AVX512-NEXT:    vmovq %xmm0, %rax
+; AVX512-NEXT:    vcvtusi2sd %rax, %xmm2, %xmm2
+; AVX512-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm0
+; AVX512-NEXT:    vmovq %xmm0, %rax
+; AVX512-NEXT:    vcvtusi2sd %rax, %xmm3, %xmm0
+; AVX512-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX512-NEXT:    retq
 entry:
-  %round = call <2 x double> @llvm.experimental.constrained.round.v2f64(
-                                <2 x double> <double 1.1, double 1.9>,
-                                metadata !"round.dynamic",
-                                metadata !"fpexcept.strict")
-  ret <2 x double> %round
+  %result = call <3 x double>
+           @llvm.experimental.constrained.uitofp.v3f64.v3i64(<3 x i64> %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret <3 x double> %result
 }
 
-define <3 x float> @constrained_vector_round_v3f32() {
-; CHECK-LABEL: constrained_vector_round_v3f32:
+define <3 x float> @constrained_vector_uitofp_v3f32_v3i64(<3 x i64> %x) #0 {
+; CHECK-LABEL: constrained_vector_uitofp_v3f32_v3i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    subq $40, %rsp
-; CHECK-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; CHECK-NEXT:    callq roundf
-; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; CHECK-NEXT:    callq roundf
-; CHECK-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
-; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; CHECK-NEXT:    callq roundf
-; CHECK-NEXT:    movaps (%rsp), %xmm1 # 16-byte Reload
-; CHECK-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; CHECK-NEXT:    unpcklpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
-; CHECK-NEXT:    # xmm1 = xmm1[0],mem[0]
-; CHECK-NEXT:    movaps %xmm1, %xmm0
-; CHECK-NEXT:    addq $40, %rsp
-; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    movq %rsi, %rax
+; CHECK-NEXT:    shrq %rax
+; CHECK-NEXT:    movl %esi, %ecx
+; CHECK-NEXT:    andl $1, %ecx
+; CHECK-NEXT:    orq %rax, %rcx
+; CHECK-NEXT:    testq %rsi, %rsi
+; CHECK-NEXT:    cmovnsq %rsi, %rcx
+; CHECK-NEXT:    cvtsi2ss %rcx, %xmm1
+; CHECK-NEXT:    jns .LBB178_2
+; CHECK-NEXT:  # %bb.1:
+; CHECK-NEXT:    addss %xmm1, %xmm1
+; CHECK-NEXT:  .LBB178_2: # %entry
+; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    shrq %rax
+; CHECK-NEXT:    movl %edi, %ecx
+; CHECK-NEXT:    andl $1, %ecx
+; CHECK-NEXT:    orq %rax, %rcx
+; CHECK-NEXT:    testq %rdi, %rdi
+; CHECK-NEXT:    cmovnsq %rdi, %rcx
+; CHECK-NEXT:    cvtsi2ss %rcx, %xmm0
+; CHECK-NEXT:    jns .LBB178_4
+; CHECK-NEXT:  # %bb.3:
+; CHECK-NEXT:    addss %xmm0, %xmm0
+; CHECK-NEXT:  .LBB178_4: # %entry
+; CHECK-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; CHECK-NEXT:    movq %rdx, %rax
+; CHECK-NEXT:    shrq %rax
+; CHECK-NEXT:    movl %edx, %ecx
+; CHECK-NEXT:    andl $1, %ecx
+; CHECK-NEXT:    orq %rax, %rcx
+; CHECK-NEXT:    testq %rdx, %rdx
+; CHECK-NEXT:    cmovnsq %rdx, %rcx
+; CHECK-NEXT:    xorps %xmm1, %xmm1
+; CHECK-NEXT:    cvtsi2ss %rcx, %xmm1
+; CHECK-NEXT:    jns .LBB178_6
+; CHECK-NEXT:  # %bb.5:
+; CHECK-NEXT:    addss %xmm1, %xmm1
+; CHECK-NEXT:  .LBB178_6: # %entry
+; CHECK-NEXT:    movlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
 ; CHECK-NEXT:    retq
 ;
-; AVX-LABEL: constrained_vector_round_v3f32:
-; AVX:       # %bb.0: # %entry
-; AVX-NEXT:    subq $40, %rsp
-; AVX-NEXT:    .cfi_def_cfa_offset 48
-; AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX-NEXT:    callq roundf
-; AVX-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX-NEXT:    callq roundf
-; AVX-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
-; AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX-NEXT:    callq roundf
-; AVX-NEXT:    vmovaps (%rsp), %xmm1 # 16-byte Reload
-; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3]
-; AVX-NEXT:    vinsertps $32, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
-; AVX-NEXT:    # xmm0 = xmm0[0,1],mem[0],xmm0[3]
-; AVX-NEXT:    addq $40, %rsp
-; AVX-NEXT:    .cfi_def_cfa_offset 8
-; AVX-NEXT:    retq
+; AVX1-LABEL: constrained_vector_uitofp_v3f32_v3i64:
+; AVX1:       # %bb.0: # %entry
+; AVX1-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX1-NEXT:    movq %rax, %rcx
+; AVX1-NEXT:    shrq %rcx
+; AVX1-NEXT:    movl %eax, %edx
+; AVX1-NEXT:    andl $1, %edx
+; AVX1-NEXT:    orq %rcx, %rdx
+; AVX1-NEXT:    testq %rax, %rax
+; AVX1-NEXT:    cmovnsq %rax, %rdx
+; AVX1-NEXT:    vcvtsi2ss %rdx, %xmm1, %xmm1
+; AVX1-NEXT:    jns .LBB178_2
+; AVX1-NEXT:  # %bb.1:
+; AVX1-NEXT:    vaddss %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:  .LBB178_2: # %entry
+; AVX1-NEXT:    vmovq %xmm0, %rax
+; AVX1-NEXT:    movq %rax, %rcx
+; AVX1-NEXT:    shrq %rcx
+; AVX1-NEXT:    movl %eax, %edx
+; AVX1-NEXT:    andl $1, %edx
+; AVX1-NEXT:    orq %rcx, %rdx
+; AVX1-NEXT:    testq %rax, %rax
+; AVX1-NEXT:    cmovnsq %rax, %rdx
+; AVX1-NEXT:    vcvtsi2ss %rdx, %xmm2, %xmm2
+; AVX1-NEXT:    jns .LBB178_4
+; AVX1-NEXT:  # %bb.3:
+; AVX1-NEXT:    vaddss %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:  .LBB178_4: # %entry
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[2,3]
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vmovq %xmm0, %rax
+; AVX1-NEXT:    movq %rax, %rcx
+; AVX1-NEXT:    shrq %rcx
+; AVX1-NEXT:    movl %eax, %edx
+; AVX1-NEXT:    andl $1, %edx
+; AVX1-NEXT:    orq %rcx, %rdx
+; AVX1-NEXT:    testq %rax, %rax
+; AVX1-NEXT:    cmovnsq %rax, %rdx
+; AVX1-NEXT:    vcvtsi2ss %rdx, %xmm3, %xmm0
+; AVX1-NEXT:    jns .LBB178_6
+; AVX1-NEXT:  # %bb.5:
+; AVX1-NEXT:    vaddss %xmm0, %xmm0, %xmm0
+; AVX1-NEXT:  .LBB178_6: # %entry
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3]
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX512-LABEL: constrained_vector_uitofp_v3f32_v3i64:
+; AVX512:       # %bb.0: # %entry
+; AVX512-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX512-NEXT:    vcvtusi2ss %rax, %xmm1, %xmm1
+; AVX512-NEXT:    vmovq %xmm0, %rax
+; AVX512-NEXT:    vcvtusi2ss %rax, %xmm2, %xmm2
+; AVX512-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[2,3]
+; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm0
+; AVX512-NEXT:    vmovq %xmm0, %rax
+; AVX512-NEXT:    vcvtusi2ss %rax, %xmm3, %xmm0
+; AVX512-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3]
+; AVX512-NEXT:    vzeroupper
+; AVX512-NEXT:    retq
 entry:
-  %round = call <3 x float> @llvm.experimental.constrained.round.v3f32(
-                              <3 x float> <float 1.5, float 2.5, float 3.5>,
-                              metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
-  ret <3 x float> %round
+  %result = call <3 x float>
+           @llvm.experimental.constrained.uitofp.v3f32.v3i64(<3 x i64> %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret <3 x float> %result
 }
 
-
-define <3 x double> @constrained_vector_round_v3f64() {
-; CHECK-LABEL: constrained_vector_round_v3f64:
+define <4 x double> @constrained_vector_uitofp_v4f64_v4i32(<4 x i32> %x) #0 {
+; CHECK-LABEL: constrained_vector_uitofp_v4f64_v4i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    subq $24, %rsp
-; CHECK-NEXT:    .cfi_def_cfa_offset 32
-; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; CHECK-NEXT:    callq round
-; CHECK-NEXT:    movsd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; CHECK-NEXT:    callq round
-; CHECK-NEXT:    movsd %xmm0, (%rsp) # 8-byte Spill
-; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; CHECK-NEXT:    callq round
-; CHECK-NEXT:    movsd %xmm0, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    fldl {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Reload
-; CHECK-NEXT:    # xmm0 = mem[0],zero
-; CHECK-NEXT:    movsd (%rsp), %xmm1 # 8-byte Reload
-; CHECK-NEXT:    # xmm1 = mem[0],zero
-; CHECK-NEXT:    addq $24, %rsp
-; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    xorpd %xmm2, %xmm2
+; CHECK-NEXT:    movapd %xmm0, %xmm1
+; CHECK-NEXT:    unpckhps {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; CHECK-NEXT:    movapd {{.*#+}} xmm3 = [4.503599627370496E+15,4.503599627370496E+15]
+; CHECK-NEXT:    orpd %xmm3, %xmm1
+; CHECK-NEXT:    subpd %xmm3, %xmm1
+; CHECK-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; CHECK-NEXT:    orpd %xmm3, %xmm0
+; CHECK-NEXT:    subpd %xmm3, %xmm0
 ; CHECK-NEXT:    retq
 ;
-; AVX-LABEL: constrained_vector_round_v3f64:
-; AVX:       # %bb.0: # %entry
-; AVX-NEXT:    subq $56, %rsp
-; AVX-NEXT:    .cfi_def_cfa_offset 64
-; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; AVX-NEXT:    callq round
-; AVX-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
-; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; AVX-NEXT:    callq round
-; AVX-NEXT:    vunpcklpd (%rsp), %xmm0, %xmm0 # 16-byte Folded Reload
-; AVX-NEXT:    # xmm0 = xmm0[0],mem[0]
-; AVX-NEXT:    vmovups %ymm0, (%rsp) # 32-byte Spill
-; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; AVX-NEXT:    vzeroupper
-; AVX-NEXT:    callq round
-; AVX-NEXT:    vmovups (%rsp), %ymm1 # 32-byte Reload
-; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; AVX-NEXT:    addq $56, %rsp
-; AVX-NEXT:    .cfi_def_cfa_offset 8
-; AVX-NEXT:    retq
+; AVX1-LABEL: constrained_vector_uitofp_v4f64_v4i32:
+; AVX1:       # %bb.0: # %entry
+; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm1 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    vbroadcastsd {{.*#+}} ymm1 = [4.503599627370496E+15,4.503599627370496E+15,4.503599627370496E+15,4.503599627370496E+15]
+; AVX1-NEXT:    vorpd %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vsubpd %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX512-LABEL: constrained_vector_uitofp_v4f64_v4i32:
+; AVX512:       # %bb.0: # %entry
+; AVX512-NEXT:    vmovaps %xmm0, %xmm0
+; AVX512-NEXT:    vcvtudq2pd %ymm0, %zmm0
+; AVX512-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512-NEXT:    retq
 entry:
-  %round = call <3 x double> @llvm.experimental.constrained.round.v3f64(
-                          <3 x double> <double 1.1, double 1.9, double 1.5>,
-                          metadata !"round.dynamic",
-                          metadata !"fpexcept.strict")
-  ret <3 x double> %round
+  %result = call <4 x double>
+           @llvm.experimental.constrained.uitofp.v4f64.v4i32(<4 x i32> %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret <4 x double> %result
 }
 
-define <1 x float> @constrained_vector_trunc_v1f32() {
-; CHECK-LABEL: constrained_vector_trunc_v1f32:
+define <4 x float> @constrained_vector_uitofp_v4f32_v4i32(<4 x i32> %x) #0 {
+; CHECK-LABEL: constrained_vector_uitofp_v4f32_v4i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    pushq %rax
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; CHECK-NEXT:    callq truncf
-; CHECK-NEXT:    popq %rax
-; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    movdqa {{.*#+}} xmm1 = [65535,65535,65535,65535]
+; CHECK-NEXT:    pand %xmm0, %xmm1
+; CHECK-NEXT:    por {{.*}}(%rip), %xmm1
+; CHECK-NEXT:    psrld $16, %xmm0
+; CHECK-NEXT:    por {{.*}}(%rip), %xmm0
+; CHECK-NEXT:    subps {{.*}}(%rip), %xmm0
+; CHECK-NEXT:    addps %xmm1, %xmm0
 ; CHECK-NEXT:    retq
 ;
-; AVX-LABEL: constrained_vector_trunc_v1f32:
-; AVX:       # %bb.0: # %entry
-; AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX-NEXT:    vroundss $11, %xmm0, %xmm0, %xmm0
-; AVX-NEXT:    retq
+; AVX1-LABEL: constrained_vector_uitofp_v4f32_v4i32:
+; AVX1:       # %bb.0: # %entry
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],mem[1],xmm0[2],mem[3],xmm0[4],mem[5],xmm0[6],mem[7]
+; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm0
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],mem[1],xmm0[2],mem[3],xmm0[4],mem[5],xmm0[6],mem[7]
+; AVX1-NEXT:    vsubps {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vaddps %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX512-LABEL: constrained_vector_uitofp_v4f32_v4i32:
+; AVX512:       # %bb.0: # %entry
+; AVX512-NEXT:    vmovaps %xmm0, %xmm0
+; AVX512-NEXT:    vcvtudq2ps %zmm0, %zmm0
+; AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512-NEXT:    vzeroupper
+; AVX512-NEXT:    retq
 entry:
-  %trunc = call <1 x float> @llvm.experimental.constrained.trunc.v1f32(
-                               <1 x float> <float 1.5>,
-                               metadata !"round.dynamic",
-                               metadata !"fpexcept.strict")
-  ret <1 x float> %trunc
+  %result = call <4 x float>
+           @llvm.experimental.constrained.uitofp.v4f32.v4i32(<4 x i32> %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret <4 x float> %result
 }
 
-define <2 x double> @constrained_vector_trunc_v2f64() {
-; CHECK-LABEL: constrained_vector_trunc_v2f64:
+define <4 x double> @constrained_vector_uitofp_v4f64_v4i64(<4 x i64> %x) #0 {
+; CHECK-LABEL: constrained_vector_uitofp_v4f64_v4i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    subq $24, %rsp
-; CHECK-NEXT:    .cfi_def_cfa_offset 32
-; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; CHECK-NEXT:    callq trunc
-; CHECK-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
-; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; CHECK-NEXT:    callq trunc
-; CHECK-NEXT:    unpcklpd (%rsp), %xmm0 # 16-byte Folded Reload
-; CHECK-NEXT:    # xmm0 = xmm0[0],mem[0]
-; CHECK-NEXT:    addq $24, %rsp
-; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    movdqa {{.*#+}} xmm2 = [4294967295,4294967295]
+; CHECK-NEXT:    movdqa %xmm1, %xmm3
+; CHECK-NEXT:    pand %xmm2, %xmm3
+; CHECK-NEXT:    movdqa {{.*#+}} xmm4 = [4841369599423283200,4841369599423283200]
+; CHECK-NEXT:    por %xmm4, %xmm3
+; CHECK-NEXT:    psrlq $32, %xmm1
+; CHECK-NEXT:    movdqa {{.*#+}} xmm5 = [4985484787499139072,4985484787499139072]
+; CHECK-NEXT:    por %xmm5, %xmm1
+; CHECK-NEXT:    movapd {{.*#+}} xmm6 = [1.9342813118337666E+25,1.9342813118337666E+25]
+; CHECK-NEXT:    subpd %xmm6, %xmm1
+; CHECK-NEXT:    addpd %xmm3, %xmm1
+; CHECK-NEXT:    pand %xmm0, %xmm2
+; CHECK-NEXT:    por %xmm4, %xmm2
+; CHECK-NEXT:    psrlq $32, %xmm0
+; CHECK-NEXT:    por %xmm5, %xmm0
+; CHECK-NEXT:    subpd %xmm6, %xmm0
+; CHECK-NEXT:    addpd %xmm2, %xmm0
 ; CHECK-NEXT:    retq
 ;
-; AVX-LABEL: constrained_vector_trunc_v2f64:
-; AVX:       # %bb.0: # %entry
-; AVX-NEXT:    vroundpd $11, {{.*}}(%rip), %xmm0
-; AVX-NEXT:    retq
+; AVX1-LABEL: constrained_vector_uitofp_v4f64_v4i64:
+; AVX1:       # %bb.0: # %entry
+; AVX1-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm0[0],ymm1[1],ymm0[2],ymm1[3],ymm0[4],ymm1[5],ymm0[6],ymm1[7]
+; AVX1-NEXT:    vorps {{.*}}(%rip), %ymm1, %ymm1
+; AVX1-NEXT:    vpsrlq $32, %xmm0, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vpsrlq $32, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
+; AVX1-NEXT:    vorpd {{.*}}(%rip), %ymm0, %ymm0
+; AVX1-NEXT:    vsubpd {{.*}}(%rip), %ymm0, %ymm0
+; AVX1-NEXT:    vaddpd %ymm0, %ymm1, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX512F-LABEL: constrained_vector_uitofp_v4f64_v4i64:
+; AVX512F:       # %bb.0: # %entry
+; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0],ymm1[1],ymm0[2],ymm1[3],ymm0[4],ymm1[5],ymm0[6],ymm1[7]
+; AVX512F-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200]
+; AVX512F-NEXT:    vpor %ymm2, %ymm1, %ymm1
+; AVX512F-NEXT:    vpsrlq $32, %ymm0, %ymm0
+; AVX512F-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [4985484787499139072,4985484787499139072,4985484787499139072,4985484787499139072]
+; AVX512F-NEXT:    vpor %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vbroadcastsd {{.*#+}} ymm2 = [1.9342813118337666E+25,1.9342813118337666E+25,1.9342813118337666E+25,1.9342813118337666E+25]
+; AVX512F-NEXT:    vsubpd %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vaddpd %ymm0, %ymm1, %ymm0
+; AVX512F-NEXT:    retq
+;
+; AVX512DQ-LABEL: constrained_vector_uitofp_v4f64_v4i64:
+; AVX512DQ:       # %bb.0: # %entry
+; AVX512DQ-NEXT:    vmovaps %ymm0, %ymm0
+; AVX512DQ-NEXT:    vcvtuqq2pd %zmm0, %zmm0
+; AVX512DQ-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512DQ-NEXT:    retq
 entry:
-  %trunc = call <2 x double> @llvm.experimental.constrained.trunc.v2f64(
-                                <2 x double> <double 1.1, double 1.9>,
-                                metadata !"round.dynamic",
-                                metadata !"fpexcept.strict")
-  ret <2 x double> %trunc
+  %result = call <4 x double>
+           @llvm.experimental.constrained.uitofp.v4f64.v4i64(<4 x i64> %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret <4 x double> %result
 }
 
-define <3 x float> @constrained_vector_trunc_v3f32() {
-; CHECK-LABEL: constrained_vector_trunc_v3f32:
+define <4 x float> @constrained_vector_uitofp_v4f32_v4i64(<4 x i64> %x) #0 {
+; CHECK-LABEL: constrained_vector_uitofp_v4f32_v4i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    subq $40, %rsp
-; CHECK-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; CHECK-NEXT:    callq truncf
-; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; CHECK-NEXT:    callq truncf
-; CHECK-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
-; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; CHECK-NEXT:    callq truncf
-; CHECK-NEXT:    movaps (%rsp), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    movq %xmm1, %rax
+; CHECK-NEXT:    movq %rax, %rcx
+; CHECK-NEXT:    shrq %rcx
+; CHECK-NEXT:    movl %eax, %edx
+; CHECK-NEXT:    andl $1, %edx
+; CHECK-NEXT:    orq %rcx, %rdx
+; CHECK-NEXT:    testq %rax, %rax
+; CHECK-NEXT:    cmovnsq %rax, %rdx
+; CHECK-NEXT:    cvtsi2ss %rdx, %xmm2
+; CHECK-NEXT:    jns .LBB182_2
+; CHECK-NEXT:  # %bb.1:
+; CHECK-NEXT:    addss %xmm2, %xmm2
+; CHECK-NEXT:  .LBB182_2: # %entry
+; CHECK-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; CHECK-NEXT:    movq %xmm1, %rax
+; CHECK-NEXT:    movq %rax, %rcx
+; CHECK-NEXT:    shrq %rcx
+; CHECK-NEXT:    movl %eax, %edx
+; CHECK-NEXT:    andl $1, %edx
+; CHECK-NEXT:    orq %rcx, %rdx
+; CHECK-NEXT:    testq %rax, %rax
+; CHECK-NEXT:    cmovnsq %rax, %rdx
+; CHECK-NEXT:    cvtsi2ss %rdx, %xmm3
+; CHECK-NEXT:    jns .LBB182_4
+; CHECK-NEXT:  # %bb.3:
+; CHECK-NEXT:    addss %xmm3, %xmm3
+; CHECK-NEXT:  .LBB182_4: # %entry
+; CHECK-NEXT:    movq %xmm0, %rax
+; CHECK-NEXT:    movq %rax, %rcx
+; CHECK-NEXT:    shrq %rcx
+; CHECK-NEXT:    movl %eax, %edx
+; CHECK-NEXT:    andl $1, %edx
+; CHECK-NEXT:    orq %rcx, %rdx
+; CHECK-NEXT:    testq %rax, %rax
+; CHECK-NEXT:    cmovnsq %rax, %rdx
+; CHECK-NEXT:    xorps %xmm1, %xmm1
+; CHECK-NEXT:    cvtsi2ss %rdx, %xmm1
+; CHECK-NEXT:    jns .LBB182_6
+; CHECK-NEXT:  # %bb.5:
+; CHECK-NEXT:    addss %xmm1, %xmm1
+; CHECK-NEXT:  .LBB182_6: # %entry
+; CHECK-NEXT:    unpcklps {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; CHECK-NEXT:    movq %xmm0, %rax
+; CHECK-NEXT:    movq %rax, %rcx
+; CHECK-NEXT:    shrq %rcx
+; CHECK-NEXT:    movl %eax, %edx
+; CHECK-NEXT:    andl $1, %edx
+; CHECK-NEXT:    orq %rcx, %rdx
+; CHECK-NEXT:    testq %rax, %rax
+; CHECK-NEXT:    cmovnsq %rax, %rdx
+; CHECK-NEXT:    xorps %xmm0, %xmm0
+; CHECK-NEXT:    cvtsi2ss %rdx, %xmm0
+; CHECK-NEXT:    jns .LBB182_8
+; CHECK-NEXT:  # %bb.7:
+; CHECK-NEXT:    addss %xmm0, %xmm0
+; CHECK-NEXT:  .LBB182_8: # %entry
 ; CHECK-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; CHECK-NEXT:    unpcklpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
-; CHECK-NEXT:    # xmm1 = xmm1[0],mem[0]
+; CHECK-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm2[0]
 ; CHECK-NEXT:    movaps %xmm1, %xmm0
-; CHECK-NEXT:    addq $40, %rsp
-; CHECK-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK-NEXT:    retq
 ;
-; AVX-LABEL: constrained_vector_trunc_v3f32:
-; AVX:       # %bb.0: # %entry
-; AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX-NEXT:    vroundss $11, %xmm0, %xmm0, %xmm0
-; AVX-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX-NEXT:    vroundss $11, %xmm1, %xmm1, %xmm1
-; AVX-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; AVX-NEXT:    vroundss $11, %xmm2, %xmm2, %xmm2
-; AVX-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[2,3]
-; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3]
-; AVX-NEXT:    retq
+; AVX1-LABEL: constrained_vector_uitofp_v4f32_v4i64:
+; AVX1:       # %bb.0: # %entry
+; AVX1-NEXT:    vpsrlq $1, %xmm0, %xmm1
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT:    vpsrlq $1, %xmm2, %xmm3
+; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
+; AVX1-NEXT:    vandpd {{.*}}(%rip), %ymm0, %ymm3
+; AVX1-NEXT:    vorpd %ymm3, %ymm1, %ymm1
+; AVX1-NEXT:    vblendvpd %xmm0, %xmm1, %xmm0, %xmm3
+; AVX1-NEXT:    vpextrq $1, %xmm3, %rax
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm4, %xmm4
+; AVX1-NEXT:    vmovq %xmm3, %rax
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm5, %xmm3
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[2,3]
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm1
+; AVX1-NEXT:    vblendvpd %xmm2, %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vmovq %xmm1, %rax
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm5, %xmm4
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm3 = xmm3[0,1],xmm4[0],xmm3[3]
+; AVX1-NEXT:    vpextrq $1, %xmm1, %rax
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm5, %xmm1
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[0]
+; AVX1-NEXT:    vaddps %xmm1, %xmm1, %xmm3
+; AVX1-NEXT:    vxorps %xmm4, %xmm4, %xmm4
+; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm4, %xmm2
+; AVX1-NEXT:    vpackssdw %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vblendvps %xmm0, %xmm3, %xmm1, %xmm0
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX512F-LABEL: constrained_vector_uitofp_v4f32_v4i64:
+; AVX512F:       # %bb.0: # %entry
+; AVX512F-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX512F-NEXT:    vcvtusi2ss %rax, %xmm1, %xmm1
+; AVX512F-NEXT:    vmovq %xmm0, %rax
+; AVX512F-NEXT:    vcvtusi2ss %rax, %xmm2, %xmm2
+; AVX512F-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[2,3]
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm0
+; AVX512F-NEXT:    vmovq %xmm0, %rax
+; AVX512F-NEXT:    vcvtusi2ss %rax, %xmm3, %xmm2
+; AVX512F-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0],xmm1[3]
+; AVX512F-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX512F-NEXT:    vcvtusi2ss %rax, %xmm3, %xmm0
+; AVX512F-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512DQ-LABEL: constrained_vector_uitofp_v4f32_v4i64:
+; AVX512DQ:       # %bb.0: # %entry
+; AVX512DQ-NEXT:    vmovaps %ymm0, %ymm0
+; AVX512DQ-NEXT:    vcvtuqq2ps %zmm0, %ymm0
+; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    retq
 entry:
-  %trunc = call <3 x float> @llvm.experimental.constrained.trunc.v3f32(
-                              <3 x float> <float 1.5, float 2.5, float 3.5>,
-                              metadata !"round.dynamic",
-                              metadata !"fpexcept.strict")
-  ret <3 x float> %trunc
+  %result = call <4 x float>
+           @llvm.experimental.constrained.uitofp.v4f32.v4i64(<4 x i64> %x,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret <4 x float> %result
 }
 
-define <3 x double> @constrained_vector_trunc_v3f64() {
-; CHECK-LABEL: constrained_vector_trunc_v3f64:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    subq $24, %rsp
-; CHECK-NEXT:    .cfi_def_cfa_offset 32
-; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; CHECK-NEXT:    callq trunc
-; CHECK-NEXT:    movsd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; CHECK-NEXT:    callq trunc
-; CHECK-NEXT:    movsd %xmm0, (%rsp) # 8-byte Spill
-; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; CHECK-NEXT:    callq trunc
-; CHECK-NEXT:    movsd %xmm0, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    fldl {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Reload
-; CHECK-NEXT:    # xmm0 = mem[0],zero
-; CHECK-NEXT:    movsd (%rsp), %xmm1 # 8-byte Reload
-; CHECK-NEXT:    # xmm1 = mem[0],zero
-; CHECK-NEXT:    addq $24, %rsp
-; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; Simple test to make sure we don't fuse vselect+strict_fadd into a masked operation.
+define <16 x float> @vpaddd_mask_test(<16 x float> %i, <16 x float> %j, <16 x i32> %mask1) nounwind readnone strictfp {
+; CHECK-LABEL: vpaddd_mask_test:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pxor %xmm10, %xmm10
+; CHECK-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm8
+; CHECK-NEXT:    pcmpeqd %xmm10, %xmm8
+; CHECK-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm9
+; CHECK-NEXT:    pcmpeqd %xmm10, %xmm9
+; CHECK-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm11
+; CHECK-NEXT:    pcmpeqd %xmm10, %xmm11
+; CHECK-NEXT:    pcmpeqd {{[0-9]+}}(%rsp), %xmm10
+; CHECK-NEXT:    addps %xmm3, %xmm7
+; CHECK-NEXT:    addps %xmm2, %xmm6
+; CHECK-NEXT:    addps %xmm1, %xmm5
+; CHECK-NEXT:    addps %xmm0, %xmm4
+; CHECK-NEXT:    andps %xmm10, %xmm0
+; CHECK-NEXT:    andnps %xmm4, %xmm10
+; CHECK-NEXT:    orps %xmm10, %xmm0
+; CHECK-NEXT:    andps %xmm11, %xmm1
+; CHECK-NEXT:    andnps %xmm5, %xmm11
+; CHECK-NEXT:    orps %xmm11, %xmm1
+; CHECK-NEXT:    andps %xmm9, %xmm2
+; CHECK-NEXT:    andnps %xmm6, %xmm9
+; CHECK-NEXT:    orps %xmm9, %xmm2
+; CHECK-NEXT:    andps %xmm8, %xmm3
+; CHECK-NEXT:    andnps %xmm7, %xmm8
+; CHECK-NEXT:    orps %xmm8, %xmm3
 ; CHECK-NEXT:    retq
 ;
-; AVX-LABEL: constrained_vector_trunc_v3f64:
-; AVX:       # %bb.0: # %entry
-; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; AVX-NEXT:    vroundsd $11, %xmm0, %xmm0, %xmm0
-; AVX-NEXT:    vroundpd $11, {{.*}}(%rip), %xmm1
-; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; AVX-NEXT:    retq
-entry:
-  %trunc = call <3 x double> @llvm.experimental.constrained.trunc.v3f64(
-                          <3 x double> <double 1.1, double 1.9, double 1.5>,
-                          metadata !"round.dynamic",
-                          metadata !"fpexcept.strict")
-  ret <3 x double> %trunc
+; AVX1-LABEL: vpaddd_mask_test:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm5, %xmm6
+; AVX1-NEXT:    vpxor %xmm7, %xmm7, %xmm7
+; AVX1-NEXT:    vpcmpeqd %xmm7, %xmm6, %xmm6
+; AVX1-NEXT:    vpcmpeqd %xmm7, %xmm5, %xmm5
+; AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm5, %ymm5
+; AVX1-NEXT:    vextractf128 $1, %ymm4, %xmm6
+; AVX1-NEXT:    vpcmpeqd %xmm7, %xmm6, %xmm6
+; AVX1-NEXT:    vpcmpeqd %xmm7, %xmm4, %xmm4
+; AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm4, %ymm4
+; AVX1-NEXT:    vaddps %ymm3, %ymm1, %ymm3
+; AVX1-NEXT:    vaddps %ymm2, %ymm0, %ymm2
+; AVX1-NEXT:    vblendvps %ymm4, %ymm0, %ymm2, %ymm0
+; AVX1-NEXT:    vblendvps %ymm5, %ymm1, %ymm3, %ymm1
+; AVX1-NEXT:    retq
+;
+; AVX512-LABEL: vpaddd_mask_test:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vptestmd %zmm2, %zmm2, %k1
+; AVX512-NEXT:    vaddps %zmm1, %zmm0, %zmm1
+; AVX512-NEXT:    vmovaps %zmm1, %zmm0 {%k1}
+; AVX512-NEXT:    retq
+  %mask = icmp ne <16 x i32> %mask1, zeroinitializer
+  %x = call <16 x float> @llvm.experimental.constrained.fadd.v16f32(<16 x float> %i, <16 x float> %j, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  %r = select <16 x i1> %mask, <16 x float> %x, <16 x float> %i
+  ret <16 x float> %r
 }
+declare  <16 x float> @llvm.experimental.constrained.fadd.v16f32(<16 x float>, <16 x float>, metadata, metadata)
 
+attributes #0 = { strictfp }
 
 ; Single width declarations
 declare <2 x double> @llvm.experimental.constrained.fadd.v2f64(<2 x double>, <2 x double>, metadata, metadata)
@@ -4622,14 +7598,30 @@ declare <2 x double> @llvm.experimental.constrained.log10.v2f64(<2 x double>, me
 declare <2 x double> @llvm.experimental.constrained.log2.v2f64(<2 x double>, metadata, metadata)
 declare <2 x double> @llvm.experimental.constrained.rint.v2f64(<2 x double>, metadata, metadata)
 declare <2 x double> @llvm.experimental.constrained.nearbyint.v2f64(<2 x double>, metadata, metadata)
-declare <2 x double> @llvm.experimental.constrained.maxnum.v2f64(<2 x double>, <2 x double>, metadata, metadata)
-declare <2 x double> @llvm.experimental.constrained.minnum.v2f64(<2 x double>, <2 x double>, metadata, metadata)
+declare <2 x double> @llvm.experimental.constrained.maxnum.v2f64(<2 x double>, <2 x double>, metadata)
+declare <2 x double> @llvm.experimental.constrained.minnum.v2f64(<2 x double>, <2 x double>, metadata)
+declare <2 x i32> @llvm.experimental.constrained.fptosi.v2i32.v2f32(<2 x float>, metadata)
+declare <2 x i64> @llvm.experimental.constrained.fptosi.v2i64.v2f32(<2 x float>, metadata)
+declare <2 x i32> @llvm.experimental.constrained.fptosi.v2i32.v2f64(<2 x double>, metadata)
+declare <2 x i64> @llvm.experimental.constrained.fptosi.v2i64.v2f64(<2 x double>, metadata)
+declare <2 x i32> @llvm.experimental.constrained.fptoui.v2i32.v2f32(<2 x float>, metadata)
+declare <2 x i64> @llvm.experimental.constrained.fptoui.v2i64.v2f32(<2 x float>, metadata)
+declare <2 x i32> @llvm.experimental.constrained.fptoui.v2i32.v2f64(<2 x double>, metadata)
+declare <2 x i64> @llvm.experimental.constrained.fptoui.v2i64.v2f64(<2 x double>, metadata)
 declare <2 x float> @llvm.experimental.constrained.fptrunc.v2f32.v2f64(<2 x double>, metadata, metadata)
 declare <2 x double> @llvm.experimental.constrained.fpext.v2f64.v2f32(<2 x float>, metadata)
-declare <2 x double> @llvm.experimental.constrained.ceil.v2f64(<2 x double>, metadata, metadata)
-declare <2 x double> @llvm.experimental.constrained.floor.v2f64(<2 x double>, metadata, metadata)
-declare <2 x double> @llvm.experimental.constrained.round.v2f64(<2 x double>, metadata, metadata)
-declare <2 x double> @llvm.experimental.constrained.trunc.v2f64(<2 x double>, metadata, metadata)
+declare <2 x double> @llvm.experimental.constrained.ceil.v2f64(<2 x double>, metadata)
+declare <2 x double> @llvm.experimental.constrained.floor.v2f64(<2 x double>, metadata)
+declare <2 x double> @llvm.experimental.constrained.round.v2f64(<2 x double>, metadata)
+declare <2 x double> @llvm.experimental.constrained.trunc.v2f64(<2 x double>, metadata)
+declare <2 x double> @llvm.experimental.constrained.sitofp.v2f64.v2i32(<2 x i32>, metadata, metadata)
+declare <2 x float> @llvm.experimental.constrained.sitofp.v2f32.v2i32(<2 x i32>, metadata, metadata)
+declare <2 x double> @llvm.experimental.constrained.sitofp.v2f64.v2i64(<2 x i64>, metadata, metadata)
+declare <2 x float> @llvm.experimental.constrained.sitofp.v2f32.v2i64(<2 x i64>, metadata, metadata)
+declare <2 x double> @llvm.experimental.constrained.uitofp.v2f64.v2i32(<2 x i32>, metadata, metadata)
+declare <2 x float> @llvm.experimental.constrained.uitofp.v2f32.v2i32(<2 x i32>, metadata, metadata)
+declare <2 x double> @llvm.experimental.constrained.uitofp.v2f64.v2i64(<2 x i64>, metadata, metadata)
+declare <2 x float> @llvm.experimental.constrained.uitofp.v2f32.v2i64(<2 x i64>, metadata, metadata)
 
 ; Scalar width declarations
 declare <1 x float> @llvm.experimental.constrained.fadd.v1f32(<1 x float>, <1 x float>, metadata, metadata)
@@ -4649,14 +7641,30 @@ declare <1 x float> @llvm.experimental.constrained.log10.v1f32(<1 x float>, meta
 declare <1 x float> @llvm.experimental.constrained.log2.v1f32(<1 x float>, metadata, metadata)
 declare <1 x float> @llvm.experimental.constrained.rint.v1f32(<1 x float>, metadata, metadata)
 declare <1 x float> @llvm.experimental.constrained.nearbyint.v1f32(<1 x float>, metadata, metadata)
-declare <1 x float> @llvm.experimental.constrained.maxnum.v1f32(<1 x float>, <1 x float>, metadata, metadata)
-declare <1 x float> @llvm.experimental.constrained.minnum.v1f32(<1 x float>, <1 x float>, metadata, metadata)
+declare <1 x float> @llvm.experimental.constrained.maxnum.v1f32(<1 x float>, <1 x float>, metadata)
+declare <1 x float> @llvm.experimental.constrained.minnum.v1f32(<1 x float>, <1 x float>, metadata)
+declare <1 x i32> @llvm.experimental.constrained.fptosi.v1i32.v1f32(<1 x float>, metadata)
+declare <1 x i64> @llvm.experimental.constrained.fptosi.v1i64.v1f32(<1 x float>, metadata)
+declare <1 x i32> @llvm.experimental.constrained.fptosi.v1i32.v1f64(<1 x double>, metadata)
+declare <1 x i64> @llvm.experimental.constrained.fptosi.v1i64.v1f64(<1 x double>, metadata)
+declare <1 x i32> @llvm.experimental.constrained.fptoui.v1i32.v1f32(<1 x float>, metadata)
+declare <1 x i64> @llvm.experimental.constrained.fptoui.v1i64.v1f32(<1 x float>, metadata)
+declare <1 x i32> @llvm.experimental.constrained.fptoui.v1i32.v1f64(<1 x double>, metadata)
+declare <1 x i64> @llvm.experimental.constrained.fptoui.v1i64.v1f64(<1 x double>, metadata)
 declare <1 x float> @llvm.experimental.constrained.fptrunc.v1f32.v1f64(<1 x double>, metadata, metadata)
 declare <1 x double> @llvm.experimental.constrained.fpext.v1f64.v1f32(<1 x float>, metadata)
-declare <1 x float> @llvm.experimental.constrained.ceil.v1f32(<1 x float>, metadata, metadata)
-declare <1 x float> @llvm.experimental.constrained.floor.v1f32(<1 x float>, metadata, metadata)
-declare <1 x float> @llvm.experimental.constrained.round.v1f32(<1 x float>, metadata, metadata)
-declare <1 x float> @llvm.experimental.constrained.trunc.v1f32(<1 x float>, metadata, metadata)
+declare <1 x float> @llvm.experimental.constrained.ceil.v1f32(<1 x float>, metadata)
+declare <1 x float> @llvm.experimental.constrained.floor.v1f32(<1 x float>, metadata)
+declare <1 x float> @llvm.experimental.constrained.round.v1f32(<1 x float>, metadata)
+declare <1 x float> @llvm.experimental.constrained.trunc.v1f32(<1 x float>, metadata)
+declare <1 x double> @llvm.experimental.constrained.sitofp.v1f64.v1i32(<1 x i32>, metadata, metadata)
+declare <1 x float> @llvm.experimental.constrained.sitofp.v1f32.v1i32(<1 x i32>, metadata, metadata)
+declare <1 x double> @llvm.experimental.constrained.sitofp.v1f64.v1i64(<1 x i64>, metadata, metadata)
+declare <1 x float> @llvm.experimental.constrained.sitofp.v1f32.v1i64(<1 x i64>, metadata, metadata)
+declare <1 x double> @llvm.experimental.constrained.uitofp.v1f64.v1i32(<1 x i32>, metadata, metadata)
+declare <1 x float> @llvm.experimental.constrained.uitofp.v1f32.v1i32(<1 x i32>, metadata, metadata)
+declare <1 x double> @llvm.experimental.constrained.uitofp.v1f64.v1i64(<1 x i64>, metadata, metadata)
+declare <1 x float> @llvm.experimental.constrained.uitofp.v1f32.v1i64(<1 x i64>, metadata, metadata)
 
 ; Illegal width declarations
 declare <3 x float> @llvm.experimental.constrained.fadd.v3f32(<3 x float>, <3 x float>, metadata, metadata)
@@ -4693,20 +7701,36 @@ declare <3 x float> @llvm.experimental.constrained.rint.v3f32(<3 x float>, metad
 declare <3 x double> @llvm.experimental.constrained.rint.v3f64(<3 x double>, metadata, metadata)
 declare <3 x float> @llvm.experimental.constrained.nearbyint.v3f32(<3 x float>, metadata, metadata)
 declare <3 x double> @llvm.experimental.constrained.nearbyint.v3f64(<3 x double>, metadata, metadata)
-declare <3 x float> @llvm.experimental.constrained.maxnum.v3f32(<3 x float>, <3 x float>, metadata, metadata)
-declare <3 x double> @llvm.experimental.constrained.maxnum.v3f64(<3 x double>, <3 x double>, metadata, metadata)
-declare <3 x float> @llvm.experimental.constrained.minnum.v3f32(<3 x float>, <3 x float>, metadata, metadata)
-declare <3 x double> @llvm.experimental.constrained.minnum.v3f64(<3 x double>, <3 x double>, metadata, metadata)
+declare <3 x float> @llvm.experimental.constrained.maxnum.v3f32(<3 x float>, <3 x float>, metadata)
+declare <3 x double> @llvm.experimental.constrained.maxnum.v3f64(<3 x double>, <3 x double>, metadata)
+declare <3 x float> @llvm.experimental.constrained.minnum.v3f32(<3 x float>, <3 x float>, metadata)
+declare <3 x double> @llvm.experimental.constrained.minnum.v3f64(<3 x double>, <3 x double>, metadata)
+declare <3 x i32> @llvm.experimental.constrained.fptosi.v3i32.v3f32(<3 x float>, metadata)
+declare <3 x i64> @llvm.experimental.constrained.fptosi.v3i64.v3f32(<3 x float>, metadata)
+declare <3 x i32> @llvm.experimental.constrained.fptosi.v3i32.v3f64(<3 x double>, metadata)
+declare <3 x i64> @llvm.experimental.constrained.fptosi.v3i64.v3f64(<3 x double>, metadata)
+declare <3 x i32> @llvm.experimental.constrained.fptoui.v3i32.v3f32(<3 x float>, metadata)
+declare <3 x i64> @llvm.experimental.constrained.fptoui.v3i64.v3f32(<3 x float>, metadata)
+declare <3 x i32> @llvm.experimental.constrained.fptoui.v3i32.v3f64(<3 x double>, metadata)
+declare <3 x i64> @llvm.experimental.constrained.fptoui.v3i64.v3f64(<3 x double>, metadata)
 declare <3 x float> @llvm.experimental.constrained.fptrunc.v3f32.v3f64(<3 x double>, metadata, metadata)
 declare <3 x double> @llvm.experimental.constrained.fpext.v3f64.v3f32(<3 x float>, metadata)
-declare <3 x float> @llvm.experimental.constrained.ceil.v3f32(<3 x float>, metadata, metadata)
-declare <3 x double> @llvm.experimental.constrained.ceil.v3f64(<3 x double>, metadata, metadata)
-declare <3 x float> @llvm.experimental.constrained.floor.v3f32(<3 x float>, metadata, metadata)
-declare <3 x double> @llvm.experimental.constrained.floor.v3f64(<3 x double>, metadata, metadata)
-declare <3 x float> @llvm.experimental.constrained.round.v3f32(<3 x float>, metadata, metadata)
-declare <3 x double> @llvm.experimental.constrained.round.v3f64(<3 x double>, metadata, metadata)
-declare <3 x float> @llvm.experimental.constrained.trunc.v3f32(<3 x float>, metadata, metadata)
-declare <3 x double> @llvm.experimental.constrained.trunc.v3f64(<3 x double>, metadata, metadata)
+declare <3 x float> @llvm.experimental.constrained.ceil.v3f32(<3 x float>, metadata)
+declare <3 x double> @llvm.experimental.constrained.ceil.v3f64(<3 x double>, metadata)
+declare <3 x float> @llvm.experimental.constrained.floor.v3f32(<3 x float>, metadata)
+declare <3 x double> @llvm.experimental.constrained.floor.v3f64(<3 x double>, metadata)
+declare <3 x float> @llvm.experimental.constrained.round.v3f32(<3 x float>, metadata)
+declare <3 x double> @llvm.experimental.constrained.round.v3f64(<3 x double>, metadata)
+declare <3 x float> @llvm.experimental.constrained.trunc.v3f32(<3 x float>, metadata)
+declare <3 x double> @llvm.experimental.constrained.trunc.v3f64(<3 x double>, metadata)
+declare <3 x double> @llvm.experimental.constrained.sitofp.v3f64.v3i32(<3 x i32>, metadata, metadata)
+declare <3 x float> @llvm.experimental.constrained.sitofp.v3f32.v3i32(<3 x i32>, metadata, metadata)
+declare <3 x double> @llvm.experimental.constrained.sitofp.v3f64.v3i64(<3 x i64>, metadata, metadata)
+declare <3 x float> @llvm.experimental.constrained.sitofp.v3f32.v3i64(<3 x i64>, metadata, metadata)
+declare <3 x double> @llvm.experimental.constrained.uitofp.v3f64.v3i32(<3 x i32>, metadata, metadata)
+declare <3 x float> @llvm.experimental.constrained.uitofp.v3f32.v3i32(<3 x i32>, metadata, metadata)
+declare <3 x double> @llvm.experimental.constrained.uitofp.v3f64.v3i64(<3 x i64>, metadata, metadata)
+declare <3 x float> @llvm.experimental.constrained.uitofp.v3f32.v3i64(<3 x i64>, metadata, metadata)
 
 ; Double width declarations
 declare <4 x double> @llvm.experimental.constrained.fadd.v4f64(<4 x double>, <4 x double>, metadata, metadata)
@@ -4726,11 +7750,28 @@ declare <4 x double> @llvm.experimental.constrained.log10.v4f64(<4 x double>, me
 declare <4 x double> @llvm.experimental.constrained.log2.v4f64(<4 x double>, metadata, metadata)
 declare <4 x double> @llvm.experimental.constrained.rint.v4f64(<4 x double>, metadata, metadata)
 declare <4 x double> @llvm.experimental.constrained.nearbyint.v4f64(<4 x double>, metadata, metadata)
-declare <4 x double> @llvm.experimental.constrained.maxnum.v4f64(<4 x double>, <4 x double>, metadata, metadata)
-declare <4 x double> @llvm.experimental.constrained.minnum.v4f64(<4 x double>, <4 x double>, metadata, metadata)
+declare <4 x double> @llvm.experimental.constrained.maxnum.v4f64(<4 x double>, <4 x double>, metadata)
+declare <4 x double> @llvm.experimental.constrained.minnum.v4f64(<4 x double>, <4 x double>, metadata)
+declare <4 x i32> @llvm.experimental.constrained.fptosi.v4i32.v4f32(<4 x float>, metadata)
+declare <4 x i64> @llvm.experimental.constrained.fptosi.v4i64.v4f32(<4 x float>, metadata)
+declare <4 x i32> @llvm.experimental.constrained.fptosi.v4i32.v4f64(<4 x double>, metadata)
+declare <4 x i64> @llvm.experimental.constrained.fptosi.v4i64.v4f64(<4 x double>, metadata)
+declare <4 x i32> @llvm.experimental.constrained.fptoui.v4i32.v4f32(<4 x float>, metadata)
+declare <4 x i64> @llvm.experimental.constrained.fptoui.v4i64.v4f32(<4 x float>, metadata)
+declare <4 x i32> @llvm.experimental.constrained.fptoui.v4i32.v4f64(<4 x double>, metadata)
+declare <4 x i64> @llvm.experimental.constrained.fptoui.v4i64.v4f64(<4 x double>, metadata)
 declare <4 x float> @llvm.experimental.constrained.fptrunc.v4f32.v4f64(<4 x double>, metadata, metadata)
 declare <4 x double> @llvm.experimental.constrained.fpext.v4f64.v4f32(<4 x float>, metadata)
-declare <4 x double> @llvm.experimental.constrained.ceil.v4f64(<4 x double>, metadata, metadata)
-declare <4 x double> @llvm.experimental.constrained.floor.v4f64(<4 x double>, metadata, metadata)
-declare <4 x double> @llvm.experimental.constrained.round.v4f64(<4 x double>, metadata, metadata)
-declare <4 x double> @llvm.experimental.constrained.trunc.v4f64(<4 x double>, metadata, metadata)
+declare <4 x double> @llvm.experimental.constrained.ceil.v4f64(<4 x double>, metadata)
+declare <4 x double> @llvm.experimental.constrained.floor.v4f64(<4 x double>, metadata)
+declare <4 x double> @llvm.experimental.constrained.round.v4f64(<4 x double>, metadata)
+declare <4 x double> @llvm.experimental.constrained.trunc.v4f64(<4 x double>, metadata)
+declare <4 x double> @llvm.experimental.constrained.sitofp.v4f64.v4i32(<4 x i32>, metadata, metadata)
+declare <4 x float> @llvm.experimental.constrained.sitofp.v4f32.v4i32(<4 x i32>, metadata, metadata)
+declare <4 x double> @llvm.experimental.constrained.sitofp.v4f64.v4i64(<4 x i64>, metadata, metadata)
+declare <4 x float> @llvm.experimental.constrained.sitofp.v4f32.v4i64(<4 x i64>, metadata, metadata)
+declare <4 x double> @llvm.experimental.constrained.uitofp.v4f64.v4i32(<4 x i32>, metadata, metadata)
+declare <4 x float> @llvm.experimental.constrained.uitofp.v4f32.v4i32(<4 x i32>, metadata, metadata)
+declare <4 x double> @llvm.experimental.constrained.uitofp.v4f64.v4i64(<4 x i64>, metadata, metadata)
+declare <4 x float> @llvm.experimental.constrained.uitofp.v4f32.v4i64(<4 x i64>, metadata, metadata)
+
diff --git a/llvm/test/CodeGen/X86/vector-ext-logic.ll b/llvm/test/CodeGen/X86/vector-ext-logic.ll
index f81721ed1..0d53db323 100644
--- a/llvm/test/CodeGen/X86/vector-ext-logic.ll
+++ b/llvm/test/CodeGen/X86/vector-ext-logic.ll
@@ -140,14 +140,17 @@ define <8 x i32> @sext_xor_v8i32(<8 x i16> %x, <8 x i16> %y) {
 define <8 x i16> @zext_and_v8i16(<8 x i8> %x, <8 x i8> %y) {
 ; SSE2-LABEL: zext_and_v8i16:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    andps %xmm1, %xmm0
-; SSE2-NEXT:    andps {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; SSE2-NEXT:    pand %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; AVX2-LABEL: zext_and_v8i16:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vandps %xmm0, %xmm1, %xmm0
-; AVX2-NEXT:    vandps {{.*}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    retq
   %xz = zext <8 x i8> %x to <8 x i16>
   %yz = zext <8 x i8> %y to <8 x i16>
@@ -158,14 +161,17 @@ define <8 x i16> @zext_and_v8i16(<8 x i8> %x, <8 x i8> %y) {
 define <8 x i16> @zext_or_v8i16(<8 x i8> %x, <8 x i8> %y) {
 ; SSE2-LABEL: zext_or_v8i16:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    orps %xmm1, %xmm0
-; SSE2-NEXT:    andps {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; AVX2-LABEL: zext_or_v8i16:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vorps %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vandps {{.*}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    retq
   %xz = zext <8 x i8> %x to <8 x i16>
   %yz = zext <8 x i8> %y to <8 x i16>
@@ -176,14 +182,17 @@ define <8 x i16> @zext_or_v8i16(<8 x i8> %x, <8 x i8> %y) {
 define <8 x i16> @zext_xor_v8i16(<8 x i8> %x, <8 x i8> %y) {
 ; SSE2-LABEL: zext_xor_v8i16:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    xorps %xmm1, %xmm0
-; SSE2-NEXT:    andps {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; SSE2-NEXT:    pxor %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; AVX2-LABEL: zext_xor_v8i16:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vxorps %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vandps {{.*}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    retq
   %xz = zext <8 x i8> %x to <8 x i16>
   %yz = zext <8 x i8> %y to <8 x i16>
@@ -194,19 +203,17 @@ define <8 x i16> @zext_xor_v8i16(<8 x i8> %x, <8 x i8> %y) {
 define <8 x i16> @sext_and_v8i16(<8 x i8> %x, <8 x i8> %y) {
 ; SSE2-LABEL: sext_and_v8i16:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    psllw $8, %xmm0
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE2-NEXT:    psraw $8, %xmm2
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
 ; SSE2-NEXT:    psraw $8, %xmm0
-; SSE2-NEXT:    psllw $8, %xmm1
-; SSE2-NEXT:    psraw $8, %xmm1
-; SSE2-NEXT:    pand %xmm1, %xmm0
+; SSE2-NEXT:    pand %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; AVX2-LABEL: sext_and_v8i16:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpsllw $8, %xmm0, %xmm0
-; AVX2-NEXT:    vpsraw $8, %xmm0, %xmm0
-; AVX2-NEXT:    vpsllw $8, %xmm1, %xmm1
-; AVX2-NEXT:    vpsraw $8, %xmm1, %xmm1
+; AVX2-NEXT:    vpmovsxbw %xmm0, %xmm0
+; AVX2-NEXT:    vpmovsxbw %xmm1, %xmm1
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    retq
   %xs = sext <8 x i8> %x to <8 x i16>
@@ -218,19 +225,17 @@ define <8 x i16> @sext_and_v8i16(<8 x i8> %x, <8 x i8> %y) {
 define <8 x i16> @sext_or_v8i16(<8 x i8> %x, <8 x i8> %y) {
 ; SSE2-LABEL: sext_or_v8i16:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    psllw $8, %xmm0
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE2-NEXT:    psraw $8, %xmm2
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
 ; SSE2-NEXT:    psraw $8, %xmm0
-; SSE2-NEXT:    psllw $8, %xmm1
-; SSE2-NEXT:    psraw $8, %xmm1
-; SSE2-NEXT:    por %xmm1, %xmm0
+; SSE2-NEXT:    por %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; AVX2-LABEL: sext_or_v8i16:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpsllw $8, %xmm0, %xmm0
-; AVX2-NEXT:    vpsraw $8, %xmm0, %xmm0
-; AVX2-NEXT:    vpsllw $8, %xmm1, %xmm1
-; AVX2-NEXT:    vpsraw $8, %xmm1, %xmm1
+; AVX2-NEXT:    vpmovsxbw %xmm0, %xmm0
+; AVX2-NEXT:    vpmovsxbw %xmm1, %xmm1
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    retq
   %xs = sext <8 x i8> %x to <8 x i16>
@@ -242,19 +247,17 @@ define <8 x i16> @sext_or_v8i16(<8 x i8> %x, <8 x i8> %y) {
 define <8 x i16> @sext_xor_v8i16(<8 x i8> %x, <8 x i8> %y) {
 ; SSE2-LABEL: sext_xor_v8i16:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    psllw $8, %xmm0
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE2-NEXT:    psraw $8, %xmm2
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
 ; SSE2-NEXT:    psraw $8, %xmm0
-; SSE2-NEXT:    psllw $8, %xmm1
-; SSE2-NEXT:    psraw $8, %xmm1
-; SSE2-NEXT:    pxor %xmm1, %xmm0
+; SSE2-NEXT:    pxor %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; AVX2-LABEL: sext_xor_v8i16:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpsllw $8, %xmm0, %xmm0
-; AVX2-NEXT:    vpsraw $8, %xmm0, %xmm0
-; AVX2-NEXT:    vpsllw $8, %xmm1, %xmm1
-; AVX2-NEXT:    vpsraw $8, %xmm1, %xmm1
+; AVX2-NEXT:    vpmovsxbw %xmm0, %xmm0
+; AVX2-NEXT:    vpmovsxbw %xmm1, %xmm1
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    retq
   %xs = sext <8 x i8> %x to <8 x i16>
@@ -295,18 +298,13 @@ define <8 x i32> @bool_zext_and(<8 x i1> %x, <8 x i1> %y) {
 define <8 x i32> @bool_zext_or(<8 x i1> %x, <8 x i1> %y) {
 ; SSE2-LABEL: bool_zext_or:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [1,1,1,1,1,1,1,1]
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    movdqa %xmm0, %xmm4
-; SSE2-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm2
-; SSE2-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]
-; SSE2-NEXT:    por %xmm4, %xmm2
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
-; SSE2-NEXT:    por %xmm1, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    por %xmm1, %xmm2
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm2
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
 ; SSE2-NEXT:    movdqa %xmm2, %xmm1
 ; SSE2-NEXT:    retq
 ;
@@ -325,18 +323,13 @@ define <8 x i32> @bool_zext_or(<8 x i1> %x, <8 x i1> %y) {
 define <8 x i32> @bool_zext_xor(<8 x i1> %x, <8 x i1> %y) {
 ; SSE2-LABEL: bool_zext_xor:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [1,1,1,1,1,1,1,1]
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    movdqa %xmm0, %xmm4
-; SSE2-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm2
-; SSE2-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]
-; SSE2-NEXT:    pxor %xmm4, %xmm2
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
-; SSE2-NEXT:    pxor %xmm1, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    pxor %xmm1, %xmm2
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm2
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
 ; SSE2-NEXT:    movdqa %xmm2, %xmm1
 ; SSE2-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/vector-fshl-128.ll b/llvm/test/CodeGen/X86/vector-fshl-128.ll
index b40c88169..5fb1c6fb0 100644
--- a/llvm/test/CodeGen/X86/vector-fshl-128.ll
+++ b/llvm/test/CodeGen/X86/vector-fshl-128.ll
@@ -498,7 +498,6 @@ define <4 x i32> @var_funnnel_v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i32> %amt)
 define <8 x i16> @var_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %amt) nounwind {
 ; SSE2-LABEL: var_funnnel_v8i16:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm2
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
 ; SSE2-NEXT:    psubw %xmm2, %xmm3
 ; SSE2-NEXT:    psllw $12, %xmm3
@@ -531,32 +530,32 @@ define <8 x i16> @var_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %amt)
 ; SSE2-NEXT:    pandn %xmm1, %xmm4
 ; SSE2-NEXT:    psrlw $1, %xmm1
 ; SSE2-NEXT:    pand %xmm3, %xmm1
-; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    movdqa %xmm2, %xmm5
-; SSE2-NEXT:    punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm3[4],xmm5[5],xmm3[5],xmm5[6],xmm3[6],xmm5[7],xmm3[7]
-; SSE2-NEXT:    pslld $23, %xmm5
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm3
+; SSE2-NEXT:    pxor %xmm5, %xmm5
+; SSE2-NEXT:    pcmpeqw %xmm2, %xmm5
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE2-NEXT:    pslld $23, %xmm2
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm6 = [1065353216,1065353216,1065353216,1065353216]
-; SSE2-NEXT:    paddd %xmm6, %xmm5
-; SSE2-NEXT:    cvttps2dq %xmm5, %xmm5
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm5 = xmm5[0,2,2,3,4,5,6,7]
-; SSE2-NEXT:    pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,6,6,7]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[0,2,2,3]
-; SSE2-NEXT:    movdqa %xmm2, %xmm7
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm7 = xmm7[0],xmm3[0],xmm7[1],xmm3[1],xmm7[2],xmm3[2],xmm7[3],xmm3[3]
-; SSE2-NEXT:    pslld $23, %xmm7
-; SSE2-NEXT:    paddd %xmm6, %xmm7
-; SSE2-NEXT:    cvttps2dq %xmm7, %xmm6
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm6 = xmm6[0,2,2,3,4,5,6,7]
-; SSE2-NEXT:    pshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,4,6,6,7]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[0,2,2,3]
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm6 = xmm6[0],xmm5[0]
-; SSE2-NEXT:    pmullw %xmm0, %xmm6
-; SSE2-NEXT:    por %xmm4, %xmm6
-; SSE2-NEXT:    por %xmm1, %xmm6
-; SSE2-NEXT:    pcmpeqw %xmm3, %xmm2
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    pandn %xmm6, %xmm2
-; SSE2-NEXT:    por %xmm2, %xmm0
+; SSE2-NEXT:    paddd %xmm6, %xmm2
+; SSE2-NEXT:    cvttps2dq %xmm2, %xmm2
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,2,2,3,4,5,6,7]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,6,6,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3]
+; SSE2-NEXT:    pslld $23, %xmm3
+; SSE2-NEXT:    paddd %xmm6, %xmm3
+; SSE2-NEXT:    cvttps2dq %xmm3, %xmm3
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm3[0,2,2,3,4,5,6,7]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,6,6,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
+; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm2[0]
+; SSE2-NEXT:    pmullw %xmm0, %xmm3
+; SSE2-NEXT:    por %xmm4, %xmm3
+; SSE2-NEXT:    por %xmm1, %xmm3
+; SSE2-NEXT:    pand %xmm5, %xmm0
+; SSE2-NEXT:    pandn %xmm3, %xmm5
+; SSE2-NEXT:    por %xmm5, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: var_funnnel_v8i16:
@@ -588,23 +587,22 @@ define <8 x i16> @var_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %amt)
 ; SSE41-NEXT:    paddw %xmm4, %xmm4
 ; SSE41-NEXT:    movdqa %xmm4, %xmm0
 ; SSE41-NEXT:    pblendvb %xmm0, %xmm5, %xmm1
+; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    movdqa %xmm2, %xmm4
-; SSE41-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
-; SSE41-NEXT:    pslld $23, %xmm4
+; SSE41-NEXT:    pcmpeqw %xmm2, %xmm0
+; SSE41-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE41-NEXT:    pslld $23, %xmm2
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm5 = [1065353216,1065353216,1065353216,1065353216]
-; SSE41-NEXT:    paddd %xmm5, %xmm4
-; SSE41-NEXT:    cvttps2dq %xmm4, %xmm6
-; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
+; SSE41-NEXT:    paddd %xmm5, %xmm2
+; SSE41-NEXT:    cvttps2dq %xmm2, %xmm6
 ; SSE41-NEXT:    pslld $23, %xmm4
 ; SSE41-NEXT:    paddd %xmm5, %xmm4
-; SSE41-NEXT:    cvttps2dq %xmm4, %xmm4
-; SSE41-NEXT:    packusdw %xmm6, %xmm4
-; SSE41-NEXT:    pmullw %xmm3, %xmm4
-; SSE41-NEXT:    por %xmm1, %xmm4
-; SSE41-NEXT:    pcmpeqw %xmm2, %xmm0
-; SSE41-NEXT:    pblendvb %xmm0, %xmm3, %xmm4
-; SSE41-NEXT:    movdqa %xmm4, %xmm0
+; SSE41-NEXT:    cvttps2dq %xmm4, %xmm2
+; SSE41-NEXT:    packusdw %xmm6, %xmm2
+; SSE41-NEXT:    pmullw %xmm3, %xmm2
+; SSE41-NEXT:    por %xmm1, %xmm2
+; SSE41-NEXT:    pblendvb %xmm0, %xmm3, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: var_funnnel_v8i16:
@@ -626,39 +624,38 @@ define <8 x i16> @var_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %amt)
 ; AVX1-NEXT:    vpsrlw $1, %xmm1, %xmm3
 ; AVX1-NEXT:    vpaddw %xmm4, %xmm4, %xmm4
 ; AVX1-NEXT:    vpblendvb %xmm4, %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]
-; AVX1-NEXT:    vpslld $23, %xmm4, %xmm4
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm5 = [1065353216,1065353216,1065353216,1065353216]
-; AVX1-NEXT:    vpaddd %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; AVX1-NEXT:    vpslld $23, %xmm3, %xmm3
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [1065353216,1065353216,1065353216,1065353216]
+; AVX1-NEXT:    vpaddd %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vcvttps2dq %xmm3, %xmm3
+; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm5 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
+; AVX1-NEXT:    vpslld $23, %xmm5, %xmm5
+; AVX1-NEXT:    vpaddd %xmm4, %xmm5, %xmm4
 ; AVX1-NEXT:    vcvttps2dq %xmm4, %xmm4
-; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm6 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
-; AVX1-NEXT:    vpslld $23, %xmm6, %xmm6
-; AVX1-NEXT:    vpaddd %xmm5, %xmm6, %xmm5
-; AVX1-NEXT:    vcvttps2dq %xmm5, %xmm5
-; AVX1-NEXT:    vpackusdw %xmm4, %xmm5, %xmm4
-; AVX1-NEXT:    vpmullw %xmm4, %xmm0, %xmm4
-; AVX1-NEXT:    vpor %xmm1, %xmm4, %xmm1
+; AVX1-NEXT:    vpackusdw %xmm3, %xmm4, %xmm3
+; AVX1-NEXT:    vpmullw %xmm3, %xmm0, %xmm3
+; AVX1-NEXT:    vpor %xmm1, %xmm3, %xmm1
+; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
 ; AVX1-NEXT:    vpcmpeqw %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: var_funnnel_v8i16:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; AVX2-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
+; AVX2-NEXT:    vpsubw %xmm2, %xmm3, %xmm3
+; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
+; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX2-NEXT:    vpsrlvd %ymm3, %ymm1, %ymm1
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm3
+; AVX2-NEXT:    vpackusdw %xmm3, %xmm1, %xmm1
+; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
 ; AVX2-NEXT:    vpsllvd %ymm4, %ymm3, %ymm3
-; AVX2-NEXT:    vmovdqa {{.*#+}} ymm4 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; AVX2-NEXT:    vpshufb %ymm4, %ymm3, %ymm3
+; AVX2-NEXT:    vpshufb {{.*#+}} ymm3 = ymm3[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,2,2,3]
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm5 = [16,16,16,16,16,16,16,16]
-; AVX2-NEXT:    vpsubw %xmm2, %xmm5, %xmm5
-; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm5 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
-; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
-; AVX2-NEXT:    vpsrlvd %ymm5, %ymm1, %ymm1
-; AVX2-NEXT:    vpshufb %ymm4, %ymm1, %ymm1
-; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3]
 ; AVX2-NEXT:    vpor %xmm1, %xmm3, %xmm1
 ; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
 ; AVX2-NEXT:    vpcmpeqw %xmm3, %xmm2, %xmm2
@@ -768,7 +765,6 @@ define <8 x i16> @var_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %amt)
 ;
 ; X32-SSE-LABEL: var_funnnel_v8i16:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm2
 ; X32-SSE-NEXT:    movdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
 ; X32-SSE-NEXT:    psubw %xmm2, %xmm3
 ; X32-SSE-NEXT:    psllw $12, %xmm3
@@ -801,32 +797,32 @@ define <8 x i16> @var_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %amt)
 ; X32-SSE-NEXT:    pandn %xmm1, %xmm4
 ; X32-SSE-NEXT:    psrlw $1, %xmm1
 ; X32-SSE-NEXT:    pand %xmm3, %xmm1
-; X32-SSE-NEXT:    pxor %xmm3, %xmm3
-; X32-SSE-NEXT:    movdqa %xmm2, %xmm5
-; X32-SSE-NEXT:    punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm3[4],xmm5[5],xmm3[5],xmm5[6],xmm3[6],xmm5[7],xmm3[7]
-; X32-SSE-NEXT:    pslld $23, %xmm5
+; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm2
+; X32-SSE-NEXT:    movdqa %xmm2, %xmm3
+; X32-SSE-NEXT:    pxor %xmm5, %xmm5
+; X32-SSE-NEXT:    pcmpeqw %xmm2, %xmm5
+; X32-SSE-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; X32-SSE-NEXT:    pslld $23, %xmm2
 ; X32-SSE-NEXT:    movdqa {{.*#+}} xmm6 = [1065353216,1065353216,1065353216,1065353216]
-; X32-SSE-NEXT:    paddd %xmm6, %xmm5
-; X32-SSE-NEXT:    cvttps2dq %xmm5, %xmm5
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm5 = xmm5[0,2,2,3,4,5,6,7]
-; X32-SSE-NEXT:    pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,6,6,7]
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[0,2,2,3]
-; X32-SSE-NEXT:    movdqa %xmm2, %xmm7
-; X32-SSE-NEXT:    punpcklwd {{.*#+}} xmm7 = xmm7[0],xmm3[0],xmm7[1],xmm3[1],xmm7[2],xmm3[2],xmm7[3],xmm3[3]
-; X32-SSE-NEXT:    pslld $23, %xmm7
-; X32-SSE-NEXT:    paddd %xmm6, %xmm7
-; X32-SSE-NEXT:    cvttps2dq %xmm7, %xmm6
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm6 = xmm6[0,2,2,3,4,5,6,7]
-; X32-SSE-NEXT:    pshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,4,6,6,7]
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[0,2,2,3]
-; X32-SSE-NEXT:    punpcklqdq {{.*#+}} xmm6 = xmm6[0],xmm5[0]
-; X32-SSE-NEXT:    pmullw %xmm0, %xmm6
-; X32-SSE-NEXT:    por %xmm4, %xmm6
-; X32-SSE-NEXT:    por %xmm1, %xmm6
-; X32-SSE-NEXT:    pcmpeqw %xmm3, %xmm2
-; X32-SSE-NEXT:    pand %xmm2, %xmm0
-; X32-SSE-NEXT:    pandn %xmm6, %xmm2
-; X32-SSE-NEXT:    por %xmm2, %xmm0
+; X32-SSE-NEXT:    paddd %xmm6, %xmm2
+; X32-SSE-NEXT:    cvttps2dq %xmm2, %xmm2
+; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,2,2,3,4,5,6,7]
+; X32-SSE-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,6,6,7]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; X32-SSE-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3]
+; X32-SSE-NEXT:    pslld $23, %xmm3
+; X32-SSE-NEXT:    paddd %xmm6, %xmm3
+; X32-SSE-NEXT:    cvttps2dq %xmm3, %xmm3
+; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm3 = xmm3[0,2,2,3,4,5,6,7]
+; X32-SSE-NEXT:    pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,6,6,7]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
+; X32-SSE-NEXT:    punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm2[0]
+; X32-SSE-NEXT:    pmullw %xmm0, %xmm3
+; X32-SSE-NEXT:    por %xmm4, %xmm3
+; X32-SSE-NEXT:    por %xmm1, %xmm3
+; X32-SSE-NEXT:    pand %xmm5, %xmm0
+; X32-SSE-NEXT:    pandn %xmm3, %xmm5
+; X32-SSE-NEXT:    por %xmm5, %xmm0
 ; X32-SSE-NEXT:    retl
   %res = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %amt)
   ret <8 x i16> %res
@@ -2983,14 +2979,53 @@ define <16 x i8> @splatconstant_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y) nounwi
 ; AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
-; AVX512-LABEL: splatconstant_funnnel_v16i8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsrlw $4, %xmm1, %xmm1
-; AVX512-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
-; AVX512-NEXT:    vpsllw $4, %xmm0, %xmm0
-; AVX512-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: splatconstant_funnnel_v16i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsrlw $4, %xmm1, %xmm1
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512F-NEXT:    vpsllw $4, %xmm0, %xmm0
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX512F-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: splatconstant_funnnel_v16i8:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsllw $4, %xmm0, %xmm2
+; AVX512VL-NEXT:    vpsrlw $4, %xmm1, %xmm0
+; AVX512VL-NEXT:    vpternlogq $216, {{.*}}(%rip), %xmm2, %xmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: splatconstant_funnnel_v16i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsrlw $4, %xmm1, %xmm1
+; AVX512BW-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512BW-NEXT:    vpsllw $4, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX512BW-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: splatconstant_funnnel_v16i8:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpsrlw $4, %xmm1, %xmm1
+; AVX512VBMI2-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512VBMI2-NEXT:    vpsllw $4, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLBW-LABEL: splatconstant_funnnel_v16i8:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpsllw $4, %xmm0, %xmm2
+; AVX512VLBW-NEXT:    vpsrlw $4, %xmm1, %xmm0
+; AVX512VLBW-NEXT:    vpternlogq $216, {{.*}}(%rip), %xmm2, %xmm0
+; AVX512VLBW-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v16i8:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpsllw $4, %xmm0, %xmm2
+; AVX512VLVBMI2-NEXT:    vpsrlw $4, %xmm1, %xmm0
+; AVX512VLVBMI2-NEXT:    vpternlogq $216, {{.*}}(%rip), %xmm2, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
 ;
 ; XOP-LABEL: splatconstant_funnnel_v16i8:
 ; XOP:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/vector-fshl-256.ll b/llvm/test/CodeGen/X86/vector-fshl-256.ll
index 1e30cb274..211aea7f5 100644
--- a/llvm/test/CodeGen/X86/vector-fshl-256.ll
+++ b/llvm/test/CodeGen/X86/vector-fshl-256.ll
@@ -230,8 +230,8 @@ define <8 x i32> @var_funnnel_v8i32(<8 x i32> %x, <8 x i32> %y, <8 x i32> %amt)
 ; AVX1-NEXT:    vpmulld %xmm5, %xmm0, %xmm5
 ; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm5, %ymm4
 ; AVX1-NEXT:    vorps %ymm1, %ymm4, %ymm1
-; AVX1-NEXT:    vpcmpeqd %xmm9, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpeqd %xmm9, %xmm2, %xmm2
+; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm9, %xmm3
+; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm9, %xmm2
 ; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
 ; AVX1-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
@@ -403,32 +403,32 @@ define <16 x i16> @var_funnnel_v16i16(<16 x i16> %x, <16 x i16> %y, <16 x i16> %
 ; AVX1-NEXT:    vpsrlw $1, %xmm1, %xmm5
 ; AVX1-NEXT:    vpaddw %xmm6, %xmm6, %xmm6
 ; AVX1-NEXT:    vpblendvb %xmm6, %xmm5, %xmm1, %xmm1
-; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm1, %ymm8
-; AVX1-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7]
-; AVX1-NEXT:    vpslld $23, %xmm5, %xmm5
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [1065353216,1065353216,1065353216,1065353216]
-; AVX1-NEXT:    vpaddd %xmm6, %xmm5, %xmm5
-; AVX1-NEXT:    vcvttps2dq %xmm5, %xmm5
-; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm7 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero
-; AVX1-NEXT:    vpslld $23, %xmm7, %xmm7
-; AVX1-NEXT:    vpaddd %xmm6, %xmm7, %xmm7
-; AVX1-NEXT:    vcvttps2dq %xmm7, %xmm7
-; AVX1-NEXT:    vpackusdw %xmm5, %xmm7, %xmm5
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm7
-; AVX1-NEXT:    vpmullw %xmm5, %xmm7, %xmm5
-; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm7 = xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7]
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm1, %ymm1
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; AVX1-NEXT:    vpslld $23, %xmm4, %xmm4
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm5 = [1065353216,1065353216,1065353216,1065353216]
+; AVX1-NEXT:    vpaddd %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vcvttps2dq %xmm4, %xmm4
+; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm6 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero
+; AVX1-NEXT:    vpslld $23, %xmm6, %xmm6
+; AVX1-NEXT:    vpaddd %xmm5, %xmm6, %xmm6
+; AVX1-NEXT:    vcvttps2dq %xmm6, %xmm6
+; AVX1-NEXT:    vpackusdw %xmm4, %xmm6, %xmm4
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm6
+; AVX1-NEXT:    vpmullw %xmm4, %xmm6, %xmm4
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm6 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; AVX1-NEXT:    vpslld $23, %xmm6, %xmm6
+; AVX1-NEXT:    vpaddd %xmm5, %xmm6, %xmm6
+; AVX1-NEXT:    vcvttps2dq %xmm6, %xmm6
+; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm7 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
 ; AVX1-NEXT:    vpslld $23, %xmm7, %xmm7
-; AVX1-NEXT:    vpaddd %xmm6, %xmm7, %xmm7
-; AVX1-NEXT:    vcvttps2dq %xmm7, %xmm7
-; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm1 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
-; AVX1-NEXT:    vpslld $23, %xmm1, %xmm1
-; AVX1-NEXT:    vpaddd %xmm6, %xmm1, %xmm1
-; AVX1-NEXT:    vcvttps2dq %xmm1, %xmm1
-; AVX1-NEXT:    vpackusdw %xmm7, %xmm1, %xmm1
-; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm1, %ymm1
-; AVX1-NEXT:    vorps %ymm8, %ymm1, %ymm1
+; AVX1-NEXT:    vpaddd %xmm5, %xmm7, %xmm5
+; AVX1-NEXT:    vcvttps2dq %xmm5, %xmm5
+; AVX1-NEXT:    vpackusdw %xmm6, %xmm5, %xmm5
+; AVX1-NEXT:    vpmullw %xmm5, %xmm0, %xmm5
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm5, %ymm4
+; AVX1-NEXT:    vorps %ymm1, %ymm4, %ymm1
+; AVX1-NEXT:    vxorps %xmm4, %xmm4, %xmm4
 ; AVX1-NEXT:    vpcmpeqw %xmm4, %xmm3, %xmm3
 ; AVX1-NEXT:    vpcmpeqw %xmm4, %xmm2, %xmm2
 ; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
@@ -606,7 +606,7 @@ define <32 x i8> @var_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y, <32 x i8> %amt)
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm4
 ; AVX1-NEXT:    vpsrlw $4, %xmm4, %xmm3
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm8 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX1-NEXT:    vpand %xmm8, %xmm3, %xmm6
+; AVX1-NEXT:    vpand %xmm3, %xmm8, %xmm6
 ; AVX1-NEXT:    vandps {{.*}}(%rip), %ymm2, %ymm2
 ; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm3
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm9 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
@@ -615,7 +615,7 @@ define <32 x i8> @var_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y, <32 x i8> %amt)
 ; AVX1-NEXT:    vpblendvb %xmm5, %xmm6, %xmm4, %xmm4
 ; AVX1-NEXT:    vpsrlw $2, %xmm4, %xmm6
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm10 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
-; AVX1-NEXT:    vpand %xmm10, %xmm6, %xmm6
+; AVX1-NEXT:    vpand %xmm6, %xmm10, %xmm6
 ; AVX1-NEXT:    vpaddb %xmm5, %xmm5, %xmm5
 ; AVX1-NEXT:    vpblendvb %xmm5, %xmm6, %xmm4, %xmm4
 ; AVX1-NEXT:    vpsrlw $1, %xmm4, %xmm6
@@ -624,12 +624,12 @@ define <32 x i8> @var_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y, <32 x i8> %amt)
 ; AVX1-NEXT:    vpaddb %xmm5, %xmm5, %xmm5
 ; AVX1-NEXT:    vpblendvb %xmm5, %xmm6, %xmm4, %xmm4
 ; AVX1-NEXT:    vpsrlw $4, %xmm1, %xmm5
-; AVX1-NEXT:    vpand %xmm8, %xmm5, %xmm5
+; AVX1-NEXT:    vpand %xmm5, %xmm8, %xmm5
 ; AVX1-NEXT:    vpsubb %xmm2, %xmm9, %xmm6
 ; AVX1-NEXT:    vpsllw $5, %xmm6, %xmm6
 ; AVX1-NEXT:    vpblendvb %xmm6, %xmm5, %xmm1, %xmm1
 ; AVX1-NEXT:    vpsrlw $2, %xmm1, %xmm5
-; AVX1-NEXT:    vpand %xmm10, %xmm5, %xmm5
+; AVX1-NEXT:    vpand %xmm5, %xmm10, %xmm5
 ; AVX1-NEXT:    vpaddb %xmm6, %xmm6, %xmm6
 ; AVX1-NEXT:    vpblendvb %xmm6, %xmm5, %xmm1, %xmm1
 ; AVX1-NEXT:    vpsrlw $1, %xmm1, %xmm5
@@ -663,7 +663,7 @@ define <32 x i8> @var_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y, <32 x i8> %amt)
 ; AVX1-NEXT:    vpaddb %xmm6, %xmm6, %xmm6
 ; AVX1-NEXT:    vpblendvb %xmm6, %xmm5, %xmm1, %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm1, %ymm1
-; AVX1-NEXT:    vorps %ymm8, %ymm1, %ymm1
+; AVX1-NEXT:    vorps %ymm1, %ymm8, %ymm1
 ; AVX1-NEXT:    vpxor %xmm4, %xmm4, %xmm4
 ; AVX1-NEXT:    vpcmpeqb %xmm4, %xmm3, %xmm3
 ; AVX1-NEXT:    vpcmpeqb %xmm4, %xmm2, %xmm2
@@ -1477,8 +1477,8 @@ define <32 x i8> @splatvar_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y, <32 x i8> %
 ; AVX1-NEXT:    vpand %xmm6, %xmm1, %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
 ; AVX1-NEXT:    vorps %ymm1, %ymm4, %ymm1
-; AVX1-NEXT:    vpcmpeqb %xmm8, %xmm5, %xmm3
-; AVX1-NEXT:    vpcmpeqb %xmm8, %xmm2, %xmm2
+; AVX1-NEXT:    vpcmpeqb %xmm5, %xmm8, %xmm3
+; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm8, %xmm2
 ; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
 ; AVX1-NEXT:    vandnps %ymm1, %ymm2, %ymm1
 ; AVX1-NEXT:    vandps %ymm2, %ymm0, %ymm0
@@ -2503,14 +2503,53 @@ define <32 x i8> @splatconstant_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y) nounwi
 ; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    retq
 ;
-; AVX512-LABEL: splatconstant_funnnel_v32i8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsrlw $4, %ymm1, %ymm1
-; AVX512-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
-; AVX512-NEXT:    vpsllw $4, %ymm0, %ymm0
-; AVX512-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
-; AVX512-NEXT:    vpor %ymm1, %ymm0, %ymm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: splatconstant_funnnel_v32i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsrlw $4, %ymm1, %ymm1
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
+; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm0
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
+; AVX512F-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: splatconstant_funnnel_v32i8:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm2
+; AVX512VL-NEXT:    vpsrlw $4, %ymm1, %ymm0
+; AVX512VL-NEXT:    vpternlogq $216, {{.*}}(%rip), %ymm2, %ymm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: splatconstant_funnnel_v32i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsrlw $4, %ymm1, %ymm1
+; AVX512BW-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
+; AVX512BW-NEXT:    vpsllw $4, %ymm0, %ymm0
+; AVX512BW-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
+; AVX512BW-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: splatconstant_funnnel_v32i8:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpsrlw $4, %ymm1, %ymm1
+; AVX512VBMI2-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
+; AVX512VBMI2-NEXT:    vpsllw $4, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLBW-LABEL: splatconstant_funnnel_v32i8:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpsllw $4, %ymm0, %ymm2
+; AVX512VLBW-NEXT:    vpsrlw $4, %ymm1, %ymm0
+; AVX512VLBW-NEXT:    vpternlogq $216, {{.*}}(%rip), %ymm2, %ymm0
+; AVX512VLBW-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v32i8:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpsllw $4, %ymm0, %ymm2
+; AVX512VLVBMI2-NEXT:    vpsrlw $4, %ymm1, %ymm0
+; AVX512VLVBMI2-NEXT:    vpternlogq $216, {{.*}}(%rip), %ymm2, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: splatconstant_funnnel_v32i8:
 ; XOPAVX1:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/vector-fshl-512.ll b/llvm/test/CodeGen/X86/vector-fshl-512.ll
index 0e07ddb4b..e45428d8f 100644
--- a/llvm/test/CodeGen/X86/vector-fshl-512.ll
+++ b/llvm/test/CodeGen/X86/vector-fshl-512.ll
@@ -150,64 +150,72 @@ define <16 x i32> @var_funnnel_v16i32(<16 x i32> %x, <16 x i32> %y, <16 x i32> %
 define <32 x i16> @var_funnnel_v32i16(<32 x i16> %x, <32 x i16> %y, <32 x i16> %amt) nounwind {
 ; AVX512F-LABEL: var_funnnel_v32i16:
 ; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm4
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm2, %ymm5
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm6 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512F-NEXT:    vpand %ymm6, %ymm4, %ymm4
-; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm7 = ymm4[0],zero,ymm4[1],zero,ymm4[2],zero,ymm4[3],zero,ymm4[4],zero,ymm4[5],zero,ymm4[6],zero,ymm4[7],zero,ymm4[8],zero,ymm4[9],zero,ymm4[10],zero,ymm4[11],zero,ymm4[12],zero,ymm4[13],zero,ymm4[14],zero,ymm4[15],zero
-; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm8 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
+; AVX512F-NEXT:    vpand %ymm6, %ymm5, %ymm5
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm7 = ymm5[0],zero,ymm5[1],zero,ymm5[2],zero,ymm5[3],zero,ymm5[4],zero,ymm5[5],zero,ymm5[6],zero,ymm5[7],zero,ymm5[8],zero,ymm5[9],zero,ymm5[10],zero,ymm5[11],zero,ymm5[12],zero,ymm5[13],zero,ymm5[14],zero,ymm5[15],zero
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm8 = ymm4[0],zero,ymm4[1],zero,ymm4[2],zero,ymm4[3],zero,ymm4[4],zero,ymm4[5],zero,ymm4[6],zero,ymm4[7],zero,ymm4[8],zero,ymm4[9],zero,ymm4[10],zero,ymm4[11],zero,ymm4[12],zero,ymm4[13],zero,ymm4[14],zero,ymm4[15],zero
 ; AVX512F-NEXT:    vpsllvd %zmm7, %zmm8, %zmm7
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm8 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
-; AVX512F-NEXT:    vpsubw %ymm4, %ymm8, %ymm9
+; AVX512F-NEXT:    vpsubw %ymm5, %ymm8, %ymm9
 ; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm9 = ymm9[0],zero,ymm9[1],zero,ymm9[2],zero,ymm9[3],zero,ymm9[4],zero,ymm9[5],zero,ymm9[6],zero,ymm9[7],zero,ymm9[8],zero,ymm9[9],zero,ymm9[10],zero,ymm9[11],zero,ymm9[12],zero,ymm9[13],zero,ymm9[14],zero,ymm9[15],zero
-; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
-; AVX512F-NEXT:    vpsrlvd %zmm9, %zmm2, %zmm2
-; AVX512F-NEXT:    vpord %zmm2, %zmm7, %zmm2
-; AVX512F-NEXT:    vpmovdw %zmm2, %ymm2
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm3 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
+; AVX512F-NEXT:    vpsrlvd %zmm9, %zmm3, %zmm3
+; AVX512F-NEXT:    vpord %zmm3, %zmm7, %zmm3
+; AVX512F-NEXT:    vpmovdw %zmm3, %ymm3
 ; AVX512F-NEXT:    vpxor %xmm7, %xmm7, %xmm7
-; AVX512F-NEXT:    vpcmpeqw %ymm7, %ymm4, %ymm4
-; AVX512F-NEXT:    vpblendvb %ymm4, %ymm0, %ymm2, %ymm0
-; AVX512F-NEXT:    vpand %ymm6, %ymm5, %ymm2
+; AVX512F-NEXT:    vpcmpeqw %ymm7, %ymm5, %ymm5
+; AVX512F-NEXT:    vpblendvb %ymm5, %ymm4, %ymm3, %ymm3
+; AVX512F-NEXT:    vpand %ymm6, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm4 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
-; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm5 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm5 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
 ; AVX512F-NEXT:    vpsllvd %zmm4, %zmm5, %zmm4
 ; AVX512F-NEXT:    vpsubw %ymm2, %ymm8, %ymm5
 ; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm5 = ymm5[0],zero,ymm5[1],zero,ymm5[2],zero,ymm5[3],zero,ymm5[4],zero,ymm5[5],zero,ymm5[6],zero,ymm5[7],zero,ymm5[8],zero,ymm5[9],zero,ymm5[10],zero,ymm5[11],zero,ymm5[12],zero,ymm5[13],zero,ymm5[14],zero,ymm5[15],zero
-; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm3 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
-; AVX512F-NEXT:    vpsrlvd %zmm5, %zmm3, %zmm3
-; AVX512F-NEXT:    vpord %zmm3, %zmm4, %zmm3
-; AVX512F-NEXT:    vpmovdw %zmm3, %ymm3
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
+; AVX512F-NEXT:    vpsrlvd %zmm5, %zmm1, %zmm1
+; AVX512F-NEXT:    vpord %zmm1, %zmm4, %zmm1
+; AVX512F-NEXT:    vpmovdw %zmm1, %ymm1
 ; AVX512F-NEXT:    vpcmpeqw %ymm7, %ymm2, %ymm2
-; AVX512F-NEXT:    vpblendvb %ymm2, %ymm1, %ymm3, %ymm1
+; AVX512F-NEXT:    vpblendvb %ymm2, %ymm0, %ymm1, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: var_funnnel_v32i16:
 ; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm4
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm2, %ymm5
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm6 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512VL-NEXT:    vpand %ymm6, %ymm4, %ymm4
-; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm7 = ymm4[0],zero,ymm4[1],zero,ymm4[2],zero,ymm4[3],zero,ymm4[4],zero,ymm4[5],zero,ymm4[6],zero,ymm4[7],zero,ymm4[8],zero,ymm4[9],zero,ymm4[10],zero,ymm4[11],zero,ymm4[12],zero,ymm4[13],zero,ymm4[14],zero,ymm4[15],zero
-; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm8 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
+; AVX512VL-NEXT:    vpand %ymm6, %ymm5, %ymm5
+; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm7 = ymm5[0],zero,ymm5[1],zero,ymm5[2],zero,ymm5[3],zero,ymm5[4],zero,ymm5[5],zero,ymm5[6],zero,ymm5[7],zero,ymm5[8],zero,ymm5[9],zero,ymm5[10],zero,ymm5[11],zero,ymm5[12],zero,ymm5[13],zero,ymm5[14],zero,ymm5[15],zero
+; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm8 = ymm4[0],zero,ymm4[1],zero,ymm4[2],zero,ymm4[3],zero,ymm4[4],zero,ymm4[5],zero,ymm4[6],zero,ymm4[7],zero,ymm4[8],zero,ymm4[9],zero,ymm4[10],zero,ymm4[11],zero,ymm4[12],zero,ymm4[13],zero,ymm4[14],zero,ymm4[15],zero
 ; AVX512VL-NEXT:    vpsllvd %zmm7, %zmm8, %zmm7
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm8 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
-; AVX512VL-NEXT:    vpsubw %ymm4, %ymm8, %ymm9
+; AVX512VL-NEXT:    vpsubw %ymm5, %ymm8, %ymm9
 ; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm9 = ymm9[0],zero,ymm9[1],zero,ymm9[2],zero,ymm9[3],zero,ymm9[4],zero,ymm9[5],zero,ymm9[6],zero,ymm9[7],zero,ymm9[8],zero,ymm9[9],zero,ymm9[10],zero,ymm9[11],zero,ymm9[12],zero,ymm9[13],zero,ymm9[14],zero,ymm9[15],zero
-; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
-; AVX512VL-NEXT:    vpsrlvd %zmm9, %zmm2, %zmm2
-; AVX512VL-NEXT:    vpord %zmm2, %zmm7, %zmm2
-; AVX512VL-NEXT:    vpmovdw %zmm2, %ymm2
+; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm3 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
+; AVX512VL-NEXT:    vpsrlvd %zmm9, %zmm3, %zmm3
+; AVX512VL-NEXT:    vpord %zmm3, %zmm7, %zmm3
+; AVX512VL-NEXT:    vpmovdw %zmm3, %ymm3
 ; AVX512VL-NEXT:    vpxor %xmm7, %xmm7, %xmm7
-; AVX512VL-NEXT:    vpcmpeqw %ymm7, %ymm4, %ymm4
-; AVX512VL-NEXT:    vpblendvb %ymm4, %ymm0, %ymm2, %ymm0
-; AVX512VL-NEXT:    vpand %ymm6, %ymm5, %ymm2
+; AVX512VL-NEXT:    vpcmpeqw %ymm7, %ymm5, %ymm5
+; AVX512VL-NEXT:    vpblendvb %ymm5, %ymm4, %ymm3, %ymm3
+; AVX512VL-NEXT:    vpand %ymm6, %ymm2, %ymm2
 ; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm4 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
-; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm5 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
+; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm5 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
 ; AVX512VL-NEXT:    vpsllvd %zmm4, %zmm5, %zmm4
 ; AVX512VL-NEXT:    vpsubw %ymm2, %ymm8, %ymm5
 ; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm5 = ymm5[0],zero,ymm5[1],zero,ymm5[2],zero,ymm5[3],zero,ymm5[4],zero,ymm5[5],zero,ymm5[6],zero,ymm5[7],zero,ymm5[8],zero,ymm5[9],zero,ymm5[10],zero,ymm5[11],zero,ymm5[12],zero,ymm5[13],zero,ymm5[14],zero,ymm5[15],zero
-; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm3 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
-; AVX512VL-NEXT:    vpsrlvd %zmm5, %zmm3, %zmm3
-; AVX512VL-NEXT:    vpord %zmm3, %zmm4, %zmm3
-; AVX512VL-NEXT:    vpmovdw %zmm3, %ymm3
+; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
+; AVX512VL-NEXT:    vpsrlvd %zmm5, %zmm1, %zmm1
+; AVX512VL-NEXT:    vpord %zmm1, %zmm4, %zmm1
+; AVX512VL-NEXT:    vpmovdw %zmm1, %ymm1
 ; AVX512VL-NEXT:    vpcmpeqw %ymm7, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm1, %ymm3, %ymm1
+; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm0, %ymm1, %ymm0
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: var_funnnel_v32i16:
@@ -252,81 +260,88 @@ define <32 x i16> @var_funnnel_v32i16(<32 x i16> %x, <32 x i16> %y, <32 x i16> %
 define <64 x i8> @var_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> %amt) nounwind {
 ; AVX512F-LABEL: var_funnnel_v64i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm7
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm6 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX512F-NEXT:    vpand %ymm6, %ymm7, %ymm8
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm7 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; AVX512F-NEXT:    vpand %ymm7, %ymm4, %ymm9
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm8
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm2, %ymm6
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
+; AVX512F-NEXT:    vpsllw $4, %ymm3, %ymm5
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm4 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
+; AVX512F-NEXT:    vpand %ymm4, %ymm5, %ymm7
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm5 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; AVX512F-NEXT:    vpand %ymm5, %ymm6, %ymm9
 ; AVX512F-NEXT:    vpsllw $5, %ymm9, %ymm10
-; AVX512F-NEXT:    vpblendvb %ymm10, %ymm8, %ymm0, %ymm8
-; AVX512F-NEXT:    vpsllw $2, %ymm8, %ymm11
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm4 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
-; AVX512F-NEXT:    vpand %ymm4, %ymm11, %ymm11
+; AVX512F-NEXT:    vpblendvb %ymm10, %ymm7, %ymm3, %ymm7
+; AVX512F-NEXT:    vpsllw $2, %ymm7, %ymm11
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm6 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
+; AVX512F-NEXT:    vpand %ymm6, %ymm11, %ymm11
 ; AVX512F-NEXT:    vpaddb %ymm10, %ymm10, %ymm10
-; AVX512F-NEXT:    vpblendvb %ymm10, %ymm11, %ymm8, %ymm8
-; AVX512F-NEXT:    vpaddb %ymm8, %ymm8, %ymm11
+; AVX512F-NEXT:    vpblendvb %ymm10, %ymm11, %ymm7, %ymm7
+; AVX512F-NEXT:    vpaddb %ymm7, %ymm7, %ymm11
 ; AVX512F-NEXT:    vpaddb %ymm10, %ymm10, %ymm10
-; AVX512F-NEXT:    vpblendvb %ymm10, %ymm11, %ymm8, %ymm10
-; AVX512F-NEXT:    vpsrlw $4, %ymm2, %ymm11
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm8 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512F-NEXT:    vpand %ymm8, %ymm11, %ymm11
+; AVX512F-NEXT:    vpblendvb %ymm10, %ymm11, %ymm7, %ymm10
+; AVX512F-NEXT:    vpsrlw $4, %ymm8, %ymm11
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm7 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX512F-NEXT:    vpand %ymm7, %ymm11, %ymm11
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm12 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
 ; AVX512F-NEXT:    vpsubb %ymm9, %ymm12, %ymm13
 ; AVX512F-NEXT:    vpsllw $5, %ymm13, %ymm13
-; AVX512F-NEXT:    vpblendvb %ymm13, %ymm11, %ymm2, %ymm2
-; AVX512F-NEXT:    vpsrlw $2, %ymm2, %ymm11
+; AVX512F-NEXT:    vpblendvb %ymm13, %ymm11, %ymm8, %ymm8
+; AVX512F-NEXT:    vpsrlw $2, %ymm8, %ymm11
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm14 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
 ; AVX512F-NEXT:    vpand %ymm14, %ymm11, %ymm11
 ; AVX512F-NEXT:    vpaddb %ymm13, %ymm13, %ymm13
-; AVX512F-NEXT:    vpblendvb %ymm13, %ymm11, %ymm2, %ymm2
-; AVX512F-NEXT:    vpsrlw $1, %ymm2, %ymm11
+; AVX512F-NEXT:    vpblendvb %ymm13, %ymm11, %ymm8, %ymm8
+; AVX512F-NEXT:    vpsrlw $1, %ymm8, %ymm11
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm15 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
 ; AVX512F-NEXT:    vpand %ymm15, %ymm11, %ymm11
 ; AVX512F-NEXT:    vpaddb %ymm13, %ymm13, %ymm13
-; AVX512F-NEXT:    vpblendvb %ymm13, %ymm11, %ymm2, %ymm2
-; AVX512F-NEXT:    vpor %ymm2, %ymm10, %ymm2
+; AVX512F-NEXT:    vpblendvb %ymm13, %ymm11, %ymm8, %ymm8
+; AVX512F-NEXT:    vpor %ymm8, %ymm10, %ymm8
 ; AVX512F-NEXT:    vpxor %xmm10, %xmm10, %xmm10
 ; AVX512F-NEXT:    vpcmpeqb %ymm10, %ymm9, %ymm9
-; AVX512F-NEXT:    vpblendvb %ymm9, %ymm0, %ymm2, %ymm0
-; AVX512F-NEXT:    vpsllw $4, %ymm1, %ymm2
-; AVX512F-NEXT:    vpand %ymm6, %ymm2, %ymm2
+; AVX512F-NEXT:    vpblendvb %ymm9, %ymm3, %ymm8, %ymm3
+; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm8
+; AVX512F-NEXT:    vpand %ymm4, %ymm8, %ymm4
+; AVX512F-NEXT:    vpand %ymm5, %ymm2, %ymm2
+; AVX512F-NEXT:    vpsllw $5, %ymm2, %ymm5
+; AVX512F-NEXT:    vpblendvb %ymm5, %ymm4, %ymm0, %ymm4
+; AVX512F-NEXT:    vpsllw $2, %ymm4, %ymm8
+; AVX512F-NEXT:    vpand %ymm6, %ymm8, %ymm6
+; AVX512F-NEXT:    vpaddb %ymm5, %ymm5, %ymm5
+; AVX512F-NEXT:    vpblendvb %ymm5, %ymm6, %ymm4, %ymm4
+; AVX512F-NEXT:    vpaddb %ymm4, %ymm4, %ymm6
+; AVX512F-NEXT:    vpaddb %ymm5, %ymm5, %ymm5
+; AVX512F-NEXT:    vpblendvb %ymm5, %ymm6, %ymm4, %ymm4
+; AVX512F-NEXT:    vpsrlw $4, %ymm1, %ymm5
 ; AVX512F-NEXT:    vpand %ymm7, %ymm5, %ymm5
-; AVX512F-NEXT:    vpsllw $5, %ymm5, %ymm6
-; AVX512F-NEXT:    vpblendvb %ymm6, %ymm2, %ymm1, %ymm2
-; AVX512F-NEXT:    vpsllw $2, %ymm2, %ymm7
-; AVX512F-NEXT:    vpand %ymm4, %ymm7, %ymm4
-; AVX512F-NEXT:    vpaddb %ymm6, %ymm6, %ymm6
-; AVX512F-NEXT:    vpblendvb %ymm6, %ymm4, %ymm2, %ymm2
-; AVX512F-NEXT:    vpaddb %ymm2, %ymm2, %ymm4
-; AVX512F-NEXT:    vpaddb %ymm6, %ymm6, %ymm6
-; AVX512F-NEXT:    vpblendvb %ymm6, %ymm4, %ymm2, %ymm2
-; AVX512F-NEXT:    vpsrlw $4, %ymm3, %ymm4
-; AVX512F-NEXT:    vpand %ymm8, %ymm4, %ymm4
-; AVX512F-NEXT:    vpsubb %ymm5, %ymm12, %ymm6
+; AVX512F-NEXT:    vpsubb %ymm2, %ymm12, %ymm6
 ; AVX512F-NEXT:    vpsllw $5, %ymm6, %ymm6
-; AVX512F-NEXT:    vpblendvb %ymm6, %ymm4, %ymm3, %ymm3
-; AVX512F-NEXT:    vpsrlw $2, %ymm3, %ymm4
-; AVX512F-NEXT:    vpand %ymm14, %ymm4, %ymm4
+; AVX512F-NEXT:    vpblendvb %ymm6, %ymm5, %ymm1, %ymm1
+; AVX512F-NEXT:    vpsrlw $2, %ymm1, %ymm5
+; AVX512F-NEXT:    vpand %ymm5, %ymm14, %ymm5
 ; AVX512F-NEXT:    vpaddb %ymm6, %ymm6, %ymm6
-; AVX512F-NEXT:    vpblendvb %ymm6, %ymm4, %ymm3, %ymm3
-; AVX512F-NEXT:    vpsrlw $1, %ymm3, %ymm4
-; AVX512F-NEXT:    vpand %ymm15, %ymm4, %ymm4
+; AVX512F-NEXT:    vpblendvb %ymm6, %ymm5, %ymm1, %ymm1
+; AVX512F-NEXT:    vpsrlw $1, %ymm1, %ymm5
+; AVX512F-NEXT:    vpand %ymm5, %ymm15, %ymm5
 ; AVX512F-NEXT:    vpaddb %ymm6, %ymm6, %ymm6
-; AVX512F-NEXT:    vpblendvb %ymm6, %ymm4, %ymm3, %ymm3
-; AVX512F-NEXT:    vpor %ymm3, %ymm2, %ymm2
-; AVX512F-NEXT:    vpcmpeqb %ymm10, %ymm5, %ymm3
-; AVX512F-NEXT:    vpblendvb %ymm3, %ymm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vpblendvb %ymm6, %ymm5, %ymm1, %ymm1
+; AVX512F-NEXT:    vpor %ymm1, %ymm4, %ymm1
+; AVX512F-NEXT:    vpcmpeqb %ymm2, %ymm10, %ymm2
+; AVX512F-NEXT:    vpblendvb %ymm2, %ymm0, %ymm1, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: var_funnnel_v64i8:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm6
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm1, %ymm4
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm2, %ymm5
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
+; AVX512VL-NEXT:    vpsllw $4, %ymm3, %ymm6
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm7 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
 ; AVX512VL-NEXT:    vpand %ymm7, %ymm6, %ymm6
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm8 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; AVX512VL-NEXT:    vpand %ymm8, %ymm4, %ymm4
-; AVX512VL-NEXT:    vpsllw $5, %ymm4, %ymm9
-; AVX512VL-NEXT:    vpblendvb %ymm9, %ymm6, %ymm0, %ymm6
+; AVX512VL-NEXT:    vpand %ymm5, %ymm8, %ymm5
+; AVX512VL-NEXT:    vpsllw $5, %ymm5, %ymm9
+; AVX512VL-NEXT:    vpblendvb %ymm9, %ymm6, %ymm3, %ymm6
 ; AVX512VL-NEXT:    vpsllw $2, %ymm6, %ymm10
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm11 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
 ; AVX512VL-NEXT:    vpand %ymm11, %ymm10, %ymm10
@@ -335,55 +350,56 @@ define <64 x i8> @var_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> %amt)
 ; AVX512VL-NEXT:    vpaddb %ymm6, %ymm6, %ymm10
 ; AVX512VL-NEXT:    vpaddb %ymm9, %ymm9, %ymm9
 ; AVX512VL-NEXT:    vpblendvb %ymm9, %ymm10, %ymm6, %ymm6
-; AVX512VL-NEXT:    vpsrlw $4, %ymm2, %ymm9
+; AVX512VL-NEXT:    vpsrlw $4, %ymm4, %ymm9
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm10 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
 ; AVX512VL-NEXT:    vpand %ymm10, %ymm9, %ymm9
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm12 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; AVX512VL-NEXT:    vpsubb %ymm4, %ymm12, %ymm13
+; AVX512VL-NEXT:    vpsubb %ymm5, %ymm12, %ymm13
 ; AVX512VL-NEXT:    vpsllw $5, %ymm13, %ymm13
-; AVX512VL-NEXT:    vpblendvb %ymm13, %ymm9, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpsrlw $2, %ymm2, %ymm9
+; AVX512VL-NEXT:    vpblendvb %ymm13, %ymm9, %ymm4, %ymm4
+; AVX512VL-NEXT:    vpsrlw $2, %ymm4, %ymm9
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm14 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
 ; AVX512VL-NEXT:    vpand %ymm14, %ymm9, %ymm9
 ; AVX512VL-NEXT:    vpaddb %ymm13, %ymm13, %ymm13
-; AVX512VL-NEXT:    vpblendvb %ymm13, %ymm9, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpsrlw $1, %ymm2, %ymm9
+; AVX512VL-NEXT:    vpblendvb %ymm13, %ymm9, %ymm4, %ymm4
+; AVX512VL-NEXT:    vpsrlw $1, %ymm4, %ymm9
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm15 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
 ; AVX512VL-NEXT:    vpand %ymm15, %ymm9, %ymm9
 ; AVX512VL-NEXT:    vpaddb %ymm13, %ymm13, %ymm13
-; AVX512VL-NEXT:    vpblendvb %ymm13, %ymm9, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpor %ymm2, %ymm6, %ymm2
+; AVX512VL-NEXT:    vpblendvb %ymm13, %ymm9, %ymm4, %ymm4
+; AVX512VL-NEXT:    vpor %ymm4, %ymm6, %ymm4
 ; AVX512VL-NEXT:    vpxor %xmm6, %xmm6, %xmm6
-; AVX512VL-NEXT:    vpcmpeqb %ymm6, %ymm4, %ymm4
-; AVX512VL-NEXT:    vpblendvb %ymm4, %ymm0, %ymm2, %ymm0
-; AVX512VL-NEXT:    vpsllw $4, %ymm1, %ymm2
-; AVX512VL-NEXT:    vpand %ymm7, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpand %ymm8, %ymm5, %ymm4
-; AVX512VL-NEXT:    vpsllw $5, %ymm4, %ymm5
-; AVX512VL-NEXT:    vpblendvb %ymm5, %ymm2, %ymm1, %ymm2
-; AVX512VL-NEXT:    vpsllw $2, %ymm2, %ymm7
-; AVX512VL-NEXT:    vpand %ymm11, %ymm7, %ymm7
+; AVX512VL-NEXT:    vpcmpeqb %ymm6, %ymm5, %ymm5
+; AVX512VL-NEXT:    vpblendvb %ymm5, %ymm3, %ymm4, %ymm3
+; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm4
+; AVX512VL-NEXT:    vpand %ymm7, %ymm4, %ymm4
+; AVX512VL-NEXT:    vpand %ymm2, %ymm8, %ymm2
+; AVX512VL-NEXT:    vpsllw $5, %ymm2, %ymm5
+; AVX512VL-NEXT:    vpblendvb %ymm5, %ymm4, %ymm0, %ymm4
+; AVX512VL-NEXT:    vpsllw $2, %ymm4, %ymm7
+; AVX512VL-NEXT:    vpand %ymm7, %ymm11, %ymm7
 ; AVX512VL-NEXT:    vpaddb %ymm5, %ymm5, %ymm5
-; AVX512VL-NEXT:    vpblendvb %ymm5, %ymm7, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpaddb %ymm2, %ymm2, %ymm7
+; AVX512VL-NEXT:    vpblendvb %ymm5, %ymm7, %ymm4, %ymm4
+; AVX512VL-NEXT:    vpaddb %ymm4, %ymm4, %ymm7
 ; AVX512VL-NEXT:    vpaddb %ymm5, %ymm5, %ymm5
-; AVX512VL-NEXT:    vpblendvb %ymm5, %ymm7, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpsrlw $4, %ymm3, %ymm5
-; AVX512VL-NEXT:    vpand %ymm10, %ymm5, %ymm5
-; AVX512VL-NEXT:    vpsubb %ymm4, %ymm12, %ymm7
+; AVX512VL-NEXT:    vpblendvb %ymm5, %ymm7, %ymm4, %ymm4
+; AVX512VL-NEXT:    vpsrlw $4, %ymm1, %ymm5
+; AVX512VL-NEXT:    vpand %ymm5, %ymm10, %ymm5
+; AVX512VL-NEXT:    vpsubb %ymm2, %ymm12, %ymm7
 ; AVX512VL-NEXT:    vpsllw $5, %ymm7, %ymm7
-; AVX512VL-NEXT:    vpblendvb %ymm7, %ymm5, %ymm3, %ymm3
-; AVX512VL-NEXT:    vpsrlw $2, %ymm3, %ymm5
-; AVX512VL-NEXT:    vpand %ymm14, %ymm5, %ymm5
+; AVX512VL-NEXT:    vpblendvb %ymm7, %ymm5, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpsrlw $2, %ymm1, %ymm5
+; AVX512VL-NEXT:    vpand %ymm5, %ymm14, %ymm5
 ; AVX512VL-NEXT:    vpaddb %ymm7, %ymm7, %ymm7
-; AVX512VL-NEXT:    vpblendvb %ymm7, %ymm5, %ymm3, %ymm3
-; AVX512VL-NEXT:    vpsrlw $1, %ymm3, %ymm5
-; AVX512VL-NEXT:    vpand %ymm15, %ymm5, %ymm5
+; AVX512VL-NEXT:    vpblendvb %ymm7, %ymm5, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpsrlw $1, %ymm1, %ymm5
+; AVX512VL-NEXT:    vpand %ymm5, %ymm15, %ymm5
 ; AVX512VL-NEXT:    vpaddb %ymm7, %ymm7, %ymm7
-; AVX512VL-NEXT:    vpblendvb %ymm7, %ymm5, %ymm3, %ymm3
-; AVX512VL-NEXT:    vpor %ymm3, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpcmpeqb %ymm6, %ymm4, %ymm3
-; AVX512VL-NEXT:    vpblendvb %ymm3, %ymm1, %ymm2, %ymm1
+; AVX512VL-NEXT:    vpblendvb %ymm7, %ymm5, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpor %ymm1, %ymm4, %ymm1
+; AVX512VL-NEXT:    vpcmpeqb %ymm6, %ymm2, %ymm2
+; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm0, %ymm1, %ymm0
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: var_funnnel_v64i8:
@@ -706,42 +722,48 @@ define <16 x i32> @splatvar_funnnel_v16i32(<16 x i32> %x, <16 x i32> %y, <16 x i
 define <32 x i16> @splatvar_funnnel_v32i16(<32 x i16> %x, <32 x i16> %y, <32 x i16> %amt) nounwind {
 ; AVX512F-LABEL: splatvar_funnnel_v32i16:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpbroadcastw %xmm4, %ymm4
-; AVX512F-NEXT:    vpand {{.*}}(%rip), %ymm4, %ymm4
-; AVX512F-NEXT:    vpmovzxwq {{.*#+}} xmm5 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
-; AVX512F-NEXT:    vpsllw %xmm5, %ymm0, %ymm6
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm4
+; AVX512F-NEXT:    vpbroadcastw %xmm2, %ymm2
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
+; AVX512F-NEXT:    vpmovzxwq {{.*#+}} xmm5 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
+; AVX512F-NEXT:    vpsllw %xmm5, %ymm4, %ymm6
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm7 = [16,16,16,16,16,16,16,16]
-; AVX512F-NEXT:    vpsubw %xmm4, %xmm7, %xmm7
+; AVX512F-NEXT:    vpsubw %xmm2, %xmm7, %xmm7
 ; AVX512F-NEXT:    vpmovzxwq {{.*#+}} xmm7 = xmm7[0],zero,zero,zero,xmm7[1],zero,zero,zero
-; AVX512F-NEXT:    vpsrlw %xmm7, %ymm2, %ymm2
-; AVX512F-NEXT:    vpor %ymm2, %ymm6, %ymm2
-; AVX512F-NEXT:    vpxor %xmm6, %xmm6, %xmm6
-; AVX512F-NEXT:    vpcmpeqw %ymm6, %ymm4, %ymm4
-; AVX512F-NEXT:    vpblendvb %ymm4, %ymm0, %ymm2, %ymm0
-; AVX512F-NEXT:    vpsllw %xmm5, %ymm1, %ymm2
 ; AVX512F-NEXT:    vpsrlw %xmm7, %ymm3, %ymm3
-; AVX512F-NEXT:    vpor %ymm3, %ymm2, %ymm2
-; AVX512F-NEXT:    vpblendvb %ymm4, %ymm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vpor %ymm3, %ymm6, %ymm3
+; AVX512F-NEXT:    vpxor %xmm6, %xmm6, %xmm6
+; AVX512F-NEXT:    vpcmpeqw %ymm6, %ymm2, %ymm2
+; AVX512F-NEXT:    vpblendvb %ymm2, %ymm4, %ymm3, %ymm3
+; AVX512F-NEXT:    vpsllw %xmm5, %ymm0, %ymm4
+; AVX512F-NEXT:    vpsrlw %xmm7, %ymm1, %ymm1
+; AVX512F-NEXT:    vpor %ymm1, %ymm4, %ymm1
+; AVX512F-NEXT:    vpblendvb %ymm2, %ymm0, %ymm1, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatvar_funnnel_v32i16:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpbroadcastw %xmm4, %ymm4
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm4, %ymm4
-; AVX512VL-NEXT:    vpmovzxwq {{.*#+}} xmm5 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
-; AVX512VL-NEXT:    vpsllw %xmm5, %ymm0, %ymm6
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm4
+; AVX512VL-NEXT:    vpbroadcastw %xmm2, %ymm2
+; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
+; AVX512VL-NEXT:    vpmovzxwq {{.*#+}} xmm5 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
+; AVX512VL-NEXT:    vpsllw %xmm5, %ymm4, %ymm6
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm7 = [16,16,16,16,16,16,16,16]
-; AVX512VL-NEXT:    vpsubw %xmm4, %xmm7, %xmm7
+; AVX512VL-NEXT:    vpsubw %xmm2, %xmm7, %xmm7
 ; AVX512VL-NEXT:    vpmovzxwq {{.*#+}} xmm7 = xmm7[0],zero,zero,zero,xmm7[1],zero,zero,zero
-; AVX512VL-NEXT:    vpsrlw %xmm7, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpor %ymm2, %ymm6, %ymm2
-; AVX512VL-NEXT:    vpxor %xmm6, %xmm6, %xmm6
-; AVX512VL-NEXT:    vpcmpeqw %ymm6, %ymm4, %ymm4
-; AVX512VL-NEXT:    vpblendvb %ymm4, %ymm0, %ymm2, %ymm0
-; AVX512VL-NEXT:    vpsllw %xmm5, %ymm1, %ymm2
 ; AVX512VL-NEXT:    vpsrlw %xmm7, %ymm3, %ymm3
-; AVX512VL-NEXT:    vpor %ymm3, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpblendvb %ymm4, %ymm1, %ymm2, %ymm1
+; AVX512VL-NEXT:    vpor %ymm3, %ymm6, %ymm3
+; AVX512VL-NEXT:    vpxor %xmm6, %xmm6, %xmm6
+; AVX512VL-NEXT:    vpcmpeqw %ymm6, %ymm2, %ymm2
+; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm4, %ymm3, %ymm3
+; AVX512VL-NEXT:    vpsllw %xmm5, %ymm0, %ymm4
+; AVX512VL-NEXT:    vpsrlw %xmm7, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpor %ymm1, %ymm4, %ymm1
+; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm0, %ymm1, %ymm0
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatvar_funnnel_v32i16:
@@ -795,62 +817,68 @@ define <32 x i16> @splatvar_funnnel_v32i16(<32 x i16> %x, <32 x i16> %y, <32 x i
 define <64 x i8> @splatvar_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> %amt) nounwind {
 ; AVX512F-LABEL: splatvar_funnnel_v64i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpbroadcastb %xmm4, %ymm4
-; AVX512F-NEXT:    vpand {{.*}}(%rip), %ymm4, %ymm4
-; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm5 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512F-NEXT:    vpsllw %xmm5, %ymm0, %ymm6
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm9
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm4
+; AVX512F-NEXT:    vpbroadcastb %xmm2, %ymm2
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
+; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm5 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512F-NEXT:    vpsllw %xmm5, %ymm4, %ymm6
 ; AVX512F-NEXT:    vpcmpeqd %xmm8, %xmm8, %xmm8
 ; AVX512F-NEXT:    vpsllw %xmm5, %xmm8, %xmm7
 ; AVX512F-NEXT:    vpbroadcastb %xmm7, %ymm7
-; AVX512F-NEXT:    vpand %ymm7, %ymm6, %ymm9
-; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm6 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; AVX512F-NEXT:    vpsubb %xmm4, %xmm6, %xmm6
-; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm6 = xmm6[0],zero,zero,zero,zero,zero,zero,zero,xmm6[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512F-NEXT:    vpsrlw %xmm6, %ymm2, %ymm10
-; AVX512F-NEXT:    vpsrlw %xmm6, %xmm8, %xmm2
-; AVX512F-NEXT:    vpsrlw $8, %xmm2, %xmm2
-; AVX512F-NEXT:    vpbroadcastb %xmm2, %ymm2
-; AVX512F-NEXT:    vpand %ymm2, %ymm10, %ymm8
-; AVX512F-NEXT:    vpor %ymm8, %ymm9, %ymm8
+; AVX512F-NEXT:    vpand %ymm7, %ymm6, %ymm10
+; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm3 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512F-NEXT:    vpsubb %xmm2, %xmm3, %xmm3
+; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512F-NEXT:    vpsrlw %xmm3, %ymm9, %ymm9
+; AVX512F-NEXT:    vpsrlw %xmm3, %xmm8, %xmm6
+; AVX512F-NEXT:    vpsrlw $8, %xmm6, %xmm6
+; AVX512F-NEXT:    vpbroadcastb %xmm6, %ymm6
+; AVX512F-NEXT:    vpand %ymm6, %ymm9, %ymm8
+; AVX512F-NEXT:    vpor %ymm8, %ymm10, %ymm8
 ; AVX512F-NEXT:    vpxor %xmm9, %xmm9, %xmm9
-; AVX512F-NEXT:    vpcmpeqb %ymm9, %ymm4, %ymm4
-; AVX512F-NEXT:    vpblendvb %ymm4, %ymm0, %ymm8, %ymm0
-; AVX512F-NEXT:    vpsllw %xmm5, %ymm1, %ymm5
+; AVX512F-NEXT:    vpcmpeqb %ymm2, %ymm9, %ymm2
+; AVX512F-NEXT:    vpblendvb %ymm2, %ymm4, %ymm8, %ymm4
+; AVX512F-NEXT:    vpsllw %xmm5, %ymm0, %ymm5
 ; AVX512F-NEXT:    vpand %ymm7, %ymm5, %ymm5
-; AVX512F-NEXT:    vpsrlw %xmm6, %ymm3, %ymm3
-; AVX512F-NEXT:    vpand %ymm2, %ymm3, %ymm2
-; AVX512F-NEXT:    vpor %ymm2, %ymm5, %ymm2
-; AVX512F-NEXT:    vpblendvb %ymm4, %ymm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vpsrlw %xmm3, %ymm1, %ymm1
+; AVX512F-NEXT:    vpand %ymm6, %ymm1, %ymm1
+; AVX512F-NEXT:    vpor %ymm1, %ymm5, %ymm1
+; AVX512F-NEXT:    vpblendvb %ymm2, %ymm0, %ymm1, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm4, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatvar_funnnel_v64i8:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpbroadcastb %xmm4, %ymm4
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm4, %ymm4
-; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm5 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512VL-NEXT:    vpsllw %xmm5, %ymm0, %ymm6
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm1, %ymm9
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm4
+; AVX512VL-NEXT:    vpbroadcastb %xmm2, %ymm2
+; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
+; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm5 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VL-NEXT:    vpsllw %xmm5, %ymm4, %ymm6
 ; AVX512VL-NEXT:    vpcmpeqd %xmm8, %xmm8, %xmm8
 ; AVX512VL-NEXT:    vpsllw %xmm5, %xmm8, %xmm7
 ; AVX512VL-NEXT:    vpbroadcastb %xmm7, %ymm7
-; AVX512VL-NEXT:    vpand %ymm7, %ymm6, %ymm9
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm6 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; AVX512VL-NEXT:    vpsubb %xmm4, %xmm6, %xmm6
-; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm6 = xmm6[0],zero,zero,zero,zero,zero,zero,zero,xmm6[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512VL-NEXT:    vpsrlw %xmm6, %ymm2, %ymm10
-; AVX512VL-NEXT:    vpsrlw %xmm6, %xmm8, %xmm2
-; AVX512VL-NEXT:    vpsrlw $8, %xmm2, %xmm2
-; AVX512VL-NEXT:    vpbroadcastb %xmm2, %ymm2
-; AVX512VL-NEXT:    vpand %ymm2, %ymm10, %ymm8
-; AVX512VL-NEXT:    vpor %ymm8, %ymm9, %ymm8
+; AVX512VL-NEXT:    vpand %ymm7, %ymm6, %ymm10
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm3 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VL-NEXT:    vpsubb %xmm2, %xmm3, %xmm3
+; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VL-NEXT:    vpsrlw %xmm3, %ymm9, %ymm9
+; AVX512VL-NEXT:    vpsrlw %xmm3, %xmm8, %xmm6
+; AVX512VL-NEXT:    vpsrlw $8, %xmm6, %xmm6
+; AVX512VL-NEXT:    vpbroadcastb %xmm6, %ymm6
+; AVX512VL-NEXT:    vpand %ymm6, %ymm9, %ymm8
+; AVX512VL-NEXT:    vpor %ymm8, %ymm10, %ymm8
 ; AVX512VL-NEXT:    vpxor %xmm9, %xmm9, %xmm9
-; AVX512VL-NEXT:    vpcmpeqb %ymm9, %ymm4, %ymm4
-; AVX512VL-NEXT:    vpblendvb %ymm4, %ymm0, %ymm8, %ymm0
-; AVX512VL-NEXT:    vpsllw %xmm5, %ymm1, %ymm5
+; AVX512VL-NEXT:    vpcmpeqb %ymm2, %ymm9, %ymm2
+; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm4, %ymm8, %ymm4
+; AVX512VL-NEXT:    vpsllw %xmm5, %ymm0, %ymm5
 ; AVX512VL-NEXT:    vpand %ymm7, %ymm5, %ymm5
-; AVX512VL-NEXT:    vpsrlw %xmm6, %ymm3, %ymm3
-; AVX512VL-NEXT:    vpand %ymm2, %ymm3, %ymm2
-; AVX512VL-NEXT:    vpor %ymm2, %ymm5, %ymm2
-; AVX512VL-NEXT:    vpblendvb %ymm4, %ymm1, %ymm2, %ymm1
+; AVX512VL-NEXT:    vpsrlw %xmm3, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpand %ymm6, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpor %ymm1, %ymm5, %ymm1
+; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm0, %ymm1, %ymm0
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm4, %zmm0, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatvar_funnnel_v64i8:
@@ -1044,32 +1072,38 @@ define <16 x i32> @constant_funnnel_v16i32(<16 x i32> %x, <16 x i32> %y) nounwin
 define <32 x i16> @constant_funnnel_v32i16(<32 x i16> %x, <32 x i16> %y) nounwind {
 ; AVX512F-LABEL: constant_funnnel_v32i16:
 ; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm4 = <u,2,4,8,16,32,64,128,256,512,1024,2048,4096,8192,16384,32768>
-; AVX512F-NEXT:    vpmulhuw %ymm4, %ymm2, %ymm2
-; AVX512F-NEXT:    vpmullw %ymm4, %ymm0, %ymm5
-; AVX512F-NEXT:    vpor %ymm2, %ymm5, %ymm2
-; AVX512F-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm2[1,2,3,4,5,6,7],ymm0[8],ymm2[9,10,11,12,13,14,15]
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
-; AVX512F-NEXT:    vpmulhuw %ymm4, %ymm3, %ymm2
-; AVX512F-NEXT:    vpmullw %ymm4, %ymm1, %ymm3
-; AVX512F-NEXT:    vpor %ymm2, %ymm3, %ymm2
-; AVX512F-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm2[1,2,3,4,5,6,7],ymm1[8],ymm2[9,10,11,12,13,14,15]
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
+; AVX512F-NEXT:    vpmulhuw %ymm4, %ymm3, %ymm3
+; AVX512F-NEXT:    vpmullw %ymm4, %ymm2, %ymm5
+; AVX512F-NEXT:    vpor %ymm3, %ymm5, %ymm3
+; AVX512F-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0],ymm3[1,2,3,4,5,6,7],ymm2[8],ymm3[9,10,11,12,13,14,15]
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4,5,6,7]
+; AVX512F-NEXT:    vpmulhuw %ymm4, %ymm1, %ymm1
+; AVX512F-NEXT:    vpmullw %ymm4, %ymm0, %ymm3
+; AVX512F-NEXT:    vpor %ymm1, %ymm3, %ymm1
+; AVX512F-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3,4,5,6,7],ymm0[8],ymm1[9,10,11,12,13,14,15]
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: constant_funnnel_v32i16:
 ; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm4 = <u,2,4,8,16,32,64,128,256,512,1024,2048,4096,8192,16384,32768>
-; AVX512VL-NEXT:    vpmulhuw %ymm4, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpmullw %ymm4, %ymm0, %ymm5
-; AVX512VL-NEXT:    vpor %ymm2, %ymm5, %ymm2
-; AVX512VL-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm2[1,2,3,4,5,6,7],ymm0[8],ymm2[9,10,11,12,13,14,15]
-; AVX512VL-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
-; AVX512VL-NEXT:    vpmulhuw %ymm4, %ymm3, %ymm2
-; AVX512VL-NEXT:    vpmullw %ymm4, %ymm1, %ymm3
-; AVX512VL-NEXT:    vpor %ymm2, %ymm3, %ymm2
-; AVX512VL-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm2[1,2,3,4,5,6,7],ymm1[8],ymm2[9,10,11,12,13,14,15]
-; AVX512VL-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
+; AVX512VL-NEXT:    vpmulhuw %ymm4, %ymm3, %ymm3
+; AVX512VL-NEXT:    vpmullw %ymm4, %ymm2, %ymm5
+; AVX512VL-NEXT:    vpor %ymm3, %ymm5, %ymm3
+; AVX512VL-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0],ymm3[1,2,3,4,5,6,7],ymm2[8],ymm3[9,10,11,12,13,14,15]
+; AVX512VL-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4,5,6,7]
+; AVX512VL-NEXT:    vpmulhuw %ymm4, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpmullw %ymm4, %ymm0, %ymm3
+; AVX512VL-NEXT:    vpor %ymm1, %ymm3, %ymm1
+; AVX512VL-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3,4,5,6,7],ymm0[8],ymm1[9,10,11,12,13,14,15]
+; AVX512VL-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: constant_funnnel_v32i16:
@@ -1110,15 +1144,17 @@ define <32 x i16> @constant_funnnel_v32i16(<32 x i16> %x, <32 x i16> %y) nounwin
 define <64 x i8> @constant_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y) nounwind {
 ; AVX512F-LABEL: constant_funnnel_v64i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm4
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
+; AVX512F-NEXT:    vpsllw $4, %ymm3, %ymm4
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm5 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
 ; AVX512F-NEXT:    vpand %ymm5, %ymm4, %ymm4
 ; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm6 = [8192,24640,41088,57536,57344,41152,24704,8256,8192,24640,41088,57536,57344,41152,24704,8256]
 ; AVX512F-NEXT:    # ymm6 = mem[0,1,0,1]
-; AVX512F-NEXT:    vpblendvb %ymm6, %ymm4, %ymm0, %ymm4
+; AVX512F-NEXT:    vpblendvb %ymm6, %ymm4, %ymm3, %ymm4
 ; AVX512F-NEXT:    vpsllw $2, %ymm4, %ymm7
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm8 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
-; AVX512F-NEXT:    vpand %ymm8, %ymm7, %ymm7
+; AVX512F-NEXT:    vpand %ymm7, %ymm8, %ymm7
 ; AVX512F-NEXT:    vpaddb %ymm6, %ymm6, %ymm9
 ; AVX512F-NEXT:    vpblendvb %ymm9, %ymm7, %ymm4, %ymm4
 ; AVX512F-NEXT:    vpaddb %ymm4, %ymm4, %ymm7
@@ -1133,42 +1169,45 @@ define <64 x i8> @constant_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y) nounwind {
 ; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm7[0],ymm2[1],ymm7[1],ymm2[2],ymm7[2],ymm2[3],ymm7[3],ymm2[4],ymm7[4],ymm2[5],ymm7[5],ymm2[6],ymm7[6],ymm2[7],ymm7[7],ymm2[16],ymm7[16],ymm2[17],ymm7[17],ymm2[18],ymm7[18],ymm2[19],ymm7[19],ymm2[20],ymm7[20],ymm2[21],ymm7[21],ymm2[22],ymm7[22],ymm2[23],ymm7[23]
 ; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm13 = [256,2,4,8,16,32,64,128,256,2,4,8,16,32,64,128]
 ; AVX512F-NEXT:    # ymm13 = mem[0,1,0,1]
-; AVX512F-NEXT:    vpmullw %ymm13, %ymm2, %ymm2
+; AVX512F-NEXT:    vpmullw %ymm2, %ymm13, %ymm2
 ; AVX512F-NEXT:    vpsrlw $8, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpackuswb %ymm11, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpor %ymm2, %ymm4, %ymm2
 ; AVX512F-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [18446744073709551360,18446744073709551360,18446744073709551360,18446744073709551360]
-; AVX512F-NEXT:    vpblendvb %ymm4, %ymm2, %ymm0, %ymm0
-; AVX512F-NEXT:    vpsllw $4, %ymm1, %ymm2
-; AVX512F-NEXT:    vpand %ymm5, %ymm2, %ymm2
-; AVX512F-NEXT:    vpblendvb %ymm6, %ymm2, %ymm1, %ymm2
-; AVX512F-NEXT:    vpsllw $2, %ymm2, %ymm5
-; AVX512F-NEXT:    vpand %ymm8, %ymm5, %ymm5
-; AVX512F-NEXT:    vpblendvb %ymm9, %ymm5, %ymm2, %ymm2
-; AVX512F-NEXT:    vpaddb %ymm2, %ymm2, %ymm5
-; AVX512F-NEXT:    vpblendvb %ymm10, %ymm5, %ymm2, %ymm2
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm3[8],ymm7[8],ymm3[9],ymm7[9],ymm3[10],ymm7[10],ymm3[11],ymm7[11],ymm3[12],ymm7[12],ymm3[13],ymm7[13],ymm3[14],ymm7[14],ymm3[15],ymm7[15],ymm3[24],ymm7[24],ymm3[25],ymm7[25],ymm3[26],ymm7[26],ymm3[27],ymm7[27],ymm3[28],ymm7[28],ymm3[29],ymm7[29],ymm3[30],ymm7[30],ymm3[31],ymm7[31]
-; AVX512F-NEXT:    vpmullw %ymm12, %ymm5, %ymm5
+; AVX512F-NEXT:    vpblendvb %ymm4, %ymm2, %ymm3, %ymm2
+; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm3
+; AVX512F-NEXT:    vpand %ymm5, %ymm3, %ymm3
+; AVX512F-NEXT:    vpblendvb %ymm6, %ymm3, %ymm0, %ymm3
+; AVX512F-NEXT:    vpsllw $2, %ymm3, %ymm5
+; AVX512F-NEXT:    vpand %ymm5, %ymm8, %ymm5
+; AVX512F-NEXT:    vpblendvb %ymm9, %ymm5, %ymm3, %ymm3
+; AVX512F-NEXT:    vpaddb %ymm3, %ymm3, %ymm5
+; AVX512F-NEXT:    vpblendvb %ymm10, %ymm5, %ymm3, %ymm3
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm1[8],ymm7[8],ymm1[9],ymm7[9],ymm1[10],ymm7[10],ymm1[11],ymm7[11],ymm1[12],ymm7[12],ymm1[13],ymm7[13],ymm1[14],ymm7[14],ymm1[15],ymm7[15],ymm1[24],ymm7[24],ymm1[25],ymm7[25],ymm1[26],ymm7[26],ymm1[27],ymm7[27],ymm1[28],ymm7[28],ymm1[29],ymm7[29],ymm1[30],ymm7[30],ymm1[31],ymm7[31]
+; AVX512F-NEXT:    vpmullw %ymm5, %ymm12, %ymm5
 ; AVX512F-NEXT:    vpsrlw $8, %ymm5, %ymm5
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm7[0],ymm3[1],ymm7[1],ymm3[2],ymm7[2],ymm3[3],ymm7[3],ymm3[4],ymm7[4],ymm3[5],ymm7[5],ymm3[6],ymm7[6],ymm3[7],ymm7[7],ymm3[16],ymm7[16],ymm3[17],ymm7[17],ymm3[18],ymm7[18],ymm3[19],ymm7[19],ymm3[20],ymm7[20],ymm3[21],ymm7[21],ymm3[22],ymm7[22],ymm3[23],ymm7[23]
-; AVX512F-NEXT:    vpmullw %ymm13, %ymm3, %ymm3
-; AVX512F-NEXT:    vpsrlw $8, %ymm3, %ymm3
-; AVX512F-NEXT:    vpackuswb %ymm5, %ymm3, %ymm3
-; AVX512F-NEXT:    vpor %ymm3, %ymm2, %ymm2
-; AVX512F-NEXT:    vpblendvb %ymm4, %ymm2, %ymm1, %ymm1
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm7[0],ymm1[1],ymm7[1],ymm1[2],ymm7[2],ymm1[3],ymm7[3],ymm1[4],ymm7[4],ymm1[5],ymm7[5],ymm1[6],ymm7[6],ymm1[7],ymm7[7],ymm1[16],ymm7[16],ymm1[17],ymm7[17],ymm1[18],ymm7[18],ymm1[19],ymm7[19],ymm1[20],ymm7[20],ymm1[21],ymm7[21],ymm1[22],ymm7[22],ymm1[23],ymm7[23]
+; AVX512F-NEXT:    vpmullw %ymm1, %ymm13, %ymm1
+; AVX512F-NEXT:    vpsrlw $8, %ymm1, %ymm1
+; AVX512F-NEXT:    vpackuswb %ymm5, %ymm1, %ymm1
+; AVX512F-NEXT:    vpor %ymm1, %ymm3, %ymm1
+; AVX512F-NEXT:    vpblendvb %ymm4, %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: constant_funnnel_v64i8:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm4
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
+; AVX512VL-NEXT:    vpsllw $4, %ymm3, %ymm4
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm5 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
 ; AVX512VL-NEXT:    vpand %ymm5, %ymm4, %ymm4
 ; AVX512VL-NEXT:    vbroadcasti128 {{.*#+}} ymm6 = [8192,24640,41088,57536,57344,41152,24704,8256,8192,24640,41088,57536,57344,41152,24704,8256]
 ; AVX512VL-NEXT:    # ymm6 = mem[0,1,0,1]
-; AVX512VL-NEXT:    vpblendvb %ymm6, %ymm4, %ymm0, %ymm4
+; AVX512VL-NEXT:    vpblendvb %ymm6, %ymm4, %ymm3, %ymm4
 ; AVX512VL-NEXT:    vpsllw $2, %ymm4, %ymm7
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm8 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
-; AVX512VL-NEXT:    vpand %ymm8, %ymm7, %ymm7
+; AVX512VL-NEXT:    vpand %ymm7, %ymm8, %ymm7
 ; AVX512VL-NEXT:    vpaddb %ymm6, %ymm6, %ymm9
 ; AVX512VL-NEXT:    vpblendvb %ymm9, %ymm7, %ymm4, %ymm4
 ; AVX512VL-NEXT:    vpaddb %ymm4, %ymm4, %ymm7
@@ -1178,37 +1217,38 @@ define <64 x i8> @constant_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y) nounwind {
 ; AVX512VL-NEXT:    vpsrlw $8, %ymm7, %ymm7
 ; AVX512VL-NEXT:    vbroadcasti128 {{.*#+}} ymm11 = [256,128,64,32,16,8,4,2,256,128,64,32,16,8,4,2]
 ; AVX512VL-NEXT:    # ymm11 = mem[0,1,0,1]
-; AVX512VL-NEXT:    vpmullw %ymm11, %ymm7, %ymm7
+; AVX512VL-NEXT:    vpmullw %ymm7, %ymm11, %ymm7
 ; AVX512VL-NEXT:    vpsrlw $8, %ymm7, %ymm7
 ; AVX512VL-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
 ; AVX512VL-NEXT:    vpsrlw $8, %ymm2, %ymm2
 ; AVX512VL-NEXT:    vbroadcasti128 {{.*#+}} ymm12 = [256,2,4,8,16,32,64,128,256,2,4,8,16,32,64,128]
 ; AVX512VL-NEXT:    # ymm12 = mem[0,1,0,1]
-; AVX512VL-NEXT:    vpmullw %ymm12, %ymm2, %ymm2
+; AVX512VL-NEXT:    vpmullw %ymm2, %ymm12, %ymm2
 ; AVX512VL-NEXT:    vpsrlw $8, %ymm2, %ymm2
 ; AVX512VL-NEXT:    vpackuswb %ymm7, %ymm2, %ymm2
 ; AVX512VL-NEXT:    vpor %ymm2, %ymm4, %ymm2
 ; AVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [18446744073709551360,18446744073709551360,18446744073709551360,18446744073709551360]
-; AVX512VL-NEXT:    vpblendvb %ymm4, %ymm2, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpsllw $4, %ymm1, %ymm2
-; AVX512VL-NEXT:    vpand %ymm5, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpblendvb %ymm6, %ymm2, %ymm1, %ymm2
-; AVX512VL-NEXT:    vpsllw $2, %ymm2, %ymm5
-; AVX512VL-NEXT:    vpand %ymm8, %ymm5, %ymm5
-; AVX512VL-NEXT:    vpblendvb %ymm9, %ymm5, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpaddb %ymm2, %ymm2, %ymm5
-; AVX512VL-NEXT:    vpblendvb %ymm10, %ymm5, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX512VL-NEXT:    vpblendvb %ymm4, %ymm2, %ymm3, %ymm2
+; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm3
+; AVX512VL-NEXT:    vpand %ymm5, %ymm3, %ymm3
+; AVX512VL-NEXT:    vpblendvb %ymm6, %ymm3, %ymm0, %ymm3
+; AVX512VL-NEXT:    vpsllw $2, %ymm3, %ymm5
+; AVX512VL-NEXT:    vpand %ymm5, %ymm8, %ymm5
+; AVX512VL-NEXT:    vpblendvb %ymm9, %ymm5, %ymm3, %ymm3
+; AVX512VL-NEXT:    vpaddb %ymm3, %ymm3, %ymm5
+; AVX512VL-NEXT:    vpblendvb %ymm10, %ymm5, %ymm3, %ymm3
+; AVX512VL-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
 ; AVX512VL-NEXT:    vpsrlw $8, %ymm5, %ymm5
-; AVX512VL-NEXT:    vpmullw %ymm11, %ymm5, %ymm5
+; AVX512VL-NEXT:    vpmullw %ymm5, %ymm11, %ymm5
 ; AVX512VL-NEXT:    vpsrlw $8, %ymm5, %ymm5
-; AVX512VL-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512VL-NEXT:    vpsrlw $8, %ymm3, %ymm3
-; AVX512VL-NEXT:    vpmullw %ymm12, %ymm3, %ymm3
-; AVX512VL-NEXT:    vpsrlw $8, %ymm3, %ymm3
-; AVX512VL-NEXT:    vpackuswb %ymm5, %ymm3, %ymm3
-; AVX512VL-NEXT:    vpor %ymm3, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpblendvb %ymm4, %ymm2, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512VL-NEXT:    vpsrlw $8, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpmullw %ymm1, %ymm12, %ymm1
+; AVX512VL-NEXT:    vpsrlw $8, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpackuswb %ymm5, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpor %ymm1, %ymm3, %ymm1
+; AVX512VL-NEXT:    vpblendvb %ymm4, %ymm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: constant_funnnel_v64i8:
@@ -1433,22 +1473,28 @@ define <16 x i32> @splatconstant_funnnel_v16i32(<16 x i32> %x, <16 x i32> %y) no
 define <32 x i16> @splatconstant_funnnel_v32i16(<32 x i16> %x, <32 x i16> %y) nounwind {
 ; AVX512F-LABEL: splatconstant_funnnel_v32i16:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpsrlw $9, %ymm2, %ymm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
+; AVX512F-NEXT:    vpsrlw $9, %ymm3, %ymm3
+; AVX512F-NEXT:    vpsllw $7, %ymm2, %ymm2
+; AVX512F-NEXT:    vpor %ymm3, %ymm2, %ymm2
+; AVX512F-NEXT:    vpsrlw $9, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpsllw $7, %ymm0, %ymm0
-; AVX512F-NEXT:    vpor %ymm2, %ymm0, %ymm0
-; AVX512F-NEXT:    vpsrlw $9, %ymm3, %ymm2
-; AVX512F-NEXT:    vpsllw $7, %ymm1, %ymm1
-; AVX512F-NEXT:    vpor %ymm2, %ymm1, %ymm1
+; AVX512F-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatconstant_funnnel_v32i16:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsrlw $9, %ymm2, %ymm2
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
+; AVX512VL-NEXT:    vpsrlw $9, %ymm3, %ymm3
+; AVX512VL-NEXT:    vpsllw $7, %ymm2, %ymm2
+; AVX512VL-NEXT:    vpor %ymm3, %ymm2, %ymm2
+; AVX512VL-NEXT:    vpsrlw $9, %ymm1, %ymm1
 ; AVX512VL-NEXT:    vpsllw $7, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpor %ymm2, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpsrlw $9, %ymm3, %ymm2
-; AVX512VL-NEXT:    vpsllw $7, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpor %ymm2, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatconstant_funnnel_v32i16:
@@ -1481,68 +1527,62 @@ define <32 x i16> @splatconstant_funnnel_v32i16(<32 x i16> %x, <32 x i16> %y) no
 define <64 x i8> @splatconstant_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y) nounwind {
 ; AVX512F-LABEL: splatconstant_funnnel_v64i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpsrlw $4, %ymm2, %ymm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
+; AVX512F-NEXT:    vpsrlw $4, %ymm3, %ymm3
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm4 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX512F-NEXT:    vpandn %ymm2, %ymm4, %ymm2
+; AVX512F-NEXT:    vpandn %ymm3, %ymm4, %ymm3
+; AVX512F-NEXT:    vpsllw $4, %ymm2, %ymm2
+; AVX512F-NEXT:    vpand %ymm4, %ymm2, %ymm2
+; AVX512F-NEXT:    vpor %ymm3, %ymm2, %ymm2
+; AVX512F-NEXT:    vpsrlw $4, %ymm1, %ymm1
+; AVX512F-NEXT:    vpandn %ymm1, %ymm4, %ymm1
 ; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpand %ymm4, %ymm0, %ymm0
-; AVX512F-NEXT:    vpor %ymm2, %ymm0, %ymm0
-; AVX512F-NEXT:    vpsrlw $4, %ymm3, %ymm2
-; AVX512F-NEXT:    vpandn %ymm2, %ymm4, %ymm2
-; AVX512F-NEXT:    vpsllw $4, %ymm1, %ymm1
-; AVX512F-NEXT:    vpand %ymm4, %ymm1, %ymm1
-; AVX512F-NEXT:    vpor %ymm2, %ymm1, %ymm1
+; AVX512F-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatconstant_funnnel_v64i8:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsrlw $4, %ymm2, %ymm2
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
+; AVX512VL-NEXT:    vpsrlw $4, %ymm3, %ymm3
+; AVX512VL-NEXT:    vpsllw $4, %ymm2, %ymm2
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm4 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX512VL-NEXT:    vpandn %ymm2, %ymm4, %ymm2
+; AVX512VL-NEXT:    vpternlogq $226, %ymm3, %ymm4, %ymm2
+; AVX512VL-NEXT:    vpsrlw $4, %ymm1, %ymm1
 ; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpand %ymm4, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpor %ymm2, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpsrlw $4, %ymm3, %ymm2
-; AVX512VL-NEXT:    vpandn %ymm2, %ymm4, %ymm2
-; AVX512VL-NEXT:    vpsllw $4, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpand %ymm4, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpor %ymm2, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpternlogq $226, %ymm1, %ymm4, %ymm0
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatconstant_funnnel_v64i8:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpsrlw $4, %zmm1, %zmm1
-; AVX512BW-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
-; AVX512BW-NEXT:    vpsllw $4, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
-; AVX512BW-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsllw $4, %zmm0, %zmm2
+; AVX512BW-NEXT:    vpsrlw $4, %zmm1, %zmm0
+; AVX512BW-NEXT:    vpternlogq $216, {{.*}}(%rip), %zmm2, %zmm0
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512VBMI2-LABEL: splatconstant_funnnel_v64i8:
 ; AVX512VBMI2:       # %bb.0:
-; AVX512VBMI2-NEXT:    vpsrlw $4, %zmm1, %zmm1
-; AVX512VBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
-; AVX512VBMI2-NEXT:    vpsllw $4, %zmm0, %zmm0
-; AVX512VBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
-; AVX512VBMI2-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    vpsllw $4, %zmm0, %zmm2
+; AVX512VBMI2-NEXT:    vpsrlw $4, %zmm1, %zmm0
+; AVX512VBMI2-NEXT:    vpternlogq $216, {{.*}}(%rip), %zmm2, %zmm0
 ; AVX512VBMI2-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: splatconstant_funnnel_v64i8:
 ; AVX512VLBW:       # %bb.0:
-; AVX512VLBW-NEXT:    vpsrlw $4, %zmm1, %zmm1
-; AVX512VLBW-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
-; AVX512VLBW-NEXT:    vpsllw $4, %zmm0, %zmm0
-; AVX512VLBW-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
-; AVX512VLBW-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512VLBW-NEXT:    vpsllw $4, %zmm0, %zmm2
+; AVX512VLBW-NEXT:    vpsrlw $4, %zmm1, %zmm0
+; AVX512VLBW-NEXT:    vpternlogq $216, {{.*}}(%rip), %zmm2, %zmm0
 ; AVX512VLBW-NEXT:    retq
 ;
 ; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v64i8:
 ; AVX512VLVBMI2:       # %bb.0:
-; AVX512VLVBMI2-NEXT:    vpsrlw $4, %zmm1, %zmm1
-; AVX512VLVBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
-; AVX512VLVBMI2-NEXT:    vpsllw $4, %zmm0, %zmm0
-; AVX512VLVBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
-; AVX512VLVBMI2-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    vpsllw $4, %zmm0, %zmm2
+; AVX512VLVBMI2-NEXT:    vpsrlw $4, %zmm1, %zmm0
+; AVX512VLVBMI2-NEXT:    vpternlogq $216, {{.*}}(%rip), %zmm2, %zmm0
 ; AVX512VLVBMI2-NEXT:    retq
   %res = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>)
   ret <64 x i8> %res
diff --git a/llvm/test/CodeGen/X86/vector-fshl-rot-128.ll b/llvm/test/CodeGen/X86/vector-fshl-rot-128.ll
index 2e12eeabf..7ef1ed243 100644
--- a/llvm/test/CodeGen/X86/vector-fshl-rot-128.ll
+++ b/llvm/test/CodeGen/X86/vector-fshl-rot-128.ll
@@ -276,24 +276,23 @@ define <8 x i16> @var_funnnel_v8i16(<8 x i16> %x, <8 x i16> %amt) nounwind {
 ; SSE2-LABEL: var_funnnel_v8i16:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    movdqa %xmm1, %xmm3
-; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
-; SSE2-NEXT:    pslld $23, %xmm3
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [1065353216,1065353216,1065353216,1065353216]
-; SSE2-NEXT:    paddd %xmm4, %xmm3
-; SSE2-NEXT:    cvttps2dq %xmm3, %xmm3
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm3[0,2,2,3,4,5,6,7]
-; SSE2-NEXT:    pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,6,6,7]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE2-NEXT:    pslld $23, %xmm2
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [1065353216,1065353216,1065353216,1065353216]
+; SSE2-NEXT:    paddd %xmm3, %xmm2
+; SSE2-NEXT:    cvttps2dq %xmm2, %xmm2
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,2,2,3,4,5,6,7]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,6,6,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
 ; SSE2-NEXT:    pslld $23, %xmm1
-; SSE2-NEXT:    paddd %xmm4, %xmm1
+; SSE2-NEXT:    paddd %xmm3, %xmm1
 ; SSE2-NEXT:    cvttps2dq %xmm1, %xmm1
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
 ; SSE2-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,6,6,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm2
 ; SSE2-NEXT:    pmulhuw %xmm1, %xmm2
 ; SSE2-NEXT:    pmullw %xmm1, %xmm0
@@ -339,20 +338,19 @@ define <8 x i16> @var_funnnel_v8i16(<8 x i16> %x, <8 x i16> %amt) nounwind {
 ;
 ; AVX2-LABEL: var_funnnel_v8i16:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; AVX2-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
-; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
-; AVX2-NEXT:    vpsllvd %ymm2, %ymm0, %ymm2
-; AVX2-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; AVX2-NEXT:    vpshufb %ymm3, %ymm2, %ymm2
-; AVX2-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,2,3]
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm4 = [16,16,16,16,16,16,16,16]
-; AVX2-NEXT:    vpsubw %xmm1, %xmm4, %xmm1
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [16,16,16,16,16,16,16,16]
+; AVX2-NEXT:    vpsubw %xmm1, %xmm2, %xmm2
+; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
+; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2-NEXT:    vpsrlvd %ymm2, %ymm0, %ymm2
+; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm3
+; AVX2-NEXT:    vpackusdw %xmm3, %xmm2, %xmm2
 ; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
-; AVX2-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpshufb %ymm3, %ymm0, %ymm0
+; AVX2-NEXT:    vpsllvd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-NEXT:    vpor %xmm0, %xmm2, %xmm0
+; AVX2-NEXT:    vpor %xmm2, %xmm0, %xmm0
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
@@ -417,24 +415,23 @@ define <8 x i16> @var_funnnel_v8i16(<8 x i16> %x, <8 x i16> %amt) nounwind {
 ; X32-SSE-LABEL: var_funnnel_v8i16:
 ; X32-SSE:       # %bb.0:
 ; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm1
-; X32-SSE-NEXT:    pxor %xmm2, %xmm2
-; X32-SSE-NEXT:    movdqa %xmm1, %xmm3
-; X32-SSE-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
-; X32-SSE-NEXT:    pslld $23, %xmm3
-; X32-SSE-NEXT:    movdqa {{.*#+}} xmm4 = [1065353216,1065353216,1065353216,1065353216]
-; X32-SSE-NEXT:    paddd %xmm4, %xmm3
-; X32-SSE-NEXT:    cvttps2dq %xmm3, %xmm3
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm3 = xmm3[0,2,2,3,4,5,6,7]
-; X32-SSE-NEXT:    pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,6,6,7]
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
-; X32-SSE-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
+; X32-SSE-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; X32-SSE-NEXT:    pslld $23, %xmm2
+; X32-SSE-NEXT:    movdqa {{.*#+}} xmm3 = [1065353216,1065353216,1065353216,1065353216]
+; X32-SSE-NEXT:    paddd %xmm3, %xmm2
+; X32-SSE-NEXT:    cvttps2dq %xmm2, %xmm2
+; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,2,2,3,4,5,6,7]
+; X32-SSE-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,6,6,7]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; X32-SSE-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
 ; X32-SSE-NEXT:    pslld $23, %xmm1
-; X32-SSE-NEXT:    paddd %xmm4, %xmm1
+; X32-SSE-NEXT:    paddd %xmm3, %xmm1
 ; X32-SSE-NEXT:    cvttps2dq %xmm1, %xmm1
 ; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
 ; X32-SSE-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,6,6,7]
 ; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; X32-SSE-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; X32-SSE-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
 ; X32-SSE-NEXT:    movdqa %xmm0, %xmm2
 ; X32-SSE-NEXT:    pmulhuw %xmm1, %xmm2
 ; X32-SSE-NEXT:    pmullw %xmm1, %xmm0
@@ -1837,14 +1834,37 @@ define <16 x i8> @splatconstant_funnnel_v16i8(<16 x i8> %x) nounwind {
 ; AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
-; AVX512-LABEL: splatconstant_funnnel_v16i8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsrlw $4, %xmm0, %xmm1
-; AVX512-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
-; AVX512-NEXT:    vpsllw $4, %xmm0, %xmm0
-; AVX512-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: splatconstant_funnnel_v16i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsrlw $4, %xmm0, %xmm1
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512F-NEXT:    vpsllw $4, %xmm0, %xmm0
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX512F-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: splatconstant_funnnel_v16i8:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsllw $4, %xmm0, %xmm1
+; AVX512VL-NEXT:    vpsrlw $4, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpternlogq $216, {{.*}}(%rip), %xmm1, %xmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: splatconstant_funnnel_v16i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsrlw $4, %xmm0, %xmm1
+; AVX512BW-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512BW-NEXT:    vpsllw $4, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX512BW-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VLBW-LABEL: splatconstant_funnnel_v16i8:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpsllw $4, %xmm0, %xmm1
+; AVX512VLBW-NEXT:    vpsrlw $4, %xmm0, %xmm0
+; AVX512VLBW-NEXT:    vpternlogq $216, {{.*}}(%rip), %xmm1, %xmm0
+; AVX512VLBW-NEXT:    retq
 ;
 ; XOP-LABEL: splatconstant_funnnel_v16i8:
 ; XOP:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/vector-fshl-rot-256.ll b/llvm/test/CodeGen/X86/vector-fshl-rot-256.ll
index f2da60536..ba4279f8d 100644
--- a/llvm/test/CodeGen/X86/vector-fshl-rot-256.ll
+++ b/llvm/test/CodeGen/X86/vector-fshl-rot-256.ll
@@ -210,29 +210,28 @@ define <16 x i16> @var_funnnel_v16i16(<16 x i16> %x, <16 x i16> %amt) nounwind {
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15]
 ; AVX1-NEXT:    vpand %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7]
-; AVX1-NEXT:    vpslld $23, %xmm5, %xmm5
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [1065353216,1065353216,1065353216,1065353216]
-; AVX1-NEXT:    vpaddd %xmm6, %xmm5, %xmm5
-; AVX1-NEXT:    vcvttps2dq %xmm5, %xmm5
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; AVX1-NEXT:    vpslld $23, %xmm4, %xmm4
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm5 = [1065353216,1065353216,1065353216,1065353216]
+; AVX1-NEXT:    vpaddd %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vcvttps2dq %xmm4, %xmm4
 ; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
 ; AVX1-NEXT:    vpslld $23, %xmm2, %xmm2
-; AVX1-NEXT:    vpaddd %xmm6, %xmm2, %xmm2
+; AVX1-NEXT:    vpaddd %xmm5, %xmm2, %xmm2
 ; AVX1-NEXT:    vcvttps2dq %xmm2, %xmm2
-; AVX1-NEXT:    vpackusdw %xmm5, %xmm2, %xmm2
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm5
-; AVX1-NEXT:    vpmulhuw %xmm2, %xmm5, %xmm7
-; AVX1-NEXT:    vpmullw %xmm2, %xmm5, %xmm2
-; AVX1-NEXT:    vpor %xmm7, %xmm2, %xmm2
+; AVX1-NEXT:    vpackusdw %xmm4, %xmm2, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
+; AVX1-NEXT:    vpmulhuw %xmm2, %xmm4, %xmm6
+; AVX1-NEXT:    vpmullw %xmm2, %xmm4, %xmm2
+; AVX1-NEXT:    vpor %xmm6, %xmm2, %xmm2
 ; AVX1-NEXT:    vpand %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm1[4],xmm4[4],xmm1[5],xmm4[5],xmm1[6],xmm4[6],xmm1[7],xmm4[7]
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
 ; AVX1-NEXT:    vpslld $23, %xmm3, %xmm3
-; AVX1-NEXT:    vpaddd %xmm6, %xmm3, %xmm3
+; AVX1-NEXT:    vpaddd %xmm5, %xmm3, %xmm3
 ; AVX1-NEXT:    vcvttps2dq %xmm3, %xmm3
 ; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
 ; AVX1-NEXT:    vpslld $23, %xmm1, %xmm1
-; AVX1-NEXT:    vpaddd %xmm6, %xmm1, %xmm1
+; AVX1-NEXT:    vpaddd %xmm5, %xmm1, %xmm1
 ; AVX1-NEXT:    vcvttps2dq %xmm1, %xmm1
 ; AVX1-NEXT:    vpackusdw %xmm3, %xmm1, %xmm1
 ; AVX1-NEXT:    vpmulhuw %xmm1, %xmm0, %xmm3
@@ -359,7 +358,7 @@ define <32 x i8> @var_funnnel_v32i8(<32 x i8> %x, <32 x i8> %amt) nounwind {
 ; AVX1-NEXT:    vpblendvb %xmm5, %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vpsrlw $7, %xmm2, %xmm3
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm8 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; AVX1-NEXT:    vpand %xmm8, %xmm3, %xmm3
+; AVX1-NEXT:    vpand %xmm3, %xmm8, %xmm3
 ; AVX1-NEXT:    vpaddb %xmm2, %xmm2, %xmm7
 ; AVX1-NEXT:    vpor %xmm3, %xmm7, %xmm3
 ; AVX1-NEXT:    vpaddb %xmm5, %xmm5, %xmm5
@@ -379,7 +378,7 @@ define <32 x i8> @var_funnnel_v32i8(<32 x i8> %x, <32 x i8> %amt) nounwind {
 ; AVX1-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
 ; AVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrlw $7, %xmm0, %xmm3
-; AVX1-NEXT:    vpand %xmm8, %xmm3, %xmm3
+; AVX1-NEXT:    vpand %xmm3, %xmm8, %xmm3
 ; AVX1-NEXT:    vpaddb %xmm0, %xmm0, %xmm4
 ; AVX1-NEXT:    vpor %xmm3, %xmm4, %xmm3
 ; AVX1-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
@@ -437,20 +436,16 @@ define <32 x i8> @var_funnnel_v32i8(<32 x i8> %x, <32 x i8> %amt) nounwind {
 ;
 ; AVX512VL-LABEL: var_funnnel_v32i8:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm2
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
-; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm3
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm3, %ymm3
-; AVX512VL-NEXT:    vpor %ymm2, %ymm3, %ymm2
+; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm2
+; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm3
+; AVX512VL-NEXT:    vpternlogq $216, {{.*}}(%rip), %ymm2, %ymm3
 ; AVX512VL-NEXT:    vpsllw $5, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpsrlw $6, %ymm0, %ymm2
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
-; AVX512VL-NEXT:    vpsllw $2, %ymm0, %ymm3
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm3, %ymm3
-; AVX512VL-NEXT:    vpor %ymm2, %ymm3, %ymm2
+; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpsllw $2, %ymm0, %ymm2
+; AVX512VL-NEXT:    vpsrlw $6, %ymm0, %ymm3
+; AVX512VL-NEXT:    vpternlogq $216, {{.*}}(%rip), %ymm2, %ymm3
 ; AVX512VL-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vpaddb %ymm0, %ymm0, %ymm2
 ; AVX512VL-NEXT:    vpsrlw $7, %ymm0, %ymm3
 ; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm3, %ymm3
@@ -1146,7 +1141,7 @@ define <32 x i8> @constant_funnnel_v32i8(<32 x i8> %x) nounwind {
 ; AVX1-NEXT:    vpxor %xmm8, %xmm8, %xmm8
 ; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm1[8],xmm8[8],xmm1[9],xmm8[9],xmm1[10],xmm8[10],xmm1[11],xmm8[11],xmm1[12],xmm8[12],xmm1[13],xmm8[13],xmm1[14],xmm8[14],xmm1[15],xmm8[15]
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm9 = [256,128,64,32,16,8,4,2]
-; AVX1-NEXT:    vpmullw %xmm9, %xmm3, %xmm3
+; AVX1-NEXT:    vpmullw %xmm3, %xmm9, %xmm3
 ; AVX1-NEXT:    vpsrlw $8, %xmm3, %xmm3
 ; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm5 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [256,2,4,8,16,32,64,128]
@@ -1164,7 +1159,7 @@ define <32 x i8> @constant_funnnel_v32i8(<32 x i8> %x) nounwind {
 ; AVX1-NEXT:    vpackuswb %xmm1, %xmm5, %xmm1
 ; AVX1-NEXT:    vpor %xmm3, %xmm1, %xmm1
 ; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm0[8],xmm8[8],xmm0[9],xmm8[9],xmm0[10],xmm8[10],xmm0[11],xmm8[11],xmm0[12],xmm8[12],xmm0[13],xmm8[13],xmm0[14],xmm8[14],xmm0[15],xmm8[15]
-; AVX1-NEXT:    vpmullw %xmm9, %xmm3, %xmm3
+; AVX1-NEXT:    vpmullw %xmm3, %xmm9, %xmm3
 ; AVX1-NEXT:    vpsrlw $8, %xmm3, %xmm3
 ; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm5 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; AVX1-NEXT:    vpmullw %xmm6, %xmm5, %xmm6
@@ -1498,14 +1493,37 @@ define <32 x i8> @splatconstant_funnnel_v32i8(<32 x i8> %x) nounwind {
 ; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    retq
 ;
-; AVX512-LABEL: splatconstant_funnnel_v32i8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsrlw $4, %ymm0, %ymm1
-; AVX512-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
-; AVX512-NEXT:    vpsllw $4, %ymm0, %ymm0
-; AVX512-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
-; AVX512-NEXT:    vpor %ymm1, %ymm0, %ymm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: splatconstant_funnnel_v32i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsrlw $4, %ymm0, %ymm1
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
+; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm0
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
+; AVX512F-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: splatconstant_funnnel_v32i8:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm1
+; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpternlogq $216, {{.*}}(%rip), %ymm1, %ymm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: splatconstant_funnnel_v32i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsrlw $4, %ymm0, %ymm1
+; AVX512BW-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
+; AVX512BW-NEXT:    vpsllw $4, %ymm0, %ymm0
+; AVX512BW-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
+; AVX512BW-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VLBW-LABEL: splatconstant_funnnel_v32i8:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpsllw $4, %ymm0, %ymm1
+; AVX512VLBW-NEXT:    vpsrlw $4, %ymm0, %ymm0
+; AVX512VLBW-NEXT:    vpternlogq $216, {{.*}}(%rip), %ymm1, %ymm0
+; AVX512VLBW-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: splatconstant_funnnel_v32i8:
 ; XOPAVX1:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/vector-fshl-rot-512.ll b/llvm/test/CodeGen/X86/vector-fshl-rot-512.ll
index de7959d6b..6a7865e51 100644
--- a/llvm/test/CodeGen/X86/vector-fshl-rot-512.ll
+++ b/llvm/test/CodeGen/X86/vector-fshl-rot-512.ll
@@ -34,50 +34,56 @@ define <16 x i32> @var_funnnel_v16i32(<16 x i32> %x, <16 x i32> %amt) nounwind {
 define <32 x i16> @var_funnnel_v32i16(<32 x i16> %x, <32 x i16> %amt) nounwind {
 ; AVX512F-LABEL: var_funnnel_v32i16:
 ; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512F-NEXT:    vpand %ymm4, %ymm2, %ymm2
-; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm5 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
-; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
-; AVX512F-NEXT:    vpsllvd %zmm5, %zmm0, %zmm5
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm6 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
-; AVX512F-NEXT:    vpsubw %ymm2, %ymm6, %ymm2
+; AVX512F-NEXT:    vpand %ymm4, %ymm3, %ymm3
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm5 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
 ; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
-; AVX512F-NEXT:    vpsrlvd %zmm2, %zmm0, %zmm0
-; AVX512F-NEXT:    vpord %zmm0, %zmm5, %zmm0
-; AVX512F-NEXT:    vpmovdw %zmm0, %ymm0
-; AVX512F-NEXT:    vpand %ymm4, %ymm3, %ymm2
-; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm3 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
+; AVX512F-NEXT:    vpsllvd %zmm5, %zmm2, %zmm5
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm6 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
+; AVX512F-NEXT:    vpsubw %ymm3, %ymm6, %ymm3
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm3 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
+; AVX512F-NEXT:    vpsrlvd %zmm3, %zmm2, %zmm2
+; AVX512F-NEXT:    vpord %zmm2, %zmm5, %zmm2
+; AVX512F-NEXT:    vpmovdw %zmm2, %ymm2
+; AVX512F-NEXT:    vpand %ymm4, %ymm1, %ymm1
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm3 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
+; AVX512F-NEXT:    vpsllvd %zmm3, %zmm0, %zmm3
+; AVX512F-NEXT:    vpsubw %ymm1, %ymm6, %ymm1
 ; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
-; AVX512F-NEXT:    vpsllvd %zmm3, %zmm1, %zmm3
-; AVX512F-NEXT:    vpsubw %ymm2, %ymm6, %ymm2
-; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
-; AVX512F-NEXT:    vpsrlvd %zmm2, %zmm1, %zmm1
-; AVX512F-NEXT:    vpord %zmm1, %zmm3, %zmm1
-; AVX512F-NEXT:    vpmovdw %zmm1, %ymm1
+; AVX512F-NEXT:    vpsrlvd %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vpord %zmm0, %zmm3, %zmm0
+; AVX512F-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: var_funnnel_v32i16:
 ; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512VL-NEXT:    vpand %ymm4, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm5 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
-; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
-; AVX512VL-NEXT:    vpsllvd %zmm5, %zmm0, %zmm5
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm6 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
-; AVX512VL-NEXT:    vpsubw %ymm2, %ymm6, %ymm2
+; AVX512VL-NEXT:    vpand %ymm4, %ymm3, %ymm3
+; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm5 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
 ; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
-; AVX512VL-NEXT:    vpsrlvd %zmm2, %zmm0, %zmm0
-; AVX512VL-NEXT:    vpord %zmm0, %zmm5, %zmm0
-; AVX512VL-NEXT:    vpmovdw %zmm0, %ymm0
-; AVX512VL-NEXT:    vpand %ymm4, %ymm3, %ymm2
-; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm3 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
+; AVX512VL-NEXT:    vpsllvd %zmm5, %zmm2, %zmm5
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm6 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
+; AVX512VL-NEXT:    vpsubw %ymm3, %ymm6, %ymm3
+; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm3 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
+; AVX512VL-NEXT:    vpsrlvd %zmm3, %zmm2, %zmm2
+; AVX512VL-NEXT:    vpord %zmm2, %zmm5, %zmm2
+; AVX512VL-NEXT:    vpmovdw %zmm2, %ymm2
+; AVX512VL-NEXT:    vpand %ymm4, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm3 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
+; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
+; AVX512VL-NEXT:    vpsllvd %zmm3, %zmm0, %zmm3
+; AVX512VL-NEXT:    vpsubw %ymm1, %ymm6, %ymm1
 ; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
-; AVX512VL-NEXT:    vpsllvd %zmm3, %zmm1, %zmm3
-; AVX512VL-NEXT:    vpsubw %ymm2, %ymm6, %ymm2
-; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
-; AVX512VL-NEXT:    vpsrlvd %zmm2, %zmm1, %zmm1
-; AVX512VL-NEXT:    vpord %zmm1, %zmm3, %zmm1
-; AVX512VL-NEXT:    vpmovdw %zmm1, %ymm1
+; AVX512VL-NEXT:    vpsrlvd %zmm1, %zmm0, %zmm0
+; AVX512VL-NEXT:    vpord %zmm0, %zmm3, %zmm0
+; AVX512VL-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: var_funnnel_v32i16:
@@ -110,102 +116,100 @@ define <32 x i16> @var_funnnel_v32i16(<32 x i16> %x, <32 x i16> %amt) nounwind {
 define <64 x i8> @var_funnnel_v64i8(<64 x i8> %x, <64 x i8> %amt) nounwind {
 ; AVX512F-LABEL: var_funnnel_v64i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpsrlw $4, %ymm0, %ymm4
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
+; AVX512F-NEXT:    vpsrlw $4, %ymm3, %ymm4
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm5 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
 ; AVX512F-NEXT:    vpandn %ymm4, %ymm5, %ymm4
-; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm6
+; AVX512F-NEXT:    vpsllw $4, %ymm3, %ymm6
 ; AVX512F-NEXT:    vpand %ymm5, %ymm6, %ymm6
 ; AVX512F-NEXT:    vpor %ymm4, %ymm6, %ymm4
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm6 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
 ; AVX512F-NEXT:    vpand %ymm6, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpsllw $5, %ymm2, %ymm2
-; AVX512F-NEXT:    vpblendvb %ymm2, %ymm4, %ymm0, %ymm0
-; AVX512F-NEXT:    vpsrlw $6, %ymm0, %ymm4
+; AVX512F-NEXT:    vpblendvb %ymm2, %ymm4, %ymm3, %ymm3
+; AVX512F-NEXT:    vpsrlw $6, %ymm3, %ymm4
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm7 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
 ; AVX512F-NEXT:    vpandn %ymm4, %ymm7, %ymm4
-; AVX512F-NEXT:    vpsllw $2, %ymm0, %ymm8
+; AVX512F-NEXT:    vpsllw $2, %ymm3, %ymm8
 ; AVX512F-NEXT:    vpand %ymm7, %ymm8, %ymm8
 ; AVX512F-NEXT:    vpor %ymm4, %ymm8, %ymm4
 ; AVX512F-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
-; AVX512F-NEXT:    vpblendvb %ymm2, %ymm4, %ymm0, %ymm0
-; AVX512F-NEXT:    vpsrlw $7, %ymm0, %ymm4
+; AVX512F-NEXT:    vpblendvb %ymm2, %ymm4, %ymm3, %ymm3
+; AVX512F-NEXT:    vpsrlw $7, %ymm3, %ymm4
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm8 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; AVX512F-NEXT:    vpand %ymm8, %ymm4, %ymm4
-; AVX512F-NEXT:    vpaddb %ymm0, %ymm0, %ymm9
+; AVX512F-NEXT:    vpand %ymm4, %ymm8, %ymm4
+; AVX512F-NEXT:    vpaddb %ymm3, %ymm3, %ymm9
 ; AVX512F-NEXT:    vpor %ymm4, %ymm9, %ymm4
 ; AVX512F-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
-; AVX512F-NEXT:    vpblendvb %ymm2, %ymm4, %ymm0, %ymm0
-; AVX512F-NEXT:    vpsrlw $4, %ymm1, %ymm2
-; AVX512F-NEXT:    vpandn %ymm2, %ymm5, %ymm2
-; AVX512F-NEXT:    vpsllw $4, %ymm1, %ymm4
+; AVX512F-NEXT:    vpblendvb %ymm2, %ymm4, %ymm3, %ymm2
+; AVX512F-NEXT:    vpsrlw $4, %ymm0, %ymm3
+; AVX512F-NEXT:    vpandn %ymm3, %ymm5, %ymm3
+; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm4
 ; AVX512F-NEXT:    vpand %ymm5, %ymm4, %ymm4
-; AVX512F-NEXT:    vpor %ymm2, %ymm4, %ymm2
-; AVX512F-NEXT:    vpand %ymm6, %ymm3, %ymm3
-; AVX512F-NEXT:    vpsllw $5, %ymm3, %ymm3
-; AVX512F-NEXT:    vpblendvb %ymm3, %ymm2, %ymm1, %ymm1
-; AVX512F-NEXT:    vpsrlw $6, %ymm1, %ymm2
-; AVX512F-NEXT:    vpandn %ymm2, %ymm7, %ymm2
-; AVX512F-NEXT:    vpsllw $2, %ymm1, %ymm4
+; AVX512F-NEXT:    vpor %ymm3, %ymm4, %ymm3
+; AVX512F-NEXT:    vpand %ymm6, %ymm1, %ymm1
+; AVX512F-NEXT:    vpsllw $5, %ymm1, %ymm1
+; AVX512F-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
+; AVX512F-NEXT:    vpsrlw $6, %ymm0, %ymm3
+; AVX512F-NEXT:    vpandn %ymm3, %ymm7, %ymm3
+; AVX512F-NEXT:    vpsllw $2, %ymm0, %ymm4
 ; AVX512F-NEXT:    vpand %ymm7, %ymm4, %ymm4
-; AVX512F-NEXT:    vpor %ymm2, %ymm4, %ymm2
-; AVX512F-NEXT:    vpaddb %ymm3, %ymm3, %ymm3
-; AVX512F-NEXT:    vpblendvb %ymm3, %ymm2, %ymm1, %ymm1
-; AVX512F-NEXT:    vpsrlw $7, %ymm1, %ymm2
-; AVX512F-NEXT:    vpand %ymm8, %ymm2, %ymm2
-; AVX512F-NEXT:    vpaddb %ymm1, %ymm1, %ymm4
-; AVX512F-NEXT:    vpor %ymm2, %ymm4, %ymm2
-; AVX512F-NEXT:    vpaddb %ymm3, %ymm3, %ymm3
-; AVX512F-NEXT:    vpblendvb %ymm3, %ymm2, %ymm1, %ymm1
+; AVX512F-NEXT:    vpor %ymm3, %ymm4, %ymm3
+; AVX512F-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
+; AVX512F-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
+; AVX512F-NEXT:    vpsrlw $7, %ymm0, %ymm3
+; AVX512F-NEXT:    vpand %ymm3, %ymm8, %ymm3
+; AVX512F-NEXT:    vpaddb %ymm0, %ymm0, %ymm4
+; AVX512F-NEXT:    vpor %ymm3, %ymm4, %ymm3
+; AVX512F-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
+; AVX512F-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: var_funnnel_v64i8:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm4
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm5 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX512VL-NEXT:    vpandn %ymm4, %ymm5, %ymm4
-; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm6
-; AVX512VL-NEXT:    vpand %ymm5, %ymm6, %ymm6
-; AVX512VL-NEXT:    vpor %ymm4, %ymm6, %ymm4
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm6 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; AVX512VL-NEXT:    vpand %ymm6, %ymm2, %ymm2
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
+; AVX512VL-NEXT:    vpsrlw $4, %ymm3, %ymm4
+; AVX512VL-NEXT:    vpsllw $4, %ymm3, %ymm5
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm6 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
+; AVX512VL-NEXT:    vpternlogq $226, %ymm4, %ymm6, %ymm5
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm4 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; AVX512VL-NEXT:    vpand %ymm4, %ymm2, %ymm2
 ; AVX512VL-NEXT:    vpsllw $5, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm4, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpsrlw $6, %ymm0, %ymm4
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm7 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
-; AVX512VL-NEXT:    vpandn %ymm4, %ymm7, %ymm4
-; AVX512VL-NEXT:    vpsllw $2, %ymm0, %ymm8
-; AVX512VL-NEXT:    vpand %ymm7, %ymm8, %ymm8
-; AVX512VL-NEXT:    vpor %ymm4, %ymm8, %ymm4
+; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm5, %ymm3, %ymm3
+; AVX512VL-NEXT:    vpsrlw $6, %ymm3, %ymm5
+; AVX512VL-NEXT:    vpsllw $2, %ymm3, %ymm7
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm8 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
+; AVX512VL-NEXT:    vpternlogq $226, %ymm5, %ymm8, %ymm7
 ; AVX512VL-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm4, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpsrlw $7, %ymm0, %ymm4
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm8 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; AVX512VL-NEXT:    vpand %ymm8, %ymm4, %ymm4
-; AVX512VL-NEXT:    vpaddb %ymm0, %ymm0, %ymm9
-; AVX512VL-NEXT:    vpor %ymm4, %ymm9, %ymm4
+; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm7, %ymm3, %ymm3
+; AVX512VL-NEXT:    vpsrlw $7, %ymm3, %ymm5
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm7 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; AVX512VL-NEXT:    vpand %ymm7, %ymm5, %ymm5
+; AVX512VL-NEXT:    vpaddb %ymm3, %ymm3, %ymm9
+; AVX512VL-NEXT:    vpor %ymm5, %ymm9, %ymm5
 ; AVX512VL-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm4, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpsrlw $4, %ymm1, %ymm2
-; AVX512VL-NEXT:    vpandn %ymm2, %ymm5, %ymm2
-; AVX512VL-NEXT:    vpsllw $4, %ymm1, %ymm4
-; AVX512VL-NEXT:    vpand %ymm5, %ymm4, %ymm4
-; AVX512VL-NEXT:    vpor %ymm2, %ymm4, %ymm2
-; AVX512VL-NEXT:    vpand %ymm6, %ymm3, %ymm3
-; AVX512VL-NEXT:    vpsllw $5, %ymm3, %ymm3
-; AVX512VL-NEXT:    vpblendvb %ymm3, %ymm2, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpsrlw $6, %ymm1, %ymm2
-; AVX512VL-NEXT:    vpandn %ymm2, %ymm7, %ymm2
-; AVX512VL-NEXT:    vpsllw $2, %ymm1, %ymm4
-; AVX512VL-NEXT:    vpand %ymm7, %ymm4, %ymm4
-; AVX512VL-NEXT:    vpor %ymm2, %ymm4, %ymm2
-; AVX512VL-NEXT:    vpaddb %ymm3, %ymm3, %ymm3
-; AVX512VL-NEXT:    vpblendvb %ymm3, %ymm2, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpsrlw $7, %ymm1, %ymm2
-; AVX512VL-NEXT:    vpand %ymm8, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpaddb %ymm1, %ymm1, %ymm4
-; AVX512VL-NEXT:    vpor %ymm2, %ymm4, %ymm2
-; AVX512VL-NEXT:    vpaddb %ymm3, %ymm3, %ymm3
-; AVX512VL-NEXT:    vpblendvb %ymm3, %ymm2, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm5, %ymm3, %ymm2
+; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm3
+; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm5
+; AVX512VL-NEXT:    vpternlogq $226, %ymm3, %ymm6, %ymm5
+; AVX512VL-NEXT:    vpand %ymm4, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpsllw $5, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm5, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpsrlw $6, %ymm0, %ymm3
+; AVX512VL-NEXT:    vpsllw $2, %ymm0, %ymm4
+; AVX512VL-NEXT:    vpternlogq $226, %ymm3, %ymm8, %ymm4
+; AVX512VL-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm4, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpsrlw $7, %ymm0, %ymm3
+; AVX512VL-NEXT:    vpand %ymm7, %ymm3, %ymm3
+; AVX512VL-NEXT:    vpaddb %ymm0, %ymm0, %ymm4
+; AVX512VL-NEXT:    vpor %ymm3, %ymm4, %ymm3
+; AVX512VL-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: var_funnnel_v64i8:
@@ -316,34 +320,38 @@ define <16 x i32> @splatvar_funnnel_v16i32(<16 x i32> %x, <16 x i32> %amt) nounw
 define <32 x i16> @splatvar_funnnel_v32i16(<32 x i16> %x, <32 x i16> %amt) nounwind {
 ; AVX512F-LABEL: splatvar_funnnel_v32i16:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpbroadcastw %xmm2, %xmm2
-; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
-; AVX512F-NEXT:    vpmovzxwq {{.*#+}} xmm3 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
-; AVX512F-NEXT:    vpsllw %xmm3, %ymm0, %ymm4
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512F-NEXT:    vpbroadcastw %xmm1, %xmm1
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512F-NEXT:    vpmovzxwq {{.*#+}} xmm3 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512F-NEXT:    vpsllw %xmm3, %ymm2, %ymm4
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm5 = [16,16,16,16,16,16,16,16]
-; AVX512F-NEXT:    vpsubw %xmm2, %xmm5, %xmm2
-; AVX512F-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
-; AVX512F-NEXT:    vpsrlw %xmm2, %ymm0, %ymm0
-; AVX512F-NEXT:    vpor %ymm0, %ymm4, %ymm0
-; AVX512F-NEXT:    vpsllw %xmm3, %ymm1, %ymm3
-; AVX512F-NEXT:    vpsrlw %xmm2, %ymm1, %ymm1
-; AVX512F-NEXT:    vpor %ymm1, %ymm3, %ymm1
+; AVX512F-NEXT:    vpsubw %xmm1, %xmm5, %xmm1
+; AVX512F-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512F-NEXT:    vpsrlw %xmm1, %ymm2, %ymm2
+; AVX512F-NEXT:    vpor %ymm2, %ymm4, %ymm2
+; AVX512F-NEXT:    vpsllw %xmm3, %ymm0, %ymm3
+; AVX512F-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vpor %ymm0, %ymm3, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatvar_funnnel_v32i16:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpbroadcastw %xmm2, %xmm2
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
-; AVX512VL-NEXT:    vpmovzxwq {{.*#+}} xmm3 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
-; AVX512VL-NEXT:    vpsllw %xmm3, %ymm0, %ymm4
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512VL-NEXT:    vpbroadcastw %xmm1, %xmm1
+; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512VL-NEXT:    vpmovzxwq {{.*#+}} xmm3 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512VL-NEXT:    vpsllw %xmm3, %ymm2, %ymm4
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm5 = [16,16,16,16,16,16,16,16]
-; AVX512VL-NEXT:    vpsubw %xmm2, %xmm5, %xmm2
-; AVX512VL-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
-; AVX512VL-NEXT:    vpsrlw %xmm2, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpor %ymm0, %ymm4, %ymm0
-; AVX512VL-NEXT:    vpsllw %xmm3, %ymm1, %ymm3
-; AVX512VL-NEXT:    vpsrlw %xmm2, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpor %ymm1, %ymm3, %ymm1
+; AVX512VL-NEXT:    vpsubw %xmm1, %xmm5, %xmm1
+; AVX512VL-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512VL-NEXT:    vpsrlw %xmm1, %ymm2, %ymm2
+; AVX512VL-NEXT:    vpor %ymm2, %ymm4, %ymm2
+; AVX512VL-NEXT:    vpsllw %xmm3, %ymm0, %ymm3
+; AVX512VL-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpor %ymm0, %ymm3, %ymm0
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatvar_funnnel_v32i16:
@@ -383,54 +391,58 @@ define <32 x i16> @splatvar_funnnel_v32i16(<32 x i16> %x, <32 x i16> %amt) nounw
 define <64 x i8> @splatvar_funnnel_v64i8(<64 x i8> %x, <64 x i8> %amt) nounwind {
 ; AVX512F-LABEL: splatvar_funnnel_v64i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpbroadcastb %xmm2, %xmm2
-; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
-; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512F-NEXT:    vpsllw %xmm3, %ymm0, %ymm4
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512F-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512F-NEXT:    vpsllw %xmm3, %ymm2, %ymm4
 ; AVX512F-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
 ; AVX512F-NEXT:    vpsllw %xmm3, %xmm5, %xmm6
 ; AVX512F-NEXT:    vpbroadcastb %xmm6, %ymm6
 ; AVX512F-NEXT:    vpand %ymm6, %ymm4, %ymm4
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm7 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; AVX512F-NEXT:    vpsubb %xmm2, %xmm7, %xmm2
-; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512F-NEXT:    vpsrlw %xmm2, %ymm0, %ymm0
-; AVX512F-NEXT:    vpsrlw %xmm2, %xmm5, %xmm5
+; AVX512F-NEXT:    vpsubb %xmm1, %xmm7, %xmm1
+; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512F-NEXT:    vpsrlw %xmm1, %ymm2, %ymm2
+; AVX512F-NEXT:    vpsrlw %xmm1, %xmm5, %xmm5
 ; AVX512F-NEXT:    vpsrlw $8, %xmm5, %xmm5
 ; AVX512F-NEXT:    vpbroadcastb %xmm5, %ymm5
-; AVX512F-NEXT:    vpand %ymm5, %ymm0, %ymm0
-; AVX512F-NEXT:    vpor %ymm0, %ymm4, %ymm0
-; AVX512F-NEXT:    vpsllw %xmm3, %ymm1, %ymm3
+; AVX512F-NEXT:    vpand %ymm5, %ymm2, %ymm2
+; AVX512F-NEXT:    vpor %ymm2, %ymm4, %ymm2
+; AVX512F-NEXT:    vpsllw %xmm3, %ymm0, %ymm3
 ; AVX512F-NEXT:    vpand %ymm6, %ymm3, %ymm3
-; AVX512F-NEXT:    vpsrlw %xmm2, %ymm1, %ymm1
-; AVX512F-NEXT:    vpand %ymm5, %ymm1, %ymm1
-; AVX512F-NEXT:    vpor %ymm1, %ymm3, %ymm1
+; AVX512F-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vpand %ymm5, %ymm0, %ymm0
+; AVX512F-NEXT:    vpor %ymm0, %ymm3, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatvar_funnnel_v64i8:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpbroadcastb %xmm2, %xmm2
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
-; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512VL-NEXT:    vpsllw %xmm3, %ymm0, %ymm4
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512VL-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VL-NEXT:    vpsllw %xmm3, %ymm2, %ymm4
 ; AVX512VL-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
 ; AVX512VL-NEXT:    vpsllw %xmm3, %xmm5, %xmm6
 ; AVX512VL-NEXT:    vpbroadcastb %xmm6, %ymm6
 ; AVX512VL-NEXT:    vpand %ymm6, %ymm4, %ymm4
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm7 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; AVX512VL-NEXT:    vpsubb %xmm2, %xmm7, %xmm2
-; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512VL-NEXT:    vpsrlw %xmm2, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpsrlw %xmm2, %xmm5, %xmm5
+; AVX512VL-NEXT:    vpsubb %xmm1, %xmm7, %xmm1
+; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VL-NEXT:    vpsrlw %xmm1, %ymm2, %ymm2
+; AVX512VL-NEXT:    vpsrlw %xmm1, %xmm5, %xmm5
 ; AVX512VL-NEXT:    vpsrlw $8, %xmm5, %xmm5
 ; AVX512VL-NEXT:    vpbroadcastb %xmm5, %ymm5
-; AVX512VL-NEXT:    vpand %ymm5, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpor %ymm0, %ymm4, %ymm0
-; AVX512VL-NEXT:    vpsllw %xmm3, %ymm1, %ymm3
+; AVX512VL-NEXT:    vpand %ymm5, %ymm2, %ymm2
+; AVX512VL-NEXT:    vpor %ymm2, %ymm4, %ymm2
+; AVX512VL-NEXT:    vpsllw %xmm3, %ymm0, %ymm3
 ; AVX512VL-NEXT:    vpand %ymm6, %ymm3, %ymm3
-; AVX512VL-NEXT:    vpsrlw %xmm2, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpand %ymm5, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpor %ymm1, %ymm3, %ymm1
+; AVX512VL-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpand %ymm5, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpor %ymm0, %ymm3, %ymm0
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatvar_funnnel_v64i8:
@@ -508,34 +520,38 @@ define <16 x i32> @constant_funnnel_v16i32(<16 x i32> %x) nounwind {
 define <32 x i16> @constant_funnnel_v32i16(<32 x i16> %x) nounwind {
 ; AVX512F-LABEL: constant_funnnel_v32i16:
 ; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm2 = <u,2,4,8,16,32,64,128,256,512,1024,2048,4096,8192,16384,32768>
-; AVX512F-NEXT:    vpmulhuw %ymm2, %ymm0, %ymm3
-; AVX512F-NEXT:    vpblendw {{.*#+}} ymm4 = ymm0[0],ymm3[1,2,3,4,5,6,7],ymm0[8],ymm3[9,10,11,12,13,14,15]
+; AVX512F-NEXT:    vpmulhuw %ymm2, %ymm1, %ymm3
+; AVX512F-NEXT:    vpblendw {{.*#+}} ymm4 = ymm1[0],ymm3[1,2,3,4,5,6,7],ymm1[8],ymm3[9,10,11,12,13,14,15]
 ; AVX512F-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm4 = [1,2,4,8,16,32,64,128,256,512,1024,2048,4096,8192,16384,32768]
-; AVX512F-NEXT:    vpmullw %ymm4, %ymm0, %ymm0
-; AVX512F-NEXT:    vpor %ymm3, %ymm0, %ymm0
-; AVX512F-NEXT:    vpmulhuw %ymm2, %ymm1, %ymm2
-; AVX512F-NEXT:    vpblendw {{.*#+}} ymm3 = ymm1[0],ymm2[1,2,3,4,5,6,7],ymm1[8],ymm2[9,10,11,12,13,14,15]
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
 ; AVX512F-NEXT:    vpmullw %ymm4, %ymm1, %ymm1
-; AVX512F-NEXT:    vpor %ymm2, %ymm1, %ymm1
+; AVX512F-NEXT:    vpor %ymm3, %ymm1, %ymm1
+; AVX512F-NEXT:    vpmulhuw %ymm2, %ymm0, %ymm2
+; AVX512F-NEXT:    vpblendw {{.*#+}} ymm3 = ymm0[0],ymm2[1,2,3,4,5,6,7],ymm0[8],ymm2[9,10,11,12,13,14,15]
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
+; AVX512F-NEXT:    vpmullw %ymm4, %ymm0, %ymm0
+; AVX512F-NEXT:    vpor %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: constant_funnnel_v32i16:
 ; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm2 = <u,2,4,8,16,32,64,128,256,512,1024,2048,4096,8192,16384,32768>
-; AVX512VL-NEXT:    vpmulhuw %ymm2, %ymm0, %ymm3
-; AVX512VL-NEXT:    vpblendw {{.*#+}} ymm4 = ymm0[0],ymm3[1,2,3,4,5,6,7],ymm0[8],ymm3[9,10,11,12,13,14,15]
+; AVX512VL-NEXT:    vpmulhuw %ymm2, %ymm1, %ymm3
+; AVX512VL-NEXT:    vpblendw {{.*#+}} ymm4 = ymm1[0],ymm3[1,2,3,4,5,6,7],ymm1[8],ymm3[9,10,11,12,13,14,15]
 ; AVX512VL-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm4 = [1,2,4,8,16,32,64,128,256,512,1024,2048,4096,8192,16384,32768]
-; AVX512VL-NEXT:    vpmullw %ymm4, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpor %ymm3, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpmulhuw %ymm2, %ymm1, %ymm2
-; AVX512VL-NEXT:    vpblendw {{.*#+}} ymm3 = ymm1[0],ymm2[1,2,3,4,5,6,7],ymm1[8],ymm2[9,10,11,12,13,14,15]
-; AVX512VL-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
 ; AVX512VL-NEXT:    vpmullw %ymm4, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpor %ymm2, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpor %ymm3, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpmulhuw %ymm2, %ymm0, %ymm2
+; AVX512VL-NEXT:    vpblendw {{.*#+}} ymm3 = ymm0[0],ymm2[1,2,3,4,5,6,7],ymm0[8],ymm2[9,10,11,12,13,14,15]
+; AVX512VL-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
+; AVX512VL-NEXT:    vpmullw %ymm4, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpor %ymm2, %ymm0, %ymm0
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: constant_funnnel_v32i16:
@@ -558,12 +574,13 @@ define <32 x i16> @constant_funnnel_v32i16(<32 x i16> %x) nounwind {
 define <64 x i8> @constant_funnnel_v64i8(<64 x i8> %x) nounwind {
 ; AVX512F-LABEL: constant_funnnel_v64i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
+; AVX512F-NEXT:    vpsllw $4, %ymm1, %ymm2
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
 ; AVX512F-NEXT:    vpand %ymm3, %ymm2, %ymm2
 ; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm4 = [8192,24640,41088,57536,57344,41152,24704,8256,8192,24640,41088,57536,57344,41152,24704,8256]
 ; AVX512F-NEXT:    # ymm4 = mem[0,1,0,1]
-; AVX512F-NEXT:    vpblendvb %ymm4, %ymm2, %ymm0, %ymm2
+; AVX512F-NEXT:    vpblendvb %ymm4, %ymm2, %ymm1, %ymm2
 ; AVX512F-NEXT:    vpsllw $2, %ymm2, %ymm5
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm6 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
 ; AVX512F-NEXT:    vpand %ymm6, %ymm5, %ymm5
@@ -573,44 +590,46 @@ define <64 x i8> @constant_funnnel_v64i8(<64 x i8> %x) nounwind {
 ; AVX512F-NEXT:    vpaddb %ymm7, %ymm7, %ymm8
 ; AVX512F-NEXT:    vpblendvb %ymm8, %ymm5, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpxor %xmm5, %xmm5, %xmm5
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm9 = ymm0[8],ymm5[8],ymm0[9],ymm5[9],ymm0[10],ymm5[10],ymm0[11],ymm5[11],ymm0[12],ymm5[12],ymm0[13],ymm5[13],ymm0[14],ymm5[14],ymm0[15],ymm5[15],ymm0[24],ymm5[24],ymm0[25],ymm5[25],ymm0[26],ymm5[26],ymm0[27],ymm5[27],ymm0[28],ymm5[28],ymm0[29],ymm5[29],ymm0[30],ymm5[30],ymm0[31],ymm5[31]
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm9 = ymm1[8],ymm5[8],ymm1[9],ymm5[9],ymm1[10],ymm5[10],ymm1[11],ymm5[11],ymm1[12],ymm5[12],ymm1[13],ymm5[13],ymm1[14],ymm5[14],ymm1[15],ymm5[15],ymm1[24],ymm5[24],ymm1[25],ymm5[25],ymm1[26],ymm5[26],ymm1[27],ymm5[27],ymm1[28],ymm5[28],ymm1[29],ymm5[29],ymm1[30],ymm5[30],ymm1[31],ymm5[31]
 ; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm10 = [256,128,64,32,16,8,4,2,256,128,64,32,16,8,4,2]
 ; AVX512F-NEXT:    # ymm10 = mem[0,1,0,1]
 ; AVX512F-NEXT:    vpmullw %ymm10, %ymm9, %ymm9
 ; AVX512F-NEXT:    vpsrlw $8, %ymm9, %ymm9
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0],ymm5[0],ymm0[1],ymm5[1],ymm0[2],ymm5[2],ymm0[3],ymm5[3],ymm0[4],ymm5[4],ymm0[5],ymm5[5],ymm0[6],ymm5[6],ymm0[7],ymm5[7],ymm0[16],ymm5[16],ymm0[17],ymm5[17],ymm0[18],ymm5[18],ymm0[19],ymm5[19],ymm0[20],ymm5[20],ymm0[21],ymm5[21],ymm0[22],ymm5[22],ymm0[23],ymm5[23]
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm5[0],ymm1[1],ymm5[1],ymm1[2],ymm5[2],ymm1[3],ymm5[3],ymm1[4],ymm5[4],ymm1[5],ymm5[5],ymm1[6],ymm5[6],ymm1[7],ymm5[7],ymm1[16],ymm5[16],ymm1[17],ymm5[17],ymm1[18],ymm5[18],ymm1[19],ymm5[19],ymm1[20],ymm5[20],ymm1[21],ymm5[21],ymm1[22],ymm5[22],ymm1[23],ymm5[23]
 ; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm11 = [256,2,4,8,16,32,64,128,256,2,4,8,16,32,64,128]
 ; AVX512F-NEXT:    # ymm11 = mem[0,1,0,1]
-; AVX512F-NEXT:    vpmullw %ymm11, %ymm0, %ymm0
-; AVX512F-NEXT:    vpsrlw $8, %ymm0, %ymm0
-; AVX512F-NEXT:    vpackuswb %ymm9, %ymm0, %ymm0
-; AVX512F-NEXT:    vpor %ymm0, %ymm2, %ymm0
-; AVX512F-NEXT:    vpsllw $4, %ymm1, %ymm2
+; AVX512F-NEXT:    vpmullw %ymm1, %ymm11, %ymm1
+; AVX512F-NEXT:    vpsrlw $8, %ymm1, %ymm1
+; AVX512F-NEXT:    vpackuswb %ymm9, %ymm1, %ymm1
+; AVX512F-NEXT:    vpor %ymm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm2
 ; AVX512F-NEXT:    vpand %ymm3, %ymm2, %ymm2
-; AVX512F-NEXT:    vpblendvb %ymm4, %ymm2, %ymm1, %ymm2
+; AVX512F-NEXT:    vpblendvb %ymm4, %ymm2, %ymm0, %ymm2
 ; AVX512F-NEXT:    vpsllw $2, %ymm2, %ymm3
 ; AVX512F-NEXT:    vpand %ymm6, %ymm3, %ymm3
 ; AVX512F-NEXT:    vpblendvb %ymm7, %ymm3, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpaddb %ymm2, %ymm2, %ymm3
 ; AVX512F-NEXT:    vpblendvb %ymm8, %ymm3, %ymm2, %ymm2
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm1[8],ymm5[8],ymm1[9],ymm5[9],ymm1[10],ymm5[10],ymm1[11],ymm5[11],ymm1[12],ymm5[12],ymm1[13],ymm5[13],ymm1[14],ymm5[14],ymm1[15],ymm5[15],ymm1[24],ymm5[24],ymm1[25],ymm5[25],ymm1[26],ymm5[26],ymm1[27],ymm5[27],ymm1[28],ymm5[28],ymm1[29],ymm5[29],ymm1[30],ymm5[30],ymm1[31],ymm5[31]
-; AVX512F-NEXT:    vpmullw %ymm10, %ymm3, %ymm3
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm0[8],ymm5[8],ymm0[9],ymm5[9],ymm0[10],ymm5[10],ymm0[11],ymm5[11],ymm0[12],ymm5[12],ymm0[13],ymm5[13],ymm0[14],ymm5[14],ymm0[15],ymm5[15],ymm0[24],ymm5[24],ymm0[25],ymm5[25],ymm0[26],ymm5[26],ymm0[27],ymm5[27],ymm0[28],ymm5[28],ymm0[29],ymm5[29],ymm0[30],ymm5[30],ymm0[31],ymm5[31]
+; AVX512F-NEXT:    vpmullw %ymm3, %ymm10, %ymm3
 ; AVX512F-NEXT:    vpsrlw $8, %ymm3, %ymm3
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm5[0],ymm1[1],ymm5[1],ymm1[2],ymm5[2],ymm1[3],ymm5[3],ymm1[4],ymm5[4],ymm1[5],ymm5[5],ymm1[6],ymm5[6],ymm1[7],ymm5[7],ymm1[16],ymm5[16],ymm1[17],ymm5[17],ymm1[18],ymm5[18],ymm1[19],ymm5[19],ymm1[20],ymm5[20],ymm1[21],ymm5[21],ymm1[22],ymm5[22],ymm1[23],ymm5[23]
-; AVX512F-NEXT:    vpmullw %ymm11, %ymm1, %ymm1
-; AVX512F-NEXT:    vpsrlw $8, %ymm1, %ymm1
-; AVX512F-NEXT:    vpackuswb %ymm3, %ymm1, %ymm1
-; AVX512F-NEXT:    vpor %ymm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0],ymm5[0],ymm0[1],ymm5[1],ymm0[2],ymm5[2],ymm0[3],ymm5[3],ymm0[4],ymm5[4],ymm0[5],ymm5[5],ymm0[6],ymm5[6],ymm0[7],ymm5[7],ymm0[16],ymm5[16],ymm0[17],ymm5[17],ymm0[18],ymm5[18],ymm0[19],ymm5[19],ymm0[20],ymm5[20],ymm0[21],ymm5[21],ymm0[22],ymm5[22],ymm0[23],ymm5[23]
+; AVX512F-NEXT:    vpmullw %ymm0, %ymm11, %ymm0
+; AVX512F-NEXT:    vpsrlw $8, %ymm0, %ymm0
+; AVX512F-NEXT:    vpackuswb %ymm3, %ymm0, %ymm0
+; AVX512F-NEXT:    vpor %ymm0, %ymm2, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: constant_funnnel_v64i8:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm2
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
+; AVX512VL-NEXT:    vpsllw $4, %ymm1, %ymm2
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
 ; AVX512VL-NEXT:    vpand %ymm3, %ymm2, %ymm2
 ; AVX512VL-NEXT:    vbroadcasti128 {{.*#+}} ymm4 = [8192,24640,41088,57536,57344,41152,24704,8256,8192,24640,41088,57536,57344,41152,24704,8256]
 ; AVX512VL-NEXT:    # ymm4 = mem[0,1,0,1]
-; AVX512VL-NEXT:    vpblendvb %ymm4, %ymm2, %ymm0, %ymm2
+; AVX512VL-NEXT:    vpblendvb %ymm4, %ymm2, %ymm1, %ymm2
 ; AVX512VL-NEXT:    vpsllw $2, %ymm2, %ymm5
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm6 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
 ; AVX512VL-NEXT:    vpand %ymm6, %ymm5, %ymm5
@@ -619,38 +638,39 @@ define <64 x i8> @constant_funnnel_v64i8(<64 x i8> %x) nounwind {
 ; AVX512VL-NEXT:    vpaddb %ymm2, %ymm2, %ymm5
 ; AVX512VL-NEXT:    vpaddb %ymm7, %ymm7, %ymm8
 ; AVX512VL-NEXT:    vpblendvb %ymm8, %ymm5, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX512VL-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
 ; AVX512VL-NEXT:    vpsrlw $8, %ymm5, %ymm5
 ; AVX512VL-NEXT:    vbroadcasti128 {{.*#+}} ymm9 = [256,128,64,32,16,8,4,2,256,128,64,32,16,8,4,2]
 ; AVX512VL-NEXT:    # ymm9 = mem[0,1,0,1]
-; AVX512VL-NEXT:    vpmullw %ymm9, %ymm5, %ymm5
+; AVX512VL-NEXT:    vpmullw %ymm5, %ymm9, %ymm5
 ; AVX512VL-NEXT:    vpsrlw $8, %ymm5, %ymm5
-; AVX512VL-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512VL-NEXT:    vpsrlw $8, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512VL-NEXT:    vpsrlw $8, %ymm1, %ymm1
 ; AVX512VL-NEXT:    vbroadcasti128 {{.*#+}} ymm10 = [256,2,4,8,16,32,64,128,256,2,4,8,16,32,64,128]
 ; AVX512VL-NEXT:    # ymm10 = mem[0,1,0,1]
-; AVX512VL-NEXT:    vpmullw %ymm10, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpsrlw $8, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpackuswb %ymm5, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpor %ymm0, %ymm2, %ymm0
-; AVX512VL-NEXT:    vpsllw $4, %ymm1, %ymm2
+; AVX512VL-NEXT:    vpmullw %ymm1, %ymm10, %ymm1
+; AVX512VL-NEXT:    vpsrlw $8, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpackuswb %ymm5, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpor %ymm1, %ymm2, %ymm1
+; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm2
 ; AVX512VL-NEXT:    vpand %ymm3, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpblendvb %ymm4, %ymm2, %ymm1, %ymm2
+; AVX512VL-NEXT:    vpblendvb %ymm4, %ymm2, %ymm0, %ymm2
 ; AVX512VL-NEXT:    vpsllw $2, %ymm2, %ymm3
 ; AVX512VL-NEXT:    vpand %ymm6, %ymm3, %ymm3
 ; AVX512VL-NEXT:    vpblendvb %ymm7, %ymm3, %ymm2, %ymm2
 ; AVX512VL-NEXT:    vpaddb %ymm2, %ymm2, %ymm3
 ; AVX512VL-NEXT:    vpblendvb %ymm8, %ymm3, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX512VL-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
 ; AVX512VL-NEXT:    vpsrlw $8, %ymm3, %ymm3
-; AVX512VL-NEXT:    vpmullw %ymm9, %ymm3, %ymm3
+; AVX512VL-NEXT:    vpmullw %ymm3, %ymm9, %ymm3
 ; AVX512VL-NEXT:    vpsrlw $8, %ymm3, %ymm3
-; AVX512VL-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512VL-NEXT:    vpsrlw $8, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpmullw %ymm10, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpsrlw $8, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpackuswb %ymm3, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpor %ymm1, %ymm2, %ymm1
+; AVX512VL-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512VL-NEXT:    vpsrlw $8, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpmullw %ymm0, %ymm10, %ymm0
+; AVX512VL-NEXT:    vpsrlw $8, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpackuswb %ymm3, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpor %ymm0, %ymm2, %ymm0
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: constant_funnnel_v64i8:
@@ -737,22 +757,26 @@ define <16 x i32> @splatconstant_funnnel_v16i32(<16 x i32> %x) nounwind {
 define <32 x i16> @splatconstant_funnnel_v32i16(<32 x i16> %x) nounwind {
 ; AVX512F-LABEL: splatconstant_funnnel_v32i16:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpsrlw $9, %ymm0, %ymm2
-; AVX512F-NEXT:    vpsllw $7, %ymm0, %ymm0
-; AVX512F-NEXT:    vpor %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512F-NEXT:    vpsrlw $9, %ymm1, %ymm2
 ; AVX512F-NEXT:    vpsllw $7, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpor %ymm2, %ymm1, %ymm1
+; AVX512F-NEXT:    vpsrlw $9, %ymm0, %ymm2
+; AVX512F-NEXT:    vpsllw $7, %ymm0, %ymm0
+; AVX512F-NEXT:    vpor %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatconstant_funnnel_v32i16:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsrlw $9, %ymm0, %ymm2
-; AVX512VL-NEXT:    vpsllw $7, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpor %ymm2, %ymm0, %ymm0
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512VL-NEXT:    vpsrlw $9, %ymm1, %ymm2
 ; AVX512VL-NEXT:    vpsllw $7, %ymm1, %ymm1
 ; AVX512VL-NEXT:    vpor %ymm2, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpsrlw $9, %ymm0, %ymm2
+; AVX512VL-NEXT:    vpsllw $7, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpor %ymm2, %ymm0, %ymm0
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatconstant_funnnel_v32i16:
@@ -775,50 +799,46 @@ define <32 x i16> @splatconstant_funnnel_v32i16(<32 x i16> %x) nounwind {
 define <64 x i8> @splatconstant_funnnel_v64i8(<64 x i8> %x) nounwind {
 ; AVX512F-LABEL: splatconstant_funnnel_v64i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpsrlw $4, %ymm0, %ymm2
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX512F-NEXT:    vpandn %ymm2, %ymm3, %ymm2
-; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm0
-; AVX512F-NEXT:    vpand %ymm3, %ymm0, %ymm0
-; AVX512F-NEXT:    vpor %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512F-NEXT:    vpsrlw $4, %ymm1, %ymm2
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
 ; AVX512F-NEXT:    vpandn %ymm2, %ymm3, %ymm2
 ; AVX512F-NEXT:    vpsllw $4, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpand %ymm3, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpor %ymm2, %ymm1, %ymm1
+; AVX512F-NEXT:    vpsrlw $4, %ymm0, %ymm2
+; AVX512F-NEXT:    vpandn %ymm2, %ymm3, %ymm2
+; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm0
+; AVX512F-NEXT:    vpand %ymm3, %ymm0, %ymm0
+; AVX512F-NEXT:    vpor %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatconstant_funnnel_v64i8:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm2
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX512VL-NEXT:    vpandn %ymm2, %ymm3, %ymm2
-; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpand %ymm3, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpor %ymm2, %ymm0, %ymm0
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512VL-NEXT:    vpsrlw $4, %ymm1, %ymm2
-; AVX512VL-NEXT:    vpandn %ymm2, %ymm3, %ymm2
 ; AVX512VL-NEXT:    vpsllw $4, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpand %ymm3, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpor %ymm2, %ymm1, %ymm1
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
+; AVX512VL-NEXT:    vpternlogq $226, %ymm2, %ymm3, %ymm1
+; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm2
+; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpternlogq $226, %ymm2, %ymm3, %ymm0
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatconstant_funnnel_v64i8:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpsrlw $4, %zmm0, %zmm1
-; AVX512BW-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
-; AVX512BW-NEXT:    vpsllw $4, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
-; AVX512BW-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsllw $4, %zmm0, %zmm1
+; AVX512BW-NEXT:    vpsrlw $4, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpternlogq $216, {{.*}}(%rip), %zmm1, %zmm0
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: splatconstant_funnnel_v64i8:
 ; AVX512VLBW:       # %bb.0:
-; AVX512VLBW-NEXT:    vpsrlw $4, %zmm0, %zmm1
-; AVX512VLBW-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
-; AVX512VLBW-NEXT:    vpsllw $4, %zmm0, %zmm0
-; AVX512VLBW-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
-; AVX512VLBW-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512VLBW-NEXT:    vpsllw $4, %zmm0, %zmm1
+; AVX512VLBW-NEXT:    vpsrlw $4, %zmm0, %zmm0
+; AVX512VLBW-NEXT:    vpternlogq $216, {{.*}}(%rip), %zmm1, %zmm0
 ; AVX512VLBW-NEXT:    retq
   %res = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %x, <64 x i8> %x, <64 x i8> <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>)
   ret <64 x i8> %res
diff --git a/llvm/test/CodeGen/X86/vector-fshr-128.ll b/llvm/test/CodeGen/X86/vector-fshr-128.ll
index 29035a5e3..ee982383c 100644
--- a/llvm/test/CodeGen/X86/vector-fshr-128.ll
+++ b/llvm/test/CodeGen/X86/vector-fshr-128.ll
@@ -505,114 +505,113 @@ define <8 x i16> @var_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %amt)
 ; SSE2-LABEL: var_funnnel_v8i16:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pand {{.*}}(%rip), %xmm2
-; SSE2-NEXT:    movdqa %xmm2, %xmm4
-; SSE2-NEXT:    psllw $12, %xmm4
-; SSE2-NEXT:    movdqa %xmm4, %xmm3
-; SSE2-NEXT:    psraw $15, %xmm3
-; SSE2-NEXT:    movdqa %xmm1, %xmm5
-; SSE2-NEXT:    psrlw $8, %xmm5
-; SSE2-NEXT:    pand %xmm3, %xmm5
-; SSE2-NEXT:    pandn %xmm1, %xmm3
-; SSE2-NEXT:    por %xmm5, %xmm3
-; SSE2-NEXT:    paddw %xmm4, %xmm4
-; SSE2-NEXT:    movdqa %xmm4, %xmm5
-; SSE2-NEXT:    psraw $15, %xmm5
-; SSE2-NEXT:    movdqa %xmm5, %xmm6
-; SSE2-NEXT:    pandn %xmm3, %xmm6
-; SSE2-NEXT:    psrlw $4, %xmm3
-; SSE2-NEXT:    pand %xmm5, %xmm3
-; SSE2-NEXT:    por %xmm6, %xmm3
-; SSE2-NEXT:    paddw %xmm4, %xmm4
-; SSE2-NEXT:    movdqa %xmm4, %xmm5
-; SSE2-NEXT:    psraw $15, %xmm5
-; SSE2-NEXT:    movdqa %xmm5, %xmm6
-; SSE2-NEXT:    pandn %xmm3, %xmm6
-; SSE2-NEXT:    psrlw $2, %xmm3
-; SSE2-NEXT:    pand %xmm5, %xmm3
-; SSE2-NEXT:    por %xmm6, %xmm3
-; SSE2-NEXT:    paddw %xmm4, %xmm4
-; SSE2-NEXT:    psraw $15, %xmm4
-; SSE2-NEXT:    movdqa %xmm4, %xmm5
-; SSE2-NEXT:    pandn %xmm3, %xmm5
-; SSE2-NEXT:    psrlw $1, %xmm3
-; SSE2-NEXT:    pand %xmm4, %xmm3
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [16,16,16,16,16,16,16,16]
 ; SSE2-NEXT:    psubw %xmm2, %xmm4
-; SSE2-NEXT:    pxor %xmm8, %xmm8
-; SSE2-NEXT:    movdqa %xmm4, %xmm7
-; SSE2-NEXT:    punpckhwd {{.*#+}} xmm7 = xmm7[4],xmm8[4],xmm7[5],xmm8[5],xmm7[6],xmm8[6],xmm7[7],xmm8[7]
-; SSE2-NEXT:    pslld $23, %xmm7
-; SSE2-NEXT:    movdqa {{.*#+}} xmm6 = [1065353216,1065353216,1065353216,1065353216]
-; SSE2-NEXT:    paddd %xmm6, %xmm7
-; SSE2-NEXT:    cvttps2dq %xmm7, %xmm7
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm7 = xmm7[0,2,2,3,4,5,6,7]
-; SSE2-NEXT:    pshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,4,6,6,7]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[0,2,2,3]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm8[0],xmm4[1],xmm8[1],xmm4[2],xmm8[2],xmm4[3],xmm8[3]
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    pcmpeqw %xmm2, %xmm3
+; SSE2-NEXT:    psllw $12, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm5
+; SSE2-NEXT:    psraw $15, %xmm5
+; SSE2-NEXT:    movdqa %xmm1, %xmm6
+; SSE2-NEXT:    psrlw $8, %xmm6
+; SSE2-NEXT:    pand %xmm5, %xmm6
+; SSE2-NEXT:    pandn %xmm1, %xmm5
+; SSE2-NEXT:    por %xmm6, %xmm5
+; SSE2-NEXT:    paddw %xmm2, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm6
+; SSE2-NEXT:    psraw $15, %xmm6
+; SSE2-NEXT:    movdqa %xmm6, %xmm7
+; SSE2-NEXT:    pandn %xmm5, %xmm7
+; SSE2-NEXT:    psrlw $4, %xmm5
+; SSE2-NEXT:    pand %xmm6, %xmm5
+; SSE2-NEXT:    por %xmm7, %xmm5
+; SSE2-NEXT:    paddw %xmm2, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm6
+; SSE2-NEXT:    psraw $15, %xmm6
+; SSE2-NEXT:    movdqa %xmm6, %xmm7
+; SSE2-NEXT:    pandn %xmm5, %xmm7
+; SSE2-NEXT:    psrlw $2, %xmm5
+; SSE2-NEXT:    pand %xmm6, %xmm5
+; SSE2-NEXT:    por %xmm7, %xmm5
+; SSE2-NEXT:    paddw %xmm2, %xmm2
+; SSE2-NEXT:    psraw $15, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm6
+; SSE2-NEXT:    pandn %xmm5, %xmm6
+; SSE2-NEXT:    psrlw $1, %xmm5
+; SSE2-NEXT:    pand %xmm2, %xmm5
+; SSE2-NEXT:    movdqa %xmm4, %xmm2
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE2-NEXT:    pslld $23, %xmm2
+; SSE2-NEXT:    movdqa {{.*#+}} xmm7 = [1065353216,1065353216,1065353216,1065353216]
+; SSE2-NEXT:    paddd %xmm7, %xmm2
+; SSE2-NEXT:    cvttps2dq %xmm2, %xmm2
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,2,2,3,4,5,6,7]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,6,6,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
 ; SSE2-NEXT:    pslld $23, %xmm4
-; SSE2-NEXT:    paddd %xmm6, %xmm4
+; SSE2-NEXT:    paddd %xmm7, %xmm4
 ; SSE2-NEXT:    cvttps2dq %xmm4, %xmm4
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm4 = xmm4[0,2,2,3,4,5,6,7]
 ; SSE2-NEXT:    pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,6,6,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[0,2,2,3]
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm7[0]
+; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm2[0]
 ; SSE2-NEXT:    pmullw %xmm0, %xmm4
+; SSE2-NEXT:    por %xmm6, %xmm4
 ; SSE2-NEXT:    por %xmm5, %xmm4
-; SSE2-NEXT:    por %xmm3, %xmm4
-; SSE2-NEXT:    pcmpeqw %xmm8, %xmm2
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pandn %xmm4, %xmm2
-; SSE2-NEXT:    por %xmm1, %xmm2
-; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    pand %xmm3, %xmm1
+; SSE2-NEXT:    pandn %xmm4, %xmm3
+; SSE2-NEXT:    por %xmm1, %xmm3
+; SSE2-NEXT:    movdqa %xmm3, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: var_funnnel_v8i16:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa %xmm0, %xmm8
+; SSE41-NEXT:    movdqa %xmm0, %xmm3
 ; SSE41-NEXT:    pand {{.*}}(%rip), %xmm2
 ; SSE41-NEXT:    movdqa %xmm2, %xmm0
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm5 = [16,16,16,16,16,16,16,16]
 ; SSE41-NEXT:    psubw %xmm2, %xmm5
 ; SSE41-NEXT:    pxor %xmm4, %xmm4
-; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm6 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero
-; SSE41-NEXT:    punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]
 ; SSE41-NEXT:    pcmpeqw %xmm2, %xmm4
 ; SSE41-NEXT:    psllw $12, %xmm2
 ; SSE41-NEXT:    psllw $4, %xmm0
 ; SSE41-NEXT:    por %xmm2, %xmm0
 ; SSE41-NEXT:    movdqa %xmm0, %xmm2
 ; SSE41-NEXT:    paddw %xmm0, %xmm2
+; SSE41-NEXT:    movdqa %xmm1, %xmm6
+; SSE41-NEXT:    psrlw $8, %xmm6
 ; SSE41-NEXT:    movdqa %xmm1, %xmm7
-; SSE41-NEXT:    psrlw $8, %xmm7
-; SSE41-NEXT:    movdqa %xmm1, %xmm3
-; SSE41-NEXT:    pblendvb %xmm0, %xmm7, %xmm3
-; SSE41-NEXT:    movdqa %xmm3, %xmm7
-; SSE41-NEXT:    psrlw $4, %xmm7
+; SSE41-NEXT:    pblendvb %xmm0, %xmm6, %xmm7
+; SSE41-NEXT:    movdqa %xmm7, %xmm6
+; SSE41-NEXT:    psrlw $4, %xmm6
 ; SSE41-NEXT:    movdqa %xmm2, %xmm0
-; SSE41-NEXT:    pblendvb %xmm0, %xmm7, %xmm3
-; SSE41-NEXT:    movdqa %xmm3, %xmm7
-; SSE41-NEXT:    psrlw $2, %xmm7
+; SSE41-NEXT:    pblendvb %xmm0, %xmm6, %xmm7
+; SSE41-NEXT:    movdqa %xmm7, %xmm6
+; SSE41-NEXT:    psrlw $2, %xmm6
 ; SSE41-NEXT:    paddw %xmm2, %xmm2
 ; SSE41-NEXT:    movdqa %xmm2, %xmm0
-; SSE41-NEXT:    pblendvb %xmm0, %xmm7, %xmm3
-; SSE41-NEXT:    movdqa %xmm3, %xmm7
-; SSE41-NEXT:    psrlw $1, %xmm7
+; SSE41-NEXT:    pblendvb %xmm0, %xmm6, %xmm7
+; SSE41-NEXT:    movdqa %xmm7, %xmm6
+; SSE41-NEXT:    psrlw $1, %xmm6
 ; SSE41-NEXT:    paddw %xmm2, %xmm2
 ; SSE41-NEXT:    movdqa %xmm2, %xmm0
-; SSE41-NEXT:    pblendvb %xmm0, %xmm7, %xmm3
+; SSE41-NEXT:    pblendvb %xmm0, %xmm6, %xmm7
+; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero
+; SSE41-NEXT:    punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm0[4],xmm5[5],xmm0[5],xmm5[6],xmm0[6],xmm5[7],xmm0[7]
 ; SSE41-NEXT:    pslld $23, %xmm5
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [1065353216,1065353216,1065353216,1065353216]
-; SSE41-NEXT:    paddd %xmm0, %xmm5
-; SSE41-NEXT:    cvttps2dq %xmm5, %xmm2
-; SSE41-NEXT:    pslld $23, %xmm6
-; SSE41-NEXT:    paddd %xmm0, %xmm6
-; SSE41-NEXT:    cvttps2dq %xmm6, %xmm0
-; SSE41-NEXT:    packusdw %xmm2, %xmm0
-; SSE41-NEXT:    pmullw %xmm0, %xmm8
-; SSE41-NEXT:    por %xmm3, %xmm8
+; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [1065353216,1065353216,1065353216,1065353216]
+; SSE41-NEXT:    paddd %xmm2, %xmm5
+; SSE41-NEXT:    cvttps2dq %xmm5, %xmm5
+; SSE41-NEXT:    pslld $23, %xmm0
+; SSE41-NEXT:    paddd %xmm2, %xmm0
+; SSE41-NEXT:    cvttps2dq %xmm0, %xmm0
+; SSE41-NEXT:    packusdw %xmm5, %xmm0
+; SSE41-NEXT:    pmullw %xmm0, %xmm3
+; SSE41-NEXT:    por %xmm7, %xmm3
 ; SSE41-NEXT:    movdqa %xmm4, %xmm0
-; SSE41-NEXT:    pblendvb %xmm0, %xmm1, %xmm8
-; SSE41-NEXT:    movdqa %xmm8, %xmm0
+; SSE41-NEXT:    pblendvb %xmm0, %xmm1, %xmm3
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: var_funnnel_v8i16:
@@ -634,20 +633,20 @@ define <8 x i16> @var_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %amt)
 ; AVX1-NEXT:    vpblendvb %xmm4, %xmm5, %xmm3, %xmm3
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [16,16,16,16,16,16,16,16]
 ; AVX1-NEXT:    vpsubw %xmm2, %xmm4, %xmm4
-; AVX1-NEXT:    vpxor %xmm5, %xmm5, %xmm5
-; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm6 = xmm4[4],xmm5[4],xmm4[5],xmm5[5],xmm4[6],xmm5[6],xmm4[7],xmm5[7]
-; AVX1-NEXT:    vpslld $23, %xmm6, %xmm6
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm7 = [1065353216,1065353216,1065353216,1065353216]
-; AVX1-NEXT:    vpaddd %xmm7, %xmm6, %xmm6
-; AVX1-NEXT:    vcvttps2dq %xmm6, %xmm6
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
+; AVX1-NEXT:    vpslld $23, %xmm5, %xmm5
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [1065353216,1065353216,1065353216,1065353216]
+; AVX1-NEXT:    vpaddd %xmm6, %xmm5, %xmm5
+; AVX1-NEXT:    vcvttps2dq %xmm5, %xmm5
 ; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero
 ; AVX1-NEXT:    vpslld $23, %xmm4, %xmm4
-; AVX1-NEXT:    vpaddd %xmm7, %xmm4, %xmm4
+; AVX1-NEXT:    vpaddd %xmm6, %xmm4, %xmm4
 ; AVX1-NEXT:    vcvttps2dq %xmm4, %xmm4
-; AVX1-NEXT:    vpackusdw %xmm6, %xmm4, %xmm4
+; AVX1-NEXT:    vpackusdw %xmm5, %xmm4, %xmm4
 ; AVX1-NEXT:    vpmullw %xmm4, %xmm0, %xmm0
 ; AVX1-NEXT:    vpor %xmm3, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpeqw %xmm5, %xmm2, %xmm2
+; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpeqw %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    retq
 ;
@@ -657,15 +656,14 @@ define <8 x i16> @var_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %amt)
 ; AVX2-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
 ; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
 ; AVX2-NEXT:    vpsrlvd %ymm4, %ymm3, %ymm3
-; AVX2-NEXT:    vmovdqa {{.*#+}} ymm4 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; AVX2-NEXT:    vpshufb %ymm4, %ymm3, %ymm3
-; AVX2-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,2,2,3]
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm5 = [16,16,16,16,16,16,16,16]
-; AVX2-NEXT:    vpsubw %xmm2, %xmm5, %xmm5
-; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm5 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
+; AVX2-NEXT:    vextracti128 $1, %ymm3, %xmm4
+; AVX2-NEXT:    vpackusdw %xmm4, %xmm3, %xmm3
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm4 = [16,16,16,16,16,16,16,16]
+; AVX2-NEXT:    vpsubw %xmm2, %xmm4, %xmm4
+; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero
 ; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX2-NEXT:    vpsllvd %ymm5, %ymm0, %ymm0
-; AVX2-NEXT:    vpshufb %ymm4, %ymm0, %ymm0
+; AVX2-NEXT:    vpsllvd %ymm4, %ymm0, %ymm0
+; AVX2-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
 ; AVX2-NEXT:    vpor %xmm3, %xmm0, %xmm0
 ; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
@@ -778,70 +776,65 @@ define <8 x i16> @var_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %amt)
 ;
 ; X32-SSE-LABEL: var_funnnel_v8i16:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    subl $28, %esp
-; X32-SSE-NEXT:    movups %xmm0, (%esp) # 16-byte Spill
 ; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm2
-; X32-SSE-NEXT:    movdqa %xmm2, %xmm4
-; X32-SSE-NEXT:    psllw $12, %xmm4
-; X32-SSE-NEXT:    movdqa %xmm4, %xmm3
-; X32-SSE-NEXT:    psraw $15, %xmm3
-; X32-SSE-NEXT:    movdqa %xmm1, %xmm5
-; X32-SSE-NEXT:    psrlw $8, %xmm5
-; X32-SSE-NEXT:    pand %xmm3, %xmm5
-; X32-SSE-NEXT:    pandn %xmm1, %xmm3
-; X32-SSE-NEXT:    por %xmm5, %xmm3
-; X32-SSE-NEXT:    paddw %xmm4, %xmm4
-; X32-SSE-NEXT:    movdqa %xmm4, %xmm5
-; X32-SSE-NEXT:    psraw $15, %xmm5
-; X32-SSE-NEXT:    movdqa %xmm5, %xmm6
-; X32-SSE-NEXT:    pandn %xmm3, %xmm6
-; X32-SSE-NEXT:    psrlw $4, %xmm3
-; X32-SSE-NEXT:    pand %xmm5, %xmm3
-; X32-SSE-NEXT:    por %xmm6, %xmm3
-; X32-SSE-NEXT:    paddw %xmm4, %xmm4
-; X32-SSE-NEXT:    movdqa %xmm4, %xmm5
-; X32-SSE-NEXT:    psraw $15, %xmm5
-; X32-SSE-NEXT:    movdqa %xmm5, %xmm6
-; X32-SSE-NEXT:    pandn %xmm3, %xmm6
-; X32-SSE-NEXT:    psrlw $2, %xmm3
-; X32-SSE-NEXT:    pand %xmm5, %xmm3
-; X32-SSE-NEXT:    por %xmm6, %xmm3
-; X32-SSE-NEXT:    paddw %xmm4, %xmm4
-; X32-SSE-NEXT:    psraw $15, %xmm4
-; X32-SSE-NEXT:    movdqa %xmm4, %xmm5
-; X32-SSE-NEXT:    pandn %xmm3, %xmm5
-; X32-SSE-NEXT:    psrlw $1, %xmm3
-; X32-SSE-NEXT:    pand %xmm4, %xmm3
 ; X32-SSE-NEXT:    movdqa {{.*#+}} xmm4 = [16,16,16,16,16,16,16,16]
 ; X32-SSE-NEXT:    psubw %xmm2, %xmm4
-; X32-SSE-NEXT:    pxor %xmm6, %xmm6
-; X32-SSE-NEXT:    movdqa %xmm4, %xmm7
-; X32-SSE-NEXT:    punpckhwd {{.*#+}} xmm7 = xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7]
-; X32-SSE-NEXT:    pslld $23, %xmm7
-; X32-SSE-NEXT:    movdqa {{.*#+}} xmm0 = [1065353216,1065353216,1065353216,1065353216]
-; X32-SSE-NEXT:    paddd %xmm0, %xmm7
-; X32-SSE-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm6[0],xmm4[1],xmm6[1],xmm4[2],xmm6[2],xmm4[3],xmm6[3]
+; X32-SSE-NEXT:    pxor %xmm3, %xmm3
+; X32-SSE-NEXT:    pcmpeqw %xmm2, %xmm3
+; X32-SSE-NEXT:    psllw $12, %xmm2
+; X32-SSE-NEXT:    movdqa %xmm2, %xmm5
+; X32-SSE-NEXT:    psraw $15, %xmm5
+; X32-SSE-NEXT:    movdqa %xmm1, %xmm6
+; X32-SSE-NEXT:    psrlw $8, %xmm6
+; X32-SSE-NEXT:    pand %xmm5, %xmm6
+; X32-SSE-NEXT:    pandn %xmm1, %xmm5
+; X32-SSE-NEXT:    por %xmm6, %xmm5
+; X32-SSE-NEXT:    paddw %xmm2, %xmm2
+; X32-SSE-NEXT:    movdqa %xmm2, %xmm6
+; X32-SSE-NEXT:    psraw $15, %xmm6
+; X32-SSE-NEXT:    movdqa %xmm6, %xmm7
+; X32-SSE-NEXT:    pandn %xmm5, %xmm7
+; X32-SSE-NEXT:    psrlw $4, %xmm5
+; X32-SSE-NEXT:    pand %xmm6, %xmm5
+; X32-SSE-NEXT:    por %xmm7, %xmm5
+; X32-SSE-NEXT:    paddw %xmm2, %xmm2
+; X32-SSE-NEXT:    movdqa %xmm2, %xmm6
+; X32-SSE-NEXT:    psraw $15, %xmm6
+; X32-SSE-NEXT:    movdqa %xmm6, %xmm7
+; X32-SSE-NEXT:    pandn %xmm5, %xmm7
+; X32-SSE-NEXT:    psrlw $2, %xmm5
+; X32-SSE-NEXT:    pand %xmm6, %xmm5
+; X32-SSE-NEXT:    por %xmm7, %xmm5
+; X32-SSE-NEXT:    paddw %xmm2, %xmm2
+; X32-SSE-NEXT:    psraw $15, %xmm2
+; X32-SSE-NEXT:    movdqa %xmm2, %xmm6
+; X32-SSE-NEXT:    pandn %xmm5, %xmm6
+; X32-SSE-NEXT:    psrlw $1, %xmm5
+; X32-SSE-NEXT:    pand %xmm2, %xmm5
+; X32-SSE-NEXT:    movdqa %xmm4, %xmm2
+; X32-SSE-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; X32-SSE-NEXT:    pslld $23, %xmm2
+; X32-SSE-NEXT:    movdqa {{.*#+}} xmm7 = [1065353216,1065353216,1065353216,1065353216]
+; X32-SSE-NEXT:    paddd %xmm7, %xmm2
+; X32-SSE-NEXT:    cvttps2dq %xmm2, %xmm2
+; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,2,2,3,4,5,6,7]
+; X32-SSE-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,6,6,7]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; X32-SSE-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
 ; X32-SSE-NEXT:    pslld $23, %xmm4
-; X32-SSE-NEXT:    paddd %xmm0, %xmm4
-; X32-SSE-NEXT:    cvttps2dq %xmm7, %xmm0
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
-; X32-SSE-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,6,6,7]
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X32-SSE-NEXT:    paddd %xmm7, %xmm4
 ; X32-SSE-NEXT:    cvttps2dq %xmm4, %xmm4
 ; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm4 = xmm4[0,2,2,3,4,5,6,7]
 ; X32-SSE-NEXT:    pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,6,6,7]
 ; X32-SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[0,2,2,3]
-; X32-SSE-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm0[0]
-; X32-SSE-NEXT:    movdqu (%esp), %xmm0 # 16-byte Reload
+; X32-SSE-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm2[0]
 ; X32-SSE-NEXT:    pmullw %xmm0, %xmm4
+; X32-SSE-NEXT:    por %xmm6, %xmm4
 ; X32-SSE-NEXT:    por %xmm5, %xmm4
-; X32-SSE-NEXT:    por %xmm3, %xmm4
-; X32-SSE-NEXT:    pcmpeqw %xmm6, %xmm2
-; X32-SSE-NEXT:    pand %xmm2, %xmm1
-; X32-SSE-NEXT:    pandn %xmm4, %xmm2
-; X32-SSE-NEXT:    por %xmm1, %xmm2
-; X32-SSE-NEXT:    movdqa %xmm2, %xmm0
-; X32-SSE-NEXT:    addl $28, %esp
+; X32-SSE-NEXT:    pand %xmm3, %xmm1
+; X32-SSE-NEXT:    pandn %xmm4, %xmm3
+; X32-SSE-NEXT:    por %xmm1, %xmm3
+; X32-SSE-NEXT:    movdqa %xmm3, %xmm0
 ; X32-SSE-NEXT:    retl
   %res = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %amt)
   ret <8 x i16> %res
@@ -3007,14 +3000,53 @@ define <16 x i8> @splatconstant_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y) nounwi
 ; AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
-; AVX512-LABEL: splatconstant_funnnel_v16i8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsrlw $4, %xmm1, %xmm1
-; AVX512-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
-; AVX512-NEXT:    vpsllw $4, %xmm0, %xmm0
-; AVX512-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: splatconstant_funnnel_v16i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsrlw $4, %xmm1, %xmm1
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512F-NEXT:    vpsllw $4, %xmm0, %xmm0
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX512F-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: splatconstant_funnnel_v16i8:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsllw $4, %xmm0, %xmm2
+; AVX512VL-NEXT:    vpsrlw $4, %xmm1, %xmm0
+; AVX512VL-NEXT:    vpternlogq $216, {{.*}}(%rip), %xmm2, %xmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: splatconstant_funnnel_v16i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsrlw $4, %xmm1, %xmm1
+; AVX512BW-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512BW-NEXT:    vpsllw $4, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX512BW-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: splatconstant_funnnel_v16i8:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpsrlw $4, %xmm1, %xmm1
+; AVX512VBMI2-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512VBMI2-NEXT:    vpsllw $4, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLBW-LABEL: splatconstant_funnnel_v16i8:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpsllw $4, %xmm0, %xmm2
+; AVX512VLBW-NEXT:    vpsrlw $4, %xmm1, %xmm0
+; AVX512VLBW-NEXT:    vpternlogq $216, {{.*}}(%rip), %xmm2, %xmm0
+; AVX512VLBW-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v16i8:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpsllw $4, %xmm0, %xmm2
+; AVX512VLVBMI2-NEXT:    vpsrlw $4, %xmm1, %xmm0
+; AVX512VLVBMI2-NEXT:    vpternlogq $216, {{.*}}(%rip), %xmm2, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
 ;
 ; XOP-LABEL: splatconstant_funnnel_v16i8:
 ; XOP:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/vector-fshr-256.ll b/llvm/test/CodeGen/X86/vector-fshr-256.ll
index 0f67bbec8..984588caf 100644
--- a/llvm/test/CodeGen/X86/vector-fshr-256.ll
+++ b/llvm/test/CodeGen/X86/vector-fshr-256.ll
@@ -231,8 +231,8 @@ define <8 x i32> @var_funnnel_v8i32(<8 x i32> %x, <8 x i32> %y, <8 x i32> %amt)
 ; AVX1-NEXT:    vpmulld %xmm6, %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm0, %ymm0
 ; AVX1-NEXT:    vorps %ymm4, %ymm0, %ymm0
-; AVX1-NEXT:    vpcmpeqd %xmm8, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpeqd %xmm8, %xmm2, %xmm2
+; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm8, %xmm3
+; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm8, %xmm2
 ; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
 ; AVX1-NEXT:    vblendvps %ymm2, %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
@@ -403,36 +403,36 @@ define <16 x i16> @var_funnnel_v16i16(<16 x i16> %x, <16 x i16> %y, <16 x i16> %
 ; AVX1-NEXT:    vpaddw %xmm6, %xmm6, %xmm6
 ; AVX1-NEXT:    vpblendvb %xmm6, %xmm7, %xmm5, %xmm5
 ; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm5, %ymm8
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm9 = [16,16,16,16,16,16,16,16]
-; AVX1-NEXT:    vpsubw %xmm3, %xmm9, %xmm6
-; AVX1-NEXT:    vpxor %xmm10, %xmm10, %xmm10
-; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm6[4],xmm10[4],xmm6[5],xmm10[5],xmm6[6],xmm10[6],xmm6[7],xmm10[7]
-; AVX1-NEXT:    vpslld $23, %xmm4, %xmm4
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm5 = [1065353216,1065353216,1065353216,1065353216]
-; AVX1-NEXT:    vpaddd %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vcvttps2dq %xmm4, %xmm4
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm5 = [16,16,16,16,16,16,16,16]
+; AVX1-NEXT:    vpsubw %xmm3, %xmm5, %xmm6
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm7 = xmm6[4],xmm0[4],xmm6[5],xmm0[5],xmm6[6],xmm0[6],xmm6[7],xmm0[7]
+; AVX1-NEXT:    vpslld $23, %xmm7, %xmm7
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [1065353216,1065353216,1065353216,1065353216]
+; AVX1-NEXT:    vpaddd %xmm4, %xmm7, %xmm7
+; AVX1-NEXT:    vcvttps2dq %xmm7, %xmm7
 ; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm6 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero
 ; AVX1-NEXT:    vpslld $23, %xmm6, %xmm6
-; AVX1-NEXT:    vpaddd %xmm5, %xmm6, %xmm6
+; AVX1-NEXT:    vpaddd %xmm4, %xmm6, %xmm6
 ; AVX1-NEXT:    vcvttps2dq %xmm6, %xmm6
-; AVX1-NEXT:    vpackusdw %xmm4, %xmm6, %xmm4
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm6
-; AVX1-NEXT:    vpmullw %xmm4, %xmm6, %xmm4
-; AVX1-NEXT:    vpsubw %xmm2, %xmm9, %xmm6
-; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm7 = xmm6[4],xmm10[4],xmm6[5],xmm10[5],xmm6[6],xmm10[6],xmm6[7],xmm10[7]
+; AVX1-NEXT:    vpackusdw %xmm7, %xmm6, %xmm6
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm7
+; AVX1-NEXT:    vpmullw %xmm6, %xmm7, %xmm6
+; AVX1-NEXT:    vpsubw %xmm2, %xmm5, %xmm5
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm7 = xmm5[4],xmm0[4],xmm5[5],xmm0[5],xmm5[6],xmm0[6],xmm5[7],xmm0[7]
 ; AVX1-NEXT:    vpslld $23, %xmm7, %xmm7
-; AVX1-NEXT:    vpaddd %xmm5, %xmm7, %xmm7
+; AVX1-NEXT:    vpaddd %xmm4, %xmm7, %xmm7
 ; AVX1-NEXT:    vcvttps2dq %xmm7, %xmm7
-; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm6 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero
-; AVX1-NEXT:    vpslld $23, %xmm6, %xmm6
-; AVX1-NEXT:    vpaddd %xmm5, %xmm6, %xmm5
-; AVX1-NEXT:    vcvttps2dq %xmm5, %xmm5
-; AVX1-NEXT:    vpackusdw %xmm7, %xmm5, %xmm5
-; AVX1-NEXT:    vpmullw %xmm5, %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
-; AVX1-NEXT:    vorps %ymm8, %ymm0, %ymm0
-; AVX1-NEXT:    vpcmpeqw %xmm10, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpeqw %xmm10, %xmm2, %xmm2
+; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm5 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero
+; AVX1-NEXT:    vpslld $23, %xmm5, %xmm5
+; AVX1-NEXT:    vpaddd %xmm4, %xmm5, %xmm4
+; AVX1-NEXT:    vcvttps2dq %xmm4, %xmm4
+; AVX1-NEXT:    vpackusdw %xmm7, %xmm4, %xmm4
+; AVX1-NEXT:    vpmullw %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm0, %ymm0
+; AVX1-NEXT:    vorps %ymm0, %ymm8, %ymm0
+; AVX1-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; AVX1-NEXT:    vpcmpeqw %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpeqw %xmm4, %xmm2, %xmm2
 ; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
 ; AVX1-NEXT:    vandnps %ymm0, %ymm2, %ymm0
 ; AVX1-NEXT:    vandps %ymm2, %ymm1, %ymm1
@@ -610,7 +610,7 @@ define <32 x i8> @var_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y, <32 x i8> %amt)
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
 ; AVX1-NEXT:    vpsllw $4, %xmm4, %xmm3
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm8 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX1-NEXT:    vpand %xmm8, %xmm3, %xmm6
+; AVX1-NEXT:    vpand %xmm3, %xmm8, %xmm6
 ; AVX1-NEXT:    vandps {{.*}}(%rip), %ymm2, %ymm2
 ; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm3
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm9 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
@@ -626,7 +626,7 @@ define <32 x i8> @var_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y, <32 x i8> %amt)
 ; AVX1-NEXT:    vpaddb %xmm5, %xmm5, %xmm5
 ; AVX1-NEXT:    vpblendvb %xmm5, %xmm6, %xmm4, %xmm4
 ; AVX1-NEXT:    vpsllw $4, %xmm0, %xmm5
-; AVX1-NEXT:    vpand %xmm8, %xmm5, %xmm5
+; AVX1-NEXT:    vpand %xmm5, %xmm8, %xmm5
 ; AVX1-NEXT:    vpsubb %xmm2, %xmm9, %xmm6
 ; AVX1-NEXT:    vpsllw $5, %xmm6, %xmm6
 ; AVX1-NEXT:    vpblendvb %xmm6, %xmm5, %xmm0, %xmm0
@@ -641,12 +641,12 @@ define <32 x i8> @var_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y, <32 x i8> %amt)
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm4
 ; AVX1-NEXT:    vpsrlw $4, %xmm4, %xmm5
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm9 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX1-NEXT:    vpand %xmm9, %xmm5, %xmm5
+; AVX1-NEXT:    vpand %xmm5, %xmm9, %xmm5
 ; AVX1-NEXT:    vpsllw $5, %xmm3, %xmm7
 ; AVX1-NEXT:    vpblendvb %xmm7, %xmm5, %xmm4, %xmm4
 ; AVX1-NEXT:    vpsrlw $2, %xmm4, %xmm5
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm10 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
-; AVX1-NEXT:    vpand %xmm10, %xmm5, %xmm5
+; AVX1-NEXT:    vpand %xmm5, %xmm10, %xmm5
 ; AVX1-NEXT:    vpaddb %xmm7, %xmm7, %xmm7
 ; AVX1-NEXT:    vpblendvb %xmm7, %xmm5, %xmm4, %xmm4
 ; AVX1-NEXT:    vpsrlw $1, %xmm4, %xmm5
@@ -655,11 +655,11 @@ define <32 x i8> @var_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y, <32 x i8> %amt)
 ; AVX1-NEXT:    vpaddb %xmm7, %xmm7, %xmm7
 ; AVX1-NEXT:    vpblendvb %xmm7, %xmm5, %xmm4, %xmm4
 ; AVX1-NEXT:    vpsrlw $4, %xmm1, %xmm5
-; AVX1-NEXT:    vpand %xmm9, %xmm5, %xmm5
+; AVX1-NEXT:    vpand %xmm5, %xmm9, %xmm5
 ; AVX1-NEXT:    vpsllw $5, %xmm2, %xmm7
 ; AVX1-NEXT:    vpblendvb %xmm7, %xmm5, %xmm1, %xmm5
 ; AVX1-NEXT:    vpsrlw $2, %xmm5, %xmm0
-; AVX1-NEXT:    vpand %xmm10, %xmm0, %xmm0
+; AVX1-NEXT:    vpand %xmm0, %xmm10, %xmm0
 ; AVX1-NEXT:    vpaddb %xmm7, %xmm7, %xmm7
 ; AVX1-NEXT:    vpblendvb %xmm7, %xmm0, %xmm5, %xmm0
 ; AVX1-NEXT:    vpsrlw $1, %xmm0, %xmm5
@@ -1477,7 +1477,7 @@ define <32 x i8> @splatvar_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y, <32 x i8> %
 ; AVX1-NEXT:    vpshufb %xmm3, %xmm6, %xmm6
 ; AVX1-NEXT:    vpand %xmm6, %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
-; AVX1-NEXT:    vorps %ymm9, %ymm0, %ymm0
+; AVX1-NEXT:    vorps %ymm0, %ymm9, %ymm0
 ; AVX1-NEXT:    vpcmpeqb %xmm3, %xmm5, %xmm4
 ; AVX1-NEXT:    vpcmpeqb %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm2, %ymm2
@@ -2504,14 +2504,53 @@ define <32 x i8> @splatconstant_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y) nounwi
 ; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    retq
 ;
-; AVX512-LABEL: splatconstant_funnnel_v32i8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsrlw $4, %ymm1, %ymm1
-; AVX512-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
-; AVX512-NEXT:    vpsllw $4, %ymm0, %ymm0
-; AVX512-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
-; AVX512-NEXT:    vpor %ymm1, %ymm0, %ymm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: splatconstant_funnnel_v32i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsrlw $4, %ymm1, %ymm1
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
+; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm0
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
+; AVX512F-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: splatconstant_funnnel_v32i8:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm2
+; AVX512VL-NEXT:    vpsrlw $4, %ymm1, %ymm0
+; AVX512VL-NEXT:    vpternlogq $216, {{.*}}(%rip), %ymm2, %ymm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: splatconstant_funnnel_v32i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsrlw $4, %ymm1, %ymm1
+; AVX512BW-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
+; AVX512BW-NEXT:    vpsllw $4, %ymm0, %ymm0
+; AVX512BW-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
+; AVX512BW-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: splatconstant_funnnel_v32i8:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpsrlw $4, %ymm1, %ymm1
+; AVX512VBMI2-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
+; AVX512VBMI2-NEXT:    vpsllw $4, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLBW-LABEL: splatconstant_funnnel_v32i8:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpsllw $4, %ymm0, %ymm2
+; AVX512VLBW-NEXT:    vpsrlw $4, %ymm1, %ymm0
+; AVX512VLBW-NEXT:    vpternlogq $216, {{.*}}(%rip), %ymm2, %ymm0
+; AVX512VLBW-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v32i8:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpsllw $4, %ymm0, %ymm2
+; AVX512VLVBMI2-NEXT:    vpsrlw $4, %ymm1, %ymm0
+; AVX512VLVBMI2-NEXT:    vpternlogq $216, {{.*}}(%rip), %ymm2, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: splatconstant_funnnel_v32i8:
 ; XOPAVX1:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/vector-fshr-512.ll b/llvm/test/CodeGen/X86/vector-fshr-512.ll
index 26dbc9291..6df29e86c 100644
--- a/llvm/test/CodeGen/X86/vector-fshr-512.ll
+++ b/llvm/test/CodeGen/X86/vector-fshr-512.ll
@@ -146,64 +146,72 @@ define <16 x i32> @var_funnnel_v16i32(<16 x i32> %x, <16 x i32> %y, <16 x i32> %
 define <32 x i16> @var_funnnel_v32i16(<32 x i16> %x, <32 x i16> %y, <32 x i16> %amt) nounwind {
 ; AVX512F-LABEL: var_funnnel_v32i16:
 ; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm4
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm2, %ymm5
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm6 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512F-NEXT:    vpand %ymm6, %ymm4, %ymm4
-; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm7 = ymm4[0],zero,ymm4[1],zero,ymm4[2],zero,ymm4[3],zero,ymm4[4],zero,ymm4[5],zero,ymm4[6],zero,ymm4[7],zero,ymm4[8],zero,ymm4[9],zero,ymm4[10],zero,ymm4[11],zero,ymm4[12],zero,ymm4[13],zero,ymm4[14],zero,ymm4[15],zero
-; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm8 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
+; AVX512F-NEXT:    vpand %ymm6, %ymm5, %ymm5
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm7 = ymm5[0],zero,ymm5[1],zero,ymm5[2],zero,ymm5[3],zero,ymm5[4],zero,ymm5[5],zero,ymm5[6],zero,ymm5[7],zero,ymm5[8],zero,ymm5[9],zero,ymm5[10],zero,ymm5[11],zero,ymm5[12],zero,ymm5[13],zero,ymm5[14],zero,ymm5[15],zero
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm8 = ymm4[0],zero,ymm4[1],zero,ymm4[2],zero,ymm4[3],zero,ymm4[4],zero,ymm4[5],zero,ymm4[6],zero,ymm4[7],zero,ymm4[8],zero,ymm4[9],zero,ymm4[10],zero,ymm4[11],zero,ymm4[12],zero,ymm4[13],zero,ymm4[14],zero,ymm4[15],zero
 ; AVX512F-NEXT:    vpsrlvd %zmm7, %zmm8, %zmm7
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm8 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
-; AVX512F-NEXT:    vpsubw %ymm4, %ymm8, %ymm9
+; AVX512F-NEXT:    vpsubw %ymm5, %ymm8, %ymm9
 ; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm9 = ymm9[0],zero,ymm9[1],zero,ymm9[2],zero,ymm9[3],zero,ymm9[4],zero,ymm9[5],zero,ymm9[6],zero,ymm9[7],zero,ymm9[8],zero,ymm9[9],zero,ymm9[10],zero,ymm9[11],zero,ymm9[12],zero,ymm9[13],zero,ymm9[14],zero,ymm9[15],zero
-; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
-; AVX512F-NEXT:    vpsllvd %zmm9, %zmm0, %zmm0
-; AVX512F-NEXT:    vpord %zmm7, %zmm0, %zmm0
-; AVX512F-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm3 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
+; AVX512F-NEXT:    vpsllvd %zmm9, %zmm3, %zmm3
+; AVX512F-NEXT:    vpord %zmm7, %zmm3, %zmm3
+; AVX512F-NEXT:    vpmovdw %zmm3, %ymm3
 ; AVX512F-NEXT:    vpxor %xmm7, %xmm7, %xmm7
-; AVX512F-NEXT:    vpcmpeqw %ymm7, %ymm4, %ymm4
-; AVX512F-NEXT:    vpblendvb %ymm4, %ymm2, %ymm0, %ymm0
-; AVX512F-NEXT:    vpand %ymm6, %ymm5, %ymm2
+; AVX512F-NEXT:    vpcmpeqw %ymm7, %ymm5, %ymm5
+; AVX512F-NEXT:    vpblendvb %ymm5, %ymm4, %ymm3, %ymm3
+; AVX512F-NEXT:    vpand %ymm6, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm4 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
-; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm5 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm5 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
 ; AVX512F-NEXT:    vpsrlvd %zmm4, %zmm5, %zmm4
 ; AVX512F-NEXT:    vpsubw %ymm2, %ymm8, %ymm5
 ; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm5 = ymm5[0],zero,ymm5[1],zero,ymm5[2],zero,ymm5[3],zero,ymm5[4],zero,ymm5[5],zero,ymm5[6],zero,ymm5[7],zero,ymm5[8],zero,ymm5[9],zero,ymm5[10],zero,ymm5[11],zero,ymm5[12],zero,ymm5[13],zero,ymm5[14],zero,ymm5[15],zero
-; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
-; AVX512F-NEXT:    vpsllvd %zmm5, %zmm1, %zmm1
-; AVX512F-NEXT:    vpord %zmm4, %zmm1, %zmm1
-; AVX512F-NEXT:    vpmovdw %zmm1, %ymm1
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
+; AVX512F-NEXT:    vpsllvd %zmm5, %zmm0, %zmm0
+; AVX512F-NEXT:    vpord %zmm4, %zmm0, %zmm0
+; AVX512F-NEXT:    vpmovdw %zmm0, %ymm0
 ; AVX512F-NEXT:    vpcmpeqw %ymm7, %ymm2, %ymm2
-; AVX512F-NEXT:    vpblendvb %ymm2, %ymm3, %ymm1, %ymm1
+; AVX512F-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: var_funnnel_v32i16:
 ; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm1, %ymm4
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm2, %ymm5
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm6 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512VL-NEXT:    vpand %ymm6, %ymm4, %ymm4
-; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm7 = ymm4[0],zero,ymm4[1],zero,ymm4[2],zero,ymm4[3],zero,ymm4[4],zero,ymm4[5],zero,ymm4[6],zero,ymm4[7],zero,ymm4[8],zero,ymm4[9],zero,ymm4[10],zero,ymm4[11],zero,ymm4[12],zero,ymm4[13],zero,ymm4[14],zero,ymm4[15],zero
-; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm8 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
+; AVX512VL-NEXT:    vpand %ymm6, %ymm5, %ymm5
+; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm7 = ymm5[0],zero,ymm5[1],zero,ymm5[2],zero,ymm5[3],zero,ymm5[4],zero,ymm5[5],zero,ymm5[6],zero,ymm5[7],zero,ymm5[8],zero,ymm5[9],zero,ymm5[10],zero,ymm5[11],zero,ymm5[12],zero,ymm5[13],zero,ymm5[14],zero,ymm5[15],zero
+; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm8 = ymm4[0],zero,ymm4[1],zero,ymm4[2],zero,ymm4[3],zero,ymm4[4],zero,ymm4[5],zero,ymm4[6],zero,ymm4[7],zero,ymm4[8],zero,ymm4[9],zero,ymm4[10],zero,ymm4[11],zero,ymm4[12],zero,ymm4[13],zero,ymm4[14],zero,ymm4[15],zero
 ; AVX512VL-NEXT:    vpsrlvd %zmm7, %zmm8, %zmm7
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm8 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
-; AVX512VL-NEXT:    vpsubw %ymm4, %ymm8, %ymm9
+; AVX512VL-NEXT:    vpsubw %ymm5, %ymm8, %ymm9
 ; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm9 = ymm9[0],zero,ymm9[1],zero,ymm9[2],zero,ymm9[3],zero,ymm9[4],zero,ymm9[5],zero,ymm9[6],zero,ymm9[7],zero,ymm9[8],zero,ymm9[9],zero,ymm9[10],zero,ymm9[11],zero,ymm9[12],zero,ymm9[13],zero,ymm9[14],zero,ymm9[15],zero
-; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
-; AVX512VL-NEXT:    vpsllvd %zmm9, %zmm0, %zmm0
-; AVX512VL-NEXT:    vpord %zmm7, %zmm0, %zmm0
-; AVX512VL-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm3 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
+; AVX512VL-NEXT:    vpsllvd %zmm9, %zmm3, %zmm3
+; AVX512VL-NEXT:    vpord %zmm7, %zmm3, %zmm3
+; AVX512VL-NEXT:    vpmovdw %zmm3, %ymm3
 ; AVX512VL-NEXT:    vpxor %xmm7, %xmm7, %xmm7
-; AVX512VL-NEXT:    vpcmpeqw %ymm7, %ymm4, %ymm4
-; AVX512VL-NEXT:    vpblendvb %ymm4, %ymm2, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpand %ymm6, %ymm5, %ymm2
+; AVX512VL-NEXT:    vpcmpeqw %ymm7, %ymm5, %ymm5
+; AVX512VL-NEXT:    vpblendvb %ymm5, %ymm4, %ymm3, %ymm3
+; AVX512VL-NEXT:    vpand %ymm6, %ymm2, %ymm2
 ; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm4 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
-; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm5 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
+; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm5 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
 ; AVX512VL-NEXT:    vpsrlvd %zmm4, %zmm5, %zmm4
 ; AVX512VL-NEXT:    vpsubw %ymm2, %ymm8, %ymm5
 ; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm5 = ymm5[0],zero,ymm5[1],zero,ymm5[2],zero,ymm5[3],zero,ymm5[4],zero,ymm5[5],zero,ymm5[6],zero,ymm5[7],zero,ymm5[8],zero,ymm5[9],zero,ymm5[10],zero,ymm5[11],zero,ymm5[12],zero,ymm5[13],zero,ymm5[14],zero,ymm5[15],zero
-; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
-; AVX512VL-NEXT:    vpsllvd %zmm5, %zmm1, %zmm1
-; AVX512VL-NEXT:    vpord %zmm4, %zmm1, %zmm1
-; AVX512VL-NEXT:    vpmovdw %zmm1, %ymm1
+; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
+; AVX512VL-NEXT:    vpsllvd %zmm5, %zmm0, %zmm0
+; AVX512VL-NEXT:    vpord %zmm4, %zmm0, %zmm0
+; AVX512VL-NEXT:    vpmovdw %zmm0, %ymm0
 ; AVX512VL-NEXT:    vpcmpeqw %ymm7, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm3, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: var_funnnel_v32i16:
@@ -248,81 +256,88 @@ define <32 x i16> @var_funnnel_v32i16(<32 x i16> %x, <32 x i16> %y, <32 x i16> %
 define <64 x i8> @var_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> %amt) nounwind {
 ; AVX512F-LABEL: var_funnnel_v64i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpsrlw $4, %ymm2, %ymm7
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm6 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512F-NEXT:    vpand %ymm6, %ymm7, %ymm8
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm7 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; AVX512F-NEXT:    vpand %ymm7, %ymm4, %ymm9
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm8
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm2, %ymm6
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
+; AVX512F-NEXT:    vpsrlw $4, %ymm3, %ymm5
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX512F-NEXT:    vpand %ymm4, %ymm5, %ymm7
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm5 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; AVX512F-NEXT:    vpand %ymm5, %ymm6, %ymm9
 ; AVX512F-NEXT:    vpsllw $5, %ymm9, %ymm10
-; AVX512F-NEXT:    vpblendvb %ymm10, %ymm8, %ymm2, %ymm8
-; AVX512F-NEXT:    vpsrlw $2, %ymm8, %ymm11
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm4 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
-; AVX512F-NEXT:    vpand %ymm4, %ymm11, %ymm11
+; AVX512F-NEXT:    vpblendvb %ymm10, %ymm7, %ymm3, %ymm7
+; AVX512F-NEXT:    vpsrlw $2, %ymm7, %ymm11
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm6 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
+; AVX512F-NEXT:    vpand %ymm6, %ymm11, %ymm11
 ; AVX512F-NEXT:    vpaddb %ymm10, %ymm10, %ymm10
-; AVX512F-NEXT:    vpblendvb %ymm10, %ymm11, %ymm8, %ymm11
+; AVX512F-NEXT:    vpblendvb %ymm10, %ymm11, %ymm7, %ymm11
 ; AVX512F-NEXT:    vpsrlw $1, %ymm11, %ymm12
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm8 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; AVX512F-NEXT:    vpand %ymm8, %ymm12, %ymm12
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm7 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
+; AVX512F-NEXT:    vpand %ymm7, %ymm12, %ymm12
 ; AVX512F-NEXT:    vpaddb %ymm10, %ymm10, %ymm10
 ; AVX512F-NEXT:    vpblendvb %ymm10, %ymm12, %ymm11, %ymm10
-; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm11
+; AVX512F-NEXT:    vpsllw $4, %ymm8, %ymm11
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm12 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
 ; AVX512F-NEXT:    vpand %ymm12, %ymm11, %ymm11
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm13 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
 ; AVX512F-NEXT:    vpsubb %ymm9, %ymm13, %ymm14
 ; AVX512F-NEXT:    vpsllw $5, %ymm14, %ymm14
-; AVX512F-NEXT:    vpblendvb %ymm14, %ymm11, %ymm0, %ymm0
-; AVX512F-NEXT:    vpsllw $2, %ymm0, %ymm11
+; AVX512F-NEXT:    vpblendvb %ymm14, %ymm11, %ymm8, %ymm8
+; AVX512F-NEXT:    vpsllw $2, %ymm8, %ymm11
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm15 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
 ; AVX512F-NEXT:    vpand %ymm15, %ymm11, %ymm11
 ; AVX512F-NEXT:    vpaddb %ymm14, %ymm14, %ymm14
-; AVX512F-NEXT:    vpblendvb %ymm14, %ymm11, %ymm0, %ymm0
-; AVX512F-NEXT:    vpaddb %ymm0, %ymm0, %ymm11
+; AVX512F-NEXT:    vpblendvb %ymm14, %ymm11, %ymm8, %ymm8
+; AVX512F-NEXT:    vpaddb %ymm8, %ymm8, %ymm11
 ; AVX512F-NEXT:    vpaddb %ymm14, %ymm14, %ymm14
-; AVX512F-NEXT:    vpblendvb %ymm14, %ymm11, %ymm0, %ymm0
-; AVX512F-NEXT:    vpor %ymm10, %ymm0, %ymm0
+; AVX512F-NEXT:    vpblendvb %ymm14, %ymm11, %ymm8, %ymm8
+; AVX512F-NEXT:    vpor %ymm10, %ymm8, %ymm8
 ; AVX512F-NEXT:    vpxor %xmm10, %xmm10, %xmm10
 ; AVX512F-NEXT:    vpcmpeqb %ymm10, %ymm9, %ymm9
-; AVX512F-NEXT:    vpblendvb %ymm9, %ymm2, %ymm0, %ymm0
-; AVX512F-NEXT:    vpsrlw $4, %ymm3, %ymm2
-; AVX512F-NEXT:    vpand %ymm6, %ymm2, %ymm2
-; AVX512F-NEXT:    vpand %ymm7, %ymm5, %ymm5
-; AVX512F-NEXT:    vpsllw $5, %ymm5, %ymm6
-; AVX512F-NEXT:    vpblendvb %ymm6, %ymm2, %ymm3, %ymm2
-; AVX512F-NEXT:    vpsrlw $2, %ymm2, %ymm7
-; AVX512F-NEXT:    vpand %ymm4, %ymm7, %ymm4
-; AVX512F-NEXT:    vpaddb %ymm6, %ymm6, %ymm6
-; AVX512F-NEXT:    vpblendvb %ymm6, %ymm4, %ymm2, %ymm2
-; AVX512F-NEXT:    vpsrlw $1, %ymm2, %ymm4
-; AVX512F-NEXT:    vpand %ymm8, %ymm4, %ymm4
-; AVX512F-NEXT:    vpaddb %ymm6, %ymm6, %ymm6
-; AVX512F-NEXT:    vpblendvb %ymm6, %ymm4, %ymm2, %ymm2
-; AVX512F-NEXT:    vpsllw $4, %ymm1, %ymm4
-; AVX512F-NEXT:    vpand %ymm12, %ymm4, %ymm4
-; AVX512F-NEXT:    vpsubb %ymm5, %ymm13, %ymm6
+; AVX512F-NEXT:    vpblendvb %ymm9, %ymm3, %ymm8, %ymm3
+; AVX512F-NEXT:    vpsrlw $4, %ymm1, %ymm8
+; AVX512F-NEXT:    vpand %ymm4, %ymm8, %ymm4
+; AVX512F-NEXT:    vpand %ymm5, %ymm2, %ymm2
+; AVX512F-NEXT:    vpsllw $5, %ymm2, %ymm5
+; AVX512F-NEXT:    vpblendvb %ymm5, %ymm4, %ymm1, %ymm4
+; AVX512F-NEXT:    vpsrlw $2, %ymm4, %ymm8
+; AVX512F-NEXT:    vpand %ymm6, %ymm8, %ymm6
+; AVX512F-NEXT:    vpaddb %ymm5, %ymm5, %ymm5
+; AVX512F-NEXT:    vpblendvb %ymm5, %ymm6, %ymm4, %ymm4
+; AVX512F-NEXT:    vpsrlw $1, %ymm4, %ymm6
+; AVX512F-NEXT:    vpand %ymm7, %ymm6, %ymm6
+; AVX512F-NEXT:    vpaddb %ymm5, %ymm5, %ymm5
+; AVX512F-NEXT:    vpblendvb %ymm5, %ymm6, %ymm4, %ymm4
+; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm5
+; AVX512F-NEXT:    vpand %ymm5, %ymm12, %ymm5
+; AVX512F-NEXT:    vpsubb %ymm2, %ymm13, %ymm6
 ; AVX512F-NEXT:    vpsllw $5, %ymm6, %ymm6
-; AVX512F-NEXT:    vpblendvb %ymm6, %ymm4, %ymm1, %ymm1
-; AVX512F-NEXT:    vpsllw $2, %ymm1, %ymm4
-; AVX512F-NEXT:    vpand %ymm15, %ymm4, %ymm4
+; AVX512F-NEXT:    vpblendvb %ymm6, %ymm5, %ymm0, %ymm0
+; AVX512F-NEXT:    vpsllw $2, %ymm0, %ymm5
+; AVX512F-NEXT:    vpand %ymm5, %ymm15, %ymm5
 ; AVX512F-NEXT:    vpaddb %ymm6, %ymm6, %ymm6
-; AVX512F-NEXT:    vpblendvb %ymm6, %ymm4, %ymm1, %ymm1
-; AVX512F-NEXT:    vpaddb %ymm1, %ymm1, %ymm4
+; AVX512F-NEXT:    vpblendvb %ymm6, %ymm5, %ymm0, %ymm0
+; AVX512F-NEXT:    vpaddb %ymm0, %ymm0, %ymm5
 ; AVX512F-NEXT:    vpaddb %ymm6, %ymm6, %ymm6
-; AVX512F-NEXT:    vpblendvb %ymm6, %ymm4, %ymm1, %ymm1
-; AVX512F-NEXT:    vpor %ymm2, %ymm1, %ymm1
-; AVX512F-NEXT:    vpcmpeqb %ymm10, %ymm5, %ymm2
-; AVX512F-NEXT:    vpblendvb %ymm2, %ymm3, %ymm1, %ymm1
+; AVX512F-NEXT:    vpblendvb %ymm6, %ymm5, %ymm0, %ymm0
+; AVX512F-NEXT:    vpor %ymm4, %ymm0, %ymm0
+; AVX512F-NEXT:    vpcmpeqb %ymm2, %ymm10, %ymm2
+; AVX512F-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: var_funnnel_v64i8:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsrlw $4, %ymm2, %ymm6
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm4
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm2, %ymm5
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
+; AVX512VL-NEXT:    vpsrlw $4, %ymm3, %ymm6
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm7 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
 ; AVX512VL-NEXT:    vpand %ymm7, %ymm6, %ymm6
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm8 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; AVX512VL-NEXT:    vpand %ymm8, %ymm4, %ymm4
-; AVX512VL-NEXT:    vpsllw $5, %ymm4, %ymm9
-; AVX512VL-NEXT:    vpblendvb %ymm9, %ymm6, %ymm2, %ymm6
+; AVX512VL-NEXT:    vpand %ymm5, %ymm8, %ymm5
+; AVX512VL-NEXT:    vpsllw $5, %ymm5, %ymm9
+; AVX512VL-NEXT:    vpblendvb %ymm9, %ymm6, %ymm3, %ymm6
 ; AVX512VL-NEXT:    vpsrlw $2, %ymm6, %ymm10
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm11 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
 ; AVX512VL-NEXT:    vpand %ymm11, %ymm10, %ymm10
@@ -333,53 +348,54 @@ define <64 x i8> @var_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> %amt)
 ; AVX512VL-NEXT:    vpand %ymm12, %ymm10, %ymm10
 ; AVX512VL-NEXT:    vpaddb %ymm9, %ymm9, %ymm9
 ; AVX512VL-NEXT:    vpblendvb %ymm9, %ymm10, %ymm6, %ymm6
-; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm9
+; AVX512VL-NEXT:    vpsllw $4, %ymm4, %ymm9
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm10 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
 ; AVX512VL-NEXT:    vpand %ymm10, %ymm9, %ymm9
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm13 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; AVX512VL-NEXT:    vpsubb %ymm4, %ymm13, %ymm14
+; AVX512VL-NEXT:    vpsubb %ymm5, %ymm13, %ymm14
 ; AVX512VL-NEXT:    vpsllw $5, %ymm14, %ymm14
-; AVX512VL-NEXT:    vpblendvb %ymm14, %ymm9, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpsllw $2, %ymm0, %ymm9
+; AVX512VL-NEXT:    vpblendvb %ymm14, %ymm9, %ymm4, %ymm4
+; AVX512VL-NEXT:    vpsllw $2, %ymm4, %ymm9
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm15 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
 ; AVX512VL-NEXT:    vpand %ymm15, %ymm9, %ymm9
 ; AVX512VL-NEXT:    vpaddb %ymm14, %ymm14, %ymm14
-; AVX512VL-NEXT:    vpblendvb %ymm14, %ymm9, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpaddb %ymm0, %ymm0, %ymm9
+; AVX512VL-NEXT:    vpblendvb %ymm14, %ymm9, %ymm4, %ymm4
+; AVX512VL-NEXT:    vpaddb %ymm4, %ymm4, %ymm9
 ; AVX512VL-NEXT:    vpaddb %ymm14, %ymm14, %ymm14
-; AVX512VL-NEXT:    vpblendvb %ymm14, %ymm9, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpor %ymm6, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpblendvb %ymm14, %ymm9, %ymm4, %ymm4
+; AVX512VL-NEXT:    vpor %ymm6, %ymm4, %ymm4
 ; AVX512VL-NEXT:    vpxor %xmm6, %xmm6, %xmm6
-; AVX512VL-NEXT:    vpcmpeqb %ymm6, %ymm4, %ymm4
-; AVX512VL-NEXT:    vpblendvb %ymm4, %ymm2, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpsrlw $4, %ymm3, %ymm2
-; AVX512VL-NEXT:    vpand %ymm7, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpand %ymm8, %ymm5, %ymm4
-; AVX512VL-NEXT:    vpsllw $5, %ymm4, %ymm5
-; AVX512VL-NEXT:    vpblendvb %ymm5, %ymm2, %ymm3, %ymm2
-; AVX512VL-NEXT:    vpsrlw $2, %ymm2, %ymm7
-; AVX512VL-NEXT:    vpand %ymm11, %ymm7, %ymm7
+; AVX512VL-NEXT:    vpcmpeqb %ymm6, %ymm5, %ymm5
+; AVX512VL-NEXT:    vpblendvb %ymm5, %ymm3, %ymm4, %ymm3
+; AVX512VL-NEXT:    vpsrlw $4, %ymm1, %ymm4
+; AVX512VL-NEXT:    vpand %ymm7, %ymm4, %ymm4
+; AVX512VL-NEXT:    vpand %ymm2, %ymm8, %ymm2
+; AVX512VL-NEXT:    vpsllw $5, %ymm2, %ymm5
+; AVX512VL-NEXT:    vpblendvb %ymm5, %ymm4, %ymm1, %ymm4
+; AVX512VL-NEXT:    vpsrlw $2, %ymm4, %ymm7
+; AVX512VL-NEXT:    vpand %ymm7, %ymm11, %ymm7
 ; AVX512VL-NEXT:    vpaddb %ymm5, %ymm5, %ymm5
-; AVX512VL-NEXT:    vpblendvb %ymm5, %ymm7, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpsrlw $1, %ymm2, %ymm7
-; AVX512VL-NEXT:    vpand %ymm12, %ymm7, %ymm7
+; AVX512VL-NEXT:    vpblendvb %ymm5, %ymm7, %ymm4, %ymm4
+; AVX512VL-NEXT:    vpsrlw $1, %ymm4, %ymm7
+; AVX512VL-NEXT:    vpand %ymm7, %ymm12, %ymm7
 ; AVX512VL-NEXT:    vpaddb %ymm5, %ymm5, %ymm5
-; AVX512VL-NEXT:    vpblendvb %ymm5, %ymm7, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpsllw $4, %ymm1, %ymm5
-; AVX512VL-NEXT:    vpand %ymm10, %ymm5, %ymm5
-; AVX512VL-NEXT:    vpsubb %ymm4, %ymm13, %ymm7
+; AVX512VL-NEXT:    vpblendvb %ymm5, %ymm7, %ymm4, %ymm4
+; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm5
+; AVX512VL-NEXT:    vpand %ymm5, %ymm10, %ymm5
+; AVX512VL-NEXT:    vpsubb %ymm2, %ymm13, %ymm7
 ; AVX512VL-NEXT:    vpsllw $5, %ymm7, %ymm7
-; AVX512VL-NEXT:    vpblendvb %ymm7, %ymm5, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpsllw $2, %ymm1, %ymm5
-; AVX512VL-NEXT:    vpand %ymm15, %ymm5, %ymm5
+; AVX512VL-NEXT:    vpblendvb %ymm7, %ymm5, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpsllw $2, %ymm0, %ymm5
+; AVX512VL-NEXT:    vpand %ymm5, %ymm15, %ymm5
 ; AVX512VL-NEXT:    vpaddb %ymm7, %ymm7, %ymm7
-; AVX512VL-NEXT:    vpblendvb %ymm7, %ymm5, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpaddb %ymm1, %ymm1, %ymm5
+; AVX512VL-NEXT:    vpblendvb %ymm7, %ymm5, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpaddb %ymm0, %ymm0, %ymm5
 ; AVX512VL-NEXT:    vpaddb %ymm7, %ymm7, %ymm7
-; AVX512VL-NEXT:    vpblendvb %ymm7, %ymm5, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpor %ymm2, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpcmpeqb %ymm6, %ymm4, %ymm2
-; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm3, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpblendvb %ymm7, %ymm5, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpor %ymm4, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpcmpeqb %ymm6, %ymm2, %ymm2
+; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: var_funnnel_v64i8:
@@ -694,42 +710,48 @@ define <16 x i32> @splatvar_funnnel_v16i32(<16 x i32> %x, <16 x i32> %y, <16 x i
 define <32 x i16> @splatvar_funnnel_v32i16(<32 x i16> %x, <32 x i16> %y, <32 x i16> %amt) nounwind {
 ; AVX512F-LABEL: splatvar_funnnel_v32i16:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpbroadcastw %xmm4, %ymm4
-; AVX512F-NEXT:    vpand {{.*}}(%rip), %ymm4, %ymm4
-; AVX512F-NEXT:    vpmovzxwq {{.*#+}} xmm5 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
-; AVX512F-NEXT:    vpsrlw %xmm5, %ymm2, %ymm6
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm4
+; AVX512F-NEXT:    vpbroadcastw %xmm2, %ymm2
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
+; AVX512F-NEXT:    vpmovzxwq {{.*#+}} xmm5 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
+; AVX512F-NEXT:    vpsrlw %xmm5, %ymm4, %ymm6
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm7 = [16,16,16,16,16,16,16,16]
-; AVX512F-NEXT:    vpsubw %xmm4, %xmm7, %xmm7
+; AVX512F-NEXT:    vpsubw %xmm2, %xmm7, %xmm7
 ; AVX512F-NEXT:    vpmovzxwq {{.*#+}} xmm7 = xmm7[0],zero,zero,zero,xmm7[1],zero,zero,zero
-; AVX512F-NEXT:    vpsllw %xmm7, %ymm0, %ymm0
-; AVX512F-NEXT:    vpor %ymm6, %ymm0, %ymm0
+; AVX512F-NEXT:    vpsllw %xmm7, %ymm3, %ymm3
+; AVX512F-NEXT:    vpor %ymm6, %ymm3, %ymm3
 ; AVX512F-NEXT:    vpxor %xmm6, %xmm6, %xmm6
-; AVX512F-NEXT:    vpcmpeqw %ymm6, %ymm4, %ymm4
-; AVX512F-NEXT:    vpblendvb %ymm4, %ymm2, %ymm0, %ymm0
-; AVX512F-NEXT:    vpsrlw %xmm5, %ymm3, %ymm2
-; AVX512F-NEXT:    vpsllw %xmm7, %ymm1, %ymm1
-; AVX512F-NEXT:    vpor %ymm2, %ymm1, %ymm1
-; AVX512F-NEXT:    vpblendvb %ymm4, %ymm3, %ymm1, %ymm1
+; AVX512F-NEXT:    vpcmpeqw %ymm6, %ymm2, %ymm2
+; AVX512F-NEXT:    vpblendvb %ymm2, %ymm4, %ymm3, %ymm3
+; AVX512F-NEXT:    vpsrlw %xmm5, %ymm1, %ymm4
+; AVX512F-NEXT:    vpsllw %xmm7, %ymm0, %ymm0
+; AVX512F-NEXT:    vpor %ymm4, %ymm0, %ymm0
+; AVX512F-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatvar_funnnel_v32i16:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpbroadcastw %xmm4, %ymm4
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm4, %ymm4
-; AVX512VL-NEXT:    vpmovzxwq {{.*#+}} xmm5 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
-; AVX512VL-NEXT:    vpsrlw %xmm5, %ymm2, %ymm6
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm1, %ymm4
+; AVX512VL-NEXT:    vpbroadcastw %xmm2, %ymm2
+; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
+; AVX512VL-NEXT:    vpmovzxwq {{.*#+}} xmm5 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
+; AVX512VL-NEXT:    vpsrlw %xmm5, %ymm4, %ymm6
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm7 = [16,16,16,16,16,16,16,16]
-; AVX512VL-NEXT:    vpsubw %xmm4, %xmm7, %xmm7
+; AVX512VL-NEXT:    vpsubw %xmm2, %xmm7, %xmm7
 ; AVX512VL-NEXT:    vpmovzxwq {{.*#+}} xmm7 = xmm7[0],zero,zero,zero,xmm7[1],zero,zero,zero
-; AVX512VL-NEXT:    vpsllw %xmm7, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpor %ymm6, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpsllw %xmm7, %ymm3, %ymm3
+; AVX512VL-NEXT:    vpor %ymm6, %ymm3, %ymm3
 ; AVX512VL-NEXT:    vpxor %xmm6, %xmm6, %xmm6
-; AVX512VL-NEXT:    vpcmpeqw %ymm6, %ymm4, %ymm4
-; AVX512VL-NEXT:    vpblendvb %ymm4, %ymm2, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpsrlw %xmm5, %ymm3, %ymm2
-; AVX512VL-NEXT:    vpsllw %xmm7, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpor %ymm2, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpblendvb %ymm4, %ymm3, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpcmpeqw %ymm6, %ymm2, %ymm2
+; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm4, %ymm3, %ymm3
+; AVX512VL-NEXT:    vpsrlw %xmm5, %ymm1, %ymm4
+; AVX512VL-NEXT:    vpsllw %xmm7, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpor %ymm4, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatvar_funnnel_v32i16:
@@ -783,62 +805,68 @@ define <32 x i16> @splatvar_funnnel_v32i16(<32 x i16> %x, <32 x i16> %y, <32 x i
 define <64 x i8> @splatvar_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> %amt) nounwind {
 ; AVX512F-LABEL: splatvar_funnnel_v64i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpbroadcastb %xmm4, %ymm4
-; AVX512F-NEXT:    vpand {{.*}}(%rip), %ymm4, %ymm4
-; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm5 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512F-NEXT:    vpsrlw %xmm5, %ymm2, %ymm6
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm9
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm4
+; AVX512F-NEXT:    vpbroadcastb %xmm2, %ymm2
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
+; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm5 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512F-NEXT:    vpsrlw %xmm5, %ymm4, %ymm6
 ; AVX512F-NEXT:    vpcmpeqd %xmm8, %xmm8, %xmm8
 ; AVX512F-NEXT:    vpsrlw %xmm5, %xmm8, %xmm7
 ; AVX512F-NEXT:    vpsrlw $8, %xmm7, %xmm7
 ; AVX512F-NEXT:    vpbroadcastb %xmm7, %ymm7
-; AVX512F-NEXT:    vpand %ymm7, %ymm6, %ymm9
-; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm6 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; AVX512F-NEXT:    vpsubb %xmm4, %xmm6, %xmm6
-; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm6 = xmm6[0],zero,zero,zero,zero,zero,zero,zero,xmm6[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512F-NEXT:    vpsllw %xmm6, %ymm0, %ymm10
-; AVX512F-NEXT:    vpsllw %xmm6, %xmm8, %xmm0
-; AVX512F-NEXT:    vpbroadcastb %xmm0, %ymm8
-; AVX512F-NEXT:    vpand %ymm8, %ymm10, %ymm0
-; AVX512F-NEXT:    vpor %ymm9, %ymm0, %ymm0
+; AVX512F-NEXT:    vpand %ymm7, %ymm6, %ymm10
+; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm3 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512F-NEXT:    vpsubb %xmm2, %xmm3, %xmm3
+; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512F-NEXT:    vpsllw %xmm3, %ymm9, %ymm9
+; AVX512F-NEXT:    vpsllw %xmm3, %xmm8, %xmm6
+; AVX512F-NEXT:    vpbroadcastb %xmm6, %ymm6
+; AVX512F-NEXT:    vpand %ymm6, %ymm9, %ymm8
+; AVX512F-NEXT:    vpor %ymm10, %ymm8, %ymm8
 ; AVX512F-NEXT:    vpxor %xmm9, %xmm9, %xmm9
-; AVX512F-NEXT:    vpcmpeqb %ymm9, %ymm4, %ymm4
-; AVX512F-NEXT:    vpblendvb %ymm4, %ymm2, %ymm0, %ymm0
-; AVX512F-NEXT:    vpsrlw %xmm5, %ymm3, %ymm2
-; AVX512F-NEXT:    vpand %ymm7, %ymm2, %ymm2
-; AVX512F-NEXT:    vpsllw %xmm6, %ymm1, %ymm1
-; AVX512F-NEXT:    vpand %ymm8, %ymm1, %ymm1
-; AVX512F-NEXT:    vpor %ymm2, %ymm1, %ymm1
-; AVX512F-NEXT:    vpblendvb %ymm4, %ymm3, %ymm1, %ymm1
+; AVX512F-NEXT:    vpcmpeqb %ymm2, %ymm9, %ymm2
+; AVX512F-NEXT:    vpblendvb %ymm2, %ymm4, %ymm8, %ymm4
+; AVX512F-NEXT:    vpsrlw %xmm5, %ymm1, %ymm5
+; AVX512F-NEXT:    vpand %ymm7, %ymm5, %ymm5
+; AVX512F-NEXT:    vpsllw %xmm3, %ymm0, %ymm0
+; AVX512F-NEXT:    vpand %ymm6, %ymm0, %ymm0
+; AVX512F-NEXT:    vpor %ymm5, %ymm0, %ymm0
+; AVX512F-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm4, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatvar_funnnel_v64i8:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpbroadcastb %xmm4, %ymm4
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm4, %ymm4
-; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm5 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512VL-NEXT:    vpsrlw %xmm5, %ymm2, %ymm6
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm9
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm1, %ymm4
+; AVX512VL-NEXT:    vpbroadcastb %xmm2, %ymm2
+; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
+; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm5 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VL-NEXT:    vpsrlw %xmm5, %ymm4, %ymm6
 ; AVX512VL-NEXT:    vpcmpeqd %xmm8, %xmm8, %xmm8
 ; AVX512VL-NEXT:    vpsrlw %xmm5, %xmm8, %xmm7
 ; AVX512VL-NEXT:    vpsrlw $8, %xmm7, %xmm7
 ; AVX512VL-NEXT:    vpbroadcastb %xmm7, %ymm7
-; AVX512VL-NEXT:    vpand %ymm7, %ymm6, %ymm9
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm6 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; AVX512VL-NEXT:    vpsubb %xmm4, %xmm6, %xmm6
-; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm6 = xmm6[0],zero,zero,zero,zero,zero,zero,zero,xmm6[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512VL-NEXT:    vpsllw %xmm6, %ymm0, %ymm10
-; AVX512VL-NEXT:    vpsllw %xmm6, %xmm8, %xmm0
-; AVX512VL-NEXT:    vpbroadcastb %xmm0, %ymm8
-; AVX512VL-NEXT:    vpand %ymm8, %ymm10, %ymm0
-; AVX512VL-NEXT:    vpor %ymm9, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpand %ymm7, %ymm6, %ymm10
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm3 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VL-NEXT:    vpsubb %xmm2, %xmm3, %xmm3
+; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VL-NEXT:    vpsllw %xmm3, %ymm9, %ymm9
+; AVX512VL-NEXT:    vpsllw %xmm3, %xmm8, %xmm6
+; AVX512VL-NEXT:    vpbroadcastb %xmm6, %ymm6
+; AVX512VL-NEXT:    vpand %ymm6, %ymm9, %ymm8
+; AVX512VL-NEXT:    vpor %ymm10, %ymm8, %ymm8
 ; AVX512VL-NEXT:    vpxor %xmm9, %xmm9, %xmm9
-; AVX512VL-NEXT:    vpcmpeqb %ymm9, %ymm4, %ymm4
-; AVX512VL-NEXT:    vpblendvb %ymm4, %ymm2, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpsrlw %xmm5, %ymm3, %ymm2
-; AVX512VL-NEXT:    vpand %ymm7, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpsllw %xmm6, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpand %ymm8, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpor %ymm2, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpblendvb %ymm4, %ymm3, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpcmpeqb %ymm2, %ymm9, %ymm2
+; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm4, %ymm8, %ymm4
+; AVX512VL-NEXT:    vpsrlw %xmm5, %ymm1, %ymm5
+; AVX512VL-NEXT:    vpand %ymm7, %ymm5, %ymm5
+; AVX512VL-NEXT:    vpsllw %xmm3, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpand %ymm6, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpor %ymm5, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm4, %zmm0, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatvar_funnnel_v64i8:
@@ -1032,32 +1060,38 @@ define <16 x i32> @constant_funnnel_v16i32(<16 x i32> %x, <16 x i32> %y) nounwin
 define <32 x i16> @constant_funnnel_v32i16(<32 x i16> %x, <32 x i16> %y) nounwind {
 ; AVX512F-LABEL: constant_funnnel_v32i16:
 ; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm4 = <u,32768,16384,8192,4096,2048,1024,512,256,128,64,32,16,8,4,2>
-; AVX512F-NEXT:    vpmulhuw %ymm4, %ymm2, %ymm5
+; AVX512F-NEXT:    vpmulhuw %ymm4, %ymm3, %ymm5
+; AVX512F-NEXT:    vpmullw %ymm4, %ymm2, %ymm2
+; AVX512F-NEXT:    vpor %ymm5, %ymm2, %ymm2
+; AVX512F-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm2[1,2,3,4,5,6,7],ymm3[8],ymm2[9,10,11,12,13,14,15]
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
+; AVX512F-NEXT:    vpmulhuw %ymm4, %ymm1, %ymm3
 ; AVX512F-NEXT:    vpmullw %ymm4, %ymm0, %ymm0
-; AVX512F-NEXT:    vpor %ymm5, %ymm0, %ymm0
-; AVX512F-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0],ymm0[1,2,3,4,5,6,7],ymm2[8],ymm0[9,10,11,12,13,14,15]
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
-; AVX512F-NEXT:    vpmulhuw %ymm4, %ymm3, %ymm2
-; AVX512F-NEXT:    vpmullw %ymm4, %ymm1, %ymm1
-; AVX512F-NEXT:    vpor %ymm2, %ymm1, %ymm1
-; AVX512F-NEXT:    vpblendw {{.*#+}} ymm2 = ymm3[0],ymm1[1,2,3,4,5,6,7],ymm3[8],ymm1[9,10,11,12,13,14,15]
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    vpor %ymm3, %ymm0, %ymm0
+; AVX512F-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm0[1,2,3,4,5,6,7],ymm1[8],ymm0[9,10,11,12,13,14,15]
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: constant_funnnel_v32i16:
 ; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm4 = <u,32768,16384,8192,4096,2048,1024,512,256,128,64,32,16,8,4,2>
-; AVX512VL-NEXT:    vpmulhuw %ymm4, %ymm2, %ymm5
+; AVX512VL-NEXT:    vpmulhuw %ymm4, %ymm3, %ymm5
+; AVX512VL-NEXT:    vpmullw %ymm4, %ymm2, %ymm2
+; AVX512VL-NEXT:    vpor %ymm5, %ymm2, %ymm2
+; AVX512VL-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm2[1,2,3,4,5,6,7],ymm3[8],ymm2[9,10,11,12,13,14,15]
+; AVX512VL-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
+; AVX512VL-NEXT:    vpmulhuw %ymm4, %ymm1, %ymm3
 ; AVX512VL-NEXT:    vpmullw %ymm4, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpor %ymm5, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0],ymm0[1,2,3,4,5,6,7],ymm2[8],ymm0[9,10,11,12,13,14,15]
-; AVX512VL-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
-; AVX512VL-NEXT:    vpmulhuw %ymm4, %ymm3, %ymm2
-; AVX512VL-NEXT:    vpmullw %ymm4, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpor %ymm2, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpblendw {{.*#+}} ymm2 = ymm3[0],ymm1[1,2,3,4,5,6,7],ymm3[8],ymm1[9,10,11,12,13,14,15]
-; AVX512VL-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512VL-NEXT:    vpor %ymm3, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm0[1,2,3,4,5,6,7],ymm1[8],ymm0[9,10,11,12,13,14,15]
+; AVX512VL-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: constant_funnnel_v32i16:
@@ -1098,20 +1132,22 @@ define <32 x i16> @constant_funnnel_v32i16(<32 x i16> %x, <32 x i16> %y) nounwin
 define <64 x i8> @constant_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y) nounwind {
 ; AVX512F-LABEL: constant_funnnel_v64i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm4
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
+; AVX512F-NEXT:    vpsllw $4, %ymm3, %ymm4
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm5 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
 ; AVX512F-NEXT:    vpand %ymm5, %ymm4, %ymm4
 ; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm6 = [57344,41152,24704,8256,8192,24640,41088,57536,57344,41152,24704,8256,8192,24640,41088,57536]
 ; AVX512F-NEXT:    # ymm6 = mem[0,1,0,1]
-; AVX512F-NEXT:    vpblendvb %ymm6, %ymm4, %ymm0, %ymm0
-; AVX512F-NEXT:    vpsllw $2, %ymm0, %ymm4
+; AVX512F-NEXT:    vpblendvb %ymm6, %ymm4, %ymm3, %ymm3
+; AVX512F-NEXT:    vpsllw $2, %ymm3, %ymm4
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm7 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
 ; AVX512F-NEXT:    vpand %ymm7, %ymm4, %ymm4
 ; AVX512F-NEXT:    vpaddb %ymm6, %ymm6, %ymm8
-; AVX512F-NEXT:    vpblendvb %ymm8, %ymm4, %ymm0, %ymm0
-; AVX512F-NEXT:    vpaddb %ymm0, %ymm0, %ymm4
+; AVX512F-NEXT:    vpblendvb %ymm8, %ymm4, %ymm3, %ymm3
+; AVX512F-NEXT:    vpaddb %ymm3, %ymm3, %ymm4
 ; AVX512F-NEXT:    vpaddb %ymm8, %ymm8, %ymm9
-; AVX512F-NEXT:    vpblendvb %ymm9, %ymm4, %ymm0, %ymm0
+; AVX512F-NEXT:    vpblendvb %ymm9, %ymm4, %ymm3, %ymm3
 ; AVX512F-NEXT:    vpxor %xmm4, %xmm4, %xmm4
 ; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm10 = ymm2[8],ymm4[8],ymm2[9],ymm4[9],ymm2[10],ymm4[10],ymm2[11],ymm4[11],ymm2[12],ymm4[12],ymm2[13],ymm4[13],ymm2[14],ymm4[14],ymm2[15],ymm4[15],ymm2[24],ymm4[24],ymm2[25],ymm4[25],ymm2[26],ymm4[26],ymm2[27],ymm4[27],ymm2[28],ymm4[28],ymm2[29],ymm4[29],ymm2[30],ymm4[30],ymm2[31],ymm4[31]
 ; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm11 = [256,2,4,8,16,32,64,128,256,2,4,8,16,32,64,128]
@@ -1124,49 +1160,52 @@ define <64 x i8> @constant_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y) nounwind {
 ; AVX512F-NEXT:    vpmullw %ymm13, %ymm12, %ymm12
 ; AVX512F-NEXT:    vpsrlw $8, %ymm12, %ymm12
 ; AVX512F-NEXT:    vpackuswb %ymm10, %ymm12, %ymm10
-; AVX512F-NEXT:    vpor %ymm10, %ymm0, %ymm0
+; AVX512F-NEXT:    vpor %ymm3, %ymm10, %ymm3
 ; AVX512F-NEXT:    vpbroadcastq {{.*#+}} ymm10 = [18446744073709551360,18446744073709551360,18446744073709551360,18446744073709551360]
-; AVX512F-NEXT:    vpblendvb %ymm10, %ymm0, %ymm2, %ymm0
-; AVX512F-NEXT:    vpsllw $4, %ymm1, %ymm2
-; AVX512F-NEXT:    vpand %ymm5, %ymm2, %ymm2
-; AVX512F-NEXT:    vpblendvb %ymm6, %ymm2, %ymm1, %ymm1
-; AVX512F-NEXT:    vpsllw $2, %ymm1, %ymm2
-; AVX512F-NEXT:    vpand %ymm7, %ymm2, %ymm2
-; AVX512F-NEXT:    vpblendvb %ymm8, %ymm2, %ymm1, %ymm1
-; AVX512F-NEXT:    vpaddb %ymm1, %ymm1, %ymm2
-; AVX512F-NEXT:    vpblendvb %ymm9, %ymm2, %ymm1, %ymm1
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm3[8],ymm4[8],ymm3[9],ymm4[9],ymm3[10],ymm4[10],ymm3[11],ymm4[11],ymm3[12],ymm4[12],ymm3[13],ymm4[13],ymm3[14],ymm4[14],ymm3[15],ymm4[15],ymm3[24],ymm4[24],ymm3[25],ymm4[25],ymm3[26],ymm4[26],ymm3[27],ymm4[27],ymm3[28],ymm4[28],ymm3[29],ymm4[29],ymm3[30],ymm4[30],ymm3[31],ymm4[31]
-; AVX512F-NEXT:    vpmullw %ymm11, %ymm2, %ymm2
-; AVX512F-NEXT:    vpsrlw $8, %ymm2, %ymm2
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm3[0],ymm4[0],ymm3[1],ymm4[1],ymm3[2],ymm4[2],ymm3[3],ymm4[3],ymm3[4],ymm4[4],ymm3[5],ymm4[5],ymm3[6],ymm4[6],ymm3[7],ymm4[7],ymm3[16],ymm4[16],ymm3[17],ymm4[17],ymm3[18],ymm4[18],ymm3[19],ymm4[19],ymm3[20],ymm4[20],ymm3[21],ymm4[21],ymm3[22],ymm4[22],ymm3[23],ymm4[23]
-; AVX512F-NEXT:    vpmullw %ymm13, %ymm4, %ymm4
+; AVX512F-NEXT:    vpblendvb %ymm10, %ymm3, %ymm2, %ymm2
+; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm3
+; AVX512F-NEXT:    vpand %ymm5, %ymm3, %ymm3
+; AVX512F-NEXT:    vpblendvb %ymm6, %ymm3, %ymm0, %ymm0
+; AVX512F-NEXT:    vpsllw $2, %ymm0, %ymm3
+; AVX512F-NEXT:    vpand %ymm7, %ymm3, %ymm3
+; AVX512F-NEXT:    vpblendvb %ymm8, %ymm3, %ymm0, %ymm0
+; AVX512F-NEXT:    vpaddb %ymm0, %ymm0, %ymm3
+; AVX512F-NEXT:    vpblendvb %ymm9, %ymm3, %ymm0, %ymm0
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm1[8],ymm4[8],ymm1[9],ymm4[9],ymm1[10],ymm4[10],ymm1[11],ymm4[11],ymm1[12],ymm4[12],ymm1[13],ymm4[13],ymm1[14],ymm4[14],ymm1[15],ymm4[15],ymm1[24],ymm4[24],ymm1[25],ymm4[25],ymm1[26],ymm4[26],ymm1[27],ymm4[27],ymm1[28],ymm4[28],ymm1[29],ymm4[29],ymm1[30],ymm4[30],ymm1[31],ymm4[31]
+; AVX512F-NEXT:    vpmullw %ymm3, %ymm11, %ymm3
+; AVX512F-NEXT:    vpsrlw $8, %ymm3, %ymm3
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm1[0],ymm4[0],ymm1[1],ymm4[1],ymm1[2],ymm4[2],ymm1[3],ymm4[3],ymm1[4],ymm4[4],ymm1[5],ymm4[5],ymm1[6],ymm4[6],ymm1[7],ymm4[7],ymm1[16],ymm4[16],ymm1[17],ymm4[17],ymm1[18],ymm4[18],ymm1[19],ymm4[19],ymm1[20],ymm4[20],ymm1[21],ymm4[21],ymm1[22],ymm4[22],ymm1[23],ymm4[23]
+; AVX512F-NEXT:    vpmullw %ymm4, %ymm13, %ymm4
 ; AVX512F-NEXT:    vpsrlw $8, %ymm4, %ymm4
-; AVX512F-NEXT:    vpackuswb %ymm2, %ymm4, %ymm2
-; AVX512F-NEXT:    vpor %ymm2, %ymm1, %ymm1
-; AVX512F-NEXT:    vpblendvb %ymm10, %ymm1, %ymm3, %ymm1
+; AVX512F-NEXT:    vpackuswb %ymm3, %ymm4, %ymm3
+; AVX512F-NEXT:    vpor %ymm3, %ymm0, %ymm0
+; AVX512F-NEXT:    vpblendvb %ymm10, %ymm0, %ymm1, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: constant_funnnel_v64i8:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm4
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
+; AVX512VL-NEXT:    vpsllw $4, %ymm3, %ymm4
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm5 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
 ; AVX512VL-NEXT:    vpand %ymm5, %ymm4, %ymm4
 ; AVX512VL-NEXT:    vbroadcasti128 {{.*#+}} ymm6 = [57344,41152,24704,8256,8192,24640,41088,57536,57344,41152,24704,8256,8192,24640,41088,57536]
 ; AVX512VL-NEXT:    # ymm6 = mem[0,1,0,1]
-; AVX512VL-NEXT:    vpblendvb %ymm6, %ymm4, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpsllw $2, %ymm0, %ymm4
+; AVX512VL-NEXT:    vpblendvb %ymm6, %ymm4, %ymm3, %ymm3
+; AVX512VL-NEXT:    vpsllw $2, %ymm3, %ymm4
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm7 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
 ; AVX512VL-NEXT:    vpand %ymm7, %ymm4, %ymm4
 ; AVX512VL-NEXT:    vpaddb %ymm6, %ymm6, %ymm8
-; AVX512VL-NEXT:    vpblendvb %ymm8, %ymm4, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpaddb %ymm0, %ymm0, %ymm4
+; AVX512VL-NEXT:    vpblendvb %ymm8, %ymm4, %ymm3, %ymm3
+; AVX512VL-NEXT:    vpaddb %ymm3, %ymm3, %ymm4
 ; AVX512VL-NEXT:    vpaddb %ymm8, %ymm8, %ymm9
-; AVX512VL-NEXT:    vpblendvb %ymm9, %ymm4, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpblendvb %ymm9, %ymm4, %ymm3, %ymm3
 ; AVX512VL-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
 ; AVX512VL-NEXT:    vpsrlw $8, %ymm4, %ymm4
 ; AVX512VL-NEXT:    vbroadcasti128 {{.*#+}} ymm10 = [256,2,4,8,16,32,64,128,256,2,4,8,16,32,64,128]
 ; AVX512VL-NEXT:    # ymm10 = mem[0,1,0,1]
-; AVX512VL-NEXT:    vpmullw %ymm10, %ymm4, %ymm4
+; AVX512VL-NEXT:    vpmullw %ymm4, %ymm10, %ymm4
 ; AVX512VL-NEXT:    vpsrlw $8, %ymm4, %ymm4
 ; AVX512VL-NEXT:    vpunpcklbw {{.*#+}} ymm11 = ymm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
 ; AVX512VL-NEXT:    vpsrlw $8, %ymm11, %ymm11
@@ -1175,28 +1214,29 @@ define <64 x i8> @constant_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y) nounwind {
 ; AVX512VL-NEXT:    vpmullw %ymm12, %ymm11, %ymm11
 ; AVX512VL-NEXT:    vpsrlw $8, %ymm11, %ymm11
 ; AVX512VL-NEXT:    vpackuswb %ymm4, %ymm11, %ymm4
-; AVX512VL-NEXT:    vpor %ymm4, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpor %ymm4, %ymm3, %ymm3
 ; AVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [18446744073709551360,18446744073709551360,18446744073709551360,18446744073709551360]
-; AVX512VL-NEXT:    vpblendvb %ymm4, %ymm0, %ymm2, %ymm0
-; AVX512VL-NEXT:    vpsllw $4, %ymm1, %ymm2
-; AVX512VL-NEXT:    vpand %ymm5, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpblendvb %ymm6, %ymm2, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpsllw $2, %ymm1, %ymm2
-; AVX512VL-NEXT:    vpand %ymm7, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpblendvb %ymm8, %ymm2, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpaddb %ymm1, %ymm1, %ymm2
-; AVX512VL-NEXT:    vpblendvb %ymm9, %ymm2, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
-; AVX512VL-NEXT:    vpsrlw $8, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpmullw %ymm10, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpsrlw $8, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512VL-NEXT:    vpblendvb %ymm4, %ymm3, %ymm2, %ymm2
+; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm3
+; AVX512VL-NEXT:    vpand %ymm5, %ymm3, %ymm3
+; AVX512VL-NEXT:    vpblendvb %ymm6, %ymm3, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpsllw $2, %ymm0, %ymm3
+; AVX512VL-NEXT:    vpand %ymm7, %ymm3, %ymm3
+; AVX512VL-NEXT:    vpblendvb %ymm8, %ymm3, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpaddb %ymm0, %ymm0, %ymm3
+; AVX512VL-NEXT:    vpblendvb %ymm9, %ymm3, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX512VL-NEXT:    vpsrlw $8, %ymm3, %ymm3
+; AVX512VL-NEXT:    vpmullw %ymm3, %ymm10, %ymm3
+; AVX512VL-NEXT:    vpsrlw $8, %ymm3, %ymm3
+; AVX512VL-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
 ; AVX512VL-NEXT:    vpsrlw $8, %ymm5, %ymm5
-; AVX512VL-NEXT:    vpmullw %ymm12, %ymm5, %ymm5
+; AVX512VL-NEXT:    vpmullw %ymm5, %ymm12, %ymm5
 ; AVX512VL-NEXT:    vpsrlw $8, %ymm5, %ymm5
-; AVX512VL-NEXT:    vpackuswb %ymm2, %ymm5, %ymm2
-; AVX512VL-NEXT:    vpor %ymm2, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpblendvb %ymm4, %ymm1, %ymm3, %ymm1
+; AVX512VL-NEXT:    vpackuswb %ymm3, %ymm5, %ymm3
+; AVX512VL-NEXT:    vpor %ymm3, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpblendvb %ymm4, %ymm0, %ymm1, %ymm0
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: constant_funnnel_v64i8:
@@ -1417,22 +1457,28 @@ define <16 x i32> @splatconstant_funnnel_v16i32(<16 x i32> %x, <16 x i32> %y) no
 define <32 x i16> @splatconstant_funnnel_v32i16(<32 x i16> %x, <32 x i16> %y) nounwind {
 ; AVX512F-LABEL: splatconstant_funnnel_v32i16:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpsrlw $7, %ymm2, %ymm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
+; AVX512F-NEXT:    vpsrlw $7, %ymm3, %ymm3
+; AVX512F-NEXT:    vpsllw $9, %ymm2, %ymm2
+; AVX512F-NEXT:    vpor %ymm3, %ymm2, %ymm2
+; AVX512F-NEXT:    vpsrlw $7, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpsllw $9, %ymm0, %ymm0
-; AVX512F-NEXT:    vpor %ymm2, %ymm0, %ymm0
-; AVX512F-NEXT:    vpsrlw $7, %ymm3, %ymm2
-; AVX512F-NEXT:    vpsllw $9, %ymm1, %ymm1
-; AVX512F-NEXT:    vpor %ymm2, %ymm1, %ymm1
+; AVX512F-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatconstant_funnnel_v32i16:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsrlw $7, %ymm2, %ymm2
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
+; AVX512VL-NEXT:    vpsrlw $7, %ymm3, %ymm3
+; AVX512VL-NEXT:    vpsllw $9, %ymm2, %ymm2
+; AVX512VL-NEXT:    vpor %ymm3, %ymm2, %ymm2
+; AVX512VL-NEXT:    vpsrlw $7, %ymm1, %ymm1
 ; AVX512VL-NEXT:    vpsllw $9, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpor %ymm2, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpsrlw $7, %ymm3, %ymm2
-; AVX512VL-NEXT:    vpsllw $9, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpor %ymm2, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatconstant_funnnel_v32i16:
@@ -1465,68 +1511,62 @@ define <32 x i16> @splatconstant_funnnel_v32i16(<32 x i16> %x, <32 x i16> %y) no
 define <64 x i8> @splatconstant_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y) nounwind {
 ; AVX512F-LABEL: splatconstant_funnnel_v64i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpsrlw $4, %ymm2, %ymm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
+; AVX512F-NEXT:    vpsrlw $4, %ymm3, %ymm3
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm4 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX512F-NEXT:    vpandn %ymm2, %ymm4, %ymm2
+; AVX512F-NEXT:    vpandn %ymm3, %ymm4, %ymm3
+; AVX512F-NEXT:    vpsllw $4, %ymm2, %ymm2
+; AVX512F-NEXT:    vpand %ymm4, %ymm2, %ymm2
+; AVX512F-NEXT:    vpor %ymm3, %ymm2, %ymm2
+; AVX512F-NEXT:    vpsrlw $4, %ymm1, %ymm1
+; AVX512F-NEXT:    vpandn %ymm1, %ymm4, %ymm1
 ; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpand %ymm4, %ymm0, %ymm0
-; AVX512F-NEXT:    vpor %ymm2, %ymm0, %ymm0
-; AVX512F-NEXT:    vpsrlw $4, %ymm3, %ymm2
-; AVX512F-NEXT:    vpandn %ymm2, %ymm4, %ymm2
-; AVX512F-NEXT:    vpsllw $4, %ymm1, %ymm1
-; AVX512F-NEXT:    vpand %ymm4, %ymm1, %ymm1
-; AVX512F-NEXT:    vpor %ymm2, %ymm1, %ymm1
+; AVX512F-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatconstant_funnnel_v64i8:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsrlw $4, %ymm2, %ymm2
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
+; AVX512VL-NEXT:    vpsrlw $4, %ymm3, %ymm3
+; AVX512VL-NEXT:    vpsllw $4, %ymm2, %ymm2
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm4 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX512VL-NEXT:    vpandn %ymm2, %ymm4, %ymm2
+; AVX512VL-NEXT:    vpternlogq $226, %ymm3, %ymm4, %ymm2
+; AVX512VL-NEXT:    vpsrlw $4, %ymm1, %ymm1
 ; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpand %ymm4, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpor %ymm2, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpsrlw $4, %ymm3, %ymm2
-; AVX512VL-NEXT:    vpandn %ymm2, %ymm4, %ymm2
-; AVX512VL-NEXT:    vpsllw $4, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpand %ymm4, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpor %ymm2, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpternlogq $226, %ymm1, %ymm4, %ymm0
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatconstant_funnnel_v64i8:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpsrlw $4, %zmm1, %zmm1
-; AVX512BW-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
-; AVX512BW-NEXT:    vpsllw $4, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
-; AVX512BW-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsllw $4, %zmm0, %zmm2
+; AVX512BW-NEXT:    vpsrlw $4, %zmm1, %zmm0
+; AVX512BW-NEXT:    vpternlogq $216, {{.*}}(%rip), %zmm2, %zmm0
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512VBMI2-LABEL: splatconstant_funnnel_v64i8:
 ; AVX512VBMI2:       # %bb.0:
-; AVX512VBMI2-NEXT:    vpsrlw $4, %zmm1, %zmm1
-; AVX512VBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
-; AVX512VBMI2-NEXT:    vpsllw $4, %zmm0, %zmm0
-; AVX512VBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
-; AVX512VBMI2-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    vpsllw $4, %zmm0, %zmm2
+; AVX512VBMI2-NEXT:    vpsrlw $4, %zmm1, %zmm0
+; AVX512VBMI2-NEXT:    vpternlogq $216, {{.*}}(%rip), %zmm2, %zmm0
 ; AVX512VBMI2-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: splatconstant_funnnel_v64i8:
 ; AVX512VLBW:       # %bb.0:
-; AVX512VLBW-NEXT:    vpsrlw $4, %zmm1, %zmm1
-; AVX512VLBW-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
-; AVX512VLBW-NEXT:    vpsllw $4, %zmm0, %zmm0
-; AVX512VLBW-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
-; AVX512VLBW-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512VLBW-NEXT:    vpsllw $4, %zmm0, %zmm2
+; AVX512VLBW-NEXT:    vpsrlw $4, %zmm1, %zmm0
+; AVX512VLBW-NEXT:    vpternlogq $216, {{.*}}(%rip), %zmm2, %zmm0
 ; AVX512VLBW-NEXT:    retq
 ;
 ; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v64i8:
 ; AVX512VLVBMI2:       # %bb.0:
-; AVX512VLVBMI2-NEXT:    vpsrlw $4, %zmm1, %zmm1
-; AVX512VLVBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
-; AVX512VLVBMI2-NEXT:    vpsllw $4, %zmm0, %zmm0
-; AVX512VLVBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
-; AVX512VLVBMI2-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    vpsllw $4, %zmm0, %zmm2
+; AVX512VLVBMI2-NEXT:    vpsrlw $4, %zmm1, %zmm0
+; AVX512VLVBMI2-NEXT:    vpternlogq $216, {{.*}}(%rip), %zmm2, %zmm0
 ; AVX512VLVBMI2-NEXT:    retq
   %res = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>)
   ret <64 x i8> %res
diff --git a/llvm/test/CodeGen/X86/vector-fshr-rot-128.ll b/llvm/test/CodeGen/X86/vector-fshr-rot-128.ll
index fc915c2ea..0439459ac 100644
--- a/llvm/test/CodeGen/X86/vector-fshr-rot-128.ll
+++ b/llvm/test/CodeGen/X86/vector-fshr-rot-128.ll
@@ -290,22 +290,21 @@ define <8 x i16> @var_funnnel_v8i16(<8 x i16> %x, <8 x i16> %amt) nounwind {
 ; SSE2-LABEL: var_funnnel_v8i16:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    psubw %xmm1, %xmm3
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm3
-; SSE2-NEXT:    movdqa %xmm3, %xmm1
-; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; SSE2-NEXT:    psubw %xmm1, %xmm2
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm1
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
 ; SSE2-NEXT:    pslld $23, %xmm1
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [1065353216,1065353216,1065353216,1065353216]
-; SSE2-NEXT:    paddd %xmm4, %xmm1
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [1065353216,1065353216,1065353216,1065353216]
+; SSE2-NEXT:    paddd %xmm3, %xmm1
 ; SSE2-NEXT:    cvttps2dq %xmm1, %xmm1
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
 ; SSE2-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,6,6,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; SSE2-NEXT:    pslld $23, %xmm3
-; SSE2-NEXT:    paddd %xmm4, %xmm3
-; SSE2-NEXT:    cvttps2dq %xmm3, %xmm2
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
+; SSE2-NEXT:    pslld $23, %xmm2
+; SSE2-NEXT:    paddd %xmm3, %xmm2
+; SSE2-NEXT:    cvttps2dq %xmm2, %xmm2
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,2,2,3,4,5,6,7]
 ; SSE2-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,6,6,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
@@ -319,19 +318,18 @@ define <8 x i16> @var_funnnel_v8i16(<8 x i16> %x, <8 x i16> %amt) nounwind {
 ; SSE41-LABEL: var_funnnel_v8i16:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pxor %xmm2, %xmm2
-; SSE41-NEXT:    pxor %xmm3, %xmm3
-; SSE41-NEXT:    psubw %xmm1, %xmm3
-; SSE41-NEXT:    pand {{.*}}(%rip), %xmm3
-; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm1 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero
-; SSE41-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
-; SSE41-NEXT:    pslld $23, %xmm3
-; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [1065353216,1065353216,1065353216,1065353216]
-; SSE41-NEXT:    paddd %xmm2, %xmm3
-; SSE41-NEXT:    cvttps2dq %xmm3, %xmm3
+; SSE41-NEXT:    psubw %xmm1, %xmm2
+; SSE41-NEXT:    pand {{.*}}(%rip), %xmm2
+; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm1 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
+; SSE41-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE41-NEXT:    pslld $23, %xmm2
+; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [1065353216,1065353216,1065353216,1065353216]
+; SSE41-NEXT:    paddd %xmm3, %xmm2
+; SSE41-NEXT:    cvttps2dq %xmm2, %xmm2
 ; SSE41-NEXT:    pslld $23, %xmm1
-; SSE41-NEXT:    paddd %xmm2, %xmm1
+; SSE41-NEXT:    paddd %xmm3, %xmm1
 ; SSE41-NEXT:    cvttps2dq %xmm1, %xmm1
-; SSE41-NEXT:    packusdw %xmm3, %xmm1
+; SSE41-NEXT:    packusdw %xmm2, %xmm1
 ; SSE41-NEXT:    movdqa %xmm0, %xmm2
 ; SSE41-NEXT:    pmulhuw %xmm1, %xmm2
 ; SSE41-NEXT:    pmullw %xmm1, %xmm0
@@ -343,7 +341,7 @@ define <8 x i16> @var_funnnel_v8i16(<8 x i16> %x, <8 x i16> %amt) nounwind {
 ; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; AVX1-NEXT:    vpsubw %xmm1, %xmm2, %xmm1
 ; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
-; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
 ; AVX1-NEXT:    vpslld $23, %xmm2, %xmm2
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [1065353216,1065353216,1065353216,1065353216]
 ; AVX1-NEXT:    vpaddd %xmm3, %xmm2, %xmm2
@@ -363,19 +361,18 @@ define <8 x i16> @var_funnnel_v8i16(<8 x i16> %x, <8 x i16> %amt) nounwind {
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; AVX2-NEXT:    vpsubw %xmm1, %xmm2, %xmm1
 ; AVX2-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
-; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [16,16,16,16,16,16,16,16]
+; AVX2-NEXT:    vpsubw %xmm1, %xmm2, %xmm2
+; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
 ; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX2-NEXT:    vpsllvd %ymm2, %ymm0, %ymm2
-; AVX2-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; AVX2-NEXT:    vpshufb %ymm3, %ymm2, %ymm2
-; AVX2-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,2,3]
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm4 = [16,16,16,16,16,16,16,16]
-; AVX2-NEXT:    vpsubw %xmm1, %xmm4, %xmm1
+; AVX2-NEXT:    vpsrlvd %ymm2, %ymm0, %ymm2
+; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm3
+; AVX2-NEXT:    vpackusdw %xmm3, %xmm2, %xmm2
 ; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
-; AVX2-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpshufb %ymm3, %ymm0, %ymm0
+; AVX2-NEXT:    vpsllvd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-NEXT:    vpor %xmm0, %xmm2, %xmm0
+; AVX2-NEXT:    vpor %xmm2, %xmm0, %xmm0
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
@@ -450,22 +447,21 @@ define <8 x i16> @var_funnnel_v8i16(<8 x i16> %x, <8 x i16> %amt) nounwind {
 ; X32-SSE-LABEL: var_funnnel_v8i16:
 ; X32-SSE:       # %bb.0:
 ; X32-SSE-NEXT:    pxor %xmm2, %xmm2
-; X32-SSE-NEXT:    pxor %xmm3, %xmm3
-; X32-SSE-NEXT:    psubw %xmm1, %xmm3
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm3
-; X32-SSE-NEXT:    movdqa %xmm3, %xmm1
-; X32-SSE-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; X32-SSE-NEXT:    psubw %xmm1, %xmm2
+; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm2
+; X32-SSE-NEXT:    movdqa %xmm2, %xmm1
+; X32-SSE-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
 ; X32-SSE-NEXT:    pslld $23, %xmm1
-; X32-SSE-NEXT:    movdqa {{.*#+}} xmm4 = [1065353216,1065353216,1065353216,1065353216]
-; X32-SSE-NEXT:    paddd %xmm4, %xmm1
+; X32-SSE-NEXT:    movdqa {{.*#+}} xmm3 = [1065353216,1065353216,1065353216,1065353216]
+; X32-SSE-NEXT:    paddd %xmm3, %xmm1
 ; X32-SSE-NEXT:    cvttps2dq %xmm1, %xmm1
 ; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
 ; X32-SSE-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,6,6,7]
 ; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; X32-SSE-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; X32-SSE-NEXT:    pslld $23, %xmm3
-; X32-SSE-NEXT:    paddd %xmm4, %xmm3
-; X32-SSE-NEXT:    cvttps2dq %xmm3, %xmm2
+; X32-SSE-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
+; X32-SSE-NEXT:    pslld $23, %xmm2
+; X32-SSE-NEXT:    paddd %xmm3, %xmm2
+; X32-SSE-NEXT:    cvttps2dq %xmm2, %xmm2
 ; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,2,2,3,4,5,6,7]
 ; X32-SSE-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,6,6,7]
 ; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
@@ -1920,14 +1916,37 @@ define <16 x i8> @splatconstant_funnnel_v16i8(<16 x i8> %x) nounwind {
 ; AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
-; AVX512-LABEL: splatconstant_funnnel_v16i8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsrlw $4, %xmm0, %xmm1
-; AVX512-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
-; AVX512-NEXT:    vpsllw $4, %xmm0, %xmm0
-; AVX512-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: splatconstant_funnnel_v16i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsrlw $4, %xmm0, %xmm1
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512F-NEXT:    vpsllw $4, %xmm0, %xmm0
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX512F-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: splatconstant_funnnel_v16i8:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsllw $4, %xmm0, %xmm1
+; AVX512VL-NEXT:    vpsrlw $4, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpternlogq $216, {{.*}}(%rip), %xmm1, %xmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: splatconstant_funnnel_v16i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsrlw $4, %xmm0, %xmm1
+; AVX512BW-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512BW-NEXT:    vpsllw $4, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX512BW-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VLBW-LABEL: splatconstant_funnnel_v16i8:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpsllw $4, %xmm0, %xmm1
+; AVX512VLBW-NEXT:    vpsrlw $4, %xmm0, %xmm0
+; AVX512VLBW-NEXT:    vpternlogq $216, {{.*}}(%rip), %xmm1, %xmm0
+; AVX512VLBW-NEXT:    retq
 ;
 ; XOP-LABEL: splatconstant_funnnel_v16i8:
 ; XOP:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/vector-fshr-rot-256.ll b/llvm/test/CodeGen/X86/vector-fshr-rot-256.ll
index 6a2f6a889..7d6a5eef9 100644
--- a/llvm/test/CodeGen/X86/vector-fshr-rot-256.ll
+++ b/llvm/test/CodeGen/X86/vector-fshr-rot-256.ll
@@ -227,7 +227,7 @@ define <16 x i16> @var_funnnel_v16i16(<16 x i16> %x, <16 x i16> %amt) nounwind {
 ; AVX1-NEXT:    vpsubw %xmm2, %xmm3, %xmm2
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [15,15,15,15,15,15,15,15]
 ; AVX1-NEXT:    vpand %xmm4, %xmm2, %xmm2
-; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
 ; AVX1-NEXT:    vpslld $23, %xmm5, %xmm5
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [1065353216,1065353216,1065353216,1065353216]
 ; AVX1-NEXT:    vpaddd %xmm6, %xmm5, %xmm5
@@ -243,7 +243,7 @@ define <16 x i16> @var_funnnel_v16i16(<16 x i16> %x, <16 x i16> %amt) nounwind {
 ; AVX1-NEXT:    vpor %xmm7, %xmm2, %xmm2
 ; AVX1-NEXT:    vpsubw %xmm1, %xmm3, %xmm1
 ; AVX1-NEXT:    vpand %xmm4, %xmm1, %xmm1
-; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7]
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
 ; AVX1-NEXT:    vpslld $23, %xmm3, %xmm3
 ; AVX1-NEXT:    vpaddd %xmm6, %xmm3, %xmm3
 ; AVX1-NEXT:    vcvttps2dq %xmm3, %xmm3
@@ -392,7 +392,7 @@ define <32 x i8> @var_funnnel_v32i8(<32 x i8> %x, <32 x i8> %amt) nounwind {
 ; AVX1-NEXT:    vpblendvb %xmm5, %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vpsrlw $7, %xmm2, %xmm3
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm9 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; AVX1-NEXT:    vpand %xmm9, %xmm3, %xmm3
+; AVX1-NEXT:    vpand %xmm3, %xmm9, %xmm3
 ; AVX1-NEXT:    vpaddb %xmm2, %xmm2, %xmm6
 ; AVX1-NEXT:    vpor %xmm3, %xmm6, %xmm3
 ; AVX1-NEXT:    vpaddb %xmm5, %xmm5, %xmm5
@@ -413,7 +413,7 @@ define <32 x i8> @var_funnnel_v32i8(<32 x i8> %x, <32 x i8> %amt) nounwind {
 ; AVX1-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
 ; AVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrlw $7, %xmm0, %xmm3
-; AVX1-NEXT:    vpand %xmm9, %xmm3, %xmm3
+; AVX1-NEXT:    vpand %xmm3, %xmm9, %xmm3
 ; AVX1-NEXT:    vpaddb %xmm0, %xmm0, %xmm4
 ; AVX1-NEXT:    vpor %xmm3, %xmm4, %xmm3
 ; AVX1-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
@@ -475,22 +475,18 @@ define <32 x i8> @var_funnnel_v32i8(<32 x i8> %x, <32 x i8> %amt) nounwind {
 ;
 ; AVX512VL-LABEL: var_funnnel_v32i8:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm2
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
-; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm3
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm3, %ymm3
-; AVX512VL-NEXT:    vpor %ymm2, %ymm3, %ymm2
-; AVX512VL-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX512VL-NEXT:    vpsubb %ymm1, %ymm3, %ymm1
+; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm2
+; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm3
+; AVX512VL-NEXT:    vpternlogq $216, {{.*}}(%rip), %ymm2, %ymm3
+; AVX512VL-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512VL-NEXT:    vpsubb %ymm1, %ymm2, %ymm1
 ; AVX512VL-NEXT:    vpsllw $5, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpsrlw $6, %ymm0, %ymm2
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
-; AVX512VL-NEXT:    vpsllw $2, %ymm0, %ymm3
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm3, %ymm3
-; AVX512VL-NEXT:    vpor %ymm2, %ymm3, %ymm2
+; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpsllw $2, %ymm0, %ymm2
+; AVX512VL-NEXT:    vpsrlw $6, %ymm0, %ymm3
+; AVX512VL-NEXT:    vpternlogq $216, {{.*}}(%rip), %ymm2, %ymm3
 ; AVX512VL-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vpaddb %ymm0, %ymm0, %ymm2
 ; AVX512VL-NEXT:    vpsrlw $7, %ymm0, %ymm3
 ; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm3, %ymm3
@@ -1223,7 +1219,7 @@ define <32 x i8> @constant_funnnel_v32i8(<32 x i8> %x) nounwind {
 ; AVX1-NEXT:    vpxor %xmm8, %xmm8, %xmm8
 ; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm1[8],xmm8[8],xmm1[9],xmm8[9],xmm1[10],xmm8[10],xmm1[11],xmm8[11],xmm1[12],xmm8[12],xmm1[13],xmm8[13],xmm1[14],xmm8[14],xmm1[15],xmm8[15]
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm9 = [256,2,4,8,16,32,64,128]
-; AVX1-NEXT:    vpmullw %xmm9, %xmm3, %xmm3
+; AVX1-NEXT:    vpmullw %xmm3, %xmm9, %xmm3
 ; AVX1-NEXT:    vpsrlw $8, %xmm3, %xmm3
 ; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm5 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [256,128,64,32,16,8,4,2]
@@ -1241,7 +1237,7 @@ define <32 x i8> @constant_funnnel_v32i8(<32 x i8> %x) nounwind {
 ; AVX1-NEXT:    vpackuswb %xmm1, %xmm5, %xmm1
 ; AVX1-NEXT:    vpor %xmm3, %xmm1, %xmm1
 ; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm0[8],xmm8[8],xmm0[9],xmm8[9],xmm0[10],xmm8[10],xmm0[11],xmm8[11],xmm0[12],xmm8[12],xmm0[13],xmm8[13],xmm0[14],xmm8[14],xmm0[15],xmm8[15]
-; AVX1-NEXT:    vpmullw %xmm9, %xmm3, %xmm3
+; AVX1-NEXT:    vpmullw %xmm3, %xmm9, %xmm3
 ; AVX1-NEXT:    vpsrlw $8, %xmm3, %xmm3
 ; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm5 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; AVX1-NEXT:    vpmullw %xmm6, %xmm5, %xmm6
@@ -1575,14 +1571,37 @@ define <32 x i8> @splatconstant_funnnel_v32i8(<32 x i8> %x) nounwind {
 ; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    retq
 ;
-; AVX512-LABEL: splatconstant_funnnel_v32i8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsrlw $4, %ymm0, %ymm1
-; AVX512-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
-; AVX512-NEXT:    vpsllw $4, %ymm0, %ymm0
-; AVX512-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
-; AVX512-NEXT:    vpor %ymm1, %ymm0, %ymm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: splatconstant_funnnel_v32i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsrlw $4, %ymm0, %ymm1
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
+; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm0
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
+; AVX512F-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: splatconstant_funnnel_v32i8:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm1
+; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpternlogq $216, {{.*}}(%rip), %ymm1, %ymm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: splatconstant_funnnel_v32i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsrlw $4, %ymm0, %ymm1
+; AVX512BW-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
+; AVX512BW-NEXT:    vpsllw $4, %ymm0, %ymm0
+; AVX512BW-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
+; AVX512BW-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VLBW-LABEL: splatconstant_funnnel_v32i8:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpsllw $4, %ymm0, %ymm1
+; AVX512VLBW-NEXT:    vpsrlw $4, %ymm0, %ymm0
+; AVX512VLBW-NEXT:    vpternlogq $216, {{.*}}(%rip), %ymm1, %ymm0
+; AVX512VLBW-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: splatconstant_funnnel_v32i8:
 ; XOPAVX1:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/vector-fshr-rot-512.ll b/llvm/test/CodeGen/X86/vector-fshr-rot-512.ll
index 2a25efd50..d8d300f66 100644
--- a/llvm/test/CodeGen/X86/vector-fshr-rot-512.ll
+++ b/llvm/test/CodeGen/X86/vector-fshr-rot-512.ll
@@ -34,56 +34,62 @@ define <16 x i32> @var_funnnel_v16i32(<16 x i32> %x, <16 x i32> %amt) nounwind {
 define <32 x i16> @var_funnnel_v32i16(<32 x i16> %x, <32 x i16> %amt) nounwind {
 ; AVX512F-LABEL: var_funnnel_v32i16:
 ; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
 ; AVX512F-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX512F-NEXT:    vpsubw %ymm2, %ymm4, %ymm2
+; AVX512F-NEXT:    vpsubw %ymm3, %ymm4, %ymm3
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm5 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512F-NEXT:    vpand %ymm5, %ymm2, %ymm2
-; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm6 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
-; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
-; AVX512F-NEXT:    vpsllvd %zmm6, %zmm0, %zmm6
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm7 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
-; AVX512F-NEXT:    vpsubw %ymm2, %ymm7, %ymm2
+; AVX512F-NEXT:    vpand %ymm5, %ymm3, %ymm3
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm6 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
 ; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
-; AVX512F-NEXT:    vpsrlvd %zmm2, %zmm0, %zmm0
-; AVX512F-NEXT:    vpord %zmm0, %zmm6, %zmm0
-; AVX512F-NEXT:    vpmovdw %zmm0, %ymm0
-; AVX512F-NEXT:    vpsubw %ymm3, %ymm4, %ymm2
-; AVX512F-NEXT:    vpand %ymm5, %ymm2, %ymm2
-; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm3 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
+; AVX512F-NEXT:    vpsllvd %zmm6, %zmm2, %zmm6
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm7 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
+; AVX512F-NEXT:    vpsubw %ymm3, %ymm7, %ymm3
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm3 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
+; AVX512F-NEXT:    vpsrlvd %zmm3, %zmm2, %zmm2
+; AVX512F-NEXT:    vpord %zmm2, %zmm6, %zmm2
+; AVX512F-NEXT:    vpmovdw %zmm2, %ymm2
+; AVX512F-NEXT:    vpsubw %ymm1, %ymm4, %ymm1
+; AVX512F-NEXT:    vpand %ymm5, %ymm1, %ymm1
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm3 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
+; AVX512F-NEXT:    vpsllvd %zmm3, %zmm0, %zmm3
+; AVX512F-NEXT:    vpsubw %ymm1, %ymm7, %ymm1
 ; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
-; AVX512F-NEXT:    vpsllvd %zmm3, %zmm1, %zmm3
-; AVX512F-NEXT:    vpsubw %ymm2, %ymm7, %ymm2
-; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
-; AVX512F-NEXT:    vpsrlvd %zmm2, %zmm1, %zmm1
-; AVX512F-NEXT:    vpord %zmm1, %zmm3, %zmm1
-; AVX512F-NEXT:    vpmovdw %zmm1, %ymm1
+; AVX512F-NEXT:    vpsrlvd %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vpord %zmm0, %zmm3, %zmm0
+; AVX512F-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: var_funnnel_v32i16:
 ; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
 ; AVX512VL-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX512VL-NEXT:    vpsubw %ymm2, %ymm4, %ymm2
+; AVX512VL-NEXT:    vpsubw %ymm3, %ymm4, %ymm3
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm5 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512VL-NEXT:    vpand %ymm5, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm6 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
-; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
-; AVX512VL-NEXT:    vpsllvd %zmm6, %zmm0, %zmm6
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm7 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
-; AVX512VL-NEXT:    vpsubw %ymm2, %ymm7, %ymm2
+; AVX512VL-NEXT:    vpand %ymm5, %ymm3, %ymm3
+; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm6 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
 ; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
-; AVX512VL-NEXT:    vpsrlvd %zmm2, %zmm0, %zmm0
-; AVX512VL-NEXT:    vpord %zmm0, %zmm6, %zmm0
-; AVX512VL-NEXT:    vpmovdw %zmm0, %ymm0
-; AVX512VL-NEXT:    vpsubw %ymm3, %ymm4, %ymm2
-; AVX512VL-NEXT:    vpand %ymm5, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm3 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
+; AVX512VL-NEXT:    vpsllvd %zmm6, %zmm2, %zmm6
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm7 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
+; AVX512VL-NEXT:    vpsubw %ymm3, %ymm7, %ymm3
+; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm3 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
+; AVX512VL-NEXT:    vpsrlvd %zmm3, %zmm2, %zmm2
+; AVX512VL-NEXT:    vpord %zmm2, %zmm6, %zmm2
+; AVX512VL-NEXT:    vpmovdw %zmm2, %ymm2
+; AVX512VL-NEXT:    vpsubw %ymm1, %ymm4, %ymm1
+; AVX512VL-NEXT:    vpand %ymm5, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm3 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
+; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
+; AVX512VL-NEXT:    vpsllvd %zmm3, %zmm0, %zmm3
+; AVX512VL-NEXT:    vpsubw %ymm1, %ymm7, %ymm1
 ; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
-; AVX512VL-NEXT:    vpsllvd %zmm3, %zmm1, %zmm3
-; AVX512VL-NEXT:    vpsubw %ymm2, %ymm7, %ymm2
-; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
-; AVX512VL-NEXT:    vpsrlvd %zmm2, %zmm1, %zmm1
-; AVX512VL-NEXT:    vpord %zmm1, %zmm3, %zmm1
-; AVX512VL-NEXT:    vpmovdw %zmm1, %ymm1
+; AVX512VL-NEXT:    vpsrlvd %zmm1, %zmm0, %zmm0
+; AVX512VL-NEXT:    vpord %zmm0, %zmm3, %zmm0
+; AVX512VL-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: var_funnnel_v32i16:
@@ -116,10 +122,12 @@ define <32 x i16> @var_funnnel_v32i16(<32 x i16> %x, <32 x i16> %amt) nounwind {
 define <64 x i8> @var_funnnel_v64i8(<64 x i8> %x, <64 x i8> %amt) nounwind {
 ; AVX512F-LABEL: var_funnnel_v64i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpsrlw $4, %ymm0, %ymm4
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
+; AVX512F-NEXT:    vpsrlw $4, %ymm3, %ymm4
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm5 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
 ; AVX512F-NEXT:    vpandn %ymm4, %ymm5, %ymm4
-; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm6
+; AVX512F-NEXT:    vpsllw $4, %ymm3, %ymm6
 ; AVX512F-NEXT:    vpand %ymm5, %ymm6, %ymm6
 ; AVX512F-NEXT:    vpor %ymm4, %ymm6, %ymm4
 ; AVX512F-NEXT:    vpxor %xmm6, %xmm6, %xmm6
@@ -127,97 +135,93 @@ define <64 x i8> @var_funnnel_v64i8(<64 x i8> %x, <64 x i8> %amt) nounwind {
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm7 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
 ; AVX512F-NEXT:    vpand %ymm7, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpsllw $5, %ymm2, %ymm2
-; AVX512F-NEXT:    vpblendvb %ymm2, %ymm4, %ymm0, %ymm0
-; AVX512F-NEXT:    vpsrlw $6, %ymm0, %ymm4
+; AVX512F-NEXT:    vpblendvb %ymm2, %ymm4, %ymm3, %ymm3
+; AVX512F-NEXT:    vpsrlw $6, %ymm3, %ymm4
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm8 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
 ; AVX512F-NEXT:    vpandn %ymm4, %ymm8, %ymm4
-; AVX512F-NEXT:    vpsllw $2, %ymm0, %ymm9
+; AVX512F-NEXT:    vpsllw $2, %ymm3, %ymm9
 ; AVX512F-NEXT:    vpand %ymm8, %ymm9, %ymm9
 ; AVX512F-NEXT:    vpor %ymm4, %ymm9, %ymm4
 ; AVX512F-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
-; AVX512F-NEXT:    vpblendvb %ymm2, %ymm4, %ymm0, %ymm0
-; AVX512F-NEXT:    vpsrlw $7, %ymm0, %ymm4
+; AVX512F-NEXT:    vpblendvb %ymm2, %ymm4, %ymm3, %ymm3
+; AVX512F-NEXT:    vpsrlw $7, %ymm3, %ymm4
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm9 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; AVX512F-NEXT:    vpand %ymm9, %ymm4, %ymm4
-; AVX512F-NEXT:    vpaddb %ymm0, %ymm0, %ymm10
+; AVX512F-NEXT:    vpand %ymm4, %ymm9, %ymm4
+; AVX512F-NEXT:    vpaddb %ymm3, %ymm3, %ymm10
 ; AVX512F-NEXT:    vpor %ymm4, %ymm10, %ymm4
 ; AVX512F-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
-; AVX512F-NEXT:    vpblendvb %ymm2, %ymm4, %ymm0, %ymm0
-; AVX512F-NEXT:    vpsrlw $4, %ymm1, %ymm2
-; AVX512F-NEXT:    vpandn %ymm2, %ymm5, %ymm2
-; AVX512F-NEXT:    vpsllw $4, %ymm1, %ymm4
+; AVX512F-NEXT:    vpblendvb %ymm2, %ymm4, %ymm3, %ymm2
+; AVX512F-NEXT:    vpsrlw $4, %ymm0, %ymm3
+; AVX512F-NEXT:    vpandn %ymm3, %ymm5, %ymm3
+; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm4
 ; AVX512F-NEXT:    vpand %ymm5, %ymm4, %ymm4
-; AVX512F-NEXT:    vpor %ymm2, %ymm4, %ymm2
-; AVX512F-NEXT:    vpsubb %ymm3, %ymm6, %ymm3
-; AVX512F-NEXT:    vpand %ymm7, %ymm3, %ymm3
-; AVX512F-NEXT:    vpsllw $5, %ymm3, %ymm3
-; AVX512F-NEXT:    vpblendvb %ymm3, %ymm2, %ymm1, %ymm1
-; AVX512F-NEXT:    vpsrlw $6, %ymm1, %ymm2
-; AVX512F-NEXT:    vpandn %ymm2, %ymm8, %ymm2
-; AVX512F-NEXT:    vpsllw $2, %ymm1, %ymm4
-; AVX512F-NEXT:    vpand %ymm8, %ymm4, %ymm4
-; AVX512F-NEXT:    vpor %ymm2, %ymm4, %ymm2
-; AVX512F-NEXT:    vpaddb %ymm3, %ymm3, %ymm3
-; AVX512F-NEXT:    vpblendvb %ymm3, %ymm2, %ymm1, %ymm1
-; AVX512F-NEXT:    vpsrlw $7, %ymm1, %ymm2
-; AVX512F-NEXT:    vpand %ymm9, %ymm2, %ymm2
-; AVX512F-NEXT:    vpaddb %ymm1, %ymm1, %ymm4
-; AVX512F-NEXT:    vpor %ymm2, %ymm4, %ymm2
-; AVX512F-NEXT:    vpaddb %ymm3, %ymm3, %ymm3
-; AVX512F-NEXT:    vpblendvb %ymm3, %ymm2, %ymm1, %ymm1
+; AVX512F-NEXT:    vpor %ymm3, %ymm4, %ymm3
+; AVX512F-NEXT:    vpsubb %ymm1, %ymm6, %ymm1
+; AVX512F-NEXT:    vpand %ymm7, %ymm1, %ymm1
+; AVX512F-NEXT:    vpsllw $5, %ymm1, %ymm1
+; AVX512F-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
+; AVX512F-NEXT:    vpsrlw $6, %ymm0, %ymm3
+; AVX512F-NEXT:    vpandn %ymm3, %ymm8, %ymm3
+; AVX512F-NEXT:    vpsllw $2, %ymm0, %ymm4
+; AVX512F-NEXT:    vpand %ymm4, %ymm8, %ymm4
+; AVX512F-NEXT:    vpor %ymm3, %ymm4, %ymm3
+; AVX512F-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
+; AVX512F-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
+; AVX512F-NEXT:    vpsrlw $7, %ymm0, %ymm3
+; AVX512F-NEXT:    vpand %ymm3, %ymm9, %ymm3
+; AVX512F-NEXT:    vpaddb %ymm0, %ymm0, %ymm4
+; AVX512F-NEXT:    vpor %ymm3, %ymm4, %ymm3
+; AVX512F-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
+; AVX512F-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: var_funnnel_v64i8:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm4
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm5 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX512VL-NEXT:    vpandn %ymm4, %ymm5, %ymm4
-; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm6
-; AVX512VL-NEXT:    vpand %ymm5, %ymm6, %ymm6
-; AVX512VL-NEXT:    vpor %ymm4, %ymm6, %ymm4
-; AVX512VL-NEXT:    vpxor %xmm6, %xmm6, %xmm6
-; AVX512VL-NEXT:    vpsubb %ymm2, %ymm6, %ymm2
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
+; AVX512VL-NEXT:    vpsrlw $4, %ymm3, %ymm4
+; AVX512VL-NEXT:    vpsllw $4, %ymm3, %ymm5
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm6 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
+; AVX512VL-NEXT:    vpternlogq $226, %ymm4, %ymm6, %ymm5
+; AVX512VL-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; AVX512VL-NEXT:    vpsubb %ymm2, %ymm4, %ymm2
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm7 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
 ; AVX512VL-NEXT:    vpand %ymm7, %ymm2, %ymm2
 ; AVX512VL-NEXT:    vpsllw $5, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm4, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpsrlw $6, %ymm0, %ymm4
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm8 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
-; AVX512VL-NEXT:    vpandn %ymm4, %ymm8, %ymm4
-; AVX512VL-NEXT:    vpsllw $2, %ymm0, %ymm9
-; AVX512VL-NEXT:    vpand %ymm8, %ymm9, %ymm9
-; AVX512VL-NEXT:    vpor %ymm4, %ymm9, %ymm4
+; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm5, %ymm3, %ymm3
+; AVX512VL-NEXT:    vpsrlw $6, %ymm3, %ymm5
+; AVX512VL-NEXT:    vpsllw $2, %ymm3, %ymm8
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm9 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
+; AVX512VL-NEXT:    vpternlogq $226, %ymm5, %ymm9, %ymm8
 ; AVX512VL-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm4, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpsrlw $7, %ymm0, %ymm4
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm9 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; AVX512VL-NEXT:    vpand %ymm9, %ymm4, %ymm4
-; AVX512VL-NEXT:    vpaddb %ymm0, %ymm0, %ymm10
-; AVX512VL-NEXT:    vpor %ymm4, %ymm10, %ymm4
+; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm8, %ymm3, %ymm3
+; AVX512VL-NEXT:    vpsrlw $7, %ymm3, %ymm5
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm8 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; AVX512VL-NEXT:    vpand %ymm5, %ymm8, %ymm5
+; AVX512VL-NEXT:    vpaddb %ymm3, %ymm3, %ymm10
+; AVX512VL-NEXT:    vpor %ymm5, %ymm10, %ymm5
 ; AVX512VL-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm4, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpsrlw $4, %ymm1, %ymm2
-; AVX512VL-NEXT:    vpandn %ymm2, %ymm5, %ymm2
-; AVX512VL-NEXT:    vpsllw $4, %ymm1, %ymm4
-; AVX512VL-NEXT:    vpand %ymm5, %ymm4, %ymm4
-; AVX512VL-NEXT:    vpor %ymm2, %ymm4, %ymm2
-; AVX512VL-NEXT:    vpsubb %ymm3, %ymm6, %ymm3
-; AVX512VL-NEXT:    vpand %ymm7, %ymm3, %ymm3
-; AVX512VL-NEXT:    vpsllw $5, %ymm3, %ymm3
-; AVX512VL-NEXT:    vpblendvb %ymm3, %ymm2, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpsrlw $6, %ymm1, %ymm2
-; AVX512VL-NEXT:    vpandn %ymm2, %ymm8, %ymm2
-; AVX512VL-NEXT:    vpsllw $2, %ymm1, %ymm4
-; AVX512VL-NEXT:    vpand %ymm8, %ymm4, %ymm4
-; AVX512VL-NEXT:    vpor %ymm2, %ymm4, %ymm2
-; AVX512VL-NEXT:    vpaddb %ymm3, %ymm3, %ymm3
-; AVX512VL-NEXT:    vpblendvb %ymm3, %ymm2, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpsrlw $7, %ymm1, %ymm2
-; AVX512VL-NEXT:    vpand %ymm9, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpaddb %ymm1, %ymm1, %ymm4
-; AVX512VL-NEXT:    vpor %ymm2, %ymm4, %ymm2
-; AVX512VL-NEXT:    vpaddb %ymm3, %ymm3, %ymm3
-; AVX512VL-NEXT:    vpblendvb %ymm3, %ymm2, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm5, %ymm3, %ymm2
+; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm3
+; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm5
+; AVX512VL-NEXT:    vpternlogq $226, %ymm3, %ymm6, %ymm5
+; AVX512VL-NEXT:    vpsubb %ymm1, %ymm4, %ymm1
+; AVX512VL-NEXT:    vpand %ymm7, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpsllw $5, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm5, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpsrlw $6, %ymm0, %ymm3
+; AVX512VL-NEXT:    vpsllw $2, %ymm0, %ymm4
+; AVX512VL-NEXT:    vpternlogq $226, %ymm3, %ymm9, %ymm4
+; AVX512VL-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm4, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpsrlw $7, %ymm0, %ymm3
+; AVX512VL-NEXT:    vpand %ymm3, %ymm8, %ymm3
+; AVX512VL-NEXT:    vpaddb %ymm0, %ymm0, %ymm4
+; AVX512VL-NEXT:    vpor %ymm3, %ymm4, %ymm3
+; AVX512VL-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: var_funnnel_v64i8:
@@ -328,38 +332,42 @@ define <16 x i32> @splatvar_funnnel_v16i32(<16 x i32> %x, <16 x i32> %amt) nounw
 define <32 x i16> @splatvar_funnnel_v32i16(<32 x i16> %x, <32 x i16> %amt) nounwind {
 ; AVX512F-LABEL: splatvar_funnnel_v32i16:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpbroadcastw %xmm2, %xmm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512F-NEXT:    vpbroadcastw %xmm1, %xmm1
 ; AVX512F-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX512F-NEXT:    vpsubw %xmm2, %xmm3, %xmm2
-; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
-; AVX512F-NEXT:    vpmovzxwq {{.*#+}} xmm3 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
-; AVX512F-NEXT:    vpsllw %xmm3, %ymm0, %ymm4
+; AVX512F-NEXT:    vpsubw %xmm1, %xmm3, %xmm1
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512F-NEXT:    vpmovzxwq {{.*#+}} xmm3 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512F-NEXT:    vpsllw %xmm3, %ymm2, %ymm4
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm5 = [16,16,16,16,16,16,16,16]
-; AVX512F-NEXT:    vpsubw %xmm2, %xmm5, %xmm2
-; AVX512F-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
-; AVX512F-NEXT:    vpsrlw %xmm2, %ymm0, %ymm0
-; AVX512F-NEXT:    vpor %ymm0, %ymm4, %ymm0
-; AVX512F-NEXT:    vpsllw %xmm3, %ymm1, %ymm3
-; AVX512F-NEXT:    vpsrlw %xmm2, %ymm1, %ymm1
-; AVX512F-NEXT:    vpor %ymm1, %ymm3, %ymm1
+; AVX512F-NEXT:    vpsubw %xmm1, %xmm5, %xmm1
+; AVX512F-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512F-NEXT:    vpsrlw %xmm1, %ymm2, %ymm2
+; AVX512F-NEXT:    vpor %ymm2, %ymm4, %ymm2
+; AVX512F-NEXT:    vpsllw %xmm3, %ymm0, %ymm3
+; AVX512F-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vpor %ymm0, %ymm3, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatvar_funnnel_v32i16:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpbroadcastw %xmm2, %xmm2
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512VL-NEXT:    vpbroadcastw %xmm1, %xmm1
 ; AVX512VL-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX512VL-NEXT:    vpsubw %xmm2, %xmm3, %xmm2
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
-; AVX512VL-NEXT:    vpmovzxwq {{.*#+}} xmm3 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
-; AVX512VL-NEXT:    vpsllw %xmm3, %ymm0, %ymm4
+; AVX512VL-NEXT:    vpsubw %xmm1, %xmm3, %xmm1
+; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512VL-NEXT:    vpmovzxwq {{.*#+}} xmm3 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512VL-NEXT:    vpsllw %xmm3, %ymm2, %ymm4
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm5 = [16,16,16,16,16,16,16,16]
-; AVX512VL-NEXT:    vpsubw %xmm2, %xmm5, %xmm2
-; AVX512VL-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
-; AVX512VL-NEXT:    vpsrlw %xmm2, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpor %ymm0, %ymm4, %ymm0
-; AVX512VL-NEXT:    vpsllw %xmm3, %ymm1, %ymm3
-; AVX512VL-NEXT:    vpsrlw %xmm2, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpor %ymm1, %ymm3, %ymm1
+; AVX512VL-NEXT:    vpsubw %xmm1, %xmm5, %xmm1
+; AVX512VL-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512VL-NEXT:    vpsrlw %xmm1, %ymm2, %ymm2
+; AVX512VL-NEXT:    vpor %ymm2, %ymm4, %ymm2
+; AVX512VL-NEXT:    vpsllw %xmm3, %ymm0, %ymm3
+; AVX512VL-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpor %ymm0, %ymm3, %ymm0
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatvar_funnnel_v32i16:
@@ -399,58 +407,62 @@ define <32 x i16> @splatvar_funnnel_v32i16(<32 x i16> %x, <32 x i16> %amt) nounw
 define <64 x i8> @splatvar_funnnel_v64i8(<64 x i8> %x, <64 x i8> %amt) nounwind {
 ; AVX512F-LABEL: splatvar_funnnel_v64i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpbroadcastb %xmm2, %xmm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512F-NEXT:    vpbroadcastb %xmm1, %xmm1
 ; AVX512F-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX512F-NEXT:    vpsubb %xmm2, %xmm3, %xmm2
-; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
-; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512F-NEXT:    vpsllw %xmm3, %ymm0, %ymm4
+; AVX512F-NEXT:    vpsubb %xmm1, %xmm3, %xmm1
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512F-NEXT:    vpsllw %xmm3, %ymm2, %ymm4
 ; AVX512F-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
 ; AVX512F-NEXT:    vpsllw %xmm3, %xmm5, %xmm6
 ; AVX512F-NEXT:    vpbroadcastb %xmm6, %ymm6
 ; AVX512F-NEXT:    vpand %ymm6, %ymm4, %ymm4
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm7 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; AVX512F-NEXT:    vpsubb %xmm2, %xmm7, %xmm2
-; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512F-NEXT:    vpsrlw %xmm2, %ymm0, %ymm0
-; AVX512F-NEXT:    vpsrlw %xmm2, %xmm5, %xmm5
+; AVX512F-NEXT:    vpsubb %xmm1, %xmm7, %xmm1
+; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512F-NEXT:    vpsrlw %xmm1, %ymm2, %ymm2
+; AVX512F-NEXT:    vpsrlw %xmm1, %xmm5, %xmm5
 ; AVX512F-NEXT:    vpsrlw $8, %xmm5, %xmm5
 ; AVX512F-NEXT:    vpbroadcastb %xmm5, %ymm5
-; AVX512F-NEXT:    vpand %ymm5, %ymm0, %ymm0
-; AVX512F-NEXT:    vpor %ymm0, %ymm4, %ymm0
-; AVX512F-NEXT:    vpsllw %xmm3, %ymm1, %ymm3
+; AVX512F-NEXT:    vpand %ymm5, %ymm2, %ymm2
+; AVX512F-NEXT:    vpor %ymm2, %ymm4, %ymm2
+; AVX512F-NEXT:    vpsllw %xmm3, %ymm0, %ymm3
 ; AVX512F-NEXT:    vpand %ymm6, %ymm3, %ymm3
-; AVX512F-NEXT:    vpsrlw %xmm2, %ymm1, %ymm1
-; AVX512F-NEXT:    vpand %ymm5, %ymm1, %ymm1
-; AVX512F-NEXT:    vpor %ymm1, %ymm3, %ymm1
+; AVX512F-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vpand %ymm5, %ymm0, %ymm0
+; AVX512F-NEXT:    vpor %ymm0, %ymm3, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatvar_funnnel_v64i8:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpbroadcastb %xmm2, %xmm2
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512VL-NEXT:    vpbroadcastb %xmm1, %xmm1
 ; AVX512VL-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX512VL-NEXT:    vpsubb %xmm2, %xmm3, %xmm2
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
-; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512VL-NEXT:    vpsllw %xmm3, %ymm0, %ymm4
+; AVX512VL-NEXT:    vpsubb %xmm1, %xmm3, %xmm1
+; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VL-NEXT:    vpsllw %xmm3, %ymm2, %ymm4
 ; AVX512VL-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
 ; AVX512VL-NEXT:    vpsllw %xmm3, %xmm5, %xmm6
 ; AVX512VL-NEXT:    vpbroadcastb %xmm6, %ymm6
 ; AVX512VL-NEXT:    vpand %ymm6, %ymm4, %ymm4
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm7 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; AVX512VL-NEXT:    vpsubb %xmm2, %xmm7, %xmm2
-; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512VL-NEXT:    vpsrlw %xmm2, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpsrlw %xmm2, %xmm5, %xmm5
+; AVX512VL-NEXT:    vpsubb %xmm1, %xmm7, %xmm1
+; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VL-NEXT:    vpsrlw %xmm1, %ymm2, %ymm2
+; AVX512VL-NEXT:    vpsrlw %xmm1, %xmm5, %xmm5
 ; AVX512VL-NEXT:    vpsrlw $8, %xmm5, %xmm5
 ; AVX512VL-NEXT:    vpbroadcastb %xmm5, %ymm5
-; AVX512VL-NEXT:    vpand %ymm5, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpor %ymm0, %ymm4, %ymm0
-; AVX512VL-NEXT:    vpsllw %xmm3, %ymm1, %ymm3
+; AVX512VL-NEXT:    vpand %ymm5, %ymm2, %ymm2
+; AVX512VL-NEXT:    vpor %ymm2, %ymm4, %ymm2
+; AVX512VL-NEXT:    vpsllw %xmm3, %ymm0, %ymm3
 ; AVX512VL-NEXT:    vpand %ymm6, %ymm3, %ymm3
-; AVX512VL-NEXT:    vpsrlw %xmm2, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpand %ymm5, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpor %ymm1, %ymm3, %ymm1
+; AVX512VL-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpand %ymm5, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpor %ymm0, %ymm3, %ymm0
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatvar_funnnel_v64i8:
@@ -528,34 +540,38 @@ define <16 x i32> @constant_funnnel_v16i32(<16 x i32> %x) nounwind {
 define <32 x i16> @constant_funnnel_v32i16(<32 x i16> %x) nounwind {
 ; AVX512F-LABEL: constant_funnnel_v32i16:
 ; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm2 = <u,32768,16384,8192,4096,2048,1024,512,256,128,64,32,16,8,4,2>
-; AVX512F-NEXT:    vpmulhuw %ymm2, %ymm0, %ymm3
-; AVX512F-NEXT:    vpblendw {{.*#+}} ymm4 = ymm0[0],ymm3[1,2,3,4,5,6,7],ymm0[8],ymm3[9,10,11,12,13,14,15]
+; AVX512F-NEXT:    vpmulhuw %ymm2, %ymm1, %ymm3
+; AVX512F-NEXT:    vpblendw {{.*#+}} ymm4 = ymm1[0],ymm3[1,2,3,4,5,6,7],ymm1[8],ymm3[9,10,11,12,13,14,15]
 ; AVX512F-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm4 = [1,32768,16384,8192,4096,2048,1024,512,256,128,64,32,16,8,4,2]
-; AVX512F-NEXT:    vpmullw %ymm4, %ymm0, %ymm0
-; AVX512F-NEXT:    vpor %ymm3, %ymm0, %ymm0
-; AVX512F-NEXT:    vpmulhuw %ymm2, %ymm1, %ymm2
-; AVX512F-NEXT:    vpblendw {{.*#+}} ymm3 = ymm1[0],ymm2[1,2,3,4,5,6,7],ymm1[8],ymm2[9,10,11,12,13,14,15]
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
 ; AVX512F-NEXT:    vpmullw %ymm4, %ymm1, %ymm1
-; AVX512F-NEXT:    vpor %ymm2, %ymm1, %ymm1
+; AVX512F-NEXT:    vpor %ymm3, %ymm1, %ymm1
+; AVX512F-NEXT:    vpmulhuw %ymm2, %ymm0, %ymm2
+; AVX512F-NEXT:    vpblendw {{.*#+}} ymm3 = ymm0[0],ymm2[1,2,3,4,5,6,7],ymm0[8],ymm2[9,10,11,12,13,14,15]
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
+; AVX512F-NEXT:    vpmullw %ymm4, %ymm0, %ymm0
+; AVX512F-NEXT:    vpor %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: constant_funnnel_v32i16:
 ; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm2 = <u,32768,16384,8192,4096,2048,1024,512,256,128,64,32,16,8,4,2>
-; AVX512VL-NEXT:    vpmulhuw %ymm2, %ymm0, %ymm3
-; AVX512VL-NEXT:    vpblendw {{.*#+}} ymm4 = ymm0[0],ymm3[1,2,3,4,5,6,7],ymm0[8],ymm3[9,10,11,12,13,14,15]
+; AVX512VL-NEXT:    vpmulhuw %ymm2, %ymm1, %ymm3
+; AVX512VL-NEXT:    vpblendw {{.*#+}} ymm4 = ymm1[0],ymm3[1,2,3,4,5,6,7],ymm1[8],ymm3[9,10,11,12,13,14,15]
 ; AVX512VL-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm4 = [1,32768,16384,8192,4096,2048,1024,512,256,128,64,32,16,8,4,2]
-; AVX512VL-NEXT:    vpmullw %ymm4, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpor %ymm3, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpmulhuw %ymm2, %ymm1, %ymm2
-; AVX512VL-NEXT:    vpblendw {{.*#+}} ymm3 = ymm1[0],ymm2[1,2,3,4,5,6,7],ymm1[8],ymm2[9,10,11,12,13,14,15]
-; AVX512VL-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
 ; AVX512VL-NEXT:    vpmullw %ymm4, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpor %ymm2, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpor %ymm3, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpmulhuw %ymm2, %ymm0, %ymm2
+; AVX512VL-NEXT:    vpblendw {{.*#+}} ymm3 = ymm0[0],ymm2[1,2,3,4,5,6,7],ymm0[8],ymm2[9,10,11,12,13,14,15]
+; AVX512VL-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
+; AVX512VL-NEXT:    vpmullw %ymm4, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpor %ymm2, %ymm0, %ymm0
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: constant_funnnel_v32i16:
@@ -578,12 +594,13 @@ define <32 x i16> @constant_funnnel_v32i16(<32 x i16> %x) nounwind {
 define <64 x i8> @constant_funnnel_v64i8(<64 x i8> %x) nounwind {
 ; AVX512F-LABEL: constant_funnnel_v64i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
+; AVX512F-NEXT:    vpsllw $4, %ymm1, %ymm2
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
 ; AVX512F-NEXT:    vpand %ymm3, %ymm2, %ymm2
 ; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm4 = [57344,41152,24704,8256,8192,24640,41088,57536,57344,41152,24704,8256,8192,24640,41088,57536]
 ; AVX512F-NEXT:    # ymm4 = mem[0,1,0,1]
-; AVX512F-NEXT:    vpblendvb %ymm4, %ymm2, %ymm0, %ymm2
+; AVX512F-NEXT:    vpblendvb %ymm4, %ymm2, %ymm1, %ymm2
 ; AVX512F-NEXT:    vpsllw $2, %ymm2, %ymm5
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm6 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
 ; AVX512F-NEXT:    vpand %ymm6, %ymm5, %ymm5
@@ -593,44 +610,46 @@ define <64 x i8> @constant_funnnel_v64i8(<64 x i8> %x) nounwind {
 ; AVX512F-NEXT:    vpaddb %ymm7, %ymm7, %ymm8
 ; AVX512F-NEXT:    vpblendvb %ymm8, %ymm5, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpxor %xmm5, %xmm5, %xmm5
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm9 = ymm0[8],ymm5[8],ymm0[9],ymm5[9],ymm0[10],ymm5[10],ymm0[11],ymm5[11],ymm0[12],ymm5[12],ymm0[13],ymm5[13],ymm0[14],ymm5[14],ymm0[15],ymm5[15],ymm0[24],ymm5[24],ymm0[25],ymm5[25],ymm0[26],ymm5[26],ymm0[27],ymm5[27],ymm0[28],ymm5[28],ymm0[29],ymm5[29],ymm0[30],ymm5[30],ymm0[31],ymm5[31]
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm9 = ymm1[8],ymm5[8],ymm1[9],ymm5[9],ymm1[10],ymm5[10],ymm1[11],ymm5[11],ymm1[12],ymm5[12],ymm1[13],ymm5[13],ymm1[14],ymm5[14],ymm1[15],ymm5[15],ymm1[24],ymm5[24],ymm1[25],ymm5[25],ymm1[26],ymm5[26],ymm1[27],ymm5[27],ymm1[28],ymm5[28],ymm1[29],ymm5[29],ymm1[30],ymm5[30],ymm1[31],ymm5[31]
 ; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm10 = [256,2,4,8,16,32,64,128,256,2,4,8,16,32,64,128]
 ; AVX512F-NEXT:    # ymm10 = mem[0,1,0,1]
 ; AVX512F-NEXT:    vpmullw %ymm10, %ymm9, %ymm9
 ; AVX512F-NEXT:    vpsrlw $8, %ymm9, %ymm9
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0],ymm5[0],ymm0[1],ymm5[1],ymm0[2],ymm5[2],ymm0[3],ymm5[3],ymm0[4],ymm5[4],ymm0[5],ymm5[5],ymm0[6],ymm5[6],ymm0[7],ymm5[7],ymm0[16],ymm5[16],ymm0[17],ymm5[17],ymm0[18],ymm5[18],ymm0[19],ymm5[19],ymm0[20],ymm5[20],ymm0[21],ymm5[21],ymm0[22],ymm5[22],ymm0[23],ymm5[23]
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm5[0],ymm1[1],ymm5[1],ymm1[2],ymm5[2],ymm1[3],ymm5[3],ymm1[4],ymm5[4],ymm1[5],ymm5[5],ymm1[6],ymm5[6],ymm1[7],ymm5[7],ymm1[16],ymm5[16],ymm1[17],ymm5[17],ymm1[18],ymm5[18],ymm1[19],ymm5[19],ymm1[20],ymm5[20],ymm1[21],ymm5[21],ymm1[22],ymm5[22],ymm1[23],ymm5[23]
 ; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm11 = [256,128,64,32,16,8,4,2,256,128,64,32,16,8,4,2]
 ; AVX512F-NEXT:    # ymm11 = mem[0,1,0,1]
-; AVX512F-NEXT:    vpmullw %ymm11, %ymm0, %ymm0
-; AVX512F-NEXT:    vpsrlw $8, %ymm0, %ymm0
-; AVX512F-NEXT:    vpackuswb %ymm9, %ymm0, %ymm0
-; AVX512F-NEXT:    vpor %ymm0, %ymm2, %ymm0
-; AVX512F-NEXT:    vpsllw $4, %ymm1, %ymm2
+; AVX512F-NEXT:    vpmullw %ymm1, %ymm11, %ymm1
+; AVX512F-NEXT:    vpsrlw $8, %ymm1, %ymm1
+; AVX512F-NEXT:    vpackuswb %ymm9, %ymm1, %ymm1
+; AVX512F-NEXT:    vpor %ymm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm2
 ; AVX512F-NEXT:    vpand %ymm3, %ymm2, %ymm2
-; AVX512F-NEXT:    vpblendvb %ymm4, %ymm2, %ymm1, %ymm2
+; AVX512F-NEXT:    vpblendvb %ymm4, %ymm2, %ymm0, %ymm2
 ; AVX512F-NEXT:    vpsllw $2, %ymm2, %ymm3
 ; AVX512F-NEXT:    vpand %ymm6, %ymm3, %ymm3
 ; AVX512F-NEXT:    vpblendvb %ymm7, %ymm3, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpaddb %ymm2, %ymm2, %ymm3
 ; AVX512F-NEXT:    vpblendvb %ymm8, %ymm3, %ymm2, %ymm2
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm1[8],ymm5[8],ymm1[9],ymm5[9],ymm1[10],ymm5[10],ymm1[11],ymm5[11],ymm1[12],ymm5[12],ymm1[13],ymm5[13],ymm1[14],ymm5[14],ymm1[15],ymm5[15],ymm1[24],ymm5[24],ymm1[25],ymm5[25],ymm1[26],ymm5[26],ymm1[27],ymm5[27],ymm1[28],ymm5[28],ymm1[29],ymm5[29],ymm1[30],ymm5[30],ymm1[31],ymm5[31]
-; AVX512F-NEXT:    vpmullw %ymm10, %ymm3, %ymm3
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm0[8],ymm5[8],ymm0[9],ymm5[9],ymm0[10],ymm5[10],ymm0[11],ymm5[11],ymm0[12],ymm5[12],ymm0[13],ymm5[13],ymm0[14],ymm5[14],ymm0[15],ymm5[15],ymm0[24],ymm5[24],ymm0[25],ymm5[25],ymm0[26],ymm5[26],ymm0[27],ymm5[27],ymm0[28],ymm5[28],ymm0[29],ymm5[29],ymm0[30],ymm5[30],ymm0[31],ymm5[31]
+; AVX512F-NEXT:    vpmullw %ymm3, %ymm10, %ymm3
 ; AVX512F-NEXT:    vpsrlw $8, %ymm3, %ymm3
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm5[0],ymm1[1],ymm5[1],ymm1[2],ymm5[2],ymm1[3],ymm5[3],ymm1[4],ymm5[4],ymm1[5],ymm5[5],ymm1[6],ymm5[6],ymm1[7],ymm5[7],ymm1[16],ymm5[16],ymm1[17],ymm5[17],ymm1[18],ymm5[18],ymm1[19],ymm5[19],ymm1[20],ymm5[20],ymm1[21],ymm5[21],ymm1[22],ymm5[22],ymm1[23],ymm5[23]
-; AVX512F-NEXT:    vpmullw %ymm11, %ymm1, %ymm1
-; AVX512F-NEXT:    vpsrlw $8, %ymm1, %ymm1
-; AVX512F-NEXT:    vpackuswb %ymm3, %ymm1, %ymm1
-; AVX512F-NEXT:    vpor %ymm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0],ymm5[0],ymm0[1],ymm5[1],ymm0[2],ymm5[2],ymm0[3],ymm5[3],ymm0[4],ymm5[4],ymm0[5],ymm5[5],ymm0[6],ymm5[6],ymm0[7],ymm5[7],ymm0[16],ymm5[16],ymm0[17],ymm5[17],ymm0[18],ymm5[18],ymm0[19],ymm5[19],ymm0[20],ymm5[20],ymm0[21],ymm5[21],ymm0[22],ymm5[22],ymm0[23],ymm5[23]
+; AVX512F-NEXT:    vpmullw %ymm0, %ymm11, %ymm0
+; AVX512F-NEXT:    vpsrlw $8, %ymm0, %ymm0
+; AVX512F-NEXT:    vpackuswb %ymm3, %ymm0, %ymm0
+; AVX512F-NEXT:    vpor %ymm0, %ymm2, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: constant_funnnel_v64i8:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm2
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
+; AVX512VL-NEXT:    vpsllw $4, %ymm1, %ymm2
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
 ; AVX512VL-NEXT:    vpand %ymm3, %ymm2, %ymm2
 ; AVX512VL-NEXT:    vbroadcasti128 {{.*#+}} ymm4 = [57344,41152,24704,8256,8192,24640,41088,57536,57344,41152,24704,8256,8192,24640,41088,57536]
 ; AVX512VL-NEXT:    # ymm4 = mem[0,1,0,1]
-; AVX512VL-NEXT:    vpblendvb %ymm4, %ymm2, %ymm0, %ymm2
+; AVX512VL-NEXT:    vpblendvb %ymm4, %ymm2, %ymm1, %ymm2
 ; AVX512VL-NEXT:    vpsllw $2, %ymm2, %ymm5
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm6 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
 ; AVX512VL-NEXT:    vpand %ymm6, %ymm5, %ymm5
@@ -639,38 +658,39 @@ define <64 x i8> @constant_funnnel_v64i8(<64 x i8> %x) nounwind {
 ; AVX512VL-NEXT:    vpaddb %ymm2, %ymm2, %ymm5
 ; AVX512VL-NEXT:    vpaddb %ymm7, %ymm7, %ymm8
 ; AVX512VL-NEXT:    vpblendvb %ymm8, %ymm5, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX512VL-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
 ; AVX512VL-NEXT:    vpsrlw $8, %ymm5, %ymm5
 ; AVX512VL-NEXT:    vbroadcasti128 {{.*#+}} ymm9 = [256,2,4,8,16,32,64,128,256,2,4,8,16,32,64,128]
 ; AVX512VL-NEXT:    # ymm9 = mem[0,1,0,1]
-; AVX512VL-NEXT:    vpmullw %ymm9, %ymm5, %ymm5
+; AVX512VL-NEXT:    vpmullw %ymm5, %ymm9, %ymm5
 ; AVX512VL-NEXT:    vpsrlw $8, %ymm5, %ymm5
-; AVX512VL-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512VL-NEXT:    vpsrlw $8, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512VL-NEXT:    vpsrlw $8, %ymm1, %ymm1
 ; AVX512VL-NEXT:    vbroadcasti128 {{.*#+}} ymm10 = [256,128,64,32,16,8,4,2,256,128,64,32,16,8,4,2]
 ; AVX512VL-NEXT:    # ymm10 = mem[0,1,0,1]
-; AVX512VL-NEXT:    vpmullw %ymm10, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpsrlw $8, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpackuswb %ymm5, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpor %ymm0, %ymm2, %ymm0
-; AVX512VL-NEXT:    vpsllw $4, %ymm1, %ymm2
+; AVX512VL-NEXT:    vpmullw %ymm1, %ymm10, %ymm1
+; AVX512VL-NEXT:    vpsrlw $8, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpackuswb %ymm5, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpor %ymm1, %ymm2, %ymm1
+; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm2
 ; AVX512VL-NEXT:    vpand %ymm3, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpblendvb %ymm4, %ymm2, %ymm1, %ymm2
+; AVX512VL-NEXT:    vpblendvb %ymm4, %ymm2, %ymm0, %ymm2
 ; AVX512VL-NEXT:    vpsllw $2, %ymm2, %ymm3
 ; AVX512VL-NEXT:    vpand %ymm6, %ymm3, %ymm3
 ; AVX512VL-NEXT:    vpblendvb %ymm7, %ymm3, %ymm2, %ymm2
 ; AVX512VL-NEXT:    vpaddb %ymm2, %ymm2, %ymm3
 ; AVX512VL-NEXT:    vpblendvb %ymm8, %ymm3, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX512VL-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
 ; AVX512VL-NEXT:    vpsrlw $8, %ymm3, %ymm3
-; AVX512VL-NEXT:    vpmullw %ymm9, %ymm3, %ymm3
+; AVX512VL-NEXT:    vpmullw %ymm3, %ymm9, %ymm3
 ; AVX512VL-NEXT:    vpsrlw $8, %ymm3, %ymm3
-; AVX512VL-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512VL-NEXT:    vpsrlw $8, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpmullw %ymm10, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpsrlw $8, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpackuswb %ymm3, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpor %ymm1, %ymm2, %ymm1
+; AVX512VL-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512VL-NEXT:    vpsrlw $8, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpmullw %ymm0, %ymm10, %ymm0
+; AVX512VL-NEXT:    vpsrlw $8, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpackuswb %ymm3, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpor %ymm0, %ymm2, %ymm0
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: constant_funnnel_v64i8:
@@ -757,22 +777,26 @@ define <16 x i32> @splatconstant_funnnel_v16i32(<16 x i32> %x) nounwind {
 define <32 x i16> @splatconstant_funnnel_v32i16(<32 x i16> %x) nounwind {
 ; AVX512F-LABEL: splatconstant_funnnel_v32i16:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpsrlw $7, %ymm0, %ymm2
-; AVX512F-NEXT:    vpsllw $9, %ymm0, %ymm0
-; AVX512F-NEXT:    vpor %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512F-NEXT:    vpsrlw $7, %ymm1, %ymm2
 ; AVX512F-NEXT:    vpsllw $9, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpor %ymm2, %ymm1, %ymm1
+; AVX512F-NEXT:    vpsrlw $7, %ymm0, %ymm2
+; AVX512F-NEXT:    vpsllw $9, %ymm0, %ymm0
+; AVX512F-NEXT:    vpor %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatconstant_funnnel_v32i16:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsrlw $7, %ymm0, %ymm2
-; AVX512VL-NEXT:    vpsllw $9, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpor %ymm2, %ymm0, %ymm0
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512VL-NEXT:    vpsrlw $7, %ymm1, %ymm2
 ; AVX512VL-NEXT:    vpsllw $9, %ymm1, %ymm1
 ; AVX512VL-NEXT:    vpor %ymm2, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpsrlw $7, %ymm0, %ymm2
+; AVX512VL-NEXT:    vpsllw $9, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpor %ymm2, %ymm0, %ymm0
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatconstant_funnnel_v32i16:
@@ -795,50 +819,46 @@ define <32 x i16> @splatconstant_funnnel_v32i16(<32 x i16> %x) nounwind {
 define <64 x i8> @splatconstant_funnnel_v64i8(<64 x i8> %x) nounwind {
 ; AVX512F-LABEL: splatconstant_funnnel_v64i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpsrlw $4, %ymm0, %ymm2
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX512F-NEXT:    vpandn %ymm2, %ymm3, %ymm2
-; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm0
-; AVX512F-NEXT:    vpand %ymm3, %ymm0, %ymm0
-; AVX512F-NEXT:    vpor %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512F-NEXT:    vpsrlw $4, %ymm1, %ymm2
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
 ; AVX512F-NEXT:    vpandn %ymm2, %ymm3, %ymm2
 ; AVX512F-NEXT:    vpsllw $4, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpand %ymm3, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpor %ymm2, %ymm1, %ymm1
+; AVX512F-NEXT:    vpsrlw $4, %ymm0, %ymm2
+; AVX512F-NEXT:    vpandn %ymm2, %ymm3, %ymm2
+; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm0
+; AVX512F-NEXT:    vpand %ymm3, %ymm0, %ymm0
+; AVX512F-NEXT:    vpor %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatconstant_funnnel_v64i8:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm2
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX512VL-NEXT:    vpandn %ymm2, %ymm3, %ymm2
-; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpand %ymm3, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpor %ymm2, %ymm0, %ymm0
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512VL-NEXT:    vpsrlw $4, %ymm1, %ymm2
-; AVX512VL-NEXT:    vpandn %ymm2, %ymm3, %ymm2
 ; AVX512VL-NEXT:    vpsllw $4, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpand %ymm3, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpor %ymm2, %ymm1, %ymm1
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
+; AVX512VL-NEXT:    vpternlogq $226, %ymm2, %ymm3, %ymm1
+; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm2
+; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpternlogq $226, %ymm2, %ymm3, %ymm0
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatconstant_funnnel_v64i8:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpsrlw $4, %zmm0, %zmm1
-; AVX512BW-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
-; AVX512BW-NEXT:    vpsllw $4, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
-; AVX512BW-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsllw $4, %zmm0, %zmm1
+; AVX512BW-NEXT:    vpsrlw $4, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpternlogq $216, {{.*}}(%rip), %zmm1, %zmm0
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: splatconstant_funnnel_v64i8:
 ; AVX512VLBW:       # %bb.0:
-; AVX512VLBW-NEXT:    vpsrlw $4, %zmm0, %zmm1
-; AVX512VLBW-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
-; AVX512VLBW-NEXT:    vpsllw $4, %zmm0, %zmm0
-; AVX512VLBW-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
-; AVX512VLBW-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512VLBW-NEXT:    vpsllw $4, %zmm0, %zmm1
+; AVX512VLBW-NEXT:    vpsrlw $4, %zmm0, %zmm0
+; AVX512VLBW-NEXT:    vpternlogq $216, {{.*}}(%rip), %zmm1, %zmm0
 ; AVX512VLBW-NEXT:    retq
   %res = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %x, <64 x i8> %x, <64 x i8> <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>)
   ret <64 x i8> %res
diff --git a/llvm/test/CodeGen/X86/vector-gep.ll b/llvm/test/CodeGen/X86/vector-gep.ll
index 693380a48..f8c5a4fc2 100644
--- a/llvm/test/CodeGen/X86/vector-gep.ll
+++ b/llvm/test/CodeGen/X86/vector-gep.ll
@@ -74,8 +74,7 @@ define <4 x i16*> @AGEP4(<4 x i16*> %param, <4 x i32> %off) nounwind {
 define <4 x i8*> @AGEP5(<4 x i8*> %param, <4 x i8> %off) nounwind {
 ; CHECK-LABEL: AGEP5:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpslld $24, %xmm1, %xmm1
-; CHECK-NEXT:    vpsrad $24, %xmm1, %xmm1
+; CHECK-NEXT:    vpmovsxbd %xmm1, %xmm1
 ; CHECK-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; CHECK-NEXT:    retl
   %A = getelementptr i8, <4 x i8*> %param, <4 x i8> %off
diff --git a/llvm/test/CodeGen/X86/vector-half-conversions.ll b/llvm/test/CodeGen/X86/vector-half-conversions.ll
index f08e0ccce..4895c0697 100644
--- a/llvm/test/CodeGen/X86/vector-half-conversions.ll
+++ b/llvm/test/CodeGen/X86/vector-half-conversions.ll
@@ -24,7 +24,6 @@ define float @cvt_i16_to_f32(i16 %a0) nounwind {
 define <4 x float> @cvt_4i16_to_4f32(<4 x i16> %a0) nounwind {
 ; ALL-LABEL: cvt_4i16_to_4f32:
 ; ALL:       # %bb.0:
-; ALL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
 ; ALL-NEXT:    vmovq %xmm0, %rax
 ; ALL-NEXT:    movq %rax, %rcx
 ; ALL-NEXT:    movq %rax, %rdx
@@ -932,88 +931,20 @@ define double @cvt_i16_to_f64(i16 %a0) nounwind {
 }
 
 define <2 x double> @cvt_2i16_to_2f64(<2 x i16> %a0) nounwind {
-; AVX1-LABEL: cvt_2i16_to_2f64:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
-; AVX1-NEXT:    vmovd %xmm0, %eax
-; AVX1-NEXT:    movswl %ax, %ecx
-; AVX1-NEXT:    shrl $16, %eax
-; AVX1-NEXT:    cwtl
-; AVX1-NEXT:    vmovd %eax, %xmm0
-; AVX1-NEXT:    vcvtph2ps %xmm0, %xmm0
-; AVX1-NEXT:    vmovd %ecx, %xmm1
-; AVX1-NEXT:    vcvtph2ps %xmm1, %xmm1
-; AVX1-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
-; AVX1-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX1-NEXT:    retq
-;
-; AVX2-SLOW-LABEL: cvt_2i16_to_2f64:
-; AVX2-SLOW:       # %bb.0:
-; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vmovd %xmm0, %eax
-; AVX2-SLOW-NEXT:    movswl %ax, %ecx
-; AVX2-SLOW-NEXT:    shrl $16, %eax
-; AVX2-SLOW-NEXT:    cwtl
-; AVX2-SLOW-NEXT:    vmovd %eax, %xmm0
-; AVX2-SLOW-NEXT:    vcvtph2ps %xmm0, %xmm0
-; AVX2-SLOW-NEXT:    vmovd %ecx, %xmm1
-; AVX2-SLOW-NEXT:    vcvtph2ps %xmm1, %xmm1
-; AVX2-SLOW-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
-; AVX2-SLOW-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
-; AVX2-SLOW-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX2-SLOW-NEXT:    retq
-;
-; AVX2-FAST-LABEL: cvt_2i16_to_2f64:
-; AVX2-FAST:       # %bb.0:
-; AVX2-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,8,9,8,9,10,11,8,9,10,11,12,13,14,15]
-; AVX2-FAST-NEXT:    vmovd %xmm0, %eax
-; AVX2-FAST-NEXT:    movswl %ax, %ecx
-; AVX2-FAST-NEXT:    shrl $16, %eax
-; AVX2-FAST-NEXT:    cwtl
-; AVX2-FAST-NEXT:    vmovd %eax, %xmm0
-; AVX2-FAST-NEXT:    vcvtph2ps %xmm0, %xmm0
-; AVX2-FAST-NEXT:    vmovd %ecx, %xmm1
-; AVX2-FAST-NEXT:    vcvtph2ps %xmm1, %xmm1
-; AVX2-FAST-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
-; AVX2-FAST-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
-; AVX2-FAST-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX2-FAST-NEXT:    retq
-;
-; AVX512F-LABEL: cvt_2i16_to_2f64:
-; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; AVX512F-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
-; AVX512F-NEXT:    vmovd %xmm0, %eax
-; AVX512F-NEXT:    movswl %ax, %ecx
-; AVX512F-NEXT:    shrl $16, %eax
-; AVX512F-NEXT:    cwtl
-; AVX512F-NEXT:    vmovd %eax, %xmm0
-; AVX512F-NEXT:    vcvtph2ps %xmm0, %xmm0
-; AVX512F-NEXT:    vmovd %ecx, %xmm1
-; AVX512F-NEXT:    vcvtph2ps %xmm1, %xmm1
-; AVX512F-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
-; AVX512F-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX512F-NEXT:    retq
-;
-; AVX512VL-LABEL: cvt_2i16_to_2f64:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpmovqw %xmm0, -{{[0-9]+}}(%rsp)
-; AVX512VL-NEXT:    movl -{{[0-9]+}}(%rsp), %eax
-; AVX512VL-NEXT:    movswl %ax, %ecx
-; AVX512VL-NEXT:    shrl $16, %eax
-; AVX512VL-NEXT:    cwtl
-; AVX512VL-NEXT:    vmovd %eax, %xmm0
-; AVX512VL-NEXT:    vcvtph2ps %xmm0, %xmm0
-; AVX512VL-NEXT:    vmovd %ecx, %xmm1
-; AVX512VL-NEXT:    vcvtph2ps %xmm1, %xmm1
-; AVX512VL-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
-; AVX512VL-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
-; AVX512VL-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX512VL-NEXT:    retq
+; ALL-LABEL: cvt_2i16_to_2f64:
+; ALL:       # %bb.0:
+; ALL-NEXT:    vmovd %xmm0, %eax
+; ALL-NEXT:    movswl %ax, %ecx
+; ALL-NEXT:    shrl $16, %eax
+; ALL-NEXT:    cwtl
+; ALL-NEXT:    vmovd %eax, %xmm0
+; ALL-NEXT:    vcvtph2ps %xmm0, %xmm0
+; ALL-NEXT:    vmovd %ecx, %xmm1
+; ALL-NEXT:    vcvtph2ps %xmm1, %xmm1
+; ALL-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
+; ALL-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
+; ALL-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; ALL-NEXT:    retq
   %1 = bitcast <2 x i16> %a0 to <2 x half>
   %2 = fpext <2 x half> %1 to <2 x double>
   ret <2 x double> %2
@@ -1022,7 +953,6 @@ define <2 x double> @cvt_2i16_to_2f64(<2 x i16> %a0) nounwind {
 define <4 x double> @cvt_4i16_to_4f64(<4 x i16> %a0) nounwind {
 ; ALL-LABEL: cvt_4i16_to_4f64:
 ; ALL:       # %bb.0:
-; ALL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
 ; ALL-NEXT:    vmovq %xmm0, %rax
 ; ALL-NEXT:    movq %rax, %rcx
 ; ALL-NEXT:    movl %eax, %edx
@@ -1055,67 +985,20 @@ define <4 x double> @cvt_4i16_to_4f64(<4 x i16> %a0) nounwind {
 }
 
 define <2 x double> @cvt_8i16_to_2f64(<8 x i16> %a0) nounwind {
-; AVX1-LABEL: cvt_8i16_to_2f64:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovd %xmm0, %eax
-; AVX1-NEXT:    movswl %ax, %ecx
-; AVX1-NEXT:    shrl $16, %eax
-; AVX1-NEXT:    cwtl
-; AVX1-NEXT:    vmovd %eax, %xmm0
-; AVX1-NEXT:    vcvtph2ps %xmm0, %xmm0
-; AVX1-NEXT:    vmovd %ecx, %xmm1
-; AVX1-NEXT:    vcvtph2ps %xmm1, %xmm1
-; AVX1-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
-; AVX1-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: cvt_8i16_to_2f64:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vmovd %xmm0, %eax
-; AVX2-NEXT:    movswl %ax, %ecx
-; AVX2-NEXT:    shrl $16, %eax
-; AVX2-NEXT:    cwtl
-; AVX2-NEXT:    vmovd %eax, %xmm0
-; AVX2-NEXT:    vcvtph2ps %xmm0, %xmm0
-; AVX2-NEXT:    vmovd %ecx, %xmm1
-; AVX2-NEXT:    vcvtph2ps %xmm1, %xmm1
-; AVX2-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
-; AVX2-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX2-NEXT:    retq
-;
-; AVX512F-LABEL: cvt_8i16_to_2f64:
-; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovd %xmm0, %eax
-; AVX512F-NEXT:    movswl %ax, %ecx
-; AVX512F-NEXT:    shrl $16, %eax
-; AVX512F-NEXT:    cwtl
-; AVX512F-NEXT:    vmovd %eax, %xmm0
-; AVX512F-NEXT:    vcvtph2ps %xmm0, %xmm0
-; AVX512F-NEXT:    vmovd %ecx, %xmm1
-; AVX512F-NEXT:    vcvtph2ps %xmm1, %xmm1
-; AVX512F-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
-; AVX512F-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX512F-NEXT:    retq
-;
-; AVX512VL-LABEL: cvt_8i16_to_2f64:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; AVX512VL-NEXT:    vpmovqw %xmm0, -{{[0-9]+}}(%rsp)
-; AVX512VL-NEXT:    movl -{{[0-9]+}}(%rsp), %eax
-; AVX512VL-NEXT:    movswl %ax, %ecx
-; AVX512VL-NEXT:    shrl $16, %eax
-; AVX512VL-NEXT:    cwtl
-; AVX512VL-NEXT:    vmovd %eax, %xmm0
-; AVX512VL-NEXT:    vcvtph2ps %xmm0, %xmm0
-; AVX512VL-NEXT:    vmovd %ecx, %xmm1
-; AVX512VL-NEXT:    vcvtph2ps %xmm1, %xmm1
-; AVX512VL-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
-; AVX512VL-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
-; AVX512VL-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX512VL-NEXT:    retq
+; ALL-LABEL: cvt_8i16_to_2f64:
+; ALL:       # %bb.0:
+; ALL-NEXT:    vmovd %xmm0, %eax
+; ALL-NEXT:    movswl %ax, %ecx
+; ALL-NEXT:    shrl $16, %eax
+; ALL-NEXT:    cwtl
+; ALL-NEXT:    vmovd %eax, %xmm0
+; ALL-NEXT:    vcvtph2ps %xmm0, %xmm0
+; ALL-NEXT:    vmovd %ecx, %xmm1
+; ALL-NEXT:    vcvtph2ps %xmm1, %xmm1
+; ALL-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
+; ALL-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
+; ALL-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; ALL-NEXT:    retq
   %1 = shufflevector <8 x i16> %a0, <8 x i16> undef, <2 x i32> <i32 0, i32 1>
   %2 = bitcast <2 x i16> %1 to <2 x half>
   %3 = fpext <2 x half> %2 to <2 x double>
@@ -1594,7 +1477,7 @@ define <4 x i16> @cvt_4f32_to_4i16(<4 x float> %a0) nounwind {
 ; ALL-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
 ; ALL-NEXT:    vmovd %xmm0, %eax
 ; ALL-NEXT:    movw %ax, -{{[0-9]+}}(%rsp)
-; ALL-NEXT:    vpmovzxwd {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
+; ALL-NEXT:    vmovaps -{{[0-9]+}}(%rsp), %xmm0
 ; ALL-NEXT:    retq
   %1 = fptrunc <4 x float> %a0 to <4 x half>
   %2 = bitcast <4 x half> %1 to <4 x i16>
@@ -1619,7 +1502,7 @@ define <8 x i16> @cvt_4f32_to_8i16_undef(<4 x float> %a0) nounwind {
 ; ALL-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
 ; ALL-NEXT:    vmovd %xmm0, %eax
 ; ALL-NEXT:    movw %ax, -{{[0-9]+}}(%rsp)
-; ALL-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; ALL-NEXT:    vmovaps -{{[0-9]+}}(%rsp), %xmm0
 ; ALL-NEXT:    retq
   %1 = fptrunc <4 x float> %a0 to <4 x half>
   %2 = bitcast <4 x half> %1 to <4 x i16>
@@ -1972,7 +1855,7 @@ define void @store_cvt_4f32_to_8i16_undef(<4 x float> %a0, <8 x i16>* %a1) nounw
 ; ALL-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
 ; ALL-NEXT:    vmovd %xmm0, %eax
 ; ALL-NEXT:    movw %ax, -{{[0-9]+}}(%rsp)
-; ALL-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; ALL-NEXT:    vmovaps -{{[0-9]+}}(%rsp), %xmm0
 ; ALL-NEXT:    vmovaps %xmm0, (%rdi)
 ; ALL-NEXT:    retq
   %1 = fptrunc <4 x float> %a0 to <4 x half>
@@ -2279,12 +2162,12 @@ define <2 x i16> @cvt_2f64_to_2i16(<2 x double> %a0) nounwind {
 ; ALL-NEXT:    subq $40, %rsp
 ; ALL-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; ALL-NEXT:    callq __truncdfhf2
-; ALL-NEXT:    movw %ax, {{[0-9]+}}(%rsp)
+; ALL-NEXT:    movw %ax, (%rsp)
 ; ALL-NEXT:    vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
 ; ALL-NEXT:    # xmm0 = mem[1,0]
 ; ALL-NEXT:    callq __truncdfhf2
 ; ALL-NEXT:    movw %ax, {{[0-9]+}}(%rsp)
-; ALL-NEXT:    vpmovzxwq {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero
+; ALL-NEXT:    vmovaps (%rsp), %xmm0
 ; ALL-NEXT:    addq $40, %rsp
 ; ALL-NEXT:    retq
   %1 = fptrunc <2 x double> %a0 to <2 x half>
@@ -2306,7 +2189,7 @@ define <4 x i16> @cvt_4f64_to_4i16(<4 x double> %a0) nounwind {
 ; ALL-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
 ; ALL-NEXT:    vzeroupper
 ; ALL-NEXT:    callq __truncdfhf2
-; ALL-NEXT:    movw %ax, {{[0-9]+}}(%rsp)
+; ALL-NEXT:    movw %ax, (%rsp)
 ; ALL-NEXT:    vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
 ; ALL-NEXT:    # xmm0 = mem[1,0]
 ; ALL-NEXT:    callq __truncdfhf2
@@ -2315,7 +2198,7 @@ define <4 x i16> @cvt_4f64_to_4i16(<4 x double> %a0) nounwind {
 ; ALL-NEXT:    # xmm0 = mem[1,0]
 ; ALL-NEXT:    callq __truncdfhf2
 ; ALL-NEXT:    movw %ax, {{[0-9]+}}(%rsp)
-; ALL-NEXT:    vpmovzxwd {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
+; ALL-NEXT:    vmovaps (%rsp), %xmm0
 ; ALL-NEXT:    addq $88, %rsp
 ; ALL-NEXT:    retq
   %1 = fptrunc <4 x double> %a0 to <4 x half>
@@ -2337,7 +2220,7 @@ define <8 x i16> @cvt_4f64_to_8i16_undef(<4 x double> %a0) nounwind {
 ; ALL-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
 ; ALL-NEXT:    vzeroupper
 ; ALL-NEXT:    callq __truncdfhf2
-; ALL-NEXT:    movw %ax, {{[0-9]+}}(%rsp)
+; ALL-NEXT:    movw %ax, (%rsp)
 ; ALL-NEXT:    vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
 ; ALL-NEXT:    # xmm0 = mem[1,0]
 ; ALL-NEXT:    callq __truncdfhf2
@@ -2346,7 +2229,7 @@ define <8 x i16> @cvt_4f64_to_8i16_undef(<4 x double> %a0) nounwind {
 ; ALL-NEXT:    # xmm0 = mem[1,0]
 ; ALL-NEXT:    callq __truncdfhf2
 ; ALL-NEXT:    movw %ax, {{[0-9]+}}(%rsp)
-; ALL-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; ALL-NEXT:    vmovaps (%rsp), %xmm0
 ; ALL-NEXT:    addq $88, %rsp
 ; ALL-NEXT:    retq
   %1 = fptrunc <4 x double> %a0 to <4 x half>
@@ -2369,7 +2252,7 @@ define <8 x i16> @cvt_4f64_to_8i16_zero(<4 x double> %a0) nounwind {
 ; ALL-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
 ; ALL-NEXT:    vzeroupper
 ; ALL-NEXT:    callq __truncdfhf2
-; ALL-NEXT:    movw %ax, {{[0-9]+}}(%rsp)
+; ALL-NEXT:    movw %ax, (%rsp)
 ; ALL-NEXT:    vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
 ; ALL-NEXT:    # xmm0 = mem[1,0]
 ; ALL-NEXT:    callq __truncdfhf2
@@ -2775,7 +2658,7 @@ define void @store_cvt_4f64_to_8i16_undef(<4 x double> %a0, <8 x i16>* %a1) noun
 ; ALL-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
 ; ALL-NEXT:    vzeroupper
 ; ALL-NEXT:    callq __truncdfhf2
-; ALL-NEXT:    movw %ax, {{[0-9]+}}(%rsp)
+; ALL-NEXT:    movw %ax, (%rsp)
 ; ALL-NEXT:    vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
 ; ALL-NEXT:    # xmm0 = mem[1,0]
 ; ALL-NEXT:    callq __truncdfhf2
@@ -2784,7 +2667,7 @@ define void @store_cvt_4f64_to_8i16_undef(<4 x double> %a0, <8 x i16>* %a1) noun
 ; ALL-NEXT:    # xmm0 = mem[1,0]
 ; ALL-NEXT:    callq __truncdfhf2
 ; ALL-NEXT:    movw %ax, {{[0-9]+}}(%rsp)
-; ALL-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; ALL-NEXT:    vmovaps (%rsp), %xmm0
 ; ALL-NEXT:    vmovaps %xmm0, (%rbx)
 ; ALL-NEXT:    addq $80, %rsp
 ; ALL-NEXT:    popq %rbx
@@ -2812,7 +2695,7 @@ define void @store_cvt_4f64_to_8i16_zero(<4 x double> %a0, <8 x i16>* %a1) nounw
 ; ALL-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
 ; ALL-NEXT:    vzeroupper
 ; ALL-NEXT:    callq __truncdfhf2
-; ALL-NEXT:    movw %ax, {{[0-9]+}}(%rsp)
+; ALL-NEXT:    movw %ax, (%rsp)
 ; ALL-NEXT:    vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
 ; ALL-NEXT:    # xmm0 = mem[1,0]
 ; ALL-NEXT:    callq __truncdfhf2
diff --git a/llvm/test/CodeGen/X86/vector-idiv-sdiv-128.ll b/llvm/test/CodeGen/X86/vector-idiv-sdiv-128.ll
index 58c972164..abd4002a7 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-sdiv-128.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-sdiv-128.ll
@@ -971,215 +971,182 @@ define <16 x i8> @test_rem_variable_16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
 ; SSE2-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movb -{{[0-9]+}}(%rsp), %al
-; SSE2-NEXT:    cbtw
+; SSE2-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
 ; SSE2-NEXT:    idivb -{{[0-9]+}}(%rsp)
 ; SSE2-NEXT:    movsbl %ah, %eax
 ; SSE2-NEXT:    movd %eax, %xmm0
-; SSE2-NEXT:    movb -{{[0-9]+}}(%rsp), %al
-; SSE2-NEXT:    cbtw
+; SSE2-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
 ; SSE2-NEXT:    idivb -{{[0-9]+}}(%rsp)
 ; SSE2-NEXT:    movsbl %ah, %eax
 ; SSE2-NEXT:    movd %eax, %xmm1
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; SSE2-NEXT:    movb -{{[0-9]+}}(%rsp), %al
-; SSE2-NEXT:    cbtw
-; SSE2-NEXT:    idivb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movsbl %ah, %eax
-; SSE2-NEXT:    movd %eax, %xmm2
-; SSE2-NEXT:    movb -{{[0-9]+}}(%rsp), %al
-; SSE2-NEXT:    cbtw
+; SSE2-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
 ; SSE2-NEXT:    idivb -{{[0-9]+}}(%rsp)
 ; SSE2-NEXT:    movsbl %ah, %eax
 ; SSE2-NEXT:    movd %eax, %xmm0
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; SSE2-NEXT:    movb -{{[0-9]+}}(%rsp), %al
-; SSE2-NEXT:    cbtw
-; SSE2-NEXT:    idivb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movsbl %ah, %eax
-; SSE2-NEXT:    movd %eax, %xmm1
-; SSE2-NEXT:    movb -{{[0-9]+}}(%rsp), %al
-; SSE2-NEXT:    cbtw
+; SSE2-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
 ; SSE2-NEXT:    idivb -{{[0-9]+}}(%rsp)
 ; SSE2-NEXT:    movsbl %ah, %eax
 ; SSE2-NEXT:    movd %eax, %xmm2
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
-; SSE2-NEXT:    movb -{{[0-9]+}}(%rsp), %al
-; SSE2-NEXT:    cbtw
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; SSE2-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
 ; SSE2-NEXT:    idivb -{{[0-9]+}}(%rsp)
 ; SSE2-NEXT:    movsbl %ah, %eax
-; SSE2-NEXT:    movd %eax, %xmm3
-; SSE2-NEXT:    movb -{{[0-9]+}}(%rsp), %al
-; SSE2-NEXT:    cbtw
+; SSE2-NEXT:    movd %eax, %xmm0
+; SSE2-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
 ; SSE2-NEXT:    idivb -{{[0-9]+}}(%rsp)
 ; SSE2-NEXT:    movsbl %ah, %eax
-; SSE2-NEXT:    movd %eax, %xmm1
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3],xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; SSE2-NEXT:    movb -{{[0-9]+}}(%rsp), %al
-; SSE2-NEXT:    cbtw
+; SSE2-NEXT:    movd %eax, %xmm3
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE2-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
 ; SSE2-NEXT:    idivb -{{[0-9]+}}(%rsp)
 ; SSE2-NEXT:    movsbl %ah, %eax
 ; SSE2-NEXT:    movd %eax, %xmm0
-; SSE2-NEXT:    movb -{{[0-9]+}}(%rsp), %al
-; SSE2-NEXT:    cbtw
+; SSE2-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
 ; SSE2-NEXT:    idivb -{{[0-9]+}}(%rsp)
 ; SSE2-NEXT:    movsbl %ah, %eax
-; SSE2-NEXT:    movd %eax, %xmm3
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
-; SSE2-NEXT:    movb -{{[0-9]+}}(%rsp), %al
-; SSE2-NEXT:    cbtw
+; SSE2-NEXT:    movd %eax, %xmm1
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; SSE2-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
 ; SSE2-NEXT:    idivb -{{[0-9]+}}(%rsp)
 ; SSE2-NEXT:    movsbl %ah, %eax
 ; SSE2-NEXT:    movd %eax, %xmm0
-; SSE2-NEXT:    movb -{{[0-9]+}}(%rsp), %al
-; SSE2-NEXT:    cbtw
+; SSE2-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
 ; SSE2-NEXT:    idivb -{{[0-9]+}}(%rsp)
 ; SSE2-NEXT:    movsbl %ah, %eax
 ; SSE2-NEXT:    movd %eax, %xmm2
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
-; SSE2-NEXT:    movb -{{[0-9]+}}(%rsp), %al
-; SSE2-NEXT:    cbtw
+; SSE2-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
 ; SSE2-NEXT:    idivb -{{[0-9]+}}(%rsp)
 ; SSE2-NEXT:    movsbl %ah, %eax
 ; SSE2-NEXT:    movd %eax, %xmm0
-; SSE2-NEXT:    movb -{{[0-9]+}}(%rsp), %al
-; SSE2-NEXT:    cbtw
+; SSE2-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
 ; SSE2-NEXT:    idivb -{{[0-9]+}}(%rsp)
 ; SSE2-NEXT:    movsbl %ah, %eax
 ; SSE2-NEXT:    movd %eax, %xmm3
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
-; SSE2-NEXT:    movb -{{[0-9]+}}(%rsp), %al
-; SSE2-NEXT:    cbtw
-; SSE2-NEXT:    movb -{{[0-9]+}}(%rsp), %cl
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; SSE2-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    idivb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    movsbl %ah, %eax
+; SSE2-NEXT:    movd %eax, %xmm0
+; SSE2-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    idivb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    movsbl %ah, %eax
+; SSE2-NEXT:    movd %eax, %xmm2
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE2-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
 ; SSE2-NEXT:    idivb -{{[0-9]+}}(%rsp)
 ; SSE2-NEXT:    movsbl %ah, %eax
 ; SSE2-NEXT:    movd %eax, %xmm4
-; SSE2-NEXT:    movl %ecx, %eax
-; SSE2-NEXT:    cbtw
+; SSE2-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
 ; SSE2-NEXT:    idivb -{{[0-9]+}}(%rsp)
 ; SSE2-NEXT:    movsbl %ah, %eax
 ; SSE2-NEXT:    movd %eax, %xmm0
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
 ; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: test_rem_variable_16i8:
 ; SSE41:       # %bb.0:
+; SSE41-NEXT:    pextrb $1, %xmm1, %ecx
 ; SSE41-NEXT:    pextrb $1, %xmm0, %eax
-; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    cbtw
-; SSE41-NEXT:    pextrb $1, %xmm1, %ecx
 ; SSE41-NEXT:    idivb %cl
 ; SSE41-NEXT:    movsbl %ah, %ecx
+; SSE41-NEXT:    pextrb $0, %xmm1, %edx
 ; SSE41-NEXT:    pextrb $0, %xmm0, %eax
-; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    cbtw
-; SSE41-NEXT:    pextrb $0, %xmm1, %edx
 ; SSE41-NEXT:    idivb %dl
 ; SSE41-NEXT:    movsbl %ah, %eax
 ; SSE41-NEXT:    movd %eax, %xmm2
-; SSE41-NEXT:    pextrb $2, %xmm0, %eax
-; SSE41-NEXT:    # kill: def $al killed $al killed $eax
-; SSE41-NEXT:    cbtw
 ; SSE41-NEXT:    pinsrb $1, %ecx, %xmm2
 ; SSE41-NEXT:    pextrb $2, %xmm1, %ecx
+; SSE41-NEXT:    pextrb $2, %xmm0, %eax
+; SSE41-NEXT:    cbtw
 ; SSE41-NEXT:    idivb %cl
-; SSE41-NEXT:    movsbl %ah, %ecx
+; SSE41-NEXT:    movsbl %ah, %eax
+; SSE41-NEXT:    pinsrb $2, %eax, %xmm2
+; SSE41-NEXT:    pextrb $3, %xmm1, %ecx
 ; SSE41-NEXT:    pextrb $3, %xmm0, %eax
-; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    cbtw
-; SSE41-NEXT:    pinsrb $2, %ecx, %xmm2
-; SSE41-NEXT:    pextrb $3, %xmm1, %ecx
 ; SSE41-NEXT:    idivb %cl
-; SSE41-NEXT:    movsbl %ah, %ecx
+; SSE41-NEXT:    movsbl %ah, %eax
+; SSE41-NEXT:    pinsrb $3, %eax, %xmm2
+; SSE41-NEXT:    pextrb $4, %xmm1, %ecx
 ; SSE41-NEXT:    pextrb $4, %xmm0, %eax
-; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    cbtw
-; SSE41-NEXT:    pinsrb $3, %ecx, %xmm2
-; SSE41-NEXT:    pextrb $4, %xmm1, %ecx
 ; SSE41-NEXT:    idivb %cl
-; SSE41-NEXT:    movsbl %ah, %ecx
+; SSE41-NEXT:    movsbl %ah, %eax
+; SSE41-NEXT:    pinsrb $4, %eax, %xmm2
+; SSE41-NEXT:    pextrb $5, %xmm1, %ecx
 ; SSE41-NEXT:    pextrb $5, %xmm0, %eax
-; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    cbtw
-; SSE41-NEXT:    pinsrb $4, %ecx, %xmm2
-; SSE41-NEXT:    pextrb $5, %xmm1, %ecx
 ; SSE41-NEXT:    idivb %cl
-; SSE41-NEXT:    movsbl %ah, %ecx
+; SSE41-NEXT:    movsbl %ah, %eax
+; SSE41-NEXT:    pinsrb $5, %eax, %xmm2
+; SSE41-NEXT:    pextrb $6, %xmm1, %ecx
 ; SSE41-NEXT:    pextrb $6, %xmm0, %eax
-; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    cbtw
-; SSE41-NEXT:    pinsrb $5, %ecx, %xmm2
-; SSE41-NEXT:    pextrb $6, %xmm1, %ecx
 ; SSE41-NEXT:    idivb %cl
-; SSE41-NEXT:    movsbl %ah, %ecx
+; SSE41-NEXT:    movsbl %ah, %eax
+; SSE41-NEXT:    pinsrb $6, %eax, %xmm2
+; SSE41-NEXT:    pextrb $7, %xmm1, %ecx
 ; SSE41-NEXT:    pextrb $7, %xmm0, %eax
-; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    cbtw
-; SSE41-NEXT:    pinsrb $6, %ecx, %xmm2
-; SSE41-NEXT:    pextrb $7, %xmm1, %ecx
 ; SSE41-NEXT:    idivb %cl
-; SSE41-NEXT:    movsbl %ah, %ecx
+; SSE41-NEXT:    movsbl %ah, %eax
+; SSE41-NEXT:    pinsrb $7, %eax, %xmm2
+; SSE41-NEXT:    pextrb $8, %xmm1, %ecx
 ; SSE41-NEXT:    pextrb $8, %xmm0, %eax
-; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    cbtw
-; SSE41-NEXT:    pinsrb $7, %ecx, %xmm2
-; SSE41-NEXT:    pextrb $8, %xmm1, %ecx
 ; SSE41-NEXT:    idivb %cl
-; SSE41-NEXT:    movsbl %ah, %ecx
+; SSE41-NEXT:    movsbl %ah, %eax
+; SSE41-NEXT:    pinsrb $8, %eax, %xmm2
+; SSE41-NEXT:    pextrb $9, %xmm1, %ecx
 ; SSE41-NEXT:    pextrb $9, %xmm0, %eax
-; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    cbtw
-; SSE41-NEXT:    pinsrb $8, %ecx, %xmm2
-; SSE41-NEXT:    pextrb $9, %xmm1, %ecx
 ; SSE41-NEXT:    idivb %cl
-; SSE41-NEXT:    movsbl %ah, %ecx
+; SSE41-NEXT:    movsbl %ah, %eax
+; SSE41-NEXT:    pinsrb $9, %eax, %xmm2
+; SSE41-NEXT:    pextrb $10, %xmm1, %ecx
 ; SSE41-NEXT:    pextrb $10, %xmm0, %eax
-; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    cbtw
-; SSE41-NEXT:    pinsrb $9, %ecx, %xmm2
-; SSE41-NEXT:    pextrb $10, %xmm1, %ecx
 ; SSE41-NEXT:    idivb %cl
-; SSE41-NEXT:    movsbl %ah, %ecx
+; SSE41-NEXT:    movsbl %ah, %eax
+; SSE41-NEXT:    pinsrb $10, %eax, %xmm2
+; SSE41-NEXT:    pextrb $11, %xmm1, %ecx
 ; SSE41-NEXT:    pextrb $11, %xmm0, %eax
-; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    cbtw
-; SSE41-NEXT:    pinsrb $10, %ecx, %xmm2
-; SSE41-NEXT:    pextrb $11, %xmm1, %ecx
 ; SSE41-NEXT:    idivb %cl
-; SSE41-NEXT:    movsbl %ah, %ecx
+; SSE41-NEXT:    movsbl %ah, %eax
+; SSE41-NEXT:    pinsrb $11, %eax, %xmm2
+; SSE41-NEXT:    pextrb $12, %xmm1, %ecx
 ; SSE41-NEXT:    pextrb $12, %xmm0, %eax
-; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    cbtw
-; SSE41-NEXT:    pinsrb $11, %ecx, %xmm2
-; SSE41-NEXT:    pextrb $12, %xmm1, %ecx
 ; SSE41-NEXT:    idivb %cl
-; SSE41-NEXT:    movsbl %ah, %ecx
+; SSE41-NEXT:    movsbl %ah, %eax
+; SSE41-NEXT:    pinsrb $12, %eax, %xmm2
+; SSE41-NEXT:    pextrb $13, %xmm1, %ecx
 ; SSE41-NEXT:    pextrb $13, %xmm0, %eax
-; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    cbtw
-; SSE41-NEXT:    pinsrb $12, %ecx, %xmm2
-; SSE41-NEXT:    pextrb $13, %xmm1, %ecx
 ; SSE41-NEXT:    idivb %cl
-; SSE41-NEXT:    movsbl %ah, %ecx
+; SSE41-NEXT:    movsbl %ah, %eax
+; SSE41-NEXT:    pinsrb $13, %eax, %xmm2
+; SSE41-NEXT:    pextrb $14, %xmm1, %ecx
 ; SSE41-NEXT:    pextrb $14, %xmm0, %eax
-; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    cbtw
-; SSE41-NEXT:    pinsrb $13, %ecx, %xmm2
-; SSE41-NEXT:    pextrb $14, %xmm1, %ecx
 ; SSE41-NEXT:    idivb %cl
-; SSE41-NEXT:    movsbl %ah, %ecx
+; SSE41-NEXT:    movsbl %ah, %eax
+; SSE41-NEXT:    pinsrb $14, %eax, %xmm2
+; SSE41-NEXT:    pextrb $15, %xmm1, %ecx
 ; SSE41-NEXT:    pextrb $15, %xmm0, %eax
-; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    cbtw
-; SSE41-NEXT:    pinsrb $14, %ecx, %xmm2
-; SSE41-NEXT:    pextrb $15, %xmm1, %ecx
 ; SSE41-NEXT:    idivb %cl
 ; SSE41-NEXT:    movsbl %ah, %eax
 ; SSE41-NEXT:    pinsrb $15, %eax, %xmm2
@@ -1188,118 +1155,102 @@ define <16 x i8> @test_rem_variable_16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
 ;
 ; AVX-LABEL: test_rem_variable_16i8:
 ; AVX:       # %bb.0:
+; AVX-NEXT:    vpextrb $1, %xmm1, %ecx
 ; AVX-NEXT:    vpextrb $1, %xmm0, %eax
-; AVX-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX-NEXT:    cbtw
-; AVX-NEXT:    vpextrb $1, %xmm1, %ecx
 ; AVX-NEXT:    idivb %cl
 ; AVX-NEXT:    movsbl %ah, %ecx
+; AVX-NEXT:    vpextrb $0, %xmm1, %edx
 ; AVX-NEXT:    vpextrb $0, %xmm0, %eax
-; AVX-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX-NEXT:    cbtw
-; AVX-NEXT:    vpextrb $0, %xmm1, %edx
 ; AVX-NEXT:    idivb %dl
 ; AVX-NEXT:    movsbl %ah, %eax
 ; AVX-NEXT:    vmovd %eax, %xmm2
-; AVX-NEXT:    vpextrb $2, %xmm0, %eax
-; AVX-NEXT:    # kill: def $al killed $al killed $eax
-; AVX-NEXT:    cbtw
 ; AVX-NEXT:    vpinsrb $1, %ecx, %xmm2, %xmm2
 ; AVX-NEXT:    vpextrb $2, %xmm1, %ecx
+; AVX-NEXT:    vpextrb $2, %xmm0, %eax
+; AVX-NEXT:    cbtw
 ; AVX-NEXT:    idivb %cl
-; AVX-NEXT:    movsbl %ah, %ecx
+; AVX-NEXT:    movsbl %ah, %eax
+; AVX-NEXT:    vpinsrb $2, %eax, %xmm2, %xmm2
+; AVX-NEXT:    vpextrb $3, %xmm1, %ecx
 ; AVX-NEXT:    vpextrb $3, %xmm0, %eax
-; AVX-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX-NEXT:    cbtw
-; AVX-NEXT:    vpinsrb $2, %ecx, %xmm2, %xmm2
-; AVX-NEXT:    vpextrb $3, %xmm1, %ecx
 ; AVX-NEXT:    idivb %cl
-; AVX-NEXT:    movsbl %ah, %ecx
+; AVX-NEXT:    movsbl %ah, %eax
+; AVX-NEXT:    vpinsrb $3, %eax, %xmm2, %xmm2
+; AVX-NEXT:    vpextrb $4, %xmm1, %ecx
 ; AVX-NEXT:    vpextrb $4, %xmm0, %eax
-; AVX-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX-NEXT:    cbtw
-; AVX-NEXT:    vpinsrb $3, %ecx, %xmm2, %xmm2
-; AVX-NEXT:    vpextrb $4, %xmm1, %ecx
 ; AVX-NEXT:    idivb %cl
-; AVX-NEXT:    movsbl %ah, %ecx
+; AVX-NEXT:    movsbl %ah, %eax
+; AVX-NEXT:    vpinsrb $4, %eax, %xmm2, %xmm2
+; AVX-NEXT:    vpextrb $5, %xmm1, %ecx
 ; AVX-NEXT:    vpextrb $5, %xmm0, %eax
-; AVX-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX-NEXT:    cbtw
-; AVX-NEXT:    vpinsrb $4, %ecx, %xmm2, %xmm2
-; AVX-NEXT:    vpextrb $5, %xmm1, %ecx
 ; AVX-NEXT:    idivb %cl
-; AVX-NEXT:    movsbl %ah, %ecx
+; AVX-NEXT:    movsbl %ah, %eax
+; AVX-NEXT:    vpinsrb $5, %eax, %xmm2, %xmm2
+; AVX-NEXT:    vpextrb $6, %xmm1, %ecx
 ; AVX-NEXT:    vpextrb $6, %xmm0, %eax
-; AVX-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX-NEXT:    cbtw
-; AVX-NEXT:    vpinsrb $5, %ecx, %xmm2, %xmm2
-; AVX-NEXT:    vpextrb $6, %xmm1, %ecx
 ; AVX-NEXT:    idivb %cl
-; AVX-NEXT:    movsbl %ah, %ecx
+; AVX-NEXT:    movsbl %ah, %eax
+; AVX-NEXT:    vpinsrb $6, %eax, %xmm2, %xmm2
+; AVX-NEXT:    vpextrb $7, %xmm1, %ecx
 ; AVX-NEXT:    vpextrb $7, %xmm0, %eax
-; AVX-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX-NEXT:    cbtw
-; AVX-NEXT:    vpinsrb $6, %ecx, %xmm2, %xmm2
-; AVX-NEXT:    vpextrb $7, %xmm1, %ecx
 ; AVX-NEXT:    idivb %cl
-; AVX-NEXT:    movsbl %ah, %ecx
+; AVX-NEXT:    movsbl %ah, %eax
+; AVX-NEXT:    vpinsrb $7, %eax, %xmm2, %xmm2
+; AVX-NEXT:    vpextrb $8, %xmm1, %ecx
 ; AVX-NEXT:    vpextrb $8, %xmm0, %eax
-; AVX-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX-NEXT:    cbtw
-; AVX-NEXT:    vpinsrb $7, %ecx, %xmm2, %xmm2
-; AVX-NEXT:    vpextrb $8, %xmm1, %ecx
 ; AVX-NEXT:    idivb %cl
-; AVX-NEXT:    movsbl %ah, %ecx
+; AVX-NEXT:    movsbl %ah, %eax
+; AVX-NEXT:    vpinsrb $8, %eax, %xmm2, %xmm2
+; AVX-NEXT:    vpextrb $9, %xmm1, %ecx
 ; AVX-NEXT:    vpextrb $9, %xmm0, %eax
-; AVX-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX-NEXT:    cbtw
-; AVX-NEXT:    vpinsrb $8, %ecx, %xmm2, %xmm2
-; AVX-NEXT:    vpextrb $9, %xmm1, %ecx
 ; AVX-NEXT:    idivb %cl
-; AVX-NEXT:    movsbl %ah, %ecx
+; AVX-NEXT:    movsbl %ah, %eax
+; AVX-NEXT:    vpinsrb $9, %eax, %xmm2, %xmm2
+; AVX-NEXT:    vpextrb $10, %xmm1, %ecx
 ; AVX-NEXT:    vpextrb $10, %xmm0, %eax
-; AVX-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX-NEXT:    cbtw
-; AVX-NEXT:    vpinsrb $9, %ecx, %xmm2, %xmm2
-; AVX-NEXT:    vpextrb $10, %xmm1, %ecx
 ; AVX-NEXT:    idivb %cl
-; AVX-NEXT:    movsbl %ah, %ecx
+; AVX-NEXT:    movsbl %ah, %eax
+; AVX-NEXT:    vpinsrb $10, %eax, %xmm2, %xmm2
+; AVX-NEXT:    vpextrb $11, %xmm1, %ecx
 ; AVX-NEXT:    vpextrb $11, %xmm0, %eax
-; AVX-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX-NEXT:    cbtw
-; AVX-NEXT:    vpinsrb $10, %ecx, %xmm2, %xmm2
-; AVX-NEXT:    vpextrb $11, %xmm1, %ecx
 ; AVX-NEXT:    idivb %cl
-; AVX-NEXT:    movsbl %ah, %ecx
+; AVX-NEXT:    movsbl %ah, %eax
+; AVX-NEXT:    vpinsrb $11, %eax, %xmm2, %xmm2
+; AVX-NEXT:    vpextrb $12, %xmm1, %ecx
 ; AVX-NEXT:    vpextrb $12, %xmm0, %eax
-; AVX-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX-NEXT:    cbtw
-; AVX-NEXT:    vpinsrb $11, %ecx, %xmm2, %xmm2
-; AVX-NEXT:    vpextrb $12, %xmm1, %ecx
 ; AVX-NEXT:    idivb %cl
-; AVX-NEXT:    movsbl %ah, %ecx
+; AVX-NEXT:    movsbl %ah, %eax
+; AVX-NEXT:    vpinsrb $12, %eax, %xmm2, %xmm2
+; AVX-NEXT:    vpextrb $13, %xmm1, %ecx
 ; AVX-NEXT:    vpextrb $13, %xmm0, %eax
-; AVX-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX-NEXT:    cbtw
-; AVX-NEXT:    vpinsrb $12, %ecx, %xmm2, %xmm2
-; AVX-NEXT:    vpextrb $13, %xmm1, %ecx
 ; AVX-NEXT:    idivb %cl
-; AVX-NEXT:    movsbl %ah, %ecx
+; AVX-NEXT:    movsbl %ah, %eax
+; AVX-NEXT:    vpinsrb $13, %eax, %xmm2, %xmm2
+; AVX-NEXT:    vpextrb $14, %xmm1, %ecx
 ; AVX-NEXT:    vpextrb $14, %xmm0, %eax
-; AVX-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX-NEXT:    cbtw
-; AVX-NEXT:    vpinsrb $13, %ecx, %xmm2, %xmm2
-; AVX-NEXT:    vpextrb $14, %xmm1, %ecx
 ; AVX-NEXT:    idivb %cl
-; AVX-NEXT:    movsbl %ah, %ecx
+; AVX-NEXT:    movsbl %ah, %eax
+; AVX-NEXT:    vpinsrb $14, %eax, %xmm2, %xmm2
+; AVX-NEXT:    vpextrb $15, %xmm1, %ecx
 ; AVX-NEXT:    vpextrb $15, %xmm0, %eax
-; AVX-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX-NEXT:    cbtw
-; AVX-NEXT:    vpinsrb $14, %ecx, %xmm2, %xmm0
-; AVX-NEXT:    vpextrb $15, %xmm1, %ecx
 ; AVX-NEXT:    idivb %cl
 ; AVX-NEXT:    movsbl %ah, %eax
-; AVX-NEXT:    vpinsrb $15, %eax, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $15, %eax, %xmm2, %xmm0
 ; AVX-NEXT:    retq
   %res = srem <16 x i8> %a, %b
   ret <16 x i8> %res
diff --git a/llvm/test/CodeGen/X86/vector-idiv-sdiv-256.ll b/llvm/test/CodeGen/X86/vector-idiv-sdiv-256.ll
index eda349005..298028fd8 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-sdiv-256.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-sdiv-256.ll
@@ -582,7 +582,7 @@ define <32 x i8> @test_rem7_32i8(<32 x i8> %a) nounwind {
 ; AVX1-NEXT:    vpaddb %xmm1, %xmm2, %xmm2
 ; AVX1-NEXT:    vpsrlw $7, %xmm2, %xmm4
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm8 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; AVX1-NEXT:    vpand %xmm8, %xmm4, %xmm4
+; AVX1-NEXT:    vpand %xmm4, %xmm8, %xmm4
 ; AVX1-NEXT:    vpsrlw $2, %xmm2, %xmm2
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
 ; AVX1-NEXT:    vpand %xmm6, %xmm2, %xmm2
@@ -605,7 +605,7 @@ define <32 x i8> @test_rem7_32i8(<32 x i8> %a) nounwind {
 ; AVX1-NEXT:    vpackuswb %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vpaddb %xmm0, %xmm2, %xmm2
 ; AVX1-NEXT:    vpsrlw $7, %xmm2, %xmm3
-; AVX1-NEXT:    vpand %xmm8, %xmm3, %xmm3
+; AVX1-NEXT:    vpand %xmm3, %xmm8, %xmm3
 ; AVX1-NEXT:    vpsrlw $2, %xmm2, %xmm2
 ; AVX1-NEXT:    vpand %xmm6, %xmm2, %xmm2
 ; AVX1-NEXT:    vpxor %xmm7, %xmm2, %xmm2
diff --git a/llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll b/llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll
index 961bec56e..05422fa48 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll
@@ -102,15 +102,17 @@ define <16 x i32> @test_div7_16i32(<16 x i32> %a) nounwind {
 define <32 x i16> @test_div7_32i16(<32 x i16> %a) nounwind {
 ; AVX512F-LABEL: test_div7_32i16:
 ; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm2 = [18725,18725,18725,18725,18725,18725,18725,18725,18725,18725,18725,18725,18725,18725,18725,18725]
-; AVX512F-NEXT:    vpmulhw %ymm2, %ymm0, %ymm0
-; AVX512F-NEXT:    vpsrlw $15, %ymm0, %ymm3
-; AVX512F-NEXT:    vpsraw $1, %ymm0, %ymm0
-; AVX512F-NEXT:    vpaddw %ymm3, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpmulhw %ymm2, %ymm1, %ymm1
-; AVX512F-NEXT:    vpsrlw $15, %ymm1, %ymm2
+; AVX512F-NEXT:    vpsrlw $15, %ymm1, %ymm3
 ; AVX512F-NEXT:    vpsraw $1, %ymm1, %ymm1
-; AVX512F-NEXT:    vpaddw %ymm2, %ymm1, %ymm1
+; AVX512F-NEXT:    vpaddw %ymm3, %ymm1, %ymm1
+; AVX512F-NEXT:    vpmulhw %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vpsrlw $15, %ymm0, %ymm2
+; AVX512F-NEXT:    vpsraw $1, %ymm0, %ymm0
+; AVX512F-NEXT:    vpaddw %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: test_div7_32i16:
@@ -127,44 +129,46 @@ define <32 x i16> @test_div7_32i16(<32 x i16> %a) nounwind {
 define <64 x i8> @test_div7_64i8(<64 x i8> %a) nounwind {
 ; AVX512F-LABEL: test_div7_64i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
+; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm2
 ; AVX512F-NEXT:    vpmovsxbw %xmm2, %ymm2
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm3 = [65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427]
 ; AVX512F-NEXT:    vpmullw %ymm3, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpsrlw $8, %ymm2, %ymm2
-; AVX512F-NEXT:    vpmovsxbw %xmm0, %ymm4
+; AVX512F-NEXT:    vpmovsxbw %xmm1, %ymm4
 ; AVX512F-NEXT:    vpmullw %ymm3, %ymm4, %ymm4
 ; AVX512F-NEXT:    vpsrlw $8, %ymm4, %ymm4
 ; AVX512F-NEXT:    vpackuswb %ymm2, %ymm4, %ymm2
 ; AVX512F-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,1,3]
-; AVX512F-NEXT:    vpaddb %ymm0, %ymm2, %ymm0
-; AVX512F-NEXT:    vpsrlw $7, %ymm0, %ymm2
+; AVX512F-NEXT:    vpaddb %ymm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vpsrlw $7, %ymm1, %ymm2
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm4 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
 ; AVX512F-NEXT:    vpand %ymm4, %ymm2, %ymm2
-; AVX512F-NEXT:    vpsrlw $2, %ymm0, %ymm0
+; AVX512F-NEXT:    vpsrlw $2, %ymm1, %ymm1
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm5 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
-; AVX512F-NEXT:    vpand %ymm5, %ymm0, %ymm0
+; AVX512F-NEXT:    vpand %ymm5, %ymm1, %ymm1
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm6 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
-; AVX512F-NEXT:    vpxor %ymm6, %ymm0, %ymm0
-; AVX512F-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
-; AVX512F-NEXT:    vpsubb %ymm6, %ymm0, %ymm0
-; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm2
+; AVX512F-NEXT:    vpxor %ymm6, %ymm1, %ymm1
+; AVX512F-NEXT:    vpaddb %ymm2, %ymm1, %ymm1
+; AVX512F-NEXT:    vpsubb %ymm6, %ymm1, %ymm1
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm2
 ; AVX512F-NEXT:    vpmovsxbw %xmm2, %ymm2
 ; AVX512F-NEXT:    vpmullw %ymm3, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpsrlw $8, %ymm2, %ymm2
-; AVX512F-NEXT:    vpmovsxbw %xmm1, %ymm7
+; AVX512F-NEXT:    vpmovsxbw %xmm0, %ymm7
 ; AVX512F-NEXT:    vpmullw %ymm3, %ymm7, %ymm3
 ; AVX512F-NEXT:    vpsrlw $8, %ymm3, %ymm3
 ; AVX512F-NEXT:    vpackuswb %ymm2, %ymm3, %ymm2
 ; AVX512F-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,1,3]
-; AVX512F-NEXT:    vpaddb %ymm1, %ymm2, %ymm1
-; AVX512F-NEXT:    vpsrlw $7, %ymm1, %ymm2
+; AVX512F-NEXT:    vpaddb %ymm0, %ymm2, %ymm0
+; AVX512F-NEXT:    vpsrlw $7, %ymm0, %ymm2
 ; AVX512F-NEXT:    vpand %ymm4, %ymm2, %ymm2
-; AVX512F-NEXT:    vpsrlw $2, %ymm1, %ymm1
-; AVX512F-NEXT:    vpand %ymm5, %ymm1, %ymm1
-; AVX512F-NEXT:    vpxor %ymm6, %ymm1, %ymm1
-; AVX512F-NEXT:    vpaddb %ymm2, %ymm1, %ymm1
-; AVX512F-NEXT:    vpsubb %ymm6, %ymm1, %ymm1
+; AVX512F-NEXT:    vpsrlw $2, %ymm0, %ymm0
+; AVX512F-NEXT:    vpand %ymm5, %ymm0, %ymm0
+; AVX512F-NEXT:    vpxor %ymm6, %ymm0, %ymm0
+; AVX512F-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vpsubb %ymm6, %ymm0, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: test_div7_64i8:
@@ -201,66 +205,68 @@ define <64 x i8> @test_div7_64i8(<64 x i8> %a) nounwind {
 define <64 x i8> @test_divconstant_64i8(<64 x i8> %a) nounwind {
 ; AVX512F-LABEL: test_divconstant_64i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
-; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm2, %ymm3
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512F-NEXT:    vpand %ymm2, %ymm3, %ymm3
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm1 = ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11],ymm2[12],ymm0[12],ymm2[13],ymm0[13],ymm2[14],ymm0[14],ymm2[15],ymm0[15],ymm2[24],ymm0[24],ymm2[25],ymm0[25],ymm2[26],ymm0[26],ymm2[27],ymm0[27],ymm2[28],ymm0[28],ymm2[29],ymm0[29],ymm2[30],ymm0[30],ymm2[31],ymm0[31]
+; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm1, %ymm3
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm1 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; AVX512F-NEXT:    vpand %ymm1, %ymm3, %ymm3
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
 ; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm4, %ymm4
-; AVX512F-NEXT:    vpand %ymm2, %ymm4, %ymm4
+; AVX512F-NEXT:    vpand %ymm1, %ymm4, %ymm4
 ; AVX512F-NEXT:    vpackuswb %ymm3, %ymm4, %ymm3
-; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm4
+; AVX512F-NEXT:    vextracti128 $1, %ymm2, %xmm4
 ; AVX512F-NEXT:    vpmovsxbw %xmm4, %ymm4
 ; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm4, %ymm4
 ; AVX512F-NEXT:    vpsrlw $8, %ymm4, %ymm4
-; AVX512F-NEXT:    vpmovsxbw %xmm0, %ymm0
-; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm0, %ymm0
-; AVX512F-NEXT:    vpsrlw $8, %ymm0, %ymm0
-; AVX512F-NEXT:    vpackuswb %ymm4, %ymm0, %ymm0
-; AVX512F-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX512F-NEXT:    vpaddb %ymm3, %ymm0, %ymm0
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX512F-NEXT:    vpmovsxbw %xmm2, %ymm2
+; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm2, %ymm2
+; AVX512F-NEXT:    vpsrlw $8, %ymm2, %ymm2
+; AVX512F-NEXT:    vpackuswb %ymm4, %ymm2, %ymm2
+; AVX512F-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,1,3]
+; AVX512F-NEXT:    vpaddb %ymm3, %ymm2, %ymm2
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
 ; AVX512F-NEXT:    vpsraw $8, %ymm3, %ymm3
 ; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm3, %ymm3
 ; AVX512F-NEXT:    vpsrlw $8, %ymm3, %ymm3
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
 ; AVX512F-NEXT:    vpsraw $8, %ymm4, %ymm4
 ; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm4, %ymm4
 ; AVX512F-NEXT:    vpsrlw $8, %ymm4, %ymm4
 ; AVX512F-NEXT:    vpackuswb %ymm3, %ymm4, %ymm3
-; AVX512F-NEXT:    vpsrlw $7, %ymm0, %ymm0
+; AVX512F-NEXT:    vpsrlw $7, %ymm2, %ymm2
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm4 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; AVX512F-NEXT:    vpand %ymm4, %ymm0, %ymm0
-; AVX512F-NEXT:    vpaddb %ymm0, %ymm3, %ymm0
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31]
+; AVX512F-NEXT:    vpand %ymm4, %ymm2, %ymm2
+; AVX512F-NEXT:    vpaddb %ymm2, %ymm3, %ymm2
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
 ; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm3, %ymm3
-; AVX512F-NEXT:    vpand %ymm2, %ymm3, %ymm3
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
+; AVX512F-NEXT:    vpand %ymm1, %ymm3, %ymm3
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
 ; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm5, %ymm5
-; AVX512F-NEXT:    vpand %ymm2, %ymm5, %ymm2
-; AVX512F-NEXT:    vpackuswb %ymm3, %ymm2, %ymm2
-; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm3
+; AVX512F-NEXT:    vpand %ymm1, %ymm5, %ymm1
+; AVX512F-NEXT:    vpackuswb %ymm3, %ymm1, %ymm1
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm3
 ; AVX512F-NEXT:    vpmovsxbw %xmm3, %ymm3
 ; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm3, %ymm3
 ; AVX512F-NEXT:    vpsrlw $8, %ymm3, %ymm3
-; AVX512F-NEXT:    vpmovsxbw %xmm1, %ymm1
+; AVX512F-NEXT:    vpmovsxbw %xmm0, %ymm0
+; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm0, %ymm0
+; AVX512F-NEXT:    vpsrlw $8, %ymm0, %ymm0
+; AVX512F-NEXT:    vpackuswb %ymm3, %ymm0, %ymm0
+; AVX512F-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX512F-NEXT:    vpaddb %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm1 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX512F-NEXT:    vpsraw $8, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm1, %ymm1
 ; AVX512F-NEXT:    vpsrlw $8, %ymm1, %ymm1
-; AVX512F-NEXT:    vpackuswb %ymm3, %ymm1, %ymm1
-; AVX512F-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
-; AVX512F-NEXT:    vpaddb %ymm2, %ymm1, %ymm1
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
-; AVX512F-NEXT:    vpsraw $8, %ymm2, %ymm2
-; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm2, %ymm2
-; AVX512F-NEXT:    vpsrlw $8, %ymm2, %ymm2
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
 ; AVX512F-NEXT:    vpsraw $8, %ymm3, %ymm3
 ; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm3, %ymm3
 ; AVX512F-NEXT:    vpsrlw $8, %ymm3, %ymm3
-; AVX512F-NEXT:    vpackuswb %ymm2, %ymm3, %ymm2
-; AVX512F-NEXT:    vpsrlw $7, %ymm1, %ymm1
-; AVX512F-NEXT:    vpand %ymm4, %ymm1, %ymm1
-; AVX512F-NEXT:    vpaddb %ymm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vpackuswb %ymm1, %ymm3, %ymm1
+; AVX512F-NEXT:    vpsrlw $7, %ymm0, %ymm0
+; AVX512F-NEXT:    vpand %ymm4, %ymm0, %ymm0
+; AVX512F-NEXT:    vpaddb %ymm0, %ymm1, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: test_divconstant_64i8:
@@ -435,20 +441,22 @@ define <16 x i32> @test_rem7_16i32(<16 x i32> %a) nounwind {
 define <32 x i16> @test_rem7_32i16(<32 x i16> %a) nounwind {
 ; AVX512F-LABEL: test_rem7_32i16:
 ; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm2 = [18725,18725,18725,18725,18725,18725,18725,18725,18725,18725,18725,18725,18725,18725,18725,18725]
-; AVX512F-NEXT:    vpmulhw %ymm2, %ymm0, %ymm3
+; AVX512F-NEXT:    vpmulhw %ymm2, %ymm1, %ymm3
 ; AVX512F-NEXT:    vpsrlw $15, %ymm3, %ymm4
 ; AVX512F-NEXT:    vpsraw $1, %ymm3, %ymm3
 ; AVX512F-NEXT:    vpaddw %ymm4, %ymm3, %ymm3
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm4 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
 ; AVX512F-NEXT:    vpmullw %ymm4, %ymm3, %ymm3
-; AVX512F-NEXT:    vpsubw %ymm3, %ymm0, %ymm0
-; AVX512F-NEXT:    vpmulhw %ymm2, %ymm1, %ymm2
+; AVX512F-NEXT:    vpsubw %ymm3, %ymm1, %ymm1
+; AVX512F-NEXT:    vpmulhw %ymm2, %ymm0, %ymm2
 ; AVX512F-NEXT:    vpsrlw $15, %ymm2, %ymm3
 ; AVX512F-NEXT:    vpsraw $1, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpaddw %ymm3, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpmullw %ymm4, %ymm2, %ymm2
-; AVX512F-NEXT:    vpsubw %ymm2, %ymm1, %ymm1
+; AVX512F-NEXT:    vpsubw %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: test_rem7_32i16:
@@ -467,17 +475,18 @@ define <32 x i16> @test_rem7_32i16(<32 x i16> %a) nounwind {
 define <64 x i8> @test_rem7_64i8(<64 x i8> %a) nounwind {
 ; AVX512F-LABEL: test_rem7_64i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
+; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm2
 ; AVX512F-NEXT:    vpmovsxbw %xmm2, %ymm2
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm3 = [65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427]
 ; AVX512F-NEXT:    vpmullw %ymm3, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpsrlw $8, %ymm2, %ymm2
-; AVX512F-NEXT:    vpmovsxbw %xmm0, %ymm4
+; AVX512F-NEXT:    vpmovsxbw %xmm1, %ymm4
 ; AVX512F-NEXT:    vpmullw %ymm3, %ymm4, %ymm4
 ; AVX512F-NEXT:    vpsrlw $8, %ymm4, %ymm4
 ; AVX512F-NEXT:    vpackuswb %ymm2, %ymm4, %ymm2
 ; AVX512F-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,1,3]
-; AVX512F-NEXT:    vpaddb %ymm0, %ymm2, %ymm2
+; AVX512F-NEXT:    vpaddb %ymm1, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpsrlw $7, %ymm2, %ymm4
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm5 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
 ; AVX512F-NEXT:    vpand %ymm5, %ymm4, %ymm4
@@ -490,19 +499,19 @@ define <64 x i8> @test_rem7_64i8(<64 x i8> %a) nounwind {
 ; AVX512F-NEXT:    vpsubb %ymm7, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpsllw $3, %ymm2, %ymm4
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm8 = [248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248]
-; AVX512F-NEXT:    vpand %ymm8, %ymm4, %ymm4
+; AVX512F-NEXT:    vpand %ymm4, %ymm8, %ymm4
 ; AVX512F-NEXT:    vpsubb %ymm4, %ymm2, %ymm2
-; AVX512F-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
-; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm2
+; AVX512F-NEXT:    vpaddb %ymm2, %ymm1, %ymm1
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm2
 ; AVX512F-NEXT:    vpmovsxbw %xmm2, %ymm2
 ; AVX512F-NEXT:    vpmullw %ymm3, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpsrlw $8, %ymm2, %ymm2
-; AVX512F-NEXT:    vpmovsxbw %xmm1, %ymm4
+; AVX512F-NEXT:    vpmovsxbw %xmm0, %ymm4
 ; AVX512F-NEXT:    vpmullw %ymm3, %ymm4, %ymm3
 ; AVX512F-NEXT:    vpsrlw $8, %ymm3, %ymm3
 ; AVX512F-NEXT:    vpackuswb %ymm2, %ymm3, %ymm2
 ; AVX512F-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,1,3]
-; AVX512F-NEXT:    vpaddb %ymm1, %ymm2, %ymm2
+; AVX512F-NEXT:    vpaddb %ymm0, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpsrlw $7, %ymm2, %ymm3
 ; AVX512F-NEXT:    vpand %ymm5, %ymm3, %ymm3
 ; AVX512F-NEXT:    vpsrlw $2, %ymm2, %ymm2
@@ -511,9 +520,10 @@ define <64 x i8> @test_rem7_64i8(<64 x i8> %a) nounwind {
 ; AVX512F-NEXT:    vpaddb %ymm3, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpsubb %ymm7, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpsllw $3, %ymm2, %ymm3
-; AVX512F-NEXT:    vpand %ymm8, %ymm3, %ymm3
+; AVX512F-NEXT:    vpand %ymm3, %ymm8, %ymm3
 ; AVX512F-NEXT:    vpsubb %ymm3, %ymm2, %ymm2
-; AVX512F-NEXT:    vpaddb %ymm2, %ymm1, %ymm1
+; AVX512F-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: test_rem7_64i8:
@@ -554,19 +564,20 @@ define <64 x i8> @test_rem7_64i8(<64 x i8> %a) nounwind {
 define <64 x i8> @test_remconstant_64i8(<64 x i8> %a) nounwind {
 ; AVX512F-LABEL: test_remconstant_64i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
-; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm2, %ymm3
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512F-NEXT:    vpand %ymm2, %ymm3, %ymm3
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm1 = ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11],ymm2[12],ymm0[12],ymm2[13],ymm0[13],ymm2[14],ymm0[14],ymm2[15],ymm0[15],ymm2[24],ymm0[24],ymm2[25],ymm0[25],ymm2[26],ymm0[26],ymm2[27],ymm0[27],ymm2[28],ymm0[28],ymm2[29],ymm0[29],ymm2[30],ymm0[30],ymm2[31],ymm0[31]
+; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm1, %ymm3
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm1 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; AVX512F-NEXT:    vpand %ymm1, %ymm3, %ymm3
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
 ; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm4, %ymm4
-; AVX512F-NEXT:    vpand %ymm2, %ymm4, %ymm4
+; AVX512F-NEXT:    vpand %ymm1, %ymm4, %ymm4
 ; AVX512F-NEXT:    vpackuswb %ymm3, %ymm4, %ymm3
-; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm4
+; AVX512F-NEXT:    vextracti128 $1, %ymm2, %xmm4
 ; AVX512F-NEXT:    vpmovsxbw %xmm4, %ymm4
 ; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm4, %ymm4
 ; AVX512F-NEXT:    vpsrlw $8, %ymm4, %ymm4
-; AVX512F-NEXT:    vpmovsxbw %xmm0, %ymm5
+; AVX512F-NEXT:    vpmovsxbw %xmm2, %ymm5
 ; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm5, %ymm5
 ; AVX512F-NEXT:    vpsrlw $8, %ymm5, %ymm5
 ; AVX512F-NEXT:    vpackuswb %ymm4, %ymm5, %ymm4
@@ -587,24 +598,24 @@ define <64 x i8> @test_remconstant_64i8(<64 x i8> %a) nounwind {
 ; AVX512F-NEXT:    vpaddb %ymm5, %ymm4, %ymm4
 ; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm4[8],ymm0[8],ymm4[9],ymm0[9],ymm4[10],ymm0[10],ymm4[11],ymm0[11],ymm4[12],ymm0[12],ymm4[13],ymm0[13],ymm4[14],ymm0[14],ymm4[15],ymm0[15],ymm4[24],ymm0[24],ymm4[25],ymm0[25],ymm4[26],ymm0[26],ymm4[27],ymm0[27],ymm4[28],ymm0[28],ymm4[29],ymm0[29],ymm4[30],ymm0[30],ymm4[31],ymm0[31]
 ; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm5, %ymm5
-; AVX512F-NEXT:    vpand %ymm2, %ymm5, %ymm5
+; AVX512F-NEXT:    vpand %ymm1, %ymm5, %ymm5
 ; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm4[0],ymm0[0],ymm4[1],ymm0[1],ymm4[2],ymm0[2],ymm4[3],ymm0[3],ymm4[4],ymm0[4],ymm4[5],ymm0[5],ymm4[6],ymm0[6],ymm4[7],ymm0[7],ymm4[16],ymm0[16],ymm4[17],ymm0[17],ymm4[18],ymm0[18],ymm4[19],ymm0[19],ymm4[20],ymm0[20],ymm4[21],ymm0[21],ymm4[22],ymm0[22],ymm4[23],ymm0[23]
 ; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm4, %ymm4
-; AVX512F-NEXT:    vpand %ymm2, %ymm4, %ymm4
+; AVX512F-NEXT:    vpand %ymm1, %ymm4, %ymm4
 ; AVX512F-NEXT:    vpackuswb %ymm5, %ymm4, %ymm4
-; AVX512F-NEXT:    vpsubb %ymm4, %ymm0, %ymm0
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31]
+; AVX512F-NEXT:    vpsubb %ymm4, %ymm2, %ymm2
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
 ; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm4, %ymm4
-; AVX512F-NEXT:    vpand %ymm2, %ymm4, %ymm4
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
+; AVX512F-NEXT:    vpand %ymm1, %ymm4, %ymm4
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
 ; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm5, %ymm5
-; AVX512F-NEXT:    vpand %ymm2, %ymm5, %ymm5
+; AVX512F-NEXT:    vpand %ymm1, %ymm5, %ymm5
 ; AVX512F-NEXT:    vpackuswb %ymm4, %ymm5, %ymm4
-; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm5
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm5
 ; AVX512F-NEXT:    vpmovsxbw %xmm5, %ymm5
 ; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm5, %ymm5
 ; AVX512F-NEXT:    vpsrlw $8, %ymm5, %ymm5
-; AVX512F-NEXT:    vpmovsxbw %xmm1, %ymm6
+; AVX512F-NEXT:    vpmovsxbw %xmm0, %ymm6
 ; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm6, %ymm6
 ; AVX512F-NEXT:    vpsrlw $8, %ymm6, %ymm6
 ; AVX512F-NEXT:    vpackuswb %ymm5, %ymm6, %ymm5
@@ -624,12 +635,13 @@ define <64 x i8> @test_remconstant_64i8(<64 x i8> %a) nounwind {
 ; AVX512F-NEXT:    vpaddb %ymm3, %ymm5, %ymm3
 ; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm3[8],ymm0[8],ymm3[9],ymm0[9],ymm3[10],ymm0[10],ymm3[11],ymm0[11],ymm3[12],ymm0[12],ymm3[13],ymm0[13],ymm3[14],ymm0[14],ymm3[15],ymm0[15],ymm3[24],ymm0[24],ymm3[25],ymm0[25],ymm3[26],ymm0[26],ymm3[27],ymm0[27],ymm3[28],ymm0[28],ymm3[29],ymm0[29],ymm3[30],ymm0[30],ymm3[31],ymm0[31]
 ; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm4, %ymm4
-; AVX512F-NEXT:    vpand %ymm2, %ymm4, %ymm4
+; AVX512F-NEXT:    vpand %ymm1, %ymm4, %ymm4
 ; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23]
 ; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm3, %ymm3
-; AVX512F-NEXT:    vpand %ymm2, %ymm3, %ymm2
-; AVX512F-NEXT:    vpackuswb %ymm4, %ymm2, %ymm2
-; AVX512F-NEXT:    vpsubb %ymm2, %ymm1, %ymm1
+; AVX512F-NEXT:    vpand %ymm1, %ymm3, %ymm1
+; AVX512F-NEXT:    vpackuswb %ymm4, %ymm1, %ymm1
+; AVX512F-NEXT:    vpsubb %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: test_remconstant_64i8:
diff --git a/llvm/test/CodeGen/X86/vector-idiv-udiv-128.ll b/llvm/test/CodeGen/X86/vector-idiv-udiv-128.ll
index e599ceea7..47b7d157d 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-udiv-128.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-udiv-128.ll
@@ -309,36 +309,35 @@ define <16 x i8> @test_divconstant_16i8(<16 x i8> %a) nounwind {
 ; SSE41-NEXT:    psllw $7, %xmm3
 ; SSE41-NEXT:    pblendw {{.*#+}} xmm3 = xmm3[0],xmm2[1,2,3,4,5,6,7]
 ; SSE41-NEXT:    psrlw $8, %xmm3
-; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [147,79,171,117,205,57,57,37]
-; SSE41-NEXT:    pmullw %xmm3, %xmm2
-; SSE41-NEXT:    psrlw $8, %xmm2
-; SSE41-NEXT:    pxor %xmm4, %xmm4
-; SSE41-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3],xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm5 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; SSE41-NEXT:    psllw $7, %xmm5
-; SSE41-NEXT:    pblendw {{.*#+}} xmm5 = xmm4[0,1,2,3,4,5,6],xmm5[7]
-; SSE41-NEXT:    psrlw $8, %xmm5
-; SSE41-NEXT:    packuswb %xmm3, %xmm5
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm3 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
 ; SSE41-NEXT:    pmullw {{.*}}(%rip), %xmm3
 ; SSE41-NEXT:    psrlw $8, %xmm3
-; SSE41-NEXT:    packuswb %xmm2, %xmm3
-; SSE41-NEXT:    psubb %xmm3, %xmm0
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; SSE41-NEXT:    pxor %xmm2, %xmm2
+; SSE41-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm4 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; SSE41-NEXT:    psllw $7, %xmm4
+; SSE41-NEXT:    pblendw {{.*#+}} xmm4 = xmm2[0,1,2,3,4,5,6],xmm4[7]
+; SSE41-NEXT:    psrlw $8, %xmm4
+; SSE41-NEXT:    packuswb %xmm0, %xmm4
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm2 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero
+; SSE41-NEXT:    pmullw {{.*}}(%rip), %xmm2
+; SSE41-NEXT:    psrlw $8, %xmm2
+; SSE41-NEXT:    packuswb %xmm3, %xmm2
+; SSE41-NEXT:    psubb %xmm2, %xmm0
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
 ; SSE41-NEXT:    pmullw {{.*}}(%rip), %xmm0
 ; SSE41-NEXT:    psrlw $8, %xmm0
-; SSE41-NEXT:    pmullw {{.*}}(%rip), %xmm2
-; SSE41-NEXT:    psrlw $8, %xmm2
-; SSE41-NEXT:    packuswb %xmm0, %xmm2
-; SSE41-NEXT:    paddb %xmm3, %xmm2
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
-; SSE41-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
-; SSE41-NEXT:    pmullw {{.*}}(%rip), %xmm2
-; SSE41-NEXT:    psrlw $8, %xmm2
+; SSE41-NEXT:    pmullw {{.*}}(%rip), %xmm3
+; SSE41-NEXT:    psrlw $8, %xmm3
+; SSE41-NEXT:    packuswb %xmm0, %xmm3
+; SSE41-NEXT:    paddb %xmm2, %xmm3
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
+; SSE41-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm1[8],xmm3[9],xmm1[9],xmm3[10],xmm1[10],xmm3[11],xmm1[11],xmm3[12],xmm1[12],xmm3[13],xmm1[13],xmm3[14],xmm1[14],xmm3[15],xmm1[15]
+; SSE41-NEXT:    pmullw {{.*}}(%rip), %xmm3
+; SSE41-NEXT:    psrlw $8, %xmm3
 ; SSE41-NEXT:    pmullw {{.*}}(%rip), %xmm0
 ; SSE41-NEXT:    psrlw $8, %xmm0
-; SSE41-NEXT:    packuswb %xmm2, %xmm0
+; SSE41-NEXT:    packuswb %xmm3, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: test_divconstant_16i8:
@@ -406,7 +405,6 @@ define <16 x i8> @test_divconstant_16i8(<16 x i8> %a) nounwind {
 ; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0]
 ; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
 ; AVX512BW-NEXT:    vpsrlvw %zmm1, %zmm2, %zmm1
-; AVX512BW-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
 ; AVX512BW-NEXT:    vpmullw {{.*}}(%rip), %ymm1, %ymm1
 ; AVX512BW-NEXT:    vpsrlw $8, %ymm1, %ymm1
 ; AVX512BW-NEXT:    vpmovwb %zmm1, %ymm1
@@ -810,30 +808,29 @@ define <16 x i8> @test_remconstant_16i8(<16 x i8> %a) nounwind {
 ; SSE41-NEXT:    psllw $7, %xmm3
 ; SSE41-NEXT:    pblendw {{.*#+}} xmm3 = xmm3[0],xmm2[1,2,3,4,5,6,7]
 ; SSE41-NEXT:    psrlw $8, %xmm3
-; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [147,79,171,117,205,57,57,37]
-; SSE41-NEXT:    pmullw %xmm3, %xmm2
-; SSE41-NEXT:    psrlw $8, %xmm2
-; SSE41-NEXT:    pxor %xmm4, %xmm4
-; SSE41-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3],xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm5 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; SSE41-NEXT:    psllw $7, %xmm5
-; SSE41-NEXT:    pblendw {{.*#+}} xmm5 = xmm4[0,1,2,3,4,5,6],xmm5[7]
-; SSE41-NEXT:    psrlw $8, %xmm5
-; SSE41-NEXT:    packuswb %xmm3, %xmm5
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm3 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
 ; SSE41-NEXT:    pmullw {{.*}}(%rip), %xmm3
 ; SSE41-NEXT:    psrlw $8, %xmm3
-; SSE41-NEXT:    packuswb %xmm2, %xmm3
-; SSE41-NEXT:    movdqa %xmm0, %xmm4
-; SSE41-NEXT:    psubb %xmm3, %xmm4
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm2 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero
-; SSE41-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm1[8],xmm4[9],xmm1[9],xmm4[10],xmm1[10],xmm4[11],xmm1[11],xmm4[12],xmm1[12],xmm4[13],xmm1[13],xmm4[14],xmm1[14],xmm4[15],xmm1[15]
+; SSE41-NEXT:    pxor %xmm2, %xmm2
+; SSE41-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm4 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; SSE41-NEXT:    psllw $7, %xmm4
+; SSE41-NEXT:    pblendw {{.*#+}} xmm4 = xmm2[0,1,2,3,4,5,6],xmm4[7]
+; SSE41-NEXT:    psrlw $8, %xmm4
+; SSE41-NEXT:    packuswb %xmm0, %xmm4
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero
 ; SSE41-NEXT:    pmullw {{.*}}(%rip), %xmm4
 ; SSE41-NEXT:    psrlw $8, %xmm4
+; SSE41-NEXT:    packuswb %xmm3, %xmm4
+; SSE41-NEXT:    movdqa %xmm0, %xmm3
+; SSE41-NEXT:    psubb %xmm4, %xmm3
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm2 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
+; SSE41-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm1[8],xmm3[9],xmm1[9],xmm3[10],xmm1[10],xmm3[11],xmm1[11],xmm3[12],xmm1[12],xmm3[13],xmm1[13],xmm3[14],xmm1[14],xmm3[15],xmm1[15]
+; SSE41-NEXT:    pmullw {{.*}}(%rip), %xmm3
+; SSE41-NEXT:    psrlw $8, %xmm3
 ; SSE41-NEXT:    pmullw {{.*}}(%rip), %xmm2
 ; SSE41-NEXT:    psrlw $8, %xmm2
-; SSE41-NEXT:    packuswb %xmm4, %xmm2
-; SSE41-NEXT:    paddb %xmm3, %xmm2
+; SSE41-NEXT:    packuswb %xmm3, %xmm2
+; SSE41-NEXT:    paddb %xmm4, %xmm2
 ; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
 ; SSE41-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
 ; SSE41-NEXT:    pmullw {{.*}}(%rip), %xmm2
@@ -925,7 +922,6 @@ define <16 x i8> @test_remconstant_16i8(<16 x i8> %a) nounwind {
 ; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0]
 ; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
 ; AVX512BW-NEXT:    vpsrlvw %zmm1, %zmm2, %zmm1
-; AVX512BW-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
 ; AVX512BW-NEXT:    vpmullw {{.*}}(%rip), %ymm1, %ymm1
 ; AVX512BW-NEXT:    vpsrlw $8, %ymm1, %ymm1
 ; AVX512BW-NEXT:    vpmovwb %zmm1, %ymm1
diff --git a/llvm/test/CodeGen/X86/vector-idiv-udiv-256.ll b/llvm/test/CodeGen/X86/vector-idiv-udiv-256.ll
index 198c6de8b..155990c3d 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-udiv-256.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-udiv-256.ll
@@ -345,7 +345,6 @@ define <32 x i8> @test_divconstant_32i8(<32 x i8> %a) nounwind {
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
 ; AVX512BW-NEXT:    vpsrlvw {{.*}}(%rip), %zmm1, %zmm1
-; AVX512BW-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
 ; AVX512BW-NEXT:    vpmullw {{.*}}(%rip), %zmm1, %zmm1
 ; AVX512BW-NEXT:    vpsrlw $8, %zmm1, %zmm1
 ; AVX512BW-NEXT:    vpmovwb %zmm1, %ymm1
@@ -793,7 +792,6 @@ define <32 x i8> @test_remconstant_32i8(<32 x i8> %a) nounwind {
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
 ; AVX512BW-NEXT:    vpsrlvw {{.*}}(%rip), %zmm1, %zmm1
-; AVX512BW-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
 ; AVX512BW-NEXT:    vpmullw {{.*}}(%rip), %zmm1, %zmm1
 ; AVX512BW-NEXT:    vpsrlw $8, %zmm1, %zmm1
 ; AVX512BW-NEXT:    vpmovwb %zmm1, %ymm1
diff --git a/llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll b/llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll
index 495d35a0c..c750dcb77 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll
@@ -103,17 +103,19 @@ define <16 x i32> @test_div7_16i32(<16 x i32> %a) nounwind {
 define <32 x i16> @test_div7_32i16(<32 x i16> %a) nounwind {
 ; AVX512F-LABEL: test_div7_32i16:
 ; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm2 = [9363,9363,9363,9363,9363,9363,9363,9363,9363,9363,9363,9363,9363,9363,9363,9363]
-; AVX512F-NEXT:    vpmulhuw %ymm2, %ymm0, %ymm3
-; AVX512F-NEXT:    vpsubw %ymm3, %ymm0, %ymm0
-; AVX512F-NEXT:    vpsrlw $1, %ymm0, %ymm0
-; AVX512F-NEXT:    vpaddw %ymm3, %ymm0, %ymm0
-; AVX512F-NEXT:    vpsrlw $2, %ymm0, %ymm0
-; AVX512F-NEXT:    vpmulhuw %ymm2, %ymm1, %ymm2
-; AVX512F-NEXT:    vpsubw %ymm2, %ymm1, %ymm1
+; AVX512F-NEXT:    vpmulhuw %ymm2, %ymm1, %ymm3
+; AVX512F-NEXT:    vpsubw %ymm3, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpsrlw $1, %ymm1, %ymm1
-; AVX512F-NEXT:    vpaddw %ymm2, %ymm1, %ymm1
+; AVX512F-NEXT:    vpaddw %ymm3, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpsrlw $2, %ymm1, %ymm1
+; AVX512F-NEXT:    vpmulhuw %ymm2, %ymm0, %ymm2
+; AVX512F-NEXT:    vpsubw %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vpsrlw $1, %ymm0, %ymm0
+; AVX512F-NEXT:    vpaddw %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vpsrlw $2, %ymm0, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: test_div7_32i16:
@@ -131,36 +133,38 @@ define <32 x i16> @test_div7_32i16(<32 x i16> %a) nounwind {
 define <64 x i8> @test_div7_64i8(<64 x i8> %a) nounwind {
 ; AVX512F-LABEL: test_div7_64i8:
 ; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm0[8],ymm2[8],ymm0[9],ymm2[9],ymm0[10],ymm2[10],ymm0[11],ymm2[11],ymm0[12],ymm2[12],ymm0[13],ymm2[13],ymm0[14],ymm2[14],ymm0[15],ymm2[15],ymm0[24],ymm2[24],ymm0[25],ymm2[25],ymm0[26],ymm2[26],ymm0[27],ymm2[27],ymm0[28],ymm2[28],ymm0[29],ymm2[29],ymm0[30],ymm2[30],ymm0[31],ymm2[31]
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15],ymm1[24],ymm2[24],ymm1[25],ymm2[25],ymm1[26],ymm2[26],ymm1[27],ymm2[27],ymm1[28],ymm2[28],ymm1[29],ymm2[29],ymm1[30],ymm2[30],ymm1[31],ymm2[31]
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm4 = [37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37]
 ; AVX512F-NEXT:    vpmullw %ymm4, %ymm3, %ymm3
 ; AVX512F-NEXT:    vpsrlw $8, %ymm3, %ymm3
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm0[0],ymm2[0],ymm0[1],ymm2[1],ymm0[2],ymm2[2],ymm0[3],ymm2[3],ymm0[4],ymm2[4],ymm0[5],ymm2[5],ymm0[6],ymm2[6],ymm0[7],ymm2[7],ymm0[16],ymm2[16],ymm0[17],ymm2[17],ymm0[18],ymm2[18],ymm0[19],ymm2[19],ymm0[20],ymm2[20],ymm0[21],ymm2[21],ymm0[22],ymm2[22],ymm0[23],ymm2[23]
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[16],ymm2[16],ymm1[17],ymm2[17],ymm1[18],ymm2[18],ymm1[19],ymm2[19],ymm1[20],ymm2[20],ymm1[21],ymm2[21],ymm1[22],ymm2[22],ymm1[23],ymm2[23]
 ; AVX512F-NEXT:    vpmullw %ymm4, %ymm5, %ymm5
 ; AVX512F-NEXT:    vpsrlw $8, %ymm5, %ymm5
 ; AVX512F-NEXT:    vpackuswb %ymm3, %ymm5, %ymm3
-; AVX512F-NEXT:    vpsubb %ymm3, %ymm0, %ymm0
-; AVX512F-NEXT:    vpsrlw $1, %ymm0, %ymm0
+; AVX512F-NEXT:    vpsubb %ymm3, %ymm1, %ymm1
+; AVX512F-NEXT:    vpsrlw $1, %ymm1, %ymm1
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm5 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; AVX512F-NEXT:    vpand %ymm5, %ymm0, %ymm0
-; AVX512F-NEXT:    vpaddb %ymm3, %ymm0, %ymm0
-; AVX512F-NEXT:    vpsrlw $2, %ymm0, %ymm0
+; AVX512F-NEXT:    vpand %ymm5, %ymm1, %ymm1
+; AVX512F-NEXT:    vpaddb %ymm3, %ymm1, %ymm1
+; AVX512F-NEXT:    vpsrlw $2, %ymm1, %ymm1
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm3 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
-; AVX512F-NEXT:    vpand %ymm3, %ymm0, %ymm0
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm6 = ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15],ymm1[24],ymm2[24],ymm1[25],ymm2[25],ymm1[26],ymm2[26],ymm1[27],ymm2[27],ymm1[28],ymm2[28],ymm1[29],ymm2[29],ymm1[30],ymm2[30],ymm1[31],ymm2[31]
+; AVX512F-NEXT:    vpand %ymm3, %ymm1, %ymm1
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm6 = ymm0[8],ymm2[8],ymm0[9],ymm2[9],ymm0[10],ymm2[10],ymm0[11],ymm2[11],ymm0[12],ymm2[12],ymm0[13],ymm2[13],ymm0[14],ymm2[14],ymm0[15],ymm2[15],ymm0[24],ymm2[24],ymm0[25],ymm2[25],ymm0[26],ymm2[26],ymm0[27],ymm2[27],ymm0[28],ymm2[28],ymm0[29],ymm2[29],ymm0[30],ymm2[30],ymm0[31],ymm2[31]
 ; AVX512F-NEXT:    vpmullw %ymm4, %ymm6, %ymm6
 ; AVX512F-NEXT:    vpsrlw $8, %ymm6, %ymm6
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[16],ymm2[16],ymm1[17],ymm2[17],ymm1[18],ymm2[18],ymm1[19],ymm2[19],ymm1[20],ymm2[20],ymm1[21],ymm2[21],ymm1[22],ymm2[22],ymm1[23],ymm2[23]
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm0[0],ymm2[0],ymm0[1],ymm2[1],ymm0[2],ymm2[2],ymm0[3],ymm2[3],ymm0[4],ymm2[4],ymm0[5],ymm2[5],ymm0[6],ymm2[6],ymm0[7],ymm2[7],ymm0[16],ymm2[16],ymm0[17],ymm2[17],ymm0[18],ymm2[18],ymm0[19],ymm2[19],ymm0[20],ymm2[20],ymm0[21],ymm2[21],ymm0[22],ymm2[22],ymm0[23],ymm2[23]
 ; AVX512F-NEXT:    vpmullw %ymm4, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpsrlw $8, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpackuswb %ymm6, %ymm2, %ymm2
-; AVX512F-NEXT:    vpsubb %ymm2, %ymm1, %ymm1
-; AVX512F-NEXT:    vpsrlw $1, %ymm1, %ymm1
-; AVX512F-NEXT:    vpand %ymm5, %ymm1, %ymm1
-; AVX512F-NEXT:    vpaddb %ymm2, %ymm1, %ymm1
-; AVX512F-NEXT:    vpsrlw $2, %ymm1, %ymm1
-; AVX512F-NEXT:    vpand %ymm3, %ymm1, %ymm1
+; AVX512F-NEXT:    vpsubb %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vpsrlw $1, %ymm0, %ymm0
+; AVX512F-NEXT:    vpand %ymm5, %ymm0, %ymm0
+; AVX512F-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vpsrlw $2, %ymm0, %ymm0
+; AVX512F-NEXT:    vpand %ymm3, %ymm0, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: test_div7_64i8:
@@ -192,61 +196,63 @@ define <64 x i8> @test_div7_64i8(<64 x i8> %a) nounwind {
 define <64 x i8> @test_divconstant_64i8(<64 x i8> %a) nounwind {
 ; AVX512F-LABEL: test_divconstant_64i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm0[8],ymm2[8],ymm0[9],ymm2[9],ymm0[10],ymm2[10],ymm0[11],ymm2[11],ymm0[12],ymm2[12],ymm0[13],ymm2[13],ymm0[14],ymm2[14],ymm0[15],ymm2[15],ymm0[24],ymm2[24],ymm0[25],ymm2[25],ymm0[26],ymm2[26],ymm0[27],ymm2[27],ymm0[28],ymm2[28],ymm0[29],ymm2[29],ymm0[30],ymm2[30],ymm0[31],ymm2[31]
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm2[8],ymm1[8],ymm2[9],ymm1[9],ymm2[10],ymm1[10],ymm2[11],ymm1[11],ymm2[12],ymm1[12],ymm2[13],ymm1[13],ymm2[14],ymm1[14],ymm2[15],ymm1[15],ymm2[24],ymm1[24],ymm2[25],ymm1[25],ymm2[26],ymm1[26],ymm2[27],ymm1[27],ymm2[28],ymm1[28],ymm2[29],ymm1[29],ymm2[30],ymm1[30],ymm2[31],ymm1[31]
 ; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm3, %ymm3
 ; AVX512F-NEXT:    vpsrlw $8, %ymm3, %ymm3
 ; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm3, %ymm3
 ; AVX512F-NEXT:    vpsrlw $8, %ymm3, %ymm3
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm0[0],ymm2[0],ymm0[1],ymm2[1],ymm0[2],ymm2[2],ymm0[3],ymm2[3],ymm0[4],ymm2[4],ymm0[5],ymm2[5],ymm0[6],ymm2[6],ymm0[7],ymm2[7],ymm0[16],ymm2[16],ymm0[17],ymm2[17],ymm0[18],ymm2[18],ymm0[19],ymm2[19],ymm0[20],ymm2[20],ymm0[21],ymm2[21],ymm0[22],ymm2[22],ymm0[23],ymm2[23]
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm2[0],ymm1[0],ymm2[1],ymm1[1],ymm2[2],ymm1[2],ymm2[3],ymm1[3],ymm2[4],ymm1[4],ymm2[5],ymm1[5],ymm2[6],ymm1[6],ymm2[7],ymm1[7],ymm2[16],ymm1[16],ymm2[17],ymm1[17],ymm2[18],ymm1[18],ymm2[19],ymm1[19],ymm2[20],ymm1[20],ymm2[21],ymm1[21],ymm2[22],ymm1[22],ymm2[23],ymm1[23]
 ; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm4, %ymm4
 ; AVX512F-NEXT:    vpsrlw $8, %ymm4, %ymm4
 ; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm4, %ymm4
 ; AVX512F-NEXT:    vpsrlw $8, %ymm4, %ymm4
 ; AVX512F-NEXT:    vpackuswb %ymm3, %ymm4, %ymm3
-; AVX512F-NEXT:    vpsubb %ymm3, %ymm0, %ymm0
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm0[8],ymm2[8],ymm0[9],ymm2[9],ymm0[10],ymm2[10],ymm0[11],ymm2[11],ymm0[12],ymm2[12],ymm0[13],ymm2[13],ymm0[14],ymm2[14],ymm0[15],ymm2[15],ymm0[24],ymm2[24],ymm0[25],ymm2[25],ymm0[26],ymm2[26],ymm0[27],ymm2[27],ymm0[28],ymm2[28],ymm0[29],ymm2[29],ymm0[30],ymm2[30],ymm0[31],ymm2[31]
+; AVX512F-NEXT:    vpsubb %ymm3, %ymm2, %ymm2
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm2[8],ymm1[8],ymm2[9],ymm1[9],ymm2[10],ymm1[10],ymm2[11],ymm1[11],ymm2[12],ymm1[12],ymm2[13],ymm1[13],ymm2[14],ymm1[14],ymm2[15],ymm1[15],ymm2[24],ymm1[24],ymm2[25],ymm1[25],ymm2[26],ymm1[26],ymm2[27],ymm1[27],ymm2[28],ymm1[28],ymm2[29],ymm1[29],ymm2[30],ymm1[30],ymm2[31],ymm1[31]
 ; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm4, %ymm4
 ; AVX512F-NEXT:    vpsrlw $8, %ymm4, %ymm4
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0],ymm2[0],ymm0[1],ymm2[1],ymm0[2],ymm2[2],ymm0[3],ymm2[3],ymm0[4],ymm2[4],ymm0[5],ymm2[5],ymm0[6],ymm2[6],ymm0[7],ymm2[7],ymm0[16],ymm2[16],ymm0[17],ymm2[17],ymm0[18],ymm2[18],ymm0[19],ymm2[19],ymm0[20],ymm2[20],ymm0[21],ymm2[21],ymm0[22],ymm2[22],ymm0[23],ymm2[23]
-; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm0, %ymm0
-; AVX512F-NEXT:    vpsrlw $8, %ymm0, %ymm0
-; AVX512F-NEXT:    vpackuswb %ymm4, %ymm0, %ymm0
-; AVX512F-NEXT:    vpaddb %ymm3, %ymm0, %ymm0
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm0[8],ymm2[8],ymm0[9],ymm2[9],ymm0[10],ymm2[10],ymm0[11],ymm2[11],ymm0[12],ymm2[12],ymm0[13],ymm2[13],ymm0[14],ymm2[14],ymm0[15],ymm2[15],ymm0[24],ymm2[24],ymm0[25],ymm2[25],ymm0[26],ymm2[26],ymm0[27],ymm2[27],ymm0[28],ymm2[28],ymm0[29],ymm2[29],ymm0[30],ymm2[30],ymm0[31],ymm2[31]
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm1[0],ymm2[1],ymm1[1],ymm2[2],ymm1[2],ymm2[3],ymm1[3],ymm2[4],ymm1[4],ymm2[5],ymm1[5],ymm2[6],ymm1[6],ymm2[7],ymm1[7],ymm2[16],ymm1[16],ymm2[17],ymm1[17],ymm2[18],ymm1[18],ymm2[19],ymm1[19],ymm2[20],ymm1[20],ymm2[21],ymm1[21],ymm2[22],ymm1[22],ymm2[23],ymm1[23]
+; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm2, %ymm2
+; AVX512F-NEXT:    vpsrlw $8, %ymm2, %ymm2
+; AVX512F-NEXT:    vpackuswb %ymm4, %ymm2, %ymm2
+; AVX512F-NEXT:    vpaddb %ymm3, %ymm2, %ymm2
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm2[8],ymm1[8],ymm2[9],ymm1[9],ymm2[10],ymm1[10],ymm2[11],ymm1[11],ymm2[12],ymm1[12],ymm2[13],ymm1[13],ymm2[14],ymm1[14],ymm2[15],ymm1[15],ymm2[24],ymm1[24],ymm2[25],ymm1[25],ymm2[26],ymm1[26],ymm2[27],ymm1[27],ymm2[28],ymm1[28],ymm2[29],ymm1[29],ymm2[30],ymm1[30],ymm2[31],ymm1[31]
 ; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm3, %ymm3
 ; AVX512F-NEXT:    vpsrlw $8, %ymm3, %ymm3
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0],ymm2[0],ymm0[1],ymm2[1],ymm0[2],ymm2[2],ymm0[3],ymm2[3],ymm0[4],ymm2[4],ymm0[5],ymm2[5],ymm0[6],ymm2[6],ymm0[7],ymm2[7],ymm0[16],ymm2[16],ymm0[17],ymm2[17],ymm0[18],ymm2[18],ymm0[19],ymm2[19],ymm0[20],ymm2[20],ymm0[21],ymm2[21],ymm0[22],ymm2[22],ymm0[23],ymm2[23]
-; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm0, %ymm0
-; AVX512F-NEXT:    vpsrlw $8, %ymm0, %ymm0
-; AVX512F-NEXT:    vpackuswb %ymm3, %ymm0, %ymm0
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15],ymm1[24],ymm2[24],ymm1[25],ymm2[25],ymm1[26],ymm2[26],ymm1[27],ymm2[27],ymm1[28],ymm2[28],ymm1[29],ymm2[29],ymm1[30],ymm2[30],ymm1[31],ymm2[31]
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm1[0],ymm2[1],ymm1[1],ymm2[2],ymm1[2],ymm2[3],ymm1[3],ymm2[4],ymm1[4],ymm2[5],ymm1[5],ymm2[6],ymm1[6],ymm2[7],ymm1[7],ymm2[16],ymm1[16],ymm2[17],ymm1[17],ymm2[18],ymm1[18],ymm2[19],ymm1[19],ymm2[20],ymm1[20],ymm2[21],ymm1[21],ymm2[22],ymm1[22],ymm2[23],ymm1[23]
+; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm2, %ymm2
+; AVX512F-NEXT:    vpsrlw $8, %ymm2, %ymm2
+; AVX512F-NEXT:    vpackuswb %ymm3, %ymm2, %ymm2
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15],ymm0[24],ymm1[24],ymm0[25],ymm1[25],ymm0[26],ymm1[26],ymm0[27],ymm1[27],ymm0[28],ymm1[28],ymm0[29],ymm1[29],ymm0[30],ymm1[30],ymm0[31],ymm1[31]
 ; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm3, %ymm3
 ; AVX512F-NEXT:    vpsrlw $8, %ymm3, %ymm3
 ; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm3, %ymm3
 ; AVX512F-NEXT:    vpsrlw $8, %ymm3, %ymm3
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[16],ymm2[16],ymm1[17],ymm2[17],ymm1[18],ymm2[18],ymm1[19],ymm2[19],ymm1[20],ymm2[20],ymm1[21],ymm2[21],ymm1[22],ymm2[22],ymm1[23],ymm2[23]
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[16],ymm1[16],ymm0[17],ymm1[17],ymm0[18],ymm1[18],ymm0[19],ymm1[19],ymm0[20],ymm1[20],ymm0[21],ymm1[21],ymm0[22],ymm1[22],ymm0[23],ymm1[23]
 ; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm4, %ymm4
 ; AVX512F-NEXT:    vpsrlw $8, %ymm4, %ymm4
 ; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm4, %ymm4
 ; AVX512F-NEXT:    vpsrlw $8, %ymm4, %ymm4
 ; AVX512F-NEXT:    vpackuswb %ymm3, %ymm4, %ymm3
-; AVX512F-NEXT:    vpsubb %ymm3, %ymm1, %ymm1
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15],ymm1[24],ymm2[24],ymm1[25],ymm2[25],ymm1[26],ymm2[26],ymm1[27],ymm2[27],ymm1[28],ymm2[28],ymm1[29],ymm2[29],ymm1[30],ymm2[30],ymm1[31],ymm2[31]
+; AVX512F-NEXT:    vpsubb %ymm3, %ymm0, %ymm0
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15],ymm0[24],ymm1[24],ymm0[25],ymm1[25],ymm0[26],ymm1[26],ymm0[27],ymm1[27],ymm0[28],ymm1[28],ymm0[29],ymm1[29],ymm0[30],ymm1[30],ymm0[31],ymm1[31]
 ; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm4, %ymm4
 ; AVX512F-NEXT:    vpsrlw $8, %ymm4, %ymm4
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[16],ymm2[16],ymm1[17],ymm2[17],ymm1[18],ymm2[18],ymm1[19],ymm2[19],ymm1[20],ymm2[20],ymm1[21],ymm2[21],ymm1[22],ymm2[22],ymm1[23],ymm2[23]
-; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm1, %ymm1
-; AVX512F-NEXT:    vpsrlw $8, %ymm1, %ymm1
-; AVX512F-NEXT:    vpackuswb %ymm4, %ymm1, %ymm1
-; AVX512F-NEXT:    vpaddb %ymm3, %ymm1, %ymm1
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15],ymm1[24],ymm2[24],ymm1[25],ymm2[25],ymm1[26],ymm2[26],ymm1[27],ymm2[27],ymm1[28],ymm2[28],ymm1[29],ymm2[29],ymm1[30],ymm2[30],ymm1[31],ymm2[31]
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[16],ymm1[16],ymm0[17],ymm1[17],ymm0[18],ymm1[18],ymm0[19],ymm1[19],ymm0[20],ymm1[20],ymm0[21],ymm1[21],ymm0[22],ymm1[22],ymm0[23],ymm1[23]
+; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm0, %ymm0
+; AVX512F-NEXT:    vpsrlw $8, %ymm0, %ymm0
+; AVX512F-NEXT:    vpackuswb %ymm4, %ymm0, %ymm0
+; AVX512F-NEXT:    vpaddb %ymm3, %ymm0, %ymm0
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15],ymm0[24],ymm1[24],ymm0[25],ymm1[25],ymm0[26],ymm1[26],ymm0[27],ymm1[27],ymm0[28],ymm1[28],ymm0[29],ymm1[29],ymm0[30],ymm1[30],ymm0[31],ymm1[31]
 ; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm3, %ymm3
 ; AVX512F-NEXT:    vpsrlw $8, %ymm3, %ymm3
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[16],ymm2[16],ymm1[17],ymm2[17],ymm1[18],ymm2[18],ymm1[19],ymm2[19],ymm1[20],ymm2[20],ymm1[21],ymm2[21],ymm1[22],ymm2[22],ymm1[23],ymm2[23]
-; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm1, %ymm1
-; AVX512F-NEXT:    vpsrlw $8, %ymm1, %ymm1
-; AVX512F-NEXT:    vpackuswb %ymm3, %ymm1, %ymm1
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[16],ymm1[16],ymm0[17],ymm1[17],ymm0[18],ymm1[18],ymm0[19],ymm1[19],ymm0[20],ymm1[20],ymm0[21],ymm1[21],ymm0[22],ymm1[22],ymm0[23],ymm1[23]
+; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm0, %ymm0
+; AVX512F-NEXT:    vpsrlw $8, %ymm0, %ymm0
+; AVX512F-NEXT:    vpackuswb %ymm3, %ymm0, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: test_divconstant_64i8:
@@ -433,22 +439,24 @@ define <16 x i32> @test_rem7_16i32(<16 x i32> %a) nounwind {
 define <32 x i16> @test_rem7_32i16(<32 x i16> %a) nounwind {
 ; AVX512F-LABEL: test_rem7_32i16:
 ; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm2 = [9363,9363,9363,9363,9363,9363,9363,9363,9363,9363,9363,9363,9363,9363,9363,9363]
-; AVX512F-NEXT:    vpmulhuw %ymm2, %ymm0, %ymm3
-; AVX512F-NEXT:    vpsubw %ymm3, %ymm0, %ymm4
+; AVX512F-NEXT:    vpmulhuw %ymm2, %ymm1, %ymm3
+; AVX512F-NEXT:    vpsubw %ymm3, %ymm1, %ymm4
 ; AVX512F-NEXT:    vpsrlw $1, %ymm4, %ymm4
 ; AVX512F-NEXT:    vpaddw %ymm3, %ymm4, %ymm3
 ; AVX512F-NEXT:    vpsrlw $2, %ymm3, %ymm3
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm4 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
 ; AVX512F-NEXT:    vpmullw %ymm4, %ymm3, %ymm3
-; AVX512F-NEXT:    vpsubw %ymm3, %ymm0, %ymm0
-; AVX512F-NEXT:    vpmulhuw %ymm2, %ymm1, %ymm2
-; AVX512F-NEXT:    vpsubw %ymm2, %ymm1, %ymm3
+; AVX512F-NEXT:    vpsubw %ymm3, %ymm1, %ymm1
+; AVX512F-NEXT:    vpmulhuw %ymm2, %ymm0, %ymm2
+; AVX512F-NEXT:    vpsubw %ymm2, %ymm0, %ymm3
 ; AVX512F-NEXT:    vpsrlw $1, %ymm3, %ymm3
 ; AVX512F-NEXT:    vpaddw %ymm2, %ymm3, %ymm2
 ; AVX512F-NEXT:    vpsrlw $2, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpmullw %ymm4, %ymm2, %ymm2
-; AVX512F-NEXT:    vpsubw %ymm2, %ymm1, %ymm1
+; AVX512F-NEXT:    vpsubw %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: test_rem7_32i16:
@@ -468,16 +476,17 @@ define <32 x i16> @test_rem7_32i16(<32 x i16> %a) nounwind {
 define <64 x i8> @test_rem7_64i8(<64 x i8> %a) nounwind {
 ; AVX512F-LABEL: test_rem7_64i8:
 ; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm0[8],ymm2[8],ymm0[9],ymm2[9],ymm0[10],ymm2[10],ymm0[11],ymm2[11],ymm0[12],ymm2[12],ymm0[13],ymm2[13],ymm0[14],ymm2[14],ymm0[15],ymm2[15],ymm0[24],ymm2[24],ymm0[25],ymm2[25],ymm0[26],ymm2[26],ymm0[27],ymm2[27],ymm0[28],ymm2[28],ymm0[29],ymm2[29],ymm0[30],ymm2[30],ymm0[31],ymm2[31]
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15],ymm1[24],ymm2[24],ymm1[25],ymm2[25],ymm1[26],ymm2[26],ymm1[27],ymm2[27],ymm1[28],ymm2[28],ymm1[29],ymm2[29],ymm1[30],ymm2[30],ymm1[31],ymm2[31]
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm4 = [37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37]
 ; AVX512F-NEXT:    vpmullw %ymm4, %ymm3, %ymm3
 ; AVX512F-NEXT:    vpsrlw $8, %ymm3, %ymm3
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm0[0],ymm2[0],ymm0[1],ymm2[1],ymm0[2],ymm2[2],ymm0[3],ymm2[3],ymm0[4],ymm2[4],ymm0[5],ymm2[5],ymm0[6],ymm2[6],ymm0[7],ymm2[7],ymm0[16],ymm2[16],ymm0[17],ymm2[17],ymm0[18],ymm2[18],ymm0[19],ymm2[19],ymm0[20],ymm2[20],ymm0[21],ymm2[21],ymm0[22],ymm2[22],ymm0[23],ymm2[23]
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[16],ymm2[16],ymm1[17],ymm2[17],ymm1[18],ymm2[18],ymm1[19],ymm2[19],ymm1[20],ymm2[20],ymm1[21],ymm2[21],ymm1[22],ymm2[22],ymm1[23],ymm2[23]
 ; AVX512F-NEXT:    vpmullw %ymm4, %ymm5, %ymm5
 ; AVX512F-NEXT:    vpsrlw $8, %ymm5, %ymm5
 ; AVX512F-NEXT:    vpackuswb %ymm3, %ymm5, %ymm3
-; AVX512F-NEXT:    vpsubb %ymm3, %ymm0, %ymm5
+; AVX512F-NEXT:    vpsubb %ymm3, %ymm1, %ymm5
 ; AVX512F-NEXT:    vpsrlw $1, %ymm5, %ymm5
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm6 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
 ; AVX512F-NEXT:    vpand %ymm6, %ymm5, %ymm5
@@ -487,26 +496,27 @@ define <64 x i8> @test_rem7_64i8(<64 x i8> %a) nounwind {
 ; AVX512F-NEXT:    vpand %ymm5, %ymm3, %ymm3
 ; AVX512F-NEXT:    vpsllw $3, %ymm3, %ymm7
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm8 = [248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248]
-; AVX512F-NEXT:    vpand %ymm8, %ymm7, %ymm7
+; AVX512F-NEXT:    vpand %ymm7, %ymm8, %ymm7
 ; AVX512F-NEXT:    vpsubb %ymm7, %ymm3, %ymm3
-; AVX512F-NEXT:    vpaddb %ymm3, %ymm0, %ymm0
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15],ymm1[24],ymm2[24],ymm1[25],ymm2[25],ymm1[26],ymm2[26],ymm1[27],ymm2[27],ymm1[28],ymm2[28],ymm1[29],ymm2[29],ymm1[30],ymm2[30],ymm1[31],ymm2[31]
+; AVX512F-NEXT:    vpaddb %ymm3, %ymm1, %ymm1
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm0[8],ymm2[8],ymm0[9],ymm2[9],ymm0[10],ymm2[10],ymm0[11],ymm2[11],ymm0[12],ymm2[12],ymm0[13],ymm2[13],ymm0[14],ymm2[14],ymm0[15],ymm2[15],ymm0[24],ymm2[24],ymm0[25],ymm2[25],ymm0[26],ymm2[26],ymm0[27],ymm2[27],ymm0[28],ymm2[28],ymm0[29],ymm2[29],ymm0[30],ymm2[30],ymm0[31],ymm2[31]
 ; AVX512F-NEXT:    vpmullw %ymm4, %ymm3, %ymm3
 ; AVX512F-NEXT:    vpsrlw $8, %ymm3, %ymm3
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[16],ymm2[16],ymm1[17],ymm2[17],ymm1[18],ymm2[18],ymm1[19],ymm2[19],ymm1[20],ymm2[20],ymm1[21],ymm2[21],ymm1[22],ymm2[22],ymm1[23],ymm2[23]
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm0[0],ymm2[0],ymm0[1],ymm2[1],ymm0[2],ymm2[2],ymm0[3],ymm2[3],ymm0[4],ymm2[4],ymm0[5],ymm2[5],ymm0[6],ymm2[6],ymm0[7],ymm2[7],ymm0[16],ymm2[16],ymm0[17],ymm2[17],ymm0[18],ymm2[18],ymm0[19],ymm2[19],ymm0[20],ymm2[20],ymm0[21],ymm2[21],ymm0[22],ymm2[22],ymm0[23],ymm2[23]
 ; AVX512F-NEXT:    vpmullw %ymm4, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpsrlw $8, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpackuswb %ymm3, %ymm2, %ymm2
-; AVX512F-NEXT:    vpsubb %ymm2, %ymm1, %ymm3
+; AVX512F-NEXT:    vpsubb %ymm2, %ymm0, %ymm3
 ; AVX512F-NEXT:    vpsrlw $1, %ymm3, %ymm3
 ; AVX512F-NEXT:    vpand %ymm6, %ymm3, %ymm3
 ; AVX512F-NEXT:    vpaddb %ymm2, %ymm3, %ymm2
 ; AVX512F-NEXT:    vpsrlw $2, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpand %ymm5, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpsllw $3, %ymm2, %ymm3
-; AVX512F-NEXT:    vpand %ymm8, %ymm3, %ymm3
+; AVX512F-NEXT:    vpand %ymm3, %ymm8, %ymm3
 ; AVX512F-NEXT:    vpsubb %ymm3, %ymm2, %ymm2
-; AVX512F-NEXT:    vpaddb %ymm2, %ymm1, %ymm1
+; AVX512F-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: test_rem7_64i8:
@@ -542,72 +552,74 @@ define <64 x i8> @test_rem7_64i8(<64 x i8> %a) nounwind {
 define <64 x i8> @test_remconstant_64i8(<64 x i8> %a) nounwind {
 ; AVX512F-LABEL: test_remconstant_64i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm0[8],ymm2[8],ymm0[9],ymm2[9],ymm0[10],ymm2[10],ymm0[11],ymm2[11],ymm0[12],ymm2[12],ymm0[13],ymm2[13],ymm0[14],ymm2[14],ymm0[15],ymm2[15],ymm0[24],ymm2[24],ymm0[25],ymm2[25],ymm0[26],ymm2[26],ymm0[27],ymm2[27],ymm0[28],ymm2[28],ymm0[29],ymm2[29],ymm0[30],ymm2[30],ymm0[31],ymm2[31]
-; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm3, %ymm3
-; AVX512F-NEXT:    vpsrlw $8, %ymm3, %ymm3
-; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm3, %ymm3
-; AVX512F-NEXT:    vpsrlw $8, %ymm3, %ymm3
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm0[0],ymm2[0],ymm0[1],ymm2[1],ymm0[2],ymm2[2],ymm0[3],ymm2[3],ymm0[4],ymm2[4],ymm0[5],ymm2[5],ymm0[6],ymm2[6],ymm0[7],ymm2[7],ymm0[16],ymm2[16],ymm0[17],ymm2[17],ymm0[18],ymm2[18],ymm0[19],ymm2[19],ymm0[20],ymm2[20],ymm0[21],ymm2[21],ymm0[22],ymm2[22],ymm0[23],ymm2[23]
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
+; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm3[8],ymm1[8],ymm3[9],ymm1[9],ymm3[10],ymm1[10],ymm3[11],ymm1[11],ymm3[12],ymm1[12],ymm3[13],ymm1[13],ymm3[14],ymm1[14],ymm3[15],ymm1[15],ymm3[24],ymm1[24],ymm3[25],ymm1[25],ymm3[26],ymm1[26],ymm3[27],ymm1[27],ymm3[28],ymm1[28],ymm3[29],ymm1[29],ymm3[30],ymm1[30],ymm3[31],ymm1[31]
+; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm2, %ymm2
+; AVX512F-NEXT:    vpsrlw $8, %ymm2, %ymm2
+; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm2, %ymm2
+; AVX512F-NEXT:    vpsrlw $8, %ymm2, %ymm2
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm3[0],ymm1[0],ymm3[1],ymm1[1],ymm3[2],ymm1[2],ymm3[3],ymm1[3],ymm3[4],ymm1[4],ymm3[5],ymm1[5],ymm3[6],ymm1[6],ymm3[7],ymm1[7],ymm3[16],ymm1[16],ymm3[17],ymm1[17],ymm3[18],ymm1[18],ymm3[19],ymm1[19],ymm3[20],ymm1[20],ymm3[21],ymm1[21],ymm3[22],ymm1[22],ymm3[23],ymm1[23]
 ; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm4, %ymm4
 ; AVX512F-NEXT:    vpsrlw $8, %ymm4, %ymm4
 ; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm4, %ymm4
 ; AVX512F-NEXT:    vpsrlw $8, %ymm4, %ymm4
-; AVX512F-NEXT:    vpackuswb %ymm3, %ymm4, %ymm3
-; AVX512F-NEXT:    vpsubb %ymm3, %ymm0, %ymm4
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm4[8],ymm2[8],ymm4[9],ymm2[9],ymm4[10],ymm2[10],ymm4[11],ymm2[11],ymm4[12],ymm2[12],ymm4[13],ymm2[13],ymm4[14],ymm2[14],ymm4[15],ymm2[15],ymm4[24],ymm2[24],ymm4[25],ymm2[25],ymm4[26],ymm2[26],ymm4[27],ymm2[27],ymm4[28],ymm2[28],ymm4[29],ymm2[29],ymm4[30],ymm2[30],ymm4[31],ymm2[31]
+; AVX512F-NEXT:    vpackuswb %ymm2, %ymm4, %ymm2
+; AVX512F-NEXT:    vpsubb %ymm2, %ymm3, %ymm4
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm4[8],ymm1[8],ymm4[9],ymm1[9],ymm4[10],ymm1[10],ymm4[11],ymm1[11],ymm4[12],ymm1[12],ymm4[13],ymm1[13],ymm4[14],ymm1[14],ymm4[15],ymm1[15],ymm4[24],ymm1[24],ymm4[25],ymm1[25],ymm4[26],ymm1[26],ymm4[27],ymm1[27],ymm4[28],ymm1[28],ymm4[29],ymm1[29],ymm4[30],ymm1[30],ymm4[31],ymm1[31]
 ; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm5, %ymm5
 ; AVX512F-NEXT:    vpsrlw $8, %ymm5, %ymm5
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm4[0],ymm2[0],ymm4[1],ymm2[1],ymm4[2],ymm2[2],ymm4[3],ymm2[3],ymm4[4],ymm2[4],ymm4[5],ymm2[5],ymm4[6],ymm2[6],ymm4[7],ymm2[7],ymm4[16],ymm2[16],ymm4[17],ymm2[17],ymm4[18],ymm2[18],ymm4[19],ymm2[19],ymm4[20],ymm2[20],ymm4[21],ymm2[21],ymm4[22],ymm2[22],ymm4[23],ymm2[23]
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm4[0],ymm1[0],ymm4[1],ymm1[1],ymm4[2],ymm1[2],ymm4[3],ymm1[3],ymm4[4],ymm1[4],ymm4[5],ymm1[5],ymm4[6],ymm1[6],ymm4[7],ymm1[7],ymm4[16],ymm1[16],ymm4[17],ymm1[17],ymm4[18],ymm1[18],ymm4[19],ymm1[19],ymm4[20],ymm1[20],ymm4[21],ymm1[21],ymm4[22],ymm1[22],ymm4[23],ymm1[23]
 ; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm4, %ymm4
 ; AVX512F-NEXT:    vpsrlw $8, %ymm4, %ymm4
 ; AVX512F-NEXT:    vpackuswb %ymm5, %ymm4, %ymm4
-; AVX512F-NEXT:    vpaddb %ymm3, %ymm4, %ymm4
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm4[8],ymm2[8],ymm4[9],ymm2[9],ymm4[10],ymm2[10],ymm4[11],ymm2[11],ymm4[12],ymm2[12],ymm4[13],ymm2[13],ymm4[14],ymm2[14],ymm4[15],ymm2[15],ymm4[24],ymm2[24],ymm4[25],ymm2[25],ymm4[26],ymm2[26],ymm4[27],ymm2[27],ymm4[28],ymm2[28],ymm4[29],ymm2[29],ymm4[30],ymm2[30],ymm4[31],ymm2[31]
-; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm3, %ymm3
-; AVX512F-NEXT:    vpsrlw $8, %ymm3, %ymm3
-; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm3, %ymm5
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512F-NEXT:    vpand %ymm3, %ymm5, %ymm5
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm4[0],ymm2[0],ymm4[1],ymm2[1],ymm4[2],ymm2[2],ymm4[3],ymm2[3],ymm4[4],ymm2[4],ymm4[5],ymm2[5],ymm4[6],ymm2[6],ymm4[7],ymm2[7],ymm4[16],ymm2[16],ymm4[17],ymm2[17],ymm4[18],ymm2[18],ymm4[19],ymm2[19],ymm4[20],ymm2[20],ymm4[21],ymm2[21],ymm4[22],ymm2[22],ymm4[23],ymm2[23]
+; AVX512F-NEXT:    vpaddb %ymm2, %ymm4, %ymm4
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm4[8],ymm1[8],ymm4[9],ymm1[9],ymm4[10],ymm1[10],ymm4[11],ymm1[11],ymm4[12],ymm1[12],ymm4[13],ymm1[13],ymm4[14],ymm1[14],ymm4[15],ymm1[15],ymm4[24],ymm1[24],ymm4[25],ymm1[25],ymm4[26],ymm1[26],ymm4[27],ymm1[27],ymm4[28],ymm1[28],ymm4[29],ymm1[29],ymm4[30],ymm1[30],ymm4[31],ymm1[31]
+; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm2, %ymm2
+; AVX512F-NEXT:    vpsrlw $8, %ymm2, %ymm2
+; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm2, %ymm5
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; AVX512F-NEXT:    vpand %ymm2, %ymm5, %ymm5
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm4[0],ymm1[0],ymm4[1],ymm1[1],ymm4[2],ymm1[2],ymm4[3],ymm1[3],ymm4[4],ymm1[4],ymm4[5],ymm1[5],ymm4[6],ymm1[6],ymm4[7],ymm1[7],ymm4[16],ymm1[16],ymm4[17],ymm1[17],ymm4[18],ymm1[18],ymm4[19],ymm1[19],ymm4[20],ymm1[20],ymm4[21],ymm1[21],ymm4[22],ymm1[22],ymm4[23],ymm1[23]
 ; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm4, %ymm4
 ; AVX512F-NEXT:    vpsrlw $8, %ymm4, %ymm4
 ; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm4, %ymm4
-; AVX512F-NEXT:    vpand %ymm3, %ymm4, %ymm4
+; AVX512F-NEXT:    vpand %ymm2, %ymm4, %ymm4
 ; AVX512F-NEXT:    vpackuswb %ymm5, %ymm4, %ymm4
-; AVX512F-NEXT:    vpsubb %ymm4, %ymm0, %ymm0
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15],ymm1[24],ymm2[24],ymm1[25],ymm2[25],ymm1[26],ymm2[26],ymm1[27],ymm2[27],ymm1[28],ymm2[28],ymm1[29],ymm2[29],ymm1[30],ymm2[30],ymm1[31],ymm2[31]
+; AVX512F-NEXT:    vpsubb %ymm4, %ymm3, %ymm3
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15],ymm0[24],ymm1[24],ymm0[25],ymm1[25],ymm0[26],ymm1[26],ymm0[27],ymm1[27],ymm0[28],ymm1[28],ymm0[29],ymm1[29],ymm0[30],ymm1[30],ymm0[31],ymm1[31]
 ; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm4, %ymm4
 ; AVX512F-NEXT:    vpsrlw $8, %ymm4, %ymm4
 ; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm4, %ymm4
 ; AVX512F-NEXT:    vpsrlw $8, %ymm4, %ymm4
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[16],ymm2[16],ymm1[17],ymm2[17],ymm1[18],ymm2[18],ymm1[19],ymm2[19],ymm1[20],ymm2[20],ymm1[21],ymm2[21],ymm1[22],ymm2[22],ymm1[23],ymm2[23]
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[16],ymm1[16],ymm0[17],ymm1[17],ymm0[18],ymm1[18],ymm0[19],ymm1[19],ymm0[20],ymm1[20],ymm0[21],ymm1[21],ymm0[22],ymm1[22],ymm0[23],ymm1[23]
 ; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm5, %ymm5
 ; AVX512F-NEXT:    vpsrlw $8, %ymm5, %ymm5
 ; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm5, %ymm5
 ; AVX512F-NEXT:    vpsrlw $8, %ymm5, %ymm5
 ; AVX512F-NEXT:    vpackuswb %ymm4, %ymm5, %ymm4
-; AVX512F-NEXT:    vpsubb %ymm4, %ymm1, %ymm5
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm6 = ymm5[8],ymm2[8],ymm5[9],ymm2[9],ymm5[10],ymm2[10],ymm5[11],ymm2[11],ymm5[12],ymm2[12],ymm5[13],ymm2[13],ymm5[14],ymm2[14],ymm5[15],ymm2[15],ymm5[24],ymm2[24],ymm5[25],ymm2[25],ymm5[26],ymm2[26],ymm5[27],ymm2[27],ymm5[28],ymm2[28],ymm5[29],ymm2[29],ymm5[30],ymm2[30],ymm5[31],ymm2[31]
+; AVX512F-NEXT:    vpsubb %ymm4, %ymm0, %ymm5
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm6 = ymm5[8],ymm1[8],ymm5[9],ymm1[9],ymm5[10],ymm1[10],ymm5[11],ymm1[11],ymm5[12],ymm1[12],ymm5[13],ymm1[13],ymm5[14],ymm1[14],ymm5[15],ymm1[15],ymm5[24],ymm1[24],ymm5[25],ymm1[25],ymm5[26],ymm1[26],ymm5[27],ymm1[27],ymm5[28],ymm1[28],ymm5[29],ymm1[29],ymm5[30],ymm1[30],ymm5[31],ymm1[31]
 ; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm6, %ymm6
 ; AVX512F-NEXT:    vpsrlw $8, %ymm6, %ymm6
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm5[0],ymm2[0],ymm5[1],ymm2[1],ymm5[2],ymm2[2],ymm5[3],ymm2[3],ymm5[4],ymm2[4],ymm5[5],ymm2[5],ymm5[6],ymm2[6],ymm5[7],ymm2[7],ymm5[16],ymm2[16],ymm5[17],ymm2[17],ymm5[18],ymm2[18],ymm5[19],ymm2[19],ymm5[20],ymm2[20],ymm5[21],ymm2[21],ymm5[22],ymm2[22],ymm5[23],ymm2[23]
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm5[0],ymm1[0],ymm5[1],ymm1[1],ymm5[2],ymm1[2],ymm5[3],ymm1[3],ymm5[4],ymm1[4],ymm5[5],ymm1[5],ymm5[6],ymm1[6],ymm5[7],ymm1[7],ymm5[16],ymm1[16],ymm5[17],ymm1[17],ymm5[18],ymm1[18],ymm5[19],ymm1[19],ymm5[20],ymm1[20],ymm5[21],ymm1[21],ymm5[22],ymm1[22],ymm5[23],ymm1[23]
 ; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm5, %ymm5
 ; AVX512F-NEXT:    vpsrlw $8, %ymm5, %ymm5
 ; AVX512F-NEXT:    vpackuswb %ymm6, %ymm5, %ymm5
 ; AVX512F-NEXT:    vpaddb %ymm4, %ymm5, %ymm4
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm4[8],ymm2[8],ymm4[9],ymm2[9],ymm4[10],ymm2[10],ymm4[11],ymm2[11],ymm4[12],ymm2[12],ymm4[13],ymm2[13],ymm4[14],ymm2[14],ymm4[15],ymm2[15],ymm4[24],ymm2[24],ymm4[25],ymm2[25],ymm4[26],ymm2[26],ymm4[27],ymm2[27],ymm4[28],ymm2[28],ymm4[29],ymm2[29],ymm4[30],ymm2[30],ymm4[31],ymm2[31]
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm4[8],ymm1[8],ymm4[9],ymm1[9],ymm4[10],ymm1[10],ymm4[11],ymm1[11],ymm4[12],ymm1[12],ymm4[13],ymm1[13],ymm4[14],ymm1[14],ymm4[15],ymm1[15],ymm4[24],ymm1[24],ymm4[25],ymm1[25],ymm4[26],ymm1[26],ymm4[27],ymm1[27],ymm4[28],ymm1[28],ymm4[29],ymm1[29],ymm4[30],ymm1[30],ymm4[31],ymm1[31]
 ; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm5, %ymm5
 ; AVX512F-NEXT:    vpsrlw $8, %ymm5, %ymm5
 ; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm5, %ymm5
-; AVX512F-NEXT:    vpand %ymm3, %ymm5, %ymm5
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm4[0],ymm2[0],ymm4[1],ymm2[1],ymm4[2],ymm2[2],ymm4[3],ymm2[3],ymm4[4],ymm2[4],ymm4[5],ymm2[5],ymm4[6],ymm2[6],ymm4[7],ymm2[7],ymm4[16],ymm2[16],ymm4[17],ymm2[17],ymm4[18],ymm2[18],ymm4[19],ymm2[19],ymm4[20],ymm2[20],ymm4[21],ymm2[21],ymm4[22],ymm2[22],ymm4[23],ymm2[23]
-; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm2, %ymm2
-; AVX512F-NEXT:    vpsrlw $8, %ymm2, %ymm2
-; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm2, %ymm2
-; AVX512F-NEXT:    vpand %ymm3, %ymm2, %ymm2
-; AVX512F-NEXT:    vpackuswb %ymm5, %ymm2, %ymm2
-; AVX512F-NEXT:    vpsubb %ymm2, %ymm1, %ymm1
+; AVX512F-NEXT:    vpand %ymm2, %ymm5, %ymm5
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm4[0],ymm1[0],ymm4[1],ymm1[1],ymm4[2],ymm1[2],ymm4[3],ymm1[3],ymm4[4],ymm1[4],ymm4[5],ymm1[5],ymm4[6],ymm1[6],ymm4[7],ymm1[7],ymm4[16],ymm1[16],ymm4[17],ymm1[17],ymm4[18],ymm1[18],ymm4[19],ymm1[19],ymm4[20],ymm1[20],ymm4[21],ymm1[21],ymm4[22],ymm1[22],ymm4[23],ymm1[23]
+; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm1, %ymm1
+; AVX512F-NEXT:    vpsrlw $8, %ymm1, %ymm1
+; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm1, %ymm1
+; AVX512F-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX512F-NEXT:    vpackuswb %ymm5, %ymm1, %ymm1
+; AVX512F-NEXT:    vpsubb %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: test_remconstant_64i8:
diff --git a/llvm/test/CodeGen/X86/vector-idiv-v2i32.ll b/llvm/test/CodeGen/X86/vector-idiv-v2i32.ll
index c0825262e..6e6dd6982 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-v2i32.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-v2i32.ll
@@ -1,105 +1,46 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=X64
 ; RUN: llc < %s -mtriple=i386-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=X86
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 -x86-experimental-vector-widening-legalization | FileCheck %s --check-prefix=X64_WIDEN
-; RUN: llc < %s -mtriple=i386-unknown-unknown -mattr=+sse2 -x86-experimental-vector-widening-legalization | FileCheck %s --check-prefix=X86_WIDEN
 
 define void @test_udiv7_v2i32(<2 x i32>* %x, <2 x i32>* %y) nounwind {
 ; X64-LABEL: test_udiv7_v2i32:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,3,0,1]
-; X64-NEXT:    movd %xmm1, %eax
-; X64-NEXT:    imulq $613566757, %rax, %rcx # imm = 0x24924925
-; X64-NEXT:    shrq $32, %rcx
-; X64-NEXT:    subl %ecx, %eax
-; X64-NEXT:    shrl %eax
-; X64-NEXT:    addl %ecx, %eax
-; X64-NEXT:    shrl $2, %eax
-; X64-NEXT:    movd %xmm0, %ecx
-; X64-NEXT:    imulq $613566757, %rcx, %rdx # imm = 0x24924925
-; X64-NEXT:    shrq $32, %rdx
-; X64-NEXT:    subl %edx, %ecx
-; X64-NEXT:    shrl %ecx
-; X64-NEXT:    addl %edx, %ecx
-; X64-NEXT:    shrl $2, %ecx
-; X64-NEXT:    movd %ecx, %xmm0
-; X64-NEXT:    movd %eax, %xmm1
-; X64-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; X64-NEXT:    movdqa {{.*#+}} xmm1 = [613566757,613566757,613566757,613566757]
+; X64-NEXT:    movdqa %xmm0, %xmm2
+; X64-NEXT:    pmuludq %xmm1, %xmm2
+; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
+; X64-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; X64-NEXT:    pmuludq %xmm1, %xmm3
+; X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,3,2,3]
+; X64-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; X64-NEXT:    psubd %xmm2, %xmm0
+; X64-NEXT:    psrld $1, %xmm0
+; X64-NEXT:    paddd %xmm2, %xmm0
+; X64-NEXT:    psrld $2, %xmm0
 ; X64-NEXT:    movq %xmm0, (%rsi)
 ; X64-NEXT:    retq
 ;
 ; X86-LABEL: test_udiv7_v2i32:
 ; X86:       # %bb.0:
-; X86-NEXT:    pushl %ebx
-; X86-NEXT:    pushl %edi
-; X86-NEXT:    pushl %esi
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X86-NEXT:    movd %xmm0, %ecx
-; X86-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,0,1]
-; X86-NEXT:    movd %xmm0, %esi
-; X86-NEXT:    movl $613566757, %ebx # imm = 0x24924925
-; X86-NEXT:    movl %esi, %eax
-; X86-NEXT:    mull %ebx
-; X86-NEXT:    subl %edx, %esi
-; X86-NEXT:    shrl %esi
-; X86-NEXT:    addl %edx, %esi
-; X86-NEXT:    shrl $2, %esi
-; X86-NEXT:    movl %ecx, %eax
-; X86-NEXT:    mull %ebx
-; X86-NEXT:    subl %edx, %ecx
-; X86-NEXT:    shrl %ecx
-; X86-NEXT:    addl %edx, %ecx
-; X86-NEXT:    shrl $2, %ecx
-; X86-NEXT:    movd %ecx, %xmm0
-; X86-NEXT:    movd %esi, %xmm1
-; X86-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; X86-NEXT:    movq %xmm0, (%edi)
-; X86-NEXT:    popl %esi
-; X86-NEXT:    popl %edi
-; X86-NEXT:    popl %ebx
+; X86-NEXT:    movdqa {{.*#+}} xmm1 = [613566757,613566757,613566757,613566757]
+; X86-NEXT:    movdqa %xmm0, %xmm2
+; X86-NEXT:    pmuludq %xmm1, %xmm2
+; X86-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
+; X86-NEXT:    movdqa %xmm0, %xmm3
+; X86-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm0[2,3]
+; X86-NEXT:    pmuludq %xmm1, %xmm3
+; X86-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,3,2,3]
+; X86-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; X86-NEXT:    psubd %xmm2, %xmm0
+; X86-NEXT:    psrld $1, %xmm0
+; X86-NEXT:    paddd %xmm2, %xmm0
+; X86-NEXT:    psrld $2, %xmm0
+; X86-NEXT:    movq %xmm0, (%eax)
 ; X86-NEXT:    retl
-;
-; X64_WIDEN-LABEL: test_udiv7_v2i32:
-; X64_WIDEN:       # %bb.0:
-; X64_WIDEN-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X64_WIDEN-NEXT:    movdqa {{.*#+}} xmm1 = [613566757,613566757,613566757,613566757]
-; X64_WIDEN-NEXT:    movdqa %xmm0, %xmm2
-; X64_WIDEN-NEXT:    pmuludq %xmm1, %xmm2
-; X64_WIDEN-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
-; X64_WIDEN-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
-; X64_WIDEN-NEXT:    pmuludq %xmm1, %xmm3
-; X64_WIDEN-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,3,2,3]
-; X64_WIDEN-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; X64_WIDEN-NEXT:    psubd %xmm2, %xmm0
-; X64_WIDEN-NEXT:    psrld $1, %xmm0
-; X64_WIDEN-NEXT:    paddd %xmm2, %xmm0
-; X64_WIDEN-NEXT:    psrld $2, %xmm0
-; X64_WIDEN-NEXT:    movq %xmm0, (%rsi)
-; X64_WIDEN-NEXT:    retq
-;
-; X86_WIDEN-LABEL: test_udiv7_v2i32:
-; X86_WIDEN:       # %bb.0:
-; X86_WIDEN-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86_WIDEN-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86_WIDEN-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X86_WIDEN-NEXT:    movdqa {{.*#+}} xmm1 = [613566757,613566757,613566757,613566757]
-; X86_WIDEN-NEXT:    movdqa %xmm0, %xmm2
-; X86_WIDEN-NEXT:    pmuludq %xmm1, %xmm2
-; X86_WIDEN-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
-; X86_WIDEN-NEXT:    movdqa %xmm0, %xmm3
-; X86_WIDEN-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm0[2,3]
-; X86_WIDEN-NEXT:    pmuludq %xmm1, %xmm3
-; X86_WIDEN-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,3,2,3]
-; X86_WIDEN-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; X86_WIDEN-NEXT:    psubd %xmm2, %xmm0
-; X86_WIDEN-NEXT:    psrld $1, %xmm0
-; X86_WIDEN-NEXT:    paddd %xmm2, %xmm0
-; X86_WIDEN-NEXT:    psrld $2, %xmm0
-; X86_WIDEN-NEXT:    movq %xmm0, (%eax)
-; X86_WIDEN-NEXT:    retl
   %a = load <2 x i32>, <2 x i32>* %x
   %b = udiv <2 x i32> %a, <i32 7, i32 7>
   store <2 x i32> %b, <2 x i32>* %y
@@ -110,126 +51,51 @@ define void @test_urem7_v2i32(<2 x i32>* %x, <2 x i32>* %y) nounwind {
 ; X64-LABEL: test_urem7_v2i32:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,3,0,1]
-; X64-NEXT:    movd %xmm1, %ecx
-; X64-NEXT:    imulq $613566757, %rcx, %rdx # imm = 0x24924925
-; X64-NEXT:    shrq $32, %rdx
-; X64-NEXT:    movl %ecx, %eax
-; X64-NEXT:    subl %edx, %eax
-; X64-NEXT:    shrl %eax
-; X64-NEXT:    addl %edx, %eax
-; X64-NEXT:    shrl $2, %eax
-; X64-NEXT:    leal (,%rax,8), %edx
-; X64-NEXT:    subl %edx, %eax
-; X64-NEXT:    addl %ecx, %eax
-; X64-NEXT:    movd %xmm0, %ecx
-; X64-NEXT:    imulq $613566757, %rcx, %rdx # imm = 0x24924925
-; X64-NEXT:    shrq $32, %rdx
-; X64-NEXT:    movl %ecx, %edi
-; X64-NEXT:    subl %edx, %edi
-; X64-NEXT:    shrl %edi
-; X64-NEXT:    addl %edx, %edi
-; X64-NEXT:    shrl $2, %edi
-; X64-NEXT:    leal (,%rdi,8), %edx
-; X64-NEXT:    subl %edx, %edi
-; X64-NEXT:    addl %ecx, %edi
-; X64-NEXT:    movd %edi, %xmm0
-; X64-NEXT:    movd %eax, %xmm1
-; X64-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; X64-NEXT:    movq %xmm0, (%rsi)
+; X64-NEXT:    movdqa {{.*#+}} xmm1 = [613566757,613566757,613566757,613566757]
+; X64-NEXT:    movdqa %xmm0, %xmm2
+; X64-NEXT:    pmuludq %xmm1, %xmm2
+; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
+; X64-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; X64-NEXT:    pmuludq %xmm1, %xmm3
+; X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,3,2,3]
+; X64-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; X64-NEXT:    movdqa %xmm0, %xmm1
+; X64-NEXT:    psubd %xmm2, %xmm1
+; X64-NEXT:    psrld $1, %xmm1
+; X64-NEXT:    paddd %xmm2, %xmm1
+; X64-NEXT:    psrld $2, %xmm1
+; X64-NEXT:    movdqa %xmm1, %xmm2
+; X64-NEXT:    pslld $3, %xmm2
+; X64-NEXT:    psubd %xmm2, %xmm1
+; X64-NEXT:    paddd %xmm0, %xmm1
+; X64-NEXT:    movq %xmm1, (%rsi)
 ; X64-NEXT:    retq
 ;
 ; X86-LABEL: test_urem7_v2i32:
 ; X86:       # %bb.0:
-; X86-NEXT:    pushl %ebp
-; X86-NEXT:    pushl %ebx
-; X86-NEXT:    pushl %edi
-; X86-NEXT:    pushl %esi
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X86-NEXT:    movd %xmm0, %ecx
-; X86-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,0,1]
-; X86-NEXT:    movd %xmm0, %esi
-; X86-NEXT:    movl $613566757, %edi # imm = 0x24924925
-; X86-NEXT:    movl %esi, %eax
-; X86-NEXT:    mull %edi
-; X86-NEXT:    movl %esi, %ebx
-; X86-NEXT:    subl %edx, %ebx
-; X86-NEXT:    shrl %ebx
-; X86-NEXT:    addl %edx, %ebx
-; X86-NEXT:    shrl $2, %ebx
-; X86-NEXT:    leal (,%ebx,8), %eax
-; X86-NEXT:    subl %eax, %ebx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebp
-; X86-NEXT:    addl %esi, %ebx
-; X86-NEXT:    movl %ecx, %eax
-; X86-NEXT:    mull %edi
-; X86-NEXT:    movl %ecx, %eax
-; X86-NEXT:    subl %edx, %eax
-; X86-NEXT:    shrl %eax
-; X86-NEXT:    addl %edx, %eax
-; X86-NEXT:    shrl $2, %eax
-; X86-NEXT:    leal (,%eax,8), %edx
-; X86-NEXT:    subl %edx, %eax
-; X86-NEXT:    addl %ecx, %eax
-; X86-NEXT:    movd %eax, %xmm0
-; X86-NEXT:    movd %ebx, %xmm1
-; X86-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; X86-NEXT:    movq %xmm0, (%ebp)
-; X86-NEXT:    popl %esi
-; X86-NEXT:    popl %edi
-; X86-NEXT:    popl %ebx
-; X86-NEXT:    popl %ebp
+; X86-NEXT:    movdqa {{.*#+}} xmm1 = [613566757,613566757,613566757,613566757]
+; X86-NEXT:    movdqa %xmm0, %xmm2
+; X86-NEXT:    pmuludq %xmm1, %xmm2
+; X86-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
+; X86-NEXT:    movdqa %xmm0, %xmm3
+; X86-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm0[2,3]
+; X86-NEXT:    pmuludq %xmm1, %xmm3
+; X86-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,3,2,3]
+; X86-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; X86-NEXT:    movdqa %xmm0, %xmm1
+; X86-NEXT:    psubd %xmm2, %xmm1
+; X86-NEXT:    psrld $1, %xmm1
+; X86-NEXT:    paddd %xmm2, %xmm1
+; X86-NEXT:    psrld $2, %xmm1
+; X86-NEXT:    movdqa %xmm1, %xmm2
+; X86-NEXT:    pslld $3, %xmm2
+; X86-NEXT:    psubd %xmm2, %xmm1
+; X86-NEXT:    paddd %xmm0, %xmm1
+; X86-NEXT:    movq %xmm1, (%eax)
 ; X86-NEXT:    retl
-;
-; X64_WIDEN-LABEL: test_urem7_v2i32:
-; X64_WIDEN:       # %bb.0:
-; X64_WIDEN-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X64_WIDEN-NEXT:    movdqa {{.*#+}} xmm1 = [613566757,613566757,613566757,613566757]
-; X64_WIDEN-NEXT:    movdqa %xmm0, %xmm2
-; X64_WIDEN-NEXT:    pmuludq %xmm1, %xmm2
-; X64_WIDEN-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
-; X64_WIDEN-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
-; X64_WIDEN-NEXT:    pmuludq %xmm1, %xmm3
-; X64_WIDEN-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,3,2,3]
-; X64_WIDEN-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; X64_WIDEN-NEXT:    movdqa %xmm0, %xmm1
-; X64_WIDEN-NEXT:    psubd %xmm2, %xmm1
-; X64_WIDEN-NEXT:    psrld $1, %xmm1
-; X64_WIDEN-NEXT:    paddd %xmm2, %xmm1
-; X64_WIDEN-NEXT:    psrld $2, %xmm1
-; X64_WIDEN-NEXT:    movdqa %xmm1, %xmm2
-; X64_WIDEN-NEXT:    pslld $3, %xmm2
-; X64_WIDEN-NEXT:    psubd %xmm2, %xmm1
-; X64_WIDEN-NEXT:    paddd %xmm0, %xmm1
-; X64_WIDEN-NEXT:    movq %xmm1, (%rsi)
-; X64_WIDEN-NEXT:    retq
-;
-; X86_WIDEN-LABEL: test_urem7_v2i32:
-; X86_WIDEN:       # %bb.0:
-; X86_WIDEN-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86_WIDEN-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86_WIDEN-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X86_WIDEN-NEXT:    movdqa {{.*#+}} xmm1 = [613566757,613566757,613566757,613566757]
-; X86_WIDEN-NEXT:    movdqa %xmm0, %xmm2
-; X86_WIDEN-NEXT:    pmuludq %xmm1, %xmm2
-; X86_WIDEN-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
-; X86_WIDEN-NEXT:    movdqa %xmm0, %xmm3
-; X86_WIDEN-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm0[2,3]
-; X86_WIDEN-NEXT:    pmuludq %xmm1, %xmm3
-; X86_WIDEN-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,3,2,3]
-; X86_WIDEN-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; X86_WIDEN-NEXT:    movdqa %xmm0, %xmm1
-; X86_WIDEN-NEXT:    psubd %xmm2, %xmm1
-; X86_WIDEN-NEXT:    psrld $1, %xmm1
-; X86_WIDEN-NEXT:    paddd %xmm2, %xmm1
-; X86_WIDEN-NEXT:    psrld $2, %xmm1
-; X86_WIDEN-NEXT:    movdqa %xmm1, %xmm2
-; X86_WIDEN-NEXT:    pslld $3, %xmm2
-; X86_WIDEN-NEXT:    psubd %xmm2, %xmm1
-; X86_WIDEN-NEXT:    paddd %xmm0, %xmm1
-; X86_WIDEN-NEXT:    movq %xmm1, (%eax)
-; X86_WIDEN-NEXT:    retl
   %a = load <2 x i32>, <2 x i32>* %x
   %b = urem <2 x i32> %a, <i32 7, i32 7>
   store <2 x i32> %b, <2 x i32>* %y
@@ -240,119 +106,53 @@ define void @test_sdiv7_v2i32(<2 x i32>* %x, <2 x i32>* %y) nounwind {
 ; X64-LABEL: test_sdiv7_v2i32:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,3,0,1]
-; X64-NEXT:    movd %xmm1, %eax
-; X64-NEXT:    cltq
-; X64-NEXT:    imulq $-1840700269, %rax, %rcx # imm = 0x92492493
-; X64-NEXT:    shrq $32, %rcx
-; X64-NEXT:    addl %ecx, %eax
-; X64-NEXT:    movl %eax, %ecx
-; X64-NEXT:    shrl $31, %ecx
-; X64-NEXT:    sarl $2, %eax
-; X64-NEXT:    addl %ecx, %eax
-; X64-NEXT:    movd %xmm0, %ecx
-; X64-NEXT:    movslq %ecx, %rcx
-; X64-NEXT:    imulq $-1840700269, %rcx, %rdx # imm = 0x92492493
-; X64-NEXT:    shrq $32, %rdx
-; X64-NEXT:    addl %edx, %ecx
-; X64-NEXT:    movl %ecx, %edx
-; X64-NEXT:    shrl $31, %edx
-; X64-NEXT:    sarl $2, %ecx
-; X64-NEXT:    addl %edx, %ecx
-; X64-NEXT:    movd %ecx, %xmm0
-; X64-NEXT:    movd %eax, %xmm1
-; X64-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; X64-NEXT:    movq %xmm0, (%rsi)
+; X64-NEXT:    movdqa {{.*#+}} xmm1 = [2454267027,2454267027,2454267027,2454267027]
+; X64-NEXT:    movdqa %xmm0, %xmm2
+; X64-NEXT:    pmuludq %xmm1, %xmm2
+; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
+; X64-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; X64-NEXT:    pmuludq %xmm1, %xmm3
+; X64-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,3,2,3]
+; X64-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; X64-NEXT:    pxor %xmm3, %xmm3
+; X64-NEXT:    pcmpgtd %xmm0, %xmm3
+; X64-NEXT:    pand %xmm1, %xmm3
+; X64-NEXT:    paddd %xmm0, %xmm3
+; X64-NEXT:    psubd %xmm3, %xmm2
+; X64-NEXT:    paddd %xmm0, %xmm2
+; X64-NEXT:    movdqa %xmm2, %xmm0
+; X64-NEXT:    psrld $31, %xmm0
+; X64-NEXT:    psrad $2, %xmm2
+; X64-NEXT:    paddd %xmm0, %xmm2
+; X64-NEXT:    movq %xmm2, (%rsi)
 ; X64-NEXT:    retq
 ;
 ; X86-LABEL: test_sdiv7_v2i32:
 ; X86:       # %bb.0:
-; X86-NEXT:    pushl %ebp
-; X86-NEXT:    pushl %ebx
-; X86-NEXT:    pushl %edi
-; X86-NEXT:    pushl %esi
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X86-NEXT:    movd %xmm0, %ecx
-; X86-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,0,1]
-; X86-NEXT:    movd %xmm0, %esi
-; X86-NEXT:    movl $-1840700269, %ebp # imm = 0x92492493
-; X86-NEXT:    movl %esi, %eax
-; X86-NEXT:    imull %ebp
-; X86-NEXT:    movl %edx, %edi
-; X86-NEXT:    addl %esi, %edi
-; X86-NEXT:    movl %edi, %eax
-; X86-NEXT:    shrl $31, %eax
-; X86-NEXT:    sarl $2, %edi
-; X86-NEXT:    addl %eax, %edi
-; X86-NEXT:    movl %ecx, %eax
-; X86-NEXT:    imull %ebp
-; X86-NEXT:    addl %ecx, %edx
-; X86-NEXT:    movl %edx, %eax
-; X86-NEXT:    shrl $31, %eax
-; X86-NEXT:    sarl $2, %edx
-; X86-NEXT:    addl %eax, %edx
-; X86-NEXT:    movd %edx, %xmm0
-; X86-NEXT:    movd %edi, %xmm1
-; X86-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; X86-NEXT:    movq %xmm0, (%ebx)
-; X86-NEXT:    popl %esi
-; X86-NEXT:    popl %edi
-; X86-NEXT:    popl %ebx
-; X86-NEXT:    popl %ebp
+; X86-NEXT:    movdqa {{.*#+}} xmm1 = [2454267027,2454267027,2454267027,2454267027]
+; X86-NEXT:    movdqa %xmm0, %xmm2
+; X86-NEXT:    pmuludq %xmm1, %xmm2
+; X86-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
+; X86-NEXT:    movdqa %xmm0, %xmm3
+; X86-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm0[2,3]
+; X86-NEXT:    pmuludq %xmm1, %xmm3
+; X86-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,3,2,3]
+; X86-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; X86-NEXT:    pxor %xmm3, %xmm3
+; X86-NEXT:    pcmpgtd %xmm0, %xmm3
+; X86-NEXT:    pand %xmm1, %xmm3
+; X86-NEXT:    paddd %xmm0, %xmm3
+; X86-NEXT:    psubd %xmm3, %xmm2
+; X86-NEXT:    paddd %xmm0, %xmm2
+; X86-NEXT:    movdqa %xmm2, %xmm0
+; X86-NEXT:    psrld $31, %xmm0
+; X86-NEXT:    psrad $2, %xmm2
+; X86-NEXT:    paddd %xmm0, %xmm2
+; X86-NEXT:    movq %xmm2, (%eax)
 ; X86-NEXT:    retl
-;
-; X64_WIDEN-LABEL: test_sdiv7_v2i32:
-; X64_WIDEN:       # %bb.0:
-; X64_WIDEN-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X64_WIDEN-NEXT:    movdqa {{.*#+}} xmm1 = [2454267027,2454267027,2454267027,2454267027]
-; X64_WIDEN-NEXT:    movdqa %xmm0, %xmm2
-; X64_WIDEN-NEXT:    pmuludq %xmm1, %xmm2
-; X64_WIDEN-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
-; X64_WIDEN-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
-; X64_WIDEN-NEXT:    pmuludq %xmm1, %xmm3
-; X64_WIDEN-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,3,2,3]
-; X64_WIDEN-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; X64_WIDEN-NEXT:    pxor %xmm3, %xmm3
-; X64_WIDEN-NEXT:    pcmpgtd %xmm0, %xmm3
-; X64_WIDEN-NEXT:    pand %xmm1, %xmm3
-; X64_WIDEN-NEXT:    paddd %xmm0, %xmm3
-; X64_WIDEN-NEXT:    psubd %xmm3, %xmm2
-; X64_WIDEN-NEXT:    paddd %xmm0, %xmm2
-; X64_WIDEN-NEXT:    movdqa %xmm2, %xmm0
-; X64_WIDEN-NEXT:    psrld $31, %xmm0
-; X64_WIDEN-NEXT:    psrad $2, %xmm2
-; X64_WIDEN-NEXT:    paddd %xmm0, %xmm2
-; X64_WIDEN-NEXT:    movq %xmm2, (%rsi)
-; X64_WIDEN-NEXT:    retq
-;
-; X86_WIDEN-LABEL: test_sdiv7_v2i32:
-; X86_WIDEN:       # %bb.0:
-; X86_WIDEN-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86_WIDEN-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86_WIDEN-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X86_WIDEN-NEXT:    movdqa {{.*#+}} xmm1 = [2454267027,2454267027,2454267027,2454267027]
-; X86_WIDEN-NEXT:    movdqa %xmm0, %xmm2
-; X86_WIDEN-NEXT:    pmuludq %xmm1, %xmm2
-; X86_WIDEN-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
-; X86_WIDEN-NEXT:    movdqa %xmm0, %xmm3
-; X86_WIDEN-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm0[2,3]
-; X86_WIDEN-NEXT:    pmuludq %xmm1, %xmm3
-; X86_WIDEN-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,3,2,3]
-; X86_WIDEN-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; X86_WIDEN-NEXT:    pxor %xmm3, %xmm3
-; X86_WIDEN-NEXT:    pcmpgtd %xmm0, %xmm3
-; X86_WIDEN-NEXT:    pand %xmm1, %xmm3
-; X86_WIDEN-NEXT:    paddd %xmm0, %xmm3
-; X86_WIDEN-NEXT:    psubd %xmm3, %xmm2
-; X86_WIDEN-NEXT:    paddd %xmm0, %xmm2
-; X86_WIDEN-NEXT:    movdqa %xmm2, %xmm0
-; X86_WIDEN-NEXT:    psrld $31, %xmm0
-; X86_WIDEN-NEXT:    psrad $2, %xmm2
-; X86_WIDEN-NEXT:    paddd %xmm0, %xmm2
-; X86_WIDEN-NEXT:    movq %xmm2, (%eax)
-; X86_WIDEN-NEXT:    retl
   %a = load <2 x i32>, <2 x i32>* %x
   %b = sdiv <2 x i32> %a, <i32 7, i32 7>
   store <2 x i32> %b, <2 x i32>* %y
@@ -363,139 +163,61 @@ define void @test_srem7_v2i32(<2 x i32>* %x, <2 x i32>* %y) nounwind {
 ; X64-LABEL: test_srem7_v2i32:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,3,0,1]
-; X64-NEXT:    movd %xmm1, %eax
-; X64-NEXT:    movslq %eax, %rcx
-; X64-NEXT:    imulq $-1840700269, %rcx, %rax # imm = 0x92492493
-; X64-NEXT:    shrq $32, %rax
-; X64-NEXT:    addl %ecx, %eax
-; X64-NEXT:    movl %eax, %edx
-; X64-NEXT:    shrl $31, %edx
-; X64-NEXT:    sarl $2, %eax
-; X64-NEXT:    addl %edx, %eax
-; X64-NEXT:    leal (,%rax,8), %edx
-; X64-NEXT:    subl %edx, %eax
-; X64-NEXT:    addl %ecx, %eax
-; X64-NEXT:    movd %xmm0, %ecx
-; X64-NEXT:    movslq %ecx, %rcx
-; X64-NEXT:    imulq $-1840700269, %rcx, %rdx # imm = 0x92492493
-; X64-NEXT:    shrq $32, %rdx
-; X64-NEXT:    addl %ecx, %edx
-; X64-NEXT:    movl %edx, %edi
-; X64-NEXT:    shrl $31, %edi
-; X64-NEXT:    sarl $2, %edx
-; X64-NEXT:    addl %edi, %edx
-; X64-NEXT:    leal (,%rdx,8), %edi
-; X64-NEXT:    subl %edi, %edx
-; X64-NEXT:    addl %ecx, %edx
-; X64-NEXT:    movd %edx, %xmm0
-; X64-NEXT:    movd %eax, %xmm1
-; X64-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; X64-NEXT:    movq %xmm0, (%rsi)
+; X64-NEXT:    movdqa {{.*#+}} xmm1 = [2454267027,2454267027,2454267027,2454267027]
+; X64-NEXT:    movdqa %xmm0, %xmm2
+; X64-NEXT:    pmuludq %xmm1, %xmm2
+; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
+; X64-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; X64-NEXT:    pmuludq %xmm1, %xmm3
+; X64-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,3,2,3]
+; X64-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; X64-NEXT:    pxor %xmm3, %xmm3
+; X64-NEXT:    pcmpgtd %xmm0, %xmm3
+; X64-NEXT:    pand %xmm1, %xmm3
+; X64-NEXT:    paddd %xmm0, %xmm3
+; X64-NEXT:    psubd %xmm3, %xmm2
+; X64-NEXT:    paddd %xmm0, %xmm2
+; X64-NEXT:    movdqa %xmm2, %xmm1
+; X64-NEXT:    psrld $31, %xmm1
+; X64-NEXT:    psrad $2, %xmm2
+; X64-NEXT:    paddd %xmm1, %xmm2
+; X64-NEXT:    movdqa %xmm2, %xmm1
+; X64-NEXT:    pslld $3, %xmm1
+; X64-NEXT:    psubd %xmm1, %xmm2
+; X64-NEXT:    paddd %xmm0, %xmm2
+; X64-NEXT:    movq %xmm2, (%rsi)
 ; X64-NEXT:    retq
 ;
 ; X86-LABEL: test_srem7_v2i32:
 ; X86:       # %bb.0:
-; X86-NEXT:    pushl %ebp
-; X86-NEXT:    pushl %ebx
-; X86-NEXT:    pushl %edi
-; X86-NEXT:    pushl %esi
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X86-NEXT:    movd %xmm0, %ecx
-; X86-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,0,1]
-; X86-NEXT:    movd %xmm0, %esi
-; X86-NEXT:    movl $-1840700269, %ebx # imm = 0x92492493
-; X86-NEXT:    movl %esi, %eax
-; X86-NEXT:    imull %ebx
-; X86-NEXT:    movl %edx, %edi
-; X86-NEXT:    addl %esi, %edi
-; X86-NEXT:    movl %edi, %eax
-; X86-NEXT:    shrl $31, %eax
-; X86-NEXT:    sarl $2, %edi
-; X86-NEXT:    addl %eax, %edi
-; X86-NEXT:    leal (,%edi,8), %eax
-; X86-NEXT:    subl %eax, %edi
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebp
-; X86-NEXT:    addl %esi, %edi
-; X86-NEXT:    movl %ecx, %eax
-; X86-NEXT:    imull %ebx
-; X86-NEXT:    addl %ecx, %edx
-; X86-NEXT:    movl %edx, %eax
-; X86-NEXT:    shrl $31, %eax
-; X86-NEXT:    sarl $2, %edx
-; X86-NEXT:    addl %eax, %edx
-; X86-NEXT:    leal (,%edx,8), %eax
-; X86-NEXT:    subl %eax, %edx
-; X86-NEXT:    addl %ecx, %edx
-; X86-NEXT:    movd %edx, %xmm0
-; X86-NEXT:    movd %edi, %xmm1
-; X86-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; X86-NEXT:    movq %xmm0, (%ebp)
-; X86-NEXT:    popl %esi
-; X86-NEXT:    popl %edi
-; X86-NEXT:    popl %ebx
-; X86-NEXT:    popl %ebp
+; X86-NEXT:    movdqa {{.*#+}} xmm1 = [2454267027,2454267027,2454267027,2454267027]
+; X86-NEXT:    movdqa %xmm0, %xmm2
+; X86-NEXT:    pmuludq %xmm1, %xmm2
+; X86-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
+; X86-NEXT:    movdqa %xmm0, %xmm3
+; X86-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm0[2,3]
+; X86-NEXT:    pmuludq %xmm1, %xmm3
+; X86-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,3,2,3]
+; X86-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; X86-NEXT:    pxor %xmm3, %xmm3
+; X86-NEXT:    pcmpgtd %xmm0, %xmm3
+; X86-NEXT:    pand %xmm1, %xmm3
+; X86-NEXT:    paddd %xmm0, %xmm3
+; X86-NEXT:    psubd %xmm3, %xmm2
+; X86-NEXT:    paddd %xmm0, %xmm2
+; X86-NEXT:    movdqa %xmm2, %xmm1
+; X86-NEXT:    psrld $31, %xmm1
+; X86-NEXT:    psrad $2, %xmm2
+; X86-NEXT:    paddd %xmm1, %xmm2
+; X86-NEXT:    movdqa %xmm2, %xmm1
+; X86-NEXT:    pslld $3, %xmm1
+; X86-NEXT:    psubd %xmm1, %xmm2
+; X86-NEXT:    paddd %xmm0, %xmm2
+; X86-NEXT:    movq %xmm2, (%eax)
 ; X86-NEXT:    retl
-;
-; X64_WIDEN-LABEL: test_srem7_v2i32:
-; X64_WIDEN:       # %bb.0:
-; X64_WIDEN-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X64_WIDEN-NEXT:    movdqa {{.*#+}} xmm1 = [2454267027,2454267027,2454267027,2454267027]
-; X64_WIDEN-NEXT:    movdqa %xmm0, %xmm2
-; X64_WIDEN-NEXT:    pmuludq %xmm1, %xmm2
-; X64_WIDEN-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
-; X64_WIDEN-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
-; X64_WIDEN-NEXT:    pmuludq %xmm1, %xmm3
-; X64_WIDEN-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,3,2,3]
-; X64_WIDEN-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; X64_WIDEN-NEXT:    pxor %xmm3, %xmm3
-; X64_WIDEN-NEXT:    pcmpgtd %xmm0, %xmm3
-; X64_WIDEN-NEXT:    pand %xmm1, %xmm3
-; X64_WIDEN-NEXT:    paddd %xmm0, %xmm3
-; X64_WIDEN-NEXT:    psubd %xmm3, %xmm2
-; X64_WIDEN-NEXT:    paddd %xmm0, %xmm2
-; X64_WIDEN-NEXT:    movdqa %xmm2, %xmm1
-; X64_WIDEN-NEXT:    psrld $31, %xmm1
-; X64_WIDEN-NEXT:    psrad $2, %xmm2
-; X64_WIDEN-NEXT:    paddd %xmm1, %xmm2
-; X64_WIDEN-NEXT:    movdqa %xmm2, %xmm1
-; X64_WIDEN-NEXT:    pslld $3, %xmm1
-; X64_WIDEN-NEXT:    psubd %xmm1, %xmm2
-; X64_WIDEN-NEXT:    paddd %xmm0, %xmm2
-; X64_WIDEN-NEXT:    movq %xmm2, (%rsi)
-; X64_WIDEN-NEXT:    retq
-;
-; X86_WIDEN-LABEL: test_srem7_v2i32:
-; X86_WIDEN:       # %bb.0:
-; X86_WIDEN-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86_WIDEN-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86_WIDEN-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X86_WIDEN-NEXT:    movdqa {{.*#+}} xmm1 = [2454267027,2454267027,2454267027,2454267027]
-; X86_WIDEN-NEXT:    movdqa %xmm0, %xmm2
-; X86_WIDEN-NEXT:    pmuludq %xmm1, %xmm2
-; X86_WIDEN-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
-; X86_WIDEN-NEXT:    movdqa %xmm0, %xmm3
-; X86_WIDEN-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm0[2,3]
-; X86_WIDEN-NEXT:    pmuludq %xmm1, %xmm3
-; X86_WIDEN-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,3,2,3]
-; X86_WIDEN-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; X86_WIDEN-NEXT:    pxor %xmm3, %xmm3
-; X86_WIDEN-NEXT:    pcmpgtd %xmm0, %xmm3
-; X86_WIDEN-NEXT:    pand %xmm1, %xmm3
-; X86_WIDEN-NEXT:    paddd %xmm0, %xmm3
-; X86_WIDEN-NEXT:    psubd %xmm3, %xmm2
-; X86_WIDEN-NEXT:    paddd %xmm0, %xmm2
-; X86_WIDEN-NEXT:    movdqa %xmm2, %xmm1
-; X86_WIDEN-NEXT:    psrld $31, %xmm1
-; X86_WIDEN-NEXT:    psrad $2, %xmm2
-; X86_WIDEN-NEXT:    paddd %xmm1, %xmm2
-; X86_WIDEN-NEXT:    movdqa %xmm2, %xmm1
-; X86_WIDEN-NEXT:    pslld $3, %xmm1
-; X86_WIDEN-NEXT:    psubd %xmm1, %xmm2
-; X86_WIDEN-NEXT:    paddd %xmm0, %xmm2
-; X86_WIDEN-NEXT:    movq %xmm2, (%eax)
-; X86_WIDEN-NEXT:    retl
   %a = load <2 x i32>, <2 x i32>* %x
   %b = srem <2 x i32> %a, <i32 7, i32 7>
   store <2 x i32> %b, <2 x i32>* %y
@@ -506,10 +228,7 @@ define void @test_udiv_pow2_v2i32(<2 x i32>* %x, <2 x i32>* %y) nounwind {
 ; X64-LABEL: test_udiv_pow2_v2i32:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X64-NEXT:    pxor %xmm1, %xmm1
-; X64-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; X64-NEXT:    psrlq $3, %xmm0
-; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X64-NEXT:    psrld $3, %xmm0
 ; X64-NEXT:    movq %xmm0, (%rsi)
 ; X64-NEXT:    retq
 ;
@@ -518,28 +237,9 @@ define void @test_udiv_pow2_v2i32(<2 x i32>* %x, <2 x i32>* %y) nounwind {
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X86-NEXT:    pxor %xmm1, %xmm1
-; X86-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; X86-NEXT:    psrlq $3, %xmm0
-; X86-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X86-NEXT:    psrld $3, %xmm0
 ; X86-NEXT:    movq %xmm0, (%eax)
 ; X86-NEXT:    retl
-;
-; X64_WIDEN-LABEL: test_udiv_pow2_v2i32:
-; X64_WIDEN:       # %bb.0:
-; X64_WIDEN-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X64_WIDEN-NEXT:    psrld $3, %xmm0
-; X64_WIDEN-NEXT:    movq %xmm0, (%rsi)
-; X64_WIDEN-NEXT:    retq
-;
-; X86_WIDEN-LABEL: test_udiv_pow2_v2i32:
-; X86_WIDEN:       # %bb.0:
-; X86_WIDEN-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86_WIDEN-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86_WIDEN-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X86_WIDEN-NEXT:    psrld $3, %xmm0
-; X86_WIDEN-NEXT:    movq %xmm0, (%eax)
-; X86_WIDEN-NEXT:    retl
   %a = load <2 x i32>, <2 x i32>* %x
   %b = udiv <2 x i32> %a, <i32 8, i32 8>
   store <2 x i32> %b, <2 x i32>* %y
@@ -549,14 +249,9 @@ define void @test_udiv_pow2_v2i32(<2 x i32>* %x, <2 x i32>* %y) nounwind {
 define void @test_urem_pow2_v2i32(<2 x i32>* %x, <2 x i32>* %y) nounwind {
 ; X64-LABEL: test_urem_pow2_v2i32:
 ; X64:       # %bb.0:
-; X64-NEXT:    movl (%rdi), %eax
-; X64-NEXT:    movl 4(%rdi), %ecx
-; X64-NEXT:    movq %rcx, %xmm0
-; X64-NEXT:    movq %rax, %xmm1
-; X64-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
-; X64-NEXT:    pand {{.*}}(%rip), %xmm1
-; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
-; X64-NEXT:    movq %xmm0, (%rsi)
+; X64-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X64-NEXT:    andps {{.*}}(%rip), %xmm0
+; X64-NEXT:    movlps %xmm0, (%rsi)
 ; X64-NEXT:    retq
 ;
 ; X86-LABEL: test_urem_pow2_v2i32:
@@ -564,27 +259,9 @@ define void @test_urem_pow2_v2i32(<2 x i32>* %x, <2 x i32>* %y) nounwind {
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; X86-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1,1,3]
 ; X86-NEXT:    andps {{\.LCPI.*}}, %xmm0
-; X86-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; X86-NEXT:    movq %xmm0, (%eax)
+; X86-NEXT:    movlps %xmm0, (%eax)
 ; X86-NEXT:    retl
-;
-; X64_WIDEN-LABEL: test_urem_pow2_v2i32:
-; X64_WIDEN:       # %bb.0:
-; X64_WIDEN-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; X64_WIDEN-NEXT:    andps {{.*}}(%rip), %xmm0
-; X64_WIDEN-NEXT:    movlps %xmm0, (%rsi)
-; X64_WIDEN-NEXT:    retq
-;
-; X86_WIDEN-LABEL: test_urem_pow2_v2i32:
-; X86_WIDEN:       # %bb.0:
-; X86_WIDEN-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86_WIDEN-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86_WIDEN-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; X86_WIDEN-NEXT:    andps {{\.LCPI.*}}, %xmm0
-; X86_WIDEN-NEXT:    movlps %xmm0, (%eax)
-; X86_WIDEN-NEXT:    retl
   %a = load <2 x i32>, <2 x i32>* %x
   %b = urem <2 x i32> %a, <i32 8, i32 8>
   store <2 x i32> %b, <2 x i32>* %y
@@ -595,23 +272,12 @@ define void @test_sdiv_pow2_v2i32(<2 x i32>* %x, <2 x i32>* %y) nounwind {
 ; X64-LABEL: test_sdiv_pow2_v2i32:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,1]
+; X64-NEXT:    movdqa %xmm0, %xmm1
 ; X64-NEXT:    psrad $31, %xmm1
-; X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
-; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,1,1,3]
-; X64-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; X64-NEXT:    psrlq $31, %xmm0
-; X64-NEXT:    pand {{.*}}(%rip), %xmm0
-; X64-NEXT:    psrlq $29, %xmm0
-; X64-NEXT:    paddq %xmm2, %xmm0
-; X64-NEXT:    psllq $32, %xmm0
-; X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,3,2,3]
-; X64-NEXT:    psrad $31, %xmm0
-; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
-; X64-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; X64-NEXT:    psrlq $3, %xmm1
-; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
-; X64-NEXT:    movq %xmm0, (%rsi)
+; X64-NEXT:    psrld $29, %xmm1
+; X64-NEXT:    paddd %xmm0, %xmm1
+; X64-NEXT:    psrad $3, %xmm1
+; X64-NEXT:    movq %xmm1, (%rsi)
 ; X64-NEXT:    retq
 ;
 ; X86-LABEL: test_sdiv_pow2_v2i32:
@@ -619,53 +285,13 @@ define void @test_sdiv_pow2_v2i32(<2 x i32>* %x, <2 x i32>* %y) nounwind {
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X86-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; X86-NEXT:    movdqa %xmm0, %xmm1
 ; X86-NEXT:    psrad $31, %xmm1
-; X86-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
-; X86-NEXT:    movdqa %xmm0, %xmm2
-; X86-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; X86-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1,1,3]
-; X86-NEXT:    psrlq $31, %xmm2
-; X86-NEXT:    movsd {{.*#+}} xmm2 = xmm2[0,1]
-; X86-NEXT:    movapd {{.*#+}} xmm1 = [2.1219957909652723E-314,2.1219957909652723E-314]
-; X86-NEXT:    xorpd %xmm1, %xmm2
-; X86-NEXT:    psubq %xmm1, %xmm2
-; X86-NEXT:    pand {{\.LCPI.*}}, %xmm2
-; X86-NEXT:    psrlq $29, %xmm2
-; X86-NEXT:    paddq %xmm0, %xmm2
-; X86-NEXT:    psllq $32, %xmm2
-; X86-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,3,2,3]
-; X86-NEXT:    psrad $31, %xmm2
-; X86-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,3,2,3]
-; X86-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; X86-NEXT:    psrlq $3, %xmm0
-; X86-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; X86-NEXT:    movq %xmm0, (%eax)
+; X86-NEXT:    psrld $29, %xmm1
+; X86-NEXT:    paddd %xmm0, %xmm1
+; X86-NEXT:    psrad $3, %xmm1
+; X86-NEXT:    movq %xmm1, (%eax)
 ; X86-NEXT:    retl
-;
-; X64_WIDEN-LABEL: test_sdiv_pow2_v2i32:
-; X64_WIDEN:       # %bb.0:
-; X64_WIDEN-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X64_WIDEN-NEXT:    movdqa %xmm0, %xmm1
-; X64_WIDEN-NEXT:    psrad $31, %xmm1
-; X64_WIDEN-NEXT:    psrld $29, %xmm1
-; X64_WIDEN-NEXT:    paddd %xmm0, %xmm1
-; X64_WIDEN-NEXT:    psrad $3, %xmm1
-; X64_WIDEN-NEXT:    movq %xmm1, (%rsi)
-; X64_WIDEN-NEXT:    retq
-;
-; X86_WIDEN-LABEL: test_sdiv_pow2_v2i32:
-; X86_WIDEN:       # %bb.0:
-; X86_WIDEN-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86_WIDEN-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86_WIDEN-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X86_WIDEN-NEXT:    movdqa %xmm0, %xmm1
-; X86_WIDEN-NEXT:    psrad $31, %xmm1
-; X86_WIDEN-NEXT:    psrld $29, %xmm1
-; X86_WIDEN-NEXT:    paddd %xmm0, %xmm1
-; X86_WIDEN-NEXT:    psrad $3, %xmm1
-; X86_WIDEN-NEXT:    movq %xmm1, (%eax)
-; X86_WIDEN-NEXT:    retl
   %a = load <2 x i32>, <2 x i32>* %x
   %b = sdiv <2 x i32> %a, <i32 8, i32 8>
   store <2 x i32> %b, <2 x i32>* %y
@@ -676,10 +302,7 @@ define void @test_srem_pow2_v2i32(<2 x i32>* %x, <2 x i32>* %y) nounwind {
 ; X64-LABEL: test_srem_pow2_v2i32:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X64-NEXT:    pxor %xmm1, %xmm1
-; X64-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; X64-NEXT:    psrlq $3, %xmm0
-; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X64-NEXT:    psrld $3, %xmm0
 ; X64-NEXT:    movq %xmm0, (%rsi)
 ; X64-NEXT:    retq
 ;
@@ -688,28 +311,9 @@ define void @test_srem_pow2_v2i32(<2 x i32>* %x, <2 x i32>* %y) nounwind {
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X86-NEXT:    pxor %xmm1, %xmm1
-; X86-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; X86-NEXT:    psrlq $3, %xmm0
-; X86-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X86-NEXT:    psrld $3, %xmm0
 ; X86-NEXT:    movq %xmm0, (%eax)
 ; X86-NEXT:    retl
-;
-; X64_WIDEN-LABEL: test_srem_pow2_v2i32:
-; X64_WIDEN:       # %bb.0:
-; X64_WIDEN-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X64_WIDEN-NEXT:    psrld $3, %xmm0
-; X64_WIDEN-NEXT:    movq %xmm0, (%rsi)
-; X64_WIDEN-NEXT:    retq
-;
-; X86_WIDEN-LABEL: test_srem_pow2_v2i32:
-; X86_WIDEN:       # %bb.0:
-; X86_WIDEN-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86_WIDEN-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86_WIDEN-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X86_WIDEN-NEXT:    psrld $3, %xmm0
-; X86_WIDEN-NEXT:    movq %xmm0, (%eax)
-; X86_WIDEN-NEXT:    retl
   %a = load <2 x i32>, <2 x i32>* %x
   %b = udiv <2 x i32> %a, <i32 8, i32 8>
   store <2 x i32> %b, <2 x i32>* %y
@@ -722,99 +326,46 @@ define void @test_udiv_v2i32(<2 x i32>* %x, <2 x i32>* %y, <2 x i32>* %z) nounwi
 ; X64-NEXT:    movq %rdx, %rcx
 ; X64-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
 ; X64-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
-; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,3,0,1]
-; X64-NEXT:    movd %xmm2, %eax
-; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,3,0,1]
-; X64-NEXT:    movd %xmm2, %esi
+; X64-NEXT:    movd %xmm0, %eax
+; X64-NEXT:    movd %xmm1, %esi
 ; X64-NEXT:    xorl %edx, %edx
 ; X64-NEXT:    divl %esi
-; X64-NEXT:    movl %eax, %esi
+; X64-NEXT:    movd %eax, %xmm2
+; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
 ; X64-NEXT:    movd %xmm0, %eax
-; X64-NEXT:    movd %xmm1, %edi
+; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X64-NEXT:    movd %xmm0, %esi
 ; X64-NEXT:    xorl %edx, %edx
-; X64-NEXT:    divl %edi
+; X64-NEXT:    divl %esi
 ; X64-NEXT:    movd %eax, %xmm0
-; X64-NEXT:    movd %esi, %xmm1
-; X64-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; X64-NEXT:    movq %xmm0, (%rcx)
+; X64-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
+; X64-NEXT:    movq %xmm2, (%rcx)
 ; X64-NEXT:    retq
 ;
 ; X86-LABEL: test_udiv_v2i32:
 ; X86:       # %bb.0:
-; X86-NEXT:    pushl %ebx
-; X86-NEXT:    pushl %edi
 ; X86-NEXT:    pushl %esi
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
 ; X86-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
-; X86-NEXT:    movd %xmm0, %ecx
-; X86-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,0,1]
 ; X86-NEXT:    movd %xmm0, %eax
-; X86-NEXT:    movd %xmm1, %ebx
-; X86-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,0,1]
 ; X86-NEXT:    movd %xmm1, %esi
 ; X86-NEXT:    xorl %edx, %edx
 ; X86-NEXT:    divl %esi
-; X86-NEXT:    movl %eax, %esi
-; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    movd %eax, %xmm2
+; X86-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; X86-NEXT:    movd %xmm0, %eax
+; X86-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[2,3]
+; X86-NEXT:    movd %xmm1, %esi
 ; X86-NEXT:    xorl %edx, %edx
-; X86-NEXT:    divl %ebx
+; X86-NEXT:    divl %esi
 ; X86-NEXT:    movd %eax, %xmm0
-; X86-NEXT:    movd %esi, %xmm1
-; X86-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; X86-NEXT:    movq %xmm0, (%edi)
+; X86-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
+; X86-NEXT:    movq %xmm2, (%ecx)
 ; X86-NEXT:    popl %esi
-; X86-NEXT:    popl %edi
-; X86-NEXT:    popl %ebx
 ; X86-NEXT:    retl
-;
-; X64_WIDEN-LABEL: test_udiv_v2i32:
-; X64_WIDEN:       # %bb.0:
-; X64_WIDEN-NEXT:    movq %rdx, %rcx
-; X64_WIDEN-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X64_WIDEN-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
-; X64_WIDEN-NEXT:    movd %xmm0, %eax
-; X64_WIDEN-NEXT:    movd %xmm1, %esi
-; X64_WIDEN-NEXT:    xorl %edx, %edx
-; X64_WIDEN-NEXT:    divl %esi
-; X64_WIDEN-NEXT:    movd %eax, %xmm2
-; X64_WIDEN-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; X64_WIDEN-NEXT:    movd %xmm0, %eax
-; X64_WIDEN-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
-; X64_WIDEN-NEXT:    movd %xmm0, %esi
-; X64_WIDEN-NEXT:    xorl %edx, %edx
-; X64_WIDEN-NEXT:    divl %esi
-; X64_WIDEN-NEXT:    movd %eax, %xmm0
-; X64_WIDEN-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; X64_WIDEN-NEXT:    movq %xmm2, (%rcx)
-; X64_WIDEN-NEXT:    retq
-;
-; X86_WIDEN-LABEL: test_udiv_v2i32:
-; X86_WIDEN:       # %bb.0:
-; X86_WIDEN-NEXT:    pushl %esi
-; X86_WIDEN-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86_WIDEN-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86_WIDEN-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86_WIDEN-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X86_WIDEN-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
-; X86_WIDEN-NEXT:    movd %xmm0, %eax
-; X86_WIDEN-NEXT:    movd %xmm1, %esi
-; X86_WIDEN-NEXT:    xorl %edx, %edx
-; X86_WIDEN-NEXT:    divl %esi
-; X86_WIDEN-NEXT:    movd %eax, %xmm2
-; X86_WIDEN-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; X86_WIDEN-NEXT:    movd %xmm0, %eax
-; X86_WIDEN-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[2,3]
-; X86_WIDEN-NEXT:    movd %xmm1, %esi
-; X86_WIDEN-NEXT:    xorl %edx, %edx
-; X86_WIDEN-NEXT:    divl %esi
-; X86_WIDEN-NEXT:    movd %eax, %xmm0
-; X86_WIDEN-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; X86_WIDEN-NEXT:    movq %xmm2, (%ecx)
-; X86_WIDEN-NEXT:    popl %esi
-; X86_WIDEN-NEXT:    retl
   %a = load <2 x i32>, <2 x i32>* %x
   %b = load <2 x i32>, <2 x i32>* %y
   %c = udiv <2 x i32> %a, %b
@@ -828,99 +379,46 @@ define void @test_urem_v2i32(<2 x i32>* %x, <2 x i32>* %y, <2 x i32>* %z) nounwi
 ; X64-NEXT:    movq %rdx, %rcx
 ; X64-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
 ; X64-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
-; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,3,0,1]
-; X64-NEXT:    movd %xmm2, %eax
-; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,3,0,1]
-; X64-NEXT:    movd %xmm2, %esi
+; X64-NEXT:    movd %xmm0, %eax
+; X64-NEXT:    movd %xmm1, %esi
 ; X64-NEXT:    xorl %edx, %edx
 ; X64-NEXT:    divl %esi
-; X64-NEXT:    movl %edx, %esi
+; X64-NEXT:    movd %edx, %xmm2
+; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
 ; X64-NEXT:    movd %xmm0, %eax
-; X64-NEXT:    movd %xmm1, %edi
+; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X64-NEXT:    movd %xmm0, %esi
 ; X64-NEXT:    xorl %edx, %edx
-; X64-NEXT:    divl %edi
+; X64-NEXT:    divl %esi
 ; X64-NEXT:    movd %edx, %xmm0
-; X64-NEXT:    movd %esi, %xmm1
-; X64-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; X64-NEXT:    movq %xmm0, (%rcx)
+; X64-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
+; X64-NEXT:    movq %xmm2, (%rcx)
 ; X64-NEXT:    retq
 ;
 ; X86-LABEL: test_urem_v2i32:
 ; X86:       # %bb.0:
-; X86-NEXT:    pushl %ebx
-; X86-NEXT:    pushl %edi
 ; X86-NEXT:    pushl %esi
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
 ; X86-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
-; X86-NEXT:    movd %xmm0, %ecx
-; X86-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,0,1]
 ; X86-NEXT:    movd %xmm0, %eax
-; X86-NEXT:    movd %xmm1, %ebx
-; X86-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,0,1]
 ; X86-NEXT:    movd %xmm1, %esi
 ; X86-NEXT:    xorl %edx, %edx
 ; X86-NEXT:    divl %esi
-; X86-NEXT:    movl %edx, %esi
-; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    movd %edx, %xmm2
+; X86-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; X86-NEXT:    movd %xmm0, %eax
+; X86-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[2,3]
+; X86-NEXT:    movd %xmm1, %esi
 ; X86-NEXT:    xorl %edx, %edx
-; X86-NEXT:    divl %ebx
+; X86-NEXT:    divl %esi
 ; X86-NEXT:    movd %edx, %xmm0
-; X86-NEXT:    movd %esi, %xmm1
-; X86-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; X86-NEXT:    movq %xmm0, (%edi)
+; X86-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
+; X86-NEXT:    movq %xmm2, (%ecx)
 ; X86-NEXT:    popl %esi
-; X86-NEXT:    popl %edi
-; X86-NEXT:    popl %ebx
 ; X86-NEXT:    retl
-;
-; X64_WIDEN-LABEL: test_urem_v2i32:
-; X64_WIDEN:       # %bb.0:
-; X64_WIDEN-NEXT:    movq %rdx, %rcx
-; X64_WIDEN-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X64_WIDEN-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
-; X64_WIDEN-NEXT:    movd %xmm0, %eax
-; X64_WIDEN-NEXT:    movd %xmm1, %esi
-; X64_WIDEN-NEXT:    xorl %edx, %edx
-; X64_WIDEN-NEXT:    divl %esi
-; X64_WIDEN-NEXT:    movd %edx, %xmm2
-; X64_WIDEN-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; X64_WIDEN-NEXT:    movd %xmm0, %eax
-; X64_WIDEN-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
-; X64_WIDEN-NEXT:    movd %xmm0, %esi
-; X64_WIDEN-NEXT:    xorl %edx, %edx
-; X64_WIDEN-NEXT:    divl %esi
-; X64_WIDEN-NEXT:    movd %edx, %xmm0
-; X64_WIDEN-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; X64_WIDEN-NEXT:    movq %xmm2, (%rcx)
-; X64_WIDEN-NEXT:    retq
-;
-; X86_WIDEN-LABEL: test_urem_v2i32:
-; X86_WIDEN:       # %bb.0:
-; X86_WIDEN-NEXT:    pushl %esi
-; X86_WIDEN-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86_WIDEN-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86_WIDEN-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86_WIDEN-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X86_WIDEN-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
-; X86_WIDEN-NEXT:    movd %xmm0, %eax
-; X86_WIDEN-NEXT:    movd %xmm1, %esi
-; X86_WIDEN-NEXT:    xorl %edx, %edx
-; X86_WIDEN-NEXT:    divl %esi
-; X86_WIDEN-NEXT:    movd %edx, %xmm2
-; X86_WIDEN-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; X86_WIDEN-NEXT:    movd %xmm0, %eax
-; X86_WIDEN-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[2,3]
-; X86_WIDEN-NEXT:    movd %xmm1, %esi
-; X86_WIDEN-NEXT:    xorl %edx, %edx
-; X86_WIDEN-NEXT:    divl %esi
-; X86_WIDEN-NEXT:    movd %edx, %xmm0
-; X86_WIDEN-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; X86_WIDEN-NEXT:    movq %xmm2, (%ecx)
-; X86_WIDEN-NEXT:    popl %esi
-; X86_WIDEN-NEXT:    retl
   %a = load <2 x i32>, <2 x i32>* %x
   %b = load <2 x i32>, <2 x i32>* %y
   %c = urem <2 x i32> %a, %b
@@ -934,21 +432,20 @@ define void @test_sdiv_v2i32(<2 x i32>* %x, <2 x i32>* %y, <2 x i32>* %z) nounwi
 ; X64-NEXT:    movq %rdx, %rcx
 ; X64-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
 ; X64-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
-; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,3,0,1]
-; X64-NEXT:    movd %xmm2, %eax
-; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,3,0,1]
-; X64-NEXT:    movd %xmm2, %esi
+; X64-NEXT:    movd %xmm0, %eax
+; X64-NEXT:    movd %xmm1, %esi
 ; X64-NEXT:    cltd
 ; X64-NEXT:    idivl %esi
-; X64-NEXT:    movl %eax, %esi
+; X64-NEXT:    movd %eax, %xmm2
+; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
 ; X64-NEXT:    movd %xmm0, %eax
-; X64-NEXT:    movd %xmm1, %edi
+; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X64-NEXT:    movd %xmm0, %esi
 ; X64-NEXT:    cltd
-; X64-NEXT:    idivl %edi
+; X64-NEXT:    idivl %esi
 ; X64-NEXT:    movd %eax, %xmm0
-; X64-NEXT:    movd %esi, %xmm1
-; X64-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; X64-NEXT:    movq %xmm0, (%rcx)
+; X64-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
+; X64-NEXT:    movq %xmm2, (%rcx)
 ; X64-NEXT:    retq
 ;
 ; X86-LABEL: test_sdiv_v2i32:
@@ -956,82 +453,30 @@ define void @test_sdiv_v2i32(<2 x i32>* %x, <2 x i32>* %y, <2 x i32>* %z) nounwi
 ; X86-NEXT:    pushl %ebx
 ; X86-NEXT:    pushl %edi
 ; X86-NEXT:    pushl %esi
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
 ; X86-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
 ; X86-NEXT:    movd %xmm0, %ecx
-; X86-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,0,1]
+; X86-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
 ; X86-NEXT:    movd %xmm0, %eax
+; X86-NEXT:    movd %xmm1, %edi
+; X86-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[2,3]
 ; X86-NEXT:    movd %xmm1, %ebx
-; X86-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,0,1]
-; X86-NEXT:    movd %xmm1, %esi
-; X86-NEXT:    cltd
-; X86-NEXT:    idivl %esi
-; X86-NEXT:    movl %eax, %esi
-; X86-NEXT:    movl %ecx, %eax
 ; X86-NEXT:    cltd
 ; X86-NEXT:    idivl %ebx
 ; X86-NEXT:    movd %eax, %xmm0
-; X86-NEXT:    movd %esi, %xmm1
-; X86-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; X86-NEXT:    movq %xmm0, (%edi)
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    cltd
+; X86-NEXT:    idivl %edi
+; X86-NEXT:    movd %eax, %xmm1
+; X86-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; X86-NEXT:    movq %xmm1, (%esi)
 ; X86-NEXT:    popl %esi
 ; X86-NEXT:    popl %edi
 ; X86-NEXT:    popl %ebx
 ; X86-NEXT:    retl
-;
-; X64_WIDEN-LABEL: test_sdiv_v2i32:
-; X64_WIDEN:       # %bb.0:
-; X64_WIDEN-NEXT:    movq %rdx, %rcx
-; X64_WIDEN-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X64_WIDEN-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
-; X64_WIDEN-NEXT:    movd %xmm0, %eax
-; X64_WIDEN-NEXT:    movd %xmm1, %esi
-; X64_WIDEN-NEXT:    cltd
-; X64_WIDEN-NEXT:    idivl %esi
-; X64_WIDEN-NEXT:    movd %eax, %xmm2
-; X64_WIDEN-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; X64_WIDEN-NEXT:    movd %xmm0, %eax
-; X64_WIDEN-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
-; X64_WIDEN-NEXT:    movd %xmm0, %esi
-; X64_WIDEN-NEXT:    cltd
-; X64_WIDEN-NEXT:    idivl %esi
-; X64_WIDEN-NEXT:    movd %eax, %xmm0
-; X64_WIDEN-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; X64_WIDEN-NEXT:    movq %xmm2, (%rcx)
-; X64_WIDEN-NEXT:    retq
-;
-; X86_WIDEN-LABEL: test_sdiv_v2i32:
-; X86_WIDEN:       # %bb.0:
-; X86_WIDEN-NEXT:    pushl %ebx
-; X86_WIDEN-NEXT:    pushl %edi
-; X86_WIDEN-NEXT:    pushl %esi
-; X86_WIDEN-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X86_WIDEN-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86_WIDEN-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86_WIDEN-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X86_WIDEN-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
-; X86_WIDEN-NEXT:    movd %xmm0, %ecx
-; X86_WIDEN-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; X86_WIDEN-NEXT:    movd %xmm0, %eax
-; X86_WIDEN-NEXT:    movd %xmm1, %edi
-; X86_WIDEN-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[2,3]
-; X86_WIDEN-NEXT:    movd %xmm1, %ebx
-; X86_WIDEN-NEXT:    cltd
-; X86_WIDEN-NEXT:    idivl %ebx
-; X86_WIDEN-NEXT:    movd %eax, %xmm0
-; X86_WIDEN-NEXT:    movl %ecx, %eax
-; X86_WIDEN-NEXT:    cltd
-; X86_WIDEN-NEXT:    idivl %edi
-; X86_WIDEN-NEXT:    movd %eax, %xmm1
-; X86_WIDEN-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; X86_WIDEN-NEXT:    movq %xmm1, (%esi)
-; X86_WIDEN-NEXT:    popl %esi
-; X86_WIDEN-NEXT:    popl %edi
-; X86_WIDEN-NEXT:    popl %ebx
-; X86_WIDEN-NEXT:    retl
   %a = load <2 x i32>, <2 x i32>* %x
   %b = load <2 x i32>, <2 x i32>* %y
   %c = sdiv <2 x i32> %a, %b
@@ -1045,21 +490,20 @@ define void @test_srem_v2i32(<2 x i32>* %x, <2 x i32>* %y, <2 x i32>* %z) nounwi
 ; X64-NEXT:    movq %rdx, %rcx
 ; X64-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
 ; X64-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
-; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,3,0,1]
-; X64-NEXT:    movd %xmm2, %eax
-; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,3,0,1]
-; X64-NEXT:    movd %xmm2, %esi
+; X64-NEXT:    movd %xmm0, %eax
+; X64-NEXT:    movd %xmm1, %esi
 ; X64-NEXT:    cltd
 ; X64-NEXT:    idivl %esi
-; X64-NEXT:    movl %eax, %esi
+; X64-NEXT:    movd %eax, %xmm2
+; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
 ; X64-NEXT:    movd %xmm0, %eax
-; X64-NEXT:    movd %xmm1, %edi
+; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X64-NEXT:    movd %xmm0, %esi
 ; X64-NEXT:    cltd
-; X64-NEXT:    idivl %edi
+; X64-NEXT:    idivl %esi
 ; X64-NEXT:    movd %eax, %xmm0
-; X64-NEXT:    movd %esi, %xmm1
-; X64-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; X64-NEXT:    movq %xmm0, (%rcx)
+; X64-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
+; X64-NEXT:    movq %xmm2, (%rcx)
 ; X64-NEXT:    retq
 ;
 ; X86-LABEL: test_srem_v2i32:
@@ -1067,82 +511,30 @@ define void @test_srem_v2i32(<2 x i32>* %x, <2 x i32>* %y, <2 x i32>* %z) nounwi
 ; X86-NEXT:    pushl %ebx
 ; X86-NEXT:    pushl %edi
 ; X86-NEXT:    pushl %esi
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
 ; X86-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
 ; X86-NEXT:    movd %xmm0, %ecx
-; X86-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,0,1]
+; X86-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
 ; X86-NEXT:    movd %xmm0, %eax
+; X86-NEXT:    movd %xmm1, %edi
+; X86-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[2,3]
 ; X86-NEXT:    movd %xmm1, %ebx
-; X86-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,0,1]
-; X86-NEXT:    movd %xmm1, %esi
-; X86-NEXT:    cltd
-; X86-NEXT:    idivl %esi
-; X86-NEXT:    movl %eax, %esi
-; X86-NEXT:    movl %ecx, %eax
 ; X86-NEXT:    cltd
 ; X86-NEXT:    idivl %ebx
 ; X86-NEXT:    movd %eax, %xmm0
-; X86-NEXT:    movd %esi, %xmm1
-; X86-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; X86-NEXT:    movq %xmm0, (%edi)
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    cltd
+; X86-NEXT:    idivl %edi
+; X86-NEXT:    movd %eax, %xmm1
+; X86-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; X86-NEXT:    movq %xmm1, (%esi)
 ; X86-NEXT:    popl %esi
 ; X86-NEXT:    popl %edi
 ; X86-NEXT:    popl %ebx
 ; X86-NEXT:    retl
-;
-; X64_WIDEN-LABEL: test_srem_v2i32:
-; X64_WIDEN:       # %bb.0:
-; X64_WIDEN-NEXT:    movq %rdx, %rcx
-; X64_WIDEN-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X64_WIDEN-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
-; X64_WIDEN-NEXT:    movd %xmm0, %eax
-; X64_WIDEN-NEXT:    movd %xmm1, %esi
-; X64_WIDEN-NEXT:    cltd
-; X64_WIDEN-NEXT:    idivl %esi
-; X64_WIDEN-NEXT:    movd %eax, %xmm2
-; X64_WIDEN-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; X64_WIDEN-NEXT:    movd %xmm0, %eax
-; X64_WIDEN-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
-; X64_WIDEN-NEXT:    movd %xmm0, %esi
-; X64_WIDEN-NEXT:    cltd
-; X64_WIDEN-NEXT:    idivl %esi
-; X64_WIDEN-NEXT:    movd %eax, %xmm0
-; X64_WIDEN-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; X64_WIDEN-NEXT:    movq %xmm2, (%rcx)
-; X64_WIDEN-NEXT:    retq
-;
-; X86_WIDEN-LABEL: test_srem_v2i32:
-; X86_WIDEN:       # %bb.0:
-; X86_WIDEN-NEXT:    pushl %ebx
-; X86_WIDEN-NEXT:    pushl %edi
-; X86_WIDEN-NEXT:    pushl %esi
-; X86_WIDEN-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X86_WIDEN-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86_WIDEN-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86_WIDEN-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X86_WIDEN-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
-; X86_WIDEN-NEXT:    movd %xmm0, %ecx
-; X86_WIDEN-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; X86_WIDEN-NEXT:    movd %xmm0, %eax
-; X86_WIDEN-NEXT:    movd %xmm1, %edi
-; X86_WIDEN-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[2,3]
-; X86_WIDEN-NEXT:    movd %xmm1, %ebx
-; X86_WIDEN-NEXT:    cltd
-; X86_WIDEN-NEXT:    idivl %ebx
-; X86_WIDEN-NEXT:    movd %eax, %xmm0
-; X86_WIDEN-NEXT:    movl %ecx, %eax
-; X86_WIDEN-NEXT:    cltd
-; X86_WIDEN-NEXT:    idivl %edi
-; X86_WIDEN-NEXT:    movd %eax, %xmm1
-; X86_WIDEN-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; X86_WIDEN-NEXT:    movq %xmm1, (%esi)
-; X86_WIDEN-NEXT:    popl %esi
-; X86_WIDEN-NEXT:    popl %edi
-; X86_WIDEN-NEXT:    popl %ebx
-; X86_WIDEN-NEXT:    retl
   %a = load <2 x i32>, <2 x i32>* %x
   %b = load <2 x i32>, <2 x i32>* %y
   %c = sdiv <2 x i32> %a, %b
diff --git a/llvm/test/CodeGen/X86/vector-interleave.ll b/llvm/test/CodeGen/X86/vector-interleave.ll
index 0c71d12b9..635d18904 100644
--- a/llvm/test/CodeGen/X86/vector-interleave.ll
+++ b/llvm/test/CodeGen/X86/vector-interleave.ll
@@ -137,3 +137,193 @@ define <64 x i16> @interleave8x8(<8 x i16> %a, <8 x i16> %b, <8 x i16> %c, <8 x
   %result = shufflevector <32 x i16> %abcd, <32 x i16> %efgh, <64 x i32> <i32 0, i32 32, i32 1, i32 33, i32 2, i32 34, i32 3, i32 35, i32 4, i32 36, i32 5, i32 37, i32 6, i32 38, i32 7, i32 39, i32 8, i32 40, i32 9, i32 41, i32 10, i32 42, i32 11, i32 43, i32 12, i32 44, i32 13, i32 45, i32 14, i32 46, i32 15, i32 47, i32 16, i32 48, i32 17, i32 49, i32 18, i32 50, i32 19, i32 51, i32 20, i32 52, i32 21, i32 53, i32 22, i32 54, i32 23, i32 55, i32 24, i32 56, i32 25, i32 57, i32 26, i32 58, i32 27, i32 59, i32 28, i32 60, i32 29, i32 61, i32 30, i32 62, i32 31, i32 63>
   ret <64 x i16> %result
 }
+
+define void @splat2_i8(<32 x i8>* %s, <64 x i8>* %d) {
+; SSE-LABEL: splat2_i8:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqu (%rdi), %xmm0
+; SSE-NEXT:    movdqu 16(%rdi), %xmm1
+; SSE-NEXT:    movdqa %xmm0, %xmm2
+; SSE-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSE-NEXT:    movdqa %xmm1, %xmm3
+; SSE-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3],xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
+; SSE-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSE-NEXT:    movdqu %xmm1, 48(%rsi)
+; SSE-NEXT:    movdqu %xmm3, 32(%rsi)
+; SSE-NEXT:    movdqu %xmm0, 16(%rsi)
+; SSE-NEXT:    movdqu %xmm2, (%rsi)
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: splat2_i8:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vmovdqu (%rdi), %xmm0
+; AVX1-NEXT:    vmovdqu 16(%rdi), %xmm1
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm3 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX1-NEXT:    vmovdqu %xmm1, 48(%rsi)
+; AVX1-NEXT:    vmovdqu %xmm3, 32(%rsi)
+; AVX1-NEXT:    vmovdqu %xmm0, 16(%rsi)
+; AVX1-NEXT:    vmovdqu %xmm2, (%rsi)
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: splat2_i8:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vmovdqu (%rdi), %ymm0
+; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm0[0,1,0,1]
+; AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX2-NEXT:    vpshufb %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,3,2,3]
+; AVX2-NEXT:    vpshufb %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vmovdqu %ymm0, 32(%rsi)
+; AVX2-NEXT:    vmovdqu %ymm1, (%rsi)
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+  %ld32 = load <32 x i8>, <32 x i8>* %s, align 1
+  %cat = shufflevector <32 x i8> %ld32, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+  %cat2 = shufflevector <64 x i8> %cat, <64 x i8> undef, <64 x i32> <i32 0, i32 32, i32 1, i32 33, i32 2, i32 34, i32 3, i32 35, i32 4, i32 36, i32 5, i32 37, i32 6, i32 38, i32 7, i32 39, i32 8, i32 40, i32 9, i32 41, i32 10, i32 42, i32 11, i32 43, i32 12, i32 44, i32 13, i32 45, i32 14, i32 46, i32 15, i32 47, i32 16, i32 48, i32 17, i32 49, i32 18, i32 50, i32 19, i32 51, i32 20, i32 52, i32 21, i32 53, i32 22, i32 54, i32 23, i32 55, i32 24, i32 56, i32 25, i32 57, i32 26, i32 58, i32 27, i32 59, i32 28, i32 60, i32 29, i32 61, i32 30, i32 62, i32 31, i32 63>
+  store <64 x i8> %cat2, <64 x i8>* %d, align 1
+  ret void
+}
+
+define void @splat2_i16(<16 x i16>* %s, <32 x i16>* %d) {
+; SSE-LABEL: splat2_i16:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqu (%rdi), %xmm0
+; SSE-NEXT:    movdqu 16(%rdi), %xmm1
+; SSE-NEXT:    movdqa %xmm0, %xmm2
+; SSE-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
+; SSE-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]
+; SSE-NEXT:    movdqa %xmm1, %xmm3
+; SSE-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
+; SSE-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
+; SSE-NEXT:    movdqu %xmm1, 48(%rsi)
+; SSE-NEXT:    movdqu %xmm3, 32(%rsi)
+; SSE-NEXT:    movdqu %xmm0, 16(%rsi)
+; SSE-NEXT:    movdqu %xmm2, (%rsi)
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: splat2_i16:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vmovdqu (%rdi), %xmm0
+; AVX1-NEXT:    vmovdqu 16(%rdi), %xmm1
+; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm0[0,0,1,1,2,2,3,3]
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]
+; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm1[0,0,1,1,2,2,3,3]
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
+; AVX1-NEXT:    vmovdqu %xmm1, 48(%rsi)
+; AVX1-NEXT:    vmovdqu %xmm3, 32(%rsi)
+; AVX1-NEXT:    vmovdqu %xmm0, 16(%rsi)
+; AVX1-NEXT:    vmovdqu %xmm2, (%rsi)
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: splat2_i16:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vmovdqu (%rdi), %ymm0
+; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm0[0,1,0,1]
+; AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,1,0,1,2,3,2,3,4,5,4,5,6,7,6,7,8,9,8,9,10,11,10,11,12,13,12,13,14,15,14,15]
+; AVX2-NEXT:    vpshufb %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,3,2,3]
+; AVX2-NEXT:    vpshufb %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vmovdqu %ymm0, 32(%rsi)
+; AVX2-NEXT:    vmovdqu %ymm1, (%rsi)
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+  %ld32 = load <16 x i16>, <16 x i16>* %s, align 1
+  %cat = shufflevector <16 x i16> %ld32, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %cat2 = shufflevector <32 x i16> %cat, <32 x i16> undef, <32 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23, i32 8, i32 24, i32 9, i32 25, i32 10, i32 26, i32 11, i32 27, i32 12, i32 28, i32 13, i32 29, i32 14, i32 30, i32 15, i32 31>
+  store <32 x i16> %cat2, <32 x i16>* %d, align 1
+  ret void
+}
+
+define void @splat2_i32(<8 x i32>* %s, <16 x i32>* %d) {
+; SSE-LABEL: splat2_i32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqu (%rdi), %xmm0
+; SSE-NEXT:    movdqu 16(%rdi), %xmm1
+; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,1,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,1,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,2,3,3]
+; SSE-NEXT:    movdqu %xmm1, 48(%rsi)
+; SSE-NEXT:    movdqu %xmm3, 32(%rsi)
+; SSE-NEXT:    movdqu %xmm0, 16(%rsi)
+; SSE-NEXT:    movdqu %xmm2, (%rsi)
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: splat2_i32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vmovups (%rdi), %xmm0
+; AVX1-NEXT:    vmovups 16(%rdi), %xmm1
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm2 = xmm0[0,0,1,1]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[2,2,3,3]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm3 = xmm1[0,0,1,1]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[2,2,3,3]
+; AVX1-NEXT:    vmovups %xmm1, 48(%rsi)
+; AVX1-NEXT:    vmovups %xmm3, 32(%rsi)
+; AVX1-NEXT:    vmovups %xmm0, 16(%rsi)
+; AVX1-NEXT:    vmovups %xmm2, (%rsi)
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: splat2_i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vmovups (%rdi), %ymm0
+; AVX2-NEXT:    vmovaps {{.*#+}} ymm1 = [0,0,1,1,2,2,3,3]
+; AVX2-NEXT:    vpermps %ymm0, %ymm1, %ymm1
+; AVX2-NEXT:    vmovaps {{.*#+}} ymm2 = [4,4,5,5,6,6,7,7]
+; AVX2-NEXT:    vpermps %ymm0, %ymm2, %ymm0
+; AVX2-NEXT:    vmovups %ymm0, 32(%rsi)
+; AVX2-NEXT:    vmovups %ymm1, (%rsi)
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+  %ld32 = load <8 x i32>, <8 x i32>* %s, align 1
+  %cat = shufflevector <8 x i32> %ld32, <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %cat2 = shufflevector <16 x i32> %cat, <16 x i32> undef, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
+  store <16 x i32> %cat2, <16 x i32>* %d, align 1
+  ret void
+}
+
+define void @splat2_i64(<4 x i64>* %s, <8 x i64>* %d) {
+; SSE-LABEL: splat2_i64:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqu (%rdi), %xmm0
+; SSE-NEXT:    movdqu 16(%rdi), %xmm1
+; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,1,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,1,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
+; SSE-NEXT:    movdqu %xmm1, 48(%rsi)
+; SSE-NEXT:    movdqu %xmm3, 32(%rsi)
+; SSE-NEXT:    movdqu %xmm0, 16(%rsi)
+; SSE-NEXT:    movdqu %xmm2, (%rsi)
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: splat2_i64:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vmovupd (%rdi), %ymm0
+; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm0[0,1,0,1]
+; AVX1-NEXT:    vpermilpd {{.*#+}} ymm1 = ymm1[0,0,3,3]
+; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,2,3]
+; AVX1-NEXT:    vpermilpd {{.*#+}} ymm0 = ymm0[0,0,3,3]
+; AVX1-NEXT:    vmovupd %ymm0, 32(%rsi)
+; AVX1-NEXT:    vmovupd %ymm1, (%rsi)
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: splat2_i64:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vmovups (%rdi), %ymm0
+; AVX2-NEXT:    vpermpd {{.*#+}} ymm1 = ymm0[0,0,1,1]
+; AVX2-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[2,2,3,3]
+; AVX2-NEXT:    vmovups %ymm0, 32(%rsi)
+; AVX2-NEXT:    vmovups %ymm1, (%rsi)
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+  %ld32 = load <4 x i64>, <4 x i64>* %s, align 1
+  %cat = shufflevector <4 x i64> %ld32, <4 x i64> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+  %cat2 = shufflevector <8 x i64> %cat, <8 x i64> undef, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+  store <8 x i64> %cat2, <8 x i64>* %d, align 1
+  ret void
+}
diff --git a/llvm/test/CodeGen/X86/vector-intrinsics.ll b/llvm/test/CodeGen/X86/vector-intrinsics.ll
index 436644e9f..becb28323 100644
--- a/llvm/test/CodeGen/X86/vector-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/vector-intrinsics.ll
@@ -1,4 +1,5 @@
-; RUN: llc < %s -mtriple=x86_64-- | grep call | count 43
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-- | FileCheck %s
 
 declare <4 x double> @llvm.sin.v4f64(<4 x double> %p)
 declare <4 x double> @llvm.cos.v4f64(<4 x double> %p)
@@ -6,21 +7,142 @@ declare <4 x double> @llvm.pow.v4f64(<4 x double> %p, <4 x double> %q)
 declare <4 x double> @llvm.powi.v4f64(<4 x double> %p, i32)
 
 define <4 x double> @foo(<4 x double> %p)
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subq $56, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
+; CHECK-NEXT:    callq sin
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps (%rsp), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
+; CHECK-NEXT:    callq sin
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    callq sin
+; CHECK-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
+; CHECK-NEXT:    callq sin
+; CHECK-NEXT:    movaps (%rsp), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    addq $56, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
 {
   %t = call <4 x double> @llvm.sin.v4f64(<4 x double> %p)
   ret <4 x double> %t
 }
 define <4 x double> @goo(<4 x double> %p)
+; CHECK-LABEL: goo:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subq $56, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
+; CHECK-NEXT:    callq cos
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps (%rsp), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
+; CHECK-NEXT:    callq cos
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    callq cos
+; CHECK-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
+; CHECK-NEXT:    callq cos
+; CHECK-NEXT:    movaps (%rsp), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    addq $56, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
 {
   %t = call <4 x double> @llvm.cos.v4f64(<4 x double> %p)
   ret <4 x double> %t
 }
 define <4 x double> @moo(<4 x double> %p, <4 x double> %q)
+; CHECK-LABEL: moo:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subq $88, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 96
+; CHECK-NEXT:    movaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps %xmm2, (%rsp) # 16-byte Spill
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps %xmm2, %xmm1
+; CHECK-NEXT:    callq pow
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
+; CHECK-NEXT:    movaps (%rsp), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    movhlps {{.*#+}} xmm1 = xmm1[1,1]
+; CHECK-NEXT:    callq pow
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    callq pow
+; CHECK-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    movhlps {{.*#+}} xmm1 = xmm1[1,1]
+; CHECK-NEXT:    callq pow
+; CHECK-NEXT:    movaps (%rsp), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    addq $88, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
 {
   %t = call <4 x double> @llvm.pow.v4f64(<4 x double> %p, <4 x double> %q)
   ret <4 x double> %t
 }
 define <4 x double> @zoo(<4 x double> %p, i32 %q)
+; CHECK-LABEL: zoo:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    subq $48, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-NEXT:    .cfi_offset %rbx, -16
+; CHECK-NEXT:    movl %edi, %ebx
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
+; CHECK-NEXT:    callq __powidf2
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps (%rsp), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
+; CHECK-NEXT:    movl %ebx, %edi
+; CHECK-NEXT:    callq __powidf2
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    movl %ebx, %edi
+; CHECK-NEXT:    callq __powidf2
+; CHECK-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
+; CHECK-NEXT:    movl %ebx, %edi
+; CHECK-NEXT:    callq __powidf2
+; CHECK-NEXT:    movaps (%rsp), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    addq $48, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
 {
   %t = call <4 x double> @llvm.powi.v4f64(<4 x double> %p, i32 %q)
   ret <4 x double> %t
@@ -32,12 +154,166 @@ declare <9 x double> @llvm.pow.v9f64(<9 x double> %a, <9 x double> %b)
 declare <9 x double> @llvm.powi.v9f64(<9 x double> %a, i32)
 
 define void @a(<9 x double>* %p) nounwind {
+; CHECK-LABEL: a:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    subq $96, %rsp
+; CHECK-NEXT:    movq %rdi, %rbx
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    movsd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    movaps (%rdi), %xmm0
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps 16(%rdi), %xmm0
+; CHECK-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
+; CHECK-NEXT:    movaps 32(%rdi), %xmm0
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps 48(%rdi), %xmm0
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    callq exp
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
+; CHECK-NEXT:    callq exp
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    callq exp
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
+; CHECK-NEXT:    callq exp
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps (%rsp), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    callq exp
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps (%rsp), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
+; CHECK-NEXT:    callq exp
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    callq exp
+; CHECK-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
+; CHECK-NEXT:    callq exp
+; CHECK-NEXT:    movaps (%rsp), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; CHECK-NEXT:    movaps %xmm1, (%rsp) # 16-byte Spill
+; CHECK-NEXT:    movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Reload
+; CHECK-NEXT:    # xmm0 = mem[0],zero
+; CHECK-NEXT:    callq exp
+; CHECK-NEXT:    movsd %xmm0, 64(%rbx)
+; CHECK-NEXT:    movaps (%rsp), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    movaps %xmm0, (%rbx)
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    movaps %xmm0, 16(%rbx)
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    movaps %xmm0, 32(%rbx)
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    movaps %xmm0, 48(%rbx)
+; CHECK-NEXT:    addq $96, %rsp
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    retq
   %a = load <9 x double>, <9 x double>* %p
   %r = call <9 x double> @llvm.exp.v9f64(<9 x double> %a)
   store <9 x double> %r, <9 x double>* %p
   ret void
 }
 define void @b(<9 x double>* %p, <9 x double>* %q) nounwind {
+; CHECK-LABEL: b:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    subq $160, %rsp
+; CHECK-NEXT:    movq %rdi, %rbx
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    movsd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    movaps (%rdi), %xmm0
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps 16(%rdi), %xmm0
+; CHECK-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
+; CHECK-NEXT:    movaps 32(%rdi), %xmm0
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps 48(%rdi), %xmm2
+; CHECK-NEXT:    movaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    movsd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    movaps (%rsi), %xmm0
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps 16(%rsi), %xmm0
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps 32(%rsi), %xmm0
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps 48(%rsi), %xmm1
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps %xmm2, %xmm0
+; CHECK-NEXT:    callq pow
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    movhlps {{.*#+}} xmm1 = xmm1[1,1]
+; CHECK-NEXT:    callq pow
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    callq pow
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    movhlps {{.*#+}} xmm1 = xmm1[1,1]
+; CHECK-NEXT:    callq pow
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps (%rsp), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    callq pow
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps (%rsp), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    movhlps {{.*#+}} xmm1 = xmm1[1,1]
+; CHECK-NEXT:    callq pow
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    callq pow
+; CHECK-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    movhlps {{.*#+}} xmm1 = xmm1[1,1]
+; CHECK-NEXT:    callq pow
+; CHECK-NEXT:    movaps (%rsp), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; CHECK-NEXT:    movaps %xmm1, (%rsp) # 16-byte Spill
+; CHECK-NEXT:    movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Reload
+; CHECK-NEXT:    # xmm0 = mem[0],zero
+; CHECK-NEXT:    movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 8-byte Reload
+; CHECK-NEXT:    # xmm1 = mem[0],zero
+; CHECK-NEXT:    callq pow
+; CHECK-NEXT:    movsd %xmm0, 64(%rbx)
+; CHECK-NEXT:    movaps (%rsp), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    movaps %xmm0, (%rbx)
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    movaps %xmm0, 16(%rbx)
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    movaps %xmm0, 32(%rbx)
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    movaps %xmm0, 48(%rbx)
+; CHECK-NEXT:    addq $160, %rsp
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    retq
   %a = load <9 x double>, <9 x double>* %p
   %b = load <9 x double>, <9 x double>* %q
   %r = call <9 x double> @llvm.pow.v9f64(<9 x double> %a, <9 x double> %b)
@@ -45,6 +321,83 @@ define void @b(<9 x double>* %p, <9 x double>* %q) nounwind {
   ret void
 }
 define void @c(<9 x double>* %p, i32 %n) nounwind {
+; CHECK-LABEL: c:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    subq $104, %rsp
+; CHECK-NEXT:    movl %esi, %ebp
+; CHECK-NEXT:    movq %rdi, %rbx
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    movsd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    movaps (%rdi), %xmm0
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps 16(%rdi), %xmm0
+; CHECK-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
+; CHECK-NEXT:    movaps 32(%rdi), %xmm0
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps 48(%rdi), %xmm0
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movl %esi, %edi
+; CHECK-NEXT:    callq __powidf2
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
+; CHECK-NEXT:    movl %ebp, %edi
+; CHECK-NEXT:    callq __powidf2
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    movl %ebp, %edi
+; CHECK-NEXT:    callq __powidf2
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
+; CHECK-NEXT:    movl %ebp, %edi
+; CHECK-NEXT:    callq __powidf2
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps (%rsp), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    movl %ebp, %edi
+; CHECK-NEXT:    callq __powidf2
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps (%rsp), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
+; CHECK-NEXT:    movl %ebp, %edi
+; CHECK-NEXT:    callq __powidf2
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    movl %ebp, %edi
+; CHECK-NEXT:    callq __powidf2
+; CHECK-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
+; CHECK-NEXT:    movl %ebp, %edi
+; CHECK-NEXT:    callq __powidf2
+; CHECK-NEXT:    movaps (%rsp), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; CHECK-NEXT:    movaps %xmm1, (%rsp) # 16-byte Spill
+; CHECK-NEXT:    movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Reload
+; CHECK-NEXT:    # xmm0 = mem[0],zero
+; CHECK-NEXT:    movl %ebp, %edi
+; CHECK-NEXT:    callq __powidf2
+; CHECK-NEXT:    movsd %xmm0, 64(%rbx)
+; CHECK-NEXT:    movaps (%rsp), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    movaps %xmm0, (%rbx)
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    movaps %xmm0, 16(%rbx)
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    movaps %xmm0, 32(%rbx)
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    movaps %xmm0, 48(%rbx)
+; CHECK-NEXT:    addq $104, %rsp
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    retq
   %a = load <9 x double>, <9 x double>* %p
   %r = call <9 x double> @llvm.powi.v9f64(<9 x double> %a, i32 %n)
   store <9 x double> %r, <9 x double>* %p
diff --git a/llvm/test/CodeGen/X86/vector-lzcnt-512.ll b/llvm/test/CodeGen/X86/vector-lzcnt-512.ll
index 7c54724f5..5a9aadd20 100644
--- a/llvm/test/CodeGen/X86/vector-lzcnt-512.ll
+++ b/llvm/test/CodeGen/X86/vector-lzcnt-512.ll
@@ -331,15 +331,17 @@ define <16 x i32> @testv16i32u(<16 x i32> %in) nounwind {
 define <32 x i16> @testv32i16(<32 x i16> %in) nounwind {
 ; AVX512CD-LABEL: testv32i16:
 ; AVX512CD:       # %bb.0:
-; AVX512CD-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
-; AVX512CD-NEXT:    vplzcntd %zmm0, %zmm0
-; AVX512CD-NEXT:    vpmovdw %zmm0, %ymm0
-; AVX512CD-NEXT:    vmovdqa {{.*#+}} ymm2 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
-; AVX512CD-NEXT:    vpsubw %ymm2, %ymm0, %ymm0
+; AVX512CD-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512CD-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
 ; AVX512CD-NEXT:    vplzcntd %zmm1, %zmm1
 ; AVX512CD-NEXT:    vpmovdw %zmm1, %ymm1
+; AVX512CD-NEXT:    vmovdqa {{.*#+}} ymm2 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
 ; AVX512CD-NEXT:    vpsubw %ymm2, %ymm1, %ymm1
+; AVX512CD-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
+; AVX512CD-NEXT:    vplzcntd %zmm0, %zmm0
+; AVX512CD-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512CD-NEXT:    vpsubw %ymm2, %ymm0, %ymm0
+; AVX512CD-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512CD-NEXT:    retq
 ;
 ; AVX512CDBW-LABEL: testv32i16:
@@ -378,9 +380,10 @@ define <32 x i16> @testv32i16(<32 x i16> %in) nounwind {
 ;
 ; AVX512DQ-LABEL: testv32i16:
 ; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm2 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
-; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm2, %ymm3
-; AVX512DQ-NEXT:    vpsrlw $4, %ymm0, %ymm4
+; AVX512DQ-NEXT:    vpshufb %ymm1, %ymm2, %ymm3
+; AVX512DQ-NEXT:    vpsrlw $4, %ymm1, %ymm4
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm5 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
 ; AVX512DQ-NEXT:    vpand %ymm5, %ymm4, %ymm4
 ; AVX512DQ-NEXT:    vpxor %xmm6, %xmm6, %xmm6
@@ -388,23 +391,24 @@ define <32 x i16> @testv32i16(<32 x i16> %in) nounwind {
 ; AVX512DQ-NEXT:    vpand %ymm7, %ymm3, %ymm3
 ; AVX512DQ-NEXT:    vpshufb %ymm4, %ymm2, %ymm4
 ; AVX512DQ-NEXT:    vpaddb %ymm4, %ymm3, %ymm3
-; AVX512DQ-NEXT:    vpcmpeqb %ymm6, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpsrlw $8, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpand %ymm0, %ymm3, %ymm0
+; AVX512DQ-NEXT:    vpcmpeqb %ymm6, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpsrlw $8, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpand %ymm1, %ymm3, %ymm1
 ; AVX512DQ-NEXT:    vpsrlw $8, %ymm3, %ymm3
-; AVX512DQ-NEXT:    vpaddw %ymm0, %ymm3, %ymm0
-; AVX512DQ-NEXT:    vpshufb %ymm1, %ymm2, %ymm3
-; AVX512DQ-NEXT:    vpsrlw $4, %ymm1, %ymm4
+; AVX512DQ-NEXT:    vpaddw %ymm1, %ymm3, %ymm1
+; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm2, %ymm3
+; AVX512DQ-NEXT:    vpsrlw $4, %ymm0, %ymm4
 ; AVX512DQ-NEXT:    vpand %ymm5, %ymm4, %ymm4
 ; AVX512DQ-NEXT:    vpcmpeqb %ymm6, %ymm4, %ymm5
 ; AVX512DQ-NEXT:    vpand %ymm5, %ymm3, %ymm3
 ; AVX512DQ-NEXT:    vpshufb %ymm4, %ymm2, %ymm2
 ; AVX512DQ-NEXT:    vpaddb %ymm2, %ymm3, %ymm2
-; AVX512DQ-NEXT:    vpcmpeqb %ymm6, %ymm1, %ymm1
-; AVX512DQ-NEXT:    vpsrlw $8, %ymm1, %ymm1
-; AVX512DQ-NEXT:    vpand %ymm1, %ymm2, %ymm1
+; AVX512DQ-NEXT:    vpcmpeqb %ymm6, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpsrlw $8, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpand %ymm0, %ymm2, %ymm0
 ; AVX512DQ-NEXT:    vpsrlw $8, %ymm2, %ymm2
-; AVX512DQ-NEXT:    vpaddw %ymm1, %ymm2, %ymm1
+; AVX512DQ-NEXT:    vpaddw %ymm0, %ymm2, %ymm0
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    retq
   %out = call <32 x i16> @llvm.ctlz.v32i16(<32 x i16> %in, i1 0)
   ret <32 x i16> %out
@@ -413,15 +417,17 @@ define <32 x i16> @testv32i16(<32 x i16> %in) nounwind {
 define <32 x i16> @testv32i16u(<32 x i16> %in) nounwind {
 ; AVX512CD-LABEL: testv32i16u:
 ; AVX512CD:       # %bb.0:
-; AVX512CD-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
-; AVX512CD-NEXT:    vplzcntd %zmm0, %zmm0
-; AVX512CD-NEXT:    vpmovdw %zmm0, %ymm0
-; AVX512CD-NEXT:    vmovdqa {{.*#+}} ymm2 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
-; AVX512CD-NEXT:    vpsubw %ymm2, %ymm0, %ymm0
+; AVX512CD-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512CD-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
 ; AVX512CD-NEXT:    vplzcntd %zmm1, %zmm1
 ; AVX512CD-NEXT:    vpmovdw %zmm1, %ymm1
+; AVX512CD-NEXT:    vmovdqa {{.*#+}} ymm2 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
 ; AVX512CD-NEXT:    vpsubw %ymm2, %ymm1, %ymm1
+; AVX512CD-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
+; AVX512CD-NEXT:    vplzcntd %zmm0, %zmm0
+; AVX512CD-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512CD-NEXT:    vpsubw %ymm2, %ymm0, %ymm0
+; AVX512CD-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512CD-NEXT:    retq
 ;
 ; AVX512CDBW-LABEL: testv32i16u:
@@ -460,9 +466,10 @@ define <32 x i16> @testv32i16u(<32 x i16> %in) nounwind {
 ;
 ; AVX512DQ-LABEL: testv32i16u:
 ; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm2 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
-; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm2, %ymm3
-; AVX512DQ-NEXT:    vpsrlw $4, %ymm0, %ymm4
+; AVX512DQ-NEXT:    vpshufb %ymm1, %ymm2, %ymm3
+; AVX512DQ-NEXT:    vpsrlw $4, %ymm1, %ymm4
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm5 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
 ; AVX512DQ-NEXT:    vpand %ymm5, %ymm4, %ymm4
 ; AVX512DQ-NEXT:    vpxor %xmm6, %xmm6, %xmm6
@@ -470,23 +477,24 @@ define <32 x i16> @testv32i16u(<32 x i16> %in) nounwind {
 ; AVX512DQ-NEXT:    vpand %ymm7, %ymm3, %ymm3
 ; AVX512DQ-NEXT:    vpshufb %ymm4, %ymm2, %ymm4
 ; AVX512DQ-NEXT:    vpaddb %ymm4, %ymm3, %ymm3
-; AVX512DQ-NEXT:    vpcmpeqb %ymm6, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpsrlw $8, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpand %ymm0, %ymm3, %ymm0
+; AVX512DQ-NEXT:    vpcmpeqb %ymm6, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpsrlw $8, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpand %ymm1, %ymm3, %ymm1
 ; AVX512DQ-NEXT:    vpsrlw $8, %ymm3, %ymm3
-; AVX512DQ-NEXT:    vpaddw %ymm0, %ymm3, %ymm0
-; AVX512DQ-NEXT:    vpshufb %ymm1, %ymm2, %ymm3
-; AVX512DQ-NEXT:    vpsrlw $4, %ymm1, %ymm4
+; AVX512DQ-NEXT:    vpaddw %ymm1, %ymm3, %ymm1
+; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm2, %ymm3
+; AVX512DQ-NEXT:    vpsrlw $4, %ymm0, %ymm4
 ; AVX512DQ-NEXT:    vpand %ymm5, %ymm4, %ymm4
 ; AVX512DQ-NEXT:    vpcmpeqb %ymm6, %ymm4, %ymm5
 ; AVX512DQ-NEXT:    vpand %ymm5, %ymm3, %ymm3
 ; AVX512DQ-NEXT:    vpshufb %ymm4, %ymm2, %ymm2
 ; AVX512DQ-NEXT:    vpaddb %ymm2, %ymm3, %ymm2
-; AVX512DQ-NEXT:    vpcmpeqb %ymm6, %ymm1, %ymm1
-; AVX512DQ-NEXT:    vpsrlw $8, %ymm1, %ymm1
-; AVX512DQ-NEXT:    vpand %ymm1, %ymm2, %ymm1
+; AVX512DQ-NEXT:    vpcmpeqb %ymm6, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpsrlw $8, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpand %ymm0, %ymm2, %ymm0
 ; AVX512DQ-NEXT:    vpsrlw $8, %ymm2, %ymm2
-; AVX512DQ-NEXT:    vpaddw %ymm1, %ymm2, %ymm1
+; AVX512DQ-NEXT:    vpaddw %ymm0, %ymm2, %ymm0
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    retq
   %out = call <32 x i16> @llvm.ctlz.v32i16(<32 x i16> %in, i1 -1)
   ret <32 x i16> %out
@@ -495,27 +503,29 @@ define <32 x i16> @testv32i16u(<32 x i16> %in) nounwind {
 define <64 x i8> @testv64i8(<64 x i8> %in) nounwind {
 ; AVX512CD-LABEL: testv64i8:
 ; AVX512CD:       # %bb.0:
-; AVX512CD-NEXT:    vextracti128 $1, %ymm0, %xmm2
-; AVX512CD-NEXT:    vpmovzxbd {{.*#+}} zmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero
-; AVX512CD-NEXT:    vplzcntd %zmm2, %zmm2
-; AVX512CD-NEXT:    vpmovdb %zmm2, %xmm2
-; AVX512CD-NEXT:    vmovdqa {{.*#+}} xmm3 = [24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24]
-; AVX512CD-NEXT:    vpsubb %xmm3, %xmm2, %xmm2
-; AVX512CD-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
-; AVX512CD-NEXT:    vplzcntd %zmm0, %zmm0
-; AVX512CD-NEXT:    vpmovdb %zmm0, %xmm0
-; AVX512CD-NEXT:    vpsubb %xmm3, %xmm0, %xmm0
-; AVX512CD-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX512CD-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512CD-NEXT:    vextracti128 $1, %ymm1, %xmm2
 ; AVX512CD-NEXT:    vpmovzxbd {{.*#+}} zmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero
 ; AVX512CD-NEXT:    vplzcntd %zmm2, %zmm2
 ; AVX512CD-NEXT:    vpmovdb %zmm2, %xmm2
+; AVX512CD-NEXT:    vmovdqa {{.*#+}} xmm3 = [24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24]
 ; AVX512CD-NEXT:    vpsubb %xmm3, %xmm2, %xmm2
 ; AVX512CD-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
 ; AVX512CD-NEXT:    vplzcntd %zmm1, %zmm1
 ; AVX512CD-NEXT:    vpmovdb %zmm1, %xmm1
 ; AVX512CD-NEXT:    vpsubb %xmm3, %xmm1, %xmm1
 ; AVX512CD-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512CD-NEXT:    vextracti128 $1, %ymm0, %xmm2
+; AVX512CD-NEXT:    vpmovzxbd {{.*#+}} zmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero
+; AVX512CD-NEXT:    vplzcntd %zmm2, %zmm2
+; AVX512CD-NEXT:    vpmovdb %zmm2, %xmm2
+; AVX512CD-NEXT:    vpsubb %xmm3, %xmm2, %xmm2
+; AVX512CD-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; AVX512CD-NEXT:    vplzcntd %zmm0, %zmm0
+; AVX512CD-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512CD-NEXT:    vpsubb %xmm3, %xmm0, %xmm0
+; AVX512CD-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX512CD-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512CD-NEXT:    retq
 ;
 ; AVX512CDBW-LABEL: testv64i8:
@@ -560,23 +570,25 @@ define <64 x i8> @testv64i8(<64 x i8> %in) nounwind {
 ;
 ; AVX512DQ-LABEL: testv64i8:
 ; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm2 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
-; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm2, %ymm3
-; AVX512DQ-NEXT:    vpsrlw $4, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512DQ-NEXT:    vpand %ymm4, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpxor %xmm5, %xmm5, %xmm5
-; AVX512DQ-NEXT:    vpcmpeqb %ymm5, %ymm0, %ymm6
-; AVX512DQ-NEXT:    vpand %ymm6, %ymm3, %ymm3
-; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm2, %ymm0
-; AVX512DQ-NEXT:    vpaddb %ymm0, %ymm3, %ymm0
 ; AVX512DQ-NEXT:    vpshufb %ymm1, %ymm2, %ymm3
 ; AVX512DQ-NEXT:    vpsrlw $4, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
 ; AVX512DQ-NEXT:    vpand %ymm4, %ymm1, %ymm1
-; AVX512DQ-NEXT:    vpcmpeqb %ymm5, %ymm1, %ymm4
-; AVX512DQ-NEXT:    vpand %ymm4, %ymm3, %ymm3
+; AVX512DQ-NEXT:    vpxor %xmm5, %xmm5, %xmm5
+; AVX512DQ-NEXT:    vpcmpeqb %ymm5, %ymm1, %ymm6
+; AVX512DQ-NEXT:    vpand %ymm6, %ymm3, %ymm3
 ; AVX512DQ-NEXT:    vpshufb %ymm1, %ymm2, %ymm1
 ; AVX512DQ-NEXT:    vpaddb %ymm1, %ymm3, %ymm1
+; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm2, %ymm3
+; AVX512DQ-NEXT:    vpsrlw $4, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpand %ymm4, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpcmpeqb %ymm5, %ymm0, %ymm4
+; AVX512DQ-NEXT:    vpand %ymm4, %ymm3, %ymm3
+; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm2, %ymm0
+; AVX512DQ-NEXT:    vpaddb %ymm0, %ymm3, %ymm0
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    retq
   %out = call <64 x i8> @llvm.ctlz.v64i8(<64 x i8> %in, i1 0)
   ret <64 x i8> %out
@@ -585,27 +597,29 @@ define <64 x i8> @testv64i8(<64 x i8> %in) nounwind {
 define <64 x i8> @testv64i8u(<64 x i8> %in) nounwind {
 ; AVX512CD-LABEL: testv64i8u:
 ; AVX512CD:       # %bb.0:
-; AVX512CD-NEXT:    vextracti128 $1, %ymm0, %xmm2
-; AVX512CD-NEXT:    vpmovzxbd {{.*#+}} zmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero
-; AVX512CD-NEXT:    vplzcntd %zmm2, %zmm2
-; AVX512CD-NEXT:    vpmovdb %zmm2, %xmm2
-; AVX512CD-NEXT:    vmovdqa {{.*#+}} xmm3 = [24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24]
-; AVX512CD-NEXT:    vpsubb %xmm3, %xmm2, %xmm2
-; AVX512CD-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
-; AVX512CD-NEXT:    vplzcntd %zmm0, %zmm0
-; AVX512CD-NEXT:    vpmovdb %zmm0, %xmm0
-; AVX512CD-NEXT:    vpsubb %xmm3, %xmm0, %xmm0
-; AVX512CD-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX512CD-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512CD-NEXT:    vextracti128 $1, %ymm1, %xmm2
 ; AVX512CD-NEXT:    vpmovzxbd {{.*#+}} zmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero
 ; AVX512CD-NEXT:    vplzcntd %zmm2, %zmm2
 ; AVX512CD-NEXT:    vpmovdb %zmm2, %xmm2
+; AVX512CD-NEXT:    vmovdqa {{.*#+}} xmm3 = [24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24]
 ; AVX512CD-NEXT:    vpsubb %xmm3, %xmm2, %xmm2
 ; AVX512CD-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
 ; AVX512CD-NEXT:    vplzcntd %zmm1, %zmm1
 ; AVX512CD-NEXT:    vpmovdb %zmm1, %xmm1
 ; AVX512CD-NEXT:    vpsubb %xmm3, %xmm1, %xmm1
 ; AVX512CD-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512CD-NEXT:    vextracti128 $1, %ymm0, %xmm2
+; AVX512CD-NEXT:    vpmovzxbd {{.*#+}} zmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero
+; AVX512CD-NEXT:    vplzcntd %zmm2, %zmm2
+; AVX512CD-NEXT:    vpmovdb %zmm2, %xmm2
+; AVX512CD-NEXT:    vpsubb %xmm3, %xmm2, %xmm2
+; AVX512CD-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; AVX512CD-NEXT:    vplzcntd %zmm0, %zmm0
+; AVX512CD-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512CD-NEXT:    vpsubb %xmm3, %xmm0, %xmm0
+; AVX512CD-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX512CD-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512CD-NEXT:    retq
 ;
 ; AVX512CDBW-LABEL: testv64i8u:
@@ -650,23 +664,25 @@ define <64 x i8> @testv64i8u(<64 x i8> %in) nounwind {
 ;
 ; AVX512DQ-LABEL: testv64i8u:
 ; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm2 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
-; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm2, %ymm3
-; AVX512DQ-NEXT:    vpsrlw $4, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512DQ-NEXT:    vpand %ymm4, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpxor %xmm5, %xmm5, %xmm5
-; AVX512DQ-NEXT:    vpcmpeqb %ymm5, %ymm0, %ymm6
-; AVX512DQ-NEXT:    vpand %ymm6, %ymm3, %ymm3
-; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm2, %ymm0
-; AVX512DQ-NEXT:    vpaddb %ymm0, %ymm3, %ymm0
 ; AVX512DQ-NEXT:    vpshufb %ymm1, %ymm2, %ymm3
 ; AVX512DQ-NEXT:    vpsrlw $4, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
 ; AVX512DQ-NEXT:    vpand %ymm4, %ymm1, %ymm1
-; AVX512DQ-NEXT:    vpcmpeqb %ymm5, %ymm1, %ymm4
-; AVX512DQ-NEXT:    vpand %ymm4, %ymm3, %ymm3
+; AVX512DQ-NEXT:    vpxor %xmm5, %xmm5, %xmm5
+; AVX512DQ-NEXT:    vpcmpeqb %ymm5, %ymm1, %ymm6
+; AVX512DQ-NEXT:    vpand %ymm6, %ymm3, %ymm3
 ; AVX512DQ-NEXT:    vpshufb %ymm1, %ymm2, %ymm1
 ; AVX512DQ-NEXT:    vpaddb %ymm1, %ymm3, %ymm1
+; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm2, %ymm3
+; AVX512DQ-NEXT:    vpsrlw $4, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpand %ymm4, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpcmpeqb %ymm5, %ymm0, %ymm4
+; AVX512DQ-NEXT:    vpand %ymm4, %ymm3, %ymm3
+; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm2, %ymm0
+; AVX512DQ-NEXT:    vpaddb %ymm0, %ymm3, %ymm0
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    retq
   %out = call <64 x i8> @llvm.ctlz.v64i8(<64 x i8> %in, i1 -1)
   ret <64 x i8> %out
diff --git a/llvm/test/CodeGen/X86/vector-lzcnt-sub128.ll b/llvm/test/CodeGen/X86/vector-lzcnt-sub128.ll
new file mode 100644
index 000000000..c1e7e42ac
--- /dev/null
+++ b/llvm/test/CodeGen/X86/vector-lzcnt-sub128.ll
@@ -0,0 +1,85 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s
+
+declare <2 x i32> @llvm.ctlz.v2i32(<2 x i32>, i1 immarg)
+
+define <2 x i32> @illegal_ctlz(<2 x i32> %v1) {
+; CHECK-LABEL: illegal_ctlz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movdqa %xmm0, %xmm1
+; CHECK-NEXT:    psrld $1, %xmm1
+; CHECK-NEXT:    por %xmm0, %xmm1
+; CHECK-NEXT:    movdqa %xmm1, %xmm0
+; CHECK-NEXT:    psrld $2, %xmm0
+; CHECK-NEXT:    por %xmm1, %xmm0
+; CHECK-NEXT:    movdqa %xmm0, %xmm1
+; CHECK-NEXT:    psrld $4, %xmm1
+; CHECK-NEXT:    por %xmm0, %xmm1
+; CHECK-NEXT:    movdqa %xmm1, %xmm0
+; CHECK-NEXT:    psrld $8, %xmm0
+; CHECK-NEXT:    por %xmm1, %xmm0
+; CHECK-NEXT:    movdqa %xmm0, %xmm1
+; CHECK-NEXT:    psrld $16, %xmm1
+; CHECK-NEXT:    por %xmm0, %xmm1
+; CHECK-NEXT:    pcmpeqd %xmm2, %xmm2
+; CHECK-NEXT:    pxor %xmm1, %xmm2
+; CHECK-NEXT:    movdqa %xmm2, %xmm0
+; CHECK-NEXT:    psrlw $1, %xmm0
+; CHECK-NEXT:    pand {{.*}}(%rip), %xmm0
+; CHECK-NEXT:    psubb %xmm0, %xmm2
+; CHECK-NEXT:    movdqa {{.*#+}} xmm0 = [51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51]
+; CHECK-NEXT:    movdqa %xmm2, %xmm1
+; CHECK-NEXT:    pand %xmm0, %xmm1
+; CHECK-NEXT:    psrlw $2, %xmm2
+; CHECK-NEXT:    pand %xmm0, %xmm2
+; CHECK-NEXT:    paddb %xmm1, %xmm2
+; CHECK-NEXT:    movdqa %xmm2, %xmm0
+; CHECK-NEXT:    psrlw $4, %xmm0
+; CHECK-NEXT:    paddb %xmm2, %xmm0
+; CHECK-NEXT:    pand {{.*}}(%rip), %xmm0
+; CHECK-NEXT:    pxor %xmm1, %xmm1
+; CHECK-NEXT:    movdqa %xmm0, %xmm2
+; CHECK-NEXT:    punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; CHECK-NEXT:    psadbw %xmm1, %xmm2
+; CHECK-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; CHECK-NEXT:    psadbw %xmm1, %xmm0
+; CHECK-NEXT:    packuswb %xmm2, %xmm0
+; CHECK-NEXT:    retq
+  %v2 = call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> %v1, i1 true)
+  ret <2 x i32> %v2
+}
+
+declare <2 x i32> @llvm.cttz.v2i32(<2 x i32>, i1 immarg)
+
+define <2 x i32> @illegal_cttz(<2 x i32> %v1) {
+; CHECK-LABEL: illegal_cttz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pcmpeqd %xmm1, %xmm1
+; CHECK-NEXT:    paddd %xmm0, %xmm1
+; CHECK-NEXT:    pandn %xmm1, %xmm0
+; CHECK-NEXT:    movdqa %xmm0, %xmm1
+; CHECK-NEXT:    psrlw $1, %xmm1
+; CHECK-NEXT:    pand {{.*}}(%rip), %xmm1
+; CHECK-NEXT:    psubb %xmm1, %xmm0
+; CHECK-NEXT:    movdqa {{.*#+}} xmm1 = [51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51]
+; CHECK-NEXT:    movdqa %xmm0, %xmm2
+; CHECK-NEXT:    pand %xmm1, %xmm2
+; CHECK-NEXT:    psrlw $2, %xmm0
+; CHECK-NEXT:    pand %xmm1, %xmm0
+; CHECK-NEXT:    paddb %xmm2, %xmm0
+; CHECK-NEXT:    movdqa %xmm0, %xmm1
+; CHECK-NEXT:    psrlw $4, %xmm1
+; CHECK-NEXT:    paddb %xmm0, %xmm1
+; CHECK-NEXT:    pand {{.*}}(%rip), %xmm1
+; CHECK-NEXT:    pxor %xmm0, %xmm0
+; CHECK-NEXT:    movdqa %xmm1, %xmm2
+; CHECK-NEXT:    punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm0[2],xmm2[3],xmm0[3]
+; CHECK-NEXT:    psadbw %xmm0, %xmm2
+; CHECK-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; CHECK-NEXT:    psadbw %xmm0, %xmm1
+; CHECK-NEXT:    packuswb %xmm2, %xmm1
+; CHECK-NEXT:    movdqa %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %v2 = call <2 x i32> @llvm.cttz.v2i32(<2 x i32> %v1, i1 true)
+  ret <2 x i32> %v2
+}
diff --git a/llvm/test/CodeGen/X86/vector-mul.ll b/llvm/test/CodeGen/X86/vector-mul.ll
index d16e18791..805ff9f69 100644
--- a/llvm/test/CodeGen/X86/vector-mul.ll
+++ b/llvm/test/CodeGen/X86/vector-mul.ll
@@ -3,6 +3,7 @@
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.2 | FileCheck %s --check-prefix=X64
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop  | FileCheck %s --check-prefix=X64-AVX --check-prefix=X64-XOP
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=X64-AVX --check-prefix=X64-AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512dq,+avx512vl | FileCheck %s --check-prefix=X64-AVX --check-prefix=X64-AVX512DQ
 
 ;
 ; PowOf2 (uniform)
@@ -88,6 +89,12 @@ define <16 x i8> @mul_v16i8_32(<16 x i8> %a0) nounwind {
 ; X64-AVX2-NEXT:    vpsllw $5, %xmm0, %xmm0
 ; X64-AVX2-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
 ; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512DQ-LABEL: mul_v16i8_32:
+; X64-AVX512DQ:       # %bb.0:
+; X64-AVX512DQ-NEXT:    vpsllw $5, %xmm0, %xmm0
+; X64-AVX512DQ-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; X64-AVX512DQ-NEXT:    retq
   %1 = mul <16 x i8> %a0, <i8 32, i8 32, i8 32, i8 32, i8 32, i8 32, i8 32, i8 32, i8 32, i8 32, i8 32, i8 32, i8 32, i8 32, i8 32, i8 32>
   ret <16 x i8> %1
 }
@@ -122,6 +129,11 @@ define <2 x i64> @mul_v2i64_32_8(<2 x i64> %a0) nounwind {
 ; X64-AVX2:       # %bb.0:
 ; X64-AVX2-NEXT:    vpsllvq {{.*}}(%rip), %xmm0, %xmm0
 ; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512DQ-LABEL: mul_v2i64_32_8:
+; X64-AVX512DQ:       # %bb.0:
+; X64-AVX512DQ-NEXT:    vpsllvq {{.*}}(%rip), %xmm0, %xmm0
+; X64-AVX512DQ-NEXT:    retq
   %1 = mul <2 x i64> %a0, <i64 32, i64 8>
   ret <2 x i64> %1
 }
@@ -146,6 +158,11 @@ define <4 x i32> @mul_v4i32_1_2_4_8(<4 x i32> %a0) nounwind {
 ; X64-AVX2:       # %bb.0:
 ; X64-AVX2-NEXT:    vpsllvd {{.*}}(%rip), %xmm0, %xmm0
 ; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512DQ-LABEL: mul_v4i32_1_2_4_8:
+; X64-AVX512DQ:       # %bb.0:
+; X64-AVX512DQ-NEXT:    vpsllvd {{.*}}(%rip), %xmm0, %xmm0
+; X64-AVX512DQ-NEXT:    retq
   %1 = mul <4 x i32> %a0, <i32 1, i32 2, i32 4, i32 8>
   ret <4 x i32> %1
 }
@@ -170,6 +187,11 @@ define <8 x i16> @mul_v8i16_1_2_4_8_16_32_64_128(<8 x i16> %a0) nounwind {
 ; X64-AVX2:       # %bb.0:
 ; X64-AVX2-NEXT:    vpmullw {{.*}}(%rip), %xmm0, %xmm0
 ; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512DQ-LABEL: mul_v8i16_1_2_4_8_16_32_64_128:
+; X64-AVX512DQ:       # %bb.0:
+; X64-AVX512DQ-NEXT:    vpmullw {{.*}}(%rip), %xmm0, %xmm0
+; X64-AVX512DQ-NEXT:    retq
   %1 = mul <8 x i16> %a0, <i16 1, i16 2, i16 4, i16 8, i16 16, i16 32, i16 64, i16 128>
   ret <8 x i16> %1
 }
@@ -217,6 +239,14 @@ define <16 x i8> @mul_v16i8_1_2_4_8_1_2_4_8_1_2_4_8_1_2_4_8(<16 x i8> %a0) nounw
 ; X64-AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
 ; X64-AVX2-NEXT:    vzeroupper
 ; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512DQ-LABEL: mul_v16i8_1_2_4_8_1_2_4_8_1_2_4_8_1_2_4_8:
+; X64-AVX512DQ:       # %bb.0:
+; X64-AVX512DQ-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; X64-AVX512DQ-NEXT:    vpsllvd {{.*}}(%rip), %zmm0, %zmm0
+; X64-AVX512DQ-NEXT:    vpmovdb %zmm0, %xmm0
+; X64-AVX512DQ-NEXT:    vzeroupper
+; X64-AVX512DQ-NEXT:    retq
   %1 = mul <16 x i8> %a0, <i8 1, i8 2, i8 4, i8 8, i8 1, i8 2, i8 4, i8 8, i8 1, i8 2, i8 4, i8 8, i8 1, i8 2, i8 4, i8 8>
   ret <16 x i8> %1
 }
@@ -242,11 +272,22 @@ define <2 x i64> @mul_v2i64_17(<2 x i64> %a0) nounwind {
 ; X64-NEXT:    movdqa %xmm1, %xmm0
 ; X64-NEXT:    retq
 ;
-; X64-AVX-LABEL: mul_v2i64_17:
-; X64-AVX:       # %bb.0:
-; X64-AVX-NEXT:    vpsllq $4, %xmm0, %xmm1
-; X64-AVX-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
-; X64-AVX-NEXT:    retq
+; X64-XOP-LABEL: mul_v2i64_17:
+; X64-XOP:       # %bb.0:
+; X64-XOP-NEXT:    vpsllq $4, %xmm0, %xmm1
+; X64-XOP-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
+; X64-XOP-NEXT:    retq
+;
+; X64-AVX2-LABEL: mul_v2i64_17:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    vpsllq $4, %xmm0, %xmm1
+; X64-AVX2-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
+; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512DQ-LABEL: mul_v2i64_17:
+; X64-AVX512DQ:       # %bb.0:
+; X64-AVX512DQ-NEXT:    vpmullq {{.*}}(%rip), %xmm0, %xmm0
+; X64-AVX512DQ-NEXT:    retq
   %1 = mul <2 x i64> %a0, <i64 17, i64 17>
   ret <2 x i64> %1
 }
@@ -272,6 +313,11 @@ define <4 x i32> @mul_v4i32_17(<4 x i32> %a0) nounwind {
 ; X64-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [17,17,17,17]
 ; X64-AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512DQ-LABEL: mul_v4i32_17:
+; X64-AVX512DQ:       # %bb.0:
+; X64-AVX512DQ-NEXT:    vpmulld {{.*}}(%rip){1to4}, %xmm0, %xmm0
+; X64-AVX512DQ-NEXT:    retq
   %1 = mul <4 x i32> %a0, <i32 17, i32 17, i32 17, i32 17>
   ret <4 x i32> %1
 }
@@ -326,10 +372,202 @@ define <16 x i8> @mul_v16i8_17(<16 x i8> %a0) nounwind {
 ; X64-AVX2-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
 ; X64-AVX2-NEXT:    vpaddb %xmm0, %xmm1, %xmm0
 ; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512DQ-LABEL: mul_v16i8_17:
+; X64-AVX512DQ:       # %bb.0:
+; X64-AVX512DQ-NEXT:    vpsllw $4, %xmm0, %xmm1
+; X64-AVX512DQ-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; X64-AVX512DQ-NEXT:    vpaddb %xmm0, %xmm1, %xmm0
+; X64-AVX512DQ-NEXT:    retq
   %1 = mul <16 x i8> %a0, <i8 17, i8 17, i8 17, i8 17, i8 17, i8 17, i8 17, i8 17, i8 17, i8 17, i8 17, i8 17, i8 17, i8 17, i8 17, i8 17>
   ret <16 x i8> %1
 }
 
+define <4 x i64> @mul_v4i64_17(<4 x i64> %a0) nounwind {
+; X86-LABEL: mul_v4i64_17:
+; X86:       # %bb.0:
+; X86-NEXT:    movdqa %xmm0, %xmm2
+; X86-NEXT:    psllq $4, %xmm2
+; X86-NEXT:    paddq %xmm0, %xmm2
+; X86-NEXT:    movdqa %xmm1, %xmm3
+; X86-NEXT:    psllq $4, %xmm3
+; X86-NEXT:    paddq %xmm1, %xmm3
+; X86-NEXT:    movdqa %xmm2, %xmm0
+; X86-NEXT:    movdqa %xmm3, %xmm1
+; X86-NEXT:    retl
+;
+; X64-LABEL: mul_v4i64_17:
+; X64:       # %bb.0:
+; X64-NEXT:    movdqa %xmm0, %xmm2
+; X64-NEXT:    psllq $4, %xmm2
+; X64-NEXT:    paddq %xmm0, %xmm2
+; X64-NEXT:    movdqa %xmm1, %xmm3
+; X64-NEXT:    psllq $4, %xmm3
+; X64-NEXT:    paddq %xmm1, %xmm3
+; X64-NEXT:    movdqa %xmm2, %xmm0
+; X64-NEXT:    movdqa %xmm3, %xmm1
+; X64-NEXT:    retq
+;
+; X64-XOP-LABEL: mul_v4i64_17:
+; X64-XOP:       # %bb.0:
+; X64-XOP-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; X64-XOP-NEXT:    vpsllq $4, %xmm1, %xmm2
+; X64-XOP-NEXT:    vpaddq %xmm1, %xmm2, %xmm1
+; X64-XOP-NEXT:    vpsllq $4, %xmm0, %xmm2
+; X64-XOP-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
+; X64-XOP-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; X64-XOP-NEXT:    retq
+;
+; X64-AVX2-LABEL: mul_v4i64_17:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    vpsllq $4, %ymm0, %ymm1
+; X64-AVX2-NEXT:    vpaddq %ymm0, %ymm1, %ymm0
+; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512DQ-LABEL: mul_v4i64_17:
+; X64-AVX512DQ:       # %bb.0:
+; X64-AVX512DQ-NEXT:    vpmullq {{.*}}(%rip){1to4}, %ymm0, %ymm0
+; X64-AVX512DQ-NEXT:    retq
+  %1 = mul <4 x i64> %a0, <i64 17, i64 17, i64 17, i64 17>
+  ret <4 x i64> %1
+}
+
+define <8 x i32> @mul_v8i32_17(<8 x i32> %a0) nounwind {
+; X86-LABEL: mul_v8i32_17:
+; X86:       # %bb.0:
+; X86-NEXT:    movdqa {{.*#+}} xmm2 = [17,17,17,17]
+; X86-NEXT:    pmulld %xmm2, %xmm0
+; X86-NEXT:    pmulld %xmm2, %xmm1
+; X86-NEXT:    retl
+;
+; X64-LABEL: mul_v8i32_17:
+; X64:       # %bb.0:
+; X64-NEXT:    movdqa {{.*#+}} xmm2 = [17,17,17,17]
+; X64-NEXT:    pmulld %xmm2, %xmm0
+; X64-NEXT:    pmulld %xmm2, %xmm1
+; X64-NEXT:    retq
+;
+; X64-XOP-LABEL: mul_v8i32_17:
+; X64-XOP:       # %bb.0:
+; X64-XOP-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; X64-XOP-NEXT:    vpslld $4, %xmm1, %xmm2
+; X64-XOP-NEXT:    vpaddd %xmm1, %xmm2, %xmm1
+; X64-XOP-NEXT:    vpslld $4, %xmm0, %xmm2
+; X64-XOP-NEXT:    vpaddd %xmm0, %xmm2, %xmm0
+; X64-XOP-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; X64-XOP-NEXT:    retq
+;
+; X64-AVX2-LABEL: mul_v8i32_17:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [17,17,17,17,17,17,17,17]
+; X64-AVX2-NEXT:    vpmulld %ymm1, %ymm0, %ymm0
+; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512DQ-LABEL: mul_v8i32_17:
+; X64-AVX512DQ:       # %bb.0:
+; X64-AVX512DQ-NEXT:    vpmulld {{.*}}(%rip){1to8}, %ymm0, %ymm0
+; X64-AVX512DQ-NEXT:    retq
+  %1 = mul <8 x i32> %a0, <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
+  ret <8 x i32> %1
+}
+
+define <16 x i16> @mul_v16i16_17(<16 x i16> %a0) nounwind {
+; X86-LABEL: mul_v16i16_17:
+; X86:       # %bb.0:
+; X86-NEXT:    movdqa {{.*#+}} xmm2 = [17,17,17,17,17,17,17,17]
+; X86-NEXT:    pmullw %xmm2, %xmm0
+; X86-NEXT:    pmullw %xmm2, %xmm1
+; X86-NEXT:    retl
+;
+; X64-LABEL: mul_v16i16_17:
+; X64:       # %bb.0:
+; X64-NEXT:    movdqa {{.*#+}} xmm2 = [17,17,17,17,17,17,17,17]
+; X64-NEXT:    pmullw %xmm2, %xmm0
+; X64-NEXT:    pmullw %xmm2, %xmm1
+; X64-NEXT:    retq
+;
+; X64-XOP-LABEL: mul_v16i16_17:
+; X64-XOP:       # %bb.0:
+; X64-XOP-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; X64-XOP-NEXT:    vpsllw $4, %xmm1, %xmm2
+; X64-XOP-NEXT:    vpaddw %xmm1, %xmm2, %xmm1
+; X64-XOP-NEXT:    vpsllw $4, %xmm0, %xmm2
+; X64-XOP-NEXT:    vpaddw %xmm0, %xmm2, %xmm0
+; X64-XOP-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; X64-XOP-NEXT:    retq
+;
+; X64-AVX2-LABEL: mul_v16i16_17:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    vpmullw {{.*}}(%rip), %ymm0, %ymm0
+; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512DQ-LABEL: mul_v16i16_17:
+; X64-AVX512DQ:       # %bb.0:
+; X64-AVX512DQ-NEXT:    vpmullw {{.*}}(%rip), %ymm0, %ymm0
+; X64-AVX512DQ-NEXT:    retq
+  %1 = mul <16 x i16> %a0, <i16 17, i16 17, i16 17, i16 17, i16 17, i16 17, i16 17, i16 17, i16 17, i16 17, i16 17, i16 17, i16 17, i16 17, i16 17, i16 17>
+  ret <16 x i16> %1
+}
+
+define <32 x i8> @mul_v32i8_17(<32 x i8> %a0) nounwind {
+; X86-LABEL: mul_v32i8_17:
+; X86:       # %bb.0:
+; X86-NEXT:    movdqa %xmm0, %xmm2
+; X86-NEXT:    psllw $4, %xmm2
+; X86-NEXT:    movdqa {{.*#+}} xmm4 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
+; X86-NEXT:    pand %xmm4, %xmm2
+; X86-NEXT:    paddb %xmm0, %xmm2
+; X86-NEXT:    movdqa %xmm1, %xmm3
+; X86-NEXT:    psllw $4, %xmm3
+; X86-NEXT:    pand %xmm4, %xmm3
+; X86-NEXT:    paddb %xmm1, %xmm3
+; X86-NEXT:    movdqa %xmm2, %xmm0
+; X86-NEXT:    movdqa %xmm3, %xmm1
+; X86-NEXT:    retl
+;
+; X64-LABEL: mul_v32i8_17:
+; X64:       # %bb.0:
+; X64-NEXT:    movdqa %xmm0, %xmm2
+; X64-NEXT:    psllw $4, %xmm2
+; X64-NEXT:    movdqa {{.*#+}} xmm4 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
+; X64-NEXT:    pand %xmm4, %xmm2
+; X64-NEXT:    paddb %xmm0, %xmm2
+; X64-NEXT:    movdqa %xmm1, %xmm3
+; X64-NEXT:    psllw $4, %xmm3
+; X64-NEXT:    pand %xmm4, %xmm3
+; X64-NEXT:    paddb %xmm1, %xmm3
+; X64-NEXT:    movdqa %xmm2, %xmm0
+; X64-NEXT:    movdqa %xmm3, %xmm1
+; X64-NEXT:    retq
+;
+; X64-XOP-LABEL: mul_v32i8_17:
+; X64-XOP:       # %bb.0:
+; X64-XOP-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; X64-XOP-NEXT:    vmovdqa {{.*#+}} xmm2 = [4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4]
+; X64-XOP-NEXT:    vpshlb %xmm2, %xmm1, %xmm3
+; X64-XOP-NEXT:    vpaddb %xmm1, %xmm3, %xmm1
+; X64-XOP-NEXT:    vpshlb %xmm2, %xmm0, %xmm2
+; X64-XOP-NEXT:    vpaddb %xmm0, %xmm2, %xmm0
+; X64-XOP-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; X64-XOP-NEXT:    retq
+;
+; X64-AVX2-LABEL: mul_v32i8_17:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    vpsllw $4, %ymm0, %ymm1
+; X64-AVX2-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
+; X64-AVX2-NEXT:    vpaddb %ymm0, %ymm1, %ymm0
+; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512DQ-LABEL: mul_v32i8_17:
+; X64-AVX512DQ:       # %bb.0:
+; X64-AVX512DQ-NEXT:    vpsllw $4, %ymm0, %ymm1
+; X64-AVX512DQ-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
+; X64-AVX512DQ-NEXT:    vpaddb %ymm0, %ymm1, %ymm0
+; X64-AVX512DQ-NEXT:    retq
+  %1 = mul <32 x i8> %a0, <i8 17, i8 17, i8 17, i8 17, i8 17, i8 17, i8 17, i8 17, i8 17, i8 17, i8 17, i8 17, i8 17, i8 17, i8 17, i8 17, i8 17, i8 17, i8 17, i8 17, i8 17, i8 17, i8 17, i8 17, i8 17, i8 17, i8 17, i8 17, i8 17, i8 17, i8 17, i8 17>
+  ret <32 x i8> %1
+}
+
 ;
 ; -(PowOf2 + 1) (uniform)
 ;
@@ -353,13 +591,26 @@ define <2 x i64> @mul_v2i64_neg1025(<2 x i64> %a0) nounwind {
 ; X64-NEXT:    psubq %xmm1, %xmm0
 ; X64-NEXT:    retq
 ;
-; X64-AVX-LABEL: mul_v2i64_neg1025:
-; X64-AVX:       # %bb.0:
-; X64-AVX-NEXT:    vpsllq $10, %xmm0, %xmm1
-; X64-AVX-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
-; X64-AVX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; X64-AVX-NEXT:    vpsubq %xmm0, %xmm1, %xmm0
-; X64-AVX-NEXT:    retq
+; X64-XOP-LABEL: mul_v2i64_neg1025:
+; X64-XOP:       # %bb.0:
+; X64-XOP-NEXT:    vpsllq $10, %xmm0, %xmm1
+; X64-XOP-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
+; X64-XOP-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; X64-XOP-NEXT:    vpsubq %xmm0, %xmm1, %xmm0
+; X64-XOP-NEXT:    retq
+;
+; X64-AVX2-LABEL: mul_v2i64_neg1025:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    vpsllq $10, %xmm0, %xmm1
+; X64-AVX2-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
+; X64-AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX2-NEXT:    vpsubq %xmm0, %xmm1, %xmm0
+; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512DQ-LABEL: mul_v2i64_neg1025:
+; X64-AVX512DQ:       # %bb.0:
+; X64-AVX512DQ-NEXT:    vpmullq {{.*}}(%rip), %xmm0, %xmm0
+; X64-AVX512DQ-NEXT:    retq
   %1 = mul <2 x i64> %a0, <i64 -1025, i64 -1025>
   ret <2 x i64> %1
 }
@@ -385,6 +636,11 @@ define <4 x i32> @mul_v4i32_neg33(<4 x i32> %a0) nounwind {
 ; X64-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [4294967263,4294967263,4294967263,4294967263]
 ; X64-AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512DQ-LABEL: mul_v4i32_neg33:
+; X64-AVX512DQ:       # %bb.0:
+; X64-AVX512DQ-NEXT:    vpmulld {{.*}}(%rip){1to4}, %xmm0, %xmm0
+; X64-AVX512DQ-NEXT:    retq
   %1 = mul <4 x i32> %a0, <i32 -33, i32 -33, i32 -33, i32 -33>
   ret <4 x i32> %1
 }
@@ -445,10 +701,234 @@ define <16 x i8> @mul_v16i8_neg5(<16 x i8> %a0) nounwind {
 ; X64-AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; X64-AVX2-NEXT:    vpsubb %xmm0, %xmm1, %xmm0
 ; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512DQ-LABEL: mul_v16i8_neg5:
+; X64-AVX512DQ:       # %bb.0:
+; X64-AVX512DQ-NEXT:    vpsllw $2, %xmm0, %xmm1
+; X64-AVX512DQ-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; X64-AVX512DQ-NEXT:    vpaddb %xmm0, %xmm1, %xmm0
+; X64-AVX512DQ-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX512DQ-NEXT:    vpsubb %xmm0, %xmm1, %xmm0
+; X64-AVX512DQ-NEXT:    retq
   %1 = mul <16 x i8> %a0, <i8 -5, i8 -5, i8 -5, i8 -5, i8 -5, i8 -5, i8 -5, i8 -5, i8 -5, i8 -5, i8 -5, i8 -5, i8 -5, i8 -5, i8 -5, i8 -5>
   ret <16 x i8> %1
 }
 
+define <4 x i64> @mul_v4i64_neg1025(<4 x i64> %a0) nounwind {
+; X86-LABEL: mul_v4i64_neg1025:
+; X86:       # %bb.0:
+; X86-NEXT:    movdqa %xmm0, %xmm3
+; X86-NEXT:    psllq $10, %xmm3
+; X86-NEXT:    paddq %xmm0, %xmm3
+; X86-NEXT:    pxor %xmm2, %xmm2
+; X86-NEXT:    pxor %xmm0, %xmm0
+; X86-NEXT:    psubq %xmm3, %xmm0
+; X86-NEXT:    movdqa %xmm1, %xmm3
+; X86-NEXT:    psllq $10, %xmm3
+; X86-NEXT:    paddq %xmm1, %xmm3
+; X86-NEXT:    psubq %xmm3, %xmm2
+; X86-NEXT:    movdqa %xmm2, %xmm1
+; X86-NEXT:    retl
+;
+; X64-LABEL: mul_v4i64_neg1025:
+; X64:       # %bb.0:
+; X64-NEXT:    movdqa %xmm0, %xmm3
+; X64-NEXT:    psllq $10, %xmm3
+; X64-NEXT:    paddq %xmm0, %xmm3
+; X64-NEXT:    pxor %xmm2, %xmm2
+; X64-NEXT:    pxor %xmm0, %xmm0
+; X64-NEXT:    psubq %xmm3, %xmm0
+; X64-NEXT:    movdqa %xmm1, %xmm3
+; X64-NEXT:    psllq $10, %xmm3
+; X64-NEXT:    paddq %xmm1, %xmm3
+; X64-NEXT:    psubq %xmm3, %xmm2
+; X64-NEXT:    movdqa %xmm2, %xmm1
+; X64-NEXT:    retq
+;
+; X64-XOP-LABEL: mul_v4i64_neg1025:
+; X64-XOP:       # %bb.0:
+; X64-XOP-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; X64-XOP-NEXT:    vpsllq $10, %xmm1, %xmm2
+; X64-XOP-NEXT:    vpaddq %xmm1, %xmm2, %xmm1
+; X64-XOP-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; X64-XOP-NEXT:    vpsubq %xmm1, %xmm2, %xmm1
+; X64-XOP-NEXT:    vpsllq $10, %xmm0, %xmm3
+; X64-XOP-NEXT:    vpaddq %xmm0, %xmm3, %xmm0
+; X64-XOP-NEXT:    vpsubq %xmm0, %xmm2, %xmm0
+; X64-XOP-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; X64-XOP-NEXT:    retq
+;
+; X64-AVX2-LABEL: mul_v4i64_neg1025:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    vpsllq $10, %ymm0, %ymm1
+; X64-AVX2-NEXT:    vpaddq %ymm0, %ymm1, %ymm0
+; X64-AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX2-NEXT:    vpsubq %ymm0, %ymm1, %ymm0
+; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512DQ-LABEL: mul_v4i64_neg1025:
+; X64-AVX512DQ:       # %bb.0:
+; X64-AVX512DQ-NEXT:    vpmullq {{.*}}(%rip){1to4}, %ymm0, %ymm0
+; X64-AVX512DQ-NEXT:    retq
+  %1 = mul <4 x i64> %a0, <i64 -1025, i64 -1025, i64 -1025, i64 -1025>
+  ret <4 x i64> %1
+}
+
+define <8 x i32> @mul_v8i32_neg33(<8 x i32> %a0) nounwind {
+; X86-LABEL: mul_v8i32_neg33:
+; X86:       # %bb.0:
+; X86-NEXT:    movdqa {{.*#+}} xmm2 = [4294967263,4294967263,4294967263,4294967263]
+; X86-NEXT:    pmulld %xmm2, %xmm0
+; X86-NEXT:    pmulld %xmm2, %xmm1
+; X86-NEXT:    retl
+;
+; X64-LABEL: mul_v8i32_neg33:
+; X64:       # %bb.0:
+; X64-NEXT:    movdqa {{.*#+}} xmm2 = [4294967263,4294967263,4294967263,4294967263]
+; X64-NEXT:    pmulld %xmm2, %xmm0
+; X64-NEXT:    pmulld %xmm2, %xmm1
+; X64-NEXT:    retq
+;
+; X64-XOP-LABEL: mul_v8i32_neg33:
+; X64-XOP:       # %bb.0:
+; X64-XOP-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; X64-XOP-NEXT:    vpslld $5, %xmm1, %xmm2
+; X64-XOP-NEXT:    vpaddd %xmm1, %xmm2, %xmm1
+; X64-XOP-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; X64-XOP-NEXT:    vpsubd %xmm1, %xmm2, %xmm1
+; X64-XOP-NEXT:    vpslld $5, %xmm0, %xmm3
+; X64-XOP-NEXT:    vpaddd %xmm0, %xmm3, %xmm0
+; X64-XOP-NEXT:    vpsubd %xmm0, %xmm2, %xmm0
+; X64-XOP-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; X64-XOP-NEXT:    retq
+;
+; X64-AVX2-LABEL: mul_v8i32_neg33:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [4294967263,4294967263,4294967263,4294967263,4294967263,4294967263,4294967263,4294967263]
+; X64-AVX2-NEXT:    vpmulld %ymm1, %ymm0, %ymm0
+; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512DQ-LABEL: mul_v8i32_neg33:
+; X64-AVX512DQ:       # %bb.0:
+; X64-AVX512DQ-NEXT:    vpmulld {{.*}}(%rip){1to8}, %ymm0, %ymm0
+; X64-AVX512DQ-NEXT:    retq
+  %1 = mul <8 x i32> %a0, <i32 -33, i32 -33, i32 -33, i32 -33, i32 -33, i32 -33, i32 -33, i32 -33>
+  ret <8 x i32> %1
+}
+
+define <16 x i16> @mul_v16i16_neg9(<16 x i16> %a0) nounwind {
+; X86-LABEL: mul_v16i16_neg9:
+; X86:       # %bb.0:
+; X86-NEXT:    movdqa {{.*#+}} xmm2 = [65527,65527,65527,65527,65527,65527,65527,65527]
+; X86-NEXT:    pmullw %xmm2, %xmm0
+; X86-NEXT:    pmullw %xmm2, %xmm1
+; X86-NEXT:    retl
+;
+; X64-LABEL: mul_v16i16_neg9:
+; X64:       # %bb.0:
+; X64-NEXT:    movdqa {{.*#+}} xmm2 = [65527,65527,65527,65527,65527,65527,65527,65527]
+; X64-NEXT:    pmullw %xmm2, %xmm0
+; X64-NEXT:    pmullw %xmm2, %xmm1
+; X64-NEXT:    retq
+;
+; X64-XOP-LABEL: mul_v16i16_neg9:
+; X64-XOP:       # %bb.0:
+; X64-XOP-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; X64-XOP-NEXT:    vpsllw $3, %xmm1, %xmm2
+; X64-XOP-NEXT:    vpaddw %xmm1, %xmm2, %xmm1
+; X64-XOP-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; X64-XOP-NEXT:    vpsubw %xmm1, %xmm2, %xmm1
+; X64-XOP-NEXT:    vpsllw $3, %xmm0, %xmm3
+; X64-XOP-NEXT:    vpaddw %xmm0, %xmm3, %xmm0
+; X64-XOP-NEXT:    vpsubw %xmm0, %xmm2, %xmm0
+; X64-XOP-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; X64-XOP-NEXT:    retq
+;
+; X64-AVX2-LABEL: mul_v16i16_neg9:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    vpmullw {{.*}}(%rip), %ymm0, %ymm0
+; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512DQ-LABEL: mul_v16i16_neg9:
+; X64-AVX512DQ:       # %bb.0:
+; X64-AVX512DQ-NEXT:    vpmullw {{.*}}(%rip), %ymm0, %ymm0
+; X64-AVX512DQ-NEXT:    retq
+  %1 = mul <16 x i16> %a0, <i16 -9, i16 -9, i16 -9, i16 -9, i16 -9, i16 -9, i16 -9, i16 -9, i16 -9, i16 -9, i16 -9, i16 -9, i16 -9, i16 -9, i16 -9, i16 -9>
+  ret <16 x i16> %1
+}
+
+define <32 x i8> @mul_v32i8_neg5(<32 x i8> %a0) nounwind {
+; X86-LABEL: mul_v32i8_neg5:
+; X86:       # %bb.0:
+; X86-NEXT:    movdqa %xmm0, %xmm3
+; X86-NEXT:    psllw $2, %xmm3
+; X86-NEXT:    movdqa {{.*#+}} xmm4 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
+; X86-NEXT:    pand %xmm4, %xmm3
+; X86-NEXT:    paddb %xmm0, %xmm3
+; X86-NEXT:    pxor %xmm2, %xmm2
+; X86-NEXT:    pxor %xmm0, %xmm0
+; X86-NEXT:    psubb %xmm3, %xmm0
+; X86-NEXT:    movdqa %xmm1, %xmm3
+; X86-NEXT:    psllw $2, %xmm3
+; X86-NEXT:    pand %xmm4, %xmm3
+; X86-NEXT:    paddb %xmm1, %xmm3
+; X86-NEXT:    psubb %xmm3, %xmm2
+; X86-NEXT:    movdqa %xmm2, %xmm1
+; X86-NEXT:    retl
+;
+; X64-LABEL: mul_v32i8_neg5:
+; X64:       # %bb.0:
+; X64-NEXT:    movdqa %xmm0, %xmm3
+; X64-NEXT:    psllw $2, %xmm3
+; X64-NEXT:    movdqa {{.*#+}} xmm4 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
+; X64-NEXT:    pand %xmm4, %xmm3
+; X64-NEXT:    paddb %xmm0, %xmm3
+; X64-NEXT:    pxor %xmm2, %xmm2
+; X64-NEXT:    pxor %xmm0, %xmm0
+; X64-NEXT:    psubb %xmm3, %xmm0
+; X64-NEXT:    movdqa %xmm1, %xmm3
+; X64-NEXT:    psllw $2, %xmm3
+; X64-NEXT:    pand %xmm4, %xmm3
+; X64-NEXT:    paddb %xmm1, %xmm3
+; X64-NEXT:    psubb %xmm3, %xmm2
+; X64-NEXT:    movdqa %xmm2, %xmm1
+; X64-NEXT:    retq
+;
+; X64-XOP-LABEL: mul_v32i8_neg5:
+; X64-XOP:       # %bb.0:
+; X64-XOP-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; X64-XOP-NEXT:    vmovdqa {{.*#+}} xmm2 = [2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2]
+; X64-XOP-NEXT:    vpshlb %xmm2, %xmm1, %xmm3
+; X64-XOP-NEXT:    vpaddb %xmm1, %xmm3, %xmm1
+; X64-XOP-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; X64-XOP-NEXT:    vpsubb %xmm1, %xmm3, %xmm1
+; X64-XOP-NEXT:    vpshlb %xmm2, %xmm0, %xmm2
+; X64-XOP-NEXT:    vpaddb %xmm0, %xmm2, %xmm0
+; X64-XOP-NEXT:    vpsubb %xmm0, %xmm3, %xmm0
+; X64-XOP-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; X64-XOP-NEXT:    retq
+;
+; X64-AVX2-LABEL: mul_v32i8_neg5:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    vpsllw $2, %ymm0, %ymm1
+; X64-AVX2-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
+; X64-AVX2-NEXT:    vpaddb %ymm0, %ymm1, %ymm0
+; X64-AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX2-NEXT:    vpsubb %ymm0, %ymm1, %ymm0
+; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512DQ-LABEL: mul_v32i8_neg5:
+; X64-AVX512DQ:       # %bb.0:
+; X64-AVX512DQ-NEXT:    vpsllw $2, %ymm0, %ymm1
+; X64-AVX512DQ-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
+; X64-AVX512DQ-NEXT:    vpaddb %ymm0, %ymm1, %ymm0
+; X64-AVX512DQ-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX512DQ-NEXT:    vpsubb %ymm0, %ymm1, %ymm0
+; X64-AVX512DQ-NEXT:    retq
+  %1 = mul <32 x i8> %a0, <i8 -5, i8 -5, i8 -5, i8 -5, i8 -5, i8 -5, i8 -5, i8 -5, i8 -5, i8 -5, i8 -5, i8 -5, i8 -5, i8 -5, i8 -5, i8 -5, i8 -5, i8 -5, i8 -5, i8 -5, i8 -5, i8 -5, i8 -5, i8 -5, i8 -5, i8 -5, i8 -5, i8 -5, i8 -5, i8 -5, i8 -5, i8 -5>
+  ret <32 x i8> %1
+}
+
 ;
 ; PowOf2 + 1 (non-uniform)
 ;
@@ -476,15 +956,30 @@ define <2 x i64> @mul_v2i64_17_65(<2 x i64> %a0) nounwind {
 ; X64-NEXT:    paddq %xmm2, %xmm0
 ; X64-NEXT:    retq
 ;
-; X64-AVX-LABEL: mul_v2i64_17_65:
-; X64-AVX:       # %bb.0:
-; X64-AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [17,65]
-; X64-AVX-NEXT:    vpmuludq %xmm1, %xmm0, %xmm2
-; X64-AVX-NEXT:    vpsrlq $32, %xmm0, %xmm0
-; X64-AVX-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
-; X64-AVX-NEXT:    vpsllq $32, %xmm0, %xmm0
-; X64-AVX-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
-; X64-AVX-NEXT:    retq
+; X64-XOP-LABEL: mul_v2i64_17_65:
+; X64-XOP:       # %bb.0:
+; X64-XOP-NEXT:    vmovdqa {{.*#+}} xmm1 = [17,65]
+; X64-XOP-NEXT:    vpmuludq %xmm1, %xmm0, %xmm2
+; X64-XOP-NEXT:    vpsrlq $32, %xmm0, %xmm0
+; X64-XOP-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
+; X64-XOP-NEXT:    vpsllq $32, %xmm0, %xmm0
+; X64-XOP-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
+; X64-XOP-NEXT:    retq
+;
+; X64-AVX2-LABEL: mul_v2i64_17_65:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [17,65]
+; X64-AVX2-NEXT:    vpmuludq %xmm1, %xmm0, %xmm2
+; X64-AVX2-NEXT:    vpsrlq $32, %xmm0, %xmm0
+; X64-AVX2-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT:    vpsllq $32, %xmm0, %xmm0
+; X64-AVX2-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
+; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512DQ-LABEL: mul_v2i64_17_65:
+; X64-AVX512DQ:       # %bb.0:
+; X64-AVX512DQ-NEXT:    vpmullq {{.*}}(%rip), %xmm0, %xmm0
+; X64-AVX512DQ-NEXT:    retq
   %1 = mul <2 x i64> %a0, <i64 17, i64 65>
   ret <2 x i64> %1
 }
@@ -572,6 +1067,15 @@ define <16 x i8> @mul_v16i8_2_3_9_17_33_65_129_2_3_9_17_33_65_129_2_3(<16 x i8>
 ; X64-AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
 ; X64-AVX2-NEXT:    vzeroupper
 ; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512DQ-LABEL: mul_v16i8_2_3_9_17_33_65_129_2_3_9_17_33_65_129_2_3:
+; X64-AVX512DQ:       # %bb.0:
+; X64-AVX512DQ-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; X64-AVX512DQ-NEXT:    vpmullw {{.*}}(%rip), %ymm0, %ymm0
+; X64-AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
+; X64-AVX512DQ-NEXT:    vpmovdb %zmm0, %xmm0
+; X64-AVX512DQ-NEXT:    vzeroupper
+; X64-AVX512DQ-NEXT:    retq
   %1 = mul <16 x i8> %a0, <i8 2, i8 3, i8 9, i8 17, i8 33, i8 65, i8 129, i8 2, i8 3, i8 9, i8 17, i8 33, i8 65, i8 129, i8 2, i8 3>
   ret <16 x i8> %1
 }
@@ -597,11 +1101,22 @@ define <2 x i64> @mul_v2i64_7(<2 x i64> %a0) nounwind {
 ; X64-NEXT:    movdqa %xmm1, %xmm0
 ; X64-NEXT:    retq
 ;
-; X64-AVX-LABEL: mul_v2i64_7:
-; X64-AVX:       # %bb.0:
-; X64-AVX-NEXT:    vpsllq $3, %xmm0, %xmm1
-; X64-AVX-NEXT:    vpsubq %xmm0, %xmm1, %xmm0
-; X64-AVX-NEXT:    retq
+; X64-XOP-LABEL: mul_v2i64_7:
+; X64-XOP:       # %bb.0:
+; X64-XOP-NEXT:    vpsllq $3, %xmm0, %xmm1
+; X64-XOP-NEXT:    vpsubq %xmm0, %xmm1, %xmm0
+; X64-XOP-NEXT:    retq
+;
+; X64-AVX2-LABEL: mul_v2i64_7:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    vpsllq $3, %xmm0, %xmm1
+; X64-AVX2-NEXT:    vpsubq %xmm0, %xmm1, %xmm0
+; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512DQ-LABEL: mul_v2i64_7:
+; X64-AVX512DQ:       # %bb.0:
+; X64-AVX512DQ-NEXT:    vpmullq {{.*}}(%rip), %xmm0, %xmm0
+; X64-AVX512DQ-NEXT:    retq
   %1 = mul <2 x i64> %a0, <i64 7, i64 7>
   ret <2 x i64> %1
 }
@@ -627,6 +1142,11 @@ define <4 x i32> @mul_v4i32_7(<4 x i32> %a0) nounwind {
 ; X64-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [7,7,7,7]
 ; X64-AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512DQ-LABEL: mul_v4i32_7:
+; X64-AVX512DQ:       # %bb.0:
+; X64-AVX512DQ-NEXT:    vpmulld {{.*}}(%rip){1to4}, %xmm0, %xmm0
+; X64-AVX512DQ-NEXT:    retq
   %1 = mul <4 x i32> %a0, <i32 7, i32 7, i32 7, i32 7>
   ret <4 x i32> %1
 }
@@ -681,6 +1201,13 @@ define <16 x i8> @mul_v16i8_31(<16 x i8> %a0) nounwind {
 ; X64-AVX2-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
 ; X64-AVX2-NEXT:    vpsubb %xmm0, %xmm1, %xmm0
 ; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512DQ-LABEL: mul_v16i8_31:
+; X64-AVX512DQ:       # %bb.0:
+; X64-AVX512DQ-NEXT:    vpsllw $5, %xmm0, %xmm1
+; X64-AVX512DQ-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; X64-AVX512DQ-NEXT:    vpsubb %xmm0, %xmm1, %xmm0
+; X64-AVX512DQ-NEXT:    retq
   %1 = mul <16 x i8> %a0, <i8 31, i8 31, i8 31, i8 31, i8 31, i8 31, i8 31, i8 31, i8 31, i8 31, i8 31, i8 31, i8 31, i8 31, i8 31, i8 31>
   ret <16 x i8> %1
 }
@@ -704,11 +1231,22 @@ define <2 x i64> @mul_v2i64_neg7(<2 x i64> %a0) nounwind {
 ; X64-NEXT:    psubq %xmm1, %xmm0
 ; X64-NEXT:    retq
 ;
-; X64-AVX-LABEL: mul_v2i64_neg7:
-; X64-AVX:       # %bb.0:
-; X64-AVX-NEXT:    vpsllq $3, %xmm0, %xmm1
-; X64-AVX-NEXT:    vpsubq %xmm1, %xmm0, %xmm0
-; X64-AVX-NEXT:    retq
+; X64-XOP-LABEL: mul_v2i64_neg7:
+; X64-XOP:       # %bb.0:
+; X64-XOP-NEXT:    vpsllq $3, %xmm0, %xmm1
+; X64-XOP-NEXT:    vpsubq %xmm1, %xmm0, %xmm0
+; X64-XOP-NEXT:    retq
+;
+; X64-AVX2-LABEL: mul_v2i64_neg7:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    vpsllq $3, %xmm0, %xmm1
+; X64-AVX2-NEXT:    vpsubq %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512DQ-LABEL: mul_v2i64_neg7:
+; X64-AVX512DQ:       # %bb.0:
+; X64-AVX512DQ-NEXT:    vpmullq {{.*}}(%rip), %xmm0, %xmm0
+; X64-AVX512DQ-NEXT:    retq
   %1 = mul <2 x i64> %a0, <i64 -7, i64 -7>
   ret <2 x i64> %1
 }
@@ -734,6 +1272,11 @@ define <4 x i32> @mul_v4i32_neg63(<4 x i32> %a0) nounwind {
 ; X64-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [4294967233,4294967233,4294967233,4294967233]
 ; X64-AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512DQ-LABEL: mul_v4i32_neg63:
+; X64-AVX512DQ:       # %bb.0:
+; X64-AVX512DQ-NEXT:    vpmulld {{.*}}(%rip){1to4}, %xmm0, %xmm0
+; X64-AVX512DQ-NEXT:    retq
   %1 = mul <4 x i32> %a0, <i32 -63, i32 -63, i32 -63, i32 -63>
   ret <4 x i32> %1
 }
@@ -786,6 +1329,13 @@ define <16 x i8> @mul_v16i8_neg15(<16 x i8> %a0) nounwind {
 ; X64-AVX2-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
 ; X64-AVX2-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
 ; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512DQ-LABEL: mul_v16i8_neg15:
+; X64-AVX512DQ:       # %bb.0:
+; X64-AVX512DQ-NEXT:    vpsllw $4, %xmm0, %xmm1
+; X64-AVX512DQ-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; X64-AVX512DQ-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
+; X64-AVX512DQ-NEXT:    retq
   %1 = mul <16 x i8> %a0, <i8 -15, i8 -15, i8 -15, i8 -15, i8 -15, i8 -15, i8 -15, i8 -15, i8 -15, i8 -15, i8 -15, i8 -15, i8 -15, i8 -15, i8 -15, i8 -15>
   ret <16 x i8> %1
 }
@@ -817,15 +1367,30 @@ define <2 x i64> @mul_v2i64_15_63(<2 x i64> %a0) nounwind {
 ; X64-NEXT:    paddq %xmm2, %xmm0
 ; X64-NEXT:    retq
 ;
-; X64-AVX-LABEL: mul_v2i64_15_63:
-; X64-AVX:       # %bb.0:
-; X64-AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [15,63]
-; X64-AVX-NEXT:    vpmuludq %xmm1, %xmm0, %xmm2
-; X64-AVX-NEXT:    vpsrlq $32, %xmm0, %xmm0
-; X64-AVX-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
-; X64-AVX-NEXT:    vpsllq $32, %xmm0, %xmm0
-; X64-AVX-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
-; X64-AVX-NEXT:    retq
+; X64-XOP-LABEL: mul_v2i64_15_63:
+; X64-XOP:       # %bb.0:
+; X64-XOP-NEXT:    vmovdqa {{.*#+}} xmm1 = [15,63]
+; X64-XOP-NEXT:    vpmuludq %xmm1, %xmm0, %xmm2
+; X64-XOP-NEXT:    vpsrlq $32, %xmm0, %xmm0
+; X64-XOP-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
+; X64-XOP-NEXT:    vpsllq $32, %xmm0, %xmm0
+; X64-XOP-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
+; X64-XOP-NEXT:    retq
+;
+; X64-AVX2-LABEL: mul_v2i64_15_63:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [15,63]
+; X64-AVX2-NEXT:    vpmuludq %xmm1, %xmm0, %xmm2
+; X64-AVX2-NEXT:    vpsrlq $32, %xmm0, %xmm0
+; X64-AVX2-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT:    vpsllq $32, %xmm0, %xmm0
+; X64-AVX2-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
+; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512DQ-LABEL: mul_v2i64_15_63:
+; X64-AVX512DQ:       # %bb.0:
+; X64-AVX512DQ-NEXT:    vpmullq {{.*}}(%rip), %xmm0, %xmm0
+; X64-AVX512DQ-NEXT:    retq
   %1 = mul <2 x i64> %a0, <i64 15, i64 63>
   ret <2 x i64> %1
 }
@@ -859,17 +1424,34 @@ define <2 x i64> @mul_v2i64_neg_15_63(<2 x i64> %a0) nounwind {
 ; X64-NEXT:    paddq %xmm2, %xmm0
 ; X64-NEXT:    retq
 ;
-; X64-AVX-LABEL: mul_v2i64_neg_15_63:
-; X64-AVX:       # %bb.0:
-; X64-AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [18446744073709551601,18446744073709551553]
-; X64-AVX-NEXT:    vpmuludq %xmm1, %xmm0, %xmm2
-; X64-AVX-NEXT:    vpsrlq $32, %xmm0, %xmm3
-; X64-AVX-NEXT:    vpmuludq %xmm1, %xmm3, %xmm1
-; X64-AVX-NEXT:    vpmuludq {{.*}}(%rip), %xmm0, %xmm0
-; X64-AVX-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
-; X64-AVX-NEXT:    vpsllq $32, %xmm0, %xmm0
-; X64-AVX-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
-; X64-AVX-NEXT:    retq
+; X64-XOP-LABEL: mul_v2i64_neg_15_63:
+; X64-XOP:       # %bb.0:
+; X64-XOP-NEXT:    vmovdqa {{.*#+}} xmm1 = [18446744073709551601,18446744073709551553]
+; X64-XOP-NEXT:    vpmuludq %xmm1, %xmm0, %xmm2
+; X64-XOP-NEXT:    vpsrlq $32, %xmm0, %xmm3
+; X64-XOP-NEXT:    vpmuludq %xmm1, %xmm3, %xmm1
+; X64-XOP-NEXT:    vpmuludq {{.*}}(%rip), %xmm0, %xmm0
+; X64-XOP-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
+; X64-XOP-NEXT:    vpsllq $32, %xmm0, %xmm0
+; X64-XOP-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
+; X64-XOP-NEXT:    retq
+;
+; X64-AVX2-LABEL: mul_v2i64_neg_15_63:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [18446744073709551601,18446744073709551553]
+; X64-AVX2-NEXT:    vpmuludq %xmm1, %xmm0, %xmm2
+; X64-AVX2-NEXT:    vpsrlq $32, %xmm0, %xmm3
+; X64-AVX2-NEXT:    vpmuludq %xmm1, %xmm3, %xmm1
+; X64-AVX2-NEXT:    vpmuludq {{.*}}(%rip), %xmm0, %xmm0
+; X64-AVX2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT:    vpsllq $32, %xmm0, %xmm0
+; X64-AVX2-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
+; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512DQ-LABEL: mul_v2i64_neg_15_63:
+; X64-AVX512DQ:       # %bb.0:
+; X64-AVX512DQ-NEXT:    vpmullq {{.*}}(%rip), %xmm0, %xmm0
+; X64-AVX512DQ-NEXT:    retq
   %1 = mul <2 x i64> %a0, <i64 -15, i64 -63>
   ret <2 x i64> %1
 }
@@ -903,17 +1485,34 @@ define <2 x i64> @mul_v2i64_neg_17_65(<2 x i64> %a0) nounwind {
 ; X64-NEXT:    paddq %xmm2, %xmm0
 ; X64-NEXT:    retq
 ;
-; X64-AVX-LABEL: mul_v2i64_neg_17_65:
-; X64-AVX:       # %bb.0:
-; X64-AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [18446744073709551599,18446744073709551551]
-; X64-AVX-NEXT:    vpmuludq %xmm1, %xmm0, %xmm2
-; X64-AVX-NEXT:    vpsrlq $32, %xmm0, %xmm3
-; X64-AVX-NEXT:    vpmuludq %xmm1, %xmm3, %xmm1
-; X64-AVX-NEXT:    vpmuludq {{.*}}(%rip), %xmm0, %xmm0
-; X64-AVX-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
-; X64-AVX-NEXT:    vpsllq $32, %xmm0, %xmm0
-; X64-AVX-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
-; X64-AVX-NEXT:    retq
+; X64-XOP-LABEL: mul_v2i64_neg_17_65:
+; X64-XOP:       # %bb.0:
+; X64-XOP-NEXT:    vmovdqa {{.*#+}} xmm1 = [18446744073709551599,18446744073709551551]
+; X64-XOP-NEXT:    vpmuludq %xmm1, %xmm0, %xmm2
+; X64-XOP-NEXT:    vpsrlq $32, %xmm0, %xmm3
+; X64-XOP-NEXT:    vpmuludq %xmm1, %xmm3, %xmm1
+; X64-XOP-NEXT:    vpmuludq {{.*}}(%rip), %xmm0, %xmm0
+; X64-XOP-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
+; X64-XOP-NEXT:    vpsllq $32, %xmm0, %xmm0
+; X64-XOP-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
+; X64-XOP-NEXT:    retq
+;
+; X64-AVX2-LABEL: mul_v2i64_neg_17_65:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [18446744073709551599,18446744073709551551]
+; X64-AVX2-NEXT:    vpmuludq %xmm1, %xmm0, %xmm2
+; X64-AVX2-NEXT:    vpsrlq $32, %xmm0, %xmm3
+; X64-AVX2-NEXT:    vpmuludq %xmm1, %xmm3, %xmm1
+; X64-AVX2-NEXT:    vpmuludq {{.*}}(%rip), %xmm0, %xmm0
+; X64-AVX2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT:    vpsllq $32, %xmm0, %xmm0
+; X64-AVX2-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
+; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512DQ-LABEL: mul_v2i64_neg_17_65:
+; X64-AVX512DQ:       # %bb.0:
+; X64-AVX512DQ-NEXT:    vpmullq {{.*}}(%rip), %xmm0, %xmm0
+; X64-AVX512DQ-NEXT:    retq
   %1 = mul <2 x i64> %a0, <i64 -17, i64 -65>
   ret <2 x i64> %1
 }
@@ -941,15 +1540,30 @@ define <2 x i64> @mul_v2i64_0_1(<2 x i64> %a0) nounwind {
 ; X64-NEXT:    paddq %xmm2, %xmm0
 ; X64-NEXT:    retq
 ;
-; X64-AVX-LABEL: mul_v2i64_0_1:
-; X64-AVX:       # %bb.0:
-; X64-AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0]
-; X64-AVX-NEXT:    vpmuludq %xmm1, %xmm0, %xmm2
-; X64-AVX-NEXT:    vpsrlq $32, %xmm0, %xmm0
-; X64-AVX-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
-; X64-AVX-NEXT:    vpsllq $32, %xmm0, %xmm0
-; X64-AVX-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
-; X64-AVX-NEXT:    retq
+; X64-XOP-LABEL: mul_v2i64_0_1:
+; X64-XOP:       # %bb.0:
+; X64-XOP-NEXT:    vmovdqa {{.*#+}} xmm1 = [0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0]
+; X64-XOP-NEXT:    vpmuludq %xmm1, %xmm0, %xmm2
+; X64-XOP-NEXT:    vpsrlq $32, %xmm0, %xmm0
+; X64-XOP-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
+; X64-XOP-NEXT:    vpsllq $32, %xmm0, %xmm0
+; X64-XOP-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
+; X64-XOP-NEXT:    retq
+;
+; X64-AVX2-LABEL: mul_v2i64_0_1:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0]
+; X64-AVX2-NEXT:    vpmuludq %xmm1, %xmm0, %xmm2
+; X64-AVX2-NEXT:    vpsrlq $32, %xmm0, %xmm0
+; X64-AVX2-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT:    vpsllq $32, %xmm0, %xmm0
+; X64-AVX2-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
+; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512DQ-LABEL: mul_v2i64_0_1:
+; X64-AVX512DQ:       # %bb.0:
+; X64-AVX512DQ-NEXT:    vpmullq {{.*}}(%rip), %xmm0, %xmm0
+; X64-AVX512DQ-NEXT:    retq
   %1 = mul <2 x i64> %a0, <i64 0, i64 1>
   ret <2 x i64> %1
 }
@@ -984,17 +1598,34 @@ define <2 x i64> @mul_v2i64_neg_0_1(<2 x i64> %a0) nounwind {
 ; X64-NEXT:    paddq %xmm2, %xmm0
 ; X64-NEXT:    retq
 ;
-; X64-AVX-LABEL: mul_v2i64_neg_0_1:
-; X64-AVX:       # %bb.0:
-; X64-AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255]
-; X64-AVX-NEXT:    vpmuludq %xmm1, %xmm0, %xmm2
-; X64-AVX-NEXT:    vpsrlq $32, %xmm0, %xmm3
-; X64-AVX-NEXT:    vpmuludq %xmm1, %xmm3, %xmm1
-; X64-AVX-NEXT:    vpmuludq {{.*}}(%rip), %xmm0, %xmm0
-; X64-AVX-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
-; X64-AVX-NEXT:    vpsllq $32, %xmm0, %xmm0
-; X64-AVX-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
-; X64-AVX-NEXT:    retq
+; X64-XOP-LABEL: mul_v2i64_neg_0_1:
+; X64-XOP:       # %bb.0:
+; X64-XOP-NEXT:    vmovdqa {{.*#+}} xmm1 = [0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255]
+; X64-XOP-NEXT:    vpmuludq %xmm1, %xmm0, %xmm2
+; X64-XOP-NEXT:    vpsrlq $32, %xmm0, %xmm3
+; X64-XOP-NEXT:    vpmuludq %xmm1, %xmm3, %xmm1
+; X64-XOP-NEXT:    vpmuludq {{.*}}(%rip), %xmm0, %xmm0
+; X64-XOP-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
+; X64-XOP-NEXT:    vpsllq $32, %xmm0, %xmm0
+; X64-XOP-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
+; X64-XOP-NEXT:    retq
+;
+; X64-AVX2-LABEL: mul_v2i64_neg_0_1:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255]
+; X64-AVX2-NEXT:    vpmuludq %xmm1, %xmm0, %xmm2
+; X64-AVX2-NEXT:    vpsrlq $32, %xmm0, %xmm3
+; X64-AVX2-NEXT:    vpmuludq %xmm1, %xmm3, %xmm1
+; X64-AVX2-NEXT:    vpmuludq {{.*}}(%rip), %xmm0, %xmm0
+; X64-AVX2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT:    vpsllq $32, %xmm0, %xmm0
+; X64-AVX2-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
+; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512DQ-LABEL: mul_v2i64_neg_0_1:
+; X64-AVX512DQ:       # %bb.0:
+; X64-AVX512DQ-NEXT:    vpmullq {{.*}}(%rip), %xmm0, %xmm0
+; X64-AVX512DQ-NEXT:    retq
   %1 = mul <2 x i64> %a0, <i64 0, i64 -1>
   ret <2 x i64> %1
 }
@@ -1029,17 +1660,34 @@ define <2 x i64> @mul_v2i64_15_neg_63(<2 x i64> %a0) nounwind {
 ; X64-NEXT:    paddq %xmm2, %xmm0
 ; X64-NEXT:    retq
 ;
-; X64-AVX-LABEL: mul_v2i64_15_neg_63:
-; X64-AVX:       # %bb.0:
-; X64-AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [15,18446744073709551553]
-; X64-AVX-NEXT:    vpmuludq %xmm1, %xmm0, %xmm2
-; X64-AVX-NEXT:    vpsrlq $32, %xmm0, %xmm3
-; X64-AVX-NEXT:    vpmuludq %xmm1, %xmm3, %xmm1
-; X64-AVX-NEXT:    vpmuludq {{.*}}(%rip), %xmm0, %xmm0
-; X64-AVX-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
-; X64-AVX-NEXT:    vpsllq $32, %xmm0, %xmm0
-; X64-AVX-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
-; X64-AVX-NEXT:    retq
+; X64-XOP-LABEL: mul_v2i64_15_neg_63:
+; X64-XOP:       # %bb.0:
+; X64-XOP-NEXT:    vmovdqa {{.*#+}} xmm1 = [15,18446744073709551553]
+; X64-XOP-NEXT:    vpmuludq %xmm1, %xmm0, %xmm2
+; X64-XOP-NEXT:    vpsrlq $32, %xmm0, %xmm3
+; X64-XOP-NEXT:    vpmuludq %xmm1, %xmm3, %xmm1
+; X64-XOP-NEXT:    vpmuludq {{.*}}(%rip), %xmm0, %xmm0
+; X64-XOP-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
+; X64-XOP-NEXT:    vpsllq $32, %xmm0, %xmm0
+; X64-XOP-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
+; X64-XOP-NEXT:    retq
+;
+; X64-AVX2-LABEL: mul_v2i64_15_neg_63:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [15,18446744073709551553]
+; X64-AVX2-NEXT:    vpmuludq %xmm1, %xmm0, %xmm2
+; X64-AVX2-NEXT:    vpsrlq $32, %xmm0, %xmm3
+; X64-AVX2-NEXT:    vpmuludq %xmm1, %xmm3, %xmm1
+; X64-AVX2-NEXT:    vpmuludq {{.*}}(%rip), %xmm0, %xmm0
+; X64-AVX2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT:    vpsllq $32, %xmm0, %xmm0
+; X64-AVX2-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
+; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512DQ-LABEL: mul_v2i64_15_neg_63:
+; X64-AVX512DQ:       # %bb.0:
+; X64-AVX512DQ-NEXT:    vpmullq {{.*}}(%rip), %xmm0, %xmm0
+; X64-AVX512DQ-NEXT:    retq
   %1 = mul <2 x i64> %a0, <i64 15, i64 -63>
   ret <2 x i64> %1
 }
@@ -1130,6 +1778,15 @@ define <16 x i8> @mul_v16i8_0_1_3_7_15_31_63_127_0_1_3_7_15_31_63_127(<16 x i8>
 ; X64-AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
 ; X64-AVX2-NEXT:    vzeroupper
 ; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512DQ-LABEL: mul_v16i8_0_1_3_7_15_31_63_127_0_1_3_7_15_31_63_127:
+; X64-AVX512DQ:       # %bb.0:
+; X64-AVX512DQ-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; X64-AVX512DQ-NEXT:    vpmullw {{.*}}(%rip), %ymm0, %ymm0
+; X64-AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
+; X64-AVX512DQ-NEXT:    vpmovdb %zmm0, %xmm0
+; X64-AVX512DQ-NEXT:    vzeroupper
+; X64-AVX512DQ-NEXT:    retq
   %1 = mul <16 x i8> %a0, <i8 0, i8 1, i8 3, i8 7, i8 15, i8 31, i8 63, i8 127, i8 0, i8 1, i8 3, i8 7, i8 15, i8 31, i8 63, i8 127>
   ret <16 x i8> %1
 }
@@ -1157,15 +1814,30 @@ define <2 x i64> @mul_v2i64_68_132(<2 x i64> %x) nounwind {
 ; X64-NEXT:    paddq %xmm2, %xmm0
 ; X64-NEXT:    retq
 ;
-; X64-AVX-LABEL: mul_v2i64_68_132:
-; X64-AVX:       # %bb.0:
-; X64-AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [68,132]
-; X64-AVX-NEXT:    vpmuludq %xmm1, %xmm0, %xmm2
-; X64-AVX-NEXT:    vpsrlq $32, %xmm0, %xmm0
-; X64-AVX-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
-; X64-AVX-NEXT:    vpsllq $32, %xmm0, %xmm0
-; X64-AVX-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
-; X64-AVX-NEXT:    retq
+; X64-XOP-LABEL: mul_v2i64_68_132:
+; X64-XOP:       # %bb.0:
+; X64-XOP-NEXT:    vmovdqa {{.*#+}} xmm1 = [68,132]
+; X64-XOP-NEXT:    vpmuludq %xmm1, %xmm0, %xmm2
+; X64-XOP-NEXT:    vpsrlq $32, %xmm0, %xmm0
+; X64-XOP-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
+; X64-XOP-NEXT:    vpsllq $32, %xmm0, %xmm0
+; X64-XOP-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
+; X64-XOP-NEXT:    retq
+;
+; X64-AVX2-LABEL: mul_v2i64_68_132:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [68,132]
+; X64-AVX2-NEXT:    vpmuludq %xmm1, %xmm0, %xmm2
+; X64-AVX2-NEXT:    vpsrlq $32, %xmm0, %xmm0
+; X64-AVX2-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT:    vpsllq $32, %xmm0, %xmm0
+; X64-AVX2-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
+; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512DQ-LABEL: mul_v2i64_68_132:
+; X64-AVX512DQ:       # %bb.0:
+; X64-AVX512DQ-NEXT:    vpmullq {{.*}}(%rip), %xmm0, %xmm0
+; X64-AVX512DQ-NEXT:    retq
   %mul = mul <2 x i64> %x, <i64 68, i64 132>
   ret <2 x i64> %mul
 }
@@ -1193,15 +1865,30 @@ define <2 x i64> @mul_v2i64_60_120(<2 x i64> %x) nounwind {
 ; X64-NEXT:    paddq %xmm2, %xmm0
 ; X64-NEXT:    retq
 ;
-; X64-AVX-LABEL: mul_v2i64_60_120:
-; X64-AVX:       # %bb.0:
-; X64-AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [60,124]
-; X64-AVX-NEXT:    vpmuludq %xmm1, %xmm0, %xmm2
-; X64-AVX-NEXT:    vpsrlq $32, %xmm0, %xmm0
-; X64-AVX-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
-; X64-AVX-NEXT:    vpsllq $32, %xmm0, %xmm0
-; X64-AVX-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
-; X64-AVX-NEXT:    retq
+; X64-XOP-LABEL: mul_v2i64_60_120:
+; X64-XOP:       # %bb.0:
+; X64-XOP-NEXT:    vmovdqa {{.*#+}} xmm1 = [60,124]
+; X64-XOP-NEXT:    vpmuludq %xmm1, %xmm0, %xmm2
+; X64-XOP-NEXT:    vpsrlq $32, %xmm0, %xmm0
+; X64-XOP-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
+; X64-XOP-NEXT:    vpsllq $32, %xmm0, %xmm0
+; X64-XOP-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
+; X64-XOP-NEXT:    retq
+;
+; X64-AVX2-LABEL: mul_v2i64_60_120:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [60,124]
+; X64-AVX2-NEXT:    vpmuludq %xmm1, %xmm0, %xmm2
+; X64-AVX2-NEXT:    vpsrlq $32, %xmm0, %xmm0
+; X64-AVX2-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT:    vpsllq $32, %xmm0, %xmm0
+; X64-AVX2-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
+; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512DQ-LABEL: mul_v2i64_60_120:
+; X64-AVX512DQ:       # %bb.0:
+; X64-AVX512DQ-NEXT:    vpmullq {{.*}}(%rip), %xmm0, %xmm0
+; X64-AVX512DQ-NEXT:    retq
   %mul = mul <2 x i64> %x, <i64 60, i64 124>
   ret <2 x i64> %mul
 }
@@ -1287,6 +1974,13 @@ define <4 x i64> @mul_v4i64_zext_cross_bb(<4 x i32>* %in, <4 x i32>* %y) {
 ; X64-AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
 ; X64-AVX2-NEXT:    vpmuludq %ymm1, %ymm0, %ymm0
 ; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512DQ-LABEL: mul_v4i64_zext_cross_bb:
+; X64-AVX512DQ:       # %bb.0:
+; X64-AVX512DQ-NEXT:    vpmovzxdq {{.*#+}} ymm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
+; X64-AVX512DQ-NEXT:    vpmovzxdq {{.*#+}} ymm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
+; X64-AVX512DQ-NEXT:    vpmuludq %ymm1, %ymm0, %ymm0
+; X64-AVX512DQ-NEXT:    retq
   %a = load <4 x i32>, <4 x i32>* %in
   %b = zext <4 x i32> %a to <4 x i64>
   br label %foo
diff --git a/llvm/test/CodeGen/X86/vector-mulfix-legalize.ll b/llvm/test/CodeGen/X86/vector-mulfix-legalize.ll
new file mode 100644
index 000000000..b36979b6e
--- /dev/null
+++ b/llvm/test/CodeGen/X86/vector-mulfix-legalize.ll
@@ -0,0 +1,146 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -O1 -mtriple=x86_64-unknown-unknown -o - | FileCheck %s
+
+; We used to assert on widening the SMULFIX/UMULFIX/SMULFIXSAT node result,
+; so primiary goal with the test is to see that we support legalization for
+; such vectors.
+
+declare <4 x i16> @llvm.smul.fix.v4i16(<4 x i16>, <4 x i16>, i32 immarg)
+declare <4 x i16> @llvm.umul.fix.v4i16(<4 x i16>, <4 x i16>, i32 immarg)
+declare <4 x i16> @llvm.smul.fix.sat.v4i16(<4 x i16>, <4 x i16>, i32 immarg)
+declare <4 x i16> @llvm.umul.fix.sat.v4i16(<4 x i16>, <4 x i16>, i32 immarg)
+
+define <4 x i16> @smulfix(<4 x i16> %a) {
+; CHECK-LABEL: smulfix:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movdqa {{.*#+}} xmm1 = <1,2,3,4,u,u,u,u>
+; CHECK-NEXT:    movdqa %xmm0, %xmm2
+; CHECK-NEXT:    pmullw %xmm1, %xmm2
+; CHECK-NEXT:    psrlw $15, %xmm2
+; CHECK-NEXT:    pmulhw %xmm1, %xmm0
+; CHECK-NEXT:    psllw $1, %xmm0
+; CHECK-NEXT:    por %xmm2, %xmm0
+; CHECK-NEXT:    retq
+  %t = call <4 x i16> @llvm.smul.fix.v4i16(<4 x i16> <i16 1, i16 2, i16 3, i16 4>, <4 x i16> %a, i32 15)
+  ret <4 x i16> %t
+}
+
+define <4 x i16> @umulfix(<4 x i16> %a) {
+; CHECK-LABEL: umulfix:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movdqa {{.*#+}} xmm1 = <1,2,3,4,u,u,u,u>
+; CHECK-NEXT:    movdqa %xmm0, %xmm2
+; CHECK-NEXT:    pmullw %xmm1, %xmm2
+; CHECK-NEXT:    psrlw $15, %xmm2
+; CHECK-NEXT:    pmulhuw %xmm1, %xmm0
+; CHECK-NEXT:    psllw $1, %xmm0
+; CHECK-NEXT:    por %xmm2, %xmm0
+; CHECK-NEXT:    retq
+  %t = call <4 x i16> @llvm.umul.fix.v4i16(<4 x i16> <i16 1, i16 2, i16 3, i16 4>, <4 x i16> %a, i32 15)
+  ret <4 x i16> %t
+}
+
+define <4 x i16> @smulfixsat(<4 x i16> %a) {
+; CHECK-LABEL: smulfixsat:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pextrw $2, %xmm0, %eax
+; CHECK-NEXT:    cwtl
+; CHECK-NEXT:    leal (%rax,%rax,2), %ecx
+; CHECK-NEXT:    movl %ecx, %edx
+; CHECK-NEXT:    shrl $16, %edx
+; CHECK-NEXT:    shldw $1, %cx, %dx
+; CHECK-NEXT:    sarl $16, %ecx
+; CHECK-NEXT:    cmpl $16383, %ecx # imm = 0x3FFF
+; CHECK-NEXT:    movl $32767, %r8d # imm = 0x7FFF
+; CHECK-NEXT:    cmovgl %r8d, %edx
+; CHECK-NEXT:    cmpl $-16384, %ecx # imm = 0xC000
+; CHECK-NEXT:    movl $32768, %ecx # imm = 0x8000
+; CHECK-NEXT:    cmovll %ecx, %edx
+; CHECK-NEXT:    pextrw $1, %xmm0, %esi
+; CHECK-NEXT:    movswl %si, %edi
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    shrl $15, %eax
+; CHECK-NEXT:    leal (%rdi,%rdi), %esi
+; CHECK-NEXT:    shrdw $15, %ax, %si
+; CHECK-NEXT:    sarl $15, %edi
+; CHECK-NEXT:    cmpl $16383, %edi # imm = 0x3FFF
+; CHECK-NEXT:    cmovgl %r8d, %esi
+; CHECK-NEXT:    cmpl $-16384, %edi # imm = 0xC000
+; CHECK-NEXT:    cmovll %ecx, %esi
+; CHECK-NEXT:    movd %xmm0, %eax
+; CHECK-NEXT:    cwtl
+; CHECK-NEXT:    movl %eax, %edi
+; CHECK-NEXT:    shrl $16, %edi
+; CHECK-NEXT:    shldw $1, %ax, %di
+; CHECK-NEXT:    sarl $16, %eax
+; CHECK-NEXT:    cmpl $16383, %eax # imm = 0x3FFF
+; CHECK-NEXT:    cmovgl %r8d, %edi
+; CHECK-NEXT:    cmpl $-16384, %eax # imm = 0xC000
+; CHECK-NEXT:    cmovll %ecx, %edi
+; CHECK-NEXT:    pxor %xmm1, %xmm1
+; CHECK-NEXT:    pinsrw $0, %edi, %xmm1
+; CHECK-NEXT:    pinsrw $1, %esi, %xmm1
+; CHECK-NEXT:    pinsrw $2, %edx, %xmm1
+; CHECK-NEXT:    pextrw $3, %xmm0, %eax
+; CHECK-NEXT:    cwtl
+; CHECK-NEXT:    movl %eax, %edx
+; CHECK-NEXT:    shrl $14, %edx
+; CHECK-NEXT:    leal (,%rax,4), %esi
+; CHECK-NEXT:    shrdw $15, %dx, %si
+; CHECK-NEXT:    sarl $14, %eax
+; CHECK-NEXT:    cmpl $16383, %eax # imm = 0x3FFF
+; CHECK-NEXT:    cmovgl %r8d, %esi
+; CHECK-NEXT:    cmpl $-16384, %eax # imm = 0xC000
+; CHECK-NEXT:    cmovll %ecx, %esi
+; CHECK-NEXT:    pinsrw $3, %esi, %xmm1
+; CHECK-NEXT:    movdqa %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %t = call <4 x i16> @llvm.smul.fix.sat.v4i16(<4 x i16> <i16 1, i16 2, i16 3, i16 4>, <4 x i16> %a, i32 15)
+  ret <4 x i16> %t
+}
+
+
+define <4 x i16> @umulfixsat(<4 x i16> %a) {
+; CHECK-LABEL: umulfixsat:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pextrw $2, %xmm0, %eax
+; CHECK-NEXT:    leal (%rax,%rax,2), %eax
+; CHECK-NEXT:    movl %eax, %edx
+; CHECK-NEXT:    shrl $16, %edx
+; CHECK-NEXT:    movl %edx, %ecx
+; CHECK-NEXT:    shldw $1, %ax, %cx
+; CHECK-NEXT:    cmpl $32767, %edx # imm = 0x7FFF
+; CHECK-NEXT:    movl $65535, %eax # imm = 0xFFFF
+; CHECK-NEXT:    cmoval %eax, %ecx
+; CHECK-NEXT:    pextrw $1, %xmm0, %edx
+; CHECK-NEXT:    addl %edx, %edx
+; CHECK-NEXT:    movl %edx, %esi
+; CHECK-NEXT:    shrl $16, %esi
+; CHECK-NEXT:    movl %esi, %edi
+; CHECK-NEXT:    shldw $1, %dx, %di
+; CHECK-NEXT:    cmpl $32767, %esi # imm = 0x7FFF
+; CHECK-NEXT:    cmoval %eax, %edi
+; CHECK-NEXT:    movd %xmm0, %edx
+; CHECK-NEXT:    xorl %esi, %esi
+; CHECK-NEXT:    shldw $1, %dx, %si
+; CHECK-NEXT:    movl $32767, %edx # imm = 0x7FFF
+; CHECK-NEXT:    negl %edx
+; CHECK-NEXT:    cmoval %eax, %esi
+; CHECK-NEXT:    pxor %xmm1, %xmm1
+; CHECK-NEXT:    pinsrw $0, %esi, %xmm1
+; CHECK-NEXT:    pinsrw $1, %edi, %xmm1
+; CHECK-NEXT:    pinsrw $2, %ecx, %xmm1
+; CHECK-NEXT:    pextrw $3, %xmm0, %ecx
+; CHECK-NEXT:    shll $2, %ecx
+; CHECK-NEXT:    movl %ecx, %edx
+; CHECK-NEXT:    shrl $16, %edx
+; CHECK-NEXT:    movl %edx, %esi
+; CHECK-NEXT:    shldw $1, %cx, %si
+; CHECK-NEXT:    cmpl $32767, %edx # imm = 0x7FFF
+; CHECK-NEXT:    cmoval %eax, %esi
+; CHECK-NEXT:    pinsrw $3, %esi, %xmm1
+; CHECK-NEXT:    movdqa %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %t = call <4 x i16> @llvm.umul.fix.sat.v4i16(<4 x i16> <i16 1, i16 2, i16 3, i16 4>, <4 x i16> %a, i32 15)
+  ret <4 x i16> %t
+}
diff --git a/llvm/test/CodeGen/X86/vector-narrow-binop.ll b/llvm/test/CodeGen/X86/vector-narrow-binop.ll
index aaeb54dd7..d76f0dc54 100644
--- a/llvm/test/CodeGen/X86/vector-narrow-binop.ll
+++ b/llvm/test/CodeGen/X86/vector-narrow-binop.ll
@@ -107,34 +107,20 @@ define <2 x i8> @PR39893(<2 x i32> %x, <8 x i8> %y) {
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pxor %xmm2, %xmm2
 ; SSE-NEXT:    psubd %xmm0, %xmm2
+; SSE-NEXT:    psrld $16, %xmm2
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
-; SSE-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1],xmm1[2,3]
-; SSE-NEXT:    movaps %xmm2, %xmm0
+; SSE-NEXT:    movdqa %xmm2, %xmm0
 ; SSE-NEXT:    retq
 ;
-; AVX1-LABEL: PR39893:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpsubd %xmm0, %xmm2, %xmm0
-; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[2],zero,xmm0[3],zero,xmm0[2],zero,xmm0[3],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4,5,6,7]
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: PR39893:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpsubd %xmm0, %xmm2, %xmm0
-; AVX2-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[2],zero,xmm0[3],zero,xmm0[2],zero,xmm0[3],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: PR39893:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpsubd %xmm0, %xmm2, %xmm0
-; AVX512-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[2],zero,xmm0[3],zero,xmm0[2],zero,xmm0[3],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero
-; AVX512-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
-; AVX512-NEXT:    retq
+; AVX-LABEL: PR39893:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX-NEXT:    vpsubd %xmm0, %xmm2, %xmm0
+; AVX-NEXT:    vpsrld $16, %xmm0, %xmm0
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; AVX-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; AVX-NEXT:    retq
   %sub = sub <2 x i32> <i32 0, i32 undef>, %x
   %bc = bitcast <2 x i32> %sub to <8 x i8>
   %shuffle = shufflevector <8 x i8> %y, <8 x i8> %bc, <2 x i32> <i32 10, i32 4>
@@ -146,16 +132,13 @@ define <2 x i8> @PR39893_2(<2 x float> %x) {
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    xorps %xmm1, %xmm1
 ; SSE-NEXT:    subps %xmm0, %xmm1
-; SSE-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; SSE-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,1,1,3]
+; SSE-NEXT:    movaps %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: PR39893_2:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
 ; AVX-NEXT:    vsubps %xmm0, %xmm1, %xmm0
-; AVX-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX-NEXT:    retq
   %fsub = fsub <2 x float> zeroinitializer, %x
   %bc = bitcast <2 x float> %fsub to <8 x i8>
diff --git a/llvm/test/CodeGen/X86/vector-pcmp.ll b/llvm/test/CodeGen/X86/vector-pcmp.ll
index 89eaad82f..2900ce2c8 100644
--- a/llvm/test/CodeGen/X86/vector-pcmp.ll
+++ b/llvm/test/CodeGen/X86/vector-pcmp.ll
@@ -61,10 +61,9 @@ define <4 x i32> @test_pcmpgtd(<4 x i32> %x) {
 define <2 x i64> @test_pcmpgtq(<2 x i64> %x) {
 ; SSE2-LABEL: test_pcmpgtq:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    psrad $31, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE2-NEXT:    pxor %xmm1, %xmm0
+; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
 ; SSE2-NEXT:    retq
 ;
 ; SSE42-LABEL: test_pcmpgtq:
@@ -187,13 +186,11 @@ define <8 x i32> @test_pcmpgtd_256(<8 x i32> %x) {
 define <4 x i64> @test_pcmpgtq_256(<4 x i64> %x) {
 ; SSE2-LABEL: test_pcmpgtq_256:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    psrad $31, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    psrad $31, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
 ; SSE2-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT:    pxor %xmm2, %xmm0
-; SSE2-NEXT:    pxor %xmm2, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
 ; SSE2-NEXT:    retq
 ;
 ; SSE42-LABEL: test_pcmpgtq_256:
diff --git a/llvm/test/CodeGen/X86/vector-popcnt-512.ll b/llvm/test/CodeGen/X86/vector-popcnt-512.ll
index eae9e6c79..a4afda297 100644
--- a/llvm/test/CodeGen/X86/vector-popcnt-512.ll
+++ b/llvm/test/CodeGen/X86/vector-popcnt-512.ll
@@ -130,26 +130,28 @@ define <16 x i32> @testv16i32(<16 x i32> %in) nounwind {
 define <32 x i16> @testv32i16(<32 x i16> %in) nounwind {
 ; AVX512F-LABEL: testv32i16:
 ; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512F-NEXT:    vpand %ymm2, %ymm0, %ymm3
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
-; AVX512F-NEXT:    vpshufb %ymm3, %ymm4, %ymm3
-; AVX512F-NEXT:    vpsrlw $4, %ymm0, %ymm0
-; AVX512F-NEXT:    vpand %ymm2, %ymm0, %ymm0
-; AVX512F-NEXT:    vpshufb %ymm0, %ymm4, %ymm0
-; AVX512F-NEXT:    vpaddb %ymm3, %ymm0, %ymm0
-; AVX512F-NEXT:    vpsllw $8, %ymm0, %ymm3
-; AVX512F-NEXT:    vpaddb %ymm0, %ymm3, %ymm0
-; AVX512F-NEXT:    vpsrlw $8, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpand %ymm2, %ymm1, %ymm3
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
 ; AVX512F-NEXT:    vpshufb %ymm3, %ymm4, %ymm3
 ; AVX512F-NEXT:    vpsrlw $4, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpand %ymm2, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpshufb %ymm1, %ymm4, %ymm1
 ; AVX512F-NEXT:    vpaddb %ymm3, %ymm1, %ymm1
-; AVX512F-NEXT:    vpsllw $8, %ymm1, %ymm2
-; AVX512F-NEXT:    vpaddb %ymm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vpsllw $8, %ymm1, %ymm3
+; AVX512F-NEXT:    vpaddb %ymm1, %ymm3, %ymm1
 ; AVX512F-NEXT:    vpsrlw $8, %ymm1, %ymm1
+; AVX512F-NEXT:    vpand %ymm2, %ymm0, %ymm3
+; AVX512F-NEXT:    vpshufb %ymm3, %ymm4, %ymm3
+; AVX512F-NEXT:    vpsrlw $4, %ymm0, %ymm0
+; AVX512F-NEXT:    vpand %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vpshufb %ymm0, %ymm4, %ymm0
+; AVX512F-NEXT:    vpaddb %ymm3, %ymm0, %ymm0
+; AVX512F-NEXT:    vpsllw $8, %ymm0, %ymm2
+; AVX512F-NEXT:    vpaddb %ymm0, %ymm2, %ymm0
+; AVX512F-NEXT:    vpsrlw $8, %ymm0, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: testv32i16:
@@ -169,12 +171,14 @@ define <32 x i16> @testv32i16(<32 x i16> %in) nounwind {
 ;
 ; AVX512VPOPCNTDQ-NOBW-LABEL: testv32i16:
 ; AVX512VPOPCNTDQ-NOBW:       # %bb.0:
-; AVX512VPOPCNTDQ-NOBW-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
-; AVX512VPOPCNTDQ-NOBW-NEXT:    vpopcntd %zmm0, %zmm0
-; AVX512VPOPCNTDQ-NOBW-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512VPOPCNTDQ-NOBW-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512VPOPCNTDQ-NOBW-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
 ; AVX512VPOPCNTDQ-NOBW-NEXT:    vpopcntd %zmm1, %zmm1
 ; AVX512VPOPCNTDQ-NOBW-NEXT:    vpmovdw %zmm1, %ymm1
+; AVX512VPOPCNTDQ-NOBW-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
+; AVX512VPOPCNTDQ-NOBW-NEXT:    vpopcntd %zmm0, %zmm0
+; AVX512VPOPCNTDQ-NOBW-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512VPOPCNTDQ-NOBW-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512VPOPCNTDQ-NOBW-NEXT:    retq
 ;
 ; AVX512VPOPCNTDQ-BW-LABEL: testv32i16:
@@ -203,20 +207,22 @@ define <32 x i16> @testv32i16(<32 x i16> %in) nounwind {
 define <64 x i8> @testv64i8(<64 x i8> %in) nounwind {
 ; AVX512F-LABEL: testv64i8:
 ; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512F-NEXT:    vpand %ymm2, %ymm0, %ymm3
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
-; AVX512F-NEXT:    vpshufb %ymm3, %ymm4, %ymm3
-; AVX512F-NEXT:    vpsrlw $4, %ymm0, %ymm0
-; AVX512F-NEXT:    vpand %ymm2, %ymm0, %ymm0
-; AVX512F-NEXT:    vpshufb %ymm0, %ymm4, %ymm0
-; AVX512F-NEXT:    vpaddb %ymm3, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpand %ymm2, %ymm1, %ymm3
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
 ; AVX512F-NEXT:    vpshufb %ymm3, %ymm4, %ymm3
 ; AVX512F-NEXT:    vpsrlw $4, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpand %ymm2, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpshufb %ymm1, %ymm4, %ymm1
 ; AVX512F-NEXT:    vpaddb %ymm3, %ymm1, %ymm1
+; AVX512F-NEXT:    vpand %ymm2, %ymm0, %ymm3
+; AVX512F-NEXT:    vpshufb %ymm3, %ymm4, %ymm3
+; AVX512F-NEXT:    vpsrlw $4, %ymm0, %ymm0
+; AVX512F-NEXT:    vpand %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vpshufb %ymm0, %ymm4, %ymm0
+; AVX512F-NEXT:    vpaddb %ymm3, %ymm0, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: testv64i8:
@@ -233,20 +239,22 @@ define <64 x i8> @testv64i8(<64 x i8> %in) nounwind {
 ;
 ; AVX512VPOPCNTDQ-NOBW-LABEL: testv64i8:
 ; AVX512VPOPCNTDQ-NOBW:       # %bb.0:
+; AVX512VPOPCNTDQ-NOBW-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512VPOPCNTDQ-NOBW-NEXT:    vmovdqa {{.*#+}} ymm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512VPOPCNTDQ-NOBW-NEXT:    vpand %ymm2, %ymm0, %ymm3
-; AVX512VPOPCNTDQ-NOBW-NEXT:    vmovdqa {{.*#+}} ymm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
-; AVX512VPOPCNTDQ-NOBW-NEXT:    vpshufb %ymm3, %ymm4, %ymm3
-; AVX512VPOPCNTDQ-NOBW-NEXT:    vpsrlw $4, %ymm0, %ymm0
-; AVX512VPOPCNTDQ-NOBW-NEXT:    vpand %ymm2, %ymm0, %ymm0
-; AVX512VPOPCNTDQ-NOBW-NEXT:    vpshufb %ymm0, %ymm4, %ymm0
-; AVX512VPOPCNTDQ-NOBW-NEXT:    vpaddb %ymm3, %ymm0, %ymm0
 ; AVX512VPOPCNTDQ-NOBW-NEXT:    vpand %ymm2, %ymm1, %ymm3
+; AVX512VPOPCNTDQ-NOBW-NEXT:    vmovdqa {{.*#+}} ymm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
 ; AVX512VPOPCNTDQ-NOBW-NEXT:    vpshufb %ymm3, %ymm4, %ymm3
 ; AVX512VPOPCNTDQ-NOBW-NEXT:    vpsrlw $4, %ymm1, %ymm1
 ; AVX512VPOPCNTDQ-NOBW-NEXT:    vpand %ymm2, %ymm1, %ymm1
 ; AVX512VPOPCNTDQ-NOBW-NEXT:    vpshufb %ymm1, %ymm4, %ymm1
 ; AVX512VPOPCNTDQ-NOBW-NEXT:    vpaddb %ymm3, %ymm1, %ymm1
+; AVX512VPOPCNTDQ-NOBW-NEXT:    vpand %ymm2, %ymm0, %ymm3
+; AVX512VPOPCNTDQ-NOBW-NEXT:    vpshufb %ymm3, %ymm4, %ymm3
+; AVX512VPOPCNTDQ-NOBW-NEXT:    vpsrlw $4, %ymm0, %ymm0
+; AVX512VPOPCNTDQ-NOBW-NEXT:    vpand %ymm2, %ymm0, %ymm0
+; AVX512VPOPCNTDQ-NOBW-NEXT:    vpshufb %ymm0, %ymm4, %ymm0
+; AVX512VPOPCNTDQ-NOBW-NEXT:    vpaddb %ymm3, %ymm0, %ymm0
+; AVX512VPOPCNTDQ-NOBW-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512VPOPCNTDQ-NOBW-NEXT:    retq
 ;
 ; AVX512VPOPCNTDQ-BW-LABEL: testv64i8:
diff --git a/llvm/test/CodeGen/X86/vector-reduce-add.ll b/llvm/test/CodeGen/X86/vector-reduce-add.ll
index 630299a18..60636a86d 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-add.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-add.ll
@@ -198,22 +198,35 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 define i32 @test_v2i32(<2 x i32> %a0) {
 ; SSE-LABEL: test_v2i32:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; SSE-NEXT:    paddq %xmm0, %xmm1
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE-NEXT:    paddd %xmm0, %xmm1
 ; SSE-NEXT:    movd %xmm1, %eax
 ; SSE-NEXT:    retq
 ;
-; AVX-LABEL: test_v2i32:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vmovd %xmm0, %eax
-; AVX-NEXT:    retq
+; AVX1-SLOW-LABEL: test_v2i32:
+; AVX1-SLOW:       # %bb.0:
+; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX1-SLOW-NEXT:    vmovd %xmm0, %eax
+; AVX1-SLOW-NEXT:    retq
+;
+; AVX1-FAST-LABEL: test_v2i32:
+; AVX1-FAST:       # %bb.0:
+; AVX1-FAST-NEXT:    vphaddd %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT:    vmovd %xmm0, %eax
+; AVX1-FAST-NEXT:    retq
+;
+; AVX2-LABEL: test_v2i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vmovd %xmm0, %eax
+; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: test_v2i32:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
 ; AVX512-NEXT:    retq
   %1 = call i32 @llvm.experimental.vector.reduce.add.v2i32(<2 x i32> %a0)
@@ -497,24 +510,40 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 define i16 @test_v2i16(<2 x i16> %a0) {
 ; SSE-LABEL: test_v2i16:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; SSE-NEXT:    paddq %xmm0, %xmm1
+; SSE-NEXT:    movdqa %xmm0, %xmm1
+; SSE-NEXT:    psrld $16, %xmm1
+; SSE-NEXT:    paddw %xmm0, %xmm1
 ; SSE-NEXT:    movd %xmm1, %eax
 ; SSE-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE-NEXT:    retq
 ;
-; AVX-LABEL: test_v2i16:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vmovd %xmm0, %eax
-; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX-NEXT:    retq
+; AVX1-SLOW-LABEL: test_v2i16:
+; AVX1-SLOW:       # %bb.0:
+; AVX1-SLOW-NEXT:    vpsrld $16, %xmm0, %xmm1
+; AVX1-SLOW-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
+; AVX1-SLOW-NEXT:    vmovd %xmm0, %eax
+; AVX1-SLOW-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX1-SLOW-NEXT:    retq
+;
+; AVX1-FAST-LABEL: test_v2i16:
+; AVX1-FAST:       # %bb.0:
+; AVX1-FAST-NEXT:    vphaddw %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT:    vmovd %xmm0, %eax
+; AVX1-FAST-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX1-FAST-NEXT:    retq
+;
+; AVX2-LABEL: test_v2i16:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
+; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vmovd %xmm0, %eax
+; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: test_v2i16:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
+; AVX512-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
 ; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX512-NEXT:    retq
@@ -525,49 +554,50 @@ define i16 @test_v2i16(<2 x i16> %a0) {
 define i16 @test_v4i16(<4 x i16> %a0) {
 ; SSE-LABEL: test_v4i16:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; SSE-NEXT:    paddd %xmm0, %xmm1
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
-; SSE-NEXT:    paddd %xmm1, %xmm0
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE-NEXT:    paddw %xmm0, %xmm1
+; SSE-NEXT:    movdqa %xmm1, %xmm0
+; SSE-NEXT:    psrld $16, %xmm0
+; SSE-NEXT:    paddw %xmm1, %xmm0
 ; SSE-NEXT:    movd %xmm0, %eax
 ; SSE-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE-NEXT:    retq
 ;
 ; AVX1-SLOW-LABEL: test_v4i16:
 ; AVX1-SLOW:       # %bb.0:
-; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX1-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX1-SLOW-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
+; AVX1-SLOW-NEXT:    vpsrld $16, %xmm0, %xmm1
+; AVX1-SLOW-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX1-SLOW-NEXT:    vmovd %xmm0, %eax
 ; AVX1-SLOW-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX1-SLOW-NEXT:    retq
 ;
 ; AVX1-FAST-LABEL: test_v4i16:
 ; AVX1-FAST:       # %bb.0:
-; AVX1-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-FAST-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-FAST-NEXT:    vphaddd %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-FAST-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
+; AVX1-FAST-NEXT:    vphaddw %xmm0, %xmm0, %xmm0
 ; AVX1-FAST-NEXT:    vmovd %xmm0, %eax
 ; AVX1-FAST-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX1-FAST-NEXT:    retq
 ;
 ; AVX2-LABEL: test_v4i16:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
+; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
 ; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: test_v4i16:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
+; AVX512-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
 ; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX512-NEXT:    retq
@@ -914,35 +944,84 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 define i8 @test_v2i8(<2 x i8> %a0) {
 ; SSE2-LABEL: test_v2i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; SSE2-NEXT:    paddq %xmm0, %xmm1
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psrlw $8, %xmm1
+; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    movd %xmm1, %eax
 ; SSE2-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: test_v2i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; SSE41-NEXT:    paddq %xmm0, %xmm1
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    psrlw $8, %xmm1
+; SSE41-NEXT:    paddb %xmm0, %xmm1
 ; SSE41-NEXT:    pextrb $0, %xmm1, %eax
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: test_v2i8:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpextrb $0, %xmm0, %eax
 ; AVX-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: test_v2i8:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX512-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX512-NEXT:    # kill: def $al killed $al killed $eax
+; AVX512-NEXT:    retq
+  %1 = call i8 @llvm.experimental.vector.reduce.add.v2i8(<2 x i8> %a0)
+  ret i8 %1
+}
+
+define i8 @test_v2i8_load(<2 x i8>* %p) {
+; SSE2-LABEL: test_v2i8_load:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movzwl (%rdi), %eax
+; SSE2-NEXT:    movd %eax, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psrlw $8, %xmm1
+; SSE2-NEXT:    paddb %xmm0, %xmm1
+; SSE2-NEXT:    movd %xmm1, %eax
+; SSE2-NEXT:    # kill: def $al killed $al killed $eax
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: test_v2i8_load:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movzwl (%rdi), %eax
+; SSE41-NEXT:    movd %eax, %xmm0
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    psrlw $8, %xmm1
+; SSE41-NEXT:    paddb %xmm0, %xmm1
+; SSE41-NEXT:    pextrb $0, %xmm1, %eax
+; SSE41-NEXT:    # kill: def $al killed $al killed $eax
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: test_v2i8_load:
+; AVX:       # %bb.0:
+; AVX-NEXT:    movzwl (%rdi), %eax
+; AVX-NEXT:    vmovd %eax, %xmm0
+; AVX-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX-NEXT:    # kill: def $al killed $al killed $eax
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: test_v2i8_load:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    movzwl (%rdi), %eax
+; AVX512-NEXT:    vmovd %eax, %xmm0
+; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX512-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
 ; AVX512-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512-NEXT:    retq
+  %a0 = load <2 x i8>, <2 x i8>* %p
   %1 = call i8 @llvm.experimental.vector.reduce.add.v2i8(<2 x i8> %a0)
   ret i8 %1
 }
@@ -950,62 +1029,81 @@ define i8 @test_v2i8(<2 x i8> %a0) {
 define i8 @test_v4i8(<4 x i8> %a0) {
 ; SSE2-LABEL: test_v4i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; SSE2-NEXT:    paddd %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
-; SSE2-NEXT:    paddd %xmm1, %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
 ; SSE2-NEXT:    movd %xmm0, %eax
 ; SSE2-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: test_v4i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; SSE41-NEXT:    paddd %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
-; SSE41-NEXT:    paddd %xmm1, %xmm0
+; SSE41-NEXT:    pxor %xmm1, %xmm1
+; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7]
+; SSE41-NEXT:    psadbw %xmm1, %xmm0
 ; SSE41-NEXT:    pextrb $0, %xmm0, %eax
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    retq
 ;
-; AVX1-SLOW-LABEL: test_v4i8:
-; AVX1-SLOW:       # %bb.0:
-; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX1-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-SLOW-NEXT:    vpextrb $0, %xmm0, %eax
-; AVX1-SLOW-NEXT:    # kill: def $al killed $al killed $eax
-; AVX1-SLOW-NEXT:    retq
+; AVX-LABEL: test_v4i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7]
+; AVX-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX-NEXT:    # kill: def $al killed $al killed $eax
+; AVX-NEXT:    retq
 ;
-; AVX1-FAST-LABEL: test_v4i8:
-; AVX1-FAST:       # %bb.0:
-; AVX1-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-FAST-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-FAST-NEXT:    vphaddd %xmm0, %xmm0, %xmm0
-; AVX1-FAST-NEXT:    vpextrb $0, %xmm0, %eax
-; AVX1-FAST-NEXT:    # kill: def $al killed $al killed $eax
-; AVX1-FAST-NEXT:    retq
+; AVX512-LABEL: test_v4i8:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7]
+; AVX512-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX512-NEXT:    # kill: def $al killed $al killed $eax
+; AVX512-NEXT:    retq
+  %1 = call i8 @llvm.experimental.vector.reduce.add.v4i8(<4 x i8> %a0)
+  ret i8 %1
+}
+
+define i8 @test_v4i8_load(<4 x i8>* %p) {
+; SSE2-LABEL: test_v4i8_load:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    movd %xmm1, %eax
+; SSE2-NEXT:    # kill: def $al killed $al killed $eax
+; SSE2-NEXT:    retq
 ;
-; AVX2-LABEL: test_v4i8:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpextrb $0, %xmm0, %eax
-; AVX2-NEXT:    # kill: def $al killed $al killed $eax
-; AVX2-NEXT:    retq
+; SSE41-LABEL: test_v4i8_load:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE41-NEXT:    pxor %xmm1, %xmm1
+; SSE41-NEXT:    psadbw %xmm0, %xmm1
+; SSE41-NEXT:    pextrb $0, %xmm1, %eax
+; SSE41-NEXT:    # kill: def $al killed $al killed $eax
+; SSE41-NEXT:    retq
 ;
-; AVX512-LABEL: test_v4i8:
+; AVX-LABEL: test_v4i8_load:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX-NEXT:    # kill: def $al killed $al killed $eax
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: test_v4i8_load:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX512-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
 ; AVX512-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512-NEXT:    retq
+  %a0 = load <4 x i8>, <4 x i8>* %p
   %1 = call i8 @llvm.experimental.vector.reduce.add.v4i8(<4 x i8> %a0)
   ret i8 %1
 }
@@ -1013,76 +1111,76 @@ define i8 @test_v4i8(<4 x i8> %a0) {
 define i8 @test_v8i8(<8 x i8> %a0) {
 ; SSE2-LABEL: test_v8i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; SSE2-NEXT:    paddw %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
-; SSE2-NEXT:    paddw %xmm1, %xmm0
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    psrld $16, %xmm1
-; SSE2-NEXT:    paddw %xmm0, %xmm1
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    psadbw %xmm0, %xmm1
 ; SSE2-NEXT:    movd %xmm1, %eax
 ; SSE2-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: test_v8i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; SSE41-NEXT:    paddw %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
-; SSE41-NEXT:    paddw %xmm1, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    psrld $16, %xmm1
-; SSE41-NEXT:    paddw %xmm0, %xmm1
+; SSE41-NEXT:    pxor %xmm1, %xmm1
+; SSE41-NEXT:    psadbw %xmm0, %xmm1
 ; SSE41-NEXT:    pextrb $0, %xmm1, %eax
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    retq
 ;
-; AVX1-SLOW-LABEL: test_v8i8:
-; AVX1-SLOW:       # %bb.0:
-; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-SLOW-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
-; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX1-SLOW-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
-; AVX1-SLOW-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX1-SLOW-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
-; AVX1-SLOW-NEXT:    vpextrb $0, %xmm0, %eax
-; AVX1-SLOW-NEXT:    # kill: def $al killed $al killed $eax
-; AVX1-SLOW-NEXT:    retq
+; AVX-LABEL: test_v8i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX-NEXT:    # kill: def $al killed $al killed $eax
+; AVX-NEXT:    retq
 ;
-; AVX1-FAST-LABEL: test_v8i8:
-; AVX1-FAST:       # %bb.0:
-; AVX1-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-FAST-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
-; AVX1-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX1-FAST-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
-; AVX1-FAST-NEXT:    vphaddw %xmm0, %xmm0, %xmm0
-; AVX1-FAST-NEXT:    vpextrb $0, %xmm0, %eax
-; AVX1-FAST-NEXT:    # kill: def $al killed $al killed $eax
-; AVX1-FAST-NEXT:    retq
+; AVX512-LABEL: test_v8i8:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX512-NEXT:    # kill: def $al killed $al killed $eax
+; AVX512-NEXT:    retq
+  %1 = call i8 @llvm.experimental.vector.reduce.add.v8i8(<8 x i8> %a0)
+  ret i8 %1
+}
+
+define i8 @test_v8i8_load(<8 x i8>* %p) {
+; SSE2-LABEL: test_v8i8_load:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    movd %xmm1, %eax
+; SSE2-NEXT:    # kill: def $al killed $al killed $eax
+; SSE2-NEXT:    retq
 ;
-; AVX2-LABEL: test_v8i8:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpextrb $0, %xmm0, %eax
-; AVX2-NEXT:    # kill: def $al killed $al killed $eax
-; AVX2-NEXT:    retq
+; SSE41-LABEL: test_v8i8_load:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; SSE41-NEXT:    pxor %xmm1, %xmm1
+; SSE41-NEXT:    psadbw %xmm0, %xmm1
+; SSE41-NEXT:    pextrb $0, %xmm1, %eax
+; SSE41-NEXT:    # kill: def $al killed $al killed $eax
+; SSE41-NEXT:    retq
 ;
-; AVX512-LABEL: test_v8i8:
+; AVX-LABEL: test_v8i8_load:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
+; AVX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX-NEXT:    # kill: def $al killed $al killed $eax
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: test_v8i8_load:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX512-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
+; AVX512-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
 ; AVX512-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512-NEXT:    retq
+  %a0 = load <8 x i8>, <8 x i8>* %p
   %1 = call i8 @llvm.experimental.vector.reduce.add.v8i8(<8 x i8> %a0)
   ret i8 %1
 }
@@ -1092,14 +1190,8 @@ define i8 @test_v16i8(<16 x i8> %a0) {
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
-; SSE2-NEXT:    paddb %xmm1, %xmm0
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    psrld $16, %xmm1
-; SSE2-NEXT:    paddb %xmm0, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm0
-; SSE2-NEXT:    psrlw $8, %xmm0
-; SSE2-NEXT:    paddb %xmm1, %xmm0
+; SSE2-NEXT:    pxor %xmm0, %xmm0
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
 ; SSE2-NEXT:    movd %xmm0, %eax
 ; SSE2-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE2-NEXT:    retq
@@ -1108,14 +1200,8 @@ define i8 @test_v16i8(<16 x i8> %a0) {
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; SSE41-NEXT:    paddb %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
-; SSE41-NEXT:    paddb %xmm1, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    psrld $16, %xmm1
-; SSE41-NEXT:    paddb %xmm0, %xmm1
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    psrlw $8, %xmm0
-; SSE41-NEXT:    paddb %xmm1, %xmm0
+; SSE41-NEXT:    pxor %xmm0, %xmm0
+; SSE41-NEXT:    psadbw %xmm1, %xmm0
 ; SSE41-NEXT:    pextrb $0, %xmm0, %eax
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    retq
@@ -1124,12 +1210,8 @@ define i8 @test_v16i8(<16 x i8> %a0) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vpsrlw $8, %xmm0, %xmm1
-; AVX-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpextrb $0, %xmm0, %eax
 ; AVX-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX-NEXT:    retq
@@ -1138,12 +1220,8 @@ define i8 @test_v16i8(<16 x i8> %a0) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX512-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX512-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
-; AVX512-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
 ; AVX512-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512-NEXT:    retq
@@ -1157,14 +1235,8 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; SSE2-NEXT:    paddb %xmm1, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
-; SSE2-NEXT:    paddb %xmm1, %xmm0
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    psrld $16, %xmm1
-; SSE2-NEXT:    paddb %xmm0, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm0
-; SSE2-NEXT:    psrlw $8, %xmm0
-; SSE2-NEXT:    paddb %xmm1, %xmm0
+; SSE2-NEXT:    pxor %xmm0, %xmm0
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
 ; SSE2-NEXT:    movd %xmm0, %eax
 ; SSE2-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE2-NEXT:    retq
@@ -1174,14 +1246,8 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; SSE41-NEXT:    paddb %xmm1, %xmm0
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; SSE41-NEXT:    paddb %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
-; SSE41-NEXT:    paddb %xmm1, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    psrld $16, %xmm1
-; SSE41-NEXT:    paddb %xmm0, %xmm1
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    psrlw $8, %xmm0
-; SSE41-NEXT:    paddb %xmm1, %xmm0
+; SSE41-NEXT:    pxor %xmm0, %xmm0
+; SSE41-NEXT:    psadbw %xmm1, %xmm0
 ; SSE41-NEXT:    pextrb $0, %xmm0, %eax
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    retq
@@ -1192,12 +1258,8 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX1-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX1-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX1-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX1-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
-; AVX1-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpextrb $0, %xmm0, %eax
 ; AVX1-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX1-NEXT:    vzeroupper
@@ -1209,12 +1271,8 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX2-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX2-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX2-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
-; AVX2-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpextrb $0, %xmm0, %eax
 ; AVX2-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX2-NEXT:    vzeroupper
@@ -1226,12 +1284,8 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX512-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX512-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX512-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
-; AVX512-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
 ; AVX512-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512-NEXT:    vzeroupper
@@ -1248,14 +1302,8 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
 ; SSE2-NEXT:    paddb %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; SSE2-NEXT:    paddb %xmm0, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm0
-; SSE2-NEXT:    psrld $16, %xmm0
-; SSE2-NEXT:    paddb %xmm1, %xmm0
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    psrlw $8, %xmm1
-; SSE2-NEXT:    paddb %xmm0, %xmm1
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    psadbw %xmm0, %xmm1
 ; SSE2-NEXT:    movd %xmm1, %eax
 ; SSE2-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE2-NEXT:    retq
@@ -1267,14 +1315,8 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; SSE41-NEXT:    paddb %xmm0, %xmm1
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
 ; SSE41-NEXT:    paddb %xmm1, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; SSE41-NEXT:    paddb %xmm0, %xmm1
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    psrld $16, %xmm0
-; SSE41-NEXT:    paddb %xmm1, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    psrlw $8, %xmm1
-; SSE41-NEXT:    paddb %xmm0, %xmm1
+; SSE41-NEXT:    pxor %xmm1, %xmm1
+; SSE41-NEXT:    psadbw %xmm0, %xmm1
 ; SSE41-NEXT:    pextrb $0, %xmm1, %eax
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    retq
@@ -1288,12 +1330,8 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX1-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX1-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX1-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX1-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
-; AVX1-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpextrb $0, %xmm0, %eax
 ; AVX1-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX1-NEXT:    vzeroupper
@@ -1306,12 +1344,8 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX2-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX2-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX2-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
-; AVX2-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpextrb $0, %xmm0, %eax
 ; AVX2-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX2-NEXT:    vzeroupper
@@ -1320,17 +1354,13 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX512-LABEL: test_v64i8:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
-; AVX512-NEXT:    vpaddb %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpaddb %ymm1, %ymm0, %ymm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX512-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX512-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
-; AVX512-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
 ; AVX512-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512-NEXT:    vzeroupper
@@ -1342,71 +1372,55 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 define i8 @test_v128i8(<128 x i8> %a0) {
 ; SSE2-LABEL: test_v128i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    paddb %xmm6, %xmm2
 ; SSE2-NEXT:    paddb %xmm7, %xmm3
 ; SSE2-NEXT:    paddb %xmm5, %xmm3
 ; SSE2-NEXT:    paddb %xmm1, %xmm3
+; SSE2-NEXT:    paddb %xmm6, %xmm2
 ; SSE2-NEXT:    paddb %xmm4, %xmm2
 ; SSE2-NEXT:    paddb %xmm3, %xmm2
 ; SSE2-NEXT:    paddb %xmm0, %xmm2
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
 ; SSE2-NEXT:    paddb %xmm2, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; SSE2-NEXT:    paddb %xmm0, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm0
-; SSE2-NEXT:    psrld $16, %xmm0
-; SSE2-NEXT:    paddb %xmm1, %xmm0
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    psrlw $8, %xmm1
-; SSE2-NEXT:    paddb %xmm0, %xmm1
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    psadbw %xmm0, %xmm1
 ; SSE2-NEXT:    movd %xmm1, %eax
 ; SSE2-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: test_v128i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    paddb %xmm6, %xmm2
 ; SSE41-NEXT:    paddb %xmm7, %xmm3
 ; SSE41-NEXT:    paddb %xmm5, %xmm3
 ; SSE41-NEXT:    paddb %xmm1, %xmm3
+; SSE41-NEXT:    paddb %xmm6, %xmm2
 ; SSE41-NEXT:    paddb %xmm4, %xmm2
 ; SSE41-NEXT:    paddb %xmm3, %xmm2
 ; SSE41-NEXT:    paddb %xmm0, %xmm2
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
 ; SSE41-NEXT:    paddb %xmm2, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; SSE41-NEXT:    paddb %xmm0, %xmm1
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    psrld $16, %xmm0
-; SSE41-NEXT:    paddb %xmm1, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    psrlw $8, %xmm1
-; SSE41-NEXT:    paddb %xmm0, %xmm1
+; SSE41-NEXT:    pxor %xmm1, %xmm1
+; SSE41-NEXT:    psadbw %xmm0, %xmm1
 ; SSE41-NEXT:    pextrb $0, %xmm1, %eax
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: test_v128i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpaddb %xmm3, %xmm1, %xmm4
-; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm3
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm1
+; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm4
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm5
+; AVX1-NEXT:    vpaddb %xmm4, %xmm5, %xmm4
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm5
+; AVX1-NEXT:    vpaddb %xmm4, %xmm5, %xmm4
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm5
+; AVX1-NEXT:    vpaddb %xmm4, %xmm5, %xmm4
 ; AVX1-NEXT:    vpaddb %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm3
-; AVX1-NEXT:    vpaddb %xmm1, %xmm3, %xmm1
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
-; AVX1-NEXT:    vpaddb %xmm1, %xmm3, %xmm1
-; AVX1-NEXT:    vpaddb %xmm4, %xmm2, %xmm2
 ; AVX1-NEXT:    vpaddb %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vpaddb %xmm4, %xmm1, %xmm1
 ; AVX1-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX1-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX1-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX1-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
-; AVX1-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpextrb $0, %xmm0, %eax
 ; AVX1-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX1-NEXT:    vzeroupper
@@ -1421,12 +1435,8 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX2-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX2-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX2-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
-; AVX2-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpextrb $0, %xmm0, %eax
 ; AVX2-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX2-NEXT:    vzeroupper
@@ -1436,17 +1446,13 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpaddb %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
-; AVX512-NEXT:    vpaddb %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpaddb %ymm1, %ymm0, %ymm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX512-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX512-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
-; AVX512-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
 ; AVX512-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512-NEXT:    vzeroupper
diff --git a/llvm/test/CodeGen/X86/vector-reduce-and-bool.ll b/llvm/test/CodeGen/X86/vector-reduce-and-bool.ll
index b8d57f695..37bd440e7 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-and-bool.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-and-bool.ll
@@ -55,7 +55,6 @@ define i1 @trunc_v2i64_v2i1(<2 x i64>) {
 ; AVX512VL-NEXT:    vpsllq $63, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vptestmq %xmm0, %xmm0, %k0
 ; AVX512VL-NEXT:    kmovd %k0, %eax
-; AVX512VL-NEXT:    andb $3, %al
 ; AVX512VL-NEXT:    cmpb $3, %al
 ; AVX512VL-NEXT:    sete %al
 ; AVX512VL-NEXT:    retq
@@ -108,7 +107,6 @@ define i1 @trunc_v4i32_v4i1(<4 x i32>) {
 ; AVX512VL-NEXT:    vpslld $31, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vptestmd %xmm0, %xmm0, %k0
 ; AVX512VL-NEXT:    kmovd %k0, %eax
-; AVX512VL-NEXT:    andb $15, %al
 ; AVX512VL-NEXT:    cmpb $15, %al
 ; AVX512VL-NEXT:    sete %al
 ; AVX512VL-NEXT:    retq
@@ -118,17 +116,29 @@ define i1 @trunc_v4i32_v4i1(<4 x i32>) {
 }
 
 define i1 @trunc_v8i16_v8i1(<8 x i8>) {
-; SSE-LABEL: trunc_v8i16_v8i1:
-; SSE:       # %bb.0:
-; SSE-NEXT:    psllw $15, %xmm0
-; SSE-NEXT:    packsswb %xmm0, %xmm0
-; SSE-NEXT:    pmovmskb %xmm0, %eax
-; SSE-NEXT:    cmpb $-1, %al
-; SSE-NEXT:    sete %al
-; SSE-NEXT:    retq
+; SSE2-LABEL: trunc_v8i16_v8i1:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    psllw $15, %xmm0
+; SSE2-NEXT:    packsswb %xmm0, %xmm0
+; SSE2-NEXT:    pmovmskb %xmm0, %eax
+; SSE2-NEXT:    cmpb $-1, %al
+; SSE2-NEXT:    sete %al
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: trunc_v8i16_v8i1:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; SSE41-NEXT:    psllw $15, %xmm0
+; SSE41-NEXT:    packsswb %xmm0, %xmm0
+; SSE41-NEXT:    pmovmskb %xmm0, %eax
+; SSE41-NEXT:    cmpb $-1, %al
+; SSE41-NEXT:    sete %al
+; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: trunc_v8i16_v8i1:
 ; AVX:       # %bb.0:
+; AVX-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; AVX-NEXT:    vpsllw $15, %xmm0, %xmm0
 ; AVX-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
 ; AVX-NEXT:    vpmovmskb %xmm0, %eax
@@ -138,9 +148,9 @@ define i1 @trunc_v8i16_v8i1(<8 x i8>) {
 ;
 ; AVX512F-LABEL: trunc_v8i16_v8i1:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpmovsxwq %xmm0, %zmm0
-; AVX512F-NEXT:    vpsllq $63, %zmm0, %zmm0
-; AVX512F-NEXT:    vptestmq %zmm0, %zmm0, %k0
+; AVX512F-NEXT:    vpmovsxbd %xmm0, %zmm0
+; AVX512F-NEXT:    vpslld $31, %zmm0, %zmm0
+; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k0
 ; AVX512F-NEXT:    kmovw %k0, %eax
 ; AVX512F-NEXT:    cmpb $-1, %al
 ; AVX512F-NEXT:    sete %al
@@ -149,8 +159,8 @@ define i1 @trunc_v8i16_v8i1(<8 x i8>) {
 ;
 ; AVX512BW-LABEL: trunc_v8i16_v8i1:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpsllw $15, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpmovw2m %zmm0, %k0
+; AVX512BW-NEXT:    vpsllw $7, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpmovb2m %zmm0, %k0
 ; AVX512BW-NEXT:    kmovd %k0, %eax
 ; AVX512BW-NEXT:    cmpb $-1, %al
 ; AVX512BW-NEXT:    sete %al
@@ -159,8 +169,8 @@ define i1 @trunc_v8i16_v8i1(<8 x i8>) {
 ;
 ; AVX512VL-LABEL: trunc_v8i16_v8i1:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsllw $15, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpmovw2m %xmm0, %k0
+; AVX512VL-NEXT:    vpsllw $7, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpmovb2m %xmm0, %k0
 ; AVX512VL-NEXT:    kmovd %k0, %eax
 ; AVX512VL-NEXT:    cmpb $-1, %al
 ; AVX512VL-NEXT:    sete %al
@@ -247,7 +257,6 @@ define i1 @trunc_v4i64_v4i1(<4 x i64>) {
 ; AVX512VL-NEXT:    vpsllq $63, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vptestmq %ymm0, %ymm0, %k0
 ; AVX512VL-NEXT:    kmovd %k0, %eax
-; AVX512VL-NEXT:    andb $15, %al
 ; AVX512VL-NEXT:    cmpb $15, %al
 ; AVX512VL-NEXT:    sete %al
 ; AVX512VL-NEXT:    vzeroupper
@@ -349,7 +358,7 @@ define i1 @trunc_v8i32_v8i1(<8 x i32>) {
 define i1 @trunc_v16i16_v16i1(<16 x i16>) {
 ; SSE2-LABEL: trunc_v16i16_v16i1:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
 ; SSE2-NEXT:    pand %xmm2, %xmm1
 ; SSE2-NEXT:    pand %xmm2, %xmm0
 ; SSE2-NEXT:    packuswb %xmm1, %xmm0
@@ -690,7 +699,7 @@ define i1 @trunc_v16i32_v16i1(<16 x i32>) {
 define i1 @trunc_v32i16_v32i1(<32 x i16>) {
 ; SSE2-LABEL: trunc_v32i16_v32i1:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
 ; SSE2-NEXT:    pand %xmm4, %xmm3
 ; SSE2-NEXT:    pand %xmm4, %xmm2
 ; SSE2-NEXT:    packuswb %xmm3, %xmm2
@@ -751,6 +760,7 @@ define i1 @trunc_v32i16_v32i1(<32 x i16>) {
 ;
 ; AVX512F-LABEL: trunc_v32i16_v32i1:
 ; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512F-NEXT:    vpand %ymm1, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpmovsxwd %ymm0, %zmm0
 ; AVX512F-NEXT:    vpslld $31, %zmm0, %zmm0
@@ -791,82 +801,42 @@ define i1 @trunc_v32i16_v32i1(<32 x i16>) {
 }
 
 define i1 @trunc_v64i8_v64i1(<64 x i8>) {
-; SSE2-LABEL: trunc_v64i8_v64i1:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    pand %xmm3, %xmm1
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pand %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
-; SSE2-NEXT:    pand %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; SSE2-NEXT:    pand %xmm0, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm0
-; SSE2-NEXT:    psrld $16, %xmm0
-; SSE2-NEXT:    pand %xmm1, %xmm0
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    psrlw $8, %xmm1
-; SSE2-NEXT:    pand %xmm0, %xmm1
-; SSE2-NEXT:    movd %xmm1, %eax
-; SSE2-NEXT:    # kill: def $al killed $al killed $eax
-; SSE2-NEXT:    retq
-;
-; SSE41-LABEL: trunc_v64i8_v64i1:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pand %xmm3, %xmm1
-; SSE41-NEXT:    pand %xmm2, %xmm1
-; SSE41-NEXT:    pand %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
-; SSE41-NEXT:    pand %xmm1, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; SSE41-NEXT:    pand %xmm0, %xmm1
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    psrld $16, %xmm0
-; SSE41-NEXT:    pand %xmm1, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    psrlw $8, %xmm1
-; SSE41-NEXT:    pand %xmm0, %xmm1
-; SSE41-NEXT:    pextrb $0, %xmm1, %eax
-; SSE41-NEXT:    # kill: def $al killed $al killed $eax
-; SSE41-NEXT:    retq
+; SSE-LABEL: trunc_v64i8_v64i1:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pand %xmm3, %xmm1
+; SSE-NEXT:    pand %xmm2, %xmm1
+; SSE-NEXT:    pand %xmm0, %xmm1
+; SSE-NEXT:    psllw $7, %xmm1
+; SSE-NEXT:    pmovmskb %xmm1, %eax
+; SSE-NEXT:    cmpw $-1, %ax
+; SSE-NEXT:    sete %al
+; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: trunc_v64i8_v64i1:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
-; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpextrb $0, %xmm0, %eax
-; AVX1-NEXT:    # kill: def $al killed $al killed $eax
+; AVX1-NEXT:    vpsllw $7, %xmm0, %xmm0
+; AVX1-NEXT:    vpmovmskb %xmm0, %eax
+; AVX1-NEXT:    cmpw $-1, %ax
+; AVX1-NEXT:    sete %al
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: trunc_v64i8_v64i1:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
-; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpextrb $0, %xmm0, %eax
-; AVX2-NEXT:    # kill: def $al killed $al killed $eax
+; AVX2-NEXT:    vpsllw $7, %ymm0, %ymm0
+; AVX2-NEXT:    vpmovmskb %ymm0, %eax
+; AVX2-NEXT:    cmpl $-1, %eax
+; AVX2-NEXT:    sete %al
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: trunc_v64i8_v64i1:
 ; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm2
 ; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm3
 ; AVX512F-NEXT:    vpand %xmm2, %xmm3, %xmm2
@@ -892,20 +862,8 @@ define i1 @trunc_v64i8_v64i1(<64 x i8>) {
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    vpsllw $7, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpmovb2m %zmm0, %k0
-; AVX512BW-NEXT:    kshiftrq $32, %k0, %k1
-; AVX512BW-NEXT:    kandq %k1, %k0, %k0
-; AVX512BW-NEXT:    kshiftrq $16, %k0, %k1
-; AVX512BW-NEXT:    kandq %k1, %k0, %k0
-; AVX512BW-NEXT:    kshiftrq $8, %k0, %k1
-; AVX512BW-NEXT:    kandq %k1, %k0, %k0
-; AVX512BW-NEXT:    kshiftrq $4, %k0, %k1
-; AVX512BW-NEXT:    kandq %k1, %k0, %k0
-; AVX512BW-NEXT:    kshiftrq $2, %k0, %k1
-; AVX512BW-NEXT:    kandq %k1, %k0, %k0
-; AVX512BW-NEXT:    kshiftrq $1, %k0, %k1
-; AVX512BW-NEXT:    kandq %k1, %k0, %k0
-; AVX512BW-NEXT:    kmovd %k0, %eax
-; AVX512BW-NEXT:    # kill: def $al killed $al killed $eax
+; AVX512BW-NEXT:    kortestq %k0, %k0
+; AVX512BW-NEXT:    setb %al
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
@@ -913,20 +871,8 @@ define i1 @trunc_v64i8_v64i1(<64 x i8>) {
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vpsllw $7, %zmm0, %zmm0
 ; AVX512VL-NEXT:    vpmovb2m %zmm0, %k0
-; AVX512VL-NEXT:    kshiftrq $32, %k0, %k1
-; AVX512VL-NEXT:    kandq %k1, %k0, %k0
-; AVX512VL-NEXT:    kshiftrq $16, %k0, %k1
-; AVX512VL-NEXT:    kandq %k1, %k0, %k0
-; AVX512VL-NEXT:    kshiftrq $8, %k0, %k1
-; AVX512VL-NEXT:    kandq %k1, %k0, %k0
-; AVX512VL-NEXT:    kshiftrq $4, %k0, %k1
-; AVX512VL-NEXT:    kandq %k1, %k0, %k0
-; AVX512VL-NEXT:    kshiftrq $2, %k0, %k1
-; AVX512VL-NEXT:    kandq %k1, %k0, %k0
-; AVX512VL-NEXT:    kshiftrq $1, %k0, %k1
-; AVX512VL-NEXT:    kandq %k1, %k0, %k0
-; AVX512VL-NEXT:    kmovd %k0, %eax
-; AVX512VL-NEXT:    # kill: def $al killed $al killed $eax
+; AVX512VL-NEXT:    kortestq %k0, %k0
+; AVX512VL-NEXT:    setb %al
 ; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq
   %a = trunc <64 x i8> %0 to <64 x i1>
@@ -994,7 +940,6 @@ define i1 @icmp_v2i64_v2i1(<2 x i64>) {
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vptestnmq %xmm0, %xmm0, %k0
 ; AVX512VL-NEXT:    kmovd %k0, %eax
-; AVX512VL-NEXT:    andb $3, %al
 ; AVX512VL-NEXT:    cmpb $3, %al
 ; AVX512VL-NEXT:    sete %al
 ; AVX512VL-NEXT:    retq
@@ -1048,7 +993,6 @@ define i1 @icmp_v4i32_v4i1(<4 x i32>) {
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vptestnmd %xmm0, %xmm0, %k0
 ; AVX512VL-NEXT:    kmovd %k0, %eax
-; AVX512VL-NEXT:    andb $15, %al
 ; AVX512VL-NEXT:    cmpb $15, %al
 ; AVX512VL-NEXT:    sete %al
 ; AVX512VL-NEXT:    retq
@@ -1058,22 +1002,33 @@ define i1 @icmp_v4i32_v4i1(<4 x i32>) {
 }
 
 define i1 @icmp_v8i16_v8i1(<8 x i8>) {
-; SSE-LABEL: icmp_v8i16_v8i1:
-; SSE:       # %bb.0:
-; SSE-NEXT:    pand {{.*}}(%rip), %xmm0
-; SSE-NEXT:    pxor %xmm1, %xmm1
-; SSE-NEXT:    pcmpeqw %xmm0, %xmm1
-; SSE-NEXT:    packsswb %xmm0, %xmm1
-; SSE-NEXT:    pmovmskb %xmm1, %eax
-; SSE-NEXT:    cmpb $-1, %al
-; SSE-NEXT:    sete %al
-; SSE-NEXT:    retq
+; SSE2-LABEL: icmp_v8i16_v8i1:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    pcmpeqb %xmm0, %xmm1
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSE2-NEXT:    packsswb %xmm0, %xmm0
+; SSE2-NEXT:    pmovmskb %xmm0, %eax
+; SSE2-NEXT:    cmpb $-1, %al
+; SSE2-NEXT:    sete %al
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: icmp_v8i16_v8i1:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pxor %xmm1, %xmm1
+; SSE41-NEXT:    pcmpeqb %xmm0, %xmm1
+; SSE41-NEXT:    pmovsxbw %xmm1, %xmm0
+; SSE41-NEXT:    packsswb %xmm0, %xmm0
+; SSE41-NEXT:    pmovmskb %xmm0, %eax
+; SSE41-NEXT:    cmpb $-1, %al
+; SSE41-NEXT:    sete %al
+; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: icmp_v8i16_v8i1:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
 ; AVX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX-NEXT:    vpcmpeqw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpcmpeqb %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpmovsxbw %xmm0, %xmm0
 ; AVX-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
 ; AVX-NEXT:    vpmovmskb %xmm0, %eax
 ; AVX-NEXT:    cmpb $-1, %al
@@ -1082,11 +1037,10 @@ define i1 @icmp_v8i16_v8i1(<8 x i8>) {
 ;
 ; AVX512F-LABEL: icmp_v8i16_v8i1:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT:    vpcmpeqw %xmm1, %xmm0, %xmm0
-; AVX512F-NEXT:    vpmovsxwq %xmm0, %zmm0
-; AVX512F-NEXT:    vptestmq %zmm0, %zmm0, %k0
+; AVX512F-NEXT:    vpcmpeqb %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    vpmovsxbd %xmm0, %zmm0
+; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k0
 ; AVX512F-NEXT:    kmovw %k0, %eax
 ; AVX512F-NEXT:    cmpb $-1, %al
 ; AVX512F-NEXT:    sete %al
@@ -1096,8 +1050,7 @@ define i1 @icmp_v8i16_v8i1(<8 x i8>) {
 ; AVX512BW-LABEL: icmp_v8i16_v8i1:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm1 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
-; AVX512BW-NEXT:    vptestnmw %zmm1, %zmm0, %k0
+; AVX512BW-NEXT:    vptestnmb %zmm0, %zmm0, %k0
 ; AVX512BW-NEXT:    kmovd %k0, %eax
 ; AVX512BW-NEXT:    cmpb $-1, %al
 ; AVX512BW-NEXT:    sete %al
@@ -1106,7 +1059,7 @@ define i1 @icmp_v8i16_v8i1(<8 x i8>) {
 ;
 ; AVX512VL-LABEL: icmp_v8i16_v8i1:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vptestnmw {{.*}}(%rip), %xmm0, %k0
+; AVX512VL-NEXT:    vptestnmb %xmm0, %xmm0, %k0
 ; AVX512VL-NEXT:    kmovd %k0, %eax
 ; AVX512VL-NEXT:    cmpb $-1, %al
 ; AVX512VL-NEXT:    sete %al
@@ -1240,7 +1193,6 @@ define i1 @icmp_v4i64_v4i1(<4 x i64>) {
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vptestnmq %ymm0, %ymm0, %k0
 ; AVX512VL-NEXT:    kmovd %k0, %eax
-; AVX512VL-NEXT:    andb $15, %al
 ; AVX512VL-NEXT:    cmpb $15, %al
 ; AVX512VL-NEXT:    sete %al
 ; AVX512VL-NEXT:    vzeroupper
@@ -1678,6 +1630,7 @@ define i1 @icmp_v32i16_v32i1(<32 x i16>) {
 ;
 ; AVX512F-LABEL: icmp_v32i16_v32i1:
 ; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512F-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX512F-NEXT:    vpcmpeqw %ymm1, %ymm0, %ymm0
@@ -1725,9 +1678,8 @@ define i1 @icmp_v64i8_v64i1(<64 x i8>) {
 ; SSE-NEXT:    por %xmm0, %xmm1
 ; SSE-NEXT:    pcmpeqb %xmm3, %xmm1
 ; SSE-NEXT:    pmovmskb %xmm1, %eax
-; SSE-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; SSE-NEXT:    cmpw $-1, %ax
 ; SSE-NEXT:    sete %al
-; SSE-NEXT:    negb %al
 ; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: icmp_v64i8_v64i1:
@@ -1738,10 +1690,10 @@ define i1 @icmp_v64i8_v64i1(<64 x i8>) {
 ; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm1
 ; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpsllw $7, %xmm0, %xmm0
 ; AVX1-NEXT:    vpmovmskb %xmm0, %eax
-; AVX1-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; AVX1-NEXT:    cmpw $-1, %ax
 ; AVX1-NEXT:    sete %al
-; AVX1-NEXT:    negb %al
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
@@ -1753,12 +1705,12 @@ define i1 @icmp_v64i8_v64i1(<64 x i8>) {
 ; AVX2-NEXT:    vpmovmskb %ymm0, %eax
 ; AVX2-NEXT:    cmpl $-1, %eax
 ; AVX2-NEXT:    sete %al
-; AVX2-NEXT:    negb %al
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: icmp_v64i8_v64i1:
 ; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; AVX512F-NEXT:    vpcmpeqb %ymm2, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpcmpeqb %ymm2, %ymm0, %ymm0
@@ -1785,40 +1737,16 @@ define i1 @icmp_v64i8_v64i1(<64 x i8>) {
 ; AVX512BW-LABEL: icmp_v64i8_v64i1:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    vptestnmb %zmm0, %zmm0, %k0
-; AVX512BW-NEXT:    kshiftrq $32, %k0, %k1
-; AVX512BW-NEXT:    kandq %k1, %k0, %k0
-; AVX512BW-NEXT:    kshiftrq $16, %k0, %k1
-; AVX512BW-NEXT:    kandq %k1, %k0, %k0
-; AVX512BW-NEXT:    kshiftrq $8, %k0, %k1
-; AVX512BW-NEXT:    kandq %k1, %k0, %k0
-; AVX512BW-NEXT:    kshiftrq $4, %k0, %k1
-; AVX512BW-NEXT:    kandq %k1, %k0, %k0
-; AVX512BW-NEXT:    kshiftrq $2, %k0, %k1
-; AVX512BW-NEXT:    kandq %k1, %k0, %k0
-; AVX512BW-NEXT:    kshiftrq $1, %k0, %k1
-; AVX512BW-NEXT:    kandq %k1, %k0, %k0
-; AVX512BW-NEXT:    kmovd %k0, %eax
-; AVX512BW-NEXT:    # kill: def $al killed $al killed $eax
+; AVX512BW-NEXT:    kortestq %k0, %k0
+; AVX512BW-NEXT:    setb %al
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512VL-LABEL: icmp_v64i8_v64i1:
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vptestnmb %zmm0, %zmm0, %k0
-; AVX512VL-NEXT:    kshiftrq $32, %k0, %k1
-; AVX512VL-NEXT:    kandq %k1, %k0, %k0
-; AVX512VL-NEXT:    kshiftrq $16, %k0, %k1
-; AVX512VL-NEXT:    kandq %k1, %k0, %k0
-; AVX512VL-NEXT:    kshiftrq $8, %k0, %k1
-; AVX512VL-NEXT:    kandq %k1, %k0, %k0
-; AVX512VL-NEXT:    kshiftrq $4, %k0, %k1
-; AVX512VL-NEXT:    kandq %k1, %k0, %k0
-; AVX512VL-NEXT:    kshiftrq $2, %k0, %k1
-; AVX512VL-NEXT:    kandq %k1, %k0, %k0
-; AVX512VL-NEXT:    kshiftrq $1, %k0, %k1
-; AVX512VL-NEXT:    kandq %k1, %k0, %k0
-; AVX512VL-NEXT:    kmovd %k0, %eax
-; AVX512VL-NEXT:    # kill: def $al killed $al killed $eax
+; AVX512VL-NEXT:    kortestq %k0, %k0
+; AVX512VL-NEXT:    setb %al
 ; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq
   %a = icmp eq <64 x i8> %0, zeroinitializer
diff --git a/llvm/test/CodeGen/X86/vector-reduce-and.ll b/llvm/test/CodeGen/X86/vector-reduce-and.ll
index de102b44a..d3f63f6c8 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-and.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-and.ll
@@ -182,14 +182,14 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 define i32 @test_v2i32(<2 x i32> %a0) {
 ; SSE-LABEL: test_v2i32:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE-NEXT:    pand %xmm0, %xmm1
 ; SSE-NEXT:    movd %xmm1, %eax
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: test_v2i32:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovd %xmm0, %eax
 ; AVX-NEXT:    retq
@@ -397,7 +397,8 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 define i16 @test_v2i16(<2 x i16> %a0) {
 ; SSE-LABEL: test_v2i16:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE-NEXT:    movdqa %xmm0, %xmm1
+; SSE-NEXT:    psrld $16, %xmm1
 ; SSE-NEXT:    pand %xmm0, %xmm1
 ; SSE-NEXT:    movd %xmm1, %eax
 ; SSE-NEXT:    # kill: def $ax killed $ax killed $eax
@@ -405,7 +406,7 @@ define i16 @test_v2i16(<2 x i16> %a0) {
 ;
 ; AVX-LABEL: test_v2i16:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovd %xmm0, %eax
 ; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
@@ -417,9 +418,10 @@ define i16 @test_v2i16(<2 x i16> %a0) {
 define i16 @test_v4i16(<4 x i16> %a0) {
 ; SSE-LABEL: test_v4i16:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE-NEXT:    pand %xmm0, %xmm1
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE-NEXT:    movdqa %xmm1, %xmm0
+; SSE-NEXT:    psrld $16, %xmm0
 ; SSE-NEXT:    pand %xmm1, %xmm0
 ; SSE-NEXT:    movd %xmm0, %eax
 ; SSE-NEXT:    # kill: def $ax killed $ax killed $eax
@@ -427,10 +429,10 @@ define i16 @test_v4i16(<4 x i16> %a0) {
 ;
 ; AVX-LABEL: test_v4i16:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpsrld $16, %xmm0, %xmm1
+; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovd %xmm0, %eax
 ; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX-NEXT:    retq
@@ -684,7 +686,8 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 define i8 @test_v2i8(<2 x i8> %a0) {
 ; SSE2-LABEL: test_v2i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psrlw $8, %xmm1
 ; SSE2-NEXT:    pand %xmm0, %xmm1
 ; SSE2-NEXT:    movd %xmm1, %eax
 ; SSE2-NEXT:    # kill: def $al killed $al killed $eax
@@ -692,7 +695,8 @@ define i8 @test_v2i8(<2 x i8> %a0) {
 ;
 ; SSE41-LABEL: test_v2i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    psrlw $8, %xmm1
 ; SSE41-NEXT:    pand %xmm0, %xmm1
 ; SSE41-NEXT:    pextrb $0, %xmm1, %eax
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
@@ -700,7 +704,7 @@ define i8 @test_v2i8(<2 x i8> %a0) {
 ;
 ; AVX-LABEL: test_v2i8:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpextrb $0, %xmm0, %eax
 ; AVX-NEXT:    # kill: def $al killed $al killed $eax
@@ -712,9 +716,11 @@ define i8 @test_v2i8(<2 x i8> %a0) {
 define i8 @test_v4i8(<4 x i8> %a0) {
 ; SSE2-LABEL: test_v4i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psrld $16, %xmm1
 ; SSE2-NEXT:    pand %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    psrlw $8, %xmm0
 ; SSE2-NEXT:    pand %xmm1, %xmm0
 ; SSE2-NEXT:    movd %xmm0, %eax
 ; SSE2-NEXT:    # kill: def $al killed $al killed $eax
@@ -722,9 +728,11 @@ define i8 @test_v4i8(<4 x i8> %a0) {
 ;
 ; SSE41-LABEL: test_v4i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    psrld $16, %xmm1
 ; SSE41-NEXT:    pand %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    psrlw $8, %xmm0
 ; SSE41-NEXT:    pand %xmm1, %xmm0
 ; SSE41-NEXT:    pextrb $0, %xmm0, %eax
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
@@ -732,9 +740,9 @@ define i8 @test_v4i8(<4 x i8> %a0) {
 ;
 ; AVX-LABEL: test_v4i8:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpextrb $0, %xmm0, %eax
 ; AVX-NEXT:    # kill: def $al killed $al killed $eax
@@ -746,12 +754,13 @@ define i8 @test_v4i8(<4 x i8> %a0) {
 define i8 @test_v8i8(<8 x i8> %a0) {
 ; SSE2-LABEL: test_v8i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE2-NEXT:    pand %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    psrld $16, %xmm0
 ; SSE2-NEXT:    pand %xmm1, %xmm0
 ; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    psrld $16, %xmm1
+; SSE2-NEXT:    psrlw $8, %xmm1
 ; SSE2-NEXT:    pand %xmm0, %xmm1
 ; SSE2-NEXT:    movd %xmm1, %eax
 ; SSE2-NEXT:    # kill: def $al killed $al killed $eax
@@ -759,12 +768,13 @@ define i8 @test_v8i8(<8 x i8> %a0) {
 ;
 ; SSE41-LABEL: test_v8i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE41-NEXT:    pand %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    psrld $16, %xmm0
 ; SSE41-NEXT:    pand %xmm1, %xmm0
 ; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    psrld $16, %xmm1
+; SSE41-NEXT:    psrlw $8, %xmm1
 ; SSE41-NEXT:    pand %xmm0, %xmm1
 ; SSE41-NEXT:    pextrb $0, %xmm1, %eax
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
@@ -772,12 +782,12 @@ define i8 @test_v8i8(<8 x i8> %a0) {
 ;
 ; AVX-LABEL: test_v8i8:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpextrb $0, %xmm0, %eax
 ; AVX-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/vector-reduce-fadd-fast.ll b/llvm/test/CodeGen/X86/vector-reduce-fadd-fast.ll
index acc02a2d8..c94f584df 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-fadd-fast.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-fadd-fast.ll
@@ -90,8 +90,7 @@ define float @test_v4f32(float %a0, <4 x float> %a1) {
 ;
 ; AVX1-FAST-LABEL: test_v4f32:
 ; AVX1-FAST:       # %bb.0:
-; AVX1-FAST-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm1[1,0]
-; AVX1-FAST-NEXT:    vaddps %xmm2, %xmm1, %xmm1
+; AVX1-FAST-NEXT:    vhaddps %xmm1, %xmm1, %xmm1
 ; AVX1-FAST-NEXT:    vhaddps %xmm1, %xmm1, %xmm1
 ; AVX1-FAST-NEXT:    vaddss %xmm1, %xmm0, %xmm0
 ; AVX1-FAST-NEXT:    retq
@@ -156,9 +155,8 @@ define float @test_v8f32(float %a0, <8 x float> %a1) {
 ; AVX1-FAST-LABEL: test_v8f32:
 ; AVX1-FAST:       # %bb.0:
 ; AVX1-FAST-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX1-FAST-NEXT:    vaddps %xmm2, %xmm1, %xmm1
-; AVX1-FAST-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm1[1,0]
-; AVX1-FAST-NEXT:    vaddps %xmm2, %xmm1, %xmm1
+; AVX1-FAST-NEXT:    vhaddps %xmm1, %xmm2, %xmm1
+; AVX1-FAST-NEXT:    vhaddps %xmm1, %xmm1, %xmm1
 ; AVX1-FAST-NEXT:    vhaddps %xmm1, %xmm1, %xmm1
 ; AVX1-FAST-NEXT:    vaddss %xmm1, %xmm0, %xmm0
 ; AVX1-FAST-NEXT:    vzeroupper
@@ -350,8 +348,7 @@ define float @test_v4f32_zero(<4 x float> %a0) {
 ;
 ; AVX1-FAST-LABEL: test_v4f32_zero:
 ; AVX1-FAST:       # %bb.0:
-; AVX1-FAST-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
-; AVX1-FAST-NEXT:    vaddps %xmm1, %xmm0, %xmm0
+; AVX1-FAST-NEXT:    vhaddps %xmm0, %xmm0, %xmm0
 ; AVX1-FAST-NEXT:    vhaddps %xmm0, %xmm0, %xmm0
 ; AVX1-FAST-NEXT:    retq
 ;
@@ -411,9 +408,8 @@ define float @test_v8f32_zero(<8 x float> %a0) {
 ; AVX1-FAST-LABEL: test_v8f32_zero:
 ; AVX1-FAST:       # %bb.0:
 ; AVX1-FAST-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-FAST-NEXT:    vaddps %xmm1, %xmm0, %xmm0
-; AVX1-FAST-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
-; AVX1-FAST-NEXT:    vaddps %xmm1, %xmm0, %xmm0
+; AVX1-FAST-NEXT:    vhaddps %xmm0, %xmm1, %xmm0
+; AVX1-FAST-NEXT:    vhaddps %xmm0, %xmm0, %xmm0
 ; AVX1-FAST-NEXT:    vhaddps %xmm0, %xmm0, %xmm0
 ; AVX1-FAST-NEXT:    vzeroupper
 ; AVX1-FAST-NEXT:    retq
@@ -597,8 +593,7 @@ define float @test_v4f32_undef(<4 x float> %a0) {
 ;
 ; AVX1-FAST-LABEL: test_v4f32_undef:
 ; AVX1-FAST:       # %bb.0:
-; AVX1-FAST-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
-; AVX1-FAST-NEXT:    vaddps %xmm1, %xmm0, %xmm0
+; AVX1-FAST-NEXT:    vhaddps %xmm0, %xmm0, %xmm0
 ; AVX1-FAST-NEXT:    vhaddps %xmm0, %xmm0, %xmm0
 ; AVX1-FAST-NEXT:    retq
 ;
@@ -658,9 +653,8 @@ define float @test_v8f32_undef(<8 x float> %a0) {
 ; AVX1-FAST-LABEL: test_v8f32_undef:
 ; AVX1-FAST:       # %bb.0:
 ; AVX1-FAST-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-FAST-NEXT:    vaddps %xmm1, %xmm0, %xmm0
-; AVX1-FAST-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
-; AVX1-FAST-NEXT:    vaddps %xmm1, %xmm0, %xmm0
+; AVX1-FAST-NEXT:    vhaddps %xmm0, %xmm1, %xmm0
+; AVX1-FAST-NEXT:    vhaddps %xmm0, %xmm0, %xmm0
 ; AVX1-FAST-NEXT:    vhaddps %xmm0, %xmm0, %xmm0
 ; AVX1-FAST-NEXT:    vzeroupper
 ; AVX1-FAST-NEXT:    retq
@@ -834,7 +828,7 @@ define double @test_v4f64(double %a0, <4 x double> %a1) {
 ; AVX1-FAST-LABEL: test_v4f64:
 ; AVX1-FAST:       # %bb.0:
 ; AVX1-FAST-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX1-FAST-NEXT:    vaddpd %xmm2, %xmm1, %xmm1
+; AVX1-FAST-NEXT:    vhaddpd %xmm1, %xmm2, %xmm1
 ; AVX1-FAST-NEXT:    vhaddpd %xmm1, %xmm1, %xmm1
 ; AVX1-FAST-NEXT:    vaddsd %xmm1, %xmm0, %xmm0
 ; AVX1-FAST-NEXT:    vzeroupper
@@ -1053,7 +1047,7 @@ define double @test_v4f64_zero(<4 x double> %a0) {
 ; AVX1-FAST-LABEL: test_v4f64_zero:
 ; AVX1-FAST:       # %bb.0:
 ; AVX1-FAST-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-FAST-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
+; AVX1-FAST-NEXT:    vhaddpd %xmm0, %xmm1, %xmm0
 ; AVX1-FAST-NEXT:    vhaddpd %xmm0, %xmm0, %xmm0
 ; AVX1-FAST-NEXT:    vzeroupper
 ; AVX1-FAST-NEXT:    retq
@@ -1260,7 +1254,7 @@ define double @test_v4f64_undef(<4 x double> %a0) {
 ; AVX1-FAST-LABEL: test_v4f64_undef:
 ; AVX1-FAST:       # %bb.0:
 ; AVX1-FAST-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-FAST-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
+; AVX1-FAST-NEXT:    vhaddpd %xmm0, %xmm1, %xmm0
 ; AVX1-FAST-NEXT:    vhaddpd %xmm0, %xmm0, %xmm0
 ; AVX1-FAST-NEXT:    vzeroupper
 ; AVX1-FAST-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/vector-reduce-mul.ll b/llvm/test/CodeGen/X86/vector-reduce-mul.ll
index e89857b25..927f51e9b 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-mul.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-mul.ll
@@ -776,24 +776,31 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ;
 
 define i32 @test_v2i32(<2 x i32> %a0) {
-; SSE-LABEL: test_v2i32:
-; SSE:       # %bb.0:
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; SSE-NEXT:    pmuludq %xmm0, %xmm1
-; SSE-NEXT:    movd %xmm1, %eax
-; SSE-NEXT:    retq
+; SSE2-LABEL: test_v2i32:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE2-NEXT:    pmuludq %xmm0, %xmm1
+; SSE2-NEXT:    movd %xmm1, %eax
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: test_v2i32:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE41-NEXT:    pmulld %xmm0, %xmm1
+; SSE41-NEXT:    movd %xmm1, %eax
+; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: test_v2i32:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovd %xmm0, %eax
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: test_v2i32:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
 ; AVX512-NEXT:    retq
   %1 = call i32 @llvm.experimental.vector.reduce.mul.v2i32(<2 x i32> %a0)
@@ -807,12 +814,8 @@ define i32 @test_v4i32(<4 x i32> %a0) {
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,3,1,1]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
 ; SSE2-NEXT:    pmuludq %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,2,2,3]
-; SSE2-NEXT:    pmuludq %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE2-NEXT:    pmuludq %xmm0, %xmm1
+; SSE2-NEXT:    pmuludq %xmm3, %xmm1
 ; SSE2-NEXT:    movd %xmm1, %eax
 ; SSE2-NEXT:    retq
 ;
@@ -849,23 +852,16 @@ define i32 @test_v4i32(<4 x i32> %a0) {
 define i32 @test_v8i32(<8 x i32> %a0) {
 ; SSE2-LABEL: test_v8i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pmuludq %xmm2, %xmm3
 ; SSE2-NEXT:    pmuludq %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pmuludq %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[0,2,2,3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
-; SSE2-NEXT:    pmuludq %xmm0, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,2,0,0]
-; SSE2-NEXT:    pmuludq %xmm1, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; SSE2-NEXT:    pmuludq %xmm0, %xmm1
-; SSE2-NEXT:    movd %xmm1, %eax
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[2,2,0,0]
+; SSE2-NEXT:    pmuludq %xmm3, %xmm0
+; SSE2-NEXT:    pmuludq %xmm1, %xmm0
+; SSE2-NEXT:    movd %xmm0, %eax
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: test_v8i32:
@@ -920,29 +916,22 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 define i32 @test_v16i32(<16 x i32> %a0) {
 ; SSE2-LABEL: test_v16i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pmuludq %xmm4, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,1,3,3]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pmuludq %xmm4, %xmm6
-; SSE2-NEXT:    pmuludq %xmm5, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,3,3]
 ; SSE2-NEXT:    pmuludq %xmm3, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[1,1,3,3]
 ; SSE2-NEXT:    pmuludq %xmm2, %xmm0
 ; SSE2-NEXT:    pmuludq %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm6[0,2,2,3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
+; SSE2-NEXT:    pmuludq %xmm4, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE2-NEXT:    pmuludq %xmm5, %xmm2
+; SSE2-NEXT:    pmuludq %xmm1, %xmm2
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; SSE2-NEXT:    pmuludq %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm6[2,2,0,0]
-; SSE2-NEXT:    pmuludq %xmm6, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; SSE2-NEXT:    pmuludq %xmm0, %xmm1
-; SSE2-NEXT:    movd %xmm1, %eax
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,2,0,0]
+; SSE2-NEXT:    pmuludq %xmm2, %xmm0
+; SSE2-NEXT:    pmuludq %xmm1, %xmm0
+; SSE2-NEXT:    movd %xmm0, %eax
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: test_v16i32:
@@ -1005,39 +994,32 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 define i32 @test_v32i32(<32 x i32> %a0) {
 ; SSE2-LABEL: test_v32i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm2[1,1,3,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm6[1,1,3,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm2[1,1,3,3]
+; SSE2-NEXT:    pmuludq %xmm8, %xmm9
+; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm4[1,1,3,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pmuludq %xmm8, %xmm10
+; SSE2-NEXT:    pmuludq %xmm9, %xmm10
+; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm7[1,1,3,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm3[1,1,3,3]
+; SSE2-NEXT:    pmuludq %xmm8, %xmm9
+; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm5[1,1,3,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pmuludq %xmm8, %xmm11
+; SSE2-NEXT:    pmuludq %xmm9, %xmm11
+; SSE2-NEXT:    pmuludq %xmm10, %xmm11
 ; SSE2-NEXT:    pmuludq %xmm6, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm0[1,1,3,3]
 ; SSE2-NEXT:    pmuludq %xmm4, %xmm0
 ; SSE2-NEXT:    pmuludq %xmm2, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,1,3,3]
 ; SSE2-NEXT:    pmuludq %xmm7, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm1[1,1,3,3]
 ; SSE2-NEXT:    pmuludq %xmm5, %xmm1
 ; SSE2-NEXT:    pmuludq %xmm3, %xmm1
 ; SSE2-NEXT:    pmuludq %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3]
-; SSE2-NEXT:    pmuludq %xmm8, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
-; SSE2-NEXT:    pmuludq %xmm9, %xmm3
-; SSE2-NEXT:    pmuludq %xmm0, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[1,1,3,3]
-; SSE2-NEXT:    pmuludq %xmm2, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm5[1,1,3,3]
-; SSE2-NEXT:    pmuludq %xmm10, %xmm2
-; SSE2-NEXT:    pmuludq %xmm0, %xmm2
-; SSE2-NEXT:    pmuludq %xmm3, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; SSE2-NEXT:    pmuludq %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,2,0,0]
-; SSE2-NEXT:    pmuludq %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pmuludq %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm11[2,2,0,0]
+; SSE2-NEXT:    pmuludq %xmm11, %xmm1
 ; SSE2-NEXT:    pmuludq %xmm0, %xmm1
 ; SSE2-NEXT:    movd %xmm1, %eax
 ; SSE2-NEXT:    retq
@@ -1119,24 +1101,25 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 define i16 @test_v2i16(<2 x i16> %a0) {
 ; SSE-LABEL: test_v2i16:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; SSE-NEXT:    pmuludq %xmm0, %xmm1
+; SSE-NEXT:    movdqa %xmm0, %xmm1
+; SSE-NEXT:    psrld $16, %xmm1
+; SSE-NEXT:    pmullw %xmm0, %xmm1
 ; SSE-NEXT:    movd %xmm1, %eax
 ; SSE-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: test_v2i16:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpsrld $16, %xmm0, %xmm1
+; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovd %xmm0, %eax
 ; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: test_v2i16:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
+; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
 ; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX512-NEXT:    retq
@@ -1145,48 +1128,33 @@ define i16 @test_v2i16(<2 x i16> %a0) {
 }
 
 define i16 @test_v4i16(<4 x i16> %a0) {
-; SSE2-LABEL: test_v4i16:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,3,1,1]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pmuludq %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,2,2,3]
-; SSE2-NEXT:    pmuludq %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; SSE2-NEXT:    pmuludq %xmm0, %xmm1
-; SSE2-NEXT:    pextrw $0, %xmm1, %eax
-; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT:    retq
-;
-; SSE41-LABEL: test_v4i16:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; SSE41-NEXT:    pmulld %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
-; SSE41-NEXT:    pmulld %xmm1, %xmm0
-; SSE41-NEXT:    movd %xmm0, %eax
-; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT:    retq
+; SSE-LABEL: test_v4i16:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE-NEXT:    pmullw %xmm0, %xmm1
+; SSE-NEXT:    movdqa %xmm1, %xmm0
+; SSE-NEXT:    psrld $16, %xmm0
+; SSE-NEXT:    pmullw %xmm1, %xmm0
+; SSE-NEXT:    movd %xmm0, %eax
+; SSE-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: test_v4i16:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpsrld $16, %xmm0, %xmm1
+; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovd %xmm0, %eax
 ; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: test_v4i16:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
+; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
 ; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX512-NEXT:    retq
@@ -1385,6 +1353,7 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ;
 ; AVX512DQ-LABEL: test_v32i16:
 ; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512DQ-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
@@ -1401,6 +1370,7 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ;
 ; AVX512DQVL-LABEL: test_v32i16:
 ; AVX512DQVL:       # %bb.0:
+; AVX512DQVL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512DQVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
@@ -1563,32 +1533,34 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 define i8 @test_v2i8(<2 x i8> %a0) {
 ; SSE2-LABEL: test_v2i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; SSE2-NEXT:    pmuludq %xmm0, %xmm1
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psrlw $8, %xmm1
+; SSE2-NEXT:    pmullw %xmm0, %xmm1
 ; SSE2-NEXT:    movd %xmm1, %eax
 ; SSE2-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: test_v2i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; SSE41-NEXT:    pmuludq %xmm0, %xmm1
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    psrlw $8, %xmm1
+; SSE41-NEXT:    pmullw %xmm0, %xmm1
 ; SSE41-NEXT:    pextrb $0, %xmm1, %eax
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: test_v2i8:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpextrb $0, %xmm0, %eax
 ; AVX-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: test_v2i8:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
 ; AVX512-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512-NEXT:    retq
@@ -1599,49 +1571,76 @@ define i8 @test_v2i8(<2 x i8> %a0) {
 define i8 @test_v4i8(<4 x i8> %a0) {
 ; SSE2-LABEL: test_v4i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,3,1,1]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pmuludq %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,2,2,3]
-; SSE2-NEXT:    pmuludq %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; SSE2-NEXT:    pmuludq %xmm0, %xmm1
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    pmullw %xmm1, %xmm0
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    packuswb %xmm0, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psrlw $8, %xmm1
+; SSE2-NEXT:    pmullw %xmm0, %xmm1
 ; SSE2-NEXT:    movd %xmm1, %eax
 ; SSE2-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: test_v4i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; SSE41-NEXT:    pmulld %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
-; SSE41-NEXT:    pmulld %xmm1, %xmm0
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE41-NEXT:    pmullw %xmm0, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[2],zero,xmm0[6],zero,xmm0[u],zero,xmm0[u],zero,xmm0[u],zero,xmm0[u],zero,xmm0[u],zero,xmm0[u],zero
+; SSE41-NEXT:    pmullw %xmm1, %xmm0
 ; SSE41-NEXT:    pextrb $0, %xmm0, %eax
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: test_v4i8:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpshufb {{.*#+}} xmm1 = xmm0[2],zero,xmm0[6],zero,xmm0[u],zero,xmm0[u],zero,xmm0[u],zero,xmm0[u],zero,xmm0[u],zero,xmm0[u],zero
+; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpextrb $0, %xmm0, %eax
 ; AVX-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX-NEXT:    retq
 ;
-; AVX512-LABEL: test_v4i8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
-; AVX512-NEXT:    # kill: def $al killed $al killed $eax
-; AVX512-NEXT:    retq
+; AVX512BW-LABEL: test_v4i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm1 = xmm0[2],zero,xmm0[6],zero,xmm0[u],zero,xmm0[u],zero,xmm0[u],zero,xmm0[u],zero,xmm0[u],zero,xmm0[u],zero
+; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX512BW-NEXT:    # kill: def $al killed $al killed $eax
+; AVX512BW-NEXT:    retq
+;
+; AVX512VL-LABEL: test_v4i8:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; AVX512VL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; AVX512VL-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512VL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpshufb {{.*#+}} xmm1 = xmm0[2],zero,xmm0[6],zero,xmm0[u],zero,xmm0[u],zero,xmm0[u],zero,xmm0[u],zero,xmm0[u],zero,xmm0[u],zero
+; AVX512VL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX512VL-NEXT:    # kill: def $al killed $al killed $eax
+; AVX512VL-NEXT:    retq
+;
+; AVX512DQ-LABEL: test_v4i8:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm1 = xmm0[2],zero,xmm0[6],zero,xmm0[u],zero,xmm0[u],zero,xmm0[u],zero,xmm0[u],zero,xmm0[u],zero,xmm0[u],zero
+; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX512DQ-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX512DQ-NEXT:    # kill: def $al killed $al killed $eax
+; AVX512DQ-NEXT:    retq
   %1 = call i8 @llvm.experimental.vector.reduce.mul.v4i8(<4 x i8> %a0)
   ret i8 %1
 }
@@ -1649,12 +1648,16 @@ define i8 @test_v4i8(<4 x i8> %a0) {
 define i8 @test_v8i8(<8 x i8> %a0) {
 ; SSE2-LABEL: test_v8i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,2,3,3]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
 ; SSE2-NEXT:    pmullw %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,2,3,0]
 ; SSE2-NEXT:    pmullw %xmm1, %xmm0
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    packuswb %xmm0, %xmm0
 ; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    psrld $16, %xmm1
+; SSE2-NEXT:    psrlw $8, %xmm1
 ; SSE2-NEXT:    pmullw %xmm0, %xmm1
 ; SSE2-NEXT:    movd %xmm1, %eax
 ; SSE2-NEXT:    # kill: def $al killed $al killed $eax
@@ -1662,40 +1665,72 @@ define i8 @test_v8i8(<8 x i8> %a0) {
 ;
 ; SSE41-LABEL: test_v8i8:
 ; SSE41:       # %bb.0:
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; SSE41-NEXT:    pmullw %xmm0, %xmm1
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
-; SSE41-NEXT:    pmullw %xmm1, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    psrld $16, %xmm1
+; SSE41-NEXT:    pshufb {{.*#+}} xmm1 = xmm1[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
 ; SSE41-NEXT:    pmullw %xmm0, %xmm1
-; SSE41-NEXT:    pextrb $0, %xmm1, %eax
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[2],zero,xmm0[6],zero,xmm0[10],zero,xmm0[14],zero,xmm0[u],zero,xmm0[u],zero,xmm0[u],zero,xmm0[u],zero
+; SSE41-NEXT:    pmullw %xmm1, %xmm0
+; SSE41-NEXT:    pextrb $0, %xmm0, %eax
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: test_v8i8:
 ; AVX:       # %bb.0:
+; AVX-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vpsrld $16, %xmm0, %xmm1
+; AVX-NEXT:    vpshufb {{.*#+}} xmm1 = xmm0[2],zero,xmm0[6],zero,xmm0[10],zero,xmm0[14],zero,xmm0[u],zero,xmm0[u],zero,xmm0[u],zero,xmm0[u],zero
 ; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpextrb $0, %xmm0, %eax
 ; AVX-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX-NEXT:    retq
 ;
-; AVX512-LABEL: test_v8i8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
-; AVX512-NEXT:    # kill: def $al killed $al killed $eax
-; AVX512-NEXT:    retq
+; AVX512BW-LABEL: test_v8i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm1 = xmm0[2],zero,xmm0[6],zero,xmm0[10],zero,xmm0[14],zero,xmm0[u],zero,xmm0[u],zero,xmm0[u],zero,xmm0[u],zero
+; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX512BW-NEXT:    # kill: def $al killed $al killed $eax
+; AVX512BW-NEXT:    retq
+;
+; AVX512VL-LABEL: test_v8i8:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; AVX512VL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; AVX512VL-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512VL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpalignr {{.*#+}} xmm1 = xmm0[4,5,6,7,8,9,10,11,12,13,14,15,0,1,2,3]
+; AVX512VL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpshufb {{.*#+}} xmm1 = xmm0[2],zero,xmm0[6],zero,xmm0[10],zero,xmm0[14],zero,xmm0[u],zero,xmm0[u],zero,xmm0[u],zero,xmm0[u],zero
+; AVX512VL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX512VL-NEXT:    # kill: def $al killed $al killed $eax
+; AVX512VL-NEXT:    retq
+;
+; AVX512DQ-LABEL: test_v8i8:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm1 = xmm0[2],zero,xmm0[6],zero,xmm0[10],zero,xmm0[14],zero,xmm0[u],zero,xmm0[u],zero,xmm0[u],zero,xmm0[u],zero
+; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX512DQ-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX512DQ-NEXT:    # kill: def $al killed $al killed $eax
+; AVX512DQ-NEXT:    retq
   %1 = call i8 @llvm.experimental.vector.reduce.mul.v8i8(<8 x i8> %a0)
   ret i8 %1
 }
@@ -1721,14 +1756,11 @@ define i8 @test_v16i8(<16 x i8> %a0) {
 ; SSE2-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    pmullw %xmm2, %xmm0
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    pand %xmm1, %xmm2
-; SSE2-NEXT:    packuswb %xmm3, %xmm2
-; SSE2-NEXT:    psrlw $8, %xmm2
-; SSE2-NEXT:    pmullw %xmm0, %xmm2
-; SSE2-NEXT:    pand %xmm1, %xmm2
-; SSE2-NEXT:    packuswb %xmm3, %xmm2
-; SSE2-NEXT:    movd %xmm2, %eax
+; SSE2-NEXT:    pand %xmm0, %xmm1
+; SSE2-NEXT:    packuswb %xmm3, %xmm1
+; SSE2-NEXT:    psrlw $8, %xmm1
+; SSE2-NEXT:    pmullw %xmm0, %xmm1
+; SSE2-NEXT:    movd %xmm1, %eax
 ; SSE2-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE2-NEXT:    retq
 ;
@@ -1738,21 +1770,19 @@ define i8 @test_v16i8(<16 x i8> %a0) {
 ; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; SSE41-NEXT:    pmullw %xmm1, %xmm0
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
 ; SSE41-NEXT:    pand %xmm1, %xmm0
-; SSE41-NEXT:    pxor %xmm2, %xmm2
-; SSE41-NEXT:    packuswb %xmm2, %xmm0
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; SSE41-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; SSE41-NEXT:    pmullw %xmm3, %xmm0
+; SSE41-NEXT:    pxor %xmm3, %xmm3
+; SSE41-NEXT:    packuswb %xmm3, %xmm0
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; SSE41-NEXT:    pmullw %xmm2, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
 ; SSE41-NEXT:    pand %xmm1, %xmm0
-; SSE41-NEXT:    packuswb %xmm2, %xmm0
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; SSE41-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; SSE41-NEXT:    pmullw %xmm3, %xmm0
+; SSE41-NEXT:    packuswb %xmm3, %xmm0
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; SSE41-NEXT:    pmullw %xmm2, %xmm0
 ; SSE41-NEXT:    pand %xmm1, %xmm0
-; SSE41-NEXT:    packuswb %xmm2, %xmm0
+; SSE41-NEXT:    packuswb %xmm3, %xmm0
 ; SSE41-NEXT:    movdqa %xmm0, %xmm1
 ; SSE41-NEXT:    psrlw $8, %xmm1
 ; SSE41-NEXT:    pmullw %xmm0, %xmm1
@@ -1765,23 +1795,14 @@ define i8 @test_v16i8(<16 x i8> %a0) {
 ; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
-; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm1
 ; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vpmullw %xmm0, %xmm3, %xmm0
-; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; AVX1-NEXT:    vpmullw %xmm0, %xmm3, %xmm0
-; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX1-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpsrlw $8, %xmm1, %xmm1
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpextrb $0, %xmm0, %eax
 ; AVX1-NEXT:    # kill: def $al killed $al killed $eax
@@ -1792,15 +1813,12 @@ define i8 @test_v16i8(<16 x i8> %a0) {
 ; AVX2-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; AVX2-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
-; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
-; AVX2-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
-; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
-; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm0[1,1,2,3,4,5,6,7]
+; AVX2-NEXT:    vpshufb {{.*#+}} xmm1 = xmm0[8],zero,xmm0[10],zero,xmm0[12],zero,xmm0[14],zero,xmm0[8],zero,xmm0[10],zero,xmm0[12],zero,xmm0[14],zero
+; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufb {{.*#+}} xmm1 = xmm0[2],zero,xmm0[2],zero,xmm0[4],zero,xmm0[6],zero,xmm0[8],zero,xmm0[10],zero,xmm0[12],zero,xmm0[14],zero
 ; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpextrb $0, %xmm0, %eax
 ; AVX2-NEXT:    # kill: def $al killed $al killed $eax
@@ -1822,9 +1840,7 @@ define i8 @test_v16i8(<16 x i8> %a0) {
 ; AVX512BW-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
 ; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm1
 ; AVX512BW-NEXT:    vpsrlw $8, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
-; AVX512BW-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
-; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vpextrb $0, %xmm0, %eax
 ; AVX512BW-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512BW-NEXT:    vzeroupper
@@ -1846,9 +1862,7 @@ define i8 @test_v16i8(<16 x i8> %a0) {
 ; AVX512BWVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
 ; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm1
 ; AVX512BWVL-NEXT:    vpsrlw $8, %xmm1, %xmm1
-; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
-; AVX512BWVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
-; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm0
+; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    vpextrb $0, %xmm0, %eax
 ; AVX512BWVL-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512BWVL-NEXT:    vzeroupper
@@ -1873,10 +1887,7 @@ define i8 @test_v16i8(<16 x i8> %a0) {
 ; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
 ; AVX512DQ-NEXT:    vpmovdb %zmm1, %xmm1
 ; AVX512DQ-NEXT:    vpsrlw $8, %xmm1, %xmm1
-; AVX512DQ-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
-; AVX512DQ-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
-; AVX512DQ-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512DQ-NEXT:    vpextrb $0, %xmm0, %eax
 ; AVX512DQ-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512DQ-NEXT:    vzeroupper
@@ -1901,10 +1912,7 @@ define i8 @test_v16i8(<16 x i8> %a0) {
 ; AVX512DQVL-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
 ; AVX512DQVL-NEXT:    vpmovdb %zmm1, %xmm1
 ; AVX512DQVL-NEXT:    vpsrlw $8, %xmm1, %xmm1
-; AVX512DQVL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
-; AVX512DQVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
-; AVX512DQVL-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
-; AVX512DQVL-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512DQVL-NEXT:    vpextrb $0, %xmm0, %eax
 ; AVX512DQVL-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512DQVL-NEXT:    vzeroupper
@@ -1939,14 +1947,11 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; SSE2-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    pmullw %xmm2, %xmm0
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    pand %xmm1, %xmm2
-; SSE2-NEXT:    packuswb %xmm3, %xmm2
-; SSE2-NEXT:    psrlw $8, %xmm2
-; SSE2-NEXT:    pmullw %xmm0, %xmm2
-; SSE2-NEXT:    pand %xmm1, %xmm2
-; SSE2-NEXT:    packuswb %xmm3, %xmm2
-; SSE2-NEXT:    movd %xmm2, %eax
+; SSE2-NEXT:    pand %xmm0, %xmm1
+; SSE2-NEXT:    packuswb %xmm3, %xmm1
+; SSE2-NEXT:    psrlw $8, %xmm1
+; SSE2-NEXT:    pmullw %xmm0, %xmm1
+; SSE2-NEXT:    movd %xmm1, %eax
 ; SSE2-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE2-NEXT:    retq
 ;
@@ -1956,30 +1961,31 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; SSE41-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
 ; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSE41-NEXT:    pmullw %xmm2, %xmm3
-; SSE41-NEXT:    pshufb {{.*#+}} xmm3 = xmm3[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
-; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
-; SSE41-NEXT:    pmullw %xmm1, %xmm3
-; SSE41-NEXT:    pmullw %xmm0, %xmm3
-; SSE41-NEXT:    pand %xmm2, %xmm3
-; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    packuswb %xmm0, %xmm3
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
-; SSE41-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[2,3,0,1]
-; SSE41-NEXT:    pmullw %xmm1, %xmm3
-; SSE41-NEXT:    pand %xmm2, %xmm3
-; SSE41-NEXT:    packuswb %xmm0, %xmm3
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
-; SSE41-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,2,3]
-; SSE41-NEXT:    pmullw %xmm1, %xmm3
-; SSE41-NEXT:    pand %xmm2, %xmm3
-; SSE41-NEXT:    packuswb %xmm0, %xmm3
-; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    pmullw %xmm1, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
+; SSE41-NEXT:    movdqa %xmm0, %xmm4
+; SSE41-NEXT:    pand %xmm1, %xmm4
+; SSE41-NEXT:    pmullw %xmm2, %xmm3
+; SSE41-NEXT:    pand %xmm1, %xmm3
+; SSE41-NEXT:    packuswb %xmm4, %xmm3
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm2 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
+; SSE41-NEXT:    pmullw %xmm0, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE41-NEXT:    pand %xmm1, %xmm2
+; SSE41-NEXT:    pxor %xmm3, %xmm3
+; SSE41-NEXT:    packuswb %xmm3, %xmm2
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
+; SSE41-NEXT:    pmullw %xmm0, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
+; SSE41-NEXT:    pand %xmm1, %xmm2
+; SSE41-NEXT:    packuswb %xmm3, %xmm2
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
+; SSE41-NEXT:    pmullw %xmm0, %xmm2
+; SSE41-NEXT:    pand %xmm1, %xmm2
+; SSE41-NEXT:    packuswb %xmm3, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
 ; SSE41-NEXT:    psrlw $8, %xmm0
-; SSE41-NEXT:    pmullw %xmm3, %xmm0
+; SSE41-NEXT:    pmullw %xmm2, %xmm0
 ; SSE41-NEXT:    pextrb $0, %xmm0, %eax
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    retq
@@ -1994,22 +2000,13 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX1-NEXT:    vpmullw %xmm2, %xmm1, %xmm1
 ; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
-; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vpmullw %xmm0, %xmm3, %xmm0
-; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; AVX1-NEXT:    vpmullw %xmm0, %xmm3, %xmm0
-; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpextrb $0, %xmm0, %eax
@@ -2023,24 +2020,25 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX2-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
 ; AVX2-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; AVX2-NEXT:    vpmullw %xmm2, %xmm3, %xmm2
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
+; AVX2-NEXT:    vpand %xmm3, %xmm2, %xmm2
 ; AVX2-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
 ; AVX2-NEXT:    vpmullw %xmm2, %xmm1, %xmm1
 ; AVX2-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
-; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm2
-; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    vpackuswb %xmm3, %xmm2, %xmm2
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,2,3]
-; AVX2-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
-; AVX2-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
-; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm2
-; AVX2-NEXT:    vpackuswb %xmm3, %xmm2, %xmm2
-; AVX2-NEXT:    vpsrld $16, %xmm2, %xmm2
-; AVX2-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
-; AVX2-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
-; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm1
-; AVX2-NEXT:    vpackuswb %xmm3, %xmm1, %xmm1
+; AVX2-NEXT:    vpand %xmm3, %xmm0, %xmm1
+; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; AVX2-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpand %xmm3, %xmm0, %xmm1
+; AVX2-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpsrld $16, %xmm1, %xmm1
+; AVX2-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpand %xmm3, %xmm0, %xmm1
+; AVX2-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
 ; AVX2-NEXT:    vpsrlw $8, %xmm1, %xmm1
 ; AVX2-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
 ; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
@@ -2111,24 +2109,25 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
 ; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; AVX512DQ-NEXT:    vpmullw %xmm2, %xmm3, %xmm2
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
+; AVX512DQ-NEXT:    vpand %xmm3, %xmm2, %xmm2
 ; AVX512DQ-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
 ; AVX512DQ-NEXT:    vpmullw %xmm2, %xmm1, %xmm1
 ; AVX512DQ-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
-; AVX512DQ-NEXT:    vpand %xmm1, %xmm0, %xmm2
-; AVX512DQ-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX512DQ-NEXT:    vpackuswb %xmm3, %xmm2, %xmm2
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,2,3]
-; AVX512DQ-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
-; AVX512DQ-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
-; AVX512DQ-NEXT:    vpand %xmm1, %xmm0, %xmm2
-; AVX512DQ-NEXT:    vpackuswb %xmm3, %xmm2, %xmm2
-; AVX512DQ-NEXT:    vpsrld $16, %xmm2, %xmm2
-; AVX512DQ-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
-; AVX512DQ-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
-; AVX512DQ-NEXT:    vpand %xmm1, %xmm0, %xmm1
-; AVX512DQ-NEXT:    vpackuswb %xmm3, %xmm1, %xmm1
+; AVX512DQ-NEXT:    vpand %xmm3, %xmm0, %xmm1
+; AVX512DQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512DQ-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; AVX512DQ-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX512DQ-NEXT:    vpand %xmm3, %xmm0, %xmm1
+; AVX512DQ-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
+; AVX512DQ-NEXT:    vpsrld $16, %xmm1, %xmm1
+; AVX512DQ-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX512DQ-NEXT:    vpand %xmm3, %xmm0, %xmm1
+; AVX512DQ-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
 ; AVX512DQ-NEXT:    vpsrlw $8, %xmm1, %xmm1
 ; AVX512DQ-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
 ; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
@@ -2143,24 +2142,25 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
 ; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; AVX512DQVL-NEXT:    vpmullw %xmm2, %xmm3, %xmm2
+; AVX512DQVL-NEXT:    vmovdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
+; AVX512DQVL-NEXT:    vpand %xmm3, %xmm2, %xmm2
 ; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
 ; AVX512DQVL-NEXT:    vpmullw %xmm2, %xmm1, %xmm1
 ; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512DQVL-NEXT:    vmovdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
-; AVX512DQVL-NEXT:    vpand %xmm1, %xmm0, %xmm2
-; AVX512DQVL-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX512DQVL-NEXT:    vpackuswb %xmm3, %xmm2, %xmm2
-; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,2,3]
-; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
-; AVX512DQVL-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
-; AVX512DQVL-NEXT:    vpand %xmm1, %xmm0, %xmm2
-; AVX512DQVL-NEXT:    vpackuswb %xmm3, %xmm2, %xmm2
-; AVX512DQVL-NEXT:    vpsrld $16, %xmm2, %xmm2
-; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
-; AVX512DQVL-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
-; AVX512DQVL-NEXT:    vpand %xmm1, %xmm0, %xmm1
-; AVX512DQVL-NEXT:    vpackuswb %xmm3, %xmm1, %xmm1
+; AVX512DQVL-NEXT:    vpand %xmm3, %xmm0, %xmm1
+; AVX512DQVL-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512DQVL-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
+; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX512DQVL-NEXT:    vpand %xmm3, %xmm0, %xmm1
+; AVX512DQVL-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
+; AVX512DQVL-NEXT:    vpsrld $16, %xmm1, %xmm1
+; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX512DQVL-NEXT:    vpand %xmm3, %xmm0, %xmm1
+; AVX512DQVL-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
 ; AVX512DQVL-NEXT:    vpsrlw $8, %xmm1, %xmm1
 ; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
 ; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
@@ -2208,14 +2208,11 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; SSE2-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    pmullw %xmm2, %xmm0
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    pand %xmm1, %xmm2
-; SSE2-NEXT:    packuswb %xmm3, %xmm2
-; SSE2-NEXT:    psrlw $8, %xmm2
-; SSE2-NEXT:    pmullw %xmm0, %xmm2
-; SSE2-NEXT:    pand %xmm1, %xmm2
-; SSE2-NEXT:    packuswb %xmm3, %xmm2
-; SSE2-NEXT:    movd %xmm2, %eax
+; SSE2-NEXT:    pand %xmm0, %xmm1
+; SSE2-NEXT:    packuswb %xmm3, %xmm1
+; SSE2-NEXT:    psrlw $8, %xmm1
+; SSE2-NEXT:    pmullw %xmm0, %xmm1
+; SSE2-NEXT:    movd %xmm1, %eax
 ; SSE2-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE2-NEXT:    retq
 ;
@@ -2229,39 +2226,40 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
 ; SSE41-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
 ; SSE41-NEXT:    pmullw %xmm1, %xmm2
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm6 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSE41-NEXT:    pmullw %xmm3, %xmm6
-; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
-; SSE41-NEXT:    pshufb %xmm3, %xmm6
+; SSE41-NEXT:    pmullw %xmm2, %xmm0
+; SSE41-NEXT:    pmullw %xmm3, %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
+; SSE41-NEXT:    pshufb %xmm2, %xmm1
 ; SSE41-NEXT:    pmullw %xmm4, %xmm5
-; SSE41-NEXT:    pshufb %xmm3, %xmm5
-; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
+; SSE41-NEXT:    pshufb %xmm2, %xmm5
+; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; SSE41-NEXT:    movdqa %xmm0, %xmm3
+; SSE41-NEXT:    pand %xmm2, %xmm3
 ; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm4 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm5 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero,xmm6[4],zero,xmm6[5],zero,xmm6[6],zero,xmm6[7],zero
-; SSE41-NEXT:    pmullw %xmm4, %xmm5
-; SSE41-NEXT:    pshufb %xmm3, %xmm5
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm3 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
-; SSE41-NEXT:    pmullw %xmm2, %xmm3
-; SSE41-NEXT:    pmullw %xmm0, %xmm3
-; SSE41-NEXT:    pand %xmm1, %xmm3
-; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    packuswb %xmm0, %xmm3
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm2 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
-; SSE41-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[2,3,0,1]
-; SSE41-NEXT:    pmullw %xmm2, %xmm3
-; SSE41-NEXT:    pand %xmm1, %xmm3
-; SSE41-NEXT:    packuswb %xmm0, %xmm3
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm2 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
-; SSE41-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,2,3]
-; SSE41-NEXT:    pmullw %xmm2, %xmm3
-; SSE41-NEXT:    pand %xmm1, %xmm3
-; SSE41-NEXT:    packuswb %xmm0, %xmm3
-; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; SSE41-NEXT:    pmullw %xmm4, %xmm1
+; SSE41-NEXT:    pand %xmm2, %xmm1
+; SSE41-NEXT:    packuswb %xmm3, %xmm1
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; SSE41-NEXT:    pmullw %xmm0, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE41-NEXT:    pand %xmm2, %xmm1
+; SSE41-NEXT:    pxor %xmm3, %xmm3
+; SSE41-NEXT:    packuswb %xmm3, %xmm1
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; SSE41-NEXT:    pmullw %xmm0, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE41-NEXT:    pand %xmm2, %xmm1
+; SSE41-NEXT:    packuswb %xmm3, %xmm1
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; SSE41-NEXT:    pmullw %xmm0, %xmm1
+; SSE41-NEXT:    pand %xmm2, %xmm1
+; SSE41-NEXT:    packuswb %xmm3, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
 ; SSE41-NEXT:    psrlw $8, %xmm0
-; SSE41-NEXT:    pmullw %xmm3, %xmm0
+; SSE41-NEXT:    pmullw %xmm1, %xmm0
 ; SSE41-NEXT:    pextrb $0, %xmm0, %eax
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    retq
@@ -2277,31 +2275,21 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX1-NEXT:    vpmullw %xmm3, %xmm5, %xmm3
 ; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; AVX1-NEXT:    vpmullw %xmm3, %xmm5, %xmm3
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
 ; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero
 ; AVX1-NEXT:    vpmullw %xmm2, %xmm4, %xmm2
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
-; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
+; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
 ; AVX1-NEXT:    vpmullw %xmm2, %xmm1, %xmm1
 ; AVX1-NEXT:    vpmullw %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpand %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
 ; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vpmullw %xmm0, %xmm2, %xmm0
-; AVX1-NEXT:    vpand %xmm4, %xmm0, %xmm0
-; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; AVX1-NEXT:    vpmullw %xmm0, %xmm2, %xmm0
-; AVX1-NEXT:    vpand %xmm4, %xmm0, %xmm0
-; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpextrb $0, %xmm0, %eax
@@ -2311,25 +2299,19 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ;
 ; AVX2-LABEL: test_v64i8:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31]
-; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
+; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
 ; AVX2-NEXT:    vpmullw %ymm2, %ymm3, %ymm2
-; AVX2-NEXT:    vmovdqa {{.*#+}} ymm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX2-NEXT:    vpand %ymm3, %ymm2, %ymm2
-; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
-; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm1 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31]
+; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm0 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
 ; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpand %ymm3, %ymm0, %ymm0
-; AVX2-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
-; AVX2-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX2-NEXT:    vpmullw %xmm2, %xmm3, %xmm2
-; AVX2-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
-; AVX2-NEXT:    vpmullw %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm3
+; AVX2-NEXT:    vpmullw %xmm0, %xmm3, %xmm0
+; AVX2-NEXT:    vpmullw %xmm0, %xmm2, %xmm0
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm2
 ; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
 ; AVX2-NEXT:    vpackuswb %xmm3, %xmm2, %xmm2
@@ -2357,35 +2339,34 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm2 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
 ; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm3 = zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[12],zmm0[12],zmm1[13],zmm0[13],zmm1[14],zmm0[14],zmm1[15],zmm0[15],zmm1[24],zmm0[24],zmm1[25],zmm0[25],zmm1[26],zmm0[26],zmm1[27],zmm0[27],zmm1[28],zmm0[28],zmm1[29],zmm0[29],zmm1[30],zmm0[30],zmm1[31],zmm0[31],zmm1[40],zmm0[40],zmm1[41],zmm0[41],zmm1[42],zmm0[42],zmm1[43],zmm0[43],zmm1[44],zmm0[44],zmm1[45],zmm0[45],zmm1[46],zmm0[46],zmm1[47],zmm0[47],zmm1[56],zmm0[56],zmm1[57],zmm0[57],zmm1[58],zmm0[58],zmm1[59],zmm0[59],zmm1[60],zmm0[60],zmm1[61],zmm0[61],zmm1[62],zmm0[62],zmm1[63],zmm0[63]
 ; AVX512BW-NEXT:    vpmullw %zmm3, %zmm2, %zmm2
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512BW-NEXT:    vpandq %zmm3, %zmm2, %zmm2
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; AVX512BW-NEXT:    vpand %ymm3, %ymm2, %ymm4
 ; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
 ; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm1 = zmm1[0],zmm0[0],zmm1[1],zmm0[1],zmm1[2],zmm0[2],zmm1[3],zmm0[3],zmm1[4],zmm0[4],zmm1[5],zmm0[5],zmm1[6],zmm0[6],zmm1[7],zmm0[7],zmm1[16],zmm0[16],zmm1[17],zmm0[17],zmm1[18],zmm0[18],zmm1[19],zmm0[19],zmm1[20],zmm0[20],zmm1[21],zmm0[21],zmm1[22],zmm0[22],zmm1[23],zmm0[23],zmm1[32],zmm0[32],zmm1[33],zmm0[33],zmm1[34],zmm0[34],zmm1[35],zmm0[35],zmm1[36],zmm0[36],zmm1[37],zmm0[37],zmm1[38],zmm0[38],zmm1[39],zmm0[39],zmm1[48],zmm0[48],zmm1[49],zmm0[49],zmm1[50],zmm0[50],zmm1[51],zmm0[51],zmm1[52],zmm0[52],zmm1[53],zmm0[53],zmm1[54],zmm0[54],zmm1[55],zmm0[55]
 ; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpandq %zmm3, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpackuswb %zmm2, %zmm0, %zmm0
-; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX512BW-NEXT:    vpand %ymm3, %ymm0, %ymm1
+; AVX512BW-NEXT:    vpackuswb %ymm4, %ymm1, %ymm1
+; AVX512BW-NEXT:    vextracti128 $1, %ymm1, %xmm1
 ; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
 ; AVX512BW-NEXT:    vpmullw %xmm3, %xmm2, %xmm2
-; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
+; AVX512BW-NEXT:    vpand %xmm3, %xmm2, %xmm2
 ; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
 ; AVX512BW-NEXT:    vpmullw %xmm2, %xmm1, %xmm1
 ; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
-; AVX512BW-NEXT:    vpand %xmm1, %xmm0, %xmm2
-; AVX512BW-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX512BW-NEXT:    vpackuswb %xmm3, %xmm2, %xmm2
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,2,3]
-; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
-; AVX512BW-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpand %xmm1, %xmm0, %xmm2
-; AVX512BW-NEXT:    vpackuswb %xmm3, %xmm2, %xmm2
-; AVX512BW-NEXT:    vpsrld $16, %xmm2, %xmm2
-; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
-; AVX512BW-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpand %xmm1, %xmm0, %xmm1
-; AVX512BW-NEXT:    vpackuswb %xmm3, %xmm1, %xmm1
+; AVX512BW-NEXT:    vpand %xmm3, %xmm0, %xmm1
+; AVX512BW-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512BW-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpand %xmm3, %xmm0, %xmm1
+; AVX512BW-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
+; AVX512BW-NEXT:    vpsrld $16, %xmm1, %xmm1
+; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpand %xmm3, %xmm0, %xmm1
+; AVX512BW-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
 ; AVX512BW-NEXT:    vpsrlw $8, %xmm1, %xmm1
 ; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
 ; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
@@ -2400,35 +2381,34 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} zmm2 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
 ; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} zmm3 = zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[12],zmm0[12],zmm1[13],zmm0[13],zmm1[14],zmm0[14],zmm1[15],zmm0[15],zmm1[24],zmm0[24],zmm1[25],zmm0[25],zmm1[26],zmm0[26],zmm1[27],zmm0[27],zmm1[28],zmm0[28],zmm1[29],zmm0[29],zmm1[30],zmm0[30],zmm1[31],zmm0[31],zmm1[40],zmm0[40],zmm1[41],zmm0[41],zmm1[42],zmm0[42],zmm1[43],zmm0[43],zmm1[44],zmm0[44],zmm1[45],zmm0[45],zmm1[46],zmm0[46],zmm1[47],zmm0[47],zmm1[56],zmm0[56],zmm1[57],zmm0[57],zmm1[58],zmm0[58],zmm1[59],zmm0[59],zmm1[60],zmm0[60],zmm1[61],zmm0[61],zmm1[62],zmm0[62],zmm1[63],zmm0[63]
 ; AVX512BWVL-NEXT:    vpmullw %zmm3, %zmm2, %zmm2
-; AVX512BWVL-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512BWVL-NEXT:    vpandq %zmm3, %zmm2, %zmm2
+; AVX512BWVL-NEXT:    vmovdqa {{.*#+}} ymm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; AVX512BWVL-NEXT:    vpand %ymm3, %ymm2, %ymm4
 ; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
 ; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm1 = zmm1[0],zmm0[0],zmm1[1],zmm0[1],zmm1[2],zmm0[2],zmm1[3],zmm0[3],zmm1[4],zmm0[4],zmm1[5],zmm0[5],zmm1[6],zmm0[6],zmm1[7],zmm0[7],zmm1[16],zmm0[16],zmm1[17],zmm0[17],zmm1[18],zmm0[18],zmm1[19],zmm0[19],zmm1[20],zmm0[20],zmm1[21],zmm0[21],zmm1[22],zmm0[22],zmm1[23],zmm0[23],zmm1[32],zmm0[32],zmm1[33],zmm0[33],zmm1[34],zmm0[34],zmm1[35],zmm0[35],zmm1[36],zmm0[36],zmm1[37],zmm0[37],zmm1[38],zmm0[38],zmm1[39],zmm0[39],zmm1[48],zmm0[48],zmm1[49],zmm0[49],zmm1[50],zmm0[50],zmm1[51],zmm0[51],zmm1[52],zmm0[52],zmm1[53],zmm0[53],zmm1[54],zmm0[54],zmm1[55],zmm0[55]
 ; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
-; AVX512BWVL-NEXT:    vpandq %zmm3, %zmm0, %zmm0
-; AVX512BWVL-NEXT:    vpackuswb %zmm2, %zmm0, %zmm0
-; AVX512BWVL-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX512BWVL-NEXT:    vpand %ymm3, %ymm0, %ymm1
+; AVX512BWVL-NEXT:    vpackuswb %ymm4, %ymm1, %ymm1
+; AVX512BWVL-NEXT:    vextracti128 $1, %ymm1, %xmm1
 ; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
 ; AVX512BWVL-NEXT:    vpmullw %xmm3, %xmm2, %xmm2
-; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; AVX512BWVL-NEXT:    vmovdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
+; AVX512BWVL-NEXT:    vpand %xmm3, %xmm2, %xmm2
 ; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
 ; AVX512BWVL-NEXT:    vpmullw %xmm2, %xmm1, %xmm1
 ; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512BWVL-NEXT:    vmovdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
-; AVX512BWVL-NEXT:    vpand %xmm1, %xmm0, %xmm2
-; AVX512BWVL-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX512BWVL-NEXT:    vpackuswb %xmm3, %xmm2, %xmm2
-; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,2,3]
-; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
-; AVX512BWVL-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
-; AVX512BWVL-NEXT:    vpand %xmm1, %xmm0, %xmm2
-; AVX512BWVL-NEXT:    vpackuswb %xmm3, %xmm2, %xmm2
-; AVX512BWVL-NEXT:    vpsrld $16, %xmm2, %xmm2
-; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
-; AVX512BWVL-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
-; AVX512BWVL-NEXT:    vpand %xmm1, %xmm0, %xmm1
-; AVX512BWVL-NEXT:    vpackuswb %xmm3, %xmm1, %xmm1
+; AVX512BWVL-NEXT:    vpand %xmm3, %xmm0, %xmm1
+; AVX512BWVL-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512BWVL-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
+; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX512BWVL-NEXT:    vpand %xmm3, %xmm0, %xmm1
+; AVX512BWVL-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
+; AVX512BWVL-NEXT:    vpsrld $16, %xmm1, %xmm1
+; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX512BWVL-NEXT:    vpand %xmm3, %xmm0, %xmm1
+; AVX512BWVL-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
 ; AVX512BWVL-NEXT:    vpsrlw $8, %xmm1, %xmm1
 ; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
 ; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
@@ -2439,25 +2419,20 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ;
 ; AVX512DQ-LABEL: test_v64i8:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31]
-; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
 ; AVX512DQ-NEXT:    vpmullw %ymm2, %ymm3, %ymm2
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512DQ-NEXT:    vpand %ymm3, %ymm2, %ymm2
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} ymm1 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31]
+; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} ymm0 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
 ; AVX512DQ-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpand %ymm3, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
-; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX512DQ-NEXT:    vpmullw %xmm2, %xmm3, %xmm2
-; AVX512DQ-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
-; AVX512DQ-NEXT:    vpmullw %xmm2, %xmm1, %xmm1
-; AVX512DQ-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
+; AVX512DQ-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; AVX512DQ-NEXT:    vextracti128 $1, %ymm2, %xmm3
+; AVX512DQ-NEXT:    vpmullw %xmm0, %xmm3, %xmm0
+; AVX512DQ-NEXT:    vpmullw %xmm0, %xmm2, %xmm0
 ; AVX512DQ-NEXT:    vpand %xmm1, %xmm0, %xmm2
 ; AVX512DQ-NEXT:    vpxor %xmm3, %xmm3, %xmm3
 ; AVX512DQ-NEXT:    vpackuswb %xmm3, %xmm2, %xmm2
@@ -2481,38 +2456,38 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ;
 ; AVX512DQVL-LABEL: test_v64i8:
 ; AVX512DQVL:       # %bb.0:
+; AVX512DQVL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31]
 ; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
 ; AVX512DQVL-NEXT:    vpmullw %ymm2, %ymm3, %ymm2
 ; AVX512DQVL-NEXT:    vmovdqa {{.*#+}} ymm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512DQVL-NEXT:    vpand %ymm3, %ymm2, %ymm2
+; AVX512DQVL-NEXT:    vpand %ymm3, %ymm2, %ymm4
 ; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
 ; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
 ; AVX512DQVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
-; AVX512DQVL-NEXT:    vpand %ymm3, %ymm0, %ymm0
-; AVX512DQVL-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
-; AVX512DQVL-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
-; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX512DQVL-NEXT:    vpmullw %xmm2, %xmm3, %xmm2
+; AVX512DQVL-NEXT:    vpand %ymm3, %ymm0, %ymm1
+; AVX512DQVL-NEXT:    vpackuswb %ymm4, %ymm1, %ymm1
+; AVX512DQVL-NEXT:    vextracti128 $1, %ymm1, %xmm1
+; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
+; AVX512DQVL-NEXT:    vpmullw %xmm3, %xmm2, %xmm2
+; AVX512DQVL-NEXT:    vmovdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
+; AVX512DQVL-NEXT:    vpand %xmm3, %xmm2, %xmm2
 ; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
 ; AVX512DQVL-NEXT:    vpmullw %xmm2, %xmm1, %xmm1
-; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512DQVL-NEXT:    vmovdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
-; AVX512DQVL-NEXT:    vpand %xmm1, %xmm0, %xmm2
-; AVX512DQVL-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX512DQVL-NEXT:    vpackuswb %xmm3, %xmm2, %xmm2
-; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,2,3]
-; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
-; AVX512DQVL-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
-; AVX512DQVL-NEXT:    vpand %xmm1, %xmm0, %xmm2
-; AVX512DQVL-NEXT:    vpackuswb %xmm3, %xmm2, %xmm2
-; AVX512DQVL-NEXT:    vpsrld $16, %xmm2, %xmm2
-; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
-; AVX512DQVL-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
-; AVX512DQVL-NEXT:    vpand %xmm1, %xmm0, %xmm1
-; AVX512DQVL-NEXT:    vpackuswb %xmm3, %xmm1, %xmm1
+; AVX512DQVL-NEXT:    vpand %xmm3, %xmm0, %xmm1
+; AVX512DQVL-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512DQVL-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
+; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX512DQVL-NEXT:    vpand %xmm3, %xmm0, %xmm1
+; AVX512DQVL-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
+; AVX512DQVL-NEXT:    vpsrld $16, %xmm1, %xmm1
+; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX512DQVL-NEXT:    vpand %xmm3, %xmm0, %xmm1
+; AVX512DQVL-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
 ; AVX512DQVL-NEXT:    vpsrlw $8, %xmm1, %xmm1
 ; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
 ; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
@@ -2580,85 +2555,92 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; SSE2-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
 ; SSE2-NEXT:    pmullw %xmm2, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm2
-; SSE2-NEXT:    pand %xmm0, %xmm2
-; SSE2-NEXT:    packuswb %xmm3, %xmm2
-; SSE2-NEXT:    psrlw $8, %xmm2
-; SSE2-NEXT:    pmullw %xmm1, %xmm2
-; SSE2-NEXT:    pand %xmm0, %xmm2
-; SSE2-NEXT:    packuswb %xmm3, %xmm2
-; SSE2-NEXT:    movd %xmm2, %eax
+; SSE2-NEXT:    pand %xmm1, %xmm0
+; SSE2-NEXT:    packuswb %xmm3, %xmm0
+; SSE2-NEXT:    psrlw $8, %xmm0
+; SSE2-NEXT:    pmullw %xmm1, %xmm0
+; SSE2-NEXT:    movd %xmm0, %eax
 ; SSE2-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: test_v128i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm8 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero,xmm6[4],zero,xmm6[5],zero,xmm6[6],zero,xmm6[7],zero
-; SSE41-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm0[8],xmm6[9],xmm0[9],xmm6[10],xmm0[10],xmm6[11],xmm0[11],xmm6[12],xmm0[12],xmm6[13],xmm0[13],xmm6[14],xmm0[14],xmm6[15],xmm0[15]
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm9 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
-; SSE41-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
-; SSE41-NEXT:    pmullw %xmm6, %xmm2
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm10 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero
-; SSE41-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm0[8],xmm4[9],xmm0[9],xmm4[10],xmm0[10],xmm4[11],xmm0[11],xmm4[12],xmm0[12],xmm4[13],xmm0[13],xmm4[14],xmm0[14],xmm4[15],xmm0[15]
-; SSE41-NEXT:    pmullw %xmm2, %xmm4
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm11 = xmm7[0],zero,xmm7[1],zero,xmm7[2],zero,xmm7[3],zero,xmm7[4],zero,xmm7[5],zero,xmm7[6],zero,xmm7[7],zero
-; SSE41-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm0[8],xmm7[9],xmm0[9],xmm7[10],xmm0[10],xmm7[11],xmm0[11],xmm7[12],xmm0[12],xmm7[13],xmm0[13],xmm7[14],xmm0[14],xmm7[15],xmm0[15]
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm6 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
-; SSE41-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm0[8],xmm3[9],xmm0[9],xmm3[10],xmm0[10],xmm3[11],xmm0[11],xmm3[12],xmm0[12],xmm3[13],xmm0[13],xmm3[14],xmm0[14],xmm3[15],xmm0[15]
-; SSE41-NEXT:    pmullw %xmm7, %xmm3
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm7 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm10 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
 ; SSE41-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm0[8],xmm5[9],xmm0[9],xmm5[10],xmm0[10],xmm5[11],xmm0[11],xmm5[12],xmm0[12],xmm5[13],xmm0[13],xmm5[14],xmm0[14],xmm5[15],xmm0[15]
-; SSE41-NEXT:    pmullw %xmm3, %xmm5
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm3 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm8 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
 ; SSE41-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
 ; SSE41-NEXT:    pmullw %xmm5, %xmm1
-; SSE41-NEXT:    pmullw %xmm4, %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm9 = [255,255,255,255,255,255,255,255]
+; SSE41-NEXT:    pmullw %xmm10, %xmm8
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm10 = xmm7[0],zero,xmm7[1],zero,xmm7[2],zero,xmm7[3],zero,xmm7[4],zero,xmm7[5],zero,xmm7[6],zero,xmm7[7],zero
+; SSE41-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm0[8],xmm7[9],xmm0[9],xmm7[10],xmm0[10],xmm7[11],xmm0[11],xmm7[12],xmm0[12],xmm7[13],xmm0[13],xmm7[14],xmm0[14],xmm7[15],xmm0[15]
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm5 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
+; SSE41-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm0[8],xmm3[9],xmm0[9],xmm3[10],xmm0[10],xmm3[11],xmm0[11],xmm3[12],xmm0[12],xmm3[13],xmm0[13],xmm3[14],xmm0[14],xmm3[15],xmm0[15]
 ; SSE41-NEXT:    pmullw %xmm7, %xmm3
-; SSE41-NEXT:    movdqa {{.*#+}} xmm5 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
-; SSE41-NEXT:    pshufb %xmm5, %xmm3
-; SSE41-NEXT:    pmullw %xmm11, %xmm6
-; SSE41-NEXT:    pshufb %xmm5, %xmm6
-; SSE41-NEXT:    pmullw %xmm10, %xmm2
-; SSE41-NEXT:    pshufb %xmm5, %xmm2
-; SSE41-NEXT:    pmullw %xmm8, %xmm9
-; SSE41-NEXT:    pshufb %xmm5, %xmm9
-; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm7 = xmm9[0],zero,xmm9[1],zero,xmm9[2],zero,xmm9[3],zero,xmm9[4],zero,xmm9[5],zero,xmm9[6],zero,xmm9[7],zero
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
-; SSE41-NEXT:    pmullw %xmm7, %xmm2
-; SSE41-NEXT:    pshufb %xmm5, %xmm2
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm6 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero,xmm6[4],zero,xmm6[5],zero,xmm6[6],zero,xmm6[7],zero
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
-; SSE41-NEXT:    pmullw %xmm6, %xmm3
-; SSE41-NEXT:    pshufb %xmm5, %xmm3
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
-; SSE41-NEXT:    pmullw %xmm3, %xmm2
-; SSE41-NEXT:    pshufb %xmm5, %xmm2
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
-; SSE41-NEXT:    pmullw %xmm1, %xmm2
-; SSE41-NEXT:    pmullw %xmm0, %xmm2
-; SSE41-NEXT:    pand %xmm4, %xmm2
-; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    packuswb %xmm0, %xmm2
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
-; SSE41-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
-; SSE41-NEXT:    pmullw %xmm1, %xmm2
-; SSE41-NEXT:    pand %xmm4, %xmm2
-; SSE41-NEXT:    packuswb %xmm0, %xmm2
+; SSE41-NEXT:    movdqa %xmm3, %xmm7
+; SSE41-NEXT:    pmullw %xmm1, %xmm3
+; SSE41-NEXT:    pand %xmm9, %xmm1
+; SSE41-NEXT:    pand %xmm9, %xmm8
+; SSE41-NEXT:    packuswb %xmm1, %xmm8
+; SSE41-NEXT:    pand %xmm9, %xmm7
+; SSE41-NEXT:    pmullw %xmm10, %xmm5
+; SSE41-NEXT:    pand %xmm9, %xmm5
+; SSE41-NEXT:    packuswb %xmm7, %xmm5
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm10 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero
+; SSE41-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm0[8],xmm4[9],xmm0[9],xmm4[10],xmm0[10],xmm4[11],xmm0[11],xmm4[12],xmm0[12],xmm4[13],xmm0[13],xmm4[14],xmm0[14],xmm4[15],xmm0[15]
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm7 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSE41-NEXT:    pmullw %xmm4, %xmm0
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm4 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero,xmm6[4],zero,xmm6[5],zero,xmm6[6],zero,xmm6[7],zero
+; SSE41-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm0[8],xmm6[9],xmm0[9],xmm6[10],xmm0[10],xmm6[11],xmm0[11],xmm6[12],xmm0[12],xmm6[13],xmm0[13],xmm6[14],xmm0[14],xmm6[15],xmm0[15]
 ; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
-; SSE41-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,2,3]
-; SSE41-NEXT:    pmullw %xmm1, %xmm2
-; SSE41-NEXT:    pand %xmm4, %xmm2
-; SSE41-NEXT:    packuswb %xmm0, %xmm2
-; SSE41-NEXT:    movdqa %xmm2, %xmm0
-; SSE41-NEXT:    psrlw $8, %xmm0
-; SSE41-NEXT:    pmullw %xmm2, %xmm0
-; SSE41-NEXT:    pextrb $0, %xmm0, %eax
+; SSE41-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
+; SSE41-NEXT:    pmullw %xmm6, %xmm2
+; SSE41-NEXT:    pmullw %xmm2, %xmm3
+; SSE41-NEXT:    pmullw %xmm0, %xmm3
+; SSE41-NEXT:    pand %xmm9, %xmm0
+; SSE41-NEXT:    pmullw %xmm10, %xmm7
+; SSE41-NEXT:    pand %xmm9, %xmm7
+; SSE41-NEXT:    packuswb %xmm0, %xmm7
+; SSE41-NEXT:    pand %xmm9, %xmm2
+; SSE41-NEXT:    pmullw %xmm4, %xmm1
+; SSE41-NEXT:    pand %xmm9, %xmm1
+; SSE41-NEXT:    packuswb %xmm2, %xmm1
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm7[0],zero,xmm7[1],zero,xmm7[2],zero,xmm7[3],zero,xmm7[4],zero,xmm7[5],zero,xmm7[6],zero,xmm7[7],zero
+; SSE41-NEXT:    pmullw %xmm0, %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
+; SSE41-NEXT:    pshufb %xmm0, %xmm1
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm2 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm4 = xmm8[0],zero,xmm8[1],zero,xmm8[2],zero,xmm8[3],zero,xmm8[4],zero,xmm8[5],zero,xmm8[6],zero,xmm8[7],zero
+; SSE41-NEXT:    pmullw %xmm2, %xmm4
+; SSE41-NEXT:    pshufb %xmm0, %xmm4
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    pand %xmm9, %xmm0
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm2 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; SSE41-NEXT:    pmullw %xmm2, %xmm1
+; SSE41-NEXT:    pand %xmm9, %xmm1
+; SSE41-NEXT:    packuswb %xmm0, %xmm1
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; SSE41-NEXT:    pmullw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pand %xmm9, %xmm0
+; SSE41-NEXT:    pxor %xmm2, %xmm2
+; SSE41-NEXT:    packuswb %xmm2, %xmm0
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; SSE41-NEXT:    pmullw %xmm1, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE41-NEXT:    pand %xmm9, %xmm0
+; SSE41-NEXT:    packuswb %xmm2, %xmm0
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; SSE41-NEXT:    pmullw %xmm1, %xmm0
+; SSE41-NEXT:    pand %xmm9, %xmm0
+; SSE41-NEXT:    packuswb %xmm2, %xmm0
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    psrlw $8, %xmm1
+; SSE41-NEXT:    pmullw %xmm0, %xmm1
+; SSE41-NEXT:    pextrb $0, %xmm1, %eax
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    retq
 ;
@@ -2670,56 +2652,42 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
 ; AVX1-NEXT:    vpmullw %xmm4, %xmm5, %xmm8
 ; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm9 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm11
-; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm7 = xmm11[8],xmm0[8],xmm11[9],xmm0[9],xmm11[10],xmm0[10],xmm11[11],xmm0[11],xmm11[12],xmm0[12],xmm11[13],xmm0[13],xmm11[14],xmm0[14],xmm11[15],xmm0[15]
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm4
-; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm4[8],xmm0[8],xmm4[9],xmm0[9],xmm4[10],xmm0[10],xmm4[11],xmm0[11],xmm4[12],xmm0[12],xmm4[13],xmm0[13],xmm4[14],xmm0[14],xmm4[15],xmm0[15]
-; AVX1-NEXT:    vpmullw %xmm7, %xmm5, %xmm10
-; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm7
-; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm7[8],xmm0[8],xmm7[9],xmm0[9],xmm7[10],xmm0[10],xmm7[11],xmm0[11],xmm7[12],xmm0[12],xmm7[13],xmm0[13],xmm7[14],xmm0[14],xmm7[15],xmm0[15]
-; AVX1-NEXT:    vpmullw %xmm10, %xmm5, %xmm10
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm5
-; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm6 = xmm5[8],xmm0[8],xmm5[9],xmm0[9],xmm5[10],xmm0[10],xmm5[11],xmm0[11],xmm5[12],xmm0[12],xmm5[13],xmm0[13],xmm5[14],xmm0[14],xmm5[15],xmm0[15]
-; AVX1-NEXT:    vpmullw %xmm10, %xmm6, %xmm6
-; AVX1-NEXT:    vpmullw %xmm6, %xmm8, %xmm6
-; AVX1-NEXT:    vpmullw %xmm6, %xmm9, %xmm6
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm7 = xmm7[0],zero,xmm7[1],zero,xmm7[2],zero,xmm7[3],zero,xmm7[4],zero,xmm7[5],zero,xmm7[6],zero,xmm7[7],zero
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm5 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
+; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm10
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm7 = xmm10[8],xmm0[8],xmm10[9],xmm0[9],xmm10[10],xmm0[10],xmm10[11],xmm0[11],xmm10[12],xmm0[12],xmm10[13],xmm0[13],xmm10[14],xmm0[14],xmm10[15],xmm0[15]
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm11
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm11[8],xmm0[8],xmm11[9],xmm0[9],xmm11[10],xmm0[10],xmm11[11],xmm0[11],xmm11[12],xmm0[12],xmm11[13],xmm0[13],xmm11[14],xmm0[14],xmm11[15],xmm0[15]
 ; AVX1-NEXT:    vpmullw %xmm7, %xmm5, %xmm5
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm7 = xmm11[0],zero,xmm11[1],zero,xmm11[2],zero,xmm11[3],zero,xmm11[4],zero,xmm11[5],zero,xmm11[6],zero,xmm11[7],zero
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero
-; AVX1-NEXT:    vpmullw %xmm7, %xmm4, %xmm4
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm7
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm6 = xmm7[8],xmm0[8],xmm7[9],xmm0[9],xmm7[10],xmm0[10],xmm7[11],xmm0[11],xmm7[12],xmm0[12],xmm7[13],xmm0[13],xmm7[14],xmm0[14],xmm7[15],xmm0[15]
+; AVX1-NEXT:    vpmullw %xmm5, %xmm6, %xmm5
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm6
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm6[8],xmm0[8],xmm6[9],xmm0[9],xmm6[10],xmm0[10],xmm6[11],xmm0[11],xmm6[12],xmm0[12],xmm6[13],xmm0[13],xmm6[14],xmm0[14],xmm6[15],xmm0[15]
+; AVX1-NEXT:    vpmullw %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vpmullw %xmm4, %xmm8, %xmm4
+; AVX1-NEXT:    vpmullw %xmm4, %xmm9, %xmm4
+; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
+; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX1-NEXT:    vpmullw %xmm3, %xmm1, %xmm1
 ; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
+; AVX1-NEXT:    vpmullw %xmm1, %xmm2, %xmm1
 ; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX1-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm10[0],zero,xmm10[1],zero,xmm10[2],zero,xmm10[3],zero,xmm10[4],zero,xmm10[5],zero,xmm10[6],zero,xmm10[7],zero
+; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm3 = xmm11[0],zero,xmm11[1],zero,xmm11[2],zero,xmm11[3],zero,xmm11[4],zero,xmm11[5],zero,xmm11[6],zero,xmm11[7],zero
+; AVX1-NEXT:    vpmullw %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm3 = xmm7[0],zero,xmm7[1],zero,xmm7[2],zero,xmm7[3],zero,xmm7[4],zero,xmm7[5],zero,xmm7[6],zero,xmm7[7],zero
+; AVX1-NEXT:    vpmullw %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm3 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero,xmm6[4],zero,xmm6[5],zero,xmm6[6],zero,xmm6[7],zero
+; AVX1-NEXT:    vpmullw %xmm2, %xmm3, %xmm2
 ; AVX1-NEXT:    vpmullw %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
-; AVX1-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpand %xmm2, %xmm4, %xmm3
-; AVX1-NEXT:    vpand %xmm2, %xmm5, %xmm4
-; AVX1-NEXT:    vpmullw %xmm3, %xmm4, %xmm3
-; AVX1-NEXT:    vpand %xmm2, %xmm3, %xmm3
-; AVX1-NEXT:    vpmullw %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpmullw %xmm6, %xmm1, %xmm1
+; AVX1-NEXT:    vpmullw %xmm4, %xmm1, %xmm1
+; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
 ; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vpmullw %xmm0, %xmm3, %xmm0
-; AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; AVX1-NEXT:    vpmullw %xmm0, %xmm3, %xmm0
-; AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpextrb $0, %xmm0, %eax
@@ -2729,45 +2697,39 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ;
 ; AVX2-LABEL: test_v128i8:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23]
-; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
+; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm3[8],ymm0[8],ymm3[9],ymm0[9],ymm3[10],ymm0[10],ymm3[11],ymm0[11],ymm3[12],ymm0[12],ymm3[13],ymm0[13],ymm3[14],ymm0[14],ymm3[15],ymm0[15],ymm3[24],ymm0[24],ymm3[25],ymm0[25],ymm3[26],ymm0[26],ymm3[27],ymm0[27],ymm3[28],ymm0[28],ymm3[29],ymm0[29],ymm3[30],ymm0[30],ymm3[31],ymm0[31]
+; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31]
 ; AVX2-NEXT:    vpmullw %ymm4, %ymm5, %ymm4
-; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
+; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11],ymm2[12],ymm0[12],ymm2[13],ymm0[13],ymm2[14],ymm0[14],ymm2[15],ymm0[15],ymm2[24],ymm0[24],ymm2[25],ymm0[25],ymm2[26],ymm0[26],ymm2[27],ymm0[27],ymm2[28],ymm0[28],ymm2[29],ymm0[29],ymm2[30],ymm0[30],ymm2[31],ymm0[31]
 ; AVX2-NEXT:    vpmullw %ymm4, %ymm5, %ymm4
-; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
 ; AVX2-NEXT:    vpmullw %ymm4, %ymm5, %ymm4
-; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm3[8],ymm0[8],ymm3[9],ymm0[9],ymm3[10],ymm0[10],ymm3[11],ymm0[11],ymm3[12],ymm0[12],ymm3[13],ymm0[13],ymm3[14],ymm0[14],ymm3[15],ymm0[15],ymm3[24],ymm0[24],ymm3[25],ymm0[25],ymm3[26],ymm0[26],ymm3[27],ymm0[27],ymm3[28],ymm0[28],ymm3[29],ymm0[29],ymm3[30],ymm0[30],ymm3[31],ymm0[31]
-; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm1 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31]
+; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23]
+; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
 ; AVX2-NEXT:    vpmullw %ymm3, %ymm1, %ymm1
-; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11],ymm2[12],ymm0[12],ymm2[13],ymm0[13],ymm2[14],ymm0[14],ymm2[15],ymm0[15],ymm2[24],ymm0[24],ymm2[25],ymm0[25],ymm2[26],ymm0[26],ymm2[27],ymm0[27],ymm2[28],ymm0[28],ymm2[29],ymm0[29],ymm2[30],ymm0[30],ymm2[31],ymm0[31]
+; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
 ; AVX2-NEXT:    vpmullw %ymm1, %ymm2, %ymm1
-; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm0 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
 ; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vmovdqa {{.*#+}} ymm1 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpand %ymm1, %ymm4, %ymm1
-; AVX2-NEXT:    vpackuswb %ymm0, %ymm1, %ymm0
-; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
-; AVX2-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX2-NEXT:    vpmullw %xmm2, %xmm3, %xmm2
-; AVX2-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
-; AVX2-NEXT:    vpmullw %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2-NEXT:    vextracti128 $1, %ymm4, %xmm1
+; AVX2-NEXT:    vpmullw %xmm1, %xmm4, %xmm1
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; AVX2-NEXT:    vpmullw %xmm1, %xmm3, %xmm1
 ; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
-; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm2
+; AVX2-NEXT:    vpand %xmm2, %xmm0, %xmm1
 ; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    vpackuswb %xmm3, %xmm2, %xmm2
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,2,3]
-; AVX2-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
-; AVX2-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
-; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm2
-; AVX2-NEXT:    vpackuswb %xmm3, %xmm2, %xmm2
-; AVX2-NEXT:    vpsrld $16, %xmm2, %xmm2
-; AVX2-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
-; AVX2-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
-; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm1
+; AVX2-NEXT:    vpackuswb %xmm3, %xmm1, %xmm1
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; AVX2-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpand %xmm2, %xmm0, %xmm1
+; AVX2-NEXT:    vpackuswb %xmm3, %xmm1, %xmm1
+; AVX2-NEXT:    vpsrld $16, %xmm1, %xmm1
+; AVX2-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpand %xmm2, %xmm0, %xmm1
 ; AVX2-NEXT:    vpackuswb %xmm3, %xmm1, %xmm1
 ; AVX2-NEXT:    vpsrlw $8, %xmm1, %xmm1
 ; AVX2-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
@@ -2790,35 +2752,35 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX512BW-NEXT:    vpandq %zmm3, %zmm0, %zmm1
 ; AVX512BW-NEXT:    vpackuswb %zmm4, %zmm1, %zmm1
 ; AVX512BW-NEXT:    vextracti64x4 $1, %zmm1, %ymm1
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm4 = zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[12],zmm0[12],zmm1[13],zmm0[13],zmm1[14],zmm0[14],zmm1[15],zmm0[15],zmm1[24],zmm0[24],zmm1[25],zmm0[25],zmm1[26],zmm0[26],zmm1[27],zmm0[27],zmm1[28],zmm0[28],zmm1[29],zmm0[29],zmm1[30],zmm0[30],zmm1[31],zmm0[31],zmm1[40],zmm0[40],zmm1[41],zmm0[41],zmm1[42],zmm0[42],zmm1[43],zmm0[43],zmm1[44],zmm0[44],zmm1[45],zmm0[45],zmm1[46],zmm0[46],zmm1[47],zmm0[47],zmm1[56],zmm0[56],zmm1[57],zmm0[57],zmm1[58],zmm0[58],zmm1[59],zmm0[59],zmm1[60],zmm0[60],zmm1[61],zmm0[61],zmm1[62],zmm0[62],zmm1[63],zmm0[63]
-; AVX512BW-NEXT:    vpmullw %zmm4, %zmm2, %zmm2
-; AVX512BW-NEXT:    vpandq %zmm3, %zmm2, %zmm2
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm3 = zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[12],zmm0[12],zmm1[13],zmm0[13],zmm1[14],zmm0[14],zmm1[15],zmm0[15],zmm1[24],zmm0[24],zmm1[25],zmm0[25],zmm1[26],zmm0[26],zmm1[27],zmm0[27],zmm1[28],zmm0[28],zmm1[29],zmm0[29],zmm1[30],zmm0[30],zmm1[31],zmm0[31],zmm1[40],zmm0[40],zmm1[41],zmm0[41],zmm1[42],zmm0[42],zmm1[43],zmm0[43],zmm1[44],zmm0[44],zmm1[45],zmm0[45],zmm1[46],zmm0[46],zmm1[47],zmm0[47],zmm1[56],zmm0[56],zmm1[57],zmm0[57],zmm1[58],zmm0[58],zmm1[59],zmm0[59],zmm1[60],zmm0[60],zmm1[61],zmm0[61],zmm1[62],zmm0[62],zmm1[63],zmm0[63]
+; AVX512BW-NEXT:    vpmullw %zmm3, %zmm2, %zmm2
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; AVX512BW-NEXT:    vpand %ymm3, %ymm2, %ymm4
 ; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm1 = zmm1[0],zmm0[0],zmm1[1],zmm0[1],zmm1[2],zmm0[2],zmm1[3],zmm0[3],zmm1[4],zmm0[4],zmm1[5],zmm0[5],zmm1[6],zmm0[6],zmm1[7],zmm0[7],zmm1[16],zmm0[16],zmm1[17],zmm0[17],zmm1[18],zmm0[18],zmm1[19],zmm0[19],zmm1[20],zmm0[20],zmm1[21],zmm0[21],zmm1[22],zmm0[22],zmm1[23],zmm0[23],zmm1[32],zmm0[32],zmm1[33],zmm0[33],zmm1[34],zmm0[34],zmm1[35],zmm0[35],zmm1[36],zmm0[36],zmm1[37],zmm0[37],zmm1[38],zmm0[38],zmm1[39],zmm0[39],zmm1[48],zmm0[48],zmm1[49],zmm0[49],zmm1[50],zmm0[50],zmm1[51],zmm0[51],zmm1[52],zmm0[52],zmm1[53],zmm0[53],zmm1[54],zmm0[54],zmm1[55],zmm0[55]
 ; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpandq %zmm3, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpackuswb %zmm2, %zmm0, %zmm0
-; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX512BW-NEXT:    vpand %ymm3, %ymm0, %ymm1
+; AVX512BW-NEXT:    vpackuswb %ymm4, %ymm1, %ymm1
+; AVX512BW-NEXT:    vextracti128 $1, %ymm1, %xmm1
 ; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
 ; AVX512BW-NEXT:    vpmullw %xmm3, %xmm2, %xmm2
-; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
+; AVX512BW-NEXT:    vpand %xmm3, %xmm2, %xmm2
 ; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
 ; AVX512BW-NEXT:    vpmullw %xmm2, %xmm1, %xmm1
 ; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
-; AVX512BW-NEXT:    vpand %xmm1, %xmm0, %xmm2
-; AVX512BW-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX512BW-NEXT:    vpackuswb %xmm3, %xmm2, %xmm2
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,2,3]
-; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
-; AVX512BW-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpand %xmm1, %xmm0, %xmm2
-; AVX512BW-NEXT:    vpackuswb %xmm3, %xmm2, %xmm2
-; AVX512BW-NEXT:    vpsrld $16, %xmm2, %xmm2
-; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
-; AVX512BW-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpand %xmm1, %xmm0, %xmm1
-; AVX512BW-NEXT:    vpackuswb %xmm3, %xmm1, %xmm1
+; AVX512BW-NEXT:    vpand %xmm3, %xmm0, %xmm1
+; AVX512BW-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512BW-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpand %xmm3, %xmm0, %xmm1
+; AVX512BW-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
+; AVX512BW-NEXT:    vpsrld $16, %xmm1, %xmm1
+; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpand %xmm3, %xmm0, %xmm1
+; AVX512BW-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
 ; AVX512BW-NEXT:    vpsrlw $8, %xmm1, %xmm1
 ; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
 ; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
@@ -2840,35 +2802,35 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX512BWVL-NEXT:    vpandq %zmm3, %zmm0, %zmm1
 ; AVX512BWVL-NEXT:    vpackuswb %zmm4, %zmm1, %zmm1
 ; AVX512BWVL-NEXT:    vextracti64x4 $1, %zmm1, %ymm1
-; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} zmm4 = zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[12],zmm0[12],zmm1[13],zmm0[13],zmm1[14],zmm0[14],zmm1[15],zmm0[15],zmm1[24],zmm0[24],zmm1[25],zmm0[25],zmm1[26],zmm0[26],zmm1[27],zmm0[27],zmm1[28],zmm0[28],zmm1[29],zmm0[29],zmm1[30],zmm0[30],zmm1[31],zmm0[31],zmm1[40],zmm0[40],zmm1[41],zmm0[41],zmm1[42],zmm0[42],zmm1[43],zmm0[43],zmm1[44],zmm0[44],zmm1[45],zmm0[45],zmm1[46],zmm0[46],zmm1[47],zmm0[47],zmm1[56],zmm0[56],zmm1[57],zmm0[57],zmm1[58],zmm0[58],zmm1[59],zmm0[59],zmm1[60],zmm0[60],zmm1[61],zmm0[61],zmm1[62],zmm0[62],zmm1[63],zmm0[63]
-; AVX512BWVL-NEXT:    vpmullw %zmm4, %zmm2, %zmm2
-; AVX512BWVL-NEXT:    vpandq %zmm3, %zmm2, %zmm2
+; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} zmm3 = zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[12],zmm0[12],zmm1[13],zmm0[13],zmm1[14],zmm0[14],zmm1[15],zmm0[15],zmm1[24],zmm0[24],zmm1[25],zmm0[25],zmm1[26],zmm0[26],zmm1[27],zmm0[27],zmm1[28],zmm0[28],zmm1[29],zmm0[29],zmm1[30],zmm0[30],zmm1[31],zmm0[31],zmm1[40],zmm0[40],zmm1[41],zmm0[41],zmm1[42],zmm0[42],zmm1[43],zmm0[43],zmm1[44],zmm0[44],zmm1[45],zmm0[45],zmm1[46],zmm0[46],zmm1[47],zmm0[47],zmm1[56],zmm0[56],zmm1[57],zmm0[57],zmm1[58],zmm0[58],zmm1[59],zmm0[59],zmm1[60],zmm0[60],zmm1[61],zmm0[61],zmm1[62],zmm0[62],zmm1[63],zmm0[63]
+; AVX512BWVL-NEXT:    vpmullw %zmm3, %zmm2, %zmm2
+; AVX512BWVL-NEXT:    vmovdqa {{.*#+}} ymm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; AVX512BWVL-NEXT:    vpand %ymm3, %ymm2, %ymm4
 ; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm1 = zmm1[0],zmm0[0],zmm1[1],zmm0[1],zmm1[2],zmm0[2],zmm1[3],zmm0[3],zmm1[4],zmm0[4],zmm1[5],zmm0[5],zmm1[6],zmm0[6],zmm1[7],zmm0[7],zmm1[16],zmm0[16],zmm1[17],zmm0[17],zmm1[18],zmm0[18],zmm1[19],zmm0[19],zmm1[20],zmm0[20],zmm1[21],zmm0[21],zmm1[22],zmm0[22],zmm1[23],zmm0[23],zmm1[32],zmm0[32],zmm1[33],zmm0[33],zmm1[34],zmm0[34],zmm1[35],zmm0[35],zmm1[36],zmm0[36],zmm1[37],zmm0[37],zmm1[38],zmm0[38],zmm1[39],zmm0[39],zmm1[48],zmm0[48],zmm1[49],zmm0[49],zmm1[50],zmm0[50],zmm1[51],zmm0[51],zmm1[52],zmm0[52],zmm1[53],zmm0[53],zmm1[54],zmm0[54],zmm1[55],zmm0[55]
 ; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
-; AVX512BWVL-NEXT:    vpandq %zmm3, %zmm0, %zmm0
-; AVX512BWVL-NEXT:    vpackuswb %zmm2, %zmm0, %zmm0
-; AVX512BWVL-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX512BWVL-NEXT:    vpand %ymm3, %ymm0, %ymm1
+; AVX512BWVL-NEXT:    vpackuswb %ymm4, %ymm1, %ymm1
+; AVX512BWVL-NEXT:    vextracti128 $1, %ymm1, %xmm1
 ; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
 ; AVX512BWVL-NEXT:    vpmullw %xmm3, %xmm2, %xmm2
-; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; AVX512BWVL-NEXT:    vmovdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
+; AVX512BWVL-NEXT:    vpand %xmm3, %xmm2, %xmm2
 ; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
 ; AVX512BWVL-NEXT:    vpmullw %xmm2, %xmm1, %xmm1
 ; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512BWVL-NEXT:    vmovdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
-; AVX512BWVL-NEXT:    vpand %xmm1, %xmm0, %xmm2
-; AVX512BWVL-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX512BWVL-NEXT:    vpackuswb %xmm3, %xmm2, %xmm2
-; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,2,3]
-; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
-; AVX512BWVL-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
-; AVX512BWVL-NEXT:    vpand %xmm1, %xmm0, %xmm2
-; AVX512BWVL-NEXT:    vpackuswb %xmm3, %xmm2, %xmm2
-; AVX512BWVL-NEXT:    vpsrld $16, %xmm2, %xmm2
-; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
-; AVX512BWVL-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
-; AVX512BWVL-NEXT:    vpand %xmm1, %xmm0, %xmm1
-; AVX512BWVL-NEXT:    vpackuswb %xmm3, %xmm1, %xmm1
+; AVX512BWVL-NEXT:    vpand %xmm3, %xmm0, %xmm1
+; AVX512BWVL-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512BWVL-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
+; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX512BWVL-NEXT:    vpand %xmm3, %xmm0, %xmm1
+; AVX512BWVL-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
+; AVX512BWVL-NEXT:    vpsrld $16, %xmm1, %xmm1
+; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX512BWVL-NEXT:    vpand %xmm3, %xmm0, %xmm1
+; AVX512BWVL-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
 ; AVX512BWVL-NEXT:    vpsrlw $8, %xmm1, %xmm1
 ; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
 ; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
@@ -2879,45 +2841,39 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ;
 ; AVX512DQ-LABEL: test_v128i8:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23]
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
+; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm3[8],ymm0[8],ymm3[9],ymm0[9],ymm3[10],ymm0[10],ymm3[11],ymm0[11],ymm3[12],ymm0[12],ymm3[13],ymm0[13],ymm3[14],ymm0[14],ymm3[15],ymm0[15],ymm3[24],ymm0[24],ymm3[25],ymm0[25],ymm3[26],ymm0[26],ymm3[27],ymm0[27],ymm3[28],ymm0[28],ymm3[29],ymm0[29],ymm3[30],ymm0[30],ymm3[31],ymm0[31]
+; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31]
 ; AVX512DQ-NEXT:    vpmullw %ymm4, %ymm5, %ymm4
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
+; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11],ymm2[12],ymm0[12],ymm2[13],ymm0[13],ymm2[14],ymm0[14],ymm2[15],ymm0[15],ymm2[24],ymm0[24],ymm2[25],ymm0[25],ymm2[26],ymm0[26],ymm2[27],ymm0[27],ymm2[28],ymm0[28],ymm2[29],ymm0[29],ymm2[30],ymm0[30],ymm2[31],ymm0[31]
 ; AVX512DQ-NEXT:    vpmullw %ymm4, %ymm5, %ymm4
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
 ; AVX512DQ-NEXT:    vpmullw %ymm4, %ymm5, %ymm4
-; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm3[8],ymm0[8],ymm3[9],ymm0[9],ymm3[10],ymm0[10],ymm3[11],ymm0[11],ymm3[12],ymm0[12],ymm3[13],ymm0[13],ymm3[14],ymm0[14],ymm3[15],ymm0[15],ymm3[24],ymm0[24],ymm3[25],ymm0[25],ymm3[26],ymm0[26],ymm3[27],ymm0[27],ymm3[28],ymm0[28],ymm3[29],ymm0[29],ymm3[30],ymm0[30],ymm3[31],ymm0[31]
-; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} ymm1 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31]
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23]
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
 ; AVX512DQ-NEXT:    vpmullw %ymm3, %ymm1, %ymm1
-; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11],ymm2[12],ymm0[12],ymm2[13],ymm0[13],ymm2[14],ymm0[14],ymm2[15],ymm0[15],ymm2[24],ymm0[24],ymm2[25],ymm0[25],ymm2[26],ymm0[26],ymm2[27],ymm0[27],ymm2[28],ymm0[28],ymm2[29],ymm0[29],ymm2[30],ymm0[30],ymm2[31],ymm0[31]
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
 ; AVX512DQ-NEXT:    vpmullw %ymm1, %ymm2, %ymm1
-; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} ymm0 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
 ; AVX512DQ-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm1 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512DQ-NEXT:    vpand %ymm1, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpand %ymm1, %ymm4, %ymm1
-; AVX512DQ-NEXT:    vpackuswb %ymm0, %ymm1, %ymm0
-; AVX512DQ-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
-; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX512DQ-NEXT:    vpmullw %xmm2, %xmm3, %xmm2
-; AVX512DQ-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
-; AVX512DQ-NEXT:    vpmullw %xmm2, %xmm1, %xmm1
-; AVX512DQ-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512DQ-NEXT:    vextracti128 $1, %ymm4, %xmm1
+; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm4, %xmm1
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; AVX512DQ-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX512DQ-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm3, %xmm1
 ; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
-; AVX512DQ-NEXT:    vpand %xmm1, %xmm0, %xmm2
+; AVX512DQ-NEXT:    vpand %xmm2, %xmm0, %xmm1
 ; AVX512DQ-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX512DQ-NEXT:    vpackuswb %xmm3, %xmm2, %xmm2
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,2,3]
-; AVX512DQ-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
-; AVX512DQ-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
-; AVX512DQ-NEXT:    vpand %xmm1, %xmm0, %xmm2
-; AVX512DQ-NEXT:    vpackuswb %xmm3, %xmm2, %xmm2
-; AVX512DQ-NEXT:    vpsrld $16, %xmm2, %xmm2
-; AVX512DQ-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
-; AVX512DQ-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
-; AVX512DQ-NEXT:    vpand %xmm1, %xmm0, %xmm1
+; AVX512DQ-NEXT:    vpackuswb %xmm3, %xmm1, %xmm1
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; AVX512DQ-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX512DQ-NEXT:    vpand %xmm2, %xmm0, %xmm1
+; AVX512DQ-NEXT:    vpackuswb %xmm3, %xmm1, %xmm1
+; AVX512DQ-NEXT:    vpsrld $16, %xmm1, %xmm1
+; AVX512DQ-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX512DQ-NEXT:    vpand %xmm2, %xmm0, %xmm1
 ; AVX512DQ-NEXT:    vpackuswb %xmm3, %xmm1, %xmm1
 ; AVX512DQ-NEXT:    vpsrlw $8, %xmm1, %xmm1
 ; AVX512DQ-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
@@ -2944,30 +2900,29 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} ymm0 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
 ; AVX512DQVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vmovdqa {{.*#+}} ymm1 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512DQVL-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX512DQVL-NEXT:    vpand %ymm1, %ymm0, %ymm2
 ; AVX512DQVL-NEXT:    vpand %ymm1, %ymm4, %ymm1
-; AVX512DQVL-NEXT:    vpackuswb %ymm0, %ymm1, %ymm0
-; AVX512DQVL-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512DQVL-NEXT:    vpackuswb %ymm2, %ymm1, %ymm1
+; AVX512DQVL-NEXT:    vextracti128 $1, %ymm1, %xmm1
 ; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
-; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX512DQVL-NEXT:    vpmullw %xmm2, %xmm3, %xmm2
+; AVX512DQVL-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
+; AVX512DQVL-NEXT:    vmovdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; AVX512DQVL-NEXT:    vpand %xmm2, %xmm0, %xmm0
 ; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; AVX512DQVL-NEXT:    vpmullw %xmm2, %xmm1, %xmm1
-; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512DQVL-NEXT:    vmovdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
-; AVX512DQVL-NEXT:    vpand %xmm1, %xmm0, %xmm2
+; AVX512DQVL-NEXT:    vpmullw %xmm0, %xmm1, %xmm0
+; AVX512DQVL-NEXT:    vpmullw %xmm0, %xmm4, %xmm0
+; AVX512DQVL-NEXT:    vpand %xmm2, %xmm0, %xmm1
 ; AVX512DQVL-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX512DQVL-NEXT:    vpackuswb %xmm3, %xmm2, %xmm2
-; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,2,3]
-; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
-; AVX512DQVL-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
-; AVX512DQVL-NEXT:    vpand %xmm1, %xmm0, %xmm2
-; AVX512DQVL-NEXT:    vpackuswb %xmm3, %xmm2, %xmm2
-; AVX512DQVL-NEXT:    vpsrld $16, %xmm2, %xmm2
-; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
-; AVX512DQVL-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
-; AVX512DQVL-NEXT:    vpand %xmm1, %xmm0, %xmm1
+; AVX512DQVL-NEXT:    vpackuswb %xmm3, %xmm1, %xmm1
+; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX512DQVL-NEXT:    vpand %xmm2, %xmm0, %xmm1
+; AVX512DQVL-NEXT:    vpackuswb %xmm3, %xmm1, %xmm1
+; AVX512DQVL-NEXT:    vpsrld $16, %xmm1, %xmm1
+; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX512DQVL-NEXT:    vpand %xmm2, %xmm0, %xmm1
 ; AVX512DQVL-NEXT:    vpackuswb %xmm3, %xmm1, %xmm1
 ; AVX512DQVL-NEXT:    vpsrlw $8, %xmm1, %xmm1
 ; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
diff --git a/llvm/test/CodeGen/X86/vector-reduce-or-bool.ll b/llvm/test/CodeGen/X86/vector-reduce-or-bool.ll
index de5f52ffc..07079ece2 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-or-bool.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-or-bool.ll
@@ -53,7 +53,7 @@ define i1 @trunc_v2i64_v2i1(<2 x i64>) {
 ; AVX512VL-NEXT:    vpsllq $63, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vptestmq %xmm0, %xmm0, %k0
 ; AVX512VL-NEXT:    kmovd %k0, %eax
-; AVX512VL-NEXT:    testb $3, %al
+; AVX512VL-NEXT:    testb %al, %al
 ; AVX512VL-NEXT:    setne %al
 ; AVX512VL-NEXT:    retq
   %a = trunc <2 x i64> %0 to <2 x i1>
@@ -103,7 +103,7 @@ define i1 @trunc_v4i32_v4i1(<4 x i32>) {
 ; AVX512VL-NEXT:    vpslld $31, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vptestmd %xmm0, %xmm0, %k0
 ; AVX512VL-NEXT:    kmovd %k0, %eax
-; AVX512VL-NEXT:    testb $15, %al
+; AVX512VL-NEXT:    testb %al, %al
 ; AVX512VL-NEXT:    setne %al
 ; AVX512VL-NEXT:    retq
   %a = trunc <4 x i32> %0 to <4 x i1>
@@ -112,17 +112,29 @@ define i1 @trunc_v4i32_v4i1(<4 x i32>) {
 }
 
 define i1 @trunc_v8i16_v8i1(<8 x i8>) {
-; SSE-LABEL: trunc_v8i16_v8i1:
-; SSE:       # %bb.0:
-; SSE-NEXT:    psllw $15, %xmm0
-; SSE-NEXT:    packsswb %xmm0, %xmm0
-; SSE-NEXT:    pmovmskb %xmm0, %eax
-; SSE-NEXT:    testb %al, %al
-; SSE-NEXT:    setne %al
-; SSE-NEXT:    retq
+; SSE2-LABEL: trunc_v8i16_v8i1:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    psllw $15, %xmm0
+; SSE2-NEXT:    packsswb %xmm0, %xmm0
+; SSE2-NEXT:    pmovmskb %xmm0, %eax
+; SSE2-NEXT:    testb %al, %al
+; SSE2-NEXT:    setne %al
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: trunc_v8i16_v8i1:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; SSE41-NEXT:    psllw $15, %xmm0
+; SSE41-NEXT:    packsswb %xmm0, %xmm0
+; SSE41-NEXT:    pmovmskb %xmm0, %eax
+; SSE41-NEXT:    testb %al, %al
+; SSE41-NEXT:    setne %al
+; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: trunc_v8i16_v8i1:
 ; AVX:       # %bb.0:
+; AVX-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; AVX-NEXT:    vpsllw $15, %xmm0, %xmm0
 ; AVX-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
 ; AVX-NEXT:    vpmovmskb %xmm0, %eax
@@ -132,9 +144,9 @@ define i1 @trunc_v8i16_v8i1(<8 x i8>) {
 ;
 ; AVX512F-LABEL: trunc_v8i16_v8i1:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpmovsxwq %xmm0, %zmm0
-; AVX512F-NEXT:    vpsllq $63, %zmm0, %zmm0
-; AVX512F-NEXT:    vptestmq %zmm0, %zmm0, %k0
+; AVX512F-NEXT:    vpmovsxbd %xmm0, %zmm0
+; AVX512F-NEXT:    vpslld $31, %zmm0, %zmm0
+; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k0
 ; AVX512F-NEXT:    kmovw %k0, %eax
 ; AVX512F-NEXT:    testb %al, %al
 ; AVX512F-NEXT:    setne %al
@@ -143,8 +155,8 @@ define i1 @trunc_v8i16_v8i1(<8 x i8>) {
 ;
 ; AVX512BW-LABEL: trunc_v8i16_v8i1:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpsllw $15, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpmovw2m %zmm0, %k0
+; AVX512BW-NEXT:    vpsllw $7, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpmovb2m %zmm0, %k0
 ; AVX512BW-NEXT:    kmovd %k0, %eax
 ; AVX512BW-NEXT:    testb %al, %al
 ; AVX512BW-NEXT:    setne %al
@@ -153,8 +165,8 @@ define i1 @trunc_v8i16_v8i1(<8 x i8>) {
 ;
 ; AVX512VL-LABEL: trunc_v8i16_v8i1:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsllw $15, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpmovw2m %xmm0, %k0
+; AVX512VL-NEXT:    vpsllw $7, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpmovb2m %xmm0, %k0
 ; AVX512VL-NEXT:    kmovd %k0, %eax
 ; AVX512VL-NEXT:    testb %al, %al
 ; AVX512VL-NEXT:    setne %al
@@ -239,7 +251,7 @@ define i1 @trunc_v4i64_v4i1(<4 x i64>) {
 ; AVX512VL-NEXT:    vpsllq $63, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vptestmq %ymm0, %ymm0, %k0
 ; AVX512VL-NEXT:    kmovd %k0, %eax
-; AVX512VL-NEXT:    testb $15, %al
+; AVX512VL-NEXT:    testb %al, %al
 ; AVX512VL-NEXT:    setne %al
 ; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq
@@ -340,7 +352,7 @@ define i1 @trunc_v8i32_v8i1(<8 x i32>) {
 define i1 @trunc_v16i16_v16i1(<16 x i16>) {
 ; SSE2-LABEL: trunc_v16i16_v16i1:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
 ; SSE2-NEXT:    pand %xmm2, %xmm1
 ; SSE2-NEXT:    pand %xmm2, %xmm0
 ; SSE2-NEXT:    packuswb %xmm1, %xmm0
@@ -681,7 +693,7 @@ define i1 @trunc_v16i32_v16i1(<16 x i32>) {
 define i1 @trunc_v32i16_v32i1(<32 x i16>) {
 ; SSE2-LABEL: trunc_v32i16_v32i1:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
 ; SSE2-NEXT:    pand %xmm4, %xmm3
 ; SSE2-NEXT:    pand %xmm4, %xmm2
 ; SSE2-NEXT:    packuswb %xmm3, %xmm2
@@ -742,6 +754,7 @@ define i1 @trunc_v32i16_v32i1(<32 x i16>) {
 ;
 ; AVX512F-LABEL: trunc_v32i16_v32i1:
 ; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512F-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpmovsxwd %ymm0, %zmm0
 ; AVX512F-NEXT:    vpslld $31, %zmm0, %zmm0
@@ -782,82 +795,42 @@ define i1 @trunc_v32i16_v32i1(<32 x i16>) {
 }
 
 define i1 @trunc_v64i8_v64i1(<64 x i8>) {
-; SSE2-LABEL: trunc_v64i8_v64i1:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    por %xmm3, %xmm1
-; SSE2-NEXT:    por %xmm2, %xmm1
-; SSE2-NEXT:    por %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
-; SSE2-NEXT:    por %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; SSE2-NEXT:    por %xmm0, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm0
-; SSE2-NEXT:    psrld $16, %xmm0
-; SSE2-NEXT:    por %xmm1, %xmm0
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    psrlw $8, %xmm1
-; SSE2-NEXT:    por %xmm0, %xmm1
-; SSE2-NEXT:    movd %xmm1, %eax
-; SSE2-NEXT:    # kill: def $al killed $al killed $eax
-; SSE2-NEXT:    retq
-;
-; SSE41-LABEL: trunc_v64i8_v64i1:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    por %xmm3, %xmm1
-; SSE41-NEXT:    por %xmm2, %xmm1
-; SSE41-NEXT:    por %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
-; SSE41-NEXT:    por %xmm1, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; SSE41-NEXT:    por %xmm0, %xmm1
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    psrld $16, %xmm0
-; SSE41-NEXT:    por %xmm1, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    psrlw $8, %xmm1
-; SSE41-NEXT:    por %xmm0, %xmm1
-; SSE41-NEXT:    pextrb $0, %xmm1, %eax
-; SSE41-NEXT:    # kill: def $al killed $al killed $eax
-; SSE41-NEXT:    retq
+; SSE-LABEL: trunc_v64i8_v64i1:
+; SSE:       # %bb.0:
+; SSE-NEXT:    por %xmm3, %xmm1
+; SSE-NEXT:    por %xmm2, %xmm1
+; SSE-NEXT:    por %xmm0, %xmm1
+; SSE-NEXT:    psllw $7, %xmm1
+; SSE-NEXT:    pmovmskb %xmm1, %eax
+; SSE-NEXT:    testw %ax, %ax
+; SSE-NEXT:    setne %al
+; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: trunc_v64i8_v64i1:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
-; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpextrb $0, %xmm0, %eax
-; AVX1-NEXT:    # kill: def $al killed $al killed $eax
+; AVX1-NEXT:    vpsllw $7, %xmm0, %xmm0
+; AVX1-NEXT:    vpmovmskb %xmm0, %eax
+; AVX1-NEXT:    testw %ax, %ax
+; AVX1-NEXT:    setne %al
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: trunc_v64i8_v64i1:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
-; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpextrb $0, %xmm0, %eax
-; AVX2-NEXT:    # kill: def $al killed $al killed $eax
+; AVX2-NEXT:    vpsllw $7, %ymm0, %ymm0
+; AVX2-NEXT:    vpmovmskb %ymm0, %eax
+; AVX2-NEXT:    testl %eax, %eax
+; AVX2-NEXT:    setne %al
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: trunc_v64i8_v64i1:
 ; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm2
 ; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm3
 ; AVX512F-NEXT:    vpor %xmm2, %xmm3, %xmm2
@@ -883,20 +856,8 @@ define i1 @trunc_v64i8_v64i1(<64 x i8>) {
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    vpsllw $7, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpmovb2m %zmm0, %k0
-; AVX512BW-NEXT:    kshiftrq $32, %k0, %k1
-; AVX512BW-NEXT:    korq %k1, %k0, %k0
-; AVX512BW-NEXT:    kshiftrq $16, %k0, %k1
-; AVX512BW-NEXT:    korq %k1, %k0, %k0
-; AVX512BW-NEXT:    kshiftrq $8, %k0, %k1
-; AVX512BW-NEXT:    korq %k1, %k0, %k0
-; AVX512BW-NEXT:    kshiftrq $4, %k0, %k1
-; AVX512BW-NEXT:    korq %k1, %k0, %k0
-; AVX512BW-NEXT:    kshiftrq $2, %k0, %k1
-; AVX512BW-NEXT:    korq %k1, %k0, %k0
-; AVX512BW-NEXT:    kshiftrq $1, %k0, %k1
-; AVX512BW-NEXT:    korq %k1, %k0, %k0
-; AVX512BW-NEXT:    kmovd %k0, %eax
-; AVX512BW-NEXT:    # kill: def $al killed $al killed $eax
+; AVX512BW-NEXT:    kortestq %k0, %k0
+; AVX512BW-NEXT:    setne %al
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
@@ -904,20 +865,8 @@ define i1 @trunc_v64i8_v64i1(<64 x i8>) {
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vpsllw $7, %zmm0, %zmm0
 ; AVX512VL-NEXT:    vpmovb2m %zmm0, %k0
-; AVX512VL-NEXT:    kshiftrq $32, %k0, %k1
-; AVX512VL-NEXT:    korq %k1, %k0, %k0
-; AVX512VL-NEXT:    kshiftrq $16, %k0, %k1
-; AVX512VL-NEXT:    korq %k1, %k0, %k0
-; AVX512VL-NEXT:    kshiftrq $8, %k0, %k1
-; AVX512VL-NEXT:    korq %k1, %k0, %k0
-; AVX512VL-NEXT:    kshiftrq $4, %k0, %k1
-; AVX512VL-NEXT:    korq %k1, %k0, %k0
-; AVX512VL-NEXT:    kshiftrq $2, %k0, %k1
-; AVX512VL-NEXT:    korq %k1, %k0, %k0
-; AVX512VL-NEXT:    kshiftrq $1, %k0, %k1
-; AVX512VL-NEXT:    korq %k1, %k0, %k0
-; AVX512VL-NEXT:    kmovd %k0, %eax
-; AVX512VL-NEXT:    # kill: def $al killed $al killed $eax
+; AVX512VL-NEXT:    kortestq %k0, %k0
+; AVX512VL-NEXT:    setne %al
 ; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq
   %a = trunc <64 x i8> %0 to <64 x i1>
@@ -983,7 +932,7 @@ define i1 @icmp_v2i64_v2i1(<2 x i64>) {
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vptestnmq %xmm0, %xmm0, %k0
 ; AVX512VL-NEXT:    kmovd %k0, %eax
-; AVX512VL-NEXT:    testb $3, %al
+; AVX512VL-NEXT:    testb %al, %al
 ; AVX512VL-NEXT:    setne %al
 ; AVX512VL-NEXT:    retq
   %a = icmp eq <2 x i64> %0, zeroinitializer
@@ -1034,7 +983,7 @@ define i1 @icmp_v4i32_v4i1(<4 x i32>) {
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vptestnmd %xmm0, %xmm0, %k0
 ; AVX512VL-NEXT:    kmovd %k0, %eax
-; AVX512VL-NEXT:    testb $15, %al
+; AVX512VL-NEXT:    testb %al, %al
 ; AVX512VL-NEXT:    setne %al
 ; AVX512VL-NEXT:    retq
   %a = icmp eq <4 x i32> %0, zeroinitializer
@@ -1043,22 +992,33 @@ define i1 @icmp_v4i32_v4i1(<4 x i32>) {
 }
 
 define i1 @icmp_v8i16_v8i1(<8 x i8>) {
-; SSE-LABEL: icmp_v8i16_v8i1:
-; SSE:       # %bb.0:
-; SSE-NEXT:    pand {{.*}}(%rip), %xmm0
-; SSE-NEXT:    pxor %xmm1, %xmm1
-; SSE-NEXT:    pcmpeqw %xmm0, %xmm1
-; SSE-NEXT:    packsswb %xmm0, %xmm1
-; SSE-NEXT:    pmovmskb %xmm1, %eax
-; SSE-NEXT:    testb %al, %al
-; SSE-NEXT:    setne %al
-; SSE-NEXT:    retq
+; SSE2-LABEL: icmp_v8i16_v8i1:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    pcmpeqb %xmm0, %xmm1
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSE2-NEXT:    packsswb %xmm0, %xmm0
+; SSE2-NEXT:    pmovmskb %xmm0, %eax
+; SSE2-NEXT:    testb %al, %al
+; SSE2-NEXT:    setne %al
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: icmp_v8i16_v8i1:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pxor %xmm1, %xmm1
+; SSE41-NEXT:    pcmpeqb %xmm0, %xmm1
+; SSE41-NEXT:    pmovsxbw %xmm1, %xmm0
+; SSE41-NEXT:    packsswb %xmm0, %xmm0
+; SSE41-NEXT:    pmovmskb %xmm0, %eax
+; SSE41-NEXT:    testb %al, %al
+; SSE41-NEXT:    setne %al
+; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: icmp_v8i16_v8i1:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
 ; AVX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX-NEXT:    vpcmpeqw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpcmpeqb %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpmovsxbw %xmm0, %xmm0
 ; AVX-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
 ; AVX-NEXT:    vpmovmskb %xmm0, %eax
 ; AVX-NEXT:    testb %al, %al
@@ -1067,11 +1027,10 @@ define i1 @icmp_v8i16_v8i1(<8 x i8>) {
 ;
 ; AVX512F-LABEL: icmp_v8i16_v8i1:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT:    vpcmpeqw %xmm1, %xmm0, %xmm0
-; AVX512F-NEXT:    vpmovsxwq %xmm0, %zmm0
-; AVX512F-NEXT:    vptestmq %zmm0, %zmm0, %k0
+; AVX512F-NEXT:    vpcmpeqb %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    vpmovsxbd %xmm0, %zmm0
+; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k0
 ; AVX512F-NEXT:    kmovw %k0, %eax
 ; AVX512F-NEXT:    testb %al, %al
 ; AVX512F-NEXT:    setne %al
@@ -1081,8 +1040,7 @@ define i1 @icmp_v8i16_v8i1(<8 x i8>) {
 ; AVX512BW-LABEL: icmp_v8i16_v8i1:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm1 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
-; AVX512BW-NEXT:    vptestnmw %zmm1, %zmm0, %k0
+; AVX512BW-NEXT:    vptestnmb %zmm0, %zmm0, %k0
 ; AVX512BW-NEXT:    kmovd %k0, %eax
 ; AVX512BW-NEXT:    testb %al, %al
 ; AVX512BW-NEXT:    setne %al
@@ -1091,7 +1049,7 @@ define i1 @icmp_v8i16_v8i1(<8 x i8>) {
 ;
 ; AVX512VL-LABEL: icmp_v8i16_v8i1:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vptestnmw {{.*}}(%rip), %xmm0, %k0
+; AVX512VL-NEXT:    vptestnmb %xmm0, %xmm0, %k0
 ; AVX512VL-NEXT:    kmovd %k0, %eax
 ; AVX512VL-NEXT:    testb %al, %al
 ; AVX512VL-NEXT:    setne %al
@@ -1223,7 +1181,7 @@ define i1 @icmp_v4i64_v4i1(<4 x i64>) {
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vptestnmq %ymm0, %ymm0, %k0
 ; AVX512VL-NEXT:    kmovd %k0, %eax
-; AVX512VL-NEXT:    testb $15, %al
+; AVX512VL-NEXT:    testb %al, %al
 ; AVX512VL-NEXT:    setne %al
 ; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq
@@ -1661,6 +1619,7 @@ define i1 @icmp_v32i16_v32i1(<32 x i16>) {
 ;
 ; AVX512F-LABEL: icmp_v32i16_v32i1:
 ; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; AVX512F-NEXT:    vpcmpeqw %ymm2, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpcmpeqw %ymm2, %ymm0, %ymm0
@@ -1711,9 +1670,10 @@ define i1 @icmp_v64i8_v64i1(<64 x i8>) {
 ; SSE-NEXT:    por %xmm3, %xmm1
 ; SSE-NEXT:    por %xmm2, %xmm1
 ; SSE-NEXT:    por %xmm0, %xmm1
+; SSE-NEXT:    psllw $7, %xmm1
 ; SSE-NEXT:    pmovmskb %xmm1, %eax
-; SSE-NEXT:    negl %eax
-; SSE-NEXT:    sbbb %al, %al
+; SSE-NEXT:    testw %ax, %ax
+; SSE-NEXT:    setne %al
 ; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: icmp_v64i8_v64i1:
@@ -1728,9 +1688,10 @@ define i1 @icmp_v64i8_v64i1(<64 x i8>) {
 ; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpor %xmm0, %xmm3, %xmm0
 ; AVX1-NEXT:    vpor %xmm0, %xmm4, %xmm0
+; AVX1-NEXT:    vpsllw $7, %xmm0, %xmm0
 ; AVX1-NEXT:    vpmovmskb %xmm0, %eax
-; AVX1-NEXT:    negl %eax
-; AVX1-NEXT:    sbbb %al, %al
+; AVX1-NEXT:    testw %ax, %ax
+; AVX1-NEXT:    setne %al
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
@@ -1741,13 +1702,14 @@ define i1 @icmp_v64i8_v64i1(<64 x i8>) {
 ; AVX2-NEXT:    vpcmpeqb %ymm2, %ymm0, %ymm0
 ; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vpmovmskb %ymm0, %eax
-; AVX2-NEXT:    negl %eax
-; AVX2-NEXT:    sbbb %al, %al
+; AVX2-NEXT:    testl %eax, %eax
+; AVX2-NEXT:    setne %al
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: icmp_v64i8_v64i1:
 ; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; AVX512F-NEXT:    vpcmpeqb %ymm2, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpcmpeqb %ymm2, %ymm0, %ymm0
@@ -1774,40 +1736,16 @@ define i1 @icmp_v64i8_v64i1(<64 x i8>) {
 ; AVX512BW-LABEL: icmp_v64i8_v64i1:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    vptestnmb %zmm0, %zmm0, %k0
-; AVX512BW-NEXT:    kshiftrq $32, %k0, %k1
-; AVX512BW-NEXT:    korq %k1, %k0, %k0
-; AVX512BW-NEXT:    kshiftrq $16, %k0, %k1
-; AVX512BW-NEXT:    korq %k1, %k0, %k0
-; AVX512BW-NEXT:    kshiftrq $8, %k0, %k1
-; AVX512BW-NEXT:    korq %k1, %k0, %k0
-; AVX512BW-NEXT:    kshiftrq $4, %k0, %k1
-; AVX512BW-NEXT:    korq %k1, %k0, %k0
-; AVX512BW-NEXT:    kshiftrq $2, %k0, %k1
-; AVX512BW-NEXT:    korq %k1, %k0, %k0
-; AVX512BW-NEXT:    kshiftrq $1, %k0, %k1
-; AVX512BW-NEXT:    korq %k1, %k0, %k0
-; AVX512BW-NEXT:    kmovd %k0, %eax
-; AVX512BW-NEXT:    # kill: def $al killed $al killed $eax
+; AVX512BW-NEXT:    kortestq %k0, %k0
+; AVX512BW-NEXT:    setne %al
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512VL-LABEL: icmp_v64i8_v64i1:
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vptestnmb %zmm0, %zmm0, %k0
-; AVX512VL-NEXT:    kshiftrq $32, %k0, %k1
-; AVX512VL-NEXT:    korq %k1, %k0, %k0
-; AVX512VL-NEXT:    kshiftrq $16, %k0, %k1
-; AVX512VL-NEXT:    korq %k1, %k0, %k0
-; AVX512VL-NEXT:    kshiftrq $8, %k0, %k1
-; AVX512VL-NEXT:    korq %k1, %k0, %k0
-; AVX512VL-NEXT:    kshiftrq $4, %k0, %k1
-; AVX512VL-NEXT:    korq %k1, %k0, %k0
-; AVX512VL-NEXT:    kshiftrq $2, %k0, %k1
-; AVX512VL-NEXT:    korq %k1, %k0, %k0
-; AVX512VL-NEXT:    kshiftrq $1, %k0, %k1
-; AVX512VL-NEXT:    korq %k1, %k0, %k0
-; AVX512VL-NEXT:    kmovd %k0, %eax
-; AVX512VL-NEXT:    # kill: def $al killed $al killed $eax
+; AVX512VL-NEXT:    kortestq %k0, %k0
+; AVX512VL-NEXT:    setne %al
 ; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq
   %a = icmp eq <64 x i8> %0, zeroinitializer
diff --git a/llvm/test/CodeGen/X86/vector-reduce-or.ll b/llvm/test/CodeGen/X86/vector-reduce-or.ll
index ddfe176d5..0ad1283a5 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-or.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-or.ll
@@ -182,14 +182,14 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 define i32 @test_v2i32(<2 x i32> %a0) {
 ; SSE-LABEL: test_v2i32:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE-NEXT:    por %xmm0, %xmm1
 ; SSE-NEXT:    movd %xmm1, %eax
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: test_v2i32:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovd %xmm0, %eax
 ; AVX-NEXT:    retq
@@ -397,7 +397,8 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 define i16 @test_v2i16(<2 x i16> %a0) {
 ; SSE-LABEL: test_v2i16:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE-NEXT:    movdqa %xmm0, %xmm1
+; SSE-NEXT:    psrld $16, %xmm1
 ; SSE-NEXT:    por %xmm0, %xmm1
 ; SSE-NEXT:    movd %xmm1, %eax
 ; SSE-NEXT:    # kill: def $ax killed $ax killed $eax
@@ -405,7 +406,7 @@ define i16 @test_v2i16(<2 x i16> %a0) {
 ;
 ; AVX-LABEL: test_v2i16:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovd %xmm0, %eax
 ; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
@@ -417,9 +418,10 @@ define i16 @test_v2i16(<2 x i16> %a0) {
 define i16 @test_v4i16(<4 x i16> %a0) {
 ; SSE-LABEL: test_v4i16:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE-NEXT:    por %xmm0, %xmm1
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE-NEXT:    movdqa %xmm1, %xmm0
+; SSE-NEXT:    psrld $16, %xmm0
 ; SSE-NEXT:    por %xmm1, %xmm0
 ; SSE-NEXT:    movd %xmm0, %eax
 ; SSE-NEXT:    # kill: def $ax killed $ax killed $eax
@@ -427,10 +429,10 @@ define i16 @test_v4i16(<4 x i16> %a0) {
 ;
 ; AVX-LABEL: test_v4i16:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpsrld $16, %xmm0, %xmm1
+; AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovd %xmm0, %eax
 ; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX-NEXT:    retq
@@ -684,7 +686,8 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 define i8 @test_v2i8(<2 x i8> %a0) {
 ; SSE2-LABEL: test_v2i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psrlw $8, %xmm1
 ; SSE2-NEXT:    por %xmm0, %xmm1
 ; SSE2-NEXT:    movd %xmm1, %eax
 ; SSE2-NEXT:    # kill: def $al killed $al killed $eax
@@ -692,7 +695,8 @@ define i8 @test_v2i8(<2 x i8> %a0) {
 ;
 ; SSE41-LABEL: test_v2i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    psrlw $8, %xmm1
 ; SSE41-NEXT:    por %xmm0, %xmm1
 ; SSE41-NEXT:    pextrb $0, %xmm1, %eax
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
@@ -700,7 +704,7 @@ define i8 @test_v2i8(<2 x i8> %a0) {
 ;
 ; AVX-LABEL: test_v2i8:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpextrb $0, %xmm0, %eax
 ; AVX-NEXT:    # kill: def $al killed $al killed $eax
@@ -712,9 +716,11 @@ define i8 @test_v2i8(<2 x i8> %a0) {
 define i8 @test_v4i8(<4 x i8> %a0) {
 ; SSE2-LABEL: test_v4i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psrld $16, %xmm1
 ; SSE2-NEXT:    por %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    psrlw $8, %xmm0
 ; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    movd %xmm0, %eax
 ; SSE2-NEXT:    # kill: def $al killed $al killed $eax
@@ -722,9 +728,11 @@ define i8 @test_v4i8(<4 x i8> %a0) {
 ;
 ; SSE41-LABEL: test_v4i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    psrld $16, %xmm1
 ; SSE41-NEXT:    por %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    psrlw $8, %xmm0
 ; SSE41-NEXT:    por %xmm1, %xmm0
 ; SSE41-NEXT:    pextrb $0, %xmm0, %eax
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
@@ -732,9 +740,9 @@ define i8 @test_v4i8(<4 x i8> %a0) {
 ;
 ; AVX-LABEL: test_v4i8:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpextrb $0, %xmm0, %eax
 ; AVX-NEXT:    # kill: def $al killed $al killed $eax
@@ -746,12 +754,13 @@ define i8 @test_v4i8(<4 x i8> %a0) {
 define i8 @test_v8i8(<8 x i8> %a0) {
 ; SSE2-LABEL: test_v8i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE2-NEXT:    por %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    psrld $16, %xmm0
 ; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    psrld $16, %xmm1
+; SSE2-NEXT:    psrlw $8, %xmm1
 ; SSE2-NEXT:    por %xmm0, %xmm1
 ; SSE2-NEXT:    movd %xmm1, %eax
 ; SSE2-NEXT:    # kill: def $al killed $al killed $eax
@@ -759,12 +768,13 @@ define i8 @test_v8i8(<8 x i8> %a0) {
 ;
 ; SSE41-LABEL: test_v8i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE41-NEXT:    por %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    psrld $16, %xmm0
 ; SSE41-NEXT:    por %xmm1, %xmm0
 ; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    psrld $16, %xmm1
+; SSE41-NEXT:    psrlw $8, %xmm1
 ; SSE41-NEXT:    por %xmm0, %xmm1
 ; SSE41-NEXT:    pextrb $0, %xmm1, %eax
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
@@ -772,12 +782,12 @@ define i8 @test_v8i8(<8 x i8> %a0) {
 ;
 ; AVX-LABEL: test_v8i8:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpextrb $0, %xmm0, %eax
 ; AVX-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/vector-reduce-smax.ll b/llvm/test/CodeGen/X86/vector-reduce-smax.ll
index 057174249..8a184cd78 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-smax.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-smax.ll
@@ -41,12 +41,11 @@ define i64 @test_v2i64(<2 x i64> %a0) {
 ; SSE41-NEXT:    pxor %xmm3, %xmm4
 ; SSE41-NEXT:    pxor %xmm2, %xmm3
 ; SSE41-NEXT:    movdqa %xmm4, %xmm0
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm4, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm3
-; SSE41-NEXT:    por %xmm3, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm3, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pand %xmm3, %xmm0
+; SSE41-NEXT:    por %xmm4, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
 ; SSE41-NEXT:    movq %xmm2, %rax
 ; SSE41-NEXT:    retq
@@ -124,22 +123,20 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; SSE41-NEXT:    movdqa %xmm0, %xmm5
 ; SSE41-NEXT:    pxor %xmm3, %xmm5
 ; SSE41-NEXT:    movdqa %xmm5, %xmm0
-; SSE41-NEXT:    pcmpgtd %xmm4, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm4, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm5[1,1,3,3]
-; SSE41-NEXT:    pand %xmm6, %xmm4
-; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm4, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm4, %xmm5
+; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm5[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm5, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    pxor %xmm3, %xmm0
+; SSE41-NEXT:    movdqa %xmm1, %xmm4
+; SSE41-NEXT:    pxor %xmm3, %xmm4
 ; SSE41-NEXT:    pxor %xmm2, %xmm3
-; SSE41-NEXT:    movdqa %xmm0, %xmm4
+; SSE41-NEXT:    movdqa %xmm4, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm3, %xmm5
 ; SSE41-NEXT:    pcmpgtd %xmm3, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2]
 ; SSE41-NEXT:    pand %xmm5, %xmm0
 ; SSE41-NEXT:    por %xmm4, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
@@ -270,46 +267,42 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; SSE41-NEXT:    movdqa %xmm0, %xmm7
 ; SSE41-NEXT:    pxor %xmm5, %xmm7
 ; SSE41-NEXT:    movdqa %xmm7, %xmm0
-; SSE41-NEXT:    pcmpgtd %xmm6, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm8 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm6, %xmm7
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm7[1,1,3,3]
-; SSE41-NEXT:    pand %xmm8, %xmm6
-; SSE41-NEXT:    por %xmm6, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm6, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm6, %xmm7
+; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm7[0,0,2,2]
+; SSE41-NEXT:    pand %xmm6, %xmm0
+; SSE41-NEXT:    por %xmm7, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm2
 ; SSE41-NEXT:    movdqa %xmm3, %xmm0
 ; SSE41-NEXT:    pxor %xmm5, %xmm0
 ; SSE41-NEXT:    movdqa %xmm1, %xmm4
 ; SSE41-NEXT:    pxor %xmm5, %xmm4
 ; SSE41-NEXT:    movdqa %xmm4, %xmm6
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3]
-; SSE41-NEXT:    pand %xmm7, %xmm0
-; SSE41-NEXT:    por %xmm6, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pand %xmm6, %xmm0
+; SSE41-NEXT:    por %xmm4, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm3
 ; SSE41-NEXT:    movapd %xmm3, %xmm0
 ; SSE41-NEXT:    xorpd %xmm5, %xmm0
 ; SSE41-NEXT:    movapd %xmm2, %xmm1
 ; SSE41-NEXT:    xorpd %xmm5, %xmm1
 ; SSE41-NEXT:    movapd %xmm1, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE41-NEXT:    pand %xmm6, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm1, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm3
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[2,3,0,1]
-; SSE41-NEXT:    movdqa %xmm3, %xmm0
-; SSE41-NEXT:    pxor %xmm5, %xmm0
+; SSE41-NEXT:    movdqa %xmm3, %xmm2
+; SSE41-NEXT:    pxor %xmm5, %xmm2
 ; SSE41-NEXT:    pxor %xmm1, %xmm5
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm5, %xmm4
 ; SSE41-NEXT:    pcmpgtd %xmm5, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[1,1,3,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,0,2,2]
 ; SSE41-NEXT:    pand %xmm4, %xmm0
 ; SSE41-NEXT:    por %xmm2, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm1
@@ -505,99 +498,91 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movdqa %xmm0, %xmm8
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm9 = [2147483648,2147483648]
-; SSE41-NEXT:    movdqa %xmm5, %xmm10
-; SSE41-NEXT:    pxor %xmm9, %xmm10
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    movdqa %xmm5, %xmm0
 ; SSE41-NEXT:    pxor %xmm9, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm11
-; SSE41-NEXT:    pcmpgtd %xmm10, %xmm11
-; SSE41-NEXT:    pshufd {{.*#+}} xmm12 = xmm11[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm12, %xmm0
-; SSE41-NEXT:    por %xmm11, %xmm0
+; SSE41-NEXT:    movdqa %xmm1, %xmm10
+; SSE41-NEXT:    pxor %xmm9, %xmm10
+; SSE41-NEXT:    movdqa %xmm10, %xmm11
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm11
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm10
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm10[0,0,2,2]
+; SSE41-NEXT:    pand %xmm11, %xmm0
+; SSE41-NEXT:    por %xmm10, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm5
 ; SSE41-NEXT:    movdqa %xmm7, %xmm0
 ; SSE41-NEXT:    pxor %xmm9, %xmm0
 ; SSE41-NEXT:    movdqa %xmm3, %xmm1
 ; SSE41-NEXT:    pxor %xmm9, %xmm1
 ; SSE41-NEXT:    movdqa %xmm1, %xmm10
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm10
-; SSE41-NEXT:    pshufd {{.*#+}} xmm11 = xmm10[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE41-NEXT:    pand %xmm11, %xmm0
-; SSE41-NEXT:    por %xmm10, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm10
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2]
+; SSE41-NEXT:    pand %xmm10, %xmm0
+; SSE41-NEXT:    por %xmm1, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm7
 ; SSE41-NEXT:    movdqa %xmm4, %xmm1
 ; SSE41-NEXT:    pxor %xmm9, %xmm1
 ; SSE41-NEXT:    movdqa %xmm8, %xmm3
 ; SSE41-NEXT:    pxor %xmm9, %xmm3
 ; SSE41-NEXT:    movdqa %xmm3, %xmm0
-; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm10 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm1, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pand %xmm10, %xmm1
-; SSE41-NEXT:    por %xmm1, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm1, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pand %xmm1, %xmm0
+; SSE41-NEXT:    por %xmm3, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm8, %xmm4
 ; SSE41-NEXT:    movdqa %xmm6, %xmm0
 ; SSE41-NEXT:    pxor %xmm9, %xmm0
 ; SSE41-NEXT:    movdqa %xmm2, %xmm1
 ; SSE41-NEXT:    pxor %xmm9, %xmm1
 ; SSE41-NEXT:    movdqa %xmm1, %xmm3
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm8 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE41-NEXT:    pand %xmm8, %xmm0
-; SSE41-NEXT:    por %xmm3, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm3
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2]
+; SSE41-NEXT:    pand %xmm3, %xmm0
+; SSE41-NEXT:    por %xmm1, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm6
 ; SSE41-NEXT:    movapd %xmm6, %xmm0
 ; SSE41-NEXT:    xorpd %xmm9, %xmm0
 ; SSE41-NEXT:    movapd %xmm4, %xmm1
 ; SSE41-NEXT:    xorpd %xmm9, %xmm1
 ; SSE41-NEXT:    movapd %xmm1, %xmm2
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE41-NEXT:    pand %xmm3, %xmm0
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm2
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2]
+; SSE41-NEXT:    pand %xmm2, %xmm0
+; SSE41-NEXT:    por %xmm1, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm6
 ; SSE41-NEXT:    movapd %xmm7, %xmm0
 ; SSE41-NEXT:    xorpd %xmm9, %xmm0
 ; SSE41-NEXT:    movapd %xmm5, %xmm1
 ; SSE41-NEXT:    xorpd %xmm9, %xmm1
 ; SSE41-NEXT:    movapd %xmm1, %xmm2
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE41-NEXT:    pand %xmm3, %xmm0
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm2
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2]
+; SSE41-NEXT:    pand %xmm2, %xmm0
+; SSE41-NEXT:    por %xmm1, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm5, %xmm7
 ; SSE41-NEXT:    movapd %xmm7, %xmm0
 ; SSE41-NEXT:    xorpd %xmm9, %xmm0
 ; SSE41-NEXT:    movapd %xmm6, %xmm1
 ; SSE41-NEXT:    xorpd %xmm9, %xmm1
 ; SSE41-NEXT:    movapd %xmm1, %xmm2
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE41-NEXT:    pand %xmm3, %xmm0
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm2
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2]
+; SSE41-NEXT:    pand %xmm2, %xmm0
+; SSE41-NEXT:    por %xmm1, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm6, %xmm7
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm7[2,3,0,1]
-; SSE41-NEXT:    movdqa %xmm7, %xmm0
-; SSE41-NEXT:    pxor %xmm9, %xmm0
+; SSE41-NEXT:    movdqa %xmm7, %xmm2
+; SSE41-NEXT:    pxor %xmm9, %xmm2
 ; SSE41-NEXT:    pxor %xmm1, %xmm9
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm3
+; SSE41-NEXT:    pcmpeqd %xmm9, %xmm3
 ; SSE41-NEXT:    pcmpgtd %xmm9, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm9
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm9[1,1,3,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,0,2,2]
 ; SSE41-NEXT:    pand %xmm3, %xmm0
 ; SSE41-NEXT:    por %xmm2, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm7, %xmm1
@@ -685,109 +670,35 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 define i32 @test_v2i32(<2 x i32> %a0) {
 ; SSE2-LABEL: test_v2i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; SSE2-NEXT:    psllq $32, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,3,2,3]
-; SSE2-NEXT:    psrad $31, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,3,2,3]
-; SSE2-NEXT:    psrad $31, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [2147483648,2147483648]
-; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    pxor %xmm1, %xmm3
-; SSE2-NEXT:    pxor %xmm2, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pand %xmm5, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm3
-; SSE2-NEXT:    pand %xmm3, %xmm2
-; SSE2-NEXT:    pandn %xmm0, %xmm3
-; SSE2-NEXT:    por %xmm2, %xmm3
-; SSE2-NEXT:    movd %xmm3, %eax
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    pandn %xmm1, %xmm2
+; SSE2-NEXT:    por %xmm0, %xmm2
+; SSE2-NEXT:    movd %xmm2, %eax
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: test_v2i32:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    psllq $32, %xmm1
-; SSE41-NEXT:    psrad $31, %xmm1
-; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,2,3,3]
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[0,2,2,3]
-; SSE41-NEXT:    psrad $31, %xmm3
-; SSE41-NEXT:    pblendw {{.*#+}} xmm3 = xmm2[0,1],xmm3[2,3],xmm2[4,5],xmm3[6,7]
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483648,2147483648]
-; SSE41-NEXT:    movdqa %xmm3, %xmm2
-; SSE41-NEXT:    pxor %xmm0, %xmm2
-; SSE41-NEXT:    pxor %xmm1, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm2, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm2, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
-; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm3
-; SSE41-NEXT:    movd %xmm3, %eax
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE41-NEXT:    pmaxsd %xmm0, %xmm1
+; SSE41-NEXT:    movd %xmm1, %eax
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: test_v2i32:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpsllq $32, %xmm0, %xmm1
-; AVX1-NEXT:    vpsrad $31, %xmm1, %xmm1
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[2,2,3,3]
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; AVX1-NEXT:    vpsrad $31, %xmm0, %xmm0
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3],xmm2[4,5],xmm0[6,7]
-; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm2
-; AVX1-NEXT:    vblendvpd %xmm2, %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vmovd %xmm0, %eax
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: test_v2i32:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpsllq $32, %xmm0, %xmm1
-; AVX2-NEXT:    vpsrad $31, %xmm1, %xmm1
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[2,2,3,3]
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; AVX2-NEXT:    vpsrad $31, %xmm0, %xmm0
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm2[0],xmm0[1],xmm2[2],xmm0[3]
-; AVX2-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm2
-; AVX2-NEXT:    vblendvpd %xmm2, %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vmovd %xmm0, %eax
-; AVX2-NEXT:    retq
-;
-; AVX512BW-LABEL: test_v2i32:
-; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpsllq $32, %xmm0, %xmm1
-; AVX512BW-NEXT:    vpsraq $32, %zmm1, %zmm1
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; AVX512BW-NEXT:    vpsraq $32, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpmaxsq %zmm0, %zmm1, %zmm0
-; AVX512BW-NEXT:    vmovd %xmm0, %eax
-; AVX512BW-NEXT:    vzeroupper
-; AVX512BW-NEXT:    retq
+; AVX-LABEL: test_v2i32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vmovd %xmm0, %eax
+; AVX-NEXT:    retq
 ;
-; AVX512VL-LABEL: test_v2i32:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsllq $32, %xmm0, %xmm1
-; AVX512VL-NEXT:    vpsraq $32, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX512VL-NEXT:    vpsllq $32, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpsraq $32, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpmaxsq %xmm0, %xmm1, %xmm0
-; AVX512VL-NEXT:    vmovd %xmm0, %eax
-; AVX512VL-NEXT:    retq
+; AVX512-LABEL: test_v2i32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vmovd %xmm0, %eax
+; AVX512-NEXT:    retq
   %1 = call i32 @llvm.experimental.vector.reduce.smax.v2i32(<2 x i32> %a0)
   ret i32 %1
 }
@@ -1130,200 +1041,62 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ;
 
 define i16 @test_v2i16(<2 x i16> %a0) {
-; SSE2-LABEL: test_v2i16:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; SSE2-NEXT:    psllq $48, %xmm0
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    psrad $31, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
-; SSE2-NEXT:    psrad $16, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; SSE2-NEXT:    psllq $48, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm2
-; SSE2-NEXT:    psrad $31, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
-; SSE2-NEXT:    psrad $16, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
-; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    pxor %xmm2, %xmm3
-; SSE2-NEXT:    pxor %xmm1, %xmm2
-; SSE2-NEXT:    movdqa %xmm3, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT:    pand %xmm5, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm2, %xmm3
-; SSE2-NEXT:    pand %xmm3, %xmm0
-; SSE2-NEXT:    pandn %xmm1, %xmm3
-; SSE2-NEXT:    por %xmm0, %xmm3
-; SSE2-NEXT:    movd %xmm3, %eax
-; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT:    retq
-;
-; SSE41-LABEL: test_v2i16:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; SSE41-NEXT:    psllq $48, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    psrad $31, %xmm2
-; SSE41-NEXT:    psrad $16, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pblendw {{.*#+}} xmm3 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
-; SSE41-NEXT:    psllq $48, %xmm1
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    psrad $31, %xmm0
-; SSE41-NEXT:    psrad $16, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm0[2,3],xmm1[4,5],xmm0[6,7]
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483648,2147483648]
-; SSE41-NEXT:    movdqa %xmm3, %xmm2
-; SSE41-NEXT:    pxor %xmm0, %xmm2
-; SSE41-NEXT:    pxor %xmm1, %xmm0
-; SSE41-NEXT:    movdqa %xmm2, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm2, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
-; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm1
-; SSE41-NEXT:    movd %xmm1, %eax
-; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT:    retq
-;
-; AVX1-LABEL: test_v2i16:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpsllq $48, %xmm0, %xmm1
-; AVX1-NEXT:    vpsrad $31, %xmm1, %xmm2
-; AVX1-NEXT:    vpsrad $16, %xmm1, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vpsllq $48, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrad $31, %xmm0, %xmm2
-; AVX1-NEXT:    vpsrad $16, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
-; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm2
-; AVX1-NEXT:    vblendvpd %xmm2, %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vmovd %xmm0, %eax
-; AVX1-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: test_v2i16:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpsllq $48, %xmm0, %xmm1
-; AVX2-NEXT:    vpsrad $31, %xmm1, %xmm2
-; AVX2-NEXT:    vpsrad $16, %xmm1, %xmm1
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpsllq $48, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrad $31, %xmm0, %xmm2
-; AVX2-NEXT:    vpsrad $16, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
-; AVX2-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm2
-; AVX2-NEXT:    vblendvpd %xmm2, %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vmovd %xmm0, %eax
-; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT:    retq
-;
-; AVX512BW-LABEL: test_v2i16:
-; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpsllq $48, %xmm0, %xmm1
-; AVX512BW-NEXT:    vpsraq $48, %zmm1, %zmm1
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX512BW-NEXT:    vpsllq $48, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpsraq $48, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpmaxsq %zmm0, %zmm1, %zmm0
-; AVX512BW-NEXT:    vmovd %xmm0, %eax
-; AVX512BW-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX512BW-NEXT:    vzeroupper
-; AVX512BW-NEXT:    retq
+; SSE-LABEL: test_v2i16:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqa %xmm0, %xmm1
+; SSE-NEXT:    psrld $16, %xmm1
+; SSE-NEXT:    pmaxsw %xmm0, %xmm1
+; SSE-NEXT:    movd %xmm1, %eax
+; SSE-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: test_v2i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpsrld $16, %xmm0, %xmm1
+; AVX-NEXT:    vpmaxsw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vmovd %xmm0, %eax
+; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX-NEXT:    retq
 ;
-; AVX512VL-LABEL: test_v2i16:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsllq $48, %xmm0, %xmm1
-; AVX512VL-NEXT:    vpsraq $48, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX512VL-NEXT:    vpsllq $48, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpsraq $48, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpmaxsq %xmm0, %xmm1, %xmm0
-; AVX512VL-NEXT:    vmovd %xmm0, %eax
-; AVX512VL-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX512VL-NEXT:    retq
+; AVX512-LABEL: test_v2i16:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
+; AVX512-NEXT:    vpmaxsw %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vmovd %xmm0, %eax
+; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX512-NEXT:    retq
   %1 = call i16 @llvm.experimental.vector.reduce.smax.v2i16(<2 x i16> %a0)
   ret i16 %1
 }
 
 define i16 @test_v4i16(<4 x i16> %a0) {
-; SSE2-LABEL: test_v4i16:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; SSE2-NEXT:    pslld $16, %xmm0
-; SSE2-NEXT:    psrad $16, %xmm0
-; SSE2-NEXT:    pslld $16, %xmm1
-; SSE2-NEXT:    psrad $16, %xmm1
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    pandn %xmm1, %xmm2
-; SSE2-NEXT:    por %xmm0, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
-; SSE2-NEXT:    movdqa %xmm2, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
-; SSE2-NEXT:    pand %xmm1, %xmm2
-; SSE2-NEXT:    pandn %xmm0, %xmm1
-; SSE2-NEXT:    por %xmm2, %xmm1
-; SSE2-NEXT:    movd %xmm1, %eax
-; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT:    retq
-;
-; SSE41-LABEL: test_v4i16:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; SSE41-NEXT:    pslld $16, %xmm0
-; SSE41-NEXT:    psrad $16, %xmm0
-; SSE41-NEXT:    pslld $16, %xmm1
-; SSE41-NEXT:    psrad $16, %xmm1
-; SSE41-NEXT:    pmaxsd %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
-; SSE41-NEXT:    pmaxsd %xmm1, %xmm0
-; SSE41-NEXT:    movd %xmm0, %eax
-; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT:    retq
+; SSE-LABEL: test_v4i16:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE-NEXT:    pmaxsw %xmm0, %xmm1
+; SSE-NEXT:    movdqa %xmm1, %xmm0
+; SSE-NEXT:    psrld $16, %xmm0
+; SSE-NEXT:    pmaxsw %xmm1, %xmm0
+; SSE-NEXT:    movd %xmm0, %eax
+; SSE-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: test_v4i16:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpslld $16, %xmm0, %xmm1
-; AVX-NEXT:    vpsrad $16, %xmm1, %xmm1
-; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX-NEXT:    vpslld $16, %xmm0, %xmm0
-; AVX-NEXT:    vpsrad $16, %xmm0, %xmm0
-; AVX-NEXT:    vpmaxsd %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpmaxsw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpsrld $16, %xmm0, %xmm1
+; AVX-NEXT:    vpmaxsw %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovd %xmm0, %eax
 ; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: test_v4i16:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpslld $16, %xmm0, %xmm1
-; AVX512-NEXT:    vpsrad $16, %xmm1, %xmm1
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpslld $16, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrad $16, %xmm0, %xmm0
-; AVX512-NEXT:    vpmaxsd %xmm0, %xmm1, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpmaxsw %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
+; AVX512-NEXT:    vpmaxsw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
 ; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX512-NEXT:    retq
@@ -1611,133 +1384,41 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 define i8 @test_v2i8(<2 x i8> %a0) {
 ; SSE2-LABEL: test_v2i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; SSE2-NEXT:    psllq $56, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psrlw $8, %xmm1
 ; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    psrad $31, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
-; SSE2-NEXT:    psrad $24, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; SSE2-NEXT:    psllq $56, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm2
-; SSE2-NEXT:    psrad $31, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
-; SSE2-NEXT:    psrad $24, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
-; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    pxor %xmm2, %xmm3
-; SSE2-NEXT:    pxor %xmm1, %xmm2
-; SSE2-NEXT:    movdqa %xmm3, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT:    pand %xmm5, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm2, %xmm3
-; SSE2-NEXT:    pand %xmm3, %xmm0
-; SSE2-NEXT:    pandn %xmm1, %xmm3
-; SSE2-NEXT:    por %xmm0, %xmm3
-; SSE2-NEXT:    movd %xmm3, %eax
+; SSE2-NEXT:    pcmpgtb %xmm1, %xmm2
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    pandn %xmm1, %xmm2
+; SSE2-NEXT:    por %xmm0, %xmm2
+; SSE2-NEXT:    movd %xmm2, %eax
 ; SSE2-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: test_v2i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; SSE41-NEXT:    psllq $56, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    psrad $31, %xmm2
-; SSE41-NEXT:    psrad $24, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pblendw {{.*#+}} xmm3 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
-; SSE41-NEXT:    psllq $56, %xmm1
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    psrad $31, %xmm0
-; SSE41-NEXT:    psrad $24, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm0[2,3],xmm1[4,5],xmm0[6,7]
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483648,2147483648]
-; SSE41-NEXT:    movdqa %xmm3, %xmm2
-; SSE41-NEXT:    pxor %xmm0, %xmm2
-; SSE41-NEXT:    pxor %xmm1, %xmm0
-; SSE41-NEXT:    movdqa %xmm2, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm2, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
-; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm1
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    psrlw $8, %xmm1
+; SSE41-NEXT:    pmaxsb %xmm0, %xmm1
 ; SSE41-NEXT:    pextrb $0, %xmm1, %eax
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: test_v2i8:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpsllq $56, %xmm0, %xmm1
-; AVX1-NEXT:    vpsrad $31, %xmm1, %xmm2
-; AVX1-NEXT:    vpsrad $24, %xmm1, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vpsllq $56, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrad $31, %xmm0, %xmm2
-; AVX1-NEXT:    vpsrad $24, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
-; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm2
-; AVX1-NEXT:    vblendvpd %xmm2, %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpextrb $0, %xmm0, %eax
-; AVX1-NEXT:    # kill: def $al killed $al killed $eax
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: test_v2i8:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpsllq $56, %xmm0, %xmm1
-; AVX2-NEXT:    vpsrad $31, %xmm1, %xmm2
-; AVX2-NEXT:    vpsrad $24, %xmm1, %xmm1
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpsllq $56, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrad $31, %xmm0, %xmm2
-; AVX2-NEXT:    vpsrad $24, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
-; AVX2-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm2
-; AVX2-NEXT:    vblendvpd %xmm2, %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpextrb $0, %xmm0, %eax
-; AVX2-NEXT:    # kill: def $al killed $al killed $eax
-; AVX2-NEXT:    retq
-;
-; AVX512BW-LABEL: test_v2i8:
-; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpsllq $56, %xmm0, %xmm1
-; AVX512BW-NEXT:    vpsraq $56, %zmm1, %zmm1
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX512BW-NEXT:    vpsllq $56, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpsraq $56, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpmaxsq %zmm0, %zmm1, %zmm0
-; AVX512BW-NEXT:    vpextrb $0, %xmm0, %eax
-; AVX512BW-NEXT:    # kill: def $al killed $al killed $eax
-; AVX512BW-NEXT:    vzeroupper
-; AVX512BW-NEXT:    retq
+; AVX-LABEL: test_v2i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX-NEXT:    vpmaxsb %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX-NEXT:    # kill: def $al killed $al killed $eax
+; AVX-NEXT:    retq
 ;
-; AVX512VL-LABEL: test_v2i8:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsllq $56, %xmm0, %xmm1
-; AVX512VL-NEXT:    vpsraq $56, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX512VL-NEXT:    vpsllq $56, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpsraq $56, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpmaxsq %xmm0, %xmm1, %xmm0
-; AVX512VL-NEXT:    vpextrb $0, %xmm0, %eax
-; AVX512VL-NEXT:    # kill: def $al killed $al killed $eax
-; AVX512VL-NEXT:    retq
+; AVX512-LABEL: test_v2i8:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX512-NEXT:    vpmaxsb %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX512-NEXT:    # kill: def $al killed $al killed $eax
+; AVX512-NEXT:    retq
   %1 = call i8 @llvm.experimental.vector.reduce.smax.v2i8(<2 x i8> %a0)
   ret i8 %1
 }
@@ -1745,19 +1426,17 @@ define i8 @test_v2i8(<2 x i8> %a0) {
 define i8 @test_v4i8(<4 x i8> %a0) {
 ; SSE2-LABEL: test_v4i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; SSE2-NEXT:    pslld $24, %xmm0
-; SSE2-NEXT:    psrad $24, %xmm0
-; SSE2-NEXT:    pslld $24, %xmm1
-; SSE2-NEXT:    psrad $24, %xmm1
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psrld $16, %xmm1
 ; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSE2-NEXT:    pcmpgtb %xmm1, %xmm2
 ; SSE2-NEXT:    pand %xmm2, %xmm0
 ; SSE2-NEXT:    pandn %xmm1, %xmm2
 ; SSE2-NEXT:    por %xmm0, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
+; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    psrlw $8, %xmm0
 ; SSE2-NEXT:    movdqa %xmm2, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
+; SSE2-NEXT:    pcmpgtb %xmm0, %xmm1
 ; SSE2-NEXT:    pand %xmm1, %xmm2
 ; SSE2-NEXT:    pandn %xmm0, %xmm1
 ; SSE2-NEXT:    por %xmm2, %xmm1
@@ -1767,42 +1446,32 @@ define i8 @test_v4i8(<4 x i8> %a0) {
 ;
 ; SSE41-LABEL: test_v4i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; SSE41-NEXT:    pslld $24, %xmm0
-; SSE41-NEXT:    psrad $24, %xmm0
-; SSE41-NEXT:    pslld $24, %xmm1
-; SSE41-NEXT:    psrad $24, %xmm1
-; SSE41-NEXT:    pmaxsd %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
-; SSE41-NEXT:    pmaxsd %xmm1, %xmm0
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    psrld $16, %xmm1
+; SSE41-NEXT:    pmaxsb %xmm0, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    psrlw $8, %xmm0
+; SSE41-NEXT:    pmaxsb %xmm1, %xmm0
 ; SSE41-NEXT:    pextrb $0, %xmm0, %eax
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: test_v4i8:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpslld $24, %xmm0, %xmm1
-; AVX-NEXT:    vpsrad $24, %xmm1, %xmm1
-; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX-NEXT:    vpslld $24, %xmm0, %xmm0
-; AVX-NEXT:    vpsrad $24, %xmm0, %xmm0
-; AVX-NEXT:    vpmaxsd %xmm0, %xmm1, %xmm0
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpsrld $16, %xmm0, %xmm1
+; AVX-NEXT:    vpmaxsb %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX-NEXT:    vpmaxsb %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpextrb $0, %xmm0, %eax
 ; AVX-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: test_v4i8:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpslld $24, %xmm0, %xmm1
-; AVX512-NEXT:    vpsrad $24, %xmm1, %xmm1
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpslld $24, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrad $24, %xmm0, %xmm0
-; AVX512-NEXT:    vpmaxsd %xmm0, %xmm1, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
+; AVX512-NEXT:    vpmaxsb %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX512-NEXT:    vpmaxsb %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
 ; AVX512-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512-NEXT:    retq
@@ -1813,82 +1482,64 @@ define i8 @test_v4i8(<4 x i8> %a0) {
 define i8 @test_v8i8(<8 x i8> %a0) {
 ; SSE2-LABEL: test_v8i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; SSE2-NEXT:    psllw $8, %xmm0
-; SSE2-NEXT:    psraw $8, %xmm0
-; SSE2-NEXT:    psllw $8, %xmm1
-; SSE2-NEXT:    psraw $8, %xmm1
-; SSE2-NEXT:    pmaxsw %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
-; SSE2-NEXT:    psllw $8, %xmm0
-; SSE2-NEXT:    psraw $8, %xmm0
-; SSE2-NEXT:    pmaxsw %xmm1, %xmm0
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    psrld $16, %xmm1
-; SSE2-NEXT:    psllw $8, %xmm1
-; SSE2-NEXT:    psraw $8, %xmm1
-; SSE2-NEXT:    pmaxsw %xmm0, %xmm1
-; SSE2-NEXT:    movd %xmm1, %eax
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    pcmpgtb %xmm1, %xmm2
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    pandn %xmm1, %xmm2
+; SSE2-NEXT:    por %xmm0, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    psrld $16, %xmm0
+; SSE2-NEXT:    movdqa %xmm2, %xmm1
+; SSE2-NEXT:    pcmpgtb %xmm0, %xmm1
+; SSE2-NEXT:    pand %xmm1, %xmm2
+; SSE2-NEXT:    pandn %xmm0, %xmm1
+; SSE2-NEXT:    por %xmm2, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    psrlw $8, %xmm0
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    pcmpgtb %xmm0, %xmm2
+; SSE2-NEXT:    pand %xmm2, %xmm1
+; SSE2-NEXT:    pandn %xmm0, %xmm2
+; SSE2-NEXT:    por %xmm1, %xmm2
+; SSE2-NEXT:    movd %xmm2, %eax
 ; SSE2-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: test_v8i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; SSE41-NEXT:    psllw $8, %xmm0
-; SSE41-NEXT:    psraw $8, %xmm0
-; SSE41-NEXT:    psllw $8, %xmm1
-; SSE41-NEXT:    psraw $8, %xmm1
-; SSE41-NEXT:    pmaxsw %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
-; SSE41-NEXT:    psllw $8, %xmm0
-; SSE41-NEXT:    psraw $8, %xmm0
-; SSE41-NEXT:    pmaxsw %xmm1, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE41-NEXT:    pmaxsb %xmm0, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    psrld $16, %xmm0
+; SSE41-NEXT:    pmaxsb %xmm1, %xmm0
 ; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    psrld $16, %xmm1
-; SSE41-NEXT:    psllw $8, %xmm1
-; SSE41-NEXT:    psraw $8, %xmm1
-; SSE41-NEXT:    pmaxsw %xmm0, %xmm1
+; SSE41-NEXT:    psrlw $8, %xmm1
+; SSE41-NEXT:    pmaxsb %xmm0, %xmm1
 ; SSE41-NEXT:    pextrb $0, %xmm1, %eax
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: test_v8i8:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpsllw $8, %xmm0, %xmm1
-; AVX-NEXT:    vpsraw $8, %xmm1, %xmm1
-; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX-NEXT:    vpsllw $8, %xmm0, %xmm0
-; AVX-NEXT:    vpsraw $8, %xmm0, %xmm0
-; AVX-NEXT:    vpmaxsw %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX-NEXT:    vpsllw $8, %xmm1, %xmm1
-; AVX-NEXT:    vpsraw $8, %xmm1, %xmm1
-; AVX-NEXT:    vpmaxsw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpmaxsb %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX-NEXT:    vpsllw $8, %xmm1, %xmm1
-; AVX-NEXT:    vpsraw $8, %xmm1, %xmm1
-; AVX-NEXT:    vpmaxsw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpmaxsb %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX-NEXT:    vpmaxsb %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpextrb $0, %xmm0, %eax
 ; AVX-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: test_v8i8:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsllw $8, %xmm0, %xmm1
-; AVX512-NEXT:    vpsraw $8, %xmm1, %xmm1
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpsllw $8, %xmm0, %xmm0
-; AVX512-NEXT:    vpsraw $8, %xmm0, %xmm0
-; AVX512-NEXT:    vpmaxsw %xmm0, %xmm1, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpsllw $8, %xmm1, %xmm1
-; AVX512-NEXT:    vpsraw $8, %xmm1, %xmm1
-; AVX512-NEXT:    vpmaxsw %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpmaxsb %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX512-NEXT:    vpsllw $8, %xmm1, %xmm1
-; AVX512-NEXT:    vpsraw $8, %xmm1, %xmm1
-; AVX512-NEXT:    vpmaxsw %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpmaxsb %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX512-NEXT:    vpmaxsb %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
 ; AVX512-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/vector-reduce-smin.ll b/llvm/test/CodeGen/X86/vector-reduce-smin.ll
index 30692a65b..116b2ed18 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-smin.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-smin.ll
@@ -36,17 +36,15 @@ define i64 @test_v2i64(<2 x i64> %a0) {
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movdqa %xmm0, %xmm1
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483648,2147483648]
-; SSE41-NEXT:    movdqa %xmm1, %xmm3
-; SSE41-NEXT:    pxor %xmm0, %xmm3
-; SSE41-NEXT:    pxor %xmm2, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [2147483648,2147483648]
+; SSE41-NEXT:    pxor %xmm3, %xmm0
+; SSE41-NEXT:    pxor %xmm2, %xmm3
+; SSE41-NEXT:    movdqa %xmm3, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm3, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
 ; SSE41-NEXT:    movq %xmm2, %rax
 ; SSE41-NEXT:    retq
@@ -123,24 +121,22 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; SSE41-NEXT:    movdqa %xmm1, %xmm4
 ; SSE41-NEXT:    pxor %xmm3, %xmm4
 ; SSE41-NEXT:    movdqa %xmm4, %xmm5
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3]
-; SSE41-NEXT:    pand %xmm6, %xmm0
-; SSE41-NEXT:    por %xmm5, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm0
+; SSE41-NEXT:    por %xmm4, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
 ; SSE41-NEXT:    movdqa %xmm1, %xmm0
 ; SSE41-NEXT:    pxor %xmm3, %xmm0
 ; SSE41-NEXT:    pxor %xmm2, %xmm3
 ; SSE41-NEXT:    movdqa %xmm3, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm3, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
 ; SSE41-NEXT:    movq %xmm2, %rax
 ; SSE41-NEXT:    retq
@@ -262,55 +258,51 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ;
 ; SSE41-LABEL: test_v8i64:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa %xmm0, %xmm8
+; SSE41-NEXT:    movdqa %xmm0, %xmm4
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm5 = [2147483648,2147483648]
 ; SSE41-NEXT:    movdqa %xmm1, %xmm0
 ; SSE41-NEXT:    pxor %xmm5, %xmm0
 ; SSE41-NEXT:    movdqa %xmm3, %xmm6
 ; SSE41-NEXT:    pxor %xmm5, %xmm6
 ; SSE41-NEXT:    movdqa %xmm6, %xmm7
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm7
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm7[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3]
-; SSE41-NEXT:    pand %xmm4, %xmm0
-; SSE41-NEXT:    por %xmm7, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm7
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
+; SSE41-NEXT:    pand %xmm7, %xmm0
+; SSE41-NEXT:    por %xmm6, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm3
-; SSE41-NEXT:    movdqa %xmm8, %xmm0
+; SSE41-NEXT:    movdqa %xmm4, %xmm0
 ; SSE41-NEXT:    pxor %xmm5, %xmm0
 ; SSE41-NEXT:    movdqa %xmm2, %xmm1
 ; SSE41-NEXT:    pxor %xmm5, %xmm1
-; SSE41-NEXT:    movdqa %xmm1, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
+; SSE41-NEXT:    movdqa %xmm1, %xmm6
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2]
 ; SSE41-NEXT:    pand %xmm6, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
-; SSE41-NEXT:    blendvpd %xmm0, %xmm8, %xmm2
+; SSE41-NEXT:    por %xmm1, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm2
 ; SSE41-NEXT:    movapd %xmm2, %xmm0
 ; SSE41-NEXT:    xorpd %xmm5, %xmm0
 ; SSE41-NEXT:    movapd %xmm3, %xmm1
 ; SSE41-NEXT:    xorpd %xmm5, %xmm1
 ; SSE41-NEXT:    movapd %xmm1, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE41-NEXT:    pand %xmm6, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm1, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm3
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[2,3,0,1]
 ; SSE41-NEXT:    movdqa %xmm3, %xmm0
 ; SSE41-NEXT:    pxor %xmm5, %xmm0
 ; SSE41-NEXT:    pxor %xmm1, %xmm5
 ; SSE41-NEXT:    movdqa %xmm5, %xmm2
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[1,1,3,3]
-; SSE41-NEXT:    pand %xmm4, %xmm0
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm2
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm5
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2]
+; SSE41-NEXT:    pand %xmm2, %xmm0
+; SSE41-NEXT:    por %xmm5, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm1
 ; SSE41-NEXT:    movq %xmm1, %rax
 ; SSE41-NEXT:    retq
@@ -504,101 +496,93 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movdqa %xmm0, %xmm8
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm9 = [2147483648,2147483648]
-; SSE41-NEXT:    movdqa %xmm2, %xmm10
-; SSE41-NEXT:    pxor %xmm9, %xmm10
-; SSE41-NEXT:    movdqa %xmm6, %xmm0
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
 ; SSE41-NEXT:    pxor %xmm9, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm11
-; SSE41-NEXT:    pcmpgtd %xmm10, %xmm11
-; SSE41-NEXT:    pshufd {{.*#+}} xmm12 = xmm11[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm12, %xmm0
-; SSE41-NEXT:    por %xmm11, %xmm0
+; SSE41-NEXT:    movdqa %xmm6, %xmm10
+; SSE41-NEXT:    pxor %xmm9, %xmm10
+; SSE41-NEXT:    movdqa %xmm10, %xmm11
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm11
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm10
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm10[0,0,2,2]
+; SSE41-NEXT:    pand %xmm11, %xmm0
+; SSE41-NEXT:    por %xmm10, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm6
 ; SSE41-NEXT:    movdqa %xmm8, %xmm0
 ; SSE41-NEXT:    pxor %xmm9, %xmm0
 ; SSE41-NEXT:    movdqa %xmm4, %xmm2
 ; SSE41-NEXT:    pxor %xmm9, %xmm2
 ; SSE41-NEXT:    movdqa %xmm2, %xmm10
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm10
-; SSE41-NEXT:    pshufd {{.*#+}} xmm11 = xmm10[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3]
-; SSE41-NEXT:    pand %xmm11, %xmm0
-; SSE41-NEXT:    por %xmm10, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm10
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,0,2,2]
+; SSE41-NEXT:    pand %xmm10, %xmm0
+; SSE41-NEXT:    por %xmm2, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm8, %xmm4
 ; SSE41-NEXT:    movdqa %xmm3, %xmm0
 ; SSE41-NEXT:    pxor %xmm9, %xmm0
 ; SSE41-NEXT:    movdqa %xmm7, %xmm2
 ; SSE41-NEXT:    pxor %xmm9, %xmm2
 ; SSE41-NEXT:    movdqa %xmm2, %xmm8
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm8
-; SSE41-NEXT:    pshufd {{.*#+}} xmm10 = xmm8[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3]
-; SSE41-NEXT:    pand %xmm10, %xmm0
-; SSE41-NEXT:    por %xmm8, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm8
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,0,2,2]
+; SSE41-NEXT:    pand %xmm8, %xmm0
+; SSE41-NEXT:    por %xmm2, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm7
 ; SSE41-NEXT:    movdqa %xmm1, %xmm0
 ; SSE41-NEXT:    pxor %xmm9, %xmm0
 ; SSE41-NEXT:    movdqa %xmm5, %xmm2
 ; SSE41-NEXT:    pxor %xmm9, %xmm2
 ; SSE41-NEXT:    movdqa %xmm2, %xmm3
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm8 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3]
-; SSE41-NEXT:    pand %xmm8, %xmm0
-; SSE41-NEXT:    por %xmm3, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm3
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,0,2,2]
+; SSE41-NEXT:    pand %xmm3, %xmm0
+; SSE41-NEXT:    por %xmm2, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm5
 ; SSE41-NEXT:    movapd %xmm5, %xmm0
 ; SSE41-NEXT:    xorpd %xmm9, %xmm0
 ; SSE41-NEXT:    movapd %xmm7, %xmm1
 ; SSE41-NEXT:    xorpd %xmm9, %xmm1
 ; SSE41-NEXT:    movapd %xmm1, %xmm2
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE41-NEXT:    pand %xmm3, %xmm0
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm2
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2]
+; SSE41-NEXT:    pand %xmm2, %xmm0
+; SSE41-NEXT:    por %xmm1, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm5, %xmm7
 ; SSE41-NEXT:    movapd %xmm4, %xmm0
 ; SSE41-NEXT:    xorpd %xmm9, %xmm0
 ; SSE41-NEXT:    movapd %xmm6, %xmm1
 ; SSE41-NEXT:    xorpd %xmm9, %xmm1
 ; SSE41-NEXT:    movapd %xmm1, %xmm2
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE41-NEXT:    pand %xmm3, %xmm0
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm2
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2]
+; SSE41-NEXT:    pand %xmm2, %xmm0
+; SSE41-NEXT:    por %xmm1, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm6
 ; SSE41-NEXT:    movapd %xmm6, %xmm0
 ; SSE41-NEXT:    xorpd %xmm9, %xmm0
 ; SSE41-NEXT:    movapd %xmm7, %xmm1
 ; SSE41-NEXT:    xorpd %xmm9, %xmm1
 ; SSE41-NEXT:    movapd %xmm1, %xmm2
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE41-NEXT:    pand %xmm3, %xmm0
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm2
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2]
+; SSE41-NEXT:    pand %xmm2, %xmm0
+; SSE41-NEXT:    por %xmm1, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm6, %xmm7
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm7[2,3,0,1]
 ; SSE41-NEXT:    movdqa %xmm7, %xmm0
 ; SSE41-NEXT:    pxor %xmm9, %xmm0
 ; SSE41-NEXT:    pxor %xmm1, %xmm9
 ; SSE41-NEXT:    movdqa %xmm9, %xmm2
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm9
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm9[1,1,3,3]
-; SSE41-NEXT:    pand %xmm3, %xmm0
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm2
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm9
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm9[0,0,2,2]
+; SSE41-NEXT:    pand %xmm2, %xmm0
+; SSE41-NEXT:    por %xmm9, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm7, %xmm1
 ; SSE41-NEXT:    movq %xmm1, %rax
 ; SSE41-NEXT:    retq
@@ -684,109 +668,35 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 define i32 @test_v2i32(<2 x i32> %a0) {
 ; SSE2-LABEL: test_v2i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; SSE2-NEXT:    psllq $32, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,3,2,3]
-; SSE2-NEXT:    psrad $31, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,3,2,3]
-; SSE2-NEXT:    psrad $31, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [2147483648,2147483648]
-; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    pxor %xmm1, %xmm3
-; SSE2-NEXT:    pxor %xmm2, %xmm1
-; SSE2-NEXT:    movdqa %xmm3, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pand %xmm5, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm3
-; SSE2-NEXT:    pand %xmm3, %xmm2
-; SSE2-NEXT:    pandn %xmm0, %xmm3
-; SSE2-NEXT:    por %xmm2, %xmm3
-; SSE2-NEXT:    movd %xmm3, %eax
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    pandn %xmm1, %xmm2
+; SSE2-NEXT:    por %xmm0, %xmm2
+; SSE2-NEXT:    movd %xmm2, %eax
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: test_v2i32:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    psllq $32, %xmm1
-; SSE41-NEXT:    psrad $31, %xmm1
-; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,2,3,3]
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[0,2,2,3]
-; SSE41-NEXT:    psrad $31, %xmm3
-; SSE41-NEXT:    pblendw {{.*#+}} xmm3 = xmm2[0,1],xmm3[2,3],xmm2[4,5],xmm3[6,7]
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483648,2147483648]
-; SSE41-NEXT:    movdqa %xmm1, %xmm2
-; SSE41-NEXT:    pxor %xmm0, %xmm2
-; SSE41-NEXT:    pxor %xmm3, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm2, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm2, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
-; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm3
-; SSE41-NEXT:    movd %xmm3, %eax
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE41-NEXT:    pminsd %xmm0, %xmm1
+; SSE41-NEXT:    movd %xmm1, %eax
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: test_v2i32:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpsllq $32, %xmm0, %xmm1
-; AVX1-NEXT:    vpsrad $31, %xmm1, %xmm1
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[2,2,3,3]
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; AVX1-NEXT:    vpsrad $31, %xmm0, %xmm0
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3],xmm2[4,5],xmm0[6,7]
-; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm2
-; AVX1-NEXT:    vblendvpd %xmm2, %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vmovd %xmm0, %eax
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: test_v2i32:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpsllq $32, %xmm0, %xmm1
-; AVX2-NEXT:    vpsrad $31, %xmm1, %xmm1
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[2,2,3,3]
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; AVX2-NEXT:    vpsrad $31, %xmm0, %xmm0
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm2[0],xmm0[1],xmm2[2],xmm0[3]
-; AVX2-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm2
-; AVX2-NEXT:    vblendvpd %xmm2, %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vmovd %xmm0, %eax
-; AVX2-NEXT:    retq
-;
-; AVX512BW-LABEL: test_v2i32:
-; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpsllq $32, %xmm0, %xmm1
-; AVX512BW-NEXT:    vpsraq $32, %zmm1, %zmm1
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; AVX512BW-NEXT:    vpsraq $32, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpminsq %zmm0, %zmm1, %zmm0
-; AVX512BW-NEXT:    vmovd %xmm0, %eax
-; AVX512BW-NEXT:    vzeroupper
-; AVX512BW-NEXT:    retq
+; AVX-LABEL: test_v2i32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vmovd %xmm0, %eax
+; AVX-NEXT:    retq
 ;
-; AVX512VL-LABEL: test_v2i32:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsllq $32, %xmm0, %xmm1
-; AVX512VL-NEXT:    vpsraq $32, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX512VL-NEXT:    vpsllq $32, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpsraq $32, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpminsq %xmm0, %xmm1, %xmm0
-; AVX512VL-NEXT:    vmovd %xmm0, %eax
-; AVX512VL-NEXT:    retq
+; AVX512-LABEL: test_v2i32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vmovd %xmm0, %eax
+; AVX512-NEXT:    retq
   %1 = call i32 @llvm.experimental.vector.reduce.smin.v2i32(<2 x i32> %a0)
   ret i32 %1
 }
@@ -1129,200 +1039,62 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ;
 
 define i16 @test_v2i16(<2 x i16> %a0) {
-; SSE2-LABEL: test_v2i16:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; SSE2-NEXT:    psllq $48, %xmm0
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    psrad $31, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
-; SSE2-NEXT:    psrad $16, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; SSE2-NEXT:    psllq $48, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm2
-; SSE2-NEXT:    psrad $31, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
-; SSE2-NEXT:    psrad $16, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
-; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    pxor %xmm2, %xmm3
-; SSE2-NEXT:    pxor %xmm1, %xmm2
-; SSE2-NEXT:    movdqa %xmm2, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT:    pand %xmm5, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm2, %xmm3
-; SSE2-NEXT:    pand %xmm3, %xmm0
-; SSE2-NEXT:    pandn %xmm1, %xmm3
-; SSE2-NEXT:    por %xmm0, %xmm3
-; SSE2-NEXT:    movd %xmm3, %eax
-; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT:    retq
-;
-; SSE41-LABEL: test_v2i16:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; SSE41-NEXT:    psllq $48, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    psrad $31, %xmm2
-; SSE41-NEXT:    psrad $16, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pblendw {{.*#+}} xmm3 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
-; SSE41-NEXT:    psllq $48, %xmm1
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    psrad $31, %xmm0
-; SSE41-NEXT:    psrad $16, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm0[2,3],xmm1[4,5],xmm0[6,7]
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483648,2147483648]
-; SSE41-NEXT:    movdqa %xmm3, %xmm2
-; SSE41-NEXT:    pxor %xmm0, %xmm2
-; SSE41-NEXT:    pxor %xmm1, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm2, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm2, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
-; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm1
-; SSE41-NEXT:    movd %xmm1, %eax
-; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT:    retq
-;
-; AVX1-LABEL: test_v2i16:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpsllq $48, %xmm0, %xmm1
-; AVX1-NEXT:    vpsrad $31, %xmm1, %xmm2
-; AVX1-NEXT:    vpsrad $16, %xmm1, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vpsllq $48, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrad $31, %xmm0, %xmm2
-; AVX1-NEXT:    vpsrad $16, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
-; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm2
-; AVX1-NEXT:    vblendvpd %xmm2, %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vmovd %xmm0, %eax
-; AVX1-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: test_v2i16:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpsllq $48, %xmm0, %xmm1
-; AVX2-NEXT:    vpsrad $31, %xmm1, %xmm2
-; AVX2-NEXT:    vpsrad $16, %xmm1, %xmm1
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpsllq $48, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrad $31, %xmm0, %xmm2
-; AVX2-NEXT:    vpsrad $16, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
-; AVX2-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm2
-; AVX2-NEXT:    vblendvpd %xmm2, %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vmovd %xmm0, %eax
-; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT:    retq
-;
-; AVX512BW-LABEL: test_v2i16:
-; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpsllq $48, %xmm0, %xmm1
-; AVX512BW-NEXT:    vpsraq $48, %zmm1, %zmm1
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX512BW-NEXT:    vpsllq $48, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpsraq $48, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpminsq %zmm0, %zmm1, %zmm0
-; AVX512BW-NEXT:    vmovd %xmm0, %eax
-; AVX512BW-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX512BW-NEXT:    vzeroupper
-; AVX512BW-NEXT:    retq
+; SSE-LABEL: test_v2i16:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqa %xmm0, %xmm1
+; SSE-NEXT:    psrld $16, %xmm1
+; SSE-NEXT:    pminsw %xmm0, %xmm1
+; SSE-NEXT:    movd %xmm1, %eax
+; SSE-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: test_v2i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpsrld $16, %xmm0, %xmm1
+; AVX-NEXT:    vpminsw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vmovd %xmm0, %eax
+; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX-NEXT:    retq
 ;
-; AVX512VL-LABEL: test_v2i16:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsllq $48, %xmm0, %xmm1
-; AVX512VL-NEXT:    vpsraq $48, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX512VL-NEXT:    vpsllq $48, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpsraq $48, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpminsq %xmm0, %xmm1, %xmm0
-; AVX512VL-NEXT:    vmovd %xmm0, %eax
-; AVX512VL-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX512VL-NEXT:    retq
+; AVX512-LABEL: test_v2i16:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
+; AVX512-NEXT:    vpminsw %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vmovd %xmm0, %eax
+; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX512-NEXT:    retq
   %1 = call i16 @llvm.experimental.vector.reduce.smin.v2i16(<2 x i16> %a0)
   ret i16 %1
 }
 
 define i16 @test_v4i16(<4 x i16> %a0) {
-; SSE2-LABEL: test_v4i16:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; SSE2-NEXT:    pslld $16, %xmm0
-; SSE2-NEXT:    psrad $16, %xmm0
-; SSE2-NEXT:    pslld $16, %xmm1
-; SSE2-NEXT:    psrad $16, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm2
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm2
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    pandn %xmm1, %xmm2
-; SSE2-NEXT:    por %xmm0, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm1
-; SSE2-NEXT:    pand %xmm1, %xmm2
-; SSE2-NEXT:    pandn %xmm0, %xmm1
-; SSE2-NEXT:    por %xmm2, %xmm1
-; SSE2-NEXT:    movd %xmm1, %eax
-; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT:    retq
-;
-; SSE41-LABEL: test_v4i16:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; SSE41-NEXT:    pslld $16, %xmm0
-; SSE41-NEXT:    psrad $16, %xmm0
-; SSE41-NEXT:    pslld $16, %xmm1
-; SSE41-NEXT:    psrad $16, %xmm1
-; SSE41-NEXT:    pminsd %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
-; SSE41-NEXT:    pminsd %xmm1, %xmm0
-; SSE41-NEXT:    movd %xmm0, %eax
-; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT:    retq
+; SSE-LABEL: test_v4i16:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE-NEXT:    pminsw %xmm0, %xmm1
+; SSE-NEXT:    movdqa %xmm1, %xmm0
+; SSE-NEXT:    psrld $16, %xmm0
+; SSE-NEXT:    pminsw %xmm1, %xmm0
+; SSE-NEXT:    movd %xmm0, %eax
+; SSE-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: test_v4i16:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpslld $16, %xmm0, %xmm1
-; AVX-NEXT:    vpsrad $16, %xmm1, %xmm1
-; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX-NEXT:    vpslld $16, %xmm0, %xmm0
-; AVX-NEXT:    vpsrad $16, %xmm0, %xmm0
-; AVX-NEXT:    vpminsd %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpminsw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpsrld $16, %xmm0, %xmm1
+; AVX-NEXT:    vpminsw %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovd %xmm0, %eax
 ; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: test_v4i16:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpslld $16, %xmm0, %xmm1
-; AVX512-NEXT:    vpsrad $16, %xmm1, %xmm1
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpslld $16, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrad $16, %xmm0, %xmm0
-; AVX512-NEXT:    vpminsd %xmm0, %xmm1, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpminsw %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
+; AVX512-NEXT:    vpminsw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
 ; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX512-NEXT:    retq
@@ -1610,133 +1382,41 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 define i8 @test_v2i8(<2 x i8> %a0) {
 ; SSE2-LABEL: test_v2i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; SSE2-NEXT:    psllq $56, %xmm0
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    psrad $31, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
-; SSE2-NEXT:    psrad $24, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; SSE2-NEXT:    psllq $56, %xmm1
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psrlw $8, %xmm1
 ; SSE2-NEXT:    movdqa %xmm1, %xmm2
-; SSE2-NEXT:    psrad $31, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
-; SSE2-NEXT:    psrad $24, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
-; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    pxor %xmm2, %xmm3
-; SSE2-NEXT:    pxor %xmm1, %xmm2
-; SSE2-NEXT:    movdqa %xmm2, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT:    pand %xmm5, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm2, %xmm3
-; SSE2-NEXT:    pand %xmm3, %xmm0
-; SSE2-NEXT:    pandn %xmm1, %xmm3
-; SSE2-NEXT:    por %xmm0, %xmm3
-; SSE2-NEXT:    movd %xmm3, %eax
+; SSE2-NEXT:    pcmpgtb %xmm0, %xmm2
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    pandn %xmm1, %xmm2
+; SSE2-NEXT:    por %xmm0, %xmm2
+; SSE2-NEXT:    movd %xmm2, %eax
 ; SSE2-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: test_v2i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; SSE41-NEXT:    psllq $56, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    psrad $31, %xmm2
-; SSE41-NEXT:    psrad $24, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pblendw {{.*#+}} xmm3 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
-; SSE41-NEXT:    psllq $56, %xmm1
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    psrad $31, %xmm0
-; SSE41-NEXT:    psrad $24, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm0[2,3],xmm1[4,5],xmm0[6,7]
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483648,2147483648]
-; SSE41-NEXT:    movdqa %xmm3, %xmm2
-; SSE41-NEXT:    pxor %xmm0, %xmm2
-; SSE41-NEXT:    pxor %xmm1, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm2, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm2, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
-; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm1
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    psrlw $8, %xmm1
+; SSE41-NEXT:    pminsb %xmm0, %xmm1
 ; SSE41-NEXT:    pextrb $0, %xmm1, %eax
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: test_v2i8:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpsllq $56, %xmm0, %xmm1
-; AVX1-NEXT:    vpsrad $31, %xmm1, %xmm2
-; AVX1-NEXT:    vpsrad $24, %xmm1, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vpsllq $56, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrad $31, %xmm0, %xmm2
-; AVX1-NEXT:    vpsrad $24, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
-; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm2
-; AVX1-NEXT:    vblendvpd %xmm2, %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpextrb $0, %xmm0, %eax
-; AVX1-NEXT:    # kill: def $al killed $al killed $eax
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: test_v2i8:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpsllq $56, %xmm0, %xmm1
-; AVX2-NEXT:    vpsrad $31, %xmm1, %xmm2
-; AVX2-NEXT:    vpsrad $24, %xmm1, %xmm1
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpsllq $56, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrad $31, %xmm0, %xmm2
-; AVX2-NEXT:    vpsrad $24, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
-; AVX2-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm2
-; AVX2-NEXT:    vblendvpd %xmm2, %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpextrb $0, %xmm0, %eax
-; AVX2-NEXT:    # kill: def $al killed $al killed $eax
-; AVX2-NEXT:    retq
-;
-; AVX512BW-LABEL: test_v2i8:
-; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpsllq $56, %xmm0, %xmm1
-; AVX512BW-NEXT:    vpsraq $56, %zmm1, %zmm1
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX512BW-NEXT:    vpsllq $56, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpsraq $56, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpminsq %zmm0, %zmm1, %zmm0
-; AVX512BW-NEXT:    vpextrb $0, %xmm0, %eax
-; AVX512BW-NEXT:    # kill: def $al killed $al killed $eax
-; AVX512BW-NEXT:    vzeroupper
-; AVX512BW-NEXT:    retq
+; AVX-LABEL: test_v2i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX-NEXT:    vpminsb %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX-NEXT:    # kill: def $al killed $al killed $eax
+; AVX-NEXT:    retq
 ;
-; AVX512VL-LABEL: test_v2i8:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsllq $56, %xmm0, %xmm1
-; AVX512VL-NEXT:    vpsraq $56, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX512VL-NEXT:    vpsllq $56, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpsraq $56, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpminsq %xmm0, %xmm1, %xmm0
-; AVX512VL-NEXT:    vpextrb $0, %xmm0, %eax
-; AVX512VL-NEXT:    # kill: def $al killed $al killed $eax
-; AVX512VL-NEXT:    retq
+; AVX512-LABEL: test_v2i8:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX512-NEXT:    vpminsb %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX512-NEXT:    # kill: def $al killed $al killed $eax
+; AVX512-NEXT:    retq
   %1 = call i8 @llvm.experimental.vector.reduce.smin.v2i8(<2 x i8> %a0)
   ret i8 %1
 }
@@ -1744,19 +1424,17 @@ define i8 @test_v2i8(<2 x i8> %a0) {
 define i8 @test_v4i8(<4 x i8> %a0) {
 ; SSE2-LABEL: test_v4i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; SSE2-NEXT:    pslld $24, %xmm0
-; SSE2-NEXT:    psrad $24, %xmm0
-; SSE2-NEXT:    pslld $24, %xmm1
-; SSE2-NEXT:    psrad $24, %xmm1
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psrld $16, %xmm1
 ; SSE2-NEXT:    movdqa %xmm1, %xmm2
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSE2-NEXT:    pcmpgtb %xmm0, %xmm2
 ; SSE2-NEXT:    pand %xmm2, %xmm0
 ; SSE2-NEXT:    pandn %xmm1, %xmm2
 ; SSE2-NEXT:    por %xmm0, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
+; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    psrlw $8, %xmm0
 ; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm1
+; SSE2-NEXT:    pcmpgtb %xmm2, %xmm1
 ; SSE2-NEXT:    pand %xmm1, %xmm2
 ; SSE2-NEXT:    pandn %xmm0, %xmm1
 ; SSE2-NEXT:    por %xmm2, %xmm1
@@ -1766,42 +1444,32 @@ define i8 @test_v4i8(<4 x i8> %a0) {
 ;
 ; SSE41-LABEL: test_v4i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; SSE41-NEXT:    pslld $24, %xmm0
-; SSE41-NEXT:    psrad $24, %xmm0
-; SSE41-NEXT:    pslld $24, %xmm1
-; SSE41-NEXT:    psrad $24, %xmm1
-; SSE41-NEXT:    pminsd %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
-; SSE41-NEXT:    pminsd %xmm1, %xmm0
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    psrld $16, %xmm1
+; SSE41-NEXT:    pminsb %xmm0, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    psrlw $8, %xmm0
+; SSE41-NEXT:    pminsb %xmm1, %xmm0
 ; SSE41-NEXT:    pextrb $0, %xmm0, %eax
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: test_v4i8:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpslld $24, %xmm0, %xmm1
-; AVX-NEXT:    vpsrad $24, %xmm1, %xmm1
-; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX-NEXT:    vpslld $24, %xmm0, %xmm0
-; AVX-NEXT:    vpsrad $24, %xmm0, %xmm0
-; AVX-NEXT:    vpminsd %xmm0, %xmm1, %xmm0
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpsrld $16, %xmm0, %xmm1
+; AVX-NEXT:    vpminsb %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX-NEXT:    vpminsb %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpextrb $0, %xmm0, %eax
 ; AVX-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: test_v4i8:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpslld $24, %xmm0, %xmm1
-; AVX512-NEXT:    vpsrad $24, %xmm1, %xmm1
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpslld $24, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrad $24, %xmm0, %xmm0
-; AVX512-NEXT:    vpminsd %xmm0, %xmm1, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
+; AVX512-NEXT:    vpminsb %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX512-NEXT:    vpminsb %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
 ; AVX512-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512-NEXT:    retq
@@ -1812,82 +1480,64 @@ define i8 @test_v4i8(<4 x i8> %a0) {
 define i8 @test_v8i8(<8 x i8> %a0) {
 ; SSE2-LABEL: test_v8i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; SSE2-NEXT:    psllw $8, %xmm0
-; SSE2-NEXT:    psraw $8, %xmm0
-; SSE2-NEXT:    psllw $8, %xmm1
-; SSE2-NEXT:    psraw $8, %xmm1
-; SSE2-NEXT:    pminsw %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
-; SSE2-NEXT:    psllw $8, %xmm0
-; SSE2-NEXT:    psraw $8, %xmm0
-; SSE2-NEXT:    pminsw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    pcmpgtb %xmm0, %xmm2
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    pandn %xmm1, %xmm2
+; SSE2-NEXT:    por %xmm0, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    psrld $16, %xmm0
 ; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    psrld $16, %xmm1
-; SSE2-NEXT:    psllw $8, %xmm1
-; SSE2-NEXT:    psraw $8, %xmm1
-; SSE2-NEXT:    pminsw %xmm0, %xmm1
-; SSE2-NEXT:    movd %xmm1, %eax
+; SSE2-NEXT:    pcmpgtb %xmm2, %xmm1
+; SSE2-NEXT:    pand %xmm1, %xmm2
+; SSE2-NEXT:    pandn %xmm0, %xmm1
+; SSE2-NEXT:    por %xmm2, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    psrlw $8, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    pcmpgtb %xmm1, %xmm2
+; SSE2-NEXT:    pand %xmm2, %xmm1
+; SSE2-NEXT:    pandn %xmm0, %xmm2
+; SSE2-NEXT:    por %xmm1, %xmm2
+; SSE2-NEXT:    movd %xmm2, %eax
 ; SSE2-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: test_v8i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; SSE41-NEXT:    psllw $8, %xmm0
-; SSE41-NEXT:    psraw $8, %xmm0
-; SSE41-NEXT:    psllw $8, %xmm1
-; SSE41-NEXT:    psraw $8, %xmm1
-; SSE41-NEXT:    pminsw %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
-; SSE41-NEXT:    psllw $8, %xmm0
-; SSE41-NEXT:    psraw $8, %xmm0
-; SSE41-NEXT:    pminsw %xmm1, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE41-NEXT:    pminsb %xmm0, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    psrld $16, %xmm0
+; SSE41-NEXT:    pminsb %xmm1, %xmm0
 ; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    psrld $16, %xmm1
-; SSE41-NEXT:    psllw $8, %xmm1
-; SSE41-NEXT:    psraw $8, %xmm1
-; SSE41-NEXT:    pminsw %xmm0, %xmm1
+; SSE41-NEXT:    psrlw $8, %xmm1
+; SSE41-NEXT:    pminsb %xmm0, %xmm1
 ; SSE41-NEXT:    pextrb $0, %xmm1, %eax
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: test_v8i8:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpsllw $8, %xmm0, %xmm1
-; AVX-NEXT:    vpsraw $8, %xmm1, %xmm1
-; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX-NEXT:    vpsllw $8, %xmm0, %xmm0
-; AVX-NEXT:    vpsraw $8, %xmm0, %xmm0
-; AVX-NEXT:    vpminsw %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX-NEXT:    vpsllw $8, %xmm1, %xmm1
-; AVX-NEXT:    vpsraw $8, %xmm1, %xmm1
-; AVX-NEXT:    vpminsw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpminsb %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX-NEXT:    vpsllw $8, %xmm1, %xmm1
-; AVX-NEXT:    vpsraw $8, %xmm1, %xmm1
-; AVX-NEXT:    vpminsw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpminsb %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX-NEXT:    vpminsb %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpextrb $0, %xmm0, %eax
 ; AVX-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: test_v8i8:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsllw $8, %xmm0, %xmm1
-; AVX512-NEXT:    vpsraw $8, %xmm1, %xmm1
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpsllw $8, %xmm0, %xmm0
-; AVX512-NEXT:    vpsraw $8, %xmm0, %xmm0
-; AVX512-NEXT:    vpminsw %xmm0, %xmm1, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpsllw $8, %xmm1, %xmm1
-; AVX512-NEXT:    vpsraw $8, %xmm1, %xmm1
-; AVX512-NEXT:    vpminsw %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpminsb %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX512-NEXT:    vpsllw $8, %xmm1, %xmm1
-; AVX512-NEXT:    vpsraw $8, %xmm1, %xmm1
-; AVX512-NEXT:    vpminsw %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpminsb %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX512-NEXT:    vpminsb %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
 ; AVX512-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/vector-reduce-umax.ll b/llvm/test/CodeGen/X86/vector-reduce-umax.ll
index d748c3b39..33a8fdace 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-umax.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-umax.ll
@@ -41,12 +41,11 @@ define i64 @test_v2i64(<2 x i64> %a0) {
 ; SSE41-NEXT:    pxor %xmm3, %xmm4
 ; SSE41-NEXT:    pxor %xmm2, %xmm3
 ; SSE41-NEXT:    movdqa %xmm4, %xmm0
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm4, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm3
-; SSE41-NEXT:    por %xmm3, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm3, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pand %xmm3, %xmm0
+; SSE41-NEXT:    por %xmm4, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
 ; SSE41-NEXT:    movq %xmm2, %rax
 ; SSE41-NEXT:    retq
@@ -127,22 +126,20 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; SSE41-NEXT:    movdqa %xmm0, %xmm5
 ; SSE41-NEXT:    pxor %xmm3, %xmm5
 ; SSE41-NEXT:    movdqa %xmm5, %xmm0
-; SSE41-NEXT:    pcmpgtd %xmm4, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm4, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm5[1,1,3,3]
-; SSE41-NEXT:    pand %xmm6, %xmm4
-; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm4, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm4, %xmm5
+; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm5[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm5, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    pxor %xmm3, %xmm0
+; SSE41-NEXT:    movdqa %xmm1, %xmm4
+; SSE41-NEXT:    pxor %xmm3, %xmm4
 ; SSE41-NEXT:    pxor %xmm2, %xmm3
-; SSE41-NEXT:    movdqa %xmm0, %xmm4
+; SSE41-NEXT:    movdqa %xmm4, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm3, %xmm5
 ; SSE41-NEXT:    pcmpgtd %xmm3, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2]
 ; SSE41-NEXT:    pand %xmm5, %xmm0
 ; SSE41-NEXT:    por %xmm4, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
@@ -283,46 +280,42 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; SSE41-NEXT:    movdqa %xmm0, %xmm7
 ; SSE41-NEXT:    pxor %xmm5, %xmm7
 ; SSE41-NEXT:    movdqa %xmm7, %xmm0
-; SSE41-NEXT:    pcmpgtd %xmm6, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm8 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm6, %xmm7
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm7[1,1,3,3]
-; SSE41-NEXT:    pand %xmm8, %xmm6
-; SSE41-NEXT:    por %xmm6, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm6, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm6, %xmm7
+; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm7[0,0,2,2]
+; SSE41-NEXT:    pand %xmm6, %xmm0
+; SSE41-NEXT:    por %xmm7, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm2
 ; SSE41-NEXT:    movdqa %xmm3, %xmm0
 ; SSE41-NEXT:    pxor %xmm5, %xmm0
 ; SSE41-NEXT:    movdqa %xmm1, %xmm4
 ; SSE41-NEXT:    pxor %xmm5, %xmm4
 ; SSE41-NEXT:    movdqa %xmm4, %xmm6
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3]
-; SSE41-NEXT:    pand %xmm7, %xmm0
-; SSE41-NEXT:    por %xmm6, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pand %xmm6, %xmm0
+; SSE41-NEXT:    por %xmm4, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm3
 ; SSE41-NEXT:    movapd %xmm3, %xmm0
 ; SSE41-NEXT:    xorpd %xmm5, %xmm0
 ; SSE41-NEXT:    movapd %xmm2, %xmm1
 ; SSE41-NEXT:    xorpd %xmm5, %xmm1
 ; SSE41-NEXT:    movapd %xmm1, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE41-NEXT:    pand %xmm6, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm1, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm3
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[2,3,0,1]
-; SSE41-NEXT:    movdqa %xmm3, %xmm0
-; SSE41-NEXT:    pxor %xmm5, %xmm0
+; SSE41-NEXT:    movdqa %xmm3, %xmm2
+; SSE41-NEXT:    pxor %xmm5, %xmm2
 ; SSE41-NEXT:    pxor %xmm1, %xmm5
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm5, %xmm4
 ; SSE41-NEXT:    pcmpgtd %xmm5, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[1,1,3,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,0,2,2]
 ; SSE41-NEXT:    pand %xmm4, %xmm0
 ; SSE41-NEXT:    por %xmm2, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm1
@@ -534,99 +527,91 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movdqa %xmm0, %xmm8
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm9 = [9223372039002259456,9223372039002259456]
-; SSE41-NEXT:    movdqa %xmm5, %xmm10
-; SSE41-NEXT:    pxor %xmm9, %xmm10
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    movdqa %xmm5, %xmm0
 ; SSE41-NEXT:    pxor %xmm9, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm11
-; SSE41-NEXT:    pcmpgtd %xmm10, %xmm11
-; SSE41-NEXT:    pshufd {{.*#+}} xmm12 = xmm11[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm12, %xmm0
-; SSE41-NEXT:    por %xmm11, %xmm0
+; SSE41-NEXT:    movdqa %xmm1, %xmm10
+; SSE41-NEXT:    pxor %xmm9, %xmm10
+; SSE41-NEXT:    movdqa %xmm10, %xmm11
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm11
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm10
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm10[0,0,2,2]
+; SSE41-NEXT:    pand %xmm11, %xmm0
+; SSE41-NEXT:    por %xmm10, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm5
 ; SSE41-NEXT:    movdqa %xmm7, %xmm0
 ; SSE41-NEXT:    pxor %xmm9, %xmm0
 ; SSE41-NEXT:    movdqa %xmm3, %xmm1
 ; SSE41-NEXT:    pxor %xmm9, %xmm1
 ; SSE41-NEXT:    movdqa %xmm1, %xmm10
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm10
-; SSE41-NEXT:    pshufd {{.*#+}} xmm11 = xmm10[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE41-NEXT:    pand %xmm11, %xmm0
-; SSE41-NEXT:    por %xmm10, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm10
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2]
+; SSE41-NEXT:    pand %xmm10, %xmm0
+; SSE41-NEXT:    por %xmm1, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm7
 ; SSE41-NEXT:    movdqa %xmm4, %xmm1
 ; SSE41-NEXT:    pxor %xmm9, %xmm1
 ; SSE41-NEXT:    movdqa %xmm8, %xmm3
 ; SSE41-NEXT:    pxor %xmm9, %xmm3
 ; SSE41-NEXT:    movdqa %xmm3, %xmm0
-; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm10 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm1, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pand %xmm10, %xmm1
-; SSE41-NEXT:    por %xmm1, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm1, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pand %xmm1, %xmm0
+; SSE41-NEXT:    por %xmm3, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm8, %xmm4
 ; SSE41-NEXT:    movdqa %xmm6, %xmm0
 ; SSE41-NEXT:    pxor %xmm9, %xmm0
 ; SSE41-NEXT:    movdqa %xmm2, %xmm1
 ; SSE41-NEXT:    pxor %xmm9, %xmm1
 ; SSE41-NEXT:    movdqa %xmm1, %xmm3
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm8 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE41-NEXT:    pand %xmm8, %xmm0
-; SSE41-NEXT:    por %xmm3, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm3
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2]
+; SSE41-NEXT:    pand %xmm3, %xmm0
+; SSE41-NEXT:    por %xmm1, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm6
 ; SSE41-NEXT:    movapd %xmm6, %xmm0
 ; SSE41-NEXT:    xorpd %xmm9, %xmm0
 ; SSE41-NEXT:    movapd %xmm4, %xmm1
 ; SSE41-NEXT:    xorpd %xmm9, %xmm1
 ; SSE41-NEXT:    movapd %xmm1, %xmm2
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE41-NEXT:    pand %xmm3, %xmm0
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm2
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2]
+; SSE41-NEXT:    pand %xmm2, %xmm0
+; SSE41-NEXT:    por %xmm1, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm6
 ; SSE41-NEXT:    movapd %xmm7, %xmm0
 ; SSE41-NEXT:    xorpd %xmm9, %xmm0
 ; SSE41-NEXT:    movapd %xmm5, %xmm1
 ; SSE41-NEXT:    xorpd %xmm9, %xmm1
 ; SSE41-NEXT:    movapd %xmm1, %xmm2
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE41-NEXT:    pand %xmm3, %xmm0
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm2
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2]
+; SSE41-NEXT:    pand %xmm2, %xmm0
+; SSE41-NEXT:    por %xmm1, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm5, %xmm7
 ; SSE41-NEXT:    movapd %xmm7, %xmm0
 ; SSE41-NEXT:    xorpd %xmm9, %xmm0
 ; SSE41-NEXT:    movapd %xmm6, %xmm1
 ; SSE41-NEXT:    xorpd %xmm9, %xmm1
 ; SSE41-NEXT:    movapd %xmm1, %xmm2
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE41-NEXT:    pand %xmm3, %xmm0
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm2
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2]
+; SSE41-NEXT:    pand %xmm2, %xmm0
+; SSE41-NEXT:    por %xmm1, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm6, %xmm7
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm7[2,3,0,1]
-; SSE41-NEXT:    movdqa %xmm7, %xmm0
-; SSE41-NEXT:    pxor %xmm9, %xmm0
+; SSE41-NEXT:    movdqa %xmm7, %xmm2
+; SSE41-NEXT:    pxor %xmm9, %xmm2
 ; SSE41-NEXT:    pxor %xmm1, %xmm9
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm3
+; SSE41-NEXT:    pcmpeqd %xmm9, %xmm3
 ; SSE41-NEXT:    pcmpgtd %xmm9, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm9
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm9[1,1,3,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,0,2,2]
 ; SSE41-NEXT:    pand %xmm3, %xmm0
 ; SSE41-NEXT:    por %xmm2, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm7, %xmm1
@@ -742,87 +727,38 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 define i32 @test_v2i32(<2 x i32> %a0) {
 ; SSE2-LABEL: test_v2i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [4294967295,0,4294967295,0]
-; SSE2-NEXT:    pand %xmm0, %xmm1
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm2[2],xmm0[3],xmm2[3]
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm3
 ; SSE2-NEXT:    pxor %xmm2, %xmm3
 ; SSE2-NEXT:    pxor %xmm1, %xmm2
-; SSE2-NEXT:    movdqa %xmm2, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT:    pand %xmm5, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm2, %xmm3
-; SSE2-NEXT:    pand %xmm3, %xmm1
-; SSE2-NEXT:    pandn %xmm0, %xmm3
-; SSE2-NEXT:    por %xmm1, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm3
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    pandn %xmm1, %xmm3
+; SSE2-NEXT:    por %xmm0, %xmm3
 ; SSE2-NEXT:    movd %xmm3, %eax
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: test_v2i32:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pxor %xmm1, %xmm1
-; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483648,2147483648]
-; SSE41-NEXT:    movdqa %xmm1, %xmm3
-; SSE41-NEXT:    pxor %xmm0, %xmm3
-; SSE41-NEXT:    pxor %xmm2, %xmm0
-; SSE41-NEXT:    movdqa %xmm3, %xmm4
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pand %xmm4, %xmm0
-; SSE41-NEXT:    por %xmm3, %xmm0
-; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
-; SSE41-NEXT:    movd %xmm2, %eax
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE41-NEXT:    pmaxud %xmm0, %xmm1
+; SSE41-NEXT:    movd %xmm1, %eax
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: test_v2i32:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
-; AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm1
-; AVX1-NEXT:    vblendvpd %xmm1, %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vmovd %xmm0, %eax
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: test_v2i32:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm2 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
-; AVX2-NEXT:    vpunpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX2-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm1
-; AVX2-NEXT:    vblendvpd %xmm1, %xmm2, %xmm0, %xmm0
-; AVX2-NEXT:    vmovd %xmm0, %eax
-; AVX2-NEXT:    retq
-;
-; AVX512BW-LABEL: test_v2i32:
-; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm2 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
-; AVX512BW-NEXT:    vpunpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX512BW-NEXT:    vpmaxuq %zmm0, %zmm2, %zmm0
-; AVX512BW-NEXT:    vmovd %xmm0, %eax
-; AVX512BW-NEXT:    vzeroupper
-; AVX512BW-NEXT:    retq
+; AVX-LABEL: test_v2i32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vmovd %xmm0, %eax
+; AVX-NEXT:    retq
 ;
-; AVX512VL-LABEL: test_v2i32:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpblendd {{.*#+}} xmm1 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
-; AVX512VL-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX512VL-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
-; AVX512VL-NEXT:    vpmaxuq %xmm0, %xmm1, %xmm0
-; AVX512VL-NEXT:    vmovd %xmm0, %eax
-; AVX512VL-NEXT:    retq
+; AVX512-LABEL: test_v2i32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vmovd %xmm0, %eax
+; AVX512-NEXT:    retq
   %1 = call i32 @llvm.experimental.vector.reduce.umax.v2i32(<2 x i32> %a0)
   ret i32 %1
 }
@@ -1224,85 +1160,41 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 define i16 @test_v2i16(<2 x i16> %a0) {
 ; SSE2-LABEL: test_v2i16:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [65535,0,0,0,65535,0,0,0]
-; SSE2-NEXT:    pand %xmm0, %xmm1
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
-; SSE2-NEXT:    movdqa %xmm1, %xmm3
-; SSE2-NEXT:    por %xmm2, %xmm3
-; SSE2-NEXT:    por %xmm0, %xmm2
-; SSE2-NEXT:    movdqa %xmm3, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT:    pand %xmm5, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm2, %xmm3
-; SSE2-NEXT:    pand %xmm3, %xmm1
-; SSE2-NEXT:    pandn %xmm0, %xmm3
-; SSE2-NEXT:    por %xmm1, %xmm3
-; SSE2-NEXT:    movd %xmm3, %eax
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psrld $16, %xmm1
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [32768,32768,32768,32768,32768,32768,32768,32768]
+; SSE2-NEXT:    pxor %xmm2, %xmm0
+; SSE2-NEXT:    pxor %xmm2, %xmm1
+; SSE2-NEXT:    pmaxsw %xmm0, %xmm1
+; SSE2-NEXT:    movd %xmm1, %eax
+; SSE2-NEXT:    xorl $32768, %eax # imm = 0x8000
 ; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: test_v2i16:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pxor %xmm1, %xmm1
-; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3],xmm0[4],xmm1[5,6,7]
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm2 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483648,2147483648]
-; SSE41-NEXT:    movdqa %xmm1, %xmm3
-; SSE41-NEXT:    por %xmm0, %xmm3
-; SSE41-NEXT:    por %xmm2, %xmm0
-; SSE41-NEXT:    movdqa %xmm3, %xmm4
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pand %xmm4, %xmm0
-; SSE41-NEXT:    por %xmm3, %xmm0
-; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
-; SSE41-NEXT:    movd %xmm2, %eax
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    psrld $16, %xmm1
+; SSE41-NEXT:    pmaxuw %xmm0, %xmm1
+; SSE41-NEXT:    movd %xmm1, %eax
 ; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: test_v2i16:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3],xmm0[4],xmm1[5,6,7]
-; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; AVX-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm2
-; AVX-NEXT:    vblendvpd %xmm2, %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpsrld $16, %xmm0, %xmm1
+; AVX-NEXT:    vpmaxuw %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovd %xmm0, %eax
 ; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX-NEXT:    retq
 ;
-; AVX512BW-LABEL: test_v2i16:
-; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3],xmm0[4],xmm1[5,6,7]
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; AVX512BW-NEXT:    vpmaxuq %zmm0, %zmm1, %zmm0
-; AVX512BW-NEXT:    vmovd %xmm0, %eax
-; AVX512BW-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX512BW-NEXT:    vzeroupper
-; AVX512BW-NEXT:    retq
-;
-; AVX512VL-LABEL: test_v2i16:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3],xmm0[4],xmm1[5,6,7]
-; AVX512VL-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX512VL-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; AVX512VL-NEXT:    vpmaxuq %xmm0, %xmm1, %xmm0
-; AVX512VL-NEXT:    vmovd %xmm0, %eax
-; AVX512VL-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX512VL-NEXT:    retq
+; AVX512-LABEL: test_v2i16:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
+; AVX512-NEXT:    vpmaxuw %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vmovd %xmm0, %eax
+; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX512-NEXT:    retq
   %1 = call i16 @llvm.experimental.vector.reduce.umax.v2i16(<2 x i16> %a0)
   ret i16 %1
 }
@@ -1310,70 +1202,48 @@ define i16 @test_v2i16(<2 x i16> %a0) {
 define i16 @test_v4i16(<4 x i16> %a0) {
 ; SSE2-LABEL: test_v4i16:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [65535,0,65535,0,65535,0,65535,0]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
-; SSE2-NEXT:    pand %xmm1, %xmm0
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm2
-; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm3
-; SSE2-NEXT:    pand %xmm3, %xmm0
-; SSE2-NEXT:    pandn %xmm2, %xmm3
-; SSE2-NEXT:    por %xmm0, %xmm3
-; SSE2-NEXT:    pand %xmm3, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,2,3]
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; SSE2-NEXT:    movdqa %xmm1, %xmm2
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm2
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pandn %xmm0, %xmm2
-; SSE2-NEXT:    por %xmm1, %xmm2
-; SSE2-NEXT:    movd %xmm2, %eax
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [32768,32768,32768,32768,32768,32768,32768,32768]
+; SSE2-NEXT:    pxor %xmm2, %xmm0
+; SSE2-NEXT:    pxor %xmm2, %xmm1
+; SSE2-NEXT:    pmaxsw %xmm0, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    pxor %xmm2, %xmm0
+; SSE2-NEXT:    psrld $16, %xmm0
+; SSE2-NEXT:    pxor %xmm2, %xmm0
+; SSE2-NEXT:    pmaxsw %xmm1, %xmm0
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    xorl $32768, %eax # imm = 0x8000
 ; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: test_v4i16:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pxor %xmm1, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7]
-; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0],xmm1[1],xmm2[2],xmm1[3,4,5,6,7]
-; SSE41-NEXT:    pmaxud %xmm0, %xmm2
-; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2],xmm1[3],xmm2[4],xmm1[5],xmm2[6],xmm1[7]
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
-; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; SSE41-NEXT:    pmaxud %xmm1, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE41-NEXT:    pmaxuw %xmm0, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    psrld $16, %xmm0
+; SSE41-NEXT:    pmaxuw %xmm1, %xmm0
 ; SSE41-NEXT:    movd %xmm0, %eax
 ; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: test_v4i16:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX-NEXT:    vpblendw {{.*#+}} xmm2 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7]
-; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3,4,5,6,7]
-; AVX-NEXT:    vpmaxud %xmm0, %xmm2, %xmm0
-; AVX-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7]
-; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; AVX-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; AVX-NEXT:    vpmaxud %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpmaxuw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpsrld $16, %xmm0, %xmm1
+; AVX-NEXT:    vpmaxuw %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovd %xmm0, %eax
 ; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: test_v4i16:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm2 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7]
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3,4,5,6,7]
-; AVX512-NEXT:    vpmaxud %xmm0, %xmm2, %xmm0
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7]
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; AVX512-NEXT:    vpmaxud %xmm0, %xmm1, %xmm0
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpmaxuw %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
+; AVX512-NEXT:    vpmaxuw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
 ; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX512-NEXT:    retq
@@ -1777,83 +1647,37 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 define i8 @test_v2i8(<2 x i8> %a0) {
 ; SSE2-LABEL: test_v2i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [255,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0]
-; SSE2-NEXT:    pand %xmm0, %xmm1
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
-; SSE2-NEXT:    movdqa %xmm1, %xmm3
-; SSE2-NEXT:    por %xmm2, %xmm3
-; SSE2-NEXT:    por %xmm0, %xmm2
-; SSE2-NEXT:    movdqa %xmm3, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT:    pand %xmm5, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm2, %xmm3
-; SSE2-NEXT:    pand %xmm3, %xmm1
-; SSE2-NEXT:    pandn %xmm0, %xmm3
-; SSE2-NEXT:    por %xmm1, %xmm3
-; SSE2-NEXT:    movd %xmm3, %eax
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psrlw $8, %xmm1
+; SSE2-NEXT:    pmaxub %xmm0, %xmm1
+; SSE2-NEXT:    movd %xmm1, %eax
 ; SSE2-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: test_v2i8:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; SSE41-NEXT:    pand {{.*}}(%rip), %xmm1
-; SSE41-NEXT:    pmovzxbq {{.*#+}} xmm2 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
-; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [2147483648,2147483648]
-; SSE41-NEXT:    movdqa %xmm1, %xmm4
-; SSE41-NEXT:    por %xmm3, %xmm4
-; SSE41-NEXT:    por %xmm2, %xmm3
-; SSE41-NEXT:    movdqa %xmm4, %xmm0
-; SSE41-NEXT:    pcmpeqd %xmm3, %xmm0
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pand %xmm3, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
-; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
-; SSE41-NEXT:    pextrb $0, %xmm2, %eax
+; SSE41-NEXT:    psrlw $8, %xmm1
+; SSE41-NEXT:    pmaxub %xmm0, %xmm1
+; SSE41-NEXT:    pextrb $0, %xmm1, %eax
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: test_v2i8:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm1
-; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
-; AVX-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm2
-; AVX-NEXT:    vblendvpd %xmm2, %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX-NEXT:    vpmaxub %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpextrb $0, %xmm0, %eax
 ; AVX-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX-NEXT:    retq
 ;
-; AVX512BW-LABEL: test_v2i8:
-; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm1
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512BW-NEXT:    vpmaxuq %zmm0, %zmm1, %zmm0
-; AVX512BW-NEXT:    vpextrb $0, %xmm0, %eax
-; AVX512BW-NEXT:    # kill: def $al killed $al killed $eax
-; AVX512BW-NEXT:    vzeroupper
-; AVX512BW-NEXT:    retq
-;
-; AVX512VL-LABEL: test_v2i8:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm1
-; AVX512VL-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512VL-NEXT:    vpmaxuq %xmm0, %xmm1, %xmm0
-; AVX512VL-NEXT:    vpextrb $0, %xmm0, %eax
-; AVX512VL-NEXT:    # kill: def $al killed $al killed $eax
-; AVX512VL-NEXT:    retq
+; AVX512-LABEL: test_v2i8:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX512-NEXT:    vpmaxub %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX512-NEXT:    # kill: def $al killed $al killed $eax
+; AVX512-NEXT:    retq
   %1 = call i8 @llvm.experimental.vector.reduce.umax.v2i8(<2 x i8> %a0)
   ret i8 %1
 }
@@ -1861,83 +1685,44 @@ define i8 @test_v2i8(<2 x i8> %a0) {
 define i8 @test_v4i8(<4 x i8> %a0) {
 ; SSE2-LABEL: test_v4i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
-; SSE2-NEXT:    pand %xmm1, %xmm0
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm2
-; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm3
-; SSE2-NEXT:    pand %xmm3, %xmm0
-; SSE2-NEXT:    pandn %xmm2, %xmm3
-; SSE2-NEXT:    por %xmm0, %xmm3
-; SSE2-NEXT:    pand %xmm3, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,2,3]
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; SSE2-NEXT:    movdqa %xmm1, %xmm2
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm2
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pandn %xmm0, %xmm2
-; SSE2-NEXT:    por %xmm1, %xmm2
-; SSE2-NEXT:    movd %xmm2, %eax
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psrld $16, %xmm1
+; SSE2-NEXT:    pmaxub %xmm0, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    psrlw $8, %xmm0
+; SSE2-NEXT:    pmaxub %xmm1, %xmm0
+; SSE2-NEXT:    movd %xmm0, %eax
 ; SSE2-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: test_v4i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0]
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[8],zero,zero,zero,xmm0[12],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; SSE41-NEXT:    pmaxud %xmm2, %xmm0
-; SSE41-NEXT:    pand %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; SSE41-NEXT:    pmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
-; SSE41-NEXT:    pmaxud %xmm1, %xmm0
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    psrld $16, %xmm1
+; SSE41-NEXT:    pmaxub %xmm0, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    psrlw $8, %xmm0
+; SSE41-NEXT:    pmaxub %xmm1, %xmm0
 ; SSE41-NEXT:    pextrb $0, %xmm0, %eax
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: test_v4i8:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm1 = [3.57331108E-43,3.57331108E-43,3.57331108E-43,3.57331108E-43]
-; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm2
-; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[8],zero,zero,zero,xmm0[12],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX1-NEXT:    vpmaxud %xmm0, %xmm2, %xmm0
-; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; AVX1-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
-; AVX1-NEXT:    vpmaxud %xmm0, %xmm1, %xmm0
-; AVX1-NEXT:    vpextrb $0, %xmm0, %eax
-; AVX1-NEXT:    # kill: def $al killed $al killed $eax
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: test_v4i8:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [255,255,255,255]
-; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm2
-; AVX2-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[8],zero,zero,zero,xmm0[12],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX2-NEXT:    vpmaxud %xmm0, %xmm2, %xmm0
-; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm1
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
-; AVX2-NEXT:    vpmaxud %xmm0, %xmm1, %xmm0
-; AVX2-NEXT:    vpextrb $0, %xmm0, %eax
-; AVX2-NEXT:    # kill: def $al killed $al killed $eax
-; AVX2-NEXT:    retq
+; AVX-LABEL: test_v4i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpsrld $16, %xmm0, %xmm1
+; AVX-NEXT:    vpmaxub %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX-NEXT:    vpmaxub %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX-NEXT:    # kill: def $al killed $al killed $eax
+; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: test_v4i8:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [255,255,255,255]
-; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm2
-; AVX512-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[8],zero,zero,zero,xmm0[12],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT:    vpmaxud %xmm0, %xmm2, %xmm0
-; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm1
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT:    vpmaxud %xmm0, %xmm1, %xmm0
+; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
+; AVX512-NEXT:    vpmaxub %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX512-NEXT:    vpmaxub %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
 ; AVX512-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512-NEXT:    retq
@@ -1948,92 +1733,52 @@ define i8 @test_v4i8(<4 x i8> %a0) {
 define i8 @test_v8i8(<8 x i8> %a0) {
 ; SSE2-LABEL: test_v8i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
-; SSE2-NEXT:    pand %xmm1, %xmm0
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm2
-; SSE2-NEXT:    pmaxsw %xmm0, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
-; SSE2-NEXT:    pand %xmm1, %xmm2
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
-; SSE2-NEXT:    pmaxsw %xmm2, %xmm0
-; SSE2-NEXT:    pand %xmm0, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE2-NEXT:    pmaxub %xmm0, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
 ; SSE2-NEXT:    psrld $16, %xmm0
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; SSE2-NEXT:    pmaxsw %xmm1, %xmm0
-; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    pmaxub %xmm1, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psrlw $8, %xmm1
+; SSE2-NEXT:    pmaxub %xmm0, %xmm1
+; SSE2-NEXT:    movd %xmm1, %eax
 ; SSE2-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: test_v8i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[8],zero,xmm0[10],zero,xmm0[12],zero,xmm0[14],zero,zero,zero,zero,zero,zero,zero,zero,zero
-; SSE41-NEXT:    pmaxuw %xmm2, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[4],zero,xmm0[6],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; SSE41-NEXT:    pmaxuw %xmm2, %xmm0
-; SSE41-NEXT:    pand %xmm0, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE41-NEXT:    pmaxub %xmm0, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
 ; SSE41-NEXT:    psrld $16, %xmm0
-; SSE41-NEXT:    pmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
-; SSE41-NEXT:    pmaxuw %xmm1, %xmm0
-; SSE41-NEXT:    pextrb $0, %xmm0, %eax
+; SSE41-NEXT:    pmaxub %xmm1, %xmm0
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    psrlw $8, %xmm1
+; SSE41-NEXT:    pmaxub %xmm0, %xmm1
+; SSE41-NEXT:    pextrb $0, %xmm1, %eax
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: test_v8i8:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
-; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm2
-; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[8],zero,xmm0[10],zero,xmm0[12],zero,xmm0[14],zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX1-NEXT:    vpmaxuw %xmm0, %xmm2, %xmm0
-; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm2
-; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[4],zero,xmm0[6],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX1-NEXT:    vpmaxuw %xmm0, %xmm2, %xmm0
-; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm0
-; AVX1-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
-; AVX1-NEXT:    vpmaxuw %xmm0, %xmm1, %xmm0
-; AVX1-NEXT:    vpextrb $0, %xmm0, %eax
-; AVX1-NEXT:    # kill: def $al killed $al killed $eax
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: test_v8i8:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpbroadcastw {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
-; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm2
-; AVX2-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[8],zero,xmm0[10],zero,xmm0[12],zero,xmm0[14],zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX2-NEXT:    vpmaxuw %xmm0, %xmm2, %xmm0
-; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm2
-; AVX2-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[4],zero,xmm0[6],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX2-NEXT:    vpmaxuw %xmm0, %xmm2, %xmm0
-; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm1
-; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm0
-; AVX2-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
-; AVX2-NEXT:    vpmaxuw %xmm0, %xmm1, %xmm0
-; AVX2-NEXT:    vpextrb $0, %xmm0, %eax
-; AVX2-NEXT:    # kill: def $al killed $al killed $eax
-; AVX2-NEXT:    retq
+; AVX-LABEL: test_v8i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpmaxub %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpsrld $16, %xmm0, %xmm1
+; AVX-NEXT:    vpmaxub %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX-NEXT:    vpmaxub %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX-NEXT:    # kill: def $al killed $al killed $eax
+; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: test_v8i8:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpbroadcastw {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
-; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm2
-; AVX512-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[8],zero,xmm0[10],zero,xmm0[12],zero,xmm0[14],zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT:    vpmaxuw %xmm0, %xmm2, %xmm0
-; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm2
-; AVX512-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[4],zero,xmm0[6],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT:    vpmaxuw %xmm0, %xmm2, %xmm0
-; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm1
-; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm0
-; AVX512-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT:    vpmaxuw %xmm0, %xmm1, %xmm0
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpmaxub %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
+; AVX512-NEXT:    vpmaxub %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX512-NEXT:    vpmaxub %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
 ; AVX512-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/vector-reduce-umin.ll b/llvm/test/CodeGen/X86/vector-reduce-umin.ll
index cde6c4065..7225c270a 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-umin.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-umin.ll
@@ -36,17 +36,15 @@ define i64 @test_v2i64(<2 x i64> %a0) {
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movdqa %xmm0, %xmm1
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [9223372039002259456,9223372039002259456]
-; SSE41-NEXT:    movdqa %xmm1, %xmm3
-; SSE41-NEXT:    pxor %xmm0, %xmm3
-; SSE41-NEXT:    pxor %xmm2, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [9223372039002259456,9223372039002259456]
+; SSE41-NEXT:    pxor %xmm3, %xmm0
+; SSE41-NEXT:    pxor %xmm2, %xmm3
+; SSE41-NEXT:    movdqa %xmm3, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm3, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
 ; SSE41-NEXT:    movq %xmm2, %rax
 ; SSE41-NEXT:    retq
@@ -126,24 +124,22 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; SSE41-NEXT:    movdqa %xmm1, %xmm4
 ; SSE41-NEXT:    pxor %xmm3, %xmm4
 ; SSE41-NEXT:    movdqa %xmm4, %xmm5
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3]
-; SSE41-NEXT:    pand %xmm6, %xmm0
-; SSE41-NEXT:    por %xmm5, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm0
+; SSE41-NEXT:    por %xmm4, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
 ; SSE41-NEXT:    movdqa %xmm1, %xmm0
 ; SSE41-NEXT:    pxor %xmm3, %xmm0
 ; SSE41-NEXT:    pxor %xmm2, %xmm3
 ; SSE41-NEXT:    movdqa %xmm3, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm3, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
 ; SSE41-NEXT:    movq %xmm2, %rax
 ; SSE41-NEXT:    retq
@@ -275,55 +271,51 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ;
 ; SSE41-LABEL: test_v8i64:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa %xmm0, %xmm8
+; SSE41-NEXT:    movdqa %xmm0, %xmm4
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm5 = [9223372039002259456,9223372039002259456]
 ; SSE41-NEXT:    movdqa %xmm1, %xmm0
 ; SSE41-NEXT:    pxor %xmm5, %xmm0
 ; SSE41-NEXT:    movdqa %xmm3, %xmm6
 ; SSE41-NEXT:    pxor %xmm5, %xmm6
 ; SSE41-NEXT:    movdqa %xmm6, %xmm7
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm7
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm7[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3]
-; SSE41-NEXT:    pand %xmm4, %xmm0
-; SSE41-NEXT:    por %xmm7, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm7
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
+; SSE41-NEXT:    pand %xmm7, %xmm0
+; SSE41-NEXT:    por %xmm6, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm3
-; SSE41-NEXT:    movdqa %xmm8, %xmm0
+; SSE41-NEXT:    movdqa %xmm4, %xmm0
 ; SSE41-NEXT:    pxor %xmm5, %xmm0
 ; SSE41-NEXT:    movdqa %xmm2, %xmm1
 ; SSE41-NEXT:    pxor %xmm5, %xmm1
-; SSE41-NEXT:    movdqa %xmm1, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
+; SSE41-NEXT:    movdqa %xmm1, %xmm6
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2]
 ; SSE41-NEXT:    pand %xmm6, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
-; SSE41-NEXT:    blendvpd %xmm0, %xmm8, %xmm2
+; SSE41-NEXT:    por %xmm1, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm2
 ; SSE41-NEXT:    movapd %xmm2, %xmm0
 ; SSE41-NEXT:    xorpd %xmm5, %xmm0
 ; SSE41-NEXT:    movapd %xmm3, %xmm1
 ; SSE41-NEXT:    xorpd %xmm5, %xmm1
 ; SSE41-NEXT:    movapd %xmm1, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE41-NEXT:    pand %xmm6, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm1, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm3
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[2,3,0,1]
 ; SSE41-NEXT:    movdqa %xmm3, %xmm0
 ; SSE41-NEXT:    pxor %xmm5, %xmm0
 ; SSE41-NEXT:    pxor %xmm1, %xmm5
 ; SSE41-NEXT:    movdqa %xmm5, %xmm2
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[1,1,3,3]
-; SSE41-NEXT:    pand %xmm4, %xmm0
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm2
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm5
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2]
+; SSE41-NEXT:    pand %xmm2, %xmm0
+; SSE41-NEXT:    por %xmm5, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm1
 ; SSE41-NEXT:    movq %xmm1, %rax
 ; SSE41-NEXT:    retq
@@ -533,101 +525,93 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movdqa %xmm0, %xmm8
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm9 = [9223372039002259456,9223372039002259456]
-; SSE41-NEXT:    movdqa %xmm2, %xmm10
-; SSE41-NEXT:    pxor %xmm9, %xmm10
-; SSE41-NEXT:    movdqa %xmm6, %xmm0
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
 ; SSE41-NEXT:    pxor %xmm9, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm11
-; SSE41-NEXT:    pcmpgtd %xmm10, %xmm11
-; SSE41-NEXT:    pshufd {{.*#+}} xmm12 = xmm11[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm12, %xmm0
-; SSE41-NEXT:    por %xmm11, %xmm0
+; SSE41-NEXT:    movdqa %xmm6, %xmm10
+; SSE41-NEXT:    pxor %xmm9, %xmm10
+; SSE41-NEXT:    movdqa %xmm10, %xmm11
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm11
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm10
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm10[0,0,2,2]
+; SSE41-NEXT:    pand %xmm11, %xmm0
+; SSE41-NEXT:    por %xmm10, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm6
 ; SSE41-NEXT:    movdqa %xmm8, %xmm0
 ; SSE41-NEXT:    pxor %xmm9, %xmm0
 ; SSE41-NEXT:    movdqa %xmm4, %xmm2
 ; SSE41-NEXT:    pxor %xmm9, %xmm2
 ; SSE41-NEXT:    movdqa %xmm2, %xmm10
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm10
-; SSE41-NEXT:    pshufd {{.*#+}} xmm11 = xmm10[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3]
-; SSE41-NEXT:    pand %xmm11, %xmm0
-; SSE41-NEXT:    por %xmm10, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm10
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,0,2,2]
+; SSE41-NEXT:    pand %xmm10, %xmm0
+; SSE41-NEXT:    por %xmm2, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm8, %xmm4
 ; SSE41-NEXT:    movdqa %xmm3, %xmm0
 ; SSE41-NEXT:    pxor %xmm9, %xmm0
 ; SSE41-NEXT:    movdqa %xmm7, %xmm2
 ; SSE41-NEXT:    pxor %xmm9, %xmm2
 ; SSE41-NEXT:    movdqa %xmm2, %xmm8
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm8
-; SSE41-NEXT:    pshufd {{.*#+}} xmm10 = xmm8[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3]
-; SSE41-NEXT:    pand %xmm10, %xmm0
-; SSE41-NEXT:    por %xmm8, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm8
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,0,2,2]
+; SSE41-NEXT:    pand %xmm8, %xmm0
+; SSE41-NEXT:    por %xmm2, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm7
 ; SSE41-NEXT:    movdqa %xmm1, %xmm0
 ; SSE41-NEXT:    pxor %xmm9, %xmm0
 ; SSE41-NEXT:    movdqa %xmm5, %xmm2
 ; SSE41-NEXT:    pxor %xmm9, %xmm2
 ; SSE41-NEXT:    movdqa %xmm2, %xmm3
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm8 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3]
-; SSE41-NEXT:    pand %xmm8, %xmm0
-; SSE41-NEXT:    por %xmm3, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm3
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,0,2,2]
+; SSE41-NEXT:    pand %xmm3, %xmm0
+; SSE41-NEXT:    por %xmm2, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm5
 ; SSE41-NEXT:    movapd %xmm5, %xmm0
 ; SSE41-NEXT:    xorpd %xmm9, %xmm0
 ; SSE41-NEXT:    movapd %xmm7, %xmm1
 ; SSE41-NEXT:    xorpd %xmm9, %xmm1
 ; SSE41-NEXT:    movapd %xmm1, %xmm2
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE41-NEXT:    pand %xmm3, %xmm0
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm2
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2]
+; SSE41-NEXT:    pand %xmm2, %xmm0
+; SSE41-NEXT:    por %xmm1, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm5, %xmm7
 ; SSE41-NEXT:    movapd %xmm4, %xmm0
 ; SSE41-NEXT:    xorpd %xmm9, %xmm0
 ; SSE41-NEXT:    movapd %xmm6, %xmm1
 ; SSE41-NEXT:    xorpd %xmm9, %xmm1
 ; SSE41-NEXT:    movapd %xmm1, %xmm2
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE41-NEXT:    pand %xmm3, %xmm0
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm2
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2]
+; SSE41-NEXT:    pand %xmm2, %xmm0
+; SSE41-NEXT:    por %xmm1, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm6
 ; SSE41-NEXT:    movapd %xmm6, %xmm0
 ; SSE41-NEXT:    xorpd %xmm9, %xmm0
 ; SSE41-NEXT:    movapd %xmm7, %xmm1
 ; SSE41-NEXT:    xorpd %xmm9, %xmm1
 ; SSE41-NEXT:    movapd %xmm1, %xmm2
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE41-NEXT:    pand %xmm3, %xmm0
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm2
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2]
+; SSE41-NEXT:    pand %xmm2, %xmm0
+; SSE41-NEXT:    por %xmm1, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm6, %xmm7
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm7[2,3,0,1]
 ; SSE41-NEXT:    movdqa %xmm7, %xmm0
 ; SSE41-NEXT:    pxor %xmm9, %xmm0
 ; SSE41-NEXT:    pxor %xmm1, %xmm9
 ; SSE41-NEXT:    movdqa %xmm9, %xmm2
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm9
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm9[1,1,3,3]
-; SSE41-NEXT:    pand %xmm3, %xmm0
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm2
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm9
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm9[0,0,2,2]
+; SSE41-NEXT:    pand %xmm2, %xmm0
+; SSE41-NEXT:    por %xmm9, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm7, %xmm1
 ; SSE41-NEXT:    movq %xmm1, %rax
 ; SSE41-NEXT:    retq
@@ -741,87 +725,38 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 define i32 @test_v2i32(<2 x i32> %a0) {
 ; SSE2-LABEL: test_v2i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [4294967295,0,4294967295,0]
-; SSE2-NEXT:    pand %xmm0, %xmm1
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm2[2],xmm0[3],xmm2[3]
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
-; SSE2-NEXT:    movdqa %xmm1, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm0, %xmm3
 ; SSE2-NEXT:    pxor %xmm2, %xmm3
-; SSE2-NEXT:    pxor %xmm0, %xmm2
-; SSE2-NEXT:    movdqa %xmm2, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT:    pand %xmm5, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm2, %xmm3
-; SSE2-NEXT:    pand %xmm3, %xmm1
-; SSE2-NEXT:    pandn %xmm0, %xmm3
-; SSE2-NEXT:    por %xmm1, %xmm3
-; SSE2-NEXT:    movd %xmm3, %eax
+; SSE2-NEXT:    pxor %xmm1, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm2
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    pandn %xmm1, %xmm2
+; SSE2-NEXT:    por %xmm0, %xmm2
+; SSE2-NEXT:    movd %xmm2, %eax
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: test_v2i32:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pxor %xmm1, %xmm1
-; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero
-; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [2147483648,2147483648]
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    pxor %xmm3, %xmm0
-; SSE41-NEXT:    pxor %xmm2, %xmm3
-; SSE41-NEXT:    movdqa %xmm3, %xmm4
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pand %xmm4, %xmm0
-; SSE41-NEXT:    por %xmm3, %xmm0
-; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
-; SSE41-NEXT:    movd %xmm2, %eax
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE41-NEXT:    pminud %xmm0, %xmm1
+; SSE41-NEXT:    movd %xmm1, %eax
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: test_v2i32:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
-; AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm0, %xmm1
-; AVX1-NEXT:    vblendvpd %xmm1, %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vmovd %xmm0, %eax
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: test_v2i32:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm2 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
-; AVX2-NEXT:    vpunpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX2-NEXT:    vpcmpgtq %xmm2, %xmm0, %xmm1
-; AVX2-NEXT:    vblendvpd %xmm1, %xmm2, %xmm0, %xmm0
-; AVX2-NEXT:    vmovd %xmm0, %eax
-; AVX2-NEXT:    retq
-;
-; AVX512BW-LABEL: test_v2i32:
-; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm2 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
-; AVX512BW-NEXT:    vpunpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX512BW-NEXT:    vpminuq %zmm0, %zmm2, %zmm0
-; AVX512BW-NEXT:    vmovd %xmm0, %eax
-; AVX512BW-NEXT:    vzeroupper
-; AVX512BW-NEXT:    retq
+; AVX-LABEL: test_v2i32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpminud %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vmovd %xmm0, %eax
+; AVX-NEXT:    retq
 ;
-; AVX512VL-LABEL: test_v2i32:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpblendd {{.*#+}} xmm1 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
-; AVX512VL-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX512VL-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
-; AVX512VL-NEXT:    vpminuq %xmm0, %xmm1, %xmm0
-; AVX512VL-NEXT:    vmovd %xmm0, %eax
-; AVX512VL-NEXT:    retq
+; AVX512-LABEL: test_v2i32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpminud %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vmovd %xmm0, %eax
+; AVX512-NEXT:    retq
   %1 = call i32 @llvm.experimental.vector.reduce.umin.v2i32(<2 x i32> %a0)
   ret i32 %1
 }
@@ -1223,85 +1158,41 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 define i16 @test_v2i16(<2 x i16> %a0) {
 ; SSE2-LABEL: test_v2i16:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [65535,0,0,0,65535,0,0,0]
-; SSE2-NEXT:    pand %xmm0, %xmm1
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
-; SSE2-NEXT:    movdqa %xmm1, %xmm3
-; SSE2-NEXT:    por %xmm2, %xmm3
-; SSE2-NEXT:    por %xmm0, %xmm2
-; SSE2-NEXT:    movdqa %xmm2, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT:    pand %xmm5, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm2, %xmm3
-; SSE2-NEXT:    pand %xmm3, %xmm1
-; SSE2-NEXT:    pandn %xmm0, %xmm3
-; SSE2-NEXT:    por %xmm1, %xmm3
-; SSE2-NEXT:    movd %xmm3, %eax
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psrld $16, %xmm1
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [32768,32768,32768,32768,32768,32768,32768,32768]
+; SSE2-NEXT:    pxor %xmm2, %xmm0
+; SSE2-NEXT:    pxor %xmm2, %xmm1
+; SSE2-NEXT:    pminsw %xmm0, %xmm1
+; SSE2-NEXT:    movd %xmm1, %eax
+; SSE2-NEXT:    xorl $32768, %eax # imm = 0x8000
 ; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: test_v2i16:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pxor %xmm1, %xmm1
-; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3],xmm0[4],xmm1[5,6,7]
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm2 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [2147483648,2147483648]
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    por %xmm3, %xmm0
-; SSE41-NEXT:    por %xmm2, %xmm3
-; SSE41-NEXT:    movdqa %xmm3, %xmm4
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pand %xmm4, %xmm0
-; SSE41-NEXT:    por %xmm3, %xmm0
-; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
-; SSE41-NEXT:    movd %xmm2, %eax
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    psrld $16, %xmm1
+; SSE41-NEXT:    pminuw %xmm0, %xmm1
+; SSE41-NEXT:    movd %xmm1, %eax
 ; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: test_v2i16:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3],xmm0[4],xmm1[5,6,7]
-; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; AVX-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm2
-; AVX-NEXT:    vblendvpd %xmm2, %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpsrld $16, %xmm0, %xmm1
+; AVX-NEXT:    vpminuw %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovd %xmm0, %eax
 ; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX-NEXT:    retq
 ;
-; AVX512BW-LABEL: test_v2i16:
-; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3],xmm0[4],xmm1[5,6,7]
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; AVX512BW-NEXT:    vpminuq %zmm0, %zmm1, %zmm0
-; AVX512BW-NEXT:    vmovd %xmm0, %eax
-; AVX512BW-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX512BW-NEXT:    vzeroupper
-; AVX512BW-NEXT:    retq
-;
-; AVX512VL-LABEL: test_v2i16:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3],xmm0[4],xmm1[5,6,7]
-; AVX512VL-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX512VL-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; AVX512VL-NEXT:    vpminuq %xmm0, %xmm1, %xmm0
-; AVX512VL-NEXT:    vmovd %xmm0, %eax
-; AVX512VL-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX512VL-NEXT:    retq
+; AVX512-LABEL: test_v2i16:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
+; AVX512-NEXT:    vpminuw %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vmovd %xmm0, %eax
+; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX512-NEXT:    retq
   %1 = call i16 @llvm.experimental.vector.reduce.umin.v2i16(<2 x i16> %a0)
   ret i16 %1
 }
@@ -1309,70 +1200,48 @@ define i16 @test_v2i16(<2 x i16> %a0) {
 define i16 @test_v4i16(<4 x i16> %a0) {
 ; SSE2-LABEL: test_v4i16:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [65535,0,65535,0,65535,0,65535,0]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
-; SSE2-NEXT:    pand %xmm1, %xmm0
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm2
-; SSE2-NEXT:    movdqa %xmm2, %xmm3
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSE2-NEXT:    pand %xmm3, %xmm0
-; SSE2-NEXT:    pandn %xmm2, %xmm3
-; SSE2-NEXT:    por %xmm0, %xmm3
-; SSE2-NEXT:    pand %xmm3, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,2,3]
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pandn %xmm0, %xmm2
-; SSE2-NEXT:    por %xmm1, %xmm2
-; SSE2-NEXT:    movd %xmm2, %eax
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [32768,32768,32768,32768,32768,32768,32768,32768]
+; SSE2-NEXT:    pxor %xmm2, %xmm0
+; SSE2-NEXT:    pxor %xmm2, %xmm1
+; SSE2-NEXT:    pminsw %xmm0, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    pxor %xmm2, %xmm0
+; SSE2-NEXT:    psrld $16, %xmm0
+; SSE2-NEXT:    pxor %xmm2, %xmm0
+; SSE2-NEXT:    pminsw %xmm1, %xmm0
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    xorl $32768, %eax # imm = 0x8000
 ; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: test_v4i16:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pxor %xmm1, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7]
-; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0],xmm1[1],xmm2[2],xmm1[3,4,5,6,7]
-; SSE41-NEXT:    pminud %xmm0, %xmm2
-; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2],xmm1[3],xmm2[4],xmm1[5],xmm2[6],xmm1[7]
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
-; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; SSE41-NEXT:    pminud %xmm1, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE41-NEXT:    pminuw %xmm0, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    psrld $16, %xmm0
+; SSE41-NEXT:    pminuw %xmm1, %xmm0
 ; SSE41-NEXT:    movd %xmm0, %eax
 ; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: test_v4i16:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX-NEXT:    vpblendw {{.*#+}} xmm2 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7]
-; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3,4,5,6,7]
-; AVX-NEXT:    vpminud %xmm0, %xmm2, %xmm0
-; AVX-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7]
-; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; AVX-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; AVX-NEXT:    vpminud %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpminuw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpsrld $16, %xmm0, %xmm1
+; AVX-NEXT:    vpminuw %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovd %xmm0, %eax
 ; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: test_v4i16:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm2 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7]
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3,4,5,6,7]
-; AVX512-NEXT:    vpminud %xmm0, %xmm2, %xmm0
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7]
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; AVX512-NEXT:    vpminud %xmm0, %xmm1, %xmm0
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpminuw %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
+; AVX512-NEXT:    vpminuw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
 ; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX512-NEXT:    retq
@@ -1683,83 +1552,37 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 define i8 @test_v2i8(<2 x i8> %a0) {
 ; SSE2-LABEL: test_v2i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [255,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0]
-; SSE2-NEXT:    pand %xmm0, %xmm1
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
-; SSE2-NEXT:    movdqa %xmm1, %xmm3
-; SSE2-NEXT:    por %xmm2, %xmm3
-; SSE2-NEXT:    por %xmm0, %xmm2
-; SSE2-NEXT:    movdqa %xmm2, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT:    pand %xmm5, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm2, %xmm3
-; SSE2-NEXT:    pand %xmm3, %xmm1
-; SSE2-NEXT:    pandn %xmm0, %xmm3
-; SSE2-NEXT:    por %xmm1, %xmm3
-; SSE2-NEXT:    movd %xmm3, %eax
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psrlw $8, %xmm1
+; SSE2-NEXT:    pminub %xmm0, %xmm1
+; SSE2-NEXT:    movd %xmm1, %eax
 ; SSE2-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: test_v2i8:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; SSE41-NEXT:    pand {{.*}}(%rip), %xmm1
-; SSE41-NEXT:    pmovzxbq {{.*#+}} xmm2 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
-; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [2147483648,2147483648]
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    por %xmm3, %xmm0
-; SSE41-NEXT:    por %xmm2, %xmm3
-; SSE41-NEXT:    movdqa %xmm3, %xmm4
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pand %xmm4, %xmm0
-; SSE41-NEXT:    por %xmm3, %xmm0
-; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
-; SSE41-NEXT:    pextrb $0, %xmm2, %eax
+; SSE41-NEXT:    psrlw $8, %xmm1
+; SSE41-NEXT:    pminub %xmm0, %xmm1
+; SSE41-NEXT:    pextrb $0, %xmm1, %eax
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: test_v2i8:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm1
-; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
-; AVX-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm2
-; AVX-NEXT:    vblendvpd %xmm2, %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpextrb $0, %xmm0, %eax
 ; AVX-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX-NEXT:    retq
 ;
-; AVX512BW-LABEL: test_v2i8:
-; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm1
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512BW-NEXT:    vpminuq %zmm0, %zmm1, %zmm0
-; AVX512BW-NEXT:    vpextrb $0, %xmm0, %eax
-; AVX512BW-NEXT:    # kill: def $al killed $al killed $eax
-; AVX512BW-NEXT:    vzeroupper
-; AVX512BW-NEXT:    retq
-;
-; AVX512VL-LABEL: test_v2i8:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm1
-; AVX512VL-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512VL-NEXT:    vpminuq %xmm0, %xmm1, %xmm0
-; AVX512VL-NEXT:    vpextrb $0, %xmm0, %eax
-; AVX512VL-NEXT:    # kill: def $al killed $al killed $eax
-; AVX512VL-NEXT:    retq
+; AVX512-LABEL: test_v2i8:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX512-NEXT:    vpminub %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX512-NEXT:    # kill: def $al killed $al killed $eax
+; AVX512-NEXT:    retq
   %1 = call i8 @llvm.experimental.vector.reduce.umin.v2i8(<2 x i8> %a0)
   ret i8 %1
 }
@@ -1767,83 +1590,44 @@ define i8 @test_v2i8(<2 x i8> %a0) {
 define i8 @test_v4i8(<4 x i8> %a0) {
 ; SSE2-LABEL: test_v4i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
-; SSE2-NEXT:    pand %xmm1, %xmm0
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm2
-; SSE2-NEXT:    movdqa %xmm2, %xmm3
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSE2-NEXT:    pand %xmm3, %xmm0
-; SSE2-NEXT:    pandn %xmm2, %xmm3
-; SSE2-NEXT:    por %xmm0, %xmm3
-; SSE2-NEXT:    pand %xmm3, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,2,3]
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pandn %xmm0, %xmm2
-; SSE2-NEXT:    por %xmm1, %xmm2
-; SSE2-NEXT:    movd %xmm2, %eax
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psrld $16, %xmm1
+; SSE2-NEXT:    pminub %xmm0, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    psrlw $8, %xmm0
+; SSE2-NEXT:    pminub %xmm1, %xmm0
+; SSE2-NEXT:    movd %xmm0, %eax
 ; SSE2-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: test_v4i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0]
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[8],zero,zero,zero,xmm0[12],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; SSE41-NEXT:    pminud %xmm2, %xmm0
-; SSE41-NEXT:    pand %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; SSE41-NEXT:    pmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
-; SSE41-NEXT:    pminud %xmm1, %xmm0
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    psrld $16, %xmm1
+; SSE41-NEXT:    pminub %xmm0, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    psrlw $8, %xmm0
+; SSE41-NEXT:    pminub %xmm1, %xmm0
 ; SSE41-NEXT:    pextrb $0, %xmm0, %eax
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: test_v4i8:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm1 = [3.57331108E-43,3.57331108E-43,3.57331108E-43,3.57331108E-43]
-; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm2
-; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[8],zero,zero,zero,xmm0[12],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX1-NEXT:    vpminud %xmm0, %xmm2, %xmm0
-; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; AVX1-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
-; AVX1-NEXT:    vpminud %xmm0, %xmm1, %xmm0
-; AVX1-NEXT:    vpextrb $0, %xmm0, %eax
-; AVX1-NEXT:    # kill: def $al killed $al killed $eax
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: test_v4i8:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [255,255,255,255]
-; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm2
-; AVX2-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[8],zero,zero,zero,xmm0[12],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX2-NEXT:    vpminud %xmm0, %xmm2, %xmm0
-; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm1
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
-; AVX2-NEXT:    vpminud %xmm0, %xmm1, %xmm0
-; AVX2-NEXT:    vpextrb $0, %xmm0, %eax
-; AVX2-NEXT:    # kill: def $al killed $al killed $eax
-; AVX2-NEXT:    retq
+; AVX-LABEL: test_v4i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpsrld $16, %xmm0, %xmm1
+; AVX-NEXT:    vpminub %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX-NEXT:    vpminub %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX-NEXT:    # kill: def $al killed $al killed $eax
+; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: test_v4i8:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [255,255,255,255]
-; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm2
-; AVX512-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[8],zero,zero,zero,xmm0[12],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT:    vpminud %xmm0, %xmm2, %xmm0
-; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm1
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT:    vpminud %xmm0, %xmm1, %xmm0
+; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
+; AVX512-NEXT:    vpminub %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX512-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
 ; AVX512-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512-NEXT:    retq
@@ -1854,92 +1638,52 @@ define i8 @test_v4i8(<4 x i8> %a0) {
 define i8 @test_v8i8(<8 x i8> %a0) {
 ; SSE2-LABEL: test_v8i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
-; SSE2-NEXT:    pand %xmm1, %xmm0
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm2
-; SSE2-NEXT:    pminsw %xmm0, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
-; SSE2-NEXT:    pand %xmm1, %xmm2
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
-; SSE2-NEXT:    pminsw %xmm2, %xmm0
-; SSE2-NEXT:    pand %xmm0, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE2-NEXT:    pminub %xmm0, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
 ; SSE2-NEXT:    psrld $16, %xmm0
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; SSE2-NEXT:    pminsw %xmm1, %xmm0
-; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    pminub %xmm1, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psrlw $8, %xmm1
+; SSE2-NEXT:    pminub %xmm0, %xmm1
+; SSE2-NEXT:    movd %xmm1, %eax
 ; SSE2-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: test_v8i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[8],zero,xmm0[10],zero,xmm0[12],zero,xmm0[14],zero,zero,zero,zero,zero,zero,zero,zero,zero
-; SSE41-NEXT:    pminuw %xmm2, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[4],zero,xmm0[6],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; SSE41-NEXT:    pminuw %xmm2, %xmm0
-; SSE41-NEXT:    pand %xmm0, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE41-NEXT:    pminub %xmm0, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
 ; SSE41-NEXT:    psrld $16, %xmm0
-; SSE41-NEXT:    pmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
-; SSE41-NEXT:    pminuw %xmm1, %xmm0
-; SSE41-NEXT:    pextrb $0, %xmm0, %eax
+; SSE41-NEXT:    pminub %xmm1, %xmm0
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    psrlw $8, %xmm1
+; SSE41-NEXT:    pminub %xmm0, %xmm1
+; SSE41-NEXT:    pextrb $0, %xmm1, %eax
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: test_v8i8:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
-; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm2
-; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[8],zero,xmm0[10],zero,xmm0[12],zero,xmm0[14],zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX1-NEXT:    vpminuw %xmm0, %xmm2, %xmm0
-; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm2
-; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[4],zero,xmm0[6],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX1-NEXT:    vpminuw %xmm0, %xmm2, %xmm0
-; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm0
-; AVX1-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
-; AVX1-NEXT:    vpminuw %xmm0, %xmm1, %xmm0
-; AVX1-NEXT:    vpextrb $0, %xmm0, %eax
-; AVX1-NEXT:    # kill: def $al killed $al killed $eax
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: test_v8i8:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpbroadcastw {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
-; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm2
-; AVX2-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[8],zero,xmm0[10],zero,xmm0[12],zero,xmm0[14],zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX2-NEXT:    vpminuw %xmm0, %xmm2, %xmm0
-; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm2
-; AVX2-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[4],zero,xmm0[6],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX2-NEXT:    vpminuw %xmm0, %xmm2, %xmm0
-; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm1
-; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm0
-; AVX2-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
-; AVX2-NEXT:    vpminuw %xmm0, %xmm1, %xmm0
-; AVX2-NEXT:    vpextrb $0, %xmm0, %eax
-; AVX2-NEXT:    # kill: def $al killed $al killed $eax
-; AVX2-NEXT:    retq
+; AVX-LABEL: test_v8i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpminub %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpsrld $16, %xmm0, %xmm1
+; AVX-NEXT:    vpminub %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX-NEXT:    vpminub %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX-NEXT:    # kill: def $al killed $al killed $eax
+; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: test_v8i8:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpbroadcastw {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
-; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm2
-; AVX512-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[8],zero,xmm0[10],zero,xmm0[12],zero,xmm0[14],zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT:    vpminuw %xmm0, %xmm2, %xmm0
-; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm2
-; AVX512-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[4],zero,xmm0[6],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT:    vpminuw %xmm0, %xmm2, %xmm0
-; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm1
-; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm0
-; AVX512-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT:    vpminuw %xmm0, %xmm1, %xmm0
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpminub %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
+; AVX512-NEXT:    vpminub %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX512-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
 ; AVX512-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/vector-reduce-xor-bool.ll b/llvm/test/CodeGen/X86/vector-reduce-xor-bool.ll
index eb0ba63fc..1dc839734 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-xor-bool.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-xor-bool.ll
@@ -118,17 +118,29 @@ define i1 @trunc_v4i32_v4i1(<4 x i32>) {
 }
 
 define i1 @trunc_v8i16_v8i1(<8 x i8>) {
-; SSE-LABEL: trunc_v8i16_v8i1:
-; SSE:       # %bb.0:
-; SSE-NEXT:    psllw $15, %xmm0
-; SSE-NEXT:    packsswb %xmm0, %xmm0
-; SSE-NEXT:    pmovmskb %xmm0, %eax
-; SSE-NEXT:    xorb $0, %al
-; SSE-NEXT:    setnp %al
-; SSE-NEXT:    retq
+; SSE2-LABEL: trunc_v8i16_v8i1:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    psllw $15, %xmm0
+; SSE2-NEXT:    packsswb %xmm0, %xmm0
+; SSE2-NEXT:    pmovmskb %xmm0, %eax
+; SSE2-NEXT:    xorb $0, %al
+; SSE2-NEXT:    setnp %al
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: trunc_v8i16_v8i1:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; SSE41-NEXT:    psllw $15, %xmm0
+; SSE41-NEXT:    packsswb %xmm0, %xmm0
+; SSE41-NEXT:    pmovmskb %xmm0, %eax
+; SSE41-NEXT:    xorb $0, %al
+; SSE41-NEXT:    setnp %al
+; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: trunc_v8i16_v8i1:
 ; AVX:       # %bb.0:
+; AVX-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; AVX-NEXT:    vpsllw $15, %xmm0, %xmm0
 ; AVX-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
 ; AVX-NEXT:    vpmovmskb %xmm0, %eax
@@ -138,9 +150,9 @@ define i1 @trunc_v8i16_v8i1(<8 x i8>) {
 ;
 ; AVX512F-LABEL: trunc_v8i16_v8i1:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpmovsxwq %xmm0, %zmm0
-; AVX512F-NEXT:    vpsllq $63, %zmm0, %zmm0
-; AVX512F-NEXT:    vptestmq %zmm0, %zmm0, %k0
+; AVX512F-NEXT:    vpmovsxbd %xmm0, %zmm0
+; AVX512F-NEXT:    vpslld $31, %zmm0, %zmm0
+; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k0
 ; AVX512F-NEXT:    kmovw %k0, %eax
 ; AVX512F-NEXT:    xorb $0, %al
 ; AVX512F-NEXT:    setnp %al
@@ -149,8 +161,8 @@ define i1 @trunc_v8i16_v8i1(<8 x i8>) {
 ;
 ; AVX512BW-LABEL: trunc_v8i16_v8i1:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpsllw $15, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpmovw2m %zmm0, %k0
+; AVX512BW-NEXT:    vpsllw $7, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpmovb2m %zmm0, %k0
 ; AVX512BW-NEXT:    kmovd %k0, %eax
 ; AVX512BW-NEXT:    xorb $0, %al
 ; AVX512BW-NEXT:    setnp %al
@@ -159,8 +171,8 @@ define i1 @trunc_v8i16_v8i1(<8 x i8>) {
 ;
 ; AVX512VL-LABEL: trunc_v8i16_v8i1:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsllw $15, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpmovw2m %xmm0, %k0
+; AVX512VL-NEXT:    vpsllw $7, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpmovb2m %xmm0, %k0
 ; AVX512VL-NEXT:    kmovd %k0, %eax
 ; AVX512VL-NEXT:    xorb $0, %al
 ; AVX512VL-NEXT:    setnp %al
@@ -355,7 +367,7 @@ define i1 @trunc_v8i32_v8i1(<8 x i32>) {
 define i1 @trunc_v16i16_v16i1(<16 x i16>) {
 ; SSE2-LABEL: trunc_v16i16_v16i1:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
 ; SSE2-NEXT:    pand %xmm2, %xmm1
 ; SSE2-NEXT:    pand %xmm2, %xmm0
 ; SSE2-NEXT:    packuswb %xmm1, %xmm0
@@ -767,7 +779,7 @@ define i1 @trunc_v16i32_v16i1(<16 x i32>) {
 define i1 @trunc_v32i16_v32i1(<32 x i16>) {
 ; SSE2-LABEL: trunc_v32i16_v32i1:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
 ; SSE2-NEXT:    pand %xmm4, %xmm3
 ; SSE2-NEXT:    pand %xmm4, %xmm2
 ; SSE2-NEXT:    packuswb %xmm3, %xmm2
@@ -839,6 +851,7 @@ define i1 @trunc_v32i16_v32i1(<32 x i16>) {
 ;
 ; AVX512F-LABEL: trunc_v32i16_v32i1:
 ; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512F-NEXT:    vpxor %ymm1, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpmovsxwd %ymm0, %zmm0
 ; AVX512F-NEXT:    vpslld $31, %zmm0, %zmm0
@@ -891,82 +904,51 @@ define i1 @trunc_v32i16_v32i1(<32 x i16>) {
 }
 
 define i1 @trunc_v64i8_v64i1(<64 x i8>) {
-; SSE2-LABEL: trunc_v64i8_v64i1:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    pxor %xmm3, %xmm1
-; SSE2-NEXT:    pxor %xmm2, %xmm1
-; SSE2-NEXT:    pxor %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
-; SSE2-NEXT:    pxor %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; SSE2-NEXT:    pxor %xmm0, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm0
-; SSE2-NEXT:    psrld $16, %xmm0
-; SSE2-NEXT:    pxor %xmm1, %xmm0
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    psrlw $8, %xmm1
-; SSE2-NEXT:    pxor %xmm0, %xmm1
-; SSE2-NEXT:    movd %xmm1, %eax
-; SSE2-NEXT:    # kill: def $al killed $al killed $eax
-; SSE2-NEXT:    retq
-;
-; SSE41-LABEL: trunc_v64i8_v64i1:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pxor %xmm3, %xmm1
-; SSE41-NEXT:    pxor %xmm2, %xmm1
-; SSE41-NEXT:    pxor %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
-; SSE41-NEXT:    pxor %xmm1, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; SSE41-NEXT:    pxor %xmm0, %xmm1
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    psrld $16, %xmm0
-; SSE41-NEXT:    pxor %xmm1, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    psrlw $8, %xmm1
-; SSE41-NEXT:    pxor %xmm0, %xmm1
-; SSE41-NEXT:    pextrb $0, %xmm1, %eax
-; SSE41-NEXT:    # kill: def $al killed $al killed $eax
-; SSE41-NEXT:    retq
+; SSE-LABEL: trunc_v64i8_v64i1:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pxor %xmm3, %xmm1
+; SSE-NEXT:    pxor %xmm2, %xmm1
+; SSE-NEXT:    pxor %xmm0, %xmm1
+; SSE-NEXT:    psllw $7, %xmm1
+; SSE-NEXT:    pmovmskb %xmm1, %eax
+; SSE-NEXT:    movl %eax, %ecx
+; SSE-NEXT:    shrl $8, %ecx
+; SSE-NEXT:    xorb %al, %cl
+; SSE-NEXT:    setnp %al
+; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: trunc_v64i8_v64i1:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpextrb $0, %xmm0, %eax
-; AVX1-NEXT:    # kill: def $al killed $al killed $eax
+; AVX1-NEXT:    vpsllw $7, %xmm0, %xmm0
+; AVX1-NEXT:    vpmovmskb %xmm0, %eax
+; AVX1-NEXT:    movl %eax, %ecx
+; AVX1-NEXT:    shrl $8, %ecx
+; AVX1-NEXT:    xorb %al, %cl
+; AVX1-NEXT:    setnp %al
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: trunc_v64i8_v64i1:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpextrb $0, %xmm0, %eax
-; AVX2-NEXT:    # kill: def $al killed $al killed $eax
+; AVX2-NEXT:    vpsllw $7, %ymm0, %ymm0
+; AVX2-NEXT:    vpmovmskb %ymm0, %eax
+; AVX2-NEXT:    movl %eax, %ecx
+; AVX2-NEXT:    shrl $16, %ecx
+; AVX2-NEXT:    xorl %eax, %ecx
+; AVX2-NEXT:    movl %ecx, %eax
+; AVX2-NEXT:    shrl $8, %eax
+; AVX2-NEXT:    xorb %cl, %al
+; AVX2-NEXT:    setnp %al
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: trunc_v64i8_v64i1:
 ; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm2
 ; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm3
 ; AVX512F-NEXT:    vpxor %xmm2, %xmm3, %xmm2
@@ -992,20 +974,17 @@ define i1 @trunc_v64i8_v64i1(<64 x i8>) {
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    vpsllw $7, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpmovb2m %zmm0, %k0
-; AVX512BW-NEXT:    kshiftrq $32, %k0, %k1
-; AVX512BW-NEXT:    kxorq %k1, %k0, %k0
-; AVX512BW-NEXT:    kshiftrq $16, %k0, %k1
-; AVX512BW-NEXT:    kxorq %k1, %k0, %k0
-; AVX512BW-NEXT:    kshiftrq $8, %k0, %k1
-; AVX512BW-NEXT:    kxorq %k1, %k0, %k0
-; AVX512BW-NEXT:    kshiftrq $4, %k0, %k1
-; AVX512BW-NEXT:    kxorq %k1, %k0, %k0
-; AVX512BW-NEXT:    kshiftrq $2, %k0, %k1
-; AVX512BW-NEXT:    kxorq %k1, %k0, %k0
-; AVX512BW-NEXT:    kshiftrq $1, %k0, %k1
-; AVX512BW-NEXT:    kxorq %k1, %k0, %k0
-; AVX512BW-NEXT:    kmovd %k0, %eax
-; AVX512BW-NEXT:    # kill: def $al killed $al killed $eax
+; AVX512BW-NEXT:    kmovq %k0, %rax
+; AVX512BW-NEXT:    movq %rax, %rcx
+; AVX512BW-NEXT:    shrq $32, %rcx
+; AVX512BW-NEXT:    xorl %eax, %ecx
+; AVX512BW-NEXT:    movl %ecx, %eax
+; AVX512BW-NEXT:    shrl $16, %eax
+; AVX512BW-NEXT:    xorl %ecx, %eax
+; AVX512BW-NEXT:    movl %eax, %ecx
+; AVX512BW-NEXT:    shrl $8, %ecx
+; AVX512BW-NEXT:    xorb %al, %cl
+; AVX512BW-NEXT:    setnp %al
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
@@ -1013,20 +992,17 @@ define i1 @trunc_v64i8_v64i1(<64 x i8>) {
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vpsllw $7, %zmm0, %zmm0
 ; AVX512VL-NEXT:    vpmovb2m %zmm0, %k0
-; AVX512VL-NEXT:    kshiftrq $32, %k0, %k1
-; AVX512VL-NEXT:    kxorq %k1, %k0, %k0
-; AVX512VL-NEXT:    kshiftrq $16, %k0, %k1
-; AVX512VL-NEXT:    kxorq %k1, %k0, %k0
-; AVX512VL-NEXT:    kshiftrq $8, %k0, %k1
-; AVX512VL-NEXT:    kxorq %k1, %k0, %k0
-; AVX512VL-NEXT:    kshiftrq $4, %k0, %k1
-; AVX512VL-NEXT:    kxorq %k1, %k0, %k0
-; AVX512VL-NEXT:    kshiftrq $2, %k0, %k1
-; AVX512VL-NEXT:    kxorq %k1, %k0, %k0
-; AVX512VL-NEXT:    kshiftrq $1, %k0, %k1
-; AVX512VL-NEXT:    kxorq %k1, %k0, %k0
-; AVX512VL-NEXT:    kmovd %k0, %eax
-; AVX512VL-NEXT:    # kill: def $al killed $al killed $eax
+; AVX512VL-NEXT:    kmovq %k0, %rax
+; AVX512VL-NEXT:    movq %rax, %rcx
+; AVX512VL-NEXT:    shrq $32, %rcx
+; AVX512VL-NEXT:    xorl %eax, %ecx
+; AVX512VL-NEXT:    movl %ecx, %eax
+; AVX512VL-NEXT:    shrl $16, %eax
+; AVX512VL-NEXT:    xorl %ecx, %eax
+; AVX512VL-NEXT:    movl %eax, %ecx
+; AVX512VL-NEXT:    shrl $8, %ecx
+; AVX512VL-NEXT:    xorb %al, %cl
+; AVX512VL-NEXT:    setnp %al
 ; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq
   %a = trunc <64 x i8> %0 to <64 x i1>
@@ -1158,22 +1134,33 @@ define i1 @icmp_v4i32_v4i1(<4 x i32>) {
 }
 
 define i1 @icmp_v8i16_v8i1(<8 x i8>) {
-; SSE-LABEL: icmp_v8i16_v8i1:
-; SSE:       # %bb.0:
-; SSE-NEXT:    pand {{.*}}(%rip), %xmm0
-; SSE-NEXT:    pxor %xmm1, %xmm1
-; SSE-NEXT:    pcmpeqw %xmm0, %xmm1
-; SSE-NEXT:    packsswb %xmm0, %xmm1
-; SSE-NEXT:    pmovmskb %xmm1, %eax
-; SSE-NEXT:    xorb $0, %al
-; SSE-NEXT:    setnp %al
-; SSE-NEXT:    retq
+; SSE2-LABEL: icmp_v8i16_v8i1:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    pcmpeqb %xmm0, %xmm1
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSE2-NEXT:    packsswb %xmm0, %xmm0
+; SSE2-NEXT:    pmovmskb %xmm0, %eax
+; SSE2-NEXT:    xorb $0, %al
+; SSE2-NEXT:    setnp %al
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: icmp_v8i16_v8i1:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pxor %xmm1, %xmm1
+; SSE41-NEXT:    pcmpeqb %xmm0, %xmm1
+; SSE41-NEXT:    pmovsxbw %xmm1, %xmm0
+; SSE41-NEXT:    packsswb %xmm0, %xmm0
+; SSE41-NEXT:    pmovmskb %xmm0, %eax
+; SSE41-NEXT:    xorb $0, %al
+; SSE41-NEXT:    setnp %al
+; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: icmp_v8i16_v8i1:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
 ; AVX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX-NEXT:    vpcmpeqw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpcmpeqb %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpmovsxbw %xmm0, %xmm0
 ; AVX-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
 ; AVX-NEXT:    vpmovmskb %xmm0, %eax
 ; AVX-NEXT:    xorb $0, %al
@@ -1182,11 +1169,10 @@ define i1 @icmp_v8i16_v8i1(<8 x i8>) {
 ;
 ; AVX512F-LABEL: icmp_v8i16_v8i1:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT:    vpcmpeqw %xmm1, %xmm0, %xmm0
-; AVX512F-NEXT:    vpmovsxwq %xmm0, %zmm0
-; AVX512F-NEXT:    vptestmq %zmm0, %zmm0, %k0
+; AVX512F-NEXT:    vpcmpeqb %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    vpmovsxbd %xmm0, %zmm0
+; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k0
 ; AVX512F-NEXT:    kmovw %k0, %eax
 ; AVX512F-NEXT:    xorb $0, %al
 ; AVX512F-NEXT:    setnp %al
@@ -1196,8 +1182,7 @@ define i1 @icmp_v8i16_v8i1(<8 x i8>) {
 ; AVX512BW-LABEL: icmp_v8i16_v8i1:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm1 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
-; AVX512BW-NEXT:    vptestnmw %zmm1, %zmm0, %k0
+; AVX512BW-NEXT:    vptestnmb %zmm0, %zmm0, %k0
 ; AVX512BW-NEXT:    kmovd %k0, %eax
 ; AVX512BW-NEXT:    xorb $0, %al
 ; AVX512BW-NEXT:    setnp %al
@@ -1206,7 +1191,7 @@ define i1 @icmp_v8i16_v8i1(<8 x i8>) {
 ;
 ; AVX512VL-LABEL: icmp_v8i16_v8i1:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vptestnmw {{.*}}(%rip), %xmm0, %k0
+; AVX512VL-NEXT:    vptestnmb %xmm0, %xmm0, %k0
 ; AVX512VL-NEXT:    kmovd %k0, %eax
 ; AVX512VL-NEXT:    xorb $0, %al
 ; AVX512VL-NEXT:    setnp %al
@@ -1867,6 +1852,7 @@ define i1 @icmp_v32i16_v32i1(<32 x i16>) {
 ;
 ; AVX512F-LABEL: icmp_v32i16_v32i1:
 ; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; AVX512F-NEXT:    vpcmpeqw %ymm2, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpcmpeqw %ymm2, %ymm0, %ymm0
@@ -1919,53 +1905,23 @@ define i1 @icmp_v32i16_v32i1(<32 x i16>) {
 }
 
 define i1 @icmp_v64i8_v64i1(<64 x i8>) {
-; SSE2-LABEL: icmp_v64i8_v64i1:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    pxor %xmm4, %xmm4
-; SSE2-NEXT:    pcmpeqb %xmm4, %xmm2
-; SSE2-NEXT:    pcmpeqb %xmm4, %xmm0
-; SSE2-NEXT:    pcmpeqb %xmm4, %xmm3
-; SSE2-NEXT:    pcmpeqb %xmm4, %xmm1
-; SSE2-NEXT:    pxor %xmm3, %xmm1
-; SSE2-NEXT:    pxor %xmm2, %xmm1
-; SSE2-NEXT:    pxor %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
-; SSE2-NEXT:    pxor %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; SSE2-NEXT:    pxor %xmm0, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm0
-; SSE2-NEXT:    psrld $16, %xmm0
-; SSE2-NEXT:    pxor %xmm1, %xmm0
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    psrlw $8, %xmm1
-; SSE2-NEXT:    pxor %xmm0, %xmm1
-; SSE2-NEXT:    movd %xmm1, %eax
-; SSE2-NEXT:    # kill: def $al killed $al killed $eax
-; SSE2-NEXT:    retq
-;
-; SSE41-LABEL: icmp_v64i8_v64i1:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pxor %xmm4, %xmm4
-; SSE41-NEXT:    pcmpeqb %xmm4, %xmm2
-; SSE41-NEXT:    pcmpeqb %xmm4, %xmm0
-; SSE41-NEXT:    pcmpeqb %xmm4, %xmm3
-; SSE41-NEXT:    pcmpeqb %xmm4, %xmm1
-; SSE41-NEXT:    pxor %xmm3, %xmm1
-; SSE41-NEXT:    pxor %xmm2, %xmm1
-; SSE41-NEXT:    pxor %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
-; SSE41-NEXT:    pxor %xmm1, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; SSE41-NEXT:    pxor %xmm0, %xmm1
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    psrld $16, %xmm0
-; SSE41-NEXT:    pxor %xmm1, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    psrlw $8, %xmm1
-; SSE41-NEXT:    pxor %xmm0, %xmm1
-; SSE41-NEXT:    pextrb $0, %xmm1, %eax
-; SSE41-NEXT:    # kill: def $al killed $al killed $eax
-; SSE41-NEXT:    retq
+; SSE-LABEL: icmp_v64i8_v64i1:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pxor %xmm4, %xmm4
+; SSE-NEXT:    pcmpeqb %xmm4, %xmm2
+; SSE-NEXT:    pcmpeqb %xmm4, %xmm0
+; SSE-NEXT:    pcmpeqb %xmm4, %xmm3
+; SSE-NEXT:    pcmpeqb %xmm4, %xmm1
+; SSE-NEXT:    pxor %xmm3, %xmm1
+; SSE-NEXT:    pxor %xmm2, %xmm1
+; SSE-NEXT:    pxor %xmm0, %xmm1
+; SSE-NEXT:    psllw $7, %xmm1
+; SSE-NEXT:    pmovmskb %xmm1, %eax
+; SSE-NEXT:    movl %eax, %ecx
+; SSE-NEXT:    shrl $8, %ecx
+; SSE-NEXT:    xorb %al, %cl
+; SSE-NEXT:    setnp %al
+; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: icmp_v64i8_v64i1:
 ; AVX1:       # %bb.0:
@@ -1979,16 +1935,12 @@ define i1 @icmp_v64i8_v64i1(<64 x i8>) {
 ; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpxor %xmm0, %xmm3, %xmm0
 ; AVX1-NEXT:    vpxor %xmm0, %xmm4, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpextrb $0, %xmm0, %eax
-; AVX1-NEXT:    # kill: def $al killed $al killed $eax
+; AVX1-NEXT:    vpsllw $7, %xmm0, %xmm0
+; AVX1-NEXT:    vpmovmskb %xmm0, %eax
+; AVX1-NEXT:    movl %eax, %ecx
+; AVX1-NEXT:    shrl $8, %ecx
+; AVX1-NEXT:    xorb %al, %cl
+; AVX1-NEXT:    setnp %al
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
@@ -1998,23 +1950,20 @@ define i1 @icmp_v64i8_v64i1(<64 x i8>) {
 ; AVX2-NEXT:    vpcmpeqb %ymm2, %ymm1, %ymm1
 ; AVX2-NEXT:    vpcmpeqb %ymm2, %ymm0, %ymm0
 ; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpextrb $0, %xmm0, %eax
-; AVX2-NEXT:    # kill: def $al killed $al killed $eax
+; AVX2-NEXT:    vpmovmskb %ymm0, %eax
+; AVX2-NEXT:    movl %eax, %ecx
+; AVX2-NEXT:    shrl $16, %ecx
+; AVX2-NEXT:    xorl %eax, %ecx
+; AVX2-NEXT:    movl %ecx, %eax
+; AVX2-NEXT:    shrl $8, %eax
+; AVX2-NEXT:    xorb %cl, %al
+; AVX2-NEXT:    setnp %al
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: icmp_v64i8_v64i1:
 ; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; AVX512F-NEXT:    vpcmpeqb %ymm2, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpcmpeqb %ymm2, %ymm0, %ymm0
@@ -2041,40 +1990,34 @@ define i1 @icmp_v64i8_v64i1(<64 x i8>) {
 ; AVX512BW-LABEL: icmp_v64i8_v64i1:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    vptestnmb %zmm0, %zmm0, %k0
-; AVX512BW-NEXT:    kshiftrq $32, %k0, %k1
-; AVX512BW-NEXT:    kxorq %k1, %k0, %k0
-; AVX512BW-NEXT:    kshiftrq $16, %k0, %k1
-; AVX512BW-NEXT:    kxorq %k1, %k0, %k0
-; AVX512BW-NEXT:    kshiftrq $8, %k0, %k1
-; AVX512BW-NEXT:    kxorq %k1, %k0, %k0
-; AVX512BW-NEXT:    kshiftrq $4, %k0, %k1
-; AVX512BW-NEXT:    kxorq %k1, %k0, %k0
-; AVX512BW-NEXT:    kshiftrq $2, %k0, %k1
-; AVX512BW-NEXT:    kxorq %k1, %k0, %k0
-; AVX512BW-NEXT:    kshiftrq $1, %k0, %k1
-; AVX512BW-NEXT:    kxorq %k1, %k0, %k0
-; AVX512BW-NEXT:    kmovd %k0, %eax
-; AVX512BW-NEXT:    # kill: def $al killed $al killed $eax
+; AVX512BW-NEXT:    kmovq %k0, %rax
+; AVX512BW-NEXT:    movq %rax, %rcx
+; AVX512BW-NEXT:    shrq $32, %rcx
+; AVX512BW-NEXT:    xorl %eax, %ecx
+; AVX512BW-NEXT:    movl %ecx, %eax
+; AVX512BW-NEXT:    shrl $16, %eax
+; AVX512BW-NEXT:    xorl %ecx, %eax
+; AVX512BW-NEXT:    movl %eax, %ecx
+; AVX512BW-NEXT:    shrl $8, %ecx
+; AVX512BW-NEXT:    xorb %al, %cl
+; AVX512BW-NEXT:    setnp %al
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512VL-LABEL: icmp_v64i8_v64i1:
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vptestnmb %zmm0, %zmm0, %k0
-; AVX512VL-NEXT:    kshiftrq $32, %k0, %k1
-; AVX512VL-NEXT:    kxorq %k1, %k0, %k0
-; AVX512VL-NEXT:    kshiftrq $16, %k0, %k1
-; AVX512VL-NEXT:    kxorq %k1, %k0, %k0
-; AVX512VL-NEXT:    kshiftrq $8, %k0, %k1
-; AVX512VL-NEXT:    kxorq %k1, %k0, %k0
-; AVX512VL-NEXT:    kshiftrq $4, %k0, %k1
-; AVX512VL-NEXT:    kxorq %k1, %k0, %k0
-; AVX512VL-NEXT:    kshiftrq $2, %k0, %k1
-; AVX512VL-NEXT:    kxorq %k1, %k0, %k0
-; AVX512VL-NEXT:    kshiftrq $1, %k0, %k1
-; AVX512VL-NEXT:    kxorq %k1, %k0, %k0
-; AVX512VL-NEXT:    kmovd %k0, %eax
-; AVX512VL-NEXT:    # kill: def $al killed $al killed $eax
+; AVX512VL-NEXT:    kmovq %k0, %rax
+; AVX512VL-NEXT:    movq %rax, %rcx
+; AVX512VL-NEXT:    shrq $32, %rcx
+; AVX512VL-NEXT:    xorl %eax, %ecx
+; AVX512VL-NEXT:    movl %ecx, %eax
+; AVX512VL-NEXT:    shrl $16, %eax
+; AVX512VL-NEXT:    xorl %ecx, %eax
+; AVX512VL-NEXT:    movl %eax, %ecx
+; AVX512VL-NEXT:    shrl $8, %ecx
+; AVX512VL-NEXT:    xorb %al, %cl
+; AVX512VL-NEXT:    setnp %al
 ; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq
   %a = icmp eq <64 x i8> %0, zeroinitializer
diff --git a/llvm/test/CodeGen/X86/vector-reduce-xor.ll b/llvm/test/CodeGen/X86/vector-reduce-xor.ll
index 7d590cd80..812259a33 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-xor.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-xor.ll
@@ -182,14 +182,14 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 define i32 @test_v2i32(<2 x i32> %a0) {
 ; SSE-LABEL: test_v2i32:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE-NEXT:    pxor %xmm0, %xmm1
 ; SSE-NEXT:    movd %xmm1, %eax
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: test_v2i32:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovd %xmm0, %eax
 ; AVX-NEXT:    retq
@@ -397,7 +397,8 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 define i16 @test_v2i16(<2 x i16> %a0) {
 ; SSE-LABEL: test_v2i16:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE-NEXT:    movdqa %xmm0, %xmm1
+; SSE-NEXT:    psrld $16, %xmm1
 ; SSE-NEXT:    pxor %xmm0, %xmm1
 ; SSE-NEXT:    movd %xmm1, %eax
 ; SSE-NEXT:    # kill: def $ax killed $ax killed $eax
@@ -405,7 +406,7 @@ define i16 @test_v2i16(<2 x i16> %a0) {
 ;
 ; AVX-LABEL: test_v2i16:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovd %xmm0, %eax
 ; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
@@ -417,9 +418,10 @@ define i16 @test_v2i16(<2 x i16> %a0) {
 define i16 @test_v4i16(<4 x i16> %a0) {
 ; SSE-LABEL: test_v4i16:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE-NEXT:    pxor %xmm0, %xmm1
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE-NEXT:    movdqa %xmm1, %xmm0
+; SSE-NEXT:    psrld $16, %xmm0
 ; SSE-NEXT:    pxor %xmm1, %xmm0
 ; SSE-NEXT:    movd %xmm0, %eax
 ; SSE-NEXT:    # kill: def $ax killed $ax killed $eax
@@ -427,10 +429,10 @@ define i16 @test_v4i16(<4 x i16> %a0) {
 ;
 ; AVX-LABEL: test_v4i16:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpsrld $16, %xmm0, %xmm1
+; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovd %xmm0, %eax
 ; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX-NEXT:    retq
@@ -684,7 +686,8 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 define i8 @test_v2i8(<2 x i8> %a0) {
 ; SSE2-LABEL: test_v2i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psrlw $8, %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm1
 ; SSE2-NEXT:    movd %xmm1, %eax
 ; SSE2-NEXT:    # kill: def $al killed $al killed $eax
@@ -692,7 +695,8 @@ define i8 @test_v2i8(<2 x i8> %a0) {
 ;
 ; SSE41-LABEL: test_v2i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    psrlw $8, %xmm1
 ; SSE41-NEXT:    pxor %xmm0, %xmm1
 ; SSE41-NEXT:    pextrb $0, %xmm1, %eax
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
@@ -700,7 +704,7 @@ define i8 @test_v2i8(<2 x i8> %a0) {
 ;
 ; AVX-LABEL: test_v2i8:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpextrb $0, %xmm0, %eax
 ; AVX-NEXT:    # kill: def $al killed $al killed $eax
@@ -712,9 +716,11 @@ define i8 @test_v2i8(<2 x i8> %a0) {
 define i8 @test_v4i8(<4 x i8> %a0) {
 ; SSE2-LABEL: test_v4i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psrld $16, %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    psrlw $8, %xmm0
 ; SSE2-NEXT:    pxor %xmm1, %xmm0
 ; SSE2-NEXT:    movd %xmm0, %eax
 ; SSE2-NEXT:    # kill: def $al killed $al killed $eax
@@ -722,9 +728,11 @@ define i8 @test_v4i8(<4 x i8> %a0) {
 ;
 ; SSE41-LABEL: test_v4i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    psrld $16, %xmm1
 ; SSE41-NEXT:    pxor %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    psrlw $8, %xmm0
 ; SSE41-NEXT:    pxor %xmm1, %xmm0
 ; SSE41-NEXT:    pextrb $0, %xmm0, %eax
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
@@ -732,9 +740,9 @@ define i8 @test_v4i8(<4 x i8> %a0) {
 ;
 ; AVX-LABEL: test_v4i8:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpextrb $0, %xmm0, %eax
 ; AVX-NEXT:    # kill: def $al killed $al killed $eax
@@ -746,12 +754,13 @@ define i8 @test_v4i8(<4 x i8> %a0) {
 define i8 @test_v8i8(<8 x i8> %a0) {
 ; SSE2-LABEL: test_v8i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE2-NEXT:    pxor %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    psrld $16, %xmm0
 ; SSE2-NEXT:    pxor %xmm1, %xmm0
 ; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    psrld $16, %xmm1
+; SSE2-NEXT:    psrlw $8, %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm1
 ; SSE2-NEXT:    movd %xmm1, %eax
 ; SSE2-NEXT:    # kill: def $al killed $al killed $eax
@@ -759,12 +768,13 @@ define i8 @test_v8i8(<8 x i8> %a0) {
 ;
 ; SSE41-LABEL: test_v8i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE41-NEXT:    pxor %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    psrld $16, %xmm0
 ; SSE41-NEXT:    pxor %xmm1, %xmm0
 ; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    psrld $16, %xmm1
+; SSE41-NEXT:    psrlw $8, %xmm1
 ; SSE41-NEXT:    pxor %xmm0, %xmm1
 ; SSE41-NEXT:    pextrb $0, %xmm1, %eax
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
@@ -772,12 +782,12 @@ define i8 @test_v8i8(<8 x i8> %a0) {
 ;
 ; AVX-LABEL: test_v8i8:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpextrb $0, %xmm0, %eax
 ; AVX-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/vector-rem.ll b/llvm/test/CodeGen/X86/vector-rem.ll
index 3f57bd833..15e119869 100644
--- a/llvm/test/CodeGen/X86/vector-rem.ll
+++ b/llvm/test/CodeGen/X86/vector-rem.ll
@@ -81,30 +81,30 @@ define <4 x float> @qux(<4 x float> %t, <4 x float> %u) nounwind {
 ; CHECK-LABEL: qux:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    subq $72, %rsp
-; CHECK-NEXT:    movaps %xmm1, {{[0-9]+}}(%rsp) # 16-byte Spill
-; CHECK-NEXT:    movaps %xmm0, {{[0-9]+}}(%rsp) # 16-byte Spill
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; CHECK-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
 ; CHECK-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,1,2,3]
 ; CHECK-NEXT:    callq fmodf
 ; CHECK-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
-; CHECK-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
 ; CHECK-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
-; CHECK-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
 ; CHECK-NEXT:    movhlps {{.*#+}} xmm1 = xmm1[1,1]
 ; CHECK-NEXT:    callq fmodf
 ; CHECK-NEXT:    unpcklps (%rsp), %xmm0 # 16-byte Folded Reload
 ; CHECK-NEXT:    # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
 ; CHECK-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
-; CHECK-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm0 # 16-byte Reload
-; CHECK-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
 ; CHECK-NEXT:    callq fmodf
-; CHECK-NEXT:    movaps %xmm0, {{[0-9]+}}(%rsp) # 16-byte Spill
-; CHECK-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
 ; CHECK-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; CHECK-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
 ; CHECK-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,2,3]
 ; CHECK-NEXT:    callq fmodf
-; CHECK-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
 ; CHECK-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
 ; CHECK-NEXT:    unpcklpd (%rsp), %xmm1 # 16-byte Folded Reload
 ; CHECK-NEXT:    # xmm1 = xmm1[0],mem[0]
diff --git a/llvm/test/CodeGen/X86/vector-rotate-128.ll b/llvm/test/CodeGen/X86/vector-rotate-128.ll
index 4c528a552..35dbb1099 100644
--- a/llvm/test/CodeGen/X86/vector-rotate-128.ll
+++ b/llvm/test/CodeGen/X86/vector-rotate-128.ll
@@ -262,24 +262,23 @@ define <8 x i16> @var_rotate_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind {
 ; SSE2-LABEL: var_rotate_v8i16:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    movdqa %xmm1, %xmm3
-; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
-; SSE2-NEXT:    pslld $23, %xmm3
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [1065353216,1065353216,1065353216,1065353216]
-; SSE2-NEXT:    paddd %xmm4, %xmm3
-; SSE2-NEXT:    cvttps2dq %xmm3, %xmm3
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm3[0,2,2,3,4,5,6,7]
-; SSE2-NEXT:    pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,6,6,7]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE2-NEXT:    pslld $23, %xmm2
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [1065353216,1065353216,1065353216,1065353216]
+; SSE2-NEXT:    paddd %xmm3, %xmm2
+; SSE2-NEXT:    cvttps2dq %xmm2, %xmm2
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,2,2,3,4,5,6,7]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,6,6,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
 ; SSE2-NEXT:    pslld $23, %xmm1
-; SSE2-NEXT:    paddd %xmm4, %xmm1
+; SSE2-NEXT:    paddd %xmm3, %xmm1
 ; SSE2-NEXT:    cvttps2dq %xmm1, %xmm1
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
 ; SSE2-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,6,6,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm2
 ; SSE2-NEXT:    pmulhuw %xmm1, %xmm2
 ; SSE2-NEXT:    pmullw %xmm1, %xmm0
@@ -325,20 +324,19 @@ define <8 x i16> @var_rotate_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind {
 ;
 ; AVX2-LABEL: var_rotate_v8i16:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; AVX2-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
-; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
-; AVX2-NEXT:    vpsllvd %ymm2, %ymm0, %ymm2
-; AVX2-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; AVX2-NEXT:    vpshufb %ymm3, %ymm2, %ymm2
-; AVX2-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,2,3]
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm4 = [16,16,16,16,16,16,16,16]
-; AVX2-NEXT:    vpsubw %xmm1, %xmm4, %xmm1
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [16,16,16,16,16,16,16,16]
+; AVX2-NEXT:    vpsubw %xmm1, %xmm2, %xmm2
+; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
+; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2-NEXT:    vpsrlvd %ymm2, %ymm0, %ymm2
+; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm3
+; AVX2-NEXT:    vpackusdw %xmm3, %xmm2, %xmm2
 ; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
-; AVX2-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpshufb %ymm3, %ymm0, %ymm0
+; AVX2-NEXT:    vpsllvd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-NEXT:    vpor %xmm0, %xmm2, %xmm0
+; AVX2-NEXT:    vpor %xmm2, %xmm0, %xmm0
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
@@ -403,24 +401,23 @@ define <8 x i16> @var_rotate_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind {
 ; X32-SSE-LABEL: var_rotate_v8i16:
 ; X32-SSE:       # %bb.0:
 ; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm1
-; X32-SSE-NEXT:    pxor %xmm2, %xmm2
-; X32-SSE-NEXT:    movdqa %xmm1, %xmm3
-; X32-SSE-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
-; X32-SSE-NEXT:    pslld $23, %xmm3
-; X32-SSE-NEXT:    movdqa {{.*#+}} xmm4 = [1065353216,1065353216,1065353216,1065353216]
-; X32-SSE-NEXT:    paddd %xmm4, %xmm3
-; X32-SSE-NEXT:    cvttps2dq %xmm3, %xmm3
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm3 = xmm3[0,2,2,3,4,5,6,7]
-; X32-SSE-NEXT:    pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,6,6,7]
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
-; X32-SSE-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
+; X32-SSE-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; X32-SSE-NEXT:    pslld $23, %xmm2
+; X32-SSE-NEXT:    movdqa {{.*#+}} xmm3 = [1065353216,1065353216,1065353216,1065353216]
+; X32-SSE-NEXT:    paddd %xmm3, %xmm2
+; X32-SSE-NEXT:    cvttps2dq %xmm2, %xmm2
+; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,2,2,3,4,5,6,7]
+; X32-SSE-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,6,6,7]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; X32-SSE-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
 ; X32-SSE-NEXT:    pslld $23, %xmm1
-; X32-SSE-NEXT:    paddd %xmm4, %xmm1
+; X32-SSE-NEXT:    paddd %xmm3, %xmm1
 ; X32-SSE-NEXT:    cvttps2dq %xmm1, %xmm1
 ; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
 ; X32-SSE-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,6,6,7]
 ; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; X32-SSE-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; X32-SSE-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
 ; X32-SSE-NEXT:    movdqa %xmm0, %xmm2
 ; X32-SSE-NEXT:    pmulhuw %xmm1, %xmm2
 ; X32-SSE-NEXT:    pmullw %xmm1, %xmm0
@@ -1803,14 +1800,37 @@ define <16 x i8> @splatconstant_rotate_v16i8(<16 x i8> %a) nounwind {
 ; AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
-; AVX512-LABEL: splatconstant_rotate_v16i8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsllw $4, %xmm0, %xmm1
-; AVX512-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
-; AVX512-NEXT:    vpsrlw $4, %xmm0, %xmm0
-; AVX512-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX512-NEXT:    vpor %xmm0, %xmm1, %xmm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: splatconstant_rotate_v16i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsllw $4, %xmm0, %xmm1
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512F-NEXT:    vpsrlw $4, %xmm0, %xmm0
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX512F-NEXT:    vpor %xmm0, %xmm1, %xmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: splatconstant_rotate_v16i8:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsllw $4, %xmm0, %xmm1
+; AVX512VL-NEXT:    vpsrlw $4, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpternlogq $216, {{.*}}(%rip), %xmm1, %xmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: splatconstant_rotate_v16i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsllw $4, %xmm0, %xmm1
+; AVX512BW-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512BW-NEXT:    vpsrlw $4, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX512BW-NEXT:    vpor %xmm0, %xmm1, %xmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VLBW-LABEL: splatconstant_rotate_v16i8:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpsllw $4, %xmm0, %xmm1
+; AVX512VLBW-NEXT:    vpsrlw $4, %xmm0, %xmm0
+; AVX512VLBW-NEXT:    vpternlogq $216, {{.*}}(%rip), %xmm1, %xmm0
+; AVX512VLBW-NEXT:    retq
 ;
 ; XOP-LABEL: splatconstant_rotate_v16i8:
 ; XOP:       # %bb.0:
@@ -2066,3 +2086,146 @@ define <16 x i8> @splatconstant_rotate_mask_v16i8(<16 x i8> %a) nounwind {
   %or = or <16 x i8> %lmask, %rmask
   ret <16 x i8> %or
 }
+
+define <4 x i32> @rot16_demandedbits(<4 x i32> %x, <4 x i32> %y) nounwind {
+; X32-LABEL: rot16_demandedbits:
+; X32:       # %bb.0:
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    shrl $11, %ecx
+; X32-NEXT:    shll $5, %eax
+; X32-NEXT:    orl %ecx, %eax
+; X32-NEXT:    andl $65536, %eax # imm = 0x10000
+; X32-NEXT:    retl
+;
+; X64-LABEL: rot16_demandedbits:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    movl %edi, %ecx
+; X64-NEXT:    shrl $11, %ecx
+; X64-NEXT:    shll $5, %eax
+; X64-NEXT:    orl %ecx, %eax
+; X64-NEXT:    andl $65536, %eax # imm = 0x10000
+; X64-NEXT:    retq
+; SSE2-LABEL: rot16_demandedbits:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psrld $11, %xmm1
+; SSE2-NEXT:    pslld $11, %xmm0
+; SSE2-NEXT:    por %xmm1, %xmm0
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: rot16_demandedbits:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    psrld $11, %xmm1
+; SSE41-NEXT:    pslld $11, %xmm0
+; SSE41-NEXT:    por %xmm1, %xmm0
+; SSE41-NEXT:    pxor %xmm1, %xmm1
+; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7]
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: rot16_demandedbits:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpsrld $11, %xmm0, %xmm1
+; AVX-NEXT:    vpslld $11, %xmm0, %xmm0
+; AVX-NEXT:    vpor %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7]
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: rot16_demandedbits:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpsrld $11, %xmm0, %xmm1
+; AVX512-NEXT:    vpslld $11, %xmm0, %xmm0
+; AVX512-NEXT:    vpor %xmm0, %xmm1, %xmm0
+; AVX512-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7]
+; AVX512-NEXT:    retq
+;
+; XOP-LABEL: rot16_demandedbits:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vpsrld $11, %xmm0, %xmm1
+; XOP-NEXT:    vpslld $11, %xmm0, %xmm0
+; XOP-NEXT:    vpor %xmm0, %xmm1, %xmm0
+; XOP-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; XOP-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7]
+; XOP-NEXT:    retq
+;
+; X32-SSE-LABEL: rot16_demandedbits:
+; X32-SSE:       # %bb.0:
+; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
+; X32-SSE-NEXT:    psrld $11, %xmm1
+; X32-SSE-NEXT:    pslld $11, %xmm0
+; X32-SSE-NEXT:    por %xmm1, %xmm0
+; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X32-SSE-NEXT:    retl
+  %t0 = lshr <4 x i32> %x, <i32 11, i32 11, i32 11, i32 11>
+  %t1 = shl <4 x i32> %x, <i32 11, i32 11, i32 11, i32 11>
+  %t2 = or <4 x i32> %t0, %t1
+  %t3 = and <4 x i32> %t2, <i32 65535, i32 65535, i32 65535, i32 65535>
+  ret <4 x i32> %t3
+}
+
+define <4 x i16> @rot16_trunc(<4 x i32> %x, <4 x i32> %y) nounwind {
+; SSE2-LABEL: rot16_trunc:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psrld $11, %xmm1
+; SSE2-NEXT:    pslld $5, %xmm0
+; SSE2-NEXT:    por %xmm1, %xmm0
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,6,6,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: rot16_trunc:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    psrld $11, %xmm1
+; SSE41-NEXT:    pslld $5, %xmm0
+; SSE41-NEXT:    por %xmm1, %xmm0
+; SSE41-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: rot16_trunc:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpsrld $11, %xmm0, %xmm1
+; AVX-NEXT:    vpslld $5, %xmm0, %xmm0
+; AVX-NEXT:    vpor %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: rot16_trunc:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpsrld $11, %xmm0, %xmm1
+; AVX512-NEXT:    vpslld $5, %xmm0, %xmm0
+; AVX512-NEXT:    vpor %xmm0, %xmm1, %xmm0
+; AVX512-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
+; AVX512-NEXT:    retq
+;
+; XOP-LABEL: rot16_trunc:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vpsrld $11, %xmm0, %xmm1
+; XOP-NEXT:    vpslld $5, %xmm0, %xmm0
+; XOP-NEXT:    vpor %xmm0, %xmm1, %xmm0
+; XOP-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
+; XOP-NEXT:    retq
+;
+; X32-SSE-LABEL: rot16_trunc:
+; X32-SSE:       # %bb.0:
+; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
+; X32-SSE-NEXT:    psrld $11, %xmm1
+; X32-SSE-NEXT:    pslld $5, %xmm0
+; X32-SSE-NEXT:    por %xmm1, %xmm0
+; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; X32-SSE-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,6,6,7]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X32-SSE-NEXT:    retl
+  %t0 = lshr <4 x i32> %x, <i32 11, i32 11, i32 11, i32 11>
+  %t1 = shl <4 x i32> %x, <i32 5, i32 5, i32 5, i32 5>
+  %t2 = or <4 x i32> %t0, %t1
+  %t3 = trunc <4 x i32> %t2 to <4 x i16>
+  ret <4 x i16> %t3
+}
diff --git a/llvm/test/CodeGen/X86/vector-rotate-256.ll b/llvm/test/CodeGen/X86/vector-rotate-256.ll
index 606067f9e..33f14d7e2 100644
--- a/llvm/test/CodeGen/X86/vector-rotate-256.ll
+++ b/llvm/test/CodeGen/X86/vector-rotate-256.ll
@@ -203,29 +203,28 @@ define <16 x i16> @var_rotate_v16i16(<16 x i16> %a, <16 x i16> %b) nounwind {
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15]
 ; AVX1-NEXT:    vpand %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7]
-; AVX1-NEXT:    vpslld $23, %xmm5, %xmm5
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [1065353216,1065353216,1065353216,1065353216]
-; AVX1-NEXT:    vpaddd %xmm6, %xmm5, %xmm5
-; AVX1-NEXT:    vcvttps2dq %xmm5, %xmm5
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; AVX1-NEXT:    vpslld $23, %xmm4, %xmm4
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm5 = [1065353216,1065353216,1065353216,1065353216]
+; AVX1-NEXT:    vpaddd %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vcvttps2dq %xmm4, %xmm4
 ; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
 ; AVX1-NEXT:    vpslld $23, %xmm2, %xmm2
-; AVX1-NEXT:    vpaddd %xmm6, %xmm2, %xmm2
+; AVX1-NEXT:    vpaddd %xmm5, %xmm2, %xmm2
 ; AVX1-NEXT:    vcvttps2dq %xmm2, %xmm2
-; AVX1-NEXT:    vpackusdw %xmm5, %xmm2, %xmm2
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm5
-; AVX1-NEXT:    vpmulhuw %xmm2, %xmm5, %xmm7
-; AVX1-NEXT:    vpmullw %xmm2, %xmm5, %xmm2
-; AVX1-NEXT:    vpor %xmm7, %xmm2, %xmm2
+; AVX1-NEXT:    vpackusdw %xmm4, %xmm2, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
+; AVX1-NEXT:    vpmulhuw %xmm2, %xmm4, %xmm6
+; AVX1-NEXT:    vpmullw %xmm2, %xmm4, %xmm2
+; AVX1-NEXT:    vpor %xmm6, %xmm2, %xmm2
 ; AVX1-NEXT:    vpand %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm1[4],xmm4[4],xmm1[5],xmm4[5],xmm1[6],xmm4[6],xmm1[7],xmm4[7]
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
 ; AVX1-NEXT:    vpslld $23, %xmm3, %xmm3
-; AVX1-NEXT:    vpaddd %xmm6, %xmm3, %xmm3
+; AVX1-NEXT:    vpaddd %xmm5, %xmm3, %xmm3
 ; AVX1-NEXT:    vcvttps2dq %xmm3, %xmm3
 ; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
 ; AVX1-NEXT:    vpslld $23, %xmm1, %xmm1
-; AVX1-NEXT:    vpaddd %xmm6, %xmm1, %xmm1
+; AVX1-NEXT:    vpaddd %xmm5, %xmm1, %xmm1
 ; AVX1-NEXT:    vcvttps2dq %xmm1, %xmm1
 ; AVX1-NEXT:    vpackusdw %xmm3, %xmm1, %xmm1
 ; AVX1-NEXT:    vpmulhuw %xmm1, %xmm0, %xmm3
@@ -355,7 +354,7 @@ define <32 x i8> @var_rotate_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
 ; AVX1-NEXT:    vpblendvb %xmm5, %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vpsrlw $7, %xmm2, %xmm3
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm8 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; AVX1-NEXT:    vpand %xmm8, %xmm3, %xmm3
+; AVX1-NEXT:    vpand %xmm3, %xmm8, %xmm3
 ; AVX1-NEXT:    vpaddb %xmm2, %xmm2, %xmm7
 ; AVX1-NEXT:    vpor %xmm3, %xmm7, %xmm3
 ; AVX1-NEXT:    vpaddb %xmm5, %xmm5, %xmm5
@@ -375,7 +374,7 @@ define <32 x i8> @var_rotate_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
 ; AVX1-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
 ; AVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrlw $7, %xmm0, %xmm3
-; AVX1-NEXT:    vpand %xmm8, %xmm3, %xmm3
+; AVX1-NEXT:    vpand %xmm3, %xmm8, %xmm3
 ; AVX1-NEXT:    vpaddb %xmm0, %xmm0, %xmm4
 ; AVX1-NEXT:    vpor %xmm3, %xmm4, %xmm3
 ; AVX1-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
@@ -433,20 +432,16 @@ define <32 x i8> @var_rotate_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
 ;
 ; AVX512VL-LABEL: var_rotate_v32i8:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm2
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
-; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm3
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm3, %ymm3
-; AVX512VL-NEXT:    vpor %ymm2, %ymm3, %ymm2
+; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm2
+; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm3
+; AVX512VL-NEXT:    vpternlogq $216, {{.*}}(%rip), %ymm2, %ymm3
 ; AVX512VL-NEXT:    vpsllw $5, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpsrlw $6, %ymm0, %ymm2
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
-; AVX512VL-NEXT:    vpsllw $2, %ymm0, %ymm3
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm3, %ymm3
-; AVX512VL-NEXT:    vpor %ymm2, %ymm3, %ymm2
+; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpsllw $2, %ymm0, %ymm2
+; AVX512VL-NEXT:    vpsrlw $6, %ymm0, %ymm3
+; AVX512VL-NEXT:    vpternlogq $216, {{.*}}(%rip), %ymm2, %ymm3
 ; AVX512VL-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vpaddb %ymm0, %ymm0, %ymm2
 ; AVX512VL-NEXT:    vpsrlw $7, %ymm0, %ymm3
 ; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm3, %ymm3
@@ -1139,7 +1134,7 @@ define <32 x i8> @constant_rotate_v32i8(<32 x i8> %a) nounwind {
 ; AVX1-NEXT:    vpxor %xmm8, %xmm8, %xmm8
 ; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm1[8],xmm8[8],xmm1[9],xmm8[9],xmm1[10],xmm8[10],xmm1[11],xmm8[11],xmm1[12],xmm8[12],xmm1[13],xmm8[13],xmm1[14],xmm8[14],xmm1[15],xmm8[15]
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm9 = [256,128,64,32,16,8,4,2]
-; AVX1-NEXT:    vpmullw %xmm9, %xmm3, %xmm3
+; AVX1-NEXT:    vpmullw %xmm3, %xmm9, %xmm3
 ; AVX1-NEXT:    vpsrlw $8, %xmm3, %xmm3
 ; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm5 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [256,2,4,8,16,32,64,128]
@@ -1157,7 +1152,7 @@ define <32 x i8> @constant_rotate_v32i8(<32 x i8> %a) nounwind {
 ; AVX1-NEXT:    vpackuswb %xmm1, %xmm5, %xmm1
 ; AVX1-NEXT:    vpor %xmm3, %xmm1, %xmm1
 ; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm0[8],xmm8[8],xmm0[9],xmm8[9],xmm0[10],xmm8[10],xmm0[11],xmm8[11],xmm0[12],xmm8[12],xmm0[13],xmm8[13],xmm0[14],xmm8[14],xmm0[15],xmm8[15]
-; AVX1-NEXT:    vpmullw %xmm9, %xmm3, %xmm3
+; AVX1-NEXT:    vpmullw %xmm3, %xmm9, %xmm3
 ; AVX1-NEXT:    vpsrlw $8, %xmm3, %xmm3
 ; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm5 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; AVX1-NEXT:    vpmullw %xmm6, %xmm5, %xmm6
@@ -1510,11 +1505,9 @@ define <32 x i8> @splatconstant_rotate_v32i8(<32 x i8> %a) nounwind {
 ;
 ; AVX512VL-LABEL: splatconstant_rotate_v32i8:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm1
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
-; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
-; AVX512VL-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm1
+; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpternlogq $216, {{.*}}(%rip), %ymm1, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatconstant_rotate_v32i8:
@@ -1529,10 +1522,8 @@ define <32 x i8> @splatconstant_rotate_v32i8(<32 x i8> %a) nounwind {
 ; AVX512VLBW-LABEL: splatconstant_rotate_v32i8:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpsllw $4, %ymm0, %ymm1
-; AVX512VLBW-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
 ; AVX512VLBW-NEXT:    vpsrlw $4, %ymm0, %ymm0
-; AVX512VLBW-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
-; AVX512VLBW-NEXT:    vpor %ymm0, %ymm1, %ymm0
+; AVX512VLBW-NEXT:    vpternlogq $216, {{.*}}(%rip), %ymm1, %ymm0
 ; AVX512VLBW-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: splatconstant_rotate_v32i8:
@@ -1796,11 +1787,9 @@ define <32 x i8> @splatconstant_rotate_mask_v32i8(<32 x i8> %a) nounwind {
 ;
 ; AVX512VL-LABEL: splatconstant_rotate_mask_v32i8:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm1
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
-; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
-; AVX512VL-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm1
+; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpternlogq $216, {{.*}}(%rip), %ymm1, %ymm0
 ; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/vector-rotate-512.ll b/llvm/test/CodeGen/X86/vector-rotate-512.ll
index b7793ed6c..7a210264f 100644
--- a/llvm/test/CodeGen/X86/vector-rotate-512.ll
+++ b/llvm/test/CodeGen/X86/vector-rotate-512.ll
@@ -35,50 +35,56 @@ define <16 x i32> @var_rotate_v16i32(<16 x i32> %a, <16 x i32> %b) nounwind {
 define <32 x i16> @var_rotate_v32i16(<32 x i16> %a, <32 x i16> %b) nounwind {
 ; AVX512F-LABEL: var_rotate_v32i16:
 ; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512F-NEXT:    vpand %ymm4, %ymm2, %ymm2
-; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm5 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
-; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
-; AVX512F-NEXT:    vpsllvd %zmm5, %zmm0, %zmm5
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm6 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
-; AVX512F-NEXT:    vpsubw %ymm2, %ymm6, %ymm2
+; AVX512F-NEXT:    vpand %ymm4, %ymm3, %ymm3
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm5 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
 ; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
-; AVX512F-NEXT:    vpsrlvd %zmm2, %zmm0, %zmm0
-; AVX512F-NEXT:    vpord %zmm0, %zmm5, %zmm0
-; AVX512F-NEXT:    vpmovdw %zmm0, %ymm0
-; AVX512F-NEXT:    vpand %ymm4, %ymm3, %ymm2
-; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm3 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
+; AVX512F-NEXT:    vpsllvd %zmm5, %zmm2, %zmm5
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm6 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
+; AVX512F-NEXT:    vpsubw %ymm3, %ymm6, %ymm3
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm3 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
+; AVX512F-NEXT:    vpsrlvd %zmm3, %zmm2, %zmm2
+; AVX512F-NEXT:    vpord %zmm2, %zmm5, %zmm2
+; AVX512F-NEXT:    vpmovdw %zmm2, %ymm2
+; AVX512F-NEXT:    vpand %ymm4, %ymm1, %ymm1
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm3 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
+; AVX512F-NEXT:    vpsllvd %zmm3, %zmm0, %zmm3
+; AVX512F-NEXT:    vpsubw %ymm1, %ymm6, %ymm1
 ; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
-; AVX512F-NEXT:    vpsllvd %zmm3, %zmm1, %zmm3
-; AVX512F-NEXT:    vpsubw %ymm2, %ymm6, %ymm2
-; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
-; AVX512F-NEXT:    vpsrlvd %zmm2, %zmm1, %zmm1
-; AVX512F-NEXT:    vpord %zmm1, %zmm3, %zmm1
-; AVX512F-NEXT:    vpmovdw %zmm1, %ymm1
+; AVX512F-NEXT:    vpsrlvd %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vpord %zmm0, %zmm3, %zmm0
+; AVX512F-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: var_rotate_v32i16:
 ; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512VL-NEXT:    vpand %ymm4, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm5 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
-; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
-; AVX512VL-NEXT:    vpsllvd %zmm5, %zmm0, %zmm5
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm6 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
-; AVX512VL-NEXT:    vpsubw %ymm2, %ymm6, %ymm2
+; AVX512VL-NEXT:    vpand %ymm4, %ymm3, %ymm3
+; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm5 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
 ; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
-; AVX512VL-NEXT:    vpsrlvd %zmm2, %zmm0, %zmm0
-; AVX512VL-NEXT:    vpord %zmm0, %zmm5, %zmm0
-; AVX512VL-NEXT:    vpmovdw %zmm0, %ymm0
-; AVX512VL-NEXT:    vpand %ymm4, %ymm3, %ymm2
-; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm3 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
+; AVX512VL-NEXT:    vpsllvd %zmm5, %zmm2, %zmm5
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm6 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
+; AVX512VL-NEXT:    vpsubw %ymm3, %ymm6, %ymm3
+; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm3 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
+; AVX512VL-NEXT:    vpsrlvd %zmm3, %zmm2, %zmm2
+; AVX512VL-NEXT:    vpord %zmm2, %zmm5, %zmm2
+; AVX512VL-NEXT:    vpmovdw %zmm2, %ymm2
+; AVX512VL-NEXT:    vpand %ymm4, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm3 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
+; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
+; AVX512VL-NEXT:    vpsllvd %zmm3, %zmm0, %zmm3
+; AVX512VL-NEXT:    vpsubw %ymm1, %ymm6, %ymm1
 ; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
-; AVX512VL-NEXT:    vpsllvd %zmm3, %zmm1, %zmm3
-; AVX512VL-NEXT:    vpsubw %ymm2, %ymm6, %ymm2
-; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
-; AVX512VL-NEXT:    vpsrlvd %zmm2, %zmm1, %zmm1
-; AVX512VL-NEXT:    vpord %zmm1, %zmm3, %zmm1
-; AVX512VL-NEXT:    vpmovdw %zmm1, %ymm1
+; AVX512VL-NEXT:    vpsrlvd %zmm1, %zmm0, %zmm0
+; AVX512VL-NEXT:    vpord %zmm0, %zmm3, %zmm0
+; AVX512VL-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: var_rotate_v32i16:
@@ -108,96 +114,94 @@ define <32 x i16> @var_rotate_v32i16(<32 x i16> %a, <32 x i16> %b) nounwind {
 define <64 x i8> @var_rotate_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
 ; AVX512F-LABEL: var_rotate_v64i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpsrlw $4, %ymm0, %ymm4
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
+; AVX512F-NEXT:    vpsrlw $4, %ymm3, %ymm4
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm5 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
 ; AVX512F-NEXT:    vpandn %ymm4, %ymm5, %ymm4
-; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm6
+; AVX512F-NEXT:    vpsllw $4, %ymm3, %ymm6
 ; AVX512F-NEXT:    vpand %ymm5, %ymm6, %ymm6
 ; AVX512F-NEXT:    vpor %ymm4, %ymm6, %ymm4
 ; AVX512F-NEXT:    vpsllw $5, %ymm2, %ymm2
-; AVX512F-NEXT:    vpblendvb %ymm2, %ymm4, %ymm0, %ymm0
-; AVX512F-NEXT:    vpsrlw $6, %ymm0, %ymm4
+; AVX512F-NEXT:    vpblendvb %ymm2, %ymm4, %ymm3, %ymm3
+; AVX512F-NEXT:    vpsrlw $6, %ymm3, %ymm4
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm6 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
 ; AVX512F-NEXT:    vpandn %ymm4, %ymm6, %ymm4
-; AVX512F-NEXT:    vpsllw $2, %ymm0, %ymm7
+; AVX512F-NEXT:    vpsllw $2, %ymm3, %ymm7
 ; AVX512F-NEXT:    vpand %ymm6, %ymm7, %ymm7
 ; AVX512F-NEXT:    vpor %ymm4, %ymm7, %ymm4
 ; AVX512F-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
-; AVX512F-NEXT:    vpblendvb %ymm2, %ymm4, %ymm0, %ymm0
-; AVX512F-NEXT:    vpsrlw $7, %ymm0, %ymm4
+; AVX512F-NEXT:    vpblendvb %ymm2, %ymm4, %ymm3, %ymm3
+; AVX512F-NEXT:    vpsrlw $7, %ymm3, %ymm4
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm7 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
 ; AVX512F-NEXT:    vpand %ymm7, %ymm4, %ymm4
-; AVX512F-NEXT:    vpaddb %ymm0, %ymm0, %ymm8
+; AVX512F-NEXT:    vpaddb %ymm3, %ymm3, %ymm8
 ; AVX512F-NEXT:    vpor %ymm4, %ymm8, %ymm4
 ; AVX512F-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
-; AVX512F-NEXT:    vpblendvb %ymm2, %ymm4, %ymm0, %ymm0
-; AVX512F-NEXT:    vpsrlw $4, %ymm1, %ymm2
-; AVX512F-NEXT:    vpandn %ymm2, %ymm5, %ymm2
-; AVX512F-NEXT:    vpsllw $4, %ymm1, %ymm4
+; AVX512F-NEXT:    vpblendvb %ymm2, %ymm4, %ymm3, %ymm2
+; AVX512F-NEXT:    vpsrlw $4, %ymm0, %ymm3
+; AVX512F-NEXT:    vpandn %ymm3, %ymm5, %ymm3
+; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm4
 ; AVX512F-NEXT:    vpand %ymm5, %ymm4, %ymm4
-; AVX512F-NEXT:    vpor %ymm2, %ymm4, %ymm2
-; AVX512F-NEXT:    vpsllw $5, %ymm3, %ymm3
-; AVX512F-NEXT:    vpblendvb %ymm3, %ymm2, %ymm1, %ymm1
-; AVX512F-NEXT:    vpsrlw $6, %ymm1, %ymm2
-; AVX512F-NEXT:    vpandn %ymm2, %ymm6, %ymm2
-; AVX512F-NEXT:    vpsllw $2, %ymm1, %ymm4
+; AVX512F-NEXT:    vpor %ymm3, %ymm4, %ymm3
+; AVX512F-NEXT:    vpsllw $5, %ymm1, %ymm1
+; AVX512F-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
+; AVX512F-NEXT:    vpsrlw $6, %ymm0, %ymm3
+; AVX512F-NEXT:    vpandn %ymm3, %ymm6, %ymm3
+; AVX512F-NEXT:    vpsllw $2, %ymm0, %ymm4
 ; AVX512F-NEXT:    vpand %ymm6, %ymm4, %ymm4
-; AVX512F-NEXT:    vpor %ymm2, %ymm4, %ymm2
-; AVX512F-NEXT:    vpaddb %ymm3, %ymm3, %ymm3
-; AVX512F-NEXT:    vpblendvb %ymm3, %ymm2, %ymm1, %ymm1
-; AVX512F-NEXT:    vpsrlw $7, %ymm1, %ymm2
-; AVX512F-NEXT:    vpand %ymm7, %ymm2, %ymm2
-; AVX512F-NEXT:    vpaddb %ymm1, %ymm1, %ymm4
-; AVX512F-NEXT:    vpor %ymm2, %ymm4, %ymm2
-; AVX512F-NEXT:    vpaddb %ymm3, %ymm3, %ymm3
-; AVX512F-NEXT:    vpblendvb %ymm3, %ymm2, %ymm1, %ymm1
+; AVX512F-NEXT:    vpor %ymm3, %ymm4, %ymm3
+; AVX512F-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
+; AVX512F-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
+; AVX512F-NEXT:    vpsrlw $7, %ymm0, %ymm3
+; AVX512F-NEXT:    vpand %ymm7, %ymm3, %ymm3
+; AVX512F-NEXT:    vpaddb %ymm0, %ymm0, %ymm4
+; AVX512F-NEXT:    vpor %ymm3, %ymm4, %ymm3
+; AVX512F-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
+; AVX512F-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: var_rotate_v64i8:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm4
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm5 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX512VL-NEXT:    vpandn %ymm4, %ymm5, %ymm4
-; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm6
-; AVX512VL-NEXT:    vpand %ymm5, %ymm6, %ymm6
-; AVX512VL-NEXT:    vpor %ymm4, %ymm6, %ymm4
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
+; AVX512VL-NEXT:    vpsrlw $4, %ymm3, %ymm4
+; AVX512VL-NEXT:    vpsllw $4, %ymm3, %ymm5
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm6 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
+; AVX512VL-NEXT:    vpternlogq $226, %ymm4, %ymm6, %ymm5
 ; AVX512VL-NEXT:    vpsllw $5, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm4, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpsrlw $6, %ymm0, %ymm4
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm6 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
-; AVX512VL-NEXT:    vpandn %ymm4, %ymm6, %ymm4
-; AVX512VL-NEXT:    vpsllw $2, %ymm0, %ymm7
-; AVX512VL-NEXT:    vpand %ymm6, %ymm7, %ymm7
-; AVX512VL-NEXT:    vpor %ymm4, %ymm7, %ymm4
+; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm5, %ymm3, %ymm3
+; AVX512VL-NEXT:    vpsrlw $6, %ymm3, %ymm4
+; AVX512VL-NEXT:    vpsllw $2, %ymm3, %ymm5
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm7 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
+; AVX512VL-NEXT:    vpternlogq $226, %ymm4, %ymm7, %ymm5
 ; AVX512VL-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm4, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpsrlw $7, %ymm0, %ymm4
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm7 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; AVX512VL-NEXT:    vpand %ymm7, %ymm4, %ymm4
-; AVX512VL-NEXT:    vpaddb %ymm0, %ymm0, %ymm8
+; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm5, %ymm3, %ymm3
+; AVX512VL-NEXT:    vpsrlw $7, %ymm3, %ymm4
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm5 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; AVX512VL-NEXT:    vpand %ymm5, %ymm4, %ymm4
+; AVX512VL-NEXT:    vpaddb %ymm3, %ymm3, %ymm8
 ; AVX512VL-NEXT:    vpor %ymm4, %ymm8, %ymm4
 ; AVX512VL-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm4, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpsrlw $4, %ymm1, %ymm2
-; AVX512VL-NEXT:    vpandn %ymm2, %ymm5, %ymm2
-; AVX512VL-NEXT:    vpsllw $4, %ymm1, %ymm4
-; AVX512VL-NEXT:    vpand %ymm5, %ymm4, %ymm4
-; AVX512VL-NEXT:    vpor %ymm2, %ymm4, %ymm2
-; AVX512VL-NEXT:    vpsllw $5, %ymm3, %ymm3
-; AVX512VL-NEXT:    vpblendvb %ymm3, %ymm2, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpsrlw $6, %ymm1, %ymm2
-; AVX512VL-NEXT:    vpandn %ymm2, %ymm6, %ymm2
-; AVX512VL-NEXT:    vpsllw $2, %ymm1, %ymm4
-; AVX512VL-NEXT:    vpand %ymm6, %ymm4, %ymm4
-; AVX512VL-NEXT:    vpor %ymm2, %ymm4, %ymm2
-; AVX512VL-NEXT:    vpaddb %ymm3, %ymm3, %ymm3
-; AVX512VL-NEXT:    vpblendvb %ymm3, %ymm2, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpsrlw $7, %ymm1, %ymm2
-; AVX512VL-NEXT:    vpand %ymm7, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpaddb %ymm1, %ymm1, %ymm4
-; AVX512VL-NEXT:    vpor %ymm2, %ymm4, %ymm2
-; AVX512VL-NEXT:    vpaddb %ymm3, %ymm3, %ymm3
-; AVX512VL-NEXT:    vpblendvb %ymm3, %ymm2, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm4, %ymm3, %ymm2
+; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm3
+; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm4
+; AVX512VL-NEXT:    vpternlogq $226, %ymm3, %ymm6, %ymm4
+; AVX512VL-NEXT:    vpsllw $5, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm4, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpsrlw $6, %ymm0, %ymm3
+; AVX512VL-NEXT:    vpsllw $2, %ymm0, %ymm4
+; AVX512VL-NEXT:    vpternlogq $226, %ymm3, %ymm7, %ymm4
+; AVX512VL-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm4, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpsrlw $7, %ymm0, %ymm3
+; AVX512VL-NEXT:    vpand %ymm5, %ymm3, %ymm3
+; AVX512VL-NEXT:    vpaddb %ymm0, %ymm0, %ymm4
+; AVX512VL-NEXT:    vpor %ymm3, %ymm4, %ymm3
+; AVX512VL-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: var_rotate_v64i8:
@@ -311,34 +315,38 @@ define <16 x i32> @splatvar_rotate_v16i32(<16 x i32> %a, <16 x i32> %b) nounwind
 define <32 x i16> @splatvar_rotate_v32i16(<32 x i16> %a, <32 x i16> %b) nounwind {
 ; AVX512F-LABEL: splatvar_rotate_v32i16:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpbroadcastw %xmm2, %xmm2
-; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
-; AVX512F-NEXT:    vpmovzxwq {{.*#+}} xmm3 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
-; AVX512F-NEXT:    vpsllw %xmm3, %ymm0, %ymm4
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512F-NEXT:    vpbroadcastw %xmm1, %xmm1
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512F-NEXT:    vpmovzxwq {{.*#+}} xmm3 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512F-NEXT:    vpsllw %xmm3, %ymm2, %ymm4
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm5 = [16,16,16,16,16,16,16,16]
-; AVX512F-NEXT:    vpsubw %xmm2, %xmm5, %xmm2
-; AVX512F-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
-; AVX512F-NEXT:    vpsrlw %xmm2, %ymm0, %ymm0
-; AVX512F-NEXT:    vpor %ymm0, %ymm4, %ymm0
-; AVX512F-NEXT:    vpsllw %xmm3, %ymm1, %ymm3
-; AVX512F-NEXT:    vpsrlw %xmm2, %ymm1, %ymm1
-; AVX512F-NEXT:    vpor %ymm1, %ymm3, %ymm1
+; AVX512F-NEXT:    vpsubw %xmm1, %xmm5, %xmm1
+; AVX512F-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512F-NEXT:    vpsrlw %xmm1, %ymm2, %ymm2
+; AVX512F-NEXT:    vpor %ymm2, %ymm4, %ymm2
+; AVX512F-NEXT:    vpsllw %xmm3, %ymm0, %ymm3
+; AVX512F-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vpor %ymm0, %ymm3, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatvar_rotate_v32i16:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpbroadcastw %xmm2, %xmm2
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
-; AVX512VL-NEXT:    vpmovzxwq {{.*#+}} xmm3 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
-; AVX512VL-NEXT:    vpsllw %xmm3, %ymm0, %ymm4
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512VL-NEXT:    vpbroadcastw %xmm1, %xmm1
+; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512VL-NEXT:    vpmovzxwq {{.*#+}} xmm3 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512VL-NEXT:    vpsllw %xmm3, %ymm2, %ymm4
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm5 = [16,16,16,16,16,16,16,16]
-; AVX512VL-NEXT:    vpsubw %xmm2, %xmm5, %xmm2
-; AVX512VL-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
-; AVX512VL-NEXT:    vpsrlw %xmm2, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpor %ymm0, %ymm4, %ymm0
-; AVX512VL-NEXT:    vpsllw %xmm3, %ymm1, %ymm3
-; AVX512VL-NEXT:    vpsrlw %xmm2, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpor %ymm1, %ymm3, %ymm1
+; AVX512VL-NEXT:    vpsubw %xmm1, %xmm5, %xmm1
+; AVX512VL-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512VL-NEXT:    vpsrlw %xmm1, %ymm2, %ymm2
+; AVX512VL-NEXT:    vpor %ymm2, %ymm4, %ymm2
+; AVX512VL-NEXT:    vpsllw %xmm3, %ymm0, %ymm3
+; AVX512VL-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpor %ymm0, %ymm3, %ymm0
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatvar_rotate_v32i16:
@@ -373,54 +381,58 @@ define <32 x i16> @splatvar_rotate_v32i16(<32 x i16> %a, <32 x i16> %b) nounwind
 define <64 x i8> @splatvar_rotate_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
 ; AVX512F-LABEL: splatvar_rotate_v64i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpbroadcastb %xmm2, %xmm2
-; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
-; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512F-NEXT:    vpsllw %xmm3, %ymm0, %ymm4
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512F-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512F-NEXT:    vpsllw %xmm3, %ymm2, %ymm4
 ; AVX512F-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
 ; AVX512F-NEXT:    vpsllw %xmm3, %xmm5, %xmm6
 ; AVX512F-NEXT:    vpbroadcastb %xmm6, %ymm6
 ; AVX512F-NEXT:    vpand %ymm6, %ymm4, %ymm4
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm7 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; AVX512F-NEXT:    vpsubb %xmm2, %xmm7, %xmm2
-; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512F-NEXT:    vpsrlw %xmm2, %ymm0, %ymm0
-; AVX512F-NEXT:    vpsrlw %xmm2, %xmm5, %xmm5
+; AVX512F-NEXT:    vpsubb %xmm1, %xmm7, %xmm1
+; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512F-NEXT:    vpsrlw %xmm1, %ymm2, %ymm2
+; AVX512F-NEXT:    vpsrlw %xmm1, %xmm5, %xmm5
 ; AVX512F-NEXT:    vpsrlw $8, %xmm5, %xmm5
 ; AVX512F-NEXT:    vpbroadcastb %xmm5, %ymm5
-; AVX512F-NEXT:    vpand %ymm5, %ymm0, %ymm0
-; AVX512F-NEXT:    vpor %ymm0, %ymm4, %ymm0
-; AVX512F-NEXT:    vpsllw %xmm3, %ymm1, %ymm3
+; AVX512F-NEXT:    vpand %ymm5, %ymm2, %ymm2
+; AVX512F-NEXT:    vpor %ymm2, %ymm4, %ymm2
+; AVX512F-NEXT:    vpsllw %xmm3, %ymm0, %ymm3
 ; AVX512F-NEXT:    vpand %ymm6, %ymm3, %ymm3
-; AVX512F-NEXT:    vpsrlw %xmm2, %ymm1, %ymm1
-; AVX512F-NEXT:    vpand %ymm5, %ymm1, %ymm1
-; AVX512F-NEXT:    vpor %ymm1, %ymm3, %ymm1
+; AVX512F-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vpand %ymm5, %ymm0, %ymm0
+; AVX512F-NEXT:    vpor %ymm0, %ymm3, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatvar_rotate_v64i8:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpbroadcastb %xmm2, %xmm2
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
-; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512VL-NEXT:    vpsllw %xmm3, %ymm0, %ymm4
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512VL-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VL-NEXT:    vpsllw %xmm3, %ymm2, %ymm4
 ; AVX512VL-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
 ; AVX512VL-NEXT:    vpsllw %xmm3, %xmm5, %xmm6
 ; AVX512VL-NEXT:    vpbroadcastb %xmm6, %ymm6
 ; AVX512VL-NEXT:    vpand %ymm6, %ymm4, %ymm4
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm7 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; AVX512VL-NEXT:    vpsubb %xmm2, %xmm7, %xmm2
-; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512VL-NEXT:    vpsrlw %xmm2, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpsrlw %xmm2, %xmm5, %xmm5
+; AVX512VL-NEXT:    vpsubb %xmm1, %xmm7, %xmm1
+; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VL-NEXT:    vpsrlw %xmm1, %ymm2, %ymm2
+; AVX512VL-NEXT:    vpsrlw %xmm1, %xmm5, %xmm5
 ; AVX512VL-NEXT:    vpsrlw $8, %xmm5, %xmm5
 ; AVX512VL-NEXT:    vpbroadcastb %xmm5, %ymm5
-; AVX512VL-NEXT:    vpand %ymm5, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpor %ymm0, %ymm4, %ymm0
-; AVX512VL-NEXT:    vpsllw %xmm3, %ymm1, %ymm3
+; AVX512VL-NEXT:    vpand %ymm5, %ymm2, %ymm2
+; AVX512VL-NEXT:    vpor %ymm2, %ymm4, %ymm2
+; AVX512VL-NEXT:    vpsllw %xmm3, %ymm0, %ymm3
 ; AVX512VL-NEXT:    vpand %ymm6, %ymm3, %ymm3
-; AVX512VL-NEXT:    vpsrlw %xmm2, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpand %ymm5, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpor %ymm1, %ymm3, %ymm1
+; AVX512VL-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpand %ymm5, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpor %ymm0, %ymm3, %ymm0
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatvar_rotate_v64i8:
@@ -497,24 +509,28 @@ define <16 x i32> @constant_rotate_v16i32(<16 x i32> %a) nounwind {
 define <32 x i16> @constant_rotate_v32i16(<32 x i16> %a) nounwind {
 ; AVX512F-LABEL: constant_rotate_v32i16:
 ; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm2 = [1,2,4,8,16,32,64,128,256,512,1024,2048,4096,8192,16384,32768]
-; AVX512F-NEXT:    vpmulhuw %ymm2, %ymm0, %ymm3
-; AVX512F-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
-; AVX512F-NEXT:    vpor %ymm3, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpmulhuw %ymm2, %ymm1, %ymm3
 ; AVX512F-NEXT:    vpmullw %ymm2, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpor %ymm3, %ymm1, %ymm1
+; AVX512F-NEXT:    vpmulhuw %ymm2, %ymm0, %ymm3
+; AVX512F-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vpor %ymm3, %ymm0, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: constant_rotate_v32i16:
 ; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm2 = [1,2,4,8,16,32,64,128,256,512,1024,2048,4096,8192,16384,32768]
-; AVX512VL-NEXT:    vpmulhuw %ymm2, %ymm0, %ymm3
-; AVX512VL-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpor %ymm3, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vpmulhuw %ymm2, %ymm1, %ymm3
 ; AVX512VL-NEXT:    vpmullw %ymm2, %ymm1, %ymm1
 ; AVX512VL-NEXT:    vpor %ymm3, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpmulhuw %ymm2, %ymm0, %ymm3
+; AVX512VL-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpor %ymm3, %ymm0, %ymm0
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: constant_rotate_v32i16:
@@ -539,12 +555,13 @@ define <32 x i16> @constant_rotate_v32i16(<32 x i16> %a) nounwind {
 define <64 x i8> @constant_rotate_v64i8(<64 x i8> %a) nounwind {
 ; AVX512F-LABEL: constant_rotate_v64i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
+; AVX512F-NEXT:    vpsllw $4, %ymm1, %ymm2
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
 ; AVX512F-NEXT:    vpand %ymm3, %ymm2, %ymm2
 ; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm4 = [8192,24640,41088,57536,57344,41152,24704,8256,8192,24640,41088,57536,57344,41152,24704,8256]
 ; AVX512F-NEXT:    # ymm4 = mem[0,1,0,1]
-; AVX512F-NEXT:    vpblendvb %ymm4, %ymm2, %ymm0, %ymm2
+; AVX512F-NEXT:    vpblendvb %ymm4, %ymm2, %ymm1, %ymm2
 ; AVX512F-NEXT:    vpsllw $2, %ymm2, %ymm5
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm6 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
 ; AVX512F-NEXT:    vpand %ymm6, %ymm5, %ymm5
@@ -554,44 +571,46 @@ define <64 x i8> @constant_rotate_v64i8(<64 x i8> %a) nounwind {
 ; AVX512F-NEXT:    vpaddb %ymm7, %ymm7, %ymm8
 ; AVX512F-NEXT:    vpblendvb %ymm8, %ymm5, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpxor %xmm5, %xmm5, %xmm5
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm9 = ymm0[8],ymm5[8],ymm0[9],ymm5[9],ymm0[10],ymm5[10],ymm0[11],ymm5[11],ymm0[12],ymm5[12],ymm0[13],ymm5[13],ymm0[14],ymm5[14],ymm0[15],ymm5[15],ymm0[24],ymm5[24],ymm0[25],ymm5[25],ymm0[26],ymm5[26],ymm0[27],ymm5[27],ymm0[28],ymm5[28],ymm0[29],ymm5[29],ymm0[30],ymm5[30],ymm0[31],ymm5[31]
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm9 = ymm1[8],ymm5[8],ymm1[9],ymm5[9],ymm1[10],ymm5[10],ymm1[11],ymm5[11],ymm1[12],ymm5[12],ymm1[13],ymm5[13],ymm1[14],ymm5[14],ymm1[15],ymm5[15],ymm1[24],ymm5[24],ymm1[25],ymm5[25],ymm1[26],ymm5[26],ymm1[27],ymm5[27],ymm1[28],ymm5[28],ymm1[29],ymm5[29],ymm1[30],ymm5[30],ymm1[31],ymm5[31]
 ; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm10 = [256,128,64,32,16,8,4,2,256,128,64,32,16,8,4,2]
 ; AVX512F-NEXT:    # ymm10 = mem[0,1,0,1]
 ; AVX512F-NEXT:    vpmullw %ymm10, %ymm9, %ymm9
 ; AVX512F-NEXT:    vpsrlw $8, %ymm9, %ymm9
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0],ymm5[0],ymm0[1],ymm5[1],ymm0[2],ymm5[2],ymm0[3],ymm5[3],ymm0[4],ymm5[4],ymm0[5],ymm5[5],ymm0[6],ymm5[6],ymm0[7],ymm5[7],ymm0[16],ymm5[16],ymm0[17],ymm5[17],ymm0[18],ymm5[18],ymm0[19],ymm5[19],ymm0[20],ymm5[20],ymm0[21],ymm5[21],ymm0[22],ymm5[22],ymm0[23],ymm5[23]
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm5[0],ymm1[1],ymm5[1],ymm1[2],ymm5[2],ymm1[3],ymm5[3],ymm1[4],ymm5[4],ymm1[5],ymm5[5],ymm1[6],ymm5[6],ymm1[7],ymm5[7],ymm1[16],ymm5[16],ymm1[17],ymm5[17],ymm1[18],ymm5[18],ymm1[19],ymm5[19],ymm1[20],ymm5[20],ymm1[21],ymm5[21],ymm1[22],ymm5[22],ymm1[23],ymm5[23]
 ; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm11 = [256,2,4,8,16,32,64,128,256,2,4,8,16,32,64,128]
 ; AVX512F-NEXT:    # ymm11 = mem[0,1,0,1]
-; AVX512F-NEXT:    vpmullw %ymm11, %ymm0, %ymm0
-; AVX512F-NEXT:    vpsrlw $8, %ymm0, %ymm0
-; AVX512F-NEXT:    vpackuswb %ymm9, %ymm0, %ymm0
-; AVX512F-NEXT:    vpor %ymm0, %ymm2, %ymm0
-; AVX512F-NEXT:    vpsllw $4, %ymm1, %ymm2
+; AVX512F-NEXT:    vpmullw %ymm1, %ymm11, %ymm1
+; AVX512F-NEXT:    vpsrlw $8, %ymm1, %ymm1
+; AVX512F-NEXT:    vpackuswb %ymm9, %ymm1, %ymm1
+; AVX512F-NEXT:    vpor %ymm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm2
 ; AVX512F-NEXT:    vpand %ymm3, %ymm2, %ymm2
-; AVX512F-NEXT:    vpblendvb %ymm4, %ymm2, %ymm1, %ymm2
+; AVX512F-NEXT:    vpblendvb %ymm4, %ymm2, %ymm0, %ymm2
 ; AVX512F-NEXT:    vpsllw $2, %ymm2, %ymm3
 ; AVX512F-NEXT:    vpand %ymm6, %ymm3, %ymm3
 ; AVX512F-NEXT:    vpblendvb %ymm7, %ymm3, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpaddb %ymm2, %ymm2, %ymm3
 ; AVX512F-NEXT:    vpblendvb %ymm8, %ymm3, %ymm2, %ymm2
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm1[8],ymm5[8],ymm1[9],ymm5[9],ymm1[10],ymm5[10],ymm1[11],ymm5[11],ymm1[12],ymm5[12],ymm1[13],ymm5[13],ymm1[14],ymm5[14],ymm1[15],ymm5[15],ymm1[24],ymm5[24],ymm1[25],ymm5[25],ymm1[26],ymm5[26],ymm1[27],ymm5[27],ymm1[28],ymm5[28],ymm1[29],ymm5[29],ymm1[30],ymm5[30],ymm1[31],ymm5[31]
-; AVX512F-NEXT:    vpmullw %ymm10, %ymm3, %ymm3
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm0[8],ymm5[8],ymm0[9],ymm5[9],ymm0[10],ymm5[10],ymm0[11],ymm5[11],ymm0[12],ymm5[12],ymm0[13],ymm5[13],ymm0[14],ymm5[14],ymm0[15],ymm5[15],ymm0[24],ymm5[24],ymm0[25],ymm5[25],ymm0[26],ymm5[26],ymm0[27],ymm5[27],ymm0[28],ymm5[28],ymm0[29],ymm5[29],ymm0[30],ymm5[30],ymm0[31],ymm5[31]
+; AVX512F-NEXT:    vpmullw %ymm3, %ymm10, %ymm3
 ; AVX512F-NEXT:    vpsrlw $8, %ymm3, %ymm3
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm5[0],ymm1[1],ymm5[1],ymm1[2],ymm5[2],ymm1[3],ymm5[3],ymm1[4],ymm5[4],ymm1[5],ymm5[5],ymm1[6],ymm5[6],ymm1[7],ymm5[7],ymm1[16],ymm5[16],ymm1[17],ymm5[17],ymm1[18],ymm5[18],ymm1[19],ymm5[19],ymm1[20],ymm5[20],ymm1[21],ymm5[21],ymm1[22],ymm5[22],ymm1[23],ymm5[23]
-; AVX512F-NEXT:    vpmullw %ymm11, %ymm1, %ymm1
-; AVX512F-NEXT:    vpsrlw $8, %ymm1, %ymm1
-; AVX512F-NEXT:    vpackuswb %ymm3, %ymm1, %ymm1
-; AVX512F-NEXT:    vpor %ymm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0],ymm5[0],ymm0[1],ymm5[1],ymm0[2],ymm5[2],ymm0[3],ymm5[3],ymm0[4],ymm5[4],ymm0[5],ymm5[5],ymm0[6],ymm5[6],ymm0[7],ymm5[7],ymm0[16],ymm5[16],ymm0[17],ymm5[17],ymm0[18],ymm5[18],ymm0[19],ymm5[19],ymm0[20],ymm5[20],ymm0[21],ymm5[21],ymm0[22],ymm5[22],ymm0[23],ymm5[23]
+; AVX512F-NEXT:    vpmullw %ymm0, %ymm11, %ymm0
+; AVX512F-NEXT:    vpsrlw $8, %ymm0, %ymm0
+; AVX512F-NEXT:    vpackuswb %ymm3, %ymm0, %ymm0
+; AVX512F-NEXT:    vpor %ymm0, %ymm2, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: constant_rotate_v64i8:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm2
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
+; AVX512VL-NEXT:    vpsllw $4, %ymm1, %ymm2
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
 ; AVX512VL-NEXT:    vpand %ymm3, %ymm2, %ymm2
 ; AVX512VL-NEXT:    vbroadcasti128 {{.*#+}} ymm4 = [8192,24640,41088,57536,57344,41152,24704,8256,8192,24640,41088,57536,57344,41152,24704,8256]
 ; AVX512VL-NEXT:    # ymm4 = mem[0,1,0,1]
-; AVX512VL-NEXT:    vpblendvb %ymm4, %ymm2, %ymm0, %ymm2
+; AVX512VL-NEXT:    vpblendvb %ymm4, %ymm2, %ymm1, %ymm2
 ; AVX512VL-NEXT:    vpsllw $2, %ymm2, %ymm5
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm6 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
 ; AVX512VL-NEXT:    vpand %ymm6, %ymm5, %ymm5
@@ -600,38 +619,39 @@ define <64 x i8> @constant_rotate_v64i8(<64 x i8> %a) nounwind {
 ; AVX512VL-NEXT:    vpaddb %ymm2, %ymm2, %ymm5
 ; AVX512VL-NEXT:    vpaddb %ymm7, %ymm7, %ymm8
 ; AVX512VL-NEXT:    vpblendvb %ymm8, %ymm5, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX512VL-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
 ; AVX512VL-NEXT:    vpsrlw $8, %ymm5, %ymm5
 ; AVX512VL-NEXT:    vbroadcasti128 {{.*#+}} ymm9 = [256,128,64,32,16,8,4,2,256,128,64,32,16,8,4,2]
 ; AVX512VL-NEXT:    # ymm9 = mem[0,1,0,1]
-; AVX512VL-NEXT:    vpmullw %ymm9, %ymm5, %ymm5
+; AVX512VL-NEXT:    vpmullw %ymm5, %ymm9, %ymm5
 ; AVX512VL-NEXT:    vpsrlw $8, %ymm5, %ymm5
-; AVX512VL-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512VL-NEXT:    vpsrlw $8, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512VL-NEXT:    vpsrlw $8, %ymm1, %ymm1
 ; AVX512VL-NEXT:    vbroadcasti128 {{.*#+}} ymm10 = [256,2,4,8,16,32,64,128,256,2,4,8,16,32,64,128]
 ; AVX512VL-NEXT:    # ymm10 = mem[0,1,0,1]
-; AVX512VL-NEXT:    vpmullw %ymm10, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpsrlw $8, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpackuswb %ymm5, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpor %ymm0, %ymm2, %ymm0
-; AVX512VL-NEXT:    vpsllw $4, %ymm1, %ymm2
+; AVX512VL-NEXT:    vpmullw %ymm1, %ymm10, %ymm1
+; AVX512VL-NEXT:    vpsrlw $8, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpackuswb %ymm5, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpor %ymm1, %ymm2, %ymm1
+; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm2
 ; AVX512VL-NEXT:    vpand %ymm3, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpblendvb %ymm4, %ymm2, %ymm1, %ymm2
+; AVX512VL-NEXT:    vpblendvb %ymm4, %ymm2, %ymm0, %ymm2
 ; AVX512VL-NEXT:    vpsllw $2, %ymm2, %ymm3
 ; AVX512VL-NEXT:    vpand %ymm6, %ymm3, %ymm3
 ; AVX512VL-NEXT:    vpblendvb %ymm7, %ymm3, %ymm2, %ymm2
 ; AVX512VL-NEXT:    vpaddb %ymm2, %ymm2, %ymm3
 ; AVX512VL-NEXT:    vpblendvb %ymm8, %ymm3, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX512VL-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
 ; AVX512VL-NEXT:    vpsrlw $8, %ymm3, %ymm3
-; AVX512VL-NEXT:    vpmullw %ymm9, %ymm3, %ymm3
+; AVX512VL-NEXT:    vpmullw %ymm3, %ymm9, %ymm3
 ; AVX512VL-NEXT:    vpsrlw $8, %ymm3, %ymm3
-; AVX512VL-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512VL-NEXT:    vpsrlw $8, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpmullw %ymm10, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpsrlw $8, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpackuswb %ymm3, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpor %ymm1, %ymm2, %ymm1
+; AVX512VL-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512VL-NEXT:    vpsrlw $8, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpmullw %ymm0, %ymm10, %ymm0
+; AVX512VL-NEXT:    vpsrlw $8, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpackuswb %ymm3, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpor %ymm0, %ymm2, %ymm0
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: constant_rotate_v64i8:
@@ -724,22 +744,26 @@ define <16 x i32> @splatconstant_rotate_v16i32(<16 x i32> %a) nounwind {
 define <32 x i16> @splatconstant_rotate_v32i16(<32 x i16> %a) nounwind {
 ; AVX512F-LABEL: splatconstant_rotate_v32i16:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpsrlw $9, %ymm0, %ymm2
-; AVX512F-NEXT:    vpsllw $7, %ymm0, %ymm0
-; AVX512F-NEXT:    vpor %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512F-NEXT:    vpsrlw $9, %ymm1, %ymm2
 ; AVX512F-NEXT:    vpsllw $7, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpor %ymm2, %ymm1, %ymm1
+; AVX512F-NEXT:    vpsrlw $9, %ymm0, %ymm2
+; AVX512F-NEXT:    vpsllw $7, %ymm0, %ymm0
+; AVX512F-NEXT:    vpor %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatconstant_rotate_v32i16:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsrlw $9, %ymm0, %ymm2
-; AVX512VL-NEXT:    vpsllw $7, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpor %ymm2, %ymm0, %ymm0
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512VL-NEXT:    vpsrlw $9, %ymm1, %ymm2
 ; AVX512VL-NEXT:    vpsllw $7, %ymm1, %ymm1
 ; AVX512VL-NEXT:    vpor %ymm2, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpsrlw $9, %ymm0, %ymm2
+; AVX512VL-NEXT:    vpsllw $7, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpor %ymm2, %ymm0, %ymm0
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatconstant_rotate_v32i16:
@@ -764,50 +788,46 @@ define <32 x i16> @splatconstant_rotate_v32i16(<32 x i16> %a) nounwind {
 define <64 x i8> @splatconstant_rotate_v64i8(<64 x i8> %a) nounwind {
 ; AVX512F-LABEL: splatconstant_rotate_v64i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpsrlw $4, %ymm0, %ymm2
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX512F-NEXT:    vpandn %ymm2, %ymm3, %ymm2
-; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm0
-; AVX512F-NEXT:    vpand %ymm3, %ymm0, %ymm0
-; AVX512F-NEXT:    vpor %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512F-NEXT:    vpsrlw $4, %ymm1, %ymm2
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
 ; AVX512F-NEXT:    vpandn %ymm2, %ymm3, %ymm2
 ; AVX512F-NEXT:    vpsllw $4, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpand %ymm3, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpor %ymm2, %ymm1, %ymm1
+; AVX512F-NEXT:    vpsrlw $4, %ymm0, %ymm2
+; AVX512F-NEXT:    vpandn %ymm2, %ymm3, %ymm2
+; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm0
+; AVX512F-NEXT:    vpand %ymm3, %ymm0, %ymm0
+; AVX512F-NEXT:    vpor %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatconstant_rotate_v64i8:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm2
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX512VL-NEXT:    vpandn %ymm2, %ymm3, %ymm2
-; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpand %ymm3, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpor %ymm2, %ymm0, %ymm0
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512VL-NEXT:    vpsrlw $4, %ymm1, %ymm2
-; AVX512VL-NEXT:    vpandn %ymm2, %ymm3, %ymm2
 ; AVX512VL-NEXT:    vpsllw $4, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpand %ymm3, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpor %ymm2, %ymm1, %ymm1
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
+; AVX512VL-NEXT:    vpternlogq $226, %ymm2, %ymm3, %ymm1
+; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm2
+; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpternlogq $226, %ymm2, %ymm3, %ymm0
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatconstant_rotate_v64i8:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    vpsllw $4, %zmm0, %zmm1
-; AVX512BW-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
 ; AVX512BW-NEXT:    vpsrlw $4, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
-; AVX512BW-NEXT:    vporq %zmm0, %zmm1, %zmm0
+; AVX512BW-NEXT:    vpternlogq $216, {{.*}}(%rip), %zmm1, %zmm0
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: splatconstant_rotate_v64i8:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpsllw $4, %zmm0, %zmm1
-; AVX512VLBW-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
 ; AVX512VLBW-NEXT:    vpsrlw $4, %zmm0, %zmm0
-; AVX512VLBW-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
-; AVX512VLBW-NEXT:    vporq %zmm0, %zmm1, %zmm0
+; AVX512VLBW-NEXT:    vpternlogq $216, {{.*}}(%rip), %zmm1, %zmm0
 ; AVX512VLBW-NEXT:    retq
   %shl = shl <64 x i8> %a, <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>
   %lshr = lshr <64 x i8> %a, <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>
@@ -850,28 +870,32 @@ define <16 x i32> @splatconstant_rotate_mask_v16i32(<16 x i32> %a) nounwind {
 define <32 x i16> @splatconstant_rotate_mask_v32i16(<32 x i16> %a) nounwind {
 ; AVX512F-LABEL: splatconstant_rotate_mask_v32i16:
 ; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm2 = [55,55,55,55,55,55,55,55,55,55,55,55,55,55,55,55]
-; AVX512F-NEXT:    vpsrlw $11, %ymm0, %ymm3
-; AVX512F-NEXT:    vpsllw $5, %ymm0, %ymm0
-; AVX512F-NEXT:    vpor %ymm3, %ymm0, %ymm0
-; AVX512F-NEXT:    vpand %ymm2, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpsrlw $11, %ymm1, %ymm3
 ; AVX512F-NEXT:    vpsllw $5, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpor %ymm3, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX512F-NEXT:    vpsrlw $11, %ymm0, %ymm3
+; AVX512F-NEXT:    vpsllw $5, %ymm0, %ymm0
+; AVX512F-NEXT:    vpor %ymm3, %ymm0, %ymm0
+; AVX512F-NEXT:    vpand %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatconstant_rotate_mask_v32i16:
 ; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm2 = [55,55,55,55,55,55,55,55,55,55,55,55,55,55,55,55]
-; AVX512VL-NEXT:    vpsrlw $11, %ymm0, %ymm3
-; AVX512VL-NEXT:    vpsllw $5, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpor %ymm3, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpand %ymm2, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vpsrlw $11, %ymm1, %ymm3
 ; AVX512VL-NEXT:    vpsllw $5, %ymm1, %ymm1
 ; AVX512VL-NEXT:    vpor %ymm3, %ymm1, %ymm1
 ; AVX512VL-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpsrlw $11, %ymm0, %ymm3
+; AVX512VL-NEXT:    vpsllw $5, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpor %ymm3, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpand %ymm2, %ymm0, %ymm0
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatconstant_rotate_mask_v32i16:
@@ -902,38 +926,38 @@ define <32 x i16> @splatconstant_rotate_mask_v32i16(<32 x i16> %a) nounwind {
 define <64 x i8> @splatconstant_rotate_mask_v64i8(<64 x i8> %a) nounwind {
 ; AVX512F-LABEL: splatconstant_rotate_mask_v64i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpsrlw $4, %ymm0, %ymm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
+; AVX512F-NEXT:    vpsrlw $4, %ymm1, %ymm2
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
 ; AVX512F-NEXT:    vpandn %ymm2, %ymm3, %ymm2
-; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm0
-; AVX512F-NEXT:    vpand %ymm3, %ymm0, %ymm0
-; AVX512F-NEXT:    vpor %ymm2, %ymm0, %ymm0
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm2 = [39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39]
-; AVX512F-NEXT:    vpand %ymm2, %ymm0, %ymm0
-; AVX512F-NEXT:    vpsrlw $4, %ymm1, %ymm4
-; AVX512F-NEXT:    vpandn %ymm4, %ymm3, %ymm4
 ; AVX512F-NEXT:    vpsllw $4, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpand %ymm3, %ymm1, %ymm1
-; AVX512F-NEXT:    vpor %ymm4, %ymm1, %ymm1
+; AVX512F-NEXT:    vpor %ymm2, %ymm1, %ymm1
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm2 = [39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39]
 ; AVX512F-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX512F-NEXT:    vpsrlw $4, %ymm0, %ymm4
+; AVX512F-NEXT:    vpandn %ymm4, %ymm3, %ymm4
+; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm0
+; AVX512F-NEXT:    vpand %ymm3, %ymm0, %ymm0
+; AVX512F-NEXT:    vpor %ymm4, %ymm0, %ymm0
+; AVX512F-NEXT:    vpand %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatconstant_rotate_mask_v64i8:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm2
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
+; AVX512VL-NEXT:    vpsrlw $4, %ymm1, %ymm2
+; AVX512VL-NEXT:    vpsllw $4, %ymm1, %ymm1
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX512VL-NEXT:    vpandn %ymm2, %ymm3, %ymm2
-; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpand %ymm3, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpor %ymm2, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpternlogq $226, %ymm2, %ymm3, %ymm1
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm2 = [39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39]
-; AVX512VL-NEXT:    vpand %ymm2, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpsrlw $4, %ymm1, %ymm4
-; AVX512VL-NEXT:    vpandn %ymm4, %ymm3, %ymm4
-; AVX512VL-NEXT:    vpsllw $4, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpand %ymm3, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpor %ymm4, %ymm1, %ymm1
 ; AVX512VL-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm4
+; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpternlogq $226, %ymm4, %ymm3, %ymm0
+; AVX512VL-NEXT:    vpand %ymm2, %ymm0, %ymm0
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatconstant_rotate_mask_v64i8:
diff --git a/llvm/test/CodeGen/X86/vector-sext.ll b/llvm/test/CodeGen/X86/vector-sext.ll
index dd96924be..48ff42319 100644
--- a/llvm/test/CodeGen/X86/vector-sext.ll
+++ b/llvm/test/CodeGen/X86/vector-sext.ll
@@ -181,10 +181,10 @@ define <32 x i16> @sext_32i8_to_32i16(<32 x i8> %A) nounwind uwtable readnone ss
 ;
 ; AVX512F-LABEL: sext_32i8_to_32i16:
 ; AVX512F:       # %bb.0: # %entry
-; AVX512F-NEXT:    vpmovsxbw %xmm0, %ymm2
-; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; AVX512F-NEXT:    vpmovsxbw %xmm0, %ymm1
-; AVX512F-NEXT:    vmovdqa %ymm2, %ymm0
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-NEXT:    vpmovsxbw %xmm1, %ymm1
+; AVX512F-NEXT:    vpmovsxbw %xmm0, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: sext_32i8_to_32i16:
@@ -267,22 +267,20 @@ entry:
 define <8 x i32> @sext_16i8_to_8i32(<16 x i8> %A) nounwind uwtable readnone ssp {
 ; SSE2-LABEL: sext_16i8_to_8i32:
 ; SSE2:       # %bb.0: # %entry
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
-; SSE2-NEXT:    psrad $24, %xmm2
-; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSE2-NEXT:    psrad $24, %xmm0
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    psrad $24, %xmm1
-; SSE2-NEXT:    movdqa %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: sext_16i8_to_8i32:
 ; SSSE3:       # %bb.0: # %entry
-; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
-; SSSE3-NEXT:    psrad $24, %xmm2
-; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSSE3-NEXT:    psrad $24, %xmm0
+; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
 ; SSSE3-NEXT:    psrad $24, %xmm1
-; SSSE3-NEXT:    movdqa %xmm2, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: sext_16i8_to_8i32:
@@ -313,12 +311,11 @@ define <8 x i32> @sext_16i8_to_8i32(<16 x i8> %A) nounwind uwtable readnone ssp
 ;
 ; X32-SSE2-LABEL: sext_16i8_to_8i32:
 ; X32-SSE2:       # %bb.0: # %entry
-; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
-; X32-SSE2-NEXT:    psrad $24, %xmm2
-; X32-SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; X32-SSE2-NEXT:    psrad $24, %xmm0
+; X32-SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
 ; X32-SSE2-NEXT:    psrad $24, %xmm1
-; X32-SSE2-NEXT:    movdqa %xmm2, %xmm0
 ; X32-SSE2-NEXT:    retl
 ;
 ; X32-SSE41-LABEL: sext_16i8_to_8i32:
@@ -337,7 +334,6 @@ entry:
 define <16 x i32> @sext_16i8_to_16i32(<16 x i8> %A) nounwind uwtable readnone ssp {
 ; SSE2-LABEL: sext_16i8_to_16i32:
 ; SSE2:       # %bb.0: # %entry
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
 ; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
 ; SSE2-NEXT:    psrad $24, %xmm4
@@ -353,7 +349,6 @@ define <16 x i32> @sext_16i8_to_16i32(<16 x i8> %A) nounwind uwtable readnone ss
 ;
 ; SSSE3-LABEL: sext_16i8_to_16i32:
 ; SSSE3:       # %bb.0: # %entry
-; SSSE3-NEXT:    movdqa %xmm0, %xmm1
 ; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
 ; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
 ; SSSE3-NEXT:    psrad $24, %xmm4
@@ -408,7 +403,6 @@ define <16 x i32> @sext_16i8_to_16i32(<16 x i8> %A) nounwind uwtable readnone ss
 ;
 ; X32-SSE2-LABEL: sext_16i8_to_16i32:
 ; X32-SSE2:       # %bb.0: # %entry
-; X32-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
 ; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
 ; X32-SSE2-NEXT:    psrad $24, %xmm4
@@ -568,42 +562,40 @@ entry:
 define <8 x i64> @sext_16i8_to_8i64(<16 x i8> %A) nounwind uwtable readnone ssp {
 ; SSE2-LABEL: sext_16i8_to_8i64:
 ; SSE2:       # %bb.0: # %entry
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
 ; SSE2-NEXT:    psrad $24, %xmm1
-; SSE2-NEXT:    pxor %xmm5, %xmm5
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
-; SSE2-NEXT:    movdqa %xmm1, %xmm4
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
-; SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
-; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE2-NEXT:    pxor %xmm4, %xmm4
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
 ; SSE2-NEXT:    psrad $24, %xmm3
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm4
 ; SSE2-NEXT:    movdqa %xmm3, %xmm2
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
-; SSE2-NEXT:    punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm5[2],xmm3[3],xmm5[3]
-; SSE2-NEXT:    movdqa %xmm4, %xmm0
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
+; SSE2-NEXT:    punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm4[2],xmm3[3],xmm4[3]
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: sext_16i8_to_8i64:
 ; SSSE3:       # %bb.0: # %entry
-; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
 ; SSSE3-NEXT:    psrad $24, %xmm1
-; SSSE3-NEXT:    pxor %xmm5, %xmm5
-; SSSE3-NEXT:    pxor %xmm2, %xmm2
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
-; SSSE3-NEXT:    movdqa %xmm1, %xmm4
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
-; SSSE3-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
-; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSSE3-NEXT:    pxor %xmm4, %xmm4
+; SSSE3-NEXT:    pxor %xmm3, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm3
+; SSSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; SSSE3-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
 ; SSSE3-NEXT:    psrad $24, %xmm3
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm4
 ; SSSE3-NEXT:    movdqa %xmm3, %xmm2
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
-; SSSE3-NEXT:    punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm5[2],xmm3[3],xmm5[3]
-; SSSE3-NEXT:    movdqa %xmm4, %xmm0
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
+; SSSE3-NEXT:    punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm4[2],xmm3[3],xmm4[3]
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: sext_16i8_to_8i64:
@@ -648,22 +640,21 @@ define <8 x i64> @sext_16i8_to_8i64(<16 x i8> %A) nounwind uwtable readnone ssp
 ;
 ; X32-SSE2-LABEL: sext_16i8_to_8i64:
 ; X32-SSE2:       # %bb.0: # %entry
-; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
 ; X32-SSE2-NEXT:    psrad $24, %xmm1
-; X32-SSE2-NEXT:    pxor %xmm5, %xmm5
-; X32-SSE2-NEXT:    pxor %xmm2, %xmm2
-; X32-SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
-; X32-SSE2-NEXT:    movdqa %xmm1, %xmm4
-; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
-; X32-SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
-; X32-SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; X32-SSE2-NEXT:    pxor %xmm4, %xmm4
+; X32-SSE2-NEXT:    pxor %xmm3, %xmm3
+; X32-SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
+; X32-SSE2-NEXT:    movdqa %xmm1, %xmm0
+; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; X32-SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; X32-SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
 ; X32-SSE2-NEXT:    psrad $24, %xmm3
-; X32-SSE2-NEXT:    pcmpgtd %xmm3, %xmm5
+; X32-SSE2-NEXT:    pcmpgtd %xmm3, %xmm4
 ; X32-SSE2-NEXT:    movdqa %xmm3, %xmm2
-; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
-; X32-SSE2-NEXT:    punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm5[2],xmm3[3],xmm5[3]
-; X32-SSE2-NEXT:    movdqa %xmm4, %xmm0
+; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
+; X32-SSE2-NEXT:    punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm4[2],xmm3[3],xmm4[3]
 ; X32-SSE2-NEXT:    retl
 ;
 ; X32-SSE41-LABEL: sext_16i8_to_8i64:
@@ -2134,8 +2125,8 @@ entry:
 define <8 x i64> @load_sext_8i8_to_8i64(<8 x i8> *%ptr) {
 ; SSE2-LABEL: load_sext_8i8_to_8i64:
 ; SSE2:       # %bb.0: # %entry
-; SSE2-NEXT:    movq {{.*#+}} xmm2 = mem[0],zero
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
 ; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
 ; SSE2-NEXT:    psrad $24, %xmm1
 ; SSE2-NEXT:    pxor %xmm4, %xmm4
@@ -2154,8 +2145,8 @@ define <8 x i64> @load_sext_8i8_to_8i64(<8 x i8> *%ptr) {
 ;
 ; SSSE3-LABEL: load_sext_8i8_to_8i64:
 ; SSSE3:       # %bb.0: # %entry
-; SSSE3-NEXT:    movq {{.*#+}} xmm2 = mem[0],zero
-; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSSE3-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
 ; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
 ; SSSE3-NEXT:    psrad $24, %xmm1
 ; SSSE3-NEXT:    pxor %xmm4, %xmm4
@@ -2204,8 +2195,8 @@ define <8 x i64> @load_sext_8i8_to_8i64(<8 x i8> *%ptr) {
 ; X32-SSE2-LABEL: load_sext_8i8_to_8i64:
 ; X32-SSE2:       # %bb.0: # %entry
 ; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE2-NEXT:    movq {{.*#+}} xmm2 = mem[0],zero
-; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X32-SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
 ; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
 ; X32-SSE2-NEXT:    psrad $24, %xmm1
 ; X32-SSE2-NEXT:    pxor %xmm4, %xmm4
@@ -2303,8 +2294,8 @@ entry:
 define <8 x i32> @load_sext_8i8_to_8i32(<8 x i8> *%ptr) {
 ; SSE2-LABEL: load_sext_8i8_to_8i32:
 ; SSE2:       # %bb.0: # %entry
-; SSE2-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
 ; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
 ; SSE2-NEXT:    psrad $24, %xmm0
 ; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
@@ -2313,8 +2304,8 @@ define <8 x i32> @load_sext_8i8_to_8i32(<8 x i8> *%ptr) {
 ;
 ; SSSE3-LABEL: load_sext_8i8_to_8i32:
 ; SSSE3:       # %bb.0: # %entry
-; SSSE3-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
-; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSSE3-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
 ; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
 ; SSSE3-NEXT:    psrad $24, %xmm0
 ; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
@@ -2347,8 +2338,8 @@ define <8 x i32> @load_sext_8i8_to_8i32(<8 x i8> *%ptr) {
 ; X32-SSE2-LABEL: load_sext_8i8_to_8i32:
 ; X32-SSE2:       # %bb.0: # %entry
 ; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE2-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
-; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X32-SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
 ; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
 ; X32-SSE2-NEXT:    psrad $24, %xmm0
 ; X32-SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
@@ -3064,23 +3055,15 @@ define i32 @sext_2i8_to_i32(<16 x i8> %A) nounwind uwtable readnone ssp {
 ;
 ; X32-SSE2-LABEL: sext_2i8_to_i32:
 ; X32-SSE2:       # %bb.0: # %entry
-; X32-SSE2-NEXT:    pushl %eax
-; X32-SSE2-NEXT:    .cfi_def_cfa_offset 8
 ; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; X32-SSE2-NEXT:    psraw $8, %xmm0
 ; X32-SSE2-NEXT:    movd %xmm0, %eax
-; X32-SSE2-NEXT:    popl %ecx
-; X32-SSE2-NEXT:    .cfi_def_cfa_offset 4
 ; X32-SSE2-NEXT:    retl
 ;
 ; X32-SSE41-LABEL: sext_2i8_to_i32:
 ; X32-SSE41:       # %bb.0: # %entry
-; X32-SSE41-NEXT:    pushl %eax
-; X32-SSE41-NEXT:    .cfi_def_cfa_offset 8
 ; X32-SSE41-NEXT:    pmovsxbw %xmm0, %xmm0
 ; X32-SSE41-NEXT:    movd %xmm0, %eax
-; X32-SSE41-NEXT:    popl %ecx
-; X32-SSE41-NEXT:    .cfi_def_cfa_offset 4
 ; X32-SSE41-NEXT:    retl
 entry:
   %Shuf = shufflevector <16 x i8> %A, <16 x i8> undef, <2 x i32> <i32 0, i32 1>
@@ -3179,84 +3162,71 @@ define <4 x i64> @sext_4i1_to_4i64(<4 x i1> %mask) {
 define <4 x i64> @sext_4i8_to_4i64(<4 x i8> %mask) {
 ; SSE2-LABEL: sext_4i8_to_4i64:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pslld $24, %xmm0
-; SSE2-NEXT:    psrad $24, %xmm0
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; SSE2-NEXT:    psrad $24, %xmm1
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: sext_4i8_to_4i64:
 ; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    pslld $24, %xmm0
-; SSSE3-NEXT:    psrad $24, %xmm0
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; SSSE3-NEXT:    psrad $24, %xmm1
 ; SSSE3-NEXT:    pxor %xmm2, %xmm2
-; SSSE3-NEXT:    pxor %xmm3, %xmm3
-; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
 ; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; SSSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; SSSE3-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: sext_4i8_to_4i64:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pslld $24, %xmm0
-; SSE41-NEXT:    psrad $24, %xmm0
-; SSE41-NEXT:    pmovsxdq %xmm0, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; SSE41-NEXT:    pmovsxdq %xmm0, %xmm1
+; SSE41-NEXT:    pmovsxbq %xmm0, %xmm2
+; SSE41-NEXT:    psrld $16, %xmm0
+; SSE41-NEXT:    pmovsxbq %xmm0, %xmm1
 ; SSE41-NEXT:    movdqa %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: sext_4i8_to_4i64:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpslld $24, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrad $24, %xmm0, %xmm0
-; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm0
+; AVX1-NEXT:    vpmovsxbq %xmm0, %xmm1
+; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm0
+; AVX1-NEXT:    vpmovsxbq %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: sext_4i8_to_4i64:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpslld $24, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrad $24, %xmm0, %xmm0
-; AVX2-NEXT:    vpmovsxdq %xmm0, %ymm0
+; AVX2-NEXT:    vpmovsxbq %xmm0, %ymm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: sext_4i8_to_4i64:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpslld $24, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrad $24, %xmm0, %xmm0
-; AVX512-NEXT:    vpmovsxdq %xmm0, %ymm0
+; AVX512-NEXT:    vpmovsxbq %xmm0, %ymm0
 ; AVX512-NEXT:    retq
 ;
 ; X32-SSE2-LABEL: sext_4i8_to_4i64:
 ; X32-SSE2:       # %bb.0:
-; X32-SSE2-NEXT:    pslld $24, %xmm0
-; X32-SSE2-NEXT:    psrad $24, %xmm0
+; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; X32-SSE2-NEXT:    psrad $24, %xmm1
 ; X32-SSE2-NEXT:    pxor %xmm2, %xmm2
-; X32-SSE2-NEXT:    pxor %xmm3, %xmm3
-; X32-SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
-; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
 ; X32-SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
-; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; X32-SSE2-NEXT:    movdqa %xmm1, %xmm0
+; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; X32-SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
 ; X32-SSE2-NEXT:    retl
 ;
 ; X32-SSE41-LABEL: sext_4i8_to_4i64:
 ; X32-SSE41:       # %bb.0:
-; X32-SSE41-NEXT:    pslld $24, %xmm0
-; X32-SSE41-NEXT:    psrad $24, %xmm0
-; X32-SSE41-NEXT:    pmovsxdq %xmm0, %xmm2
-; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; X32-SSE41-NEXT:    pmovsxdq %xmm0, %xmm1
+; X32-SSE41-NEXT:    pmovsxbq %xmm0, %xmm2
+; X32-SSE41-NEXT:    psrld $16, %xmm0
+; X32-SSE41-NEXT:    pmovsxbq %xmm0, %xmm1
 ; X32-SSE41-NEXT:    movdqa %xmm2, %xmm0
 ; X32-SSE41-NEXT:    retl
   %extmask = sext <4 x i8> %mask to <4 x i64>
@@ -3300,11 +3270,13 @@ define <32 x i8> @sext_32xi1_to_32xi8(<32 x i16> %c1, <32 x i16> %c2)nounwind {
 ;
 ; AVX512F-LABEL: sext_32xi1_to_32xi8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpcmpeqw %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
+; AVX512F-NEXT:    vpcmpeqw %ymm2, %ymm3, %ymm2
+; AVX512F-NEXT:    vpcmpeqw %ymm1, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
 ; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
-; AVX512F-NEXT:    vpcmpeqw %ymm3, %ymm1, %ymm1
-; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
 ; AVX512F-NEXT:    vpmovdb %zmm1, %xmm1
 ; AVX512F-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
 ; AVX512F-NEXT:    retq
@@ -3346,8 +3318,7 @@ define <2 x i32> @sext_2i8_to_2i32(<2 x i8>* %addr) {
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
 ; SSE2-NEXT:    psrad $24, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,1,3]
-; SSE2-NEXT:    paddq %xmm0, %xmm0
+; SSE2-NEXT:    paddd %xmm0, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: sext_2i8_to_2i32:
@@ -3357,20 +3328,23 @@ define <2 x i32> @sext_2i8_to_2i32(<2 x i8>* %addr) {
 ; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
 ; SSSE3-NEXT:    psrad $24, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,1,3]
-; SSSE3-NEXT:    paddq %xmm0, %xmm0
+; SSSE3-NEXT:    paddd %xmm0, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: sext_2i8_to_2i32:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pmovsxbq (%rdi), %xmm0
-; SSE41-NEXT:    paddq %xmm0, %xmm0
+; SSE41-NEXT:    movzwl (%rdi), %eax
+; SSE41-NEXT:    movd %eax, %xmm0
+; SSE41-NEXT:    pmovsxbd %xmm0, %xmm0
+; SSE41-NEXT:    paddd %xmm0, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: sext_2i8_to_2i32:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpmovsxbq (%rdi), %xmm0
-; AVX-NEXT:    vpaddq %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    movzwl (%rdi), %eax
+; AVX-NEXT:    vmovd %eax, %xmm0
+; AVX-NEXT:    vpmovsxbd %xmm0, %xmm0
+; AVX-NEXT:    vpaddd %xmm0, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; X32-SSE2-LABEL: sext_2i8_to_2i32:
@@ -3381,15 +3355,16 @@ define <2 x i32> @sext_2i8_to_2i32(<2 x i8>* %addr) {
 ; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
 ; X32-SSE2-NEXT:    psrad $24, %xmm0
-; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,1,3]
-; X32-SSE2-NEXT:    paddq %xmm0, %xmm0
+; X32-SSE2-NEXT:    paddd %xmm0, %xmm0
 ; X32-SSE2-NEXT:    retl
 ;
 ; X32-SSE41-LABEL: sext_2i8_to_2i32:
 ; X32-SSE41:       # %bb.0:
 ; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE41-NEXT:    pmovsxbq (%eax), %xmm0
-; X32-SSE41-NEXT:    paddq %xmm0, %xmm0
+; X32-SSE41-NEXT:    movzwl (%eax), %eax
+; X32-SSE41-NEXT:    movd %eax, %xmm0
+; X32-SSE41-NEXT:    pmovsxbd %xmm0, %xmm0
+; X32-SSE41-NEXT:    paddd %xmm0, %xmm0
 ; X32-SSE41-NEXT:    retl
   %x = load <2 x i8>, <2 x i8>* %addr, align 1
   %y = sext <2 x i8> %x to <2 x i32>
@@ -3815,8 +3790,8 @@ entry:
 define <8 x i32> @zext_negate_sext(<8 x i8> %x) {
 ; SSE2-LABEL: zext_negate_sext:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
 ; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
 ; SSE2-NEXT:    psubw %xmm0, %xmm1
 ; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
 ; SSE2-NEXT:    psrad $16, %xmm0
@@ -3826,8 +3801,8 @@ define <8 x i32> @zext_negate_sext(<8 x i8> %x) {
 ;
 ; SSSE3-LABEL: zext_negate_sext:
 ; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    pand {{.*}}(%rip), %xmm0
 ; SSSE3-NEXT:    pxor %xmm1, %xmm1
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
 ; SSSE3-NEXT:    psubw %xmm0, %xmm1
 ; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
 ; SSSE3-NEXT:    psrad $16, %xmm0
@@ -3837,7 +3812,7 @@ define <8 x i32> @zext_negate_sext(<8 x i8> %x) {
 ;
 ; SSE41-LABEL: zext_negate_sext:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; SSE41-NEXT:    pxor %xmm1, %xmm1
 ; SSE41-NEXT:    psubw %xmm0, %xmm1
 ; SSE41-NEXT:    pmovsxwd %xmm1, %xmm0
@@ -3847,35 +3822,33 @@ define <8 x i32> @zext_negate_sext(<8 x i8> %x) {
 ;
 ; AVX1-LABEL: zext_negate_sext:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
-; AVX1-NEXT:    vpsubd %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; AVX1-NEXT:    vpsubd %xmm0, %xmm1, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
+; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpsubd %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; AVX1-NEXT:    vpsubd %xmm0, %xmm2, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: zext_negate_sext:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
 ; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX2-NEXT:    vpsubd %ymm0, %ymm1, %ymm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: zext_negate_sext:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX512-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
 ; AVX512-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX512-NEXT:    vpsubd %ymm0, %ymm1, %ymm0
 ; AVX512-NEXT:    retq
 ;
 ; X32-SSE2-LABEL: zext_negate_sext:
 ; X32-SSE2:       # %bb.0:
-; X32-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
 ; X32-SSE2-NEXT:    pxor %xmm1, %xmm1
+; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
 ; X32-SSE2-NEXT:    psubw %xmm0, %xmm1
 ; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
 ; X32-SSE2-NEXT:    psrad $16, %xmm0
@@ -3885,7 +3858,7 @@ define <8 x i32> @zext_negate_sext(<8 x i8> %x) {
 ;
 ; X32-SSE41-LABEL: zext_negate_sext:
 ; X32-SSE41:       # %bb.0:
-; X32-SSE41-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X32-SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; X32-SSE41-NEXT:    pxor %xmm1, %xmm1
 ; X32-SSE41-NEXT:    psubw %xmm0, %xmm1
 ; X32-SSE41-NEXT:    pmovsxwd %xmm1, %xmm0
@@ -3901,7 +3874,8 @@ define <8 x i32> @zext_negate_sext(<8 x i8> %x) {
 define <8 x i32> @zext_decremenet_sext(<8 x i8> %x) {
 ; SSE2-LABEL: zext_decremenet_sext:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
 ; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
 ; SSE2-NEXT:    paddw %xmm0, %xmm1
 ; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
@@ -3912,7 +3886,8 @@ define <8 x i32> @zext_decremenet_sext(<8 x i8> %x) {
 ;
 ; SSSE3-LABEL: zext_decremenet_sext:
 ; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSSE3-NEXT:    pxor %xmm1, %xmm1
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
 ; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm1
 ; SSSE3-NEXT:    paddw %xmm0, %xmm1
 ; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
@@ -3923,7 +3898,7 @@ define <8 x i32> @zext_decremenet_sext(<8 x i8> %x) {
 ;
 ; SSE41-LABEL: zext_decremenet_sext:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
 ; SSE41-NEXT:    paddw %xmm0, %xmm1
 ; SSE41-NEXT:    pmovsxwd %xmm1, %xmm0
@@ -3933,35 +3908,33 @@ define <8 x i32> @zext_decremenet_sext(<8 x i8> %x) {
 ;
 ; AVX1-LABEL: zext_decremenet_sext:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
 ; AVX1-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; AVX1-NEXT:    vpaddd %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: zext_decremenet_sext:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
 ; AVX2-NEXT:    vpcmpeqd %ymm1, %ymm1, %ymm1
 ; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: zext_decremenet_sext:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX512-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
 ; AVX512-NEXT:    vpcmpeqd %ymm1, %ymm1, %ymm1
 ; AVX512-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
 ; AVX512-NEXT:    retq
 ;
 ; X32-SSE2-LABEL: zext_decremenet_sext:
 ; X32-SSE2:       # %bb.0:
-; X32-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X32-SSE2-NEXT:    pxor %xmm1, %xmm1
+; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
 ; X32-SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
 ; X32-SSE2-NEXT:    paddw %xmm0, %xmm1
 ; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
@@ -3972,7 +3945,7 @@ define <8 x i32> @zext_decremenet_sext(<8 x i8> %x) {
 ;
 ; X32-SSE41-LABEL: zext_decremenet_sext:
 ; X32-SSE41:       # %bb.0:
-; X32-SSE41-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X32-SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; X32-SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
 ; X32-SSE41-NEXT:    paddw %xmm0, %xmm1
 ; X32-SSE41-NEXT:    pmovsxwd %xmm1, %xmm0
diff --git a/llvm/test/CodeGen/X86/vector-shift-ashr-512.ll b/llvm/test/CodeGen/X86/vector-shift-ashr-512.ll
index e4f676cec..1bb62977c 100644
--- a/llvm/test/CodeGen/X86/vector-shift-ashr-512.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-ashr-512.ll
@@ -27,14 +27,17 @@ define <16 x i32> @var_shift_v16i32(<16 x i32> %a, <16 x i32> %b) nounwind {
 define <32 x i16> @var_shift_v32i16(<32 x i16> %a, <32 x i16> %b) nounwind {
 ; AVX512DQ-LABEL: var_shift_v32i16:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
+; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} zmm3 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
+; AVX512DQ-NEXT:    vpmovsxwd %ymm2, %zmm2
+; AVX512DQ-NEXT:    vpsravd %zmm3, %zmm2, %zmm2
+; AVX512DQ-NEXT:    vpmovdw %zmm2, %ymm2
+; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
 ; AVX512DQ-NEXT:    vpmovsxwd %ymm0, %zmm0
-; AVX512DQ-NEXT:    vpsravd %zmm2, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpsravd %zmm1, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    vpmovdw %zmm0, %ymm0
-; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
-; AVX512DQ-NEXT:    vpmovsxwd %ymm1, %zmm1
-; AVX512DQ-NEXT:    vpsravd %zmm2, %zmm1, %zmm1
-; AVX512DQ-NEXT:    vpmovdw %zmm1, %ymm1
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512BW-LABEL: var_shift_v32i16:
@@ -48,9 +51,11 @@ define <32 x i16> @var_shift_v32i16(<32 x i16> %a, <32 x i16> %b) nounwind {
 define <64 x i8> @var_shift_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
 ; AVX512DQ-LABEL: var_shift_v64i8:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vpsllw $5, %ymm2, %ymm2
-; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm0[8],ymm2[8],ymm0[9],ymm2[9],ymm0[10],ymm2[10],ymm0[11],ymm2[11],ymm0[12],ymm2[12],ymm0[13],ymm2[13],ymm0[14],ymm2[14],ymm0[15],ymm2[15],ymm0[24],ymm2[24],ymm0[25],ymm2[25],ymm0[26],ymm2[26],ymm0[27],ymm2[27],ymm0[28],ymm2[28],ymm0[29],ymm2[29],ymm0[30],ymm2[30],ymm0[31],ymm2[31]
-; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
+; AVX512DQ-NEXT:    vpsllw $5, %ymm3, %ymm3
+; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm0[8],ymm3[8],ymm0[9],ymm3[9],ymm0[10],ymm3[10],ymm0[11],ymm3[11],ymm0[12],ymm3[12],ymm0[13],ymm3[13],ymm0[14],ymm3[14],ymm0[15],ymm3[15],ymm0[24],ymm3[24],ymm0[25],ymm3[25],ymm0[26],ymm3[26],ymm0[27],ymm3[27],ymm0[28],ymm3[28],ymm0[29],ymm3[29],ymm0[30],ymm3[30],ymm0[31],ymm3[31]
+; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm0[8],ymm2[8],ymm0[9],ymm2[9],ymm0[10],ymm2[10],ymm0[11],ymm2[11],ymm0[12],ymm2[12],ymm0[13],ymm2[13],ymm0[14],ymm2[14],ymm0[15],ymm2[15],ymm0[24],ymm2[24],ymm0[25],ymm2[25],ymm0[26],ymm2[26],ymm0[27],ymm2[27],ymm0[28],ymm2[28],ymm0[29],ymm2[29],ymm0[30],ymm2[30],ymm0[31],ymm2[31]
 ; AVX512DQ-NEXT:    vpsraw $4, %ymm5, %ymm6
 ; AVX512DQ-NEXT:    vpblendvb %ymm4, %ymm6, %ymm5, %ymm5
 ; AVX512DQ-NEXT:    vpsraw $2, %ymm5, %ymm6
@@ -60,21 +65,21 @@ define <64 x i8> @var_shift_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
 ; AVX512DQ-NEXT:    vpaddw %ymm4, %ymm4, %ymm4
 ; AVX512DQ-NEXT:    vpblendvb %ymm4, %ymm6, %ymm5, %ymm4
 ; AVX512DQ-NEXT:    vpsrlw $8, %ymm4, %ymm4
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm0[0],ymm3[0],ymm0[1],ymm3[1],ymm0[2],ymm3[2],ymm0[3],ymm3[3],ymm0[4],ymm3[4],ymm0[5],ymm3[5],ymm0[6],ymm3[6],ymm0[7],ymm3[7],ymm0[16],ymm3[16],ymm0[17],ymm3[17],ymm0[18],ymm3[18],ymm0[19],ymm3[19],ymm0[20],ymm3[20],ymm0[21],ymm3[21],ymm0[22],ymm3[22],ymm0[23],ymm3[23]
 ; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm0[0],ymm2[0],ymm0[1],ymm2[1],ymm0[2],ymm2[2],ymm0[3],ymm2[3],ymm0[4],ymm2[4],ymm0[5],ymm2[5],ymm0[6],ymm2[6],ymm0[7],ymm2[7],ymm0[16],ymm2[16],ymm0[17],ymm2[17],ymm0[18],ymm2[18],ymm0[19],ymm2[19],ymm0[20],ymm2[20],ymm0[21],ymm2[21],ymm0[22],ymm2[22],ymm0[23],ymm2[23]
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512DQ-NEXT:    vpsraw $4, %ymm0, %ymm5
-; AVX512DQ-NEXT:    vpblendvb %ymm2, %ymm5, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpsraw $2, %ymm0, %ymm5
-; AVX512DQ-NEXT:    vpaddw %ymm2, %ymm2, %ymm2
-; AVX512DQ-NEXT:    vpblendvb %ymm2, %ymm5, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpsraw $1, %ymm0, %ymm5
-; AVX512DQ-NEXT:    vpaddw %ymm2, %ymm2, %ymm2
-; AVX512DQ-NEXT:    vpblendvb %ymm2, %ymm5, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpsrlw $8, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpackuswb %ymm4, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpsllw $5, %ymm3, %ymm2
-; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm0[8],ymm2[8],ymm0[9],ymm2[9],ymm0[10],ymm2[10],ymm0[11],ymm2[11],ymm0[12],ymm2[12],ymm0[13],ymm2[13],ymm0[14],ymm2[14],ymm0[15],ymm2[15],ymm0[24],ymm2[24],ymm0[25],ymm2[25],ymm0[26],ymm2[26],ymm0[27],ymm2[27],ymm0[28],ymm2[28],ymm0[29],ymm2[29],ymm0[30],ymm2[30],ymm0[31],ymm2[31]
-; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15],ymm0[24],ymm1[24],ymm0[25],ymm1[25],ymm0[26],ymm1[26],ymm0[27],ymm1[27],ymm0[28],ymm1[28],ymm0[29],ymm1[29],ymm0[30],ymm1[30],ymm0[31],ymm1[31]
+; AVX512DQ-NEXT:    vpsraw $4, %ymm2, %ymm5
+; AVX512DQ-NEXT:    vpblendvb %ymm3, %ymm5, %ymm2, %ymm2
+; AVX512DQ-NEXT:    vpsraw $2, %ymm2, %ymm5
+; AVX512DQ-NEXT:    vpaddw %ymm3, %ymm3, %ymm3
+; AVX512DQ-NEXT:    vpblendvb %ymm3, %ymm5, %ymm2, %ymm2
+; AVX512DQ-NEXT:    vpsraw $1, %ymm2, %ymm5
+; AVX512DQ-NEXT:    vpaddw %ymm3, %ymm3, %ymm3
+; AVX512DQ-NEXT:    vpblendvb %ymm3, %ymm5, %ymm2, %ymm2
+; AVX512DQ-NEXT:    vpsrlw $8, %ymm2, %ymm2
+; AVX512DQ-NEXT:    vpackuswb %ymm4, %ymm2, %ymm2
+; AVX512DQ-NEXT:    vpsllw $5, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15],ymm0[24],ymm1[24],ymm0[25],ymm1[25],ymm0[26],ymm1[26],ymm0[27],ymm1[27],ymm0[28],ymm1[28],ymm0[29],ymm1[29],ymm0[30],ymm1[30],ymm0[31],ymm1[31]
+; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
 ; AVX512DQ-NEXT:    vpsraw $4, %ymm4, %ymm5
 ; AVX512DQ-NEXT:    vpblendvb %ymm3, %ymm5, %ymm4, %ymm4
 ; AVX512DQ-NEXT:    vpsraw $2, %ymm4, %ymm5
@@ -84,18 +89,19 @@ define <64 x i8> @var_shift_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
 ; AVX512DQ-NEXT:    vpaddw %ymm3, %ymm3, %ymm3
 ; AVX512DQ-NEXT:    vpblendvb %ymm3, %ymm5, %ymm4, %ymm3
 ; AVX512DQ-NEXT:    vpsrlw $8, %ymm3, %ymm3
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm0[0],ymm2[0],ymm0[1],ymm2[1],ymm0[2],ymm2[2],ymm0[3],ymm2[3],ymm0[4],ymm2[4],ymm0[5],ymm2[5],ymm0[6],ymm2[6],ymm0[7],ymm2[7],ymm0[16],ymm2[16],ymm0[17],ymm2[17],ymm0[18],ymm2[18],ymm0[19],ymm2[19],ymm0[20],ymm2[20],ymm0[21],ymm2[21],ymm0[22],ymm2[22],ymm0[23],ymm2[23]
 ; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[16],ymm1[16],ymm0[17],ymm1[17],ymm0[18],ymm1[18],ymm0[19],ymm1[19],ymm0[20],ymm1[20],ymm0[21],ymm1[21],ymm0[22],ymm1[22],ymm0[23],ymm1[23]
-; AVX512DQ-NEXT:    vpsraw $4, %ymm1, %ymm4
-; AVX512DQ-NEXT:    vpblendvb %ymm2, %ymm4, %ymm1, %ymm1
-; AVX512DQ-NEXT:    vpsraw $2, %ymm1, %ymm4
-; AVX512DQ-NEXT:    vpaddw %ymm2, %ymm2, %ymm2
-; AVX512DQ-NEXT:    vpblendvb %ymm2, %ymm4, %ymm1, %ymm1
-; AVX512DQ-NEXT:    vpsraw $1, %ymm1, %ymm4
-; AVX512DQ-NEXT:    vpaddw %ymm2, %ymm2, %ymm2
-; AVX512DQ-NEXT:    vpblendvb %ymm2, %ymm4, %ymm1, %ymm1
-; AVX512DQ-NEXT:    vpsrlw $8, %ymm1, %ymm1
-; AVX512DQ-NEXT:    vpackuswb %ymm3, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512DQ-NEXT:    vpsraw $4, %ymm0, %ymm4
+; AVX512DQ-NEXT:    vpblendvb %ymm1, %ymm4, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpsraw $2, %ymm0, %ymm4
+; AVX512DQ-NEXT:    vpaddw %ymm1, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpblendvb %ymm1, %ymm4, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpsraw $1, %ymm0, %ymm4
+; AVX512DQ-NEXT:    vpaddw %ymm1, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpblendvb %ymm1, %ymm4, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpsrlw $8, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpackuswb %ymm3, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512BW-LABEL: var_shift_v64i8:
@@ -163,9 +169,11 @@ define <16 x i32> @splatvar_shift_v16i32(<16 x i32> %a, <16 x i32> %b) nounwind
 define <32 x i16> @splatvar_shift_v32i16(<32 x i16> %a, <32 x i16> %b) nounwind {
 ; AVX512DQ-LABEL: splatvar_shift_v32i16:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
-; AVX512DQ-NEXT:    vpsraw %xmm2, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpsraw %xmm2, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512DQ-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512DQ-NEXT:    vpsraw %xmm1, %ymm2, %ymm2
+; AVX512DQ-NEXT:    vpsraw %xmm1, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatvar_shift_v32i16:
@@ -181,21 +189,23 @@ define <32 x i16> @splatvar_shift_v32i16(<32 x i16> %a, <32 x i16> %b) nounwind
 define <64 x i8> @splatvar_shift_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
 ; AVX512DQ-LABEL: splatvar_shift_v64i8:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vpmovzxbq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-NEXT:    vpsrlw %xmm2, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512DQ-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512DQ-NEXT:    vpsrlw %xmm1, %ymm2, %ymm2
 ; AVX512DQ-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX512DQ-NEXT:    vpsrlw %xmm2, %xmm3, %xmm3
+; AVX512DQ-NEXT:    vpsrlw %xmm1, %xmm3, %xmm3
 ; AVX512DQ-NEXT:    vpsrlw $8, %xmm3, %xmm3
 ; AVX512DQ-NEXT:    vpbroadcastb %xmm3, %ymm3
-; AVX512DQ-NEXT:    vpand %ymm3, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpand %ymm3, %ymm2, %ymm2
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm4 = [32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896]
-; AVX512DQ-NEXT:    vpsrlw %xmm2, %ymm4, %ymm4
+; AVX512DQ-NEXT:    vpsrlw %xmm1, %ymm4, %ymm4
+; AVX512DQ-NEXT:    vpxor %ymm4, %ymm2, %ymm2
+; AVX512DQ-NEXT:    vpsubb %ymm4, %ymm2, %ymm2
+; AVX512DQ-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpand %ymm3, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpxor %ymm4, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpsubb %ymm4, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpsrlw %xmm2, %ymm1, %ymm1
-; AVX512DQ-NEXT:    vpand %ymm3, %ymm1, %ymm1
-; AVX512DQ-NEXT:    vpxor %ymm4, %ymm1, %ymm1
-; AVX512DQ-NEXT:    vpsubb %ymm4, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatvar_shift_v64i8:
@@ -242,13 +252,15 @@ define <16 x i32> @constant_shift_v16i32(<16 x i32> %a) nounwind {
 define <32 x i16> @constant_shift_v32i16(<32 x i16> %a) nounwind {
 ; AVX512DQ-LABEL: constant_shift_v32i16:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vpmovsxwd %ymm0, %zmm0
-; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15]
-; AVX512DQ-NEXT:    vpsravd %zmm2, %zmm0, %zmm0
-; AVX512DQ-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512DQ-NEXT:    vpmovsxwd %ymm1, %zmm1
+; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15]
 ; AVX512DQ-NEXT:    vpsravd %zmm2, %zmm1, %zmm1
 ; AVX512DQ-NEXT:    vpmovdw %zmm1, %ymm1
+; AVX512DQ-NEXT:    vpmovsxwd %ymm0, %zmm0
+; AVX512DQ-NEXT:    vpsravd %zmm2, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512BW-LABEL: constant_shift_v32i16:
@@ -262,28 +274,30 @@ define <32 x i16> @constant_shift_v32i16(<32 x i16> %a) nounwind {
 define <64 x i8> @constant_shift_v64i8(<64 x i8> %a) nounwind {
 ; AVX512DQ-LABEL: constant_shift_v64i8:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
+; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
 ; AVX512DQ-NEXT:    vpsraw $8, %ymm2, %ymm2
 ; AVX512DQ-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [2,4,8,16,32,64,128,256,2,4,8,16,32,64,128,256]
 ; AVX512DQ-NEXT:    # ymm3 = mem[0,1,0,1]
 ; AVX512DQ-NEXT:    vpmullw %ymm3, %ymm2, %ymm2
 ; AVX512DQ-NEXT:    vpsrlw $8, %ymm2, %ymm2
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512DQ-NEXT:    vpsraw $8, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vbroadcasti128 {{.*#+}} ymm4 = [256,128,64,32,16,8,4,2,256,128,64,32,16,8,4,2]
-; AVX512DQ-NEXT:    # ymm4 = mem[0,1,0,1]
-; AVX512DQ-NEXT:    vpmullw %ymm4, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpsrlw $8, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
-; AVX512DQ-NEXT:    vpsraw $8, %ymm2, %ymm2
-; AVX512DQ-NEXT:    vpmullw %ymm3, %ymm2, %ymm2
-; AVX512DQ-NEXT:    vpsrlw $8, %ymm2, %ymm2
 ; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
 ; AVX512DQ-NEXT:    vpsraw $8, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vbroadcasti128 {{.*#+}} ymm4 = [256,128,64,32,16,8,4,2,256,128,64,32,16,8,4,2]
+; AVX512DQ-NEXT:    # ymm4 = mem[0,1,0,1]
 ; AVX512DQ-NEXT:    vpmullw %ymm4, %ymm1, %ymm1
 ; AVX512DQ-NEXT:    vpsrlw $8, %ymm1, %ymm1
 ; AVX512DQ-NEXT:    vpackuswb %ymm2, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX512DQ-NEXT:    vpsraw $8, %ymm2, %ymm2
+; AVX512DQ-NEXT:    vpmullw %ymm3, %ymm2, %ymm2
+; AVX512DQ-NEXT:    vpsrlw $8, %ymm2, %ymm2
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512DQ-NEXT:    vpsraw $8, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpmullw %ymm4, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpsrlw $8, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512BW-LABEL: constant_shift_v64i8:
@@ -327,8 +341,10 @@ define <16 x i32> @splatconstant_shift_v16i32(<16 x i32> %a) nounwind {
 define <32 x i16> @splatconstant_shift_v32i16(<32 x i16> %a) nounwind {
 ; AVX512DQ-LABEL: splatconstant_shift_v32i16:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vpsraw $3, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512DQ-NEXT:    vpsraw $3, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpsraw $3, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatconstant_shift_v32i16:
@@ -342,16 +358,18 @@ define <32 x i16> @splatconstant_shift_v32i16(<32 x i16> %a) nounwind {
 define <64 x i8> @splatconstant_shift_v64i8(<64 x i8> %a) nounwind {
 ; AVX512DQ-LABEL: splatconstant_shift_v64i8:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vpsrlw $3, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm2 = [31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31]
-; AVX512DQ-NEXT:    vpand %ymm2, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm3 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
-; AVX512DQ-NEXT:    vpxor %ymm3, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpsubb %ymm3, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512DQ-NEXT:    vpsrlw $3, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm2 = [31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31]
 ; AVX512DQ-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm3 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
 ; AVX512DQ-NEXT:    vpxor %ymm3, %ymm1, %ymm1
 ; AVX512DQ-NEXT:    vpsubb %ymm3, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpsrlw $3, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpand %ymm2, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpxor %ymm3, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpsubb %ymm3, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatconstant_shift_v64i8:
@@ -369,9 +387,11 @@ define <64 x i8> @splatconstant_shift_v64i8(<64 x i8> %a) nounwind {
 define <64 x i8> @ashr_const7_v64i8(<64 x i8> %a) {
 ; AVX512DQ-LABEL: ashr_const7_v64i8:
 ; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512DQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512DQ-NEXT:    vpcmpgtb %ymm0, %ymm2, %ymm0
 ; AVX512DQ-NEXT:    vpcmpgtb %ymm1, %ymm2, %ymm1
+; AVX512DQ-NEXT:    vpcmpgtb %ymm0, %ymm2, %ymm0
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512BW-LABEL: ashr_const7_v64i8:
diff --git a/llvm/test/CodeGen/X86/vector-shift-ashr-sub128.ll b/llvm/test/CodeGen/X86/vector-shift-ashr-sub128.ll
index 8e4895849..3f0345ad8 100644
--- a/llvm/test/CodeGen/X86/vector-shift-ashr-sub128.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-ashr-sub128.ll
@@ -20,157 +20,6 @@
 define <2 x i32> @var_shift_v2i32(<2 x i32> %a, <2 x i32> %b) nounwind {
 ; SSE2-LABEL: var_shift_v2i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    psllq $32, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,3,2,3]
-; SSE2-NEXT:    psrad $31, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [9223372036854775808,9223372036854775808]
-; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    psrlq %xmm1, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[2,3,0,1]
-; SSE2-NEXT:    psrlq %xmm4, %xmm0
-; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm3[0],xmm0[1]
-; SSE2-NEXT:    movdqa %xmm2, %xmm3
-; SSE2-NEXT:    psrlq %xmm1, %xmm3
-; SSE2-NEXT:    psrlq %xmm4, %xmm2
-; SSE2-NEXT:    movsd {{.*#+}} xmm2 = xmm3[0],xmm2[1]
-; SSE2-NEXT:    xorpd %xmm0, %xmm2
-; SSE2-NEXT:    psubq %xmm0, %xmm2
-; SSE2-NEXT:    movdqa %xmm2, %xmm0
-; SSE2-NEXT:    retq
-;
-; SSE41-LABEL: var_shift_v2i32:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    psllq $32, %xmm2
-; SSE41-NEXT:    psrad $31, %xmm2
-; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
-; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3],xmm1[4,5],xmm0[6,7]
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; SSE41-NEXT:    movdqa %xmm2, %xmm3
-; SSE41-NEXT:    psrlq %xmm1, %xmm3
-; SSE41-NEXT:    psrlq %xmm0, %xmm2
-; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm3[4,5,6,7]
-; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808]
-; SSE41-NEXT:    movdqa %xmm3, %xmm4
-; SSE41-NEXT:    psrlq %xmm0, %xmm4
-; SSE41-NEXT:    psrlq %xmm1, %xmm3
-; SSE41-NEXT:    pblendw {{.*#+}} xmm3 = xmm4[0,1,2,3],xmm3[4,5,6,7]
-; SSE41-NEXT:    pxor %xmm3, %xmm2
-; SSE41-NEXT:    psubq %xmm3, %xmm2
-; SSE41-NEXT:    movdqa %xmm2, %xmm0
-; SSE41-NEXT:    retq
-;
-; AVX1-LABEL: var_shift_v2i32:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpsllq $32, %xmm0, %xmm2
-; AVX1-NEXT:    vpsrad $31, %xmm2, %xmm2
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
-; AVX1-NEXT:    vpsrlq %xmm2, %xmm0, %xmm3
-; AVX1-NEXT:    vpsrlq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm3[4,5,6,7]
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808]
-; AVX1-NEXT:    vpsrlq %xmm1, %xmm3, %xmm1
-; AVX1-NEXT:    vpsrlq %xmm2, %xmm3, %xmm2
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm2[4,5,6,7]
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsubq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: var_shift_v2i32:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; AVX2-NEXT:    vpsllq $32, %xmm0, %xmm2
-; AVX2-NEXT:    vpsrad $31, %xmm2, %xmm2
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
-; AVX2-NEXT:    vpsrlvq %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
-; AVX2-NEXT:    vpsrlvq %xmm1, %xmm2, %xmm1
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpsubq %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    retq
-;
-; XOPAVX1-LABEL: var_shift_v2i32:
-; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; XOPAVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
-; XOPAVX1-NEXT:    vpsllq $32, %xmm0, %xmm0
-; XOPAVX1-NEXT:    vpshaq {{.*}}(%rip), %xmm0, %xmm0
-; XOPAVX1-NEXT:    vpsubq %xmm1, %xmm2, %xmm1
-; XOPAVX1-NEXT:    vpshaq %xmm1, %xmm0, %xmm0
-; XOPAVX1-NEXT:    retq
-;
-; XOPAVX2-LABEL: var_shift_v2i32:
-; XOPAVX2:       # %bb.0:
-; XOPAVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; XOPAVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; XOPAVX2-NEXT:    vpsllq $32, %xmm0, %xmm0
-; XOPAVX2-NEXT:    vpshaq {{.*}}(%rip), %xmm0, %xmm0
-; XOPAVX2-NEXT:    vpsubq %xmm1, %xmm2, %xmm1
-; XOPAVX2-NEXT:    vpshaq %xmm1, %xmm0, %xmm0
-; XOPAVX2-NEXT:    retq
-;
-; AVX512-LABEL: var_shift_v2i32:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsllq $32, %xmm0, %xmm0
-; AVX512-NEXT:    vpsraq $32, %zmm0, %zmm0
-; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; AVX512-NEXT:    vpsravq %zmm1, %zmm0, %zmm0
-; AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
-; AVX512-NEXT:    vzeroupper
-; AVX512-NEXT:    retq
-;
-; AVX512VL-LABEL: var_shift_v2i32:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VL-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; AVX512VL-NEXT:    vpsllq $32, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpsraq $32, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpsravq %xmm1, %xmm0, %xmm0
-; AVX512VL-NEXT:    retq
-;
-; X32-SSE-LABEL: var_shift_v2i32:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    psllq $32, %xmm0
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,3,2,3]
-; X32-SSE-NEXT:    psrad $31, %xmm0
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
-; X32-SSE-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm1
-; X32-SSE-NEXT:    movdqa {{.*#+}} xmm0 = [0,2147483648,0,2147483648]
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm3
-; X32-SSE-NEXT:    psrlq %xmm1, %xmm3
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[2,3,0,1]
-; X32-SSE-NEXT:    xorps %xmm5, %xmm5
-; X32-SSE-NEXT:    movss {{.*#+}} xmm5 = xmm4[0],xmm5[1,2,3]
-; X32-SSE-NEXT:    psrlq %xmm5, %xmm0
-; X32-SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm3[0],xmm0[1]
-; X32-SSE-NEXT:    movdqa %xmm2, %xmm3
-; X32-SSE-NEXT:    psrlq %xmm5, %xmm3
-; X32-SSE-NEXT:    psrlq %xmm1, %xmm2
-; X32-SSE-NEXT:    shufpd {{.*#+}} xmm2 = xmm2[0],xmm3[1]
-; X32-SSE-NEXT:    xorpd %xmm0, %xmm2
-; X32-SSE-NEXT:    psubq %xmm0, %xmm2
-; X32-SSE-NEXT:    movdqa %xmm2, %xmm0
-; X32-SSE-NEXT:    retl
-  %shift = ashr <2 x i32> %a, %b
-  ret <2 x i32> %shift
-}
-
-define <4 x i16> @var_shift_v4i16(<4 x i16> %a, <4 x i16> %b) nounwind {
-; SSE2-LABEL: var_shift_v4i16:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    pslld $16, %xmm0
-; SSE2-NEXT:    psrad $16, %xmm0
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm1[2,3,3,3,4,5,6,7]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm3
 ; SSE2-NEXT:    psrad %xmm2, %xmm3
@@ -189,100 +38,70 @@ define <4 x i16> @var_shift_v4i16(<4 x i16> %a, <4 x i16> %b) nounwind {
 ; SSE2-NEXT:    movaps %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: var_shift_v4i16:
+; SSE41-LABEL: var_shift_v2i32:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pxor %xmm2, %xmm2
-; SSE41-NEXT:    movdqa %xmm1, %xmm3
-; SSE41-NEXT:    pblendw {{.*#+}} xmm3 = xmm3[0],xmm2[1],xmm3[2],xmm2[3],xmm3[4],xmm2[5],xmm3[6],xmm2[7]
-; SSE41-NEXT:    pslld $16, %xmm0
-; SSE41-NEXT:    psrad $16, %xmm0
-; SSE41-NEXT:    pshuflw {{.*#+}} xmm4 = xmm3[2,3,3,3,4,5,6,7]
+; SSE41-NEXT:    pshuflw {{.*#+}} xmm2 = xmm1[2,3,3,3,4,5,6,7]
+; SSE41-NEXT:    movdqa %xmm0, %xmm3
+; SSE41-NEXT:    psrad %xmm2, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; SSE41-NEXT:    pshuflw {{.*#+}} xmm4 = xmm2[2,3,3,3,4,5,6,7]
 ; SSE41-NEXT:    movdqa %xmm0, %xmm5
 ; SSE41-NEXT:    psrad %xmm4, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[2,3,0,1]
-; SSE41-NEXT:    pshuflw {{.*#+}} xmm4 = xmm3[2,3,3,3,4,5,6,7]
-; SSE41-NEXT:    movdqa %xmm0, %xmm6
-; SSE41-NEXT:    psrad %xmm4, %xmm6
-; SSE41-NEXT:    pblendw {{.*#+}} xmm6 = xmm5[0,1,2,3],xmm6[4,5,6,7]
-; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    psrad %xmm1, %xmm2
-; SSE41-NEXT:    pshuflw {{.*#+}} xmm1 = xmm3[0,1,1,1,4,5,6,7]
+; SSE41-NEXT:    pblendw {{.*#+}} xmm5 = xmm3[0,1,2,3],xmm5[4,5,6,7]
+; SSE41-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,1,1,1,4,5,6,7]
+; SSE41-NEXT:    movdqa %xmm0, %xmm3
+; SSE41-NEXT:    psrad %xmm1, %xmm3
+; SSE41-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,1,1,1,4,5,6,7]
 ; SSE41-NEXT:    psrad %xmm1, %xmm0
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm2[0,1,2,3],xmm0[4,5,6,7]
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm6[2,3],xmm0[4,5],xmm6[6,7]
+; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm3[0,1,2,3],xmm0[4,5,6,7]
+; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm5[2,3],xmm0[4,5],xmm5[6,7]
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: var_shift_v4i16:
+; AVX1-LABEL: var_shift_v2i32:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
-; AVX1-NEXT:    vpslld $16, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrad $16, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrldq {{.*#+}} xmm3 = xmm1[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX1-NEXT:    vpsrad %xmm3, %xmm0, %xmm3
-; AVX1-NEXT:    vpsrlq $32, %xmm1, %xmm4
-; AVX1-NEXT:    vpsrad %xmm4, %xmm0, %xmm4
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm4[0,1,2,3],xmm3[4,5,6,7]
-; AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm2 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; AVX1-NEXT:    vpsrldq {{.*#+}} xmm2 = xmm1[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; AVX1-NEXT:    vpsrad %xmm2, %xmm0, %xmm2
+; AVX1-NEXT:    vpsrlq $32, %xmm1, %xmm3
+; AVX1-NEXT:    vpsrad %xmm3, %xmm0, %xmm3
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
+; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm3 = xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; AVX1-NEXT:    vpsrad %xmm3, %xmm0, %xmm3
 ; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
 ; AVX1-NEXT:    vpsrad %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm2[4,5,6,7]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm3[2,3],xmm0[4,5],xmm3[6,7]
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm3[4,5,6,7]
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
 ; AVX1-NEXT:    retq
 ;
-; AVX2-LABEL: var_shift_v4i16:
+; AVX2-LABEL: var_shift_v2i32:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
-; AVX2-NEXT:    vpslld $16, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrad $16, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsravd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    retq
 ;
-; XOPAVX1-LABEL: var_shift_v4i16:
+; XOPAVX1-LABEL: var_shift_v2i32:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; XOPAVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
-; XOPAVX1-NEXT:    vpslld $16, %xmm0, %xmm0
-; XOPAVX1-NEXT:    vpsrad $16, %xmm0, %xmm0
 ; XOPAVX1-NEXT:    vpsubd %xmm1, %xmm2, %xmm1
 ; XOPAVX1-NEXT:    vpshad %xmm1, %xmm0, %xmm0
 ; XOPAVX1-NEXT:    retq
 ;
-; XOPAVX2-LABEL: var_shift_v4i16:
+; XOPAVX2-LABEL: var_shift_v2i32:
 ; XOPAVX2:       # %bb.0:
-; XOPAVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; XOPAVX2-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
-; XOPAVX2-NEXT:    vpslld $16, %xmm0, %xmm0
-; XOPAVX2-NEXT:    vpsrad $16, %xmm0, %xmm0
 ; XOPAVX2-NEXT:    vpsravd %xmm1, %xmm0, %xmm0
 ; XOPAVX2-NEXT:    retq
 ;
-; AVX512-LABEL: var_shift_v4i16:
+; AVX512-LABEL: var_shift_v2i32:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
-; AVX512-NEXT:    vpslld $16, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrad $16, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsravd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    retq
 ;
-; AVX512VL-LABEL: var_shift_v4i16:
+; AVX512VL-LABEL: var_shift_v2i32:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VL-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
-; AVX512VL-NEXT:    vpslld $16, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpsrad $16, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vpsravd %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: var_shift_v4i16:
+; X32-SSE-LABEL: var_shift_v2i32:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pslld $16, %xmm0
-; X32-SSE-NEXT:    psrad $16, %xmm0
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm1
 ; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm1[2,3,3,3,4,5,6,7]
 ; X32-SSE-NEXT:    movdqa %xmm0, %xmm3
 ; X32-SSE-NEXT:    psrad %xmm2, %xmm3
@@ -300,167 +119,21 @@ define <4 x i16> @var_shift_v4i16(<4 x i16> %a, <4 x i16> %b) nounwind {
 ; X32-SSE-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,3],xmm0[0,3]
 ; X32-SSE-NEXT:    movaps %xmm2, %xmm0
 ; X32-SSE-NEXT:    retl
-  %shift = ashr <4 x i16> %a, %b
-  ret <4 x i16> %shift
-}
-
-define <2 x i16> @var_shift_v2i16(<2 x i16> %a, <2 x i16> %b) nounwind {
-; SSE2-LABEL: var_shift_v2i16:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    psllq $48, %xmm0
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    psrad $31, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
-; SSE2-NEXT:    psrad $16, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
-; SSE2-NEXT:    movdqa %xmm2, %xmm3
-; SSE2-NEXT:    psrlq %xmm1, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[2,3,0,1]
-; SSE2-NEXT:    psrlq %xmm4, %xmm2
-; SSE2-NEXT:    movsd {{.*#+}} xmm2 = xmm3[0],xmm2[1]
-; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    psrlq %xmm1, %xmm3
-; SSE2-NEXT:    psrlq %xmm4, %xmm0
-; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm3[0],xmm0[1]
-; SSE2-NEXT:    xorpd %xmm2, %xmm0
-; SSE2-NEXT:    psubq %xmm2, %xmm0
-; SSE2-NEXT:    retq
-;
-; SSE41-LABEL: var_shift_v2i16:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    psllq $48, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    psrad $31, %xmm2
-; SSE41-NEXT:    psrad $16, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
-; SSE41-NEXT:    pxor %xmm2, %xmm2
-; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3],xmm1[4],xmm2[5,6,7]
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,3,0,1]
-; SSE41-NEXT:    movdqa %xmm0, %xmm3
-; SSE41-NEXT:    psrlq %xmm1, %xmm3
-; SSE41-NEXT:    psrlq %xmm2, %xmm0
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm3[4,5,6,7]
-; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808]
-; SSE41-NEXT:    movdqa %xmm3, %xmm4
-; SSE41-NEXT:    psrlq %xmm2, %xmm4
-; SSE41-NEXT:    psrlq %xmm1, %xmm3
-; SSE41-NEXT:    pblendw {{.*#+}} xmm3 = xmm4[0,1,2,3],xmm3[4,5,6,7]
-; SSE41-NEXT:    pxor %xmm3, %xmm0
-; SSE41-NEXT:    psubq %xmm3, %xmm0
-; SSE41-NEXT:    retq
-;
-; AVX1-LABEL: var_shift_v2i16:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpsllq $48, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrad $31, %xmm0, %xmm2
-; AVX1-NEXT:    vpsrad $16, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3],xmm1[4],xmm2[5,6,7]
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
-; AVX1-NEXT:    vpsrlq %xmm2, %xmm0, %xmm3
-; AVX1-NEXT:    vpsrlq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm3[4,5,6,7]
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808]
-; AVX1-NEXT:    vpsrlq %xmm1, %xmm3, %xmm1
-; AVX1-NEXT:    vpsrlq %xmm2, %xmm3, %xmm2
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm2[4,5,6,7]
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsubq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: var_shift_v2i16:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3],xmm1[4],xmm2[5,6,7]
-; AVX2-NEXT:    vpsllq $48, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrad $31, %xmm0, %xmm2
-; AVX2-NEXT:    vpsrad $16, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
-; AVX2-NEXT:    vpsrlvq %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
-; AVX2-NEXT:    vpsrlvq %xmm1, %xmm2, %xmm1
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpsubq %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    retq
-;
-; XOP-LABEL: var_shift_v2i16:
-; XOP:       # %bb.0:
-; XOP-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; XOP-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3],xmm1[4],xmm2[5,6,7]
-; XOP-NEXT:    vpsllq $48, %xmm0, %xmm0
-; XOP-NEXT:    vpshaq {{.*}}(%rip), %xmm0, %xmm0
-; XOP-NEXT:    vpsubq %xmm1, %xmm2, %xmm1
-; XOP-NEXT:    vpshaq %xmm1, %xmm0, %xmm0
-; XOP-NEXT:    retq
-;
-; AVX512-LABEL: var_shift_v2i16:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsllq $48, %xmm0, %xmm0
-; AVX512-NEXT:    vpsraq $48, %zmm0, %zmm0
-; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3],xmm1[4],xmm2[5,6,7]
-; AVX512-NEXT:    vpsravq %zmm1, %zmm0, %zmm0
-; AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
-; AVX512-NEXT:    vzeroupper
-; AVX512-NEXT:    retq
-;
-; AVX512VL-LABEL: var_shift_v2i16:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VL-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3],xmm1[4],xmm2[5,6,7]
-; AVX512VL-NEXT:    vpsllq $48, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpsraq $48, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpsravq %xmm1, %xmm0, %xmm0
-; AVX512VL-NEXT:    retq
-;
-; X32-SSE-LABEL: var_shift_v2i16:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    psllq $48, %xmm0
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm2
-; X32-SSE-NEXT:    psrad $31, %xmm2
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
-; X32-SSE-NEXT:    psrad $16, %xmm0
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
-; X32-SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm1
-; X32-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [0,2147483648,0,2147483648]
-; X32-SSE-NEXT:    movdqa %xmm2, %xmm3
-; X32-SSE-NEXT:    psrlq %xmm1, %xmm3
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[2,3,0,1]
-; X32-SSE-NEXT:    psrlq %xmm4, %xmm2
-; X32-SSE-NEXT:    movsd {{.*#+}} xmm2 = xmm3[0],xmm2[1]
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm3
-; X32-SSE-NEXT:    psrlq %xmm1, %xmm3
-; X32-SSE-NEXT:    psrlq %xmm4, %xmm0
-; X32-SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm3[0],xmm0[1]
-; X32-SSE-NEXT:    xorpd %xmm2, %xmm0
-; X32-SSE-NEXT:    psubq %xmm2, %xmm0
-; X32-SSE-NEXT:    retl
-  %shift = ashr <2 x i16> %a, %b
-  ret <2 x i16> %shift
+  %shift = ashr <2 x i32> %a, %b
+  ret <2 x i32> %shift
 }
 
-define <8 x i8> @var_shift_v8i8(<8 x i8> %a, <8 x i8> %b) nounwind {
-; SSE2-LABEL: var_shift_v8i8:
+define <4 x i16> @var_shift_v4i16(<4 x i16> %a, <4 x i16> %b) nounwind {
+; SSE2-LABEL: var_shift_v4i16:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    psllw $8, %xmm2
-; SSE2-NEXT:    movdqa %xmm2, %xmm3
-; SSE2-NEXT:    psraw $8, %xmm3
 ; SSE2-NEXT:    psllw $12, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm0
-; SSE2-NEXT:    psraw $15, %xmm0
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
 ; SSE2-NEXT:    psraw $15, %xmm2
-; SSE2-NEXT:    pand %xmm0, %xmm2
-; SSE2-NEXT:    pandn %xmm3, %xmm0
-; SSE2-NEXT:    por %xmm2, %xmm0
+; SSE2-NEXT:    movdqa %xmm2, %xmm3
+; SSE2-NEXT:    pandn %xmm0, %xmm3
+; SSE2-NEXT:    psraw $8, %xmm0
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    por %xmm3, %xmm0
 ; SSE2-NEXT:    paddw %xmm1, %xmm1
 ; SSE2-NEXT:    movdqa %xmm1, %xmm2
 ; SSE2-NEXT:    psraw $15, %xmm2
@@ -486,66 +159,57 @@ define <8 x i8> @var_shift_v8i8(<8 x i8> %a, <8 x i8> %b) nounwind {
 ; SSE2-NEXT:    por %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: var_shift_v8i8:
+; SSE41-LABEL: var_shift_v4i16:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movdqa %xmm1, %xmm2
-; SSE41-NEXT:    movdqa %xmm0, %xmm3
-; SSE41-NEXT:    psllw $8, %xmm3
-; SSE41-NEXT:    movdqa %xmm3, %xmm1
-; SSE41-NEXT:    psraw $8, %xmm1
-; SSE41-NEXT:    pand {{.*}}(%rip), %xmm2
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
 ; SSE41-NEXT:    movdqa %xmm2, %xmm0
 ; SSE41-NEXT:    psllw $12, %xmm0
 ; SSE41-NEXT:    psllw $4, %xmm2
 ; SSE41-NEXT:    por %xmm0, %xmm2
-; SSE41-NEXT:    movdqa %xmm2, %xmm4
-; SSE41-NEXT:    paddw %xmm2, %xmm4
-; SSE41-NEXT:    psraw $15, %xmm3
+; SSE41-NEXT:    movdqa %xmm2, %xmm3
+; SSE41-NEXT:    paddw %xmm2, %xmm3
+; SSE41-NEXT:    movdqa %xmm1, %xmm4
+; SSE41-NEXT:    psraw $8, %xmm4
 ; SSE41-NEXT:    movdqa %xmm2, %xmm0
-; SSE41-NEXT:    pblendvb %xmm0, %xmm3, %xmm1
+; SSE41-NEXT:    pblendvb %xmm0, %xmm4, %xmm1
 ; SSE41-NEXT:    movdqa %xmm1, %xmm2
 ; SSE41-NEXT:    psraw $4, %xmm2
-; SSE41-NEXT:    movdqa %xmm4, %xmm0
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
 ; SSE41-NEXT:    pblendvb %xmm0, %xmm2, %xmm1
 ; SSE41-NEXT:    movdqa %xmm1, %xmm2
 ; SSE41-NEXT:    psraw $2, %xmm2
-; SSE41-NEXT:    paddw %xmm4, %xmm4
-; SSE41-NEXT:    movdqa %xmm4, %xmm0
+; SSE41-NEXT:    paddw %xmm3, %xmm3
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
 ; SSE41-NEXT:    pblendvb %xmm0, %xmm2, %xmm1
 ; SSE41-NEXT:    movdqa %xmm1, %xmm2
 ; SSE41-NEXT:    psraw $1, %xmm2
-; SSE41-NEXT:    paddw %xmm4, %xmm4
-; SSE41-NEXT:    movdqa %xmm4, %xmm0
+; SSE41-NEXT:    paddw %xmm3, %xmm3
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
 ; SSE41-NEXT:    pblendvb %xmm0, %xmm2, %xmm1
 ; SSE41-NEXT:    movdqa %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: var_shift_v8i8:
+; AVX1-LABEL: var_shift_v4i16:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpsllw $8, %xmm0, %xmm0
-; AVX1-NEXT:    vpsraw $8, %xmm0, %xmm2
-; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
-; AVX1-NEXT:    vpsllw $12, %xmm1, %xmm3
+; AVX1-NEXT:    vpsllw $12, %xmm1, %xmm2
 ; AVX1-NEXT:    vpsllw $4, %xmm1, %xmm1
-; AVX1-NEXT:    vpor %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpaddw %xmm1, %xmm1, %xmm3
-; AVX1-NEXT:    vpsraw $15, %xmm0, %xmm0
-; AVX1-NEXT:    vpblendvb %xmm1, %xmm0, %xmm2, %xmm0
+; AVX1-NEXT:    vpor %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpaddw %xmm1, %xmm1, %xmm2
+; AVX1-NEXT:    vpsraw $8, %xmm0, %xmm3
+; AVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsraw $4, %xmm0, %xmm1
-; AVX1-NEXT:    vpblendvb %xmm3, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsraw $2, %xmm0, %xmm1
-; AVX1-NEXT:    vpaddw %xmm3, %xmm3, %xmm2
+; AVX1-NEXT:    vpaddw %xmm2, %xmm2, %xmm2
 ; AVX1-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsraw $1, %xmm0, %xmm1
 ; AVX1-NEXT:    vpaddw %xmm2, %xmm2, %xmm2
 ; AVX1-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    retq
 ;
-; AVX2-LABEL: var_shift_v8i8:
+; AVX2-LABEL: var_shift_v4i16:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpsllw $8, %xmm0, %xmm0
-; AVX2-NEXT:    vpsraw $8, %xmm0, %xmm0
-; AVX2-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
 ; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
 ; AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
 ; AVX2-NEXT:    vpsravd %ymm1, %ymm0, %ymm0
@@ -554,21 +218,15 @@ define <8 x i8> @var_shift_v8i8(<8 x i8> %a, <8 x i8> %b) nounwind {
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
-; XOP-LABEL: var_shift_v8i8:
+; XOP-LABEL: var_shift_v4i16:
 ; XOP:       # %bb.0:
-; XOP-NEXT:    vpsllw $8, %xmm0, %xmm0
-; XOP-NEXT:    vpsraw $8, %xmm0, %xmm0
-; XOP-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
 ; XOP-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; XOP-NEXT:    vpsubw %xmm1, %xmm2, %xmm1
 ; XOP-NEXT:    vpshaw %xmm1, %xmm0, %xmm0
 ; XOP-NEXT:    retq
 ;
-; AVX512DQ-LABEL: var_shift_v8i8:
+; AVX512DQ-LABEL: var_shift_v4i16:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vpsllw $8, %xmm0, %xmm0
-; AVX512DQ-NEXT:    vpsraw $8, %xmm0, %xmm0
-; AVX512DQ-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
 ; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
 ; AVX512DQ-NEXT:    vpmovsxwd %xmm0, %ymm0
 ; AVX512DQ-NEXT:    vpsravd %ymm1, %ymm0, %ymm0
@@ -577,21 +235,17 @@ define <8 x i8> @var_shift_v8i8(<8 x i8> %a, <8 x i8> %b) nounwind {
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
 ;
-; AVX512BW-LABEL: var_shift_v8i8:
+; AVX512BW-LABEL: var_shift_v4i16:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpsllw $8, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpsraw $8, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; AVX512BW-NEXT:    vpsravw %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
-; AVX512DQVL-LABEL: var_shift_v8i8:
+; AVX512DQVL-LABEL: var_shift_v4i16:
 ; AVX512DQVL:       # %bb.0:
-; AVX512DQVL-NEXT:    vpsllw $8, %xmm0, %xmm0
-; AVX512DQVL-NEXT:    vpsraw $8, %xmm0, %xmm0
-; AVX512DQVL-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
 ; AVX512DQVL-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
 ; AVX512DQVL-NEXT:    vpmovsxwd %xmm0, %ymm0
 ; AVX512DQVL-NEXT:    vpsravd %ymm1, %ymm0, %ymm0
@@ -599,27 +253,21 @@ define <8 x i8> @var_shift_v8i8(<8 x i8> %a, <8 x i8> %b) nounwind {
 ; AVX512DQVL-NEXT:    vzeroupper
 ; AVX512DQVL-NEXT:    retq
 ;
-; AVX512BWVL-LABEL: var_shift_v8i8:
+; AVX512BWVL-LABEL: var_shift_v4i16:
 ; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
-; AVX512BWVL-NEXT:    vpsllw $8, %xmm0, %xmm0
-; AVX512BWVL-NEXT:    vpsraw $8, %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    vpsravw %xmm1, %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-SSE-LABEL: var_shift_v8i8:
+; X32-SSE-LABEL: var_shift_v4i16:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm2
-; X32-SSE-NEXT:    psllw $8, %xmm2
-; X32-SSE-NEXT:    movdqa %xmm2, %xmm3
-; X32-SSE-NEXT:    psraw $8, %xmm3
 ; X32-SSE-NEXT:    psllw $12, %xmm1
-; X32-SSE-NEXT:    movdqa %xmm1, %xmm0
-; X32-SSE-NEXT:    psraw $15, %xmm0
+; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
 ; X32-SSE-NEXT:    psraw $15, %xmm2
-; X32-SSE-NEXT:    pand %xmm0, %xmm2
-; X32-SSE-NEXT:    pandn %xmm3, %xmm0
-; X32-SSE-NEXT:    por %xmm2, %xmm0
+; X32-SSE-NEXT:    movdqa %xmm2, %xmm3
+; X32-SSE-NEXT:    pandn %xmm0, %xmm3
+; X32-SSE-NEXT:    psraw $8, %xmm0
+; X32-SSE-NEXT:    pand %xmm2, %xmm0
+; X32-SSE-NEXT:    por %xmm3, %xmm0
 ; X32-SSE-NEXT:    paddw %xmm1, %xmm1
 ; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
 ; X32-SSE-NEXT:    psraw $15, %xmm2
@@ -644,138 +292,634 @@ define <8 x i8> @var_shift_v8i8(<8 x i8> %a, <8 x i8> %b) nounwind {
 ; X32-SSE-NEXT:    pand %xmm1, %xmm0
 ; X32-SSE-NEXT:    por %xmm2, %xmm0
 ; X32-SSE-NEXT:    retl
-  %shift = ashr <8 x i8> %a, %b
-  ret <8 x i8> %shift
+  %shift = ashr <4 x i16> %a, %b
+  ret <4 x i16> %shift
 }
 
-define <4 x i8> @var_shift_v4i8(<4 x i8> %a, <4 x i8> %b) nounwind {
-; SSE2-LABEL: var_shift_v4i8:
+define <2 x i16> @var_shift_v2i16(<2 x i16> %a, <2 x i16> %b) nounwind {
+; SSE2-LABEL: var_shift_v2i16:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pslld $24, %xmm0
-; SSE2-NEXT:    psrad $24, %xmm0
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm1[2,3,3,3,4,5,6,7]
-; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    psrad %xmm2, %xmm3
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm4 = xmm1[0,1,1,1,4,5,6,7]
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    psrad %xmm4, %xmm2
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm1[2,3,3,3,4,5,6,7]
-; SSE2-NEXT:    movdqa %xmm0, %xmm4
-; SSE2-NEXT:    psrad %xmm3, %xmm4
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,1,1,1,4,5,6,7]
-; SSE2-NEXT:    psrad %xmm1, %xmm0
-; SSE2-NEXT:    punpckhqdq {{.*#+}} xmm0 = xmm0[1],xmm4[1]
-; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,3],xmm0[0,3]
-; SSE2-NEXT:    movaps %xmm2, %xmm0
+; SSE2-NEXT:    psllw $12, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    psraw $15, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm3
+; SSE2-NEXT:    pandn %xmm0, %xmm3
+; SSE2-NEXT:    psraw $8, %xmm0
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    paddw %xmm1, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    psraw $15, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm3
+; SSE2-NEXT:    pandn %xmm0, %xmm3
+; SSE2-NEXT:    psraw $4, %xmm0
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    paddw %xmm1, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    psraw $15, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm3
+; SSE2-NEXT:    pandn %xmm0, %xmm3
+; SSE2-NEXT:    psraw $2, %xmm0
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    paddw %xmm1, %xmm1
+; SSE2-NEXT:    psraw $15, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    pandn %xmm0, %xmm2
+; SSE2-NEXT:    psraw $1, %xmm0
+; SSE2-NEXT:    pand %xmm1, %xmm0
+; SSE2-NEXT:    por %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: var_shift_v4i8:
+; SSE41-LABEL: var_shift_v2i16:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pslld $24, %xmm0
-; SSE41-NEXT:    psrad $24, %xmm0
-; SSE41-NEXT:    pand {{.*}}(%rip), %xmm1
-; SSE41-NEXT:    pshuflw {{.*#+}} xmm2 = xmm1[2,3,3,3,4,5,6,7]
-; SSE41-NEXT:    movdqa %xmm0, %xmm3
-; SSE41-NEXT:    psrad %xmm2, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
-; SSE41-NEXT:    pshuflw {{.*#+}} xmm4 = xmm2[2,3,3,3,4,5,6,7]
-; SSE41-NEXT:    movdqa %xmm0, %xmm5
-; SSE41-NEXT:    psrad %xmm4, %xmm5
-; SSE41-NEXT:    pblendw {{.*#+}} xmm5 = xmm3[0,1,2,3],xmm5[4,5,6,7]
-; SSE41-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,1,1,1,4,5,6,7]
-; SSE41-NEXT:    movdqa %xmm0, %xmm3
-; SSE41-NEXT:    psrad %xmm1, %xmm3
-; SSE41-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,1,1,1,4,5,6,7]
-; SSE41-NEXT:    psrad %xmm1, %xmm0
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm3[0,1,2,3],xmm0[4,5,6,7]
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm5[2,3],xmm0[4,5],xmm5[6,7]
-; SSE41-NEXT:    retq
-;
-; AVX1-LABEL: var_shift_v4i8:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpslld $24, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrad $24, %xmm0, %xmm0
-; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
-; AVX1-NEXT:    vpsrldq {{.*#+}} xmm2 = xmm1[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX1-NEXT:    vpsrad %xmm2, %xmm0, %xmm2
-; AVX1-NEXT:    vpsrlq $32, %xmm1, %xmm3
-; AVX1-NEXT:    vpsrad %xmm3, %xmm0, %xmm3
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
-; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm3 = xmm1[2],xmm3[2],xmm1[3],xmm3[3]
-; AVX1-NEXT:    vpsrad %xmm3, %xmm0, %xmm3
-; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
-; AVX1-NEXT:    vpsrad %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm3[4,5,6,7]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
+; SSE41-NEXT:    movdqa %xmm1, %xmm2
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    psllw $12, %xmm0
+; SSE41-NEXT:    psllw $4, %xmm2
+; SSE41-NEXT:    por %xmm0, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm3
+; SSE41-NEXT:    paddw %xmm2, %xmm3
+; SSE41-NEXT:    movdqa %xmm1, %xmm4
+; SSE41-NEXT:    psraw $8, %xmm4
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    pblendvb %xmm0, %xmm4, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm2
+; SSE41-NEXT:    psraw $4, %xmm2
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    pblendvb %xmm0, %xmm2, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm2
+; SSE41-NEXT:    psraw $2, %xmm2
+; SSE41-NEXT:    paddw %xmm3, %xmm3
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    pblendvb %xmm0, %xmm2, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm2
+; SSE41-NEXT:    psraw $1, %xmm2
+; SSE41-NEXT:    paddw %xmm3, %xmm3
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    pblendvb %xmm0, %xmm2, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: var_shift_v2i16:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpsllw $12, %xmm1, %xmm2
+; AVX1-NEXT:    vpsllw $4, %xmm1, %xmm1
+; AVX1-NEXT:    vpor %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpaddw %xmm1, %xmm1, %xmm2
+; AVX1-NEXT:    vpsraw $8, %xmm0, %xmm3
+; AVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpsraw $4, %xmm0, %xmm1
+; AVX1-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpsraw $2, %xmm0, %xmm1
+; AVX1-NEXT:    vpaddw %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpsraw $1, %xmm0, %xmm1
+; AVX1-NEXT:    vpaddw %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    retq
 ;
-; AVX2-LABEL: var_shift_v4i8:
+; AVX2-LABEL: var_shift_v2i16:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
-; AVX2-NEXT:    vpslld $24, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrad $24, %xmm0, %xmm0
-; AVX2-NEXT:    vpsravd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
+; AVX2-NEXT:    vpsravd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
-; XOPAVX1-LABEL: var_shift_v4i8:
-; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vpslld $24, %xmm0, %xmm0
-; XOPAVX1-NEXT:    vpsrad $24, %xmm0, %xmm0
-; XOPAVX1-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
-; XOPAVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; XOPAVX1-NEXT:    vpsubd %xmm1, %xmm2, %xmm1
-; XOPAVX1-NEXT:    vpshad %xmm1, %xmm0, %xmm0
-; XOPAVX1-NEXT:    retq
+; XOP-LABEL: var_shift_v2i16:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; XOP-NEXT:    vpsubw %xmm1, %xmm2, %xmm1
+; XOP-NEXT:    vpshaw %xmm1, %xmm0, %xmm0
+; XOP-NEXT:    retq
 ;
-; XOPAVX2-LABEL: var_shift_v4i8:
-; XOPAVX2:       # %bb.0:
-; XOPAVX2-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
-; XOPAVX2-NEXT:    vpslld $24, %xmm0, %xmm0
-; XOPAVX2-NEXT:    vpsrad $24, %xmm0, %xmm0
-; XOPAVX2-NEXT:    vpsravd %xmm1, %xmm0, %xmm0
-; XOPAVX2-NEXT:    retq
+; AVX512DQ-LABEL: var_shift_v2i16:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX512DQ-NEXT:    vpmovsxwd %xmm0, %ymm0
+; AVX512DQ-NEXT:    vpsravd %ymm1, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    retq
 ;
-; AVX512-LABEL: var_shift_v4i8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
-; AVX512-NEXT:    vpslld $24, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrad $24, %xmm0, %xmm0
-; AVX512-NEXT:    vpsravd %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    retq
+; AVX512BW-LABEL: var_shift_v2i16:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512BW-NEXT:    vpsravw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
 ;
-; AVX512VL-LABEL: var_shift_v4i8:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
-; AVX512VL-NEXT:    vpslld $24, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpsrad $24, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpsravd %xmm1, %xmm0, %xmm0
-; AVX512VL-NEXT:    retq
+; AVX512DQVL-LABEL: var_shift_v2i16:
+; AVX512DQVL:       # %bb.0:
+; AVX512DQVL-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX512DQVL-NEXT:    vpmovsxwd %xmm0, %ymm0
+; AVX512DQVL-NEXT:    vpsravd %ymm1, %ymm0, %ymm0
+; AVX512DQVL-NEXT:    vpmovdw %ymm0, %xmm0
+; AVX512DQVL-NEXT:    vzeroupper
+; AVX512DQVL-NEXT:    retq
+;
+; AVX512BWVL-LABEL: var_shift_v2i16:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpsravw %xmm1, %xmm0, %xmm0
+; AVX512BWVL-NEXT:    retq
+;
+; X32-SSE-LABEL: var_shift_v2i16:
+; X32-SSE:       # %bb.0:
+; X32-SSE-NEXT:    psllw $12, %xmm1
+; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
+; X32-SSE-NEXT:    psraw $15, %xmm2
+; X32-SSE-NEXT:    movdqa %xmm2, %xmm3
+; X32-SSE-NEXT:    pandn %xmm0, %xmm3
+; X32-SSE-NEXT:    psraw $8, %xmm0
+; X32-SSE-NEXT:    pand %xmm2, %xmm0
+; X32-SSE-NEXT:    por %xmm3, %xmm0
+; X32-SSE-NEXT:    paddw %xmm1, %xmm1
+; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
+; X32-SSE-NEXT:    psraw $15, %xmm2
+; X32-SSE-NEXT:    movdqa %xmm2, %xmm3
+; X32-SSE-NEXT:    pandn %xmm0, %xmm3
+; X32-SSE-NEXT:    psraw $4, %xmm0
+; X32-SSE-NEXT:    pand %xmm2, %xmm0
+; X32-SSE-NEXT:    por %xmm3, %xmm0
+; X32-SSE-NEXT:    paddw %xmm1, %xmm1
+; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
+; X32-SSE-NEXT:    psraw $15, %xmm2
+; X32-SSE-NEXT:    movdqa %xmm2, %xmm3
+; X32-SSE-NEXT:    pandn %xmm0, %xmm3
+; X32-SSE-NEXT:    psraw $2, %xmm0
+; X32-SSE-NEXT:    pand %xmm2, %xmm0
+; X32-SSE-NEXT:    por %xmm3, %xmm0
+; X32-SSE-NEXT:    paddw %xmm1, %xmm1
+; X32-SSE-NEXT:    psraw $15, %xmm1
+; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
+; X32-SSE-NEXT:    pandn %xmm0, %xmm2
+; X32-SSE-NEXT:    psraw $1, %xmm0
+; X32-SSE-NEXT:    pand %xmm1, %xmm0
+; X32-SSE-NEXT:    por %xmm2, %xmm0
+; X32-SSE-NEXT:    retl
+  %shift = ashr <2 x i16> %a, %b
+  ret <2 x i16> %shift
+}
+
+define <8 x i8> @var_shift_v8i8(<8 x i8> %a, <8 x i8> %b) nounwind {
+; SSE2-LABEL: var_shift_v8i8:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
+; SSE2-NEXT:    psllw $5, %xmm1
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm1[8],xmm4[9],xmm1[9],xmm4[10],xmm1[10],xmm4[11],xmm1[11],xmm4[12],xmm1[12],xmm4[13],xmm1[13],xmm4[14],xmm1[14],xmm4[15],xmm1[15]
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    pxor %xmm5, %xmm5
+; SSE2-NEXT:    pcmpgtw %xmm4, %xmm5
+; SSE2-NEXT:    movdqa %xmm5, %xmm6
+; SSE2-NEXT:    pandn %xmm2, %xmm6
+; SSE2-NEXT:    psraw $4, %xmm2
+; SSE2-NEXT:    pand %xmm5, %xmm2
+; SSE2-NEXT:    por %xmm6, %xmm2
+; SSE2-NEXT:    paddw %xmm4, %xmm4
+; SSE2-NEXT:    pxor %xmm5, %xmm5
+; SSE2-NEXT:    pcmpgtw %xmm4, %xmm5
+; SSE2-NEXT:    movdqa %xmm5, %xmm6
+; SSE2-NEXT:    pandn %xmm2, %xmm6
+; SSE2-NEXT:    psraw $2, %xmm2
+; SSE2-NEXT:    pand %xmm5, %xmm2
+; SSE2-NEXT:    por %xmm6, %xmm2
+; SSE2-NEXT:    paddw %xmm4, %xmm4
+; SSE2-NEXT:    pxor %xmm5, %xmm5
+; SSE2-NEXT:    pcmpgtw %xmm4, %xmm5
+; SSE2-NEXT:    movdqa %xmm5, %xmm4
+; SSE2-NEXT:    pandn %xmm2, %xmm4
+; SSE2-NEXT:    psraw $1, %xmm2
+; SSE2-NEXT:    pand %xmm5, %xmm2
+; SSE2-NEXT:    por %xmm4, %xmm2
+; SSE2-NEXT:    psrlw $8, %xmm2
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    pxor %xmm4, %xmm4
+; SSE2-NEXT:    pcmpgtw %xmm1, %xmm4
+; SSE2-NEXT:    movdqa %xmm4, %xmm5
+; SSE2-NEXT:    pandn %xmm0, %xmm5
+; SSE2-NEXT:    psraw $4, %xmm0
+; SSE2-NEXT:    pand %xmm4, %xmm0
+; SSE2-NEXT:    por %xmm5, %xmm0
+; SSE2-NEXT:    paddw %xmm1, %xmm1
+; SSE2-NEXT:    pxor %xmm4, %xmm4
+; SSE2-NEXT:    pcmpgtw %xmm1, %xmm4
+; SSE2-NEXT:    movdqa %xmm4, %xmm5
+; SSE2-NEXT:    pandn %xmm0, %xmm5
+; SSE2-NEXT:    psraw $2, %xmm0
+; SSE2-NEXT:    pand %xmm4, %xmm0
+; SSE2-NEXT:    por %xmm5, %xmm0
+; SSE2-NEXT:    paddw %xmm1, %xmm1
+; SSE2-NEXT:    pcmpgtw %xmm1, %xmm3
+; SSE2-NEXT:    movdqa %xmm3, %xmm1
+; SSE2-NEXT:    pandn %xmm0, %xmm1
+; SSE2-NEXT:    psraw $1, %xmm0
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    por %xmm1, %xmm0
+; SSE2-NEXT:    psrlw $8, %xmm0
+; SSE2-NEXT:    packuswb %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: var_shift_v8i8:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    psllw $5, %xmm1
+; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
+; SSE41-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm2[8],xmm3[9],xmm2[9],xmm3[10],xmm2[10],xmm3[11],xmm2[11],xmm3[12],xmm2[12],xmm3[13],xmm2[13],xmm3[14],xmm2[14],xmm3[15],xmm2[15]
+; SSE41-NEXT:    movdqa %xmm3, %xmm4
+; SSE41-NEXT:    psraw $4, %xmm4
+; SSE41-NEXT:    pblendvb %xmm0, %xmm4, %xmm3
+; SSE41-NEXT:    movdqa %xmm3, %xmm4
+; SSE41-NEXT:    psraw $2, %xmm4
+; SSE41-NEXT:    paddw %xmm0, %xmm0
+; SSE41-NEXT:    pblendvb %xmm0, %xmm4, %xmm3
+; SSE41-NEXT:    movdqa %xmm3, %xmm4
+; SSE41-NEXT:    psraw $1, %xmm4
+; SSE41-NEXT:    paddw %xmm0, %xmm0
+; SSE41-NEXT:    pblendvb %xmm0, %xmm4, %xmm3
+; SSE41-NEXT:    psrlw $8, %xmm3
+; SSE41-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSE41-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; SSE41-NEXT:    movdqa %xmm1, %xmm2
+; SSE41-NEXT:    psraw $4, %xmm2
+; SSE41-NEXT:    pblendvb %xmm0, %xmm2, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm2
+; SSE41-NEXT:    psraw $2, %xmm2
+; SSE41-NEXT:    paddw %xmm0, %xmm0
+; SSE41-NEXT:    pblendvb %xmm0, %xmm2, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm2
+; SSE41-NEXT:    psraw $1, %xmm2
+; SSE41-NEXT:    paddw %xmm0, %xmm0
+; SSE41-NEXT:    pblendvb %xmm0, %xmm2, %xmm1
+; SSE41-NEXT:    psrlw $8, %xmm1
+; SSE41-NEXT:    packuswb %xmm3, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: var_shift_v8i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpsllw $5, %xmm1, %xmm1
+; AVX-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
+; AVX-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX-NEXT:    vpsraw $4, %xmm3, %xmm4
+; AVX-NEXT:    vpblendvb %xmm2, %xmm4, %xmm3, %xmm3
+; AVX-NEXT:    vpsraw $2, %xmm3, %xmm4
+; AVX-NEXT:    vpaddw %xmm2, %xmm2, %xmm2
+; AVX-NEXT:    vpblendvb %xmm2, %xmm4, %xmm3, %xmm3
+; AVX-NEXT:    vpsraw $1, %xmm3, %xmm4
+; AVX-NEXT:    vpaddw %xmm2, %xmm2, %xmm2
+; AVX-NEXT:    vpblendvb %xmm2, %xmm4, %xmm3, %xmm2
+; AVX-NEXT:    vpsrlw $8, %xmm2, %xmm2
+; AVX-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; AVX-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; AVX-NEXT:    vpsraw $4, %xmm0, %xmm3
+; AVX-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
+; AVX-NEXT:    vpsraw $2, %xmm0, %xmm3
+; AVX-NEXT:    vpaddw %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
+; AVX-NEXT:    vpsraw $1, %xmm0, %xmm3
+; AVX-NEXT:    vpaddw %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
+; AVX-NEXT:    vpsrlw $8, %xmm0, %xmm0
+; AVX-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; XOP-LABEL: var_shift_v8i8:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; XOP-NEXT:    vpsubb %xmm1, %xmm2, %xmm1
+; XOP-NEXT:    vpshab %xmm1, %xmm0, %xmm0
+; XOP-NEXT:    retq
+;
+; AVX512DQ-LABEL: var_shift_v8i8:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; AVX512DQ-NEXT:    vpmovsxbd %xmm0, %zmm0
+; AVX512DQ-NEXT:    vpsravd %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    retq
+;
+; AVX512BW-LABEL: var_shift_v8i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512BW-NEXT:    vpmovsxbw %xmm0, %ymm0
+; AVX512BW-NEXT:    vpsravw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512DQVL-LABEL: var_shift_v8i8:
+; AVX512DQVL:       # %bb.0:
+; AVX512DQVL-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; AVX512DQVL-NEXT:    vpmovsxbd %xmm0, %zmm0
+; AVX512DQVL-NEXT:    vpsravd %zmm1, %zmm0, %zmm0
+; AVX512DQVL-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512DQVL-NEXT:    vzeroupper
+; AVX512DQVL-NEXT:    retq
+;
+; AVX512BWVL-LABEL: var_shift_v8i8:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512BWVL-NEXT:    vpmovsxbw %xmm0, %ymm0
+; AVX512BWVL-NEXT:    vpsravw %ymm1, %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm0
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
+;
+; X32-SSE-LABEL: var_shift_v8i8:
+; X32-SSE:       # %bb.0:
+; X32-SSE-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
+; X32-SSE-NEXT:    psllw $5, %xmm1
+; X32-SSE-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm1[8],xmm4[9],xmm1[9],xmm4[10],xmm1[10],xmm4[11],xmm1[11],xmm4[12],xmm1[12],xmm4[13],xmm1[13],xmm4[14],xmm1[14],xmm4[15],xmm1[15]
+; X32-SSE-NEXT:    pxor %xmm3, %xmm3
+; X32-SSE-NEXT:    pxor %xmm5, %xmm5
+; X32-SSE-NEXT:    pcmpgtw %xmm4, %xmm5
+; X32-SSE-NEXT:    movdqa %xmm5, %xmm6
+; X32-SSE-NEXT:    pandn %xmm2, %xmm6
+; X32-SSE-NEXT:    psraw $4, %xmm2
+; X32-SSE-NEXT:    pand %xmm5, %xmm2
+; X32-SSE-NEXT:    por %xmm6, %xmm2
+; X32-SSE-NEXT:    paddw %xmm4, %xmm4
+; X32-SSE-NEXT:    pxor %xmm5, %xmm5
+; X32-SSE-NEXT:    pcmpgtw %xmm4, %xmm5
+; X32-SSE-NEXT:    movdqa %xmm5, %xmm6
+; X32-SSE-NEXT:    pandn %xmm2, %xmm6
+; X32-SSE-NEXT:    psraw $2, %xmm2
+; X32-SSE-NEXT:    pand %xmm5, %xmm2
+; X32-SSE-NEXT:    por %xmm6, %xmm2
+; X32-SSE-NEXT:    paddw %xmm4, %xmm4
+; X32-SSE-NEXT:    pxor %xmm5, %xmm5
+; X32-SSE-NEXT:    pcmpgtw %xmm4, %xmm5
+; X32-SSE-NEXT:    movdqa %xmm5, %xmm4
+; X32-SSE-NEXT:    pandn %xmm2, %xmm4
+; X32-SSE-NEXT:    psraw $1, %xmm2
+; X32-SSE-NEXT:    pand %xmm5, %xmm2
+; X32-SSE-NEXT:    por %xmm4, %xmm2
+; X32-SSE-NEXT:    psrlw $8, %xmm2
+; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X32-SSE-NEXT:    pxor %xmm4, %xmm4
+; X32-SSE-NEXT:    pcmpgtw %xmm1, %xmm4
+; X32-SSE-NEXT:    movdqa %xmm4, %xmm5
+; X32-SSE-NEXT:    pandn %xmm0, %xmm5
+; X32-SSE-NEXT:    psraw $4, %xmm0
+; X32-SSE-NEXT:    pand %xmm4, %xmm0
+; X32-SSE-NEXT:    por %xmm5, %xmm0
+; X32-SSE-NEXT:    paddw %xmm1, %xmm1
+; X32-SSE-NEXT:    pxor %xmm4, %xmm4
+; X32-SSE-NEXT:    pcmpgtw %xmm1, %xmm4
+; X32-SSE-NEXT:    movdqa %xmm4, %xmm5
+; X32-SSE-NEXT:    pandn %xmm0, %xmm5
+; X32-SSE-NEXT:    psraw $2, %xmm0
+; X32-SSE-NEXT:    pand %xmm4, %xmm0
+; X32-SSE-NEXT:    por %xmm5, %xmm0
+; X32-SSE-NEXT:    paddw %xmm1, %xmm1
+; X32-SSE-NEXT:    pcmpgtw %xmm1, %xmm3
+; X32-SSE-NEXT:    movdqa %xmm3, %xmm1
+; X32-SSE-NEXT:    pandn %xmm0, %xmm1
+; X32-SSE-NEXT:    psraw $1, %xmm0
+; X32-SSE-NEXT:    pand %xmm3, %xmm0
+; X32-SSE-NEXT:    por %xmm1, %xmm0
+; X32-SSE-NEXT:    psrlw $8, %xmm0
+; X32-SSE-NEXT:    packuswb %xmm2, %xmm0
+; X32-SSE-NEXT:    retl
+  %shift = ashr <8 x i8> %a, %b
+  ret <8 x i8> %shift
+}
+
+define <4 x i8> @var_shift_v4i8(<4 x i8> %a, <4 x i8> %b) nounwind {
+; SSE2-LABEL: var_shift_v4i8:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
+; SSE2-NEXT:    psllw $5, %xmm1
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm1[8],xmm4[9],xmm1[9],xmm4[10],xmm1[10],xmm4[11],xmm1[11],xmm4[12],xmm1[12],xmm4[13],xmm1[13],xmm4[14],xmm1[14],xmm4[15],xmm1[15]
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    pxor %xmm5, %xmm5
+; SSE2-NEXT:    pcmpgtw %xmm4, %xmm5
+; SSE2-NEXT:    movdqa %xmm5, %xmm6
+; SSE2-NEXT:    pandn %xmm2, %xmm6
+; SSE2-NEXT:    psraw $4, %xmm2
+; SSE2-NEXT:    pand %xmm5, %xmm2
+; SSE2-NEXT:    por %xmm6, %xmm2
+; SSE2-NEXT:    paddw %xmm4, %xmm4
+; SSE2-NEXT:    pxor %xmm5, %xmm5
+; SSE2-NEXT:    pcmpgtw %xmm4, %xmm5
+; SSE2-NEXT:    movdqa %xmm5, %xmm6
+; SSE2-NEXT:    pandn %xmm2, %xmm6
+; SSE2-NEXT:    psraw $2, %xmm2
+; SSE2-NEXT:    pand %xmm5, %xmm2
+; SSE2-NEXT:    por %xmm6, %xmm2
+; SSE2-NEXT:    paddw %xmm4, %xmm4
+; SSE2-NEXT:    pxor %xmm5, %xmm5
+; SSE2-NEXT:    pcmpgtw %xmm4, %xmm5
+; SSE2-NEXT:    movdqa %xmm5, %xmm4
+; SSE2-NEXT:    pandn %xmm2, %xmm4
+; SSE2-NEXT:    psraw $1, %xmm2
+; SSE2-NEXT:    pand %xmm5, %xmm2
+; SSE2-NEXT:    por %xmm4, %xmm2
+; SSE2-NEXT:    psrlw $8, %xmm2
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    pxor %xmm4, %xmm4
+; SSE2-NEXT:    pcmpgtw %xmm1, %xmm4
+; SSE2-NEXT:    movdqa %xmm4, %xmm5
+; SSE2-NEXT:    pandn %xmm0, %xmm5
+; SSE2-NEXT:    psraw $4, %xmm0
+; SSE2-NEXT:    pand %xmm4, %xmm0
+; SSE2-NEXT:    por %xmm5, %xmm0
+; SSE2-NEXT:    paddw %xmm1, %xmm1
+; SSE2-NEXT:    pxor %xmm4, %xmm4
+; SSE2-NEXT:    pcmpgtw %xmm1, %xmm4
+; SSE2-NEXT:    movdqa %xmm4, %xmm5
+; SSE2-NEXT:    pandn %xmm0, %xmm5
+; SSE2-NEXT:    psraw $2, %xmm0
+; SSE2-NEXT:    pand %xmm4, %xmm0
+; SSE2-NEXT:    por %xmm5, %xmm0
+; SSE2-NEXT:    paddw %xmm1, %xmm1
+; SSE2-NEXT:    pcmpgtw %xmm1, %xmm3
+; SSE2-NEXT:    movdqa %xmm3, %xmm1
+; SSE2-NEXT:    pandn %xmm0, %xmm1
+; SSE2-NEXT:    psraw $1, %xmm0
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    por %xmm1, %xmm0
+; SSE2-NEXT:    psrlw $8, %xmm0
+; SSE2-NEXT:    packuswb %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: var_shift_v4i8:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    psllw $5, %xmm1
+; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
+; SSE41-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm2[8],xmm3[9],xmm2[9],xmm3[10],xmm2[10],xmm3[11],xmm2[11],xmm3[12],xmm2[12],xmm3[13],xmm2[13],xmm3[14],xmm2[14],xmm3[15],xmm2[15]
+; SSE41-NEXT:    movdqa %xmm3, %xmm4
+; SSE41-NEXT:    psraw $4, %xmm4
+; SSE41-NEXT:    pblendvb %xmm0, %xmm4, %xmm3
+; SSE41-NEXT:    movdqa %xmm3, %xmm4
+; SSE41-NEXT:    psraw $2, %xmm4
+; SSE41-NEXT:    paddw %xmm0, %xmm0
+; SSE41-NEXT:    pblendvb %xmm0, %xmm4, %xmm3
+; SSE41-NEXT:    movdqa %xmm3, %xmm4
+; SSE41-NEXT:    psraw $1, %xmm4
+; SSE41-NEXT:    paddw %xmm0, %xmm0
+; SSE41-NEXT:    pblendvb %xmm0, %xmm4, %xmm3
+; SSE41-NEXT:    psrlw $8, %xmm3
+; SSE41-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSE41-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; SSE41-NEXT:    movdqa %xmm1, %xmm2
+; SSE41-NEXT:    psraw $4, %xmm2
+; SSE41-NEXT:    pblendvb %xmm0, %xmm2, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm2
+; SSE41-NEXT:    psraw $2, %xmm2
+; SSE41-NEXT:    paddw %xmm0, %xmm0
+; SSE41-NEXT:    pblendvb %xmm0, %xmm2, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm2
+; SSE41-NEXT:    psraw $1, %xmm2
+; SSE41-NEXT:    paddw %xmm0, %xmm0
+; SSE41-NEXT:    pblendvb %xmm0, %xmm2, %xmm1
+; SSE41-NEXT:    psrlw $8, %xmm1
+; SSE41-NEXT:    packuswb %xmm3, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: var_shift_v4i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpsllw $5, %xmm1, %xmm1
+; AVX-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
+; AVX-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX-NEXT:    vpsraw $4, %xmm3, %xmm4
+; AVX-NEXT:    vpblendvb %xmm2, %xmm4, %xmm3, %xmm3
+; AVX-NEXT:    vpsraw $2, %xmm3, %xmm4
+; AVX-NEXT:    vpaddw %xmm2, %xmm2, %xmm2
+; AVX-NEXT:    vpblendvb %xmm2, %xmm4, %xmm3, %xmm3
+; AVX-NEXT:    vpsraw $1, %xmm3, %xmm4
+; AVX-NEXT:    vpaddw %xmm2, %xmm2, %xmm2
+; AVX-NEXT:    vpblendvb %xmm2, %xmm4, %xmm3, %xmm2
+; AVX-NEXT:    vpsrlw $8, %xmm2, %xmm2
+; AVX-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; AVX-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; AVX-NEXT:    vpsraw $4, %xmm0, %xmm3
+; AVX-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
+; AVX-NEXT:    vpsraw $2, %xmm0, %xmm3
+; AVX-NEXT:    vpaddw %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
+; AVX-NEXT:    vpsraw $1, %xmm0, %xmm3
+; AVX-NEXT:    vpaddw %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
+; AVX-NEXT:    vpsrlw $8, %xmm0, %xmm0
+; AVX-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; XOP-LABEL: var_shift_v4i8:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; XOP-NEXT:    vpsubb %xmm1, %xmm2, %xmm1
+; XOP-NEXT:    vpshab %xmm1, %xmm0, %xmm0
+; XOP-NEXT:    retq
+;
+; AVX512DQ-LABEL: var_shift_v4i8:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; AVX512DQ-NEXT:    vpmovsxbd %xmm0, %zmm0
+; AVX512DQ-NEXT:    vpsravd %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    retq
+;
+; AVX512BW-LABEL: var_shift_v4i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512BW-NEXT:    vpmovsxbw %xmm0, %ymm0
+; AVX512BW-NEXT:    vpsravw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512DQVL-LABEL: var_shift_v4i8:
+; AVX512DQVL:       # %bb.0:
+; AVX512DQVL-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; AVX512DQVL-NEXT:    vpmovsxbd %xmm0, %zmm0
+; AVX512DQVL-NEXT:    vpsravd %zmm1, %zmm0, %zmm0
+; AVX512DQVL-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512DQVL-NEXT:    vzeroupper
+; AVX512DQVL-NEXT:    retq
+;
+; AVX512BWVL-LABEL: var_shift_v4i8:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512BWVL-NEXT:    vpmovsxbw %xmm0, %ymm0
+; AVX512BWVL-NEXT:    vpsravw %ymm1, %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm0
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
 ;
 ; X32-SSE-LABEL: var_shift_v4i8:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pslld $24, %xmm0
-; X32-SSE-NEXT:    psrad $24, %xmm0
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm1
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm1[2,3,3,3,4,5,6,7]
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm3
-; X32-SSE-NEXT:    psrad %xmm2, %xmm3
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm4 = xmm1[0,1,1,1,4,5,6,7]
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm2
-; X32-SSE-NEXT:    psrad %xmm4, %xmm2
-; X32-SSE-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm3 = xmm1[2,3,3,3,4,5,6,7]
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm4
-; X32-SSE-NEXT:    psrad %xmm3, %xmm4
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,1,1,1,4,5,6,7]
-; X32-SSE-NEXT:    psrad %xmm1, %xmm0
-; X32-SSE-NEXT:    punpckhqdq {{.*#+}} xmm0 = xmm0[1],xmm4[1]
-; X32-SSE-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,3],xmm0[0,3]
-; X32-SSE-NEXT:    movaps %xmm2, %xmm0
+; X32-SSE-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
+; X32-SSE-NEXT:    psllw $5, %xmm1
+; X32-SSE-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm1[8],xmm4[9],xmm1[9],xmm4[10],xmm1[10],xmm4[11],xmm1[11],xmm4[12],xmm1[12],xmm4[13],xmm1[13],xmm4[14],xmm1[14],xmm4[15],xmm1[15]
+; X32-SSE-NEXT:    pxor %xmm3, %xmm3
+; X32-SSE-NEXT:    pxor %xmm5, %xmm5
+; X32-SSE-NEXT:    pcmpgtw %xmm4, %xmm5
+; X32-SSE-NEXT:    movdqa %xmm5, %xmm6
+; X32-SSE-NEXT:    pandn %xmm2, %xmm6
+; X32-SSE-NEXT:    psraw $4, %xmm2
+; X32-SSE-NEXT:    pand %xmm5, %xmm2
+; X32-SSE-NEXT:    por %xmm6, %xmm2
+; X32-SSE-NEXT:    paddw %xmm4, %xmm4
+; X32-SSE-NEXT:    pxor %xmm5, %xmm5
+; X32-SSE-NEXT:    pcmpgtw %xmm4, %xmm5
+; X32-SSE-NEXT:    movdqa %xmm5, %xmm6
+; X32-SSE-NEXT:    pandn %xmm2, %xmm6
+; X32-SSE-NEXT:    psraw $2, %xmm2
+; X32-SSE-NEXT:    pand %xmm5, %xmm2
+; X32-SSE-NEXT:    por %xmm6, %xmm2
+; X32-SSE-NEXT:    paddw %xmm4, %xmm4
+; X32-SSE-NEXT:    pxor %xmm5, %xmm5
+; X32-SSE-NEXT:    pcmpgtw %xmm4, %xmm5
+; X32-SSE-NEXT:    movdqa %xmm5, %xmm4
+; X32-SSE-NEXT:    pandn %xmm2, %xmm4
+; X32-SSE-NEXT:    psraw $1, %xmm2
+; X32-SSE-NEXT:    pand %xmm5, %xmm2
+; X32-SSE-NEXT:    por %xmm4, %xmm2
+; X32-SSE-NEXT:    psrlw $8, %xmm2
+; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X32-SSE-NEXT:    pxor %xmm4, %xmm4
+; X32-SSE-NEXT:    pcmpgtw %xmm1, %xmm4
+; X32-SSE-NEXT:    movdqa %xmm4, %xmm5
+; X32-SSE-NEXT:    pandn %xmm0, %xmm5
+; X32-SSE-NEXT:    psraw $4, %xmm0
+; X32-SSE-NEXT:    pand %xmm4, %xmm0
+; X32-SSE-NEXT:    por %xmm5, %xmm0
+; X32-SSE-NEXT:    paddw %xmm1, %xmm1
+; X32-SSE-NEXT:    pxor %xmm4, %xmm4
+; X32-SSE-NEXT:    pcmpgtw %xmm1, %xmm4
+; X32-SSE-NEXT:    movdqa %xmm4, %xmm5
+; X32-SSE-NEXT:    pandn %xmm0, %xmm5
+; X32-SSE-NEXT:    psraw $2, %xmm0
+; X32-SSE-NEXT:    pand %xmm4, %xmm0
+; X32-SSE-NEXT:    por %xmm5, %xmm0
+; X32-SSE-NEXT:    paddw %xmm1, %xmm1
+; X32-SSE-NEXT:    pcmpgtw %xmm1, %xmm3
+; X32-SSE-NEXT:    movdqa %xmm3, %xmm1
+; X32-SSE-NEXT:    pandn %xmm0, %xmm1
+; X32-SSE-NEXT:    psraw $1, %xmm0
+; X32-SSE-NEXT:    pand %xmm3, %xmm0
+; X32-SSE-NEXT:    por %xmm1, %xmm0
+; X32-SSE-NEXT:    psrlw $8, %xmm0
+; X32-SSE-NEXT:    packuswb %xmm2, %xmm0
 ; X32-SSE-NEXT:    retl
   %shift = ashr <4 x i8> %a, %b
   ret <4 x i8> %shift
@@ -784,136 +928,226 @@ define <4 x i8> @var_shift_v4i8(<4 x i8> %a, <4 x i8> %b) nounwind {
 define <2 x i8> @var_shift_v2i8(<2 x i8> %a, <2 x i8> %b) nounwind {
 ; SSE2-LABEL: var_shift_v2i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    psllq $56, %xmm0
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    psrad $31, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
-; SSE2-NEXT:    psrad $24, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
-; SSE2-NEXT:    movdqa %xmm2, %xmm3
-; SSE2-NEXT:    psrlq %xmm1, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[2,3,0,1]
-; SSE2-NEXT:    psrlq %xmm4, %xmm2
-; SSE2-NEXT:    movsd {{.*#+}} xmm2 = xmm3[0],xmm2[1]
-; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    psrlq %xmm1, %xmm3
-; SSE2-NEXT:    psrlq %xmm4, %xmm0
-; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm3[0],xmm0[1]
-; SSE2-NEXT:    xorpd %xmm2, %xmm0
-; SSE2-NEXT:    psubq %xmm2, %xmm0
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
+; SSE2-NEXT:    psllw $5, %xmm1
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm1[8],xmm4[9],xmm1[9],xmm4[10],xmm1[10],xmm4[11],xmm1[11],xmm4[12],xmm1[12],xmm4[13],xmm1[13],xmm4[14],xmm1[14],xmm4[15],xmm1[15]
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    pxor %xmm5, %xmm5
+; SSE2-NEXT:    pcmpgtw %xmm4, %xmm5
+; SSE2-NEXT:    movdqa %xmm5, %xmm6
+; SSE2-NEXT:    pandn %xmm2, %xmm6
+; SSE2-NEXT:    psraw $4, %xmm2
+; SSE2-NEXT:    pand %xmm5, %xmm2
+; SSE2-NEXT:    por %xmm6, %xmm2
+; SSE2-NEXT:    paddw %xmm4, %xmm4
+; SSE2-NEXT:    pxor %xmm5, %xmm5
+; SSE2-NEXT:    pcmpgtw %xmm4, %xmm5
+; SSE2-NEXT:    movdqa %xmm5, %xmm6
+; SSE2-NEXT:    pandn %xmm2, %xmm6
+; SSE2-NEXT:    psraw $2, %xmm2
+; SSE2-NEXT:    pand %xmm5, %xmm2
+; SSE2-NEXT:    por %xmm6, %xmm2
+; SSE2-NEXT:    paddw %xmm4, %xmm4
+; SSE2-NEXT:    pxor %xmm5, %xmm5
+; SSE2-NEXT:    pcmpgtw %xmm4, %xmm5
+; SSE2-NEXT:    movdqa %xmm5, %xmm4
+; SSE2-NEXT:    pandn %xmm2, %xmm4
+; SSE2-NEXT:    psraw $1, %xmm2
+; SSE2-NEXT:    pand %xmm5, %xmm2
+; SSE2-NEXT:    por %xmm4, %xmm2
+; SSE2-NEXT:    psrlw $8, %xmm2
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    pxor %xmm4, %xmm4
+; SSE2-NEXT:    pcmpgtw %xmm1, %xmm4
+; SSE2-NEXT:    movdqa %xmm4, %xmm5
+; SSE2-NEXT:    pandn %xmm0, %xmm5
+; SSE2-NEXT:    psraw $4, %xmm0
+; SSE2-NEXT:    pand %xmm4, %xmm0
+; SSE2-NEXT:    por %xmm5, %xmm0
+; SSE2-NEXT:    paddw %xmm1, %xmm1
+; SSE2-NEXT:    pxor %xmm4, %xmm4
+; SSE2-NEXT:    pcmpgtw %xmm1, %xmm4
+; SSE2-NEXT:    movdqa %xmm4, %xmm5
+; SSE2-NEXT:    pandn %xmm0, %xmm5
+; SSE2-NEXT:    psraw $2, %xmm0
+; SSE2-NEXT:    pand %xmm4, %xmm0
+; SSE2-NEXT:    por %xmm5, %xmm0
+; SSE2-NEXT:    paddw %xmm1, %xmm1
+; SSE2-NEXT:    pcmpgtw %xmm1, %xmm3
+; SSE2-NEXT:    movdqa %xmm3, %xmm1
+; SSE2-NEXT:    pandn %xmm0, %xmm1
+; SSE2-NEXT:    psraw $1, %xmm0
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    por %xmm1, %xmm0
+; SSE2-NEXT:    psrlw $8, %xmm0
+; SSE2-NEXT:    packuswb %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: var_shift_v2i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    psllq $56, %xmm0
 ; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    psrad $31, %xmm2
-; SSE41-NEXT:    psrad $24, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
-; SSE41-NEXT:    pand {{.*}}(%rip), %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
-; SSE41-NEXT:    movdqa %xmm2, %xmm3
-; SSE41-NEXT:    psrlq %xmm1, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[2,3,0,1]
-; SSE41-NEXT:    psrlq %xmm4, %xmm2
-; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
-; SSE41-NEXT:    movdqa %xmm0, %xmm3
-; SSE41-NEXT:    psrlq %xmm4, %xmm3
-; SSE41-NEXT:    psrlq %xmm1, %xmm0
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm3[4,5,6,7]
-; SSE41-NEXT:    pxor %xmm2, %xmm0
-; SSE41-NEXT:    psubq %xmm2, %xmm0
+; SSE41-NEXT:    psllw $5, %xmm1
+; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
+; SSE41-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm2[8],xmm3[9],xmm2[9],xmm3[10],xmm2[10],xmm3[11],xmm2[11],xmm3[12],xmm2[12],xmm3[13],xmm2[13],xmm3[14],xmm2[14],xmm3[15],xmm2[15]
+; SSE41-NEXT:    movdqa %xmm3, %xmm4
+; SSE41-NEXT:    psraw $4, %xmm4
+; SSE41-NEXT:    pblendvb %xmm0, %xmm4, %xmm3
+; SSE41-NEXT:    movdqa %xmm3, %xmm4
+; SSE41-NEXT:    psraw $2, %xmm4
+; SSE41-NEXT:    paddw %xmm0, %xmm0
+; SSE41-NEXT:    pblendvb %xmm0, %xmm4, %xmm3
+; SSE41-NEXT:    movdqa %xmm3, %xmm4
+; SSE41-NEXT:    psraw $1, %xmm4
+; SSE41-NEXT:    paddw %xmm0, %xmm0
+; SSE41-NEXT:    pblendvb %xmm0, %xmm4, %xmm3
+; SSE41-NEXT:    psrlw $8, %xmm3
+; SSE41-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSE41-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; SSE41-NEXT:    movdqa %xmm1, %xmm2
+; SSE41-NEXT:    psraw $4, %xmm2
+; SSE41-NEXT:    pblendvb %xmm0, %xmm2, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm2
+; SSE41-NEXT:    psraw $2, %xmm2
+; SSE41-NEXT:    paddw %xmm0, %xmm0
+; SSE41-NEXT:    pblendvb %xmm0, %xmm2, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm2
+; SSE41-NEXT:    psraw $1, %xmm2
+; SSE41-NEXT:    paddw %xmm0, %xmm0
+; SSE41-NEXT:    pblendvb %xmm0, %xmm2, %xmm1
+; SSE41-NEXT:    psrlw $8, %xmm1
+; SSE41-NEXT:    packuswb %xmm3, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: var_shift_v2i8:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpsllq $56, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrad $31, %xmm0, %xmm2
-; AVX1-NEXT:    vpsrad $24, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
-; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
-; AVX1-NEXT:    vpsrlq %xmm1, %xmm2, %xmm3
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm1[2,3,0,1]
-; AVX1-NEXT:    vpsrlq %xmm4, %xmm2, %xmm2
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
-; AVX1-NEXT:    vpsrlq %xmm4, %xmm0, %xmm3
-; AVX1-NEXT:    vpsrlq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm3[4,5,6,7]
-; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpsubq %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: var_shift_v2i8:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpsllq $56, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrad $31, %xmm0, %xmm2
-; AVX2-NEXT:    vpsrad $24, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
-; AVX2-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
-; AVX2-NEXT:    vpsrlvq %xmm1, %xmm2, %xmm2
-; AVX2-NEXT:    vpsrlvq %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm0
-; AVX2-NEXT:    vpsubq %xmm2, %xmm0, %xmm0
-; AVX2-NEXT:    retq
+; AVX-LABEL: var_shift_v2i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpsllw $5, %xmm1, %xmm1
+; AVX-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
+; AVX-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX-NEXT:    vpsraw $4, %xmm3, %xmm4
+; AVX-NEXT:    vpblendvb %xmm2, %xmm4, %xmm3, %xmm3
+; AVX-NEXT:    vpsraw $2, %xmm3, %xmm4
+; AVX-NEXT:    vpaddw %xmm2, %xmm2, %xmm2
+; AVX-NEXT:    vpblendvb %xmm2, %xmm4, %xmm3, %xmm3
+; AVX-NEXT:    vpsraw $1, %xmm3, %xmm4
+; AVX-NEXT:    vpaddw %xmm2, %xmm2, %xmm2
+; AVX-NEXT:    vpblendvb %xmm2, %xmm4, %xmm3, %xmm2
+; AVX-NEXT:    vpsrlw $8, %xmm2, %xmm2
+; AVX-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; AVX-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; AVX-NEXT:    vpsraw $4, %xmm0, %xmm3
+; AVX-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
+; AVX-NEXT:    vpsraw $2, %xmm0, %xmm3
+; AVX-NEXT:    vpaddw %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
+; AVX-NEXT:    vpsraw $1, %xmm0, %xmm3
+; AVX-NEXT:    vpaddw %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
+; AVX-NEXT:    vpsrlw $8, %xmm0, %xmm0
+; AVX-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    retq
 ;
 ; XOP-LABEL: var_shift_v2i8:
 ; XOP:       # %bb.0:
-; XOP-NEXT:    vpsllq $56, %xmm0, %xmm0
-; XOP-NEXT:    vpshaq {{.*}}(%rip), %xmm0, %xmm0
-; XOP-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
 ; XOP-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; XOP-NEXT:    vpsubq %xmm1, %xmm2, %xmm1
-; XOP-NEXT:    vpshaq %xmm1, %xmm0, %xmm0
+; XOP-NEXT:    vpsubb %xmm1, %xmm2, %xmm1
+; XOP-NEXT:    vpshab %xmm1, %xmm0, %xmm0
 ; XOP-NEXT:    retq
 ;
-; AVX512-LABEL: var_shift_v2i8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsllq $56, %xmm0, %xmm0
-; AVX512-NEXT:    vpsraq $56, %zmm0, %zmm0
-; AVX512-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
-; AVX512-NEXT:    vpsravq %zmm1, %zmm0, %zmm0
-; AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
-; AVX512-NEXT:    vzeroupper
-; AVX512-NEXT:    retq
+; AVX512DQ-LABEL: var_shift_v2i8:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; AVX512DQ-NEXT:    vpmovsxbd %xmm0, %zmm0
+; AVX512DQ-NEXT:    vpsravd %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    retq
 ;
-; AVX512VL-LABEL: var_shift_v2i8:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
-; AVX512VL-NEXT:    vpsllq $56, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpsraq $56, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpsravq %xmm1, %xmm0, %xmm0
-; AVX512VL-NEXT:    retq
+; AVX512BW-LABEL: var_shift_v2i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512BW-NEXT:    vpmovsxbw %xmm0, %ymm0
+; AVX512BW-NEXT:    vpsravw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512DQVL-LABEL: var_shift_v2i8:
+; AVX512DQVL:       # %bb.0:
+; AVX512DQVL-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; AVX512DQVL-NEXT:    vpmovsxbd %xmm0, %zmm0
+; AVX512DQVL-NEXT:    vpsravd %zmm1, %zmm0, %zmm0
+; AVX512DQVL-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512DQVL-NEXT:    vzeroupper
+; AVX512DQVL-NEXT:    retq
+;
+; AVX512BWVL-LABEL: var_shift_v2i8:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512BWVL-NEXT:    vpmovsxbw %xmm0, %ymm0
+; AVX512BWVL-NEXT:    vpsravw %ymm1, %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm0
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
 ;
 ; X32-SSE-LABEL: var_shift_v2i8:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    psllq $56, %xmm0
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm2
-; X32-SSE-NEXT:    psrad $31, %xmm2
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
-; X32-SSE-NEXT:    psrad $24, %xmm0
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
-; X32-SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm1
-; X32-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [0,2147483648,0,2147483648]
-; X32-SSE-NEXT:    movdqa %xmm2, %xmm3
-; X32-SSE-NEXT:    psrlq %xmm1, %xmm3
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[2,3,0,1]
-; X32-SSE-NEXT:    psrlq %xmm4, %xmm2
-; X32-SSE-NEXT:    movsd {{.*#+}} xmm2 = xmm3[0],xmm2[1]
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm3
-; X32-SSE-NEXT:    psrlq %xmm1, %xmm3
-; X32-SSE-NEXT:    psrlq %xmm4, %xmm0
-; X32-SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm3[0],xmm0[1]
-; X32-SSE-NEXT:    xorpd %xmm2, %xmm0
-; X32-SSE-NEXT:    psubq %xmm2, %xmm0
+; X32-SSE-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
+; X32-SSE-NEXT:    psllw $5, %xmm1
+; X32-SSE-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm1[8],xmm4[9],xmm1[9],xmm4[10],xmm1[10],xmm4[11],xmm1[11],xmm4[12],xmm1[12],xmm4[13],xmm1[13],xmm4[14],xmm1[14],xmm4[15],xmm1[15]
+; X32-SSE-NEXT:    pxor %xmm3, %xmm3
+; X32-SSE-NEXT:    pxor %xmm5, %xmm5
+; X32-SSE-NEXT:    pcmpgtw %xmm4, %xmm5
+; X32-SSE-NEXT:    movdqa %xmm5, %xmm6
+; X32-SSE-NEXT:    pandn %xmm2, %xmm6
+; X32-SSE-NEXT:    psraw $4, %xmm2
+; X32-SSE-NEXT:    pand %xmm5, %xmm2
+; X32-SSE-NEXT:    por %xmm6, %xmm2
+; X32-SSE-NEXT:    paddw %xmm4, %xmm4
+; X32-SSE-NEXT:    pxor %xmm5, %xmm5
+; X32-SSE-NEXT:    pcmpgtw %xmm4, %xmm5
+; X32-SSE-NEXT:    movdqa %xmm5, %xmm6
+; X32-SSE-NEXT:    pandn %xmm2, %xmm6
+; X32-SSE-NEXT:    psraw $2, %xmm2
+; X32-SSE-NEXT:    pand %xmm5, %xmm2
+; X32-SSE-NEXT:    por %xmm6, %xmm2
+; X32-SSE-NEXT:    paddw %xmm4, %xmm4
+; X32-SSE-NEXT:    pxor %xmm5, %xmm5
+; X32-SSE-NEXT:    pcmpgtw %xmm4, %xmm5
+; X32-SSE-NEXT:    movdqa %xmm5, %xmm4
+; X32-SSE-NEXT:    pandn %xmm2, %xmm4
+; X32-SSE-NEXT:    psraw $1, %xmm2
+; X32-SSE-NEXT:    pand %xmm5, %xmm2
+; X32-SSE-NEXT:    por %xmm4, %xmm2
+; X32-SSE-NEXT:    psrlw $8, %xmm2
+; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X32-SSE-NEXT:    pxor %xmm4, %xmm4
+; X32-SSE-NEXT:    pcmpgtw %xmm1, %xmm4
+; X32-SSE-NEXT:    movdqa %xmm4, %xmm5
+; X32-SSE-NEXT:    pandn %xmm0, %xmm5
+; X32-SSE-NEXT:    psraw $4, %xmm0
+; X32-SSE-NEXT:    pand %xmm4, %xmm0
+; X32-SSE-NEXT:    por %xmm5, %xmm0
+; X32-SSE-NEXT:    paddw %xmm1, %xmm1
+; X32-SSE-NEXT:    pxor %xmm4, %xmm4
+; X32-SSE-NEXT:    pcmpgtw %xmm1, %xmm4
+; X32-SSE-NEXT:    movdqa %xmm4, %xmm5
+; X32-SSE-NEXT:    pandn %xmm0, %xmm5
+; X32-SSE-NEXT:    psraw $2, %xmm0
+; X32-SSE-NEXT:    pand %xmm4, %xmm0
+; X32-SSE-NEXT:    por %xmm5, %xmm0
+; X32-SSE-NEXT:    paddw %xmm1, %xmm1
+; X32-SSE-NEXT:    pcmpgtw %xmm1, %xmm3
+; X32-SSE-NEXT:    movdqa %xmm3, %xmm1
+; X32-SSE-NEXT:    pandn %xmm0, %xmm1
+; X32-SSE-NEXT:    psraw $1, %xmm0
+; X32-SSE-NEXT:    pand %xmm3, %xmm0
+; X32-SSE-NEXT:    por %xmm1, %xmm0
+; X32-SSE-NEXT:    psrlw $8, %xmm0
+; X32-SSE-NEXT:    packuswb %xmm2, %xmm0
 ; X32-SSE-NEXT:    retl
   %shift = ashr <2 x i8> %a, %b
   ret <2 x i8> %shift
@@ -926,154 +1160,46 @@ define <2 x i8> @var_shift_v2i8(<2 x i8> %a, <2 x i8> %b) nounwind {
 define <2 x i32> @splatvar_shift_v2i32(<2 x i32> %a, <2 x i32> %b) nounwind {
 ; SSE2-LABEL: splatvar_shift_v2i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    psllq $32, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,3,2,3]
-; SSE2-NEXT:    psrad $31, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,1,0,1]
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
-; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [9223372036854775808,9223372036854775808]
-; SSE2-NEXT:    movdqa %xmm1, %xmm3
-; SSE2-NEXT:    psrlq %xmm0, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[2,3,0,1]
-; SSE2-NEXT:    psrlq %xmm4, %xmm1
-; SSE2-NEXT:    movsd {{.*#+}} xmm1 = xmm3[0],xmm1[1]
-; SSE2-NEXT:    movdqa %xmm2, %xmm3
-; SSE2-NEXT:    psrlq %xmm0, %xmm3
-; SSE2-NEXT:    psrlq %xmm4, %xmm2
-; SSE2-NEXT:    movsd {{.*#+}} xmm2 = xmm3[0],xmm2[1]
-; SSE2-NEXT:    xorpd %xmm1, %xmm2
-; SSE2-NEXT:    psubq %xmm1, %xmm2
-; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    xorps %xmm2, %xmm2
+; SSE2-NEXT:    movss {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
+; SSE2-NEXT:    psrad %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: splatvar_shift_v2i32:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    psllq $32, %xmm2
-; SSE41-NEXT:    psrad $31, %xmm2
-; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,1,0,1]
-; SSE41-NEXT:    pxor %xmm1, %xmm1
-; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
-; SSE41-NEXT:    movdqa %xmm2, %xmm3
-; SSE41-NEXT:    psrlq %xmm0, %xmm3
-; SSE41-NEXT:    psrlq %xmm1, %xmm2
-; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm3[4,5,6,7]
-; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808]
-; SSE41-NEXT:    movdqa %xmm3, %xmm4
-; SSE41-NEXT:    psrlq %xmm1, %xmm4
-; SSE41-NEXT:    psrlq %xmm0, %xmm3
-; SSE41-NEXT:    pblendw {{.*#+}} xmm3 = xmm4[0,1,2,3],xmm3[4,5,6,7]
-; SSE41-NEXT:    pxor %xmm3, %xmm2
-; SSE41-NEXT:    psubq %xmm3, %xmm2
-; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
+; SSE41-NEXT:    psrad %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: splatvar_shift_v2i32:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpsllq $32, %xmm0, %xmm2
-; AVX1-NEXT:    vpsrad $31, %xmm2, %xmm2
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
-; AVX1-NEXT:    vpsrlq %xmm2, %xmm0, %xmm3
-; AVX1-NEXT:    vpsrlq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm3[4,5,6,7]
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808]
-; AVX1-NEXT:    vpsrlq %xmm1, %xmm3, %xmm1
-; AVX1-NEXT:    vpsrlq %xmm2, %xmm3, %xmm2
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm2[4,5,6,7]
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsubq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: splatvar_shift_v2i32:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpsllq $32, %xmm0, %xmm2
-; AVX2-NEXT:    vpsrad $31, %xmm2, %xmm2
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
-; AVX2-NEXT:    vpbroadcastq %xmm1, %xmm1
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; AVX2-NEXT:    vpsrlvq %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
-; AVX2-NEXT:    vpsrlvq %xmm1, %xmm2, %xmm1
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpsubq %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    retq
-;
-; XOPAVX1-LABEL: splatvar_shift_v2i32:
-; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vpsllq $32, %xmm0, %xmm0
-; XOPAVX1-NEXT:    vpshaq {{.*}}(%rip), %xmm0, %xmm0
-; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
-; XOPAVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; XOPAVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
-; XOPAVX1-NEXT:    vpsubq %xmm1, %xmm2, %xmm1
-; XOPAVX1-NEXT:    vpshaq %xmm1, %xmm0, %xmm0
-; XOPAVX1-NEXT:    retq
+; AVX-LABEL: splatvar_shift_v2i32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
+; AVX-NEXT:    vpsrad %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
 ;
-; XOPAVX2-LABEL: splatvar_shift_v2i32:
-; XOPAVX2:       # %bb.0:
-; XOPAVX2-NEXT:    vpsllq $32, %xmm0, %xmm0
-; XOPAVX2-NEXT:    vpshaq {{.*}}(%rip), %xmm0, %xmm0
-; XOPAVX2-NEXT:    vpbroadcastq %xmm1, %xmm1
-; XOPAVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; XOPAVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; XOPAVX2-NEXT:    vpsubq %xmm1, %xmm2, %xmm1
-; XOPAVX2-NEXT:    vpshaq %xmm1, %xmm0, %xmm0
-; XOPAVX2-NEXT:    retq
+; XOP-LABEL: splatvar_shift_v2i32:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
+; XOP-NEXT:    vpsrad %xmm1, %xmm0, %xmm0
+; XOP-NEXT:    retq
 ;
 ; AVX512-LABEL: splatvar_shift_v2i32:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsllq $32, %xmm0, %xmm0
-; AVX512-NEXT:    vpsraq $32, %zmm0, %zmm0
-; AVX512-NEXT:    vpbroadcastq %xmm1, %xmm1
-; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; AVX512-NEXT:    vpsravq %zmm1, %zmm0, %zmm0
-; AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
-; AVX512-NEXT:    vzeroupper
+; AVX512-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
+; AVX512-NEXT:    vpsrad %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatvar_shift_v2i32:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsllq $32, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpsraq $32, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpbroadcastq %xmm1, %xmm1
-; AVX512VL-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VL-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; AVX512VL-NEXT:    vpsravq %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
+; AVX512VL-NEXT:    vpsrad %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; X32-SSE-LABEL: splatvar_shift_v2i32:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    psllq $32, %xmm0
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,3,2,3]
-; X32-SSE-NEXT:    psrad $31, %xmm0
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
-; X32-SSE-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; X32-SSE-NEXT:    movdqa {{.*#+}} xmm0 = [4294967295,0,4294967295,0]
-; X32-SSE-NEXT:    pand %xmm1, %xmm0
-; X32-SSE-NEXT:    movdqa {{.*#+}} xmm3 = [0,2147483648,0,2147483648]
-; X32-SSE-NEXT:    movdqa %xmm3, %xmm4
-; X32-SSE-NEXT:    psrlq %xmm0, %xmm4
-; X32-SSE-NEXT:    xorps %xmm5, %xmm5
-; X32-SSE-NEXT:    movss {{.*#+}} xmm5 = xmm1[0],xmm5[1,2,3]
-; X32-SSE-NEXT:    psrlq %xmm5, %xmm3
-; X32-SSE-NEXT:    movsd {{.*#+}} xmm3 = xmm4[0],xmm3[1]
-; X32-SSE-NEXT:    movdqa %xmm2, %xmm1
-; X32-SSE-NEXT:    psrlq %xmm5, %xmm1
-; X32-SSE-NEXT:    psrlq %xmm0, %xmm2
-; X32-SSE-NEXT:    shufpd {{.*#+}} xmm2 = xmm2[0],xmm1[1]
-; X32-SSE-NEXT:    xorpd %xmm3, %xmm2
-; X32-SSE-NEXT:    psubq %xmm3, %xmm2
-; X32-SSE-NEXT:    movdqa %xmm2, %xmm0
+; X32-SSE-NEXT:    xorps %xmm2, %xmm2
+; X32-SSE-NEXT:    movss {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
+; X32-SSE-NEXT:    psrad %xmm2, %xmm0
 ; X32-SSE-NEXT:    retl
   %splat = shufflevector <2 x i32> %b, <2 x i32> undef, <2 x i32> zeroinitializer
   %shift = ashr <2 x i32> %a, %splat
@@ -1083,146 +1209,46 @@ define <2 x i32> @splatvar_shift_v2i32(<2 x i32> %a, <2 x i32> %b) nounwind {
 define <4 x i16> @splatvar_shift_v4i16(<4 x i16> %a, <4 x i16> %b) nounwind {
 ; SSE2-LABEL: splatvar_shift_v4i16:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pslld $16, %xmm0
-; SSE2-NEXT:    psrad $16, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[0,0,0,0]
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm2
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[2,3,3,3,4,5,6,7]
-; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    psrad %xmm1, %xmm3
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm4 = xmm2[0,1,1,1,4,5,6,7]
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    psrad %xmm4, %xmm1
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm2[2,3,3,3,4,5,6,7]
-; SSE2-NEXT:    movdqa %xmm0, %xmm4
-; SSE2-NEXT:    psrad %xmm3, %xmm4
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,1,1,1,4,5,6,7]
-; SSE2-NEXT:    psrad %xmm2, %xmm0
-; SSE2-NEXT:    punpckhqdq {{.*#+}} xmm0 = xmm0[1],xmm4[1]
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,3],xmm0[0,3]
-; SSE2-NEXT:    movaps %xmm1, %xmm0
+; SSE2-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1]
+; SSE2-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; SSE2-NEXT:    psraw %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: splatvar_shift_v4i16:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pslld $16, %xmm0
-; SSE41-NEXT:    psrad $16, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; SSE41-NEXT:    pxor %xmm2, %xmm2
-; SSE41-NEXT:    movdqa %xmm1, %xmm3
-; SSE41-NEXT:    pblendw {{.*#+}} xmm3 = xmm3[0],xmm2[1],xmm3[2],xmm2[3],xmm3[4],xmm2[5],xmm3[6],xmm2[7]
-; SSE41-NEXT:    pshuflw {{.*#+}} xmm4 = xmm3[2,3,3,3,4,5,6,7]
-; SSE41-NEXT:    movdqa %xmm0, %xmm5
-; SSE41-NEXT:    psrad %xmm4, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[2,3,0,1]
-; SSE41-NEXT:    pshuflw {{.*#+}} xmm4 = xmm3[2,3,3,3,4,5,6,7]
-; SSE41-NEXT:    movdqa %xmm0, %xmm6
-; SSE41-NEXT:    psrad %xmm4, %xmm6
-; SSE41-NEXT:    pblendw {{.*#+}} xmm6 = xmm5[0,1,2,3],xmm6[4,5,6,7]
-; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    psrad %xmm1, %xmm2
-; SSE41-NEXT:    pshuflw {{.*#+}} xmm1 = xmm3[0,1,1,1,4,5,6,7]
-; SSE41-NEXT:    psrad %xmm1, %xmm0
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm2[0,1,2,3],xmm0[4,5,6,7]
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm6[2,3],xmm0[4,5],xmm6[6,7]
+; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; SSE41-NEXT:    psraw %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: splatvar_shift_v4i16:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpslld $16, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrad $16, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
-; AVX1-NEXT:    vpsrldq {{.*#+}} xmm3 = xmm1[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX1-NEXT:    vpsrad %xmm3, %xmm0, %xmm3
-; AVX1-NEXT:    vpsrlq $32, %xmm1, %xmm4
-; AVX1-NEXT:    vpsrad %xmm4, %xmm0, %xmm4
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm4[0,1,2,3],xmm3[4,5,6,7]
-; AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm2 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
-; AVX1-NEXT:    vpsrad %xmm2, %xmm0, %xmm2
-; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
-; AVX1-NEXT:    vpsrad %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm2[4,5,6,7]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm3[2,3],xmm0[4,5],xmm3[6,7]
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: splatvar_shift_v4i16:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpslld $16, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrad $16, %xmm0, %xmm0
-; AVX2-NEXT:    vpbroadcastd %xmm1, %xmm1
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
-; AVX2-NEXT:    vpsravd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    retq
-;
-; XOPAVX1-LABEL: splatvar_shift_v4i16:
-; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vpslld $16, %xmm0, %xmm0
-; XOPAVX1-NEXT:    vpsrad $16, %xmm0, %xmm0
-; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; XOPAVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; XOPAVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
-; XOPAVX1-NEXT:    vpsubd %xmm1, %xmm2, %xmm1
-; XOPAVX1-NEXT:    vpshad %xmm1, %xmm0, %xmm0
-; XOPAVX1-NEXT:    retq
+; AVX-LABEL: splatvar_shift_v4i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX-NEXT:    vpsraw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
 ;
-; XOPAVX2-LABEL: splatvar_shift_v4i16:
-; XOPAVX2:       # %bb.0:
-; XOPAVX2-NEXT:    vpslld $16, %xmm0, %xmm0
-; XOPAVX2-NEXT:    vpsrad $16, %xmm0, %xmm0
-; XOPAVX2-NEXT:    vpbroadcastd %xmm1, %xmm1
-; XOPAVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; XOPAVX2-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
-; XOPAVX2-NEXT:    vpsravd %xmm1, %xmm0, %xmm0
-; XOPAVX2-NEXT:    retq
+; XOP-LABEL: splatvar_shift_v4i16:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; XOP-NEXT:    vpsraw %xmm1, %xmm0, %xmm0
+; XOP-NEXT:    retq
 ;
 ; AVX512-LABEL: splatvar_shift_v4i16:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpslld $16, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrad $16, %xmm0, %xmm0
-; AVX512-NEXT:    vpbroadcastd %xmm1, %xmm1
-; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
-; AVX512-NEXT:    vpsravd %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512-NEXT:    vpsraw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatvar_shift_v4i16:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpslld $16, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpsrad $16, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpbroadcastd %xmm1, %xmm1
-; AVX512VL-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VL-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
-; AVX512VL-NEXT:    vpsravd %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512VL-NEXT:    vpsraw %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; X32-SSE-LABEL: splatvar_shift_v4i16:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pslld $16, %xmm0
-; X32-SSE-NEXT:    psrad $16, %xmm0
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[0,0,0,0]
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm2
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[2,3,3,3,4,5,6,7]
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm3
-; X32-SSE-NEXT:    psrad %xmm1, %xmm3
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm4 = xmm2[0,1,1,1,4,5,6,7]
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
-; X32-SSE-NEXT:    psrad %xmm4, %xmm1
-; X32-SSE-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm3 = xmm2[2,3,3,3,4,5,6,7]
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm4
-; X32-SSE-NEXT:    psrad %xmm3, %xmm4
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,1,1,1,4,5,6,7]
-; X32-SSE-NEXT:    psrad %xmm2, %xmm0
-; X32-SSE-NEXT:    punpckhqdq {{.*#+}} xmm0 = xmm0[1],xmm4[1]
-; X32-SSE-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,3],xmm0[0,3]
-; X32-SSE-NEXT:    movaps %xmm1, %xmm0
+; X32-SSE-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1]
+; X32-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X32-SSE-NEXT:    psraw %xmm1, %xmm0
 ; X32-SSE-NEXT:    retl
   %splat = shufflevector <4 x i16> %b, <4 x i16> undef, <4 x i32> zeroinitializer
   %shift = ashr <4 x i16> %a, %splat
@@ -1232,366 +1258,188 @@ define <4 x i16> @splatvar_shift_v4i16(<4 x i16> %a, <4 x i16> %b) nounwind {
 define <2 x i16> @splatvar_shift_v2i16(<2 x i16> %a, <2 x i16> %b) nounwind {
 ; SSE2-LABEL: splatvar_shift_v2i16:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    psllq $48, %xmm0
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    psrad $31, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
-; SSE2-NEXT:    psrad $16, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
-; SSE2-NEXT:    movdqa %xmm2, %xmm3
-; SSE2-NEXT:    psrlq %xmm1, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[2,3,0,1]
-; SSE2-NEXT:    psrlq %xmm4, %xmm2
-; SSE2-NEXT:    movsd {{.*#+}} xmm2 = xmm3[0],xmm2[1]
-; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    psrlq %xmm1, %xmm3
-; SSE2-NEXT:    psrlq %xmm4, %xmm0
-; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm3[0],xmm0[1]
-; SSE2-NEXT:    xorpd %xmm2, %xmm0
-; SSE2-NEXT:    psubq %xmm2, %xmm0
+; SSE2-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1]
+; SSE2-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; SSE2-NEXT:    psraw %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: splatvar_shift_v2i16:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    psllq $48, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    psrad $31, %xmm2
-; SSE41-NEXT:    psrad $16, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
-; SSE41-NEXT:    pxor %xmm2, %xmm2
-; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3],xmm1[4],xmm2[5,6,7]
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,3,0,1]
-; SSE41-NEXT:    movdqa %xmm0, %xmm3
-; SSE41-NEXT:    psrlq %xmm1, %xmm3
-; SSE41-NEXT:    psrlq %xmm2, %xmm0
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm3[4,5,6,7]
-; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808]
-; SSE41-NEXT:    movdqa %xmm3, %xmm4
-; SSE41-NEXT:    psrlq %xmm2, %xmm4
-; SSE41-NEXT:    psrlq %xmm1, %xmm3
-; SSE41-NEXT:    pblendw {{.*#+}} xmm3 = xmm4[0,1,2,3],xmm3[4,5,6,7]
-; SSE41-NEXT:    pxor %xmm3, %xmm0
-; SSE41-NEXT:    psubq %xmm3, %xmm0
+; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; SSE41-NEXT:    psraw %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: splatvar_shift_v2i16:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpsllq $48, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrad $31, %xmm0, %xmm2
-; AVX1-NEXT:    vpsrad $16, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3],xmm1[4],xmm2[5,6,7]
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
-; AVX1-NEXT:    vpsrlq %xmm2, %xmm0, %xmm3
-; AVX1-NEXT:    vpsrlq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm3[4,5,6,7]
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808]
-; AVX1-NEXT:    vpsrlq %xmm1, %xmm3, %xmm1
-; AVX1-NEXT:    vpsrlq %xmm2, %xmm3, %xmm2
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm2[4,5,6,7]
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsubq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: splatvar_shift_v2i16:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpsllq $48, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrad $31, %xmm0, %xmm2
-; AVX2-NEXT:    vpsrad $16, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
-; AVX2-NEXT:    vpbroadcastq %xmm1, %xmm1
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3],xmm1[4],xmm2[5,6,7]
-; AVX2-NEXT:    vpsrlvq %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
-; AVX2-NEXT:    vpsrlvq %xmm1, %xmm2, %xmm1
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpsubq %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    retq
-;
-; XOPAVX1-LABEL: splatvar_shift_v2i16:
-; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vpsllq $48, %xmm0, %xmm0
-; XOPAVX1-NEXT:    vpshaq {{.*}}(%rip), %xmm0, %xmm0
-; XOPAVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
-; XOPAVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3],xmm1[4],xmm2[5,6,7]
-; XOPAVX1-NEXT:    vpsubq %xmm1, %xmm2, %xmm1
-; XOPAVX1-NEXT:    vpshaq %xmm1, %xmm0, %xmm0
-; XOPAVX1-NEXT:    retq
+; AVX-LABEL: splatvar_shift_v2i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX-NEXT:    vpsraw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
 ;
-; XOPAVX2-LABEL: splatvar_shift_v2i16:
-; XOPAVX2:       # %bb.0:
-; XOPAVX2-NEXT:    vpsllq $48, %xmm0, %xmm0
-; XOPAVX2-NEXT:    vpshaq {{.*}}(%rip), %xmm0, %xmm0
-; XOPAVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; XOPAVX2-NEXT:    vpbroadcastq %xmm1, %xmm1
-; XOPAVX2-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3],xmm1[4],xmm2[5,6,7]
-; XOPAVX2-NEXT:    vpsubq %xmm1, %xmm2, %xmm1
-; XOPAVX2-NEXT:    vpshaq %xmm1, %xmm0, %xmm0
-; XOPAVX2-NEXT:    retq
+; XOP-LABEL: splatvar_shift_v2i16:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; XOP-NEXT:    vpsraw %xmm1, %xmm0, %xmm0
+; XOP-NEXT:    retq
 ;
 ; AVX512-LABEL: splatvar_shift_v2i16:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsllq $48, %xmm0, %xmm0
-; AVX512-NEXT:    vpsraq $48, %zmm0, %zmm0
-; AVX512-NEXT:    vpbroadcastq %xmm1, %xmm1
-; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3],xmm1[4],xmm2[5,6,7]
-; AVX512-NEXT:    vpsravq %zmm1, %zmm0, %zmm0
-; AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
-; AVX512-NEXT:    vzeroupper
+; AVX512-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512-NEXT:    vpsraw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatvar_shift_v2i16:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsllq $48, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpsraq $48, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpbroadcastq %xmm1, %xmm1
-; AVX512VL-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VL-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3],xmm1[4],xmm2[5,6,7]
-; AVX512VL-NEXT:    vpsravq %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512VL-NEXT:    vpsraw %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; X32-SSE-LABEL: splatvar_shift_v2i16:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    psllq $48, %xmm0
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm2
-; X32-SSE-NEXT:    psrad $31, %xmm2
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
-; X32-SSE-NEXT:    psrad $16, %xmm0
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
-; X32-SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm1
-; X32-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [0,2147483648,0,2147483648]
-; X32-SSE-NEXT:    movdqa %xmm2, %xmm3
-; X32-SSE-NEXT:    psrlq %xmm1, %xmm3
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[2,3,0,1]
-; X32-SSE-NEXT:    psrlq %xmm4, %xmm2
-; X32-SSE-NEXT:    movsd {{.*#+}} xmm2 = xmm3[0],xmm2[1]
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm3
-; X32-SSE-NEXT:    psrlq %xmm1, %xmm3
-; X32-SSE-NEXT:    psrlq %xmm4, %xmm0
-; X32-SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm3[0],xmm0[1]
-; X32-SSE-NEXT:    xorpd %xmm2, %xmm0
-; X32-SSE-NEXT:    psubq %xmm2, %xmm0
+; X32-SSE-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1]
+; X32-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X32-SSE-NEXT:    psraw %xmm1, %xmm0
 ; X32-SSE-NEXT:    retl
   %splat = shufflevector <2 x i16> %b, <2 x i16> undef, <2 x i32> zeroinitializer
   %shift = ashr <2 x i16> %a, %splat
   ret <2 x i16> %shift
 }
 
-define <8 x i8> @splatvar_shift_v8i8(<8 x i8> %a, <8 x i8> %b) nounwind {
-; SSE2-LABEL: splatvar_shift_v8i8:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    psllw $8, %xmm2
-; SSE2-NEXT:    movdqa %xmm2, %xmm3
-; SSE2-NEXT:    psraw $8, %xmm3
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm1[0,0,2,3,4,5,6,7]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,0,0]
-; SSE2-NEXT:    psllw $12, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm0
-; SSE2-NEXT:    psraw $15, %xmm0
-; SSE2-NEXT:    psraw $15, %xmm2
-; SSE2-NEXT:    pand %xmm0, %xmm2
-; SSE2-NEXT:    pandn %xmm3, %xmm0
-; SSE2-NEXT:    por %xmm2, %xmm0
-; SSE2-NEXT:    paddw %xmm1, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm2
-; SSE2-NEXT:    psraw $15, %xmm2
-; SSE2-NEXT:    movdqa %xmm2, %xmm3
-; SSE2-NEXT:    pandn %xmm0, %xmm3
-; SSE2-NEXT:    psraw $4, %xmm0
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    por %xmm3, %xmm0
-; SSE2-NEXT:    paddw %xmm1, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm2
-; SSE2-NEXT:    psraw $15, %xmm2
-; SSE2-NEXT:    movdqa %xmm2, %xmm3
-; SSE2-NEXT:    pandn %xmm0, %xmm3
-; SSE2-NEXT:    psraw $2, %xmm0
+define <8 x i8> @splatvar_shift_v8i8(<8 x i8> %a, <8 x i8> %b) nounwind {
+; SSE2-LABEL: splatvar_shift_v8i8:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0]
+; SSE2-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; SSE2-NEXT:    psrlw %xmm1, %xmm0
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm2
+; SSE2-NEXT:    psrlw %xmm1, %xmm2
+; SSE2-NEXT:    psrlw $8, %xmm2
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,0,2,3,4,5,6,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
 ; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    por %xmm3, %xmm0
-; SSE2-NEXT:    paddw %xmm1, %xmm1
-; SSE2-NEXT:    psraw $15, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm2
-; SSE2-NEXT:    pandn %xmm0, %xmm2
-; SSE2-NEXT:    psraw $1, %xmm0
-; SSE2-NEXT:    pand %xmm1, %xmm0
-; SSE2-NEXT:    por %xmm2, %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [32896,32896,32896,32896,32896,32896,32896,32896]
+; SSE2-NEXT:    psrlw %xmm1, %xmm2
+; SSE2-NEXT:    pxor %xmm2, %xmm0
+; SSE2-NEXT:    psubb %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: splatvar_shift_v8i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa %xmm1, %xmm2
-; SSE41-NEXT:    movdqa %xmm0, %xmm3
-; SSE41-NEXT:    psllw $8, %xmm3
-; SSE41-NEXT:    movdqa %xmm3, %xmm1
-; SSE41-NEXT:    psraw $8, %xmm1
-; SSE41-NEXT:    pshufb {{.*#+}} xmm2 = xmm2[0],zero,xmm2[0],zero,xmm2[0],zero,xmm2[0],zero,xmm2[0],zero,xmm2[0],zero,xmm2[0],zero,xmm2[0],zero
-; SSE41-NEXT:    movdqa %xmm2, %xmm0
-; SSE41-NEXT:    psllw $12, %xmm0
-; SSE41-NEXT:    psllw $4, %xmm2
-; SSE41-NEXT:    por %xmm0, %xmm2
-; SSE41-NEXT:    movdqa %xmm2, %xmm4
-; SSE41-NEXT:    paddw %xmm2, %xmm4
-; SSE41-NEXT:    psraw $15, %xmm3
-; SSE41-NEXT:    movdqa %xmm2, %xmm0
-; SSE41-NEXT:    pblendvb %xmm0, %xmm3, %xmm1
-; SSE41-NEXT:    movdqa %xmm1, %xmm2
-; SSE41-NEXT:    psraw $4, %xmm2
-; SSE41-NEXT:    movdqa %xmm4, %xmm0
-; SSE41-NEXT:    pblendvb %xmm0, %xmm2, %xmm1
-; SSE41-NEXT:    movdqa %xmm1, %xmm2
-; SSE41-NEXT:    psraw $2, %xmm2
-; SSE41-NEXT:    paddw %xmm4, %xmm4
-; SSE41-NEXT:    movdqa %xmm4, %xmm0
-; SSE41-NEXT:    pblendvb %xmm0, %xmm2, %xmm1
-; SSE41-NEXT:    movdqa %xmm1, %xmm2
-; SSE41-NEXT:    psraw $1, %xmm2
-; SSE41-NEXT:    paddw %xmm4, %xmm4
-; SSE41-NEXT:    movdqa %xmm4, %xmm0
-; SSE41-NEXT:    pblendvb %xmm0, %xmm2, %xmm1
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; SSE41-NEXT:    psrlw %xmm1, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm2, %xmm2
+; SSE41-NEXT:    psrlw %xmm1, %xmm2
+; SSE41-NEXT:    pshufb {{.*#+}} xmm2 = xmm2[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; SSE41-NEXT:    pand %xmm2, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [32896,32896,32896,32896,32896,32896,32896,32896]
+; SSE41-NEXT:    psrlw %xmm1, %xmm2
+; SSE41-NEXT:    pxor %xmm2, %xmm0
+; SSE41-NEXT:    psubb %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: splatvar_shift_v8i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpsllw $8, %xmm0, %xmm0
-; AVX1-NEXT:    vpsraw $8, %xmm0, %xmm2
-; AVX1-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero
-; AVX1-NEXT:    vpsllw $12, %xmm1, %xmm3
-; AVX1-NEXT:    vpsllw $4, %xmm1, %xmm1
-; AVX1-NEXT:    vpor %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpaddw %xmm1, %xmm1, %xmm3
-; AVX1-NEXT:    vpsraw $15, %xmm0, %xmm0
-; AVX1-NEXT:    vpblendvb %xmm1, %xmm0, %xmm2, %xmm0
-; AVX1-NEXT:    vpsraw $4, %xmm0, %xmm1
-; AVX1-NEXT:    vpblendvb %xmm3, %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsraw $2, %xmm0, %xmm1
-; AVX1-NEXT:    vpaddw %xmm3, %xmm3, %xmm2
-; AVX1-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsraw $1, %xmm0, %xmm1
-; AVX1-NEXT:    vpaddw %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX1-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpsrlw %xmm1, %xmm2, %xmm2
+; AVX1-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [32896,32896,32896,32896,32896,32896,32896,32896]
+; AVX1-NEXT:    vpsrlw %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: splatvar_shift_v8i8:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpsllw $8, %xmm0, %xmm0
-; AVX2-NEXT:    vpsraw $8, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero
-; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
-; AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
-; AVX2-NEXT:    vpsravd %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX2-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpsrlw %xmm1, %xmm2, %xmm2
+; AVX2-NEXT:    vpsrlw $8, %xmm2, %xmm2
+; AVX2-NEXT:    vpbroadcastb %xmm2, %xmm2
+; AVX2-NEXT:    vpand %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [32896,32896,32896,32896,32896,32896,32896,32896]
+; AVX2-NEXT:    vpsrlw %xmm1, %xmm2, %xmm1
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    retq
 ;
-; XOP-LABEL: splatvar_shift_v8i8:
-; XOP:       # %bb.0:
-; XOP-NEXT:    vpsllw $8, %xmm0, %xmm0
-; XOP-NEXT:    vpsraw $8, %xmm0, %xmm0
-; XOP-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero
-; XOP-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; XOP-NEXT:    vpsubw %xmm1, %xmm2, %xmm1
-; XOP-NEXT:    vpshaw %xmm1, %xmm0, %xmm0
-; XOP-NEXT:    retq
+; XOPAVX1-LABEL: splatvar_shift_v8i8:
+; XOPAVX1:       # %bb.0:
+; XOPAVX1-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,0,0,0,4,5,6,7]
+; XOPAVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; XOPAVX1-NEXT:    vpsubb %xmm1, %xmm2, %xmm1
+; XOPAVX1-NEXT:    vpshab %xmm1, %xmm0, %xmm0
+; XOPAVX1-NEXT:    retq
+;
+; XOPAVX2-LABEL: splatvar_shift_v8i8:
+; XOPAVX2:       # %bb.0:
+; XOPAVX2-NEXT:    vpbroadcastb %xmm1, %xmm1
+; XOPAVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; XOPAVX2-NEXT:    vpsubb %xmm1, %xmm2, %xmm1
+; XOPAVX2-NEXT:    vpshab %xmm1, %xmm0, %xmm0
+; XOPAVX2-NEXT:    retq
 ;
 ; AVX512DQ-LABEL: splatvar_shift_v8i8:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vpsllw $8, %xmm0, %xmm0
-; AVX512DQ-NEXT:    vpsraw $8, %xmm0, %xmm0
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero
-; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
-; AVX512DQ-NEXT:    vpmovsxwd %xmm0, %ymm0
-; AVX512DQ-NEXT:    vpsravd %ymm1, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpmovdw %zmm0, %ymm0
-; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512DQ-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512DQ-NEXT:    vpmovsxbd %xmm0, %zmm0
+; AVX512DQ-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; AVX512DQ-NEXT:    vpsravd %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpmovdb %zmm0, %xmm0
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatvar_shift_v8i8:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpsllw $8, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpsraw $8, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero
+; AVX512BW-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512BW-NEXT:    vpmovsxbw %xmm0, %ymm0
+; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
 ; AVX512BW-NEXT:    vpsravw %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512DQVL-LABEL: splatvar_shift_v8i8:
 ; AVX512DQVL:       # %bb.0:
-; AVX512DQVL-NEXT:    vpsllw $8, %xmm0, %xmm0
-; AVX512DQVL-NEXT:    vpsraw $8, %xmm0, %xmm0
-; AVX512DQVL-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero
-; AVX512DQVL-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
-; AVX512DQVL-NEXT:    vpmovsxwd %xmm0, %ymm0
-; AVX512DQVL-NEXT:    vpsravd %ymm1, %ymm0, %ymm0
-; AVX512DQVL-NEXT:    vpmovdw %ymm0, %xmm0
+; AVX512DQVL-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512DQVL-NEXT:    vpmovsxbd %xmm0, %zmm0
+; AVX512DQVL-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; AVX512DQVL-NEXT:    vpsravd %zmm1, %zmm0, %zmm0
+; AVX512DQVL-NEXT:    vpmovdb %zmm0, %xmm0
 ; AVX512DQVL-NEXT:    vzeroupper
 ; AVX512DQVL-NEXT:    retq
 ;
 ; AVX512BWVL-LABEL: splatvar_shift_v8i8:
 ; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vpsllw $8, %xmm0, %xmm0
-; AVX512BWVL-NEXT:    vpsraw $8, %xmm0, %xmm0
-; AVX512BWVL-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero
-; AVX512BWVL-NEXT:    vpsravw %xmm1, %xmm0, %xmm0
+; AVX512BWVL-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512BWVL-NEXT:    vpmovsxbw %xmm0, %ymm0
+; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512BWVL-NEXT:    vpsravw %ymm1, %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm0
+; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
 ;
 ; X32-SSE-LABEL: splatvar_shift_v8i8:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm2
-; X32-SSE-NEXT:    psllw $8, %xmm2
-; X32-SSE-NEXT:    movdqa %xmm2, %xmm3
-; X32-SSE-NEXT:    psraw $8, %xmm3
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm1[0,0,2,3,4,5,6,7]
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,0,0]
-; X32-SSE-NEXT:    psllw $12, %xmm1
-; X32-SSE-NEXT:    movdqa %xmm1, %xmm0
-; X32-SSE-NEXT:    psraw $15, %xmm0
-; X32-SSE-NEXT:    psraw $15, %xmm2
-; X32-SSE-NEXT:    pand %xmm0, %xmm2
-; X32-SSE-NEXT:    pandn %xmm3, %xmm0
-; X32-SSE-NEXT:    por %xmm2, %xmm0
-; X32-SSE-NEXT:    paddw %xmm1, %xmm1
-; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
-; X32-SSE-NEXT:    psraw $15, %xmm2
-; X32-SSE-NEXT:    movdqa %xmm2, %xmm3
-; X32-SSE-NEXT:    pandn %xmm0, %xmm3
-; X32-SSE-NEXT:    psraw $4, %xmm0
-; X32-SSE-NEXT:    pand %xmm2, %xmm0
-; X32-SSE-NEXT:    por %xmm3, %xmm0
-; X32-SSE-NEXT:    paddw %xmm1, %xmm1
-; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
-; X32-SSE-NEXT:    psraw $15, %xmm2
-; X32-SSE-NEXT:    movdqa %xmm2, %xmm3
-; X32-SSE-NEXT:    pandn %xmm0, %xmm3
-; X32-SSE-NEXT:    psraw $2, %xmm0
+; X32-SSE-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0]
+; X32-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X32-SSE-NEXT:    psrlw %xmm1, %xmm0
+; X32-SSE-NEXT:    pcmpeqd %xmm2, %xmm2
+; X32-SSE-NEXT:    psrlw %xmm1, %xmm2
+; X32-SSE-NEXT:    psrlw $8, %xmm2
+; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,0,2,3,4,5,6,7]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
 ; X32-SSE-NEXT:    pand %xmm2, %xmm0
-; X32-SSE-NEXT:    por %xmm3, %xmm0
-; X32-SSE-NEXT:    paddw %xmm1, %xmm1
-; X32-SSE-NEXT:    psraw $15, %xmm1
-; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
-; X32-SSE-NEXT:    pandn %xmm0, %xmm2
-; X32-SSE-NEXT:    psraw $1, %xmm0
-; X32-SSE-NEXT:    pand %xmm1, %xmm0
-; X32-SSE-NEXT:    por %xmm2, %xmm0
+; X32-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [32896,32896,32896,32896,32896,32896,32896,32896]
+; X32-SSE-NEXT:    psrlw %xmm1, %xmm2
+; X32-SSE-NEXT:    pxor %xmm2, %xmm0
+; X32-SSE-NEXT:    psubb %xmm2, %xmm0
 ; X32-SSE-NEXT:    retl
   %splat = shufflevector <8 x i8> %b, <8 x i8> undef, <8 x i32> zeroinitializer
   %shift = ashr <8 x i8> %a, %splat
@@ -1601,133 +1449,139 @@ define <8 x i8> @splatvar_shift_v8i8(<8 x i8> %a, <8 x i8> %b) nounwind {
 define <4 x i8> @splatvar_shift_v4i8(<4 x i8> %a, <4 x i8> %b) nounwind {
 ; SSE2-LABEL: splatvar_shift_v4i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pslld $24, %xmm0
-; SSE2-NEXT:    psrad $24, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[0,0,0,0]
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm2
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[2,3,3,3,4,5,6,7]
-; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    psrad %xmm1, %xmm3
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm4 = xmm2[0,1,1,1,4,5,6,7]
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    psrad %xmm4, %xmm1
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm2[2,3,3,3,4,5,6,7]
-; SSE2-NEXT:    movdqa %xmm0, %xmm4
-; SSE2-NEXT:    psrad %xmm3, %xmm4
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,1,1,1,4,5,6,7]
-; SSE2-NEXT:    psrad %xmm2, %xmm0
-; SSE2-NEXT:    punpckhqdq {{.*#+}} xmm0 = xmm0[1],xmm4[1]
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,3],xmm0[0,3]
-; SSE2-NEXT:    movaps %xmm1, %xmm0
+; SSE2-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0]
+; SSE2-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; SSE2-NEXT:    psrlw %xmm1, %xmm0
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm2
+; SSE2-NEXT:    psrlw %xmm1, %xmm2
+; SSE2-NEXT:    psrlw $8, %xmm2
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,0,2,3,4,5,6,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [32896,32896,32896,32896,32896,32896,32896,32896]
+; SSE2-NEXT:    psrlw %xmm1, %xmm2
+; SSE2-NEXT:    pxor %xmm2, %xmm0
+; SSE2-NEXT:    psubb %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: splatvar_shift_v4i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pslld $24, %xmm0
-; SSE41-NEXT:    psrad $24, %xmm0
-; SSE41-NEXT:    pshufb {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[0],zero,zero,zero,xmm1[0],zero,zero,zero,xmm1[0],zero,zero,zero
-; SSE41-NEXT:    pshuflw {{.*#+}} xmm2 = xmm1[2,3,3,3,4,5,6,7]
-; SSE41-NEXT:    movdqa %xmm0, %xmm3
-; SSE41-NEXT:    psrad %xmm2, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
-; SSE41-NEXT:    pshuflw {{.*#+}} xmm4 = xmm2[2,3,3,3,4,5,6,7]
-; SSE41-NEXT:    movdqa %xmm0, %xmm5
-; SSE41-NEXT:    psrad %xmm4, %xmm5
-; SSE41-NEXT:    pblendw {{.*#+}} xmm5 = xmm3[0,1,2,3],xmm5[4,5,6,7]
-; SSE41-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,1,1,1,4,5,6,7]
-; SSE41-NEXT:    movdqa %xmm0, %xmm3
-; SSE41-NEXT:    psrad %xmm1, %xmm3
-; SSE41-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,1,1,1,4,5,6,7]
-; SSE41-NEXT:    psrad %xmm1, %xmm0
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm3[0,1,2,3],xmm0[4,5,6,7]
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm5[2,3],xmm0[4,5],xmm5[6,7]
+; SSE41-NEXT:    pmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; SSE41-NEXT:    psrlw %xmm1, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm2, %xmm2
+; SSE41-NEXT:    psrlw %xmm1, %xmm2
+; SSE41-NEXT:    pshufb {{.*#+}} xmm2 = xmm2[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; SSE41-NEXT:    pand %xmm2, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [32896,32896,32896,32896,32896,32896,32896,32896]
+; SSE41-NEXT:    psrlw %xmm1, %xmm2
+; SSE41-NEXT:    pxor %xmm2, %xmm0
+; SSE41-NEXT:    psubb %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: splatvar_shift_v4i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpslld $24, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrad $24, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[0],zero,zero,zero,xmm1[0],zero,zero,zero,xmm1[0],zero,zero,zero
-; AVX1-NEXT:    vpsrldq {{.*#+}} xmm2 = xmm1[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX1-NEXT:    vpsrad %xmm2, %xmm0, %xmm2
-; AVX1-NEXT:    vpsrlq $32, %xmm1, %xmm3
-; AVX1-NEXT:    vpsrad %xmm3, %xmm0, %xmm3
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
-; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm3 = xmm1[2],xmm3[2],xmm1[3],xmm3[3]
-; AVX1-NEXT:    vpsrad %xmm3, %xmm0, %xmm3
-; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
-; AVX1-NEXT:    vpsrad %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm3[4,5,6,7]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
+; AVX1-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX1-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpsrlw %xmm1, %xmm2, %xmm2
+; AVX1-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [32896,32896,32896,32896,32896,32896,32896,32896]
+; AVX1-NEXT:    vpsrlw %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: splatvar_shift_v4i8:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpslld $24, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrad $24, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[0],zero,zero,zero,xmm1[0],zero,zero,zero,xmm1[0],zero,zero,zero
-; AVX2-NEXT:    vpsravd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX2-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpsrlw %xmm1, %xmm2, %xmm2
+; AVX2-NEXT:    vpsrlw $8, %xmm2, %xmm2
+; AVX2-NEXT:    vpbroadcastb %xmm2, %xmm2
+; AVX2-NEXT:    vpand %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [32896,32896,32896,32896,32896,32896,32896,32896]
+; AVX2-NEXT:    vpsrlw %xmm1, %xmm2, %xmm1
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: splatvar_shift_v4i8:
 ; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vpslld $24, %xmm0, %xmm0
-; XOPAVX1-NEXT:    vpsrad $24, %xmm0, %xmm0
-; XOPAVX1-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[0],zero,zero,zero,xmm1[0],zero,zero,zero,xmm1[0],zero,zero,zero
+; XOPAVX1-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,0,2,3,4,5,6,7]
 ; XOPAVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; XOPAVX1-NEXT:    vpsubd %xmm1, %xmm2, %xmm1
-; XOPAVX1-NEXT:    vpshad %xmm1, %xmm0, %xmm0
+; XOPAVX1-NEXT:    vpsubb %xmm1, %xmm2, %xmm1
+; XOPAVX1-NEXT:    vpshab %xmm1, %xmm0, %xmm0
 ; XOPAVX1-NEXT:    retq
 ;
 ; XOPAVX2-LABEL: splatvar_shift_v4i8:
 ; XOPAVX2:       # %bb.0:
-; XOPAVX2-NEXT:    vpslld $24, %xmm0, %xmm0
-; XOPAVX2-NEXT:    vpsrad $24, %xmm0, %xmm0
-; XOPAVX2-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[0],zero,zero,zero,xmm1[0],zero,zero,zero,xmm1[0],zero,zero,zero
-; XOPAVX2-NEXT:    vpsravd %xmm1, %xmm0, %xmm0
+; XOPAVX2-NEXT:    vpbroadcastb %xmm1, %xmm1
+; XOPAVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; XOPAVX2-NEXT:    vpsubb %xmm1, %xmm2, %xmm1
+; XOPAVX2-NEXT:    vpshab %xmm1, %xmm0, %xmm0
 ; XOPAVX2-NEXT:    retq
 ;
-; AVX512-LABEL: splatvar_shift_v4i8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpslld $24, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrad $24, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[0],zero,zero,zero,xmm1[0],zero,zero,zero,xmm1[0],zero,zero,zero
-; AVX512-NEXT:    vpsravd %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    retq
+; AVX512DQ-LABEL: splatvar_shift_v4i8:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512DQ-NEXT:    vpmovsxbd %xmm0, %zmm0
+; AVX512DQ-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; AVX512DQ-NEXT:    vpsravd %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    retq
 ;
-; AVX512VL-LABEL: splatvar_shift_v4i8:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpslld $24, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpsrad $24, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[0],zero,zero,zero,xmm1[0],zero,zero,zero,xmm1[0],zero,zero,zero
-; AVX512VL-NEXT:    vpsravd %xmm1, %xmm0, %xmm0
-; AVX512VL-NEXT:    retq
+; AVX512BW-LABEL: splatvar_shift_v4i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512BW-NEXT:    vpmovsxbw %xmm0, %ymm0
+; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512BW-NEXT:    vpsravw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512DQVL-LABEL: splatvar_shift_v4i8:
+; AVX512DQVL:       # %bb.0:
+; AVX512DQVL-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512DQVL-NEXT:    vpmovsxbd %xmm0, %zmm0
+; AVX512DQVL-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; AVX512DQVL-NEXT:    vpsravd %zmm1, %zmm0, %zmm0
+; AVX512DQVL-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512DQVL-NEXT:    vzeroupper
+; AVX512DQVL-NEXT:    retq
+;
+; AVX512BWVL-LABEL: splatvar_shift_v4i8:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512BWVL-NEXT:    vpmovsxbw %xmm0, %ymm0
+; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512BWVL-NEXT:    vpsravw %ymm1, %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm0
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
 ;
 ; X32-SSE-LABEL: splatvar_shift_v4i8:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pslld $24, %xmm0
-; X32-SSE-NEXT:    psrad $24, %xmm0
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[0,0,0,0]
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm2
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[2,3,3,3,4,5,6,7]
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm3
-; X32-SSE-NEXT:    psrad %xmm1, %xmm3
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm4 = xmm2[0,1,1,1,4,5,6,7]
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
-; X32-SSE-NEXT:    psrad %xmm4, %xmm1
-; X32-SSE-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm3 = xmm2[2,3,3,3,4,5,6,7]
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm4
-; X32-SSE-NEXT:    psrad %xmm3, %xmm4
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,1,1,1,4,5,6,7]
-; X32-SSE-NEXT:    psrad %xmm2, %xmm0
-; X32-SSE-NEXT:    punpckhqdq {{.*#+}} xmm0 = xmm0[1],xmm4[1]
-; X32-SSE-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,3],xmm0[0,3]
-; X32-SSE-NEXT:    movaps %xmm1, %xmm0
+; X32-SSE-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0]
+; X32-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X32-SSE-NEXT:    psrlw %xmm1, %xmm0
+; X32-SSE-NEXT:    pcmpeqd %xmm2, %xmm2
+; X32-SSE-NEXT:    psrlw %xmm1, %xmm2
+; X32-SSE-NEXT:    psrlw $8, %xmm2
+; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,0,2,3,4,5,6,7]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
+; X32-SSE-NEXT:    pand %xmm2, %xmm0
+; X32-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [32896,32896,32896,32896,32896,32896,32896,32896]
+; X32-SSE-NEXT:    psrlw %xmm1, %xmm2
+; X32-SSE-NEXT:    pxor %xmm2, %xmm0
+; X32-SSE-NEXT:    psubb %xmm2, %xmm0
 ; X32-SSE-NEXT:    retl
   %splat = shufflevector <4 x i8> %b, <4 x i8> undef, <4 x i32> zeroinitializer
   %shift = ashr <4 x i8> %a, %splat
@@ -1737,138 +1591,130 @@ define <4 x i8> @splatvar_shift_v4i8(<4 x i8> %a, <4 x i8> %b) nounwind {
 define <2 x i8> @splatvar_shift_v2i8(<2 x i8> %a, <2 x i8> %b) nounwind {
 ; SSE2-LABEL: splatvar_shift_v2i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    psllq $56, %xmm0
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    psrad $31, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
-; SSE2-NEXT:    psrad $24, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
-; SSE2-NEXT:    movdqa %xmm2, %xmm3
-; SSE2-NEXT:    psrlq %xmm1, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[2,3,0,1]
-; SSE2-NEXT:    psrlq %xmm4, %xmm2
-; SSE2-NEXT:    movsd {{.*#+}} xmm2 = xmm3[0],xmm2[1]
-; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    psrlq %xmm1, %xmm3
-; SSE2-NEXT:    psrlq %xmm4, %xmm0
-; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm3[0],xmm0[1]
-; SSE2-NEXT:    xorpd %xmm2, %xmm0
-; SSE2-NEXT:    psubq %xmm2, %xmm0
+; SSE2-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0]
+; SSE2-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; SSE2-NEXT:    psrlw %xmm1, %xmm0
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm2
+; SSE2-NEXT:    psrlw %xmm1, %xmm2
+; SSE2-NEXT:    psrlw $8, %xmm2
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,0,2,3,4,5,6,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [32896,32896,32896,32896,32896,32896,32896,32896]
+; SSE2-NEXT:    psrlw %xmm1, %xmm2
+; SSE2-NEXT:    pxor %xmm2, %xmm0
+; SSE2-NEXT:    psubb %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: splatvar_shift_v2i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    psllq $56, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    psrad $31, %xmm2
-; SSE41-NEXT:    psrad $24, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
-; SSE41-NEXT:    pshufb {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[0],zero,zero,zero,zero,zero,zero,zero
-; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
-; SSE41-NEXT:    movdqa %xmm2, %xmm3
-; SSE41-NEXT:    psrlq %xmm1, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[2,3,0,1]
-; SSE41-NEXT:    psrlq %xmm4, %xmm2
-; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
-; SSE41-NEXT:    movdqa %xmm0, %xmm3
-; SSE41-NEXT:    psrlq %xmm4, %xmm3
-; SSE41-NEXT:    psrlq %xmm1, %xmm0
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm3[4,5,6,7]
+; SSE41-NEXT:    pmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; SSE41-NEXT:    psrlw %xmm1, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm2, %xmm2
+; SSE41-NEXT:    psrlw %xmm1, %xmm2
+; SSE41-NEXT:    pshufb {{.*#+}} xmm2 = xmm2[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; SSE41-NEXT:    pand %xmm2, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [32896,32896,32896,32896,32896,32896,32896,32896]
+; SSE41-NEXT:    psrlw %xmm1, %xmm2
 ; SSE41-NEXT:    pxor %xmm2, %xmm0
-; SSE41-NEXT:    psubq %xmm2, %xmm0
+; SSE41-NEXT:    psubb %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: splatvar_shift_v2i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpsllq $56, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrad $31, %xmm0, %xmm2
-; AVX1-NEXT:    vpsrad $24, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
-; AVX1-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[0],zero,zero,zero,zero,zero,zero,zero
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
-; AVX1-NEXT:    vpsrlq %xmm1, %xmm2, %xmm3
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm1[2,3,0,1]
-; AVX1-NEXT:    vpsrlq %xmm4, %xmm2, %xmm2
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
-; AVX1-NEXT:    vpsrlq %xmm4, %xmm0, %xmm3
-; AVX1-NEXT:    vpsrlq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm3[4,5,6,7]
-; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpsubq %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX1-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpsrlw %xmm1, %xmm2, %xmm2
+; AVX1-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [32896,32896,32896,32896,32896,32896,32896,32896]
+; AVX1-NEXT:    vpsrlw %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: splatvar_shift_v2i8:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpsllq $56, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrad $31, %xmm0, %xmm2
-; AVX2-NEXT:    vpsrad $24, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
-; AVX2-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[0],zero,zero,zero,zero,zero,zero,zero
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
-; AVX2-NEXT:    vpsrlvq %xmm1, %xmm2, %xmm2
-; AVX2-NEXT:    vpsrlvq %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm0
-; AVX2-NEXT:    vpsubq %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX2-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpsrlw %xmm1, %xmm2, %xmm2
+; AVX2-NEXT:    vpsrlw $8, %xmm2, %xmm2
+; AVX2-NEXT:    vpbroadcastb %xmm2, %xmm2
+; AVX2-NEXT:    vpand %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [32896,32896,32896,32896,32896,32896,32896,32896]
+; AVX2-NEXT:    vpsrlw %xmm1, %xmm2, %xmm1
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    retq
 ;
 ; XOP-LABEL: splatvar_shift_v2i8:
 ; XOP:       # %bb.0:
-; XOP-NEXT:    vpsllq $56, %xmm0, %xmm0
-; XOP-NEXT:    vpshaq {{.*}}(%rip), %xmm0, %xmm0
+; XOP-NEXT:    insertq {{.*#+}} xmm1 = xmm1[0,0,2,3,4,5,6,7,u,u,u,u,u,u,u,u]
 ; XOP-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; XOP-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[0],zero,zero,zero,zero,zero,zero,zero
-; XOP-NEXT:    vpsubq %xmm1, %xmm2, %xmm1
-; XOP-NEXT:    vpshaq %xmm1, %xmm0, %xmm0
+; XOP-NEXT:    vpsubb %xmm1, %xmm2, %xmm1
+; XOP-NEXT:    vpshab %xmm1, %xmm0, %xmm0
 ; XOP-NEXT:    retq
 ;
-; AVX512-LABEL: splatvar_shift_v2i8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsllq $56, %xmm0, %xmm0
-; AVX512-NEXT:    vpsraq $56, %zmm0, %zmm0
-; AVX512-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[0],zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT:    vpsravq %zmm1, %zmm0, %zmm0
-; AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
-; AVX512-NEXT:    vzeroupper
-; AVX512-NEXT:    retq
+; AVX512DQ-LABEL: splatvar_shift_v2i8:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512DQ-NEXT:    vpmovsxbd %xmm0, %zmm0
+; AVX512DQ-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; AVX512DQ-NEXT:    vpsravd %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    retq
 ;
-; AVX512VL-LABEL: splatvar_shift_v2i8:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsllq $56, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpsraq $56, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[0],zero,zero,zero,zero,zero,zero,zero
-; AVX512VL-NEXT:    vpsravq %xmm1, %xmm0, %xmm0
-; AVX512VL-NEXT:    retq
+; AVX512BW-LABEL: splatvar_shift_v2i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512BW-NEXT:    vpmovsxbw %xmm0, %ymm0
+; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512BW-NEXT:    vpsravw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512DQVL-LABEL: splatvar_shift_v2i8:
+; AVX512DQVL:       # %bb.0:
+; AVX512DQVL-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512DQVL-NEXT:    vpmovsxbd %xmm0, %zmm0
+; AVX512DQVL-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; AVX512DQVL-NEXT:    vpsravd %zmm1, %zmm0, %zmm0
+; AVX512DQVL-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512DQVL-NEXT:    vzeroupper
+; AVX512DQVL-NEXT:    retq
+;
+; AVX512BWVL-LABEL: splatvar_shift_v2i8:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512BWVL-NEXT:    vpmovsxbw %xmm0, %ymm0
+; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512BWVL-NEXT:    vpsravw %ymm1, %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm0
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
 ;
 ; X32-SSE-LABEL: splatvar_shift_v2i8:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    psllq $56, %xmm0
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm2
-; X32-SSE-NEXT:    psrad $31, %xmm2
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
-; X32-SSE-NEXT:    psrad $24, %xmm0
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
-; X32-SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm1
-; X32-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [0,2147483648,0,2147483648]
-; X32-SSE-NEXT:    movdqa %xmm2, %xmm3
-; X32-SSE-NEXT:    psrlq %xmm1, %xmm3
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[2,3,0,1]
-; X32-SSE-NEXT:    psrlq %xmm4, %xmm2
-; X32-SSE-NEXT:    movsd {{.*#+}} xmm2 = xmm3[0],xmm2[1]
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm3
-; X32-SSE-NEXT:    psrlq %xmm1, %xmm3
-; X32-SSE-NEXT:    psrlq %xmm4, %xmm0
-; X32-SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm3[0],xmm0[1]
-; X32-SSE-NEXT:    xorpd %xmm2, %xmm0
-; X32-SSE-NEXT:    psubq %xmm2, %xmm0
+; X32-SSE-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0]
+; X32-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X32-SSE-NEXT:    psrlw %xmm1, %xmm0
+; X32-SSE-NEXT:    pcmpeqd %xmm2, %xmm2
+; X32-SSE-NEXT:    psrlw %xmm1, %xmm2
+; X32-SSE-NEXT:    psrlw $8, %xmm2
+; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,0,2,3,4,5,6,7]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
+; X32-SSE-NEXT:    pand %xmm2, %xmm0
+; X32-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [32896,32896,32896,32896,32896,32896,32896,32896]
+; X32-SSE-NEXT:    psrlw %xmm1, %xmm2
+; X32-SSE-NEXT:    pxor %xmm2, %xmm0
+; X32-SSE-NEXT:    psubb %xmm2, %xmm0
 ; X32-SSE-NEXT:    retl
   %splat = shufflevector <2 x i8> %b, <2 x i8> undef, <2 x i32> zeroinitializer
   %shift = ashr <2 x i8> %a, %splat
@@ -1882,100 +1728,62 @@ define <2 x i8> @splatvar_shift_v2i8(<2 x i8> %a, <2 x i8> %b) nounwind {
 define <2 x i32> @constant_shift_v2i32(<2 x i32> %a) nounwind {
 ; SSE2-LABEL: constant_shift_v2i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    psllq $32, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,3,2,3]
-; SSE2-NEXT:    psrad $31, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psrad $4, %xmm1
+; SSE2-NEXT:    psrad $5, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
 ; SSE2-NEXT:    movdqa %xmm1, %xmm0
-; SSE2-NEXT:    psrlq $4, %xmm0
-; SSE2-NEXT:    psrlq $5, %xmm1
-; SSE2-NEXT:    movsd {{.*#+}} xmm1 = xmm0[0],xmm1[1]
-; SSE2-NEXT:    movapd {{.*#+}} xmm0 = [576460752303423488,288230376151711744]
-; SSE2-NEXT:    xorpd %xmm0, %xmm1
-; SSE2-NEXT:    psubq %xmm0, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: constant_shift_v2i32:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    psllq $32, %xmm1
-; SSE41-NEXT:    psrad $31, %xmm1
-; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    psrlq $5, %xmm0
-; SSE41-NEXT:    psrlq $4, %xmm1
-; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm0[4,5,6,7]
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [576460752303423488,288230376151711744]
-; SSE41-NEXT:    pxor %xmm0, %xmm1
-; SSE41-NEXT:    psubq %xmm0, %xmm1
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    psrad $5, %xmm1
+; SSE41-NEXT:    psrad $4, %xmm0
+; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5,6,7]
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: constant_shift_v2i32:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpsllq $32, %xmm0, %xmm1
-; AVX1-NEXT:    vpsrad $31, %xmm1, %xmm1
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
-; AVX1-NEXT:    vpsrlq $5, %xmm0, %xmm1
-; AVX1-NEXT:    vpsrlq $4, %xmm0, %xmm0
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4,5,6,7]
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [576460752303423488,288230376151711744]
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsubq %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpsrad $5, %xmm0, %xmm1
+; AVX1-NEXT:    vpsrad $4, %xmm0, %xmm0
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5,6,7]
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: constant_shift_v2i32:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpsllq $32, %xmm0, %xmm1
-; AVX2-NEXT:    vpsrad $31, %xmm1, %xmm1
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
-; AVX2-NEXT:    vpsrlvq {{.*}}(%rip), %xmm0, %xmm0
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [576460752303423488,288230376151711744]
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpsubq %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpsravd {{.*}}(%rip), %xmm0, %xmm0
 ; AVX2-NEXT:    retq
 ;
-; XOP-LABEL: constant_shift_v2i32:
-; XOP:       # %bb.0:
-; XOP-NEXT:    vpsllq $32, %xmm0, %xmm0
-; XOP-NEXT:    vpshaq {{.*}}(%rip), %xmm0, %xmm0
-; XOP-NEXT:    vpshaq {{.*}}(%rip), %xmm0, %xmm0
-; XOP-NEXT:    retq
+; XOPAVX1-LABEL: constant_shift_v2i32:
+; XOPAVX1:       # %bb.0:
+; XOPAVX1-NEXT:    vpshad {{.*}}(%rip), %xmm0, %xmm0
+; XOPAVX1-NEXT:    retq
+;
+; XOPAVX2-LABEL: constant_shift_v2i32:
+; XOPAVX2:       # %bb.0:
+; XOPAVX2-NEXT:    vpsravd {{.*}}(%rip), %xmm0, %xmm0
+; XOPAVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: constant_shift_v2i32:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsllq $32, %xmm0, %xmm0
-; AVX512-NEXT:    vpsraq $32, %zmm0, %zmm0
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = [4,5]
-; AVX512-NEXT:    vpsravq %zmm1, %zmm0, %zmm0
-; AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
-; AVX512-NEXT:    vzeroupper
+; AVX512-NEXT:    vpsravd {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512-NEXT:    retq
 ;
 ; AVX512VL-LABEL: constant_shift_v2i32:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsllq $32, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpsraq $32, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpsravq {{.*}}(%rip), %xmm0, %xmm0
+; AVX512VL-NEXT:    vpsravd {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; X32-SSE-LABEL: constant_shift_v2i32:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    psllq $32, %xmm0
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,3,2,3]
-; X32-SSE-NEXT:    psrad $31, %xmm0
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
+; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
+; X32-SSE-NEXT:    psrad $4, %xmm1
+; X32-SSE-NEXT:    psrad $5, %xmm0
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
 ; X32-SSE-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
 ; X32-SSE-NEXT:    movdqa %xmm1, %xmm0
-; X32-SSE-NEXT:    psrlq $4, %xmm0
-; X32-SSE-NEXT:    psrlq $5, %xmm1
-; X32-SSE-NEXT:    movsd {{.*#+}} xmm1 = xmm0[0],xmm1[1]
-; X32-SSE-NEXT:    movapd {{.*#+}} xmm0 = [3.7857669957336791E-270,2.0522684006491881E-289]
-; X32-SSE-NEXT:    xorpd %xmm0, %xmm1
-; X32-SSE-NEXT:    psubq %xmm0, %xmm1
-; X32-SSE-NEXT:    movdqa %xmm1, %xmm0
 ; X32-SSE-NEXT:    retl
   %shift = ashr <2 x i32> %a, <i32 4, i32 5>
   ret <2 x i32> %shift
@@ -1984,96 +1792,83 @@ define <2 x i32> @constant_shift_v2i32(<2 x i32> %a) nounwind {
 define <4 x i16> @constant_shift_v4i16(<4 x i16> %a) nounwind {
 ; SSE2-LABEL: constant_shift_v4i16:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pslld $16, %xmm0
 ; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    psrad $16, %xmm1
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    psrad $19, %xmm2
-; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    psrad $18, %xmm3
-; SSE2-NEXT:    punpckhqdq {{.*#+}} xmm3 = xmm3[1],xmm2[1]
-; SSE2-NEXT:    psrad $17, %xmm0
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,3],xmm3[0,3]
+; SSE2-NEXT:    psraw $2, %xmm1
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,0],xmm0[0,0]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[2,0],xmm0[2,3]
+; SSE2-NEXT:    movaps {{.*#+}} xmm2 = [65535,0,65535,0,65535,65535,65535,65535]
 ; SSE2-NEXT:    movaps %xmm1, %xmm0
+; SSE2-NEXT:    andps %xmm2, %xmm0
+; SSE2-NEXT:    psraw $1, %xmm1
+; SSE2-NEXT:    andnps %xmm1, %xmm2
+; SSE2-NEXT:    orps %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: constant_shift_v4i16:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pslld $16, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    psrad $16, %xmm1
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    psrad $19, %xmm2
-; SSE41-NEXT:    movdqa %xmm0, %xmm3
-; SSE41-NEXT:    psrad $17, %xmm3
-; SSE41-NEXT:    pblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm2[4,5,6,7]
-; SSE41-NEXT:    psrad $18, %xmm0
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4,5,6,7]
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm3[2,3],xmm0[4,5],xmm3[6,7]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = <u,32768,16384,8192,u,u,u,u>
+; SSE41-NEXT:    pmulhw %xmm0, %xmm1
+; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3,4,5,6,7]
+; SSE41-NEXT:    psraw $1, %xmm0
+; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2,3,4,5,6,7]
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: constant_shift_v4i16:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpslld $16, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrad $16, %xmm0, %xmm1
-; AVX1-NEXT:    vpsrad $19, %xmm0, %xmm2
-; AVX1-NEXT:    vpsrad $17, %xmm0, %xmm3
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
-; AVX1-NEXT:    vpsrad $18, %xmm0, %xmm0
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4,5,6,7]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
-; AVX1-NEXT:    retq
+; AVX-LABEL: constant_shift_v4i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpmulhw {{.*}}(%rip), %xmm0, %xmm1
+; AVX-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3,4,5,6,7]
+; AVX-NEXT:    vpsraw $1, %xmm0, %xmm0
+; AVX-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2,3,4,5,6,7]
+; AVX-NEXT:    retq
 ;
-; AVX2-LABEL: constant_shift_v4i16:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpslld $16, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrad $16, %xmm0, %xmm0
-; AVX2-NEXT:    vpsravd {{.*}}(%rip), %xmm0, %xmm0
-; AVX2-NEXT:    retq
+; XOP-LABEL: constant_shift_v4i16:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vpshaw {{.*}}(%rip), %xmm0, %xmm0
+; XOP-NEXT:    retq
 ;
-; XOPAVX1-LABEL: constant_shift_v4i16:
-; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vpslld $16, %xmm0, %xmm0
-; XOPAVX1-NEXT:    vpsrad $16, %xmm0, %xmm0
-; XOPAVX1-NEXT:    vpshad {{.*}}(%rip), %xmm0, %xmm0
-; XOPAVX1-NEXT:    retq
+; AVX512DQ-LABEL: constant_shift_v4i16:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vpmovsxwd %xmm0, %ymm0
+; AVX512DQ-NEXT:    vpsravd {{.*}}(%rip), %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    retq
 ;
-; XOPAVX2-LABEL: constant_shift_v4i16:
-; XOPAVX2:       # %bb.0:
-; XOPAVX2-NEXT:    vpslld $16, %xmm0, %xmm0
-; XOPAVX2-NEXT:    vpsrad $16, %xmm0, %xmm0
-; XOPAVX2-NEXT:    vpsravd {{.*}}(%rip), %xmm0, %xmm0
-; XOPAVX2-NEXT:    retq
+; AVX512BW-LABEL: constant_shift_v4i16:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm1 = <0,1,2,3,u,u,u,u>
+; AVX512BW-NEXT:    vpsravw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
 ;
-; AVX512-LABEL: constant_shift_v4i16:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpslld $16, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrad $16, %xmm0, %xmm0
-; AVX512-NEXT:    vpsravd {{.*}}(%rip), %xmm0, %xmm0
-; AVX512-NEXT:    retq
+; AVX512DQVL-LABEL: constant_shift_v4i16:
+; AVX512DQVL:       # %bb.0:
+; AVX512DQVL-NEXT:    vpmovsxwd %xmm0, %ymm0
+; AVX512DQVL-NEXT:    vpsravd {{.*}}(%rip), %ymm0, %ymm0
+; AVX512DQVL-NEXT:    vpmovdw %ymm0, %xmm0
+; AVX512DQVL-NEXT:    vzeroupper
+; AVX512DQVL-NEXT:    retq
 ;
-; AVX512VL-LABEL: constant_shift_v4i16:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpslld $16, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpsrad $16, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpsravd {{.*}}(%rip), %xmm0, %xmm0
-; AVX512VL-NEXT:    retq
+; AVX512BWVL-LABEL: constant_shift_v4i16:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpsravw {{.*}}(%rip), %xmm0, %xmm0
+; AVX512BWVL-NEXT:    retq
 ;
 ; X32-SSE-LABEL: constant_shift_v4i16:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pslld $16, %xmm0
 ; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
-; X32-SSE-NEXT:    psrad $16, %xmm1
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm2
-; X32-SSE-NEXT:    psrad $19, %xmm2
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm3
-; X32-SSE-NEXT:    psrad $18, %xmm3
-; X32-SSE-NEXT:    punpckhqdq {{.*#+}} xmm3 = xmm3[1],xmm2[1]
-; X32-SSE-NEXT:    psrad $17, %xmm0
-; X32-SSE-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
-; X32-SSE-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,3],xmm3[0,3]
+; X32-SSE-NEXT:    psraw $2, %xmm1
+; X32-SSE-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,0],xmm0[0,0]
+; X32-SSE-NEXT:    shufps {{.*#+}} xmm1 = xmm1[2,0],xmm0[2,3]
+; X32-SSE-NEXT:    movaps {{.*#+}} xmm2 = [65535,0,65535,0,65535,65535,65535,65535]
 ; X32-SSE-NEXT:    movaps %xmm1, %xmm0
+; X32-SSE-NEXT:    andps %xmm2, %xmm0
+; X32-SSE-NEXT:    psraw $1, %xmm1
+; X32-SSE-NEXT:    andnps %xmm1, %xmm2
+; X32-SSE-NEXT:    orps %xmm2, %xmm0
 ; X32-SSE-NEXT:    retl
   %shift = ashr <4 x i16> %a, <i16 0, i16 1, i16 2, i16 3>
   ret <4 x i16> %shift
@@ -2082,425 +1877,336 @@ define <4 x i16> @constant_shift_v4i16(<4 x i16> %a) nounwind {
 define <2 x i16> @constant_shift_v2i16(<2 x i16> %a) nounwind {
 ; SSE2-LABEL: constant_shift_v2i16:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    psllq $48, %xmm0
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    psrad $31, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
-; SSE2-NEXT:    psrad $16, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    psrlq $2, %xmm1
-; SSE2-NEXT:    psrlq $3, %xmm0
-; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
-; SSE2-NEXT:    movapd {{.*#+}} xmm1 = [2305843009213693952,1152921504606846976]
-; SSE2-NEXT:    xorpd %xmm1, %xmm0
-; SSE2-NEXT:    psubq %xmm1, %xmm0
+; SSE2-NEXT:    psraw $3, %xmm1
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [65535,0,65535,65535,65535,65535,65535,65535]
+; SSE2-NEXT:    psraw $2, %xmm0
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    pandn %xmm1, %xmm2
+; SSE2-NEXT:    por %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: constant_shift_v2i16:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    psllq $48, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    psrad $31, %xmm1
-; SSE41-NEXT:    psrad $16, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
 ; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    psrlq $3, %xmm1
-; SSE41-NEXT:    psrlq $2, %xmm0
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4,5,6,7]
-; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [2305843009213693952,1152921504606846976]
-; SSE41-NEXT:    pxor %xmm1, %xmm0
-; SSE41-NEXT:    psubq %xmm1, %xmm0
+; SSE41-NEXT:    psraw $3, %xmm1
+; SSE41-NEXT:    psraw $2, %xmm0
+; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3,4,5,6,7]
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: constant_shift_v2i16:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpsllq $48, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrad $31, %xmm0, %xmm1
-; AVX1-NEXT:    vpsrad $16, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
-; AVX1-NEXT:    vpsrlq $3, %xmm0, %xmm1
-; AVX1-NEXT:    vpsrlq $2, %xmm0, %xmm0
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4,5,6,7]
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [2305843009213693952,1152921504606846976]
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsubq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: constant_shift_v2i16:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpsllq $48, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrad $31, %xmm0, %xmm1
-; AVX2-NEXT:    vpsrad $16, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
-; AVX2-NEXT:    vpsrlvq {{.*}}(%rip), %xmm0, %xmm0
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [2305843009213693952,1152921504606846976]
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpsubq %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    retq
+; AVX-LABEL: constant_shift_v2i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpsraw $3, %xmm0, %xmm1
+; AVX-NEXT:    vpsraw $2, %xmm0, %xmm0
+; AVX-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3,4,5,6,7]
+; AVX-NEXT:    retq
 ;
 ; XOP-LABEL: constant_shift_v2i16:
 ; XOP:       # %bb.0:
-; XOP-NEXT:    vpsllq $48, %xmm0, %xmm0
-; XOP-NEXT:    vpshaq {{.*}}(%rip), %xmm0, %xmm0
-; XOP-NEXT:    vpshaq {{.*}}(%rip), %xmm0, %xmm0
+; XOP-NEXT:    vpshaw {{.*}}(%rip), %xmm0, %xmm0
 ; XOP-NEXT:    retq
 ;
-; AVX512-LABEL: constant_shift_v2i16:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsllq $48, %xmm0, %xmm0
-; AVX512-NEXT:    vpsraq $48, %zmm0, %zmm0
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = [2,3]
-; AVX512-NEXT:    vpsravq %zmm1, %zmm0, %zmm0
-; AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
-; AVX512-NEXT:    vzeroupper
-; AVX512-NEXT:    retq
+; AVX512DQ-LABEL: constant_shift_v2i16:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vpsraw $3, %xmm0, %xmm1
+; AVX512DQ-NEXT:    vpsraw $2, %xmm0, %xmm0
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3,4,5,6,7]
+; AVX512DQ-NEXT:    retq
 ;
-; AVX512VL-LABEL: constant_shift_v2i16:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsllq $48, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpsraq $48, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpsravq {{.*}}(%rip), %xmm0, %xmm0
-; AVX512VL-NEXT:    retq
+; AVX512BW-LABEL: constant_shift_v2i16:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm1 = <2,3,u,u,u,u,u,u>
+; AVX512BW-NEXT:    vpsravw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512DQVL-LABEL: constant_shift_v2i16:
+; AVX512DQVL:       # %bb.0:
+; AVX512DQVL-NEXT:    vpsraw $3, %xmm0, %xmm1
+; AVX512DQVL-NEXT:    vpsraw $2, %xmm0, %xmm0
+; AVX512DQVL-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3,4,5,6,7]
+; AVX512DQVL-NEXT:    retq
+;
+; AVX512BWVL-LABEL: constant_shift_v2i16:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpsravw {{.*}}(%rip), %xmm0, %xmm0
+; AVX512BWVL-NEXT:    retq
 ;
 ; X32-SSE-LABEL: constant_shift_v2i16:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    psllq $48, %xmm0
 ; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
-; X32-SSE-NEXT:    psrad $31, %xmm1
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
-; X32-SSE-NEXT:    psrad $16, %xmm0
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
-; X32-SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
-; X32-SSE-NEXT:    psrlq $2, %xmm1
-; X32-SSE-NEXT:    psrlq $3, %xmm0
-; X32-SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
-; X32-SSE-NEXT:    movapd {{.*#+}} xmm1 = [1.4916681462400413E-154,1.2882297539194267E-231]
-; X32-SSE-NEXT:    xorpd %xmm1, %xmm0
-; X32-SSE-NEXT:    psubq %xmm1, %xmm0
+; X32-SSE-NEXT:    psraw $3, %xmm1
+; X32-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [65535,0,65535,65535,65535,65535,65535,65535]
+; X32-SSE-NEXT:    psraw $2, %xmm0
+; X32-SSE-NEXT:    pand %xmm2, %xmm0
+; X32-SSE-NEXT:    pandn %xmm1, %xmm2
+; X32-SSE-NEXT:    por %xmm2, %xmm0
 ; X32-SSE-NEXT:    retl
   %shift = ashr <2 x i16> %a, <i16 2, i16 3>
   ret <2 x i16> %shift
 }
 
 define <8 x i8> @constant_shift_v8i8(<8 x i8> %a) nounwind {
-; SSE2-LABEL: constant_shift_v8i8:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    psllw $8, %xmm0
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    psraw $8, %xmm1
-; SSE2-NEXT:    psraw $12, %xmm0
-; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
-; SSE2-NEXT:    movapd %xmm0, %xmm1
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,2],xmm0[2,3]
-; SSE2-NEXT:    psraw $2, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
-; SSE2-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; SSE2-NEXT:    movaps {{.*#+}} xmm2 = [65535,0,65535,0,65535,0,65535,0]
-; SSE2-NEXT:    movaps %xmm1, %xmm0
-; SSE2-NEXT:    andps %xmm2, %xmm0
-; SSE2-NEXT:    psraw $1, %xmm1
-; SSE2-NEXT:    andnps %xmm1, %xmm2
-; SSE2-NEXT:    orps %xmm2, %xmm0
-; SSE2-NEXT:    retq
+; SSE-LABEL: constant_shift_v8i8:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pxor %xmm1, %xmm1
+; SSE-NEXT:    movdqa %xmm0, %xmm2
+; SSE-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
+; SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE-NEXT:    psraw $8, %xmm0
+; SSE-NEXT:    pmullw {{.*}}(%rip), %xmm0
+; SSE-NEXT:    psrlw $8, %xmm0
+; SSE-NEXT:    packuswb %xmm2, %xmm0
+; SSE-NEXT:    retq
 ;
-; SSE41-LABEL: constant_shift_v8i8:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    psllw $8, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    psraw $8, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = <u,32768,16384,8192,4096,2048,1024,512>
-; SSE41-NEXT:    pmulhw %xmm1, %xmm2
-; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3,4,5,6,7]
-; SSE41-NEXT:    psraw $9, %xmm0
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2,3,4,5,6,7]
-; SSE41-NEXT:    retq
+; AVX1-LABEL: constant_shift_v8i8:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; AVX1-NEXT:    vpsraw $8, %xmm0, %xmm0
+; AVX1-NEXT:    vpmullw {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm0
+; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    retq
 ;
-; AVX-LABEL: constant_shift_v8i8:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vpsllw $8, %xmm0, %xmm0
-; AVX-NEXT:    vpsraw $8, %xmm0, %xmm1
-; AVX-NEXT:    vpmulhw {{.*}}(%rip), %xmm1, %xmm2
-; AVX-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3,4,5,6,7]
-; AVX-NEXT:    vpsraw $9, %xmm0, %xmm0
-; AVX-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2,3,4,5,6,7]
-; AVX-NEXT:    retq
+; AVX2-LABEL: constant_shift_v8i8:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpmovsxbw %xmm0, %ymm0
+; AVX2-NEXT:    vpmullw {{.*}}(%rip), %ymm0, %ymm0
+; AVX2-NEXT:    vpsrlw $8, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
 ;
 ; XOP-LABEL: constant_shift_v8i8:
 ; XOP:       # %bb.0:
-; XOP-NEXT:    vpsllw $8, %xmm0, %xmm0
-; XOP-NEXT:    vpsraw $8, %xmm0, %xmm0
-; XOP-NEXT:    vpshaw {{.*}}(%rip), %xmm0, %xmm0
+; XOP-NEXT:    vpshab {{.*}}(%rip), %xmm0, %xmm0
 ; XOP-NEXT:    retq
 ;
 ; AVX512DQ-LABEL: constant_shift_v8i8:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vpsllw $8, %xmm0, %xmm0
-; AVX512DQ-NEXT:    vpsraw $8, %xmm0, %xmm0
-; AVX512DQ-NEXT:    vpmovsxwd %xmm0, %ymm0
-; AVX512DQ-NEXT:    vpsravd {{.*}}(%rip), %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpmovdw %zmm0, %ymm0
-; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512DQ-NEXT:    vpmovsxbd %xmm0, %zmm0
+; AVX512DQ-NEXT:    vpsravd {{.*}}(%rip), %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpmovdb %zmm0, %xmm0
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512BW-LABEL: constant_shift_v8i8:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpsllw $8, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpsraw $8, %xmm0, %xmm0
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm1 = [0,1,2,3,4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,1,2,3,4,5,6,7,0,0,0,0,0,0,0,0]
+; AVX512BW-NEXT:    vpmovsxbw %xmm0, %ymm0
 ; AVX512BW-NEXT:    vpsravw %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512DQVL-LABEL: constant_shift_v8i8:
 ; AVX512DQVL:       # %bb.0:
-; AVX512DQVL-NEXT:    vpsllw $8, %xmm0, %xmm0
-; AVX512DQVL-NEXT:    vpsraw $8, %xmm0, %xmm0
-; AVX512DQVL-NEXT:    vpmovsxwd %xmm0, %ymm0
-; AVX512DQVL-NEXT:    vpsravd {{.*}}(%rip), %ymm0, %ymm0
-; AVX512DQVL-NEXT:    vpmovdw %ymm0, %xmm0
+; AVX512DQVL-NEXT:    vpmovsxbd %xmm0, %zmm0
+; AVX512DQVL-NEXT:    vpsravd {{.*}}(%rip), %zmm0, %zmm0
+; AVX512DQVL-NEXT:    vpmovdb %zmm0, %xmm0
 ; AVX512DQVL-NEXT:    vzeroupper
 ; AVX512DQVL-NEXT:    retq
 ;
 ; AVX512BWVL-LABEL: constant_shift_v8i8:
 ; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vpsllw $8, %xmm0, %xmm0
-; AVX512BWVL-NEXT:    vpsraw $8, %xmm0, %xmm0
-; AVX512BWVL-NEXT:    vpsravw {{.*}}(%rip), %xmm0, %xmm0
+; AVX512BWVL-NEXT:    vpmovsxbw %xmm0, %ymm0
+; AVX512BWVL-NEXT:    vpsravw {{.*}}(%rip), %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm0
+; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
 ;
 ; X32-SSE-LABEL: constant_shift_v8i8:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    psllw $8, %xmm0
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
-; X32-SSE-NEXT:    psraw $8, %xmm1
-; X32-SSE-NEXT:    psraw $12, %xmm0
-; X32-SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
-; X32-SSE-NEXT:    movapd %xmm0, %xmm1
-; X32-SSE-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,2],xmm0[2,3]
-; X32-SSE-NEXT:    psraw $2, %xmm0
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
-; X32-SSE-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; X32-SSE-NEXT:    movaps {{.*#+}} xmm2 = [65535,0,65535,0,65535,0,65535,0]
-; X32-SSE-NEXT:    movaps %xmm1, %xmm0
-; X32-SSE-NEXT:    andps %xmm2, %xmm0
-; X32-SSE-NEXT:    psraw $1, %xmm1
-; X32-SSE-NEXT:    andnps %xmm1, %xmm2
-; X32-SSE-NEXT:    orps %xmm2, %xmm0
+; X32-SSE-NEXT:    pxor %xmm1, %xmm1
+; X32-SSE-NEXT:    movdqa %xmm0, %xmm2
+; X32-SSE-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
+; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X32-SSE-NEXT:    psraw $8, %xmm0
+; X32-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
+; X32-SSE-NEXT:    psrlw $8, %xmm0
+; X32-SSE-NEXT:    packuswb %xmm2, %xmm0
 ; X32-SSE-NEXT:    retl
   %shift = ashr <8 x i8> %a, <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7>
   ret <8 x i8> %shift
 }
 
 define <4 x i8> @constant_shift_v4i8(<4 x i8> %a) nounwind {
-; SSE2-LABEL: constant_shift_v4i8:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    pslld $24, %xmm0
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    psrad $24, %xmm1
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    psrad $27, %xmm2
-; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    psrad $26, %xmm3
-; SSE2-NEXT:    punpckhqdq {{.*#+}} xmm3 = xmm3[1],xmm2[1]
-; SSE2-NEXT:    psrad $25, %xmm0
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,3],xmm3[0,3]
-; SSE2-NEXT:    movaps %xmm1, %xmm0
-; SSE2-NEXT:    retq
-;
-; SSE41-LABEL: constant_shift_v4i8:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pslld $24, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    psrad $24, %xmm1
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    psrad $27, %xmm2
-; SSE41-NEXT:    movdqa %xmm0, %xmm3
-; SSE41-NEXT:    psrad $25, %xmm3
-; SSE41-NEXT:    pblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm2[4,5,6,7]
-; SSE41-NEXT:    psrad $26, %xmm0
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4,5,6,7]
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm3[2,3],xmm0[4,5],xmm3[6,7]
-; SSE41-NEXT:    retq
+; SSE-LABEL: constant_shift_v4i8:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pxor %xmm1, %xmm1
+; SSE-NEXT:    movdqa %xmm0, %xmm2
+; SSE-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
+; SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE-NEXT:    psraw $8, %xmm0
+; SSE-NEXT:    pmullw {{.*}}(%rip), %xmm0
+; SSE-NEXT:    psrlw $8, %xmm0
+; SSE-NEXT:    packuswb %xmm2, %xmm0
+; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: constant_shift_v4i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpslld $24, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrad $24, %xmm0, %xmm1
-; AVX1-NEXT:    vpsrad $27, %xmm0, %xmm2
-; AVX1-NEXT:    vpsrad $25, %xmm0, %xmm3
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
-; AVX1-NEXT:    vpsrad $26, %xmm0, %xmm0
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4,5,6,7]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
+; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; AVX1-NEXT:    vpsraw $8, %xmm0, %xmm0
+; AVX1-NEXT:    vpmullw {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm0
+; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: constant_shift_v4i8:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpslld $24, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrad $24, %xmm0, %xmm0
-; AVX2-NEXT:    vpsravd {{.*}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT:    vpmovsxbw %xmm0, %ymm0
+; AVX2-NEXT:    vpmullw {{.*}}(%rip), %ymm0, %ymm0
+; AVX2-NEXT:    vpsrlw $8, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
-; XOPAVX1-LABEL: constant_shift_v4i8:
-; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vpslld $24, %xmm0, %xmm0
-; XOPAVX1-NEXT:    vpsrad $24, %xmm0, %xmm0
-; XOPAVX1-NEXT:    vpshad {{.*}}(%rip), %xmm0, %xmm0
-; XOPAVX1-NEXT:    retq
+; XOP-LABEL: constant_shift_v4i8:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vpshab {{.*}}(%rip), %xmm0, %xmm0
+; XOP-NEXT:    retq
 ;
-; XOPAVX2-LABEL: constant_shift_v4i8:
-; XOPAVX2:       # %bb.0:
-; XOPAVX2-NEXT:    vpslld $24, %xmm0, %xmm0
-; XOPAVX2-NEXT:    vpsrad $24, %xmm0, %xmm0
-; XOPAVX2-NEXT:    vpsravd {{.*}}(%rip), %xmm0, %xmm0
-; XOPAVX2-NEXT:    retq
+; AVX512DQ-LABEL: constant_shift_v4i8:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vpmovsxbd %xmm0, %zmm0
+; AVX512DQ-NEXT:    vpsravd {{.*}}(%rip), %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    retq
 ;
-; AVX512-LABEL: constant_shift_v4i8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpslld $24, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrad $24, %xmm0, %xmm0
-; AVX512-NEXT:    vpsravd {{.*}}(%rip), %xmm0, %xmm0
-; AVX512-NEXT:    retq
+; AVX512BW-LABEL: constant_shift_v4i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,1,2,3,0,0,0,0,0,0,0,0,0,0,0,0]
+; AVX512BW-NEXT:    vpmovsxbw %xmm0, %ymm0
+; AVX512BW-NEXT:    vpsravw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
 ;
-; AVX512VL-LABEL: constant_shift_v4i8:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpslld $24, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpsrad $24, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpsravd {{.*}}(%rip), %xmm0, %xmm0
-; AVX512VL-NEXT:    retq
+; AVX512DQVL-LABEL: constant_shift_v4i8:
+; AVX512DQVL:       # %bb.0:
+; AVX512DQVL-NEXT:    vpmovsxbd %xmm0, %zmm0
+; AVX512DQVL-NEXT:    vpsravd {{.*}}(%rip), %zmm0, %zmm0
+; AVX512DQVL-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512DQVL-NEXT:    vzeroupper
+; AVX512DQVL-NEXT:    retq
+;
+; AVX512BWVL-LABEL: constant_shift_v4i8:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpmovsxbw %xmm0, %ymm0
+; AVX512BWVL-NEXT:    vpsravw {{.*}}(%rip), %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm0
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
 ;
 ; X32-SSE-LABEL: constant_shift_v4i8:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pslld $24, %xmm0
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
-; X32-SSE-NEXT:    psrad $24, %xmm1
+; X32-SSE-NEXT:    pxor %xmm1, %xmm1
 ; X32-SSE-NEXT:    movdqa %xmm0, %xmm2
-; X32-SSE-NEXT:    psrad $27, %xmm2
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm3
-; X32-SSE-NEXT:    psrad $26, %xmm3
-; X32-SSE-NEXT:    punpckhqdq {{.*#+}} xmm3 = xmm3[1],xmm2[1]
-; X32-SSE-NEXT:    psrad $25, %xmm0
-; X32-SSE-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
-; X32-SSE-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,3],xmm3[0,3]
-; X32-SSE-NEXT:    movaps %xmm1, %xmm0
+; X32-SSE-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
+; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X32-SSE-NEXT:    psraw $8, %xmm0
+; X32-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
+; X32-SSE-NEXT:    psrlw $8, %xmm0
+; X32-SSE-NEXT:    packuswb %xmm2, %xmm0
 ; X32-SSE-NEXT:    retl
   %shift = ashr <4 x i8> %a, <i8 0, i8 1, i8 2, i8 3>
   ret <4 x i8> %shift
 }
 
 define <2 x i8> @constant_shift_v2i8(<2 x i8> %a) nounwind {
-; SSE2-LABEL: constant_shift_v2i8:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    psllq $56, %xmm0
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    psrad $31, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
-; SSE2-NEXT:    psrad $24, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    psrlq $2, %xmm1
-; SSE2-NEXT:    psrlq $3, %xmm0
-; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
-; SSE2-NEXT:    movapd {{.*#+}} xmm1 = [2305843009213693952,1152921504606846976]
-; SSE2-NEXT:    xorpd %xmm1, %xmm0
-; SSE2-NEXT:    psubq %xmm1, %xmm0
-; SSE2-NEXT:    retq
-;
-; SSE41-LABEL: constant_shift_v2i8:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    psllq $56, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    psrad $31, %xmm1
-; SSE41-NEXT:    psrad $24, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
-; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    psrlq $3, %xmm1
-; SSE41-NEXT:    psrlq $2, %xmm0
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4,5,6,7]
-; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [2305843009213693952,1152921504606846976]
-; SSE41-NEXT:    pxor %xmm1, %xmm0
-; SSE41-NEXT:    psubq %xmm1, %xmm0
-; SSE41-NEXT:    retq
+; SSE-LABEL: constant_shift_v2i8:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pxor %xmm1, %xmm1
+; SSE-NEXT:    movdqa %xmm0, %xmm2
+; SSE-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
+; SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE-NEXT:    psraw $8, %xmm0
+; SSE-NEXT:    pmullw {{.*}}(%rip), %xmm0
+; SSE-NEXT:    psrlw $8, %xmm0
+; SSE-NEXT:    packuswb %xmm2, %xmm0
+; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: constant_shift_v2i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpsllq $56, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrad $31, %xmm0, %xmm1
-; AVX1-NEXT:    vpsrad $24, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
-; AVX1-NEXT:    vpsrlq $3, %xmm0, %xmm1
-; AVX1-NEXT:    vpsrlq $2, %xmm0, %xmm0
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4,5,6,7]
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [2305843009213693952,1152921504606846976]
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsubq %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; AVX1-NEXT:    vpsraw $8, %xmm0, %xmm0
+; AVX1-NEXT:    vpmullw {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm0
+; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: constant_shift_v2i8:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpsllq $56, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrad $31, %xmm0, %xmm1
-; AVX2-NEXT:    vpsrad $24, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
-; AVX2-NEXT:    vpsrlvq {{.*}}(%rip), %xmm0, %xmm0
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [2305843009213693952,1152921504606846976]
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpsubq %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpmovsxbw %xmm0, %ymm0
+; AVX2-NEXT:    vpmullw {{.*}}(%rip), %ymm0, %ymm0
+; AVX2-NEXT:    vpsrlw $8, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
 ; XOP-LABEL: constant_shift_v2i8:
 ; XOP:       # %bb.0:
-; XOP-NEXT:    vpsllq $56, %xmm0, %xmm0
-; XOP-NEXT:    vpshaq {{.*}}(%rip), %xmm0, %xmm0
-; XOP-NEXT:    vpshaq {{.*}}(%rip), %xmm0, %xmm0
+; XOP-NEXT:    vpshab {{.*}}(%rip), %xmm0, %xmm0
 ; XOP-NEXT:    retq
 ;
-; AVX512-LABEL: constant_shift_v2i8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsllq $56, %xmm0, %xmm0
-; AVX512-NEXT:    vpsraq $56, %zmm0, %zmm0
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = [2,3]
-; AVX512-NEXT:    vpsravq %zmm1, %zmm0, %zmm0
-; AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
-; AVX512-NEXT:    vzeroupper
-; AVX512-NEXT:    retq
+; AVX512DQ-LABEL: constant_shift_v2i8:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vpmovsxbd %xmm0, %zmm0
+; AVX512DQ-NEXT:    vpsravd {{.*}}(%rip), %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    retq
 ;
-; AVX512VL-LABEL: constant_shift_v2i8:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsllq $56, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpsraq $56, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpsravq {{.*}}(%rip), %xmm0, %xmm0
-; AVX512VL-NEXT:    retq
+; AVX512BW-LABEL: constant_shift_v2i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm1 = [2,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+; AVX512BW-NEXT:    vpmovsxbw %xmm0, %ymm0
+; AVX512BW-NEXT:    vpsravw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512DQVL-LABEL: constant_shift_v2i8:
+; AVX512DQVL:       # %bb.0:
+; AVX512DQVL-NEXT:    vpmovsxbd %xmm0, %zmm0
+; AVX512DQVL-NEXT:    vpsravd {{.*}}(%rip), %zmm0, %zmm0
+; AVX512DQVL-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512DQVL-NEXT:    vzeroupper
+; AVX512DQVL-NEXT:    retq
+;
+; AVX512BWVL-LABEL: constant_shift_v2i8:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpmovsxbw %xmm0, %ymm0
+; AVX512BWVL-NEXT:    vpsravw {{.*}}(%rip), %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm0
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
 ;
 ; X32-SSE-LABEL: constant_shift_v2i8:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    psllq $56, %xmm0
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
-; X32-SSE-NEXT:    psrad $31, %xmm1
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
-; X32-SSE-NEXT:    psrad $24, %xmm0
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
-; X32-SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
-; X32-SSE-NEXT:    psrlq $2, %xmm1
-; X32-SSE-NEXT:    psrlq $3, %xmm0
-; X32-SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
-; X32-SSE-NEXT:    movapd {{.*#+}} xmm1 = [1.4916681462400413E-154,1.2882297539194267E-231]
-; X32-SSE-NEXT:    xorpd %xmm1, %xmm0
-; X32-SSE-NEXT:    psubq %xmm1, %xmm0
+; X32-SSE-NEXT:    pxor %xmm1, %xmm1
+; X32-SSE-NEXT:    movdqa %xmm0, %xmm2
+; X32-SSE-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
+; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X32-SSE-NEXT:    psraw $8, %xmm0
+; X32-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
+; X32-SSE-NEXT:    psrlw $8, %xmm0
+; X32-SSE-NEXT:    packuswb %xmm2, %xmm0
 ; X32-SSE-NEXT:    retl
   %shift = ashr <2 x i8> %a, <i8 2, i8 3>
   ret <2 x i8> %shift
@@ -2511,88 +2217,34 @@ define <2 x i8> @constant_shift_v2i8(<2 x i8> %a) nounwind {
 ;
 
 define <2 x i32> @splatconstant_shift_v2i32(<2 x i32> %a) nounwind {
-; SSE2-LABEL: splatconstant_shift_v2i32:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    psllq $32, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,3,2,3]
-; SSE2-NEXT:    psrad $31, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; SSE2-NEXT:    movdqa %xmm1, %xmm0
-; SSE2-NEXT:    psrad $5, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,3,2,3]
-; SSE2-NEXT:    psrlq $5, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; SSE2-NEXT:    retq
-;
-; SSE41-LABEL: splatconstant_shift_v2i32:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    psllq $32, %xmm1
-; SSE41-NEXT:    psrad $31, %xmm1
-; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    psrad $5, %xmm0
-; SSE41-NEXT:    psrlq $5, %xmm1
-; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm0[2,3],xmm1[4,5],xmm0[6,7]
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    retq
-;
-; AVX1-LABEL: splatconstant_shift_v2i32:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpsllq $32, %xmm0, %xmm1
-; AVX1-NEXT:    vpsrad $31, %xmm1, %xmm1
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
-; AVX1-NEXT:    vpsrad $5, %xmm0, %xmm1
-; AVX1-NEXT:    vpsrlq $5, %xmm0, %xmm0
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
-; AVX1-NEXT:    retq
+; SSE-LABEL: splatconstant_shift_v2i32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    psrad $5, %xmm0
+; SSE-NEXT:    retq
 ;
-; AVX2-LABEL: splatconstant_shift_v2i32:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpsllq $32, %xmm0, %xmm1
-; AVX2-NEXT:    vpsrad $31, %xmm1, %xmm1
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
-; AVX2-NEXT:    vpsrad $5, %xmm0, %xmm1
-; AVX2-NEXT:    vpsrlq $5, %xmm0, %xmm0
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
-; AVX2-NEXT:    retq
+; AVX-LABEL: splatconstant_shift_v2i32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpsrad $5, %xmm0, %xmm0
+; AVX-NEXT:    retq
 ;
 ; XOP-LABEL: splatconstant_shift_v2i32:
 ; XOP:       # %bb.0:
-; XOP-NEXT:    vpsllq $32, %xmm0, %xmm0
-; XOP-NEXT:    vpshaq {{.*}}(%rip), %xmm0, %xmm0
-; XOP-NEXT:    vpshaq {{.*}}(%rip), %xmm0, %xmm0
+; XOP-NEXT:    vpsrad $5, %xmm0, %xmm0
 ; XOP-NEXT:    retq
 ;
 ; AVX512-LABEL: splatconstant_shift_v2i32:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsllq $32, %xmm0, %xmm0
-; AVX512-NEXT:    vpsraq $37, %zmm0, %zmm0
-; AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
-; AVX512-NEXT:    vzeroupper
+; AVX512-NEXT:    vpsrad $5, %xmm0, %xmm0
 ; AVX512-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatconstant_shift_v2i32:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsllq $32, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpsraq $37, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpsrad $5, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; X32-SSE-LABEL: splatconstant_shift_v2i32:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    psllq $32, %xmm0
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,3,2,3]
-; X32-SSE-NEXT:    psrad $31, %xmm0
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
-; X32-SSE-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; X32-SSE-NEXT:    movdqa %xmm1, %xmm0
 ; X32-SSE-NEXT:    psrad $5, %xmm0
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,3,2,3]
-; X32-SSE-NEXT:    psrlq $5, %xmm1
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
-; X32-SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
 ; X32-SSE-NEXT:    retl
   %shift = ashr <2 x i32> %a, <i32 5, i32 5>
   ret <2 x i32> %shift
@@ -2601,135 +2253,66 @@ define <2 x i32> @splatconstant_shift_v2i32(<2 x i32> %a) nounwind {
 define <4 x i16> @splatconstant_shift_v4i16(<4 x i16> %a) nounwind {
 ; SSE-LABEL: splatconstant_shift_v4i16:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pslld $16, %xmm0
-; SSE-NEXT:    psrad $19, %xmm0
+; SSE-NEXT:    psraw $3, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: splatconstant_shift_v4i16:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpslld $16, %xmm0, %xmm0
-; AVX-NEXT:    vpsrad $19, %xmm0, %xmm0
+; AVX-NEXT:    vpsraw $3, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; XOP-LABEL: splatconstant_shift_v4i16:
 ; XOP:       # %bb.0:
-; XOP-NEXT:    vpslld $16, %xmm0, %xmm0
-; XOP-NEXT:    vpsrad $19, %xmm0, %xmm0
+; XOP-NEXT:    vpsraw $3, %xmm0, %xmm0
 ; XOP-NEXT:    retq
 ;
 ; AVX512-LABEL: splatconstant_shift_v4i16:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpslld $16, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrad $19, %xmm0, %xmm0
+; AVX512-NEXT:    vpsraw $3, %xmm0, %xmm0
 ; AVX512-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatconstant_shift_v4i16:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpslld $16, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpsrad $19, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpsraw $3, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; X32-SSE-LABEL: splatconstant_shift_v4i16:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pslld $16, %xmm0
-; X32-SSE-NEXT:    psrad $19, %xmm0
+; X32-SSE-NEXT:    psraw $3, %xmm0
 ; X32-SSE-NEXT:    retl
   %shift = ashr <4 x i16> %a, <i16 3, i16 3, i16 3, i16 3>
   ret <4 x i16> %shift
 }
 
 define <2 x i16> @splatconstant_shift_v2i16(<2 x i16> %a) nounwind {
-; SSE2-LABEL: splatconstant_shift_v2i16:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    psllq $48, %xmm0
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    psrad $31, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
-; SSE2-NEXT:    psrad $16, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    psrad $3, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
-; SSE2-NEXT:    psrlq $3, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; SSE2-NEXT:    retq
-;
-; SSE41-LABEL: splatconstant_shift_v2i16:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    psllq $48, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    psrad $31, %xmm1
-; SSE41-NEXT:    psrad $16, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
-; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    psrad $3, %xmm1
-; SSE41-NEXT:    psrlq $3, %xmm0
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
-; SSE41-NEXT:    retq
-;
-; AVX1-LABEL: splatconstant_shift_v2i16:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpsllq $48, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrad $31, %xmm0, %xmm1
-; AVX1-NEXT:    vpsrad $16, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
-; AVX1-NEXT:    vpsrad $3, %xmm0, %xmm1
-; AVX1-NEXT:    vpsrlq $3, %xmm0, %xmm0
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
-; AVX1-NEXT:    retq
+; SSE-LABEL: splatconstant_shift_v2i16:
+; SSE:       # %bb.0:
+; SSE-NEXT:    psraw $3, %xmm0
+; SSE-NEXT:    retq
 ;
-; AVX2-LABEL: splatconstant_shift_v2i16:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpsllq $48, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrad $31, %xmm0, %xmm1
-; AVX2-NEXT:    vpsrad $16, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
-; AVX2-NEXT:    vpsrad $3, %xmm0, %xmm1
-; AVX2-NEXT:    vpsrlq $3, %xmm0, %xmm0
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
-; AVX2-NEXT:    retq
+; AVX-LABEL: splatconstant_shift_v2i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpsraw $3, %xmm0, %xmm0
+; AVX-NEXT:    retq
 ;
 ; XOP-LABEL: splatconstant_shift_v2i16:
 ; XOP:       # %bb.0:
-; XOP-NEXT:    vpsllq $48, %xmm0, %xmm0
-; XOP-NEXT:    vpshaq {{.*}}(%rip), %xmm0, %xmm0
-; XOP-NEXT:    vpshaq {{.*}}(%rip), %xmm0, %xmm0
+; XOP-NEXT:    vpsraw $3, %xmm0, %xmm0
 ; XOP-NEXT:    retq
 ;
 ; AVX512-LABEL: splatconstant_shift_v2i16:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsllq $48, %xmm0, %xmm0
-; AVX512-NEXT:    vpsraq $51, %zmm0, %zmm0
-; AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
-; AVX512-NEXT:    vzeroupper
+; AVX512-NEXT:    vpsraw $3, %xmm0, %xmm0
 ; AVX512-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatconstant_shift_v2i16:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsllq $48, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpsraq $51, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpsraw $3, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; X32-SSE-LABEL: splatconstant_shift_v2i16:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    psllq $48, %xmm0
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
-; X32-SSE-NEXT:    psrad $31, %xmm1
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
-; X32-SSE-NEXT:    psrad $16, %xmm0
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
-; X32-SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
-; X32-SSE-NEXT:    psrad $3, %xmm1
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
-; X32-SSE-NEXT:    psrlq $3, %xmm0
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; X32-SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; X32-SSE-NEXT:    psraw $3, %xmm0
 ; X32-SSE-NEXT:    retl
   %shift = ashr <2 x i16> %a, <i16 3, i16 3>
   ret <2 x i16> %shift
@@ -2738,38 +2321,52 @@ define <2 x i16> @splatconstant_shift_v2i16(<2 x i16> %a) nounwind {
 define <8 x i8> @splatconstant_shift_v8i8(<8 x i8> %a) nounwind {
 ; SSE-LABEL: splatconstant_shift_v8i8:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    psllw $8, %xmm0
-; SSE-NEXT:    psraw $11, %xmm0
+; SSE-NEXT:    psrlw $3, %xmm0
+; SSE-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE-NEXT:    movdqa {{.*#+}} xmm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
+; SSE-NEXT:    pxor %xmm1, %xmm0
+; SSE-NEXT:    psubb %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: splatconstant_shift_v8i8:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpsllw $8, %xmm0, %xmm0
-; AVX-NEXT:    vpsraw $11, %xmm0, %xmm0
+; AVX-NEXT:    vpsrlw $3, %xmm0, %xmm0
+; AVX-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
+; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; XOP-LABEL: splatconstant_shift_v8i8:
 ; XOP:       # %bb.0:
-; XOP-NEXT:    vpsllw $8, %xmm0, %xmm0
-; XOP-NEXT:    vpsraw $11, %xmm0, %xmm0
+; XOP-NEXT:    vpshab {{.*}}(%rip), %xmm0, %xmm0
 ; XOP-NEXT:    retq
 ;
 ; AVX512-LABEL: splatconstant_shift_v8i8:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsllw $8, %xmm0, %xmm0
-; AVX512-NEXT:    vpsraw $11, %xmm0, %xmm0
+; AVX512-NEXT:    vpsrlw $3, %xmm0, %xmm0
+; AVX512-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
+; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatconstant_shift_v8i8:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsllw $8, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpsraw $11, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpsrlw $3, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
+; AVX512VL-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; X32-SSE-LABEL: splatconstant_shift_v8i8:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    psllw $8, %xmm0
-; X32-SSE-NEXT:    psraw $11, %xmm0
+; X32-SSE-NEXT:    psrlw $3, %xmm0
+; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X32-SSE-NEXT:    movdqa {{.*#+}} xmm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
+; X32-SSE-NEXT:    pxor %xmm1, %xmm0
+; X32-SSE-NEXT:    psubb %xmm1, %xmm0
 ; X32-SSE-NEXT:    retl
   %shift = ashr <8 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
   ret <8 x i8> %shift
@@ -2778,135 +2375,106 @@ define <8 x i8> @splatconstant_shift_v8i8(<8 x i8> %a) nounwind {
 define <4 x i8> @splatconstant_shift_v4i8(<4 x i8> %a) nounwind {
 ; SSE-LABEL: splatconstant_shift_v4i8:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pslld $24, %xmm0
-; SSE-NEXT:    psrad $27, %xmm0
+; SSE-NEXT:    psrlw $3, %xmm0
+; SSE-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE-NEXT:    movdqa {{.*#+}} xmm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
+; SSE-NEXT:    pxor %xmm1, %xmm0
+; SSE-NEXT:    psubb %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: splatconstant_shift_v4i8:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpslld $24, %xmm0, %xmm0
-; AVX-NEXT:    vpsrad $27, %xmm0, %xmm0
+; AVX-NEXT:    vpsrlw $3, %xmm0, %xmm0
+; AVX-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
+; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; XOP-LABEL: splatconstant_shift_v4i8:
 ; XOP:       # %bb.0:
-; XOP-NEXT:    vpslld $24, %xmm0, %xmm0
-; XOP-NEXT:    vpsrad $27, %xmm0, %xmm0
+; XOP-NEXT:    vpshab {{.*}}(%rip), %xmm0, %xmm0
 ; XOP-NEXT:    retq
 ;
 ; AVX512-LABEL: splatconstant_shift_v4i8:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpslld $24, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrad $27, %xmm0, %xmm0
+; AVX512-NEXT:    vpsrlw $3, %xmm0, %xmm0
+; AVX512-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
+; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatconstant_shift_v4i8:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpslld $24, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpsrad $27, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpsrlw $3, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
+; AVX512VL-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; X32-SSE-LABEL: splatconstant_shift_v4i8:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pslld $24, %xmm0
-; X32-SSE-NEXT:    psrad $27, %xmm0
+; X32-SSE-NEXT:    psrlw $3, %xmm0
+; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X32-SSE-NEXT:    movdqa {{.*#+}} xmm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
+; X32-SSE-NEXT:    pxor %xmm1, %xmm0
+; X32-SSE-NEXT:    psubb %xmm1, %xmm0
 ; X32-SSE-NEXT:    retl
   %shift = ashr <4 x i8> %a, <i8 3, i8 3, i8 3, i8 3>
   ret <4 x i8> %shift
 }
 
 define <2 x i8> @splatconstant_shift_v2i8(<2 x i8> %a) nounwind {
-; SSE2-LABEL: splatconstant_shift_v2i8:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    psllq $56, %xmm0
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    psrad $31, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
-; SSE2-NEXT:    psrad $24, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    psrad $3, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
-; SSE2-NEXT:    psrlq $3, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; SSE2-NEXT:    retq
-;
-; SSE41-LABEL: splatconstant_shift_v2i8:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    psllq $56, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    psrad $31, %xmm1
-; SSE41-NEXT:    psrad $24, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
-; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    psrad $3, %xmm1
-; SSE41-NEXT:    psrlq $3, %xmm0
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
-; SSE41-NEXT:    retq
-;
-; AVX1-LABEL: splatconstant_shift_v2i8:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpsllq $56, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrad $31, %xmm0, %xmm1
-; AVX1-NEXT:    vpsrad $24, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
-; AVX1-NEXT:    vpsrad $3, %xmm0, %xmm1
-; AVX1-NEXT:    vpsrlq $3, %xmm0, %xmm0
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
-; AVX1-NEXT:    retq
+; SSE-LABEL: splatconstant_shift_v2i8:
+; SSE:       # %bb.0:
+; SSE-NEXT:    psrlw $3, %xmm0
+; SSE-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE-NEXT:    movdqa {{.*#+}} xmm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
+; SSE-NEXT:    pxor %xmm1, %xmm0
+; SSE-NEXT:    psubb %xmm1, %xmm0
+; SSE-NEXT:    retq
 ;
-; AVX2-LABEL: splatconstant_shift_v2i8:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpsllq $56, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrad $31, %xmm0, %xmm1
-; AVX2-NEXT:    vpsrad $24, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
-; AVX2-NEXT:    vpsrad $3, %xmm0, %xmm1
-; AVX2-NEXT:    vpsrlq $3, %xmm0, %xmm0
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
-; AVX2-NEXT:    retq
+; AVX-LABEL: splatconstant_shift_v2i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpsrlw $3, %xmm0, %xmm0
+; AVX-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
+; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
 ;
 ; XOP-LABEL: splatconstant_shift_v2i8:
 ; XOP:       # %bb.0:
-; XOP-NEXT:    vpsllq $56, %xmm0, %xmm0
-; XOP-NEXT:    vpshaq {{.*}}(%rip), %xmm0, %xmm0
-; XOP-NEXT:    vpshaq {{.*}}(%rip), %xmm0, %xmm0
+; XOP-NEXT:    vpshab {{.*}}(%rip), %xmm0, %xmm0
 ; XOP-NEXT:    retq
 ;
 ; AVX512-LABEL: splatconstant_shift_v2i8:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsllq $56, %xmm0, %xmm0
-; AVX512-NEXT:    vpsraq $59, %zmm0, %zmm0
-; AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
-; AVX512-NEXT:    vzeroupper
+; AVX512-NEXT:    vpsrlw $3, %xmm0, %xmm0
+; AVX512-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
+; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatconstant_shift_v2i8:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsllq $56, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpsraq $59, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpsrlw $3, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
+; AVX512VL-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; X32-SSE-LABEL: splatconstant_shift_v2i8:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    psllq $56, %xmm0
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
-; X32-SSE-NEXT:    psrad $31, %xmm1
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
-; X32-SSE-NEXT:    psrad $24, %xmm0
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
-; X32-SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
-; X32-SSE-NEXT:    psrad $3, %xmm1
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
-; X32-SSE-NEXT:    psrlq $3, %xmm0
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; X32-SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; X32-SSE-NEXT:    psrlw $3, %xmm0
+; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X32-SSE-NEXT:    movdqa {{.*#+}} xmm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
+; X32-SSE-NEXT:    pxor %xmm1, %xmm0
+; X32-SSE-NEXT:    psubb %xmm1, %xmm0
 ; X32-SSE-NEXT:    retl
   %shift = ashr <2 x i8> %a, <i8 3, i8 3>
   ret <2 x i8> %shift
diff --git a/llvm/test/CodeGen/X86/vector-shift-by-select-loop.ll b/llvm/test/CodeGen/X86/vector-shift-by-select-loop.ll
index 26c267d0c..b69b35e73 100644
--- a/llvm/test/CodeGen/X86/vector-shift-by-select-loop.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-by-select-loop.ll
@@ -29,91 +29,87 @@ define void @vector_variable_shift_left_loop(i32* nocapture %arr, i8* nocapture
 ; SSE-NEXT:    movd %r9d, %xmm0
 ; SSE-NEXT:    movd %r8d, %xmm1
 ; SSE-NEXT:    xorl %ecx, %ecx
-; SSE-NEXT:    pmovzxdq {{.*#+}} xmm13 = xmm1[0],zero,xmm1[1],zero
-; SSE-NEXT:    pmovzxdq {{.*#+}} xmm14 = xmm0[0],zero,xmm0[1],zero
+; SSE-NEXT:    pmovzxdq {{.*#+}} xmm14 = xmm1[0],zero,xmm1[1],zero
+; SSE-NEXT:    pmovzxdq {{.*#+}} xmm15 = xmm0[0],zero,xmm0[1],zero
 ; SSE-NEXT:    .p2align 4, 0x90
 ; SSE-NEXT:  .LBB0_4: # %vector.body
 ; SSE-NEXT:    # =>This Inner Loop Header: Depth=1
-; SSE-NEXT:    pmovzxbw {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
-; SSE-NEXT:    pmovzxbw {{.*#+}} xmm3 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
-; SSE-NEXT:    pmovzxbw {{.*#+}} xmm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
-; SSE-NEXT:    pmovzxbw {{.*#+}} xmm15 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
+; SSE-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; SSE-NEXT:    movq {{.*#+}} xmm3 = mem[0],zero
+; SSE-NEXT:    movq {{.*#+}} xmm4 = mem[0],zero
+; SSE-NEXT:    movq {{.*#+}} xmm5 = mem[0],zero
 ; SSE-NEXT:    pxor %xmm1, %xmm1
-; SSE-NEXT:    pcmpeqw %xmm1, %xmm0
-; SSE-NEXT:    pmovzxwd {{.*#+}} xmm12 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; SSE-NEXT:    pslld $24, %xmm12
-; SSE-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]
-; SSE-NEXT:    pslld $24, %xmm0
-; SSE-NEXT:    pcmpeqw %xmm1, %xmm3
-; SSE-NEXT:    pmovzxwd {{.*#+}} xmm11 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero
-; SSE-NEXT:    pslld $24, %xmm11
-; SSE-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
-; SSE-NEXT:    pslld $24, %xmm3
-; SSE-NEXT:    pcmpeqw %xmm1, %xmm2
-; SSE-NEXT:    pmovzxwd {{.*#+}} xmm9 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
-; SSE-NEXT:    pslld $24, %xmm9
-; SSE-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
-; SSE-NEXT:    pslld $24, %xmm2
-; SSE-NEXT:    pcmpeqw %xmm1, %xmm15
-; SSE-NEXT:    pmovzxwd {{.*#+}} xmm8 = xmm15[0],zero,xmm15[1],zero,xmm15[2],zero,xmm15[3],zero
-; SSE-NEXT:    pslld $24, %xmm8
-; SSE-NEXT:    punpckhwd {{.*#+}} xmm15 = xmm15[4],xmm0[4],xmm15[5],xmm0[5],xmm15[6],xmm0[6],xmm15[7],xmm0[7]
-; SSE-NEXT:    pslld $24, %xmm15
-; SSE-NEXT:    movdqu 16(%rdi,%rcx,4), %xmm6
-; SSE-NEXT:    movdqa %xmm6, %xmm4
-; SSE-NEXT:    pslld %xmm14, %xmm4
-; SSE-NEXT:    pslld %xmm13, %xmm6
-; SSE-NEXT:    blendvps %xmm0, %xmm4, %xmm6
+; SSE-NEXT:    pcmpeqb %xmm1, %xmm0
+; SSE-NEXT:    pmovsxbd %xmm0, %xmm7
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE-NEXT:    pmovsxbd %xmm0, %xmm0
+; SSE-NEXT:    pcmpeqb %xmm1, %xmm3
+; SSE-NEXT:    pmovsxbd %xmm3, %xmm13
+; SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,2,3]
+; SSE-NEXT:    pmovsxbd %xmm3, %xmm6
+; SSE-NEXT:    pcmpeqb %xmm1, %xmm4
+; SSE-NEXT:    pmovsxbd %xmm4, %xmm11
+; SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,2,3]
+; SSE-NEXT:    pmovsxbd %xmm3, %xmm2
+; SSE-NEXT:    pcmpeqb %xmm1, %xmm5
+; SSE-NEXT:    pmovsxbd %xmm5, %xmm8
+; SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm5[1,1,2,3]
+; SSE-NEXT:    pmovsxbd %xmm3, %xmm9
+; SSE-NEXT:    movdqu 16(%rdi,%rcx,4), %xmm3
+; SSE-NEXT:    movdqa %xmm3, %xmm4
+; SSE-NEXT:    pslld %xmm15, %xmm4
+; SSE-NEXT:    pslld %xmm14, %xmm3
+; SSE-NEXT:    blendvps %xmm0, %xmm4, %xmm3
 ; SSE-NEXT:    movdqu (%rdi,%rcx,4), %xmm10
-; SSE-NEXT:    movdqa %xmm10, %xmm4
-; SSE-NEXT:    pslld %xmm14, %xmm4
-; SSE-NEXT:    pslld %xmm13, %xmm10
-; SSE-NEXT:    movdqa %xmm12, %xmm0
-; SSE-NEXT:    blendvps %xmm0, %xmm4, %xmm10
+; SSE-NEXT:    movdqa %xmm10, %xmm5
+; SSE-NEXT:    pslld %xmm15, %xmm5
+; SSE-NEXT:    pslld %xmm14, %xmm10
+; SSE-NEXT:    movdqa %xmm7, %xmm0
+; SSE-NEXT:    blendvps %xmm0, %xmm5, %xmm10
 ; SSE-NEXT:    movdqu 48(%rdi,%rcx,4), %xmm12
 ; SSE-NEXT:    movdqa %xmm12, %xmm5
-; SSE-NEXT:    pslld %xmm14, %xmm5
-; SSE-NEXT:    pslld %xmm13, %xmm12
-; SSE-NEXT:    movdqa %xmm3, %xmm0
+; SSE-NEXT:    pslld %xmm15, %xmm5
+; SSE-NEXT:    pslld %xmm14, %xmm12
+; SSE-NEXT:    movdqa %xmm6, %xmm0
 ; SSE-NEXT:    blendvps %xmm0, %xmm5, %xmm12
-; SSE-NEXT:    movdqu 32(%rdi,%rcx,4), %xmm3
-; SSE-NEXT:    movdqa %xmm3, %xmm5
-; SSE-NEXT:    pslld %xmm14, %xmm5
-; SSE-NEXT:    pslld %xmm13, %xmm3
-; SSE-NEXT:    movdqa %xmm11, %xmm0
-; SSE-NEXT:    blendvps %xmm0, %xmm5, %xmm3
+; SSE-NEXT:    movdqu 32(%rdi,%rcx,4), %xmm6
+; SSE-NEXT:    movdqa %xmm6, %xmm5
+; SSE-NEXT:    pslld %xmm15, %xmm5
+; SSE-NEXT:    pslld %xmm14, %xmm6
+; SSE-NEXT:    movdqa %xmm13, %xmm0
+; SSE-NEXT:    blendvps %xmm0, %xmm5, %xmm6
 ; SSE-NEXT:    movdqu 80(%rdi,%rcx,4), %xmm1
 ; SSE-NEXT:    movdqa %xmm1, %xmm5
-; SSE-NEXT:    pslld %xmm14, %xmm5
-; SSE-NEXT:    pslld %xmm13, %xmm1
+; SSE-NEXT:    pslld %xmm15, %xmm5
+; SSE-NEXT:    pslld %xmm14, %xmm1
 ; SSE-NEXT:    movdqa %xmm2, %xmm0
 ; SSE-NEXT:    blendvps %xmm0, %xmm5, %xmm1
-; SSE-NEXT:    movdqu 64(%rdi,%rcx,4), %xmm2
-; SSE-NEXT:    movdqa %xmm2, %xmm5
+; SSE-NEXT:    movdqu 64(%rdi,%rcx,4), %xmm5
+; SSE-NEXT:    movdqa %xmm5, %xmm2
+; SSE-NEXT:    pslld %xmm15, %xmm2
 ; SSE-NEXT:    pslld %xmm14, %xmm5
-; SSE-NEXT:    pslld %xmm13, %xmm2
+; SSE-NEXT:    movdqa %xmm11, %xmm0
+; SSE-NEXT:    blendvps %xmm0, %xmm2, %xmm5
+; SSE-NEXT:    movdqu 112(%rdi,%rcx,4), %xmm2
+; SSE-NEXT:    movdqa %xmm2, %xmm4
+; SSE-NEXT:    pslld %xmm15, %xmm4
+; SSE-NEXT:    pslld %xmm14, %xmm2
 ; SSE-NEXT:    movdqa %xmm9, %xmm0
-; SSE-NEXT:    blendvps %xmm0, %xmm5, %xmm2
-; SSE-NEXT:    movdqu 112(%rdi,%rcx,4), %xmm5
-; SSE-NEXT:    movdqa %xmm5, %xmm7
-; SSE-NEXT:    pslld %xmm14, %xmm7
-; SSE-NEXT:    pslld %xmm13, %xmm5
-; SSE-NEXT:    movdqa %xmm15, %xmm0
-; SSE-NEXT:    blendvps %xmm0, %xmm7, %xmm5
-; SSE-NEXT:    movdqu 96(%rdi,%rcx,4), %xmm7
-; SSE-NEXT:    movdqa %xmm7, %xmm4
+; SSE-NEXT:    blendvps %xmm0, %xmm4, %xmm2
+; SSE-NEXT:    movdqu 96(%rdi,%rcx,4), %xmm4
+; SSE-NEXT:    movdqa %xmm4, %xmm7
+; SSE-NEXT:    pslld %xmm15, %xmm7
 ; SSE-NEXT:    pslld %xmm14, %xmm4
-; SSE-NEXT:    pslld %xmm13, %xmm7
 ; SSE-NEXT:    movdqa %xmm8, %xmm0
-; SSE-NEXT:    blendvps %xmm0, %xmm4, %xmm7
+; SSE-NEXT:    blendvps %xmm0, %xmm7, %xmm4
 ; SSE-NEXT:    movups %xmm10, (%rdi,%rcx,4)
-; SSE-NEXT:    movups %xmm6, 16(%rdi,%rcx,4)
-; SSE-NEXT:    movups %xmm3, 32(%rdi,%rcx,4)
+; SSE-NEXT:    movups %xmm3, 16(%rdi,%rcx,4)
+; SSE-NEXT:    movups %xmm6, 32(%rdi,%rcx,4)
 ; SSE-NEXT:    movups %xmm12, 48(%rdi,%rcx,4)
-; SSE-NEXT:    movups %xmm2, 64(%rdi,%rcx,4)
+; SSE-NEXT:    movups %xmm5, 64(%rdi,%rcx,4)
 ; SSE-NEXT:    movups %xmm1, 80(%rdi,%rcx,4)
-; SSE-NEXT:    movups %xmm7, 96(%rdi,%rcx,4)
-; SSE-NEXT:    movups %xmm5, 112(%rdi,%rcx,4)
+; SSE-NEXT:    movups %xmm4, 96(%rdi,%rcx,4)
+; SSE-NEXT:    movups %xmm2, 112(%rdi,%rcx,4)
 ; SSE-NEXT:    addq $32, %rcx
 ; SSE-NEXT:    cmpq %rcx, %rdx
 ; SSE-NEXT:    jne .LBB0_4
@@ -179,33 +175,33 @@ define void @vector_variable_shift_left_loop(i32* nocapture %arr, i8* nocapture
 ; AVX1-NEXT:    # xmm1 = mem[0],zero,mem[1],zero
 ; AVX1-NEXT:    vpmovzxdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
 ; AVX1-NEXT:    # xmm2 = mem[0],zero,mem[1],zero
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm3 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm4 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm5 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm6 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
-; AVX1-NEXT:    vpcmpeqw %xmm11, %xmm3, %xmm3
-; AVX1-NEXT:    vpmovsxwd %xmm3, %xmm7
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,3,0,1]
-; AVX1-NEXT:    vpmovsxwd %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpeqw %xmm11, %xmm4, %xmm4
-; AVX1-NEXT:    vpmovsxwd %xmm4, %xmm8
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[2,3,0,1]
-; AVX1-NEXT:    vpmovsxwd %xmm4, %xmm4
-; AVX1-NEXT:    vpcmpeqw %xmm11, %xmm5, %xmm5
+; AVX1-NEXT:    vmovq {{.*#+}} xmm3 = mem[0],zero
+; AVX1-NEXT:    vmovq {{.*#+}} xmm4 = mem[0],zero
+; AVX1-NEXT:    vmovq {{.*#+}} xmm5 = mem[0],zero
+; AVX1-NEXT:    vmovq {{.*#+}} xmm6 = mem[0],zero
+; AVX1-NEXT:    vpcmpeqb %xmm3, %xmm11, %xmm3
+; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm7
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,2,3]
+; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpeqb %xmm4, %xmm11, %xmm4
+; AVX1-NEXT:    vpmovsxbd %xmm4, %xmm8
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[1,1,2,3]
+; AVX1-NEXT:    vpmovsxbd %xmm4, %xmm4
+; AVX1-NEXT:    vpcmpeqb %xmm5, %xmm11, %xmm5
 ; AVX1-NEXT:    vmovdqu (%rdi,%rcx,4), %xmm9
 ; AVX1-NEXT:    vpslld %xmm2, %xmm9, %xmm10
 ; AVX1-NEXT:    vpslld %xmm1, %xmm9, %xmm0
 ; AVX1-NEXT:    vblendvps %xmm7, %xmm10, %xmm0, %xmm9
-; AVX1-NEXT:    vpmovsxwd %xmm5, %xmm7
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[2,3,0,1]
-; AVX1-NEXT:    vpmovsxwd %xmm5, %xmm5
-; AVX1-NEXT:    vpcmpeqw %xmm11, %xmm6, %xmm6
+; AVX1-NEXT:    vpmovsxbd %xmm5, %xmm7
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[1,1,2,3]
+; AVX1-NEXT:    vpmovsxbd %xmm5, %xmm5
+; AVX1-NEXT:    vpcmpeqb %xmm6, %xmm11, %xmm6
 ; AVX1-NEXT:    vmovdqu 16(%rdi,%rcx,4), %xmm0
 ; AVX1-NEXT:    vpslld %xmm2, %xmm0, %xmm2
 ; AVX1-NEXT:    vpslld %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpmovsxwd %xmm6, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[2,3,0,1]
-; AVX1-NEXT:    vpmovsxwd %xmm6, %xmm6
+; AVX1-NEXT:    vpmovsxbd %xmm6, %xmm1
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[1,1,2,3]
+; AVX1-NEXT:    vpmovsxbd %xmm6, %xmm6
 ; AVX1-NEXT:    vblendvps %xmm3, %xmm2, %xmm0, %xmm10
 ; AVX1-NEXT:    vmovdqu 32(%rdi,%rcx,4), %xmm2
 ; AVX1-NEXT:    vpslld %xmm15, %xmm2, %xmm3
diff --git a/llvm/test/CodeGen/X86/vector-shift-lshr-128.ll b/llvm/test/CodeGen/X86/vector-shift-lshr-128.ll
index 21a011aba..a587a43f1 100644
--- a/llvm/test/CodeGen/X86/vector-shift-lshr-128.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-lshr-128.ll
@@ -279,9 +279,8 @@ define <8 x i16> @var_shift_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind {
 ; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
 ; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; AVX2-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/vector-shift-lshr-256.ll b/llvm/test/CodeGen/X86/vector-shift-lshr-256.ll
index 698a45fad..24395c916 100644
--- a/llvm/test/CodeGen/X86/vector-shift-lshr-256.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-lshr-256.ll
@@ -1394,3 +1394,86 @@ define <32 x i8> @splatconstant_shift_v32i8(<32 x i8> %a) nounwind {
   %shift = lshr <32 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
   ret <32 x i8> %shift
 }
+
+define <4 x i32> @sh_trunc_sh_vec(<4 x i64> %x) {
+; AVX1-LABEL: sh_trunc_sh_vec:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vpsrlq $36, %xmm1, %xmm1
+; AVX1-NEXT:    vpsrlq $36, %xmm0, %xmm0
+; AVX1-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
+; AVX1-NEXT:    vandps {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: sh_trunc_sh_vec:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsrlq $36, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
+; AVX2-NEXT:    vbroadcastss {{.*#+}} xmm1 = [1048575,1048575,1048575,1048575]
+; AVX2-NEXT:    vandps %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+;
+; XOPAVX1-LABEL: sh_trunc_sh_vec:
+; XOPAVX1:       # %bb.0:
+; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; XOPAVX1-NEXT:    vpsrlq $36, %xmm1, %xmm1
+; XOPAVX1-NEXT:    vpsrlq $36, %xmm0, %xmm0
+; XOPAVX1-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
+; XOPAVX1-NEXT:    vandps {{.*}}(%rip), %xmm0, %xmm0
+; XOPAVX1-NEXT:    vzeroupper
+; XOPAVX1-NEXT:    retq
+;
+; XOPAVX2-LABEL: sh_trunc_sh_vec:
+; XOPAVX2:       # %bb.0:
+; XOPAVX2-NEXT:    vpsrlq $36, %ymm0, %ymm0
+; XOPAVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; XOPAVX2-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
+; XOPAVX2-NEXT:    vbroadcastss {{.*#+}} xmm1 = [1048575,1048575,1048575,1048575]
+; XOPAVX2-NEXT:    vandps %xmm1, %xmm0, %xmm0
+; XOPAVX2-NEXT:    vzeroupper
+; XOPAVX2-NEXT:    retq
+;
+; AVX512-LABEL: sh_trunc_sh_vec:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpsrlq $36, %ymm0, %ymm0
+; AVX512-NEXT:    vpmovqd %zmm0, %ymm0
+; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [1048575,1048575,1048575,1048575]
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vzeroupper
+; AVX512-NEXT:    retq
+;
+; AVX512VL-LABEL: sh_trunc_sh_vec:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsrlq $36, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpmovqd %ymm0, %xmm0
+; AVX512VL-NEXT:    vpandd {{.*}}(%rip){1to4}, %xmm0, %xmm0
+; AVX512VL-NEXT:    vzeroupper
+; AVX512VL-NEXT:    retq
+;
+; X32-AVX1-LABEL: sh_trunc_sh_vec:
+; X32-AVX1:       # %bb.0:
+; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; X32-AVX1-NEXT:    vpsrlq $36, %xmm1, %xmm1
+; X32-AVX1-NEXT:    vpsrlq $36, %xmm0, %xmm0
+; X32-AVX1-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
+; X32-AVX1-NEXT:    vandps {{\.LCPI.*}}, %xmm0, %xmm0
+; X32-AVX1-NEXT:    vzeroupper
+; X32-AVX1-NEXT:    retl
+;
+; X32-AVX2-LABEL: sh_trunc_sh_vec:
+; X32-AVX2:       # %bb.0:
+; X32-AVX2-NEXT:    vpsrlq $36, %ymm0, %ymm0
+; X32-AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; X32-AVX2-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
+; X32-AVX2-NEXT:    vbroadcastss {{.*#+}} xmm1 = [1048575,1048575,1048575,1048575]
+; X32-AVX2-NEXT:    vandps %xmm1, %xmm0, %xmm0
+; X32-AVX2-NEXT:    vzeroupper
+; X32-AVX2-NEXT:    retl
+  %s = lshr <4 x i64> %x, <i64 24, i64 24, i64 24, i64 24>
+  %t = trunc <4 x i64> %s to <4 x i32>
+  %r = lshr <4 x i32> %t, <i32 12, i32 12, i32 12, i32 12>
+  ret <4 x i32> %r
+}
diff --git a/llvm/test/CodeGen/X86/vector-shift-lshr-512.ll b/llvm/test/CodeGen/X86/vector-shift-lshr-512.ll
index f32b56d60..81ab84315 100644
--- a/llvm/test/CodeGen/X86/vector-shift-lshr-512.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-lshr-512.ll
@@ -27,14 +27,17 @@ define <16 x i32> @var_shift_v16i32(<16 x i32> %a, <16 x i32> %b) nounwind {
 define <32 x i16> @var_shift_v32i16(<32 x i16> %a, <32 x i16> %b) nounwind {
 ; AVX512DQ-LABEL: var_shift_v32i16:
 ; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
+; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} zmm3 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
 ; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
+; AVX512DQ-NEXT:    vpsrlvd %zmm3, %zmm2, %zmm2
+; AVX512DQ-NEXT:    vpmovdw %zmm2, %ymm2
+; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
 ; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
-; AVX512DQ-NEXT:    vpsrlvd %zmm2, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpsrlvd %zmm1, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    vpmovdw %zmm0, %ymm0
-; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
-; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
-; AVX512DQ-NEXT:    vpsrlvd %zmm2, %zmm1, %zmm1
-; AVX512DQ-NEXT:    vpmovdw %zmm1, %ymm1
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512BW-LABEL: var_shift_v32i16:
@@ -48,33 +51,36 @@ define <32 x i16> @var_shift_v32i16(<32 x i16> %a, <32 x i16> %b) nounwind {
 define <64 x i8> @var_shift_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
 ; AVX512DQ-LABEL: var_shift_v64i8:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vpsrlw $4, %ymm0, %ymm4
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
+; AVX512DQ-NEXT:    vpsrlw $4, %ymm3, %ymm4
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm5 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
 ; AVX512DQ-NEXT:    vpand %ymm5, %ymm4, %ymm4
 ; AVX512DQ-NEXT:    vpsllw $5, %ymm2, %ymm2
-; AVX512DQ-NEXT:    vpblendvb %ymm2, %ymm4, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpsrlw $2, %ymm0, %ymm4
+; AVX512DQ-NEXT:    vpblendvb %ymm2, %ymm4, %ymm3, %ymm3
+; AVX512DQ-NEXT:    vpsrlw $2, %ymm3, %ymm4
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm6 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
 ; AVX512DQ-NEXT:    vpand %ymm6, %ymm4, %ymm4
 ; AVX512DQ-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
-; AVX512DQ-NEXT:    vpblendvb %ymm2, %ymm4, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpsrlw $1, %ymm0, %ymm4
+; AVX512DQ-NEXT:    vpblendvb %ymm2, %ymm4, %ymm3, %ymm3
+; AVX512DQ-NEXT:    vpsrlw $1, %ymm3, %ymm4
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm7 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
 ; AVX512DQ-NEXT:    vpand %ymm7, %ymm4, %ymm4
 ; AVX512DQ-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
-; AVX512DQ-NEXT:    vpblendvb %ymm2, %ymm4, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpsrlw $4, %ymm1, %ymm2
-; AVX512DQ-NEXT:    vpand %ymm5, %ymm2, %ymm2
-; AVX512DQ-NEXT:    vpsllw $5, %ymm3, %ymm3
-; AVX512DQ-NEXT:    vpblendvb %ymm3, %ymm2, %ymm1, %ymm1
-; AVX512DQ-NEXT:    vpsrlw $2, %ymm1, %ymm2
-; AVX512DQ-NEXT:    vpand %ymm6, %ymm2, %ymm2
-; AVX512DQ-NEXT:    vpaddb %ymm3, %ymm3, %ymm3
-; AVX512DQ-NEXT:    vpblendvb %ymm3, %ymm2, %ymm1, %ymm1
-; AVX512DQ-NEXT:    vpsrlw $1, %ymm1, %ymm2
-; AVX512DQ-NEXT:    vpand %ymm7, %ymm2, %ymm2
-; AVX512DQ-NEXT:    vpaddb %ymm3, %ymm3, %ymm3
-; AVX512DQ-NEXT:    vpblendvb %ymm3, %ymm2, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpblendvb %ymm2, %ymm4, %ymm3, %ymm2
+; AVX512DQ-NEXT:    vpsrlw $4, %ymm0, %ymm3
+; AVX512DQ-NEXT:    vpand %ymm5, %ymm3, %ymm3
+; AVX512DQ-NEXT:    vpsllw $5, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpsrlw $2, %ymm0, %ymm3
+; AVX512DQ-NEXT:    vpand %ymm6, %ymm3, %ymm3
+; AVX512DQ-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpsrlw $1, %ymm0, %ymm3
+; AVX512DQ-NEXT:    vpand %ymm7, %ymm3, %ymm3
+; AVX512DQ-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512BW-LABEL: var_shift_v64i8:
@@ -127,9 +133,11 @@ define <16 x i32> @splatvar_shift_v16i32(<16 x i32> %a, <16 x i32> %b) nounwind
 define <32 x i16> @splatvar_shift_v32i16(<32 x i16> %a, <32 x i16> %b) nounwind {
 ; AVX512DQ-LABEL: splatvar_shift_v32i16:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
-; AVX512DQ-NEXT:    vpsrlw %xmm2, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpsrlw %xmm2, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512DQ-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512DQ-NEXT:    vpsrlw %xmm1, %ymm2, %ymm2
+; AVX512DQ-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatvar_shift_v32i16:
@@ -145,15 +153,17 @@ define <32 x i16> @splatvar_shift_v32i16(<32 x i16> %a, <32 x i16> %b) nounwind
 define <64 x i8> @splatvar_shift_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
 ; AVX512DQ-LABEL: splatvar_shift_v64i8:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vpmovzxbq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-NEXT:    vpsrlw %xmm2, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512DQ-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512DQ-NEXT:    vpsrlw %xmm1, %ymm2, %ymm2
 ; AVX512DQ-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX512DQ-NEXT:    vpsrlw %xmm2, %xmm3, %xmm3
+; AVX512DQ-NEXT:    vpsrlw %xmm1, %xmm3, %xmm3
 ; AVX512DQ-NEXT:    vpsrlw $8, %xmm3, %xmm3
 ; AVX512DQ-NEXT:    vpbroadcastb %xmm3, %ymm3
+; AVX512DQ-NEXT:    vpand %ymm3, %ymm2, %ymm2
+; AVX512DQ-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpand %ymm3, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpsrlw %xmm2, %ymm1, %ymm1
-; AVX512DQ-NEXT:    vpand %ymm3, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatvar_shift_v64i8:
@@ -196,13 +206,15 @@ define <16 x i32> @constant_shift_v16i32(<16 x i32> %a) nounwind {
 define <32 x i16> @constant_shift_v32i16(<32 x i16> %a) nounwind {
 ; AVX512DQ-LABEL: constant_shift_v32i16:
 ; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm2 = <u,32768,16384,8192,4096,2048,1024,512,256,128,64,32,16,8,4,2>
-; AVX512DQ-NEXT:    vpmulhuw %ymm2, %ymm0, %ymm3
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm3[1,2,3,4,5,6,7],ymm0[8],ymm3[9,10,11,12,13,14,15]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm3[4,5,6,7]
-; AVX512DQ-NEXT:    vpmulhuw %ymm2, %ymm1, %ymm2
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm2[1,2,3,4,5,6,7],ymm1[8],ymm2[9,10,11,12,13,14,15]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
+; AVX512DQ-NEXT:    vpmulhuw %ymm2, %ymm1, %ymm3
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm3[1,2,3,4,5,6,7],ymm1[8],ymm3[9,10,11,12,13,14,15]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm3[4,5,6,7]
+; AVX512DQ-NEXT:    vpmulhuw %ymm2, %ymm0, %ymm2
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm2[1,2,3,4,5,6,7],ymm0[8],ymm2[9,10,11,12,13,14,15]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512BW-LABEL: constant_shift_v32i16:
@@ -216,25 +228,27 @@ define <32 x i16> @constant_shift_v32i16(<32 x i16> %a) nounwind {
 define <64 x i8> @constant_shift_v64i8(<64 x i8> %a) nounwind {
 ; AVX512DQ-LABEL: constant_shift_v64i8:
 ; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512DQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm0[8],ymm2[8],ymm0[9],ymm2[9],ymm0[10],ymm2[10],ymm0[11],ymm2[11],ymm0[12],ymm2[12],ymm0[13],ymm2[13],ymm0[14],ymm2[14],ymm0[15],ymm2[15],ymm0[24],ymm2[24],ymm0[25],ymm2[25],ymm0[26],ymm2[26],ymm0[27],ymm2[27],ymm0[28],ymm2[28],ymm0[29],ymm2[29],ymm0[30],ymm2[30],ymm0[31],ymm2[31]
+; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15],ymm1[24],ymm2[24],ymm1[25],ymm2[25],ymm1[26],ymm2[26],ymm1[27],ymm2[27],ymm1[28],ymm2[28],ymm1[29],ymm2[29],ymm1[30],ymm2[30],ymm1[31],ymm2[31]
 ; AVX512DQ-NEXT:    vbroadcasti128 {{.*#+}} ymm4 = [2,4,8,16,32,64,128,256,2,4,8,16,32,64,128,256]
 ; AVX512DQ-NEXT:    # ymm4 = mem[0,1,0,1]
 ; AVX512DQ-NEXT:    vpmullw %ymm4, %ymm3, %ymm3
 ; AVX512DQ-NEXT:    vpsrlw $8, %ymm3, %ymm3
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0],ymm2[0],ymm0[1],ymm2[1],ymm0[2],ymm2[2],ymm0[3],ymm2[3],ymm0[4],ymm2[4],ymm0[5],ymm2[5],ymm0[6],ymm2[6],ymm0[7],ymm2[7],ymm0[16],ymm2[16],ymm0[17],ymm2[17],ymm0[18],ymm2[18],ymm0[19],ymm2[19],ymm0[20],ymm2[20],ymm0[21],ymm2[21],ymm0[22],ymm2[22],ymm0[23],ymm2[23]
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[16],ymm2[16],ymm1[17],ymm2[17],ymm1[18],ymm2[18],ymm1[19],ymm2[19],ymm1[20],ymm2[20],ymm1[21],ymm2[21],ymm1[22],ymm2[22],ymm1[23],ymm2[23]
 ; AVX512DQ-NEXT:    vbroadcasti128 {{.*#+}} ymm5 = [256,128,64,32,16,8,4,2,256,128,64,32,16,8,4,2]
 ; AVX512DQ-NEXT:    # ymm5 = mem[0,1,0,1]
-; AVX512DQ-NEXT:    vpmullw %ymm5, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpsrlw $8, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpackuswb %ymm3, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15],ymm1[24],ymm2[24],ymm1[25],ymm2[25],ymm1[26],ymm2[26],ymm1[27],ymm2[27],ymm1[28],ymm2[28],ymm1[29],ymm2[29],ymm1[30],ymm2[30],ymm1[31],ymm2[31]
-; AVX512DQ-NEXT:    vpmullw %ymm4, %ymm3, %ymm3
-; AVX512DQ-NEXT:    vpsrlw $8, %ymm3, %ymm3
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[16],ymm2[16],ymm1[17],ymm2[17],ymm1[18],ymm2[18],ymm1[19],ymm2[19],ymm1[20],ymm2[20],ymm1[21],ymm2[21],ymm1[22],ymm2[22],ymm1[23],ymm2[23]
 ; AVX512DQ-NEXT:    vpmullw %ymm5, %ymm1, %ymm1
 ; AVX512DQ-NEXT:    vpsrlw $8, %ymm1, %ymm1
 ; AVX512DQ-NEXT:    vpackuswb %ymm3, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm0[8],ymm2[8],ymm0[9],ymm2[9],ymm0[10],ymm2[10],ymm0[11],ymm2[11],ymm0[12],ymm2[12],ymm0[13],ymm2[13],ymm0[14],ymm2[14],ymm0[15],ymm2[15],ymm0[24],ymm2[24],ymm0[25],ymm2[25],ymm0[26],ymm2[26],ymm0[27],ymm2[27],ymm0[28],ymm2[28],ymm0[29],ymm2[29],ymm0[30],ymm2[30],ymm0[31],ymm2[31]
+; AVX512DQ-NEXT:    vpmullw %ymm4, %ymm3, %ymm3
+; AVX512DQ-NEXT:    vpsrlw $8, %ymm3, %ymm3
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0],ymm2[0],ymm0[1],ymm2[1],ymm0[2],ymm2[2],ymm0[3],ymm2[3],ymm0[4],ymm2[4],ymm0[5],ymm2[5],ymm0[6],ymm2[6],ymm0[7],ymm2[7],ymm0[16],ymm2[16],ymm0[17],ymm2[17],ymm0[18],ymm2[18],ymm0[19],ymm2[19],ymm0[20],ymm2[20],ymm0[21],ymm2[21],ymm0[22],ymm2[22],ymm0[23],ymm2[23]
+; AVX512DQ-NEXT:    vpmullw %ymm5, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpsrlw $8, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpackuswb %ymm3, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512BW-LABEL: constant_shift_v64i8:
@@ -278,8 +292,10 @@ define <16 x i32> @splatconstant_shift_v16i32(<16 x i32> %a) nounwind {
 define <32 x i16> @splatconstant_shift_v32i16(<32 x i16> %a) nounwind {
 ; AVX512DQ-LABEL: splatconstant_shift_v32i16:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vpsrlw $3, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512DQ-NEXT:    vpsrlw $3, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpsrlw $3, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatconstant_shift_v32i16:
@@ -293,11 +309,13 @@ define <32 x i16> @splatconstant_shift_v32i16(<32 x i16> %a) nounwind {
 define <64 x i8> @splatconstant_shift_v64i8(<64 x i8> %a) nounwind {
 ; AVX512DQ-LABEL: splatconstant_shift_v64i8:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vpsrlw $3, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm2 = [31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31]
-; AVX512DQ-NEXT:    vpand %ymm2, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512DQ-NEXT:    vpsrlw $3, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm2 = [31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31]
 ; AVX512DQ-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpsrlw $3, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpand %ymm2, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatconstant_shift_v64i8:
diff --git a/llvm/test/CodeGen/X86/vector-shift-lshr-sub128.ll b/llvm/test/CodeGen/X86/vector-shift-lshr-sub128.ll
index 546f723e6..b540421fb 100644
--- a/llvm/test/CodeGen/X86/vector-shift-lshr-sub128.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-lshr-sub128.ll
@@ -20,103 +20,6 @@
 define <2 x i32> @var_shift_v2i32(<2 x i32> %a, <2 x i32> %b) nounwind {
 ; SSE2-LABEL: var_shift_v2i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [4294967295,0,4294967295,0]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    psrlq %xmm1, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
-; SSE2-NEXT:    psrlq %xmm1, %xmm0
-; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm2[0],xmm0[1]
-; SSE2-NEXT:    retq
-;
-; SSE41-LABEL: var_shift_v2i32:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pxor %xmm2, %xmm2
-; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    psrlq %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
-; SSE41-NEXT:    psrlq %xmm1, %xmm0
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm2[0,1,2,3],xmm0[4,5,6,7]
-; SSE41-NEXT:    retq
-;
-; AVX1-LABEL: var_shift_v2i32:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
-; AVX1-NEXT:    vpsrlq %xmm1, %xmm0, %xmm2
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
-; AVX1-NEXT:    vpsrlq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm2[0,1,2,3],xmm0[4,5,6,7]
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: var_shift_v2i32:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
-; AVX2-NEXT:    vpsrlvq %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    retq
-;
-; XOPAVX1-LABEL: var_shift_v2i32:
-; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; XOPAVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
-; XOPAVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
-; XOPAVX1-NEXT:    vpsubq %xmm1, %xmm2, %xmm1
-; XOPAVX1-NEXT:    vpshlq %xmm1, %xmm0, %xmm0
-; XOPAVX1-NEXT:    retq
-;
-; XOPAVX2-LABEL: var_shift_v2i32:
-; XOPAVX2:       # %bb.0:
-; XOPAVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; XOPAVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; XOPAVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
-; XOPAVX2-NEXT:    vpsrlvq %xmm1, %xmm0, %xmm0
-; XOPAVX2-NEXT:    retq
-;
-; AVX512-LABEL: var_shift_v2i32:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; AVX512-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
-; AVX512-NEXT:    vpsrlvq %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    retq
-;
-; AVX512VL-LABEL: var_shift_v2i32:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VL-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; AVX512VL-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
-; AVX512VL-NEXT:    vpsrlvq %xmm1, %xmm0, %xmm0
-; AVX512VL-NEXT:    retq
-;
-; X32-SSE-LABEL: var_shift_v2i32:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [4294967295,0,4294967295,0]
-; X32-SSE-NEXT:    pand %xmm2, %xmm1
-; X32-SSE-NEXT:    pand %xmm2, %xmm0
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm2
-; X32-SSE-NEXT:    psrlq %xmm1, %xmm2
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
-; X32-SSE-NEXT:    xorps %xmm3, %xmm3
-; X32-SSE-NEXT:    movss {{.*#+}} xmm3 = xmm1[0],xmm3[1,2,3]
-; X32-SSE-NEXT:    psrlq %xmm3, %xmm0
-; X32-SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm2[0],xmm0[1]
-; X32-SSE-NEXT:    retl
-  %shift = lshr <2 x i32> %a, %b
-  ret <2 x i32> %shift
-}
-
-define <4 x i16> @var_shift_v4i16(<4 x i16> %a, <4 x i16> %b) nounwind {
-; SSE2-LABEL: var_shift_v4i16:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [65535,0,65535,0,65535,0,65535,0]
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    pand %xmm2, %xmm1
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm1[2,3,3,3,4,5,6,7]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm3
 ; SSE2-NEXT:    psrld %xmm2, %xmm3
@@ -135,93 +38,70 @@ define <4 x i16> @var_shift_v4i16(<4 x i16> %a, <4 x i16> %b) nounwind {
 ; SSE2-NEXT:    movaps %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: var_shift_v4i16:
+; SSE41-LABEL: var_shift_v2i32:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pxor %xmm2, %xmm2
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3],xmm0[4],xmm2[5],xmm0[6],xmm2[7]
-; SSE41-NEXT:    movdqa %xmm1, %xmm3
-; SSE41-NEXT:    pblendw {{.*#+}} xmm3 = xmm3[0],xmm2[1],xmm3[2],xmm2[3],xmm3[4],xmm2[5],xmm3[6],xmm2[7]
-; SSE41-NEXT:    pshuflw {{.*#+}} xmm4 = xmm3[2,3,3,3,4,5,6,7]
+; SSE41-NEXT:    pshuflw {{.*#+}} xmm2 = xmm1[2,3,3,3,4,5,6,7]
+; SSE41-NEXT:    movdqa %xmm0, %xmm3
+; SSE41-NEXT:    psrld %xmm2, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; SSE41-NEXT:    pshuflw {{.*#+}} xmm4 = xmm2[2,3,3,3,4,5,6,7]
 ; SSE41-NEXT:    movdqa %xmm0, %xmm5
 ; SSE41-NEXT:    psrld %xmm4, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[2,3,0,1]
-; SSE41-NEXT:    pshuflw {{.*#+}} xmm4 = xmm3[2,3,3,3,4,5,6,7]
-; SSE41-NEXT:    movdqa %xmm0, %xmm6
-; SSE41-NEXT:    psrld %xmm4, %xmm6
-; SSE41-NEXT:    pblendw {{.*#+}} xmm6 = xmm5[0,1,2,3],xmm6[4,5,6,7]
-; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    psrld %xmm1, %xmm2
-; SSE41-NEXT:    pshuflw {{.*#+}} xmm1 = xmm3[0,1,1,1,4,5,6,7]
+; SSE41-NEXT:    pblendw {{.*#+}} xmm5 = xmm3[0,1,2,3],xmm5[4,5,6,7]
+; SSE41-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,1,1,1,4,5,6,7]
+; SSE41-NEXT:    movdqa %xmm0, %xmm3
+; SSE41-NEXT:    psrld %xmm1, %xmm3
+; SSE41-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,1,1,1,4,5,6,7]
 ; SSE41-NEXT:    psrld %xmm1, %xmm0
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm2[0,1,2,3],xmm0[4,5,6,7]
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm6[2,3],xmm0[4,5],xmm6[6,7]
+; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm3[0,1,2,3],xmm0[4,5,6,7]
+; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm5[2,3],xmm0[4,5],xmm5[6,7]
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: var_shift_v4i16:
+; AVX1-LABEL: var_shift_v2i32:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3],xmm0[4],xmm2[5],xmm0[6],xmm2[7]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
-; AVX1-NEXT:    vpsrldq {{.*#+}} xmm3 = xmm1[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX1-NEXT:    vpsrld %xmm3, %xmm0, %xmm3
-; AVX1-NEXT:    vpsrlq $32, %xmm1, %xmm4
-; AVX1-NEXT:    vpsrld %xmm4, %xmm0, %xmm4
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm4[0,1,2,3],xmm3[4,5,6,7]
-; AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm2 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; AVX1-NEXT:    vpsrldq {{.*#+}} xmm2 = xmm1[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; AVX1-NEXT:    vpsrld %xmm2, %xmm0, %xmm2
+; AVX1-NEXT:    vpsrlq $32, %xmm1, %xmm3
+; AVX1-NEXT:    vpsrld %xmm3, %xmm0, %xmm3
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
+; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm3 = xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; AVX1-NEXT:    vpsrld %xmm3, %xmm0, %xmm3
 ; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
 ; AVX1-NEXT:    vpsrld %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm2[4,5,6,7]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm3[2,3],xmm0[4,5],xmm3[6,7]
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm3[4,5,6,7]
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
 ; AVX1-NEXT:    retq
 ;
-; AVX2-LABEL: var_shift_v4i16:
+; AVX2-LABEL: var_shift_v2i32:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
-; AVX2-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3],xmm0[4],xmm2[5],xmm0[6],xmm2[7]
 ; AVX2-NEXT:    vpsrlvd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    retq
 ;
-; XOPAVX1-LABEL: var_shift_v4i16:
+; XOPAVX1-LABEL: var_shift_v2i32:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; XOPAVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3],xmm0[4],xmm2[5],xmm0[6],xmm2[7]
-; XOPAVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
 ; XOPAVX1-NEXT:    vpsubd %xmm1, %xmm2, %xmm1
 ; XOPAVX1-NEXT:    vpshld %xmm1, %xmm0, %xmm0
 ; XOPAVX1-NEXT:    retq
 ;
-; XOPAVX2-LABEL: var_shift_v4i16:
+; XOPAVX2-LABEL: var_shift_v2i32:
 ; XOPAVX2:       # %bb.0:
-; XOPAVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; XOPAVX2-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
-; XOPAVX2-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3],xmm0[4],xmm2[5],xmm0[6],xmm2[7]
 ; XOPAVX2-NEXT:    vpsrlvd %xmm1, %xmm0, %xmm0
 ; XOPAVX2-NEXT:    retq
 ;
-; AVX512-LABEL: var_shift_v4i16:
+; AVX512-LABEL: var_shift_v2i32:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3],xmm0[4],xmm2[5],xmm0[6],xmm2[7]
 ; AVX512-NEXT:    vpsrlvd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    retq
 ;
-; AVX512VL-LABEL: var_shift_v4i16:
+; AVX512VL-LABEL: var_shift_v2i32:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VL-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
-; AVX512VL-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3],xmm0[4],xmm2[5],xmm0[6],xmm2[7]
 ; AVX512VL-NEXT:    vpsrlvd %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: var_shift_v4i16:
+; X32-SSE-LABEL: var_shift_v2i32:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [65535,0,65535,0,65535,0,65535,0]
-; X32-SSE-NEXT:    pand %xmm2, %xmm0
-; X32-SSE-NEXT:    pand %xmm2, %xmm1
 ; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm1[2,3,3,3,4,5,6,7]
 ; X32-SSE-NEXT:    movdqa %xmm0, %xmm3
 ; X32-SSE-NEXT:    psrld %xmm2, %xmm3
@@ -239,111 +119,21 @@ define <4 x i16> @var_shift_v4i16(<4 x i16> %a, <4 x i16> %b) nounwind {
 ; X32-SSE-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,3],xmm0[0,3]
 ; X32-SSE-NEXT:    movaps %xmm2, %xmm0
 ; X32-SSE-NEXT:    retl
-  %shift = lshr <4 x i16> %a, %b
-  ret <4 x i16> %shift
-}
-
-define <2 x i16> @var_shift_v2i16(<2 x i16> %a, <2 x i16> %b) nounwind {
-; SSE2-LABEL: var_shift_v2i16:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [65535,0,0,0,65535,0,0,0]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    psrlq %xmm1, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
-; SSE2-NEXT:    psrlq %xmm1, %xmm0
-; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm2[0],xmm0[1]
-; SSE2-NEXT:    retq
-;
-; SSE41-LABEL: var_shift_v2i16:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pxor %xmm2, %xmm2
-; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3],xmm1[4],xmm2[5,6,7]
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3],xmm0[4],xmm2[5,6,7]
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    psrlq %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
-; SSE41-NEXT:    psrlq %xmm1, %xmm0
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm2[0,1,2,3],xmm0[4,5,6,7]
-; SSE41-NEXT:    retq
-;
-; AVX1-LABEL: var_shift_v2i16:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3],xmm1[4],xmm2[5,6,7]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3],xmm0[4],xmm2[5,6,7]
-; AVX1-NEXT:    vpsrlq %xmm1, %xmm0, %xmm2
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
-; AVX1-NEXT:    vpsrlq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm2[0,1,2,3],xmm0[4,5,6,7]
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: var_shift_v2i16:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3],xmm1[4],xmm2[5,6,7]
-; AVX2-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3],xmm0[4],xmm2[5,6,7]
-; AVX2-NEXT:    vpsrlvq %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    retq
-;
-; XOPAVX1-LABEL: var_shift_v2i16:
-; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; XOPAVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3],xmm0[4],xmm2[5,6,7]
-; XOPAVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3],xmm1[4],xmm2[5,6,7]
-; XOPAVX1-NEXT:    vpsubq %xmm1, %xmm2, %xmm1
-; XOPAVX1-NEXT:    vpshlq %xmm1, %xmm0, %xmm0
-; XOPAVX1-NEXT:    retq
-;
-; XOPAVX2-LABEL: var_shift_v2i16:
-; XOPAVX2:       # %bb.0:
-; XOPAVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; XOPAVX2-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3],xmm1[4],xmm2[5,6,7]
-; XOPAVX2-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3],xmm0[4],xmm2[5,6,7]
-; XOPAVX2-NEXT:    vpsrlvq %xmm1, %xmm0, %xmm0
-; XOPAVX2-NEXT:    retq
-;
-; AVX512-LABEL: var_shift_v2i16:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3],xmm1[4],xmm2[5,6,7]
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3],xmm0[4],xmm2[5,6,7]
-; AVX512-NEXT:    vpsrlvq %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    retq
-;
-; AVX512VL-LABEL: var_shift_v2i16:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VL-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3],xmm1[4],xmm2[5,6,7]
-; AVX512VL-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3],xmm0[4],xmm2[5,6,7]
-; AVX512VL-NEXT:    vpsrlvq %xmm1, %xmm0, %xmm0
-; AVX512VL-NEXT:    retq
-;
-; X32-SSE-LABEL: var_shift_v2i16:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [65535,0,0,0,65535,0,0,0]
-; X32-SSE-NEXT:    pand %xmm2, %xmm1
-; X32-SSE-NEXT:    pand %xmm2, %xmm0
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm2
-; X32-SSE-NEXT:    psrlq %xmm1, %xmm2
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
-; X32-SSE-NEXT:    psrlq %xmm1, %xmm0
-; X32-SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm2[0],xmm0[1]
-; X32-SSE-NEXT:    retl
-  %shift = lshr <2 x i16> %a, %b
-  ret <2 x i16> %shift
+  %shift = lshr <2 x i32> %a, %b
+  ret <2 x i32> %shift
 }
 
-define <8 x i8> @var_shift_v8i8(<8 x i8> %a, <8 x i8> %b) nounwind {
-; SSE2-LABEL: var_shift_v8i8:
+define <4 x i16> @var_shift_v4i16(<4 x i16> %a, <4 x i16> %b) nounwind {
+; SSE2-LABEL: var_shift_v4i16:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm2
 ; SSE2-NEXT:    psllw $12, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm0
-; SSE2-NEXT:    psraw $15, %xmm0
-; SSE2-NEXT:    pandn %xmm2, %xmm0
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    psraw $15, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm3
+; SSE2-NEXT:    pandn %xmm0, %xmm3
+; SSE2-NEXT:    psrlw $8, %xmm0
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    por %xmm3, %xmm0
 ; SSE2-NEXT:    paddw %xmm1, %xmm1
 ; SSE2-NEXT:    movdqa %xmm1, %xmm2
 ; SSE2-NEXT:    psraw $15, %xmm2
@@ -369,20 +159,18 @@ define <8 x i8> @var_shift_v8i8(<8 x i8> %a, <8 x i8> %b) nounwind {
 ; SSE2-NEXT:    por %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: var_shift_v8i8:
+; SSE41-LABEL: var_shift_v4i16:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movdqa %xmm1, %xmm2
 ; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
-; SSE41-NEXT:    pand %xmm0, %xmm1
-; SSE41-NEXT:    pand %xmm0, %xmm2
 ; SSE41-NEXT:    movdqa %xmm2, %xmm0
 ; SSE41-NEXT:    psllw $12, %xmm0
 ; SSE41-NEXT:    psllw $4, %xmm2
 ; SSE41-NEXT:    por %xmm0, %xmm2
 ; SSE41-NEXT:    movdqa %xmm2, %xmm3
 ; SSE41-NEXT:    paddw %xmm2, %xmm3
-; SSE41-NEXT:    pxor %xmm4, %xmm4
+; SSE41-NEXT:    movdqa %xmm1, %xmm4
+; SSE41-NEXT:    psrlw $8, %xmm4
 ; SSE41-NEXT:    movdqa %xmm2, %xmm0
 ; SSE41-NEXT:    pblendvb %xmm0, %xmm4, %xmm1
 ; SSE41-NEXT:    movdqa %xmm1, %xmm2
@@ -402,16 +190,13 @@ define <8 x i8> @var_shift_v8i8(<8 x i8> %a, <8 x i8> %b) nounwind {
 ; SSE41-NEXT:    movdqa %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: var_shift_v8i8:
+; AVX1-LABEL: var_shift_v4i16:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
-; AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpand %xmm2, %xmm1, %xmm1
 ; AVX1-NEXT:    vpsllw $12, %xmm1, %xmm2
 ; AVX1-NEXT:    vpsllw $4, %xmm1, %xmm1
 ; AVX1-NEXT:    vpor %xmm2, %xmm1, %xmm1
 ; AVX1-NEXT:    vpaddw %xmm1, %xmm1, %xmm2
-; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm3
 ; AVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm1
 ; AVX1-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
@@ -423,45 +208,25 @@ define <8 x i8> @var_shift_v8i8(<8 x i8> %a, <8 x i8> %b) nounwind {
 ; AVX1-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    retq
 ;
-; AVX2-LABEL: var_shift_v8i8:
+; AVX2-LABEL: var_shift_v4i16:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpbroadcastw {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
-; AVX2-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
 ; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
 ; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; AVX2-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
-; XOPAVX1-LABEL: var_shift_v8i8:
-; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
-; XOPAVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; XOPAVX1-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; XOPAVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; XOPAVX1-NEXT:    vpsubw %xmm1, %xmm2, %xmm1
-; XOPAVX1-NEXT:    vpshlw %xmm1, %xmm0, %xmm0
-; XOPAVX1-NEXT:    retq
-;
-; XOPAVX2-LABEL: var_shift_v8i8:
-; XOPAVX2:       # %bb.0:
-; XOPAVX2-NEXT:    vpbroadcastw {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
-; XOPAVX2-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; XOPAVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; XOPAVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; XOPAVX2-NEXT:    vpsubw %xmm1, %xmm2, %xmm1
-; XOPAVX2-NEXT:    vpshlw %xmm1, %xmm0, %xmm0
-; XOPAVX2-NEXT:    retq
+; XOP-LABEL: var_shift_v4i16:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; XOP-NEXT:    vpsubw %xmm1, %xmm2, %xmm1
+; XOP-NEXT:    vpshlw %xmm1, %xmm0, %xmm0
+; XOP-NEXT:    retq
 ;
-; AVX512DQ-LABEL: var_shift_v8i8:
+; AVX512DQ-LABEL: var_shift_v4i16:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vpbroadcastw {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
-; AVX512DQ-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; AVX512DQ-NEXT:    vpand %xmm2, %xmm1, %xmm1
 ; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
 ; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; AVX512DQ-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm0
@@ -470,21 +235,17 @@ define <8 x i8> @var_shift_v8i8(<8 x i8> %a, <8 x i8> %b) nounwind {
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
 ;
-; AVX512BW-LABEL: var_shift_v8i8:
+; AVX512BW-LABEL: var_shift_v4i16:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
-; AVX512BW-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; AVX512BW-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
-; AVX512DQVL-LABEL: var_shift_v8i8:
+; AVX512DQVL-LABEL: var_shift_v4i16:
 ; AVX512DQVL:       # %bb.0:
-; AVX512DQVL-NEXT:    vpbroadcastw {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
-; AVX512DQVL-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; AVX512DQVL-NEXT:    vpand %xmm2, %xmm1, %xmm1
 ; AVX512DQVL-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
 ; AVX512DQVL-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; AVX512DQVL-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm0
@@ -492,22 +253,21 @@ define <8 x i8> @var_shift_v8i8(<8 x i8> %a, <8 x i8> %b) nounwind {
 ; AVX512DQVL-NEXT:    vzeroupper
 ; AVX512DQVL-NEXT:    retq
 ;
-; AVX512BWVL-LABEL: var_shift_v8i8:
+; AVX512BWVL-LABEL: var_shift_v4i16:
 ; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vpbroadcastw {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
-; AVX512BWVL-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; AVX512BWVL-NEXT:    vpand %xmm2, %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    vpsrlvw %xmm1, %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-SSE-LABEL: var_shift_v8i8:
+; X32-SSE-LABEL: var_shift_v4i16:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm2
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm2
 ; X32-SSE-NEXT:    psllw $12, %xmm1
-; X32-SSE-NEXT:    movdqa %xmm1, %xmm0
-; X32-SSE-NEXT:    psraw $15, %xmm0
-; X32-SSE-NEXT:    pandn %xmm2, %xmm0
+; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
+; X32-SSE-NEXT:    psraw $15, %xmm2
+; X32-SSE-NEXT:    movdqa %xmm2, %xmm3
+; X32-SSE-NEXT:    pandn %xmm0, %xmm3
+; X32-SSE-NEXT:    psrlw $8, %xmm0
+; X32-SSE-NEXT:    pand %xmm2, %xmm0
+; X32-SSE-NEXT:    por %xmm3, %xmm0
 ; X32-SSE-NEXT:    paddw %xmm1, %xmm1
 ; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
 ; X32-SSE-NEXT:    psraw $15, %xmm2
@@ -532,138 +292,478 @@ define <8 x i8> @var_shift_v8i8(<8 x i8> %a, <8 x i8> %b) nounwind {
 ; X32-SSE-NEXT:    pand %xmm1, %xmm0
 ; X32-SSE-NEXT:    por %xmm2, %xmm0
 ; X32-SSE-NEXT:    retl
-  %shift = lshr <8 x i8> %a, %b
-  ret <8 x i8> %shift
+  %shift = lshr <4 x i16> %a, %b
+  ret <4 x i16> %shift
 }
 
-define <4 x i8> @var_shift_v4i8(<4 x i8> %a, <4 x i8> %b) nounwind {
-; SSE2-LABEL: var_shift_v4i8:
+define <2 x i16> @var_shift_v2i16(<2 x i16> %a, <2 x i16> %b) nounwind {
+; SSE2-LABEL: var_shift_v2i16:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0]
+; SSE2-NEXT:    psllw $12, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    psraw $15, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm3
+; SSE2-NEXT:    pandn %xmm0, %xmm3
+; SSE2-NEXT:    psrlw $8, %xmm0
 ; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm1[2,3,3,3,4,5,6,7]
-; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    psrld %xmm2, %xmm3
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm4 = xmm1[0,1,1,1,4,5,6,7]
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    psrld %xmm4, %xmm2
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm1[2,3,3,3,4,5,6,7]
-; SSE2-NEXT:    movdqa %xmm0, %xmm4
-; SSE2-NEXT:    psrld %xmm3, %xmm4
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,1,1,1,4,5,6,7]
-; SSE2-NEXT:    psrld %xmm1, %xmm0
-; SSE2-NEXT:    punpckhqdq {{.*#+}} xmm0 = xmm0[1],xmm4[1]
-; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,3],xmm0[0,3]
-; SSE2-NEXT:    movaps %xmm2, %xmm0
+; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    paddw %xmm1, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    psraw $15, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm3
+; SSE2-NEXT:    pandn %xmm0, %xmm3
+; SSE2-NEXT:    psrlw $4, %xmm0
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    paddw %xmm1, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    psraw $15, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm3
+; SSE2-NEXT:    pandn %xmm0, %xmm3
+; SSE2-NEXT:    psrlw $2, %xmm0
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    paddw %xmm1, %xmm1
+; SSE2-NEXT:    psraw $15, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    pandn %xmm0, %xmm2
+; SSE2-NEXT:    psrlw $1, %xmm0
+; SSE2-NEXT:    pand %xmm1, %xmm0
+; SSE2-NEXT:    por %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: var_shift_v4i8:
+; SSE41-LABEL: var_shift_v2i16:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0]
-; SSE41-NEXT:    pand %xmm2, %xmm0
-; SSE41-NEXT:    pand %xmm2, %xmm1
-; SSE41-NEXT:    pshuflw {{.*#+}} xmm2 = xmm1[2,3,3,3,4,5,6,7]
-; SSE41-NEXT:    movdqa %xmm0, %xmm3
-; SSE41-NEXT:    psrld %xmm2, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
-; SSE41-NEXT:    pshuflw {{.*#+}} xmm4 = xmm2[2,3,3,3,4,5,6,7]
-; SSE41-NEXT:    movdqa %xmm0, %xmm5
-; SSE41-NEXT:    psrld %xmm4, %xmm5
-; SSE41-NEXT:    pblendw {{.*#+}} xmm5 = xmm3[0,1,2,3],xmm5[4,5,6,7]
-; SSE41-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,1,1,1,4,5,6,7]
-; SSE41-NEXT:    movdqa %xmm0, %xmm3
-; SSE41-NEXT:    psrld %xmm1, %xmm3
-; SSE41-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,1,1,1,4,5,6,7]
-; SSE41-NEXT:    psrld %xmm1, %xmm0
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm3[0,1,2,3],xmm0[4,5,6,7]
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm5[2,3],xmm0[4,5],xmm5[6,7]
-; SSE41-NEXT:    retq
-;
-; AVX1-LABEL: var_shift_v4i8:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm2 = [3.57331108E-43,3.57331108E-43,3.57331108E-43,3.57331108E-43]
-; AVX1-NEXT:    vandps %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vandps %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpsrldq {{.*#+}} xmm2 = xmm1[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX1-NEXT:    vpsrld %xmm2, %xmm0, %xmm2
-; AVX1-NEXT:    vpsrlq $32, %xmm1, %xmm3
-; AVX1-NEXT:    vpsrld %xmm3, %xmm0, %xmm3
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
-; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm3 = xmm1[2],xmm3[2],xmm1[3],xmm3[3]
-; AVX1-NEXT:    vpsrld %xmm3, %xmm0, %xmm3
-; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
-; AVX1-NEXT:    vpsrld %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm3[4,5,6,7]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
+; SSE41-NEXT:    movdqa %xmm1, %xmm2
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    psllw $12, %xmm0
+; SSE41-NEXT:    psllw $4, %xmm2
+; SSE41-NEXT:    por %xmm0, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm3
+; SSE41-NEXT:    paddw %xmm2, %xmm3
+; SSE41-NEXT:    movdqa %xmm1, %xmm4
+; SSE41-NEXT:    psrlw $8, %xmm4
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    pblendvb %xmm0, %xmm4, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm2
+; SSE41-NEXT:    psrlw $4, %xmm2
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    pblendvb %xmm0, %xmm2, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm2
+; SSE41-NEXT:    psrlw $2, %xmm2
+; SSE41-NEXT:    paddw %xmm3, %xmm3
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    pblendvb %xmm0, %xmm2, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm2
+; SSE41-NEXT:    psrlw $1, %xmm2
+; SSE41-NEXT:    paddw %xmm3, %xmm3
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    pblendvb %xmm0, %xmm2, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: var_shift_v2i16:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpsllw $12, %xmm1, %xmm2
+; AVX1-NEXT:    vpsllw $4, %xmm1, %xmm1
+; AVX1-NEXT:    vpor %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpaddw %xmm1, %xmm1, %xmm2
+; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm3
+; AVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm1
+; AVX1-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpsrlw $2, %xmm0, %xmm1
+; AVX1-NEXT:    vpaddw %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpsrlw $1, %xmm0, %xmm1
+; AVX1-NEXT:    vpaddw %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    retq
 ;
-; AVX2-LABEL: var_shift_v4i8:
+; AVX2-LABEL: var_shift_v2i16:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [255,255,255,255]
-; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrlvd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
-; XOPAVX1-LABEL: var_shift_v4i8:
-; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vbroadcastss {{.*#+}} xmm2 = [3.57331108E-43,3.57331108E-43,3.57331108E-43,3.57331108E-43]
-; XOPAVX1-NEXT:    vandps %xmm2, %xmm0, %xmm0
-; XOPAVX1-NEXT:    vandps %xmm2, %xmm1, %xmm1
-; XOPAVX1-NEXT:    vxorps %xmm2, %xmm2, %xmm2
-; XOPAVX1-NEXT:    vpsubd %xmm1, %xmm2, %xmm1
-; XOPAVX1-NEXT:    vpshld %xmm1, %xmm0, %xmm0
-; XOPAVX1-NEXT:    retq
+; XOP-LABEL: var_shift_v2i16:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; XOP-NEXT:    vpsubw %xmm1, %xmm2, %xmm1
+; XOP-NEXT:    vpshlw %xmm1, %xmm0, %xmm0
+; XOP-NEXT:    retq
 ;
-; XOPAVX2-LABEL: var_shift_v4i8:
-; XOPAVX2:       # %bb.0:
-; XOPAVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [255,255,255,255]
-; XOPAVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; XOPAVX2-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; XOPAVX2-NEXT:    vpsrlvd %xmm1, %xmm0, %xmm0
-; XOPAVX2-NEXT:    retq
+; AVX512DQ-LABEL: var_shift_v2i16:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512DQ-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    retq
 ;
-; AVX512-LABEL: var_shift_v4i8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [255,255,255,255]
-; AVX512-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; AVX512-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrlvd %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    retq
+; AVX512BW-LABEL: var_shift_v2i16:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512BW-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
 ;
-; AVX512VL-LABEL: var_shift_v4i8:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [255,255,255,255]
-; AVX512VL-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpsrlvd %xmm1, %xmm0, %xmm0
-; AVX512VL-NEXT:    retq
+; AVX512DQVL-LABEL: var_shift_v2i16:
+; AVX512DQVL:       # %bb.0:
+; AVX512DQVL-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX512DQVL-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512DQVL-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm0
+; AVX512DQVL-NEXT:    vpmovdw %ymm0, %xmm0
+; AVX512DQVL-NEXT:    vzeroupper
+; AVX512DQVL-NEXT:    retq
+;
+; AVX512BWVL-LABEL: var_shift_v2i16:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpsrlvw %xmm1, %xmm0, %xmm0
+; AVX512BWVL-NEXT:    retq
+;
+; X32-SSE-LABEL: var_shift_v2i16:
+; X32-SSE:       # %bb.0:
+; X32-SSE-NEXT:    psllw $12, %xmm1
+; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
+; X32-SSE-NEXT:    psraw $15, %xmm2
+; X32-SSE-NEXT:    movdqa %xmm2, %xmm3
+; X32-SSE-NEXT:    pandn %xmm0, %xmm3
+; X32-SSE-NEXT:    psrlw $8, %xmm0
+; X32-SSE-NEXT:    pand %xmm2, %xmm0
+; X32-SSE-NEXT:    por %xmm3, %xmm0
+; X32-SSE-NEXT:    paddw %xmm1, %xmm1
+; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
+; X32-SSE-NEXT:    psraw $15, %xmm2
+; X32-SSE-NEXT:    movdqa %xmm2, %xmm3
+; X32-SSE-NEXT:    pandn %xmm0, %xmm3
+; X32-SSE-NEXT:    psrlw $4, %xmm0
+; X32-SSE-NEXT:    pand %xmm2, %xmm0
+; X32-SSE-NEXT:    por %xmm3, %xmm0
+; X32-SSE-NEXT:    paddw %xmm1, %xmm1
+; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
+; X32-SSE-NEXT:    psraw $15, %xmm2
+; X32-SSE-NEXT:    movdqa %xmm2, %xmm3
+; X32-SSE-NEXT:    pandn %xmm0, %xmm3
+; X32-SSE-NEXT:    psrlw $2, %xmm0
+; X32-SSE-NEXT:    pand %xmm2, %xmm0
+; X32-SSE-NEXT:    por %xmm3, %xmm0
+; X32-SSE-NEXT:    paddw %xmm1, %xmm1
+; X32-SSE-NEXT:    psraw $15, %xmm1
+; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
+; X32-SSE-NEXT:    pandn %xmm0, %xmm2
+; X32-SSE-NEXT:    psrlw $1, %xmm0
+; X32-SSE-NEXT:    pand %xmm1, %xmm0
+; X32-SSE-NEXT:    por %xmm2, %xmm0
+; X32-SSE-NEXT:    retl
+  %shift = lshr <2 x i16> %a, %b
+  ret <2 x i16> %shift
+}
+
+define <8 x i8> @var_shift_v8i8(<8 x i8> %a, <8 x i8> %b) nounwind {
+; SSE2-LABEL: var_shift_v8i8:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    psllw $5, %xmm1
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    pcmpgtb %xmm1, %xmm3
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    pandn %xmm0, %xmm4
+; SSE2-NEXT:    psrlw $4, %xmm0
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    por %xmm4, %xmm0
+; SSE2-NEXT:    paddb %xmm1, %xmm1
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    pcmpgtb %xmm1, %xmm3
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    pandn %xmm0, %xmm4
+; SSE2-NEXT:    psrlw $2, %xmm0
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    por %xmm4, %xmm0
+; SSE2-NEXT:    paddb %xmm1, %xmm1
+; SSE2-NEXT:    pcmpgtb %xmm1, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm1
+; SSE2-NEXT:    pandn %xmm0, %xmm1
+; SSE2-NEXT:    psrlw $1, %xmm0
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    por %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: var_shift_v8i8:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    psllw $5, %xmm1
+; SSE41-NEXT:    movdqa %xmm0, %xmm3
+; SSE41-NEXT:    psrlw $4, %xmm3
+; SSE41-NEXT:    pand {{.*}}(%rip), %xmm3
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pblendvb %xmm0, %xmm3, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm3
+; SSE41-NEXT:    psrlw $2, %xmm3
+; SSE41-NEXT:    pand {{.*}}(%rip), %xmm3
+; SSE41-NEXT:    paddb %xmm1, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pblendvb %xmm0, %xmm3, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm3
+; SSE41-NEXT:    psrlw $1, %xmm3
+; SSE41-NEXT:    pand {{.*}}(%rip), %xmm3
+; SSE41-NEXT:    paddb %xmm1, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pblendvb %xmm0, %xmm3, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: var_shift_v8i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpsllw $5, %xmm1, %xmm1
+; AVX-NEXT:    vpsrlw $4, %xmm0, %xmm2
+; AVX-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
+; AVX-NEXT:    vpblendvb %xmm1, %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    vpsrlw $2, %xmm0, %xmm2
+; AVX-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
+; AVX-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vpblendvb %xmm1, %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    vpsrlw $1, %xmm0, %xmm2
+; AVX-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
+; AVX-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vpblendvb %xmm1, %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; XOP-LABEL: var_shift_v8i8:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; XOP-NEXT:    vpsubb %xmm1, %xmm2, %xmm1
+; XOP-NEXT:    vpshlb %xmm1, %xmm0, %xmm0
+; XOP-NEXT:    retq
+;
+; AVX512DQ-LABEL: var_shift_v8i8:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; AVX512DQ-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; AVX512DQ-NEXT:    vpsrlvd %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    retq
+;
+; AVX512BW-LABEL: var_shift_v8i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512BW-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512DQVL-LABEL: var_shift_v8i8:
+; AVX512DQVL:       # %bb.0:
+; AVX512DQVL-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; AVX512DQVL-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; AVX512DQVL-NEXT:    vpsrlvd %zmm1, %zmm0, %zmm0
+; AVX512DQVL-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512DQVL-NEXT:    vzeroupper
+; AVX512DQVL-NEXT:    retq
+;
+; AVX512BWVL-LABEL: var_shift_v8i8:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512BWVL-NEXT:    vpsrlvw %ymm1, %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm0
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
+;
+; X32-SSE-LABEL: var_shift_v8i8:
+; X32-SSE:       # %bb.0:
+; X32-SSE-NEXT:    psllw $5, %xmm1
+; X32-SSE-NEXT:    pxor %xmm2, %xmm2
+; X32-SSE-NEXT:    pxor %xmm3, %xmm3
+; X32-SSE-NEXT:    pcmpgtb %xmm1, %xmm3
+; X32-SSE-NEXT:    movdqa %xmm3, %xmm4
+; X32-SSE-NEXT:    pandn %xmm0, %xmm4
+; X32-SSE-NEXT:    psrlw $4, %xmm0
+; X32-SSE-NEXT:    pand %xmm3, %xmm0
+; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X32-SSE-NEXT:    por %xmm4, %xmm0
+; X32-SSE-NEXT:    paddb %xmm1, %xmm1
+; X32-SSE-NEXT:    pxor %xmm3, %xmm3
+; X32-SSE-NEXT:    pcmpgtb %xmm1, %xmm3
+; X32-SSE-NEXT:    movdqa %xmm3, %xmm4
+; X32-SSE-NEXT:    pandn %xmm0, %xmm4
+; X32-SSE-NEXT:    psrlw $2, %xmm0
+; X32-SSE-NEXT:    pand %xmm3, %xmm0
+; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X32-SSE-NEXT:    por %xmm4, %xmm0
+; X32-SSE-NEXT:    paddb %xmm1, %xmm1
+; X32-SSE-NEXT:    pcmpgtb %xmm1, %xmm2
+; X32-SSE-NEXT:    movdqa %xmm2, %xmm1
+; X32-SSE-NEXT:    pandn %xmm0, %xmm1
+; X32-SSE-NEXT:    psrlw $1, %xmm0
+; X32-SSE-NEXT:    pand %xmm2, %xmm0
+; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X32-SSE-NEXT:    por %xmm1, %xmm0
+; X32-SSE-NEXT:    retl
+  %shift = lshr <8 x i8> %a, %b
+  ret <8 x i8> %shift
+}
+
+define <4 x i8> @var_shift_v4i8(<4 x i8> %a, <4 x i8> %b) nounwind {
+; SSE2-LABEL: var_shift_v4i8:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    psllw $5, %xmm1
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    pcmpgtb %xmm1, %xmm3
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    pandn %xmm0, %xmm4
+; SSE2-NEXT:    psrlw $4, %xmm0
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    por %xmm4, %xmm0
+; SSE2-NEXT:    paddb %xmm1, %xmm1
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    pcmpgtb %xmm1, %xmm3
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    pandn %xmm0, %xmm4
+; SSE2-NEXT:    psrlw $2, %xmm0
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    por %xmm4, %xmm0
+; SSE2-NEXT:    paddb %xmm1, %xmm1
+; SSE2-NEXT:    pcmpgtb %xmm1, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm1
+; SSE2-NEXT:    pandn %xmm0, %xmm1
+; SSE2-NEXT:    psrlw $1, %xmm0
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    por %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: var_shift_v4i8:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    psllw $5, %xmm1
+; SSE41-NEXT:    movdqa %xmm0, %xmm3
+; SSE41-NEXT:    psrlw $4, %xmm3
+; SSE41-NEXT:    pand {{.*}}(%rip), %xmm3
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pblendvb %xmm0, %xmm3, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm3
+; SSE41-NEXT:    psrlw $2, %xmm3
+; SSE41-NEXT:    pand {{.*}}(%rip), %xmm3
+; SSE41-NEXT:    paddb %xmm1, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pblendvb %xmm0, %xmm3, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm3
+; SSE41-NEXT:    psrlw $1, %xmm3
+; SSE41-NEXT:    pand {{.*}}(%rip), %xmm3
+; SSE41-NEXT:    paddb %xmm1, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pblendvb %xmm0, %xmm3, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: var_shift_v4i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpsllw $5, %xmm1, %xmm1
+; AVX-NEXT:    vpsrlw $4, %xmm0, %xmm2
+; AVX-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
+; AVX-NEXT:    vpblendvb %xmm1, %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    vpsrlw $2, %xmm0, %xmm2
+; AVX-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
+; AVX-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vpblendvb %xmm1, %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    vpsrlw $1, %xmm0, %xmm2
+; AVX-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
+; AVX-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vpblendvb %xmm1, %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; XOP-LABEL: var_shift_v4i8:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; XOP-NEXT:    vpsubb %xmm1, %xmm2, %xmm1
+; XOP-NEXT:    vpshlb %xmm1, %xmm0, %xmm0
+; XOP-NEXT:    retq
+;
+; AVX512DQ-LABEL: var_shift_v4i8:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; AVX512DQ-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; AVX512DQ-NEXT:    vpsrlvd %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    retq
+;
+; AVX512BW-LABEL: var_shift_v4i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512BW-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512DQVL-LABEL: var_shift_v4i8:
+; AVX512DQVL:       # %bb.0:
+; AVX512DQVL-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; AVX512DQVL-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; AVX512DQVL-NEXT:    vpsrlvd %zmm1, %zmm0, %zmm0
+; AVX512DQVL-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512DQVL-NEXT:    vzeroupper
+; AVX512DQVL-NEXT:    retq
+;
+; AVX512BWVL-LABEL: var_shift_v4i8:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512BWVL-NEXT:    vpsrlvw %ymm1, %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm0
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
 ;
 ; X32-SSE-LABEL: var_shift_v4i8:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0]
+; X32-SSE-NEXT:    psllw $5, %xmm1
+; X32-SSE-NEXT:    pxor %xmm2, %xmm2
+; X32-SSE-NEXT:    pxor %xmm3, %xmm3
+; X32-SSE-NEXT:    pcmpgtb %xmm1, %xmm3
+; X32-SSE-NEXT:    movdqa %xmm3, %xmm4
+; X32-SSE-NEXT:    pandn %xmm0, %xmm4
+; X32-SSE-NEXT:    psrlw $4, %xmm0
+; X32-SSE-NEXT:    pand %xmm3, %xmm0
+; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X32-SSE-NEXT:    por %xmm4, %xmm0
+; X32-SSE-NEXT:    paddb %xmm1, %xmm1
+; X32-SSE-NEXT:    pxor %xmm3, %xmm3
+; X32-SSE-NEXT:    pcmpgtb %xmm1, %xmm3
+; X32-SSE-NEXT:    movdqa %xmm3, %xmm4
+; X32-SSE-NEXT:    pandn %xmm0, %xmm4
+; X32-SSE-NEXT:    psrlw $2, %xmm0
+; X32-SSE-NEXT:    pand %xmm3, %xmm0
+; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X32-SSE-NEXT:    por %xmm4, %xmm0
+; X32-SSE-NEXT:    paddb %xmm1, %xmm1
+; X32-SSE-NEXT:    pcmpgtb %xmm1, %xmm2
+; X32-SSE-NEXT:    movdqa %xmm2, %xmm1
+; X32-SSE-NEXT:    pandn %xmm0, %xmm1
+; X32-SSE-NEXT:    psrlw $1, %xmm0
 ; X32-SSE-NEXT:    pand %xmm2, %xmm0
-; X32-SSE-NEXT:    pand %xmm2, %xmm1
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm1[2,3,3,3,4,5,6,7]
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm3
-; X32-SSE-NEXT:    psrld %xmm2, %xmm3
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm4 = xmm1[0,1,1,1,4,5,6,7]
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm2
-; X32-SSE-NEXT:    psrld %xmm4, %xmm2
-; X32-SSE-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm3 = xmm1[2,3,3,3,4,5,6,7]
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm4
-; X32-SSE-NEXT:    psrld %xmm3, %xmm4
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,1,1,1,4,5,6,7]
-; X32-SSE-NEXT:    psrld %xmm1, %xmm0
-; X32-SSE-NEXT:    punpckhqdq {{.*#+}} xmm0 = xmm0[1],xmm4[1]
-; X32-SSE-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,3],xmm0[0,3]
-; X32-SSE-NEXT:    movaps %xmm2, %xmm0
+; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X32-SSE-NEXT:    por %xmm1, %xmm0
 ; X32-SSE-NEXT:    retl
   %shift = lshr <4 x i8> %a, %b
   ret <4 x i8> %shift
@@ -672,93 +772,148 @@ define <4 x i8> @var_shift_v4i8(<4 x i8> %a, <4 x i8> %b) nounwind {
 define <2 x i8> @var_shift_v2i8(<2 x i8> %a, <2 x i8> %b) nounwind {
 ; SSE2-LABEL: var_shift_v2i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0]
-; SSE2-NEXT:    pand %xmm2, %xmm1
+; SSE2-NEXT:    psllw $5, %xmm1
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    pcmpgtb %xmm1, %xmm3
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    pandn %xmm0, %xmm4
+; SSE2-NEXT:    psrlw $4, %xmm0
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    por %xmm4, %xmm0
+; SSE2-NEXT:    paddb %xmm1, %xmm1
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    pcmpgtb %xmm1, %xmm3
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    pandn %xmm0, %xmm4
+; SSE2-NEXT:    psrlw $2, %xmm0
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    por %xmm4, %xmm0
+; SSE2-NEXT:    paddb %xmm1, %xmm1
+; SSE2-NEXT:    pcmpgtb %xmm1, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm1
+; SSE2-NEXT:    pandn %xmm0, %xmm1
+; SSE2-NEXT:    psrlw $1, %xmm0
 ; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    psrlq %xmm1, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
-; SSE2-NEXT:    psrlq %xmm1, %xmm0
-; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm2[0],xmm0[1]
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: var_shift_v2i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [255,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0]
-; SSE41-NEXT:    pand %xmm2, %xmm1
-; SSE41-NEXT:    pand %xmm2, %xmm0
 ; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    psrlq %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
-; SSE41-NEXT:    psrlq %xmm1, %xmm0
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm2[0,1,2,3],xmm0[4,5,6,7]
+; SSE41-NEXT:    psllw $5, %xmm1
+; SSE41-NEXT:    movdqa %xmm0, %xmm3
+; SSE41-NEXT:    psrlw $4, %xmm3
+; SSE41-NEXT:    pand {{.*}}(%rip), %xmm3
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pblendvb %xmm0, %xmm3, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm3
+; SSE41-NEXT:    psrlw $2, %xmm3
+; SSE41-NEXT:    pand {{.*}}(%rip), %xmm3
+; SSE41-NEXT:    paddb %xmm1, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pblendvb %xmm0, %xmm3, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm3
+; SSE41-NEXT:    psrlw $1, %xmm3
+; SSE41-NEXT:    pand {{.*}}(%rip), %xmm3
+; SSE41-NEXT:    paddb %xmm1, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pblendvb %xmm0, %xmm3, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: var_shift_v2i8:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovddup {{.*#+}} xmm2 = [1.2598673968951787E-321,1.2598673968951787E-321]
-; AVX1-NEXT:    # xmm2 = mem[0,0]
-; AVX1-NEXT:    vandps %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vandps %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrlq %xmm1, %xmm0, %xmm2
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
-; AVX1-NEXT:    vpsrlq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm2[0,1,2,3],xmm0[4,5,6,7]
-; AVX1-NEXT:    retq
+; AVX-LABEL: var_shift_v2i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpsllw $5, %xmm1, %xmm1
+; AVX-NEXT:    vpsrlw $4, %xmm0, %xmm2
+; AVX-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
+; AVX-NEXT:    vpblendvb %xmm1, %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    vpsrlw $2, %xmm0, %xmm2
+; AVX-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
+; AVX-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vpblendvb %xmm1, %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    vpsrlw $1, %xmm0, %xmm2
+; AVX-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
+; AVX-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vpblendvb %xmm1, %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    retq
 ;
-; AVX2-LABEL: var_shift_v2i8:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpbroadcastq {{.*#+}} xmm2 = [255,255]
-; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrlvq %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    retq
+; XOP-LABEL: var_shift_v2i8:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; XOP-NEXT:    vpsubb %xmm1, %xmm2, %xmm1
+; XOP-NEXT:    vpshlb %xmm1, %xmm0, %xmm0
+; XOP-NEXT:    retq
 ;
-; XOPAVX1-LABEL: var_shift_v2i8:
-; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vmovddup {{.*#+}} xmm2 = [1.2598673968951787E-321,1.2598673968951787E-321]
-; XOPAVX1-NEXT:    # xmm2 = mem[0,0]
-; XOPAVX1-NEXT:    vandps %xmm2, %xmm0, %xmm0
-; XOPAVX1-NEXT:    vandps %xmm2, %xmm1, %xmm1
-; XOPAVX1-NEXT:    vxorps %xmm2, %xmm2, %xmm2
-; XOPAVX1-NEXT:    vpsubq %xmm1, %xmm2, %xmm1
-; XOPAVX1-NEXT:    vpshlq %xmm1, %xmm0, %xmm0
-; XOPAVX1-NEXT:    retq
+; AVX512DQ-LABEL: var_shift_v2i8:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; AVX512DQ-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; AVX512DQ-NEXT:    vpsrlvd %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    retq
 ;
-; XOPAVX2-LABEL: var_shift_v2i8:
-; XOPAVX2:       # %bb.0:
-; XOPAVX2-NEXT:    vpbroadcastq {{.*#+}} xmm2 = [255,255]
-; XOPAVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; XOPAVX2-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; XOPAVX2-NEXT:    vpsrlvq %xmm1, %xmm0, %xmm0
-; XOPAVX2-NEXT:    retq
+; AVX512BW-LABEL: var_shift_v2i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512BW-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
 ;
-; AVX512-LABEL: var_shift_v2i8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpbroadcastq {{.*#+}} xmm2 = [255,255]
-; AVX512-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; AVX512-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrlvq %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    retq
+; AVX512DQVL-LABEL: var_shift_v2i8:
+; AVX512DQVL:       # %bb.0:
+; AVX512DQVL-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; AVX512DQVL-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; AVX512DQVL-NEXT:    vpsrlvd %zmm1, %zmm0, %zmm0
+; AVX512DQVL-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512DQVL-NEXT:    vzeroupper
+; AVX512DQVL-NEXT:    retq
 ;
-; AVX512VL-LABEL: var_shift_v2i8:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpbroadcastq {{.*#+}} xmm2 = [255,255]
-; AVX512VL-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpsrlvq %xmm1, %xmm0, %xmm0
-; AVX512VL-NEXT:    retq
+; AVX512BWVL-LABEL: var_shift_v2i8:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512BWVL-NEXT:    vpsrlvw %ymm1, %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm0
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
 ;
 ; X32-SSE-LABEL: var_shift_v2i8:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [255,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0]
-; X32-SSE-NEXT:    pand %xmm2, %xmm1
+; X32-SSE-NEXT:    psllw $5, %xmm1
+; X32-SSE-NEXT:    pxor %xmm2, %xmm2
+; X32-SSE-NEXT:    pxor %xmm3, %xmm3
+; X32-SSE-NEXT:    pcmpgtb %xmm1, %xmm3
+; X32-SSE-NEXT:    movdqa %xmm3, %xmm4
+; X32-SSE-NEXT:    pandn %xmm0, %xmm4
+; X32-SSE-NEXT:    psrlw $4, %xmm0
+; X32-SSE-NEXT:    pand %xmm3, %xmm0
+; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X32-SSE-NEXT:    por %xmm4, %xmm0
+; X32-SSE-NEXT:    paddb %xmm1, %xmm1
+; X32-SSE-NEXT:    pxor %xmm3, %xmm3
+; X32-SSE-NEXT:    pcmpgtb %xmm1, %xmm3
+; X32-SSE-NEXT:    movdqa %xmm3, %xmm4
+; X32-SSE-NEXT:    pandn %xmm0, %xmm4
+; X32-SSE-NEXT:    psrlw $2, %xmm0
+; X32-SSE-NEXT:    pand %xmm3, %xmm0
+; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X32-SSE-NEXT:    por %xmm4, %xmm0
+; X32-SSE-NEXT:    paddb %xmm1, %xmm1
+; X32-SSE-NEXT:    pcmpgtb %xmm1, %xmm2
+; X32-SSE-NEXT:    movdqa %xmm2, %xmm1
+; X32-SSE-NEXT:    pandn %xmm0, %xmm1
+; X32-SSE-NEXT:    psrlw $1, %xmm0
 ; X32-SSE-NEXT:    pand %xmm2, %xmm0
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm2
-; X32-SSE-NEXT:    psrlq %xmm1, %xmm2
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
-; X32-SSE-NEXT:    psrlq %xmm1, %xmm0
-; X32-SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm2[0],xmm0[1]
+; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X32-SSE-NEXT:    por %xmm1, %xmm0
 ; X32-SSE-NEXT:    retl
   %shift = lshr <2 x i8> %a, %b
   ret <2 x i8> %shift
@@ -771,99 +926,46 @@ define <2 x i8> @var_shift_v2i8(<2 x i8> %a, <2 x i8> %b) nounwind {
 define <2 x i32> @splatvar_shift_v2i32(<2 x i32> %a, <2 x i32> %b) nounwind {
 ; SSE2-LABEL: splatvar_shift_v2i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [4294967295,0,4294967295,0]
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    psrlq %xmm1, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
-; SSE2-NEXT:    psrlq %xmm1, %xmm0
-; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm2[0],xmm0[1]
+; SSE2-NEXT:    xorps %xmm2, %xmm2
+; SSE2-NEXT:    movss {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
+; SSE2-NEXT:    psrld %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: splatvar_shift_v2i32:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pxor %xmm2, %xmm2
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
-; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    psrlq %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
-; SSE41-NEXT:    psrlq %xmm1, %xmm0
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm2[0,1,2,3],xmm0[4,5,6,7]
+; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
+; SSE41-NEXT:    psrld %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: splatvar_shift_v2i32:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
-; AVX1-NEXT:    vpsrlq %xmm1, %xmm0, %xmm2
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
-; AVX1-NEXT:    vpsrlq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm2[0,1,2,3],xmm0[4,5,6,7]
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: splatvar_shift_v2i32:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
-; AVX2-NEXT:    vpbroadcastq %xmm1, %xmm1
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; AVX2-NEXT:    vpsrlvq %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    retq
-;
-; XOPAVX1-LABEL: splatvar_shift_v2i32:
-; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; XOPAVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
-; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
-; XOPAVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
-; XOPAVX1-NEXT:    vpsubq %xmm1, %xmm2, %xmm1
-; XOPAVX1-NEXT:    vpshlq %xmm1, %xmm0, %xmm0
-; XOPAVX1-NEXT:    retq
+; AVX-LABEL: splatvar_shift_v2i32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
+; AVX-NEXT:    vpsrld %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
 ;
-; XOPAVX2-LABEL: splatvar_shift_v2i32:
-; XOPAVX2:       # %bb.0:
-; XOPAVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; XOPAVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
-; XOPAVX2-NEXT:    vpbroadcastq %xmm1, %xmm1
-; XOPAVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; XOPAVX2-NEXT:    vpsrlvq %xmm1, %xmm0, %xmm0
-; XOPAVX2-NEXT:    retq
+; XOP-LABEL: splatvar_shift_v2i32:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
+; XOP-NEXT:    vpsrld %xmm1, %xmm0, %xmm0
+; XOP-NEXT:    retq
 ;
 ; AVX512-LABEL: splatvar_shift_v2i32:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
-; AVX512-NEXT:    vpbroadcastq %xmm1, %xmm1
-; AVX512-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; AVX512-NEXT:    vpsrlvq %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
+; AVX512-NEXT:    vpsrld %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatvar_shift_v2i32:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VL-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
-; AVX512VL-NEXT:    vpbroadcastq %xmm1, %xmm1
-; AVX512VL-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; AVX512VL-NEXT:    vpsrlvq %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
+; AVX512VL-NEXT:    vpsrld %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; X32-SSE-LABEL: splatvar_shift_v2i32:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [4294967295,0,4294967295,0]
-; X32-SSE-NEXT:    pand %xmm2, %xmm0
-; X32-SSE-NEXT:    pand %xmm1, %xmm2
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm3
-; X32-SSE-NEXT:    psrlq %xmm2, %xmm3
-; X32-SSE-NEXT:    pxor %xmm2, %xmm2
+; X32-SSE-NEXT:    xorps %xmm2, %xmm2
 ; X32-SSE-NEXT:    movss {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
-; X32-SSE-NEXT:    psrlq %xmm2, %xmm0
-; X32-SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm3[0],xmm0[1]
+; X32-SSE-NEXT:    psrld %xmm2, %xmm0
 ; X32-SSE-NEXT:    retl
   %splat = shufflevector <2 x i32> %b, <2 x i32> undef, <2 x i32> zeroinitializer
   %shift = lshr <2 x i32> %a, %splat
@@ -873,139 +975,46 @@ define <2 x i32> @splatvar_shift_v2i32(<2 x i32> %a, <2 x i32> %b) nounwind {
 define <4 x i16> @splatvar_shift_v4i16(<4 x i16> %a, <4 x i16> %b) nounwind {
 ; SSE2-LABEL: splatvar_shift_v4i16:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [65535,0,65535,0,65535,0,65535,0]
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,0,0]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm3[2,3,3,3,4,5,6,7]
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    psrld %xmm1, %xmm2
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm4 = xmm3[0,1,1,1,4,5,6,7]
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    psrld %xmm4, %xmm1
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,3,0,1]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm2[2,3,3,3,4,5,6,7]
-; SSE2-NEXT:    movdqa %xmm0, %xmm4
-; SSE2-NEXT:    psrld %xmm3, %xmm4
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,1,1,1,4,5,6,7]
-; SSE2-NEXT:    psrld %xmm2, %xmm0
-; SSE2-NEXT:    punpckhqdq {{.*#+}} xmm0 = xmm0[1],xmm4[1]
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,3],xmm0[0,3]
-; SSE2-NEXT:    movaps %xmm1, %xmm0
+; SSE2-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1]
+; SSE2-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; SSE2-NEXT:    psrlw %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: splatvar_shift_v4i16:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pxor %xmm2, %xmm2
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3],xmm0[4],xmm2[5],xmm0[6],xmm2[7]
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; SSE41-NEXT:    movdqa %xmm1, %xmm3
-; SSE41-NEXT:    pblendw {{.*#+}} xmm3 = xmm3[0],xmm2[1],xmm3[2],xmm2[3],xmm3[4],xmm2[5],xmm3[6],xmm2[7]
-; SSE41-NEXT:    pshuflw {{.*#+}} xmm4 = xmm3[2,3,3,3,4,5,6,7]
-; SSE41-NEXT:    movdqa %xmm0, %xmm5
-; SSE41-NEXT:    psrld %xmm4, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[2,3,0,1]
-; SSE41-NEXT:    pshuflw {{.*#+}} xmm4 = xmm3[2,3,3,3,4,5,6,7]
-; SSE41-NEXT:    movdqa %xmm0, %xmm6
-; SSE41-NEXT:    psrld %xmm4, %xmm6
-; SSE41-NEXT:    pblendw {{.*#+}} xmm6 = xmm5[0,1,2,3],xmm6[4,5,6,7]
-; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    psrld %xmm1, %xmm2
-; SSE41-NEXT:    pshuflw {{.*#+}} xmm1 = xmm3[0,1,1,1,4,5,6,7]
-; SSE41-NEXT:    psrld %xmm1, %xmm0
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm2[0,1,2,3],xmm0[4,5,6,7]
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm6[2,3],xmm0[4,5],xmm6[6,7]
+; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; SSE41-NEXT:    psrlw %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: splatvar_shift_v4i16:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3],xmm0[4],xmm2[5],xmm0[6],xmm2[7]
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
-; AVX1-NEXT:    vpsrldq {{.*#+}} xmm3 = xmm1[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX1-NEXT:    vpsrld %xmm3, %xmm0, %xmm3
-; AVX1-NEXT:    vpsrlq $32, %xmm1, %xmm4
-; AVX1-NEXT:    vpsrld %xmm4, %xmm0, %xmm4
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm4[0,1,2,3],xmm3[4,5,6,7]
-; AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm2 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
-; AVX1-NEXT:    vpsrld %xmm2, %xmm0, %xmm2
-; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
-; AVX1-NEXT:    vpsrld %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm2[4,5,6,7]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm3[2,3],xmm0[4,5],xmm3[6,7]
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: splatvar_shift_v4i16:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3],xmm0[4],xmm2[5],xmm0[6],xmm2[7]
-; AVX2-NEXT:    vpbroadcastd %xmm1, %xmm1
-; AVX2-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
-; AVX2-NEXT:    vpsrlvd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    retq
-;
-; XOPAVX1-LABEL: splatvar_shift_v4i16:
-; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; XOPAVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3],xmm0[4],xmm2[5],xmm0[6],xmm2[7]
-; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; XOPAVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
-; XOPAVX1-NEXT:    vpsubd %xmm1, %xmm2, %xmm1
-; XOPAVX1-NEXT:    vpshld %xmm1, %xmm0, %xmm0
-; XOPAVX1-NEXT:    retq
+; AVX-LABEL: splatvar_shift_v4i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
 ;
-; XOPAVX2-LABEL: splatvar_shift_v4i16:
-; XOPAVX2:       # %bb.0:
-; XOPAVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; XOPAVX2-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3],xmm0[4],xmm2[5],xmm0[6],xmm2[7]
-; XOPAVX2-NEXT:    vpbroadcastd %xmm1, %xmm1
-; XOPAVX2-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
-; XOPAVX2-NEXT:    vpsrlvd %xmm1, %xmm0, %xmm0
-; XOPAVX2-NEXT:    retq
+; XOP-LABEL: splatvar_shift_v4i16:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; XOP-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
+; XOP-NEXT:    retq
 ;
 ; AVX512-LABEL: splatvar_shift_v4i16:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3],xmm0[4],xmm2[5],xmm0[6],xmm2[7]
-; AVX512-NEXT:    vpbroadcastd %xmm1, %xmm1
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
-; AVX512-NEXT:    vpsrlvd %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatvar_shift_v4i16:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VL-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3],xmm0[4],xmm2[5],xmm0[6],xmm2[7]
-; AVX512VL-NEXT:    vpbroadcastd %xmm1, %xmm1
-; AVX512VL-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
-; AVX512VL-NEXT:    vpsrlvd %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512VL-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; X32-SSE-LABEL: splatvar_shift_v4i16:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [65535,0,65535,0,65535,0,65535,0]
-; X32-SSE-NEXT:    pand %xmm2, %xmm0
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,0,0]
-; X32-SSE-NEXT:    pand %xmm2, %xmm3
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm3[2,3,3,3,4,5,6,7]
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm2
-; X32-SSE-NEXT:    psrld %xmm1, %xmm2
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm4 = xmm3[0,1,1,1,4,5,6,7]
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
-; X32-SSE-NEXT:    psrld %xmm4, %xmm1
-; X32-SSE-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,3,0,1]
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm3 = xmm2[2,3,3,3,4,5,6,7]
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm4
-; X32-SSE-NEXT:    psrld %xmm3, %xmm4
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,1,1,1,4,5,6,7]
-; X32-SSE-NEXT:    psrld %xmm2, %xmm0
-; X32-SSE-NEXT:    punpckhqdq {{.*#+}} xmm0 = xmm0[1],xmm4[1]
-; X32-SSE-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,3],xmm0[0,3]
-; X32-SSE-NEXT:    movaps %xmm1, %xmm0
+; X32-SSE-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1]
+; X32-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X32-SSE-NEXT:    psrlw %xmm1, %xmm0
 ; X32-SSE-NEXT:    retl
   %splat = shufflevector <4 x i16> %b, <4 x i16> undef, <4 x i32> zeroinitializer
   %shift = lshr <4 x i16> %a, %splat
@@ -1015,302 +1024,168 @@ define <4 x i16> @splatvar_shift_v4i16(<4 x i16> %a, <4 x i16> %b) nounwind {
 define <2 x i16> @splatvar_shift_v2i16(<2 x i16> %a, <2 x i16> %b) nounwind {
 ; SSE2-LABEL: splatvar_shift_v2i16:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [65535,0,0,0,65535,0,0,0]
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    psrlq %xmm1, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
-; SSE2-NEXT:    psrlq %xmm1, %xmm0
-; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm2[0],xmm0[1]
+; SSE2-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1]
+; SSE2-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; SSE2-NEXT:    psrlw %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: splatvar_shift_v2i16:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pxor %xmm2, %xmm2
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3],xmm0[4],xmm2[5,6,7]
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
-; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3],xmm1[4],xmm2[5,6,7]
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    psrlq %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
-; SSE41-NEXT:    psrlq %xmm1, %xmm0
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm2[0,1,2,3],xmm0[4,5,6,7]
+; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; SSE41-NEXT:    psrlw %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: splatvar_shift_v2i16:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3],xmm0[4],xmm2[5,6,7]
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3],xmm1[4],xmm2[5,6,7]
-; AVX1-NEXT:    vpsrlq %xmm1, %xmm0, %xmm2
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
-; AVX1-NEXT:    vpsrlq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm2[0,1,2,3],xmm0[4,5,6,7]
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: splatvar_shift_v2i16:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3],xmm0[4],xmm2[5,6,7]
-; AVX2-NEXT:    vpbroadcastq %xmm1, %xmm1
-; AVX2-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3],xmm1[4],xmm2[5,6,7]
-; AVX2-NEXT:    vpsrlvq %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    retq
-;
-; XOPAVX1-LABEL: splatvar_shift_v2i16:
-; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; XOPAVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3],xmm0[4],xmm2[5,6,7]
-; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
-; XOPAVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3],xmm1[4],xmm2[5,6,7]
-; XOPAVX1-NEXT:    vpsubq %xmm1, %xmm2, %xmm1
-; XOPAVX1-NEXT:    vpshlq %xmm1, %xmm0, %xmm0
-; XOPAVX1-NEXT:    retq
+; AVX-LABEL: splatvar_shift_v2i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
 ;
-; XOPAVX2-LABEL: splatvar_shift_v2i16:
-; XOPAVX2:       # %bb.0:
-; XOPAVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; XOPAVX2-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3],xmm0[4],xmm2[5,6,7]
-; XOPAVX2-NEXT:    vpbroadcastq %xmm1, %xmm1
-; XOPAVX2-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3],xmm1[4],xmm2[5,6,7]
-; XOPAVX2-NEXT:    vpsrlvq %xmm1, %xmm0, %xmm0
-; XOPAVX2-NEXT:    retq
+; XOP-LABEL: splatvar_shift_v2i16:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; XOP-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
+; XOP-NEXT:    retq
 ;
 ; AVX512-LABEL: splatvar_shift_v2i16:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3],xmm0[4],xmm2[5,6,7]
-; AVX512-NEXT:    vpbroadcastq %xmm1, %xmm1
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3],xmm1[4],xmm2[5,6,7]
-; AVX512-NEXT:    vpsrlvq %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatvar_shift_v2i16:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VL-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3],xmm0[4],xmm2[5,6,7]
-; AVX512VL-NEXT:    vpbroadcastq %xmm1, %xmm1
-; AVX512VL-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3],xmm1[4],xmm2[5,6,7]
-; AVX512VL-NEXT:    vpsrlvq %xmm1, %xmm0, %xmm0
-; AVX512VL-NEXT:    retq
-;
-; X32-SSE-LABEL: splatvar_shift_v2i16:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [65535,0,0,0,65535,0,0,0]
-; X32-SSE-NEXT:    pand %xmm2, %xmm0
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
-; X32-SSE-NEXT:    pand %xmm2, %xmm1
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm2
-; X32-SSE-NEXT:    psrlq %xmm1, %xmm2
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
-; X32-SSE-NEXT:    psrlq %xmm1, %xmm0
-; X32-SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm2[0],xmm0[1]
-; X32-SSE-NEXT:    retl
-  %splat = shufflevector <2 x i16> %b, <2 x i16> undef, <2 x i32> zeroinitializer
-  %shift = lshr <2 x i16> %a, %splat
-  ret <2 x i16> %shift
-}
-
-define <8 x i8> @splatvar_shift_v8i8(<8 x i8> %a, <8 x i8> %b) nounwind {
-; SSE2-LABEL: splatvar_shift_v8i8:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm2
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm1[0,0,2,3,4,5,6,7]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,0,0]
-; SSE2-NEXT:    psllw $12, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm0
-; SSE2-NEXT:    psraw $15, %xmm0
-; SSE2-NEXT:    pandn %xmm2, %xmm0
-; SSE2-NEXT:    paddw %xmm1, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm2
-; SSE2-NEXT:    psraw $15, %xmm2
-; SSE2-NEXT:    movdqa %xmm2, %xmm3
-; SSE2-NEXT:    pandn %xmm0, %xmm3
-; SSE2-NEXT:    psrlw $4, %xmm0
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    por %xmm3, %xmm0
-; SSE2-NEXT:    paddw %xmm1, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm2
-; SSE2-NEXT:    psraw $15, %xmm2
-; SSE2-NEXT:    movdqa %xmm2, %xmm3
-; SSE2-NEXT:    pandn %xmm0, %xmm3
-; SSE2-NEXT:    psrlw $2, %xmm0
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    por %xmm3, %xmm0
-; SSE2-NEXT:    paddw %xmm1, %xmm1
-; SSE2-NEXT:    psraw $15, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm2
-; SSE2-NEXT:    pandn %xmm0, %xmm2
-; SSE2-NEXT:    psrlw $1, %xmm0
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512VL-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT:    retq
+;
+; X32-SSE-LABEL: splatvar_shift_v2i16:
+; X32-SSE:       # %bb.0:
+; X32-SSE-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1]
+; X32-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X32-SSE-NEXT:    psrlw %xmm1, %xmm0
+; X32-SSE-NEXT:    retl
+  %splat = shufflevector <2 x i16> %b, <2 x i16> undef, <2 x i32> zeroinitializer
+  %shift = lshr <2 x i16> %a, %splat
+  ret <2 x i16> %shift
+}
+
+define <8 x i8> @splatvar_shift_v8i8(<8 x i8> %a, <8 x i8> %b) nounwind {
+; SSE2-LABEL: splatvar_shift_v8i8:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0]
+; SSE2-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; SSE2-NEXT:    psrlw %xmm1, %xmm0
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm2
+; SSE2-NEXT:    psrlw %xmm1, %xmm2
+; SSE2-NEXT:    psrlw $8, %xmm2
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,0,2,3,4,5,6,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; SSE2-NEXT:    pand %xmm1, %xmm0
-; SSE2-NEXT:    por %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: splatvar_shift_v8i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa %xmm1, %xmm2
-; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    pand {{.*}}(%rip), %xmm1
-; SSE41-NEXT:    pshufb {{.*#+}} xmm2 = xmm2[0],zero,xmm2[0],zero,xmm2[0],zero,xmm2[0],zero,xmm2[0],zero,xmm2[0],zero,xmm2[0],zero,xmm2[0],zero
-; SSE41-NEXT:    movdqa %xmm2, %xmm0
-; SSE41-NEXT:    psllw $12, %xmm0
-; SSE41-NEXT:    psllw $4, %xmm2
-; SSE41-NEXT:    por %xmm0, %xmm2
-; SSE41-NEXT:    movdqa %xmm2, %xmm3
-; SSE41-NEXT:    paddw %xmm2, %xmm3
-; SSE41-NEXT:    pxor %xmm4, %xmm4
-; SSE41-NEXT:    movdqa %xmm2, %xmm0
-; SSE41-NEXT:    pblendvb %xmm0, %xmm4, %xmm1
-; SSE41-NEXT:    movdqa %xmm1, %xmm2
-; SSE41-NEXT:    psrlw $4, %xmm2
-; SSE41-NEXT:    movdqa %xmm3, %xmm0
-; SSE41-NEXT:    pblendvb %xmm0, %xmm2, %xmm1
-; SSE41-NEXT:    movdqa %xmm1, %xmm2
-; SSE41-NEXT:    psrlw $2, %xmm2
-; SSE41-NEXT:    paddw %xmm3, %xmm3
-; SSE41-NEXT:    movdqa %xmm3, %xmm0
-; SSE41-NEXT:    pblendvb %xmm0, %xmm2, %xmm1
-; SSE41-NEXT:    movdqa %xmm1, %xmm2
-; SSE41-NEXT:    psrlw $1, %xmm2
-; SSE41-NEXT:    paddw %xmm3, %xmm3
-; SSE41-NEXT:    movdqa %xmm3, %xmm0
-; SSE41-NEXT:    pblendvb %xmm0, %xmm2, %xmm1
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; SSE41-NEXT:    psrlw %xmm1, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm2, %xmm2
+; SSE41-NEXT:    psrlw %xmm1, %xmm2
+; SSE41-NEXT:    pshufb {{.*#+}} xmm2 = xmm2[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; SSE41-NEXT:    pand %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: splatvar_shift_v8i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX1-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero
-; AVX1-NEXT:    vpsllw $12, %xmm1, %xmm2
-; AVX1-NEXT:    vpsllw $4, %xmm1, %xmm1
-; AVX1-NEXT:    vpor %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpaddw %xmm1, %xmm1, %xmm2
-; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm1
-; AVX1-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrlw $2, %xmm0, %xmm1
-; AVX1-NEXT:    vpaddw %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrlw $1, %xmm0, %xmm1
-; AVX1-NEXT:    vpaddw %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX1-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpsrlw %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: splatvar_shift_v8i8:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpbroadcastw {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
-; AVX2-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero
-; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
-; AVX2-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
-; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX2-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpsrlw %xmm1, %xmm2, %xmm1
+; AVX2-NEXT:    vpsrlw $8, %xmm1, %xmm1
+; AVX2-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: splatvar_shift_v8i8:
 ; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; XOPAVX1-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero
+; XOPAVX1-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,0,0,0,4,5,6,7]
 ; XOPAVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; XOPAVX1-NEXT:    vpsubw %xmm1, %xmm2, %xmm1
-; XOPAVX1-NEXT:    vpshlw %xmm1, %xmm0, %xmm0
+; XOPAVX1-NEXT:    vpsubb %xmm1, %xmm2, %xmm1
+; XOPAVX1-NEXT:    vpshlb %xmm1, %xmm0, %xmm0
 ; XOPAVX1-NEXT:    retq
 ;
 ; XOPAVX2-LABEL: splatvar_shift_v8i8:
 ; XOPAVX2:       # %bb.0:
-; XOPAVX2-NEXT:    vpbroadcastw {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
-; XOPAVX2-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; XOPAVX2-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero
+; XOPAVX2-NEXT:    vpbroadcastb %xmm1, %xmm1
 ; XOPAVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; XOPAVX2-NEXT:    vpsubw %xmm1, %xmm2, %xmm1
-; XOPAVX2-NEXT:    vpshlw %xmm1, %xmm0, %xmm0
+; XOPAVX2-NEXT:    vpsubb %xmm1, %xmm2, %xmm1
+; XOPAVX2-NEXT:    vpshlb %xmm1, %xmm0, %xmm0
 ; XOPAVX2-NEXT:    retq
 ;
 ; AVX512DQ-LABEL: splatvar_shift_v8i8:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vpbroadcastw {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
-; AVX512DQ-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero
-; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
-; AVX512DQ-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpmovdw %zmm0, %ymm0
-; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512DQ-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512DQ-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; AVX512DQ-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; AVX512DQ-NEXT:    vpsrlvd %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpmovdb %zmm0, %xmm0
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatvar_shift_v8i8:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
-; AVX512BW-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero
+; AVX512BW-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
 ; AVX512BW-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512DQVL-LABEL: splatvar_shift_v8i8:
 ; AVX512DQVL:       # %bb.0:
-; AVX512DQVL-NEXT:    vpbroadcastw {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
-; AVX512DQVL-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; AVX512DQVL-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero
-; AVX512DQVL-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX512DQVL-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
-; AVX512DQVL-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm0
-; AVX512DQVL-NEXT:    vpmovdw %ymm0, %xmm0
+; AVX512DQVL-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512DQVL-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; AVX512DQVL-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; AVX512DQVL-NEXT:    vpsrlvd %zmm1, %zmm0, %zmm0
+; AVX512DQVL-NEXT:    vpmovdb %zmm0, %xmm0
 ; AVX512DQVL-NEXT:    vzeroupper
 ; AVX512DQVL-NEXT:    retq
 ;
 ; AVX512BWVL-LABEL: splatvar_shift_v8i8:
 ; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vpbroadcastw {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
-; AVX512BWVL-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; AVX512BWVL-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero
-; AVX512BWVL-NEXT:    vpsrlvw %xmm1, %xmm0, %xmm0
+; AVX512BWVL-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512BWVL-NEXT:    vpsrlvw %ymm1, %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm0
+; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
 ;
 ; X32-SSE-LABEL: splatvar_shift_v8i8:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm2
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm2
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm1[0,0,2,3,4,5,6,7]
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,0,0]
-; X32-SSE-NEXT:    psllw $12, %xmm1
-; X32-SSE-NEXT:    movdqa %xmm1, %xmm0
-; X32-SSE-NEXT:    psraw $15, %xmm0
-; X32-SSE-NEXT:    pandn %xmm2, %xmm0
-; X32-SSE-NEXT:    paddw %xmm1, %xmm1
-; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
-; X32-SSE-NEXT:    psraw $15, %xmm2
-; X32-SSE-NEXT:    movdqa %xmm2, %xmm3
-; X32-SSE-NEXT:    pandn %xmm0, %xmm3
-; X32-SSE-NEXT:    psrlw $4, %xmm0
-; X32-SSE-NEXT:    pand %xmm2, %xmm0
-; X32-SSE-NEXT:    por %xmm3, %xmm0
-; X32-SSE-NEXT:    paddw %xmm1, %xmm1
-; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
-; X32-SSE-NEXT:    psraw $15, %xmm2
-; X32-SSE-NEXT:    movdqa %xmm2, %xmm3
-; X32-SSE-NEXT:    pandn %xmm0, %xmm3
-; X32-SSE-NEXT:    psrlw $2, %xmm0
-; X32-SSE-NEXT:    pand %xmm2, %xmm0
-; X32-SSE-NEXT:    por %xmm3, %xmm0
-; X32-SSE-NEXT:    paddw %xmm1, %xmm1
-; X32-SSE-NEXT:    psraw $15, %xmm1
-; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
-; X32-SSE-NEXT:    pandn %xmm0, %xmm2
-; X32-SSE-NEXT:    psrlw $1, %xmm0
+; X32-SSE-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0]
+; X32-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X32-SSE-NEXT:    psrlw %xmm1, %xmm0
+; X32-SSE-NEXT:    pcmpeqd %xmm2, %xmm2
+; X32-SSE-NEXT:    psrlw %xmm1, %xmm2
+; X32-SSE-NEXT:    psrlw $8, %xmm2
+; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,0,2,3,4,5,6,7]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; X32-SSE-NEXT:    pand %xmm1, %xmm0
-; X32-SSE-NEXT:    por %xmm2, %xmm0
 ; X32-SSE-NEXT:    retl
   %splat = shufflevector <8 x i8> %b, <8 x i8> undef, <8 x i32> zeroinitializer
   %shift = lshr <8 x i8> %a, %splat
@@ -1320,132 +1195,119 @@ define <8 x i8> @splatvar_shift_v8i8(<8 x i8> %a, <8 x i8> %b) nounwind {
 define <4 x i8> @splatvar_shift_v4i8(<4 x i8> %a, <4 x i8> %b) nounwind {
 ; SSE2-LABEL: splatvar_shift_v4i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0]
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,0,0]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm3[2,3,3,3,4,5,6,7]
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    psrld %xmm1, %xmm2
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm4 = xmm3[0,1,1,1,4,5,6,7]
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    psrld %xmm4, %xmm1
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,3,0,1]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm2[2,3,3,3,4,5,6,7]
-; SSE2-NEXT:    movdqa %xmm0, %xmm4
-; SSE2-NEXT:    psrld %xmm3, %xmm4
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,1,1,1,4,5,6,7]
-; SSE2-NEXT:    psrld %xmm2, %xmm0
-; SSE2-NEXT:    punpckhqdq {{.*#+}} xmm0 = xmm0[1],xmm4[1]
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,3],xmm0[0,3]
-; SSE2-NEXT:    movaps %xmm1, %xmm0
+; SSE2-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0]
+; SSE2-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; SSE2-NEXT:    psrlw %xmm1, %xmm0
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm2
+; SSE2-NEXT:    psrlw %xmm1, %xmm2
+; SSE2-NEXT:    psrlw $8, %xmm2
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,0,2,3,4,5,6,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
+; SSE2-NEXT:    pand %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: splatvar_shift_v4i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pand {{.*}}(%rip), %xmm0
-; SSE41-NEXT:    pshufb {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[0],zero,zero,zero,xmm1[0],zero,zero,zero,xmm1[0],zero,zero,zero
-; SSE41-NEXT:    pshuflw {{.*#+}} xmm2 = xmm1[2,3,3,3,4,5,6,7]
-; SSE41-NEXT:    movdqa %xmm0, %xmm3
-; SSE41-NEXT:    psrld %xmm2, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
-; SSE41-NEXT:    pshuflw {{.*#+}} xmm4 = xmm2[2,3,3,3,4,5,6,7]
-; SSE41-NEXT:    movdqa %xmm0, %xmm5
-; SSE41-NEXT:    psrld %xmm4, %xmm5
-; SSE41-NEXT:    pblendw {{.*#+}} xmm5 = xmm3[0,1,2,3],xmm5[4,5,6,7]
-; SSE41-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,1,1,1,4,5,6,7]
-; SSE41-NEXT:    movdqa %xmm0, %xmm3
-; SSE41-NEXT:    psrld %xmm1, %xmm3
-; SSE41-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,1,1,1,4,5,6,7]
-; SSE41-NEXT:    psrld %xmm1, %xmm0
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm3[0,1,2,3],xmm0[4,5,6,7]
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm5[2,3],xmm0[4,5],xmm5[6,7]
+; SSE41-NEXT:    pmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; SSE41-NEXT:    psrlw %xmm1, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm2, %xmm2
+; SSE41-NEXT:    psrlw %xmm1, %xmm2
+; SSE41-NEXT:    pshufb {{.*#+}} xmm2 = xmm2[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; SSE41-NEXT:    pand %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: splatvar_shift_v4i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm2 = [3.57331108E-43,3.57331108E-43,3.57331108E-43,3.57331108E-43]
-; AVX1-NEXT:    vandps %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[0],zero,zero,zero,xmm1[0],zero,zero,zero,xmm1[0],zero,zero,zero
-; AVX1-NEXT:    vpsrldq {{.*#+}} xmm2 = xmm1[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX1-NEXT:    vpsrld %xmm2, %xmm0, %xmm2
-; AVX1-NEXT:    vpsrlq $32, %xmm1, %xmm3
-; AVX1-NEXT:    vpsrld %xmm3, %xmm0, %xmm3
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
-; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm3 = xmm1[2],xmm3[2],xmm1[3],xmm3[3]
-; AVX1-NEXT:    vpsrld %xmm3, %xmm0, %xmm3
-; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
-; AVX1-NEXT:    vpsrld %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm3[4,5,6,7]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
+; AVX1-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX1-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpsrlw %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: splatvar_shift_v4i8:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [255,255,255,255]
-; AVX2-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[0],zero,zero,zero,xmm1[0],zero,zero,zero,xmm1[0],zero,zero,zero
-; AVX2-NEXT:    vpsrlvd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX2-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpsrlw %xmm1, %xmm2, %xmm1
+; AVX2-NEXT:    vpsrlw $8, %xmm1, %xmm1
+; AVX2-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: splatvar_shift_v4i8:
 ; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vbroadcastss {{.*#+}} xmm2 = [3.57331108E-43,3.57331108E-43,3.57331108E-43,3.57331108E-43]
-; XOPAVX1-NEXT:    vandps %xmm2, %xmm0, %xmm0
-; XOPAVX1-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[0],zero,zero,zero,xmm1[0],zero,zero,zero,xmm1[0],zero,zero,zero
-; XOPAVX1-NEXT:    vxorps %xmm2, %xmm2, %xmm2
-; XOPAVX1-NEXT:    vpsubd %xmm1, %xmm2, %xmm1
-; XOPAVX1-NEXT:    vpshld %xmm1, %xmm0, %xmm0
+; XOPAVX1-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,0,2,3,4,5,6,7]
+; XOPAVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; XOPAVX1-NEXT:    vpsubb %xmm1, %xmm2, %xmm1
+; XOPAVX1-NEXT:    vpshlb %xmm1, %xmm0, %xmm0
 ; XOPAVX1-NEXT:    retq
 ;
 ; XOPAVX2-LABEL: splatvar_shift_v4i8:
 ; XOPAVX2:       # %bb.0:
-; XOPAVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [255,255,255,255]
-; XOPAVX2-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; XOPAVX2-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[0],zero,zero,zero,xmm1[0],zero,zero,zero,xmm1[0],zero,zero,zero
-; XOPAVX2-NEXT:    vpsrlvd %xmm1, %xmm0, %xmm0
+; XOPAVX2-NEXT:    vpbroadcastb %xmm1, %xmm1
+; XOPAVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; XOPAVX2-NEXT:    vpsubb %xmm1, %xmm2, %xmm1
+; XOPAVX2-NEXT:    vpshlb %xmm1, %xmm0, %xmm0
 ; XOPAVX2-NEXT:    retq
 ;
-; AVX512-LABEL: splatvar_shift_v4i8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [255,255,255,255]
-; AVX512-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[0],zero,zero,zero,xmm1[0],zero,zero,zero,xmm1[0],zero,zero,zero
-; AVX512-NEXT:    vpsrlvd %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    retq
+; AVX512DQ-LABEL: splatvar_shift_v4i8:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512DQ-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; AVX512DQ-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; AVX512DQ-NEXT:    vpsrlvd %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    retq
 ;
-; AVX512VL-LABEL: splatvar_shift_v4i8:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [255,255,255,255]
-; AVX512VL-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[0],zero,zero,zero,xmm1[0],zero,zero,zero,xmm1[0],zero,zero,zero
-; AVX512VL-NEXT:    vpsrlvd %xmm1, %xmm0, %xmm0
-; AVX512VL-NEXT:    retq
+; AVX512BW-LABEL: splatvar_shift_v4i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512BW-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512DQVL-LABEL: splatvar_shift_v4i8:
+; AVX512DQVL:       # %bb.0:
+; AVX512DQVL-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512DQVL-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; AVX512DQVL-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; AVX512DQVL-NEXT:    vpsrlvd %zmm1, %zmm0, %zmm0
+; AVX512DQVL-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512DQVL-NEXT:    vzeroupper
+; AVX512DQVL-NEXT:    retq
+;
+; AVX512BWVL-LABEL: splatvar_shift_v4i8:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512BWVL-NEXT:    vpsrlvw %ymm1, %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm0
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
 ;
 ; X32-SSE-LABEL: splatvar_shift_v4i8:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0]
-; X32-SSE-NEXT:    pand %xmm2, %xmm0
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,0,0]
-; X32-SSE-NEXT:    pand %xmm2, %xmm3
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm3[2,3,3,3,4,5,6,7]
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm2
-; X32-SSE-NEXT:    psrld %xmm1, %xmm2
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm4 = xmm3[0,1,1,1,4,5,6,7]
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
-; X32-SSE-NEXT:    psrld %xmm4, %xmm1
-; X32-SSE-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,3,0,1]
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm3 = xmm2[2,3,3,3,4,5,6,7]
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm4
-; X32-SSE-NEXT:    psrld %xmm3, %xmm4
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,1,1,1,4,5,6,7]
-; X32-SSE-NEXT:    psrld %xmm2, %xmm0
-; X32-SSE-NEXT:    punpckhqdq {{.*#+}} xmm0 = xmm0[1],xmm4[1]
-; X32-SSE-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,3],xmm0[0,3]
-; X32-SSE-NEXT:    movaps %xmm1, %xmm0
+; X32-SSE-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0]
+; X32-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X32-SSE-NEXT:    psrlw %xmm1, %xmm0
+; X32-SSE-NEXT:    pcmpeqd %xmm2, %xmm2
+; X32-SSE-NEXT:    psrlw %xmm1, %xmm2
+; X32-SSE-NEXT:    psrlw $8, %xmm2
+; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,0,2,3,4,5,6,7]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
+; X32-SSE-NEXT:    pand %xmm1, %xmm0
 ; X32-SSE-NEXT:    retl
   %splat = shufflevector <4 x i8> %b, <4 x i8> undef, <4 x i32> zeroinitializer
   %shift = lshr <4 x i8> %a, %splat
@@ -1455,94 +1317,110 @@ define <4 x i8> @splatvar_shift_v4i8(<4 x i8> %a, <4 x i8> %b) nounwind {
 define <2 x i8> @splatvar_shift_v2i8(<2 x i8> %a, <2 x i8> %b) nounwind {
 ; SSE2-LABEL: splatvar_shift_v2i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0]
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    psrlq %xmm1, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
-; SSE2-NEXT:    psrlq %xmm1, %xmm0
-; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm2[0],xmm0[1]
+; SSE2-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0]
+; SSE2-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; SSE2-NEXT:    psrlw %xmm1, %xmm0
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm2
+; SSE2-NEXT:    psrlw %xmm1, %xmm2
+; SSE2-NEXT:    psrlw $8, %xmm2
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,0,2,3,4,5,6,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
+; SSE2-NEXT:    pand %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: splatvar_shift_v2i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pand {{.*}}(%rip), %xmm0
-; SSE41-NEXT:    pshufb {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[0],zero,zero,zero,zero,zero,zero,zero
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    psrlq %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
-; SSE41-NEXT:    psrlq %xmm1, %xmm0
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm2[0,1,2,3],xmm0[4,5,6,7]
+; SSE41-NEXT:    pmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; SSE41-NEXT:    psrlw %xmm1, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm2, %xmm2
+; SSE41-NEXT:    psrlw %xmm1, %xmm2
+; SSE41-NEXT:    pshufb {{.*#+}} xmm2 = xmm2[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; SSE41-NEXT:    pand %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: splatvar_shift_v2i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovddup {{.*#+}} xmm2 = [1.2598673968951787E-321,1.2598673968951787E-321]
-; AVX1-NEXT:    # xmm2 = mem[0,0]
-; AVX1-NEXT:    vandps %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[0],zero,zero,zero,zero,zero,zero,zero
-; AVX1-NEXT:    vpsrlq %xmm1, %xmm0, %xmm2
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
-; AVX1-NEXT:    vpsrlq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm2[0,1,2,3],xmm0[4,5,6,7]
+; AVX1-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX1-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpsrlw %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: splatvar_shift_v2i8:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpbroadcastq {{.*#+}} xmm2 = [255,255]
-; AVX2-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[0],zero,zero,zero,zero,zero,zero,zero
-; AVX2-NEXT:    vpsrlvq %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX2-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpsrlw %xmm1, %xmm2, %xmm1
+; AVX2-NEXT:    vpsrlw $8, %xmm1, %xmm1
+; AVX2-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    retq
 ;
-; XOPAVX1-LABEL: splatvar_shift_v2i8:
-; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vmovddup {{.*#+}} xmm2 = [1.2598673968951787E-321,1.2598673968951787E-321]
-; XOPAVX1-NEXT:    # xmm2 = mem[0,0]
-; XOPAVX1-NEXT:    vandps %xmm2, %xmm0, %xmm0
-; XOPAVX1-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[0],zero,zero,zero,zero,zero,zero,zero
-; XOPAVX1-NEXT:    vxorps %xmm2, %xmm2, %xmm2
-; XOPAVX1-NEXT:    vpsubq %xmm1, %xmm2, %xmm1
-; XOPAVX1-NEXT:    vpshlq %xmm1, %xmm0, %xmm0
-; XOPAVX1-NEXT:    retq
+; XOP-LABEL: splatvar_shift_v2i8:
+; XOP:       # %bb.0:
+; XOP-NEXT:    insertq {{.*#+}} xmm1 = xmm1[0,0,2,3,4,5,6,7,u,u,u,u,u,u,u,u]
+; XOP-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; XOP-NEXT:    vpsubb %xmm1, %xmm2, %xmm1
+; XOP-NEXT:    vpshlb %xmm1, %xmm0, %xmm0
+; XOP-NEXT:    retq
 ;
-; XOPAVX2-LABEL: splatvar_shift_v2i8:
-; XOPAVX2:       # %bb.0:
-; XOPAVX2-NEXT:    vpbroadcastq {{.*#+}} xmm2 = [255,255]
-; XOPAVX2-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; XOPAVX2-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[0],zero,zero,zero,zero,zero,zero,zero
-; XOPAVX2-NEXT:    vpsrlvq %xmm1, %xmm0, %xmm0
-; XOPAVX2-NEXT:    retq
+; AVX512DQ-LABEL: splatvar_shift_v2i8:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512DQ-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; AVX512DQ-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; AVX512DQ-NEXT:    vpsrlvd %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    retq
 ;
-; AVX512-LABEL: splatvar_shift_v2i8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpbroadcastq {{.*#+}} xmm2 = [255,255]
-; AVX512-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[0],zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT:    vpsrlvq %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    retq
+; AVX512BW-LABEL: splatvar_shift_v2i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512BW-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
 ;
-; AVX512VL-LABEL: splatvar_shift_v2i8:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpbroadcastq {{.*#+}} xmm2 = [255,255]
-; AVX512VL-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[0],zero,zero,zero,zero,zero,zero,zero
-; AVX512VL-NEXT:    vpsrlvq %xmm1, %xmm0, %xmm0
-; AVX512VL-NEXT:    retq
+; AVX512DQVL-LABEL: splatvar_shift_v2i8:
+; AVX512DQVL:       # %bb.0:
+; AVX512DQVL-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512DQVL-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; AVX512DQVL-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; AVX512DQVL-NEXT:    vpsrlvd %zmm1, %zmm0, %zmm0
+; AVX512DQVL-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512DQVL-NEXT:    vzeroupper
+; AVX512DQVL-NEXT:    retq
+;
+; AVX512BWVL-LABEL: splatvar_shift_v2i8:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512BWVL-NEXT:    vpsrlvw %ymm1, %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm0
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
 ;
 ; X32-SSE-LABEL: splatvar_shift_v2i8:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [255,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0]
-; X32-SSE-NEXT:    pand %xmm2, %xmm0
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
-; X32-SSE-NEXT:    pand %xmm2, %xmm1
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm2
-; X32-SSE-NEXT:    psrlq %xmm1, %xmm2
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
-; X32-SSE-NEXT:    psrlq %xmm1, %xmm0
-; X32-SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm2[0],xmm0[1]
+; X32-SSE-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0]
+; X32-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X32-SSE-NEXT:    psrlw %xmm1, %xmm0
+; X32-SSE-NEXT:    pcmpeqd %xmm2, %xmm2
+; X32-SSE-NEXT:    psrlw %xmm1, %xmm2
+; X32-SSE-NEXT:    psrlw $8, %xmm2
+; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,0,2,3,4,5,6,7]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
+; X32-SSE-NEXT:    pand %xmm1, %xmm0
 ; X32-SSE-NEXT:    retl
   %splat = shufflevector <2 x i8> %b, <2 x i8> undef, <2 x i32> zeroinitializer
   %shift = lshr <2 x i8> %a, %splat
@@ -1556,75 +1434,62 @@ define <2 x i8> @splatvar_shift_v2i8(<2 x i8> %a, <2 x i8> %b) nounwind {
 define <2 x i32> @constant_shift_v2i32(<2 x i32> %a) nounwind {
 ; SSE2-LABEL: constant_shift_v2i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
 ; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    psrlq $4, %xmm1
-; SSE2-NEXT:    psrlq $5, %xmm0
-; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; SSE2-NEXT:    psrld $4, %xmm1
+; SSE2-NEXT:    psrld $5, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: constant_shift_v2i32:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pxor %xmm1, %xmm1
-; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    psrlq $5, %xmm0
-; SSE41-NEXT:    psrlq $4, %xmm1
-; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm0[4,5,6,7]
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    psrld $5, %xmm1
+; SSE41-NEXT:    psrld $4, %xmm0
+; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5,6,7]
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: constant_shift_v2i32:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
-; AVX1-NEXT:    vpsrlq $5, %xmm0, %xmm1
-; AVX1-NEXT:    vpsrlq $4, %xmm0, %xmm0
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4,5,6,7]
+; AVX1-NEXT:    vpsrld $5, %xmm0, %xmm1
+; AVX1-NEXT:    vpsrld $4, %xmm0, %xmm0
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5,6,7]
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: constant_shift_v2i32:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
-; AVX2-NEXT:    vpsrlvq {{.*}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT:    vpsrlvd {{.*}}(%rip), %xmm0, %xmm0
 ; AVX2-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: constant_shift_v2i32:
 ; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; XOPAVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
-; XOPAVX1-NEXT:    vpshlq {{.*}}(%rip), %xmm0, %xmm0
+; XOPAVX1-NEXT:    vpshld {{.*}}(%rip), %xmm0, %xmm0
 ; XOPAVX1-NEXT:    retq
 ;
 ; XOPAVX2-LABEL: constant_shift_v2i32:
 ; XOPAVX2:       # %bb.0:
-; XOPAVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; XOPAVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
-; XOPAVX2-NEXT:    vpsrlvq {{.*}}(%rip), %xmm0, %xmm0
+; XOPAVX2-NEXT:    vpsrlvd {{.*}}(%rip), %xmm0, %xmm0
 ; XOPAVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: constant_shift_v2i32:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
-; AVX512-NEXT:    vpsrlvq {{.*}}(%rip), %xmm0, %xmm0
+; AVX512-NEXT:    vpsrlvd {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512-NEXT:    retq
 ;
 ; AVX512VL-LABEL: constant_shift_v2i32:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
-; AVX512VL-NEXT:    vpsrlvq {{.*}}(%rip), %xmm0, %xmm0
+; AVX512VL-NEXT:    vpsrlvd {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; X32-SSE-LABEL: constant_shift_v2i32:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
 ; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
-; X32-SSE-NEXT:    psrlq $4, %xmm1
-; X32-SSE-NEXT:    psrlq $5, %xmm0
-; X32-SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; X32-SSE-NEXT:    psrld $4, %xmm1
+; X32-SSE-NEXT:    psrld $5, %xmm0
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; X32-SSE-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; X32-SSE-NEXT:    movdqa %xmm1, %xmm0
 ; X32-SSE-NEXT:    retl
   %shift = lshr <2 x i32> %a, <i32 4, i32 5>
   ret <2 x i32> %shift
@@ -1633,91 +1498,66 @@ define <2 x i32> @constant_shift_v2i32(<2 x i32> %a) nounwind {
 define <4 x i16> @constant_shift_v4i16(<4 x i16> %a) nounwind {
 ; SSE2-LABEL: constant_shift_v4i16:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    psrld $3, %xmm1
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    psrld $2, %xmm2
-; SSE2-NEXT:    punpckhqdq {{.*#+}} xmm2 = xmm2[1],xmm1[1]
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    psrld $1, %xmm1
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,3],xmm2[0,3]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [0,65535,65535,65535,65535,65535,65535,65535]
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    pandn %xmm0, %xmm2
+; SSE2-NEXT:    pmulhuw {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    pand %xmm1, %xmm0
+; SSE2-NEXT:    por %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: constant_shift_v4i16:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pxor %xmm1, %xmm1
-; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7]
-; SSE41-NEXT:    movdqa %xmm1, %xmm2
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    psrld $2, %xmm0
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4,5,6,7]
-; SSE41-NEXT:    psrld $3, %xmm1
-; SSE41-NEXT:    psrld $1, %xmm2
-; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm1[4,5,6,7]
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = <u,32768,16384,8192,u,u,u,u>
+; SSE41-NEXT:    pmulhuw %xmm0, %xmm1
+; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: constant_shift_v4i16:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7]
-; AVX1-NEXT:    vpsrld $3, %xmm0, %xmm1
-; AVX1-NEXT:    vpsrld $1, %xmm0, %xmm2
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1,2,3],xmm1[4,5,6,7]
-; AVX1-NEXT:    vpsrld $2, %xmm0, %xmm2
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm2[4,5,6,7]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: constant_shift_v4i16:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7]
-; AVX2-NEXT:    vpsrlvd {{.*}}(%rip), %xmm0, %xmm0
-; AVX2-NEXT:    retq
+; AVX-LABEL: constant_shift_v4i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpmulhuw {{.*}}(%rip), %xmm0, %xmm1
+; AVX-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]
+; AVX-NEXT:    retq
 ;
-; XOPAVX1-LABEL: constant_shift_v4i16:
-; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; XOPAVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7]
-; XOPAVX1-NEXT:    vpshld {{.*}}(%rip), %xmm0, %xmm0
-; XOPAVX1-NEXT:    retq
+; XOP-LABEL: constant_shift_v4i16:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vpshlw {{.*}}(%rip), %xmm0, %xmm0
+; XOP-NEXT:    retq
 ;
-; XOPAVX2-LABEL: constant_shift_v4i16:
-; XOPAVX2:       # %bb.0:
-; XOPAVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; XOPAVX2-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7]
-; XOPAVX2-NEXT:    vpsrlvd {{.*}}(%rip), %xmm0, %xmm0
-; XOPAVX2-NEXT:    retq
+; AVX512DQ-LABEL: constant_shift_v4i16:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vpmulhuw {{.*}}(%rip), %xmm0, %xmm1
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]
+; AVX512DQ-NEXT:    retq
 ;
-; AVX512-LABEL: constant_shift_v4i16:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7]
-; AVX512-NEXT:    vpsrlvd {{.*}}(%rip), %xmm0, %xmm0
-; AVX512-NEXT:    retq
+; AVX512BW-LABEL: constant_shift_v4i16:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm1 = <0,1,2,3,u,u,u,u>
+; AVX512BW-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
 ;
-; AVX512VL-LABEL: constant_shift_v4i16:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7]
-; AVX512VL-NEXT:    vpsrlvd {{.*}}(%rip), %xmm0, %xmm0
-; AVX512VL-NEXT:    retq
+; AVX512DQVL-LABEL: constant_shift_v4i16:
+; AVX512DQVL:       # %bb.0:
+; AVX512DQVL-NEXT:    vpmulhuw {{.*}}(%rip), %xmm0, %xmm1
+; AVX512DQVL-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]
+; AVX512DQVL-NEXT:    retq
+;
+; AVX512BWVL-LABEL: constant_shift_v4i16:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpsrlvw {{.*}}(%rip), %xmm0, %xmm0
+; AVX512BWVL-NEXT:    retq
 ;
 ; X32-SSE-LABEL: constant_shift_v4i16:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
-; X32-SSE-NEXT:    psrld $3, %xmm1
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm2
-; X32-SSE-NEXT:    psrld $2, %xmm2
-; X32-SSE-NEXT:    punpckhqdq {{.*#+}} xmm2 = xmm2[1],xmm1[1]
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
-; X32-SSE-NEXT:    psrld $1, %xmm1
-; X32-SSE-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; X32-SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,3],xmm2[0,3]
+; X32-SSE-NEXT:    movdqa {{.*#+}} xmm1 = [0,65535,65535,65535,65535,65535,65535,65535]
+; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
+; X32-SSE-NEXT:    pandn %xmm0, %xmm2
+; X32-SSE-NEXT:    pmulhuw {{\.LCPI.*}}, %xmm0
+; X32-SSE-NEXT:    pand %xmm1, %xmm0
+; X32-SSE-NEXT:    por %xmm2, %xmm0
 ; X32-SSE-NEXT:    retl
   %shift = lshr <4 x i16> %a, <i16 0, i16 1, i16 2, i16 3>
   ret <4 x i16> %shift
@@ -1726,75 +1566,72 @@ define <4 x i16> @constant_shift_v4i16(<4 x i16> %a) nounwind {
 define <2 x i16> @constant_shift_v2i16(<2 x i16> %a) nounwind {
 ; SSE2-LABEL: constant_shift_v2i16:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
 ; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    psrlq $2, %xmm1
-; SSE2-NEXT:    psrlq $3, %xmm0
-; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; SSE2-NEXT:    psrlw $3, %xmm1
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [65535,0,65535,65535,65535,65535,65535,65535]
+; SSE2-NEXT:    psrlw $2, %xmm0
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    pandn %xmm1, %xmm2
+; SSE2-NEXT:    por %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: constant_shift_v2i16:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pxor %xmm1, %xmm1
-; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3],xmm0[4],xmm1[5,6,7]
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    psrlq $3, %xmm0
-; SSE41-NEXT:    psrlq $2, %xmm1
-; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm0[4,5,6,7]
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    psrlw $3, %xmm1
+; SSE41-NEXT:    psrlw $2, %xmm0
+; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3,4,5,6,7]
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: constant_shift_v2i16:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3],xmm0[4],xmm1[5,6,7]
-; AVX1-NEXT:    vpsrlq $3, %xmm0, %xmm1
-; AVX1-NEXT:    vpsrlq $2, %xmm0, %xmm0
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4,5,6,7]
-; AVX1-NEXT:    retq
+; AVX-LABEL: constant_shift_v2i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpsrlw $3, %xmm0, %xmm1
+; AVX-NEXT:    vpsrlw $2, %xmm0, %xmm0
+; AVX-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3,4,5,6,7]
+; AVX-NEXT:    retq
 ;
-; AVX2-LABEL: constant_shift_v2i16:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3],xmm0[4],xmm1[5,6,7]
-; AVX2-NEXT:    vpsrlvq {{.*}}(%rip), %xmm0, %xmm0
-; AVX2-NEXT:    retq
+; XOP-LABEL: constant_shift_v2i16:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vpshlw {{.*}}(%rip), %xmm0, %xmm0
+; XOP-NEXT:    retq
 ;
-; XOPAVX1-LABEL: constant_shift_v2i16:
-; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; XOPAVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3],xmm0[4],xmm1[5,6,7]
-; XOPAVX1-NEXT:    vpshlq {{.*}}(%rip), %xmm0, %xmm0
-; XOPAVX1-NEXT:    retq
+; AVX512DQ-LABEL: constant_shift_v2i16:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vpsrlw $3, %xmm0, %xmm1
+; AVX512DQ-NEXT:    vpsrlw $2, %xmm0, %xmm0
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3,4,5,6,7]
+; AVX512DQ-NEXT:    retq
 ;
-; XOPAVX2-LABEL: constant_shift_v2i16:
-; XOPAVX2:       # %bb.0:
-; XOPAVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; XOPAVX2-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3],xmm0[4],xmm1[5,6,7]
-; XOPAVX2-NEXT:    vpsrlvq {{.*}}(%rip), %xmm0, %xmm0
-; XOPAVX2-NEXT:    retq
+; AVX512BW-LABEL: constant_shift_v2i16:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm1 = <2,3,u,u,u,u,u,u>
+; AVX512BW-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
 ;
-; AVX512-LABEL: constant_shift_v2i16:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3],xmm0[4],xmm1[5,6,7]
-; AVX512-NEXT:    vpsrlvq {{.*}}(%rip), %xmm0, %xmm0
-; AVX512-NEXT:    retq
+; AVX512DQVL-LABEL: constant_shift_v2i16:
+; AVX512DQVL:       # %bb.0:
+; AVX512DQVL-NEXT:    vpsrlw $3, %xmm0, %xmm1
+; AVX512DQVL-NEXT:    vpsrlw $2, %xmm0, %xmm0
+; AVX512DQVL-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3,4,5,6,7]
+; AVX512DQVL-NEXT:    retq
 ;
-; AVX512VL-LABEL: constant_shift_v2i16:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3],xmm0[4],xmm1[5,6,7]
-; AVX512VL-NEXT:    vpsrlvq {{.*}}(%rip), %xmm0, %xmm0
-; AVX512VL-NEXT:    retq
+; AVX512BWVL-LABEL: constant_shift_v2i16:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpsrlvw {{.*}}(%rip), %xmm0, %xmm0
+; AVX512BWVL-NEXT:    retq
 ;
 ; X32-SSE-LABEL: constant_shift_v2i16:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
 ; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
-; X32-SSE-NEXT:    psrlq $2, %xmm1
-; X32-SSE-NEXT:    psrlq $3, %xmm0
-; X32-SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; X32-SSE-NEXT:    psrlw $3, %xmm1
+; X32-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [65535,0,65535,65535,65535,65535,65535,65535]
+; X32-SSE-NEXT:    psrlw $2, %xmm0
+; X32-SSE-NEXT:    pand %xmm2, %xmm0
+; X32-SSE-NEXT:    pandn %xmm1, %xmm2
+; X32-SSE-NEXT:    por %xmm2, %xmm0
 ; X32-SSE-NEXT:    retl
   %shift = lshr <2 x i16> %a, <i16 2, i16 3>
   ret <2 x i16> %shift
@@ -1803,74 +1640,94 @@ define <2 x i16> @constant_shift_v2i16(<2 x i16> %a) nounwind {
 define <8 x i8> @constant_shift_v8i8(<8 x i8> %a) nounwind {
 ; SSE2-LABEL: constant_shift_v8i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
-; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [0,65535,65535,65535,65535,65535,65535,65535]
-; SSE2-NEXT:    movdqa %xmm1, %xmm2
-; SSE2-NEXT:    pandn %xmm0, %xmm2
-; SSE2-NEXT:    pmulhuw {{.*}}(%rip), %xmm0
-; SSE2-NEXT:    pand %xmm1, %xmm0
-; SSE2-NEXT:    por %xmm2, %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSE2-NEXT:    pmullw {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    psrlw $8, %xmm0
+; SSE2-NEXT:    packuswb %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: constant_shift_v8i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pand {{.*}}(%rip), %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = <u,32768,16384,8192,4096,2048,1024,512>
-; SSE41-NEXT:    pmulhuw %xmm0, %xmm1
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]
+; SSE41-NEXT:    pxor %xmm2, %xmm2
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15]
+; SSE41-NEXT:    pmullw {{.*}}(%rip), %xmm1
+; SSE41-NEXT:    psrlw $8, %xmm1
+; SSE41-NEXT:    packuswb %xmm0, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
-; AVX-LABEL: constant_shift_v8i8:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX-NEXT:    vpmulhuw {{.*}}(%rip), %xmm0, %xmm1
-; AVX-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]
-; AVX-NEXT:    retq
+; AVX1-LABEL: constant_shift_v8i8:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
+; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX1-NEXT:    vpmullw {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm0
+; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: constant_shift_v8i8:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX2-NEXT:    vpmullw {{.*}}(%rip), %ymm0, %ymm0
+; AVX2-NEXT:    vpsrlw $8, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
 ;
 ; XOP-LABEL: constant_shift_v8i8:
 ; XOP:       # %bb.0:
-; XOP-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; XOP-NEXT:    vpshlw {{.*}}(%rip), %xmm0, %xmm0
+; XOP-NEXT:    vpshlb {{.*}}(%rip), %xmm0, %xmm0
 ; XOP-NEXT:    retq
 ;
 ; AVX512DQ-LABEL: constant_shift_v8i8:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX512DQ-NEXT:    vpmulhuw {{.*}}(%rip), %xmm0, %xmm1
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]
+; AVX512DQ-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; AVX512DQ-NEXT:    vpsrlvd {{.*}}(%rip), %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512BW-LABEL: constant_shift_v8i8:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm1 = [0,1,2,3,4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,1,2,3,4,5,6,7,0,0,0,0,0,0,0,0]
+; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
 ; AVX512BW-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512DQVL-LABEL: constant_shift_v8i8:
 ; AVX512DQVL:       # %bb.0:
-; AVX512DQVL-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX512DQVL-NEXT:    vpmulhuw {{.*}}(%rip), %xmm0, %xmm1
-; AVX512DQVL-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]
+; AVX512DQVL-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; AVX512DQVL-NEXT:    vpsrlvd {{.*}}(%rip), %zmm0, %zmm0
+; AVX512DQVL-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512DQVL-NEXT:    vzeroupper
 ; AVX512DQVL-NEXT:    retq
 ;
 ; AVX512BWVL-LABEL: constant_shift_v8i8:
 ; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX512BWVL-NEXT:    vpsrlvw {{.*}}(%rip), %xmm0, %xmm0
+; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512BWVL-NEXT:    vpsrlvw {{.*}}(%rip), %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm0
+; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
 ;
 ; X32-SSE-LABEL: constant_shift_v8i8:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    movdqa {{.*#+}} xmm1 = [0,65535,65535,65535,65535,65535,65535,65535]
-; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
-; X32-SSE-NEXT:    pandn %xmm0, %xmm2
-; X32-SSE-NEXT:    pmulhuw {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    pand %xmm1, %xmm0
-; X32-SSE-NEXT:    por %xmm2, %xmm0
+; X32-SSE-NEXT:    pxor %xmm1, %xmm1
+; X32-SSE-NEXT:    movdqa %xmm0, %xmm2
+; X32-SSE-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
+; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; X32-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
+; X32-SSE-NEXT:    psrlw $8, %xmm0
+; X32-SSE-NEXT:    packuswb %xmm2, %xmm0
 ; X32-SSE-NEXT:    retl
   %shift = lshr <8 x i8> %a, <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7>
   ret <8 x i8> %shift
@@ -1879,85 +1736,94 @@ define <8 x i8> @constant_shift_v8i8(<8 x i8> %a) nounwind {
 define <4 x i8> @constant_shift_v4i8(<4 x i8> %a) nounwind {
 ; SSE2-LABEL: constant_shift_v4i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    psrld $3, %xmm1
+; SSE2-NEXT:    pxor %xmm1, %xmm1
 ; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    psrld $2, %xmm2
-; SSE2-NEXT:    punpckhqdq {{.*#+}} xmm2 = xmm2[1],xmm1[1]
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    psrld $1, %xmm1
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,3],xmm2[0,3]
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSE2-NEXT:    pmullw {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    psrlw $8, %xmm0
+; SSE2-NEXT:    packuswb %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: constant_shift_v4i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pand {{.*}}(%rip), %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    psrld $2, %xmm1
-; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm0[0,1,2,3],xmm1[4,5,6,7]
-; SSE41-NEXT:    psrld $3, %xmm0
-; SSE41-NEXT:    psrld $1, %xmm2
-; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm0[4,5,6,7]
-; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
+; SSE41-NEXT:    pxor %xmm2, %xmm2
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15]
+; SSE41-NEXT:    pmullw {{.*}}(%rip), %xmm1
+; SSE41-NEXT:    psrlw $8, %xmm1
+; SSE41-NEXT:    packuswb %xmm0, %xmm1
 ; SSE41-NEXT:    movdqa %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: constant_shift_v4i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX1-NEXT:    vpsrld $3, %xmm0, %xmm1
-; AVX1-NEXT:    vpsrld $1, %xmm0, %xmm2
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1,2,3],xmm1[4,5,6,7]
-; AVX1-NEXT:    vpsrld $2, %xmm0, %xmm2
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm2[4,5,6,7]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
+; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
+; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX1-NEXT:    vpmullw {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm0
+; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: constant_shift_v4i8:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX2-NEXT:    vpsrlvd {{.*}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX2-NEXT:    vpmullw {{.*}}(%rip), %ymm0, %ymm0
+; AVX2-NEXT:    vpsrlw $8, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
-; XOPAVX1-LABEL: constant_shift_v4i8:
-; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; XOPAVX1-NEXT:    vpshld {{.*}}(%rip), %xmm0, %xmm0
-; XOPAVX1-NEXT:    retq
+; XOP-LABEL: constant_shift_v4i8:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vpshlb {{.*}}(%rip), %xmm0, %xmm0
+; XOP-NEXT:    retq
 ;
-; XOPAVX2-LABEL: constant_shift_v4i8:
-; XOPAVX2:       # %bb.0:
-; XOPAVX2-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; XOPAVX2-NEXT:    vpsrlvd {{.*}}(%rip), %xmm0, %xmm0
-; XOPAVX2-NEXT:    retq
+; AVX512DQ-LABEL: constant_shift_v4i8:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; AVX512DQ-NEXT:    vpsrlvd {{.*}}(%rip), %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    retq
 ;
-; AVX512-LABEL: constant_shift_v4i8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX512-NEXT:    vpsrlvd {{.*}}(%rip), %xmm0, %xmm0
-; AVX512-NEXT:    retq
+; AVX512BW-LABEL: constant_shift_v4i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,1,2,3,0,0,0,0,0,0,0,0,0,0,0,0]
+; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512BW-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
 ;
-; AVX512VL-LABEL: constant_shift_v4i8:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX512VL-NEXT:    vpsrlvd {{.*}}(%rip), %xmm0, %xmm0
-; AVX512VL-NEXT:    retq
+; AVX512DQVL-LABEL: constant_shift_v4i8:
+; AVX512DQVL:       # %bb.0:
+; AVX512DQVL-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; AVX512DQVL-NEXT:    vpsrlvd {{.*}}(%rip), %zmm0, %zmm0
+; AVX512DQVL-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512DQVL-NEXT:    vzeroupper
+; AVX512DQVL-NEXT:    retq
+;
+; AVX512BWVL-LABEL: constant_shift_v4i8:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512BWVL-NEXT:    vpsrlvw {{.*}}(%rip), %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm0
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
 ;
 ; X32-SSE-LABEL: constant_shift_v4i8:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
-; X32-SSE-NEXT:    psrld $3, %xmm1
+; X32-SSE-NEXT:    pxor %xmm1, %xmm1
 ; X32-SSE-NEXT:    movdqa %xmm0, %xmm2
-; X32-SSE-NEXT:    psrld $2, %xmm2
-; X32-SSE-NEXT:    punpckhqdq {{.*#+}} xmm2 = xmm2[1],xmm1[1]
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
-; X32-SSE-NEXT:    psrld $1, %xmm1
-; X32-SSE-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; X32-SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,3],xmm2[0,3]
+; X32-SSE-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
+; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; X32-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
+; X32-SSE-NEXT:    psrlw $8, %xmm0
+; X32-SSE-NEXT:    packuswb %xmm2, %xmm0
 ; X32-SSE-NEXT:    retl
   %shift = lshr <4 x i8> %a, <i8 0, i8 1, i8 2, i8 3>
   ret <4 x i8> %shift
@@ -1966,67 +1832,94 @@ define <4 x i8> @constant_shift_v4i8(<4 x i8> %a) nounwind {
 define <2 x i8> @constant_shift_v2i8(<2 x i8> %a) nounwind {
 ; SSE2-LABEL: constant_shift_v2i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    psrlq $2, %xmm1
-; SSE2-NEXT:    psrlq $3, %xmm0
-; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSE2-NEXT:    pmullw {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    psrlw $8, %xmm0
+; SSE2-NEXT:    packuswb %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: constant_shift_v2i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pand {{.*}}(%rip), %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    psrlq $3, %xmm1
-; SSE41-NEXT:    psrlq $2, %xmm0
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4,5,6,7]
+; SSE41-NEXT:    pxor %xmm2, %xmm2
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15]
+; SSE41-NEXT:    pmullw {{.*}}(%rip), %xmm1
+; SSE41-NEXT:    psrlw $8, %xmm1
+; SSE41-NEXT:    packuswb %xmm0, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: constant_shift_v2i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX1-NEXT:    vpsrlq $3, %xmm0, %xmm1
-; AVX1-NEXT:    vpsrlq $2, %xmm0, %xmm0
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4,5,6,7]
+; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
+; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX1-NEXT:    vpmullw {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm0
+; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: constant_shift_v2i8:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX2-NEXT:    vpsrlvq {{.*}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX2-NEXT:    vpmullw {{.*}}(%rip), %ymm0, %ymm0
+; AVX2-NEXT:    vpsrlw $8, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
-; XOPAVX1-LABEL: constant_shift_v2i8:
-; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; XOPAVX1-NEXT:    vpshlq {{.*}}(%rip), %xmm0, %xmm0
-; XOPAVX1-NEXT:    retq
+; XOP-LABEL: constant_shift_v2i8:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vpshlb {{.*}}(%rip), %xmm0, %xmm0
+; XOP-NEXT:    retq
 ;
-; XOPAVX2-LABEL: constant_shift_v2i8:
-; XOPAVX2:       # %bb.0:
-; XOPAVX2-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; XOPAVX2-NEXT:    vpsrlvq {{.*}}(%rip), %xmm0, %xmm0
-; XOPAVX2-NEXT:    retq
+; AVX512DQ-LABEL: constant_shift_v2i8:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; AVX512DQ-NEXT:    vpsrlvd {{.*}}(%rip), %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    retq
 ;
-; AVX512-LABEL: constant_shift_v2i8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX512-NEXT:    vpsrlvq {{.*}}(%rip), %xmm0, %xmm0
-; AVX512-NEXT:    retq
+; AVX512BW-LABEL: constant_shift_v2i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm1 = [2,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512BW-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
 ;
-; AVX512VL-LABEL: constant_shift_v2i8:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX512VL-NEXT:    vpsrlvq {{.*}}(%rip), %xmm0, %xmm0
-; AVX512VL-NEXT:    retq
+; AVX512DQVL-LABEL: constant_shift_v2i8:
+; AVX512DQVL:       # %bb.0:
+; AVX512DQVL-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; AVX512DQVL-NEXT:    vpsrlvd {{.*}}(%rip), %zmm0, %zmm0
+; AVX512DQVL-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512DQVL-NEXT:    vzeroupper
+; AVX512DQVL-NEXT:    retq
+;
+; AVX512BWVL-LABEL: constant_shift_v2i8:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512BWVL-NEXT:    vpsrlvw {{.*}}(%rip), %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm0
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
 ;
 ; X32-SSE-LABEL: constant_shift_v2i8:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
-; X32-SSE-NEXT:    psrlq $2, %xmm1
-; X32-SSE-NEXT:    psrlq $3, %xmm0
-; X32-SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; X32-SSE-NEXT:    pxor %xmm1, %xmm1
+; X32-SSE-NEXT:    movdqa %xmm0, %xmm2
+; X32-SSE-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
+; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; X32-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
+; X32-SSE-NEXT:    psrlw $8, %xmm0
+; X32-SSE-NEXT:    packuswb %xmm2, %xmm0
 ; X32-SSE-NEXT:    retl
   %shift = lshr <2 x i8> %a, <i8 2, i8 3>
   ret <2 x i8> %shift
@@ -2037,167 +1930,102 @@ define <2 x i8> @constant_shift_v2i8(<2 x i8> %a) nounwind {
 ;
 
 define <2 x i32> @splatconstant_shift_v2i32(<2 x i32> %a) nounwind {
-; SSE2-LABEL: splatconstant_shift_v2i32:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
-; SSE2-NEXT:    psrlq $5, %xmm0
-; SSE2-NEXT:    retq
-;
-; SSE41-LABEL: splatconstant_shift_v2i32:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pxor %xmm1, %xmm1
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
-; SSE41-NEXT:    psrlq $5, %xmm0
-; SSE41-NEXT:    retq
-;
-; AVX1-LABEL: splatconstant_shift_v2i32:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
-; AVX1-NEXT:    vpsrlq $5, %xmm0, %xmm0
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: splatconstant_shift_v2i32:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
-; AVX2-NEXT:    vpsrlq $5, %xmm0, %xmm0
-; AVX2-NEXT:    retq
+; SSE-LABEL: splatconstant_shift_v2i32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    psrld $5, %xmm0
+; SSE-NEXT:    retq
 ;
-; XOPAVX1-LABEL: splatconstant_shift_v2i32:
-; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; XOPAVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
-; XOPAVX1-NEXT:    vpsrlq $5, %xmm0, %xmm0
-; XOPAVX1-NEXT:    retq
+; AVX-LABEL: splatconstant_shift_v2i32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpsrld $5, %xmm0, %xmm0
+; AVX-NEXT:    retq
 ;
-; XOPAVX2-LABEL: splatconstant_shift_v2i32:
-; XOPAVX2:       # %bb.0:
-; XOPAVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; XOPAVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
-; XOPAVX2-NEXT:    vpsrlq $5, %xmm0, %xmm0
-; XOPAVX2-NEXT:    retq
+; XOP-LABEL: splatconstant_shift_v2i32:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vpsrld $5, %xmm0, %xmm0
+; XOP-NEXT:    retq
 ;
 ; AVX512-LABEL: splatconstant_shift_v2i32:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
-; AVX512-NEXT:    vpsrlq $5, %xmm0, %xmm0
+; AVX512-NEXT:    vpsrld $5, %xmm0, %xmm0
 ; AVX512-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatconstant_shift_v2i32:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
-; AVX512VL-NEXT:    vpsrlq $5, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpsrld $5, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; X32-SSE-LABEL: splatconstant_shift_v2i32:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    psrlq $5, %xmm0
+; X32-SSE-NEXT:    psrld $5, %xmm0
 ; X32-SSE-NEXT:    retl
   %shift = lshr <2 x i32> %a, <i32 5, i32 5>
   ret <2 x i32> %shift
 }
 
 define <4 x i16> @splatconstant_shift_v4i16(<4 x i16> %a) nounwind {
-; SSE2-LABEL: splatconstant_shift_v4i16:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
-; SSE2-NEXT:    psrld $3, %xmm0
-; SSE2-NEXT:    retq
-;
-; SSE41-LABEL: splatconstant_shift_v4i16:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pxor %xmm1, %xmm1
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7]
-; SSE41-NEXT:    psrld $3, %xmm0
-; SSE41-NEXT:    retq
+; SSE-LABEL: splatconstant_shift_v4i16:
+; SSE:       # %bb.0:
+; SSE-NEXT:    psrlw $3, %xmm0
+; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: splatconstant_shift_v4i16:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7]
-; AVX-NEXT:    vpsrld $3, %xmm0, %xmm0
+; AVX-NEXT:    vpsrlw $3, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; XOP-LABEL: splatconstant_shift_v4i16:
 ; XOP:       # %bb.0:
-; XOP-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; XOP-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7]
-; XOP-NEXT:    vpsrld $3, %xmm0, %xmm0
+; XOP-NEXT:    vpsrlw $3, %xmm0, %xmm0
 ; XOP-NEXT:    retq
 ;
 ; AVX512-LABEL: splatconstant_shift_v4i16:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7]
-; AVX512-NEXT:    vpsrld $3, %xmm0, %xmm0
+; AVX512-NEXT:    vpsrlw $3, %xmm0, %xmm0
 ; AVX512-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatconstant_shift_v4i16:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7]
-; AVX512VL-NEXT:    vpsrld $3, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpsrlw $3, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; X32-SSE-LABEL: splatconstant_shift_v4i16:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    psrld $3, %xmm0
+; X32-SSE-NEXT:    psrlw $3, %xmm0
 ; X32-SSE-NEXT:    retl
   %shift = lshr <4 x i16> %a, <i16 3, i16 3, i16 3, i16 3>
   ret <4 x i16> %shift
 }
 
 define <2 x i16> @splatconstant_shift_v2i16(<2 x i16> %a) nounwind {
-; SSE2-LABEL: splatconstant_shift_v2i16:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
-; SSE2-NEXT:    psrlq $3, %xmm0
-; SSE2-NEXT:    retq
-;
-; SSE41-LABEL: splatconstant_shift_v2i16:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pxor %xmm1, %xmm1
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3],xmm0[4],xmm1[5,6,7]
-; SSE41-NEXT:    psrlq $3, %xmm0
-; SSE41-NEXT:    retq
+; SSE-LABEL: splatconstant_shift_v2i16:
+; SSE:       # %bb.0:
+; SSE-NEXT:    psrlw $3, %xmm0
+; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: splatconstant_shift_v2i16:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3],xmm0[4],xmm1[5,6,7]
-; AVX-NEXT:    vpsrlq $3, %xmm0, %xmm0
+; AVX-NEXT:    vpsrlw $3, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; XOP-LABEL: splatconstant_shift_v2i16:
 ; XOP:       # %bb.0:
-; XOP-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; XOP-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3],xmm0[4],xmm1[5,6,7]
-; XOP-NEXT:    vpsrlq $3, %xmm0, %xmm0
+; XOP-NEXT:    vpsrlw $3, %xmm0, %xmm0
 ; XOP-NEXT:    retq
 ;
 ; AVX512-LABEL: splatconstant_shift_v2i16:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3],xmm0[4],xmm1[5,6,7]
-; AVX512-NEXT:    vpsrlq $3, %xmm0, %xmm0
+; AVX512-NEXT:    vpsrlw $3, %xmm0, %xmm0
 ; AVX512-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatconstant_shift_v2i16:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3],xmm0[4],xmm1[5,6,7]
-; AVX512VL-NEXT:    vpsrlq $3, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpsrlw $3, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; X32-SSE-LABEL: splatconstant_shift_v2i16:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    psrlq $3, %xmm0
+; X32-SSE-NEXT:    psrlw $3, %xmm0
 ; X32-SSE-NEXT:    retl
   %shift = lshr <2 x i16> %a, <i16 3, i16 3>
   ret <2 x i16> %shift
@@ -2206,38 +2034,37 @@ define <2 x i16> @splatconstant_shift_v2i16(<2 x i16> %a) nounwind {
 define <8 x i8> @splatconstant_shift_v8i8(<8 x i8> %a) nounwind {
 ; SSE-LABEL: splatconstant_shift_v8i8:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pand {{.*}}(%rip), %xmm0
 ; SSE-NEXT:    psrlw $3, %xmm0
+; SSE-NEXT:    pand {{.*}}(%rip), %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: splatconstant_shift_v8i8:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
 ; AVX-NEXT:    vpsrlw $3, %xmm0, %xmm0
+; AVX-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; XOP-LABEL: splatconstant_shift_v8i8:
 ; XOP:       # %bb.0:
-; XOP-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; XOP-NEXT:    vpsrlw $3, %xmm0, %xmm0
+; XOP-NEXT:    vpshlb {{.*}}(%rip), %xmm0, %xmm0
 ; XOP-NEXT:    retq
 ;
 ; AVX512-LABEL: splatconstant_shift_v8i8:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrlw $3, %xmm0, %xmm0
+; AVX512-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatconstant_shift_v8i8:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512VL-NEXT:    vpsrlw $3, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; X32-SSE-LABEL: splatconstant_shift_v8i8:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
 ; X32-SSE-NEXT:    psrlw $3, %xmm0
+; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
 ; X32-SSE-NEXT:    retl
   %shift = lshr <8 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
   ret <8 x i8> %shift
@@ -2246,38 +2073,37 @@ define <8 x i8> @splatconstant_shift_v8i8(<8 x i8> %a) nounwind {
 define <4 x i8> @splatconstant_shift_v4i8(<4 x i8> %a) nounwind {
 ; SSE-LABEL: splatconstant_shift_v4i8:
 ; SSE:       # %bb.0:
+; SSE-NEXT:    psrlw $3, %xmm0
 ; SSE-NEXT:    pand {{.*}}(%rip), %xmm0
-; SSE-NEXT:    psrld $3, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: splatconstant_shift_v4i8:
 ; AVX:       # %bb.0:
+; AVX-NEXT:    vpsrlw $3, %xmm0, %xmm0
 ; AVX-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX-NEXT:    vpsrld $3, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; XOP-LABEL: splatconstant_shift_v4i8:
 ; XOP:       # %bb.0:
-; XOP-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; XOP-NEXT:    vpsrld $3, %xmm0, %xmm0
+; XOP-NEXT:    vpshlb {{.*}}(%rip), %xmm0, %xmm0
 ; XOP-NEXT:    retq
 ;
 ; AVX512-LABEL: splatconstant_shift_v4i8:
 ; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpsrlw $3, %xmm0, %xmm0
 ; AVX512-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX512-NEXT:    vpsrld $3, %xmm0, %xmm0
 ; AVX512-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatconstant_shift_v4i8:
 ; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsrlw $3, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX512VL-NEXT:    vpsrld $3, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; X32-SSE-LABEL: splatconstant_shift_v4i8:
 ; X32-SSE:       # %bb.0:
+; X32-SSE-NEXT:    psrlw $3, %xmm0
 ; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    psrld $3, %xmm0
 ; X32-SSE-NEXT:    retl
   %shift = lshr <4 x i8> %a, <i8 3, i8 3, i8 3, i8 3>
   ret <4 x i8> %shift
@@ -2286,38 +2112,37 @@ define <4 x i8> @splatconstant_shift_v4i8(<4 x i8> %a) nounwind {
 define <2 x i8> @splatconstant_shift_v2i8(<2 x i8> %a) nounwind {
 ; SSE-LABEL: splatconstant_shift_v2i8:
 ; SSE:       # %bb.0:
+; SSE-NEXT:    psrlw $3, %xmm0
 ; SSE-NEXT:    pand {{.*}}(%rip), %xmm0
-; SSE-NEXT:    psrlq $3, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: splatconstant_shift_v2i8:
 ; AVX:       # %bb.0:
+; AVX-NEXT:    vpsrlw $3, %xmm0, %xmm0
 ; AVX-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX-NEXT:    vpsrlq $3, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; XOP-LABEL: splatconstant_shift_v2i8:
 ; XOP:       # %bb.0:
-; XOP-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; XOP-NEXT:    vpsrlq $3, %xmm0, %xmm0
+; XOP-NEXT:    vpshlb {{.*}}(%rip), %xmm0, %xmm0
 ; XOP-NEXT:    retq
 ;
 ; AVX512-LABEL: splatconstant_shift_v2i8:
 ; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpsrlw $3, %xmm0, %xmm0
 ; AVX512-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX512-NEXT:    vpsrlq $3, %xmm0, %xmm0
 ; AVX512-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatconstant_shift_v2i8:
 ; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsrlw $3, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX512VL-NEXT:    vpsrlq $3, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; X32-SSE-LABEL: splatconstant_shift_v2i8:
 ; X32-SSE:       # %bb.0:
+; X32-SSE-NEXT:    psrlw $3, %xmm0
 ; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    psrlq $3, %xmm0
 ; X32-SSE-NEXT:    retl
   %shift = lshr <2 x i8> %a, <i8 3, i8 3>
   ret <2 x i8> %shift
diff --git a/llvm/test/CodeGen/X86/vector-shift-shl-128.ll b/llvm/test/CodeGen/X86/vector-shift-shl-128.ll
index 6088c965c..b9caaa8b4 100644
--- a/llvm/test/CodeGen/X86/vector-shift-shl-128.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-shl-128.ll
@@ -157,24 +157,23 @@ define <4 x i32> @var_shift_v4i32(<4 x i32> %a, <4 x i32> %b) nounwind {
 define <8 x i16> @var_shift_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind {
 ; SSE2-LABEL: var_shift_v8i16:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    movdqa %xmm1, %xmm3
-; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
-; SSE2-NEXT:    pslld $23, %xmm3
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [1065353216,1065353216,1065353216,1065353216]
-; SSE2-NEXT:    paddd %xmm4, %xmm3
-; SSE2-NEXT:    cvttps2dq %xmm3, %xmm3
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm3[0,2,2,3,4,5,6,7]
-; SSE2-NEXT:    pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,6,6,7]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE2-NEXT:    pslld $23, %xmm2
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [1065353216,1065353216,1065353216,1065353216]
+; SSE2-NEXT:    paddd %xmm3, %xmm2
+; SSE2-NEXT:    cvttps2dq %xmm2, %xmm2
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,2,2,3,4,5,6,7]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,6,6,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
 ; SSE2-NEXT:    pslld $23, %xmm1
-; SSE2-NEXT:    paddd %xmm4, %xmm1
+; SSE2-NEXT:    paddd %xmm3, %xmm1
 ; SSE2-NEXT:    cvttps2dq %xmm1, %xmm1
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
 ; SSE2-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,6,6,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
 ; SSE2-NEXT:    pmullw %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
@@ -259,24 +258,23 @@ define <8 x i16> @var_shift_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind {
 ;
 ; X32-SSE-LABEL: var_shift_v8i16:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pxor %xmm2, %xmm2
-; X32-SSE-NEXT:    movdqa %xmm1, %xmm3
-; X32-SSE-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
-; X32-SSE-NEXT:    pslld $23, %xmm3
-; X32-SSE-NEXT:    movdqa {{.*#+}} xmm4 = [1065353216,1065353216,1065353216,1065353216]
-; X32-SSE-NEXT:    paddd %xmm4, %xmm3
-; X32-SSE-NEXT:    cvttps2dq %xmm3, %xmm3
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm3 = xmm3[0,2,2,3,4,5,6,7]
-; X32-SSE-NEXT:    pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,6,6,7]
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
-; X32-SSE-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
+; X32-SSE-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; X32-SSE-NEXT:    pslld $23, %xmm2
+; X32-SSE-NEXT:    movdqa {{.*#+}} xmm3 = [1065353216,1065353216,1065353216,1065353216]
+; X32-SSE-NEXT:    paddd %xmm3, %xmm2
+; X32-SSE-NEXT:    cvttps2dq %xmm2, %xmm2
+; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,2,2,3,4,5,6,7]
+; X32-SSE-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,6,6,7]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; X32-SSE-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
 ; X32-SSE-NEXT:    pslld $23, %xmm1
-; X32-SSE-NEXT:    paddd %xmm4, %xmm1
+; X32-SSE-NEXT:    paddd %xmm3, %xmm1
 ; X32-SSE-NEXT:    cvttps2dq %xmm1, %xmm1
 ; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
 ; X32-SSE-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,6,6,7]
 ; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; X32-SSE-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; X32-SSE-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
 ; X32-SSE-NEXT:    pmullw %xmm1, %xmm0
 ; X32-SSE-NEXT:    retl
   %shift = shl <8 x i16> %a, %b
diff --git a/llvm/test/CodeGen/X86/vector-shift-shl-256.ll b/llvm/test/CodeGen/X86/vector-shift-shl-256.ll
index 273ff325f..8e3aa9543 100644
--- a/llvm/test/CodeGen/X86/vector-shift-shl-256.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-shl-256.ll
@@ -158,26 +158,25 @@ define <16 x i16> @var_shift_v16i16(<16 x i16> %a, <16 x i16> %b) nounwind {
 ; AVX1-LABEL: var_shift_v16i16:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]
-; AVX1-NEXT:    vpslld $23, %xmm4, %xmm4
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm5 = [1065353216,1065353216,1065353216,1065353216]
-; AVX1-NEXT:    vpaddd %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vcvttps2dq %xmm4, %xmm4
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; AVX1-NEXT:    vpslld $23, %xmm3, %xmm3
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [1065353216,1065353216,1065353216,1065353216]
+; AVX1-NEXT:    vpaddd %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vcvttps2dq %xmm3, %xmm3
 ; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
 ; AVX1-NEXT:    vpslld $23, %xmm2, %xmm2
-; AVX1-NEXT:    vpaddd %xmm5, %xmm2, %xmm2
+; AVX1-NEXT:    vpaddd %xmm4, %xmm2, %xmm2
 ; AVX1-NEXT:    vcvttps2dq %xmm2, %xmm2
-; AVX1-NEXT:    vpackusdw %xmm4, %xmm2, %xmm2
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
-; AVX1-NEXT:    vpmullw %xmm2, %xmm4, %xmm2
-; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7]
+; AVX1-NEXT:    vpackusdw %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; AVX1-NEXT:    vpmullw %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
 ; AVX1-NEXT:    vpslld $23, %xmm3, %xmm3
-; AVX1-NEXT:    vpaddd %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vpaddd %xmm4, %xmm3, %xmm3
 ; AVX1-NEXT:    vcvttps2dq %xmm3, %xmm3
 ; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
 ; AVX1-NEXT:    vpslld $23, %xmm1, %xmm1
-; AVX1-NEXT:    vpaddd %xmm5, %xmm1, %xmm1
+; AVX1-NEXT:    vpaddd %xmm4, %xmm1, %xmm1
 ; AVX1-NEXT:    vcvttps2dq %xmm1, %xmm1
 ; AVX1-NEXT:    vpackusdw %xmm3, %xmm1, %xmm1
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
@@ -248,26 +247,25 @@ define <16 x i16> @var_shift_v16i16(<16 x i16> %a, <16 x i16> %b) nounwind {
 ; X32-AVX1-LABEL: var_shift_v16i16:
 ; X32-AVX1:       # %bb.0:
 ; X32-AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; X32-AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; X32-AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]
-; X32-AVX1-NEXT:    vpslld $23, %xmm4, %xmm4
-; X32-AVX1-NEXT:    vmovdqa {{.*#+}} xmm5 = [1065353216,1065353216,1065353216,1065353216]
-; X32-AVX1-NEXT:    vpaddd %xmm5, %xmm4, %xmm4
-; X32-AVX1-NEXT:    vcvttps2dq %xmm4, %xmm4
+; X32-AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; X32-AVX1-NEXT:    vpslld $23, %xmm3, %xmm3
+; X32-AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [1065353216,1065353216,1065353216,1065353216]
+; X32-AVX1-NEXT:    vpaddd %xmm4, %xmm3, %xmm3
+; X32-AVX1-NEXT:    vcvttps2dq %xmm3, %xmm3
 ; X32-AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
 ; X32-AVX1-NEXT:    vpslld $23, %xmm2, %xmm2
-; X32-AVX1-NEXT:    vpaddd %xmm5, %xmm2, %xmm2
+; X32-AVX1-NEXT:    vpaddd %xmm4, %xmm2, %xmm2
 ; X32-AVX1-NEXT:    vcvttps2dq %xmm2, %xmm2
-; X32-AVX1-NEXT:    vpackusdw %xmm4, %xmm2, %xmm2
-; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
-; X32-AVX1-NEXT:    vpmullw %xmm2, %xmm4, %xmm2
-; X32-AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7]
+; X32-AVX1-NEXT:    vpackusdw %xmm3, %xmm2, %xmm2
+; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; X32-AVX1-NEXT:    vpmullw %xmm2, %xmm3, %xmm2
+; X32-AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
 ; X32-AVX1-NEXT:    vpslld $23, %xmm3, %xmm3
-; X32-AVX1-NEXT:    vpaddd %xmm5, %xmm3, %xmm3
+; X32-AVX1-NEXT:    vpaddd %xmm4, %xmm3, %xmm3
 ; X32-AVX1-NEXT:    vcvttps2dq %xmm3, %xmm3
 ; X32-AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
 ; X32-AVX1-NEXT:    vpslld $23, %xmm1, %xmm1
-; X32-AVX1-NEXT:    vpaddd %xmm5, %xmm1, %xmm1
+; X32-AVX1-NEXT:    vpaddd %xmm4, %xmm1, %xmm1
 ; X32-AVX1-NEXT:    vcvttps2dq %xmm1, %xmm1
 ; X32-AVX1-NEXT:    vpackusdw %xmm3, %xmm1, %xmm1
 ; X32-AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
diff --git a/llvm/test/CodeGen/X86/vector-shift-shl-512.ll b/llvm/test/CodeGen/X86/vector-shift-shl-512.ll
index f63e1ab8d..b8af18448 100644
--- a/llvm/test/CodeGen/X86/vector-shift-shl-512.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-shl-512.ll
@@ -27,14 +27,17 @@ define <16 x i32> @var_shift_v16i32(<16 x i32> %a, <16 x i32> %b) nounwind {
 define <32 x i16> @var_shift_v32i16(<32 x i16> %a, <32 x i16> %b) nounwind {
 ; AVX512DQ-LABEL: var_shift_v32i16:
 ; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
+; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} zmm3 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
 ; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
+; AVX512DQ-NEXT:    vpsllvd %zmm3, %zmm2, %zmm2
+; AVX512DQ-NEXT:    vpmovdw %zmm2, %ymm2
+; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
 ; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
-; AVX512DQ-NEXT:    vpsllvd %zmm2, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpsllvd %zmm1, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    vpmovdw %zmm0, %ymm0
-; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
-; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
-; AVX512DQ-NEXT:    vpsllvd %zmm2, %zmm1, %zmm1
-; AVX512DQ-NEXT:    vpmovdw %zmm1, %ymm1
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512BW-LABEL: var_shift_v32i16:
@@ -48,30 +51,33 @@ define <32 x i16> @var_shift_v32i16(<32 x i16> %a, <32 x i16> %b) nounwind {
 define <64 x i8> @var_shift_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
 ; AVX512DQ-LABEL: var_shift_v64i8:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vpsllw $4, %ymm0, %ymm4
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
+; AVX512DQ-NEXT:    vpsllw $4, %ymm3, %ymm4
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm5 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
 ; AVX512DQ-NEXT:    vpand %ymm5, %ymm4, %ymm4
 ; AVX512DQ-NEXT:    vpsllw $5, %ymm2, %ymm2
-; AVX512DQ-NEXT:    vpblendvb %ymm2, %ymm4, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpsllw $2, %ymm0, %ymm4
+; AVX512DQ-NEXT:    vpblendvb %ymm2, %ymm4, %ymm3, %ymm3
+; AVX512DQ-NEXT:    vpsllw $2, %ymm3, %ymm4
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm6 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
 ; AVX512DQ-NEXT:    vpand %ymm6, %ymm4, %ymm4
 ; AVX512DQ-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
-; AVX512DQ-NEXT:    vpblendvb %ymm2, %ymm4, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpaddb %ymm0, %ymm0, %ymm4
+; AVX512DQ-NEXT:    vpblendvb %ymm2, %ymm4, %ymm3, %ymm3
+; AVX512DQ-NEXT:    vpaddb %ymm3, %ymm3, %ymm4
 ; AVX512DQ-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
-; AVX512DQ-NEXT:    vpblendvb %ymm2, %ymm4, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpsllw $4, %ymm1, %ymm2
-; AVX512DQ-NEXT:    vpand %ymm5, %ymm2, %ymm2
-; AVX512DQ-NEXT:    vpsllw $5, %ymm3, %ymm3
-; AVX512DQ-NEXT:    vpblendvb %ymm3, %ymm2, %ymm1, %ymm1
-; AVX512DQ-NEXT:    vpsllw $2, %ymm1, %ymm2
-; AVX512DQ-NEXT:    vpand %ymm6, %ymm2, %ymm2
-; AVX512DQ-NEXT:    vpaddb %ymm3, %ymm3, %ymm3
-; AVX512DQ-NEXT:    vpblendvb %ymm3, %ymm2, %ymm1, %ymm1
-; AVX512DQ-NEXT:    vpaddb %ymm1, %ymm1, %ymm2
-; AVX512DQ-NEXT:    vpaddb %ymm3, %ymm3, %ymm3
-; AVX512DQ-NEXT:    vpblendvb %ymm3, %ymm2, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpblendvb %ymm2, %ymm4, %ymm3, %ymm2
+; AVX512DQ-NEXT:    vpsllw $4, %ymm0, %ymm3
+; AVX512DQ-NEXT:    vpand %ymm5, %ymm3, %ymm3
+; AVX512DQ-NEXT:    vpsllw $5, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpsllw $2, %ymm0, %ymm3
+; AVX512DQ-NEXT:    vpand %ymm6, %ymm3, %ymm3
+; AVX512DQ-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpaddb %ymm0, %ymm0, %ymm3
+; AVX512DQ-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512BW-LABEL: var_shift_v64i8:
@@ -122,9 +128,11 @@ define <16 x i32> @splatvar_shift_v16i32(<16 x i32> %a, <16 x i32> %b) nounwind
 define <32 x i16> @splatvar_shift_v32i16(<32 x i16> %a, <32 x i16> %b) nounwind {
 ; AVX512DQ-LABEL: splatvar_shift_v32i16:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
-; AVX512DQ-NEXT:    vpsllw %xmm2, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpsllw %xmm2, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512DQ-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512DQ-NEXT:    vpsllw %xmm1, %ymm2, %ymm2
+; AVX512DQ-NEXT:    vpsllw %xmm1, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatvar_shift_v32i16:
@@ -140,14 +148,16 @@ define <32 x i16> @splatvar_shift_v32i16(<32 x i16> %a, <32 x i16> %b) nounwind
 define <64 x i8> @splatvar_shift_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
 ; AVX512DQ-LABEL: splatvar_shift_v64i8:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vpmovzxbq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-NEXT:    vpsllw %xmm2, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512DQ-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512DQ-NEXT:    vpsllw %xmm1, %ymm2, %ymm2
 ; AVX512DQ-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX512DQ-NEXT:    vpsllw %xmm2, %xmm3, %xmm3
+; AVX512DQ-NEXT:    vpsllw %xmm1, %xmm3, %xmm3
 ; AVX512DQ-NEXT:    vpbroadcastb %xmm3, %ymm3
+; AVX512DQ-NEXT:    vpand %ymm3, %ymm2, %ymm2
+; AVX512DQ-NEXT:    vpsllw %xmm1, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpand %ymm3, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpsllw %xmm2, %ymm1, %ymm1
-; AVX512DQ-NEXT:    vpand %ymm3, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatvar_shift_v64i8:
@@ -189,9 +199,11 @@ define <16 x i32> @constant_shift_v16i32(<16 x i32> %a) nounwind {
 define <32 x i16> @constant_shift_v32i16(<32 x i16> %a) nounwind {
 ; AVX512DQ-LABEL: constant_shift_v32i16:
 ; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm2 = [1,2,4,8,16,32,64,128,256,512,1024,2048,4096,8192,16384,32768]
-; AVX512DQ-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpmullw %ymm2, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512BW-LABEL: constant_shift_v32i16:
@@ -205,28 +217,30 @@ define <32 x i16> @constant_shift_v32i16(<32 x i16> %a) nounwind {
 define <64 x i8> @constant_shift_v64i8(<64 x i8> %a) nounwind {
 ; AVX512DQ-LABEL: constant_shift_v64i8:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vpsllw $4, %ymm0, %ymm2
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
+; AVX512DQ-NEXT:    vpsllw $4, %ymm1, %ymm2
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
 ; AVX512DQ-NEXT:    vpand %ymm3, %ymm2, %ymm2
 ; AVX512DQ-NEXT:    vbroadcasti128 {{.*#+}} ymm4 = [8192,24640,41088,57536,49376,32928,16480,32,8192,24640,41088,57536,49376,32928,16480,32]
 ; AVX512DQ-NEXT:    # ymm4 = mem[0,1,0,1]
-; AVX512DQ-NEXT:    vpblendvb %ymm4, %ymm2, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpsllw $2, %ymm0, %ymm2
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm5 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
-; AVX512DQ-NEXT:    vpand %ymm5, %ymm2, %ymm2
-; AVX512DQ-NEXT:    vpaddb %ymm4, %ymm4, %ymm6
-; AVX512DQ-NEXT:    vpblendvb %ymm6, %ymm2, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpaddb %ymm0, %ymm0, %ymm2
-; AVX512DQ-NEXT:    vpaddb %ymm6, %ymm6, %ymm7
-; AVX512DQ-NEXT:    vpblendvb %ymm7, %ymm2, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpsllw $4, %ymm1, %ymm2
-; AVX512DQ-NEXT:    vpand %ymm3, %ymm2, %ymm2
 ; AVX512DQ-NEXT:    vpblendvb %ymm4, %ymm2, %ymm1, %ymm1
 ; AVX512DQ-NEXT:    vpsllw $2, %ymm1, %ymm2
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm5 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
 ; AVX512DQ-NEXT:    vpand %ymm5, %ymm2, %ymm2
+; AVX512DQ-NEXT:    vpaddb %ymm4, %ymm4, %ymm6
 ; AVX512DQ-NEXT:    vpblendvb %ymm6, %ymm2, %ymm1, %ymm1
 ; AVX512DQ-NEXT:    vpaddb %ymm1, %ymm1, %ymm2
+; AVX512DQ-NEXT:    vpaddb %ymm6, %ymm6, %ymm7
 ; AVX512DQ-NEXT:    vpblendvb %ymm7, %ymm2, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpsllw $4, %ymm0, %ymm2
+; AVX512DQ-NEXT:    vpand %ymm3, %ymm2, %ymm2
+; AVX512DQ-NEXT:    vpblendvb %ymm4, %ymm2, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpsllw $2, %ymm0, %ymm2
+; AVX512DQ-NEXT:    vpand %ymm5, %ymm2, %ymm2
+; AVX512DQ-NEXT:    vpblendvb %ymm6, %ymm2, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpaddb %ymm0, %ymm0, %ymm2
+; AVX512DQ-NEXT:    vpblendvb %ymm7, %ymm2, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512BW-LABEL: constant_shift_v64i8:
@@ -275,8 +289,10 @@ define <16 x i32> @splatconstant_shift_v16i32(<16 x i32> %a) nounwind {
 define <32 x i16> @splatconstant_shift_v32i16(<32 x i16> %a) nounwind {
 ; AVX512DQ-LABEL: splatconstant_shift_v32i16:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vpsllw $3, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512DQ-NEXT:    vpsllw $3, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpsllw $3, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatconstant_shift_v32i16:
@@ -290,11 +306,13 @@ define <32 x i16> @splatconstant_shift_v32i16(<32 x i16> %a) nounwind {
 define <64 x i8> @splatconstant_shift_v64i8(<64 x i8> %a) nounwind {
 ; AVX512DQ-LABEL: splatconstant_shift_v64i8:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vpsllw $3, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm2 = [248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248]
-; AVX512DQ-NEXT:    vpand %ymm2, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512DQ-NEXT:    vpsllw $3, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm2 = [248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248]
 ; AVX512DQ-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpsllw $3, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpand %ymm2, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatconstant_shift_v64i8:
diff --git a/llvm/test/CodeGen/X86/vector-shift-shl-sub128.ll b/llvm/test/CodeGen/X86/vector-shift-shl-sub128.ll
index 1d663839a..65bc99ec6 100644
--- a/llvm/test/CodeGen/X86/vector-shift-shl-sub128.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-shl-sub128.ll
@@ -20,88 +20,6 @@
 define <2 x i32> @var_shift_v2i32(<2 x i32> %a, <2 x i32> %b) nounwind {
 ; SSE2-LABEL: var_shift_v2i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    psllq %xmm1, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
-; SSE2-NEXT:    psllq %xmm1, %xmm0
-; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm2[0],xmm0[1]
-; SSE2-NEXT:    retq
-;
-; SSE41-LABEL: var_shift_v2i32:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pxor %xmm2, %xmm2
-; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
-; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    psllq %xmm2, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
-; SSE41-NEXT:    psllq %xmm2, %xmm0
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4,5,6,7]
-; SSE41-NEXT:    retq
-;
-; AVX1-LABEL: var_shift_v2i32:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
-; AVX1-NEXT:    vpsllq %xmm1, %xmm0, %xmm2
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
-; AVX1-NEXT:    vpsllq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm2[0,1,2,3],xmm0[4,5,6,7]
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: var_shift_v2i32:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; AVX2-NEXT:    vpsllvq %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    retq
-;
-; XOPAVX1-LABEL: var_shift_v2i32:
-; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; XOPAVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
-; XOPAVX1-NEXT:    vpshlq %xmm1, %xmm0, %xmm0
-; XOPAVX1-NEXT:    retq
-;
-; XOPAVX2-LABEL: var_shift_v2i32:
-; XOPAVX2:       # %bb.0:
-; XOPAVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; XOPAVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; XOPAVX2-NEXT:    vpsllvq %xmm1, %xmm0, %xmm0
-; XOPAVX2-NEXT:    retq
-;
-; AVX512-LABEL: var_shift_v2i32:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; AVX512-NEXT:    vpsllvq %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    retq
-;
-; AVX512VL-LABEL: var_shift_v2i32:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VL-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; AVX512VL-NEXT:    vpsllvq %xmm1, %xmm0, %xmm0
-; AVX512VL-NEXT:    retq
-;
-; X32-SSE-LABEL: var_shift_v2i32:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm1
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm2
-; X32-SSE-NEXT:    psllq %xmm1, %xmm2
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
-; X32-SSE-NEXT:    xorps %xmm3, %xmm3
-; X32-SSE-NEXT:    movss {{.*#+}} xmm3 = xmm1[0],xmm3[1,2,3]
-; X32-SSE-NEXT:    psllq %xmm3, %xmm0
-; X32-SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm2[0],xmm0[1]
-; X32-SSE-NEXT:    retl
-  %shift = shl <2 x i32> %a, %b
-  ret <2 x i32> %shift
-}
-
-define <4 x i16> @var_shift_v4i16(<4 x i16> %a, <4 x i16> %b) nounwind {
-; SSE2-LABEL: var_shift_v4i16:
-; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pslld $23, %xmm1
 ; SSE2-NEXT:    paddd {{.*}}(%rip), %xmm1
 ; SSE2-NEXT:    cvttps2dq %xmm1, %xmm1
@@ -114,7 +32,7 @@ define <4 x i16> @var_shift_v4i16(<4 x i16> %a, <4 x i16> %b) nounwind {
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: var_shift_v4i16:
+; SSE41-LABEL: var_shift_v2i32:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pslld $23, %xmm1
 ; SSE41-NEXT:    paddd {{.*}}(%rip), %xmm1
@@ -122,7 +40,7 @@ define <4 x i16> @var_shift_v4i16(<4 x i16> %a, <4 x i16> %b) nounwind {
 ; SSE41-NEXT:    pmulld %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: var_shift_v4i16:
+; AVX1-LABEL: var_shift_v2i32:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpslld $23, %xmm1, %xmm1
 ; AVX1-NEXT:    vpaddd {{.*}}(%rip), %xmm1, %xmm1
@@ -130,42 +48,32 @@ define <4 x i16> @var_shift_v4i16(<4 x i16> %a, <4 x i16> %b) nounwind {
 ; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    retq
 ;
-; AVX2-LABEL: var_shift_v4i16:
+; AVX2-LABEL: var_shift_v2i32:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
 ; AVX2-NEXT:    vpsllvd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    retq
 ;
-; XOPAVX1-LABEL: var_shift_v4i16:
+; XOPAVX1-LABEL: var_shift_v2i32:
 ; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; XOPAVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
 ; XOPAVX1-NEXT:    vpshld %xmm1, %xmm0, %xmm0
 ; XOPAVX1-NEXT:    retq
 ;
-; XOPAVX2-LABEL: var_shift_v4i16:
+; XOPAVX2-LABEL: var_shift_v2i32:
 ; XOPAVX2:       # %bb.0:
-; XOPAVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; XOPAVX2-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
 ; XOPAVX2-NEXT:    vpsllvd %xmm1, %xmm0, %xmm0
 ; XOPAVX2-NEXT:    retq
 ;
-; AVX512-LABEL: var_shift_v4i16:
+; AVX512-LABEL: var_shift_v2i32:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
 ; AVX512-NEXT:    vpsllvd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    retq
 ;
-; AVX512VL-LABEL: var_shift_v4i16:
+; AVX512VL-LABEL: var_shift_v2i32:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VL-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
 ; AVX512VL-NEXT:    vpsllvd %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: var_shift_v4i16:
+; X32-SSE-LABEL: var_shift_v2i32:
 ; X32-SSE:       # %bb.0:
 ; X32-SSE-NEXT:    pslld $23, %xmm1
 ; X32-SSE-NEXT:    paddd {{\.LCPI.*}}, %xmm1
@@ -178,126 +86,168 @@ define <4 x i16> @var_shift_v4i16(<4 x i16> %a, <4 x i16> %b) nounwind {
 ; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
 ; X32-SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; X32-SSE-NEXT:    retl
-  %shift = shl <4 x i16> %a, %b
-  ret <4 x i16> %shift
+  %shift = shl <2 x i32> %a, %b
+  ret <2 x i32> %shift
 }
 
-define <2 x i16> @var_shift_v2i16(<2 x i16> %a, <2 x i16> %b) nounwind {
-; SSE2-LABEL: var_shift_v2i16:
+define <4 x i16> @var_shift_v4i16(<4 x i16> %a, <4 x i16> %b) nounwind {
+; SSE2-LABEL: var_shift_v4i16:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    psllq %xmm1, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
-; SSE2-NEXT:    psllq %xmm1, %xmm0
-; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm2[0],xmm0[1]
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE2-NEXT:    pslld $23, %xmm2
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [1065353216,1065353216,1065353216,1065353216]
+; SSE2-NEXT:    paddd %xmm3, %xmm2
+; SSE2-NEXT:    cvttps2dq %xmm2, %xmm2
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,2,2,3,4,5,6,7]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,6,6,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; SSE2-NEXT:    pslld $23, %xmm1
+; SSE2-NEXT:    paddd %xmm3, %xmm1
+; SSE2-NEXT:    cvttps2dq %xmm1, %xmm1
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,6,6,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; SSE2-NEXT:    pmullw %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: var_shift_v2i16:
+; SSE41-LABEL: var_shift_v4i16:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pxor %xmm2, %xmm2
-; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3],xmm1[4],xmm2[5,6,7]
-; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    psllq %xmm2, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
-; SSE41-NEXT:    psllq %xmm2, %xmm0
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4,5,6,7]
+; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; SSE41-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE41-NEXT:    pslld $23, %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [1065353216,1065353216,1065353216,1065353216]
+; SSE41-NEXT:    paddd %xmm3, %xmm1
+; SSE41-NEXT:    cvttps2dq %xmm1, %xmm1
+; SSE41-NEXT:    pslld $23, %xmm2
+; SSE41-NEXT:    paddd %xmm3, %xmm2
+; SSE41-NEXT:    cvttps2dq %xmm2, %xmm2
+; SSE41-NEXT:    packusdw %xmm1, %xmm2
+; SSE41-NEXT:    pmullw %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: var_shift_v2i16:
+; AVX1-LABEL: var_shift_v4i16:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3],xmm1[4],xmm2[5,6,7]
-; AVX1-NEXT:    vpsllq %xmm1, %xmm0, %xmm2
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
-; AVX1-NEXT:    vpsllq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm2[0,1,2,3],xmm0[4,5,6,7]
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX1-NEXT:    vpslld $23, %xmm2, %xmm2
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [1065353216,1065353216,1065353216,1065353216]
+; AVX1-NEXT:    vpaddd %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vcvttps2dq %xmm2, %xmm2
+; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; AVX1-NEXT:    vpslld $23, %xmm1, %xmm1
+; AVX1-NEXT:    vpaddd %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vcvttps2dq %xmm1, %xmm1
+; AVX1-NEXT:    vpackusdw %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    retq
 ;
-; AVX2-LABEL: var_shift_v2i16:
+; AVX2-LABEL: var_shift_v4i16:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3],xmm1[4],xmm2[5,6,7]
-; AVX2-NEXT:    vpsllvq %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2-NEXT:    vpsllvd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
+; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; AVX2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
-; XOPAVX1-LABEL: var_shift_v2i16:
-; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; XOPAVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3],xmm1[4],xmm2[5,6,7]
-; XOPAVX1-NEXT:    vpshlq %xmm1, %xmm0, %xmm0
-; XOPAVX1-NEXT:    retq
+; XOP-LABEL: var_shift_v4i16:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vpshlw %xmm1, %xmm0, %xmm0
+; XOP-NEXT:    retq
 ;
-; XOPAVX2-LABEL: var_shift_v2i16:
-; XOPAVX2:       # %bb.0:
-; XOPAVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; XOPAVX2-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3],xmm1[4],xmm2[5,6,7]
-; XOPAVX2-NEXT:    vpsllvq %xmm1, %xmm0, %xmm0
-; XOPAVX2-NEXT:    retq
+; AVX512DQ-LABEL: var_shift_v4i16:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512DQ-NEXT:    vpsllvd %ymm1, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    retq
 ;
-; AVX512-LABEL: var_shift_v2i16:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3],xmm1[4],xmm2[5,6,7]
-; AVX512-NEXT:    vpsllvq %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    retq
+; AVX512BW-LABEL: var_shift_v4i16:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512BW-NEXT:    vpsllvw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
 ;
-; AVX512VL-LABEL: var_shift_v2i16:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VL-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3],xmm1[4],xmm2[5,6,7]
-; AVX512VL-NEXT:    vpsllvq %xmm1, %xmm0, %xmm0
-; AVX512VL-NEXT:    retq
+; AVX512DQVL-LABEL: var_shift_v4i16:
+; AVX512DQVL:       # %bb.0:
+; AVX512DQVL-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX512DQVL-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512DQVL-NEXT:    vpsllvd %ymm1, %ymm0, %ymm0
+; AVX512DQVL-NEXT:    vpmovdw %ymm0, %xmm0
+; AVX512DQVL-NEXT:    vzeroupper
+; AVX512DQVL-NEXT:    retq
 ;
-; X32-SSE-LABEL: var_shift_v2i16:
+; AVX512BWVL-LABEL: var_shift_v4i16:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpsllvw %xmm1, %xmm0, %xmm0
+; AVX512BWVL-NEXT:    retq
+;
+; X32-SSE-LABEL: var_shift_v4i16:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm1
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm2
-; X32-SSE-NEXT:    psllq %xmm1, %xmm2
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
-; X32-SSE-NEXT:    psllq %xmm1, %xmm0
-; X32-SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm2[0],xmm0[1]
+; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
+; X32-SSE-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; X32-SSE-NEXT:    pslld $23, %xmm2
+; X32-SSE-NEXT:    movdqa {{.*#+}} xmm3 = [1065353216,1065353216,1065353216,1065353216]
+; X32-SSE-NEXT:    paddd %xmm3, %xmm2
+; X32-SSE-NEXT:    cvttps2dq %xmm2, %xmm2
+; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,2,2,3,4,5,6,7]
+; X32-SSE-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,6,6,7]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; X32-SSE-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; X32-SSE-NEXT:    pslld $23, %xmm1
+; X32-SSE-NEXT:    paddd %xmm3, %xmm1
+; X32-SSE-NEXT:    cvttps2dq %xmm1, %xmm1
+; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
+; X32-SSE-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,6,6,7]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; X32-SSE-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; X32-SSE-NEXT:    pmullw %xmm1, %xmm0
 ; X32-SSE-NEXT:    retl
-  %shift = shl <2 x i16> %a, %b
-  ret <2 x i16> %shift
+  %shift = shl <4 x i16> %a, %b
+  ret <4 x i16> %shift
 }
 
-define <8 x i8> @var_shift_v8i8(<8 x i8> %a, <8 x i8> %b) nounwind {
-; SSE2-LABEL: var_shift_v8i8:
+define <2 x i16> @var_shift_v2i16(<2 x i16> %a, <2 x i16> %b) nounwind {
+; SSE2-LABEL: var_shift_v2i16:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    movdqa %xmm1, %xmm3
-; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
-; SSE2-NEXT:    pslld $23, %xmm3
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [1065353216,1065353216,1065353216,1065353216]
-; SSE2-NEXT:    paddd %xmm4, %xmm3
-; SSE2-NEXT:    cvttps2dq %xmm3, %xmm3
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm3[0,2,2,3,4,5,6,7]
-; SSE2-NEXT:    pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,6,6,7]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE2-NEXT:    pslld $23, %xmm2
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [1065353216,1065353216,1065353216,1065353216]
+; SSE2-NEXT:    paddd %xmm3, %xmm2
+; SSE2-NEXT:    cvttps2dq %xmm2, %xmm2
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,2,2,3,4,5,6,7]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,6,6,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
 ; SSE2-NEXT:    pslld $23, %xmm1
-; SSE2-NEXT:    paddd %xmm4, %xmm1
+; SSE2-NEXT:    paddd %xmm3, %xmm1
 ; SSE2-NEXT:    cvttps2dq %xmm1, %xmm1
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
 ; SSE2-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,6,6,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
 ; SSE2-NEXT:    pmullw %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: var_shift_v8i8:
+; SSE41-LABEL: var_shift_v2i16:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pxor %xmm3, %xmm3
-; SSE41-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm3[8],xmm1[9],xmm3[9],xmm1[10],xmm3[10],xmm1[11],xmm3[11],xmm1[12],xmm3[12],xmm1[13],xmm3[13],xmm1[14],xmm3[14],xmm1[15],xmm3[15]
+; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; SSE41-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
 ; SSE41-NEXT:    pslld $23, %xmm1
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [1065353216,1065353216,1065353216,1065353216]
 ; SSE41-NEXT:    paddd %xmm3, %xmm1
 ; SSE41-NEXT:    cvttps2dq %xmm1, %xmm1
-; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
 ; SSE41-NEXT:    pslld $23, %xmm2
 ; SSE41-NEXT:    paddd %xmm3, %xmm2
 ; SSE41-NEXT:    cvttps2dq %xmm2, %xmm2
@@ -305,15 +255,14 @@ define <8 x i8> @var_shift_v8i8(<8 x i8> %a, <8 x i8> %b) nounwind {
 ; SSE41-NEXT:    pmullw %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: var_shift_v8i8:
+; AVX1-LABEL: var_shift_v2i16:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15]
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
 ; AVX1-NEXT:    vpslld $23, %xmm2, %xmm2
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [1065353216,1065353216,1065353216,1065353216]
 ; AVX1-NEXT:    vpaddd %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vcvttps2dq %xmm2, %xmm2
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
 ; AVX1-NEXT:    vpslld $23, %xmm1, %xmm1
 ; AVX1-NEXT:    vpaddd %xmm3, %xmm1, %xmm1
 ; AVX1-NEXT:    vcvttps2dq %xmm1, %xmm1
@@ -321,11 +270,10 @@ define <8 x i8> @var_shift_v8i8(<8 x i8> %a, <8 x i8> %b) nounwind {
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    retq
 ;
-; AVX2-LABEL: var_shift_v8i8:
+; AVX2-LABEL: var_shift_v2i16:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
-; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; AVX2-NEXT:    vpsllvd %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
@@ -333,70 +281,208 @@ define <8 x i8> @var_shift_v8i8(<8 x i8> %a, <8 x i8> %b) nounwind {
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
-; XOP-LABEL: var_shift_v8i8:
+; XOP-LABEL: var_shift_v2i16:
 ; XOP:       # %bb.0:
-; XOP-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
 ; XOP-NEXT:    vpshlw %xmm1, %xmm0, %xmm0
 ; XOP-NEXT:    retq
 ;
-; AVX512DQ-LABEL: var_shift_v8i8:
+; AVX512DQ-LABEL: var_shift_v2i16:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
-; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; AVX512DQ-NEXT:    vpsllvd %ymm1, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpmovdw %zmm0, %ymm0
 ; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
 ;
-; AVX512BW-LABEL: var_shift_v8i8:
+; AVX512BW-LABEL: var_shift_v2i16:
 ; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
 ; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; AVX512BW-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
 ; AVX512BW-NEXT:    vpsllvw %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
-; AVX512DQVL-LABEL: var_shift_v8i8:
+; AVX512DQVL-LABEL: var_shift_v2i16:
 ; AVX512DQVL:       # %bb.0:
-; AVX512DQVL-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
-; AVX512DQVL-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; AVX512DQVL-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX512DQVL-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; AVX512DQVL-NEXT:    vpsllvd %ymm1, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vpmovdw %ymm0, %xmm0
 ; AVX512DQVL-NEXT:    vzeroupper
 ; AVX512DQVL-NEXT:    retq
 ;
-; AVX512BWVL-LABEL: var_shift_v8i8:
+; AVX512BWVL-LABEL: var_shift_v2i16:
 ; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
 ; AVX512BWVL-NEXT:    vpsllvw %xmm1, %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-SSE-LABEL: var_shift_v8i8:
+; X32-SSE-LABEL: var_shift_v2i16:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm1
-; X32-SSE-NEXT:    pxor %xmm2, %xmm2
-; X32-SSE-NEXT:    movdqa %xmm1, %xmm3
-; X32-SSE-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
-; X32-SSE-NEXT:    pslld $23, %xmm3
-; X32-SSE-NEXT:    movdqa {{.*#+}} xmm4 = [1065353216,1065353216,1065353216,1065353216]
-; X32-SSE-NEXT:    paddd %xmm4, %xmm3
-; X32-SSE-NEXT:    cvttps2dq %xmm3, %xmm3
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm3 = xmm3[0,2,2,3,4,5,6,7]
-; X32-SSE-NEXT:    pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,6,6,7]
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
-; X32-SSE-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
+; X32-SSE-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; X32-SSE-NEXT:    pslld $23, %xmm2
+; X32-SSE-NEXT:    movdqa {{.*#+}} xmm3 = [1065353216,1065353216,1065353216,1065353216]
+; X32-SSE-NEXT:    paddd %xmm3, %xmm2
+; X32-SSE-NEXT:    cvttps2dq %xmm2, %xmm2
+; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,2,2,3,4,5,6,7]
+; X32-SSE-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,6,6,7]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; X32-SSE-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
 ; X32-SSE-NEXT:    pslld $23, %xmm1
-; X32-SSE-NEXT:    paddd %xmm4, %xmm1
+; X32-SSE-NEXT:    paddd %xmm3, %xmm1
 ; X32-SSE-NEXT:    cvttps2dq %xmm1, %xmm1
 ; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
 ; X32-SSE-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,6,6,7]
 ; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; X32-SSE-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; X32-SSE-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
 ; X32-SSE-NEXT:    pmullw %xmm1, %xmm0
+; X32-SSE-NEXT:    retl
+  %shift = shl <2 x i16> %a, %b
+  ret <2 x i16> %shift
+}
+
+define <8 x i8> @var_shift_v8i8(<8 x i8> %a, <8 x i8> %b) nounwind {
+; SSE2-LABEL: var_shift_v8i8:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    psllw $5, %xmm1
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    pcmpgtb %xmm1, %xmm3
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    pandn %xmm0, %xmm4
+; SSE2-NEXT:    psllw $4, %xmm0
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    por %xmm4, %xmm0
+; SSE2-NEXT:    paddb %xmm1, %xmm1
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    pcmpgtb %xmm1, %xmm3
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    pandn %xmm0, %xmm4
+; SSE2-NEXT:    psllw $2, %xmm0
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    por %xmm4, %xmm0
+; SSE2-NEXT:    paddb %xmm1, %xmm1
+; SSE2-NEXT:    pcmpgtb %xmm1, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm1
+; SSE2-NEXT:    pandn %xmm0, %xmm1
+; SSE2-NEXT:    paddb %xmm0, %xmm0
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    por %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: var_shift_v8i8:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    psllw $5, %xmm1
+; SSE41-NEXT:    movdqa %xmm0, %xmm3
+; SSE41-NEXT:    psllw $4, %xmm3
+; SSE41-NEXT:    pand {{.*}}(%rip), %xmm3
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pblendvb %xmm0, %xmm3, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm3
+; SSE41-NEXT:    psllw $2, %xmm3
+; SSE41-NEXT:    pand {{.*}}(%rip), %xmm3
+; SSE41-NEXT:    paddb %xmm1, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pblendvb %xmm0, %xmm3, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm3
+; SSE41-NEXT:    paddb %xmm2, %xmm3
+; SSE41-NEXT:    paddb %xmm1, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pblendvb %xmm0, %xmm3, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: var_shift_v8i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpsllw $5, %xmm1, %xmm1
+; AVX-NEXT:    vpsllw $4, %xmm0, %xmm2
+; AVX-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
+; AVX-NEXT:    vpblendvb %xmm1, %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    vpsllw $2, %xmm0, %xmm2
+; AVX-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
+; AVX-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vpblendvb %xmm1, %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    vpaddb %xmm0, %xmm0, %xmm2
+; AVX-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vpblendvb %xmm1, %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; XOP-LABEL: var_shift_v8i8:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vpshlb %xmm1, %xmm0, %xmm0
+; XOP-NEXT:    retq
+;
+; AVX512DQ-LABEL: var_shift_v8i8:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; AVX512DQ-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; AVX512DQ-NEXT:    vpsllvd %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    retq
+;
+; AVX512BW-LABEL: var_shift_v8i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512BW-NEXT:    vpsllvw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512DQVL-LABEL: var_shift_v8i8:
+; AVX512DQVL:       # %bb.0:
+; AVX512DQVL-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; AVX512DQVL-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; AVX512DQVL-NEXT:    vpsllvd %zmm1, %zmm0, %zmm0
+; AVX512DQVL-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512DQVL-NEXT:    vzeroupper
+; AVX512DQVL-NEXT:    retq
+;
+; AVX512BWVL-LABEL: var_shift_v8i8:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512BWVL-NEXT:    vpsllvw %ymm1, %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm0
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
+;
+; X32-SSE-LABEL: var_shift_v8i8:
+; X32-SSE:       # %bb.0:
+; X32-SSE-NEXT:    psllw $5, %xmm1
+; X32-SSE-NEXT:    pxor %xmm2, %xmm2
+; X32-SSE-NEXT:    pxor %xmm3, %xmm3
+; X32-SSE-NEXT:    pcmpgtb %xmm1, %xmm3
+; X32-SSE-NEXT:    movdqa %xmm3, %xmm4
+; X32-SSE-NEXT:    pandn %xmm0, %xmm4
+; X32-SSE-NEXT:    psllw $4, %xmm0
+; X32-SSE-NEXT:    pand %xmm3, %xmm0
+; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X32-SSE-NEXT:    por %xmm4, %xmm0
+; X32-SSE-NEXT:    paddb %xmm1, %xmm1
+; X32-SSE-NEXT:    pxor %xmm3, %xmm3
+; X32-SSE-NEXT:    pcmpgtb %xmm1, %xmm3
+; X32-SSE-NEXT:    movdqa %xmm3, %xmm4
+; X32-SSE-NEXT:    pandn %xmm0, %xmm4
+; X32-SSE-NEXT:    psllw $2, %xmm0
+; X32-SSE-NEXT:    pand %xmm3, %xmm0
+; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X32-SSE-NEXT:    por %xmm4, %xmm0
+; X32-SSE-NEXT:    paddb %xmm1, %xmm1
+; X32-SSE-NEXT:    pcmpgtb %xmm1, %xmm2
+; X32-SSE-NEXT:    movdqa %xmm2, %xmm1
+; X32-SSE-NEXT:    pandn %xmm0, %xmm1
+; X32-SSE-NEXT:    paddb %xmm0, %xmm0
+; X32-SSE-NEXT:    pand %xmm2, %xmm0
+; X32-SSE-NEXT:    por %xmm1, %xmm0
 ; X32-SSE-NEXT:    retl
   %shift = shl <8 x i8> %a, %b
   ret <8 x i8> %shift
@@ -405,80 +491,142 @@ define <8 x i8> @var_shift_v8i8(<8 x i8> %a, <8 x i8> %b) nounwind {
 define <4 x i8> @var_shift_v4i8(<4 x i8> %a, <4 x i8> %b) nounwind {
 ; SSE2-LABEL: var_shift_v4i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
-; SSE2-NEXT:    pslld $23, %xmm1
-; SSE2-NEXT:    paddd {{.*}}(%rip), %xmm1
-; SSE2-NEXT:    cvttps2dq %xmm1, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pmuludq %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pmuludq %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT:    psllw $5, %xmm1
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    pcmpgtb %xmm1, %xmm3
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    pandn %xmm0, %xmm4
+; SSE2-NEXT:    psllw $4, %xmm0
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    por %xmm4, %xmm0
+; SSE2-NEXT:    paddb %xmm1, %xmm1
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    pcmpgtb %xmm1, %xmm3
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    pandn %xmm0, %xmm4
+; SSE2-NEXT:    psllw $2, %xmm0
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    por %xmm4, %xmm0
+; SSE2-NEXT:    paddb %xmm1, %xmm1
+; SSE2-NEXT:    pcmpgtb %xmm1, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm1
+; SSE2-NEXT:    pandn %xmm0, %xmm1
+; SSE2-NEXT:    paddb %xmm0, %xmm0
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: var_shift_v4i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pand {{.*}}(%rip), %xmm1
-; SSE41-NEXT:    pslld $23, %xmm1
-; SSE41-NEXT:    paddd {{.*}}(%rip), %xmm1
-; SSE41-NEXT:    cvttps2dq %xmm1, %xmm1
-; SSE41-NEXT:    pmulld %xmm1, %xmm0
+; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    psllw $5, %xmm1
+; SSE41-NEXT:    movdqa %xmm0, %xmm3
+; SSE41-NEXT:    psllw $4, %xmm3
+; SSE41-NEXT:    pand {{.*}}(%rip), %xmm3
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pblendvb %xmm0, %xmm3, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm3
+; SSE41-NEXT:    psllw $2, %xmm3
+; SSE41-NEXT:    pand {{.*}}(%rip), %xmm3
+; SSE41-NEXT:    paddb %xmm1, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pblendvb %xmm0, %xmm3, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm3
+; SSE41-NEXT:    paddb %xmm2, %xmm3
+; SSE41-NEXT:    paddb %xmm1, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pblendvb %xmm0, %xmm3, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: var_shift_v4i8:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
-; AVX1-NEXT:    vpslld $23, %xmm1, %xmm1
-; AVX1-NEXT:    vpaddd {{.*}}(%rip), %xmm1, %xmm1
-; AVX1-NEXT:    vcvttps2dq %xmm1, %xmm1
-; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    retq
+; AVX-LABEL: var_shift_v4i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpsllw $5, %xmm1, %xmm1
+; AVX-NEXT:    vpsllw $4, %xmm0, %xmm2
+; AVX-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
+; AVX-NEXT:    vpblendvb %xmm1, %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    vpsllw $2, %xmm0, %xmm2
+; AVX-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
+; AVX-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vpblendvb %xmm1, %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    vpaddb %xmm0, %xmm0, %xmm2
+; AVX-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vpblendvb %xmm1, %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    retq
 ;
-; AVX2-LABEL: var_shift_v4i8:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
-; AVX2-NEXT:    vpsllvd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    retq
+; XOP-LABEL: var_shift_v4i8:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vpshlb %xmm1, %xmm0, %xmm0
+; XOP-NEXT:    retq
 ;
-; XOPAVX1-LABEL: var_shift_v4i8:
-; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
-; XOPAVX1-NEXT:    vpshld %xmm1, %xmm0, %xmm0
-; XOPAVX1-NEXT:    retq
+; AVX512DQ-LABEL: var_shift_v4i8:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; AVX512DQ-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; AVX512DQ-NEXT:    vpsllvd %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    retq
 ;
-; XOPAVX2-LABEL: var_shift_v4i8:
-; XOPAVX2:       # %bb.0:
-; XOPAVX2-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
-; XOPAVX2-NEXT:    vpsllvd %xmm1, %xmm0, %xmm0
-; XOPAVX2-NEXT:    retq
+; AVX512BW-LABEL: var_shift_v4i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512BW-NEXT:    vpsllvw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
 ;
-; AVX512-LABEL: var_shift_v4i8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
-; AVX512-NEXT:    vpsllvd %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    retq
+; AVX512DQVL-LABEL: var_shift_v4i8:
+; AVX512DQVL:       # %bb.0:
+; AVX512DQVL-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; AVX512DQVL-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; AVX512DQVL-NEXT:    vpsllvd %zmm1, %zmm0, %zmm0
+; AVX512DQVL-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512DQVL-NEXT:    vzeroupper
+; AVX512DQVL-NEXT:    retq
 ;
-; AVX512VL-LABEL: var_shift_v4i8:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
-; AVX512VL-NEXT:    vpsllvd %xmm1, %xmm0, %xmm0
-; AVX512VL-NEXT:    retq
+; AVX512BWVL-LABEL: var_shift_v4i8:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512BWVL-NEXT:    vpsllvw %ymm1, %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm0
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
 ;
 ; X32-SSE-LABEL: var_shift_v4i8:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm1
-; X32-SSE-NEXT:    pslld $23, %xmm1
-; X32-SSE-NEXT:    paddd {{\.LCPI.*}}, %xmm1
-; X32-SSE-NEXT:    cvttps2dq %xmm1, %xmm1
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
-; X32-SSE-NEXT:    pmuludq %xmm1, %xmm0
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; X32-SSE-NEXT:    pmuludq %xmm2, %xmm1
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; X32-SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; X32-SSE-NEXT:    psllw $5, %xmm1
+; X32-SSE-NEXT:    pxor %xmm2, %xmm2
+; X32-SSE-NEXT:    pxor %xmm3, %xmm3
+; X32-SSE-NEXT:    pcmpgtb %xmm1, %xmm3
+; X32-SSE-NEXT:    movdqa %xmm3, %xmm4
+; X32-SSE-NEXT:    pandn %xmm0, %xmm4
+; X32-SSE-NEXT:    psllw $4, %xmm0
+; X32-SSE-NEXT:    pand %xmm3, %xmm0
+; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X32-SSE-NEXT:    por %xmm4, %xmm0
+; X32-SSE-NEXT:    paddb %xmm1, %xmm1
+; X32-SSE-NEXT:    pxor %xmm3, %xmm3
+; X32-SSE-NEXT:    pcmpgtb %xmm1, %xmm3
+; X32-SSE-NEXT:    movdqa %xmm3, %xmm4
+; X32-SSE-NEXT:    pandn %xmm0, %xmm4
+; X32-SSE-NEXT:    psllw $2, %xmm0
+; X32-SSE-NEXT:    pand %xmm3, %xmm0
+; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X32-SSE-NEXT:    por %xmm4, %xmm0
+; X32-SSE-NEXT:    paddb %xmm1, %xmm1
+; X32-SSE-NEXT:    pcmpgtb %xmm1, %xmm2
+; X32-SSE-NEXT:    movdqa %xmm2, %xmm1
+; X32-SSE-NEXT:    pandn %xmm0, %xmm1
+; X32-SSE-NEXT:    paddb %xmm0, %xmm0
+; X32-SSE-NEXT:    pand %xmm2, %xmm0
+; X32-SSE-NEXT:    por %xmm1, %xmm0
 ; X32-SSE-NEXT:    retl
   %shift = shl <4 x i8> %a, %b
   ret <4 x i8> %shift
@@ -487,71 +635,142 @@ define <4 x i8> @var_shift_v4i8(<4 x i8> %a, <4 x i8> %b) nounwind {
 define <2 x i8> @var_shift_v2i8(<2 x i8> %a, <2 x i8> %b) nounwind {
 ; SSE2-LABEL: var_shift_v2i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    psllq %xmm1, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
-; SSE2-NEXT:    psllq %xmm1, %xmm0
-; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm2[0],xmm0[1]
+; SSE2-NEXT:    psllw $5, %xmm1
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    pcmpgtb %xmm1, %xmm3
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    pandn %xmm0, %xmm4
+; SSE2-NEXT:    psllw $4, %xmm0
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    por %xmm4, %xmm0
+; SSE2-NEXT:    paddb %xmm1, %xmm1
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    pcmpgtb %xmm1, %xmm3
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    pandn %xmm0, %xmm4
+; SSE2-NEXT:    psllw $2, %xmm0
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    por %xmm4, %xmm0
+; SSE2-NEXT:    paddb %xmm1, %xmm1
+; SSE2-NEXT:    pcmpgtb %xmm1, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm1
+; SSE2-NEXT:    pandn %xmm0, %xmm1
+; SSE2-NEXT:    paddb %xmm0, %xmm0
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: var_shift_v2i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pand {{.*}}(%rip), %xmm1
 ; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    psllq %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
-; SSE41-NEXT:    psllq %xmm1, %xmm0
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm2[0,1,2,3],xmm0[4,5,6,7]
+; SSE41-NEXT:    psllw $5, %xmm1
+; SSE41-NEXT:    movdqa %xmm0, %xmm3
+; SSE41-NEXT:    psllw $4, %xmm3
+; SSE41-NEXT:    pand {{.*}}(%rip), %xmm3
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pblendvb %xmm0, %xmm3, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm3
+; SSE41-NEXT:    psllw $2, %xmm3
+; SSE41-NEXT:    pand {{.*}}(%rip), %xmm3
+; SSE41-NEXT:    paddb %xmm1, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pblendvb %xmm0, %xmm3, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm3
+; SSE41-NEXT:    paddb %xmm2, %xmm3
+; SSE41-NEXT:    paddb %xmm1, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pblendvb %xmm0, %xmm3, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: var_shift_v2i8:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
-; AVX1-NEXT:    vpsllq %xmm1, %xmm0, %xmm2
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
-; AVX1-NEXT:    vpsllq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm2[0,1,2,3],xmm0[4,5,6,7]
-; AVX1-NEXT:    retq
+; AVX-LABEL: var_shift_v2i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpsllw $5, %xmm1, %xmm1
+; AVX-NEXT:    vpsllw $4, %xmm0, %xmm2
+; AVX-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
+; AVX-NEXT:    vpblendvb %xmm1, %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    vpsllw $2, %xmm0, %xmm2
+; AVX-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
+; AVX-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vpblendvb %xmm1, %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    vpaddb %xmm0, %xmm0, %xmm2
+; AVX-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vpblendvb %xmm1, %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    retq
 ;
-; AVX2-LABEL: var_shift_v2i8:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
-; AVX2-NEXT:    vpsllvq %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    retq
+; XOP-LABEL: var_shift_v2i8:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vpshlb %xmm1, %xmm0, %xmm0
+; XOP-NEXT:    retq
 ;
-; XOPAVX1-LABEL: var_shift_v2i8:
-; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
-; XOPAVX1-NEXT:    vpshlq %xmm1, %xmm0, %xmm0
-; XOPAVX1-NEXT:    retq
+; AVX512DQ-LABEL: var_shift_v2i8:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; AVX512DQ-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; AVX512DQ-NEXT:    vpsllvd %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    retq
 ;
-; XOPAVX2-LABEL: var_shift_v2i8:
-; XOPAVX2:       # %bb.0:
-; XOPAVX2-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
-; XOPAVX2-NEXT:    vpsllvq %xmm1, %xmm0, %xmm0
-; XOPAVX2-NEXT:    retq
+; AVX512BW-LABEL: var_shift_v2i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512BW-NEXT:    vpsllvw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
 ;
-; AVX512-LABEL: var_shift_v2i8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
-; AVX512-NEXT:    vpsllvq %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    retq
+; AVX512DQVL-LABEL: var_shift_v2i8:
+; AVX512DQVL:       # %bb.0:
+; AVX512DQVL-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; AVX512DQVL-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; AVX512DQVL-NEXT:    vpsllvd %zmm1, %zmm0, %zmm0
+; AVX512DQVL-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512DQVL-NEXT:    vzeroupper
+; AVX512DQVL-NEXT:    retq
 ;
-; AVX512VL-LABEL: var_shift_v2i8:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
-; AVX512VL-NEXT:    vpsllvq %xmm1, %xmm0, %xmm0
-; AVX512VL-NEXT:    retq
+; AVX512BWVL-LABEL: var_shift_v2i8:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512BWVL-NEXT:    vpsllvw %ymm1, %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm0
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
 ;
 ; X32-SSE-LABEL: var_shift_v2i8:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm1
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm2
-; X32-SSE-NEXT:    psllq %xmm1, %xmm2
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
-; X32-SSE-NEXT:    psllq %xmm1, %xmm0
-; X32-SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm2[0],xmm0[1]
+; X32-SSE-NEXT:    psllw $5, %xmm1
+; X32-SSE-NEXT:    pxor %xmm2, %xmm2
+; X32-SSE-NEXT:    pxor %xmm3, %xmm3
+; X32-SSE-NEXT:    pcmpgtb %xmm1, %xmm3
+; X32-SSE-NEXT:    movdqa %xmm3, %xmm4
+; X32-SSE-NEXT:    pandn %xmm0, %xmm4
+; X32-SSE-NEXT:    psllw $4, %xmm0
+; X32-SSE-NEXT:    pand %xmm3, %xmm0
+; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X32-SSE-NEXT:    por %xmm4, %xmm0
+; X32-SSE-NEXT:    paddb %xmm1, %xmm1
+; X32-SSE-NEXT:    pxor %xmm3, %xmm3
+; X32-SSE-NEXT:    pcmpgtb %xmm1, %xmm3
+; X32-SSE-NEXT:    movdqa %xmm3, %xmm4
+; X32-SSE-NEXT:    pandn %xmm0, %xmm4
+; X32-SSE-NEXT:    psllw $2, %xmm0
+; X32-SSE-NEXT:    pand %xmm3, %xmm0
+; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X32-SSE-NEXT:    por %xmm4, %xmm0
+; X32-SSE-NEXT:    paddb %xmm1, %xmm1
+; X32-SSE-NEXT:    pcmpgtb %xmm1, %xmm2
+; X32-SSE-NEXT:    movdqa %xmm2, %xmm1
+; X32-SSE-NEXT:    pandn %xmm0, %xmm1
+; X32-SSE-NEXT:    paddb %xmm0, %xmm0
+; X32-SSE-NEXT:    pand %xmm2, %xmm0
+; X32-SSE-NEXT:    por %xmm1, %xmm0
 ; X32-SSE-NEXT:    retl
   %shift = shl <2 x i8> %a, %b
   ret <2 x i8> %shift
@@ -564,88 +783,46 @@ define <2 x i8> @var_shift_v2i8(<2 x i8> %a, <2 x i8> %b) nounwind {
 define <2 x i32> @splatvar_shift_v2i32(<2 x i32> %a, <2 x i32> %b) nounwind {
 ; SSE2-LABEL: splatvar_shift_v2i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    psllq %xmm1, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
-; SSE2-NEXT:    psllq %xmm1, %xmm0
-; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm2[0],xmm0[1]
+; SSE2-NEXT:    xorps %xmm2, %xmm2
+; SSE2-NEXT:    movss {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
+; SSE2-NEXT:    pslld %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: splatvar_shift_v2i32:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
-; SSE41-NEXT:    pxor %xmm2, %xmm2
-; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
-; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    psllq %xmm2, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
-; SSE41-NEXT:    psllq %xmm2, %xmm0
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4,5,6,7]
+; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
+; SSE41-NEXT:    pslld %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: splatvar_shift_v2i32:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
-; AVX1-NEXT:    vpsllq %xmm1, %xmm0, %xmm2
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
-; AVX1-NEXT:    vpsllq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm2[0,1,2,3],xmm0[4,5,6,7]
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: splatvar_shift_v2i32:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpbroadcastq %xmm1, %xmm1
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; AVX2-NEXT:    vpsllvq %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    retq
-;
-; XOPAVX1-LABEL: splatvar_shift_v2i32:
-; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
-; XOPAVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; XOPAVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
-; XOPAVX1-NEXT:    vpshlq %xmm1, %xmm0, %xmm0
-; XOPAVX1-NEXT:    retq
+; AVX-LABEL: splatvar_shift_v2i32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
+; AVX-NEXT:    vpslld %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
 ;
-; XOPAVX2-LABEL: splatvar_shift_v2i32:
-; XOPAVX2:       # %bb.0:
-; XOPAVX2-NEXT:    vpbroadcastq %xmm1, %xmm1
-; XOPAVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; XOPAVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; XOPAVX2-NEXT:    vpsllvq %xmm1, %xmm0, %xmm0
-; XOPAVX2-NEXT:    retq
+; XOP-LABEL: splatvar_shift_v2i32:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
+; XOP-NEXT:    vpslld %xmm1, %xmm0, %xmm0
+; XOP-NEXT:    retq
 ;
 ; AVX512-LABEL: splatvar_shift_v2i32:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpbroadcastq %xmm1, %xmm1
-; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; AVX512-NEXT:    vpsllvq %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
+; AVX512-NEXT:    vpslld %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatvar_shift_v2i32:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpbroadcastq %xmm1, %xmm1
-; AVX512VL-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VL-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; AVX512VL-NEXT:    vpsllvq %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
+; AVX512VL-NEXT:    vpslld %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; X32-SSE-LABEL: splatvar_shift_v2i32:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [4294967295,0,4294967295,0]
-; X32-SSE-NEXT:    pand %xmm1, %xmm2
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm3
-; X32-SSE-NEXT:    psllq %xmm2, %xmm3
-; X32-SSE-NEXT:    pxor %xmm2, %xmm2
+; X32-SSE-NEXT:    xorps %xmm2, %xmm2
 ; X32-SSE-NEXT:    movss {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
-; X32-SSE-NEXT:    psllq %xmm2, %xmm0
-; X32-SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm3[0],xmm0[1]
+; X32-SSE-NEXT:    pslld %xmm2, %xmm0
 ; X32-SSE-NEXT:    retl
   %splat = shufflevector <2 x i32> %b, <2 x i32> undef, <2 x i32> zeroinitializer
   %shift = shl <2 x i32> %a, %splat
@@ -655,90 +832,46 @@ define <2 x i32> @splatvar_shift_v2i32(<2 x i32> %a, <2 x i32> %b) nounwind {
 define <4 x i16> @splatvar_shift_v4i16(<4 x i16> %a, <4 x i16> %b) nounwind {
 ; SSE2-LABEL: splatvar_shift_v4i16:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; SSE2-NEXT:    pslld $23, %xmm1
-; SSE2-NEXT:    paddd {{.*}}(%rip), %xmm1
-; SSE2-NEXT:    cvttps2dq %xmm1, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pmuludq %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pmuludq %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1]
+; SSE2-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; SSE2-NEXT:    psllw %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: splatvar_shift_v4i16:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; SSE41-NEXT:    pslld $23, %xmm1
-; SSE41-NEXT:    paddd {{.*}}(%rip), %xmm1
-; SSE41-NEXT:    cvttps2dq %xmm1, %xmm1
-; SSE41-NEXT:    pmulld %xmm1, %xmm0
+; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; SSE41-NEXT:    psllw %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: splatvar_shift_v4i16:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; AVX1-NEXT:    vpslld $23, %xmm1, %xmm1
-; AVX1-NEXT:    vpaddd {{.*}}(%rip), %xmm1, %xmm1
-; AVX1-NEXT:    vcvttps2dq %xmm1, %xmm1
-; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: splatvar_shift_v4i16:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpbroadcastd %xmm1, %xmm1
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
-; AVX2-NEXT:    vpsllvd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    retq
-;
-; XOPAVX1-LABEL: splatvar_shift_v4i16:
-; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; XOPAVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; XOPAVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
-; XOPAVX1-NEXT:    vpshld %xmm1, %xmm0, %xmm0
-; XOPAVX1-NEXT:    retq
+; AVX-LABEL: splatvar_shift_v4i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX-NEXT:    vpsllw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
 ;
-; XOPAVX2-LABEL: splatvar_shift_v4i16:
-; XOPAVX2:       # %bb.0:
-; XOPAVX2-NEXT:    vpbroadcastd %xmm1, %xmm1
-; XOPAVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; XOPAVX2-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
-; XOPAVX2-NEXT:    vpsllvd %xmm1, %xmm0, %xmm0
-; XOPAVX2-NEXT:    retq
+; XOP-LABEL: splatvar_shift_v4i16:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; XOP-NEXT:    vpsllw %xmm1, %xmm0, %xmm0
+; XOP-NEXT:    retq
 ;
 ; AVX512-LABEL: splatvar_shift_v4i16:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpbroadcastd %xmm1, %xmm1
-; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
-; AVX512-NEXT:    vpsllvd %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512-NEXT:    vpsllw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatvar_shift_v4i16:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpbroadcastd %xmm1, %xmm1
-; AVX512VL-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VL-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
-; AVX512VL-NEXT:    vpsllvd %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512VL-NEXT:    vpsllw %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; X32-SSE-LABEL: splatvar_shift_v4i16:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; X32-SSE-NEXT:    pslld $23, %xmm1
-; X32-SSE-NEXT:    paddd {{\.LCPI.*}}, %xmm1
-; X32-SSE-NEXT:    cvttps2dq %xmm1, %xmm1
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
-; X32-SSE-NEXT:    pmuludq %xmm1, %xmm0
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; X32-SSE-NEXT:    pmuludq %xmm2, %xmm1
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; X32-SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; X32-SSE-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1]
+; X32-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X32-SSE-NEXT:    psllw %xmm1, %xmm0
 ; X32-SSE-NEXT:    retl
   %splat = shufflevector <4 x i16> %b, <4 x i16> undef, <4 x i32> zeroinitializer
   %shift = shl <4 x i16> %a, %splat
@@ -748,87 +881,46 @@ define <4 x i16> @splatvar_shift_v4i16(<4 x i16> %a, <4 x i16> %b) nounwind {
 define <2 x i16> @splatvar_shift_v2i16(<2 x i16> %a, <2 x i16> %b) nounwind {
 ; SSE2-LABEL: splatvar_shift_v2i16:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    psllq %xmm1, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
-; SSE2-NEXT:    psllq %xmm1, %xmm0
-; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm2[0],xmm0[1]
+; SSE2-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1]
+; SSE2-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; SSE2-NEXT:    psllw %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: splatvar_shift_v2i16:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
-; SSE41-NEXT:    pxor %xmm2, %xmm2
-; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3],xmm1[4],xmm2[5,6,7]
-; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    psllq %xmm2, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
-; SSE41-NEXT:    psllq %xmm2, %xmm0
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4,5,6,7]
+; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; SSE41-NEXT:    psllw %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: splatvar_shift_v2i16:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3],xmm1[4],xmm2[5,6,7]
-; AVX1-NEXT:    vpsllq %xmm1, %xmm0, %xmm2
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
-; AVX1-NEXT:    vpsllq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm2[0,1,2,3],xmm0[4,5,6,7]
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: splatvar_shift_v2i16:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpbroadcastq %xmm1, %xmm1
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3],xmm1[4],xmm2[5,6,7]
-; AVX2-NEXT:    vpsllvq %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    retq
-;
-; XOPAVX1-LABEL: splatvar_shift_v2i16:
-; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
-; XOPAVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; XOPAVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3],xmm1[4],xmm2[5,6,7]
-; XOPAVX1-NEXT:    vpshlq %xmm1, %xmm0, %xmm0
-; XOPAVX1-NEXT:    retq
+; AVX-LABEL: splatvar_shift_v2i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX-NEXT:    vpsllw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
 ;
-; XOPAVX2-LABEL: splatvar_shift_v2i16:
-; XOPAVX2:       # %bb.0:
-; XOPAVX2-NEXT:    vpbroadcastq %xmm1, %xmm1
-; XOPAVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; XOPAVX2-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3],xmm1[4],xmm2[5,6,7]
-; XOPAVX2-NEXT:    vpsllvq %xmm1, %xmm0, %xmm0
-; XOPAVX2-NEXT:    retq
+; XOP-LABEL: splatvar_shift_v2i16:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; XOP-NEXT:    vpsllw %xmm1, %xmm0, %xmm0
+; XOP-NEXT:    retq
 ;
 ; AVX512-LABEL: splatvar_shift_v2i16:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpbroadcastq %xmm1, %xmm1
-; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3],xmm1[4],xmm2[5,6,7]
-; AVX512-NEXT:    vpsllvq %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512-NEXT:    vpsllw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatvar_shift_v2i16:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpbroadcastq %xmm1, %xmm1
-; AVX512VL-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VL-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3],xmm1[4],xmm2[5,6,7]
-; AVX512VL-NEXT:    vpsllvq %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512VL-NEXT:    vpsllw %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; X32-SSE-LABEL: splatvar_shift_v2i16:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm1
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm2
-; X32-SSE-NEXT:    psllq %xmm1, %xmm2
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
-; X32-SSE-NEXT:    psllq %xmm1, %xmm0
-; X32-SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm2[0],xmm0[1]
+; X32-SSE-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1]
+; X32-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X32-SSE-NEXT:    psllw %xmm1, %xmm0
 ; X32-SSE-NEXT:    retl
   %splat = shufflevector <2 x i16> %b, <2 x i16> undef, <2 x i32> zeroinitializer
   %shift = shl <2 x i16> %a, %splat
@@ -838,140 +930,114 @@ define <2 x i16> @splatvar_shift_v2i16(<2 x i16> %a, <2 x i16> %b) nounwind {
 define <8 x i8> @splatvar_shift_v8i8(<8 x i8> %a, <8 x i8> %b) nounwind {
 ; SSE2-LABEL: splatvar_shift_v8i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,0,2,3,4,5,6,7]
+; SSE2-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0]
+; SSE2-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; SSE2-NEXT:    psllw %xmm1, %xmm0
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm2
+; SSE2-NEXT:    psllw %xmm1, %xmm2
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,0,2,3,4,5,6,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    movdqa %xmm1, %xmm3
-; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
-; SSE2-NEXT:    pslld $23, %xmm3
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [1065353216,1065353216,1065353216,1065353216]
-; SSE2-NEXT:    paddd %xmm4, %xmm3
-; SSE2-NEXT:    cvttps2dq %xmm3, %xmm3
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm3[0,2,2,3,4,5,6,7]
-; SSE2-NEXT:    pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,6,6,7]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
-; SSE2-NEXT:    pslld $23, %xmm1
-; SSE2-NEXT:    paddd %xmm4, %xmm1
-; SSE2-NEXT:    cvttps2dq %xmm1, %xmm1
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
-; SSE2-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,6,6,7]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
-; SSE2-NEXT:    pmullw %xmm1, %xmm0
+; SSE2-NEXT:    pand %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: splatvar_shift_v8i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pshufb {{.*#+}} xmm1 = xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero
-; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
-; SSE41-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; SSE41-NEXT:    pslld $23, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [1065353216,1065353216,1065353216,1065353216]
-; SSE41-NEXT:    paddd %xmm3, %xmm1
-; SSE41-NEXT:    cvttps2dq %xmm1, %xmm1
-; SSE41-NEXT:    pslld $23, %xmm2
-; SSE41-NEXT:    paddd %xmm3, %xmm2
-; SSE41-NEXT:    cvttps2dq %xmm2, %xmm2
-; SSE41-NEXT:    packusdw %xmm1, %xmm2
-; SSE41-NEXT:    pmullw %xmm2, %xmm0
+; SSE41-NEXT:    pmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; SSE41-NEXT:    psllw %xmm1, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm2, %xmm2
+; SSE41-NEXT:    psllw %xmm1, %xmm2
+; SSE41-NEXT:    pxor %xmm1, %xmm1
+; SSE41-NEXT:    pshufb %xmm1, %xmm2
+; SSE41-NEXT:    pand %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: splatvar_shift_v8i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero
-; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; AVX1-NEXT:    vpslld $23, %xmm2, %xmm2
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [1065353216,1065353216,1065353216,1065353216]
-; AVX1-NEXT:    vpaddd %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vcvttps2dq %xmm2, %xmm2
-; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
-; AVX1-NEXT:    vpslld $23, %xmm1, %xmm1
-; AVX1-NEXT:    vpaddd %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vcvttps2dq %xmm1, %xmm1
-; AVX1-NEXT:    vpackusdw %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX1-NEXT:    vpsllw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpsllw %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: splatvar_shift_v8i8:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero
-; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
-; AVX2-NEXT:    vpsllvd %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
-; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX2-NEXT:    vpsllw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpsllw %xmm1, %xmm2, %xmm1
+; AVX2-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    retq
 ;
-; XOP-LABEL: splatvar_shift_v8i8:
-; XOP:       # %bb.0:
-; XOP-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero
-; XOP-NEXT:    vpshlw %xmm1, %xmm0, %xmm0
-; XOP-NEXT:    retq
+; XOPAVX1-LABEL: splatvar_shift_v8i8:
+; XOPAVX1:       # %bb.0:
+; XOPAVX1-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,0,0,0,4,5,6,7]
+; XOPAVX1-NEXT:    vpshlb %xmm1, %xmm0, %xmm0
+; XOPAVX1-NEXT:    retq
+;
+; XOPAVX2-LABEL: splatvar_shift_v8i8:
+; XOPAVX2:       # %bb.0:
+; XOPAVX2-NEXT:    vpbroadcastb %xmm1, %xmm1
+; XOPAVX2-NEXT:    vpshlb %xmm1, %xmm0, %xmm0
+; XOPAVX2-NEXT:    retq
 ;
 ; AVX512DQ-LABEL: splatvar_shift_v8i8:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero
-; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
-; AVX512DQ-NEXT:    vpsllvd %ymm1, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpmovdw %zmm0, %ymm0
-; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512DQ-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512DQ-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; AVX512DQ-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; AVX512DQ-NEXT:    vpsllvd %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpmovdb %zmm0, %xmm0
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatvar_shift_v8i8:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero
+; AVX512BW-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
 ; AVX512BW-NEXT:    vpsllvw %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512DQVL-LABEL: splatvar_shift_v8i8:
 ; AVX512DQVL:       # %bb.0:
-; AVX512DQVL-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero
-; AVX512DQVL-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX512DQVL-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
-; AVX512DQVL-NEXT:    vpsllvd %ymm1, %ymm0, %ymm0
-; AVX512DQVL-NEXT:    vpmovdw %ymm0, %xmm0
+; AVX512DQVL-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512DQVL-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; AVX512DQVL-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; AVX512DQVL-NEXT:    vpsllvd %zmm1, %zmm0, %zmm0
+; AVX512DQVL-NEXT:    vpmovdb %zmm0, %xmm0
 ; AVX512DQVL-NEXT:    vzeroupper
 ; AVX512DQVL-NEXT:    retq
 ;
 ; AVX512BWVL-LABEL: splatvar_shift_v8i8:
 ; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero,xmm1[0],zero
-; AVX512BWVL-NEXT:    vpsllvw %xmm1, %xmm0, %xmm0
+; AVX512BWVL-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512BWVL-NEXT:    vpsllvw %ymm1, %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm0
+; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
 ;
 ; X32-SSE-LABEL: splatvar_shift_v8i8:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,0,2,3,4,5,6,7]
+; X32-SSE-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0]
+; X32-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X32-SSE-NEXT:    psllw %xmm1, %xmm0
+; X32-SSE-NEXT:    pcmpeqd %xmm2, %xmm2
+; X32-SSE-NEXT:    psllw %xmm1, %xmm2
+; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,0,2,3,4,5,6,7]
 ; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm1
-; X32-SSE-NEXT:    pxor %xmm2, %xmm2
-; X32-SSE-NEXT:    movdqa %xmm1, %xmm3
-; X32-SSE-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
-; X32-SSE-NEXT:    pslld $23, %xmm3
-; X32-SSE-NEXT:    movdqa {{.*#+}} xmm4 = [1065353216,1065353216,1065353216,1065353216]
-; X32-SSE-NEXT:    paddd %xmm4, %xmm3
-; X32-SSE-NEXT:    cvttps2dq %xmm3, %xmm3
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm3 = xmm3[0,2,2,3,4,5,6,7]
-; X32-SSE-NEXT:    pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,6,6,7]
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
-; X32-SSE-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
-; X32-SSE-NEXT:    pslld $23, %xmm1
-; X32-SSE-NEXT:    paddd %xmm4, %xmm1
-; X32-SSE-NEXT:    cvttps2dq %xmm1, %xmm1
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
-; X32-SSE-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,6,6,7]
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; X32-SSE-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
-; X32-SSE-NEXT:    pmullw %xmm1, %xmm0
+; X32-SSE-NEXT:    pand %xmm1, %xmm0
 ; X32-SSE-NEXT:    retl
   %splat = shufflevector <8 x i8> %b, <8 x i8> undef, <8 x i32> zeroinitializer
   %shift = shl <8 x i8> %a, %splat
@@ -981,82 +1047,114 @@ define <8 x i8> @splatvar_shift_v8i8(<8 x i8> %a, <8 x i8> %b) nounwind {
 define <4 x i8> @splatvar_shift_v4i8(<4 x i8> %a, <4 x i8> %b) nounwind {
 ; SSE2-LABEL: splatvar_shift_v4i8:
 ; SSE2:       # %bb.0:
+; SSE2-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0]
+; SSE2-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; SSE2-NEXT:    psllw %xmm1, %xmm0
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm2
+; SSE2-NEXT:    psllw %xmm1, %xmm2
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,0,2,3,4,5,6,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
-; SSE2-NEXT:    pslld $23, %xmm1
-; SSE2-NEXT:    paddd {{.*}}(%rip), %xmm1
-; SSE2-NEXT:    cvttps2dq %xmm1, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pmuludq %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pmuludq %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT:    pand %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: splatvar_shift_v4i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pshufb {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[0],zero,zero,zero,xmm1[0],zero,zero,zero,xmm1[0],zero,zero,zero
-; SSE41-NEXT:    pslld $23, %xmm1
-; SSE41-NEXT:    paddd {{.*}}(%rip), %xmm1
-; SSE41-NEXT:    cvttps2dq %xmm1, %xmm1
-; SSE41-NEXT:    pmulld %xmm1, %xmm0
+; SSE41-NEXT:    pmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; SSE41-NEXT:    psllw %xmm1, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm2, %xmm2
+; SSE41-NEXT:    psllw %xmm1, %xmm2
+; SSE41-NEXT:    pxor %xmm1, %xmm1
+; SSE41-NEXT:    pshufb %xmm1, %xmm2
+; SSE41-NEXT:    pand %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: splatvar_shift_v4i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[0],zero,zero,zero,xmm1[0],zero,zero,zero,xmm1[0],zero,zero,zero
-; AVX1-NEXT:    vpslld $23, %xmm1, %xmm1
-; AVX1-NEXT:    vpaddd {{.*}}(%rip), %xmm1, %xmm1
-; AVX1-NEXT:    vcvttps2dq %xmm1, %xmm1
-; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX1-NEXT:    vpsllw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpsllw %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: splatvar_shift_v4i8:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[0],zero,zero,zero,xmm1[0],zero,zero,zero,xmm1[0],zero,zero,zero
-; AVX2-NEXT:    vpsllvd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX2-NEXT:    vpsllw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpsllw %xmm1, %xmm2, %xmm1
+; AVX2-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: splatvar_shift_v4i8:
 ; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[0],zero,zero,zero,xmm1[0],zero,zero,zero,xmm1[0],zero,zero,zero
-; XOPAVX1-NEXT:    vpshld %xmm1, %xmm0, %xmm0
+; XOPAVX1-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,0,2,3,4,5,6,7]
+; XOPAVX1-NEXT:    vpshlb %xmm1, %xmm0, %xmm0
 ; XOPAVX1-NEXT:    retq
 ;
 ; XOPAVX2-LABEL: splatvar_shift_v4i8:
 ; XOPAVX2:       # %bb.0:
-; XOPAVX2-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[0],zero,zero,zero,xmm1[0],zero,zero,zero,xmm1[0],zero,zero,zero
-; XOPAVX2-NEXT:    vpsllvd %xmm1, %xmm0, %xmm0
+; XOPAVX2-NEXT:    vpbroadcastb %xmm1, %xmm1
+; XOPAVX2-NEXT:    vpshlb %xmm1, %xmm0, %xmm0
 ; XOPAVX2-NEXT:    retq
 ;
-; AVX512-LABEL: splatvar_shift_v4i8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[0],zero,zero,zero,xmm1[0],zero,zero,zero,xmm1[0],zero,zero,zero
-; AVX512-NEXT:    vpsllvd %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    retq
+; AVX512DQ-LABEL: splatvar_shift_v4i8:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512DQ-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; AVX512DQ-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; AVX512DQ-NEXT:    vpsllvd %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    retq
 ;
-; AVX512VL-LABEL: splatvar_shift_v4i8:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[0],zero,zero,zero,xmm1[0],zero,zero,zero,xmm1[0],zero,zero,zero
-; AVX512VL-NEXT:    vpsllvd %xmm1, %xmm0, %xmm0
-; AVX512VL-NEXT:    retq
+; AVX512BW-LABEL: splatvar_shift_v4i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512BW-NEXT:    vpsllvw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512DQVL-LABEL: splatvar_shift_v4i8:
+; AVX512DQVL:       # %bb.0:
+; AVX512DQVL-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512DQVL-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; AVX512DQVL-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; AVX512DQVL-NEXT:    vpsllvd %zmm1, %zmm0, %zmm0
+; AVX512DQVL-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512DQVL-NEXT:    vzeroupper
+; AVX512DQVL-NEXT:    retq
+;
+; AVX512BWVL-LABEL: splatvar_shift_v4i8:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512BWVL-NEXT:    vpsllvw %ymm1, %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm0
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
 ;
 ; X32-SSE-LABEL: splatvar_shift_v4i8:
 ; X32-SSE:       # %bb.0:
+; X32-SSE-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0]
+; X32-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X32-SSE-NEXT:    psllw %xmm1, %xmm0
+; X32-SSE-NEXT:    pcmpeqd %xmm2, %xmm2
+; X32-SSE-NEXT:    psllw %xmm1, %xmm2
+; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,0,2,3,4,5,6,7]
 ; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm1
-; X32-SSE-NEXT:    pslld $23, %xmm1
-; X32-SSE-NEXT:    paddd {{\.LCPI.*}}, %xmm1
-; X32-SSE-NEXT:    cvttps2dq %xmm1, %xmm1
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
-; X32-SSE-NEXT:    pmuludq %xmm1, %xmm0
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; X32-SSE-NEXT:    pmuludq %xmm2, %xmm1
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; X32-SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; X32-SSE-NEXT:    pand %xmm1, %xmm0
 ; X32-SSE-NEXT:    retl
   %splat = shufflevector <4 x i8> %b, <4 x i8> undef, <4 x i32> zeroinitializer
   %shift = shl <4 x i8> %a, %splat
@@ -1066,73 +1164,107 @@ define <4 x i8> @splatvar_shift_v4i8(<4 x i8> %a, <4 x i8> %b) nounwind {
 define <2 x i8> @splatvar_shift_v2i8(<2 x i8> %a, <2 x i8> %b) nounwind {
 ; SSE2-LABEL: splatvar_shift_v2i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    psllq %xmm1, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
-; SSE2-NEXT:    psllq %xmm1, %xmm0
-; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm2[0],xmm0[1]
+; SSE2-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0]
+; SSE2-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; SSE2-NEXT:    psllw %xmm1, %xmm0
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm2
+; SSE2-NEXT:    psllw %xmm1, %xmm2
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,0,2,3,4,5,6,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
+; SSE2-NEXT:    pand %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: splatvar_shift_v2i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pshufb {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[0],zero,zero,zero,zero,zero,zero,zero
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    psllq %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
-; SSE41-NEXT:    psllq %xmm1, %xmm0
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm2[0,1,2,3],xmm0[4,5,6,7]
+; SSE41-NEXT:    pmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; SSE41-NEXT:    psllw %xmm1, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm2, %xmm2
+; SSE41-NEXT:    psllw %xmm1, %xmm2
+; SSE41-NEXT:    pxor %xmm1, %xmm1
+; SSE41-NEXT:    pshufb %xmm1, %xmm2
+; SSE41-NEXT:    pand %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: splatvar_shift_v2i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[0],zero,zero,zero,zero,zero,zero,zero
-; AVX1-NEXT:    vpsllq %xmm1, %xmm0, %xmm2
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
-; AVX1-NEXT:    vpsllq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm2[0,1,2,3],xmm0[4,5,6,7]
+; AVX1-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX1-NEXT:    vpsllw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpsllw %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: splatvar_shift_v2i8:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[0],zero,zero,zero,zero,zero,zero,zero
-; AVX2-NEXT:    vpsllvq %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX2-NEXT:    vpsllw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpsllw %xmm1, %xmm2, %xmm1
+; AVX2-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    retq
 ;
-; XOPAVX1-LABEL: splatvar_shift_v2i8:
-; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[0],zero,zero,zero,zero,zero,zero,zero
-; XOPAVX1-NEXT:    vpshlq %xmm1, %xmm0, %xmm0
-; XOPAVX1-NEXT:    retq
+; XOP-LABEL: splatvar_shift_v2i8:
+; XOP:       # %bb.0:
+; XOP-NEXT:    insertq {{.*#+}} xmm1 = xmm1[0,0,2,3,4,5,6,7,u,u,u,u,u,u,u,u]
+; XOP-NEXT:    vpshlb %xmm1, %xmm0, %xmm0
+; XOP-NEXT:    retq
 ;
-; XOPAVX2-LABEL: splatvar_shift_v2i8:
-; XOPAVX2:       # %bb.0:
-; XOPAVX2-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[0],zero,zero,zero,zero,zero,zero,zero
-; XOPAVX2-NEXT:    vpsllvq %xmm1, %xmm0, %xmm0
-; XOPAVX2-NEXT:    retq
+; AVX512DQ-LABEL: splatvar_shift_v2i8:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512DQ-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; AVX512DQ-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; AVX512DQ-NEXT:    vpsllvd %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    retq
 ;
-; AVX512-LABEL: splatvar_shift_v2i8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[0],zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT:    vpsllvq %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    retq
+; AVX512BW-LABEL: splatvar_shift_v2i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512BW-NEXT:    vpsllvw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
 ;
-; AVX512VL-LABEL: splatvar_shift_v2i8:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[0],zero,zero,zero,zero,zero,zero,zero
-; AVX512VL-NEXT:    vpsllvq %xmm1, %xmm0, %xmm0
-; AVX512VL-NEXT:    retq
+; AVX512DQVL-LABEL: splatvar_shift_v2i8:
+; AVX512DQVL:       # %bb.0:
+; AVX512DQVL-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512DQVL-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; AVX512DQVL-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; AVX512DQVL-NEXT:    vpsllvd %zmm1, %zmm0, %zmm0
+; AVX512DQVL-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512DQVL-NEXT:    vzeroupper
+; AVX512DQVL-NEXT:    retq
+;
+; AVX512BWVL-LABEL: splatvar_shift_v2i8:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512BWVL-NEXT:    vpsllvw %ymm1, %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm0
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
 ;
 ; X32-SSE-LABEL: splatvar_shift_v2i8:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm1
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm2
-; X32-SSE-NEXT:    psllq %xmm1, %xmm2
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
-; X32-SSE-NEXT:    psllq %xmm1, %xmm0
-; X32-SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm2[0],xmm0[1]
+; X32-SSE-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0]
+; X32-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X32-SSE-NEXT:    psllw %xmm1, %xmm0
+; X32-SSE-NEXT:    pcmpeqd %xmm2, %xmm2
+; X32-SSE-NEXT:    psllw %xmm1, %xmm2
+; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,0,2,3,4,5,6,7]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
+; X32-SSE-NEXT:    pand %xmm1, %xmm0
 ; X32-SSE-NEXT:    retl
   %splat = shufflevector <2 x i8> %b, <2 x i8> undef, <2 x i32> zeroinitializer
   %shift = shl <2 x i8> %a, %splat
@@ -1147,120 +1279,109 @@ define <2 x i32> @constant_shift_v2i32(<2 x i32> %a) nounwind {
 ; SSE2-LABEL: constant_shift_v2i32:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    psllq $4, %xmm1
-; SSE2-NEXT:    psllq $5, %xmm0
-; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; SSE2-NEXT:    pslld $4, %xmm1
+; SSE2-NEXT:    pslld $5, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: constant_shift_v2i32:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    psllq $5, %xmm1
-; SSE41-NEXT:    psllq $4, %xmm0
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4,5,6,7]
+; SSE41-NEXT:    pslld $5, %xmm1
+; SSE41-NEXT:    pslld $4, %xmm0
+; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5,6,7]
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: constant_shift_v2i32:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpsllq $5, %xmm0, %xmm1
-; AVX1-NEXT:    vpsllq $4, %xmm0, %xmm0
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4,5,6,7]
+; AVX1-NEXT:    vpslld $5, %xmm0, %xmm1
+; AVX1-NEXT:    vpslld $4, %xmm0, %xmm0
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5,6,7]
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: constant_shift_v2i32:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpsllvq {{.*}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT:    vpsllvd {{.*}}(%rip), %xmm0, %xmm0
 ; AVX2-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: constant_shift_v2i32:
 ; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vpshlq {{.*}}(%rip), %xmm0, %xmm0
+; XOPAVX1-NEXT:    vpshld {{.*}}(%rip), %xmm0, %xmm0
 ; XOPAVX1-NEXT:    retq
 ;
 ; XOPAVX2-LABEL: constant_shift_v2i32:
 ; XOPAVX2:       # %bb.0:
-; XOPAVX2-NEXT:    vpsllvq {{.*}}(%rip), %xmm0, %xmm0
+; XOPAVX2-NEXT:    vpsllvd {{.*}}(%rip), %xmm0, %xmm0
 ; XOPAVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: constant_shift_v2i32:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsllvq {{.*}}(%rip), %xmm0, %xmm0
+; AVX512-NEXT:    vpsllvd {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512-NEXT:    retq
 ;
 ; AVX512VL-LABEL: constant_shift_v2i32:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsllvq {{.*}}(%rip), %xmm0, %xmm0
+; AVX512VL-NEXT:    vpsllvd {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; X32-SSE-LABEL: constant_shift_v2i32:
 ; X32-SSE:       # %bb.0:
 ; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
-; X32-SSE-NEXT:    psllq $4, %xmm1
-; X32-SSE-NEXT:    psllq $5, %xmm0
-; X32-SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; X32-SSE-NEXT:    pslld $4, %xmm1
+; X32-SSE-NEXT:    pslld $5, %xmm0
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; X32-SSE-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; X32-SSE-NEXT:    movdqa %xmm1, %xmm0
 ; X32-SSE-NEXT:    retl
   %shift = shl <2 x i32> %a, <i32 4, i32 5>
   ret <2 x i32> %shift
 }
 
 define <4 x i16> @constant_shift_v4i16(<4 x i16> %a) nounwind {
-; SSE2-LABEL: constant_shift_v4i16:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [1,2,4,8]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pmuludq %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pmuludq %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; SSE2-NEXT:    retq
-;
-; SSE41-LABEL: constant_shift_v4i16:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm0
-; SSE41-NEXT:    retq
+; SSE-LABEL: constant_shift_v4i16:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pmullw {{.*}}(%rip), %xmm0
+; SSE-NEXT:    retq
 ;
-; AVX1-LABEL: constant_shift_v4i16:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
-; AVX1-NEXT:    retq
+; AVX-LABEL: constant_shift_v4i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpmullw {{.*}}(%rip), %xmm0, %xmm0
+; AVX-NEXT:    retq
 ;
-; AVX2-LABEL: constant_shift_v4i16:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpsllvd {{.*}}(%rip), %xmm0, %xmm0
-; AVX2-NEXT:    retq
+; XOP-LABEL: constant_shift_v4i16:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vpshlw {{.*}}(%rip), %xmm0, %xmm0
+; XOP-NEXT:    retq
 ;
-; XOPAVX1-LABEL: constant_shift_v4i16:
-; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vpshld {{.*}}(%rip), %xmm0, %xmm0
-; XOPAVX1-NEXT:    retq
+; AVX512DQ-LABEL: constant_shift_v4i16:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vpmullw {{.*}}(%rip), %xmm0, %xmm0
+; AVX512DQ-NEXT:    retq
 ;
-; XOPAVX2-LABEL: constant_shift_v4i16:
-; XOPAVX2:       # %bb.0:
-; XOPAVX2-NEXT:    vpsllvd {{.*}}(%rip), %xmm0, %xmm0
-; XOPAVX2-NEXT:    retq
+; AVX512BW-LABEL: constant_shift_v4i16:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm1 = <0,1,2,3,u,u,u,u>
+; AVX512BW-NEXT:    vpsllvw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
 ;
-; AVX512-LABEL: constant_shift_v4i16:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsllvd {{.*}}(%rip), %xmm0, %xmm0
-; AVX512-NEXT:    retq
+; AVX512DQVL-LABEL: constant_shift_v4i16:
+; AVX512DQVL:       # %bb.0:
+; AVX512DQVL-NEXT:    vpmullw {{.*}}(%rip), %xmm0, %xmm0
+; AVX512DQVL-NEXT:    retq
 ;
-; AVX512VL-LABEL: constant_shift_v4i16:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsllvd {{.*}}(%rip), %xmm0, %xmm0
-; AVX512VL-NEXT:    retq
+; AVX512BWVL-LABEL: constant_shift_v4i16:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpsllvw {{.*}}(%rip), %xmm0, %xmm0
+; AVX512BWVL-NEXT:    retq
 ;
 ; X32-SSE-LABEL: constant_shift_v4i16:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    movdqa {{.*#+}} xmm1 = [1,2,4,8]
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
-; X32-SSE-NEXT:    pmuludq %xmm1, %xmm0
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; X32-SSE-NEXT:    pmuludq %xmm2, %xmm1
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; X32-SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; X32-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
 ; X32-SSE-NEXT:    retl
   %shift = shl <4 x i16> %a, <i16 0, i16 1, i16 2, i16 3>
   ret <4 x i16> %shift
@@ -1269,106 +1390,149 @@ define <4 x i16> @constant_shift_v4i16(<4 x i16> %a) nounwind {
 define <2 x i16> @constant_shift_v2i16(<2 x i16> %a) nounwind {
 ; SSE2-LABEL: constant_shift_v2i16:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    psllq $2, %xmm1
-; SSE2-NEXT:    psllq $3, %xmm0
-; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; SSE2-NEXT:    pmullw {{.*}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: constant_shift_v2i16:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    psllq $3, %xmm1
-; SSE41-NEXT:    psllq $2, %xmm0
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4,5,6,7]
+; SSE41-NEXT:    psllw $3, %xmm1
+; SSE41-NEXT:    psllw $2, %xmm0
+; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3,4,5,6,7]
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: constant_shift_v2i16:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpsllq $3, %xmm0, %xmm1
-; AVX1-NEXT:    vpsllq $2, %xmm0, %xmm0
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4,5,6,7]
-; AVX1-NEXT:    retq
+; AVX-LABEL: constant_shift_v2i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpsllw $3, %xmm0, %xmm1
+; AVX-NEXT:    vpsllw $2, %xmm0, %xmm0
+; AVX-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3,4,5,6,7]
+; AVX-NEXT:    retq
 ;
-; AVX2-LABEL: constant_shift_v2i16:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpsllvq {{.*}}(%rip), %xmm0, %xmm0
-; AVX2-NEXT:    retq
+; XOP-LABEL: constant_shift_v2i16:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vpshlw {{.*}}(%rip), %xmm0, %xmm0
+; XOP-NEXT:    retq
 ;
-; XOPAVX1-LABEL: constant_shift_v2i16:
-; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vpshlq {{.*}}(%rip), %xmm0, %xmm0
-; XOPAVX1-NEXT:    retq
+; AVX512DQ-LABEL: constant_shift_v2i16:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vpsllw $3, %xmm0, %xmm1
+; AVX512DQ-NEXT:    vpsllw $2, %xmm0, %xmm0
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3,4,5,6,7]
+; AVX512DQ-NEXT:    retq
 ;
-; XOPAVX2-LABEL: constant_shift_v2i16:
-; XOPAVX2:       # %bb.0:
-; XOPAVX2-NEXT:    vpsllvq {{.*}}(%rip), %xmm0, %xmm0
-; XOPAVX2-NEXT:    retq
+; AVX512BW-LABEL: constant_shift_v2i16:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm1 = <2,3,u,u,u,u,u,u>
+; AVX512BW-NEXT:    vpsllvw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
 ;
-; AVX512-LABEL: constant_shift_v2i16:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsllvq {{.*}}(%rip), %xmm0, %xmm0
-; AVX512-NEXT:    retq
+; AVX512DQVL-LABEL: constant_shift_v2i16:
+; AVX512DQVL:       # %bb.0:
+; AVX512DQVL-NEXT:    vpsllw $3, %xmm0, %xmm1
+; AVX512DQVL-NEXT:    vpsllw $2, %xmm0, %xmm0
+; AVX512DQVL-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3,4,5,6,7]
+; AVX512DQVL-NEXT:    retq
 ;
-; AVX512VL-LABEL: constant_shift_v2i16:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsllvq {{.*}}(%rip), %xmm0, %xmm0
-; AVX512VL-NEXT:    retq
+; AVX512BWVL-LABEL: constant_shift_v2i16:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpsllvw {{.*}}(%rip), %xmm0, %xmm0
+; AVX512BWVL-NEXT:    retq
 ;
 ; X32-SSE-LABEL: constant_shift_v2i16:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
-; X32-SSE-NEXT:    psllq $2, %xmm1
-; X32-SSE-NEXT:    psllq $3, %xmm0
-; X32-SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; X32-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
 ; X32-SSE-NEXT:    retl
   %shift = shl <2 x i16> %a, <i16 2, i16 3>
   ret <2 x i16> %shift
 }
 
 define <8 x i8> @constant_shift_v8i8(<8 x i8> %a) nounwind {
-; SSE-LABEL: constant_shift_v8i8:
-; SSE:       # %bb.0:
-; SSE-NEXT:    pmullw {{.*}}(%rip), %xmm0
-; SSE-NEXT:    retq
+; SSE2-LABEL: constant_shift_v8i8:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    pmullw {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    packuswb %xmm1, %xmm0
+; SSE2-NEXT:    retq
 ;
-; AVX-LABEL: constant_shift_v8i8:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vpmullw {{.*}}(%rip), %xmm0, %xmm0
-; AVX-NEXT:    retq
+; SSE41-LABEL: constant_shift_v8i8:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; SSE41-NEXT:    pmullw {{.*}}(%rip), %xmm0
+; SSE41-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE41-NEXT:    pxor %xmm1, %xmm1
+; SSE41-NEXT:    packuswb %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: constant_shift_v8i8:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX1-NEXT:    vpmullw {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: constant_shift_v8i8:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX2-NEXT:    vpmullw {{.*}}(%rip), %ymm0, %ymm0
+; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
 ;
 ; XOP-LABEL: constant_shift_v8i8:
 ; XOP:       # %bb.0:
-; XOP-NEXT:    vpshlw {{.*}}(%rip), %xmm0, %xmm0
+; XOP-NEXT:    vpshlb {{.*}}(%rip), %xmm0, %xmm0
 ; XOP-NEXT:    retq
 ;
 ; AVX512DQ-LABEL: constant_shift_v8i8:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vpmullw {{.*}}(%rip), %xmm0, %xmm0
+; AVX512DQ-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; AVX512DQ-NEXT:    vpsllvd {{.*}}(%rip), %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512BW-LABEL: constant_shift_v8i8:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm1 = [0,1,2,3,4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,1,2,3,4,5,6,7,0,0,0,0,0,0,0,0]
+; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
 ; AVX512BW-NEXT:    vpsllvw %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512DQVL-LABEL: constant_shift_v8i8:
 ; AVX512DQVL:       # %bb.0:
-; AVX512DQVL-NEXT:    vpmullw {{.*}}(%rip), %xmm0, %xmm0
+; AVX512DQVL-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; AVX512DQVL-NEXT:    vpsllvd {{.*}}(%rip), %zmm0, %zmm0
+; AVX512DQVL-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512DQVL-NEXT:    vzeroupper
 ; AVX512DQVL-NEXT:    retq
 ;
 ; AVX512BWVL-LABEL: constant_shift_v8i8:
 ; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vpsllvw {{.*}}(%rip), %xmm0, %xmm0
+; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512BWVL-NEXT:    vpsllvw {{.*}}(%rip), %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm0
+; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
 ;
 ; X32-SSE-LABEL: constant_shift_v8i8:
 ; X32-SSE:       # %bb.0:
+; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; X32-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
+; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X32-SSE-NEXT:    pxor %xmm1, %xmm1
+; X32-SSE-NEXT:    packuswb %xmm1, %xmm0
 ; X32-SSE-NEXT:    retl
   %shift = shl <8 x i8> %a, <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7>
   ret <8 x i8> %shift
@@ -1377,61 +1541,87 @@ define <8 x i8> @constant_shift_v8i8(<8 x i8> %a) nounwind {
 define <4 x i8> @constant_shift_v4i8(<4 x i8> %a) nounwind {
 ; SSE2-LABEL: constant_shift_v4i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [1,2,4,8]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pmuludq %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pmuludq %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    pmullw {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    packuswb %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: constant_shift_v4i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm0
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; SSE41-NEXT:    pmullw {{.*}}(%rip), %xmm0
+; SSE41-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE41-NEXT:    pxor %xmm1, %xmm1
+; SSE41-NEXT:    packuswb %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: constant_shift_v4i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX1-NEXT:    vpmullw {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: constant_shift_v4i8:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpsllvd {{.*}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX2-NEXT:    vpmullw {{.*}}(%rip), %ymm0, %ymm0
+; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
-; XOPAVX1-LABEL: constant_shift_v4i8:
-; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vpshld {{.*}}(%rip), %xmm0, %xmm0
-; XOPAVX1-NEXT:    retq
+; XOP-LABEL: constant_shift_v4i8:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vpshlb {{.*}}(%rip), %xmm0, %xmm0
+; XOP-NEXT:    retq
 ;
-; XOPAVX2-LABEL: constant_shift_v4i8:
-; XOPAVX2:       # %bb.0:
-; XOPAVX2-NEXT:    vpsllvd {{.*}}(%rip), %xmm0, %xmm0
-; XOPAVX2-NEXT:    retq
+; AVX512DQ-LABEL: constant_shift_v4i8:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; AVX512DQ-NEXT:    vpsllvd {{.*}}(%rip), %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    retq
 ;
-; AVX512-LABEL: constant_shift_v4i8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsllvd {{.*}}(%rip), %xmm0, %xmm0
-; AVX512-NEXT:    retq
+; AVX512BW-LABEL: constant_shift_v4i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,1,2,3,0,0,0,0,0,0,0,0,0,0,0,0]
+; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512BW-NEXT:    vpsllvw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
 ;
-; AVX512VL-LABEL: constant_shift_v4i8:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsllvd {{.*}}(%rip), %xmm0, %xmm0
-; AVX512VL-NEXT:    retq
+; AVX512DQVL-LABEL: constant_shift_v4i8:
+; AVX512DQVL:       # %bb.0:
+; AVX512DQVL-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; AVX512DQVL-NEXT:    vpsllvd {{.*}}(%rip), %zmm0, %zmm0
+; AVX512DQVL-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512DQVL-NEXT:    vzeroupper
+; AVX512DQVL-NEXT:    retq
+;
+; AVX512BWVL-LABEL: constant_shift_v4i8:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512BWVL-NEXT:    vpsllvw {{.*}}(%rip), %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm0
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
 ;
 ; X32-SSE-LABEL: constant_shift_v4i8:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    movdqa {{.*#+}} xmm1 = [1,2,4,8]
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
-; X32-SSE-NEXT:    pmuludq %xmm1, %xmm0
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; X32-SSE-NEXT:    pmuludq %xmm2, %xmm1
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; X32-SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X32-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
+; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X32-SSE-NEXT:    pxor %xmm1, %xmm1
+; X32-SSE-NEXT:    packuswb %xmm1, %xmm0
 ; X32-SSE-NEXT:    retl
   %shift = shl <4 x i8> %a, <i8 0, i8 1, i8 2, i8 3>
   ret <4 x i8> %shift
@@ -1440,58 +1630,87 @@ define <4 x i8> @constant_shift_v4i8(<4 x i8> %a) nounwind {
 define <2 x i8> @constant_shift_v2i8(<2 x i8> %a) nounwind {
 ; SSE2-LABEL: constant_shift_v2i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    psllq $2, %xmm1
-; SSE2-NEXT:    psllq $3, %xmm0
-; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    pmullw {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    packuswb %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: constant_shift_v2i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    psllq $3, %xmm1
-; SSE41-NEXT:    psllq $2, %xmm0
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4,5,6,7]
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; SSE41-NEXT:    pmullw {{.*}}(%rip), %xmm0
+; SSE41-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE41-NEXT:    pxor %xmm1, %xmm1
+; SSE41-NEXT:    packuswb %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: constant_shift_v2i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpsllq $3, %xmm0, %xmm1
-; AVX1-NEXT:    vpsllq $2, %xmm0, %xmm0
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4,5,6,7]
+; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX1-NEXT:    vpmullw {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: constant_shift_v2i8:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpsllvq {{.*}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX2-NEXT:    vpmullw {{.*}}(%rip), %ymm0, %ymm0
+; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
-; XOPAVX1-LABEL: constant_shift_v2i8:
-; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vpshlq {{.*}}(%rip), %xmm0, %xmm0
-; XOPAVX1-NEXT:    retq
+; XOP-LABEL: constant_shift_v2i8:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vpshlb {{.*}}(%rip), %xmm0, %xmm0
+; XOP-NEXT:    retq
 ;
-; XOPAVX2-LABEL: constant_shift_v2i8:
-; XOPAVX2:       # %bb.0:
-; XOPAVX2-NEXT:    vpsllvq {{.*}}(%rip), %xmm0, %xmm0
-; XOPAVX2-NEXT:    retq
+; AVX512DQ-LABEL: constant_shift_v2i8:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; AVX512DQ-NEXT:    vpsllvd {{.*}}(%rip), %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    retq
 ;
-; AVX512-LABEL: constant_shift_v2i8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsllvq {{.*}}(%rip), %xmm0, %xmm0
-; AVX512-NEXT:    retq
+; AVX512BW-LABEL: constant_shift_v2i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm1 = [2,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512BW-NEXT:    vpsllvw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
 ;
-; AVX512VL-LABEL: constant_shift_v2i8:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsllvq {{.*}}(%rip), %xmm0, %xmm0
-; AVX512VL-NEXT:    retq
+; AVX512DQVL-LABEL: constant_shift_v2i8:
+; AVX512DQVL:       # %bb.0:
+; AVX512DQVL-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; AVX512DQVL-NEXT:    vpsllvd {{.*}}(%rip), %zmm0, %zmm0
+; AVX512DQVL-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512DQVL-NEXT:    vzeroupper
+; AVX512DQVL-NEXT:    retq
+;
+; AVX512BWVL-LABEL: constant_shift_v2i8:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512BWVL-NEXT:    vpsllvw {{.*}}(%rip), %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm0
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
 ;
 ; X32-SSE-LABEL: constant_shift_v2i8:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
-; X32-SSE-NEXT:    psllq $2, %xmm1
-; X32-SSE-NEXT:    psllq $3, %xmm0
-; X32-SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X32-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
+; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X32-SSE-NEXT:    pxor %xmm1, %xmm1
+; X32-SSE-NEXT:    packuswb %xmm1, %xmm0
 ; X32-SSE-NEXT:    retl
   %shift = shl <2 x i8> %a, <i8 2, i8 3>
   ret <2 x i8> %shift
@@ -1504,32 +1723,32 @@ define <2 x i8> @constant_shift_v2i8(<2 x i8> %a) nounwind {
 define <2 x i32> @splatconstant_shift_v2i32(<2 x i32> %a) nounwind {
 ; SSE-LABEL: splatconstant_shift_v2i32:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    psllq $5, %xmm0
+; SSE-NEXT:    pslld $5, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: splatconstant_shift_v2i32:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpsllq $5, %xmm0, %xmm0
+; AVX-NEXT:    vpslld $5, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; XOP-LABEL: splatconstant_shift_v2i32:
 ; XOP:       # %bb.0:
-; XOP-NEXT:    vpsllq $5, %xmm0, %xmm0
+; XOP-NEXT:    vpslld $5, %xmm0, %xmm0
 ; XOP-NEXT:    retq
 ;
 ; AVX512-LABEL: splatconstant_shift_v2i32:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsllq $5, %xmm0, %xmm0
+; AVX512-NEXT:    vpslld $5, %xmm0, %xmm0
 ; AVX512-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatconstant_shift_v2i32:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsllq $5, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpslld $5, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; X32-SSE-LABEL: splatconstant_shift_v2i32:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    psllq $5, %xmm0
+; X32-SSE-NEXT:    pslld $5, %xmm0
 ; X32-SSE-NEXT:    retl
   %shift = shl <2 x i32> %a, <i32 5, i32 5>
   ret <2 x i32> %shift
@@ -1538,32 +1757,32 @@ define <2 x i32> @splatconstant_shift_v2i32(<2 x i32> %a) nounwind {
 define <4 x i16> @splatconstant_shift_v4i16(<4 x i16> %a) nounwind {
 ; SSE-LABEL: splatconstant_shift_v4i16:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pslld $3, %xmm0
+; SSE-NEXT:    psllw $3, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: splatconstant_shift_v4i16:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpslld $3, %xmm0, %xmm0
+; AVX-NEXT:    vpsllw $3, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; XOP-LABEL: splatconstant_shift_v4i16:
 ; XOP:       # %bb.0:
-; XOP-NEXT:    vpslld $3, %xmm0, %xmm0
+; XOP-NEXT:    vpsllw $3, %xmm0, %xmm0
 ; XOP-NEXT:    retq
 ;
 ; AVX512-LABEL: splatconstant_shift_v4i16:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpslld $3, %xmm0, %xmm0
+; AVX512-NEXT:    vpsllw $3, %xmm0, %xmm0
 ; AVX512-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatconstant_shift_v4i16:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpslld $3, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpsllw $3, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; X32-SSE-LABEL: splatconstant_shift_v4i16:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pslld $3, %xmm0
+; X32-SSE-NEXT:    psllw $3, %xmm0
 ; X32-SSE-NEXT:    retl
   %shift = shl <4 x i16> %a, <i16 3, i16 3, i16 3, i16 3>
   ret <4 x i16> %shift
@@ -1572,32 +1791,32 @@ define <4 x i16> @splatconstant_shift_v4i16(<4 x i16> %a) nounwind {
 define <2 x i16> @splatconstant_shift_v2i16(<2 x i16> %a) nounwind {
 ; SSE-LABEL: splatconstant_shift_v2i16:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    psllq $3, %xmm0
+; SSE-NEXT:    psllw $3, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: splatconstant_shift_v2i16:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpsllq $3, %xmm0, %xmm0
+; AVX-NEXT:    vpsllw $3, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; XOP-LABEL: splatconstant_shift_v2i16:
 ; XOP:       # %bb.0:
-; XOP-NEXT:    vpsllq $3, %xmm0, %xmm0
+; XOP-NEXT:    vpsllw $3, %xmm0, %xmm0
 ; XOP-NEXT:    retq
 ;
 ; AVX512-LABEL: splatconstant_shift_v2i16:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsllq $3, %xmm0, %xmm0
+; AVX512-NEXT:    vpsllw $3, %xmm0, %xmm0
 ; AVX512-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatconstant_shift_v2i16:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsllq $3, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpsllw $3, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; X32-SSE-LABEL: splatconstant_shift_v2i16:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    psllq $3, %xmm0
+; X32-SSE-NEXT:    psllw $3, %xmm0
 ; X32-SSE-NEXT:    retl
   %shift = shl <2 x i16> %a, <i16 3, i16 3>
   ret <2 x i16> %shift
@@ -1607,31 +1826,36 @@ define <8 x i8> @splatconstant_shift_v8i8(<8 x i8> %a) nounwind {
 ; SSE-LABEL: splatconstant_shift_v8i8:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    psllw $3, %xmm0
+; SSE-NEXT:    pand {{.*}}(%rip), %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: splatconstant_shift_v8i8:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpsllw $3, %xmm0, %xmm0
+; AVX-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; XOP-LABEL: splatconstant_shift_v8i8:
 ; XOP:       # %bb.0:
-; XOP-NEXT:    vpsllw $3, %xmm0, %xmm0
+; XOP-NEXT:    vpshlb {{.*}}(%rip), %xmm0, %xmm0
 ; XOP-NEXT:    retq
 ;
 ; AVX512-LABEL: splatconstant_shift_v8i8:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpsllw $3, %xmm0, %xmm0
+; AVX512-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatconstant_shift_v8i8:
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vpsllw $3, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; X32-SSE-LABEL: splatconstant_shift_v8i8:
 ; X32-SSE:       # %bb.0:
 ; X32-SSE-NEXT:    psllw $3, %xmm0
+; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
 ; X32-SSE-NEXT:    retl
   %shift = shl <8 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
   ret <8 x i8> %shift
@@ -1640,32 +1864,37 @@ define <8 x i8> @splatconstant_shift_v8i8(<8 x i8> %a) nounwind {
 define <4 x i8> @splatconstant_shift_v4i8(<4 x i8> %a) nounwind {
 ; SSE-LABEL: splatconstant_shift_v4i8:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pslld $3, %xmm0
+; SSE-NEXT:    psllw $3, %xmm0
+; SSE-NEXT:    pand {{.*}}(%rip), %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: splatconstant_shift_v4i8:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpslld $3, %xmm0, %xmm0
+; AVX-NEXT:    vpsllw $3, %xmm0, %xmm0
+; AVX-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; XOP-LABEL: splatconstant_shift_v4i8:
 ; XOP:       # %bb.0:
-; XOP-NEXT:    vpslld $3, %xmm0, %xmm0
+; XOP-NEXT:    vpshlb {{.*}}(%rip), %xmm0, %xmm0
 ; XOP-NEXT:    retq
 ;
 ; AVX512-LABEL: splatconstant_shift_v4i8:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpslld $3, %xmm0, %xmm0
+; AVX512-NEXT:    vpsllw $3, %xmm0, %xmm0
+; AVX512-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatconstant_shift_v4i8:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpslld $3, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpsllw $3, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; X32-SSE-LABEL: splatconstant_shift_v4i8:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pslld $3, %xmm0
+; X32-SSE-NEXT:    psllw $3, %xmm0
+; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
 ; X32-SSE-NEXT:    retl
   %shift = shl <4 x i8> %a, <i8 3, i8 3, i8 3, i8 3>
   ret <4 x i8> %shift
@@ -1674,32 +1903,37 @@ define <4 x i8> @splatconstant_shift_v4i8(<4 x i8> %a) nounwind {
 define <2 x i8> @splatconstant_shift_v2i8(<2 x i8> %a) nounwind {
 ; SSE-LABEL: splatconstant_shift_v2i8:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    psllq $3, %xmm0
+; SSE-NEXT:    psllw $3, %xmm0
+; SSE-NEXT:    pand {{.*}}(%rip), %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: splatconstant_shift_v2i8:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpsllq $3, %xmm0, %xmm0
+; AVX-NEXT:    vpsllw $3, %xmm0, %xmm0
+; AVX-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; XOP-LABEL: splatconstant_shift_v2i8:
 ; XOP:       # %bb.0:
-; XOP-NEXT:    vpsllq $3, %xmm0, %xmm0
+; XOP-NEXT:    vpshlb {{.*}}(%rip), %xmm0, %xmm0
 ; XOP-NEXT:    retq
 ;
 ; AVX512-LABEL: splatconstant_shift_v2i8:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsllq $3, %xmm0, %xmm0
+; AVX512-NEXT:    vpsllw $3, %xmm0, %xmm0
+; AVX512-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatconstant_shift_v2i8:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsllq $3, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpsllw $3, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; X32-SSE-LABEL: splatconstant_shift_v2i8:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    psllq $3, %xmm0
+; X32-SSE-NEXT:    psllw $3, %xmm0
+; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
 ; X32-SSE-NEXT:    retl
   %shift = shl <2 x i8> %a, <i8 3, i8 3>
   ret <2 x i8> %shift
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-128-v16.ll b/llvm/test/CodeGen/X86/vector-shuffle-128-v16.ll
index 4df979adb..21fe12a45 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-128-v16.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-128-v16.ll
@@ -841,26 +841,14 @@ entry:
 }
 
 define <16 x i8> @PR20540(<8 x i8> %a) {
-; SSE2-LABEL: PR20540:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
-; SSE2-NEXT:    packuswb %xmm0, %xmm0
-; SSE2-NEXT:    movq {{.*#+}} xmm0 = xmm0[0],zero
-; SSE2-NEXT:    retq
-;
-; SSSE3-LABEL: PR20540:
-; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14],zero,zero,zero,zero,zero,zero,zero,zero
-; SSSE3-NEXT:    retq
-;
-; SSE41-LABEL: PR20540:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14],zero,zero,zero,zero,zero,zero,zero,zero
-; SSE41-NEXT:    retq
+; SSE-LABEL: PR20540:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movq {{.*#+}} xmm0 = xmm0[0],zero
+; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: PR20540:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14],zero,zero,zero,zero,zero,zero,zero,zero
+; AVX-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; AVX-NEXT:    retq
   %shuffle = shufflevector <8 x i8> %a, <8 x i8> zeroinitializer, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8>
   ret <16 x i8> %shuffle
@@ -1205,7 +1193,7 @@ define <16 x i8> @shuffle_v16i8_00_uu_uu_uu_uu_uu_uu_uu_01_uu_uu_uu_uu_uu_uu_uu(
 ;
 ; SSSE3-LABEL: shuffle_v16i8_00_uu_uu_uu_uu_uu_uu_uu_01_uu_uu_uu_uu_uu_uu_uu:
 ; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
+; SSSE3-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,u,u,u,u,u,u,u,1,u,u,u,u,u,u,u]
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: shuffle_v16i8_00_uu_uu_uu_uu_uu_uu_uu_01_uu_uu_uu_uu_uu_uu_uu:
@@ -1688,7 +1676,7 @@ define <16 x i8> @shuffle_v16i8_zz_zz_zz_zz_zz_zz_zz_zz_zz_zz_01_02_03_04_05_06(
 define <16 x i8> @PR12412(<16 x i8> %inval1, <16 x i8> %inval2) {
 ; SSE2-LABEL: PR12412:
 ; SSE2:       # %bb.0: # %entry
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
 ; SSE2-NEXT:    pand %xmm2, %xmm1
 ; SSE2-NEXT:    pand %xmm2, %xmm0
 ; SSE2-NEXT:    packuswb %xmm1, %xmm0
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-128-v2.ll b/llvm/test/CodeGen/X86/vector-shuffle-128-v2.ll
index 2a5ac3ebc..02b6dff3b 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-128-v2.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-128-v2.ll
@@ -718,20 +718,23 @@ define <2 x i64> @shuffle_v2i64_z0(<2 x i64> %a) {
 define <2 x i64> @shuffle_v2i64_z1(<2 x i64> %a) {
 ; SSE2-LABEL: shuffle_v2i64_z1:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    xorpd %xmm1, %xmm1
-; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; SSE2-NEXT:    xorps %xmm1, %xmm1
+; SSE2-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
+; SSE2-NEXT:    movaps %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: shuffle_v2i64_z1:
 ; SSE3:       # %bb.0:
-; SSE3-NEXT:    xorpd %xmm1, %xmm1
-; SSE3-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; SSE3-NEXT:    xorps %xmm1, %xmm1
+; SSE3-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
+; SSE3-NEXT:    movaps %xmm1, %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: shuffle_v2i64_z1:
 ; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    xorpd %xmm1, %xmm1
-; SSSE3-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; SSSE3-NEXT:    xorps %xmm1, %xmm1
+; SSSE3-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
+; SSSE3-NEXT:    movaps %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: shuffle_v2i64_z1:
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-128-v4.ll b/llvm/test/CodeGen/X86/vector-shuffle-128-v4.ll
index f4a4bb5e6..4b012e73f 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-128-v4.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-128-v4.ll
@@ -354,17 +354,11 @@ define <4 x i32> @shuffle_v4i32_0124(<4 x i32> %a, <4 x i32> %b) {
 ; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5],xmm1[6,7]
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: shuffle_v4i32_0124:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[0,1,2,0]
-; AVX1-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[3]
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: shuffle_v4i32_0124:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vbroadcastss %xmm1, %xmm1
-; AVX2-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[3]
-; AVX2-NEXT:    retq
+; AVX1OR2-LABEL: shuffle_v4i32_0124:
+; AVX1OR2:       # %bb.0:
+; AVX1OR2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[0,1,2,0]
+; AVX1OR2-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[3]
+; AVX1OR2-NEXT:    retq
 ;
 ; AVX512VL-LABEL: shuffle_v4i32_0124:
 ; AVX512VL:       # %bb.0:
@@ -451,19 +445,12 @@ define <4 x i32> @shuffle_v4i32_0412(<4 x i32> %a, <4 x i32> %b) {
 ; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5,6,7]
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: shuffle_v4i32_0412:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[0,0,1,1]
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,1,1,2]
-; AVX1-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3]
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: shuffle_v4i32_0412:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vbroadcastss %xmm1, %xmm1
-; AVX2-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,1,1,2]
-; AVX2-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3]
-; AVX2-NEXT:    retq
+; AVX1OR2-LABEL: shuffle_v4i32_0412:
+; AVX1OR2:       # %bb.0:
+; AVX1OR2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[0,0,1,1]
+; AVX1OR2-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,1,1,2]
+; AVX1OR2-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3]
+; AVX1OR2-NEXT:    retq
 ;
 ; AVX512VL-LABEL: shuffle_v4i32_0412:
 ; AVX512VL:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-128-v8.ll b/llvm/test/CodeGen/X86/vector-shuffle-128-v8.ll
index c756fe7d1..95669b32b 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-128-v8.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-128-v8.ll
@@ -1571,7 +1571,7 @@ define <8 x i16> @shuffle_v8i16_XXX1X579(<8 x i16> %a, <8 x i16> %b) {
 ;
 ; AVX2-SLOW-LABEL: shuffle_v8i16_XXX1X579:
 ; AVX2-SLOW:       # %bb.0:
-; AVX2-SLOW-NEXT:    vpbroadcastd %xmm1, %xmm1
+; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,1,2,0]
 ; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,1,2,1,4,5,6,7]
 ; AVX2-SLOW-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,7,7]
 ; AVX2-SLOW-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,6],xmm1[7]
@@ -1579,7 +1579,7 @@ define <8 x i16> @shuffle_v8i16_XXX1X579(<8 x i16> %a, <8 x i16> %b) {
 ;
 ; AVX2-FAST-LABEL: shuffle_v8i16_XXX1X579:
 ; AVX2-FAST:       # %bb.0:
-; AVX2-FAST-NEXT:    vpbroadcastd %xmm1, %xmm1
+; AVX2-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,1,2,0]
 ; AVX2-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,2,3,8,9,10,11,14,15,14,15]
 ; AVX2-FAST-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,6],xmm1[7]
 ; AVX2-FAST-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-256-v32.ll b/llvm/test/CodeGen/X86/vector-shuffle-256-v32.ll
index 8dd211b55..31c71354e 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-256-v32.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-256-v32.ll
@@ -3106,16 +3106,14 @@ define <4 x i64> @PR28136(<32 x i8> %a0, <32 x i8> %a1) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
 ; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [8,8,10,10,12,12,14,14,9,9,11,11,13,13,15,15]
-; AVX1-NEXT:    vpshufb %xmm2, %xmm0, %xmm3
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm4
-; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3],xmm1[4],xmm4[4],xmm1[5],xmm4[5],xmm1[6],xmm4[6],xmm1[7],xmm4[7]
-; AVX1-NEXT:    vpshufb %xmm2, %xmm1, %xmm2
+; AVX1-NEXT:    vpshufb {{.*#+}} xmm2 = xmm0[8,u,10,u,12,u,14,u,9,u,11,u,13,u,15,u]
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3],xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7]
+; AVX1-NEXT:    vpshufb {{.*#+}} xmm3 = xmm1[u,8,u,10,u,12,u,14,u,9,u,11,u,13,u,15]
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
-; AVX1-NEXT:    vpblendvb %xmm4, %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,0,2,2,4,4,6,6,1,1,3,3,5,5,7,7]
-; AVX1-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vpblendvb %xmm4, %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,u,2,u,4,u,6,u,1,u,3,u,5,u,7,u]
+; AVX1-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[u,0,u,2,u,4,u,6,u,1,u,3,u,5,u,7]
 ; AVX1-NEXT:    vpblendvb %xmm4, %xmm0, %xmm1, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-256-v4.ll b/llvm/test/CodeGen/X86/vector-shuffle-256-v4.ll
index 8fc0eabbc..5195f5f0e 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-256-v4.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-256-v4.ll
@@ -48,10 +48,8 @@ define <4 x double> @shuffle_v4f64_0001(<4 x double> %a, <4 x double> %b) {
 define <4 x double> @shuffle_v4f64_0020(<4 x double> %a, <4 x double> %b) {
 ; AVX1-LABEL: shuffle_v4f64_0020:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vmovlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0]
-; AVX1-NEXT:    vmovddup {{.*#+}} xmm0 = xmm0[0,0]
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm1
+; AVX1-NEXT:    vunpcklpd {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: shuffle_v4f64_0020:
@@ -71,8 +69,8 @@ define <4 x double> @shuffle_v4f64_0300(<4 x double> %a, <4 x double> %b) {
 ; AVX1-LABEL: shuffle_v4f64_0300:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm0[2,3,0,1]
-; AVX1-NEXT:    vpermilpd {{.*#+}} ymm1 = ymm1[0,1,2,2]
-; AVX1-NEXT:    vblendpd {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3]
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
+; AVX1-NEXT:    vshufpd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[2]
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: shuffle_v4f64_0300:
@@ -194,8 +192,8 @@ define <4 x double> @shuffle_v4f64_3330(<4 x double> %a, <4 x double> %b) {
 ; AVX1-LABEL: shuffle_v4f64_3330:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm0[2,3,0,1]
-; AVX1-NEXT:    vblendpd {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3]
-; AVX1-NEXT:    vpermilpd {{.*#+}} ymm0 = ymm0[1,1,3,2]
+; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,2,3]
+; AVX1-NEXT:    vshufpd {{.*#+}} ymm0 = ymm0[1],ymm1[1],ymm0[3],ymm1[2]
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: shuffle_v4f64_3330:
@@ -299,9 +297,9 @@ define <4 x double> @shuffle_v4f64_1022(<4 x double> %a, <4 x double> %b) {
 define <4 x double> @shuffle_v4f64_0213(<4 x double> %a, <4 x double> %b) {
 ; AVX1-LABEL: shuffle_v4f64_0213:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm0[2,3,0,1]
-; AVX1-NEXT:    vpermilpd {{.*#+}} ymm1 = ymm1[0,0,3,2]
-; AVX1-NEXT:    vblendpd {{.*#+}} ymm0 = ymm0[0],ymm1[1,2],ymm0[3]
+; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm0[2,3,2,3]
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
+; AVX1-NEXT:    vshufpd {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[3],ymm1[3]
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: shuffle_v4f64_0213:
@@ -625,75 +623,45 @@ define <4 x double> @shuffle_v4f64_3333(<4 x double> %a, <4 x double> %b) {
   ret <4 x double> %shuffle
 }
 
-define <4 x double> @shuffle_v4f64_0z3z(<4 x double> %a, <4 x double> %b) {
-; AVX1-LABEL: shuffle_v4f64_0z3z:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpermilpd {{.*#+}} ymm0 = ymm0[0,0,3,2]
-; AVX1-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vblendpd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3]
-; AVX1-NEXT:    retq
-;
-; AVX2-SLOW-LABEL: shuffle_v4f64_0z3z:
-; AVX2-SLOW:       # %bb.0:
-; AVX2-SLOW-NEXT:    vpermilpd {{.*#+}} ymm0 = ymm0[0,0,3,2]
-; AVX2-SLOW-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
-; AVX2-SLOW-NEXT:    vblendpd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3]
-; AVX2-SLOW-NEXT:    retq
-;
-; AVX2-FAST-LABEL: shuffle_v4f64_0z3z:
-; AVX2-FAST:       # %bb.0:
-; AVX2-FAST-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,6,7],zero,zero,zero,zero,zero,zero,zero,zero,ymm0[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero
-; AVX2-FAST-NEXT:    retq
+define <4 x double> @shuffle_v4f64_0456(<4 x double> %a, <4 x double> %b) {
+; AVX1OR2-LABEL: shuffle_v4f64_0456:
+; AVX1OR2:       # %bb.0:
+; AVX1OR2-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1OR2-NEXT:    vshufpd {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[3],ymm1[2]
+; AVX1OR2-NEXT:    retq
 ;
-; AVX512VL-SLOW-LABEL: shuffle_v4f64_0z3z:
+; AVX512VL-SLOW-LABEL: shuffle_v4f64_0456:
 ; AVX512VL-SLOW:       # %bb.0:
-; AVX512VL-SLOW-NEXT:    vpermilpd {{.*#+}} ymm0 = ymm0[0,0,3,2]
-; AVX512VL-SLOW-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
-; AVX512VL-SLOW-NEXT:    vblendpd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3]
+; AVX512VL-SLOW-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX512VL-SLOW-NEXT:    vshufpd {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[3],ymm1[2]
 ; AVX512VL-SLOW-NEXT:    retq
 ;
-; AVX512VL-FAST-LABEL: shuffle_v4f64_0z3z:
+; AVX512VL-FAST-LABEL: shuffle_v4f64_0456:
 ; AVX512VL-FAST:       # %bb.0:
-; AVX512VL-FAST-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,6,7],zero,zero,zero,zero,zero,zero,zero,zero,ymm0[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512VL-FAST-NEXT:    vmovapd {{.*#+}} ymm2 = [4,0,1,2]
+; AVX512VL-FAST-NEXT:    vpermi2pd %ymm0, %ymm1, %ymm2
+; AVX512VL-FAST-NEXT:    vmovapd %ymm2, %ymm0
 ; AVX512VL-FAST-NEXT:    retq
+  %shuffle = shufflevector <4 x double> %a, <4 x double> %b, <4 x i32> <i32 0, i32 4, i32 5, i32 6>
+  ret <4 x double> %shuffle
+}
+
+define <4 x double> @shuffle_v4f64_0z3z(<4 x double> %a, <4 x double> %b) {
+; ALL-LABEL: shuffle_v4f64_0z3z:
+; ALL:       # %bb.0:
+; ALL-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; ALL-NEXT:    vshufpd {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[3],ymm1[2]
+; ALL-NEXT:    retq
   %shuffle = shufflevector <4 x double> %a, <4 x double> <double 0.000000e+00, double undef, double undef, double undef>, <4 x i32> <i32 0, i32 4, i32 3, i32 4>
   ret <4 x double> %shuffle
 }
 
 define <4 x double> @shuffle_v4f64_1z2z(<4 x double> %a, <4 x double> %b) {
-; AVX1-LABEL: shuffle_v4f64_1z2z:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vblendpd {{.*#+}} ymm0 = ymm1[0],ymm0[1,2,3]
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm1[0],xmm0[0]
-; AVX1-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX1-NEXT:    retq
-;
-; AVX2-SLOW-LABEL: shuffle_v4f64_1z2z:
-; AVX2-SLOW:       # %bb.0:
-; AVX2-SLOW-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[1,0,2,0]
-; AVX2-SLOW-NEXT:    retq
-;
-; AVX2-FAST-LABEL: shuffle_v4f64_1z2z:
-; AVX2-FAST:       # %bb.0:
-; AVX2-FAST-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm0[16,17,18,19,20,21,22,23],zero,zero,zero,zero,zero,zero,zero,zero
-; AVX2-FAST-NEXT:    retq
-;
-; AVX512VL-SLOW-LABEL: shuffle_v4f64_1z2z:
-; AVX512VL-SLOW:       # %bb.0:
-; AVX512VL-SLOW-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; AVX512VL-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3,4,5,6,7]
-; AVX512VL-SLOW-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[1,0,2,0]
-; AVX512VL-SLOW-NEXT:    retq
-;
-; AVX512VL-FAST-LABEL: shuffle_v4f64_1z2z:
-; AVX512VL-FAST:       # %bb.0:
-; AVX512VL-FAST-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm0[16,17,18,19,20,21,22,23],zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512VL-FAST-NEXT:    retq
+; ALL-LABEL: shuffle_v4f64_1z2z:
+; ALL:       # %bb.0:
+; ALL-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; ALL-NEXT:    vshufpd {{.*#+}} ymm0 = ymm0[1],ymm1[0],ymm0[2],ymm1[2]
+; ALL-NEXT:    retq
   %1 = shufflevector <4 x double> %a, <4 x double> <double 0.000000e+00, double undef, double undef, double undef>, <4 x i32> <i32 1, i32 4, i32 2, i32 4>
   ret <4 x double> %1
 }
@@ -807,10 +775,8 @@ define <4 x i64> @shuffle_v4i64_0001(<4 x i64> %a, <4 x i64> %b) {
 define <4 x i64> @shuffle_v4i64_0020(<4 x i64> %a, <4 x i64> %b) {
 ; AVX1-LABEL: shuffle_v4i64_0020:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vmovlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0]
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,1,0,1]
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm1
+; AVX1-NEXT:    vunpcklpd {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: shuffle_v4i64_0020:
@@ -829,9 +795,8 @@ define <4 x i64> @shuffle_v4i64_0020(<4 x i64> %a, <4 x i64> %b) {
 define <4 x i64> @shuffle_v4i64_0112(<4 x i64> %a, <4 x i64> %b) {
 ; AVX1-LABEL: shuffle_v4i64_0112:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vpalignr {{.*#+}} xmm1 = xmm0[8,9,10,11,12,13,14,15],xmm1[0,1,2,3,4,5,6,7]
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm1
+; AVX1-NEXT:    vshufpd {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[3],ymm0[2]
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: shuffle_v4i64_0112:
@@ -851,8 +816,8 @@ define <4 x i64> @shuffle_v4i64_0300(<4 x i64> %a, <4 x i64> %b) {
 ; AVX1-LABEL: shuffle_v4i64_0300:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm0[2,3,0,1]
-; AVX1-NEXT:    vpermilpd {{.*#+}} ymm1 = ymm1[0,1,2,2]
-; AVX1-NEXT:    vblendpd {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3]
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
+; AVX1-NEXT:    vshufpd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[2]
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: shuffle_v4i64_0300:
@@ -912,8 +877,8 @@ define <4 x i64> @shuffle_v4i64_3330(<4 x i64> %a, <4 x i64> %b) {
 ; AVX1-LABEL: shuffle_v4i64_3330:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm0[2,3,0,1]
-; AVX1-NEXT:    vblendpd {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3]
-; AVX1-NEXT:    vpermilpd {{.*#+}} ymm0 = ymm0[1,1,3,2]
+; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,2,3]
+; AVX1-NEXT:    vshufpd {{.*#+}} ymm0 = ymm0[1],ymm1[1],ymm0[3],ymm1[2]
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: shuffle_v4i64_3330:
@@ -952,9 +917,9 @@ define <4 x i64> @shuffle_v4i64_3210(<4 x i64> %a, <4 x i64> %b) {
 define <4 x i64> @shuffle_v4i64_0213(<4 x i64> %a, <4 x i64> %b) {
 ; AVX1-LABEL: shuffle_v4i64_0213:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm0[2,3,0,1]
-; AVX1-NEXT:    vpermilpd {{.*#+}} ymm1 = ymm1[0,0,3,2]
-; AVX1-NEXT:    vblendpd {{.*#+}} ymm0 = ymm0[0],ymm1[1,2],ymm0[3]
+; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm0[2,3,2,3]
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
+; AVX1-NEXT:    vshufpd {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[3],ymm1[3]
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: shuffle_v4i64_0213:
@@ -973,9 +938,8 @@ define <4 x i64> @shuffle_v4i64_0213(<4 x i64> %a, <4 x i64> %b) {
 define <4 x i64> @shuffle_v4i64_0124(<4 x i64> %a, <4 x i64> %b) {
 ; AVX1-LABEL: shuffle_v4i64_0124:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[0,1,0,1]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1
-; AVX1-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
+; AVX1-NEXT:    vshufpd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[2]
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: shuffle_v4i64_0124:
@@ -1003,8 +967,7 @@ define <4 x i64> @shuffle_v4i64_0142(<4 x i64> %a, <4 x i64> %b) {
 ; AVX1-LABEL: shuffle_v4i64_0142:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1
-; AVX1-NEXT:    vpermilpd {{.*#+}} ymm0 = ymm0[0,1,2,2]
-; AVX1-NEXT:    vblendpd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2],ymm0[3]
+; AVX1-NEXT:    vshufpd {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[2]
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: shuffle_v4i64_0142:
@@ -1026,11 +989,9 @@ define <4 x i64> @shuffle_v4i64_0142(<4 x i64> %a, <4 x i64> %b) {
 define <4 x i64> @shuffle_v4i64_0412(<4 x i64> %a, <4 x i64> %b) {
 ; AVX1-LABEL: shuffle_v4i64_0412:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[0,1,0,1]
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX1-NEXT:    vpalignr {{.*#+}} xmm2 = xmm0[8,9,10,11,12,13,14,15],xmm2[0,1,2,3,4,5,6,7]
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; AVX1-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5,6,7]
+; AVX1-NEXT:    vblendpd {{.*#+}} ymm1 = ymm1[0,1],ymm0[2],ymm1[3]
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
+; AVX1-NEXT:    vshufpd {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[3],ymm1[2]
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: shuffle_v4i64_0412:
@@ -1052,11 +1013,8 @@ define <4 x i64> @shuffle_v4i64_0412(<4 x i64> %a, <4 x i64> %b) {
 define <4 x i64> @shuffle_v4i64_4012(<4 x i64> %a, <4 x i64> %b) {
 ; AVX1-LABEL: shuffle_v4i64_4012:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX1-NEXT:    vpalignr {{.*#+}} xmm2 = xmm0[8,9,10,11,12,13,14,15],xmm2[0,1,2,3,4,5,6,7]
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; AVX1-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3,4,5,6,7]
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm1
+; AVX1-NEXT:    vshufpd {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[3],ymm0[2]
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: shuffle_v4i64_4012:
@@ -1149,9 +1107,9 @@ define <4 x i64> @shuffle_v4i64_4015(<4 x i64> %a, <4 x i64> %b) {
 define <4 x i64> @shuffle_v4i64_2u35(<4 x i64> %a, <4 x i64> %b) {
 ; AVX1-LABEL: shuffle_v4i64_2u35:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT:    vunpckhpd {{.*#+}} xmm1 = xmm0[1],xmm1[1]
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,2,3]
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1
+; AVX1-NEXT:    vshufpd {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[3],ymm1[3]
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: shuffle_v4i64_2u35:
@@ -1179,10 +1137,8 @@ define <4 x i64> @shuffle_v4i64_1251(<4 x i64> %a, <4 x i64> %b) {
 ; AVX1-LABEL: shuffle_v4i64_1251:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm2 = ymm0[2,3,0,1]
-; AVX1-NEXT:    vshufpd {{.*#+}} ymm0 = ymm0[1],ymm2[0],ymm0[2],ymm2[3]
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[2,3,0,1]
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1
-; AVX1-NEXT:    vblendpd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2],ymm0[3]
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    vshufpd {{.*#+}} ymm0 = ymm0[1],ymm2[0],ymm0[3],ymm2[3]
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: shuffle_v4i64_1251:
@@ -1744,9 +1700,8 @@ define <4 x i64> @shuffle_v4i64_1230(<4 x i64> %a) {
 define <4 x i64> @shuffle_v4i64_z0z3(<4 x i64> %a, <4 x i64> %b) {
 ; AVX1-LABEL: shuffle_v4i64_z0z3:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpermilpd {{.*#+}} ymm0 = ymm0[0,0,2,3]
 ; AVX1-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vblendpd {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[3]
+; AVX1-NEXT:    vshufpd {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[2],ymm0[3]
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-SLOW-LABEL: shuffle_v4i64_z0z3:
@@ -1780,11 +1735,7 @@ define <4 x i64> @shuffle_v4i64_1z2z(<4 x i64> %a, <4 x i64> %b) {
 ; AVX1-LABEL: shuffle_v4i64_1z2z:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vblendpd {{.*#+}} ymm0 = ymm1[0],ymm0[1,2,3]
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm1[0],xmm0[0]
-; AVX1-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    vshufpd {{.*#+}} ymm0 = ymm0[1],ymm1[0],ymm0[2],ymm1[2]
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-SLOW-LABEL: shuffle_v4i64_1z2z:
@@ -1815,39 +1766,12 @@ define <4 x i64> @shuffle_v4i64_1z2z(<4 x i64> %a, <4 x i64> %b) {
 }
 
 define <4 x double> @add_v4f64_0246_1357(<4 x double> %a, <4 x double> %b) {
-; AVX1-LABEL: add_v4f64_0246_1357:
-; AVX1:       # %bb.0: # %entry
-; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm2 = ymm0[2,3],ymm1[2,3]
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX1-NEXT:    vhaddpd %ymm2, %ymm0, %ymm0
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: add_v4f64_0246_1357:
-; AVX2:       # %bb.0: # %entry
-; AVX2-NEXT:    vunpcklpd {{.*#+}} ymm2 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
-; AVX2-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[0,2,1,3]
-; AVX2-NEXT:    vunpckhpd {{.*#+}} ymm0 = ymm0[1],ymm1[1],ymm0[3],ymm1[3]
-; AVX2-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX2-NEXT:    vaddpd %ymm0, %ymm2, %ymm0
-; AVX2-NEXT:    retq
-;
-; AVX512VL-SLOW-LABEL: add_v4f64_0246_1357:
-; AVX512VL-SLOW:       # %bb.0: # %entry
-; AVX512VL-SLOW-NEXT:    vunpcklpd {{.*#+}} ymm2 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
-; AVX512VL-SLOW-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[0,2,1,3]
-; AVX512VL-SLOW-NEXT:    vunpckhpd {{.*#+}} ymm0 = ymm0[1],ymm1[1],ymm0[3],ymm1[3]
-; AVX512VL-SLOW-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX512VL-SLOW-NEXT:    vaddpd %ymm0, %ymm2, %ymm0
-; AVX512VL-SLOW-NEXT:    retq
-;
-; AVX512VL-FAST-LABEL: add_v4f64_0246_1357:
-; AVX512VL-FAST:       # %bb.0: # %entry
-; AVX512VL-FAST-NEXT:    vmovapd {{.*#+}} ymm2 = [0,2,4,6]
-; AVX512VL-FAST-NEXT:    vpermi2pd %ymm1, %ymm0, %ymm2
-; AVX512VL-FAST-NEXT:    vmovapd {{.*#+}} ymm3 = [1,3,5,7]
-; AVX512VL-FAST-NEXT:    vpermi2pd %ymm1, %ymm0, %ymm3
-; AVX512VL-FAST-NEXT:    vaddpd %ymm3, %ymm2, %ymm0
-; AVX512VL-FAST-NEXT:    retq
+; ALL-LABEL: add_v4f64_0246_1357:
+; ALL:       # %bb.0: # %entry
+; ALL-NEXT:    vperm2f128 {{.*#+}} ymm2 = ymm0[2,3],ymm1[2,3]
+; ALL-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; ALL-NEXT:    vhaddpd %ymm2, %ymm0, %ymm0
+; ALL-NEXT:    retq
 entry:
   %shuffle = shufflevector <4 x double> %a, <4 x double> %b, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
   %shuffle1 = shufflevector <4 x double> %a, <4 x double> %b, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
@@ -1856,39 +1780,12 @@ entry:
 }
 
 define <4 x double> @add_v4f64_4602_5713(<4 x double> %a, <4 x double> %b) {
-; AVX1-LABEL: add_v4f64_4602_5713:
-; AVX1:       # %bb.0: # %entry
-; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm2 = ymm1[2,3],ymm0[2,3]
-; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; AVX1-NEXT:    vhaddpd %ymm2, %ymm0, %ymm0
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: add_v4f64_4602_5713:
-; AVX2:       # %bb.0: # %entry
-; AVX2-NEXT:    vunpcklpd {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
-; AVX2-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[0,2,1,3]
-; AVX2-NEXT:    vunpckhpd {{.*#+}} ymm0 = ymm1[1],ymm0[1],ymm1[3],ymm0[3]
-; AVX2-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX2-NEXT:    vaddpd %ymm0, %ymm2, %ymm0
-; AVX2-NEXT:    retq
-;
-; AVX512VL-SLOW-LABEL: add_v4f64_4602_5713:
-; AVX512VL-SLOW:       # %bb.0: # %entry
-; AVX512VL-SLOW-NEXT:    vunpcklpd {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
-; AVX512VL-SLOW-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[0,2,1,3]
-; AVX512VL-SLOW-NEXT:    vunpckhpd {{.*#+}} ymm0 = ymm1[1],ymm0[1],ymm1[3],ymm0[3]
-; AVX512VL-SLOW-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX512VL-SLOW-NEXT:    vaddpd %ymm0, %ymm2, %ymm0
-; AVX512VL-SLOW-NEXT:    retq
-;
-; AVX512VL-FAST-LABEL: add_v4f64_4602_5713:
-; AVX512VL-FAST:       # %bb.0: # %entry
-; AVX512VL-FAST-NEXT:    vmovapd {{.*#+}} ymm2 = [0,2,4,6]
-; AVX512VL-FAST-NEXT:    vpermi2pd %ymm0, %ymm1, %ymm2
-; AVX512VL-FAST-NEXT:    vmovapd {{.*#+}} ymm3 = [1,3,5,7]
-; AVX512VL-FAST-NEXT:    vpermi2pd %ymm0, %ymm1, %ymm3
-; AVX512VL-FAST-NEXT:    vaddpd %ymm3, %ymm2, %ymm0
-; AVX512VL-FAST-NEXT:    retq
+; ALL-LABEL: add_v4f64_4602_5713:
+; ALL:       # %bb.0: # %entry
+; ALL-NEXT:    vperm2f128 {{.*#+}} ymm2 = ymm1[2,3],ymm0[2,3]
+; ALL-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; ALL-NEXT:    vhaddpd %ymm2, %ymm0, %ymm0
+; ALL-NEXT:    retq
 entry:
   %shuffle = shufflevector <4 x double> %a, <4 x double> %b, <4 x i32> <i32 4, i32 6, i32 0, i32 2>
   %shuffle1 = shufflevector <4 x double> %a, <4 x double> %b, <4 x i32> <i32 5, i32 7, i32 1, i32 3>
@@ -2028,6 +1925,56 @@ define <8 x i32> @shuffle_v8i32_0zzzzzzz_optsize(<8 x i32> %a) optsize {
   ret <8 x i32> %b
 }
 
+define <4 x double> @shuffle_v4f64_0zzz_pgso(<4 x double> %a) !prof !14 {
+; ALL-LABEL: shuffle_v4f64_0zzz_pgso:
+; ALL:       # %bb.0:
+; ALL-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
+; ALL-NEXT:    retq
+  %b = shufflevector <4 x double> %a, <4 x double> zeroinitializer, <4 x i32> <i32 0, i32 5, i32 6, i32 7>
+  ret <4 x double> %b
+}
+
+define <4 x i64> @shuffle_v4i64_0zzz_pgso(<4 x i64> %a) !prof !14 {
+; ALL-LABEL: shuffle_v4i64_0zzz_pgso:
+; ALL:       # %bb.0:
+; ALL-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
+; ALL-NEXT:    retq
+  %b = shufflevector <4 x i64> %a, <4 x i64> zeroinitializer, <4 x i32> <i32 0, i32 5, i32 6, i32 7>
+  ret <4 x i64> %b
+}
+
+define <8 x float> @shuffle_v8f32_0zzzzzzz_pgso(<8 x float> %a) !prof !14 {
+; AVX1OR2-LABEL: shuffle_v8f32_0zzzzzzz_pgso:
+; AVX1OR2:       # %bb.0:
+; AVX1OR2-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; AVX1OR2-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX1OR2-NEXT:    retq
+;
+; AVX512VL-LABEL: shuffle_v8f32_0zzzzzzz_pgso:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; AVX512VL-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX512VL-NEXT:    retq
+  %b = shufflevector <8 x float> %a, <8 x float> zeroinitializer, <8 x i32> <i32 0, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  ret <8 x float> %b
+}
+
+define <8 x i32> @shuffle_v8i32_0zzzzzzz_pgso(<8 x i32> %a) !prof !14 {
+; AVX1OR2-LABEL: shuffle_v8i32_0zzzzzzz_pgso:
+; AVX1OR2:       # %bb.0:
+; AVX1OR2-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; AVX1OR2-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX1OR2-NEXT:    retq
+;
+; AVX512VL-LABEL: shuffle_v8i32_0zzzzzzz_pgso:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; AVX512VL-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX512VL-NEXT:    retq
+  %b = shufflevector <8 x i32> %a, <8 x i32> zeroinitializer, <8 x i32> <i32 0, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  ret <8 x i32> %b
+}
+
 define <4 x i64> @unpckh_v4i64(<4 x i64> %x, <4 x i64> %y) {
 ; ALL-LABEL: unpckh_v4i64:
 ; ALL:       # %bb.0:
@@ -2048,3 +1995,19 @@ define <4 x double> @unpckh_v4f64(<4 x double> %x, <4 x double> %y) {
   ret <4 x double> %unpckh
 }
 
+!llvm.module.flags = !{!0}
+!0 = !{i32 1, !"ProfileSummary", !1}
+!1 = !{!2, !3, !4, !5, !6, !7, !8, !9}
+!2 = !{!"ProfileFormat", !"InstrProf"}
+!3 = !{!"TotalCount", i64 10000}
+!4 = !{!"MaxCount", i64 10}
+!5 = !{!"MaxInternalCount", i64 1}
+!6 = !{!"MaxFunctionCount", i64 1000}
+!7 = !{!"NumCounts", i64 3}
+!8 = !{!"NumFunctions", i64 3}
+!9 = !{!"DetailedSummary", !10}
+!10 = !{!11, !12, !13}
+!11 = !{i32 10000, i64 100, i32 1}
+!12 = !{i32 999000, i64 100, i32 1}
+!13 = !{i32 999999, i64 1, i32 2}
+!14 = !{!"function_entry_count", i64 0}
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-256-v8.ll b/llvm/test/CodeGen/X86/vector-shuffle-256-v8.ll
index f4a11f443..99429560d 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-256-v8.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-256-v8.ll
@@ -1151,15 +1151,15 @@ define <8 x float> @shuffle_v8f32_084c195d(<8 x float> %a, <8 x float> %b) {
   ret <8 x float> %shuffle
 }
 
-define <8 x float> @shuffle_v8f32_01452367d(<8 x float> %a) {
-; AVX1-LABEL: shuffle_v8f32_01452367d:
+define <8 x float> @shuffle_v8f32_01452367(<8 x float> %a) {
+; AVX1-LABEL: shuffle_v8f32_01452367:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm0[2,3,0,1]
-; AVX1-NEXT:    vpermilpd {{.*#+}} ymm1 = ymm1[0,0,3,2]
-; AVX1-NEXT:    vblendpd {{.*#+}} ymm0 = ymm0[0],ymm1[1,2],ymm0[3]
+; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm0[2,3,2,3]
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
+; AVX1-NEXT:    vshufpd {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[3],ymm1[3]
 ; AVX1-NEXT:    retq
 ;
-; AVX2OR512VL-LABEL: shuffle_v8f32_01452367d:
+; AVX2OR512VL-LABEL: shuffle_v8f32_01452367:
 ; AVX2OR512VL:       # %bb.0:
 ; AVX2OR512VL-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2OR512VL-NEXT:    retq
@@ -1167,6 +1167,57 @@ define <8 x float> @shuffle_v8f32_01452367d(<8 x float> %a) {
   ret <8 x float> %shuffle
 }
 
+define <8 x float> @shuffle_v8f32_089abcde(<8 x float> %a, <8 x float> %b) {
+; AVX1-LABEL: shuffle_v8f32_089abcde:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm2 = xmm1[0,0,1,2]
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
+; AVX1-NEXT:    vblendps {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[3]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[3,0,1,2]
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
+; AVX1-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3,4,5,6,7]
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: shuffle_v8f32_089abcde:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vmovaps {{.*#+}} ymm2 = <u,0,1,2,3,4,5,6>
+; AVX2-NEXT:    vpermps %ymm1, %ymm2, %ymm1
+; AVX2-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3,4,5,6,7]
+; AVX2-NEXT:    retq
+;
+; AVX512VL-LABEL: shuffle_v8f32_089abcde:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vmovaps {{.*#+}} ymm2 = [8,0,1,2,3,4,5,6]
+; AVX512VL-NEXT:    vpermi2ps %ymm0, %ymm1, %ymm2
+; AVX512VL-NEXT:    vmovaps %ymm2, %ymm0
+; AVX512VL-NEXT:    retq
+  %shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> <i32 0, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14>
+  ret <8 x float> %shuffle
+}
+
+define <8 x float> @shuffle_v8f32_0189abcd(<8 x float> %a, <8 x float> %b) {
+; AVX1OR2-LABEL: shuffle_v8f32_0189abcd:
+; AVX1OR2:       # %bb.0:
+; AVX1OR2-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1OR2-NEXT:    vshufpd {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[3],ymm1[2]
+; AVX1OR2-NEXT:    retq
+;
+; AVX512VL-SLOW-LABEL: shuffle_v8f32_0189abcd:
+; AVX512VL-SLOW:       # %bb.0:
+; AVX512VL-SLOW-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX512VL-SLOW-NEXT:    vshufpd {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[3],ymm1[2]
+; AVX512VL-SLOW-NEXT:    retq
+;
+; AVX512VL-FAST-LABEL: shuffle_v8f32_0189abcd:
+; AVX512VL-FAST:       # %bb.0:
+; AVX512VL-FAST-NEXT:    vmovapd {{.*#+}} ymm2 = [4,0,1,2]
+; AVX512VL-FAST-NEXT:    vpermi2pd %ymm0, %ymm1, %ymm2
+; AVX512VL-FAST-NEXT:    vmovapd %ymm2, %ymm0
+; AVX512VL-FAST-NEXT:    retq
+  %shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13>
+  ret <8 x float> %shuffle
+}
+
 define <8 x float> @shuffle_v8f32_uuuu1111(<8 x float> %a, <8 x float> %b) {
 ; ALL-LABEL: shuffle_v8f32_uuuu1111:
 ; ALL:       # %bb.0:
@@ -1636,14 +1687,23 @@ define <8 x i32> @shuffle_v8i32_08991abb(<8 x i32> %a, <8 x i32> %b) {
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
 ; AVX1-NEXT:    retq
 ;
-; AVX2-LABEL: shuffle_v8i32_08991abb:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = <u,0,1,1,u,2,3,3>
-; AVX2-NEXT:    vpermd %ymm1, %ymm2, %ymm1
-; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,1,3]
-; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7]
-; AVX2-NEXT:    retq
+; AVX2-SLOW-LABEL: shuffle_v8i32_08991abb:
+; AVX2-SLOW:       # %bb.0:
+; AVX2-SLOW-NEXT:    vmovdqa {{.*#+}} ymm2 = <u,0,1,1,u,2,3,3>
+; AVX2-SLOW-NEXT:    vpermd %ymm1, %ymm2, %ymm1
+; AVX2-SLOW-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
+; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,1,3]
+; AVX2-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7]
+; AVX2-SLOW-NEXT:    retq
+;
+; AVX2-FAST-LABEL: shuffle_v8i32_08991abb:
+; AVX2-FAST:       # %bb.0:
+; AVX2-FAST-NEXT:    vmovaps {{.*#+}} ymm2 = <0,u,1,u,1,u,u,u>
+; AVX2-FAST-NEXT:    vpermps %ymm0, %ymm2, %ymm0
+; AVX2-FAST-NEXT:    vmovaps {{.*#+}} ymm2 = <u,0,1,1,u,2,3,3>
+; AVX2-FAST-NEXT:    vpermps %ymm1, %ymm2, %ymm1
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7]
+; AVX2-FAST-NEXT:    retq
 ;
 ; AVX512VL-LABEL: shuffle_v8i32_08991abb:
 ; AVX512VL:       # %bb.0:
@@ -1693,12 +1753,19 @@ define <8 x i32> @shuffle_v8i32_09ab1def(<8 x i32> %a, <8 x i32> %b) {
 ; AVX1-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7]
 ; AVX1-NEXT:    retq
 ;
-; AVX2-LABEL: shuffle_v8i32_09ab1def:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,1,3]
-; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7]
-; AVX2-NEXT:    retq
+; AVX2-SLOW-LABEL: shuffle_v8i32_09ab1def:
+; AVX2-SLOW:       # %bb.0:
+; AVX2-SLOW-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
+; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,1,3]
+; AVX2-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7]
+; AVX2-SLOW-NEXT:    retq
+;
+; AVX2-FAST-LABEL: shuffle_v8i32_09ab1def:
+; AVX2-FAST:       # %bb.0:
+; AVX2-FAST-NEXT:    vmovaps {{.*#+}} ymm2 = <0,u,1,u,1,u,u,u>
+; AVX2-FAST-NEXT:    vpermps %ymm0, %ymm2, %ymm0
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7]
+; AVX2-FAST-NEXT:    retq
 ;
 ; AVX512VL-LABEL: shuffle_v8i32_09ab1def:
 ; AVX512VL:       # %bb.0:
@@ -2399,6 +2466,61 @@ define <8 x i32> @shuffle_v8i32_ba983210(<8 x i32> %a, <8 x i32> %b) {
   ret <8 x i32> %shuffle
 }
 
+define <8 x i32> @shuffle_v8i32_089abcde(<8 x i32> %a, <8 x i32> %b) {
+; AVX1-LABEL: shuffle_v8i32_089abcde:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT:    vpalignr {{.*#+}} xmm2 = xmm1[12,13,14,15],xmm2[0,1,2,3,4,5,6,7,8,9,10,11]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,1,2]
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; AVX1-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3,4,5,6,7]
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: shuffle_v8i32_089abcde:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vmovaps {{.*#+}} ymm2 = <u,0,1,2,3,4,5,6>
+; AVX2-NEXT:    vpermps %ymm1, %ymm2, %ymm1
+; AVX2-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3,4,5,6,7]
+; AVX2-NEXT:    retq
+;
+; AVX512VL-LABEL: shuffle_v8i32_089abcde:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    valignd {{.*#+}} ymm1 = ymm1[7,0,1,2,3,4,5,6]
+; AVX512VL-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3,4,5,6,7]
+; AVX512VL-NEXT:    retq
+  %shuffle = shufflevector <8 x i32> %a, <8 x i32> %b, <8 x i32> <i32 0, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14>
+  ret <8 x i32> %shuffle
+}
+
+define <8 x i32> @shuffle_v8i32_0189abcd(<8 x i32> %a, <8 x i32> %b) {
+; AVX1-LABEL: shuffle_v8i32_0189abcd:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    vshufpd {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[3],ymm1[2]
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: shuffle_v8i32_0189abcd:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,0,1,2]
+; AVX2-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3,4,5,6,7]
+; AVX2-NEXT:    retq
+;
+; AVX512VL-SLOW-LABEL: shuffle_v8i32_0189abcd:
+; AVX512VL-SLOW:       # %bb.0:
+; AVX512VL-SLOW-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,0,1,2]
+; AVX512VL-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3,4,5,6,7]
+; AVX512VL-SLOW-NEXT:    retq
+;
+; AVX512VL-FAST-LABEL: shuffle_v8i32_0189abcd:
+; AVX512VL-FAST:       # %bb.0:
+; AVX512VL-FAST-NEXT:    vmovdqa {{.*#+}} ymm2 = [4,0,1,2]
+; AVX512VL-FAST-NEXT:    vpermi2q %ymm0, %ymm1, %ymm2
+; AVX512VL-FAST-NEXT:    vmovdqa %ymm2, %ymm0
+; AVX512VL-FAST-NEXT:    retq
+  %shuffle = shufflevector <8 x i32> %a, <8 x i32> %b, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13>
+  ret <8 x i32> %shuffle
+}
+
 define <8 x i32> @shuffle_v8i32_zuu8zuuc(<8 x i32> %a) {
 ; AVX1-LABEL: shuffle_v8i32_zuu8zuuc:
 ; AVX1:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-512-v16.ll b/llvm/test/CodeGen/X86/vector-shuffle-512-v16.ll
index b74a0b1c0..d04704241 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-512-v16.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-512-v16.ll
@@ -309,7 +309,7 @@ define <4 x i32> @test_v16i32_0_1_2_12 (<16 x i32> %v) {
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vextractf64x4 $1, %zmm0, %ymm1
 ; ALL-NEXT:    vextractf128 $1, %ymm1, %xmm1
-; ALL-NEXT:    vbroadcastss %xmm1, %xmm1
+; ALL-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[0,1,2,0]
 ; ALL-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[3]
 ; ALL-NEXT:    vzeroupper
 ; ALL-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-512-v32.ll b/llvm/test/CodeGen/X86/vector-shuffle-512-v32.ll
index a77099abe..c350e7f6d 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-512-v32.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-512-v32.ll
@@ -8,7 +8,7 @@ define <32 x i16> @shuffle_v32i16(<32 x i16> %a)  {
 ; KNL-LABEL: shuffle_v32i16:
 ; KNL:       ## %bb.0:
 ; KNL-NEXT:    vpbroadcastw %xmm0, %ymm0
-; KNL-NEXT:    vmovdqa %ymm0, %ymm1
+; KNL-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: shuffle_v32i16:
@@ -24,7 +24,7 @@ define <32 x i16> @shuffle_v32i16_08_08_08_08_08_08_08_08_08_08_08_08_08_08_08_0
 ; KNL:       ## %bb.0:
 ; KNL-NEXT:    vextracti128 $1, %ymm0, %xmm0
 ; KNL-NEXT:    vpbroadcastw %xmm0, %ymm0
-; KNL-NEXT:    vmovdqa %ymm0, %ymm1
+; KNL-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: shuffle_v32i16_08_08_08_08_08_08_08_08_08_08_08_08_08_08_08_08_08_08_08_08_08_08_08_08_08_08_08_08_08_08_08_08:
@@ -39,16 +39,18 @@ define <32 x i16> @shuffle_v32i16_08_08_08_08_08_08_08_08_08_08_08_08_08_08_08_0
 define <32 x i16> @shuffle_v32i16_02_05_u_u_07_u_0a_01_00_05_u_04_07_u_0a_01_02_05_u_u_07_u_0a_01_00_05_u_04_07_u_0a_1f(<32 x i16> %a)  {
 ; KNL-LABEL: shuffle_v32i16_02_05_u_u_07_u_0a_01_00_05_u_04_07_u_0a_01_02_05_u_u_07_u_0a_01_00_05_u_04_07_u_0a_1f:
 ; KNL:       ## %bb.0:
-; KNL-NEXT:    vpshufb {{.*#+}} ymm2 = ymm0[4,5,10,11,4,5,6,7,14,15,2,3,4,5,2,3,20,21,26,27,20,21,22,23,30,31,18,19,20,21,18,19]
-; KNL-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,3,0,1]
-; KNL-NEXT:    vpshufb {{.*#+}} ymm3 = ymm0[0,1,10,11,8,9,8,9,14,15,6,7,4,5,14,15,16,17,26,27,24,25,24,25,30,31,22,23,20,21,30,31]
+; KNL-NEXT:    vpshufb {{.*#+}} ymm1 = ymm0[4,5,10,11,4,5,6,7,14,15,2,3,4,5,2,3,20,21,26,27,20,21,22,23,30,31,18,19,20,21,18,19]
+; KNL-NEXT:    vpermq {{.*#+}} ymm2 = ymm0[2,3,0,1]
+; KNL-NEXT:    vpshufb {{.*#+}} ymm3 = ymm2[0,1,10,11,8,9,8,9,14,15,6,7,4,5,14,15,16,17,26,27,24,25,24,25,30,31,22,23,20,21,30,31]
+; KNL-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
 ; KNL-NEXT:    vmovdqa {{.*#+}} ymm4 = <255,255,255,255,u,u,u,u,255,255,u,u,0,0,255,255,0,0,0,0,u,u,0,0,0,0,u,u,255,255,u,u>
-; KNL-NEXT:    vpblendvb %ymm4, %ymm2, %ymm3, %ymm3
-; KNL-NEXT:    vpblendw {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5,6],ymm1[7],ymm3[8,9,10,11,12,13,14],ymm1[15]
-; KNL-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,10,11,8,9,8,9,14,15,2,3,4,5,2,3,16,17,26,27,24,25,24,25,30,31,18,19,20,21,18,19]
-; KNL-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
+; KNL-NEXT:    vpblendvb %ymm4, %ymm1, %ymm3, %ymm3
+; KNL-NEXT:    vpblendw {{.*#+}} ymm0 = ymm3[0,1,2,3,4,5,6],ymm0[7],ymm3[8,9,10,11,12,13,14],ymm0[15]
+; KNL-NEXT:    vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3],ymm0[4,5,6,7]
+; KNL-NEXT:    vpshufb {{.*#+}} ymm2 = ymm2[0,1,10,11,8,9,8,9,14,15,2,3,4,5,2,3,16,17,26,27,24,25,24,25,30,31,18,19,20,21,18,19]
 ; KNL-NEXT:    vmovdqa {{.*#+}} ymm3 = <0,0,0,0,u,u,u,u,0,0,u,u,255,255,0,0,255,255,255,255,u,u,255,255,255,255,u,u,0,0,255,255>
-; KNL-NEXT:    vpblendvb %ymm3, %ymm0, %ymm2, %ymm0
+; KNL-NEXT:    vpblendvb %ymm3, %ymm2, %ymm1, %ymm1
+; KNL-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: shuffle_v32i16_02_05_u_u_07_u_0a_01_00_05_u_04_07_u_0a_01_02_05_u_u_07_u_0a_01_00_05_u_04_07_u_0a_1f:
@@ -63,19 +65,22 @@ define <32 x i16> @shuffle_v32i16_02_05_u_u_07_u_0a_01_00_05_u_04_07_u_0a_01_02_
 define <32 x i16> @shuffle_v32i16_0f_1f_0e_16_0d_1d_04_1e_0b_1b_0a_1a_09_19_08_18_0f_1f_0e_16_0d_1d_04_1e_0b_1b_0a_1a_09_19_08_38(<32 x i16> %a, <32 x i16> %b)  {
 ; KNL-LABEL: shuffle_v32i16_0f_1f_0e_16_0d_1d_04_1e_0b_1b_0a_1a_09_19_08_18_0f_1f_0e_16_0d_1d_04_1e_0b_1b_0a_1a_09_19_08_38:
 ; KNL:       ## %bb.0:
-; KNL-NEXT:    vpermq {{.*#+}} ymm2 = ymm1[2,3,0,1]
-; KNL-NEXT:    vpblendw {{.*#+}} ymm2 = ymm1[0,1,2,3],ymm2[4,5],ymm1[6],ymm2[7],ymm1[8,9,10,11],ymm2[12,13],ymm1[14],ymm2[15]
-; KNL-NEXT:    vpshufb {{.*#+}} ymm1 = ymm2[u,u,14,15,u,u,12,13,u,u,10,11,u,u,8,9,u,u,22,23,u,u,20,21,u,u,18,19,u,u,u,u]
+; KNL-NEXT:    vextracti64x4 $1, %zmm1, %ymm1
+; KNL-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; KNL-NEXT:    vpermq {{.*#+}} ymm3 = ymm2[2,3,0,1]
+; KNL-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4,5],ymm2[6],ymm3[7],ymm2[8,9,10,11],ymm3[12,13],ymm2[14],ymm3[15]
+; KNL-NEXT:    vpshufb {{.*#+}} ymm3 = ymm2[u,u,14,15,u,u,12,13,u,u,10,11,u,u,8,9,u,u,22,23,u,u,20,21,u,u,18,19,u,u,u,u]
 ; KNL-NEXT:    vextracti128 $1, %ymm0, %xmm4
 ; KNL-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm4[5,6,7],ymm0[8,9,10,11,12],ymm4[13,14,15]
 ; KNL-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[14,15,u,u,12,13,u,u,10,11,u,u,8,9,u,u,22,23,u,u,20,21,u,u,18,19,u,u,16,17,u,u]
-; KNL-NEXT:    vpblendw {{.*#+}} ymm1 = ymm0[0],ymm1[1],ymm0[2],ymm1[3],ymm0[4],ymm1[5],ymm0[6],ymm1[7],ymm0[8],ymm1[9],ymm0[10],ymm1[11],ymm0[12],ymm1[13],ymm0[14],ymm1[15]
-; KNL-NEXT:    vextracti128 $1, %ymm3, %xmm3
-; KNL-NEXT:    vpbroadcastw %xmm3, %ymm3
-; KNL-NEXT:    vpblendw {{.*#+}} ymm3 = ymm1[0,1,2,3,4,5,6],ymm3[7],ymm1[8,9,10,11,12,13,14],ymm3[15]
-; KNL-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm3[4,5,6,7]
+; KNL-NEXT:    vpblendw {{.*#+}} ymm3 = ymm0[0],ymm3[1],ymm0[2],ymm3[3],ymm0[4],ymm3[5],ymm0[6],ymm3[7],ymm0[8],ymm3[9],ymm0[10],ymm3[11],ymm0[12],ymm3[13],ymm0[14],ymm3[15]
+; KNL-NEXT:    vextracti128 $1, %ymm1, %xmm1
+; KNL-NEXT:    vpbroadcastw %xmm1, %ymm1
+; KNL-NEXT:    vpblendw {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5,6],ymm1[7],ymm3[8,9,10,11,12,13,14],ymm1[15]
+; KNL-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
 ; KNL-NEXT:    vpshufb {{.*#+}} ymm2 = ymm2[u,u,14,15,u,u,12,13,u,u,10,11,u,u,8,9,u,u,22,23,u,u,20,21,u,u,18,19,u,u,16,17]
 ; KNL-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm2[1],ymm0[2],ymm2[3],ymm0[4],ymm2[5],ymm0[6],ymm2[7],ymm0[8],ymm2[9],ymm0[10],ymm2[11],ymm0[12],ymm2[13],ymm0[14],ymm2[15]
+; KNL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: shuffle_v32i16_0f_1f_0e_16_0d_1d_04_1e_0b_1b_0a_1a_09_19_08_18_0f_1f_0e_16_0d_1d_04_1e_0b_1b_0a_1a_09_19_08_38:
@@ -88,29 +93,19 @@ define <32 x i16> @shuffle_v32i16_0f_1f_0e_16_0d_1d_04_1e_0b_1b_0a_1a_09_19_08_1
 }
 
 define <32 x i16> @shuffle_v16i32_0_32_1_33_2_34_3_35_8_40_9_41_u_u_u_u(<32 x i16> %a, <32 x i16> %b)  {
-; KNL-LABEL: shuffle_v16i32_0_32_1_33_2_34_3_35_8_40_9_41_u_u_u_u:
-; KNL:       ## %bb.0:
-; KNL-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm0[0],ymm2[0],ymm0[1],ymm2[1],ymm0[2],ymm2[2],ymm0[3],ymm2[3],ymm0[8],ymm2[8],ymm0[9],ymm2[9],ymm0[10],ymm2[10],ymm0[11],ymm2[11]
-; KNL-NEXT:    retq
-;
-; SKX-LABEL: shuffle_v16i32_0_32_1_33_2_34_3_35_8_40_9_41_u_u_u_u:
-; SKX:       ## %bb.0:
-; SKX-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11]
-; SKX-NEXT:    retq
+; ALL-LABEL: shuffle_v16i32_0_32_1_33_2_34_3_35_8_40_9_41_u_u_u_u:
+; ALL:       ## %bb.0:
+; ALL-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11]
+; ALL-NEXT:    retq
   %c = shufflevector <32 x i16> %a, <32 x i16> %b, <32 x i32> <i32 0, i32 32, i32 1, i32 33, i32 2, i32 34, i32 3, i32 35, i32 8, i32 40, i32 9, i32 41, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
   ret <32 x i16> %c
 }
 
 define <32 x i16> @shuffle_v16i32_4_36_5_37_6_38_7_39_12_44_13_45_u_u_u_u(<32 x i16> %a, <32 x i16> %b)  {
-; KNL-LABEL: shuffle_v16i32_4_36_5_37_6_38_7_39_12_44_13_45_u_u_u_u:
-; KNL:       ## %bb.0:
-; KNL-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm0[4],ymm2[4],ymm0[5],ymm2[5],ymm0[6],ymm2[6],ymm0[7],ymm2[7],ymm0[12],ymm2[12],ymm0[13],ymm2[13],ymm0[14],ymm2[14],ymm0[15],ymm2[15]
-; KNL-NEXT:    retq
-;
-; SKX-LABEL: shuffle_v16i32_4_36_5_37_6_38_7_39_12_44_13_45_u_u_u_u:
-; SKX:       ## %bb.0:
-; SKX-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15]
-; SKX-NEXT:    retq
+; ALL-LABEL: shuffle_v16i32_4_36_5_37_6_38_7_39_12_44_13_45_u_u_u_u:
+; ALL:       ## %bb.0:
+; ALL-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15]
+; ALL-NEXT:    retq
   %c = shufflevector <32 x i16> %a, <32 x i16> %b, <32 x i32> <i32 4, i32 36, i32 5, i32 37, i32 6, i32 38, i32 7, i32 39, i32 12, i32 44, i32 13, i32 45, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
   ret <32 x i16> %c
 }
@@ -118,8 +113,10 @@ define <32 x i16> @shuffle_v16i32_4_36_5_37_6_38_7_39_12_44_13_45_u_u_u_u(<32 x
 define <32 x i16> @shuffle_v32i16_1_z_3_z_5_z_7_z_9_z_11_z_13_z_15_z_17_z_19_z_21_z_23_z_25_z_27_z_29_z_31_z(<32 x i16> %a, <32 x i16> %b)  {
 ; KNL-LABEL: shuffle_v32i16_1_z_3_z_5_z_7_z_9_z_11_z_13_z_15_z_17_z_19_z_21_z_23_z_25_z_27_z_29_z_31_z:
 ; KNL:       ## %bb.0:
-; KNL-NEXT:    vpsrld $16, %ymm0, %ymm0
+; KNL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; KNL-NEXT:    vpsrld $16, %ymm1, %ymm1
+; KNL-NEXT:    vpsrld $16, %ymm0, %ymm0
+; KNL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: shuffle_v32i16_1_z_3_z_5_z_7_z_9_z_11_z_13_z_15_z_17_z_19_z_21_z_23_z_25_z_27_z_29_z_31_z:
@@ -133,8 +130,10 @@ define <32 x i16> @shuffle_v32i16_1_z_3_z_5_z_7_z_9_z_11_z_13_z_15_z_17_z_19_z_2
 define <32 x i16> @shuffle_v32i16_z_0_z_2_z_4_z_6_z_8_z_10_z_12_z_14_z_16_z_18_z_20_z_22_z_24_z_26_z_28_z_30(<32 x i16> %a, <32 x i16> %b)  {
 ; KNL-LABEL: shuffle_v32i16_z_0_z_2_z_4_z_6_z_8_z_10_z_12_z_14_z_16_z_18_z_20_z_22_z_24_z_26_z_28_z_30:
 ; KNL:       ## %bb.0:
-; KNL-NEXT:    vpslld $16, %ymm0, %ymm0
+; KNL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; KNL-NEXT:    vpslld $16, %ymm1, %ymm1
+; KNL-NEXT:    vpslld $16, %ymm0, %ymm0
+; KNL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: shuffle_v32i16_z_0_z_2_z_4_z_6_z_8_z_10_z_12_z_14_z_16_z_18_z_20_z_22_z_24_z_26_z_28_z_30:
@@ -148,8 +147,10 @@ define <32 x i16> @shuffle_v32i16_z_0_z_2_z_4_z_6_z_8_z_10_z_12_z_14_z_16_z_18_z
 define <32 x i16> @shuffle_v32i16_1_1_0_0_4_5_6_7_9_9_8_8_12_13_14_15_17_17_16_16_20_21_22_23_25_25_24_24_28_29_30_31(<32 x i16> %a, <32 x i16> %b)  {
 ; KNL-LABEL: shuffle_v32i16_1_1_0_0_4_5_6_7_9_9_8_8_12_13_14_15_17_17_16_16_20_21_22_23_25_25_24_24_28_29_30_31:
 ; KNL:       ## %bb.0:
-; KNL-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm0[1,1,0,0,4,5,6,7,9,9,8,8,12,13,14,15]
+; KNL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; KNL-NEXT:    vpshuflw {{.*#+}} ymm1 = ymm1[1,1,0,0,4,5,6,7,9,9,8,8,12,13,14,15]
+; KNL-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm0[1,1,0,0,4,5,6,7,9,9,8,8,12,13,14,15]
+; KNL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: shuffle_v32i16_1_1_0_0_4_5_6_7_9_9_8_8_12_13_14_15_17_17_16_16_20_21_22_23_25_25_24_24_28_29_30_31:
@@ -163,8 +164,10 @@ define <32 x i16> @shuffle_v32i16_1_1_0_0_4_5_6_7_9_9_8_8_12_13_14_15_17_17_16_1
 define <32 x i16> @shuffle_v32i16_0_1_2_3_5_5_4_4_8_9_10_11_13_13_12_12_16_17_18_19_21_21_20_20_24_25_26_27_29_29_28_28(<32 x i16> %a, <32 x i16> %b)  {
 ; KNL-LABEL: shuffle_v32i16_0_1_2_3_5_5_4_4_8_9_10_11_13_13_12_12_16_17_18_19_21_21_20_20_24_25_26_27_29_29_28_28:
 ; KNL:       ## %bb.0:
-; KNL-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm0[0,1,2,3,5,5,4,4,8,9,10,11,13,13,12,12]
+; KNL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; KNL-NEXT:    vpshufhw {{.*#+}} ymm1 = ymm1[0,1,2,3,5,5,4,4,8,9,10,11,13,13,12,12]
+; KNL-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm0[0,1,2,3,5,5,4,4,8,9,10,11,13,13,12,12]
+; KNL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: shuffle_v32i16_0_1_2_3_5_5_4_4_8_9_10_11_13_13_12_12_16_17_18_19_21_21_20_20_24_25_26_27_29_29_28_28:
@@ -178,10 +181,12 @@ define <32 x i16> @shuffle_v32i16_0_1_2_3_5_5_4_4_8_9_10_11_13_13_12_12_16_17_18
 define <32 x i16> @shuffle_v32i16_1_1_0_0_5_5_4_4_9_9_11_11_13_13_12_12_17_17_19_19_21_21_20_20_25_25_27_27_29_29_28_28(<32 x i16> %a, <32 x i16> %b)  {
 ; KNL-LABEL: shuffle_v32i16_1_1_0_0_5_5_4_4_9_9_11_11_13_13_12_12_17_17_19_19_21_21_20_20_25_25_27_27_29_29_28_28:
 ; KNL:       ## %bb.0:
-; KNL-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm0[1,1,0,0,4,5,6,7,9,9,8,8,12,13,14,15]
-; KNL-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm0[0,1,2,3,5,5,4,4,8,9,10,11,13,13,12,12]
+; KNL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; KNL-NEXT:    vpshuflw {{.*#+}} ymm1 = ymm1[1,1,0,0,4,5,6,7,9,9,8,8,12,13,14,15]
 ; KNL-NEXT:    vpshufhw {{.*#+}} ymm1 = ymm1[0,1,2,3,5,5,4,4,8,9,10,11,13,13,12,12]
+; KNL-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm0[1,1,0,0,4,5,6,7,9,9,8,8,12,13,14,15]
+; KNL-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm0[0,1,2,3,5,5,4,4,8,9,10,11,13,13,12,12]
+; KNL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: shuffle_v32i16_1_1_0_0_5_5_4_4_9_9_11_11_13_13_12_12_17_17_19_19_21_21_20_20_25_25_27_27_29_29_28_28:
@@ -198,7 +203,6 @@ define <32 x i16> @shuffle_v32i16_0zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz(<32 x i16> %a
 ; KNL-NEXT:    movl $65535, %eax ## imm = 0xFFFF
 ; KNL-NEXT:    vmovd %eax, %xmm1
 ; KNL-NEXT:    vpand %ymm1, %ymm0, %ymm0
-; KNL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: shuffle_v32i16_0zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz:
@@ -215,7 +219,7 @@ define <32 x i16> @insert_dup_mem_v32i16_i32(i32* %ptr) {
 ; KNL-LABEL: insert_dup_mem_v32i16_i32:
 ; KNL:       ## %bb.0:
 ; KNL-NEXT:    vpbroadcastw (%rdi), %ymm0
-; KNL-NEXT:    vmovdqa %ymm0, %ymm1
+; KNL-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: insert_dup_mem_v32i16_i32:
@@ -233,7 +237,7 @@ define <32 x i16> @insert_dup_mem_v32i16_sext_i16(i16* %ptr) {
 ; KNL-LABEL: insert_dup_mem_v32i16_sext_i16:
 ; KNL:       ## %bb.0:
 ; KNL-NEXT:    vpbroadcastw (%rdi), %ymm0
-; KNL-NEXT:    vmovdqa %ymm0, %ymm1
+; KNL-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: insert_dup_mem_v32i16_sext_i16:
@@ -252,7 +256,7 @@ define <32 x i16> @insert_dup_elt1_mem_v32i16_i32(i32* %ptr) #0 {
 ; KNL-LABEL: insert_dup_elt1_mem_v32i16_i32:
 ; KNL:       ## %bb.0:
 ; KNL-NEXT:    vpbroadcastw 2(%rdi), %ymm0
-; KNL-NEXT:    vmovdqa %ymm0, %ymm1
+; KNL-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: insert_dup_elt1_mem_v32i16_i32:
@@ -270,7 +274,7 @@ define <32 x i16> @insert_dup_elt3_mem_v32i16_i32(i32* %ptr) #0 {
 ; KNL-LABEL: insert_dup_elt3_mem_v32i16_i32:
 ; KNL:       ## %bb.0:
 ; KNL-NEXT:    vpbroadcastw 2(%rdi), %ymm0
-; KNL-NEXT:    vmovdqa %ymm0, %ymm1
+; KNL-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: insert_dup_elt3_mem_v32i16_i32:
@@ -287,10 +291,10 @@ define <32 x i16> @insert_dup_elt3_mem_v32i16_i32(i32* %ptr) #0 {
 define <32 x i16> @shuffle_v32i16_32_zz_zz_zz_33_zz_zz_zz_34_zz_zz_zz_35_zz_zz_zz_36_zz_zz_zz_37_zz_zz_zz_38_zz_zz_zz_39_zz_zz_zz(<32 x i16> %a) {
 ; KNL-LABEL: shuffle_v32i16_32_zz_zz_zz_33_zz_zz_zz_34_zz_zz_zz_35_zz_zz_zz_36_zz_zz_zz_37_zz_zz_zz_38_zz_zz_zz_39_zz_zz_zz:
 ; KNL:       ## %bb.0:
-; KNL-NEXT:    vpmovzxwq {{.*#+}} ymm2 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; KNL-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; KNL-NEXT:    vpmovzxwq {{.*#+}} ymm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; KNL-NEXT:    vmovdqa %ymm2, %ymm0
+; KNL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; KNL-NEXT:    vpmovzxwq {{.*#+}} ymm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
+; KNL-NEXT:    vpmovzxwq {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; KNL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: shuffle_v32i16_32_zz_zz_zz_33_zz_zz_zz_34_zz_zz_zz_35_zz_zz_zz_36_zz_zz_zz_37_zz_zz_zz_38_zz_zz_zz_39_zz_zz_zz:
@@ -304,10 +308,10 @@ define <32 x i16> @shuffle_v32i16_32_zz_zz_zz_33_zz_zz_zz_34_zz_zz_zz_35_zz_zz_z
 define <32 x i16> @shuffle_v32i16_32_zz_33_zz_34_zz_35_zz_36_zz_37_zz_38_zz_39_zz_40_zz_41_zz_42_zz_43_zz_44_zz_45_zz_46_zz_47_zz(<32 x i16> %a) {
 ; KNL-LABEL: shuffle_v32i16_32_zz_33_zz_34_zz_35_zz_36_zz_37_zz_38_zz_39_zz_40_zz_41_zz_42_zz_43_zz_44_zz_45_zz_46_zz_47_zz:
 ; KNL:       ## %bb.0:
-; KNL-NEXT:    vpmovzxwd {{.*#+}} ymm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; KNL-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; KNL-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; KNL-NEXT:    vmovdqa %ymm2, %ymm0
+; KNL-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; KNL-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; KNL-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; KNL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: shuffle_v32i16_32_zz_33_zz_34_zz_35_zz_36_zz_37_zz_38_zz_39_zz_40_zz_41_zz_42_zz_43_zz_44_zz_45_zz_46_zz_47_zz:
@@ -321,6 +325,7 @@ define <32 x i16> @shuffle_v32i16_32_zz_33_zz_34_zz_35_zz_36_zz_37_zz_38_zz_39_z
 define <8 x i16> @pr32967(<32 x i16> %v) {
 ; KNL-LABEL: pr32967:
 ; KNL:       ## %bb.0:
+; KNL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; KNL-NEXT:    vextracti128 $1, %ymm1, %xmm2
 ; KNL-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
 ; KNL-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm2[0,1,1,3,4,5,6,7]
@@ -351,8 +356,10 @@ define <8 x i16> @pr32967(<32 x i16> %v) {
 define <32 x i16> @shuffle_v32i16_07_zz_05_zz_03_zz_01_zz_15_zz_13_zz_11_zz_09_zz_23_zz_21_zz_19_zz_17_zz_31_zz_29_zz_27_zz_25_zz(<32 x i16> %a) {
 ; KNL-LABEL: shuffle_v32i16_07_zz_05_zz_03_zz_01_zz_15_zz_13_zz_11_zz_09_zz_23_zz_21_zz_19_zz_17_zz_31_zz_29_zz_27_zz_25_zz:
 ; KNL:       ## %bb.0:
-; KNL-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[14,15],zero,zero,ymm0[10,11],zero,zero,ymm0[6,7],zero,zero,ymm0[2,3],zero,zero,ymm0[30,31],zero,zero,ymm0[26,27],zero,zero,ymm0[22,23],zero,zero,ymm0[18,19],zero,zero
+; KNL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; KNL-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[14,15],zero,zero,ymm1[10,11],zero,zero,ymm1[6,7],zero,zero,ymm1[2,3],zero,zero,ymm1[30,31],zero,zero,ymm1[26,27],zero,zero,ymm1[22,23],zero,zero,ymm1[20,21],zero,zero
+; KNL-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[14,15],zero,zero,ymm0[10,11],zero,zero,ymm0[6,7],zero,zero,ymm0[2,3],zero,zero,ymm0[30,31],zero,zero,ymm0[26,27],zero,zero,ymm0[22,23],zero,zero,ymm0[18,19],zero,zero
+; KNL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: shuffle_v32i16_07_zz_05_zz_03_zz_01_zz_15_zz_13_zz_11_zz_09_zz_23_zz_21_zz_19_zz_17_zz_31_zz_29_zz_27_zz_25_zz:
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-512-v64.ll b/llvm/test/CodeGen/X86/vector-shuffle-512-v64.ll
index 2f1c59882..7e5847eb1 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-512-v64.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-512-v64.ll
@@ -16,8 +16,10 @@ define <64 x i8> @shuffle_v64i8_02_03_u_u_u_u_u_u_u_u_u_u_u_u_u_u_u_u_u_u_u_u_u_
 define <64 x i8> @shuffle_v64i8_zz_00_01_02_03_04_05_06_07_08_09_10_11_12_13_14_zz_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_zz_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_zz_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62(<64 x i8> %a, <64 x i8> %b) {
 ; AVX512F-LABEL: shuffle_v64i8_zz_00_01_02_03_04_05_06_07_08_09_10_11_12_13_14_zz_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_zz_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_zz_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpslldq {{.*#+}} ymm0 = zero,ymm0[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14],zero,ymm0[16,17,18,19,20,21,22,23,24,25,26,27,28,29,30]
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512F-NEXT:    vpslldq {{.*#+}} ymm1 = zero,ymm1[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14],zero,ymm1[16,17,18,19,20,21,22,23,24,25,26,27,28,29,30]
+; AVX512F-NEXT:    vpslldq {{.*#+}} ymm0 = zero,ymm0[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14],zero,ymm0[16,17,18,19,20,21,22,23,24,25,26,27,28,29,30]
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: shuffle_v64i8_zz_00_01_02_03_04_05_06_07_08_09_10_11_12_13_14_zz_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_zz_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_zz_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62:
@@ -27,8 +29,10 @@ define <64 x i8> @shuffle_v64i8_zz_00_01_02_03_04_05_06_07_08_09_10_11_12_13_14_
 ;
 ; AVX512DQ-LABEL: shuffle_v64i8_zz_00_01_02_03_04_05_06_07_08_09_10_11_12_13_14_zz_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_zz_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_zz_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vpslldq {{.*#+}} ymm0 = zero,ymm0[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14],zero,ymm0[16,17,18,19,20,21,22,23,24,25,26,27,28,29,30]
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512DQ-NEXT:    vpslldq {{.*#+}} ymm1 = zero,ymm1[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14],zero,ymm1[16,17,18,19,20,21,22,23,24,25,26,27,28,29,30]
+; AVX512DQ-NEXT:    vpslldq {{.*#+}} ymm0 = zero,ymm0[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14],zero,ymm0[16,17,18,19,20,21,22,23,24,25,26,27,28,29,30]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512VBMI-LABEL: shuffle_v64i8_zz_00_01_02_03_04_05_06_07_08_09_10_11_12_13_14_zz_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_zz_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_zz_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62:
@@ -42,8 +46,10 @@ define <64 x i8> @shuffle_v64i8_zz_00_01_02_03_04_05_06_07_08_09_10_11_12_13_14_
 define <64 x i8> @shuffle_v64i8_02_03_04_05_06_07_08_09_10_11_12_13_14_15_zz_zz_18_19_20_21_22_23_24_25_26_27_28_29_30_31_zz_zz_34_35_36_37_38_39_40_41_42_43_44_45_46_47_zz_zz_50_51_52_53_54_55_56_57_58_59_60_61_62_63_zz_zz(<64 x i8> %a, <64 x i8> %b) {
 ; AVX512F-LABEL: shuffle_v64i8_02_03_04_05_06_07_08_09_10_11_12_13_14_15_zz_zz_18_19_20_21_22_23_24_25_26_27_28_29_30_31_zz_zz_34_35_36_37_38_39_40_41_42_43_44_45_46_47_zz_zz_50_51_52_53_54_55_56_57_58_59_60_61_62_63_zz_zz:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpsrldq {{.*#+}} ymm0 = ymm0[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero,ymm0[18,19,20,21,22,23,24,25,26,27,28,29,30,31],zero,zero
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512F-NEXT:    vpsrldq {{.*#+}} ymm1 = ymm1[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero,ymm1[18,19,20,21,22,23,24,25,26,27,28,29,30,31],zero,zero
+; AVX512F-NEXT:    vpsrldq {{.*#+}} ymm0 = ymm0[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero,ymm0[18,19,20,21,22,23,24,25,26,27,28,29,30,31],zero,zero
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: shuffle_v64i8_02_03_04_05_06_07_08_09_10_11_12_13_14_15_zz_zz_18_19_20_21_22_23_24_25_26_27_28_29_30_31_zz_zz_34_35_36_37_38_39_40_41_42_43_44_45_46_47_zz_zz_50_51_52_53_54_55_56_57_58_59_60_61_62_63_zz_zz:
@@ -53,8 +59,10 @@ define <64 x i8> @shuffle_v64i8_02_03_04_05_06_07_08_09_10_11_12_13_14_15_zz_zz_
 ;
 ; AVX512DQ-LABEL: shuffle_v64i8_02_03_04_05_06_07_08_09_10_11_12_13_14_15_zz_zz_18_19_20_21_22_23_24_25_26_27_28_29_30_31_zz_zz_34_35_36_37_38_39_40_41_42_43_44_45_46_47_zz_zz_50_51_52_53_54_55_56_57_58_59_60_61_62_63_zz_zz:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vpsrldq {{.*#+}} ymm0 = ymm0[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero,ymm0[18,19,20,21,22,23,24,25,26,27,28,29,30,31],zero,zero
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512DQ-NEXT:    vpsrldq {{.*#+}} ymm1 = ymm1[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero,ymm1[18,19,20,21,22,23,24,25,26,27,28,29,30,31],zero,zero
+; AVX512DQ-NEXT:    vpsrldq {{.*#+}} ymm0 = ymm0[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero,ymm0[18,19,20,21,22,23,24,25,26,27,28,29,30,31],zero,zero
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512VBMI-LABEL: shuffle_v64i8_02_03_04_05_06_07_08_09_10_11_12_13_14_15_zz_zz_18_19_20_21_22_23_24_25_26_27_28_29_30_31_zz_zz_34_35_36_37_38_39_40_41_42_43_44_45_46_47_zz_zz_50_51_52_53_54_55_56_57_58_59_60_61_62_63_zz_zz:
@@ -68,8 +76,11 @@ define <64 x i8> @shuffle_v64i8_02_03_04_05_06_07_08_09_10_11_12_13_14_15_zz_zz_
 define <64 x i8> @shuffle_v64i8_79_00_01_02_03_04_05_06_07_08_09_10_11_12_13_14_95_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_111_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_127_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62(<64 x i8> %a, <64 x i8> %b) {
 ; AVX512F-LABEL: shuffle_v64i8_79_00_01_02_03_04_05_06_07_08_09_10_11_12_13_14_95_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_111_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_127_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpalignr {{.*#+}} ymm0 = ymm2[15],ymm0[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14],ymm2[31],ymm0[16,17,18,19,20,21,22,23,24,25,26,27,28,29,30]
-; AVX512F-NEXT:    vpalignr {{.*#+}} ymm1 = ymm3[15],ymm1[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14],ymm3[31],ymm1[16,17,18,19,20,21,22,23,24,25,26,27,28,29,30]
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
+; AVX512F-NEXT:    vpalignr {{.*#+}} ymm2 = ymm2[15],ymm3[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14],ymm2[31],ymm3[16,17,18,19,20,21,22,23,24,25,26,27,28,29,30]
+; AVX512F-NEXT:    vpalignr {{.*#+}} ymm0 = ymm1[15],ymm0[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14],ymm1[31],ymm0[16,17,18,19,20,21,22,23,24,25,26,27,28,29,30]
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: shuffle_v64i8_79_00_01_02_03_04_05_06_07_08_09_10_11_12_13_14_95_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_111_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_127_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62:
@@ -79,8 +90,11 @@ define <64 x i8> @shuffle_v64i8_79_00_01_02_03_04_05_06_07_08_09_10_11_12_13_14_
 ;
 ; AVX512DQ-LABEL: shuffle_v64i8_79_00_01_02_03_04_05_06_07_08_09_10_11_12_13_14_95_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_111_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_127_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vpalignr {{.*#+}} ymm0 = ymm2[15],ymm0[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14],ymm2[31],ymm0[16,17,18,19,20,21,22,23,24,25,26,27,28,29,30]
-; AVX512DQ-NEXT:    vpalignr {{.*#+}} ymm1 = ymm3[15],ymm1[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14],ymm3[31],ymm1[16,17,18,19,20,21,22,23,24,25,26,27,28,29,30]
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
+; AVX512DQ-NEXT:    vpalignr {{.*#+}} ymm2 = ymm2[15],ymm3[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14],ymm2[31],ymm3[16,17,18,19,20,21,22,23,24,25,26,27,28,29,30]
+; AVX512DQ-NEXT:    vpalignr {{.*#+}} ymm0 = ymm1[15],ymm0[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14],ymm1[31],ymm0[16,17,18,19,20,21,22,23,24,25,26,27,28,29,30]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512VBMI-LABEL: shuffle_v64i8_79_00_01_02_03_04_05_06_07_08_09_10_11_12_13_14_95_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_111_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_127_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62:
@@ -98,7 +112,6 @@ define <64 x i8> @shuffle_v64i8_0zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz(<64 x i8> %a) {
 ; AVX512F-NEXT:    movl $255, %eax
 ; AVX512F-NEXT:    vmovd %eax, %xmm1
 ; AVX512F-NEXT:    vpand %ymm1, %ymm0, %ymm0
-; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: shuffle_v64i8_0zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz:
@@ -111,7 +124,6 @@ define <64 x i8> @shuffle_v64i8_0zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz(<64 x i8> %a) {
 ; AVX512DQ-NEXT:    movl $255, %eax
 ; AVX512DQ-NEXT:    vmovd %eax, %xmm1
 ; AVX512DQ-NEXT:    vpand %ymm1, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512VBMI-LABEL: shuffle_v64i8_0zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz:
@@ -126,7 +138,7 @@ define <64 x i8> @shuffle_v64i8_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_
 ; AVX512F-LABEL: shuffle_v64i8_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpbroadcastb %xmm0, %ymm0
-; AVX512F-NEXT:    vmovdqa %ymm0, %ymm1
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: shuffle_v64i8_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00:
@@ -137,7 +149,7 @@ define <64 x i8> @shuffle_v64i8_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_
 ; AVX512DQ-LABEL: shuffle_v64i8_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00:
 ; AVX512DQ:       # %bb.0:
 ; AVX512DQ-NEXT:    vpbroadcastb %xmm0, %ymm0
-; AVX512DQ-NEXT:    vmovdqa %ymm0, %ymm1
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512VBMI-LABEL: shuffle_v64i8_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00:
@@ -151,12 +163,13 @@ define <64 x i8> @shuffle_v64i8_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_
 define <64 x i8> @shuffle_v64i8_63_62_61_60_59_58_57_56_55_54_53_52_51_50_49_48_47_46_45_44_43_42_41_40_39_38_37_36_35_34_33_32_31_30_29_28_27_26_25_24_23_22_21_20_19_18_17_16_15_14_13_12_11_10_09_08_07_06_05_04_03_02_01_00(<64 x i8> %a) {
 ; AVX512F-LABEL: shuffle_v64i8_63_62_61_60_59_58_57_56_55_54_53_52_51_50_49_48_47_46_45_44_43_42_41_40_39_38_37_36_35_34_33_32_31_30_29_28_27_26_25_24_23_22_21_20_19_18_17_16_15_14_13_12_11_10_09_08_07_06_05_04_03_02_01_00:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm3 = [15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0,15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0]
-; AVX512F-NEXT:    vpshufb %ymm3, %ymm1, %ymm1
-; AVX512F-NEXT:    vpermq {{.*#+}} ymm2 = ymm1[2,3,0,1]
-; AVX512F-NEXT:    vpshufb %ymm3, %ymm0, %ymm0
-; AVX512F-NEXT:    vpermq {{.*#+}} ymm1 = ymm0[2,3,0,1]
-; AVX512F-NEXT:    vmovdqa %ymm2, %ymm0
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm2 = [15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0,15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0]
+; AVX512F-NEXT:    vpshufb %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,3,0,1]
+; AVX512F-NEXT:    vpshufb %ymm2, %ymm1, %ymm1
+; AVX512F-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,3,0,1]
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: shuffle_v64i8_63_62_61_60_59_58_57_56_55_54_53_52_51_50_49_48_47_46_45_44_43_42_41_40_39_38_37_36_35_34_33_32_31_30_29_28_27_26_25_24_23_22_21_20_19_18_17_16_15_14_13_12_11_10_09_08_07_06_05_04_03_02_01_00:
@@ -167,12 +180,13 @@ define <64 x i8> @shuffle_v64i8_63_62_61_60_59_58_57_56_55_54_53_52_51_50_49_48_
 ;
 ; AVX512DQ-LABEL: shuffle_v64i8_63_62_61_60_59_58_57_56_55_54_53_52_51_50_49_48_47_46_45_44_43_42_41_40_39_38_37_36_35_34_33_32_31_30_29_28_27_26_25_24_23_22_21_20_19_18_17_16_15_14_13_12_11_10_09_08_07_06_05_04_03_02_01_00:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm3 = [15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0,15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0]
-; AVX512DQ-NEXT:    vpshufb %ymm3, %ymm1, %ymm1
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm1[2,3,0,1]
-; AVX512DQ-NEXT:    vpshufb %ymm3, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm1 = ymm0[2,3,0,1]
-; AVX512DQ-NEXT:    vmovdqa %ymm2, %ymm0
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm2 = [15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0,15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0]
+; AVX512DQ-NEXT:    vpshufb %ymm2, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,3,0,1]
+; AVX512DQ-NEXT:    vpshufb %ymm2, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,3,0,1]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512VBMI-LABEL: shuffle_v64i8_63_62_61_60_59_58_57_56_55_54_53_52_51_50_49_48_47_46_45_44_43_42_41_40_39_38_37_36_35_34_33_32_31_30_29_28_27_26_25_24_23_22_21_20_19_18_17_16_15_14_13_12_11_10_09_08_07_06_05_04_03_02_01_00:
@@ -188,7 +202,7 @@ define <64 x i8> @insert_dup_mem_v64i8_i32(i32* %ptr) {
 ; AVX512F-LABEL: insert_dup_mem_v64i8_i32:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpbroadcastb (%rdi), %ymm0
-; AVX512F-NEXT:    vmovdqa %ymm0, %ymm1
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: insert_dup_mem_v64i8_i32:
@@ -199,7 +213,7 @@ define <64 x i8> @insert_dup_mem_v64i8_i32(i32* %ptr) {
 ; AVX512DQ-LABEL: insert_dup_mem_v64i8_i32:
 ; AVX512DQ:       # %bb.0:
 ; AVX512DQ-NEXT:    vpbroadcastb (%rdi), %ymm0
-; AVX512DQ-NEXT:    vmovdqa %ymm0, %ymm1
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512VBMI-LABEL: insert_dup_mem_v64i8_i32:
@@ -217,7 +231,7 @@ define <64 x i8> @insert_dup_mem_v64i8_sext_i8(i8* %ptr) {
 ; AVX512F-LABEL: insert_dup_mem_v64i8_sext_i8:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpbroadcastb (%rdi), %ymm0
-; AVX512F-NEXT:    vmovdqa %ymm0, %ymm1
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: insert_dup_mem_v64i8_sext_i8:
@@ -228,7 +242,7 @@ define <64 x i8> @insert_dup_mem_v64i8_sext_i8(i8* %ptr) {
 ; AVX512DQ-LABEL: insert_dup_mem_v64i8_sext_i8:
 ; AVX512DQ:       # %bb.0:
 ; AVX512DQ-NEXT:    vpbroadcastb (%rdi), %ymm0
-; AVX512DQ-NEXT:    vmovdqa %ymm0, %ymm1
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512VBMI-LABEL: insert_dup_mem_v64i8_sext_i8:
@@ -247,7 +261,7 @@ define <64 x i8> @insert_dup_elt1_mem_v64i8_i32(i32* %ptr) {
 ; AVX512F-LABEL: insert_dup_elt1_mem_v64i8_i32:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpbroadcastb 1(%rdi), %ymm0
-; AVX512F-NEXT:    vmovdqa %ymm0, %ymm1
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: insert_dup_elt1_mem_v64i8_i32:
@@ -258,7 +272,7 @@ define <64 x i8> @insert_dup_elt1_mem_v64i8_i32(i32* %ptr) {
 ; AVX512DQ-LABEL: insert_dup_elt1_mem_v64i8_i32:
 ; AVX512DQ:       # %bb.0:
 ; AVX512DQ-NEXT:    vpbroadcastb 1(%rdi), %ymm0
-; AVX512DQ-NEXT:    vmovdqa %ymm0, %ymm1
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512VBMI-LABEL: insert_dup_elt1_mem_v64i8_i32:
@@ -276,7 +290,7 @@ define <64 x i8> @insert_dup_elt3_mem_v64i8_i32(i32* %ptr) {
 ; AVX512F-LABEL: insert_dup_elt3_mem_v64i8_i32:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpbroadcastb 3(%rdi), %ymm0
-; AVX512F-NEXT:    vmovdqa %ymm0, %ymm1
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: insert_dup_elt3_mem_v64i8_i32:
@@ -287,7 +301,7 @@ define <64 x i8> @insert_dup_elt3_mem_v64i8_i32(i32* %ptr) {
 ; AVX512DQ-LABEL: insert_dup_elt3_mem_v64i8_i32:
 ; AVX512DQ:       # %bb.0:
 ; AVX512DQ-NEXT:    vpbroadcastb 3(%rdi), %ymm0
-; AVX512DQ-NEXT:    vmovdqa %ymm0, %ymm1
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512VBMI-LABEL: insert_dup_elt3_mem_v64i8_i32:
@@ -308,7 +322,7 @@ define <64 x i8> @insert_dup_elt1_mem_v64i8_sext_i8(i8* %ptr) {
 ; AVX512F-NEXT:    shrl $8, %eax
 ; AVX512F-NEXT:    vmovd %eax, %xmm0
 ; AVX512F-NEXT:    vpbroadcastb %xmm0, %ymm0
-; AVX512F-NEXT:    vmovdqa %ymm0, %ymm1
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: insert_dup_elt1_mem_v64i8_sext_i8:
@@ -324,7 +338,7 @@ define <64 x i8> @insert_dup_elt1_mem_v64i8_sext_i8(i8* %ptr) {
 ; AVX512DQ-NEXT:    shrl $8, %eax
 ; AVX512DQ-NEXT:    vmovd %eax, %xmm0
 ; AVX512DQ-NEXT:    vpbroadcastb %xmm0, %ymm0
-; AVX512DQ-NEXT:    vmovdqa %ymm0, %ymm1
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512VBMI-LABEL: insert_dup_elt1_mem_v64i8_sext_i8:
@@ -344,10 +358,10 @@ define <64 x i8> @insert_dup_elt1_mem_v64i8_sext_i8(i8* %ptr) {
 define <64 x i8> @shuffle_v64i8_64_zz_zz_zz_zz_zz_zz_zz_65_zz_zz_zz_zz_zz_zz_zz_66_zz_zz_zz_zz_zz_zz_zz_67_zz_zz_zz_zz_zz_zz_zz_68_zz_zz_zz_zz_zz_zz_zz_69_zz_zz_zz_zz_zz_zz_zz_70_zz_zz_zz_zz_zz_zz_zz_71_zz_zz_zz_zz_zz_zz_zz(<64 x i8> %a) {
 ; AVX512F-LABEL: shuffle_v64i8_64_zz_zz_zz_zz_zz_zz_zz_65_zz_zz_zz_zz_zz_zz_zz_66_zz_zz_zz_zz_zz_zz_zz_67_zz_zz_zz_zz_zz_zz_zz_68_zz_zz_zz_zz_zz_zz_zz_69_zz_zz_zz_zz_zz_zz_zz_70_zz_zz_zz_zz_zz_zz_zz_71_zz_zz_zz_zz_zz_zz_zz:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpmovzxbq {{.*#+}} ymm2 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero,xmm0[2],zero,zero,zero,zero,zero,zero,zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero
-; AVX512F-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; AVX512F-NEXT:    vpmovzxbq {{.*#+}} ymm1 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero,xmm0[2],zero,zero,zero,zero,zero,zero,zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero
-; AVX512F-NEXT:    vmovdqa %ymm2, %ymm0
+; AVX512F-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512F-NEXT:    vpmovzxbq {{.*#+}} ymm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero,xmm1[2],zero,zero,zero,zero,zero,zero,zero,xmm1[3],zero,zero,zero,zero,zero,zero,zero
+; AVX512F-NEXT:    vpmovzxbq {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero,xmm0[2],zero,zero,zero,zero,zero,zero,zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: shuffle_v64i8_64_zz_zz_zz_zz_zz_zz_zz_65_zz_zz_zz_zz_zz_zz_zz_66_zz_zz_zz_zz_zz_zz_zz_67_zz_zz_zz_zz_zz_zz_zz_68_zz_zz_zz_zz_zz_zz_zz_69_zz_zz_zz_zz_zz_zz_zz_70_zz_zz_zz_zz_zz_zz_zz_71_zz_zz_zz_zz_zz_zz_zz:
@@ -357,10 +371,10 @@ define <64 x i8> @shuffle_v64i8_64_zz_zz_zz_zz_zz_zz_zz_65_zz_zz_zz_zz_zz_zz_zz_
 ;
 ; AVX512DQ-LABEL: shuffle_v64i8_64_zz_zz_zz_zz_zz_zz_zz_65_zz_zz_zz_zz_zz_zz_zz_66_zz_zz_zz_zz_zz_zz_zz_67_zz_zz_zz_zz_zz_zz_zz_68_zz_zz_zz_zz_zz_zz_zz_69_zz_zz_zz_zz_zz_zz_zz_70_zz_zz_zz_zz_zz_zz_zz_71_zz_zz_zz_zz_zz_zz_zz:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vpmovzxbq {{.*#+}} ymm2 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero,xmm0[2],zero,zero,zero,zero,zero,zero,zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; AVX512DQ-NEXT:    vpmovzxbq {{.*#+}} ymm1 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero,xmm0[2],zero,zero,zero,zero,zero,zero,zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-NEXT:    vmovdqa %ymm2, %ymm0
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512DQ-NEXT:    vpmovzxbq {{.*#+}} ymm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero,xmm1[2],zero,zero,zero,zero,zero,zero,zero,xmm1[3],zero,zero,zero,zero,zero,zero,zero
+; AVX512DQ-NEXT:    vpmovzxbq {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero,xmm0[2],zero,zero,zero,zero,zero,zero,zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512VBMI-LABEL: shuffle_v64i8_64_zz_zz_zz_zz_zz_zz_zz_65_zz_zz_zz_zz_zz_zz_zz_66_zz_zz_zz_zz_zz_zz_zz_67_zz_zz_zz_zz_zz_zz_zz_68_zz_zz_zz_zz_zz_zz_zz_69_zz_zz_zz_zz_zz_zz_zz_70_zz_zz_zz_zz_zz_zz_zz_71_zz_zz_zz_zz_zz_zz_zz:
@@ -374,10 +388,10 @@ define <64 x i8> @shuffle_v64i8_64_zz_zz_zz_zz_zz_zz_zz_65_zz_zz_zz_zz_zz_zz_zz_
 define <64 x i8> @shuffle_v64i8_64_zz_zz_zz_65_zz_zz_zz_66_zz_zz_zz_67_zz_zz_zz_68_zz_zz_zz_69_zz_zz_zz_70_zz_zz_zz_71_zz_zz_zz_72_zz_zz_zz_73_zz_zz_zz_74_zz_zz_zz_75_zz_zz_zz_76_zz_zz_zz_77_zz_zz_zz_78_zz_zz_zz_79_zz_zz_zz(<64 x i8> %a) {
 ; AVX512F-LABEL: shuffle_v64i8_64_zz_zz_zz_65_zz_zz_zz_66_zz_zz_zz_67_zz_zz_zz_68_zz_zz_zz_69_zz_zz_zz_70_zz_zz_zz_71_zz_zz_zz_72_zz_zz_zz_73_zz_zz_zz_74_zz_zz_zz_75_zz_zz_zz_76_zz_zz_zz_77_zz_zz_zz_78_zz_zz_zz_79_zz_zz_zz:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpmovzxbd {{.*#+}} ymm2 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
-; AVX512F-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX512F-NEXT:    vpmovzxbd {{.*#+}} ymm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
-; AVX512F-NEXT:    vmovdqa %ymm2, %ymm0
+; AVX512F-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512F-NEXT:    vpmovzxbd {{.*#+}} ymm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero
+; AVX512F-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: shuffle_v64i8_64_zz_zz_zz_65_zz_zz_zz_66_zz_zz_zz_67_zz_zz_zz_68_zz_zz_zz_69_zz_zz_zz_70_zz_zz_zz_71_zz_zz_zz_72_zz_zz_zz_73_zz_zz_zz_74_zz_zz_zz_75_zz_zz_zz_76_zz_zz_zz_77_zz_zz_zz_78_zz_zz_zz_79_zz_zz_zz:
@@ -387,10 +401,10 @@ define <64 x i8> @shuffle_v64i8_64_zz_zz_zz_65_zz_zz_zz_66_zz_zz_zz_67_zz_zz_zz_
 ;
 ; AVX512DQ-LABEL: shuffle_v64i8_64_zz_zz_zz_65_zz_zz_zz_66_zz_zz_zz_67_zz_zz_zz_68_zz_zz_zz_69_zz_zz_zz_70_zz_zz_zz_71_zz_zz_zz_72_zz_zz_zz_73_zz_zz_zz_74_zz_zz_zz_75_zz_zz_zz_76_zz_zz_zz_77_zz_zz_zz_78_zz_zz_zz_79_zz_zz_zz:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vpmovzxbd {{.*#+}} ymm2 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX512DQ-NEXT:    vpmovzxbd {{.*#+}} ymm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
-; AVX512DQ-NEXT:    vmovdqa %ymm2, %ymm0
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512DQ-NEXT:    vpmovzxbd {{.*#+}} ymm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero
+; AVX512DQ-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512VBMI-LABEL: shuffle_v64i8_64_zz_zz_zz_65_zz_zz_zz_66_zz_zz_zz_67_zz_zz_zz_68_zz_zz_zz_69_zz_zz_zz_70_zz_zz_zz_71_zz_zz_zz_72_zz_zz_zz_73_zz_zz_zz_74_zz_zz_zz_75_zz_zz_zz_76_zz_zz_zz_77_zz_zz_zz_78_zz_zz_zz_79_zz_zz_zz:
@@ -404,10 +418,10 @@ define <64 x i8> @shuffle_v64i8_64_zz_zz_zz_65_zz_zz_zz_66_zz_zz_zz_67_zz_zz_zz_
 define <64 x i8> @shuffle_v64i8_64_zz_65_zz_66_zz_67_zz_68_zz_69_zz_70_zz_71_zz_72_zz_73_zz_74_zz_75_zz_76_zz_77_zz_78_zz_79_zz_80_zz_81_zz_82_zz_83_zz_84_zz_85_zz_86_zz_87_zz_88_zz_89_zz_90_zz_91_zz_92_zz_93_zz_94_zz_95_zz(<64 x i8> %a) {
 ; AVX512F-LABEL: shuffle_v64i8_64_zz_65_zz_66_zz_67_zz_68_zz_69_zz_70_zz_71_zz_72_zz_73_zz_74_zz_75_zz_76_zz_77_zz_78_zz_79_zz_80_zz_81_zz_82_zz_83_zz_84_zz_85_zz_86_zz_87_zz_88_zz_89_zz_90_zz_91_zz_92_zz_93_zz_94_zz_95_zz:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
-; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
-; AVX512F-NEXT:    vmovdqa %ymm2, %ymm0
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: shuffle_v64i8_64_zz_65_zz_66_zz_67_zz_68_zz_69_zz_70_zz_71_zz_72_zz_73_zz_74_zz_75_zz_76_zz_77_zz_78_zz_79_zz_80_zz_81_zz_82_zz_83_zz_84_zz_85_zz_86_zz_87_zz_88_zz_89_zz_90_zz_91_zz_92_zz_93_zz_94_zz_95_zz:
@@ -417,10 +431,10 @@ define <64 x i8> @shuffle_v64i8_64_zz_65_zz_66_zz_67_zz_68_zz_69_zz_70_zz_71_zz_
 ;
 ; AVX512DQ-LABEL: shuffle_v64i8_64_zz_65_zz_66_zz_67_zz_68_zz_69_zz_70_zz_71_zz_72_zz_73_zz_74_zz_75_zz_76_zz_77_zz_78_zz_79_zz_80_zz_81_zz_82_zz_83_zz_84_zz_85_zz_86_zz_87_zz_88_zz_89_zz_90_zz_91_zz_92_zz_93_zz_94_zz_95_zz:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vpmovzxbw {{.*#+}} ymm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
-; AVX512DQ-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; AVX512DQ-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
-; AVX512DQ-NEXT:    vmovdqa %ymm2, %ymm0
+; AVX512DQ-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512DQ-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512DQ-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512VBMI-LABEL: shuffle_v64i8_64_zz_65_zz_66_zz_67_zz_68_zz_69_zz_70_zz_71_zz_72_zz_73_zz_74_zz_75_zz_76_zz_77_zz_78_zz_79_zz_80_zz_81_zz_82_zz_83_zz_84_zz_85_zz_86_zz_87_zz_88_zz_89_zz_90_zz_91_zz_92_zz_93_zz_94_zz_95_zz:
@@ -434,12 +448,13 @@ define <64 x i8> @shuffle_v64i8_64_zz_65_zz_66_zz_67_zz_68_zz_69_zz_70_zz_71_zz_
 define <64 x i8> @shuffle_v64i8_63_zz_61_zz_59_zz_57_zz_55_zz_53_zz_51_zz_49_zz_47_zz_45_zz_43_zz_41_zz_39_zz_37_zz_35_zz_33_zz_31_zz_29_zz_27_zz_25_zz_23_zz_21_zz_19_zz_17_zz_15_zz_13_zz_11_zz_9_zz_7_zz_5_zz_3_zz_1_zz(<64 x i8> %a) {
 ; AVX512F-LABEL: shuffle_v64i8_63_zz_61_zz_59_zz_57_zz_55_zz_53_zz_51_zz_49_zz_47_zz_45_zz_43_zz_41_zz_39_zz_37_zz_35_zz_33_zz_31_zz_29_zz_27_zz_25_zz_23_zz_21_zz_19_zz_17_zz_15_zz_13_zz_11_zz_9_zz_7_zz_5_zz_3_zz_1_zz:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,3,0,1]
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm3 = [15,128,13,128,11,128,9,128,7,128,5,128,3,128,1,128,15,128,13,128,11,128,9,128,7,128,5,128,3,128,1,128]
-; AVX512F-NEXT:    vpshufb %ymm3, %ymm1, %ymm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512F-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,3,0,1]
-; AVX512F-NEXT:    vpshufb %ymm3, %ymm0, %ymm1
-; AVX512F-NEXT:    vmovdqa %ymm2, %ymm0
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm2 = [15,128,13,128,11,128,9,128,7,128,5,128,3,128,1,128,15,128,13,128,11,128,9,128,7,128,5,128,3,128,1,128]
+; AVX512F-NEXT:    vpshufb %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,3,0,1]
+; AVX512F-NEXT:    vpshufb %ymm2, %ymm1, %ymm1
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: shuffle_v64i8_63_zz_61_zz_59_zz_57_zz_55_zz_53_zz_51_zz_49_zz_47_zz_45_zz_43_zz_41_zz_39_zz_37_zz_35_zz_33_zz_31_zz_29_zz_27_zz_25_zz_23_zz_21_zz_19_zz_17_zz_15_zz_13_zz_11_zz_9_zz_7_zz_5_zz_3_zz_1_zz:
@@ -450,12 +465,13 @@ define <64 x i8> @shuffle_v64i8_63_zz_61_zz_59_zz_57_zz_55_zz_53_zz_51_zz_49_zz_
 ;
 ; AVX512DQ-LABEL: shuffle_v64i8_63_zz_61_zz_59_zz_57_zz_55_zz_53_zz_51_zz_49_zz_47_zz_45_zz_43_zz_41_zz_39_zz_37_zz_35_zz_33_zz_31_zz_29_zz_27_zz_25_zz_23_zz_21_zz_19_zz_17_zz_15_zz_13_zz_11_zz_9_zz_7_zz_5_zz_3_zz_1_zz:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,3,0,1]
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm3 = [15,128,13,128,11,128,9,128,7,128,5,128,3,128,1,128,15,128,13,128,11,128,9,128,7,128,5,128,3,128,1,128]
-; AVX512DQ-NEXT:    vpshufb %ymm3, %ymm1, %ymm2
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,3,0,1]
-; AVX512DQ-NEXT:    vpshufb %ymm3, %ymm0, %ymm1
-; AVX512DQ-NEXT:    vmovdqa %ymm2, %ymm0
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm2 = [15,128,13,128,11,128,9,128,7,128,5,128,3,128,1,128,15,128,13,128,11,128,9,128,7,128,5,128,3,128,1,128]
+; AVX512DQ-NEXT:    vpshufb %ymm2, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,3,0,1]
+; AVX512DQ-NEXT:    vpshufb %ymm2, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512VBMI-LABEL: shuffle_v64i8_63_zz_61_zz_59_zz_57_zz_55_zz_53_zz_51_zz_49_zz_47_zz_45_zz_43_zz_41_zz_39_zz_37_zz_35_zz_33_zz_31_zz_29_zz_27_zz_25_zz_23_zz_21_zz_19_zz_17_zz_15_zz_13_zz_11_zz_9_zz_7_zz_5_zz_3_zz_1_zz:
@@ -471,15 +487,17 @@ define <64 x i8> @shuffle_v64i8_63_zz_61_zz_59_zz_57_zz_55_zz_53_zz_51_zz_49_zz_
 define <64 x i8> @shuffle_v64i8_63_64_61_66_59_68_57_70_55_72_53_74_51_76_49_78_47_80_45_82_43_84_41_86_39_88_37_90_35_92_33_94_31_96_29_98_27_100_25_102_23_104_21_106_19_108_17_110_15_112_13_114_11_116_9_118_7_120_5_122_3_124_1_126(<64 x i8> %a, <64 x i8> %b) {
 ; AVX512F-LABEL: shuffle_v64i8_63_64_61_66_59_68_57_70_55_72_53_74_51_76_49_78_47_80_45_82_43_84_41_86_39_88_37_90_35_92_33_94_31_96_29_98_27_100_25_102_23_104_21_106_19_108_17_110_15_112_13_114_11_116_9_118_7_120_5_122_3_124_1_126:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,3,0,1]
-; AVX512F-NEXT:    vpbroadcastw {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512F-NEXT:    vpblendvb %ymm4, %ymm2, %ymm1, %ymm1
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm5 = [15,0,13,2,11,4,9,6,7,8,5,10,3,12,1,14,15,0,13,2,11,4,9,6,7,8,5,10,3,12,1,14]
-; AVX512F-NEXT:    vpshufb %ymm5, %ymm1, %ymm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
 ; AVX512F-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,3,0,1]
+; AVX512F-NEXT:    vpbroadcastw {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
 ; AVX512F-NEXT:    vpblendvb %ymm4, %ymm3, %ymm0, %ymm0
-; AVX512F-NEXT:    vpshufb %ymm5, %ymm0, %ymm1
-; AVX512F-NEXT:    vmovdqa %ymm2, %ymm0
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm3 = [15,0,13,2,11,4,9,6,7,8,5,10,3,12,1,14,15,0,13,2,11,4,9,6,7,8,5,10,3,12,1,14]
+; AVX512F-NEXT:    vpshufb %ymm3, %ymm0, %ymm0
+; AVX512F-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[2,3,0,1]
+; AVX512F-NEXT:    vpblendvb %ymm4, %ymm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vpshufb %ymm3, %ymm1, %ymm1
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: shuffle_v64i8_63_64_61_66_59_68_57_70_55_72_53_74_51_76_49_78_47_80_45_82_43_84_41_86_39_88_37_90_35_92_33_94_31_96_29_98_27_100_25_102_23_104_21_106_19_108_17_110_15_112_13_114_11_116_9_118_7_120_5_122_3_124_1_126:
@@ -498,15 +516,17 @@ define <64 x i8> @shuffle_v64i8_63_64_61_66_59_68_57_70_55_72_53_74_51_76_49_78_
 ;
 ; AVX512DQ-LABEL: shuffle_v64i8_63_64_61_66_59_68_57_70_55_72_53_74_51_76_49_78_47_80_45_82_43_84_41_86_39_88_37_90_35_92_33_94_31_96_29_98_27_100_25_102_23_104_21_106_19_108_17_110_15_112_13_114_11_116_9_118_7_120_5_122_3_124_1_126:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,3,0,1]
-; AVX512DQ-NEXT:    vpbroadcastw {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512DQ-NEXT:    vpblendvb %ymm4, %ymm2, %ymm1, %ymm1
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm5 = [15,0,13,2,11,4,9,6,7,8,5,10,3,12,1,14,15,0,13,2,11,4,9,6,7,8,5,10,3,12,1,14]
-; AVX512DQ-NEXT:    vpshufb %ymm5, %ymm1, %ymm2
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,3,0,1]
+; AVX512DQ-NEXT:    vpbroadcastw {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
 ; AVX512DQ-NEXT:    vpblendvb %ymm4, %ymm3, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpshufb %ymm5, %ymm0, %ymm1
-; AVX512DQ-NEXT:    vmovdqa %ymm2, %ymm0
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm3 = [15,0,13,2,11,4,9,6,7,8,5,10,3,12,1,14,15,0,13,2,11,4,9,6,7,8,5,10,3,12,1,14]
+; AVX512DQ-NEXT:    vpshufb %ymm3, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[2,3,0,1]
+; AVX512DQ-NEXT:    vpblendvb %ymm4, %ymm1, %ymm2, %ymm1
+; AVX512DQ-NEXT:    vpshufb %ymm3, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512VBMI-LABEL: shuffle_v64i8_63_64_61_66_59_68_57_70_55_72_53_74_51_76_49_78_47_80_45_82_43_84_41_86_39_88_37_90_35_92_33_94_31_96_29_98_27_100_25_102_23_104_21_106_19_108_17_110_15_112_13_114_11_116_9_118_7_120_5_122_3_124_1_126:
@@ -521,14 +541,17 @@ define <64 x i8> @shuffle_v64i8_63_64_61_66_59_68_57_70_55_72_53_74_51_76_49_78_
 define <64 x i8> @shuffle_v64i8_shift_00_02_04_06_08_10_12_14_16_18_20_22_24_26_28_30_32_34_36_38_40_42_44_46_48_50_52_54_56_58_60_62_64_66_68_70_72_74_76_78_80_82_84_86_88_90_92_94_96_98_100_102_104_106_108_110_112_114_116_118_120_122_124_126(<32 x i16> %a0, <32 x i16> %a1) {
 ; AVX512F-LABEL: shuffle_v64i8_shift_00_02_04_06_08_10_12_14_16_18_20_22_24_26_28_30_32_34_36_38_40_42_44_46_48_50_52_54_56_58_60_62_64_66_68_70_72_74_76_78_80_82_84_86_88_90_92_94_96_98_100_102_104_106_108_110_112_114_116_118_120_122_124_126:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpsrlw $8, %ymm1, %ymm1
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
+; AVX512F-NEXT:    vpsrlw $8, %ymm3, %ymm3
 ; AVX512F-NEXT:    vpsrlw $8, %ymm0, %ymm0
-; AVX512F-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
-; AVX512F-NEXT:    vpsrlw $8, %ymm3, %ymm1
+; AVX512F-NEXT:    vpackuswb %ymm3, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpsrlw $8, %ymm2, %ymm2
-; AVX512F-NEXT:    vpackuswb %ymm1, %ymm2, %ymm1
-; AVX512F-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX512F-NEXT:    vpsrlw $8, %ymm1, %ymm1
+; AVX512F-NEXT:    vpackuswb %ymm2, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
+; AVX512F-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: shuffle_v64i8_shift_00_02_04_06_08_10_12_14_16_18_20_22_24_26_28_30_32_34_36_38_40_42_44_46_48_50_52_54_56_58_60_62_64_66_68_70_72_74_76_78_80_82_84_86_88_90_92_94_96_98_100_102_104_106_108_110_112_114_116_118_120_122_124_126:
@@ -542,14 +565,17 @@ define <64 x i8> @shuffle_v64i8_shift_00_02_04_06_08_10_12_14_16_18_20_22_24_26_
 ;
 ; AVX512DQ-LABEL: shuffle_v64i8_shift_00_02_04_06_08_10_12_14_16_18_20_22_24_26_28_30_32_34_36_38_40_42_44_46_48_50_52_54_56_58_60_62_64_66_68_70_72_74_76_78_80_82_84_86_88_90_92_94_96_98_100_102_104_106_108_110_112_114_116_118_120_122_124_126:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vpsrlw $8, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
+; AVX512DQ-NEXT:    vpsrlw $8, %ymm3, %ymm3
 ; AVX512DQ-NEXT:    vpsrlw $8, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpsrlw $8, %ymm3, %ymm1
+; AVX512DQ-NEXT:    vpackuswb %ymm3, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpsrlw $8, %ymm2, %ymm2
-; AVX512DQ-NEXT:    vpackuswb %ymm1, %ymm2, %ymm1
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX512DQ-NEXT:    vpsrlw $8, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpackuswb %ymm2, %ymm1, %ymm1
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512VBMI-LABEL: shuffle_v64i8_shift_00_02_04_06_08_10_12_14_16_18_20_22_24_26_28_30_32_34_36_38_40_42_44_46_48_50_52_54_56_58_60_62_64_66_68_70_72_74_76_78_80_82_84_86_88_90_92_94_96_98_100_102_104_106_108_110_112_114_116_118_120_122_124_126:
@@ -568,12 +594,15 @@ define <64 x i8> @shuffle_v64i8_shift_00_02_04_06_08_10_12_14_16_18_20_22_24_26_
 define <64 x i8> @shuffle_v64i8_shift_00_02_04_06_08_10_12_14_64_66_68_70_72_74_76_78_16_18_20_22_24_26_28_30_80_82_84_86_88_90_92_94_32_34_36_38_40_42_44_46_96_98_100_102_104_106_108_110_48_50_52_54_56_58_60_62_112_114_116_118_120_122_124_126(<32 x i16> %a0, <32 x i16> %a1) {
 ; AVX512F-LABEL: shuffle_v64i8_shift_00_02_04_06_08_10_12_14_64_66_68_70_72_74_76_78_16_18_20_22_24_26_28_30_80_82_84_86_88_90_92_94_32_34_36_38_40_42_44_46_96_98_100_102_104_106_108_110_48_50_52_54_56_58_60_62_112_114_116_118_120_122_124_126:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpsrlw $8, %ymm1, %ymm1
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
 ; AVX512F-NEXT:    vpsrlw $8, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpsrlw $8, %ymm3, %ymm3
-; AVX512F-NEXT:    vpackuswb %ymm3, %ymm1, %ymm1
-; AVX512F-NEXT:    vpsrlw $8, %ymm2, %ymm2
-; AVX512F-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vpsrlw $8, %ymm1, %ymm1
+; AVX512F-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vpsrlw $8, %ymm2, %ymm1
+; AVX512F-NEXT:    vpackuswb %ymm1, %ymm3, %ymm1
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: shuffle_v64i8_shift_00_02_04_06_08_10_12_14_64_66_68_70_72_74_76_78_16_18_20_22_24_26_28_30_80_82_84_86_88_90_92_94_32_34_36_38_40_42_44_46_96_98_100_102_104_106_108_110_48_50_52_54_56_58_60_62_112_114_116_118_120_122_124_126:
@@ -585,12 +614,15 @@ define <64 x i8> @shuffle_v64i8_shift_00_02_04_06_08_10_12_14_64_66_68_70_72_74_
 ;
 ; AVX512DQ-LABEL: shuffle_v64i8_shift_00_02_04_06_08_10_12_14_64_66_68_70_72_74_76_78_16_18_20_22_24_26_28_30_80_82_84_86_88_90_92_94_32_34_36_38_40_42_44_46_96_98_100_102_104_106_108_110_48_50_52_54_56_58_60_62_112_114_116_118_120_122_124_126:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vpsrlw $8, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
 ; AVX512DQ-NEXT:    vpsrlw $8, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpsrlw $8, %ymm3, %ymm3
-; AVX512DQ-NEXT:    vpackuswb %ymm3, %ymm1, %ymm1
-; AVX512DQ-NEXT:    vpsrlw $8, %ymm2, %ymm2
-; AVX512DQ-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpsrlw $8, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpsrlw $8, %ymm2, %ymm1
+; AVX512DQ-NEXT:    vpackuswb %ymm1, %ymm3, %ymm1
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512VBMI-LABEL: shuffle_v64i8_shift_00_02_04_06_08_10_12_14_64_66_68_70_72_74_76_78_16_18_20_22_24_26_28_30_80_82_84_86_88_90_92_94_32_34_36_38_40_42_44_46_96_98_100_102_104_106_108_110_48_50_52_54_56_58_60_62_112_114_116_118_120_122_124_126:
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-512-v8.ll b/llvm/test/CodeGen/X86/vector-shuffle-512-v8.ll
index 52b04fdc8..dfe2d4db1 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-512-v8.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-512-v8.ll
@@ -841,19 +841,11 @@ define <8 x double> @shuffle_v8f64_f511235a(<8 x double> %a, <8 x double> %b) {
 }
 
 define <8 x double> @shuffle_v8f64_1z2z5z6z(<8 x double> %a, <8 x double> %b) {
-; AVX512F-LABEL: shuffle_v8f64_1z2z5z6z:
-; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT:    vmovapd {{.*#+}} zmm2 = [1,8,2,8,5,8,6,8]
-; AVX512F-NEXT:    vpermt2pd %zmm1, %zmm2, %zmm0
-; AVX512F-NEXT:    retq
-;
-; AVX512F-32-LABEL: shuffle_v8f64_1z2z5z6z:
-; AVX512F-32:       # %bb.0:
-; AVX512F-32-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
-; AVX512F-32-NEXT:    vmovapd {{.*#+}} zmm2 = [1,0,8,0,2,0,8,0,5,0,8,0,6,0,8,0]
-; AVX512F-32-NEXT:    vpermt2pd %zmm1, %zmm2, %zmm0
-; AVX512F-32-NEXT:    retl
+; ALL-LABEL: shuffle_v8f64_1z2z5z6z:
+; ALL:       # %bb.0:
+; ALL-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; ALL-NEXT:    vshufpd {{.*#+}} zmm0 = zmm0[1],zmm1[0],zmm0[2],zmm1[2],zmm0[5],zmm1[4],zmm0[6],zmm1[6]
+; ALL-NEXT:    ret{{[l|q]}}
   %shuffle = shufflevector <8 x double> %a, <8 x double> <double 0.000000e+00, double undef, double undef, double undef, double undef, double undef, double undef, double undef>, <8 x i32> <i32 1, i32 8, i32 2, i32 8, i32 5, i32 8, i32 6, i32 8>
   ret <8 x double> %shuffle
 }
@@ -1767,8 +1759,8 @@ define <8 x double> @shuffle_v8f64_0z2z4z6z(<8 x double> %a, <8 x double> %b) {
 ;
 ; ALL-LABEL: shuffle_v8f64_0z2z4z6z:
 ; ALL:       # %bb.0:
-; ALL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; ALL-NEXT:    vunpcklpd {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[2],zmm1[2],zmm0[4],zmm1[4],zmm0[6],zmm1[6]
+; ALL-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; ALL-NEXT:    vshufpd {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[2],zmm1[2],zmm0[4],zmm1[4],zmm0[6],zmm1[6]
 ; ALL-NEXT:    ret{{[l|q]}}
   %shuffle = shufflevector <8 x double> %a, <8 x double> zeroinitializer, <8 x i32><i32 0, i32 8, i32 2, i32 8, i32 4, i32 8, i32 6, i32 8>
   ret <8 x double> %shuffle
@@ -1809,8 +1801,8 @@ define <8 x double> @shuffle_v8f64_z9zbzdzf(<8 x double> %a, <8 x double> %b) {
 ;
 ; ALL-LABEL: shuffle_v8f64_z9zbzdzf:
 ; ALL:       # %bb.0:
-; ALL-NEXT:    vxorps %xmm0, %xmm0, %xmm0
-; ALL-NEXT:    vunpckhpd {{.*#+}} zmm0 = zmm0[1],zmm1[1],zmm0[3],zmm1[3],zmm0[5],zmm1[5],zmm0[7],zmm1[7]
+; ALL-NEXT:    vxorpd %xmm0, %xmm0, %xmm0
+; ALL-NEXT:    vshufpd {{.*#+}} zmm0 = zmm0[0],zmm1[1],zmm0[2],zmm1[3],zmm0[4],zmm1[5],zmm0[6],zmm1[7]
 ; ALL-NEXT:    ret{{[l|q]}}
   %shuffle = shufflevector <8 x double> zeroinitializer, <8 x double> %b, <8 x i32><i32 0, i32 9, i32 0, i32 11, i32 0, i32 13, i32 0, i32 15>
   ret <8 x double> %shuffle
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-avx512.ll b/llvm/test/CodeGen/X86/vector-shuffle-avx512.ll
index 65472aaee..c669562dc 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-avx512.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-avx512.ll
@@ -1,501 +1,256 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-pc-linux-gnu -mcpu=skx | FileCheck %s --check-prefix=SKX64
-; RUN: llc < %s -mtriple=x86_64-pc-linux-gnu -mcpu=knl | FileCheck %s --check-prefix=KNL64
-; RUN: llc < %s -mtriple=i386-pc-linux-gnu -mcpu=skx | FileCheck %s --check-prefix=SKX32
-; RUN: llc < %s -mtriple=i386-pc-linux-gnu -mcpu=knl | FileCheck %s --check-prefix=KNL32
+; RUN: llc < %s -mtriple=x86_64-pc-linux-gnu -mcpu=skx | FileCheck %s --check-prefixes=CHECK,SKX,X64,SKX64
+; RUN: llc < %s -mtriple=x86_64-pc-linux-gnu -mcpu=knl | FileCheck %s --check-prefixes=CHECK,KNL,X64,KNL64
+; RUN: llc < %s -mtriple=i386-pc-linux-gnu -mcpu=skx | FileCheck %s --check-prefixes=CHECK,SKX,X86,SKX32
+; RUN: llc < %s -mtriple=i386-pc-linux-gnu -mcpu=knl | FileCheck %s --check-prefixes=CHECK,KNL,X86,KNL32
 
 ;expand 128 -> 256 include <4 x float> <2 x double>
 define <8 x float> @expand(<4 x float> %a) {
-; SKX64-LABEL: expand:
-; SKX64:       # %bb.0:
-; SKX64-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
-; SKX64-NEXT:    movb $5, %al
-; SKX64-NEXT:    kmovd %eax, %k1
-; SKX64-NEXT:    vexpandps %ymm0, %ymm0 {%k1} {z}
-; SKX64-NEXT:    retq
-;
-; KNL64-LABEL: expand:
-; KNL64:       # %bb.0:
-; KNL64-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,1,1,3]
-; KNL64-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; KNL64-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3,4,5,6,7]
-; KNL64-NEXT:    retq
-;
-; SKX32-LABEL: expand:
-; SKX32:       # %bb.0:
-; SKX32-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
-; SKX32-NEXT:    movb $5, %al
-; SKX32-NEXT:    kmovd %eax, %k1
-; SKX32-NEXT:    vexpandps %ymm0, %ymm0 {%k1} {z}
-; SKX32-NEXT:    retl
-;
-; KNL32-LABEL: expand:
-; KNL32:       # %bb.0:
-; KNL32-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,1,1,3]
-; KNL32-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; KNL32-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3,4,5,6,7]
-; KNL32-NEXT:    retl
+; SKX-LABEL: expand:
+; SKX:       # %bb.0:
+; SKX-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; SKX-NEXT:    movb $5, %al
+; SKX-NEXT:    kmovd %eax, %k1
+; SKX-NEXT:    vexpandps %ymm0, %ymm0 {%k1} {z}
+; SKX-NEXT:    ret{{[l|q]}}
+;
+; KNL-LABEL: expand:
+; KNL:       # %bb.0:
+; KNL-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,1,1,3]
+; KNL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; KNL-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3,4,5,6,7]
+; KNL-NEXT:    ret{{[l|q]}}
    %res = shufflevector <4 x float> %a, <4 x float> zeroinitializer, <8 x i32> <i32 0, i32 5, i32 1, i32 5, i32 5, i32 5, i32 5, i32 5>
    ret <8 x float> %res
 }
 
 define <8 x float> @expand1(<4 x float> %a ) {
-; SKX64-LABEL: expand1:
-; SKX64:       # %bb.0:
-; SKX64-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
-; SKX64-NEXT:    movb $-86, %al
-; SKX64-NEXT:    kmovd %eax, %k1
-; SKX64-NEXT:    vexpandps %ymm0, %ymm0 {%k1} {z}
-; SKX64-NEXT:    retq
-;
-; KNL64-LABEL: expand1:
-; KNL64:       # %bb.0:
-; KNL64-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
-; KNL64-NEXT:    vmovaps {{.*#+}} ymm1 = <u,0,u,1,u,2,u,3>
-; KNL64-NEXT:    vpermps %ymm0, %ymm1, %ymm0
-; KNL64-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; KNL64-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4],ymm0[5],ymm1[6],ymm0[7]
-; KNL64-NEXT:    retq
-;
-; SKX32-LABEL: expand1:
-; SKX32:       # %bb.0:
-; SKX32-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
-; SKX32-NEXT:    movb $-86, %al
-; SKX32-NEXT:    kmovd %eax, %k1
-; SKX32-NEXT:    vexpandps %ymm0, %ymm0 {%k1} {z}
-; SKX32-NEXT:    retl
-;
-; KNL32-LABEL: expand1:
-; KNL32:       # %bb.0:
-; KNL32-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
-; KNL32-NEXT:    vmovaps {{.*#+}} ymm1 = <u,0,u,1,u,2,u,3>
-; KNL32-NEXT:    vpermps %ymm0, %ymm1, %ymm0
-; KNL32-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; KNL32-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4],ymm0[5],ymm1[6],ymm0[7]
-; KNL32-NEXT:    retl
+; SKX-LABEL: expand1:
+; SKX:       # %bb.0:
+; SKX-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; SKX-NEXT:    movb $-86, %al
+; SKX-NEXT:    kmovd %eax, %k1
+; SKX-NEXT:    vexpandps %ymm0, %ymm0 {%k1} {z}
+; SKX-NEXT:    ret{{[l|q]}}
+;
+; KNL-LABEL: expand1:
+; KNL:       # %bb.0:
+; KNL-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; KNL-NEXT:    vmovaps {{.*#+}} ymm1 = <u,0,u,1,u,2,u,3>
+; KNL-NEXT:    vpermps %ymm0, %ymm1, %ymm0
+; KNL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; KNL-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4],ymm0[5],ymm1[6],ymm0[7]
+; KNL-NEXT:    ret{{[l|q]}}
    %res = shufflevector <4 x float> zeroinitializer, <4 x float> %a, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
    ret <8 x float> %res
 }
 
 ;Expand 128 -> 256 test <2 x double> -> <4 x double>
 define <4 x double> @expand2(<2 x double> %a) {
-; SKX64-LABEL: expand2:
-; SKX64:       # %bb.0:
-; SKX64-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
-; SKX64-NEXT:    vperm2f128 {{.*#+}} ymm1 = zero,zero,ymm0[0,1]
-; SKX64-NEXT:    vmovaps %xmm0, %xmm0
-; SKX64-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
-; SKX64-NEXT:    retq
-;
-; KNL64-LABEL: expand2:
-; KNL64:       # %bb.0:
-; KNL64-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
-; KNL64-NEXT:    vperm2f128 {{.*#+}} ymm1 = zero,zero,ymm0[0,1]
-; KNL64-NEXT:    vmovaps %xmm0, %xmm0
-; KNL64-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
-; KNL64-NEXT:    retq
-;
-; SKX32-LABEL: expand2:
-; SKX32:       # %bb.0:
-; SKX32-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
-; SKX32-NEXT:    vperm2f128 {{.*#+}} ymm1 = zero,zero,ymm0[0,1]
-; SKX32-NEXT:    vmovaps %xmm0, %xmm0
-; SKX32-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
-; SKX32-NEXT:    retl
-;
-; KNL32-LABEL: expand2:
-; KNL32:       # %bb.0:
-; KNL32-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
-; KNL32-NEXT:    vperm2f128 {{.*#+}} ymm1 = zero,zero,ymm0[0,1]
-; KNL32-NEXT:    vmovaps %xmm0, %xmm0
-; KNL32-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
-; KNL32-NEXT:    retl
+; CHECK-LABEL: expand2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; CHECK-NEXT:    vperm2f128 {{.*#+}} ymm1 = zero,zero,ymm0[0,1]
+; CHECK-NEXT:    vmovaps %xmm0, %xmm0
+; CHECK-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
+; CHECK-NEXT:    ret{{[l|q]}}
    %res = shufflevector <2 x double> %a, <2 x double> zeroinitializer, <4 x i32> <i32 0, i32 2, i32 2, i32 1>
    ret <4 x double> %res
 }
 
 ;expand 128 -> 256 include case <4 x i32> <8 x i32>
 define <8 x i32> @expand3(<4 x i32> %a ) {
-; SKX64-LABEL: expand3:
-; SKX64:       # %bb.0:
-; SKX64-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
-; SKX64-NEXT:    movb $-127, %al
-; SKX64-NEXT:    kmovd %eax, %k1
-; SKX64-NEXT:    vpexpandd %ymm0, %ymm0 {%k1} {z}
-; SKX64-NEXT:    retq
-;
-; KNL64-LABEL: expand3:
-; KNL64:       # %bb.0:
-; KNL64-NEXT:    vbroadcastsd %xmm0, %ymm0
-; KNL64-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; KNL64-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3,4,5,6],ymm0[7]
-; KNL64-NEXT:    retq
-;
-; SKX32-LABEL: expand3:
-; SKX32:       # %bb.0:
-; SKX32-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
-; SKX32-NEXT:    movb $-127, %al
-; SKX32-NEXT:    kmovd %eax, %k1
-; SKX32-NEXT:    vpexpandd %ymm0, %ymm0 {%k1} {z}
-; SKX32-NEXT:    retl
-;
-; KNL32-LABEL: expand3:
-; KNL32:       # %bb.0:
-; KNL32-NEXT:    vbroadcastsd %xmm0, %ymm0
-; KNL32-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; KNL32-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3,4,5,6],ymm0[7]
-; KNL32-NEXT:    retl
+; SKX-LABEL: expand3:
+; SKX:       # %bb.0:
+; SKX-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; SKX-NEXT:    movb $-127, %al
+; SKX-NEXT:    kmovd %eax, %k1
+; SKX-NEXT:    vpexpandd %ymm0, %ymm0 {%k1} {z}
+; SKX-NEXT:    ret{{[l|q]}}
+;
+; KNL-LABEL: expand3:
+; KNL:       # %bb.0:
+; KNL-NEXT:    vbroadcastsd %xmm0, %ymm0
+; KNL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; KNL-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3,4,5,6],ymm0[7]
+; KNL-NEXT:    ret{{[l|q]}}
    %res = shufflevector <4 x i32> zeroinitializer, <4 x i32> %a, <8 x i32> <i32 4, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0,i32 5>
    ret <8 x i32> %res
 }
 
 ;expand 128 -> 256 include case <2 x i64> <4 x i64>
 define <4 x i64> @expand4(<2 x i64> %a ) {
-; SKX64-LABEL: expand4:
-; SKX64:       # %bb.0:
-; SKX64-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
-; SKX64-NEXT:    movb $9, %al
-; SKX64-NEXT:    kmovd %eax, %k1
-; SKX64-NEXT:    vpexpandq %ymm0, %ymm0 {%k1} {z}
-; SKX64-NEXT:    retq
-;
-; KNL64-LABEL: expand4:
-; KNL64:       # %bb.0:
-; KNL64-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
-; KNL64-NEXT:    vperm2f128 {{.*#+}} ymm1 = zero,zero,ymm0[0,1]
-; KNL64-NEXT:    vmovaps %xmm0, %xmm0
-; KNL64-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
-; KNL64-NEXT:    retq
-;
-; SKX32-LABEL: expand4:
-; SKX32:       # %bb.0:
-; SKX32-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
-; SKX32-NEXT:    movb $9, %al
-; SKX32-NEXT:    kmovd %eax, %k1
-; SKX32-NEXT:    vpexpandq %ymm0, %ymm0 {%k1} {z}
-; SKX32-NEXT:    retl
-;
-; KNL32-LABEL: expand4:
-; KNL32:       # %bb.0:
-; KNL32-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
-; KNL32-NEXT:    vperm2f128 {{.*#+}} ymm1 = zero,zero,ymm0[0,1]
-; KNL32-NEXT:    vmovaps %xmm0, %xmm0
-; KNL32-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
-; KNL32-NEXT:    retl
+; SKX-LABEL: expand4:
+; SKX:       # %bb.0:
+; SKX-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; SKX-NEXT:    movb $9, %al
+; SKX-NEXT:    kmovd %eax, %k1
+; SKX-NEXT:    vpexpandq %ymm0, %ymm0 {%k1} {z}
+; SKX-NEXT:    ret{{[l|q]}}
+;
+; KNL-LABEL: expand4:
+; KNL:       # %bb.0:
+; KNL-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; KNL-NEXT:    vperm2f128 {{.*#+}} ymm1 = zero,zero,ymm0[0,1]
+; KNL-NEXT:    vmovaps %xmm0, %xmm0
+; KNL-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
+; KNL-NEXT:    ret{{[l|q]}}
    %res = shufflevector <2 x i64> zeroinitializer, <2 x i64> %a, <4 x i32> <i32 2, i32 0, i32 0, i32 3>
    ret <4 x i64> %res
 }
 
 ;Negative test for 128-> 256
 define <8 x float> @expand5(<4 x float> %a ) {
-; SKX64-LABEL: expand5:
-; SKX64:       # %bb.0:
-; SKX64-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
-; SKX64-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; SKX64-NEXT:    vmovaps {{.*#+}} ymm2 = [8,0,10,0,12,0,14,0]
-; SKX64-NEXT:    vpermt2ps %ymm1, %ymm2, %ymm0
-; SKX64-NEXT:    retq
-;
-; KNL64-LABEL: expand5:
-; KNL64:       # %bb.0:
-; KNL64-NEXT:    vbroadcastss %xmm0, %ymm0
-; KNL64-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; KNL64-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4],ymm0[5],ymm1[6],ymm0[7]
-; KNL64-NEXT:    retq
-;
-; SKX32-LABEL: expand5:
-; SKX32:       # %bb.0:
-; SKX32-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
-; SKX32-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; SKX32-NEXT:    vmovaps {{.*#+}} ymm2 = [8,0,10,0,12,0,14,0]
-; SKX32-NEXT:    vpermt2ps %ymm1, %ymm2, %ymm0
-; SKX32-NEXT:    retl
-;
-; KNL32-LABEL: expand5:
-; KNL32:       # %bb.0:
-; KNL32-NEXT:    vbroadcastss %xmm0, %ymm0
-; KNL32-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; KNL32-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4],ymm0[5],ymm1[6],ymm0[7]
-; KNL32-NEXT:    retl
+; SKX-LABEL: expand5:
+; SKX:       # %bb.0:
+; SKX-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; SKX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; SKX-NEXT:    vmovaps {{.*#+}} ymm2 = [8,0,10,0,12,0,14,0]
+; SKX-NEXT:    vpermt2ps %ymm1, %ymm2, %ymm0
+; SKX-NEXT:    ret{{[l|q]}}
+;
+; KNL-LABEL: expand5:
+; KNL:       # %bb.0:
+; KNL-NEXT:    vbroadcastss %xmm0, %ymm0
+; KNL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; KNL-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4],ymm0[5],ymm1[6],ymm0[7]
+; KNL-NEXT:    ret{{[l|q]}}
    %res = shufflevector <4 x float> zeroinitializer, <4 x float> %a, <8 x i32> <i32 0, i32 4, i32 1, i32 4, i32 2, i32 4, i32 3, i32 4>
    ret <8 x float> %res
 }
 
 ;expand 256 -> 512 include <8 x float> <16 x float>
 define <8 x float> @expand6(<4 x float> %a ) {
-; SKX64-LABEL: expand6:
-; SKX64:       # %bb.0:
-; SKX64-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; SKX64-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; SKX64-NEXT:    retq
-;
-; KNL64-LABEL: expand6:
-; KNL64:       # %bb.0:
-; KNL64-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; KNL64-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; KNL64-NEXT:    retq
-;
-; SKX32-LABEL: expand6:
-; SKX32:       # %bb.0:
-; SKX32-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; SKX32-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; SKX32-NEXT:    retl
-;
-; KNL32-LABEL: expand6:
-; KNL32:       # %bb.0:
-; KNL32-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; KNL32-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; KNL32-NEXT:    retl
+; CHECK-LABEL: expand6:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
    %res = shufflevector <4 x float> zeroinitializer, <4 x float> %a, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
    ret <8 x float> %res
 }
 
 define <16 x float> @expand7(<8 x float> %a) {
-; SKX64-LABEL: expand7:
-; SKX64:       # %bb.0:
-; SKX64-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; SKX64-NEXT:    movw $1285, %ax # imm = 0x505
-; SKX64-NEXT:    kmovd %eax, %k1
-; SKX64-NEXT:    vexpandps %zmm0, %zmm0 {%k1} {z}
-; SKX64-NEXT:    retq
-;
-; KNL64-LABEL: expand7:
-; KNL64:       # %bb.0:
-; KNL64-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; KNL64-NEXT:    movw $1285, %ax # imm = 0x505
-; KNL64-NEXT:    kmovw %eax, %k1
-; KNL64-NEXT:    vexpandps %zmm0, %zmm0 {%k1} {z}
-; KNL64-NEXT:    retq
-;
-; SKX32-LABEL: expand7:
-; SKX32:       # %bb.0:
-; SKX32-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; SKX32-NEXT:    movw $1285, %ax # imm = 0x505
-; SKX32-NEXT:    kmovd %eax, %k1
-; SKX32-NEXT:    vexpandps %zmm0, %zmm0 {%k1} {z}
-; SKX32-NEXT:    retl
-;
-; KNL32-LABEL: expand7:
-; KNL32:       # %bb.0:
-; KNL32-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; KNL32-NEXT:    movw $1285, %ax # imm = 0x505
-; KNL32-NEXT:    kmovw %eax, %k1
-; KNL32-NEXT:    vexpandps %zmm0, %zmm0 {%k1} {z}
-; KNL32-NEXT:    retl
+; SKX-LABEL: expand7:
+; SKX:       # %bb.0:
+; SKX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; SKX-NEXT:    movw $1285, %ax # imm = 0x505
+; SKX-NEXT:    kmovd %eax, %k1
+; SKX-NEXT:    vexpandps %zmm0, %zmm0 {%k1} {z}
+; SKX-NEXT:    ret{{[l|q]}}
+;
+; KNL-LABEL: expand7:
+; KNL:       # %bb.0:
+; KNL-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; KNL-NEXT:    movw $1285, %ax # imm = 0x505
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    vexpandps %zmm0, %zmm0 {%k1} {z}
+; KNL-NEXT:    ret{{[l|q]}}
    %res = shufflevector <8 x float> %a, <8 x float> zeroinitializer, <16 x i32> <i32 0, i32 8, i32 1, i32 8, i32 8, i32 8, i32 8, i32 8, i32 2, i32 8, i32 3, i32 8, i32 8, i32 8, i32 8, i32 8>
    ret <16 x float> %res
 }
 
 define <16 x float> @expand8(<8 x float> %a ) {
-; SKX64-LABEL: expand8:
-; SKX64:       # %bb.0:
-; SKX64-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; SKX64-NEXT:    movw $-21846, %ax # imm = 0xAAAA
-; SKX64-NEXT:    kmovd %eax, %k1
-; SKX64-NEXT:    vexpandps %zmm0, %zmm0 {%k1} {z}
-; SKX64-NEXT:    retq
-;
-; KNL64-LABEL: expand8:
-; KNL64:       # %bb.0:
-; KNL64-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; KNL64-NEXT:    movw $-21846, %ax # imm = 0xAAAA
-; KNL64-NEXT:    kmovw %eax, %k1
-; KNL64-NEXT:    vexpandps %zmm0, %zmm0 {%k1} {z}
-; KNL64-NEXT:    retq
-;
-; SKX32-LABEL: expand8:
-; SKX32:       # %bb.0:
-; SKX32-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; SKX32-NEXT:    movw $-21846, %ax # imm = 0xAAAA
-; SKX32-NEXT:    kmovd %eax, %k1
-; SKX32-NEXT:    vexpandps %zmm0, %zmm0 {%k1} {z}
-; SKX32-NEXT:    retl
-;
-; KNL32-LABEL: expand8:
-; KNL32:       # %bb.0:
-; KNL32-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; KNL32-NEXT:    movw $-21846, %ax # imm = 0xAAAA
-; KNL32-NEXT:    kmovw %eax, %k1
-; KNL32-NEXT:    vexpandps %zmm0, %zmm0 {%k1} {z}
-; KNL32-NEXT:    retl
+; SKX-LABEL: expand8:
+; SKX:       # %bb.0:
+; SKX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; SKX-NEXT:    movw $-21846, %ax # imm = 0xAAAA
+; SKX-NEXT:    kmovd %eax, %k1
+; SKX-NEXT:    vexpandps %zmm0, %zmm0 {%k1} {z}
+; SKX-NEXT:    ret{{[l|q]}}
+;
+; KNL-LABEL: expand8:
+; KNL:       # %bb.0:
+; KNL-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; KNL-NEXT:    movw $-21846, %ax # imm = 0xAAAA
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    vexpandps %zmm0, %zmm0 {%k1} {z}
+; KNL-NEXT:    ret{{[l|q]}}
    %res = shufflevector <8 x float> zeroinitializer, <8 x float> %a, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
    ret <16 x float> %res
 }
 
 ;expand 256 -> 512 include <4 x double> <8 x double>
 define <8 x double> @expand9(<4 x double> %a) {
-; SKX64-LABEL: expand9:
-; SKX64:       # %bb.0:
-; SKX64-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; SKX64-NEXT:    movb $-127, %al
-; SKX64-NEXT:    kmovd %eax, %k1
-; SKX64-NEXT:    vexpandpd %zmm0, %zmm0 {%k1} {z}
-; SKX64-NEXT:    retq
-;
-; KNL64-LABEL: expand9:
-; KNL64:       # %bb.0:
-; KNL64-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; KNL64-NEXT:    movb $-127, %al
-; KNL64-NEXT:    kmovw %eax, %k1
-; KNL64-NEXT:    vexpandpd %zmm0, %zmm0 {%k1} {z}
-; KNL64-NEXT:    retq
-;
-; SKX32-LABEL: expand9:
-; SKX32:       # %bb.0:
-; SKX32-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; SKX32-NEXT:    movb $-127, %al
-; SKX32-NEXT:    kmovd %eax, %k1
-; SKX32-NEXT:    vexpandpd %zmm0, %zmm0 {%k1} {z}
-; SKX32-NEXT:    retl
-;
-; KNL32-LABEL: expand9:
-; KNL32:       # %bb.0:
-; KNL32-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; KNL32-NEXT:    movb $-127, %al
-; KNL32-NEXT:    kmovw %eax, %k1
-; KNL32-NEXT:    vexpandpd %zmm0, %zmm0 {%k1} {z}
-; KNL32-NEXT:    retl
+; SKX-LABEL: expand9:
+; SKX:       # %bb.0:
+; SKX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; SKX-NEXT:    movb $-127, %al
+; SKX-NEXT:    kmovd %eax, %k1
+; SKX-NEXT:    vexpandpd %zmm0, %zmm0 {%k1} {z}
+; SKX-NEXT:    ret{{[l|q]}}
+;
+; KNL-LABEL: expand9:
+; KNL:       # %bb.0:
+; KNL-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; KNL-NEXT:    movb $-127, %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    vexpandpd %zmm0, %zmm0 {%k1} {z}
+; KNL-NEXT:    ret{{[l|q]}}
    %res = shufflevector <4 x double> %a, <4 x double> zeroinitializer, <8 x i32> <i32 0, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 1>
    ret <8 x double> %res
 }
 
 define <16 x i32> @expand10(<8 x i32> %a ) {
-; SKX64-LABEL: expand10:
-; SKX64:       # %bb.0:
-; SKX64-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; SKX64-NEXT:    movw $-21846, %ax # imm = 0xAAAA
-; SKX64-NEXT:    kmovd %eax, %k1
-; SKX64-NEXT:    vpexpandd %zmm0, %zmm0 {%k1} {z}
-; SKX64-NEXT:    retq
-;
-; KNL64-LABEL: expand10:
-; KNL64:       # %bb.0:
-; KNL64-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; KNL64-NEXT:    movw $-21846, %ax # imm = 0xAAAA
-; KNL64-NEXT:    kmovw %eax, %k1
-; KNL64-NEXT:    vpexpandd %zmm0, %zmm0 {%k1} {z}
-; KNL64-NEXT:    retq
-;
-; SKX32-LABEL: expand10:
-; SKX32:       # %bb.0:
-; SKX32-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; SKX32-NEXT:    movw $-21846, %ax # imm = 0xAAAA
-; SKX32-NEXT:    kmovd %eax, %k1
-; SKX32-NEXT:    vpexpandd %zmm0, %zmm0 {%k1} {z}
-; SKX32-NEXT:    retl
-;
-; KNL32-LABEL: expand10:
-; KNL32:       # %bb.0:
-; KNL32-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; KNL32-NEXT:    movw $-21846, %ax # imm = 0xAAAA
-; KNL32-NEXT:    kmovw %eax, %k1
-; KNL32-NEXT:    vpexpandd %zmm0, %zmm0 {%k1} {z}
-; KNL32-NEXT:    retl
+; SKX-LABEL: expand10:
+; SKX:       # %bb.0:
+; SKX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; SKX-NEXT:    movw $-21846, %ax # imm = 0xAAAA
+; SKX-NEXT:    kmovd %eax, %k1
+; SKX-NEXT:    vpexpandd %zmm0, %zmm0 {%k1} {z}
+; SKX-NEXT:    ret{{[l|q]}}
+;
+; KNL-LABEL: expand10:
+; KNL:       # %bb.0:
+; KNL-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; KNL-NEXT:    movw $-21846, %ax # imm = 0xAAAA
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    vpexpandd %zmm0, %zmm0 {%k1} {z}
+; KNL-NEXT:    ret{{[l|q]}}
    %res = shufflevector <8 x i32> zeroinitializer, <8 x i32> %a, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
    ret <16 x i32> %res
 }
 
 define <8 x i64> @expand11(<4 x i64> %a) {
-; SKX64-LABEL: expand11:
-; SKX64:       # %bb.0:
-; SKX64-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; SKX64-NEXT:    movb $-127, %al
-; SKX64-NEXT:    kmovd %eax, %k1
-; SKX64-NEXT:    vpexpandq %zmm0, %zmm0 {%k1} {z}
-; SKX64-NEXT:    retq
-;
-; KNL64-LABEL: expand11:
-; KNL64:       # %bb.0:
-; KNL64-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; KNL64-NEXT:    movb $-127, %al
-; KNL64-NEXT:    kmovw %eax, %k1
-; KNL64-NEXT:    vpexpandq %zmm0, %zmm0 {%k1} {z}
-; KNL64-NEXT:    retq
-;
-; SKX32-LABEL: expand11:
-; SKX32:       # %bb.0:
-; SKX32-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; SKX32-NEXT:    movb $-127, %al
-; SKX32-NEXT:    kmovd %eax, %k1
-; SKX32-NEXT:    vpexpandq %zmm0, %zmm0 {%k1} {z}
-; SKX32-NEXT:    retl
-;
-; KNL32-LABEL: expand11:
-; KNL32:       # %bb.0:
-; KNL32-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; KNL32-NEXT:    movb $-127, %al
-; KNL32-NEXT:    kmovw %eax, %k1
-; KNL32-NEXT:    vpexpandq %zmm0, %zmm0 {%k1} {z}
-; KNL32-NEXT:    retl
+; SKX-LABEL: expand11:
+; SKX:       # %bb.0:
+; SKX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; SKX-NEXT:    movb $-127, %al
+; SKX-NEXT:    kmovd %eax, %k1
+; SKX-NEXT:    vpexpandq %zmm0, %zmm0 {%k1} {z}
+; SKX-NEXT:    ret{{[l|q]}}
+;
+; KNL-LABEL: expand11:
+; KNL:       # %bb.0:
+; KNL-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; KNL-NEXT:    movb $-127, %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    vpexpandq %zmm0, %zmm0 {%k1} {z}
+; KNL-NEXT:    ret{{[l|q]}}
    %res = shufflevector <4 x i64> %a, <4 x i64> zeroinitializer, <8 x i32> <i32 0, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 1>
    ret <8 x i64> %res
 }
 
 ;Negative test for 256-> 512
 define <16 x float> @expand12(<8 x float> %a) {
-; SKX64-LABEL: expand12:
-; SKX64:       # %bb.0:
-; SKX64-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; SKX64-NEXT:    vmovaps {{.*#+}} zmm2 = [0,16,2,16,4,16,6,16,0,16,1,16,2,16,3,16]
-; SKX64-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; SKX64-NEXT:    vpermt2ps %zmm0, %zmm2, %zmm1
-; SKX64-NEXT:    vmovaps %zmm1, %zmm0
-; SKX64-NEXT:    retq
-;
-; KNL64-LABEL: expand12:
-; KNL64:       # %bb.0:
-; KNL64-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; KNL64-NEXT:    vmovaps {{.*#+}} zmm2 = [0,16,2,16,4,16,6,16,0,16,1,16,2,16,3,16]
-; KNL64-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; KNL64-NEXT:    vpermt2ps %zmm0, %zmm2, %zmm1
-; KNL64-NEXT:    vmovaps %zmm1, %zmm0
-; KNL64-NEXT:    retq
-;
-; SKX32-LABEL: expand12:
-; SKX32:       # %bb.0:
-; SKX32-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; SKX32-NEXT:    vmovaps {{.*#+}} zmm2 = [0,16,2,16,4,16,6,16,0,16,1,16,2,16,3,16]
-; SKX32-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; SKX32-NEXT:    vpermt2ps %zmm0, %zmm2, %zmm1
-; SKX32-NEXT:    vmovaps %zmm1, %zmm0
-; SKX32-NEXT:    retl
-;
-; KNL32-LABEL: expand12:
-; KNL32:       # %bb.0:
-; KNL32-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; KNL32-NEXT:    vmovaps {{.*#+}} zmm2 = [0,16,2,16,4,16,6,16,0,16,1,16,2,16,3,16]
-; KNL32-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; KNL32-NEXT:    vpermt2ps %zmm0, %zmm2, %zmm1
-; KNL32-NEXT:    vmovaps %zmm1, %zmm0
-; KNL32-NEXT:    retl
+; CHECK-LABEL: expand12:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; CHECK-NEXT:    vmovaps {{.*#+}} zmm2 = [0,16,2,16,4,16,6,16,0,16,1,16,2,16,3,16]
+; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vpermt2ps %zmm0, %zmm2, %zmm1
+; CHECK-NEXT:    vmovaps %zmm1, %zmm0
+; CHECK-NEXT:    ret{{[l|q]}}
    %res = shufflevector <8 x float> zeroinitializer, <8 x float> %a, <16 x i32> <i32 0, i32 8, i32 1, i32 8, i32 2, i32 8, i32 3, i32 8,i32 0, i32 8, i32 1, i32 8, i32 2, i32 8, i32 3, i32 8>
    ret <16 x float> %res
 }
 
 define <16 x float> @expand13(<8 x float> %a ) {
-; SKX64-LABEL: expand13:
-; SKX64:       # %bb.0:
-; SKX64-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; SKX64-NEXT:    vinsertf64x4 $1, %ymm0, %zmm1, %zmm0
-; SKX64-NEXT:    retq
-;
-; KNL64-LABEL: expand13:
-; KNL64:       # %bb.0:
-; KNL64-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; KNL64-NEXT:    vinsertf64x4 $1, %ymm0, %zmm1, %zmm0
-; KNL64-NEXT:    retq
-;
-; SKX32-LABEL: expand13:
-; SKX32:       # %bb.0:
-; SKX32-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; SKX32-NEXT:    vinsertf64x4 $1, %ymm0, %zmm1, %zmm0
-; SKX32-NEXT:    retl
-;
-; KNL32-LABEL: expand13:
-; KNL32:       # %bb.0:
-; KNL32-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; KNL32-NEXT:    vinsertf64x4 $1, %ymm0, %zmm1, %zmm0
-; KNL32-NEXT:    retl
+; CHECK-LABEL: expand13:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vinsertf64x4 $1, %ymm0, %zmm1, %zmm0
+; CHECK-NEXT:    ret{{[l|q]}}
    %res = shufflevector <8 x float> zeroinitializer, <8 x float> %a, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
    ret <16 x float> %res
 }
@@ -503,37 +258,21 @@ define <16 x float> @expand13(<8 x float> %a ) {
 ; The function checks for a case where the vector is mixed values vector ,and the mask points on zero elements from this vector.
 
 define <8 x float> @expand14(<4 x float> %a) {
-; SKX64-LABEL: expand14:
-; SKX64:       # %bb.0:
-; SKX64-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
-; SKX64-NEXT:    movb $20, %al
-; SKX64-NEXT:    kmovd %eax, %k1
-; SKX64-NEXT:    vexpandps %ymm0, %ymm0 {%k1} {z}
-; SKX64-NEXT:    retq
-;
-; KNL64-LABEL: expand14:
-; KNL64:       # %bb.0:
-; KNL64-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,1,1,3]
-; KNL64-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,0,1,3]
-; KNL64-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; KNL64-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3],ymm0[4],ymm1[5,6,7]
-; KNL64-NEXT:    retq
-;
-; SKX32-LABEL: expand14:
-; SKX32:       # %bb.0:
-; SKX32-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
-; SKX32-NEXT:    movb $20, %al
-; SKX32-NEXT:    kmovd %eax, %k1
-; SKX32-NEXT:    vexpandps %ymm0, %ymm0 {%k1} {z}
-; SKX32-NEXT:    retl
-;
-; KNL32-LABEL: expand14:
-; KNL32:       # %bb.0:
-; KNL32-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,1,1,3]
-; KNL32-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,0,1,3]
-; KNL32-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; KNL32-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3],ymm0[4],ymm1[5,6,7]
-; KNL32-NEXT:    retl
+; SKX-LABEL: expand14:
+; SKX:       # %bb.0:
+; SKX-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; SKX-NEXT:    movb $20, %al
+; SKX-NEXT:    kmovd %eax, %k1
+; SKX-NEXT:    vexpandps %ymm0, %ymm0 {%k1} {z}
+; SKX-NEXT:    ret{{[l|q]}}
+;
+; KNL-LABEL: expand14:
+; KNL:       # %bb.0:
+; KNL-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,1,1,3]
+; KNL-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,0,1,3]
+; KNL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; KNL-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3],ymm0[4],ymm1[5,6,7]
+; KNL-NEXT:    ret{{[l|q]}}
    %addV = fadd <4 x float> <float 0.0,float 1.0,float 2.0,float 0.0> , <float 0.0,float 1.0,float 2.0,float 0.0>
    %res = shufflevector <4 x float> %addV, <4 x float> %a, <8 x i32> <i32 3, i32 3, i32 4, i32 0, i32 5, i32 0, i32 0, i32 0>
    ret <8 x float> %res
@@ -541,35 +280,20 @@ define <8 x float> @expand14(<4 x float> %a) {
 
 ;Negative test.
 define <8 x float> @expand15(<4 x float> %a) {
-; SKX64-LABEL: expand15:
-; SKX64:       # %bb.0:
-; SKX64-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
-; SKX64-NEXT:    vmovaps {{.*#+}} ymm1 = <0,1,0,1,1,3,u,u>
-; SKX64-NEXT:    vpermps %ymm0, %ymm1, %ymm0
-; SKX64-NEXT:    vblendps {{.*#+}} ymm0 = mem[0,1],ymm0[2],mem[3],ymm0[4],mem[5,6,7]
-; SKX64-NEXT:    retq
-;
-; KNL64-LABEL: expand15:
-; KNL64:       # %bb.0:
-; KNL64-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,1,1,3]
-; KNL64-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,0,1,3]
-; KNL64-NEXT:    vblendps {{.*#+}} ymm0 = mem[0,1],ymm0[2],mem[3],ymm0[4],mem[5,6,7]
-; KNL64-NEXT:    retq
-;
-; SKX32-LABEL: expand15:
-; SKX32:       # %bb.0:
-; SKX32-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
-; SKX32-NEXT:    vmovaps {{.*#+}} ymm1 = <0,1,0,1,1,3,u,u>
-; SKX32-NEXT:    vpermps %ymm0, %ymm1, %ymm0
-; SKX32-NEXT:    vblendps {{.*#+}} ymm0 = mem[0,1],ymm0[2],mem[3],ymm0[4],mem[5,6,7]
-; SKX32-NEXT:    retl
-;
-; KNL32-LABEL: expand15:
-; KNL32:       # %bb.0:
-; KNL32-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,1,1,3]
-; KNL32-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,0,1,3]
-; KNL32-NEXT:    vblendps {{.*#+}} ymm0 = mem[0,1],ymm0[2],mem[3],ymm0[4],mem[5,6,7]
-; KNL32-NEXT:    retl
+; SKX-LABEL: expand15:
+; SKX:       # %bb.0:
+; SKX-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; SKX-NEXT:    vmovaps {{.*#+}} ymm1 = <0,1,0,1,1,3,u,u>
+; SKX-NEXT:    vpermps %ymm0, %ymm1, %ymm0
+; SKX-NEXT:    vblendps {{.*#+}} ymm0 = mem[0,1],ymm0[2],mem[3],ymm0[4],mem[5,6,7]
+; SKX-NEXT:    ret{{[l|q]}}
+;
+; KNL-LABEL: expand15:
+; KNL:       # %bb.0:
+; KNL-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,1,1,3]
+; KNL-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,0,1,3]
+; KNL-NEXT:    vblendps {{.*#+}} ymm0 = mem[0,1],ymm0[2],mem[3],ymm0[4],mem[5,6,7]
+; KNL-NEXT:    ret{{[l|q]}}
    %addV = fadd <4 x float> <float 0.0,float 1.0,float 2.0,float 0.0> , <float 0.0,float 1.0,float 2.0,float 0.0>
    %res = shufflevector <4 x float> %addV, <4 x float> %a, <8 x i32> <i32 0, i32 1, i32 4, i32 0, i32 5, i32 0, i32 0, i32 0>
    ret <8 x float> %res
@@ -586,12 +310,15 @@ define <64 x i8> @test_mm512_mask_blend_epi8(<64 x i8> %A, <64 x i8> %W){
 ; SKX64-NEXT:    vpblendmb %zmm0, %zmm1, %zmm0 {%k1}
 ; SKX64-NEXT:    retq
 ;
-; KNL64-LABEL: test_mm512_mask_blend_epi8:
-; KNL64:       # %bb.0: # %entry
-; KNL64-NEXT:    vpbroadcastw {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; KNL64-NEXT:    vpblendvb %ymm4, %ymm2, %ymm0, %ymm0
-; KNL64-NEXT:    vpblendvb %ymm4, %ymm3, %ymm1, %ymm1
-; KNL64-NEXT:    retq
+; KNL-LABEL: test_mm512_mask_blend_epi8:
+; KNL:       # %bb.0: # %entry
+; KNL-NEXT:    vpbroadcastw {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; KNL-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
+; KNL-NEXT:    vextracti64x4 $1, %zmm0, %ymm4
+; KNL-NEXT:    vpblendvb %ymm2, %ymm3, %ymm4, %ymm3
+; KNL-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
+; KNL-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm0
+; KNL-NEXT:    ret{{[l|q]}}
 ;
 ; SKX32-LABEL: test_mm512_mask_blend_epi8:
 ; SKX32:       # %bb.0: # %entry
@@ -600,196 +327,103 @@ define <64 x i8> @test_mm512_mask_blend_epi8(<64 x i8> %A, <64 x i8> %W){
 ; SKX32-NEXT:    kunpckdq %k0, %k0, %k1
 ; SKX32-NEXT:    vpblendmb %zmm0, %zmm1, %zmm0 {%k1}
 ; SKX32-NEXT:    retl
-;
-; KNL32-LABEL: test_mm512_mask_blend_epi8:
-; KNL32:       # %bb.0: # %entry
-; KNL32-NEXT:    pushl %ebp
-; KNL32-NEXT:    .cfi_def_cfa_offset 8
-; KNL32-NEXT:    .cfi_offset %ebp, -8
-; KNL32-NEXT:    movl %esp, %ebp
-; KNL32-NEXT:    .cfi_def_cfa_register %ebp
-; KNL32-NEXT:    andl $-32, %esp
-; KNL32-NEXT:    subl $32, %esp
-; KNL32-NEXT:    vpbroadcastw {{.*#+}} ymm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; KNL32-NEXT:    vpblendvb %ymm3, %ymm2, %ymm0, %ymm0
-; KNL32-NEXT:    vpblendvb %ymm3, 8(%ebp), %ymm1, %ymm1
-; KNL32-NEXT:    movl %ebp, %esp
-; KNL32-NEXT:    popl %ebp
-; KNL32-NEXT:    .cfi_def_cfa %esp, 4
-; KNL32-NEXT:    retl
 entry:
   %0 = shufflevector <64 x i8> %A, <64 x i8> %W, <64 x i32>  <i32 64, i32 1, i32 66, i32 3, i32 68, i32 5, i32 70, i32 7, i32 72, i32 9, i32 74, i32 11, i32 76, i32 13, i32 78, i32 15, i32 80, i32 17, i32 82, i32 19, i32 84, i32 21, i32 86, i32 23, i32 88, i32 25, i32 90, i32 27, i32 92, i32 29, i32 94, i32 31, i32 96, i32 33, i32 98, i32 35, i32 100, i32 37, i32 102, i32 39, i32 104, i32 41, i32 106, i32 43, i32 108, i32 45, i32 110, i32 47, i32 112, i32 49, i32 114, i32 51, i32 116, i32 53, i32 118, i32 55, i32 120, i32 57, i32 122, i32 59, i32 124, i32 61, i32 126, i32 63>
   ret <64 x i8> %0
 }
 
 define <32 x i16> @test_mm512_mask_blend_epi16(<32 x i16> %A, <32 x i16> %W){
-; SKX64-LABEL: test_mm512_mask_blend_epi16:
-; SKX64:       # %bb.0: # %entry
-; SKX64-NEXT:    movl $-1431655766, %eax # imm = 0xAAAAAAAA
-; SKX64-NEXT:    kmovd %eax, %k1
-; SKX64-NEXT:    vpblendmw %zmm0, %zmm1, %zmm0 {%k1}
-; SKX64-NEXT:    retq
-;
-; KNL64-LABEL: test_mm512_mask_blend_epi16:
-; KNL64:       # %bb.0: # %entry
-; KNL64-NEXT:    vpblendw {{.*#+}} ymm0 = ymm2[0],ymm0[1],ymm2[2],ymm0[3],ymm2[4],ymm0[5],ymm2[6],ymm0[7],ymm2[8],ymm0[9],ymm2[10],ymm0[11],ymm2[12],ymm0[13],ymm2[14],ymm0[15]
-; KNL64-NEXT:    vpblendw {{.*#+}} ymm1 = ymm3[0],ymm1[1],ymm3[2],ymm1[3],ymm3[4],ymm1[5],ymm3[6],ymm1[7],ymm3[8],ymm1[9],ymm3[10],ymm1[11],ymm3[12],ymm1[13],ymm3[14],ymm1[15]
-; KNL64-NEXT:    retq
-;
-; SKX32-LABEL: test_mm512_mask_blend_epi16:
-; SKX32:       # %bb.0: # %entry
-; SKX32-NEXT:    movl $-1431655766, %eax # imm = 0xAAAAAAAA
-; SKX32-NEXT:    kmovd %eax, %k1
-; SKX32-NEXT:    vpblendmw %zmm0, %zmm1, %zmm0 {%k1}
-; SKX32-NEXT:    retl
-;
-; KNL32-LABEL: test_mm512_mask_blend_epi16:
-; KNL32:       # %bb.0: # %entry
-; KNL32-NEXT:    pushl %ebp
-; KNL32-NEXT:    .cfi_def_cfa_offset 8
-; KNL32-NEXT:    .cfi_offset %ebp, -8
-; KNL32-NEXT:    movl %esp, %ebp
-; KNL32-NEXT:    .cfi_def_cfa_register %ebp
-; KNL32-NEXT:    andl $-32, %esp
-; KNL32-NEXT:    subl $32, %esp
-; KNL32-NEXT:    vpblendw {{.*#+}} ymm0 = ymm2[0],ymm0[1],ymm2[2],ymm0[3],ymm2[4],ymm0[5],ymm2[6],ymm0[7],ymm2[8],ymm0[9],ymm2[10],ymm0[11],ymm2[12],ymm0[13],ymm2[14],ymm0[15]
-; KNL32-NEXT:    vpblendw {{.*#+}} ymm1 = mem[0],ymm1[1],mem[2],ymm1[3],mem[4],ymm1[5],mem[6],ymm1[7],mem[8],ymm1[9],mem[10],ymm1[11],mem[12],ymm1[13],mem[14],ymm1[15]
-; KNL32-NEXT:    movl %ebp, %esp
-; KNL32-NEXT:    popl %ebp
-; KNL32-NEXT:    .cfi_def_cfa %esp, 4
-; KNL32-NEXT:    retl
+; SKX-LABEL: test_mm512_mask_blend_epi16:
+; SKX:       # %bb.0: # %entry
+; SKX-NEXT:    movl $-1431655766, %eax # imm = 0xAAAAAAAA
+; SKX-NEXT:    kmovd %eax, %k1
+; SKX-NEXT:    vpblendmw %zmm0, %zmm1, %zmm0 {%k1}
+; SKX-NEXT:    ret{{[l|q]}}
+;
+; KNL-LABEL: test_mm512_mask_blend_epi16:
+; KNL:       # %bb.0: # %entry
+; KNL-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; KNL-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
+; KNL-NEXT:    vpblendw {{.*#+}} ymm2 = ymm3[0],ymm2[1],ymm3[2],ymm2[3],ymm3[4],ymm2[5],ymm3[6],ymm2[7],ymm3[8],ymm2[9],ymm3[10],ymm2[11],ymm3[12],ymm2[13],ymm3[14],ymm2[15]
+; KNL-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4],ymm0[5],ymm1[6],ymm0[7],ymm1[8],ymm0[9],ymm1[10],ymm0[11],ymm1[12],ymm0[13],ymm1[14],ymm0[15]
+; KNL-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; KNL-NEXT:    ret{{[l|q]}}
 entry:
   %0 = shufflevector <32 x i16> %A, <32 x i16> %W, <32 x i32>  <i32 32, i32 1, i32 34, i32 3, i32 36, i32 5, i32 38, i32 7, i32 40, i32 9, i32 42, i32 11, i32 44, i32 13, i32 46, i32 15, i32 48, i32 17, i32 50, i32 19, i32 52, i32 21, i32 54, i32 23, i32 56, i32 25, i32 58, i32 27, i32 60, i32 29, i32 62, i32 31>
   ret <32 x i16> %0
 }
 
 define <16 x i32> @test_mm512_mask_blend_epi32(<16 x i32> %A, <16 x i32> %W){
-; SKX64-LABEL: test_mm512_mask_blend_epi32:
-; SKX64:       # %bb.0: # %entry
-; SKX64-NEXT:    movw $-21846, %ax # imm = 0xAAAA
-; SKX64-NEXT:    kmovd %eax, %k1
-; SKX64-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
-; SKX64-NEXT:    retq
-;
-; KNL64-LABEL: test_mm512_mask_blend_epi32:
-; KNL64:       # %bb.0: # %entry
-; KNL64-NEXT:    movw $-21846, %ax # imm = 0xAAAA
-; KNL64-NEXT:    kmovw %eax, %k1
-; KNL64-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
-; KNL64-NEXT:    retq
-;
-; SKX32-LABEL: test_mm512_mask_blend_epi32:
-; SKX32:       # %bb.0: # %entry
-; SKX32-NEXT:    movw $-21846, %ax # imm = 0xAAAA
-; SKX32-NEXT:    kmovd %eax, %k1
-; SKX32-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
-; SKX32-NEXT:    retl
-;
-; KNL32-LABEL: test_mm512_mask_blend_epi32:
-; KNL32:       # %bb.0: # %entry
-; KNL32-NEXT:    movw $-21846, %ax # imm = 0xAAAA
-; KNL32-NEXT:    kmovw %eax, %k1
-; KNL32-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
-; KNL32-NEXT:    retl
+; SKX-LABEL: test_mm512_mask_blend_epi32:
+; SKX:       # %bb.0: # %entry
+; SKX-NEXT:    movw $-21846, %ax # imm = 0xAAAA
+; SKX-NEXT:    kmovd %eax, %k1
+; SKX-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
+; SKX-NEXT:    ret{{[l|q]}}
+;
+; KNL-LABEL: test_mm512_mask_blend_epi32:
+; KNL:       # %bb.0: # %entry
+; KNL-NEXT:    movw $-21846, %ax # imm = 0xAAAA
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
+; KNL-NEXT:    ret{{[l|q]}}
 entry:
   %0 = shufflevector <16 x i32> %A, <16 x i32> %W, <16 x i32>  <i32 16, i32 1, i32 18, i32 3, i32 20, i32 5, i32 22, i32 7, i32 24, i32 9, i32 26, i32 11, i32 28, i32 13, i32 30, i32 15>
   ret <16 x i32> %0
 }
 
 define <8 x i64> @test_mm512_mask_blend_epi64(<8 x i64> %A, <8 x i64> %W){
-; SKX64-LABEL: test_mm512_mask_blend_epi64:
-; SKX64:       # %bb.0: # %entry
-; SKX64-NEXT:    movb $-86, %al
-; SKX64-NEXT:    kmovd %eax, %k1
-; SKX64-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
-; SKX64-NEXT:    retq
-;
-; KNL64-LABEL: test_mm512_mask_blend_epi64:
-; KNL64:       # %bb.0: # %entry
-; KNL64-NEXT:    movb $-86, %al
-; KNL64-NEXT:    kmovw %eax, %k1
-; KNL64-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
-; KNL64-NEXT:    retq
-;
-; SKX32-LABEL: test_mm512_mask_blend_epi64:
-; SKX32:       # %bb.0: # %entry
-; SKX32-NEXT:    movb $-86, %al
-; SKX32-NEXT:    kmovd %eax, %k1
-; SKX32-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
-; SKX32-NEXT:    retl
-;
-; KNL32-LABEL: test_mm512_mask_blend_epi64:
-; KNL32:       # %bb.0: # %entry
-; KNL32-NEXT:    movb $-86, %al
-; KNL32-NEXT:    kmovw %eax, %k1
-; KNL32-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
-; KNL32-NEXT:    retl
+; SKX-LABEL: test_mm512_mask_blend_epi64:
+; SKX:       # %bb.0: # %entry
+; SKX-NEXT:    movb $-86, %al
+; SKX-NEXT:    kmovd %eax, %k1
+; SKX-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
+; SKX-NEXT:    ret{{[l|q]}}
+;
+; KNL-LABEL: test_mm512_mask_blend_epi64:
+; KNL:       # %bb.0: # %entry
+; KNL-NEXT:    movb $-86, %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
+; KNL-NEXT:    ret{{[l|q]}}
 entry:
   %0 = shufflevector <8 x i64> %A, <8 x i64> %W, <8 x i32>  <i32 8, i32 1, i32 10, i32 3, i32 12, i32 5, i32 14, i32 7>
   ret <8 x i64> %0
 }
 
 define <16 x float> @test_mm512_mask_blend_ps(<16 x float> %A, <16 x float> %W){
-; SKX64-LABEL: test_mm512_mask_blend_ps:
-; SKX64:       # %bb.0: # %entry
-; SKX64-NEXT:    movw $-21846, %ax # imm = 0xAAAA
-; SKX64-NEXT:    kmovd %eax, %k1
-; SKX64-NEXT:    vblendmps %zmm0, %zmm1, %zmm0 {%k1}
-; SKX64-NEXT:    retq
-;
-; KNL64-LABEL: test_mm512_mask_blend_ps:
-; KNL64:       # %bb.0: # %entry
-; KNL64-NEXT:    movw $-21846, %ax # imm = 0xAAAA
-; KNL64-NEXT:    kmovw %eax, %k1
-; KNL64-NEXT:    vblendmps %zmm0, %zmm1, %zmm0 {%k1}
-; KNL64-NEXT:    retq
-;
-; SKX32-LABEL: test_mm512_mask_blend_ps:
-; SKX32:       # %bb.0: # %entry
-; SKX32-NEXT:    movw $-21846, %ax # imm = 0xAAAA
-; SKX32-NEXT:    kmovd %eax, %k1
-; SKX32-NEXT:    vblendmps %zmm0, %zmm1, %zmm0 {%k1}
-; SKX32-NEXT:    retl
-;
-; KNL32-LABEL: test_mm512_mask_blend_ps:
-; KNL32:       # %bb.0: # %entry
-; KNL32-NEXT:    movw $-21846, %ax # imm = 0xAAAA
-; KNL32-NEXT:    kmovw %eax, %k1
-; KNL32-NEXT:    vblendmps %zmm0, %zmm1, %zmm0 {%k1}
-; KNL32-NEXT:    retl
+; SKX-LABEL: test_mm512_mask_blend_ps:
+; SKX:       # %bb.0: # %entry
+; SKX-NEXT:    movw $-21846, %ax # imm = 0xAAAA
+; SKX-NEXT:    kmovd %eax, %k1
+; SKX-NEXT:    vblendmps %zmm0, %zmm1, %zmm0 {%k1}
+; SKX-NEXT:    ret{{[l|q]}}
+;
+; KNL-LABEL: test_mm512_mask_blend_ps:
+; KNL:       # %bb.0: # %entry
+; KNL-NEXT:    movw $-21846, %ax # imm = 0xAAAA
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    vblendmps %zmm0, %zmm1, %zmm0 {%k1}
+; KNL-NEXT:    ret{{[l|q]}}
 entry:
   %0 = shufflevector <16 x float> %A, <16 x float> %W, <16 x i32>  <i32 16, i32 1, i32 18, i32 3, i32 20, i32 5, i32 22, i32 7, i32 24, i32 9, i32 26, i32 11, i32 28, i32 13, i32 30, i32 15>
   ret <16 x float> %0
 }
 
 define <8 x double> @test_mm512_mask_blend_pd(<8 x double> %A, <8 x double> %W){
-; SKX64-LABEL: test_mm512_mask_blend_pd:
-; SKX64:       # %bb.0: # %entry
-; SKX64-NEXT:    movb $-88, %al
-; SKX64-NEXT:    kmovd %eax, %k1
-; SKX64-NEXT:    vblendmpd %zmm0, %zmm1, %zmm0 {%k1}
-; SKX64-NEXT:    retq
-;
-; KNL64-LABEL: test_mm512_mask_blend_pd:
-; KNL64:       # %bb.0: # %entry
-; KNL64-NEXT:    movb $-88, %al
-; KNL64-NEXT:    kmovw %eax, %k1
-; KNL64-NEXT:    vblendmpd %zmm0, %zmm1, %zmm0 {%k1}
-; KNL64-NEXT:    retq
-;
-; SKX32-LABEL: test_mm512_mask_blend_pd:
-; SKX32:       # %bb.0: # %entry
-; SKX32-NEXT:    movb $-88, %al
-; SKX32-NEXT:    kmovd %eax, %k1
-; SKX32-NEXT:    vblendmpd %zmm0, %zmm1, %zmm0 {%k1}
-; SKX32-NEXT:    retl
-;
-; KNL32-LABEL: test_mm512_mask_blend_pd:
-; KNL32:       # %bb.0: # %entry
-; KNL32-NEXT:    movb $-88, %al
-; KNL32-NEXT:    kmovw %eax, %k1
-; KNL32-NEXT:    vblendmpd %zmm0, %zmm1, %zmm0 {%k1}
-; KNL32-NEXT:    retl
+; SKX-LABEL: test_mm512_mask_blend_pd:
+; SKX:       # %bb.0: # %entry
+; SKX-NEXT:    movb $-88, %al
+; SKX-NEXT:    kmovd %eax, %k1
+; SKX-NEXT:    vblendmpd %zmm0, %zmm1, %zmm0 {%k1}
+; SKX-NEXT:    ret{{[l|q]}}
+;
+; KNL-LABEL: test_mm512_mask_blend_pd:
+; KNL:       # %bb.0: # %entry
+; KNL-NEXT:    movb $-88, %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    vblendmpd %zmm0, %zmm1, %zmm0 {%k1}
+; KNL-NEXT:    ret{{[l|q]}}
 entry:
   %0 = shufflevector <8 x double> %A, <8 x double> %W, <8 x i32>  <i32 8, i32 9, i32 10, i32 3, i32 12, i32 5, i32 14, i32 7>
   ret <8 x double> %0
@@ -797,62 +431,36 @@ entry:
 
 
 define <32 x i8> @test_mm256_mask_blend_epi8(<32 x i8> %A, <32 x i8> %W){
-; SKX64-LABEL: test_mm256_mask_blend_epi8:
-; SKX64:       # %bb.0: # %entry
-; SKX64-NEXT:    movl $-1431655766, %eax # imm = 0xAAAAAAAA
-; SKX64-NEXT:    kmovd %eax, %k1
-; SKX64-NEXT:    vpblendmb %ymm0, %ymm1, %ymm0 {%k1}
-; SKX64-NEXT:    retq
-;
-; KNL64-LABEL: test_mm256_mask_blend_epi8:
-; KNL64:       # %bb.0: # %entry
-; KNL64-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
-; KNL64-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
-; KNL64-NEXT:    retq
-;
-; SKX32-LABEL: test_mm256_mask_blend_epi8:
-; SKX32:       # %bb.0: # %entry
-; SKX32-NEXT:    movl $-1431655766, %eax # imm = 0xAAAAAAAA
-; SKX32-NEXT:    kmovd %eax, %k1
-; SKX32-NEXT:    vpblendmb %ymm0, %ymm1, %ymm0 {%k1}
-; SKX32-NEXT:    retl
-;
-; KNL32-LABEL: test_mm256_mask_blend_epi8:
-; KNL32:       # %bb.0: # %entry
-; KNL32-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
-; KNL32-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
-; KNL32-NEXT:    retl
+; SKX-LABEL: test_mm256_mask_blend_epi8:
+; SKX:       # %bb.0: # %entry
+; SKX-NEXT:    movl $-1431655766, %eax # imm = 0xAAAAAAAA
+; SKX-NEXT:    kmovd %eax, %k1
+; SKX-NEXT:    vpblendmb %ymm0, %ymm1, %ymm0 {%k1}
+; SKX-NEXT:    ret{{[l|q]}}
+;
+; KNL-LABEL: test_mm256_mask_blend_epi8:
+; KNL:       # %bb.0: # %entry
+; KNL-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; KNL-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
+; KNL-NEXT:    ret{{[l|q]}}
 entry:
   %0 = shufflevector <32 x i8> %A, <32 x i8> %W, <32 x i32>  <i32 32, i32 1, i32 34, i32 3, i32 36, i32 5, i32 38, i32 7, i32 40, i32 9, i32 42, i32 11, i32 44, i32 13, i32 46, i32 15, i32 48, i32 17, i32 50, i32 19, i32 52, i32 21, i32 54, i32 23, i32 56, i32 25, i32 58, i32 27, i32 60, i32 29, i32 62, i32 31>
   ret <32 x i8> %0
 }
 
 define <16 x i8> @test_mm_mask_blend_epi8(<16 x i8> %A, <16 x i8> %W){
-; SKX64-LABEL: test_mm_mask_blend_epi8:
-; SKX64:       # %bb.0: # %entry
-; SKX64-NEXT:    movw $-21846, %ax # imm = 0xAAAA
-; SKX64-NEXT:    kmovd %eax, %k1
-; SKX64-NEXT:    vpblendmb %xmm0, %xmm1, %xmm0 {%k1}
-; SKX64-NEXT:    retq
-;
-; KNL64-LABEL: test_mm_mask_blend_epi8:
-; KNL64:       # %bb.0: # %entry
-; KNL64-NEXT:    vmovdqa {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
-; KNL64-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
-; KNL64-NEXT:    retq
-;
-; SKX32-LABEL: test_mm_mask_blend_epi8:
-; SKX32:       # %bb.0: # %entry
-; SKX32-NEXT:    movw $-21846, %ax # imm = 0xAAAA
-; SKX32-NEXT:    kmovd %eax, %k1
-; SKX32-NEXT:    vpblendmb %xmm0, %xmm1, %xmm0 {%k1}
-; SKX32-NEXT:    retl
-;
-; KNL32-LABEL: test_mm_mask_blend_epi8:
-; KNL32:       # %bb.0: # %entry
-; KNL32-NEXT:    vmovdqa {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
-; KNL32-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
-; KNL32-NEXT:    retl
+; SKX-LABEL: test_mm_mask_blend_epi8:
+; SKX:       # %bb.0: # %entry
+; SKX-NEXT:    movw $-21846, %ax # imm = 0xAAAA
+; SKX-NEXT:    kmovd %eax, %k1
+; SKX-NEXT:    vpblendmb %xmm0, %xmm1, %xmm0 {%k1}
+; SKX-NEXT:    ret{{[l|q]}}
+;
+; KNL-LABEL: test_mm_mask_blend_epi8:
+; KNL:       # %bb.0: # %entry
+; KNL-NEXT:    vmovdqa {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; KNL-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
+; KNL-NEXT:    ret{{[l|q]}}
 entry:
   %0 = shufflevector <16 x i8> %A, <16 x i8> %W, <16 x i32>  <i32 16, i32 1, i32 18, i32 3, i32 20, i32 5, i32 22, i32 7, i32 24, i32 9, i32 26, i32 11, i32 28, i32 13, i32 30, i32 15>
   ret <16 x i8> %0
@@ -898,39 +506,22 @@ define <8 x float> @test_masked_permps_v8f32(<8 x float>* %vp, <8 x float> %vec2
 }
 
 define <16 x float> @test_masked_permps_v16f32(<16 x float>* %vp, <16 x float> %vec2) {
-; SKX64-LABEL: test_masked_permps_v16f32:
-; SKX64:       # %bb.0:
-; SKX64-NEXT:    vmovaps (%rdi), %zmm2
-; SKX64-NEXT:    vmovaps {{.*#+}} zmm1 = [15,13,11,19,14,12,22,23,7,6,3,27,7,29,3,31]
-; SKX64-NEXT:    vpermi2ps %zmm0, %zmm2, %zmm1
-; SKX64-NEXT:    vmovaps %zmm1, %zmm0
-; SKX64-NEXT:    retq
-;
-; KNL64-LABEL: test_masked_permps_v16f32:
-; KNL64:       # %bb.0:
-; KNL64-NEXT:    vmovaps (%rdi), %zmm2
-; KNL64-NEXT:    vmovaps {{.*#+}} zmm1 = [15,13,11,19,14,12,22,23,7,6,3,27,7,29,3,31]
-; KNL64-NEXT:    vpermi2ps %zmm0, %zmm2, %zmm1
-; KNL64-NEXT:    vmovaps %zmm1, %zmm0
-; KNL64-NEXT:    retq
-;
-; SKX32-LABEL: test_masked_permps_v16f32:
-; SKX32:       # %bb.0:
-; SKX32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; SKX32-NEXT:    vmovaps (%eax), %zmm2
-; SKX32-NEXT:    vmovaps {{.*#+}} zmm1 = [15,13,11,19,14,12,22,23,7,6,3,27,7,29,3,31]
-; SKX32-NEXT:    vpermi2ps %zmm0, %zmm2, %zmm1
-; SKX32-NEXT:    vmovaps %zmm1, %zmm0
-; SKX32-NEXT:    retl
-;
-; KNL32-LABEL: test_masked_permps_v16f32:
-; KNL32:       # %bb.0:
-; KNL32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; KNL32-NEXT:    vmovaps (%eax), %zmm2
-; KNL32-NEXT:    vmovaps {{.*#+}} zmm1 = [15,13,11,19,14,12,22,23,7,6,3,27,7,29,3,31]
-; KNL32-NEXT:    vpermi2ps %zmm0, %zmm2, %zmm1
-; KNL32-NEXT:    vmovaps %zmm1, %zmm0
-; KNL32-NEXT:    retl
+; X64-LABEL: test_masked_permps_v16f32:
+; X64:       # %bb.0:
+; X64-NEXT:    vmovaps (%rdi), %zmm2
+; X64-NEXT:    vmovaps {{.*#+}} zmm1 = [15,13,11,19,14,12,22,23,7,6,3,27,7,29,3,31]
+; X64-NEXT:    vpermi2ps %zmm0, %zmm2, %zmm1
+; X64-NEXT:    vmovaps %zmm1, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_masked_permps_v16f32:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    vmovaps (%eax), %zmm2
+; X86-NEXT:    vmovaps {{.*#+}} zmm1 = [15,13,11,19,14,12,22,23,7,6,3,27,7,29,3,31]
+; X86-NEXT:    vpermi2ps %zmm0, %zmm2, %zmm1
+; X86-NEXT:    vmovaps %zmm1, %zmm0
+; X86-NEXT:    retl
   %vec = load <16 x float>, <16 x float>* %vp
   %shuf = shufflevector <16 x float> %vec, <16 x float> undef, <16 x i32> <i32 15, i32 13, i32 11, i32 9, i32 14, i32 12, i32 10, i32 8, i32 7, i32 6, i32 3, i32 0, i32 7, i32 6, i32 3, i32 0>
   %res = select <16 x i1> <i1 1, i1 1, i1 1, i1 0, i1 1, i1 1, i1 0, i1 0, i1 1, i1 1, i1 1, i1 0, i1 1, i1 0, i1 1, i1 0>, <16 x float> %shuf, <16 x float> %vec2
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-combining-avx.ll b/llvm/test/CodeGen/X86/vector-shuffle-combining-avx.ll
index 45c2abc03..9a759e35f 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-combining-avx.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-combining-avx.ll
@@ -417,3 +417,60 @@ entry:
   store <8 x float> %1, <8 x float>* undef, align 16
   unreachable
 }
+
+define <4 x i64> @concat_self_v4i64(<2 x i64> %x) {
+; AVX1-LABEL: concat_self_v4i64:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
+; AVX1-NEXT:    vpermilpd {{.*#+}} ymm0 = ymm0[0,0,3,3]
+; AVX1-NEXT:    ret{{[l|q]}}
+;
+; AVX2-LABEL: concat_self_v4i64:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; AVX2-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,0,1,1]
+; AVX2-NEXT:    ret{{[l|q]}}
+;
+; AVX512-LABEL: concat_self_v4i64:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; AVX512-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,0,1,1]
+; AVX512-NEXT:    ret{{[l|q]}}
+  %cat = shufflevector <2 x i64> %x, <2 x i64> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+  %s = shufflevector <4 x i64> %cat, <4 x i64> undef, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+  ret <4 x i64> %s
+}
+
+define <8 x i32> @concat_self_v8i32(<4 x i32> %x) {
+; AVX1-LABEL: concat_self_v8i32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[3,2,1,0]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[0,2,1,3]
+; AVX1-NEXT:    vpaddd %xmm0, %xmm2, %xmm2
+; AVX1-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    ret{{[l|q]}}
+;
+; AVX2-LABEL: concat_self_v8i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm1
+; AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [3,2,1,0,0,2,1,3]
+; AVX2-NEXT:    vpermd %ymm0, %ymm2, %ymm0
+; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    ret{{[l|q]}}
+;
+; AVX512-LABEL: concat_self_v8i32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; AVX512-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm1
+; AVX512-NEXT:    vmovdqa {{.*#+}} ymm2 = [3,2,1,0,0,2,1,3]
+; AVX512-NEXT:    vpermd %ymm0, %ymm2, %ymm0
+; AVX512-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    ret{{[l|q]}}
+  %cat = shufflevector <4 x i32> %x, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+  %s = shufflevector <8 x i32> %cat, <8 x i32> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 0, i32 2, i32 1, i32 3>
+  %a = add <8 x i32> %s, %cat
+  ret <8 x i32> %a
+}
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-combining-ssse3.ll b/llvm/test/CodeGen/X86/vector-shuffle-combining-ssse3.ll
index b5696e33b..16a174c0b 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-combining-ssse3.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-combining-ssse3.ll
@@ -746,12 +746,14 @@ define <16 x i8> @constant_fold_pshufb() {
 define <16 x i8> @constant_fold_pshufb_2() {
 ; SSE-LABEL: constant_fold_pshufb_2:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    movaps {{.*#+}} xmm0 = [2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2]
+; SSE-NEXT:    movl $2, %eax
+; SSE-NEXT:    movd %eax, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: constant_fold_pshufb_2:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2]
+; AVX-NEXT:    movl $2, %eax
+; AVX-NEXT:    vmovd %eax, %xmm0
 ; AVX-NEXT:    retq
   %1 = tail call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> <i8 2, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>, <16 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>)
   ret <16 x i8> %1
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-combining.ll b/llvm/test/CodeGen/X86/vector-shuffle-combining.ll
index 58b7be1ec..8cf8cab8b 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-combining.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-combining.ll
@@ -864,19 +864,12 @@ define <4 x i32> @combine_nested_undef_test15(<4 x i32> %A, <4 x i32> %B) {
 ; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5,6,7]
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: combine_nested_undef_test15:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[0,0,1,1]
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,0,1]
-; AVX1-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3]
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: combine_nested_undef_test15:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vbroadcastss %xmm1, %xmm1
-; AVX2-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,0,1]
-; AVX2-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3]
-; AVX2-NEXT:    retq
+; AVX-LABEL: combine_nested_undef_test15:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[0,0,1,1]
+; AVX-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,0,1]
+; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3]
+; AVX-NEXT:    retq
   %1 = shufflevector <4 x i32> %A, <4 x i32> %B, <4 x i32> <i32 0, i32 4, i32 3, i32 1>
   %2 = shufflevector <4 x i32> %1, <4 x i32> undef, <4 x i32> <i32 2, i32 1, i32 0, i32 3>
   ret <4 x i32> %2
@@ -1715,41 +1708,37 @@ define <4 x i8> @combine_test1c(<4 x i8>* %a, <4 x i8>* %b) {
 ; SSE2-LABEL: combine_test1c:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
-; SSE2-NEXT:    movss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
+; SSE2-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE2-NEXT:    movaps {{.*#+}} xmm0 = [0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; SSE2-NEXT:    andps %xmm0, %xmm2
+; SSE2-NEXT:    andnps %xmm1, %xmm0
+; SSE2-NEXT:    orps %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: combine_test1c:
 ; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSSE3-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; SSSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
-; SSSE3-NEXT:    movss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSSE3-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[1,2,4,6,u,u,u,u,u,u,u,u,u,u,u,u]
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: combine_test1c:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
-; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3,4,5,6,7]
+; SSE41-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE41-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE41-NEXT:    movaps {{.*#+}} xmm0 = <0,255,255,255,u,u,u,u,u,u,u,u,u,u,u,u>
+; SSE41-NEXT:    pblendvb %xmm0, %xmm2, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: combine_test1c:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
-; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7]
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: combine_test1c:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpmovzxbd {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
-; AVX2-NEXT:    vpmovzxbd {{.*#+}} xmm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
-; AVX2-NEXT:    retq
+; AVX-LABEL: combine_test1c:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm2 = <0,255,255,255,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
   %A = load <4 x i8>, <4 x i8>* %a
   %B = load <4 x i8>, <4 x i8>* %b
   %1 = shufflevector <4 x i8> %A, <4 x i8> %B, <4 x i32> <i32 0, i32 5, i32 2, i32 7>
@@ -1758,40 +1747,18 @@ define <4 x i8> @combine_test1c(<4 x i8>* %a, <4 x i8>* %b) {
 }
 
 define <4 x i8> @combine_test2c(<4 x i8>* %a, <4 x i8>* %b) {
-; SSE2-LABEL: combine_test2c:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
-; SSE2-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSE2-NEXT:    retq
-;
-; SSSE3-LABEL: combine_test2c:
-; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
-; SSSE3-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSSE3-NEXT:    retq
-;
-; SSE41-LABEL: combine_test2c:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
-; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
-; SSE41-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSE41-NEXT:    retq
+; SSE-LABEL: combine_test2c:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: combine_test2c:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpmovzxbd {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
-; AVX-NEXT:    vpmovzxbd {{.*#+}} xmm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
-; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
 ; AVX-NEXT:    retq
   %A = load <4 x i8>, <4 x i8>* %a
   %B = load <4 x i8>, <4 x i8>* %b
@@ -1801,40 +1768,20 @@ define <4 x i8> @combine_test2c(<4 x i8>* %a, <4 x i8>* %b) {
 }
 
 define <4 x i8> @combine_test3c(<4 x i8>* %a, <4 x i8>* %b) {
-; SSE2-LABEL: combine_test3c:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
-; SSE2-NEXT:    punpckhqdq {{.*#+}} xmm0 = xmm0[1],xmm1[1]
-; SSE2-NEXT:    retq
-;
-; SSSE3-LABEL: combine_test3c:
-; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; SSSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
-; SSSE3-NEXT:    punpckhqdq {{.*#+}} xmm0 = xmm0[1],xmm1[1]
-; SSSE3-NEXT:    retq
-;
-; SSE41-LABEL: combine_test3c:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
-; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
-; SSE41-NEXT:    punpckhqdq {{.*#+}} xmm0 = xmm0[1],xmm1[1]
-; SSE41-NEXT:    retq
+; SSE-LABEL: combine_test3c:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: combine_test3c:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpmovzxbd {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
-; AVX-NEXT:    vpmovzxbd {{.*#+}} xmm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
-; AVX-NEXT:    vpunpckhqdq {{.*#+}} xmm0 = xmm1[1],xmm0[1]
+; AVX-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
 ; AVX-NEXT:    retq
   %A = load <4 x i8>, <4 x i8>* %a
   %B = load <4 x i8>, <4 x i8>* %b
@@ -1846,48 +1793,38 @@ define <4 x i8> @combine_test3c(<4 x i8>* %a, <4 x i8>* %b) {
 define <4 x i8> @combine_test4c(<4 x i8>* %a, <4 x i8>* %b) {
 ; SSE2-LABEL: combine_test4c:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
 ; SSE2-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,0],xmm1[0,0]
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[2,0],xmm1[2,3]
+; SSE2-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE2-NEXT:    movaps {{.*#+}} xmm0 = [255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; SSE2-NEXT:    andps %xmm0, %xmm2
+; SSE2-NEXT:    andnps %xmm1, %xmm0
+; SSE2-NEXT:    orps %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: combine_test4c:
 ; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; SSSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
-; SSSE3-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; SSSE3-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,0],xmm1[0,0]
-; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; SSSE3-NEXT:    shufps {{.*#+}} xmm0 = xmm0[2,0],xmm1[2,3]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSSE3-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,3,4,6,u,u,u,u,u,u,u,u,u,u,u,u]
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: combine_test4c:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
-; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5,6,7]
+; SSE41-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE41-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE41-NEXT:    movaps {{.*#+}} xmm0 = <255,0,255,255,u,u,u,u,u,u,u,u,u,u,u,u>
+; SSE41-NEXT:    pblendvb %xmm0, %xmm2, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: combine_test4c:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
-; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3],xmm1[4,5,6,7]
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: combine_test4c:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpmovzxbd {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
-; AVX2-NEXT:    vpmovzxbd {{.*#+}} xmm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2,3]
-; AVX2-NEXT:    retq
+; AVX-LABEL: combine_test4c:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm2 = <255,0,255,255,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
   %A = load <4 x i8>, <4 x i8>* %a
   %B = load <4 x i8>, <4 x i8>* %b
   %1 = shufflevector <4 x i8> %A, <4 x i8> %B, <4 x i32> <i32 4, i32 1, i32 6, i32 3>
@@ -2882,3 +2819,211 @@ define <4 x i32> @PR41545(<4 x i32> %a0, <16 x i8> %a1) {
   %15 = add <4 x i32> %a0, %14
   ret <4 x i32> %15
 }
+
+define <8 x i16> @shuffle_extract_insert(<8 x i16> %a) {
+; SSE-LABEL: shuffle_extract_insert:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[2,1,0,3,4,5,6,7]
+; SSE-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,4,7]
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: shuffle_extract_insert:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[2,1,0,3,4,5,6,7]
+; AVX1-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,4,7]
+; AVX1-NEXT:    retq
+;
+; AVX2-SLOW-LABEL: shuffle_extract_insert:
+; AVX2-SLOW:       # %bb.0:
+; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[2,1,0,3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,4,7]
+; AVX2-SLOW-NEXT:    retq
+;
+; AVX2-FAST-LABEL: shuffle_extract_insert:
+; AVX2-FAST:       # %bb.0:
+; AVX2-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[4,5,2,3,0,1,6,7,12,13,10,11,8,9,14,15]
+; AVX2-FAST-NEXT:    retq
+  %a0 = extractelement <8 x i16> %a, i32 0
+  %a1 = extractelement <8 x i16> %a, i32 1
+  %a3 = extractelement <8 x i16> %a, i32 3
+  %a4 = extractelement <8 x i16> %a, i32 4
+  %a5 = extractelement <8 x i16> %a, i32 5
+  %a6 = extractelement <8 x i16> %a, i32 6
+  %a7 = extractelement <8 x i16> %a, i32 7
+  %1 = shufflevector <8 x i16> %a, <8 x i16> %a, <8 x i32> <i32 2, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %2 = insertelement <8 x i16> %1, i16 %a1, i32 1
+  %3 = insertelement <8 x i16> %2, i16 %a0, i32 2
+  %4 = insertelement <8 x i16> %3, i16 %a3, i32 3
+  %5 = insertelement <8 x i16> %4, i16 %a6, i32 4
+  %6 = insertelement <8 x i16> %5, i16 %a5, i32 5
+  %7 = insertelement <8 x i16> %6, i16 %a4, i32 6
+  %8 = insertelement <8 x i16> %7, i16 %a7, i32 7
+  ret <8 x i16> %8
+}
+
+define <8 x i16> @shuffle_extract_insert_double(<8 x i16> %a, <8 x i16> %b) {
+; SSE2-LABEL: shuffle_extract_insert_double:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,3,2,3,4,5,6,7]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,7,5,6,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,1,3,2,4,5,6,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,6,6,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[1,0,3,2,4,5,6,7]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: shuffle_extract_insert_double:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    pshufb {{.*#+}} xmm1 = xmm1[0,1,6,7,10,11,14,15,14,15,10,11,12,13,14,15]
+; SSSE3-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[4,5,0,1,12,13,8,9,8,9,12,13,12,13,14,15]
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: shuffle_extract_insert_double:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pshufb {{.*#+}} xmm1 = xmm1[0,1,6,7,10,11,14,15,14,15,10,11,12,13,14,15]
+; SSE41-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[4,5,0,1,12,13,8,9,8,9,12,13,12,13,14,15]
+; SSE41-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: shuffle_extract_insert_double:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0,1,6,7,10,11,14,15,14,15,10,11,12,13,14,15]
+; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[4,5,0,1,12,13,8,9,8,9,12,13,12,13,14,15]
+; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX-NEXT:    retq
+  %a0 = extractelement <8 x i16> %a, i32 0
+  %a4 = extractelement <8 x i16> %a, i32 4
+  %a6 = extractelement <8 x i16> %a, i32 6
+  %b11 = extractelement <8 x i16> %b, i32 3
+  %b13 = extractelement <8 x i16> %b, i32 5
+  %b15 = extractelement <8 x i16> %b, i32 7
+  %1 = shufflevector <8 x i16> %a, <8 x i16> %b, <8 x i32> <i32 2, i32 8, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %2 = insertelement <8 x i16> %1, i16 %a0, i32 2
+  %3 = insertelement <8 x i16> %2, i16 %b11, i32 3
+  %4 = insertelement <8 x i16> %3, i16 %a6, i32 4
+  %5 = insertelement <8 x i16> %4, i16 %b13, i32 5
+  %6 = insertelement <8 x i16> %5, i16 %a4, i32 6
+  %7 = insertelement <8 x i16> %6, i16 %b15, i32 7
+  ret <8 x i16> %7
+}
+
+define <8 x i16> @shuffle_extract_concat_insert(<4 x i16> %lhsa, <4 x i16> %rhsa, <8 x i16> %b) {
+; SSE2-LABEL: shuffle_extract_concat_insert:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,6,6,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[1,0,3,2,4,5,6,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,3,2,3,4,5,6,7]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,7,5,6,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,1,3,2,4,5,6,7]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: shuffle_extract_concat_insert:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSSE3-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[4,5,0,1,12,13,8,9,8,9,12,13,12,13,14,15]
+; SSSE3-NEXT:    pshufb {{.*#+}} xmm2 = xmm2[0,1,6,7,10,11,14,15,14,15,10,11,12,13,14,15]
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: shuffle_extract_concat_insert:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE41-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[4,5,0,1,12,13,8,9,8,9,12,13,12,13,14,15]
+; SSE41-NEXT:    pshufb {{.*#+}} xmm2 = xmm2[0,1,6,7,10,11,14,15,14,15,10,11,12,13,14,15]
+; SSE41-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: shuffle_extract_concat_insert:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[4,5,0,1,12,13,8,9,8,9,12,13,12,13,14,15]
+; AVX-NEXT:    vpshufb {{.*#+}} xmm1 = xmm2[0,1,6,7,10,11,14,15,14,15,10,11,12,13,14,15]
+; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX-NEXT:    retq
+  %a = shufflevector <4 x i16> %lhsa, <4 x i16> %rhsa, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %a0 = extractelement <8 x i16> %a, i32 0
+  %a4 = extractelement <8 x i16> %a, i32 4
+  %a6 = extractelement <8 x i16> %a, i32 6
+  %b11 = extractelement <8 x i16> %b, i32 3
+  %b13 = extractelement <8 x i16> %b, i32 5
+  %b15 = extractelement <8 x i16> %b, i32 7
+  %1 = shufflevector <8 x i16> %a, <8 x i16> %b, <8 x i32> <i32 2, i32 8, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %2 = insertelement <8 x i16> %1, i16 %a0, i32 2
+  %3 = insertelement <8 x i16> %2, i16 %b11, i32 3
+  %4 = insertelement <8 x i16> %3, i16 %a6, i32 4
+  %5 = insertelement <8 x i16> %4, i16 %b13, i32 5
+  %6 = insertelement <8 x i16> %5, i16 %a4, i32 6
+  %7 = insertelement <8 x i16> %6, i16 %b15, i32 7
+  ret <8 x i16> %7
+}
+
+define void @PR43024() {
+; SSE2-LABEL: PR43024:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movaps {{.*#+}} xmm0 = [NaN,NaN,0.0E+0,0.0E+0]
+; SSE2-NEXT:    movaps %xmm0, (%rax)
+; SSE2-NEXT:    movaps %xmm0, %xmm1
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[2,3]
+; SSE2-NEXT:    addss %xmm0, %xmm1
+; SSE2-NEXT:    xorps %xmm0, %xmm0
+; SSE2-NEXT:    addss %xmm0, %xmm1
+; SSE2-NEXT:    addss %xmm0, %xmm1
+; SSE2-NEXT:    movss %xmm1, (%rax)
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: PR43024:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movaps {{.*#+}} xmm0 = [NaN,NaN,0.0E+0,0.0E+0]
+; SSSE3-NEXT:    movaps %xmm0, (%rax)
+; SSSE3-NEXT:    movshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    addss %xmm0, %xmm1
+; SSSE3-NEXT:    xorps %xmm0, %xmm0
+; SSSE3-NEXT:    addss %xmm0, %xmm1
+; SSSE3-NEXT:    addss %xmm0, %xmm1
+; SSSE3-NEXT:    movss %xmm1, (%rax)
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: PR43024:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movaps {{.*#+}} xmm0 = [NaN,NaN,0.0E+0,0.0E+0]
+; SSE41-NEXT:    movaps %xmm0, (%rax)
+; SSE41-NEXT:    movshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; SSE41-NEXT:    addss %xmm0, %xmm1
+; SSE41-NEXT:    xorps %xmm0, %xmm0
+; SSE41-NEXT:    addss %xmm0, %xmm1
+; SSE41-NEXT:    addss %xmm0, %xmm1
+; SSE41-NEXT:    movss %xmm1, (%rax)
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: PR43024:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [NaN,NaN,0.0E+0,0.0E+0]
+; AVX-NEXT:    vmovaps %xmm0, (%rax)
+; AVX-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX-NEXT:    vaddss %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vaddss %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vaddss %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vmovss %xmm0, (%rax)
+; AVX-NEXT:    retq
+  store <4 x float> <float 0x7FF8000000000000, float 0x7FF8000000000000, float 0x0, float 0x0>, <4 x float>* undef, align 16
+  %1 = load <4 x float>, <4 x float>* undef, align 16
+  %2 = fmul <4 x float> %1, <float 0x0, float 0x0, float 0x0, float 0x0>
+  %3 = shufflevector <4 x float> %2, <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+  %4 = fadd <4 x float> %2, %3
+  %5 = fadd <4 x float> zeroinitializer, %4
+  %6 = shufflevector <4 x float> %2, <4 x float> undef, <4 x i32> <i32 3, i32 undef, i32 undef, i32 undef>
+  %7 = fadd <4 x float> %6, %5
+  %8 = extractelement <4 x float> %7, i32 0
+  store float %8, float* undef, align 8
+  ret void
+}
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-v1.ll b/llvm/test/CodeGen/X86/vector-shuffle-v1.ll
index d926e5427..94b00fbd9 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-v1.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-v1.ll
@@ -267,42 +267,50 @@ define <32 x i1> @shuf32i1_3_6_22_12_3_7_7_0_3_6_1_13_3_21_7_0_3_6_22_12_3_7_7_0
 define <32 x i16> @shuf32i1_3_6_22_12_3_7_7_0_3_6_1_13_3_21_7_0_3_6_22_12_3_7_7_0_3_6_1_13_3_21_7_0_icmp_v32i16(<32 x i16> %a, <32 x i16> %c, <32 x i16> %d) {
 ; AVX512F-LABEL: shuf32i1_3_6_22_12_3_7_7_0_3_6_1_13_3_21_7_0_3_6_22_12_3_7_7_0_3_6_1_13_3_21_7_0_icmp_v32i16:
 ; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm2, %ymm4
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm5
 ; AVX512F-NEXT:    vpxor %xmm6, %xmm6, %xmm6
 ; AVX512F-NEXT:    vpcmpeqw %ymm6, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpmovsxwd %ymm0, %zmm0
 ; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k1
-; AVX512F-NEXT:    vpcmpeqw %ymm6, %ymm1, %ymm0
+; AVX512F-NEXT:    vpcmpeqw %ymm6, %ymm5, %ymm0
 ; AVX512F-NEXT:    vpmovsxwd %ymm0, %zmm0
 ; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k2
 ; AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k2} {z}
-; AVX512F-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k1} {z}
+; AVX512F-NEXT:    vpternlogd $255, %zmm5, %zmm5, %zmm5 {%k1} {z}
 ; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm6 = [3,6,22,12,3,7,7,0,3,6,1,13,3,21,7,0]
-; AVX512F-NEXT:    vpermi2d %zmm0, %zmm1, %zmm6
+; AVX512F-NEXT:    vpermi2d %zmm0, %zmm5, %zmm6
 ; AVX512F-NEXT:    vptestmd %zmm6, %zmm6, %k1
 ; AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
-; AVX512F-NEXT:    vpmovdw %zmm0, %ymm1
-; AVX512F-NEXT:    vpblendvb %ymm1, %ymm2, %ymm4, %ymm0
-; AVX512F-NEXT:    vpblendvb %ymm1, %ymm3, %ymm5, %ymm1
+; AVX512F-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512F-NEXT:    vpblendvb %ymm0, %ymm3, %ymm4, %ymm3
+; AVX512F-NEXT:    vpblendvb %ymm0, %ymm1, %ymm2, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: shuf32i1_3_6_22_12_3_7_7_0_3_6_1_13_3_21_7_0_3_6_22_12_3_7_7_0_3_6_1_13_3_21_7_0_icmp_v32i16:
 ; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm2, %ymm4
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm5
 ; AVX512VL-NEXT:    vpxor %xmm6, %xmm6, %xmm6
 ; AVX512VL-NEXT:    vpcmpeqw %ymm6, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vpmovsxwd %ymm0, %zmm0
 ; AVX512VL-NEXT:    vptestmd %zmm0, %zmm0, %k1
-; AVX512VL-NEXT:    vpcmpeqw %ymm6, %ymm1, %ymm0
+; AVX512VL-NEXT:    vpcmpeqw %ymm6, %ymm5, %ymm0
 ; AVX512VL-NEXT:    vpmovsxwd %ymm0, %zmm0
 ; AVX512VL-NEXT:    vptestmd %zmm0, %zmm0, %k2
 ; AVX512VL-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k2} {z}
-; AVX512VL-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k1} {z}
+; AVX512VL-NEXT:    vpternlogd $255, %zmm5, %zmm5, %zmm5 {%k1} {z}
 ; AVX512VL-NEXT:    vmovdqa64 {{.*#+}} zmm6 = [3,6,22,12,3,7,7,0,3,6,1,13,3,21,7,0]
-; AVX512VL-NEXT:    vpermi2d %zmm0, %zmm1, %zmm6
+; AVX512VL-NEXT:    vpermi2d %zmm0, %zmm5, %zmm6
 ; AVX512VL-NEXT:    vptestmd %zmm6, %zmm6, %k1
 ; AVX512VL-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
-; AVX512VL-NEXT:    vpmovdw %zmm0, %ymm1
-; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm2, %ymm4, %ymm0
-; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm3, %ymm5, %ymm1
+; AVX512VL-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512VL-NEXT:    vpblendvb %ymm0, %ymm3, %ymm4, %ymm3
+; AVX512VL-NEXT:    vpblendvb %ymm0, %ymm1, %ymm2, %ymm0
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; VL_BW_DQ-LABEL: shuf32i1_3_6_22_12_3_7_7_0_3_6_1_13_3_21_7_0_3_6_22_12_3_7_7_0_3_6_1_13_3_21_7_0_icmp_v32i16:
@@ -381,6 +389,8 @@ define <32 x i8> @shuf32i1_3_6_22_12_3_7_7_0_3_6_1_13_3_21_7_0_3_6_22_12_3_7_7_0
 define <32 x i16> @shuf32i1_3_6_22_12_3_7_7_0_3_6_1_13_3_21_7_0_3_6_22_12_3_7_7_0_3_6_1_13_3_21_7_0_icmp_v32i16_split(<16 x i32> %a, <16 x i32> %b, <32 x i16> %c, <32 x i16> %d) {
 ; AVX512F-LABEL: shuf32i1_3_6_22_12_3_7_7_0_3_6_1_13_3_21_7_0_3_6_22_12_3_7_7_0_3_6_1_13_3_21_7_0_icmp_v32i16_split:
 ; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm2, %ymm4
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm3, %ymm5
 ; AVX512F-NEXT:    vptestnmd %zmm0, %zmm0, %k1
 ; AVX512F-NEXT:    vptestnmd %zmm1, %zmm1, %k2
 ; AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k2} {z}
@@ -389,13 +399,16 @@ define <32 x i16> @shuf32i1_3_6_22_12_3_7_7_0_3_6_1_13_3_21_7_0_3_6_22_12_3_7_7_
 ; AVX512F-NEXT:    vpermi2d %zmm0, %zmm1, %zmm6
 ; AVX512F-NEXT:    vptestmd %zmm6, %zmm6, %k1
 ; AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
-; AVX512F-NEXT:    vpmovdw %zmm0, %ymm1
-; AVX512F-NEXT:    vpblendvb %ymm1, %ymm2, %ymm4, %ymm0
-; AVX512F-NEXT:    vpblendvb %ymm1, %ymm3, %ymm5, %ymm1
+; AVX512F-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512F-NEXT:    vpblendvb %ymm0, %ymm4, %ymm5, %ymm1
+; AVX512F-NEXT:    vpblendvb %ymm0, %ymm2, %ymm3, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: shuf32i1_3_6_22_12_3_7_7_0_3_6_1_13_3_21_7_0_3_6_22_12_3_7_7_0_3_6_1_13_3_21_7_0_icmp_v32i16_split:
 ; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm2, %ymm4
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm3, %ymm5
 ; AVX512VL-NEXT:    vptestnmd %zmm0, %zmm0, %k1
 ; AVX512VL-NEXT:    vptestnmd %zmm1, %zmm1, %k2
 ; AVX512VL-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k2} {z}
@@ -404,9 +417,10 @@ define <32 x i16> @shuf32i1_3_6_22_12_3_7_7_0_3_6_1_13_3_21_7_0_3_6_22_12_3_7_7_
 ; AVX512VL-NEXT:    vpermi2d %zmm0, %zmm1, %zmm6
 ; AVX512VL-NEXT:    vptestmd %zmm6, %zmm6, %k1
 ; AVX512VL-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
-; AVX512VL-NEXT:    vpmovdw %zmm0, %ymm1
-; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm2, %ymm4, %ymm0
-; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm3, %ymm5, %ymm1
+; AVX512VL-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512VL-NEXT:    vpblendvb %ymm0, %ymm4, %ymm5, %ymm1
+; AVX512VL-NEXT:    vpblendvb %ymm0, %ymm2, %ymm3, %ymm0
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; VL_BW_DQ-LABEL: shuf32i1_3_6_22_12_3_7_7_0_3_6_1_13_3_21_7_0_3_6_22_12_3_7_7_0_3_6_1_13_3_21_7_0_icmp_v32i16_split:
diff --git a/llvm/test/CodeGen/X86/vector-trunc-math.ll b/llvm/test/CodeGen/X86/vector-trunc-math.ll
index 3d2202e38..38cd2a3ae 100644
--- a/llvm/test/CodeGen/X86/vector-trunc-math.ll
+++ b/llvm/test/CodeGen/X86/vector-trunc-math.ll
@@ -389,7 +389,7 @@ define <16 x i8> @trunc_add_v16i16_v16i8(<16 x i16> %a0, <16 x i16> %a1) nounwin
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    paddw %xmm2, %xmm0
 ; SSE-NEXT:    paddw %xmm3, %xmm1
-; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
 ; SSE-NEXT:    pand %xmm2, %xmm1
 ; SSE-NEXT:    pand %xmm2, %xmm0
 ; SSE-NEXT:    packuswb %xmm1, %xmm0
@@ -812,7 +812,7 @@ define <16 x i8> @trunc_add_const_v16i32_v16i8(<16 x i32> %a0) nounwind {
 define <16 x i8> @trunc_add_const_v16i16_v16i8(<16 x i16> %a0) nounwind {
 ; SSE-LABEL: trunc_add_const_v16i16_v16i8:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
 ; SSE-NEXT:    pand %xmm2, %xmm1
 ; SSE-NEXT:    pand %xmm2, %xmm0
 ; SSE-NEXT:    packuswb %xmm1, %xmm0
@@ -1247,7 +1247,7 @@ define <16 x i8> @trunc_sub_v16i16_v16i8(<16 x i16> %a0, <16 x i16> %a1) nounwin
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    psubw %xmm2, %xmm0
 ; SSE-NEXT:    psubw %xmm3, %xmm1
-; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
 ; SSE-NEXT:    pand %xmm2, %xmm1
 ; SSE-NEXT:    pand %xmm2, %xmm0
 ; SSE-NEXT:    packuswb %xmm1, %xmm0
@@ -1638,7 +1638,7 @@ define <16 x i8> @trunc_sub_const_v16i32_v16i8(<16 x i32> %a0) nounwind {
 define <16 x i8> @trunc_sub_const_v16i16_v16i8(<16 x i16> %a0) nounwind {
 ; SSE-LABEL: trunc_sub_const_v16i16_v16i8:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
 ; SSE-NEXT:    pand %xmm2, %xmm1
 ; SSE-NEXT:    pand %xmm2, %xmm0
 ; SSE-NEXT:    packuswb %xmm1, %xmm0
@@ -2240,7 +2240,7 @@ define <16 x i8> @trunc_mul_v16i16_v16i8(<16 x i16> %a0, <16 x i16> %a1) nounwin
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pmullw %xmm2, %xmm0
 ; SSE-NEXT:    pmullw %xmm3, %xmm1
-; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
 ; SSE-NEXT:    pand %xmm2, %xmm1
 ; SSE-NEXT:    pand %xmm2, %xmm0
 ; SSE-NEXT:    packuswb %xmm1, %xmm0
@@ -2728,7 +2728,7 @@ define <16 x i8> @trunc_mul_const_v16i16_v16i8(<16 x i16> %a0) nounwind {
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pmullw {{.*}}(%rip), %xmm0
 ; SSE-NEXT:    pmullw {{.*}}(%rip), %xmm1
-; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
 ; SSE-NEXT:    pand %xmm2, %xmm1
 ; SSE-NEXT:    pand %xmm2, %xmm0
 ; SSE-NEXT:    packuswb %xmm1, %xmm0
@@ -2983,20 +2983,20 @@ define <16 x i8> @trunc_and_v16i64_v16i8(<16 x i64> %a0, <16 x i64> %a1) nounwin
 ; AVX1-LABEL: trunc_and_v16i64_v16i8:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vmovaps {{.*#+}} ymm8 = [255,255,255,255]
-; AVX1-NEXT:    vandps %ymm8, %ymm7, %ymm7
+; AVX1-NEXT:    vandps %ymm7, %ymm8, %ymm7
 ; AVX1-NEXT:    vandps %ymm7, %ymm3, %ymm3
 ; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm7
 ; AVX1-NEXT:    vpackusdw %xmm7, %xmm3, %xmm3
-; AVX1-NEXT:    vandps %ymm8, %ymm6, %ymm6
+; AVX1-NEXT:    vandps %ymm6, %ymm8, %ymm6
 ; AVX1-NEXT:    vandps %ymm6, %ymm2, %ymm2
 ; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm6
 ; AVX1-NEXT:    vpackusdw %xmm6, %xmm2, %xmm2
 ; AVX1-NEXT:    vpackusdw %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vandps %ymm8, %ymm5, %ymm3
+; AVX1-NEXT:    vandps %ymm5, %ymm8, %ymm3
 ; AVX1-NEXT:    vandps %ymm3, %ymm1, %ymm1
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
 ; AVX1-NEXT:    vpackusdw %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vandps %ymm8, %ymm4, %ymm3
+; AVX1-NEXT:    vandps %ymm4, %ymm8, %ymm3
 ; AVX1-NEXT:    vandps %ymm3, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
 ; AVX1-NEXT:    vpackusdw %xmm3, %xmm0, %xmm0
@@ -3135,7 +3135,7 @@ define <16 x i8> @trunc_and_v16i32_v16i8(<16 x i32> %a0, <16 x i32> %a1) nounwin
 define <16 x i8> @trunc_and_v16i16_v16i8(<16 x i16> %a0, <16 x i16> %a1) nounwind {
 ; SSE-LABEL: trunc_and_v16i16_v16i8:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    movdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
+; SSE-NEXT:    movdqa {{.*#+}} xmm4 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
 ; SSE-NEXT:    pand %xmm4, %xmm3
 ; SSE-NEXT:    pand %xmm1, %xmm3
 ; SSE-NEXT:    pand %xmm4, %xmm2
@@ -3507,7 +3507,7 @@ define <16 x i8> @trunc_and_const_v16i32_v16i8(<16 x i32> %a0) nounwind {
 define <16 x i8> @trunc_and_const_v16i16_v16i8(<16 x i16> %a0) nounwind {
 ; SSE-LABEL: trunc_and_const_v16i16_v16i8:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
 ; SSE-NEXT:    pand %xmm2, %xmm1
 ; SSE-NEXT:    pand %xmm2, %xmm0
 ; SSE-NEXT:    packuswb %xmm1, %xmm0
@@ -3914,7 +3914,7 @@ define <16 x i8> @trunc_xor_v16i16_v16i8(<16 x i16> %a0, <16 x i16> %a1) nounwin
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pxor %xmm2, %xmm0
 ; SSE-NEXT:    pxor %xmm3, %xmm1
-; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
 ; SSE-NEXT:    pand %xmm2, %xmm1
 ; SSE-NEXT:    pand %xmm2, %xmm0
 ; SSE-NEXT:    packuswb %xmm1, %xmm0
@@ -4284,7 +4284,7 @@ define <16 x i8> @trunc_xor_const_v16i32_v16i8(<16 x i32> %a0) nounwind {
 define <16 x i8> @trunc_xor_const_v16i16_v16i8(<16 x i16> %a0) nounwind {
 ; SSE-LABEL: trunc_xor_const_v16i16_v16i8:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
 ; SSE-NEXT:    pand %xmm2, %xmm1
 ; SSE-NEXT:    pand %xmm2, %xmm0
 ; SSE-NEXT:    packuswb %xmm1, %xmm0
@@ -4691,7 +4691,7 @@ define <16 x i8> @trunc_or_v16i16_v16i8(<16 x i16> %a0, <16 x i16> %a1) nounwind
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    por %xmm2, %xmm0
 ; SSE-NEXT:    por %xmm3, %xmm1
-; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
 ; SSE-NEXT:    pand %xmm2, %xmm1
 ; SSE-NEXT:    pand %xmm2, %xmm0
 ; SSE-NEXT:    packuswb %xmm1, %xmm0
@@ -5061,7 +5061,7 @@ define <16 x i8> @trunc_or_const_v16i32_v16i8(<16 x i32> %a0) nounwind {
 define <16 x i8> @trunc_or_const_v16i16_v16i8(<16 x i16> %a0) nounwind {
 ; SSE-LABEL: trunc_or_const_v16i16_v16i8:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
 ; SSE-NEXT:    pand %xmm2, %xmm1
 ; SSE-NEXT:    pand %xmm2, %xmm0
 ; SSE-NEXT:    packuswb %xmm1, %xmm0
diff --git a/llvm/test/CodeGen/X86/vector-trunc-packus.ll b/llvm/test/CodeGen/X86/vector-trunc-packus.ll
index 18917cf96..80ce22ca9 100644
--- a/llvm/test/CodeGen/X86/vector-trunc-packus.ll
+++ b/llvm/test/CodeGen/X86/vector-trunc-packus.ll
@@ -9,11 +9,315 @@
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vl,+fast-variable-shuffle | FileCheck %s --check-prefixes=AVX512,AVX512VL
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+fast-variable-shuffle | FileCheck %s --check-prefixes=AVX512,AVX512BW
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl,+fast-variable-shuffle | FileCheck %s --check-prefixes=AVX512,AVX512BWVL
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=skx | FileCheck %s --check-prefixes=SKX
 
 ;
 ; PACKUS saturation truncation to vXi32
 ;
 
+define <2 x i32> @trunc_packus_v2i64_v2i32(<2 x i64> %a0) {
+; SSE2-LABEL: trunc_packus_v2i64_v2i32:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    pxor %xmm1, %xmm2
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [2147483647,2147483647]
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm3, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE2-NEXT:    pand %xmm5, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm2, %xmm3
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm3
+; SSE2-NEXT:    por %xmm0, %xmm3
+; SSE2-NEXT:    movdqa %xmm3, %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pand %xmm4, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,3,3]
+; SSE2-NEXT:    por %xmm0, %xmm1
+; SSE2-NEXT:    pand %xmm3, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: trunc_packus_v2i64_v2i32:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [2147483648,2147483648]
+; SSSE3-NEXT:    movdqa %xmm0, %xmm2
+; SSSE3-NEXT:    pxor %xmm1, %xmm2
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm3 = [2147483647,2147483647]
+; SSSE3-NEXT:    movdqa %xmm3, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm5, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm2, %xmm3
+; SSSE3-NEXT:    pand %xmm3, %xmm0
+; SSSE3-NEXT:    pandn {{.*}}(%rip), %xmm3
+; SSSE3-NEXT:    por %xmm0, %xmm3
+; SSSE3-NEXT:    movdqa %xmm3, %xmm0
+; SSSE3-NEXT:    pxor %xmm1, %xmm0
+; SSSE3-NEXT:    movdqa %xmm0, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm4, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    por %xmm0, %xmm1
+; SSSE3-NEXT:    pand %xmm3, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: trunc_packus_v2i64_v2i32:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    movapd {{.*#+}} xmm2 = [4294967295,4294967295]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [2147483648,2147483648]
+; SSE41-NEXT:    pxor %xmm3, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [2147483647,2147483647]
+; SSE41-NEXT:    movdqa %xmm4, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm0
+; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
+; SSE41-NEXT:    xorpd %xmm1, %xmm1
+; SSE41-NEXT:    movapd %xmm2, %xmm4
+; SSE41-NEXT:    xorpd %xmm3, %xmm4
+; SSE41-NEXT:    movapd %xmm4, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm3, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm0
+; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: trunc_packus_v2i64_v2i32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [4294967295,4294967295]
+; AVX-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm2
+; AVX-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm1
+; AVX-NEXT:    vpand %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX-NEXT:    retq
+;
+; AVX512F-LABEL: trunc_packus_v2i64_v2i32:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vpmovusqd %zmm0, %ymm0
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: trunc_packus_v2i64_v2i32:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512VL-NEXT:    vpmaxsq %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpmovusqd %xmm0, %xmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: trunc_packus_v2i64_v2i32:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512BW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512BW-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmovusqd %zmm0, %ymm0
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: trunc_packus_v2i64_v2i32:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512BWVL-NEXT:    vpmaxsq %xmm1, %xmm0, %xmm0
+; AVX512BWVL-NEXT:    vpmovusqd %xmm0, %xmm0
+; AVX512BWVL-NEXT:    retq
+;
+; SKX-LABEL: trunc_packus_v2i64_v2i32:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; SKX-NEXT:    vpmaxsq %xmm1, %xmm0, %xmm0
+; SKX-NEXT:    vpmovusqd %xmm0, %xmm0
+; SKX-NEXT:    retq
+  %1 = icmp slt <2 x i64> %a0, <i64 4294967295, i64 4294967295>
+  %2 = select <2 x i1> %1, <2 x i64> %a0, <2 x i64> <i64 4294967295, i64 4294967295>
+  %3 = icmp sgt <2 x i64> %2, zeroinitializer
+  %4 = select <2 x i1> %3, <2 x i64> %2, <2 x i64> zeroinitializer
+  %5 = trunc <2 x i64> %4 to <2 x i32>
+  ret <2 x i32> %5
+}
+
+define void @trunc_packus_v2i64_v2i32_store(<2 x i64> %a0, <2 x i32>* %p1) {
+; SSE2-LABEL: trunc_packus_v2i64_v2i32_store:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    pxor %xmm1, %xmm2
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [2147483647,2147483647]
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm3, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE2-NEXT:    pand %xmm5, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm2, %xmm3
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm3
+; SSE2-NEXT:    por %xmm0, %xmm3
+; SSE2-NEXT:    movdqa %xmm3, %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pand %xmm4, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,3,3]
+; SSE2-NEXT:    por %xmm0, %xmm1
+; SSE2-NEXT:    pand %xmm3, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
+; SSE2-NEXT:    movq %xmm0, (%rdi)
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: trunc_packus_v2i64_v2i32_store:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [2147483648,2147483648]
+; SSSE3-NEXT:    movdqa %xmm0, %xmm2
+; SSSE3-NEXT:    pxor %xmm1, %xmm2
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm3 = [2147483647,2147483647]
+; SSSE3-NEXT:    movdqa %xmm3, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm5, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm2, %xmm3
+; SSSE3-NEXT:    pand %xmm3, %xmm0
+; SSSE3-NEXT:    pandn {{.*}}(%rip), %xmm3
+; SSSE3-NEXT:    por %xmm0, %xmm3
+; SSSE3-NEXT:    movdqa %xmm3, %xmm0
+; SSSE3-NEXT:    pxor %xmm1, %xmm0
+; SSSE3-NEXT:    movdqa %xmm0, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm4, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    por %xmm0, %xmm1
+; SSSE3-NEXT:    pand %xmm3, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
+; SSSE3-NEXT:    movq %xmm0, (%rdi)
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: trunc_packus_v2i64_v2i32_store:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    movapd {{.*#+}} xmm2 = [4294967295,4294967295]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [2147483648,2147483648]
+; SSE41-NEXT:    pxor %xmm3, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [2147483647,2147483647]
+; SSE41-NEXT:    movdqa %xmm4, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm0
+; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
+; SSE41-NEXT:    xorpd %xmm1, %xmm1
+; SSE41-NEXT:    movapd %xmm2, %xmm4
+; SSE41-NEXT:    xorpd %xmm3, %xmm4
+; SSE41-NEXT:    movapd %xmm4, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm3, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm0
+; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
+; SSE41-NEXT:    movq %xmm0, (%rdi)
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: trunc_packus_v2i64_v2i32_store:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [4294967295,4294967295]
+; AVX-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm2
+; AVX-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm1
+; AVX-NEXT:    vpand %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX-NEXT:    vmovq %xmm0, (%rdi)
+; AVX-NEXT:    retq
+;
+; AVX512F-LABEL: trunc_packus_v2i64_v2i32_store:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vpmovusqd %zmm0, %ymm0
+; AVX512F-NEXT:    vmovq %xmm0, (%rdi)
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: trunc_packus_v2i64_v2i32_store:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512VL-NEXT:    vpmaxsq %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpmovusqd %xmm0, (%rdi)
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: trunc_packus_v2i64_v2i32_store:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512BW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512BW-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmovusqd %zmm0, %ymm0
+; AVX512BW-NEXT:    vmovq %xmm0, (%rdi)
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: trunc_packus_v2i64_v2i32_store:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512BWVL-NEXT:    vpmaxsq %xmm1, %xmm0, %xmm0
+; AVX512BWVL-NEXT:    vpmovusqd %xmm0, (%rdi)
+; AVX512BWVL-NEXT:    retq
+;
+; SKX-LABEL: trunc_packus_v2i64_v2i32_store:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; SKX-NEXT:    vpmaxsq %xmm1, %xmm0, %xmm0
+; SKX-NEXT:    vpmovusqd %xmm0, (%rdi)
+; SKX-NEXT:    retq
+  %1 = icmp slt <2 x i64> %a0, <i64 4294967295, i64 4294967295>
+  %2 = select <2 x i1> %1, <2 x i64> %a0, <2 x i64> <i64 4294967295, i64 4294967295>
+  %3 = icmp sgt <2 x i64> %2, zeroinitializer
+  %4 = select <2 x i1> %3, <2 x i64> %2, <2 x i64> zeroinitializer
+  %5 = trunc <2 x i64> %4 to <2 x i32>
+  store <2 x i32> %5, <2 x i32>* %p1
+  ret void
+}
+
 define <4 x i32> @trunc_packus_v4i64_v4i32(<4 x i64> %a0) {
 ; SSE2-LABEL: trunc_packus_v4i64_v4i32:
 ; SSE2:       # %bb.0:
@@ -135,43 +439,40 @@ define <4 x i32> @trunc_packus_v4i64_v4i32(<4 x i64> %a0) {
 ; SSE41-NEXT:    pxor %xmm3, %xmm0
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm6 = [2147483647,2147483647]
 ; SSE41-NEXT:    movdqa %xmm6, %xmm5
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm5[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm6, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm7, %xmm0
-; SSE41-NEXT:    por %xmm5, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm5
+; SSE41-NEXT:    movdqa %xmm6, %xmm7
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm7
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm0
+; SSE41-NEXT:    por %xmm7, %xmm0
 ; SSE41-NEXT:    movapd %xmm4, %xmm5
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm5
 ; SSE41-NEXT:    movdqa %xmm1, %xmm0
 ; SSE41-NEXT:    pxor %xmm3, %xmm0
 ; SSE41-NEXT:    movdqa %xmm6, %xmm2
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm2[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm6, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm7, %xmm0
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm2
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
+; SSE41-NEXT:    pand %xmm2, %xmm0
+; SSE41-NEXT:    por %xmm6, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm4
 ; SSE41-NEXT:    xorpd %xmm1, %xmm1
-; SSE41-NEXT:    movapd %xmm4, %xmm0
-; SSE41-NEXT:    xorpd %xmm3, %xmm0
-; SSE41-NEXT:    movapd %xmm0, %xmm2
+; SSE41-NEXT:    movapd %xmm4, %xmm2
+; SSE41-NEXT:    xorpd %xmm3, %xmm2
+; SSE41-NEXT:    movapd %xmm2, %xmm6
+; SSE41-NEXT:    pcmpeqd %xmm3, %xmm6
 ; SSE41-NEXT:    pcmpgtd %xmm3, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,0,2,2]
 ; SSE41-NEXT:    pand %xmm6, %xmm0
 ; SSE41-NEXT:    por %xmm2, %xmm0
 ; SSE41-NEXT:    pxor %xmm2, %xmm2
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm2
-; SSE41-NEXT:    movapd %xmm5, %xmm0
-; SSE41-NEXT:    xorpd %xmm3, %xmm0
-; SSE41-NEXT:    movapd %xmm0, %xmm4
+; SSE41-NEXT:    movapd %xmm5, %xmm4
+; SSE41-NEXT:    xorpd %xmm3, %xmm4
+; SSE41-NEXT:    movapd %xmm4, %xmm6
+; SSE41-NEXT:    pcmpeqd %xmm3, %xmm6
 ; SSE41-NEXT:    pcmpgtd %xmm3, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2]
 ; SSE41-NEXT:    pand %xmm6, %xmm0
 ; SSE41-NEXT:    por %xmm4, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm5, %xmm1
@@ -226,11 +527,9 @@ define <4 x i32> @trunc_packus_v4i64_v4i32(<4 x i64> %a0) {
 ; AVX512F-LABEL: trunc_packus_v4i64_v4i32:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; AVX512F-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [4294967295,4294967295,4294967295,4294967295]
-; AVX512F-NEXT:    vpminsq %zmm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX512F-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm0
-; AVX512F-NEXT:    vpmovqd %zmm0, %ymm0
+; AVX512F-NEXT:    vpmovusqd %zmm0, %ymm0
 ; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
@@ -246,11 +545,9 @@ define <4 x i32> @trunc_packus_v4i64_v4i32(<4 x i64> %a0) {
 ; AVX512BW-LABEL: trunc_packus_v4i64_v4i32:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; AVX512BW-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [4294967295,4294967295,4294967295,4294967295]
-; AVX512BW-NEXT:    vpminsq %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX512BW-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpmovqd %zmm0, %ymm0
+; AVX512BW-NEXT:    vpmovusqd %zmm0, %ymm0
 ; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -262,6 +559,14 @@ define <4 x i32> @trunc_packus_v4i64_v4i32(<4 x i64> %a0) {
 ; AVX512BWVL-NEXT:    vpmovusqd %ymm0, %xmm0
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
+;
+; SKX-LABEL: trunc_packus_v4i64_v4i32:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; SKX-NEXT:    vpmaxsq %ymm1, %ymm0, %ymm0
+; SKX-NEXT:    vpmovusqd %ymm0, %xmm0
+; SKX-NEXT:    vzeroupper
+; SKX-NEXT:    retq
   %1 = icmp slt <4 x i64> %a0, <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>
   %2 = select <4 x i1> %1, <4 x i64> %a0, <4 x i64> <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>
   %3 = icmp sgt <4 x i64> %2, zeroinitializer
@@ -271,343 +576,354 @@ define <4 x i32> @trunc_packus_v4i64_v4i32(<4 x i64> %a0) {
 }
 
 
-define <8 x i32> @trunc_packus_v8i64_v8i32(<8 x i64> %a0) {
+define <8 x i32> @trunc_packus_v8i64_v8i32(<8 x i64>* %p0) "min-legal-vector-width"="256" {
 ; SSE2-LABEL: trunc_packus_v8i64_v8i32:
 ; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa (%rdi), %xmm3
+; SSE2-NEXT:    movdqa 16(%rdi), %xmm7
+; SSE2-NEXT:    movdqa 32(%rdi), %xmm6
+; SSE2-NEXT:    movdqa 48(%rdi), %xmm9
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [4294967295,4294967295]
-; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [2147483648,2147483648]
-; SSE2-NEXT:    movdqa %xmm0, %xmm5
-; SSE2-NEXT:    pxor %xmm10, %xmm5
-; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [2147483647,2147483647]
-; SSE2-NEXT:    movdqa %xmm9, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm5, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm5[1,1,3,3]
-; SSE2-NEXT:    pand %xmm7, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm6[1,1,3,3]
-; SSE2-NEXT:    por %xmm4, %xmm5
-; SSE2-NEXT:    pand %xmm5, %xmm0
-; SSE2-NEXT:    pandn %xmm8, %xmm5
-; SSE2-NEXT:    por %xmm0, %xmm5
-; SSE2-NEXT:    movdqa %xmm1, %xmm0
-; SSE2-NEXT:    pxor %xmm10, %xmm0
-; SSE2-NEXT:    movdqa %xmm9, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm7, %xmm0
-; SSE2-NEXT:    pand %xmm0, %xmm1
-; SSE2-NEXT:    pandn %xmm8, %xmm0
-; SSE2-NEXT:    por %xmm1, %xmm0
-; SSE2-NEXT:    movdqa %xmm2, %xmm1
-; SSE2-NEXT:    pxor %xmm10, %xmm1
-; SSE2-NEXT:    movdqa %xmm9, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm6
-; SSE2-NEXT:    pand %xmm6, %xmm2
-; SSE2-NEXT:    pandn %xmm8, %xmm6
-; SSE2-NEXT:    por %xmm2, %xmm6
-; SSE2-NEXT:    movdqa %xmm3, %xmm1
-; SSE2-NEXT:    pxor %xmm10, %xmm1
-; SSE2-NEXT:    movdqa %xmm9, %xmm2
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pand %xmm4, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm2
+; SSE2-NEXT:    movdqa {{.*#+}} xmm11 = [2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm3, %xmm2
+; SSE2-NEXT:    pxor %xmm11, %xmm2
+; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [2147483647,2147483647]
+; SSE2-NEXT:    movdqa %xmm10, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm5[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,1,3,3]
+; SSE2-NEXT:    pand %xmm1, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm5[1,1,3,3]
+; SSE2-NEXT:    por %xmm4, %xmm2
 ; SSE2-NEXT:    pand %xmm2, %xmm3
 ; SSE2-NEXT:    pandn %xmm8, %xmm2
 ; SSE2-NEXT:    por %xmm3, %xmm2
-; SSE2-NEXT:    movdqa %xmm2, %xmm1
-; SSE2-NEXT:    pxor %xmm10, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm3
-; SSE2-NEXT:    pcmpgtd %xmm10, %xmm3
+; SSE2-NEXT:    movdqa %xmm7, %xmm1
+; SSE2-NEXT:    pxor %xmm11, %xmm1
+; SSE2-NEXT:    movdqa %xmm10, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
 ; SSE2-NEXT:    pcmpeqd %xmm10, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm4, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
 ; SSE2-NEXT:    por %xmm1, %xmm3
-; SSE2-NEXT:    pand %xmm2, %xmm3
+; SSE2-NEXT:    pand %xmm3, %xmm7
+; SSE2-NEXT:    pandn %xmm8, %xmm3
+; SSE2-NEXT:    por %xmm7, %xmm3
 ; SSE2-NEXT:    movdqa %xmm6, %xmm1
-; SSE2-NEXT:    pxor %xmm10, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm2
-; SSE2-NEXT:    pcmpgtd %xmm10, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
+; SSE2-NEXT:    pxor %xmm11, %xmm1
+; SSE2-NEXT:    movdqa %xmm10, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
 ; SSE2-NEXT:    pcmpeqd %xmm10, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pand %xmm4, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,3,3]
-; SSE2-NEXT:    por %xmm7, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pand %xmm5, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm1, %xmm7
+; SSE2-NEXT:    pand %xmm7, %xmm6
+; SSE2-NEXT:    pandn %xmm8, %xmm7
+; SSE2-NEXT:    por %xmm6, %xmm7
+; SSE2-NEXT:    movdqa %xmm9, %xmm1
+; SSE2-NEXT:    pxor %xmm11, %xmm1
+; SSE2-NEXT:    movdqa %xmm10, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pand %xmm5, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm1, %xmm4
+; SSE2-NEXT:    pand %xmm4, %xmm9
+; SSE2-NEXT:    pandn %xmm8, %xmm4
+; SSE2-NEXT:    por %xmm9, %xmm4
+; SSE2-NEXT:    movdqa %xmm4, %xmm1
+; SSE2-NEXT:    pxor %xmm11, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm11, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm11, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm6, %xmm1
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,2],xmm3[0,2]
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    pxor %xmm10, %xmm2
-; SSE2-NEXT:    movdqa %xmm2, %xmm3
-; SSE2-NEXT:    pcmpgtd %xmm10, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm10, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT:    pand %xmm4, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE2-NEXT:    por %xmm2, %xmm3
-; SSE2-NEXT:    pand %xmm0, %xmm3
-; SSE2-NEXT:    movdqa %xmm5, %xmm0
-; SSE2-NEXT:    pxor %xmm10, %xmm0
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd %xmm10, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm10, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pand %xmm4, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3]
-; SSE2-NEXT:    por %xmm6, %xmm0
-; SSE2-NEXT:    pand %xmm5, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm3[0,2]
-; SSE2-NEXT:    retq
-;
-; SSSE3-LABEL: trunc_packus_v8i64_v8i32:
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSE2-NEXT:    por %xmm1, %xmm5
+; SSE2-NEXT:    pand %xmm4, %xmm5
+; SSE2-NEXT:    movdqa %xmm7, %xmm1
+; SSE2-NEXT:    pxor %xmm11, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm11, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm11, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pand %xmm6, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm0, %xmm1
+; SSE2-NEXT:    pand %xmm7, %xmm1
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,2],xmm5[0,2]
+; SSE2-NEXT:    movdqa %xmm3, %xmm0
+; SSE2-NEXT:    pxor %xmm11, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm11, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm11, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pand %xmm5, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm0, %xmm4
+; SSE2-NEXT:    pand %xmm3, %xmm4
+; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    pxor %xmm11, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm11, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm11, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pand %xmm5, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
+; SSE2-NEXT:    por %xmm6, %xmm0
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm4[0,2]
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: trunc_packus_v8i64_v8i32:
 ; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa (%rdi), %xmm3
+; SSSE3-NEXT:    movdqa 16(%rdi), %xmm7
+; SSSE3-NEXT:    movdqa 32(%rdi), %xmm6
+; SSSE3-NEXT:    movdqa 48(%rdi), %xmm9
 ; SSSE3-NEXT:    movdqa {{.*#+}} xmm8 = [4294967295,4294967295]
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm10 = [2147483648,2147483648]
-; SSSE3-NEXT:    movdqa %xmm0, %xmm5
-; SSSE3-NEXT:    pxor %xmm10, %xmm5
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm9 = [2147483647,2147483647]
-; SSSE3-NEXT:    movdqa %xmm9, %xmm6
-; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm5
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm5[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm7, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm6[1,1,3,3]
-; SSSE3-NEXT:    por %xmm4, %xmm5
-; SSSE3-NEXT:    pand %xmm5, %xmm0
-; SSSE3-NEXT:    pandn %xmm8, %xmm5
-; SSSE3-NEXT:    por %xmm0, %xmm5
-; SSSE3-NEXT:    movdqa %xmm1, %xmm0
-; SSSE3-NEXT:    pxor %xmm10, %xmm0
-; SSSE3-NEXT:    movdqa %xmm9, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm6, %xmm7
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    por %xmm7, %xmm0
-; SSSE3-NEXT:    pand %xmm0, %xmm1
-; SSSE3-NEXT:    pandn %xmm8, %xmm0
-; SSSE3-NEXT:    por %xmm1, %xmm0
-; SSSE3-NEXT:    movdqa %xmm2, %xmm1
-; SSSE3-NEXT:    pxor %xmm10, %xmm1
-; SSSE3-NEXT:    movdqa %xmm9, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm6, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    por %xmm1, %xmm6
-; SSSE3-NEXT:    pand %xmm6, %xmm2
-; SSSE3-NEXT:    pandn %xmm8, %xmm6
-; SSSE3-NEXT:    por %xmm2, %xmm6
-; SSSE3-NEXT:    movdqa %xmm3, %xmm1
-; SSSE3-NEXT:    pxor %xmm10, %xmm1
-; SSSE3-NEXT:    movdqa %xmm9, %xmm2
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm4, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSSE3-NEXT:    por %xmm1, %xmm2
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm11 = [2147483648,2147483648]
+; SSSE3-NEXT:    movdqa %xmm3, %xmm2
+; SSSE3-NEXT:    pxor %xmm11, %xmm2
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm10 = [2147483647,2147483647]
+; SSSE3-NEXT:    movdqa %xmm10, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm5[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm1, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm5[1,1,3,3]
+; SSSE3-NEXT:    por %xmm4, %xmm2
 ; SSSE3-NEXT:    pand %xmm2, %xmm3
 ; SSSE3-NEXT:    pandn %xmm8, %xmm2
 ; SSSE3-NEXT:    por %xmm3, %xmm2
-; SSSE3-NEXT:    movdqa %xmm2, %xmm1
-; SSSE3-NEXT:    pxor %xmm10, %xmm1
-; SSSE3-NEXT:    movdqa %xmm1, %xmm3
-; SSSE3-NEXT:    pcmpgtd %xmm10, %xmm3
+; SSSE3-NEXT:    movdqa %xmm7, %xmm1
+; SSSE3-NEXT:    pxor %xmm11, %xmm1
+; SSSE3-NEXT:    movdqa %xmm10, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm3
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
 ; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm1
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
 ; SSSE3-NEXT:    pand %xmm4, %xmm1
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
 ; SSSE3-NEXT:    por %xmm1, %xmm3
-; SSSE3-NEXT:    pand %xmm2, %xmm3
+; SSSE3-NEXT:    pand %xmm3, %xmm7
+; SSSE3-NEXT:    pandn %xmm8, %xmm3
+; SSSE3-NEXT:    por %xmm7, %xmm3
 ; SSSE3-NEXT:    movdqa %xmm6, %xmm1
-; SSSE3-NEXT:    pxor %xmm10, %xmm1
-; SSSE3-NEXT:    movdqa %xmm1, %xmm2
-; SSSE3-NEXT:    pcmpgtd %xmm10, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
+; SSSE3-NEXT:    pxor %xmm11, %xmm1
+; SSSE3-NEXT:    movdqa %xmm10, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
 ; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm4, %xmm7
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,3,3]
-; SSSE3-NEXT:    por %xmm7, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm5, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm1, %xmm7
+; SSSE3-NEXT:    pand %xmm7, %xmm6
+; SSSE3-NEXT:    pandn %xmm8, %xmm7
+; SSSE3-NEXT:    por %xmm6, %xmm7
+; SSSE3-NEXT:    movdqa %xmm9, %xmm1
+; SSSE3-NEXT:    pxor %xmm11, %xmm1
+; SSSE3-NEXT:    movdqa %xmm10, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm5, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm1, %xmm4
+; SSSE3-NEXT:    pand %xmm4, %xmm9
+; SSSE3-NEXT:    pandn %xmm8, %xmm4
+; SSSE3-NEXT:    por %xmm9, %xmm4
+; SSSE3-NEXT:    movdqa %xmm4, %xmm1
+; SSSE3-NEXT:    pxor %xmm11, %xmm1
+; SSSE3-NEXT:    movdqa %xmm1, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm11, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm11, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
 ; SSSE3-NEXT:    pand %xmm6, %xmm1
-; SSSE3-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,2],xmm3[0,2]
-; SSSE3-NEXT:    movdqa %xmm0, %xmm2
-; SSSE3-NEXT:    pxor %xmm10, %xmm2
-; SSSE3-NEXT:    movdqa %xmm2, %xmm3
-; SSSE3-NEXT:    pcmpgtd %xmm10, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm4, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm3
-; SSSE3-NEXT:    pand %xmm0, %xmm3
-; SSSE3-NEXT:    movdqa %xmm5, %xmm0
-; SSSE3-NEXT:    pxor %xmm10, %xmm0
-; SSSE3-NEXT:    movdqa %xmm0, %xmm2
-; SSSE3-NEXT:    pcmpgtd %xmm10, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSSE3-NEXT:    por %xmm1, %xmm5
+; SSSE3-NEXT:    pand %xmm4, %xmm5
+; SSSE3-NEXT:    movdqa %xmm7, %xmm1
+; SSSE3-NEXT:    pxor %xmm11, %xmm1
+; SSSE3-NEXT:    movdqa %xmm1, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm11, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm11, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm6, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm0, %xmm1
+; SSSE3-NEXT:    pand %xmm7, %xmm1
+; SSSE3-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,2],xmm5[0,2]
+; SSSE3-NEXT:    movdqa %xmm3, %xmm0
+; SSSE3-NEXT:    pxor %xmm11, %xmm0
+; SSSE3-NEXT:    movdqa %xmm0, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm11, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm11, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm5, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm0, %xmm4
+; SSSE3-NEXT:    pand %xmm3, %xmm4
+; SSSE3-NEXT:    movdqa %xmm2, %xmm0
+; SSSE3-NEXT:    pxor %xmm11, %xmm0
+; SSSE3-NEXT:    movdqa %xmm0, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm11, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm11, %xmm0
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm4, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm5, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
 ; SSSE3-NEXT:    por %xmm6, %xmm0
-; SSSE3-NEXT:    pand %xmm5, %xmm0
-; SSSE3-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm3[0,2]
+; SSSE3-NEXT:    pand %xmm2, %xmm0
+; SSSE3-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm4[0,2]
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: trunc_packus_v8i64_v8i32:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa %xmm0, %xmm4
-; SSE41-NEXT:    movapd {{.*#+}} xmm7 = [4294967295,4294967295]
-; SSE41-NEXT:    movdqa {{.*#+}} xmm10 = [2147483648,2147483648]
-; SSE41-NEXT:    pxor %xmm10, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm11 = [2147483647,2147483647]
-; SSE41-NEXT:    movdqa %xmm11, %xmm6
+; SSE41-NEXT:    movdqa (%rdi), %xmm5
+; SSE41-NEXT:    movdqa 16(%rdi), %xmm4
+; SSE41-NEXT:    movdqa 32(%rdi), %xmm10
+; SSE41-NEXT:    movdqa 48(%rdi), %xmm9
+; SSE41-NEXT:    movapd {{.*#+}} xmm1 = [4294967295,4294967295]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [2147483648,2147483648]
+; SSE41-NEXT:    movdqa %xmm5, %xmm0
+; SSE41-NEXT:    pxor %xmm3, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [2147483647,2147483647]
+; SSE41-NEXT:    movdqa %xmm2, %xmm7
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm7
+; SSE41-NEXT:    movdqa %xmm2, %xmm6
 ; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm6[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm11, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
+; SSE41-NEXT:    pand %xmm7, %xmm0
 ; SSE41-NEXT:    por %xmm6, %xmm0
-; SSE41-NEXT:    movapd %xmm7, %xmm8
-; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm8
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    pxor %xmm10, %xmm0
-; SSE41-NEXT:    movdqa %xmm11, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm11, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE41-NEXT:    movapd %xmm1, %xmm8
+; SSE41-NEXT:    blendvpd %xmm0, %xmm5, %xmm8
+; SSE41-NEXT:    movdqa %xmm4, %xmm0
+; SSE41-NEXT:    pxor %xmm3, %xmm0
+; SSE41-NEXT:    movdqa %xmm2, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm5
+; SSE41-NEXT:    movdqa %xmm2, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
 ; SSE41-NEXT:    pand %xmm5, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
-; SSE41-NEXT:    movapd %xmm7, %xmm9
-; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm9
-; SSE41-NEXT:    movdqa %xmm2, %xmm0
-; SSE41-NEXT:    pxor %xmm10, %xmm0
-; SSE41-NEXT:    movdqa %xmm11, %xmm1
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm11, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE41-NEXT:    por %xmm6, %xmm0
+; SSE41-NEXT:    movapd %xmm1, %xmm5
+; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm5
+; SSE41-NEXT:    movdqa %xmm10, %xmm0
+; SSE41-NEXT:    pxor %xmm3, %xmm0
+; SSE41-NEXT:    movdqa %xmm2, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT:    movdqa %xmm2, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
 ; SSE41-NEXT:    pand %xmm4, %xmm0
-; SSE41-NEXT:    por %xmm1, %xmm0
-; SSE41-NEXT:    movapd %xmm7, %xmm4
-; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm4
-; SSE41-NEXT:    movdqa %xmm3, %xmm0
-; SSE41-NEXT:    pxor %xmm10, %xmm0
-; SSE41-NEXT:    movdqa %xmm11, %xmm1
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm11, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm2, %xmm0
-; SSE41-NEXT:    por %xmm1, %xmm0
-; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm7
+; SSE41-NEXT:    por %xmm6, %xmm0
+; SSE41-NEXT:    movapd %xmm1, %xmm4
+; SSE41-NEXT:    blendvpd %xmm0, %xmm10, %xmm4
+; SSE41-NEXT:    movdqa %xmm9, %xmm0
+; SSE41-NEXT:    pxor %xmm3, %xmm0
+; SSE41-NEXT:    movdqa %xmm2, %xmm6
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,0,2,2]
+; SSE41-NEXT:    pand %xmm6, %xmm0
+; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm9, %xmm1
 ; SSE41-NEXT:    pxor %xmm2, %xmm2
-; SSE41-NEXT:    movapd %xmm7, %xmm0
-; SSE41-NEXT:    xorpd %xmm10, %xmm0
-; SSE41-NEXT:    movapd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd %xmm10, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm3, %xmm0
-; SSE41-NEXT:    por %xmm1, %xmm0
-; SSE41-NEXT:    pxor %xmm3, %xmm3
-; SSE41-NEXT:    blendvpd %xmm0, %xmm7, %xmm3
-; SSE41-NEXT:    movapd %xmm4, %xmm0
-; SSE41-NEXT:    xorpd %xmm10, %xmm0
-; SSE41-NEXT:    movapd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd %xmm10, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm0
+; SSE41-NEXT:    movapd %xmm1, %xmm6
+; SSE41-NEXT:    xorpd %xmm3, %xmm6
+; SSE41-NEXT:    movapd %xmm6, %xmm7
+; SSE41-NEXT:    pcmpeqd %xmm3, %xmm7
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
+; SSE41-NEXT:    pand %xmm7, %xmm0
+; SSE41-NEXT:    por %xmm6, %xmm0
+; SSE41-NEXT:    pxor %xmm6, %xmm6
+; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm6
+; SSE41-NEXT:    movapd %xmm4, %xmm1
+; SSE41-NEXT:    xorpd %xmm3, %xmm1
+; SSE41-NEXT:    movapd %xmm1, %xmm7
+; SSE41-NEXT:    pcmpeqd %xmm3, %xmm7
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2]
+; SSE41-NEXT:    pand %xmm7, %xmm0
 ; SSE41-NEXT:    por %xmm1, %xmm0
 ; SSE41-NEXT:    pxor %xmm1, %xmm1
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm1
-; SSE41-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,2],xmm3[0,2]
-; SSE41-NEXT:    movapd %xmm9, %xmm0
-; SSE41-NEXT:    xorpd %xmm10, %xmm0
-; SSE41-NEXT:    movapd %xmm0, %xmm3
-; SSE41-NEXT:    pcmpgtd %xmm10, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm4, %xmm0
-; SSE41-NEXT:    por %xmm3, %xmm0
-; SSE41-NEXT:    pxor %xmm3, %xmm3
-; SSE41-NEXT:    blendvpd %xmm0, %xmm9, %xmm3
-; SSE41-NEXT:    movapd %xmm8, %xmm0
-; SSE41-NEXT:    xorpd %xmm10, %xmm0
-; SSE41-NEXT:    movapd %xmm0, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm10, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm0
+; SSE41-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,2],xmm6[0,2]
+; SSE41-NEXT:    movapd %xmm5, %xmm4
+; SSE41-NEXT:    xorpd %xmm3, %xmm4
+; SSE41-NEXT:    movapd %xmm4, %xmm6
+; SSE41-NEXT:    pcmpeqd %xmm3, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pand %xmm6, %xmm0
 ; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    pxor %xmm4, %xmm4
+; SSE41-NEXT:    blendvpd %xmm0, %xmm5, %xmm4
+; SSE41-NEXT:    movapd %xmm8, %xmm5
+; SSE41-NEXT:    xorpd %xmm3, %xmm5
+; SSE41-NEXT:    movapd %xmm5, %xmm6
+; SSE41-NEXT:    pcmpeqd %xmm3, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm5
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2]
+; SSE41-NEXT:    pand %xmm6, %xmm0
+; SSE41-NEXT:    por %xmm5, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm8, %xmm2
-; SSE41-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,2],xmm3[0,2]
+; SSE41-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,2],xmm4[0,2]
 ; SSE41-NEXT:    movaps %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: trunc_packus_v8i64_v8i32:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [4294967295,4294967295]
-; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm3, %xmm8
-; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm3, %xmm5
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm6
-; AVX1-NEXT:    vpcmpgtq %xmm6, %xmm3, %xmm7
-; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm3, %xmm4
-; AVX1-NEXT:    vblendvpd %xmm4, %xmm0, %xmm3, %xmm0
-; AVX1-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm0, %xmm9
-; AVX1-NEXT:    vblendvpd %xmm7, %xmm6, %xmm3, %xmm6
-; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm6, %xmm7
-; AVX1-NEXT:    vblendvpd %xmm5, %xmm1, %xmm3, %xmm1
-; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm1, %xmm5
-; AVX1-NEXT:    vblendvpd %xmm8, %xmm2, %xmm3, %xmm2
-; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm2, %xmm3
-; AVX1-NEXT:    vpand %xmm2, %xmm3, %xmm2
-; AVX1-NEXT:    vpand %xmm1, %xmm5, %xmm1
-; AVX1-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[0,2],xmm2[0,2]
-; AVX1-NEXT:    vpand %xmm6, %xmm7, %xmm2
+; AVX1-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX1-NEXT:    vmovdqa 16(%rdi), %xmm1
+; AVX1-NEXT:    vmovdqa 32(%rdi), %xmm2
+; AVX1-NEXT:    vmovdqa 48(%rdi), %xmm3
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [4294967295,4294967295]
+; AVX1-NEXT:    vpcmpgtq %xmm3, %xmm4, %xmm8
+; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm4, %xmm6
+; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm4, %xmm7
+; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm4, %xmm5
+; AVX1-NEXT:    vblendvpd %xmm5, %xmm0, %xmm4, %xmm0
+; AVX1-NEXT:    vpxor %xmm5, %xmm5, %xmm5
+; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm0, %xmm9
+; AVX1-NEXT:    vblendvpd %xmm7, %xmm1, %xmm4, %xmm1
+; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm1, %xmm7
+; AVX1-NEXT:    vblendvpd %xmm6, %xmm2, %xmm4, %xmm2
+; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm2, %xmm6
+; AVX1-NEXT:    vblendvpd %xmm8, %xmm3, %xmm4, %xmm3
+; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm3, %xmm4
+; AVX1-NEXT:    vpand %xmm3, %xmm4, %xmm3
+; AVX1-NEXT:    vpand %xmm2, %xmm6, %xmm2
+; AVX1-NEXT:    vshufps {{.*#+}} xmm2 = xmm2[0,2],xmm3[0,2]
+; AVX1-NEXT:    vpand %xmm1, %xmm7, %xmm1
 ; AVX1-NEXT:    vpand %xmm0, %xmm9, %xmm0
-; AVX1-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm2[0,2]
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-SLOW-LABEL: trunc_packus_v8i64_v8i32:
 ; AVX2-SLOW:       # %bb.0:
+; AVX2-SLOW-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX2-SLOW-NEXT:    vmovdqa 32(%rdi), %ymm1
 ; AVX2-SLOW-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [4294967295,4294967295,4294967295,4294967295]
 ; AVX2-SLOW-NEXT:    vpcmpgtq %ymm1, %ymm2, %ymm3
 ; AVX2-SLOW-NEXT:    vblendvpd %ymm3, %ymm1, %ymm2, %ymm1
@@ -627,6 +943,8 @@ define <8 x i32> @trunc_packus_v8i64_v8i32(<8 x i64> %a0) {
 ;
 ; AVX2-FAST-LABEL: trunc_packus_v8i64_v8i32:
 ; AVX2-FAST:       # %bb.0:
+; AVX2-FAST-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX2-FAST-NEXT:    vmovdqa 32(%rdi), %ymm1
 ; AVX2-FAST-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [4294967295,4294967295,4294967295,4294967295]
 ; AVX2-FAST-NEXT:    vpcmpgtq %ymm0, %ymm2, %ymm3
 ; AVX2-FAST-NEXT:    vblendvpd %ymm3, %ymm0, %ymm2, %ymm0
@@ -645,10 +963,21 @@ define <8 x i32> @trunc_packus_v8i64_v8i32(<8 x i64> %a0) {
 ;
 ; AVX512-LABEL: trunc_packus_v8i64_v8i32:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; AVX512-NEXT:    vpmaxsq (%rdi), %zmm0, %zmm0
 ; AVX512-NEXT:    vpmovusqd %zmm0, %ymm0
 ; AVX512-NEXT:    retq
+;
+; SKX-LABEL: trunc_packus_v8i64_v8i32:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; SKX-NEXT:    vpmaxsq (%rdi), %ymm0, %ymm1
+; SKX-NEXT:    vpmovusqd %ymm1, %xmm1
+; SKX-NEXT:    vpmaxsq 32(%rdi), %ymm0, %ymm0
+; SKX-NEXT:    vpmovusqd %ymm0, %xmm0
+; SKX-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
+; SKX-NEXT:    retq
+  %a0 = load <8 x i64>, <8 x i64>* %p0
   %1 = icmp slt <8 x i64> %a0, <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>
   %2 = select <8 x i1> %1, <8 x i64> %a0, <8 x i64> <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>
   %3 = icmp sgt <8 x i64> %2, zeroinitializer
@@ -661,1018 +990,2989 @@ define <8 x i32> @trunc_packus_v8i64_v8i32(<8 x i64> %a0) {
 ; PACKUS saturation truncation to vXi16
 ;
 
-define <8 x i16> @trunc_packus_v8i64_v8i16(<8 x i64> %a0) {
-; SSE2-LABEL: trunc_packus_v8i64_v8i16:
+define <2 x i16> @trunc_packus_v2i64_v2i16(<2 x i64> %a0) {
+; SSE2-LABEL: trunc_packus_v2i64_v2i16:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [65535,65535]
-; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [2147483648,2147483648]
-; SSE2-NEXT:    movdqa %xmm1, %xmm5
-; SSE2-NEXT:    pxor %xmm10, %xmm5
-; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [2147549183,2147549183]
-; SSE2-NEXT:    movdqa %xmm9, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm5, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm5[1,1,3,3]
-; SSE2-NEXT:    pand %xmm7, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm6[1,1,3,3]
-; SSE2-NEXT:    por %xmm4, %xmm5
-; SSE2-NEXT:    pand %xmm5, %xmm1
-; SSE2-NEXT:    pandn %xmm8, %xmm5
-; SSE2-NEXT:    por %xmm1, %xmm5
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    pxor %xmm10, %xmm1
-; SSE2-NEXT:    movdqa %xmm9, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm7, %xmm1
-; SSE2-NEXT:    pand %xmm1, %xmm0
-; SSE2-NEXT:    pandn %xmm8, %xmm1
-; SSE2-NEXT:    por %xmm0, %xmm1
-; SSE2-NEXT:    movdqa %xmm3, %xmm0
-; SSE2-NEXT:    pxor %xmm10, %xmm0
-; SSE2-NEXT:    movdqa %xmm9, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm0, %xmm6
-; SSE2-NEXT:    pand %xmm6, %xmm3
-; SSE2-NEXT:    pandn %xmm8, %xmm6
-; SSE2-NEXT:    por %xmm3, %xmm6
-; SSE2-NEXT:    movdqa %xmm2, %xmm0
-; SSE2-NEXT:    pxor %xmm10, %xmm0
-; SSE2-NEXT:    movdqa %xmm9, %xmm3
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pand %xmm4, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE2-NEXT:    por %xmm0, %xmm3
-; SSE2-NEXT:    pand %xmm3, %xmm2
-; SSE2-NEXT:    pandn %xmm8, %xmm3
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    pxor %xmm1, %xmm2
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [2147549183,2147549183]
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm3, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE2-NEXT:    pand %xmm5, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
 ; SSE2-NEXT:    por %xmm2, %xmm3
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm3
+; SSE2-NEXT:    por %xmm0, %xmm3
 ; SSE2-NEXT:    movdqa %xmm3, %xmm0
-; SSE2-NEXT:    pxor %xmm10, %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm0
 ; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd %xmm10, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm10, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pand %xmm4, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3]
-; SSE2-NEXT:    por %xmm7, %xmm0
-; SSE2-NEXT:    pand %xmm3, %xmm0
-; SSE2-NEXT:    movdqa %xmm6, %xmm2
-; SSE2-NEXT:    pxor %xmm10, %xmm2
-; SSE2-NEXT:    movdqa %xmm2, %xmm3
-; SSE2-NEXT:    pcmpgtd %xmm10, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm10, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm2[1,1,3,3]
-; SSE2-NEXT:    pand %xmm4, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,1,3,3]
-; SSE2-NEXT:    por %xmm7, %xmm2
-; SSE2-NEXT:    pand %xmm6, %xmm2
-; SSE2-NEXT:    movdqa %xmm1, %xmm3
-; SSE2-NEXT:    pxor %xmm10, %xmm3
-; SSE2-NEXT:    movdqa %xmm3, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm10, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm10, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm4
-; SSE2-NEXT:    pand %xmm1, %xmm4
-; SSE2-NEXT:    movdqa %xmm5, %xmm1
-; SSE2-NEXT:    pxor %xmm10, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm3
-; SSE2-NEXT:    pcmpgtd %xmm10, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm10, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm3
-; SSE2-NEXT:    pand %xmm5, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[0,2,2,3]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[0,2,2,3]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm3[0,2,2,3,4,5,6,7]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,1,0,2,4,5,6,7]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,1,0,2,4,5,6,7]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm3[0],xmm0[1]
+; SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pand %xmm4, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,3,3]
+; SSE2-NEXT:    por %xmm0, %xmm1
+; SSE2-NEXT:    pand %xmm3, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
 ; SSE2-NEXT:    retq
 ;
-; SSSE3-LABEL: trunc_packus_v8i64_v8i16:
+; SSSE3-LABEL: trunc_packus_v2i64_v2i16:
 ; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm8 = [65535,65535]
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm10 = [2147483648,2147483648]
-; SSSE3-NEXT:    movdqa %xmm1, %xmm5
-; SSSE3-NEXT:    pxor %xmm10, %xmm5
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm9 = [2147549183,2147549183]
-; SSSE3-NEXT:    movdqa %xmm9, %xmm6
-; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm5
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm5[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm7, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm6[1,1,3,3]
-; SSSE3-NEXT:    por %xmm4, %xmm5
-; SSSE3-NEXT:    pand %xmm5, %xmm1
-; SSSE3-NEXT:    pandn %xmm8, %xmm5
-; SSSE3-NEXT:    por %xmm1, %xmm5
-; SSSE3-NEXT:    movdqa %xmm0, %xmm1
-; SSSE3-NEXT:    pxor %xmm10, %xmm1
-; SSSE3-NEXT:    movdqa %xmm9, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm6, %xmm7
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    por %xmm7, %xmm1
-; SSSE3-NEXT:    pand %xmm1, %xmm0
-; SSSE3-NEXT:    pandn %xmm8, %xmm1
-; SSSE3-NEXT:    por %xmm0, %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [2147483648,2147483648]
+; SSSE3-NEXT:    movdqa %xmm0, %xmm2
+; SSSE3-NEXT:    pxor %xmm1, %xmm2
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm3 = [2147549183,2147549183]
+; SSSE3-NEXT:    movdqa %xmm3, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm5, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm2, %xmm3
+; SSSE3-NEXT:    pand %xmm3, %xmm0
+; SSSE3-NEXT:    pandn {{.*}}(%rip), %xmm3
+; SSSE3-NEXT:    por %xmm0, %xmm3
 ; SSSE3-NEXT:    movdqa %xmm3, %xmm0
-; SSSE3-NEXT:    pxor %xmm10, %xmm0
-; SSSE3-NEXT:    movdqa %xmm9, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm6, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    por %xmm0, %xmm6
-; SSSE3-NEXT:    pand %xmm6, %xmm3
-; SSSE3-NEXT:    pandn %xmm8, %xmm6
-; SSSE3-NEXT:    por %xmm3, %xmm6
-; SSSE3-NEXT:    movdqa %xmm2, %xmm0
-; SSSE3-NEXT:    pxor %xmm10, %xmm0
-; SSSE3-NEXT:    movdqa %xmm9, %xmm3
-; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm0
+; SSSE3-NEXT:    pxor %xmm1, %xmm0
+; SSSE3-NEXT:    movdqa %xmm0, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm0
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
 ; SSSE3-NEXT:    pand %xmm4, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm0, %xmm3
-; SSSE3-NEXT:    pand %xmm3, %xmm2
-; SSSE3-NEXT:    pandn %xmm8, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    por %xmm0, %xmm1
+; SSSE3-NEXT:    pand %xmm3, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
+; SSSE3-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: trunc_packus_v2i64_v2i16:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    movapd {{.*#+}} xmm2 = [65535,65535]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [2147483648,2147483648]
+; SSE41-NEXT:    pxor %xmm3, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [2147549183,2147549183]
+; SSE41-NEXT:    movdqa %xmm4, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm0
+; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
+; SSE41-NEXT:    xorpd %xmm1, %xmm1
+; SSE41-NEXT:    movapd %xmm2, %xmm4
+; SSE41-NEXT:    xorpd %xmm3, %xmm4
+; SSE41-NEXT:    movapd %xmm4, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm3, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm0
+; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
+; SSE41-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: trunc_packus_v2i64_v2i16:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [65535,65535]
+; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm2
+; AVX1-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vpand %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; AVX1-NEXT:    retq
+;
+; AVX2-SLOW-LABEL: trunc_packus_v2i64_v2i16:
+; AVX2-SLOW:       # %bb.0:
+; AVX2-SLOW-NEXT:    vmovdqa {{.*#+}} xmm1 = [65535,65535]
+; AVX2-SLOW-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm2
+; AVX2-SLOW-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX2-SLOW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX2-SLOW-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm1
+; AVX2-SLOW-NEXT:    vpand %xmm0, %xmm1, %xmm0
+; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; AVX2-SLOW-NEXT:    retq
+;
+; AVX2-FAST-LABEL: trunc_packus_v2i64_v2i16:
+; AVX2-FAST:       # %bb.0:
+; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} xmm1 = [65535,65535]
+; AVX2-FAST-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm2
+; AVX2-FAST-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX2-FAST-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX2-FAST-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm1
+; AVX2-FAST-NEXT:    vpand %xmm0, %xmm1, %xmm0
+; AVX2-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,8,9,8,9,10,11,8,9,10,11,12,13,14,15]
+; AVX2-FAST-NEXT:    retq
+;
+; AVX512F-LABEL: trunc_packus_v2i64_v2i16:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vpmovusqw %zmm0, %xmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: trunc_packus_v2i64_v2i16:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512VL-NEXT:    vpmaxsq %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpmovusqw %xmm0, %xmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: trunc_packus_v2i64_v2i16:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512BW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512BW-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmovusqw %zmm0, %xmm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: trunc_packus_v2i64_v2i16:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512BWVL-NEXT:    vpmaxsq %xmm1, %xmm0, %xmm0
+; AVX512BWVL-NEXT:    vpmovusqw %xmm0, %xmm0
+; AVX512BWVL-NEXT:    retq
+;
+; SKX-LABEL: trunc_packus_v2i64_v2i16:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; SKX-NEXT:    vpmaxsq %xmm1, %xmm0, %xmm0
+; SKX-NEXT:    vpmovusqw %xmm0, %xmm0
+; SKX-NEXT:    retq
+  %1 = icmp slt <2 x i64> %a0, <i64 65535, i64 65535>
+  %2 = select <2 x i1> %1, <2 x i64> %a0, <2 x i64> <i64 65535, i64 65535>
+  %3 = icmp sgt <2 x i64> %2, zeroinitializer
+  %4 = select <2 x i1> %3, <2 x i64> %2, <2 x i64> zeroinitializer
+  %5 = trunc <2 x i64> %4 to <2 x i16>
+  ret <2 x i16> %5
+}
+
+define void @trunc_packus_v2i64_v2i16_store(<2 x i64> %a0, <2 x i16> *%p1) {
+; SSE2-LABEL: trunc_packus_v2i64_v2i16_store:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    pxor %xmm1, %xmm2
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [2147549183,2147549183]
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm3, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE2-NEXT:    pand %xmm5, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm2, %xmm3
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm3
+; SSE2-NEXT:    por %xmm0, %xmm3
+; SSE2-NEXT:    movdqa %xmm3, %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pand %xmm4, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,3,3]
+; SSE2-NEXT:    por %xmm0, %xmm1
+; SSE2-NEXT:    pand %xmm3, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; SSE2-NEXT:    movd %xmm0, (%rdi)
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: trunc_packus_v2i64_v2i16_store:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [2147483648,2147483648]
+; SSSE3-NEXT:    movdqa %xmm0, %xmm2
+; SSSE3-NEXT:    pxor %xmm1, %xmm2
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm3 = [2147549183,2147549183]
+; SSSE3-NEXT:    movdqa %xmm3, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm5, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
 ; SSSE3-NEXT:    por %xmm2, %xmm3
+; SSSE3-NEXT:    pand %xmm3, %xmm0
+; SSSE3-NEXT:    pandn {{.*}}(%rip), %xmm3
+; SSSE3-NEXT:    por %xmm0, %xmm3
 ; SSSE3-NEXT:    movdqa %xmm3, %xmm0
-; SSSE3-NEXT:    pxor %xmm10, %xmm0
+; SSSE3-NEXT:    pxor %xmm1, %xmm0
 ; SSSE3-NEXT:    movdqa %xmm0, %xmm2
-; SSSE3-NEXT:    pcmpgtd %xmm10, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm4, %xmm7
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3]
-; SSSE3-NEXT:    por %xmm7, %xmm0
-; SSSE3-NEXT:    pand %xmm3, %xmm0
-; SSSE3-NEXT:    movdqa %xmm6, %xmm2
-; SSSE3-NEXT:    pxor %xmm10, %xmm2
-; SSSE3-NEXT:    movdqa %xmm2, %xmm3
-; SSSE3-NEXT:    pcmpgtd %xmm10, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm2[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm4, %xmm7
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm7, %xmm2
-; SSSE3-NEXT:    pand %xmm6, %xmm2
+; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm4, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    por %xmm0, %xmm1
+; SSSE3-NEXT:    pand %xmm3, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
+; SSSE3-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; SSSE3-NEXT:    movd %xmm0, (%rdi)
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: trunc_packus_v2i64_v2i16_store:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    movapd {{.*#+}} xmm2 = [65535,65535]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [2147483648,2147483648]
+; SSE41-NEXT:    pxor %xmm3, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [2147549183,2147549183]
+; SSE41-NEXT:    movdqa %xmm4, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm0
+; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
+; SSE41-NEXT:    xorpd %xmm1, %xmm1
+; SSE41-NEXT:    movapd %xmm2, %xmm4
+; SSE41-NEXT:    xorpd %xmm3, %xmm4
+; SSE41-NEXT:    movapd %xmm4, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm3, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm0
+; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
+; SSE41-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; SSE41-NEXT:    movd %xmm0, (%rdi)
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: trunc_packus_v2i64_v2i16_store:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [65535,65535]
+; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm2
+; AVX1-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vpand %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; AVX1-NEXT:    vmovd %xmm0, (%rdi)
+; AVX1-NEXT:    retq
+;
+; AVX2-SLOW-LABEL: trunc_packus_v2i64_v2i16_store:
+; AVX2-SLOW:       # %bb.0:
+; AVX2-SLOW-NEXT:    vmovdqa {{.*#+}} xmm1 = [65535,65535]
+; AVX2-SLOW-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm2
+; AVX2-SLOW-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX2-SLOW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX2-SLOW-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm1
+; AVX2-SLOW-NEXT:    vpand %xmm0, %xmm1, %xmm0
+; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vmovd %xmm0, (%rdi)
+; AVX2-SLOW-NEXT:    retq
+;
+; AVX2-FAST-LABEL: trunc_packus_v2i64_v2i16_store:
+; AVX2-FAST:       # %bb.0:
+; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} xmm1 = [65535,65535]
+; AVX2-FAST-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm2
+; AVX2-FAST-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX2-FAST-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX2-FAST-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm1
+; AVX2-FAST-NEXT:    vpand %xmm0, %xmm1, %xmm0
+; AVX2-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,8,9,8,9,10,11,8,9,10,11,12,13,14,15]
+; AVX2-FAST-NEXT:    vmovd %xmm0, (%rdi)
+; AVX2-FAST-NEXT:    retq
+;
+; AVX512F-LABEL: trunc_packus_v2i64_v2i16_store:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vpmovusqw %zmm0, %xmm0
+; AVX512F-NEXT:    vmovd %xmm0, (%rdi)
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: trunc_packus_v2i64_v2i16_store:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512VL-NEXT:    vpmaxsq %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpmovusqw %xmm0, (%rdi)
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: trunc_packus_v2i64_v2i16_store:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512BW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512BW-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmovusqw %zmm0, %xmm0
+; AVX512BW-NEXT:    vmovd %xmm0, (%rdi)
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: trunc_packus_v2i64_v2i16_store:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512BWVL-NEXT:    vpmaxsq %xmm1, %xmm0, %xmm0
+; AVX512BWVL-NEXT:    vpmovusqw %xmm0, (%rdi)
+; AVX512BWVL-NEXT:    retq
+;
+; SKX-LABEL: trunc_packus_v2i64_v2i16_store:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; SKX-NEXT:    vpmaxsq %xmm1, %xmm0, %xmm0
+; SKX-NEXT:    vpmovusqw %xmm0, (%rdi)
+; SKX-NEXT:    retq
+  %1 = icmp slt <2 x i64> %a0, <i64 65535, i64 65535>
+  %2 = select <2 x i1> %1, <2 x i64> %a0, <2 x i64> <i64 65535, i64 65535>
+  %3 = icmp sgt <2 x i64> %2, zeroinitializer
+  %4 = select <2 x i1> %3, <2 x i64> %2, <2 x i64> zeroinitializer
+  %5 = trunc <2 x i64> %4 to <2 x i16>
+  store <2 x i16> %5, <2 x i16> *%p1
+  ret void
+}
+
+define <4 x i16> @trunc_packus_v4i64_v4i16(<4 x i64> %a0) {
+; SSE2-LABEL: trunc_packus_v4i64_v4i16:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [65535,65535]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm1, %xmm3
+; SSE2-NEXT:    pxor %xmm2, %xmm3
+; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [2147549183,2147549183]
+; SSE2-NEXT:    movdqa %xmm5, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
+; SSE2-NEXT:    pand %xmm7, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm6[1,1,3,3]
+; SSE2-NEXT:    por %xmm4, %xmm3
+; SSE2-NEXT:    pand %xmm3, %xmm1
+; SSE2-NEXT:    pandn %xmm8, %xmm3
+; SSE2-NEXT:    por %xmm1, %xmm3
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    pxor %xmm2, %xmm1
+; SSE2-NEXT:    movdqa %xmm5, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pand %xmm6, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm1, %xmm4
+; SSE2-NEXT:    pand %xmm4, %xmm0
+; SSE2-NEXT:    pandn %xmm8, %xmm4
+; SSE2-NEXT:    por %xmm0, %xmm4
+; SSE2-NEXT:    movdqa %xmm4, %xmm0
+; SSE2-NEXT:    pxor %xmm2, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pand %xmm5, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    por %xmm0, %xmm1
+; SSE2-NEXT:    pand %xmm4, %xmm1
+; SSE2-NEXT:    movdqa %xmm3, %xmm0
+; SSE2-NEXT:    pxor %xmm2, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pand %xmm5, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm0, %xmm2
+; SSE2-NEXT:    pand %xmm3, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm0[0,2,2,3,4,5,6,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: trunc_packus_v4i64_v4i16:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm8 = [65535,65535]
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
 ; SSSE3-NEXT:    movdqa %xmm1, %xmm3
-; SSSE3-NEXT:    pxor %xmm10, %xmm3
-; SSSE3-NEXT:    movdqa %xmm3, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm10, %xmm4
+; SSSE3-NEXT:    pxor %xmm2, %xmm3
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm5 = [2147549183,2147549183]
+; SSSE3-NEXT:    movdqa %xmm5, %xmm6
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm7, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm6[1,1,3,3]
+; SSSE3-NEXT:    por %xmm4, %xmm3
+; SSSE3-NEXT:    pand %xmm3, %xmm1
+; SSSE3-NEXT:    pandn %xmm8, %xmm3
+; SSSE3-NEXT:    por %xmm1, %xmm3
+; SSSE3-NEXT:    movdqa %xmm0, %xmm1
+; SSSE3-NEXT:    pxor %xmm2, %xmm1
+; SSSE3-NEXT:    movdqa %xmm5, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm6, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm1, %xmm4
+; SSSE3-NEXT:    pand %xmm4, %xmm0
+; SSSE3-NEXT:    pandn %xmm8, %xmm4
+; SSSE3-NEXT:    por %xmm0, %xmm4
+; SSSE3-NEXT:    movdqa %xmm4, %xmm0
+; SSSE3-NEXT:    pxor %xmm2, %xmm0
+; SSSE3-NEXT:    movdqa %xmm0, %xmm1
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm5, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    por %xmm0, %xmm1
+; SSSE3-NEXT:    pand %xmm4, %xmm1
+; SSSE3-NEXT:    movdqa %xmm3, %xmm0
+; SSSE3-NEXT:    pxor %xmm2, %xmm0
+; SSSE3-NEXT:    movdqa %xmm0, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm5, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm0, %xmm2
+; SSSE3-NEXT:    pand %xmm3, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
+; SSSE3-NEXT:    pshuflw {{.*#+}} xmm2 = xmm0[0,2,2,3,4,5,6,7]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
+; SSSE3-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: trunc_packus_v4i64_v4i16:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    movapd {{.*#+}} xmm4 = [65535,65535]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [2147483648,2147483648]
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pxor %xmm3, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm6 = [2147549183,2147549183]
+; SSE41-NEXT:    movdqa %xmm6, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm5
+; SSE41-NEXT:    movdqa %xmm6, %xmm7
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm7
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm0
+; SSE41-NEXT:    por %xmm7, %xmm0
+; SSE41-NEXT:    movapd %xmm4, %xmm5
+; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm5
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    pxor %xmm3, %xmm0
+; SSE41-NEXT:    movdqa %xmm6, %xmm1
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
+; SSE41-NEXT:    pand %xmm1, %xmm0
+; SSE41-NEXT:    por %xmm6, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm4
+; SSE41-NEXT:    pxor %xmm1, %xmm1
+; SSE41-NEXT:    movapd %xmm4, %xmm2
+; SSE41-NEXT:    xorpd %xmm3, %xmm2
+; SSE41-NEXT:    movapd %xmm2, %xmm6
+; SSE41-NEXT:    pcmpeqd %xmm3, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,0,2,2]
+; SSE41-NEXT:    pand %xmm6, %xmm0
+; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    pxor %xmm2, %xmm2
+; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm2
+; SSE41-NEXT:    movapd %xmm5, %xmm4
+; SSE41-NEXT:    xorpd %xmm3, %xmm4
+; SSE41-NEXT:    movapd %xmm4, %xmm6
+; SSE41-NEXT:    pcmpeqd %xmm3, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pand %xmm6, %xmm0
+; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm5, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
+; SSE41-NEXT:    pshuflw {{.*#+}} xmm1 = xmm0[0,2,2,3,4,5,6,7]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
+; SSE41-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; SSE41-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: trunc_packus_v4i64_v4i16:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [65535,65535]
+; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm2, %xmm3
+; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm4
+; AVX1-NEXT:    vblendvpd %xmm4, %xmm0, %xmm2, %xmm0
+; AVX1-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm0, %xmm5
+; AVX1-NEXT:    vblendvpd %xmm3, %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm1, %xmm2
+; AVX1-NEXT:    vpand %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
+; AVX1-NEXT:    vpand %xmm0, %xmm5, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2-SLOW-LABEL: trunc_packus_v4i64_v4i16:
+; AVX2-SLOW:       # %bb.0:
+; AVX2-SLOW-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [65535,65535,65535,65535]
+; AVX2-SLOW-NEXT:    vpcmpgtq %ymm0, %ymm1, %ymm2
+; AVX2-SLOW-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX2-SLOW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX2-SLOW-NEXT:    vpcmpgtq %ymm1, %ymm0, %ymm1
+; AVX2-SLOW-NEXT:    vpand %ymm0, %ymm1, %ymm0
+; AVX2-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX2-SLOW-NEXT:    vzeroupper
+; AVX2-SLOW-NEXT:    retq
+;
+; AVX2-FAST-LABEL: trunc_packus_v4i64_v4i16:
+; AVX2-FAST:       # %bb.0:
+; AVX2-FAST-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [65535,65535,65535,65535]
+; AVX2-FAST-NEXT:    vpcmpgtq %ymm0, %ymm1, %ymm2
+; AVX2-FAST-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX2-FAST-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX2-FAST-NEXT:    vpcmpgtq %ymm1, %ymm0, %ymm1
+; AVX2-FAST-NEXT:    vpand %ymm0, %ymm1, %ymm0
+; AVX2-FAST-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,1,8,9,8,9,10,11,8,9,10,11,12,13,14,15]
+; AVX2-FAST-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
+; AVX2-FAST-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
+; AVX2-FAST-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX2-FAST-NEXT:    vzeroupper
+; AVX2-FAST-NEXT:    retq
+;
+; AVX512F-LABEL: trunc_packus_v4i64_v4i16:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vpmovusqw %zmm0, %xmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: trunc_packus_v4i64_v4i16:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512VL-NEXT:    vpmaxsq %ymm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpmovusqw %ymm0, %xmm0
+; AVX512VL-NEXT:    vzeroupper
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: trunc_packus_v4i64_v4i16:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512BW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512BW-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmovusqw %zmm0, %xmm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: trunc_packus_v4i64_v4i16:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512BWVL-NEXT:    vpmaxsq %ymm1, %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpmovusqw %ymm0, %xmm0
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
+;
+; SKX-LABEL: trunc_packus_v4i64_v4i16:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; SKX-NEXT:    vpmaxsq %ymm1, %ymm0, %ymm0
+; SKX-NEXT:    vpmovusqw %ymm0, %xmm0
+; SKX-NEXT:    vzeroupper
+; SKX-NEXT:    retq
+  %1 = icmp slt <4 x i64> %a0, <i64 65535, i64 65535, i64 65535, i64 65535>
+  %2 = select <4 x i1> %1, <4 x i64> %a0, <4 x i64> <i64 65535, i64 65535, i64 65535, i64 65535>
+  %3 = icmp sgt <4 x i64> %2, zeroinitializer
+  %4 = select <4 x i1> %3, <4 x i64> %2, <4 x i64> zeroinitializer
+  %5 = trunc <4 x i64> %4 to <4 x i16>
+  ret <4 x i16> %5
+}
+
+define void @trunc_packus_v4i64_v4i16_store(<4 x i64> %a0, <4 x i16> *%p1) {
+; SSE2-LABEL: trunc_packus_v4i64_v4i16_store:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [65535,65535]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm1, %xmm3
+; SSE2-NEXT:    pxor %xmm2, %xmm3
+; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [2147549183,2147549183]
+; SSE2-NEXT:    movdqa %xmm5, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
+; SSE2-NEXT:    pand %xmm7, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm6[1,1,3,3]
+; SSE2-NEXT:    por %xmm4, %xmm3
+; SSE2-NEXT:    pand %xmm3, %xmm1
+; SSE2-NEXT:    pandn %xmm8, %xmm3
+; SSE2-NEXT:    por %xmm1, %xmm3
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    pxor %xmm2, %xmm1
+; SSE2-NEXT:    movdqa %xmm5, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pand %xmm6, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm1, %xmm4
+; SSE2-NEXT:    pand %xmm4, %xmm0
+; SSE2-NEXT:    pandn %xmm8, %xmm4
+; SSE2-NEXT:    por %xmm0, %xmm4
+; SSE2-NEXT:    movdqa %xmm4, %xmm0
+; SSE2-NEXT:    pxor %xmm2, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pand %xmm5, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    por %xmm0, %xmm1
+; SSE2-NEXT:    pand %xmm4, %xmm1
+; SSE2-NEXT:    movdqa %xmm3, %xmm0
+; SSE2-NEXT:    pxor %xmm2, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pand %xmm5, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm0, %xmm2
+; SSE2-NEXT:    pand %xmm3, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE2-NEXT:    movq %xmm1, (%rdi)
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: trunc_packus_v4i64_v4i16_store:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm8 = [65535,65535]
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
+; SSSE3-NEXT:    movdqa %xmm1, %xmm3
+; SSSE3-NEXT:    pxor %xmm2, %xmm3
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm5 = [2147549183,2147549183]
+; SSSE3-NEXT:    movdqa %xmm5, %xmm6
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm7, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm6[1,1,3,3]
+; SSSE3-NEXT:    por %xmm4, %xmm3
+; SSSE3-NEXT:    pand %xmm3, %xmm1
+; SSSE3-NEXT:    pandn %xmm8, %xmm3
+; SSSE3-NEXT:    por %xmm1, %xmm3
+; SSSE3-NEXT:    movdqa %xmm0, %xmm1
+; SSSE3-NEXT:    pxor %xmm2, %xmm1
+; SSSE3-NEXT:    movdqa %xmm5, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm6, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm1, %xmm4
+; SSSE3-NEXT:    pand %xmm4, %xmm0
+; SSSE3-NEXT:    pandn %xmm8, %xmm4
+; SSSE3-NEXT:    por %xmm0, %xmm4
+; SSSE3-NEXT:    movdqa %xmm4, %xmm0
+; SSSE3-NEXT:    pxor %xmm2, %xmm0
+; SSSE3-NEXT:    movdqa %xmm0, %xmm1
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm5, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    por %xmm0, %xmm1
+; SSSE3-NEXT:    pand %xmm4, %xmm1
+; SSSE3-NEXT:    movdqa %xmm3, %xmm0
+; SSSE3-NEXT:    pxor %xmm2, %xmm0
+; SSSE3-NEXT:    movdqa %xmm0, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm5, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm0, %xmm2
+; SSSE3-NEXT:    pand %xmm3, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
+; SSSE3-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; SSSE3-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSSE3-NEXT:    movq %xmm1, (%rdi)
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: trunc_packus_v4i64_v4i16_store:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    movapd {{.*#+}} xmm4 = [65535,65535]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [2147483648,2147483648]
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pxor %xmm3, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm6 = [2147549183,2147549183]
+; SSE41-NEXT:    movdqa %xmm6, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm5
+; SSE41-NEXT:    movdqa %xmm6, %xmm7
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm7
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm0
+; SSE41-NEXT:    por %xmm7, %xmm0
+; SSE41-NEXT:    movapd %xmm4, %xmm5
+; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm5
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    pxor %xmm3, %xmm0
+; SSE41-NEXT:    movdqa %xmm6, %xmm1
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
+; SSE41-NEXT:    pand %xmm1, %xmm0
+; SSE41-NEXT:    por %xmm6, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm4
+; SSE41-NEXT:    pxor %xmm1, %xmm1
+; SSE41-NEXT:    movapd %xmm4, %xmm2
+; SSE41-NEXT:    xorpd %xmm3, %xmm2
+; SSE41-NEXT:    movapd %xmm2, %xmm6
+; SSE41-NEXT:    pcmpeqd %xmm3, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,0,2,2]
+; SSE41-NEXT:    pand %xmm6, %xmm0
+; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    pxor %xmm2, %xmm2
+; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm2
+; SSE41-NEXT:    movapd %xmm5, %xmm4
+; SSE41-NEXT:    xorpd %xmm3, %xmm4
+; SSE41-NEXT:    movapd %xmm4, %xmm6
+; SSE41-NEXT:    pcmpeqd %xmm3, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pand %xmm6, %xmm0
+; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm5, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
+; SSE41-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
+; SSE41-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
+; SSE41-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE41-NEXT:    movq %xmm1, (%rdi)
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: trunc_packus_v4i64_v4i16_store:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [65535,65535]
+; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm2, %xmm3
+; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm4
+; AVX1-NEXT:    vblendvpd %xmm4, %xmm0, %xmm2, %xmm0
+; AVX1-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm0, %xmm5
+; AVX1-NEXT:    vblendvpd %xmm3, %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm1, %xmm2
+; AVX1-NEXT:    vpand %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
+; AVX1-NEXT:    vpand %xmm0, %xmm5, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX1-NEXT:    vmovq %xmm0, (%rdi)
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2-SLOW-LABEL: trunc_packus_v4i64_v4i16_store:
+; AVX2-SLOW:       # %bb.0:
+; AVX2-SLOW-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [65535,65535,65535,65535]
+; AVX2-SLOW-NEXT:    vpcmpgtq %ymm0, %ymm1, %ymm2
+; AVX2-SLOW-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX2-SLOW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX2-SLOW-NEXT:    vpcmpgtq %ymm1, %ymm0, %ymm1
+; AVX2-SLOW-NEXT:    vpand %ymm0, %ymm1, %ymm0
+; AVX2-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX2-SLOW-NEXT:    vmovq %xmm0, (%rdi)
+; AVX2-SLOW-NEXT:    vzeroupper
+; AVX2-SLOW-NEXT:    retq
+;
+; AVX2-FAST-LABEL: trunc_packus_v4i64_v4i16_store:
+; AVX2-FAST:       # %bb.0:
+; AVX2-FAST-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [65535,65535,65535,65535]
+; AVX2-FAST-NEXT:    vpcmpgtq %ymm0, %ymm1, %ymm2
+; AVX2-FAST-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX2-FAST-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX2-FAST-NEXT:    vpcmpgtq %ymm1, %ymm0, %ymm1
+; AVX2-FAST-NEXT:    vpand %ymm0, %ymm1, %ymm0
+; AVX2-FAST-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,1,8,9,8,9,10,11,8,9,10,11,12,13,14,15]
+; AVX2-FAST-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
+; AVX2-FAST-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
+; AVX2-FAST-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX2-FAST-NEXT:    vmovq %xmm0, (%rdi)
+; AVX2-FAST-NEXT:    vzeroupper
+; AVX2-FAST-NEXT:    retq
+;
+; AVX512F-LABEL: trunc_packus_v4i64_v4i16_store:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vpmovusqw %zmm0, %xmm0
+; AVX512F-NEXT:    vmovq %xmm0, (%rdi)
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: trunc_packus_v4i64_v4i16_store:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512VL-NEXT:    vpmaxsq %ymm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpmovusqw %ymm0, (%rdi)
+; AVX512VL-NEXT:    vzeroupper
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: trunc_packus_v4i64_v4i16_store:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512BW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512BW-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmovusqw %zmm0, %xmm0
+; AVX512BW-NEXT:    vmovq %xmm0, (%rdi)
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: trunc_packus_v4i64_v4i16_store:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512BWVL-NEXT:    vpmaxsq %ymm1, %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpmovusqw %ymm0, (%rdi)
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
+;
+; SKX-LABEL: trunc_packus_v4i64_v4i16_store:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; SKX-NEXT:    vpmaxsq %ymm1, %ymm0, %ymm0
+; SKX-NEXT:    vpmovusqw %ymm0, (%rdi)
+; SKX-NEXT:    vzeroupper
+; SKX-NEXT:    retq
+  %1 = icmp slt <4 x i64> %a0, <i64 65535, i64 65535, i64 65535, i64 65535>
+  %2 = select <4 x i1> %1, <4 x i64> %a0, <4 x i64> <i64 65535, i64 65535, i64 65535, i64 65535>
+  %3 = icmp sgt <4 x i64> %2, zeroinitializer
+  %4 = select <4 x i1> %3, <4 x i64> %2, <4 x i64> zeroinitializer
+  %5 = trunc <4 x i64> %4 to <4 x i16>
+  store <4 x i16> %5, <4 x i16> *%p1
+  ret void
+}
+
+define <8 x i16> @trunc_packus_v8i64_v8i16(<8 x i64>* %p0) "min-legal-vector-width"="256" {
+; SSE2-LABEL: trunc_packus_v8i64_v8i16:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa (%rdi), %xmm7
+; SSE2-NEXT:    movdqa 16(%rdi), %xmm2
+; SSE2-NEXT:    movdqa 32(%rdi), %xmm9
+; SSE2-NEXT:    movdqa 48(%rdi), %xmm6
+; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [65535,65535]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm11 = [2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm2, %xmm1
+; SSE2-NEXT:    pxor %xmm11, %xmm1
+; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [2147549183,2147549183]
+; SSE2-NEXT:    movdqa %xmm10, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm5[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pand %xmm3, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm5[1,1,3,3]
+; SSE2-NEXT:    por %xmm4, %xmm1
+; SSE2-NEXT:    pand %xmm1, %xmm2
+; SSE2-NEXT:    pandn %xmm8, %xmm1
+; SSE2-NEXT:    por %xmm2, %xmm1
+; SSE2-NEXT:    movdqa %xmm7, %xmm2
+; SSE2-NEXT:    pxor %xmm11, %xmm2
+; SSE2-NEXT:    movdqa %xmm10, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[1,1,3,3]
+; SSE2-NEXT:    pand %xmm4, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,1,3,3]
+; SSE2-NEXT:    por %xmm5, %xmm2
+; SSE2-NEXT:    pand %xmm2, %xmm7
+; SSE2-NEXT:    pandn %xmm8, %xmm2
+; SSE2-NEXT:    por %xmm7, %xmm2
+; SSE2-NEXT:    movdqa %xmm6, %xmm3
+; SSE2-NEXT:    pxor %xmm11, %xmm3
+; SSE2-NEXT:    movdqa %xmm10, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSE2-NEXT:    pand %xmm5, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm3, %xmm7
+; SSE2-NEXT:    pand %xmm7, %xmm6
+; SSE2-NEXT:    pandn %xmm8, %xmm7
+; SSE2-NEXT:    por %xmm6, %xmm7
+; SSE2-NEXT:    movdqa %xmm9, %xmm3
+; SSE2-NEXT:    pxor %xmm11, %xmm3
+; SSE2-NEXT:    movdqa %xmm10, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSE2-NEXT:    pand %xmm5, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm3, %xmm4
+; SSE2-NEXT:    pand %xmm4, %xmm9
+; SSE2-NEXT:    pandn %xmm8, %xmm4
+; SSE2-NEXT:    por %xmm9, %xmm4
+; SSE2-NEXT:    movdqa %xmm4, %xmm3
+; SSE2-NEXT:    pxor %xmm11, %xmm3
+; SSE2-NEXT:    movdqa %xmm3, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm11, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm11, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
+; SSE2-NEXT:    pand %xmm6, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm5[1,1,3,3]
+; SSE2-NEXT:    por %xmm0, %xmm3
+; SSE2-NEXT:    pand %xmm4, %xmm3
+; SSE2-NEXT:    movdqa %xmm7, %xmm0
+; SSE2-NEXT:    pxor %xmm11, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm11, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm11, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pand %xmm5, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm0, %xmm4
+; SSE2-NEXT:    pand %xmm7, %xmm4
+; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    pxor %xmm11, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm11, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm11, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pand %xmm6, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSE2-NEXT:    por %xmm0, %xmm5
+; SSE2-NEXT:    pand %xmm2, %xmm5
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    pxor %xmm11, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm11, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm11, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pand %xmm6, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE2-NEXT:    por %xmm0, %xmm2
+; SSE2-NEXT:    pand %xmm1, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm5[0,2,2,3]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,2,2,3]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm0[0,1,0,2,4,5,6,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,2,2,3]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,1,0,2,4,5,6,7]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: trunc_packus_v8i64_v8i16:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa (%rdi), %xmm7
+; SSSE3-NEXT:    movdqa 16(%rdi), %xmm2
+; SSSE3-NEXT:    movdqa 32(%rdi), %xmm9
+; SSSE3-NEXT:    movdqa 48(%rdi), %xmm6
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm8 = [65535,65535]
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm11 = [2147483648,2147483648]
+; SSSE3-NEXT:    movdqa %xmm2, %xmm1
+; SSSE3-NEXT:    pxor %xmm11, %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm10 = [2147549183,2147549183]
+; SSSE3-NEXT:    movdqa %xmm10, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm5[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm3, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm5[1,1,3,3]
+; SSSE3-NEXT:    por %xmm4, %xmm1
+; SSSE3-NEXT:    pand %xmm1, %xmm2
+; SSSE3-NEXT:    pandn %xmm8, %xmm1
+; SSSE3-NEXT:    por %xmm2, %xmm1
+; SSSE3-NEXT:    movdqa %xmm7, %xmm2
+; SSSE3-NEXT:    pxor %xmm11, %xmm2
+; SSSE3-NEXT:    movdqa %xmm10, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm4, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,1,3,3]
+; SSSE3-NEXT:    por %xmm5, %xmm2
+; SSSE3-NEXT:    pand %xmm2, %xmm7
+; SSSE3-NEXT:    pandn %xmm8, %xmm2
+; SSSE3-NEXT:    por %xmm7, %xmm2
+; SSSE3-NEXT:    movdqa %xmm6, %xmm3
+; SSSE3-NEXT:    pxor %xmm11, %xmm3
+; SSSE3-NEXT:    movdqa %xmm10, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm5, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm3, %xmm7
+; SSSE3-NEXT:    pand %xmm7, %xmm6
+; SSSE3-NEXT:    pandn %xmm8, %xmm7
+; SSSE3-NEXT:    por %xmm6, %xmm7
+; SSSE3-NEXT:    movdqa %xmm9, %xmm3
+; SSSE3-NEXT:    pxor %xmm11, %xmm3
+; SSSE3-NEXT:    movdqa %xmm10, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm5, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm3, %xmm4
+; SSSE3-NEXT:    pand %xmm4, %xmm9
+; SSSE3-NEXT:    pandn %xmm8, %xmm4
+; SSSE3-NEXT:    por %xmm9, %xmm4
+; SSSE3-NEXT:    movdqa %xmm4, %xmm3
+; SSSE3-NEXT:    pxor %xmm11, %xmm3
+; SSSE3-NEXT:    movdqa %xmm3, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm11, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm11, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm6, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm5[1,1,3,3]
+; SSSE3-NEXT:    por %xmm0, %xmm3
+; SSSE3-NEXT:    pand %xmm4, %xmm3
+; SSSE3-NEXT:    movdqa %xmm7, %xmm0
+; SSSE3-NEXT:    pxor %xmm11, %xmm0
+; SSSE3-NEXT:    movdqa %xmm0, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm11, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm11, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm5, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm0, %xmm4
+; SSSE3-NEXT:    pand %xmm7, %xmm4
+; SSSE3-NEXT:    movdqa %xmm2, %xmm0
+; SSSE3-NEXT:    pxor %xmm11, %xmm0
+; SSSE3-NEXT:    movdqa %xmm0, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm11, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm11, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm6, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSSE3-NEXT:    por %xmm0, %xmm5
+; SSSE3-NEXT:    pand %xmm2, %xmm5
+; SSSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSSE3-NEXT:    pxor %xmm11, %xmm0
+; SSSE3-NEXT:    movdqa %xmm0, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm11, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm11, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm6, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    por %xmm0, %xmm2
+; SSSE3-NEXT:    pand %xmm1, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
+; SSSE3-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm5[0,2,2,3]
+; SSSE3-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,2,2,3]
+; SSSE3-NEXT:    pshuflw {{.*#+}} xmm2 = xmm0[0,1,0,2,4,5,6,7]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,2,2,3]
+; SSSE3-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,1,0,2,4,5,6,7]
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; SSSE3-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: trunc_packus_v8i64_v8i16:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa (%rdi), %xmm10
+; SSE41-NEXT:    movdqa 16(%rdi), %xmm9
+; SSE41-NEXT:    movdqa 32(%rdi), %xmm3
+; SSE41-NEXT:    movdqa 48(%rdi), %xmm5
+; SSE41-NEXT:    movapd {{.*#+}} xmm1 = [65535,65535]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    pxor %xmm2, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [2147549183,2147549183]
+; SSE41-NEXT:    movdqa %xmm4, %xmm7
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm7
+; SSE41-NEXT:    movdqa %xmm4, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
+; SSE41-NEXT:    pand %xmm7, %xmm0
+; SSE41-NEXT:    por %xmm6, %xmm0
+; SSE41-NEXT:    movapd %xmm1, %xmm8
+; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm8
+; SSE41-NEXT:    movdqa %xmm5, %xmm0
+; SSE41-NEXT:    pxor %xmm2, %xmm0
+; SSE41-NEXT:    movdqa %xmm4, %xmm3
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm3
+; SSE41-NEXT:    movdqa %xmm4, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
+; SSE41-NEXT:    pand %xmm3, %xmm0
+; SSE41-NEXT:    por %xmm6, %xmm0
+; SSE41-NEXT:    movapd %xmm1, %xmm6
+; SSE41-NEXT:    blendvpd %xmm0, %xmm5, %xmm6
+; SSE41-NEXT:    movdqa %xmm10, %xmm0
+; SSE41-NEXT:    pxor %xmm2, %xmm0
+; SSE41-NEXT:    movdqa %xmm4, %xmm3
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm3
+; SSE41-NEXT:    movdqa %xmm4, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm5
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2]
+; SSE41-NEXT:    pand %xmm3, %xmm0
+; SSE41-NEXT:    por %xmm5, %xmm0
+; SSE41-NEXT:    movapd %xmm1, %xmm3
+; SSE41-NEXT:    blendvpd %xmm0, %xmm10, %xmm3
+; SSE41-NEXT:    movdqa %xmm9, %xmm0
+; SSE41-NEXT:    pxor %xmm2, %xmm0
+; SSE41-NEXT:    movdqa %xmm4, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm0
+; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm9, %xmm1
+; SSE41-NEXT:    pxor %xmm5, %xmm5
+; SSE41-NEXT:    movapd %xmm1, %xmm4
+; SSE41-NEXT:    xorpd %xmm2, %xmm4
+; SSE41-NEXT:    movapd %xmm4, %xmm7
+; SSE41-NEXT:    pcmpeqd %xmm2, %xmm7
+; SSE41-NEXT:    pcmpgtd %xmm2, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pand %xmm7, %xmm0
+; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    pxor %xmm4, %xmm4
+; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm4
+; SSE41-NEXT:    movapd %xmm3, %xmm1
+; SSE41-NEXT:    xorpd %xmm2, %xmm1
+; SSE41-NEXT:    movapd %xmm1, %xmm7
+; SSE41-NEXT:    pcmpeqd %xmm2, %xmm7
+; SSE41-NEXT:    pcmpgtd %xmm2, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2]
+; SSE41-NEXT:    pand %xmm7, %xmm0
+; SSE41-NEXT:    por %xmm1, %xmm0
+; SSE41-NEXT:    pxor %xmm1, %xmm1
+; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm1
+; SSE41-NEXT:    packusdw %xmm4, %xmm1
+; SSE41-NEXT:    movapd %xmm6, %xmm3
+; SSE41-NEXT:    xorpd %xmm2, %xmm3
+; SSE41-NEXT:    movapd %xmm3, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm2, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm2, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm3, %xmm0
+; SSE41-NEXT:    pxor %xmm3, %xmm3
+; SSE41-NEXT:    blendvpd %xmm0, %xmm6, %xmm3
+; SSE41-NEXT:    movapd %xmm8, %xmm4
+; SSE41-NEXT:    xorpd %xmm2, %xmm4
+; SSE41-NEXT:    movapd %xmm4, %xmm6
+; SSE41-NEXT:    pcmpeqd %xmm2, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm2, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pand %xmm6, %xmm0
+; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm8, %xmm5
+; SSE41-NEXT:    packusdw %xmm3, %xmm5
+; SSE41-NEXT:    packusdw %xmm5, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: trunc_packus_v8i64_v8i16:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX1-NEXT:    vmovdqa 16(%rdi), %xmm1
+; AVX1-NEXT:    vmovdqa 32(%rdi), %xmm2
+; AVX1-NEXT:    vmovdqa 48(%rdi), %xmm3
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [65535,65535]
+; AVX1-NEXT:    vpcmpgtq %xmm3, %xmm4, %xmm8
+; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm4, %xmm6
+; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm4, %xmm7
+; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm4, %xmm5
+; AVX1-NEXT:    vblendvpd %xmm5, %xmm0, %xmm4, %xmm0
+; AVX1-NEXT:    vpxor %xmm5, %xmm5, %xmm5
+; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm0, %xmm9
+; AVX1-NEXT:    vblendvpd %xmm7, %xmm1, %xmm4, %xmm1
+; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm1, %xmm7
+; AVX1-NEXT:    vblendvpd %xmm6, %xmm2, %xmm4, %xmm2
+; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm2, %xmm6
+; AVX1-NEXT:    vblendvpd %xmm8, %xmm3, %xmm4, %xmm3
+; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm3, %xmm4
+; AVX1-NEXT:    vpand %xmm3, %xmm4, %xmm3
+; AVX1-NEXT:    vpand %xmm2, %xmm6, %xmm2
+; AVX1-NEXT:    vpackusdw %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpand %xmm1, %xmm7, %xmm1
+; AVX1-NEXT:    vpand %xmm0, %xmm9, %xmm0
+; AVX1-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpackusdw %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: trunc_packus_v8i64_v8i16:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX2-NEXT:    vmovdqa 32(%rdi), %ymm1
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [65535,65535,65535,65535]
+; AVX2-NEXT:    vpcmpgtq %ymm0, %ymm2, %ymm3
+; AVX2-NEXT:    vblendvpd %ymm3, %ymm0, %ymm2, %ymm0
+; AVX2-NEXT:    vpcmpgtq %ymm1, %ymm2, %ymm3
+; AVX2-NEXT:    vblendvpd %ymm3, %ymm1, %ymm2, %ymm1
+; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpcmpgtq %ymm2, %ymm1, %ymm3
+; AVX2-NEXT:    vpand %ymm1, %ymm3, %ymm1
+; AVX2-NEXT:    vpcmpgtq %ymm2, %ymm0, %ymm2
+; AVX2-NEXT:    vpand %ymm0, %ymm2, %ymm0
+; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: trunc_packus_v8i64_v8i16:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; AVX512-NEXT:    vpmaxsq (%rdi), %zmm0, %zmm0
+; AVX512-NEXT:    vpmovusqw %zmm0, %xmm0
+; AVX512-NEXT:    vzeroupper
+; AVX512-NEXT:    retq
+;
+; SKX-LABEL: trunc_packus_v8i64_v8i16:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; SKX-NEXT:    vpmaxsq 32(%rdi), %ymm0, %ymm1
+; SKX-NEXT:    vpmovusqw %ymm1, %xmm1
+; SKX-NEXT:    vpmaxsq (%rdi), %ymm0, %ymm0
+; SKX-NEXT:    vpmovusqw %ymm0, %xmm0
+; SKX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SKX-NEXT:    vzeroupper
+; SKX-NEXT:    retq
+  %a0 = load <8 x i64>, <8 x i64>* %p0
+  %1 = icmp slt <8 x i64> %a0, <i64 65535, i64 65535, i64 65535, i64 65535, i64 65535, i64 65535, i64 65535, i64 65535>
+  %2 = select <8 x i1> %1, <8 x i64> %a0, <8 x i64> <i64 65535, i64 65535, i64 65535, i64 65535, i64 65535, i64 65535, i64 65535, i64 65535>
+  %3 = icmp sgt <8 x i64> %2, zeroinitializer
+  %4 = select <8 x i1> %3, <8 x i64> %2, <8 x i64> zeroinitializer
+  %5 = trunc <8 x i64> %4 to <8 x i16>
+  ret <8 x i16> %5
+}
+
+define <4 x i16> @trunc_packus_v4i32_v4i16(<4 x i32> %a0) {
+; SSE2-LABEL: trunc_packus_v4i32_v4i16:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [65535,65535,65535,65535]
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    pandn %xmm1, %xmm2
+; SSE2-NEXT:    por %xmm0, %xmm2
+; SSE2-NEXT:    pxor %xmm0, %xmm0
+; SSE2-NEXT:    movdqa %xmm2, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
+; SSE2-NEXT:    pand %xmm2, %xmm1
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm1[0,2,2,3,4,5,6,7]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,6,6,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: trunc_packus_v4i32_v4i16:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [65535,65535,65535,65535]
+; SSSE3-NEXT:    movdqa %xmm1, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSSE3-NEXT:    pand %xmm2, %xmm0
+; SSSE3-NEXT:    pandn %xmm1, %xmm2
+; SSSE3-NEXT:    por %xmm0, %xmm2
+; SSSE3-NEXT:    pxor %xmm1, %xmm1
+; SSSE3-NEXT:    movdqa %xmm2, %xmm0
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
+; SSSE3-NEXT:    pand %xmm2, %xmm0
+; SSSE3-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: trunc_packus_v4i32_v4i16:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    packusdw %xmm0, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: trunc_packus_v4i32_v4i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpackusdw %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: trunc_packus_v4i32_v4i16:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpackusdw %xmm0, %xmm0, %xmm0
+; AVX512-NEXT:    retq
+;
+; SKX-LABEL: trunc_packus_v4i32_v4i16:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpackusdw %xmm0, %xmm0, %xmm0
+; SKX-NEXT:    retq
+  %1 = icmp slt <4 x i32> %a0, <i32 65535, i32 65535, i32 65535, i32 65535>
+  %2 = select <4 x i1> %1, <4 x i32> %a0, <4 x i32> <i32 65535, i32 65535, i32 65535, i32 65535>
+  %3 = icmp sgt <4 x i32> %2, zeroinitializer
+  %4 = select <4 x i1> %3, <4 x i32> %2, <4 x i32> zeroinitializer
+  %5 = trunc <4 x i32> %4 to <4 x i16>
+  ret <4 x i16> %5
+}
+
+define void @trunc_packus_v4i32_v4i16_store(<4 x i32> %a0, <4 x i16> *%p1) {
+; SSE2-LABEL: trunc_packus_v4i32_v4i16_store:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [65535,65535,65535,65535]
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    pandn %xmm1, %xmm2
+; SSE2-NEXT:    por %xmm0, %xmm2
+; SSE2-NEXT:    pxor %xmm0, %xmm0
+; SSE2-NEXT:    movdqa %xmm2, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
+; SSE2-NEXT:    pand %xmm2, %xmm1
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm1[0,2,2,3,4,5,6,7]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,6,6,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSE2-NEXT:    movq %xmm0, (%rdi)
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: trunc_packus_v4i32_v4i16_store:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [65535,65535,65535,65535]
+; SSSE3-NEXT:    movdqa %xmm1, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSSE3-NEXT:    pand %xmm2, %xmm0
+; SSSE3-NEXT:    pandn %xmm1, %xmm2
+; SSSE3-NEXT:    por %xmm0, %xmm2
+; SSSE3-NEXT:    pxor %xmm0, %xmm0
+; SSSE3-NEXT:    movdqa %xmm2, %xmm1
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm1
+; SSSE3-NEXT:    pand %xmm2, %xmm1
+; SSSE3-NEXT:    pshufb {{.*#+}} xmm1 = xmm1[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
+; SSSE3-NEXT:    movq %xmm1, (%rdi)
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: trunc_packus_v4i32_v4i16_store:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    packusdw %xmm0, %xmm0
+; SSE41-NEXT:    movq %xmm0, (%rdi)
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: trunc_packus_v4i32_v4i16_store:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpackusdw %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vmovq %xmm0, (%rdi)
+; AVX-NEXT:    retq
+;
+; AVX512F-LABEL: trunc_packus_v4i32_v4i16_store:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpackusdw %xmm0, %xmm0, %xmm0
+; AVX512F-NEXT:    vmovq %xmm0, (%rdi)
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: trunc_packus_v4i32_v4i16_store:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512VL-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpmovusdw %xmm0, (%rdi)
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: trunc_packus_v4i32_v4i16_store:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpackusdw %xmm0, %xmm0, %xmm0
+; AVX512BW-NEXT:    vmovq %xmm0, (%rdi)
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: trunc_packus_v4i32_v4i16_store:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512BWVL-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
+; AVX512BWVL-NEXT:    vpmovusdw %xmm0, (%rdi)
+; AVX512BWVL-NEXT:    retq
+;
+; SKX-LABEL: trunc_packus_v4i32_v4i16_store:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; SKX-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
+; SKX-NEXT:    vpmovusdw %xmm0, (%rdi)
+; SKX-NEXT:    retq
+  %1 = icmp slt <4 x i32> %a0, <i32 65535, i32 65535, i32 65535, i32 65535>
+  %2 = select <4 x i1> %1, <4 x i32> %a0, <4 x i32> <i32 65535, i32 65535, i32 65535, i32 65535>
+  %3 = icmp sgt <4 x i32> %2, zeroinitializer
+  %4 = select <4 x i1> %3, <4 x i32> %2, <4 x i32> zeroinitializer
+  %5 = trunc <4 x i32> %4 to <4 x i16>
+  store <4 x i16> %5, <4 x i16> *%p1
+  ret void
+}
+
+define <8 x i16> @trunc_packus_v8i32_v8i16(<8 x i32> %a0) {
+; SSE2-LABEL: trunc_packus_v8i32_v8i16:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [65535,65535,65535,65535]
+; SSE2-NEXT:    movdqa %xmm2, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
+; SSE2-NEXT:    pand %xmm3, %xmm1
+; SSE2-NEXT:    pandn %xmm2, %xmm3
+; SSE2-NEXT:    por %xmm1, %xmm3
+; SSE2-NEXT:    movdqa %xmm2, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
+; SSE2-NEXT:    pand %xmm1, %xmm0
+; SSE2-NEXT:    pandn %xmm2, %xmm1
+; SSE2-NEXT:    por %xmm0, %xmm1
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm0
+; SSE2-NEXT:    pand %xmm1, %xmm0
+; SSE2-NEXT:    movdqa %xmm3, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm1
+; SSE2-NEXT:    pand %xmm3, %xmm1
+; SSE2-NEXT:    pslld $16, %xmm1
+; SSE2-NEXT:    psrad $16, %xmm1
+; SSE2-NEXT:    pslld $16, %xmm0
+; SSE2-NEXT:    psrad $16, %xmm0
+; SSE2-NEXT:    packssdw %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: trunc_packus_v8i32_v8i16:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [65535,65535,65535,65535]
+; SSSE3-NEXT:    movdqa %xmm2, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm3
+; SSSE3-NEXT:    pand %xmm3, %xmm1
+; SSSE3-NEXT:    pandn %xmm2, %xmm3
+; SSSE3-NEXT:    por %xmm1, %xmm3
+; SSSE3-NEXT:    movdqa %xmm2, %xmm1
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm1
+; SSSE3-NEXT:    pand %xmm1, %xmm0
+; SSSE3-NEXT:    pandn %xmm2, %xmm1
+; SSSE3-NEXT:    por %xmm0, %xmm1
+; SSSE3-NEXT:    pxor %xmm2, %xmm2
+; SSSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm0
+; SSSE3-NEXT:    pand %xmm1, %xmm0
+; SSSE3-NEXT:    movdqa %xmm3, %xmm1
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm1
+; SSSE3-NEXT:    pand %xmm3, %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
+; SSSE3-NEXT:    pshufb %xmm2, %xmm1
+; SSSE3-NEXT:    pshufb %xmm2, %xmm0
+; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: trunc_packus_v8i32_v8i16:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    packusdw %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: trunc_packus_v8i32_v8i16:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: trunc_packus_v8i32_v8i16:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+;
+; AVX512F-LABEL: trunc_packus_v8i32_v8i16:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: trunc_packus_v8i32_v8i16:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512VL-NEXT:    vpmaxsd %ymm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpmovusdw %ymm0, %xmm0
+; AVX512VL-NEXT:    vzeroupper
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: trunc_packus_v8i32_v8i16:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512BW-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: trunc_packus_v8i32_v8i16:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512BWVL-NEXT:    vpmaxsd %ymm1, %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpmovusdw %ymm0, %xmm0
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
+;
+; SKX-LABEL: trunc_packus_v8i32_v8i16:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; SKX-NEXT:    vpmaxsd %ymm1, %ymm0, %ymm0
+; SKX-NEXT:    vpmovusdw %ymm0, %xmm0
+; SKX-NEXT:    vzeroupper
+; SKX-NEXT:    retq
+  %1 = icmp slt <8 x i32> %a0, <i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535>
+  %2 = select <8 x i1> %1, <8 x i32> %a0, <8 x i32> <i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535>
+  %3 = icmp sgt <8 x i32> %2, zeroinitializer
+  %4 = select <8 x i1> %3, <8 x i32> %2, <8 x i32> zeroinitializer
+  %5 = trunc <8 x i32> %4 to <8 x i16>
+  ret <8 x i16> %5
+}
+
+define <16 x i16> @trunc_packus_v16i32_v16i16(<16 x i32>* %p0) "min-legal-vector-width"="256" {
+; SSE2-LABEL: trunc_packus_v16i32_v16i16:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa (%rdi), %xmm1
+; SSE2-NEXT:    movdqa 16(%rdi), %xmm3
+; SSE2-NEXT:    movdqa 32(%rdi), %xmm0
+; SSE2-NEXT:    movdqa 48(%rdi), %xmm4
+; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [65535,65535,65535,65535]
+; SSE2-NEXT:    movdqa %xmm5, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm2
+; SSE2-NEXT:    pand %xmm2, %xmm3
+; SSE2-NEXT:    pandn %xmm5, %xmm2
+; SSE2-NEXT:    por %xmm3, %xmm2
+; SSE2-NEXT:    movdqa %xmm5, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
+; SSE2-NEXT:    pand %xmm3, %xmm1
+; SSE2-NEXT:    pandn %xmm5, %xmm3
+; SSE2-NEXT:    por %xmm1, %xmm3
+; SSE2-NEXT:    movdqa %xmm5, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm6
+; SSE2-NEXT:    pand %xmm6, %xmm4
+; SSE2-NEXT:    pandn %xmm5, %xmm6
+; SSE2-NEXT:    por %xmm4, %xmm6
+; SSE2-NEXT:    movdqa %xmm5, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSE2-NEXT:    pand %xmm4, %xmm0
+; SSE2-NEXT:    pandn %xmm5, %xmm4
+; SSE2-NEXT:    por %xmm0, %xmm4
+; SSE2-NEXT:    pxor %xmm5, %xmm5
+; SSE2-NEXT:    movdqa %xmm4, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm5, %xmm1
+; SSE2-NEXT:    pand %xmm4, %xmm1
+; SSE2-NEXT:    movdqa %xmm6, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm5, %xmm4
+; SSE2-NEXT:    pand %xmm6, %xmm4
+; SSE2-NEXT:    movdqa %xmm3, %xmm0
+; SSE2-NEXT:    pcmpgtd %xmm5, %xmm0
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    movdqa %xmm2, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm5, %xmm3
+; SSE2-NEXT:    pand %xmm2, %xmm3
+; SSE2-NEXT:    pslld $16, %xmm3
+; SSE2-NEXT:    psrad $16, %xmm3
+; SSE2-NEXT:    pslld $16, %xmm0
+; SSE2-NEXT:    psrad $16, %xmm0
+; SSE2-NEXT:    packssdw %xmm3, %xmm0
+; SSE2-NEXT:    pslld $16, %xmm4
+; SSE2-NEXT:    psrad $16, %xmm4
+; SSE2-NEXT:    pslld $16, %xmm1
+; SSE2-NEXT:    psrad $16, %xmm1
+; SSE2-NEXT:    packssdw %xmm4, %xmm1
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: trunc_packus_v16i32_v16i16:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa (%rdi), %xmm1
+; SSSE3-NEXT:    movdqa 16(%rdi), %xmm3
+; SSSE3-NEXT:    movdqa 32(%rdi), %xmm0
+; SSSE3-NEXT:    movdqa 48(%rdi), %xmm4
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm5 = [65535,65535,65535,65535]
+; SSSE3-NEXT:    movdqa %xmm5, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm2
+; SSSE3-NEXT:    pand %xmm2, %xmm3
+; SSSE3-NEXT:    pandn %xmm5, %xmm2
+; SSSE3-NEXT:    por %xmm3, %xmm2
+; SSSE3-NEXT:    movdqa %xmm5, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm3
+; SSSE3-NEXT:    pand %xmm3, %xmm1
+; SSSE3-NEXT:    pandn %xmm5, %xmm3
+; SSSE3-NEXT:    por %xmm1, %xmm3
+; SSSE3-NEXT:    movdqa %xmm5, %xmm6
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm6
+; SSSE3-NEXT:    pand %xmm6, %xmm4
+; SSSE3-NEXT:    pandn %xmm5, %xmm6
+; SSSE3-NEXT:    por %xmm4, %xmm6
+; SSSE3-NEXT:    movdqa %xmm5, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSSE3-NEXT:    pand %xmm4, %xmm0
+; SSSE3-NEXT:    pandn %xmm5, %xmm4
+; SSSE3-NEXT:    por %xmm0, %xmm4
+; SSSE3-NEXT:    pxor %xmm5, %xmm5
+; SSSE3-NEXT:    movdqa %xmm4, %xmm1
+; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm1
+; SSSE3-NEXT:    pand %xmm4, %xmm1
+; SSSE3-NEXT:    movdqa %xmm6, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm4
+; SSSE3-NEXT:    pand %xmm6, %xmm4
+; SSSE3-NEXT:    movdqa %xmm3, %xmm0
+; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm0
+; SSSE3-NEXT:    pand %xmm3, %xmm0
+; SSSE3-NEXT:    movdqa %xmm2, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm3
+; SSSE3-NEXT:    pand %xmm2, %xmm3
+; SSSE3-NEXT:    pslld $16, %xmm3
+; SSSE3-NEXT:    psrad $16, %xmm3
+; SSSE3-NEXT:    pslld $16, %xmm0
+; SSSE3-NEXT:    psrad $16, %xmm0
+; SSSE3-NEXT:    packssdw %xmm3, %xmm0
+; SSSE3-NEXT:    pslld $16, %xmm4
+; SSSE3-NEXT:    psrad $16, %xmm4
+; SSSE3-NEXT:    pslld $16, %xmm1
+; SSSE3-NEXT:    psrad $16, %xmm1
+; SSSE3-NEXT:    packssdw %xmm4, %xmm1
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: trunc_packus_v16i32_v16i16:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa (%rdi), %xmm0
+; SSE41-NEXT:    movdqa 32(%rdi), %xmm1
+; SSE41-NEXT:    packusdw 16(%rdi), %xmm0
+; SSE41-NEXT:    packusdw 48(%rdi), %xmm1
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: trunc_packus_v16i32_v16i16:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX1-NEXT:    vmovdqa 32(%rdi), %xmm1
+; AVX1-NEXT:    vpackusdw 48(%rdi), %xmm1, %xmm1
+; AVX1-NEXT:    vpackusdw 16(%rdi), %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: trunc_packus_v16i32_v16i16:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX2-NEXT:    vpackusdw 32(%rdi), %ymm0, %ymm0
+; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: trunc_packus_v16i32_v16i16:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; AVX512-NEXT:    vpmaxsd (%rdi), %zmm0, %zmm0
+; AVX512-NEXT:    vpmovusdw %zmm0, %ymm0
+; AVX512-NEXT:    retq
+;
+; SKX-LABEL: trunc_packus_v16i32_v16i16:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vmovdqa (%rdi), %ymm0
+; SKX-NEXT:    vpackusdw 32(%rdi), %ymm0, %ymm0
+; SKX-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; SKX-NEXT:    retq
+  %a0 = load <16 x i32>, <16 x i32>* %p0
+  %1 = icmp slt <16 x i32> %a0, <i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535>
+  %2 = select <16 x i1> %1, <16 x i32> %a0, <16 x i32> <i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535>
+  %3 = icmp sgt <16 x i32> %2, zeroinitializer
+  %4 = select <16 x i1> %3, <16 x i32> %2, <16 x i32> zeroinitializer
+  %5 = trunc <16 x i32> %4 to <16 x i16>
+  ret <16 x i16> %5
+}
+
+;
+; PACKUS saturation truncation to vXi8
+;
+
+define <2 x i8> @trunc_packus_v2i64_v2i8(<2 x i64> %a0) {
+; SSE2-LABEL: trunc_packus_v2i64_v2i8:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    pxor %xmm1, %xmm2
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [2147483903,2147483903]
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm3, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE2-NEXT:    pand %xmm5, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm2, %xmm3
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm3
+; SSE2-NEXT:    por %xmm0, %xmm3
+; SSE2-NEXT:    movdqa %xmm3, %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pand %xmm4, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3]
+; SSE2-NEXT:    por %xmm1, %xmm0
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    packuswb %xmm0, %xmm0
+; SSE2-NEXT:    packuswb %xmm0, %xmm0
+; SSE2-NEXT:    packuswb %xmm0, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: trunc_packus_v2i64_v2i8:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [2147483648,2147483648]
+; SSSE3-NEXT:    movdqa %xmm0, %xmm2
+; SSSE3-NEXT:    pxor %xmm1, %xmm2
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm3 = [2147483903,2147483903]
+; SSSE3-NEXT:    movdqa %xmm3, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm5, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm2, %xmm3
+; SSSE3-NEXT:    pand %xmm3, %xmm0
+; SSSE3-NEXT:    pandn {{.*}}(%rip), %xmm3
+; SSSE3-NEXT:    por %xmm0, %xmm3
+; SSSE3-NEXT:    movdqa %xmm3, %xmm0
+; SSSE3-NEXT:    pxor %xmm1, %xmm0
+; SSSE3-NEXT:    movdqa %xmm0, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm4, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    por %xmm1, %xmm0
+; SSSE3-NEXT:    pand %xmm3, %xmm0
+; SSSE3-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: trunc_packus_v2i64_v2i8:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    movapd {{.*#+}} xmm2 = [255,255]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [2147483648,2147483648]
+; SSE41-NEXT:    pxor %xmm3, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [2147483903,2147483903]
+; SSE41-NEXT:    movdqa %xmm4, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm0
+; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
+; SSE41-NEXT:    xorpd %xmm1, %xmm1
+; SSE41-NEXT:    movapd %xmm2, %xmm4
+; SSE41-NEXT:    xorpd %xmm3, %xmm4
+; SSE41-NEXT:    movapd %xmm4, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm3, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm0
+; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
+; SSE41-NEXT:    pshufb {{.*#+}} xmm1 = xmm1[0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: trunc_packus_v2i64_v2i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [255,255]
+; AVX-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm2
+; AVX-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm1
+; AVX-NEXT:    vpand %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX-NEXT:    retq
+;
+; AVX512F-LABEL: trunc_packus_v2i64_v2i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vpmovusqb %zmm0, %xmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: trunc_packus_v2i64_v2i8:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512VL-NEXT:    vpmaxsq %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpmovusqb %xmm0, %xmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: trunc_packus_v2i64_v2i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512BW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512BW-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmovusqb %zmm0, %xmm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: trunc_packus_v2i64_v2i8:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512BWVL-NEXT:    vpmaxsq %xmm1, %xmm0, %xmm0
+; AVX512BWVL-NEXT:    vpmovusqb %xmm0, %xmm0
+; AVX512BWVL-NEXT:    retq
+;
+; SKX-LABEL: trunc_packus_v2i64_v2i8:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; SKX-NEXT:    vpmaxsq %xmm1, %xmm0, %xmm0
+; SKX-NEXT:    vpmovusqb %xmm0, %xmm0
+; SKX-NEXT:    retq
+  %1 = icmp slt <2 x i64> %a0, <i64 255, i64 255>
+  %2 = select <2 x i1> %1, <2 x i64> %a0, <2 x i64> <i64 255, i64 255>
+  %3 = icmp sgt <2 x i64> %2, zeroinitializer
+  %4 = select <2 x i1> %3, <2 x i64> %2, <2 x i64> zeroinitializer
+  %5 = trunc <2 x i64> %4 to <2 x i8>
+  ret <2 x i8> %5
+}
+
+define void @trunc_packus_v2i64_v2i8_store(<2 x i64> %a0, <2 x i8> *%p1) {
+; SSE2-LABEL: trunc_packus_v2i64_v2i8_store:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    pxor %xmm1, %xmm2
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [2147483903,2147483903]
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm3, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE2-NEXT:    pand %xmm5, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm2, %xmm3
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm3
+; SSE2-NEXT:    por %xmm0, %xmm3
+; SSE2-NEXT:    movdqa %xmm3, %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pand %xmm4, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,3,3]
+; SSE2-NEXT:    por %xmm0, %xmm1
+; SSE2-NEXT:    pand %xmm3, %xmm1
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE2-NEXT:    packuswb %xmm1, %xmm1
+; SSE2-NEXT:    packuswb %xmm0, %xmm1
+; SSE2-NEXT:    packuswb %xmm0, %xmm1
+; SSE2-NEXT:    movd %xmm1, %eax
+; SSE2-NEXT:    movw %ax, (%rdi)
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: trunc_packus_v2i64_v2i8_store:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [2147483648,2147483648]
+; SSSE3-NEXT:    movdqa %xmm0, %xmm2
+; SSSE3-NEXT:    pxor %xmm1, %xmm2
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm3 = [2147483903,2147483903]
+; SSSE3-NEXT:    movdqa %xmm3, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm5, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm2, %xmm3
+; SSSE3-NEXT:    pand %xmm3, %xmm0
+; SSSE3-NEXT:    pandn {{.*}}(%rip), %xmm3
+; SSSE3-NEXT:    por %xmm0, %xmm3
+; SSSE3-NEXT:    movdqa %xmm3, %xmm0
+; SSSE3-NEXT:    pxor %xmm1, %xmm0
+; SSSE3-NEXT:    movdqa %xmm0, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm4, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    por %xmm0, %xmm1
+; SSSE3-NEXT:    pand %xmm3, %xmm1
+; SSSE3-NEXT:    pshufb {{.*#+}} xmm1 = xmm1[0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; SSSE3-NEXT:    movd %xmm1, %eax
+; SSSE3-NEXT:    movw %ax, (%rdi)
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: trunc_packus_v2i64_v2i8_store:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    movapd {{.*#+}} xmm2 = [255,255]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [2147483648,2147483648]
+; SSE41-NEXT:    pxor %xmm3, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [2147483903,2147483903]
+; SSE41-NEXT:    movdqa %xmm4, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm0
+; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
+; SSE41-NEXT:    xorpd %xmm1, %xmm1
+; SSE41-NEXT:    movapd %xmm2, %xmm4
+; SSE41-NEXT:    xorpd %xmm3, %xmm4
+; SSE41-NEXT:    movapd %xmm4, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm3, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm0
+; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
+; SSE41-NEXT:    pshufb {{.*#+}} xmm1 = xmm1[0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; SSE41-NEXT:    pextrw $0, %xmm1, (%rdi)
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: trunc_packus_v2i64_v2i8_store:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [255,255]
+; AVX-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm2
+; AVX-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm1
+; AVX-NEXT:    vpand %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX-NEXT:    vpextrw $0, %xmm0, (%rdi)
+; AVX-NEXT:    retq
+;
+; AVX512F-LABEL: trunc_packus_v2i64_v2i8_store:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vpmovusqb %zmm0, %xmm0
+; AVX512F-NEXT:    vpextrw $0, %xmm0, (%rdi)
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: trunc_packus_v2i64_v2i8_store:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512VL-NEXT:    vpmaxsq %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpmovusqb %xmm0, (%rdi)
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: trunc_packus_v2i64_v2i8_store:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512BW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512BW-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmovusqb %zmm0, %xmm0
+; AVX512BW-NEXT:    vpextrw $0, %xmm0, (%rdi)
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: trunc_packus_v2i64_v2i8_store:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512BWVL-NEXT:    vpmaxsq %xmm1, %xmm0, %xmm0
+; AVX512BWVL-NEXT:    vpmovusqb %xmm0, (%rdi)
+; AVX512BWVL-NEXT:    retq
+;
+; SKX-LABEL: trunc_packus_v2i64_v2i8_store:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; SKX-NEXT:    vpmaxsq %xmm1, %xmm0, %xmm0
+; SKX-NEXT:    vpmovusqb %xmm0, (%rdi)
+; SKX-NEXT:    retq
+  %1 = icmp slt <2 x i64> %a0, <i64 255, i64 255>
+  %2 = select <2 x i1> %1, <2 x i64> %a0, <2 x i64> <i64 255, i64 255>
+  %3 = icmp sgt <2 x i64> %2, zeroinitializer
+  %4 = select <2 x i1> %3, <2 x i64> %2, <2 x i64> zeroinitializer
+  %5 = trunc <2 x i64> %4 to <2 x i8>
+  store <2 x i8> %5, <2 x i8> *%p1
+  ret void
+}
+
+define <4 x i8> @trunc_packus_v4i64_v4i8(<4 x i64> %a0) {
+; SSE2-LABEL: trunc_packus_v4i64_v4i8:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [255,255]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm1, %xmm4
+; SSE2-NEXT:    pxor %xmm3, %xmm4
+; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [2147483903,2147483903]
+; SSE2-NEXT:    movdqa %xmm5, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3]
+; SSE2-NEXT:    pand %xmm7, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[1,1,3,3]
+; SSE2-NEXT:    por %xmm2, %xmm4
+; SSE2-NEXT:    pand %xmm4, %xmm1
+; SSE2-NEXT:    pandn %xmm8, %xmm4
+; SSE2-NEXT:    por %xmm1, %xmm4
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    pxor %xmm3, %xmm1
+; SSE2-NEXT:    movdqa %xmm5, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pand %xmm6, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,3,3]
+; SSE2-NEXT:    por %xmm5, %xmm1
+; SSE2-NEXT:    pand %xmm1, %xmm0
+; SSE2-NEXT:    pandn %xmm8, %xmm1
+; SSE2-NEXT:    por %xmm0, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    pxor %xmm3, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm3, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pand %xmm5, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3]
+; SSE2-NEXT:    por %xmm6, %xmm0
+; SSE2-NEXT:    movdqa %xmm4, %xmm2
+; SSE2-NEXT:    pxor %xmm3, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm3, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE2-NEXT:    pand %xmm6, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm5[1,1,3,3]
+; SSE2-NEXT:    por %xmm2, %xmm3
+; SSE2-NEXT:    pand %xmm8, %xmm3
+; SSE2-NEXT:    pand %xmm4, %xmm3
+; SSE2-NEXT:    pand %xmm8, %xmm0
+; SSE2-NEXT:    pand %xmm1, %xmm0
+; SSE2-NEXT:    packuswb %xmm3, %xmm0
+; SSE2-NEXT:    packuswb %xmm0, %xmm0
+; SSE2-NEXT:    packuswb %xmm0, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: trunc_packus_v4i64_v4i8:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm8 = [255,255]
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
+; SSSE3-NEXT:    movdqa %xmm1, %xmm3
+; SSSE3-NEXT:    pxor %xmm2, %xmm3
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm5 = [2147483903,2147483903]
+; SSSE3-NEXT:    movdqa %xmm5, %xmm6
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm7, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm6[1,1,3,3]
+; SSSE3-NEXT:    por %xmm4, %xmm3
+; SSSE3-NEXT:    pand %xmm3, %xmm1
+; SSSE3-NEXT:    pandn %xmm8, %xmm3
+; SSSE3-NEXT:    por %xmm1, %xmm3
+; SSSE3-NEXT:    movdqa %xmm0, %xmm1
+; SSSE3-NEXT:    pxor %xmm2, %xmm1
+; SSSE3-NEXT:    movdqa %xmm5, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm4
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm6, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    por %xmm3, %xmm4
-; SSSE3-NEXT:    pand %xmm1, %xmm4
-; SSSE3-NEXT:    movdqa %xmm5, %xmm1
-; SSSE3-NEXT:    pxor %xmm10, %xmm1
-; SSSE3-NEXT:    movdqa %xmm1, %xmm3
-; SSSE3-NEXT:    pcmpgtd %xmm10, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm1
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm1
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
 ; SSSE3-NEXT:    pand %xmm6, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm1, %xmm3
-; SSSE3-NEXT:    pand %xmm5, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[0,2,2,3]
-; SSSE3-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[0,2,2,3]
-; SSSE3-NEXT:    pshuflw {{.*#+}} xmm3 = xmm3[0,2,2,3,4,5,6,7]
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
-; SSSE3-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,1,0,2,4,5,6,7]
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; SSSE3-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,1,0,2,4,5,6,7]
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; SSSE3-NEXT:    movsd {{.*#+}} xmm0 = xmm3[0],xmm0[1]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm1, %xmm4
+; SSSE3-NEXT:    pand %xmm4, %xmm0
+; SSSE3-NEXT:    pandn %xmm8, %xmm4
+; SSSE3-NEXT:    por %xmm0, %xmm4
+; SSSE3-NEXT:    movdqa %xmm4, %xmm0
+; SSSE3-NEXT:    pxor %xmm2, %xmm0
+; SSSE3-NEXT:    movdqa %xmm0, %xmm1
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm5, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    por %xmm6, %xmm0
+; SSSE3-NEXT:    pand %xmm4, %xmm0
+; SSSE3-NEXT:    movdqa %xmm3, %xmm1
+; SSSE3-NEXT:    pxor %xmm2, %xmm1
+; SSSE3-NEXT:    movdqa %xmm1, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm5, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm1, %xmm2
+; SSSE3-NEXT:    pand %xmm3, %xmm2
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = <0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; SSSE3-NEXT:    pshufb %xmm1, %xmm2
+; SSSE3-NEXT:    pshufb %xmm1, %xmm0
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
 ; SSSE3-NEXT:    retq
 ;
-; SSE41-LABEL: trunc_packus_v8i64_v8i16:
+; SSE41-LABEL: trunc_packus_v4i64_v4i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa %xmm0, %xmm9
-; SSE41-NEXT:    movapd {{.*#+}} xmm7 = [65535,65535]
-; SSE41-NEXT:    movdqa {{.*#+}} xmm10 = [2147483648,2147483648]
-; SSE41-NEXT:    movdqa %xmm2, %xmm0
-; SSE41-NEXT:    pxor %xmm10, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm6 = [2147549183,2147549183]
-; SSE41-NEXT:    movdqa %xmm6, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm6, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
-; SSE41-NEXT:    movapd %xmm7, %xmm8
-; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm8
-; SSE41-NEXT:    movdqa %xmm3, %xmm0
-; SSE41-NEXT:    pxor %xmm10, %xmm0
-; SSE41-NEXT:    movdqa %xmm6, %xmm2
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm6, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm4, %xmm0
-; SSE41-NEXT:    por %xmm2, %xmm0
-; SSE41-NEXT:    movapd %xmm7, %xmm2
-; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm2
-; SSE41-NEXT:    movdqa %xmm9, %xmm0
-; SSE41-NEXT:    pxor %xmm10, %xmm0
-; SSE41-NEXT:    movdqa %xmm6, %xmm3
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm6, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm4, %xmm0
-; SSE41-NEXT:    por %xmm3, %xmm0
-; SSE41-NEXT:    movapd %xmm7, %xmm4
-; SSE41-NEXT:    blendvpd %xmm0, %xmm9, %xmm4
+; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    movapd {{.*#+}} xmm4 = [255,255]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [2147483648,2147483648]
 ; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    pxor %xmm10, %xmm0
-; SSE41-NEXT:    movdqa %xmm6, %xmm3
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm6, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm0
-; SSE41-NEXT:    por %xmm3, %xmm0
-; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm7
-; SSE41-NEXT:    pxor %xmm3, %xmm3
-; SSE41-NEXT:    movapd %xmm7, %xmm0
-; SSE41-NEXT:    xorpd %xmm10, %xmm0
-; SSE41-NEXT:    movapd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd %xmm10, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE41-NEXT:    pxor %xmm3, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm6 = [2147483903,2147483903]
+; SSE41-NEXT:    movdqa %xmm6, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm5
+; SSE41-NEXT:    movdqa %xmm6, %xmm7
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm7
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[0,0,2,2]
 ; SSE41-NEXT:    pand %xmm5, %xmm0
-; SSE41-NEXT:    por %xmm1, %xmm0
-; SSE41-NEXT:    pxor %xmm5, %xmm5
-; SSE41-NEXT:    blendvpd %xmm0, %xmm7, %xmm5
-; SSE41-NEXT:    movapd %xmm4, %xmm0
-; SSE41-NEXT:    xorpd %xmm10, %xmm0
-; SSE41-NEXT:    movapd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd %xmm10, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE41-NEXT:    por %xmm7, %xmm0
+; SSE41-NEXT:    movapd %xmm4, %xmm5
+; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm5
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    pxor %xmm3, %xmm0
+; SSE41-NEXT:    movdqa %xmm6, %xmm1
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
+; SSE41-NEXT:    pand %xmm1, %xmm0
+; SSE41-NEXT:    por %xmm6, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm4
+; SSE41-NEXT:    xorpd %xmm2, %xmm2
+; SSE41-NEXT:    movapd %xmm4, %xmm1
+; SSE41-NEXT:    xorpd %xmm3, %xmm1
+; SSE41-NEXT:    movapd %xmm1, %xmm6
+; SSE41-NEXT:    pcmpeqd %xmm3, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2]
 ; SSE41-NEXT:    pand %xmm6, %xmm0
 ; SSE41-NEXT:    por %xmm1, %xmm0
 ; SSE41-NEXT:    pxor %xmm1, %xmm1
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm1
-; SSE41-NEXT:    packusdw %xmm5, %xmm1
-; SSE41-NEXT:    movapd %xmm2, %xmm0
-; SSE41-NEXT:    xorpd %xmm10, %xmm0
-; SSE41-NEXT:    movapd %xmm0, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm10, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm0
+; SSE41-NEXT:    movapd %xmm5, %xmm4
+; SSE41-NEXT:    xorpd %xmm3, %xmm4
+; SSE41-NEXT:    movapd %xmm4, %xmm6
+; SSE41-NEXT:    pcmpeqd %xmm3, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pand %xmm6, %xmm0
 ; SSE41-NEXT:    por %xmm4, %xmm0
-; SSE41-NEXT:    pxor %xmm4, %xmm4
-; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm4
-; SSE41-NEXT:    movapd %xmm8, %xmm0
-; SSE41-NEXT:    xorpd %xmm10, %xmm0
-; SSE41-NEXT:    movapd %xmm0, %xmm2
-; SSE41-NEXT:    pcmpgtd %xmm10, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm0
-; SSE41-NEXT:    por %xmm2, %xmm0
-; SSE41-NEXT:    blendvpd %xmm0, %xmm8, %xmm3
-; SSE41-NEXT:    packusdw %xmm4, %xmm3
-; SSE41-NEXT:    packusdw %xmm3, %xmm1
+; SSE41-NEXT:    blendvpd %xmm0, %xmm5, %xmm2
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = <0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; SSE41-NEXT:    pshufb %xmm0, %xmm2
+; SSE41-NEXT:    pshufb %xmm0, %xmm1
+; SSE41-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
 ; SSE41-NEXT:    movdqa %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: trunc_packus_v8i64_v8i16:
+; AVX1-LABEL: trunc_packus_v4i64_v4i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [65535,65535]
-; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm3, %xmm8
-; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm3, %xmm5
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm6
-; AVX1-NEXT:    vpcmpgtq %xmm6, %xmm3, %xmm7
-; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm3, %xmm4
-; AVX1-NEXT:    vblendvpd %xmm4, %xmm0, %xmm3, %xmm0
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [255,255]
+; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm2, %xmm3
+; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm4
+; AVX1-NEXT:    vblendvpd %xmm4, %xmm0, %xmm2, %xmm0
 ; AVX1-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm0, %xmm9
-; AVX1-NEXT:    vblendvpd %xmm7, %xmm6, %xmm3, %xmm6
-; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm6, %xmm7
-; AVX1-NEXT:    vblendvpd %xmm5, %xmm1, %xmm3, %xmm1
-; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm1, %xmm5
-; AVX1-NEXT:    vblendvpd %xmm8, %xmm2, %xmm3, %xmm2
-; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm2, %xmm3
-; AVX1-NEXT:    vpand %xmm2, %xmm3, %xmm2
-; AVX1-NEXT:    vpand %xmm1, %xmm5, %xmm1
-; AVX1-NEXT:    vpackusdw %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpand %xmm6, %xmm7, %xmm2
-; AVX1-NEXT:    vpand %xmm0, %xmm9, %xmm0
-; AVX1-NEXT:    vpackusdw %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm0, %xmm5
+; AVX1-NEXT:    vblendvpd %xmm3, %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm1, %xmm2
+; AVX1-NEXT:    vpand %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = <0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX1-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpand %xmm0, %xmm5, %xmm0
+; AVX1-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
-; AVX2-LABEL: trunc_packus_v8i64_v8i16:
+; AVX2-LABEL: trunc_packus_v4i64_v4i8:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [65535,65535,65535,65535]
-; AVX2-NEXT:    vpcmpgtq %ymm0, %ymm2, %ymm3
-; AVX2-NEXT:    vblendvpd %ymm3, %ymm0, %ymm2, %ymm0
-; AVX2-NEXT:    vpcmpgtq %ymm1, %ymm2, %ymm3
-; AVX2-NEXT:    vblendvpd %ymm3, %ymm1, %ymm2, %ymm1
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtq %ymm2, %ymm1, %ymm3
-; AVX2-NEXT:    vpand %ymm1, %ymm3, %ymm1
-; AVX2-NEXT:    vpcmpgtq %ymm2, %ymm0, %ymm2
-; AVX2-NEXT:    vpand %ymm0, %ymm2, %ymm0
-; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [255,255,255,255]
+; AVX2-NEXT:    vpcmpgtq %ymm0, %ymm1, %ymm2
+; AVX2-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT:    vpcmpgtq %ymm1, %ymm0, %ymm1
+; AVX2-NEXT:    vpand %ymm0, %ymm1, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = <0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX2-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
-; AVX512-LABEL: trunc_packus_v8i64_v8i16:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm0
-; AVX512-NEXT:    vpmovusqw %zmm0, %xmm0
-; AVX512-NEXT:    vzeroupper
-; AVX512-NEXT:    retq
-  %1 = icmp slt <8 x i64> %a0, <i64 65535, i64 65535, i64 65535, i64 65535, i64 65535, i64 65535, i64 65535, i64 65535>
-  %2 = select <8 x i1> %1, <8 x i64> %a0, <8 x i64> <i64 65535, i64 65535, i64 65535, i64 65535, i64 65535, i64 65535, i64 65535, i64 65535>
-  %3 = icmp sgt <8 x i64> %2, zeroinitializer
-  %4 = select <8 x i1> %3, <8 x i64> %2, <8 x i64> zeroinitializer
-  %5 = trunc <8 x i64> %4 to <8 x i16>
-  ret <8 x i16> %5
+; AVX512F-LABEL: trunc_packus_v4i64_v4i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vpmovusqb %zmm0, %xmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: trunc_packus_v4i64_v4i8:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512VL-NEXT:    vpmaxsq %ymm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpmovusqb %ymm0, %xmm0
+; AVX512VL-NEXT:    vzeroupper
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: trunc_packus_v4i64_v4i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512BW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512BW-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmovusqb %zmm0, %xmm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: trunc_packus_v4i64_v4i8:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512BWVL-NEXT:    vpmaxsq %ymm1, %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpmovusqb %ymm0, %xmm0
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
+;
+; SKX-LABEL: trunc_packus_v4i64_v4i8:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; SKX-NEXT:    vpmaxsq %ymm1, %ymm0, %ymm0
+; SKX-NEXT:    vpmovusqb %ymm0, %xmm0
+; SKX-NEXT:    vzeroupper
+; SKX-NEXT:    retq
+  %1 = icmp slt <4 x i64> %a0, <i64 255, i64 255, i64 255, i64 255>
+  %2 = select <4 x i1> %1, <4 x i64> %a0, <4 x i64> <i64 255, i64 255, i64 255, i64 255>
+  %3 = icmp sgt <4 x i64> %2, zeroinitializer
+  %4 = select <4 x i1> %3, <4 x i64> %2, <4 x i64> zeroinitializer
+  %5 = trunc <4 x i64> %4 to <4 x i8>
+  ret <4 x i8> %5
 }
 
-define <8 x i16> @trunc_packus_v8i32_v8i16(<8 x i32> %a0) {
-; SSE2-LABEL: trunc_packus_v8i32_v8i16:
+define void @trunc_packus_v4i64_v4i8_store(<4 x i64> %a0, <4 x i8> *%p1) {
+; SSE2-LABEL: trunc_packus_v4i64_v4i8_store:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [65535,65535,65535,65535]
-; SSE2-NEXT:    movdqa %xmm2, %xmm3
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
-; SSE2-NEXT:    pand %xmm3, %xmm1
-; SSE2-NEXT:    pandn %xmm2, %xmm3
-; SSE2-NEXT:    por %xmm1, %xmm3
-; SSE2-NEXT:    movdqa %xmm2, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
+; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [255,255]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm1, %xmm4
+; SSE2-NEXT:    pxor %xmm3, %xmm4
+; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [2147483903,2147483903]
+; SSE2-NEXT:    movdqa %xmm5, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3]
+; SSE2-NEXT:    pand %xmm7, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[1,1,3,3]
+; SSE2-NEXT:    por %xmm2, %xmm4
+; SSE2-NEXT:    pand %xmm4, %xmm1
+; SSE2-NEXT:    pandn %xmm8, %xmm4
+; SSE2-NEXT:    por %xmm1, %xmm4
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    pxor %xmm3, %xmm1
+; SSE2-NEXT:    movdqa %xmm5, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pand %xmm6, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,3,3]
+; SSE2-NEXT:    por %xmm5, %xmm1
 ; SSE2-NEXT:    pand %xmm1, %xmm0
-; SSE2-NEXT:    pandn %xmm2, %xmm1
+; SSE2-NEXT:    pandn %xmm8, %xmm1
 ; SSE2-NEXT:    por %xmm0, %xmm1
-; SSE2-NEXT:    pxor %xmm2, %xmm2
 ; SSE2-NEXT:    movdqa %xmm1, %xmm0
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm0
-; SSE2-NEXT:    pand %xmm1, %xmm0
-; SSE2-NEXT:    movdqa %xmm3, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm1
-; SSE2-NEXT:    pand %xmm3, %xmm1
-; SSE2-NEXT:    pslld $16, %xmm1
-; SSE2-NEXT:    psrad $16, %xmm1
-; SSE2-NEXT:    pslld $16, %xmm0
-; SSE2-NEXT:    psrad $16, %xmm0
-; SSE2-NEXT:    packssdw %xmm1, %xmm0
+; SSE2-NEXT:    pxor %xmm3, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm3, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pand %xmm5, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE2-NEXT:    por %xmm0, %xmm2
+; SSE2-NEXT:    movdqa %xmm4, %xmm0
+; SSE2-NEXT:    pxor %xmm3, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm3, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pand %xmm6, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm5[1,1,3,3]
+; SSE2-NEXT:    por %xmm0, %xmm3
+; SSE2-NEXT:    pand %xmm8, %xmm3
+; SSE2-NEXT:    pand %xmm4, %xmm3
+; SSE2-NEXT:    pand %xmm8, %xmm2
+; SSE2-NEXT:    pand %xmm1, %xmm2
+; SSE2-NEXT:    packuswb %xmm3, %xmm2
+; SSE2-NEXT:    packuswb %xmm0, %xmm2
+; SSE2-NEXT:    packuswb %xmm0, %xmm2
+; SSE2-NEXT:    movd %xmm2, (%rdi)
 ; SSE2-NEXT:    retq
 ;
-; SSSE3-LABEL: trunc_packus_v8i32_v8i16:
+; SSSE3-LABEL: trunc_packus_v4i64_v4i8_store:
 ; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [65535,65535,65535,65535]
-; SSSE3-NEXT:    movdqa %xmm2, %xmm3
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm3
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm8 = [255,255]
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
+; SSSE3-NEXT:    movdqa %xmm1, %xmm3
+; SSSE3-NEXT:    pxor %xmm2, %xmm3
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm5 = [2147483903,2147483903]
+; SSSE3-NEXT:    movdqa %xmm5, %xmm6
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm7, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm6[1,1,3,3]
+; SSSE3-NEXT:    por %xmm4, %xmm3
 ; SSSE3-NEXT:    pand %xmm3, %xmm1
-; SSSE3-NEXT:    pandn %xmm2, %xmm3
+; SSSE3-NEXT:    pandn %xmm8, %xmm3
 ; SSSE3-NEXT:    por %xmm1, %xmm3
-; SSSE3-NEXT:    movdqa %xmm2, %xmm1
-; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm1
-; SSSE3-NEXT:    pand %xmm1, %xmm0
-; SSSE3-NEXT:    pandn %xmm2, %xmm1
-; SSSE3-NEXT:    por %xmm0, %xmm1
-; SSSE3-NEXT:    pxor %xmm2, %xmm2
-; SSSE3-NEXT:    movdqa %xmm1, %xmm0
-; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm0
-; SSSE3-NEXT:    pand %xmm1, %xmm0
-; SSSE3-NEXT:    movdqa %xmm3, %xmm1
+; SSSE3-NEXT:    movdqa %xmm0, %xmm1
+; SSSE3-NEXT:    pxor %xmm2, %xmm1
+; SSSE3-NEXT:    movdqa %xmm5, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm6, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm1, %xmm4
+; SSSE3-NEXT:    pand %xmm4, %xmm0
+; SSSE3-NEXT:    pandn %xmm8, %xmm4
+; SSSE3-NEXT:    por %xmm0, %xmm4
+; SSSE3-NEXT:    movdqa %xmm4, %xmm0
+; SSSE3-NEXT:    pxor %xmm2, %xmm0
+; SSSE3-NEXT:    movdqa %xmm0, %xmm1
 ; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm1
-; SSSE3-NEXT:    pand %xmm3, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
-; SSSE3-NEXT:    pshufb %xmm2, %xmm1
-; SSSE3-NEXT:    pshufb %xmm2, %xmm0
-; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm5, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    por %xmm0, %xmm1
+; SSSE3-NEXT:    pand %xmm4, %xmm1
+; SSSE3-NEXT:    movdqa %xmm3, %xmm0
+; SSSE3-NEXT:    pxor %xmm2, %xmm0
+; SSSE3-NEXT:    movdqa %xmm0, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm5, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm0, %xmm2
+; SSSE3-NEXT:    pand %xmm3, %xmm2
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = <0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; SSSE3-NEXT:    pshufb %xmm0, %xmm2
+; SSSE3-NEXT:    pshufb %xmm0, %xmm1
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; SSSE3-NEXT:    movd %xmm1, (%rdi)
 ; SSSE3-NEXT:    retq
 ;
-; SSE41-LABEL: trunc_packus_v8i32_v8i16:
+; SSE41-LABEL: trunc_packus_v4i64_v4i8_store:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    packusdw %xmm1, %xmm0
+; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    movapd {{.*#+}} xmm4 = [255,255]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [2147483648,2147483648]
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pxor %xmm3, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm6 = [2147483903,2147483903]
+; SSE41-NEXT:    movdqa %xmm6, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm5
+; SSE41-NEXT:    movdqa %xmm6, %xmm7
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm7
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm0
+; SSE41-NEXT:    por %xmm7, %xmm0
+; SSE41-NEXT:    movapd %xmm4, %xmm5
+; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm5
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    pxor %xmm3, %xmm0
+; SSE41-NEXT:    movdqa %xmm6, %xmm1
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
+; SSE41-NEXT:    pand %xmm1, %xmm0
+; SSE41-NEXT:    por %xmm6, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm4
+; SSE41-NEXT:    pxor %xmm1, %xmm1
+; SSE41-NEXT:    movapd %xmm4, %xmm2
+; SSE41-NEXT:    xorpd %xmm3, %xmm2
+; SSE41-NEXT:    movapd %xmm2, %xmm6
+; SSE41-NEXT:    pcmpeqd %xmm3, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,0,2,2]
+; SSE41-NEXT:    pand %xmm6, %xmm0
+; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    pxor %xmm2, %xmm2
+; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm2
+; SSE41-NEXT:    movapd %xmm5, %xmm4
+; SSE41-NEXT:    xorpd %xmm3, %xmm4
+; SSE41-NEXT:    movapd %xmm4, %xmm6
+; SSE41-NEXT:    pcmpeqd %xmm3, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pand %xmm6, %xmm0
+; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm5, %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = <0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; SSE41-NEXT:    pshufb %xmm0, %xmm1
+; SSE41-NEXT:    pshufb %xmm0, %xmm2
+; SSE41-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; SSE41-NEXT:    movd %xmm2, (%rdi)
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: trunc_packus_v8i32_v8i16:
+; AVX1-LABEL: trunc_packus_v4i64_v4i8_store:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [255,255]
+; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm2, %xmm3
+; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm4
+; AVX1-NEXT:    vblendvpd %xmm4, %xmm0, %xmm2, %xmm0
+; AVX1-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm0, %xmm5
+; AVX1-NEXT:    vblendvpd %xmm3, %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm1, %xmm2
+; AVX1-NEXT:    vpand %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = <0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX1-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpand %xmm0, %xmm5, %xmm0
+; AVX1-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX1-NEXT:    vmovd %xmm0, (%rdi)
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
-; AVX2-LABEL: trunc_packus_v8i32_v8i16:
+; AVX2-LABEL: trunc_packus_v4i64_v4i8_store:
 ; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [255,255,255,255]
+; AVX2-NEXT:    vpcmpgtq %ymm0, %ymm1, %ymm2
+; AVX2-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT:    vpcmpgtq %ymm1, %ymm0, %ymm1
+; AVX2-NEXT:    vpand %ymm0, %ymm1, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = <0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX2-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX2-NEXT:    vmovd %xmm0, (%rdi)
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
-; AVX512F-LABEL: trunc_packus_v8i32_v8i16:
+; AVX512F-LABEL: trunc_packus_v4i64_v4i8_store:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [65535,65535,65535,65535,65535,65535,65535,65535]
-; AVX512F-NEXT:    vpminsd %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
 ; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT:    vpmaxsd %ymm1, %ymm0, %ymm0
-; AVX512F-NEXT:    vpmovdw %zmm0, %ymm0
-; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512F-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vpmovusqb %zmm0, %xmm0
+; AVX512F-NEXT:    vmovd %xmm0, (%rdi)
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
-; AVX512VL-LABEL: trunc_packus_v8i32_v8i16:
+; AVX512VL-LABEL: trunc_packus_v4i64_v4i8_store:
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpmaxsd %ymm1, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpmovusdw %ymm0, %xmm0
+; AVX512VL-NEXT:    vpmaxsq %ymm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpmovusqb %ymm0, (%rdi)
 ; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq
 ;
-; AVX512BW-LABEL: trunc_packus_v8i32_v8i16:
+; AVX512BW-LABEL: trunc_packus_v4i64_v4i8_store:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [65535,65535,65535,65535,65535,65535,65535,65535]
-; AVX512BW-NEXT:    vpminsd %ymm1, %ymm0, %ymm0
+; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
 ; AVX512BW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpmaxsd %ymm1, %ymm0, %ymm0
-; AVX512BW-NEXT:    vpmovdw %zmm0, %ymm0
-; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512BW-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmovusqb %zmm0, %xmm0
+; AVX512BW-NEXT:    vmovd %xmm0, (%rdi)
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
-; AVX512BWVL-LABEL: trunc_packus_v8i32_v8i16:
+; AVX512BWVL-LABEL: trunc_packus_v4i64_v4i8_store:
 ; AVX512BWVL:       # %bb.0:
 ; AVX512BWVL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512BWVL-NEXT:    vpmaxsd %ymm1, %ymm0, %ymm0
-; AVX512BWVL-NEXT:    vpmovusdw %ymm0, %xmm0
+; AVX512BWVL-NEXT:    vpmaxsq %ymm1, %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpmovusqb %ymm0, (%rdi)
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
-  %1 = icmp slt <8 x i32> %a0, <i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535>
-  %2 = select <8 x i1> %1, <8 x i32> %a0, <8 x i32> <i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535>
-  %3 = icmp sgt <8 x i32> %2, zeroinitializer
-  %4 = select <8 x i1> %3, <8 x i32> %2, <8 x i32> zeroinitializer
-  %5 = trunc <8 x i32> %4 to <8 x i16>
-  ret <8 x i16> %5
-}
-
-define <16 x i16> @trunc_packus_v16i32_v16i16(<16 x i32> %a0) {
-; SSE2-LABEL: trunc_packus_v16i32_v16i16:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm6 = [65535,65535,65535,65535]
-; SSE2-NEXT:    movdqa %xmm6, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm4
-; SSE2-NEXT:    pand %xmm4, %xmm1
-; SSE2-NEXT:    pandn %xmm6, %xmm4
-; SSE2-NEXT:    por %xmm1, %xmm4
-; SSE2-NEXT:    movdqa %xmm6, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm5
-; SSE2-NEXT:    pand %xmm5, %xmm0
-; SSE2-NEXT:    pandn %xmm6, %xmm5
-; SSE2-NEXT:    por %xmm0, %xmm5
-; SSE2-NEXT:    movdqa %xmm6, %xmm0
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE2-NEXT:    pand %xmm0, %xmm3
-; SSE2-NEXT:    pandn %xmm6, %xmm0
-; SSE2-NEXT:    por %xmm3, %xmm0
-; SSE2-NEXT:    movdqa %xmm6, %xmm3
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm3
-; SSE2-NEXT:    pand %xmm3, %xmm2
-; SSE2-NEXT:    pandn %xmm6, %xmm3
-; SSE2-NEXT:    por %xmm2, %xmm3
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    movdqa %xmm3, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm1
-; SSE2-NEXT:    pand %xmm3, %xmm1
-; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm3
-; SSE2-NEXT:    pand %xmm0, %xmm3
-; SSE2-NEXT:    movdqa %xmm5, %xmm0
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm0
-; SSE2-NEXT:    pand %xmm5, %xmm0
-; SSE2-NEXT:    movdqa %xmm4, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm5
-; SSE2-NEXT:    pand %xmm4, %xmm5
-; SSE2-NEXT:    pslld $16, %xmm5
-; SSE2-NEXT:    psrad $16, %xmm5
-; SSE2-NEXT:    pslld $16, %xmm0
-; SSE2-NEXT:    psrad $16, %xmm0
-; SSE2-NEXT:    packssdw %xmm5, %xmm0
-; SSE2-NEXT:    pslld $16, %xmm3
-; SSE2-NEXT:    psrad $16, %xmm3
-; SSE2-NEXT:    pslld $16, %xmm1
-; SSE2-NEXT:    psrad $16, %xmm1
-; SSE2-NEXT:    packssdw %xmm3, %xmm1
-; SSE2-NEXT:    retq
-;
-; SSSE3-LABEL: trunc_packus_v16i32_v16i16:
-; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm6 = [65535,65535,65535,65535]
-; SSSE3-NEXT:    movdqa %xmm6, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm4
-; SSSE3-NEXT:    pand %xmm4, %xmm1
-; SSSE3-NEXT:    pandn %xmm6, %xmm4
-; SSSE3-NEXT:    por %xmm1, %xmm4
-; SSSE3-NEXT:    movdqa %xmm6, %xmm5
-; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm5
-; SSSE3-NEXT:    pand %xmm5, %xmm0
-; SSSE3-NEXT:    pandn %xmm6, %xmm5
-; SSSE3-NEXT:    por %xmm0, %xmm5
-; SSSE3-NEXT:    movdqa %xmm6, %xmm0
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSSE3-NEXT:    pand %xmm0, %xmm3
-; SSSE3-NEXT:    pandn %xmm6, %xmm0
-; SSSE3-NEXT:    por %xmm3, %xmm0
-; SSSE3-NEXT:    movdqa %xmm6, %xmm3
-; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm3
-; SSSE3-NEXT:    pand %xmm3, %xmm2
-; SSSE3-NEXT:    pandn %xmm6, %xmm3
-; SSSE3-NEXT:    por %xmm2, %xmm3
-; SSSE3-NEXT:    pxor %xmm2, %xmm2
-; SSSE3-NEXT:    movdqa %xmm3, %xmm1
-; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm1
-; SSSE3-NEXT:    pand %xmm3, %xmm1
-; SSSE3-NEXT:    movdqa %xmm0, %xmm3
-; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm3
-; SSSE3-NEXT:    pand %xmm0, %xmm3
-; SSSE3-NEXT:    movdqa %xmm5, %xmm0
-; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm0
-; SSSE3-NEXT:    pand %xmm5, %xmm0
-; SSSE3-NEXT:    movdqa %xmm4, %xmm5
-; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm5
-; SSSE3-NEXT:    pand %xmm4, %xmm5
-; SSSE3-NEXT:    pslld $16, %xmm5
-; SSSE3-NEXT:    psrad $16, %xmm5
-; SSSE3-NEXT:    pslld $16, %xmm0
-; SSSE3-NEXT:    psrad $16, %xmm0
-; SSSE3-NEXT:    packssdw %xmm5, %xmm0
-; SSSE3-NEXT:    pslld $16, %xmm3
-; SSSE3-NEXT:    psrad $16, %xmm3
-; SSSE3-NEXT:    pslld $16, %xmm1
-; SSSE3-NEXT:    psrad $16, %xmm1
-; SSSE3-NEXT:    packssdw %xmm3, %xmm1
-; SSSE3-NEXT:    retq
-;
-; SSE41-LABEL: trunc_packus_v16i32_v16i16:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    packusdw %xmm1, %xmm0
-; SSE41-NEXT:    packusdw %xmm3, %xmm2
-; SSE41-NEXT:    movdqa %xmm2, %xmm1
-; SSE41-NEXT:    retq
-;
-; AVX1-LABEL: trunc_packus_v16i32_v16i16:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT:    vpackusdw %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX1-NEXT:    vpackusdw %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: trunc_packus_v16i32_v16i16:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX2-NEXT:    retq
 ;
-; AVX512-LABEL: trunc_packus_v16i32_v16i16:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512-NEXT:    vpmaxsd %zmm1, %zmm0, %zmm0
-; AVX512-NEXT:    vpmovusdw %zmm0, %ymm0
-; AVX512-NEXT:    retq
-  %1 = icmp slt <16 x i32> %a0, <i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535>
-  %2 = select <16 x i1> %1, <16 x i32> %a0, <16 x i32> <i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535>
-  %3 = icmp sgt <16 x i32> %2, zeroinitializer
-  %4 = select <16 x i1> %3, <16 x i32> %2, <16 x i32> zeroinitializer
-  %5 = trunc <16 x i32> %4 to <16 x i16>
-  ret <16 x i16> %5
+; SKX-LABEL: trunc_packus_v4i64_v4i8_store:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; SKX-NEXT:    vpmaxsq %ymm1, %ymm0, %ymm0
+; SKX-NEXT:    vpmovusqb %ymm0, (%rdi)
+; SKX-NEXT:    vzeroupper
+; SKX-NEXT:    retq
+  %1 = icmp slt <4 x i64> %a0, <i64 255, i64 255, i64 255, i64 255>
+  %2 = select <4 x i1> %1, <4 x i64> %a0, <4 x i64> <i64 255, i64 255, i64 255, i64 255>
+  %3 = icmp sgt <4 x i64> %2, zeroinitializer
+  %4 = select <4 x i1> %3, <4 x i64> %2, <4 x i64> zeroinitializer
+  %5 = trunc <4 x i64> %4 to <4 x i8>
+  store <4 x i8> %5, <4 x i8> *%p1
+  ret void
 }
 
-;
-; PACKUS saturation truncation to v16i8
-;
-
-define <8 x i8> @trunc_packus_v8i64_v8i8(<8 x i64> %a0) {
+define <8 x i8> @trunc_packus_v8i64_v8i8(<8 x i64>* %p0) "min-legal-vector-width"="256" {
 ; SSE2-LABEL: trunc_packus_v8i64_v8i8:
 ; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa (%rdi), %xmm5
+; SSE2-NEXT:    movdqa 16(%rdi), %xmm9
+; SSE2-NEXT:    movdqa 32(%rdi), %xmm3
+; SSE2-NEXT:    movdqa 48(%rdi), %xmm7
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [255,255]
-; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [2147483648,2147483648]
-; SSE2-NEXT:    movdqa %xmm2, %xmm5
-; SSE2-NEXT:    pxor %xmm10, %xmm5
-; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [2147483903,2147483903]
-; SSE2-NEXT:    movdqa %xmm9, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm5, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm7[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm5[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm7[1,1,3,3]
-; SSE2-NEXT:    por %xmm4, %xmm5
-; SSE2-NEXT:    pand %xmm5, %xmm2
-; SSE2-NEXT:    pandn %xmm8, %xmm5
-; SSE2-NEXT:    por %xmm2, %xmm5
+; SSE2-NEXT:    movdqa {{.*#+}} xmm11 = [2147483648,2147483648]
 ; SSE2-NEXT:    movdqa %xmm3, %xmm2
-; SSE2-NEXT:    pxor %xmm10, %xmm2
-; SSE2-NEXT:    movdqa %xmm9, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm2[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm7, %xmm2
+; SSE2-NEXT:    pxor %xmm11, %xmm2
+; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [2147483903,2147483903]
+; SSE2-NEXT:    movdqa %xmm10, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,1,3,3]
+; SSE2-NEXT:    pand %xmm0, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[1,1,3,3]
+; SSE2-NEXT:    por %xmm4, %xmm2
 ; SSE2-NEXT:    pand %xmm2, %xmm3
 ; SSE2-NEXT:    pandn %xmm8, %xmm2
 ; SSE2-NEXT:    por %xmm3, %xmm2
-; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    pxor %xmm10, %xmm3
-; SSE2-NEXT:    movdqa %xmm9, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm3[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm7, %xmm3
-; SSE2-NEXT:    pand %xmm3, %xmm0
-; SSE2-NEXT:    pandn %xmm8, %xmm3
+; SSE2-NEXT:    movdqa %xmm7, %xmm0
+; SSE2-NEXT:    pxor %xmm11, %xmm0
+; SSE2-NEXT:    movdqa %xmm10, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pand %xmm4, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
 ; SSE2-NEXT:    por %xmm0, %xmm3
-; SSE2-NEXT:    movdqa %xmm1, %xmm0
-; SSE2-NEXT:    pxor %xmm10, %xmm0
-; SSE2-NEXT:    movdqa %xmm9, %xmm4
+; SSE2-NEXT:    pand %xmm3, %xmm7
+; SSE2-NEXT:    pandn %xmm8, %xmm3
+; SSE2-NEXT:    por %xmm7, %xmm3
+; SSE2-NEXT:    movdqa %xmm5, %xmm0
+; SSE2-NEXT:    pxor %xmm11, %xmm0
+; SSE2-NEXT:    movdqa %xmm10, %xmm4
 ; SSE2-NEXT:    pcmpgtd %xmm0, %xmm4
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm0
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm6, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm0, %xmm7
+; SSE2-NEXT:    pand %xmm7, %xmm5
+; SSE2-NEXT:    pandn %xmm8, %xmm7
+; SSE2-NEXT:    por %xmm5, %xmm7
+; SSE2-NEXT:    movdqa %xmm9, %xmm0
+; SSE2-NEXT:    pxor %xmm11, %xmm0
+; SSE2-NEXT:    movdqa %xmm10, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pand %xmm5, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
 ; SSE2-NEXT:    por %xmm0, %xmm4
-; SSE2-NEXT:    pand %xmm4, %xmm1
+; SSE2-NEXT:    pand %xmm4, %xmm9
 ; SSE2-NEXT:    pandn %xmm8, %xmm4
-; SSE2-NEXT:    por %xmm1, %xmm4
+; SSE2-NEXT:    por %xmm9, %xmm4
 ; SSE2-NEXT:    movdqa %xmm4, %xmm0
-; SSE2-NEXT:    pxor %xmm10, %xmm0
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm10, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm10, %xmm0
+; SSE2-NEXT:    pxor %xmm11, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm11, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm11, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm6, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm0, %xmm1
-; SSE2-NEXT:    pand %xmm4, %xmm1
-; SSE2-NEXT:    movdqa %xmm3, %xmm0
-; SSE2-NEXT:    pxor %xmm10, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSE2-NEXT:    por %xmm0, %xmm5
+; SSE2-NEXT:    pand %xmm4, %xmm5
+; SSE2-NEXT:    movdqa %xmm7, %xmm0
+; SSE2-NEXT:    pxor %xmm11, %xmm0
 ; SSE2-NEXT:    movdqa %xmm0, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm10, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm11, %xmm4
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm10, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm7
+; SSE2-NEXT:    pcmpeqd %xmm11, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pand %xmm6, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm7, %xmm0
-; SSE2-NEXT:    pand %xmm3, %xmm0
-; SSE2-NEXT:    packuswb %xmm1, %xmm0
+; SSE2-NEXT:    por %xmm1, %xmm0
+; SSE2-NEXT:    pand %xmm7, %xmm0
+; SSE2-NEXT:    packuswb %xmm5, %xmm0
+; SSE2-NEXT:    movdqa %xmm3, %xmm1
+; SSE2-NEXT:    pxor %xmm11, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm11, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm11, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pand %xmm5, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm1, %xmm4
+; SSE2-NEXT:    pand %xmm3, %xmm4
 ; SSE2-NEXT:    movdqa %xmm2, %xmm1
-; SSE2-NEXT:    pxor %xmm10, %xmm1
+; SSE2-NEXT:    pxor %xmm11, %xmm1
 ; SSE2-NEXT:    movdqa %xmm1, %xmm3
-; SSE2-NEXT:    pcmpgtd %xmm10, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm10, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm11, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm11, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pand %xmm4, %xmm1
+; SSE2-NEXT:    pand %xmm5, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
 ; SSE2-NEXT:    por %xmm1, %xmm3
 ; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    movdqa %xmm5, %xmm1
-; SSE2-NEXT:    pxor %xmm10, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm2
-; SSE2-NEXT:    pcmpgtd %xmm10, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm10, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pand %xmm4, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm2
-; SSE2-NEXT:    pand %xmm5, %xmm2
-; SSE2-NEXT:    packuswb %xmm3, %xmm2
-; SSE2-NEXT:    packuswb %xmm2, %xmm0
+; SSE2-NEXT:    packuswb %xmm4, %xmm3
+; SSE2-NEXT:    packuswb %xmm3, %xmm0
+; SSE2-NEXT:    packuswb %xmm0, %xmm0
 ; SSE2-NEXT:    retq
 ;
-; SSSE3-LABEL: trunc_packus_v8i64_v8i8:
-; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm8 = [255,255]
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm10 = [2147483648,2147483648]
-; SSSE3-NEXT:    movdqa %xmm2, %xmm5
-; SSSE3-NEXT:    pxor %xmm10, %xmm5
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm9 = [2147483903,2147483903]
-; SSSE3-NEXT:    movdqa %xmm9, %xmm7
-; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm7
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm7[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm5
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm5[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm6, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm7[1,1,3,3]
-; SSSE3-NEXT:    por %xmm4, %xmm5
-; SSSE3-NEXT:    pand %xmm5, %xmm2
-; SSSE3-NEXT:    pandn %xmm8, %xmm5
-; SSSE3-NEXT:    por %xmm2, %xmm5
+; SSSE3-LABEL: trunc_packus_v8i64_v8i8:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa (%rdi), %xmm5
+; SSSE3-NEXT:    movdqa 16(%rdi), %xmm9
+; SSSE3-NEXT:    movdqa 32(%rdi), %xmm3
+; SSSE3-NEXT:    movdqa 48(%rdi), %xmm7
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm8 = [255,255]
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm11 = [2147483648,2147483648]
 ; SSSE3-NEXT:    movdqa %xmm3, %xmm2
-; SSSE3-NEXT:    pxor %xmm10, %xmm2
-; SSSE3-NEXT:    movdqa %xmm9, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm2[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm6, %xmm7
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    por %xmm7, %xmm2
+; SSSE3-NEXT:    pxor %xmm11, %xmm2
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm10 = [2147483903,2147483903]
+; SSSE3-NEXT:    movdqa %xmm10, %xmm6
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm0, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[1,1,3,3]
+; SSSE3-NEXT:    por %xmm4, %xmm2
 ; SSSE3-NEXT:    pand %xmm2, %xmm3
 ; SSSE3-NEXT:    pandn %xmm8, %xmm2
 ; SSSE3-NEXT:    por %xmm3, %xmm2
-; SSSE3-NEXT:    movdqa %xmm0, %xmm3
-; SSSE3-NEXT:    pxor %xmm10, %xmm3
-; SSSE3-NEXT:    movdqa %xmm9, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm6, %xmm7
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    por %xmm7, %xmm3
-; SSSE3-NEXT:    pand %xmm3, %xmm0
-; SSSE3-NEXT:    pandn %xmm8, %xmm3
+; SSSE3-NEXT:    movdqa %xmm7, %xmm0
+; SSSE3-NEXT:    pxor %xmm11, %xmm0
+; SSSE3-NEXT:    movdqa %xmm10, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm4, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
 ; SSSE3-NEXT:    por %xmm0, %xmm3
-; SSSE3-NEXT:    movdqa %xmm1, %xmm0
-; SSSE3-NEXT:    pxor %xmm10, %xmm0
-; SSSE3-NEXT:    movdqa %xmm9, %xmm4
+; SSSE3-NEXT:    pand %xmm3, %xmm7
+; SSSE3-NEXT:    pandn %xmm8, %xmm3
+; SSSE3-NEXT:    por %xmm7, %xmm3
+; SSSE3-NEXT:    movdqa %xmm5, %xmm0
+; SSSE3-NEXT:    pxor %xmm11, %xmm0
+; SSSE3-NEXT:    movdqa %xmm10, %xmm4
 ; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm4
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm0
+; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm0
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
 ; SSSE3-NEXT:    pand %xmm6, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm0, %xmm7
+; SSSE3-NEXT:    pand %xmm7, %xmm5
+; SSSE3-NEXT:    pandn %xmm8, %xmm7
+; SSSE3-NEXT:    por %xmm5, %xmm7
+; SSSE3-NEXT:    movdqa %xmm9, %xmm0
+; SSSE3-NEXT:    pxor %xmm11, %xmm0
+; SSSE3-NEXT:    movdqa %xmm10, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm5, %xmm0
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
 ; SSSE3-NEXT:    por %xmm0, %xmm4
-; SSSE3-NEXT:    pand %xmm4, %xmm1
+; SSSE3-NEXT:    pand %xmm4, %xmm9
 ; SSSE3-NEXT:    pandn %xmm8, %xmm4
-; SSSE3-NEXT:    por %xmm1, %xmm4
+; SSSE3-NEXT:    por %xmm9, %xmm4
 ; SSSE3-NEXT:    movdqa %xmm4, %xmm0
-; SSSE3-NEXT:    pxor %xmm10, %xmm0
-; SSSE3-NEXT:    movdqa %xmm0, %xmm1
-; SSSE3-NEXT:    pcmpgtd %xmm10, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm0
+; SSSE3-NEXT:    pxor %xmm11, %xmm0
+; SSSE3-NEXT:    movdqa %xmm0, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm11, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm11, %xmm0
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
 ; SSSE3-NEXT:    pand %xmm6, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    por %xmm0, %xmm1
-; SSSE3-NEXT:    pand %xmm4, %xmm1
-; SSSE3-NEXT:    movdqa %xmm3, %xmm0
-; SSSE3-NEXT:    pxor %xmm10, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSSE3-NEXT:    por %xmm0, %xmm5
+; SSSE3-NEXT:    pand %xmm4, %xmm5
+; SSSE3-NEXT:    movdqa %xmm7, %xmm0
+; SSSE3-NEXT:    pxor %xmm11, %xmm0
 ; SSSE3-NEXT:    movdqa %xmm0, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm10, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm11, %xmm4
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm6, %xmm7
+; SSSE3-NEXT:    pcmpeqd %xmm11, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm6, %xmm1
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    por %xmm7, %xmm0
-; SSSE3-NEXT:    pand %xmm3, %xmm0
-; SSSE3-NEXT:    packuswb %xmm1, %xmm0
+; SSSE3-NEXT:    por %xmm1, %xmm0
+; SSSE3-NEXT:    pand %xmm7, %xmm0
+; SSSE3-NEXT:    packuswb %xmm5, %xmm0
+; SSSE3-NEXT:    movdqa %xmm3, %xmm1
+; SSSE3-NEXT:    pxor %xmm11, %xmm1
+; SSSE3-NEXT:    movdqa %xmm1, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm11, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm11, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm5, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm1, %xmm4
+; SSSE3-NEXT:    pand %xmm3, %xmm4
 ; SSSE3-NEXT:    movdqa %xmm2, %xmm1
-; SSSE3-NEXT:    pxor %xmm10, %xmm1
+; SSSE3-NEXT:    pxor %xmm11, %xmm1
 ; SSSE3-NEXT:    movdqa %xmm1, %xmm3
-; SSSE3-NEXT:    pcmpgtd %xmm10, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm1
+; SSSE3-NEXT:    pcmpgtd %xmm11, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm11, %xmm1
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm4, %xmm1
+; SSSE3-NEXT:    pand %xmm5, %xmm1
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
 ; SSSE3-NEXT:    por %xmm1, %xmm3
 ; SSSE3-NEXT:    pand %xmm2, %xmm3
-; SSSE3-NEXT:    movdqa %xmm5, %xmm1
-; SSSE3-NEXT:    pxor %xmm10, %xmm1
-; SSSE3-NEXT:    movdqa %xmm1, %xmm2
-; SSSE3-NEXT:    pcmpgtd %xmm10, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm4, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSSE3-NEXT:    por %xmm1, %xmm2
-; SSSE3-NEXT:    pand %xmm5, %xmm2
-; SSSE3-NEXT:    packuswb %xmm3, %xmm2
-; SSSE3-NEXT:    packuswb %xmm2, %xmm0
+; SSSE3-NEXT:    packuswb %xmm4, %xmm3
+; SSSE3-NEXT:    packuswb %xmm3, %xmm0
+; SSSE3-NEXT:    packuswb %xmm0, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: trunc_packus_v8i64_v8i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa %xmm0, %xmm9
-; SSE41-NEXT:    movapd {{.*#+}} xmm7 = [255,255]
-; SSE41-NEXT:    movdqa {{.*#+}} xmm10 = [2147483648,2147483648]
-; SSE41-NEXT:    movdqa %xmm2, %xmm0
-; SSE41-NEXT:    pxor %xmm10, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm6 = [2147483903,2147483903]
-; SSE41-NEXT:    movdqa %xmm6, %xmm4
+; SSE41-NEXT:    movdqa (%rdi), %xmm10
+; SSE41-NEXT:    movdqa 16(%rdi), %xmm9
+; SSE41-NEXT:    movdqa 32(%rdi), %xmm3
+; SSE41-NEXT:    movdqa 48(%rdi), %xmm5
+; SSE41-NEXT:    movapd {{.*#+}} xmm1 = [255,255]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    pxor %xmm2, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [2147483903,2147483903]
+; SSE41-NEXT:    movdqa %xmm4, %xmm7
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm7
+; SSE41-NEXT:    movdqa %xmm4, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
+; SSE41-NEXT:    pand %xmm7, %xmm0
+; SSE41-NEXT:    por %xmm6, %xmm0
+; SSE41-NEXT:    movapd %xmm1, %xmm8
+; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm8
+; SSE41-NEXT:    movdqa %xmm5, %xmm0
+; SSE41-NEXT:    pxor %xmm2, %xmm0
+; SSE41-NEXT:    movdqa %xmm4, %xmm3
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm3
+; SSE41-NEXT:    movdqa %xmm4, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
+; SSE41-NEXT:    pand %xmm3, %xmm0
+; SSE41-NEXT:    por %xmm6, %xmm0
+; SSE41-NEXT:    movapd %xmm1, %xmm6
+; SSE41-NEXT:    blendvpd %xmm0, %xmm5, %xmm6
+; SSE41-NEXT:    movdqa %xmm10, %xmm0
+; SSE41-NEXT:    pxor %xmm2, %xmm0
+; SSE41-NEXT:    movdqa %xmm4, %xmm3
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm3
+; SSE41-NEXT:    movdqa %xmm4, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm5
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2]
+; SSE41-NEXT:    pand %xmm3, %xmm0
+; SSE41-NEXT:    por %xmm5, %xmm0
+; SSE41-NEXT:    movapd %xmm1, %xmm3
+; SSE41-NEXT:    blendvpd %xmm0, %xmm10, %xmm3
+; SSE41-NEXT:    movdqa %xmm9, %xmm0
+; SSE41-NEXT:    pxor %xmm2, %xmm0
+; SSE41-NEXT:    movdqa %xmm4, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm5
 ; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm6, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2]
 ; SSE41-NEXT:    pand %xmm5, %xmm0
 ; SSE41-NEXT:    por %xmm4, %xmm0
-; SSE41-NEXT:    movapd %xmm7, %xmm8
-; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm8
-; SSE41-NEXT:    movdqa %xmm3, %xmm0
-; SSE41-NEXT:    pxor %xmm10, %xmm0
-; SSE41-NEXT:    movdqa %xmm6, %xmm2
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm6, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm4, %xmm0
-; SSE41-NEXT:    por %xmm2, %xmm0
-; SSE41-NEXT:    movapd %xmm7, %xmm2
-; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm2
-; SSE41-NEXT:    movdqa %xmm9, %xmm0
-; SSE41-NEXT:    pxor %xmm10, %xmm0
-; SSE41-NEXT:    movdqa %xmm6, %xmm3
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm6, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE41-NEXT:    blendvpd %xmm0, %xmm9, %xmm1
+; SSE41-NEXT:    pxor %xmm5, %xmm5
+; SSE41-NEXT:    movapd %xmm1, %xmm4
+; SSE41-NEXT:    xorpd %xmm2, %xmm4
+; SSE41-NEXT:    movapd %xmm4, %xmm7
+; SSE41-NEXT:    pcmpeqd %xmm2, %xmm7
+; SSE41-NEXT:    pcmpgtd %xmm2, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pand %xmm7, %xmm0
+; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    pxor %xmm4, %xmm4
+; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm4
+; SSE41-NEXT:    movapd %xmm3, %xmm1
+; SSE41-NEXT:    xorpd %xmm2, %xmm1
+; SSE41-NEXT:    movapd %xmm1, %xmm7
+; SSE41-NEXT:    pcmpeqd %xmm2, %xmm7
+; SSE41-NEXT:    pcmpgtd %xmm2, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2]
+; SSE41-NEXT:    pand %xmm7, %xmm0
+; SSE41-NEXT:    por %xmm1, %xmm0
+; SSE41-NEXT:    pxor %xmm1, %xmm1
+; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm1
+; SSE41-NEXT:    packusdw %xmm4, %xmm1
+; SSE41-NEXT:    movapd %xmm6, %xmm3
+; SSE41-NEXT:    xorpd %xmm2, %xmm3
+; SSE41-NEXT:    movapd %xmm3, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm2, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm2, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
 ; SSE41-NEXT:    pand %xmm4, %xmm0
 ; SSE41-NEXT:    por %xmm3, %xmm0
-; SSE41-NEXT:    movapd %xmm7, %xmm4
-; SSE41-NEXT:    blendvpd %xmm0, %xmm9, %xmm4
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    pxor %xmm10, %xmm0
-; SSE41-NEXT:    movdqa %xmm6, %xmm3
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm6, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm0
-; SSE41-NEXT:    por %xmm3, %xmm0
-; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm7
 ; SSE41-NEXT:    pxor %xmm3, %xmm3
-; SSE41-NEXT:    movapd %xmm7, %xmm0
-; SSE41-NEXT:    xorpd %xmm10, %xmm0
-; SSE41-NEXT:    movapd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd %xmm10, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm0
-; SSE41-NEXT:    por %xmm1, %xmm0
-; SSE41-NEXT:    pxor %xmm5, %xmm5
-; SSE41-NEXT:    blendvpd %xmm0, %xmm7, %xmm5
-; SSE41-NEXT:    movapd %xmm4, %xmm0
-; SSE41-NEXT:    xorpd %xmm10, %xmm0
-; SSE41-NEXT:    movapd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd %xmm10, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE41-NEXT:    blendvpd %xmm0, %xmm6, %xmm3
+; SSE41-NEXT:    movapd %xmm8, %xmm4
+; SSE41-NEXT:    xorpd %xmm2, %xmm4
+; SSE41-NEXT:    movapd %xmm4, %xmm6
+; SSE41-NEXT:    pcmpeqd %xmm2, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm2, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2]
 ; SSE41-NEXT:    pand %xmm6, %xmm0
-; SSE41-NEXT:    por %xmm1, %xmm0
-; SSE41-NEXT:    pxor %xmm1, %xmm1
-; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm1
-; SSE41-NEXT:    packusdw %xmm5, %xmm1
-; SSE41-NEXT:    movapd %xmm2, %xmm0
-; SSE41-NEXT:    xorpd %xmm10, %xmm0
-; SSE41-NEXT:    movapd %xmm0, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm10, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm0
 ; SSE41-NEXT:    por %xmm4, %xmm0
-; SSE41-NEXT:    pxor %xmm4, %xmm4
-; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm4
-; SSE41-NEXT:    movapd %xmm8, %xmm0
-; SSE41-NEXT:    xorpd %xmm10, %xmm0
-; SSE41-NEXT:    movapd %xmm0, %xmm2
-; SSE41-NEXT:    pcmpgtd %xmm10, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm0
-; SSE41-NEXT:    por %xmm2, %xmm0
-; SSE41-NEXT:    blendvpd %xmm0, %xmm8, %xmm3
-; SSE41-NEXT:    packusdw %xmm4, %xmm3
-; SSE41-NEXT:    packusdw %xmm3, %xmm1
+; SSE41-NEXT:    blendvpd %xmm0, %xmm8, %xmm5
+; SSE41-NEXT:    packusdw %xmm3, %xmm5
+; SSE41-NEXT:    packusdw %xmm5, %xmm1
+; SSE41-NEXT:    packuswb %xmm1, %xmm1
 ; SSE41-NEXT:    movdqa %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: trunc_packus_v8i64_v8i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [255,255]
-; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm3, %xmm8
-; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm3, %xmm5
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm6
-; AVX1-NEXT:    vpcmpgtq %xmm6, %xmm3, %xmm7
-; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm3, %xmm4
-; AVX1-NEXT:    vblendvpd %xmm4, %xmm0, %xmm3, %xmm0
-; AVX1-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm0, %xmm9
-; AVX1-NEXT:    vblendvpd %xmm7, %xmm6, %xmm3, %xmm6
-; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm6, %xmm7
-; AVX1-NEXT:    vblendvpd %xmm5, %xmm1, %xmm3, %xmm1
-; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm1, %xmm5
-; AVX1-NEXT:    vblendvpd %xmm8, %xmm2, %xmm3, %xmm2
-; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm2, %xmm3
-; AVX1-NEXT:    vpand %xmm2, %xmm3, %xmm2
-; AVX1-NEXT:    vpand %xmm1, %xmm5, %xmm1
-; AVX1-NEXT:    vpackusdw %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpand %xmm6, %xmm7, %xmm2
+; AVX1-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX1-NEXT:    vmovdqa 16(%rdi), %xmm1
+; AVX1-NEXT:    vmovdqa 32(%rdi), %xmm2
+; AVX1-NEXT:    vmovdqa 48(%rdi), %xmm3
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [255,255]
+; AVX1-NEXT:    vpcmpgtq %xmm3, %xmm4, %xmm8
+; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm4, %xmm6
+; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm4, %xmm7
+; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm4, %xmm5
+; AVX1-NEXT:    vblendvpd %xmm5, %xmm0, %xmm4, %xmm0
+; AVX1-NEXT:    vpxor %xmm5, %xmm5, %xmm5
+; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm0, %xmm9
+; AVX1-NEXT:    vblendvpd %xmm7, %xmm1, %xmm4, %xmm1
+; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm1, %xmm7
+; AVX1-NEXT:    vblendvpd %xmm6, %xmm2, %xmm4, %xmm2
+; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm2, %xmm6
+; AVX1-NEXT:    vblendvpd %xmm8, %xmm3, %xmm4, %xmm3
+; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm3, %xmm4
+; AVX1-NEXT:    vpand %xmm3, %xmm4, %xmm3
+; AVX1-NEXT:    vpand %xmm2, %xmm6, %xmm2
+; AVX1-NEXT:    vpackusdw %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpand %xmm1, %xmm7, %xmm1
 ; AVX1-NEXT:    vpand %xmm0, %xmm9, %xmm0
-; AVX1-NEXT:    vpackusdw %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    vpackusdw %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: trunc_packus_v8i64_v8i8:
 ; AVX2:       # %bb.0:
+; AVX2-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX2-NEXT:    vmovdqa 32(%rdi), %ymm1
 ; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [255,255,255,255]
-; AVX2-NEXT:    vpcmpgtq %ymm0, %ymm2, %ymm3
-; AVX2-NEXT:    vblendvpd %ymm3, %ymm0, %ymm2, %ymm0
 ; AVX2-NEXT:    vpcmpgtq %ymm1, %ymm2, %ymm3
 ; AVX2-NEXT:    vblendvpd %ymm3, %ymm1, %ymm2, %ymm1
+; AVX2-NEXT:    vpcmpgtq %ymm0, %ymm2, %ymm3
+; AVX2-NEXT:    vblendvpd %ymm3, %ymm0, %ymm2, %ymm0
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtq %ymm2, %ymm1, %ymm3
-; AVX2-NEXT:    vpand %ymm1, %ymm3, %ymm1
-; AVX2-NEXT:    vpcmpgtq %ymm2, %ymm0, %ymm2
-; AVX2-NEXT:    vpand %ymm0, %ymm2, %ymm0
-; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpcmpgtq %ymm2, %ymm0, %ymm3
+; AVX2-NEXT:    vpand %ymm0, %ymm3, %ymm0
+; AVX2-NEXT:    vpcmpgtq %ymm2, %ymm1, %ymm2
+; AVX2-NEXT:    vpand %ymm1, %ymm2, %ymm1
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm3 = <u,u,0,8,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX2-NEXT:    vpshufb %xmm3, %xmm2, %xmm2
+; AVX2-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
+; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm2
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm3 = <0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX2-NEXT:    vpshufb %xmm3, %xmm2, %xmm2
+; AVX2-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
+; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3]
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: trunc_packus_v8i64_v8i8:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpminsq {{.*}}(%rip){1to8}, %zmm0, %zmm0
-; AVX512-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm0
-; AVX512-NEXT:    vpmovqw %zmm0, %xmm0
+; AVX512-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; AVX512-NEXT:    vpmaxsq (%rdi), %zmm0, %zmm0
+; AVX512-NEXT:    vpmovusqb %zmm0, %xmm0
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
+;
+; SKX-LABEL: trunc_packus_v8i64_v8i8:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; SKX-NEXT:    vpmaxsq 32(%rdi), %ymm0, %ymm1
+; SKX-NEXT:    vpmovusqb %ymm1, %xmm1
+; SKX-NEXT:    vpmaxsq (%rdi), %ymm0, %ymm0
+; SKX-NEXT:    vpmovusqb %ymm0, %xmm0
+; SKX-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SKX-NEXT:    vzeroupper
+; SKX-NEXT:    retq
+  %a0 = load <8 x i64>, <8 x i64>* %p0
   %1 = icmp slt <8 x i64> %a0, <i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255>
   %2 = select <8 x i1> %1, <8 x i64> %a0, <8 x i64> <i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255>
   %3 = icmp sgt <8 x i64> %2, zeroinitializer
@@ -1681,355 +3981,364 @@ define <8 x i8> @trunc_packus_v8i64_v8i8(<8 x i64> %a0) {
   ret <8 x i8> %5
 }
 
-define void @trunc_packus_v8i64_v8i8_store(<8 x i64> %a0, <8 x i8> *%p1) {
+define void @trunc_packus_v8i64_v8i8_store(<8 x i64>* %p0, <8 x i8> *%p1) "min-legal-vector-width"="256" {
 ; SSE2-LABEL: trunc_packus_v8i64_v8i8_store:
 ; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa (%rdi), %xmm5
+; SSE2-NEXT:    movdqa 16(%rdi), %xmm9
+; SSE2-NEXT:    movdqa 32(%rdi), %xmm2
+; SSE2-NEXT:    movdqa 48(%rdi), %xmm7
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [255,255]
-; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [2147483648,2147483648]
-; SSE2-NEXT:    movdqa %xmm2, %xmm5
-; SSE2-NEXT:    pxor %xmm10, %xmm5
-; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [2147483903,2147483903]
-; SSE2-NEXT:    movdqa %xmm9, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm5, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm7[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm5[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm7[1,1,3,3]
-; SSE2-NEXT:    por %xmm4, %xmm5
-; SSE2-NEXT:    pand %xmm5, %xmm2
-; SSE2-NEXT:    pandn %xmm8, %xmm5
-; SSE2-NEXT:    por %xmm2, %xmm5
-; SSE2-NEXT:    movdqa %xmm3, %xmm2
-; SSE2-NEXT:    pxor %xmm10, %xmm2
-; SSE2-NEXT:    movdqa %xmm9, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm2[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm7, %xmm2
-; SSE2-NEXT:    pand %xmm2, %xmm3
+; SSE2-NEXT:    movdqa {{.*#+}} xmm11 = [2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm2, %xmm1
+; SSE2-NEXT:    pxor %xmm11, %xmm1
+; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [2147483903,2147483903]
+; SSE2-NEXT:    movdqa %xmm10, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm6[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pand %xmm3, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm6[1,1,3,3]
+; SSE2-NEXT:    por %xmm4, %xmm1
+; SSE2-NEXT:    pand %xmm1, %xmm2
+; SSE2-NEXT:    pandn %xmm8, %xmm1
+; SSE2-NEXT:    por %xmm2, %xmm1
+; SSE2-NEXT:    movdqa %xmm7, %xmm2
+; SSE2-NEXT:    pxor %xmm11, %xmm2
+; SSE2-NEXT:    movdqa %xmm10, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[1,1,3,3]
+; SSE2-NEXT:    pand %xmm4, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,1,3,3]
+; SSE2-NEXT:    por %xmm6, %xmm2
+; SSE2-NEXT:    pand %xmm2, %xmm7
 ; SSE2-NEXT:    pandn %xmm8, %xmm2
-; SSE2-NEXT:    por %xmm3, %xmm2
-; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    pxor %xmm10, %xmm3
-; SSE2-NEXT:    movdqa %xmm9, %xmm4
+; SSE2-NEXT:    por %xmm7, %xmm2
+; SSE2-NEXT:    movdqa %xmm5, %xmm3
+; SSE2-NEXT:    pxor %xmm11, %xmm3
+; SSE2-NEXT:    movdqa %xmm10, %xmm4
 ; SSE2-NEXT:    pcmpgtd %xmm3, %xmm4
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm3[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm7
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSE2-NEXT:    pand %xmm6, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm3, %xmm7
+; SSE2-NEXT:    pand %xmm7, %xmm5
+; SSE2-NEXT:    pandn %xmm8, %xmm7
+; SSE2-NEXT:    por %xmm5, %xmm7
+; SSE2-NEXT:    movdqa %xmm9, %xmm3
+; SSE2-NEXT:    pxor %xmm11, %xmm3
+; SSE2-NEXT:    movdqa %xmm10, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSE2-NEXT:    pand %xmm5, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm3, %xmm4
+; SSE2-NEXT:    pand %xmm4, %xmm9
+; SSE2-NEXT:    pandn %xmm8, %xmm4
+; SSE2-NEXT:    por %xmm9, %xmm4
+; SSE2-NEXT:    movdqa %xmm4, %xmm3
+; SSE2-NEXT:    pxor %xmm11, %xmm3
+; SSE2-NEXT:    movdqa %xmm3, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm11, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm11, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSE2-NEXT:    pand %xmm6, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSE2-NEXT:    por %xmm3, %xmm5
+; SSE2-NEXT:    pand %xmm4, %xmm5
+; SSE2-NEXT:    movdqa %xmm7, %xmm3
+; SSE2-NEXT:    pxor %xmm11, %xmm3
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm11, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm11, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
+; SSE2-NEXT:    pand %xmm6, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm7, %xmm3
-; SSE2-NEXT:    pand %xmm3, %xmm0
-; SSE2-NEXT:    pandn %xmm8, %xmm3
 ; SSE2-NEXT:    por %xmm0, %xmm3
-; SSE2-NEXT:    movdqa %xmm1, %xmm0
-; SSE2-NEXT:    pxor %xmm10, %xmm0
-; SSE2-NEXT:    movdqa %xmm9, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm0
+; SSE2-NEXT:    pand %xmm7, %xmm3
+; SSE2-NEXT:    packuswb %xmm5, %xmm3
+; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    pxor %xmm11, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm11, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm11, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm0
+; SSE2-NEXT:    pand %xmm5, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
 ; SSE2-NEXT:    por %xmm0, %xmm4
-; SSE2-NEXT:    pand %xmm4, %xmm1
-; SSE2-NEXT:    pandn %xmm8, %xmm4
-; SSE2-NEXT:    por %xmm1, %xmm4
-; SSE2-NEXT:    movdqa %xmm4, %xmm0
-; SSE2-NEXT:    pxor %xmm10, %xmm0
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm10, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm10, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm0, %xmm1
-; SSE2-NEXT:    pand %xmm4, %xmm1
-; SSE2-NEXT:    movdqa %xmm3, %xmm0
-; SSE2-NEXT:    pxor %xmm10, %xmm0
-; SSE2-NEXT:    movdqa %xmm0, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm10, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm10, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm7, %xmm0
-; SSE2-NEXT:    pand %xmm3, %xmm0
-; SSE2-NEXT:    packuswb %xmm1, %xmm0
-; SSE2-NEXT:    movdqa %xmm2, %xmm1
-; SSE2-NEXT:    pxor %xmm10, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm3
-; SSE2-NEXT:    pcmpgtd %xmm10, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm10, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pand %xmm4, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm3
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    movdqa %xmm5, %xmm1
-; SSE2-NEXT:    pxor %xmm10, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm2
-; SSE2-NEXT:    pcmpgtd %xmm10, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm10, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pand %xmm4, %xmm1
+; SSE2-NEXT:    pand %xmm2, %xmm4
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    pxor %xmm11, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm11, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm11, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pand %xmm5, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm2
-; SSE2-NEXT:    pand %xmm5, %xmm2
-; SSE2-NEXT:    packuswb %xmm3, %xmm2
-; SSE2-NEXT:    packuswb %xmm2, %xmm0
-; SSE2-NEXT:    packuswb %xmm0, %xmm0
-; SSE2-NEXT:    movq %xmm0, (%rdi)
+; SSE2-NEXT:    por %xmm0, %xmm2
+; SSE2-NEXT:    pand %xmm1, %xmm2
+; SSE2-NEXT:    packuswb %xmm4, %xmm2
+; SSE2-NEXT:    packuswb %xmm2, %xmm3
+; SSE2-NEXT:    packuswb %xmm0, %xmm3
+; SSE2-NEXT:    movq %xmm3, (%rsi)
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: trunc_packus_v8i64_v8i8_store:
 ; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa (%rdi), %xmm5
+; SSSE3-NEXT:    movdqa 16(%rdi), %xmm9
+; SSSE3-NEXT:    movdqa 32(%rdi), %xmm2
+; SSSE3-NEXT:    movdqa 48(%rdi), %xmm7
 ; SSSE3-NEXT:    movdqa {{.*#+}} xmm8 = [255,255]
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm10 = [2147483648,2147483648]
-; SSSE3-NEXT:    movdqa %xmm2, %xmm5
-; SSSE3-NEXT:    pxor %xmm10, %xmm5
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm9 = [2147483903,2147483903]
-; SSSE3-NEXT:    movdqa %xmm9, %xmm7
-; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm7
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm7[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm5
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm5[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm6, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm7[1,1,3,3]
-; SSSE3-NEXT:    por %xmm4, %xmm5
-; SSSE3-NEXT:    pand %xmm5, %xmm2
-; SSSE3-NEXT:    pandn %xmm8, %xmm5
-; SSSE3-NEXT:    por %xmm2, %xmm5
-; SSSE3-NEXT:    movdqa %xmm3, %xmm2
-; SSSE3-NEXT:    pxor %xmm10, %xmm2
-; SSSE3-NEXT:    movdqa %xmm9, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm2[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm6, %xmm7
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    por %xmm7, %xmm2
-; SSSE3-NEXT:    pand %xmm2, %xmm3
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm11 = [2147483648,2147483648]
+; SSSE3-NEXT:    movdqa %xmm2, %xmm1
+; SSSE3-NEXT:    pxor %xmm11, %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm10 = [2147483903,2147483903]
+; SSSE3-NEXT:    movdqa %xmm10, %xmm6
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm6[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm3, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm6[1,1,3,3]
+; SSSE3-NEXT:    por %xmm4, %xmm1
+; SSSE3-NEXT:    pand %xmm1, %xmm2
+; SSSE3-NEXT:    pandn %xmm8, %xmm1
+; SSSE3-NEXT:    por %xmm2, %xmm1
+; SSSE3-NEXT:    movdqa %xmm7, %xmm2
+; SSSE3-NEXT:    pxor %xmm11, %xmm2
+; SSSE3-NEXT:    movdqa %xmm10, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm4, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,1,3,3]
+; SSSE3-NEXT:    por %xmm6, %xmm2
+; SSSE3-NEXT:    pand %xmm2, %xmm7
 ; SSSE3-NEXT:    pandn %xmm8, %xmm2
-; SSSE3-NEXT:    por %xmm3, %xmm2
-; SSSE3-NEXT:    movdqa %xmm0, %xmm3
-; SSSE3-NEXT:    pxor %xmm10, %xmm3
-; SSSE3-NEXT:    movdqa %xmm9, %xmm4
+; SSSE3-NEXT:    por %xmm7, %xmm2
+; SSSE3-NEXT:    movdqa %xmm5, %xmm3
+; SSSE3-NEXT:    pxor %xmm11, %xmm3
+; SSSE3-NEXT:    movdqa %xmm10, %xmm4
 ; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm4
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm6, %xmm7
+; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm6, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm3, %xmm7
+; SSSE3-NEXT:    pand %xmm7, %xmm5
+; SSSE3-NEXT:    pandn %xmm8, %xmm7
+; SSSE3-NEXT:    por %xmm5, %xmm7
+; SSSE3-NEXT:    movdqa %xmm9, %xmm3
+; SSSE3-NEXT:    pxor %xmm11, %xmm3
+; SSSE3-NEXT:    movdqa %xmm10, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm5, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm3, %xmm4
+; SSSE3-NEXT:    pand %xmm4, %xmm9
+; SSSE3-NEXT:    pandn %xmm8, %xmm4
+; SSSE3-NEXT:    por %xmm9, %xmm4
+; SSSE3-NEXT:    movdqa %xmm4, %xmm3
+; SSSE3-NEXT:    pxor %xmm11, %xmm3
+; SSSE3-NEXT:    movdqa %xmm3, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm11, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm11, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm6, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSSE3-NEXT:    por %xmm3, %xmm5
+; SSSE3-NEXT:    pand %xmm4, %xmm5
+; SSSE3-NEXT:    movdqa %xmm7, %xmm3
+; SSSE3-NEXT:    pxor %xmm11, %xmm3
+; SSSE3-NEXT:    movdqa %xmm3, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm11, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm11, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm6, %xmm0
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    por %xmm7, %xmm3
-; SSSE3-NEXT:    pand %xmm3, %xmm0
-; SSSE3-NEXT:    pandn %xmm8, %xmm3
 ; SSSE3-NEXT:    por %xmm0, %xmm3
-; SSSE3-NEXT:    movdqa %xmm1, %xmm0
-; SSSE3-NEXT:    pxor %xmm10, %xmm0
-; SSSE3-NEXT:    movdqa %xmm9, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm0
+; SSSE3-NEXT:    pand %xmm7, %xmm3
+; SSSE3-NEXT:    packuswb %xmm5, %xmm3
+; SSSE3-NEXT:    movdqa %xmm2, %xmm0
+; SSSE3-NEXT:    pxor %xmm11, %xmm0
+; SSSE3-NEXT:    movdqa %xmm0, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm11, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm11, %xmm0
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm6, %xmm0
+; SSSE3-NEXT:    pand %xmm5, %xmm0
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
 ; SSSE3-NEXT:    por %xmm0, %xmm4
-; SSSE3-NEXT:    pand %xmm4, %xmm1
-; SSSE3-NEXT:    pandn %xmm8, %xmm4
-; SSSE3-NEXT:    por %xmm1, %xmm4
-; SSSE3-NEXT:    movdqa %xmm4, %xmm0
-; SSSE3-NEXT:    pxor %xmm10, %xmm0
-; SSSE3-NEXT:    movdqa %xmm0, %xmm1
-; SSSE3-NEXT:    pcmpgtd %xmm10, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm0
+; SSSE3-NEXT:    pand %xmm2, %xmm4
+; SSSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSSE3-NEXT:    pxor %xmm11, %xmm0
+; SSSE3-NEXT:    movdqa %xmm0, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm11, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm11, %xmm0
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm6, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    por %xmm0, %xmm1
-; SSSE3-NEXT:    pand %xmm4, %xmm1
-; SSSE3-NEXT:    movdqa %xmm3, %xmm0
-; SSSE3-NEXT:    pxor %xmm10, %xmm0
-; SSSE3-NEXT:    movdqa %xmm0, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm10, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm6, %xmm7
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    por %xmm7, %xmm0
-; SSSE3-NEXT:    pand %xmm3, %xmm0
-; SSSE3-NEXT:    packuswb %xmm1, %xmm0
-; SSSE3-NEXT:    movdqa %xmm2, %xmm1
-; SSSE3-NEXT:    pxor %xmm10, %xmm1
-; SSSE3-NEXT:    movdqa %xmm1, %xmm3
-; SSSE3-NEXT:    pcmpgtd %xmm10, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm4, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm1, %xmm3
-; SSSE3-NEXT:    pand %xmm2, %xmm3
-; SSSE3-NEXT:    movdqa %xmm5, %xmm1
-; SSSE3-NEXT:    pxor %xmm10, %xmm1
-; SSSE3-NEXT:    movdqa %xmm1, %xmm2
-; SSSE3-NEXT:    pcmpgtd %xmm10, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm4, %xmm1
+; SSSE3-NEXT:    pand %xmm5, %xmm0
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSSE3-NEXT:    por %xmm1, %xmm2
-; SSSE3-NEXT:    pand %xmm5, %xmm2
-; SSSE3-NEXT:    packuswb %xmm3, %xmm2
-; SSSE3-NEXT:    packuswb %xmm2, %xmm0
-; SSSE3-NEXT:    packuswb %xmm0, %xmm0
-; SSSE3-NEXT:    movq %xmm0, (%rdi)
+; SSSE3-NEXT:    por %xmm0, %xmm2
+; SSSE3-NEXT:    pand %xmm1, %xmm2
+; SSSE3-NEXT:    packuswb %xmm4, %xmm2
+; SSSE3-NEXT:    packuswb %xmm2, %xmm3
+; SSSE3-NEXT:    packuswb %xmm0, %xmm3
+; SSSE3-NEXT:    movq %xmm3, (%rsi)
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: trunc_packus_v8i64_v8i8_store:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa %xmm0, %xmm9
-; SSE41-NEXT:    movapd {{.*#+}} xmm7 = [255,255]
-; SSE41-NEXT:    movdqa {{.*#+}} xmm10 = [2147483648,2147483648]
+; SSE41-NEXT:    movdqa (%rdi), %xmm10
+; SSE41-NEXT:    movdqa 16(%rdi), %xmm9
+; SSE41-NEXT:    movdqa 32(%rdi), %xmm2
+; SSE41-NEXT:    movdqa 48(%rdi), %xmm5
+; SSE41-NEXT:    movapd {{.*#+}} xmm4 = [255,255]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [2147483648,2147483648]
 ; SSE41-NEXT:    movdqa %xmm2, %xmm0
-; SSE41-NEXT:    pxor %xmm10, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [2147483903,2147483903]
-; SSE41-NEXT:    movdqa %xmm4, %xmm6
+; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [2147483903,2147483903]
+; SSE41-NEXT:    movdqa %xmm3, %xmm7
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm7
+; SSE41-NEXT:    movdqa %xmm3, %xmm6
 ; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm6[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm4, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
+; SSE41-NEXT:    pand %xmm7, %xmm0
 ; SSE41-NEXT:    por %xmm6, %xmm0
-; SSE41-NEXT:    movapd %xmm7, %xmm8
+; SSE41-NEXT:    movapd %xmm4, %xmm8
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm8
-; SSE41-NEXT:    movdqa %xmm3, %xmm0
-; SSE41-NEXT:    pxor %xmm10, %xmm0
-; SSE41-NEXT:    movdqa %xmm4, %xmm2
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm4, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm0
-; SSE41-NEXT:    por %xmm2, %xmm0
-; SSE41-NEXT:    movapd %xmm7, %xmm2
-; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm2
+; SSE41-NEXT:    movdqa %xmm5, %xmm0
+; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    movdqa %xmm3, %xmm2
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm2
+; SSE41-NEXT:    movdqa %xmm3, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
+; SSE41-NEXT:    pand %xmm2, %xmm0
+; SSE41-NEXT:    por %xmm6, %xmm0
+; SSE41-NEXT:    movapd %xmm4, %xmm6
+; SSE41-NEXT:    blendvpd %xmm0, %xmm5, %xmm6
+; SSE41-NEXT:    movdqa %xmm10, %xmm0
+; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    movdqa %xmm3, %xmm2
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm2
+; SSE41-NEXT:    movdqa %xmm3, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm5
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2]
+; SSE41-NEXT:    pand %xmm2, %xmm0
+; SSE41-NEXT:    por %xmm5, %xmm0
+; SSE41-NEXT:    movapd %xmm4, %xmm2
+; SSE41-NEXT:    blendvpd %xmm0, %xmm10, %xmm2
 ; SSE41-NEXT:    movdqa %xmm9, %xmm0
-; SSE41-NEXT:    pxor %xmm10, %xmm0
-; SSE41-NEXT:    movdqa %xmm4, %xmm3
+; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    movdqa %xmm3, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm5
 ; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm4, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
 ; SSE41-NEXT:    pand %xmm5, %xmm0
 ; SSE41-NEXT:    por %xmm3, %xmm0
-; SSE41-NEXT:    movapd %xmm7, %xmm3
-; SSE41-NEXT:    blendvpd %xmm0, %xmm9, %xmm3
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    pxor %xmm10, %xmm0
-; SSE41-NEXT:    movdqa %xmm4, %xmm5
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm4, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm6, %xmm0
-; SSE41-NEXT:    por %xmm5, %xmm0
-; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm7
-; SSE41-NEXT:    xorpd %xmm1, %xmm1
-; SSE41-NEXT:    movapd %xmm7, %xmm0
-; SSE41-NEXT:    xorpd %xmm10, %xmm0
-; SSE41-NEXT:    movapd %xmm0, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm10, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm9, %xmm4
 ; SSE41-NEXT:    pxor %xmm5, %xmm5
-; SSE41-NEXT:    blendvpd %xmm0, %xmm7, %xmm5
-; SSE41-NEXT:    movapd %xmm3, %xmm0
-; SSE41-NEXT:    xorpd %xmm10, %xmm0
-; SSE41-NEXT:    movapd %xmm0, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm10, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm6, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
-; SSE41-NEXT:    pxor %xmm4, %xmm4
-; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm4
-; SSE41-NEXT:    packusdw %xmm5, %xmm4
-; SSE41-NEXT:    movapd %xmm2, %xmm0
-; SSE41-NEXT:    xorpd %xmm10, %xmm0
-; SSE41-NEXT:    movapd %xmm0, %xmm3
-; SSE41-NEXT:    pcmpgtd %xmm10, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm0
+; SSE41-NEXT:    movapd %xmm4, %xmm3
+; SSE41-NEXT:    xorpd %xmm1, %xmm3
+; SSE41-NEXT:    movapd %xmm3, %xmm7
+; SSE41-NEXT:    pcmpeqd %xmm1, %xmm7
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pand %xmm7, %xmm0
 ; SSE41-NEXT:    por %xmm3, %xmm0
 ; SSE41-NEXT:    pxor %xmm3, %xmm3
-; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm3
-; SSE41-NEXT:    movapd %xmm8, %xmm0
-; SSE41-NEXT:    xorpd %xmm10, %xmm0
-; SSE41-NEXT:    movapd %xmm0, %xmm2
-; SSE41-NEXT:    pcmpgtd %xmm10, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm3
+; SSE41-NEXT:    movapd %xmm2, %xmm4
+; SSE41-NEXT:    xorpd %xmm1, %xmm4
+; SSE41-NEXT:    movapd %xmm4, %xmm7
+; SSE41-NEXT:    pcmpeqd %xmm1, %xmm7
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pand %xmm7, %xmm0
+; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    pxor %xmm4, %xmm4
+; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm4
+; SSE41-NEXT:    packusdw %xmm3, %xmm4
+; SSE41-NEXT:    movapd %xmm6, %xmm2
+; SSE41-NEXT:    xorpd %xmm1, %xmm2
+; SSE41-NEXT:    movapd %xmm2, %xmm3
+; SSE41-NEXT:    pcmpeqd %xmm1, %xmm3
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,0,2,2]
+; SSE41-NEXT:    pand %xmm3, %xmm0
 ; SSE41-NEXT:    por %xmm2, %xmm0
-; SSE41-NEXT:    blendvpd %xmm0, %xmm8, %xmm1
-; SSE41-NEXT:    packusdw %xmm3, %xmm1
-; SSE41-NEXT:    packusdw %xmm1, %xmm4
-; SSE41-NEXT:    packuswb %xmm4, %xmm4
-; SSE41-NEXT:    movq %xmm4, (%rdi)
+; SSE41-NEXT:    pxor %xmm2, %xmm2
+; SSE41-NEXT:    blendvpd %xmm0, %xmm6, %xmm2
+; SSE41-NEXT:    movapd %xmm8, %xmm3
+; SSE41-NEXT:    xorpd %xmm1, %xmm3
+; SSE41-NEXT:    movapd %xmm3, %xmm6
+; SSE41-NEXT:    pcmpeqd %xmm1, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pand %xmm6, %xmm0
+; SSE41-NEXT:    por %xmm3, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm8, %xmm5
+; SSE41-NEXT:    packusdw %xmm2, %xmm5
+; SSE41-NEXT:    packusdw %xmm5, %xmm4
+; SSE41-NEXT:    packuswb %xmm0, %xmm4
+; SSE41-NEXT:    movq %xmm4, (%rsi)
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: trunc_packus_v8i64_v8i8_store:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [255,255]
-; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm3, %xmm8
-; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm3, %xmm5
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm6
-; AVX1-NEXT:    vpcmpgtq %xmm6, %xmm3, %xmm7
-; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm3, %xmm4
-; AVX1-NEXT:    vblendvpd %xmm4, %xmm0, %xmm3, %xmm0
-; AVX1-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm0, %xmm9
-; AVX1-NEXT:    vblendvpd %xmm7, %xmm6, %xmm3, %xmm6
-; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm6, %xmm7
-; AVX1-NEXT:    vblendvpd %xmm5, %xmm1, %xmm3, %xmm1
-; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm1, %xmm5
-; AVX1-NEXT:    vblendvpd %xmm8, %xmm2, %xmm3, %xmm2
-; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm2, %xmm3
-; AVX1-NEXT:    vpand %xmm2, %xmm3, %xmm2
-; AVX1-NEXT:    vpand %xmm1, %xmm5, %xmm1
-; AVX1-NEXT:    vpackusdw %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpand %xmm6, %xmm7, %xmm2
+; AVX1-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX1-NEXT:    vmovdqa 16(%rdi), %xmm1
+; AVX1-NEXT:    vmovdqa 32(%rdi), %xmm2
+; AVX1-NEXT:    vmovdqa 48(%rdi), %xmm3
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [255,255]
+; AVX1-NEXT:    vpcmpgtq %xmm3, %xmm4, %xmm8
+; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm4, %xmm6
+; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm4, %xmm7
+; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm4, %xmm5
+; AVX1-NEXT:    vblendvpd %xmm5, %xmm0, %xmm4, %xmm0
+; AVX1-NEXT:    vpxor %xmm5, %xmm5, %xmm5
+; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm0, %xmm9
+; AVX1-NEXT:    vblendvpd %xmm7, %xmm1, %xmm4, %xmm1
+; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm1, %xmm7
+; AVX1-NEXT:    vblendvpd %xmm6, %xmm2, %xmm4, %xmm2
+; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm2, %xmm6
+; AVX1-NEXT:    vblendvpd %xmm8, %xmm3, %xmm4, %xmm3
+; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm3, %xmm4
+; AVX1-NEXT:    vpand %xmm3, %xmm4, %xmm3
+; AVX1-NEXT:    vpand %xmm2, %xmm6, %xmm2
+; AVX1-NEXT:    vpackusdw %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpand %xmm1, %xmm7, %xmm1
 ; AVX1-NEXT:    vpand %xmm0, %xmm9, %xmm0
-; AVX1-NEXT:    vpackusdw %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpackusdw %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
-; AVX1-NEXT:    vmovq %xmm0, (%rdi)
-; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    vmovq %xmm0, (%rsi)
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: trunc_packus_v8i64_v8i8_store:
 ; AVX2:       # %bb.0:
+; AVX2-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX2-NEXT:    vmovdqa 32(%rdi), %ymm1
 ; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [255,255,255,255]
 ; AVX2-NEXT:    vpcmpgtq %ymm1, %ymm2, %ymm3
 ; AVX2-NEXT:    vblendvpd %ymm3, %ymm1, %ymm2, %ymm1
@@ -2041,25 +4350,40 @@ define void @trunc_packus_v8i64_v8i8_store(<8 x i64> %a0, <8 x i8> *%p1) {
 ; AVX2-NEXT:    vpcmpgtq %ymm2, %ymm1, %ymm2
 ; AVX2-NEXT:    vpand %ymm1, %ymm2, %ymm1
 ; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX2-NEXT:    vpackusdw %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
-; AVX2-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm3
-; AVX2-NEXT:    vpackusdw %xmm3, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; AVX2-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
-; AVX2-NEXT:    vmovq %xmm0, (%rdi)
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm3 = <u,u,0,8,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX2-NEXT:    vpshufb %xmm3, %xmm2, %xmm2
+; AVX2-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
+; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm2
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm3 = <0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX2-NEXT:    vpshufb %xmm3, %xmm2, %xmm2
+; AVX2-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
+; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3]
+; AVX2-NEXT:    vmovq %xmm0, (%rsi)
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: trunc_packus_v8i64_v8i8_store:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm0
-; AVX512-NEXT:    vpmovusqb %zmm0, (%rdi)
+; AVX512-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; AVX512-NEXT:    vpmaxsq (%rdi), %zmm0, %zmm0
+; AVX512-NEXT:    vpmovusqb %zmm0, (%rsi)
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
+;
+; SKX-LABEL: trunc_packus_v8i64_v8i8_store:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; SKX-NEXT:    vpmaxsq 32(%rdi), %ymm0, %ymm1
+; SKX-NEXT:    vpmovusqb %ymm1, %xmm1
+; SKX-NEXT:    vpmaxsq (%rdi), %ymm0, %ymm0
+; SKX-NEXT:    vpmovusqb %ymm0, %xmm0
+; SKX-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SKX-NEXT:    vmovq %xmm0, (%rsi)
+; SKX-NEXT:    vzeroupper
+; SKX-NEXT:    retq
+  %a0 = load <8 x i64>, <8 x i64>* %p0
   %1 = icmp slt <8 x i64> %a0, <i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255>
   %2 = select <8 x i1> %1, <8 x i64> %a0, <8 x i64> <i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255>
   %3 = icmp sgt <8 x i64> %2, zeroinitializer
@@ -2069,662 +4393,683 @@ define void @trunc_packus_v8i64_v8i8_store(<8 x i64> %a0, <8 x i8> *%p1) {
   ret void
 }
 
-define <16 x i8> @trunc_packus_v16i64_v16i8(<16 x i64> %a0) {
+define <16 x i8> @trunc_packus_v16i64_v16i8(<16 x i64>* %p0) "min-legal-vector-width"="256" {
 ; SSE2-LABEL: trunc_packus_v16i64_v16i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [255,255]
-; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [2147483648,2147483648]
-; SSE2-NEXT:    movdqa %xmm6, %xmm9
-; SSE2-NEXT:    pxor %xmm8, %xmm9
-; SSE2-NEXT:    movdqa {{.*#+}} xmm11 = [2147483903,2147483903]
-; SSE2-NEXT:    movdqa %xmm11, %xmm12
-; SSE2-NEXT:    pcmpgtd %xmm9, %xmm12
-; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm12[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm11, %xmm9
-; SSE2-NEXT:    pshufd {{.*#+}} xmm14 = xmm9[1,1,3,3]
-; SSE2-NEXT:    pand %xmm13, %xmm14
-; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm12[1,1,3,3]
-; SSE2-NEXT:    por %xmm14, %xmm9
-; SSE2-NEXT:    pand %xmm9, %xmm6
-; SSE2-NEXT:    pandn %xmm10, %xmm9
-; SSE2-NEXT:    por %xmm6, %xmm9
-; SSE2-NEXT:    movdqa %xmm7, %xmm6
-; SSE2-NEXT:    pxor %xmm8, %xmm6
-; SSE2-NEXT:    movdqa %xmm11, %xmm12
-; SSE2-NEXT:    pcmpgtd %xmm6, %xmm12
-; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm12[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm11, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT:    pand %xmm13, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm12[1,1,3,3]
-; SSE2-NEXT:    por %xmm6, %xmm12
-; SSE2-NEXT:    pand %xmm12, %xmm7
-; SSE2-NEXT:    pandn %xmm10, %xmm12
-; SSE2-NEXT:    por %xmm7, %xmm12
-; SSE2-NEXT:    movdqa %xmm4, %xmm6
-; SSE2-NEXT:    pxor %xmm8, %xmm6
-; SSE2-NEXT:    movdqa %xmm11, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm6, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm7[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm11, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT:    pand %xmm13, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm7[1,1,3,3]
-; SSE2-NEXT:    por %xmm6, %xmm13
-; SSE2-NEXT:    pand %xmm13, %xmm4
-; SSE2-NEXT:    pandn %xmm10, %xmm13
-; SSE2-NEXT:    por %xmm4, %xmm13
-; SSE2-NEXT:    movdqa %xmm5, %xmm4
-; SSE2-NEXT:    pxor %xmm8, %xmm4
-; SSE2-NEXT:    movdqa %xmm11, %xmm6
+; SSE2-NEXT:    movdqa (%rdi), %xmm10
+; SSE2-NEXT:    movdqa 16(%rdi), %xmm9
+; SSE2-NEXT:    movdqa 32(%rdi), %xmm15
+; SSE2-NEXT:    movdqa 48(%rdi), %xmm13
+; SSE2-NEXT:    movdqa 80(%rdi), %xmm7
+; SSE2-NEXT:    movdqa 64(%rdi), %xmm5
+; SSE2-NEXT:    movdqa 112(%rdi), %xmm3
+; SSE2-NEXT:    movdqa 96(%rdi), %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [255,255]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm0, %xmm4
+; SSE2-NEXT:    pxor %xmm1, %xmm4
+; SSE2-NEXT:    movdqa {{.*#+}} xmm14 = [2147483903,2147483903]
+; SSE2-NEXT:    movdqa %xmm14, %xmm6
 ; SSE2-NEXT:    pcmpgtd %xmm4, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm11, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    pand %xmm7, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm14 = xmm6[1,1,3,3]
-; SSE2-NEXT:    por %xmm4, %xmm14
-; SSE2-NEXT:    pand %xmm14, %xmm5
-; SSE2-NEXT:    pandn %xmm10, %xmm14
-; SSE2-NEXT:    por %xmm5, %xmm14
-; SSE2-NEXT:    movdqa %xmm2, %xmm4
-; SSE2-NEXT:    pxor %xmm8, %xmm4
-; SSE2-NEXT:    movdqa %xmm11, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm4, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm5[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm11, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm14, %xmm4
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    pand %xmm7, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE2-NEXT:    por %xmm4, %xmm5
-; SSE2-NEXT:    pand %xmm5, %xmm2
-; SSE2-NEXT:    pandn %xmm10, %xmm5
-; SSE2-NEXT:    por %xmm2, %xmm5
-; SSE2-NEXT:    movdqa %xmm3, %xmm2
-; SSE2-NEXT:    pxor %xmm8, %xmm2
-; SSE2-NEXT:    movdqa %xmm11, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm11, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[1,1,3,3]
-; SSE2-NEXT:    pand %xmm7, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm6, %xmm2
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pandn %xmm10, %xmm2
-; SSE2-NEXT:    por %xmm3, %xmm2
-; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    pxor %xmm8, %xmm3
-; SSE2-NEXT:    movdqa %xmm11, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm11, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm3[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm7, %xmm3
-; SSE2-NEXT:    pand %xmm3, %xmm0
-; SSE2-NEXT:    pandn %xmm10, %xmm3
-; SSE2-NEXT:    por %xmm0, %xmm3
-; SSE2-NEXT:    movdqa %xmm1, %xmm0
-; SSE2-NEXT:    pxor %xmm8, %xmm0
-; SSE2-NEXT:    movdqa %xmm11, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm11, %xmm0
+; SSE2-NEXT:    pand %xmm2, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm6[1,1,3,3]
+; SSE2-NEXT:    por %xmm4, %xmm11
+; SSE2-NEXT:    pand %xmm11, %xmm0
+; SSE2-NEXT:    pandn %xmm8, %xmm11
+; SSE2-NEXT:    por %xmm0, %xmm11
+; SSE2-NEXT:    movdqa %xmm3, %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm0
+; SSE2-NEXT:    movdqa %xmm14, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm14, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    pand %xmm4, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm2[1,1,3,3]
+; SSE2-NEXT:    por %xmm0, %xmm12
+; SSE2-NEXT:    pand %xmm12, %xmm3
+; SSE2-NEXT:    pandn %xmm8, %xmm12
+; SSE2-NEXT:    por %xmm3, %xmm12
+; SSE2-NEXT:    movdqa %xmm5, %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm0
+; SSE2-NEXT:    movdqa %xmm14, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm14, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,1,3,3]
 ; SSE2-NEXT:    por %xmm0, %xmm4
-; SSE2-NEXT:    pand %xmm4, %xmm1
-; SSE2-NEXT:    pandn %xmm10, %xmm4
-; SSE2-NEXT:    por %xmm1, %xmm4
-; SSE2-NEXT:    movdqa %xmm4, %xmm0
-; SSE2-NEXT:    pxor %xmm8, %xmm0
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm8, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm8, %xmm0
+; SSE2-NEXT:    pand %xmm4, %xmm5
+; SSE2-NEXT:    pandn %xmm8, %xmm4
+; SSE2-NEXT:    por %xmm5, %xmm4
+; SSE2-NEXT:    movdqa %xmm7, %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm0
+; SSE2-NEXT:    movdqa %xmm14, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm14, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm0, %xmm1
-; SSE2-NEXT:    pand %xmm4, %xmm1
-; SSE2-NEXT:    movdqa %xmm3, %xmm0
-; SSE2-NEXT:    pxor %xmm8, %xmm0
-; SSE2-NEXT:    movdqa %xmm0, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm8, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm8, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm7, %xmm0
 ; SSE2-NEXT:    pand %xmm3, %xmm0
-; SSE2-NEXT:    packuswb %xmm1, %xmm0
-; SSE2-NEXT:    movdqa %xmm2, %xmm1
-; SSE2-NEXT:    pxor %xmm8, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm3
-; SSE2-NEXT:    pcmpgtd %xmm8, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm8, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pand %xmm4, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm3
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    movdqa %xmm5, %xmm1
-; SSE2-NEXT:    pxor %xmm8, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm2
-; SSE2-NEXT:    pcmpgtd %xmm8, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm8, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pand %xmm4, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[1,1,3,3]
+; SSE2-NEXT:    por %xmm0, %xmm5
+; SSE2-NEXT:    pand %xmm5, %xmm7
+; SSE2-NEXT:    pandn %xmm8, %xmm5
+; SSE2-NEXT:    por %xmm7, %xmm5
+; SSE2-NEXT:    movdqa %xmm15, %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm0
+; SSE2-NEXT:    movdqa %xmm14, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm14, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm2[1,1,3,3]
+; SSE2-NEXT:    por %xmm0, %xmm7
+; SSE2-NEXT:    pand %xmm7, %xmm15
+; SSE2-NEXT:    pandn %xmm8, %xmm7
+; SSE2-NEXT:    por %xmm15, %xmm7
+; SSE2-NEXT:    movdqa %xmm13, %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm0
+; SSE2-NEXT:    movdqa %xmm14, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm14, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm15 = xmm2[1,1,3,3]
+; SSE2-NEXT:    por %xmm0, %xmm15
+; SSE2-NEXT:    pand %xmm15, %xmm13
+; SSE2-NEXT:    pandn %xmm8, %xmm15
+; SSE2-NEXT:    por %xmm13, %xmm15
+; SSE2-NEXT:    movdqa %xmm10, %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm0
+; SSE2-NEXT:    movdqa %xmm14, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm14, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pand %xmm6, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm3[1,1,3,3]
+; SSE2-NEXT:    por %xmm0, %xmm13
+; SSE2-NEXT:    pand %xmm13, %xmm10
+; SSE2-NEXT:    pandn %xmm8, %xmm13
+; SSE2-NEXT:    por %xmm10, %xmm13
+; SSE2-NEXT:    movdqa %xmm9, %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm0
+; SSE2-NEXT:    movdqa %xmm14, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm6[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm14, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pand %xmm10, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSE2-NEXT:    por %xmm0, %xmm6
+; SSE2-NEXT:    pand %xmm6, %xmm9
+; SSE2-NEXT:    pandn %xmm8, %xmm6
+; SSE2-NEXT:    por %xmm9, %xmm6
+; SSE2-NEXT:    movdqa %xmm6, %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm2[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pand %xmm8, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm2
-; SSE2-NEXT:    pand %xmm5, %xmm2
-; SSE2-NEXT:    packuswb %xmm3, %xmm2
+; SSE2-NEXT:    por %xmm0, %xmm2
+; SSE2-NEXT:    pand %xmm6, %xmm2
+; SSE2-NEXT:    movdqa %xmm13, %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm6[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pand %xmm8, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3]
+; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    pand %xmm13, %xmm0
 ; SSE2-NEXT:    packuswb %xmm2, %xmm0
-; SSE2-NEXT:    movdqa %xmm14, %xmm1
-; SSE2-NEXT:    pxor %xmm8, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm2
-; SSE2-NEXT:    pcmpgtd %xmm8, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm8, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pand %xmm3, %xmm1
+; SSE2-NEXT:    movdqa %xmm15, %xmm2
+; SSE2-NEXT:    pxor %xmm1, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm1, %xmm2
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm2
-; SSE2-NEXT:    pand %xmm14, %xmm2
-; SSE2-NEXT:    movdqa %xmm13, %xmm1
-; SSE2-NEXT:    pxor %xmm8, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm3
-; SSE2-NEXT:    pcmpgtd %xmm8, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm8, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pand %xmm4, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE2-NEXT:    por %xmm5, %xmm1
-; SSE2-NEXT:    pand %xmm13, %xmm1
-; SSE2-NEXT:    packuswb %xmm2, %xmm1
-; SSE2-NEXT:    movdqa %xmm12, %xmm2
-; SSE2-NEXT:    pxor %xmm8, %xmm2
+; SSE2-NEXT:    pand %xmm6, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSE2-NEXT:    por %xmm2, %xmm3
+; SSE2-NEXT:    pand %xmm15, %xmm3
+; SSE2-NEXT:    movdqa %xmm7, %xmm2
+; SSE2-NEXT:    pxor %xmm1, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm6[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm1, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE2-NEXT:    pand %xmm8, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSE2-NEXT:    por %xmm2, %xmm6
+; SSE2-NEXT:    pand %xmm7, %xmm6
+; SSE2-NEXT:    packuswb %xmm3, %xmm6
+; SSE2-NEXT:    packuswb %xmm6, %xmm0
+; SSE2-NEXT:    movdqa %xmm5, %xmm2
+; SSE2-NEXT:    pxor %xmm1, %xmm2
 ; SSE2-NEXT:    movdqa %xmm2, %xmm3
-; SSE2-NEXT:    pcmpgtd %xmm8, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm8, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm1, %xmm2
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT:    pand %xmm4, %xmm2
+; SSE2-NEXT:    pand %xmm6, %xmm2
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
 ; SSE2-NEXT:    por %xmm2, %xmm3
-; SSE2-NEXT:    pand %xmm12, %xmm3
-; SSE2-NEXT:    movdqa %xmm9, %xmm2
-; SSE2-NEXT:    pxor %xmm8, %xmm2
-; SSE2-NEXT:    movdqa %xmm2, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm8, %xmm4
+; SSE2-NEXT:    pand %xmm5, %xmm3
+; SSE2-NEXT:    movdqa %xmm4, %xmm2
+; SSE2-NEXT:    pxor %xmm1, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm1, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm2[1,1,3,3]
+; SSE2-NEXT:    pand %xmm6, %xmm7
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm5[1,1,3,3]
+; SSE2-NEXT:    por %xmm7, %xmm2
+; SSE2-NEXT:    pand %xmm4, %xmm2
+; SSE2-NEXT:    packuswb %xmm3, %xmm2
+; SSE2-NEXT:    movdqa %xmm12, %xmm3
+; SSE2-NEXT:    pxor %xmm1, %xmm3
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm4
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm8, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT:    pand %xmm5, %xmm2
+; SSE2-NEXT:    pcmpeqd %xmm1, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSE2-NEXT:    pand %xmm5, %xmm3
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm2, %xmm4
-; SSE2-NEXT:    pand %xmm9, %xmm4
-; SSE2-NEXT:    packuswb %xmm3, %xmm4
-; SSE2-NEXT:    packuswb %xmm4, %xmm1
-; SSE2-NEXT:    packuswb %xmm1, %xmm0
+; SSE2-NEXT:    por %xmm3, %xmm4
+; SSE2-NEXT:    pand %xmm12, %xmm4
+; SSE2-NEXT:    movdqa %xmm11, %xmm3
+; SSE2-NEXT:    pxor %xmm1, %xmm3
+; SSE2-NEXT:    movdqa %xmm3, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm1, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
+; SSE2-NEXT:    pand %xmm6, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm5[1,1,3,3]
+; SSE2-NEXT:    por %xmm1, %xmm3
+; SSE2-NEXT:    pand %xmm11, %xmm3
+; SSE2-NEXT:    packuswb %xmm4, %xmm3
+; SSE2-NEXT:    packuswb %xmm3, %xmm2
+; SSE2-NEXT:    packuswb %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: trunc_packus_v16i64_v16i8:
 ; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm10 = [255,255]
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm8 = [2147483648,2147483648]
-; SSSE3-NEXT:    movdqa %xmm6, %xmm9
-; SSSE3-NEXT:    pxor %xmm8, %xmm9
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm11 = [2147483903,2147483903]
-; SSSE3-NEXT:    movdqa %xmm11, %xmm12
-; SSSE3-NEXT:    pcmpgtd %xmm9, %xmm12
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm13 = xmm12[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm11, %xmm9
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm14 = xmm9[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm13, %xmm14
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm9 = xmm12[1,1,3,3]
-; SSSE3-NEXT:    por %xmm14, %xmm9
-; SSSE3-NEXT:    pand %xmm9, %xmm6
-; SSSE3-NEXT:    pandn %xmm10, %xmm9
-; SSSE3-NEXT:    por %xmm6, %xmm9
-; SSSE3-NEXT:    movdqa %xmm7, %xmm6
-; SSSE3-NEXT:    pxor %xmm8, %xmm6
-; SSSE3-NEXT:    movdqa %xmm11, %xmm12
-; SSSE3-NEXT:    pcmpgtd %xmm6, %xmm12
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm13 = xmm12[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm11, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm13, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm12 = xmm12[1,1,3,3]
-; SSSE3-NEXT:    por %xmm6, %xmm12
-; SSSE3-NEXT:    pand %xmm12, %xmm7
-; SSSE3-NEXT:    pandn %xmm10, %xmm12
-; SSSE3-NEXT:    por %xmm7, %xmm12
-; SSSE3-NEXT:    movdqa %xmm4, %xmm6
-; SSSE3-NEXT:    pxor %xmm8, %xmm6
-; SSSE3-NEXT:    movdqa %xmm11, %xmm7
-; SSSE3-NEXT:    pcmpgtd %xmm6, %xmm7
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm13 = xmm7[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm11, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm13, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm13 = xmm7[1,1,3,3]
-; SSSE3-NEXT:    por %xmm6, %xmm13
-; SSSE3-NEXT:    pand %xmm13, %xmm4
-; SSSE3-NEXT:    pandn %xmm10, %xmm13
-; SSSE3-NEXT:    por %xmm4, %xmm13
-; SSSE3-NEXT:    movdqa %xmm5, %xmm4
-; SSSE3-NEXT:    pxor %xmm8, %xmm4
-; SSSE3-NEXT:    movdqa %xmm11, %xmm6
+; SSSE3-NEXT:    movdqa (%rdi), %xmm10
+; SSSE3-NEXT:    movdqa 16(%rdi), %xmm9
+; SSSE3-NEXT:    movdqa 32(%rdi), %xmm15
+; SSSE3-NEXT:    movdqa 48(%rdi), %xmm13
+; SSSE3-NEXT:    movdqa 80(%rdi), %xmm7
+; SSSE3-NEXT:    movdqa 64(%rdi), %xmm5
+; SSSE3-NEXT:    movdqa 112(%rdi), %xmm3
+; SSSE3-NEXT:    movdqa 96(%rdi), %xmm0
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm8 = [255,255]
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [2147483648,2147483648]
+; SSSE3-NEXT:    movdqa %xmm0, %xmm4
+; SSSE3-NEXT:    pxor %xmm1, %xmm4
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm14 = [2147483903,2147483903]
+; SSSE3-NEXT:    movdqa %xmm14, %xmm6
 ; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm11, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm14, %xmm4
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm7, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm14 = xmm6[1,1,3,3]
-; SSSE3-NEXT:    por %xmm4, %xmm14
-; SSSE3-NEXT:    pand %xmm14, %xmm5
-; SSSE3-NEXT:    pandn %xmm10, %xmm14
-; SSSE3-NEXT:    por %xmm5, %xmm14
-; SSSE3-NEXT:    movdqa %xmm2, %xmm4
-; SSSE3-NEXT:    pxor %xmm8, %xmm4
-; SSSE3-NEXT:    movdqa %xmm11, %xmm5
-; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm5
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm5[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm11, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm7, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSSE3-NEXT:    por %xmm4, %xmm5
-; SSSE3-NEXT:    pand %xmm5, %xmm2
-; SSSE3-NEXT:    pandn %xmm10, %xmm5
-; SSSE3-NEXT:    por %xmm2, %xmm5
-; SSSE3-NEXT:    movdqa %xmm3, %xmm2
-; SSSE3-NEXT:    pxor %xmm8, %xmm2
-; SSSE3-NEXT:    movdqa %xmm11, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm11, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm7, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    por %xmm6, %xmm2
-; SSSE3-NEXT:    pand %xmm2, %xmm3
-; SSSE3-NEXT:    pandn %xmm10, %xmm2
-; SSSE3-NEXT:    por %xmm3, %xmm2
-; SSSE3-NEXT:    movdqa %xmm0, %xmm3
-; SSSE3-NEXT:    pxor %xmm8, %xmm3
-; SSSE3-NEXT:    movdqa %xmm11, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm11, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm6, %xmm7
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    por %xmm7, %xmm3
-; SSSE3-NEXT:    pand %xmm3, %xmm0
-; SSSE3-NEXT:    pandn %xmm10, %xmm3
-; SSSE3-NEXT:    por %xmm0, %xmm3
-; SSSE3-NEXT:    movdqa %xmm1, %xmm0
-; SSSE3-NEXT:    pxor %xmm8, %xmm0
-; SSSE3-NEXT:    movdqa %xmm11, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm11, %xmm0
+; SSSE3-NEXT:    pand %xmm2, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm11 = xmm6[1,1,3,3]
+; SSSE3-NEXT:    por %xmm4, %xmm11
+; SSSE3-NEXT:    pand %xmm11, %xmm0
+; SSSE3-NEXT:    pandn %xmm8, %xmm11
+; SSSE3-NEXT:    por %xmm0, %xmm11
+; SSSE3-NEXT:    movdqa %xmm3, %xmm0
+; SSSE3-NEXT:    pxor %xmm1, %xmm0
+; SSSE3-NEXT:    movdqa %xmm14, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm14, %xmm0
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm6, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm4, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm12 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    por %xmm0, %xmm12
+; SSSE3-NEXT:    pand %xmm12, %xmm3
+; SSSE3-NEXT:    pandn %xmm8, %xmm12
+; SSSE3-NEXT:    por %xmm3, %xmm12
+; SSSE3-NEXT:    movdqa %xmm5, %xmm0
+; SSSE3-NEXT:    pxor %xmm1, %xmm0
+; SSSE3-NEXT:    movdqa %xmm14, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm14, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,1,3,3]
 ; SSSE3-NEXT:    por %xmm0, %xmm4
-; SSSE3-NEXT:    pand %xmm4, %xmm1
-; SSSE3-NEXT:    pandn %xmm10, %xmm4
-; SSSE3-NEXT:    por %xmm1, %xmm4
-; SSSE3-NEXT:    movdqa %xmm4, %xmm0
-; SSSE3-NEXT:    pxor %xmm8, %xmm0
-; SSSE3-NEXT:    movdqa %xmm0, %xmm1
-; SSSE3-NEXT:    pcmpgtd %xmm8, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm8, %xmm0
+; SSSE3-NEXT:    pand %xmm4, %xmm5
+; SSSE3-NEXT:    pandn %xmm8, %xmm4
+; SSSE3-NEXT:    por %xmm5, %xmm4
+; SSSE3-NEXT:    movdqa %xmm7, %xmm0
+; SSSE3-NEXT:    pxor %xmm1, %xmm0
+; SSSE3-NEXT:    movdqa %xmm14, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm14, %xmm0
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm6, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    por %xmm0, %xmm1
-; SSSE3-NEXT:    pand %xmm4, %xmm1
-; SSSE3-NEXT:    movdqa %xmm3, %xmm0
-; SSSE3-NEXT:    pxor %xmm8, %xmm0
-; SSSE3-NEXT:    movdqa %xmm0, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm8, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm8, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm6, %xmm7
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    por %xmm7, %xmm0
 ; SSSE3-NEXT:    pand %xmm3, %xmm0
-; SSSE3-NEXT:    packuswb %xmm1, %xmm0
-; SSSE3-NEXT:    movdqa %xmm2, %xmm1
-; SSSE3-NEXT:    pxor %xmm8, %xmm1
-; SSSE3-NEXT:    movdqa %xmm1, %xmm3
-; SSSE3-NEXT:    pcmpgtd %xmm8, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm8, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm4, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm1, %xmm3
-; SSSE3-NEXT:    pand %xmm2, %xmm3
-; SSSE3-NEXT:    movdqa %xmm5, %xmm1
-; SSSE3-NEXT:    pxor %xmm8, %xmm1
-; SSSE3-NEXT:    movdqa %xmm1, %xmm2
-; SSSE3-NEXT:    pcmpgtd %xmm8, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm8, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm4, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    por %xmm0, %xmm5
+; SSSE3-NEXT:    pand %xmm5, %xmm7
+; SSSE3-NEXT:    pandn %xmm8, %xmm5
+; SSSE3-NEXT:    por %xmm7, %xmm5
+; SSSE3-NEXT:    movdqa %xmm15, %xmm0
+; SSSE3-NEXT:    pxor %xmm1, %xmm0
+; SSSE3-NEXT:    movdqa %xmm14, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm14, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    por %xmm0, %xmm7
+; SSSE3-NEXT:    pand %xmm7, %xmm15
+; SSSE3-NEXT:    pandn %xmm8, %xmm7
+; SSSE3-NEXT:    por %xmm15, %xmm7
+; SSSE3-NEXT:    movdqa %xmm13, %xmm0
+; SSSE3-NEXT:    pxor %xmm1, %xmm0
+; SSSE3-NEXT:    movdqa %xmm14, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm14, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm15 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    por %xmm0, %xmm15
+; SSSE3-NEXT:    pand %xmm15, %xmm13
+; SSSE3-NEXT:    pandn %xmm8, %xmm15
+; SSSE3-NEXT:    por %xmm13, %xmm15
+; SSSE3-NEXT:    movdqa %xmm10, %xmm0
+; SSSE3-NEXT:    pxor %xmm1, %xmm0
+; SSSE3-NEXT:    movdqa %xmm14, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm14, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm6, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm13 = xmm3[1,1,3,3]
+; SSSE3-NEXT:    por %xmm0, %xmm13
+; SSSE3-NEXT:    pand %xmm13, %xmm10
+; SSSE3-NEXT:    pandn %xmm8, %xmm13
+; SSSE3-NEXT:    por %xmm10, %xmm13
+; SSSE3-NEXT:    movdqa %xmm9, %xmm0
+; SSSE3-NEXT:    pxor %xmm1, %xmm0
+; SSSE3-NEXT:    movdqa %xmm14, %xmm6
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm10 = xmm6[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm14, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm10, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSSE3-NEXT:    por %xmm0, %xmm6
+; SSSE3-NEXT:    pand %xmm6, %xmm9
+; SSSE3-NEXT:    pandn %xmm8, %xmm6
+; SSSE3-NEXT:    por %xmm9, %xmm6
+; SSSE3-NEXT:    movdqa %xmm6, %xmm0
+; SSSE3-NEXT:    pxor %xmm1, %xmm0
+; SSSE3-NEXT:    movdqa %xmm0, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm2[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm8, %xmm0
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSSE3-NEXT:    por %xmm1, %xmm2
-; SSSE3-NEXT:    pand %xmm5, %xmm2
-; SSSE3-NEXT:    packuswb %xmm3, %xmm2
+; SSSE3-NEXT:    por %xmm0, %xmm2
+; SSSE3-NEXT:    pand %xmm6, %xmm2
+; SSSE3-NEXT:    movdqa %xmm13, %xmm0
+; SSSE3-NEXT:    pxor %xmm1, %xmm0
+; SSSE3-NEXT:    movdqa %xmm0, %xmm6
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm6[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm8, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3]
+; SSSE3-NEXT:    por %xmm3, %xmm0
+; SSSE3-NEXT:    pand %xmm13, %xmm0
 ; SSSE3-NEXT:    packuswb %xmm2, %xmm0
-; SSSE3-NEXT:    movdqa %xmm14, %xmm1
-; SSSE3-NEXT:    pxor %xmm8, %xmm1
-; SSSE3-NEXT:    movdqa %xmm1, %xmm2
-; SSSE3-NEXT:    pcmpgtd %xmm8, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm8, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm3, %xmm1
+; SSSE3-NEXT:    movdqa %xmm15, %xmm2
+; SSSE3-NEXT:    pxor %xmm1, %xmm2
+; SSSE3-NEXT:    movdqa %xmm2, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm2
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSSE3-NEXT:    por %xmm1, %xmm2
-; SSSE3-NEXT:    pand %xmm14, %xmm2
-; SSSE3-NEXT:    movdqa %xmm13, %xmm1
-; SSSE3-NEXT:    pxor %xmm8, %xmm1
-; SSSE3-NEXT:    movdqa %xmm1, %xmm3
-; SSSE3-NEXT:    pcmpgtd %xmm8, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm8, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm4, %xmm5
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm5, %xmm1
-; SSSE3-NEXT:    pand %xmm13, %xmm1
-; SSSE3-NEXT:    packuswb %xmm2, %xmm1
-; SSSE3-NEXT:    movdqa %xmm12, %xmm2
-; SSSE3-NEXT:    pxor %xmm8, %xmm2
+; SSSE3-NEXT:    pand %xmm6, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSSE3-NEXT:    por %xmm2, %xmm3
+; SSSE3-NEXT:    pand %xmm15, %xmm3
+; SSSE3-NEXT:    movdqa %xmm7, %xmm2
+; SSSE3-NEXT:    pxor %xmm1, %xmm2
+; SSSE3-NEXT:    movdqa %xmm2, %xmm6
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm6[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm8, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSSE3-NEXT:    por %xmm2, %xmm6
+; SSSE3-NEXT:    pand %xmm7, %xmm6
+; SSSE3-NEXT:    packuswb %xmm3, %xmm6
+; SSSE3-NEXT:    packuswb %xmm6, %xmm0
+; SSSE3-NEXT:    movdqa %xmm5, %xmm2
+; SSSE3-NEXT:    pxor %xmm1, %xmm2
 ; SSSE3-NEXT:    movdqa %xmm2, %xmm3
-; SSSE3-NEXT:    pcmpgtd %xmm8, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm8, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm2
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm4, %xmm2
+; SSSE3-NEXT:    pand %xmm6, %xmm2
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
 ; SSSE3-NEXT:    por %xmm2, %xmm3
-; SSSE3-NEXT:    pand %xmm12, %xmm3
-; SSSE3-NEXT:    movdqa %xmm9, %xmm2
-; SSSE3-NEXT:    pxor %xmm8, %xmm2
-; SSSE3-NEXT:    movdqa %xmm2, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm8, %xmm4
+; SSSE3-NEXT:    pand %xmm5, %xmm3
+; SSSE3-NEXT:    movdqa %xmm4, %xmm2
+; SSSE3-NEXT:    pxor %xmm1, %xmm2
+; SSSE3-NEXT:    movdqa %xmm2, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm6, %xmm7
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm5[1,1,3,3]
+; SSSE3-NEXT:    por %xmm7, %xmm2
+; SSSE3-NEXT:    pand %xmm4, %xmm2
+; SSSE3-NEXT:    packuswb %xmm3, %xmm2
+; SSSE3-NEXT:    movdqa %xmm12, %xmm3
+; SSSE3-NEXT:    pxor %xmm1, %xmm3
+; SSSE3-NEXT:    movdqa %xmm3, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm4
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm8, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm5, %xmm2
+; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm5, %xmm3
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm4
-; SSSE3-NEXT:    pand %xmm9, %xmm4
-; SSSE3-NEXT:    packuswb %xmm3, %xmm4
-; SSSE3-NEXT:    packuswb %xmm4, %xmm1
-; SSSE3-NEXT:    packuswb %xmm1, %xmm0
+; SSSE3-NEXT:    por %xmm3, %xmm4
+; SSSE3-NEXT:    pand %xmm12, %xmm4
+; SSSE3-NEXT:    movdqa %xmm11, %xmm3
+; SSSE3-NEXT:    pxor %xmm1, %xmm3
+; SSSE3-NEXT:    movdqa %xmm3, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm6, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm5[1,1,3,3]
+; SSSE3-NEXT:    por %xmm1, %xmm3
+; SSSE3-NEXT:    pand %xmm11, %xmm3
+; SSSE3-NEXT:    packuswb %xmm4, %xmm3
+; SSSE3-NEXT:    packuswb %xmm3, %xmm2
+; SSSE3-NEXT:    packuswb %xmm2, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: trunc_packus_v16i64_v16i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa %xmm0, %xmm8
-; SSE41-NEXT:    movapd {{.*#+}} xmm11 = [255,255]
-; SSE41-NEXT:    movdqa {{.*#+}} xmm9 = [2147483648,2147483648]
-; SSE41-NEXT:    movdqa %xmm6, %xmm0
-; SSE41-NEXT:    pxor %xmm9, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm12 = [2147483903,2147483903]
-; SSE41-NEXT:    movdqa %xmm12, %xmm10
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm10
-; SSE41-NEXT:    pshufd {{.*#+}} xmm13 = xmm10[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm12, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm13, %xmm0
-; SSE41-NEXT:    por %xmm10, %xmm0
-; SSE41-NEXT:    movapd %xmm11, %xmm10
-; SSE41-NEXT:    blendvpd %xmm0, %xmm6, %xmm10
-; SSE41-NEXT:    movdqa %xmm7, %xmm0
-; SSE41-NEXT:    pxor %xmm9, %xmm0
-; SSE41-NEXT:    movdqa %xmm12, %xmm6
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm13 = xmm6[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm12, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm13, %xmm0
-; SSE41-NEXT:    por %xmm6, %xmm0
-; SSE41-NEXT:    movapd %xmm11, %xmm13
-; SSE41-NEXT:    blendvpd %xmm0, %xmm7, %xmm13
+; SSE41-NEXT:    movdqa (%rdi), %xmm10
+; SSE41-NEXT:    movdqa 16(%rdi), %xmm9
+; SSE41-NEXT:    movdqa 32(%rdi), %xmm14
+; SSE41-NEXT:    movdqa 48(%rdi), %xmm12
+; SSE41-NEXT:    movdqa 80(%rdi), %xmm15
+; SSE41-NEXT:    movdqa 64(%rdi), %xmm6
+; SSE41-NEXT:    movdqa 112(%rdi), %xmm13
+; SSE41-NEXT:    movdqa 96(%rdi), %xmm4
+; SSE41-NEXT:    movapd {{.*#+}} xmm1 = [255,255]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
 ; SSE41-NEXT:    movdqa %xmm4, %xmm0
-; SSE41-NEXT:    pxor %xmm9, %xmm0
-; SSE41-NEXT:    movdqa %xmm12, %xmm6
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm12, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm7, %xmm0
-; SSE41-NEXT:    por %xmm6, %xmm0
-; SSE41-NEXT:    movapd %xmm11, %xmm14
-; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm14
-; SSE41-NEXT:    movdqa %xmm5, %xmm0
-; SSE41-NEXT:    pxor %xmm9, %xmm0
-; SSE41-NEXT:    movdqa %xmm12, %xmm4
+; SSE41-NEXT:    pxor %xmm2, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm7 = [2147483903,2147483903]
+; SSE41-NEXT:    movdqa %xmm7, %xmm3
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm3
+; SSE41-NEXT:    movdqa %xmm7, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm5
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2]
+; SSE41-NEXT:    pand %xmm3, %xmm0
+; SSE41-NEXT:    por %xmm5, %xmm0
+; SSE41-NEXT:    movapd %xmm1, %xmm8
+; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm8
+; SSE41-NEXT:    movdqa %xmm13, %xmm0
+; SSE41-NEXT:    pxor %xmm2, %xmm0
+; SSE41-NEXT:    movdqa %xmm7, %xmm3
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm3
+; SSE41-NEXT:    movdqa %xmm7, %xmm4
 ; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm12, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm6, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pand %xmm3, %xmm0
 ; SSE41-NEXT:    por %xmm4, %xmm0
-; SSE41-NEXT:    movapd %xmm11, %xmm15
-; SSE41-NEXT:    blendvpd %xmm0, %xmm5, %xmm15
-; SSE41-NEXT:    movdqa %xmm2, %xmm0
-; SSE41-NEXT:    pxor %xmm9, %xmm0
-; SSE41-NEXT:    movdqa %xmm12, %xmm5
+; SSE41-NEXT:    movapd %xmm1, %xmm11
+; SSE41-NEXT:    blendvpd %xmm0, %xmm13, %xmm11
+; SSE41-NEXT:    movdqa %xmm6, %xmm0
+; SSE41-NEXT:    pxor %xmm2, %xmm0
+; SSE41-NEXT:    movdqa %xmm7, %xmm3
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm3
+; SSE41-NEXT:    movdqa %xmm7, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pand %xmm3, %xmm0
+; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    movapd %xmm1, %xmm13
+; SSE41-NEXT:    blendvpd %xmm0, %xmm6, %xmm13
+; SSE41-NEXT:    movdqa %xmm15, %xmm0
+; SSE41-NEXT:    pxor %xmm2, %xmm0
+; SSE41-NEXT:    movdqa %xmm7, %xmm3
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm3
+; SSE41-NEXT:    movdqa %xmm7, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pand %xmm3, %xmm0
+; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    movapd %xmm1, %xmm6
+; SSE41-NEXT:    blendvpd %xmm0, %xmm15, %xmm6
+; SSE41-NEXT:    movdqa %xmm14, %xmm0
+; SSE41-NEXT:    pxor %xmm2, %xmm0
+; SSE41-NEXT:    movdqa %xmm7, %xmm3
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm3
+; SSE41-NEXT:    movdqa %xmm7, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pand %xmm3, %xmm0
+; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    movapd %xmm1, %xmm15
+; SSE41-NEXT:    blendvpd %xmm0, %xmm14, %xmm15
+; SSE41-NEXT:    movdqa %xmm12, %xmm0
+; SSE41-NEXT:    pxor %xmm2, %xmm0
+; SSE41-NEXT:    movdqa %xmm7, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT:    movdqa %xmm7, %xmm5
 ; SSE41-NEXT:    pcmpgtd %xmm0, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm12, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm6, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
 ; SSE41-NEXT:    por %xmm5, %xmm0
-; SSE41-NEXT:    movapd %xmm11, %xmm5
-; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm5
-; SSE41-NEXT:    movdqa %xmm3, %xmm0
-; SSE41-NEXT:    pxor %xmm9, %xmm0
-; SSE41-NEXT:    movdqa %xmm12, %xmm2
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm12, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm6, %xmm0
-; SSE41-NEXT:    por %xmm2, %xmm0
-; SSE41-NEXT:    movapd %xmm11, %xmm6
-; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm6
-; SSE41-NEXT:    movdqa %xmm8, %xmm0
-; SSE41-NEXT:    pxor %xmm9, %xmm0
-; SSE41-NEXT:    movdqa %xmm12, %xmm2
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm12, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE41-NEXT:    movapd %xmm1, %xmm4
+; SSE41-NEXT:    blendvpd %xmm0, %xmm12, %xmm4
+; SSE41-NEXT:    movdqa %xmm10, %xmm0
+; SSE41-NEXT:    pxor %xmm2, %xmm0
+; SSE41-NEXT:    movdqa %xmm7, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm5
+; SSE41-NEXT:    movdqa %xmm7, %xmm3
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm0
+; SSE41-NEXT:    por %xmm3, %xmm0
+; SSE41-NEXT:    movapd %xmm1, %xmm5
+; SSE41-NEXT:    blendvpd %xmm0, %xmm10, %xmm5
+; SSE41-NEXT:    movdqa %xmm9, %xmm0
+; SSE41-NEXT:    pxor %xmm2, %xmm0
+; SSE41-NEXT:    movdqa %xmm7, %xmm3
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm3
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm7
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[0,0,2,2]
 ; SSE41-NEXT:    pand %xmm3, %xmm0
-; SSE41-NEXT:    por %xmm2, %xmm0
-; SSE41-NEXT:    movapd %xmm11, %xmm3
-; SSE41-NEXT:    blendvpd %xmm0, %xmm8, %xmm3
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    pxor %xmm9, %xmm0
-; SSE41-NEXT:    movdqa %xmm12, %xmm2
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm2[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm12, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE41-NEXT:    por %xmm7, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm9, %xmm1
+; SSE41-NEXT:    xorpd %xmm9, %xmm9
+; SSE41-NEXT:    movapd %xmm1, %xmm3
+; SSE41-NEXT:    xorpd %xmm2, %xmm3
+; SSE41-NEXT:    movapd %xmm3, %xmm7
+; SSE41-NEXT:    pcmpeqd %xmm2, %xmm7
+; SSE41-NEXT:    pcmpgtd %xmm2, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
 ; SSE41-NEXT:    pand %xmm7, %xmm0
-; SSE41-NEXT:    por %xmm2, %xmm0
-; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm11
-; SSE41-NEXT:    pxor %xmm2, %xmm2
-; SSE41-NEXT:    movapd %xmm11, %xmm0
-; SSE41-NEXT:    xorpd %xmm9, %xmm0
-; SSE41-NEXT:    movapd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd %xmm9, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm1[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm9, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE41-NEXT:    por %xmm3, %xmm0
+; SSE41-NEXT:    pxor %xmm3, %xmm3
+; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm3
+; SSE41-NEXT:    movapd %xmm5, %xmm1
+; SSE41-NEXT:    xorpd %xmm2, %xmm1
+; SSE41-NEXT:    movapd %xmm1, %xmm7
+; SSE41-NEXT:    pcmpeqd %xmm2, %xmm7
+; SSE41-NEXT:    pcmpgtd %xmm2, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2]
 ; SSE41-NEXT:    pand %xmm7, %xmm0
 ; SSE41-NEXT:    por %xmm1, %xmm0
-; SSE41-NEXT:    pxor %xmm7, %xmm7
-; SSE41-NEXT:    blendvpd %xmm0, %xmm11, %xmm7
-; SSE41-NEXT:    movapd %xmm3, %xmm0
-; SSE41-NEXT:    xorpd %xmm9, %xmm0
-; SSE41-NEXT:    movapd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd %xmm9, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm9, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm4, %xmm0
-; SSE41-NEXT:    por %xmm1, %xmm0
 ; SSE41-NEXT:    pxor %xmm1, %xmm1
-; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm1
-; SSE41-NEXT:    packusdw %xmm7, %xmm1
-; SSE41-NEXT:    movapd %xmm6, %xmm0
-; SSE41-NEXT:    xorpd %xmm9, %xmm0
-; SSE41-NEXT:    movapd %xmm0, %xmm3
-; SSE41-NEXT:    pcmpgtd %xmm9, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm9, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm5, %xmm1
+; SSE41-NEXT:    packusdw %xmm3, %xmm1
+; SSE41-NEXT:    movapd %xmm4, %xmm3
+; SSE41-NEXT:    xorpd %xmm2, %xmm3
+; SSE41-NEXT:    movapd %xmm3, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm2, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm2, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm0
 ; SSE41-NEXT:    por %xmm3, %xmm0
 ; SSE41-NEXT:    pxor %xmm3, %xmm3
-; SSE41-NEXT:    blendvpd %xmm0, %xmm6, %xmm3
-; SSE41-NEXT:    movapd %xmm5, %xmm0
-; SSE41-NEXT:    xorpd %xmm9, %xmm0
-; SSE41-NEXT:    movapd %xmm0, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm9, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm9, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm6, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm3
+; SSE41-NEXT:    movapd %xmm15, %xmm4
+; SSE41-NEXT:    xorpd %xmm2, %xmm4
+; SSE41-NEXT:    movapd %xmm4, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm2, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm2, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm0
 ; SSE41-NEXT:    por %xmm4, %xmm0
 ; SSE41-NEXT:    pxor %xmm4, %xmm4
-; SSE41-NEXT:    blendvpd %xmm0, %xmm5, %xmm4
+; SSE41-NEXT:    blendvpd %xmm0, %xmm15, %xmm4
 ; SSE41-NEXT:    packusdw %xmm3, %xmm4
 ; SSE41-NEXT:    packusdw %xmm4, %xmm1
-; SSE41-NEXT:    movapd %xmm15, %xmm0
-; SSE41-NEXT:    xorpd %xmm9, %xmm0
-; SSE41-NEXT:    movapd %xmm0, %xmm3
-; SSE41-NEXT:    pcmpgtd %xmm9, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm9, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE41-NEXT:    movapd %xmm6, %xmm3
+; SSE41-NEXT:    xorpd %xmm2, %xmm3
+; SSE41-NEXT:    movapd %xmm3, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm2, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm2, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
 ; SSE41-NEXT:    pand %xmm4, %xmm0
 ; SSE41-NEXT:    por %xmm3, %xmm0
 ; SSE41-NEXT:    pxor %xmm4, %xmm4
-; SSE41-NEXT:    blendvpd %xmm0, %xmm15, %xmm4
-; SSE41-NEXT:    movapd %xmm14, %xmm0
-; SSE41-NEXT:    xorpd %xmm9, %xmm0
-; SSE41-NEXT:    movapd %xmm0, %xmm3
-; SSE41-NEXT:    pcmpgtd %xmm9, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm9, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE41-NEXT:    blendvpd %xmm0, %xmm6, %xmm4
+; SSE41-NEXT:    movapd %xmm13, %xmm3
+; SSE41-NEXT:    xorpd %xmm2, %xmm3
+; SSE41-NEXT:    movapd %xmm3, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm2, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm2, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
 ; SSE41-NEXT:    pand %xmm5, %xmm0
 ; SSE41-NEXT:    por %xmm3, %xmm0
 ; SSE41-NEXT:    pxor %xmm3, %xmm3
-; SSE41-NEXT:    blendvpd %xmm0, %xmm14, %xmm3
+; SSE41-NEXT:    blendvpd %xmm0, %xmm13, %xmm3
 ; SSE41-NEXT:    packusdw %xmm4, %xmm3
-; SSE41-NEXT:    movapd %xmm13, %xmm0
-; SSE41-NEXT:    xorpd %xmm9, %xmm0
-; SSE41-NEXT:    movapd %xmm0, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm9, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm9, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE41-NEXT:    movapd %xmm11, %xmm4
+; SSE41-NEXT:    xorpd %xmm2, %xmm4
+; SSE41-NEXT:    movapd %xmm4, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm2, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm2, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2]
 ; SSE41-NEXT:    pand %xmm5, %xmm0
 ; SSE41-NEXT:    por %xmm4, %xmm0
 ; SSE41-NEXT:    pxor %xmm4, %xmm4
-; SSE41-NEXT:    blendvpd %xmm0, %xmm13, %xmm4
-; SSE41-NEXT:    movapd %xmm10, %xmm0
-; SSE41-NEXT:    xorpd %xmm9, %xmm0
-; SSE41-NEXT:    movapd %xmm0, %xmm5
-; SSE41-NEXT:    pcmpgtd %xmm9, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm9, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE41-NEXT:    blendvpd %xmm0, %xmm11, %xmm4
+; SSE41-NEXT:    movapd %xmm8, %xmm5
+; SSE41-NEXT:    xorpd %xmm2, %xmm5
+; SSE41-NEXT:    movapd %xmm5, %xmm6
+; SSE41-NEXT:    pcmpeqd %xmm2, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm2, %xmm5
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2]
 ; SSE41-NEXT:    pand %xmm6, %xmm0
 ; SSE41-NEXT:    por %xmm5, %xmm0
-; SSE41-NEXT:    blendvpd %xmm0, %xmm10, %xmm2
-; SSE41-NEXT:    packusdw %xmm4, %xmm2
-; SSE41-NEXT:    packusdw %xmm2, %xmm3
+; SSE41-NEXT:    blendvpd %xmm0, %xmm8, %xmm9
+; SSE41-NEXT:    packusdw %xmm4, %xmm9
+; SSE41-NEXT:    packusdw %xmm9, %xmm3
 ; SSE41-NEXT:    packuswb %xmm3, %xmm1
 ; SSE41-NEXT:    movdqa %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: trunc_packus_v16i64_v16i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm8
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm5 = [255,255]
-; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm9
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm7
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
-; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm5, %xmm6
-; AVX1-NEXT:    vblendvpd %xmm6, %xmm0, %xmm5, %xmm10
-; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm5, %xmm6
-; AVX1-NEXT:    vblendvpd %xmm6, %xmm4, %xmm5, %xmm11
-; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm5, %xmm6
-; AVX1-NEXT:    vblendvpd %xmm6, %xmm1, %xmm5, %xmm1
-; AVX1-NEXT:    vpcmpgtq %xmm7, %xmm5, %xmm6
-; AVX1-NEXT:    vblendvpd %xmm6, %xmm7, %xmm5, %xmm6
-; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm5, %xmm7
-; AVX1-NEXT:    vblendvpd %xmm7, %xmm2, %xmm5, %xmm2
-; AVX1-NEXT:    vpcmpgtq %xmm9, %xmm5, %xmm7
-; AVX1-NEXT:    vblendvpd %xmm7, %xmm9, %xmm5, %xmm7
-; AVX1-NEXT:    vpcmpgtq %xmm3, %xmm5, %xmm0
-; AVX1-NEXT:    vblendvpd %xmm0, %xmm3, %xmm5, %xmm0
-; AVX1-NEXT:    vpcmpgtq %xmm8, %xmm5, %xmm3
-; AVX1-NEXT:    vblendvpd %xmm3, %xmm8, %xmm5, %xmm3
-; AVX1-NEXT:    vpxor %xmm5, %xmm5, %xmm5
-; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm3, %xmm4
-; AVX1-NEXT:    vpand %xmm3, %xmm4, %xmm3
-; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm0, %xmm4
-; AVX1-NEXT:    vpand %xmm0, %xmm4, %xmm0
-; AVX1-NEXT:    vpackusdw %xmm3, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm7, %xmm3
-; AVX1-NEXT:    vpand %xmm7, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm2, %xmm4
-; AVX1-NEXT:    vpand %xmm2, %xmm4, %xmm2
+; AVX1-NEXT:    vmovdqa 112(%rdi), %xmm8
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [255,255]
+; AVX1-NEXT:    vmovdqa 96(%rdi), %xmm9
+; AVX1-NEXT:    vmovdqa 80(%rdi), %xmm3
+; AVX1-NEXT:    vmovdqa 64(%rdi), %xmm4
+; AVX1-NEXT:    vmovdqa (%rdi), %xmm5
+; AVX1-NEXT:    vmovdqa 16(%rdi), %xmm6
+; AVX1-NEXT:    vmovdqa 32(%rdi), %xmm7
+; AVX1-NEXT:    vmovdqa 48(%rdi), %xmm0
+; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm1, %xmm2
+; AVX1-NEXT:    vblendvpd %xmm2, %xmm5, %xmm1, %xmm10
+; AVX1-NEXT:    vpcmpgtq %xmm6, %xmm1, %xmm5
+; AVX1-NEXT:    vblendvpd %xmm5, %xmm6, %xmm1, %xmm11
+; AVX1-NEXT:    vpcmpgtq %xmm7, %xmm1, %xmm6
+; AVX1-NEXT:    vblendvpd %xmm6, %xmm7, %xmm1, %xmm6
+; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm7
+; AVX1-NEXT:    vblendvpd %xmm7, %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm1, %xmm7
+; AVX1-NEXT:    vblendvpd %xmm7, %xmm4, %xmm1, %xmm4
+; AVX1-NEXT:    vpcmpgtq %xmm3, %xmm1, %xmm7
+; AVX1-NEXT:    vblendvpd %xmm7, %xmm3, %xmm1, %xmm3
+; AVX1-NEXT:    vpcmpgtq %xmm9, %xmm1, %xmm7
+; AVX1-NEXT:    vblendvpd %xmm7, %xmm9, %xmm1, %xmm7
+; AVX1-NEXT:    vpcmpgtq %xmm8, %xmm1, %xmm2
+; AVX1-NEXT:    vblendvpd %xmm2, %xmm8, %xmm1, %xmm1
+; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm1, %xmm5
+; AVX1-NEXT:    vpand %xmm1, %xmm5, %xmm1
+; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm7, %xmm5
+; AVX1-NEXT:    vpand %xmm7, %xmm5, %xmm5
+; AVX1-NEXT:    vpackusdw %xmm1, %xmm5, %xmm1
+; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm3, %xmm5
+; AVX1-NEXT:    vpand %xmm3, %xmm5, %xmm3
+; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm4, %xmm5
+; AVX1-NEXT:    vpand %xmm4, %xmm5, %xmm4
+; AVX1-NEXT:    vpackusdw %xmm3, %xmm4, %xmm3
+; AVX1-NEXT:    vpackusdw %xmm1, %xmm3, %xmm1
+; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm0, %xmm3
+; AVX1-NEXT:    vpand %xmm0, %xmm3, %xmm0
+; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm6, %xmm3
+; AVX1-NEXT:    vpand %xmm6, %xmm3, %xmm3
+; AVX1-NEXT:    vpackusdw %xmm0, %xmm3, %xmm0
+; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm11, %xmm3
+; AVX1-NEXT:    vpand %xmm3, %xmm11, %xmm3
+; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm10, %xmm2
+; AVX1-NEXT:    vpand %xmm2, %xmm10, %xmm2
 ; AVX1-NEXT:    vpackusdw %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vpackusdw %xmm0, %xmm2, %xmm0
-; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm6, %xmm2
-; AVX1-NEXT:    vpand %xmm6, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm1, %xmm3
-; AVX1-NEXT:    vpand %xmm1, %xmm3, %xmm1
-; AVX1-NEXT:    vpackusdw %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm11, %xmm2
-; AVX1-NEXT:    vpand %xmm11, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm10, %xmm3
-; AVX1-NEXT:    vpand %xmm10, %xmm3, %xmm3
-; AVX1-NEXT:    vpackusdw %xmm2, %xmm3, %xmm2
-; AVX1-NEXT:    vpackusdw %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vpackuswb %xmm0, %xmm1, %xmm0
-; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: trunc_packus_v16i64_v16i8:
 ; AVX2:       # %bb.0:
+; AVX2-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX2-NEXT:    vmovdqa 32(%rdi), %ymm1
+; AVX2-NEXT:    vmovdqa 64(%rdi), %ymm2
+; AVX2-NEXT:    vmovdqa 96(%rdi), %ymm3
 ; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [255,255,255,255]
 ; AVX2-NEXT:    vpcmpgtq %ymm2, %ymm4, %ymm5
 ; AVX2-NEXT:    vblendvpd %ymm5, %ymm2, %ymm4, %ymm2
@@ -2754,20 +5099,75 @@ define <16 x i8> @trunc_packus_v16i64_v16i8(<16 x i64> %a0) {
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
-; AVX512-LABEL: trunc_packus_v16i64_v16i8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpbroadcastq {{.*#+}} zmm2 = [255,255,255,255,255,255,255,255]
-; AVX512-NEXT:    vpminsq %zmm2, %zmm0, %zmm0
-; AVX512-NEXT:    vpminsq %zmm2, %zmm1, %zmm1
-; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpmaxsq %zmm2, %zmm1, %zmm1
-; AVX512-NEXT:    vpmaxsq %zmm2, %zmm0, %zmm0
-; AVX512-NEXT:    vpmovqd %zmm0, %ymm0
-; AVX512-NEXT:    vpmovqd %zmm1, %ymm1
-; AVX512-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512-NEXT:    vpmovdb %zmm0, %xmm0
-; AVX512-NEXT:    vzeroupper
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: trunc_packus_v16i64_v16i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpbroadcastq {{.*#+}} zmm0 = [255,255,255,255,255,255,255,255]
+; AVX512F-NEXT:    vpminsq (%rdi), %zmm0, %zmm1
+; AVX512F-NEXT:    vpminsq 64(%rdi), %zmm0, %zmm0
+; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512F-NEXT:    vpmaxsq %zmm2, %zmm0, %zmm0
+; AVX512F-NEXT:    vpmaxsq %zmm2, %zmm1, %zmm1
+; AVX512F-NEXT:    vpmovqd %zmm1, %ymm1
+; AVX512F-NEXT:    vpmovqd %zmm0, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: trunc_packus_v16i64_v16i8:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpmaxsq 64(%rdi), %zmm0, %zmm1
+; AVX512VL-NEXT:    vpmovusqb %zmm1, %xmm1
+; AVX512VL-NEXT:    vpmaxsq (%rdi), %zmm0, %zmm0
+; AVX512VL-NEXT:    vpmovusqb %zmm0, %xmm0
+; AVX512VL-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX512VL-NEXT:    vzeroupper
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: trunc_packus_v16i64_v16i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpbroadcastq {{.*#+}} zmm0 = [255,255,255,255,255,255,255,255]
+; AVX512BW-NEXT:    vpminsq (%rdi), %zmm0, %zmm1
+; AVX512BW-NEXT:    vpminsq 64(%rdi), %zmm0, %zmm0
+; AVX512BW-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512BW-NEXT:    vpmaxsq %zmm2, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmaxsq %zmm2, %zmm1, %zmm1
+; AVX512BW-NEXT:    vpmovqd %zmm1, %ymm1
+; AVX512BW-NEXT:    vpmovqd %zmm0, %ymm0
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512BW-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: trunc_packus_v16i64_v16i8:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; AVX512BWVL-NEXT:    vpmaxsq 64(%rdi), %zmm0, %zmm1
+; AVX512BWVL-NEXT:    vpmovusqb %zmm1, %xmm1
+; AVX512BWVL-NEXT:    vpmaxsq (%rdi), %zmm0, %zmm0
+; AVX512BWVL-NEXT:    vpmovusqb %zmm0, %xmm0
+; AVX512BWVL-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
+;
+; SKX-LABEL: trunc_packus_v16i64_v16i8:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; SKX-NEXT:    vpmaxsq 96(%rdi), %ymm0, %ymm1
+; SKX-NEXT:    vpmovusqb %ymm1, %xmm1
+; SKX-NEXT:    vpmaxsq 64(%rdi), %ymm0, %ymm2
+; SKX-NEXT:    vpmovusqb %ymm2, %xmm2
+; SKX-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SKX-NEXT:    vpmaxsq 32(%rdi), %ymm0, %ymm2
+; SKX-NEXT:    vpmovusqb %ymm2, %xmm2
+; SKX-NEXT:    vpmaxsq (%rdi), %ymm0, %ymm0
+; SKX-NEXT:    vpmovusqb %ymm0, %xmm0
+; SKX-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; SKX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SKX-NEXT:    vzeroupper
+; SKX-NEXT:    retq
+  %a0 = load <16 x i64>, <16 x i64>* %p0
   %1 = icmp slt <16 x i64> %a0, <i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255>
   %2 = select <16 x i1> %1, <16 x i64> %a0, <16 x i64> <i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255>
   %3 = icmp sgt <16 x i64> %2, zeroinitializer
@@ -2776,127 +5176,280 @@ define <16 x i8> @trunc_packus_v16i64_v16i8(<16 x i64> %a0) {
   ret <16 x i8> %5
 }
 
-define <8 x i8> @trunc_packus_v8i32_v8i8(<8 x i32> %a0) {
-; SSE2-LABEL: trunc_packus_v8i32_v8i8:
+define <4 x i8> @trunc_packus_v4i32_v4i8(<4 x i32> %a0) "min-legal-vector-width"="256" {
+; SSE2-LABEL: trunc_packus_v4i32_v4i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255]
-; SSE2-NEXT:    movdqa %xmm2, %xmm3
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSE2-NEXT:    pand %xmm3, %xmm0
-; SSE2-NEXT:    pandn %xmm2, %xmm3
-; SSE2-NEXT:    por %xmm0, %xmm3
-; SSE2-NEXT:    movdqa %xmm2, %xmm0
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pand %xmm0, %xmm1
-; SSE2-NEXT:    pandn %xmm2, %xmm0
-; SSE2-NEXT:    por %xmm1, %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,255,255]
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    pandn %xmm1, %xmm2
+; SSE2-NEXT:    por %xmm0, %xmm2
 ; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
-; SSE2-NEXT:    pand %xmm0, %xmm2
-; SSE2-NEXT:    movdqa %xmm3, %xmm0
+; SSE2-NEXT:    movdqa %xmm2, %xmm0
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pand %xmm3, %xmm0
-; SSE2-NEXT:    packuswb %xmm2, %xmm0
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    packuswb %xmm0, %xmm0
+; SSE2-NEXT:    packuswb %xmm0, %xmm0
 ; SSE2-NEXT:    retq
 ;
-; SSSE3-LABEL: trunc_packus_v8i32_v8i8:
+; SSSE3-LABEL: trunc_packus_v4i32_v4i8:
 ; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255]
-; SSSE3-NEXT:    movdqa %xmm2, %xmm3
-; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSSE3-NEXT:    pand %xmm3, %xmm0
-; SSSE3-NEXT:    pandn %xmm2, %xmm3
-; SSSE3-NEXT:    por %xmm0, %xmm3
-; SSSE3-NEXT:    movdqa %xmm2, %xmm0
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSSE3-NEXT:    pand %xmm0, %xmm1
-; SSSE3-NEXT:    pandn %xmm2, %xmm0
-; SSSE3-NEXT:    por %xmm1, %xmm0
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,255,255]
+; SSSE3-NEXT:    movdqa %xmm1, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSSE3-NEXT:    pand %xmm2, %xmm0
+; SSSE3-NEXT:    pandn %xmm1, %xmm2
+; SSSE3-NEXT:    por %xmm0, %xmm2
 ; SSSE3-NEXT:    pxor %xmm1, %xmm1
-; SSSE3-NEXT:    movdqa %xmm0, %xmm2
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
-; SSSE3-NEXT:    pand %xmm0, %xmm2
-; SSSE3-NEXT:    movdqa %xmm3, %xmm0
+; SSSE3-NEXT:    movdqa %xmm2, %xmm0
 ; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSSE3-NEXT:    pand %xmm3, %xmm0
-; SSSE3-NEXT:    packuswb %xmm2, %xmm0
+; SSSE3-NEXT:    pand %xmm2, %xmm0
+; SSSE3-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
 ; SSSE3-NEXT:    retq
 ;
-; SSE41-LABEL: trunc_packus_v8i32_v8i8:
+; SSE41-LABEL: trunc_packus_v4i32_v4i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255]
-; SSE41-NEXT:    pminsd %xmm2, %xmm0
-; SSE41-NEXT:    pminsd %xmm2, %xmm1
-; SSE41-NEXT:    pxor %xmm2, %xmm2
-; SSE41-NEXT:    pmaxsd %xmm2, %xmm1
-; SSE41-NEXT:    pmaxsd %xmm2, %xmm0
-; SSE41-NEXT:    packusdw %xmm1, %xmm0
+; SSE41-NEXT:    pminsd {{.*}}(%rip), %xmm0
+; SSE41-NEXT:    pxor %xmm1, %xmm1
+; SSE41-NEXT:    pmaxsd %xmm1, %xmm0
+; SSE41-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: trunc_packus_v8i32_v8i8:
+; AVX1-LABEL: trunc_packus_v4i32_v4i8:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpminsd {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: trunc_packus_v4i32_v4i8:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [255,255,255,255]
+; AVX2-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX2-NEXT:    retq
+;
+; AVX512F-LABEL: trunc_packus_v4i32_v4i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    vpmovusdb %zmm0, %xmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: trunc_packus_v4i32_v4i8:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512VL-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpmovusdb %xmm0, %xmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: trunc_packus_v4i32_v4i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512BW-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpmovusdb %zmm0, %xmm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: trunc_packus_v4i32_v4i8:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512BWVL-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
+; AVX512BWVL-NEXT:    vpmovusdb %xmm0, %xmm0
+; AVX512BWVL-NEXT:    retq
+;
+; SKX-LABEL: trunc_packus_v4i32_v4i8:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; SKX-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
+; SKX-NEXT:    vpmovusdb %xmm0, %xmm0
+; SKX-NEXT:    retq
+  %1 = icmp slt <4 x i32> %a0, <i32 255, i32 255, i32 255, i32 255>
+  %2 = select <4 x i1> %1, <4 x i32> %a0, <4 x i32> <i32 255, i32 255, i32 255, i32 255>
+  %3 = icmp sgt <4 x i32> %2, zeroinitializer
+  %4 = select <4 x i1> %3, <4 x i32> %2, <4 x i32> zeroinitializer
+  %5 = trunc <4 x i32> %4 to <4 x i8>
+  ret <4 x i8> %5
+}
+
+define void @trunc_packus_v4i32_v4i8_store(<4 x i32> %a0, <4 x i8> *%p1) {
+; SSE2-LABEL: trunc_packus_v4i32_v4i8_store:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,255,255]
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    pandn %xmm1, %xmm2
+; SSE2-NEXT:    por %xmm0, %xmm2
+; SSE2-NEXT:    pxor %xmm0, %xmm0
+; SSE2-NEXT:    movdqa %xmm2, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
+; SSE2-NEXT:    pand %xmm2, %xmm1
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE2-NEXT:    packuswb %xmm0, %xmm1
+; SSE2-NEXT:    packuswb %xmm0, %xmm1
+; SSE2-NEXT:    movd %xmm1, (%rdi)
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: trunc_packus_v4i32_v4i8_store:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,255,255]
+; SSSE3-NEXT:    movdqa %xmm1, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSSE3-NEXT:    pand %xmm2, %xmm0
+; SSSE3-NEXT:    pandn %xmm1, %xmm2
+; SSSE3-NEXT:    por %xmm0, %xmm2
+; SSSE3-NEXT:    pxor %xmm0, %xmm0
+; SSSE3-NEXT:    movdqa %xmm2, %xmm1
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm1
+; SSSE3-NEXT:    pand %xmm2, %xmm1
+; SSSE3-NEXT:    pshufb {{.*#+}} xmm1 = xmm1[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; SSSE3-NEXT:    movd %xmm1, (%rdi)
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: trunc_packus_v4i32_v4i8_store:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pminsd {{.*}}(%rip), %xmm0
+; SSE41-NEXT:    pxor %xmm1, %xmm1
+; SSE41-NEXT:    pmaxsd %xmm0, %xmm1
+; SSE41-NEXT:    pshufb {{.*#+}} xmm1 = xmm1[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; SSE41-NEXT:    movd %xmm1, (%rdi)
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: trunc_packus_v4i32_v4i8_store:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [255,255,255,255]
-; AVX1-NEXT:    vpminsd %xmm1, %xmm0, %xmm2
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpminsd {{.*}}(%rip), %xmm0, %xmm0
 ; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX1-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpmaxsd %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vpackusdw %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX1-NEXT:    vmovd %xmm0, (%rdi)
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: trunc_packus_v4i32_v4i8_store:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [255,255,255,255]
+; AVX2-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX2-NEXT:    vmovd %xmm0, (%rdi)
+; AVX2-NEXT:    retq
+;
+; AVX512F-LABEL: trunc_packus_v4i32_v4i8_store:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    vpmovusdb %zmm0, %xmm0
+; AVX512F-NEXT:    vmovd %xmm0, (%rdi)
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: trunc_packus_v4i32_v4i8_store:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512VL-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpmovusdb %xmm0, (%rdi)
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: trunc_packus_v4i32_v4i8_store:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512BW-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpmovusdb %zmm0, %xmm0
+; AVX512BW-NEXT:    vmovd %xmm0, (%rdi)
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: trunc_packus_v4i32_v4i8_store:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512BWVL-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
+; AVX512BWVL-NEXT:    vpmovusdb %xmm0, (%rdi)
+; AVX512BWVL-NEXT:    retq
+;
+; SKX-LABEL: trunc_packus_v4i32_v4i8_store:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; SKX-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
+; SKX-NEXT:    vpmovusdb %xmm0, (%rdi)
+; SKX-NEXT:    retq
+  %1 = icmp slt <4 x i32> %a0, <i32 255, i32 255, i32 255, i32 255>
+  %2 = select <4 x i1> %1, <4 x i32> %a0, <4 x i32> <i32 255, i32 255, i32 255, i32 255>
+  %3 = icmp sgt <4 x i32> %2, zeroinitializer
+  %4 = select <4 x i1> %3, <4 x i32> %2, <4 x i32> zeroinitializer
+  %5 = trunc <4 x i32> %4 to <4 x i8>
+  store <4 x i8> %5, <4 x i8> *%p1
+  ret void
+}
+
+define <8 x i8> @trunc_packus_v8i32_v8i8(<8 x i32> %a0) {
+; SSE-LABEL: trunc_packus_v8i32_v8i8:
+; SSE:       # %bb.0:
+; SSE-NEXT:    packssdw %xmm1, %xmm0
+; SSE-NEXT:    packuswb %xmm0, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: trunc_packus_v8i32_v8i8:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: trunc_packus_v8i32_v8i8:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [255,255,255,255,255,255,255,255]
-; AVX2-NEXT:    vpminsd %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpmaxsd %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: trunc_packus_v8i32_v8i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [255,255,255,255,255,255,255,255]
-; AVX512F-NEXT:    vpminsd %ymm1, %ymm0, %ymm0
-; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT:    vpmaxsd %ymm1, %ymm0, %ymm0
-; AVX512F-NEXT:    vpmovdw %zmm0, %ymm0
-; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: trunc_packus_v8i32_v8i8:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpminsd {{.*}}(%rip){1to8}, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX512VL-NEXT:    vpmaxsd %ymm1, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpmovdw %ymm0, %xmm0
+; AVX512VL-NEXT:    vpmovusdb %ymm0, %xmm0
 ; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: trunc_packus_v8i32_v8i8:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [255,255,255,255,255,255,255,255]
-; AVX512BW-NEXT:    vpminsd %ymm1, %ymm0, %ymm0
-; AVX512BW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpmaxsd %ymm1, %ymm0, %ymm0
-; AVX512BW-NEXT:    vpmovdw %zmm0, %ymm0
-; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512BW-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512BWVL-LABEL: trunc_packus_v8i32_v8i8:
 ; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vpminsd {{.*}}(%rip){1to8}, %ymm0, %ymm0
 ; AVX512BWVL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX512BWVL-NEXT:    vpmaxsd %ymm1, %ymm0, %ymm0
-; AVX512BWVL-NEXT:    vpmovdw %ymm0, %xmm0
+; AVX512BWVL-NEXT:    vpmovusdb %ymm0, %xmm0
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
+;
+; SKX-LABEL: trunc_packus_v8i32_v8i8:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; SKX-NEXT:    vpmaxsd %ymm1, %ymm0, %ymm0
+; SKX-NEXT:    vpmovusdb %ymm0, %xmm0
+; SKX-NEXT:    vzeroupper
+; SKX-NEXT:    retq
   %1 = icmp slt <8 x i32> %a0, <i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255>
   %2 = select <8 x i1> %1, <8 x i32> %a0, <8 x i32> <i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255>
   %3 = icmp sgt <8 x i32> %2, zeroinitializer
@@ -2933,11 +5486,8 @@ define void @trunc_packus_v8i32_v8i8_store(<8 x i32> %a0, <8 x i8> *%p1) {
 ;
 ; AVX512F-LABEL: trunc_packus_v8i32_v8i8_store:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [255,255,255,255,255,255,255,255]
-; AVX512F-NEXT:    vpminsd %ymm1, %ymm0, %ymm0
-; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT:    vpmaxsd %ymm1, %ymm0, %ymm0
-; AVX512F-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
 ; AVX512F-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
 ; AVX512F-NEXT:    vmovq %xmm0, (%rdi)
 ; AVX512F-NEXT:    vzeroupper
@@ -2953,11 +5503,8 @@ define void @trunc_packus_v8i32_v8i8_store(<8 x i32> %a0, <8 x i8> *%p1) {
 ;
 ; AVX512BW-LABEL: trunc_packus_v8i32_v8i8_store:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [255,255,255,255,255,255,255,255]
-; AVX512BW-NEXT:    vpminsd %ymm1, %ymm0, %ymm0
-; AVX512BW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpmaxsd %ymm1, %ymm0, %ymm0
-; AVX512BW-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512BW-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vmovq %xmm0, (%rdi)
 ; AVX512BW-NEXT:    vzeroupper
@@ -2970,6 +5517,14 @@ define void @trunc_packus_v8i32_v8i8_store(<8 x i32> %a0, <8 x i8> *%p1) {
 ; AVX512BWVL-NEXT:    vpmovusdb %ymm0, (%rdi)
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
+;
+; SKX-LABEL: trunc_packus_v8i32_v8i8_store:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; SKX-NEXT:    vpmaxsd %ymm1, %ymm0, %ymm0
+; SKX-NEXT:    vpmovusdb %ymm0, (%rdi)
+; SKX-NEXT:    vzeroupper
+; SKX-NEXT:    retq
   %1 = icmp slt <8 x i32> %a0, <i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255>
   %2 = select <8 x i1> %1, <8 x i32> %a0, <8 x i32> <i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255>
   %3 = icmp sgt <8 x i32> %2, zeroinitializer
@@ -2979,27 +5534,29 @@ define void @trunc_packus_v8i32_v8i8_store(<8 x i32> %a0, <8 x i8> *%p1) {
   ret void
 }
 
-define <16 x i8> @trunc_packus_v16i32_v16i8(<16 x i32> %a0) {
+define <16 x i8> @trunc_packus_v16i32_v16i8(<16 x i32>* %p0) "min-legal-vector-width"="256" {
 ; SSE-LABEL: trunc_packus_v16i32_v16i8:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    packssdw %xmm3, %xmm2
-; SSE-NEXT:    packssdw %xmm1, %xmm0
-; SSE-NEXT:    packuswb %xmm2, %xmm0
+; SSE-NEXT:    movdqa (%rdi), %xmm0
+; SSE-NEXT:    movdqa 32(%rdi), %xmm1
+; SSE-NEXT:    packssdw 48(%rdi), %xmm1
+; SSE-NEXT:    packssdw 16(%rdi), %xmm0
+; SSE-NEXT:    packuswb %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: trunc_packus_v16i32_v16i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT:    vpackssdw %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX1-NEXT:    vpackssdw %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX1-NEXT:    vmovdqa 32(%rdi), %xmm1
+; AVX1-NEXT:    vpackssdw 48(%rdi), %xmm1, %xmm1
+; AVX1-NEXT:    vpackssdw 16(%rdi), %xmm0, %xmm0
 ; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: trunc_packus_v16i32_v16i8:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpackssdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX2-NEXT:    vpackssdw 32(%rdi), %ymm0, %ymm0
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
@@ -3008,11 +5565,21 @@ define <16 x i8> @trunc_packus_v16i32_v16i8(<16 x i32> %a0) {
 ;
 ; AVX512-LABEL: trunc_packus_v16i32_v16i8:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512-NEXT:    vpmaxsd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; AVX512-NEXT:    vpmaxsd (%rdi), %zmm0, %zmm0
 ; AVX512-NEXT:    vpmovusdb %zmm0, %xmm0
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
+;
+; SKX-LABEL: trunc_packus_v16i32_v16i8:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vmovdqa (%rdi), %ymm0
+; SKX-NEXT:    vpackusdw 32(%rdi), %ymm0, %ymm0
+; SKX-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; SKX-NEXT:    vpmovuswb %ymm0, %xmm0
+; SKX-NEXT:    vzeroupper
+; SKX-NEXT:    retq
+  %a0 = load <16 x i32>, <16 x i32>* %p0
   %1 = icmp slt <16 x i32> %a0, <i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255>
   %2 = select <16 x i1> %1, <16 x i32> %a0, <16 x i32> <i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255>
   %3 = icmp sgt <16 x i32> %2, zeroinitializer
@@ -3021,6 +5588,145 @@ define <16 x i8> @trunc_packus_v16i32_v16i8(<16 x i32> %a0) {
   ret <16 x i8> %5
 }
 
+define void @trunc_packus_v16i32_v16i8_store(<16 x i32>* %p0, <16 x i8>* %p1) "min-legal-vector-width"="256" {
+; SSE-LABEL: trunc_packus_v16i32_v16i8_store:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqa (%rdi), %xmm0
+; SSE-NEXT:    movdqa 32(%rdi), %xmm1
+; SSE-NEXT:    packssdw 48(%rdi), %xmm1
+; SSE-NEXT:    packssdw 16(%rdi), %xmm0
+; SSE-NEXT:    packuswb %xmm1, %xmm0
+; SSE-NEXT:    movdqa %xmm0, (%rsi)
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: trunc_packus_v16i32_v16i8_store:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX1-NEXT:    vmovdqa 32(%rdi), %xmm1
+; AVX1-NEXT:    vpackssdw 48(%rdi), %xmm1, %xmm1
+; AVX1-NEXT:    vpackssdw 16(%rdi), %xmm0, %xmm0
+; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vmovdqa %xmm0, (%rsi)
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: trunc_packus_v16i32_v16i8_store:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX2-NEXT:    vpackssdw 32(%rdi), %ymm0, %ymm0
+; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vmovdqa %xmm0, (%rsi)
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: trunc_packus_v16i32_v16i8_store:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; AVX512-NEXT:    vpmaxsd (%rdi), %zmm0, %zmm0
+; AVX512-NEXT:    vpmovusdb %zmm0, (%rsi)
+; AVX512-NEXT:    vzeroupper
+; AVX512-NEXT:    retq
+;
+; SKX-LABEL: trunc_packus_v16i32_v16i8_store:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vmovdqa (%rdi), %ymm0
+; SKX-NEXT:    vpackusdw 32(%rdi), %ymm0, %ymm0
+; SKX-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; SKX-NEXT:    vpmovuswb %ymm0, (%rsi)
+; SKX-NEXT:    vzeroupper
+; SKX-NEXT:    retq
+  %a = load <16 x i32>, <16 x i32>* %p0
+  %b = icmp slt <16 x i32> %a, <i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255>
+  %c = select <16 x i1> %b, <16 x i32> %a, <16 x i32> <i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255>
+  %d = icmp sgt <16 x i32> %c, zeroinitializer
+  %e = select <16 x i1> %d, <16 x i32> %c, <16 x i32> zeroinitializer
+  %f = trunc <16 x i32> %e to <16 x i8>
+  store <16 x i8> %f, <16 x i8>* %p1
+  ret void
+}
+
+define <8 x i8> @trunc_packus_v8i16_v8i8(<8 x i16> %a0) {
+; SSE-LABEL: trunc_packus_v8i16_v8i8:
+; SSE:       # %bb.0:
+; SSE-NEXT:    packuswb %xmm0, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: trunc_packus_v8i16_v8i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: trunc_packus_v8i16_v8i8:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
+; AVX512-NEXT:    retq
+;
+; SKX-LABEL: trunc_packus_v8i16_v8i8:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
+; SKX-NEXT:    retq
+  %1 = icmp slt <8 x i16> %a0, <i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255>
+  %2 = select <8 x i1> %1, <8 x i16> %a0, <8 x i16> <i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255>
+  %3 = icmp sgt <8 x i16> %2, zeroinitializer
+  %4 = select <8 x i1> %3, <8 x i16> %2, <8 x i16> zeroinitializer
+  %5 = trunc <8 x i16> %4 to <8 x i8>
+  ret <8 x i8> %5
+}
+
+define void @trunc_packus_v8i16_v8i8_store(<8 x i16> %a0, <8 x i8> *%p1) {
+; SSE-LABEL: trunc_packus_v8i16_v8i8_store:
+; SSE:       # %bb.0:
+; SSE-NEXT:    packuswb %xmm0, %xmm0
+; SSE-NEXT:    movq %xmm0, (%rdi)
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: trunc_packus_v8i16_v8i8_store:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vmovq %xmm0, (%rdi)
+; AVX-NEXT:    retq
+;
+; AVX512F-LABEL: trunc_packus_v8i16_v8i8_store:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
+; AVX512F-NEXT:    vmovq %xmm0, (%rdi)
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: trunc_packus_v8i16_v8i8_store:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
+; AVX512VL-NEXT:    vmovq %xmm0, (%rdi)
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: trunc_packus_v8i16_v8i8_store:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
+; AVX512BW-NEXT:    vmovq %xmm0, (%rdi)
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: trunc_packus_v8i16_v8i8_store:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512BWVL-NEXT:    vpmaxsw %xmm1, %xmm0, %xmm0
+; AVX512BWVL-NEXT:    vpmovuswb %xmm0, (%rdi)
+; AVX512BWVL-NEXT:    retq
+;
+; SKX-LABEL: trunc_packus_v8i16_v8i8_store:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; SKX-NEXT:    vpmaxsw %xmm1, %xmm0, %xmm0
+; SKX-NEXT:    vpmovuswb %xmm0, (%rdi)
+; SKX-NEXT:    retq
+  %1 = icmp slt <8 x i16> %a0, <i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255>
+  %2 = select <8 x i1> %1, <8 x i16> %a0, <8 x i16> <i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255>
+  %3 = icmp sgt <8 x i16> %2, zeroinitializer
+  %4 = select <8 x i1> %3, <8 x i16> %2, <8 x i16> zeroinitializer
+  %5 = trunc <8 x i16> %4 to <8 x i8>
+  store <8 x i8> %5, <8 x i8> *%p1
+  ret void
+}
+
 define <16 x i8> @trunc_packus_v16i16_v16i8(<16 x i16> %a0) {
 ; SSE-LABEL: trunc_packus_v16i16_v16i8:
 ; SSE:       # %bb.0:
@@ -3043,31 +5749,22 @@ define <16 x i8> @trunc_packus_v16i16_v16i8(<16 x i16> %a0) {
 ;
 ; AVX512F-LABEL: trunc_packus_v16i16_v16i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpminsw {{.*}}(%rip), %ymm0, %ymm0
-; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT:    vpmaxsw %ymm1, %ymm0, %ymm0
-; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
-; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: trunc_packus_v16i16_v16i8:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpminsw {{.*}}(%rip), %ymm0, %ymm0
-; AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpmaxsw %ymm1, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
-; AVX512VL-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512VL-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512VL-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: trunc_packus_v16i16_v16i8:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpminsw {{.*}}(%rip), %ymm0, %ymm0
-; AVX512BW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpmaxsw %ymm1, %ymm0, %ymm0
-; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
-; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512BW-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
@@ -3078,6 +5775,14 @@ define <16 x i8> @trunc_packus_v16i16_v16i8(<16 x i16> %a0) {
 ; AVX512BWVL-NEXT:    vpmovuswb %ymm0, %xmm0
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
+;
+; SKX-LABEL: trunc_packus_v16i16_v16i8:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; SKX-NEXT:    vpmaxsw %ymm1, %ymm0, %ymm0
+; SKX-NEXT:    vpmovuswb %ymm0, %xmm0
+; SKX-NEXT:    vzeroupper
+; SKX-NEXT:    retq
   %1 = icmp slt <16 x i16> %a0, <i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255>
   %2 = select <16 x i1> %1, <16 x i16> %a0, <16 x i16> <i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255>
   %3 = icmp sgt <16 x i16> %2, zeroinitializer
@@ -3086,72 +5791,66 @@ define <16 x i8> @trunc_packus_v16i16_v16i8(<16 x i16> %a0) {
   ret <16 x i8> %5
 }
 
-define <32 x i8> @trunc_packus_v32i16_v32i8(<32 x i16> %a0) {
+define <32 x i8> @trunc_packus_v32i16_v32i8(<32 x i16>* %p0) "min-legal-vector-width"="256" {
 ; SSE-LABEL: trunc_packus_v32i16_v32i8:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    packuswb %xmm1, %xmm0
-; SSE-NEXT:    packuswb %xmm3, %xmm2
-; SSE-NEXT:    movdqa %xmm2, %xmm1
+; SSE-NEXT:    movdqa (%rdi), %xmm0
+; SSE-NEXT:    movdqa 32(%rdi), %xmm1
+; SSE-NEXT:    packuswb 16(%rdi), %xmm0
+; SSE-NEXT:    packuswb 48(%rdi), %xmm1
 ; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: trunc_packus_v32i16_v32i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX1-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX1-NEXT:    vmovdqa 32(%rdi), %xmm1
+; AVX1-NEXT:    vpackuswb 48(%rdi), %xmm1, %xmm1
+; AVX1-NEXT:    vpackuswb 16(%rdi), %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: trunc_packus_v32i16_v32i8:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX2-NEXT:    vpackuswb 32(%rdi), %ymm0, %ymm0
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: trunc_packus_v32i16_v32i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512F-NEXT:    vpminsw %ymm2, %ymm0, %ymm0
-; AVX512F-NEXT:    vpminsw %ymm2, %ymm1, %ymm1
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpmaxsw %ymm2, %ymm1, %ymm1
-; AVX512F-NEXT:    vpmaxsw %ymm2, %ymm0, %ymm0
-; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
-; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
-; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
-; AVX512F-NEXT:    vpmovdb %zmm1, %xmm1
-; AVX512F-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX512F-NEXT:    vpackuswb 32(%rdi), %ymm0, %ymm0
+; AVX512F-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: trunc_packus_v32i16_v32i8:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512VL-NEXT:    vpminsw %ymm2, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpminsw %ymm2, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VL-NEXT:    vpmaxsw %ymm2, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpmaxsw %ymm2, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
-; AVX512VL-NEXT:    vpmovdb %zmm0, %xmm0
-; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
-; AVX512VL-NEXT:    vpmovdb %zmm1, %xmm1
-; AVX512VL-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX512VL-NEXT:    vpackuswb 32(%rdi), %ymm0, %ymm0
+; AVX512VL-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: trunc_packus_v32i16_v32i8:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpmaxsw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpmaxsw (%rdi), %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpmovuswb %zmm0, %ymm0
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512BWVL-LABEL: trunc_packus_v32i16_v32i8:
 ; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512BWVL-NEXT:    vpmaxsw %zmm1, %zmm0, %zmm0
+; AVX512BWVL-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; AVX512BWVL-NEXT:    vpmaxsw (%rdi), %zmm0, %zmm0
 ; AVX512BWVL-NEXT:    vpmovuswb %zmm0, %ymm0
 ; AVX512BWVL-NEXT:    retq
+;
+; SKX-LABEL: trunc_packus_v32i16_v32i8:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vmovdqa (%rdi), %ymm0
+; SKX-NEXT:    vpackuswb 32(%rdi), %ymm0, %ymm0
+; SKX-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; SKX-NEXT:    retq
+  %a0 = load <32 x i16>, <32 x i16>* %p0
   %1 = icmp slt <32 x i16> %a0, <i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255>
   %2 = select <32 x i1> %1, <32 x i16> %a0, <32 x i16> <i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255>
   %3 = icmp sgt <32 x i16> %2, zeroinitializer
@@ -3159,3 +5858,75 @@ define <32 x i8> @trunc_packus_v32i16_v32i8(<32 x i16> %a0) {
   %5 = trunc <32 x i16> %4 to <32 x i8>
   ret <32 x i8> %5
 }
+
+define <32 x i8> @trunc_packus_v32i32_v32i8(<32 x i32>* %p0) "min-legal-vector-width"="256" {
+; SSE-LABEL: trunc_packus_v32i32_v32i8:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqa (%rdi), %xmm0
+; SSE-NEXT:    movdqa 32(%rdi), %xmm2
+; SSE-NEXT:    movdqa 64(%rdi), %xmm1
+; SSE-NEXT:    movdqa 96(%rdi), %xmm3
+; SSE-NEXT:    packssdw 48(%rdi), %xmm2
+; SSE-NEXT:    packssdw 16(%rdi), %xmm0
+; SSE-NEXT:    packuswb %xmm2, %xmm0
+; SSE-NEXT:    packssdw 112(%rdi), %xmm3
+; SSE-NEXT:    packssdw 80(%rdi), %xmm1
+; SSE-NEXT:    packuswb %xmm3, %xmm1
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: trunc_packus_v32i32_v32i8:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX1-NEXT:    vmovdqa 32(%rdi), %xmm1
+; AVX1-NEXT:    vmovdqa 64(%rdi), %xmm2
+; AVX1-NEXT:    vmovdqa 96(%rdi), %xmm3
+; AVX1-NEXT:    vpackssdw 112(%rdi), %xmm3, %xmm3
+; AVX1-NEXT:    vpackssdw 80(%rdi), %xmm2, %xmm2
+; AVX1-NEXT:    vpackuswb %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpackssdw 48(%rdi), %xmm1, %xmm1
+; AVX1-NEXT:    vpackssdw 16(%rdi), %xmm0, %xmm0
+; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: trunc_packus_v32i32_v32i8:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX2-NEXT:    vmovdqa 64(%rdi), %ymm1
+; AVX2-NEXT:    vpackssdw 96(%rdi), %ymm1, %ymm1
+; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
+; AVX2-NEXT:    vpackssdw 32(%rdi), %ymm0, %ymm0
+; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: trunc_packus_v32i32_v32i8:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; AVX512-NEXT:    vpmaxsd (%rdi), %zmm0, %zmm1
+; AVX512-NEXT:    vpmovusdb %zmm1, %xmm1
+; AVX512-NEXT:    vpmaxsd 64(%rdi), %zmm0, %zmm0
+; AVX512-NEXT:    vpmovusdb %zmm0, %xmm0
+; AVX512-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX512-NEXT:    retq
+;
+; SKX-LABEL: trunc_packus_v32i32_v32i8:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vmovdqa (%rdi), %ymm0
+; SKX-NEXT:    vmovdqa 64(%rdi), %ymm1
+; SKX-NEXT:    vpackssdw 96(%rdi), %ymm1, %ymm1
+; SKX-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
+; SKX-NEXT:    vpackssdw 32(%rdi), %ymm0, %ymm0
+; SKX-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; SKX-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
+; SKX-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; SKX-NEXT:    retq
+  %a0 = load <32 x i32>, <32 x i32>* %p0
+  %1 = icmp slt <32 x i32> %a0, <i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255>
+  %2 = select <32 x i1> %1, <32 x i32> %a0, <32 x i32> <i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255>
+  %3 = icmp sgt <32 x i32> %2, zeroinitializer
+  %4 = select <32 x i1> %3, <32 x i32> %2, <32 x i32> zeroinitializer
+  %5 = trunc <32 x i32> %4 to <32 x i8>
+  ret <32 x i8> %5
+}
diff --git a/llvm/test/CodeGen/X86/vector-trunc-ssat.ll b/llvm/test/CodeGen/X86/vector-trunc-ssat.ll
index d8be79cc6..543708caa 100644
--- a/llvm/test/CodeGen/X86/vector-trunc-ssat.ll
+++ b/llvm/test/CodeGen/X86/vector-trunc-ssat.ll
@@ -9,11 +9,303 @@
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vl,+fast-variable-shuffle | FileCheck %s --check-prefixes=AVX512,AVX512VL
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+fast-variable-shuffle | FileCheck %s --check-prefixes=AVX512,AVX512BW
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl,+fast-variable-shuffle | FileCheck %s --check-prefixes=AVX512,AVX512BWVL
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=skx | FileCheck %s --check-prefixes=SKX
 
 ;
 ; Signed saturation truncation to vXi32
 ;
 
+define <2 x i32> @trunc_ssat_v2i64_v2i32(<2 x i64> %a0) {
+; SSE2-LABEL: trunc_ssat_v2i64_v2i32:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    pxor %xmm1, %xmm2
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [4294967295,4294967295]
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm3, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE2-NEXT:    pand %xmm5, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm2, %xmm3
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm3
+; SSE2-NEXT:    por %xmm0, %xmm3
+; SSE2-NEXT:    pxor %xmm3, %xmm1
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [18446744069414584320,18446744069414584320]
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm0, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pand %xmm4, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,3,3]
+; SSE2-NEXT:    por %xmm0, %xmm1
+; SSE2-NEXT:    pand %xmm1, %xmm3
+; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm1
+; SSE2-NEXT:    por %xmm3, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: trunc_ssat_v2i64_v2i32:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [2147483648,2147483648]
+; SSSE3-NEXT:    movdqa %xmm0, %xmm2
+; SSSE3-NEXT:    pxor %xmm1, %xmm2
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm3 = [4294967295,4294967295]
+; SSSE3-NEXT:    movdqa %xmm3, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm5, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm2, %xmm3
+; SSSE3-NEXT:    pand %xmm3, %xmm0
+; SSSE3-NEXT:    pandn {{.*}}(%rip), %xmm3
+; SSSE3-NEXT:    por %xmm0, %xmm3
+; SSSE3-NEXT:    pxor %xmm3, %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [18446744069414584320,18446744069414584320]
+; SSSE3-NEXT:    movdqa %xmm1, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm4, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    por %xmm0, %xmm1
+; SSSE3-NEXT:    pand %xmm1, %xmm3
+; SSSE3-NEXT:    pandn {{.*}}(%rip), %xmm1
+; SSSE3-NEXT:    por %xmm3, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: trunc_ssat_v2i64_v2i32:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    movapd {{.*#+}} xmm2 = [2147483647,2147483647]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [2147483648,2147483648]
+; SSE41-NEXT:    pxor %xmm3, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [4294967295,4294967295]
+; SSE41-NEXT:    movdqa %xmm4, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm0
+; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
+; SSE41-NEXT:    movapd {{.*#+}} xmm1 = [18446744071562067968,18446744071562067968]
+; SSE41-NEXT:    pxor %xmm2, %xmm3
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [18446744069414584320,18446744069414584320]
+; SSE41-NEXT:    movdqa %xmm3, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm3, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: trunc_ssat_v2i64_v2i32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [2147483647,2147483647]
+; AVX-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm2
+; AVX-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [18446744071562067968,18446744071562067968]
+; AVX-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm2
+; AVX-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX-NEXT:    retq
+;
+; AVX512F-LABEL: trunc_ssat_v2i64_v2i32:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512F-NEXT:    vpmovsqd %zmm0, %ymm0
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: trunc_ssat_v2i64_v2i32:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpmovsqd %xmm0, %xmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: trunc_ssat_v2i64_v2i32:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512BW-NEXT:    vpmovsqd %zmm0, %ymm0
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: trunc_ssat_v2i64_v2i32:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpmovsqd %xmm0, %xmm0
+; AVX512BWVL-NEXT:    retq
+;
+; SKX-LABEL: trunc_ssat_v2i64_v2i32:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpmovsqd %xmm0, %xmm0
+; SKX-NEXT:    retq
+  %1 = icmp slt <2 x i64> %a0, <i64 2147483647, i64 2147483647>
+  %2 = select <2 x i1> %1, <2 x i64> %a0, <2 x i64> <i64 2147483647, i64 2147483647>
+  %3 = icmp sgt <2 x i64> %2, <i64 -2147483648, i64 -2147483648>
+  %4 = select <2 x i1> %3, <2 x i64> %2, <2 x i64> <i64 -2147483648, i64 -2147483648>
+  %5 = trunc <2 x i64> %4 to <2 x i32>
+  ret <2 x i32> %5
+}
+
+define void @trunc_ssat_v2i64_v2i32_store(<2 x i64> %a0, <2 x i32>* %p1) {
+; SSE2-LABEL: trunc_ssat_v2i64_v2i32_store:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    pxor %xmm1, %xmm2
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [4294967295,4294967295]
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm3, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE2-NEXT:    pand %xmm5, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm2, %xmm3
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm3
+; SSE2-NEXT:    por %xmm0, %xmm3
+; SSE2-NEXT:    pxor %xmm3, %xmm1
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [18446744069414584320,18446744069414584320]
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm0, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pand %xmm4, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,3,3]
+; SSE2-NEXT:    por %xmm0, %xmm1
+; SSE2-NEXT:    pand %xmm1, %xmm3
+; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm1
+; SSE2-NEXT:    por %xmm3, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
+; SSE2-NEXT:    movq %xmm0, (%rdi)
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: trunc_ssat_v2i64_v2i32_store:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [2147483648,2147483648]
+; SSSE3-NEXT:    movdqa %xmm0, %xmm2
+; SSSE3-NEXT:    pxor %xmm1, %xmm2
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm3 = [4294967295,4294967295]
+; SSSE3-NEXT:    movdqa %xmm3, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm5, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm2, %xmm3
+; SSSE3-NEXT:    pand %xmm3, %xmm0
+; SSSE3-NEXT:    pandn {{.*}}(%rip), %xmm3
+; SSSE3-NEXT:    por %xmm0, %xmm3
+; SSSE3-NEXT:    pxor %xmm3, %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [18446744069414584320,18446744069414584320]
+; SSSE3-NEXT:    movdqa %xmm1, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm4, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    por %xmm0, %xmm1
+; SSSE3-NEXT:    pand %xmm1, %xmm3
+; SSSE3-NEXT:    pandn {{.*}}(%rip), %xmm1
+; SSSE3-NEXT:    por %xmm3, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
+; SSSE3-NEXT:    movq %xmm0, (%rdi)
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: trunc_ssat_v2i64_v2i32_store:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    movapd {{.*#+}} xmm2 = [2147483647,2147483647]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [2147483648,2147483648]
+; SSE41-NEXT:    pxor %xmm3, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [4294967295,4294967295]
+; SSE41-NEXT:    movdqa %xmm4, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm0
+; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
+; SSE41-NEXT:    movapd {{.*#+}} xmm1 = [18446744071562067968,18446744071562067968]
+; SSE41-NEXT:    pxor %xmm2, %xmm3
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [18446744069414584320,18446744069414584320]
+; SSE41-NEXT:    movdqa %xmm3, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm3, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
+; SSE41-NEXT:    movq %xmm0, (%rdi)
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: trunc_ssat_v2i64_v2i32_store:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [2147483647,2147483647]
+; AVX-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm2
+; AVX-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [18446744071562067968,18446744071562067968]
+; AVX-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm2
+; AVX-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX-NEXT:    vmovlpd %xmm0, (%rdi)
+; AVX-NEXT:    retq
+;
+; AVX512F-LABEL: trunc_ssat_v2i64_v2i32_store:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512F-NEXT:    vpmovsqd %zmm0, %ymm0
+; AVX512F-NEXT:    vmovq %xmm0, (%rdi)
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: trunc_ssat_v2i64_v2i32_store:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpmovsqd %xmm0, (%rdi)
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: trunc_ssat_v2i64_v2i32_store:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512BW-NEXT:    vpmovsqd %zmm0, %ymm0
+; AVX512BW-NEXT:    vmovq %xmm0, (%rdi)
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: trunc_ssat_v2i64_v2i32_store:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpmovsqd %xmm0, (%rdi)
+; AVX512BWVL-NEXT:    retq
+;
+; SKX-LABEL: trunc_ssat_v2i64_v2i32_store:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpmovsqd %xmm0, (%rdi)
+; SKX-NEXT:    retq
+  %1 = icmp slt <2 x i64> %a0, <i64 2147483647, i64 2147483647>
+  %2 = select <2 x i1> %1, <2 x i64> %a0, <2 x i64> <i64 2147483647, i64 2147483647>
+  %3 = icmp sgt <2 x i64> %2, <i64 -2147483648, i64 -2147483648>
+  %4 = select <2 x i1> %3, <2 x i64> %2, <2 x i64> <i64 -2147483648, i64 -2147483648>
+  %5 = trunc <2 x i64> %4 to <2 x i32>
+  store <2 x i32> %5, <2 x i32>* %p1
+  ret void
+}
+
 define <4 x i32> @trunc_ssat_v4i64_v4i32(<4 x i64> %a0) {
 ; SSE2-LABEL: trunc_ssat_v4i64_v4i32:
 ; SSE2:       # %bb.0:
@@ -145,47 +437,44 @@ define <4 x i32> @trunc_ssat_v4i64_v4i32(<4 x i64> %a0) {
 ; SSE41-NEXT:    pxor %xmm3, %xmm0
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm6 = [4294967295,4294967295]
 ; SSE41-NEXT:    movdqa %xmm6, %xmm5
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm5[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm6, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm7, %xmm0
-; SSE41-NEXT:    por %xmm5, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm5
+; SSE41-NEXT:    movdqa %xmm6, %xmm7
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm7
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm0
+; SSE41-NEXT:    por %xmm7, %xmm0
 ; SSE41-NEXT:    movapd %xmm4, %xmm5
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm5
 ; SSE41-NEXT:    movdqa %xmm1, %xmm0
 ; SSE41-NEXT:    pxor %xmm3, %xmm0
 ; SSE41-NEXT:    movdqa %xmm6, %xmm2
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm2[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm6, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm7, %xmm0
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm2
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
+; SSE41-NEXT:    pand %xmm2, %xmm0
+; SSE41-NEXT:    por %xmm6, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm4
 ; SSE41-NEXT:    movapd {{.*#+}} xmm1 = [18446744071562067968,18446744071562067968]
-; SSE41-NEXT:    movapd %xmm4, %xmm0
-; SSE41-NEXT:    xorpd %xmm3, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [18446744069414584320,18446744069414584320]
-; SSE41-NEXT:    movapd %xmm0, %xmm6
-; SSE41-NEXT:    pcmpgtd %xmm2, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm2, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE41-NEXT:    movapd %xmm4, %xmm2
+; SSE41-NEXT:    xorpd %xmm3, %xmm2
+; SSE41-NEXT:    movdqa {{.*#+}} xmm6 = [18446744069414584320,18446744069414584320]
+; SSE41-NEXT:    movapd %xmm2, %xmm7
+; SSE41-NEXT:    pcmpeqd %xmm6, %xmm7
+; SSE41-NEXT:    pcmpgtd %xmm6, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,0,2,2]
 ; SSE41-NEXT:    pand %xmm7, %xmm0
-; SSE41-NEXT:    por %xmm6, %xmm0
-; SSE41-NEXT:    movapd %xmm1, %xmm6
-; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm6
+; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    movapd %xmm1, %xmm2
+; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm2
 ; SSE41-NEXT:    xorpd %xmm5, %xmm3
 ; SSE41-NEXT:    movapd %xmm3, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm2, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm2, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pand %xmm7, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm6, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm6, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm3, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm5, %xmm1
-; SSE41-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,2],xmm6[0,2]
+; SSE41-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,2],xmm2[0,2]
 ; SSE41-NEXT:    movaps %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
@@ -236,11 +525,7 @@ define <4 x i32> @trunc_ssat_v4i64_v4i32(<4 x i64> %a0) {
 ; AVX512F-LABEL: trunc_ssat_v4i64_v4i32:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; AVX512F-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [2147483647,2147483647,2147483647,2147483647]
-; AVX512F-NEXT:    vpminsq %zmm1, %zmm0, %zmm0
-; AVX512F-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [18446744071562067968,18446744071562067968,18446744071562067968,18446744071562067968]
-; AVX512F-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm0
-; AVX512F-NEXT:    vpmovqd %zmm0, %ymm0
+; AVX512F-NEXT:    vpmovsqd %zmm0, %ymm0
 ; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
@@ -254,11 +539,7 @@ define <4 x i32> @trunc_ssat_v4i64_v4i32(<4 x i64> %a0) {
 ; AVX512BW-LABEL: trunc_ssat_v4i64_v4i32:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; AVX512BW-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [2147483647,2147483647,2147483647,2147483647]
-; AVX512BW-NEXT:    vpminsq %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [18446744071562067968,18446744071562067968,18446744071562067968,18446744071562067968]
-; AVX512BW-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpmovqd %zmm0, %ymm0
+; AVX512BW-NEXT:    vpmovsqd %zmm0, %ymm0
 ; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -268,6 +549,12 @@ define <4 x i32> @trunc_ssat_v4i64_v4i32(<4 x i64> %a0) {
 ; AVX512BWVL-NEXT:    vpmovsqd %ymm0, %xmm0
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
+;
+; SKX-LABEL: trunc_ssat_v4i64_v4i32:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpmovsqd %ymm0, %xmm0
+; SKX-NEXT:    vzeroupper
+; SKX-NEXT:    retq
   %1 = icmp slt <4 x i64> %a0, <i64 2147483647, i64 2147483647, i64 2147483647, i64 2147483647>
   %2 = select <4 x i1> %1, <4 x i64> %a0, <4 x i64> <i64 2147483647, i64 2147483647, i64 2147483647, i64 2147483647>
   %3 = icmp sgt <4 x i64> %2, <i64 -2147483648, i64 -2147483648, i64 -2147483648, i64 -2147483648>
@@ -277,327 +564,334 @@ define <4 x i32> @trunc_ssat_v4i64_v4i32(<4 x i64> %a0) {
 }
 
 
-define <8 x i32> @trunc_ssat_v8i64_v8i32(<8 x i64> %a0) {
+define <8 x i32> @trunc_ssat_v8i64_v8i32(<8 x i64>* %p0) "min-legal-vector-width"="256" {
 ; SSE2-LABEL: trunc_ssat_v8i64_v8i32:
 ; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa (%rdi), %xmm3
+; SSE2-NEXT:    movdqa 16(%rdi), %xmm5
+; SSE2-NEXT:    movdqa 32(%rdi), %xmm7
+; SSE2-NEXT:    movdqa 48(%rdi), %xmm9
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [2147483647,2147483647]
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [2147483648,2147483648]
-; SSE2-NEXT:    movdqa %xmm0, %xmm5
-; SSE2-NEXT:    pxor %xmm4, %xmm5
-; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [4294967295,4294967295]
-; SSE2-NEXT:    movdqa %xmm9, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm5, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm7[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[1,1,3,3]
-; SSE2-NEXT:    pand %xmm10, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm7[1,1,3,3]
-; SSE2-NEXT:    por %xmm6, %xmm5
-; SSE2-NEXT:    pand %xmm5, %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm3, %xmm2
+; SSE2-NEXT:    pxor %xmm0, %xmm2
+; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [4294967295,4294967295]
+; SSE2-NEXT:    movdqa %xmm10, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm6[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,1,3,3]
+; SSE2-NEXT:    pand %xmm1, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[1,1,3,3]
+; SSE2-NEXT:    por %xmm4, %xmm2
+; SSE2-NEXT:    pand %xmm2, %xmm3
+; SSE2-NEXT:    pandn %xmm8, %xmm2
+; SSE2-NEXT:    por %xmm3, %xmm2
+; SSE2-NEXT:    movdqa %xmm5, %xmm1
+; SSE2-NEXT:    pxor %xmm0, %xmm1
+; SSE2-NEXT:    movdqa %xmm10, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pand %xmm4, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSE2-NEXT:    por %xmm1, %xmm3
+; SSE2-NEXT:    pand %xmm3, %xmm5
+; SSE2-NEXT:    pandn %xmm8, %xmm3
+; SSE2-NEXT:    por %xmm5, %xmm3
+; SSE2-NEXT:    movdqa %xmm7, %xmm1
+; SSE2-NEXT:    pxor %xmm0, %xmm1
+; SSE2-NEXT:    movdqa %xmm10, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pand %xmm5, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm1, %xmm5
+; SSE2-NEXT:    pand %xmm5, %xmm7
 ; SSE2-NEXT:    pandn %xmm8, %xmm5
-; SSE2-NEXT:    por %xmm0, %xmm5
-; SSE2-NEXT:    movdqa %xmm1, %xmm0
-; SSE2-NEXT:    pxor %xmm4, %xmm0
-; SSE2-NEXT:    movdqa %xmm9, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pand %xmm10, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3]
-; SSE2-NEXT:    por %xmm7, %xmm0
-; SSE2-NEXT:    pand %xmm0, %xmm1
-; SSE2-NEXT:    pandn %xmm8, %xmm0
-; SSE2-NEXT:    por %xmm1, %xmm0
-; SSE2-NEXT:    movdqa %xmm2, %xmm1
-; SSE2-NEXT:    pxor %xmm4, %xmm1
-; SSE2-NEXT:    movdqa %xmm9, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm1
+; SSE2-NEXT:    por %xmm7, %xmm5
+; SSE2-NEXT:    movdqa %xmm9, %xmm1
+; SSE2-NEXT:    pxor %xmm0, %xmm1
+; SSE2-NEXT:    movdqa %xmm10, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pand %xmm7, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[1,1,3,3]
+; SSE2-NEXT:    pand %xmm6, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[1,1,3,3]
 ; SSE2-NEXT:    por %xmm1, %xmm7
-; SSE2-NEXT:    pand %xmm7, %xmm2
+; SSE2-NEXT:    pand %xmm7, %xmm9
 ; SSE2-NEXT:    pandn %xmm8, %xmm7
-; SSE2-NEXT:    por %xmm2, %xmm7
-; SSE2-NEXT:    movdqa %xmm3, %xmm1
-; SSE2-NEXT:    pxor %xmm4, %xmm1
-; SSE2-NEXT:    movdqa %xmm9, %xmm2
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm6
-; SSE2-NEXT:    pand %xmm6, %xmm3
-; SSE2-NEXT:    pandn %xmm8, %xmm6
-; SSE2-NEXT:    por %xmm3, %xmm6
+; SSE2-NEXT:    por %xmm9, %xmm7
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [18446744071562067968,18446744071562067968]
-; SSE2-NEXT:    movdqa %xmm6, %xmm1
-; SSE2-NEXT:    pxor %xmm4, %xmm1
+; SSE2-NEXT:    movdqa %xmm7, %xmm1
+; SSE2-NEXT:    pxor %xmm0, %xmm1
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [18446744069414584320,18446744069414584320]
-; SSE2-NEXT:    movdqa %xmm1, %xmm2
-; SSE2-NEXT:    pcmpgtd %xmm9, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
+; SSE2-NEXT:    movdqa %xmm1, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm9, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
 ; SSE2-NEXT:    pcmpeqd %xmm9, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pand %xmm3, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm2
-; SSE2-NEXT:    pand %xmm2, %xmm6
-; SSE2-NEXT:    pandn %xmm8, %xmm2
-; SSE2-NEXT:    por %xmm6, %xmm2
-; SSE2-NEXT:    movdqa %xmm7, %xmm1
-; SSE2-NEXT:    pxor %xmm4, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm3
-; SSE2-NEXT:    pcmpgtd %xmm9, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm3[0,0,2,2]
+; SSE2-NEXT:    pand %xmm6, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm1, %xmm4
+; SSE2-NEXT:    pand %xmm4, %xmm7
+; SSE2-NEXT:    pandn %xmm8, %xmm4
+; SSE2-NEXT:    por %xmm7, %xmm4
+; SSE2-NEXT:    movdqa %xmm5, %xmm1
+; SSE2-NEXT:    pxor %xmm0, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm9, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm6[0,0,2,2]
 ; SSE2-NEXT:    pcmpeqd %xmm9, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pand %xmm10, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE2-NEXT:    por %xmm6, %xmm1
-; SSE2-NEXT:    pand %xmm1, %xmm7
-; SSE2-NEXT:    pandn %xmm8, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pand %xmm10, %xmm7
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm6[1,1,3,3]
 ; SSE2-NEXT:    por %xmm7, %xmm1
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,2],xmm2[0,2]
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    pxor %xmm4, %xmm2
-; SSE2-NEXT:    movdqa %xmm2, %xmm3
-; SSE2-NEXT:    pcmpgtd %xmm9, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE2-NEXT:    por %xmm2, %xmm3
-; SSE2-NEXT:    pand %xmm3, %xmm0
-; SSE2-NEXT:    pandn %xmm8, %xmm3
-; SSE2-NEXT:    por %xmm0, %xmm3
-; SSE2-NEXT:    pxor %xmm5, %xmm4
-; SSE2-NEXT:    movdqa %xmm4, %xmm0
-; SSE2-NEXT:    pcmpgtd %xmm9, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
+; SSE2-NEXT:    pand %xmm1, %xmm5
+; SSE2-NEXT:    pandn %xmm8, %xmm1
+; SSE2-NEXT:    por %xmm5, %xmm1
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,2],xmm4[0,2]
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    pxor %xmm0, %xmm4
+; SSE2-NEXT:    movdqa %xmm4, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm9, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
 ; SSE2-NEXT:    pcmpeqd %xmm9, %xmm4
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    pand %xmm2, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm4, %xmm0
-; SSE2-NEXT:    pand %xmm0, %xmm5
-; SSE2-NEXT:    pandn %xmm8, %xmm0
-; SSE2-NEXT:    por %xmm5, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm3[0,2]
-; SSE2-NEXT:    retq
-;
-; SSSE3-LABEL: trunc_ssat_v8i64_v8i32:
-; SSSE3:       # %bb.0:
+; SSE2-NEXT:    pand %xmm6, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSE2-NEXT:    por %xmm4, %xmm5
+; SSE2-NEXT:    pand %xmm5, %xmm3
+; SSE2-NEXT:    pandn %xmm8, %xmm5
+; SSE2-NEXT:    por %xmm3, %xmm5
+; SSE2-NEXT:    pxor %xmm2, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm9, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm9, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pand %xmm4, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
+; SSE2-NEXT:    por %xmm6, %xmm0
+; SSE2-NEXT:    pand %xmm0, %xmm2
+; SSE2-NEXT:    pandn %xmm8, %xmm0
+; SSE2-NEXT:    por %xmm2, %xmm0
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm5[0,2]
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: trunc_ssat_v8i64_v8i32:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa (%rdi), %xmm3
+; SSSE3-NEXT:    movdqa 16(%rdi), %xmm5
+; SSSE3-NEXT:    movdqa 32(%rdi), %xmm7
+; SSSE3-NEXT:    movdqa 48(%rdi), %xmm9
 ; SSSE3-NEXT:    movdqa {{.*#+}} xmm8 = [2147483647,2147483647]
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm4 = [2147483648,2147483648]
-; SSSE3-NEXT:    movdqa %xmm0, %xmm5
-; SSSE3-NEXT:    pxor %xmm4, %xmm5
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm9 = [4294967295,4294967295]
-; SSSE3-NEXT:    movdqa %xmm9, %xmm7
-; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm7
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm10 = xmm7[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm5
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm10, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm7[1,1,3,3]
-; SSSE3-NEXT:    por %xmm6, %xmm5
-; SSSE3-NEXT:    pand %xmm5, %xmm0
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483648,2147483648]
+; SSSE3-NEXT:    movdqa %xmm3, %xmm2
+; SSSE3-NEXT:    pxor %xmm0, %xmm2
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm10 = [4294967295,4294967295]
+; SSSE3-NEXT:    movdqa %xmm10, %xmm6
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm6[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm1, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[1,1,3,3]
+; SSSE3-NEXT:    por %xmm4, %xmm2
+; SSSE3-NEXT:    pand %xmm2, %xmm3
+; SSSE3-NEXT:    pandn %xmm8, %xmm2
+; SSSE3-NEXT:    por %xmm3, %xmm2
+; SSSE3-NEXT:    movdqa %xmm5, %xmm1
+; SSSE3-NEXT:    pxor %xmm0, %xmm1
+; SSSE3-NEXT:    movdqa %xmm10, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm4, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSSE3-NEXT:    por %xmm1, %xmm3
+; SSSE3-NEXT:    pand %xmm3, %xmm5
+; SSSE3-NEXT:    pandn %xmm8, %xmm3
+; SSSE3-NEXT:    por %xmm5, %xmm3
+; SSSE3-NEXT:    movdqa %xmm7, %xmm1
+; SSSE3-NEXT:    pxor %xmm0, %xmm1
+; SSSE3-NEXT:    movdqa %xmm10, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm5, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm1, %xmm5
+; SSSE3-NEXT:    pand %xmm5, %xmm7
 ; SSSE3-NEXT:    pandn %xmm8, %xmm5
-; SSSE3-NEXT:    por %xmm0, %xmm5
-; SSSE3-NEXT:    movdqa %xmm1, %xmm0
-; SSSE3-NEXT:    pxor %xmm4, %xmm0
-; SSSE3-NEXT:    movdqa %xmm9, %xmm6
-; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm10 = xmm6[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm10, %xmm7
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3]
-; SSSE3-NEXT:    por %xmm7, %xmm0
-; SSSE3-NEXT:    pand %xmm0, %xmm1
-; SSSE3-NEXT:    pandn %xmm8, %xmm0
-; SSSE3-NEXT:    por %xmm1, %xmm0
-; SSSE3-NEXT:    movdqa %xmm2, %xmm1
-; SSSE3-NEXT:    pxor %xmm4, %xmm1
-; SSSE3-NEXT:    movdqa %xmm9, %xmm6
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm1
+; SSSE3-NEXT:    por %xmm7, %xmm5
+; SSSE3-NEXT:    movdqa %xmm9, %xmm1
+; SSSE3-NEXT:    pxor %xmm0, %xmm1
+; SSSE3-NEXT:    movdqa %xmm10, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm1
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm7, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm6, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[1,1,3,3]
 ; SSSE3-NEXT:    por %xmm1, %xmm7
-; SSSE3-NEXT:    pand %xmm7, %xmm2
+; SSSE3-NEXT:    pand %xmm7, %xmm9
 ; SSSE3-NEXT:    pandn %xmm8, %xmm7
-; SSSE3-NEXT:    por %xmm2, %xmm7
-; SSSE3-NEXT:    movdqa %xmm3, %xmm1
-; SSSE3-NEXT:    pxor %xmm4, %xmm1
-; SSSE3-NEXT:    movdqa %xmm9, %xmm2
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm6, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[1,1,3,3]
-; SSSE3-NEXT:    por %xmm1, %xmm6
-; SSSE3-NEXT:    pand %xmm6, %xmm3
-; SSSE3-NEXT:    pandn %xmm8, %xmm6
-; SSSE3-NEXT:    por %xmm3, %xmm6
+; SSSE3-NEXT:    por %xmm9, %xmm7
 ; SSSE3-NEXT:    movdqa {{.*#+}} xmm8 = [18446744071562067968,18446744071562067968]
-; SSSE3-NEXT:    movdqa %xmm6, %xmm1
-; SSSE3-NEXT:    pxor %xmm4, %xmm1
+; SSSE3-NEXT:    movdqa %xmm7, %xmm1
+; SSSE3-NEXT:    pxor %xmm0, %xmm1
 ; SSSE3-NEXT:    movdqa {{.*#+}} xmm9 = [18446744069414584320,18446744069414584320]
-; SSSE3-NEXT:    movdqa %xmm1, %xmm2
-; SSSE3-NEXT:    pcmpgtd %xmm9, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
+; SSSE3-NEXT:    movdqa %xmm1, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm9, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
 ; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm1
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm3, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSSE3-NEXT:    por %xmm1, %xmm2
-; SSSE3-NEXT:    pand %xmm2, %xmm6
-; SSSE3-NEXT:    pandn %xmm8, %xmm2
-; SSSE3-NEXT:    por %xmm6, %xmm2
-; SSSE3-NEXT:    movdqa %xmm7, %xmm1
-; SSSE3-NEXT:    pxor %xmm4, %xmm1
-; SSSE3-NEXT:    movdqa %xmm1, %xmm3
-; SSSE3-NEXT:    pcmpgtd %xmm9, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm10 = xmm3[0,0,2,2]
+; SSSE3-NEXT:    pand %xmm6, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm1, %xmm4
+; SSSE3-NEXT:    pand %xmm4, %xmm7
+; SSSE3-NEXT:    pandn %xmm8, %xmm4
+; SSSE3-NEXT:    por %xmm7, %xmm4
+; SSSE3-NEXT:    movdqa %xmm5, %xmm1
+; SSSE3-NEXT:    pxor %xmm0, %xmm1
+; SSSE3-NEXT:    movdqa %xmm1, %xmm6
+; SSSE3-NEXT:    pcmpgtd %xmm9, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm10 = xmm6[0,0,2,2]
 ; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm10, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm6, %xmm1
-; SSSE3-NEXT:    pand %xmm1, %xmm7
-; SSSE3-NEXT:    pandn %xmm8, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm10, %xmm7
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm6[1,1,3,3]
 ; SSSE3-NEXT:    por %xmm7, %xmm1
-; SSSE3-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,2],xmm2[0,2]
-; SSSE3-NEXT:    movdqa %xmm0, %xmm2
-; SSSE3-NEXT:    pxor %xmm4, %xmm2
-; SSSE3-NEXT:    movdqa %xmm2, %xmm3
-; SSSE3-NEXT:    pcmpgtd %xmm9, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm6, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm3
-; SSSE3-NEXT:    pand %xmm3, %xmm0
-; SSSE3-NEXT:    pandn %xmm8, %xmm3
-; SSSE3-NEXT:    por %xmm0, %xmm3
-; SSSE3-NEXT:    pxor %xmm5, %xmm4
-; SSSE3-NEXT:    movdqa %xmm4, %xmm0
-; SSSE3-NEXT:    pcmpgtd %xmm9, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    pand %xmm1, %xmm5
+; SSSE3-NEXT:    pandn %xmm8, %xmm1
+; SSSE3-NEXT:    por %xmm5, %xmm1
+; SSSE3-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,2],xmm4[0,2]
+; SSSE3-NEXT:    movdqa %xmm3, %xmm4
+; SSSE3-NEXT:    pxor %xmm0, %xmm4
+; SSSE3-NEXT:    movdqa %xmm4, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm9, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
 ; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm4
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm2, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    por %xmm4, %xmm0
-; SSSE3-NEXT:    pand %xmm0, %xmm5
+; SSSE3-NEXT:    pand %xmm6, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSSE3-NEXT:    por %xmm4, %xmm5
+; SSSE3-NEXT:    pand %xmm5, %xmm3
+; SSSE3-NEXT:    pandn %xmm8, %xmm5
+; SSSE3-NEXT:    por %xmm3, %xmm5
+; SSSE3-NEXT:    pxor %xmm2, %xmm0
+; SSSE3-NEXT:    movdqa %xmm0, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm9, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm4, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
+; SSSE3-NEXT:    por %xmm6, %xmm0
+; SSSE3-NEXT:    pand %xmm0, %xmm2
 ; SSSE3-NEXT:    pandn %xmm8, %xmm0
-; SSSE3-NEXT:    por %xmm5, %xmm0
-; SSSE3-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm3[0,2]
+; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm5[0,2]
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: trunc_ssat_v8i64_v8i32:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa %xmm0, %xmm4
-; SSE41-NEXT:    movapd {{.*#+}} xmm7 = [2147483647,2147483647]
-; SSE41-NEXT:    movdqa {{.*#+}} xmm5 = [2147483648,2147483648]
-; SSE41-NEXT:    pxor %xmm5, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm10 = [4294967295,4294967295]
-; SSE41-NEXT:    movdqa %xmm10, %xmm6
+; SSE41-NEXT:    movdqa (%rdi), %xmm5
+; SSE41-NEXT:    movdqa 16(%rdi), %xmm4
+; SSE41-NEXT:    movdqa 32(%rdi), %xmm10
+; SSE41-NEXT:    movdqa 48(%rdi), %xmm9
+; SSE41-NEXT:    movapd {{.*#+}} xmm1 = [2147483647,2147483647]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [2147483648,2147483648]
+; SSE41-NEXT:    movdqa %xmm5, %xmm0
+; SSE41-NEXT:    pxor %xmm3, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [4294967295,4294967295]
+; SSE41-NEXT:    movdqa %xmm2, %xmm7
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm7
+; SSE41-NEXT:    movdqa %xmm2, %xmm6
 ; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm8 = xmm6[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm8, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
+; SSE41-NEXT:    pand %xmm7, %xmm0
 ; SSE41-NEXT:    por %xmm6, %xmm0
-; SSE41-NEXT:    movapd %xmm7, %xmm8
-; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm8
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    pxor %xmm5, %xmm0
-; SSE41-NEXT:    movdqa %xmm10, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm6, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
-; SSE41-NEXT:    movapd %xmm7, %xmm9
-; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm9
-; SSE41-NEXT:    movdqa %xmm2, %xmm0
-; SSE41-NEXT:    pxor %xmm5, %xmm0
-; SSE41-NEXT:    movdqa %xmm10, %xmm1
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE41-NEXT:    movapd %xmm1, %xmm8
+; SSE41-NEXT:    blendvpd %xmm0, %xmm5, %xmm8
+; SSE41-NEXT:    movdqa %xmm4, %xmm0
+; SSE41-NEXT:    pxor %xmm3, %xmm0
+; SSE41-NEXT:    movdqa %xmm2, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm5
+; SSE41-NEXT:    movdqa %xmm2, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm0
+; SSE41-NEXT:    por %xmm6, %xmm0
+; SSE41-NEXT:    movapd %xmm1, %xmm11
+; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm11
+; SSE41-NEXT:    movdqa %xmm10, %xmm0
+; SSE41-NEXT:    pxor %xmm3, %xmm0
+; SSE41-NEXT:    movdqa %xmm2, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT:    movdqa %xmm2, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
 ; SSE41-NEXT:    pand %xmm4, %xmm0
-; SSE41-NEXT:    por %xmm1, %xmm0
-; SSE41-NEXT:    movapd %xmm7, %xmm4
-; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm4
-; SSE41-NEXT:    movdqa %xmm3, %xmm0
-; SSE41-NEXT:    pxor %xmm5, %xmm0
-; SSE41-NEXT:    movdqa %xmm10, %xmm1
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm2, %xmm0
-; SSE41-NEXT:    por %xmm1, %xmm0
-; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm7
-; SSE41-NEXT:    movapd {{.*#+}} xmm2 = [18446744071562067968,18446744071562067968]
-; SSE41-NEXT:    movapd %xmm7, %xmm0
-; SSE41-NEXT:    xorpd %xmm5, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [18446744069414584320,18446744069414584320]
-; SSE41-NEXT:    movapd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE41-NEXT:    por %xmm6, %xmm0
+; SSE41-NEXT:    movapd %xmm1, %xmm4
+; SSE41-NEXT:    blendvpd %xmm0, %xmm10, %xmm4
+; SSE41-NEXT:    movdqa %xmm9, %xmm0
+; SSE41-NEXT:    pxor %xmm3, %xmm0
+; SSE41-NEXT:    movdqa %xmm2, %xmm6
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,0,2,2]
 ; SSE41-NEXT:    pand %xmm6, %xmm0
-; SSE41-NEXT:    por %xmm1, %xmm0
-; SSE41-NEXT:    movapd %xmm2, %xmm6
-; SSE41-NEXT:    blendvpd %xmm0, %xmm7, %xmm6
-; SSE41-NEXT:    movapd %xmm4, %xmm0
-; SSE41-NEXT:    xorpd %xmm5, %xmm0
-; SSE41-NEXT:    movapd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm1[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm9, %xmm1
+; SSE41-NEXT:    movapd {{.*#+}} xmm2 = [18446744071562067968,18446744071562067968]
+; SSE41-NEXT:    movapd %xmm1, %xmm7
+; SSE41-NEXT:    xorpd %xmm3, %xmm7
+; SSE41-NEXT:    movdqa {{.*#+}} xmm6 = [18446744069414584320,18446744069414584320]
+; SSE41-NEXT:    movapd %xmm7, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm6, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm6, %xmm7
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm0
+; SSE41-NEXT:    por %xmm7, %xmm0
+; SSE41-NEXT:    movapd %xmm2, %xmm5
+; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm5
+; SSE41-NEXT:    movapd %xmm4, %xmm1
+; SSE41-NEXT:    xorpd %xmm3, %xmm1
+; SSE41-NEXT:    movapd %xmm1, %xmm7
+; SSE41-NEXT:    pcmpeqd %xmm6, %xmm7
+; SSE41-NEXT:    pcmpgtd %xmm6, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2]
 ; SSE41-NEXT:    pand %xmm7, %xmm0
 ; SSE41-NEXT:    por %xmm1, %xmm0
 ; SSE41-NEXT:    movapd %xmm2, %xmm1
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm1
-; SSE41-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,2],xmm6[0,2]
-; SSE41-NEXT:    movapd %xmm9, %xmm0
-; SSE41-NEXT:    xorpd %xmm5, %xmm0
-; SSE41-NEXT:    movapd %xmm0, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm6, %xmm0
+; SSE41-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,2],xmm5[0,2]
+; SSE41-NEXT:    movapd %xmm11, %xmm4
+; SSE41-NEXT:    xorpd %xmm3, %xmm4
+; SSE41-NEXT:    movapd %xmm4, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm6, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm6, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm0
 ; SSE41-NEXT:    por %xmm4, %xmm0
 ; SSE41-NEXT:    movapd %xmm2, %xmm4
-; SSE41-NEXT:    blendvpd %xmm0, %xmm9, %xmm4
-; SSE41-NEXT:    xorpd %xmm8, %xmm5
-; SSE41-NEXT:    movapd %xmm5, %xmm6
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm3, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[1,1,3,3]
-; SSE41-NEXT:    pand %xmm7, %xmm0
-; SSE41-NEXT:    por %xmm6, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm11, %xmm4
+; SSE41-NEXT:    xorpd %xmm8, %xmm3
+; SSE41-NEXT:    movapd %xmm3, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm6, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm6, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm0
+; SSE41-NEXT:    por %xmm3, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm8, %xmm2
 ; SSE41-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,2],xmm4[0,2]
 ; SSE41-NEXT:    movaps %xmm2, %xmm0
@@ -605,33 +899,37 @@ define <8 x i32> @trunc_ssat_v8i64_v8i32(<8 x i64> %a0) {
 ;
 ; AVX1-LABEL: trunc_ssat_v8i64_v8i32:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [2147483647,2147483647]
-; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm3, %xmm8
-; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm3, %xmm5
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm6
-; AVX1-NEXT:    vpcmpgtq %xmm6, %xmm3, %xmm7
-; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm3, %xmm4
-; AVX1-NEXT:    vblendvpd %xmm4, %xmm0, %xmm3, %xmm0
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [18446744071562067968,18446744071562067968]
-; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm0, %xmm9
-; AVX1-NEXT:    vblendvpd %xmm7, %xmm6, %xmm3, %xmm6
-; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm6, %xmm7
-; AVX1-NEXT:    vblendvpd %xmm5, %xmm1, %xmm3, %xmm1
-; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm1, %xmm5
-; AVX1-NEXT:    vblendvpd %xmm8, %xmm2, %xmm3, %xmm2
-; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm2, %xmm3
-; AVX1-NEXT:    vblendvpd %xmm3, %xmm2, %xmm4, %xmm2
-; AVX1-NEXT:    vblendvpd %xmm5, %xmm1, %xmm4, %xmm1
-; AVX1-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[0,2],xmm2[0,2]
-; AVX1-NEXT:    vblendvpd %xmm7, %xmm6, %xmm4, %xmm2
-; AVX1-NEXT:    vblendvpd %xmm9, %xmm0, %xmm4, %xmm0
-; AVX1-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm2[0,2]
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX1-NEXT:    vmovdqa 16(%rdi), %xmm1
+; AVX1-NEXT:    vmovdqa 32(%rdi), %xmm2
+; AVX1-NEXT:    vmovdqa 48(%rdi), %xmm3
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [2147483647,2147483647]
+; AVX1-NEXT:    vpcmpgtq %xmm3, %xmm4, %xmm8
+; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm4, %xmm6
+; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm4, %xmm7
+; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm4, %xmm5
+; AVX1-NEXT:    vblendvpd %xmm5, %xmm0, %xmm4, %xmm0
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm5 = [18446744071562067968,18446744071562067968]
+; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm0, %xmm9
+; AVX1-NEXT:    vblendvpd %xmm7, %xmm1, %xmm4, %xmm1
+; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm1, %xmm7
+; AVX1-NEXT:    vblendvpd %xmm6, %xmm2, %xmm4, %xmm2
+; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm2, %xmm6
+; AVX1-NEXT:    vblendvpd %xmm8, %xmm3, %xmm4, %xmm3
+; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm3, %xmm4
+; AVX1-NEXT:    vblendvpd %xmm4, %xmm3, %xmm5, %xmm3
+; AVX1-NEXT:    vblendvpd %xmm6, %xmm2, %xmm5, %xmm2
+; AVX1-NEXT:    vshufps {{.*#+}} xmm2 = xmm2[0,2],xmm3[0,2]
+; AVX1-NEXT:    vblendvpd %xmm7, %xmm1, %xmm5, %xmm1
+; AVX1-NEXT:    vblendvpd %xmm9, %xmm0, %xmm5, %xmm0
+; AVX1-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-SLOW-LABEL: trunc_ssat_v8i64_v8i32:
 ; AVX2-SLOW:       # %bb.0:
+; AVX2-SLOW-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX2-SLOW-NEXT:    vmovdqa 32(%rdi), %ymm1
 ; AVX2-SLOW-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [2147483647,2147483647,2147483647,2147483647]
 ; AVX2-SLOW-NEXT:    vpcmpgtq %ymm1, %ymm2, %ymm3
 ; AVX2-SLOW-NEXT:    vblendvpd %ymm3, %ymm1, %ymm2, %ymm1
@@ -651,6 +949,8 @@ define <8 x i32> @trunc_ssat_v8i64_v8i32(<8 x i64> %a0) {
 ;
 ; AVX2-FAST-LABEL: trunc_ssat_v8i64_v8i32:
 ; AVX2-FAST:       # %bb.0:
+; AVX2-FAST-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX2-FAST-NEXT:    vmovdqa 32(%rdi), %ymm1
 ; AVX2-FAST-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [2147483647,2147483647,2147483647,2147483647]
 ; AVX2-FAST-NEXT:    vpcmpgtq %ymm0, %ymm2, %ymm3
 ; AVX2-FAST-NEXT:    vblendvpd %ymm3, %ymm0, %ymm2, %ymm0
@@ -669,8 +969,19 @@ define <8 x i32> @trunc_ssat_v8i64_v8i32(<8 x i64> %a0) {
 ;
 ; AVX512-LABEL: trunc_ssat_v8i64_v8i32:
 ; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovdqa64 (%rdi), %zmm0
 ; AVX512-NEXT:    vpmovsqd %zmm0, %ymm0
 ; AVX512-NEXT:    retq
+;
+; SKX-LABEL: trunc_ssat_v8i64_v8i32:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vmovdqa (%rdi), %ymm0
+; SKX-NEXT:    vmovdqa 32(%rdi), %ymm1
+; SKX-NEXT:    vpmovsqd %ymm0, %xmm0
+; SKX-NEXT:    vpmovsqd %ymm1, %xmm1
+; SKX-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; SKX-NEXT:    retq
+  %a0 = load <8 x i64>, <8 x i64>* %p0
   %1 = icmp slt <8 x i64> %a0, <i64 2147483647, i64 2147483647, i64 2147483647, i64 2147483647, i64 2147483647, i64 2147483647, i64 2147483647, i64 2147483647>
   %2 = select <8 x i1> %1, <8 x i64> %a0, <8 x i64> <i64 2147483647, i64 2147483647, i64 2147483647, i64 2147483647, i64 2147483647, i64 2147483647, i64 2147483647, i64 2147483647>
   %3 = icmp sgt <8 x i64> %2, <i64 -2147483648, i64 -2147483648, i64 -2147483648, i64 -2147483648, i64 -2147483648, i64 -2147483648, i64 -2147483648, i64 -2147483648>
@@ -683,880 +994,2774 @@ define <8 x i32> @trunc_ssat_v8i64_v8i32(<8 x i64> %a0) {
 ; Signed saturation truncation to vXi16
 ;
 
-define <8 x i16> @trunc_ssat_v8i64_v8i16(<8 x i64> %a0) {
-; SSE2-LABEL: trunc_ssat_v8i64_v8i16:
+define <2 x i16> @trunc_ssat_v2i64_v2i16(<2 x i64> %a0) {
+; SSE2-LABEL: trunc_ssat_v2i64_v2i16:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [32767,32767]
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [2147483648,2147483648]
-; SSE2-NEXT:    movdqa %xmm2, %xmm5
-; SSE2-NEXT:    pxor %xmm4, %xmm5
-; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [2147516415,2147516415]
-; SSE2-NEXT:    movdqa %xmm9, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm5, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm7[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[1,1,3,3]
-; SSE2-NEXT:    pand %xmm10, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm7[1,1,3,3]
-; SSE2-NEXT:    por %xmm6, %xmm5
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    pxor %xmm1, %xmm2
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [2147516415,2147516415]
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm3, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm5, %xmm2
-; SSE2-NEXT:    pandn %xmm8, %xmm5
-; SSE2-NEXT:    por %xmm2, %xmm5
-; SSE2-NEXT:    movdqa %xmm3, %xmm2
-; SSE2-NEXT:    pxor %xmm4, %xmm2
-; SSE2-NEXT:    movdqa %xmm9, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm2[1,1,3,3]
-; SSE2-NEXT:    pand %xmm10, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[1,1,3,3]
-; SSE2-NEXT:    por %xmm7, %xmm2
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pandn %xmm8, %xmm2
-; SSE2-NEXT:    por %xmm3, %xmm2
-; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    pxor %xmm4, %xmm3
-; SSE2-NEXT:    movdqa %xmm9, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm3[1,1,3,3]
-; SSE2-NEXT:    pand %xmm10, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm6[1,1,3,3]
-; SSE2-NEXT:    por %xmm7, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm2, %xmm3
 ; SSE2-NEXT:    pand %xmm3, %xmm0
-; SSE2-NEXT:    pandn %xmm8, %xmm3
+; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm3
 ; SSE2-NEXT:    por %xmm0, %xmm3
-; SSE2-NEXT:    movdqa %xmm1, %xmm0
-; SSE2-NEXT:    pxor %xmm4, %xmm0
-; SSE2-NEXT:    movdqa %xmm9, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pand %xmm7, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[1,1,3,3]
-; SSE2-NEXT:    por %xmm0, %xmm7
-; SSE2-NEXT:    pand %xmm7, %xmm1
-; SSE2-NEXT:    pandn %xmm8, %xmm7
-; SSE2-NEXT:    por %xmm1, %xmm7
-; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [18446744073709518848,18446744073709518848]
-; SSE2-NEXT:    movdqa %xmm7, %xmm0
-; SSE2-NEXT:    pxor %xmm4, %xmm0
-; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [18446744071562035200,18446744071562035200]
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm9, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pxor %xmm3, %xmm1
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [18446744071562035200,18446744071562035200]
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm0, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pand %xmm4, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,3,3]
 ; SSE2-NEXT:    por %xmm0, %xmm1
-; SSE2-NEXT:    pand %xmm1, %xmm7
-; SSE2-NEXT:    pandn %xmm8, %xmm1
-; SSE2-NEXT:    por %xmm7, %xmm1
-; SSE2-NEXT:    movdqa %xmm3, %xmm0
-; SSE2-NEXT:    pxor %xmm4, %xmm0
-; SSE2-NEXT:    movdqa %xmm0, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm9, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pand %xmm10, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3]
-; SSE2-NEXT:    por %xmm7, %xmm0
-; SSE2-NEXT:    pand %xmm0, %xmm3
-; SSE2-NEXT:    pandn %xmm8, %xmm0
+; SSE2-NEXT:    pand %xmm1, %xmm3
+; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm1
+; SSE2-NEXT:    por %xmm3, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: trunc_ssat_v2i64_v2i16:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [2147483648,2147483648]
+; SSSE3-NEXT:    movdqa %xmm0, %xmm2
+; SSSE3-NEXT:    pxor %xmm1, %xmm2
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm3 = [2147516415,2147516415]
+; SSSE3-NEXT:    movdqa %xmm3, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm5, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm2, %xmm3
+; SSSE3-NEXT:    pand %xmm3, %xmm0
+; SSSE3-NEXT:    pandn {{.*}}(%rip), %xmm3
+; SSSE3-NEXT:    por %xmm0, %xmm3
+; SSSE3-NEXT:    pxor %xmm3, %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [18446744071562035200,18446744071562035200]
+; SSSE3-NEXT:    movdqa %xmm1, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm4, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    por %xmm0, %xmm1
+; SSSE3-NEXT:    pand %xmm1, %xmm3
+; SSSE3-NEXT:    pandn {{.*}}(%rip), %xmm1
+; SSSE3-NEXT:    por %xmm3, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
+; SSSE3-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: trunc_ssat_v2i64_v2i16:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    movapd {{.*#+}} xmm2 = [32767,32767]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [2147483648,2147483648]
+; SSE41-NEXT:    pxor %xmm3, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [2147516415,2147516415]
+; SSE41-NEXT:    movdqa %xmm4, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm0
+; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
+; SSE41-NEXT:    movapd {{.*#+}} xmm1 = [18446744073709518848,18446744073709518848]
+; SSE41-NEXT:    pxor %xmm2, %xmm3
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [18446744071562035200,18446744071562035200]
+; SSE41-NEXT:    movdqa %xmm3, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm3, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
+; SSE41-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: trunc_ssat_v2i64_v2i16:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [32767,32767]
+; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm2
+; AVX1-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [18446744073709518848,18446744073709518848]
+; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm2
+; AVX1-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; AVX1-NEXT:    retq
+;
+; AVX2-SLOW-LABEL: trunc_ssat_v2i64_v2i16:
+; AVX2-SLOW:       # %bb.0:
+; AVX2-SLOW-NEXT:    vmovdqa {{.*#+}} xmm1 = [32767,32767]
+; AVX2-SLOW-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm2
+; AVX2-SLOW-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX2-SLOW-NEXT:    vmovdqa {{.*#+}} xmm1 = [18446744073709518848,18446744073709518848]
+; AVX2-SLOW-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm2
+; AVX2-SLOW-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX2-SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; AVX2-SLOW-NEXT:    retq
+;
+; AVX2-FAST-LABEL: trunc_ssat_v2i64_v2i16:
+; AVX2-FAST:       # %bb.0:
+; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} xmm1 = [32767,32767]
+; AVX2-FAST-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm2
+; AVX2-FAST-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} xmm1 = [18446744073709518848,18446744073709518848]
+; AVX2-FAST-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm2
+; AVX2-FAST-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX2-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,8,9,8,9,10,11,8,9,10,11,12,13,14,15]
+; AVX2-FAST-NEXT:    retq
+;
+; AVX512F-LABEL: trunc_ssat_v2i64_v2i16:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512F-NEXT:    vpmovsqw %zmm0, %xmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: trunc_ssat_v2i64_v2i16:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpmovsqw %xmm0, %xmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: trunc_ssat_v2i64_v2i16:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512BW-NEXT:    vpmovsqw %zmm0, %xmm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: trunc_ssat_v2i64_v2i16:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpmovsqw %xmm0, %xmm0
+; AVX512BWVL-NEXT:    retq
+;
+; SKX-LABEL: trunc_ssat_v2i64_v2i16:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpmovsqw %xmm0, %xmm0
+; SKX-NEXT:    retq
+  %1 = icmp slt <2 x i64> %a0, <i64 32767, i64 32767>
+  %2 = select <2 x i1> %1, <2 x i64> %a0, <2 x i64> <i64 32767, i64 32767>
+  %3 = icmp sgt <2 x i64> %2, <i64 -32768, i64 -32768>
+  %4 = select <2 x i1> %3, <2 x i64> %2, <2 x i64> <i64 -32768, i64 -32768>
+  %5 = trunc <2 x i64> %4 to <2 x i16>
+  ret <2 x i16> %5
+}
+
+define void @trunc_ssat_v2i64_v2i16_store(<2 x i64> %a0, <2 x i16> *%p1) {
+; SSE2-LABEL: trunc_ssat_v2i64_v2i16_store:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    pxor %xmm1, %xmm2
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [2147516415,2147516415]
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm3, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE2-NEXT:    pand %xmm5, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm2, %xmm3
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm3
+; SSE2-NEXT:    por %xmm0, %xmm3
+; SSE2-NEXT:    pxor %xmm3, %xmm1
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [18446744071562035200,18446744071562035200]
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm0, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pand %xmm4, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,3,3]
+; SSE2-NEXT:    por %xmm0, %xmm1
+; SSE2-NEXT:    pand %xmm1, %xmm3
+; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm1
+; SSE2-NEXT:    por %xmm3, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; SSE2-NEXT:    movd %xmm0, (%rdi)
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: trunc_ssat_v2i64_v2i16_store:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [2147483648,2147483648]
+; SSSE3-NEXT:    movdqa %xmm0, %xmm2
+; SSSE3-NEXT:    pxor %xmm1, %xmm2
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm3 = [2147516415,2147516415]
+; SSSE3-NEXT:    movdqa %xmm3, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm5, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm2, %xmm3
+; SSSE3-NEXT:    pand %xmm3, %xmm0
+; SSSE3-NEXT:    pandn {{.*}}(%rip), %xmm3
+; SSSE3-NEXT:    por %xmm0, %xmm3
+; SSSE3-NEXT:    pxor %xmm3, %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [18446744071562035200,18446744071562035200]
+; SSSE3-NEXT:    movdqa %xmm1, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm4, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    por %xmm0, %xmm1
+; SSSE3-NEXT:    pand %xmm1, %xmm3
+; SSSE3-NEXT:    pandn {{.*}}(%rip), %xmm1
+; SSSE3-NEXT:    por %xmm3, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
+; SSSE3-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; SSSE3-NEXT:    movd %xmm0, (%rdi)
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: trunc_ssat_v2i64_v2i16_store:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    movapd {{.*#+}} xmm2 = [32767,32767]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [2147483648,2147483648]
+; SSE41-NEXT:    pxor %xmm3, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [2147516415,2147516415]
+; SSE41-NEXT:    movdqa %xmm4, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm0
+; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
+; SSE41-NEXT:    movapd {{.*#+}} xmm1 = [18446744073709518848,18446744073709518848]
+; SSE41-NEXT:    pxor %xmm2, %xmm3
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [18446744071562035200,18446744071562035200]
+; SSE41-NEXT:    movdqa %xmm3, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm3, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
+; SSE41-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; SSE41-NEXT:    movd %xmm0, (%rdi)
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: trunc_ssat_v2i64_v2i16_store:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [32767,32767]
+; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm2
+; AVX1-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [18446744073709518848,18446744073709518848]
+; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm2
+; AVX1-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; AVX1-NEXT:    vmovd %xmm0, (%rdi)
+; AVX1-NEXT:    retq
+;
+; AVX2-SLOW-LABEL: trunc_ssat_v2i64_v2i16_store:
+; AVX2-SLOW:       # %bb.0:
+; AVX2-SLOW-NEXT:    vmovdqa {{.*#+}} xmm1 = [32767,32767]
+; AVX2-SLOW-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm2
+; AVX2-SLOW-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX2-SLOW-NEXT:    vmovdqa {{.*#+}} xmm1 = [18446744073709518848,18446744073709518848]
+; AVX2-SLOW-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm2
+; AVX2-SLOW-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX2-SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vmovd %xmm0, (%rdi)
+; AVX2-SLOW-NEXT:    retq
+;
+; AVX2-FAST-LABEL: trunc_ssat_v2i64_v2i16_store:
+; AVX2-FAST:       # %bb.0:
+; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} xmm1 = [32767,32767]
+; AVX2-FAST-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm2
+; AVX2-FAST-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} xmm1 = [18446744073709518848,18446744073709518848]
+; AVX2-FAST-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm2
+; AVX2-FAST-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX2-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,8,9,8,9,10,11,8,9,10,11,12,13,14,15]
+; AVX2-FAST-NEXT:    vmovd %xmm0, (%rdi)
+; AVX2-FAST-NEXT:    retq
+;
+; AVX512F-LABEL: trunc_ssat_v2i64_v2i16_store:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512F-NEXT:    vpmovsqw %zmm0, %xmm0
+; AVX512F-NEXT:    vmovd %xmm0, (%rdi)
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: trunc_ssat_v2i64_v2i16_store:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpmovsqw %xmm0, (%rdi)
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: trunc_ssat_v2i64_v2i16_store:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512BW-NEXT:    vpmovsqw %zmm0, %xmm0
+; AVX512BW-NEXT:    vmovd %xmm0, (%rdi)
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: trunc_ssat_v2i64_v2i16_store:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpmovsqw %xmm0, (%rdi)
+; AVX512BWVL-NEXT:    retq
+;
+; SKX-LABEL: trunc_ssat_v2i64_v2i16_store:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpmovsqw %xmm0, (%rdi)
+; SKX-NEXT:    retq
+  %1 = icmp slt <2 x i64> %a0, <i64 32767, i64 32767>
+  %2 = select <2 x i1> %1, <2 x i64> %a0, <2 x i64> <i64 32767, i64 32767>
+  %3 = icmp sgt <2 x i64> %2, <i64 -32768, i64 -32768>
+  %4 = select <2 x i1> %3, <2 x i64> %2, <2 x i64> <i64 -32768, i64 -32768>
+  %5 = trunc <2 x i64> %4 to <2 x i16>
+  store <2 x i16> %5, <2 x i16> *%p1
+  ret void
+}
+
+define <4 x i16> @trunc_ssat_v4i64_v4i16(<4 x i64> %a0) {
+; SSE2-LABEL: trunc_ssat_v4i64_v4i16:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [32767,32767]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm1, %xmm3
+; SSE2-NEXT:    pxor %xmm2, %xmm3
+; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [2147516415,2147516415]
+; SSE2-NEXT:    movdqa %xmm5, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
+; SSE2-NEXT:    pand %xmm7, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm6[1,1,3,3]
+; SSE2-NEXT:    por %xmm4, %xmm3
+; SSE2-NEXT:    pand %xmm3, %xmm1
+; SSE2-NEXT:    pandn %xmm8, %xmm3
+; SSE2-NEXT:    por %xmm1, %xmm3
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    pxor %xmm2, %xmm1
+; SSE2-NEXT:    movdqa %xmm5, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pand %xmm6, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm1, %xmm4
+; SSE2-NEXT:    pand %xmm4, %xmm0
+; SSE2-NEXT:    pandn %xmm8, %xmm4
+; SSE2-NEXT:    por %xmm0, %xmm4
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [18446744073709518848,18446744073709518848]
+; SSE2-NEXT:    movdqa %xmm4, %xmm1
+; SSE2-NEXT:    pxor %xmm2, %xmm1
+; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [18446744071562035200,18446744071562035200]
+; SSE2-NEXT:    movdqa %xmm1, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm5, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pand %xmm7, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSE2-NEXT:    por %xmm1, %xmm6
+; SSE2-NEXT:    pand %xmm6, %xmm4
+; SSE2-NEXT:    pandn %xmm0, %xmm6
+; SSE2-NEXT:    por %xmm4, %xmm6
+; SSE2-NEXT:    pxor %xmm3, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm5, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE2-NEXT:    pand %xmm4, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    por %xmm2, %xmm1
+; SSE2-NEXT:    pand %xmm1, %xmm3
+; SSE2-NEXT:    pandn %xmm0, %xmm1
+; SSE2-NEXT:    por %xmm3, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm0[0,2,2,3,4,5,6,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,2,2,3]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: trunc_ssat_v4i64_v4i16:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm8 = [32767,32767]
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
+; SSSE3-NEXT:    movdqa %xmm1, %xmm3
+; SSSE3-NEXT:    pxor %xmm2, %xmm3
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm5 = [2147516415,2147516415]
+; SSSE3-NEXT:    movdqa %xmm5, %xmm6
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm7, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm6[1,1,3,3]
+; SSSE3-NEXT:    por %xmm4, %xmm3
+; SSSE3-NEXT:    pand %xmm3, %xmm1
+; SSSE3-NEXT:    pandn %xmm8, %xmm3
+; SSSE3-NEXT:    por %xmm1, %xmm3
+; SSSE3-NEXT:    movdqa %xmm0, %xmm1
+; SSSE3-NEXT:    pxor %xmm2, %xmm1
+; SSSE3-NEXT:    movdqa %xmm5, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm6, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm1, %xmm4
+; SSSE3-NEXT:    pand %xmm4, %xmm0
+; SSSE3-NEXT:    pandn %xmm8, %xmm4
+; SSSE3-NEXT:    por %xmm0, %xmm4
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [18446744073709518848,18446744073709518848]
+; SSSE3-NEXT:    movdqa %xmm4, %xmm1
+; SSSE3-NEXT:    pxor %xmm2, %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm5 = [18446744071562035200,18446744071562035200]
+; SSSE3-NEXT:    movdqa %xmm1, %xmm6
+; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm7, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSSE3-NEXT:    por %xmm1, %xmm6
+; SSSE3-NEXT:    pand %xmm6, %xmm4
+; SSSE3-NEXT:    pandn %xmm0, %xmm6
+; SSSE3-NEXT:    por %xmm4, %xmm6
+; SSSE3-NEXT:    pxor %xmm3, %xmm2
+; SSSE3-NEXT:    movdqa %xmm2, %xmm1
+; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm4, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    por %xmm2, %xmm1
+; SSSE3-NEXT:    pand %xmm1, %xmm3
+; SSSE3-NEXT:    pandn %xmm0, %xmm1
+; SSSE3-NEXT:    por %xmm3, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
+; SSSE3-NEXT:    pshuflw {{.*#+}} xmm1 = xmm0[0,2,2,3,4,5,6,7]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,2,2,3]
+; SSSE3-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: trunc_ssat_v4i64_v4i16:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    movapd {{.*#+}} xmm4 = [32767,32767]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [2147483648,2147483648]
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pxor %xmm3, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm6 = [2147516415,2147516415]
+; SSE41-NEXT:    movdqa %xmm6, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm5
+; SSE41-NEXT:    movdqa %xmm6, %xmm7
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm7
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm0
+; SSE41-NEXT:    por %xmm7, %xmm0
+; SSE41-NEXT:    movapd %xmm4, %xmm5
+; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm5
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    pxor %xmm3, %xmm0
+; SSE41-NEXT:    movdqa %xmm6, %xmm1
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
+; SSE41-NEXT:    pand %xmm1, %xmm0
+; SSE41-NEXT:    por %xmm6, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm4
+; SSE41-NEXT:    movapd {{.*#+}} xmm1 = [18446744073709518848,18446744073709518848]
+; SSE41-NEXT:    movapd %xmm4, %xmm2
+; SSE41-NEXT:    xorpd %xmm3, %xmm2
+; SSE41-NEXT:    movdqa {{.*#+}} xmm6 = [18446744071562035200,18446744071562035200]
+; SSE41-NEXT:    movapd %xmm2, %xmm7
+; SSE41-NEXT:    pcmpeqd %xmm6, %xmm7
+; SSE41-NEXT:    pcmpgtd %xmm6, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,0,2,2]
+; SSE41-NEXT:    pand %xmm7, %xmm0
+; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    movapd %xmm1, %xmm2
+; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm2
+; SSE41-NEXT:    xorpd %xmm5, %xmm3
+; SSE41-NEXT:    movapd %xmm3, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm6, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm6, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm3, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm5, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
+; SSE41-NEXT:    pshuflw {{.*#+}} xmm1 = xmm0[0,2,2,3,4,5,6,7]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
+; SSE41-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; SSE41-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: trunc_ssat_v4i64_v4i16:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [32767,32767]
+; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm2, %xmm3
+; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm4
+; AVX1-NEXT:    vblendvpd %xmm4, %xmm0, %xmm2, %xmm0
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [18446744073709518848,18446744073709518848]
+; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm0, %xmm5
+; AVX1-NEXT:    vblendvpd %xmm3, %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm1, %xmm2
+; AVX1-NEXT:    vblendvpd %xmm2, %xmm1, %xmm4, %xmm1
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
+; AVX1-NEXT:    vblendvpd %xmm5, %xmm0, %xmm4, %xmm0
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2-SLOW-LABEL: trunc_ssat_v4i64_v4i16:
+; AVX2-SLOW:       # %bb.0:
+; AVX2-SLOW-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [32767,32767,32767,32767]
+; AVX2-SLOW-NEXT:    vpcmpgtq %ymm0, %ymm1, %ymm2
+; AVX2-SLOW-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX2-SLOW-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [18446744073709518848,18446744073709518848,18446744073709518848,18446744073709518848]
+; AVX2-SLOW-NEXT:    vpcmpgtq %ymm1, %ymm0, %ymm2
+; AVX2-SLOW-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX2-SLOW-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX2-SLOW-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX2-SLOW-NEXT:    vzeroupper
+; AVX2-SLOW-NEXT:    retq
+;
+; AVX2-FAST-LABEL: trunc_ssat_v4i64_v4i16:
+; AVX2-FAST:       # %bb.0:
+; AVX2-FAST-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [32767,32767,32767,32767]
+; AVX2-FAST-NEXT:    vpcmpgtq %ymm0, %ymm1, %ymm2
+; AVX2-FAST-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX2-FAST-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [18446744073709518848,18446744073709518848,18446744073709518848,18446744073709518848]
+; AVX2-FAST-NEXT:    vpcmpgtq %ymm1, %ymm0, %ymm2
+; AVX2-FAST-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX2-FAST-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,1,8,9,8,9,10,11,8,9,10,11,12,13,14,15]
+; AVX2-FAST-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
+; AVX2-FAST-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
+; AVX2-FAST-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX2-FAST-NEXT:    vzeroupper
+; AVX2-FAST-NEXT:    retq
+;
+; AVX512F-LABEL: trunc_ssat_v4i64_v4i16:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512F-NEXT:    vpmovsqw %zmm0, %xmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: trunc_ssat_v4i64_v4i16:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpmovsqw %ymm0, %xmm0
+; AVX512VL-NEXT:    vzeroupper
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: trunc_ssat_v4i64_v4i16:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512BW-NEXT:    vpmovsqw %zmm0, %xmm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: trunc_ssat_v4i64_v4i16:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpmovsqw %ymm0, %xmm0
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
+;
+; SKX-LABEL: trunc_ssat_v4i64_v4i16:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpmovsqw %ymm0, %xmm0
+; SKX-NEXT:    vzeroupper
+; SKX-NEXT:    retq
+  %1 = icmp slt <4 x i64> %a0, <i64 32767, i64 32767, i64 32767, i64 32767>
+  %2 = select <4 x i1> %1, <4 x i64> %a0, <4 x i64> <i64 32767, i64 32767, i64 32767, i64 32767>
+  %3 = icmp sgt <4 x i64> %2, <i64 -32768, i64 -32768, i64 -32768, i64 -32768>
+  %4 = select <4 x i1> %3, <4 x i64> %2, <4 x i64> <i64 -32768, i64 -32768, i64 -32768, i64 -32768>
+  %5 = trunc <4 x i64> %4 to <4 x i16>
+  ret <4 x i16> %5
+}
+
+define void @trunc_ssat_v4i64_v4i16_store(<4 x i64> %a0, <4 x i16> *%p1) {
+; SSE2-LABEL: trunc_ssat_v4i64_v4i16_store:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [32767,32767]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm1, %xmm3
+; SSE2-NEXT:    pxor %xmm2, %xmm3
+; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [2147516415,2147516415]
+; SSE2-NEXT:    movdqa %xmm5, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
+; SSE2-NEXT:    pand %xmm7, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm6[1,1,3,3]
+; SSE2-NEXT:    por %xmm4, %xmm3
+; SSE2-NEXT:    pand %xmm3, %xmm1
+; SSE2-NEXT:    pandn %xmm8, %xmm3
+; SSE2-NEXT:    por %xmm1, %xmm3
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    pxor %xmm2, %xmm1
+; SSE2-NEXT:    movdqa %xmm5, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pand %xmm6, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm1, %xmm4
+; SSE2-NEXT:    pand %xmm4, %xmm0
+; SSE2-NEXT:    pandn %xmm8, %xmm4
+; SSE2-NEXT:    por %xmm0, %xmm4
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [18446744073709518848,18446744073709518848]
+; SSE2-NEXT:    movdqa %xmm4, %xmm1
+; SSE2-NEXT:    pxor %xmm2, %xmm1
+; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [18446744071562035200,18446744071562035200]
+; SSE2-NEXT:    movdqa %xmm1, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm5, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pand %xmm7, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSE2-NEXT:    por %xmm1, %xmm6
+; SSE2-NEXT:    pand %xmm6, %xmm4
+; SSE2-NEXT:    pandn %xmm0, %xmm6
+; SSE2-NEXT:    por %xmm4, %xmm6
+; SSE2-NEXT:    pxor %xmm3, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm5, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE2-NEXT:    pand %xmm4, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    por %xmm2, %xmm1
+; SSE2-NEXT:    pand %xmm1, %xmm3
+; SSE2-NEXT:    pandn %xmm0, %xmm1
+; SSE2-NEXT:    por %xmm3, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm6[0,2,2,3]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE2-NEXT:    movq %xmm1, (%rdi)
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: trunc_ssat_v4i64_v4i16_store:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm8 = [32767,32767]
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
+; SSSE3-NEXT:    movdqa %xmm1, %xmm3
+; SSSE3-NEXT:    pxor %xmm2, %xmm3
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm5 = [2147516415,2147516415]
+; SSSE3-NEXT:    movdqa %xmm5, %xmm6
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm7, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm6[1,1,3,3]
+; SSSE3-NEXT:    por %xmm4, %xmm3
+; SSSE3-NEXT:    pand %xmm3, %xmm1
+; SSSE3-NEXT:    pandn %xmm8, %xmm3
+; SSSE3-NEXT:    por %xmm1, %xmm3
+; SSSE3-NEXT:    movdqa %xmm0, %xmm1
+; SSSE3-NEXT:    pxor %xmm2, %xmm1
+; SSSE3-NEXT:    movdqa %xmm5, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm6, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm1, %xmm4
+; SSSE3-NEXT:    pand %xmm4, %xmm0
+; SSSE3-NEXT:    pandn %xmm8, %xmm4
+; SSSE3-NEXT:    por %xmm0, %xmm4
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [18446744073709518848,18446744073709518848]
+; SSSE3-NEXT:    movdqa %xmm4, %xmm1
+; SSSE3-NEXT:    pxor %xmm2, %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm5 = [18446744071562035200,18446744071562035200]
+; SSSE3-NEXT:    movdqa %xmm1, %xmm6
+; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm7, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSSE3-NEXT:    por %xmm1, %xmm6
+; SSSE3-NEXT:    pand %xmm6, %xmm4
+; SSSE3-NEXT:    pandn %xmm0, %xmm6
+; SSSE3-NEXT:    por %xmm4, %xmm6
+; SSSE3-NEXT:    pxor %xmm3, %xmm2
+; SSSE3-NEXT:    movdqa %xmm2, %xmm1
+; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm4, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    por %xmm2, %xmm1
+; SSSE3-NEXT:    pand %xmm1, %xmm3
+; SSSE3-NEXT:    pandn %xmm0, %xmm1
+; SSSE3-NEXT:    por %xmm3, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
+; SSSE3-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm6[0,2,2,3]
+; SSSE3-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSSE3-NEXT:    movq %xmm1, (%rdi)
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: trunc_ssat_v4i64_v4i16_store:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    movapd {{.*#+}} xmm4 = [32767,32767]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [2147483648,2147483648]
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pxor %xmm3, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm6 = [2147516415,2147516415]
+; SSE41-NEXT:    movdqa %xmm6, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm5
+; SSE41-NEXT:    movdqa %xmm6, %xmm7
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm7
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm0
+; SSE41-NEXT:    por %xmm7, %xmm0
+; SSE41-NEXT:    movapd %xmm4, %xmm5
+; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm5
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    pxor %xmm3, %xmm0
+; SSE41-NEXT:    movdqa %xmm6, %xmm1
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
+; SSE41-NEXT:    pand %xmm1, %xmm0
+; SSE41-NEXT:    por %xmm6, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm4
+; SSE41-NEXT:    movapd {{.*#+}} xmm1 = [18446744073709518848,18446744073709518848]
+; SSE41-NEXT:    movapd %xmm4, %xmm2
+; SSE41-NEXT:    xorpd %xmm3, %xmm2
+; SSE41-NEXT:    movdqa {{.*#+}} xmm6 = [18446744071562035200,18446744071562035200]
+; SSE41-NEXT:    movapd %xmm2, %xmm7
+; SSE41-NEXT:    pcmpeqd %xmm6, %xmm7
+; SSE41-NEXT:    pcmpgtd %xmm6, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,0,2,2]
+; SSE41-NEXT:    pand %xmm7, %xmm0
+; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    movapd %xmm1, %xmm2
+; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm2
+; SSE41-NEXT:    xorpd %xmm5, %xmm3
+; SSE41-NEXT:    movapd %xmm3, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm6, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm6, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm3, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm5, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
+; SSE41-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
+; SSE41-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
+; SSE41-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE41-NEXT:    movq %xmm1, (%rdi)
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: trunc_ssat_v4i64_v4i16_store:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [32767,32767]
+; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm2, %xmm3
+; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm4
+; AVX1-NEXT:    vblendvpd %xmm4, %xmm0, %xmm2, %xmm0
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [18446744073709518848,18446744073709518848]
+; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm0, %xmm5
+; AVX1-NEXT:    vblendvpd %xmm3, %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm1, %xmm2
+; AVX1-NEXT:    vblendvpd %xmm2, %xmm1, %xmm4, %xmm1
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
+; AVX1-NEXT:    vblendvpd %xmm5, %xmm0, %xmm4, %xmm0
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX1-NEXT:    vmovq %xmm0, (%rdi)
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2-SLOW-LABEL: trunc_ssat_v4i64_v4i16_store:
+; AVX2-SLOW:       # %bb.0:
+; AVX2-SLOW-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [32767,32767,32767,32767]
+; AVX2-SLOW-NEXT:    vpcmpgtq %ymm0, %ymm1, %ymm2
+; AVX2-SLOW-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX2-SLOW-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [18446744073709518848,18446744073709518848,18446744073709518848,18446744073709518848]
+; AVX2-SLOW-NEXT:    vpcmpgtq %ymm1, %ymm0, %ymm2
+; AVX2-SLOW-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX2-SLOW-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX2-SLOW-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX2-SLOW-NEXT:    vmovq %xmm0, (%rdi)
+; AVX2-SLOW-NEXT:    vzeroupper
+; AVX2-SLOW-NEXT:    retq
+;
+; AVX2-FAST-LABEL: trunc_ssat_v4i64_v4i16_store:
+; AVX2-FAST:       # %bb.0:
+; AVX2-FAST-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [32767,32767,32767,32767]
+; AVX2-FAST-NEXT:    vpcmpgtq %ymm0, %ymm1, %ymm2
+; AVX2-FAST-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX2-FAST-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [18446744073709518848,18446744073709518848,18446744073709518848,18446744073709518848]
+; AVX2-FAST-NEXT:    vpcmpgtq %ymm1, %ymm0, %ymm2
+; AVX2-FAST-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX2-FAST-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,1,8,9,8,9,10,11,8,9,10,11,12,13,14,15]
+; AVX2-FAST-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
+; AVX2-FAST-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
+; AVX2-FAST-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX2-FAST-NEXT:    vmovq %xmm0, (%rdi)
+; AVX2-FAST-NEXT:    vzeroupper
+; AVX2-FAST-NEXT:    retq
+;
+; AVX512F-LABEL: trunc_ssat_v4i64_v4i16_store:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512F-NEXT:    vpmovsqw %zmm0, %xmm0
+; AVX512F-NEXT:    vmovq %xmm0, (%rdi)
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: trunc_ssat_v4i64_v4i16_store:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpmovsqw %ymm0, (%rdi)
+; AVX512VL-NEXT:    vzeroupper
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: trunc_ssat_v4i64_v4i16_store:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512BW-NEXT:    vpmovsqw %zmm0, %xmm0
+; AVX512BW-NEXT:    vmovq %xmm0, (%rdi)
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: trunc_ssat_v4i64_v4i16_store:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpmovsqw %ymm0, (%rdi)
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
+;
+; SKX-LABEL: trunc_ssat_v4i64_v4i16_store:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpmovsqw %ymm0, (%rdi)
+; SKX-NEXT:    vzeroupper
+; SKX-NEXT:    retq
+  %1 = icmp slt <4 x i64> %a0, <i64 32767, i64 32767, i64 32767, i64 32767>
+  %2 = select <4 x i1> %1, <4 x i64> %a0, <4 x i64> <i64 32767, i64 32767, i64 32767, i64 32767>
+  %3 = icmp sgt <4 x i64> %2, <i64 -32768, i64 -32768, i64 -32768, i64 -32768>
+  %4 = select <4 x i1> %3, <4 x i64> %2, <4 x i64> <i64 -32768, i64 -32768, i64 -32768, i64 -32768>
+  %5 = trunc <4 x i64> %4 to <4 x i16>
+  store <4 x i16> %5, <4 x i16> *%p1
+  ret void
+}
+
+define <8 x i16> @trunc_ssat_v8i64_v8i16(<8 x i64>* %p0) "min-legal-vector-width"="256" {
+; SSE2-LABEL: trunc_ssat_v8i64_v8i16:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa (%rdi), %xmm6
+; SSE2-NEXT:    movdqa 16(%rdi), %xmm9
+; SSE2-NEXT:    movdqa 32(%rdi), %xmm3
+; SSE2-NEXT:    movdqa 48(%rdi), %xmm5
+; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [32767,32767]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm3, %xmm2
+; SSE2-NEXT:    pxor %xmm1, %xmm2
+; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [2147516415,2147516415]
+; SSE2-NEXT:    movdqa %xmm10, %xmm7
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm7
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,1,3,3]
+; SSE2-NEXT:    pand %xmm0, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm7[1,1,3,3]
+; SSE2-NEXT:    por %xmm4, %xmm2
+; SSE2-NEXT:    pand %xmm2, %xmm3
+; SSE2-NEXT:    pandn %xmm8, %xmm2
+; SSE2-NEXT:    por %xmm3, %xmm2
+; SSE2-NEXT:    movdqa %xmm5, %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm0
+; SSE2-NEXT:    movdqa %xmm10, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pand %xmm4, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSE2-NEXT:    por %xmm0, %xmm3
+; SSE2-NEXT:    pand %xmm3, %xmm5
+; SSE2-NEXT:    pandn %xmm8, %xmm3
+; SSE2-NEXT:    por %xmm5, %xmm3
+; SSE2-NEXT:    movdqa %xmm6, %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm0
+; SSE2-NEXT:    movdqa %xmm10, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pand %xmm5, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm0, %xmm5
+; SSE2-NEXT:    pand %xmm5, %xmm6
+; SSE2-NEXT:    pandn %xmm8, %xmm5
+; SSE2-NEXT:    por %xmm6, %xmm5
+; SSE2-NEXT:    movdqa %xmm9, %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm0
+; SSE2-NEXT:    movdqa %xmm10, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pand %xmm6, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm0, %xmm7
+; SSE2-NEXT:    pand %xmm7, %xmm9
+; SSE2-NEXT:    pandn %xmm8, %xmm7
+; SSE2-NEXT:    por %xmm9, %xmm7
+; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [18446744073709518848,18446744073709518848]
+; SSE2-NEXT:    movdqa %xmm7, %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [18446744071562035200,18446744071562035200]
+; SSE2-NEXT:    movdqa %xmm0, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm9, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm9, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pand %xmm6, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm0, %xmm4
+; SSE2-NEXT:    pand %xmm4, %xmm7
+; SSE2-NEXT:    pandn %xmm8, %xmm4
+; SSE2-NEXT:    por %xmm7, %xmm4
+; SSE2-NEXT:    movdqa %xmm5, %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm9, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm6[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm9, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pand %xmm10, %xmm7
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3]
+; SSE2-NEXT:    por %xmm7, %xmm0
+; SSE2-NEXT:    pand %xmm0, %xmm5
+; SSE2-NEXT:    pandn %xmm8, %xmm0
+; SSE2-NEXT:    por %xmm5, %xmm0
+; SSE2-NEXT:    packssdw %xmm4, %xmm0
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    pxor %xmm1, %xmm4
+; SSE2-NEXT:    movdqa %xmm4, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm9, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm9, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    pand %xmm6, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSE2-NEXT:    por %xmm4, %xmm5
+; SSE2-NEXT:    pand %xmm5, %xmm3
+; SSE2-NEXT:    pandn %xmm8, %xmm5
+; SSE2-NEXT:    por %xmm3, %xmm5
+; SSE2-NEXT:    pxor %xmm2, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm9, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm9, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pand %xmm4, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSE2-NEXT:    por %xmm1, %xmm3
+; SSE2-NEXT:    pand %xmm3, %xmm2
+; SSE2-NEXT:    pandn %xmm8, %xmm3
+; SSE2-NEXT:    por %xmm2, %xmm3
+; SSE2-NEXT:    packssdw %xmm5, %xmm3
+; SSE2-NEXT:    packssdw %xmm3, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: trunc_ssat_v8i64_v8i16:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa (%rdi), %xmm6
+; SSSE3-NEXT:    movdqa 16(%rdi), %xmm9
+; SSSE3-NEXT:    movdqa 32(%rdi), %xmm3
+; SSSE3-NEXT:    movdqa 48(%rdi), %xmm5
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm8 = [32767,32767]
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [2147483648,2147483648]
+; SSSE3-NEXT:    movdqa %xmm3, %xmm2
+; SSSE3-NEXT:    pxor %xmm1, %xmm2
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm10 = [2147516415,2147516415]
+; SSSE3-NEXT:    movdqa %xmm10, %xmm7
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm7
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm0, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm7[1,1,3,3]
+; SSSE3-NEXT:    por %xmm4, %xmm2
+; SSSE3-NEXT:    pand %xmm2, %xmm3
+; SSSE3-NEXT:    pandn %xmm8, %xmm2
+; SSSE3-NEXT:    por %xmm3, %xmm2
+; SSSE3-NEXT:    movdqa %xmm5, %xmm0
+; SSSE3-NEXT:    pxor %xmm1, %xmm0
+; SSSE3-NEXT:    movdqa %xmm10, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm4, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSSE3-NEXT:    por %xmm0, %xmm3
+; SSSE3-NEXT:    pand %xmm3, %xmm5
+; SSSE3-NEXT:    pandn %xmm8, %xmm3
+; SSSE3-NEXT:    por %xmm5, %xmm3
+; SSSE3-NEXT:    movdqa %xmm6, %xmm0
+; SSSE3-NEXT:    pxor %xmm1, %xmm0
+; SSSE3-NEXT:    movdqa %xmm10, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm5, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm0, %xmm5
+; SSSE3-NEXT:    pand %xmm5, %xmm6
+; SSSE3-NEXT:    pandn %xmm8, %xmm5
+; SSSE3-NEXT:    por %xmm6, %xmm5
+; SSSE3-NEXT:    movdqa %xmm9, %xmm0
+; SSSE3-NEXT:    pxor %xmm1, %xmm0
+; SSSE3-NEXT:    movdqa %xmm10, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm6, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm0, %xmm7
+; SSSE3-NEXT:    pand %xmm7, %xmm9
+; SSSE3-NEXT:    pandn %xmm8, %xmm7
+; SSSE3-NEXT:    por %xmm9, %xmm7
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm8 = [18446744073709518848,18446744073709518848]
+; SSSE3-NEXT:    movdqa %xmm7, %xmm0
+; SSSE3-NEXT:    pxor %xmm1, %xmm0
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm9 = [18446744071562035200,18446744071562035200]
+; SSSE3-NEXT:    movdqa %xmm0, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm9, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm6, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm0, %xmm4
+; SSSE3-NEXT:    pand %xmm4, %xmm7
+; SSSE3-NEXT:    pandn %xmm8, %xmm4
+; SSSE3-NEXT:    por %xmm7, %xmm4
+; SSSE3-NEXT:    movdqa %xmm5, %xmm0
+; SSSE3-NEXT:    pxor %xmm1, %xmm0
+; SSSE3-NEXT:    movdqa %xmm0, %xmm6
+; SSSE3-NEXT:    pcmpgtd %xmm9, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm10 = xmm6[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm10, %xmm7
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3]
+; SSSE3-NEXT:    por %xmm7, %xmm0
+; SSSE3-NEXT:    pand %xmm0, %xmm5
+; SSSE3-NEXT:    pandn %xmm8, %xmm0
+; SSSE3-NEXT:    por %xmm5, %xmm0
+; SSSE3-NEXT:    packssdw %xmm4, %xmm0
+; SSSE3-NEXT:    movdqa %xmm3, %xmm4
+; SSSE3-NEXT:    pxor %xmm1, %xmm4
+; SSSE3-NEXT:    movdqa %xmm4, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm9, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm6, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSSE3-NEXT:    por %xmm4, %xmm5
+; SSSE3-NEXT:    pand %xmm5, %xmm3
+; SSSE3-NEXT:    pandn %xmm8, %xmm5
+; SSSE3-NEXT:    por %xmm3, %xmm5
+; SSSE3-NEXT:    pxor %xmm2, %xmm1
+; SSSE3-NEXT:    movdqa %xmm1, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm9, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm4, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSSE3-NEXT:    por %xmm1, %xmm3
+; SSSE3-NEXT:    pand %xmm3, %xmm2
+; SSSE3-NEXT:    pandn %xmm8, %xmm3
+; SSSE3-NEXT:    por %xmm2, %xmm3
+; SSSE3-NEXT:    packssdw %xmm5, %xmm3
+; SSSE3-NEXT:    packssdw %xmm3, %xmm0
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: trunc_ssat_v8i64_v8i16:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa (%rdi), %xmm10
+; SSE41-NEXT:    movdqa 16(%rdi), %xmm9
+; SSE41-NEXT:    movdqa 32(%rdi), %xmm3
+; SSE41-NEXT:    movdqa 48(%rdi), %xmm5
+; SSE41-NEXT:    movapd {{.*#+}} xmm1 = [32767,32767]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    pxor %xmm2, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [2147516415,2147516415]
+; SSE41-NEXT:    movdqa %xmm4, %xmm7
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm7
+; SSE41-NEXT:    movdqa %xmm4, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
+; SSE41-NEXT:    pand %xmm7, %xmm0
+; SSE41-NEXT:    por %xmm6, %xmm0
+; SSE41-NEXT:    movapd %xmm1, %xmm8
+; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm8
+; SSE41-NEXT:    movdqa %xmm5, %xmm0
+; SSE41-NEXT:    pxor %xmm2, %xmm0
+; SSE41-NEXT:    movdqa %xmm4, %xmm3
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm3
+; SSE41-NEXT:    movdqa %xmm4, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
+; SSE41-NEXT:    pand %xmm3, %xmm0
+; SSE41-NEXT:    por %xmm6, %xmm0
+; SSE41-NEXT:    movapd %xmm1, %xmm11
+; SSE41-NEXT:    blendvpd %xmm0, %xmm5, %xmm11
+; SSE41-NEXT:    movdqa %xmm10, %xmm0
+; SSE41-NEXT:    pxor %xmm2, %xmm0
+; SSE41-NEXT:    movdqa %xmm4, %xmm3
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm3
+; SSE41-NEXT:    movdqa %xmm4, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm5
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2]
+; SSE41-NEXT:    pand %xmm3, %xmm0
+; SSE41-NEXT:    por %xmm5, %xmm0
+; SSE41-NEXT:    movapd %xmm1, %xmm3
+; SSE41-NEXT:    blendvpd %xmm0, %xmm10, %xmm3
+; SSE41-NEXT:    movdqa %xmm9, %xmm0
+; SSE41-NEXT:    pxor %xmm2, %xmm0
+; SSE41-NEXT:    movdqa %xmm4, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm0
+; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm9, %xmm1
+; SSE41-NEXT:    movapd {{.*#+}} xmm5 = [18446744073709518848,18446744073709518848]
+; SSE41-NEXT:    movapd %xmm1, %xmm4
+; SSE41-NEXT:    xorpd %xmm2, %xmm4
+; SSE41-NEXT:    movdqa {{.*#+}} xmm6 = [18446744071562035200,18446744071562035200]
+; SSE41-NEXT:    movapd %xmm4, %xmm7
+; SSE41-NEXT:    pcmpeqd %xmm6, %xmm7
+; SSE41-NEXT:    pcmpgtd %xmm6, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pand %xmm7, %xmm0
+; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    movapd %xmm5, %xmm4
+; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm4
+; SSE41-NEXT:    movapd %xmm3, %xmm1
+; SSE41-NEXT:    xorpd %xmm2, %xmm1
+; SSE41-NEXT:    movapd %xmm1, %xmm7
+; SSE41-NEXT:    pcmpeqd %xmm6, %xmm7
+; SSE41-NEXT:    pcmpgtd %xmm6, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2]
+; SSE41-NEXT:    pand %xmm7, %xmm0
+; SSE41-NEXT:    por %xmm1, %xmm0
+; SSE41-NEXT:    movapd %xmm5, %xmm1
+; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm1
+; SSE41-NEXT:    packssdw %xmm4, %xmm1
+; SSE41-NEXT:    movapd %xmm11, %xmm3
+; SSE41-NEXT:    xorpd %xmm2, %xmm3
+; SSE41-NEXT:    movapd %xmm3, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm6, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm6, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm3, %xmm0
+; SSE41-NEXT:    movapd %xmm5, %xmm3
+; SSE41-NEXT:    blendvpd %xmm0, %xmm11, %xmm3
+; SSE41-NEXT:    xorpd %xmm8, %xmm2
+; SSE41-NEXT:    movapd %xmm2, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm6, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm6, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm8, %xmm5
+; SSE41-NEXT:    packssdw %xmm3, %xmm5
+; SSE41-NEXT:    packssdw %xmm5, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: trunc_ssat_v8i64_v8i16:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX1-NEXT:    vmovdqa 16(%rdi), %xmm1
+; AVX1-NEXT:    vmovdqa 32(%rdi), %xmm2
+; AVX1-NEXT:    vmovdqa 48(%rdi), %xmm3
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [32767,32767]
+; AVX1-NEXT:    vpcmpgtq %xmm3, %xmm4, %xmm8
+; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm4, %xmm6
+; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm4, %xmm7
+; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm4, %xmm5
+; AVX1-NEXT:    vblendvpd %xmm5, %xmm0, %xmm4, %xmm0
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm5 = [18446744073709518848,18446744073709518848]
+; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm0, %xmm9
+; AVX1-NEXT:    vblendvpd %xmm7, %xmm1, %xmm4, %xmm1
+; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm1, %xmm7
+; AVX1-NEXT:    vblendvpd %xmm6, %xmm2, %xmm4, %xmm2
+; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm2, %xmm6
+; AVX1-NEXT:    vblendvpd %xmm8, %xmm3, %xmm4, %xmm3
+; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm3, %xmm4
+; AVX1-NEXT:    vblendvpd %xmm4, %xmm3, %xmm5, %xmm3
+; AVX1-NEXT:    vblendvpd %xmm6, %xmm2, %xmm5, %xmm2
+; AVX1-NEXT:    vpackssdw %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vblendvpd %xmm7, %xmm1, %xmm5, %xmm1
+; AVX1-NEXT:    vblendvpd %xmm9, %xmm0, %xmm5, %xmm0
+; AVX1-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpackssdw %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: trunc_ssat_v8i64_v8i16:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX2-NEXT:    vmovdqa 32(%rdi), %ymm1
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [32767,32767,32767,32767]
+; AVX2-NEXT:    vpcmpgtq %ymm0, %ymm2, %ymm3
+; AVX2-NEXT:    vblendvpd %ymm3, %ymm0, %ymm2, %ymm0
+; AVX2-NEXT:    vpcmpgtq %ymm1, %ymm2, %ymm3
+; AVX2-NEXT:    vblendvpd %ymm3, %ymm1, %ymm2, %ymm1
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [18446744073709518848,18446744073709518848,18446744073709518848,18446744073709518848]
+; AVX2-NEXT:    vpcmpgtq %ymm2, %ymm1, %ymm3
+; AVX2-NEXT:    vblendvpd %ymm3, %ymm1, %ymm2, %ymm1
+; AVX2-NEXT:    vpcmpgtq %ymm2, %ymm0, %ymm3
+; AVX2-NEXT:    vblendvpd %ymm3, %ymm0, %ymm2, %ymm0
+; AVX2-NEXT:    vpackssdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: trunc_ssat_v8i64_v8i16:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovdqa64 (%rdi), %zmm0
+; AVX512-NEXT:    vpmovsqw %zmm0, %xmm0
+; AVX512-NEXT:    vzeroupper
+; AVX512-NEXT:    retq
+;
+; SKX-LABEL: trunc_ssat_v8i64_v8i16:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vmovdqa (%rdi), %ymm0
+; SKX-NEXT:    vmovdqa 32(%rdi), %ymm1
+; SKX-NEXT:    vpmovsqw %ymm1, %xmm1
+; SKX-NEXT:    vpmovsqw %ymm0, %xmm0
+; SKX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SKX-NEXT:    vzeroupper
+; SKX-NEXT:    retq
+  %a0 = load <8 x i64>, <8 x i64>* %p0
+  %1 = icmp slt <8 x i64> %a0, <i64 32767, i64 32767, i64 32767, i64 32767, i64 32767, i64 32767, i64 32767, i64 32767>
+  %2 = select <8 x i1> %1, <8 x i64> %a0, <8 x i64> <i64 32767, i64 32767, i64 32767, i64 32767, i64 32767, i64 32767, i64 32767, i64 32767>
+  %3 = icmp sgt <8 x i64> %2, <i64 -32768, i64 -32768, i64 -32768, i64 -32768, i64 -32768, i64 -32768, i64 -32768, i64 -32768>
+  %4 = select <8 x i1> %3, <8 x i64> %2, <8 x i64> <i64 -32768, i64 -32768, i64 -32768, i64 -32768, i64 -32768, i64 -32768, i64 -32768, i64 -32768>
+  %5 = trunc <8 x i64> %4 to <8 x i16>
+  ret <8 x i16> %5
+}
+
+define <4 x i16> @trunc_ssat_v4i32_v4i16(<4 x i32> %a0) {
+; SSE-LABEL: trunc_ssat_v4i32_v4i16:
+; SSE:       # %bb.0:
+; SSE-NEXT:    packssdw %xmm0, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: trunc_ssat_v4i32_v4i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpackssdw %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: trunc_ssat_v4i32_v4i16:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpackssdw %xmm0, %xmm0, %xmm0
+; AVX512-NEXT:    retq
+;
+; SKX-LABEL: trunc_ssat_v4i32_v4i16:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpackssdw %xmm0, %xmm0, %xmm0
+; SKX-NEXT:    retq
+  %1 = icmp slt <4 x i32> %a0, <i32 32767, i32 32767, i32 32767, i32 32767>
+  %2 = select <4 x i1> %1, <4 x i32> %a0, <4 x i32> <i32 32767, i32 32767, i32 32767, i32 32767>
+  %3 = icmp sgt <4 x i32> %2, <i32 -32768, i32 -32768, i32 -32768, i32 -32768>
+  %4 = select <4 x i1> %3, <4 x i32> %2, <4 x i32> <i32 -32768, i32 -32768, i32 -32768, i32 -32768>
+  %5 = trunc <4 x i32> %4 to <4 x i16>
+  ret <4 x i16> %5
+}
+
+define void @trunc_ssat_v4i32_v4i16_store(<4 x i32> %a0, <4 x i16> *%p1) {
+; SSE-LABEL: trunc_ssat_v4i32_v4i16_store:
+; SSE:       # %bb.0:
+; SSE-NEXT:    packssdw %xmm0, %xmm0
+; SSE-NEXT:    movq %xmm0, (%rdi)
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: trunc_ssat_v4i32_v4i16_store:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpackssdw %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vmovq %xmm0, (%rdi)
+; AVX-NEXT:    retq
+;
+; AVX512F-LABEL: trunc_ssat_v4i32_v4i16_store:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpackssdw %xmm0, %xmm0, %xmm0
+; AVX512F-NEXT:    vmovq %xmm0, (%rdi)
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: trunc_ssat_v4i32_v4i16_store:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpmovsdw %xmm0, (%rdi)
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: trunc_ssat_v4i32_v4i16_store:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpackssdw %xmm0, %xmm0, %xmm0
+; AVX512BW-NEXT:    vmovq %xmm0, (%rdi)
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: trunc_ssat_v4i32_v4i16_store:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpmovsdw %xmm0, (%rdi)
+; AVX512BWVL-NEXT:    retq
+;
+; SKX-LABEL: trunc_ssat_v4i32_v4i16_store:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpmovsdw %xmm0, (%rdi)
+; SKX-NEXT:    retq
+  %1 = icmp slt <4 x i32> %a0, <i32 32767, i32 32767, i32 32767, i32 32767>
+  %2 = select <4 x i1> %1, <4 x i32> %a0, <4 x i32> <i32 32767, i32 32767, i32 32767, i32 32767>
+  %3 = icmp sgt <4 x i32> %2, <i32 -32768, i32 -32768, i32 -32768, i32 -32768>
+  %4 = select <4 x i1> %3, <4 x i32> %2, <4 x i32> <i32 -32768, i32 -32768, i32 -32768, i32 -32768>
+  %5 = trunc <4 x i32> %4 to <4 x i16>
+  store <4 x i16> %5, <4 x i16> *%p1
+  ret void
+}
+
+define <8 x i16> @trunc_ssat_v8i32_v8i16(<8 x i32> %a0) {
+; SSE-LABEL: trunc_ssat_v8i32_v8i16:
+; SSE:       # %bb.0:
+; SSE-NEXT:    packssdw %xmm1, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: trunc_ssat_v8i32_v8i16:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: trunc_ssat_v8i32_v8i16:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+;
+; AVX512F-LABEL: trunc_ssat_v8i32_v8i16:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: trunc_ssat_v8i32_v8i16:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpmovsdw %ymm0, %xmm0
+; AVX512VL-NEXT:    vzeroupper
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: trunc_ssat_v8i32_v8i16:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512BW-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: trunc_ssat_v8i32_v8i16:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpmovsdw %ymm0, %xmm0
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
+;
+; SKX-LABEL: trunc_ssat_v8i32_v8i16:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpmovsdw %ymm0, %xmm0
+; SKX-NEXT:    vzeroupper
+; SKX-NEXT:    retq
+  %1 = icmp slt <8 x i32> %a0, <i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767>
+  %2 = select <8 x i1> %1, <8 x i32> %a0, <8 x i32> <i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767>
+  %3 = icmp sgt <8 x i32> %2, <i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768>
+  %4 = select <8 x i1> %3, <8 x i32> %2, <8 x i32> <i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768>
+  %5 = trunc <8 x i32> %4 to <8 x i16>
+  ret <8 x i16> %5
+}
+
+define <16 x i16> @trunc_ssat_v16i32_v16i16(<16 x i32>* %p0) "min-legal-vector-width"="256" {
+; SSE-LABEL: trunc_ssat_v16i32_v16i16:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqa (%rdi), %xmm0
+; SSE-NEXT:    movdqa 32(%rdi), %xmm1
+; SSE-NEXT:    packssdw 16(%rdi), %xmm0
+; SSE-NEXT:    packssdw 48(%rdi), %xmm1
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: trunc_ssat_v16i32_v16i16:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX1-NEXT:    vmovdqa 32(%rdi), %xmm1
+; AVX1-NEXT:    vpackssdw 48(%rdi), %xmm1, %xmm1
+; AVX1-NEXT:    vpackssdw 16(%rdi), %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: trunc_ssat_v16i32_v16i16:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX2-NEXT:    vpackssdw 32(%rdi), %ymm0, %ymm0
+; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: trunc_ssat_v16i32_v16i16:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovdqa64 (%rdi), %zmm0
+; AVX512-NEXT:    vpmovsdw %zmm0, %ymm0
+; AVX512-NEXT:    retq
+;
+; SKX-LABEL: trunc_ssat_v16i32_v16i16:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vmovdqa (%rdi), %ymm0
+; SKX-NEXT:    vpackssdw 32(%rdi), %ymm0, %ymm0
+; SKX-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; SKX-NEXT:    retq
+  %a0 = load <16 x i32>, <16 x i32>* %p0
+  %1 = icmp slt <16 x i32> %a0, <i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767>
+  %2 = select <16 x i1> %1, <16 x i32> %a0, <16 x i32> <i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767>
+  %3 = icmp sgt <16 x i32> %2, <i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768>
+  %4 = select <16 x i1> %3, <16 x i32> %2, <16 x i32> <i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768>
+  %5 = trunc <16 x i32> %4 to <16 x i16>
+  ret <16 x i16> %5
+}
+
+;
+; Signed saturation truncation to vXi8
+;
+
+define <2 x i8> @trunc_ssat_v2i64_v2i8(<2 x i64> %a0) {
+; SSE2-LABEL: trunc_ssat_v2i64_v2i8:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    pxor %xmm1, %xmm2
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [2147483775,2147483775]
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm3, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE2-NEXT:    pand %xmm5, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm2, %xmm3
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm3
+; SSE2-NEXT:    por %xmm0, %xmm3
+; SSE2-NEXT:    pxor %xmm3, %xmm1
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [18446744071562067840,18446744071562067840]
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm0, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pand %xmm4, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3]
+; SSE2-NEXT:    por %xmm1, %xmm0
+; SSE2-NEXT:    pand %xmm0, %xmm3
+; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm0
 ; SSE2-NEXT:    por %xmm3, %xmm0
-; SSE2-NEXT:    packssdw %xmm1, %xmm0
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    packuswb %xmm0, %xmm0
+; SSE2-NEXT:    packuswb %xmm0, %xmm0
+; SSE2-NEXT:    packuswb %xmm0, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: trunc_ssat_v2i64_v2i8:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [2147483648,2147483648]
+; SSSE3-NEXT:    movdqa %xmm0, %xmm2
+; SSSE3-NEXT:    pxor %xmm1, %xmm2
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm3 = [2147483775,2147483775]
+; SSSE3-NEXT:    movdqa %xmm3, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm5, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm2, %xmm3
+; SSSE3-NEXT:    pand %xmm3, %xmm0
+; SSSE3-NEXT:    pandn {{.*}}(%rip), %xmm3
+; SSSE3-NEXT:    por %xmm0, %xmm3
+; SSSE3-NEXT:    pxor %xmm3, %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [18446744071562067840,18446744071562067840]
+; SSSE3-NEXT:    movdqa %xmm1, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm4, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    por %xmm1, %xmm0
+; SSSE3-NEXT:    pand %xmm0, %xmm3
+; SSSE3-NEXT:    pandn {{.*}}(%rip), %xmm0
+; SSSE3-NEXT:    por %xmm3, %xmm0
+; SSSE3-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: trunc_ssat_v2i64_v2i8:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    movapd {{.*#+}} xmm2 = [127,127]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [2147483648,2147483648]
+; SSE41-NEXT:    pxor %xmm3, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [2147483775,2147483775]
+; SSE41-NEXT:    movdqa %xmm4, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm0
+; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
+; SSE41-NEXT:    movapd {{.*#+}} xmm1 = [18446744073709551488,18446744073709551488]
+; SSE41-NEXT:    pxor %xmm2, %xmm3
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [18446744071562067840,18446744071562067840]
+; SSE41-NEXT:    movdqa %xmm3, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm3, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
+; SSE41-NEXT:    pshufb {{.*#+}} xmm1 = xmm1[0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: trunc_ssat_v2i64_v2i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [127,127]
+; AVX-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm2
+; AVX-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [18446744073709551488,18446744073709551488]
+; AVX-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm2
+; AVX-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX-NEXT:    retq
+;
+; AVX512F-LABEL: trunc_ssat_v2i64_v2i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512F-NEXT:    vpmovsqb %zmm0, %xmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: trunc_ssat_v2i64_v2i8:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpmovsqb %xmm0, %xmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: trunc_ssat_v2i64_v2i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512BW-NEXT:    vpmovsqb %zmm0, %xmm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: trunc_ssat_v2i64_v2i8:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpmovsqb %xmm0, %xmm0
+; AVX512BWVL-NEXT:    retq
+;
+; SKX-LABEL: trunc_ssat_v2i64_v2i8:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpmovsqb %xmm0, %xmm0
+; SKX-NEXT:    retq
+  %1 = icmp slt <2 x i64> %a0, <i64 127, i64 127>
+  %2 = select <2 x i1> %1, <2 x i64> %a0, <2 x i64> <i64 127, i64 127>
+  %3 = icmp sgt <2 x i64> %2, <i64 -128, i64 -128>
+  %4 = select <2 x i1> %3, <2 x i64> %2, <2 x i64> <i64 -128, i64 -128>
+  %5 = trunc <2 x i64> %4 to <2 x i8>
+  ret <2 x i8> %5
+}
+
+define void @trunc_ssat_v2i64_v2i8_store(<2 x i64> %a0, <2 x i8> *%p1) {
+; SSE2-LABEL: trunc_ssat_v2i64_v2i8_store:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    pxor %xmm1, %xmm2
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [2147483775,2147483775]
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm3, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE2-NEXT:    pand %xmm5, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm2, %xmm3
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm3
+; SSE2-NEXT:    por %xmm0, %xmm3
+; SSE2-NEXT:    pxor %xmm3, %xmm1
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [18446744071562067840,18446744071562067840]
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm0, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pand %xmm4, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,3,3]
+; SSE2-NEXT:    por %xmm0, %xmm1
+; SSE2-NEXT:    pand %xmm1, %xmm3
+; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm1
+; SSE2-NEXT:    por %xmm3, %xmm1
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE2-NEXT:    packuswb %xmm1, %xmm1
+; SSE2-NEXT:    packuswb %xmm0, %xmm1
+; SSE2-NEXT:    packuswb %xmm0, %xmm1
+; SSE2-NEXT:    movd %xmm1, %eax
+; SSE2-NEXT:    movw %ax, (%rdi)
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: trunc_ssat_v2i64_v2i8_store:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [2147483648,2147483648]
+; SSSE3-NEXT:    movdqa %xmm0, %xmm2
+; SSSE3-NEXT:    pxor %xmm1, %xmm2
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm3 = [2147483775,2147483775]
+; SSSE3-NEXT:    movdqa %xmm3, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm5, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm2, %xmm3
+; SSSE3-NEXT:    pand %xmm3, %xmm0
+; SSSE3-NEXT:    pandn {{.*}}(%rip), %xmm3
+; SSSE3-NEXT:    por %xmm0, %xmm3
+; SSSE3-NEXT:    pxor %xmm3, %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [18446744071562067840,18446744071562067840]
+; SSSE3-NEXT:    movdqa %xmm1, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm4, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    por %xmm0, %xmm1
+; SSSE3-NEXT:    pand %xmm1, %xmm3
+; SSSE3-NEXT:    pandn {{.*}}(%rip), %xmm1
+; SSSE3-NEXT:    por %xmm3, %xmm1
+; SSSE3-NEXT:    pshufb {{.*#+}} xmm1 = xmm1[0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; SSSE3-NEXT:    movd %xmm1, %eax
+; SSSE3-NEXT:    movw %ax, (%rdi)
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: trunc_ssat_v2i64_v2i8_store:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    movapd {{.*#+}} xmm2 = [127,127]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [2147483648,2147483648]
+; SSE41-NEXT:    pxor %xmm3, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [2147483775,2147483775]
+; SSE41-NEXT:    movdqa %xmm4, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm0
+; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
+; SSE41-NEXT:    movapd {{.*#+}} xmm1 = [18446744073709551488,18446744073709551488]
+; SSE41-NEXT:    pxor %xmm2, %xmm3
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [18446744071562067840,18446744071562067840]
+; SSE41-NEXT:    movdqa %xmm3, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm3, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
+; SSE41-NEXT:    pshufb {{.*#+}} xmm1 = xmm1[0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; SSE41-NEXT:    pextrw $0, %xmm1, (%rdi)
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: trunc_ssat_v2i64_v2i8_store:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [127,127]
+; AVX-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm2
+; AVX-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [18446744073709551488,18446744073709551488]
+; AVX-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm2
+; AVX-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX-NEXT:    vpextrw $0, %xmm0, (%rdi)
+; AVX-NEXT:    retq
+;
+; AVX512F-LABEL: trunc_ssat_v2i64_v2i8_store:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512F-NEXT:    vpmovsqb %zmm0, %xmm0
+; AVX512F-NEXT:    vpextrw $0, %xmm0, (%rdi)
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: trunc_ssat_v2i64_v2i8_store:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpmovsqb %xmm0, (%rdi)
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: trunc_ssat_v2i64_v2i8_store:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512BW-NEXT:    vpmovsqb %zmm0, %xmm0
+; AVX512BW-NEXT:    vpextrw $0, %xmm0, (%rdi)
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: trunc_ssat_v2i64_v2i8_store:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpmovsqb %xmm0, (%rdi)
+; AVX512BWVL-NEXT:    retq
+;
+; SKX-LABEL: trunc_ssat_v2i64_v2i8_store:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpmovsqb %xmm0, (%rdi)
+; SKX-NEXT:    retq
+  %1 = icmp slt <2 x i64> %a0, <i64 127, i64 127>
+  %2 = select <2 x i1> %1, <2 x i64> %a0, <2 x i64> <i64 127, i64 127>
+  %3 = icmp sgt <2 x i64> %2, <i64 -128, i64 -128>
+  %4 = select <2 x i1> %3, <2 x i64> %2, <2 x i64> <i64 -128, i64 -128>
+  %5 = trunc <2 x i64> %4 to <2 x i8>
+  store <2 x i8> %5, <2 x i8> *%p1
+  ret void
+}
+
+define <4 x i8> @trunc_ssat_v4i64_v4i8(<4 x i64> %a0) {
+; SSE2-LABEL: trunc_ssat_v4i64_v4i8:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [127,127]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm1, %xmm3
+; SSE2-NEXT:    pxor %xmm2, %xmm3
+; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [2147483775,2147483775]
+; SSE2-NEXT:    movdqa %xmm5, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
+; SSE2-NEXT:    pand %xmm7, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm6[1,1,3,3]
+; SSE2-NEXT:    por %xmm4, %xmm3
+; SSE2-NEXT:    pand %xmm3, %xmm1
+; SSE2-NEXT:    pandn %xmm8, %xmm3
+; SSE2-NEXT:    por %xmm1, %xmm3
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    pxor %xmm2, %xmm1
+; SSE2-NEXT:    movdqa %xmm5, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pand %xmm6, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm1, %xmm4
+; SSE2-NEXT:    pand %xmm4, %xmm0
+; SSE2-NEXT:    pandn %xmm8, %xmm4
+; SSE2-NEXT:    por %xmm0, %xmm4
+; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [18446744073709551488,18446744073709551488]
+; SSE2-NEXT:    movdqa %xmm4, %xmm0
+; SSE2-NEXT:    pxor %xmm2, %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [18446744071562067840,18446744071562067840]
+; SSE2-NEXT:    movdqa %xmm0, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm5, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pand %xmm7, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3]
+; SSE2-NEXT:    por %xmm1, %xmm0
+; SSE2-NEXT:    pand %xmm0, %xmm4
+; SSE2-NEXT:    pandn %xmm8, %xmm0
+; SSE2-NEXT:    por %xmm4, %xmm0
+; SSE2-NEXT:    pxor %xmm3, %xmm2
 ; SSE2-NEXT:    movdqa %xmm2, %xmm1
-; SSE2-NEXT:    pxor %xmm4, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm5, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE2-NEXT:    pand %xmm4, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    por %xmm2, %xmm1
+; SSE2-NEXT:    pand %xmm1, %xmm3
+; SSE2-NEXT:    pandn %xmm8, %xmm1
+; SSE2-NEXT:    por %xmm3, %xmm1
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0]
+; SSE2-NEXT:    pand %xmm2, %xmm1
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    packuswb %xmm1, %xmm0
+; SSE2-NEXT:    packuswb %xmm0, %xmm0
+; SSE2-NEXT:    packuswb %xmm0, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: trunc_ssat_v4i64_v4i8:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm8 = [127,127]
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
+; SSSE3-NEXT:    movdqa %xmm1, %xmm3
+; SSSE3-NEXT:    pxor %xmm2, %xmm3
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm5 = [2147483775,2147483775]
+; SSSE3-NEXT:    movdqa %xmm5, %xmm6
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm7, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm6[1,1,3,3]
+; SSSE3-NEXT:    por %xmm4, %xmm3
+; SSSE3-NEXT:    pand %xmm3, %xmm1
+; SSSE3-NEXT:    pandn %xmm8, %xmm3
+; SSSE3-NEXT:    por %xmm1, %xmm3
+; SSSE3-NEXT:    movdqa %xmm0, %xmm1
+; SSSE3-NEXT:    pxor %xmm2, %xmm1
+; SSSE3-NEXT:    movdqa %xmm5, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm6, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm1, %xmm4
+; SSSE3-NEXT:    pand %xmm4, %xmm0
+; SSSE3-NEXT:    pandn %xmm8, %xmm4
+; SSSE3-NEXT:    por %xmm0, %xmm4
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm8 = [18446744073709551488,18446744073709551488]
+; SSSE3-NEXT:    movdqa %xmm4, %xmm0
+; SSSE3-NEXT:    pxor %xmm2, %xmm0
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm5 = [18446744071562067840,18446744071562067840]
+; SSSE3-NEXT:    movdqa %xmm0, %xmm6
+; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm7, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3]
+; SSSE3-NEXT:    por %xmm1, %xmm0
+; SSSE3-NEXT:    pand %xmm0, %xmm4
+; SSSE3-NEXT:    pandn %xmm8, %xmm0
+; SSSE3-NEXT:    por %xmm4, %xmm0
+; SSSE3-NEXT:    pxor %xmm3, %xmm2
+; SSSE3-NEXT:    movdqa %xmm2, %xmm1
+; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm4, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    por %xmm2, %xmm1
+; SSSE3-NEXT:    pand %xmm1, %xmm3
+; SSSE3-NEXT:    pandn %xmm8, %xmm1
+; SSSE3-NEXT:    por %xmm3, %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = <0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; SSSE3-NEXT:    pshufb %xmm2, %xmm1
+; SSSE3-NEXT:    pshufb %xmm2, %xmm0
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: trunc_ssat_v4i64_v4i8:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    movapd {{.*#+}} xmm4 = [127,127]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [2147483648,2147483648]
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pxor %xmm3, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm6 = [2147483775,2147483775]
+; SSE41-NEXT:    movdqa %xmm6, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm5
+; SSE41-NEXT:    movdqa %xmm6, %xmm7
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm7
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm0
+; SSE41-NEXT:    por %xmm7, %xmm0
+; SSE41-NEXT:    movapd %xmm4, %xmm5
+; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm5
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    pxor %xmm3, %xmm0
+; SSE41-NEXT:    movdqa %xmm6, %xmm1
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
+; SSE41-NEXT:    pand %xmm1, %xmm0
+; SSE41-NEXT:    por %xmm6, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm4
+; SSE41-NEXT:    movapd {{.*#+}} xmm2 = [18446744073709551488,18446744073709551488]
+; SSE41-NEXT:    movapd %xmm4, %xmm1
+; SSE41-NEXT:    xorpd %xmm3, %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm6 = [18446744071562067840,18446744071562067840]
+; SSE41-NEXT:    movapd %xmm1, %xmm7
+; SSE41-NEXT:    pcmpeqd %xmm6, %xmm7
+; SSE41-NEXT:    pcmpgtd %xmm6, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2]
+; SSE41-NEXT:    pand %xmm7, %xmm0
+; SSE41-NEXT:    por %xmm1, %xmm0
+; SSE41-NEXT:    movapd %xmm2, %xmm1
+; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm1
+; SSE41-NEXT:    xorpd %xmm5, %xmm3
+; SSE41-NEXT:    movapd %xmm3, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm6, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm6, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm3, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm5, %xmm2
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = <0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; SSE41-NEXT:    pshufb %xmm0, %xmm2
+; SSE41-NEXT:    pshufb %xmm0, %xmm1
+; SSE41-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: trunc_ssat_v4i64_v4i8:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [127,127]
+; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm2, %xmm3
+; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm4
+; AVX1-NEXT:    vblendvpd %xmm4, %xmm0, %xmm2, %xmm0
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [18446744073709551488,18446744073709551488]
+; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm0, %xmm5
+; AVX1-NEXT:    vblendvpd %xmm3, %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm1, %xmm2
+; AVX1-NEXT:    vblendvpd %xmm2, %xmm1, %xmm4, %xmm1
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = <0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX1-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vblendvpd %xmm5, %xmm0, %xmm4, %xmm0
+; AVX1-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: trunc_ssat_v4i64_v4i8:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [127,127,127,127]
+; AVX2-NEXT:    vpcmpgtq %ymm0, %ymm1, %ymm2
+; AVX2-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [18446744073709551488,18446744073709551488,18446744073709551488,18446744073709551488]
+; AVX2-NEXT:    vpcmpgtq %ymm1, %ymm0, %ymm2
+; AVX2-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = <0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX2-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+;
+; AVX512F-LABEL: trunc_ssat_v4i64_v4i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512F-NEXT:    vpmovsqb %zmm0, %xmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: trunc_ssat_v4i64_v4i8:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpmovsqb %ymm0, %xmm0
+; AVX512VL-NEXT:    vzeroupper
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: trunc_ssat_v4i64_v4i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512BW-NEXT:    vpmovsqb %zmm0, %xmm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: trunc_ssat_v4i64_v4i8:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpmovsqb %ymm0, %xmm0
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
+;
+; SKX-LABEL: trunc_ssat_v4i64_v4i8:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpmovsqb %ymm0, %xmm0
+; SKX-NEXT:    vzeroupper
+; SKX-NEXT:    retq
+  %1 = icmp slt <4 x i64> %a0, <i64 127, i64 127, i64 127, i64 127>
+  %2 = select <4 x i1> %1, <4 x i64> %a0, <4 x i64> <i64 127, i64 127, i64 127, i64 127>
+  %3 = icmp sgt <4 x i64> %2, <i64 -128, i64 -128, i64 -128, i64 -128>
+  %4 = select <4 x i1> %3, <4 x i64> %2, <4 x i64> <i64 -128, i64 -128, i64 -128, i64 -128>
+  %5 = trunc <4 x i64> %4 to <4 x i8>
+  ret <4 x i8> %5
+}
+
+define void @trunc_ssat_v4i64_v4i8_store(<4 x i64> %a0, <4 x i8> *%p1) {
+; SSE2-LABEL: trunc_ssat_v4i64_v4i8_store:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [127,127]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
 ; SSE2-NEXT:    movdqa %xmm1, %xmm3
-; SSE2-NEXT:    pcmpgtd %xmm9, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm1
+; SSE2-NEXT:    pxor %xmm2, %xmm3
+; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [2147483775,2147483775]
+; SSE2-NEXT:    movdqa %xmm5, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
+; SSE2-NEXT:    pand %xmm7, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm6[1,1,3,3]
+; SSE2-NEXT:    por %xmm4, %xmm3
+; SSE2-NEXT:    pand %xmm3, %xmm1
+; SSE2-NEXT:    pandn %xmm8, %xmm3
+; SSE2-NEXT:    por %xmm1, %xmm3
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    pxor %xmm2, %xmm1
+; SSE2-NEXT:    movdqa %xmm5, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm6, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm3
-; SSE2-NEXT:    pand %xmm3, %xmm2
-; SSE2-NEXT:    pandn %xmm8, %xmm3
-; SSE2-NEXT:    por %xmm2, %xmm3
-; SSE2-NEXT:    pxor %xmm5, %xmm4
-; SSE2-NEXT:    movdqa %xmm4, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm9, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm4
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    pand %xmm2, %xmm4
+; SSE2-NEXT:    por %xmm1, %xmm4
+; SSE2-NEXT:    pand %xmm4, %xmm0
+; SSE2-NEXT:    pandn %xmm8, %xmm4
+; SSE2-NEXT:    por %xmm0, %xmm4
+; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [18446744073709551488,18446744073709551488]
+; SSE2-NEXT:    movdqa %xmm4, %xmm0
+; SSE2-NEXT:    pxor %xmm2, %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [18446744071562067840,18446744071562067840]
+; SSE2-NEXT:    movdqa %xmm0, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm5, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pand %xmm7, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3]
+; SSE2-NEXT:    por %xmm1, %xmm0
+; SSE2-NEXT:    pand %xmm0, %xmm4
+; SSE2-NEXT:    pandn %xmm8, %xmm0
+; SSE2-NEXT:    por %xmm4, %xmm0
+; SSE2-NEXT:    pxor %xmm3, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm5, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE2-NEXT:    pand %xmm4, %xmm2
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm4, %xmm1
-; SSE2-NEXT:    pand %xmm1, %xmm5
+; SSE2-NEXT:    por %xmm2, %xmm1
+; SSE2-NEXT:    pand %xmm1, %xmm3
 ; SSE2-NEXT:    pandn %xmm8, %xmm1
-; SSE2-NEXT:    por %xmm5, %xmm1
-; SSE2-NEXT:    packssdw %xmm3, %xmm1
-; SSE2-NEXT:    packssdw %xmm1, %xmm0
+; SSE2-NEXT:    por %xmm3, %xmm1
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0]
+; SSE2-NEXT:    pand %xmm2, %xmm1
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    packuswb %xmm1, %xmm0
+; SSE2-NEXT:    packuswb %xmm0, %xmm0
+; SSE2-NEXT:    packuswb %xmm0, %xmm0
+; SSE2-NEXT:    movd %xmm0, (%rdi)
 ; SSE2-NEXT:    retq
 ;
-; SSSE3-LABEL: trunc_ssat_v8i64_v8i16:
+; SSSE3-LABEL: trunc_ssat_v4i64_v4i8_store:
 ; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm8 = [32767,32767]
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm4 = [2147483648,2147483648]
-; SSSE3-NEXT:    movdqa %xmm2, %xmm5
-; SSSE3-NEXT:    pxor %xmm4, %xmm5
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm9 = [2147516415,2147516415]
-; SSSE3-NEXT:    movdqa %xmm9, %xmm7
-; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm7
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm10 = xmm7[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm5
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm10, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm7[1,1,3,3]
-; SSSE3-NEXT:    por %xmm6, %xmm5
-; SSSE3-NEXT:    pand %xmm5, %xmm2
-; SSSE3-NEXT:    pandn %xmm8, %xmm5
-; SSSE3-NEXT:    por %xmm2, %xmm5
-; SSSE3-NEXT:    movdqa %xmm3, %xmm2
-; SSSE3-NEXT:    pxor %xmm4, %xmm2
-; SSSE3-NEXT:    movdqa %xmm9, %xmm6
-; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm10 = xmm6[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm2[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm10, %xmm7
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[1,1,3,3]
-; SSSE3-NEXT:    por %xmm7, %xmm2
-; SSSE3-NEXT:    pand %xmm2, %xmm3
-; SSSE3-NEXT:    pandn %xmm8, %xmm2
-; SSSE3-NEXT:    por %xmm3, %xmm2
-; SSSE3-NEXT:    movdqa %xmm0, %xmm3
-; SSSE3-NEXT:    pxor %xmm4, %xmm3
-; SSSE3-NEXT:    movdqa %xmm9, %xmm6
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm8 = [127,127]
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
+; SSSE3-NEXT:    movdqa %xmm1, %xmm3
+; SSSE3-NEXT:    pxor %xmm2, %xmm3
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm5 = [2147483775,2147483775]
+; SSSE3-NEXT:    movdqa %xmm5, %xmm6
 ; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm10 = xmm6[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm10, %xmm7
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm7, %xmm4
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm6[1,1,3,3]
-; SSSE3-NEXT:    por %xmm7, %xmm3
-; SSSE3-NEXT:    pand %xmm3, %xmm0
+; SSSE3-NEXT:    por %xmm4, %xmm3
+; SSSE3-NEXT:    pand %xmm3, %xmm1
 ; SSSE3-NEXT:    pandn %xmm8, %xmm3
-; SSSE3-NEXT:    por %xmm0, %xmm3
-; SSSE3-NEXT:    movdqa %xmm1, %xmm0
-; SSSE3-NEXT:    pxor %xmm4, %xmm0
-; SSSE3-NEXT:    movdqa %xmm9, %xmm6
-; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm7, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[1,1,3,3]
-; SSSE3-NEXT:    por %xmm0, %xmm7
-; SSSE3-NEXT:    pand %xmm7, %xmm1
-; SSSE3-NEXT:    pandn %xmm8, %xmm7
-; SSSE3-NEXT:    por %xmm1, %xmm7
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm8 = [18446744073709518848,18446744073709518848]
-; SSSE3-NEXT:    movdqa %xmm7, %xmm0
-; SSSE3-NEXT:    pxor %xmm4, %xmm0
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm9 = [18446744071562035200,18446744071562035200]
+; SSSE3-NEXT:    por %xmm1, %xmm3
 ; SSSE3-NEXT:    movdqa %xmm0, %xmm1
-; SSSE3-NEXT:    pcmpgtd %xmm9, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm6, %xmm0
+; SSSE3-NEXT:    pxor %xmm2, %xmm1
+; SSSE3-NEXT:    movdqa %xmm5, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm1
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm6, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm1, %xmm4
+; SSSE3-NEXT:    pand %xmm4, %xmm0
+; SSSE3-NEXT:    pandn %xmm8, %xmm4
+; SSSE3-NEXT:    por %xmm0, %xmm4
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm8 = [18446744073709551488,18446744073709551488]
+; SSSE3-NEXT:    movdqa %xmm4, %xmm1
+; SSSE3-NEXT:    pxor %xmm2, %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm5 = [18446744071562067840,18446744071562067840]
+; SSSE3-NEXT:    movdqa %xmm1, %xmm6
+; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm7, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm6[1,1,3,3]
 ; SSSE3-NEXT:    por %xmm0, %xmm1
-; SSSE3-NEXT:    pand %xmm1, %xmm7
+; SSSE3-NEXT:    pand %xmm1, %xmm4
 ; SSSE3-NEXT:    pandn %xmm8, %xmm1
-; SSSE3-NEXT:    por %xmm7, %xmm1
-; SSSE3-NEXT:    movdqa %xmm3, %xmm0
-; SSSE3-NEXT:    pxor %xmm4, %xmm0
-; SSSE3-NEXT:    movdqa %xmm0, %xmm6
-; SSSE3-NEXT:    pcmpgtd %xmm9, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm10 = xmm6[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm10, %xmm7
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3]
-; SSSE3-NEXT:    por %xmm7, %xmm0
+; SSSE3-NEXT:    por %xmm4, %xmm1
+; SSSE3-NEXT:    pxor %xmm3, %xmm2
+; SSSE3-NEXT:    movdqa %xmm2, %xmm0
+; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm4, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    por %xmm2, %xmm0
 ; SSSE3-NEXT:    pand %xmm0, %xmm3
 ; SSSE3-NEXT:    pandn %xmm8, %xmm0
 ; SSSE3-NEXT:    por %xmm3, %xmm0
-; SSSE3-NEXT:    packssdw %xmm1, %xmm0
-; SSSE3-NEXT:    movdqa %xmm2, %xmm1
-; SSSE3-NEXT:    pxor %xmm4, %xmm1
-; SSSE3-NEXT:    movdqa %xmm1, %xmm3
-; SSSE3-NEXT:    pcmpgtd %xmm9, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm6, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm1, %xmm3
-; SSSE3-NEXT:    pand %xmm3, %xmm2
-; SSSE3-NEXT:    pandn %xmm8, %xmm3
-; SSSE3-NEXT:    por %xmm2, %xmm3
-; SSSE3-NEXT:    pxor %xmm5, %xmm4
-; SSSE3-NEXT:    movdqa %xmm4, %xmm1
-; SSSE3-NEXT:    pcmpgtd %xmm9, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm2, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    por %xmm4, %xmm1
-; SSSE3-NEXT:    pand %xmm1, %xmm5
-; SSSE3-NEXT:    pandn %xmm8, %xmm1
-; SSSE3-NEXT:    por %xmm5, %xmm1
-; SSSE3-NEXT:    packssdw %xmm3, %xmm1
-; SSSE3-NEXT:    packssdw %xmm1, %xmm0
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = <0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; SSSE3-NEXT:    pshufb %xmm2, %xmm0
+; SSSE3-NEXT:    pshufb %xmm2, %xmm1
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; SSSE3-NEXT:    movd %xmm1, (%rdi)
 ; SSSE3-NEXT:    retq
 ;
-; SSE41-LABEL: trunc_ssat_v8i64_v8i16:
+; SSE41-LABEL: trunc_ssat_v4i64_v4i8_store:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa %xmm0, %xmm10
-; SSE41-NEXT:    movapd {{.*#+}} xmm7 = [32767,32767]
-; SSE41-NEXT:    movdqa {{.*#+}} xmm5 = [2147483648,2147483648]
+; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    movapd {{.*#+}} xmm4 = [127,127]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [2147483648,2147483648]
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pxor %xmm3, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm6 = [2147483775,2147483775]
+; SSE41-NEXT:    movdqa %xmm6, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm5
+; SSE41-NEXT:    movdqa %xmm6, %xmm7
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm7
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm0
+; SSE41-NEXT:    por %xmm7, %xmm0
+; SSE41-NEXT:    movapd %xmm4, %xmm5
+; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm5
 ; SSE41-NEXT:    movdqa %xmm2, %xmm0
-; SSE41-NEXT:    pxor %xmm5, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm11 = [2147516415,2147516415]
-; SSE41-NEXT:    movdqa %xmm11, %xmm6
+; SSE41-NEXT:    pxor %xmm3, %xmm0
+; SSE41-NEXT:    movdqa %xmm6, %xmm1
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
 ; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm11, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
+; SSE41-NEXT:    pand %xmm1, %xmm0
 ; SSE41-NEXT:    por %xmm6, %xmm0
-; SSE41-NEXT:    movapd %xmm7, %xmm8
-; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm8
-; SSE41-NEXT:    movdqa %xmm3, %xmm0
-; SSE41-NEXT:    pxor %xmm5, %xmm0
-; SSE41-NEXT:    movdqa %xmm11, %xmm2
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm11, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm4, %xmm0
-; SSE41-NEXT:    por %xmm2, %xmm0
-; SSE41-NEXT:    movapd %xmm7, %xmm9
-; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm9
-; SSE41-NEXT:    movdqa %xmm10, %xmm0
-; SSE41-NEXT:    pxor %xmm5, %xmm0
-; SSE41-NEXT:    movdqa %xmm11, %xmm2
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm11, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm3, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm4
+; SSE41-NEXT:    movapd {{.*#+}} xmm1 = [18446744073709551488,18446744073709551488]
+; SSE41-NEXT:    movapd %xmm4, %xmm2
+; SSE41-NEXT:    xorpd %xmm3, %xmm2
+; SSE41-NEXT:    movdqa {{.*#+}} xmm6 = [18446744071562067840,18446744071562067840]
+; SSE41-NEXT:    movapd %xmm2, %xmm7
+; SSE41-NEXT:    pcmpeqd %xmm6, %xmm7
+; SSE41-NEXT:    pcmpgtd %xmm6, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,0,2,2]
+; SSE41-NEXT:    pand %xmm7, %xmm0
 ; SSE41-NEXT:    por %xmm2, %xmm0
-; SSE41-NEXT:    movapd %xmm7, %xmm2
-; SSE41-NEXT:    blendvpd %xmm0, %xmm10, %xmm2
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    pxor %xmm5, %xmm0
-; SSE41-NEXT:    movdqa %xmm11, %xmm3
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm11, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE41-NEXT:    movapd %xmm1, %xmm2
+; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm2
+; SSE41-NEXT:    xorpd %xmm5, %xmm3
+; SSE41-NEXT:    movapd %xmm3, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm6, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm6, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
 ; SSE41-NEXT:    pand %xmm4, %xmm0
 ; SSE41-NEXT:    por %xmm3, %xmm0
-; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm7
-; SSE41-NEXT:    movapd {{.*#+}} xmm3 = [18446744073709518848,18446744073709518848]
-; SSE41-NEXT:    movapd %xmm7, %xmm0
-; SSE41-NEXT:    xorpd %xmm5, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [18446744071562035200,18446744071562035200]
-; SSE41-NEXT:    movapd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd %xmm4, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm4, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm6, %xmm0
-; SSE41-NEXT:    por %xmm1, %xmm0
-; SSE41-NEXT:    movapd %xmm3, %xmm6
-; SSE41-NEXT:    blendvpd %xmm0, %xmm7, %xmm6
-; SSE41-NEXT:    movapd %xmm2, %xmm0
-; SSE41-NEXT:    xorpd %xmm5, %xmm0
-; SSE41-NEXT:    movapd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd %xmm4, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm1[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm4, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm7, %xmm0
-; SSE41-NEXT:    por %xmm1, %xmm0
-; SSE41-NEXT:    movapd %xmm3, %xmm1
-; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
-; SSE41-NEXT:    packssdw %xmm6, %xmm1
-; SSE41-NEXT:    movapd %xmm9, %xmm0
-; SSE41-NEXT:    xorpd %xmm5, %xmm0
-; SSE41-NEXT:    movapd %xmm0, %xmm2
-; SSE41-NEXT:    pcmpgtd %xmm4, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm4, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm6, %xmm0
-; SSE41-NEXT:    por %xmm2, %xmm0
-; SSE41-NEXT:    movapd %xmm3, %xmm2
-; SSE41-NEXT:    blendvpd %xmm0, %xmm9, %xmm2
-; SSE41-NEXT:    xorpd %xmm8, %xmm5
-; SSE41-NEXT:    movapd %xmm5, %xmm6
-; SSE41-NEXT:    pcmpgtd %xmm4, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm4, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[1,1,3,3]
-; SSE41-NEXT:    pand %xmm7, %xmm0
-; SSE41-NEXT:    por %xmm6, %xmm0
-; SSE41-NEXT:    blendvpd %xmm0, %xmm8, %xmm3
-; SSE41-NEXT:    packssdw %xmm2, %xmm3
-; SSE41-NEXT:    packssdw %xmm3, %xmm1
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm5, %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = <0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; SSE41-NEXT:    pshufb %xmm0, %xmm1
+; SSE41-NEXT:    pshufb %xmm0, %xmm2
+; SSE41-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; SSE41-NEXT:    movd %xmm2, (%rdi)
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: trunc_ssat_v8i64_v8i16:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [32767,32767]
-; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm3, %xmm8
-; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm3, %xmm5
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm6
-; AVX1-NEXT:    vpcmpgtq %xmm6, %xmm3, %xmm7
-; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm3, %xmm4
-; AVX1-NEXT:    vblendvpd %xmm4, %xmm0, %xmm3, %xmm0
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [18446744073709518848,18446744073709518848]
-; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm0, %xmm9
-; AVX1-NEXT:    vblendvpd %xmm7, %xmm6, %xmm3, %xmm6
-; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm6, %xmm7
-; AVX1-NEXT:    vblendvpd %xmm5, %xmm1, %xmm3, %xmm1
-; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm1, %xmm5
-; AVX1-NEXT:    vblendvpd %xmm8, %xmm2, %xmm3, %xmm2
-; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm2, %xmm3
-; AVX1-NEXT:    vblendvpd %xmm3, %xmm2, %xmm4, %xmm2
-; AVX1-NEXT:    vblendvpd %xmm5, %xmm1, %xmm4, %xmm1
-; AVX1-NEXT:    vpackssdw %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vblendvpd %xmm7, %xmm6, %xmm4, %xmm2
-; AVX1-NEXT:    vblendvpd %xmm9, %xmm0, %xmm4, %xmm0
-; AVX1-NEXT:    vpackssdw %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vzeroupper
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: trunc_ssat_v8i64_v8i16:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [32767,32767,32767,32767]
-; AVX2-NEXT:    vpcmpgtq %ymm0, %ymm2, %ymm3
-; AVX2-NEXT:    vblendvpd %ymm3, %ymm0, %ymm2, %ymm0
-; AVX2-NEXT:    vpcmpgtq %ymm1, %ymm2, %ymm3
-; AVX2-NEXT:    vblendvpd %ymm3, %ymm1, %ymm2, %ymm1
-; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [18446744073709518848,18446744073709518848,18446744073709518848,18446744073709518848]
-; AVX2-NEXT:    vpcmpgtq %ymm2, %ymm1, %ymm3
-; AVX2-NEXT:    vblendvpd %ymm3, %ymm1, %ymm2, %ymm1
-; AVX2-NEXT:    vpcmpgtq %ymm2, %ymm0, %ymm3
-; AVX2-NEXT:    vblendvpd %ymm3, %ymm0, %ymm2, %ymm0
-; AVX2-NEXT:    vpackssdw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vzeroupper
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: trunc_ssat_v8i64_v8i16:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpmovsqw %zmm0, %xmm0
-; AVX512-NEXT:    vzeroupper
-; AVX512-NEXT:    retq
-  %1 = icmp slt <8 x i64> %a0, <i64 32767, i64 32767, i64 32767, i64 32767, i64 32767, i64 32767, i64 32767, i64 32767>
-  %2 = select <8 x i1> %1, <8 x i64> %a0, <8 x i64> <i64 32767, i64 32767, i64 32767, i64 32767, i64 32767, i64 32767, i64 32767, i64 32767>
-  %3 = icmp sgt <8 x i64> %2, <i64 -32768, i64 -32768, i64 -32768, i64 -32768, i64 -32768, i64 -32768, i64 -32768, i64 -32768>
-  %4 = select <8 x i1> %3, <8 x i64> %2, <8 x i64> <i64 -32768, i64 -32768, i64 -32768, i64 -32768, i64 -32768, i64 -32768, i64 -32768, i64 -32768>
-  %5 = trunc <8 x i64> %4 to <8 x i16>
-  ret <8 x i16> %5
-}
-
-define <8 x i16> @trunc_ssat_v8i32_v8i16(<8 x i32> %a0) {
-; SSE-LABEL: trunc_ssat_v8i32_v8i16:
-; SSE:       # %bb.0:
-; SSE-NEXT:    packssdw %xmm1, %xmm0
-; SSE-NEXT:    retq
-;
-; AVX1-LABEL: trunc_ssat_v8i32_v8i16:
+; AVX1-LABEL: trunc_ssat_v4i64_v4i8_store:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [127,127]
+; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm2, %xmm3
+; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm4
+; AVX1-NEXT:    vblendvpd %xmm4, %xmm0, %xmm2, %xmm0
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [18446744073709551488,18446744073709551488]
+; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm0, %xmm5
+; AVX1-NEXT:    vblendvpd %xmm3, %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm1, %xmm2
+; AVX1-NEXT:    vblendvpd %xmm2, %xmm1, %xmm4, %xmm1
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = <0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX1-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vblendvpd %xmm5, %xmm0, %xmm4, %xmm0
+; AVX1-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX1-NEXT:    vmovd %xmm0, (%rdi)
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
-; AVX2-LABEL: trunc_ssat_v8i32_v8i16:
+; AVX2-LABEL: trunc_ssat_v4i64_v4i8_store:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [127,127,127,127]
+; AVX2-NEXT:    vpcmpgtq %ymm0, %ymm1, %ymm2
+; AVX2-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [18446744073709551488,18446744073709551488,18446744073709551488,18446744073709551488]
+; AVX2-NEXT:    vpcmpgtq %ymm1, %ymm0, %ymm2
+; AVX2-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = <0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX2-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX2-NEXT:    vmovd %xmm0, (%rdi)
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
-; AVX512F-LABEL: trunc_ssat_v8i32_v8i16:
+; AVX512F-LABEL: trunc_ssat_v4i64_v4i8_store:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [32767,32767,32767,32767,32767,32767,32767,32767]
-; AVX512F-NEXT:    vpminsd %ymm1, %ymm0, %ymm0
-; AVX512F-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [4294934528,4294934528,4294934528,4294934528,4294934528,4294934528,4294934528,4294934528]
-; AVX512F-NEXT:    vpmaxsd %ymm1, %ymm0, %ymm0
-; AVX512F-NEXT:    vpmovdw %zmm0, %ymm0
-; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512F-NEXT:    vpmovsqb %zmm0, %xmm0
+; AVX512F-NEXT:    vmovd %xmm0, (%rdi)
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
-; AVX512VL-LABEL: trunc_ssat_v8i32_v8i16:
+; AVX512VL-LABEL: trunc_ssat_v4i64_v4i8_store:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpmovsdw %ymm0, %xmm0
+; AVX512VL-NEXT:    vpmovsqb %ymm0, (%rdi)
 ; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq
 ;
-; AVX512BW-LABEL: trunc_ssat_v8i32_v8i16:
+; AVX512BW-LABEL: trunc_ssat_v4i64_v4i8_store:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [32767,32767,32767,32767,32767,32767,32767,32767]
-; AVX512BW-NEXT:    vpminsd %ymm1, %ymm0, %ymm0
-; AVX512BW-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [4294934528,4294934528,4294934528,4294934528,4294934528,4294934528,4294934528,4294934528]
-; AVX512BW-NEXT:    vpmaxsd %ymm1, %ymm0, %ymm0
-; AVX512BW-NEXT:    vpmovdw %zmm0, %ymm0
-; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512BW-NEXT:    vpmovsqb %zmm0, %xmm0
+; AVX512BW-NEXT:    vmovd %xmm0, (%rdi)
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
-; AVX512BWVL-LABEL: trunc_ssat_v8i32_v8i16:
+; AVX512BWVL-LABEL: trunc_ssat_v4i64_v4i8_store:
 ; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vpmovsdw %ymm0, %xmm0
+; AVX512BWVL-NEXT:    vpmovsqb %ymm0, (%rdi)
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
-  %1 = icmp slt <8 x i32> %a0, <i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767>
-  %2 = select <8 x i1> %1, <8 x i32> %a0, <8 x i32> <i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767>
-  %3 = icmp sgt <8 x i32> %2, <i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768>
-  %4 = select <8 x i1> %3, <8 x i32> %2, <8 x i32> <i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768>
-  %5 = trunc <8 x i32> %4 to <8 x i16>
-  ret <8 x i16> %5
-}
-
-define <16 x i16> @trunc_ssat_v16i32_v16i16(<16 x i32> %a0) {
-; SSE-LABEL: trunc_ssat_v16i32_v16i16:
-; SSE:       # %bb.0:
-; SSE-NEXT:    packssdw %xmm1, %xmm0
-; SSE-NEXT:    packssdw %xmm3, %xmm2
-; SSE-NEXT:    movdqa %xmm2, %xmm1
-; SSE-NEXT:    retq
-;
-; AVX1-LABEL: trunc_ssat_v16i32_v16i16:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT:    vpackssdw %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX1-NEXT:    vpackssdw %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: trunc_ssat_v16i32_v16i16:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpackssdw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX2-NEXT:    retq
 ;
-; AVX512-LABEL: trunc_ssat_v16i32_v16i16:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpmovsdw %zmm0, %ymm0
-; AVX512-NEXT:    retq
-  %1 = icmp slt <16 x i32> %a0, <i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767>
-  %2 = select <16 x i1> %1, <16 x i32> %a0, <16 x i32> <i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767>
-  %3 = icmp sgt <16 x i32> %2, <i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768>
-  %4 = select <16 x i1> %3, <16 x i32> %2, <16 x i32> <i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768>
-  %5 = trunc <16 x i32> %4 to <16 x i16>
-  ret <16 x i16> %5
+; SKX-LABEL: trunc_ssat_v4i64_v4i8_store:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpmovsqb %ymm0, (%rdi)
+; SKX-NEXT:    vzeroupper
+; SKX-NEXT:    retq
+  %1 = icmp slt <4 x i64> %a0, <i64 127, i64 127, i64 127, i64 127>
+  %2 = select <4 x i1> %1, <4 x i64> %a0, <4 x i64> <i64 127, i64 127, i64 127, i64 127>
+  %3 = icmp sgt <4 x i64> %2, <i64 -128, i64 -128, i64 -128, i64 -128>
+  %4 = select <4 x i1> %3, <4 x i64> %2, <4 x i64> <i64 -128, i64 -128, i64 -128, i64 -128>
+  %5 = trunc <4 x i64> %4 to <4 x i8>
+  store <4 x i8> %5, <4 x i8> *%p1
+  ret void
 }
 
-;
-; Signed saturation truncation to v16i8
-;
-
-define <8 x i8> @trunc_ssat_v8i64_v8i8(<8 x i64> %a0) {
+define <8 x i8> @trunc_ssat_v8i64_v8i8(<8 x i64>* %p0) "min-legal-vector-width"="256" {
 ; SSE2-LABEL: trunc_ssat_v8i64_v8i8:
 ; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa (%rdi), %xmm9
+; SSE2-NEXT:    movdqa 16(%rdi), %xmm7
+; SSE2-NEXT:    movdqa 32(%rdi), %xmm5
+; SSE2-NEXT:    movdqa 48(%rdi), %xmm3
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [127,127]
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [2147483648,2147483648]
-; SSE2-NEXT:    movdqa %xmm2, %xmm5
-; SSE2-NEXT:    pxor %xmm4, %xmm5
-; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [2147483775,2147483775]
-; SSE2-NEXT:    movdqa %xmm9, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm5, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm7[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[1,1,3,3]
-; SSE2-NEXT:    pand %xmm10, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm7[1,1,3,3]
-; SSE2-NEXT:    por %xmm6, %xmm5
-; SSE2-NEXT:    pand %xmm5, %xmm2
-; SSE2-NEXT:    pandn %xmm8, %xmm5
-; SSE2-NEXT:    por %xmm2, %xmm5
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [2147483648,2147483648]
 ; SSE2-NEXT:    movdqa %xmm3, %xmm2
-; SSE2-NEXT:    pxor %xmm4, %xmm2
-; SSE2-NEXT:    movdqa %xmm9, %xmm6
+; SSE2-NEXT:    pxor %xmm1, %xmm2
+; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [2147483775,2147483775]
+; SSE2-NEXT:    movdqa %xmm10, %xmm6
 ; SSE2-NEXT:    pcmpgtd %xmm2, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm2[1,1,3,3]
-; SSE2-NEXT:    pand %xmm10, %xmm7
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,1,3,3]
+; SSE2-NEXT:    pand %xmm0, %xmm4
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[1,1,3,3]
-; SSE2-NEXT:    por %xmm7, %xmm2
+; SSE2-NEXT:    por %xmm4, %xmm2
 ; SSE2-NEXT:    pand %xmm2, %xmm3
 ; SSE2-NEXT:    pandn %xmm8, %xmm2
 ; SSE2-NEXT:    por %xmm3, %xmm2
-; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    pxor %xmm4, %xmm3
-; SSE2-NEXT:    movdqa %xmm9, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm3[1,1,3,3]
-; SSE2-NEXT:    pand %xmm10, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm6[1,1,3,3]
-; SSE2-NEXT:    por %xmm7, %xmm3
-; SSE2-NEXT:    pand %xmm3, %xmm0
-; SSE2-NEXT:    pandn %xmm8, %xmm3
+; SSE2-NEXT:    movdqa %xmm5, %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm0
+; SSE2-NEXT:    movdqa %xmm10, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pand %xmm4, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
 ; SSE2-NEXT:    por %xmm0, %xmm3
-; SSE2-NEXT:    movdqa %xmm1, %xmm0
-; SSE2-NEXT:    pxor %xmm4, %xmm0
-; SSE2-NEXT:    movdqa %xmm9, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm0
+; SSE2-NEXT:    pand %xmm3, %xmm5
+; SSE2-NEXT:    pandn %xmm8, %xmm3
+; SSE2-NEXT:    por %xmm5, %xmm3
+; SSE2-NEXT:    movdqa %xmm7, %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm0
+; SSE2-NEXT:    movdqa %xmm10, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pand %xmm7, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[1,1,3,3]
+; SSE2-NEXT:    pand %xmm5, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm0, %xmm5
+; SSE2-NEXT:    pand %xmm5, %xmm7
+; SSE2-NEXT:    pandn %xmm8, %xmm5
+; SSE2-NEXT:    por %xmm7, %xmm5
+; SSE2-NEXT:    movdqa %xmm9, %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm0
+; SSE2-NEXT:    movdqa %xmm10, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pand %xmm6, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[1,1,3,3]
 ; SSE2-NEXT:    por %xmm0, %xmm7
-; SSE2-NEXT:    pand %xmm7, %xmm1
+; SSE2-NEXT:    pand %xmm7, %xmm9
 ; SSE2-NEXT:    pandn %xmm8, %xmm7
-; SSE2-NEXT:    por %xmm1, %xmm7
+; SSE2-NEXT:    por %xmm9, %xmm7
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [18446744073709551488,18446744073709551488]
 ; SSE2-NEXT:    movdqa %xmm7, %xmm0
-; SSE2-NEXT:    pxor %xmm4, %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm0
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [18446744071562067840,18446744071562067840]
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm9, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm0, %xmm1
-; SSE2-NEXT:    pand %xmm1, %xmm7
-; SSE2-NEXT:    pandn %xmm8, %xmm1
-; SSE2-NEXT:    por %xmm7, %xmm1
-; SSE2-NEXT:    movdqa %xmm3, %xmm0
-; SSE2-NEXT:    pxor %xmm4, %xmm0
-; SSE2-NEXT:    movdqa %xmm0, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm9, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm6[0,0,2,2]
+; SSE2-NEXT:    movdqa %xmm0, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm9, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm4[0,0,2,2]
 ; SSE2-NEXT:    pcmpeqd %xmm9, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pand %xmm10, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3]
-; SSE2-NEXT:    por %xmm7, %xmm0
-; SSE2-NEXT:    pand %xmm0, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pand %xmm10, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm6, %xmm0
+; SSE2-NEXT:    pand %xmm0, %xmm7
 ; SSE2-NEXT:    pandn %xmm8, %xmm0
-; SSE2-NEXT:    por %xmm3, %xmm0
-; SSE2-NEXT:    packssdw %xmm1, %xmm0
-; SSE2-NEXT:    movdqa %xmm2, %xmm1
-; SSE2-NEXT:    pxor %xmm4, %xmm1
+; SSE2-NEXT:    por %xmm7, %xmm0
+; SSE2-NEXT:    movdqa %xmm5, %xmm4
+; SSE2-NEXT:    pxor %xmm1, %xmm4
+; SSE2-NEXT:    movdqa %xmm4, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm9, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm9, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    pand %xmm7, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[1,1,3,3]
+; SSE2-NEXT:    por %xmm4, %xmm7
+; SSE2-NEXT:    pand %xmm7, %xmm5
+; SSE2-NEXT:    pandn %xmm8, %xmm7
+; SSE2-NEXT:    por %xmm5, %xmm7
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    pxor %xmm1, %xmm4
+; SSE2-NEXT:    movdqa %xmm4, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm9, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm9, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    pand %xmm6, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSE2-NEXT:    por %xmm4, %xmm5
+; SSE2-NEXT:    pand %xmm5, %xmm3
+; SSE2-NEXT:    pandn %xmm8, %xmm5
+; SSE2-NEXT:    por %xmm3, %xmm5
+; SSE2-NEXT:    pxor %xmm2, %xmm1
 ; SSE2-NEXT:    movdqa %xmm1, %xmm3
 ; SSE2-NEXT:    pcmpgtd %xmm9, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[0,0,2,2]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
 ; SSE2-NEXT:    pcmpeqd %xmm9, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm1
+; SSE2-NEXT:    pand %xmm4, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
 ; SSE2-NEXT:    por %xmm1, %xmm3
 ; SSE2-NEXT:    pand %xmm3, %xmm2
 ; SSE2-NEXT:    pandn %xmm8, %xmm3
 ; SSE2-NEXT:    por %xmm2, %xmm3
-; SSE2-NEXT:    pxor %xmm5, %xmm4
-; SSE2-NEXT:    movdqa %xmm4, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm9, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    pand %xmm2, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm4, %xmm1
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [255,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0]
+; SSE2-NEXT:    pand %xmm1, %xmm3
 ; SSE2-NEXT:    pand %xmm1, %xmm5
-; SSE2-NEXT:    pandn %xmm8, %xmm1
-; SSE2-NEXT:    por %xmm5, %xmm1
-; SSE2-NEXT:    packssdw %xmm3, %xmm1
-; SSE2-NEXT:    packssdw %xmm1, %xmm0
+; SSE2-NEXT:    packuswb %xmm3, %xmm5
+; SSE2-NEXT:    pand %xmm1, %xmm7
+; SSE2-NEXT:    pand %xmm1, %xmm0
+; SSE2-NEXT:    packuswb %xmm7, %xmm0
+; SSE2-NEXT:    packuswb %xmm5, %xmm0
+; SSE2-NEXT:    packuswb %xmm0, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: trunc_ssat_v8i64_v8i8:
 ; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa (%rdi), %xmm9
+; SSSE3-NEXT:    movdqa 16(%rdi), %xmm7
+; SSSE3-NEXT:    movdqa 32(%rdi), %xmm5
+; SSSE3-NEXT:    movdqa 48(%rdi), %xmm3
 ; SSSE3-NEXT:    movdqa {{.*#+}} xmm8 = [127,127]
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm4 = [2147483648,2147483648]
-; SSSE3-NEXT:    movdqa %xmm2, %xmm5
-; SSSE3-NEXT:    pxor %xmm4, %xmm5
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm9 = [2147483775,2147483775]
-; SSSE3-NEXT:    movdqa %xmm9, %xmm7
-; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm7
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm10 = xmm7[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm5
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm10, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm7[1,1,3,3]
-; SSSE3-NEXT:    por %xmm6, %xmm5
-; SSSE3-NEXT:    pand %xmm5, %xmm2
-; SSSE3-NEXT:    pandn %xmm8, %xmm5
-; SSSE3-NEXT:    por %xmm2, %xmm5
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [2147483648,2147483648]
 ; SSSE3-NEXT:    movdqa %xmm3, %xmm2
-; SSSE3-NEXT:    pxor %xmm4, %xmm2
-; SSSE3-NEXT:    movdqa %xmm9, %xmm6
+; SSSE3-NEXT:    pxor %xmm1, %xmm2
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm10 = [2147483775,2147483775]
+; SSSE3-NEXT:    movdqa %xmm10, %xmm6
 ; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm10 = xmm6[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm2[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm10, %xmm7
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm0, %xmm4
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[1,1,3,3]
-; SSSE3-NEXT:    por %xmm7, %xmm2
+; SSSE3-NEXT:    por %xmm4, %xmm2
 ; SSSE3-NEXT:    pand %xmm2, %xmm3
 ; SSSE3-NEXT:    pandn %xmm8, %xmm2
 ; SSSE3-NEXT:    por %xmm3, %xmm2
-; SSSE3-NEXT:    movdqa %xmm0, %xmm3
-; SSSE3-NEXT:    pxor %xmm4, %xmm3
-; SSSE3-NEXT:    movdqa %xmm9, %xmm6
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm10 = xmm6[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm10, %xmm7
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm6[1,1,3,3]
-; SSSE3-NEXT:    por %xmm7, %xmm3
-; SSSE3-NEXT:    pand %xmm3, %xmm0
-; SSSE3-NEXT:    pandn %xmm8, %xmm3
+; SSSE3-NEXT:    movdqa %xmm5, %xmm0
+; SSSE3-NEXT:    pxor %xmm1, %xmm0
+; SSSE3-NEXT:    movdqa %xmm10, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm4, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
 ; SSSE3-NEXT:    por %xmm0, %xmm3
-; SSSE3-NEXT:    movdqa %xmm1, %xmm0
-; SSSE3-NEXT:    pxor %xmm4, %xmm0
-; SSSE3-NEXT:    movdqa %xmm9, %xmm6
-; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm0
+; SSSE3-NEXT:    pand %xmm3, %xmm5
+; SSSE3-NEXT:    pandn %xmm8, %xmm3
+; SSSE3-NEXT:    por %xmm5, %xmm3
+; SSSE3-NEXT:    movdqa %xmm7, %xmm0
+; SSSE3-NEXT:    pxor %xmm1, %xmm0
+; SSSE3-NEXT:    movdqa %xmm10, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm0
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm7, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm5, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm0, %xmm5
+; SSSE3-NEXT:    pand %xmm5, %xmm7
+; SSSE3-NEXT:    pandn %xmm8, %xmm5
+; SSSE3-NEXT:    por %xmm7, %xmm5
+; SSSE3-NEXT:    movdqa %xmm9, %xmm0
+; SSSE3-NEXT:    pxor %xmm1, %xmm0
+; SSSE3-NEXT:    movdqa %xmm10, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm6, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[1,1,3,3]
 ; SSSE3-NEXT:    por %xmm0, %xmm7
-; SSSE3-NEXT:    pand %xmm7, %xmm1
+; SSSE3-NEXT:    pand %xmm7, %xmm9
 ; SSSE3-NEXT:    pandn %xmm8, %xmm7
-; SSSE3-NEXT:    por %xmm1, %xmm7
+; SSSE3-NEXT:    por %xmm9, %xmm7
 ; SSSE3-NEXT:    movdqa {{.*#+}} xmm8 = [18446744073709551488,18446744073709551488]
 ; SSSE3-NEXT:    movdqa %xmm7, %xmm0
-; SSSE3-NEXT:    pxor %xmm4, %xmm0
+; SSSE3-NEXT:    pxor %xmm1, %xmm0
 ; SSSE3-NEXT:    movdqa {{.*#+}} xmm9 = [18446744071562067840,18446744071562067840]
-; SSSE3-NEXT:    movdqa %xmm0, %xmm1
-; SSSE3-NEXT:    pcmpgtd %xmm9, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm6, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    por %xmm0, %xmm1
-; SSSE3-NEXT:    pand %xmm1, %xmm7
-; SSSE3-NEXT:    pandn %xmm8, %xmm1
-; SSSE3-NEXT:    por %xmm7, %xmm1
-; SSSE3-NEXT:    movdqa %xmm3, %xmm0
-; SSSE3-NEXT:    pxor %xmm4, %xmm0
-; SSSE3-NEXT:    movdqa %xmm0, %xmm6
-; SSSE3-NEXT:    pcmpgtd %xmm9, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm10 = xmm6[0,0,2,2]
+; SSSE3-NEXT:    movdqa %xmm0, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm9, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm10 = xmm4[0,0,2,2]
 ; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm10, %xmm7
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3]
-; SSSE3-NEXT:    por %xmm7, %xmm0
-; SSSE3-NEXT:    pand %xmm0, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm10, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm6, %xmm0
+; SSSE3-NEXT:    pand %xmm0, %xmm7
 ; SSSE3-NEXT:    pandn %xmm8, %xmm0
-; SSSE3-NEXT:    por %xmm3, %xmm0
-; SSSE3-NEXT:    packssdw %xmm1, %xmm0
-; SSSE3-NEXT:    movdqa %xmm2, %xmm1
-; SSSE3-NEXT:    pxor %xmm4, %xmm1
+; SSSE3-NEXT:    por %xmm7, %xmm0
+; SSSE3-NEXT:    movdqa %xmm5, %xmm4
+; SSSE3-NEXT:    pxor %xmm1, %xmm4
+; SSSE3-NEXT:    movdqa %xmm4, %xmm6
+; SSSE3-NEXT:    pcmpgtd %xmm9, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm7, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[1,1,3,3]
+; SSSE3-NEXT:    por %xmm4, %xmm7
+; SSSE3-NEXT:    pand %xmm7, %xmm5
+; SSSE3-NEXT:    pandn %xmm8, %xmm7
+; SSSE3-NEXT:    por %xmm5, %xmm7
+; SSSE3-NEXT:    movdqa %xmm3, %xmm4
+; SSSE3-NEXT:    pxor %xmm1, %xmm4
+; SSSE3-NEXT:    movdqa %xmm4, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm9, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm6, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSSE3-NEXT:    por %xmm4, %xmm5
+; SSSE3-NEXT:    pand %xmm5, %xmm3
+; SSSE3-NEXT:    pandn %xmm8, %xmm5
+; SSSE3-NEXT:    por %xmm3, %xmm5
+; SSSE3-NEXT:    pxor %xmm2, %xmm1
 ; SSSE3-NEXT:    movdqa %xmm1, %xmm3
 ; SSSE3-NEXT:    pcmpgtd %xmm9, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[0,0,2,2]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
 ; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm1
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm6, %xmm1
+; SSSE3-NEXT:    pand %xmm4, %xmm1
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
 ; SSSE3-NEXT:    por %xmm1, %xmm3
 ; SSSE3-NEXT:    pand %xmm3, %xmm2
 ; SSSE3-NEXT:    pandn %xmm8, %xmm3
 ; SSSE3-NEXT:    por %xmm2, %xmm3
-; SSSE3-NEXT:    pxor %xmm5, %xmm4
-; SSSE3-NEXT:    movdqa %xmm4, %xmm1
-; SSSE3-NEXT:    pcmpgtd %xmm9, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm2, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    por %xmm4, %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [255,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0]
+; SSSE3-NEXT:    pand %xmm1, %xmm3
 ; SSSE3-NEXT:    pand %xmm1, %xmm5
-; SSSE3-NEXT:    pandn %xmm8, %xmm1
-; SSSE3-NEXT:    por %xmm5, %xmm1
-; SSSE3-NEXT:    packssdw %xmm3, %xmm1
-; SSSE3-NEXT:    packssdw %xmm1, %xmm0
+; SSSE3-NEXT:    packuswb %xmm3, %xmm5
+; SSSE3-NEXT:    pand %xmm1, %xmm7
+; SSSE3-NEXT:    pand %xmm1, %xmm0
+; SSSE3-NEXT:    packuswb %xmm7, %xmm0
+; SSSE3-NEXT:    packuswb %xmm5, %xmm0
+; SSSE3-NEXT:    packuswb %xmm0, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: trunc_ssat_v8i64_v8i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa %xmm0, %xmm10
-; SSE41-NEXT:    movapd {{.*#+}} xmm7 = [127,127]
-; SSE41-NEXT:    movdqa {{.*#+}} xmm5 = [2147483648,2147483648]
-; SSE41-NEXT:    movdqa %xmm2, %xmm0
-; SSE41-NEXT:    pxor %xmm5, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm11 = [2147483775,2147483775]
-; SSE41-NEXT:    movdqa %xmm11, %xmm6
+; SSE41-NEXT:    movdqa (%rdi), %xmm9
+; SSE41-NEXT:    movdqa 16(%rdi), %xmm10
+; SSE41-NEXT:    movdqa 32(%rdi), %xmm3
+; SSE41-NEXT:    movdqa 48(%rdi), %xmm5
+; SSE41-NEXT:    movapd {{.*#+}} xmm4 = [127,127]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
+; SSE41-NEXT:    movdqa %xmm5, %xmm0
+; SSE41-NEXT:    pxor %xmm2, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [2147483775,2147483775]
+; SSE41-NEXT:    movdqa %xmm1, %xmm7
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm7
+; SSE41-NEXT:    movdqa %xmm1, %xmm6
 ; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm11, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
+; SSE41-NEXT:    pand %xmm7, %xmm0
 ; SSE41-NEXT:    por %xmm6, %xmm0
-; SSE41-NEXT:    movapd %xmm7, %xmm8
-; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm8
+; SSE41-NEXT:    movapd %xmm4, %xmm8
+; SSE41-NEXT:    blendvpd %xmm0, %xmm5, %xmm8
 ; SSE41-NEXT:    movdqa %xmm3, %xmm0
-; SSE41-NEXT:    pxor %xmm5, %xmm0
-; SSE41-NEXT:    movdqa %xmm11, %xmm2
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm11, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm4, %xmm0
-; SSE41-NEXT:    por %xmm2, %xmm0
-; SSE41-NEXT:    movapd %xmm7, %xmm9
-; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm9
+; SSE41-NEXT:    pxor %xmm2, %xmm0
+; SSE41-NEXT:    movdqa %xmm1, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm5
+; SSE41-NEXT:    movdqa %xmm1, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm0
+; SSE41-NEXT:    por %xmm6, %xmm0
+; SSE41-NEXT:    movapd %xmm4, %xmm11
+; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm11
 ; SSE41-NEXT:    movdqa %xmm10, %xmm0
-; SSE41-NEXT:    pxor %xmm5, %xmm0
-; SSE41-NEXT:    movdqa %xmm11, %xmm2
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm11, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE41-NEXT:    pxor %xmm2, %xmm0
+; SSE41-NEXT:    movdqa %xmm1, %xmm3
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm3
+; SSE41-NEXT:    movdqa %xmm1, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
 ; SSE41-NEXT:    pand %xmm3, %xmm0
-; SSE41-NEXT:    por %xmm2, %xmm0
-; SSE41-NEXT:    movapd %xmm7, %xmm2
-; SSE41-NEXT:    blendvpd %xmm0, %xmm10, %xmm2
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    pxor %xmm5, %xmm0
-; SSE41-NEXT:    movdqa %xmm11, %xmm3
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm11, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm4, %xmm0
-; SSE41-NEXT:    por %xmm3, %xmm0
-; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm7
-; SSE41-NEXT:    movapd {{.*#+}} xmm3 = [18446744073709551488,18446744073709551488]
-; SSE41-NEXT:    movapd %xmm7, %xmm0
-; SSE41-NEXT:    xorpd %xmm5, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [18446744071562067840,18446744071562067840]
-; SSE41-NEXT:    movapd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd %xmm4, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm4, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE41-NEXT:    por %xmm6, %xmm0
+; SSE41-NEXT:    movapd %xmm4, %xmm3
+; SSE41-NEXT:    blendvpd %xmm0, %xmm10, %xmm3
+; SSE41-NEXT:    movdqa %xmm9, %xmm0
+; SSE41-NEXT:    pxor %xmm2, %xmm0
+; SSE41-NEXT:    movdqa %xmm1, %xmm6
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2]
 ; SSE41-NEXT:    pand %xmm6, %xmm0
 ; SSE41-NEXT:    por %xmm1, %xmm0
-; SSE41-NEXT:    movapd %xmm3, %xmm6
-; SSE41-NEXT:    blendvpd %xmm0, %xmm7, %xmm6
-; SSE41-NEXT:    movapd %xmm2, %xmm0
-; SSE41-NEXT:    xorpd %xmm5, %xmm0
-; SSE41-NEXT:    movapd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd %xmm4, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm1[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm4, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm7, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm9, %xmm4
+; SSE41-NEXT:    movapd {{.*#+}} xmm6 = [18446744073709551488,18446744073709551488]
+; SSE41-NEXT:    movapd %xmm4, %xmm1
+; SSE41-NEXT:    xorpd %xmm2, %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm7 = [18446744071562067840,18446744071562067840]
+; SSE41-NEXT:    movapd %xmm1, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm7, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm7, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm0
 ; SSE41-NEXT:    por %xmm1, %xmm0
-; SSE41-NEXT:    movapd %xmm3, %xmm1
-; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
-; SSE41-NEXT:    packssdw %xmm6, %xmm1
-; SSE41-NEXT:    movapd %xmm9, %xmm0
-; SSE41-NEXT:    xorpd %xmm5, %xmm0
-; SSE41-NEXT:    movapd %xmm0, %xmm2
-; SSE41-NEXT:    pcmpgtd %xmm4, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm4, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm6, %xmm0
+; SSE41-NEXT:    movapd %xmm6, %xmm1
+; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm1
+; SSE41-NEXT:    movapd %xmm3, %xmm4
+; SSE41-NEXT:    xorpd %xmm2, %xmm4
+; SSE41-NEXT:    movapd %xmm4, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm7, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm7, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm0
+; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    movapd %xmm6, %xmm4
+; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm4
+; SSE41-NEXT:    movapd %xmm11, %xmm3
+; SSE41-NEXT:    xorpd %xmm2, %xmm3
+; SSE41-NEXT:    movapd %xmm3, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm7, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm7, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm0
+; SSE41-NEXT:    por %xmm3, %xmm0
+; SSE41-NEXT:    movapd %xmm6, %xmm3
+; SSE41-NEXT:    blendvpd %xmm0, %xmm11, %xmm3
+; SSE41-NEXT:    xorpd %xmm8, %xmm2
+; SSE41-NEXT:    movapd %xmm2, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm7, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm7, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm0
 ; SSE41-NEXT:    por %xmm2, %xmm0
-; SSE41-NEXT:    movapd %xmm3, %xmm2
-; SSE41-NEXT:    blendvpd %xmm0, %xmm9, %xmm2
-; SSE41-NEXT:    xorpd %xmm8, %xmm5
-; SSE41-NEXT:    movapd %xmm5, %xmm6
-; SSE41-NEXT:    pcmpgtd %xmm4, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm4, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[1,1,3,3]
-; SSE41-NEXT:    pand %xmm7, %xmm0
-; SSE41-NEXT:    por %xmm6, %xmm0
-; SSE41-NEXT:    blendvpd %xmm0, %xmm8, %xmm3
-; SSE41-NEXT:    packssdw %xmm2, %xmm3
-; SSE41-NEXT:    packssdw %xmm3, %xmm1
+; SSE41-NEXT:    blendvpd %xmm0, %xmm8, %xmm6
+; SSE41-NEXT:    movapd {{.*#+}} xmm0 = [255,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0]
+; SSE41-NEXT:    andpd %xmm0, %xmm6
+; SSE41-NEXT:    andpd %xmm0, %xmm3
+; SSE41-NEXT:    packusdw %xmm6, %xmm3
+; SSE41-NEXT:    andpd %xmm0, %xmm4
+; SSE41-NEXT:    andpd %xmm0, %xmm1
+; SSE41-NEXT:    packusdw %xmm4, %xmm1
+; SSE41-NEXT:    packusdw %xmm3, %xmm1
+; SSE41-NEXT:    packuswb %xmm1, %xmm1
 ; SSE41-NEXT:    movdqa %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: trunc_ssat_v8i64_v8i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [127,127]
-; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm3, %xmm8
-; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm3, %xmm5
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm6
-; AVX1-NEXT:    vpcmpgtq %xmm6, %xmm3, %xmm7
-; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm3, %xmm4
-; AVX1-NEXT:    vblendvpd %xmm4, %xmm0, %xmm3, %xmm0
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [18446744073709551488,18446744073709551488]
-; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm0, %xmm9
-; AVX1-NEXT:    vblendvpd %xmm7, %xmm6, %xmm3, %xmm6
-; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm6, %xmm7
-; AVX1-NEXT:    vblendvpd %xmm5, %xmm1, %xmm3, %xmm1
-; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm1, %xmm5
-; AVX1-NEXT:    vblendvpd %xmm8, %xmm2, %xmm3, %xmm2
-; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm2, %xmm3
-; AVX1-NEXT:    vblendvpd %xmm3, %xmm2, %xmm4, %xmm2
-; AVX1-NEXT:    vblendvpd %xmm5, %xmm1, %xmm4, %xmm1
-; AVX1-NEXT:    vpackssdw %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vblendvpd %xmm7, %xmm6, %xmm4, %xmm2
-; AVX1-NEXT:    vblendvpd %xmm9, %xmm0, %xmm4, %xmm0
-; AVX1-NEXT:    vpackssdw %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vmovapd {{.*#+}} ymm9 = [127,127,127,127]
+; AVX1-NEXT:    vmovdqa 48(%rdi), %xmm1
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [127,127]
+; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm2, %xmm3
+; AVX1-NEXT:    vmovdqa (%rdi), %xmm4
+; AVX1-NEXT:    vmovdqa 16(%rdi), %xmm5
+; AVX1-NEXT:    vmovdqa 32(%rdi), %xmm6
+; AVX1-NEXT:    vpcmpgtq %xmm6, %xmm2, %xmm7
+; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm7, %ymm8
+; AVX1-NEXT:    vblendvpd %ymm8, 32(%rdi), %ymm9, %ymm8
+; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm2, %xmm0
+; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm2, %xmm10
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm10, %ymm11
+; AVX1-NEXT:    vblendvpd %ymm11, (%rdi), %ymm9, %ymm9
+; AVX1-NEXT:    vmovapd {{.*#+}} ymm11 = [18446744073709551488,18446744073709551488,18446744073709551488,18446744073709551488]
+; AVX1-NEXT:    vblendvpd %xmm0, %xmm5, %xmm2, %xmm0
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm5 = [18446744073709551488,18446744073709551488]
+; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm0, %xmm0
+; AVX1-NEXT:    vblendvpd %xmm10, %xmm4, %xmm2, %xmm4
+; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm4, %ymm0
+; AVX1-NEXT:    vblendvpd %ymm0, %ymm9, %ymm11, %ymm0
+; AVX1-NEXT:    vblendvpd %xmm3, %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm1, %xmm1
+; AVX1-NEXT:    vblendvpd %xmm7, %xmm6, %xmm2, %xmm2
+; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm2, %xmm2
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
+; AVX1-NEXT:    vblendvpd %ymm1, %ymm8, %ymm11, %ymm1
+; AVX1-NEXT:    vmovapd {{.*#+}} ymm2 = [255,255,255,255]
+; AVX1-NEXT:    vandpd %ymm2, %ymm1, %ymm1
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
+; AVX1-NEXT:    vpackusdw %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vandpd %ymm2, %ymm0, %ymm0
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT:    vpackusdw %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: trunc_ssat_v8i64_v8i8:
 ; AVX2:       # %bb.0:
+; AVX2-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX2-NEXT:    vmovdqa 32(%rdi), %ymm1
 ; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [127,127,127,127]
-; AVX2-NEXT:    vpcmpgtq %ymm0, %ymm2, %ymm3
-; AVX2-NEXT:    vblendvpd %ymm3, %ymm0, %ymm2, %ymm0
 ; AVX2-NEXT:    vpcmpgtq %ymm1, %ymm2, %ymm3
 ; AVX2-NEXT:    vblendvpd %ymm3, %ymm1, %ymm2, %ymm1
+; AVX2-NEXT:    vpcmpgtq %ymm0, %ymm2, %ymm3
+; AVX2-NEXT:    vblendvpd %ymm3, %ymm0, %ymm2, %ymm0
 ; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [18446744073709551488,18446744073709551488,18446744073709551488,18446744073709551488]
-; AVX2-NEXT:    vpcmpgtq %ymm2, %ymm1, %ymm3
-; AVX2-NEXT:    vblendvpd %ymm3, %ymm1, %ymm2, %ymm1
 ; AVX2-NEXT:    vpcmpgtq %ymm2, %ymm0, %ymm3
 ; AVX2-NEXT:    vblendvpd %ymm3, %ymm0, %ymm2, %ymm0
-; AVX2-NEXT:    vpackssdw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpcmpgtq %ymm2, %ymm1, %ymm3
+; AVX2-NEXT:    vblendvpd %ymm3, %ymm1, %ymm2, %ymm1
+; AVX2-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm3 = <u,u,0,8,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX2-NEXT:    vpshufb %xmm3, %xmm2, %xmm2
+; AVX2-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
+; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm3 = <0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX2-NEXT:    vpshufb %xmm3, %xmm2, %xmm2
+; AVX2-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
+; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3]
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: trunc_ssat_v8i64_v8i8:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpminsq {{.*}}(%rip){1to8}, %zmm0, %zmm0
-; AVX512-NEXT:    vpmaxsq {{.*}}(%rip){1to8}, %zmm0, %zmm0
-; AVX512-NEXT:    vpmovqw %zmm0, %xmm0
+; AVX512-NEXT:    vmovdqa64 (%rdi), %zmm0
+; AVX512-NEXT:    vpmovsqb %zmm0, %xmm0
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
+;
+; SKX-LABEL: trunc_ssat_v8i64_v8i8:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vmovdqa (%rdi), %ymm0
+; SKX-NEXT:    vmovdqa 32(%rdi), %ymm1
+; SKX-NEXT:    vpmovsqb %ymm1, %xmm1
+; SKX-NEXT:    vpmovsqb %ymm0, %xmm0
+; SKX-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SKX-NEXT:    vzeroupper
+; SKX-NEXT:    retq
+  %a0 = load <8 x i64>, <8 x i64>* %p0
   %1 = icmp slt <8 x i64> %a0, <i64 127, i64 127, i64 127, i64 127, i64 127, i64 127, i64 127, i64 127>
   %2 = select <8 x i1> %1, <8 x i64> %a0, <8 x i64> <i64 127, i64 127, i64 127, i64 127, i64 127, i64 127, i64 127, i64 127>
   %3 = icmp sgt <8 x i64> %2, <i64 -128, i64 -128, i64 -128, i64 -128, i64 -128, i64 -128, i64 -128, i64 -128>
@@ -1567,384 +3772,392 @@ define <8 x i8> @trunc_ssat_v8i64_v8i8(<8 x i64> %a0) {
 
 ; TODO: The AVX1 codegen shows a missed opportunity to narrow blendv+logic to 128-bit.
 
-define void @trunc_ssat_v8i64_v8i8_store(<8 x i64> %a0, <8 x i8> *%p1) {
+define void @trunc_ssat_v8i64_v8i8_store(<8 x i64>* %p0, <8 x i8> *%p1) "min-legal-vector-width"="256" {
 ; SSE2-LABEL: trunc_ssat_v8i64_v8i8_store:
 ; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa (%rdi), %xmm9
+; SSE2-NEXT:    movdqa 16(%rdi), %xmm7
+; SSE2-NEXT:    movdqa 32(%rdi), %xmm5
+; SSE2-NEXT:    movdqa 48(%rdi), %xmm2
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [127,127]
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [2147483648,2147483648]
-; SSE2-NEXT:    movdqa %xmm3, %xmm5
-; SSE2-NEXT:    pxor %xmm4, %xmm5
-; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [2147483775,2147483775]
-; SSE2-NEXT:    movdqa %xmm9, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm5, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm7[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[1,1,3,3]
-; SSE2-NEXT:    pand %xmm10, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm7[1,1,3,3]
-; SSE2-NEXT:    por %xmm6, %xmm5
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm2, %xmm1
+; SSE2-NEXT:    pxor %xmm0, %xmm1
+; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [2147483775,2147483775]
+; SSE2-NEXT:    movdqa %xmm10, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm6[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pand %xmm3, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm6[1,1,3,3]
+; SSE2-NEXT:    por %xmm4, %xmm1
+; SSE2-NEXT:    pand %xmm1, %xmm2
+; SSE2-NEXT:    pandn %xmm8, %xmm1
+; SSE2-NEXT:    por %xmm2, %xmm1
+; SSE2-NEXT:    movdqa %xmm5, %xmm2
+; SSE2-NEXT:    pxor %xmm0, %xmm2
+; SSE2-NEXT:    movdqa %xmm10, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[1,1,3,3]
+; SSE2-NEXT:    pand %xmm4, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,1,3,3]
+; SSE2-NEXT:    por %xmm6, %xmm2
+; SSE2-NEXT:    pand %xmm2, %xmm5
+; SSE2-NEXT:    pandn %xmm8, %xmm2
+; SSE2-NEXT:    por %xmm5, %xmm2
+; SSE2-NEXT:    movdqa %xmm7, %xmm3
+; SSE2-NEXT:    pxor %xmm0, %xmm3
+; SSE2-NEXT:    movdqa %xmm10, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm5, %xmm3
-; SSE2-NEXT:    pandn %xmm8, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[1,1,3,3]
 ; SSE2-NEXT:    por %xmm3, %xmm5
-; SSE2-NEXT:    movdqa %xmm2, %xmm3
-; SSE2-NEXT:    pxor %xmm4, %xmm3
-; SSE2-NEXT:    movdqa %xmm9, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm3[1,1,3,3]
-; SSE2-NEXT:    pand %xmm10, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm6[1,1,3,3]
-; SSE2-NEXT:    por %xmm7, %xmm3
-; SSE2-NEXT:    pand %xmm3, %xmm2
-; SSE2-NEXT:    pandn %xmm8, %xmm3
-; SSE2-NEXT:    por %xmm2, %xmm3
-; SSE2-NEXT:    movdqa %xmm1, %xmm2
-; SSE2-NEXT:    pxor %xmm4, %xmm2
-; SSE2-NEXT:    movdqa %xmm9, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm2[1,1,3,3]
-; SSE2-NEXT:    pand %xmm10, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[1,1,3,3]
-; SSE2-NEXT:    por %xmm7, %xmm2
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pandn %xmm8, %xmm2
-; SSE2-NEXT:    por %xmm1, %xmm2
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    pxor %xmm4, %xmm1
-; SSE2-NEXT:    movdqa %xmm9, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pand %xmm7, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm7
-; SSE2-NEXT:    pand %xmm7, %xmm0
+; SSE2-NEXT:    pand %xmm5, %xmm7
+; SSE2-NEXT:    pandn %xmm8, %xmm5
+; SSE2-NEXT:    por %xmm7, %xmm5
+; SSE2-NEXT:    movdqa %xmm9, %xmm3
+; SSE2-NEXT:    pxor %xmm0, %xmm3
+; SSE2-NEXT:    movdqa %xmm10, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSE2-NEXT:    pand %xmm6, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm3, %xmm7
+; SSE2-NEXT:    pand %xmm7, %xmm9
 ; SSE2-NEXT:    pandn %xmm8, %xmm7
-; SSE2-NEXT:    por %xmm0, %xmm7
+; SSE2-NEXT:    por %xmm9, %xmm7
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [18446744073709551488,18446744073709551488]
-; SSE2-NEXT:    movdqa %xmm7, %xmm0
-; SSE2-NEXT:    pxor %xmm4, %xmm0
+; SSE2-NEXT:    movdqa %xmm7, %xmm3
+; SSE2-NEXT:    pxor %xmm0, %xmm3
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [18446744071562067840,18446744071562067840]
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm9, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm0[1,1,3,3]
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm9, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm9, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm10, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm6, %xmm0
-; SSE2-NEXT:    pand %xmm0, %xmm7
-; SSE2-NEXT:    pandn %xmm8, %xmm0
-; SSE2-NEXT:    por %xmm7, %xmm0
-; SSE2-NEXT:    movdqa %xmm2, %xmm1
-; SSE2-NEXT:    pxor %xmm4, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm6, %xmm3
+; SSE2-NEXT:    pand %xmm3, %xmm7
+; SSE2-NEXT:    pandn %xmm8, %xmm3
+; SSE2-NEXT:    por %xmm7, %xmm3
+; SSE2-NEXT:    movdqa %xmm5, %xmm4
+; SSE2-NEXT:    pxor %xmm0, %xmm4
+; SSE2-NEXT:    movdqa %xmm4, %xmm6
 ; SSE2-NEXT:    pcmpgtd %xmm9, %xmm6
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pand %xmm7, %xmm1
+; SSE2-NEXT:    pcmpeqd %xmm9, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    pand %xmm7, %xmm4
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm7
-; SSE2-NEXT:    pand %xmm7, %xmm2
+; SSE2-NEXT:    por %xmm4, %xmm7
+; SSE2-NEXT:    pand %xmm7, %xmm5
 ; SSE2-NEXT:    pandn %xmm8, %xmm7
-; SSE2-NEXT:    por %xmm2, %xmm7
-; SSE2-NEXT:    movdqa %xmm3, %xmm1
-; SSE2-NEXT:    pxor %xmm4, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    por %xmm5, %xmm7
+; SSE2-NEXT:    movdqa %xmm2, %xmm4
+; SSE2-NEXT:    pxor %xmm0, %xmm4
+; SSE2-NEXT:    movdqa %xmm4, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm9, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm9, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    pand %xmm6, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSE2-NEXT:    por %xmm4, %xmm5
+; SSE2-NEXT:    pand %xmm5, %xmm2
+; SSE2-NEXT:    pandn %xmm8, %xmm5
+; SSE2-NEXT:    por %xmm2, %xmm5
+; SSE2-NEXT:    pxor %xmm1, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
 ; SSE2-NEXT:    pcmpgtd %xmm9, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm9, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pand %xmm4, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm2
-; SSE2-NEXT:    pand %xmm2, %xmm3
+; SSE2-NEXT:    por %xmm0, %xmm2
+; SSE2-NEXT:    pand %xmm2, %xmm1
 ; SSE2-NEXT:    pandn %xmm8, %xmm2
-; SSE2-NEXT:    por %xmm3, %xmm2
-; SSE2-NEXT:    pxor %xmm5, %xmm4
-; SSE2-NEXT:    movdqa %xmm4, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm9, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    pand %xmm3, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm4, %xmm1
-; SSE2-NEXT:    pand %xmm1, %xmm5
-; SSE2-NEXT:    pandn %xmm8, %xmm1
-; SSE2-NEXT:    por %xmm5, %xmm1
-; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [255,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0]
-; SSE2-NEXT:    pand %xmm3, %xmm1
-; SSE2-NEXT:    pand %xmm3, %xmm2
-; SSE2-NEXT:    packuswb %xmm1, %xmm2
-; SSE2-NEXT:    pand %xmm3, %xmm7
-; SSE2-NEXT:    pand %xmm3, %xmm0
-; SSE2-NEXT:    packuswb %xmm7, %xmm0
-; SSE2-NEXT:    packuswb %xmm2, %xmm0
-; SSE2-NEXT:    packuswb %xmm0, %xmm0
-; SSE2-NEXT:    movq %xmm0, (%rdi)
+; SSE2-NEXT:    por %xmm1, %xmm2
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0]
+; SSE2-NEXT:    pand %xmm0, %xmm2
+; SSE2-NEXT:    pand %xmm0, %xmm5
+; SSE2-NEXT:    packuswb %xmm2, %xmm5
+; SSE2-NEXT:    pand %xmm0, %xmm7
+; SSE2-NEXT:    pand %xmm0, %xmm3
+; SSE2-NEXT:    packuswb %xmm7, %xmm3
+; SSE2-NEXT:    packuswb %xmm5, %xmm3
+; SSE2-NEXT:    packuswb %xmm0, %xmm3
+; SSE2-NEXT:    movq %xmm3, (%rsi)
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: trunc_ssat_v8i64_v8i8_store:
 ; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa (%rdi), %xmm9
+; SSSE3-NEXT:    movdqa 16(%rdi), %xmm7
+; SSSE3-NEXT:    movdqa 32(%rdi), %xmm5
+; SSSE3-NEXT:    movdqa 48(%rdi), %xmm2
 ; SSSE3-NEXT:    movdqa {{.*#+}} xmm8 = [127,127]
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm4 = [2147483648,2147483648]
-; SSSE3-NEXT:    movdqa %xmm3, %xmm5
-; SSSE3-NEXT:    pxor %xmm4, %xmm5
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm9 = [2147483775,2147483775]
-; SSSE3-NEXT:    movdqa %xmm9, %xmm7
-; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm7
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm10 = xmm7[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm5
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm10, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm7[1,1,3,3]
-; SSSE3-NEXT:    por %xmm6, %xmm5
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483648,2147483648]
+; SSSE3-NEXT:    movdqa %xmm2, %xmm1
+; SSSE3-NEXT:    pxor %xmm0, %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm10 = [2147483775,2147483775]
+; SSSE3-NEXT:    movdqa %xmm10, %xmm6
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm6[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm3, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm6[1,1,3,3]
+; SSSE3-NEXT:    por %xmm4, %xmm1
+; SSSE3-NEXT:    pand %xmm1, %xmm2
+; SSSE3-NEXT:    pandn %xmm8, %xmm1
+; SSSE3-NEXT:    por %xmm2, %xmm1
+; SSSE3-NEXT:    movdqa %xmm5, %xmm2
+; SSSE3-NEXT:    pxor %xmm0, %xmm2
+; SSSE3-NEXT:    movdqa %xmm10, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm4, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,1,3,3]
+; SSSE3-NEXT:    por %xmm6, %xmm2
+; SSSE3-NEXT:    pand %xmm2, %xmm5
+; SSSE3-NEXT:    pandn %xmm8, %xmm2
+; SSSE3-NEXT:    por %xmm5, %xmm2
+; SSSE3-NEXT:    movdqa %xmm7, %xmm3
+; SSSE3-NEXT:    pxor %xmm0, %xmm3
+; SSSE3-NEXT:    movdqa %xmm10, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
 ; SSSE3-NEXT:    pand %xmm5, %xmm3
-; SSSE3-NEXT:    pandn %xmm8, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[1,1,3,3]
 ; SSSE3-NEXT:    por %xmm3, %xmm5
-; SSSE3-NEXT:    movdqa %xmm2, %xmm3
-; SSSE3-NEXT:    pxor %xmm4, %xmm3
-; SSSE3-NEXT:    movdqa %xmm9, %xmm6
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm10 = xmm6[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm10, %xmm7
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm6[1,1,3,3]
-; SSSE3-NEXT:    por %xmm7, %xmm3
-; SSSE3-NEXT:    pand %xmm3, %xmm2
-; SSSE3-NEXT:    pandn %xmm8, %xmm3
-; SSSE3-NEXT:    por %xmm2, %xmm3
-; SSSE3-NEXT:    movdqa %xmm1, %xmm2
-; SSSE3-NEXT:    pxor %xmm4, %xmm2
-; SSSE3-NEXT:    movdqa %xmm9, %xmm6
-; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm10 = xmm6[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm2[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm10, %xmm7
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[1,1,3,3]
-; SSSE3-NEXT:    por %xmm7, %xmm2
-; SSSE3-NEXT:    pand %xmm2, %xmm1
-; SSSE3-NEXT:    pandn %xmm8, %xmm2
-; SSSE3-NEXT:    por %xmm1, %xmm2
-; SSSE3-NEXT:    movdqa %xmm0, %xmm1
-; SSSE3-NEXT:    pxor %xmm4, %xmm1
-; SSSE3-NEXT:    movdqa %xmm9, %xmm6
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm7, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[1,1,3,3]
-; SSSE3-NEXT:    por %xmm1, %xmm7
-; SSSE3-NEXT:    pand %xmm7, %xmm0
+; SSSE3-NEXT:    pand %xmm5, %xmm7
+; SSSE3-NEXT:    pandn %xmm8, %xmm5
+; SSSE3-NEXT:    por %xmm7, %xmm5
+; SSSE3-NEXT:    movdqa %xmm9, %xmm3
+; SSSE3-NEXT:    pxor %xmm0, %xmm3
+; SSSE3-NEXT:    movdqa %xmm10, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm6, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm3, %xmm7
+; SSSE3-NEXT:    pand %xmm7, %xmm9
 ; SSSE3-NEXT:    pandn %xmm8, %xmm7
-; SSSE3-NEXT:    por %xmm0, %xmm7
+; SSSE3-NEXT:    por %xmm9, %xmm7
 ; SSSE3-NEXT:    movdqa {{.*#+}} xmm8 = [18446744073709551488,18446744073709551488]
-; SSSE3-NEXT:    movdqa %xmm7, %xmm0
-; SSSE3-NEXT:    pxor %xmm4, %xmm0
+; SSSE3-NEXT:    movdqa %xmm7, %xmm3
+; SSSE3-NEXT:    pxor %xmm0, %xmm3
 ; SSSE3-NEXT:    movdqa {{.*#+}} xmm9 = [18446744071562067840,18446744071562067840]
-; SSSE3-NEXT:    movdqa %xmm0, %xmm1
-; SSSE3-NEXT:    pcmpgtd %xmm9, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm10 = xmm1[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm10, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    por %xmm6, %xmm0
-; SSSE3-NEXT:    pand %xmm0, %xmm7
-; SSSE3-NEXT:    pandn %xmm8, %xmm0
-; SSSE3-NEXT:    por %xmm7, %xmm0
-; SSSE3-NEXT:    movdqa %xmm2, %xmm1
-; SSSE3-NEXT:    pxor %xmm4, %xmm1
-; SSSE3-NEXT:    movdqa %xmm1, %xmm6
+; SSSE3-NEXT:    movdqa %xmm3, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm9, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm10 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm10, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm6, %xmm3
+; SSSE3-NEXT:    pand %xmm3, %xmm7
+; SSSE3-NEXT:    pandn %xmm8, %xmm3
+; SSSE3-NEXT:    por %xmm7, %xmm3
+; SSSE3-NEXT:    movdqa %xmm5, %xmm4
+; SSSE3-NEXT:    pxor %xmm0, %xmm4
+; SSSE3-NEXT:    movdqa %xmm4, %xmm6
 ; SSSE3-NEXT:    pcmpgtd %xmm9, %xmm6
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm7, %xmm1
+; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm7, %xmm4
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[1,1,3,3]
-; SSSE3-NEXT:    por %xmm1, %xmm7
-; SSSE3-NEXT:    pand %xmm7, %xmm2
+; SSSE3-NEXT:    por %xmm4, %xmm7
+; SSSE3-NEXT:    pand %xmm7, %xmm5
 ; SSSE3-NEXT:    pandn %xmm8, %xmm7
-; SSSE3-NEXT:    por %xmm2, %xmm7
-; SSSE3-NEXT:    movdqa %xmm3, %xmm1
-; SSSE3-NEXT:    pxor %xmm4, %xmm1
-; SSSE3-NEXT:    movdqa %xmm1, %xmm2
+; SSSE3-NEXT:    por %xmm5, %xmm7
+; SSSE3-NEXT:    movdqa %xmm2, %xmm4
+; SSSE3-NEXT:    pxor %xmm0, %xmm4
+; SSSE3-NEXT:    movdqa %xmm4, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm9, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm6, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSSE3-NEXT:    por %xmm4, %xmm5
+; SSSE3-NEXT:    pand %xmm5, %xmm2
+; SSSE3-NEXT:    pandn %xmm8, %xmm5
+; SSSE3-NEXT:    por %xmm2, %xmm5
+; SSSE3-NEXT:    pxor %xmm1, %xmm0
+; SSSE3-NEXT:    movdqa %xmm0, %xmm2
 ; SSSE3-NEXT:    pcmpgtd %xmm9, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm6, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm4, %xmm0
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSSE3-NEXT:    por %xmm1, %xmm2
-; SSSE3-NEXT:    pand %xmm2, %xmm3
+; SSSE3-NEXT:    por %xmm0, %xmm2
+; SSSE3-NEXT:    pand %xmm2, %xmm1
 ; SSSE3-NEXT:    pandn %xmm8, %xmm2
-; SSSE3-NEXT:    por %xmm3, %xmm2
-; SSSE3-NEXT:    pxor %xmm5, %xmm4
-; SSSE3-NEXT:    movdqa %xmm4, %xmm1
-; SSSE3-NEXT:    pcmpgtd %xmm9, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm3, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    por %xmm4, %xmm1
-; SSSE3-NEXT:    pand %xmm1, %xmm5
-; SSSE3-NEXT:    pandn %xmm8, %xmm1
-; SSSE3-NEXT:    por %xmm5, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm3 = [255,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0]
-; SSSE3-NEXT:    pand %xmm3, %xmm1
-; SSSE3-NEXT:    pand %xmm3, %xmm2
-; SSSE3-NEXT:    packuswb %xmm1, %xmm2
-; SSSE3-NEXT:    pand %xmm3, %xmm7
-; SSSE3-NEXT:    pand %xmm3, %xmm0
-; SSSE3-NEXT:    packuswb %xmm7, %xmm0
-; SSSE3-NEXT:    packuswb %xmm2, %xmm0
-; SSSE3-NEXT:    packuswb %xmm0, %xmm0
-; SSSE3-NEXT:    movq %xmm0, (%rdi)
+; SSSE3-NEXT:    por %xmm1, %xmm2
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [255,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0]
+; SSSE3-NEXT:    pand %xmm0, %xmm2
+; SSSE3-NEXT:    pand %xmm0, %xmm5
+; SSSE3-NEXT:    packuswb %xmm2, %xmm5
+; SSSE3-NEXT:    pand %xmm0, %xmm7
+; SSSE3-NEXT:    pand %xmm0, %xmm3
+; SSSE3-NEXT:    packuswb %xmm7, %xmm3
+; SSSE3-NEXT:    packuswb %xmm5, %xmm3
+; SSSE3-NEXT:    packuswb %xmm0, %xmm3
+; SSSE3-NEXT:    movq %xmm3, (%rsi)
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: trunc_ssat_v8i64_v8i8_store:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa %xmm0, %xmm8
-; SSE41-NEXT:    movapd {{.*#+}} xmm7 = [127,127]
-; SSE41-NEXT:    movdqa {{.*#+}} xmm5 = [2147483648,2147483648]
-; SSE41-NEXT:    movdqa %xmm3, %xmm0
-; SSE41-NEXT:    pxor %xmm5, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm10 = [2147483775,2147483775]
-; SSE41-NEXT:    movdqa %xmm10, %xmm6
+; SSE41-NEXT:    movdqa (%rdi), %xmm9
+; SSE41-NEXT:    movdqa 16(%rdi), %xmm10
+; SSE41-NEXT:    movdqa 32(%rdi), %xmm2
+; SSE41-NEXT:    movdqa 48(%rdi), %xmm4
+; SSE41-NEXT:    movapd {{.*#+}} xmm3 = [127,127]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [2147483648,2147483648]
+; SSE41-NEXT:    movdqa %xmm4, %xmm0
+; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm5 = [2147483775,2147483775]
+; SSE41-NEXT:    movdqa %xmm5, %xmm7
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm7
+; SSE41-NEXT:    movdqa %xmm5, %xmm6
 ; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
+; SSE41-NEXT:    pand %xmm7, %xmm0
 ; SSE41-NEXT:    por %xmm6, %xmm0
-; SSE41-NEXT:    movapd %xmm7, %xmm9
-; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm9
+; SSE41-NEXT:    movapd %xmm3, %xmm8
+; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm8
 ; SSE41-NEXT:    movdqa %xmm2, %xmm0
-; SSE41-NEXT:    pxor %xmm5, %xmm0
-; SSE41-NEXT:    movdqa %xmm10, %xmm3
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    movdqa %xmm5, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT:    movdqa %xmm5, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
 ; SSE41-NEXT:    pand %xmm4, %xmm0
-; SSE41-NEXT:    por %xmm3, %xmm0
-; SSE41-NEXT:    movapd %xmm7, %xmm11
+; SSE41-NEXT:    por %xmm6, %xmm0
+; SSE41-NEXT:    movapd %xmm3, %xmm11
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm11
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    pxor %xmm5, %xmm0
-; SSE41-NEXT:    movdqa %xmm10, %xmm2
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm4, %xmm0
-; SSE41-NEXT:    por %xmm2, %xmm0
-; SSE41-NEXT:    movapd %xmm7, %xmm6
-; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm6
-; SSE41-NEXT:    movdqa %xmm8, %xmm0
-; SSE41-NEXT:    pxor %xmm5, %xmm0
-; SSE41-NEXT:    movdqa %xmm10, %xmm1
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE41-NEXT:    movdqa %xmm10, %xmm0
+; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    movdqa %xmm5, %xmm2
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm2
+; SSE41-NEXT:    movdqa %xmm5, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
 ; SSE41-NEXT:    pand %xmm2, %xmm0
-; SSE41-NEXT:    por %xmm1, %xmm0
-; SSE41-NEXT:    blendvpd %xmm0, %xmm8, %xmm7
-; SSE41-NEXT:    movapd {{.*#+}} xmm1 = [18446744073709551488,18446744073709551488]
-; SSE41-NEXT:    movapd %xmm7, %xmm0
-; SSE41-NEXT:    xorpd %xmm5, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm8 = [18446744071562067840,18446744071562067840]
-; SSE41-NEXT:    movapd %xmm0, %xmm2
-; SSE41-NEXT:    pcmpgtd %xmm8, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm8, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm3, %xmm0
-; SSE41-NEXT:    por %xmm2, %xmm0
-; SSE41-NEXT:    movapd %xmm1, %xmm2
-; SSE41-NEXT:    blendvpd %xmm0, %xmm7, %xmm2
-; SSE41-NEXT:    movapd %xmm6, %xmm0
-; SSE41-NEXT:    xorpd %xmm5, %xmm0
-; SSE41-NEXT:    movapd %xmm0, %xmm3
-; SSE41-NEXT:    pcmpgtd %xmm8, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm8, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm7, %xmm0
-; SSE41-NEXT:    por %xmm3, %xmm0
-; SSE41-NEXT:    movapd %xmm1, %xmm7
-; SSE41-NEXT:    blendvpd %xmm0, %xmm6, %xmm7
-; SSE41-NEXT:    movapd %xmm11, %xmm0
-; SSE41-NEXT:    xorpd %xmm5, %xmm0
-; SSE41-NEXT:    movapd %xmm0, %xmm3
-; SSE41-NEXT:    pcmpgtd %xmm8, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm8, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE41-NEXT:    por %xmm6, %xmm0
+; SSE41-NEXT:    movapd %xmm3, %xmm2
+; SSE41-NEXT:    blendvpd %xmm0, %xmm10, %xmm2
+; SSE41-NEXT:    movdqa %xmm9, %xmm0
+; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    movdqa %xmm5, %xmm6
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm5
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2]
 ; SSE41-NEXT:    pand %xmm6, %xmm0
-; SSE41-NEXT:    por %xmm3, %xmm0
-; SSE41-NEXT:    movapd %xmm1, %xmm3
-; SSE41-NEXT:    blendvpd %xmm0, %xmm11, %xmm3
-; SSE41-NEXT:    xorpd %xmm9, %xmm5
-; SSE41-NEXT:    movapd %xmm5, %xmm6
-; SSE41-NEXT:    pcmpgtd %xmm8, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm8, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[1,1,3,3]
+; SSE41-NEXT:    por %xmm5, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm9, %xmm3
+; SSE41-NEXT:    movapd {{.*#+}} xmm5 = [18446744073709551488,18446744073709551488]
+; SSE41-NEXT:    movapd %xmm3, %xmm6
+; SSE41-NEXT:    xorpd %xmm1, %xmm6
+; SSE41-NEXT:    movdqa {{.*#+}} xmm7 = [18446744071562067840,18446744071562067840]
+; SSE41-NEXT:    movapd %xmm6, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm7, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm7, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
 ; SSE41-NEXT:    pand %xmm4, %xmm0
 ; SSE41-NEXT:    por %xmm6, %xmm0
-; SSE41-NEXT:    blendvpd %xmm0, %xmm9, %xmm1
+; SSE41-NEXT:    movapd %xmm5, %xmm6
+; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm6
+; SSE41-NEXT:    movapd %xmm2, %xmm3
+; SSE41-NEXT:    xorpd %xmm1, %xmm3
+; SSE41-NEXT:    movapd %xmm3, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm7, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm7, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm3, %xmm0
+; SSE41-NEXT:    movapd %xmm5, %xmm3
+; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm3
+; SSE41-NEXT:    movapd %xmm11, %xmm2
+; SSE41-NEXT:    xorpd %xmm1, %xmm2
+; SSE41-NEXT:    movapd %xmm2, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm7, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm7, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    movapd %xmm5, %xmm2
+; SSE41-NEXT:    blendvpd %xmm0, %xmm11, %xmm2
+; SSE41-NEXT:    xorpd %xmm8, %xmm1
+; SSE41-NEXT:    movapd %xmm1, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm7, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm7, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm1, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm8, %xmm5
 ; SSE41-NEXT:    movapd {{.*#+}} xmm0 = [255,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0]
-; SSE41-NEXT:    andpd %xmm0, %xmm1
-; SSE41-NEXT:    andpd %xmm0, %xmm3
-; SSE41-NEXT:    packusdw %xmm1, %xmm3
-; SSE41-NEXT:    andpd %xmm0, %xmm7
+; SSE41-NEXT:    andpd %xmm0, %xmm5
 ; SSE41-NEXT:    andpd %xmm0, %xmm2
-; SSE41-NEXT:    packusdw %xmm7, %xmm2
-; SSE41-NEXT:    packusdw %xmm3, %xmm2
-; SSE41-NEXT:    packuswb %xmm2, %xmm2
-; SSE41-NEXT:    movq %xmm2, (%rdi)
+; SSE41-NEXT:    packusdw %xmm5, %xmm2
+; SSE41-NEXT:    andpd %xmm0, %xmm3
+; SSE41-NEXT:    andpd %xmm0, %xmm6
+; SSE41-NEXT:    packusdw %xmm3, %xmm6
+; SSE41-NEXT:    packusdw %xmm2, %xmm6
+; SSE41-NEXT:    packuswb %xmm0, %xmm6
+; SSE41-NEXT:    movq %xmm6, (%rsi)
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: trunc_ssat_v8i64_v8i8_store:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovapd {{.*#+}} ymm8 = [127,127,127,127]
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [127,127]
-; AVX1-NEXT:    vpcmpgtq %xmm3, %xmm4, %xmm5
-; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm4, %xmm6
-; AVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm6, %ymm7
-; AVX1-NEXT:    vblendvpd %ymm7, %ymm1, %ymm8, %ymm9
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm4, %xmm7
-; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm4, %xmm10
-; AVX1-NEXT:    vinsertf128 $1, %xmm7, %ymm10, %ymm11
-; AVX1-NEXT:    vblendvpd %ymm11, %ymm0, %ymm8, %ymm8
+; AVX1-NEXT:    vmovapd {{.*#+}} ymm9 = [127,127,127,127]
+; AVX1-NEXT:    vmovdqa 48(%rdi), %xmm1
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [127,127]
+; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm2, %xmm3
+; AVX1-NEXT:    vmovdqa (%rdi), %xmm4
+; AVX1-NEXT:    vmovdqa 16(%rdi), %xmm5
+; AVX1-NEXT:    vmovdqa 32(%rdi), %xmm6
+; AVX1-NEXT:    vpcmpgtq %xmm6, %xmm2, %xmm7
+; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm7, %ymm8
+; AVX1-NEXT:    vblendvpd %ymm8, 32(%rdi), %ymm9, %ymm8
+; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm2, %xmm0
+; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm2, %xmm10
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm10, %ymm11
+; AVX1-NEXT:    vblendvpd %ymm11, (%rdi), %ymm9, %ymm9
 ; AVX1-NEXT:    vmovapd {{.*#+}} ymm11 = [18446744073709551488,18446744073709551488,18446744073709551488,18446744073709551488]
-; AVX1-NEXT:    vblendvpd %xmm7, %xmm2, %xmm4, %xmm2
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm7 = [18446744073709551488,18446744073709551488]
-; AVX1-NEXT:    vpcmpgtq %xmm7, %xmm2, %xmm2
-; AVX1-NEXT:    vblendvpd %xmm10, %xmm0, %xmm4, %xmm0
-; AVX1-NEXT:    vpcmpgtq %xmm7, %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; AVX1-NEXT:    vblendvpd %ymm0, %ymm8, %ymm11, %ymm0
-; AVX1-NEXT:    vblendvpd %xmm5, %xmm3, %xmm4, %xmm2
-; AVX1-NEXT:    vpcmpgtq %xmm7, %xmm2, %xmm2
-; AVX1-NEXT:    vblendvpd %xmm6, %xmm1, %xmm4, %xmm1
-; AVX1-NEXT:    vpcmpgtq %xmm7, %xmm1, %xmm1
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
-; AVX1-NEXT:    vblendvpd %ymm1, %ymm9, %ymm11, %ymm1
+; AVX1-NEXT:    vblendvpd %xmm0, %xmm5, %xmm2, %xmm0
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm5 = [18446744073709551488,18446744073709551488]
+; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm0, %xmm0
+; AVX1-NEXT:    vblendvpd %xmm10, %xmm4, %xmm2, %xmm4
+; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm4, %ymm0
+; AVX1-NEXT:    vblendvpd %ymm0, %ymm9, %ymm11, %ymm0
+; AVX1-NEXT:    vblendvpd %xmm3, %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm1, %xmm1
+; AVX1-NEXT:    vblendvpd %xmm7, %xmm6, %xmm2, %xmm2
+; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm2, %xmm2
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
+; AVX1-NEXT:    vblendvpd %ymm1, %ymm8, %ymm11, %ymm1
 ; AVX1-NEXT:    vmovapd {{.*#+}} ymm2 = [255,255,255,255]
 ; AVX1-NEXT:    vandpd %ymm2, %ymm1, %ymm1
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
@@ -1954,12 +4167,14 @@ define void @trunc_ssat_v8i64_v8i8_store(<8 x i64> %a0, <8 x i8> *%p1) {
 ; AVX1-NEXT:    vpackusdw %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
-; AVX1-NEXT:    vmovq %xmm0, (%rdi)
+; AVX1-NEXT:    vmovq %xmm0, (%rsi)
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: trunc_ssat_v8i64_v8i8_store:
 ; AVX2:       # %bb.0:
+; AVX2-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX2-NEXT:    vmovdqa 32(%rdi), %ymm1
 ; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [127,127,127,127]
 ; AVX2-NEXT:    vpcmpgtq %ymm1, %ymm2, %ymm3
 ; AVX2-NEXT:    vblendvpd %ymm3, %ymm1, %ymm2, %ymm1
@@ -1971,23 +4186,38 @@ define void @trunc_ssat_v8i64_v8i8_store(<8 x i64> %a0, <8 x i8> *%p1) {
 ; AVX2-NEXT:    vpcmpgtq %ymm2, %ymm1, %ymm3
 ; AVX2-NEXT:    vblendvpd %ymm3, %ymm1, %ymm2, %ymm1
 ; AVX2-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX2-NEXT:    vpackssdw %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
-; AVX2-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm3
-; AVX2-NEXT:    vpackssdw %xmm3, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; AVX2-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
-; AVX2-NEXT:    vmovq %xmm0, (%rdi)
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm3 = <u,u,0,8,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX2-NEXT:    vpshufb %xmm3, %xmm2, %xmm2
+; AVX2-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
+; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm3 = <0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX2-NEXT:    vpshufb %xmm3, %xmm2, %xmm2
+; AVX2-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
+; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3]
+; AVX2-NEXT:    vmovq %xmm0, (%rsi)
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: trunc_ssat_v8i64_v8i8_store:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpmovsqb %zmm0, (%rdi)
+; AVX512-NEXT:    vmovdqa64 (%rdi), %zmm0
+; AVX512-NEXT:    vpmovsqb %zmm0, (%rsi)
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
+;
+; SKX-LABEL: trunc_ssat_v8i64_v8i8_store:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vmovdqa (%rdi), %ymm0
+; SKX-NEXT:    vmovdqa 32(%rdi), %ymm1
+; SKX-NEXT:    vpmovsqb %ymm1, %xmm1
+; SKX-NEXT:    vpmovsqb %ymm0, %xmm0
+; SKX-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SKX-NEXT:    vmovq %xmm0, (%rsi)
+; SKX-NEXT:    vzeroupper
+; SKX-NEXT:    retq
+  %a0 = load <8 x i64>, <8 x i64>* %p0
   %1 = icmp slt <8 x i64> %a0, <i64 127, i64 127, i64 127, i64 127, i64 127, i64 127, i64 127, i64 127>
   %2 = select <8 x i1> %1, <8 x i64> %a0, <8 x i64> <i64 127, i64 127, i64 127, i64 127, i64 127, i64 127, i64 127, i64 127>
   %3 = icmp sgt <8 x i64> %2, <i64 -128, i64 -128, i64 -128, i64 -128, i64 -128, i64 -128, i64 -128, i64 -128>
@@ -1997,696 +4227,717 @@ define void @trunc_ssat_v8i64_v8i8_store(<8 x i64> %a0, <8 x i8> *%p1) {
   ret void
 }
 
-define <16 x i8> @trunc_ssat_v16i64_v16i8(<16 x i64> %a0) {
+define <16 x i8> @trunc_ssat_v16i64_v16i8(<16 x i64>* %p0) "min-legal-vector-width"="256" {
 ; SSE2-LABEL: trunc_ssat_v16i64_v16i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [127,127]
-; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [2147483648,2147483648]
-; SSE2-NEXT:    movdqa %xmm6, %xmm9
-; SSE2-NEXT:    pxor %xmm8, %xmm9
-; SSE2-NEXT:    movdqa {{.*#+}} xmm11 = [2147483775,2147483775]
-; SSE2-NEXT:    movdqa %xmm11, %xmm12
-; SSE2-NEXT:    pcmpgtd %xmm9, %xmm12
-; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm12[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm11, %xmm9
-; SSE2-NEXT:    pshufd {{.*#+}} xmm14 = xmm9[1,1,3,3]
-; SSE2-NEXT:    pand %xmm13, %xmm14
-; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm12[1,1,3,3]
-; SSE2-NEXT:    por %xmm14, %xmm9
-; SSE2-NEXT:    pand %xmm9, %xmm6
-; SSE2-NEXT:    pandn %xmm10, %xmm9
-; SSE2-NEXT:    por %xmm6, %xmm9
-; SSE2-NEXT:    movdqa %xmm7, %xmm6
-; SSE2-NEXT:    pxor %xmm8, %xmm6
-; SSE2-NEXT:    movdqa %xmm11, %xmm12
-; SSE2-NEXT:    pcmpgtd %xmm6, %xmm12
-; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm12[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm11, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT:    pand %xmm13, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm12[1,1,3,3]
-; SSE2-NEXT:    por %xmm6, %xmm12
-; SSE2-NEXT:    pand %xmm12, %xmm7
-; SSE2-NEXT:    pandn %xmm10, %xmm12
-; SSE2-NEXT:    por %xmm7, %xmm12
-; SSE2-NEXT:    movdqa %xmm4, %xmm6
-; SSE2-NEXT:    pxor %xmm8, %xmm6
-; SSE2-NEXT:    movdqa %xmm11, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm6, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm7[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm11, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT:    pand %xmm13, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm7[1,1,3,3]
-; SSE2-NEXT:    por %xmm6, %xmm13
-; SSE2-NEXT:    pand %xmm13, %xmm4
-; SSE2-NEXT:    pandn %xmm10, %xmm13
-; SSE2-NEXT:    por %xmm4, %xmm13
-; SSE2-NEXT:    movdqa %xmm5, %xmm4
-; SSE2-NEXT:    pxor %xmm8, %xmm4
-; SSE2-NEXT:    movdqa %xmm11, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm4, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm11, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    pand %xmm7, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm14 = xmm6[1,1,3,3]
-; SSE2-NEXT:    por %xmm4, %xmm14
-; SSE2-NEXT:    pand %xmm14, %xmm5
-; SSE2-NEXT:    pandn %xmm10, %xmm14
-; SSE2-NEXT:    por %xmm5, %xmm14
-; SSE2-NEXT:    movdqa %xmm2, %xmm4
-; SSE2-NEXT:    pxor %xmm8, %xmm4
-; SSE2-NEXT:    movdqa %xmm11, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm4, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm11, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm4
+; SSE2-NEXT:    movdqa (%rdi), %xmm10
+; SSE2-NEXT:    movdqa 16(%rdi), %xmm9
+; SSE2-NEXT:    movdqa 32(%rdi), %xmm15
+; SSE2-NEXT:    movdqa 48(%rdi), %xmm13
+; SSE2-NEXT:    movdqa 80(%rdi), %xmm6
+; SSE2-NEXT:    movdqa 64(%rdi), %xmm3
+; SSE2-NEXT:    movdqa 112(%rdi), %xmm4
+; SSE2-NEXT:    movdqa 96(%rdi), %xmm7
+; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [127,127]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm7, %xmm5
+; SSE2-NEXT:    pxor %xmm1, %xmm5
+; SSE2-NEXT:    movdqa {{.*#+}} xmm14 = [2147483775,2147483775]
+; SSE2-NEXT:    movdqa %xmm14, %xmm0
+; SSE2-NEXT:    pcmpgtd %xmm5, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm14, %xmm5
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE2-NEXT:    por %xmm4, %xmm5
-; SSE2-NEXT:    pand %xmm5, %xmm2
-; SSE2-NEXT:    pandn %xmm10, %xmm5
-; SSE2-NEXT:    por %xmm2, %xmm5
-; SSE2-NEXT:    movdqa %xmm3, %xmm2
-; SSE2-NEXT:    pxor %xmm8, %xmm2
-; SSE2-NEXT:    movdqa %xmm11, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm11, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm2, %xmm6
-; SSE2-NEXT:    pand %xmm6, %xmm3
-; SSE2-NEXT:    pandn %xmm10, %xmm6
-; SSE2-NEXT:    por %xmm3, %xmm6
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    pxor %xmm8, %xmm2
-; SSE2-NEXT:    movdqa %xmm11, %xmm3
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm11, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT:    pand %xmm4, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE2-NEXT:    por %xmm2, %xmm3
-; SSE2-NEXT:    pand %xmm3, %xmm0
-; SSE2-NEXT:    pandn %xmm10, %xmm3
-; SSE2-NEXT:    por %xmm0, %xmm3
-; SSE2-NEXT:    movdqa %xmm1, %xmm0
-; SSE2-NEXT:    pxor %xmm8, %xmm0
-; SSE2-NEXT:    movdqa %xmm11, %xmm2
+; SSE2-NEXT:    pand %xmm2, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm0[1,1,3,3]
+; SSE2-NEXT:    por %xmm5, %xmm11
+; SSE2-NEXT:    pand %xmm11, %xmm7
+; SSE2-NEXT:    pandn %xmm8, %xmm11
+; SSE2-NEXT:    por %xmm7, %xmm11
+; SSE2-NEXT:    movdqa %xmm4, %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm0
+; SSE2-NEXT:    movdqa %xmm14, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm14, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pand %xmm5, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm2[1,1,3,3]
+; SSE2-NEXT:    por %xmm0, %xmm12
+; SSE2-NEXT:    pand %xmm12, %xmm4
+; SSE2-NEXT:    pandn %xmm8, %xmm12
+; SSE2-NEXT:    por %xmm4, %xmm12
+; SSE2-NEXT:    movdqa %xmm3, %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm0
+; SSE2-NEXT:    movdqa %xmm14, %xmm2
 ; SSE2-NEXT:    pcmpgtd %xmm0, %xmm2
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm11, %xmm0
+; SSE2-NEXT:    pcmpeqd %xmm14, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm4, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,1,3,3]
 ; SSE2-NEXT:    por %xmm0, %xmm4
-; SSE2-NEXT:    pand %xmm4, %xmm1
-; SSE2-NEXT:    pandn %xmm10, %xmm4
-; SSE2-NEXT:    por %xmm1, %xmm4
-; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [18446744073709551488,18446744073709551488]
-; SSE2-NEXT:    movdqa %xmm4, %xmm0
-; SSE2-NEXT:    pxor %xmm8, %xmm0
-; SSE2-NEXT:    movdqa {{.*#+}} xmm11 = [18446744071562067840,18446744071562067840]
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm11, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm11, %xmm0
+; SSE2-NEXT:    pand %xmm4, %xmm3
+; SSE2-NEXT:    pandn %xmm8, %xmm4
+; SSE2-NEXT:    por %xmm3, %xmm4
+; SSE2-NEXT:    movdqa %xmm6, %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm0
+; SSE2-NEXT:    movdqa %xmm14, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm14, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm0, %xmm1
-; SSE2-NEXT:    pand %xmm1, %xmm4
-; SSE2-NEXT:    pandn %xmm10, %xmm1
-; SSE2-NEXT:    por %xmm4, %xmm1
-; SSE2-NEXT:    movdqa %xmm3, %xmm0
-; SSE2-NEXT:    pxor %xmm8, %xmm0
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[1,1,3,3]
+; SSE2-NEXT:    por %xmm0, %xmm5
+; SSE2-NEXT:    pand %xmm5, %xmm6
+; SSE2-NEXT:    pandn %xmm8, %xmm5
+; SSE2-NEXT:    por %xmm6, %xmm5
+; SSE2-NEXT:    movdqa %xmm15, %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm0
+; SSE2-NEXT:    movdqa %xmm14, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm14, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[1,1,3,3]
+; SSE2-NEXT:    por %xmm0, %xmm6
+; SSE2-NEXT:    pand %xmm6, %xmm15
+; SSE2-NEXT:    pandn %xmm8, %xmm6
+; SSE2-NEXT:    por %xmm15, %xmm6
+; SSE2-NEXT:    movdqa %xmm13, %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm0
+; SSE2-NEXT:    movdqa %xmm14, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm14, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm15 = xmm2[1,1,3,3]
+; SSE2-NEXT:    por %xmm0, %xmm15
+; SSE2-NEXT:    pand %xmm15, %xmm13
+; SSE2-NEXT:    pandn %xmm8, %xmm15
+; SSE2-NEXT:    por %xmm13, %xmm15
+; SSE2-NEXT:    movdqa %xmm10, %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm0
+; SSE2-NEXT:    movdqa %xmm14, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm3[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm14, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pand %xmm7, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm3[1,1,3,3]
+; SSE2-NEXT:    por %xmm0, %xmm13
+; SSE2-NEXT:    pand %xmm13, %xmm10
+; SSE2-NEXT:    pandn %xmm8, %xmm13
+; SSE2-NEXT:    por %xmm10, %xmm13
+; SSE2-NEXT:    movdqa %xmm9, %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm0
+; SSE2-NEXT:    movdqa %xmm14, %xmm7
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm7
+; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm7[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm14, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pand %xmm10, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
+; SSE2-NEXT:    por %xmm0, %xmm7
+; SSE2-NEXT:    pand %xmm7, %xmm9
+; SSE2-NEXT:    pandn %xmm8, %xmm7
+; SSE2-NEXT:    por %xmm9, %xmm7
+; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [18446744073709551488,18446744073709551488]
+; SSE2-NEXT:    movdqa %xmm7, %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [18446744071562067840,18446744071562067840]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd %xmm11, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm11, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pand %xmm4, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3]
-; SSE2-NEXT:    por %xmm7, %xmm0
-; SSE2-NEXT:    pand %xmm0, %xmm3
-; SSE2-NEXT:    pandn %xmm10, %xmm0
+; SSE2-NEXT:    pcmpgtd %xmm9, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm2[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm9, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pand %xmm10, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE2-NEXT:    por %xmm0, %xmm2
+; SSE2-NEXT:    pand %xmm2, %xmm7
+; SSE2-NEXT:    pandn %xmm8, %xmm2
+; SSE2-NEXT:    por %xmm7, %xmm2
+; SSE2-NEXT:    movdqa %xmm13, %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm7
+; SSE2-NEXT:    pcmpgtd %xmm9, %xmm7
+; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm7[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm9, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pand %xmm10, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[1,1,3,3]
 ; SSE2-NEXT:    por %xmm3, %xmm0
-; SSE2-NEXT:    packssdw %xmm1, %xmm0
-; SSE2-NEXT:    movdqa %xmm6, %xmm1
-; SSE2-NEXT:    pxor %xmm8, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm2
-; SSE2-NEXT:    pcmpgtd %xmm11, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm11, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pand %xmm3, %xmm1
+; SSE2-NEXT:    pand %xmm0, %xmm13
+; SSE2-NEXT:    pandn %xmm8, %xmm0
+; SSE2-NEXT:    por %xmm13, %xmm0
+; SSE2-NEXT:    packssdw %xmm2, %xmm0
+; SSE2-NEXT:    movdqa %xmm15, %xmm2
+; SSE2-NEXT:    pxor %xmm1, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm9, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm3[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm9, %xmm2
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm2
-; SSE2-NEXT:    pand %xmm2, %xmm6
-; SSE2-NEXT:    pandn %xmm10, %xmm2
-; SSE2-NEXT:    por %xmm6, %xmm2
-; SSE2-NEXT:    movdqa %xmm5, %xmm1
-; SSE2-NEXT:    pxor %xmm8, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm3
-; SSE2-NEXT:    pcmpgtd %xmm11, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm11, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pand %xmm4, %xmm1
+; SSE2-NEXT:    pand %xmm7, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSE2-NEXT:    por %xmm2, %xmm3
+; SSE2-NEXT:    pand %xmm3, %xmm15
+; SSE2-NEXT:    pandn %xmm8, %xmm3
+; SSE2-NEXT:    por %xmm15, %xmm3
+; SSE2-NEXT:    movdqa %xmm6, %xmm2
+; SSE2-NEXT:    pxor %xmm1, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm7
+; SSE2-NEXT:    pcmpgtd %xmm9, %xmm7
+; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm7[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm9, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE2-NEXT:    pand %xmm10, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
+; SSE2-NEXT:    por %xmm2, %xmm7
+; SSE2-NEXT:    pand %xmm7, %xmm6
+; SSE2-NEXT:    pandn %xmm8, %xmm7
+; SSE2-NEXT:    por %xmm6, %xmm7
+; SSE2-NEXT:    packssdw %xmm3, %xmm7
+; SSE2-NEXT:    packssdw %xmm7, %xmm0
+; SSE2-NEXT:    movdqa %xmm5, %xmm2
+; SSE2-NEXT:    pxor %xmm1, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm9, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm9, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE2-NEXT:    pand %xmm6, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSE2-NEXT:    por %xmm2, %xmm3
+; SSE2-NEXT:    pand %xmm3, %xmm5
+; SSE2-NEXT:    pandn %xmm8, %xmm3
+; SSE2-NEXT:    por %xmm5, %xmm3
+; SSE2-NEXT:    movdqa %xmm4, %xmm2
+; SSE2-NEXT:    pxor %xmm1, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm9, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm9, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm2[1,1,3,3]
+; SSE2-NEXT:    pand %xmm6, %xmm7
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm5[1,1,3,3]
+; SSE2-NEXT:    por %xmm7, %xmm2
+; SSE2-NEXT:    pand %xmm2, %xmm4
+; SSE2-NEXT:    pandn %xmm8, %xmm2
+; SSE2-NEXT:    por %xmm4, %xmm2
+; SSE2-NEXT:    packssdw %xmm3, %xmm2
+; SSE2-NEXT:    movdqa %xmm12, %xmm3
+; SSE2-NEXT:    pxor %xmm1, %xmm3
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm9, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm9, %xmm3
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm3
-; SSE2-NEXT:    pand %xmm3, %xmm5
-; SSE2-NEXT:    pandn %xmm10, %xmm3
-; SSE2-NEXT:    por %xmm5, %xmm3
-; SSE2-NEXT:    packssdw %xmm2, %xmm3
-; SSE2-NEXT:    packssdw %xmm3, %xmm0
-; SSE2-NEXT:    movdqa %xmm14, %xmm1
-; SSE2-NEXT:    pxor %xmm8, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm2
-; SSE2-NEXT:    pcmpgtd %xmm11, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm11, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pand %xmm3, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm2
-; SSE2-NEXT:    pand %xmm2, %xmm14
-; SSE2-NEXT:    pandn %xmm10, %xmm2
-; SSE2-NEXT:    por %xmm14, %xmm2
-; SSE2-NEXT:    movdqa %xmm13, %xmm1
-; SSE2-NEXT:    pxor %xmm8, %xmm1
+; SSE2-NEXT:    pand %xmm5, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm3, %xmm4
+; SSE2-NEXT:    pand %xmm4, %xmm12
+; SSE2-NEXT:    pandn %xmm8, %xmm4
+; SSE2-NEXT:    por %xmm12, %xmm4
+; SSE2-NEXT:    pxor %xmm11, %xmm1
 ; SSE2-NEXT:    movdqa %xmm1, %xmm3
-; SSE2-NEXT:    pcmpgtd %xmm11, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm11, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm9, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm9, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pand %xmm4, %xmm1
+; SSE2-NEXT:    pand %xmm5, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
 ; SSE2-NEXT:    por %xmm1, %xmm3
-; SSE2-NEXT:    pand %xmm3, %xmm13
-; SSE2-NEXT:    pandn %xmm10, %xmm3
-; SSE2-NEXT:    por %xmm13, %xmm3
-; SSE2-NEXT:    packssdw %xmm2, %xmm3
-; SSE2-NEXT:    movdqa %xmm12, %xmm1
-; SSE2-NEXT:    pxor %xmm8, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm2
-; SSE2-NEXT:    pcmpgtd %xmm11, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm11, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pand %xmm4, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm2
-; SSE2-NEXT:    pand %xmm2, %xmm12
-; SSE2-NEXT:    pandn %xmm10, %xmm2
-; SSE2-NEXT:    por %xmm12, %xmm2
-; SSE2-NEXT:    pxor %xmm9, %xmm8
-; SSE2-NEXT:    movdqa %xmm8, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm11, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm11, %xmm8
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm8[1,1,3,3]
-; SSE2-NEXT:    pand %xmm4, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm5, %xmm1
-; SSE2-NEXT:    pand %xmm1, %xmm9
-; SSE2-NEXT:    pandn %xmm10, %xmm1
-; SSE2-NEXT:    por %xmm9, %xmm1
-; SSE2-NEXT:    packssdw %xmm2, %xmm1
-; SSE2-NEXT:    packssdw %xmm1, %xmm3
-; SSE2-NEXT:    packsswb %xmm3, %xmm0
+; SSE2-NEXT:    pand %xmm3, %xmm11
+; SSE2-NEXT:    pandn %xmm8, %xmm3
+; SSE2-NEXT:    por %xmm11, %xmm3
+; SSE2-NEXT:    packssdw %xmm4, %xmm3
+; SSE2-NEXT:    packssdw %xmm3, %xmm2
+; SSE2-NEXT:    packsswb %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: trunc_ssat_v16i64_v16i8:
 ; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm10 = [127,127]
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm8 = [2147483648,2147483648]
-; SSSE3-NEXT:    movdqa %xmm6, %xmm9
-; SSSE3-NEXT:    pxor %xmm8, %xmm9
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm11 = [2147483775,2147483775]
-; SSSE3-NEXT:    movdqa %xmm11, %xmm12
-; SSSE3-NEXT:    pcmpgtd %xmm9, %xmm12
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm13 = xmm12[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm11, %xmm9
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm14 = xmm9[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm13, %xmm14
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm9 = xmm12[1,1,3,3]
-; SSSE3-NEXT:    por %xmm14, %xmm9
-; SSSE3-NEXT:    pand %xmm9, %xmm6
-; SSSE3-NEXT:    pandn %xmm10, %xmm9
-; SSSE3-NEXT:    por %xmm6, %xmm9
-; SSSE3-NEXT:    movdqa %xmm7, %xmm6
-; SSSE3-NEXT:    pxor %xmm8, %xmm6
-; SSSE3-NEXT:    movdqa %xmm11, %xmm12
-; SSSE3-NEXT:    pcmpgtd %xmm6, %xmm12
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm13 = xmm12[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm11, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm13, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm12 = xmm12[1,1,3,3]
-; SSSE3-NEXT:    por %xmm6, %xmm12
-; SSSE3-NEXT:    pand %xmm12, %xmm7
-; SSSE3-NEXT:    pandn %xmm10, %xmm12
-; SSSE3-NEXT:    por %xmm7, %xmm12
-; SSSE3-NEXT:    movdqa %xmm4, %xmm6
-; SSSE3-NEXT:    pxor %xmm8, %xmm6
-; SSSE3-NEXT:    movdqa %xmm11, %xmm7
-; SSSE3-NEXT:    pcmpgtd %xmm6, %xmm7
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm13 = xmm7[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm11, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm13, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm13 = xmm7[1,1,3,3]
-; SSSE3-NEXT:    por %xmm6, %xmm13
-; SSSE3-NEXT:    pand %xmm13, %xmm4
-; SSSE3-NEXT:    pandn %xmm10, %xmm13
-; SSSE3-NEXT:    por %xmm4, %xmm13
-; SSSE3-NEXT:    movdqa %xmm5, %xmm4
-; SSSE3-NEXT:    pxor %xmm8, %xmm4
-; SSSE3-NEXT:    movdqa %xmm11, %xmm6
-; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm11, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm7, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm14 = xmm6[1,1,3,3]
-; SSSE3-NEXT:    por %xmm4, %xmm14
-; SSSE3-NEXT:    pand %xmm14, %xmm5
-; SSSE3-NEXT:    pandn %xmm10, %xmm14
-; SSSE3-NEXT:    por %xmm5, %xmm14
-; SSSE3-NEXT:    movdqa %xmm2, %xmm4
-; SSSE3-NEXT:    pxor %xmm8, %xmm4
-; SSSE3-NEXT:    movdqa %xmm11, %xmm5
-; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm5
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm11, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm6, %xmm4
+; SSSE3-NEXT:    movdqa (%rdi), %xmm10
+; SSSE3-NEXT:    movdqa 16(%rdi), %xmm9
+; SSSE3-NEXT:    movdqa 32(%rdi), %xmm15
+; SSSE3-NEXT:    movdqa 48(%rdi), %xmm13
+; SSSE3-NEXT:    movdqa 80(%rdi), %xmm6
+; SSSE3-NEXT:    movdqa 64(%rdi), %xmm3
+; SSSE3-NEXT:    movdqa 112(%rdi), %xmm4
+; SSSE3-NEXT:    movdqa 96(%rdi), %xmm7
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm8 = [127,127]
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [2147483648,2147483648]
+; SSSE3-NEXT:    movdqa %xmm7, %xmm5
+; SSSE3-NEXT:    pxor %xmm1, %xmm5
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm14 = [2147483775,2147483775]
+; SSSE3-NEXT:    movdqa %xmm14, %xmm0
+; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm14, %xmm5
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSSE3-NEXT:    por %xmm4, %xmm5
-; SSSE3-NEXT:    pand %xmm5, %xmm2
-; SSSE3-NEXT:    pandn %xmm10, %xmm5
-; SSSE3-NEXT:    por %xmm2, %xmm5
-; SSSE3-NEXT:    movdqa %xmm3, %xmm2
-; SSSE3-NEXT:    pxor %xmm8, %xmm2
-; SSSE3-NEXT:    movdqa %xmm11, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm11, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm6, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm6
-; SSSE3-NEXT:    pand %xmm6, %xmm3
-; SSSE3-NEXT:    pandn %xmm10, %xmm6
-; SSSE3-NEXT:    por %xmm3, %xmm6
-; SSSE3-NEXT:    movdqa %xmm0, %xmm2
-; SSSE3-NEXT:    pxor %xmm8, %xmm2
-; SSSE3-NEXT:    movdqa %xmm11, %xmm3
-; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm11, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm4, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm3
-; SSSE3-NEXT:    pand %xmm3, %xmm0
-; SSSE3-NEXT:    pandn %xmm10, %xmm3
-; SSSE3-NEXT:    por %xmm0, %xmm3
-; SSSE3-NEXT:    movdqa %xmm1, %xmm0
-; SSSE3-NEXT:    pxor %xmm8, %xmm0
-; SSSE3-NEXT:    movdqa %xmm11, %xmm2
+; SSSE3-NEXT:    pand %xmm2, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm11 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    por %xmm5, %xmm11
+; SSSE3-NEXT:    pand %xmm11, %xmm7
+; SSSE3-NEXT:    pandn %xmm8, %xmm11
+; SSSE3-NEXT:    por %xmm7, %xmm11
+; SSSE3-NEXT:    movdqa %xmm4, %xmm0
+; SSSE3-NEXT:    pxor %xmm1, %xmm0
+; SSSE3-NEXT:    movdqa %xmm14, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm14, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm5, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm12 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    por %xmm0, %xmm12
+; SSSE3-NEXT:    pand %xmm12, %xmm4
+; SSSE3-NEXT:    pandn %xmm8, %xmm12
+; SSSE3-NEXT:    por %xmm4, %xmm12
+; SSSE3-NEXT:    movdqa %xmm3, %xmm0
+; SSSE3-NEXT:    pxor %xmm1, %xmm0
+; SSSE3-NEXT:    movdqa %xmm14, %xmm2
 ; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm2
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm11, %xmm0
+; SSSE3-NEXT:    pcmpeqd %xmm14, %xmm0
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
 ; SSSE3-NEXT:    pand %xmm4, %xmm0
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,1,3,3]
 ; SSSE3-NEXT:    por %xmm0, %xmm4
-; SSSE3-NEXT:    pand %xmm4, %xmm1
-; SSSE3-NEXT:    pandn %xmm10, %xmm4
-; SSSE3-NEXT:    por %xmm1, %xmm4
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm10 = [18446744073709551488,18446744073709551488]
-; SSSE3-NEXT:    movdqa %xmm4, %xmm0
-; SSSE3-NEXT:    pxor %xmm8, %xmm0
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm11 = [18446744071562067840,18446744071562067840]
-; SSSE3-NEXT:    movdqa %xmm0, %xmm1
-; SSSE3-NEXT:    pcmpgtd %xmm11, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm11, %xmm0
+; SSSE3-NEXT:    pand %xmm4, %xmm3
+; SSSE3-NEXT:    pandn %xmm8, %xmm4
+; SSSE3-NEXT:    por %xmm3, %xmm4
+; SSSE3-NEXT:    movdqa %xmm6, %xmm0
+; SSSE3-NEXT:    pxor %xmm1, %xmm0
+; SSSE3-NEXT:    movdqa %xmm14, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm14, %xmm0
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm2, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    por %xmm0, %xmm1
-; SSSE3-NEXT:    pand %xmm1, %xmm4
-; SSSE3-NEXT:    pandn %xmm10, %xmm1
-; SSSE3-NEXT:    por %xmm4, %xmm1
-; SSSE3-NEXT:    movdqa %xmm3, %xmm0
-; SSSE3-NEXT:    pxor %xmm8, %xmm0
+; SSSE3-NEXT:    pand %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    por %xmm0, %xmm5
+; SSSE3-NEXT:    pand %xmm5, %xmm6
+; SSSE3-NEXT:    pandn %xmm8, %xmm5
+; SSSE3-NEXT:    por %xmm6, %xmm5
+; SSSE3-NEXT:    movdqa %xmm15, %xmm0
+; SSSE3-NEXT:    pxor %xmm1, %xmm0
+; SSSE3-NEXT:    movdqa %xmm14, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm14, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    por %xmm0, %xmm6
+; SSSE3-NEXT:    pand %xmm6, %xmm15
+; SSSE3-NEXT:    pandn %xmm8, %xmm6
+; SSSE3-NEXT:    por %xmm15, %xmm6
+; SSSE3-NEXT:    movdqa %xmm13, %xmm0
+; SSSE3-NEXT:    pxor %xmm1, %xmm0
+; SSSE3-NEXT:    movdqa %xmm14, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm14, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm15 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    por %xmm0, %xmm15
+; SSSE3-NEXT:    pand %xmm15, %xmm13
+; SSSE3-NEXT:    pandn %xmm8, %xmm15
+; SSSE3-NEXT:    por %xmm13, %xmm15
+; SSSE3-NEXT:    movdqa %xmm10, %xmm0
+; SSSE3-NEXT:    pxor %xmm1, %xmm0
+; SSSE3-NEXT:    movdqa %xmm14, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm3[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm14, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm7, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm13 = xmm3[1,1,3,3]
+; SSSE3-NEXT:    por %xmm0, %xmm13
+; SSSE3-NEXT:    pand %xmm13, %xmm10
+; SSSE3-NEXT:    pandn %xmm8, %xmm13
+; SSSE3-NEXT:    por %xmm10, %xmm13
+; SSSE3-NEXT:    movdqa %xmm9, %xmm0
+; SSSE3-NEXT:    pxor %xmm1, %xmm0
+; SSSE3-NEXT:    movdqa %xmm14, %xmm7
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm7
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm10 = xmm7[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm14, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm10, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
+; SSSE3-NEXT:    por %xmm0, %xmm7
+; SSSE3-NEXT:    pand %xmm7, %xmm9
+; SSSE3-NEXT:    pandn %xmm8, %xmm7
+; SSSE3-NEXT:    por %xmm9, %xmm7
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm8 = [18446744073709551488,18446744073709551488]
+; SSSE3-NEXT:    movdqa %xmm7, %xmm0
+; SSSE3-NEXT:    pxor %xmm1, %xmm0
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm9 = [18446744071562067840,18446744071562067840]
 ; SSSE3-NEXT:    movdqa %xmm0, %xmm2
-; SSSE3-NEXT:    pcmpgtd %xmm11, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm11, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm4, %xmm7
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3]
-; SSSE3-NEXT:    por %xmm7, %xmm0
-; SSSE3-NEXT:    pand %xmm0, %xmm3
-; SSSE3-NEXT:    pandn %xmm10, %xmm0
+; SSSE3-NEXT:    pcmpgtd %xmm9, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm10 = xmm2[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm10, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    por %xmm0, %xmm2
+; SSSE3-NEXT:    pand %xmm2, %xmm7
+; SSSE3-NEXT:    pandn %xmm8, %xmm2
+; SSSE3-NEXT:    por %xmm7, %xmm2
+; SSSE3-NEXT:    movdqa %xmm13, %xmm0
+; SSSE3-NEXT:    pxor %xmm1, %xmm0
+; SSSE3-NEXT:    movdqa %xmm0, %xmm7
+; SSSE3-NEXT:    pcmpgtd %xmm9, %xmm7
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm10 = xmm7[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm10, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[1,1,3,3]
 ; SSSE3-NEXT:    por %xmm3, %xmm0
-; SSSE3-NEXT:    packssdw %xmm1, %xmm0
-; SSSE3-NEXT:    movdqa %xmm6, %xmm1
-; SSSE3-NEXT:    pxor %xmm8, %xmm1
-; SSSE3-NEXT:    movdqa %xmm1, %xmm2
-; SSSE3-NEXT:    pcmpgtd %xmm11, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm11, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm3, %xmm1
+; SSSE3-NEXT:    pand %xmm0, %xmm13
+; SSSE3-NEXT:    pandn %xmm8, %xmm0
+; SSSE3-NEXT:    por %xmm13, %xmm0
+; SSSE3-NEXT:    packssdw %xmm2, %xmm0
+; SSSE3-NEXT:    movdqa %xmm15, %xmm2
+; SSSE3-NEXT:    pxor %xmm1, %xmm2
+; SSSE3-NEXT:    movdqa %xmm2, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm9, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm3[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm2
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSSE3-NEXT:    por %xmm1, %xmm2
-; SSSE3-NEXT:    pand %xmm2, %xmm6
-; SSSE3-NEXT:    pandn %xmm10, %xmm2
-; SSSE3-NEXT:    por %xmm6, %xmm2
-; SSSE3-NEXT:    movdqa %xmm5, %xmm1
-; SSSE3-NEXT:    pxor %xmm8, %xmm1
-; SSSE3-NEXT:    movdqa %xmm1, %xmm3
-; SSSE3-NEXT:    pcmpgtd %xmm11, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm11, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm4, %xmm1
+; SSSE3-NEXT:    pand %xmm7, %xmm2
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm1, %xmm3
+; SSSE3-NEXT:    por %xmm2, %xmm3
+; SSSE3-NEXT:    pand %xmm3, %xmm15
+; SSSE3-NEXT:    pandn %xmm8, %xmm3
+; SSSE3-NEXT:    por %xmm15, %xmm3
+; SSSE3-NEXT:    movdqa %xmm6, %xmm2
+; SSSE3-NEXT:    pxor %xmm1, %xmm2
+; SSSE3-NEXT:    movdqa %xmm2, %xmm7
+; SSSE3-NEXT:    pcmpgtd %xmm9, %xmm7
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm10 = xmm7[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm10, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
+; SSSE3-NEXT:    por %xmm2, %xmm7
+; SSSE3-NEXT:    pand %xmm7, %xmm6
+; SSSE3-NEXT:    pandn %xmm8, %xmm7
+; SSSE3-NEXT:    por %xmm6, %xmm7
+; SSSE3-NEXT:    packssdw %xmm3, %xmm7
+; SSSE3-NEXT:    packssdw %xmm7, %xmm0
+; SSSE3-NEXT:    movdqa %xmm5, %xmm2
+; SSSE3-NEXT:    pxor %xmm1, %xmm2
+; SSSE3-NEXT:    movdqa %xmm2, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm9, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm6, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSSE3-NEXT:    por %xmm2, %xmm3
 ; SSSE3-NEXT:    pand %xmm3, %xmm5
-; SSSE3-NEXT:    pandn %xmm10, %xmm3
+; SSSE3-NEXT:    pandn %xmm8, %xmm3
 ; SSSE3-NEXT:    por %xmm5, %xmm3
-; SSSE3-NEXT:    packssdw %xmm2, %xmm3
-; SSSE3-NEXT:    packssdw %xmm3, %xmm0
-; SSSE3-NEXT:    movdqa %xmm14, %xmm1
-; SSSE3-NEXT:    pxor %xmm8, %xmm1
-; SSSE3-NEXT:    movdqa %xmm1, %xmm2
-; SSSE3-NEXT:    pcmpgtd %xmm11, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm11, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm3, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSSE3-NEXT:    por %xmm1, %xmm2
-; SSSE3-NEXT:    pand %xmm2, %xmm14
-; SSSE3-NEXT:    pandn %xmm10, %xmm2
-; SSSE3-NEXT:    por %xmm14, %xmm2
-; SSSE3-NEXT:    movdqa %xmm13, %xmm1
-; SSSE3-NEXT:    pxor %xmm8, %xmm1
+; SSSE3-NEXT:    movdqa %xmm4, %xmm2
+; SSSE3-NEXT:    pxor %xmm1, %xmm2
+; SSSE3-NEXT:    movdqa %xmm2, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm9, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm6, %xmm7
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm5[1,1,3,3]
+; SSSE3-NEXT:    por %xmm7, %xmm2
+; SSSE3-NEXT:    pand %xmm2, %xmm4
+; SSSE3-NEXT:    pandn %xmm8, %xmm2
+; SSSE3-NEXT:    por %xmm4, %xmm2
+; SSSE3-NEXT:    packssdw %xmm3, %xmm2
+; SSSE3-NEXT:    movdqa %xmm12, %xmm3
+; SSSE3-NEXT:    pxor %xmm1, %xmm3
+; SSSE3-NEXT:    movdqa %xmm3, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm9, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm5, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm3, %xmm4
+; SSSE3-NEXT:    pand %xmm4, %xmm12
+; SSSE3-NEXT:    pandn %xmm8, %xmm4
+; SSSE3-NEXT:    por %xmm12, %xmm4
+; SSSE3-NEXT:    pxor %xmm11, %xmm1
 ; SSSE3-NEXT:    movdqa %xmm1, %xmm3
-; SSSE3-NEXT:    pcmpgtd %xmm11, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm11, %xmm1
+; SSSE3-NEXT:    pcmpgtd %xmm9, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm1
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm4, %xmm1
+; SSSE3-NEXT:    pand %xmm5, %xmm1
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
 ; SSSE3-NEXT:    por %xmm1, %xmm3
-; SSSE3-NEXT:    pand %xmm3, %xmm13
-; SSSE3-NEXT:    pandn %xmm10, %xmm3
-; SSSE3-NEXT:    por %xmm13, %xmm3
-; SSSE3-NEXT:    packssdw %xmm2, %xmm3
-; SSSE3-NEXT:    movdqa %xmm12, %xmm1
-; SSSE3-NEXT:    pxor %xmm8, %xmm1
-; SSSE3-NEXT:    movdqa %xmm1, %xmm2
-; SSSE3-NEXT:    pcmpgtd %xmm11, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm11, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm4, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSSE3-NEXT:    por %xmm1, %xmm2
-; SSSE3-NEXT:    pand %xmm2, %xmm12
-; SSSE3-NEXT:    pandn %xmm10, %xmm2
-; SSSE3-NEXT:    por %xmm12, %xmm2
-; SSSE3-NEXT:    pxor %xmm9, %xmm8
-; SSSE3-NEXT:    movdqa %xmm8, %xmm1
-; SSSE3-NEXT:    pcmpgtd %xmm11, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm11, %xmm8
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm8[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm4, %xmm5
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    por %xmm5, %xmm1
-; SSSE3-NEXT:    pand %xmm1, %xmm9
-; SSSE3-NEXT:    pandn %xmm10, %xmm1
-; SSSE3-NEXT:    por %xmm9, %xmm1
-; SSSE3-NEXT:    packssdw %xmm2, %xmm1
-; SSSE3-NEXT:    packssdw %xmm1, %xmm3
-; SSSE3-NEXT:    packsswb %xmm3, %xmm0
+; SSSE3-NEXT:    pand %xmm3, %xmm11
+; SSSE3-NEXT:    pandn %xmm8, %xmm3
+; SSSE3-NEXT:    por %xmm11, %xmm3
+; SSSE3-NEXT:    packssdw %xmm4, %xmm3
+; SSSE3-NEXT:    packssdw %xmm3, %xmm2
+; SSSE3-NEXT:    packsswb %xmm2, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: trunc_ssat_v16i64_v16i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa %xmm0, %xmm8
-; SSE41-NEXT:    movapd {{.*#+}} xmm11 = [127,127]
-; SSE41-NEXT:    movdqa {{.*#+}} xmm9 = [2147483648,2147483648]
-; SSE41-NEXT:    movdqa %xmm6, %xmm0
-; SSE41-NEXT:    pxor %xmm9, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm12 = [2147483775,2147483775]
-; SSE41-NEXT:    movdqa %xmm12, %xmm10
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm10
-; SSE41-NEXT:    pshufd {{.*#+}} xmm13 = xmm10[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm12, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm13, %xmm0
-; SSE41-NEXT:    por %xmm10, %xmm0
-; SSE41-NEXT:    movapd %xmm11, %xmm10
-; SSE41-NEXT:    blendvpd %xmm0, %xmm6, %xmm10
-; SSE41-NEXT:    movdqa %xmm7, %xmm0
-; SSE41-NEXT:    pxor %xmm9, %xmm0
-; SSE41-NEXT:    movdqa %xmm12, %xmm6
+; SSE41-NEXT:    movdqa (%rdi), %xmm11
+; SSE41-NEXT:    movdqa 16(%rdi), %xmm9
+; SSE41-NEXT:    movdqa 32(%rdi), %xmm15
+; SSE41-NEXT:    movdqa 48(%rdi), %xmm12
+; SSE41-NEXT:    movdqa 80(%rdi), %xmm4
+; SSE41-NEXT:    movdqa 64(%rdi), %xmm14
+; SSE41-NEXT:    movdqa 112(%rdi), %xmm13
+; SSE41-NEXT:    movdqa 96(%rdi), %xmm3
+; SSE41-NEXT:    movapd {{.*#+}} xmm1 = [127,127]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    pxor %xmm2, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm7 = [2147483775,2147483775]
+; SSE41-NEXT:    movdqa %xmm7, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm5
+; SSE41-NEXT:    movdqa %xmm7, %xmm6
 ; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm13 = xmm6[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm12, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm13, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm0
 ; SSE41-NEXT:    por %xmm6, %xmm0
-; SSE41-NEXT:    movapd %xmm11, %xmm13
-; SSE41-NEXT:    blendvpd %xmm0, %xmm7, %xmm13
+; SSE41-NEXT:    movapd %xmm1, %xmm8
+; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm8
+; SSE41-NEXT:    movdqa %xmm13, %xmm0
+; SSE41-NEXT:    pxor %xmm2, %xmm0
+; SSE41-NEXT:    movdqa %xmm7, %xmm3
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm3
+; SSE41-NEXT:    movdqa %xmm7, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm5
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2]
+; SSE41-NEXT:    pand %xmm3, %xmm0
+; SSE41-NEXT:    por %xmm5, %xmm0
+; SSE41-NEXT:    movapd %xmm1, %xmm10
+; SSE41-NEXT:    blendvpd %xmm0, %xmm13, %xmm10
+; SSE41-NEXT:    movdqa %xmm14, %xmm0
+; SSE41-NEXT:    pxor %xmm2, %xmm0
+; SSE41-NEXT:    movdqa %xmm7, %xmm3
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm3
+; SSE41-NEXT:    movdqa %xmm7, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm5
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2]
+; SSE41-NEXT:    pand %xmm3, %xmm0
+; SSE41-NEXT:    por %xmm5, %xmm0
+; SSE41-NEXT:    movapd %xmm1, %xmm13
+; SSE41-NEXT:    blendvpd %xmm0, %xmm14, %xmm13
 ; SSE41-NEXT:    movdqa %xmm4, %xmm0
-; SSE41-NEXT:    pxor %xmm9, %xmm0
-; SSE41-NEXT:    movdqa %xmm12, %xmm6
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm12, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm7, %xmm0
-; SSE41-NEXT:    por %xmm6, %xmm0
-; SSE41-NEXT:    movapd %xmm11, %xmm14
+; SSE41-NEXT:    pxor %xmm2, %xmm0
+; SSE41-NEXT:    movdqa %xmm7, %xmm3
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm3
+; SSE41-NEXT:    movdqa %xmm7, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm5
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2]
+; SSE41-NEXT:    pand %xmm3, %xmm0
+; SSE41-NEXT:    por %xmm5, %xmm0
+; SSE41-NEXT:    movapd %xmm1, %xmm14
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm14
-; SSE41-NEXT:    movdqa %xmm5, %xmm0
-; SSE41-NEXT:    pxor %xmm9, %xmm0
-; SSE41-NEXT:    movdqa %xmm12, %xmm4
+; SSE41-NEXT:    movdqa %xmm15, %xmm0
+; SSE41-NEXT:    pxor %xmm2, %xmm0
+; SSE41-NEXT:    movdqa %xmm7, %xmm3
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm3
+; SSE41-NEXT:    movdqa %xmm7, %xmm4
 ; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm12, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm6, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pand %xmm3, %xmm0
 ; SSE41-NEXT:    por %xmm4, %xmm0
-; SSE41-NEXT:    movapd %xmm11, %xmm15
-; SSE41-NEXT:    blendvpd %xmm0, %xmm5, %xmm15
-; SSE41-NEXT:    movdqa %xmm2, %xmm0
-; SSE41-NEXT:    pxor %xmm9, %xmm0
-; SSE41-NEXT:    movdqa %xmm12, %xmm5
+; SSE41-NEXT:    movapd %xmm1, %xmm4
+; SSE41-NEXT:    blendvpd %xmm0, %xmm15, %xmm4
+; SSE41-NEXT:    movdqa %xmm12, %xmm0
+; SSE41-NEXT:    pxor %xmm2, %xmm0
+; SSE41-NEXT:    movdqa %xmm7, %xmm3
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm3
+; SSE41-NEXT:    movdqa %xmm7, %xmm5
 ; SSE41-NEXT:    pcmpgtd %xmm0, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm12, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm6, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2]
+; SSE41-NEXT:    pand %xmm3, %xmm0
 ; SSE41-NEXT:    por %xmm5, %xmm0
-; SSE41-NEXT:    movapd %xmm11, %xmm5
-; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm5
-; SSE41-NEXT:    movdqa %xmm3, %xmm0
-; SSE41-NEXT:    pxor %xmm9, %xmm0
-; SSE41-NEXT:    movdqa %xmm12, %xmm2
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm12, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm6, %xmm0
-; SSE41-NEXT:    por %xmm2, %xmm0
-; SSE41-NEXT:    movapd %xmm11, %xmm6
-; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm6
-; SSE41-NEXT:    movdqa %xmm8, %xmm0
-; SSE41-NEXT:    pxor %xmm9, %xmm0
-; SSE41-NEXT:    movdqa %xmm12, %xmm2
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm12, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE41-NEXT:    movapd %xmm1, %xmm15
+; SSE41-NEXT:    blendvpd %xmm0, %xmm12, %xmm15
+; SSE41-NEXT:    movdqa %xmm11, %xmm0
+; SSE41-NEXT:    pxor %xmm2, %xmm0
+; SSE41-NEXT:    movdqa %xmm7, %xmm3
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm3
+; SSE41-NEXT:    movdqa %xmm7, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
 ; SSE41-NEXT:    pand %xmm3, %xmm0
-; SSE41-NEXT:    por %xmm2, %xmm0
-; SSE41-NEXT:    movapd %xmm11, %xmm7
-; SSE41-NEXT:    blendvpd %xmm0, %xmm8, %xmm7
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    pxor %xmm9, %xmm0
-; SSE41-NEXT:    movdqa %xmm12, %xmm2
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm12, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE41-NEXT:    por %xmm6, %xmm0
+; SSE41-NEXT:    movapd %xmm1, %xmm6
+; SSE41-NEXT:    blendvpd %xmm0, %xmm11, %xmm6
+; SSE41-NEXT:    movdqa %xmm9, %xmm0
+; SSE41-NEXT:    pxor %xmm2, %xmm0
+; SSE41-NEXT:    movdqa %xmm7, %xmm3
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm3
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm7
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[0,0,2,2]
 ; SSE41-NEXT:    pand %xmm3, %xmm0
-; SSE41-NEXT:    por %xmm2, %xmm0
-; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm11
-; SSE41-NEXT:    movapd {{.*#+}} xmm2 = [18446744073709551488,18446744073709551488]
-; SSE41-NEXT:    movapd %xmm11, %xmm0
-; SSE41-NEXT:    xorpd %xmm9, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm8 = [18446744071562067840,18446744071562067840]
-; SSE41-NEXT:    movapd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd %xmm8, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm8, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm4, %xmm0
-; SSE41-NEXT:    por %xmm1, %xmm0
-; SSE41-NEXT:    movapd %xmm2, %xmm4
-; SSE41-NEXT:    blendvpd %xmm0, %xmm11, %xmm4
-; SSE41-NEXT:    movapd %xmm7, %xmm0
-; SSE41-NEXT:    xorpd %xmm9, %xmm0
-; SSE41-NEXT:    movapd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd %xmm8, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm8, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE41-NEXT:    por %xmm7, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm9, %xmm1
+; SSE41-NEXT:    movapd {{.*#+}} xmm7 = [18446744073709551488,18446744073709551488]
+; SSE41-NEXT:    movapd %xmm1, %xmm5
+; SSE41-NEXT:    xorpd %xmm2, %xmm5
+; SSE41-NEXT:    movdqa {{.*#+}} xmm9 = [18446744071562067840,18446744071562067840]
+; SSE41-NEXT:    movapd %xmm5, %xmm3
+; SSE41-NEXT:    pcmpeqd %xmm9, %xmm3
+; SSE41-NEXT:    pcmpgtd %xmm9, %xmm5
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2]
 ; SSE41-NEXT:    pand %xmm3, %xmm0
+; SSE41-NEXT:    por %xmm5, %xmm0
+; SSE41-NEXT:    movapd %xmm7, %xmm3
+; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm3
+; SSE41-NEXT:    movapd %xmm6, %xmm1
+; SSE41-NEXT:    xorpd %xmm2, %xmm1
+; SSE41-NEXT:    movapd %xmm1, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm9, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm9, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm0
 ; SSE41-NEXT:    por %xmm1, %xmm0
-; SSE41-NEXT:    movapd %xmm2, %xmm1
-; SSE41-NEXT:    blendvpd %xmm0, %xmm7, %xmm1
-; SSE41-NEXT:    packssdw %xmm4, %xmm1
-; SSE41-NEXT:    movapd %xmm6, %xmm0
-; SSE41-NEXT:    xorpd %xmm9, %xmm0
-; SSE41-NEXT:    movapd %xmm0, %xmm3
-; SSE41-NEXT:    pcmpgtd %xmm8, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm8, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    movapd %xmm7, %xmm1
+; SSE41-NEXT:    blendvpd %xmm0, %xmm6, %xmm1
+; SSE41-NEXT:    packssdw %xmm3, %xmm1
+; SSE41-NEXT:    movapd %xmm15, %xmm3
+; SSE41-NEXT:    xorpd %xmm2, %xmm3
+; SSE41-NEXT:    movapd %xmm3, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm9, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm9, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm0
 ; SSE41-NEXT:    por %xmm3, %xmm0
-; SSE41-NEXT:    movapd %xmm2, %xmm3
-; SSE41-NEXT:    blendvpd %xmm0, %xmm6, %xmm3
-; SSE41-NEXT:    movapd %xmm5, %xmm0
-; SSE41-NEXT:    xorpd %xmm9, %xmm0
-; SSE41-NEXT:    movapd %xmm0, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm8, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm8, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE41-NEXT:    movapd %xmm7, %xmm3
+; SSE41-NEXT:    blendvpd %xmm0, %xmm15, %xmm3
+; SSE41-NEXT:    movapd %xmm4, %xmm5
+; SSE41-NEXT:    xorpd %xmm2, %xmm5
+; SSE41-NEXT:    movapd %xmm5, %xmm6
+; SSE41-NEXT:    pcmpeqd %xmm9, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm9, %xmm5
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2]
 ; SSE41-NEXT:    pand %xmm6, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
-; SSE41-NEXT:    movapd %xmm2, %xmm4
-; SSE41-NEXT:    blendvpd %xmm0, %xmm5, %xmm4
-; SSE41-NEXT:    packssdw %xmm3, %xmm4
-; SSE41-NEXT:    packssdw %xmm4, %xmm1
-; SSE41-NEXT:    movapd %xmm15, %xmm0
-; SSE41-NEXT:    xorpd %xmm9, %xmm0
-; SSE41-NEXT:    movapd %xmm0, %xmm3
-; SSE41-NEXT:    pcmpgtd %xmm8, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm8, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE41-NEXT:    por %xmm5, %xmm0
+; SSE41-NEXT:    movapd %xmm7, %xmm5
+; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm5
+; SSE41-NEXT:    packssdw %xmm3, %xmm5
+; SSE41-NEXT:    packssdw %xmm5, %xmm1
+; SSE41-NEXT:    movapd %xmm14, %xmm3
+; SSE41-NEXT:    xorpd %xmm2, %xmm3
+; SSE41-NEXT:    movapd %xmm3, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm9, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm9, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
 ; SSE41-NEXT:    pand %xmm4, %xmm0
 ; SSE41-NEXT:    por %xmm3, %xmm0
-; SSE41-NEXT:    movapd %xmm2, %xmm3
-; SSE41-NEXT:    blendvpd %xmm0, %xmm15, %xmm3
-; SSE41-NEXT:    movapd %xmm14, %xmm0
-; SSE41-NEXT:    xorpd %xmm9, %xmm0
-; SSE41-NEXT:    movapd %xmm0, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm8, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm8, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE41-NEXT:    movapd %xmm7, %xmm3
+; SSE41-NEXT:    blendvpd %xmm0, %xmm14, %xmm3
+; SSE41-NEXT:    movapd %xmm13, %xmm4
+; SSE41-NEXT:    xorpd %xmm2, %xmm4
+; SSE41-NEXT:    movapd %xmm4, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm9, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm9, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2]
 ; SSE41-NEXT:    pand %xmm5, %xmm0
 ; SSE41-NEXT:    por %xmm4, %xmm0
-; SSE41-NEXT:    movapd %xmm2, %xmm4
-; SSE41-NEXT:    blendvpd %xmm0, %xmm14, %xmm4
+; SSE41-NEXT:    movapd %xmm7, %xmm4
+; SSE41-NEXT:    blendvpd %xmm0, %xmm13, %xmm4
 ; SSE41-NEXT:    packssdw %xmm3, %xmm4
-; SSE41-NEXT:    movapd %xmm13, %xmm0
-; SSE41-NEXT:    xorpd %xmm9, %xmm0
-; SSE41-NEXT:    movapd %xmm0, %xmm3
-; SSE41-NEXT:    pcmpgtd %xmm8, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm8, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE41-NEXT:    movapd %xmm10, %xmm3
+; SSE41-NEXT:    xorpd %xmm2, %xmm3
+; SSE41-NEXT:    movapd %xmm3, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm9, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm9, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
 ; SSE41-NEXT:    pand %xmm5, %xmm0
 ; SSE41-NEXT:    por %xmm3, %xmm0
-; SSE41-NEXT:    movapd %xmm2, %xmm3
-; SSE41-NEXT:    blendvpd %xmm0, %xmm13, %xmm3
-; SSE41-NEXT:    xorpd %xmm10, %xmm9
-; SSE41-NEXT:    movapd %xmm9, %xmm5
-; SSE41-NEXT:    pcmpgtd %xmm8, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm8, %xmm9
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm9[1,1,3,3]
-; SSE41-NEXT:    pand %xmm6, %xmm0
-; SSE41-NEXT:    por %xmm5, %xmm0
-; SSE41-NEXT:    blendvpd %xmm0, %xmm10, %xmm2
-; SSE41-NEXT:    packssdw %xmm3, %xmm2
-; SSE41-NEXT:    packssdw %xmm2, %xmm4
+; SSE41-NEXT:    movapd %xmm7, %xmm3
+; SSE41-NEXT:    blendvpd %xmm0, %xmm10, %xmm3
+; SSE41-NEXT:    xorpd %xmm8, %xmm2
+; SSE41-NEXT:    movapd %xmm2, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm9, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm9, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm0
+; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm8, %xmm7
+; SSE41-NEXT:    packssdw %xmm3, %xmm7
+; SSE41-NEXT:    packssdw %xmm7, %xmm4
 ; SSE41-NEXT:    packsswb %xmm4, %xmm1
 ; SSE41-NEXT:    movdqa %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: trunc_ssat_v16i64_v16i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm8
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm5 = [127,127]
-; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm9
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm7
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
-; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm5, %xmm6
-; AVX1-NEXT:    vblendvpd %xmm6, %xmm0, %xmm5, %xmm10
-; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm5, %xmm6
-; AVX1-NEXT:    vblendvpd %xmm6, %xmm4, %xmm5, %xmm11
-; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm5, %xmm6
-; AVX1-NEXT:    vblendvpd %xmm6, %xmm1, %xmm5, %xmm1
-; AVX1-NEXT:    vpcmpgtq %xmm7, %xmm5, %xmm6
-; AVX1-NEXT:    vblendvpd %xmm6, %xmm7, %xmm5, %xmm6
-; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm5, %xmm7
-; AVX1-NEXT:    vblendvpd %xmm7, %xmm2, %xmm5, %xmm2
-; AVX1-NEXT:    vpcmpgtq %xmm9, %xmm5, %xmm7
-; AVX1-NEXT:    vblendvpd %xmm7, %xmm9, %xmm5, %xmm7
-; AVX1-NEXT:    vpcmpgtq %xmm3, %xmm5, %xmm0
-; AVX1-NEXT:    vblendvpd %xmm0, %xmm3, %xmm5, %xmm0
-; AVX1-NEXT:    vpcmpgtq %xmm8, %xmm5, %xmm3
-; AVX1-NEXT:    vblendvpd %xmm3, %xmm8, %xmm5, %xmm3
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm5 = [18446744073709551488,18446744073709551488]
-; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm3, %xmm4
-; AVX1-NEXT:    vblendvpd %xmm4, %xmm3, %xmm5, %xmm8
-; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm0, %xmm4
-; AVX1-NEXT:    vblendvpd %xmm4, %xmm0, %xmm5, %xmm0
-; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm7, %xmm4
-; AVX1-NEXT:    vblendvpd %xmm4, %xmm7, %xmm5, %xmm4
-; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm2, %xmm7
-; AVX1-NEXT:    vblendvpd %xmm7, %xmm2, %xmm5, %xmm2
-; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm6, %xmm7
-; AVX1-NEXT:    vblendvpd %xmm7, %xmm6, %xmm5, %xmm6
-; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm1, %xmm7
-; AVX1-NEXT:    vblendvpd %xmm7, %xmm1, %xmm5, %xmm1
-; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm11, %xmm7
-; AVX1-NEXT:    vblendvpd %xmm7, %xmm11, %xmm5, %xmm7
-; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm10, %xmm3
-; AVX1-NEXT:    vblendvpd %xmm3, %xmm10, %xmm5, %xmm3
-; AVX1-NEXT:    vpackssdw %xmm8, %xmm0, %xmm0
-; AVX1-NEXT:    vpackssdw %xmm4, %xmm2, %xmm2
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm2, %xmm0
-; AVX1-NEXT:    vpackssdw %xmm6, %xmm1, %xmm1
-; AVX1-NEXT:    vpackssdw %xmm7, %xmm3, %xmm2
-; AVX1-NEXT:    vpackssdw %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm1, %xmm0
-; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    vmovdqa 112(%rdi), %xmm8
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [127,127]
+; AVX1-NEXT:    vmovdqa 96(%rdi), %xmm9
+; AVX1-NEXT:    vmovdqa 80(%rdi), %xmm3
+; AVX1-NEXT:    vmovdqa 64(%rdi), %xmm4
+; AVX1-NEXT:    vmovdqa (%rdi), %xmm5
+; AVX1-NEXT:    vmovdqa 16(%rdi), %xmm6
+; AVX1-NEXT:    vmovdqa 32(%rdi), %xmm7
+; AVX1-NEXT:    vmovdqa 48(%rdi), %xmm0
+; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm1, %xmm2
+; AVX1-NEXT:    vblendvpd %xmm2, %xmm5, %xmm1, %xmm10
+; AVX1-NEXT:    vpcmpgtq %xmm6, %xmm1, %xmm5
+; AVX1-NEXT:    vblendvpd %xmm5, %xmm6, %xmm1, %xmm11
+; AVX1-NEXT:    vpcmpgtq %xmm7, %xmm1, %xmm6
+; AVX1-NEXT:    vblendvpd %xmm6, %xmm7, %xmm1, %xmm6
+; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm7
+; AVX1-NEXT:    vblendvpd %xmm7, %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm1, %xmm7
+; AVX1-NEXT:    vblendvpd %xmm7, %xmm4, %xmm1, %xmm4
+; AVX1-NEXT:    vpcmpgtq %xmm3, %xmm1, %xmm7
+; AVX1-NEXT:    vblendvpd %xmm7, %xmm3, %xmm1, %xmm3
+; AVX1-NEXT:    vpcmpgtq %xmm9, %xmm1, %xmm7
+; AVX1-NEXT:    vblendvpd %xmm7, %xmm9, %xmm1, %xmm7
+; AVX1-NEXT:    vpcmpgtq %xmm8, %xmm1, %xmm2
+; AVX1-NEXT:    vblendvpd %xmm2, %xmm8, %xmm1, %xmm1
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [18446744073709551488,18446744073709551488]
+; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm1, %xmm5
+; AVX1-NEXT:    vblendvpd %xmm5, %xmm1, %xmm2, %xmm8
+; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm7, %xmm5
+; AVX1-NEXT:    vblendvpd %xmm5, %xmm7, %xmm2, %xmm5
+; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm3, %xmm7
+; AVX1-NEXT:    vblendvpd %xmm7, %xmm3, %xmm2, %xmm3
+; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm4, %xmm7
+; AVX1-NEXT:    vblendvpd %xmm7, %xmm4, %xmm2, %xmm4
+; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm0, %xmm7
+; AVX1-NEXT:    vblendvpd %xmm7, %xmm0, %xmm2, %xmm0
+; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm6, %xmm7
+; AVX1-NEXT:    vblendvpd %xmm7, %xmm6, %xmm2, %xmm6
+; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm11, %xmm7
+; AVX1-NEXT:    vblendvpd %xmm7, %xmm11, %xmm2, %xmm7
+; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm10, %xmm1
+; AVX1-NEXT:    vblendvpd %xmm1, %xmm10, %xmm2, %xmm1
+; AVX1-NEXT:    vpackssdw %xmm8, %xmm5, %xmm2
+; AVX1-NEXT:    vpackssdw %xmm3, %xmm4, %xmm3
+; AVX1-NEXT:    vpackssdw %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vpackssdw %xmm0, %xmm6, %xmm0
+; AVX1-NEXT:    vpackssdw %xmm7, %xmm1, %xmm1
+; AVX1-NEXT:    vpackssdw %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vpacksswb %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: trunc_ssat_v16i64_v16i8:
 ; AVX2:       # %bb.0:
+; AVX2-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX2-NEXT:    vmovdqa 32(%rdi), %ymm1
+; AVX2-NEXT:    vmovdqa 64(%rdi), %ymm2
+; AVX2-NEXT:    vmovdqa 96(%rdi), %ymm3
 ; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [127,127,127,127]
 ; AVX2-NEXT:    vpcmpgtq %ymm2, %ymm4, %ymm5
 ; AVX2-NEXT:    vblendvpd %ymm5, %ymm2, %ymm4, %ymm2
@@ -2716,20 +4967,72 @@ define <16 x i8> @trunc_ssat_v16i64_v16i8(<16 x i64> %a0) {
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
-; AVX512-LABEL: trunc_ssat_v16i64_v16i8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpbroadcastq {{.*#+}} zmm2 = [127,127,127,127,127,127,127,127]
-; AVX512-NEXT:    vpminsq %zmm2, %zmm0, %zmm0
-; AVX512-NEXT:    vpminsq %zmm2, %zmm1, %zmm1
-; AVX512-NEXT:    vpbroadcastq {{.*#+}} zmm2 = [18446744073709551488,18446744073709551488,18446744073709551488,18446744073709551488,18446744073709551488,18446744073709551488,18446744073709551488,18446744073709551488]
-; AVX512-NEXT:    vpmaxsq %zmm2, %zmm1, %zmm1
-; AVX512-NEXT:    vpmaxsq %zmm2, %zmm0, %zmm0
-; AVX512-NEXT:    vpmovqd %zmm0, %ymm0
-; AVX512-NEXT:    vpmovqd %zmm1, %ymm1
-; AVX512-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512-NEXT:    vpmovdb %zmm0, %xmm0
-; AVX512-NEXT:    vzeroupper
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: trunc_ssat_v16i64_v16i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpbroadcastq {{.*#+}} zmm0 = [127,127,127,127,127,127,127,127]
+; AVX512F-NEXT:    vpminsq (%rdi), %zmm0, %zmm1
+; AVX512F-NEXT:    vpminsq 64(%rdi), %zmm0, %zmm0
+; AVX512F-NEXT:    vpbroadcastq {{.*#+}} zmm2 = [18446744073709551488,18446744073709551488,18446744073709551488,18446744073709551488,18446744073709551488,18446744073709551488,18446744073709551488,18446744073709551488]
+; AVX512F-NEXT:    vpmaxsq %zmm2, %zmm0, %zmm0
+; AVX512F-NEXT:    vpmaxsq %zmm2, %zmm1, %zmm1
+; AVX512F-NEXT:    vpmovqd %zmm1, %ymm1
+; AVX512F-NEXT:    vpmovqd %zmm0, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: trunc_ssat_v16i64_v16i8:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vmovdqa64 (%rdi), %zmm0
+; AVX512VL-NEXT:    vmovdqa64 64(%rdi), %zmm1
+; AVX512VL-NEXT:    vpmovsqb %zmm1, %xmm1
+; AVX512VL-NEXT:    vpmovsqb %zmm0, %xmm0
+; AVX512VL-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX512VL-NEXT:    vzeroupper
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: trunc_ssat_v16i64_v16i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpbroadcastq {{.*#+}} zmm0 = [127,127,127,127,127,127,127,127]
+; AVX512BW-NEXT:    vpminsq (%rdi), %zmm0, %zmm1
+; AVX512BW-NEXT:    vpminsq 64(%rdi), %zmm0, %zmm0
+; AVX512BW-NEXT:    vpbroadcastq {{.*#+}} zmm2 = [18446744073709551488,18446744073709551488,18446744073709551488,18446744073709551488,18446744073709551488,18446744073709551488,18446744073709551488,18446744073709551488]
+; AVX512BW-NEXT:    vpmaxsq %zmm2, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmaxsq %zmm2, %zmm1, %zmm1
+; AVX512BW-NEXT:    vpmovqd %zmm1, %ymm1
+; AVX512BW-NEXT:    vpmovqd %zmm0, %ymm0
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512BW-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: trunc_ssat_v16i64_v16i8:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vmovdqa64 (%rdi), %zmm0
+; AVX512BWVL-NEXT:    vmovdqa64 64(%rdi), %zmm1
+; AVX512BWVL-NEXT:    vpmovsqb %zmm1, %xmm1
+; AVX512BWVL-NEXT:    vpmovsqb %zmm0, %xmm0
+; AVX512BWVL-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
+;
+; SKX-LABEL: trunc_ssat_v16i64_v16i8:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vmovdqa (%rdi), %ymm0
+; SKX-NEXT:    vmovdqa 32(%rdi), %ymm1
+; SKX-NEXT:    vmovdqa 64(%rdi), %ymm2
+; SKX-NEXT:    vmovdqa 96(%rdi), %ymm3
+; SKX-NEXT:    vpmovsqb %ymm3, %xmm3
+; SKX-NEXT:    vpmovsqb %ymm2, %xmm2
+; SKX-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; SKX-NEXT:    vpmovsqb %ymm1, %xmm1
+; SKX-NEXT:    vpmovsqb %ymm0, %xmm0
+; SKX-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SKX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SKX-NEXT:    vzeroupper
+; SKX-NEXT:    retq
+  %a0 = load <16 x i64>, <16 x i64>* %p0
   %1 = icmp slt <16 x i64> %a0, <i64 127, i64 127, i64 127, i64 127, i64 127, i64 127, i64 127, i64 127, i64 127, i64 127, i64 127, i64 127, i64 127, i64 127, i64 127, i64 127>
   %2 = select <16 x i1> %1, <16 x i64> %a0, <16 x i64> <i64 127, i64 127, i64 127, i64 127, i64 127, i64 127, i64 127, i64 127, i64 127, i64 127, i64 127, i64 127, i64 127, i64 127, i64 127, i64 127>
   %3 = icmp sgt <16 x i64> %2, <i64 -128, i64 -128, i64 -128, i64 -128, i64 -128, i64 -128, i64 -128, i64 -128, i64 -128, i64 -128, i64 -128, i64 -128, i64 -128, i64 -128, i64 -128, i64 -128>
@@ -2738,133 +5041,262 @@ define <16 x i8> @trunc_ssat_v16i64_v16i8(<16 x i64> %a0) {
   ret <16 x i8> %5
 }
 
-define <8 x i8> @trunc_ssat_v8i32_v8i8(<8 x i32> %a0) {
-; SSE2-LABEL: trunc_ssat_v8i32_v8i8:
+define <4 x i8> @trunc_ssat_v4i32_v4i8(<4 x i32> %a0) {
+; SSE2-LABEL: trunc_ssat_v4i32_v4i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [127,127,127,127]
-; SSE2-NEXT:    movdqa %xmm2, %xmm3
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSE2-NEXT:    pand %xmm3, %xmm0
-; SSE2-NEXT:    pandn %xmm2, %xmm3
-; SSE2-NEXT:    por %xmm0, %xmm3
-; SSE2-NEXT:    movdqa %xmm2, %xmm0
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pand %xmm0, %xmm1
-; SSE2-NEXT:    pandn %xmm2, %xmm0
-; SSE2-NEXT:    por %xmm1, %xmm0
-; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [4294967168,4294967168,4294967168,4294967168]
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [127,127,127,127]
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm2
 ; SSE2-NEXT:    pand %xmm2, %xmm0
 ; SSE2-NEXT:    pandn %xmm1, %xmm2
 ; SSE2-NEXT:    por %xmm0, %xmm2
-; SSE2-NEXT:    movdqa %xmm3, %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [4294967168,4294967168,4294967168,4294967168]
+; SSE2-NEXT:    movdqa %xmm2, %xmm0
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pand %xmm0, %xmm3
+; SSE2-NEXT:    pand %xmm0, %xmm2
 ; SSE2-NEXT:    pandn %xmm1, %xmm0
-; SSE2-NEXT:    por %xmm3, %xmm0
-; SSE2-NEXT:    packssdw %xmm2, %xmm0
+; SSE2-NEXT:    por %xmm2, %xmm0
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    packuswb %xmm0, %xmm0
+; SSE2-NEXT:    packuswb %xmm0, %xmm0
 ; SSE2-NEXT:    retq
 ;
-; SSSE3-LABEL: trunc_ssat_v8i32_v8i8:
+; SSSE3-LABEL: trunc_ssat_v4i32_v4i8:
 ; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [127,127,127,127]
-; SSSE3-NEXT:    movdqa %xmm2, %xmm3
-; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSSE3-NEXT:    pand %xmm3, %xmm0
-; SSSE3-NEXT:    pandn %xmm2, %xmm3
-; SSSE3-NEXT:    por %xmm0, %xmm3
-; SSSE3-NEXT:    movdqa %xmm2, %xmm0
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSSE3-NEXT:    pand %xmm0, %xmm1
-; SSSE3-NEXT:    pandn %xmm2, %xmm0
-; SSSE3-NEXT:    por %xmm1, %xmm0
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [4294967168,4294967168,4294967168,4294967168]
-; SSSE3-NEXT:    movdqa %xmm0, %xmm2
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [127,127,127,127]
+; SSSE3-NEXT:    movdqa %xmm1, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm2
 ; SSSE3-NEXT:    pand %xmm2, %xmm0
 ; SSSE3-NEXT:    pandn %xmm1, %xmm2
 ; SSSE3-NEXT:    por %xmm0, %xmm2
-; SSSE3-NEXT:    movdqa %xmm3, %xmm0
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [4294967168,4294967168,4294967168,4294967168]
+; SSSE3-NEXT:    movdqa %xmm2, %xmm0
 ; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSSE3-NEXT:    pand %xmm0, %xmm3
+; SSSE3-NEXT:    pand %xmm0, %xmm2
 ; SSSE3-NEXT:    pandn %xmm1, %xmm0
-; SSSE3-NEXT:    por %xmm3, %xmm0
-; SSSE3-NEXT:    packssdw %xmm2, %xmm0
+; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: trunc_ssat_v4i32_v4i8:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pminsd {{.*}}(%rip), %xmm0
+; SSE41-NEXT:    pmaxsd {{.*}}(%rip), %xmm0
+; SSE41-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: trunc_ssat_v4i32_v4i8:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpminsd {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vpmaxsd {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: trunc_ssat_v4i32_v4i8:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [127,127,127,127]
+; AVX2-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [4294967168,4294967168,4294967168,4294967168]
+; AVX2-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX2-NEXT:    retq
+;
+; AVX512F-LABEL: trunc_ssat_v4i32_v4i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512F-NEXT:    vpmovsdb %zmm0, %xmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: trunc_ssat_v4i32_v4i8:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpmovsdb %xmm0, %xmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: trunc_ssat_v4i32_v4i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512BW-NEXT:    vpmovsdb %zmm0, %xmm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: trunc_ssat_v4i32_v4i8:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpmovsdb %xmm0, %xmm0
+; AVX512BWVL-NEXT:    retq
+;
+; SKX-LABEL: trunc_ssat_v4i32_v4i8:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpmovsdb %xmm0, %xmm0
+; SKX-NEXT:    retq
+  %1 = icmp slt <4 x i32> %a0, <i32 127, i32 127, i32 127, i32 127>
+  %2 = select <4 x i1> %1, <4 x i32> %a0, <4 x i32> <i32 127, i32 127, i32 127, i32 127>
+  %3 = icmp sgt <4 x i32> %2, <i32 -128, i32 -128, i32 -128, i32 -128>
+  %4 = select <4 x i1> %3, <4 x i32> %2, <4 x i32> <i32 -128, i32 -128, i32 -128, i32 -128>
+  %5 = trunc <4 x i32> %4 to <4 x i8>
+  ret <4 x i8> %5
+}
+
+define void @trunc_ssat_v4i32_v4i8_store(<4 x i32> %a0, <4 x i8> *%p1) {
+; SSE2-LABEL: trunc_ssat_v4i32_v4i8_store:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [127,127,127,127]
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    pandn %xmm1, %xmm2
+; SSE2-NEXT:    por %xmm0, %xmm2
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [4294967168,4294967168,4294967168,4294967168]
+; SSE2-NEXT:    movdqa %xmm2, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
+; SSE2-NEXT:    pand %xmm1, %xmm2
+; SSE2-NEXT:    pandn %xmm0, %xmm1
+; SSE2-NEXT:    por %xmm2, %xmm1
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE2-NEXT:    packuswb %xmm0, %xmm1
+; SSE2-NEXT:    packuswb %xmm0, %xmm1
+; SSE2-NEXT:    movd %xmm1, (%rdi)
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: trunc_ssat_v4i32_v4i8_store:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [127,127,127,127]
+; SSSE3-NEXT:    movdqa %xmm1, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSSE3-NEXT:    pand %xmm2, %xmm0
+; SSSE3-NEXT:    pandn %xmm1, %xmm2
+; SSSE3-NEXT:    por %xmm0, %xmm2
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [4294967168,4294967168,4294967168,4294967168]
+; SSSE3-NEXT:    movdqa %xmm2, %xmm1
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm1
+; SSSE3-NEXT:    pand %xmm1, %xmm2
+; SSSE3-NEXT:    pandn %xmm0, %xmm1
+; SSSE3-NEXT:    por %xmm2, %xmm1
+; SSSE3-NEXT:    pshufb {{.*#+}} xmm1 = xmm1[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; SSSE3-NEXT:    movd %xmm1, (%rdi)
 ; SSSE3-NEXT:    retq
 ;
-; SSE41-LABEL: trunc_ssat_v8i32_v8i8:
+; SSE41-LABEL: trunc_ssat_v4i32_v4i8_store:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [127,127,127,127]
-; SSE41-NEXT:    pminsd %xmm2, %xmm0
-; SSE41-NEXT:    pminsd %xmm2, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [4294967168,4294967168,4294967168,4294967168]
-; SSE41-NEXT:    pmaxsd %xmm2, %xmm1
-; SSE41-NEXT:    pmaxsd %xmm2, %xmm0
-; SSE41-NEXT:    packssdw %xmm1, %xmm0
+; SSE41-NEXT:    pminsd {{.*}}(%rip), %xmm0
+; SSE41-NEXT:    pmaxsd {{.*}}(%rip), %xmm0
+; SSE41-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; SSE41-NEXT:    movd %xmm0, (%rdi)
 ; SSE41-NEXT:    retq
 ;
+; AVX1-LABEL: trunc_ssat_v4i32_v4i8_store:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpminsd {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vpmaxsd {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX1-NEXT:    vmovd %xmm0, (%rdi)
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: trunc_ssat_v4i32_v4i8_store:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [127,127,127,127]
+; AVX2-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [4294967168,4294967168,4294967168,4294967168]
+; AVX2-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX2-NEXT:    vmovd %xmm0, (%rdi)
+; AVX2-NEXT:    retq
+;
+; AVX512F-LABEL: trunc_ssat_v4i32_v4i8_store:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512F-NEXT:    vpmovsdb %zmm0, %xmm0
+; AVX512F-NEXT:    vmovd %xmm0, (%rdi)
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: trunc_ssat_v4i32_v4i8_store:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpmovsdb %xmm0, (%rdi)
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: trunc_ssat_v4i32_v4i8_store:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512BW-NEXT:    vpmovsdb %zmm0, %xmm0
+; AVX512BW-NEXT:    vmovd %xmm0, (%rdi)
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: trunc_ssat_v4i32_v4i8_store:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpmovsdb %xmm0, (%rdi)
+; AVX512BWVL-NEXT:    retq
+;
+; SKX-LABEL: trunc_ssat_v4i32_v4i8_store:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpmovsdb %xmm0, (%rdi)
+; SKX-NEXT:    retq
+  %1 = icmp slt <4 x i32> %a0, <i32 127, i32 127, i32 127, i32 127>
+  %2 = select <4 x i1> %1, <4 x i32> %a0, <4 x i32> <i32 127, i32 127, i32 127, i32 127>
+  %3 = icmp sgt <4 x i32> %2, <i32 -128, i32 -128, i32 -128, i32 -128>
+  %4 = select <4 x i1> %3, <4 x i32> %2, <4 x i32> <i32 -128, i32 -128, i32 -128, i32 -128>
+  %5 = trunc <4 x i32> %4 to <4 x i8>
+  store <4 x i8> %5, <4 x i8> *%p1
+  ret void
+}
+
+define <8 x i8> @trunc_ssat_v8i32_v8i8(<8 x i32> %a0) {
+; SSE-LABEL: trunc_ssat_v8i32_v8i8:
+; SSE:       # %bb.0:
+; SSE-NEXT:    packssdw %xmm1, %xmm0
+; SSE-NEXT:    packsswb %xmm0, %xmm0
+; SSE-NEXT:    retq
+;
 ; AVX1-LABEL: trunc_ssat_v8i32_v8i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [127,127,127,127]
-; AVX1-NEXT:    vpminsd %xmm1, %xmm0, %xmm2
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [4294967168,4294967168,4294967168,4294967168]
-; AVX1-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpmaxsd %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vpackssdw %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: trunc_ssat_v8i32_v8i8:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [127,127,127,127,127,127,127,127]
-; AVX2-NEXT:    vpminsd %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [4294967168,4294967168,4294967168,4294967168,4294967168,4294967168,4294967168,4294967168]
-; AVX2-NEXT:    vpmaxsd %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: trunc_ssat_v8i32_v8i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [127,127,127,127,127,127,127,127]
-; AVX512F-NEXT:    vpminsd %ymm1, %ymm0, %ymm0
-; AVX512F-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [4294967168,4294967168,4294967168,4294967168,4294967168,4294967168,4294967168,4294967168]
-; AVX512F-NEXT:    vpmaxsd %ymm1, %ymm0, %ymm0
-; AVX512F-NEXT:    vpmovdw %zmm0, %ymm0
-; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: trunc_ssat_v8i32_v8i8:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpminsd {{.*}}(%rip){1to8}, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpmaxsd {{.*}}(%rip){1to8}, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpmovdw %ymm0, %xmm0
+; AVX512VL-NEXT:    vpmovsdb %ymm0, %xmm0
 ; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: trunc_ssat_v8i32_v8i8:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [127,127,127,127,127,127,127,127]
-; AVX512BW-NEXT:    vpminsd %ymm1, %ymm0, %ymm0
-; AVX512BW-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [4294967168,4294967168,4294967168,4294967168,4294967168,4294967168,4294967168,4294967168]
-; AVX512BW-NEXT:    vpmaxsd %ymm1, %ymm0, %ymm0
-; AVX512BW-NEXT:    vpmovdw %zmm0, %ymm0
-; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512BW-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512BWVL-LABEL: trunc_ssat_v8i32_v8i8:
 ; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vpminsd {{.*}}(%rip){1to8}, %ymm0, %ymm0
-; AVX512BWVL-NEXT:    vpmaxsd {{.*}}(%rip){1to8}, %ymm0, %ymm0
-; AVX512BWVL-NEXT:    vpmovdw %ymm0, %xmm0
+; AVX512BWVL-NEXT:    vpmovsdb %ymm0, %xmm0
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
+;
+; SKX-LABEL: trunc_ssat_v8i32_v8i8:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpmovsdb %ymm0, %xmm0
+; SKX-NEXT:    vzeroupper
+; SKX-NEXT:    retq
   %1 = icmp slt <8 x i32> %a0, <i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127>
   %2 = select <8 x i1> %1, <8 x i32> %a0, <8 x i32> <i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127>
   %3 = icmp sgt <8 x i32> %2, <i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128>
@@ -2901,11 +5333,8 @@ define void @trunc_ssat_v8i32_v8i8_store(<8 x i32> %a0, <8 x i8> *%p1) {
 ;
 ; AVX512F-LABEL: trunc_ssat_v8i32_v8i8_store:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [127,127,127,127,127,127,127,127]
-; AVX512F-NEXT:    vpminsd %ymm1, %ymm0, %ymm0
-; AVX512F-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [4294967168,4294967168,4294967168,4294967168,4294967168,4294967168,4294967168,4294967168]
-; AVX512F-NEXT:    vpmaxsd %ymm1, %ymm0, %ymm0
-; AVX512F-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
 ; AVX512F-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
 ; AVX512F-NEXT:    vmovq %xmm0, (%rdi)
 ; AVX512F-NEXT:    vzeroupper
@@ -2919,11 +5348,8 @@ define void @trunc_ssat_v8i32_v8i8_store(<8 x i32> %a0, <8 x i8> *%p1) {
 ;
 ; AVX512BW-LABEL: trunc_ssat_v8i32_v8i8_store:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [127,127,127,127,127,127,127,127]
-; AVX512BW-NEXT:    vpminsd %ymm1, %ymm0, %ymm0
-; AVX512BW-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [4294967168,4294967168,4294967168,4294967168,4294967168,4294967168,4294967168,4294967168]
-; AVX512BW-NEXT:    vpmaxsd %ymm1, %ymm0, %ymm0
-; AVX512BW-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512BW-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vmovq %xmm0, (%rdi)
 ; AVX512BW-NEXT:    vzeroupper
@@ -2934,6 +5360,12 @@ define void @trunc_ssat_v8i32_v8i8_store(<8 x i32> %a0, <8 x i8> *%p1) {
 ; AVX512BWVL-NEXT:    vpmovsdb %ymm0, (%rdi)
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
+;
+; SKX-LABEL: trunc_ssat_v8i32_v8i8_store:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpmovsdb %ymm0, (%rdi)
+; SKX-NEXT:    vzeroupper
+; SKX-NEXT:    retq
   %1 = icmp slt <8 x i32> %a0, <i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127>
   %2 = select <8 x i1> %1, <8 x i32> %a0, <8 x i32> <i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127>
   %3 = icmp sgt <8 x i32> %2, <i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128>
@@ -2943,27 +5375,29 @@ define void @trunc_ssat_v8i32_v8i8_store(<8 x i32> %a0, <8 x i8> *%p1) {
   ret void
 }
 
-define <16 x i8> @trunc_ssat_v16i32_v16i8(<16 x i32> %a0) {
+define <16 x i8> @trunc_ssat_v16i32_v16i8(<16 x i32>* %p0) "min-legal-vector-width"="256" {
 ; SSE-LABEL: trunc_ssat_v16i32_v16i8:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    packssdw %xmm3, %xmm2
-; SSE-NEXT:    packssdw %xmm1, %xmm0
-; SSE-NEXT:    packsswb %xmm2, %xmm0
+; SSE-NEXT:    movdqa (%rdi), %xmm0
+; SSE-NEXT:    movdqa 32(%rdi), %xmm1
+; SSE-NEXT:    packssdw 48(%rdi), %xmm1
+; SSE-NEXT:    packssdw 16(%rdi), %xmm0
+; SSE-NEXT:    packsswb %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: trunc_ssat_v16i32_v16i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT:    vpackssdw %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX1-NEXT:    vpackssdw %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX1-NEXT:    vmovdqa 32(%rdi), %xmm1
+; AVX1-NEXT:    vpackssdw 48(%rdi), %xmm1, %xmm1
+; AVX1-NEXT:    vpackssdw 16(%rdi), %xmm0, %xmm0
 ; AVX1-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: trunc_ssat_v16i32_v16i8:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpackssdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX2-NEXT:    vpackssdw 32(%rdi), %ymm0, %ymm0
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
@@ -2972,9 +5406,21 @@ define <16 x i8> @trunc_ssat_v16i32_v16i8(<16 x i32> %a0) {
 ;
 ; AVX512-LABEL: trunc_ssat_v16i32_v16i8:
 ; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovdqa64 (%rdi), %zmm0
 ; AVX512-NEXT:    vpmovsdb %zmm0, %xmm0
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
+;
+; SKX-LABEL: trunc_ssat_v16i32_v16i8:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vmovdqa (%rdi), %ymm0
+; SKX-NEXT:    vmovdqa 32(%rdi), %ymm1
+; SKX-NEXT:    vpmovsdb %ymm1, %xmm1
+; SKX-NEXT:    vpmovsdb %ymm0, %xmm0
+; SKX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SKX-NEXT:    vzeroupper
+; SKX-NEXT:    retq
+  %a0 = load <16 x i32>, <16 x i32>* %p0
   %1 = icmp slt <16 x i32> %a0, <i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127>
   %2 = select <16 x i1> %1, <16 x i32> %a0, <16 x i32> <i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127>
   %3 = icmp sgt <16 x i32> %2, <i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128>
@@ -2983,6 +5429,142 @@ define <16 x i8> @trunc_ssat_v16i32_v16i8(<16 x i32> %a0) {
   ret <16 x i8> %5
 }
 
+define void @trunc_ssat_v16i32_v16i8_store(<16 x i32>* %p0, <16 x i8>* %p1) "min-legal-vector-width"="256" {
+; SSE-LABEL: trunc_ssat_v16i32_v16i8_store:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqa (%rdi), %xmm0
+; SSE-NEXT:    movdqa 32(%rdi), %xmm1
+; SSE-NEXT:    packssdw 48(%rdi), %xmm1
+; SSE-NEXT:    packssdw 16(%rdi), %xmm0
+; SSE-NEXT:    packsswb %xmm1, %xmm0
+; SSE-NEXT:    movdqa %xmm0, (%rsi)
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: trunc_ssat_v16i32_v16i8_store:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX1-NEXT:    vmovdqa 32(%rdi), %xmm1
+; AVX1-NEXT:    vpackssdw 48(%rdi), %xmm1, %xmm1
+; AVX1-NEXT:    vpackssdw 16(%rdi), %xmm0, %xmm0
+; AVX1-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vmovdqa %xmm0, (%rsi)
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: trunc_ssat_v16i32_v16i8_store:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX2-NEXT:    vpackssdw 32(%rdi), %ymm0, %ymm0
+; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vmovdqa %xmm0, (%rsi)
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: trunc_ssat_v16i32_v16i8_store:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovdqa64 (%rdi), %zmm0
+; AVX512-NEXT:    vpmovsdb %zmm0, (%rsi)
+; AVX512-NEXT:    vzeroupper
+; AVX512-NEXT:    retq
+;
+; SKX-LABEL: trunc_ssat_v16i32_v16i8_store:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vmovdqa (%rdi), %ymm0
+; SKX-NEXT:    vmovdqa 32(%rdi), %ymm1
+; SKX-NEXT:    vpmovsdb %ymm1, %xmm1
+; SKX-NEXT:    vpmovsdb %ymm0, %xmm0
+; SKX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SKX-NEXT:    vmovdqa %xmm0, (%rsi)
+; SKX-NEXT:    vzeroupper
+; SKX-NEXT:    retq
+  %a0 = load <16 x i32>, <16 x i32>* %p0
+  %1 = icmp slt <16 x i32> %a0, <i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127>
+  %2 = select <16 x i1> %1, <16 x i32> %a0, <16 x i32> <i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127>
+  %3 = icmp sgt <16 x i32> %2, <i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128>
+  %4 = select <16 x i1> %3, <16 x i32> %2, <16 x i32> <i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128>
+  %5 = trunc <16 x i32> %4 to <16 x i8>
+  store <16 x i8> %5, <16 x i8>* %p1
+  ret void
+}
+
+define <8 x i8> @trunc_ssat_v8i16_v8i8(<8 x i16> %a0) {
+; SSE-LABEL: trunc_ssat_v8i16_v8i8:
+; SSE:       # %bb.0:
+; SSE-NEXT:    packsswb %xmm0, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: trunc_ssat_v8i16_v8i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: trunc_ssat_v8i16_v8i8:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
+; AVX512-NEXT:    retq
+;
+; SKX-LABEL: trunc_ssat_v8i16_v8i8:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
+; SKX-NEXT:    retq
+  %1 = icmp slt <8 x i16> %a0, <i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127>
+  %2 = select <8 x i1> %1, <8 x i16> %a0, <8 x i16> <i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127>
+  %3 = icmp sgt <8 x i16> %2, <i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128>
+  %4 = select <8 x i1> %3, <8 x i16> %2, <8 x i16> <i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128>
+  %5 = trunc <8 x i16> %4 to <8 x i8>
+  ret <8 x i8> %5
+}
+
+define void @trunc_ssat_v8i16_v8i8_store(<8 x i16> %a0, <8 x i8> *%p1) {
+; SSE-LABEL: trunc_ssat_v8i16_v8i8_store:
+; SSE:       # %bb.0:
+; SSE-NEXT:    packsswb %xmm0, %xmm0
+; SSE-NEXT:    movq %xmm0, (%rdi)
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: trunc_ssat_v8i16_v8i8_store:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vmovq %xmm0, (%rdi)
+; AVX-NEXT:    retq
+;
+; AVX512F-LABEL: trunc_ssat_v8i16_v8i8_store:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
+; AVX512F-NEXT:    vmovq %xmm0, (%rdi)
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: trunc_ssat_v8i16_v8i8_store:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
+; AVX512VL-NEXT:    vmovq %xmm0, (%rdi)
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: trunc_ssat_v8i16_v8i8_store:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
+; AVX512BW-NEXT:    vmovq %xmm0, (%rdi)
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: trunc_ssat_v8i16_v8i8_store:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpmovswb %xmm0, (%rdi)
+; AVX512BWVL-NEXT:    retq
+;
+; SKX-LABEL: trunc_ssat_v8i16_v8i8_store:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpmovswb %xmm0, (%rdi)
+; SKX-NEXT:    retq
+  %1 = icmp slt <8 x i16> %a0, <i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127>
+  %2 = select <8 x i1> %1, <8 x i16> %a0, <8 x i16> <i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127>
+  %3 = icmp sgt <8 x i16> %2, <i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128>
+  %4 = select <8 x i1> %3, <8 x i16> %2, <8 x i16> <i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128>
+  %5 = trunc <8 x i16> %4 to <8 x i8>
+  store <8 x i8> %5, <8 x i8> *%p1
+  ret void
+}
+
 define <16 x i8> @trunc_ssat_v16i16_v16i8(<16 x i16> %a0) {
 ; SSE-LABEL: trunc_ssat_v16i16_v16i8:
 ; SSE:       # %bb.0:
@@ -3005,28 +5587,22 @@ define <16 x i8> @trunc_ssat_v16i16_v16i8(<16 x i16> %a0) {
 ;
 ; AVX512F-LABEL: trunc_ssat_v16i16_v16i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpminsw {{.*}}(%rip), %ymm0, %ymm0
-; AVX512F-NEXT:    vpmaxsw {{.*}}(%rip), %ymm0, %ymm0
-; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
-; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: trunc_ssat_v16i16_v16i8:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpminsw {{.*}}(%rip), %ymm0, %ymm0
-; AVX512VL-NEXT:    vpmaxsw {{.*}}(%rip), %ymm0, %ymm0
-; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
-; AVX512VL-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512VL-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512VL-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: trunc_ssat_v16i16_v16i8:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpminsw {{.*}}(%rip), %ymm0, %ymm0
-; AVX512BW-NEXT:    vpmaxsw {{.*}}(%rip), %ymm0, %ymm0
-; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
-; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512BW-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
@@ -3035,6 +5611,12 @@ define <16 x i8> @trunc_ssat_v16i16_v16i8(<16 x i16> %a0) {
 ; AVX512BWVL-NEXT:    vpmovswb %ymm0, %xmm0
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
+;
+; SKX-LABEL: trunc_ssat_v16i16_v16i8:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpmovswb %ymm0, %xmm0
+; SKX-NEXT:    vzeroupper
+; SKX-NEXT:    retq
   %1 = icmp slt <16 x i16> %a0, <i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127>
   %2 = select <16 x i1> %1, <16 x i16> %a0, <16 x i16> <i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127>
   %3 = icmp sgt <16 x i16> %2, <i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128>
@@ -3043,68 +5625,64 @@ define <16 x i8> @trunc_ssat_v16i16_v16i8(<16 x i16> %a0) {
   ret <16 x i8> %5
 }
 
-define <32 x i8> @trunc_ssat_v32i16_v32i8(<32 x i16> %a0) {
+define <32 x i8> @trunc_ssat_v32i16_v32i8(<32 x i16>* %p0) "min-legal-vector-width"="256" {
 ; SSE-LABEL: trunc_ssat_v32i16_v32i8:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    packsswb %xmm1, %xmm0
-; SSE-NEXT:    packsswb %xmm3, %xmm2
-; SSE-NEXT:    movdqa %xmm2, %xmm1
+; SSE-NEXT:    movdqa (%rdi), %xmm0
+; SSE-NEXT:    movdqa 32(%rdi), %xmm1
+; SSE-NEXT:    packsswb 16(%rdi), %xmm0
+; SSE-NEXT:    packsswb 48(%rdi), %xmm1
 ; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: trunc_ssat_v32i16_v32i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT:    vpacksswb %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX1-NEXT:    vpacksswb %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX1-NEXT:    vmovdqa 32(%rdi), %xmm1
+; AVX1-NEXT:    vpacksswb 48(%rdi), %xmm1, %xmm1
+; AVX1-NEXT:    vpacksswb 16(%rdi), %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: trunc_ssat_v32i16_v32i8:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpacksswb %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX2-NEXT:    vpacksswb 32(%rdi), %ymm0, %ymm0
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: trunc_ssat_v32i16_v32i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm2 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; AVX512F-NEXT:    vpminsw %ymm2, %ymm0, %ymm0
-; AVX512F-NEXT:    vpminsw %ymm2, %ymm1, %ymm1
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm2 = [65408,65408,65408,65408,65408,65408,65408,65408,65408,65408,65408,65408,65408,65408,65408,65408]
-; AVX512F-NEXT:    vpmaxsw %ymm2, %ymm1, %ymm1
-; AVX512F-NEXT:    vpmaxsw %ymm2, %ymm0, %ymm0
-; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
-; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
-; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
-; AVX512F-NEXT:    vpmovdb %zmm1, %xmm1
-; AVX512F-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX512F-NEXT:    vpacksswb 32(%rdi), %ymm0, %ymm0
+; AVX512F-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: trunc_ssat_v32i16_v32i8:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm2 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; AVX512VL-NEXT:    vpminsw %ymm2, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpminsw %ymm2, %ymm1, %ymm1
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm2 = [65408,65408,65408,65408,65408,65408,65408,65408,65408,65408,65408,65408,65408,65408,65408,65408]
-; AVX512VL-NEXT:    vpmaxsw %ymm2, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpmaxsw %ymm2, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
-; AVX512VL-NEXT:    vpmovdb %zmm0, %xmm0
-; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
-; AVX512VL-NEXT:    vpmovdb %zmm1, %xmm1
-; AVX512VL-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX512VL-NEXT:    vpacksswb 32(%rdi), %ymm0, %ymm0
+; AVX512VL-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: trunc_ssat_v32i16_v32i8:
 ; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm0
 ; AVX512BW-NEXT:    vpmovswb %zmm0, %ymm0
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512BWVL-LABEL: trunc_ssat_v32i16_v32i8:
 ; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vmovdqa64 (%rdi), %zmm0
 ; AVX512BWVL-NEXT:    vpmovswb %zmm0, %ymm0
 ; AVX512BWVL-NEXT:    retq
+;
+; SKX-LABEL: trunc_ssat_v32i16_v32i8:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vmovdqa (%rdi), %ymm0
+; SKX-NEXT:    vpacksswb 32(%rdi), %ymm0, %ymm0
+; SKX-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; SKX-NEXT:    retq
+  %a0 = load <32 x i16>, <32 x i16>* %p0
   %1 = icmp slt <32 x i16> %a0, <i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127>
   %2 = select <32 x i1> %1, <32 x i16> %a0, <32 x i16> <i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127>
   %3 = icmp sgt <32 x i16> %2, <i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128>
@@ -3112,3 +5690,74 @@ define <32 x i8> @trunc_ssat_v32i16_v32i8(<32 x i16> %a0) {
   %5 = trunc <32 x i16> %4 to <32 x i8>
   ret <32 x i8> %5
 }
+
+define <32 x i8> @trunc_ssat_v32i32_v32i8(<32 x i32>* %p0) "min-legal-vector-width"="256" {
+; SSE-LABEL: trunc_ssat_v32i32_v32i8:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqa (%rdi), %xmm0
+; SSE-NEXT:    movdqa 32(%rdi), %xmm2
+; SSE-NEXT:    movdqa 64(%rdi), %xmm1
+; SSE-NEXT:    movdqa 96(%rdi), %xmm3
+; SSE-NEXT:    packssdw 48(%rdi), %xmm2
+; SSE-NEXT:    packssdw 16(%rdi), %xmm0
+; SSE-NEXT:    packsswb %xmm2, %xmm0
+; SSE-NEXT:    packssdw 112(%rdi), %xmm3
+; SSE-NEXT:    packssdw 80(%rdi), %xmm1
+; SSE-NEXT:    packsswb %xmm3, %xmm1
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: trunc_ssat_v32i32_v32i8:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX1-NEXT:    vmovdqa 32(%rdi), %xmm1
+; AVX1-NEXT:    vmovdqa 64(%rdi), %xmm2
+; AVX1-NEXT:    vmovdqa 96(%rdi), %xmm3
+; AVX1-NEXT:    vpackssdw 112(%rdi), %xmm3, %xmm3
+; AVX1-NEXT:    vpackssdw 80(%rdi), %xmm2, %xmm2
+; AVX1-NEXT:    vpacksswb %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpackssdw 48(%rdi), %xmm1, %xmm1
+; AVX1-NEXT:    vpackssdw 16(%rdi), %xmm0, %xmm0
+; AVX1-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: trunc_ssat_v32i32_v32i8:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX2-NEXT:    vmovdqa 64(%rdi), %ymm1
+; AVX2-NEXT:    vpackssdw 96(%rdi), %ymm1, %ymm1
+; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
+; AVX2-NEXT:    vpackssdw 32(%rdi), %ymm0, %ymm0
+; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2-NEXT:    vpacksswb %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: trunc_ssat_v32i32_v32i8:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovdqa64 (%rdi), %zmm0
+; AVX512-NEXT:    vmovdqa64 64(%rdi), %zmm1
+; AVX512-NEXT:    vpmovsdb %zmm0, %xmm0
+; AVX512-NEXT:    vpmovsdb %zmm1, %xmm1
+; AVX512-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512-NEXT:    retq
+;
+; SKX-LABEL: trunc_ssat_v32i32_v32i8:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vmovdqa (%rdi), %ymm0
+; SKX-NEXT:    vmovdqa 64(%rdi), %ymm1
+; SKX-NEXT:    vpackssdw 96(%rdi), %ymm1, %ymm1
+; SKX-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
+; SKX-NEXT:    vpackssdw 32(%rdi), %ymm0, %ymm0
+; SKX-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; SKX-NEXT:    vpacksswb %ymm1, %ymm0, %ymm0
+; SKX-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; SKX-NEXT:    retq
+  %a0 = load <32 x i32>, <32 x i32>* %p0
+  %1 = icmp slt <32 x i32> %a0, <i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127>
+  %2 = select <32 x i1> %1, <32 x i32> %a0, <32 x i32> <i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127>
+  %3 = icmp sgt <32 x i32> %2, <i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128>
+  %4 = select <32 x i1> %3, <32 x i32> %2, <32 x i32> <i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128>
+  %5 = trunc <32 x i32> %4 to <32 x i8>
+  ret <32 x i8> %5
+}
diff --git a/llvm/test/CodeGen/X86/vector-trunc-usat.ll b/llvm/test/CodeGen/X86/vector-trunc-usat.ll
index 23cb25df8..7afdb04f6 100644
--- a/llvm/test/CodeGen/X86/vector-trunc-usat.ll
+++ b/llvm/test/CodeGen/X86/vector-trunc-usat.ll
@@ -9,11 +9,221 @@
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vl,+fast-variable-shuffle | FileCheck %s --check-prefixes=AVX512,AVX512VL
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+fast-variable-shuffle | FileCheck %s --check-prefixes=AVX512,AVX512BW
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl,+fast-variable-shuffle | FileCheck %s --check-prefixes=AVX512,AVX512BWVL
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=skx | FileCheck %s --check-prefixes=SKX
 
 ;
 ; Unsigned saturation truncation to vXi32
 ;
 
+define <2 x i32> @trunc_usat_v2i64_v2i32(<2 x i64> %a0) {
+; SSE2-LABEL: trunc_usat_v2i64_v2i32:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [9223372039002259456,9223372039002259456]
+; SSE2-NEXT:    pxor %xmm0, %xmm1
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [9223372039002259455,9223372039002259455]
+; SSE2-NEXT:    movdqa %xmm2, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pand %xmm4, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,1,3,3]
+; SSE2-NEXT:    por %xmm1, %xmm2
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm2
+; SSE2-NEXT:    por %xmm0, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: trunc_usat_v2i64_v2i32:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [9223372039002259456,9223372039002259456]
+; SSSE3-NEXT:    pxor %xmm0, %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [9223372039002259455,9223372039002259455]
+; SSSE3-NEXT:    movdqa %xmm2, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm4, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,1,3,3]
+; SSSE3-NEXT:    por %xmm1, %xmm2
+; SSSE3-NEXT:    pand %xmm2, %xmm0
+; SSSE3-NEXT:    pandn {{.*}}(%rip), %xmm2
+; SSSE3-NEXT:    por %xmm0, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: trunc_usat_v2i64_v2i32:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    movapd {{.*#+}} xmm2 = [4294967295,4294967295]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [9223372039002259456,9223372039002259456]
+; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [9223372039002259455,9223372039002259455]
+; SSE41-NEXT:    movdqa %xmm3, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm3, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: trunc_usat_v2i64_v2i32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovapd {{.*#+}} xmm1 = [4294967295,4294967295]
+; AVX-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm2
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm3 = [9223372041149743103,9223372041149743103]
+; AVX-NEXT:    vpcmpgtq %xmm2, %xmm3, %xmm2
+; AVX-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX-NEXT:    retq
+;
+; AVX512F-LABEL: trunc_usat_v2i64_v2i32:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512F-NEXT:    vpmovusqd %zmm0, %ymm0
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: trunc_usat_v2i64_v2i32:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpmovusqd %xmm0, %xmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: trunc_usat_v2i64_v2i32:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512BW-NEXT:    vpmovusqd %zmm0, %ymm0
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: trunc_usat_v2i64_v2i32:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpmovusqd %xmm0, %xmm0
+; AVX512BWVL-NEXT:    retq
+;
+; SKX-LABEL: trunc_usat_v2i64_v2i32:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpmovusqd %xmm0, %xmm0
+; SKX-NEXT:    retq
+  %1 = icmp ult <2 x i64> %a0, <i64 4294967295, i64 4294967295>
+  %2 = select <2 x i1> %1, <2 x i64> %a0, <2 x i64> <i64 4294967295, i64 4294967295>
+  %3 = trunc <2 x i64> %2 to <2 x i32>
+  ret <2 x i32> %3
+}
+
+define void @trunc_usat_v2i64_v2i32_store(<2 x i64> %a0, <2 x i32>* %p1) {
+; SSE2-LABEL: trunc_usat_v2i64_v2i32_store:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [9223372039002259456,9223372039002259456]
+; SSE2-NEXT:    pxor %xmm0, %xmm1
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [9223372039002259455,9223372039002259455]
+; SSE2-NEXT:    movdqa %xmm2, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pand %xmm4, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,1,3,3]
+; SSE2-NEXT:    por %xmm1, %xmm2
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm2
+; SSE2-NEXT:    por %xmm0, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
+; SSE2-NEXT:    movq %xmm0, (%rdi)
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: trunc_usat_v2i64_v2i32_store:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [9223372039002259456,9223372039002259456]
+; SSSE3-NEXT:    pxor %xmm0, %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [9223372039002259455,9223372039002259455]
+; SSSE3-NEXT:    movdqa %xmm2, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm4, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,1,3,3]
+; SSSE3-NEXT:    por %xmm1, %xmm2
+; SSSE3-NEXT:    pand %xmm2, %xmm0
+; SSSE3-NEXT:    pandn {{.*}}(%rip), %xmm2
+; SSSE3-NEXT:    por %xmm0, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
+; SSSE3-NEXT:    movq %xmm0, (%rdi)
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: trunc_usat_v2i64_v2i32_store:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    movapd {{.*#+}} xmm2 = [4294967295,4294967295]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [9223372039002259456,9223372039002259456]
+; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [9223372039002259455,9223372039002259455]
+; SSE41-NEXT:    movdqa %xmm3, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm3, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
+; SSE41-NEXT:    movq %xmm0, (%rdi)
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: trunc_usat_v2i64_v2i32_store:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovapd {{.*#+}} xmm1 = [4294967295,4294967295]
+; AVX-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm2
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm3 = [9223372041149743103,9223372041149743103]
+; AVX-NEXT:    vpcmpgtq %xmm2, %xmm3, %xmm2
+; AVX-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX-NEXT:    vmovlpd %xmm0, (%rdi)
+; AVX-NEXT:    retq
+;
+; AVX512F-LABEL: trunc_usat_v2i64_v2i32_store:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512F-NEXT:    vpmovusqd %zmm0, %ymm0
+; AVX512F-NEXT:    vmovq %xmm0, (%rdi)
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: trunc_usat_v2i64_v2i32_store:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpmovusqd %xmm0, (%rdi)
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: trunc_usat_v2i64_v2i32_store:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512BW-NEXT:    vpmovusqd %zmm0, %ymm0
+; AVX512BW-NEXT:    vmovq %xmm0, (%rdi)
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: trunc_usat_v2i64_v2i32_store:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpmovusqd %xmm0, (%rdi)
+; AVX512BWVL-NEXT:    retq
+;
+; SKX-LABEL: trunc_usat_v2i64_v2i32_store:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpmovusqd %xmm0, (%rdi)
+; SKX-NEXT:    retq
+  %1 = icmp ult <2 x i64> %a0, <i64 4294967295, i64 4294967295>
+  %2 = select <2 x i1> %1, <2 x i64> %a0, <2 x i64> <i64 4294967295, i64 4294967295>
+  %3 = trunc <2 x i64> %2 to <2 x i32>
+  store <2 x i32> %3, <2 x i32>* %p1
+  ret void
+}
+
 define <4 x i32> @trunc_usat_v4i64_v4i32(<4 x i64> %a0) {
 ; SSE2-LABEL: trunc_usat_v4i64_v4i32:
 ; SSE2:       # %bb.0:
@@ -87,20 +297,19 @@ define <4 x i32> @trunc_usat_v4i64_v4i32(<4 x i64> %a0) {
 ; SSE41-NEXT:    pxor %xmm0, %xmm3
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [9223372039002259455,9223372039002259455]
 ; SSE41-NEXT:    movdqa %xmm4, %xmm5
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm4, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pand %xmm6, %xmm3
-; SSE41-NEXT:    por %xmm5, %xmm3
+; SSE41-NEXT:    pcmpeqd %xmm3, %xmm5
+; SSE41-NEXT:    movdqa %xmm4, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm6[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm3
+; SSE41-NEXT:    por %xmm6, %xmm3
 ; SSE41-NEXT:    pxor %xmm1, %xmm0
 ; SSE41-NEXT:    movdqa %xmm4, %xmm5
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm4, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm6, %xmm0
-; SSE41-NEXT:    por %xmm5, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm0
+; SSE41-NEXT:    por %xmm4, %xmm0
 ; SSE41-NEXT:    movapd {{.*#+}} xmm4 = [4294967295,4294967295]
 ; SSE41-NEXT:    movapd {{.*#+}} xmm5 = [4294967295,429496729]
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm5
@@ -157,8 +366,7 @@ define <4 x i32> @trunc_usat_v4i64_v4i32(<4 x i64> %a0) {
 ; AVX512F-LABEL: trunc_usat_v4i64_v4i32:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; AVX512F-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [4294967295,4294967295,4294967295,4294967295]
-; AVX512F-NEXT:    vpcmpltuq %zmm1, %zmm0, %k1
+; AVX512F-NEXT:    vpcmpltuq {{.*}}(%rip){1to8}, %zmm0, %k1
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm1 = [4294967295,4294967295,4294967295,429496729]
 ; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
 ; AVX512F-NEXT:    vpmovqd %zmm1, %ymm0
@@ -178,8 +386,7 @@ define <4 x i32> @trunc_usat_v4i64_v4i32(<4 x i64> %a0) {
 ; AVX512BW-LABEL: trunc_usat_v4i64_v4i32:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; AVX512BW-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [4294967295,4294967295,4294967295,4294967295]
-; AVX512BW-NEXT:    vpcmpltuq %zmm1, %zmm0, %k1
+; AVX512BW-NEXT:    vpcmpltuq {{.*}}(%rip){1to8}, %zmm0, %k1
 ; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm1 = [4294967295,4294967295,4294967295,429496729]
 ; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
 ; AVX512BW-NEXT:    vpmovqd %zmm1, %ymm0
@@ -195,215 +402,235 @@ define <4 x i32> @trunc_usat_v4i64_v4i32(<4 x i64> %a0) {
 ; AVX512BWVL-NEXT:    vpmovqd %ymm1, %xmm0
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
+;
+; SKX-LABEL: trunc_usat_v4i64_v4i32:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpcmpltuq {{.*}}(%rip){1to4}, %ymm0, %k1
+; SKX-NEXT:    vmovdqa {{.*#+}} ymm1 = [4294967295,4294967295,4294967295,429496729]
+; SKX-NEXT:    vmovdqa64 %ymm0, %ymm1 {%k1}
+; SKX-NEXT:    vpmovqd %ymm1, %xmm0
+; SKX-NEXT:    vzeroupper
+; SKX-NEXT:    retq
   %1 = icmp ult <4 x i64> %a0, <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>
   %2 = select <4 x i1> %1, <4 x i64> %a0, <4 x i64> <i64 4294967295, i64 4294967295, i64 4294967295, i64 429496729>
   %3 = trunc <4 x i64> %2 to <4 x i32>
   ret <4 x i32> %3
 }
 
-define <8 x i32> @trunc_usat_v8i64_v8i32(<8 x i64> %a0) {
+define <8 x i32> @trunc_usat_v8i64_v8i32(<8 x i64>* %p0) {
 ; SSE2-LABEL: trunc_usat_v8i64_v8i32:
 ; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa (%rdi), %xmm9
+; SSE2-NEXT:    movdqa 16(%rdi), %xmm5
+; SSE2-NEXT:    movdqa 32(%rdi), %xmm6
+; SSE2-NEXT:    movdqa 48(%rdi), %xmm1
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [4294967295,4294967295]
-; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [9223372039002259456,9223372039002259456]
-; SSE2-NEXT:    movdqa %xmm3, %xmm7
-; SSE2-NEXT:    pxor %xmm5, %xmm7
-; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [9223372039002259455,9223372039002259455]
-; SSE2-NEXT:    movdqa %xmm9, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm7, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm7
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [9223372039002259456,9223372039002259456]
+; SSE2-NEXT:    movdqa %xmm1, %xmm7
+; SSE2-NEXT:    pxor %xmm0, %xmm7
+; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [9223372039002259455,9223372039002259455]
+; SSE2-NEXT:    movdqa %xmm10, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm7, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm7
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT:    pand %xmm4, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[1,1,3,3]
-; SSE2-NEXT:    por %xmm7, %xmm4
-; SSE2-NEXT:    pand %xmm4, %xmm3
-; SSE2-NEXT:    pandn %xmm8, %xmm4
-; SSE2-NEXT:    por %xmm3, %xmm4
-; SSE2-NEXT:    movdqa %xmm2, %xmm3
-; SSE2-NEXT:    pxor %xmm5, %xmm3
-; SSE2-NEXT:    movdqa %xmm9, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm3[1,1,3,3]
-; SSE2-NEXT:    pand %xmm10, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm6[1,1,3,3]
-; SSE2-NEXT:    por %xmm7, %xmm3
-; SSE2-NEXT:    pand %xmm3, %xmm2
-; SSE2-NEXT:    pandn %xmm8, %xmm3
-; SSE2-NEXT:    por %xmm2, %xmm3
-; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,2],xmm4[0,2]
-; SSE2-NEXT:    movdqa %xmm1, %xmm2
-; SSE2-NEXT:    pxor %xmm5, %xmm2
-; SSE2-NEXT:    movdqa %xmm9, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm2, %xmm4
-; SSE2-NEXT:    pand %xmm4, %xmm1
-; SSE2-NEXT:    pandn %xmm8, %xmm4
-; SSE2-NEXT:    por %xmm1, %xmm4
-; SSE2-NEXT:    pxor %xmm0, %xmm5
-; SSE2-NEXT:    movdqa %xmm9, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm5, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE2-NEXT:    pand %xmm2, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm5, %xmm1
-; SSE2-NEXT:    pand %xmm1, %xmm0
+; SSE2-NEXT:    pand %xmm2, %xmm7
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm7, %xmm2
+; SSE2-NEXT:    pand %xmm2, %xmm1
+; SSE2-NEXT:    pandn %xmm8, %xmm2
+; SSE2-NEXT:    por %xmm1, %xmm2
+; SSE2-NEXT:    movdqa %xmm6, %xmm1
+; SSE2-NEXT:    pxor %xmm0, %xmm1
+; SSE2-NEXT:    movdqa %xmm10, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pand %xmm7, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm3, %xmm1
+; SSE2-NEXT:    pand %xmm1, %xmm6
 ; SSE2-NEXT:    pandn %xmm8, %xmm1
-; SSE2-NEXT:    por %xmm1, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm4[0,2]
-; SSE2-NEXT:    movaps %xmm3, %xmm1
+; SSE2-NEXT:    por %xmm6, %xmm1
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,2],xmm2[0,2]
+; SSE2-NEXT:    movdqa %xmm5, %xmm2
+; SSE2-NEXT:    pxor %xmm0, %xmm2
+; SSE2-NEXT:    movdqa %xmm10, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE2-NEXT:    pand %xmm4, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSE2-NEXT:    por %xmm2, %xmm3
+; SSE2-NEXT:    pand %xmm3, %xmm5
+; SSE2-NEXT:    pandn %xmm8, %xmm3
+; SSE2-NEXT:    por %xmm5, %xmm3
+; SSE2-NEXT:    pxor %xmm9, %xmm0
+; SSE2-NEXT:    movdqa %xmm10, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pand %xmm4, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3]
+; SSE2-NEXT:    por %xmm5, %xmm0
+; SSE2-NEXT:    pand %xmm0, %xmm9
+; SSE2-NEXT:    pandn %xmm8, %xmm0
+; SSE2-NEXT:    por %xmm9, %xmm0
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm3[0,2]
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: trunc_usat_v8i64_v8i32:
 ; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa (%rdi), %xmm9
+; SSSE3-NEXT:    movdqa 16(%rdi), %xmm5
+; SSSE3-NEXT:    movdqa 32(%rdi), %xmm6
+; SSSE3-NEXT:    movdqa 48(%rdi), %xmm1
 ; SSSE3-NEXT:    movdqa {{.*#+}} xmm8 = [4294967295,4294967295]
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm5 = [9223372039002259456,9223372039002259456]
-; SSSE3-NEXT:    movdqa %xmm3, %xmm7
-; SSSE3-NEXT:    pxor %xmm5, %xmm7
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm9 = [9223372039002259455,9223372039002259455]
-; SSSE3-NEXT:    movdqa %xmm9, %xmm6
-; SSSE3-NEXT:    pcmpgtd %xmm7, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm7
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [9223372039002259456,9223372039002259456]
+; SSSE3-NEXT:    movdqa %xmm1, %xmm7
+; SSSE3-NEXT:    pxor %xmm0, %xmm7
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm10 = [9223372039002259455,9223372039002259455]
+; SSSE3-NEXT:    movdqa %xmm10, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm7, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm7
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm4, %xmm7
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[1,1,3,3]
-; SSSE3-NEXT:    por %xmm7, %xmm4
-; SSSE3-NEXT:    pand %xmm4, %xmm3
-; SSSE3-NEXT:    pandn %xmm8, %xmm4
-; SSSE3-NEXT:    por %xmm3, %xmm4
-; SSSE3-NEXT:    movdqa %xmm2, %xmm3
-; SSSE3-NEXT:    pxor %xmm5, %xmm3
-; SSSE3-NEXT:    movdqa %xmm9, %xmm6
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm10 = xmm6[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm10, %xmm7
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm6[1,1,3,3]
-; SSSE3-NEXT:    por %xmm7, %xmm3
-; SSSE3-NEXT:    pand %xmm3, %xmm2
-; SSSE3-NEXT:    pandn %xmm8, %xmm3
-; SSSE3-NEXT:    por %xmm2, %xmm3
-; SSSE3-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,2],xmm4[0,2]
-; SSSE3-NEXT:    movdqa %xmm1, %xmm2
-; SSSE3-NEXT:    pxor %xmm5, %xmm2
-; SSSE3-NEXT:    movdqa %xmm9, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm6, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm4
-; SSSE3-NEXT:    pand %xmm4, %xmm1
-; SSSE3-NEXT:    pandn %xmm8, %xmm4
-; SSSE3-NEXT:    por %xmm1, %xmm4
-; SSSE3-NEXT:    pxor %xmm0, %xmm5
-; SSSE3-NEXT:    movdqa %xmm9, %xmm1
-; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm5
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm2, %xmm5
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    por %xmm5, %xmm1
-; SSSE3-NEXT:    pand %xmm1, %xmm0
+; SSSE3-NEXT:    pand %xmm2, %xmm7
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm7, %xmm2
+; SSSE3-NEXT:    pand %xmm2, %xmm1
+; SSSE3-NEXT:    pandn %xmm8, %xmm2
+; SSSE3-NEXT:    por %xmm1, %xmm2
+; SSSE3-NEXT:    movdqa %xmm6, %xmm1
+; SSSE3-NEXT:    pxor %xmm0, %xmm1
+; SSSE3-NEXT:    movdqa %xmm10, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm7, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm3, %xmm1
+; SSSE3-NEXT:    pand %xmm1, %xmm6
 ; SSSE3-NEXT:    pandn %xmm8, %xmm1
-; SSSE3-NEXT:    por %xmm1, %xmm0
-; SSSE3-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm4[0,2]
-; SSSE3-NEXT:    movaps %xmm3, %xmm1
+; SSSE3-NEXT:    por %xmm6, %xmm1
+; SSSE3-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,2],xmm2[0,2]
+; SSSE3-NEXT:    movdqa %xmm5, %xmm2
+; SSSE3-NEXT:    pxor %xmm0, %xmm2
+; SSSE3-NEXT:    movdqa %xmm10, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm4, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSSE3-NEXT:    por %xmm2, %xmm3
+; SSSE3-NEXT:    pand %xmm3, %xmm5
+; SSSE3-NEXT:    pandn %xmm8, %xmm3
+; SSSE3-NEXT:    por %xmm5, %xmm3
+; SSSE3-NEXT:    pxor %xmm9, %xmm0
+; SSSE3-NEXT:    movdqa %xmm10, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm4, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    por %xmm5, %xmm0
+; SSSE3-NEXT:    pand %xmm0, %xmm9
+; SSSE3-NEXT:    pandn %xmm8, %xmm0
+; SSSE3-NEXT:    por %xmm9, %xmm0
+; SSSE3-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm3[0,2]
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: trunc_usat_v8i64_v8i32:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa %xmm0, %xmm8
-; SSE41-NEXT:    movapd {{.*#+}} xmm6 = [4294967295,4294967295]
-; SSE41-NEXT:    movdqa {{.*#+}} xmm7 = [9223372039002259456,9223372039002259456]
-; SSE41-NEXT:    movdqa %xmm3, %xmm0
-; SSE41-NEXT:    pxor %xmm7, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm9 = [9223372039002259455,9223372039002259455]
-; SSE41-NEXT:    movdqa %xmm9, %xmm4
+; SSE41-NEXT:    movdqa (%rdi), %xmm8
+; SSE41-NEXT:    movdqa 16(%rdi), %xmm9
+; SSE41-NEXT:    movdqa 32(%rdi), %xmm7
+; SSE41-NEXT:    movdqa 48(%rdi), %xmm1
+; SSE41-NEXT:    movapd {{.*#+}} xmm2 = [4294967295,4294967295]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm5 = [9223372039002259456,9223372039002259456]
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pxor %xmm5, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [9223372039002259455,9223372039002259455]
+; SSE41-NEXT:    movdqa %xmm3, %xmm6
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm6
+; SSE41-NEXT:    movdqa %xmm3, %xmm4
 ; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm9, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pand %xmm6, %xmm0
 ; SSE41-NEXT:    por %xmm4, %xmm0
-; SSE41-NEXT:    movapd %xmm6, %xmm5
-; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm5
-; SSE41-NEXT:    movdqa %xmm2, %xmm0
-; SSE41-NEXT:    pxor %xmm7, %xmm0
-; SSE41-NEXT:    movdqa %xmm9, %xmm3
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm9, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE41-NEXT:    movapd %xmm2, %xmm4
+; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm4
+; SSE41-NEXT:    movdqa %xmm7, %xmm0
+; SSE41-NEXT:    pxor %xmm5, %xmm0
+; SSE41-NEXT:    movdqa %xmm3, %xmm1
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
+; SSE41-NEXT:    movdqa %xmm3, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
+; SSE41-NEXT:    pand %xmm1, %xmm0
+; SSE41-NEXT:    por %xmm6, %xmm0
+; SSE41-NEXT:    movapd %xmm2, %xmm1
+; SSE41-NEXT:    blendvpd %xmm0, %xmm7, %xmm1
+; SSE41-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,2],xmm4[0,2]
+; SSE41-NEXT:    movdqa %xmm9, %xmm0
+; SSE41-NEXT:    pxor %xmm5, %xmm0
+; SSE41-NEXT:    movdqa %xmm3, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT:    movdqa %xmm3, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
 ; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm6, %xmm0
+; SSE41-NEXT:    movapd %xmm2, %xmm4
+; SSE41-NEXT:    blendvpd %xmm0, %xmm9, %xmm4
+; SSE41-NEXT:    pxor %xmm8, %xmm5
+; SSE41-NEXT:    movdqa %xmm3, %xmm6
+; SSE41-NEXT:    pcmpeqd %xmm5, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm5, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pand %xmm6, %xmm0
 ; SSE41-NEXT:    por %xmm3, %xmm0
-; SSE41-NEXT:    movapd %xmm6, %xmm4
-; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm4
-; SSE41-NEXT:    shufps {{.*#+}} xmm4 = xmm4[0,2],xmm5[0,2]
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    pxor %xmm7, %xmm0
-; SSE41-NEXT:    movdqa %xmm9, %xmm2
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm9, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm3, %xmm0
-; SSE41-NEXT:    por %xmm2, %xmm0
-; SSE41-NEXT:    movapd %xmm6, %xmm2
-; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
-; SSE41-NEXT:    pxor %xmm8, %xmm7
-; SSE41-NEXT:    movdqa %xmm9, %xmm1
-; SSE41-NEXT:    pcmpgtd %xmm7, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm9, %xmm7
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[1,1,3,3]
-; SSE41-NEXT:    pand %xmm3, %xmm0
-; SSE41-NEXT:    por %xmm1, %xmm0
-; SSE41-NEXT:    blendvpd %xmm0, %xmm8, %xmm6
-; SSE41-NEXT:    shufps {{.*#+}} xmm6 = xmm6[0,2],xmm2[0,2]
-; SSE41-NEXT:    movaps %xmm6, %xmm0
-; SSE41-NEXT:    movaps %xmm4, %xmm1
+; SSE41-NEXT:    blendvpd %xmm0, %xmm8, %xmm2
+; SSE41-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,2],xmm4[0,2]
+; SSE41-NEXT:    movaps %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: trunc_usat_v8i64_v8i32:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
-; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm3
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [9223372041149743103,9223372041149743103]
-; AVX1-NEXT:    vpcmpgtq %xmm3, %xmm4, %xmm8
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm5
-; AVX1-NEXT:    vpxor %xmm2, %xmm5, %xmm6
-; AVX1-NEXT:    vpcmpgtq %xmm6, %xmm4, %xmm6
-; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm7
-; AVX1-NEXT:    vpcmpgtq %xmm7, %xmm4, %xmm7
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
-; AVX1-NEXT:    vpxor %xmm2, %xmm3, %xmm2
-; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm4, %xmm2
-; AVX1-NEXT:    vmovapd {{.*#+}} xmm4 = [4294967295,4294967295]
-; AVX1-NEXT:    vblendvpd %xmm2, %xmm3, %xmm4, %xmm2
-; AVX1-NEXT:    vblendvpd %xmm7, %xmm1, %xmm4, %xmm1
-; AVX1-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[0,2],xmm2[0,2]
-; AVX1-NEXT:    vblendvpd %xmm6, %xmm5, %xmm4, %xmm2
-; AVX1-NEXT:    vblendvpd %xmm8, %xmm0, %xmm4, %xmm0
-; AVX1-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm2[0,2]
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX1-NEXT:    vmovdqa 16(%rdi), %xmm1
+; AVX1-NEXT:    vmovdqa 32(%rdi), %xmm2
+; AVX1-NEXT:    vmovdqa 48(%rdi), %xmm3
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [9223372036854775808,9223372036854775808]
+; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm5
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [9223372041149743103,9223372041149743103]
+; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm6, %xmm8
+; AVX1-NEXT:    vpxor %xmm4, %xmm1, %xmm7
+; AVX1-NEXT:    vpcmpgtq %xmm7, %xmm6, %xmm7
+; AVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm5
+; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm6, %xmm5
+; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm4
+; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm6, %xmm4
+; AVX1-NEXT:    vmovapd {{.*#+}} xmm6 = [4294967295,4294967295]
+; AVX1-NEXT:    vblendvpd %xmm4, %xmm3, %xmm6, %xmm3
+; AVX1-NEXT:    vblendvpd %xmm5, %xmm2, %xmm6, %xmm2
+; AVX1-NEXT:    vshufps {{.*#+}} xmm2 = xmm2[0,2],xmm3[0,2]
+; AVX1-NEXT:    vblendvpd %xmm7, %xmm1, %xmm6, %xmm1
+; AVX1-NEXT:    vblendvpd %xmm8, %xmm0, %xmm6, %xmm0
+; AVX1-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-SLOW-LABEL: trunc_usat_v8i64_v8i32:
 ; AVX2-SLOW:       # %bb.0:
+; AVX2-SLOW-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX2-SLOW-NEXT:    vmovdqa 32(%rdi), %ymm1
 ; AVX2-SLOW-NEXT:    vbroadcastsd {{.*#+}} ymm2 = [4294967295,4294967295,4294967295,4294967295]
 ; AVX2-SLOW-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
 ; AVX2-SLOW-NEXT:    vpxor %ymm3, %ymm0, %ymm4
@@ -422,6 +649,8 @@ define <8 x i32> @trunc_usat_v8i64_v8i32(<8 x i64> %a0) {
 ;
 ; AVX2-FAST-LABEL: trunc_usat_v8i64_v8i32:
 ; AVX2-FAST:       # %bb.0:
+; AVX2-FAST-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX2-FAST-NEXT:    vmovdqa 32(%rdi), %ymm1
 ; AVX2-FAST-NEXT:    vbroadcastsd {{.*#+}} ymm2 = [4294967295,4294967295,4294967295,4294967295]
 ; AVX2-FAST-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
 ; AVX2-FAST-NEXT:    vpxor %ymm3, %ymm1, %ymm4
@@ -439,8 +668,16 @@ define <8 x i32> @trunc_usat_v8i64_v8i32(<8 x i64> %a0) {
 ;
 ; AVX512-LABEL: trunc_usat_v8i64_v8i32:
 ; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovdqa64 (%rdi), %zmm0
 ; AVX512-NEXT:    vpmovusqd %zmm0, %ymm0
 ; AVX512-NEXT:    retq
+;
+; SKX-LABEL: trunc_usat_v8i64_v8i32:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vmovdqa64 (%rdi), %zmm0
+; SKX-NEXT:    vpmovusqd %zmm0, %ymm0
+; SKX-NEXT:    retq
+  %a0 = load <8 x i64>, <8 x i64>* %p0
   %1 = icmp ult <8 x i64> %a0, <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>
   %2 = select <8 x i1> %1, <8 x i64> %a0, <8 x i64> <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>
   %3 = trunc <8 x i64> %2 to <8 x i32>
@@ -451,944 +688,2437 @@ define <8 x i32> @trunc_usat_v8i64_v8i32(<8 x i64> %a0) {
 ; Unsigned saturation truncation to vXi16
 ;
 
-define <8 x i16> @trunc_usat_v8i64_v8i16(<8 x i64> %a0) {
-; SSE2-LABEL: trunc_usat_v8i64_v8i16:
+define <2 x i16> @trunc_usat_v2i64_v2i16(<2 x i64> %a0) {
+; SSE2-LABEL: trunc_usat_v2i64_v2i16:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [65535,65535]
-; SSE2-NEXT:    movdqa {{.*#+}} xmm6 = [9223372039002259456,9223372039002259456]
-; SSE2-NEXT:    movdqa %xmm2, %xmm5
-; SSE2-NEXT:    pxor %xmm6, %xmm5
-; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [9223372039002324991,9223372039002324991]
-; SSE2-NEXT:    movdqa %xmm9, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm5, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm7[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm5[1,1,3,3]
-; SSE2-NEXT:    pand %xmm10, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm7[1,1,3,3]
-; SSE2-NEXT:    por %xmm4, %xmm5
-; SSE2-NEXT:    pand %xmm5, %xmm2
-; SSE2-NEXT:    pandn %xmm8, %xmm5
-; SSE2-NEXT:    por %xmm2, %xmm5
-; SSE2-NEXT:    movdqa %xmm3, %xmm2
-; SSE2-NEXT:    pxor %xmm6, %xmm2
-; SSE2-NEXT:    movdqa %xmm9, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm2[1,1,3,3]
-; SSE2-NEXT:    pand %xmm10, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm7, %xmm2
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pandn %xmm8, %xmm2
-; SSE2-NEXT:    por %xmm3, %xmm2
-; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    pxor %xmm6, %xmm3
-; SSE2-NEXT:    movdqa %xmm9, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE2-NEXT:    pand %xmm7, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm4
-; SSE2-NEXT:    pand %xmm4, %xmm0
-; SSE2-NEXT:    pandn %xmm8, %xmm4
-; SSE2-NEXT:    por %xmm0, %xmm4
-; SSE2-NEXT:    pxor %xmm1, %xmm6
-; SSE2-NEXT:    movdqa %xmm9, %xmm0
-; SSE2-NEXT:    pcmpgtd %xmm6, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT:    pand %xmm3, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm6, %xmm0
-; SSE2-NEXT:    pand %xmm0, %xmm1
-; SSE2-NEXT:    pandn %xmm8, %xmm0
-; SSE2-NEXT:    por %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[0,2,2,3]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [9223372039002259456,9223372039002259456]
+; SSE2-NEXT:    pxor %xmm0, %xmm1
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [9223372039002324991,9223372039002324991]
+; SSE2-NEXT:    movdqa %xmm2, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pand %xmm4, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,1,3,3]
+; SSE2-NEXT:    por %xmm1, %xmm2
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm2
+; SSE2-NEXT:    por %xmm0, %xmm2
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm0[0,1,0,2,4,5,6,7]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[0,2,2,3]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,1,0,2,4,5,6,7]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
 ; SSE2-NEXT:    retq
 ;
-; SSSE3-LABEL: trunc_usat_v8i64_v8i16:
+; SSSE3-LABEL: trunc_usat_v2i64_v2i16:
 ; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm8 = [65535,65535]
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm6 = [9223372039002259456,9223372039002259456]
-; SSSE3-NEXT:    movdqa %xmm2, %xmm5
-; SSSE3-NEXT:    pxor %xmm6, %xmm5
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm9 = [9223372039002324991,9223372039002324991]
-; SSSE3-NEXT:    movdqa %xmm9, %xmm7
-; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm7
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm10 = xmm7[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm5
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm5[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm10, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm7[1,1,3,3]
-; SSSE3-NEXT:    por %xmm4, %xmm5
-; SSSE3-NEXT:    pand %xmm5, %xmm2
-; SSSE3-NEXT:    pandn %xmm8, %xmm5
-; SSSE3-NEXT:    por %xmm2, %xmm5
-; SSSE3-NEXT:    movdqa %xmm3, %xmm2
-; SSSE3-NEXT:    pxor %xmm6, %xmm2
-; SSSE3-NEXT:    movdqa %xmm9, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm10 = xmm4[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm2[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm10, %xmm7
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    por %xmm7, %xmm2
-; SSSE3-NEXT:    pand %xmm2, %xmm3
-; SSSE3-NEXT:    pandn %xmm8, %xmm2
-; SSSE3-NEXT:    por %xmm3, %xmm2
-; SSSE3-NEXT:    movdqa %xmm0, %xmm3
-; SSSE3-NEXT:    pxor %xmm6, %xmm3
-; SSSE3-NEXT:    movdqa %xmm9, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm7, %xmm3
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [9223372039002259456,9223372039002259456]
+; SSSE3-NEXT:    pxor %xmm0, %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [9223372039002324991,9223372039002324991]
+; SSSE3-NEXT:    movdqa %xmm2, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm4, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,1,3,3]
+; SSSE3-NEXT:    por %xmm1, %xmm2
+; SSSE3-NEXT:    pand %xmm2, %xmm0
+; SSSE3-NEXT:    pandn {{.*}}(%rip), %xmm2
+; SSSE3-NEXT:    por %xmm0, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
+; SSSE3-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: trunc_usat_v2i64_v2i16:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    movapd {{.*#+}} xmm2 = [65535,65535]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [9223372039002259456,9223372039002259456]
+; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [9223372039002324991,9223372039002324991]
+; SSE41-NEXT:    movdqa %xmm3, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm3, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
+; SSE41-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: trunc_usat_v2i64_v2i16:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vmovapd {{.*#+}} xmm1 = [65535,65535]
+; AVX1-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm2
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [9223372036854841343,9223372036854841343]
+; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; AVX1-NEXT:    retq
+;
+; AVX2-SLOW-LABEL: trunc_usat_v2i64_v2i16:
+; AVX2-SLOW:       # %bb.0:
+; AVX2-SLOW-NEXT:    vmovapd {{.*#+}} xmm1 = [65535,65535]
+; AVX2-SLOW-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm2
+; AVX2-SLOW-NEXT:    vmovdqa {{.*#+}} xmm3 = [9223372036854841343,9223372036854841343]
+; AVX2-SLOW-NEXT:    vpcmpgtq %xmm2, %xmm3, %xmm2
+; AVX2-SLOW-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX2-SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; AVX2-SLOW-NEXT:    retq
+;
+; AVX2-FAST-LABEL: trunc_usat_v2i64_v2i16:
+; AVX2-FAST:       # %bb.0:
+; AVX2-FAST-NEXT:    vmovapd {{.*#+}} xmm1 = [65535,65535]
+; AVX2-FAST-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm2
+; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} xmm3 = [9223372036854841343,9223372036854841343]
+; AVX2-FAST-NEXT:    vpcmpgtq %xmm2, %xmm3, %xmm2
+; AVX2-FAST-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX2-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,8,9,8,9,10,11,8,9,10,11,12,13,14,15]
+; AVX2-FAST-NEXT:    retq
+;
+; AVX512F-LABEL: trunc_usat_v2i64_v2i16:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512F-NEXT:    vpmovusqw %zmm0, %xmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: trunc_usat_v2i64_v2i16:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpmovusqw %xmm0, %xmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: trunc_usat_v2i64_v2i16:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512BW-NEXT:    vpmovusqw %zmm0, %xmm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: trunc_usat_v2i64_v2i16:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpmovusqw %xmm0, %xmm0
+; AVX512BWVL-NEXT:    retq
+;
+; SKX-LABEL: trunc_usat_v2i64_v2i16:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpmovusqw %xmm0, %xmm0
+; SKX-NEXT:    retq
+  %1 = icmp ult <2 x i64> %a0, <i64 65535, i64 65535>
+  %2 = select <2 x i1> %1, <2 x i64> %a0, <2 x i64> <i64 65535, i64 65535>
+  %3 = trunc <2 x i64> %2 to <2 x i16>
+  ret <2 x i16> %3
+}
+
+define void @trunc_usat_v2i64_v2i16_store(<2 x i64> %a0, <2 x i16>* %p1) {
+; SSE2-LABEL: trunc_usat_v2i64_v2i16_store:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [9223372039002259456,9223372039002259456]
+; SSE2-NEXT:    pxor %xmm0, %xmm1
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [9223372039002324991,9223372039002324991]
+; SSE2-NEXT:    movdqa %xmm2, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pand %xmm4, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,1,3,3]
+; SSE2-NEXT:    por %xmm1, %xmm2
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm2
+; SSE2-NEXT:    por %xmm0, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; SSE2-NEXT:    movd %xmm0, (%rdi)
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: trunc_usat_v2i64_v2i16_store:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [9223372039002259456,9223372039002259456]
+; SSSE3-NEXT:    pxor %xmm0, %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [9223372039002324991,9223372039002324991]
+; SSSE3-NEXT:    movdqa %xmm2, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm4, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,1,3,3]
+; SSSE3-NEXT:    por %xmm1, %xmm2
+; SSSE3-NEXT:    pand %xmm2, %xmm0
+; SSSE3-NEXT:    pandn {{.*}}(%rip), %xmm2
+; SSSE3-NEXT:    por %xmm0, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
+; SSSE3-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; SSSE3-NEXT:    movd %xmm0, (%rdi)
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: trunc_usat_v2i64_v2i16_store:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    movapd {{.*#+}} xmm2 = [65535,65535]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [9223372039002259456,9223372039002259456]
+; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [9223372039002324991,9223372039002324991]
+; SSE41-NEXT:    movdqa %xmm3, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm3, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
+; SSE41-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; SSE41-NEXT:    movd %xmm0, (%rdi)
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: trunc_usat_v2i64_v2i16_store:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vmovapd {{.*#+}} xmm1 = [65535,65535]
+; AVX1-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm2
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [9223372036854841343,9223372036854841343]
+; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; AVX1-NEXT:    vmovd %xmm0, (%rdi)
+; AVX1-NEXT:    retq
+;
+; AVX2-SLOW-LABEL: trunc_usat_v2i64_v2i16_store:
+; AVX2-SLOW:       # %bb.0:
+; AVX2-SLOW-NEXT:    vmovapd {{.*#+}} xmm1 = [65535,65535]
+; AVX2-SLOW-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm2
+; AVX2-SLOW-NEXT:    vmovdqa {{.*#+}} xmm3 = [9223372036854841343,9223372036854841343]
+; AVX2-SLOW-NEXT:    vpcmpgtq %xmm2, %xmm3, %xmm2
+; AVX2-SLOW-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX2-SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vmovd %xmm0, (%rdi)
+; AVX2-SLOW-NEXT:    retq
+;
+; AVX2-FAST-LABEL: trunc_usat_v2i64_v2i16_store:
+; AVX2-FAST:       # %bb.0:
+; AVX2-FAST-NEXT:    vmovapd {{.*#+}} xmm1 = [65535,65535]
+; AVX2-FAST-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm2
+; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} xmm3 = [9223372036854841343,9223372036854841343]
+; AVX2-FAST-NEXT:    vpcmpgtq %xmm2, %xmm3, %xmm2
+; AVX2-FAST-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX2-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,8,9,8,9,10,11,8,9,10,11,12,13,14,15]
+; AVX2-FAST-NEXT:    vmovd %xmm0, (%rdi)
+; AVX2-FAST-NEXT:    retq
+;
+; AVX512F-LABEL: trunc_usat_v2i64_v2i16_store:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512F-NEXT:    vpmovusqw %zmm0, %xmm0
+; AVX512F-NEXT:    vmovd %xmm0, (%rdi)
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: trunc_usat_v2i64_v2i16_store:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpmovusqw %xmm0, (%rdi)
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: trunc_usat_v2i64_v2i16_store:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512BW-NEXT:    vpmovusqw %zmm0, %xmm0
+; AVX512BW-NEXT:    vmovd %xmm0, (%rdi)
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: trunc_usat_v2i64_v2i16_store:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpmovusqw %xmm0, (%rdi)
+; AVX512BWVL-NEXT:    retq
+;
+; SKX-LABEL: trunc_usat_v2i64_v2i16_store:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpmovusqw %xmm0, (%rdi)
+; SKX-NEXT:    retq
+  %1 = icmp ult <2 x i64> %a0, <i64 65535, i64 65535>
+  %2 = select <2 x i1> %1, <2 x i64> %a0, <2 x i64> <i64 65535, i64 65535>
+  %3 = trunc <2 x i64> %2 to <2 x i16>
+  store <2 x i16> %3, <2 x i16>* %p1
+  ret void
+}
+
+define <4 x i16> @trunc_usat_v4i64_v4i16(<4 x i64> %a0) {
+; SSE2-LABEL: trunc_usat_v4i64_v4i16:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [65535,65535]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [9223372039002259456,9223372039002259456]
+; SSE2-NEXT:    movdqa %xmm0, %xmm4
+; SSE2-NEXT:    pxor %xmm3, %xmm4
+; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [9223372039002324991,9223372039002324991]
+; SSE2-NEXT:    movdqa %xmm5, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    pand %xmm7, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSE2-NEXT:    por %xmm4, %xmm6
+; SSE2-NEXT:    pand %xmm6, %xmm0
+; SSE2-NEXT:    pandn %xmm2, %xmm6
+; SSE2-NEXT:    por %xmm0, %xmm6
+; SSE2-NEXT:    pxor %xmm1, %xmm3
+; SSE2-NEXT:    movdqa %xmm5, %xmm0
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSE2-NEXT:    pand %xmm4, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    pand %xmm0, %xmm1
+; SSE2-NEXT:    pandn %xmm2, %xmm0
+; SSE2-NEXT:    por %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm0[0,2,2,3,4,5,6,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,2,2,3]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: trunc_usat_v4i64_v4i16:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [65535,65535]
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm3 = [9223372039002259456,9223372039002259456]
+; SSSE3-NEXT:    movdqa %xmm0, %xmm4
+; SSSE3-NEXT:    pxor %xmm3, %xmm4
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm5 = [9223372039002324991,9223372039002324991]
+; SSSE3-NEXT:    movdqa %xmm5, %xmm6
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm4
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    por %xmm3, %xmm4
-; SSSE3-NEXT:    pand %xmm4, %xmm0
-; SSSE3-NEXT:    pandn %xmm8, %xmm4
-; SSSE3-NEXT:    por %xmm0, %xmm4
-; SSSE3-NEXT:    pxor %xmm1, %xmm6
-; SSSE3-NEXT:    movdqa %xmm9, %xmm0
-; SSSE3-NEXT:    pcmpgtd %xmm6, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm6
+; SSSE3-NEXT:    pand %xmm7, %xmm4
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm3, %xmm6
+; SSSE3-NEXT:    por %xmm4, %xmm6
+; SSSE3-NEXT:    pand %xmm6, %xmm0
+; SSSE3-NEXT:    pandn %xmm2, %xmm6
+; SSSE3-NEXT:    por %xmm0, %xmm6
+; SSSE3-NEXT:    pxor %xmm1, %xmm3
+; SSSE3-NEXT:    movdqa %xmm5, %xmm0
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm4, %xmm3
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    por %xmm6, %xmm0
+; SSSE3-NEXT:    por %xmm3, %xmm0
 ; SSSE3-NEXT:    pand %xmm0, %xmm1
-; SSSE3-NEXT:    pandn %xmm8, %xmm0
+; SSSE3-NEXT:    pandn %xmm2, %xmm0
 ; SSSE3-NEXT:    por %xmm1, %xmm0
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSSE3-NEXT:    pshuflw {{.*#+}} xmm1 = xmm0[0,2,2,3,4,5,6,7]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,2,2,3]
 ; SSSE3-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[0,2,2,3]
-; SSSE3-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
-; SSSE3-NEXT:    pshuflw {{.*#+}} xmm2 = xmm0[0,1,0,2,4,5,6,7]
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[0,2,2,3]
-; SSSE3-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,1,0,2,4,5,6,7]
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; SSSE3-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; SSSE3-NEXT:    retq
 ;
-; SSE41-LABEL: trunc_usat_v8i64_v8i16:
+; SSE41-LABEL: trunc_usat_v4i64_v4i16:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa %xmm0, %xmm8
-; SSE41-NEXT:    movapd {{.*#+}} xmm5 = [65535,65535]
-; SSE41-NEXT:    movdqa {{.*#+}} xmm6 = [9223372039002259456,9223372039002259456]
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    pxor %xmm6, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm9 = [9223372039002324991,9223372039002324991]
-; SSE41-NEXT:    movdqa %xmm9, %xmm7
+; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    movapd {{.*#+}} xmm3 = [65535,65535]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [9223372039002259456,9223372039002259456]
+; SSE41-NEXT:    pxor %xmm4, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm5 = [9223372039002324991,9223372039002324991]
+; SSE41-NEXT:    movdqa %xmm5, %xmm6
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm6
+; SSE41-NEXT:    movdqa %xmm5, %xmm7
 ; SSE41-NEXT:    pcmpgtd %xmm0, %xmm7
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm7[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm9, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[0,0,2,2]
+; SSE41-NEXT:    pand %xmm6, %xmm0
 ; SSE41-NEXT:    por %xmm7, %xmm0
-; SSE41-NEXT:    movapd %xmm5, %xmm4
-; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm4
-; SSE41-NEXT:    movdqa %xmm8, %xmm0
-; SSE41-NEXT:    pxor %xmm6, %xmm0
-; SSE41-NEXT:    movdqa %xmm9, %xmm1
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm1[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm9, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm7, %xmm0
-; SSE41-NEXT:    por %xmm1, %xmm0
-; SSE41-NEXT:    movapd %xmm5, %xmm1
-; SSE41-NEXT:    blendvpd %xmm0, %xmm8, %xmm1
-; SSE41-NEXT:    packusdw %xmm4, %xmm1
-; SSE41-NEXT:    movdqa %xmm3, %xmm0
-; SSE41-NEXT:    pxor %xmm6, %xmm0
-; SSE41-NEXT:    movdqa %xmm9, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm9, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm7, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
-; SSE41-NEXT:    movapd %xmm5, %xmm4
-; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm4
-; SSE41-NEXT:    pxor %xmm2, %xmm6
-; SSE41-NEXT:    movdqa %xmm9, %xmm3
-; SSE41-NEXT:    pcmpgtd %xmm6, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm9, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3]
-; SSE41-NEXT:    pand %xmm7, %xmm0
-; SSE41-NEXT:    por %xmm3, %xmm0
-; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm5
-; SSE41-NEXT:    packusdw %xmm4, %xmm5
-; SSE41-NEXT:    packusdw %xmm5, %xmm1
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    movapd %xmm3, %xmm6
+; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm6
+; SSE41-NEXT:    pxor %xmm1, %xmm4
+; SSE41-NEXT:    movdqa %xmm5, %xmm2
+; SSE41-NEXT:    pcmpeqd %xmm4, %xmm2
+; SSE41-NEXT:    pcmpgtd %xmm4, %xmm5
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2]
+; SSE41-NEXT:    pand %xmm2, %xmm0
+; SSE41-NEXT:    por %xmm5, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,2,2,3]
+; SSE41-NEXT:    pshuflw {{.*#+}} xmm1 = xmm0[0,2,2,3,4,5,6,7]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,2,2,3]
+; SSE41-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; SSE41-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: trunc_usat_v8i64_v8i16:
+; AVX1-LABEL: trunc_usat_v4i64_v4i16:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
-; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm3
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [9223372036854841343,9223372036854841343]
-; AVX1-NEXT:    vpcmpgtq %xmm3, %xmm4, %xmm8
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm5
-; AVX1-NEXT:    vpxor %xmm2, %xmm5, %xmm6
-; AVX1-NEXT:    vpcmpgtq %xmm6, %xmm4, %xmm6
-; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm7
-; AVX1-NEXT:    vpcmpgtq %xmm7, %xmm4, %xmm7
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
-; AVX1-NEXT:    vpxor %xmm2, %xmm3, %xmm2
-; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm4, %xmm2
-; AVX1-NEXT:    vmovapd {{.*#+}} xmm4 = [65535,65535]
-; AVX1-NEXT:    vblendvpd %xmm2, %xmm3, %xmm4, %xmm2
-; AVX1-NEXT:    vblendvpd %xmm7, %xmm1, %xmm4, %xmm1
-; AVX1-NEXT:    vpackusdw %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vblendvpd %xmm6, %xmm5, %xmm4, %xmm2
-; AVX1-NEXT:    vblendvpd %xmm8, %xmm0, %xmm4, %xmm0
-; AVX1-NEXT:    vpackusdw %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [9223372036854775808,9223372036854775808]
+; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm2
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [9223372036854841343,9223372036854841343]
+; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
+; AVX1-NEXT:    vpxor %xmm1, %xmm4, %xmm1
+; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm3, %xmm1
+; AVX1-NEXT:    vmovapd {{.*#+}} xmm3 = [65535,65535]
+; AVX1-NEXT:    vblendvpd %xmm1, %xmm4, %xmm3, %xmm1
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
+; AVX1-NEXT:    vblendvpd %xmm2, %xmm0, %xmm3, %xmm0
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
-; AVX2-LABEL: trunc_usat_v8i64_v8i16:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm2 = [65535,65535,65535,65535]
-; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
-; AVX2-NEXT:    vpxor %ymm3, %ymm1, %ymm4
-; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm5 = [9223372036854841343,9223372036854841343,9223372036854841343,9223372036854841343]
-; AVX2-NEXT:    vpcmpgtq %ymm4, %ymm5, %ymm4
-; AVX2-NEXT:    vblendvpd %ymm4, %ymm1, %ymm2, %ymm1
-; AVX2-NEXT:    vpxor %ymm3, %ymm0, %ymm3
-; AVX2-NEXT:    vpcmpgtq %ymm3, %ymm5, %ymm3
-; AVX2-NEXT:    vblendvpd %ymm3, %ymm0, %ymm2, %ymm0
-; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vzeroupper
-; AVX2-NEXT:    retq
+; AVX2-SLOW-LABEL: trunc_usat_v4i64_v4i16:
+; AVX2-SLOW:       # %bb.0:
+; AVX2-SLOW-NEXT:    vbroadcastsd {{.*#+}} ymm1 = [65535,65535,65535,65535]
+; AVX2-SLOW-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
+; AVX2-SLOW-NEXT:    vpxor %ymm2, %ymm0, %ymm2
+; AVX2-SLOW-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [9223372036854841343,9223372036854841343,9223372036854841343,9223372036854841343]
+; AVX2-SLOW-NEXT:    vpcmpgtq %ymm2, %ymm3, %ymm2
+; AVX2-SLOW-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX2-SLOW-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX2-SLOW-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX2-SLOW-NEXT:    vzeroupper
+; AVX2-SLOW-NEXT:    retq
 ;
-; AVX512-LABEL: trunc_usat_v8i64_v8i16:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpmovusqw %zmm0, %xmm0
-; AVX512-NEXT:    vzeroupper
-; AVX512-NEXT:    retq
-  %1 = icmp ult <8 x i64> %a0, <i64 65535, i64 65535, i64 65535, i64 65535, i64 65535, i64 65535, i64 65535, i64 65535>
-  %2 = select <8 x i1> %1, <8 x i64> %a0, <8 x i64> <i64 65535, i64 65535, i64 65535, i64 65535, i64 65535, i64 65535, i64 65535, i64 65535>
-  %3 = trunc <8 x i64> %2 to <8 x i16>
-  ret <8 x i16> %3
+; AVX2-FAST-LABEL: trunc_usat_v4i64_v4i16:
+; AVX2-FAST:       # %bb.0:
+; AVX2-FAST-NEXT:    vbroadcastsd {{.*#+}} ymm1 = [65535,65535,65535,65535]
+; AVX2-FAST-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
+; AVX2-FAST-NEXT:    vpxor %ymm2, %ymm0, %ymm2
+; AVX2-FAST-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [9223372036854841343,9223372036854841343,9223372036854841343,9223372036854841343]
+; AVX2-FAST-NEXT:    vpcmpgtq %ymm2, %ymm3, %ymm2
+; AVX2-FAST-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX2-FAST-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,1,8,9,8,9,10,11,8,9,10,11,12,13,14,15]
+; AVX2-FAST-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
+; AVX2-FAST-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
+; AVX2-FAST-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX2-FAST-NEXT:    vzeroupper
+; AVX2-FAST-NEXT:    retq
+;
+; AVX512F-LABEL: trunc_usat_v4i64_v4i16:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512F-NEXT:    vpmovusqw %zmm0, %xmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: trunc_usat_v4i64_v4i16:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpmovusqw %ymm0, %xmm0
+; AVX512VL-NEXT:    vzeroupper
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: trunc_usat_v4i64_v4i16:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512BW-NEXT:    vpmovusqw %zmm0, %xmm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: trunc_usat_v4i64_v4i16:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpmovusqw %ymm0, %xmm0
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
+;
+; SKX-LABEL: trunc_usat_v4i64_v4i16:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpmovusqw %ymm0, %xmm0
+; SKX-NEXT:    vzeroupper
+; SKX-NEXT:    retq
+  %1 = icmp ult <4 x i64> %a0, <i64 65535, i64 65535, i64 65535, i64 65535>
+  %2 = select <4 x i1> %1, <4 x i64> %a0, <4 x i64> <i64 65535, i64 65535, i64 65535, i64 65535>
+  %3 = trunc <4 x i64> %2 to <4 x i16>
+  ret <4 x i16> %3
+}
+
+define void @trunc_usat_v4i64_v4i16_store(<4 x i64> %a0, <4 x i16> *%p1) {
+; SSE2-LABEL: trunc_usat_v4i64_v4i16_store:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [65535,65535]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [9223372039002259456,9223372039002259456]
+; SSE2-NEXT:    movdqa %xmm0, %xmm4
+; SSE2-NEXT:    pxor %xmm3, %xmm4
+; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [9223372039002324991,9223372039002324991]
+; SSE2-NEXT:    movdqa %xmm5, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    pand %xmm7, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSE2-NEXT:    por %xmm4, %xmm6
+; SSE2-NEXT:    pand %xmm6, %xmm0
+; SSE2-NEXT:    pandn %xmm2, %xmm6
+; SSE2-NEXT:    por %xmm0, %xmm6
+; SSE2-NEXT:    pxor %xmm1, %xmm3
+; SSE2-NEXT:    movdqa %xmm5, %xmm0
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSE2-NEXT:    pand %xmm4, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    pand %xmm0, %xmm1
+; SSE2-NEXT:    pandn %xmm2, %xmm0
+; SSE2-NEXT:    por %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm6[0,2,2,3]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE2-NEXT:    movq %xmm1, (%rdi)
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: trunc_usat_v4i64_v4i16_store:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [65535,65535]
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm3 = [9223372039002259456,9223372039002259456]
+; SSSE3-NEXT:    movdqa %xmm0, %xmm4
+; SSSE3-NEXT:    pxor %xmm3, %xmm4
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm5 = [9223372039002324991,9223372039002324991]
+; SSSE3-NEXT:    movdqa %xmm5, %xmm6
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm7, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSSE3-NEXT:    por %xmm4, %xmm6
+; SSSE3-NEXT:    pand %xmm6, %xmm0
+; SSSE3-NEXT:    pandn %xmm2, %xmm6
+; SSSE3-NEXT:    por %xmm0, %xmm6
+; SSSE3-NEXT:    pxor %xmm1, %xmm3
+; SSSE3-NEXT:    movdqa %xmm5, %xmm0
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm4, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    por %xmm3, %xmm0
+; SSSE3-NEXT:    pand %xmm0, %xmm1
+; SSSE3-NEXT:    pandn %xmm2, %xmm0
+; SSSE3-NEXT:    por %xmm1, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSSE3-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm6[0,2,2,3]
+; SSSE3-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSSE3-NEXT:    movq %xmm1, (%rdi)
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: trunc_usat_v4i64_v4i16_store:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    movapd {{.*#+}} xmm3 = [65535,65535]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [9223372039002259456,9223372039002259456]
+; SSE41-NEXT:    pxor %xmm4, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm5 = [9223372039002324991,9223372039002324991]
+; SSE41-NEXT:    movdqa %xmm5, %xmm6
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm6
+; SSE41-NEXT:    movdqa %xmm5, %xmm7
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm7
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[0,0,2,2]
+; SSE41-NEXT:    pand %xmm6, %xmm0
+; SSE41-NEXT:    por %xmm7, %xmm0
+; SSE41-NEXT:    movapd %xmm3, %xmm6
+; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm6
+; SSE41-NEXT:    pxor %xmm1, %xmm4
+; SSE41-NEXT:    movdqa %xmm5, %xmm2
+; SSE41-NEXT:    pcmpeqd %xmm4, %xmm2
+; SSE41-NEXT:    pcmpgtd %xmm4, %xmm5
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2]
+; SSE41-NEXT:    pand %xmm2, %xmm0
+; SSE41-NEXT:    por %xmm5, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,2,2,3]
+; SSE41-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm6[0,2,2,3]
+; SSE41-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
+; SSE41-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE41-NEXT:    movq %xmm1, (%rdi)
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: trunc_usat_v4i64_v4i16_store:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [9223372036854775808,9223372036854775808]
+; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm2
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [9223372036854841343,9223372036854841343]
+; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
+; AVX1-NEXT:    vpxor %xmm1, %xmm4, %xmm1
+; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm3, %xmm1
+; AVX1-NEXT:    vmovapd {{.*#+}} xmm3 = [65535,65535]
+; AVX1-NEXT:    vblendvpd %xmm1, %xmm4, %xmm3, %xmm1
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
+; AVX1-NEXT:    vblendvpd %xmm2, %xmm0, %xmm3, %xmm0
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX1-NEXT:    vmovq %xmm0, (%rdi)
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2-SLOW-LABEL: trunc_usat_v4i64_v4i16_store:
+; AVX2-SLOW:       # %bb.0:
+; AVX2-SLOW-NEXT:    vbroadcastsd {{.*#+}} ymm1 = [65535,65535,65535,65535]
+; AVX2-SLOW-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
+; AVX2-SLOW-NEXT:    vpxor %ymm2, %ymm0, %ymm2
+; AVX2-SLOW-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [9223372036854841343,9223372036854841343,9223372036854841343,9223372036854841343]
+; AVX2-SLOW-NEXT:    vpcmpgtq %ymm2, %ymm3, %ymm2
+; AVX2-SLOW-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX2-SLOW-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX2-SLOW-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX2-SLOW-NEXT:    vmovq %xmm0, (%rdi)
+; AVX2-SLOW-NEXT:    vzeroupper
+; AVX2-SLOW-NEXT:    retq
+;
+; AVX2-FAST-LABEL: trunc_usat_v4i64_v4i16_store:
+; AVX2-FAST:       # %bb.0:
+; AVX2-FAST-NEXT:    vbroadcastsd {{.*#+}} ymm1 = [65535,65535,65535,65535]
+; AVX2-FAST-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
+; AVX2-FAST-NEXT:    vpxor %ymm2, %ymm0, %ymm2
+; AVX2-FAST-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [9223372036854841343,9223372036854841343,9223372036854841343,9223372036854841343]
+; AVX2-FAST-NEXT:    vpcmpgtq %ymm2, %ymm3, %ymm2
+; AVX2-FAST-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX2-FAST-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,1,8,9,8,9,10,11,8,9,10,11,12,13,14,15]
+; AVX2-FAST-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
+; AVX2-FAST-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
+; AVX2-FAST-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX2-FAST-NEXT:    vmovq %xmm0, (%rdi)
+; AVX2-FAST-NEXT:    vzeroupper
+; AVX2-FAST-NEXT:    retq
+;
+; AVX512F-LABEL: trunc_usat_v4i64_v4i16_store:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512F-NEXT:    vpmovusqw %zmm0, %xmm0
+; AVX512F-NEXT:    vmovq %xmm0, (%rdi)
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: trunc_usat_v4i64_v4i16_store:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpmovusqw %ymm0, (%rdi)
+; AVX512VL-NEXT:    vzeroupper
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: trunc_usat_v4i64_v4i16_store:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512BW-NEXT:    vpmovusqw %zmm0, %xmm0
+; AVX512BW-NEXT:    vmovq %xmm0, (%rdi)
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: trunc_usat_v4i64_v4i16_store:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpmovusqw %ymm0, (%rdi)
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
+;
+; SKX-LABEL: trunc_usat_v4i64_v4i16_store:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpmovusqw %ymm0, (%rdi)
+; SKX-NEXT:    vzeroupper
+; SKX-NEXT:    retq
+  %1 = icmp ult <4 x i64> %a0, <i64 65535, i64 65535, i64 65535, i64 65535>
+  %2 = select <4 x i1> %1, <4 x i64> %a0, <4 x i64> <i64 65535, i64 65535, i64 65535, i64 65535>
+  %3 = trunc <4 x i64> %2 to <4 x i16>
+  store <4 x i16> %3, <4 x i16> *%p1
+  ret void
+}
+
+define <8 x i16> @trunc_usat_v8i64_v8i16(<8 x i64>* %p0) {
+; SSE2-LABEL: trunc_usat_v8i64_v8i16:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa (%rdi), %xmm4
+; SSE2-NEXT:    movdqa 16(%rdi), %xmm9
+; SSE2-NEXT:    movdqa 32(%rdi), %xmm6
+; SSE2-NEXT:    movdqa 48(%rdi), %xmm7
+; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [65535,65535]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [9223372039002259456,9223372039002259456]
+; SSE2-NEXT:    movdqa %xmm6, %xmm2
+; SSE2-NEXT:    pxor %xmm3, %xmm2
+; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [9223372039002324991,9223372039002324991]
+; SSE2-NEXT:    movdqa %xmm10, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,3,3]
+; SSE2-NEXT:    pand %xmm0, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm5[1,1,3,3]
+; SSE2-NEXT:    por %xmm1, %xmm2
+; SSE2-NEXT:    pand %xmm2, %xmm6
+; SSE2-NEXT:    pandn %xmm8, %xmm2
+; SSE2-NEXT:    por %xmm6, %xmm2
+; SSE2-NEXT:    movdqa %xmm7, %xmm0
+; SSE2-NEXT:    pxor %xmm3, %xmm0
+; SSE2-NEXT:    movdqa %xmm10, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pand %xmm5, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[1,1,3,3]
+; SSE2-NEXT:    por %xmm0, %xmm6
+; SSE2-NEXT:    pand %xmm6, %xmm7
+; SSE2-NEXT:    pandn %xmm8, %xmm6
+; SSE2-NEXT:    por %xmm7, %xmm6
+; SSE2-NEXT:    movdqa %xmm4, %xmm0
+; SSE2-NEXT:    pxor %xmm3, %xmm0
+; SSE2-NEXT:    movdqa %xmm10, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pand %xmm5, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    por %xmm0, %xmm1
+; SSE2-NEXT:    pand %xmm1, %xmm4
+; SSE2-NEXT:    pandn %xmm8, %xmm1
+; SSE2-NEXT:    por %xmm4, %xmm1
+; SSE2-NEXT:    pxor %xmm9, %xmm3
+; SSE2-NEXT:    movdqa %xmm10, %xmm0
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSE2-NEXT:    pand %xmm4, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    pand %xmm0, %xmm9
+; SSE2-NEXT:    pandn %xmm8, %xmm0
+; SSE2-NEXT:    por %xmm9, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,2,2,3]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm0[0,1,0,2,4,5,6,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,1,0,2,4,5,6,7]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: trunc_usat_v8i64_v8i16:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa (%rdi), %xmm4
+; SSSE3-NEXT:    movdqa 16(%rdi), %xmm9
+; SSSE3-NEXT:    movdqa 32(%rdi), %xmm6
+; SSSE3-NEXT:    movdqa 48(%rdi), %xmm7
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm8 = [65535,65535]
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm3 = [9223372039002259456,9223372039002259456]
+; SSSE3-NEXT:    movdqa %xmm6, %xmm2
+; SSSE3-NEXT:    pxor %xmm3, %xmm2
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm10 = [9223372039002324991,9223372039002324991]
+; SSSE3-NEXT:    movdqa %xmm10, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm0, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm5[1,1,3,3]
+; SSSE3-NEXT:    por %xmm1, %xmm2
+; SSSE3-NEXT:    pand %xmm2, %xmm6
+; SSSE3-NEXT:    pandn %xmm8, %xmm2
+; SSSE3-NEXT:    por %xmm6, %xmm2
+; SSSE3-NEXT:    movdqa %xmm7, %xmm0
+; SSSE3-NEXT:    pxor %xmm3, %xmm0
+; SSSE3-NEXT:    movdqa %xmm10, %xmm1
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm5, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    por %xmm0, %xmm6
+; SSSE3-NEXT:    pand %xmm6, %xmm7
+; SSSE3-NEXT:    pandn %xmm8, %xmm6
+; SSSE3-NEXT:    por %xmm7, %xmm6
+; SSSE3-NEXT:    movdqa %xmm4, %xmm0
+; SSSE3-NEXT:    pxor %xmm3, %xmm0
+; SSSE3-NEXT:    movdqa %xmm10, %xmm1
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm5, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    por %xmm0, %xmm1
+; SSSE3-NEXT:    pand %xmm1, %xmm4
+; SSSE3-NEXT:    pandn %xmm8, %xmm1
+; SSSE3-NEXT:    por %xmm4, %xmm1
+; SSSE3-NEXT:    pxor %xmm9, %xmm3
+; SSSE3-NEXT:    movdqa %xmm10, %xmm0
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm4, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    por %xmm3, %xmm0
+; SSSE3-NEXT:    pand %xmm0, %xmm9
+; SSSE3-NEXT:    pandn %xmm8, %xmm0
+; SSSE3-NEXT:    por %xmm9, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSSE3-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; SSSE3-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,2,2,3]
+; SSSE3-NEXT:    pshuflw {{.*#+}} xmm3 = xmm0[0,1,0,2,4,5,6,7]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
+; SSSE3-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,1,0,2,4,5,6,7]
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; SSSE3-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: trunc_usat_v8i64_v8i16:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa (%rdi), %xmm7
+; SSE41-NEXT:    movdqa 16(%rdi), %xmm1
+; SSE41-NEXT:    movdqa 32(%rdi), %xmm8
+; SSE41-NEXT:    movdqa 48(%rdi), %xmm9
+; SSE41-NEXT:    movapd {{.*#+}} xmm2 = [65535,65535]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm5 = [9223372039002259456,9223372039002259456]
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pxor %xmm5, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [9223372039002324991,9223372039002324991]
+; SSE41-NEXT:    movdqa %xmm3, %xmm6
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm6
+; SSE41-NEXT:    movdqa %xmm3, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pand %xmm6, %xmm0
+; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    movapd %xmm2, %xmm4
+; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm4
+; SSE41-NEXT:    movdqa %xmm7, %xmm0
+; SSE41-NEXT:    pxor %xmm5, %xmm0
+; SSE41-NEXT:    movdqa %xmm3, %xmm1
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
+; SSE41-NEXT:    movdqa %xmm3, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
+; SSE41-NEXT:    pand %xmm1, %xmm0
+; SSE41-NEXT:    por %xmm6, %xmm0
+; SSE41-NEXT:    movapd %xmm2, %xmm1
+; SSE41-NEXT:    blendvpd %xmm0, %xmm7, %xmm1
+; SSE41-NEXT:    packusdw %xmm4, %xmm1
+; SSE41-NEXT:    movdqa %xmm9, %xmm0
+; SSE41-NEXT:    pxor %xmm5, %xmm0
+; SSE41-NEXT:    movdqa %xmm3, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT:    movdqa %xmm3, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm6, %xmm0
+; SSE41-NEXT:    movapd %xmm2, %xmm4
+; SSE41-NEXT:    blendvpd %xmm0, %xmm9, %xmm4
+; SSE41-NEXT:    pxor %xmm8, %xmm5
+; SSE41-NEXT:    movdqa %xmm3, %xmm6
+; SSE41-NEXT:    pcmpeqd %xmm5, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm5, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pand %xmm6, %xmm0
+; SSE41-NEXT:    por %xmm3, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm8, %xmm2
+; SSE41-NEXT:    packusdw %xmm4, %xmm2
+; SSE41-NEXT:    packusdw %xmm2, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: trunc_usat_v8i64_v8i16:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX1-NEXT:    vmovdqa 16(%rdi), %xmm1
+; AVX1-NEXT:    vmovdqa 32(%rdi), %xmm2
+; AVX1-NEXT:    vmovdqa 48(%rdi), %xmm3
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [9223372036854775808,9223372036854775808]
+; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm5
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [9223372036854841343,9223372036854841343]
+; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm6, %xmm8
+; AVX1-NEXT:    vpxor %xmm4, %xmm1, %xmm7
+; AVX1-NEXT:    vpcmpgtq %xmm7, %xmm6, %xmm7
+; AVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm5
+; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm6, %xmm5
+; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm4
+; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm6, %xmm4
+; AVX1-NEXT:    vmovapd {{.*#+}} xmm6 = [65535,65535]
+; AVX1-NEXT:    vblendvpd %xmm4, %xmm3, %xmm6, %xmm3
+; AVX1-NEXT:    vblendvpd %xmm5, %xmm2, %xmm6, %xmm2
+; AVX1-NEXT:    vpackusdw %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vblendvpd %xmm7, %xmm1, %xmm6, %xmm1
+; AVX1-NEXT:    vblendvpd %xmm8, %xmm0, %xmm6, %xmm0
+; AVX1-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpackusdw %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: trunc_usat_v8i64_v8i16:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX2-NEXT:    vmovdqa 32(%rdi), %ymm1
+; AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm2 = [65535,65535,65535,65535]
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
+; AVX2-NEXT:    vpxor %ymm3, %ymm1, %ymm4
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm5 = [9223372036854841343,9223372036854841343,9223372036854841343,9223372036854841343]
+; AVX2-NEXT:    vpcmpgtq %ymm4, %ymm5, %ymm4
+; AVX2-NEXT:    vblendvpd %ymm4, %ymm1, %ymm2, %ymm1
+; AVX2-NEXT:    vpxor %ymm3, %ymm0, %ymm3
+; AVX2-NEXT:    vpcmpgtq %ymm3, %ymm5, %ymm3
+; AVX2-NEXT:    vblendvpd %ymm3, %ymm0, %ymm2, %ymm0
+; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: trunc_usat_v8i64_v8i16:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovdqa64 (%rdi), %zmm0
+; AVX512-NEXT:    vpmovusqw %zmm0, %xmm0
+; AVX512-NEXT:    vzeroupper
+; AVX512-NEXT:    retq
+;
+; SKX-LABEL: trunc_usat_v8i64_v8i16:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vmovdqa64 (%rdi), %zmm0
+; SKX-NEXT:    vpmovusqw %zmm0, %xmm0
+; SKX-NEXT:    vzeroupper
+; SKX-NEXT:    retq
+  %a0 = load <8 x i64>, <8 x i64>* %p0
+  %1 = icmp ult <8 x i64> %a0, <i64 65535, i64 65535, i64 65535, i64 65535, i64 65535, i64 65535, i64 65535, i64 65535>
+  %2 = select <8 x i1> %1, <8 x i64> %a0, <8 x i64> <i64 65535, i64 65535, i64 65535, i64 65535, i64 65535, i64 65535, i64 65535, i64 65535>
+  %3 = trunc <8 x i64> %2 to <8 x i16>
+  ret <8 x i16> %3
+}
+
+define <4 x i16> @trunc_usat_v4i32_v4i16(<4 x i32> %a0) {
+; SSE2-LABEL: trunc_usat_v4i32_v4i16:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [2147483648,2147483648,2147483648,2147483648]
+; SSE2-NEXT:    pxor %xmm0, %xmm1
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147549183,2147549183,2147549183,2147549183]
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm2
+; SSE2-NEXT:    por %xmm0, %xmm2
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm2[0,2,2,3,4,5,6,7]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,6,6,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: trunc_usat_v4i32_v4i16:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [2147483648,2147483648,2147483648,2147483648]
+; SSSE3-NEXT:    pxor %xmm0, %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [2147549183,2147549183,2147549183,2147549183]
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSSE3-NEXT:    pand %xmm2, %xmm0
+; SSSE3-NEXT:    pandn {{.*}}(%rip), %xmm2
+; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: trunc_usat_v4i32_v4i16:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pminud {{.*}}(%rip), %xmm0
+; SSE41-NEXT:    packusdw %xmm0, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: trunc_usat_v4i32_v4i16:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vpackusdw %xmm0, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: trunc_usat_v4i32_v4i16:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [65535,65535,65535,65535]
+; AVX2-NEXT:    vpminud %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpackusdw %xmm0, %xmm0, %xmm0
+; AVX2-NEXT:    retq
+;
+; AVX512F-LABEL: trunc_usat_v4i32_v4i16:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512F-NEXT:    vpmovusdw %zmm0, %ymm0
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: trunc_usat_v4i32_v4i16:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpmovusdw %xmm0, %xmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: trunc_usat_v4i32_v4i16:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512BW-NEXT:    vpmovusdw %zmm0, %ymm0
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: trunc_usat_v4i32_v4i16:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpmovusdw %xmm0, %xmm0
+; AVX512BWVL-NEXT:    retq
+;
+; SKX-LABEL: trunc_usat_v4i32_v4i16:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpmovusdw %xmm0, %xmm0
+; SKX-NEXT:    retq
+  %1 = icmp ult <4 x i32> %a0, <i32 65535, i32 65535, i32 65535, i32 65535>
+  %2 = select <4 x i1> %1, <4 x i32> %a0, <4 x i32> <i32 65535, i32 65535, i32 65535, i32 65535>
+  %3 = trunc <4 x i32> %2 to <4 x i16>
+  ret <4 x i16> %3
+}
+
+define void @trunc_usat_v4i32_v4i16_store(<4 x i32> %a0, <4 x i16> *%p1) {
+; SSE2-LABEL: trunc_usat_v4i32_v4i16_store:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [2147483648,2147483648,2147483648,2147483648]
+; SSE2-NEXT:    pxor %xmm0, %xmm1
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147549183,2147549183,2147549183,2147549183]
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm2
+; SSE2-NEXT:    por %xmm0, %xmm2
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm2[0,2,2,3,4,5,6,7]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,6,6,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSE2-NEXT:    movq %xmm0, (%rdi)
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: trunc_usat_v4i32_v4i16_store:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [2147483648,2147483648,2147483648,2147483648]
+; SSSE3-NEXT:    pxor %xmm0, %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [2147549183,2147549183,2147549183,2147549183]
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSSE3-NEXT:    pand %xmm2, %xmm0
+; SSSE3-NEXT:    pandn {{.*}}(%rip), %xmm2
+; SSSE3-NEXT:    por %xmm0, %xmm2
+; SSSE3-NEXT:    pshufb {{.*#+}} xmm2 = xmm2[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
+; SSSE3-NEXT:    movq %xmm2, (%rdi)
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: trunc_usat_v4i32_v4i16_store:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pminud {{.*}}(%rip), %xmm0
+; SSE41-NEXT:    packusdw %xmm0, %xmm0
+; SSE41-NEXT:    movq %xmm0, (%rdi)
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: trunc_usat_v4i32_v4i16_store:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vpackusdw %xmm0, %xmm0, %xmm0
+; AVX1-NEXT:    vmovq %xmm0, (%rdi)
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: trunc_usat_v4i32_v4i16_store:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [65535,65535,65535,65535]
+; AVX2-NEXT:    vpminud %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpackusdw %xmm0, %xmm0, %xmm0
+; AVX2-NEXT:    vmovq %xmm0, (%rdi)
+; AVX2-NEXT:    retq
+;
+; AVX512F-LABEL: trunc_usat_v4i32_v4i16_store:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512F-NEXT:    vpmovusdw %zmm0, %ymm0
+; AVX512F-NEXT:    vmovq %xmm0, (%rdi)
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: trunc_usat_v4i32_v4i16_store:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpmovusdw %xmm0, (%rdi)
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: trunc_usat_v4i32_v4i16_store:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512BW-NEXT:    vpmovusdw %zmm0, %ymm0
+; AVX512BW-NEXT:    vmovq %xmm0, (%rdi)
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: trunc_usat_v4i32_v4i16_store:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpmovusdw %xmm0, (%rdi)
+; AVX512BWVL-NEXT:    retq
+;
+; SKX-LABEL: trunc_usat_v4i32_v4i16_store:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpmovusdw %xmm0, (%rdi)
+; SKX-NEXT:    retq
+  %1 = icmp ult <4 x i32> %a0, <i32 65535, i32 65535, i32 65535, i32 65535>
+  %2 = select <4 x i1> %1, <4 x i32> %a0, <4 x i32> <i32 65535, i32 65535, i32 65535, i32 65535>
+  %3 = trunc <4 x i32> %2 to <4 x i16>
+  store <4 x i16> %3, <4 x i16> *%p1
+  ret void
 }
 
 define <8 x i16> @trunc_usat_v8i32_v8i16(<8 x i32> %a0) {
 ; SSE2-LABEL: trunc_usat_v8i32_v8i16:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm0, %xmm3
+; SSE2-NEXT:    pxor %xmm2, %xmm3
+; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [2147549183,2147549183,2147549183,2147549183]
+; SSE2-NEXT:    movdqa %xmm4, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm5
+; SSE2-NEXT:    pcmpeqd %xmm3, %xmm3
+; SSE2-NEXT:    pand %xmm5, %xmm0
+; SSE2-NEXT:    pxor %xmm3, %xmm5
+; SSE2-NEXT:    por %xmm5, %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm4
+; SSE2-NEXT:    pxor %xmm4, %xmm3
+; SSE2-NEXT:    pand %xmm1, %xmm4
+; SSE2-NEXT:    por %xmm3, %xmm4
+; SSE2-NEXT:    pslld $16, %xmm4
+; SSE2-NEXT:    psrad $16, %xmm4
+; SSE2-NEXT:    pslld $16, %xmm0
+; SSE2-NEXT:    psrad $16, %xmm0
+; SSE2-NEXT:    packssdw %xmm4, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: trunc_usat_v8i32_v8i16:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [65535,65535,65535,65535]
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm3 = [2147483648,2147483648,2147483648,2147483648]
+; SSSE3-NEXT:    movdqa %xmm0, %xmm4
+; SSSE3-NEXT:    pxor %xmm3, %xmm4
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm5 = [2147549183,2147549183,2147549183,2147549183]
+; SSSE3-NEXT:    movdqa %xmm5, %xmm6
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm6
+; SSSE3-NEXT:    pand %xmm6, %xmm0
+; SSSE3-NEXT:    pandn %xmm2, %xmm6
+; SSSE3-NEXT:    por %xmm6, %xmm0
+; SSSE3-NEXT:    pxor %xmm1, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm5
+; SSSE3-NEXT:    pand %xmm5, %xmm1
+; SSSE3-NEXT:    pandn %xmm2, %xmm5
+; SSSE3-NEXT:    por %xmm1, %xmm5
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
+; SSSE3-NEXT:    pshufb %xmm1, %xmm5
+; SSSE3-NEXT:    pshufb %xmm1, %xmm0
+; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm5[0]
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: trunc_usat_v8i32_v8i16:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [65535,65535,65535,65535]
+; SSE41-NEXT:    pminud %xmm2, %xmm1
+; SSE41-NEXT:    pminud %xmm2, %xmm0
+; SSE41-NEXT:    packusdw %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: trunc_usat_v8i32_v8i16:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [65535,65535,65535,65535]
+; AVX1-NEXT:    vpminud %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpminud %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: trunc_usat_v8i32_v8i16:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [65535,65535,65535,65535,65535,65535,65535,65535]
+; AVX2-NEXT:    vpminud %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+;
+; AVX512F-LABEL: trunc_usat_v8i32_v8i16:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512F-NEXT:    vpmovusdw %zmm0, %ymm0
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: trunc_usat_v8i32_v8i16:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpmovusdw %ymm0, %xmm0
+; AVX512VL-NEXT:    vzeroupper
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: trunc_usat_v8i32_v8i16:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512BW-NEXT:    vpmovusdw %zmm0, %ymm0
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: trunc_usat_v8i32_v8i16:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpmovusdw %ymm0, %xmm0
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
+;
+; SKX-LABEL: trunc_usat_v8i32_v8i16:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpmovusdw %ymm0, %xmm0
+; SKX-NEXT:    vzeroupper
+; SKX-NEXT:    retq
+  %1 = icmp ult <8 x i32> %a0, <i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535>
+  %2 = select <8 x i1> %1, <8 x i32> %a0, <8 x i32> <i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535>
+  %3 = trunc <8 x i32> %2 to <8 x i16>
+  ret <8 x i16> %3
+}
+
+define <16 x i16> @trunc_usat_v16i32_v16i16(<16 x i32>* %p0) {
+; SSE2-LABEL: trunc_usat_v16i32_v16i16:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa (%rdi), %xmm5
+; SSE2-NEXT:    movdqa 16(%rdi), %xmm8
+; SSE2-NEXT:    movdqa 32(%rdi), %xmm0
+; SSE2-NEXT:    movdqa 48(%rdi), %xmm4
+; SSE2-NEXT:    movdqa {{.*#+}} xmm6 = [2147483648,2147483648,2147483648,2147483648]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    pxor %xmm2, %xmm3
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [2147549183,2147549183,2147549183,2147549183]
-; SSE2-NEXT:    movdqa %xmm4, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm5
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm3
-; SSE2-NEXT:    pand %xmm5, %xmm0
-; SSE2-NEXT:    pxor %xmm3, %xmm5
+; SSE2-NEXT:    pxor %xmm6, %xmm3
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147549183,2147549183,2147549183,2147549183]
+; SSE2-NEXT:    movdqa %xmm2, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm1
+; SSE2-NEXT:    pcmpeqd %xmm7, %xmm7
+; SSE2-NEXT:    pand %xmm1, %xmm0
+; SSE2-NEXT:    pxor %xmm7, %xmm1
+; SSE2-NEXT:    por %xmm0, %xmm1
+; SSE2-NEXT:    movdqa %xmm4, %xmm0
+; SSE2-NEXT:    pxor %xmm6, %xmm0
+; SSE2-NEXT:    movdqa %xmm2, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSE2-NEXT:    pand %xmm3, %xmm4
+; SSE2-NEXT:    pxor %xmm7, %xmm3
+; SSE2-NEXT:    por %xmm4, %xmm3
+; SSE2-NEXT:    movdqa %xmm5, %xmm4
+; SSE2-NEXT:    pxor %xmm6, %xmm4
+; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm0
+; SSE2-NEXT:    pand %xmm0, %xmm5
+; SSE2-NEXT:    pxor %xmm7, %xmm0
 ; SSE2-NEXT:    por %xmm5, %xmm0
-; SSE2-NEXT:    pxor %xmm1, %xmm2
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm4
-; SSE2-NEXT:    pxor %xmm4, %xmm3
-; SSE2-NEXT:    pand %xmm1, %xmm4
-; SSE2-NEXT:    por %xmm3, %xmm4
-; SSE2-NEXT:    pslld $16, %xmm4
-; SSE2-NEXT:    psrad $16, %xmm4
+; SSE2-NEXT:    pxor %xmm8, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm6, %xmm2
+; SSE2-NEXT:    pxor %xmm2, %xmm7
+; SSE2-NEXT:    pand %xmm8, %xmm2
+; SSE2-NEXT:    por %xmm7, %xmm2
+; SSE2-NEXT:    pslld $16, %xmm2
+; SSE2-NEXT:    psrad $16, %xmm2
 ; SSE2-NEXT:    pslld $16, %xmm0
 ; SSE2-NEXT:    psrad $16, %xmm0
-; SSE2-NEXT:    packssdw %xmm4, %xmm0
+; SSE2-NEXT:    packssdw %xmm2, %xmm0
+; SSE2-NEXT:    pslld $16, %xmm3
+; SSE2-NEXT:    psrad $16, %xmm3
+; SSE2-NEXT:    pslld $16, %xmm1
+; SSE2-NEXT:    psrad $16, %xmm1
+; SSE2-NEXT:    packssdw %xmm3, %xmm1
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: trunc_usat_v16i32_v16i16:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa (%rdi), %xmm5
+; SSSE3-NEXT:    movdqa 16(%rdi), %xmm8
+; SSSE3-NEXT:    movdqa 32(%rdi), %xmm0
+; SSSE3-NEXT:    movdqa 48(%rdi), %xmm4
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm6 = [2147483648,2147483648,2147483648,2147483648]
+; SSSE3-NEXT:    movdqa %xmm0, %xmm3
+; SSSE3-NEXT:    pxor %xmm6, %xmm3
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [2147549183,2147549183,2147549183,2147549183]
+; SSSE3-NEXT:    movdqa %xmm2, %xmm1
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm1
+; SSSE3-NEXT:    pcmpeqd %xmm7, %xmm7
+; SSSE3-NEXT:    pand %xmm1, %xmm0
+; SSSE3-NEXT:    pxor %xmm7, %xmm1
+; SSSE3-NEXT:    por %xmm0, %xmm1
+; SSSE3-NEXT:    movdqa %xmm4, %xmm0
+; SSSE3-NEXT:    pxor %xmm6, %xmm0
+; SSSE3-NEXT:    movdqa %xmm2, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSSE3-NEXT:    pand %xmm3, %xmm4
+; SSSE3-NEXT:    pxor %xmm7, %xmm3
+; SSSE3-NEXT:    por %xmm4, %xmm3
+; SSSE3-NEXT:    movdqa %xmm5, %xmm4
+; SSSE3-NEXT:    pxor %xmm6, %xmm4
+; SSSE3-NEXT:    movdqa %xmm2, %xmm0
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm0
+; SSSE3-NEXT:    pand %xmm0, %xmm5
+; SSSE3-NEXT:    pxor %xmm7, %xmm0
+; SSSE3-NEXT:    por %xmm5, %xmm0
+; SSSE3-NEXT:    pxor %xmm8, %xmm6
+; SSSE3-NEXT:    pcmpgtd %xmm6, %xmm2
+; SSSE3-NEXT:    pxor %xmm2, %xmm7
+; SSSE3-NEXT:    pand %xmm8, %xmm2
+; SSSE3-NEXT:    por %xmm7, %xmm2
+; SSSE3-NEXT:    pslld $16, %xmm2
+; SSSE3-NEXT:    psrad $16, %xmm2
+; SSSE3-NEXT:    pslld $16, %xmm0
+; SSSE3-NEXT:    psrad $16, %xmm0
+; SSSE3-NEXT:    packssdw %xmm2, %xmm0
+; SSSE3-NEXT:    pslld $16, %xmm3
+; SSSE3-NEXT:    psrad $16, %xmm3
+; SSSE3-NEXT:    pslld $16, %xmm1
+; SSSE3-NEXT:    psrad $16, %xmm1
+; SSSE3-NEXT:    packssdw %xmm3, %xmm1
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: trunc_usat_v16i32_v16i16:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [65535,65535,65535,65535]
+; SSE41-NEXT:    movdqa 48(%rdi), %xmm2
+; SSE41-NEXT:    pminud %xmm0, %xmm2
+; SSE41-NEXT:    movdqa 32(%rdi), %xmm1
+; SSE41-NEXT:    pminud %xmm0, %xmm1
+; SSE41-NEXT:    packusdw %xmm2, %xmm1
+; SSE41-NEXT:    movdqa 16(%rdi), %xmm2
+; SSE41-NEXT:    pminud %xmm0, %xmm2
+; SSE41-NEXT:    pminud (%rdi), %xmm0
+; SSE41-NEXT:    packusdw %xmm2, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: trunc_usat_v16i32_v16i16:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm0 = [65535,65535,65535,65535]
+; AVX1-NEXT:    vpminud 16(%rdi), %xmm0, %xmm1
+; AVX1-NEXT:    vpminud (%rdi), %xmm0, %xmm2
+; AVX1-NEXT:    vpackusdw %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vpminud 48(%rdi), %xmm0, %xmm2
+; AVX1-NEXT:    vpminud 32(%rdi), %xmm0, %xmm0
+; AVX1-NEXT:    vpackusdw %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: trunc_usat_v16i32_v16i16:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm0 = [65535,65535,65535,65535,65535,65535,65535,65535]
+; AVX2-NEXT:    vpminud 32(%rdi), %ymm0, %ymm1
+; AVX2-NEXT:    vpminud (%rdi), %ymm0, %ymm0
+; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: trunc_usat_v16i32_v16i16:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovdqa64 (%rdi), %zmm0
+; AVX512-NEXT:    vpmovusdw %zmm0, %ymm0
+; AVX512-NEXT:    retq
+;
+; SKX-LABEL: trunc_usat_v16i32_v16i16:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vmovdqa64 (%rdi), %zmm0
+; SKX-NEXT:    vpmovusdw %zmm0, %ymm0
+; SKX-NEXT:    retq
+  %a0 = load <16 x i32>, <16 x i32>* %p0
+  %1 = icmp ult <16 x i32> %a0, <i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535>
+  %2 = select <16 x i1> %1, <16 x i32> %a0, <16 x i32> <i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535>
+  %3 = trunc <16 x i32> %2 to <16 x i16>
+  ret <16 x i16> %3
+}
+
+;
+; Unsigned saturation truncation to vXi8
+;
+
+define <2 x i8> @trunc_usat_v2i64_v2i8(<2 x i64> %a0) {
+; SSE2-LABEL: trunc_usat_v2i64_v2i8:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [9223372039002259456,9223372039002259456]
+; SSE2-NEXT:    pxor %xmm0, %xmm1
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [9223372039002259711,9223372039002259711]
+; SSE2-NEXT:    movdqa %xmm2, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pand %xmm4, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
+; SSE2-NEXT:    por %xmm2, %xmm1
+; SSE2-NEXT:    pand %xmm1, %xmm0
+; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm1
+; SSE2-NEXT:    por %xmm0, %xmm1
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE2-NEXT:    packuswb %xmm1, %xmm1
+; SSE2-NEXT:    packuswb %xmm1, %xmm1
+; SSE2-NEXT:    packuswb %xmm1, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: trunc_usat_v2i64_v2i8:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [9223372039002259456,9223372039002259456]
+; SSSE3-NEXT:    pxor %xmm0, %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [9223372039002259711,9223372039002259711]
+; SSSE3-NEXT:    movdqa %xmm2, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm4, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,1,3,3]
+; SSSE3-NEXT:    por %xmm1, %xmm2
+; SSSE3-NEXT:    pand %xmm2, %xmm0
+; SSSE3-NEXT:    pandn {{.*}}(%rip), %xmm2
+; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: trunc_usat_v2i64_v2i8:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    movapd {{.*#+}} xmm2 = [255,255]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [9223372039002259456,9223372039002259456]
+; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [9223372039002259711,9223372039002259711]
+; SSE41-NEXT:    movdqa %xmm3, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm3, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
+; SSE41-NEXT:    pshufb {{.*#+}} xmm2 = xmm2[0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: trunc_usat_v2i64_v2i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovapd {{.*#+}} xmm1 = [255,255]
+; AVX-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm2
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm3 = [9223372036854776063,9223372036854776063]
+; AVX-NEXT:    vpcmpgtq %xmm2, %xmm3, %xmm2
+; AVX-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX-NEXT:    retq
+;
+; AVX512F-LABEL: trunc_usat_v2i64_v2i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512F-NEXT:    vpmovusqb %zmm0, %xmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: trunc_usat_v2i64_v2i8:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpmovusqb %xmm0, %xmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: trunc_usat_v2i64_v2i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512BW-NEXT:    vpmovusqb %zmm0, %xmm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: trunc_usat_v2i64_v2i8:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpmovusqb %xmm0, %xmm0
+; AVX512BWVL-NEXT:    retq
+;
+; SKX-LABEL: trunc_usat_v2i64_v2i8:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpmovusqb %xmm0, %xmm0
+; SKX-NEXT:    retq
+  %1 = icmp ult <2 x i64> %a0, <i64 255, i64 255>
+  %2 = select <2 x i1> %1, <2 x i64> %a0, <2 x i64> <i64 255, i64 255>
+  %3 = trunc <2 x i64> %2 to <2 x i8>
+  ret <2 x i8> %3
+}
+
+define void @trunc_usat_v2i64_v2i8_store(<2 x i64> %a0, <2 x i8>* %p1) {
+; SSE2-LABEL: trunc_usat_v2i64_v2i8_store:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [9223372039002259456,9223372039002259456]
+; SSE2-NEXT:    pxor %xmm0, %xmm1
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [9223372039002259711,9223372039002259711]
+; SSE2-NEXT:    movdqa %xmm2, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pand %xmm4, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,1,3,3]
+; SSE2-NEXT:    por %xmm1, %xmm2
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm2
+; SSE2-NEXT:    por %xmm0, %xmm2
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm2
+; SSE2-NEXT:    packuswb %xmm2, %xmm2
+; SSE2-NEXT:    packuswb %xmm0, %xmm2
+; SSE2-NEXT:    packuswb %xmm0, %xmm2
+; SSE2-NEXT:    movd %xmm2, %eax
+; SSE2-NEXT:    movw %ax, (%rdi)
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: trunc_usat_v2i64_v2i8_store:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [9223372039002259456,9223372039002259456]
+; SSSE3-NEXT:    pxor %xmm0, %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [9223372039002259711,9223372039002259711]
+; SSSE3-NEXT:    movdqa %xmm2, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm4, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,1,3,3]
+; SSSE3-NEXT:    por %xmm1, %xmm2
+; SSSE3-NEXT:    pand %xmm2, %xmm0
+; SSSE3-NEXT:    pandn {{.*}}(%rip), %xmm2
+; SSSE3-NEXT:    por %xmm0, %xmm2
+; SSSE3-NEXT:    pshufb {{.*#+}} xmm2 = xmm2[0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; SSSE3-NEXT:    movd %xmm2, %eax
+; SSSE3-NEXT:    movw %ax, (%rdi)
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: trunc_usat_v2i64_v2i8_store:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    movapd {{.*#+}} xmm2 = [255,255]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [9223372039002259456,9223372039002259456]
+; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [9223372039002259711,9223372039002259711]
+; SSE41-NEXT:    movdqa %xmm3, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm3, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
+; SSE41-NEXT:    pshufb {{.*#+}} xmm2 = xmm2[0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; SSE41-NEXT:    pextrw $0, %xmm2, (%rdi)
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: trunc_usat_v2i64_v2i8_store:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovapd {{.*#+}} xmm1 = [255,255]
+; AVX-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm2
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm3 = [9223372036854776063,9223372036854776063]
+; AVX-NEXT:    vpcmpgtq %xmm2, %xmm3, %xmm2
+; AVX-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX-NEXT:    vpextrw $0, %xmm0, (%rdi)
+; AVX-NEXT:    retq
+;
+; AVX512F-LABEL: trunc_usat_v2i64_v2i8_store:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512F-NEXT:    vpmovusqb %zmm0, %xmm0
+; AVX512F-NEXT:    vpextrw $0, %xmm0, (%rdi)
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: trunc_usat_v2i64_v2i8_store:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpmovusqb %xmm0, (%rdi)
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: trunc_usat_v2i64_v2i8_store:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512BW-NEXT:    vpmovusqb %zmm0, %xmm0
+; AVX512BW-NEXT:    vpextrw $0, %xmm0, (%rdi)
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: trunc_usat_v2i64_v2i8_store:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpmovusqb %xmm0, (%rdi)
+; AVX512BWVL-NEXT:    retq
+;
+; SKX-LABEL: trunc_usat_v2i64_v2i8_store:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpmovusqb %xmm0, (%rdi)
+; SKX-NEXT:    retq
+  %1 = icmp ult <2 x i64> %a0, <i64 255, i64 255>
+  %2 = select <2 x i1> %1, <2 x i64> %a0, <2 x i64> <i64 255, i64 255>
+  %3 = trunc <2 x i64> %2 to <2 x i8>
+  store <2 x i8> %3, <2 x i8>* %p1
+  ret void
+}
+
+define <4 x i8> @trunc_usat_v4i64_v4i8(<4 x i64> %a0) {
+; SSE2-LABEL: trunc_usat_v4i64_v4i8:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [255,255]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [9223372039002259456,9223372039002259456]
+; SSE2-NEXT:    pxor %xmm4, %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [9223372039002259711,9223372039002259711]
+; SSE2-NEXT:    movdqa %xmm5, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pand %xmm7, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3]
+; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    pand %xmm0, %xmm2
+; SSE2-NEXT:    pandn %xmm8, %xmm0
+; SSE2-NEXT:    por %xmm2, %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm4
+; SSE2-NEXT:    movdqa %xmm5, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    pand %xmm3, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE2-NEXT:    por %xmm4, %xmm2
+; SSE2-NEXT:    pand %xmm2, %xmm1
+; SSE2-NEXT:    pandn %xmm8, %xmm2
+; SSE2-NEXT:    por %xmm1, %xmm2
+; SSE2-NEXT:    pand %xmm8, %xmm2
+; SSE2-NEXT:    pand %xmm8, %xmm0
+; SSE2-NEXT:    packuswb %xmm2, %xmm0
+; SSE2-NEXT:    packuswb %xmm0, %xmm0
+; SSE2-NEXT:    packuswb %xmm0, %xmm0
 ; SSE2-NEXT:    retq
 ;
-; SSSE3-LABEL: trunc_usat_v8i32_v8i16:
+; SSSE3-LABEL: trunc_usat_v4i64_v4i8:
 ; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [65535,65535,65535,65535]
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm3 = [2147483648,2147483648,2147483648,2147483648]
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [255,255]
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm3 = [9223372039002259456,9223372039002259456]
 ; SSSE3-NEXT:    movdqa %xmm0, %xmm4
 ; SSSE3-NEXT:    pxor %xmm3, %xmm4
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm5 = [2147549183,2147549183,2147549183,2147549183]
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm5 = [9223372039002259711,9223372039002259711]
 ; SSSE3-NEXT:    movdqa %xmm5, %xmm6
 ; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm7, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSSE3-NEXT:    por %xmm4, %xmm6
 ; SSSE3-NEXT:    pand %xmm6, %xmm0
 ; SSSE3-NEXT:    pandn %xmm2, %xmm6
 ; SSSE3-NEXT:    por %xmm6, %xmm0
 ; SSSE3-NEXT:    pxor %xmm1, %xmm3
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm5
-; SSSE3-NEXT:    pand %xmm5, %xmm1
-; SSSE3-NEXT:    pandn %xmm2, %xmm5
-; SSSE3-NEXT:    por %xmm1, %xmm5
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
-; SSSE3-NEXT:    pshufb %xmm1, %xmm5
+; SSSE3-NEXT:    movdqa %xmm5, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm6, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm3, %xmm4
+; SSSE3-NEXT:    pand %xmm4, %xmm1
+; SSSE3-NEXT:    pandn %xmm2, %xmm4
+; SSSE3-NEXT:    por %xmm1, %xmm4
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = <0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; SSSE3-NEXT:    pshufb %xmm1, %xmm4
 ; SSSE3-NEXT:    pshufb %xmm1, %xmm0
-; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm5[0]
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
 ; SSSE3-NEXT:    retq
 ;
-; SSE41-LABEL: trunc_usat_v8i32_v8i16:
+; SSE41-LABEL: trunc_usat_v4i64_v4i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [65535,65535,65535,65535]
-; SSE41-NEXT:    pminud %xmm2, %xmm1
-; SSE41-NEXT:    pminud %xmm2, %xmm0
-; SSE41-NEXT:    packusdw %xmm1, %xmm0
+; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    movapd {{.*#+}} xmm4 = [255,255]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm5 = [9223372039002259456,9223372039002259456]
+; SSE41-NEXT:    pxor %xmm5, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm6 = [9223372039002259711,9223372039002259711]
+; SSE41-NEXT:    movdqa %xmm6, %xmm3
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm3
+; SSE41-NEXT:    movdqa %xmm6, %xmm7
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm7
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[0,0,2,2]
+; SSE41-NEXT:    pand %xmm3, %xmm0
+; SSE41-NEXT:    por %xmm7, %xmm0
+; SSE41-NEXT:    movapd %xmm4, %xmm3
+; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm3
+; SSE41-NEXT:    pxor %xmm1, %xmm5
+; SSE41-NEXT:    movdqa %xmm6, %xmm2
+; SSE41-NEXT:    pcmpeqd %xmm5, %xmm2
+; SSE41-NEXT:    pcmpgtd %xmm5, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
+; SSE41-NEXT:    pand %xmm2, %xmm0
+; SSE41-NEXT:    por %xmm6, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm4
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = <0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; SSE41-NEXT:    pshufb %xmm0, %xmm4
+; SSE41-NEXT:    pshufb %xmm0, %xmm3
+; SSE41-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: trunc_usat_v8i32_v8i16:
+; AVX1-LABEL: trunc_usat_v4i64_v4i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [65535,65535,65535,65535]
-; AVX1-NEXT:    vpminud %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpminud %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [9223372036854775808,9223372036854775808]
+; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm2
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [9223372036854776063,9223372036854776063]
+; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
+; AVX1-NEXT:    vpxor %xmm1, %xmm4, %xmm1
+; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm3, %xmm1
+; AVX1-NEXT:    vmovapd {{.*#+}} xmm3 = [255,255]
+; AVX1-NEXT:    vblendvpd %xmm1, %xmm4, %xmm3, %xmm1
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = <0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX1-NEXT:    vpshufb %xmm4, %xmm1, %xmm1
+; AVX1-NEXT:    vblendvpd %xmm2, %xmm0, %xmm3, %xmm0
+; AVX1-NEXT:    vpshufb %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
-; AVX2-LABEL: trunc_usat_v8i32_v8i16:
+; AVX2-LABEL: trunc_usat_v4i64_v4i8:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [65535,65535,65535,65535,65535,65535,65535,65535]
-; AVX2-NEXT:    vpminud %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm1 = [255,255,255,255]
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
+; AVX2-NEXT:    vpxor %ymm2, %ymm0, %ymm2
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [9223372036854776063,9223372036854776063,9223372036854776063,9223372036854776063]
+; AVX2-NEXT:    vpcmpgtq %ymm2, %ymm3, %ymm2
+; AVX2-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = <0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX2-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
-; AVX512F-LABEL: trunc_usat_v8i32_v8i16:
+; AVX512F-LABEL: trunc_usat_v4i64_v4i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [65535,65535,65535,65535,65535,65535,65535,65535]
-; AVX512F-NEXT:    vpminud %ymm1, %ymm0, %ymm0
-; AVX512F-NEXT:    vpmovdw %zmm0, %ymm0
-; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512F-NEXT:    vpmovusqb %zmm0, %xmm0
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
-; AVX512VL-LABEL: trunc_usat_v8i32_v8i16:
+; AVX512VL-LABEL: trunc_usat_v4i64_v4i8:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpmovusdw %ymm0, %xmm0
+; AVX512VL-NEXT:    vpmovusqb %ymm0, %xmm0
 ; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq
 ;
-; AVX512BW-LABEL: trunc_usat_v8i32_v8i16:
+; AVX512BW-LABEL: trunc_usat_v4i64_v4i8:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [65535,65535,65535,65535,65535,65535,65535,65535]
-; AVX512BW-NEXT:    vpminud %ymm1, %ymm0, %ymm0
-; AVX512BW-NEXT:    vpmovdw %zmm0, %ymm0
-; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512BW-NEXT:    vpmovusqb %zmm0, %xmm0
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
-; AVX512BWVL-LABEL: trunc_usat_v8i32_v8i16:
+; AVX512BWVL-LABEL: trunc_usat_v4i64_v4i8:
 ; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vpmovusdw %ymm0, %xmm0
+; AVX512BWVL-NEXT:    vpmovusqb %ymm0, %xmm0
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
-  %1 = icmp ult <8 x i32> %a0, <i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535>
-  %2 = select <8 x i1> %1, <8 x i32> %a0, <8 x i32> <i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535>
-  %3 = trunc <8 x i32> %2 to <8 x i16>
-  ret <8 x i16> %3
+;
+; SKX-LABEL: trunc_usat_v4i64_v4i8:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpmovusqb %ymm0, %xmm0
+; SKX-NEXT:    vzeroupper
+; SKX-NEXT:    retq
+  %1 = icmp ult <4 x i64> %a0, <i64 255, i64 255, i64 255, i64 255>
+  %2 = select <4 x i1> %1, <4 x i64> %a0, <4 x i64> <i64 255, i64 255, i64 255, i64 255>
+  %3 = trunc <4 x i64> %2 to <4 x i8>
+  ret <4 x i8> %3
 }
 
-define <16 x i16> @trunc_usat_v16i32_v16i16(<16 x i32> %a0) {
-; SSE2-LABEL: trunc_usat_v16i32_v16i16:
+define void @trunc_usat_v4i64_v4i8_store(<4 x i64> %a0, <4 x i8> *%p1) {
+; SSE2-LABEL: trunc_usat_v4i64_v4i8_store:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa %xmm1, %xmm8
-; SSE2-NEXT:    movdqa {{.*#+}} xmm6 = [2147483648,2147483648,2147483648,2147483648]
-; SSE2-NEXT:    movdqa %xmm2, %xmm7
-; SSE2-NEXT:    pxor %xmm6, %xmm7
-; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [2147549183,2147549183,2147549183,2147549183]
-; SSE2-NEXT:    movdqa %xmm5, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm7, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm7, %xmm7
-; SSE2-NEXT:    pand %xmm1, %xmm2
-; SSE2-NEXT:    pxor %xmm7, %xmm1
-; SSE2-NEXT:    por %xmm2, %xmm1
-; SSE2-NEXT:    movdqa %xmm3, %xmm4
-; SSE2-NEXT:    pxor %xmm6, %xmm4
-; SSE2-NEXT:    movdqa %xmm5, %xmm2
-; SSE2-NEXT:    pcmpgtd %xmm4, %xmm2
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pxor %xmm7, %xmm2
-; SSE2-NEXT:    por %xmm3, %xmm2
+; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [255,255]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [9223372039002259456,9223372039002259456]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    pxor %xmm6, %xmm3
-; SSE2-NEXT:    movdqa %xmm5, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm4
-; SSE2-NEXT:    pand %xmm4, %xmm0
-; SSE2-NEXT:    pxor %xmm7, %xmm4
+; SSE2-NEXT:    pxor %xmm4, %xmm3
+; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [9223372039002259711,9223372039002259711]
+; SSE2-NEXT:    movdqa %xmm5, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,1,3,3]
+; SSE2-NEXT:    pand %xmm7, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm6[1,1,3,3]
+; SSE2-NEXT:    por %xmm2, %xmm3
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    pandn %xmm8, %xmm3
+; SSE2-NEXT:    por %xmm0, %xmm3
+; SSE2-NEXT:    pxor %xmm1, %xmm4
+; SSE2-NEXT:    movdqa %xmm5, %xmm0
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    pand %xmm2, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
 ; SSE2-NEXT:    por %xmm4, %xmm0
-; SSE2-NEXT:    pxor %xmm8, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm6, %xmm5
-; SSE2-NEXT:    pxor %xmm5, %xmm7
-; SSE2-NEXT:    pand %xmm8, %xmm5
-; SSE2-NEXT:    por %xmm7, %xmm5
-; SSE2-NEXT:    pslld $16, %xmm5
-; SSE2-NEXT:    psrad $16, %xmm5
-; SSE2-NEXT:    pslld $16, %xmm0
-; SSE2-NEXT:    psrad $16, %xmm0
-; SSE2-NEXT:    packssdw %xmm5, %xmm0
-; SSE2-NEXT:    pslld $16, %xmm2
-; SSE2-NEXT:    psrad $16, %xmm2
-; SSE2-NEXT:    pslld $16, %xmm1
-; SSE2-NEXT:    psrad $16, %xmm1
-; SSE2-NEXT:    packssdw %xmm2, %xmm1
+; SSE2-NEXT:    pand %xmm0, %xmm1
+; SSE2-NEXT:    pandn %xmm8, %xmm0
+; SSE2-NEXT:    por %xmm1, %xmm0
+; SSE2-NEXT:    pand %xmm8, %xmm0
+; SSE2-NEXT:    pand %xmm8, %xmm3
+; SSE2-NEXT:    packuswb %xmm0, %xmm3
+; SSE2-NEXT:    packuswb %xmm0, %xmm3
+; SSE2-NEXT:    packuswb %xmm0, %xmm3
+; SSE2-NEXT:    movd %xmm3, (%rdi)
 ; SSE2-NEXT:    retq
 ;
-; SSSE3-LABEL: trunc_usat_v16i32_v16i16:
+; SSSE3-LABEL: trunc_usat_v4i64_v4i8_store:
 ; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    movdqa %xmm1, %xmm8
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm6 = [2147483648,2147483648,2147483648,2147483648]
-; SSSE3-NEXT:    movdqa %xmm2, %xmm7
-; SSSE3-NEXT:    pxor %xmm6, %xmm7
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm5 = [2147549183,2147549183,2147549183,2147549183]
-; SSSE3-NEXT:    movdqa %xmm5, %xmm1
-; SSSE3-NEXT:    pcmpgtd %xmm7, %xmm1
-; SSSE3-NEXT:    pcmpeqd %xmm7, %xmm7
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pxor %xmm7, %xmm1
-; SSSE3-NEXT:    por %xmm2, %xmm1
-; SSSE3-NEXT:    movdqa %xmm3, %xmm4
-; SSSE3-NEXT:    pxor %xmm6, %xmm4
-; SSSE3-NEXT:    movdqa %xmm5, %xmm2
-; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm2
-; SSSE3-NEXT:    pand %xmm2, %xmm3
-; SSSE3-NEXT:    pxor %xmm7, %xmm2
-; SSSE3-NEXT:    por %xmm3, %xmm2
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm8 = [255,255]
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm4 = [9223372039002259456,9223372039002259456]
 ; SSSE3-NEXT:    movdqa %xmm0, %xmm3
-; SSSE3-NEXT:    pxor %xmm6, %xmm3
-; SSSE3-NEXT:    movdqa %xmm5, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm4
-; SSSE3-NEXT:    pand %xmm4, %xmm0
-; SSSE3-NEXT:    pxor %xmm7, %xmm4
+; SSSE3-NEXT:    pxor %xmm4, %xmm3
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm5 = [9223372039002259711,9223372039002259711]
+; SSSE3-NEXT:    movdqa %xmm5, %xmm6
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm7, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm6[1,1,3,3]
+; SSSE3-NEXT:    por %xmm2, %xmm3
+; SSSE3-NEXT:    pand %xmm3, %xmm0
+; SSSE3-NEXT:    pandn %xmm8, %xmm3
+; SSSE3-NEXT:    por %xmm0, %xmm3
+; SSSE3-NEXT:    pxor %xmm1, %xmm4
+; SSSE3-NEXT:    movdqa %xmm5, %xmm0
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm2, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
 ; SSSE3-NEXT:    por %xmm4, %xmm0
-; SSSE3-NEXT:    pxor %xmm8, %xmm6
-; SSSE3-NEXT:    pcmpgtd %xmm6, %xmm5
-; SSSE3-NEXT:    pxor %xmm5, %xmm7
-; SSSE3-NEXT:    pand %xmm8, %xmm5
-; SSSE3-NEXT:    por %xmm7, %xmm5
-; SSSE3-NEXT:    pslld $16, %xmm5
-; SSSE3-NEXT:    psrad $16, %xmm5
-; SSSE3-NEXT:    pslld $16, %xmm0
-; SSSE3-NEXT:    psrad $16, %xmm0
-; SSSE3-NEXT:    packssdw %xmm5, %xmm0
-; SSSE3-NEXT:    pslld $16, %xmm2
-; SSSE3-NEXT:    psrad $16, %xmm2
-; SSSE3-NEXT:    pslld $16, %xmm1
-; SSSE3-NEXT:    psrad $16, %xmm1
-; SSSE3-NEXT:    packssdw %xmm2, %xmm1
+; SSSE3-NEXT:    pand %xmm0, %xmm1
+; SSSE3-NEXT:    pandn %xmm8, %xmm0
+; SSSE3-NEXT:    por %xmm1, %xmm0
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = <0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; SSSE3-NEXT:    pshufb %xmm1, %xmm0
+; SSSE3-NEXT:    pshufb %xmm1, %xmm3
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3]
+; SSSE3-NEXT:    movd %xmm3, (%rdi)
 ; SSSE3-NEXT:    retq
 ;
-; SSE41-LABEL: trunc_usat_v16i32_v16i16:
+; SSE41-LABEL: trunc_usat_v4i64_v4i8_store:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [65535,65535,65535,65535]
-; SSE41-NEXT:    pminud %xmm4, %xmm3
-; SSE41-NEXT:    pminud %xmm4, %xmm2
-; SSE41-NEXT:    packusdw %xmm3, %xmm2
-; SSE41-NEXT:    pminud %xmm4, %xmm1
-; SSE41-NEXT:    pminud %xmm4, %xmm0
-; SSE41-NEXT:    packusdw %xmm1, %xmm0
-; SSE41-NEXT:    movdqa %xmm2, %xmm1
+; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    movapd {{.*#+}} xmm3 = [255,255]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [9223372039002259456,9223372039002259456]
+; SSE41-NEXT:    pxor %xmm4, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm5 = [9223372039002259711,9223372039002259711]
+; SSE41-NEXT:    movdqa %xmm5, %xmm6
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm6
+; SSE41-NEXT:    movdqa %xmm5, %xmm7
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm7
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[0,0,2,2]
+; SSE41-NEXT:    pand %xmm6, %xmm0
+; SSE41-NEXT:    por %xmm7, %xmm0
+; SSE41-NEXT:    movapd %xmm3, %xmm6
+; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm6
+; SSE41-NEXT:    pxor %xmm1, %xmm4
+; SSE41-NEXT:    movdqa %xmm5, %xmm2
+; SSE41-NEXT:    pcmpeqd %xmm4, %xmm2
+; SSE41-NEXT:    pcmpgtd %xmm4, %xmm5
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2]
+; SSE41-NEXT:    pand %xmm2, %xmm0
+; SSE41-NEXT:    por %xmm5, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm3
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = <0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; SSE41-NEXT:    pshufb %xmm0, %xmm3
+; SSE41-NEXT:    pshufb %xmm0, %xmm6
+; SSE41-NEXT:    punpcklwd {{.*#+}} xmm6 = xmm6[0],xmm3[0],xmm6[1],xmm3[1],xmm6[2],xmm3[2],xmm6[3],xmm3[3]
+; SSE41-NEXT:    movd %xmm6, (%rdi)
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: trunc_usat_v16i32_v16i16:
+; AVX1-LABEL: trunc_usat_v4i64_v4i8_store:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [65535,65535,65535,65535]
-; AVX1-NEXT:    vpminud %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpminud %xmm3, %xmm0, %xmm0
-; AVX1-NEXT:    vpackusdw %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT:    vpminud %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpminud %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpackusdw %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [9223372036854775808,9223372036854775808]
+; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm2
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [9223372036854776063,9223372036854776063]
+; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
+; AVX1-NEXT:    vpxor %xmm1, %xmm4, %xmm1
+; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm3, %xmm1
+; AVX1-NEXT:    vmovapd {{.*#+}} xmm3 = [255,255]
+; AVX1-NEXT:    vblendvpd %xmm1, %xmm4, %xmm3, %xmm1
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = <0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX1-NEXT:    vpshufb %xmm4, %xmm1, %xmm1
+; AVX1-NEXT:    vblendvpd %xmm2, %xmm0, %xmm3, %xmm0
+; AVX1-NEXT:    vpshufb %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX1-NEXT:    vmovd %xmm0, (%rdi)
+; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
-; AVX2-LABEL: trunc_usat_v16i32_v16i16:
+; AVX2-LABEL: trunc_usat_v4i64_v4i8_store:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm2 = [65535,65535,65535,65535,65535,65535,65535,65535]
-; AVX2-NEXT:    vpminud %ymm2, %ymm1, %ymm1
-; AVX2-NEXT:    vpminud %ymm2, %ymm0, %ymm0
-; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm1 = [255,255,255,255]
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
+; AVX2-NEXT:    vpxor %ymm2, %ymm0, %ymm2
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [9223372036854776063,9223372036854776063,9223372036854776063,9223372036854776063]
+; AVX2-NEXT:    vpcmpgtq %ymm2, %ymm3, %ymm2
+; AVX2-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = <0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX2-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX2-NEXT:    vmovd %xmm0, (%rdi)
+; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
-; AVX512-LABEL: trunc_usat_v16i32_v16i16:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpmovusdw %zmm0, %ymm0
-; AVX512-NEXT:    retq
-  %1 = icmp ult <16 x i32> %a0, <i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535>
-  %2 = select <16 x i1> %1, <16 x i32> %a0, <16 x i32> <i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535>
-  %3 = trunc <16 x i32> %2 to <16 x i16>
-  ret <16 x i16> %3
-}
-
+; AVX512F-LABEL: trunc_usat_v4i64_v4i8_store:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512F-NEXT:    vpmovusqb %zmm0, %xmm0
+; AVX512F-NEXT:    vmovd %xmm0, (%rdi)
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
 ;
-; Unsigned saturation truncation to v16i8
+; AVX512VL-LABEL: trunc_usat_v4i64_v4i8_store:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpmovusqb %ymm0, (%rdi)
+; AVX512VL-NEXT:    vzeroupper
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: trunc_usat_v4i64_v4i8_store:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512BW-NEXT:    vpmovusqb %zmm0, %xmm0
+; AVX512BW-NEXT:    vmovd %xmm0, (%rdi)
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
 ;
+; AVX512BWVL-LABEL: trunc_usat_v4i64_v4i8_store:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpmovusqb %ymm0, (%rdi)
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
+;
+; SKX-LABEL: trunc_usat_v4i64_v4i8_store:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpmovusqb %ymm0, (%rdi)
+; SKX-NEXT:    vzeroupper
+; SKX-NEXT:    retq
+  %1 = icmp ult <4 x i64> %a0, <i64 255, i64 255, i64 255, i64 255>
+  %2 = select <4 x i1> %1, <4 x i64> %a0, <4 x i64> <i64 255, i64 255, i64 255, i64 255>
+  %3 = trunc <4 x i64> %2 to <4 x i8>
+  store <4 x i8> %3, <4 x i8> *%p1
+  ret void
+}
 
-define <8 x i8> @trunc_usat_v8i64_v8i8(<8 x i64> %a0) {
+define <8 x i8> @trunc_usat_v8i64_v8i8(<8 x i64>* %p0) {
 ; SSE2-LABEL: trunc_usat_v8i64_v8i8:
 ; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa (%rdi), %xmm6
+; SSE2-NEXT:    movdqa 16(%rdi), %xmm0
+; SSE2-NEXT:    movdqa 32(%rdi), %xmm9
+; SSE2-NEXT:    movdqa 48(%rdi), %xmm5
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [255,255]
-; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [9223372039002259456,9223372039002259456]
-; SSE2-NEXT:    movdqa %xmm1, %xmm7
-; SSE2-NEXT:    pxor %xmm5, %xmm7
-; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [9223372039002259711,9223372039002259711]
-; SSE2-NEXT:    movdqa %xmm9, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm7, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm7
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [9223372039002259456,9223372039002259456]
+; SSE2-NEXT:    movdqa %xmm0, %xmm7
+; SSE2-NEXT:    pxor %xmm3, %xmm7
+; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [9223372039002259711,9223372039002259711]
+; SSE2-NEXT:    movdqa %xmm10, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm7, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm7
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT:    pand %xmm4, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[1,1,3,3]
-; SSE2-NEXT:    por %xmm7, %xmm4
-; SSE2-NEXT:    pand %xmm4, %xmm1
-; SSE2-NEXT:    pandn %xmm8, %xmm4
-; SSE2-NEXT:    por %xmm1, %xmm4
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    pxor %xmm5, %xmm1
-; SSE2-NEXT:    movdqa %xmm9, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pand %xmm7, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm6
-; SSE2-NEXT:    pand %xmm6, %xmm0
-; SSE2-NEXT:    pandn %xmm8, %xmm6
+; SSE2-NEXT:    pand %xmm1, %xmm7
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm7, %xmm1
+; SSE2-NEXT:    pand %xmm1, %xmm0
+; SSE2-NEXT:    pandn %xmm8, %xmm1
+; SSE2-NEXT:    por %xmm0, %xmm1
+; SSE2-NEXT:    movdqa %xmm6, %xmm0
+; SSE2-NEXT:    pxor %xmm3, %xmm0
+; SSE2-NEXT:    movdqa %xmm10, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pand %xmm7, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm2, %xmm0
+; SSE2-NEXT:    pand %xmm0, %xmm6
+; SSE2-NEXT:    pandn %xmm8, %xmm0
 ; SSE2-NEXT:    por %xmm6, %xmm0
-; SSE2-NEXT:    packuswb %xmm4, %xmm0
-; SSE2-NEXT:    movdqa %xmm3, %xmm1
-; SSE2-NEXT:    pxor %xmm5, %xmm1
-; SSE2-NEXT:    movdqa %xmm9, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm1
+; SSE2-NEXT:    packuswb %xmm1, %xmm0
+; SSE2-NEXT:    movdqa %xmm5, %xmm1
+; SSE2-NEXT:    pxor %xmm3, %xmm1
+; SSE2-NEXT:    movdqa %xmm10, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm4
+; SSE2-NEXT:    pand %xmm4, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE2-NEXT:    por %xmm1, %xmm2
+; SSE2-NEXT:    pand %xmm2, %xmm5
+; SSE2-NEXT:    pandn %xmm8, %xmm2
+; SSE2-NEXT:    por %xmm5, %xmm2
+; SSE2-NEXT:    pxor %xmm9, %xmm3
+; SSE2-NEXT:    movdqa %xmm10, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm4, %xmm3
-; SSE2-NEXT:    pandn %xmm8, %xmm4
-; SSE2-NEXT:    por %xmm3, %xmm4
-; SSE2-NEXT:    pxor %xmm2, %xmm5
-; SSE2-NEXT:    movdqa %xmm9, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm5, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE2-NEXT:    pand %xmm3, %xmm5
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm5, %xmm1
-; SSE2-NEXT:    pand %xmm1, %xmm2
+; SSE2-NEXT:    por %xmm3, %xmm1
+; SSE2-NEXT:    pand %xmm1, %xmm9
 ; SSE2-NEXT:    pandn %xmm8, %xmm1
-; SSE2-NEXT:    por %xmm2, %xmm1
-; SSE2-NEXT:    packuswb %xmm4, %xmm1
+; SSE2-NEXT:    por %xmm9, %xmm1
+; SSE2-NEXT:    packuswb %xmm2, %xmm1
 ; SSE2-NEXT:    packuswb %xmm1, %xmm0
+; SSE2-NEXT:    packuswb %xmm0, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: trunc_usat_v8i64_v8i8:
 ; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa (%rdi), %xmm6
+; SSSE3-NEXT:    movdqa 16(%rdi), %xmm0
+; SSSE3-NEXT:    movdqa 32(%rdi), %xmm9
+; SSSE3-NEXT:    movdqa 48(%rdi), %xmm5
 ; SSSE3-NEXT:    movdqa {{.*#+}} xmm8 = [255,255]
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm5 = [9223372039002259456,9223372039002259456]
-; SSSE3-NEXT:    movdqa %xmm1, %xmm7
-; SSSE3-NEXT:    pxor %xmm5, %xmm7
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm9 = [9223372039002259711,9223372039002259711]
-; SSSE3-NEXT:    movdqa %xmm9, %xmm6
-; SSSE3-NEXT:    pcmpgtd %xmm7, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm7
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm3 = [9223372039002259456,9223372039002259456]
+; SSSE3-NEXT:    movdqa %xmm0, %xmm7
+; SSSE3-NEXT:    pxor %xmm3, %xmm7
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm10 = [9223372039002259711,9223372039002259711]
+; SSSE3-NEXT:    movdqa %xmm10, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm7, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm7
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm4, %xmm7
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[1,1,3,3]
-; SSSE3-NEXT:    por %xmm7, %xmm4
-; SSSE3-NEXT:    pand %xmm4, %xmm1
-; SSSE3-NEXT:    pandn %xmm8, %xmm4
-; SSSE3-NEXT:    por %xmm1, %xmm4
-; SSSE3-NEXT:    movdqa %xmm0, %xmm1
-; SSSE3-NEXT:    pxor %xmm5, %xmm1
-; SSSE3-NEXT:    movdqa %xmm9, %xmm6
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm7, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSSE3-NEXT:    por %xmm1, %xmm6
-; SSSE3-NEXT:    pand %xmm6, %xmm0
-; SSSE3-NEXT:    pandn %xmm8, %xmm6
+; SSSE3-NEXT:    pand %xmm1, %xmm7
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm7, %xmm1
+; SSSE3-NEXT:    pand %xmm1, %xmm0
+; SSSE3-NEXT:    pandn %xmm8, %xmm1
+; SSSE3-NEXT:    por %xmm0, %xmm1
+; SSSE3-NEXT:    movdqa %xmm6, %xmm0
+; SSSE3-NEXT:    pxor %xmm3, %xmm0
+; SSSE3-NEXT:    movdqa %xmm10, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm7, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    pand %xmm0, %xmm6
+; SSSE3-NEXT:    pandn %xmm8, %xmm0
 ; SSSE3-NEXT:    por %xmm6, %xmm0
-; SSSE3-NEXT:    packuswb %xmm4, %xmm0
-; SSSE3-NEXT:    movdqa %xmm3, %xmm1
-; SSSE3-NEXT:    pxor %xmm5, %xmm1
-; SSSE3-NEXT:    movdqa %xmm9, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm1
+; SSSE3-NEXT:    packuswb %xmm1, %xmm0
+; SSSE3-NEXT:    movdqa %xmm5, %xmm1
+; SSSE3-NEXT:    pxor %xmm3, %xmm1
+; SSSE3-NEXT:    movdqa %xmm10, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm1
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm6, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    por %xmm1, %xmm4
+; SSSE3-NEXT:    pand %xmm4, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    por %xmm1, %xmm2
+; SSSE3-NEXT:    pand %xmm2, %xmm5
+; SSSE3-NEXT:    pandn %xmm8, %xmm2
+; SSSE3-NEXT:    por %xmm5, %xmm2
+; SSSE3-NEXT:    pxor %xmm9, %xmm3
+; SSSE3-NEXT:    movdqa %xmm10, %xmm1
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
 ; SSSE3-NEXT:    pand %xmm4, %xmm3
-; SSSE3-NEXT:    pandn %xmm8, %xmm4
-; SSSE3-NEXT:    por %xmm3, %xmm4
-; SSSE3-NEXT:    pxor %xmm2, %xmm5
-; SSSE3-NEXT:    movdqa %xmm9, %xmm1
-; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm5
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm3, %xmm5
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    por %xmm5, %xmm1
-; SSSE3-NEXT:    pand %xmm1, %xmm2
+; SSSE3-NEXT:    por %xmm3, %xmm1
+; SSSE3-NEXT:    pand %xmm1, %xmm9
 ; SSSE3-NEXT:    pandn %xmm8, %xmm1
-; SSSE3-NEXT:    por %xmm2, %xmm1
-; SSSE3-NEXT:    packuswb %xmm4, %xmm1
+; SSSE3-NEXT:    por %xmm9, %xmm1
+; SSSE3-NEXT:    packuswb %xmm2, %xmm1
 ; SSSE3-NEXT:    packuswb %xmm1, %xmm0
+; SSSE3-NEXT:    packuswb %xmm0, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: trunc_usat_v8i64_v8i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa %xmm0, %xmm8
-; SSE41-NEXT:    movapd {{.*#+}} xmm5 = [255,255]
-; SSE41-NEXT:    movdqa {{.*#+}} xmm6 = [9223372039002259456,9223372039002259456]
+; SSE41-NEXT:    movdqa (%rdi), %xmm7
+; SSE41-NEXT:    movdqa 16(%rdi), %xmm1
+; SSE41-NEXT:    movdqa 32(%rdi), %xmm8
+; SSE41-NEXT:    movdqa 48(%rdi), %xmm9
+; SSE41-NEXT:    movapd {{.*#+}} xmm2 = [255,255]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm5 = [9223372039002259456,9223372039002259456]
 ; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    pxor %xmm6, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm9 = [9223372039002259711,9223372039002259711]
-; SSE41-NEXT:    movdqa %xmm9, %xmm7
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm7
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm7[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm9, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm4, %xmm0
-; SSE41-NEXT:    por %xmm7, %xmm0
-; SSE41-NEXT:    movapd %xmm5, %xmm4
-; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm4
-; SSE41-NEXT:    movdqa %xmm8, %xmm0
-; SSE41-NEXT:    pxor %xmm6, %xmm0
-; SSE41-NEXT:    movdqa %xmm9, %xmm1
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm1[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm9, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm7, %xmm0
-; SSE41-NEXT:    por %xmm1, %xmm0
-; SSE41-NEXT:    movapd %xmm5, %xmm1
-; SSE41-NEXT:    blendvpd %xmm0, %xmm8, %xmm1
-; SSE41-NEXT:    packusdw %xmm4, %xmm1
-; SSE41-NEXT:    movdqa %xmm3, %xmm0
-; SSE41-NEXT:    pxor %xmm6, %xmm0
-; SSE41-NEXT:    movdqa %xmm9, %xmm4
+; SSE41-NEXT:    pxor %xmm5, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [9223372039002259711,9223372039002259711]
+; SSE41-NEXT:    movdqa %xmm3, %xmm6
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm6
+; SSE41-NEXT:    movdqa %xmm3, %xmm4
 ; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm9, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm7, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pand %xmm6, %xmm0
 ; SSE41-NEXT:    por %xmm4, %xmm0
-; SSE41-NEXT:    movapd %xmm5, %xmm4
-; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm4
-; SSE41-NEXT:    pxor %xmm2, %xmm6
-; SSE41-NEXT:    movdqa %xmm9, %xmm3
-; SSE41-NEXT:    pcmpgtd %xmm6, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm9, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3]
-; SSE41-NEXT:    pand %xmm7, %xmm0
+; SSE41-NEXT:    movapd %xmm2, %xmm4
+; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm4
+; SSE41-NEXT:    movdqa %xmm7, %xmm0
+; SSE41-NEXT:    pxor %xmm5, %xmm0
+; SSE41-NEXT:    movdqa %xmm3, %xmm1
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
+; SSE41-NEXT:    movdqa %xmm3, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
+; SSE41-NEXT:    pand %xmm1, %xmm0
+; SSE41-NEXT:    por %xmm6, %xmm0
+; SSE41-NEXT:    movapd %xmm2, %xmm1
+; SSE41-NEXT:    blendvpd %xmm0, %xmm7, %xmm1
+; SSE41-NEXT:    packusdw %xmm4, %xmm1
+; SSE41-NEXT:    movdqa %xmm9, %xmm0
+; SSE41-NEXT:    pxor %xmm5, %xmm0
+; SSE41-NEXT:    movdqa %xmm3, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT:    movdqa %xmm3, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm6, %xmm0
+; SSE41-NEXT:    movapd %xmm2, %xmm4
+; SSE41-NEXT:    blendvpd %xmm0, %xmm9, %xmm4
+; SSE41-NEXT:    pxor %xmm8, %xmm5
+; SSE41-NEXT:    movdqa %xmm3, %xmm6
+; SSE41-NEXT:    pcmpeqd %xmm5, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm5, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pand %xmm6, %xmm0
 ; SSE41-NEXT:    por %xmm3, %xmm0
-; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm5
-; SSE41-NEXT:    packusdw %xmm4, %xmm5
-; SSE41-NEXT:    packusdw %xmm5, %xmm1
+; SSE41-NEXT:    blendvpd %xmm0, %xmm8, %xmm2
+; SSE41-NEXT:    packusdw %xmm4, %xmm2
+; SSE41-NEXT:    packusdw %xmm2, %xmm1
+; SSE41-NEXT:    packuswb %xmm1, %xmm1
 ; SSE41-NEXT:    movdqa %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: trunc_usat_v8i64_v8i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
-; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm3
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [9223372036854776063,9223372036854776063]
-; AVX1-NEXT:    vpcmpgtq %xmm3, %xmm4, %xmm8
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm5
-; AVX1-NEXT:    vpxor %xmm2, %xmm5, %xmm6
-; AVX1-NEXT:    vpcmpgtq %xmm6, %xmm4, %xmm6
-; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm7
-; AVX1-NEXT:    vpcmpgtq %xmm7, %xmm4, %xmm7
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
-; AVX1-NEXT:    vpxor %xmm2, %xmm3, %xmm2
-; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm4, %xmm2
-; AVX1-NEXT:    vmovapd {{.*#+}} xmm4 = [255,255]
-; AVX1-NEXT:    vblendvpd %xmm2, %xmm3, %xmm4, %xmm2
-; AVX1-NEXT:    vblendvpd %xmm7, %xmm1, %xmm4, %xmm1
-; AVX1-NEXT:    vpackusdw %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vblendvpd %xmm6, %xmm5, %xmm4, %xmm2
-; AVX1-NEXT:    vblendvpd %xmm8, %xmm0, %xmm4, %xmm0
-; AVX1-NEXT:    vpackusdw %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX1-NEXT:    vmovdqa 16(%rdi), %xmm1
+; AVX1-NEXT:    vmovdqa 32(%rdi), %xmm2
+; AVX1-NEXT:    vmovdqa 48(%rdi), %xmm3
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [9223372036854775808,9223372036854775808]
+; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm5
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [9223372036854776063,9223372036854776063]
+; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm6, %xmm8
+; AVX1-NEXT:    vpxor %xmm4, %xmm1, %xmm7
+; AVX1-NEXT:    vpcmpgtq %xmm7, %xmm6, %xmm7
+; AVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm5
+; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm6, %xmm5
+; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm4
+; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm6, %xmm4
+; AVX1-NEXT:    vmovapd {{.*#+}} xmm6 = [255,255]
+; AVX1-NEXT:    vblendvpd %xmm4, %xmm3, %xmm6, %xmm3
+; AVX1-NEXT:    vblendvpd %xmm5, %xmm2, %xmm6, %xmm2
+; AVX1-NEXT:    vpackusdw %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vblendvpd %xmm7, %xmm1, %xmm6, %xmm1
+; AVX1-NEXT:    vblendvpd %xmm8, %xmm0, %xmm6, %xmm0
 ; AVX1-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    vpackusdw %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: trunc_usat_v8i64_v8i8:
 ; AVX2:       # %bb.0:
+; AVX2-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX2-NEXT:    vmovdqa 32(%rdi), %ymm1
 ; AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm2 = [255,255,255,255]
 ; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
-; AVX2-NEXT:    vpxor %ymm3, %ymm1, %ymm4
+; AVX2-NEXT:    vpxor %ymm3, %ymm0, %ymm4
 ; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm5 = [9223372036854776063,9223372036854776063,9223372036854776063,9223372036854776063]
 ; AVX2-NEXT:    vpcmpgtq %ymm4, %ymm5, %ymm4
-; AVX2-NEXT:    vblendvpd %ymm4, %ymm1, %ymm2, %ymm1
-; AVX2-NEXT:    vpxor %ymm3, %ymm0, %ymm3
+; AVX2-NEXT:    vblendvpd %ymm4, %ymm0, %ymm2, %ymm0
+; AVX2-NEXT:    vpxor %ymm3, %ymm1, %ymm3
 ; AVX2-NEXT:    vpcmpgtq %ymm3, %ymm5, %ymm3
-; AVX2-NEXT:    vblendvpd %ymm3, %ymm0, %ymm2, %ymm0
-; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vblendvpd %ymm3, %ymm1, %ymm2, %ymm1
+; AVX2-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm3 = <u,u,0,8,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX2-NEXT:    vpshufb %xmm3, %xmm2, %xmm2
+; AVX2-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
+; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm3 = <0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX2-NEXT:    vpshufb %xmm3, %xmm2, %xmm2
+; AVX2-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
+; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3]
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: trunc_usat_v8i64_v8i8:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpminuq {{.*}}(%rip){1to8}, %zmm0, %zmm0
-; AVX512-NEXT:    vpmovqw %zmm0, %xmm0
+; AVX512-NEXT:    vmovdqa64 (%rdi), %zmm0
+; AVX512-NEXT:    vpmovusqb %zmm0, %xmm0
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
+;
+; SKX-LABEL: trunc_usat_v8i64_v8i8:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vmovdqa64 (%rdi), %zmm0
+; SKX-NEXT:    vpmovusqb %zmm0, %xmm0
+; SKX-NEXT:    vzeroupper
+; SKX-NEXT:    retq
+  %a0 = load <8 x i64>, <8 x i64>* %p0
   %1 = icmp ult <8 x i64> %a0, <i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255>
   %2 = select <8 x i1> %1, <8 x i64> %a0, <8 x i64> <i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255>
   %3 = trunc <8 x i64> %2 to <8 x i8>
   ret <8 x i8> %3
 }
 
-define void @trunc_usat_v8i64_v8i8_store(<8 x i64> %a0, <8 x i8> *%p1) {
+define void @trunc_usat_v8i64_v8i8_store(<8 x i64>* %p0, <8 x i8> *%p1) {
 ; SSE2-LABEL: trunc_usat_v8i64_v8i8_store:
 ; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa (%rdi), %xmm6
+; SSE2-NEXT:    movdqa 16(%rdi), %xmm5
+; SSE2-NEXT:    movdqa 32(%rdi), %xmm9
+; SSE2-NEXT:    movdqa 48(%rdi), %xmm4
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [255,255]
-; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [9223372039002259456,9223372039002259456]
-; SSE2-NEXT:    movdqa %xmm1, %xmm7
-; SSE2-NEXT:    pxor %xmm5, %xmm7
-; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [9223372039002259711,9223372039002259711]
-; SSE2-NEXT:    movdqa %xmm9, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm7, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm7
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [9223372039002259456,9223372039002259456]
+; SSE2-NEXT:    movdqa %xmm5, %xmm7
+; SSE2-NEXT:    pxor %xmm2, %xmm7
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [9223372039002259711,9223372039002259711]
+; SSE2-NEXT:    movdqa %xmm1, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm7, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm1, %xmm7
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT:    pand %xmm4, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[1,1,3,3]
-; SSE2-NEXT:    por %xmm7, %xmm4
-; SSE2-NEXT:    pand %xmm4, %xmm1
-; SSE2-NEXT:    pandn %xmm8, %xmm4
-; SSE2-NEXT:    por %xmm1, %xmm4
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    pxor %xmm5, %xmm1
-; SSE2-NEXT:    movdqa %xmm9, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pand %xmm10, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm6[1,1,3,3]
-; SSE2-NEXT:    por %xmm7, %xmm1
-; SSE2-NEXT:    pand %xmm1, %xmm0
-; SSE2-NEXT:    pandn %xmm8, %xmm1
-; SSE2-NEXT:    por %xmm0, %xmm1
-; SSE2-NEXT:    packuswb %xmm4, %xmm1
-; SSE2-NEXT:    movdqa %xmm3, %xmm0
-; SSE2-NEXT:    pxor %xmm5, %xmm0
-; SSE2-NEXT:    movdqa %xmm9, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm0
+; SSE2-NEXT:    pand %xmm0, %xmm7
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
+; SSE2-NEXT:    por %xmm7, %xmm0
+; SSE2-NEXT:    pand %xmm0, %xmm5
+; SSE2-NEXT:    pandn %xmm8, %xmm0
+; SSE2-NEXT:    por %xmm5, %xmm0
+; SSE2-NEXT:    movdqa %xmm6, %xmm3
+; SSE2-NEXT:    pxor %xmm2, %xmm3
+; SSE2-NEXT:    movdqa %xmm1, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm5[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm1, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSE2-NEXT:    pand %xmm7, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSE2-NEXT:    por %xmm3, %xmm5
+; SSE2-NEXT:    pand %xmm5, %xmm6
+; SSE2-NEXT:    pandn %xmm8, %xmm5
+; SSE2-NEXT:    por %xmm6, %xmm5
+; SSE2-NEXT:    packuswb %xmm0, %xmm5
+; SSE2-NEXT:    movdqa %xmm4, %xmm0
+; SSE2-NEXT:    pxor %xmm2, %xmm0
+; SSE2-NEXT:    movdqa %xmm1, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm6, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm0, %xmm4
-; SSE2-NEXT:    pand %xmm4, %xmm3
-; SSE2-NEXT:    pandn %xmm8, %xmm4
-; SSE2-NEXT:    por %xmm3, %xmm4
-; SSE2-NEXT:    pxor %xmm2, %xmm5
-; SSE2-NEXT:    movdqa %xmm9, %xmm0
-; SSE2-NEXT:    pcmpgtd %xmm5, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE2-NEXT:    pand %xmm3, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSE2-NEXT:    por %xmm0, %xmm3
+; SSE2-NEXT:    pand %xmm3, %xmm4
+; SSE2-NEXT:    pandn %xmm8, %xmm3
+; SSE2-NEXT:    por %xmm4, %xmm3
+; SSE2-NEXT:    pxor %xmm9, %xmm2
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm1, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,3,3]
+; SSE2-NEXT:    pand %xmm4, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm5, %xmm0
-; SSE2-NEXT:    pand %xmm0, %xmm2
+; SSE2-NEXT:    por %xmm1, %xmm0
+; SSE2-NEXT:    pand %xmm0, %xmm9
 ; SSE2-NEXT:    pandn %xmm8, %xmm0
-; SSE2-NEXT:    por %xmm2, %xmm0
-; SSE2-NEXT:    packuswb %xmm4, %xmm0
-; SSE2-NEXT:    packuswb %xmm0, %xmm1
-; SSE2-NEXT:    packuswb %xmm1, %xmm1
-; SSE2-NEXT:    movq %xmm1, (%rdi)
+; SSE2-NEXT:    por %xmm9, %xmm0
+; SSE2-NEXT:    packuswb %xmm3, %xmm0
+; SSE2-NEXT:    packuswb %xmm0, %xmm5
+; SSE2-NEXT:    packuswb %xmm0, %xmm5
+; SSE2-NEXT:    movq %xmm5, (%rsi)
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: trunc_usat_v8i64_v8i8_store:
 ; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa (%rdi), %xmm6
+; SSSE3-NEXT:    movdqa 16(%rdi), %xmm5
+; SSSE3-NEXT:    movdqa 32(%rdi), %xmm9
+; SSSE3-NEXT:    movdqa 48(%rdi), %xmm4
 ; SSSE3-NEXT:    movdqa {{.*#+}} xmm8 = [255,255]
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm5 = [9223372039002259456,9223372039002259456]
-; SSSE3-NEXT:    movdqa %xmm1, %xmm7
-; SSSE3-NEXT:    pxor %xmm5, %xmm7
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm9 = [9223372039002259711,9223372039002259711]
-; SSSE3-NEXT:    movdqa %xmm9, %xmm6
-; SSSE3-NEXT:    pcmpgtd %xmm7, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm7
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [9223372039002259456,9223372039002259456]
+; SSSE3-NEXT:    movdqa %xmm5, %xmm7
+; SSSE3-NEXT:    pxor %xmm2, %xmm7
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [9223372039002259711,9223372039002259711]
+; SSSE3-NEXT:    movdqa %xmm1, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm7, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm7
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm4, %xmm7
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[1,1,3,3]
-; SSSE3-NEXT:    por %xmm7, %xmm4
-; SSSE3-NEXT:    pand %xmm4, %xmm1
-; SSSE3-NEXT:    pandn %xmm8, %xmm4
-; SSSE3-NEXT:    por %xmm1, %xmm4
-; SSSE3-NEXT:    movdqa %xmm0, %xmm1
-; SSSE3-NEXT:    pxor %xmm5, %xmm1
-; SSSE3-NEXT:    movdqa %xmm9, %xmm6
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm10 = xmm6[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm10, %xmm7
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm6[1,1,3,3]
-; SSSE3-NEXT:    por %xmm7, %xmm1
-; SSSE3-NEXT:    pand %xmm1, %xmm0
-; SSSE3-NEXT:    pandn %xmm8, %xmm1
-; SSSE3-NEXT:    por %xmm0, %xmm1
-; SSSE3-NEXT:    packuswb %xmm4, %xmm1
-; SSSE3-NEXT:    movdqa %xmm3, %xmm0
-; SSSE3-NEXT:    pxor %xmm5, %xmm0
-; SSSE3-NEXT:    movdqa %xmm9, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm0
+; SSSE3-NEXT:    pand %xmm0, %xmm7
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
+; SSSE3-NEXT:    por %xmm7, %xmm0
+; SSSE3-NEXT:    pand %xmm0, %xmm5
+; SSSE3-NEXT:    pandn %xmm8, %xmm0
+; SSSE3-NEXT:    por %xmm5, %xmm0
+; SSSE3-NEXT:    movdqa %xmm6, %xmm3
+; SSSE3-NEXT:    pxor %xmm2, %xmm3
+; SSSE3-NEXT:    movdqa %xmm1, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm5[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm7, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSSE3-NEXT:    por %xmm3, %xmm5
+; SSSE3-NEXT:    pand %xmm5, %xmm6
+; SSSE3-NEXT:    pandn %xmm8, %xmm5
+; SSSE3-NEXT:    por %xmm6, %xmm5
+; SSSE3-NEXT:    packuswb %xmm0, %xmm5
+; SSSE3-NEXT:    movdqa %xmm4, %xmm0
+; SSSE3-NEXT:    pxor %xmm2, %xmm0
+; SSSE3-NEXT:    movdqa %xmm1, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm0
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
 ; SSSE3-NEXT:    pand %xmm6, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    por %xmm0, %xmm4
-; SSSE3-NEXT:    pand %xmm4, %xmm3
-; SSSE3-NEXT:    pandn %xmm8, %xmm4
-; SSSE3-NEXT:    por %xmm3, %xmm4
-; SSSE3-NEXT:    pxor %xmm2, %xmm5
-; SSSE3-NEXT:    movdqa %xmm9, %xmm0
-; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm5
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm3, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSSE3-NEXT:    por %xmm0, %xmm3
+; SSSE3-NEXT:    pand %xmm3, %xmm4
+; SSSE3-NEXT:    pandn %xmm8, %xmm3
+; SSSE3-NEXT:    por %xmm4, %xmm3
+; SSSE3-NEXT:    pxor %xmm9, %xmm2
+; SSSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm4, %xmm1
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    por %xmm5, %xmm0
-; SSSE3-NEXT:    pand %xmm0, %xmm2
+; SSSE3-NEXT:    por %xmm1, %xmm0
+; SSSE3-NEXT:    pand %xmm0, %xmm9
 ; SSSE3-NEXT:    pandn %xmm8, %xmm0
-; SSSE3-NEXT:    por %xmm2, %xmm0
-; SSSE3-NEXT:    packuswb %xmm4, %xmm0
-; SSSE3-NEXT:    packuswb %xmm0, %xmm1
-; SSSE3-NEXT:    packuswb %xmm1, %xmm1
-; SSSE3-NEXT:    movq %xmm1, (%rdi)
+; SSSE3-NEXT:    por %xmm9, %xmm0
+; SSSE3-NEXT:    packuswb %xmm3, %xmm0
+; SSSE3-NEXT:    packuswb %xmm0, %xmm5
+; SSSE3-NEXT:    packuswb %xmm0, %xmm5
+; SSSE3-NEXT:    movq %xmm5, (%rsi)
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: trunc_usat_v8i64_v8i8_store:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa %xmm0, %xmm8
-; SSE41-NEXT:    movapd {{.*#+}} xmm5 = [255,255]
-; SSE41-NEXT:    movdqa {{.*#+}} xmm6 = [9223372039002259456,9223372039002259456]
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    pxor %xmm6, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm9 = [9223372039002259711,9223372039002259711]
-; SSE41-NEXT:    movdqa %xmm9, %xmm7
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm7
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm7[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm9, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm4, %xmm0
-; SSE41-NEXT:    por %xmm7, %xmm0
-; SSE41-NEXT:    movapd %xmm5, %xmm4
-; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm4
-; SSE41-NEXT:    movdqa %xmm8, %xmm0
-; SSE41-NEXT:    pxor %xmm6, %xmm0
-; SSE41-NEXT:    movdqa %xmm9, %xmm1
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm1[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm9, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm7, %xmm0
-; SSE41-NEXT:    por %xmm1, %xmm0
-; SSE41-NEXT:    movapd %xmm5, %xmm1
-; SSE41-NEXT:    blendvpd %xmm0, %xmm8, %xmm1
-; SSE41-NEXT:    packusdw %xmm4, %xmm1
-; SSE41-NEXT:    movdqa %xmm3, %xmm0
-; SSE41-NEXT:    pxor %xmm6, %xmm0
-; SSE41-NEXT:    movdqa %xmm9, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm9, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm7, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
-; SSE41-NEXT:    movapd %xmm5, %xmm4
-; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm4
-; SSE41-NEXT:    pxor %xmm2, %xmm6
-; SSE41-NEXT:    movdqa %xmm9, %xmm3
-; SSE41-NEXT:    pcmpgtd %xmm6, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm9, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3]
-; SSE41-NEXT:    pand %xmm7, %xmm0
+; SSE41-NEXT:    movdqa (%rdi), %xmm7
+; SSE41-NEXT:    movdqa 16(%rdi), %xmm6
+; SSE41-NEXT:    movdqa 32(%rdi), %xmm8
+; SSE41-NEXT:    movdqa 48(%rdi), %xmm9
+; SSE41-NEXT:    movapd {{.*#+}} xmm1 = [255,255]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [9223372039002259456,9223372039002259456]
+; SSE41-NEXT:    movdqa %xmm6, %xmm0
+; SSE41-NEXT:    pxor %xmm4, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [9223372039002259711,9223372039002259711]
+; SSE41-NEXT:    movdqa %xmm2, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm5
+; SSE41-NEXT:    movdqa %xmm2, %xmm3
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm0
 ; SSE41-NEXT:    por %xmm3, %xmm0
-; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm5
-; SSE41-NEXT:    packusdw %xmm4, %xmm5
-; SSE41-NEXT:    packusdw %xmm5, %xmm1
-; SSE41-NEXT:    packuswb %xmm1, %xmm1
-; SSE41-NEXT:    movq %xmm1, (%rdi)
+; SSE41-NEXT:    movapd %xmm1, %xmm3
+; SSE41-NEXT:    blendvpd %xmm0, %xmm6, %xmm3
+; SSE41-NEXT:    movdqa %xmm7, %xmm0
+; SSE41-NEXT:    pxor %xmm4, %xmm0
+; SSE41-NEXT:    movdqa %xmm2, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm5
+; SSE41-NEXT:    movdqa %xmm2, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm0
+; SSE41-NEXT:    por %xmm6, %xmm0
+; SSE41-NEXT:    movapd %xmm1, %xmm6
+; SSE41-NEXT:    blendvpd %xmm0, %xmm7, %xmm6
+; SSE41-NEXT:    packusdw %xmm3, %xmm6
+; SSE41-NEXT:    movdqa %xmm9, %xmm0
+; SSE41-NEXT:    pxor %xmm4, %xmm0
+; SSE41-NEXT:    movdqa %xmm2, %xmm3
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm3
+; SSE41-NEXT:    movdqa %xmm2, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm5
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2]
+; SSE41-NEXT:    pand %xmm3, %xmm0
+; SSE41-NEXT:    por %xmm5, %xmm0
+; SSE41-NEXT:    movapd %xmm1, %xmm3
+; SSE41-NEXT:    blendvpd %xmm0, %xmm9, %xmm3
+; SSE41-NEXT:    pxor %xmm8, %xmm4
+; SSE41-NEXT:    movdqa %xmm2, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm4, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm4, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm0
+; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm8, %xmm1
+; SSE41-NEXT:    packusdw %xmm3, %xmm1
+; SSE41-NEXT:    packusdw %xmm1, %xmm6
+; SSE41-NEXT:    packuswb %xmm0, %xmm6
+; SSE41-NEXT:    movq %xmm6, (%rsi)
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: trunc_usat_v8i64_v8i8_store:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
-; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm3
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [9223372036854776063,9223372036854776063]
-; AVX1-NEXT:    vpcmpgtq %xmm3, %xmm4, %xmm8
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm5
-; AVX1-NEXT:    vpxor %xmm2, %xmm5, %xmm6
-; AVX1-NEXT:    vpcmpgtq %xmm6, %xmm4, %xmm6
-; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm7
-; AVX1-NEXT:    vpcmpgtq %xmm7, %xmm4, %xmm7
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
-; AVX1-NEXT:    vpxor %xmm2, %xmm3, %xmm2
-; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm4, %xmm2
-; AVX1-NEXT:    vmovapd {{.*#+}} xmm4 = [255,255]
-; AVX1-NEXT:    vblendvpd %xmm2, %xmm3, %xmm4, %xmm2
-; AVX1-NEXT:    vblendvpd %xmm7, %xmm1, %xmm4, %xmm1
-; AVX1-NEXT:    vpackusdw %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vblendvpd %xmm6, %xmm5, %xmm4, %xmm2
-; AVX1-NEXT:    vblendvpd %xmm8, %xmm0, %xmm4, %xmm0
-; AVX1-NEXT:    vpackusdw %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX1-NEXT:    vmovdqa 16(%rdi), %xmm1
+; AVX1-NEXT:    vmovdqa 32(%rdi), %xmm2
+; AVX1-NEXT:    vmovdqa 48(%rdi), %xmm3
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [9223372036854775808,9223372036854775808]
+; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm5
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [9223372036854776063,9223372036854776063]
+; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm6, %xmm8
+; AVX1-NEXT:    vpxor %xmm4, %xmm1, %xmm7
+; AVX1-NEXT:    vpcmpgtq %xmm7, %xmm6, %xmm7
+; AVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm5
+; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm6, %xmm5
+; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm4
+; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm6, %xmm4
+; AVX1-NEXT:    vmovapd {{.*#+}} xmm6 = [255,255]
+; AVX1-NEXT:    vblendvpd %xmm4, %xmm3, %xmm6, %xmm3
+; AVX1-NEXT:    vblendvpd %xmm5, %xmm2, %xmm6, %xmm2
+; AVX1-NEXT:    vpackusdw %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vblendvpd %xmm7, %xmm1, %xmm6, %xmm1
+; AVX1-NEXT:    vblendvpd %xmm8, %xmm0, %xmm6, %xmm0
 ; AVX1-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpackusdw %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
-; AVX1-NEXT:    vmovq %xmm0, (%rdi)
-; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    vmovq %xmm0, (%rsi)
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: trunc_usat_v8i64_v8i8_store:
 ; AVX2:       # %bb.0:
+; AVX2-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX2-NEXT:    vmovdqa 32(%rdi), %ymm1
 ; AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm2 = [255,255,255,255]
 ; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
 ; AVX2-NEXT:    vpxor %ymm3, %ymm0, %ymm4
@@ -1399,23 +3129,34 @@ define void @trunc_usat_v8i64_v8i8_store(<8 x i64> %a0, <8 x i8> *%p1) {
 ; AVX2-NEXT:    vpcmpgtq %ymm3, %ymm5, %ymm3
 ; AVX2-NEXT:    vblendvpd %ymm3, %ymm1, %ymm2, %ymm1
 ; AVX2-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX2-NEXT:    vpackusdw %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
-; AVX2-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm3
-; AVX2-NEXT:    vpackusdw %xmm3, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; AVX2-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
-; AVX2-NEXT:    vmovq %xmm0, (%rdi)
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm3 = <u,u,0,8,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX2-NEXT:    vpshufb %xmm3, %xmm2, %xmm2
+; AVX2-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
+; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm3 = <0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX2-NEXT:    vpshufb %xmm3, %xmm2, %xmm2
+; AVX2-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
+; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3]
+; AVX2-NEXT:    vmovq %xmm0, (%rsi)
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: trunc_usat_v8i64_v8i8_store:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpmovusqb %zmm0, (%rdi)
+; AVX512-NEXT:    vmovdqa64 (%rdi), %zmm0
+; AVX512-NEXT:    vpmovusqb %zmm0, (%rsi)
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
+;
+; SKX-LABEL: trunc_usat_v8i64_v8i8_store:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vmovdqa64 (%rdi), %zmm0
+; SKX-NEXT:    vpmovusqb %zmm0, (%rsi)
+; SKX-NEXT:    vzeroupper
+; SKX-NEXT:    retq
+  %a0 = load <8 x i64>, <8 x i64>* %p0
   %1 = icmp ult <8 x i64> %a0, <i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255>
   %2 = select <8 x i1> %1, <8 x i64> %a0, <8 x i64> <i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255>
   %3 = trunc <8 x i64> %2 to <8 x i8>
@@ -1423,119 +3164,127 @@ define void @trunc_usat_v8i64_v8i8_store(<8 x i64> %a0, <8 x i8> *%p1) {
   ret void
 }
 
-define <16 x i8> @trunc_usat_v16i64_v16i8(<16 x i64> %a0) {
+define <16 x i8> @trunc_usat_v16i64_v16i8(<16 x i64>* %p0) {
 ; SSE2-LABEL: trunc_usat_v16i64_v16i8:
 ; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa 96(%rdi), %xmm9
+; SSE2-NEXT:    movdqa 112(%rdi), %xmm10
+; SSE2-NEXT:    movdqa 64(%rdi), %xmm11
+; SSE2-NEXT:    movdqa 80(%rdi), %xmm12
+; SSE2-NEXT:    movdqa (%rdi), %xmm3
+; SSE2-NEXT:    movdqa 16(%rdi), %xmm6
+; SSE2-NEXT:    movdqa 32(%rdi), %xmm13
+; SSE2-NEXT:    movdqa 48(%rdi), %xmm1
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [255,255]
-; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [9223372039002259456,9223372039002259456]
-; SSE2-NEXT:    movdqa %xmm1, %xmm11
-; SSE2-NEXT:    pxor %xmm9, %xmm11
-; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [9223372039002259711,9223372039002259711]
-; SSE2-NEXT:    movdqa %xmm10, %xmm12
-; SSE2-NEXT:    pcmpgtd %xmm11, %xmm12
-; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm12[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm10, %xmm11
-; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm11[1,1,3,3]
-; SSE2-NEXT:    pand %xmm13, %xmm11
-; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm12[1,1,3,3]
-; SSE2-NEXT:    por %xmm11, %xmm12
-; SSE2-NEXT:    pand %xmm12, %xmm1
-; SSE2-NEXT:    pandn %xmm8, %xmm12
-; SSE2-NEXT:    por %xmm1, %xmm12
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    pxor %xmm9, %xmm1
-; SSE2-NEXT:    movdqa %xmm10, %xmm11
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm11
-; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm11[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm10, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm14 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pand %xmm13, %xmm14
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm11[1,1,3,3]
-; SSE2-NEXT:    por %xmm14, %xmm1
-; SSE2-NEXT:    pand %xmm1, %xmm0
-; SSE2-NEXT:    pandn %xmm8, %xmm1
-; SSE2-NEXT:    por %xmm1, %xmm0
-; SSE2-NEXT:    packuswb %xmm12, %xmm0
-; SSE2-NEXT:    movdqa %xmm3, %xmm1
-; SSE2-NEXT:    pxor %xmm9, %xmm1
-; SSE2-NEXT:    movdqa %xmm10, %xmm11
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm11
-; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm11[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm10, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pand %xmm12, %xmm13
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm11[1,1,3,3]
-; SSE2-NEXT:    por %xmm13, %xmm1
-; SSE2-NEXT:    pand %xmm1, %xmm3
-; SSE2-NEXT:    pandn %xmm8, %xmm1
-; SSE2-NEXT:    por %xmm3, %xmm1
-; SSE2-NEXT:    movdqa %xmm2, %xmm3
-; SSE2-NEXT:    pxor %xmm9, %xmm3
-; SSE2-NEXT:    movdqa %xmm10, %xmm11
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm11
-; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm11[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm10, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm3[1,1,3,3]
-; SSE2-NEXT:    pand %xmm12, %xmm13
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm11[1,1,3,3]
-; SSE2-NEXT:    por %xmm13, %xmm3
-; SSE2-NEXT:    pand %xmm3, %xmm2
-; SSE2-NEXT:    pandn %xmm8, %xmm3
+; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [9223372039002259456,9223372039002259456]
+; SSE2-NEXT:    movdqa %xmm6, %xmm0
+; SSE2-NEXT:    pxor %xmm4, %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm14 = [9223372039002259711,9223372039002259711]
+; SSE2-NEXT:    movdqa %xmm14, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm5[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm14, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm5[1,1,3,3]
+; SSE2-NEXT:    por %xmm0, %xmm2
+; SSE2-NEXT:    pand %xmm2, %xmm6
+; SSE2-NEXT:    pandn %xmm8, %xmm2
+; SSE2-NEXT:    por %xmm6, %xmm2
+; SSE2-NEXT:    movdqa %xmm3, %xmm0
+; SSE2-NEXT:    pxor %xmm4, %xmm0
+; SSE2-NEXT:    movdqa %xmm14, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm14, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pand %xmm6, %xmm7
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[1,1,3,3]
+; SSE2-NEXT:    por %xmm7, %xmm0
+; SSE2-NEXT:    pand %xmm0, %xmm3
+; SSE2-NEXT:    pandn %xmm8, %xmm0
+; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    packuswb %xmm2, %xmm0
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    pxor %xmm4, %xmm2
+; SSE2-NEXT:    movdqa %xmm14, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm14, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE2-NEXT:    pand %xmm5, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
 ; SSE2-NEXT:    por %xmm2, %xmm3
-; SSE2-NEXT:    packuswb %xmm1, %xmm3
-; SSE2-NEXT:    packuswb %xmm3, %xmm0
-; SSE2-NEXT:    movdqa %xmm5, %xmm1
-; SSE2-NEXT:    pxor %xmm9, %xmm1
-; SSE2-NEXT:    movdqa %xmm10, %xmm2
+; SSE2-NEXT:    pand %xmm3, %xmm1
+; SSE2-NEXT:    pandn %xmm8, %xmm3
+; SSE2-NEXT:    por %xmm1, %xmm3
+; SSE2-NEXT:    movdqa %xmm13, %xmm1
+; SSE2-NEXT:    pxor %xmm4, %xmm1
+; SSE2-NEXT:    movdqa %xmm14, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm14, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pand %xmm5, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE2-NEXT:    por %xmm1, %xmm2
+; SSE2-NEXT:    pand %xmm2, %xmm13
+; SSE2-NEXT:    pandn %xmm8, %xmm2
+; SSE2-NEXT:    por %xmm13, %xmm2
+; SSE2-NEXT:    packuswb %xmm3, %xmm2
+; SSE2-NEXT:    packuswb %xmm2, %xmm0
+; SSE2-NEXT:    movdqa %xmm12, %xmm1
+; SSE2-NEXT:    pxor %xmm4, %xmm1
+; SSE2-NEXT:    movdqa %xmm14, %xmm2
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm10, %xmm1
+; SSE2-NEXT:    pcmpeqd %xmm14, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm3, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
 ; SSE2-NEXT:    por %xmm1, %xmm2
-; SSE2-NEXT:    pand %xmm2, %xmm5
+; SSE2-NEXT:    pand %xmm2, %xmm12
 ; SSE2-NEXT:    pandn %xmm8, %xmm2
-; SSE2-NEXT:    por %xmm5, %xmm2
-; SSE2-NEXT:    movdqa %xmm4, %xmm1
-; SSE2-NEXT:    pxor %xmm9, %xmm1
-; SSE2-NEXT:    movdqa %xmm10, %xmm3
+; SSE2-NEXT:    por %xmm12, %xmm2
+; SSE2-NEXT:    movdqa %xmm11, %xmm1
+; SSE2-NEXT:    pxor %xmm4, %xmm1
+; SSE2-NEXT:    movdqa %xmm14, %xmm3
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm3[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm10, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pand %xmm11, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm14, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pand %xmm5, %xmm6
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE2-NEXT:    por %xmm5, %xmm1
-; SSE2-NEXT:    pand %xmm1, %xmm4
+; SSE2-NEXT:    por %xmm6, %xmm1
+; SSE2-NEXT:    pand %xmm1, %xmm11
 ; SSE2-NEXT:    pandn %xmm8, %xmm1
-; SSE2-NEXT:    por %xmm4, %xmm1
+; SSE2-NEXT:    por %xmm11, %xmm1
 ; SSE2-NEXT:    packuswb %xmm2, %xmm1
-; SSE2-NEXT:    movdqa %xmm7, %xmm2
-; SSE2-NEXT:    pxor %xmm9, %xmm2
-; SSE2-NEXT:    movdqa %xmm10, %xmm3
+; SSE2-NEXT:    movdqa %xmm10, %xmm2
+; SSE2-NEXT:    pxor %xmm4, %xmm2
+; SSE2-NEXT:    movdqa %xmm14, %xmm3
 ; SSE2-NEXT:    pcmpgtd %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm10, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm14, %xmm2
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT:    pand %xmm4, %xmm2
+; SSE2-NEXT:    pand %xmm5, %xmm2
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
 ; SSE2-NEXT:    por %xmm2, %xmm3
-; SSE2-NEXT:    pand %xmm3, %xmm7
+; SSE2-NEXT:    pand %xmm3, %xmm10
 ; SSE2-NEXT:    pandn %xmm8, %xmm3
-; SSE2-NEXT:    por %xmm7, %xmm3
-; SSE2-NEXT:    pxor %xmm6, %xmm9
-; SSE2-NEXT:    movdqa %xmm10, %xmm2
-; SSE2-NEXT:    pcmpgtd %xmm9, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm10, %xmm9
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm9[1,1,3,3]
-; SSE2-NEXT:    pand %xmm4, %xmm5
+; SSE2-NEXT:    por %xmm10, %xmm3
+; SSE2-NEXT:    pxor %xmm9, %xmm4
+; SSE2-NEXT:    movdqa %xmm14, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm14, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    pand %xmm5, %xmm4
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT:    por %xmm5, %xmm2
-; SSE2-NEXT:    pand %xmm2, %xmm6
+; SSE2-NEXT:    por %xmm4, %xmm2
+; SSE2-NEXT:    pand %xmm2, %xmm9
 ; SSE2-NEXT:    pandn %xmm8, %xmm2
-; SSE2-NEXT:    por %xmm6, %xmm2
+; SSE2-NEXT:    por %xmm9, %xmm2
 ; SSE2-NEXT:    packuswb %xmm3, %xmm2
 ; SSE2-NEXT:    packuswb %xmm2, %xmm1
 ; SSE2-NEXT:    packuswb %xmm1, %xmm0
@@ -1543,372 +3292,615 @@ define <16 x i8> @trunc_usat_v16i64_v16i8(<16 x i64> %a0) {
 ;
 ; SSSE3-LABEL: trunc_usat_v16i64_v16i8:
 ; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa 96(%rdi), %xmm9
+; SSSE3-NEXT:    movdqa 112(%rdi), %xmm10
+; SSSE3-NEXT:    movdqa 64(%rdi), %xmm11
+; SSSE3-NEXT:    movdqa 80(%rdi), %xmm12
+; SSSE3-NEXT:    movdqa (%rdi), %xmm3
+; SSSE3-NEXT:    movdqa 16(%rdi), %xmm6
+; SSSE3-NEXT:    movdqa 32(%rdi), %xmm13
+; SSSE3-NEXT:    movdqa 48(%rdi), %xmm1
 ; SSSE3-NEXT:    movdqa {{.*#+}} xmm8 = [255,255]
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm9 = [9223372039002259456,9223372039002259456]
-; SSSE3-NEXT:    movdqa %xmm1, %xmm11
-; SSSE3-NEXT:    pxor %xmm9, %xmm11
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm10 = [9223372039002259711,9223372039002259711]
-; SSSE3-NEXT:    movdqa %xmm10, %xmm12
-; SSSE3-NEXT:    pcmpgtd %xmm11, %xmm12
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm13 = xmm12[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm11
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm11 = xmm11[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm13, %xmm11
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm12 = xmm12[1,1,3,3]
-; SSSE3-NEXT:    por %xmm11, %xmm12
-; SSSE3-NEXT:    pand %xmm12, %xmm1
-; SSSE3-NEXT:    pandn %xmm8, %xmm12
-; SSSE3-NEXT:    por %xmm1, %xmm12
-; SSSE3-NEXT:    movdqa %xmm0, %xmm1
-; SSSE3-NEXT:    pxor %xmm9, %xmm1
-; SSSE3-NEXT:    movdqa %xmm10, %xmm11
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm11
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm13 = xmm11[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm14 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm13, %xmm14
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm11[1,1,3,3]
-; SSSE3-NEXT:    por %xmm14, %xmm1
-; SSSE3-NEXT:    pand %xmm1, %xmm0
-; SSSE3-NEXT:    pandn %xmm8, %xmm1
-; SSSE3-NEXT:    por %xmm1, %xmm0
-; SSSE3-NEXT:    packuswb %xmm12, %xmm0
-; SSSE3-NEXT:    movdqa %xmm3, %xmm1
-; SSSE3-NEXT:    pxor %xmm9, %xmm1
-; SSSE3-NEXT:    movdqa %xmm10, %xmm11
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm11
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm12 = xmm11[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm13 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm12, %xmm13
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm11[1,1,3,3]
-; SSSE3-NEXT:    por %xmm13, %xmm1
-; SSSE3-NEXT:    pand %xmm1, %xmm3
-; SSSE3-NEXT:    pandn %xmm8, %xmm1
-; SSSE3-NEXT:    por %xmm3, %xmm1
-; SSSE3-NEXT:    movdqa %xmm2, %xmm3
-; SSSE3-NEXT:    pxor %xmm9, %xmm3
-; SSSE3-NEXT:    movdqa %xmm10, %xmm11
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm11
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm12 = xmm11[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm13 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm12, %xmm13
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm11[1,1,3,3]
-; SSSE3-NEXT:    por %xmm13, %xmm3
-; SSSE3-NEXT:    pand %xmm3, %xmm2
-; SSSE3-NEXT:    pandn %xmm8, %xmm3
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm4 = [9223372039002259456,9223372039002259456]
+; SSSE3-NEXT:    movdqa %xmm6, %xmm0
+; SSSE3-NEXT:    pxor %xmm4, %xmm0
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm14 = [9223372039002259711,9223372039002259711]
+; SSSE3-NEXT:    movdqa %xmm14, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm5[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm14, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm2, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm5[1,1,3,3]
+; SSSE3-NEXT:    por %xmm0, %xmm2
+; SSSE3-NEXT:    pand %xmm2, %xmm6
+; SSSE3-NEXT:    pandn %xmm8, %xmm2
+; SSSE3-NEXT:    por %xmm6, %xmm2
+; SSSE3-NEXT:    movdqa %xmm3, %xmm0
+; SSSE3-NEXT:    pxor %xmm4, %xmm0
+; SSSE3-NEXT:    movdqa %xmm14, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm14, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm6, %xmm7
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[1,1,3,3]
+; SSSE3-NEXT:    por %xmm7, %xmm0
+; SSSE3-NEXT:    pand %xmm0, %xmm3
+; SSSE3-NEXT:    pandn %xmm8, %xmm0
+; SSSE3-NEXT:    por %xmm3, %xmm0
+; SSSE3-NEXT:    packuswb %xmm2, %xmm0
+; SSSE3-NEXT:    movdqa %xmm1, %xmm2
+; SSSE3-NEXT:    pxor %xmm4, %xmm2
+; SSSE3-NEXT:    movdqa %xmm14, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm14, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm5, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
 ; SSSE3-NEXT:    por %xmm2, %xmm3
-; SSSE3-NEXT:    packuswb %xmm1, %xmm3
-; SSSE3-NEXT:    packuswb %xmm3, %xmm0
-; SSSE3-NEXT:    movdqa %xmm5, %xmm1
-; SSSE3-NEXT:    pxor %xmm9, %xmm1
-; SSSE3-NEXT:    movdqa %xmm10, %xmm2
+; SSSE3-NEXT:    pand %xmm3, %xmm1
+; SSSE3-NEXT:    pandn %xmm8, %xmm3
+; SSSE3-NEXT:    por %xmm1, %xmm3
+; SSSE3-NEXT:    movdqa %xmm13, %xmm1
+; SSSE3-NEXT:    pxor %xmm4, %xmm1
+; SSSE3-NEXT:    movdqa %xmm14, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm14, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm5, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    por %xmm1, %xmm2
+; SSSE3-NEXT:    pand %xmm2, %xmm13
+; SSSE3-NEXT:    pandn %xmm8, %xmm2
+; SSSE3-NEXT:    por %xmm13, %xmm2
+; SSSE3-NEXT:    packuswb %xmm3, %xmm2
+; SSSE3-NEXT:    packuswb %xmm2, %xmm0
+; SSSE3-NEXT:    movdqa %xmm12, %xmm1
+; SSSE3-NEXT:    pxor %xmm4, %xmm1
+; SSSE3-NEXT:    movdqa %xmm14, %xmm2
 ; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm1
+; SSSE3-NEXT:    pcmpeqd %xmm14, %xmm1
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
 ; SSSE3-NEXT:    pand %xmm3, %xmm1
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
 ; SSSE3-NEXT:    por %xmm1, %xmm2
-; SSSE3-NEXT:    pand %xmm2, %xmm5
+; SSSE3-NEXT:    pand %xmm2, %xmm12
 ; SSSE3-NEXT:    pandn %xmm8, %xmm2
-; SSSE3-NEXT:    por %xmm5, %xmm2
-; SSSE3-NEXT:    movdqa %xmm4, %xmm1
-; SSSE3-NEXT:    pxor %xmm9, %xmm1
-; SSSE3-NEXT:    movdqa %xmm10, %xmm3
+; SSSE3-NEXT:    por %xmm12, %xmm2
+; SSSE3-NEXT:    movdqa %xmm11, %xmm1
+; SSSE3-NEXT:    pxor %xmm4, %xmm1
+; SSSE3-NEXT:    movdqa %xmm14, %xmm3
 ; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm11 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm11, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm14, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm5, %xmm6
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm5, %xmm1
-; SSSE3-NEXT:    pand %xmm1, %xmm4
+; SSSE3-NEXT:    por %xmm6, %xmm1
+; SSSE3-NEXT:    pand %xmm1, %xmm11
 ; SSSE3-NEXT:    pandn %xmm8, %xmm1
-; SSSE3-NEXT:    por %xmm4, %xmm1
+; SSSE3-NEXT:    por %xmm11, %xmm1
 ; SSSE3-NEXT:    packuswb %xmm2, %xmm1
-; SSSE3-NEXT:    movdqa %xmm7, %xmm2
-; SSSE3-NEXT:    pxor %xmm9, %xmm2
-; SSSE3-NEXT:    movdqa %xmm10, %xmm3
+; SSSE3-NEXT:    movdqa %xmm10, %xmm2
+; SSSE3-NEXT:    pxor %xmm4, %xmm2
+; SSSE3-NEXT:    movdqa %xmm14, %xmm3
 ; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm14, %xmm2
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm4, %xmm2
+; SSSE3-NEXT:    pand %xmm5, %xmm2
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
 ; SSSE3-NEXT:    por %xmm2, %xmm3
-; SSSE3-NEXT:    pand %xmm3, %xmm7
+; SSSE3-NEXT:    pand %xmm3, %xmm10
 ; SSSE3-NEXT:    pandn %xmm8, %xmm3
-; SSSE3-NEXT:    por %xmm7, %xmm3
-; SSSE3-NEXT:    pxor %xmm6, %xmm9
-; SSSE3-NEXT:    movdqa %xmm10, %xmm2
-; SSSE3-NEXT:    pcmpgtd %xmm9, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm9
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm9[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm4, %xmm5
+; SSSE3-NEXT:    por %xmm10, %xmm3
+; SSSE3-NEXT:    pxor %xmm9, %xmm4
+; SSSE3-NEXT:    movdqa %xmm14, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm14, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm5, %xmm4
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSSE3-NEXT:    por %xmm5, %xmm2
-; SSSE3-NEXT:    pand %xmm2, %xmm6
+; SSSE3-NEXT:    por %xmm4, %xmm2
+; SSSE3-NEXT:    pand %xmm2, %xmm9
 ; SSSE3-NEXT:    pandn %xmm8, %xmm2
-; SSSE3-NEXT:    por %xmm6, %xmm2
+; SSSE3-NEXT:    por %xmm9, %xmm2
 ; SSSE3-NEXT:    packuswb %xmm3, %xmm2
 ; SSSE3-NEXT:    packuswb %xmm2, %xmm1
 ; SSSE3-NEXT:    packuswb %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
-; SSE41-LABEL: trunc_usat_v16i64_v16i8:
+; SSE41-LABEL: trunc_usat_v16i64_v16i8:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa 96(%rdi), %xmm8
+; SSE41-NEXT:    movdqa 112(%rdi), %xmm9
+; SSE41-NEXT:    movdqa 64(%rdi), %xmm10
+; SSE41-NEXT:    movdqa 80(%rdi), %xmm11
+; SSE41-NEXT:    movdqa (%rdi), %xmm2
+; SSE41-NEXT:    movdqa 16(%rdi), %xmm1
+; SSE41-NEXT:    movdqa 32(%rdi), %xmm12
+; SSE41-NEXT:    movdqa 48(%rdi), %xmm13
+; SSE41-NEXT:    movapd {{.*#+}} xmm3 = [255,255]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm6 = [9223372039002259456,9223372039002259456]
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pxor %xmm6, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [9223372039002259711,9223372039002259711]
+; SSE41-NEXT:    movdqa %xmm4, %xmm7
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm7
+; SSE41-NEXT:    movdqa %xmm4, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm5
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2]
+; SSE41-NEXT:    pand %xmm7, %xmm0
+; SSE41-NEXT:    por %xmm5, %xmm0
+; SSE41-NEXT:    movapd %xmm3, %xmm5
+; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm5
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    pxor %xmm6, %xmm0
+; SSE41-NEXT:    movdqa %xmm4, %xmm1
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
+; SSE41-NEXT:    movdqa %xmm4, %xmm7
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm7
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[0,0,2,2]
+; SSE41-NEXT:    pand %xmm1, %xmm0
+; SSE41-NEXT:    por %xmm7, %xmm0
+; SSE41-NEXT:    movapd %xmm3, %xmm1
+; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
+; SSE41-NEXT:    packusdw %xmm5, %xmm1
+; SSE41-NEXT:    movdqa %xmm13, %xmm0
+; SSE41-NEXT:    pxor %xmm6, %xmm0
+; SSE41-NEXT:    movdqa %xmm4, %xmm2
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm2
+; SSE41-NEXT:    movdqa %xmm4, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm5
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2]
+; SSE41-NEXT:    pand %xmm2, %xmm0
+; SSE41-NEXT:    por %xmm5, %xmm0
+; SSE41-NEXT:    movapd %xmm3, %xmm2
+; SSE41-NEXT:    blendvpd %xmm0, %xmm13, %xmm2
+; SSE41-NEXT:    movdqa %xmm12, %xmm0
+; SSE41-NEXT:    pxor %xmm6, %xmm0
+; SSE41-NEXT:    movdqa %xmm4, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm5
+; SSE41-NEXT:    movdqa %xmm4, %xmm7
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm7
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm0
+; SSE41-NEXT:    por %xmm7, %xmm0
+; SSE41-NEXT:    movapd %xmm3, %xmm5
+; SSE41-NEXT:    blendvpd %xmm0, %xmm12, %xmm5
+; SSE41-NEXT:    packusdw %xmm2, %xmm5
+; SSE41-NEXT:    packusdw %xmm5, %xmm1
+; SSE41-NEXT:    movdqa %xmm11, %xmm0
+; SSE41-NEXT:    pxor %xmm6, %xmm0
+; SSE41-NEXT:    movdqa %xmm4, %xmm2
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm2
+; SSE41-NEXT:    movdqa %xmm4, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm5
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2]
+; SSE41-NEXT:    pand %xmm2, %xmm0
+; SSE41-NEXT:    por %xmm5, %xmm0
+; SSE41-NEXT:    movapd %xmm3, %xmm5
+; SSE41-NEXT:    blendvpd %xmm0, %xmm11, %xmm5
+; SSE41-NEXT:    movdqa %xmm10, %xmm0
+; SSE41-NEXT:    pxor %xmm6, %xmm0
+; SSE41-NEXT:    movdqa %xmm4, %xmm2
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm2
+; SSE41-NEXT:    movdqa %xmm4, %xmm7
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm7
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[0,0,2,2]
+; SSE41-NEXT:    pand %xmm2, %xmm0
+; SSE41-NEXT:    por %xmm7, %xmm0
+; SSE41-NEXT:    movapd %xmm3, %xmm2
+; SSE41-NEXT:    blendvpd %xmm0, %xmm10, %xmm2
+; SSE41-NEXT:    packusdw %xmm5, %xmm2
+; SSE41-NEXT:    movdqa %xmm9, %xmm0
+; SSE41-NEXT:    pxor %xmm6, %xmm0
+; SSE41-NEXT:    movdqa %xmm4, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm5
+; SSE41-NEXT:    movdqa %xmm4, %xmm7
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm7
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm0
+; SSE41-NEXT:    por %xmm7, %xmm0
+; SSE41-NEXT:    movapd %xmm3, %xmm5
+; SSE41-NEXT:    blendvpd %xmm0, %xmm9, %xmm5
+; SSE41-NEXT:    pxor %xmm8, %xmm6
+; SSE41-NEXT:    movdqa %xmm4, %xmm7
+; SSE41-NEXT:    pcmpeqd %xmm6, %xmm7
+; SSE41-NEXT:    pcmpgtd %xmm6, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pand %xmm7, %xmm0
+; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm8, %xmm3
+; SSE41-NEXT:    packusdw %xmm5, %xmm3
+; SSE41-NEXT:    packusdw %xmm3, %xmm2
+; SSE41-NEXT:    packuswb %xmm2, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: trunc_usat_v16i64_v16i8:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vmovdqa (%rdi), %xmm12
+; AVX1-NEXT:    vmovdqa 16(%rdi), %xmm13
+; AVX1-NEXT:    vmovdqa 32(%rdi), %xmm15
+; AVX1-NEXT:    vmovdqa 48(%rdi), %xmm9
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [9223372036854775808,9223372036854775808]
+; AVX1-NEXT:    vpxor %xmm4, %xmm12, %xmm5
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [9223372036854776063,9223372036854776063]
+; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm6, %xmm0
+; AVX1-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT:    vpxor %xmm4, %xmm13, %xmm7
+; AVX1-NEXT:    vpcmpgtq %xmm7, %xmm6, %xmm0
+; AVX1-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT:    vpxor %xmm4, %xmm15, %xmm5
+; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm6, %xmm10
+; AVX1-NEXT:    vpxor %xmm4, %xmm9, %xmm7
+; AVX1-NEXT:    vpcmpgtq %xmm7, %xmm6, %xmm11
+; AVX1-NEXT:    vmovdqa 64(%rdi), %xmm5
+; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm7
+; AVX1-NEXT:    vpcmpgtq %xmm7, %xmm6, %xmm14
+; AVX1-NEXT:    vmovdqa 80(%rdi), %xmm3
+; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm1
+; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm6, %xmm1
+; AVX1-NEXT:    vmovdqa 96(%rdi), %xmm7
+; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm2
+; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm6, %xmm2
+; AVX1-NEXT:    vmovdqa 112(%rdi), %xmm0
+; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm4
+; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm6, %xmm4
+; AVX1-NEXT:    vmovapd {{.*#+}} xmm6 = [255,255]
+; AVX1-NEXT:    vblendvpd %xmm4, %xmm0, %xmm6, %xmm8
+; AVX1-NEXT:    vblendvpd %xmm2, %xmm7, %xmm6, %xmm2
+; AVX1-NEXT:    vblendvpd %xmm1, %xmm3, %xmm6, %xmm1
+; AVX1-NEXT:    vblendvpd %xmm14, %xmm5, %xmm6, %xmm3
+; AVX1-NEXT:    vblendvpd %xmm11, %xmm9, %xmm6, %xmm4
+; AVX1-NEXT:    vblendvpd %xmm10, %xmm15, %xmm6, %xmm5
+; AVX1-NEXT:    vmovapd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX1-NEXT:    vblendvpd %xmm0, %xmm13, %xmm6, %xmm7
+; AVX1-NEXT:    vmovapd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX1-NEXT:    vblendvpd %xmm0, %xmm12, %xmm6, %xmm6
+; AVX1-NEXT:    vpackusdw %xmm8, %xmm2, %xmm0
+; AVX1-NEXT:    vpackusdw %xmm1, %xmm3, %xmm1
+; AVX1-NEXT:    vpackusdw %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vpackusdw %xmm4, %xmm5, %xmm1
+; AVX1-NEXT:    vpackusdw %xmm7, %xmm6, %xmm2
+; AVX1-NEXT:    vpackusdw %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vpackuswb %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: trunc_usat_v16i64_v16i8:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX2-NEXT:    vmovdqa 32(%rdi), %ymm1
+; AVX2-NEXT:    vmovdqa 64(%rdi), %ymm2
+; AVX2-NEXT:    vmovdqa 96(%rdi), %ymm3
+; AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm4 = [255,255,255,255]
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm5 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
+; AVX2-NEXT:    vpxor %ymm5, %ymm1, %ymm6
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm7 = [9223372036854776063,9223372036854776063,9223372036854776063,9223372036854776063]
+; AVX2-NEXT:    vpcmpgtq %ymm6, %ymm7, %ymm6
+; AVX2-NEXT:    vblendvpd %ymm6, %ymm1, %ymm4, %ymm1
+; AVX2-NEXT:    vpxor %ymm5, %ymm0, %ymm6
+; AVX2-NEXT:    vpcmpgtq %ymm6, %ymm7, %ymm6
+; AVX2-NEXT:    vblendvpd %ymm6, %ymm0, %ymm4, %ymm0
+; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %ymm5, %ymm3, %ymm1
+; AVX2-NEXT:    vpcmpgtq %ymm1, %ymm7, %ymm1
+; AVX2-NEXT:    vblendvpd %ymm1, %ymm3, %ymm4, %ymm1
+; AVX2-NEXT:    vpxor %ymm5, %ymm2, %ymm3
+; AVX2-NEXT:    vpcmpgtq %ymm3, %ymm7, %ymm3
+; AVX2-NEXT:    vblendvpd %ymm3, %ymm2, %ymm4, %ymm2
+; AVX2-NEXT:    vpackusdw %ymm1, %ymm2, %ymm1
+; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
+; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+;
+; AVX512F-LABEL: trunc_usat_v16i64_v16i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpbroadcastq {{.*#+}} zmm0 = [255,255,255,255,255,255,255,255]
+; AVX512F-NEXT:    vpminuq 64(%rdi), %zmm0, %zmm1
+; AVX512F-NEXT:    vpminuq (%rdi), %zmm0, %zmm0
+; AVX512F-NEXT:    vpmovqd %zmm0, %ymm0
+; AVX512F-NEXT:    vpmovqd %zmm1, %ymm1
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: trunc_usat_v16i64_v16i8:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vmovdqa64 (%rdi), %zmm0
+; AVX512VL-NEXT:    vmovdqa64 64(%rdi), %zmm1
+; AVX512VL-NEXT:    vpmovusqb %zmm1, %xmm1
+; AVX512VL-NEXT:    vpmovusqb %zmm0, %xmm0
+; AVX512VL-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX512VL-NEXT:    vzeroupper
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: trunc_usat_v16i64_v16i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpbroadcastq {{.*#+}} zmm0 = [255,255,255,255,255,255,255,255]
+; AVX512BW-NEXT:    vpminuq 64(%rdi), %zmm0, %zmm1
+; AVX512BW-NEXT:    vpminuq (%rdi), %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmovqd %zmm0, %ymm0
+; AVX512BW-NEXT:    vpmovqd %zmm1, %ymm1
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: trunc_usat_v16i64_v16i8:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vmovdqa64 (%rdi), %zmm0
+; AVX512BWVL-NEXT:    vmovdqa64 64(%rdi), %zmm1
+; AVX512BWVL-NEXT:    vpmovusqb %zmm1, %xmm1
+; AVX512BWVL-NEXT:    vpmovusqb %zmm0, %xmm0
+; AVX512BWVL-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
+;
+; SKX-LABEL: trunc_usat_v16i64_v16i8:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vmovdqa64 (%rdi), %zmm0
+; SKX-NEXT:    vmovdqa64 64(%rdi), %zmm1
+; SKX-NEXT:    vpmovusqb %zmm1, %xmm1
+; SKX-NEXT:    vpmovusqb %zmm0, %xmm0
+; SKX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SKX-NEXT:    vzeroupper
+; SKX-NEXT:    retq
+  %a0 = load <16 x i64>, <16 x i64>* %p0
+  %1 = icmp ult <16 x i64> %a0, <i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255>
+  %2 = select <16 x i1> %1, <16 x i64> %a0, <16 x i64> <i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255>
+  %3 = trunc <16 x i64> %2 to <16 x i8>
+  ret <16 x i8> %3
+}
+
+define <4 x i8> @trunc_usat_v4i32_v4i8(<4 x i32> %a0) {
+; SSE2-LABEL: trunc_usat_v4i32_v4i8:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648]
+; SSE2-NEXT:    pxor %xmm0, %xmm2
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [2147483903,2147483903,2147483903,2147483903]
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm1
+; SSE2-NEXT:    pand %xmm1, %xmm0
+; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm1
+; SSE2-NEXT:    por %xmm0, %xmm1
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE2-NEXT:    packuswb %xmm1, %xmm1
+; SSE2-NEXT:    packuswb %xmm1, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: trunc_usat_v4i32_v4i8:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [2147483648,2147483648,2147483648,2147483648]
+; SSSE3-NEXT:    pxor %xmm0, %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [2147483903,2147483903,2147483903,2147483903]
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSSE3-NEXT:    pand %xmm2, %xmm0
+; SSSE3-NEXT:    pandn {{.*}}(%rip), %xmm2
+; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: trunc_usat_v4i32_v4i8:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pminud {{.*}}(%rip), %xmm0
+; SSE41-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: trunc_usat_v4i32_v4i8:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: trunc_usat_v4i32_v4i8:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [255,255,255,255]
+; AVX2-NEXT:    vpminud %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX2-NEXT:    retq
+;
+; AVX512F-LABEL: trunc_usat_v4i32_v4i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512F-NEXT:    vpmovusdb %zmm0, %xmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: trunc_usat_v4i32_v4i8:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpmovusdb %xmm0, %xmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: trunc_usat_v4i32_v4i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512BW-NEXT:    vpmovusdb %zmm0, %xmm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: trunc_usat_v4i32_v4i8:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpmovusdb %xmm0, %xmm0
+; AVX512BWVL-NEXT:    retq
+;
+; SKX-LABEL: trunc_usat_v4i32_v4i8:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpmovusdb %xmm0, %xmm0
+; SKX-NEXT:    retq
+  %1 = icmp ult <4 x i32> %a0, <i32 255, i32 255, i32 255, i32 255>
+  %2 = select <4 x i1> %1, <4 x i32> %a0, <4 x i32> <i32 255, i32 255, i32 255, i32 255>
+  %3 = trunc <4 x i32> %2 to <4 x i8>
+  ret <4 x i8> %3
+}
+
+define void @trunc_usat_v4i32_v4i8_store(<4 x i32> %a0, <4 x i8> *%p1) {
+; SSE2-LABEL: trunc_usat_v4i32_v4i8_store:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [2147483648,2147483648,2147483648,2147483648]
+; SSE2-NEXT:    pxor %xmm0, %xmm1
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483903,2147483903,2147483903,2147483903]
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm2
+; SSE2-NEXT:    por %xmm0, %xmm2
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm2
+; SSE2-NEXT:    packuswb %xmm0, %xmm2
+; SSE2-NEXT:    packuswb %xmm0, %xmm2
+; SSE2-NEXT:    movd %xmm2, (%rdi)
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: trunc_usat_v4i32_v4i8_store:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [2147483648,2147483648,2147483648,2147483648]
+; SSSE3-NEXT:    pxor %xmm0, %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [2147483903,2147483903,2147483903,2147483903]
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSSE3-NEXT:    pand %xmm2, %xmm0
+; SSSE3-NEXT:    pandn {{.*}}(%rip), %xmm2
+; SSSE3-NEXT:    por %xmm0, %xmm2
+; SSSE3-NEXT:    pshufb {{.*#+}} xmm2 = xmm2[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; SSSE3-NEXT:    movd %xmm2, (%rdi)
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: trunc_usat_v4i32_v4i8_store:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa %xmm0, %xmm8
-; SSE41-NEXT:    movapd {{.*#+}} xmm9 = [255,255]
-; SSE41-NEXT:    movdqa {{.*#+}} xmm10 = [9223372039002259456,9223372039002259456]
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    pxor %xmm10, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm11 = [9223372039002259711,9223372039002259711]
-; SSE41-NEXT:    movdqa %xmm11, %xmm12
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm12
-; SSE41-NEXT:    pshufd {{.*#+}} xmm13 = xmm12[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm11, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm13, %xmm0
-; SSE41-NEXT:    por %xmm12, %xmm0
-; SSE41-NEXT:    movapd %xmm9, %xmm12
-; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm12
-; SSE41-NEXT:    movdqa %xmm8, %xmm0
-; SSE41-NEXT:    pxor %xmm10, %xmm0
-; SSE41-NEXT:    movdqa %xmm11, %xmm1
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm13 = xmm1[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm11, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm13, %xmm0
-; SSE41-NEXT:    por %xmm1, %xmm0
-; SSE41-NEXT:    movapd %xmm9, %xmm13
-; SSE41-NEXT:    blendvpd %xmm0, %xmm8, %xmm13
-; SSE41-NEXT:    packusdw %xmm12, %xmm13
-; SSE41-NEXT:    movdqa %xmm3, %xmm0
-; SSE41-NEXT:    pxor %xmm10, %xmm0
-; SSE41-NEXT:    movdqa %xmm11, %xmm1
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm8 = xmm1[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm11, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm8, %xmm0
-; SSE41-NEXT:    por %xmm1, %xmm0
-; SSE41-NEXT:    movapd %xmm9, %xmm1
-; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm1
-; SSE41-NEXT:    movdqa %xmm2, %xmm0
-; SSE41-NEXT:    pxor %xmm10, %xmm0
-; SSE41-NEXT:    movdqa %xmm11, %xmm3
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm8 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm11, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm8, %xmm0
-; SSE41-NEXT:    por %xmm3, %xmm0
-; SSE41-NEXT:    movapd %xmm9, %xmm3
-; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm3
-; SSE41-NEXT:    packusdw %xmm1, %xmm3
-; SSE41-NEXT:    packusdw %xmm3, %xmm13
-; SSE41-NEXT:    movdqa %xmm5, %xmm0
-; SSE41-NEXT:    pxor %xmm10, %xmm0
-; SSE41-NEXT:    movdqa %xmm11, %xmm1
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm11, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm2, %xmm0
-; SSE41-NEXT:    por %xmm1, %xmm0
-; SSE41-NEXT:    movapd %xmm9, %xmm1
-; SSE41-NEXT:    blendvpd %xmm0, %xmm5, %xmm1
-; SSE41-NEXT:    movdqa %xmm4, %xmm0
-; SSE41-NEXT:    pxor %xmm10, %xmm0
-; SSE41-NEXT:    movdqa %xmm11, %xmm2
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm11, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm3, %xmm0
-; SSE41-NEXT:    por %xmm2, %xmm0
-; SSE41-NEXT:    movapd %xmm9, %xmm2
-; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm2
-; SSE41-NEXT:    packusdw %xmm1, %xmm2
-; SSE41-NEXT:    movdqa %xmm7, %xmm0
-; SSE41-NEXT:    pxor %xmm10, %xmm0
-; SSE41-NEXT:    movdqa %xmm11, %xmm1
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm11, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm3, %xmm0
-; SSE41-NEXT:    por %xmm1, %xmm0
-; SSE41-NEXT:    movapd %xmm9, %xmm1
-; SSE41-NEXT:    blendvpd %xmm0, %xmm7, %xmm1
-; SSE41-NEXT:    pxor %xmm6, %xmm10
-; SSE41-NEXT:    movdqa %xmm11, %xmm3
-; SSE41-NEXT:    pcmpgtd %xmm10, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm11, %xmm10
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm10[1,1,3,3]
-; SSE41-NEXT:    pand %xmm4, %xmm0
-; SSE41-NEXT:    por %xmm3, %xmm0
-; SSE41-NEXT:    blendvpd %xmm0, %xmm6, %xmm9
-; SSE41-NEXT:    packusdw %xmm1, %xmm9
-; SSE41-NEXT:    packusdw %xmm9, %xmm2
-; SSE41-NEXT:    packuswb %xmm2, %xmm13
-; SSE41-NEXT:    movdqa %xmm13, %xmm0
+; SSE41-NEXT:    pminud {{.*}}(%rip), %xmm0
+; SSE41-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; SSE41-NEXT:    movd %xmm0, (%rdi)
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: trunc_usat_v16i64_v16i8:
+; AVX1-LABEL: trunc_usat_v4i32_v4i8_store:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovdqa %ymm0, %ymm8
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm5 = [9223372036854775808,9223372036854775808]
-; AVX1-NEXT:    vpxor %xmm5, %xmm8, %xmm4
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [9223372036854776063,9223372036854776063]
-; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm6, %xmm0
-; AVX1-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX1-NEXT:    vextractf128 $1, %ymm8, %xmm11
-; AVX1-NEXT:    vpxor %xmm5, %xmm11, %xmm4
-; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm6, %xmm0
-; AVX1-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX1-NEXT:    vpxor %xmm5, %xmm1, %xmm4
-; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm6, %xmm10
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm14
-; AVX1-NEXT:    vpxor %xmm5, %xmm14, %xmm7
-; AVX1-NEXT:    vpcmpgtq %xmm7, %xmm6, %xmm12
-; AVX1-NEXT:    vpxor %xmm5, %xmm2, %xmm7
-; AVX1-NEXT:    vpcmpgtq %xmm7, %xmm6, %xmm13
-; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm7
-; AVX1-NEXT:    vpxor %xmm5, %xmm7, %xmm4
-; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm6, %xmm15
-; AVX1-NEXT:    vpxor %xmm5, %xmm3, %xmm4
-; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm6, %xmm4
-; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm0
-; AVX1-NEXT:    vpxor %xmm5, %xmm0, %xmm5
-; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm6, %xmm5
-; AVX1-NEXT:    vmovapd {{.*#+}} xmm6 = [255,255]
-; AVX1-NEXT:    vblendvpd %xmm5, %xmm0, %xmm6, %xmm9
-; AVX1-NEXT:    vblendvpd %xmm4, %xmm3, %xmm6, %xmm3
-; AVX1-NEXT:    vblendvpd %xmm15, %xmm7, %xmm6, %xmm4
-; AVX1-NEXT:    vblendvpd %xmm13, %xmm2, %xmm6, %xmm2
-; AVX1-NEXT:    vblendvpd %xmm12, %xmm14, %xmm6, %xmm5
-; AVX1-NEXT:    vblendvpd %xmm10, %xmm1, %xmm6, %xmm1
-; AVX1-NEXT:    vmovapd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX1-NEXT:    vblendvpd %xmm0, %xmm11, %xmm6, %xmm7
-; AVX1-NEXT:    vmovapd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX1-NEXT:    vblendvpd %xmm0, %xmm8, %xmm6, %xmm6
-; AVX1-NEXT:    vpackusdw %xmm9, %xmm3, %xmm0
-; AVX1-NEXT:    vpackusdw %xmm4, %xmm2, %xmm2
-; AVX1-NEXT:    vpackusdw %xmm0, %xmm2, %xmm0
-; AVX1-NEXT:    vpackusdw %xmm5, %xmm1, %xmm1
-; AVX1-NEXT:    vpackusdw %xmm7, %xmm6, %xmm2
-; AVX1-NEXT:    vpackusdw %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vpackuswb %xmm0, %xmm1, %xmm0
-; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX1-NEXT:    vmovd %xmm0, (%rdi)
 ; AVX1-NEXT:    retq
 ;
-; AVX2-LABEL: trunc_usat_v16i64_v16i8:
+; AVX2-LABEL: trunc_usat_v4i32_v4i8_store:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm4 = [255,255,255,255]
-; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm5 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
-; AVX2-NEXT:    vpxor %ymm5, %ymm1, %ymm6
-; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm7 = [9223372036854776063,9223372036854776063,9223372036854776063,9223372036854776063]
-; AVX2-NEXT:    vpcmpgtq %ymm6, %ymm7, %ymm6
-; AVX2-NEXT:    vblendvpd %ymm6, %ymm1, %ymm4, %ymm1
-; AVX2-NEXT:    vpxor %ymm5, %ymm0, %ymm6
-; AVX2-NEXT:    vpcmpgtq %ymm6, %ymm7, %ymm6
-; AVX2-NEXT:    vblendvpd %ymm6, %ymm0, %ymm4, %ymm0
-; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpxor %ymm5, %ymm3, %ymm1
-; AVX2-NEXT:    vpcmpgtq %ymm1, %ymm7, %ymm1
-; AVX2-NEXT:    vblendvpd %ymm1, %ymm3, %ymm4, %ymm1
-; AVX2-NEXT:    vpxor %ymm5, %ymm2, %ymm3
-; AVX2-NEXT:    vpcmpgtq %ymm3, %ymm7, %ymm3
-; AVX2-NEXT:    vblendvpd %ymm3, %ymm2, %ymm4, %ymm2
-; AVX2-NEXT:    vpackusdw %ymm1, %ymm2, %ymm1
-; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [255,255,255,255]
+; AVX2-NEXT:    vpminud %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX2-NEXT:    vmovd %xmm0, (%rdi)
 ; AVX2-NEXT:    retq
 ;
-; AVX512-LABEL: trunc_usat_v16i64_v16i8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpbroadcastq {{.*#+}} zmm2 = [255,255,255,255,255,255,255,255]
-; AVX512-NEXT:    vpminuq %zmm2, %zmm1, %zmm1
-; AVX512-NEXT:    vpminuq %zmm2, %zmm0, %zmm0
-; AVX512-NEXT:    vpmovqd %zmm0, %ymm0
-; AVX512-NEXT:    vpmovqd %zmm1, %ymm1
-; AVX512-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512-NEXT:    vpmovdb %zmm0, %xmm0
-; AVX512-NEXT:    vzeroupper
-; AVX512-NEXT:    retq
-  %1 = icmp ult <16 x i64> %a0, <i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255>
-  %2 = select <16 x i1> %1, <16 x i64> %a0, <16 x i64> <i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255>
-  %3 = trunc <16 x i64> %2 to <16 x i8>
-  ret <16 x i8> %3
+; AVX512F-LABEL: trunc_usat_v4i32_v4i8_store:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512F-NEXT:    vpmovusdb %zmm0, %xmm0
+; AVX512F-NEXT:    vmovd %xmm0, (%rdi)
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: trunc_usat_v4i32_v4i8_store:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpmovusdb %xmm0, (%rdi)
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: trunc_usat_v4i32_v4i8_store:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512BW-NEXT:    vpmovusdb %zmm0, %xmm0
+; AVX512BW-NEXT:    vmovd %xmm0, (%rdi)
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: trunc_usat_v4i32_v4i8_store:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpmovusdb %xmm0, (%rdi)
+; AVX512BWVL-NEXT:    retq
+;
+; SKX-LABEL: trunc_usat_v4i32_v4i8_store:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpmovusdb %xmm0, (%rdi)
+; SKX-NEXT:    retq
+  %1 = icmp ult <4 x i32> %a0, <i32 255, i32 255, i32 255, i32 255>
+  %2 = select <4 x i1> %1, <4 x i32> %a0, <4 x i32> <i32 255, i32 255, i32 255, i32 255>
+  %3 = trunc <4 x i32> %2 to <4 x i8>
+  store <4 x i8> %3, <4 x i8> *%p1
+  ret void
 }
 
 define <8 x i8> @trunc_usat_v8i32_v8i8(<8 x i32> %a0) {
 ; SSE2-LABEL: trunc_usat_v8i32_v8i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255]
-; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [2147483648,2147483648,2147483648,2147483648]
-; SSE2-NEXT:    movdqa %xmm1, %xmm4
-; SSE2-NEXT:    pxor %xmm3, %xmm4
-; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [2147483903,2147483903,2147483903,2147483903]
-; SSE2-NEXT:    movdqa %xmm5, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm4, %xmm6
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [255,255,255,255]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [2147483648,2147483648,2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm1, %xmm5
+; SSE2-NEXT:    pxor %xmm4, %xmm5
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483903,2147483903,2147483903,2147483903]
+; SSE2-NEXT:    movdqa %xmm2, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm5, %xmm6
 ; SSE2-NEXT:    pand %xmm6, %xmm1
-; SSE2-NEXT:    pandn %xmm2, %xmm6
+; SSE2-NEXT:    pandn %xmm3, %xmm6
 ; SSE2-NEXT:    por %xmm1, %xmm6
-; SSE2-NEXT:    pxor %xmm0, %xmm3
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm5
-; SSE2-NEXT:    pand %xmm5, %xmm0
-; SSE2-NEXT:    pandn %xmm2, %xmm5
-; SSE2-NEXT:    por %xmm5, %xmm0
-; SSE2-NEXT:    packuswb %xmm6, %xmm0
+; SSE2-NEXT:    pxor %xmm0, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm2
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    pandn %xmm3, %xmm2
+; SSE2-NEXT:    por %xmm0, %xmm2
+; SSE2-NEXT:    packuswb %xmm6, %xmm2
+; SSE2-NEXT:    packuswb %xmm2, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: trunc_usat_v8i32_v8i8:
 ; SSSE3:       # %bb.0:
 ; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255]
 ; SSSE3-NEXT:    movdqa {{.*#+}} xmm3 = [2147483648,2147483648,2147483648,2147483648]
-; SSSE3-NEXT:    movdqa %xmm1, %xmm4
+; SSSE3-NEXT:    movdqa %xmm0, %xmm4
 ; SSSE3-NEXT:    pxor %xmm3, %xmm4
 ; SSSE3-NEXT:    movdqa {{.*#+}} xmm5 = [2147483903,2147483903,2147483903,2147483903]
 ; SSSE3-NEXT:    movdqa %xmm5, %xmm6
 ; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm6
-; SSSE3-NEXT:    pand %xmm6, %xmm1
+; SSSE3-NEXT:    pand %xmm6, %xmm0
 ; SSSE3-NEXT:    pandn %xmm2, %xmm6
-; SSSE3-NEXT:    por %xmm1, %xmm6
-; SSSE3-NEXT:    pxor %xmm0, %xmm3
+; SSSE3-NEXT:    por %xmm6, %xmm0
+; SSSE3-NEXT:    pxor %xmm1, %xmm3
 ; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm5
-; SSSE3-NEXT:    pand %xmm5, %xmm0
+; SSSE3-NEXT:    pand %xmm5, %xmm1
 ; SSSE3-NEXT:    pandn %xmm2, %xmm5
-; SSSE3-NEXT:    por %xmm5, %xmm0
-; SSSE3-NEXT:    packuswb %xmm6, %xmm0
+; SSSE3-NEXT:    por %xmm1, %xmm5
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = <0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u>
+; SSSE3-NEXT:    pshufb %xmm1, %xmm5
+; SSSE3-NEXT:    pshufb %xmm1, %xmm0
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1]
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: trunc_usat_v8i32_v8i8:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255]
-; SSE41-NEXT:    pminud %xmm2, %xmm1
 ; SSE41-NEXT:    pminud %xmm2, %xmm0
-; SSE41-NEXT:    packusdw %xmm1, %xmm0
+; SSE41-NEXT:    pminud %xmm2, %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = <0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u>
+; SSE41-NEXT:    pshufb %xmm2, %xmm1
+; SSE41-NEXT:    pshufb %xmm2, %xmm0
+; SSE41-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: trunc_usat_v8i32_v8i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [255,255,255,255]
-; AVX1-NEXT:    vpminud %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpminud %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [255,255,255,255]
+; AVX1-NEXT:    vpminud %xmm1, %xmm0, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vpminud %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = <0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX1-NEXT:    vpshufb %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufb %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
@@ -1917,41 +3909,44 @@ define <8 x i8> @trunc_usat_v8i32_v8i8(<8 x i32> %a0) {
 ; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [255,255,255,255,255,255,255,255]
 ; AVX2-NEXT:    vpminud %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = <0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX2-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: trunc_usat_v8i32_v8i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [255,255,255,255,255,255,255,255]
-; AVX512F-NEXT:    vpminud %ymm1, %ymm0, %ymm0
-; AVX512F-NEXT:    vpmovdw %zmm0, %ymm0
-; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512F-NEXT:    vpmovusdb %zmm0, %xmm0
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: trunc_usat_v8i32_v8i8:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpminud {{.*}}(%rip){1to8}, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpmovdw %ymm0, %xmm0
+; AVX512VL-NEXT:    vpmovusdb %ymm0, %xmm0
 ; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: trunc_usat_v8i32_v8i8:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [255,255,255,255,255,255,255,255]
-; AVX512BW-NEXT:    vpminud %ymm1, %ymm0, %ymm0
-; AVX512BW-NEXT:    vpmovdw %zmm0, %ymm0
-; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512BW-NEXT:    vpmovusdb %zmm0, %xmm0
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512BWVL-LABEL: trunc_usat_v8i32_v8i8:
 ; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vpminud {{.*}}(%rip){1to8}, %ymm0, %ymm0
-; AVX512BWVL-NEXT:    vpmovdw %ymm0, %xmm0
+; AVX512BWVL-NEXT:    vpmovusdb %ymm0, %xmm0
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
+;
+; SKX-LABEL: trunc_usat_v8i32_v8i8:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpmovusdb %ymm0, %xmm0
+; SKX-NEXT:    vzeroupper
+; SKX-NEXT:    retq
   %1 = icmp ult <8 x i32> %a0, <i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255>
   %2 = select <8 x i1> %1, <8 x i32> %a0, <8 x i32> <i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255>
   %3 = trunc <8 x i32> %2 to <8 x i8>
@@ -1977,7 +3972,7 @@ define void @trunc_usat_v8i32_v8i8_store(<8 x i32> %a0, <8 x i8> *%p1) {
 ; SSE2-NEXT:    pandn %xmm2, %xmm5
 ; SSE2-NEXT:    por %xmm0, %xmm5
 ; SSE2-NEXT:    packuswb %xmm6, %xmm5
-; SSE2-NEXT:    packuswb %xmm5, %xmm5
+; SSE2-NEXT:    packuswb %xmm0, %xmm5
 ; SSE2-NEXT:    movq %xmm5, (%rdi)
 ; SSE2-NEXT:    retq
 ;
@@ -1998,32 +3993,35 @@ define void @trunc_usat_v8i32_v8i8_store(<8 x i32> %a0, <8 x i8> *%p1) {
 ; SSSE3-NEXT:    pand %xmm5, %xmm1
 ; SSSE3-NEXT:    pandn %xmm2, %xmm5
 ; SSSE3-NEXT:    por %xmm1, %xmm5
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = <0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u>
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm5
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm6
-; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm6 = xmm6[0],xmm5[0]
-; SSSE3-NEXT:    pshufb {{.*#+}} xmm6 = xmm6[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm6 = xmm6[0],xmm5[0],xmm6[1],xmm5[1]
 ; SSSE3-NEXT:    movq %xmm6, (%rdi)
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: trunc_usat_v8i32_v8i8_store:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255]
-; SSE41-NEXT:    pminud %xmm2, %xmm1
 ; SSE41-NEXT:    pminud %xmm2, %xmm0
-; SSE41-NEXT:    packusdw %xmm1, %xmm0
-; SSE41-NEXT:    packuswb %xmm0, %xmm0
+; SSE41-NEXT:    pminud %xmm2, %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = <0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u>
+; SSE41-NEXT:    pshufb %xmm2, %xmm1
+; SSE41-NEXT:    pshufb %xmm2, %xmm0
+; SSE41-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; SSE41-NEXT:    movq %xmm0, (%rdi)
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: trunc_usat_v8i32_v8i8_store:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [255,255,255,255]
-; AVX1-NEXT:    vpminud %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpminud %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [255,255,255,255]
+; AVX1-NEXT:    vpminud %xmm1, %xmm0, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vpminud %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = <0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX1-NEXT:    vpshufb %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufb %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
 ; AVX1-NEXT:    vmovq %xmm0, (%rdi)
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -2033,18 +4031,18 @@ define void @trunc_usat_v8i32_v8i8_store(<8 x i32> %a0, <8 x i8> *%p1) {
 ; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [255,255,255,255,255,255,255,255]
 ; AVX2-NEXT:    vpminud %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = <0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX2-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; AVX2-NEXT:    vmovq %xmm0, (%rdi)
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: trunc_usat_v8i32_v8i8_store:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [255,255,255,255,255,255,255,255]
-; AVX512F-NEXT:    vpminud %ymm1, %ymm0, %ymm0
-; AVX512F-NEXT:    vpmovdw %zmm0, %ymm0
-; AVX512F-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
+; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512F-NEXT:    vpmovusdb %zmm0, %xmm0
 ; AVX512F-NEXT:    vmovq %xmm0, (%rdi)
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
@@ -2057,10 +4055,8 @@ define void @trunc_usat_v8i32_v8i8_store(<8 x i32> %a0, <8 x i8> *%p1) {
 ;
 ; AVX512BW-LABEL: trunc_usat_v8i32_v8i8_store:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [255,255,255,255,255,255,255,255]
-; AVX512BW-NEXT:    vpminud %ymm1, %ymm0, %ymm0
-; AVX512BW-NEXT:    vpmovdw %zmm0, %ymm0
-; AVX512BW-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
+; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512BW-NEXT:    vpmovusdb %zmm0, %xmm0
 ; AVX512BW-NEXT:    vmovq %xmm0, (%rdi)
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -2070,6 +4066,12 @@ define void @trunc_usat_v8i32_v8i8_store(<8 x i32> %a0, <8 x i8> *%p1) {
 ; AVX512BWVL-NEXT:    vpmovusdb %ymm0, (%rdi)
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
+;
+; SKX-LABEL: trunc_usat_v8i32_v8i8_store:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpmovusdb %ymm0, (%rdi)
+; SKX-NEXT:    vzeroupper
+; SKX-NEXT:    retq
   %1 = icmp ult <8 x i32> %a0, <i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255>
   %2 = select <8 x i1> %1, <8 x i32> %a0, <8 x i32> <i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255>
   %3 = trunc <8 x i32> %2 to <8 x i8>
@@ -2077,127 +4079,423 @@ define void @trunc_usat_v8i32_v8i8_store(<8 x i32> %a0, <8 x i8> *%p1) {
   ret void
 }
 
-define <16 x i8> @trunc_usat_v16i32_v16i8(<16 x i32> %a0) {
+define <16 x i8> @trunc_usat_v16i32_v16i8(<16 x i32>* %p0) {
 ; SSE2-LABEL: trunc_usat_v16i32_v16i8:
 ; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa (%rdi), %xmm6
+; SSE2-NEXT:    movdqa 16(%rdi), %xmm0
+; SSE2-NEXT:    movdqa 32(%rdi), %xmm1
+; SSE2-NEXT:    movdqa 48(%rdi), %xmm5
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [255,255,255,255]
-; SSE2-NEXT:    movdqa {{.*#+}} xmm6 = [2147483648,2147483648,2147483648,2147483648]
-; SSE2-NEXT:    movdqa %xmm1, %xmm7
-; SSE2-NEXT:    pxor %xmm6, %xmm7
-; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [2147483903,2147483903,2147483903,2147483903]
-; SSE2-NEXT:    movdqa %xmm5, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm7, %xmm4
-; SSE2-NEXT:    pand %xmm4, %xmm1
-; SSE2-NEXT:    pandn %xmm8, %xmm4
-; SSE2-NEXT:    por %xmm1, %xmm4
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    pxor %xmm6, %xmm1
-; SSE2-NEXT:    movdqa %xmm5, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm7
-; SSE2-NEXT:    pand %xmm7, %xmm0
-; SSE2-NEXT:    pandn %xmm8, %xmm7
-; SSE2-NEXT:    por %xmm7, %xmm0
-; SSE2-NEXT:    packuswb %xmm4, %xmm0
-; SSE2-NEXT:    movdqa %xmm3, %xmm1
-; SSE2-NEXT:    pxor %xmm6, %xmm1
-; SSE2-NEXT:    movdqa %xmm5, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm4
-; SSE2-NEXT:    pand %xmm4, %xmm3
-; SSE2-NEXT:    pandn %xmm8, %xmm4
-; SSE2-NEXT:    por %xmm3, %xmm4
-; SSE2-NEXT:    pxor %xmm2, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm6, %xmm5
-; SSE2-NEXT:    pand %xmm5, %xmm2
-; SSE2-NEXT:    pandn %xmm8, %xmm5
-; SSE2-NEXT:    por %xmm2, %xmm5
-; SSE2-NEXT:    packuswb %xmm4, %xmm5
-; SSE2-NEXT:    packuswb %xmm5, %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [2147483648,2147483648,2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm0, %xmm7
+; SSE2-NEXT:    pxor %xmm4, %xmm7
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [2147483903,2147483903,2147483903,2147483903]
+; SSE2-NEXT:    movdqa %xmm3, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm7, %xmm2
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    pandn %xmm8, %xmm2
+; SSE2-NEXT:    por %xmm0, %xmm2
+; SSE2-NEXT:    movdqa %xmm6, %xmm7
+; SSE2-NEXT:    pxor %xmm4, %xmm7
+; SSE2-NEXT:    movdqa %xmm3, %xmm0
+; SSE2-NEXT:    pcmpgtd %xmm7, %xmm0
+; SSE2-NEXT:    pand %xmm0, %xmm6
+; SSE2-NEXT:    pandn %xmm8, %xmm0
+; SSE2-NEXT:    por %xmm6, %xmm0
+; SSE2-NEXT:    packuswb %xmm2, %xmm0
+; SSE2-NEXT:    movdqa %xmm5, %xmm2
+; SSE2-NEXT:    pxor %xmm4, %xmm2
+; SSE2-NEXT:    movdqa %xmm3, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm6
+; SSE2-NEXT:    pand %xmm6, %xmm5
+; SSE2-NEXT:    pandn %xmm8, %xmm6
+; SSE2-NEXT:    por %xmm5, %xmm6
+; SSE2-NEXT:    pxor %xmm1, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm3
+; SSE2-NEXT:    pand %xmm3, %xmm1
+; SSE2-NEXT:    pandn %xmm8, %xmm3
+; SSE2-NEXT:    por %xmm1, %xmm3
+; SSE2-NEXT:    packuswb %xmm6, %xmm3
+; SSE2-NEXT:    packuswb %xmm3, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: trunc_usat_v16i32_v16i8:
 ; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa (%rdi), %xmm6
+; SSSE3-NEXT:    movdqa 16(%rdi), %xmm0
+; SSSE3-NEXT:    movdqa 32(%rdi), %xmm1
+; SSSE3-NEXT:    movdqa 48(%rdi), %xmm5
 ; SSSE3-NEXT:    movdqa {{.*#+}} xmm8 = [255,255,255,255]
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm6 = [2147483648,2147483648,2147483648,2147483648]
-; SSSE3-NEXT:    movdqa %xmm1, %xmm7
-; SSSE3-NEXT:    pxor %xmm6, %xmm7
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm5 = [2147483903,2147483903,2147483903,2147483903]
-; SSSE3-NEXT:    movdqa %xmm5, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm7, %xmm4
-; SSSE3-NEXT:    pand %xmm4, %xmm1
-; SSSE3-NEXT:    pandn %xmm8, %xmm4
-; SSSE3-NEXT:    por %xmm1, %xmm4
-; SSSE3-NEXT:    movdqa %xmm0, %xmm1
-; SSSE3-NEXT:    pxor %xmm6, %xmm1
-; SSSE3-NEXT:    movdqa %xmm5, %xmm7
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm7
-; SSSE3-NEXT:    pand %xmm7, %xmm0
-; SSSE3-NEXT:    pandn %xmm8, %xmm7
-; SSSE3-NEXT:    por %xmm7, %xmm0
-; SSSE3-NEXT:    packuswb %xmm4, %xmm0
-; SSSE3-NEXT:    movdqa %xmm3, %xmm1
-; SSSE3-NEXT:    pxor %xmm6, %xmm1
-; SSSE3-NEXT:    movdqa %xmm5, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm4
-; SSSE3-NEXT:    pand %xmm4, %xmm3
-; SSSE3-NEXT:    pandn %xmm8, %xmm4
-; SSSE3-NEXT:    por %xmm3, %xmm4
-; SSSE3-NEXT:    pxor %xmm2, %xmm6
-; SSSE3-NEXT:    pcmpgtd %xmm6, %xmm5
-; SSSE3-NEXT:    pand %xmm5, %xmm2
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm4 = [2147483648,2147483648,2147483648,2147483648]
+; SSSE3-NEXT:    movdqa %xmm0, %xmm7
+; SSSE3-NEXT:    pxor %xmm4, %xmm7
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm3 = [2147483903,2147483903,2147483903,2147483903]
+; SSSE3-NEXT:    movdqa %xmm3, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm7, %xmm2
+; SSSE3-NEXT:    pand %xmm2, %xmm0
+; SSSE3-NEXT:    pandn %xmm8, %xmm2
+; SSSE3-NEXT:    por %xmm0, %xmm2
+; SSSE3-NEXT:    movdqa %xmm6, %xmm7
+; SSSE3-NEXT:    pxor %xmm4, %xmm7
+; SSSE3-NEXT:    movdqa %xmm3, %xmm0
+; SSSE3-NEXT:    pcmpgtd %xmm7, %xmm0
+; SSSE3-NEXT:    pand %xmm0, %xmm6
+; SSSE3-NEXT:    pandn %xmm8, %xmm0
+; SSSE3-NEXT:    por %xmm6, %xmm0
+; SSSE3-NEXT:    packuswb %xmm2, %xmm0
+; SSSE3-NEXT:    movdqa %xmm5, %xmm2
+; SSSE3-NEXT:    pxor %xmm4, %xmm2
+; SSSE3-NEXT:    movdqa %xmm3, %xmm6
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm6
+; SSSE3-NEXT:    pand %xmm6, %xmm5
+; SSSE3-NEXT:    pandn %xmm8, %xmm6
+; SSSE3-NEXT:    por %xmm5, %xmm6
+; SSSE3-NEXT:    pxor %xmm1, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm3
+; SSSE3-NEXT:    pand %xmm3, %xmm1
+; SSSE3-NEXT:    pandn %xmm8, %xmm3
+; SSSE3-NEXT:    por %xmm1, %xmm3
+; SSSE3-NEXT:    packuswb %xmm6, %xmm3
+; SSSE3-NEXT:    packuswb %xmm3, %xmm0
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: trunc_usat_v16i32_v16i8:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,255,255]
+; SSE41-NEXT:    movdqa 16(%rdi), %xmm2
+; SSE41-NEXT:    pminud %xmm1, %xmm2
+; SSE41-NEXT:    movdqa (%rdi), %xmm0
+; SSE41-NEXT:    pminud %xmm1, %xmm0
+; SSE41-NEXT:    packusdw %xmm2, %xmm0
+; SSE41-NEXT:    movdqa 48(%rdi), %xmm2
+; SSE41-NEXT:    pminud %xmm1, %xmm2
+; SSE41-NEXT:    pminud 32(%rdi), %xmm1
+; SSE41-NEXT:    packusdw %xmm2, %xmm1
+; SSE41-NEXT:    packuswb %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: trunc_usat_v16i32_v16i8:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm0 = [255,255,255,255]
+; AVX1-NEXT:    vpminud 16(%rdi), %xmm0, %xmm1
+; AVX1-NEXT:    vpminud (%rdi), %xmm0, %xmm2
+; AVX1-NEXT:    vpackusdw %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vpminud 48(%rdi), %xmm0, %xmm2
+; AVX1-NEXT:    vpminud 32(%rdi), %xmm0, %xmm0
+; AVX1-NEXT:    vpackusdw %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpackuswb %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: trunc_usat_v16i32_v16i8:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm0 = [255,255,255,255,255,255,255,255]
+; AVX2-NEXT:    vpminud 32(%rdi), %ymm0, %ymm1
+; AVX2-NEXT:    vpminud (%rdi), %ymm0, %ymm0
+; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: trunc_usat_v16i32_v16i8:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovdqa64 (%rdi), %zmm0
+; AVX512-NEXT:    vpmovusdb %zmm0, %xmm0
+; AVX512-NEXT:    vzeroupper
+; AVX512-NEXT:    retq
+;
+; SKX-LABEL: trunc_usat_v16i32_v16i8:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vmovdqa64 (%rdi), %zmm0
+; SKX-NEXT:    vpmovusdb %zmm0, %xmm0
+; SKX-NEXT:    vzeroupper
+; SKX-NEXT:    retq
+  %a0 = load <16 x i32>, <16 x i32>* %p0
+  %1 = icmp ult <16 x i32> %a0, <i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255>
+  %2 = select <16 x i1> %1, <16 x i32> %a0, <16 x i32> <i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255>
+  %3 = trunc <16 x i32> %2 to <16 x i8>
+  ret <16 x i8> %3
+}
+
+define void @trunc_usat_v16i32_v16i8_store(<16 x i32>* %p0, <16 x i8>* %p1) {
+; SSE2-LABEL: trunc_usat_v16i32_v16i8_store:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa (%rdi), %xmm6
+; SSE2-NEXT:    movdqa 16(%rdi), %xmm5
+; SSE2-NEXT:    movdqa 32(%rdi), %xmm0
+; SSE2-NEXT:    movdqa 48(%rdi), %xmm4
+; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [255,255,255,255]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [2147483648,2147483648,2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm5, %xmm7
+; SSE2-NEXT:    pxor %xmm3, %xmm7
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483903,2147483903,2147483903,2147483903]
+; SSE2-NEXT:    movdqa %xmm2, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm7, %xmm1
+; SSE2-NEXT:    pand %xmm1, %xmm5
+; SSE2-NEXT:    pandn %xmm8, %xmm1
+; SSE2-NEXT:    por %xmm5, %xmm1
+; SSE2-NEXT:    movdqa %xmm6, %xmm7
+; SSE2-NEXT:    pxor %xmm3, %xmm7
+; SSE2-NEXT:    movdqa %xmm2, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm7, %xmm5
+; SSE2-NEXT:    pand %xmm5, %xmm6
+; SSE2-NEXT:    pandn %xmm8, %xmm5
+; SSE2-NEXT:    por %xmm6, %xmm5
+; SSE2-NEXT:    packuswb %xmm1, %xmm5
+; SSE2-NEXT:    movdqa %xmm4, %xmm1
+; SSE2-NEXT:    pxor %xmm3, %xmm1
+; SSE2-NEXT:    movdqa %xmm2, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm6
+; SSE2-NEXT:    pand %xmm6, %xmm4
+; SSE2-NEXT:    pandn %xmm8, %xmm6
+; SSE2-NEXT:    por %xmm4, %xmm6
+; SSE2-NEXT:    pxor %xmm0, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm2
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    pandn %xmm8, %xmm2
+; SSE2-NEXT:    por %xmm0, %xmm2
+; SSE2-NEXT:    packuswb %xmm6, %xmm2
+; SSE2-NEXT:    packuswb %xmm2, %xmm5
+; SSE2-NEXT:    movdqa %xmm5, (%rsi)
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: trunc_usat_v16i32_v16i8_store:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa (%rdi), %xmm6
+; SSSE3-NEXT:    movdqa 16(%rdi), %xmm5
+; SSSE3-NEXT:    movdqa 32(%rdi), %xmm0
+; SSSE3-NEXT:    movdqa 48(%rdi), %xmm4
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm8 = [255,255,255,255]
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm3 = [2147483648,2147483648,2147483648,2147483648]
+; SSSE3-NEXT:    movdqa %xmm5, %xmm7
+; SSSE3-NEXT:    pxor %xmm3, %xmm7
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [2147483903,2147483903,2147483903,2147483903]
+; SSSE3-NEXT:    movdqa %xmm2, %xmm1
+; SSSE3-NEXT:    pcmpgtd %xmm7, %xmm1
+; SSSE3-NEXT:    pand %xmm1, %xmm5
+; SSSE3-NEXT:    pandn %xmm8, %xmm1
+; SSSE3-NEXT:    por %xmm5, %xmm1
+; SSSE3-NEXT:    movdqa %xmm6, %xmm7
+; SSSE3-NEXT:    pxor %xmm3, %xmm7
+; SSSE3-NEXT:    movdqa %xmm2, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm7, %xmm5
+; SSSE3-NEXT:    pand %xmm5, %xmm6
 ; SSSE3-NEXT:    pandn %xmm8, %xmm5
-; SSSE3-NEXT:    por %xmm2, %xmm5
-; SSSE3-NEXT:    packuswb %xmm4, %xmm5
-; SSSE3-NEXT:    packuswb %xmm5, %xmm0
+; SSSE3-NEXT:    por %xmm6, %xmm5
+; SSSE3-NEXT:    packuswb %xmm1, %xmm5
+; SSSE3-NEXT:    movdqa %xmm4, %xmm1
+; SSSE3-NEXT:    pxor %xmm3, %xmm1
+; SSSE3-NEXT:    movdqa %xmm2, %xmm6
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm6
+; SSSE3-NEXT:    pand %xmm6, %xmm4
+; SSSE3-NEXT:    pandn %xmm8, %xmm6
+; SSSE3-NEXT:    por %xmm4, %xmm6
+; SSSE3-NEXT:    pxor %xmm0, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm2
+; SSSE3-NEXT:    pand %xmm2, %xmm0
+; SSSE3-NEXT:    pandn %xmm8, %xmm2
+; SSSE3-NEXT:    por %xmm0, %xmm2
+; SSSE3-NEXT:    packuswb %xmm6, %xmm2
+; SSSE3-NEXT:    packuswb %xmm2, %xmm5
+; SSSE3-NEXT:    movdqa %xmm5, (%rsi)
 ; SSSE3-NEXT:    retq
 ;
-; SSE41-LABEL: trunc_usat_v16i32_v16i8:
+; SSE41-LABEL: trunc_usat_v16i32_v16i8_store:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [255,255,255,255]
-; SSE41-NEXT:    pminud %xmm4, %xmm1
-; SSE41-NEXT:    pminud %xmm4, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,255]
+; SSE41-NEXT:    movdqa 16(%rdi), %xmm1
+; SSE41-NEXT:    pminud %xmm0, %xmm1
+; SSE41-NEXT:    movdqa (%rdi), %xmm2
+; SSE41-NEXT:    pminud %xmm0, %xmm2
+; SSE41-NEXT:    packusdw %xmm1, %xmm2
+; SSE41-NEXT:    movdqa 48(%rdi), %xmm1
+; SSE41-NEXT:    pminud %xmm0, %xmm1
+; SSE41-NEXT:    pminud 32(%rdi), %xmm0
 ; SSE41-NEXT:    packusdw %xmm1, %xmm0
-; SSE41-NEXT:    pminud %xmm4, %xmm3
-; SSE41-NEXT:    pminud %xmm4, %xmm2
-; SSE41-NEXT:    packusdw %xmm3, %xmm2
-; SSE41-NEXT:    packuswb %xmm2, %xmm0
+; SSE41-NEXT:    packuswb %xmm0, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, (%rsi)
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: trunc_usat_v16i32_v16i8:
+; AVX1-LABEL: trunc_usat_v16i32_v16i8_store:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [255,255,255,255]
-; AVX1-NEXT:    vpminud %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpminud %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm0 = [255,255,255,255]
+; AVX1-NEXT:    vpminud 16(%rdi), %xmm0, %xmm1
+; AVX1-NEXT:    vpminud (%rdi), %xmm0, %xmm2
+; AVX1-NEXT:    vpackusdw %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vpminud 48(%rdi), %xmm0, %xmm2
+; AVX1-NEXT:    vpminud 32(%rdi), %xmm0, %xmm0
 ; AVX1-NEXT:    vpackusdw %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT:    vpminud %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpminud %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpackusdw %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    vpackuswb %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vmovdqa %xmm0, (%rsi)
 ; AVX1-NEXT:    retq
 ;
-; AVX2-LABEL: trunc_usat_v16i32_v16i8:
+; AVX2-LABEL: trunc_usat_v16i32_v16i8_store:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255]
-; AVX2-NEXT:    vpminud %ymm2, %ymm1, %ymm1
-; AVX2-NEXT:    vpminud %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm0 = [255,255,255,255,255,255,255,255]
+; AVX2-NEXT:    vpminud 32(%rdi), %ymm0, %ymm1
+; AVX2-NEXT:    vpminud (%rdi), %ymm0, %ymm0
 ; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vmovdqa %xmm0, (%rsi)
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
-; AVX512-LABEL: trunc_usat_v16i32_v16i8:
+; AVX512-LABEL: trunc_usat_v16i32_v16i8_store:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpmovusdb %zmm0, %xmm0
+; AVX512-NEXT:    vmovdqa64 (%rdi), %zmm0
+; AVX512-NEXT:    vpmovusdb %zmm0, (%rsi)
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
+;
+; SKX-LABEL: trunc_usat_v16i32_v16i8_store:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vmovdqa64 (%rdi), %zmm0
+; SKX-NEXT:    vpmovusdb %zmm0, (%rsi)
+; SKX-NEXT:    vzeroupper
+; SKX-NEXT:    retq
+  %a0 = load <16 x i32>, <16 x i32>* %p0
   %1 = icmp ult <16 x i32> %a0, <i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255>
   %2 = select <16 x i1> %1, <16 x i32> %a0, <16 x i32> <i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255>
   %3 = trunc <16 x i32> %2 to <16 x i8>
-  ret <16 x i8> %3
+  store <16 x i8> %3, <16 x i8>* %p1
+  ret void
+}
+
+define <8 x i8> @trunc_usat_v8i16_v8i8(<8 x i16> %a0) {
+; SSE2-LABEL: trunc_usat_v8i16_v8i8:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pxor {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    pminsw {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    packuswb %xmm0, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: trunc_usat_v8i16_v8i8:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    pxor {{.*}}(%rip), %xmm0
+; SSSE3-NEXT:    pminsw {{.*}}(%rip), %xmm0
+; SSSE3-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: trunc_usat_v8i16_v8i8:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pminuw {{.*}}(%rip), %xmm0
+; SSE41-NEXT:    packuswb %xmm0, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: trunc_usat_v8i16_v8i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpminuw {{.*}}(%rip), %xmm0, %xmm0
+; AVX-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; AVX512F-LABEL: trunc_usat_v8i16_v8i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpminuw {{.*}}(%rip), %xmm0, %xmm0
+; AVX512F-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: trunc_usat_v8i16_v8i8:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpminuw {{.*}}(%rip), %xmm0, %xmm0
+; AVX512VL-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: trunc_usat_v8i16_v8i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512BW-NEXT:    vpmovuswb %zmm0, %ymm0
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: trunc_usat_v8i16_v8i8:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpmovuswb %xmm0, %xmm0
+; AVX512BWVL-NEXT:    retq
+;
+; SKX-LABEL: trunc_usat_v8i16_v8i8:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpmovuswb %xmm0, %xmm0
+; SKX-NEXT:    retq
+  %1 = icmp ult <8 x i16> %a0, <i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255>
+  %2 = select <8 x i1> %1, <8 x i16> %a0, <8 x i16> <i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255>
+  %3 = trunc <8 x i16> %2 to <8 x i8>
+  ret <8 x i8> %3
+}
+
+define void @trunc_usat_v8i16_v8i8_store(<8 x i16> %a0, <8 x i8> *%p1) {
+; SSE2-LABEL: trunc_usat_v8i16_v8i8_store:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pxor {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    pminsw {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    packuswb %xmm0, %xmm0
+; SSE2-NEXT:    movq %xmm0, (%rdi)
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: trunc_usat_v8i16_v8i8_store:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    pxor {{.*}}(%rip), %xmm0
+; SSSE3-NEXT:    pminsw {{.*}}(%rip), %xmm0
+; SSSE3-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
+; SSSE3-NEXT:    movq %xmm0, (%rdi)
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: trunc_usat_v8i16_v8i8_store:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pminuw {{.*}}(%rip), %xmm0
+; SSE41-NEXT:    packuswb %xmm0, %xmm0
+; SSE41-NEXT:    movq %xmm0, (%rdi)
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: trunc_usat_v8i16_v8i8_store:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpminuw {{.*}}(%rip), %xmm0, %xmm0
+; AVX-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vmovq %xmm0, (%rdi)
+; AVX-NEXT:    retq
+;
+; AVX512F-LABEL: trunc_usat_v8i16_v8i8_store:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpminuw {{.*}}(%rip), %xmm0, %xmm0
+; AVX512F-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
+; AVX512F-NEXT:    vmovq %xmm0, (%rdi)
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: trunc_usat_v8i16_v8i8_store:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpminuw {{.*}}(%rip), %xmm0, %xmm0
+; AVX512VL-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
+; AVX512VL-NEXT:    vmovq %xmm0, (%rdi)
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: trunc_usat_v8i16_v8i8_store:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512BW-NEXT:    vpmovuswb %zmm0, %ymm0
+; AVX512BW-NEXT:    vmovq %xmm0, (%rdi)
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: trunc_usat_v8i16_v8i8_store:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpmovuswb %xmm0, (%rdi)
+; AVX512BWVL-NEXT:    retq
+;
+; SKX-LABEL: trunc_usat_v8i16_v8i8_store:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpmovuswb %xmm0, (%rdi)
+; SKX-NEXT:    retq
+  %1 = icmp ult <8 x i16> %a0, <i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255>
+  %2 = select <8 x i1> %1, <8 x i16> %a0, <8 x i16> <i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255>
+  %3 = trunc <8 x i16> %2 to <8 x i8>
+  store <8 x i8> %3, <8 x i8> *%p1
+  ret void
 }
 
 define <16 x i8> @trunc_usat_v16i16_v16i8(<16 x i16> %a0) {
@@ -2271,8 +4569,8 @@ define <16 x i8> @trunc_usat_v16i16_v16i8(<16 x i16> %a0) {
 ;
 ; AVX512BW-LABEL: trunc_usat_v16i16_v16i8:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpminuw {{.*}}(%rip), %ymm0, %ymm0
-; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512BW-NEXT:    vpmovuswb %zmm0, %ymm0
 ; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -2282,95 +4580,107 @@ define <16 x i8> @trunc_usat_v16i16_v16i8(<16 x i16> %a0) {
 ; AVX512BWVL-NEXT:    vpmovuswb %ymm0, %xmm0
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
+;
+; SKX-LABEL: trunc_usat_v16i16_v16i8:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vpmovuswb %ymm0, %xmm0
+; SKX-NEXT:    vzeroupper
+; SKX-NEXT:    retq
   %1 = icmp ult <16 x i16> %a0, <i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255>
   %2 = select <16 x i1> %1, <16 x i16> %a0, <16 x i16> <i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255>
   %3 = trunc <16 x i16> %2 to <16 x i8>
   ret <16 x i8> %3
 }
 
-define <32 x i8> @trunc_usat_v32i16_v32i8(<32 x i16> %a0) {
+define <32 x i8> @trunc_usat_v32i16_v32i8(<32 x i16>* %p0) {
 ; SSE2-LABEL: trunc_usat_v32i16_v32i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [32768,32768,32768,32768,32768,32768,32768,32768]
-; SSE2-NEXT:    pxor %xmm4, %xmm3
-; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [33023,33023,33023,33023,33023,33023,33023,33023]
-; SSE2-NEXT:    pminsw %xmm5, %xmm3
-; SSE2-NEXT:    pxor %xmm4, %xmm3
-; SSE2-NEXT:    pxor %xmm4, %xmm2
-; SSE2-NEXT:    pminsw %xmm5, %xmm2
-; SSE2-NEXT:    pxor %xmm4, %xmm2
-; SSE2-NEXT:    packuswb %xmm3, %xmm2
-; SSE2-NEXT:    pxor %xmm4, %xmm1
-; SSE2-NEXT:    pminsw %xmm5, %xmm1
-; SSE2-NEXT:    pxor %xmm4, %xmm1
-; SSE2-NEXT:    pxor %xmm4, %xmm0
-; SSE2-NEXT:    pminsw %xmm5, %xmm0
-; SSE2-NEXT:    pxor %xmm4, %xmm0
-; SSE2-NEXT:    packuswb %xmm1, %xmm0
-; SSE2-NEXT:    movdqa %xmm2, %xmm1
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [32768,32768,32768,32768,32768,32768,32768,32768]
+; SSE2-NEXT:    movdqa 48(%rdi), %xmm0
+; SSE2-NEXT:    pxor %xmm2, %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [33023,33023,33023,33023,33023,33023,33023,33023]
+; SSE2-NEXT:    pminsw %xmm3, %xmm0
+; SSE2-NEXT:    pxor %xmm2, %xmm0
+; SSE2-NEXT:    movdqa 32(%rdi), %xmm1
+; SSE2-NEXT:    pxor %xmm2, %xmm1
+; SSE2-NEXT:    pminsw %xmm3, %xmm1
+; SSE2-NEXT:    pxor %xmm2, %xmm1
+; SSE2-NEXT:    packuswb %xmm0, %xmm1
+; SSE2-NEXT:    movdqa 16(%rdi), %xmm4
+; SSE2-NEXT:    pxor %xmm2, %xmm4
+; SSE2-NEXT:    pminsw %xmm3, %xmm4
+; SSE2-NEXT:    pxor %xmm2, %xmm4
+; SSE2-NEXT:    movdqa (%rdi), %xmm0
+; SSE2-NEXT:    pxor %xmm2, %xmm0
+; SSE2-NEXT:    pminsw %xmm3, %xmm0
+; SSE2-NEXT:    pxor %xmm2, %xmm0
+; SSE2-NEXT:    packuswb %xmm4, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: trunc_usat_v32i16_v32i8:
 ; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm4 = [32768,32768,32768,32768,32768,32768,32768,32768]
-; SSSE3-NEXT:    pxor %xmm4, %xmm3
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm5 = [33023,33023,33023,33023,33023,33023,33023,33023]
-; SSSE3-NEXT:    pminsw %xmm5, %xmm3
-; SSSE3-NEXT:    pxor %xmm4, %xmm3
-; SSSE3-NEXT:    pxor %xmm4, %xmm2
-; SSSE3-NEXT:    pminsw %xmm5, %xmm2
-; SSSE3-NEXT:    pxor %xmm4, %xmm2
-; SSSE3-NEXT:    packuswb %xmm3, %xmm2
-; SSSE3-NEXT:    pxor %xmm4, %xmm1
-; SSSE3-NEXT:    pminsw %xmm5, %xmm1
-; SSSE3-NEXT:    pxor %xmm4, %xmm1
-; SSSE3-NEXT:    pxor %xmm4, %xmm0
-; SSSE3-NEXT:    pminsw %xmm5, %xmm0
-; SSSE3-NEXT:    pxor %xmm4, %xmm0
-; SSSE3-NEXT:    packuswb %xmm1, %xmm0
-; SSSE3-NEXT:    movdqa %xmm2, %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [32768,32768,32768,32768,32768,32768,32768,32768]
+; SSSE3-NEXT:    movdqa 48(%rdi), %xmm0
+; SSSE3-NEXT:    pxor %xmm2, %xmm0
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm3 = [33023,33023,33023,33023,33023,33023,33023,33023]
+; SSSE3-NEXT:    pminsw %xmm3, %xmm0
+; SSSE3-NEXT:    pxor %xmm2, %xmm0
+; SSSE3-NEXT:    movdqa 32(%rdi), %xmm1
+; SSSE3-NEXT:    pxor %xmm2, %xmm1
+; SSSE3-NEXT:    pminsw %xmm3, %xmm1
+; SSSE3-NEXT:    pxor %xmm2, %xmm1
+; SSSE3-NEXT:    packuswb %xmm0, %xmm1
+; SSSE3-NEXT:    movdqa 16(%rdi), %xmm4
+; SSSE3-NEXT:    pxor %xmm2, %xmm4
+; SSSE3-NEXT:    pminsw %xmm3, %xmm4
+; SSSE3-NEXT:    pxor %xmm2, %xmm4
+; SSSE3-NEXT:    movdqa (%rdi), %xmm0
+; SSSE3-NEXT:    pxor %xmm2, %xmm0
+; SSSE3-NEXT:    pminsw %xmm3, %xmm0
+; SSSE3-NEXT:    pxor %xmm2, %xmm0
+; SSSE3-NEXT:    packuswb %xmm4, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: trunc_usat_v32i16_v32i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
-; SSE41-NEXT:    pminuw %xmm4, %xmm3
-; SSE41-NEXT:    pminuw %xmm4, %xmm2
-; SSE41-NEXT:    packuswb %xmm3, %xmm2
-; SSE41-NEXT:    pminuw %xmm4, %xmm1
-; SSE41-NEXT:    pminuw %xmm4, %xmm0
-; SSE41-NEXT:    packuswb %xmm1, %xmm0
-; SSE41-NEXT:    movdqa %xmm2, %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255]
+; SSE41-NEXT:    movdqa 48(%rdi), %xmm2
+; SSE41-NEXT:    pminuw %xmm0, %xmm2
+; SSE41-NEXT:    movdqa 32(%rdi), %xmm1
+; SSE41-NEXT:    pminuw %xmm0, %xmm1
+; SSE41-NEXT:    packuswb %xmm2, %xmm1
+; SSE41-NEXT:    movdqa 16(%rdi), %xmm2
+; SSE41-NEXT:    pminuw %xmm0, %xmm2
+; SSE41-NEXT:    pminuw (%rdi), %xmm0
+; SSE41-NEXT:    packuswb %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: trunc_usat_v32i16_v32i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
-; AVX1-NEXT:    vpminuw %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpminuw %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255]
+; AVX1-NEXT:    vpminuw 16(%rdi), %xmm0, %xmm1
+; AVX1-NEXT:    vpminuw (%rdi), %xmm0, %xmm2
+; AVX1-NEXT:    vpackuswb %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vpminuw 48(%rdi), %xmm0, %xmm2
+; AVX1-NEXT:    vpminuw 32(%rdi), %xmm0, %xmm0
 ; AVX1-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT:    vpminuw %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpminuw %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: trunc_usat_v32i16_v32i8:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX2-NEXT:    vpminuw %ymm2, %ymm1, %ymm1
-; AVX2-NEXT:    vpminuw %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vmovdqa {{.*#+}} ymm0 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; AVX2-NEXT:    vpminuw 32(%rdi), %ymm0, %ymm1
+; AVX2-NEXT:    vpminuw (%rdi), %ymm0, %ymm0
 ; AVX2-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: trunc_usat_v32i16_v32i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512F-NEXT:    vpminuw %ymm2, %ymm1, %ymm1
-; AVX512F-NEXT:    vpminuw %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm0 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; AVX512F-NEXT:    vpminuw 32(%rdi), %ymm0, %ymm1
+; AVX512F-NEXT:    vpminuw (%rdi), %ymm0, %ymm0
 ; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
 ; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
 ; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
@@ -2380,9 +4690,9 @@ define <32 x i8> @trunc_usat_v32i16_v32i8(<32 x i16> %a0) {
 ;
 ; AVX512VL-LABEL: trunc_usat_v32i16_v32i8:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512VL-NEXT:    vpminuw %ymm2, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpminuw %ymm2, %ymm0, %ymm0
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm0 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; AVX512VL-NEXT:    vpminuw 32(%rdi), %ymm0, %ymm1
+; AVX512VL-NEXT:    vpminuw (%rdi), %ymm0, %ymm0
 ; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
 ; AVX512VL-NEXT:    vpmovdb %zmm0, %xmm0
 ; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
@@ -2392,15 +4702,260 @@ define <32 x i8> @trunc_usat_v32i16_v32i8(<32 x i16> %a0) {
 ;
 ; AVX512BW-LABEL: trunc_usat_v32i16_v32i8:
 ; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm0
 ; AVX512BW-NEXT:    vpmovuswb %zmm0, %ymm0
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512BWVL-LABEL: trunc_usat_v32i16_v32i8:
 ; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vmovdqa64 (%rdi), %zmm0
 ; AVX512BWVL-NEXT:    vpmovuswb %zmm0, %ymm0
 ; AVX512BWVL-NEXT:    retq
+;
+; SKX-LABEL: trunc_usat_v32i16_v32i8:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vmovdqa64 (%rdi), %zmm0
+; SKX-NEXT:    vpmovuswb %zmm0, %ymm0
+; SKX-NEXT:    retq
+  %a0 = load <32 x i16>, <32 x i16>* %p0
   %1 = icmp ult <32 x i16> %a0, <i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255>
   %2 = select <32 x i1> %1, <32 x i16> %a0, <32 x i16> <i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255>
   %3 = trunc <32 x i16> %2 to <32 x i8>
   ret <32 x i8> %3
 }
+
+define <32 x i8> @trunc_usat_v32i32_v32i8(<32 x i32>* %p0) {
+; SSE2-LABEL: trunc_usat_v32i32_v32i8:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa (%rdi), %xmm11
+; SSE2-NEXT:    movdqa 16(%rdi), %xmm12
+; SSE2-NEXT:    movdqa 32(%rdi), %xmm9
+; SSE2-NEXT:    movdqa 48(%rdi), %xmm10
+; SSE2-NEXT:    movdqa 96(%rdi), %xmm0
+; SSE2-NEXT:    movdqa 112(%rdi), %xmm2
+; SSE2-NEXT:    movdqa 64(%rdi), %xmm5
+; SSE2-NEXT:    movdqa 80(%rdi), %xmm7
+; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [255,255,255,255]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm6 = [2147483648,2147483648,2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm7, %xmm1
+; SSE2-NEXT:    pxor %xmm6, %xmm1
+; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [2147483903,2147483903,2147483903,2147483903]
+; SSE2-NEXT:    movdqa %xmm4, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
+; SSE2-NEXT:    pand %xmm3, %xmm7
+; SSE2-NEXT:    pandn %xmm8, %xmm3
+; SSE2-NEXT:    por %xmm7, %xmm3
+; SSE2-NEXT:    movdqa %xmm5, %xmm7
+; SSE2-NEXT:    pxor %xmm6, %xmm7
+; SSE2-NEXT:    movdqa %xmm4, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm7, %xmm1
+; SSE2-NEXT:    pand %xmm1, %xmm5
+; SSE2-NEXT:    pandn %xmm8, %xmm1
+; SSE2-NEXT:    por %xmm5, %xmm1
+; SSE2-NEXT:    packuswb %xmm3, %xmm1
+; SSE2-NEXT:    movdqa %xmm2, %xmm3
+; SSE2-NEXT:    pxor %xmm6, %xmm3
+; SSE2-NEXT:    movdqa %xmm4, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm5
+; SSE2-NEXT:    pand %xmm5, %xmm2
+; SSE2-NEXT:    pandn %xmm8, %xmm5
+; SSE2-NEXT:    por %xmm2, %xmm5
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    pxor %xmm6, %xmm2
+; SSE2-NEXT:    movdqa %xmm4, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm3
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    pandn %xmm8, %xmm3
+; SSE2-NEXT:    por %xmm0, %xmm3
+; SSE2-NEXT:    packuswb %xmm5, %xmm3
+; SSE2-NEXT:    packuswb %xmm3, %xmm1
+; SSE2-NEXT:    movdqa %xmm12, %xmm0
+; SSE2-NEXT:    pxor %xmm6, %xmm0
+; SSE2-NEXT:    movdqa %xmm4, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSE2-NEXT:    pand %xmm2, %xmm12
+; SSE2-NEXT:    pandn %xmm8, %xmm2
+; SSE2-NEXT:    por %xmm12, %xmm2
+; SSE2-NEXT:    movdqa %xmm11, %xmm3
+; SSE2-NEXT:    pxor %xmm6, %xmm3
+; SSE2-NEXT:    movdqa %xmm4, %xmm0
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm0
+; SSE2-NEXT:    pand %xmm0, %xmm11
+; SSE2-NEXT:    pandn %xmm8, %xmm0
+; SSE2-NEXT:    por %xmm11, %xmm0
+; SSE2-NEXT:    packuswb %xmm2, %xmm0
+; SSE2-NEXT:    movdqa %xmm10, %xmm2
+; SSE2-NEXT:    pxor %xmm6, %xmm2
+; SSE2-NEXT:    movdqa %xmm4, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm3
+; SSE2-NEXT:    pand %xmm3, %xmm10
+; SSE2-NEXT:    pandn %xmm8, %xmm3
+; SSE2-NEXT:    por %xmm10, %xmm3
+; SSE2-NEXT:    pxor %xmm9, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm6, %xmm4
+; SSE2-NEXT:    pand %xmm4, %xmm9
+; SSE2-NEXT:    pandn %xmm8, %xmm4
+; SSE2-NEXT:    por %xmm9, %xmm4
+; SSE2-NEXT:    packuswb %xmm3, %xmm4
+; SSE2-NEXT:    packuswb %xmm4, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: trunc_usat_v32i32_v32i8:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa (%rdi), %xmm11
+; SSSE3-NEXT:    movdqa 16(%rdi), %xmm12
+; SSSE3-NEXT:    movdqa 32(%rdi), %xmm9
+; SSSE3-NEXT:    movdqa 48(%rdi), %xmm10
+; SSSE3-NEXT:    movdqa 96(%rdi), %xmm0
+; SSSE3-NEXT:    movdqa 112(%rdi), %xmm2
+; SSSE3-NEXT:    movdqa 64(%rdi), %xmm5
+; SSSE3-NEXT:    movdqa 80(%rdi), %xmm7
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm8 = [255,255,255,255]
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm6 = [2147483648,2147483648,2147483648,2147483648]
+; SSSE3-NEXT:    movdqa %xmm7, %xmm1
+; SSSE3-NEXT:    pxor %xmm6, %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm4 = [2147483903,2147483903,2147483903,2147483903]
+; SSSE3-NEXT:    movdqa %xmm4, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm3
+; SSSE3-NEXT:    pand %xmm3, %xmm7
+; SSSE3-NEXT:    pandn %xmm8, %xmm3
+; SSSE3-NEXT:    por %xmm7, %xmm3
+; SSSE3-NEXT:    movdqa %xmm5, %xmm7
+; SSSE3-NEXT:    pxor %xmm6, %xmm7
+; SSSE3-NEXT:    movdqa %xmm4, %xmm1
+; SSSE3-NEXT:    pcmpgtd %xmm7, %xmm1
+; SSSE3-NEXT:    pand %xmm1, %xmm5
+; SSSE3-NEXT:    pandn %xmm8, %xmm1
+; SSSE3-NEXT:    por %xmm5, %xmm1
+; SSSE3-NEXT:    packuswb %xmm3, %xmm1
+; SSSE3-NEXT:    movdqa %xmm2, %xmm3
+; SSSE3-NEXT:    pxor %xmm6, %xmm3
+; SSSE3-NEXT:    movdqa %xmm4, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm5
+; SSSE3-NEXT:    pand %xmm5, %xmm2
+; SSSE3-NEXT:    pandn %xmm8, %xmm5
+; SSSE3-NEXT:    por %xmm2, %xmm5
+; SSSE3-NEXT:    movdqa %xmm0, %xmm2
+; SSSE3-NEXT:    pxor %xmm6, %xmm2
+; SSSE3-NEXT:    movdqa %xmm4, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm3
+; SSSE3-NEXT:    pand %xmm3, %xmm0
+; SSSE3-NEXT:    pandn %xmm8, %xmm3
+; SSSE3-NEXT:    por %xmm0, %xmm3
+; SSSE3-NEXT:    packuswb %xmm5, %xmm3
+; SSSE3-NEXT:    packuswb %xmm3, %xmm1
+; SSSE3-NEXT:    movdqa %xmm12, %xmm0
+; SSSE3-NEXT:    pxor %xmm6, %xmm0
+; SSSE3-NEXT:    movdqa %xmm4, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSSE3-NEXT:    pand %xmm2, %xmm12
+; SSSE3-NEXT:    pandn %xmm8, %xmm2
+; SSSE3-NEXT:    por %xmm12, %xmm2
+; SSSE3-NEXT:    movdqa %xmm11, %xmm3
+; SSSE3-NEXT:    pxor %xmm6, %xmm3
+; SSSE3-NEXT:    movdqa %xmm4, %xmm0
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
+; SSSE3-NEXT:    pand %xmm0, %xmm11
+; SSSE3-NEXT:    pandn %xmm8, %xmm0
+; SSSE3-NEXT:    por %xmm11, %xmm0
+; SSSE3-NEXT:    packuswb %xmm2, %xmm0
+; SSSE3-NEXT:    movdqa %xmm10, %xmm2
+; SSSE3-NEXT:    pxor %xmm6, %xmm2
+; SSSE3-NEXT:    movdqa %xmm4, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm3
+; SSSE3-NEXT:    pand %xmm3, %xmm10
+; SSSE3-NEXT:    pandn %xmm8, %xmm3
+; SSSE3-NEXT:    por %xmm10, %xmm3
+; SSSE3-NEXT:    pxor %xmm9, %xmm6
+; SSSE3-NEXT:    pcmpgtd %xmm6, %xmm4
+; SSSE3-NEXT:    pand %xmm4, %xmm9
+; SSSE3-NEXT:    pandn %xmm8, %xmm4
+; SSSE3-NEXT:    por %xmm9, %xmm4
+; SSSE3-NEXT:    packuswb %xmm3, %xmm4
+; SSSE3-NEXT:    packuswb %xmm4, %xmm0
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: trunc_usat_v32i32_v32i8:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255]
+; SSE41-NEXT:    movdqa 80(%rdi), %xmm0
+; SSE41-NEXT:    pminud %xmm2, %xmm0
+; SSE41-NEXT:    movdqa 64(%rdi), %xmm1
+; SSE41-NEXT:    pminud %xmm2, %xmm1
+; SSE41-NEXT:    packusdw %xmm0, %xmm1
+; SSE41-NEXT:    movdqa 112(%rdi), %xmm0
+; SSE41-NEXT:    pminud %xmm2, %xmm0
+; SSE41-NEXT:    movdqa 96(%rdi), %xmm3
+; SSE41-NEXT:    pminud %xmm2, %xmm3
+; SSE41-NEXT:    packusdw %xmm0, %xmm3
+; SSE41-NEXT:    packuswb %xmm3, %xmm1
+; SSE41-NEXT:    movdqa 16(%rdi), %xmm3
+; SSE41-NEXT:    pminud %xmm2, %xmm3
+; SSE41-NEXT:    movdqa (%rdi), %xmm0
+; SSE41-NEXT:    pminud %xmm2, %xmm0
+; SSE41-NEXT:    packusdw %xmm3, %xmm0
+; SSE41-NEXT:    movdqa 48(%rdi), %xmm3
+; SSE41-NEXT:    pminud %xmm2, %xmm3
+; SSE41-NEXT:    pminud 32(%rdi), %xmm2
+; SSE41-NEXT:    packusdw %xmm3, %xmm2
+; SSE41-NEXT:    packuswb %xmm2, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: trunc_usat_v32i32_v32i8:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm0 = [255,255,255,255]
+; AVX1-NEXT:    vpminud 16(%rdi), %xmm0, %xmm1
+; AVX1-NEXT:    vpminud (%rdi), %xmm0, %xmm2
+; AVX1-NEXT:    vpackusdw %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vpminud 48(%rdi), %xmm0, %xmm2
+; AVX1-NEXT:    vpminud 32(%rdi), %xmm0, %xmm3
+; AVX1-NEXT:    vpackusdw %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpminud 80(%rdi), %xmm0, %xmm2
+; AVX1-NEXT:    vpminud 64(%rdi), %xmm0, %xmm3
+; AVX1-NEXT:    vpackusdw %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vpminud 112(%rdi), %xmm0, %xmm3
+; AVX1-NEXT:    vpminud 96(%rdi), %xmm0, %xmm0
+; AVX1-NEXT:    vpackusdw %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpackuswb %xmm0, %xmm2, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: trunc_usat_v32i32_v32i8:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm0 = [255,255,255,255,255,255,255,255]
+; AVX2-NEXT:    vpminud 32(%rdi), %ymm0, %ymm1
+; AVX2-NEXT:    vpminud (%rdi), %ymm0, %ymm2
+; AVX2-NEXT:    vpackusdw %ymm1, %ymm2, %ymm1
+; AVX2-NEXT:    vpminud 96(%rdi), %ymm0, %ymm2
+; AVX2-NEXT:    vpminud 64(%rdi), %ymm0, %ymm0
+; AVX2-NEXT:    vpackusdw %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
+; AVX2-NEXT:    vpackuswb %ymm0, %ymm1, %ymm0
+; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: trunc_usat_v32i32_v32i8:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovdqa64 (%rdi), %zmm0
+; AVX512-NEXT:    vmovdqa64 64(%rdi), %zmm1
+; AVX512-NEXT:    vpmovusdb %zmm0, %xmm0
+; AVX512-NEXT:    vpmovusdb %zmm1, %xmm1
+; AVX512-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512-NEXT:    retq
+;
+; SKX-LABEL: trunc_usat_v32i32_v32i8:
+; SKX:       # %bb.0:
+; SKX-NEXT:    vmovdqa64 (%rdi), %zmm0
+; SKX-NEXT:    vmovdqa64 64(%rdi), %zmm1
+; SKX-NEXT:    vpmovusdb %zmm0, %xmm0
+; SKX-NEXT:    vpmovusdb %zmm1, %xmm1
+; SKX-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; SKX-NEXT:    retq
+  %a0 = load <32 x i32>, <32 x i32>* %p0
+  %1 = icmp ult <32 x i32> %a0, <i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255>
+  %2 = select <32 x i1> %1, <32 x i32> %a0, <32 x i32> <i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255>
+  %3 = trunc <32 x i32> %2 to <32 x i8>
+  ret <32 x i8> %3
+}
diff --git a/llvm/test/CodeGen/X86/vector-trunc.ll b/llvm/test/CodeGen/X86/vector-trunc.ll
index 35b190b11..f3bbc15c3 100644
--- a/llvm/test/CodeGen/X86/vector-trunc.ll
+++ b/llvm/test/CodeGen/X86/vector-trunc.ll
@@ -296,32 +296,22 @@ define void @trunc8i64_8i8(<8 x i64> %a) {
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
-; AVX2-SLOW-LABEL: trunc8i64_8i8:
-; AVX2-SLOW:       # %bb.0: # %entry
-; AVX2-SLOW-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm2[0,2]
-; AVX2-SLOW-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[0,2],xmm2[0,2]
-; AVX2-SLOW-NEXT:    vmovdqa {{.*#+}} xmm2 = <0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX2-SLOW-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX2-SLOW-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX2-SLOW-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; AVX2-SLOW-NEXT:    vmovq %xmm0, (%rax)
-; AVX2-SLOW-NEXT:    vzeroupper
-; AVX2-SLOW-NEXT:    retq
-;
-; AVX2-FAST-LABEL: trunc8i64_8i8:
-; AVX2-FAST:       # %bb.0: # %entry
-; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,2,4,6,4,6,6,7]
-; AVX2-FAST-NEXT:    vpermd %ymm0, %ymm2, %ymm0
-; AVX2-FAST-NEXT:    vpermd %ymm1, %ymm2, %ymm1
-; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} xmm2 = <0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX2-FAST-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX2-FAST-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX2-FAST-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; AVX2-FAST-NEXT:    vmovq %xmm0, (%rax)
-; AVX2-FAST-NEXT:    vzeroupper
-; AVX2-FAST-NEXT:    retq
+; AVX2-LABEL: trunc8i64_8i8:
+; AVX2:       # %bb.0: # %entry
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm3 = <u,u,0,8,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX2-NEXT:    vpshufb %xmm3, %xmm2, %xmm2
+; AVX2-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
+; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm2
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm3 = <0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX2-NEXT:    vpshufb %xmm3, %xmm2, %xmm2
+; AVX2-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
+; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3]
+; AVX2-NEXT:    vmovq %xmm0, (%rax)
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: trunc8i64_8i8:
 ; AVX512:       # %bb.0: # %entry
@@ -587,9 +577,11 @@ define void @trunc8i32_8i8(<8 x i32> %a) {
 ;
 ; AVX2-LABEL: trunc8i32_8i8:
 ; AVX2:       # %bb.0: # %entry
-; AVX2-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = <0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX2-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; AVX2-NEXT:    vmovq %xmm0, (%rax)
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -597,8 +589,7 @@ define void @trunc8i32_8i8(<8 x i32> %a) {
 ; AVX512F-LABEL: trunc8i32_8i8:
 ; AVX512F:       # %bb.0: # %entry
 ; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; AVX512F-NEXT:    vpmovdw %zmm0, %ymm0
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
+; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
 ; AVX512F-NEXT:    vmovq %xmm0, (%rax)
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
@@ -612,8 +603,7 @@ define void @trunc8i32_8i8(<8 x i32> %a) {
 ; AVX512BW-LABEL: trunc8i32_8i8:
 ; AVX512BW:       # %bb.0: # %entry
 ; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; AVX512BW-NEXT:    vpmovdw %zmm0, %ymm0
-; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
+; AVX512BW-NEXT:    vpmovdb %zmm0, %xmm0
 ; AVX512BW-NEXT:    vmovq %xmm0, (%rax)
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -1052,7 +1042,7 @@ entry:
 define void @trunc16i16_16i8(<16 x i16> %a) {
 ; SSE2-LABEL: trunc16i16_16i8:
 ; SSE2:       # %bb.0: # %entry
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
 ; SSE2-NEXT:    pand %xmm2, %xmm1
 ; SSE2-NEXT:    pand %xmm2, %xmm0
 ; SSE2-NEXT:    packuswb %xmm1, %xmm0
@@ -1261,7 +1251,7 @@ entry:
 define void @trunc32i16_32i8(<32 x i16> %a) {
 ; SSE2-LABEL: trunc32i16_32i8:
 ; SSE2:       # %bb.0: # %entry
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
 ; SSE2-NEXT:    pand %xmm4, %xmm1
 ; SSE2-NEXT:    pand %xmm4, %xmm0
 ; SSE2-NEXT:    packuswb %xmm1, %xmm0
@@ -1327,6 +1317,7 @@ define void @trunc32i16_32i8(<32 x i16> %a) {
 ;
 ; AVX512F-LABEL: trunc32i16_32i8:
 ; AVX512F:       # %bb.0: # %entry
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
 ; AVX512F-NEXT:    vpmovdb %zmm1, (%rax)
 ; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
@@ -1336,6 +1327,7 @@ define void @trunc32i16_32i8(<32 x i16> %a) {
 ;
 ; AVX512VL-LABEL: trunc32i16_32i8:
 ; AVX512VL:       # %bb.0: # %entry
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
 ; AVX512VL-NEXT:    vpmovdb %zmm1, (%rax)
 ; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
@@ -1480,39 +1472,53 @@ define <8 x i16> @trunc2x4i64_8i16(<4 x i64> %a, <4 x i64> %b) {
 ;
 ; AVX1-LABEL: trunc2x4i64_8i16:
 ; AVX1:       # %bb.0: # %entry
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX1-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm2[0,2]
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[0,2],xmm2[0,2]
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
-; AVX1-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm2[0,1,0,2,4,5,6,7]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,1,0,2,4,5,6,7]
+; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm2[0,2,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4,5,6,7]
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-SLOW-LABEL: trunc2x4i64_8i16:
 ; AVX2-SLOW:       # %bb.0: # %entry
-; AVX2-SLOW-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm2[0,2]
-; AVX2-SLOW-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[0,2],xmm2[0,2]
-; AVX2-SLOW-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
-; AVX2-SLOW-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX2-SLOW-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX2-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX2-SLOW-NEXT:    vextracti128 $1, %ymm1, %xmm2
+; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm2[0,1,0,2,4,5,6,7]
+; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,1,0,2,4,5,6,7]
+; AVX2-SLOW-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; AVX2-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm2
+; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm2[0,2,2,3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; AVX2-SLOW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
 ; AVX2-SLOW-NEXT:    vzeroupper
 ; AVX2-SLOW-NEXT:    retq
 ;
 ; AVX2-FAST-LABEL: trunc2x4i64_8i16:
 ; AVX2-FAST:       # %bb.0: # %entry
-; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,2,4,6,4,6,6,7]
-; AVX2-FAST-NEXT:    vpermd %ymm0, %ymm2, %ymm0
-; AVX2-FAST-NEXT:    vpermd %ymm1, %ymm2, %ymm1
-; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
-; AVX2-FAST-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX2-FAST-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX2-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX2-FAST-NEXT:    vextracti128 $1, %ymm1, %xmm2
+; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,1,2,3,0,1,8,9,8,9,10,11,12,13,14,15]
+; AVX2-FAST-NEXT:    vpshufb %xmm3, %xmm2, %xmm2
+; AVX2-FAST-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
+; AVX2-FAST-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; AVX2-FAST-NEXT:    vextracti128 $1, %ymm0, %xmm2
+; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,1,8,9,8,9,10,11,8,9,10,11,12,13,14,15]
+; AVX2-FAST-NEXT:    vpshufb %xmm3, %xmm2, %xmm2
+; AVX2-FAST-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
+; AVX2-FAST-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; AVX2-FAST-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
 ; AVX2-FAST-NEXT:    vzeroupper
 ; AVX2-FAST-NEXT:    retq
 ;
@@ -1520,22 +1526,16 @@ define <8 x i16> @trunc2x4i64_8i16(<4 x i64> %a, <4 x i64> %b) {
 ; AVX512F:       # %bb.0: # %entry
 ; AVX512F-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
 ; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; AVX512F-NEXT:    vpmovqd %zmm0, %ymm0
-; AVX512F-NEXT:    vpmovqd %zmm1, %ymm1
-; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
-; AVX512F-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX512F-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
+; AVX512F-NEXT:    vpmovqw %zmm0, %xmm0
+; AVX512F-NEXT:    vpmovqw %zmm1, %xmm1
 ; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: trunc2x4i64_8i16:
 ; AVX512VL:       # %bb.0: # %entry
-; AVX512VL-NEXT:    vpmovqd %ymm0, %xmm0
-; AVX512VL-NEXT:    vpmovqd %ymm1, %xmm1
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
-; AVX512VL-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpmovqw %ymm0, %xmm0
+; AVX512VL-NEXT:    vpmovqw %ymm1, %xmm1
 ; AVX512VL-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
 ; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq
@@ -1544,21 +1544,17 @@ define <8 x i16> @trunc2x4i64_8i16(<4 x i64> %a, <4 x i64> %b) {
 ; AVX512BW:       # %bb.0: # %entry
 ; AVX512BW-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
 ; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; AVX512BW-NEXT:    vpmovqd %zmm0, %ymm0
-; AVX512BW-NEXT:    vpmovqd %zmm1, %ymm1
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
-; AVX512BW-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpmovqw %zmm0, %xmm0
+; AVX512BW-NEXT:    vpmovqw %zmm1, %xmm1
 ; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512BWVL-LABEL: trunc2x4i64_8i16:
 ; AVX512BWVL:       # %bb.0: # %entry
-; AVX512BWVL-NEXT:    vpmovqd %ymm0, %xmm2
-; AVX512BWVL-NEXT:    vpmovqd %ymm1, %xmm1
-; AVX512BWVL-NEXT:    vmovdqa {{.*#+}} xmm0 = [0,2,4,6,8,10,12,14]
-; AVX512BWVL-NEXT:    vpermi2w %xmm1, %xmm2, %xmm0
+; AVX512BWVL-NEXT:    vpmovqw %ymm0, %xmm0
+; AVX512BWVL-NEXT:    vpmovqw %ymm1, %xmm1
+; AVX512BWVL-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
 entry:
@@ -1728,7 +1724,7 @@ entry:
 define <16 x i8> @trunc2x8i16_16i8(<8 x i16> %a, <8 x i16> %b) {
 ; SSE2-LABEL: trunc2x8i16_16i8:
 ; SSE2:       # %bb.0: # %entry
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
 ; SSE2-NEXT:    pand %xmm2, %xmm1
 ; SSE2-NEXT:    pand %xmm2, %xmm0
 ; SSE2-NEXT:    packuswb %xmm1, %xmm0
@@ -2004,3 +2000,129 @@ define void @PR34773(i16* %a0, i8* %a1) {
   store <16 x i8> %12, <16 x i8>* %6, align 1
   ret void
 }
+
+; Store merging must not infinitely fight store splitting.
+
+define void @store_merge_split(<8 x i32> %w1, <8 x i32> %w2, i64 %idx, <8 x i16>* %p) align 2 {
+; SSE2-LABEL: store_merge_split:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pslld $16, %xmm1
+; SSE2-NEXT:    psrad $16, %xmm1
+; SSE2-NEXT:    pslld $16, %xmm0
+; SSE2-NEXT:    psrad $16, %xmm0
+; SSE2-NEXT:    packssdw %xmm1, %xmm0
+; SSE2-NEXT:    pslld $16, %xmm3
+; SSE2-NEXT:    psrad $16, %xmm3
+; SSE2-NEXT:    pslld $16, %xmm2
+; SSE2-NEXT:    psrad $16, %xmm2
+; SSE2-NEXT:    packssdw %xmm3, %xmm2
+; SSE2-NEXT:    shlq $4, %rdi
+; SSE2-NEXT:    movdqu %xmm0, (%rsi,%rdi)
+; SSE2-NEXT:    movdqu %xmm2, 16(%rsi,%rdi)
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: store_merge_split:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm4 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
+; SSSE3-NEXT:    pshufb %xmm4, %xmm1
+; SSSE3-NEXT:    pshufb %xmm4, %xmm0
+; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSSE3-NEXT:    pshufb %xmm4, %xmm3
+; SSSE3-NEXT:    pshufb %xmm4, %xmm2
+; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; SSSE3-NEXT:    shlq $4, %rdi
+; SSSE3-NEXT:    movdqu %xmm0, (%rsi,%rdi)
+; SSSE3-NEXT:    movdqu %xmm2, 16(%rsi,%rdi)
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: store_merge_split:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
+; SSE41-NEXT:    pshufb %xmm4, %xmm1
+; SSE41-NEXT:    pshufb %xmm4, %xmm0
+; SSE41-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE41-NEXT:    pshufb %xmm4, %xmm3
+; SSE41-NEXT:    pshufb %xmm4, %xmm2
+; SSE41-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; SSE41-NEXT:    shlq $4, %rdi
+; SSE41-NEXT:    movdqu %xmm0, (%rsi,%rdi)
+; SSE41-NEXT:    movdqu %xmm2, 16(%rsi,%rdi)
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: store_merge_split:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
+; AVX1-NEXT:    vpshufb %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT:    vpshufb %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; AVX1-NEXT:    shlq $4, %rdi
+; AVX1-NEXT:    vmovdqu %xmm0, (%rsi,%rdi)
+; AVX1-NEXT:    vmovdqu %xmm1, 16(%rsi,%rdi)
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: store_merge_split:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
+; AVX2-NEXT:    vpshufb %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; AVX2-NEXT:    vpshufb %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3]
+; AVX2-NEXT:    shlq $4, %rdi
+; AVX2-NEXT:    vmovdqu %xmm0, (%rsi,%rdi)
+; AVX2-NEXT:    vmovdqu %xmm1, 16(%rsi,%rdi)
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+;
+; AVX512F-LABEL: store_merge_split:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512F-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512F-NEXT:    vpmovdw %zmm1, %ymm1
+; AVX512F-NEXT:    shlq $4, %rdi
+; AVX512F-NEXT:    vmovdqu %xmm0, (%rsi,%rdi)
+; AVX512F-NEXT:    vmovdqu %xmm1, 16(%rsi,%rdi)
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: store_merge_split:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    shlq $4, %rdi
+; AVX512VL-NEXT:    vpmovdw %ymm0, (%rsi,%rdi)
+; AVX512VL-NEXT:    vpmovdw %ymm1, 16(%rsi,%rdi)
+; AVX512VL-NEXT:    vzeroupper
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: store_merge_split:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512BW-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512BW-NEXT:    vpmovdw %zmm1, %ymm1
+; AVX512BW-NEXT:    shlq $4, %rdi
+; AVX512BW-NEXT:    vmovdqu %xmm0, (%rsi,%rdi)
+; AVX512BW-NEXT:    vmovdqu %xmm1, 16(%rsi,%rdi)
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: store_merge_split:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    shlq $4, %rdi
+; AVX512BWVL-NEXT:    vpmovdw %ymm0, (%rsi,%rdi)
+; AVX512BWVL-NEXT:    vpmovdw %ymm1, 16(%rsi,%rdi)
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
+  %t1 = trunc <8 x i32> %w1 to <8 x i16>
+  %t2 = trunc <8 x i32> %w2 to <8 x i16>
+  %g1 = getelementptr inbounds <8 x i16>, <8 x i16>* %p, i64 %idx
+  %g2 = getelementptr inbounds <8 x i16>, <8 x i16>* %g1, i64 1
+  store <8 x i16> %t1, <8 x i16>* %g1, align 2
+  store <8 x i16> %t2, <8 x i16>* %g2, align 2
+  ret void
+}
diff --git a/llvm/test/CodeGen/X86/vector-truncate-combine.ll b/llvm/test/CodeGen/X86/vector-truncate-combine.ll
index 5b8498e77..ae2865863 100644
--- a/llvm/test/CodeGen/X86/vector-truncate-combine.ll
+++ b/llvm/test/CodeGen/X86/vector-truncate-combine.ll
@@ -14,12 +14,8 @@
 ; NOTE: This operation could be collapsed in to a single truncate. Once that is done
 ; this test will have to be adjusted.
 
-; CHECK:      PUNPCKLBWrr
-; CHECK:      PUNPCKLWDrr
 ; CHECK:      PANDrm
 ; CHECK:      PACKUSWBrr
-; CHECK:      PACKUSWBrr
-; CHECK:      PACKUSWBrr
 ; CHECK:      MOVPDI2DIrr
 
 define void @test(double %vec.coerce) local_unnamed_addr {
diff --git a/llvm/test/CodeGen/X86/vector-tzcnt-512.ll b/llvm/test/CodeGen/X86/vector-tzcnt-512.ll
index 501d7e968..83f0d25ee 100644
--- a/llvm/test/CodeGen/X86/vector-tzcnt-512.ll
+++ b/llvm/test/CodeGen/X86/vector-tzcnt-512.ll
@@ -260,31 +260,33 @@ define <16 x i32> @testv16i32u(<16 x i32> %in) nounwind {
 define <32 x i16> @testv32i16(<32 x i16> %in) nounwind {
 ; AVX512CD-LABEL: testv32i16:
 ; AVX512CD:       # %bb.0:
+; AVX512CD-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512CD-NEXT:    vpcmpeqd %ymm2, %ymm2, %ymm2
-; AVX512CD-NEXT:    vpaddw %ymm2, %ymm0, %ymm3
-; AVX512CD-NEXT:    vpandn %ymm3, %ymm0, %ymm0
+; AVX512CD-NEXT:    vpaddw %ymm2, %ymm1, %ymm3
+; AVX512CD-NEXT:    vpandn %ymm3, %ymm1, %ymm1
 ; AVX512CD-NEXT:    vmovdqa {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512CD-NEXT:    vpand %ymm3, %ymm0, %ymm4
+; AVX512CD-NEXT:    vpand %ymm3, %ymm1, %ymm4
 ; AVX512CD-NEXT:    vmovdqa {{.*#+}} ymm5 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
 ; AVX512CD-NEXT:    vpshufb %ymm4, %ymm5, %ymm4
-; AVX512CD-NEXT:    vpsrlw $4, %ymm0, %ymm0
-; AVX512CD-NEXT:    vpand %ymm3, %ymm0, %ymm0
-; AVX512CD-NEXT:    vpshufb %ymm0, %ymm5, %ymm0
-; AVX512CD-NEXT:    vpaddb %ymm4, %ymm0, %ymm0
-; AVX512CD-NEXT:    vpsllw $8, %ymm0, %ymm4
-; AVX512CD-NEXT:    vpaddb %ymm0, %ymm4, %ymm0
-; AVX512CD-NEXT:    vpsrlw $8, %ymm0, %ymm0
-; AVX512CD-NEXT:    vpaddw %ymm2, %ymm1, %ymm2
-; AVX512CD-NEXT:    vpandn %ymm2, %ymm1, %ymm1
-; AVX512CD-NEXT:    vpand %ymm3, %ymm1, %ymm2
-; AVX512CD-NEXT:    vpshufb %ymm2, %ymm5, %ymm2
 ; AVX512CD-NEXT:    vpsrlw $4, %ymm1, %ymm1
 ; AVX512CD-NEXT:    vpand %ymm3, %ymm1, %ymm1
 ; AVX512CD-NEXT:    vpshufb %ymm1, %ymm5, %ymm1
-; AVX512CD-NEXT:    vpaddb %ymm2, %ymm1, %ymm1
-; AVX512CD-NEXT:    vpsllw $8, %ymm1, %ymm2
-; AVX512CD-NEXT:    vpaddb %ymm1, %ymm2, %ymm1
+; AVX512CD-NEXT:    vpaddb %ymm4, %ymm1, %ymm1
+; AVX512CD-NEXT:    vpsllw $8, %ymm1, %ymm4
+; AVX512CD-NEXT:    vpaddb %ymm1, %ymm4, %ymm1
 ; AVX512CD-NEXT:    vpsrlw $8, %ymm1, %ymm1
+; AVX512CD-NEXT:    vpaddw %ymm2, %ymm0, %ymm2
+; AVX512CD-NEXT:    vpandn %ymm2, %ymm0, %ymm0
+; AVX512CD-NEXT:    vpand %ymm3, %ymm0, %ymm2
+; AVX512CD-NEXT:    vpshufb %ymm2, %ymm5, %ymm2
+; AVX512CD-NEXT:    vpsrlw $4, %ymm0, %ymm0
+; AVX512CD-NEXT:    vpand %ymm3, %ymm0, %ymm0
+; AVX512CD-NEXT:    vpshufb %ymm0, %ymm5, %ymm0
+; AVX512CD-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
+; AVX512CD-NEXT:    vpsllw $8, %ymm0, %ymm2
+; AVX512CD-NEXT:    vpaddb %ymm0, %ymm2, %ymm0
+; AVX512CD-NEXT:    vpsrlw $8, %ymm0, %ymm0
+; AVX512CD-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512CD-NEXT:    retq
 ;
 ; AVX512CDBW-LABEL: testv32i16:
@@ -325,17 +327,19 @@ define <32 x i16> @testv32i16(<32 x i16> %in) nounwind {
 ;
 ; AVX512VPOPCNTDQ-LABEL: testv32i16:
 ; AVX512VPOPCNTDQ:       # %bb.0:
+; AVX512VPOPCNTDQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512VPOPCNTDQ-NEXT:    vpcmpeqd %ymm2, %ymm2, %ymm2
-; AVX512VPOPCNTDQ-NEXT:    vpaddw %ymm2, %ymm0, %ymm3
-; AVX512VPOPCNTDQ-NEXT:    vpandn %ymm3, %ymm0, %ymm0
-; AVX512VPOPCNTDQ-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
-; AVX512VPOPCNTDQ-NEXT:    vpopcntd %zmm0, %zmm0
-; AVX512VPOPCNTDQ-NEXT:    vpmovdw %zmm0, %ymm0
-; AVX512VPOPCNTDQ-NEXT:    vpaddw %ymm2, %ymm1, %ymm2
-; AVX512VPOPCNTDQ-NEXT:    vpandn %ymm2, %ymm1, %ymm1
+; AVX512VPOPCNTDQ-NEXT:    vpaddw %ymm2, %ymm1, %ymm3
+; AVX512VPOPCNTDQ-NEXT:    vpandn %ymm3, %ymm1, %ymm1
 ; AVX512VPOPCNTDQ-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
 ; AVX512VPOPCNTDQ-NEXT:    vpopcntd %zmm1, %zmm1
 ; AVX512VPOPCNTDQ-NEXT:    vpmovdw %zmm1, %ymm1
+; AVX512VPOPCNTDQ-NEXT:    vpaddw %ymm2, %ymm0, %ymm2
+; AVX512VPOPCNTDQ-NEXT:    vpandn %ymm2, %ymm0, %ymm0
+; AVX512VPOPCNTDQ-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
+; AVX512VPOPCNTDQ-NEXT:    vpopcntd %zmm0, %zmm0
+; AVX512VPOPCNTDQ-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512VPOPCNTDQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512VPOPCNTDQ-NEXT:    retq
 ;
 ; BITALG-LABEL: testv32i16:
@@ -352,31 +356,33 @@ define <32 x i16> @testv32i16(<32 x i16> %in) nounwind {
 define <32 x i16> @testv32i16u(<32 x i16> %in) nounwind {
 ; AVX512CD-LABEL: testv32i16u:
 ; AVX512CD:       # %bb.0:
+; AVX512CD-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512CD-NEXT:    vpcmpeqd %ymm2, %ymm2, %ymm2
-; AVX512CD-NEXT:    vpaddw %ymm2, %ymm0, %ymm3
-; AVX512CD-NEXT:    vpandn %ymm3, %ymm0, %ymm0
+; AVX512CD-NEXT:    vpaddw %ymm2, %ymm1, %ymm3
+; AVX512CD-NEXT:    vpandn %ymm3, %ymm1, %ymm1
 ; AVX512CD-NEXT:    vmovdqa {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512CD-NEXT:    vpand %ymm3, %ymm0, %ymm4
+; AVX512CD-NEXT:    vpand %ymm3, %ymm1, %ymm4
 ; AVX512CD-NEXT:    vmovdqa {{.*#+}} ymm5 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
 ; AVX512CD-NEXT:    vpshufb %ymm4, %ymm5, %ymm4
-; AVX512CD-NEXT:    vpsrlw $4, %ymm0, %ymm0
-; AVX512CD-NEXT:    vpand %ymm3, %ymm0, %ymm0
-; AVX512CD-NEXT:    vpshufb %ymm0, %ymm5, %ymm0
-; AVX512CD-NEXT:    vpaddb %ymm4, %ymm0, %ymm0
-; AVX512CD-NEXT:    vpsllw $8, %ymm0, %ymm4
-; AVX512CD-NEXT:    vpaddb %ymm0, %ymm4, %ymm0
-; AVX512CD-NEXT:    vpsrlw $8, %ymm0, %ymm0
-; AVX512CD-NEXT:    vpaddw %ymm2, %ymm1, %ymm2
-; AVX512CD-NEXT:    vpandn %ymm2, %ymm1, %ymm1
-; AVX512CD-NEXT:    vpand %ymm3, %ymm1, %ymm2
-; AVX512CD-NEXT:    vpshufb %ymm2, %ymm5, %ymm2
 ; AVX512CD-NEXT:    vpsrlw $4, %ymm1, %ymm1
 ; AVX512CD-NEXT:    vpand %ymm3, %ymm1, %ymm1
 ; AVX512CD-NEXT:    vpshufb %ymm1, %ymm5, %ymm1
-; AVX512CD-NEXT:    vpaddb %ymm2, %ymm1, %ymm1
-; AVX512CD-NEXT:    vpsllw $8, %ymm1, %ymm2
-; AVX512CD-NEXT:    vpaddb %ymm1, %ymm2, %ymm1
+; AVX512CD-NEXT:    vpaddb %ymm4, %ymm1, %ymm1
+; AVX512CD-NEXT:    vpsllw $8, %ymm1, %ymm4
+; AVX512CD-NEXT:    vpaddb %ymm1, %ymm4, %ymm1
 ; AVX512CD-NEXT:    vpsrlw $8, %ymm1, %ymm1
+; AVX512CD-NEXT:    vpaddw %ymm2, %ymm0, %ymm2
+; AVX512CD-NEXT:    vpandn %ymm2, %ymm0, %ymm0
+; AVX512CD-NEXT:    vpand %ymm3, %ymm0, %ymm2
+; AVX512CD-NEXT:    vpshufb %ymm2, %ymm5, %ymm2
+; AVX512CD-NEXT:    vpsrlw $4, %ymm0, %ymm0
+; AVX512CD-NEXT:    vpand %ymm3, %ymm0, %ymm0
+; AVX512CD-NEXT:    vpshufb %ymm0, %ymm5, %ymm0
+; AVX512CD-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
+; AVX512CD-NEXT:    vpsllw $8, %ymm0, %ymm2
+; AVX512CD-NEXT:    vpaddb %ymm0, %ymm2, %ymm0
+; AVX512CD-NEXT:    vpsrlw $8, %ymm0, %ymm0
+; AVX512CD-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512CD-NEXT:    retq
 ;
 ; AVX512CDBW-LABEL: testv32i16u:
@@ -417,17 +423,19 @@ define <32 x i16> @testv32i16u(<32 x i16> %in) nounwind {
 ;
 ; AVX512VPOPCNTDQ-LABEL: testv32i16u:
 ; AVX512VPOPCNTDQ:       # %bb.0:
+; AVX512VPOPCNTDQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512VPOPCNTDQ-NEXT:    vpcmpeqd %ymm2, %ymm2, %ymm2
-; AVX512VPOPCNTDQ-NEXT:    vpaddw %ymm2, %ymm0, %ymm3
-; AVX512VPOPCNTDQ-NEXT:    vpandn %ymm3, %ymm0, %ymm0
-; AVX512VPOPCNTDQ-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
-; AVX512VPOPCNTDQ-NEXT:    vpopcntd %zmm0, %zmm0
-; AVX512VPOPCNTDQ-NEXT:    vpmovdw %zmm0, %ymm0
-; AVX512VPOPCNTDQ-NEXT:    vpaddw %ymm2, %ymm1, %ymm2
-; AVX512VPOPCNTDQ-NEXT:    vpandn %ymm2, %ymm1, %ymm1
+; AVX512VPOPCNTDQ-NEXT:    vpaddw %ymm2, %ymm1, %ymm3
+; AVX512VPOPCNTDQ-NEXT:    vpandn %ymm3, %ymm1, %ymm1
 ; AVX512VPOPCNTDQ-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
 ; AVX512VPOPCNTDQ-NEXT:    vpopcntd %zmm1, %zmm1
 ; AVX512VPOPCNTDQ-NEXT:    vpmovdw %zmm1, %ymm1
+; AVX512VPOPCNTDQ-NEXT:    vpaddw %ymm2, %ymm0, %ymm2
+; AVX512VPOPCNTDQ-NEXT:    vpandn %ymm2, %ymm0, %ymm0
+; AVX512VPOPCNTDQ-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
+; AVX512VPOPCNTDQ-NEXT:    vpopcntd %zmm0, %zmm0
+; AVX512VPOPCNTDQ-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512VPOPCNTDQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512VPOPCNTDQ-NEXT:    retq
 ;
 ; BITALG-LABEL: testv32i16u:
@@ -444,25 +452,27 @@ define <32 x i16> @testv32i16u(<32 x i16> %in) nounwind {
 define <64 x i8> @testv64i8(<64 x i8> %in) nounwind {
 ; AVX512CD-LABEL: testv64i8:
 ; AVX512CD:       # %bb.0:
+; AVX512CD-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512CD-NEXT:    vpcmpeqd %ymm2, %ymm2, %ymm2
-; AVX512CD-NEXT:    vpaddb %ymm2, %ymm0, %ymm3
-; AVX512CD-NEXT:    vpandn %ymm3, %ymm0, %ymm0
+; AVX512CD-NEXT:    vpaddb %ymm2, %ymm1, %ymm3
+; AVX512CD-NEXT:    vpandn %ymm3, %ymm1, %ymm1
 ; AVX512CD-NEXT:    vmovdqa {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512CD-NEXT:    vpand %ymm3, %ymm0, %ymm4
+; AVX512CD-NEXT:    vpand %ymm3, %ymm1, %ymm4
 ; AVX512CD-NEXT:    vmovdqa {{.*#+}} ymm5 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
 ; AVX512CD-NEXT:    vpshufb %ymm4, %ymm5, %ymm4
-; AVX512CD-NEXT:    vpsrlw $4, %ymm0, %ymm0
-; AVX512CD-NEXT:    vpand %ymm3, %ymm0, %ymm0
-; AVX512CD-NEXT:    vpshufb %ymm0, %ymm5, %ymm0
-; AVX512CD-NEXT:    vpaddb %ymm4, %ymm0, %ymm0
-; AVX512CD-NEXT:    vpaddb %ymm2, %ymm1, %ymm2
-; AVX512CD-NEXT:    vpandn %ymm2, %ymm1, %ymm1
-; AVX512CD-NEXT:    vpand %ymm3, %ymm1, %ymm2
-; AVX512CD-NEXT:    vpshufb %ymm2, %ymm5, %ymm2
 ; AVX512CD-NEXT:    vpsrlw $4, %ymm1, %ymm1
 ; AVX512CD-NEXT:    vpand %ymm3, %ymm1, %ymm1
 ; AVX512CD-NEXT:    vpshufb %ymm1, %ymm5, %ymm1
-; AVX512CD-NEXT:    vpaddb %ymm2, %ymm1, %ymm1
+; AVX512CD-NEXT:    vpaddb %ymm4, %ymm1, %ymm1
+; AVX512CD-NEXT:    vpaddb %ymm2, %ymm0, %ymm2
+; AVX512CD-NEXT:    vpandn %ymm2, %ymm0, %ymm0
+; AVX512CD-NEXT:    vpand %ymm3, %ymm0, %ymm2
+; AVX512CD-NEXT:    vpshufb %ymm2, %ymm5, %ymm2
+; AVX512CD-NEXT:    vpsrlw $4, %ymm0, %ymm0
+; AVX512CD-NEXT:    vpand %ymm3, %ymm0, %ymm0
+; AVX512CD-NEXT:    vpshufb %ymm0, %ymm5, %ymm0
+; AVX512CD-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
+; AVX512CD-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512CD-NEXT:    retq
 ;
 ; AVX512CDBW-LABEL: testv64i8:
@@ -497,25 +507,27 @@ define <64 x i8> @testv64i8(<64 x i8> %in) nounwind {
 ;
 ; AVX512VPOPCNTDQ-LABEL: testv64i8:
 ; AVX512VPOPCNTDQ:       # %bb.0:
+; AVX512VPOPCNTDQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512VPOPCNTDQ-NEXT:    vpcmpeqd %ymm2, %ymm2, %ymm2
-; AVX512VPOPCNTDQ-NEXT:    vpaddb %ymm2, %ymm0, %ymm3
-; AVX512VPOPCNTDQ-NEXT:    vpandn %ymm3, %ymm0, %ymm0
+; AVX512VPOPCNTDQ-NEXT:    vpaddb %ymm2, %ymm1, %ymm3
+; AVX512VPOPCNTDQ-NEXT:    vpandn %ymm3, %ymm1, %ymm1
 ; AVX512VPOPCNTDQ-NEXT:    vmovdqa {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512VPOPCNTDQ-NEXT:    vpand %ymm3, %ymm0, %ymm4
+; AVX512VPOPCNTDQ-NEXT:    vpand %ymm3, %ymm1, %ymm4
 ; AVX512VPOPCNTDQ-NEXT:    vmovdqa {{.*#+}} ymm5 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
 ; AVX512VPOPCNTDQ-NEXT:    vpshufb %ymm4, %ymm5, %ymm4
-; AVX512VPOPCNTDQ-NEXT:    vpsrlw $4, %ymm0, %ymm0
-; AVX512VPOPCNTDQ-NEXT:    vpand %ymm3, %ymm0, %ymm0
-; AVX512VPOPCNTDQ-NEXT:    vpshufb %ymm0, %ymm5, %ymm0
-; AVX512VPOPCNTDQ-NEXT:    vpaddb %ymm4, %ymm0, %ymm0
-; AVX512VPOPCNTDQ-NEXT:    vpaddb %ymm2, %ymm1, %ymm2
-; AVX512VPOPCNTDQ-NEXT:    vpandn %ymm2, %ymm1, %ymm1
-; AVX512VPOPCNTDQ-NEXT:    vpand %ymm3, %ymm1, %ymm2
-; AVX512VPOPCNTDQ-NEXT:    vpshufb %ymm2, %ymm5, %ymm2
 ; AVX512VPOPCNTDQ-NEXT:    vpsrlw $4, %ymm1, %ymm1
 ; AVX512VPOPCNTDQ-NEXT:    vpand %ymm3, %ymm1, %ymm1
 ; AVX512VPOPCNTDQ-NEXT:    vpshufb %ymm1, %ymm5, %ymm1
-; AVX512VPOPCNTDQ-NEXT:    vpaddb %ymm2, %ymm1, %ymm1
+; AVX512VPOPCNTDQ-NEXT:    vpaddb %ymm4, %ymm1, %ymm1
+; AVX512VPOPCNTDQ-NEXT:    vpaddb %ymm2, %ymm0, %ymm2
+; AVX512VPOPCNTDQ-NEXT:    vpandn %ymm2, %ymm0, %ymm0
+; AVX512VPOPCNTDQ-NEXT:    vpand %ymm3, %ymm0, %ymm2
+; AVX512VPOPCNTDQ-NEXT:    vpshufb %ymm2, %ymm5, %ymm2
+; AVX512VPOPCNTDQ-NEXT:    vpsrlw $4, %ymm0, %ymm0
+; AVX512VPOPCNTDQ-NEXT:    vpand %ymm3, %ymm0, %ymm0
+; AVX512VPOPCNTDQ-NEXT:    vpshufb %ymm0, %ymm5, %ymm0
+; AVX512VPOPCNTDQ-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
+; AVX512VPOPCNTDQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512VPOPCNTDQ-NEXT:    retq
 ;
 ; BITALG-LABEL: testv64i8:
@@ -532,25 +544,27 @@ define <64 x i8> @testv64i8(<64 x i8> %in) nounwind {
 define <64 x i8> @testv64i8u(<64 x i8> %in) nounwind {
 ; AVX512CD-LABEL: testv64i8u:
 ; AVX512CD:       # %bb.0:
+; AVX512CD-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512CD-NEXT:    vpcmpeqd %ymm2, %ymm2, %ymm2
-; AVX512CD-NEXT:    vpaddb %ymm2, %ymm0, %ymm3
-; AVX512CD-NEXT:    vpandn %ymm3, %ymm0, %ymm0
+; AVX512CD-NEXT:    vpaddb %ymm2, %ymm1, %ymm3
+; AVX512CD-NEXT:    vpandn %ymm3, %ymm1, %ymm1
 ; AVX512CD-NEXT:    vmovdqa {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512CD-NEXT:    vpand %ymm3, %ymm0, %ymm4
+; AVX512CD-NEXT:    vpand %ymm3, %ymm1, %ymm4
 ; AVX512CD-NEXT:    vmovdqa {{.*#+}} ymm5 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
 ; AVX512CD-NEXT:    vpshufb %ymm4, %ymm5, %ymm4
-; AVX512CD-NEXT:    vpsrlw $4, %ymm0, %ymm0
-; AVX512CD-NEXT:    vpand %ymm3, %ymm0, %ymm0
-; AVX512CD-NEXT:    vpshufb %ymm0, %ymm5, %ymm0
-; AVX512CD-NEXT:    vpaddb %ymm4, %ymm0, %ymm0
-; AVX512CD-NEXT:    vpaddb %ymm2, %ymm1, %ymm2
-; AVX512CD-NEXT:    vpandn %ymm2, %ymm1, %ymm1
-; AVX512CD-NEXT:    vpand %ymm3, %ymm1, %ymm2
-; AVX512CD-NEXT:    vpshufb %ymm2, %ymm5, %ymm2
 ; AVX512CD-NEXT:    vpsrlw $4, %ymm1, %ymm1
 ; AVX512CD-NEXT:    vpand %ymm3, %ymm1, %ymm1
 ; AVX512CD-NEXT:    vpshufb %ymm1, %ymm5, %ymm1
-; AVX512CD-NEXT:    vpaddb %ymm2, %ymm1, %ymm1
+; AVX512CD-NEXT:    vpaddb %ymm4, %ymm1, %ymm1
+; AVX512CD-NEXT:    vpaddb %ymm2, %ymm0, %ymm2
+; AVX512CD-NEXT:    vpandn %ymm2, %ymm0, %ymm0
+; AVX512CD-NEXT:    vpand %ymm3, %ymm0, %ymm2
+; AVX512CD-NEXT:    vpshufb %ymm2, %ymm5, %ymm2
+; AVX512CD-NEXT:    vpsrlw $4, %ymm0, %ymm0
+; AVX512CD-NEXT:    vpand %ymm3, %ymm0, %ymm0
+; AVX512CD-NEXT:    vpshufb %ymm0, %ymm5, %ymm0
+; AVX512CD-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
+; AVX512CD-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512CD-NEXT:    retq
 ;
 ; AVX512CDBW-LABEL: testv64i8u:
@@ -585,25 +599,27 @@ define <64 x i8> @testv64i8u(<64 x i8> %in) nounwind {
 ;
 ; AVX512VPOPCNTDQ-LABEL: testv64i8u:
 ; AVX512VPOPCNTDQ:       # %bb.0:
+; AVX512VPOPCNTDQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512VPOPCNTDQ-NEXT:    vpcmpeqd %ymm2, %ymm2, %ymm2
-; AVX512VPOPCNTDQ-NEXT:    vpaddb %ymm2, %ymm0, %ymm3
-; AVX512VPOPCNTDQ-NEXT:    vpandn %ymm3, %ymm0, %ymm0
+; AVX512VPOPCNTDQ-NEXT:    vpaddb %ymm2, %ymm1, %ymm3
+; AVX512VPOPCNTDQ-NEXT:    vpandn %ymm3, %ymm1, %ymm1
 ; AVX512VPOPCNTDQ-NEXT:    vmovdqa {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512VPOPCNTDQ-NEXT:    vpand %ymm3, %ymm0, %ymm4
+; AVX512VPOPCNTDQ-NEXT:    vpand %ymm3, %ymm1, %ymm4
 ; AVX512VPOPCNTDQ-NEXT:    vmovdqa {{.*#+}} ymm5 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
 ; AVX512VPOPCNTDQ-NEXT:    vpshufb %ymm4, %ymm5, %ymm4
-; AVX512VPOPCNTDQ-NEXT:    vpsrlw $4, %ymm0, %ymm0
-; AVX512VPOPCNTDQ-NEXT:    vpand %ymm3, %ymm0, %ymm0
-; AVX512VPOPCNTDQ-NEXT:    vpshufb %ymm0, %ymm5, %ymm0
-; AVX512VPOPCNTDQ-NEXT:    vpaddb %ymm4, %ymm0, %ymm0
-; AVX512VPOPCNTDQ-NEXT:    vpaddb %ymm2, %ymm1, %ymm2
-; AVX512VPOPCNTDQ-NEXT:    vpandn %ymm2, %ymm1, %ymm1
-; AVX512VPOPCNTDQ-NEXT:    vpand %ymm3, %ymm1, %ymm2
-; AVX512VPOPCNTDQ-NEXT:    vpshufb %ymm2, %ymm5, %ymm2
 ; AVX512VPOPCNTDQ-NEXT:    vpsrlw $4, %ymm1, %ymm1
 ; AVX512VPOPCNTDQ-NEXT:    vpand %ymm3, %ymm1, %ymm1
 ; AVX512VPOPCNTDQ-NEXT:    vpshufb %ymm1, %ymm5, %ymm1
-; AVX512VPOPCNTDQ-NEXT:    vpaddb %ymm2, %ymm1, %ymm1
+; AVX512VPOPCNTDQ-NEXT:    vpaddb %ymm4, %ymm1, %ymm1
+; AVX512VPOPCNTDQ-NEXT:    vpaddb %ymm2, %ymm0, %ymm2
+; AVX512VPOPCNTDQ-NEXT:    vpandn %ymm2, %ymm0, %ymm0
+; AVX512VPOPCNTDQ-NEXT:    vpand %ymm3, %ymm0, %ymm2
+; AVX512VPOPCNTDQ-NEXT:    vpshufb %ymm2, %ymm5, %ymm2
+; AVX512VPOPCNTDQ-NEXT:    vpsrlw $4, %ymm0, %ymm0
+; AVX512VPOPCNTDQ-NEXT:    vpand %ymm3, %ymm0, %ymm0
+; AVX512VPOPCNTDQ-NEXT:    vpshufb %ymm0, %ymm5, %ymm0
+; AVX512VPOPCNTDQ-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
+; AVX512VPOPCNTDQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512VPOPCNTDQ-NEXT:    retq
 ;
 ; BITALG-LABEL: testv64i8u:
diff --git a/llvm/test/CodeGen/X86/vector-variable-idx.ll b/llvm/test/CodeGen/X86/vector-variable-idx.ll
index 7d37b1a4c..091b553a8 100644
--- a/llvm/test/CodeGen/X86/vector-variable-idx.ll
+++ b/llvm/test/CodeGen/X86/vector-variable-idx.ll
@@ -1,11 +1,27 @@
-; RUN: llc < %s -mtriple=x86_64-- | grep movss | count 2
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-- | FileCheck %s
 ; PR2676
 
 define float @foo(<4 x float> %p, i32 %t) {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    # kill: def $edi killed $edi def $rdi
+; CHECK-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    andl $3, %edi
+; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; CHECK-NEXT:    retq
   %z = extractelement <4 x float> %p, i32 %t
   ret float %z
 }
 define <4 x float> @bar(<4 x float> %p, float %f, i32 %t) {
+; CHECK-LABEL: bar:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    # kill: def $edi killed $edi def $rdi
+; CHECK-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    andl $3, %edi
+; CHECK-NEXT:    movss %xmm1, -24(%rsp,%rdi,4)
+; CHECK-NEXT:    movaps -{{[0-9]+}}(%rsp), %xmm0
+; CHECK-NEXT:    retq
   %z = insertelement <4 x float> %p, float %f, i32 %t
   ret <4 x float> %z
 }
diff --git a/llvm/test/CodeGen/X86/vector-width-store-merge.ll b/llvm/test/CodeGen/X86/vector-width-store-merge.ll
index e316e08eb..79f04c0b3 100644
--- a/llvm/test/CodeGen/X86/vector-width-store-merge.ll
+++ b/llvm/test/CodeGen/X86/vector-width-store-merge.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-- | FileCheck %s
 
 ; This tests whether or not we generate vectors large than preferred vector width when
@@ -5,40 +6,60 @@
 
 ; Function Attrs: nounwind uwtable
 define weak_odr dso_local void @A(i8* %src, i8* %dst) local_unnamed_addr #0 {
+; CHECK-LABEL: A:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmovups (%rdi), %xmm0
+; CHECK-NEXT:    vmovups 16(%rdi), %xmm1
+; CHECK-NEXT:    vmovups %xmm1, 16(%rsi)
+; CHECK-NEXT:    vmovups %xmm0, (%rsi)
+; CHECK-NEXT:    retq
 entry:
-; CHECK: A
-; CHECK-NOT: vmovups %ymm
-; CHECK: vmovups %xmm
   call void @llvm.memmove.p0i8.p0i8.i64(i8* align 1 %dst, i8* align 1 %src, i64 32, i1 false)
   ret void
 }
 
 ; Function Attrs: nounwind uwtable
 define weak_odr dso_local void @B(i8* %src, i8* %dst) local_unnamed_addr #0 {
+; CHECK-LABEL: B:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmovups (%rdi), %xmm0
+; CHECK-NEXT:    vmovups 16(%rdi), %xmm1
+; CHECK-NEXT:    vmovups 32(%rdi), %xmm2
+; CHECK-NEXT:    vmovups 48(%rdi), %xmm3
+; CHECK-NEXT:    vmovups %xmm3, 48(%rsi)
+; CHECK-NEXT:    vmovups %xmm2, 32(%rsi)
+; CHECK-NEXT:    vmovups %xmm1, 16(%rsi)
+; CHECK-NEXT:    vmovups %xmm0, (%rsi)
+; CHECK-NEXT:    retq
 entry:
-; CHECK: B
-; CHECK-NOT: vmovups %zmm
-; CHECK: vmovups %xmm
   call void @llvm.memmove.p0i8.p0i8.i64(i8* align 1 %dst, i8* align 1 %src, i64 64, i1 false)
   ret void
 }
 
 ; Function Attrs: nounwind uwtable
 define weak_odr dso_local void @C(i8* %src, i8* %dst) local_unnamed_addr #2 {
+; CHECK-LABEL: C:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmovups (%rdi), %ymm0
+; CHECK-NEXT:    vmovups %ymm0, (%rsi)
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
 entry:
-; CHECK: C
-; CHECK-NOT: vmovups %ymm
-; CHECK: vmovups %ymm
   call void @llvm.memmove.p0i8.p0i8.i64(i8* align 1 %dst, i8* align 1 %src, i64 32, i1 false)
   ret void
 }
 
 ; Function Attrs: nounwind uwtable
 define weak_odr dso_local void @D(i8* %src, i8* %dst) local_unnamed_addr #2 {
+; CHECK-LABEL: D:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmovups (%rdi), %ymm0
+; CHECK-NEXT:    vmovups 32(%rdi), %ymm1
+; CHECK-NEXT:    vmovups %ymm1, 32(%rsi)
+; CHECK-NEXT:    vmovups %ymm0, (%rsi)
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
 entry:
-; CHECK: D
-; CHECK-NOT: vmovups %zmm
-; CHECK: vmovups %ymm
   call void @llvm.memmove.p0i8.p0i8.i64(i8* align 1 %dst, i8* align 1 %src, i64 64, i1 false)
   ret void
 }
@@ -46,8 +67,8 @@ entry:
 ; Function Attrs: argmemonly nounwind
 declare void @llvm.memmove.p0i8.p0i8.i64(i8* nocapture, i8* nocapture readonly, i64, i1 immarg) #1
 
-attributes #0 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "prefer-vector-width"="128" "stack-protector-buffer-size"="8" "target-cpu"="skylake-avx512" "target-features"="+adx,+aes,+avx,+avx2,+avx512bw,+avx512cd,+avx512dq,+avx512f,+avx512vl,+bmi,+bmi2,+clflushopt,+clwb,+cx16,+cx8,+f16c,+fma,+fsgsbase,+fxsr,+invpcid,+lzcnt,+mmx,+movbe,+mpx,+pclmul,+pku,+popcnt,+prfchw,+rdrnd,+rdseed,+sahf,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87,+xsave,+xsavec,+xsaveopt,+xsaves" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "prefer-vector-width"="128" "stack-protector-buffer-size"="8" "target-cpu"="skylake-avx512" "target-features"="+adx,+aes,+avx,+avx2,+avx512bw,+avx512cd,+avx512dq,+avx512f,+avx512vl,+bmi,+bmi2,+clflushopt,+clwb,+cx16,+cx8,+f16c,+fma,+fsgsbase,+fxsr,+invpcid,+lzcnt,+mmx,+movbe,+pclmul,+pku,+popcnt,+prfchw,+rdrnd,+rdseed,+sahf,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87,+xsave,+xsavec,+xsaveopt,+xsaves" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { argmemonly nounwind }
-attributes #2 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "prefer-vector-width"="256" "stack-protector-buffer-size"="8" "target-cpu"="skylake-avx512" "target-features"="+adx,+aes,+avx,+avx2,+avx512bw,+avx512cd,+avx512dq,+avx512f,+avx512vl,+bmi,+bmi2,+clflushopt,+clwb,+cx16,+cx8,+f16c,+fma,+fsgsbase,+fxsr,+invpcid,+lzcnt,+mmx,+movbe,+mpx,+pclmul,+pku,+popcnt,+prfchw,+rdrnd,+rdseed,+sahf,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87,+xsave,+xsavec,+xsaveopt,+xsaves" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "prefer-vector-width"="256" "stack-protector-buffer-size"="8" "target-cpu"="skylake-avx512" "target-features"="+adx,+aes,+avx,+avx2,+avx512bw,+avx512cd,+avx512dq,+avx512f,+avx512vl,+bmi,+bmi2,+clflushopt,+clwb,+cx16,+cx8,+f16c,+fma,+fsgsbase,+fxsr,+invpcid,+lzcnt,+mmx,+movbe,+pclmul,+pku,+popcnt,+prfchw,+rdrnd,+rdseed,+sahf,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87,+xsave,+xsavec,+xsaveopt,+xsaves" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !0 = !{i32 1, !"wchar_size", i32 4}
diff --git a/llvm/test/CodeGen/X86/vector-zext.ll b/llvm/test/CodeGen/X86/vector-zext.ll
index 7403eb5b8..cbf49c230 100644
--- a/llvm/test/CodeGen/X86/vector-zext.ll
+++ b/llvm/test/CodeGen/X86/vector-zext.ll
@@ -144,10 +144,10 @@ define <32 x i16> @zext_32i8_to_32i16(<32 x i8> %A) {
 ;
 ; AVX512F-LABEL: zext_32i8_to_32i16:
 ; AVX512F:       # %bb.0: # %entry
-; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
-; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
-; AVX512F-NEXT:    vmovdqa %ymm2, %ymm0
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: zext_32i8_to_32i16:
@@ -397,16 +397,15 @@ entry:
 define <8 x i64> @zext_16i8_to_8i64(<16 x i8> %A) nounwind uwtable readnone ssp {
 ; SSE2-LABEL: zext_16i8_to_8i64:
 ; SSE2:       # %bb.0: # %entry
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    movdqa %xmm0, %xmm3
 ; SSE2-NEXT:    pxor %xmm4, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,2,3]
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3],xmm1[4],xmm4[4],xmm1[5],xmm4[5],xmm1[6],xmm4[6],xmm1[7],xmm4[7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3],xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7]
+; SSE2-NEXT:    movdqa %xmm3, %xmm1
 ; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3]
 ; SSE2-NEXT:    movdqa %xmm1, %xmm0
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
 ; SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm4[2],xmm1[3],xmm4[3]
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3],xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7]
 ; SSE2-NEXT:    movdqa %xmm3, %xmm2
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
 ; SSE2-NEXT:    punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm4[2],xmm3[3],xmm4[3]
@@ -414,15 +413,13 @@ define <8 x i64> @zext_16i8_to_8i64(<16 x i8> %A) nounwind uwtable readnone ssp
 ;
 ; SSSE3-LABEL: zext_16i8_to_8i64:
 ; SSSE3:       # %bb.0: # %entry
-; SSSE3-NEXT:    movdqa %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm4 = [0,128,128,128,128,128,128,128,1,128,128,128,128,128,128,128]
-; SSSE3-NEXT:    pshufb %xmm4, %xmm0
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm5 = [2,128,128,128,128,128,128,128,3,128,128,128,128,128,128,128]
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,1,2,3]
-; SSSE3-NEXT:    pshufb %xmm5, %xmm1
+; SSSE3-NEXT:    movdqa %xmm0, %xmm3
+; SSSE3-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
+; SSSE3-NEXT:    movdqa %xmm3, %xmm1
+; SSSE3-NEXT:    pshufb {{.*#+}} xmm1 = xmm1[2],zero,zero,zero,zero,zero,zero,zero,xmm1[3],zero,zero,zero,zero,zero,zero,zero
 ; SSSE3-NEXT:    movdqa %xmm3, %xmm2
-; SSSE3-NEXT:    pshufb %xmm4, %xmm2
-; SSSE3-NEXT:    pshufb %xmm5, %xmm3
+; SSSE3-NEXT:    pshufb {{.*#+}} xmm2 = xmm2[4],zero,zero,zero,zero,zero,zero,zero,xmm2[5],zero,zero,zero,zero,zero,zero,zero
+; SSSE3-NEXT:    pshufb {{.*#+}} xmm3 = xmm3[6],zero,zero,zero,zero,zero,zero,zero,xmm3[7],zero,zero,zero,zero,zero,zero,zero
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: zext_16i8_to_8i64:
@@ -1159,16 +1156,15 @@ entry:
 define <8 x i64> @load_zext_8i8_to_8i64(<8 x i8> *%ptr) {
 ; SSE2-LABEL: load_zext_8i8_to_8i64:
 ; SSE2:       # %bb.0: # %entry
-; SSE2-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
+; SSE2-NEXT:    movq {{.*#+}} xmm3 = mem[0],zero
 ; SSE2-NEXT:    pxor %xmm4, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,1,2,3]
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3],xmm1[4],xmm4[4],xmm1[5],xmm4[5],xmm1[6],xmm4[6],xmm1[7],xmm4[7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3],xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7]
+; SSE2-NEXT:    movdqa %xmm3, %xmm1
 ; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3]
 ; SSE2-NEXT:    movdqa %xmm1, %xmm0
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
 ; SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm4[2],xmm1[3],xmm4[3]
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3],xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7]
 ; SSE2-NEXT:    movdqa %xmm3, %xmm2
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
 ; SSE2-NEXT:    punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm4[2],xmm3[3],xmm4[3]
@@ -1176,16 +1172,14 @@ define <8 x i64> @load_zext_8i8_to_8i64(<8 x i8> *%ptr) {
 ;
 ; SSSE3-LABEL: load_zext_8i8_to_8i64:
 ; SSSE3:       # %bb.0: # %entry
-; SSSE3-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm4 = [0,128,128,128,128,128,128,128,1,128,128,128,128,128,128,128]
-; SSSE3-NEXT:    movdqa %xmm1, %xmm0
-; SSSE3-NEXT:    pshufb %xmm4, %xmm0
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm5 = [2,128,128,128,128,128,128,128,3,128,128,128,128,128,128,128]
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,1,2,3]
-; SSSE3-NEXT:    pshufb %xmm5, %xmm1
+; SSSE3-NEXT:    movq {{.*#+}} xmm3 = mem[0],zero
+; SSSE3-NEXT:    movdqa %xmm3, %xmm0
+; SSSE3-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
+; SSSE3-NEXT:    movdqa %xmm3, %xmm1
+; SSSE3-NEXT:    pshufb {{.*#+}} xmm1 = xmm1[2],zero,zero,zero,zero,zero,zero,zero,xmm1[3],zero,zero,zero,zero,zero,zero,zero
 ; SSSE3-NEXT:    movdqa %xmm3, %xmm2
-; SSSE3-NEXT:    pshufb %xmm4, %xmm2
-; SSSE3-NEXT:    pshufb %xmm5, %xmm3
+; SSSE3-NEXT:    pshufb {{.*#+}} xmm2 = xmm2[4],zero,zero,zero,zero,zero,zero,zero,xmm2[5],zero,zero,zero,zero,zero,zero,zero
+; SSSE3-NEXT:    pshufb {{.*#+}} xmm3 = xmm3[6],zero,zero,zero,zero,zero,zero,zero,xmm3[7],zero,zero,zero,zero,zero,zero,zero
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: load_zext_8i8_to_8i64:
@@ -1508,8 +1502,8 @@ define <8 x i32> @zext_8i8_to_8i32(<8 x i8> %z) {
 ; SSE2-LABEL: zext_8i8_to_8i32:
 ; SSE2:       # %bb.0: # %entry
 ; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
 ; SSE2-NEXT:    movdqa %xmm1, %xmm0
 ; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
 ; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
@@ -1518,8 +1512,8 @@ define <8 x i32> @zext_8i8_to_8i32(<8 x i8> %z) {
 ; SSSE3-LABEL: zext_8i8_to_8i32:
 ; SSSE3:       # %bb.0: # %entry
 ; SSSE3-NEXT:    movdqa %xmm0, %xmm1
-; SSSE3-NEXT:    pand {{.*}}(%rip), %xmm1
 ; SSSE3-NEXT:    pxor %xmm2, %xmm2
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
 ; SSSE3-NEXT:    movdqa %xmm1, %xmm0
 ; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
 ; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
@@ -1527,32 +1521,28 @@ define <8 x i32> @zext_8i8_to_8i32(<8 x i8> %z) {
 ;
 ; SSE41-LABEL: zext_8i8_to_8i32:
 ; SSE41:       # %bb.0: # %entry
-; SSE41-NEXT:    pand {{.*}}(%rip), %xmm0
-; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm2 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SSE41-NEXT:    movdqa %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: zext_8i8_to_8i32:
 ; AVX1:       # %bb.0: # %entry
-; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
-; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: zext_8i8_to_8i32:
 ; AVX2:       # %bb.0: # %entry
-; AVX2-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: zext_8i8_to_8i32:
 ; AVX512:       # %bb.0: # %entry
-; AVX512-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX512-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
 ; AVX512-NEXT:    retq
 entry:
   %t = zext <8 x i8> %z to <8 x i32>
@@ -1659,8 +1649,8 @@ define <8 x i32> @shuf_zext_8i8_to_8i32(<8 x i8> %A) {
 ; SSE2-LABEL: shuf_zext_8i8_to_8i32:
 ; SSE2:       # %bb.0: # %entry
 ; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
 ; SSE2-NEXT:    movdqa %xmm1, %xmm0
 ; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
 ; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
@@ -1669,8 +1659,8 @@ define <8 x i32> @shuf_zext_8i8_to_8i32(<8 x i8> %A) {
 ; SSSE3-LABEL: shuf_zext_8i8_to_8i32:
 ; SSSE3:       # %bb.0: # %entry
 ; SSSE3-NEXT:    movdqa %xmm0, %xmm1
-; SSSE3-NEXT:    pand {{.*}}(%rip), %xmm1
 ; SSSE3-NEXT:    pxor %xmm2, %xmm2
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
 ; SSSE3-NEXT:    movdqa %xmm1, %xmm0
 ; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
 ; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
@@ -1678,7 +1668,6 @@ define <8 x i32> @shuf_zext_8i8_to_8i32(<8 x i8> %A) {
 ;
 ; SSE41-LABEL: shuf_zext_8i8_to_8i32:
 ; SSE41:       # %bb.0: # %entry
-; SSE41-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm2 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
@@ -1687,7 +1676,6 @@ define <8 x i32> @shuf_zext_8i8_to_8i32(<8 x i8> %A) {
 ;
 ; AVX1-LABEL: shuf_zext_8i8_to_8i32:
 ; AVX1:       # %bb.0: # %entry
-; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
@@ -1696,13 +1684,11 @@ define <8 x i32> @shuf_zext_8i8_to_8i32(<8 x i8> %A) {
 ;
 ; AVX2-LABEL: shuf_zext_8i8_to_8i32:
 ; AVX2:       # %bb.0: # %entry
-; AVX2-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
 ; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: shuf_zext_8i8_to_8i32:
 ; AVX512:       # %bb.0: # %entry
-; AVX512-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
 ; AVX512-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
 ; AVX512-NEXT:    retq
 entry:
@@ -2057,19 +2043,16 @@ define <8 x i32> @shuf_zext_16i16_to_8i32_offset8(<16 x i16> %A) nounwind uwtabl
 ;
 ; SSE41-LABEL: shuf_zext_16i16_to_8i32_offset8:
 ; SSE41:       # %bb.0: # %entry
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,2,3,3]
 ; SSE41-NEXT:    pxor %xmm2, %xmm2
-; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm0[0],xmm2[1,2,3],xmm0[4],xmm2[5,6,7]
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
-; SSE41-NEXT:    movdqa %xmm2, %xmm1
+; SSE41-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: shuf_zext_16i16_to_8i32_offset8:
 ; AVX1:       # %bb.0: # %entry
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,2,3,3]
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3],xmm1[4],xmm2[5,6,7]
+; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
 ; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
@@ -2278,11 +2261,11 @@ define <32 x i32> @zext_32i8_to_32i32(<32 x i8> %x) {
 ; AVX2-LABEL: zext_32i8_to_32i32:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm4 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
-; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[3,1,2,3]
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm2 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero,xmm3[4],zero,zero,zero,xmm3[5],zero,zero,zero,xmm3[6],zero,zero,zero,xmm3[7],zero,zero,zero
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm3[2,3,0,1]
 ; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm3 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
 ; AVX2-NEXT:    vmovdqa %ymm4, %ymm0
 ; AVX2-NEXT:    retq
@@ -2306,28 +2289,33 @@ define <2 x i32> @zext_2i8_to_2i32(<2 x i8>* %addr) {
 ; SSE2-NEXT:    pxor %xmm1, %xmm1
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
 ; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,1,3]
-; SSE2-NEXT:    paddq %xmm0, %xmm0
+; SSE2-NEXT:    paddd %xmm0, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: zext_2i8_to_2i32:
 ; SSSE3:       # %bb.0:
 ; SSSE3-NEXT:    movzwl (%rdi), %eax
 ; SSSE3-NEXT:    movd %eax, %xmm0
-; SSSE3-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[3],zero,zero,zero
-; SSSE3-NEXT:    paddq %xmm0, %xmm0
+; SSSE3-NEXT:    pxor %xmm1, %xmm1
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSSE3-NEXT:    paddd %xmm0, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: zext_2i8_to_2i32:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pmovzxbq {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero
-; SSE41-NEXT:    paddq %xmm0, %xmm0
+; SSE41-NEXT:    movzwl (%rdi), %eax
+; SSE41-NEXT:    movd %eax, %xmm0
+; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; SSE41-NEXT:    paddd %xmm0, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: zext_2i8_to_2i32:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpmovzxbq {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero
-; AVX-NEXT:    vpaddq %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    movzwl (%rdi), %eax
+; AVX-NEXT:    vmovd %eax, %xmm0
+; AVX-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; AVX-NEXT:    vpaddd %xmm0, %xmm0, %xmm0
 ; AVX-NEXT:    retq
   %x = load <2 x i8>, <2 x i8>* %addr, align 1
   %y = zext <2 x i8> %x to <2 x i32>
diff --git a/llvm/test/CodeGen/X86/viabs.ll b/llvm/test/CodeGen/X86/viabs.ll
index cbb6a9aec..4857b19ea 100644
--- a/llvm/test/CodeGen/X86/viabs.ll
+++ b/llvm/test/CodeGen/X86/viabs.ll
@@ -929,8 +929,10 @@ define <64 x i8> @test_abs_lt_v64i8(<64 x i8> %a) nounwind {
 ;
 ; AVX512F-LABEL: test_abs_lt_v64i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpabsb %ymm0, %ymm0 # encoding: [0xc4,0xe2,0x7d,0x1c,0xc0]
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1 # encoding: [0x62,0xf3,0xfd,0x48,0x3b,0xc1,0x01]
 ; AVX512F-NEXT:    vpabsb %ymm1, %ymm1 # encoding: [0xc4,0xe2,0x7d,0x1c,0xc9]
+; AVX512F-NEXT:    vpabsb %ymm0, %ymm0 # encoding: [0xc4,0xe2,0x7d,0x1c,0xc0]
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0 # encoding: [0x62,0xf3,0xfd,0x48,0x3a,0xc1,0x01]
 ; AVX512F-NEXT:    retq # encoding: [0xc3]
 ;
 ; AVX512BW-LABEL: test_abs_lt_v64i8:
@@ -1000,8 +1002,10 @@ define <32 x i16> @test_abs_gt_v32i16(<32 x i16> %a) nounwind {
 ;
 ; AVX512F-LABEL: test_abs_gt_v32i16:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpabsw %ymm0, %ymm0 # encoding: [0xc4,0xe2,0x7d,0x1d,0xc0]
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1 # encoding: [0x62,0xf3,0xfd,0x48,0x3b,0xc1,0x01]
 ; AVX512F-NEXT:    vpabsw %ymm1, %ymm1 # encoding: [0xc4,0xe2,0x7d,0x1d,0xc9]
+; AVX512F-NEXT:    vpabsw %ymm0, %ymm0 # encoding: [0xc4,0xe2,0x7d,0x1d,0xc0]
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0 # encoding: [0x62,0xf3,0xfd,0x48,0x3a,0xc1,0x01]
 ; AVX512F-NEXT:    retq # encoding: [0xc3]
 ;
 ; AVX512BW-LABEL: test_abs_gt_v32i16:
diff --git a/llvm/test/CodeGen/X86/volatile.ll b/llvm/test/CodeGen/X86/volatile.ll
index 29a57976a..d6ed45a19 100644
--- a/llvm/test/CodeGen/X86/volatile.ll
+++ b/llvm/test/CodeGen/X86/volatile.ll
@@ -1,9 +1,27 @@
-; RUN: llc < %s -mtriple=i686-- -mattr=sse2 | grep movsd | count 5
-; RUN: llc < %s -mtriple=i686-- -mattr=sse2 -O0 | grep -v esp | grep movsd | count 5
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- -mattr=sse2 | FileCheck %s -check-prefixes=ALL,OPT
+; RUN: llc < %s -mtriple=i686-- -mattr=sse2 -O0 | FileCheck %s --check-prefixes=ALL,NOOPT
 
 @x = external global double
 
 define void @foo() nounwind  {
+; OPT-LABEL: foo:
+; OPT:       # %bb.0:
+; OPT-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; OPT-NEXT:    xorps %xmm0, %xmm0
+; OPT-NEXT:    movsd %xmm0, x
+; OPT-NEXT:    movsd %xmm0, x
+; OPT-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; OPT-NEXT:    retl
+;
+; NOOPT-LABEL: foo:
+; NOOPT:       # %bb.0:
+; NOOPT-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; NOOPT-NEXT:    xorps %xmm1, %xmm1
+; NOOPT-NEXT:    movsd %xmm1, x
+; NOOPT-NEXT:    movsd %xmm1, x
+; NOOPT-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
+; NOOPT-NEXT:    retl
   %a = load volatile double, double* @x
   store volatile double 0.0, double* @x
   store volatile double 0.0, double* @x
@@ -12,6 +30,10 @@ define void @foo() nounwind  {
 }
 
 define void @bar() nounwind  {
+; ALL-LABEL: bar:
+; ALL:       # %bb.0:
+; ALL-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; ALL-NEXT:    retl
   %c = load volatile double, double* @x
   ret void
 }
diff --git a/llvm/test/CodeGen/X86/vp2intersect_multiple_pairs.ll b/llvm/test/CodeGen/X86/vp2intersect_multiple_pairs.ll
index afd213b53..ed1dcb94d 100644
--- a/llvm/test/CodeGen/X86/vp2intersect_multiple_pairs.ll
+++ b/llvm/test/CodeGen/X86/vp2intersect_multiple_pairs.ll
@@ -4,25 +4,15 @@
 
 ; Test with more than four live mask pairs
 
-define void @test(<16 x i32> %a0, <16 x i32> %b0,
-                  <16 x i32> %a1, <16 x i32> %b1,
-                  <16 x i32> %a2, <16 x i32> %b2,
-                  <16 x i32> %a3, <16 x i32> %b3,
-                  <16 x i32> %a4, <16 x i32> %b4,
-                 i16* nocapture %m0, i16* nocapture %m1) {
+define void @test(<16 x i32> %a0, <16 x i32> %b0, <16 x i32> %a1, <16 x i32> %b1, <16 x i32> %a2, <16 x i32> %b2, <16 x i32> %a3, <16 x i32> %b3, <16 x i32> %a4, <16 x i32> %b4, i16* nocapture %m0, i16* nocapture %m1) nounwind {
 ; X86-LABEL: test:
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    pushl %ebp
-; X86-NEXT:    .cfi_def_cfa_offset 8
-; X86-NEXT:    .cfi_offset %ebp, -8
 ; X86-NEXT:    movl %esp, %ebp
-; X86-NEXT:    .cfi_def_cfa_register %ebp
 ; X86-NEXT:    pushl %edi
 ; X86-NEXT:    pushl %esi
 ; X86-NEXT:    andl $-64, %esp
 ; X86-NEXT:    subl $64, %esp
-; X86-NEXT:    .cfi_offset %esi, -16
-; X86-NEXT:    .cfi_offset %edi, -12
 ; X86-NEXT:    movl 456(%ebp), %esi
 ; X86-NEXT:    vmovaps 328(%ebp), %zmm3
 ; X86-NEXT:    vmovaps 200(%ebp), %zmm4
@@ -62,20 +52,24 @@ define void @test(<16 x i32> %a0, <16 x i32> %b0,
 ; X86-NEXT:    kmovw %k2, %edi
 ; X86-NEXT:    addl %ecx, %edx
 ; X86-NEXT:    kmovw %k1, %ecx
+; X86-NEXT:    addl %edi, %ecx
+; X86-NEXT:    addl %eax, %ecx
+; X86-NEXT:    addl %edx, %ecx
+; X86-NEXT:    movw %cx, (%esi)
+; X86-NEXT:    leal -8(%ebp), %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: test:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    pushq %rbp
-; X64-NEXT:    .cfi_def_cfa_offset 16
-; X64-NEXT:    .cfi_offset %rbp, -16
 ; X64-NEXT:    movq %rsp, %rbp
-; X64-NEXT:    .cfi_def_cfa_register %rbp
 ; X64-NEXT:    pushq %r14
 ; X64-NEXT:    pushq %rbx
 ; X64-NEXT:    andq $-64, %rsp
 ; X64-NEXT:    subq $64, %rsp
-; X64-NEXT:    .cfi_offset %rbx, -32
-; X64-NEXT:    .cfi_offset %r14, -24
 ; X64-NEXT:    movq %rdi, %r14
 ; X64-NEXT:    vmovaps 16(%rbp), %zmm8
 ; X64-NEXT:    vp2intersectd %zmm1, %zmm0, %k0
@@ -111,6 +105,17 @@ define void @test(<16 x i32> %a0, <16 x i32> %b0,
 ; X64-NEXT:    kmovw {{[0-9]+}}(%rsp), %k1
 ; X64-NEXT:    kmovw %k0, %edi
 ; X64-NEXT:    kmovw %k1, %ebx
+; X64-NEXT:    addl %edi, %eax
+; X64-NEXT:    addl %ecx, %edx
+; X64-NEXT:    leal (%rbx,%rsi), %ecx
+; X64-NEXT:    addl %eax, %ecx
+; X64-NEXT:    addl %edx, %ecx
+; X64-NEXT:    movw %cx, (%r14)
+; X64-NEXT:    leaq -16(%rbp), %rsp
+; X64-NEXT:    popq %rbx
+; X64-NEXT:    popq %r14
+; X64-NEXT:    popq %rbp
+; X64-NEXT:    retq
 entry:
   %0 = call { <16 x i1>, <16 x i1> } @llvm.x86.avx512.vp2intersect.d.512(<16 x i32> %a0, <16 x i32> %b0)
   %1 = call { <16 x i1>, <16 x i1> } @llvm.x86.avx512.vp2intersect.d.512(<16 x i32> %a1, <16 x i32> %b1)
diff --git a/llvm/test/CodeGen/X86/vsel-cmp-load.ll b/llvm/test/CodeGen/X86/vsel-cmp-load.ll
index 43b975caf..c809090c8 100644
--- a/llvm/test/CodeGen/X86/vsel-cmp-load.ll
+++ b/llvm/test/CodeGen/X86/vsel-cmp-load.ll
@@ -8,12 +8,12 @@
 define <8 x i32> @eq_zero(<8 x i8>* %p, <8 x i32> %x, <8 x i32> %y) {
 ; AVX1-LABEL: eq_zero:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
+; AVX1-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero
 ; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpeqw %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpmovsxwd %xmm2, %xmm3
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
-; AVX1-NEXT:    vpmovsxwd %xmm2, %xmm2
+; AVX1-NEXT:    vpcmpeqb %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpmovsxbd %xmm2, %xmm3
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,2,3]
+; AVX1-NEXT:    vpmovsxbd %xmm2, %xmm2
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
 ; AVX1-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
@@ -28,8 +28,8 @@ define <8 x i32> @eq_zero(<8 x i8>* %p, <8 x i32> %x, <8 x i32> %y) {
 ;
 ; AVX512-LABEL: eq_zero:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpmovzxbw {{.*#+}} xmm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
-; AVX512-NEXT:    vptestnmw %xmm2, %xmm2, %k1
+; AVX512-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero
+; AVX512-NEXT:    vptestnmb %xmm2, %xmm2, %k1
 ; AVX512-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
 ; AVX512-NEXT:    retq
   %load = load <8 x i8>, <8 x i8>* %p
@@ -41,14 +41,14 @@ define <8 x i32> @eq_zero(<8 x i8>* %p, <8 x i32> %x, <8 x i32> %y) {
 define <4 x i64> @ne_zero(<4 x i16>* %p, <4 x i64> %x, <4 x i64> %y) {
 ; AVX1-LABEL: ne_zero:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
+; AVX1-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero
 ; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpcmpeqw %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
 ; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpmovsxdq %xmm2, %xmm3
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
-; AVX1-NEXT:    vpmovsxdq %xmm2, %xmm2
+; AVX1-NEXT:    vpmovsxwq %xmm2, %xmm3
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,2,3]
+; AVX1-NEXT:    vpmovsxwq %xmm2, %xmm2
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
 ; AVX1-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
@@ -63,8 +63,8 @@ define <4 x i64> @ne_zero(<4 x i16>* %p, <4 x i64> %x, <4 x i64> %y) {
 ;
 ; AVX512-LABEL: ne_zero:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpmovzxwd {{.*#+}} xmm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
-; AVX512-NEXT:    vptestmd %xmm2, %xmm2, %k1
+; AVX512-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero
+; AVX512-NEXT:    vptestmw %xmm2, %xmm2, %k1
 ; AVX512-NEXT:    vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
 ; AVX512-NEXT:    retq
   %load = load <4 x i16>, <4 x i16>* %p
@@ -111,10 +111,10 @@ define <16 x i16> @sgt_zero(<16 x i8>* %p, <16 x i16> %x, <16 x i16> %y) {
 define <8 x i32> @slt_zero(<8 x i8>* %p, <8 x i32> %x, <8 x i32> %y) {
 ; AVX1-LABEL: slt_zero:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpmovsxbw (%rdi), %xmm2
-; AVX1-NEXT:    vpmovsxwd %xmm2, %xmm3
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
-; AVX1-NEXT:    vpmovsxwd %xmm2, %xmm2
+; AVX1-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero
+; AVX1-NEXT:    vpmovsxbd %xmm2, %xmm3
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,2,3]
+; AVX1-NEXT:    vpmovsxbd %xmm2, %xmm2
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
 ; AVX1-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
@@ -127,8 +127,8 @@ define <8 x i32> @slt_zero(<8 x i8>* %p, <8 x i32> %x, <8 x i32> %y) {
 ;
 ; AVX512-LABEL: slt_zero:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpmovsxbw (%rdi), %xmm2
-; AVX512-NEXT:    vpmovw2m %xmm2, %k1
+; AVX512-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero
+; AVX512-NEXT:    vpmovb2m %xmm2, %k1
 ; AVX512-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
 ; AVX512-NEXT:    retq
   %load = load <8 x i8>, <8 x i8>* %p
@@ -140,12 +140,12 @@ define <8 x i32> @slt_zero(<8 x i8>* %p, <8 x i32> %x, <8 x i32> %y) {
 define <4 x double> @eq_zero_fp_select(<4 x i8>* %p, <4 x double> %x, <4 x double> %y) {
 ; AVX1-LABEL: eq_zero_fp_select:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm2 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
+; AVX1-NEXT:    vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
 ; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpmovsxdq %xmm2, %xmm3
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
-; AVX1-NEXT:    vpmovsxdq %xmm2, %xmm2
+; AVX1-NEXT:    vpcmpeqb %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpmovsxbq %xmm2, %xmm3
+; AVX1-NEXT:    vpsrld $16, %xmm2, %xmm2
+; AVX1-NEXT:    vpmovsxbq %xmm2, %xmm2
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
 ; AVX1-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
@@ -160,8 +160,8 @@ define <4 x double> @eq_zero_fp_select(<4 x i8>* %p, <4 x double> %x, <4 x doubl
 ;
 ; AVX512-LABEL: eq_zero_fp_select:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpmovzxbd {{.*#+}} xmm2 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
-; AVX512-NEXT:    vptestnmd %xmm2, %xmm2, %k1
+; AVX512-NEXT:    vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; AVX512-NEXT:    vptestnmb %xmm2, %xmm2, %k1
 ; AVX512-NEXT:    vblendmpd %ymm0, %ymm1, %ymm0 {%k1}
 ; AVX512-NEXT:    retq
   %load = load <4 x i8>, <4 x i8>* %p
@@ -173,14 +173,14 @@ define <4 x double> @eq_zero_fp_select(<4 x i8>* %p, <4 x double> %x, <4 x doubl
 define <8 x float> @ne_zero_fp_select(<8 x i8>* %p, <8 x float> %x, <8 x float> %y) {
 ; AVX1-LABEL: ne_zero_fp_select:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
+; AVX1-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero
 ; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpeqw %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpcmpeqb %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
 ; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpmovsxwd %xmm2, %xmm3
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
-; AVX1-NEXT:    vpmovsxwd %xmm2, %xmm2
+; AVX1-NEXT:    vpmovsxbd %xmm2, %xmm3
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,2,3]
+; AVX1-NEXT:    vpmovsxbd %xmm2, %xmm2
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
 ; AVX1-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
@@ -195,8 +195,8 @@ define <8 x float> @ne_zero_fp_select(<8 x i8>* %p, <8 x float> %x, <8 x float>
 ;
 ; AVX512-LABEL: ne_zero_fp_select:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpmovzxbw {{.*#+}} xmm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
-; AVX512-NEXT:    vptestmw %xmm2, %xmm2, %k1
+; AVX512-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero
+; AVX512-NEXT:    vptestmb %xmm2, %xmm2, %k1
 ; AVX512-NEXT:    vblendmps %ymm0, %ymm1, %ymm0 {%k1}
 ; AVX512-NEXT:    retq
   %load = load <8 x i8>, <8 x i8>* %p
@@ -208,12 +208,12 @@ define <8 x float> @ne_zero_fp_select(<8 x i8>* %p, <8 x float> %x, <8 x float>
 define <4 x double> @sgt_zero_fp_select(<4 x i8>* %p, <4 x double> %x, <4 x double> %y) {
 ; AVX1-LABEL: sgt_zero_fp_select:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpmovsxbd (%rdi), %xmm2
+; AVX1-NEXT:    vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
 ; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpgtd %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpmovsxdq %xmm2, %xmm3
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
-; AVX1-NEXT:    vpmovsxdq %xmm2, %xmm2
+; AVX1-NEXT:    vpcmpgtb %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpmovsxbq %xmm2, %xmm3
+; AVX1-NEXT:    vpsrld $16, %xmm2, %xmm2
+; AVX1-NEXT:    vpmovsxbq %xmm2, %xmm2
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
 ; AVX1-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
@@ -228,9 +228,9 @@ define <4 x double> @sgt_zero_fp_select(<4 x i8>* %p, <4 x double> %x, <4 x doub
 ;
 ; AVX512-LABEL: sgt_zero_fp_select:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpmovsxbd (%rdi), %xmm2
+; AVX512-NEXT:    vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
 ; AVX512-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX512-NEXT:    vpcmpgtd %xmm3, %xmm2, %k1
+; AVX512-NEXT:    vpcmpgtb %xmm3, %xmm2, %k1
 ; AVX512-NEXT:    vblendmpd %ymm0, %ymm1, %ymm0 {%k1}
 ; AVX512-NEXT:    retq
   %load = load <4 x i8>, <4 x i8>* %p
diff --git a/llvm/test/CodeGen/X86/vselect-avx.ll b/llvm/test/CodeGen/X86/vselect-avx.ll
index cf84c4b7e..3cd565477 100644
--- a/llvm/test/CodeGen/X86/vselect-avx.ll
+++ b/llvm/test/CodeGen/X86/vselect-avx.ll
@@ -84,45 +84,35 @@ bb:
 define void @test3(<4 x i32> %induction30, <4 x i16>* %tmp16, <4 x i16>* %tmp17,  <4 x i16> %tmp3, <4 x i16> %tmp12) {
 ; AVX1-LABEL: test3:
 ; AVX1:       ## %bb.0:
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [1431655766,1431655766,1431655766,1431655766]
-; AVX1-NEXT:    vpmuldq %xmm4, %xmm3, %xmm3
-; AVX1-NEXT:    vpmuldq %xmm4, %xmm0, %xmm4
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm4[0,1],xmm3[2,3],xmm4[4,5],xmm3[6,7]
-; AVX1-NEXT:    vpsrld $31, %xmm3, %xmm4
-; AVX1-NEXT:    vpaddd %xmm4, %xmm3, %xmm3
-; AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm3, %xmm3
-; AVX1-NEXT:    vpsubd %xmm3, %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vpaddd {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm3
 ; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm0, %xmm0
-; AVX1-NEXT:    vblendvps %xmm0, %xmm1, %xmm2, %xmm1
 ; AVX1-NEXT:    vpackssdw %xmm0, %xmm0, %xmm0
+; AVX1-NEXT:    vpblendvb %xmm0, %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    vpblendvb %xmm0, %xmm2, %xmm3, %xmm0
 ; AVX1-NEXT:    vmovq %xmm0, (%rdi)
-; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm1[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
-; AVX1-NEXT:    vmovq %xmm0, (%rsi)
+; AVX1-NEXT:    vmovq %xmm1, (%rsi)
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: test3:
 ; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
-; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm4 = [1431655766,1431655766,1431655766,1431655766]
-; AVX2-NEXT:    vpmuldq %xmm4, %xmm3, %xmm3
-; AVX2-NEXT:    vpmuldq %xmm4, %xmm0, %xmm4
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm3 = xmm4[0],xmm3[1],xmm4[2],xmm3[3]
-; AVX2-NEXT:    vpsrld $31, %xmm3, %xmm4
-; AVX2-NEXT:    vpaddd %xmm4, %xmm3, %xmm3
-; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm4 = [3,3,3,3]
-; AVX2-NEXT:    vpmulld %xmm4, %xmm3, %xmm3
-; AVX2-NEXT:    vpsubd %xmm3, %xmm0, %xmm0
-; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm3 = [2863311531,2863311531,2863311531,2863311531]
+; AVX2-NEXT:    vpmulld %xmm3, %xmm0, %xmm0
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm3 = [715827882,715827882,715827882,715827882]
+; AVX2-NEXT:    vpaddd %xmm3, %xmm0, %xmm0
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm3 = [1431655764,1431655764,1431655764,1431655764]
+; AVX2-NEXT:    vpminud %xmm3, %xmm0, %xmm3
 ; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm0, %xmm0
-; AVX2-NEXT:    vblendvps %xmm0, %xmm1, %xmm2, %xmm1
 ; AVX2-NEXT:    vpackssdw %xmm0, %xmm0, %xmm0
+; AVX2-NEXT:    vpblendvb %xmm0, %xmm1, %xmm2, %xmm1
+; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX2-NEXT:    vpblendvb %xmm0, %xmm2, %xmm3, %xmm0
 ; AVX2-NEXT:    vmovq %xmm0, (%rdi)
-; AVX2-NEXT:    vpshufb {{.*#+}} xmm0 = xmm1[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
-; AVX2-NEXT:    vmovq %xmm0, (%rsi)
+; AVX2-NEXT:    vmovq %xmm1, (%rsi)
 ; AVX2-NEXT:    retq
   %tmp6 = srem <4 x i32> %induction30, <i32 3, i32 3, i32 3, i32 3>
   %tmp7 = icmp eq <4 x i32> %tmp6, zeroinitializer
diff --git a/llvm/test/CodeGen/X86/vselect.ll b/llvm/test/CodeGen/X86/vselect.ll
index d89b1352a..e5646ae12 100644
--- a/llvm/test/CodeGen/X86/vselect.ll
+++ b/llvm/test/CodeGen/X86/vselect.ll
@@ -536,17 +536,17 @@ define <2 x i64> @shrunkblend_nonvselectuse(<2 x i1> %cond, <2 x i64> %a, <2 x i
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    psllq $63, %xmm0
 ; SSE41-NEXT:    psrad $31, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
 ; SSE41-NEXT:    paddq %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: shrunkblend_nonvselectuse:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpsllq $63, %xmm0, %xmm0
-; AVX-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX-NEXT:    vpcmpgtq %xmm0, %xmm3, %xmm0
 ; AVX-NEXT:    vblendvpd %xmm0, %xmm1, %xmm2, %xmm1
+; AVX-NEXT:    vxorpd %xmm2, %xmm2, %xmm2
+; AVX-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm0
 ; AVX-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    retq
   %x = select <2 x i1> %cond, <2 x i64> %a, <2 x i64> %b
@@ -563,36 +563,43 @@ define <2 x i64> @shrunkblend_nonvselectuse(<2 x i1> %cond, <2 x i64> %a, <2 x i
 define <2 x i32> @simplify_select(i32 %x, <2 x i1> %z) {
 ; SSE2-LABEL: simplify_select:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    psllq $63, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSE2-NEXT:    pslld $31, %xmm0
 ; SSE2-NEXT:    psrad $31, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
 ; SSE2-NEXT:    movd %edi, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[0,1,0,1]
-; SSE2-NEXT:    movdqa %xmm2, %xmm3
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm1[0]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,0,1,1]
+; SSE2-NEXT:    por %xmm1, %xmm2
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,0],xmm2[1,1]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[2,0],xmm2[2,3]
 ; SSE2-NEXT:    pand %xmm0, %xmm2
-; SSE2-NEXT:    pandn %xmm3, %xmm0
+; SSE2-NEXT:    pandn %xmm1, %xmm0
 ; SSE2-NEXT:    por %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: simplify_select:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movd %edi, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSE41-NEXT:    pslld $31, %xmm0
+; SSE41-NEXT:    movd %edi, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,0,1,1]
+; SSE41-NEXT:    por %xmm1, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,2,3]
+; SSE41-NEXT:    pinsrd $1, %edi, %xmm1
+; SSE41-NEXT:    blendvps %xmm0, %xmm2, %xmm1
+; SSE41-NEXT:    movaps %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: simplify_select:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovd %edi, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: simplify_select:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    # kill: def $edi killed $edi def $rdi
-; AVX2-NEXT:    vmovq %rdi, %xmm0
-; AVX2-NEXT:    vpbroadcastq %xmm0, %xmm0
-; AVX2-NEXT:    retq
+; AVX-LABEL: simplify_select:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX-NEXT:    vpslld $31, %xmm0, %xmm0
+; AVX-NEXT:    vmovd %edi, %xmm1
+; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,0,1,1]
+; AVX-NEXT:    vpor %xmm1, %xmm2, %xmm1
+; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,2,3]
+; AVX-NEXT:    vpinsrd $1, %edi, %xmm2, %xmm2
+; AVX-NEXT:    vblendvps %xmm0, %xmm1, %xmm2, %xmm0
+; AVX-NEXT:    retq
   %a = insertelement <2 x i32> <i32 0, i32 undef>, i32 %x, i32 1
   %b = insertelement <2 x i32> <i32 undef, i32 0>, i32 %x, i32 0
   %y = or <2 x i32> %a, %b
@@ -640,39 +647,22 @@ define void @vselect_allzeros_LHS_multiple_use_setcc(<4 x i32> %x, <4 x i32> %y,
 ; This test case previously crashed after r363802, r363850, and r363856 due
 ; any_extend_vector_inreg not being handled by the X86 backend.
 define i64 @vselect_any_extend_vector_inreg_crash(<8 x i8>* %x) {
-; SSE2-LABEL: vselect_any_extend_vector_inreg_crash:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
-; SSE2-NEXT:    pcmpeqw {{.*}}(%rip), %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,0,3]
-; SSE2-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,5,6,7]
-; SSE2-NEXT:    psllq $56, %xmm0
-; SSE2-NEXT:    psrad $24, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    movq %xmm0, %rax
-; SSE2-NEXT:    andl $32768, %eax # imm = 0x8000
-; SSE2-NEXT:    retq
-;
-; SSE41-LABEL: vselect_any_extend_vector_inreg_crash:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
-; SSE41-NEXT:    pcmpeqw {{.*}}(%rip), %xmm0
-; SSE41-NEXT:    psllq $56, %xmm0
-; SSE41-NEXT:    movl $32768, %eax # imm = 0x8000
-; SSE41-NEXT:    movq %rax, %xmm1
-; SSE41-NEXT:    xorpd %xmm2, %xmm2
-; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
-; SSE41-NEXT:    movq %xmm2, %rax
-; SSE41-NEXT:    retq
+; SSE-LABEL: vselect_any_extend_vector_inreg_crash:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; SSE-NEXT:    pcmpeqb {{.*}}(%rip), %xmm0
+; SSE-NEXT:    movq %xmm0, %rax
+; SSE-NEXT:    andl $1, %eax
+; SSE-NEXT:    shlq $15, %rax
+; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: vselect_any_extend_vector_inreg_crash:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpmovzxbw {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
-; AVX-NEXT:    vpcmpeqw {{.*}}(%rip), %xmm0, %xmm0
+; AVX-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
+; AVX-NEXT:    vpcmpeqb {{.*}}(%rip), %xmm0, %xmm0
 ; AVX-NEXT:    vmovq %xmm0, %rax
-; AVX-NEXT:    andl $32768, %eax # imm = 0x8000
+; AVX-NEXT:    andl $1, %eax
+; AVX-NEXT:    shlq $15, %rax
 ; AVX-NEXT:    retq
 0:
   %1 = load <8 x i8>, <8 x i8>* %x
diff --git a/llvm/test/CodeGen/X86/vshift-4.ll b/llvm/test/CodeGen/X86/vshift-4.ll
index d2c24fbca..9362c41a3 100644
--- a/llvm/test/CodeGen/X86/vshift-4.ll
+++ b/llvm/test/CodeGen/X86/vshift-4.ll
@@ -58,7 +58,7 @@ define void @shift2a(<4 x i32> %val, <4 x i32>* %dst, <2 x i32> %amt) nounwind {
 ; X32-LABEL: shift2a:
 ; X32:       # %bb.0: # %entry
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; X32-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
 ; X32-NEXT:    xorps %xmm2, %xmm2
 ; X32-NEXT:    movss {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
 ; X32-NEXT:    pslld %xmm2, %xmm0
@@ -67,7 +67,7 @@ define void @shift2a(<4 x i32> %val, <4 x i32>* %dst, <2 x i32> %amt) nounwind {
 ;
 ; X64-LABEL: shift2a:
 ; X64:       # %bb.0: # %entry
-; X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
 ; X64-NEXT:    xorps %xmm2, %xmm2
 ; X64-NEXT:    movss {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
 ; X64-NEXT:    pslld %xmm2, %xmm0
@@ -84,7 +84,7 @@ define void @shift2b(<4 x i32> %val, <4 x i32>* %dst, <2 x i32> %amt) nounwind {
 ; X32-LABEL: shift2b:
 ; X32:       # %bb.0: # %entry
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; X32-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
 ; X32-NEXT:    xorps %xmm2, %xmm2
 ; X32-NEXT:    movss {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
 ; X32-NEXT:    pslld %xmm2, %xmm0
@@ -93,7 +93,7 @@ define void @shift2b(<4 x i32> %val, <4 x i32>* %dst, <2 x i32> %amt) nounwind {
 ;
 ; X64-LABEL: shift2b:
 ; X64:       # %bb.0: # %entry
-; X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
 ; X64-NEXT:    xorps %xmm2, %xmm2
 ; X64-NEXT:    movss {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
 ; X64-NEXT:    pslld %xmm2, %xmm0
@@ -110,7 +110,7 @@ define void @shift2c(<4 x i32> %val, <4 x i32>* %dst, <2 x i32> %amt) nounwind {
 ; X32-LABEL: shift2c:
 ; X32:       # %bb.0: # %entry
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; X32-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
 ; X32-NEXT:    xorps %xmm2, %xmm2
 ; X32-NEXT:    movss {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
 ; X32-NEXT:    pslld %xmm2, %xmm0
@@ -119,7 +119,7 @@ define void @shift2c(<4 x i32> %val, <4 x i32>* %dst, <2 x i32> %amt) nounwind {
 ;
 ; X64-LABEL: shift2c:
 ; X64:       # %bb.0: # %entry
-; X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
 ; X64-NEXT:    xorps %xmm2, %xmm2
 ; X64-NEXT:    movss {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
 ; X64-NEXT:    pslld %xmm2, %xmm0
diff --git a/llvm/test/CodeGen/X86/vshli-simplify-demanded-bits.ll b/llvm/test/CodeGen/X86/vshli-simplify-demanded-bits.ll
new file mode 100644
index 000000000..1750cbf2c
--- /dev/null
+++ b/llvm/test/CodeGen/X86/vshli-simplify-demanded-bits.ll
@@ -0,0 +1,58 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
+
+; A combine forming X86ISD::VSHLI was missing a test and not using
+; TargetConstant for the RHS operand.
+; https://bugs.chromium.org/p/chromium/issues/detail?id=1005750
+
+define <8 x i8> @vshli_target_constant(<8 x i16> %arg, <8 x i32> %arg1) {
+; CHECK-LABEL: vshli_target_constant:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movdqa {{.*#+}} xmm0 = [2863311531,2863311531,2863311531,2863311531]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]
+; CHECK-NEXT:    pmuludq %xmm0, %xmm1
+; CHECK-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,3,2,3]
+; CHECK-NEXT:    pmuludq %xmm0, %xmm3
+; CHECK-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,3,2,3]
+; CHECK-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,3,3]
+; CHECK-NEXT:    pmuludq %xmm0, %xmm2
+; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
+; CHECK-NEXT:    pmuludq %xmm0, %xmm1
+; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,3,2,3]
+; CHECK-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
+; CHECK-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
+; CHECK-NEXT:    pslld $15, %xmm2
+; CHECK-NEXT:    psrad $16, %xmm2
+; CHECK-NEXT:    pslld $15, %xmm4
+; CHECK-NEXT:    psrad $16, %xmm4
+; CHECK-NEXT:    packssdw %xmm2, %xmm4
+; CHECK-NEXT:    pand {{.*}}(%rip), %xmm4
+; CHECK-NEXT:    pxor %xmm0, %xmm0
+; CHECK-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; CHECK-NEXT:    pmullw %xmm4, %xmm1
+; CHECK-NEXT:    movdqa %xmm1, %xmm0
+; CHECK-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; CHECK-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; CHECK-NEXT:    movdqa {{.*#+}} xmm2 = [128,128,128,128]
+; CHECK-NEXT:    paddd %xmm2, %xmm1
+; CHECK-NEXT:    paddd %xmm2, %xmm0
+; CHECK-NEXT:    psrld $8, %xmm0
+; CHECK-NEXT:    psrld $8, %xmm1
+; CHECK-NEXT:    movdqa {{.*#+}} xmm2 = [255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0]
+; CHECK-NEXT:    pand %xmm2, %xmm1
+; CHECK-NEXT:    pand %xmm2, %xmm0
+; CHECK-NEXT:    packuswb %xmm1, %xmm0
+; CHECK-NEXT:    packuswb %xmm0, %xmm0
+; CHECK-NEXT:    retq
+bb:
+  %tmp = udiv <8 x i32> %arg1, <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %tmp2 = and <8 x i32> %tmp, <i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255>
+  %tmp3 = load <8 x i8>, <8 x i8>* undef, align 1
+  %tmp4 = zext <8 x i8> %tmp3 to <8 x i32>
+  %tmp5 = mul nuw nsw <8 x i32> %tmp2, %tmp4
+  %tmp6 = add nuw nsw <8 x i32> %tmp5, <i32 128, i32 128, i32 128, i32 128, i32 128, i32 128, i32 128, i32 128>
+  %tmp7 = lshr <8 x i32> %tmp6, <i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8>
+  %tmp8 = trunc <8 x i32> %tmp7 to <8 x i8>
+  ret <8 x i8> %tmp8
+}
diff --git a/llvm/test/CodeGen/X86/widen_arith-1.ll b/llvm/test/CodeGen/X86/widen_arith-1.ll
index d5b1cbb55..a37dba14f 100644
--- a/llvm/test/CodeGen/X86/widen_arith-1.ll
+++ b/llvm/test/CodeGen/X86/widen_arith-1.ll
@@ -4,10 +4,9 @@
 define void @update(<3 x i8>* %dst, <3 x i8>* %src, i32 %n) nounwind {
 ; CHECK-LABEL: update:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    subl $12, %esp
+; CHECK-NEXT:    pushl %eax
 ; CHECK-NEXT:    movl $0, (%esp)
 ; CHECK-NEXT:    pcmpeqd %xmm0, %xmm0
-; CHECK-NEXT:    movdqa {{.*#+}} xmm1 = <0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u>
 ; CHECK-NEXT:    .p2align 4, 0x90
 ; CHECK-NEXT:  .LBB0_1: # %forcond
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
@@ -19,15 +18,14 @@ define void @update(<3 x i8>* %dst, <3 x i8>* %src, i32 %n) nounwind {
 ; CHECK-NEXT:    movl (%esp), %eax
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; CHECK-NEXT:    pmovzxbd {{.*#+}} xmm2 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
-; CHECK-NEXT:    psubd %xmm0, %xmm2
-; CHECK-NEXT:    pextrb $8, %xmm2, 2(%ecx,%eax,4)
-; CHECK-NEXT:    pshufb %xmm1, %xmm2
-; CHECK-NEXT:    pextrw $0, %xmm2, (%ecx,%eax,4)
+; CHECK-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; CHECK-NEXT:    psubb %xmm0, %xmm1
+; CHECK-NEXT:    pextrb $2, %xmm1, 2(%ecx,%eax,4)
+; CHECK-NEXT:    pextrw $0, %xmm1, (%ecx,%eax,4)
 ; CHECK-NEXT:    incl (%esp)
 ; CHECK-NEXT:    jmp .LBB0_1
 ; CHECK-NEXT:  .LBB0_3: # %afterfor
-; CHECK-NEXT:    addl $12, %esp
+; CHECK-NEXT:    popl %eax
 ; CHECK-NEXT:    retl
 entry:
 	%dst.addr = alloca <3 x i8>*
diff --git a/llvm/test/CodeGen/X86/widen_arith-2.ll b/llvm/test/CodeGen/X86/widen_arith-2.ll
index 0b8a8440a..cf76f66da 100644
--- a/llvm/test/CodeGen/X86/widen_arith-2.ll
+++ b/llvm/test/CodeGen/X86/widen_arith-2.ll
@@ -9,7 +9,7 @@ define void @update(i64* %dst_i, i64* %src_i, i32 %n) nounwind {
 ; CHECK-NEXT:    subl $12, %esp
 ; CHECK-NEXT:    movl $0, (%esp)
 ; CHECK-NEXT:    pcmpeqd %xmm0, %xmm0
-; CHECK-NEXT:    movdqa {{.*#+}} xmm1 = [4,4,4,4,4,4,4,4]
+; CHECK-NEXT:    movdqa {{.*#+}} xmm1 = <4,4,4,4,4,4,4,4,u,u,u,u,u,u,u,u>
 ; CHECK-NEXT:    .p2align 4, 0x90
 ; CHECK-NEXT:  .LBB0_1: # %forcond
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
@@ -25,10 +25,9 @@ define void @update(i64* %dst_i, i64* %src_i, i32 %n) nounwind {
 ; CHECK-NEXT:    movl %edx, {{[0-9]+}}(%esp)
 ; CHECK-NEXT:    addl {{[0-9]+}}(%esp), %ecx
 ; CHECK-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; CHECK-NEXT:    pmovzxbw {{.*#+}} xmm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
-; CHECK-NEXT:    psubw %xmm0, %xmm2
+; CHECK-NEXT:    movq {{.*#+}} xmm2 = mem[0],zero
+; CHECK-NEXT:    psubb %xmm0, %xmm2
 ; CHECK-NEXT:    pand %xmm1, %xmm2
-; CHECK-NEXT:    packuswb %xmm0, %xmm2
 ; CHECK-NEXT:    movq %xmm2, (%edx,%eax,8)
 ; CHECK-NEXT:    incl (%esp)
 ; CHECK-NEXT:    jmp .LBB0_1
diff --git a/llvm/test/CodeGen/X86/widen_arith-3.ll b/llvm/test/CodeGen/X86/widen_arith-3.ll
index 1f638a253..5b944beff 100644
--- a/llvm/test/CodeGen/X86/widen_arith-3.ll
+++ b/llvm/test/CodeGen/X86/widen_arith-3.ll
@@ -12,8 +12,7 @@ define void @update(<3 x i16>* %dst, <3 x i16>* %src, i32 %n) nounwind {
 ; CHECK-NEXT:    pushl %ebp
 ; CHECK-NEXT:    movl %esp, %ebp
 ; CHECK-NEXT:    andl $-8, %esp
-; CHECK-NEXT:    subl $32, %esp
-; CHECK-NEXT:    movdqa {{.*#+}} xmm1 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
+; CHECK-NEXT:    subl $16, %esp
 ; CHECK-NEXT:    pcmpeqd %xmm0, %xmm0
 ; CHECK-NEXT:    movw $1, {{[0-9]+}}(%esp)
 ; CHECK-NEXT:    movl $65537, {{[0-9]+}}(%esp) # imm = 0x10001
@@ -29,11 +28,11 @@ define void @update(<3 x i16>* %dst, <3 x i16>* %src, i32 %n) nounwind {
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; CHECK-NEXT:    movl 12(%ebp), %edx
 ; CHECK-NEXT:    movl 8(%ebp), %ecx
-; CHECK-NEXT:    pmovzxwd {{.*#+}} xmm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
-; CHECK-NEXT:    psubd %xmm0, %xmm2
-; CHECK-NEXT:    pextrw $4, %xmm2, 4(%ecx,%eax,8)
-; CHECK-NEXT:    pshufb %xmm1, %xmm2
-; CHECK-NEXT:    movd %xmm2, (%ecx,%eax,8)
+; CHECK-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; CHECK-NEXT:    pinsrw $2, 4(%edx,%eax,8), %xmm1
+; CHECK-NEXT:    psubw %xmm0, %xmm1
+; CHECK-NEXT:    pextrw $2, %xmm1, 4(%ecx,%eax,8)
+; CHECK-NEXT:    movd %xmm1, (%ecx,%eax,8)
 ; CHECK-NEXT:    incl {{[0-9]+}}(%esp)
 ; CHECK-NEXT:    jmp .LBB0_1
 ; CHECK-NEXT:  .LBB0_3: # %afterfor
diff --git a/llvm/test/CodeGen/X86/widen_bitops-0.ll b/llvm/test/CodeGen/X86/widen_bitops-0.ll
index 251009d36..ec7ce9934 100644
--- a/llvm/test/CodeGen/X86/widen_bitops-0.ll
+++ b/llvm/test/CodeGen/X86/widen_bitops-0.ll
@@ -132,15 +132,15 @@ define <3 x i8> @and_v3i8_as_i24(<3 x i8> %a, <3 x i8> %b) nounwind {
 ; X32-SSE-LABEL: and_v3i8_as_i24:
 ; X32-SSE:       # %bb.0:
 ; X32-SSE-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X32-SSE-NEXT:    pinsrb $4, {{[0-9]+}}(%esp), %xmm0
-; X32-SSE-NEXT:    pinsrb $8, {{[0-9]+}}(%esp), %xmm0
+; X32-SSE-NEXT:    pinsrb $1, {{[0-9]+}}(%esp), %xmm0
+; X32-SSE-NEXT:    pinsrb $2, {{[0-9]+}}(%esp), %xmm0
 ; X32-SSE-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; X32-SSE-NEXT:    pinsrb $4, {{[0-9]+}}(%esp), %xmm1
-; X32-SSE-NEXT:    pinsrb $8, {{[0-9]+}}(%esp), %xmm1
+; X32-SSE-NEXT:    pinsrb $1, {{[0-9]+}}(%esp), %xmm1
+; X32-SSE-NEXT:    pinsrb $2, {{[0-9]+}}(%esp), %xmm1
 ; X32-SSE-NEXT:    pand %xmm0, %xmm1
 ; X32-SSE-NEXT:    pextrb $0, %xmm1, %eax
-; X32-SSE-NEXT:    pextrb $4, %xmm1, %edx
-; X32-SSE-NEXT:    pextrb $8, %xmm1, %ecx
+; X32-SSE-NEXT:    pextrb $1, %xmm1, %edx
+; X32-SSE-NEXT:    pextrb $2, %xmm1, %ecx
 ; X32-SSE-NEXT:    # kill: def $al killed $al killed $eax
 ; X32-SSE-NEXT:    # kill: def $dl killed $dl killed $edx
 ; X32-SSE-NEXT:    # kill: def $cl killed $cl killed $ecx
@@ -149,15 +149,15 @@ define <3 x i8> @and_v3i8_as_i24(<3 x i8> %a, <3 x i8> %b) nounwind {
 ; X64-SSE-LABEL: and_v3i8_as_i24:
 ; X64-SSE:       # %bb.0:
 ; X64-SSE-NEXT:    movd %ecx, %xmm0
-; X64-SSE-NEXT:    pinsrd $1, %r8d, %xmm0
-; X64-SSE-NEXT:    pinsrd $2, %r9d, %xmm0
+; X64-SSE-NEXT:    pinsrb $1, %r8d, %xmm0
+; X64-SSE-NEXT:    pinsrb $2, %r9d, %xmm0
 ; X64-SSE-NEXT:    movd %edi, %xmm1
-; X64-SSE-NEXT:    pinsrd $1, %esi, %xmm1
-; X64-SSE-NEXT:    pinsrd $2, %edx, %xmm1
+; X64-SSE-NEXT:    pinsrb $1, %esi, %xmm1
+; X64-SSE-NEXT:    pinsrb $2, %edx, %xmm1
 ; X64-SSE-NEXT:    pand %xmm0, %xmm1
 ; X64-SSE-NEXT:    pextrb $0, %xmm1, %eax
-; X64-SSE-NEXT:    pextrb $4, %xmm1, %edx
-; X64-SSE-NEXT:    pextrb $8, %xmm1, %ecx
+; X64-SSE-NEXT:    pextrb $1, %xmm1, %edx
+; X64-SSE-NEXT:    pextrb $2, %xmm1, %ecx
 ; X64-SSE-NEXT:    # kill: def $al killed $al killed $eax
 ; X64-SSE-NEXT:    # kill: def $dl killed $dl killed $edx
 ; X64-SSE-NEXT:    # kill: def $cl killed $cl killed $ecx
@@ -173,15 +173,15 @@ define <3 x i8> @xor_v3i8_as_i24(<3 x i8> %a, <3 x i8> %b) nounwind {
 ; X32-SSE-LABEL: xor_v3i8_as_i24:
 ; X32-SSE:       # %bb.0:
 ; X32-SSE-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X32-SSE-NEXT:    pinsrb $4, {{[0-9]+}}(%esp), %xmm0
-; X32-SSE-NEXT:    pinsrb $8, {{[0-9]+}}(%esp), %xmm0
+; X32-SSE-NEXT:    pinsrb $1, {{[0-9]+}}(%esp), %xmm0
+; X32-SSE-NEXT:    pinsrb $2, {{[0-9]+}}(%esp), %xmm0
 ; X32-SSE-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; X32-SSE-NEXT:    pinsrb $4, {{[0-9]+}}(%esp), %xmm1
-; X32-SSE-NEXT:    pinsrb $8, {{[0-9]+}}(%esp), %xmm1
+; X32-SSE-NEXT:    pinsrb $1, {{[0-9]+}}(%esp), %xmm1
+; X32-SSE-NEXT:    pinsrb $2, {{[0-9]+}}(%esp), %xmm1
 ; X32-SSE-NEXT:    pxor %xmm0, %xmm1
 ; X32-SSE-NEXT:    pextrb $0, %xmm1, %eax
-; X32-SSE-NEXT:    pextrb $4, %xmm1, %edx
-; X32-SSE-NEXT:    pextrb $8, %xmm1, %ecx
+; X32-SSE-NEXT:    pextrb $1, %xmm1, %edx
+; X32-SSE-NEXT:    pextrb $2, %xmm1, %ecx
 ; X32-SSE-NEXT:    # kill: def $al killed $al killed $eax
 ; X32-SSE-NEXT:    # kill: def $dl killed $dl killed $edx
 ; X32-SSE-NEXT:    # kill: def $cl killed $cl killed $ecx
@@ -190,15 +190,15 @@ define <3 x i8> @xor_v3i8_as_i24(<3 x i8> %a, <3 x i8> %b) nounwind {
 ; X64-SSE-LABEL: xor_v3i8_as_i24:
 ; X64-SSE:       # %bb.0:
 ; X64-SSE-NEXT:    movd %ecx, %xmm0
-; X64-SSE-NEXT:    pinsrd $1, %r8d, %xmm0
-; X64-SSE-NEXT:    pinsrd $2, %r9d, %xmm0
+; X64-SSE-NEXT:    pinsrb $1, %r8d, %xmm0
+; X64-SSE-NEXT:    pinsrb $2, %r9d, %xmm0
 ; X64-SSE-NEXT:    movd %edi, %xmm1
-; X64-SSE-NEXT:    pinsrd $1, %esi, %xmm1
-; X64-SSE-NEXT:    pinsrd $2, %edx, %xmm1
+; X64-SSE-NEXT:    pinsrb $1, %esi, %xmm1
+; X64-SSE-NEXT:    pinsrb $2, %edx, %xmm1
 ; X64-SSE-NEXT:    pxor %xmm0, %xmm1
 ; X64-SSE-NEXT:    pextrb $0, %xmm1, %eax
-; X64-SSE-NEXT:    pextrb $4, %xmm1, %edx
-; X64-SSE-NEXT:    pextrb $8, %xmm1, %ecx
+; X64-SSE-NEXT:    pextrb $1, %xmm1, %edx
+; X64-SSE-NEXT:    pextrb $2, %xmm1, %ecx
 ; X64-SSE-NEXT:    # kill: def $al killed $al killed $eax
 ; X64-SSE-NEXT:    # kill: def $dl killed $dl killed $edx
 ; X64-SSE-NEXT:    # kill: def $cl killed $cl killed $ecx
@@ -214,15 +214,15 @@ define <3 x i8> @or_v3i8_as_i24(<3 x i8> %a, <3 x i8> %b) nounwind {
 ; X32-SSE-LABEL: or_v3i8_as_i24:
 ; X32-SSE:       # %bb.0:
 ; X32-SSE-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X32-SSE-NEXT:    pinsrb $4, {{[0-9]+}}(%esp), %xmm0
-; X32-SSE-NEXT:    pinsrb $8, {{[0-9]+}}(%esp), %xmm0
+; X32-SSE-NEXT:    pinsrb $1, {{[0-9]+}}(%esp), %xmm0
+; X32-SSE-NEXT:    pinsrb $2, {{[0-9]+}}(%esp), %xmm0
 ; X32-SSE-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; X32-SSE-NEXT:    pinsrb $4, {{[0-9]+}}(%esp), %xmm1
-; X32-SSE-NEXT:    pinsrb $8, {{[0-9]+}}(%esp), %xmm1
+; X32-SSE-NEXT:    pinsrb $1, {{[0-9]+}}(%esp), %xmm1
+; X32-SSE-NEXT:    pinsrb $2, {{[0-9]+}}(%esp), %xmm1
 ; X32-SSE-NEXT:    por %xmm0, %xmm1
 ; X32-SSE-NEXT:    pextrb $0, %xmm1, %eax
-; X32-SSE-NEXT:    pextrb $4, %xmm1, %edx
-; X32-SSE-NEXT:    pextrb $8, %xmm1, %ecx
+; X32-SSE-NEXT:    pextrb $1, %xmm1, %edx
+; X32-SSE-NEXT:    pextrb $2, %xmm1, %ecx
 ; X32-SSE-NEXT:    # kill: def $al killed $al killed $eax
 ; X32-SSE-NEXT:    # kill: def $dl killed $dl killed $edx
 ; X32-SSE-NEXT:    # kill: def $cl killed $cl killed $ecx
@@ -231,15 +231,15 @@ define <3 x i8> @or_v3i8_as_i24(<3 x i8> %a, <3 x i8> %b) nounwind {
 ; X64-SSE-LABEL: or_v3i8_as_i24:
 ; X64-SSE:       # %bb.0:
 ; X64-SSE-NEXT:    movd %ecx, %xmm0
-; X64-SSE-NEXT:    pinsrd $1, %r8d, %xmm0
-; X64-SSE-NEXT:    pinsrd $2, %r9d, %xmm0
+; X64-SSE-NEXT:    pinsrb $1, %r8d, %xmm0
+; X64-SSE-NEXT:    pinsrb $2, %r9d, %xmm0
 ; X64-SSE-NEXT:    movd %edi, %xmm1
-; X64-SSE-NEXT:    pinsrd $1, %esi, %xmm1
-; X64-SSE-NEXT:    pinsrd $2, %edx, %xmm1
+; X64-SSE-NEXT:    pinsrb $1, %esi, %xmm1
+; X64-SSE-NEXT:    pinsrb $2, %edx, %xmm1
 ; X64-SSE-NEXT:    por %xmm0, %xmm1
 ; X64-SSE-NEXT:    pextrb $0, %xmm1, %eax
-; X64-SSE-NEXT:    pextrb $4, %xmm1, %edx
-; X64-SSE-NEXT:    pextrb $8, %xmm1, %ecx
+; X64-SSE-NEXT:    pextrb $1, %xmm1, %edx
+; X64-SSE-NEXT:    pextrb $2, %xmm1, %ecx
 ; X64-SSE-NEXT:    # kill: def $al killed $al killed $eax
 ; X64-SSE-NEXT:    # kill: def $dl killed $dl killed $edx
 ; X64-SSE-NEXT:    # kill: def $cl killed $cl killed $ecx
diff --git a/llvm/test/CodeGen/X86/widen_cast-1.ll b/llvm/test/CodeGen/X86/widen_cast-1.ll
index 63bb47255..2401e005b 100644
--- a/llvm/test/CodeGen/X86/widen_cast-1.ll
+++ b/llvm/test/CodeGen/X86/widen_cast-1.ll
@@ -12,7 +12,6 @@ define void @convert(<2 x i32>* %dst, <4 x i16>* %src) nounwind {
 ; CHECK-NEXT:    pushl %eax
 ; CHECK-NEXT:    movl $0, (%esp)
 ; CHECK-NEXT:    pcmpeqd %xmm0, %xmm0
-; CHECK-NEXT:    movdqa {{.*#+}} xmm1 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
 ; CHECK-NEXT:    cmpl $3, (%esp)
 ; CHECK-NEXT:    jg .LBB0_3
 ; CHECK-NEXT:    .p2align 4, 0x90
@@ -21,10 +20,9 @@ define void @convert(<2 x i32>* %dst, <4 x i16>* %src) nounwind {
 ; CHECK-NEXT:    movl (%esp), %eax
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; CHECK-NEXT:    pmovzxwd {{.*#+}} xmm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
-; CHECK-NEXT:    psubw %xmm0, %xmm2
-; CHECK-NEXT:    pshufb %xmm1, %xmm2
-; CHECK-NEXT:    movq %xmm2, (%ecx,%eax,8)
+; CHECK-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
+; CHECK-NEXT:    psubw %xmm0, %xmm1
+; CHECK-NEXT:    movq %xmm1, (%ecx,%eax,8)
 ; CHECK-NEXT:    incl (%esp)
 ; CHECK-NEXT:    cmpl $3, (%esp)
 ; CHECK-NEXT:    jle .LBB0_2
@@ -36,7 +34,6 @@ define void @convert(<2 x i32>* %dst, <4 x i16>* %src) nounwind {
 ; ATOM:       # %bb.0: # %entry
 ; ATOM-NEXT:    pushl %eax
 ; ATOM-NEXT:    pcmpeqd %xmm0, %xmm0
-; ATOM-NEXT:    movdqa {{.*#+}} xmm1 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
 ; ATOM-NEXT:    movl $0, (%esp)
 ; ATOM-NEXT:    cmpl $3, (%esp)
 ; ATOM-NEXT:    jg .LBB0_3
@@ -45,12 +42,10 @@ define void @convert(<2 x i32>* %dst, <4 x i16>* %src) nounwind {
 ; ATOM-NEXT:    # =>This Inner Loop Header: Depth=1
 ; ATOM-NEXT:    movl (%esp), %eax
 ; ATOM-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; ATOM-NEXT:    movq {{.*#+}} xmm2 = mem[0],zero
+; ATOM-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
 ; ATOM-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; ATOM-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
-; ATOM-NEXT:    psubw %xmm0, %xmm2
-; ATOM-NEXT:    pshufb %xmm1, %xmm2
-; ATOM-NEXT:    movq %xmm2, (%ecx,%eax,8)
+; ATOM-NEXT:    psubw %xmm0, %xmm1
+; ATOM-NEXT:    movq %xmm1, (%ecx,%eax,8)
 ; ATOM-NEXT:    incl (%esp)
 ; ATOM-NEXT:    cmpl $3, (%esp)
 ; ATOM-NEXT:    jle .LBB0_2
diff --git a/llvm/test/CodeGen/X86/widen_cast-2.ll b/llvm/test/CodeGen/X86/widen_cast-2.ll
index 0bbcd391d..e7780912c 100644
--- a/llvm/test/CodeGen/X86/widen_cast-2.ll
+++ b/llvm/test/CodeGen/X86/widen_cast-2.ll
@@ -21,8 +21,9 @@ define void @convert(<7 x i32>* %dst, <14 x i16>* %src) nounwind {
 ; CHECK-NEXT:    movdqa 16(%edx,%eax), %xmm2
 ; CHECK-NEXT:    psubw %xmm0, %xmm1
 ; CHECK-NEXT:    psubw %xmm0, %xmm2
+; CHECK-NEXT:    movd %xmm2, 16(%ecx,%eax)
+; CHECK-NEXT:    pextrd $1, %xmm2, 20(%ecx,%eax)
 ; CHECK-NEXT:    pextrd $2, %xmm2, 24(%ecx,%eax)
-; CHECK-NEXT:    movq %xmm2, 16(%ecx,%eax)
 ; CHECK-NEXT:    movdqa %xmm1, (%ecx,%eax)
 ; CHECK-NEXT:    incl (%esp)
 ; CHECK-NEXT:    cmpl $3, (%esp)
diff --git a/llvm/test/CodeGen/X86/widen_cast-3.ll b/llvm/test/CodeGen/X86/widen_cast-3.ll
index a4d37823d..18a04c48a 100644
--- a/llvm/test/CodeGen/X86/widen_cast-3.ll
+++ b/llvm/test/CodeGen/X86/widen_cast-3.ll
@@ -11,7 +11,8 @@ define void @convert(<12 x i8>* %dst.addr, <3 x i32> %src) nounwind {
 ; X86-NEXT:    pcmpeqd %xmm1, %xmm1
 ; X86-NEXT:    psubd %xmm1, %xmm0
 ; X86-NEXT:    pextrd $2, %xmm0, 8(%eax)
-; X86-NEXT:    movq %xmm0, (%eax)
+; X86-NEXT:    pextrd $1, %xmm0, 4(%eax)
+; X86-NEXT:    movd %xmm0, (%eax)
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: convert:
diff --git a/llvm/test/CodeGen/X86/widen_cast-4.ll b/llvm/test/CodeGen/X86/widen_cast-4.ll
index 9b29fc8b6..f317d4b59 100644
--- a/llvm/test/CodeGen/X86/widen_cast-4.ll
+++ b/llvm/test/CodeGen/X86/widen_cast-4.ll
@@ -1,44 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse4.2 | FileCheck %s --check-prefix=NARROW
-; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse4.2 -x86-experimental-vector-widening-legalization | FileCheck %s --check-prefix=WIDE
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse4.2 | FileCheck %s --check-prefix=WIDE
 
 ; FIXME: We shouldn't require both a movd and an insert in the wide version.
 
 define void @update(i64* %dst_i, i64* %src_i, i32 %n) nounwind {
-; NARROW-LABEL: update:
-; NARROW:       # %bb.0: # %entry
-; NARROW-NEXT:    subl $12, %esp
-; NARROW-NEXT:    movl $0, (%esp)
-; NARROW-NEXT:    pcmpeqd %xmm0, %xmm0
-; NARROW-NEXT:    movdqa {{.*#+}} xmm1 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
-; NARROW-NEXT:    .p2align 4, 0x90
-; NARROW-NEXT:  .LBB0_1: # %forcond
-; NARROW-NEXT:    # =>This Inner Loop Header: Depth=1
-; NARROW-NEXT:    movl (%esp), %eax
-; NARROW-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
-; NARROW-NEXT:    jge .LBB0_3
-; NARROW-NEXT:  # %bb.2: # %forbody
-; NARROW-NEXT:    # in Loop: Header=BB0_1 Depth=1
-; NARROW-NEXT:    movl (%esp), %eax
-; NARROW-NEXT:    leal (,%eax,8), %ecx
-; NARROW-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; NARROW-NEXT:    addl %ecx, %edx
-; NARROW-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; NARROW-NEXT:    addl {{[0-9]+}}(%esp), %ecx
-; NARROW-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; NARROW-NEXT:    pmovzxbw {{.*#+}} xmm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
-; NARROW-NEXT:    psubw %xmm0, %xmm2
-; NARROW-NEXT:    psllw $8, %xmm2
-; NARROW-NEXT:    psraw $8, %xmm2
-; NARROW-NEXT:    psrlw $2, %xmm2
-; NARROW-NEXT:    pshufb %xmm1, %xmm2
-; NARROW-NEXT:    movq %xmm2, (%edx,%eax,8)
-; NARROW-NEXT:    incl (%esp)
-; NARROW-NEXT:    jmp .LBB0_1
-; NARROW-NEXT:  .LBB0_3: # %afterfor
-; NARROW-NEXT:    addl $12, %esp
-; NARROW-NEXT:    retl
-;
 ; WIDE-LABEL: update:
 ; WIDE:       # %bb.0: # %entry
 ; WIDE-NEXT:    subl $12, %esp
diff --git a/llvm/test/CodeGen/X86/widen_cast-5.ll b/llvm/test/CodeGen/X86/widen_cast-5.ll
index b0363d023..5afffd3d8 100644
--- a/llvm/test/CodeGen/X86/widen_cast-5.ll
+++ b/llvm/test/CodeGen/X86/widen_cast-5.ll
@@ -8,18 +8,15 @@ define void @convert(<2 x i32>* %dst.addr, i64 %src) nounwind {
 ; X86-LABEL: convert:
 ; X86:       ## %bb.0: ## %entry
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    pmovzxdq {{.*#+}} xmm0 = mem[0],zero,mem[1],zero
-; X86-NEXT:    pxor LCPI0_0, %xmm0
-; X86-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; X86-NEXT:    movq %xmm0, (%eax)
+; X86-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-NEXT:    xorps LCPI0_0, %xmm0
+; X86-NEXT:    movlps %xmm0, (%eax)
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: convert:
 ; X64:       ## %bb.0: ## %entry
 ; X64-NEXT:    movq %rsi, %xmm0
-; X64-NEXT:    pmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
 ; X64-NEXT:    pxor {{.*}}(%rip), %xmm0
-; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; X64-NEXT:    movq %xmm0, (%rdi)
 ; X64-NEXT:    retq
 entry:
diff --git a/llvm/test/CodeGen/X86/widen_cast-6.ll b/llvm/test/CodeGen/X86/widen_cast-6.ll
index c75c3597e..14e62e1e5 100644
--- a/llvm/test/CodeGen/X86/widen_cast-6.ll
+++ b/llvm/test/CodeGen/X86/widen_cast-6.ll
@@ -7,9 +7,7 @@
 define i32 @return_v2hi() nounwind {
 ; X86-LABEL: return_v2hi:
 ; X86:       ## %bb.0: ## %entry
-; X86-NEXT:    pushl %eax
 ; X86-NEXT:    xorl %eax, %eax
-; X86-NEXT:    popl %ecx
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: return_v2hi:
diff --git a/llvm/test/CodeGen/X86/widen_compare-1.ll b/llvm/test/CodeGen/X86/widen_compare-1.ll
index f37e764fd..fc6c807c6 100644
--- a/llvm/test/CodeGen/X86/widen_compare-1.ll
+++ b/llvm/test/CodeGen/X86/widen_compare-1.ll
@@ -7,12 +7,12 @@
 define <2 x i16> @compare_v2i64_to_v2i16_unary(<2 x i16>* %src) nounwind {
 ; X86-LABEL: compare_v2i64_to_v2i16_unary:
 ; X86:       # %bb.0:
-; X86-NEXT:    movaps {{.*#+}} xmm0 = [65535,0,65535,0]
+; X86-NEXT:    pcmpeqd %xmm0, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: compare_v2i64_to_v2i16_unary:
 ; X64:       # %bb.0:
-; X64-NEXT:    movaps {{.*#+}} xmm0 = [65535,65535]
+; X64-NEXT:    pcmpeqd %xmm0, %xmm0
 ; X64-NEXT:    retq
   %val = load <2 x i16>, <2 x i16>* %src, align 4
   %cmp = icmp uge <2 x i16> %val, %val
@@ -25,20 +25,18 @@ define <2 x i16> @compare_v2i64_to_v2i16_binary(<2 x i16>* %src0, <2 x i16>* %sr
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    pmovzxwq {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero
-; X86-NEXT:    pmovzxwq {{.*#+}} xmm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero
-; X86-NEXT:    pcmpgtq %xmm0, %xmm1
-; X86-NEXT:    pcmpeqd %xmm0, %xmm0
-; X86-NEXT:    pxor %xmm1, %xmm0
+; X86-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; X86-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-NEXT:    pmaxuw %xmm1, %xmm0
+; X86-NEXT:    pcmpeqw %xmm1, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: compare_v2i64_to_v2i16_binary:
 ; X64:       # %bb.0:
-; X64-NEXT:    pmovzxwq {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero
-; X64-NEXT:    pmovzxwq {{.*#+}} xmm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero
-; X64-NEXT:    pcmpgtq %xmm0, %xmm1
-; X64-NEXT:    pcmpeqd %xmm0, %xmm0
-; X64-NEXT:    pxor %xmm1, %xmm0
+; X64-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; X64-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X64-NEXT:    pmaxuw %xmm1, %xmm0
+; X64-NEXT:    pcmpeqw %xmm1, %xmm0
 ; X64-NEXT:    retq
   %val0 = load <2 x i16>, <2 x i16>* %src0, align 4
   %val1 = load <2 x i16>, <2 x i16>* %src1, align 4
diff --git a/llvm/test/CodeGen/X86/widen_conv-1.ll b/llvm/test/CodeGen/X86/widen_conv-1.ll
index 123e4438b..9c836eeb8 100644
--- a/llvm/test/CodeGen/X86/widen_conv-1.ll
+++ b/llvm/test/CodeGen/X86/widen_conv-1.ll
@@ -8,16 +8,17 @@ define void @convert_v2i64_to_v2i32(<2 x i32>* %dst.addr, <2 x i64> %src) nounwi
 ; X86-LABEL: convert_v2i64_to_v2i32:
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; X86-NEXT:    pcmpeqd %xmm1, %xmm1
 ; X86-NEXT:    psubd %xmm1, %xmm0
-; X86-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; X86-NEXT:    movq %xmm0, (%eax)
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: convert_v2i64_to_v2i32:
 ; X64:       # %bb.0: # %entry
-; X64-NEXT:    paddd {{.*}}(%rip), %xmm0
 ; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X64-NEXT:    pcmpeqd %xmm1, %xmm1
+; X64-NEXT:    psubd %xmm1, %xmm0
 ; X64-NEXT:    movq %xmm0, (%rdi)
 ; X64-NEXT:    retq
 entry:
@@ -32,25 +33,23 @@ entry:
 define void @convert_v3i32_to_v3i8(<3 x i8>* %dst.addr, <3 x i32>* %src.addr) nounwind {
 ; X86-LABEL: convert_v3i32_to_v3i8:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    pushl %eax
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movdqa (%ecx), %xmm0
-; X86-NEXT:    pcmpeqd %xmm1, %xmm1
-; X86-NEXT:    psubd %xmm1, %xmm0
-; X86-NEXT:    pextrb $8, %xmm0, 2(%eax)
 ; X86-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; X86-NEXT:    pcmpeqd %xmm1, %xmm1
+; X86-NEXT:    psubb %xmm1, %xmm0
+; X86-NEXT:    pextrb $2, %xmm0, 2(%eax)
 ; X86-NEXT:    pextrw $0, %xmm0, (%eax)
-; X86-NEXT:    popl %eax
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: convert_v3i32_to_v3i8:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    movdqa (%rsi), %xmm0
-; X64-NEXT:    pcmpeqd %xmm1, %xmm1
-; X64-NEXT:    psubd %xmm1, %xmm0
-; X64-NEXT:    pextrb $8, %xmm0, 2(%rdi)
 ; X64-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; X64-NEXT:    pcmpeqd %xmm1, %xmm1
+; X64-NEXT:    psubb %xmm1, %xmm0
+; X64-NEXT:    pextrb $2, %xmm0, 2(%rdi)
 ; X64-NEXT:    pextrw $0, %xmm0, (%rdi)
 ; X64-NEXT:    retq
 entry:
@@ -66,29 +65,23 @@ entry:
 define void @convert_v5i16_to_v5i8(<5 x i8>* %dst.addr, <5 x i16>* %src.addr) nounwind {
 ; X86-LABEL: convert_v5i16_to_v5i8:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    pushl %ebp
-; X86-NEXT:    movl %esp, %ebp
-; X86-NEXT:    andl $-8, %esp
-; X86-NEXT:    subl $8, %esp
-; X86-NEXT:    movl 8(%ebp), %eax
-; X86-NEXT:    movl 12(%ebp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movdqa (%ecx), %xmm0
-; X86-NEXT:    pcmpeqd %xmm1, %xmm1
-; X86-NEXT:    psubw %xmm1, %xmm0
-; X86-NEXT:    pextrb $8, %xmm0, 4(%eax)
 ; X86-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
+; X86-NEXT:    pcmpeqd %xmm1, %xmm1
+; X86-NEXT:    psubb %xmm1, %xmm0
+; X86-NEXT:    pextrb $4, %xmm0, 4(%eax)
 ; X86-NEXT:    movd %xmm0, (%eax)
-; X86-NEXT:    movl %ebp, %esp
-; X86-NEXT:    popl %ebp
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: convert_v5i16_to_v5i8:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    movdqa (%rsi), %xmm0
-; X64-NEXT:    pcmpeqd %xmm1, %xmm1
-; X64-NEXT:    psubw %xmm1, %xmm0
-; X64-NEXT:    pextrb $8, %xmm0, 4(%rdi)
 ; X64-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
+; X64-NEXT:    pcmpeqd %xmm1, %xmm1
+; X64-NEXT:    psubb %xmm1, %xmm0
+; X64-NEXT:    pextrb $4, %xmm0, 4(%rdi)
 ; X64-NEXT:    movd %xmm0, (%rdi)
 ; X64-NEXT:    retq
 entry:
diff --git a/llvm/test/CodeGen/X86/widen_conv-2.ll b/llvm/test/CodeGen/X86/widen_conv-2.ll
index 3a39cbfba..ccaadec08 100644
--- a/llvm/test/CodeGen/X86/widen_conv-2.ll
+++ b/llvm/test/CodeGen/X86/widen_conv-2.ll
@@ -8,17 +8,13 @@ define void @convert_v2i16_v2i32(<2 x i32>* %dst.addr, <2 x i16> %src) nounwind
 ; X86-LABEL: convert_v2i16_v2i32:
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    psllq $48, %xmm0
-; X86-NEXT:    psrad $16, %xmm0
-; X86-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
+; X86-NEXT:    pmovsxwd %xmm0, %xmm0
 ; X86-NEXT:    movq %xmm0, (%eax)
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: convert_v2i16_v2i32:
 ; X64:       # %bb.0: # %entry
-; X64-NEXT:    psllq $48, %xmm0
-; X64-NEXT:    psrad $16, %xmm0
-; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
+; X64-NEXT:    pmovsxwd %xmm0, %xmm0
 ; X64-NEXT:    movq %xmm0, (%rdi)
 ; X64-NEXT:    retq
 entry:
diff --git a/llvm/test/CodeGen/X86/widen_conv-3.ll b/llvm/test/CodeGen/X86/widen_conv-3.ll
index d8cdc8c6b..21e80c99d 100644
--- a/llvm/test/CodeGen/X86/widen_conv-3.ll
+++ b/llvm/test/CodeGen/X86/widen_conv-3.ll
@@ -7,24 +7,37 @@
 ; sign to float v2i16 to v2f32
 
 define void @convert_v2i16_to_v2f32(<2 x float>* %dst.addr, <2 x i16> %src) nounwind {
-; X86-LABEL: convert_v2i16_to_v2f32:
-; X86:       # %bb.0: # %entry
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    psllq $48, %xmm0
-; X86-NEXT:    psrad $16, %xmm0
-; X86-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
-; X86-NEXT:    cvtdq2ps %xmm0, %xmm0
-; X86-NEXT:    movlps %xmm0, (%eax)
-; X86-NEXT:    retl
+; X86-SSE2-LABEL: convert_v2i16_to_v2f32:
+; X86-SSE2:       # %bb.0: # %entry
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; X86-SSE2-NEXT:    psrad $16, %xmm0
+; X86-SSE2-NEXT:    cvtdq2ps %xmm0, %xmm0
+; X86-SSE2-NEXT:    movlps %xmm0, (%eax)
+; X86-SSE2-NEXT:    retl
+;
+; X86-SSE42-LABEL: convert_v2i16_to_v2f32:
+; X86-SSE42:       # %bb.0: # %entry
+; X86-SSE42-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE42-NEXT:    pmovsxwd %xmm0, %xmm0
+; X86-SSE42-NEXT:    cvtdq2ps %xmm0, %xmm0
+; X86-SSE42-NEXT:    movlps %xmm0, (%eax)
+; X86-SSE42-NEXT:    retl
 ;
-; X64-LABEL: convert_v2i16_to_v2f32:
-; X64:       # %bb.0: # %entry
-; X64-NEXT:    psllq $48, %xmm0
-; X64-NEXT:    psrad $16, %xmm0
-; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
-; X64-NEXT:    cvtdq2ps %xmm0, %xmm0
-; X64-NEXT:    movlps %xmm0, (%rdi)
-; X64-NEXT:    retq
+; X64-SSE2-LABEL: convert_v2i16_to_v2f32:
+; X64-SSE2:       # %bb.0: # %entry
+; X64-SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; X64-SSE2-NEXT:    psrad $16, %xmm0
+; X64-SSE2-NEXT:    cvtdq2ps %xmm0, %xmm0
+; X64-SSE2-NEXT:    movlps %xmm0, (%rdi)
+; X64-SSE2-NEXT:    retq
+;
+; X64-SSE42-LABEL: convert_v2i16_to_v2f32:
+; X64-SSE42:       # %bb.0: # %entry
+; X64-SSE42-NEXT:    pmovsxwd %xmm0, %xmm0
+; X64-SSE42-NEXT:    cvtdq2ps %xmm0, %xmm0
+; X64-SSE42-NEXT:    movlps %xmm0, (%rdi)
+; X64-SSE42-NEXT:    retq
 entry:
 	%val = sitofp <2 x i16> %src to <2 x float>
 	store <2 x float> %val, <2 x float>* %dst.addr, align 4
@@ -36,28 +49,19 @@ entry:
 define void @convert_v3i8_to_v3f32(<3 x float>* %dst.addr, <3 x i8>* %src.addr) nounwind {
 ; X86-SSE2-LABEL: convert_v3i8_to_v3f32:
 ; X86-SSE2:       # %bb.0: # %entry
-; X86-SSE2-NEXT:    pushl %ebp
-; X86-SSE2-NEXT:    movl %esp, %ebp
-; X86-SSE2-NEXT:    pushl %esi
-; X86-SSE2-NEXT:    andl $-16, %esp
-; X86-SSE2-NEXT:    subl $32, %esp
-; X86-SSE2-NEXT:    movl 8(%ebp), %eax
-; X86-SSE2-NEXT:    movl 12(%ebp), %ecx
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-SSE2-NEXT:    movzwl (%ecx), %edx
 ; X86-SSE2-NEXT:    movd %edx, %xmm0
-; X86-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X86-SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
-; X86-SSE2-NEXT:    movdqa %xmm0, (%esp)
-; X86-SSE2-NEXT:    movl (%esp), %edx
-; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X86-SSE2-NEXT:    shll $8, %edx
-; X86-SSE2-NEXT:    pxor %xmm0, %xmm0
-; X86-SSE2-NEXT:    pinsrw $1, %edx, %xmm0
-; X86-SSE2-NEXT:    shll $8, %esi
-; X86-SSE2-NEXT:    pinsrw $3, %esi, %xmm0
+; X86-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X86-SSE2-NEXT:    pand %xmm1, %xmm0
 ; X86-SSE2-NEXT:    movzbl 2(%ecx), %ecx
-; X86-SSE2-NEXT:    shll $8, %ecx
-; X86-SSE2-NEXT:    pinsrw $5, %ecx, %xmm0
+; X86-SSE2-NEXT:    movd %ecx, %xmm2
+; X86-SSE2-NEXT:    pslld $16, %xmm2
+; X86-SSE2-NEXT:    pandn %xmm2, %xmm1
+; X86-SSE2-NEXT:    por %xmm0, %xmm1
+; X86-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; X86-SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
 ; X86-SSE2-NEXT:    psrad $24, %xmm0
 ; X86-SSE2-NEXT:    cvtdq2ps %xmm0, %xmm0
 ; X86-SSE2-NEXT:    movss %xmm0, (%eax)
@@ -66,47 +70,35 @@ define void @convert_v3i8_to_v3f32(<3 x float>* %dst.addr, <3 x i8>* %src.addr)
 ; X86-SSE2-NEXT:    movss %xmm1, 8(%eax)
 ; X86-SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
 ; X86-SSE2-NEXT:    movss %xmm0, 4(%eax)
-; X86-SSE2-NEXT:    leal -4(%ebp), %esp
-; X86-SSE2-NEXT:    popl %esi
-; X86-SSE2-NEXT:    popl %ebp
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-SSE42-LABEL: convert_v3i8_to_v3f32:
 ; X86-SSE42:       # %bb.0: # %entry
-; X86-SSE42-NEXT:    pushl %eax
 ; X86-SSE42-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-SSE42-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-SSE42-NEXT:    movzbl 2(%ecx), %edx
-; X86-SSE42-NEXT:    movzwl (%ecx), %ecx
-; X86-SSE42-NEXT:    movd %ecx, %xmm0
-; X86-SSE42-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; X86-SSE42-NEXT:    pinsrd $2, %edx, %xmm0
-; X86-SSE42-NEXT:    pslld $24, %xmm0
-; X86-SSE42-NEXT:    psrad $24, %xmm0
+; X86-SSE42-NEXT:    movzwl (%ecx), %edx
+; X86-SSE42-NEXT:    movd %edx, %xmm0
+; X86-SSE42-NEXT:    pinsrb $2, 2(%ecx), %xmm0
+; X86-SSE42-NEXT:    pmovsxbd %xmm0, %xmm0
 ; X86-SSE42-NEXT:    cvtdq2ps %xmm0, %xmm0
 ; X86-SSE42-NEXT:    extractps $2, %xmm0, 8(%eax)
 ; X86-SSE42-NEXT:    extractps $1, %xmm0, 4(%eax)
 ; X86-SSE42-NEXT:    movss %xmm0, (%eax)
-; X86-SSE42-NEXT:    popl %eax
 ; X86-SSE42-NEXT:    retl
 ;
 ; X64-SSE2-LABEL: convert_v3i8_to_v3f32:
 ; X64-SSE2:       # %bb.0: # %entry
 ; X64-SSE2-NEXT:    movzwl (%rsi), %eax
 ; X64-SSE2-NEXT:    movd %eax, %xmm0
-; X64-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X64-SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
-; X64-SSE2-NEXT:    movdqa %xmm0, -{{[0-9]+}}(%rsp)
-; X64-SSE2-NEXT:    movl -{{[0-9]+}}(%rsp), %eax
-; X64-SSE2-NEXT:    movl -{{[0-9]+}}(%rsp), %ecx
-; X64-SSE2-NEXT:    shll $8, %eax
-; X64-SSE2-NEXT:    pxor %xmm0, %xmm0
-; X64-SSE2-NEXT:    pinsrw $1, %eax, %xmm0
-; X64-SSE2-NEXT:    shll $8, %ecx
-; X64-SSE2-NEXT:    pinsrw $3, %ecx, %xmm0
+; X64-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X64-SSE2-NEXT:    pand %xmm1, %xmm0
 ; X64-SSE2-NEXT:    movzbl 2(%rsi), %eax
-; X64-SSE2-NEXT:    shll $8, %eax
-; X64-SSE2-NEXT:    pinsrw $5, %eax, %xmm0
+; X64-SSE2-NEXT:    movd %eax, %xmm2
+; X64-SSE2-NEXT:    pslld $16, %xmm2
+; X64-SSE2-NEXT:    pandn %xmm2, %xmm1
+; X64-SSE2-NEXT:    por %xmm0, %xmm1
+; X64-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; X64-SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
 ; X64-SSE2-NEXT:    psrad $24, %xmm0
 ; X64-SSE2-NEXT:    cvtdq2ps %xmm0, %xmm0
 ; X64-SSE2-NEXT:    movlps %xmm0, (%rdi)
@@ -116,13 +108,10 @@ define void @convert_v3i8_to_v3f32(<3 x float>* %dst.addr, <3 x i8>* %src.addr)
 ;
 ; X64-SSE42-LABEL: convert_v3i8_to_v3f32:
 ; X64-SSE42:       # %bb.0: # %entry
-; X64-SSE42-NEXT:    movzbl 2(%rsi), %eax
-; X64-SSE42-NEXT:    movzwl (%rsi), %ecx
-; X64-SSE42-NEXT:    movd %ecx, %xmm0
-; X64-SSE42-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; X64-SSE42-NEXT:    pinsrd $2, %eax, %xmm0
-; X64-SSE42-NEXT:    pslld $24, %xmm0
-; X64-SSE42-NEXT:    psrad $24, %xmm0
+; X64-SSE42-NEXT:    movzwl (%rsi), %eax
+; X64-SSE42-NEXT:    movd %eax, %xmm0
+; X64-SSE42-NEXT:    pinsrb $2, 2(%rsi), %xmm0
+; X64-SSE42-NEXT:    pmovsxbd %xmm0, %xmm0
 ; X64-SSE42-NEXT:    cvtdq2ps %xmm0, %xmm0
 ; X64-SSE42-NEXT:    extractps $2, %xmm0, 8(%rdi)
 ; X64-SSE42-NEXT:    movlps %xmm0, (%rdi)
diff --git a/llvm/test/CodeGen/X86/widen_conv-4.ll b/llvm/test/CodeGen/X86/widen_conv-4.ll
index e807d6f40..9c567fc38 100644
--- a/llvm/test/CodeGen/X86/widen_conv-4.ll
+++ b/llvm/test/CodeGen/X86/widen_conv-4.ll
@@ -28,15 +28,15 @@ define void @convert_v7i16_v7f32(<7 x float>* %dst.addr, <7 x i16> %src) nounwin
 ; X86-SSE42-LABEL: convert_v7i16_v7f32:
 ; X86-SSE42:       # %bb.0: # %entry
 ; X86-SSE42-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-SSE42-NEXT:    pxor %xmm1, %xmm1
-; X86-SSE42-NEXT:    pmovzxwd {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; X86-SSE42-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE42-NEXT:    pmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; X86-SSE42-NEXT:    cvtdq2ps %xmm1, %xmm1
+; X86-SSE42-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; X86-SSE42-NEXT:    cvtdq2ps %xmm0, %xmm0
-; X86-SSE42-NEXT:    cvtdq2ps %xmm2, %xmm1
-; X86-SSE42-NEXT:    extractps $2, %xmm0, 24(%eax)
-; X86-SSE42-NEXT:    extractps $1, %xmm0, 20(%eax)
-; X86-SSE42-NEXT:    movups %xmm1, (%eax)
-; X86-SSE42-NEXT:    movss %xmm0, 16(%eax)
+; X86-SSE42-NEXT:    movups %xmm0, (%eax)
+; X86-SSE42-NEXT:    extractps $2, %xmm1, 24(%eax)
+; X86-SSE42-NEXT:    extractps $1, %xmm1, 20(%eax)
+; X86-SSE42-NEXT:    movss %xmm1, 16(%eax)
 ; X86-SSE42-NEXT:    retl
 ;
 ; X64-SSE2-LABEL: convert_v7i16_v7f32:
@@ -55,14 +55,14 @@ define void @convert_v7i16_v7f32(<7 x float>* %dst.addr, <7 x i16> %src) nounwin
 ;
 ; X64-SSE42-LABEL: convert_v7i16_v7f32:
 ; X64-SSE42:       # %bb.0: # %entry
-; X64-SSE42-NEXT:    pxor %xmm1, %xmm1
-; X64-SSE42-NEXT:    pmovzxwd {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; X64-SSE42-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-SSE42-NEXT:    pmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; X64-SSE42-NEXT:    cvtdq2ps %xmm1, %xmm1
+; X64-SSE42-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; X64-SSE42-NEXT:    cvtdq2ps %xmm0, %xmm0
-; X64-SSE42-NEXT:    cvtdq2ps %xmm2, %xmm1
-; X64-SSE42-NEXT:    extractps $2, %xmm0, 24(%rdi)
-; X64-SSE42-NEXT:    movlps %xmm0, 16(%rdi)
-; X64-SSE42-NEXT:    movups %xmm1, (%rdi)
+; X64-SSE42-NEXT:    movups %xmm0, (%rdi)
+; X64-SSE42-NEXT:    extractps $2, %xmm1, 24(%rdi)
+; X64-SSE42-NEXT:    movlps %xmm1, 16(%rdi)
 ; X64-SSE42-NEXT:    retq
 entry:
 	%val = uitofp <7 x i16> %src to <7 x float>
@@ -75,66 +75,58 @@ entry:
 define void @convert_v3i8_to_v3f32(<3 x float>* %dst.addr, <3 x i8>* %src.addr) nounwind {
 ; X86-SSE2-LABEL: convert_v3i8_to_v3f32:
 ; X86-SSE2:       # %bb.0: # %entry
-; X86-SSE2-NEXT:    pushl %ebp
-; X86-SSE2-NEXT:    movl %esp, %ebp
-; X86-SSE2-NEXT:    andl $-16, %esp
-; X86-SSE2-NEXT:    subl $32, %esp
-; X86-SSE2-NEXT:    movl 8(%ebp), %eax
-; X86-SSE2-NEXT:    movl 12(%ebp), %ecx
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-SSE2-NEXT:    movzwl (%ecx), %edx
 ; X86-SSE2-NEXT:    movd %edx, %xmm0
-; X86-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X86-SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; X86-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X86-SSE2-NEXT:    pand %xmm1, %xmm0
 ; X86-SSE2-NEXT:    movzbl 2(%ecx), %ecx
-; X86-SSE2-NEXT:    movdqa %xmm0, (%esp)
-; X86-SSE2-NEXT:    movzbl (%esp), %edx
-; X86-SSE2-NEXT:    movd %edx, %xmm0
-; X86-SSE2-NEXT:    movzbl {{[0-9]+}}(%esp), %edx
-; X86-SSE2-NEXT:    pinsrw $2, %edx, %xmm0
-; X86-SSE2-NEXT:    pinsrw $4, %ecx, %xmm0
-; X86-SSE2-NEXT:    cvtdq2ps %xmm0, %xmm0
+; X86-SSE2-NEXT:    movd %ecx, %xmm2
+; X86-SSE2-NEXT:    pslld $16, %xmm2
+; X86-SSE2-NEXT:    pandn %xmm2, %xmm1
+; X86-SSE2-NEXT:    por %xmm0, %xmm1
+; X86-SSE2-NEXT:    pxor %xmm0, %xmm0
+; X86-SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; X86-SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; X86-SSE2-NEXT:    cvtdq2ps %xmm1, %xmm0
 ; X86-SSE2-NEXT:    movss %xmm0, (%eax)
 ; X86-SSE2-NEXT:    movaps %xmm0, %xmm1
 ; X86-SSE2-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
 ; X86-SSE2-NEXT:    movss %xmm1, 8(%eax)
 ; X86-SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
 ; X86-SSE2-NEXT:    movss %xmm0, 4(%eax)
-; X86-SSE2-NEXT:    movl %ebp, %esp
-; X86-SSE2-NEXT:    popl %ebp
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-SSE42-LABEL: convert_v3i8_to_v3f32:
 ; X86-SSE42:       # %bb.0: # %entry
-; X86-SSE42-NEXT:    pushl %eax
 ; X86-SSE42-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-SSE42-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-SSE42-NEXT:    movzbl 2(%ecx), %edx
-; X86-SSE42-NEXT:    movzwl (%ecx), %ecx
-; X86-SSE42-NEXT:    movd %ecx, %xmm0
+; X86-SSE42-NEXT:    movzwl (%ecx), %edx
+; X86-SSE42-NEXT:    movd %edx, %xmm0
+; X86-SSE42-NEXT:    pinsrb $2, 2(%ecx), %xmm0
 ; X86-SSE42-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; X86-SSE42-NEXT:    pinsrd $2, %edx, %xmm0
-; X86-SSE42-NEXT:    pand {{\.LCPI.*}}, %xmm0
 ; X86-SSE42-NEXT:    cvtdq2ps %xmm0, %xmm0
 ; X86-SSE42-NEXT:    extractps $2, %xmm0, 8(%eax)
 ; X86-SSE42-NEXT:    extractps $1, %xmm0, 4(%eax)
 ; X86-SSE42-NEXT:    movss %xmm0, (%eax)
-; X86-SSE42-NEXT:    popl %eax
 ; X86-SSE42-NEXT:    retl
 ;
 ; X64-SSE2-LABEL: convert_v3i8_to_v3f32:
 ; X64-SSE2:       # %bb.0: # %entry
 ; X64-SSE2-NEXT:    movzwl (%rsi), %eax
 ; X64-SSE2-NEXT:    movd %eax, %xmm0
-; X64-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X64-SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; X64-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X64-SSE2-NEXT:    pand %xmm1, %xmm0
 ; X64-SSE2-NEXT:    movzbl 2(%rsi), %eax
-; X64-SSE2-NEXT:    movdqa %xmm0, -{{[0-9]+}}(%rsp)
-; X64-SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %ecx
-; X64-SSE2-NEXT:    movd %ecx, %xmm0
-; X64-SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %ecx
-; X64-SSE2-NEXT:    pinsrw $2, %ecx, %xmm0
-; X64-SSE2-NEXT:    pinsrw $4, %eax, %xmm0
-; X64-SSE2-NEXT:    cvtdq2ps %xmm0, %xmm0
+; X64-SSE2-NEXT:    movd %eax, %xmm2
+; X64-SSE2-NEXT:    pslld $16, %xmm2
+; X64-SSE2-NEXT:    pandn %xmm2, %xmm1
+; X64-SSE2-NEXT:    por %xmm0, %xmm1
+; X64-SSE2-NEXT:    pxor %xmm0, %xmm0
+; X64-SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; X64-SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; X64-SSE2-NEXT:    cvtdq2ps %xmm1, %xmm0
 ; X64-SSE2-NEXT:    movlps %xmm0, (%rdi)
 ; X64-SSE2-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
 ; X64-SSE2-NEXT:    movss %xmm0, 8(%rdi)
@@ -142,12 +134,10 @@ define void @convert_v3i8_to_v3f32(<3 x float>* %dst.addr, <3 x i8>* %src.addr)
 ;
 ; X64-SSE42-LABEL: convert_v3i8_to_v3f32:
 ; X64-SSE42:       # %bb.0: # %entry
-; X64-SSE42-NEXT:    movzbl 2(%rsi), %eax
-; X64-SSE42-NEXT:    movzwl (%rsi), %ecx
-; X64-SSE42-NEXT:    movd %ecx, %xmm0
+; X64-SSE42-NEXT:    movzwl (%rsi), %eax
+; X64-SSE42-NEXT:    movd %eax, %xmm0
+; X64-SSE42-NEXT:    pinsrb $2, 2(%rsi), %xmm0
 ; X64-SSE42-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; X64-SSE42-NEXT:    pinsrd $2, %eax, %xmm0
-; X64-SSE42-NEXT:    pand {{.*}}(%rip), %xmm0
 ; X64-SSE42-NEXT:    cvtdq2ps %xmm0, %xmm0
 ; X64-SSE42-NEXT:    extractps $2, %xmm0, 8(%rdi)
 ; X64-SSE42-NEXT:    movlps %xmm0, (%rdi)
diff --git a/llvm/test/CodeGen/X86/widen_conversions.ll b/llvm/test/CodeGen/X86/widen_conversions.ll
index acd8c78fa..d9d43bb4a 100644
--- a/llvm/test/CodeGen/X86/widen_conversions.ll
+++ b/llvm/test/CodeGen/X86/widen_conversions.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse2 -x86-experimental-vector-widening-legalization | FileCheck %s --check-prefix=X86
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 -x86-experimental-vector-widening-legalization | FileCheck %s --check-prefix=X64
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=X86
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=X64
 
 define <4 x i32> @zext_v4i8_to_v4i32(<4 x i8>* %ptr) {
 ; X86-LABEL: zext_v4i8_to_v4i32:
diff --git a/llvm/test/CodeGen/X86/widen_load-2.ll b/llvm/test/CodeGen/X86/widen_load-2.ll
index e4ee28a8f..a816dd0d9 100644
--- a/llvm/test/CodeGen/X86/widen_load-2.ll
+++ b/llvm/test/CodeGen/X86/widen_load-2.ll
@@ -15,7 +15,8 @@ define void @add3i32(%i32vec3*  sret %ret, %i32vec3* %ap, %i32vec3* %bp)  {
 ; X86-NEXT:    movdqa (%edx), %xmm0
 ; X86-NEXT:    paddd (%ecx), %xmm0
 ; X86-NEXT:    pextrd $2, %xmm0, 8(%eax)
-; X86-NEXT:    movq %xmm0, (%eax)
+; X86-NEXT:    pextrd $1, %xmm0, 4(%eax)
+; X86-NEXT:    movd %xmm0, (%eax)
 ; X86-NEXT:    retl $4
 ;
 ; X64-LABEL: add3i32:
@@ -39,13 +40,16 @@ define void @add3i32_2(%i32vec3*  sret %ret, %i32vec3* %ap, %i32vec3* %bp)  {
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; X86-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-NEXT:    pinsrd $1, 4(%edx), %xmm0
 ; X86-NEXT:    pinsrd $2, 8(%edx), %xmm0
-; X86-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
+; X86-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; X86-NEXT:    pinsrd $1, 4(%ecx), %xmm1
 ; X86-NEXT:    pinsrd $2, 8(%ecx), %xmm1
 ; X86-NEXT:    paddd %xmm0, %xmm1
-; X86-NEXT:    movq %xmm1, (%eax)
+; X86-NEXT:    pextrd $1, %xmm1, 4(%eax)
 ; X86-NEXT:    pextrd $2, %xmm1, 8(%eax)
+; X86-NEXT:    movd %xmm1, (%eax)
 ; X86-NEXT:    retl $4
 ;
 ; X64-LABEL: add3i32_2:
@@ -77,8 +81,9 @@ define void @add7i32(%i32vec7*  sret %ret, %i32vec7* %ap, %i32vec7* %bp)  {
 ; X86-NEXT:    movdqa 16(%edx), %xmm1
 ; X86-NEXT:    paddd (%ecx), %xmm0
 ; X86-NEXT:    paddd 16(%ecx), %xmm1
+; X86-NEXT:    movd %xmm1, 16(%eax)
+; X86-NEXT:    pextrd $1, %xmm1, 20(%eax)
 ; X86-NEXT:    pextrd $2, %xmm1, 24(%eax)
-; X86-NEXT:    movq %xmm1, 16(%eax)
 ; X86-NEXT:    movdqa %xmm0, (%eax)
 ; X86-NEXT:    retl $4
 ;
@@ -143,31 +148,25 @@ define void @add12i32(%i32vec12*  sret %ret, %i32vec12* %ap, %i32vec12* %bp)  {
 define void @add3i16(%i16vec3* nocapture sret %ret, %i16vec3* %ap, %i16vec3* %bp) nounwind {
 ; X86-LABEL: add3i16:
 ; X86:       # %bb.0:
-; X86-NEXT:    pushl %ebp
-; X86-NEXT:    movl %esp, %ebp
-; X86-NEXT:    andl $-8, %esp
-; X86-NEXT:    subl $8, %esp
-; X86-NEXT:    movl 8(%ebp), %eax
-; X86-NEXT:    movl 16(%ebp), %ecx
-; X86-NEXT:    movl 12(%ebp), %edx
-; X86-NEXT:    pmovzxwd {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
-; X86-NEXT:    pmovzxwd {{.*#+}} xmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
-; X86-NEXT:    paddd %xmm0, %xmm1
-; X86-NEXT:    pextrw $4, %xmm1, 4(%eax)
-; X86-NEXT:    pshufb {{.*#+}} xmm1 = xmm1[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-NEXT:    pinsrw $2, 4(%edx), %xmm0
+; X86-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; X86-NEXT:    pinsrw $2, 4(%ecx), %xmm1
+; X86-NEXT:    paddw %xmm0, %xmm1
+; X86-NEXT:    pextrw $2, %xmm1, 4(%eax)
 ; X86-NEXT:    movd %xmm1, (%eax)
-; X86-NEXT:    movl %ebp, %esp
-; X86-NEXT:    popl %ebp
 ; X86-NEXT:    retl $4
 ;
 ; X64-LABEL: add3i16:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movq %rdi, %rax
-; X64-NEXT:    pmovzxwd {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
-; X64-NEXT:    pmovzxwd {{.*#+}} xmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
-; X64-NEXT:    paddd %xmm0, %xmm1
-; X64-NEXT:    pextrw $4, %xmm1, 4(%rdi)
-; X64-NEXT:    pshufb {{.*#+}} xmm1 = xmm1[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
+; X64-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; X64-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
+; X64-NEXT:    paddw %xmm0, %xmm1
+; X64-NEXT:    pextrw $2, %xmm1, 4(%rdi)
 ; X64-NEXT:    movd %xmm1, (%rdi)
 ; X64-NEXT:    retq
 	%a = load %i16vec3, %i16vec3* %ap, align 16
@@ -216,7 +215,8 @@ define void @add12i16(%i16vec12* nocapture sret %ret, %i16vec12* %ap, %i16vec12*
 ; X86-NEXT:    movdqa 16(%edx), %xmm1
 ; X86-NEXT:    paddw (%ecx), %xmm0
 ; X86-NEXT:    paddw 16(%ecx), %xmm1
-; X86-NEXT:    movq %xmm1, 16(%eax)
+; X86-NEXT:    movd %xmm1, 16(%eax)
+; X86-NEXT:    pextrd $1, %xmm1, 20(%eax)
 ; X86-NEXT:    movdqa %xmm0, (%eax)
 ; X86-NEXT:    retl $4
 ;
@@ -280,27 +280,23 @@ define void @add18i16(%i16vec18* nocapture sret %ret, %i16vec18* %ap, %i16vec18*
 define void @add3i8(%i8vec3* nocapture sret %ret, %i8vec3* %ap, %i8vec3* %bp) nounwind {
 ; X86-LABEL: add3i8:
 ; X86:       # %bb.0:
-; X86-NEXT:    subl $12, %esp
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    pmovzxbd {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
-; X86-NEXT:    pmovzxbd {{.*#+}} xmm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
-; X86-NEXT:    paddd %xmm0, %xmm1
-; X86-NEXT:    pextrb $8, %xmm1, 2(%eax)
-; X86-NEXT:    pshufb {{.*#+}} xmm1 = xmm1[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; X86-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; X86-NEXT:    paddb %xmm0, %xmm1
+; X86-NEXT:    pextrb $2, %xmm1, 2(%eax)
 ; X86-NEXT:    pextrw $0, %xmm1, (%eax)
-; X86-NEXT:    addl $12, %esp
 ; X86-NEXT:    retl $4
 ;
 ; X64-LABEL: add3i8:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movq %rdi, %rax
-; X64-NEXT:    pmovzxbd {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
-; X64-NEXT:    pmovzxbd {{.*#+}} xmm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
-; X64-NEXT:    paddd %xmm0, %xmm1
-; X64-NEXT:    pextrb $8, %xmm1, 2(%rdi)
-; X64-NEXT:    pshufb {{.*#+}} xmm1 = xmm1[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; X64-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X64-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; X64-NEXT:    paddb %xmm0, %xmm1
+; X64-NEXT:    pextrb $2, %xmm1, 2(%rdi)
 ; X64-NEXT:    pextrw $0, %xmm1, (%rdi)
 ; X64-NEXT:    retq
 	%a = load %i8vec3, %i8vec3* %ap, align 16
@@ -321,10 +317,11 @@ define void @add31i8(%i8vec31* nocapture sret %ret, %i8vec31* %ap, %i8vec31* %bp
 ; X86-NEXT:    movdqa 16(%edx), %xmm1
 ; X86-NEXT:    paddb (%ecx), %xmm0
 ; X86-NEXT:    paddb 16(%ecx), %xmm1
+; X86-NEXT:    movd %xmm1, 16(%eax)
+; X86-NEXT:    pextrd $1, %xmm1, 20(%eax)
 ; X86-NEXT:    pextrd $2, %xmm1, 24(%eax)
 ; X86-NEXT:    pextrw $6, %xmm1, 28(%eax)
 ; X86-NEXT:    pextrb $14, %xmm1, 30(%eax)
-; X86-NEXT:    movq %xmm1, 16(%eax)
 ; X86-NEXT:    movdqa %xmm0, (%eax)
 ; X86-NEXT:    retl $4
 ;
@@ -353,7 +350,6 @@ define void @add31i8(%i8vec31* nocapture sret %ret, %i8vec31* %ap, %i8vec31* %bp
 define void @rot(%i8vec3pack* nocapture sret %result, %i8vec3pack* %X, %i8vec3pack* %rot) nounwind {
 ; X86-LABEL: rot:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    subl $16, %esp
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
@@ -361,12 +357,11 @@ define void @rot(%i8vec3pack* nocapture sret %result, %i8vec3pack* %X, %i8vec3pa
 ; X86-NEXT:    movw $-24930, (%edx) # imm = 0x9E9E
 ; X86-NEXT:    movb $1, 2(%ecx)
 ; X86-NEXT:    movw $257, (%ecx) # imm = 0x101
-; X86-NEXT:    pmovzxbd {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
-; X86-NEXT:    psrld $1, %xmm0
-; X86-NEXT:    pextrb $8, %xmm0, 2(%eax)
-; X86-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; X86-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-NEXT:    psrlw $1, %xmm0
+; X86-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-NEXT:    pextrb $2, %xmm0, 2(%eax)
 ; X86-NEXT:    pextrw $0, %xmm0, (%eax)
-; X86-NEXT:    addl $16, %esp
 ; X86-NEXT:    retl $4
 ;
 ; X64-LABEL: rot:
@@ -376,10 +371,10 @@ define void @rot(%i8vec3pack* nocapture sret %result, %i8vec3pack* %X, %i8vec3pa
 ; X64-NEXT:    movw $-24930, (%rsi) # imm = 0x9E9E
 ; X64-NEXT:    movb $1, 2(%rdx)
 ; X64-NEXT:    movw $257, (%rdx) # imm = 0x101
-; X64-NEXT:    pmovzxbd {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
-; X64-NEXT:    psrld $1, %xmm0
-; X64-NEXT:    pextrb $8, %xmm0, 2(%rdi)
-; X64-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; X64-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X64-NEXT:    psrlw $1, %xmm0
+; X64-NEXT:    pand {{.*}}(%rip), %xmm0
+; X64-NEXT:    pextrb $2, %xmm0, 2(%rdi)
 ; X64-NEXT:    pextrw $0, %xmm0, (%rdi)
 ; X64-NEXT:    retq
 entry:
diff --git a/llvm/test/CodeGen/X86/widen_mul.ll b/llvm/test/CodeGen/X86/widen_mul.ll
index b3ccb961c..783ca948b 100644
--- a/llvm/test/CodeGen/X86/widen_mul.ll
+++ b/llvm/test/CodeGen/X86/widen_mul.ll
@@ -1,9 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -x86-experimental-vector-widening-legalization -mtriple=x86_64-unknown-unknown | FileCheck %s --check-prefix=SSE --check-prefix=SSE2
-; RUN: llc < %s -x86-experimental-vector-widening-legalization -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=SSE --check-prefix=SSE41
-; RUN: llc < %s -x86-experimental-vector-widening-legalization -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=AVX --check-prefix=AVX2
-; RUN: llc < %s -x86-experimental-vector-widening-legalization -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefix=AVX --check-prefix=AVX512 --check-prefix=AVX512F
-; RUN: llc < %s -x86-experimental-vector-widening-legalization -mtriple=x86_64-unknown-unknown -mattr=+avx512bw | FileCheck %s --check-prefix=AVX --check-prefix=AVX512 --check-prefix=AVX512BW
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s --check-prefix=SSE --check-prefix=SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=SSE --check-prefix=SSE41
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=AVX --check-prefix=AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefix=AVX --check-prefix=AVX512 --check-prefix=AVX512F
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw | FileCheck %s --check-prefix=AVX --check-prefix=AVX512 --check-prefix=AVX512BW
 
 ; Test multiplies of various narrow types.
 
diff --git a/llvm/test/CodeGen/X86/widen_shuffle-1.ll b/llvm/test/CodeGen/X86/widen_shuffle-1.ll
index 044d76ada..acf5ca7cc 100644
--- a/llvm/test/CodeGen/X86/widen_shuffle-1.ll
+++ b/llvm/test/CodeGen/X86/widen_shuffle-1.ll
@@ -89,18 +89,12 @@ entry:
 define <8 x i8> @shuf4(<4 x i8> %a, <4 x i8> %b) nounwind readnone {
 ; X86-LABEL: shuf4:
 ; X86:       # %bb.0:
-; X86-NEXT:    movdqa {{.*#+}} xmm2 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
-; X86-NEXT:    pshufb %xmm2, %xmm1
-; X86-NEXT:    pshufb %xmm2, %xmm0
-; X86-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; X86-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: shuf4:
 ; X64:       # %bb.0:
-; X64-NEXT:    movdqa {{.*#+}} xmm2 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
-; X64-NEXT:    pshufb %xmm2, %xmm1
-; X64-NEXT:    pshufb %xmm2, %xmm0
-; X64-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; X64-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; X64-NEXT:    retq
   %vshuf = shufflevector <4 x i8> %a, <4 x i8> %b, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   ret <8 x i8> %vshuf
diff --git a/llvm/test/CodeGen/X86/win-catchpad-csrs.ll b/llvm/test/CodeGen/X86/win-catchpad-csrs.ll
index 64c7a9747..b8a9c6bce 100644
--- a/llvm/test/CodeGen/X86/win-catchpad-csrs.ll
+++ b/llvm/test/CodeGen/X86/win-catchpad-csrs.ll
@@ -87,17 +87,17 @@ handler1:
 
 ; X64-LABEL: try_catch_catch:
 ; X64: pushq %rbp
-; X64: .seh_pushreg 5
+; X64: .seh_pushreg %rbp
 ; X64: pushq %rsi
-; X64: .seh_pushreg 6
+; X64: .seh_pushreg %rsi
 ; X64: pushq %rdi
-; X64: .seh_pushreg 7
+; X64: .seh_pushreg %rdi
 ; X64: pushq %rbx
-; X64: .seh_pushreg 3
+; X64: .seh_pushreg %rbx
 ; X64: subq $40, %rsp
 ; X64: .seh_stackalloc 40
 ; X64: leaq 32(%rsp), %rbp
-; X64: .seh_setframe 5, 32
+; X64: .seh_setframe %rbp, 32
 ; X64: .seh_endprologue
 ; X64: movq $-2, (%rbp)
 ; X64: callq getint
@@ -117,13 +117,13 @@ handler1:
 ; X64: LBB0_[[catch1bb]]: # %handler1{{$}}
 ; X64: movq %rdx, 16(%rsp)
 ; X64: pushq %rbp
-; X64: .seh_pushreg 5
+; X64: .seh_pushreg %rbp
 ; X64: pushq %rsi
-; X64: .seh_pushreg 6
+; X64: .seh_pushreg %rsi
 ; X64: pushq %rdi
-; X64: .seh_pushreg 7
+; X64: .seh_pushreg %rdi
 ; X64: pushq %rbx
-; X64: .seh_pushreg 3
+; X64: .seh_pushreg %rbx
 ; X64: subq $40, %rsp
 ; X64: .seh_stackalloc 40
 ; X64: leaq 32(%rdx), %rbp
@@ -166,14 +166,14 @@ try.cont:
 
 ; X64-LABEL: try_one_csr:
 ; X64: pushq %rbp
-; X64: .seh_pushreg 5
+; X64: .seh_pushreg %rbp
 ; X64: pushq %rsi
-; X64: .seh_pushreg 6
+; X64: .seh_pushreg %rsi
 ; X64-NOT: pushq
 ; X64: subq $40, %rsp
 ; X64: .seh_stackalloc 40
 ; X64: leaq 32(%rsp), %rbp
-; X64: .seh_setframe 5, 32
+; X64: .seh_setframe %rbp, 32
 ; X64: .seh_endprologue
 ; X64: callq getint
 ; X64: callq getint
@@ -192,9 +192,9 @@ try.cont:
 ; X64: LBB1_[[catch1bb]]: # %handler1{{$}}
 ; X64: movq %rdx, 16(%rsp)
 ; X64: pushq %rbp
-; X64: .seh_pushreg 5
+; X64: .seh_pushreg %rbp
 ; X64: pushq %rsi
-; X64: .seh_pushreg 6
+; X64: .seh_pushreg %rsi
 ; X64: subq $40, %rsp
 ; X64: .seh_stackalloc 40
 ; X64: leaq 32(%rdx), %rbp
@@ -230,12 +230,12 @@ try.cont:
 
 ; X64-LABEL: try_no_csr:
 ; X64: pushq %rbp
-; X64: .seh_pushreg 5
+; X64: .seh_pushreg %rbp
 ; X64-NOT: pushq
 ; X64: subq $48, %rsp
 ; X64: .seh_stackalloc 48
 ; X64: leaq 48(%rsp), %rbp
-; X64: .seh_setframe 5, 48
+; X64: .seh_setframe %rbp, 48
 ; X64: .seh_endprologue
 ; X64: movl $1, %ecx
 ; X64: callq f
@@ -250,7 +250,7 @@ try.cont:
 ; X64: LBB2_[[catch1bb]]: # %handler1{{$}}
 ; X64: movq %rdx, 16(%rsp)
 ; X64: pushq %rbp
-; X64: .seh_pushreg 5
+; X64: .seh_pushreg %rbp
 ; X64: subq $32, %rsp
 ; X64: .seh_stackalloc 32
 ; X64: leaq 48(%rdx), %rbp
diff --git a/llvm/test/CodeGen/X86/win-catchpad.ll b/llvm/test/CodeGen/X86/win-catchpad.ll
index 48866490c..6dddcb970 100644
--- a/llvm/test/CodeGen/X86/win-catchpad.ll
+++ b/llvm/test/CodeGen/X86/win-catchpad.ll
@@ -123,11 +123,11 @@ try.cont:
 ; X64-LABEL: try_catch_catch:
 ; X64: Lfunc_begin0:
 ; X64: pushq %rbp
-; X64: .seh_pushreg 5
+; X64: .seh_pushreg %rbp
 ; X64: subq $[[STCK_ALLOC:.*]], %rsp
 ; X64: .seh_stackalloc [[STCK_ALLOC]]
 ; X64: leaq [[STCK_ALLOC]](%rsp), %rbp
-; X64: .seh_setframe 5, [[STCK_ALLOC]]
+; X64: .seh_setframe %rbp, [[STCK_ALLOC]]
 ; X64: .seh_endprologue
 ; X64: movq $-2, -16(%rbp)
 ; X64: .Ltmp0
@@ -144,7 +144,7 @@ try.cont:
 ; X64: LBB0_[[catch1bb]]: # %handler1{{$}}
 ; X64: movq %rdx, 16(%rsp)
 ; X64: pushq %rbp
-; X64: .seh_pushreg 5
+; X64: .seh_pushreg %rbp
 ; X64: subq $32, %rsp
 ; X64: .seh_stackalloc 32
 ; X64: leaq [[STCK_ALLOC]](%rdx), %rbp
@@ -161,7 +161,7 @@ try.cont:
 ; X64: LBB0_[[catch2bb]]: # %handler2{{$}}
 ; X64: movq %rdx, 16(%rsp)
 ; X64: pushq %rbp
-; X64: .seh_pushreg 5
+; X64: .seh_pushreg %rbp
 ; X64: subq $32, %rsp
 ; X64: .seh_stackalloc 32
 ; X64: leaq [[STCK_ALLOC]](%rdx), %rbp
@@ -278,11 +278,11 @@ try.cont:
 ; X64-LABEL: branch_to_normal_dest:
 ; X64: # %entry
 ; X64: pushq %rbp
-; X64: .seh_pushreg 5
+; X64: .seh_pushreg %rbp
 ; X64: subq $48, %rsp
 ; X64: .seh_stackalloc 48
 ; X64: leaq 48(%rsp), %rbp
-; X64: .seh_setframe 5, 48
+; X64: .seh_setframe %rbp, 48
 ; X64: .seh_endprologue
 ; X64: .Ltmp[[before_call:[0-9]+]]:
 ; X64: callq f
@@ -297,7 +297,7 @@ try.cont:
 ; X64: LBB1_[[catchbb]]: # %catch{{$}}
 ; X64: movq %rdx, 16(%rsp)
 ; X64: pushq %rbp
-; X64: .seh_pushreg 5
+; X64: .seh_pushreg %rbp
 ; X64: subq $32, %rsp
 ; X64: .seh_stackalloc 32
 ; X64: leaq 48(%rdx), %rbp
diff --git a/llvm/test/CodeGen/X86/win-cleanuppad.ll b/llvm/test/CodeGen/X86/win-cleanuppad.ll
index 70c1e284b..273b00f39 100644
--- a/llvm/test/CodeGen/X86/win-cleanuppad.ll
+++ b/llvm/test/CodeGen/X86/win-cleanuppad.ll
@@ -194,6 +194,6 @@ cleanup.outer:                                      ; preds = %invoke.cont.1, %c
 ; X64-NEXT: .long   .Ltmp7@IMGREL+1
 ; X64-NEXT: .long   -1
 
-attributes #0 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind }
diff --git a/llvm/test/CodeGen/X86/win-funclet-cfi.ll b/llvm/test/CodeGen/X86/win-funclet-cfi.ll
index 2151cdc7b..5e78e5766 100644
--- a/llvm/test/CodeGen/X86/win-funclet-cfi.ll
+++ b/llvm/test/CodeGen/X86/win-funclet-cfi.ll
@@ -42,7 +42,7 @@ declare i32 @__CxxFrameHandler3(...)
 ; Emit CFI for pushing RBP.
 ; CHECK: movq    %rdx, 16(%rsp)
 ; CHECK: pushq   %rbp
-; CHECK: .seh_pushreg 5
+; CHECK: .seh_pushreg %rbp
 
 ; Emit CFI for allocating from the stack pointer.
 ; CHECK: subq    $32, %rsp
@@ -72,7 +72,7 @@ declare i32 @__CxxFrameHandler3(...)
 ; Emit CFI for pushing RBP.
 ; CHECK: movq    %rdx, 16(%rsp)
 ; CHECK: pushq   %rbp
-; CHECK: .seh_pushreg 5
+; CHECK: .seh_pushreg %rbp
 
 ; Emit CFI for allocating from the stack pointer.
 ; CHECK: subq    $32, %rsp
diff --git a/llvm/test/CodeGen/X86/win32-eh-available-externally.ll b/llvm/test/CodeGen/X86/win32-eh-available-externally.ll
index 49da191de..47eee30bd 100644
--- a/llvm/test/CodeGen/X86/win32-eh-available-externally.ll
+++ b/llvm/test/CodeGen/X86/win32-eh-available-externally.ll
@@ -2,7 +2,7 @@
 ; RUN: llc < %s | FileCheck %s --check-prefix=ASM
 
 ; IR-NOT: define.*__ehhandler
-; IR: define available_externally void @foo(void ()*)
+; IR: define available_externally void @foo(void ()* %0)
 ; IR-NOT: define.*__ehhandler
 
 ; No code should be emitted.
diff --git a/llvm/test/CodeGen/X86/win32-seh-catchpad.ll b/llvm/test/CodeGen/X86/win32-seh-catchpad.ll
index f60195374..c3bc2cad9 100644
--- a/llvm/test/CodeGen/X86/win32-seh-catchpad.ll
+++ b/llvm/test/CodeGen/X86/win32-seh-catchpad.ll
@@ -225,7 +225,7 @@ declare i32 @_except_handler3(...)
 ; Function Attrs: nounwind
 declare void @llvm.localescape(...) #2
 
-attributes #0 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 attributes #2 = { nounwind }
 attributes #3 = { noinline }
diff --git a/llvm/test/CodeGen/X86/win32-seh-nested-finally.ll b/llvm/test/CodeGen/X86/win32-seh-nested-finally.ll
index b732815b8..f8d0ca000 100644
--- a/llvm/test/CodeGen/X86/win32-seh-nested-finally.ll
+++ b/llvm/test/CodeGen/X86/win32-seh-nested-finally.ll
@@ -34,8 +34,8 @@ declare void @f(i32) #0
 
 declare i32 @_except_handler3(...)
 
-attributes #0 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { noinline nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { noinline nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind readnone }
 attributes #3 = { noinline }
 
diff --git a/llvm/test/CodeGen/X86/win64-eh-empty-block.ll b/llvm/test/CodeGen/X86/win64-eh-empty-block.ll
new file mode 100644
index 000000000..c93c53b6b
--- /dev/null
+++ b/llvm/test/CodeGen/X86/win64-eh-empty-block.ll
@@ -0,0 +1,107 @@
+; RUN: llc -mtriple=x86_64-windows-gnu %s -o - | FileCheck %s
+
+; Based on this C++ code:
+; struct as {
+;     as() { at = static_cast<int *>(operator new(sizeof(int))); }
+;     ~as() { operator delete(at); }
+;     int *at;
+; };
+; void am(int) {
+;     static as au;
+;     as av;
+;     throw 0;
+; }
+
+; optnone was added to ensure that branch folding and block layout are not
+; disturbed. The key thing about this test is that it ends in an empty
+; unreachable block, which forces us to scan back across blocks.
+
+; CHECK: _Z2ami:
+; CHECK: callq   __cxa_throw
+; CHECK: # %eh.resume
+; CHECK: callq _Unwind_Resume
+; CHECK-NEXT: int3
+; CHECK-NEXT: # %unreachable
+; CHECK-NEXT: .Lfunc_end0:
+
+%struct.as = type { i32* }
+
+@_ZZ2amiE2au = internal unnamed_addr global %struct.as zeroinitializer, align 8
+@_ZGVZ2amiE2au = internal global i64 0, align 8
+@_ZTIi = external constant i8*
+
+define dso_local void @_Z2ami(i32 %0) noinline optnone personality i8* bitcast (i32 (...)* @__gxx_personality_seh0 to i8*) {
+entry:
+  %1 = load atomic i8, i8* bitcast (i64* @_ZGVZ2amiE2au to i8*) acquire, align 8
+  %guard.uninitialized = icmp eq i8 %1, 0
+  br i1 %guard.uninitialized, label %init.check, label %init.end
+
+init.check:                                       ; preds = %entry
+  %2 = tail call i32 @__cxa_guard_acquire(i64* nonnull @_ZGVZ2amiE2au)
+  %tobool = icmp eq i32 %2, 0
+  br i1 %tobool, label %init.end, label %init
+
+init:                                             ; preds = %init.check
+  %call.i3 = invoke i8* @_Znwy(i64 4)
+          to label %invoke.cont unwind label %lpad
+
+invoke.cont:                                      ; preds = %init
+  store i8* %call.i3, i8** bitcast (%struct.as* @_ZZ2amiE2au to i8**), align 8
+  %3 = tail call i32 @atexit(void ()* nonnull @__dtor__ZZ2amiE2au)
+  tail call void @__cxa_guard_release(i64* nonnull @_ZGVZ2amiE2au)
+  br label %init.end
+
+init.end:                                         ; preds = %init.check, %invoke.cont, %entry
+  %call.i = tail call i8* @_Znwy(i64 4)
+  %exception = tail call i8* @__cxa_allocate_exception(i64 4)
+  %4 = bitcast i8* %exception to i32*
+  store i32 0, i32* %4, align 16
+  invoke void @__cxa_throw(i8* %exception, i8* bitcast (i8** @_ZTIi to i8*), i8* null)
+          to label %unreachable unwind label %lpad1
+
+lpad:                                             ; preds = %init
+  %5 = landingpad { i8*, i32 }
+          cleanup
+  %6 = extractvalue { i8*, i32 } %5, 0
+  %7 = extractvalue { i8*, i32 } %5, 1
+  tail call void @__cxa_guard_abort(i64* nonnull @_ZGVZ2amiE2au)
+  br label %eh.resume
+
+lpad1:                                            ; preds = %init.end
+  %8 = landingpad { i8*, i32 }
+          cleanup
+  %9 = extractvalue { i8*, i32 } %8, 0
+  %10 = extractvalue { i8*, i32 } %8, 1
+  tail call void @_ZdlPv(i8* %call.i)
+  br label %eh.resume
+
+eh.resume:                                        ; preds = %lpad1, %lpad
+  %exn.slot.0 = phi i8* [ %9, %lpad1 ], [ %6, %lpad ]
+  %ehselector.slot.0 = phi i32 [ %10, %lpad1 ], [ %7, %lpad ]
+  %lpad.val = insertvalue { i8*, i32 } undef, i8* %exn.slot.0, 0
+  %lpad.val2 = insertvalue { i8*, i32 } %lpad.val, i32 %ehselector.slot.0, 1
+  resume { i8*, i32 } %lpad.val2
+
+unreachable:                                      ; preds = %init.end
+  unreachable
+}
+
+declare dso_local i32 @__cxa_guard_acquire(i64*)
+
+declare dso_local i32 @__gxx_personality_seh0(...)
+
+declare dso_local void @__dtor__ZZ2amiE2au()
+
+declare dso_local i32 @atexit(void ()*)
+
+declare dso_local void @__cxa_guard_abort(i64*)
+
+declare dso_local void @__cxa_guard_release(i64*)
+
+declare dso_local i8* @__cxa_allocate_exception(i64)
+
+declare dso_local void @__cxa_throw(i8*, i8*, i8*)
+
+declare dso_local noalias i8* @_Znwy(i64)
+
+declare dso_local void @_ZdlPv(i8*)
diff --git a/llvm/test/CodeGen/X86/win64-funclet-savexmm.ll b/llvm/test/CodeGen/X86/win64-funclet-savexmm.ll
new file mode 100644
index 000000000..9aca63e70
--- /dev/null
+++ b/llvm/test/CodeGen/X86/win64-funclet-savexmm.ll
@@ -0,0 +1,115 @@
+; RUN: llc -mtriple=x86_64-pc-windows-msvc -mattr=+avx < %s | FileCheck %s
+
+; void bar(int a, int b, int c, int d, int e);
+; void baz(int x);
+; 
+; void foo(int a, int b, int c, int d, int e)
+; {
+;   __asm("nop" ::: "bx", "cx", "xmm5", "xmm6", "ymm7");
+;   try {
+;     bar(a, b, c, d, e);
+;   }
+;   catch (...) {
+;     baz(a);
+;     if (a)
+;       __asm("nop" ::: "xmm8");
+;   }
+; }
+
+%rtti.TypeDescriptor2 = type { i8**, i8*, [3 x i8] }
+
+$"??_R0H@8" = comdat any
+
+@"??_7type_info@@6B@" = external constant i8*
+@"??_R0H@8" = linkonce_odr global %rtti.TypeDescriptor2 { i8** @"??_7type_info@@6B@", i8* null, [3 x i8] c".H\00" }, comdat
+
+declare dso_local i32 @__CxxFrameHandler3(...)
+declare dso_local void @"?bar@@YAXHHHHH@Z"(i32, i32, i32, i32, i32)
+declare dso_local void @"?baz@@YAXH@Z"(i32)
+
+define dso_local void @"?foo@@YAXHHHHH@Z"(i32 %a, i32 %b, i32 %c, i32 %d, i32 %e) personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
+entry:
+  %e.addr = alloca i32, align 4
+  %d.addr = alloca i32, align 4
+  %c.addr = alloca i32, align 4
+  %b.addr = alloca i32, align 4
+  %a.addr = alloca i32, align 4
+  store i32 %e, i32* %e.addr, align 4
+  store i32 %d, i32* %d.addr, align 4
+  store i32 %c, i32* %c.addr, align 4
+  store i32 %b, i32* %b.addr, align 4
+  store i32 %a, i32* %a.addr, align 4
+  call void asm sideeffect "nop", "~{bx},~{cx},~{xmm5},~{xmm6},~{ymm7}"()
+  %0 = load i32, i32* %e.addr, align 4
+  %1 = load i32, i32* %d.addr, align 4
+  %2 = load i32, i32* %c.addr, align 4
+  %3 = load i32, i32* %b.addr, align 4
+  %4 = load i32, i32* %a.addr, align 4
+  invoke void @"?bar@@YAXHHHHH@Z"(i32 %4, i32 %3, i32 %2, i32 %1, i32 %0)
+          to label %invoke.cont unwind label %catch.dispatch
+
+catch.dispatch:                                   ; preds = %entry
+  %5 = catchswitch within none [label %catch] unwind to caller
+
+catch:                                            ; preds = %catch.dispatch
+  %6 = catchpad within %5 [i8* null, i32 64, i8* null]
+  %7 = load i32, i32* %a.addr, align 4
+  call void @"?baz@@YAXH@Z"(i32 %7) [ "funclet"(token %6) ]
+  %8 = load i32, i32* %a.addr, align 4
+  %tobool = icmp ne i32 %8, 0
+  br i1 %tobool, label %if.then, label %if.end
+
+if.then:                                          ; preds = %catch
+  call void asm sideeffect "nop", "~{xmm8}"() [ "funclet"(token %6) ]
+  br label %if.end
+
+invoke.cont:                                      ; preds = %entry
+  br label %try.cont
+
+if.end:                                           ; preds = %if.then, %catch
+  catchret from %6 to label %catchret.dest
+
+catchret.dest:                                    ; preds = %if.end
+  br label %try.cont
+
+try.cont:                                         ; preds = %catchret.dest, %invoke.cont
+  ret void
+}
+
+; CHECK: # %catch
+; CHECK: movq    %rdx, 16(%rsp)
+; CHECK: pushq   %rbp
+; CHECK: .seh_pushreg %rbp
+; CHECK: pushq   %rbx
+; CHECK: .seh_pushreg %rbx
+; CHECK: subq    $88, %rsp
+; CHECK: .seh_stackalloc 88
+; CHECK: leaq    112(%rdx), %rbp
+; CHECK: vmovaps %xmm8, 32(%rsp)
+; CHECK: .seh_savexmm %xmm8, 32
+; CHECK: vmovaps %xmm7, 48(%rsp)
+; CHECK: .seh_savexmm %xmm7, 48
+; CHECK: vmovaps %xmm6, 64(%rsp)
+; CHECK: .seh_savexmm %xmm6, 64
+; CHECK: .seh_endprologue
+; CHECK: movl   -{{[0-9]+}}(%rbp), %ecx
+; CHECK: vmovaps 64(%rsp), %xmm6
+; CHECK: vmovaps 48(%rsp), %xmm7
+; CHECK: vmovaps 32(%rsp), %xmm8
+; CHECK: leaq    .LBB0_1(%rip), %rax
+; CHECK: addq    $88, %rsp
+; CHECK: popq    %rbx
+; CHECK: popq    %rbp
+; CHECK: retq # CATCHRET
+
+; CHECK-LABEL: "$handlerMap$0$?foo@@YAXHHHHH@Z":
+; CHECK-NEXT: .long   64                      # Adjectives
+; CHECK-NEXT: .long   0                       # Type
+; CHECK-NEXT: .long   0                       # CatchObjOffset
+; CHECK-NEXT: .long   "?catch$2@?0??foo@@YAXHHHHH@Z@4HA"@IMGREL # Handler
+; Sum of:
+;   16 RDX store offset
+;   16 two pushes
+;   72 stack alloc
+; CHECK-NEXT: .long   120                     # ParentFrameOffset
+
diff --git a/llvm/test/CodeGen/X86/win64-stackprobe-overflow.ll b/llvm/test/CodeGen/X86/win64-stackprobe-overflow.ll
new file mode 100644
index 000000000..9555ce032
--- /dev/null
+++ b/llvm/test/CodeGen/X86/win64-stackprobe-overflow.ll
@@ -0,0 +1,14 @@
+; RUN: llc < %s -mtriple=x86_64-windows-gnu | FileCheck %s
+
+define void @foo() unnamed_addr #0 {
+start:
+  %b = alloca i64, align 8
+  %c = alloca [4294967295 x i8], align 1
+  ret void
+}
+
+attributes #0 = { nonlazybind uwtable "probe-stack"="probe_stack" "target-cpu"="x86-64" }
+
+; CHECK-LABEL: foo:
+; CHECK: movabsq $4294967304, %rax
+; CHECK-NEXT: callq probe_stack
diff --git a/llvm/test/CodeGen/X86/win64_call_epi.ll b/llvm/test/CodeGen/X86/win64_call_epi.ll
index 096cbe41c..3e8f57e65 100644
--- a/llvm/test/CodeGen/X86/win64_call_epi.ll
+++ b/llvm/test/CodeGen/X86/win64_call_epi.ll
@@ -24,10 +24,9 @@ catch:
 ; WIN64: nop
 ; WIN64: addq ${{[0-9]+}}, %rsp
 ; WIN64: retq
-; Check for 'ud2' after noreturn call
+; Check for 'int3' after noreturn call.
 ; WIN64: callq _Unwind_Resume
-; WIN64-NEXT: ud2
-; WIN64: .seh_endproc
+; WIN64-NEXT: int3
 
 
 ; Check it still works when blocks are reordered.
diff --git a/llvm/test/CodeGen/X86/win64_eh.ll b/llvm/test/CodeGen/X86/win64_eh.ll
index e51ef0516..caadea4fe 100644
--- a/llvm/test/CodeGen/X86/win64_eh.ll
+++ b/llvm/test/CodeGen/X86/win64_eh.ll
@@ -125,11 +125,11 @@ endtryfinally:
 ; WIN64-LABEL: foo4:
 ; WIN64: .seh_proc foo4
 ; WIN64: .seh_handler _d_eh_personality, @unwind, @except
-; NORM:  subq $56, %rsp
-; ATOM:  leaq -56(%rsp), %rsp
-; WIN64: .seh_stackalloc 56
+; NORM:  subq $40, %rsp
+; ATOM:  leaq -40(%rsp), %rsp
+; WIN64: .seh_stackalloc 40
 ; WIN64: .seh_endprologue
-; WIN64: addq $56, %rsp
+; WIN64: addq $40, %rsp
 ; WIN64: ret
 ; WIN64: .seh_handlerdata
 ; WIN64: .seh_endproc
@@ -145,20 +145,20 @@ entry:
 ; WIN64-LABEL: foo5:
 ; WIN64: .seh_proc foo5
 ; WIN64: pushq %rbp
-; WIN64: .seh_pushreg 5
+; WIN64: .seh_pushreg %rbp
 ; WIN64: pushq %rdi
-; WIN64: .seh_pushreg 7
+; WIN64: .seh_pushreg %rdi
 ; WIN64: pushq %rbx
-; WIN64: .seh_pushreg 3
+; WIN64: .seh_pushreg %rbx
 ; NORM:  subq  $96, %rsp
 ; ATOM:  leaq -96(%rsp), %rsp
 ; WIN64: .seh_stackalloc 96
 ; WIN64: leaq  96(%rsp), %rbp
-; WIN64: .seh_setframe 5, 96
+; WIN64: .seh_setframe %rbp, 96
 ; WIN64: movaps  %xmm7, -16(%rbp)        # 16-byte Spill
-; WIN64: .seh_savexmm 7, 80
+; WIN64: .seh_savexmm %xmm7, 80
 ; WIN64: movaps  %xmm6, -32(%rbp)        # 16-byte Spill
-; WIN64: .seh_savexmm 6, 64
+; WIN64: .seh_savexmm %xmm6, 64
 ; WIN64: .seh_endprologue
 ; WIN64: andq  $-64, %rsp
 ; WIN64: movaps  -32(%rbp), %xmm6        # 16-byte Reload
diff --git a/llvm/test/CodeGen/X86/win64_frame.ll b/llvm/test/CodeGen/X86/win64_frame.ll
index eae02dafa..9158b19b2 100644
--- a/llvm/test/CodeGen/X86/win64_frame.ll
+++ b/llvm/test/CodeGen/X86/win64_frame.ll
@@ -2,13 +2,13 @@
 ; RUN: llc < %s -mtriple=x86_64-pc-win32              | FileCheck %s --check-prefix=ALL --check-prefix=PUSHF
 ; RUN: llc < %s -mtriple=x86_64-pc-win32 -mattr=+sahf | FileCheck %s --check-prefix=ALL --check-prefix=SAHF
 
-define i32 @f1(i32 %p1, i32 %p2, i32 %p3, i32 %p4, i32 %p5) "no-frame-pointer-elim"="true" {
+define i32 @f1(i32 %p1, i32 %p2, i32 %p3, i32 %p4, i32 %p5) "frame-pointer"="all" {
 ; ALL-LABEL: f1:
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    pushq %rbp
-; ALL-NEXT:    .seh_pushreg 5
+; ALL-NEXT:    .seh_pushreg %rbp
 ; ALL-NEXT:    movq %rsp, %rbp
-; ALL-NEXT:    .seh_setframe 5, 0
+; ALL-NEXT:    .seh_setframe %rbp, 0
 ; ALL-NEXT:    .seh_endprologue
 ; ALL-NEXT:    movl 48(%rbp), %eax
 ; ALL-NEXT:    popq %rbp
@@ -19,15 +19,15 @@ define i32 @f1(i32 %p1, i32 %p2, i32 %p3, i32 %p4, i32 %p5) "no-frame-pointer-el
   ret i32 %p5
 }
 
-define void @f2(i32 %p, ...) "no-frame-pointer-elim"="true" {
+define void @f2(i32 %p, ...) "frame-pointer"="all" {
 ; ALL-LABEL: f2:
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    pushq %rbp
-; ALL-NEXT:    .seh_pushreg 5
+; ALL-NEXT:    .seh_pushreg %rbp
 ; ALL-NEXT:    pushq %rax
 ; ALL-NEXT:    .seh_stackalloc 8
 ; ALL-NEXT:    movq %rsp, %rbp
-; ALL-NEXT:    .seh_setframe 5, 0
+; ALL-NEXT:    .seh_setframe %rbp, 0
 ; ALL-NEXT:    .seh_endprologue
 ; ALL-NEXT:    movq %rdx, 32(%rbp)
 ; ALL-NEXT:    movq %r8, 40(%rbp)
@@ -45,13 +45,13 @@ define void @f2(i32 %p, ...) "no-frame-pointer-elim"="true" {
   ret void
 }
 
-define i8* @f3() "no-frame-pointer-elim"="true" {
+define i8* @f3() "frame-pointer"="all" {
 ; ALL-LABEL: f3:
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    pushq %rbp
-; ALL-NEXT:    .seh_pushreg 5
+; ALL-NEXT:    .seh_pushreg %rbp
 ; ALL-NEXT:    movq %rsp, %rbp
-; ALL-NEXT:    .seh_setframe 5, 0
+; ALL-NEXT:    .seh_setframe %rbp, 0
 ; ALL-NEXT:    .seh_endprologue
 ; ALL-NEXT:    movq 8(%rbp), %rax
 ; ALL-NEXT:    popq %rbp
@@ -63,15 +63,15 @@ define i8* @f3() "no-frame-pointer-elim"="true" {
   ret i8* %ra
 }
 
-define i8* @f4() "no-frame-pointer-elim"="true" {
+define i8* @f4() "frame-pointer"="all" {
 ; ALL-LABEL: f4:
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    pushq %rbp
-; ALL-NEXT:    .seh_pushreg 5
+; ALL-NEXT:    .seh_pushreg %rbp
 ; ALL-NEXT:    subq $304, %rsp # imm = 0x130
 ; ALL-NEXT:    .seh_stackalloc 304
 ; ALL-NEXT:    leaq {{[0-9]+}}(%rsp), %rbp
-; ALL-NEXT:    .seh_setframe 5, 128
+; ALL-NEXT:    .seh_setframe %rbp, 128
 ; ALL-NEXT:    .seh_endprologue
 ; ALL-NEXT:    movq 184(%rbp), %rax
 ; ALL-NEXT:    addq $304, %rsp # imm = 0x130
@@ -87,15 +87,15 @@ define i8* @f4() "no-frame-pointer-elim"="true" {
 
 declare void @external(i8*)
 
-define void @f5() "no-frame-pointer-elim"="true" {
+define void @f5() "frame-pointer"="all" {
 ; ALL-LABEL: f5:
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    pushq %rbp
-; ALL-NEXT:    .seh_pushreg 5
+; ALL-NEXT:    .seh_pushreg %rbp
 ; ALL-NEXT:    subq $336, %rsp # imm = 0x150
 ; ALL-NEXT:    .seh_stackalloc 336
 ; ALL-NEXT:    leaq {{[0-9]+}}(%rsp), %rbp
-; ALL-NEXT:    .seh_setframe 5, 128
+; ALL-NEXT:    .seh_setframe %rbp, 128
 ; ALL-NEXT:    .seh_endprologue
 ; ALL-NEXT:    leaq -92(%rbp), %rcx
 ; ALL-NEXT:    callq external
@@ -112,15 +112,15 @@ define void @f5() "no-frame-pointer-elim"="true" {
   ret void
 }
 
-define void @f6(i32 %p, ...) "no-frame-pointer-elim"="true" {
+define void @f6(i32 %p, ...) "frame-pointer"="all" {
 ; ALL-LABEL: f6:
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    pushq %rbp
-; ALL-NEXT:    .seh_pushreg 5
+; ALL-NEXT:    .seh_pushreg %rbp
 ; ALL-NEXT:    subq $336, %rsp # imm = 0x150
 ; ALL-NEXT:    .seh_stackalloc 336
 ; ALL-NEXT:    leaq {{[0-9]+}}(%rsp), %rbp
-; ALL-NEXT:    .seh_setframe 5, 128
+; ALL-NEXT:    .seh_setframe %rbp, 128
 ; ALL-NEXT:    .seh_endprologue
 ; ALL-NEXT:    leaq -92(%rbp), %rcx
 ; ALL-NEXT:    callq external
@@ -137,15 +137,15 @@ define void @f6(i32 %p, ...) "no-frame-pointer-elim"="true" {
   ret void
 }
 
-define i32 @f7(i32 %a, i32 %b, i32 %c, i32 %d, i32 %e) "no-frame-pointer-elim"="true" {
+define i32 @f7(i32 %a, i32 %b, i32 %c, i32 %d, i32 %e) "frame-pointer"="all" {
 ; ALL-LABEL: f7:
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    pushq %rbp
-; ALL-NEXT:    .seh_pushreg 5
+; ALL-NEXT:    .seh_pushreg %rbp
 ; ALL-NEXT:    subq $304, %rsp # imm = 0x130
 ; ALL-NEXT:    .seh_stackalloc 304
 ; ALL-NEXT:    leaq {{[0-9]+}}(%rsp), %rbp
-; ALL-NEXT:    .seh_setframe 5, 128
+; ALL-NEXT:    .seh_setframe %rbp, 128
 ; ALL-NEXT:    .seh_endprologue
 ; ALL-NEXT:    andq $-64, %rsp
 ; ALL-NEXT:    movl 224(%rbp), %eax
@@ -159,19 +159,19 @@ define i32 @f7(i32 %a, i32 %b, i32 %c, i32 %d, i32 %e) "no-frame-pointer-elim"="
   ret i32 %e
 }
 
-define i32 @f8(i32 %a, i32 %b, i32 %c, i32 %d, i32 %e) "no-frame-pointer-elim"="true" {
+define i32 @f8(i32 %a, i32 %b, i32 %c, i32 %d, i32 %e) "frame-pointer"="all" {
 ; ALL-LABEL: f8:
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    pushq %rbp
-; ALL-NEXT:    .seh_pushreg 5
+; ALL-NEXT:    .seh_pushreg %rbp
 ; ALL-NEXT:    pushq %rsi
-; ALL-NEXT:    .seh_pushreg 6
+; ALL-NEXT:    .seh_pushreg %rsi
 ; ALL-NEXT:    pushq %rbx
-; ALL-NEXT:    .seh_pushreg 3
+; ALL-NEXT:    .seh_pushreg %rbx
 ; ALL-NEXT:    subq $352, %rsp # imm = 0x160
 ; ALL-NEXT:    .seh_stackalloc 352
 ; ALL-NEXT:    leaq {{[0-9]+}}(%rsp), %rbp
-; ALL-NEXT:    .seh_setframe 5, 128
+; ALL-NEXT:    .seh_setframe %rbp, 128
 ; ALL-NEXT:    .seh_endprologue
 ; ALL-NEXT:    andq $-64, %rsp
 ; ALL-NEXT:    movq %rsp, %rbx
@@ -205,9 +205,9 @@ define i64 @f9() {
 ; ALL-LABEL: f9:
 ; ALL:       # %bb.0: # %entry
 ; ALL-NEXT:    pushq %rbp
-; ALL-NEXT:    .seh_pushreg 5
+; ALL-NEXT:    .seh_pushreg %rbp
 ; ALL-NEXT:    movq %rsp, %rbp
-; ALL-NEXT:    .seh_setframe 5, 0
+; ALL-NEXT:    .seh_setframe %rbp, 0
 ; ALL-NEXT:    .seh_endprologue
 ; ALL-NEXT:    pushfq
 ; ALL-NEXT:    popq %rax
@@ -227,9 +227,9 @@ define i64 @f10(i64* %foo, i64 %bar, i64 %baz) {
 ; ALL-LABEL: f10:
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    pushq %rsi
-; ALL-NEXT:    .seh_pushreg 6
+; ALL-NEXT:    .seh_pushreg %rsi
 ; ALL-NEXT:    pushq %rbx
-; ALL-NEXT:    .seh_pushreg 3
+; ALL-NEXT:    .seh_pushreg %rbx
 ; ALL-NEXT:    subq $40, %rsp
 ; ALL-NEXT:    .seh_stackalloc 40
 ; ALL-NEXT:    .seh_endprologue
@@ -255,13 +255,13 @@ define i64 @f10(i64* %foo, i64 %bar, i64 %baz) {
   ret i64 %sel
 }
 
-define i8* @f11() "no-frame-pointer-elim"="true" {
+define i8* @f11() "frame-pointer"="all" {
 ; ALL-LABEL: f11:
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    pushq %rbp
-; ALL-NEXT:    .seh_pushreg 5
+; ALL-NEXT:    .seh_pushreg %rbp
 ; ALL-NEXT:    movq %rsp, %rbp
-; ALL-NEXT:    .seh_setframe 5, 0
+; ALL-NEXT:    .seh_setframe %rbp, 0
 ; ALL-NEXT:    .seh_endprologue
 ; ALL-NEXT:    leaq 8(%rbp), %rax
 ; ALL-NEXT:    popq %rbp
diff --git a/llvm/test/CodeGen/X86/win_coreclr_chkstk.ll b/llvm/test/CodeGen/X86/win_coreclr_chkstk.ll
index 86aa295b7..0017b993a 100644
--- a/llvm/test/CodeGen/X86/win_coreclr_chkstk.ll
+++ b/llvm/test/CodeGen/X86/win_coreclr_chkstk.ll
@@ -39,7 +39,7 @@ entry:
 
 ; Prolog stack allocation >= 4096 bytes will require the probe sequence
 ; Case with frame pointer
-define i32 @main4k_frame() nounwind "no-frame-pointer-elim"="true" {
+define i32 @main4k_frame() nounwind "frame-pointer"="all" {
 entry:
 ; WIN_X64-LABEL:main4k_frame:
 ; WIN_X64:      movq    %gs:16, %rcx
diff --git a/llvm/test/CodeGen/X86/wineh-coreclr.ll b/llvm/test/CodeGen/X86/wineh-coreclr.ll
index af6bef57e..fd2569e4b 100644
--- a/llvm/test/CodeGen/X86/wineh-coreclr.ll
+++ b/llvm/test/CodeGen/X86/wineh-coreclr.ll
@@ -320,6 +320,7 @@ unreachable:
 ; CHECK: [[test2_before_f2:.+]]:
 ; CHECK-NEXT: movl $2, %ecx
 ; CHECK-NEXT: callq f
+; CHECK-NEXT: int3
 ; CHECK-NEXT: [[test2_after_f2:.+]]:
 ; CHECK: [[test2_end:.*func_end.*]]:
 
@@ -511,6 +512,7 @@ unreachable:
 ; CHECK: [[test3_before_f4:.+]]:
 ; CHECK-NEXT: movl $4, %ecx
 ; CHECK-NEXT: callq f
+; CHECK-NEXT: int3
 ; CHECK-NEXT: [[test3_after_f4:.+]]:
 ; CHECK: .seh_proc [[test3_fault2:[^ ]+]]
 ; CHECK: # %fault2
@@ -518,6 +520,7 @@ unreachable:
 ; CHECK: [[test3_before_f3:.+]]:
 ; CHECK-NEXT: movl $3, %ecx
 ; CHECK-NEXT: callq f
+; CHECK-NEXT: int3
 ; CHECK-NEXT: [[test3_after_f3:.+]]:
 ; CHECK: .seh_proc [[test3_fault1:[^ ]+]]
 ; CHECK: # %fault1
@@ -525,6 +528,7 @@ unreachable:
 ; CHECK: [[test3_before_f2:.+]]:
 ; CHECK-NEXT: movl $2, %ecx
 ; CHECK-NEXT: callq f
+; CHECK-NEXT: int3
 ; CHECK-NEXT: [[test3_after_f2:.+]]:
 ; CHECK: [[test3_end:.*func_end.*]]:
 }
diff --git a/llvm/test/CodeGen/X86/x86-32-intrcc.ll b/llvm/test/CodeGen/X86/x86-32-intrcc.ll
index 2644da300..af2d96577 100644
--- a/llvm/test/CodeGen/X86/x86-32-intrcc.ll
+++ b/llvm/test/CodeGen/X86/x86-32-intrcc.ll
@@ -158,4 +158,4 @@ entry:
   ret void
 }
 
-attributes #0 = { nounwind "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" }
+attributes #0 = { nounwind "frame-pointer"="all" }
diff --git a/llvm/test/CodeGen/X86/x86-64-arg.ll b/llvm/test/CodeGen/X86/x86-64-arg.ll
index 9a959e839..f23b7f864 100644
--- a/llvm/test/CodeGen/X86/x86-64-arg.ll
+++ b/llvm/test/CodeGen/X86/x86-64-arg.ll
@@ -1,4 +1,6 @@
-; RUN: llc < %s | grep "movl	%edi, %eax"
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s | FileCheck %s
+
 ; The input value is already sign extended, don't re-extend it.
 ; This testcase corresponds to:
 ;   int test(short X) { return (int)X; }
@@ -8,6 +10,10 @@ target triple = "x86_64-apple-darwin8"
 
 
 define i32 @test(i16 signext  %X) {
+; CHECK-LABEL: test:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    retq
 entry:
         %tmp12 = sext i16 %X to i32             ; <i32> [#uses=1]
         ret i32 %tmp12
diff --git a/llvm/test/CodeGen/X86/x86-64-baseptr.ll b/llvm/test/CodeGen/X86/x86-64-baseptr.ll
index 84bb03c22..590a73964 100644
--- a/llvm/test/CodeGen/X86/x86-64-baseptr.ll
+++ b/llvm/test/CodeGen/X86/x86-64-baseptr.ll
@@ -64,4 +64,4 @@ entry:
   ret void
 }
 
-attributes #0 = { nounwind "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf"}
+attributes #0 = { nounwind "frame-pointer"="all"}
diff --git a/llvm/test/CodeGen/X86/x86-64-bittest-logic.ll b/llvm/test/CodeGen/X86/x86-64-bittest-logic.ll
index 253a5d124..ceb08cead 100644
--- a/llvm/test/CodeGen/X86/x86-64-bittest-logic.ll
+++ b/llvm/test/CodeGen/X86/x86-64-bittest-logic.ll
@@ -240,3 +240,140 @@ define i64 @xor4_optsize(i64 %x) optsize {
   %a = xor i64 %x, 9223372036854775808 ; toggle bit 63
   ret i64 %a
 }
+
+define i64 @and1_pgso(i64 %x) !prof !14 {
+; CHECK-LABEL: and1_pgso:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    btrq $31, %rax
+; CHECK-NEXT:    retq
+  %a = and i64 %x, 18446744071562067967 ; clear bit 31
+  ret i64 %a
+}
+
+define i64 @and2_pgso(i64 %x) !prof !14 {
+; CHECK-LABEL: and2_pgso:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    btrq $32, %rax
+; CHECK-NEXT:    retq
+  %a = and i64 %x, 18446744069414584319 ; clear bit 32
+  ret i64 %a
+}
+
+define i64 @and3_pgso(i64 %x) !prof !14 {
+; CHECK-LABEL: and3_pgso:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    btrq $62, %rax
+; CHECK-NEXT:    retq
+  %a = and i64 %x, 13835058055282163711 ; clear bit 62
+  ret i64 %a
+}
+
+define i64 @and4_pgso(i64 %x) !prof !14 {
+; CHECK-LABEL: and4_pgso:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    btrq $63, %rax
+; CHECK-NEXT:    retq
+  %a = and i64 %x, 9223372036854775807 ; clear bit 63
+  ret i64 %a
+}
+
+define i64 @or1_pgso(i64 %x) !prof !14 {
+; CHECK-LABEL: or1_pgso:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    btsq $31, %rax
+; CHECK-NEXT:    retq
+  %a = or i64 %x, 2147483648 ; set bit 31
+  ret i64 %a
+}
+
+define i64 @or2_pgso(i64 %x) !prof !14 {
+; CHECK-LABEL: or2_pgso:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    btsq $32, %rax
+; CHECK-NEXT:    retq
+  %a = or i64 %x, 4294967296 ; set bit 32
+  ret i64 %a
+}
+
+define i64 @or3_pgso(i64 %x) !prof !14 {
+; CHECK-LABEL: or3_pgso:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    btsq $62, %rax
+; CHECK-NEXT:    retq
+  %a = or i64 %x, 4611686018427387904 ; set bit 62
+  ret i64 %a
+}
+
+define i64 @or4_pgso(i64 %x) !prof !14 {
+; CHECK-LABEL: or4_pgso:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    btsq $63, %rax
+; CHECK-NEXT:    retq
+  %a = or i64 %x, 9223372036854775808 ; set bit 63
+  ret i64 %a
+}
+
+define i64 @xor1_pgso(i64 %x) !prof !14 {
+; CHECK-LABEL: xor1_pgso:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    btcq $31, %rax
+; CHECK-NEXT:    retq
+  %a = xor i64 %x, 2147483648 ; toggle bit 31
+  ret i64 %a
+}
+
+define i64 @xor2_pgso(i64 %x) !prof !14 {
+; CHECK-LABEL: xor2_pgso:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    btcq $32, %rax
+; CHECK-NEXT:    retq
+  %a = xor i64 %x, 4294967296 ; toggle bit 32
+  ret i64 %a
+}
+
+define i64 @xor3_pgso(i64 %x) !prof !14 {
+; CHECK-LABEL: xor3_pgso:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    btcq $62, %rax
+; CHECK-NEXT:    retq
+  %a = xor i64 %x, 4611686018427387904 ; toggle bit 62
+  ret i64 %a
+}
+
+define i64 @xor4_pgso(i64 %x) !prof !14 {
+; CHECK-LABEL: xor4_pgso:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    btcq $63, %rax
+; CHECK-NEXT:    retq
+  %a = xor i64 %x, 9223372036854775808 ; toggle bit 63
+  ret i64 %a
+}
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 1, !"ProfileSummary", !1}
+!1 = !{!2, !3, !4, !5, !6, !7, !8, !9}
+!2 = !{!"ProfileFormat", !"InstrProf"}
+!3 = !{!"TotalCount", i64 10000}
+!4 = !{!"MaxCount", i64 10}
+!5 = !{!"MaxInternalCount", i64 1}
+!6 = !{!"MaxFunctionCount", i64 1000}
+!7 = !{!"NumCounts", i64 3}
+!8 = !{!"NumFunctions", i64 3}
+!9 = !{!"DetailedSummary", !10}
+!10 = !{!11, !12, !13}
+!11 = !{i32 10000, i64 100, i32 1}
+!12 = !{i32 999000, i64 100, i32 1}
+!13 = !{i32 999999, i64 1, i32 2}
+!14 = !{!"function_entry_count", i64 0}
diff --git a/llvm/test/CodeGen/X86/x86-64-disp.ll b/llvm/test/CodeGen/X86/x86-64-disp.ll
index fd63ff1be..78168025e 100644
--- a/llvm/test/CodeGen/X86/x86-64-disp.ll
+++ b/llvm/test/CodeGen/X86/x86-64-disp.ll
@@ -1,4 +1,5 @@
-; RUN: llc < %s | grep mov | count 2
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s | FileCheck %s
 
 ; Fold an offset into an address even if it's not a 32-bit
 ; signed integer.
@@ -8,6 +9,11 @@ target triple = "x86_64-unknown-linux-gnu"
 @call_used_regs = external global [53 x i8], align 32
 
 define fastcc void @foo() nounwind {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl $4294967295, %eax # imm = 0xFFFFFFFF
+; CHECK-NEXT:    movb $1, call_used_regs(%rax)
+; CHECK-NEXT:    retq
 	%t = getelementptr [53 x i8], [53 x i8]* @call_used_regs, i64 0, i64 4294967295
 	store i8 1, i8* %t, align 1
 	ret void
diff --git a/llvm/test/CodeGen/X86/x86-64-double-shifts-Oz-Os-O2.ll b/llvm/test/CodeGen/X86/x86-64-double-shifts-Oz-Os-O2.ll
index 5a180104b..69688b583 100644
--- a/llvm/test/CodeGen/X86/x86-64-double-shifts-Oz-Os-O2.ll
+++ b/llvm/test/CodeGen/X86/x86-64-double-shifts-Oz-Os-O2.ll
@@ -24,7 +24,7 @@ entry:
   ret i64 %or
 }
 
-attributes #0 = { minsize nounwind readnone uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { minsize nounwind readnone uwtable "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 
 ; clang -Os -c test2.cpp -emit-llvm -S
@@ -50,7 +50,20 @@ entry:
   ret i64 %or
 }
 
-attributes #1 = { nounwind optsize readnone uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+define i64 @_Z8lshift11mm_pgso(i64 %a, i64 %b) !prof !14 {
+; CHECK-LABEL: _Z8lshift11mm_pgso:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    shldq $11, %rsi, %rax
+; CHECK-NEXT:    retq
+entry:
+  %shl = shl i64 %a, 11
+  %shr = lshr i64 %b, 53
+  %or = or i64 %shr, %shl
+  ret i64 %or
+}
+
+attributes #1 = { nounwind optsize readnone uwtable "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 ; clang -O2 -c test2.cpp -emit-llvm -S
 ; Verify that we do not generate shld insruction when we are not optimizing
@@ -76,5 +89,21 @@ entry:
   ret i64 %or
 }
 
-attributes #2= { nounwind readnone uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2= { nounwind readnone uwtable "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
+!llvm.module.flags = !{!0}
+!0 = !{i32 1, !"ProfileSummary", !1}
+!1 = !{!2, !3, !4, !5, !6, !7, !8, !9}
+!2 = !{!"ProfileFormat", !"InstrProf"}
+!3 = !{!"TotalCount", i64 10000}
+!4 = !{!"MaxCount", i64 10}
+!5 = !{!"MaxInternalCount", i64 1}
+!6 = !{!"MaxFunctionCount", i64 1000}
+!7 = !{!"NumCounts", i64 3}
+!8 = !{!"NumFunctions", i64 3}
+!9 = !{!"DetailedSummary", !10}
+!10 = !{!11, !12, !13}
+!11 = !{i32 10000, i64 100, i32 1}
+!12 = !{i32 999000, i64 100, i32 1}
+!13 = !{i32 999999, i64 1, i32 2}
+!14 = !{!"function_entry_count", i64 0}
diff --git a/llvm/test/CodeGen/X86/x86-64-flags-intrinsics.ll b/llvm/test/CodeGen/X86/x86-64-flags-intrinsics.ll
index 2852ef49e..2cc646855 100644
--- a/llvm/test/CodeGen/X86/x86-64-flags-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/x86-64-flags-intrinsics.ll
@@ -12,9 +12,9 @@ entry:
 
 ; CHECK-LABEL: read_flags:
 ; CHECK:      pushq   %rbp
-; CHECK:      .seh_pushreg 5
+; CHECK:      .seh_pushreg %rbp
 ; CHECK:      movq    %rsp, %rbp
-; CHECK:      .seh_setframe 5, 0
+; CHECK:      .seh_setframe %rbp, 0
 ; CHECK:      .seh_endprologue
 ; CHECK-NEXT: pushfq
 ; CHECK-NEXT: popq    %rax
@@ -28,9 +28,9 @@ entry:
 
 ; CHECK-LABEL: write_flags:
 ; CHECK:      pushq   %rbp
-; CHECK:      .seh_pushreg 5
+; CHECK:      .seh_pushreg %rbp
 ; CHECK:      movq    %rsp, %rbp
-; CHECK:      .seh_setframe 5, 0
+; CHECK:      .seh_setframe %rbp, 0
 ; CHECK:      .seh_endprologue
 ; CHECK-NEXT: pushq   %rcx
 ; CHECK-NEXT: popfq
diff --git a/llvm/test/CodeGen/X86/x86-64-gv-offset.ll b/llvm/test/CodeGen/X86/x86-64-gv-offset.ll
index f60d2cff3..3727797e5 100644
--- a/llvm/test/CodeGen/X86/x86-64-gv-offset.ll
+++ b/llvm/test/CodeGen/X86/x86-64-gv-offset.ll
@@ -1,13 +1,23 @@
-; RUN: llc < %s -mtriple=x86_64-apple-darwin | not grep lea
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-apple-darwin | FileCheck %s
 
 	%struct.x = type { float, double }
 @X = global %struct.x { float 1.000000e+00, double 2.000000e+00 }, align 16		; <%struct.x*> [#uses=2]
 
 define i32 @main() nounwind  {
+; CHECK-LABEL: main:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; CHECK-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
+; CHECK-NEXT:    callq _t
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    popq %rcx
+; CHECK-NEXT:    retq
 entry:
 	%tmp2 = load float, float* getelementptr (%struct.x, %struct.x* @X, i32 0, i32 0), align 16		; <float> [#uses=1]
 	%tmp4 = load double, double* getelementptr (%struct.x, %struct.x* @X, i32 0, i32 1), align 8		; <double> [#uses=1]
-	tail call void @t( float %tmp2, double %tmp4 ) nounwind 
+	tail call void @t( float %tmp2, double %tmp4 ) nounwind
 	ret i32 0
 }
 
diff --git a/llvm/test/CodeGen/X86/x86-64-intrcc.ll b/llvm/test/CodeGen/X86/x86-64-intrcc.ll
index 51215d780..866108e3c 100644
--- a/llvm/test/CodeGen/X86/x86-64-intrcc.ll
+++ b/llvm/test/CodeGen/X86/x86-64-intrcc.ll
@@ -177,4 +177,4 @@ entry:
 }
 
 
-attributes #0 = { nounwind "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" }
+attributes #0 = { nounwind "frame-pointer"="all" }
diff --git a/llvm/test/CodeGen/X86/x86-64-mem.ll b/llvm/test/CodeGen/X86/x86-64-mem.ll
index 557f9486a..2c46f8887 100644
--- a/llvm/test/CodeGen/X86/x86-64-mem.ll
+++ b/llvm/test/CodeGen/X86/x86-64-mem.ll
@@ -1,11 +1,6 @@
-; RUN: llc < %s -mtriple=x86_64-apple-darwin -o %t1
-; RUN: grep GOTPCREL %t1 | count 4
-; RUN: grep %%rip      %t1 | count 6
-; RUN: grep movq     %t1 | count 6
-; RUN: grep leaq     %t1 | count 1
-; RUN: llc < %s -mtriple=x86_64-pc-linux -relocation-model=static -o %t2
-; RUN: grep movl %t2 | count 2
-; RUN: grep movq %t2 | count 2
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-apple-darwin | FileCheck %s --check-prefix=DARWIN
+; RUN: llc < %s -mtriple=x86_64-pc-linux -relocation-model=static | FileCheck %s --check-prefix=LINUX
 
 @ptr = external global i32*		; <i32**> [#uses=1]
 @src = external global [0 x i32]		; <[0 x i32]*> [#uses=1]
@@ -17,17 +12,51 @@
 @bdst = internal global [500000 x i32] zeroinitializer, align 32		; <[500000 x i32]*> [#uses=0]
 
 define void @test1() nounwind {
+; DARWIN-LABEL: test1:
+; DARWIN:       ## %bb.0:
+; DARWIN-NEXT:    movq _src@{{.*}}(%rip), %rax
+; DARWIN-NEXT:    movl (%rax), %eax
+; DARWIN-NEXT:    movq _dst@{{.*}}(%rip), %rcx
+; DARWIN-NEXT:    movl %eax, (%rcx)
+; DARWIN-NEXT:    retq
+;
+; LINUX-LABEL: test1:
+; LINUX:       # %bb.0:
+; LINUX-NEXT:    movl {{.*}}(%rip), %eax
+; LINUX-NEXT:    movl %eax, {{.*}}(%rip)
+; LINUX-NEXT:    retq
 	%tmp = load i32, i32* getelementptr ([0 x i32], [0 x i32]* @src, i32 0, i32 0)		; <i32> [#uses=1]
 	store i32 %tmp, i32* getelementptr ([0 x i32], [0 x i32]* @dst, i32 0, i32 0)
 	ret void
 }
 
 define void @test2() nounwind {
+; DARWIN-LABEL: test2:
+; DARWIN:       ## %bb.0:
+; DARWIN-NEXT:    movq _dst@{{.*}}(%rip), %rax
+; DARWIN-NEXT:    movq _ptr@{{.*}}(%rip), %rcx
+; DARWIN-NEXT:    movq %rax, (%rcx)
+; DARWIN-NEXT:    retq
+;
+; LINUX-LABEL: test2:
+; LINUX:       # %bb.0:
+; LINUX-NEXT:    movq $dst, {{.*}}(%rip)
+; LINUX-NEXT:    retq
 	store i32* getelementptr ([0 x i32], [0 x i32]* @dst, i32 0, i32 0), i32** @ptr
 	ret void
 }
 
 define void @test3() nounwind {
+; DARWIN-LABEL: test3:
+; DARWIN:       ## %bb.0:
+; DARWIN-NEXT:    leaq {{.*}}(%rip), %rax
+; DARWIN-NEXT:    movq %rax, {{.*}}(%rip)
+; DARWIN-NEXT:    retq
+;
+; LINUX-LABEL: test3:
+; LINUX:       # %bb.0:
+; LINUX-NEXT:    movq $ldst, {{.*}}(%rip)
+; LINUX-NEXT:    retq
 	store i32* getelementptr ([500 x i32], [500 x i32]* @ldst, i32 0, i32 0), i32** @lptr
 	br label %return
 
diff --git a/llvm/test/CodeGen/X86/x86-64-pic-1.ll b/llvm/test/CodeGen/X86/x86-64-pic-1.ll
index 46cd4f81b..9a7e4e986 100644
--- a/llvm/test/CodeGen/X86/x86-64-pic-1.ll
+++ b/llvm/test/CodeGen/X86/x86-64-pic-1.ll
@@ -1,7 +1,15 @@
-; RUN: llc < %s -mtriple=x86_64-pc-linux -relocation-model=pic -o %t1
-; RUN: grep "callq	f@PLT" %t1
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-pc-linux -relocation-model=pic | FileCheck %s
 
 define void @g() {
+; CHECK-LABEL: g:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    callq f@PLT
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
 entry:
 	call void @f( )
 	ret void
diff --git a/llvm/test/CodeGen/X86/x86-64-pic-10.ll b/llvm/test/CodeGen/X86/x86-64-pic-10.ll
index d76cf6a17..303bd4b87 100644
--- a/llvm/test/CodeGen/X86/x86-64-pic-10.ll
+++ b/llvm/test/CodeGen/X86/x86-64-pic-10.ll
@@ -1,14 +1,26 @@
-; RUN: llc < %s -mtriple=x86_64-pc-linux -relocation-model=pic -o %t1
-; RUN: grep "callq	g@PLT" %t1
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-pc-linux -relocation-model=pic | FileCheck %s
 
 @g = weak alias i32 (), i32 ()* @f
 
 define void @h() {
+; CHECK-LABEL: h:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    callq g@PLT
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
 entry:
 	%tmp31 = call i32 @g()
         ret void
 }
 
 define weak i32 @f() {
+; CHECK-LABEL: f:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl $42, %eax
+; CHECK-NEXT:    retq
   ret i32 42
 }
diff --git a/llvm/test/CodeGen/X86/x86-64-pic-11.ll b/llvm/test/CodeGen/X86/x86-64-pic-11.ll
index fd64beb69..7155c3f7f 100644
--- a/llvm/test/CodeGen/X86/x86-64-pic-11.ll
+++ b/llvm/test/CodeGen/X86/x86-64-pic-11.ll
@@ -1,7 +1,15 @@
-; RUN: llc < %s -mtriple=x86_64-pc-linux -relocation-model=pic -o %t1
-; RUN: grep "callq	__fixunsxfti@PLT" %t1
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-pc-linux -relocation-model=pic  | FileCheck %s
 
 define i128 @f(x86_fp80 %a) nounwind {
+; CHECK-LABEL: f:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    subq $24, %rsp
+; CHECK-NEXT:    fldt {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    fstpt (%rsp)
+; CHECK-NEXT:    callq __fixunsxfti@PLT
+; CHECK-NEXT:    addq $24, %rsp
+; CHECK-NEXT:    retq
 entry:
 	%tmp78 = fptoui x86_fp80 %a to i128
 	ret i128 %tmp78
diff --git a/llvm/test/CodeGen/X86/x86-64-pic-2.ll b/llvm/test/CodeGen/X86/x86-64-pic-2.ll
index f3f7b1dff..76117a69f 100644
--- a/llvm/test/CodeGen/X86/x86-64-pic-2.ll
+++ b/llvm/test/CodeGen/X86/x86-64-pic-2.ll
@@ -1,8 +1,15 @@
-; RUN: llc < %s -mtriple=x86_64-pc-linux -relocation-model=pic -o %t1
-; RUN: grep "callq	f" %t1
-; RUN: not grep "callq	f@PLT" %t1
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-pc-linux -relocation-model=pic | FileCheck %s
 
 define void @g() {
+; CHECK-LABEL: g:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    callq f
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
 entry:
 	call void @f( )
 	ret void
diff --git a/llvm/test/CodeGen/X86/x86-64-pic-4.ll b/llvm/test/CodeGen/X86/x86-64-pic-4.ll
index 42d08cc20..57ebd8f95 100644
--- a/llvm/test/CodeGen/X86/x86-64-pic-4.ll
+++ b/llvm/test/CodeGen/X86/x86-64-pic-4.ll
@@ -1,9 +1,14 @@
-; RUN: llc < %s -mtriple=x86_64-pc-linux -relocation-model=pic -o %t1
-; RUN: grep "movq	a@GOTPCREL(%rip)," %t1
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-pc-linux -relocation-model=pic | FileCheck %s
 
 @a = global i32 0
 
 define i32 @get_a() {
+; CHECK-LABEL: get_a:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movq a@{{.*}}(%rip), %rax
+; CHECK-NEXT:    movl (%rax), %eax
+; CHECK-NEXT:    retq
 entry:
 	%tmp1 = load i32, i32* @a, align 4
 	ret i32 %tmp1
diff --git a/llvm/test/CodeGen/X86/x86-64-pic-5.ll b/llvm/test/CodeGen/X86/x86-64-pic-5.ll
index d217a5c47..a24e5a203 100644
--- a/llvm/test/CodeGen/X86/x86-64-pic-5.ll
+++ b/llvm/test/CodeGen/X86/x86-64-pic-5.ll
@@ -1,10 +1,13 @@
-; RUN: llc < %s -mtriple=x86_64-pc-linux -relocation-model=pic -o %t1
-; RUN: grep "movl	a(%rip)," %t1
-; RUN: not grep GOTPCREL %t1
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-pc-linux -relocation-model=pic | FileCheck %s
 
 @a = hidden global i32 0
 
 define i32 @get_a() {
+; CHECK-LABEL: get_a:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movl {{.*}}(%rip), %eax
+; CHECK-NEXT:    retq
 entry:
 	%tmp1 = load i32, i32* @a, align 4
 	ret i32 %tmp1
diff --git a/llvm/test/CodeGen/X86/x86-64-pic-6.ll b/llvm/test/CodeGen/X86/x86-64-pic-6.ll
index 8671023da..8301dce3e 100644
--- a/llvm/test/CodeGen/X86/x86-64-pic-6.ll
+++ b/llvm/test/CodeGen/X86/x86-64-pic-6.ll
@@ -1,10 +1,13 @@
-; RUN: llc < %s -mtriple=x86_64-pc-linux -relocation-model=pic -o %t1
-; RUN: grep "movl	a(%rip)," %t1
-; RUN: not grep GOTPCREL %t1
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-pc-linux -relocation-model=pic | FileCheck %s
 
 @a = internal global i32 0
 
 define i32 @get_a() nounwind {
+; CHECK-LABEL: get_a:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movl {{.*}}(%rip), %eax
+; CHECK-NEXT:    retq
 entry:
 	%tmp1 = load i32, i32* @a, align 4
 	ret i32 %tmp1
diff --git a/llvm/test/CodeGen/X86/x86-64-pic-7.ll b/llvm/test/CodeGen/X86/x86-64-pic-7.ll
index de240a38d..b884bec7a 100644
--- a/llvm/test/CodeGen/X86/x86-64-pic-7.ll
+++ b/llvm/test/CodeGen/X86/x86-64-pic-7.ll
@@ -1,7 +1,11 @@
-; RUN: llc < %s -mtriple=x86_64-pc-linux -relocation-model=pic -o %t1
-; RUN: grep "movq	f@GOTPCREL(%rip)," %t1
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-pc-linux -relocation-model=pic | FileCheck %s
 
 define void ()* @g() nounwind {
+; CHECK-LABEL: g:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movq f@{{.*}}(%rip), %rax
+; CHECK-NEXT:    retq
 entry:
 	ret void ()* @f
 }
diff --git a/llvm/test/CodeGen/X86/x86-64-pic-8.ll b/llvm/test/CodeGen/X86/x86-64-pic-8.ll
index db35c3362..e1967995d 100644
--- a/llvm/test/CodeGen/X86/x86-64-pic-8.ll
+++ b/llvm/test/CodeGen/X86/x86-64-pic-8.ll
@@ -1,8 +1,11 @@
-; RUN: llc < %s -mtriple=x86_64-pc-linux -relocation-model=pic -o %t1
-; RUN: grep "leaq	f(%rip)," %t1
-; RUN: not grep GOTPCREL %t1
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-pc-linux -relocation-model=pic | FileCheck %s
 
 define void ()* @g() {
+; CHECK-LABEL: g:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    leaq {{.*}}(%rip), %rax
+; CHECK-NEXT:    retq
 entry:
 	ret void ()* @f
 }
diff --git a/llvm/test/CodeGen/X86/x86-64-pic-9.ll b/llvm/test/CodeGen/X86/x86-64-pic-9.ll
index 6daea84e1..1f461bf44 100644
--- a/llvm/test/CodeGen/X86/x86-64-pic-9.ll
+++ b/llvm/test/CodeGen/X86/x86-64-pic-9.ll
@@ -1,13 +1,19 @@
-; RUN: llc < %s -mtriple=x86_64-pc-linux -relocation-model=pic -o %t1
-; RUN: grep "leaq	f(%rip)," %t1
-; RUN: not grep GOTPCREL %t1
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-pc-linux -relocation-model=pic | FileCheck %s
 
 define void ()* @g() nounwind {
+; CHECK-LABEL: g:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    leaq {{.*}}(%rip), %rax
+; CHECK-NEXT:    retq
 entry:
 	ret void ()* @f
 }
 
 define internal void @f() nounwind {
+; CHECK-LABEL: f:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    retq
 entry:
 	ret void
 }
diff --git a/llvm/test/CodeGen/X86/x86-64-ret0.ll b/llvm/test/CodeGen/X86/x86-64-ret0.ll
index 3e840ab2e..4baf11c53 100644
--- a/llvm/test/CodeGen/X86/x86-64-ret0.ll
+++ b/llvm/test/CodeGen/X86/x86-64-ret0.ll
@@ -1,7 +1,16 @@
-; RUN: llc < %s -mtriple=x86_64-- | grep mov | count 1
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-- | FileCheck %s
 
 define i32 @f() nounwind  {
-	tail call void @t( i32 1 ) nounwind 
+; CHECK-LABEL: f:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    movl $1, %edi
+; CHECK-NEXT:    callq t
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    popq %rcx
+; CHECK-NEXT:    retq
+	tail call void @t( i32 1 ) nounwind
 	ret i32 0
 }
 
diff --git a/llvm/test/CodeGen/X86/x86-64-stack-and-frame-ptr.ll b/llvm/test/CodeGen/X86/x86-64-stack-and-frame-ptr.ll
index b1f4ca562..fe1bd119e 100644
--- a/llvm/test/CodeGen/X86/x86-64-stack-and-frame-ptr.ll
+++ b/llvm/test/CodeGen/X86/x86-64-stack-and-frame-ptr.ll
@@ -29,6 +29,6 @@ entry:
   ret void
 }
 
-attributes #0 = { nounwind uwtable "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf"}
+attributes #0 = { nounwind uwtable "frame-pointer"="all"}
 
 
diff --git a/llvm/test/CodeGen/X86/x86-64-varargs.ll b/llvm/test/CodeGen/X86/x86-64-varargs.ll
index ed07bde63..58f7c82c2 100644
--- a/llvm/test/CodeGen/X86/x86-64-varargs.ll
+++ b/llvm/test/CodeGen/X86/x86-64-varargs.ll
@@ -1,10 +1,29 @@
-; RUN: llc < %s -mtriple=x86_64-apple-darwin -code-model=large -relocation-model=static | grep call | not grep rax
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-apple-darwin -code-model=large -relocation-model=static | FileCheck %s
 
 @.str = internal constant [26 x i8] c"%d, %f, %d, %lld, %d, %f\0A\00"		; <[26 x i8]*> [#uses=1]
 
-declare i32 @printf(i8*, ...) nounwind 
+declare i32 @printf(i8*, ...) nounwind
 
 define i32 @main() nounwind  {
+; CHECK-LABEL: main:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    movabsq $_.str, %rdi
+; CHECK-NEXT:    movabsq $_printf, %r9
+; CHECK-NEXT:    movabsq $LCPI0_0, %rax
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    movabsq $LCPI0_1, %rax
+; CHECK-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
+; CHECK-NEXT:    movabsq $123456677890, %rcx ## imm = 0x1CBE976802
+; CHECK-NEXT:    movl $12, %esi
+; CHECK-NEXT:    movl $120, %edx
+; CHECK-NEXT:    movl $-10, %r8d
+; CHECK-NEXT:    movb $2, %al
+; CHECK-NEXT:    callq *%r9
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    popq %rcx
+; CHECK-NEXT:    retq
 entry:
 	%tmp10.i = tail call i32 (i8*, ...) @printf( i8* getelementptr ([26 x i8], [26 x i8]* @.str, i32 0, i64 0), i32 12, double 0x3FF3EB8520000000, i32 120, i64 123456677890, i32 -10, double 4.500000e+15 ) nounwind 		; <i32> [#uses=0]
 	ret i32 0
diff --git a/llvm/test/CodeGen/X86/x86-cmov-converter.ll b/llvm/test/CodeGen/X86/x86-cmov-converter.ll
index d9a2f6b24..1f7565008 100644
--- a/llvm/test/CodeGen/X86/x86-cmov-converter.ll
+++ b/llvm/test/CodeGen/X86/x86-cmov-converter.ll
@@ -57,7 +57,7 @@
 ;;    if (a[i] > a[t])
 ;;      t = i;
 ;;  }
-;;  return a[t];
+;;  return t;
 ;;}
 ;;
 ;;
@@ -177,30 +177,24 @@ for.body.preheader:                               ; preds = %entry
   %wide.trip.count = zext i32 %n to i64
   br label %for.body
 
-for.cond.cleanup.loopexit:                        ; preds = %for.body
-  %phitmp = sext i32 %i.0.t.0 to i64
-  br label %for.cond.cleanup
-
-for.cond.cleanup:                                 ; preds = %for.cond.cleanup.loopexit, %entry
-  %t.0.lcssa = phi i64 [ 0, %entry ], [ %phitmp, %for.cond.cleanup.loopexit ]
-  %arrayidx5 = getelementptr inbounds i32, i32* %a, i64 %t.0.lcssa
-  %0 = load i32, i32* %arrayidx5, align 4
-  ret i32 %0
+for.cond.cleanup:                                 ; preds = %for.body, %entry
+  %t.0.lcssa = phi i32 [ 0, %entry ], [ %i.0.t.0, %for.body ]
+  ret i32 %t.0.lcssa
 
 for.body:                                         ; preds = %for.body.preheader, %for.body
   %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 1, %for.body.preheader ]
   %t.015 = phi i32 [ %i.0.t.0, %for.body ], [ 0, %for.body.preheader ]
   %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
-  %1 = load i32, i32* %arrayidx, align 4
+  %0 = load i32, i32* %arrayidx, align 4
   %idxprom1 = sext i32 %t.015 to i64
   %arrayidx2 = getelementptr inbounds i32, i32* %a, i64 %idxprom1
-  %2 = load i32, i32* %arrayidx2, align 4
-  %cmp3 = icmp sgt i32 %1, %2
-  %3 = trunc i64 %indvars.iv to i32
-  %i.0.t.0 = select i1 %cmp3, i32 %3, i32 %t.015
+  %1 = load i32, i32* %arrayidx2, align 4
+  %cmp3 = icmp sgt i32 %0, %1
+  %2 = trunc i64 %indvars.iv to i32
+  %i.0.t.0 = select i1 %cmp3, i32 %2, i32 %t.015
   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
   %exitcond = icmp eq i64 %indvars.iv.next, %wide.trip.count
-  br i1 %exitcond, label %for.cond.cleanup.loopexit, label %for.body
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
 }
 
 ; CHECK-LABEL: MaxValue
diff --git a/llvm/test/CodeGen/X86/x86-interleaved-access.ll b/llvm/test/CodeGen/X86/x86-interleaved-access.ll
index 2728871fc..74a83214b 100644
--- a/llvm/test/CodeGen/X86/x86-interleaved-access.ll
+++ b/llvm/test/CodeGen/X86/x86-interleaved-access.ll
@@ -260,25 +260,25 @@ define void @interleaved_store_vf32_i8_stride4(<32 x i8> %x1, <32 x i8> %x2, <32
 ; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm2[8],xmm3[8],xmm2[9],xmm3[9],xmm2[10],xmm3[10],xmm2[11],xmm3[11],xmm2[12],xmm3[12],xmm2[13],xmm3[13],xmm2[14],xmm3[14],xmm2[15],xmm3[15]
 ; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8],xmm6[8],xmm0[9],xmm6[9],xmm0[10],xmm6[10],xmm0[11],xmm6[11],xmm0[12],xmm6[12],xmm0[13],xmm6[13],xmm0[14],xmm6[14],xmm0[15],xmm6[15]
 ; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm7[0],xmm1[0],xmm7[1],xmm1[1],xmm7[2],xmm1[2],xmm7[3],xmm1[3]
+; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm3
 ; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]
-; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm6, %ymm3
 ; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm7[4],xmm1[4],xmm7[5],xmm1[5],xmm7[6],xmm1[6],xmm7[7],xmm1[7]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1
 ; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm3[2,3],ymm1[2,3]
 ; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm4[4],xmm5[4],xmm4[5],xmm5[5],xmm4[6],xmm5[6],xmm4[7],xmm5[7]
 ; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm9[0],xmm0[0],xmm9[1],xmm0[1],xmm9[2],xmm0[2],xmm9[3],xmm0[3]
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm4
 ; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm8[0],xmm2[0],xmm8[1],xmm2[1],xmm8[2],xmm2[2],xmm8[3],xmm2[3]
-; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm5, %ymm4
 ; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm9[4],xmm0[4],xmm9[5],xmm0[5],xmm9[6],xmm0[6],xmm9[7],xmm0[7]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm4[2,3],ymm0[2,3]
 ; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm8[4],xmm2[4],xmm8[5],xmm2[5],xmm8[6],xmm2[6],xmm8[7],xmm2[7]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm6, %ymm3
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm5, %ymm2
-; AVX1-NEXT:    vmovaps %ymm2, 32(%rdi)
-; AVX1-NEXT:    vmovaps %ymm3, (%rdi)
 ; AVX1-NEXT:    vmovaps %ymm0, 96(%rdi)
 ; AVX1-NEXT:    vmovaps %ymm1, 64(%rdi)
+; AVX1-NEXT:    vmovaps %ymm2, 32(%rdi)
+; AVX1-NEXT:    vmovaps %ymm3, (%rdi)
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
@@ -389,29 +389,30 @@ ret void
 define <8 x i8> @interleaved_load_vf8_i8_stride4(<32 x i8>* %ptr) {
 ; AVX-LABEL: interleaved_load_vf8_i8_stride4:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vmovdqa {{.*#+}} xmm0 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm0 = <0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u>
 ; AVX-NEXT:    vmovdqa (%rdi), %xmm1
 ; AVX-NEXT:    vmovdqa 16(%rdi), %xmm2
 ; AVX-NEXT:    vpshufb %xmm0, %xmm2, %xmm3
 ; AVX-NEXT:    vpshufb %xmm0, %xmm1, %xmm0
-; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm4 = xmm0[0],xmm3[0]
-; AVX-NEXT:    vmovdqa {{.*#+}} xmm5 = [1,1,3,3,5,5,7,7,7,7,3,3,6,6,7,7]
-; AVX-NEXT:    vpshufb %xmm5, %xmm3, %xmm3
-; AVX-NEXT:    vpshufb %xmm5, %xmm0, %xmm0
-; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
-; AVX-NEXT:    vpaddw %xmm0, %xmm4, %xmm0
-; AVX-NEXT:    vmovdqa {{.*#+}} xmm3 = [6,7,2,3,14,15,10,11,14,15,10,11,12,13,14,15]
-; AVX-NEXT:    vpshufb %xmm3, %xmm2, %xmm2
-; AVX-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm2[1,0,3,2,4,5,6,7]
-; AVX-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
-; AVX-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm1[1,0,3,2,4,5,6,7]
-; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm4[0]
-; AVX-NEXT:    vmovdqa {{.*#+}} xmm4 = [3,3,1,1,7,7,5,5,1,1,5,5,0,0,1,1]
+; AVX-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm3 = <1,5,9,13,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX-NEXT:    vpshufb %xmm3, %xmm2, %xmm4
+; AVX-NEXT:    vpshufb %xmm3, %xmm1, %xmm3
+; AVX-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
+; AVX-NEXT:    vpaddb %xmm3, %xmm0, %xmm0
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm3 = <2,6,10,14,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX-NEXT:    vpshufb %xmm3, %xmm2, %xmm4
+; AVX-NEXT:    vpshufb %xmm3, %xmm1, %xmm3
+; AVX-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm4 = <3,7,11,15,u,u,u,u,u,u,u,u,u,u,u,u>
 ; AVX-NEXT:    vpshufb %xmm4, %xmm2, %xmm2
 ; AVX-NEXT:    vpshufb %xmm4, %xmm1, %xmm1
-; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; AVX-NEXT:    vpaddw %xmm3, %xmm1, %xmm1
+; AVX-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; AVX-NEXT:    vpaddb %xmm3, %xmm1, %xmm1
+; AVX-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
 ; AVX-NEXT:    retq
   %wide.vec = load <32 x i8>, <32 x i8>* %ptr, align 16
   %v1 = shufflevector <32 x i8> %wide.vec, <32 x i8> undef, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
@@ -888,13 +889,8 @@ define <32 x i1> @interleaved_load_vf32_i8_stride4(<128 x i8>* %ptr) {
 define void @interleaved_store_vf8_i8_stride4(<8 x i8> %x1, <8 x i8> %x2, <8 x i8> %x3, <8 x i8> %x4, <32 x i8>* %p) {
 ; AVX-LABEL: interleaved_store_vf8_i8_stride4:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vmovdqa {{.*#+}} xmm4 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
-; AVX-NEXT:    vpshufb %xmm4, %xmm1, %xmm1
-; AVX-NEXT:    vpshufb %xmm4, %xmm0, %xmm0
 ; AVX-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
-; AVX-NEXT:    vpshufb %xmm4, %xmm3, %xmm1
-; AVX-NEXT:    vpshufb %xmm4, %xmm2, %xmm2
-; AVX-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; AVX-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3],xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]
 ; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
 ; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
 ; AVX-NEXT:    vmovdqa %xmm0, 16(%rdi)
@@ -1017,17 +1013,17 @@ define <8 x i8> @interleaved_load_vf8_i8_stride3(<24 x i8>* %ptr){
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vmovdqa (%rdi), %xmm0
 ; AVX-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX-NEXT:    vpshufb {{.*#+}} xmm2 = zero,xmm1[u],zero,xmm1[u],zero,xmm1[u],zero,xmm1[u],zero,xmm1[u],zero,xmm1[u,2,u,5,u]
-; AVX-NEXT:    vpshufb {{.*#+}} xmm3 = xmm0[0,u,3,u,6,u,9,u,12,u,15,u],zero,xmm0[u],zero,xmm0[u]
+; AVX-NEXT:    vpshufb {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,xmm1[2,5,u,u,u,u,u,u,u,u]
+; AVX-NEXT:    vpshufb {{.*#+}} xmm3 = xmm0[0,3,6,9,12,15],zero,zero,xmm0[u,u,u,u,u,u,u,u]
 ; AVX-NEXT:    vpor %xmm2, %xmm3, %xmm2
-; AVX-NEXT:    vpshufb {{.*#+}} xmm3 = zero,xmm1[u],zero,xmm1[u],zero,xmm1[u],zero,xmm1[u],zero,xmm1[u,0,u,3,u,6,u]
-; AVX-NEXT:    vpshufb {{.*#+}} xmm4 = xmm0[1,u,4,u,7,u,10,u,13,u],zero,xmm0[u],zero,xmm0[u],zero,xmm0[u]
+; AVX-NEXT:    vpshufb {{.*#+}} xmm3 = zero,zero,zero,zero,zero,xmm1[0,3,6,u,u,u,u,u,u,u,u]
+; AVX-NEXT:    vpshufb {{.*#+}} xmm4 = xmm0[1,4,7,10,13],zero,zero,zero,xmm0[u,u,u,u,u,u,u,u]
 ; AVX-NEXT:    vpor %xmm3, %xmm4, %xmm3
-; AVX-NEXT:    vpshufb {{.*#+}} xmm1 = zero,xmm1[u],zero,xmm1[u],zero,xmm1[u],zero,xmm1[u],zero,xmm1[u,1,u,4,u,7,u]
-; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[2,u,5,u,8,u,11,u,14,u],zero,xmm0[u],zero,xmm0[u],zero,xmm0[u]
+; AVX-NEXT:    vpshufb {{.*#+}} xmm1 = zero,zero,zero,zero,zero,xmm1[1,4,7,u,u,u,u,u,u,u,u]
+; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[2,5,8,11,14],zero,zero,zero,xmm0[u,u,u,u,u,u,u,u]
 ; AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vpaddw %xmm0, %xmm3, %xmm0
-; AVX-NEXT:    vpaddw %xmm0, %xmm2, %xmm0
+; AVX-NEXT:    vpaddb %xmm0, %xmm3, %xmm0
+; AVX-NEXT:    vpaddb %xmm0, %xmm2, %xmm0
 ; AVX-NEXT:    retq
 	%wide.vec = load <24 x i8>, <24 x i8>* %ptr
 	%v1 = shufflevector <24 x i8> %wide.vec, <24 x i8> undef,<8 x i32> <i32 0,i32 3,i32 6,i32  9,i32 12,i32 15,i32 18,i32 21>
@@ -1041,19 +1037,15 @@ define <8 x i8> @interleaved_load_vf8_i8_stride3(<24 x i8>* %ptr){
 define void @interleaved_store_vf8_i8_stride3(<8 x i8> %a, <8 x i8> %b, <8 x i8> %c, <24 x i8>* %p) {
 ; AVX-LABEL: interleaved_store_vf8_i8_stride3:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vmovdqa {{.*#+}} xmm3 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
-; AVX-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
-; AVX-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
 ; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; AVX-NEXT:    vpshufb %xmm3, %xmm2, %xmm1
-; AVX-NEXT:    vpshufb {{.*#+}} xmm2 = xmm0[0,8],zero,xmm0[1,9],zero,xmm0[2,10],zero,xmm0[3,11],zero,xmm0[4,12],zero,xmm0[5]
-; AVX-NEXT:    vpshufb {{.*#+}} xmm3 = zero,zero,xmm1[0],zero,zero,xmm1[1],zero,zero,xmm1[2],zero,zero,xmm1[3],zero,zero,xmm1[4],zero
-; AVX-NEXT:    vpor %xmm3, %xmm2, %xmm2
+; AVX-NEXT:    vpshufb {{.*#+}} xmm1 = xmm0[0,8],zero,xmm0[1,9],zero,xmm0[2,10],zero,xmm0[3,11],zero,xmm0[4,12],zero,xmm0[5]
+; AVX-NEXT:    vpshufb {{.*#+}} xmm3 = zero,zero,xmm2[0],zero,zero,xmm2[1],zero,zero,xmm2[2],zero,zero,xmm2[3],zero,zero,xmm2[4],zero
+; AVX-NEXT:    vpor %xmm3, %xmm1, %xmm1
 ; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[13],zero,xmm0[6,14],zero,xmm0[7,15],zero,xmm0[u,u,u,u,u,u,u,u]
-; AVX-NEXT:    vpshufb {{.*#+}} xmm1 = zero,xmm1[5],zero,zero,xmm1[6],zero,zero,xmm1[7,u,u,u,u,u,u,u,u]
-; AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpshufb {{.*#+}} xmm2 = zero,xmm2[5],zero,zero,xmm2[6],zero,zero,xmm2[7,u,u,u,u,u,u,u,u]
+; AVX-NEXT:    vpor %xmm2, %xmm0, %xmm0
 ; AVX-NEXT:    vmovq %xmm0, 16(%rdi)
-; AVX-NEXT:    vmovdqu %xmm2, (%rdi)
+; AVX-NEXT:    vmovdqu %xmm1, (%rdi)
 ; AVX-NEXT:    retq
 %1 = shufflevector <8 x i8> %a, <8 x i8> %b, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 %2 = shufflevector <8 x i8> %c, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -1063,64 +1055,24 @@ ret void
 }
 
 define void @interleaved_store_vf16_i8_stride3(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c, <48 x i8>* %p) {
-; AVX1-LABEL: interleaved_store_vf16_i8_stride3:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpalignr {{.*#+}} xmm0 = xmm0[6,7,8,9,10,11,12,13,14,15,0,1,2,3,4,5]
-; AVX1-NEXT:    vpalignr {{.*#+}} xmm3 = xmm1[11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,10]
-; AVX1-NEXT:    vpalignr {{.*#+}} xmm4 = xmm0[5,6,7,8,9,10,11,12,13,14,15],xmm2[0,1,2,3,4]
-; AVX1-NEXT:    vpalignr {{.*#+}} xmm0 = xmm3[5,6,7,8,9,10,11,12,13,14,15],xmm0[0,1,2,3,4]
-; AVX1-NEXT:    vpalignr {{.*#+}} xmm2 = xmm2[5,6,7,8,9,10,11,12,13,14,15],xmm3[0,1,2,3,4]
-; AVX1-NEXT:    vpalignr {{.*#+}} xmm1 = xmm4[5,6,7,8,9,10,11,12,13,14,15],xmm1[0,1,2,3,4]
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5]
-; AVX1-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpalignr {{.*#+}} xmm0 = xmm0[5,6,7,8,9,10,11,12,13,14,15],xmm2[0,1,2,3,4]
-; AVX1-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
-; AVX1-NEXT:    vpalignr {{.*#+}} xmm2 = xmm2[5,6,7,8,9,10,11,12,13,14,15],xmm4[0,1,2,3,4]
-; AVX1-NEXT:    vpshufb %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vmovdqu %xmm0, 16(%rdi)
-; AVX1-NEXT:    vmovdqu %xmm1, (%rdi)
-; AVX1-NEXT:    vmovdqu %xmm2, 32(%rdi)
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: interleaved_store_vf16_i8_stride3:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpalignr {{.*#+}} xmm0 = xmm0[6,7,8,9,10,11,12,13,14,15,0,1,2,3,4,5]
-; AVX2-NEXT:    vpalignr {{.*#+}} xmm3 = xmm1[11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,10]
-; AVX2-NEXT:    vpalignr {{.*#+}} xmm4 = xmm0[5,6,7,8,9,10,11,12,13,14,15],xmm2[0,1,2,3,4]
-; AVX2-NEXT:    vpalignr {{.*#+}} xmm0 = xmm3[5,6,7,8,9,10,11,12,13,14,15],xmm0[0,1,2,3,4]
-; AVX2-NEXT:    vpalignr {{.*#+}} xmm2 = xmm2[5,6,7,8,9,10,11,12,13,14,15],xmm3[0,1,2,3,4]
-; AVX2-NEXT:    vpalignr {{.*#+}} xmm1 = xmm4[5,6,7,8,9,10,11,12,13,14,15],xmm1[0,1,2,3,4]
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5]
-; AVX2-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
-; AVX2-NEXT:    vpalignr {{.*#+}} xmm0 = xmm0[5,6,7,8,9,10,11,12,13,14,15],xmm2[0,1,2,3,4]
-; AVX2-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
-; AVX2-NEXT:    vpalignr {{.*#+}} xmm2 = xmm2[5,6,7,8,9,10,11,12,13,14,15],xmm4[0,1,2,3,4]
-; AVX2-NEXT:    vpshufb %xmm3, %xmm2, %xmm2
-; AVX2-NEXT:    vmovdqu %xmm0, 16(%rdi)
-; AVX2-NEXT:    vmovdqu %xmm1, (%rdi)
-; AVX2-NEXT:    vmovdqu %xmm2, 32(%rdi)
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: interleaved_store_vf16_i8_stride3:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpalignr {{.*#+}} xmm0 = xmm0[6,7,8,9,10,11,12,13,14,15,0,1,2,3,4,5]
-; AVX512-NEXT:    vpalignr {{.*#+}} xmm3 = xmm1[11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,10]
-; AVX512-NEXT:    vpalignr {{.*#+}} xmm4 = xmm0[5,6,7,8,9,10,11,12,13,14,15],xmm2[0,1,2,3,4]
-; AVX512-NEXT:    vpalignr {{.*#+}} xmm0 = xmm3[5,6,7,8,9,10,11,12,13,14,15],xmm0[0,1,2,3,4]
-; AVX512-NEXT:    vpalignr {{.*#+}} xmm2 = xmm2[5,6,7,8,9,10,11,12,13,14,15],xmm3[0,1,2,3,4]
-; AVX512-NEXT:    vpalignr {{.*#+}} xmm1 = xmm4[5,6,7,8,9,10,11,12,13,14,15],xmm1[0,1,2,3,4]
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5]
-; AVX512-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
-; AVX512-NEXT:    vpalignr {{.*#+}} xmm0 = xmm0[5,6,7,8,9,10,11,12,13,14,15],xmm2[0,1,2,3,4]
-; AVX512-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
-; AVX512-NEXT:    vpalignr {{.*#+}} xmm2 = xmm2[5,6,7,8,9,10,11,12,13,14,15],xmm4[0,1,2,3,4]
-; AVX512-NEXT:    vpshufb %xmm3, %xmm2, %xmm2
-; AVX512-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
-; AVX512-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm1
-; AVX512-NEXT:    vmovdqu %ymm0, (%rdi)
-; AVX512-NEXT:    vextracti32x4 $2, %zmm1, 32(%rdi)
-; AVX512-NEXT:    vzeroupper
-; AVX512-NEXT:    retq
+; AVX-LABEL: interleaved_store_vf16_i8_stride3:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpalignr {{.*#+}} xmm0 = xmm0[6,7,8,9,10,11,12,13,14,15,0,1,2,3,4,5]
+; AVX-NEXT:    vpalignr {{.*#+}} xmm3 = xmm1[11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,10]
+; AVX-NEXT:    vpalignr {{.*#+}} xmm4 = xmm0[5,6,7,8,9,10,11,12,13,14,15],xmm2[0,1,2,3,4]
+; AVX-NEXT:    vpalignr {{.*#+}} xmm0 = xmm3[5,6,7,8,9,10,11,12,13,14,15],xmm0[0,1,2,3,4]
+; AVX-NEXT:    vpalignr {{.*#+}} xmm2 = xmm2[5,6,7,8,9,10,11,12,13,14,15],xmm3[0,1,2,3,4]
+; AVX-NEXT:    vpalignr {{.*#+}} xmm1 = xmm4[5,6,7,8,9,10,11,12,13,14,15],xmm1[0,1,2,3,4]
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5]
+; AVX-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
+; AVX-NEXT:    vpalignr {{.*#+}} xmm0 = xmm0[5,6,7,8,9,10,11,12,13,14,15],xmm2[0,1,2,3,4]
+; AVX-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
+; AVX-NEXT:    vpalignr {{.*#+}} xmm2 = xmm2[5,6,7,8,9,10,11,12,13,14,15],xmm4[0,1,2,3,4]
+; AVX-NEXT:    vpshufb %xmm3, %xmm2, %xmm2
+; AVX-NEXT:    vmovdqu %xmm0, 16(%rdi)
+; AVX-NEXT:    vmovdqu %xmm1, (%rdi)
+; AVX-NEXT:    vmovdqu %xmm2, 32(%rdi)
+; AVX-NEXT:    retq
 %1 = shufflevector <16 x i8> %a, <16 x i8> %b, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 %2 = shufflevector <16 x i8> %c, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 %interleaved.vec = shufflevector <32 x i8> %1, <32 x i8> %2, <48 x i32> <i32 0, i32 16, i32 32, i32 1, i32 17, i32 33, i32 2, i32 18, i32 34, i32 3, i32 19, i32 35, i32 4, i32 20, i32 36, i32 5, i32 21, i32 37, i32 6, i32 22, i32 38, i32 7, i32 23, i32 39, i32 8, i32 24, i32 40, i32 9, i32 25, i32 41, i32 10, i32 26, i32 42, i32 11, i32 27, i32 43, i32 12, i32 28, i32 44, i32 13, i32 29, i32 45, i32 14, i32 30, i32 46, i32 15, i32 31, i32 47>
@@ -1420,8 +1372,8 @@ define <64 x i8> @interleaved_load_vf64_i8_stride3(<192 x i8>* %ptr){
 ; AVX1-NEXT:    vorps %ymm10, %ymm14, %ymm10
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm14
 ; AVX1-NEXT:    vandnps %ymm14, %ymm12, %ymm14
-; AVX1-NEXT:    vandps %ymm12, %ymm1, %ymm1
-; AVX1-NEXT:    vorps %ymm14, %ymm1, %ymm1
+; AVX1-NEXT:    vandps %ymm1, %ymm12, %ymm1
+; AVX1-NEXT:    vorps %ymm1, %ymm14, %ymm1
 ; AVX1-NEXT:    vpalignr {{.*#+}} xmm4 = xmm13[11,12,13,14,15],xmm4[0,1,2,3,4,5,6,7,8,9,10]
 ; AVX1-NEXT:    vpalignr {{.*#+}} xmm3 = xmm3[11,12,13,14,15],xmm13[0,1,2,3,4,5,6,7,8,9,10]
 ; AVX1-NEXT:    vpalignr {{.*#+}} xmm12 = xmm15[11,12,13,14,15],xmm7[0,1,2,3,4,5,6,7,8,9,10]
@@ -1442,7 +1394,7 @@ define <64 x i8> @interleaved_load_vf64_i8_stride3(<192 x i8>* %ptr){
 ; AVX1-NEXT:    vpalignr {{.*#+}} xmm4 = xmm7[10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9]
 ; AVX1-NEXT:    vpaddb %xmm3, %xmm4, %xmm3
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm3, %ymm0
-; AVX1-NEXT:    vpaddb %xmm9, %xmm1, %xmm1
+; AVX1-NEXT:    vpaddb %xmm1, %xmm9, %xmm1
 ; AVX1-NEXT:    vpalignr {{.*#+}} xmm3 = xmm6[10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9]
 ; AVX1-NEXT:    vpaddb %xmm1, %xmm3, %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
@@ -1568,45 +1520,45 @@ define void @interleaved_store_vf64_i8_stride4(<64 x i8> %a, <64 x i8> %b, <64 x
 ; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm5[8],xmm3[8],xmm5[9],xmm3[9],xmm5[10],xmm3[10],xmm5[11],xmm3[11],xmm5[12],xmm3[12],xmm5[13],xmm3[13],xmm5[14],xmm3[14],xmm5[15],xmm3[15]
 ; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm15[0],xmm0[0],xmm15[1],xmm0[1],xmm15[2],xmm0[2],xmm15[3],xmm0[3]
 ; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm15[4],xmm0[4],xmm15[5],xmm0[5],xmm15[6],xmm0[6],xmm15[7],xmm0[7]
-; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm15 = xmm10[0],xmm6[0],xmm10[1],xmm6[1],xmm10[2],xmm6[2],xmm10[3],xmm6[3]
-; AVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm15, %ymm5
+; AVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm0, %ymm5
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
-; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm5[2,3],ymm0[2,3]
-; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm10[4],xmm6[4],xmm10[5],xmm6[5],xmm10[6],xmm6[6],xmm10[7],xmm6[7]
-; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm9[0],xmm2[0],xmm9[1],xmm2[1],xmm9[2],xmm2[2],xmm9[3],xmm2[3]
+; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm15 = ymm5[2,3],ymm0[2,3]
+; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm10[0],xmm6[0],xmm10[1],xmm6[1],xmm10[2],xmm6[2],xmm10[3],xmm6[3]
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm10 = xmm10[4],xmm6[4],xmm10[5],xmm6[5],xmm10[6],xmm6[6],xmm10[7],xmm6[7]
+; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm9[0],xmm2[0],xmm9[1],xmm2[1],xmm9[2],xmm2[2],xmm9[3],xmm2[3]
 ; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm9[4],xmm2[4],xmm9[5],xmm2[5],xmm9[6],xmm2[6],xmm9[7],xmm2[7]
-; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm8[0],xmm14[0],xmm8[1],xmm14[1],xmm8[2],xmm14[2],xmm8[3],xmm14[3]
-; AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm9, %ymm6
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm2
-; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm10 = ymm6[2,3],ymm2[2,3]
-; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm14 = xmm8[4],xmm14[4],xmm8[5],xmm14[5],xmm8[6],xmm14[6],xmm8[7],xmm14[7]
-; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm13[0],xmm3[0],xmm13[1],xmm3[1],xmm13[2],xmm3[2],xmm13[3],xmm3[3]
+; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm9 = ymm0[2,3],ymm2[2,3]
+; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm8[0],xmm14[0],xmm8[1],xmm14[1],xmm8[2],xmm14[2],xmm8[3],xmm14[3]
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm8 = xmm8[4],xmm14[4],xmm8[5],xmm14[5],xmm8[6],xmm14[6],xmm8[7],xmm14[7]
+; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm13[0],xmm3[0],xmm13[1],xmm3[1],xmm13[2],xmm3[2],xmm13[3],xmm3[3]
 ; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm13[4],xmm3[4],xmm13[5],xmm3[5],xmm13[6],xmm3[6],xmm13[7],xmm3[7]
-; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm12[0],xmm7[0],xmm12[1],xmm7[1],xmm12[2],xmm7[2],xmm12[3],xmm7[3]
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm8, %ymm2
+; AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm0, %ymm6
 ; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm3
-; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm2 = ymm2[2,3],ymm3[2,3]
-; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm12[4],xmm7[4],xmm12[5],xmm7[5],xmm12[6],xmm7[6],xmm12[7],xmm7[7]
-; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm7 = xmm11[0],xmm1[0],xmm11[1],xmm1[1],xmm11[2],xmm1[2],xmm11[3],xmm1[3]
+; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm3 = ymm6[2,3],ymm3[2,3]
+; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm12[0],xmm7[0],xmm12[1],xmm7[1],xmm12[2],xmm7[2],xmm12[3],xmm7[3]
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm7 = xmm12[4],xmm7[4],xmm12[5],xmm7[5],xmm12[6],xmm7[6],xmm12[7],xmm7[7]
+; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm11[0],xmm1[0],xmm11[1],xmm1[1],xmm11[2],xmm1[2],xmm11[3],xmm1[3]
 ; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm11[4],xmm1[4],xmm11[5],xmm1[5],xmm11[6],xmm1[6],xmm11[7],xmm1[7]
-; AVX1-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm11 = xmm6[0],xmm4[0],xmm6[1],xmm4[1],xmm6[2],xmm4[2],xmm6[3],xmm4[3]
-; AVX1-NEXT:    vinsertf128 $1, %xmm7, %ymm11, %ymm7
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1
-; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm7[2,3],ymm1[2,3]
-; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm6[4],xmm4[4],xmm6[5],xmm4[5],xmm6[6],xmm4[6],xmm6[7],xmm4[7]
-; AVX1-NEXT:    vinsertf128 $1, %xmm14, %ymm9, %ymm6
-; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm11, %ymm4
-; AVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm15, %ymm5
-; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm8, %ymm3
-; AVX1-NEXT:    vmovaps %ymm3, 160(%rdi)
-; AVX1-NEXT:    vmovaps %ymm5, 128(%rdi)
-; AVX1-NEXT:    vmovaps %ymm4, 32(%rdi)
-; AVX1-NEXT:    vmovaps %ymm6, (%rdi)
-; AVX1-NEXT:    vmovaps %ymm2, 224(%rdi)
-; AVX1-NEXT:    vmovaps %ymm0, 192(%rdi)
-; AVX1-NEXT:    vmovaps %ymm1, 96(%rdi)
-; AVX1-NEXT:    vmovaps %ymm10, 64(%rdi)
+; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm11 = ymm0[2,3],ymm1[2,3]
+; AVX1-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
+; AVX1-NEXT:    vinsertf128 $1, %xmm8, %ymm2, %ymm2
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm1, %ymm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm10, %ymm5, %ymm4
+; AVX1-NEXT:    vinsertf128 $1, %xmm7, %ymm6, %ymm5
+; AVX1-NEXT:    vmovaps %ymm3, 224(%rdi)
+; AVX1-NEXT:    vmovaps %ymm15, 192(%rdi)
+; AVX1-NEXT:    vmovaps %ymm5, 160(%rdi)
+; AVX1-NEXT:    vmovaps %ymm4, 128(%rdi)
+; AVX1-NEXT:    vmovaps %ymm11, 96(%rdi)
+; AVX1-NEXT:    vmovaps %ymm9, 64(%rdi)
+; AVX1-NEXT:    vmovaps %ymm1, 32(%rdi)
+; AVX1-NEXT:    vmovaps %ymm2, (%rdi)
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/x86-interrupt_cc.ll b/llvm/test/CodeGen/X86/x86-interrupt_cc.ll
index 09f82b46c..2043816f3 100644
--- a/llvm/test/CodeGen/X86/x86-interrupt_cc.ll
+++ b/llvm/test/CodeGen/X86/x86-interrupt_cc.ll
@@ -294,7 +294,7 @@ define x86_intrcc void @foo(i8* %frame) {
 ; CHECK64-SKX-NEXT:    kmovq %k0, {{[0-9]+}}(%rsp) ## 8-byte Spill
 ; CHECK64-SKX-NEXT:    ## encoding: [0xc4,0xe1,0xf8,0x91,0x84,0x24,0x30,0x08,0x00,0x00]
 ; CHECK64-SKX-NEXT:    vmovups %zmm31, {{[0-9]+}}(%rsp) ## 64-byte Spill
-; CHECK64-SKX-NEXT:    ## encoding: [0x62,0x61,0x7c,0x48,0x11,0xbc,0x24,0xe0,0x07,0x00,0x00]
+; CHECK64-SKX-NEXT:    ## encoding: [0x62,0x61,0x7c,0x48,0x11,0x7c,0x24,0x1f]
 ; CHECK64-SKX-NEXT:    vmovups %zmm30, {{[0-9]+}}(%rsp) ## 64-byte Spill
 ; CHECK64-SKX-NEXT:    ## encoding: [0x62,0x61,0x7c,0x48,0x11,0x74,0x24,0x1e]
 ; CHECK64-SKX-NEXT:    vmovups %zmm29, {{[0-9]+}}(%rsp) ## 64-byte Spill
@@ -398,7 +398,7 @@ define x86_intrcc void @foo(i8* %frame) {
 ; CHECK64-SKX-NEXT:    .cfi_offset %xmm28, -448
 ; CHECK64-SKX-NEXT:    .cfi_offset %xmm29, -384
 ; CHECK64-SKX-NEXT:    .cfi_offset %xmm30, -320
-; CHECK64-SKX-NEXT:    .cfi_offset %xmm31, -224
+; CHECK64-SKX-NEXT:    .cfi_offset %xmm31, -256
 ; CHECK64-SKX-NEXT:    .cfi_offset %k0, -144
 ; CHECK64-SKX-NEXT:    .cfi_offset %k1, -136
 ; CHECK64-SKX-NEXT:    .cfi_offset %k2, -128
@@ -474,7 +474,7 @@ define x86_intrcc void @foo(i8* %frame) {
 ; CHECK64-SKX-NEXT:    vmovups {{[0-9]+}}(%rsp), %zmm30 ## 64-byte Reload
 ; CHECK64-SKX-NEXT:    ## encoding: [0x62,0x61,0x7c,0x48,0x10,0x74,0x24,0x1e]
 ; CHECK64-SKX-NEXT:    vmovups {{[0-9]+}}(%rsp), %zmm31 ## 64-byte Reload
-; CHECK64-SKX-NEXT:    ## encoding: [0x62,0x61,0x7c,0x48,0x10,0xbc,0x24,0xe0,0x07,0x00,0x00]
+; CHECK64-SKX-NEXT:    ## encoding: [0x62,0x61,0x7c,0x48,0x10,0x7c,0x24,0x1f]
 ; CHECK64-SKX-NEXT:    kmovq {{[0-9]+}}(%rsp), %k0 ## 8-byte Reload
 ; CHECK64-SKX-NEXT:    ## encoding: [0xc4,0xe1,0xf8,0x90,0x84,0x24,0x30,0x08,0x00,0x00]
 ; CHECK64-SKX-NEXT:    kmovq {{[0-9]+}}(%rsp), %k1 ## 8-byte Reload
@@ -635,7 +635,7 @@ define x86_intrcc void @foo(i8* %frame) {
 ; CHECK32-SKX-NEXT:    kmovq %k0, {{[0-9]+}}(%esp) ## 8-byte Spill
 ; CHECK32-SKX-NEXT:    ## encoding: [0xc4,0xe1,0xf8,0x91,0x84,0x24,0x30,0x02,0x00,0x00]
 ; CHECK32-SKX-NEXT:    vmovups %zmm7, {{[0-9]+}}(%esp) ## 64-byte Spill
-; CHECK32-SKX-NEXT:    ## encoding: [0x62,0xf1,0x7c,0x48,0x11,0xbc,0x24,0xe0,0x01,0x00,0x00]
+; CHECK32-SKX-NEXT:    ## encoding: [0x62,0xf1,0x7c,0x48,0x11,0x7c,0x24,0x07]
 ; CHECK32-SKX-NEXT:    vmovups %zmm6, {{[0-9]+}}(%esp) ## 64-byte Spill
 ; CHECK32-SKX-NEXT:    ## encoding: [0x62,0xf1,0x7c,0x48,0x11,0x74,0x24,0x06]
 ; CHECK32-SKX-NEXT:    vmovups %zmm5, {{[0-9]+}}(%esp) ## 64-byte Spill
@@ -661,7 +661,7 @@ define x86_intrcc void @foo(i8* %frame) {
 ; CHECK32-SKX-NEXT:    .cfi_offset %xmm4, -384
 ; CHECK32-SKX-NEXT:    .cfi_offset %xmm5, -320
 ; CHECK32-SKX-NEXT:    .cfi_offset %xmm6, -256
-; CHECK32-SKX-NEXT:    .cfi_offset %xmm7, -160
+; CHECK32-SKX-NEXT:    .cfi_offset %xmm7, -192
 ; CHECK32-SKX-NEXT:    .cfi_offset %k0, -80
 ; CHECK32-SKX-NEXT:    .cfi_offset %k1, -72
 ; CHECK32-SKX-NEXT:    .cfi_offset %k2, -64
@@ -689,7 +689,7 @@ define x86_intrcc void @foo(i8* %frame) {
 ; CHECK32-SKX-NEXT:    vmovups {{[0-9]+}}(%esp), %zmm6 ## 64-byte Reload
 ; CHECK32-SKX-NEXT:    ## encoding: [0x62,0xf1,0x7c,0x48,0x10,0x74,0x24,0x06]
 ; CHECK32-SKX-NEXT:    vmovups {{[0-9]+}}(%esp), %zmm7 ## 64-byte Reload
-; CHECK32-SKX-NEXT:    ## encoding: [0x62,0xf1,0x7c,0x48,0x10,0xbc,0x24,0xe0,0x01,0x00,0x00]
+; CHECK32-SKX-NEXT:    ## encoding: [0x62,0xf1,0x7c,0x48,0x10,0x7c,0x24,0x07]
 ; CHECK32-SKX-NEXT:    kmovq {{[0-9]+}}(%esp), %k0 ## 8-byte Reload
 ; CHECK32-SKX-NEXT:    ## encoding: [0xc4,0xe1,0xf8,0x90,0x84,0x24,0x30,0x02,0x00,0x00]
 ; CHECK32-SKX-NEXT:    kmovq {{[0-9]+}}(%esp), %k1 ## 8-byte Reload
diff --git a/llvm/test/CodeGen/X86/x86-mixed-alignment-dagcombine.ll b/llvm/test/CodeGen/X86/x86-mixed-alignment-dagcombine.ll
index 8892a69ab..ae571f4a9 100644
--- a/llvm/test/CodeGen/X86/x86-mixed-alignment-dagcombine.ll
+++ b/llvm/test/CodeGen/X86/x86-mixed-alignment-dagcombine.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc  -mtriple=x86_64-apple-macosx10.9.0  -mcpu=core2 -mattr=+64bit,+sse2 < %s | FileCheck %s
 
 ; DAGCombine may choose to rewrite 2 loads feeding a select as a select of
@@ -7,6 +8,18 @@
 declare void @sink(<2 x double>)
 
 define void @test1(i1 %cmp) align 2 {
+; CHECK-LABEL: test1:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    subq $40, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    testb $1, %dil
+; CHECK-NEXT:    leaq {{[0-9]+}}(%rsp), %rax
+; CHECK-NEXT:    movq %rsp, %rcx
+; CHECK-NEXT:    cmovneq %rax, %rcx
+; CHECK-NEXT:    movups (%rcx), %xmm0
+; CHECK-NEXT:    callq _sink
+; CHECK-NEXT:    addq $40, %rsp
+; CHECK-NEXT:    retq
   %1 = alloca  <2 x double>, align 16
   %2 = alloca  <2 x double>, align 8
 
@@ -15,12 +28,21 @@ define void @test1(i1 %cmp) align 2 {
   %val3 = select i1 %cmp, <2 x double> %val, <2 x double> %val2
   call void @sink(<2 x double> %val3)
   ret void
-  ; CHECK: test1
-  ; CHECK: movups
-  ; CHECK: ret
 }
 
 define void @test2(i1 %cmp) align 2 {
+; CHECK-LABEL: test2:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    subq $40, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    testb $1, %dil
+; CHECK-NEXT:    leaq {{[0-9]+}}(%rsp), %rax
+; CHECK-NEXT:    movq %rsp, %rcx
+; CHECK-NEXT:    cmovneq %rax, %rcx
+; CHECK-NEXT:    movaps (%rcx), %xmm0
+; CHECK-NEXT:    callq _sink
+; CHECK-NEXT:    addq $40, %rsp
+; CHECK-NEXT:    retq
   %1 = alloca  <2 x double>, align 16
   %2 = alloca  <2 x double>, align 8
 
@@ -29,7 +51,4 @@ define void @test2(i1 %cmp) align 2 {
   %val3 = select i1 %cmp, <2 x double> %val, <2 x double> %val2
   call void @sink(<2 x double> %val3)
   ret void
-  ; CHECK: test2
-  ; CHECK: movaps
-  ; CHECK: ret
 }
diff --git a/llvm/test/CodeGen/X86/x86-repmov-copy-eflags.ll b/llvm/test/CodeGen/X86/x86-repmov-copy-eflags.ll
index 636f1092c..91bce1610 100644
--- a/llvm/test/CodeGen/X86/x86-repmov-copy-eflags.ll
+++ b/llvm/test/CodeGen/X86/x86-repmov-copy-eflags.ll
@@ -25,6 +25,26 @@ while.end:                                        ; preds = %while.body
   ret void
 }
 
+define void @f_pgso(i8* %p, i8* %q, i32* inalloca nocapture %unused) !prof !14 {
+entry:
+  %g = alloca %struct.T, align 8
+  %r = alloca i32, align 8
+  store i32 0, i32* %r, align 4
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 8 %p, i8* align 8 %q, i32 24, i1 false)
+  br label %while.body
+
+while.body:                                       ; preds = %while.body, %entry
+  %load = load i32, i32* %r, align 4
+  %dec = add nsw i32 %load, -1
+  store i32 %dec, i32* %r, align 4
+  call void @g(%struct.T* %g)
+  %tobool = icmp eq i32 %dec, 0
+  br i1 %tobool, label %while.end, label %while.body
+
+while.end:                                        ; preds = %while.body
+  ret void
+}
+
 ; Function Attrs: argmemonly nounwind
 declare void @llvm.memcpy.p0i8.p0i8.i32(i8* nocapture, i8* nocapture readonly, i32, i1) #1
 
@@ -46,5 +66,38 @@ declare void @g(%struct.T*)
 ; CHECK:     testb    %[[NE_REG]], %[[NE_REG]]
 ; CHECK:     jne
 
+; CHECK-LABEL: _f_pgso:
+; CHECK:     pushl %ebp
+; CHECK:     movl %esp, %ebp
+; CHECK:     andl $-8, %esp
+; CHECK-NOT: movl %esp, %esi
+; CHECK:     rep;movsl
+; CHECK:     leal 8(%esp), %esi
+
+; CHECK:     decl     (%esp)
+; CHECK:     setne    %[[NE_REG:.*]]
+; CHECK:     pushl     %esi
+; CHECK:     calll     _g
+; CHECK:     addl     $4, %esp
+; CHECK:     testb    %[[NE_REG]], %[[NE_REG]]
+; CHECK:     jne
+
 attributes #0 = { nounwind optsize }
 attributes #1 = { argmemonly nounwind }
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 1, !"ProfileSummary", !1}
+!1 = !{!2, !3, !4, !5, !6, !7, !8, !9}
+!2 = !{!"ProfileFormat", !"InstrProf"}
+!3 = !{!"TotalCount", i64 10000}
+!4 = !{!"MaxCount", i64 10}
+!5 = !{!"MaxInternalCount", i64 1}
+!6 = !{!"MaxFunctionCount", i64 1000}
+!7 = !{!"NumCounts", i64 3}
+!8 = !{!"NumFunctions", i64 3}
+!9 = !{!"DetailedSummary", !10}
+!10 = !{!11, !12, !13}
+!11 = !{i32 10000, i64 100, i32 1}
+!12 = !{i32 999000, i64 100, i32 1}
+!13 = !{i32 999999, i64 1, i32 2}
+!14 = !{!"function_entry_count", i64 0}
diff --git a/llvm/test/CodeGen/X86/x86-sanitizer-shrink-wrapping.ll b/llvm/test/CodeGen/X86/x86-sanitizer-shrink-wrapping.ll
index db3bed6e6..c0b6a199d 100644
--- a/llvm/test/CodeGen/X86/x86-sanitizer-shrink-wrapping.ll
+++ b/llvm/test/CodeGen/X86/x86-sanitizer-shrink-wrapping.ll
@@ -35,6 +35,6 @@ end:
   ret void
 }
 
-attributes #0 = { sanitize_address nounwind "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" }
+attributes #0 = { sanitize_address nounwind "frame-pointer"="all" }
 
 declare void @__asan_report_load4(i64)
diff --git a/llvm/test/CodeGen/X86/x86-shifts.ll b/llvm/test/CodeGen/X86/x86-shifts.ll
index 5688d1d28..5b0181123 100644
--- a/llvm/test/CodeGen/X86/x86-shifts.ll
+++ b/llvm/test/CodeGen/X86/x86-shifts.ll
@@ -254,16 +254,16 @@ define <2 x i32> @shl2_other(<2 x i32> %A) nounwind {
 ; X32-LABEL: shl2_other:
 ; X32:       # %bb.0: # %entry
 ; X32-NEXT:    movdqa %xmm0, %xmm1
-; X32-NEXT:    psllq $2, %xmm1
-; X32-NEXT:    psllq $9, %xmm0
+; X32-NEXT:    pslld $2, %xmm1
+; X32-NEXT:    pslld $9, %xmm0
 ; X32-NEXT:    pxor %xmm1, %xmm0
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: shl2_other:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    movdqa %xmm0, %xmm1
-; X64-NEXT:    psllq $2, %xmm1
-; X64-NEXT:    psllq $9, %xmm0
+; X64-NEXT:    pslld $2, %xmm1
+; X64-NEXT:    pslld $9, %xmm0
 ; X64-NEXT:    pxor %xmm1, %xmm0
 ; X64-NEXT:    retq
 entry:
@@ -276,19 +276,17 @@ entry:
 define <2 x i32> @shr2_other(<2 x i32> %A) nounwind {
 ; X32-LABEL: shr2_other:
 ; X32:       # %bb.0: # %entry
-; X32-NEXT:    pand {{\.LCPI.*}}, %xmm0
 ; X32-NEXT:    movdqa %xmm0, %xmm1
-; X32-NEXT:    psrlq $8, %xmm1
-; X32-NEXT:    psrlq $1, %xmm0
+; X32-NEXT:    psrld $8, %xmm1
+; X32-NEXT:    psrld $1, %xmm0
 ; X32-NEXT:    pxor %xmm1, %xmm0
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: shr2_other:
 ; X64:       # %bb.0: # %entry
-; X64-NEXT:    pand {{.*}}(%rip), %xmm0
 ; X64-NEXT:    movdqa %xmm0, %xmm1
-; X64-NEXT:    psrlq $8, %xmm1
-; X64-NEXT:    psrlq $1, %xmm0
+; X64-NEXT:    psrld $8, %xmm1
+; X64-NEXT:    psrld $1, %xmm0
 ; X64-NEXT:    pxor %xmm1, %xmm0
 ; X64-NEXT:    retq
 entry:
diff --git a/llvm/test/CodeGen/X86/x86-shrink-wrap-unwind.ll b/llvm/test/CodeGen/X86/x86-shrink-wrap-unwind.ll
index 75dcbfd84..a73c06d97 100644
--- a/llvm/test/CodeGen/X86/x86-shrink-wrap-unwind.ll
+++ b/llvm/test/CodeGen/X86/x86-shrink-wrap-unwind.ll
@@ -58,7 +58,7 @@ false:
 
 declare i32 @doSomething(i32, i32*)
 
-attributes #0 = { "no-frame-pointer-elim"="false" }
+attributes #0 = { "frame-pointer"="none" }
 
 ; Shrink-wrapping should occur here. We have a frame pointer.
 ; CHECK-LABEL: frameUnwind:
@@ -104,7 +104,7 @@ false:
   ret i32 %tmp.0
 }
 
-attributes #1 = { "no-frame-pointer-elim"="true" }
+attributes #1 = { "frame-pointer"="all" }
 
 ; Shrink-wrapping should occur here. We do not have to unwind.
 ; CHECK-LABEL: framelessnoUnwind:
@@ -150,7 +150,7 @@ false:
   ret i32 %tmp.0
 }
 
-attributes #2 = { "no-frame-pointer-elim"="false" nounwind }
+attributes #2 = { "frame-pointer"="none" nounwind }
 
 
 ; Check that we generate correct code for segmented stack.
diff --git a/llvm/test/CodeGen/X86/x86-shrink-wrapping.ll b/llvm/test/CodeGen/X86/x86-shrink-wrapping.ll
index 8c514b3b2..d91ba93ff 100644
--- a/llvm/test/CodeGen/X86/x86-shrink-wrapping.ll
+++ b/llvm/test/CodeGen/X86/x86-shrink-wrapping.ll
@@ -1489,7 +1489,7 @@ for.inc:
   br i1 %cmp, label %for.cond8, label %fn1.exit
 }
 
-attributes #4 = { "no-frame-pointer-elim"="true" }
+attributes #4 = { "frame-pointer"="all" }
 
 @x = external global i32, align 4
 @y = external global i32, align 4
@@ -1580,4 +1580,4 @@ define void @infiniteLoopNoSuccessor() #5 {
 
 declare void @somethingElse(...)
 
-attributes #5 = { nounwind  "no-frame-pointer-elim-non-leaf" }
+attributes #5 = { nounwind "frame-pointer"="non-leaf" }
diff --git a/llvm/test/CodeGen/X86/x86-store-gv-addr.ll b/llvm/test/CodeGen/X86/x86-store-gv-addr.ll
index 089517aad..11da500d3 100644
--- a/llvm/test/CodeGen/X86/x86-store-gv-addr.ll
+++ b/llvm/test/CodeGen/X86/x86-store-gv-addr.ll
@@ -1,10 +1,21 @@
-; RUN: llc < %s -mtriple=x86_64-apple-darwin -relocation-model=static | not grep lea
-; RUN: llc < %s -mtriple=x86_64-pc-linux-gnu -relocation-model=static | not grep lea
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-apple-darwin -relocation-model=static | FileCheck %s --check-prefix=x86_64-darwin
+; RUN: llc < %s -mtriple=x86_64-pc-linux-gnu -relocation-model=static | FileCheck %s --check-prefix=x86_64-linux
 
 @v = external global i32, align 8
 @v_addr = external global i32*, align 8
 
 define void @t() nounwind optsize {
+; x86_64-darwin-LABEL: t:
+; x86_64-darwin:       ## %bb.0:
+; x86_64-darwin-NEXT:    movq _v@{{.*}}(%rip), %rax
+; x86_64-darwin-NEXT:    movq _v_addr@{{.*}}(%rip), %rcx
+; x86_64-darwin-NEXT:    movq %rax, (%rcx)
+; x86_64-darwin-NEXT:    ud2
+;
+; x86_64-linux-LABEL: t:
+; x86_64-linux:       # %bb.0:
+; x86_64-linux-NEXT:    movq $v, {{.*}}(%rip)
 	store i32* @v, i32** @v_addr, align 8
 	unreachable
 }
diff --git a/llvm/test/CodeGen/X86/xmulo.ll b/llvm/test/CodeGen/X86/xmulo.ll
index 86b15cce0..6f2c229ac 100644
--- a/llvm/test/CodeGen/X86/xmulo.ll
+++ b/llvm/test/CodeGen/X86/xmulo.ll
@@ -26,18 +26,14 @@ define {i64, i1} @t1() nounwind {
 define {i64, i1} @t2() nounwind {
 ; SDAG-LABEL: t2:
 ; SDAG:       ## %bb.0:
-; SDAG-NEXT:    xorl %ecx, %ecx
-; SDAG-NEXT:    movl $9, %eax
-; SDAG-NEXT:    mulq %rcx
-; SDAG-NEXT:    seto %dl
+; SDAG-NEXT:    xorl %eax, %eax
+; SDAG-NEXT:    xorl %edx, %edx
 ; SDAG-NEXT:    retq
 ;
 ; FAST-LABEL: t2:
 ; FAST:       ## %bb.0:
-; FAST-NEXT:    xorl %ecx, %ecx
-; FAST-NEXT:    movl $9, %eax
-; FAST-NEXT:    mulq %rcx
-; FAST-NEXT:    seto %dl
+; FAST-NEXT:    xorl %eax, %eax
+; FAST-NEXT:    xorl %edx, %edx
 ; FAST-NEXT:    retq
   %1 = call {i64, i1} @llvm.umul.with.overflow.i64(i64 9, i64 0)
   ret {i64, i1} %1
diff --git a/llvm/test/CodeGen/X86/xop-ifma.ll b/llvm/test/CodeGen/X86/xop-ifma.ll
index b8ec25ec1..df226933d 100644
--- a/llvm/test/CodeGen/X86/xop-ifma.ll
+++ b/llvm/test/CodeGen/X86/xop-ifma.ll
@@ -67,15 +67,13 @@ define <8 x i32> @test_mul_v8i32_add_v8i32(<8 x i32> %a0, <8 x i32> %a1, <8 x i3
 define <4 x i64> @test_mulx_v4i32_add_v4i64(<4 x i32> %a0, <4 x i32> %a1, <4 x i64> %a2) {
 ; XOP-AVX1-LABEL: test_mulx_v4i32_add_v4i64:
 ; XOP-AVX1:       # %bb.0:
-; XOP-AVX1-NEXT:    vpmovsxdq %xmm0, %xmm3
-; XOP-AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; XOP-AVX1-NEXT:    vpmovsxdq %xmm0, %xmm0
-; XOP-AVX1-NEXT:    vpmovsxdq %xmm1, %xmm4
-; XOP-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
-; XOP-AVX1-NEXT:    vpmovsxdq %xmm1, %xmm1
+; XOP-AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm1[0],zero,xmm1[1],zero
+; XOP-AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm4 = xmm0[0],zero,xmm0[1],zero
+; XOP-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,2,3,3]
+; XOP-AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]
 ; XOP-AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm5
 ; XOP-AVX1-NEXT:    vpmacsdql %xmm5, %xmm1, %xmm0, %xmm0
-; XOP-AVX1-NEXT:    vpmacsdql %xmm2, %xmm4, %xmm3, %xmm1
+; XOP-AVX1-NEXT:    vpmacsdql %xmm2, %xmm3, %xmm4, %xmm1
 ; XOP-AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; XOP-AVX1-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/xor.ll b/llvm/test/CodeGen/X86/xor.ll
index 654382f7b..e836b00bb 100644
--- a/llvm/test/CodeGen/X86/xor.ll
+++ b/llvm/test/CodeGen/X86/xor.ll
@@ -407,13 +407,10 @@ define i32 @PR17487(i1 %tobool) {
 ; X32-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; X32-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
 ; X32-NEXT:    pandn {{\.LCPI.*}}, %xmm0
-; X32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,1,2,3]
-; X32-NEXT:    movd %xmm1, %ecx
 ; X32-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; X32-NEXT:    movd %xmm0, %edx
-; X32-NEXT:    xorl $1, %edx
+; X32-NEXT:    movd %xmm0, %ecx
 ; X32-NEXT:    xorl %eax, %eax
-; X32-NEXT:    orl %ecx, %edx
+; X32-NEXT:    cmpl $1, %ecx
 ; X32-NEXT:    setne %al
 ; X32-NEXT:    retl
 ;
diff --git a/llvm/test/CodeGen/X86/xray-custom-log.ll b/llvm/test/CodeGen/X86/xray-custom-log.ll
index e60c76a4e..3a3d6a5df 100644
--- a/llvm/test/CodeGen/X86/xray-custom-log.ll
+++ b/llvm/test/CodeGen/X86/xray-custom-log.ll
@@ -12,8 +12,8 @@ define i32 @fn() nounwind noinline uwtable "function-instrument"="xray-always" {
     ; CHECK:       .byte 0xeb, 0x0f
     ; CHECK-NEXT:  pushq %rdi
     ; CHECK-NEXT:  pushq %rsi
-    ; CHECK-NEXT:  movq {{.*}}, %rdi
-    ; CHECK-NEXT:  movq {{.*}}, %rsi
+    ; CHECK-NEXT:  movq %rcx, %rdi
+    ; CHECK-NEXT:  movq %rax, %rsi
     ; CHECK-NEXT:  callq __xray_CustomEvent
     ; CHECK-NEXT:  popq %rsi
     ; CHECK-NEXT:  popq %rdi
@@ -22,8 +22,8 @@ define i32 @fn() nounwind noinline uwtable "function-instrument"="xray-always" {
     ; PIC:       .byte 0xeb, 0x0f
     ; PIC-NEXT:  pushq %rdi
     ; PIC-NEXT:  pushq %rsi
-    ; PIC-NEXT:  movq {{.*}}, %rdi
-    ; PIC-NEXT:  movq {{.*}}, %rsi
+    ; PIC-NEXT:  movq %rcx, %rdi
+    ; PIC-NEXT:  movq %rax, %rsi
     ; PIC-NEXT:  callq __xray_CustomEvent@PLT
     ; PIC-NEXT:  popq %rsi
     ; PIC-NEXT:  popq %rdi
diff --git a/llvm/test/CodeGen/X86/xray-selective-instrumentation-miss.ll b/llvm/test/CodeGen/X86/xray-selective-instrumentation-miss.ll
index 5b57e2541..758abb11e 100644
--- a/llvm/test/CodeGen/X86/xray-selective-instrumentation-miss.ll
+++ b/llvm/test/CodeGen/X86/xray-selective-instrumentation-miss.ll
@@ -1,9 +1,14 @@
-; RUN: llc -mcpu=nehalem < %s | not grep xray_sled_
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mcpu=nehalem < %s | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
 target triple = "x86_64-apple-darwin8"
 
 define i32 @foo() nounwind uwtable "xray-instruction-threshold"="3" {
+; CHECK-LABEL: foo:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    retq
 entry:
   ret i32 0
 }
diff --git a/llvm/test/CodeGen/X86/xray-selective-instrumentation.ll b/llvm/test/CodeGen/X86/xray-selective-instrumentation.ll
index 13f8578e7..8e3e0be41 100644
--- a/llvm/test/CodeGen/X86/xray-selective-instrumentation.ll
+++ b/llvm/test/CodeGen/X86/xray-selective-instrumentation.ll
@@ -1,9 +1,21 @@
-; RUN: llc -verify-machineinstrs -mcpu=nehalem < %s | grep xray_sled_
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -verify-machineinstrs -mcpu=nehalem | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
 target triple = "x86_64-apple-darwin8"
 
 define i32 @foo() nounwind uwtable "xray-instruction-threshold"="1" {
+; CHECK-LABEL: foo:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    .p2align 1, 0x90
+; CHECK-NEXT:  Lxray_sled_0:
+; CHECK-NEXT:    .ascii "\353\t"
+; CHECK-NEXT:    nopw 512(%rax,%rax)
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    .p2align 1, 0x90
+; CHECK-NEXT:  Lxray_sled_1:
+; CHECK-NEXT:    retq
+; CHECK-NEXT:    nopw %cs:512(%rax,%rax)
 entry:
   ret i32 0
 }
diff --git a/llvm/test/CodeGen/X86/xray-typed-event-log.ll b/llvm/test/CodeGen/X86/xray-typed-event-log.ll
index a69206ed7..0ed8ed7f6 100644
--- a/llvm/test/CodeGen/X86/xray-typed-event-log.ll
+++ b/llvm/test/CodeGen/X86/xray-typed-event-log.ll
@@ -16,9 +16,9 @@ define i32 @fn() nounwind noinline uwtable "function-instrument"="xray-always" {
     ; CHECK-NEXT:  pushq %rdi
     ; CHECK-NEXT:  pushq %rsi
     ; CHECK-NEXT:  pushq %rdx
-    ; CHECK-NEXT:  movq {{.*}}, %rdi
-    ; CHECK-NEXT:  movq {{.*}}, %rsi
-    ; CHECK-NEXT:  movq {{.*}}, %rdx
+    ; CHECK-NEXT:  movq %rdx, %rdi
+    ; CHECK-NEXT:  movq %rcx, %rsi
+    ; CHECK-NEXT:  movq %rax, %rdx
     ; CHECK-NEXT:  callq __xray_TypedEvent
     ; CHECK-NEXT:  popq %rdx
     ; CHECK-NEXT:  popq %rsi
@@ -29,9 +29,9 @@ define i32 @fn() nounwind noinline uwtable "function-instrument"="xray-always" {
     ; PIC-NEXT:  pushq %rdi
     ; PIC-NEXT:  pushq %rsi
     ; PIC-NEXT:  pushq %rdx
-    ; PIC-NEXT:  movq {{.*}}, %rdi
-    ; PIC-NEXT:  movq {{.*}}, %rsi
-    ; PIC-NEXT:  movq {{.*}}, %rdx
+    ; PIC-NEXT:  movq %rdx, %rdi
+    ; PIC-NEXT:  movq %rcx, %rsi
+    ; PIC-NEXT:  movq %rax, %rdx
     ; PIC-NEXT:  callq __xray_TypedEvent@PLT
     ; PIC-NEXT:  popq %rdx
     ; PIC-NEXT:  popq %rsi
diff --git a/llvm/test/CodeGen/X86/zext-inreg-1.ll b/llvm/test/CodeGen/X86/zext-inreg-1.ll
index 580269d14..3a6628fa0 100644
--- a/llvm/test/CodeGen/X86/zext-inreg-1.ll
+++ b/llvm/test/CodeGen/X86/zext-inreg-1.ll
@@ -1,4 +1,5 @@
-; RUN: llc < %s -mtriple=i686-- | not grep and
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- | FileCheck %s
 
 ; These tests differ from the ones in zext-inreg-0.ll in that
 ; on x86-64 they do require and instructions.
@@ -7,11 +8,27 @@
 ; This related to not having ZERO_EXTEND_REG node.
 
 define i64 @l(i64 %d) nounwind  {
+; CHECK-LABEL: l:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT:    addl $1, %eax
+; CHECK-NEXT:    adcl $0, %ecx
+; CHECK-NEXT:    movzbl %cl, %edx
+; CHECK-NEXT:    retl
         %e = add i64 %d, 1
         %retval = and i64 %e, 1099511627775
         ret i64 %retval
 }
 define i64 @m(i64 %d) nounwind  {
+; CHECK-LABEL: m:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT:    addl $1, %eax
+; CHECK-NEXT:    adcl $0, %ecx
+; CHECK-NEXT:    movzwl %cx, %edx
+; CHECK-NEXT:    retl
         %e = add i64 %d, 1
         %retval = and i64 %e, 281474976710655
         ret i64 %retval
diff --git a/llvm/test/DebugInfo/AArch64/call-site-info-output.ll b/llvm/test/DebugInfo/AArch64/call-site-info-output.ll
new file mode 100644
index 000000000..d52d6962f
--- /dev/null
+++ b/llvm/test/DebugInfo/AArch64/call-site-info-output.ll
@@ -0,0 +1,41 @@
+; RUN: llc -mtriple aarch64-linux-gnu -debug-entry-values %s -o - -stop-before=finalize-isel | FileCheck %s
+; Verify that Selection DAG knows how to recognize simple function parameter forwarding registers.
+; Produced from:
+; extern int fn1(int,int,int);
+; int fn2(int a, int b, int c) {
+;   int local = fn1(a+b, c, 10);
+;   if (local > 10)
+;     return local + 10;
+;   return local;
+; }
+; clang -g -O2 -target aarch64-linux-gnu -S -emit-llvm %s
+; CHECK: callSites:
+; CHECK-NEXT:   - { bb: {{.*}}, offset: {{.*}}, fwdArgRegs:
+; CHECK-NEXT:       - { arg: 0, reg: '$w0' }
+; CHECK-NEXT:       - { arg: 1, reg: '$w1' }
+; CHECK-NEXT:       - { arg: 2, reg: '$w2' } }
+
+; ModuleID = 'call-site-info-output.c'
+source_filename = "call-site-info-output.c"
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64-unknown-linux-gnu"
+
+; Function Attrs: nounwind
+define dso_local i32 @fn2(i32 %a, i32 %b, i32 %c) local_unnamed_addr{
+entry:
+  %add = add nsw i32 %b, %a
+  %call = tail call i32 @fn1(i32 %add, i32 %c, i32 10)
+  %cmp = icmp sgt i32 %call, 10
+  %add1 = add nsw i32 %call, 10
+  %retval.0 = select i1 %cmp, i32 %add1, i32 %call
+  ret i32 %retval.0
+}
+
+declare dso_local i32 @fn1(i32, i32, i32) local_unnamed_addr
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+!llvm.ident = !{!0}
+
+!0 = !{!"clang version 10.0.0"}
diff --git a/llvm/test/DebugInfo/AArch64/line-header.ll.x b/llvm/test/DebugInfo/AArch64/line-header.ll.x
index 1d9156deb..dbe190201 100644
--- a/llvm/test/DebugInfo/AArch64/line-header.ll.x
+++ b/llvm/test/DebugInfo/AArch64/line-header.ll.x
@@ -3,4 +3,4 @@
 
 ; check line table length is correctly calculated for both big and little endian
 CHECK-LABEL: .debug_line contents:
-CHECK: total_length: 0x0000003f
+CHECK: total_length: 0x00000040
diff --git a/llvm/test/DebugInfo/AArch64/machine-outliner.ll b/llvm/test/DebugInfo/AArch64/machine-outliner.ll
index d3b5beaf9..ac046afae 100644
--- a/llvm/test/DebugInfo/AArch64/machine-outliner.ll
+++ b/llvm/test/DebugInfo/AArch64/machine-outliner.ll
@@ -51,7 +51,7 @@ entry:
   ret void, !dbg !23
 }
 
-attributes #0 = { nounwind ssp uwtable "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" }
+attributes #0 = { nounwind ssp uwtable "frame-pointer"="all" }
 attributes #1 = { nounwind readnone speculatable }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/DebugInfo/ARM/PR16736.ll b/llvm/test/DebugInfo/ARM/PR16736.ll
index 7e4d56a26..0d9e219a3 100644
--- a/llvm/test/DebugInfo/ARM/PR16736.ll
+++ b/llvm/test/DebugInfo/ARM/PR16736.ll
@@ -31,7 +31,7 @@ target datalayout = "e-p:32:32:32-i1:8:32-i8:8:32-i16:16:32-i32:32:32-i64:64:64-
 target triple = "thumbv7-apple-ios"
 
 ; Function Attrs: nounwind
-define arm_aapcscc void @_Z1hiiiif(i32, i32, i32, i32, float %x) #0 "no-frame-pointer-elim"="true" !dbg !4 {
+define arm_aapcscc void @_Z1hiiiif(i32, i32, i32, i32, float %x) #0 "frame-pointer"="all" !dbg !4 {
 entry:
   tail call void @llvm.dbg.value(metadata i32 %0, metadata !12, metadata !DIExpression()), !dbg !18
   tail call void @llvm.dbg.value(metadata i32 %1, metadata !13, metadata !DIExpression()), !dbg !18
diff --git a/llvm/test/DebugInfo/ARM/PR26163.ll b/llvm/test/DebugInfo/ARM/PR26163.ll
index 5dac6ad83..0beb942b6 100644
--- a/llvm/test/DebugInfo/ARM/PR26163.ll
+++ b/llvm/test/DebugInfo/ARM/PR26163.ll
@@ -1,4 +1,4 @@
-; RUN: llc -filetype=obj -o - < %s | llvm-dwarfdump -v -debug-info - | FileCheck %s
+; RUN: llc -filetype=obj -o - < %s | llvm-dwarfdump -debug-info - | FileCheck %s
 ;
 ; Checks that we're omitting the first range, as it is empty, and that we're
 ; emitting one that spans the rest of the function. In this case, the first
@@ -10,7 +10,7 @@
 ;
 ; CHECK: DW_TAG_inlined_subroutine
 ; CHECK: DW_TAG_variable
-; CHECK:   DW_AT_location [DW_FORM_sec_offset] ({{.*}}
+; CHECK:   DW_AT_location ({{.*}}
 ; CHECK-NEXT: [0x00000004, 0x00000014): DW_OP_lit0, DW_OP_stack_value, DW_OP_piece 0x4)
 
 ; Created form the following test case (PR26163) with
diff --git a/llvm/test/DebugInfo/ARM/call-site-info-output.ll b/llvm/test/DebugInfo/ARM/call-site-info-output.ll
new file mode 100644
index 000000000..9255a7d57
--- /dev/null
+++ b/llvm/test/DebugInfo/ARM/call-site-info-output.ll
@@ -0,0 +1,41 @@
+; RUN: llc -mtriple arm-linux-gnu -debug-entry-values %s -o - -stop-before=finalize-isel | FileCheck %s
+; Verify that Selection DAG knows how to recognize simple function parameter forwarding registers.
+; Produced from:
+; extern int fn1(int,int,int);
+; int fn2(int a, int b, int c) {
+;   int local = fn1(a+b, c, 10);
+;   if (local > 10)
+;     return local + 10;
+;   return local;
+; }
+; clang -g -O2 -target arm-linux-gnu -S -emit-llvm %s
+; CHECK: callSites:
+; CHECK-NEXT:   - { bb: {{.*}}, offset: {{.*}}, fwdArgRegs:
+; CHECK-NEXT:       - { arg: 0, reg: '$r0' }
+; CHECK-NEXT:       - { arg: 1, reg: '$r1' }
+; CHECK-NEXT:       - { arg: 2, reg: '$r2' } }
+
+; ModuleID = 'call-site-info-output.c'
+source_filename = "call-site-info-output.c"
+target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
+target triple = "armv4t-unknown-linux-gnu"
+
+; Function Attrs: nounwind
+define dso_local arm_aapcscc i32 @fn2(i32 %a, i32 %b, i32 %c) {
+entry:
+  %add = add nsw i32 %b, %a
+  %call = tail call arm_aapcscc i32 @fn1(i32 %add, i32 %c, i32 10)
+  %cmp = icmp sgt i32 %call, 10
+  %add1 = select i1 %cmp, i32 %c, i32 0
+  %retval.0 = add nsw i32 %add1, %call
+  ret i32 %retval.0
+}
+
+declare dso_local arm_aapcscc i32 @fn1(i32, i32, i32) local_unnamed_addr
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+!llvm.ident = !{!0}
+
+!0 = !{!"clang version 10.0.0"}
diff --git a/llvm/test/DebugInfo/ARM/entry-value-multi-byte-expr.ll b/llvm/test/DebugInfo/ARM/entry-value-multi-byte-expr.ll
new file mode 100644
index 000000000..71cf41782
--- /dev/null
+++ b/llvm/test/DebugInfo/ARM/entry-value-multi-byte-expr.ll
@@ -0,0 +1,91 @@
+; RUN: llc -debug-entry-values -filetype=asm -o - %s | FileCheck %s
+
+; Verify that the size operands of the DW_OP_GNU_entry_value operations are
+; correct for the multi-byte DW_OP_regx expressions.
+
+; Based on the following C reproducer:
+;
+; extern void clobber();
+; double global;
+; int f(double a, double b) {
+;   global = a + b;
+;   clobber();
+;   return 1;
+; }
+
+; This test checks the assembly output rather than the output from
+; llvm-dwarfdump, as the latter printed the DW_OP_regx D0 correctly, even
+; though the entry value's size operand did not fully cover that:
+
+; DW_OP_GNU_entry_value(DW_OP_regx D0), DW_OP_stack_value
+;
+; whereas readelf interpreted it as an DW_OP_GNU_entry_value covering one byte,
+; resulting in garbage data:
+;
+; DW_OP_GNU_entry_value: (DW_OP_regx: 0 (r0)); DW_OP_breg16 (r16): 2; DW_OP_stack_value
+
+; CHECK:      .byte   243       @ DW_OP_GNU_entry_value
+; CHECK-NEXT: .byte   3         @ 3
+; CHECK-NEXT: .byte   144       @ DW_OP_regx
+; CHECK-NEXT: .byte   128       @ 256
+; CHECK-NEXT: .byte   2         @
+; CHECK-NEXT: .byte   159       @ DW_OP_stack_value
+
+; CHECK:      .byte   243       @ DW_OP_GNU_entry_value
+; CHECK-NEXT: .byte   3         @ 3
+; CHECK-NEXT: .byte   144       @ DW_OP_regx
+; CHECK-NEXT: .byte   129       @ 257
+; CHECK-NEXT: .byte   2         @
+; CHECK-NEXT: .byte   159       @ DW_OP_stack_value
+
+target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
+target triple = "armv7-unknown-unknown"
+
+@global = common global double 0.000000e+00, align 8, !dbg !0
+
+; Function Attrs: nounwind
+define arm_aapcs_vfpcc i32 @f(double %a, double %b) #0 !dbg !12 {
+entry:
+  call void @llvm.dbg.value(metadata double %a, metadata !17, metadata !DIExpression()), !dbg !19
+  call void @llvm.dbg.value(metadata double %b, metadata !18, metadata !DIExpression()), !dbg !19
+  %add = fadd double %a, %b, !dbg !20
+  store double %add, double* @global, align 8, !dbg !20
+  tail call arm_aapcs_vfpcc void @clobber(), !dbg !21
+  ret i32 1, !dbg !22
+}
+
+declare arm_aapcs_vfpcc void @clobber()
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.value(metadata, metadata, metadata) #1
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind readnone speculatable willreturn }
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!7, !8, !9, !10}
+!llvm.ident = !{!11}
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "global", scope: !2, file: !3, line: 4, type: !6, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 10.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5, nameTableKind: None)
+!3 = !DIFile(filename: "entry-value-multi-byte-expr.c", directory: "/")
+!4 = !{}
+!5 = !{!0}
+!6 = !DIBasicType(name: "double", size: 64, encoding: DW_ATE_float)
+!7 = !{i32 2, !"Dwarf Version", i32 4}
+!8 = !{i32 2, !"Debug Info Version", i32 3}
+!9 = !{i32 1, !"wchar_size", i32 4}
+!10 = !{i32 1, !"min_enum_size", i32 4}
+!11 = !{!"clang version 10.0.0"}
+!12 = distinct !DISubprogram(name: "f", scope: !3, file: !3, line: 6, type: !13, scopeLine: 6, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !2, retainedNodes: !16)
+!13 = !DISubroutineType(types: !14)
+!14 = !{!15, !6, !6}
+!15 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!16 = !{!17, !18}
+!17 = !DILocalVariable(name: "a", arg: 1, scope: !12, file: !3, line: 6, type: !6)
+!18 = !DILocalVariable(name: "b", arg: 2, scope: !12, file: !3, line: 6, type: !6)
+!19 = !DILocation(line: 0, scope: !12)
+!20 = !DILocation(line: 7, scope: !12)
+!21 = !DILocation(line: 8, scope: !12)
+!22 = !DILocation(line: 9, scope: !12)
diff --git a/llvm/test/DebugInfo/ARM/lowerbdgdeclare_vla.ll b/llvm/test/DebugInfo/ARM/lowerbdgdeclare_vla.ll
index 6ff696698..dd3cf9b4a 100644
--- a/llvm/test/DebugInfo/ARM/lowerbdgdeclare_vla.ll
+++ b/llvm/test/DebugInfo/ARM/lowerbdgdeclare_vla.ll
@@ -60,7 +60,7 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.value(metadata, metadata, metadata) #1
 
-attributes #0 = { nounwind optsize readnone "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind optsize readnone "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/DebugInfo/ARM/multiple-constant-uses-drops-dbgloc.ll b/llvm/test/DebugInfo/ARM/multiple-constant-uses-drops-dbgloc.ll
index a8e31b054..472810bee 100644
--- a/llvm/test/DebugInfo/ARM/multiple-constant-uses-drops-dbgloc.ll
+++ b/llvm/test/DebugInfo/ARM/multiple-constant-uses-drops-dbgloc.ll
@@ -27,7 +27,7 @@ entry:
   ret void, !dbg !19
 }
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="cortex-a8" "target-features"="+neon,+vfp3" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="cortex-a8" "target-features"="+neon,+vfp3" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!2}
 !llvm.module.flags = !{!8, !9, !10, !11}
diff --git a/llvm/test/DebugInfo/COFF/array-odr-violation.ll b/llvm/test/DebugInfo/COFF/array-odr-violation.ll
index ec83a4519..dc97b62b4 100644
--- a/llvm/test/DebugInfo/COFF/array-odr-violation.ll
+++ b/llvm/test/DebugInfo/COFF/array-odr-violation.ll
@@ -17,27 +17,24 @@
 ; FIXME: sizeof(a) in the user program is 1, but we claim it is 4 because
 ; sometimes the frontend lies to us. See array-types-advanced.ll for an example.
 ;
-; CHECK: Array ({{.*}}) {
-; CHECK:   TypeLeafKind: LF_ARRAY (0x1503)
-; CHECK:   ElementType: YYSTYPE ({{.*}})
-; CHECK:   IndexType: unsigned __int64 (0x23)
-; CHECK:   SizeOf: 4
-; CHECK:   Name:
-; CHECK: }
+; CHECK:	# Array (0x1004)
+; CHECK:	.short	0xe                     # Record length
+; CHECK:	.short	0x1503                  # Record kind: LF_ARRAY
+; CHECK:	.long	0x1003                  # ElementType: YYSTYPE
+; CHECK:	.long	0x23                    # IndexType: unsigned __int64
+; CHECK:	.short	0x4                     # SizeOf
+; CHECK:	.byte	0                       # Name
+; CHECK:	.byte	241
 
-; sizeof(YYSTYPE) == 4
-; CHECK: Union ({{.*}}) {
-; CHECK:   TypeLeafKind: LF_UNION (0x1506)
-; CHECK:   MemberCount: 1
-; CHECK:   Properties [ (0x600)
-; CHECK:     HasUniqueName (0x200)
-; CHECK:     Sealed (0x400)
-; CHECK:   ]
-; CHECK:   FieldList: <field list>
-; CHECK:   SizeOf: 4
-; CHECK:   Name: YYSTYPE
-; CHECK:   LinkageName: .?ATYYSTYPE@@
-; CHECK: }
+; CHECK:	# Union (0x1006)
+; CHECK:	.short	0x22                    # Record length
+; CHECK:	.short	0x1506                  # Record kind: LF_UNION
+; CHECK:	.short	0x1                     # MemberCount
+; CHECK:	.short	0x600                   # Properties ( HasUniqueName (0x200) | Sealed (0x400) )
+; CHECK:	.long	0x1005                  # FieldList: <field list>
+; CHECK:	.short	0x4                     # SizeOf
+; CHECK:	.asciz	"YYSTYPE"               # Name
+; CHECK:	.asciz	".?ATYYSTYPE@@"         # LinkageName
 
 ; ModuleID = 'llvm-link'
 source_filename = "llvm-link"
@@ -60,7 +57,7 @@ entry:
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
 
-attributes #0 = { noinline nounwind sspstrong uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noinline nounwind sspstrong uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!2, !11}
diff --git a/llvm/test/DebugInfo/COFF/asan-module-ctor.ll b/llvm/test/DebugInfo/COFF/asan-module-ctor.ll
index 83b27305f..d1879bf98 100644
--- a/llvm/test/DebugInfo/COFF/asan-module-ctor.ll
+++ b/llvm/test/DebugInfo/COFF/asan-module-ctor.ll
@@ -76,7 +76,7 @@ declare void @__asan_register_globals(i32, i32)
 
 declare void @__asan_unregister_globals(i32, i32)
 
-attributes #0 = { nounwind sanitize_address "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind sanitize_address "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!7, !8}
diff --git a/llvm/test/DebugInfo/COFF/asm.ll b/llvm/test/DebugInfo/COFF/asm.ll
index 8790dddd5..c259a53f6 100644
--- a/llvm/test/DebugInfo/COFF/asm.ll
+++ b/llvm/test/DebugInfo/COFF/asm.ll
@@ -143,8 +143,8 @@ entry:
 
 declare void @g() #1
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/DebugInfo/COFF/class-options-common.ll b/llvm/test/DebugInfo/COFF/class-options-common.ll
index 46350552e..da2941548 100644
--- a/llvm/test/DebugInfo/COFF/class-options-common.ll
+++ b/llvm/test/DebugInfo/COFF/class-options-common.ll
@@ -1,5 +1,6 @@
 ; RUN: llc < %s -filetype=obj | llvm-readobj - --codeview | FileCheck %s
 ; RUN: llc < %s | llvm-mc -filetype=obj --triple=x86_64-windows | llvm-readobj - --codeview | FileCheck %s
+; RUN: llc < %s | FileCheck %s --check-prefix=ASM-INLINE-COMMENTS
 ;
 ; Command to generate function-options.ll
 ; $ clang++ class-options-common.cpp -S -emit-llvm -g -gcodeview -o class-options-common.ll
@@ -651,6 +652,17 @@
 ; CHECK:   }
 ; CHECK: ]
 
+; ASM-INLINE-COMMENTS: # MethodOverloadList (0x1088)
+; ASM-INLINE-COMMENTS: .short	0x12                    # Record length
+; ASM-INLINE-COMMENTS: .short	0x1206                  # Record kind: LF_METHODLIST
+; ASM-INLINE-COMMENTS: .short	0x3                     # Method
+; ASM-INLINE-COMMENTS:                                       # Attrs: Public
+; ASM-INLINE-COMMENTS: .short	0x0
+; ASM-INLINE-COMMENTS: .long	0x1083                  # Type: void Foo::()
+; ASM-INLINE-COMMENTS: .short	0x3                     # Attrs: Public
+; ASM-INLINE-COMMENTS: .short	0x0
+; ASM-INLINE-COMMENTS: .long	0x1087                  # Type: void Foo::(const Foo&)
+
 
 ; ModuleID = 'class-options-common.cpp'
 source_filename = "class-options.cpp"
@@ -830,7 +842,7 @@ entry:
   ret void, !dbg !173
 }
 
-attributes #0 = { noinline nounwind optnone uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noinline nounwind optnone uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone speculatable }
 
 !llvm.dbg.cu = !{!2}
diff --git a/llvm/test/DebugInfo/COFF/comdat.ll b/llvm/test/DebugInfo/COFF/comdat.ll
index 3536f67cc..3f9bb48ae 100644
--- a/llvm/test/DebugInfo/COFF/comdat.ll
+++ b/llvm/test/DebugInfo/COFF/comdat.ll
@@ -129,11 +129,11 @@ declare i32 @__C_specific_handler(...)
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.value(metadata, metadata, metadata) #5
 
-attributes #0 = { norecurse nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #2 = { inlinehint noinline nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #3 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #4 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { norecurse nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { inlinehint noinline nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #4 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #5 = { nounwind readnone }
 attributes #6 = { noinline }
 attributes #7 = { nounwind }
diff --git a/llvm/test/DebugInfo/COFF/cpp-mangling.ll b/llvm/test/DebugInfo/COFF/cpp-mangling.ll
index 1bbb5bd17..c48dae30a 100644
--- a/llvm/test/DebugInfo/COFF/cpp-mangling.ll
+++ b/llvm/test/DebugInfo/COFF/cpp-mangling.ll
@@ -49,7 +49,7 @@ entry:
   ret void, !dbg !24
 }
 
-attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/DebugInfo/COFF/defer-complete-type.ll b/llvm/test/DebugInfo/COFF/defer-complete-type.ll
index 8b4e57afa..67b4f2844 100644
--- a/llvm/test/DebugInfo/COFF/defer-complete-type.ll
+++ b/llvm/test/DebugInfo/COFF/defer-complete-type.ll
@@ -174,7 +174,7 @@ entry:
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
 
-attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/DebugInfo/COFF/enum-co.ll b/llvm/test/DebugInfo/COFF/enum-co.ll
index bdd4d0f70..c5366ef1f 100644
--- a/llvm/test/DebugInfo/COFF/enum-co.ll
+++ b/llvm/test/DebugInfo/COFF/enum-co.ll
@@ -124,7 +124,7 @@ entry:
   ret void, !dbg !46
 }
 
-attributes #0 = { noinline nounwind optnone uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noinline nounwind optnone uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone speculatable }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/DebugInfo/COFF/fp-stack.ll b/llvm/test/DebugInfo/COFF/fp-stack.ll
index e9b031d66..7587b8984 100644
--- a/llvm/test/DebugInfo/COFF/fp-stack.ll
+++ b/llvm/test/DebugInfo/COFF/fp-stack.ll
@@ -10,7 +10,7 @@ entry:
   ret double %sub
 }
 
-; ASM:         .cv_def_range    Ltmp1 Lfunc_end0, "A\021\200\000\000\000"
+; ASM:         .cv_def_range    Ltmp1 Lfunc_end0, reg, 128
 ; OBJ:    DefRangeRegisterSym {
 ; OBJ:      Register: ST0 (0x80)
 ; OBJ:      MayHaveNoName: 0
diff --git a/llvm/test/DebugInfo/COFF/fpo-argsize.ll b/llvm/test/DebugInfo/COFF/fpo-argsize.ll
index 41d00c18b..eb2f87601 100644
--- a/llvm/test/DebugInfo/COFF/fpo-argsize.ll
+++ b/llvm/test/DebugInfo/COFF/fpo-argsize.ll
@@ -285,7 +285,7 @@ entry:
   ret void, !dbg !159
 }
 
-attributes #0 = { noinline nounwind optnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noinline nounwind optnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone speculatable }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/DebugInfo/COFF/fpo-csrs.ll b/llvm/test/DebugInfo/COFF/fpo-csrs.ll
index 50b2e3be6..926adab35 100644
--- a/llvm/test/DebugInfo/COFF/fpo-csrs.ll
+++ b/llvm/test/DebugInfo/COFF/fpo-csrs.ll
@@ -581,8 +581,8 @@ entry:
 ; Function Attrs: nounwind readnone speculatable
 declare void @llvm.dbg.value(metadata, metadata, metadata) #2
 
-attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind readnone speculatable }
 attributes #3 = { nounwind }
 
diff --git a/llvm/test/DebugInfo/COFF/fpo-funclet.ll b/llvm/test/DebugInfo/COFF/fpo-funclet.ll
index 0e8c03afb..ea28ddcb4 100644
--- a/llvm/test/DebugInfo/COFF/fpo-funclet.ll
+++ b/llvm/test/DebugInfo/COFF/fpo-funclet.ll
@@ -58,8 +58,8 @@ declare void @"\01?g@@YAXXZ"() local_unnamed_addr #1
 
 declare i32 @__CxxFrameHandler3(...)
 
-attributes #0 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!3, !4, !5, !6}
diff --git a/llvm/test/DebugInfo/COFF/fpo-realign-alloca.ll b/llvm/test/DebugInfo/COFF/fpo-realign-alloca.ll
index 6046e2f52..4c4ad7abd 100644
--- a/llvm/test/DebugInfo/COFF/fpo-realign-alloca.ll
+++ b/llvm/test/DebugInfo/COFF/fpo-realign-alloca.ll
@@ -67,9 +67,9 @@ declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture) #1
 ; Function Attrs: nounwind readnone speculatable
 declare void @llvm.dbg.value(metadata, metadata, metadata) #3
 
-attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { argmemonly nounwind }
-attributes #2 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #3 = { nounwind readnone speculatable }
 attributes #4 = { nounwind }
 
diff --git a/llvm/test/DebugInfo/COFF/fpo-realign-vframe.ll b/llvm/test/DebugInfo/COFF/fpo-realign-vframe.ll
index f0be24d21..87e80a69e 100644
--- a/llvm/test/DebugInfo/COFF/fpo-realign-vframe.ll
+++ b/llvm/test/DebugInfo/COFF/fpo-realign-vframe.ll
@@ -189,10 +189,10 @@ declare dso_local void @usevals(i32*, i32*, double*) local_unnamed_addr #3
 ; Function Attrs: argmemonly nounwind
 declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture) #2
 
-attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone speculatable }
 attributes #2 = { argmemonly nounwind }
-attributes #3 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #4 = { nounwind }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/DebugInfo/COFF/fpo-shrink-wrap.ll b/llvm/test/DebugInfo/COFF/fpo-shrink-wrap.ll
index 63063261b..d42dcf0dd 100644
--- a/llvm/test/DebugInfo/COFF/fpo-shrink-wrap.ll
+++ b/llvm/test/DebugInfo/COFF/fpo-shrink-wrap.ll
@@ -126,8 +126,8 @@ declare i32 @doSomething(i32*) local_unnamed_addr #1
 ; Function Attrs: nounwind readnone speculatable
 declare void @llvm.dbg.value(metadata, metadata, metadata) #2
 
-attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind readnone speculatable }
 attributes #3 = { nounwind }
 
diff --git a/llvm/test/DebugInfo/COFF/fpo-stack-protect.ll b/llvm/test/DebugInfo/COFF/fpo-stack-protect.ll
index c9ea2c695..26fe7c49e 100644
--- a/llvm/test/DebugInfo/COFF/fpo-stack-protect.ll
+++ b/llvm/test/DebugInfo/COFF/fpo-stack-protect.ll
@@ -73,10 +73,10 @@ declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture) #2
 ; Function Attrs: nounwind readnone speculatable
 declare void @llvm.dbg.value(metadata, metadata, metadata) #1
 
-attributes #0 = { nounwind sspstrong "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind sspstrong "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone speculatable }
 attributes #2 = { argmemonly nounwind }
-attributes #3 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #4 = { nounwind }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/DebugInfo/COFF/frameproc-flags.ll b/llvm/test/DebugInfo/COFF/frameproc-flags.ll
index 12f4294be..567000c24 100644
--- a/llvm/test/DebugInfo/COFF/frameproc-flags.ll
+++ b/llvm/test/DebugInfo/COFF/frameproc-flags.ll
@@ -269,18 +269,18 @@ declare void @llvm.memset.p0i8.i32(i8* nocapture writeonly, i8, i32, i1) #2
 ; Function Attrs: nounwind readnone speculatable
 declare void @llvm.dbg.value(metadata, metadata, metadata) #1
 
-attributes #0 = { sspstrong "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { sspstrong "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone speculatable }
 attributes #2 = { argmemonly nounwind }
-attributes #3 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #4 = { returns_twice }
-attributes #5 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #6 = { norecurse nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #7 = { inlinehint nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #8 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #5 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #6 = { norecurse nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #7 = { inlinehint nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #8 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #9 = { nounwind readnone }
 attributes #10 = { nounwind }
-attributes #11 = { naked noinline nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #11 = { naked noinline nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #12 = { noinline }
 
 !llvm.dbg.cu = !{!2}
diff --git a/llvm/test/DebugInfo/COFF/function-options.ll b/llvm/test/DebugInfo/COFF/function-options.ll
index 76831e7d4..ee312cfb1 100644
--- a/llvm/test/DebugInfo/COFF/function-options.ll
+++ b/llvm/test/DebugInfo/COFF/function-options.ll
@@ -467,7 +467,7 @@ entry:
   ret void, !dbg !108
 }
 
-attributes #0 = { noinline nounwind optnone uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noinline nounwind optnone uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone speculatable }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/DebugInfo/COFF/global-constants.ll b/llvm/test/DebugInfo/COFF/global-constants.ll
index e4b4cdaac..7f68465da 100644
--- a/llvm/test/DebugInfo/COFF/global-constants.ll
+++ b/llvm/test/DebugInfo/COFF/global-constants.ll
@@ -101,9 +101,9 @@ entry:
 ; Function Attrs: nounwind
 declare void @llvm.debugtrap() #3
 
-attributes #0 = { noinline nounwind optnone uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noinline nounwind optnone uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone speculatable }
-attributes #2 = { noinline norecurse nounwind optnone uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { noinline norecurse nounwind optnone uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #3 = { nounwind }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/DebugInfo/COFF/global-type-hashes.ll b/llvm/test/DebugInfo/COFF/global-type-hashes.ll
index 77e69229e..70f9df156 100644
--- a/llvm/test/DebugInfo/COFF/global-type-hashes.ll
+++ b/llvm/test/DebugInfo/COFF/global-type-hashes.ll
@@ -90,9 +90,9 @@ entry:
   ret i32 %add, !dbg !43
 }
 
-attributes #0 = { noinline norecurse nounwind optnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-features"="+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noinline norecurse nounwind optnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-features"="+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone speculatable }
-attributes #2 = { noinline nounwind optnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-features"="+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { noinline nounwind optnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-features"="+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!3, !4, !5, !6, !100}
diff --git a/llvm/test/DebugInfo/COFF/global_visibility.ll b/llvm/test/DebugInfo/COFF/global_visibility.ll
index e4baf13ec..bdf603580 100644
--- a/llvm/test/DebugInfo/COFF/global_visibility.ll
+++ b/llvm/test/DebugInfo/COFF/global_visibility.ll
@@ -205,8 +205,8 @@ entry:
 ; Function Attrs: nounwind readnone speculatable
 declare void @llvm.dbg.declare(metadata, metadata, metadata) #2
 
-attributes #0 = { noinline optnone uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { noinline nounwind optnone uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noinline optnone uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { noinline nounwind optnone uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind readnone speculatable }
 
 !llvm.dbg.cu = !{!2}
diff --git a/llvm/test/DebugInfo/COFF/inheritance.ll b/llvm/test/DebugInfo/COFF/inheritance.ll
index b2f487a8f..ca1a5c218 100644
--- a/llvm/test/DebugInfo/COFF/inheritance.ll
+++ b/llvm/test/DebugInfo/COFF/inheritance.ll
@@ -123,8 +123,8 @@ entry:
 ; Function Attrs: nounwind readnone speculatable
 declare void @llvm.dbg.value(metadata, metadata, metadata) #2
 
-attributes #0 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind readnone speculatable }
 
 !llvm.dbg.cu = !{!2}
diff --git a/llvm/test/DebugInfo/COFF/inlining-files.ll b/llvm/test/DebugInfo/COFF/inlining-files.ll
index e167ce43c..ce92d20e0 100644
--- a/llvm/test/DebugInfo/COFF/inlining-files.ll
+++ b/llvm/test/DebugInfo/COFF/inlining-files.ll
@@ -71,7 +71,7 @@ entry:
   ret void, !dbg !30
 }
 
-attributes #0 = { norecurse nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { norecurse nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!2}
 !llvm.module.flags = !{!8, !9, !10}
diff --git a/llvm/test/DebugInfo/COFF/inlining-header.ll b/llvm/test/DebugInfo/COFF/inlining-header.ll
index 811414a36..de8ea595b 100644
--- a/llvm/test/DebugInfo/COFF/inlining-header.ll
+++ b/llvm/test/DebugInfo/COFF/inlining-header.ll
@@ -129,7 +129,7 @@ entry:
   ret i32 %5, !dbg !32
 }
 
-attributes #0 = { norecurse nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { norecurse nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "stack-protector-buffer-size"="8" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!2}
 !llvm.module.flags = !{!8, !9, !10}
diff --git a/llvm/test/DebugInfo/COFF/inlining-levels.ll b/llvm/test/DebugInfo/COFF/inlining-levels.ll
index 5a8a28515..c1613db87 100644
--- a/llvm/test/DebugInfo/COFF/inlining-levels.ll
+++ b/llvm/test/DebugInfo/COFF/inlining-levels.ll
@@ -53,7 +53,7 @@ entry:
   ret i32 0, !dbg !28
 }
 
-attributes #0 = { norecurse nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { norecurse nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!2}
 !llvm.module.flags = !{!8, !9, !10}
diff --git a/llvm/test/DebugInfo/COFF/inlining-padding.ll b/llvm/test/DebugInfo/COFF/inlining-padding.ll
index 404ff6df4..4437cbef9 100644
--- a/llvm/test/DebugInfo/COFF/inlining-padding.ll
+++ b/llvm/test/DebugInfo/COFF/inlining-padding.ll
@@ -64,7 +64,7 @@ entry:
   ret i32 0, !dbg !29
 }
 
-attributes #0 = { norecurse nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-features"="+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { norecurse nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-features"="+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!3, !4}
diff --git a/llvm/test/DebugInfo/COFF/inlining.ll b/llvm/test/DebugInfo/COFF/inlining.ll
index 981ae5aad..c1b6e69f2 100644
--- a/llvm/test/DebugInfo/COFF/inlining.ll
+++ b/llvm/test/DebugInfo/COFF/inlining.ll
@@ -75,33 +75,30 @@
 
 ; ASM: .section .debug$T,"dr"
 ; ASM: .long 4 # Debug section magic
-; ASM: # ArgList (0x1000) {
-; ASM: #   TypeLeafKind: LF_ARGLIST (0x1201)
-; ASM: #   NumArgs: 0
-; ASM: #   Arguments [
-; ASM: #   ]
-; ASM: # }
-; ASM: # Procedure (0x1001) {
-; ASM: #   TypeLeafKind: LF_PROCEDURE (0x1008)
-; ASM: #   ReturnType: void (0x3)
-; ASM: #   CallingConvention: NearC (0x0)
-; ASM: #   FunctionOptions [ (0x0)
-; ASM: #   ]
-; ASM: #   NumParameters: 0
-; ASM: #   ArgListType: () (0x1000)
-; ASM: # }
-; ASM: # FuncId (0x1002) {
-; ASM: #   TypeLeafKind: LF_FUNC_ID (0x1601)
-; ASM: #   ParentScope: 0x0
-; ASM: #   FunctionType: void () (0x1001)
-; ASM: #   Name: bar
-; ASM: # }
-; ASM: # FuncId (0x1003) {
-; ASM: #   TypeLeafKind: LF_FUNC_ID (0x1601)
-; ASM: #   ParentScope: 0x0
-; ASM: #   FunctionType: void () (0x1001)
-; ASM: #   Name: foo
-; ASM: # }
+; ASM: # ArgList (0x1000)
+; ASM: .short	0x6                     # Record length
+; ASM: .short	0x1201                  # Record kind: LF_ARGLIST
+; ASM: .long	0x0                     # NumArgs
+; ASM: # Procedure (0x1001)
+; ASM: .short	0xe                     # Record length
+; ASM: .short	0x1008                  # Record kind: LF_PROCEDURE
+; ASM: .long	0x3                     # ReturnType: void
+; ASM: .byte	0x0                     # CallingConvention: NearC
+; ASM: .byte	0x0                     # FunctionOptions
+; ASM: .short	0x0                     # NumParameters
+; ASM: .long	0x1000                  # ArgListType: ()
+; ASM: # FuncId (0x1002)
+; ASM: .short	0xe                     # Record length
+; ASM: .short	0x1601                  # Record kind: LF_FUNC_ID
+; ASM: .long	0x0                     # ParentScope
+; ASM: .long	0x1001                  # FunctionType: void ()
+; ASM: .asciz	"bar"                   # Name
+; ASM: # FuncId (0x1003)
+; ASM: .short	0xe                     # Record length
+; ASM: .short	0x1601                  # Record kind: LF_FUNC_ID
+; ASM: .long	0x0                     # ParentScope
+; ASM: .long	0x1001                  # FunctionType: void ()
+; ASM: .asciz	"foo"                   # Name
 
 ; We should only the LF_FUNC_ID records that we needed to reference.
 ; OBJ: CodeViewTypes [
@@ -245,7 +242,7 @@ declare void @llvm.lifetime.start(i64, i8* nocapture) #1
 ; Function Attrs: argmemonly nounwind
 declare void @llvm.lifetime.end(i64, i8* nocapture) #1
 
-attributes #0 = { norecurse nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { norecurse nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { argmemonly nounwind }
 attributes #2 = { nounwind }
 
diff --git a/llvm/test/DebugInfo/COFF/lambda.ll b/llvm/test/DebugInfo/COFF/lambda.ll
index 45a374bbb..611c651d9 100644
--- a/llvm/test/DebugInfo/COFF/lambda.ll
+++ b/llvm/test/DebugInfo/COFF/lambda.ll
@@ -104,9 +104,9 @@ entry:
   ret i32 %cond, !dbg !32
 }
 
-attributes #0 = { noinline norecurse optnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noinline norecurse optnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone speculatable }
-attributes #2 = { noinline nounwind optnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { noinline nounwind optnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!3, !4, !5, !6}
diff --git a/llvm/test/DebugInfo/COFF/lexicalblock.ll b/llvm/test/DebugInfo/COFF/lexicalblock.ll
index c7102d004..4f6aa6569 100644
--- a/llvm/test/DebugInfo/COFF/lexicalblock.ll
+++ b/llvm/test/DebugInfo/COFF/lexicalblock.ll
@@ -215,7 +215,7 @@ declare i32 @llvm.expect.i32(i32, i32) #3
 ; Function Attrs: argmemonly nounwind
 declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture) #2
 
-attributes #0 = { norecurse nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-features"="+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { norecurse nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-features"="+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone speculatable }
 attributes #2 = { argmemonly nounwind }
 attributes #3 = { nounwind readnone }
diff --git a/llvm/test/DebugInfo/COFF/line-zero.ll b/llvm/test/DebugInfo/COFF/line-zero.ll
new file mode 100644
index 000000000..3d3434564
--- /dev/null
+++ b/llvm/test/DebugInfo/COFF/line-zero.ll
@@ -0,0 +1,77 @@
+; RUN: llc < %s | FileCheck %s
+
+; C++ source to regenerate:
+; int main() {
+;   volatile int x;
+;   x = 1;
+; #line 0
+;   x = 2;
+; #line 7
+;   x = 3;
+; }
+
+
+; CHECK-LABEL: main:                                   # @main
+; CHECK:         .cv_loc 0 1 1 0                 # t.cpp:1:0
+; CHECK:         .cv_loc 0 1 3 0                 # t.cpp:3:0
+; CHECK:         movl    $1, 4(%rsp)
+; CHECK-NOT: .cv_loc {{.*}} t.cpp:0:0
+; CHECK:         movl    $2, 4(%rsp)
+; CHECK:         .cv_loc 0 1 7 0                 # t.cpp:7:0
+; CHECK:         movl    $3, 4(%rsp)
+; CHECK:         .cv_loc 0 1 8 0                 # t.cpp:8:0
+; CHECK:         xorl    %eax, %eax
+; CHECK:         retq
+
+; ModuleID = 't.cpp'
+source_filename = "t.cpp"
+target datalayout = "e-m:w-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc19.22.27905"
+
+; Function Attrs: norecurse nounwind uwtable
+define dso_local i32 @main() local_unnamed_addr #0 !dbg !8 {
+entry:
+  %x = alloca i32, align 4
+  %x.0.x.0..sroa_cast = bitcast i32* %x to i8*, !dbg !15
+  call void @llvm.dbg.declare(metadata i32* %x, metadata !13, metadata !DIExpression()), !dbg !15
+  store volatile i32 1, i32* %x, align 4, !dbg !16, !tbaa !17
+  store volatile i32 2, i32* %x, align 4, !dbg !21, !tbaa !17
+  store volatile i32 3, i32* %x, align 4, !dbg !22, !tbaa !17
+  ret i32 0, !dbg !23
+}
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #2
+
+attributes #0 = { norecurse nounwind uwtable }
+attributes #1 = { argmemonly nounwind willreturn }
+attributes #2 = { nounwind readnone speculatable willreturn }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5, !6}
+!llvm.ident = !{!7}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "t.cpp", directory: "C:\5Csrc\5Cllvm-project\5Cbuild", checksumkind: CSK_MD5, checksum: "8b6d53b166e6fa660f115eff7beedf3b")
+!2 = !{}
+!3 = !{i32 2, !"CodeView", i32 1}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 2}
+!6 = !{i32 7, !"PIC Level", i32 2}
+!7 = !{!"clang version 10.0.0"}
+!8 = distinct !DISubprogram(name: "main", scope: !1, file: !1, line: 1, type: !9, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !12)
+!9 = !DISubroutineType(types: !10)
+!10 = !{!11}
+!11 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!12 = !{!13}
+!13 = !DILocalVariable(name: "x", scope: !8, file: !1, line: 2, type: !14)
+!14 = !DIDerivedType(tag: DW_TAG_volatile_type, baseType: !11)
+!15 = !DILocation(line: 2, scope: !8)
+!16 = !DILocation(line: 3, scope: !8)
+!17 = !{!18, !18, i64 0}
+!18 = !{!"int", !19, i64 0}
+!19 = !{!"omnipotent char", !20, i64 0}
+!20 = !{!"Simple C++ TBAA"}
+!21 = !DILocation(line: 0, scope: !8)
+!22 = !DILocation(line: 7, scope: !8)
+!23 = !DILocation(line: 8, scope: !8)
diff --git a/llvm/test/DebugInfo/COFF/lines-difile.ll b/llvm/test/DebugInfo/COFF/lines-difile.ll
index ab2dec4c6..56be6b3d0 100644
--- a/llvm/test/DebugInfo/COFF/lines-difile.ll
+++ b/llvm/test/DebugInfo/COFF/lines-difile.ll
@@ -68,9 +68,9 @@ entry:
   ret void
 }
 
-attributes #0 = { noinline uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #2 = { noinline nounwind optnone uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noinline uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { noinline nounwind optnone uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!2}
 !llvm.module.flags = !{!13, !14, !15, !16}
diff --git a/llvm/test/DebugInfo/COFF/local-constant.ll b/llvm/test/DebugInfo/COFF/local-constant.ll
index 57759c904..e5809ec82 100644
--- a/llvm/test/DebugInfo/COFF/local-constant.ll
+++ b/llvm/test/DebugInfo/COFF/local-constant.ll
@@ -43,8 +43,8 @@ declare void @"\01?useint@@YAXH@Z"(i32) #1
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.value(metadata, metadata, metadata) #2
 
-attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind readnone }
 attributes #3 = { nounwind }
 
diff --git a/llvm/test/DebugInfo/COFF/local-variable-gap.ll b/llvm/test/DebugInfo/COFF/local-variable-gap.ll
index b7c69a65c..0b7f4ec5f 100644
--- a/llvm/test/DebugInfo/COFF/local-variable-gap.ll
+++ b/llvm/test/DebugInfo/COFF/local-variable-gap.ll
@@ -54,7 +54,7 @@
 ; ASM: [[p_b2:\.Ltmp[0-9]+]]:
 ; ASM:         #DEBUG_VALUE: p <- $esi
 ; ASM:         callq   call_noreturn
-; ASM:         ud2
+; ASM:         int3
 ; ASM: .Lfunc_end0:
 
 ; ASM:         .short  {{.*}}         # Record length
@@ -62,7 +62,7 @@
 ; ASM:         .long   116                     # TypeIndex
 ; ASM:         .short  0                       # Flags
 ; ASM:         .asciz  "p"
-; ASM:         .cv_def_range    [[p_b1]] [[p_e1]] [[p_b2]] .Lfunc_end0, "A\021\027\000\000\000"
+; ASM:         .cv_def_range    [[p_b1]] [[p_e1]] [[p_b2]] .Lfunc_end0, reg, 23
 ; ASM:         .short  2                       # Record length
 ; ASM:         .short  4431                    # Record kind: S_PROC_ID_END
 
@@ -133,9 +133,9 @@ declare void @use(i32) local_unnamed_addr #1
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.value(metadata, metadata, metadata) #3
 
-attributes #0 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #2 = { noreturn "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { noreturn "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #3 = { nounwind readnone }
 attributes #4 = { nounwind }
 attributes #5 = { noreturn nounwind }
diff --git a/llvm/test/DebugInfo/COFF/local-variables.ll b/llvm/test/DebugInfo/COFF/local-variables.ll
index 5b00383e5..abec75a87 100644
--- a/llvm/test/DebugInfo/COFF/local-variables.ll
+++ b/llvm/test/DebugInfo/COFF/local-variables.ll
@@ -60,7 +60,7 @@
 ; ASM:         leaq    36(%rsp), %rcx
 ; ASM: [[else_end:\.Ltmp.*]]:
 ; ASM: .LBB0_3:                                # %if.end
-; ASM:         .cv_loc 0 1 0 0                # t.cpp:0:0
+; ASM:         .cv_loc 0 1 17 1                # t.cpp:17:1
 ; ASM:         callq   capture
 ; ASM:         nop
 ; ASM:         addq    $56, %rsp
@@ -71,30 +71,30 @@
 ; ASM: .long   116                     # TypeIndex
 ; ASM: .short  1                       # Flags
 ; ASM: .asciz  "param"
-; ASM: .cv_def_range    [[prologue_end]] [[param_end]], "B\0214\000\000\000"
+; ASM: .cv_def_range    [[prologue_end]] [[param_end]], frame_ptr_rel, 52
 ; ASM: .short  4414                    # Record kind: S_LOCAL
 ; ASM: .long   116                     # TypeIndex
 ; ASM: .short  0                       # Flags
 ; ASM: .asciz  "a"
-; ASM: .cv_def_range    [[if_start]] [[else_start]], "B\021(\000\000\000"
+; ASM: .cv_def_range    [[if_start]] [[else_start]], frame_ptr_rel, 40
 ; ASM: .short  4414                    # Record kind: S_LOCAL
 ; ASM: .long   116                     # TypeIndex
 ; ASM: .short  0                       # Flags
 ; ASM: .asciz  "b"
-; ASM: .cv_def_range    [[else_start]] [[else_end]], "B\021$\000\000\000"
+; ASM: .cv_def_range    [[else_start]] [[else_end]], frame_ptr_rel, 36
 ; ASM: .short  4429                    # Record kind: S_INLINESITE
 ; ASM: .short  4414                    # Record kind: S_LOCAL
 ; ASM: .long   116                     # TypeIndex
 ; ASM: .short  0                       # Flags
 ; ASM: .asciz  "v"
-; ASM: .cv_def_range    [[inline_site1]] [[else_start]], "B\021,\000\000\000"
+; ASM: .cv_def_range    [[inline_site1]] [[else_start]], frame_ptr_rel, 44
 ; ASM: .short  4430                    # Record kind: S_INLINESITE_END
 ; ASM: .short  4429                    # Record kind: S_INLINESITE
 ; ASM: .short  4414                    # Record kind: S_LOCAL
 ; ASM: .long   116                     # TypeIndex
 ; ASM: .short  0                       # Flags
 ; ASM: .asciz  "v"
-; ASM: .cv_def_range    [[inline_site2]] [[else_end]], "B\0210\000\000\000"
+; ASM: .cv_def_range    [[inline_site2]] [[else_end]], frame_ptr_rel, 48
 ; ASM: .short  4430                    # Record kind: S_INLINESITE_END
 
 ; OBJ:  Subsection [
@@ -248,9 +248,9 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
 
 declare void @capture(i32*) #2
 
-attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
-attributes #2 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #3 = { nounwind }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/DebugInfo/COFF/long-name.ll b/llvm/test/DebugInfo/COFF/long-name.ll
index db3d8ccd8..08be48be8 100644
--- a/llvm/test/DebugInfo/COFF/long-name.ll
+++ b/llvm/test/DebugInfo/COFF/long-name.ll
@@ -35,7 +35,7 @@ entry:
   ret void, !dbg !10
 }
 
-attributes #0 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!3, !4, !5}
diff --git a/llvm/test/DebugInfo/COFF/multifile.ll b/llvm/test/DebugInfo/COFF/multifile.ll
index cc17a48ff..d40f8b788 100644
--- a/llvm/test/DebugInfo/COFF/multifile.ll
+++ b/llvm/test/DebugInfo/COFF/multifile.ll
@@ -215,8 +215,8 @@ entry:
 
 declare void @g() #1
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!9, !10}
diff --git a/llvm/test/DebugInfo/COFF/multifunction.ll b/llvm/test/DebugInfo/COFF/multifunction.ll
index 577071cd9..b890e6b0b 100644
--- a/llvm/test/DebugInfo/COFF/multifunction.ll
+++ b/llvm/test/DebugInfo/COFF/multifunction.ll
@@ -680,8 +680,8 @@ entry:
   ret void, !dbg !21
 }
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!11, !12}
diff --git a/llvm/test/DebugInfo/COFF/nrvo.ll b/llvm/test/DebugInfo/COFF/nrvo.ll
index f4b7ebfe8..77f13fcb7 100644
--- a/llvm/test/DebugInfo/COFF/nrvo.ll
+++ b/llvm/test/DebugInfo/COFF/nrvo.ll
@@ -28,7 +28,7 @@
 ; ASM-NEXT:   .asciz "foo"
 ; ASM-NEXT:   .p2align 2
 ; ASM-NEXT: .Ltmp
-; ASM:        .cv_def_range  .Ltmp{{.*}} .Ltmp{{.*}}, "B\021(\000\000\000"
+; ASM:        .cv_def_range  .Ltmp{{.*}} .Ltmp{{.*}}, frame_ptr_rel, 40
 
 ; OBJ: Subsection [
 ; OBJ:   SubSectionType: Symbols (0xF1)
@@ -95,9 +95,9 @@ entry:
   ret i32 %0, !dbg !38
 }
 
-attributes #0 = { noinline nounwind optnone uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noinline nounwind optnone uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone speculatable }
-attributes #2 = { noinline norecurse nounwind optnone uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { noinline norecurse nounwind optnone uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!3, !4, !5, !6}
diff --git a/llvm/test/DebugInfo/COFF/parameter-order.ll b/llvm/test/DebugInfo/COFF/parameter-order.ll
index 87c8ec252..bc8989e45 100644
--- a/llvm/test/DebugInfo/COFF/parameter-order.ll
+++ b/llvm/test/DebugInfo/COFF/parameter-order.ll
@@ -81,7 +81,7 @@ entry:
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
 
-attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/DebugInfo/COFF/pieces.ll b/llvm/test/DebugInfo/COFF/pieces.ll
index 2ca07aa4f..4e11dcabb 100644
--- a/llvm/test/DebugInfo/COFF/pieces.ll
+++ b/llvm/test/DebugInfo/COFF/pieces.ll
@@ -43,7 +43,6 @@
 ; ASM: [[oy_ox_start:\.Ltmp[0-9]+]]:
 ; ASM:         .p2align        4, 0x90
 ; ASM: .LBB0_3:                                # %for.body
-; ASM:        #DEBUG_VALUE: loop_csr:o <- [DW_OP_LLVM_fragment 0 32] 0
 ; ASM:        #DEBUG_VALUE: loop_csr:o <- [DW_OP_LLVM_fragment 0 32] $edi
 ; ASM:        #DEBUG_VALUE: loop_csr:o <- [DW_OP_LLVM_fragment 32 32] $esi
 ; ASM:        .cv_loc 0 1 13 11               # t.c:13:11
@@ -77,6 +76,7 @@
 ; ASM: [[pad_right_tmp:\.Ltmp[0-9]+]]:
 ; ASM:         #DEBUG_VALUE: pad_right:o <- [DW_OP_LLVM_fragment 32 32] $eax
 ; ASM:         retq
+; ASM: [[pad_right_end:\.Lfunc_end1]]:
 
 
 ; ASM-LABEL: pad_left: # @pad_left
@@ -85,6 +85,7 @@
 ; ASM: [[pad_left_tmp:\.Ltmp[0-9]+]]:
 ; ASM:         #DEBUG_VALUE: pad_left:o <- [DW_OP_LLVM_fragment 0 32] $eax
 ; ASM:         retq
+; ASM: [[pad_left_end:\.Lfunc_end2]]:
 
 
 ; ASM-LABEL: nested: # @nested
@@ -113,10 +114,10 @@
 ; ASM:        .asciz  "loop_csr"              # Function name
 ; ASM:        .short  4414                    # Record kind: S_LOCAL
 ; ASM:        .asciz  "o"
-; ASM:        .cv_def_range    [[oy_ox_start]] [[ox_start]], "C\021\030\000\000\000\000\000\000\000"
-; ASM:        .cv_def_range    [[oy_ox_start]] [[oy_start]], "C\021\027\000\000\000\004\000\000\000"
-; ASM:        .cv_def_range    [[ox_start]] [[loopskip_start]], "C\021\030\000\000\000\000\000\000\000"
-; ASM:        .cv_def_range    [[oy_start]] [[loopskip_start]], "C\021\027\000\000\000\004\000\000\000"
+; ASM:        .cv_def_range    [[oy_ox_start]] [[ox_start]], subfield_reg, 24, 0
+; ASM:        .cv_def_range    [[oy_ox_start]] [[oy_start]], subfield_reg, 23, 4
+; ASM:        .cv_def_range    [[ox_start]] [[loopskip_start]], subfield_reg, 24, 0
+; ASM:        .cv_def_range    [[oy_start]] [[loopskip_start]], subfield_reg, 23, 4
 
 
 ; OBJ-LABEL: GlobalProcIdSym {
@@ -147,7 +148,7 @@
 ; ASM:        .asciz  "pad_right"             # Function name
 ; ASM:        .short  4414                    # Record kind: S_LOCAL
 ; ASM:        .asciz  "o"
-; ASM:        .cv_def_range    [[pad_right_tmp]] [[pad_right_tmp]], "C\021\021\000\000\000\004\000\000\000"
+; ASM:        .cv_def_range    [[pad_right_tmp]] [[pad_right_end]], subfield_reg, 17, 4
 
 ; OBJ-LABEL: GlobalProcIdSym {
 ; OBJ:         Kind: S_GPROC32_ID (0x1147)
@@ -170,7 +171,7 @@
 ; ASM:        .asciz  "pad_left"              # Function name
 ; ASM:        .short  4414                    # Record kind: S_LOCAL
 ; ASM:        .asciz  "o"
-; ASM:        .cv_def_range    [[pad_left_tmp]] [[pad_left_tmp]], "C\021\021\000\000\000\000\000\000\000"
+; ASM:        .cv_def_range    [[pad_left_tmp]] [[pad_left_end]], subfield_reg, 17, 0
 
 ; OBJ-LABEL: GlobalProcIdSym {
 ; OBJ:         Kind: S_GPROC32_ID (0x1147)
@@ -193,10 +194,10 @@
 ; ASM:        .asciz  "nested"                # Function name
 ; ASM:        .short  4414                    # Record kind: S_LOCAL
 ; ASM:        .asciz  "o"
-; ASM:        .cv_def_range    .Lfunc_begin3 .Lfunc_end3, "E\021J\001\000\000\000\000\000\000"
+; ASM:        .cv_def_range    .Lfunc_begin3 .Lfunc_end3, reg_rel, 330, 0, 0
 ; ASM:        .short  4414                    # Record kind: S_LOCAL
 ; ASM:        .asciz  "p"
-; ASM:        .cv_def_range    [[p_start]] .Lfunc_end3, "C\021\021\000\000\000\004\000\000\000"
+; ASM:        .cv_def_range    [[p_start]] .Lfunc_end3, subfield_reg, 17, 4
 
 ; OBJ-LABEL: GlobalProcIdSym {
 ; OBJ:         Kind: S_GPROC32_ID (0x1147)
@@ -232,7 +233,7 @@
 ; ASM:        .asciz  "bitpiece_spill"        # Function name
 ; ASM:        .short  4414                    # Record kind: S_LOCAL
 ; ASM:        .asciz  "o"
-; ASM:        .cv_def_range    [[spill_o_x_start]] .Lfunc_end4, "E\021O\001A\000$\000\000\000"
+; ASM:        .cv_def_range    [[spill_o_x_start]] .Lfunc_end4, reg_rel, 335, 65, 36
 
 ; OBJ-LABEL: GlobalProcIdSym {
 ; OBJ:         Kind: S_GPROC32_ID (0x1147)
@@ -352,11 +353,11 @@ entry:
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.value(metadata, metadata, metadata) #1
 
-attributes #0 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
-attributes #2 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #3 = { nounwind readnone uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #4 = { nounwind readonly uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { nounwind readnone uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #4 = { nounwind readonly uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #5 = { nounwind }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/DebugInfo/COFF/pr37492.ll b/llvm/test/DebugInfo/COFF/pr37492.ll
index 7edff440b..bdf0bc77d 100644
--- a/llvm/test/DebugInfo/COFF/pr37492.ll
+++ b/llvm/test/DebugInfo/COFF/pr37492.ll
@@ -12,20 +12,22 @@
 ; collision between this LF_BITFIELD and LF_MODIFIER record, so we only emitted
 ; one under the assumption that the other was redundant. Check that we emit both.
 
-; CHECK-LABEL: # BitField ({{.*}}) {
-; CHECK-NEXT: #   TypeLeafKind: LF_BITFIELD (0x1205)
-; CHECK-NEXT: #   Type: unsigned char (0x20)
-; CHECK-NEXT: #   BitSize: 1
-; CHECK-NEXT: #   BitOffset: 0
-; CHECK-NEXT: # }
-
-; CHECK-LABEL: # Modifier ({{.*}}) {
-; CHECK-NEXT: #   TypeLeafKind: LF_MODIFIER (0x1001)
-; CHECK-NEXT: #   ModifiedType: unsigned char (0x20)
-; CHECK-NEXT: #   Modifiers [ (0x1)
-; CHECK-NEXT: #     Const (0x1)
-; CHECK-NEXT: #   ]
-; CHECK-NEXT: # }
+; CHECK-LABEL: # BitField (0x1001)
+; CHECK-NEXT: .short	0xa                     # Record length
+; CHECK-NEXT: .short	0x1205                  # Record kind: LF_BITFIELD
+; CHECK-NEXT: .long	0x20                    # Type: unsigned char
+; CHECK-NEXT: .byte	0x1                     # BitSize
+; CHECK-NEXT: .byte	0x0                     # BitOffset
+; CHECK-NEXT: .byte	242
+; CHECK-NEXT: .byte	241
+
+; CHECK-LABEL: # Modifier (0x1007)
+; CHECK-NEXT: .short	0xa                     # Record length
+; CHECK-NEXT: .short	0x1001                  # Record kind: LF_MODIFIER
+; CHECK-NEXT: .long	0x20                    # ModifiedType: unsigned char
+; CHECK-NEXT: .short	0x1                     # Modifiers ( Const (0x1) )
+; CHECK-NEXT: .byte	242
+; CHECK-NEXT: .byte	241
 
 ; ModuleID = 't.cpp'
 source_filename = "t.cpp"
diff --git a/llvm/test/DebugInfo/COFF/purge-typedef-udts.ll b/llvm/test/DebugInfo/COFF/purge-typedef-udts.ll
index b1edc470c..6aa1bc608 100644
--- a/llvm/test/DebugInfo/COFF/purge-typedef-udts.ll
+++ b/llvm/test/DebugInfo/COFF/purge-typedef-udts.ll
@@ -62,7 +62,7 @@ entry:
   ret i32 %call, !dbg !48
 }
 
-attributes #0 = { noinline nounwind optnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noinline nounwind optnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone speculatable }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/DebugInfo/COFF/register-variables.ll b/llvm/test/DebugInfo/COFF/register-variables.ll
index d0a0d27fd..60c9bb0d9 100644
--- a/llvm/test/DebugInfo/COFF/register-variables.ll
+++ b/llvm/test/DebugInfo/COFF/register-variables.ll
@@ -32,9 +32,6 @@
 ; ASM:         #DEBUG_VALUE: f:p <- $esi
 ; ASM:         callq   getint
 ; ASM: [[after_getint:\.Ltmp.*]]:
-; ASM:         #DEBUG_VALUE: a <- $eax
-; ASM:         #DEBUG_VALUE: inlineinc:a <- $eax
-; ASM:         #DEBUG_VALUE: c <- $eax
 ; ASM:         testl   %esi, %esi
 ; ASM:         je      .LBB0_2
 ; ASM: [[after_je:\.Ltmp.*]]:
@@ -45,38 +42,41 @@
 ; ASM:         addl    $1, %eax
 ; ASM: [[after_inc_eax:\.Ltmp.*]]:
 ; ASM:         #DEBUG_VALUE: inlineinc:b <- $eax
-; ASM:         #DEBUG_VALUE: b <- $eax
 ; ASM:         addl    $1, x(%rip)
 ; ASM: [[after_if:\.Ltmp.*]]:
 ; ASM: .LBB0_2:                                # %if.else
 ; ASM:         #DEBUG_VALUE: f:p <- $esi
+; ASM:         #DEBUG_VALUE: c <- $eax
 ; ASM:         movl    %eax, %ecx
 ; ASM:         addq    $32, %rsp
 ; ASM:         popq    %rsi
 ; ASM: [[func_end:\.Ltmp.*]]:
 ; ASM:         jmp     putint                  # TAILCALL
+; ASM: [[func_finished:\.Ltmp.*]]:
 
 ; ASM:         .short  4414                    # Record kind: S_LOCAL
 ; ASM:         .asciz  "p"
-; ASM:         .cv_def_range    .Lfunc_begin0 [[p_ecx_esi]], "A\021\022\000\000\000"
-; ASM:         .cv_def_range    [[p_ecx_esi]] [[func_end]], "A\021\027\000\000\000"
+; ASM:         .cv_def_range    .Lfunc_begin0 [[p_ecx_esi]], reg, 18
+; ASM:         .cv_def_range    [[p_ecx_esi]] [[func_end]], reg, 23
 ; ASM:         .short  4414                    # Record kind: S_LOCAL
 ; ASM:         .asciz  "c"
-; ASM:         .cv_def_range    [[after_getint]] [[after_je]], "A\021\021\000\000\000"
+; ASM:         .cv_def_range    [[after_if]] [[func_finished]], reg, 17
 ; ASM:         .short  4414                    # Record kind: S_LOCAL
 ; ASM:         .asciz  "a"
-; ASM:         .cv_def_range    [[after_getint]] [[after_inc_eax]], "A\021\021\000\000\000"
+; ASM:         .cv_def_range    [[after_je]] [[after_inc_eax]], reg, 17
 ; ASM:         .short  4414                    # Record kind: S_LOCAL
 ; ASM:         .asciz  "b"
-; ASM:         .cv_def_range    [[after_inc_eax]] [[after_if]], "A\021\021\000\000\000"
+; ASM:         .cv_def_range    [[after_if]] [[after_if]], reg, 17
+
+; Note: "b" is a victim of tail de-duplication / branch folding.
 
 ; ASM:         .short  4429                    # Record kind: S_INLINESITE
 ; ASM:         .short  4414                    # Record kind: S_LOCAL
 ; ASM:         .asciz  "a"
-; ASM:         .cv_def_range    [[after_getint]] [[after_inc_eax]], "A\021\021\000\000\000"
+; ASM:         .cv_def_range    [[after_je]] [[after_inc_eax]], reg, 17
 ; ASM:         .short  4414                    # Record kind: S_LOCAL
 ; ASM:         .asciz  "b"
-; ASM:         .cv_def_range    [[after_inc_eax]] [[after_if]], "A\021\021\000\000\000"
+; ASM:         .cv_def_range    [[after_inc_eax]] [[after_if]], reg, 17
 ; ASM:         .short  4430                    # Record kind: S_INLINESITE_END
 
 ; OBJ: Subsection [
@@ -116,9 +116,9 @@
 ; OBJ:   DefRangeRegisterSym {
 ; OBJ:     Register: EAX (0x11)
 ; OBJ:     LocalVariableAddrRange {
-; OBJ:       OffsetStart: .text+0xC
+; OBJ:       OffsetStart: .text+0x1A
 ; OBJ:       ISectStart: 0x0
-; OBJ:       Range: 0x4
+; OBJ:       Range: 0xC
 ; OBJ:     }
 ; OBJ:   }
 ; OBJ:   LocalSym {
@@ -130,23 +130,9 @@
 ; OBJ:   DefRangeRegisterSym {
 ; OBJ:     Register: EAX (0x11)
 ; OBJ:     LocalVariableAddrRange {
-; OBJ:       OffsetStart: .text+0xC
-; OBJ:       ISectStart: 0x0
-; OBJ:       Range: 0x7
-; OBJ:     }
-; OBJ:   }
-; OBJ:   LocalSym {
-; OBJ:     Type: int (0x74)
-; OBJ:     Flags [ (0x0)
-; OBJ:     ]
-; OBJ:     VarName: b
-; OBJ:   }
-; OBJ:   DefRangeRegisterSym {
-; OBJ:     Register: EAX (0x11)
-; OBJ:     MayHaveNoName: 0
-; OBJ:       OffsetStart: .text+0x13
+; OBJ:       OffsetStart: .text+0x10
 ; OBJ:       ISectStart: 0x0
-; OBJ:       Range: 0x7
+; OBJ:       Range: 0x3
 ; OBJ:     }
 ; OBJ:   }
 ; OBJ:   InlineSiteSym {
@@ -164,9 +150,9 @@
 ; OBJ:   DefRangeRegisterSym {
 ; OBJ:     Register: EAX (0x11)
 ; OBJ:     LocalVariableAddrRange {
-; OBJ:       OffsetStart: .text+0xC
+; OBJ:       OffsetStart: .text+0x10
 ; OBJ:       ISectStart: 0x0
-; OBJ:       Range: 0x7
+; OBJ:       Range: 0x3
 ; OBJ:     }
 ; OBJ:   }
 ; OBJ:   LocalSym {
@@ -231,8 +217,8 @@ declare void @putint(i32) #1
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.value(metadata, metadata, metadata) #2
 
-attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind readnone }
 attributes #3 = { nounwind }
 
diff --git a/llvm/test/DebugInfo/COFF/retained-types.ll b/llvm/test/DebugInfo/COFF/retained-types.ll
index 8d4253b82..64c08532e 100644
--- a/llvm/test/DebugInfo/COFF/retained-types.ll
+++ b/llvm/test/DebugInfo/COFF/retained-types.ll
@@ -64,7 +64,7 @@ entry:
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
 
-attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/DebugInfo/COFF/scopes.ll b/llvm/test/DebugInfo/COFF/scopes.ll
index 23509b4c5..facc11f06 100644
--- a/llvm/test/DebugInfo/COFF/scopes.ll
+++ b/llvm/test/DebugInfo/COFF/scopes.ll
@@ -102,7 +102,7 @@ entry:
   ret void, !dbg !34
 }
 
-attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!13}
diff --git a/llvm/test/DebugInfo/COFF/simple.ll b/llvm/test/DebugInfo/COFF/simple.ll
index 5df446fa7..6f8afb5bf 100644
--- a/llvm/test/DebugInfo/COFF/simple.ll
+++ b/llvm/test/DebugInfo/COFF/simple.ll
@@ -280,8 +280,8 @@ entry:
 
 declare void @g() #1
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!9, !10}
diff --git a/llvm/test/DebugInfo/COFF/static-methods.ll b/llvm/test/DebugInfo/COFF/static-methods.ll
index 1e51f80c5..f29dd9ae7 100644
--- a/llvm/test/DebugInfo/COFF/static-methods.ll
+++ b/llvm/test/DebugInfo/COFF/static-methods.ll
@@ -99,8 +99,8 @@ entry:
   ret void
 }
 
-attributes #0 = { noinline "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { nobuiltin "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noinline "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nobuiltin "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { builtin }
 
 !llvm.dbg.cu = !{!2}
diff --git a/llvm/test/DebugInfo/COFF/synthetic.ll b/llvm/test/DebugInfo/COFF/synthetic.ll
index aed6c33d5..246244e6b 100644
--- a/llvm/test/DebugInfo/COFF/synthetic.ll
+++ b/llvm/test/DebugInfo/COFF/synthetic.ll
@@ -40,16 +40,16 @@ entry:
 !27 = !DIDerivedType(tag: DW_TAG_member, name: "Size", scope: !1, file: !1, baseType: !26, size: 32, offset: 32)
 !28 = !DILocation(line: 1, scope: !6)
 
-; CHECK: # Struct
-; CHECK: #   TypeLeafKind: LF_STRUCTURE
-; CHECK: #   MemberCount: 0
-; CHECK: #   Properties [
-; CHECK: #     ForwardReference
-; CHECK: #   ]
-; CHECK: #   FieldList: 0x0
-; CHECK: #   DerivedFrom: 0x0
-; CHECK: #   VShape: 0x0
-; CHECK: #   SizeOf: 0
-; CHECK: #   Name: __block_descriptor
-; CHECK: # }
-
+; CHECK: # Struct (0x1003)
+; CHECK: .short	0x2a                    # Record length
+; CHECK: .short	0x1505                  # Record kind: LF_STRUCTURE
+; CHECK: .short	0x0                     # MemberCount
+; CHECK: .short	0x80                    # Properties ( ForwardReference (0x80) )
+; CHECK: .long	0x0                     # FieldList
+; CHECK: .long	0x0                     # DerivedFrom
+; CHECK: .long	0x0                     # VShape
+; CHECK: .short	0x0                     # SizeOf
+; CHECK: .asciz	"__block_descriptor"    # Name
+; CHECK: .byte	243
+; CHECK: .byte	242
+; CHECK: .byte	241
diff --git a/llvm/test/DebugInfo/COFF/tail-call-without-lexical-scopes.ll b/llvm/test/DebugInfo/COFF/tail-call-without-lexical-scopes.ll
index a352f3c49..896e76058 100644
--- a/llvm/test/DebugInfo/COFF/tail-call-without-lexical-scopes.ll
+++ b/llvm/test/DebugInfo/COFF/tail-call-without-lexical-scopes.ll
@@ -50,8 +50,8 @@ entry:
 
 declare void @"\01?foo@@YAXXZ"() #1
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/DebugInfo/COFF/thunk.ll b/llvm/test/DebugInfo/COFF/thunk.ll
index ad18872b8..39abed580 100644
--- a/llvm/test/DebugInfo/COFF/thunk.ll
+++ b/llvm/test/DebugInfo/COFF/thunk.ll
@@ -469,13 +469,13 @@ entry:
   ret i1 true, !dbg !102
 }
 
-attributes #0 = { noinline norecurse optnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noinline norecurse optnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone speculatable }
-attributes #2 = { nobuiltin "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { nobuiltin "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #3 = { argmemonly nounwind }
-attributes #4 = { noinline nounwind optnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #5 = { noinline optnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "thunk" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #6 = { noinline optnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #4 = { noinline nounwind optnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #5 = { noinline optnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "thunk" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #6 = { noinline optnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #7 = { builtin }
 attributes #8 = { nounwind }
 
diff --git a/llvm/test/DebugInfo/COFF/type-quals.ll b/llvm/test/DebugInfo/COFF/type-quals.ll
index fbe7f6c5b..c5953d384 100644
--- a/llvm/test/DebugInfo/COFF/type-quals.ll
+++ b/llvm/test/DebugInfo/COFF/type-quals.ll
@@ -476,7 +476,7 @@ entry:
   ret i32 1, !dbg !86
 }
 
-attributes #0 = { noinline nounwind optnone uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noinline nounwind optnone uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone speculatable }
 attributes #2 = { argmemonly nounwind }
 
diff --git a/llvm/test/DebugInfo/COFF/types-array.ll b/llvm/test/DebugInfo/COFF/types-array.ll
index 7473c83f1..2962f970a 100644
--- a/llvm/test/DebugInfo/COFF/types-array.ll
+++ b/llvm/test/DebugInfo/COFF/types-array.ll
@@ -108,10 +108,10 @@ declare void @llvm.memcpy.p0i8.p0i8.i32(i8* nocapture, i8* nocapture readonly, i
 
 declare void @"\01?usevars@@YAXHZZ"(i32, ...) #3
 
-attributes #0 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 attributes #2 = { argmemonly nounwind }
-attributes #3 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!3, !4}
diff --git a/llvm/test/DebugInfo/COFF/types-basic.ll b/llvm/test/DebugInfo/COFF/types-basic.ll
index 3b62a3920..81e0c25d1 100644
--- a/llvm/test/DebugInfo/COFF/types-basic.ll
+++ b/llvm/test/DebugInfo/COFF/types-basic.ll
@@ -350,367 +350,178 @@
 ; ASM: .section	.debug$T,"dr"
 ; ASM: .p2align	2
 ; ASM: .long	4                       # Debug section magic
+; ASM: # ArgList (0x1000)
 ; ASM: .short	0x12                    # Record length
 ; ASM: .short	0x1201                  # Record kind: LF_ARGLIST
 ; ASM: .long	0x3                     # NumArgs
-; ASM: .long	0x40                    # Argument
-; ASM: .long	0x41                    # Argument
-; ASM: .long	0x13                    # Argument
-; ASM: # ArgList (0x1000) {
-; ASM: #   TypeLeafKind: LF_ARGLIST (0x1201)
-; ASM: #   NumArgs: 3
-; ASM: #   Arguments [
-; ASM: #     ArgType: float (0x40)
-; ASM: #     ArgType: double (0x41)
-; ASM: #     ArgType: __int64 (0x13)
-; ASM: #   ]
-; ASM: # }
+; ASM: .long	0x40                    # Argument: float
+; ASM: .long	0x41                    # Argument: double
+; ASM: .long	0x13                    # Argument: __int64
+; ASM: # Procedure (0x1001)
 ; ASM: .short	0xe                     # Record length
 ; ASM: .short	0x1008                  # Record kind: LF_PROCEDURE
-; ASM: .long	0x3                     # ReturnType
-; ASM: .byte	0x0                     # CallingConvention
+; ASM: .long	0x3                     # ReturnType: void
+; ASM: .byte	0x0                     # CallingConvention: NearC
 ; ASM: .byte	0x0                     # FunctionOptions
 ; ASM: .short	0x3                     # NumParameters
-; ASM: .long	0x1000                  # ArgListType
-; ASM: # Procedure (0x1001) {
-; ASM: #   TypeLeafKind: LF_PROCEDURE (0x1008)
-; ASM: #   ReturnType: void (0x3)
-; ASM: #   CallingConvention: NearC (0x0)
-; ASM: #   FunctionOptions [ (0x0)
-; ASM: #   ]
-; ASM: #   NumParameters: 3
-; ASM: #   ArgListType: (float, double, __int64) (0x1000)
-; ASM: # }
+; ASM: .long	0x1000                  # ArgListType: (float, double, __int64)
+; ASM: # FuncId (0x1002)
 ; ASM: .short	0xe                     # Record length
 ; ASM: .short	0x1601                  # Record kind: LF_FUNC_ID
 ; ASM: .long	0x0                     # ParentScope
-; ASM: .long	0x1001                  # FunctionType
+; ASM: .long	0x1001                  # FunctionType: void (float, double, __int64)
 ; ASM: .asciz	"f"                     # Name
 ; ASM: .byte	242
 ; ASM: .byte	241
-; ASM: # FuncId (0x1002) {
-; ASM: #   TypeLeafKind: LF_FUNC_ID (0x1601)
-; ASM: #   ParentScope: 0x0
-; ASM: #   FunctionType: void (float, double, __int64) (0x1001)
-; ASM: #   Name: f
-; ASM: # }
+; ASM: # Modifier (0x1003)
 ; ASM: .short	0xa                     # Record length
 ; ASM: .short	0x1001                  # Record kind: LF_MODIFIER
-; ASM: .long	0x74                    # ModifiedType
-; ASM: .short	0x1                     # Modifiers
+; ASM: .long	0x74                    # ModifiedType: int
+; ASM: .short	0x1                     # Modifiers ( Const (0x1) )
 ; ASM: .byte	242
 ; ASM: .byte	241
-; ASM: # Modifier (0x1003) {
-; ASM: #   TypeLeafKind: LF_MODIFIER (0x1001)
-; ASM: #   ModifiedType: int (0x74)
-; ASM: #   Modifiers [ (0x1)
-; ASM: #     Const (0x1)
-; ASM: #   ]
-; ASM: # }
+; ASM: # Pointer (0x1004)
 ; ASM: .short	0xa                     # Record length
 ; ASM: .short	0x1002                  # Record kind: LF_POINTER
-; ASM: .long	0x1003                  # PointeeType
-; ASM: .long	0x1000c                 # Attributes
-; ASM: # Pointer (0x1004) {
-; ASM: #   TypeLeafKind: LF_POINTER (0x1002)
-; ASM: #   PointeeType: const int (0x1003)
-; ASM: #   PtrType: Near64 (0xC)
-; ASM: #   PtrMode: Pointer (0x0)
-; ASM: #   IsFlat: 0
-; ASM: #   IsConst: 0
-; ASM: #   IsVolatile: 0
-; ASM: #   IsUnaligned: 0
-; ASM: #   IsRestrict: 0
-; ASM: #   IsThisPtr&: 0
-; ASM: #   IsThisPtr&&: 0
-; ASM: #   SizeOf: 8
-; ASM: # }
+; ASM: .long	0x1003                  # PointeeType: const int
+; ASM: .long	0x1000c                 # Attrs: [ Type: Near64, Mode: Pointer, SizeOf: 8 ]
+; ASM: # Struct (0x1005)
 ; ASM: .short	0x16                    # Record length
 ; ASM: .short	0x1505                  # Record kind: LF_STRUCTURE
 ; ASM: .short	0x0                     # MemberCount
-; ASM: .short	0x80                    # Properties
+; ASM: .short	0x80                    # Properties ( ForwardReference (0x80) )
 ; ASM: .long	0x0                     # FieldList
 ; ASM: .long	0x0                     # DerivedFrom
 ; ASM: .long	0x0                     # VShape
 ; ASM: .short	0x0                     # SizeOf
 ; ASM: .asciz	"A"                     # Name
-; ASM: # Struct (0x1005) {
-; ASM: #   TypeLeafKind: LF_STRUCTURE (0x1505)
-; ASM: #   MemberCount: 0
-; ASM: #   Properties [ (0x80)
-; ASM: #     ForwardReference (0x80)
-; ASM: #   ]
-; ASM: #   FieldList: 0x0
-; ASM: #   DerivedFrom: 0x0
-; ASM: #   VShape: 0x0
-; ASM: #   SizeOf: 0
-; ASM: #   Name: A
-; ASM: # }
+; ASM: # Pointer (0x1006)
 ; ASM: .short	0x12                    # Record length
 ; ASM: .short	0x1002                  # Record kind: LF_POINTER
-; ASM: .long	0x74                    # PointeeType
-; ASM: .long	0x804c                  # Attributes
-; ASM: .long	0x1005                  # ClassType
-; ASM: .short	0x4                     # Representation
+; ASM: .long	0x74                    # PointeeType: int
+; ASM: .long	0x804c                  # Attrs: [ Type: Near64, Mode: PointerToDataMember, SizeOf: 4 ]
+; ASM: .long	0x1005                  # ClassType: A
+; ASM: .short	0x4                     # Representation: GeneralData
 ; ASM: .byte	242
 ; ASM: .byte	241
-; ASM: # Pointer (0x1006) {
-; ASM: #   TypeLeafKind: LF_POINTER (0x1002)
-; ASM: #   PointeeType: int (0x74)
-; ASM: #   PtrType: Near64 (0xC)
-; ASM: #   PtrMode: PointerToDataMember (0x2)
-; ASM: #   IsFlat: 0
-; ASM: #   IsConst: 0
-; ASM: #   IsVolatile: 0
-; ASM: #   IsUnaligned: 0
-; ASM: #   IsRestrict: 0
-; ASM: #   IsThisPtr&: 0
-; ASM: #   IsThisPtr&&: 0
-; ASM: #   SizeOf: 4
-; ASM: #   ClassType: A (0x1005)
-; ASM: #   Representation: GeneralData (0x4)
-; ASM: # }
+; ASM: # Pointer (0x1007)
 ; ASM: .short	0xa                     # Record length
 ; ASM: .short	0x1002                  # Record kind: LF_POINTER
-; ASM: .long	0x1005                  # PointeeType
-; ASM: .long	0x1040c                 # Attributes
-; ASM: # Pointer (0x1007) {
-; ASM: #   TypeLeafKind: LF_POINTER (0x1002)
-; ASM: #   PointeeType: A (0x1005)
-; ASM: #   PtrType: Near64 (0xC)
-; ASM: #   PtrMode: Pointer (0x0)
-; ASM: #   IsFlat: 0
-; ASM: #   IsConst: 1
-; ASM: #   IsVolatile: 0
-; ASM: #   IsUnaligned: 0
-; ASM: #   IsRestrict: 0
-; ASM: #   IsThisPtr&: 0
-; ASM: #   IsThisPtr&&: 0
-; ASM: #   SizeOf: 8
-; ASM: # }
+; ASM: .long	0x1005                  # PointeeType: A
+; ASM: .long	0x1040c                 # Attrs: [ Type: Near64, Mode: Pointer, SizeOf: 8, isConst ]
+; ASM: # ArgList (0x1008)
 ; ASM: .short	0x6                     # Record length
 ; ASM: .short	0x1201                  # Record kind: LF_ARGLIST
 ; ASM: .long	0x0                     # NumArgs
-; ASM: # ArgList (0x1008) {
-; ASM: #   TypeLeafKind: LF_ARGLIST (0x1201)
-; ASM: #   NumArgs: 0
-; ASM: #   Arguments [
-; ASM: #   ]
-; ASM: # }
+; ASM: # MemberFunction (0x1009)
 ; ASM: .short	0x1a                    # Record length
 ; ASM: .short	0x1009                  # Record kind: LF_MFUNCTION
-; ASM: .long	0x3                     # ReturnType
-; ASM: .long	0x1005                  # ClassType
-; ASM: .long	0x1007                  # ThisType
-; ASM: .byte	0x0                     # CallingConvention
+; ASM: .long	0x3                     # ReturnType: void
+; ASM: .long	0x1005                  # ClassType: A
+; ASM: .long	0x1007                  # ThisType: A* const
+; ASM: .byte	0x0                     # CallingConvention: NearC
 ; ASM: .byte	0x0                     # FunctionOptions
 ; ASM: .short	0x0                     # NumParameters
-; ASM: .long	0x1008                  # ArgListType
+; ASM: .long	0x1008                  # ArgListType: ()
 ; ASM: .long	0x0                     # ThisAdjustment
-; ASM: # MemberFunction (0x1009) {
-; ASM: #   TypeLeafKind: LF_MFUNCTION (0x1009)
-; ASM: #   ReturnType: void (0x3)
-; ASM: #   ClassType: A (0x1005)
-; ASM: #   ThisType: A* const (0x1007)
-; ASM: #   CallingConvention: NearC (0x0)
-; ASM: #   FunctionOptions [ (0x0)
-; ASM: #   ]
-; ASM: #   NumParameters: 0
-; ASM: #   ArgListType: () (0x1008)
-; ASM: #   ThisAdjustment: 0
-; ASM: # }
+; ASM: # FieldList (0x100A)
 ; ASM: .short	0x1e                    # Record length
 ; ASM: .short	0x1203                  # Record kind: LF_FIELDLIST
-; ASM: .byte	0x0d, 0x15, 0x03, 0x00
-; ASM: .byte	0x74, 0x00, 0x00, 0x00
-; ASM: .byte	0x00, 0x00, 0x61, 0x00
-; ASM: .byte	0x11, 0x15, 0x03, 0x00
-; ASM: .byte	0x09, 0x10, 0x00, 0x00
-; ASM: .byte	0x41, 0x3a, 0x3a, 0x66
-; ASM: .byte	0x00, 0xf3, 0xf2, 0xf1
-; ASM: # FieldList (0x100A) {
-; ASM: #   TypeLeafKind: LF_FIELDLIST (0x1203)
-; ASM: #   DataMember {
-; ASM: #     TypeLeafKind: LF_MEMBER (0x150D)
-; ASM: #     AccessSpecifier: Public (0x3)
-; ASM: #     Type: int (0x74)
-; ASM: #     FieldOffset: 0x0
-; ASM: #     Name: a
-; ASM: #   }
-; ASM: #   OneMethod {
-; ASM: #     TypeLeafKind: LF_ONEMETHOD (0x1511)
-; ASM: #     AccessSpecifier: Public (0x3)
-; ASM: #     Type: void A::() (0x1009)
-; ASM: #     Name: A::f
-; ASM: #   }
-; ASM: # }
+; ASM: .short	0x150d                  # Member kind: DataMember ( LF_MEMBER )
+; ASM: .short	0x3                     # Attrs: Public
+; ASM: .long	0x74                    # Type: int
+; ASM: .short	0x0                     # FieldOffset
+; ASM: .asciz	"a"                     # Name
+; ASM: .short	0x1511                  # Member kind: OneMethod ( LF_ONEMETHOD )
+; ASM: .short	0x3                     # Attrs: Public
+; ASM: .long	0x1009                  # Type: void A::()
+; ASM: .asciz	"A::f"                  # Name
+; ASM: .byte	243
+; ASM: .byte	242
+; ASM: .byte	241
+; ASM: # Struct (0x100B)
 ; ASM: .short	0x16                    # Record length
 ; ASM: .short	0x1505                  # Record kind: LF_STRUCTURE
 ; ASM: .short	0x2                     # MemberCount
 ; ASM: .short	0x0                     # Properties
-; ASM: .long	0x100a                  # FieldList
+; ASM: .long	0x100a                  # FieldList: <field list>
 ; ASM: .long	0x0                     # DerivedFrom
 ; ASM: .long	0x0                     # VShape
 ; ASM: .short	0x4                     # SizeOf
 ; ASM: .asciz	"A"                     # Name
-; ASM: # Struct (0x100B) {
-; ASM: #   TypeLeafKind: LF_STRUCTURE (0x1505)
-; ASM: #   MemberCount: 2
-; ASM: #   Properties [ (0x0)
-; ASM: #   ]
-; ASM: #   FieldList: <field list> (0x100A)
-; ASM: #   DerivedFrom: 0x0
-; ASM: #   VShape: 0x0
-; ASM: #   SizeOf: 4
-; ASM: #   Name: A
-; ASM: # }
+; ASM: # StringId (0x100C)
 ; ASM: .short	0x1e                    # Record length
 ; ASM: .short	0x1605                  # Record kind: LF_STRING_ID
 ; ASM: .long	0x0                     # Id
 ; ASM: .asciz	"D:\\src\\llvm\\build\\t.cpp" # StringData
-; ASM: # StringId (0x100C) {
-; ASM: #   TypeLeafKind: LF_STRING_ID (0x1605)
-; ASM: #   Id: 0x0
-; ASM: #   StringData: D:\src\llvm\build\t.cpp
-; ASM: # }
+; ASM: # UdtSourceLine (0x100D)
 ; ASM: .short	0xe                     # Record length
 ; ASM: .short	0x1606                  # Record kind: LF_UDT_SRC_LINE
-; ASM: .long	0x100b                  # UDT
-; ASM: .long	0x100c                  # SourceFile
+; ASM: .long	0x100b                  # UDT: A
+; ASM: .long	0x100c                  # SourceFile: D:\src\llvm\build\t.cpp
 ; ASM: .long	0x1                     # LineNumber
-; ASM: # UdtSourceLine (0x100D) {
-; ASM: #   TypeLeafKind: LF_UDT_SRC_LINE (0x1606)
-; ASM: #   UDT: A (0x100B)
-; ASM: #   SourceFile: D:\src\llvm\build\t.cpp (0x100C)
-; ASM: #   LineNumber: 1
-; ASM: # }
+; ASM: # Pointer (0x100E)
 ; ASM: .short	0x12                    # Record length
 ; ASM: .short	0x1002                  # Record kind: LF_POINTER
-; ASM: .long	0x1009                  # PointeeType
-; ASM: .long	0x1006c                 # Attributes
-; ASM: .long	0x1005                  # ClassType
-; ASM: .short	0x8                     # Representation
+; ASM: .long	0x1009                  # PointeeType: void A::()
+; ASM: .long	0x1006c                 # Attrs: [ Type: Near64, Mode: PointerToMemberFunction, SizeOf: 8 ]
+; ASM: .long	0x1005                  # ClassType: A
+; ASM: .short	0x8                     # Representation: GeneralFunction
 ; ASM: .byte	242
 ; ASM: .byte	241
-; ASM: # Pointer (0x100E) {
-; ASM: #   TypeLeafKind: LF_POINTER (0x1002)
-; ASM: #   PointeeType: void A::() (0x1009)
-; ASM: #   PtrType: Near64 (0xC)
-; ASM: #   PtrMode: PointerToMemberFunction (0x3)
-; ASM: #   IsFlat: 0
-; ASM: #   IsConst: 0
-; ASM: #   IsVolatile: 0
-; ASM: #   IsUnaligned: 0
-; ASM: #   IsRestrict: 0
-; ASM: #   IsThisPtr&: 0
-; ASM: #   IsThisPtr&&: 0
-; ASM: #   SizeOf: 8
-; ASM: #   ClassType: A (0x1005)
-; ASM: #   Representation: GeneralFunction (0x8)
-; ASM: # }
+; ASM: # Modifier (0x100F)
 ; ASM: .short	0xa                     # Record length
 ; ASM: .short	0x1001                  # Record kind: LF_MODIFIER
-; ASM: .long	0x3                     # ModifiedType
-; ASM: .short	0x1                     # Modifiers
+; ASM: .long	0x3                     # ModifiedType: void
+; ASM: .short	0x1                     # Modifiers ( Const (0x1) )
 ; ASM: .byte	242
 ; ASM: .byte	241
-; ASM: # Modifier (0x100F) {
-; ASM: #   TypeLeafKind: LF_MODIFIER (0x1001)
-; ASM: #   ModifiedType: void (0x3)
-; ASM: #   Modifiers [ (0x1)
-; ASM: #     Const (0x1)
-; ASM: #   ]
-; ASM: # }
+; ASM: # Pointer (0x1010)
 ; ASM: .short	0xa                     # Record length
 ; ASM: .short	0x1002                  # Record kind: LF_POINTER
-; ASM: .long	0x100f                  # PointeeType
-; ASM: .long	0x1000c                 # Attributes
-; ASM: # Pointer (0x1010) {
-; ASM: #   TypeLeafKind: LF_POINTER (0x1002)
-; ASM: #   PointeeType: const void (0x100F)
-; ASM: #   PtrType: Near64 (0xC)
-; ASM: #   PtrMode: Pointer (0x0)
-; ASM: #   IsFlat: 0
-; ASM: #   IsConst: 0
-; ASM: #   IsVolatile: 0
-; ASM: #   IsUnaligned: 0
-; ASM: #   IsRestrict: 0
-; ASM: #   IsThisPtr&: 0
-; ASM: #   IsThisPtr&&: 0
-; ASM: #   SizeOf: 8
-; ASM: # }
+; ASM: .long	0x100f                  # PointeeType: const void
+; ASM: .long	0x1000c                 # Attrs: [ Type: Near64, Mode: Pointer, SizeOf: 8 ]
+; ASM: # Procedure (0x1011)
 ; ASM: .short	0xe                     # Record length
 ; ASM: .short	0x1008                  # Record kind: LF_PROCEDURE
-; ASM: .long	0x3                     # ReturnType
-; ASM: .byte	0x0                     # CallingConvention
+; ASM: .long	0x3                     # ReturnType: void
+; ASM: .byte	0x0                     # CallingConvention: NearC
 ; ASM: .byte	0x0                     # FunctionOptions
 ; ASM: .short	0x0                     # NumParameters
-; ASM: .long	0x1008                  # ArgListType
-; ASM: # Procedure (0x1011) {
-; ASM: #   TypeLeafKind: LF_PROCEDURE (0x1008)
-; ASM: #   ReturnType: void (0x3)
-; ASM: #   CallingConvention: NearC (0x0)
-; ASM: #   FunctionOptions [ (0x0)
-; ASM: #   ]
-; ASM: #   NumParameters: 0
-; ASM: #   ArgListType: () (0x1008)
-; ASM: # }
+; ASM: .long	0x1008                  # ArgListType: ()
+; ASM: # FuncId (0x1012)
 ; ASM: .short	0x16                    # Record length
 ; ASM: .short	0x1601                  # Record kind: LF_FUNC_ID
 ; ASM: .long	0x0                     # ParentScope
-; ASM: .long	0x1011                  # FunctionType
+; ASM: .long	0x1011                  # FunctionType: void ()
 ; ASM: .asciz	"CharTypes"             # Name
 ; ASM: .byte	242
 ; ASM: .byte	241
-; ASM: # FuncId (0x1012) {
-; ASM: #   TypeLeafKind: LF_FUNC_ID (0x1601)
-; ASM: #   ParentScope: 0x0
-; ASM: #   FunctionType: void () (0x1011)
-; ASM: #   Name: CharTypes
-; ASM: # }
+; ASM: # StringId (0x1013)
 ; ASM: .short	0x1a                    # Record length
 ; ASM: .short	0x1605                  # Record kind: LF_STRING_ID
 ; ASM: .long	0x0                     # Id
 ; ASM: .asciz	"D:\\src\\llvm\\build"  # StringData
 ; ASM: .byte	242
 ; ASM: .byte	241
-; ASM: # StringId (0x1013) {
-; ASM: #   TypeLeafKind: LF_STRING_ID (0x1605)
-; ASM: #   Id: 0x0
-; ASM: #   StringData: D:\src\llvm\build
-; ASM: # }
+; ASM: # StringId (0x1014)
 ; ASM: .short	0xe                     # Record length
 ; ASM: .short	0x1605                  # Record kind: LF_STRING_ID
 ; ASM: .long	0x0                     # Id
 ; ASM: .asciz	"t.cpp"                 # StringData
 ; ASM: .byte	242
 ; ASM: .byte	241
-; ASM: # StringId (0x1014) {
-; ASM: #   TypeLeafKind: LF_STRING_ID (0x1605)
-; ASM: #   Id: 0x0
-; ASM: #   StringData: t.cpp
-; ASM: # }
+; ASM: # BuildInfo (0x1015)
 ; ASM: .short	0x1a                    # Record length
 ; ASM: .short	0x1603                  # Record kind: LF_BUILDINFO
 ; ASM: .short	0x5                     # NumArgs
-; ASM: .long	0x1013                  # Argument
+; ASM: .long	0x1013                  # Argument: D:\src\llvm\build
 ; ASM: .long	0x0                     # Argument
-; ASM: .long	0x1014                  # Argument
+; ASM: .long	0x1014                  # Argument: t.cpp
 ; ASM: .long	0x0                     # Argument
 ; ASM: .long	0x0                     # Argument
 ; ASM: .byte	242
 ; ASM: .byte	241
-; ASM: # BuildInfo (0x1015) {
-; ASM: #   TypeLeafKind: LF_BUILDINFO (0x1603)
-; ASM: #   NumArgs: 5
-; ASM: #   Arguments [
-; ASM: #     ArgType: D:\src\llvm\build (0x1013)
-; ASM: #     ArgType: 0x0
-; ASM: #     ArgType: t.cpp (0x1014)
-; ASM: #     ArgType: 0x0
-; ASM: #     ArgType: 0x0
-; ASM: #   ]
-; ASM: # }
 
 ; ModuleID = 't.cpp'
 source_filename = "t.cpp"
@@ -806,10 +617,10 @@ entry:
   ret void, !dbg !96
 }
 
-attributes #0 = { uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
-attributes #2 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #3 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!3, !4, !5}
diff --git a/llvm/test/DebugInfo/COFF/types-calling-conv.ll b/llvm/test/DebugInfo/COFF/types-calling-conv.ll
index 0adac0095..948890607 100644
--- a/llvm/test/DebugInfo/COFF/types-calling-conv.ll
+++ b/llvm/test/DebugInfo/COFF/types-calling-conv.ll
@@ -208,8 +208,8 @@ entry:
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.value(metadata, metadata, metadata) #2
 
-attributes #0 = { nounwind readnone "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { norecurse nounwind readnone "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind readnone "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { norecurse nounwind readnone "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/DebugInfo/COFF/types-cvarargs.ll b/llvm/test/DebugInfo/COFF/types-cvarargs.ll
index 80b1252e8..68c7a7740 100644
--- a/llvm/test/DebugInfo/COFF/types-cvarargs.ll
+++ b/llvm/test/DebugInfo/COFF/types-cvarargs.ll
@@ -72,9 +72,9 @@ entry:
   ret i32 1, !dbg !35
 }
 
-attributes #0 = { noinline optnone uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noinline optnone uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone speculatable }
-attributes #2 = { noinline nounwind optnone uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { noinline nounwind optnone uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!2}
 !llvm.module.flags = !{!12, !13, !14, !15}
diff --git a/llvm/test/DebugInfo/COFF/types-data-members.ll b/llvm/test/DebugInfo/COFF/types-data-members.ll
index 842df34c5..87fde74b9 100644
--- a/llvm/test/DebugInfo/COFF/types-data-members.ll
+++ b/llvm/test/DebugInfo/COFF/types-data-members.ll
@@ -1,5 +1,6 @@
 ; RUN: llc < %s -filetype=obj | llvm-readobj - --codeview | FileCheck %s
 ; RUN: llc < %s | llvm-mc -filetype=obj --triple=x86_64-windows | llvm-readobj - --codeview | FileCheck %s
+; RUN: llc < %s | FileCheck %s --check-prefix=ASM
 
 ; C++ source to regenerate:
 ; $ cat t.cpp
@@ -389,6 +390,355 @@
 ; CHECK:   }
 ; CHECK: ]
 
+; ASM: .section	.debug$T,"dr"
+; ASM: .p2align	2
+; ASM: .long	4                       # Debug section magic
+; ASM: # ArgList (0x1000)
+; ASM: .short	0x6                     # Record length
+; ASM: .short	0x1201                  # Record kind: LF_ARGLIST
+; ASM: .long	0x0                     # NumArgs
+; ASM: # Procedure (0x1001)
+; ASM: .short	0xe                     # Record length
+; ASM: .short	0x1008                  # Record kind: LF_PROCEDURE
+; ASM: .long	0x3                     # ReturnType: void
+; ASM: .byte	0x0                     # CallingConvention: NearC
+; ASM: .byte	0x0                     # FunctionOptions
+; ASM: .short	0x0                     # NumParameters
+; ASM: .long	0x1000                  # ArgListType: ()
+; ASM: # FuncId (0x1002)
+; ASM: .short	0x16                    # Record length
+; ASM: .short	0x1601                  # Record kind: LF_FUNC_ID
+; ASM: .long	0x0                     # ParentScope
+; ASM: .long	0x1001                  # FunctionType: void ()
+; ASM: .asciz	"UseTypes"              # Name
+; ASM: .byte	243
+; ASM: .byte	242
+; ASM: .byte	241
+; ASM: # Struct (0x1003)
+; ASM: .short	0x2a                    # Record length
+; ASM: .short	0x1505                  # Record kind: LF_STRUCTURE
+; ASM: .short	0x0                     # MemberCount
+; ASM: .short	0x280                   # Properties ( ForwardReference (0x80) | HasUniqueName (0x200) )
+; ASM: .long	0x0                     # FieldList
+; ASM: .long	0x0                     # DerivedFrom
+; ASM: .long	0x0                     # VShape
+; ASM: .short	0x0                     # SizeOf
+; ASM: .asciz	"Struct"                # Name
+; ASM: .asciz	".?AUStruct@@"          # LinkageName
+; ASM: .byte	242
+; ASM: .byte	241
+; ASM: # Modifier (0x1004)
+; ASM: .short	0xa                     # Record length
+; ASM: .short	0x1001                  # Record kind: LF_MODIFIER
+; ASM: .long	0x74                    # ModifiedType: int
+; ASM: .short	0x1                     # Modifiers ( Const (0x1) )
+; ASM: .byte	242
+; ASM: .byte	241
+; ASM: # FieldList (0x1005)
+; ASM: .short	0x3e                    # Record length
+; ASM: .short	0x1203                  # Record kind: LF_FIELDLIST
+; ASM: .short	0x150d                  # Member kind: DataMember ( LF_MEMBER )
+; ASM: .short	0x3                     # Attrs: Public
+; ASM: .long	0x74                    # Type: int
+; ASM: .short	0x0                     # FieldOffset
+; ASM: .asciz	"s1"                    # Name
+; ASM: .byte	243
+; ASM: .byte	242
+; ASM: .byte	241
+; ASM: .short	0x150d                  # Member kind: DataMember ( LF_MEMBER )
+; ASM: .short	0x3                     # Attrs: Public
+; ASM: .long	0x74                    # Type: int
+; ASM: .short	0x4                     # FieldOffset
+; ASM: .asciz	"s2"                    # Name
+; ASM: .byte	243
+; ASM: .byte	242
+; ASM: .byte	241
+; ASM: .short	0x150d                  # Member kind: DataMember ( LF_MEMBER )
+; ASM: .short	0x3                     # Attrs: Public
+; ASM: .long	0x74                    # Type: int
+; ASM: .short	0x8                     # FieldOffset
+; ASM: .asciz	"s3"                    # Name
+; ASM: .byte	243
+; ASM: .byte	242
+; ASM: .byte	241
+; ASM: .short	0x150e                  # Member kind: StaticDataMember ( LF_STMEMBER )
+; ASM: .short	0x3                     # Attrs: Public
+; ASM: .long	0x1004                  # Type: const int
+; ASM: .asciz	"sdm"                   # Name
+; ASM: # Struct (0x1006)
+; ASM: .short	0x2a                    # Record length
+; ASM: .short	0x1505                  # Record kind: LF_STRUCTURE
+; ASM: .short	0x4                     # MemberCount
+; ASM: .short	0x200                   # Properties ( HasUniqueName (0x200) )
+; ASM: .long	0x1005                  # FieldList: <field list>
+; ASM: .long	0x0                     # DerivedFrom
+; ASM: .long	0x0                     # VShape
+; ASM: .short	0xc                     # SizeOf
+; ASM: .asciz	"Struct"                # Name
+; ASM: .asciz	".?AUStruct@@"          # LinkageName
+; ASM: .byte	242
+; ASM: .byte	241
+; ASM: # StringId (0x1007)
+; ASM: .short	0x1e                    # Record length
+; ASM: .short	0x1605                  # Record kind: LF_STRING_ID
+; ASM: .long	0x0                     # Id
+; ASM: .asciz	"D:\\src\\llvm\\build\\t.cpp" # StringData
+; ASM: # UdtSourceLine (0x1008)
+; ASM: .short	0xe                     # Record length
+; ASM: .short	0x1606                  # Record kind: LF_UDT_SRC_LINE
+; ASM: .long	0x1006                  # UDT: Struct
+; ASM: .long	0x1007                  # SourceFile: D:\src\llvm\build\t.cpp
+; ASM: .long	0x1                     # LineNumber
+; ASM: # Union (0x1009)
+; ASM: .short	0x1e                    # Record length
+; ASM: .short	0x1506                  # Record kind: LF_UNION
+; ASM: .short	0x0                     # MemberCount
+; ASM: .short	0x280                   # Properties ( ForwardReference (0x80) | HasUniqueName (0x200) )
+; ASM: .long	0x0                     # FieldList
+; ASM: .short	0x0                     # SizeOf
+; ASM: .asciz	"Union"                 # Name
+; ASM: .asciz	".?ATUnion@@"           # LinkageName
+; ASM: # FieldList (0x100A)
+; ASM: .short	0x1a                    # Record length
+; ASM: .short	0x1203                  # Record kind: LF_FIELDLIST
+; ASM: .short	0x150d                  # Member kind: DataMember ( LF_MEMBER )
+; ASM: .short	0x3                     # Attrs: Public
+; ASM: .long	0x74                    # Type: int
+; ASM: .short	0x0                     # FieldOffset
+; ASM: .asciz	"a"                     # Name
+; ASM: .short	0x150d                  # Member kind: DataMember ( LF_MEMBER )
+; ASM: .short	0x3                     # Attrs: Public
+; ASM: .long	0x40                    # Type: float
+; ASM: .short	0x0                     # FieldOffset
+; ASM: .asciz	"b"                     # Name
+; ASM: # Union (0x100B)
+; ASM: .short	0x1e                    # Record length
+; ASM: .short	0x1506                  # Record kind: LF_UNION
+; ASM: .short	0x2                     # MemberCount
+; ASM: .short	0x600                   # Properties ( HasUniqueName (0x200) | Sealed (0x400) )
+; ASM: .long	0x100a                  # FieldList: <field list>
+; ASM: .short	0x4                     # SizeOf
+; ASM: .asciz	"Union"                 # Name
+; ASM: .asciz	".?ATUnion@@"           # LinkageName
+; ASM: # UdtSourceLine (0x100C)
+; ASM: .short	0xe                     # Record length
+; ASM: .short	0x1606                  # Record kind: LF_UDT_SRC_LINE
+; ASM: .long	0x100b                  # UDT: Union
+; ASM: .long	0x1007                  # SourceFile: D:\src\llvm\build\t.cpp
+; ASM: .long	0x7                     # LineNumber
+; ASM: # Class (0x100D)
+; ASM: .short	0x26                    # Record length
+; ASM: .short	0x1504                  # Record kind: LF_CLASS
+; ASM: .short	0x0                     # MemberCount
+; ASM: .short	0x280                   # Properties ( ForwardReference (0x80) | HasUniqueName (0x200) )
+; ASM: .long	0x0                     # FieldList
+; ASM: .long	0x0                     # DerivedFrom
+; ASM: .long	0x0                     # VShape
+; ASM: .short	0x0                     # SizeOf
+; ASM: .asciz	"Class"                 # Name
+; ASM: .asciz	".?AVClass@@"           # LinkageName
+; ASM: # FieldList (0x100E)
+; ASM: .short	0x32                    # Record length
+; ASM: .short	0x1203                  # Record kind: LF_FIELDLIST
+; ASM: .short	0x150d                  # Member kind: DataMember ( LF_MEMBER )
+; ASM: .short	0x3                     # Attrs: Public
+; ASM: .long	0x74                    # Type: int
+; ASM: .short	0x0                     # FieldOffset
+; ASM: .asciz	"pub"                   # Name
+; ASM: .byte	242
+; ASM: .byte	241
+; ASM: .short	0x150d                  # Member kind: DataMember ( LF_MEMBER )
+; ASM: .short	0x1                     # Attrs: Private
+; ASM: .long	0x74                    # Type: int
+; ASM: .short	0x4                     # FieldOffset
+; ASM: .asciz	"priv"                  # Name
+; ASM: .byte	241
+; ASM: .short	0x150d                  # Member kind: DataMember ( LF_MEMBER )
+; ASM: .short	0x2                     # Attrs: Protected
+; ASM: .long	0x74                    # Type: int
+; ASM: .short	0x8                     # FieldOffset
+; ASM: .asciz	"prot"                  # Name
+; ASM: .byte	241
+; ASM: # Class (0x100F)
+; ASM: .short	0x26                    # Record length
+; ASM: .short	0x1504                  # Record kind: LF_CLASS
+; ASM: .short	0x3                     # MemberCount
+; ASM: .short	0x200                   # Properties ( HasUniqueName (0x200) )
+; ASM: .long	0x100e                  # FieldList: <field list>
+; ASM: .long	0x0                     # DerivedFrom
+; ASM: .long	0x0                     # VShape
+; ASM: .short	0xc                     # SizeOf
+; ASM: .asciz	"Class"                 # Name
+; ASM: .asciz	".?AVClass@@"           # LinkageName
+; ASM: # UdtSourceLine (0x1010)
+; ASM: .short	0xe                     # Record length
+; ASM: .short	0x1606                  # Record kind: LF_UDT_SRC_LINE
+; ASM: .long	0x100f                  # UDT: Class
+; ASM: .long	0x1007                  # SourceFile: D:\src\llvm\build\t.cpp
+; ASM: .long	0xb                     # LineNumber
+; ASM: # Struct (0x1011)
+; ASM: .short	0x36                    # Record length
+; ASM: .short	0x1505                  # Record kind: LF_STRUCTURE
+; ASM: .short	0x0                     # MemberCount
+; ASM: .short	0x280                   # Properties ( ForwardReference (0x80) | HasUniqueName (0x200) )
+; ASM: .long	0x0                     # FieldList
+; ASM: .long	0x0                     # DerivedFrom
+; ASM: .long	0x0                     # VShape
+; ASM: .short	0x0                     # SizeOf
+; ASM: .asciz	"DerivedClass"          # Name
+; ASM: .asciz	".?AUDerivedClass@@"    # LinkageName
+; ASM: .byte	242
+; ASM: .byte	241
+; ASM: # Pointer (0x1012)
+; ASM: .short	0xa                     # Record length
+; ASM: .short	0x1002                  # Record kind: LF_POINTER
+; ASM: .long	0x1004                  # PointeeType: const int
+; ASM: .long	0x1000c                 # Attrs: [ Type: Near64, Mode: Pointer, SizeOf: 8 ]
+; ASM: # VFTableShape (0x1013)
+; ASM: .short	0x6                     # Record length
+; ASM: .short	0xa                     # Record kind: LF_VTSHAPE
+; ASM: .short	0x1                     # VFEntryCount
+; ASM: .byte	0x0
+; ASM: .byte	241
+; ASM: # Pointer (0x1014)
+; ASM: .short	0xa                     # Record length
+; ASM: .short	0x1002                  # Record kind: LF_POINTER
+; ASM: .long	0x1013                  # PointeeType: <vftable 1 methods>
+; ASM: .long	0x1000c                 # Attrs: [ Type: Near64, Mode: Pointer, SizeOf: 8 ]
+; ASM: # FieldList (0x1015)
+; ASM: .short	0x32                    # Record length
+; ASM: .short	0x1203                  # Record kind: LF_FIELDLIST
+; ASM: .short	0x1400                  # Member kind: BaseClass ( LF_BCLASS )
+; ASM: .short	0x3                     # Attrs: Public
+; ASM: .long	0x1003                  # BaseType: Struct
+; ASM: .short	0x0                     # BaseOffset
+; ASM: .byte	242
+; ASM: .byte	241
+; ASM: .short	0x1401                  # Member kind: VirtualBaseClass ( LF_VBCLASS )
+; ASM: .short	0x3                     # Attrs: Public
+; ASM: .long	0x100d                  # BaseType: Class
+; ASM: .long	0x1012                  # VBPtrType: const int*
+; ASM: .short	0x0                     # VBPtrOffset
+; ASM: .short	0x1                     # VBTableIndex
+; ASM: .short	0x1409                  # Member kind: VFPtr ( LF_VFUNCTAB )
+; ASM: .short	0x0                     # Padding
+; ASM: .long	0x1014                  # Type: <vftable 1 methods>*
+; ASM: .short	0x150d                  # Member kind: DataMember ( LF_MEMBER )
+; ASM: .short	0x3                     # Attrs: Public
+; ASM: .long	0x74                    # Type: int
+; ASM: .short	0x18                    # FieldOffset
+; ASM: .asciz	"d"                     # Name
+; ASM: # Struct (0x1016)
+; ASM: .short	0x36                    # Record length
+; ASM: .short	0x1505                  # Record kind: LF_STRUCTURE
+; ASM: .short	0x4                     # MemberCount
+; ASM: .short	0x200                   # Properties ( HasUniqueName (0x200) )
+; ASM: .long	0x1015                  # FieldList: <field list>
+; ASM: .long	0x0                     # DerivedFrom
+; ASM: .long	0x0                     # VShape
+; ASM: .short	0x30                    # SizeOf
+; ASM: .asciz	"DerivedClass"          # Name
+; ASM: .asciz	".?AUDerivedClass@@"    # LinkageName
+; ASM: .byte	242
+; ASM: .byte	241
+; ASM: # UdtSourceLine (0x1017)
+; ASM: .short	0xe                     # Record length
+; ASM: .short	0x1606                  # Record kind: LF_UDT_SRC_LINE
+; ASM: .long	0x1016                  # UDT: DerivedClass
+; ASM: .long	0x1007                  # SourceFile: D:\src\llvm\build\t.cpp
+; ASM: .long	0x14                    # LineNumber
+; ASM: # Struct (0x1018)
+; ASM: .short	0x36                    # Record length
+; ASM: .short	0x1505                  # Record kind: LF_STRUCTURE
+; ASM: .short	0x0                     # MemberCount
+; ASM: .short	0x288                   # Properties ( ForwardReference (0x80) | HasUniqueName (0x200) | Nested (0x8) )
+; ASM: .long	0x0                     # FieldList
+; ASM: .long	0x0                     # DerivedFrom
+; ASM: .long	0x0                     # VShape
+; ASM: .short	0x0                     # SizeOf
+; ASM: .asciz	"Class::Nested"         # Name
+; ASM: .asciz	".?AUNested@Class@@"    # LinkageName
+; ASM: .byte	241
+; ASM: # FieldList (0x1019)
+; ASM: .short	0xe                     # Record length
+; ASM: .short	0x1203                  # Record kind: LF_FIELDLIST
+; ASM: .short	0x150d                  # Member kind: DataMember ( LF_MEMBER )
+; ASM: .short	0x3                     # Attrs: Public
+; ASM: .long	0x74                    # Type: int
+; ASM: .short	0x0                     # FieldOffset
+; ASM: .asciz	"n"                     # Name
+; ASM: # Struct (0x101A)
+; ASM: .short	0x36                    # Record length
+; ASM: .short	0x1505                  # Record kind: LF_STRUCTURE
+; ASM: .short	0x1                     # MemberCount
+; ASM: .short	0x208                   # Properties ( HasUniqueName (0x200) | Nested (0x8) )
+; ASM: .long	0x1019                  # FieldList: <field list>
+; ASM: .long	0x0                     # DerivedFrom
+; ASM: .long	0x0                     # VShape
+; ASM: .short	0x4                     # SizeOf
+; ASM: .asciz	"Class::Nested"         # Name
+; ASM: .asciz	".?AUNested@Class@@"    # LinkageName
+; ASM: .byte	241
+; ASM: # UdtSourceLine (0x101B)
+; ASM: .short	0xe                     # Record length
+; ASM: .short	0x1606                  # Record kind: LF_UDT_SRC_LINE
+; ASM: .long	0x101a                  # UDT: Class::Nested
+; ASM: .long	0x1007                  # SourceFile: D:\src\llvm\build\t.cpp
+; ASM: .long	0x17                    # LineNumber
+; ASM: # Pointer (0x101C)
+; ASM: .short	0xa                     # Record length
+; ASM: .short	0x1002                  # Record kind: LF_POINTER
+; ASM: .long	0x1011                  # PointeeType: DerivedClass
+; ASM: .long	0x1040c                 # Attrs: [ Type: Near64, Mode: Pointer, SizeOf: 8, isConst ]
+; ASM: # MemberFunction (0x101D)
+; ASM: .short	0x1a                    # Record length
+; ASM: .short	0x1009                  # Record kind: LF_MFUNCTION
+; ASM: .long	0x3                     # ReturnType: void
+; ASM: .long	0x1011                  # ClassType: DerivedClass
+; ASM: .long	0x101c                  # ThisType: DerivedClass* const
+; ASM: .byte	0x0                     # CallingConvention: NearC
+; ASM: .byte	0x0                     # FunctionOptions
+; ASM: .short	0x0                     # NumParameters
+; ASM: .long	0x1000                  # ArgListType: ()
+; ASM: .long	0x0                     # ThisAdjustment
+; ASM: # MemberFuncId (0x101E)
+; ASM: .short	0x26                    # Record length
+; ASM: .short	0x1602                  # Record kind: LF_MFUNC_ID
+; ASM: .long	0x1011                  # ClassType: DerivedClass
+; ASM: .long	0x101d                  # FunctionType: void DerivedClass::()
+; ASM: .asciz	"DerivedClass::DerivedClass" # Name
+; ASM: .byte	241
+; ASM: # Pointer (0x101F)
+; ASM: .short	0xa                     # Record length
+; ASM: .short	0x1002                  # Record kind: LF_POINTER
+; ASM: .long	0x1011                  # PointeeType: DerivedClass
+; ASM: .long	0x1000c                 # Attrs: [ Type: Near64, Mode: Pointer, SizeOf: 8 ]
+; ASM: # StringId (0x1020)
+; ASM: .short	0x1a                    # Record length
+; ASM: .short	0x1605                  # Record kind: LF_STRING_ID
+; ASM: .long	0x0                     # Id
+; ASM: .asciz	"D:\\src\\llvm\\build"  # StringData
+; ASM: .byte	242
+; ASM: .byte	241
+; ASM: # StringId (0x1021)
+; ASM: .short	0xe                     # Record length
+; ASM: .short	0x1605                  # Record kind: LF_STRING_ID
+; ASM: .long	0x0                     # Id
+; ASM: .asciz	"t.cpp"                 # StringData
+; ASM: .byte	242
+; ASM: .byte	241
+; ASM: # BuildInfo (0x1022)
+; ASM: .short	0x1a                    # Record length
+; ASM: .short	0x1603                  # Record kind: LF_BUILDINFO
+; ASM: .short	0x5                     # NumArgs
+; ASM: .long	0x1020                  # Argument: D:\src\llvm\build
+; ASM: .long	0x0                     # Argument
+; ASM: .long	0x1021                  # Argument: t.cpp
+; ASM: .long	0x0                     # Argument
+; ASM: .long	0x0                     # Argument
+; ASM: .byte	242
+; ASM: .byte	241
+
 ; ModuleID = 't.cpp'
 source_filename = "t.cpp"
 target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
@@ -458,9 +808,9 @@ ctor.skip_vbases:                                 ; preds = %ctor.init_vbases, %
   ret %struct.DerivedClass* %5, !dbg !62
 }
 
-attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
-attributes #2 = { inlinehint nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { inlinehint nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #3 = { nounwind }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/DebugInfo/COFF/types-method-ref-qualifiers.ll b/llvm/test/DebugInfo/COFF/types-method-ref-qualifiers.ll
index 64033f595..bfb67353f 100644
--- a/llvm/test/DebugInfo/COFF/types-method-ref-qualifiers.ll
+++ b/llvm/test/DebugInfo/COFF/types-method-ref-qualifiers.ll
@@ -40,7 +40,7 @@ entry:
 ; Function Attrs: nounwind readnone speculatable
 declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
 
-attributes #0 = { noinline nounwind optnone uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noinline nounwind optnone uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone speculatable }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/DebugInfo/COFF/types-recursive-struct.ll b/llvm/test/DebugInfo/COFF/types-recursive-struct.ll
index cdefec3d0..d8697e45f 100644
--- a/llvm/test/DebugInfo/COFF/types-recursive-struct.ll
+++ b/llvm/test/DebugInfo/COFF/types-recursive-struct.ll
@@ -157,7 +157,7 @@ entry:
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
 
-attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/DebugInfo/COFF/types-recursive-unnamed.ll b/llvm/test/DebugInfo/COFF/types-recursive-unnamed.ll
index 3aecebbb9..5e8edf5eb 100644
--- a/llvm/test/DebugInfo/COFF/types-recursive-unnamed.ll
+++ b/llvm/test/DebugInfo/COFF/types-recursive-unnamed.ll
@@ -43,7 +43,7 @@ entry:
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
 
-attributes #0 = { noinline nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noinline nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/DebugInfo/COFF/udts.ll b/llvm/test/DebugInfo/COFF/udts.ll
index 340d9deaf..e13bf7c1d 100644
--- a/llvm/test/DebugInfo/COFF/udts.ll
+++ b/llvm/test/DebugInfo/COFF/udts.ll
@@ -127,7 +127,7 @@ define float @"\01?g@@YAMPEAUS@@@Z"(%struct.S*) #0 !dbg !38 {
   ret float %9, !dbg !56
 }
 
-attributes #0 = { noinline nounwind optnone uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noinline nounwind optnone uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone speculatable }
 
 !llvm.dbg.cu = !{!2}
diff --git a/llvm/test/DebugInfo/COFF/unnamed.ll b/llvm/test/DebugInfo/COFF/unnamed.ll
index 5c21d052f..103eef348 100644
--- a/llvm/test/DebugInfo/COFF/unnamed.ll
+++ b/llvm/test/DebugInfo/COFF/unnamed.ll
@@ -156,7 +156,7 @@ entry:
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
 
-attributes #0 = { noinline nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noinline nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/DebugInfo/COFF/vframe-csr.ll b/llvm/test/DebugInfo/COFF/vframe-csr.ll
index 273a5a372..0d39155db 100644
--- a/llvm/test/DebugInfo/COFF/vframe-csr.ll
+++ b/llvm/test/DebugInfo/COFF/vframe-csr.ll
@@ -138,10 +138,10 @@ declare dso_local void @usecsrs(i32, i32) local_unnamed_addr #3
 ; Function Attrs: argmemonly nounwind
 declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture) #2
 
-attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone speculatable }
 attributes #2 = { argmemonly nounwind }
-attributes #3 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #4 = { nounwind }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/DebugInfo/COFF/vframe-fpo.ll b/llvm/test/DebugInfo/COFF/vframe-fpo.ll
index e0f82893b..20b3a437d 100644
--- a/llvm/test/DebugInfo/COFF/vframe-fpo.ll
+++ b/llvm/test/DebugInfo/COFF/vframe-fpo.ll
@@ -224,10 +224,10 @@ declare dso_local void @"?g@@YAXAAH00@Z"(i32* dereferenceable(4), i32* dereferen
 ; Function Attrs: argmemonly nounwind
 declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture) #1
 
-attributes #0 = { norecurse optsize "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { norecurse optsize "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { argmemonly nounwind }
 attributes #2 = { nounwind readnone speculatable }
-attributes #3 = { optsize "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { optsize "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #4 = { nounwind }
 attributes #5 = { optsize }
 
diff --git a/llvm/test/DebugInfo/COFF/vftables.ll b/llvm/test/DebugInfo/COFF/vftables.ll
index 4acacb343..cab7ab22d 100644
--- a/llvm/test/DebugInfo/COFF/vftables.ll
+++ b/llvm/test/DebugInfo/COFF/vftables.ll
@@ -414,11 +414,11 @@ entry:
 
 declare void @"\01?f@D@@UEAAXXZ"(i8*) unnamed_addr #3
 
-attributes #0 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
-attributes #2 = { inlinehint nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #3 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #4 = { uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { inlinehint nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #4 = { uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #5 = { nounwind }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/DebugInfo/COFF/virtual-method-kinds.ll b/llvm/test/DebugInfo/COFF/virtual-method-kinds.ll
index f75332671..b1d6d02c5 100644
--- a/llvm/test/DebugInfo/COFF/virtual-method-kinds.ll
+++ b/llvm/test/DebugInfo/COFF/virtual-method-kinds.ll
@@ -223,11 +223,11 @@ entry:
   ret void
 }
 
-attributes #0 = { uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { nobuiltin "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #2 = { inlinehint nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nobuiltin "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { inlinehint nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #3 = { nounwind readnone }
-attributes #4 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #4 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #5 = { builtin }
 attributes #6 = { nounwind }
 
diff --git a/llvm/test/DebugInfo/COFF/virtual-methods.ll b/llvm/test/DebugInfo/COFF/virtual-methods.ll
index f429f035a..9abd21b4b 100644
--- a/llvm/test/DebugInfo/COFF/virtual-methods.ll
+++ b/llvm/test/DebugInfo/COFF/virtual-methods.ll
@@ -328,11 +328,11 @@ entry:
   ret i32 %2, !dbg !102
 }
 
-attributes #0 = { uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
-attributes #2 = { inlinehint nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #3 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #4 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { inlinehint nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #4 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #5 = { nounwind }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/DebugInfo/COFF/vtable-optzn-array.ll b/llvm/test/DebugInfo/COFF/vtable-optzn-array.ll
index 2c9d2b763..4bc4d6120 100644
--- a/llvm/test/DebugInfo/COFF/vtable-optzn-array.ll
+++ b/llvm/test/DebugInfo/COFF/vtable-optzn-array.ll
@@ -89,8 +89,8 @@ entry:
   ret void
 }
 
-attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-features"="+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-features"="+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-features"="+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-features"="+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind }
 attributes #3 = { nounwind readnone }
 
diff --git a/llvm/test/DebugInfo/Generic/2010-05-03-DisableFramePtr.ll b/llvm/test/DebugInfo/Generic/2010-05-03-DisableFramePtr.ll
index 98d76d5e4..c15118a87 100644
--- a/llvm/test/DebugInfo/Generic/2010-05-03-DisableFramePtr.ll
+++ b/llvm/test/DebugInfo/Generic/2010-05-03-DisableFramePtr.ll
@@ -2,7 +2,7 @@
 ; Radar 7937664
 %struct.AppleEvent = type opaque
 
-define void @DisposeDMNotificationUPP(void (%struct.AppleEvent*)* %userUPP) "no-frame-pointer-elim-non-leaf" nounwind ssp {
+define void @DisposeDMNotificationUPP(void (%struct.AppleEvent*)* %userUPP) "frame-pointer"="non-leaf" nounwind ssp {
 entry:
   %userUPP_addr = alloca void (%struct.AppleEvent*)* ; <void (%struct.AppleEvent*)**> [#uses=1]
   %"alloca point" = bitcast i32 0 to i32          ; <i32> [#uses=0]
diff --git a/llvm/test/DebugInfo/Generic/PR20038.ll b/llvm/test/DebugInfo/Generic/PR20038.ll
index 3a3fe1212..372704128 100644
--- a/llvm/test/DebugInfo/Generic/PR20038.ll
+++ b/llvm/test/DebugInfo/Generic/PR20038.ll
@@ -115,8 +115,8 @@ entry:
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.declare(metadata, metadata, metadata) #2
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { alwaysinline nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { alwaysinline nounwind "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/DebugInfo/Generic/block-asan.ll b/llvm/test/DebugInfo/Generic/block-asan.ll
index ee9a52522..a686d427f 100644
--- a/llvm/test/DebugInfo/Generic/block-asan.ll
+++ b/llvm/test/DebugInfo/Generic/block-asan.ll
@@ -49,9 +49,9 @@ declare void @bar(i32) #2
 
 declare void @_Block_object_dispose(i8*, i32)
 
-attributes #0 = { nounwind ssp uwtable sanitize_address "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind ssp uwtable sanitize_address "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
-attributes #2 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #3 = { nounwind }
 
 !llvm.dbg.cu = !{!0}
@@ -70,7 +70,7 @@ attributes #3 = { nounwind }
 !10 = !{i32 1, !"PIC Level", i32 2}
 !11 = !{!"clang version 3.6.0 (trunk 223120) (llvm/trunk 223119)"}
 !12 = !DILocalVariable(name: "x", line: 4, scope: !4, file: !5, type: !13)
-!13 = !DICompositeType(tag: DW_TAG_structure_type, size: 224, flags: DIFlagBlockByrefStruct, file: !1, scope: !5, elements: !14)
+!13 = !DICompositeType(tag: DW_TAG_structure_type, size: 224, file: !1, scope: !5, elements: !14)
 !14 = !{!15, !17, !18, !20, !21}
 !15 = !DIDerivedType(tag: DW_TAG_member, name: "__isa", size: 64, align: 64, file: !1, scope: !5, baseType: !16)
 !16 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: null)
diff --git a/llvm/test/DebugInfo/Generic/constant-pointers.ll b/llvm/test/DebugInfo/Generic/constant-pointers.ll
index 8dc2dc38d..3ae06a5dc 100644
--- a/llvm/test/DebugInfo/Generic/constant-pointers.ll
+++ b/llvm/test/DebugInfo/Generic/constant-pointers.ll
@@ -24,7 +24,7 @@ entry:
   ret void, !dbg !18
 }
 
-attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!15, !16}
diff --git a/llvm/test/DebugInfo/Generic/cross-cu-inlining.ll b/llvm/test/DebugInfo/Generic/cross-cu-inlining.ll
index c41cbd990..d32ecae59 100644
--- a/llvm/test/DebugInfo/Generic/cross-cu-inlining.ll
+++ b/llvm/test/DebugInfo/Generic/cross-cu-inlining.ll
@@ -103,8 +103,8 @@ declare void @llvm.lifetime.start(i64, i8* nocapture) #3
 ; Function Attrs: nounwind
 declare void @llvm.lifetime.end(i64, i8* nocapture) #3
 
-attributes #0 = { uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { alwaysinline nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { alwaysinline nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind readnone }
 attributes #3 = { nounwind }
 
diff --git a/llvm/test/DebugInfo/Generic/cross-cu-linkonce.ll b/llvm/test/DebugInfo/Generic/cross-cu-linkonce.ll
index 7d39de97f..95280c370 100644
--- a/llvm/test/DebugInfo/Generic/cross-cu-linkonce.ll
+++ b/llvm/test/DebugInfo/Generic/cross-cu-linkonce.ll
@@ -43,7 +43,7 @@ define linkonce_odr i32 @_Z4funci(i32 %i) #0 !dbg !19 {
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
 
-attributes #0 = { inlinehint nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { inlinehint nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!9, !13}
diff --git a/llvm/test/DebugInfo/Generic/cu-range-hole.ll b/llvm/test/DebugInfo/Generic/cu-range-hole.ll
index 3358c0974..2499a9a78 100644
--- a/llvm/test/DebugInfo/Generic/cu-range-hole.ll
+++ b/llvm/test/DebugInfo/Generic/cu-range-hole.ll
@@ -48,7 +48,7 @@ entry:
   ret i32 %add, !dbg !16
 }
 
-attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !llvm.ident = !{!0, !0}
diff --git a/llvm/test/DebugInfo/Generic/cu-ranges.ll b/llvm/test/DebugInfo/Generic/cu-ranges.ll
index cb9066fce..9ac960859 100644
--- a/llvm/test/DebugInfo/Generic/cu-ranges.ll
+++ b/llvm/test/DebugInfo/Generic/cu-ranges.ll
@@ -42,7 +42,7 @@ entry:
   ret i32 %add, !dbg !18
 }
 
-attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/DebugInfo/Generic/dead-argument-order.ll b/llvm/test/DebugInfo/Generic/dead-argument-order.ll
index f97206b79..524413cc0 100644
--- a/llvm/test/DebugInfo/Generic/dead-argument-order.ll
+++ b/llvm/test/DebugInfo/Generic/dead-argument-order.ll
@@ -50,7 +50,7 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.value(metadata, metadata, metadata) #1
 
-attributes #0 = { nounwind readnone uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind readnone uwtable "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/DebugInfo/Generic/debug-info-always-inline.ll b/llvm/test/DebugInfo/Generic/debug-info-always-inline.ll
index 5eaa67522..bc1c023a0 100644
--- a/llvm/test/DebugInfo/Generic/debug-info-always-inline.ll
+++ b/llvm/test/DebugInfo/Generic/debug-info-always-inline.ll
@@ -105,10 +105,10 @@ entry:
 
 declare void @_Z3barv() #3
 
-attributes #0 = { alwaysinline nounwind sspstrong "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { alwaysinline nounwind sspstrong "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
-attributes #2 = { nounwind sspstrong "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #3 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { nounwind sspstrong "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!11, !12}
diff --git a/llvm/test/DebugInfo/Generic/debug-label.ll b/llvm/test/DebugInfo/Generic/debug-label.ll
index 57a0e952d..9139c9802 100644
--- a/llvm/test/DebugInfo/Generic/debug-label.ll
+++ b/llvm/test/DebugInfo/Generic/debug-label.ll
@@ -70,6 +70,7 @@ declare void @llvm.dbg.label(metadata)
 !9 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
 !10 = !DILabel(scope: !6, name: "top", file: !1, line: 4)
 !11 = !DILocation(line: 4, column: 1, scope: !6)
-!12 = !DILabel(scope: !6, name: "done", file: !1, line: 7)
+!12 = !DILabel(scope: !15, name: "done", file: !1, line: 7)
 !13 = !DILocation(line: 7, column: 1, scope: !6)
 !14 = !DILocation(line: 8, column: 3, scope: !6)
+!15 = !DILexicalBlockFile(discriminator: 2, file: !1, scope: !6)
diff --git a/llvm/test/DebugInfo/Generic/def-line.ll b/llvm/test/DebugInfo/Generic/def-line.ll
index ffd3fbda0..bea74fa60 100644
--- a/llvm/test/DebugInfo/Generic/def-line.ll
+++ b/llvm/test/DebugInfo/Generic/def-line.ll
@@ -61,8 +61,8 @@ entry:
   ret void, !dbg !22
 }
 
-attributes #0 = { uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!16, !17}
diff --git a/llvm/test/DebugInfo/Generic/directives-only.ll b/llvm/test/DebugInfo/Generic/directives-only.ll
index 484fa33e6..bc15400ca 100644
--- a/llvm/test/DebugInfo/Generic/directives-only.ll
+++ b/llvm/test/DebugInfo/Generic/directives-only.ll
@@ -34,8 +34,8 @@ entry:
 
 declare void @f1(...) #1
 
-attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!8, !9}
diff --git a/llvm/test/DebugInfo/Generic/discriminator.ll b/llvm/test/DebugInfo/Generic/discriminator.ll
index 64501cd56..a0f58d807 100644
--- a/llvm/test/DebugInfo/Generic/discriminator.ll
+++ b/llvm/test/DebugInfo/Generic/discriminator.ll
@@ -26,8 +26,8 @@ define void @_Z3foov() #0 !dbg !4 {
 
 declare void @_Z3xyzv() #1
 
-attributes #0 = { uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!8, !9}
diff --git a/llvm/test/DebugInfo/Generic/enum-types.ll b/llvm/test/DebugInfo/Generic/enum-types.ll
index 90c78ee56..5b481ae49 100644
--- a/llvm/test/DebugInfo/Generic/enum-types.ll
+++ b/llvm/test/DebugInfo/Generic/enum-types.ll
@@ -41,7 +41,7 @@ entry:
   ret void, !dbg !27
 }
 
-attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!0, !12}
diff --git a/llvm/test/DebugInfo/Generic/enum.ll b/llvm/test/DebugInfo/Generic/enum.ll
index 5ba5ea6dc..a2a676523 100644
--- a/llvm/test/DebugInfo/Generic/enum.ll
+++ b/llvm/test/DebugInfo/Generic/enum.ll
@@ -49,7 +49,7 @@ entry:
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
 
-attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!8}
diff --git a/llvm/test/DebugInfo/Generic/export-symbol-anonymous-class.ll b/llvm/test/DebugInfo/Generic/export-symbol-anonymous-class.ll
new file mode 100644
index 000000000..e76e6b55e
--- /dev/null
+++ b/llvm/test/DebugInfo/Generic/export-symbol-anonymous-class.ll
@@ -0,0 +1,43 @@
+; RUN: %llc_dwarf -O0 -filetype=obj < %s | llvm-dwarfdump -debug-info - | FileCheck %s
+;
+; struct A {
+;  // Anonymous class exports its symbols into A
+;  struct {
+;      int y;
+;  };
+; } a;
+;
+; CHECK: DW_TAG_structure_type
+; CHECK-NEXT: DW_AT_calling_convention	(DW_CC_pass_by_value)
+; CHECK-NEXT: DW_AT_name	("A")
+;
+; CHECK-NOT: NULL
+;
+; CHECK: DW_TAG_structure_type
+; CHECK-NEXT: DW_AT_export_symbols	(true)
+
+%struct.A = type { %struct.anon }
+%struct.anon = type { i32 }
+
+@a = global %struct.A zeroinitializer, align 4, !dbg !0
+
+!llvm.module.flags = !{!14, !15}
+!llvm.dbg.cu = !{!2}
+!llvm.ident = !{!16}
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "a", scope: !2, file: !3, line: 5, type: !6, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !3, producer: "clang version 10.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5, nameTableKind: GNU)
+!3 = !DIFile(filename: "simple_anon_class.cpp", directory: "/dir")
+!4 = !{}
+!5 = !{!0}
+!6 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "A", file: !3, line: 1, size: 32, flags: DIFlagTypePassByValue, elements: !7, identifier: "_ZTS1A")
+!7 = !{!8}
+!8 = !DIDerivedType(tag: DW_TAG_member, scope: !6, file: !3, line: 2, baseType: !9, size: 32)
+!9 = distinct !DICompositeType(tag: DW_TAG_structure_type, scope: !6, file: !3, line: 2, size: 32, flags: DIFlagExportSymbols | DIFlagTypePassByValue, elements: !10, identifier: "_ZTSN1AUt_E")
+!10 = !{!11}
+!11 = !DIDerivedType(tag: DW_TAG_member, name: "y", scope: !9, file: !3, line: 3, baseType: !12, size: 32)
+!12 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!14 = !{i32 2, !"Dwarf Version", i32 4}
+!15 = !{i32 2, !"Debug Info Version", i32 3}
+!16 = !{!"clang version 10.0.0"}
diff --git a/llvm/test/DebugInfo/Generic/extended-loc-directive.ll b/llvm/test/DebugInfo/Generic/extended-loc-directive.ll
index f0975d6f8..3ba4428c2 100644
--- a/llvm/test/DebugInfo/Generic/extended-loc-directive.ll
+++ b/llvm/test/DebugInfo/Generic/extended-loc-directive.ll
@@ -47,8 +47,8 @@ entry:
 
 declare void @f1(...) #1
 
-attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!8, !9}
diff --git a/llvm/test/DebugInfo/Generic/global-sra-array.ll b/llvm/test/DebugInfo/Generic/global-sra-array.ll
index aea6513f8..3796e4b83 100644
--- a/llvm/test/DebugInfo/Generic/global-sra-array.ll
+++ b/llvm/test/DebugInfo/Generic/global-sra-array.ll
@@ -70,7 +70,7 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
 ; Function Attrs: nounwind readnone speculatable
 declare void @llvm.dbg.value(metadata, metadata, metadata) #1
 
-attributes #0 = { nounwind optsize ssp uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+fxsr,+mmx,+sse,+sse2,+sse3,+sse4.1,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind optsize ssp uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+fxsr,+mmx,+sse,+sse2,+sse3,+sse4.1,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone speculatable }
 attributes #2 = { optsize }
 
diff --git a/llvm/test/DebugInfo/Generic/global.ll b/llvm/test/DebugInfo/Generic/global.ll
index 90d0465db..f8f7c3aa8 100644
--- a/llvm/test/DebugInfo/Generic/global.ll
+++ b/llvm/test/DebugInfo/Generic/global.ll
@@ -23,7 +23,7 @@ entry:
   ret i32 0, !dbg !12
 }
 
-attributes #0 = { nounwind readnone uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind readnone uwtable "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!7, !8}
diff --git a/llvm/test/DebugInfo/Generic/incorrect-variable-debugloc.ll b/llvm/test/DebugInfo/Generic/incorrect-variable-debugloc.ll
index e8a50d067..2fcce698b 100644
--- a/llvm/test/DebugInfo/Generic/incorrect-variable-debugloc.ll
+++ b/llvm/test/DebugInfo/Generic/incorrect-variable-debugloc.ll
@@ -327,9 +327,9 @@ declare void @__asan_unregister_globals(i64, i64)
 
 declare void @__sanitizer_cov_module_init(i64)
 
-attributes #0 = { noreturn sanitize_address "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { sanitize_address "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #2 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noreturn sanitize_address "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { sanitize_address "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #3 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/DebugInfo/Generic/incorrect-variable-debugloc1.ll b/llvm/test/DebugInfo/Generic/incorrect-variable-debugloc1.ll
index ce24b8af2..097e6e2f3 100644
--- a/llvm/test/DebugInfo/Generic/incorrect-variable-debugloc1.ll
+++ b/llvm/test/DebugInfo/Generic/incorrect-variable-debugloc1.ll
@@ -45,8 +45,8 @@ declare i32 @_Z4funcv() #1
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.value(metadata, metadata, metadata) #2
 
-attributes #0 = { uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { uwtable "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/DebugInfo/Generic/inline-no-debug-info.ll b/llvm/test/DebugInfo/Generic/inline-no-debug-info.ll
index 878a71211..e921a542d 100644
--- a/llvm/test/DebugInfo/Generic/inline-no-debug-info.ll
+++ b/llvm/test/DebugInfo/Generic/inline-no-debug-info.ll
@@ -49,7 +49,7 @@ entry:
   ret void, !dbg !12
 }
 
-attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!8, !9}
diff --git a/llvm/test/DebugInfo/Generic/inline-scopes.ll b/llvm/test/DebugInfo/Generic/inline-scopes.ll
index 6a6073fa0..38ad15da3 100644
--- a/llvm/test/DebugInfo/Generic/inline-scopes.ll
+++ b/llvm/test/DebugInfo/Generic/inline-scopes.ll
@@ -87,9 +87,9 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
 
 declare zeroext i1 @_Z1fv() #2
 
-attributes #0 = { uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
-attributes #2 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!13, !14}
diff --git a/llvm/test/DebugInfo/Generic/inlined-arguments.ll b/llvm/test/DebugInfo/Generic/inlined-arguments.ll
index e0980dbbd..1550c3509 100644
--- a/llvm/test/DebugInfo/Generic/inlined-arguments.ll
+++ b/llvm/test/DebugInfo/Generic/inlined-arguments.ll
@@ -43,8 +43,8 @@ declare void @_Z2f3i(i32) #1
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.value(metadata, metadata, metadata) #2
 
-attributes #0 = { uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { uwtable "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/DebugInfo/Generic/inlined-strings.ll b/llvm/test/DebugInfo/Generic/inlined-strings.ll
index 817b9b971..4de8dcba9 100644
--- a/llvm/test/DebugInfo/Generic/inlined-strings.ll
+++ b/llvm/test/DebugInfo/Generic/inlined-strings.ll
@@ -24,7 +24,7 @@ entry:
   ret i32 0, !dbg !12
 }
 
-attributes #0 = { nounwind readnone uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind readnone uwtable "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!7, !8}
diff --git a/llvm/test/DebugInfo/Generic/lto-comp-dir.ll b/llvm/test/DebugInfo/Generic/lto-comp-dir.ll
index 43f272376..3628f21f1 100644
--- a/llvm/test/DebugInfo/Generic/lto-comp-dir.ll
+++ b/llvm/test/DebugInfo/Generic/lto-comp-dir.ll
@@ -58,8 +58,8 @@ entry:
   ret i32 0, !dbg !21
 }
 
-attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0, !8}
 !llvm.module.flags = !{!16, !17}
diff --git a/llvm/test/DebugInfo/Generic/mainsubprogram.ll b/llvm/test/DebugInfo/Generic/mainsubprogram.ll
index e59b3940a..9666b274b 100644
--- a/llvm/test/DebugInfo/Generic/mainsubprogram.ll
+++ b/llvm/test/DebugInfo/Generic/mainsubprogram.ll
@@ -17,7 +17,7 @@ entry:
   ret i32 0, !dbg !10
 }
 
-attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!9, !11}
diff --git a/llvm/test/DebugInfo/Generic/member-order.ll b/llvm/test/DebugInfo/Generic/member-order.ll
index 79d714328..c83403ce1 100644
--- a/llvm/test/DebugInfo/Generic/member-order.ll
+++ b/llvm/test/DebugInfo/Generic/member-order.ll
@@ -37,7 +37,7 @@ entry:
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
 
-attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/DebugInfo/Generic/missing-abstract-variable.ll b/llvm/test/DebugInfo/Generic/missing-abstract-variable.ll
index e161b5b70..115076434 100644
--- a/llvm/test/DebugInfo/Generic/missing-abstract-variable.ll
+++ b/llvm/test/DebugInfo/Generic/missing-abstract-variable.ll
@@ -122,8 +122,8 @@ declare void @_Z1fi(i32) #1
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.value(metadata, metadata, metadata) #2
 
-attributes #0 = { uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { uwtable "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/DebugInfo/Generic/multiline.ll b/llvm/test/DebugInfo/Generic/multiline.ll
index a074d1015..1b15d8b77 100644
--- a/llvm/test/DebugInfo/Generic/multiline.ll
+++ b/llvm/test/DebugInfo/Generic/multiline.ll
@@ -54,8 +54,8 @@ entry:
 
 declare void @f1(...) #1
 
-attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!8, !9}
diff --git a/llvm/test/DebugInfo/Generic/namespace.ll b/llvm/test/DebugInfo/Generic/namespace.ll
index 09c5d4789..2cd13955a 100644
--- a/llvm/test/DebugInfo/Generic/namespace.ll
+++ b/llvm/test/DebugInfo/Generic/namespace.ll
@@ -279,7 +279,7 @@ entry:
   ret void, !dbg !74
 }
 
-attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/DebugInfo/Generic/namespace_function_definition.ll b/llvm/test/DebugInfo/Generic/namespace_function_definition.ll
index d053d6d97..651febfa4 100644
--- a/llvm/test/DebugInfo/Generic/namespace_function_definition.ll
+++ b/llvm/test/DebugInfo/Generic/namespace_function_definition.ll
@@ -24,7 +24,7 @@ entry:
   ret void, !dbg !11
 }
 
-attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!8, !9}
diff --git a/llvm/test/DebugInfo/Generic/namespace_inline_function_definition.ll b/llvm/test/DebugInfo/Generic/namespace_inline_function_definition.ll
index 48e8af989..354da0c8e 100644
--- a/llvm/test/DebugInfo/Generic/namespace_inline_function_definition.ll
+++ b/llvm/test/DebugInfo/Generic/namespace_inline_function_definition.ll
@@ -62,8 +62,8 @@ entry:
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.declare(metadata, metadata, metadata) #2
 
-attributes #0 = { uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { alwaysinline nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { alwaysinline nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/DebugInfo/Generic/piece-verifier.ll b/llvm/test/DebugInfo/Generic/piece-verifier.ll
index bb666bbd4..7a374ed67 100644
--- a/llvm/test/DebugInfo/Generic/piece-verifier.ll
+++ b/llvm/test/DebugInfo/Generic/piece-verifier.ll
@@ -16,7 +16,7 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.value(metadata, metadata, metadata) #1
 
-attributes #0 = { nounwind ssp uwtable "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" }
+attributes #0 = { nounwind ssp uwtable "frame-pointer"="all" }
 attributes #1 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/DebugInfo/Generic/recursive_inlining.ll b/llvm/test/DebugInfo/Generic/recursive_inlining.ll
index c8036e47e..10c9f486a 100644
--- a/llvm/test/DebugInfo/Generic/recursive_inlining.ll
+++ b/llvm/test/DebugInfo/Generic/recursive_inlining.ll
@@ -192,8 +192,8 @@ declare void @_Z3fn2iiii(i32, i32, i32, i32) #1
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.value(metadata, metadata, metadata) #2
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind readnone }
 attributes #3 = { nounwind }
 
diff --git a/llvm/test/DebugInfo/Generic/restrict.ll b/llvm/test/DebugInfo/Generic/restrict.ll
index 89b8053d4..453b9103e 100644
--- a/llvm/test/DebugInfo/Generic/restrict.ll
+++ b/llvm/test/DebugInfo/Generic/restrict.ll
@@ -28,7 +28,7 @@ entry:
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
 
-attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/DebugInfo/Generic/tu-composite.ll b/llvm/test/DebugInfo/Generic/tu-composite.ll
index b3a721763..09fe9da40 100644
--- a/llvm/test/DebugInfo/Generic/tu-composite.ll
+++ b/llvm/test/DebugInfo/Generic/tu-composite.ll
@@ -117,7 +117,7 @@ entry:
   ret void, !dbg !58
 }
 
-attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/DebugInfo/Generic/unconditional-branch.ll b/llvm/test/DebugInfo/Generic/unconditional-branch.ll
index b2bcdf15b..366f4c197 100644
--- a/llvm/test/DebugInfo/Generic/unconditional-branch.ll
+++ b/llvm/test/DebugInfo/Generic/unconditional-branch.ll
@@ -38,7 +38,7 @@ sw.default:                                       ; preds = %entry
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/DebugInfo/Generic/version.ll b/llvm/test/DebugInfo/Generic/version.ll
index dccff2fd3..158e023a5 100644
--- a/llvm/test/DebugInfo/Generic/version.ll
+++ b/llvm/test/DebugInfo/Generic/version.ll
@@ -13,7 +13,7 @@ entry:
   ret i32 0, !dbg !10
 }
 
-attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!9, !11}
diff --git a/llvm/test/DebugInfo/Inputs/gmlt.ll b/llvm/test/DebugInfo/Inputs/gmlt.ll
index a43395b3e..c0609c246 100644
--- a/llvm/test/DebugInfo/Inputs/gmlt.ll
+++ b/llvm/test/DebugInfo/Inputs/gmlt.ll
@@ -117,8 +117,8 @@ entry:
   ret void, !dbg !19
 }
 
-attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { alwaysinline nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { alwaysinline nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/DebugInfo/Inputs/line.ll b/llvm/test/DebugInfo/Inputs/line.ll
index 22c20efb2..e9ad363a0 100644
--- a/llvm/test/DebugInfo/Inputs/line.ll
+++ b/llvm/test/DebugInfo/Inputs/line.ll
@@ -33,7 +33,7 @@ land.end:                                         ; preds = %land.rhs, %entry
   ret i32 %conv, !dbg !13
 }
 
-attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!7, !8}
diff --git a/llvm/test/DebugInfo/Mips/dsr-fixed-objects.ll b/llvm/test/DebugInfo/Mips/dsr-fixed-objects.ll
index ebbe64a0c..e931bbfb9 100644
--- a/llvm/test/DebugInfo/Mips/dsr-fixed-objects.ll
+++ b/llvm/test/DebugInfo/Mips/dsr-fixed-objects.ll
@@ -24,8 +24,8 @@ declare void @foo(i32*)
 ; CHECK: DW_AT_name {{.*}}"e"
 ; CHECK: DW_TAG_variable
 ; CHECK-NEXT: DW_AT_location [DW_FORM_sec_offset] (
-; CHECK-NEXT:   [0x00000028, 0x0000002c): DW_OP_reg1 AT_64
-; CHECK-NEXT:   [0x0000002c, 0x00000048): DW_OP_breg29 SP_64+16)
+; CHECK-NEXT:   [0x00000028, 0x0000002c) ".text": DW_OP_reg1 AT_64
+; CHECK-NEXT:   [0x0000002c, 0x00000048) ".text": DW_OP_breg29 SP_64+16)
 ; CHECK-NEXT: DW_AT_name [DW_FORM_strp]     ( .debug_str[0x0000006b] = "x")
 
 define i32 @f0(i32 signext %a, i32 signext %b, i32 signext %c, i32 signext %d, i32 signext %e) !dbg !4 {
@@ -55,8 +55,8 @@ entry:
 
 ; CHECK: DW_TAG_variable
 ; CHECK-NEXT: DW_AT_location [DW_FORM_sec_offset]  (
-; CHECK-NEXT:   [0x00000080, 0x00000084): DW_OP_reg1 AT_64
-; CHECK-NEXT:   [0x00000084, 0x00000098): DW_OP_breg29 SP_64+16)
+; CHECK-NEXT:   [0x00000080, 0x00000084) ".text": DW_OP_reg1 AT_64
+; CHECK-NEXT:   [0x00000084, 0x00000098) ".text": DW_OP_breg29 SP_64+16)
 ; CHECK-NEXT: DW_AT_name [DW_FORM_strp]     ( .debug_str[0x0000006b] = "x")
 
 define i32 @f1(i32 signext %a, i32 signext %b, i32 signext %c, i32 signext %d, i32 signext %e) !dbg !15 {
diff --git a/llvm/test/DebugInfo/Mips/eh_frame.ll b/llvm/test/DebugInfo/Mips/eh_frame.ll
index 122d0a7f6..67d59837a 100644
--- a/llvm/test/DebugInfo/Mips/eh_frame.ll
+++ b/llvm/test/DebugInfo/Mips/eh_frame.ll
@@ -1,7 +1,7 @@
 ; RUN: llc -mtriple mips-unknown-linux-gnu -mattr=+micromips -relocation-model=static -O3 -filetype=obj -o - %s | \
-; RUN:     llvm-readelf -r | FileCheck %s --check-prefix=CHECK-READELF
+; RUN:     llvm-readelf -r | FileCheck %s --check-prefixes=CHECK-READELF,CHECK-READELF-STATIC
 ; RUN: llc -mtriple mips-unknown-linux-gnu -mattr=+micromips -relocation-model=pic -O3 -filetype=obj -o - %s | \
-; RUN:     llvm-readelf -r | FileCheck %s --check-prefix=CHECK-READELF
+; RUN:     llvm-readelf -r | FileCheck %s --check-prefixes=CHECK-READELF,CHECK-READELF-PIC
 ; RUN: llc -mtriple mips-unknown-linux-gnu -mattr=+micromips -relocation-model=static -O3 -filetype=obj -o - %s | \
 ; RUN:     llvm-objdump -s -j .gcc_except_table - | FileCheck %s --check-prefix=CHECK-EXCEPT-TABLE-STATIC
 ; RUN: llc -mtriple mips-unknown-linux-gnu -mattr=+micromips -relocation-model=pic -O3 -filetype=obj -o - %s | \
@@ -9,7 +9,8 @@
 
 ; CHECK-READELF: .rel.eh_frame
 ; CHECK-READELF: DW.ref.__gxx_personality_v0
-; CHECK-READELF-NEXT: .text
+; CHECK-READELF-STATIC-NEXT: R_MIPS_32 00000000 .text
+; CHECK-READELF-PIC-NEXT: R_MIPS_PC32
 ; CHECK-READELF-NEXT: .gcc_except_table
 
 ; CHECK-EXCEPT-TABLE-STATIC: 0000 ff9b1501 0c011500 00150e23 01231e00  ...........#.#..
diff --git a/llvm/test/DebugInfo/Mips/fn-call-line.ll b/llvm/test/DebugInfo/Mips/fn-call-line.ll
index ecd713387..f15881ed6 100644
--- a/llvm/test/DebugInfo/Mips/fn-call-line.ll
+++ b/llvm/test/DebugInfo/Mips/fn-call-line.ll
@@ -61,8 +61,8 @@ entry:
 
 declare void @f1(...) #1
 
-attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!8, !9}
diff --git a/llvm/test/DebugInfo/NVPTX/cu-range-hole.ll b/llvm/test/DebugInfo/NVPTX/cu-range-hole.ll
index e7852cc6d..035009380 100644
--- a/llvm/test/DebugInfo/NVPTX/cu-range-hole.ll
+++ b/llvm/test/DebugInfo/NVPTX/cu-range-hole.ll
@@ -269,12 +269,10 @@ entry:
 ; CHECK-NEXT: .b8 4                                // DW_AT_byte_size
 ; CHECK-NEXT: .b8 0                                // End Of Children Mark
 ; CHECK-NEXT: }
-; CHECK-NEXT: .section .debug_macinfo
-; CHECK-NEXT: {
-; CHECK-NEXT: .b8 0                                // End Of Macro List Mark
-; CHECK:      }
+; CHECK-NEXT: .section .debug_loc { }
+; CHECK-NOT: debug_
 
-attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !llvm.ident = !{!0, !0}
diff --git a/llvm/test/DebugInfo/NVPTX/dbg-declare-alloca.ll b/llvm/test/DebugInfo/NVPTX/dbg-declare-alloca.ll
index 4753071b2..6bfc3d1e9 100644
--- a/llvm/test/DebugInfo/NVPTX/dbg-declare-alloca.ll
+++ b/llvm/test/DebugInfo/NVPTX/dbg-declare-alloca.ll
@@ -125,12 +125,12 @@
 ; CHECK-NEXT: }
 ; CHECK-NEXT: .section .debug_info
 ; CHECK-NEXT: {
-; CHECK-NEXT: .b32 136                             // Length of Unit
+; CHECK-NEXT: .b32 134                             // Length of Unit
 ; CHECK-NEXT: .b8 2                                // DWARF version number
 ; CHECK-NEXT: .b8 0
 ; CHECK-NEXT: .b32 .debug_abbrev                   // Offset Into Abbrev. Section
 ; CHECK-NEXT: .b8 8                                // Address Size (in bytes)
-; CHECK-NEXT: .b8 1                                // Abbrev [1] 0xb:0x81 DW_TAG_compile_unit
+; CHECK-NEXT: .b8 1                                // Abbrev [1] 0xb:0x7f DW_TAG_compile_unit
 ; CHECK-NEXT: .b8 99                               // DW_AT_producer
 ; CHECK-NEXT: .b8 108
 ; CHECK-NEXT: .b8 97
@@ -151,7 +151,7 @@
 ; CHECK-NEXT: .b8 0
 ; CHECK-NEXT: .b64 Lfunc_begin0                    // DW_AT_low_pc
 ; CHECK-NEXT: .b64 Lfunc_end0                      // DW_AT_high_pc
-; CHECK-NEXT: .b8 2                                // Abbrev [2] 0x31:0x3e DW_TAG_subprogram
+; CHECK-NEXT: .b8 2                                // Abbrev [2] 0x31:0x3c DW_TAG_subprogram
 ; CHECK-NEXT: .b64 Lfunc_begin0                    // DW_AT_low_pc
 ; CHECK-NEXT: .b64 Lfunc_end0                      // DW_AT_high_pc
 ; CHECK-NEXT: .b8 1                                // DW_AT_frame_base
@@ -176,20 +176,18 @@
 ; CHECK-NEXT: .b8 3                                // DW_AT_decl_line
 ; CHECK-NEXT: .b8 1                                // DW_AT_prototyped
 ; CHECK-NEXT: .b8 1                                // DW_AT_external
-; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x58:0x16 DW_TAG_variable
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x58:0x14 DW_TAG_variable
 ; CHECK-NEXT: .b8 6                                // DW_AT_address_class
-; CHECK-NEXT: .b8 11                               // DW_AT_location
+; CHECK-NEXT: .b8 9                                // DW_AT_location
 ; CHECK-NEXT: .b8 3
 ; CHECK-NEXT: .b64 __local_depot0
-; CHECK-NEXT: .b8 35
-; CHECK-NEXT: .b8 0
 ; CHECK-NEXT: .b8 111                              // DW_AT_name
 ; CHECK-NEXT: .b8 0
 ; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
 ; CHECK-NEXT: .b8 4                                // DW_AT_decl_line
-; CHECK-NEXT: .b32 111                             // DW_AT_type
+; CHECK-NEXT: .b32 109                             // DW_AT_type
 ; CHECK-NEXT: .b8 0                                // End Of Children Mark
-; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x6f:0x15 DW_TAG_structure_type
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x6d:0x15 DW_TAG_structure_type
 ; CHECK-NEXT: .b8 70                               // DW_AT_name
 ; CHECK-NEXT: .b8 111
 ; CHECK-NEXT: .b8 111
@@ -197,17 +195,17 @@
 ; CHECK-NEXT: .b8 4                                // DW_AT_byte_size
 ; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
 ; CHECK-NEXT: .b8 1                                // DW_AT_decl_line
-; CHECK-NEXT: .b8 5                                // Abbrev [5] 0x77:0xc DW_TAG_member
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0x75:0xc DW_TAG_member
 ; CHECK-NEXT: .b8 120                              // DW_AT_name
 ; CHECK-NEXT: .b8 0
-; CHECK-NEXT: .b32 132                             // DW_AT_type
+; CHECK-NEXT: .b32 130                             // DW_AT_type
 ; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
 ; CHECK-NEXT: .b8 1                                // DW_AT_decl_line
 ; CHECK-NEXT: .b8 2                                // DW_AT_data_member_location
 ; CHECK-NEXT: .b8 35
 ; CHECK-NEXT: .b8 0
 ; CHECK-NEXT: .b8 0                                // End Of Children Mark
-; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x84:0x7 DW_TAG_base_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x82:0x7 DW_TAG_base_type
 ; CHECK-NEXT: .b8 105                              // DW_AT_name
 ; CHECK-NEXT: .b8 110
 ; CHECK-NEXT: .b8 116
diff --git a/llvm/test/DebugInfo/NVPTX/debug-addr-class.ll b/llvm/test/DebugInfo/NVPTX/debug-addr-class.ll
index e5ca8defc..c56fcda46 100644
--- a/llvm/test/DebugInfo/NVPTX/debug-addr-class.ll
+++ b/llvm/test/DebugInfo/NVPTX/debug-addr-class.ll
@@ -331,8 +331,6 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata)
 ; CHECK-NEXT: .b8 4                                   // DW_AT_byte_size
 ; CHECK-NEXT: .b8 0                                   // End Of Children Mark
 ; CHECK-NEXT: }
-; CHECK-NEXT: .section .debug_macinfo
-; CHECK-NEXT: {
-; CHECK-NEXT: .b8 0                                   // End Of Macro List Mark
-; CHECK:      }
+; CHECK-NEXT: .section .debug_loc { }
+; CHECK-NOT: debug_
 
diff --git a/llvm/test/DebugInfo/NVPTX/debug-file-loc.ll b/llvm/test/DebugInfo/NVPTX/debug-file-loc.ll
index 36fb59c72..09d47b384 100644
--- a/llvm/test/DebugInfo/NVPTX/debug-file-loc.ll
+++ b/llvm/test/DebugInfo/NVPTX/debug-file-loc.ll
@@ -86,10 +86,8 @@ bb:
 ; CHECK-NEXT: .b64 Lfunc_begin0                    // DW_AT_low_pc
 ; CHECK-NEXT: .b64 Lfunc_end1                      // DW_AT_high_pc
 ; CHECK-NEXT: }
-; CHECK-NEXT: .section .debug_macinfo
-; CHECK-NEXT: {
-; CHECK-NEXT: .b8 0                                // End Of Macro List Mark
-; CHECK:      }
+; CHECK-NEXT: .section .debug_loc { }
+; CHECK-NOT: debug_
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!8, !9}
diff --git a/llvm/test/DebugInfo/NVPTX/debug-info.ll b/llvm/test/DebugInfo/NVPTX/debug-info.ll
index 7164bc8ec..bc74f390d 100644
--- a/llvm/test/DebugInfo/NVPTX/debug-info.ll
+++ b/llvm/test/DebugInfo/NVPTX/debug-info.ll
@@ -8388,8 +8388,8 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT: .b8 37                               // DW_AT_call_column
 ; CHECK-NEXT: .b8 43                               // Abbrev [43] 0x2711:0x23 DW_TAG_inlined_subroutine
 ; CHECK-NEXT: .b32 9791                            // DW_AT_abstract_origin
-; CHECK-NEXT: .b64 Ltmp10                          // DW_AT_low_pc
-; CHECK-NEXT: .b64 Ltmp11                          // DW_AT_high_pc
+; CHECK-NEXT: .b64 Ltmp9                           // DW_AT_low_pc
+; CHECK-NEXT: .b64 Ltmp10                          // DW_AT_high_pc
 ; CHECK-NEXT: .b8 12                               // DW_AT_call_file
 ; CHECK-NEXT: .b8 8                                // DW_AT_call_line
 ; CHECK-NEXT: .b8 5                                // DW_AT_call_column
@@ -8401,10 +8401,8 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT: .b8 0                                // End Of Children Mark
 ; CHECK-NEXT: .b8 0                                // End Of Children Mark
 ; CHECK-NEXT: }
-; CHECK-NEXT: .section .debug_macinfo
-; CHECK-NEXT: {
-; CHECK-NEXT: .b8 0                                // End Of Macro List Mark
-; CHECK:      }
+; CHECK-NEXT: .section .debug_loc { }
+; CHECK-NOT: debug_
 
 ; Function Attrs: nounwind readnone
 declare i32 @llvm.nvvm.read.ptx.sreg.ctaid.x() #1
@@ -8418,7 +8416,7 @@ declare i32 @llvm.nvvm.read.ptx.sreg.tid.x() #1
 ; Function Attrs: nounwind readnone speculatable
 declare void @llvm.dbg.value(metadata, metadata, metadata) #2
 
-attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="sm_20" "target-features"="+ptx42" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="sm_20" "target-features"="+ptx42" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 attributes #2 = { nounwind readnone speculatable }
 attributes #3 = { nounwind }
diff --git a/llvm/test/DebugInfo/NVPTX/debug-loc-offset.ll b/llvm/test/DebugInfo/NVPTX/debug-loc-offset.ll
index f1510e1b4..31f346d30 100644
--- a/llvm/test/DebugInfo/NVPTX/debug-loc-offset.ll
+++ b/llvm/test/DebugInfo/NVPTX/debug-loc-offset.ll
@@ -43,8 +43,8 @@ declare void @llvm.dbg.value(metadata, metadata, metadata) #1
 ; CHECK: .loc [[CU2:[0-9]+]] 6 0
 ; CHECK: Lfunc_begin1:
 ; CHECK: .loc [[CU2]] 6 0
-; CHECK: //DEBUG_VALUE: baz:z <- {{[0-9]+}}
-; CHECK: //DEBUG_VALUE: baz:z <- {{[0-9]+}}
+; CHECK-NOT: //DEBUG_VALUE: baz:z
+; CHECK: //DEBUG_VALUE: baz:z <- undef
 ; CHECK: .loc [[CU2]] 10 0
 ; CHECK: ret;
 ; CHECK: }
@@ -87,9 +87,9 @@ if.end4:                                          ; preds = %if.then2, %if.end
 
 declare signext i8 @_ZN1A3fooEv(%struct.A*) #2
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
-attributes #2 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0, !9}
 !llvm.module.flags = !{!18, !19}
@@ -470,7 +470,5 @@ attributes #2 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "n
 ; CHECK-NEXT: .b8 4                                // DW_AT_byte_size
 ; CHECK-NEXT: .b8 0                                // End Of Children Mark
 ; CHECK-NEXT: }
-; CHECK-NEXT: .section .debug_macinfo
-; CHECK-NEXT: {
-; CHECK-NEXT: .b8 0                                // End Of Macro List Mark
-; CHECK:      }
+; CHECK-NEXT: .section .debug_loc { }
+; CHECK-NOT: debug_
diff --git a/llvm/test/DebugInfo/RISCV/dwarf-riscv-relocs.ll b/llvm/test/DebugInfo/RISCV/dwarf-riscv-relocs.ll
index c37336c36..d5cd3183e 100644
--- a/llvm/test/DebugInfo/RISCV/dwarf-riscv-relocs.ll
+++ b/llvm/test/DebugInfo/RISCV/dwarf-riscv-relocs.ll
@@ -5,15 +5,15 @@
 ; RUN:     FileCheck -check-prefix=DWARF-DUMP %s
 
 ; Check that we actually have relocations, otherwise this is kind of pointless.
-; READOBJ-RELOCS:  Section (8) .rela.debug_info {
-; READOBJ-RELOCS-NEXT:    0x0 R_RISCV_ADD32 - 0x0
-; READOBJ-RELOCS-NEXT:    0x0 R_RISCV_SUB32 - 0x0
-; READOBJ-RELOCS:  Section (11) .rela.debug_addr {
-; READOBJ-RELOCS-NEXT:    0x0 R_RISCV_ADD32 - 0x0
-; READOBJ-RELOCS-NEXT:    0x0 R_RISCV_SUB32 - 0x0
-; READOBJ-RELOCS:  Section (17) .rela.debug_line {
-; READOBJ-RELOCS-NEXT:    0x0 R_RISCV_ADD32 - 0xFFFFFFFC
-; READOBJ-RELOCS-NEXT:    0x0 R_RISCV_SUB32 .Lline_table_start0 0x0
+; READOBJ-RELOCS:  Section ({{.*}}) .rela.debug_info {
+; READOBJ-RELOCS:    0x1B R_RISCV_ADD32 - 0x0
+; READOBJ-RELOCS-NEXT:    0x1B R_RISCV_SUB32 - 0x0
+; READOBJ-RELOCS:  Section ({{.*}}) .rela.debug_frame {
+; READOBJ-RELOCS:    0x20 R_RISCV_ADD32 - 0x0
+; READOBJ-RELOCS-NEXT:    0x20 R_RISCV_SUB32 - 0x0
+; READOBJ-RELOCS:  Section ({{.*}}) .rela.debug_line {
+; READOBJ-RELOCS:    0x5A R_RISCV_ADD16 - 0x0
+; READOBJ-RELOCS-NEXT:    0x5A R_RISCV_SUB16 - 0x0
 
 ; Check that we can print the source, even with relocations.
 ; OBJDUMP-SOURCE: Disassembly of section .text:
@@ -85,7 +85,7 @@ entry:
   ret i32 0, !dbg !11
 }
 
-attributes #0 = { noinline nounwind optnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-features"="+relax" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noinline nounwind optnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-features"="+relax" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!3, !4, !5}
diff --git a/llvm/test/DebugInfo/RISCV/relax-debug-frame.ll b/llvm/test/DebugInfo/RISCV/relax-debug-frame.ll
new file mode 100644
index 000000000..4a767aaf6
--- /dev/null
+++ b/llvm/test/DebugInfo/RISCV/relax-debug-frame.ll
@@ -0,0 +1,58 @@
+; RUN: llc -filetype=obj -mtriple=riscv32 -mattr=+relax %s -o %t.o
+; RUN: llvm-readobj -r %t.o | FileCheck -check-prefix=RELAX %s
+; RUN: llvm-dwarfdump --debug-frame %t.o 2>&1 \
+; RUN:     | FileCheck -check-prefix=RELAX-DWARFDUMP %s
+;
+; RELAX:      Section ({{.*}}) .rela.eh_frame {
+; RELAX-NEXT:   0x1C R_RISCV_32_PCREL - 0x0
+; RELAX-NEXT:   0x20 R_RISCV_ADD32 - 0x0
+; RELAX-NEXT:   0x20 R_RISCV_SUB32 - 0x0
+; RELAX-NOT:  }
+; RELAX:        0x39 R_RISCV_SET6 - 0x0
+; RELAX-NEXT:   0x39 R_RISCV_SUB6 - 0x0
+;
+; RELAX-DWARFDUMP-NOT: error: failed to compute relocation
+; RELAX-DWARFDUMP: CIE
+; RELAX-DWARFDUMP: DW_CFA_advance_loc
+; RELAX-DWARFDUMP: DW_CFA_def_cfa_offset
+; RELAX-DWARFDUMP: DW_CFA_offset
+source_filename = "frame.c"
+
+; Function Attrs: noinline nounwind optnone
+define i32 @init() {
+entry:
+  ret i32 0
+}
+
+; Function Attrs: noinline nounwind optnone
+define i32 @foo(i32 signext %value) {
+entry:
+  %value.addr = alloca i32, align 4
+  store i32 %value, i32* %value.addr, align 4
+  %0 = load i32, i32* %value.addr, align 4
+  ret i32 %0
+}
+
+; Function Attrs: noinline nounwind optnone
+define i32 @bar() {
+entry:
+  %result = alloca i32, align 4
+  %v = alloca i32, align 4
+  %call = call i32 @init()
+  store i32 %call, i32* %v, align 4
+  %0 = load i32, i32* %v, align 4
+  %call1 = call i32 @foo(i32 signext %0)
+  store i32 %call1, i32* %result, align 4
+  %1 = load i32, i32* %result, align 4
+  ret i32 %1
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!1 = !DIFile(filename: "line.c", directory: "./")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
diff --git a/llvm/test/DebugInfo/Sparc/entry-value-complex-reg-expr.ll b/llvm/test/DebugInfo/Sparc/entry-value-complex-reg-expr.ll
new file mode 100644
index 000000000..0af5619b7
--- /dev/null
+++ b/llvm/test/DebugInfo/Sparc/entry-value-complex-reg-expr.ll
@@ -0,0 +1,79 @@
+; RUN: llc -debug-entry-values -filetype=asm -o - %s | FileCheck %s
+
+; Verify that the entry value covers both of the DW_OP_regx pieces. Previously
+; the size operand of the entry value would be hardcoded to one.
+;
+; XXX: Is this really what should be emitted, or should we instead emit one
+; entry value operation per DW_OP_regx? GDB can currently not understand
+; entry values containing complex expressions like this.
+
+target datalayout = "E-m:e-i64:64-n32:64-S128"
+target triple = "sparc64"
+
+; Based on the following C reproducer:
+;
+; volatile long double global;
+; extern void clobber();
+; int foo(long double p) {
+;   global = p;
+;   clobber();
+;   return 123;
+; }
+
+; CHECK:      .byte   243       ! DW_OP_GNU_entry_value
+; CHECK-NEXT: .byte   8         ! 8
+; CHECK-NEXT: .byte   144       ! sub-register DW_OP_regx
+; CHECK-NEXT: .byte   72        ! 72
+; CHECK-NEXT: .byte   147       ! DW_OP_piece
+; CHECK-NEXT: .byte   8         ! 8
+; CHECK-NEXT: .byte   144       ! sub-register DW_OP_regx
+; CHECK-NEXT: .byte   73        ! 73
+; CHECK-NEXT: .byte   147       ! DW_OP_piece
+; CHECK-NEXT: .byte   8         ! 8
+; CHECK-NEXT: .byte   159       ! DW_OP_stack_value
+
+@global = common global fp128 0xL00000000000000000000000000000000, align 16, !dbg !0
+
+; Function Attrs: nounwind
+define signext i32 @foo(fp128 %p) #0 !dbg !12 {
+entry:
+  call void @llvm.dbg.value(metadata fp128 %p, metadata !17, metadata !DIExpression()), !dbg !18
+  store volatile fp128 %p, fp128* @global, align 16, !dbg !19
+  tail call void @clobber(), !dbg !20
+  ret i32 123, !dbg !21
+}
+
+declare void @clobber()
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.value(metadata, metadata, metadata) #1
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind readnone speculatable willreturn }
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!8, !9, !10}
+!llvm.ident = !{!11}
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "global", scope: !2, file: !3, line: 1, type: !6, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 10.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5, nameTableKind: None)
+!3 = !DIFile(filename: "entry-value-complex-reg-expr.c", directory: "/")
+!4 = !{}
+!5 = !{!0}
+!6 = !DIDerivedType(tag: DW_TAG_volatile_type, baseType: !7)
+!7 = !DIBasicType(name: "long double", size: 128, encoding: DW_ATE_float)
+!8 = !{i32 2, !"Dwarf Version", i32 4}
+!9 = !{i32 2, !"Debug Info Version", i32 3}
+!10 = !{i32 1, !"wchar_size", i32 4}
+!11 = !{!"clang version 10.0.0"}
+!12 = distinct !DISubprogram(name: "foo", scope: !3, file: !3, line: 3, type: !13, scopeLine: 3, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !2, retainedNodes: !16)
+!13 = !DISubroutineType(types: !14)
+!14 = !{!15, !7}
+!15 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!16 = !{!17}
+!17 = !DILocalVariable(name: "p", arg: 1, scope: !12, file: !3, line: 3, type: !7)
+!18 = !DILocation(line: 0, scope: !12)
+!19 = !DILocation(line: 4, scope: !12)
+!20 = !DILocation(line: 5, scope: !12)
+!21 = !DILocation(line: 6, scope: !12)
diff --git a/llvm/test/DebugInfo/Sparc/gnu-window-save.ll b/llvm/test/DebugInfo/Sparc/gnu-window-save.ll
index fb9602bac..d81bcb6ed 100644
--- a/llvm/test/DebugInfo/Sparc/gnu-window-save.ll
+++ b/llvm/test/DebugInfo/Sparc/gnu-window-save.ll
@@ -48,8 +48,8 @@ entry:
 
 declare signext i32 @printf(i8*, ...) #1
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!9, !10}
diff --git a/llvm/test/DebugInfo/Sparc/subreg.ll b/llvm/test/DebugInfo/Sparc/subreg.ll
index bb5559873..afc1a00fb 100644
--- a/llvm/test/DebugInfo/Sparc/subreg.ll
+++ b/llvm/test/DebugInfo/Sparc/subreg.ll
@@ -1,6 +1,6 @@
 ; RUN: llc -filetype=obj -O0 < %s -mtriple sparc64-unknown-linux-gnu | llvm-dwarfdump - --debug-loc | FileCheck %s
 ; The undescribable 128-bit register should be split into two 64-bit registers.
-; CHECK: [{{.*}}, {{.*}}): DW_OP_regx D0, DW_OP_piece 0x8, DW_OP_regx D1, DW_OP_piece 0x8
+; CHECK: ({{.*}}, {{.*}}): DW_OP_regx D0, DW_OP_piece 0x8, DW_OP_regx D1, DW_OP_piece 0x8
 
 target datalayout = "E-m:e-i64:64-n32:64-S128"
 target triple = "sparc64"
diff --git a/llvm/test/DebugInfo/WebAssembly/dbg-loop-loc.ll b/llvm/test/DebugInfo/WebAssembly/dbg-loop-loc.ll
index e81ee2c4d..2a919fe85 100644
--- a/llvm/test/DebugInfo/WebAssembly/dbg-loop-loc.ll
+++ b/llvm/test/DebugInfo/WebAssembly/dbg-loop-loc.ll
@@ -79,7 +79,7 @@ for.end:                                          ; preds = %for.cond
 ; Function Attrs: nounwind readnone speculatable
 declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
 
-attributes #0 = { noinline nounwind optnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="generic" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noinline nounwind optnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="generic" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone speculatable }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/DebugInfo/WebAssembly/dbg-value-dwarfdump.ll b/llvm/test/DebugInfo/WebAssembly/dbg-value-dwarfdump.ll
new file mode 100644
index 000000000..f15ebe8e8
--- /dev/null
+++ b/llvm/test/DebugInfo/WebAssembly/dbg-value-dwarfdump.ll
@@ -0,0 +1,70 @@
+; RUN: llc < %s -filetype=obj -o - | llvm-dwarfdump - | FileCheck %s
+
+; Verify if dwarfdump contains DBG_VALUE associated with locals.
+; See also dgb-value-ti.ll test.
+
+target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
+target triple = "wasm32-unknown-unknown-wasm"
+
+define hidden i32 @fib(i32 %n) local_unnamed_addr #0 !dbg !7 {
+
+entry:
+; CHECK: DW_TAG_subprogram
+; CHECK: DW_TAG_variable
+
+  call void @llvm.dbg.value(metadata i32 1, metadata !16, metadata !DIExpression()), !dbg !19
+  %cmp8 = icmp sgt i32 %n, 0, !dbg !21
+  br i1 %cmp8, label %for.body, label %for.end, !dbg !24
+
+for.body:                                         ; preds = %entry, %for.body
+  %b.011 = phi i32 [ %add, %for.body ], [ 1, %entry ]
+  %a.010 = phi i32 [ %b.011, %for.body ], [ 0, %entry ]
+  %i.09 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+
+; CHECK: DW_OP_WASM_location 0x0 +[[LOCAL_1:[0-9]+]]
+  call void @llvm.dbg.value(metadata i32 %b.011, metadata !16, metadata !DIExpression()), !dbg !19
+
+; CHECK-NOT: DW_OP_WASM_location 0x0 +[[LOCAL_1]]
+; CHECK: DW_OP_WASM_location 0x0 +[[LOCAL_2:[0-9]+]]
+  %add = add nsw i32 %b.011, %a.010, !dbg !26
+  %inc = add nuw nsw i32 %i.09, 1, !dbg !28
+  call void @llvm.dbg.value(metadata i32 %add, metadata !16, metadata !DIExpression()), !dbg !19
+  %exitcond = icmp eq i32 %inc, %n, !dbg !21
+  br i1 %exitcond, label %for.end, label %for.body, !dbg !24, !llvm.loop !29
+
+for.end:                                          ; preds = %for.body, %entry
+  %b.0.lcssa = phi i32 [ 1, %entry ], [ %add, %for.body ], !dbg !31
+  call void @llvm.dbg.value(metadata i32 %b.0.lcssa, metadata !16, metadata !DIExpression()), !dbg !19
+  ret i32 %b.0.lcssa, !dbg !32
+}
+
+declare void @llvm.dbg.value(metadata, metadata, metadata) #1
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!4}
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 8.0.0 ", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "<unknown>", directory: "")
+!2 = !{}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!7 = distinct !DISubprogram(name: "fib", scope: !1, file: !1, line: 1, type: !8, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !11)
+!8 = !DISubroutineType(types: !9)
+!9 = !{!10, !10}
+!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!11 = !{!16}
+!16 = !DILocalVariable(name: "b", scope: !7, file: !1, line: 2, type: !10)
+!17 = !DILocation(line: 1, column: 13, scope: !7)
+!18 = !DILocation(line: 2, column: 13, scope: !7)
+!19 = !DILocation(line: 2, column: 20, scope: !7)
+!20 = !DILocation(line: 2, column: 7, scope: !7)
+!21 = !DILocation(line: 3, column: 17, scope: !22)
+!22 = distinct !DILexicalBlock(scope: !23, file: !1, line: 3, column: 3)
+!23 = distinct !DILexicalBlock(scope: !7, file: !1, line: 3, column: 3)
+!24 = !DILocation(line: 3, column: 3, scope: !23)
+!25 = !DILocation(line: 2, column: 10, scope: !7)
+!26 = !DILocation(line: 6, column: 7, scope: !27)
+!27 = distinct !DILexicalBlock(scope: !22, file: !1, line: 3, column: 27)
+!28 = !DILocation(line: 3, column: 23, scope: !22)
+!29 = distinct !{!29, !24, !30}
+!30 = !DILocation(line: 7, column: 3, scope: !23)
+!31 = !DILocation(line: 0, scope: !7)
+!32 = !DILocation(line: 8, column: 3, scope: !7)
diff --git a/llvm/test/DebugInfo/WebAssembly/dbg-value-ti.ll b/llvm/test/DebugInfo/WebAssembly/dbg-value-ti.ll
new file mode 100644
index 000000000..a99ca2d43
--- /dev/null
+++ b/llvm/test/DebugInfo/WebAssembly/dbg-value-ti.ll
@@ -0,0 +1,74 @@
+; RUN: llc < %s -stop-after=wasm-explicit-locals | FileCheck %s
+
+; Checks if DBG_VALUEs that correspond to new `local.{tee,set}` are
+; using `target-index(wasm-local-start)` operands.
+
+target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
+target triple = "wasm32-unknown-unknown-wasm"
+
+define hidden i32 @fib(i32 %n) local_unnamed_addr #0 !dbg !7 {
+; CHECK: body:
+entry:
+
+; CHECK: %[[REG1:.*]]:i32 = CONST_I32 1,
+; CHECK: LOCAL_SET_I32 [[LOOP_LOCAL:.*]], %[[REG1]],
+; CHECK: DBG_VALUE 1, $noreg,
+  call void @llvm.dbg.value(metadata i32 1, metadata !16, metadata !DIExpression()), !dbg !19
+  %cmp8 = icmp sgt i32 %n, 0, !dbg !21
+  br i1 %cmp8, label %for.body, label %for.end, !dbg !24
+
+for.body:                                         ; preds = %entry, %for.body
+  %b.011 = phi i32 [ %add, %for.body ], [ 1, %entry ]
+  %a.010 = phi i32 [ %b.011, %for.body ], [ 0, %entry ]
+  %i.09 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+
+; CHECK: %[[REG2:.*]]:i32 = LOCAL_GET_I32 [[LOOP_LOCAL]],
+; CHECK: %[[REG3:.*]]:i32 = LOCAL_TEE_I32 [[TMP_LOCAL:.*]], %[[REG2]],
+; CHECK: DBG_VALUE target-index(wasm-local-start) + [[TMP_LOCAL]], $noreg,
+  call void @llvm.dbg.value(metadata i32 %b.011, metadata !16, metadata !DIExpression()), !dbg !19
+
+; CHECK: %[[REG4:.*]]:i32 = nsw ADD_I32
+; CHECK: LOCAL_SET_I32 [[LOOP_LOCAL]], %[[REG4]],
+; CHECK: DBG_VALUE target-index(wasm-local-start) + [[LOOP_LOCAL]], $noreg,
+  %add = add nsw i32 %b.011, %a.010, !dbg !26
+  %inc = add nuw nsw i32 %i.09, 1, !dbg !28
+  call void @llvm.dbg.value(metadata i32 %add, metadata !16, metadata !DIExpression()), !dbg !19
+  %exitcond = icmp eq i32 %inc, %n, !dbg !21
+  br i1 %exitcond, label %for.end, label %for.body, !dbg !24, !llvm.loop !29
+
+for.end:                                          ; preds = %for.body, %entry
+  %b.0.lcssa = phi i32 [ 1, %entry ], [ %add, %for.body ], !dbg !31
+  call void @llvm.dbg.value(metadata i32 %b.0.lcssa, metadata !16, metadata !DIExpression()), !dbg !19
+  ret i32 %b.0.lcssa, !dbg !32
+}
+
+declare void @llvm.dbg.value(metadata, metadata, metadata) #1
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!4}
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 8.0.0 ", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "<unknown>", directory: "")
+!2 = !{}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!7 = distinct !DISubprogram(name: "fib", scope: !1, file: !1, line: 1, type: !8, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !11)
+!8 = !DISubroutineType(types: !9)
+!9 = !{!10, !10}
+!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!11 = !{!16}
+!16 = !DILocalVariable(name: "b", scope: !7, file: !1, line: 2, type: !10)
+!17 = !DILocation(line: 1, column: 13, scope: !7)
+!18 = !DILocation(line: 2, column: 13, scope: !7)
+!19 = !DILocation(line: 2, column: 20, scope: !7)
+!20 = !DILocation(line: 2, column: 7, scope: !7)
+!21 = !DILocation(line: 3, column: 17, scope: !22)
+!22 = distinct !DILexicalBlock(scope: !23, file: !1, line: 3, column: 3)
+!23 = distinct !DILexicalBlock(scope: !7, file: !1, line: 3, column: 3)
+!24 = !DILocation(line: 3, column: 3, scope: !23)
+!25 = !DILocation(line: 2, column: 10, scope: !7)
+!26 = !DILocation(line: 6, column: 7, scope: !27)
+!27 = distinct !DILexicalBlock(scope: !22, file: !1, line: 3, column: 27)
+!28 = !DILocation(line: 3, column: 23, scope: !22)
+!29 = distinct !{!29, !24, !30}
+!30 = !DILocation(line: 7, column: 3, scope: !23)
+!31 = !DILocation(line: 0, scope: !7)
+!32 = !DILocation(line: 8, column: 3, scope: !7)
diff --git a/llvm/test/DebugInfo/X86/DIModule.ll b/llvm/test/DebugInfo/X86/DIModule.ll
index de0830478..888343d41 100644
--- a/llvm/test/DebugInfo/X86/DIModule.ll
+++ b/llvm/test/DebugInfo/X86/DIModule.ll
@@ -6,7 +6,7 @@
 ; CHECK-NEXT: DW_AT_name {{.*}}"DebugModule"
 ; CHECK-NEXT: DW_AT_LLVM_config_macros {{.*}}"-DMODULES=0"
 ; CHECK-NEXT: DW_AT_LLVM_include_path {{.*}}"/llvm/tools/clang/test/Modules/Inputs"
-; CHECK-NEXT: DW_AT_LLVM_isysroot {{.*}}"/"
+; CHECK-NEXT: DW_AT_LLVM_sysroot {{.*}}"/"
 
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx"
@@ -20,7 +20,7 @@ target triple = "x86_64-apple-macosx"
 !2 = !{}
 !3 = !{!4}
 !4 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !0, entity: !5, file: !1, line: 5)
-!5 = !DIModule(scope: null, name: "DebugModule", configMacros: "-DMODULES=0", includePath: "/llvm/tools/clang/test/Modules/Inputs", isysroot: "/")
+!5 = !DIModule(scope: null, name: "DebugModule", configMacros: "-DMODULES=0", includePath: "/llvm/tools/clang/test/Modules/Inputs", sysroot: "/")
 !6 = !{i32 2, !"Dwarf Version", i32 4}
 !7 = !{i32 2, !"Debug Info Version", i32 3}
 !8 = !{!"LLVM version 3.7.0"}
diff --git a/llvm/test/DebugInfo/X86/DIModuleContext.ll b/llvm/test/DebugInfo/X86/DIModuleContext.ll
index b3c17aa48..c058b3cb7 100644
--- a/llvm/test/DebugInfo/X86/DIModuleContext.ll
+++ b/llvm/test/DebugInfo/X86/DIModuleContext.ll
@@ -25,7 +25,7 @@ target triple = "x86_64-apple-macosx"
 !5 = !{!0}
 !6 = !{!7}
 !7 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !2, entity: !8, file: !3, line: 11)
-!8 = !DIModule(scope: null, name: "Module", includePath: ".", isysroot: "/")
+!8 = !DIModule(scope: null, name: "Module", includePath: ".", sysroot: "/")
 !9 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !10, size: 64, align: 64)
 !10 = !DICompositeType(tag: DW_TAG_structure_type, name: "s", scope: !8, file: !3, line: 1, flags: DIFlagFwdDecl)
 !11 = !{i32 2, !"Dwarf Version", i32 2}
diff --git a/llvm/test/DebugInfo/X86/DW_AT_calling-convention.ll b/llvm/test/DebugInfo/X86/DW_AT_calling-convention.ll
index c717fa3d3..495f0f646 100644
--- a/llvm/test/DebugInfo/X86/DW_AT_calling-convention.ll
+++ b/llvm/test/DebugInfo/X86/DW_AT_calling-convention.ll
@@ -56,7 +56,7 @@ entry:
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.value(metadata, metadata, metadata) #1
 
-attributes #0 = { nounwind readnone "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind readnone "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!2}
diff --git a/llvm/test/DebugInfo/X86/DW_AT_deleted.ll b/llvm/test/DebugInfo/X86/DW_AT_deleted.ll
new file mode 100644
index 000000000..80774d696
--- /dev/null
+++ b/llvm/test/DebugInfo/X86/DW_AT_deleted.ll
@@ -0,0 +1,110 @@
+; RUN: llc < %s -filetype=obj -o %t
+; RUN: llvm-dwarfdump -v %t | FileCheck %s
+
+; C++ source to regenerate:
+; class deleted {
+; public:
+;   // Defaulted on purpose, so as to facilitate object creation
+;    deleted() = default;
+; 
+;   deleted(const deleted &) = delete;
+;   deleted &operator=(const deleted &) = delete;
+; 
+;   deleted(deleted &&) = delete;
+;   deleted &operator=(deleted &&) = delete;
+; 
+;   ~deleted() = default;
+; };
+; 
+; void foo() {
+;   deleted obj1;
+; }
+; $ clang++ -O0 -g -gdwarf-5 debug-info-deleted.cpp -c
+
+
+; CHECK: .debug_abbrev contents:
+
+; CHECK: [7] DW_TAG_subprogram   DW_CHILDREN_yes
+; CHECK: DW_AT_deleted   DW_FORM_flag_present
+; CHECK: [9] DW_TAG_subprogram   DW_CHILDREN_yes
+; CHECK: DW_AT_deleted   DW_FORM_flag_present
+
+; CHECK: .debug_info contents:
+
+; CHECK: DW_TAG_subprogram [7]
+; CHECK-NEXT: DW_AT_name [DW_FORM_strx1]    (indexed (00000006) string = "deleted") 
+; CHECK:  DW_AT_deleted [DW_FORM_flag_present]  (true)
+
+; CHECK: DW_TAG_subprogram [9]
+; CHECK-NEXT: DW_AT_linkage_name [DW_FORM_strx1]    (indexed (00000007) string = "_ZN7deletedaSERKS_") 
+; CHECK:  DW_AT_deleted [DW_FORM_flag_present]  (true)
+
+; CHECK: DW_TAG_subprogram [7]
+; CHECK-NEXT: DW_AT_name [DW_FORM_strx1]    (indexed (00000006) string = "deleted") 
+; CHECK:  DW_AT_deleted [DW_FORM_flag_present]  (true)
+
+; CHECK: DW_TAG_subprogram [9]
+; CHECK-NEXT: DW_AT_linkage_name [DW_FORM_strx1]    (indexed (00000009) string = "_ZN7deletedaSEOS_")
+; CHECK-NEXT: DW_AT_name [DW_FORM_strx1]    (indexed (00000008) string = "operator=")
+; CHECK:  DW_AT_deleted [DW_FORM_flag_present]  (true)
+
+; ModuleID = 'debug-info-deleted.cpp'
+source_filename = "debug-info-deleted.cpp"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+%class.deleted = type { i8 }
+
+; Function Attrs: noinline nounwind optnone uwtable
+define dso_local void @_Z3foov() #0 !dbg !7 {
+  %1 = alloca %class.deleted, align 1
+  call void @llvm.dbg.declare(metadata %class.deleted* %1, metadata !10, metadata !DIExpression()), !dbg !34
+  ret void, !dbg !35
+}
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
+
+attributes #0 = { noinline nounwind optnone uwtable }
+attributes #1 = { nounwind readnone speculatable willreturn }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 10.0.0 (https://github.com/llvm/llvm-project.git 715c47d5de9aa8860050992a7aaf27dca53f7f4a)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "debug-info-deleted.cpp", directory: "/home/sourabh/work/dwarf/c_c++/c++11", checksumkind: CSK_MD5, checksum: "49dc56907586479c64634558b060292d")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 5}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{!"clang version 10.0.0 (https://github.com/llvm/llvm-project.git 715c47d5de9aa8860050992a7aaf27dca53f7f4a)"}
+!7 = distinct !DISubprogram(name: "foo", linkageName: "_Z3foov", scope: !1, file: !1, line: 14, type: !8, scopeLine: 14, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !2)
+!8 = !DISubroutineType(types: !9)
+!9 = !{null}
+!10 = !DILocalVariable(name: "obj1", scope: !7, file: !1, line: 15, type: !11)
+!11 = distinct !DICompositeType(tag: DW_TAG_class_type, name: "deleted", file: !1, line: 1, size: 8, flags: DIFlagTypePassByReference, elements: !12, identifier: "_ZTS7deleted")
+!12 = !{!13, !17, !22, !26, !30, !33}
+!13 = !DISubprogram(name: "deleted", scope: !11, file: !1, line: 3, type: !14, scopeLine: 3, flags: DIFlagPublic | DIFlagPrototyped, spFlags: 0)
+!14 = !DISubroutineType(types: !15)
+!15 = !{null, !16}
+!16 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !11, size: 64, flags: DIFlagArtificial | DIFlagObjectPointer)
+!17 = !DISubprogram(name: "deleted", scope: !11, file: !1, line: 5, type: !18, scopeLine: 5, flags: DIFlagPublic | DIFlagPrototyped, spFlags: DISPFlagDeleted)
+!18 = !DISubroutineType(types: !19)
+!19 = !{null, !16, !20}
+!20 = !DIDerivedType(tag: DW_TAG_reference_type, baseType: !21, size: 64)
+!21 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !11)
+!22 = !DISubprogram(name: "operator=", linkageName: "_ZN7deletedaSERKS_", scope: !11, file: !1, line: 6, type: !23, scopeLine: 6, flags: DIFlagPublic | DIFlagPrototyped, spFlags: DISPFlagDeleted)
+!23 = !DISubroutineType(types: !24)
+!24 = !{!25, !16, !20}
+!25 = !DIDerivedType(tag: DW_TAG_reference_type, baseType: !11, size: 64)
+!26 = !DISubprogram(name: "deleted", scope: !11, file: !1, line: 8, type: !27, scopeLine: 8, flags: DIFlagPublic | DIFlagPrototyped, spFlags: DISPFlagDeleted)
+!27 = !DISubroutineType(types: !28)
+!28 = !{null, !16, !29}
+!29 = !DIDerivedType(tag: DW_TAG_rvalue_reference_type, baseType: !11, size: 64)
+!30 = !DISubprogram(name: "operator=", linkageName: "_ZN7deletedaSEOS_", scope: !11, file: !1, line: 9, type: !31, scopeLine: 9, flags: DIFlagPublic | DIFlagPrototyped, spFlags: DISPFlagDeleted)
+!31 = !DISubroutineType(types: !32)
+!32 = !{!25, !16, !29}
+!33 = !DISubprogram(name: "~deleted", scope: !11, file: !1, line: 11, type: !14, scopeLine: 11, flags: DIFlagPublic | DIFlagPrototyped, spFlags: 0)
+!34 = !DILocation(line: 15, column: 13, scope: !7)
+!35 = !DILocation(line: 16, column: 3, scope: !7)
diff --git a/llvm/test/DebugInfo/X86/DW_AT_location-reference.ll b/llvm/test/DebugInfo/X86/DW_AT_location-reference.ll
index 3ea6aea00..74b0afb2c 100644
--- a/llvm/test/DebugInfo/X86/DW_AT_location-reference.ll
+++ b/llvm/test/DebugInfo/X86/DW_AT_location-reference.ll
@@ -32,12 +32,11 @@
 ; CHECK: .debug_info contents:
 ; CHECK:      DW_TAG_variable
 ; CHECK-NEXT:   DW_AT_location [DW_FORM_sec_offset] (0x00000000
-; Check that the location contains only 4 ranges - this verifies that the 4th
-; and 5th ranges were successfully merged into a single range.
-; CHECK-NEXT:   [0x{{[0-9a-f]*}}, 0x{{[0-9a-f]*}}):
-; CHECK-NEXT:   [0x{{[0-9a-f]*}}, 0x{{[0-9a-f]*}}):
-; CHECK-NEXT:   [0x{{[0-9a-f]*}}, 0x{{[0-9a-f]*}}):
-; CHECK-NEXT:   [0x{{[0-9a-f]*}}, 0x{{[0-9a-f]*}}): {{.*}})
+; Check that the location contains only 4 ranges.
+; CHECK-NEXT:   [0x{{[0-9a-f]*}}, 0x{{[0-9a-f]*}})
+; CHECK-NEXT:   [0x{{[0-9a-f]*}}, 0x{{[0-9a-f]*}})
+; CHECK-NEXT:   [0x{{[0-9a-f]*}}, 0x{{[0-9a-f]*}})
+; CHECK-NEXT:   [0x{{[0-9a-f]*}}, 0x{{[0-9a-f]*}}){{.*}})
 ; CHECK-NEXT:   DW_AT_name {{.*}} "x"
 ; CHECK-NEXT:   DW_AT_decl_file
 ; CHECK-NEXT:   DW_AT_decl_line
diff --git a/llvm/test/DebugInfo/X86/DW_AT_stmt_list_sec_offset.ll b/llvm/test/DebugInfo/X86/DW_AT_stmt_list_sec_offset.ll
index 21edb51d8..4e451141a 100644
--- a/llvm/test/DebugInfo/X86/DW_AT_stmt_list_sec_offset.ll
+++ b/llvm/test/DebugInfo/X86/DW_AT_stmt_list_sec_offset.ll
@@ -25,7 +25,7 @@ entry:
   ret i32 0, !dbg !10
 }
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf"="true" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!9, !11}
diff --git a/llvm/test/DebugInfo/X86/PR26148.ll b/llvm/test/DebugInfo/X86/PR26148.ll
index c42b8e514..5d524d719 100644
--- a/llvm/test/DebugInfo/X86/PR26148.ll
+++ b/llvm/test/DebugInfo/X86/PR26148.ll
@@ -19,7 +19,7 @@
 ; AS in 26163, we only expect one range as the first one is zero sized.
 ;
 ;
-; CHECK: [0x0000000000000000, 0x000000000000000f): DW_OP_lit3, DW_OP_piece 0x4, DW_OP_lit0, DW_OP_piece 0x4
+; CHECK: (0x0000000000000000, 0x000000000000000f): DW_OP_lit3, DW_OP_piece 0x4, DW_OP_lit0, DW_OP_piece 0x4
 
 source_filename = "test/DebugInfo/X86/PR26148.ll"
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/DebugInfo/X86/PR37234.ll b/llvm/test/DebugInfo/X86/PR37234.ll
index a0c8b91d6..04005ca3f 100644
--- a/llvm/test/DebugInfo/X86/PR37234.ll
+++ b/llvm/test/DebugInfo/X86/PR37234.ll
@@ -20,9 +20,8 @@
 
 ; CHECK-LABEL: # %bb.{{.*}}:
 ; CHECK:        #DEBUG_VALUE: main:aa <- 0
-; CHECK: 	#DEBUG_VALUE: main:aa <- $[[REG:[0-9a-z]+]]
 ; CHECK: .LBB0_1:
-; CHECK:        #DEBUG_VALUE: main:aa <- $[[REG]]
+; CHECK: 	#DEBUG_VALUE: main:aa <- $[[REG:[0-9a-z]+]]
 ; CHECK:        je      .LBB0_4
 ; CHECK: # %bb.{{.*}}:
 ; CHECK:        #DEBUG_VALUE: main:aa <- $[[REG]]
diff --git a/llvm/test/DebugInfo/X86/addr_comments.ll b/llvm/test/DebugInfo/X86/addr_comments.ll
index 160f380be..58050d2b1 100644
--- a/llvm/test/DebugInfo/X86/addr_comments.ll
+++ b/llvm/test/DebugInfo/X86/addr_comments.ll
@@ -16,7 +16,7 @@ entry:
   ret void, !dbg !10
 }
 
-attributes #0 = { noinline nounwind optnone uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noinline nounwind optnone uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!3, !4, !5}
diff --git a/llvm/test/DebugInfo/X86/arguments.ll b/llvm/test/DebugInfo/X86/arguments.ll
index 1b7a5fbda..a642141e9 100644
--- a/llvm/test/DebugInfo/X86/arguments.ll
+++ b/llvm/test/DebugInfo/X86/arguments.ll
@@ -43,7 +43,7 @@ entry:
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
 
-attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf"="true" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/DebugInfo/X86/clang-module.ll b/llvm/test/DebugInfo/X86/clang-module.ll
index 39023502b..5f83a6448 100644
--- a/llvm/test/DebugInfo/X86/clang-module.ll
+++ b/llvm/test/DebugInfo/X86/clang-module.ll
@@ -24,7 +24,7 @@ target triple = "x86_64-apple-macosx10.12.0"
 !1 = !DIFile(filename: "modules.m", directory: "/")
 !3 = !{!4}
 !4 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !0, entity: !5, line: 122)
-!5 = !DIModule(scope: null, name: "Foo", includePath: ".", isysroot: "/")
+!5 = !DIModule(scope: null, name: "Foo", includePath: ".", sysroot: "/")
 !6 = distinct !DICompileUnit(language: DW_LANG_ObjC, file: !7, producer: "clang version 5.0.0 (trunk 308357) (llvm/trunk 308379)", isOptimized: true, runtimeVersion: 0, splitDebugFilename: "/Foo.pcm", emissionKind: FullDebug, dwoId: 1234)
 !7 = !DIFile(filename: "Foo", directory: ".")
 !15 = !{i32 2, !"Dwarf Version", i32 4}
diff --git a/llvm/test/DebugInfo/X86/codegenprep-addrsink.ll b/llvm/test/DebugInfo/X86/codegenprep-addrsink.ll
new file mode 100644
index 000000000..137316db6
--- /dev/null
+++ b/llvm/test/DebugInfo/X86/codegenprep-addrsink.ll
@@ -0,0 +1,67 @@
+; RUN: llc -start-before=codegenprepare -stop-after=codegenprepare -mtriple=x86_64-unknown-unknown %s -o - | FileCheck %s
+;
+; CGP duplicates address calculation into each basic block that contains loads
+; or stores, so that they can be folded into instruction memory operands for
+; example. dbg.value's should be redirected to identify such local address
+; computations, to give the best opportunity for variable locations to be
+; preserved.
+; This test has two dbg.values in it, one before and one after the relevant
+; memory instruction. Test that the one before does _not_ get updated (as that
+; would either make it use-before-def or shift when the variable appears), and
+; that the dbg.value after the memory instruction does get updated.
+
+define dso_local i8 @foo(i32 *%p, i32 %cond) !dbg !7 {
+entry:
+; There should be no dbg.values in this block.
+; CHECK-LABEL: entry:
+; CHECK-NOT:   dbg.value
+  %casted = bitcast i32 *%p to i8*
+  %arith = getelementptr i8, i8 *%casted, i32 3
+  %load1 = load i8, i8 *%arith
+  %cmpresult = icmp eq i32 %cond, 0
+  br i1 %cmpresult, label %next, label %ret
+
+next:
+; Address calcs should be duplicated into this block. One dbg.value should be
+; updated, and the other should not.
+; CHECK-LABEL: next:
+; CHECK:       dbg.value(metadata i8* %arith, metadata ![[DIVAR:[0-9]+]],
+; CHECK-SAME:    metadata !DIExpression()
+; CHECK-NEXT:  %[[CASTVAR:[0-9a-zA-Z]+]] = bitcast i32* %p to i8*
+; CHECK-NEXT:  %[[GEPVAR:[0-9a-zA-Z]+]] = getelementptr i8, i8* %[[CASTVAR]],
+; CHECK-SAME:                             i64 3
+; CHECK-NEXT:  %loaded = load i8, i8* %[[GEPVAR]]
+; CHECK-NEXT:  call void @llvm.dbg.value(metadata i8* %[[GEPVAR]],
+; CHECK-SAME:                            metadata ![[DIVAR]],
+  call void @llvm.dbg.value(metadata i8 *%arith, metadata !12, metadata !DIExpression()), !dbg !14
+  %loaded = load i8, i8 *%arith
+  call void @llvm.dbg.value(metadata i8 *%arith, metadata !12, metadata !DIExpression()), !dbg !14
+  ret i8 %loaded
+
+ret:
+  ret i8 0
+}
+
+; CHECK: ![[DIVAR]] = !DILocalVariable(name: "p",
+
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "test.cpp", directory: ".")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{!"clang version 8.0.0 (trunk 348209)"}
+!7 = distinct !DISubprogram(name: "foo", linkageName: "foo", scope: !1, file: !1, line: 4, type: !8, scopeLine: 5, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !11)
+!8 = !DISubroutineType(types: !9)
+!9 = !{null, !10}
+!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!11 = !{!12}
+!12 = !DILocalVariable(name: "p", arg: 1, scope: !7, file: !1, line: 4, type: !10)
+!14 = !DILocation(line: 4, column: 15, scope: !7)
+!20 = distinct !DILexicalBlock(scope: !7, file: !1, line: 8, column: 7)
diff --git a/llvm/test/DebugInfo/X86/coff_debug_info_type.ll b/llvm/test/DebugInfo/X86/coff_debug_info_type.ll
index 6c6987a2e..03230299d 100644
--- a/llvm/test/DebugInfo/X86/coff_debug_info_type.ll
+++ b/llvm/test/DebugInfo/X86/coff_debug_info_type.ll
@@ -26,7 +26,7 @@ entry:
   ret i32 0, !dbg !10
 }
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf"="true" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!9, !11}
diff --git a/llvm/test/DebugInfo/X86/coff_relative_names.ll b/llvm/test/DebugInfo/X86/coff_relative_names.ll
index 4d4ac87e1..0b5d2cb02 100644
--- a/llvm/test/DebugInfo/X86/coff_relative_names.ll
+++ b/llvm/test/DebugInfo/X86/coff_relative_names.ll
@@ -18,7 +18,7 @@ entry:
   ret i32 0, !dbg !10
 }
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf"="true" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!9, !11}
diff --git a/llvm/test/DebugInfo/X86/constant-loclist.ll b/llvm/test/DebugInfo/X86/constant-loclist.ll
index 17643d1ea..7bc68ad00 100644
--- a/llvm/test/DebugInfo/X86/constant-loclist.ll
+++ b/llvm/test/DebugInfo/X86/constant-loclist.ll
@@ -5,17 +5,17 @@
 ; CHECK: .debug_info contents:
 ; CHECK: DW_TAG_variable
 ; CHECK-NEXT: DW_AT_location [DW_FORM_data4]	(
-; CHECK-NEXT:   [0x{{.*}}, 0x{{.*}}): DW_OP_constu 0x4000000000000000)
+; CHECK-NEXT:   {{.*}}: DW_OP_constu 0x4000000000000000)
 ; CHECK-NEXT: DW_AT_name {{.*}}"d"
 ; CHECK: DW_TAG_variable
 ; CHECK-NEXT: DW_AT_location [DW_FORM_data4]	(
-; CHECK-NEXT:   [0x{{.*}}, 0x{{.*}}): DW_OP_consts +0
-; CHECK-NEXT:   [0x{{.*}}, 0x{{.*}}): DW_OP_consts +4611686018427387904)
+; CHECK-NEXT:   {{.*}}: DW_OP_consts +0
+; CHECK-NEXT:   {{.*}}: DW_OP_consts +4611686018427387904)
 ; CHECK-NEXT: DW_AT_name {{.*}}"i"
 ; CHECK: DW_TAG_variable
 ; CHECK-NEXT: DW_AT_location [DW_FORM_data4]	(
-; CHECK-NEXT:   [0x{{.*}}, 0x{{.*}}): DW_OP_lit0
-; CHECK-NEXT:   [0x{{.*}}, 0x{{.*}}): DW_OP_constu 0x4000000000000000)
+; CHECK-NEXT:   {{.*}}: DW_OP_lit0
+; CHECK-NEXT:   {{.*}}: DW_OP_constu 0x4000000000000000)
 ; CHECK-NEXT: DW_AT_name {{.*}}"u"
 
 source_filename = "test.c"
diff --git a/llvm/test/DebugInfo/X86/containing-type-extension-rust.ll b/llvm/test/DebugInfo/X86/containing-type-extension-rust.ll
index a54677766..6a0167905 100644
--- a/llvm/test/DebugInfo/X86/containing-type-extension-rust.ll
+++ b/llvm/test/DebugInfo/X86/containing-type-extension-rust.ll
@@ -76,10 +76,10 @@ top:
 ; std::rt::lang_start
 declare i64 @_ZN3std2rt10lang_start17h2626caf1112a00beE(void ()*, i64, i8**) unnamed_addr #3
 
-attributes #0 = { uwtable "no-frame-pointer-elim"="true" "probe-stack"="__rust_probestack" }
+attributes #0 = { uwtable "frame-pointer"="all" "probe-stack"="__rust_probestack" }
 attributes #1 = { nounwind readnone }
-attributes #2 = { "no-frame-pointer-elim"="true" }
-attributes #3 = { "no-frame-pointer-elim"="true" "probe-stack"="__rust_probestack" }
+attributes #2 = { "frame-pointer"="all" }
+attributes #3 = { "frame-pointer"="all" "probe-stack"="__rust_probestack" }
 
 !llvm.module.flags = !{!6, !7}
 !llvm.dbg.cu = !{!8}
diff --git a/llvm/test/DebugInfo/X86/cu-ranges-odr.ll b/llvm/test/DebugInfo/X86/cu-ranges-odr.ll
index 896fae715..a269bccb1 100644
--- a/llvm/test/DebugInfo/X86/cu-ranges-odr.ll
+++ b/llvm/test/DebugInfo/X86/cu-ranges-odr.ll
@@ -56,7 +56,7 @@ entry:
   ret void, !dbg !31
 }
 
-attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!11}
diff --git a/llvm/test/DebugInfo/X86/cu-ranges.ll b/llvm/test/DebugInfo/X86/cu-ranges.ll
index 90911d8ef..a4cbbe6c2 100644
--- a/llvm/test/DebugInfo/X86/cu-ranges.ll
+++ b/llvm/test/DebugInfo/X86/cu-ranges.ll
@@ -60,7 +60,7 @@ entry:
   ret i32 %add, !dbg !16
 }
 
-attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/DebugInfo/X86/dbg-addr-dse.ll b/llvm/test/DebugInfo/X86/dbg-addr-dse.ll
index 58b6514da..a817474c1 100644
--- a/llvm/test/DebugInfo/X86/dbg-addr-dse.ll
+++ b/llvm/test/DebugInfo/X86/dbg-addr-dse.ll
@@ -46,8 +46,8 @@ entry:
 }
 
 ; ASM-LABEL: f: # @f
-; ASM: #DEBUG_VALUE: f:x <- [DW_OP_plus_uconst [[OFF_X:[0-9]+]]] [$rsp+0]
-; ASM: movl    %ecx, [[OFF_X]](%rsp)
+; ASM: movl    %ecx, [[OFF_X:[0-9]+]](%rsp)
+; ASM: #DEBUG_VALUE: f:x <- [DW_OP_plus_uconst [[OFF_X]]] [$rsp+0]
 ; ASM: callq   escape
 ; ASM: #DEBUG_VALUE: f:x <- 1
 ; ASM: movl    $1, global(%rip)
diff --git a/llvm/test/DebugInfo/X86/dbg-value-dropped-instcombine.ll b/llvm/test/DebugInfo/X86/dbg-value-dropped-instcombine.ll
new file mode 100644
index 000000000..46f940e11
--- /dev/null
+++ b/llvm/test/DebugInfo/X86/dbg-value-dropped-instcombine.ll
@@ -0,0 +1,76 @@
+; RUN: opt -instcombine -S %s -o - | FileCheck %s
+
+; In pr40648 one of two dbg.values used to describe the variable bumble was
+; being dropped by instcombine. Test that both dbg.values survive instcombine.
+
+; $ clang -O0 -Xclang -disable-O0-optnone -g bees.c -emit-llvm -S -o - \
+;   | opt -opt-bisect-limit=10 -O2 -o -
+; $ cat bees.c
+; struct bees {
+;  int a;
+;  int b;
+; }
+
+; int global = 0;
+
+; struct bees foo(struct bees bumble)
+; {
+;   global = 1;
+;   int temp = bumble.a + bumble.b;
+;   return bumble;
+; }
+
+; CHECK: define dso_local i64 @foo
+; CHECK: @llvm.dbg.value(metadata i64 {{.*}}, metadata ![[BEE:[0-9]+]], metadata !DIExpression(DW_OP_LLVM_fragment, 0, 32)),
+; CHECK: @llvm.dbg.value(metadata i64 {{.*}}, metadata ![[BEE]], metadata !DIExpression({{.*}}, DW_OP_LLVM_fragment, 32, 32)),
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-unknown"
+
+@global = dso_local local_unnamed_addr global i32 0, align 4, !dbg !0
+
+define dso_local i64 @foo(i64 %bumble.coerce) local_unnamed_addr !dbg !11 {
+entry:
+  %bumble.sroa.0.0.extract.trunc = trunc i64 %bumble.coerce to i32
+  call void @llvm.dbg.value(metadata i32 %bumble.sroa.0.0.extract.trunc, metadata !18, metadata !DIExpression(DW_OP_LLVM_fragment, 0, 32)), !dbg !19
+  %bumble.sroa.3.0.extract.shift = lshr i64 %bumble.coerce, 32
+  %bumble.sroa.3.0.extract.trunc = trunc i64 %bumble.sroa.3.0.extract.shift to i32
+  call void @llvm.dbg.value(metadata i32 %bumble.sroa.3.0.extract.trunc, metadata !18, metadata !DIExpression(DW_OP_LLVM_fragment, 32, 32)), !dbg !19
+  store i32 1, i32* @global, align 4, !dbg !20
+  call void @llvm.dbg.value(metadata i32 undef, metadata !21, metadata !DIExpression()), !dbg !19
+  %retval.sroa.2.0.insert.ext = zext i32 %bumble.sroa.3.0.extract.trunc to i64, !dbg !22
+  %retval.sroa.2.0.insert.shift = shl i64 %retval.sroa.2.0.insert.ext, 32, !dbg !22
+  %retval.sroa.0.0.insert.ext = zext i32 %bumble.sroa.0.0.extract.trunc to i64, !dbg !22
+  %retval.sroa.0.0.insert.insert = or i64 %retval.sroa.2.0.insert.shift, %retval.sroa.0.0.insert.ext, !dbg !22
+  ret i64 %retval.sroa.0.0.insert.insert, !dbg !22
+}
+
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!7, !8, !9}
+!llvm.ident = !{!10}
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "global", scope: !2, file: !3, line: 6, type: !6, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 10.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5, nameTableKind: None)
+!3 = !DIFile(filename: "bees.c", directory: "/")
+!4 = !{}
+!5 = !{!0}
+!6 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!7 = !{i32 2, !"Dwarf Version", i32 4}
+!8 = !{i32 2, !"Debug Info Version", i32 3}
+!9 = !{i32 1, !"wchar_size", i32 4}
+!10 = !{!"clang version 10.0.0"}
+!11 = distinct !DISubprogram(name: "foo", scope: !3, file: !3, line: 8, type: !12, scopeLine: 9, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !2, retainedNodes: !4)
+!12 = !DISubroutineType(types: !13)
+!13 = !{!14, !14}
+!14 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "bees", file: !3, line: 1, size: 64, elements: !15)
+!15 = !{!16, !17}
+!16 = !DIDerivedType(tag: DW_TAG_member, name: "a", scope: !14, file: !3, line: 2, baseType: !6, size: 32)
+!17 = !DIDerivedType(tag: DW_TAG_member, name: "b", scope: !14, file: !3, line: 3, baseType: !6, size: 32, offset: 32)
+!18 = !DILocalVariable(name: "bumble", arg: 1, scope: !11, file: !3, line: 8, type: !14)
+!19 = !DILocation(line: 0, scope: !11)
+!20 = !DILocation(line: 10, column: 10, scope: !11)
+!21 = !DILocalVariable(name: "temp", scope: !11, file: !3, line: 11, type: !6)
+!22 = !DILocation(line: 12, column: 3, scope: !11)
diff --git a/llvm/test/DebugInfo/X86/dbg-value-frame-index.ll b/llvm/test/DebugInfo/X86/dbg-value-frame-index.ll
index a2cf2cc4e..d6fdada16 100644
--- a/llvm/test/DebugInfo/X86/dbg-value-frame-index.ll
+++ b/llvm/test/DebugInfo/X86/dbg-value-frame-index.ll
@@ -1,6 +1,6 @@
 ; RUN: llc -mtriple=x86_64-unknown-unknown -o - %s | FileCheck %s
 ; RUN: llc -mtriple=x86_64-unknown-unknown -filetype=obj < %s \
-; RUN:   | llvm-dwarfdump -v - | FileCheck %s --check-prefix=DWARF
+; RUN:   | llvm-dwarfdump - | FileCheck %s --check-prefix=DWARF
 
 define i1 @test() !dbg !4 {
 entry:
@@ -22,7 +22,7 @@ while.end:
 ; CHECK-LABEL: test
 ; To get the value of the variable, we need to do [$rsp+8], i.e:
 ; CHECK:       #DEBUG_VALUE: test:w <- [DW_OP_plus_uconst 8, DW_OP_deref] $rsp
-; DWARF:  DW_AT_location [DW_FORM_sec_offset] (
+; DWARF:  DW_AT_location (
 ; DWARF-NEXT:   [{{.*}}, {{.*}}): DW_OP_breg7 RSP+8)
 
 ; Note: A previous version of this test checked for `[DW_OP_plus_uconst 8] [$rsp+0]`,
diff --git a/llvm/test/DebugInfo/X86/dbg-value-regmask-clobber.ll b/llvm/test/DebugInfo/X86/dbg-value-regmask-clobber.ll
index 22d0a701a..440498a9d 100644
--- a/llvm/test/DebugInfo/X86/dbg-value-regmask-clobber.ll
+++ b/llvm/test/DebugInfo/X86/dbg-value-regmask-clobber.ll
@@ -1,5 +1,5 @@
 ; RUN: llc < %s | FileCheck %s --check-prefix=ASM
-; RUN: llc < %s -filetype=obj | llvm-dwarfdump -v - | FileCheck %s --check-prefix=DWARF
+; RUN: llc < %s -filetype=obj | llvm-dwarfdump - | FileCheck %s --check-prefix=DWARF
 
 ; Values in registers should be clobbered by calls, which use a regmask instead
 ; of individual register def operands.
@@ -22,9 +22,9 @@
 ; argc is the first formal parameter.
 ; DWARF: .debug_info contents:
 ; DWARF:  DW_TAG_formal_parameter
-; DWARF-NEXT:    DW_AT_location [DW_FORM_sec_offset]   ({{0x.*}}
+; DWARF-NEXT:    DW_AT_location ({{0x.*}}
 ; DWARF-NEXT:      [0x0000000000000000, 0x0000000000000013): DW_OP_reg2 RCX)
-; DWARF-NEXT:    DW_AT_name [DW_FORM_strp]     {{.*}} "argc"
+; DWARF-NEXT:    DW_AT_name ("argc")
 
 ; ModuleID = 't.cpp'
 source_filename = "test/DebugInfo/X86/dbg-value-regmask-clobber.ll"
diff --git a/llvm/test/DebugInfo/X86/dbg_value_direct.ll b/llvm/test/DebugInfo/X86/dbg_value_direct.ll
index f9e3d7667..9b2d63a5e 100644
--- a/llvm/test/DebugInfo/X86/dbg_value_direct.ll
+++ b/llvm/test/DebugInfo/X86/dbg_value_direct.ll
@@ -140,9 +140,9 @@ declare void @__asan_register_globals(i64, i64)
 
 declare void @__asan_unregister_globals(i64, i64)
 
-attributes #0 = { sanitize_address uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf"="true" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "ssp-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { sanitize_address uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "ssp-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
-attributes #2 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf"="true" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "ssp-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "ssp-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!22, !27}
diff --git a/llvm/test/DebugInfo/X86/dbgcall-site-64-bit-imms.ll b/llvm/test/DebugInfo/X86/dbgcall-site-64-bit-imms.ll
new file mode 100644
index 000000000..b698f1cdb
--- /dev/null
+++ b/llvm/test/DebugInfo/X86/dbgcall-site-64-bit-imms.ll
@@ -0,0 +1,56 @@
+; RUN: llc -O1 -debug-entry-values -filetype=obj -o - %s | llvm-dwarfdump - | FileCheck %s
+
+; Verify that the 64-bit call site immediates are not truncated.
+;
+; Reproducer for PR43525.
+
+; Based on the following C program:
+;
+; #include <stdint.h>
+;
+; extern void foo(int64_t);
+;
+; int main() {
+;   foo(INT64_C(0x1122334455667788));
+;   foo(INT32_C(-100));
+; }
+
+; CHECK: DW_AT_GNU_call_site_value (DW_OP_constu 0x1122334455667788)
+; CHECK: DW_AT_GNU_call_site_value (DW_OP_constu 0xffffffffffffff9c)
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Function Attrs: nounwind uwtable
+define i32 @main() !dbg !12 {
+entry:
+  tail call void @foo(i64 1234605616436508552), !dbg !16
+  tail call void @foo(i64 -100), !dbg !17
+  ret i32 0, !dbg !18
+}
+
+declare !dbg !4 void @foo(i64)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!8, !9, !10}
+!llvm.ident = !{!11}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 10.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !3, nameTableKind: None)
+!1 = !DIFile(filename: "dbgcall-site-long-imms.c", directory: "/")
+!2 = !{}
+!3 = !{!4}
+!4 = !DISubprogram(name: "foo", scope: !1, file: !1, line: 3, type: !5, flags: DIFlagPrototyped, spFlags: DISPFlagOptimized, retainedNodes: !2)
+!5 = !DISubroutineType(types: !6)
+!6 = !{null, !7}
+!7 = !DIBasicType(name: "long int", size: 64, encoding: DW_ATE_signed)
+!8 = !{i32 2, !"Dwarf Version", i32 4}
+!9 = !{i32 2, !"Debug Info Version", i32 3}
+!10 = !{i32 1, !"wchar_size", i32 4}
+!11 = !{!"clang version 10.0.0"}
+!12 = distinct !DISubprogram(name: "main", scope: !1, file: !1, line: 5, type: !13, scopeLine: 5, flags: DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2)
+!13 = !DISubroutineType(types: !14)
+!14 = !{!15}
+!15 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!16 = !DILocation(line: 6, scope: !12)
+!17 = !DILocation(line: 7, scope: !12)
+!18 = !DILocation(line: 8, scope: !12)
diff --git a/llvm/test/DebugInfo/X86/dbgcall-site-zero-valued-imms.ll b/llvm/test/DebugInfo/X86/dbgcall-site-zero-valued-imms.ll
new file mode 100644
index 000000000..9fe67f82a
--- /dev/null
+++ b/llvm/test/DebugInfo/X86/dbgcall-site-zero-valued-imms.ll
@@ -0,0 +1,68 @@
+; RUN: llc -O3 -debug-entry-values -filetype=obj -o - %s | llvm-dwarfdump - | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Verify that we produce call site entries for the zero-valued parameters.
+;
+; Based on the following reproducer:
+;
+; #include <stdint.h>
+; extern void callee(uint8_t, uint16_t, uint32_t, void *);
+; int caller() {
+;   callee(0, 0, 0, (void *)0);
+;   return 1;
+; }
+
+; CHECK: DW_TAG_GNU_call_site_parameter
+; CHECK-NEXT: DW_AT_location	(DW_OP_reg2 RCX)
+; CHECK-NEXT: DW_AT_GNU_call_site_value	(DW_OP_lit0)
+
+; CHECK: DW_TAG_GNU_call_site_parameter
+; CHECK-NEXT: DW_AT_location	(DW_OP_reg1 RDX)
+; CHECK-NEXT: DW_AT_GNU_call_site_value	(DW_OP_lit0)
+
+; CHECK: DW_TAG_GNU_call_site_parameter
+; CHECK-NEXT: DW_AT_location	(DW_OP_reg4 RSI)
+; CHECK-NEXT: DW_AT_GNU_call_site_value	(DW_OP_lit0)
+
+; CHECK: DW_TAG_GNU_call_site_parameter
+; CHECK-NEXT: DW_AT_location	(DW_OP_reg5 RDI)
+; CHECK-NEXT: DW_AT_GNU_call_site_value	(DW_OP_lit0)
+
+; Function Attrs: nounwind uwtable
+define i32 @caller() #0 !dbg !15 {
+entry:
+  tail call void @callee(i8 zeroext 0, i16 zeroext 0, i32 0, i8* null), !dbg !19
+  ret i32 1, !dbg !20
+}
+
+declare !dbg !5 void @callee(i8 zeroext, i16 zeroext, i32, i8*)
+
+attributes #0 = { nounwind uwtable }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!11, !12, !13}
+!llvm.ident = !{!14}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 10.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !3, nameTableKind: None)
+!1 = !DIFile(filename: "zero.c", directory: "/")
+!2 = !{}
+!3 = !{!4, !5}
+!4 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: null, size: 64)
+!5 = !DISubprogram(name: "callee", scope: !1, file: !1, line: 2, type: !6, flags: DIFlagPrototyped, spFlags: DISPFlagOptimized, retainedNodes: !2)
+!6 = !DISubroutineType(types: !7)
+!7 = !{null, !8, !9, !10, !4}
+!8 = !DIBasicType(name: "unsigned char", size: 8, encoding: DW_ATE_unsigned_char)
+!9 = !DIBasicType(name: "unsigned short", size: 16, encoding: DW_ATE_unsigned)
+!10 = !DIBasicType(name: "unsigned int", size: 32, encoding: DW_ATE_unsigned)
+!11 = !{i32 2, !"Dwarf Version", i32 4}
+!12 = !{i32 2, !"Debug Info Version", i32 3}
+!13 = !{i32 1, !"wchar_size", i32 4}
+!14 = !{!"clang version 10.0.0"}
+!15 = distinct !DISubprogram(name: "caller", scope: !1, file: !1, line: 3, type: !16, scopeLine: 3, flags: DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2)
+!16 = !DISubroutineType(types: !17)
+!17 = !{!18}
+!18 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!19 = !DILocation(line: 4, scope: !15)
+!20 = !DILocation(line: 5, scope: !15)
diff --git a/llvm/test/DebugInfo/X86/debug-dead-local-var.ll b/llvm/test/DebugInfo/X86/debug-dead-local-var.ll
index 21502bbc6..6e87ecec0 100644
--- a/llvm/test/DebugInfo/X86/debug-dead-local-var.ll
+++ b/llvm/test/DebugInfo/X86/debug-dead-local-var.ll
@@ -24,7 +24,7 @@ entry:
   ret i32 1, !dbg !21
 }
 
-attributes #0 = { nounwind readnone uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind readnone uwtable "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!18, !19}
diff --git a/llvm/test/DebugInfo/X86/debug-info-blocks.ll b/llvm/test/DebugInfo/X86/debug-info-blocks.ll
index 381f0adbc..0914fa686 100644
--- a/llvm/test/DebugInfo/X86/debug-info-blocks.ll
+++ b/llvm/test/DebugInfo/X86/debug-info-blocks.ll
@@ -259,7 +259,7 @@ define i32 @main() #0 !dbg !36 {
   ret i32 0, !dbg !109
 }
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 attributes #2 = { nonlazybind }
 attributes #3 = { nounwind }
diff --git a/llvm/test/DebugInfo/X86/debug-info-template-align.ll b/llvm/test/DebugInfo/X86/debug-info-template-align.ll
new file mode 100644
index 000000000..160d88cd0
--- /dev/null
+++ b/llvm/test/DebugInfo/X86/debug-info-template-align.ll
@@ -0,0 +1,63 @@
+; RUN: llc %s -filetype=obj -o - | llvm-dwarfdump -v - | FileCheck %s
+
+; C++ source to regenerate:
+
+;typedef char  __attribute__((__aligned__(64))) alchar;
+
+;int main(){
+;    alchar newChar;
+;}
+; $ clang++ -O0 -g -gdwarf-5 debug-info-template-align.cpp -c
+
+; CHECK: .debug_abbrev contents:
+
+; CHECK: [5] DW_TAG_typedef  DW_CHILDREN_no
+; CHECK:  DW_AT_alignment DW_FORM_udata
+
+; CHECK: .debug_info contents:
+
+;CHECK: DW_TAG_typedef [5]
+;CHECK: DW_AT_name {{.*}} "alchar"
+;CHECK-NEXT: DW_AT_alignment [DW_FORM_udata] (64)
+
+
+; ModuleID = '/dir/test.cpp'
+source_filename = "/dir/test.cpp"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Function Attrs: noinline norecurse nounwind optnone uwtable
+define dso_local i32 @main() #0 !dbg !7 {
+entry:
+  %newChar = alloca i8, align 64
+  call void @llvm.dbg.declare(metadata i8* %newChar, metadata !12, metadata !DIExpression()), !dbg !15
+  ret i32 0, !dbg !16
+}
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
+
+attributes #0 = { noinline norecurse nounwind optnone uwtable }
+attributes #1 = { nounwind readnone speculatable willreturn }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 10.0.0 ", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "/dir/test.cpp", directory: "/dir/", checksumkind: CSK_MD5, checksum: "872e252efdfcb9480b4bfaf8437f58ab")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 5}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{!"clang version 10.0.0 "}
+!7 = distinct !DISubprogram(name: "main", scope: !8, file: !8, line: 12, type: !9, scopeLine: 12, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !2)
+!8 = !DIFile(filename: "test.cpp", directory: "/dir", checksumkind: CSK_MD5, checksum: "872e252efdfcb9480b4bfaf8437f58ab")
+!9 = !DISubroutineType(types: !10)
+!10 = !{!11}
+!11 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!12 = !DILocalVariable(name: "newChar", scope: !7, file: !8, line: 13, type: !13)
+!13 = !DIDerivedType(tag: DW_TAG_typedef, name: "alchar", file: !8, line: 10, baseType: !14, align: 512)
+!14 = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_signed_char)
+!15 = !DILocation(line: 13, column: 10, scope: !7)
+!16 = !DILocation(line: 14, column: 1, scope: !7)
diff --git a/llvm/test/DebugInfo/X86/debug-loc-frame.ll b/llvm/test/DebugInfo/X86/debug-loc-frame.ll
index f3bcfcb73..90a38f1d2 100644
--- a/llvm/test/DebugInfo/X86/debug-loc-frame.ll
+++ b/llvm/test/DebugInfo/X86/debug-loc-frame.ll
@@ -4,7 +4,7 @@
 ; for the stack location directly instead of generating a register+offset indirection.
 
 ; RUN: llc -O2 -filetype=obj -disable-post-ra -mtriple=x86_64-unknown-linux-gnu < %s \
-; RUN: | llvm-dwarfdump -v - | FileCheck %s
+; RUN: | llvm-dwarfdump - | FileCheck %s
 ;
 ; int data = 17;
 ; int sum  = 0;
@@ -26,7 +26,7 @@
 ; CHECK:      DW_TAG_subprogram
 ; CHECK-NOT:  NULL
 ; CHECK:      DW_TAG_variable
-; CHECK:      DW_AT_location [DW_FORM_sec_offset] ({{.*}}
+; CHECK:      DW_AT_location ({{.*}}
 ; CHECK-NEXT:   [{{0x.*}}, {{0x.*}}): DW_OP_reg0 RAX
 ;
 ; Note: This is a location, so we don't want an extra DW_OP_deref at the end.
@@ -36,7 +36,7 @@
 ;     ... [rsp+4] DW_OP_deref
 ;
 ; CHECK-NEXT:   [{{0x.*}}, {{0x.*}}): DW_OP_breg7 RSP+4)
-; CHECK-NEXT: DW_AT_name {{.*}}"val"
+; CHECK-NEXT: DW_AT_name ("val")
 
 ; ModuleID = 'frame.c'
 source_filename = "frame.c"
diff --git a/llvm/test/DebugInfo/X86/debug-macinfo-split-dwarf.ll b/llvm/test/DebugInfo/X86/debug-macinfo-split-dwarf.ll
new file mode 100644
index 000000000..7e5be6ce4
--- /dev/null
+++ b/llvm/test/DebugInfo/X86/debug-macinfo-split-dwarf.ll
@@ -0,0 +1,85 @@
+; RUN: %llc_dwarf -O0 -split-dwarf-file=foo.dwo -filetype=obj < %s | llvm-dwarfdump -v - | FileCheck %s
+
+; CHECK-LABEL:.debug_info.dwo contents:
+; CHECK:     DW_AT_GNU_dwo_name  [DW_FORM_GNU_str_index]        (indexed (00000006) string = "foo.dwo")
+; CHECK-NEXT:     DW_AT_GNU_dwo_id [DW_FORM_data8] (0xe0f109905cbe1fe4)
+; CHECK-NEXT:     DW_AT_macro_info  [DW_FORM_sec_offset] (0x00000000)
+
+;CHECK-LABEL:.debug_macinfo.dwo contents:
+;CHECK-NEXT:  DW_MACINFO_start_file - lineno: 0 filenum: 1
+;CHECK-NEXT:    DW_MACINFO_start_file - lineno: 1 filenum: 2
+;CHECK-NEXT:      DW_MACINFO_define - lineno: 1 macro: define_1 12
+;CHECK-NEXT:    DW_MACINFO_end_file
+;CHECK-NEXT:    DW_MACINFO_start_file - lineno: 2 filenum: 3
+;CHECK-NEXT:      DW_MACINFO_define - lineno: 1 macro: define_2 14
+;CHECK-NEXT:    DW_MACINFO_end_file
+;CHECK-NEXT:  DW_MACINFO_end_file
+;CHECK-NEXT:  DW_MACINFO_define - lineno: 0 macro: __llvm__ 1
+;CHECK-NEXT:  DW_MACINFO_define - lineno: 0 macro: __clang__ 1
+;CHECK-NEXT:  DW_MACINFO_define - lineno: 0 macro: __clang_major__ 10
+;CHECK-NEXT:  DW_MACINFO_define - lineno: 0 macro: __clang_minor__ 0
+;CHECK-NEXT:  DW_MACINFO_define - lineno: 0 macro: __clang_patchlevel__ 0
+;CHECK-NEXT:  DW_MACINFO_define - lineno: 0 macro: __clang_version__ 10.0.0
+;CHECK-NEXT:  DW_MACINFO_define - lineno: 0 macro: __GNUC__ 4
+
+; ModuleID = 'debug-macro-split-dwarf.c'
+source_filename = "debug-macro-split-dwarf.c"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+; Function Attrs: noinline nounwind optnone uwtable
+define dso_local void @foo(i32 %a, i32 %b) #0 !dbg !25 {
+entry:
+  %a.addr = alloca i32, align 4
+  %b.addr = alloca i32, align 4
+  store i32 %a, i32* %a.addr, align 4
+  call void @llvm.dbg.declare(metadata i32* %a.addr, metadata !29, metadata !DIExpression()), !dbg !30
+  store i32 %b, i32* %b.addr, align 4
+  call void @llvm.dbg.declare(metadata i32* %b.addr, metadata !31, metadata !DIExpression()), !dbg !32
+  store i32 26, i32* %a.addr, align 4, !dbg !33
+  ret void, !dbg !34
+}
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
+
+attributes #0 = { noinline nounwind optnone uwtable }
+attributes #1 = { nounwind readnone speculatable willreturn }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!21, !22, !23}
+!llvm.ident = !{!24}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 10.0.0", isOptimized: false, runtimeVersion: 0, splitDebugFilename: "debug-macro-split-dwarf.dwo", emissionKind: FullDebug, enums: !2, macros: !3, nameTableKind: GNU)
+!1 = !DIFile(filename: "debug-macro-split-dwarf.c", directory: "/", checksumkind: CSK_MD5, checksum: "e74d0fa8f714535c1bac6da2ffbbd898")
+!2 = !{}
+!3 = !{!4, !14, !15, !16, !17, !18, !19, !20}
+!4 = !DIMacroFile(file: !1, nodes: !5)
+!5 = !{!6, !10}
+!6 = !DIMacroFile(line: 1, file: !7, nodes: !8)
+!7 = !DIFile(filename: "./1.h", directory: "/", checksumkind: CSK_MD5, checksum: "6185a3a5ae6eb7d1fd2692718f9d95e5")
+!8 = !{!9}
+!9 = !DIMacro(type: DW_MACINFO_define, line: 1, name: "define_1", value: "12")
+!10 = !DIMacroFile(line: 2, file: !11, nodes: !12)
+!11 = !DIFile(filename: "./2.h", directory: "/", checksumkind: CSK_MD5, checksum: "d48d124c86c1b50a32517884ff962f83")
+!12 = !{!13}
+!13 = !DIMacro(type: DW_MACINFO_define, line: 1, name: "define_2", value: "14")
+!14 = !DIMacro(type: DW_MACINFO_define, name: "__llvm__", value: "1")
+!15 = !DIMacro(type: DW_MACINFO_define, name: "__clang__", value: "1")
+!16 = !DIMacro(type: DW_MACINFO_define, name: "__clang_major__", value: "10")
+!17 = !DIMacro(type: DW_MACINFO_define, name: "__clang_minor__", value: "0")
+!18 = !DIMacro(type: DW_MACINFO_define, name: "__clang_patchlevel__", value: "0")
+!19 = !DIMacro(type: DW_MACINFO_define, name: "__clang_version__", value: "10.0.0")
+!20 = !DIMacro(type: DW_MACINFO_define, name: "__GNUC__", value: "4")
+!21 = !{i32 7, !"Dwarf Version", i32 4}
+!22 = !{i32 2, !"Debug Info Version", i32 3}
+!23 = !{i32 1, !"wchar_size", i32 4}
+!24 = !{!"clang version 10.0.0 "}
+!25 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 4, type: !26, scopeLine: 4, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !2)
+!26 = !DISubroutineType(types: !27)
+!27 = !{null, !28, !28}
+!28 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!29 = !DILocalVariable(name: "a", arg: 1, scope: !25, file: !1, line: 4, type: !28)
+!30 = !DILocation(line: 4, column: 14, scope: !25)
+!31 = !DILocalVariable(name: "b", arg: 2, scope: !25, file: !1, line: 4, type: !28)
+!32 = !DILocation(line: 4, column: 21, scope: !25)
+!33 = !DILocation(line: 5, column: 4, scope: !25)
+!34 = !DILocation(line: 6, column: 1, scope: !25)
diff --git a/llvm/test/DebugInfo/X86/debug-macro.ll b/llvm/test/DebugInfo/X86/debug-macro.ll
index e286b8a73..6a5ae1e70 100644
--- a/llvm/test/DebugInfo/X86/debug-macro.ll
+++ b/llvm/test/DebugInfo/X86/debug-macro.ll
@@ -9,7 +9,7 @@
 ; CHECK: DW_TAG_compile_unit
 ; CHECK-NOT: DW_TAG
 ; CHECK:   DW_AT_name {{.*}}"debug-macro1.cpp")
-; CHECK:   DW_AT_macro_info {{.*}}(0x00000044)
+; CHECK:   DW_AT_macro_info {{.*}}(0x00000045)
 ; CHECK: DW_TAG_compile_unit
 ; CHECK-NOT: DW_TAG
 ; CHECK:   DW_AT_name {{.*}}"debug-macro2.cpp")
@@ -24,7 +24,8 @@
 ; CHECK-NEXT:   DW_MACINFO_end_file
 ; CHECK-NEXT:   DW_MACINFO_undef - lineno: 10 macro: NameUndef2
 ; CHECK-NEXT: DW_MACINFO_end_file
-; CHECK-NEXT: DW_MACINFO_start_file - lineno: 0 filenum: 1
+
+; CHECK: DW_MACINFO_start_file - lineno: 0 filenum: 1
 ; CHECK-NEXT: DW_MACINFO_end_file
 
 ; CHECK-LABEL: .debug_line contents:
diff --git a/llvm/test/DebugInfo/X86/debug-ranges-offset.ll b/llvm/test/DebugInfo/X86/debug-ranges-offset.ll
index ce74406a1..c3bd3963d 100644
--- a/llvm/test/DebugInfo/X86/debug-ranges-offset.ll
+++ b/llvm/test/DebugInfo/X86/debug-ranges-offset.ll
@@ -192,8 +192,8 @@ declare i8* @__msan_memset(i8*, i32, i64)
 ; Function Attrs: nounwind
 declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i1) #3
 
-attributes #0 = { sanitize_memory uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { nobuiltin "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { sanitize_memory uwtable "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nobuiltin "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind readnone }
 attributes #3 = { nounwind }
 attributes #4 = { builtin }
diff --git a/llvm/test/DebugInfo/X86/debug_addr.ll b/llvm/test/DebugInfo/X86/debug_addr.ll
index e6dbe7d02..424313ac2 100644
--- a/llvm/test/DebugInfo/X86/debug_addr.ll
+++ b/llvm/test/DebugInfo/X86/debug_addr.ll
@@ -28,9 +28,9 @@
 ; DWARF5: .debug_info contents:
 ; DWARF5: Compile Unit:{{.*}}version = 0x0005
 ; DWARF5-NOT: Compile Unit
-; DWARF5: DW_TAG_compile_unit
+; DWARF5: DW_TAG_skeleton_unit
 ; DWARF5-NOT: DW_TAG_{{.*}}
-; DWARF5: DW_AT_GNU_dwo_name{{.*}}test.dwo
+; DWARF5: DW_AT_dwo_name{{.*}}test.dwo
 ; DWARF5: DW_AT_addr_base{{.*}}0x00000008
 ; DWARF5: DW_AT_low_pc [DW_FORM_addrx] (indexed (00000000) address = 0x0000000000000000 ".text")
 ; DWARF5: .debug_addr contents:
diff --git a/llvm/test/DebugInfo/X86/decl-derived-member.ll b/llvm/test/DebugInfo/X86/decl-derived-member.ll
index 6e51704e1..af4f714c3 100644
--- a/llvm/test/DebugInfo/X86/decl-derived-member.ll
+++ b/llvm/test/DebugInfo/X86/decl-derived-member.ll
@@ -92,11 +92,11 @@ entry:
   ret void
 }
 
-attributes #0 = { inlinehint nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { inlinehint uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { inlinehint nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { inlinehint uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind }
 attributes #3 = { nounwind readnone }
-attributes #4 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #4 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!8}
 !llvm.module.flags = !{!12, !13}
diff --git a/llvm/test/DebugInfo/X86/discriminator.ll b/llvm/test/DebugInfo/X86/discriminator.ll
index 209e17e63..c44c5d99c 100644
--- a/llvm/test/DebugInfo/X86/discriminator.ll
+++ b/llvm/test/DebugInfo/X86/discriminator.ll
@@ -35,7 +35,7 @@ return:                                           ; preds = %if.end, %if.then
   ret i32 %2, !dbg !13
 }
 
-attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!7, !8}
diff --git a/llvm/test/DebugInfo/X86/discriminator2.ll b/llvm/test/DebugInfo/X86/discriminator2.ll
index ca3020553..a8c7d3ff9 100644
--- a/llvm/test/DebugInfo/X86/discriminator2.ll
+++ b/llvm/test/DebugInfo/X86/discriminator2.ll
@@ -32,8 +32,8 @@ declare void @_Z3fooii(i32, i32) #1
 
 declare i32 @_Z3barv() #1
 
-attributes #0 = { uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!3, !4}
diff --git a/llvm/test/DebugInfo/X86/discriminator3.ll b/llvm/test/DebugInfo/X86/discriminator3.ll
index 7ae5e19cd..66d2cfee2 100644
--- a/llvm/test/DebugInfo/X86/discriminator3.ll
+++ b/llvm/test/DebugInfo/X86/discriminator3.ll
@@ -35,9 +35,9 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
 
 declare void @_Z3fooi(i32) #2
 
-attributes #0 = { uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
-attributes #2 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!3, !4}
diff --git a/llvm/test/DebugInfo/X86/dw_op_minus_direct.ll b/llvm/test/DebugInfo/X86/dw_op_minus_direct.ll
index efd640019..2d05e7484 100644
--- a/llvm/test/DebugInfo/X86/dw_op_minus_direct.ll
+++ b/llvm/test/DebugInfo/X86/dw_op_minus_direct.ll
@@ -17,7 +17,7 @@
 
 ; CHECK: .debug_loc contents:
 ; CHECK: 0x00000000:
-; CHECK-NEXT:   [0x0000000000000003, 0x0000000000000004): DW_OP_breg0 RAX+0, DW_OP_constu 0xffffffff, DW_OP_and, DW_OP_lit1, DW_OP_minus, DW_OP_stack_value
+; CHECK-NEXT:   (0x0000000000000003, 0x0000000000000004): DW_OP_breg0 RAX+0, DW_OP_constu 0xffffffff, DW_OP_and, DW_OP_lit1, DW_OP_minus, DW_OP_stack_value
 ;        rax+0, constu 0xffffffff, and, constu 0x00000001, minus, stack-value
 
 source_filename = "minus.c"
diff --git a/llvm/test/DebugInfo/X86/dwarf-aranges-no-dwarf-labels.ll b/llvm/test/DebugInfo/X86/dwarf-aranges-no-dwarf-labels.ll
index ab53436ed..c6e3c6409 100644
--- a/llvm/test/DebugInfo/X86/dwarf-aranges-no-dwarf-labels.ll
+++ b/llvm/test/DebugInfo/X86/dwarf-aranges-no-dwarf-labels.ll
@@ -53,8 +53,8 @@ entry:
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.value(metadata, metadata, metadata) #2
 
-attributes #0 = { nounwind readnone uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { nounwind readonly uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind readnone uwtable "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readonly uwtable "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!4}
diff --git a/llvm/test/DebugInfo/X86/dwarf-callsite-related-attrs.ll b/llvm/test/DebugInfo/X86/dwarf-callsite-related-attrs.ll
index f17cd0bef..33e06faba 100644
--- a/llvm/test/DebugInfo/X86/dwarf-callsite-related-attrs.ll
+++ b/llvm/test/DebugInfo/X86/dwarf-callsite-related-attrs.ll
@@ -14,17 +14,25 @@
 
 ; REQUIRES: object-emission
 ; RUN: %llc_dwarf -mtriple=x86_64-- < %s -o - | FileCheck %s -check-prefix=ASM
-; RUN: %llc_dwarf -mtriple=x86_64-- < %s -filetype=obj -o %t.o
+; RUN: %llc_dwarf -debugger-tune=lldb -mtriple=x86_64-- < %s -filetype=obj -o %t.o
 ; RUN: llvm-dwarfdump %t.o -o - | FileCheck %s -check-prefix=OBJ -implicit-check-not=DW_TAG_call_site
 ; RUN: llvm-dwarfdump -verify %t.o 2>&1 | FileCheck %s -check-prefix=VERIFY
 ; RUN: llvm-dwarfdump -statistics %t.o | FileCheck %s -check-prefix=STATS
 ; RUN: llvm-as < %s | llvm-dis | llvm-as | llvm-dis -o /dev/null
 
 ; VERIFY: No errors.
-; STATS: "call site entries":5
+; STATS: "call site DIEs":6
 
 @sink = global i32 0, align 4, !dbg !0
 
+define void @__has_no_subprogram() {
+entry:
+  %0 = load volatile i32, i32* @sink, align 4
+  %inc = add nsw i32 %0, 1
+  store volatile i32 %inc, i32* @sink, align 4
+  ret void
+}
+
 ; ASM: DW_TAG_subprogram
 ; ASM:   DW_AT_call_all_calls
 ; OBJ: [[bat_sp:.*]]: DW_TAG_subprogram
@@ -70,6 +78,7 @@ entry:
 ; OBJ:     DW_AT_call_tail_call
 define void @_Z3foov() !dbg !25 {
 entry:
+  tail call void @__has_no_subprogram()
   tail call void @_Z3barv(), !dbg !26
   tail call void @_Z3batv(), !dbg !27
   tail call void @_Z3barv(), !dbg !26
@@ -85,6 +94,9 @@ entry:
 ; OBJ:   DW_TAG_call_site
 ; OBJ:     DW_AT_call_origin ([[foo_sp]])
 ; OBJ:     DW_AT_call_return_pc
+; OBJ:   DW_TAG_call_site
+; OBJ:     DW_AT_call_target
+; OBJ:     DW_AT_call_return_pc
 define i32 @main() !dbg !29 {
 entry:
   call void @_Z3foov(), !dbg !32
diff --git a/llvm/test/DebugInfo/X86/dwarf-linkage-names.ll b/llvm/test/DebugInfo/X86/dwarf-linkage-names.ll
index dc514eb33..2af2b8659 100644
--- a/llvm/test/DebugInfo/X86/dwarf-linkage-names.ll
+++ b/llvm/test/DebugInfo/X86/dwarf-linkage-names.ll
@@ -49,7 +49,7 @@ entry:
   ret i32 %0, !dbg !15
 }
 
-attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!5}
 !llvm.module.flags = !{!8, !9}
diff --git a/llvm/test/DebugInfo/X86/dwarf-pubnames-split.ll b/llvm/test/DebugInfo/X86/dwarf-pubnames-split.ll
index 10ec63c0f..73d4af764 100644
--- a/llvm/test/DebugInfo/X86/dwarf-pubnames-split.ll
+++ b/llvm/test/DebugInfo/X86/dwarf-pubnames-split.ll
@@ -19,7 +19,7 @@ entry:
   ret i32 0, !dbg !10
 }
 
-attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!9, !11}
diff --git a/llvm/test/DebugInfo/X86/empty_macinfo.ll b/llvm/test/DebugInfo/X86/empty_macinfo.ll
index ec3c6a113..9cde5cdfa 100644
--- a/llvm/test/DebugInfo/X86/empty_macinfo.ll
+++ b/llvm/test/DebugInfo/X86/empty_macinfo.ll
@@ -2,10 +2,8 @@
 
 ; Test that we don't pollute the start of the file with debug sections
 
-; CHECK:      .section .debug_macinfo,"",@progbits
-; CHECK-NEXT: .byte 0 # End Of Macro List Mark
-; CHECK-NEXT: .section
-; CHECK-NOT:  .debug_macinfo
+; CHECK-NOT: macinfo
+; CHECK-NOT: macro
 
 define void @f() !dbg !4 {
   ret void, !dbg !9
diff --git a/llvm/test/DebugInfo/X86/fission-inline.ll b/llvm/test/DebugInfo/X86/fission-inline.ll
index 0fb4b83bd..fbfd20fc8 100644
--- a/llvm/test/DebugInfo/X86/fission-inline.ll
+++ b/llvm/test/DebugInfo/X86/fission-inline.ll
@@ -90,8 +90,8 @@ entry:
 
 declare void @_Z2f1v() #1
 
-attributes #0 = { uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!22, !23}
diff --git a/llvm/test/DebugInfo/X86/fission-no-inlining.ll b/llvm/test/DebugInfo/X86/fission-no-inlining.ll
index a2bed1a7d..7ae5c2b82 100644
--- a/llvm/test/DebugInfo/X86/fission-no-inlining.ll
+++ b/llvm/test/DebugInfo/X86/fission-no-inlining.ll
@@ -22,8 +22,8 @@ entry:
 
 declare void @_Z2f1v() #1
 
-attributes #0 = { uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!3}
diff --git a/llvm/test/DebugInfo/X86/fission-ranges.ll b/llvm/test/DebugInfo/X86/fission-ranges.ll
index 106834d92..9584d5c19 100644
--- a/llvm/test/DebugInfo/X86/fission-ranges.ll
+++ b/llvm/test/DebugInfo/X86/fission-ranges.ll
@@ -32,10 +32,10 @@
 ; CHECK:      DW_TAG_formal_parameter
 ; CHECK-NEXT:   DW_AT_const_value [DW_FORM_sdata] (1)
 ; CHECK-NEXT:   DW_AT_name {{.*}} "p")
-; CHECK: DW_AT_location [DW_FORM_sec_offset]   ([[A:0x[0-9a-z]*]]
-; CHECK: DW_AT_location [DW_FORM_sec_offset]   ([[E:0x[0-9a-z]*]]
-; CHECK: DW_AT_location [DW_FORM_sec_offset]   ([[B:0x[0-9a-z]*]]
-; CHECK: DW_AT_location [DW_FORM_sec_offset]   ([[D:0x[0-9a-z]*]]
+; CHECK: DW_AT_location [DW_FORM_sec_offset]   ([[A:0x[0-9a-z]*]]:
+; CHECK: DW_AT_location [DW_FORM_sec_offset]   ([[E:0x[0-9a-z]*]]:
+; CHECK: DW_AT_location [DW_FORM_sec_offset]   ([[B:0x[0-9a-z]*]]:
+; CHECK: DW_AT_location [DW_FORM_sec_offset]   ([[D:0x[0-9a-z]*]]:
 ; CHECK: DW_AT_ranges [DW_FORM_sec_offset]   (0x00000000
 ; CHECK-NOT: .debug_loc contents:
 ; CHECK-NOT: Beginning address offset
@@ -45,18 +45,22 @@
 ; if they've changed due to a bugfix, change in register allocation, etc.
 
 ; CHECK:      [[A]]:
-; CHECK-NEXT:   Addr idx 2 (w/ length 169): DW_OP_consts +0, DW_OP_stack_value
-; CHECK-NEXT:   Addr idx 3 (w/ length 15): DW_OP_reg0 RAX
-; CHECK-NEXT:   Addr idx 4 (w/ length 18): DW_OP_breg7 RSP-8
+; CHECK-NEXT:   DW_LLE_startx_length (0x00000002, 0x0000000f): DW_OP_consts +0, DW_OP_stack_value
+; CHECK-NEXT:   DW_LLE_startx_length (0x00000003, 0x0000000f): DW_OP_reg0 RAX
+; CHECK-NEXT:   DW_LLE_startx_length (0x00000004, 0x00000012): DW_OP_breg7 RSP-8
+; CHECK-NEXT:   DW_LLE_end_of_list   ()
 ; CHECK:      [[E]]:
-; CHECK-NEXT:   Addr idx 5 (w/ length 9): DW_OP_reg0 RAX
-; CHECK-NEXT:   Addr idx 6 (w/ length 98): DW_OP_breg7 RSP-44
+; CHECK-NEXT:   DW_LLE_startx_length (0x00000005, 0x00000009): DW_OP_reg0 RAX
+; CHECK-NEXT:   DW_LLE_startx_length (0x00000006, 0x00000062): DW_OP_breg7 RSP-44
+; CHECK-NEXT:   DW_LLE_end_of_list   ()
 ; CHECK:      [[B]]:
-; CHECK-NEXT:   Addr idx 7 (w/ length 15): DW_OP_reg0 RAX
-; CHECK-NEXT:   Addr idx 8 (w/ length 66): DW_OP_breg7 RSP-32
+; CHECK-NEXT:   DW_LLE_startx_length (0x00000007, 0x0000000f): DW_OP_reg0 RAX
+; CHECK-NEXT:   DW_LLE_startx_length (0x00000008, 0x00000042): DW_OP_breg7 RSP-32
+; CHECK-NEXT:   DW_LLE_end_of_list   ()
 ; CHECK:      [[D]]:
-; CHECK-NEXT:   Addr idx 9 (w/ length 15): DW_OP_reg0 RAX
-; CHECK-NEXT:   Addr idx 10 (w/ length 42): DW_OP_breg7 RSP-20
+; CHECK-NEXT:   DW_LLE_startx_length (0x00000009, 0x0000000f): DW_OP_reg0 RAX
+; CHECK-NEXT:   DW_LLE_startx_length (0x0000000a, 0x0000002a): DW_OP_breg7 RSP-20
+; CHECK-NEXT:   DW_LLE_end_of_list   ()
 
 ; Make sure we don't produce any relocations in any .dwo section (though in particular, debug_info.dwo)
 ; HDR-NOT: .rela.{{.*}}.dwo
@@ -183,7 +187,7 @@ for.end18:                                        ; preds = %for.inc16
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.value(metadata, metadata, metadata) #1
 
-attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/DebugInfo/X86/generate-odr-hash.ll b/llvm/test/DebugInfo/X86/generate-odr-hash.ll
index af905c376..5dc3fb6b0 100644
--- a/llvm/test/DebugInfo/X86/generate-odr-hash.ll
+++ b/llvm/test/DebugInfo/X86/generate-odr-hash.ll
@@ -216,7 +216,7 @@ entry:
   ret void, !dbg !57
 }
 
-attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!34}
diff --git a/llvm/test/DebugInfo/X86/ghost-sdnode-dbgvalues.ll b/llvm/test/DebugInfo/X86/ghost-sdnode-dbgvalues.ll
index 40606429b..8f34667a8 100644
--- a/llvm/test/DebugInfo/X86/ghost-sdnode-dbgvalues.ll
+++ b/llvm/test/DebugInfo/X86/ghost-sdnode-dbgvalues.ll
@@ -61,7 +61,7 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.value(metadata, metadata, metadata) #1
 
-attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/DebugInfo/X86/gmlt-empty-base-address.ll b/llvm/test/DebugInfo/X86/gmlt-empty-base-address.ll
new file mode 100644
index 000000000..727a3c310
--- /dev/null
+++ b/llvm/test/DebugInfo/X86/gmlt-empty-base-address.ll
@@ -0,0 +1,36 @@
+; RUN: llc -function-sections -mtriple=x86_64-unknown-linux-gnu < %s -filetype=obj | llvm-dwarfdump -v -debug-info - | FileCheck %s
+
+; CHECK: DW_AT_ranges [DW_FORM_rnglistx]   (indexed (0x0) rangelist = 0x00000010
+; CHECK:                    [0x0000000000000000, 0x{{[0-9a-z]*}}) ".text._Z2f1v"
+; CHECK:                    [0x0000000000000000, 0x{{[0-9a-z]*}}) ".text._Z2f2v")
+; CHECK: DW_AT_addr_base [DW_FORM_sec_offset] (0x00000008)
+
+
+; Function A
+define dso_local void @_Z2f1v() #0 !dbg !7 {
+  ret void, !dbg !9
+}
+
+; Function Attrs: noinline nounwind optnone uwtable
+define dso_local void @_Z2f2v() #0 !dbg !10 {
+  ret void, !dbg !11
+}
+
+attributes #0 = { noinline nounwind optnone uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 10.0.0 (trunk 371665) (llvm/trunk 371681)", isOptimized: false, runtimeVersion: 0, emissionKind: LineTablesOnly, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "gmlt-empty-base-address.cpp", directory: "/usr/local/google/home/blaikie/dev/scratch", checksumkind: CSK_MD5, checksum: "74f7c574cd1ba04403967d02e757afeb")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 5}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{!"clang version 10.0.0 (trunk 371665) (llvm/trunk 371681)"}
+!7 = distinct !DISubprogram(name: "f1", scope: !1, file: !1, line: 1, type: !8, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !2)
+!8 = !DISubroutineType(types: !2)
+!9 = !DILocation(line: 2, column: 1, scope: !7)
+!10 = distinct !DISubprogram(name: "f2", scope: !1, file: !1, line: 3, type: !8, scopeLine: 3, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !2)
+!11 = !DILocation(line: 4, column: 1, scope: !10)
diff --git a/llvm/test/DebugInfo/X86/gnu-public-names-gmlt.ll b/llvm/test/DebugInfo/X86/gnu-public-names-gmlt.ll
index 7f04eb4db..dc1475c91 100644
--- a/llvm/test/DebugInfo/X86/gnu-public-names-gmlt.ll
+++ b/llvm/test/DebugInfo/X86/gnu-public-names-gmlt.ll
@@ -39,8 +39,8 @@ entry:
 
 declare void @_Z2f1v() #1
 
-attributes #0 = { noinline uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noinline uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!3, !4, !5}
diff --git a/llvm/test/DebugInfo/X86/gnu-public-names.ll b/llvm/test/DebugInfo/X86/gnu-public-names.ll
index 6ea2fd513..c9ec1b59c 100644
--- a/llvm/test/DebugInfo/X86/gnu-public-names.ll
+++ b/llvm/test/DebugInfo/X86/gnu-public-names.ll
@@ -345,7 +345,7 @@ entry:
   ret i32 %add5, !dbg !90
 }
 
-attributes #0 = { noinline nounwind optnone uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noinline nounwind optnone uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone speculatable }
 
 !llvm.dbg.cu = !{!2}
@@ -354,7 +354,7 @@ attributes #1 = { nounwind readnone speculatable }
 
 !0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
 !1 = distinct !DIGlobalVariable(name: "static_member_variable", linkageName: "_ZN1C22static_member_variableE", scope: !2, file: !3, line: 7, type: !13, isLocal: false, isDefinition: true, declaration: !22)
-!2 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !3, producer: "clang version 9.0.0 (trunk 363288) (llvm/trunk 363294)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, retainedTypes: !16, globals: !17, imports: !54, nameTableKind: GNU)
+!2 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_03, file: !3, producer: "clang version 9.0.0 (trunk 363288) (llvm/trunk 363294)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, retainedTypes: !16, globals: !17, imports: !54, nameTableKind: GNU)
 !3 = !DIFile(filename: "names.cpp", directory: "/usr/local/google/home/blaikie/dev/scratch")
 !4 = !{!5, !9, !12}
 !5 = !DICompositeType(tag: DW_TAG_enumeration_type, file: !3, line: 49, baseType: !6, size: 32, elements: !7)
diff --git a/llvm/test/DebugInfo/X86/inline-member-function.ll b/llvm/test/DebugInfo/X86/inline-member-function.ll
index ed60cb72c..2f4937c71 100644
--- a/llvm/test/DebugInfo/X86/inline-member-function.ll
+++ b/llvm/test/DebugInfo/X86/inline-member-function.ll
@@ -61,7 +61,7 @@ entry:
 
 declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
 
-attributes #0 = { uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!4}
diff --git a/llvm/test/DebugInfo/X86/inline-seldag-test.ll b/llvm/test/DebugInfo/X86/inline-seldag-test.ll
index 947e04903..4df3a66e9 100644
--- a/llvm/test/DebugInfo/X86/inline-seldag-test.ll
+++ b/llvm/test/DebugInfo/X86/inline-seldag-test.ll
@@ -41,7 +41,7 @@ entry:
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
 
-attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/DebugInfo/X86/length_symbol_difference.ll b/llvm/test/DebugInfo/X86/length_symbol_difference.ll
index 78684e788..6736c5017 100644
--- a/llvm/test/DebugInfo/X86/length_symbol_difference.ll
+++ b/llvm/test/DebugInfo/X86/length_symbol_difference.ll
@@ -5,7 +5,6 @@
 ; CHECK-NOT:  .byte   0
 ; CHECK:      .byte   0                       # End Of Children Mark
 ; CHECK-NEXT: .Ldebug_info_end0:
-; CHECK-NEXT: .section
 
 
 define dso_local void @_Z2f1v() !dbg !7 {
diff --git a/llvm/test/DebugInfo/X86/lexical_block.ll b/llvm/test/DebugInfo/X86/lexical_block.ll
index cb57bf92e..7feff4ac7 100644
--- a/llvm/test/DebugInfo/X86/lexical_block.ll
+++ b/llvm/test/DebugInfo/X86/lexical_block.ll
@@ -41,7 +41,7 @@ if.end:                                           ; preds = %if.then, %entry
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
 
-attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/DebugInfo/X86/line-info.ll b/llvm/test/DebugInfo/X86/line-info.ll
index 20367f67c..e928fe4fe 100644
--- a/llvm/test/DebugInfo/X86/line-info.ll
+++ b/llvm/test/DebugInfo/X86/line-info.ll
@@ -32,7 +32,7 @@ entry:
   ret i32 0, !dbg !17
 }
 
-attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf"="true" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/DebugInfo/X86/live-debug-values-expr-conflict.ll b/llvm/test/DebugInfo/X86/live-debug-values-expr-conflict.ll
new file mode 100644
index 000000000..1d02ee70e
--- /dev/null
+++ b/llvm/test/DebugInfo/X86/live-debug-values-expr-conflict.ll
@@ -0,0 +1,75 @@
+; RUN: llc %s -mtriple=x86_64-unknown-unknown -o - -stop-after=livedebugvalues | FileCheck %s
+;
+; In the C below, 'baz' is re-assigned with a value that gets salvaged, making
+; it's dbg.value base itself on 'bar', but with a complex expression.
+; LiveDebugValues should recognize that these are different locations, and not
+; propagate a location for 'baz' into the return block.
+;
+; void escape1(int bees);
+; void escape2(int bees);
+; 
+; int foo(int bar) {
+;   int baz = bar;
+;   if (baz == 12) {
+;     escape1(bar);
+;   } else {
+;     baz += 1;
+;     escape2(bar);
+;   }
+; 
+;   return bar;
+; }
+;
+; We should get a plain DBG_VALUE in the entry block, a plain one then complex
+; one in the block two, and none in block three.
+; CHECK:       ![[BAZVAR:[0-9]+]] = !DILocalVariable(name: "baz",
+; CHECK-LABEL: bb.0.entry:
+; CHECK:       DBG_VALUE {{[0-9a-zA-Z$%_]*}}, $noreg, ![[BAZVAR]],
+; CHECK-SAME:     !DIExpression()
+; CHECK-LABEL: bb.1.if.then:
+; CHECK-LABEL: bb.2.if.else:
+; CHECK:       DBG_VALUE {{[0-9a-zA-Z$%_]*}}, $noreg, ![[BAZVAR]],
+; CHECK-SAME:     !DIExpression()
+; CHECK:       DBG_VALUE {{[0-9a-zA-Z$%_]*}}, $noreg, ![[BAZVAR]],
+; CHECK-SAME:     !DIExpression(DW_OP_plus_uconst, 1, DW_OP_stack_value)
+; CHECK-LABEL: bb.3.if.end:
+; CHECK-NOT:   DBG_VALUE
+
+declare void @escape1(i32)
+declare void @escape2(i32)
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+define i32 @foo(i32 returned %bar) !dbg !7 {
+entry:
+  call void @llvm.dbg.value(metadata i32 %bar, metadata !13, metadata !DIExpression()), !dbg !14
+  %cmp = icmp eq i32 %bar, 12, !dbg !14
+  br i1 %cmp, label %if.then, label %if.else, !dbg !14
+
+if.then:
+  tail call void @escape1(i32 12) #3, !dbg !14
+  br label %if.end, !dbg !14
+
+if.else:
+  call void @llvm.dbg.value(metadata i32 %bar, metadata !13, metadata !DIExpression(DW_OP_plus_uconst, 1, DW_OP_stack_value)), !dbg !14
+  tail call void @escape2(i32 %bar) #3, !dbg !14
+  br label %if.end
+
+if.end:
+  ret i32 %bar, !dbg !14
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "exprconflict.c", directory: "/home/jmorse")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!7 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 5, type: !8, scopeLine: 5, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !11)
+!8 = !DISubroutineType(types: !9)
+!9 = !{!10, !10}
+!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!11 = !{!13}
+!13 = !DILocalVariable(name: "baz", scope: !7, file: !1, line: 6, type: !10)
+!14 = !DILocation(line: 1, scope: !7)
diff --git a/llvm/test/DebugInfo/X86/live-debug-values-remove-range.ll b/llvm/test/DebugInfo/X86/live-debug-values-remove-range.ll
new file mode 100644
index 000000000..6fec3f815
--- /dev/null
+++ b/llvm/test/DebugInfo/X86/live-debug-values-remove-range.ll
@@ -0,0 +1,114 @@
+; RUN: llc -mtriple=x86_64-unknown-unknown %s -o - -stop-after=livedebugvalues | FileCheck %s
+;
+; In the simple loop below, the location of the variable "toast" is %bar in
+; the entry block, then set to constant zero at the end of the loop. We cannot
+; know the location of "toast" at the start of the %loop block. Test that no
+; location is given until after the call to @booler.
+;
+; Second function @baz added with an even tighter loop -- this tests different
+; code-paths through LiveDebugValues. Any blocks with an incoming backedge need
+; reconsideration after the parent of the backedge has had its OutLocs
+; initialized, even if OutLocs hasn't changed.
+;
+; Third function @quux tests that we don't delete too many variable locations.
+; A variable that is live across the body of the loop should maintain its
+; location across that loop, and not be invalidated.
+;
+; CHECK: ![[FOOVARNUM:[0-9]+]] = !DILocalVariable(name: "toast"
+; CHECK: ![[BAZVARNUM:[0-9]+]] = !DILocalVariable(name: "crumpets"
+; CHECK: ![[QUUXVARNUM:[0-9]+]] = !DILocalVariable(name: "teacake"
+;
+; foo tests
+; CHECK-LABEL: bb.1.loop
+; CHECK-NOT:   DBG_VALUE
+; CHECK-LABEL: CALL64pcrel32 @booler
+; CHECK:       DBG_VALUE 0, $noreg, ![[FOOVARNUM]]
+;
+; baz tests
+; CHECK-LABEL: name: baz
+; CHECK-LABEL: bb.1.loop
+; CHECK-NOT:   DBG_VALUE
+; CHECK-LABEL: CALL64pcrel32 @booler
+; CHECK:       DBG_VALUE 0, $noreg, ![[BAZVARNUM]]
+;
+; quux tests -- the variable arrives in $edi, should get a non-undef location
+; before the loop, and its position re-stated in each block.
+; CHECK-LABEL: name: quux
+; CHECK:       DBG_VALUE $edi, $noreg, ![[QUUXVARNUM]]
+; CHECK:       DBG_VALUE [[QUUXLOC:[a-zA-Z0-9$_]+]], $noreg, ![[QUUXVARNUM]]
+; CHECK-LABEL: bb.1.loop
+; CHECK:       DBG_VALUE [[QUUXLOC]], $noreg, ![[QUUXVARNUM]]
+; CHECK-NOT:   DBG_VALUE $noreg
+; CHECK-LABEL: bb.2.exit
+; CHECK:       DBG_VALUE [[QUUXLOC]], $noreg, ![[QUUXVARNUM]]
+; CHECK-NOT:   DBG_VALUE $noreg
+
+declare i1 @booler()
+declare void @escape(i32)
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+@glob = global i32 0
+
+define i32 @foo(i32 %bar) !dbg !4 {
+entry:
+  call void @llvm.dbg.value(metadata i32 %bar, metadata !3, metadata !DIExpression()), !dbg !6
+  br label %loop
+loop:
+  call void @escape(i32 %bar)
+  %retval = call i1 @booler(), !dbg !6
+  call void @llvm.dbg.value(metadata i32 0, metadata !3, metadata !DIExpression()), !dbg !6
+  br i1 %retval, label %loop2, label %exit
+loop2:
+  store i32 %bar, i32 *@glob
+  br label %loop
+exit:
+  ret i32 %bar
+}
+
+define i32 @baz(i32 %bar) !dbg !104 {
+entry:
+  call void @llvm.dbg.value(metadata i32 %bar, metadata !103, metadata !DIExpression()), !dbg !106
+  br label %loop
+loop:
+  call void @escape(i32 %bar)
+  %retval = call i1 @booler(), !dbg !106
+  call void @llvm.dbg.value(metadata i32 0, metadata !103, metadata !DIExpression()), !dbg !106
+  br i1 %retval, label %loop, label %exit
+exit:
+  ret i32 %bar
+}
+
+define i32 @quux(i32 %bar) !dbg !204 {
+entry:
+  ; %bar will be placed in a nonvolatile or spill location for the loop,
+  ; before being returned later.
+  call void @llvm.dbg.value(metadata i32 %bar, metadata !203, metadata !DIExpression()), !dbg !206
+  br label %loop
+loop:
+  %retval = call i1 @booler(), !dbg !206
+  br i1 %retval, label %loop, label %exit
+exit:
+  ret i32 %bar
+}
+
+!llvm.module.flags = !{!0, !100}
+!llvm.dbg.cu = !{!1}
+
+!100 = !{i32 2, !"Dwarf Version", i32 4}
+!0 = !{i32 2, !"Debug Info Version", i32 3}
+!1 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !2, producer: "beards", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug)
+!2 = !DIFile(filename: "bees.cpp", directory: ".")
+!3 = !DILocalVariable(name: "toast", scope: !4, file: !2, line: 1, type: !16)
+!4 = distinct !DISubprogram(name: "nope", scope: !2, file: !2, line: 1, spFlags: DISPFlagDefinition, unit: !1, retainedNodes: !13, type: !14, isDefinition: true)
+!6 = !DILocation(line: 1, scope: !4)
+!13 = !{!3}
+!14 = !DISubroutineType(types: !15)
+!15 = !{!16}
+!16 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!103 = !DILocalVariable(name: "crumpets", scope: !104, file: !2, line: 1, type: !16)
+!104 = distinct !DISubprogram(name: "ribbit", scope: !2, file: !2, line: 1, spFlags: DISPFlagDefinition, unit: !1, retainedNodes: !113, type: !14, isDefinition: true)
+!106 = !DILocation(line: 1, scope: !104)
+!113 = !{!103}
+!203 = !DILocalVariable(name: "teacake", scope: !204, file: !2, line: 1, type: !16)
+!204 = distinct !DISubprogram(name: "toad", scope: !2, file: !2, line: 1, spFlags: DISPFlagDefinition, unit: !1, retainedNodes: !113, type: !14, isDefinition: true)
+!206 = !DILocation(line: 1, scope: !204)
+!213 = !{!203}
diff --git a/llvm/test/DebugInfo/X86/live-debug-variables.ll b/llvm/test/DebugInfo/X86/live-debug-variables.ll
index b09a152ce..cf65b1909 100644
--- a/llvm/test/DebugInfo/X86/live-debug-variables.ll
+++ b/llvm/test/DebugInfo/X86/live-debug-variables.ll
@@ -25,7 +25,7 @@
 ; CHECK:      .debug_loc contents:
 ; CHECK-NEXT: 0x00000000:
 ;   We currently emit an entry for the function prologue, too, which could be optimized away.
-; CHECK:              [0x0000000000000010, 0x0000000000000072): DW_OP_reg3 RBX
+; CHECK:              (0x0000000000000018, 0x0000000000000072): DW_OP_reg3 RBX
 ;   We should only have one entry inside the function.
 ; CHECK-NOT: :
 
diff --git a/llvm/test/DebugInfo/X86/loclists-dwp.ll b/llvm/test/DebugInfo/X86/loclists-dwp.ll
index cba81a8a9..91f838876 100644
--- a/llvm/test/DebugInfo/X86/loclists-dwp.ll
+++ b/llvm/test/DebugInfo/X86/loclists-dwp.ll
@@ -19,10 +19,10 @@
 ; void b(int i) { asm("" : : : "rdi"); }
 
 ; CHECK:      DW_AT_location [DW_FORM_sec_offset]   (0x00000000
-; CHECK-NEXT: Addr idx 0 (w/ length 6): DW_OP_reg5 RDI)
+; CHECK-NEXT: DW_LLE_startx_length (0x0000000000000000, 0x0000000000000006): DW_OP_reg5 RDI)
 
 ; CHECK:      DW_AT_location [DW_FORM_sec_offset]   (0x00000000
-; CHECK-NEXT: Addr idx 0 (w/ length 0): DW_OP_reg5 RDI)
+; CHECK-NEXT: DW_LLE_startx_length (0x0000000000000000, 0x0000000000000000): DW_OP_reg5 RDI)
 
 target triple = "x86_64-unknown-linux-gnu"
 
diff --git a/llvm/test/DebugInfo/X86/low-pc-cu.ll b/llvm/test/DebugInfo/X86/low-pc-cu.ll
index 49857da37..e56c7b9ee 100644
--- a/llvm/test/DebugInfo/X86/low-pc-cu.ll
+++ b/llvm/test/DebugInfo/X86/low-pc-cu.ll
@@ -26,7 +26,7 @@ entry:
   ret void, !dbg !11
 }
 
-attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!8, !9}
diff --git a/llvm/test/DebugInfo/X86/mi-print.ll b/llvm/test/DebugInfo/X86/mi-print.ll
index f2c2094aa..8067006aa 100644
--- a/llvm/test/DebugInfo/X86/mi-print.ll
+++ b/llvm/test/DebugInfo/X86/mi-print.ll
@@ -24,7 +24,7 @@ entry:
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.value(metadata, metadata, metadata) #1
 
-attributes #0 = { nounwind readnone ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="core2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind readnone ssp uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="core2" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/DebugInfo/X86/no_debug_ranges.ll b/llvm/test/DebugInfo/X86/no_debug_ranges.ll
index e119f0080..b0fd8151c 100644
--- a/llvm/test/DebugInfo/X86/no_debug_ranges.ll
+++ b/llvm/test/DebugInfo/X86/no_debug_ranges.ll
@@ -33,7 +33,7 @@ entry:
   ret void, !dbg !12
 }
 
-attributes #0 = { noinline nounwind optnone uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noinline nounwind optnone uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!3, !4, !5}
diff --git a/llvm/test/DebugInfo/X86/nodebug.ll b/llvm/test/DebugInfo/X86/nodebug.ll
index 0eccb89c9..845fbdebc 100644
--- a/llvm/test/DebugInfo/X86/nodebug.ll
+++ b/llvm/test/DebugInfo/X86/nodebug.ll
@@ -36,7 +36,7 @@ entry:
   ret void
 }
 
-attributes #0 = { uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!8, !9}
diff --git a/llvm/test/DebugInfo/X86/nodebug_with_debug_loc.ll b/llvm/test/DebugInfo/X86/nodebug_with_debug_loc.ll
index 76a03d892..7ae3778bf 100644
--- a/llvm/test/DebugInfo/X86/nodebug_with_debug_loc.ll
+++ b/llvm/test/DebugInfo/X86/nodebug_with_debug_loc.ll
@@ -88,8 +88,8 @@ declare void @llvm.lifetime.start(i64, i8* nocapture) #3
 ; Function Attrs: nounwind
 declare void @llvm.lifetime.end(i64, i8* nocapture) #3
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind readnone }
 attributes #3 = { nounwind }
 
diff --git a/llvm/test/DebugInfo/X86/noreturn_cpp11.ll b/llvm/test/DebugInfo/X86/noreturn_cpp11.ll
index d3b6104f7..fb10c9c4b 100644
--- a/llvm/test/DebugInfo/X86/noreturn_cpp11.ll
+++ b/llvm/test/DebugInfo/X86/noreturn_cpp11.ll
@@ -2,56 +2,88 @@
 ; REQUIRES: object-emission
 
 ; Generated by clang++ -S -c -std=c++11 --emit-llvm -g from the following C++11 source:
-; [[ noreturn ]] void f() {
-;   throw 1;
-; }
+;class foo {
+;[[noreturn]] void foo_member(){throw 1;}
+;};
+;
+;[[ noreturn ]] void f() {
+;    throw 1;
+;}
+;
+;void func(){
+;    foo object;
+;}
 
 ; CHECK: DW_TAG_subprogram
 ; CHECK-NOT: DW_TAG
 ; CHECK: DW_AT_name{{.*}}"f"
 ; CHECK-NOT: DW_TAG
 ; CHECK: DW_AT_noreturn
-
+; CHECK: DW_TAG_class_type
+; CHECK: DW_TAG_subprogram
+; CHECK: DW_AT_name{{.*}}"foo_member"
+; CHECK: DW_AT_noreturn
 ; ModuleID = 'test.cpp'
-source_filename = "test.cpp"
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+source_filename = "noreturn1.cpp"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
-@_ZTIi = external constant i8*
+%class.foo = type { i8 }
 
-; Function Attrs: noreturn
-define void @_Z1fv() #0 !dbg !6 {
-entry:
-  %exception = call i8* @__cxa_allocate_exception(i64 4) #1, !dbg !9
-  %0 = bitcast i8* %exception to i32*, !dbg !9
-  store i32 1, i32* %0, align 16, !dbg !9
-  call void @__cxa_throw(i8* %exception, i8* bitcast (i8** @_ZTIi to i8*), i8* null) #2, !dbg !10
-  unreachable, !dbg !9
+@_ZTIi = external dso_local constant i8*
 
-return:                                           ; No predecessors!
-  ret void, !dbg !12
+; Function Attrs: noinline noreturn optnone uwtable
+define dso_local void @_Z1fv() #0 !dbg !7 {
+  %1 = call i8* @__cxa_allocate_exception(i64 4) #3, !dbg !10
+  %2 = bitcast i8* %1 to i32*, !dbg !10
+  store i32 1, i32* %2, align 16, !dbg !10
+  call void @__cxa_throw(i8* %1, i8* bitcast (i8** @_ZTIi to i8*), i8* null) #4, !dbg !10
+  unreachable, !dbg !10
 }
 
-declare i8* @__cxa_allocate_exception(i64)
+declare dso_local i8* @__cxa_allocate_exception(i64)
+
+declare dso_local void @__cxa_throw(i8*, i8*, i8*)
+
+; Function Attrs: noinline nounwind optnone uwtable
+define dso_local void @_Z4funcv() #1 !dbg !11 {
+  %1 = alloca %class.foo, align 1
+  call void @llvm.dbg.declare(metadata %class.foo* %1, metadata !12, metadata !DIExpression()), !dbg !19
+  ret void, !dbg !20
+}
 
-declare void @__cxa_throw(i8*, i8*, i8*)
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #2
 
-attributes #0 = { noreturn }
+attributes #0 = { noinline noreturn optnone uwtable }
+attributes #1 = { noinline nounwind optnone uwtable }
+attributes #2 = { nounwind readnone speculatable willreturn }
+attributes #3 = { nounwind }
+attributes #4 = { noreturn }
 
 !llvm.dbg.cu = !{!0}
-!llvm.module.flags = !{!3, !4}
-!llvm.ident = !{!5}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
 
-!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 4.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
-!1 = !DIFile(filename: "test.cpp", directory: "/home/del/test")
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_11, file: !1, producer: "clang version 10.0.0 (https://github.com/llvm/llvm-project.git 3fcdd25ad5566114ac3322dcbf71d3c38bfec1ed)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "test.cpp", directory: "/home/sourabh/work/dwarf/c_c++/c++11")
 !2 = !{}
 !3 = !{i32 2, !"Dwarf Version", i32 4}
 !4 = !{i32 2, !"Debug Info Version", i32 3}
-!5 = !{!"clang version 4.0.0"}
-!6 = distinct !DISubprogram(name: "f", linkageName: "_Z1fv", scope: !1, file: !1, line: 1, type: !7, isLocal: false, isDefinition: true, scopeLine: 1, flags: DIFlagPrototyped | DIFlagNoReturn, isOptimized: false, unit: !0, retainedNodes: !2)
-!7 = !DISubroutineType(types: !8)
-!8 = !{null}
-!9 = !DILocation(line: 2, column: 5, scope: !6)
-!10 = !DILocation(line: 2, column: 5, scope: !11)
-!11 = !DILexicalBlockFile(scope: !6, file: !1, discriminator: 1)
-!12 = !DILocation(line: 3, column: 1, scope: !6)
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{!"clang version 10.0.0 (https://github.com/llvm/llvm-project.git 3fcdd25ad5566114ac3322dcbf71d3c38bfec1ed)"}
+!7 = distinct !DISubprogram(name: "f", linkageName: "_Z1fv", scope: !1, file: !1, line: 5, type: !8, scopeLine: 5, flags: DIFlagPrototyped | DIFlagNoReturn, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !2)
+!8 = !DISubroutineType(types: !9)
+!9 = !{null}
+!10 = !DILocation(line: 6, column: 5, scope: !7)
+!11 = distinct !DISubprogram(name: "func", linkageName: "_Z4funcv", scope: !1, file: !1, line: 9, type: !8, scopeLine: 9, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !2)
+!12 = !DILocalVariable(name: "object", scope: !11, file: !1, line: 10, type: !13)
+!13 = distinct !DICompositeType(tag: DW_TAG_class_type, name: "foo", file: !1, line: 1, size: 8, flags: DIFlagTypePassByValue, elements: !14, identifier: "_ZTS3foo")
+!14 = !{!15}
+!15 = !DISubprogram(name: "foo_member", linkageName: "_ZN3foo10foo_memberEv", scope: !13, file: !1, line: 2, type: !16, scopeLine: 2, flags: DIFlagPrototyped | DIFlagNoReturn, spFlags: 0)
+!16 = !DISubroutineType(types: !17)
+!17 = !{null, !18}
+!18 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !13, size: 64, flags: DIFlagArtificial | DIFlagObjectPointer)
+!19 = !DILocation(line: 10, column: 9, scope: !11)
+!20 = !DILocation(line: 11, column: 1, scope: !11)
diff --git a/llvm/test/DebugInfo/X86/objc-property-void.ll b/llvm/test/DebugInfo/X86/objc-property-void.ll
index b29beb48f..1eed1e53a 100644
--- a/llvm/test/DebugInfo/X86/objc-property-void.ll
+++ b/llvm/test/DebugInfo/X86/objc-property-void.ll
@@ -66,7 +66,7 @@ entry:
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
 
-attributes #0 = { ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { ssp uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/DebugInfo/X86/objc_direct.ll b/llvm/test/DebugInfo/X86/objc_direct.ll
new file mode 100644
index 000000000..c29dec14a
--- /dev/null
+++ b/llvm/test/DebugInfo/X86/objc_direct.ll
@@ -0,0 +1,54 @@
+; RUN: llc < %s -filetype=obj -o %t
+; RUN: llvm-dwarfdump -v %t | FileCheck %s
+
+; Source code to regenerate:
+; __attribute__((objc_root_class))
+; @interface Root
+; - (int)direct_method __attribute__((objc_direct));
+; @end
+;
+; @implementation Root
+; - (int)direct_method __attribute__((objc_direct)) {
+;   return 42;
+; }
+; @end
+;
+; clang -O0 -g -gdwarf-5 direct.m -c
+
+; CHECK: DW_TAG_subprogram [3]
+; CHECK: DW_AT_APPLE_objc_direct
+; CHECK-SAME: DW_FORM_flag_present
+; CHECK: DW_TAG_formal_parameter [4]
+
+; ModuleID = 'direct.bc'
+source_filename = "direct.m"
+
+%0 = type opaque
+
+define hidden i32 @"\01-[Root direct_method]"(%0* %self, i8* %_cmd) {
+entry:
+  %retval = alloca i32, align 4
+  %0 = load i32, i32* %retval, align 4
+  ret i32 %0
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!19, !20}
+!llvm.ident = !{}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_ObjC, file: !1, producer: "clang version 10.0.0 (https://github.com/llvm/llvm-project d6b2f33e2b6338d24cf756ba220939aecc81210d)", isOptimized: false, runtimeVersion: 2, emissionKind: FullDebug, enums: !2, retainedTypes: !3, nameTableKind: None)
+!1 = !DIFile(filename: "direct.m", directory: "/", checksumkind: CSK_MD5, checksum: "6b49fad130344b0011fc0eef65949390")
+!2 = !{}
+!3 = !{!4}
+!4 = !DICompositeType(tag: DW_TAG_structure_type, name: "Root", scope: !1, file: !1, line: 2, flags: DIFlagObjcClassComplete, elements: !5, runtimeLang: DW_LANG_ObjC)
+!5 = !{!6}
+!6 = !DISubprogram(name: "-[Root direct_method]", scope: !4, file: !1, line: 7, type: !7, scopeLine: 7, flags: DIFlagPrototyped, spFlags: DISPFlagObjCDirect, retainedNodes: !2)
+!7 = !DISubroutineType(types: !8)
+!8 = !{!9, !10, !11}
+!9 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!10 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !4, size: 64, flags: DIFlagArtificial | DIFlagObjectPointer)
+!11 = !DIDerivedType(tag: DW_TAG_typedef, name: "SEL", file: !1, baseType: !12, flags: DIFlagArtificial)
+!12 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !13, size: 64)
+!13 = !DICompositeType(tag: DW_TAG_structure_type, name: "objc_selector", file: !1, flags: DIFlagFwdDecl)
+!19 = !{i32 7, !"Dwarf Version", i32 5}
+!20 = !{i32 2, !"Debug Info Version", i32 3}
diff --git a/llvm/test/DebugInfo/X86/pieces-1.ll b/llvm/test/DebugInfo/X86/pieces-1.ll
index 9577a2823..c333f2b33 100644
--- a/llvm/test/DebugInfo/X86/pieces-1.ll
+++ b/llvm/test/DebugInfo/X86/pieces-1.ll
@@ -16,7 +16,7 @@
 ; CHECK: .debug_loc contents:
 ;
 
-; CHECK: [0x0000000000000000, 0x[[LTMP3:.*]]): DW_OP_reg5 RDI, DW_OP_piece 0x8, DW_OP_reg4 RSI, DW_OP_piece 0x4
+; CHECK: (0x0000000000000000, 0x[[LTMP3:.*]]): DW_OP_reg5 RDI, DW_OP_piece 0x8, DW_OP_reg4 RSI, DW_OP_piece 0x4
 ; 0x0000000000000006 - 0x0000000000000008: rbp-8, piece 0x8, rax, piece 0x4 )
 
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
@@ -36,7 +36,7 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.value(metadata, metadata, metadata) #1
 
-attributes #0 = { nounwind ssp uwtable "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" }
+attributes #0 = { nounwind ssp uwtable "frame-pointer"="all" }
 attributes #1 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/DebugInfo/X86/pieces-3.ll b/llvm/test/DebugInfo/X86/pieces-3.ll
index 167fb37d1..2015f43b0 100644
--- a/llvm/test/DebugInfo/X86/pieces-3.ll
+++ b/llvm/test/DebugInfo/X86/pieces-3.ll
@@ -21,8 +21,9 @@
 ; CHECK-NEXT:     [0x0000000000000007, 0x0000000000000009): DW_OP_reg5 RDI, DW_OP_piece 0x8
 ; CHECK-NEXT:   DW_AT_name {{.*}}"outer"
 ; CHECK: DW_TAG_variable
-; CHECK-NEXT:   DW_AT_name {{.*}}"i1"
-; CHECK-NOT:    DW_AT_location
+; CHECK-NEXT:   DW_AT_location [DW_FORM_data4]        (0x00000044
+; CHECK-NEXT:     [0x0000000000000007, 0x0000000000000009): DW_OP_reg0 RAX, DW_OP_piece 0x4)
+; CHECK-NEXT:   "i1"
 
 ; ModuleID = '/Volumes/Data/llvm/test/DebugInfo/X86/sroasplit-2.ll'
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
@@ -54,7 +55,7 @@ declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture readonly, i
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.value(metadata, metadata, metadata) #1
 
-attributes #0 = { nounwind ssp uwtable "no-frame-pointer-elim"="true" }
+attributes #0 = { nounwind ssp uwtable "frame-pointer"="all" }
 attributes #1 = { nounwind readnone }
 attributes #2 = { nounwind }
 
diff --git a/llvm/test/DebugInfo/X86/pieces-4.ll b/llvm/test/DebugInfo/X86/pieces-4.ll
index 6ae8597e4..8eb82f4e5 100644
--- a/llvm/test/DebugInfo/X86/pieces-4.ll
+++ b/llvm/test/DebugInfo/X86/pieces-4.ll
@@ -53,9 +53,9 @@ declare i32 @g() local_unnamed_addr #2
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.value(metadata, metadata, metadata) #1
 
-attributes #0 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
-attributes #2 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #3 = { nounwind }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/DebugInfo/X86/pr28270.ll b/llvm/test/DebugInfo/X86/pr28270.ll
index 2e28b4d32..b4ad1421c 100644
--- a/llvm/test/DebugInfo/X86/pr28270.ll
+++ b/llvm/test/DebugInfo/X86/pr28270.ll
@@ -55,9 +55,9 @@ declare void @_ZN1AC1EPKc(%class.A*, i8*) unnamed_addr #2
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.value(metadata, metadata, metadata) #3
 
-attributes #0 = { noreturn uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noreturn uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { argmemonly nounwind }
-attributes #2 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #3 = { nounwind readnone }
 attributes #4 = { nounwind }
 
diff --git a/llvm/test/DebugInfo/X86/safestack-byval.ll b/llvm/test/DebugInfo/X86/safestack-byval.ll
index 7b300f772..a653cd78b 100644
--- a/llvm/test/DebugInfo/X86/safestack-byval.ll
+++ b/llvm/test/DebugInfo/X86/safestack-byval.ll
@@ -48,7 +48,7 @@ declare void @llvm.dbg.value(metadata, metadata, metadata) #1
 ; Function Attrs: argmemonly nounwind
 declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture readonly, i64, i1) #2
 
-attributes #0 = { norecurse nounwind readonly safestack uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { norecurse nounwind readonly safestack uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 attributes #2 = { argmemonly nounwind }
 
diff --git a/llvm/test/DebugInfo/X86/sdag-transfer-dbgvalue.ll b/llvm/test/DebugInfo/X86/sdag-transfer-dbgvalue.ll
new file mode 100644
index 000000000..772eb4694
--- /dev/null
+++ b/llvm/test/DebugInfo/X86/sdag-transfer-dbgvalue.ll
@@ -0,0 +1,64 @@
+; RUN: llc -start-after=codegenprepare -stop-before finalize-isel -o - %s | FileCheck %s
+
+; This tests that transferDbgValues() changes order of SDDbgValue transferred
+; to another node and debug info for 'ADD32ri' appears *after* the instruction.
+;
+; This test case was generated from the following program
+; using: clang -g -O3 -S -emit-llvm test.c
+;
+; int foo(int a, int *b) {
+;   int c = a + 512;
+;   if (c != 0)
+;     *b = a;
+;   return c;
+; }
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; CHECK-LABEL: bb.0.entry:
+; CHECK:       %[[REG:[0-9]+]]:gr32 = ADD32ri %1, 512
+; CHECK-NEXT:  DBG_VALUE %[[REG]]
+
+; Function Attrs: nofree norecurse nounwind uwtable writeonly
+define dso_local i32 @foo(i32 %a, i32* nocapture %b) local_unnamed_addr !dbg !7 {
+entry:
+  %add = add nsw i32 %a, 512, !dbg !18
+  call void @llvm.dbg.value(metadata i32 %add, metadata !16, metadata !DIExpression()), !dbg !17
+  %cmp = icmp eq i32 %add, 0, !dbg !18
+  br i1 %cmp, label %if.end, label %if.then, !dbg !18
+
+if.then:                                          ; preds = %entry
+  store i32 %a, i32* %b, align 4, !dbg !18
+  br label %if.end, !dbg !18
+
+if.end:                                           ; preds = %entry, %if.then
+  ret i32 %add, !dbg !18
+}
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 10.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "test.c", directory: "/")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{!"clang version 10.0.0"}
+!7 = distinct !DISubprogram(name: "foo", scope: !8, file: !8, line: 1, type: !9, scopeLine: 1, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !13)
+!8 = !DIFile(filename: "test.c", directory: "/")
+!9 = !DISubroutineType(types: !10)
+!10 = !{!11, !11, !12}
+!11 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!12 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !11, size: 64)
+!13 = !{!14, !15, !16}
+!14 = !DILocalVariable(name: "a", arg: 1, scope: !7, file: !8, line: 1, type: !11)
+!15 = !DILocalVariable(name: "b", arg: 2, scope: !7, file: !8, line: 1, type: !12)
+!16 = !DILocalVariable(name: "c", scope: !7, file: !8, line: 2, type: !11)
+!17 = !DILocation(line: 0, scope: !7)
+!18 = !DILocation(line: 2, column: 13, scope: !7)
diff --git a/llvm/test/DebugInfo/X86/single-dbg_value.ll b/llvm/test/DebugInfo/X86/single-dbg_value.ll
index f2eeca84e..d98b872f3 100644
--- a/llvm/test/DebugInfo/X86/single-dbg_value.ll
+++ b/llvm/test/DebugInfo/X86/single-dbg_value.ll
@@ -42,7 +42,7 @@ declare i32 @g(...)
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.value(metadata, metadata, metadata) #1
 
-attributes #0 = { nounwind ssp uwtable "no-frame-pointer-elim"="true" }
+attributes #0 = { nounwind ssp uwtable "frame-pointer"="all" }
 attributes #1 = { nounwind readnone }
 attributes #2 = { nounwind }
 
diff --git a/llvm/test/DebugInfo/X86/spill-nospill.ll b/llvm/test/DebugInfo/X86/spill-nospill.ll
index 71588487d..742258016 100644
--- a/llvm/test/DebugInfo/X86/spill-nospill.ll
+++ b/llvm/test/DebugInfo/X86/spill-nospill.ll
@@ -76,8 +76,8 @@ declare i32 @g(i32) local_unnamed_addr #1
 ; Function Attrs: nounwind readnone speculatable
 declare void @llvm.dbg.value(metadata, metadata, metadata) #2
 
-attributes #0 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind readnone speculatable }
 attributes #3 = { nounwind }
 
diff --git a/llvm/test/DebugInfo/X86/split-dwarf-multiple-cu-hash.ll b/llvm/test/DebugInfo/X86/split-dwarf-multiple-cu-hash.ll
index 2d202a1f9..3a69c0f25 100644
--- a/llvm/test/DebugInfo/X86/split-dwarf-multiple-cu-hash.ll
+++ b/llvm/test/DebugInfo/X86/split-dwarf-multiple-cu-hash.ll
@@ -22,7 +22,7 @@ entry:
   ret void, !dbg !14
 }
 
-attributes #0 = { noinline nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noinline nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0, !3}
 !llvm.ident = !{!5, !5}
diff --git a/llvm/test/DebugInfo/X86/split-dwarf-v5-ranges.ll b/llvm/test/DebugInfo/X86/split-dwarf-v5-ranges.ll
index 74e94643b..1761c4aa8 100644
--- a/llvm/test/DebugInfo/X86/split-dwarf-v5-ranges.ll
+++ b/llvm/test/DebugInfo/X86/split-dwarf-v5-ranges.ll
@@ -3,7 +3,7 @@
 
 ; CHECK: .debug_info contents:
 ; CHECK: .debug_info.dwo contents:
-; CHECK: DW_AT_ranges [DW_FORM_rnglistx] (indexed (0x0) rangelist = 0x00000004
+; CHECK: DW_AT_ranges [DW_FORM_rnglistx] (indexed (0x0) rangelist = 0x00000010
 ; CHECK:          [0x0000000000000001, 0x000000000000000c) ".text"
 ; CHECK:          [0x000000000000000e, 0x0000000000000013) ".text")
 
diff --git a/llvm/test/DebugInfo/X86/sret.ll b/llvm/test/DebugInfo/X86/sret.ll
index 17f09db53..24648dfba 100644
--- a/llvm/test/DebugInfo/X86/sret.ll
+++ b/llvm/test/DebugInfo/X86/sret.ll
@@ -259,11 +259,11 @@ eh.resume:                                        ; preds = %lpad
 ; Function Attrs: nobuiltin nounwind
 declare void @_ZdlPv(i8*) #4
 
-attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
-attributes #2 = { uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #3 = { noinline noreturn nounwind }
-attributes #4 = { nobuiltin nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #4 = { nobuiltin nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #5 = { noreturn nounwind }
 attributes #6 = { nounwind }
 attributes #7 = { builtin nounwind }
diff --git a/llvm/test/DebugInfo/X86/sroa-after-inlining.ll b/llvm/test/DebugInfo/X86/sroa-after-inlining.ll
new file mode 100644
index 000000000..5613db3e9
--- /dev/null
+++ b/llvm/test/DebugInfo/X86/sroa-after-inlining.ll
@@ -0,0 +1,139 @@
+; RUN: opt %s -sroa -instcombine -inline -instcombine -sroa -verify -S -o - | FileCheck %s
+;
+; This test checks that SROA pass processes debug info correctly if applied twice.
+; Specifically, after SROA works first time, instcombine converts dbg.declare
+; intrinsics into dbg.value. Inlining creates new opportunities for SROA,
+; so it is called again. This time it does not handle correctly previously
+; inserted dbg.value intrinsics: current SROA implementation while doing
+; "Migrate debug information from the old alloca to the new alloca(s)" handles
+; only dbg.declare intrinsic. In this case, original dbg.declare was lowered by
+; instcombine pass into dbg.value. When it comes into SROA second time, all dbg.value
+; intrinsics, inserted by instcombine pass before second SROA, just not updated
+; (though SROA was done). The fix is to not lower dbg.declare for structures.
+
+;
+; Hand-reduced from this example (-g -O -mllvm -disable-llvm-optzns -gno-column-info):
+;
+; struct S1 {
+;     int p1;
+;
+;     bool IsNull (  ) { return p1 == 0; }
+; };
+;
+; S1 foo ( void );
+;
+; int bar (  ) {
+;     S1 result = foo();
+;
+;     if (result.IsNull())
+;         return 0;
+;
+;     return result.p1 + 1;
+; }
+
+; CHECK: _Z3barv
+; CHECK: %[[RESULT:.*]] = call i32 @_Z3foov
+; CHECK: llvm.dbg.value(metadata i32 %[[RESULT]], metadata [[METADATA_IDX1:![0-9]+]]
+; CHECK: ret
+; CHECK: DICompileUnit
+; CHECK: [[METADATA_IDX1]] = !DILocalVariable(name: "result"
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+%struct.S1 = type { i32 }
+
+$_ZN2S16IsNullEv = comdat any
+
+define dso_local i32 @_Z3barv() !dbg !7 {
+entry:
+  %retval = alloca i32, align 4
+  %result = alloca %struct.S1, align 4
+  %cleanup.dest.slot = alloca i32, align 4
+  %0 = bitcast %struct.S1* %result to i8*, !dbg !21
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* %0) #5, !dbg !21
+  call void @llvm.dbg.declare(metadata %struct.S1* %result, metadata !12, metadata !DIExpression()), !dbg !21
+  %call = call i32 @_Z3foov(), !dbg !21
+  %coerce.dive = getelementptr inbounds %struct.S1, %struct.S1* %result, i32 0, i32 0, !dbg !21
+  store i32 %call, i32* %coerce.dive, align 4, !dbg !21
+  %call1 = call zeroext i1 @_ZN2S16IsNullEv(%struct.S1* %result), !dbg !22
+  br i1 %call1, label %if.then, label %if.end, !dbg !24
+
+if.then:                                          ; preds = %entry
+  store i32 0, i32* %retval, align 4, !dbg !25
+  store i32 1, i32* %cleanup.dest.slot, align 4
+  br label %cleanup, !dbg !25
+
+if.end:                                           ; preds = %entry
+  %p1 = getelementptr inbounds %struct.S1, %struct.S1* %result, i32 0, i32 0, !dbg !26
+  %1 = load i32, i32* %p1, align 4, !dbg !26
+  %add = add nsw i32 %1, 1, !dbg !26
+  store i32 %add, i32* %retval, align 4, !dbg !26
+  store i32 1, i32* %cleanup.dest.slot, align 4
+  br label %cleanup, !dbg !26
+
+cleanup:                                          ; preds = %if.end, %if.then
+  %2 = bitcast %struct.S1* %result to i8*, !dbg !32
+  call void @llvm.lifetime.end.p0i8(i64 4, i8* %2) #5, !dbg !32
+  %3 = load i32, i32* %retval, align 4, !dbg !32
+  ret i32 %3, !dbg !32
+}
+
+declare void @llvm.lifetime.start.p0i8(i64 immarg, i8* nocapture)
+
+declare void @llvm.dbg.declare(metadata, metadata, metadata)
+
+declare dso_local i32 @_Z3foov()
+
+define linkonce_odr dso_local zeroext i1 @_ZN2S16IsNullEv(%struct.S1* %this) #4 comdat align 2 !dbg !33 {
+entry:
+  %this.addr = alloca %struct.S1*, align 8
+  store %struct.S1* %this, %struct.S1** %this.addr, align 8
+  call void @llvm.dbg.declare(metadata %struct.S1** %this.addr, metadata !35, metadata !DIExpression()), !dbg !39
+  %this1 = load %struct.S1*, %struct.S1** %this.addr, align 8
+  %p1 = getelementptr inbounds %struct.S1, %struct.S1* %this1, i32 0, i32 0, !dbg !40
+  %0 = load i32, i32* %p1, align 4, !dbg !40
+  %cmp = icmp eq i32 %0, 0, !dbg !40
+  ret i1 %cmp, !dbg !40
+}
+
+declare void @llvm.lifetime.end.p0i8(i64 immarg, i8* nocapture)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "sroa-after-inlining.cpp", directory: "")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{!"clang"}
+!7 = distinct !DISubprogram(name: "bar", linkageName: "_Z3barv", scope: !1, file: !1, line: 9, type: !8, scopeLine: 9, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !11)
+!8 = !DISubroutineType(types: !9)
+!9 = !{!10}
+!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!11 = !{!12}
+!12 = !DILocalVariable(name: "result", scope: !7, file: !1, line: 10, type: !13)
+!13 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "S1", file: !1, line: 1, size: 32, flags: DIFlagTypePassByValue, elements: !14, identifier: "_ZTS2S1")
+!14 = !{!15, !16}
+!15 = !DIDerivedType(tag: DW_TAG_member, name: "p1", scope: !13, file: !1, line: 2, baseType: !10, size: 32)
+!16 = !DISubprogram(name: "IsNull", linkageName: "_ZN2S16IsNullEv", scope: !13, file: !1, line: 4, type: !17, scopeLine: 4, flags: DIFlagPrototyped, spFlags: DISPFlagOptimized)
+!17 = !DISubroutineType(types: !18)
+!18 = !{!19, !20}
+!19 = !DIBasicType(name: "bool", size: 8, encoding: DW_ATE_boolean)
+!20 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !13, size: 64, flags: DIFlagArtificial | DIFlagObjectPointer)
+!21 = !DILocation(line: 10, scope: !7)
+!22 = !DILocation(line: 12, scope: !23)
+!23 = distinct !DILexicalBlock(scope: !7, file: !1, line: 12)
+!24 = !DILocation(line: 12, scope: !7)
+!25 = !DILocation(line: 13, scope: !23)
+!26 = !DILocation(line: 15, scope: !7)
+!32 = !DILocation(line: 16, scope: !7)
+!33 = distinct !DISubprogram(name: "IsNull", linkageName: "_ZN2S16IsNullEv", scope: !13, file: !1, line: 4, type: !17, scopeLine: 4, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, declaration: !16, retainedNodes: !34)
+!34 = !{!35}
+!35 = !DILocalVariable(name: "this", arg: 1, scope: !33, type: !36, flags: DIFlagArtificial | DIFlagObjectPointer)
+!36 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !13, size: 64)
+!39 = !DILocation(line: 0, scope: !33)
+!40 = !DILocation(line: 4, scope: !33)
diff --git a/llvm/test/DebugInfo/X86/sroasplit-2.ll b/llvm/test/DebugInfo/X86/sroasplit-2.ll
index 9cebcfd1d..19df351ca 100644
--- a/llvm/test/DebugInfo/X86/sroasplit-2.ll
+++ b/llvm/test/DebugInfo/X86/sroasplit-2.ll
@@ -61,7 +61,7 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
 ; Function Attrs: nounwind
 declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture readonly, i64, i1) #2
 
-attributes #0 = { nounwind ssp uwtable "no-frame-pointer-elim"="true" }
+attributes #0 = { nounwind ssp uwtable "frame-pointer"="all" }
 attributes #1 = { nounwind readnone }
 attributes #2 = { nounwind }
 
diff --git a/llvm/test/DebugInfo/X86/stack-value-dwarf2.ll b/llvm/test/DebugInfo/X86/stack-value-dwarf2.ll
index 737a4c4dd..f7eebde84 100644
--- a/llvm/test/DebugInfo/X86/stack-value-dwarf2.ll
+++ b/llvm/test/DebugInfo/X86/stack-value-dwarf2.ll
@@ -70,7 +70,7 @@ declare void @_ZN23ConditionPlatformHelper4waitERii(i32* dereferenceable(4), i32
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.value(metadata, metadata, metadata) #1
 
-attributes #0 = { nounwind "no-frame-pointer-elim-non-leaf" }
+attributes #0 = { nounwind "frame-pointer"="non-leaf" }
 attributes #1 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/DebugInfo/X86/stack-value-piece.ll b/llvm/test/DebugInfo/X86/stack-value-piece.ll
index 6b6795c34..5b03a10d2 100644
--- a/llvm/test/DebugInfo/X86/stack-value-piece.ll
+++ b/llvm/test/DebugInfo/X86/stack-value-piece.ll
@@ -1,4 +1,4 @@
-; RUN: llc %s -filetype=obj -o - | llvm-dwarfdump -v - | FileCheck %s
+; RUN: llc %s -filetype=obj -o - | llvm-dwarfdump -debug-info -debug-loc - | FileCheck %s
 ; Test that DW_OP_piece is emitted for constants.
 ;
 ; // Generated from:
@@ -16,25 +16,25 @@
 
 ; CHECK: .debug_info contents:
 ; CHECK: DW_TAG_subprogram
-; CHECK:   DW_AT_name {{.*}} "i"
+; CHECK:   DW_AT_name ("i")
 ; CHECK:   DW_TAG_variable
-; CHECK-NEXT:   DW_AT_location {{.*}} ([[I:.*]]
+; CHECK-NEXT:   DW_AT_location ([[I:0x[0-9a-f]+]]
 ; CHECK-NEXT:     [{{.*}}, {{.*}}): DW_OP_reg5 RDI, DW_OP_piece 0x4, DW_OP_lit0, DW_OP_stack_value, DW_OP_piece 0x4)
-; CHECK-NEXT:   DW_AT_name {{.*}} "r"
+; CHECK-NEXT:   DW_AT_name ("r")
 ;
 ; CHECK: DW_TAG_subprogram
-; CHECK:   DW_AT_name {{.*}} "f"
+; CHECK:   DW_AT_name ("f")
 ; CHECK:   DW_TAG_variable
-; CHECK-NEXT:   DW_AT_location {{.*}} ([[F:.*]]
+; CHECK-NEXT:   DW_AT_location ([[F:0x[0-9a-f]+]]
 ; CHECK-NEXT:     [{{.*}}, {{.*}}): DW_OP_reg17 XMM0, DW_OP_piece 0x4, DW_OP_lit0, DW_OP_stack_value, DW_OP_piece 0x4
 ; CHECK-NEXT:     [{{.*}}, {{.*}}): DW_OP_piece 0x4, DW_OP_lit0, DW_OP_stack_value, DW_OP_piece 0x4)
-; CHECK-NEXT:   DW_AT_name {{.*}} "r"
+; CHECK-NEXT:   DW_AT_name ("r")
 ;
 ; CHECK: .debug_loc contents:
 ; CHECK:      [[I]]:
-; CHECK-NEXT:   [{{.*}}, {{.*}}): DW_OP_reg5 RDI, DW_OP_piece 0x4, DW_OP_lit0, DW_OP_stack_value, DW_OP_piece 0x4
+; CHECK-NEXT:   ({{.*}}, {{.*}}): DW_OP_reg5 RDI, DW_OP_piece 0x4, DW_OP_lit0, DW_OP_stack_value, DW_OP_piece 0x4
 ; CHECK:      [[F]]:
-; CHECK-NEXT:   [{{.*}}, {{.*}}): DW_OP_reg17 XMM0, DW_OP_piece 0x4, DW_OP_lit0, DW_OP_stack_value, DW_OP_piece 0x4
+; CHECK-NEXT:   ({{.*}}, {{.*}}): DW_OP_reg17 XMM0, DW_OP_piece 0x4, DW_OP_lit0, DW_OP_stack_value, DW_OP_piece 0x4
 
 source_filename = "stack-value-piece.c"
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/DebugInfo/X86/string-offsets-table-order.ll b/llvm/test/DebugInfo/X86/string-offsets-table-order.ll
index e39f70c0e..ab54930fe 100644
--- a/llvm/test/DebugInfo/X86/string-offsets-table-order.ll
+++ b/llvm/test/DebugInfo/X86/string-offsets-table-order.ll
@@ -12,11 +12,11 @@
 ; in different order.
 
 ; CHECK: .debug_info contents:
-; CHECK:   DW_TAG_compile_unit
+; CHECK:   DW_TAG_skeleton_unit
 ; CHECK:     DW_AT_comp_dir [DW_FORM_strx1] (indexed (00000000) string = "X3")
-; CHECK:   DW_TAG_compile_unit
+; CHECK:   DW_TAG_skeleton_unit
 ; CHECK:     DW_AT_comp_dir [DW_FORM_strx1] (indexed (00000001) string = "X2")
-; CHECK:   DW_TAG_compile_unit
+; CHECK:   DW_TAG_skeleton_unit
 ; CHECK:     DW_AT_comp_dir [DW_FORM_strx1] (indexed (00000002) string = "X1")
 ; CHECK: .debug_info.dwo contents:
 
diff --git a/llvm/test/DebugInfo/X86/string-offsets-table.ll b/llvm/test/DebugInfo/X86/string-offsets-table.ll
index 9960fd833..cff779445 100644
--- a/llvm/test/DebugInfo/X86/string-offsets-table.ll
+++ b/llvm/test/DebugInfo/X86/string-offsets-table.ll
@@ -56,11 +56,11 @@
 ; SPLIT:      .debug_info contents:
 ; SPLIT-NEXT: 0x00000000: Compile Unit:{{.*}}DW_UT_skeleton
 ; SPLIT-NOT:  contents:
-; SPLIT:      DW_TAG_compile_unit
+; SPLIT:      DW_TAG_skeleton_unit
 ; SPLIT-NOT:  {{DW_TAG|contents:}}
 ; SPLIT:      DW_AT_str_offsets_base [DW_FORM_sec_offset] (0x00000008)
 ; SPLIT:      DW_AT_comp_dir [DW_FORM_strx1] (indexed (00000000) string = "/home/test")
-; SPLIT:      DW_AT_GNU_dwo_name [DW_FORM_strx1] (indexed (00000001) string = "foo.dwo")
+; SPLIT:      DW_AT_dwo_name [DW_FORM_strx1] (indexed (00000001) string = "foo.dwo")
 
 ; Check for the split CU in .debug_info.dwo.
 ; SPLIT:      .debug_info.dwo contents:
diff --git a/llvm/test/DebugInfo/X86/template.ll b/llvm/test/DebugInfo/X86/template.ll
index 06306af52..084eac96b 100644
--- a/llvm/test/DebugInfo/X86/template.ll
+++ b/llvm/test/DebugInfo/X86/template.ll
@@ -1,6 +1,7 @@
 ; REQUIRES: object-emission
 
 ; RUN: llc -mtriple=x86_64-linux -O0 -filetype=obj < %s | llvm-dwarfdump -v -debug-info - | FileCheck %s
+; RUN: llc -mtriple=x86_64-linux -O0 -filetype=obj < %s | not llvm-dwarfdump -verify - | FileCheck %s --check-prefix VERIFY
 
 ; IR generated with `clang++ -g -emit-llvm -S` from the following code:
 ; template<int x, int*, template<typename> class y, decltype(nullptr) n, int ...z>  int func() {
@@ -11,6 +12,9 @@
 ; int glbl = func<3, &glbl, y_impl, nullptr, 1, 2>();
 ; y_impl<int>::nested n;
 
+; VERIFY-NOT: error: DIE has DW_AT_type with incompatible tag DW_TAG_unspecified_type
+; VERIFY: error: DIEs have overlapping address ranges
+
 ; CHECK: [[INT:0x[0-9a-f]*]]:{{ *}}DW_TAG_base_type
 ; CHECK-NEXT: DW_AT_name{{.*}} = "int"
 
diff --git a/llvm/test/DebugInfo/X86/tls.ll b/llvm/test/DebugInfo/X86/tls.ll
index 071ae15b8..d03eacfee 100644
--- a/llvm/test/DebugInfo/X86/tls.ll
+++ b/llvm/test/DebugInfo/X86/tls.ll
@@ -106,7 +106,7 @@ entry:
   ret i32 0, !dbg !18
 }
 
-attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!6}
 !llvm.module.flags = !{!9, !10}
diff --git a/llvm/test/DebugInfo/X86/tu-to-non-named-type.ll b/llvm/test/DebugInfo/X86/tu-to-non-named-type.ll
index 556e2adb1..eb6844673 100644
--- a/llvm/test/DebugInfo/X86/tu-to-non-named-type.ll
+++ b/llvm/test/DebugInfo/X86/tu-to-non-named-type.ll
@@ -45,7 +45,7 @@ entry:
 ; Function Attrs: nounwind readnone speculatable
 declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
 
-attributes #0 = { noinline nounwind optnone uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noinline nounwind optnone uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone speculatable }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/DebugInfo/X86/void-typedef.ll b/llvm/test/DebugInfo/X86/void-typedef.ll
index 64a56aefd..e69073b43 100644
--- a/llvm/test/DebugInfo/X86/void-typedef.ll
+++ b/llvm/test/DebugInfo/X86/void-typedef.ll
@@ -52,7 +52,7 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
 ; Function Attrs: noreturn nounwind
 declare void @llvm.trap() #2
 
-attributes #0 = { noinline nounwind optnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-features"="+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noinline nounwind optnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-features"="+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone speculatable }
 attributes #2 = { noreturn nounwind }
 
diff --git a/llvm/test/DebugInfo/salvage-cast-debug-info.ll b/llvm/test/DebugInfo/salvage-cast-debug-info.ll
new file mode 100644
index 000000000..f39ac46a6
--- /dev/null
+++ b/llvm/test/DebugInfo/salvage-cast-debug-info.ll
@@ -0,0 +1,25 @@
+; RUN: opt %s -debugify -early-cse -S | FileCheck %s
+define i32 @foo(i64 %nose, i32 %more) {
+; CHECK-LABEL: @foo(
+; CHECK: call void @llvm.dbg.value(metadata i64 %nose, metadata [[V1:![0-9]+]], metadata !DIExpression(DW_OP_LLVM_convert, 64, DW_ATE_unsigned, DW_OP_LLVM_convert, 32, DW_ATE_unsigned
+; CHECK: call void @llvm.dbg.value(metadata i64 %nose.shift, metadata [[V2:![0-9]+]]
+; CHECK: call void @llvm.dbg.value(metadata i64 %nose.shift, metadata [[V3:![0-9]+]], metadata !DIExpression(DW_OP_LLVM_convert, 64, DW_ATE_unsigned, DW_OP_LLVM_convert, 32, DW_ATE_unsigned
+
+entry:
+  %nose.trunc = trunc i64 %nose to i32
+  %nose.shift = lshr i64 %nose, 32
+  %nose.trunc.2 = trunc i64 %nose.shift to i32
+  %add = add nsw i32 %more, 1
+  ret i32 %add
+}
+
+!llvm.module.flags = !{!0, !1}
+!llvm.ident = !{!2}
+
+!0 = !{i32 1, !"wchar_size", i32 2}
+!1 = !{i32 7, !"PIC Level", i32 2}
+!2 = !{!"clang version 10.0.0 "}
+
+; CHECK: [[V1]] = !DILocalVariable(
+; CHECK: [[V2]] = !DILocalVariable(
+; CHECK: [[V3]] = !DILocalVariable(
diff --git a/llvm/test/Examples/IRTransforms/SimplifyCFG/tut-simplify-cfg-blockaddress.ll b/llvm/test/Examples/IRTransforms/SimplifyCFG/tut-simplify-cfg-blockaddress.ll
new file mode 100644
index 000000000..faf60f3ac
--- /dev/null
+++ b/llvm/test/Examples/IRTransforms/SimplifyCFG/tut-simplify-cfg-blockaddress.ll
@@ -0,0 +1,23 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -tut-simplifycfg -tut-simplifycfg-version=v1 -S < %s | FileCheck %s
+; RUN: opt -tut-simplifycfg -tut-simplifycfg-version=v2 -S < %s | FileCheck %s
+; RUN: opt -tut-simplifycfg -tut-simplifycfg-version=v3 -S < %s | FileCheck %s
+
+define i8* @simp1(i32 %x) {
+; CHECK-LABEL: @simp1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 42
+; CHECK-NEXT:    [[ADDR:%.*]] = select i1 [[CMP]], i8* inttoptr (i32 1 to i8*), i8* inttoptr (i32 1 to i8*)
+; CHECK-NEXT:    ret i8* [[ADDR]]
+;
+entry:
+  %cmp = icmp slt i32 %x, 42
+  %addr = select i1 %cmp, i8* blockaddress(@simp1, %bb1), i8* blockaddress(@simp1, %bb2)
+  ret i8* %addr
+
+bb1:
+  ret i8* null
+
+bb2:
+  ret i8* null
+}
diff --git a/llvm/test/Examples/IRTransforms/SimplifyCFG/tut-simplify-cfg1.ll b/llvm/test/Examples/IRTransforms/SimplifyCFG/tut-simplify-cfg1.ll
new file mode 100644
index 000000000..cb0f82e37
--- /dev/null
+++ b/llvm/test/Examples/IRTransforms/SimplifyCFG/tut-simplify-cfg1.ll
@@ -0,0 +1,90 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -tut-simplifycfg -tut-simplifycfg-version=v1 -S < %s | FileCheck %s
+; RUN: opt -tut-simplifycfg -tut-simplifycfg-version=v2 -S < %s | FileCheck %s
+; RUN: opt -tut-simplifycfg -tut-simplifycfg-version=v3 -S < %s | FileCheck %s
+
+define i32 @simp1() {
+; CHECK-LABEL: @simp1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret i32 10
+;
+entry:
+  br i1 true, label %if.then, label %if.else
+
+if.then:
+  ret i32 10
+
+if.else:
+  ret i32 12
+}
+
+define i32 @simp2() {
+; CHECK-LABEL: @simp2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret i32 200
+;
+entry:
+  br i1 false, label %if.then, label %if.else
+
+if.then:
+  ret i32 99
+
+if.else:
+  ret i32 200
+}
+
+declare void @foo(i64)
+
+define i64 @merge_into_predecessor(i64 %a, i64 %b) {
+; CHECK-LABEL: @merge_into_predecessor(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[R:%.*]] = add i64 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    call void @foo(i64 [[R]])
+; CHECK-NEXT:    call void @foo(i64 [[A]])
+; CHECK-NEXT:    ret i64 [[R]]
+;
+entry:
+  br label %bb.next
+
+bb.next:
+  %r = add i64 %a, %b
+  call void @foo(i64 %r)
+  call void @foo(i64 %a)
+  br label %bb.next.next
+
+bb.next.next:
+  ret i64 %r
+}
+
+define i64 @merge_into_predecessor_with_phi(i64 %a, i64 %b, i1 %c) {
+; CHECK-LABEL: @merge_into_predecessor_with_phi(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    call void @foo(i64 [[B:%.*]])
+; CHECK-NEXT:    [[R:%.*]] = add i64 [[A:%.*]], [[B]]
+; CHECK-NEXT:    call void @foo(i64 [[R]])
+; CHECK-NEXT:    call void @foo(i64 [[A]])
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[BB_NEXT_NEXT:%.*]], label [[BB_EXIT:%.*]]
+; CHECK:       bb.next.next:
+; CHECK-NEXT:    br label [[BB_EXIT]]
+; CHECK:       bb.exit:
+; CHECK-NEXT:    [[RET:%.*]] = phi i64 [ [[R]], [[ENTRY:%.*]] ], [ 10, [[BB_NEXT_NEXT]] ]
+; CHECK-NEXT:    ret i64 [[RET]]
+;
+entry:
+  call void @foo(i64 %b)
+  br label %bb.next
+
+bb.next:
+  %r = add i64 %a, %b
+  call void @foo(i64 %r)
+  call void @foo(i64 %a)
+  br i1 %c, label %bb.next.next, label %bb.exit
+
+bb.next.next:
+  br label %bb.exit
+
+bb.exit:
+  %ret = phi i64 [ %r, %bb.next], [ 10, %bb.next.next]
+  ret i64 %ret
+
+}
diff --git a/llvm/test/Examples/IRTransforms/SimplifyCFG/tut-simplify-cfg2-dead-block-order.ll b/llvm/test/Examples/IRTransforms/SimplifyCFG/tut-simplify-cfg2-dead-block-order.ll
new file mode 100644
index 000000000..11b70fa52
--- /dev/null
+++ b/llvm/test/Examples/IRTransforms/SimplifyCFG/tut-simplify-cfg2-dead-block-order.ll
@@ -0,0 +1,109 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -tut-simplifycfg -tut-simplifycfg-version=v1 -S < %s | FileCheck %s
+; RUN: opt -tut-simplifycfg -tut-simplifycfg-version=v2 -S < %s | FileCheck %s
+; RUN: opt -tut-simplifycfg -tut-simplifycfg-version=v3 -S < %s | FileCheck %s
+
+define i32 @remove_dead_blocks() {
+; CHECK-LABEL: @remove_dead_blocks(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret i32 1
+; CHECK-NEXT:  }
+;
+entry:
+  ret i32 1
+
+bb.1:
+  ret i32 2
+
+bb.2:
+  ret i32 3
+}
+
+define i32 @simp1() {
+; CHECK-LABEL: @simp1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret i32 1
+; CHECK:       bb.1:
+; CHECK-NEXT:    ret i32 2
+; CHECK-NEXT:  }
+;
+entry:
+  ret i32 1
+
+bb.1:
+  ret i32 2
+
+bb.2:
+  br i1 undef, label %bb.1, label %bb.3
+
+bb.3:
+  ret i32 3
+}
+
+define i32 @remove_dead_block_with_phi() {
+; CHECK-LABEL: @remove_dead_block_with_phi(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[BB_2:%.*]]
+; CHECK:       bb.2:
+; CHECK-NEXT:    ret i32 1
+; CHECK-NEXT:  }
+;
+entry:
+  br label %bb.2
+
+bb.1:
+  br label %bb.2
+
+bb.2:
+  %rv = phi i32 [ 1, %entry ], [ 2, %bb.1 ]
+  ret i32 %rv
+}
+
+define i32 @remove_dead_blocks_remaining_uses(i32 %a) {
+; CHECK-LABEL: @remove_dead_blocks_remaining_uses(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret i32 1
+; CHECK-NEXT:  }
+;
+entry:
+  ret i32 1
+
+bb.2:
+  ret i32 %res
+
+bb.1:
+  %res = add i32 %a, 10
+  br label %bb.2
+}
+
+define i32 @remove_dead_blocks_remaining_uses2(i32 %a, i1 %cond) {
+; CHECK-LABEL: @remove_dead_blocks_remaining_uses2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret i32 1
+; CHECK:       bb.2:
+; CHECK-NEXT:    [[RES2:%.*]] = add i32 undef, 10
+; CHECK-NEXT:    [[RES3:%.*]] = mul i32 [[RES2]], undef
+; CHECK-NEXT:    ret i32 [[RES3]]
+; CHECK:       bb.3:
+; CHECK-NEXT:    ret i32 undef
+; CHECK-NEXT:  }
+;
+entry:
+  ret i32 1
+
+bb.2:
+  %res2 = add i32 %res, 10
+  %res3 = mul i32 %res2, %res
+  ret i32 %res3
+
+bb.3:
+  br label %bb.4
+
+bb.4:
+  ret i32 %res
+
+bb.1:
+  %res = add i32 %a, 10
+  br i1 %cond, label %bb.2, label %bb.3
+  br label %bb.2
+}
diff --git a/llvm/test/Examples/IRTransforms/SimplifyCFG/tut-simplify-cfg3-phis.ll b/llvm/test/Examples/IRTransforms/SimplifyCFG/tut-simplify-cfg3-phis.ll
new file mode 100644
index 000000000..76db503fa
--- /dev/null
+++ b/llvm/test/Examples/IRTransforms/SimplifyCFG/tut-simplify-cfg3-phis.ll
@@ -0,0 +1,70 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -tut-simplifycfg -tut-simplifycfg-version=v1 -S < %s | FileCheck %s
+; RUN: opt -tut-simplifycfg -tut-simplifycfg-version=v2 -S < %s | FileCheck %s
+; RUN: opt -tut-simplifycfg -tut-simplifycfg-version=v3 -S < %s | FileCheck %s
+
+define i32 @phi_cond_branch_eliminated() {
+; CHECK-LABEL: @phi_cond_branch_eliminated(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret i32 20
+;
+entry:
+  br i1 true, label %bb.2, label %bb.3
+
+bb.2:
+  br label %bb.3
+
+bb.3:
+  %ret = phi i32 [ 10, %entry ], [ 20, %bb.2 ]
+  ret i32 %ret
+}
+
+define i32 @phi_removed() {
+; CHECK-LABEL: @phi_removed(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[BB_3:%.*]]
+; CHECK:       bb.3:
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  br i1 false, label %bb.2, label %bb.3
+
+bb.2:
+  %pv = phi i32 [ 10, %entry ]
+  br label %bb.3
+
+bb.3:
+  ret i32 0
+}
+
+define i32 @phi_in_dead_region() {
+; CHECK-LABEL: @phi_in_dead_region(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret i32 1
+;
+entry:
+  ret i32 1
+
+bb.1:
+  br i1 true, label %bb.2, label %bb.3
+
+bb.2:
+  br label %bb.3
+
+bb.3:
+  %ret = phi i32 [ 10, %bb.1 ], [ 20, %bb.2 ]
+  ret i32 %ret
+}
+
+define i32 @phi_in_mergable_blocks() {
+; CHECK-LABEL: @phi_in_mergable_blocks(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret i32 10
+;
+entry:
+  br label %bb.1
+
+bb.1:
+  %pv = phi i32 [ 10, %entry ]
+  ret i32 %pv
+}
diff --git a/llvm/test/Examples/IRTransforms/SimplifyCFG/tut-simplify-cfg4-multiple-duplicate-cfg-updates.ll b/llvm/test/Examples/IRTransforms/SimplifyCFG/tut-simplify-cfg4-multiple-duplicate-cfg-updates.ll
new file mode 100644
index 000000000..82a0e0dac
--- /dev/null
+++ b/llvm/test/Examples/IRTransforms/SimplifyCFG/tut-simplify-cfg4-multiple-duplicate-cfg-updates.ll
@@ -0,0 +1,40 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -tut-simplifycfg -tut-simplifycfg-version=v1 < %s -S -verify-dom-info | FileCheck %s
+; RUN: opt -tut-simplifycfg -tut-simplifycfg-version=v2 < %s -S -verify-dom-info | FileCheck %s
+; RUN: opt -tut-simplifycfg -tut-simplifycfg-version=v3 < %s -S -verify-dom-info | FileCheck %s
+
+; Check that we do not crash when we remove edges multiple times in
+; the DomTreeUpdater.
+define void @test() {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    switch i8 undef, label [[IF_THEN_EPIL:%.*]] [
+; CHECK-NEXT:    i8 32, label [[FOR_INC_EPIL:%.*]]
+; CHECK-NEXT:    i8 46, label [[FOR_INC_EPIL]]
+; CHECK-NEXT:    i8 95, label [[FOR_INC_EPIL]]
+; CHECK-NEXT:    i8 45, label [[FOR_INC_EPIL]]
+; CHECK-NEXT:    i8 126, label [[FOR_INC_EPIL]]
+; CHECK-NEXT:    ]
+; CHECK:       if.then.epil:
+; CHECK-NEXT:    unreachable
+; CHECK:       for.inc.epil:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %for.body.epil
+
+for.body.epil:                                    ; preds = %entry
+  switch i8 undef, label %if.then.epil [
+  i8 32, label %for.inc.epil
+  i8 46, label %for.inc.epil
+  i8 95, label %for.inc.epil
+  i8 45, label %for.inc.epil
+  i8 126, label %for.inc.epil
+  ]
+
+if.then.epil:                                     ; preds = %for.body.epil
+  unreachable
+
+for.inc.epil:                                     ; preds = %for.body.epil, %for.body.epil, %for.body.epil, %for.body.epil, %for.body.epil
+  ret void
+}
diff --git a/llvm/test/Examples/IRTransforms/SimplifyCFG/tut-simplify-cfg5-del-phis-for-dead-block.ll b/llvm/test/Examples/IRTransforms/SimplifyCFG/tut-simplify-cfg5-del-phis-for-dead-block.ll
new file mode 100644
index 000000000..b3edd1aa5
--- /dev/null
+++ b/llvm/test/Examples/IRTransforms/SimplifyCFG/tut-simplify-cfg5-del-phis-for-dead-block.ll
@@ -0,0 +1,122 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -tut-simplifycfg -tut-simplifycfg-version=v1 < %s -S -verify-dom-info | FileCheck %s
+; RUN: opt -tut-simplifycfg -tut-simplifycfg-version=v2 < %s -S -verify-dom-info | FileCheck %s
+; RUN: opt -tut-simplifycfg -tut-simplifycfg-version=v3 < %s -S -verify-dom-info | FileCheck %s
+
+define void @test() {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    switch i32 undef, label [[SW_DEFAULT23:%.*]] [
+; CHECK-NEXT:    i32 129, label [[SW_BB:%.*]]
+; CHECK-NEXT:    i32 215, label [[SW_BB1:%.*]]
+; CHECK-NEXT:    i32 117, label [[SW_BB1]]
+; CHECK-NEXT:    i32 207, label [[SW_BB1]]
+; CHECK-NEXT:    i32 158, label [[SW_BB1]]
+; CHECK-NEXT:    i32 94, label [[SW_BB1]]
+; CHECK-NEXT:    i32 219, label [[SW_BB1]]
+; CHECK-NEXT:    i32 88, label [[SW_BB1]]
+; CHECK-NEXT:    i32 168, label [[SW_BB1]]
+; CHECK-NEXT:    i32 295, label [[SW_BB1]]
+; CHECK-NEXT:    i32 294, label [[SW_BB1]]
+; CHECK-NEXT:    i32 296, label [[SW_BB1]]
+; CHECK-NEXT:    i32 67, label [[SW_BB1]]
+; CHECK-NEXT:    i32 293, label [[SW_BB1]]
+; CHECK-NEXT:    i32 382, label [[SW_BB1]]
+; CHECK-NEXT:    i32 335, label [[SW_BB1]]
+; CHECK-NEXT:    i32 393, label [[SW_BB1]]
+; CHECK-NEXT:    i32 415, label [[SW_BB1]]
+; CHECK-NEXT:    i32 400, label [[SW_BB1]]
+; CHECK-NEXT:    i32 383, label [[SW_BB1]]
+; CHECK-NEXT:    i32 421, label [[SW_BB1]]
+; CHECK-NEXT:    i32 422, label [[SW_BB1]]
+; CHECK-NEXT:    i32 302, label [[SW_BB1]]
+; CHECK-NEXT:    i32 303, label [[SW_BB1]]
+; CHECK-NEXT:    i32 304, label [[SW_BB1]]
+; CHECK-NEXT:    i32 420, label [[SW_BB1]]
+; CHECK-NEXT:    i32 401, label [[SW_EPILOG24:%.*]]
+; CHECK-NEXT:    i32 53, label [[SW_BB12:%.*]]
+; CHECK-NEXT:    i32 44, label [[SW_BB12]]
+; CHECK-NEXT:    ]
+; CHECK:       sw.bb:
+; CHECK-NEXT:    unreachable
+; CHECK:       sw.bb1:
+; CHECK-NEXT:    br label [[SW_EPILOG24]]
+; CHECK:       sw.bb12:
+; CHECK-NEXT:    switch i32 undef, label [[SW_DEFAULT:%.*]] [
+; CHECK-NEXT:    i32 47, label [[SW_BB13:%.*]]
+; CHECK-NEXT:    i32 8, label [[SW_BB13]]
+; CHECK-NEXT:    ]
+; CHECK:       sw.bb13:
+; CHECK-NEXT:    unreachable
+; CHECK:       sw.default:
+; CHECK-NEXT:    unreachable
+; CHECK:       sw.default23:
+; CHECK-NEXT:    unreachable
+; CHECK:       sw.epilog24:
+; CHECK-NEXT:    [[PREVIOUS_3:%.*]] = phi i32 [ undef, [[SW_BB1]] ], [ 401, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    unreachable
+;
+entry:
+  br label %while.body
+
+while.body:                                       ; preds = %entry
+  switch i32 undef, label %sw.default23 [
+  i32 129, label %sw.bb
+  i32 215, label %sw.bb1
+  i32 117, label %sw.bb1
+  i32 207, label %sw.bb1
+  i32 158, label %sw.bb1
+  i32 94, label %sw.bb1
+  i32 219, label %sw.bb1
+  i32 88, label %sw.bb1
+  i32 168, label %sw.bb1
+  i32 295, label %sw.bb1
+  i32 294, label %sw.bb1
+  i32 296, label %sw.bb1
+  i32 67, label %sw.bb1
+  i32 293, label %sw.bb1
+  i32 382, label %sw.bb1
+  i32 335, label %sw.bb1
+  i32 393, label %sw.bb1
+  i32 415, label %sw.bb1
+  i32 400, label %sw.bb1
+  i32 383, label %sw.bb1
+  i32 421, label %sw.bb1
+  i32 422, label %sw.bb1
+  i32 302, label %sw.bb1
+  i32 303, label %sw.bb1
+  i32 304, label %sw.bb1
+  i32 420, label %sw.bb1
+  i32 401, label %sw.epilog24
+  i32 53, label %sw.bb12
+  i32 44, label %sw.bb12
+  ]
+
+sw.bb:                                            ; preds = %while.body
+  unreachable
+
+sw.bb1:                                           ; preds = %while.body, %while.body, %while.body, %while.body, %while.body, %while.body, %while.body, %while.body, %while.body, %while.body, %while.body, %while.body, %while.body, %while.body, %while.body, %while.body, %while.body, %while.body, %while.body, %while.body, %while.body, %while.body, %while.body, %while.body, %while.body
+  br i1 false, label %land.lhs.true, label %sw.epilog24
+
+land.lhs.true:                                    ; preds = %sw.bb1
+  br label %sw.epilog24
+
+sw.bb12:                                          ; preds = %while.body, %while.body
+  switch i32 undef, label %sw.default [
+  i32 47, label %sw.bb13
+  i32 8, label %sw.bb13
+  ]
+
+sw.bb13:                                          ; preds = %sw.bb12, %sw.bb12
+  unreachable
+
+sw.default:                                       ; preds = %sw.bb12
+  unreachable
+
+sw.default23:                                     ; preds = %while.body
+  unreachable
+
+sw.epilog24:                                      ; preds = %land.lhs.true, %sw.bb1, %while.body
+  %Previous.3 = phi i32 [ undef, %land.lhs.true ], [ undef, %sw.bb1 ], [ 401, %while.body ]
+  unreachable
+}
diff --git a/llvm/test/Examples/IRTransforms/SimplifyCFG/tut-simplify-cfg6-dead-self-loop.ll b/llvm/test/Examples/IRTransforms/SimplifyCFG/tut-simplify-cfg6-dead-self-loop.ll
new file mode 100644
index 000000000..f9705a694
--- /dev/null
+++ b/llvm/test/Examples/IRTransforms/SimplifyCFG/tut-simplify-cfg6-dead-self-loop.ll
@@ -0,0 +1,25 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -tut-simplifycfg -tut-simplifycfg-version=v1 -S < %s | FileCheck %s
+; RUN: opt -tut-simplifycfg -tut-simplifycfg-version=v2 -S < %s | FileCheck %s
+; RUN: opt -tut-simplifycfg -tut-simplifycfg-version=v3 -S < %s | FileCheck %s
+
+define i32 @simp1() {
+; CHECK-LABEL: @simp1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret i32 1
+; CHECK:       bb.1:
+; CHECK-NEXT:    br label [[BB_1:%.*]]
+; CHECK:       bb.2:
+; CHECK-NEXT:    [[P:%.*]] = phi i32 [ 0, [[BB_2:%.*]] ]
+; CHECK-NEXT:    br label [[BB_2]]
+;
+entry:
+  ret i32 1
+
+bb.1:
+  br label %bb.1
+
+bb.2:
+  %p = phi i32 [ 0, %bb.2]
+  br label %bb.2
+}
diff --git a/llvm/test/ExecutionEngine/OrcLazy/emulated-tls.ll b/llvm/test/ExecutionEngine/OrcLazy/emulated-tls.ll
new file mode 100644
index 000000000..067052c07
--- /dev/null
+++ b/llvm/test/ExecutionEngine/OrcLazy/emulated-tls.ll
@@ -0,0 +1,23 @@
+; RUN: not lli -no-process-syms -emulated-tls -jit-kind=orc-lazy %s 2>&1 \
+; RUN:   | FileCheck %s
+;
+; Test that emulated-tls does not generate any unexpected errors.
+;
+; Unfortunately we cannot test successful execution of JIT'd code with
+; emulated-tls as this would require the JIT itself, in this case lli, to be
+; built with emulated-tls, which is not a common configuration. Instead we test
+; that the only error produced by the JIT for a thread-local with emulated-tls
+; enabled is a missing symbol error for __emutls_get_address. An unresolved
+; reference to this symbol (and only this symbol) implies (1) that the emulated
+; tls lowering was applied, and (2) that thread locals defined in the JIT'd code
+; were otherwise handled correctly.
+
+; CHECK: JIT session error: Symbols not found: [ {{[^,]*}}__emutls_get_address ]
+
+@x = thread_local global i32 42, align 4
+
+define i32 @main(i32 %argc, i8** %argv) {
+entry:
+  %0 = load i32, i32* @x, align 4
+  ret i32 %0
+}
diff --git a/llvm/test/ExecutionEngine/OrcLazy/printargv.ll b/llvm/test/ExecutionEngine/OrcLazy/printargv.ll
new file mode 100644
index 000000000..b5153a4a2
--- /dev/null
+++ b/llvm/test/ExecutionEngine/OrcLazy/printargv.ll
@@ -0,0 +1,81 @@
+; RUN: lli -jit-kind=orc-lazy %s a b c | FileCheck %s
+
+; CHECK: argc = 4
+; CHECK-NEXT: argv = ["{{.*}}printargv.ll", "a", "b", "c"]
+; CHECK-NEXT; argv[4] = null
+
+@.str = private unnamed_addr constant [11 x i8] c"argc = %i\0A\00", align 1
+@.str.1 = private unnamed_addr constant [9 x i8] c"argv = [\00", align 1
+@.str.3 = private unnamed_addr constant [5 x i8] c"\22%s\22\00", align 1
+@.str.4 = private unnamed_addr constant [5 x i8] c"null\00", align 1
+@.str.5 = private unnamed_addr constant [7 x i8] c", \22%s\22\00", align 1
+@.str.6 = private unnamed_addr constant [15 x i8] c"argv[%i] = %s\0A\00", align 1
+@.str.7 = private unnamed_addr constant [5 x i8] c"junk\00", align 1
+@str.8 = private unnamed_addr constant [2 x i8] c"]\00", align 1
+
+define i32 @main(i32 %argc, i8** nocapture readonly %argv)  {
+entry:
+  %call = tail call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([11 x i8], [11 x i8]* @.str, i64 0, i64 0), i32 %argc)
+  %call1 = tail call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([9 x i8], [9 x i8]* @.str.1, i64 0, i64 0))
+  %cmp = icmp eq i32 %argc, 0
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  %puts36 = tail call i32 @puts(i8* getelementptr inbounds ([2 x i8], [2 x i8]* @str.8, i64 0, i64 0))
+  br label %if.end
+
+if.end:
+  %0 = load i8*, i8** %argv, align 8
+  %tobool = icmp eq i8* %0, null
+  br i1 %tobool, label %if.else, label %if.then3
+
+if.then3:
+  %call5 = tail call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([5 x i8], [5 x i8]* @.str.3, i64 0, i64 0), i8* %0)
+  br label %if.end7
+
+if.else:
+  %call6 = tail call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([5 x i8], [5 x i8]* @.str.4, i64 0, i64 0))
+  br label %if.end7
+
+if.end7:
+  %cmp837 = icmp eq i32 %argc, 1
+  br i1 %cmp837, label %for.cond.cleanup, label %for.body.preheader
+
+for.body.preheader:
+  %1 = zext i32 %argc to i64
+  br label %for.body
+
+for.cond.cleanup:
+  %puts = tail call i32 @puts(i8* getelementptr inbounds ([2 x i8], [2 x i8]* @str.8, i64 0, i64 0))
+  %idxprom19 = sext i32 %argc to i64
+  %arrayidx20 = getelementptr inbounds i8*, i8** %argv, i64 %idxprom19
+  %2 = load i8*, i8** %arrayidx20, align 8
+  %tobool21 = icmp eq i8* %2, null
+  %cond = select i1 %tobool21, i8* getelementptr inbounds ([5 x i8], [5 x i8]* @.str.4, i64 0, i64 0), i8* getelementptr inbounds ([5 x i8], [5 x i8]* @.str.7, i64 0, i64 0)
+  %call22 = tail call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([15 x i8], [15 x i8]* @.str.6, i64 0, i64 0), i32 %argc, i8* %cond)
+  ret i32 0
+
+for.body:
+  %indvars.iv = phi i64 [ 1, %for.body.preheader ], [ %indvars.iv.next, %for.inc ]
+  %arrayidx9 = getelementptr inbounds i8*, i8** %argv, i64 %indvars.iv
+  %3 = load i8*, i8** %arrayidx9, align 8
+  %tobool10 = icmp eq i8* %3, null
+  br i1 %tobool10, label %if.else15, label %if.then11
+
+if.then11:
+  %call14 = tail call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([7 x i8], [7 x i8]* @.str.5, i64 0, i64 0), i8* %3)
+  br label %for.inc
+
+if.else15:
+  %call16 = tail call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([5 x i8], [5 x i8]* @.str.4, i64 0, i64 0))
+  br label %for.inc
+
+for.inc:
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %cmp8 = icmp eq i64 %indvars.iv.next, %1
+  br i1 %cmp8, label %for.cond.cleanup, label %for.body
+}
+
+declare i32 @printf(i8* nocapture readonly, ...)
+
+declare i32 @puts(i8* nocapture readonly)
diff --git a/llvm/test/ExecutionEngine/OrcLazy/static-library-support.ll b/llvm/test/ExecutionEngine/OrcLazy/static-library-support.ll
new file mode 100644
index 000000000..304160c7f
--- /dev/null
+++ b/llvm/test/ExecutionEngine/OrcLazy/static-library-support.ll
@@ -0,0 +1,11 @@
+; This first line will generate the .o files for the next run line
+; RUN: llc -filetype=obj -o %t.o %p/Inputs/basic-object-source.ll
+; RUN: llvm-ar r %t.a %t.o
+; RUN: lli -jit-kind=orc-lazy -extra-archive %t.a %s
+
+declare i32 @foo()
+
+define i32 @main() {
+  %r = call i32 @foo( )   ; <i32> [#uses=1]
+  ret i32 %r
+}
diff --git a/llvm/test/Feature/fp-intrinsics.ll b/llvm/test/Feature/fp-intrinsics.ll
index 895858e5a..b92408a1b 100644
--- a/llvm/test/Feature/fp-intrinsics.ll
+++ b/llvm/test/Feature/fp-intrinsics.ll
@@ -3,13 +3,13 @@
 ; Test to verify that constants aren't folded when the rounding mode is unknown.
 ; CHECK-LABEL: @f1
 ; CHECK: call double @llvm.experimental.constrained.fdiv.f64
-define double @f1() {
+define double @f1() #0 {
 entry:
   %div = call double @llvm.experimental.constrained.fdiv.f64(
                                                double 1.000000e+00,
                                                double 1.000000e+01,
                                                metadata !"round.dynamic",
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
   ret double %div
 }
 
@@ -23,12 +23,12 @@ entry:
 ;
 ; CHECK-LABEL: @f2
 ; CHECK: call double @llvm.experimental.constrained.fsub.f64
-define double @f2(double %a) {
+define double @f2(double %a) #0 {
 entry:
   %div = call double @llvm.experimental.constrained.fsub.f64(
                                                double %a, double 0.000000e+00,
                                                metadata !"round.dynamic",
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
   ret double %div
 }
 
@@ -45,21 +45,21 @@ entry:
 ; CHECK: call double @llvm.experimental.constrained.fsub.f64
 ; CHECK: call double @llvm.experimental.constrained.fmul.f64
 ; CHECK: call double @llvm.experimental.constrained.fsub.f64
-define double @f3(double %a, double %b) {
+define double @f3(double %a, double %b) #0 {
 entry:
   %sub = call double @llvm.experimental.constrained.fsub.f64(
                                                double -0.000000e+00, double %a,
                                                metadata !"round.dynamic",
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
   %mul = call double @llvm.experimental.constrained.fmul.f64(
                                                double %sub, double %b,
                                                metadata !"round.dynamic",
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
   %ret = call double @llvm.experimental.constrained.fsub.f64(
                                                double -0.000000e+00,
                                                double %mul,
                                                metadata !"round.dynamic",
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
   ret double %ret
 }
 
@@ -77,7 +77,7 @@ entry:
 ; CHECK-LABEL: @f4
 ; CHECK-NOT: select
 ; CHECK: br i1 %cmp
-define double @f4(i32 %n, double %a) {
+define double @f4(i32 %n, double %a) #0 {
 entry:
   %cmp = icmp sgt i32 %n, 0
   br i1 %cmp, label %if.then, label %if.end
@@ -86,7 +86,7 @@ if.then:
   %add = call double @llvm.experimental.constrained.fadd.f64(
                                                double 1.000000e+00, double %a,
                                                metadata !"round.dynamic",
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
   br label %if.end
 
 if.end:
@@ -97,123 +97,123 @@ if.end:
 ; Verify that sqrt(42.0) isn't simplified when the rounding mode is unknown.
 ; CHECK-LABEL: f5
 ; CHECK: call double @llvm.experimental.constrained.sqrt
-define double @f5() {
+define double @f5() #0 {
 entry:
   %result = call double @llvm.experimental.constrained.sqrt.f64(double 42.0,
                                                metadata !"round.dynamic",
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
   ret double %result
 }
 
 ; Verify that pow(42.1, 3.0) isn't simplified when the rounding mode is unknown.
 ; CHECK-LABEL: f6
 ; CHECK: call double @llvm.experimental.constrained.pow
-define double @f6() {
+define double @f6() #0 {
 entry:
   %result = call double @llvm.experimental.constrained.pow.f64(double 42.1,
                                                double 3.0,
                                                metadata !"round.dynamic",
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
   ret double %result
 }
 
 ; Verify that powi(42.1, 3) isn't simplified when the rounding mode is unknown.
 ; CHECK-LABEL: f7
 ; CHECK: call double @llvm.experimental.constrained.powi
-define double @f7() {
+define double @f7() #0 {
 entry:
   %result = call double @llvm.experimental.constrained.powi.f64(double 42.1,
                                                i32 3,
                                                metadata !"round.dynamic",
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
   ret double %result
 }
 
 ; Verify that sin(42.0) isn't simplified when the rounding mode is unknown.
 ; CHECK-LABEL: f8
 ; CHECK: call double @llvm.experimental.constrained.sin
-define double @f8() {
+define double @f8() #0 {
 entry:
   %result = call double @llvm.experimental.constrained.sin.f64(double 42.0,
                                                metadata !"round.dynamic",
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
   ret double %result
 }
 
 ; Verify that cos(42.0) isn't simplified when the rounding mode is unknown.
 ; CHECK-LABEL: f9
 ; CHECK: call double @llvm.experimental.constrained.cos
-define double @f9() {
+define double @f9() #0 {
 entry:
   %result = call double @llvm.experimental.constrained.cos.f64(double 42.0,
                                                metadata !"round.dynamic",
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
   ret double %result
 }
 
 ; Verify that exp(42.0) isn't simplified when the rounding mode is unknown.
 ; CHECK-LABEL: f10
 ; CHECK: call double @llvm.experimental.constrained.exp
-define double @f10() {
+define double @f10() #0 {
 entry:
   %result = call double @llvm.experimental.constrained.exp.f64(double 42.0,
                                                metadata !"round.dynamic",
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
   ret double %result
 }
 
 ; Verify that exp2(42.1) isn't simplified when the rounding mode is unknown.
 ; CHECK-LABEL: f11
 ; CHECK: call double @llvm.experimental.constrained.exp2
-define double @f11() {
+define double @f11() #0 {
 entry:
   %result = call double @llvm.experimental.constrained.exp2.f64(double 42.1,
                                                metadata !"round.dynamic",
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
   ret double %result
 }
 
 ; Verify that log(42.0) isn't simplified when the rounding mode is unknown.
 ; CHECK-LABEL: f12
 ; CHECK: call double @llvm.experimental.constrained.log
-define double @f12() {
+define double @f12() #0 {
 entry:
   %result = call double @llvm.experimental.constrained.log.f64(double 42.0,
                                                metadata !"round.dynamic",
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
   ret double %result
 }
 
 ; Verify that log10(42.0) isn't simplified when the rounding mode is unknown.
 ; CHECK-LABEL: f13
 ; CHECK: call double @llvm.experimental.constrained.log10
-define double @f13() {
+define double @f13() #0 {
 entry:
   %result = call double @llvm.experimental.constrained.log10.f64(double 42.0,
                                                metadata !"round.dynamic",
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
   ret double %result
 }
 
 ; Verify that log2(42.0) isn't simplified when the rounding mode is unknown.
 ; CHECK-LABEL: f14
 ; CHECK: call double @llvm.experimental.constrained.log2
-define double @f14() {
+define double @f14() #0 {
 entry:
   %result = call double @llvm.experimental.constrained.log2.f64(double 42.0,
                                                metadata !"round.dynamic",
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
   ret double %result
 }
 
 ; Verify that rint(42.1) isn't simplified when the rounding mode is unknown.
 ; CHECK-LABEL: f15
 ; CHECK: call double @llvm.experimental.constrained.rint
-define double @f15() {
+define double @f15() #0 {
 entry:
   %result = call double @llvm.experimental.constrained.rint.f64(double 42.1,
                                                metadata !"round.dynamic",
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
   ret double %result
 }
 
@@ -221,12 +221,12 @@ entry:
 ; unknown.
 ; CHECK-LABEL: f16
 ; CHECK: call double @llvm.experimental.constrained.nearbyint
-define double @f16() {
+define double @f16() #0 {
 entry:
   %result = call double @llvm.experimental.constrained.nearbyint.f64(
                                                double 42.1,
                                                metadata !"round.dynamic",
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
   ret double %result
 }
 
@@ -234,24 +234,47 @@ entry:
 ; unknown.
 ; CHECK-LABEL: f17
 ; CHECK: call double @llvm.experimental.constrained.fma
-define double @f17() {
+define double @f17() #0 {
 entry:
   %result = call double @llvm.experimental.constrained.fma.f64(double 42.1, double 42.1, double 42.1,
                                                metadata !"round.dynamic",
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
   ret double %result
 }
 
+; Verify that fptoui(42.1) isn't simplified when the rounding mode is
+; unknown.
+; CHECK-LABEL: f18
+; CHECK: call zeroext i32 @llvm.experimental.constrained.fptoui
+define zeroext i32 @f18() #0 {
+entry:
+  %result = call zeroext i32 @llvm.experimental.constrained.fptoui.i32.f64(
+                                               double 42.1,
+                                               metadata !"fpexcept.strict") #0
+  ret i32 %result
+}
+
+; Verify that fptosi(42.1) isn't simplified when the rounding mode is
+; unknown.
+; CHECK-LABEL: f19
+; CHECK: call i32 @llvm.experimental.constrained.fptosi
+define i32 @f19() #0 {
+entry:
+  %result = call i32 @llvm.experimental.constrained.fptosi.i32.f64(double 42.1,
+                                               metadata !"fpexcept.strict") #0
+  ret i32 %result
+}
+
 ; Verify that fptrunc(42.1) isn't simplified when the rounding mode is
 ; unknown.
 ; CHECK-LABEL: f20
 ; CHECK: call float @llvm.experimental.constrained.fptrunc
-define float @f20() {
+define float @f20() #0 {
 entry:
   %result = call float @llvm.experimental.constrained.fptrunc.f32.f64(
                                                double 42.1,
                                                metadata !"round.dynamic",
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
   ret float %result
 }
 
@@ -259,13 +282,121 @@ entry:
 ; unknown.
 ; CHECK-LABEL: f21
 ; CHECK: call double @llvm.experimental.constrained.fpext
-define double @f21() {
+define double @f21() #0 {
 entry:
   %result = call double @llvm.experimental.constrained.fpext.f64.f32(float 42.0,
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
+  ret double %result
+}
+
+; Verify that lrint(42.1) isn't simplified when the rounding mode is unknown.
+; CHECK-LABEL: f22
+; CHECK: call i32 @llvm.experimental.constrained.lrint
+define i32 @f22() #0 {
+entry:
+  %result = call i32 @llvm.experimental.constrained.lrint.i32.f64(double 42.1,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret i32 %result
+}
+
+; Verify that lrintf(42.0) isn't simplified when the rounding mode is unknown.
+; CHECK-LABEL: f23
+; CHECK: call i32 @llvm.experimental.constrained.lrint
+define i32 @f23() #0 {
+entry:
+  %result = call i32 @llvm.experimental.constrained.lrint.i32.f32(float 42.0,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret i32 %result
+}
+
+; Verify that llrint(42.1) isn't simplified when the rounding mode is unknown.
+; CHECK-LABEL: f24
+; CHECK: call i64 @llvm.experimental.constrained.llrint
+define i64 @f24() #0 {
+entry:
+  %result = call i64 @llvm.experimental.constrained.llrint.i64.f64(double 42.1,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret i64 %result
+}
+
+; Verify that llrint(42.0) isn't simplified when the rounding mode is unknown.
+; CHECK-LABEL: f25
+; CHECK: call i64 @llvm.experimental.constrained.llrint
+define i64 @f25() #0 {
+entry:
+  %result = call i64 @llvm.experimental.constrained.llrint.i64.f32(float 42.0,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret i64 %result
+}
+
+; Verify that lround(42.1) isn't simplified when the rounding mode is unknown.
+; CHECK-LABEL: f26
+; CHECK: call i32 @llvm.experimental.constrained.lround
+define i32 @f26() #0 {
+entry:
+  %result = call i32 @llvm.experimental.constrained.lround.i32.f64(double 42.1,
+                                               metadata !"fpexcept.strict") #0
+  ret i32 %result
+}
+
+; Verify that lround(42.0) isn't simplified when the rounding mode is unknown.
+; CHECK-LABEL: f27
+; CHECK: call i32 @llvm.experimental.constrained.lround
+define i32 @f27() #0 {
+entry:
+  %result = call i32 @llvm.experimental.constrained.lround.i32.f32(float 42.0,
+                                               metadata !"fpexcept.strict") #0
+  ret i32 %result
+}
+
+; Verify that llround(42.1) isn't simplified when the rounding mode is unknown.
+; CHECK-LABEL: f28
+; CHECK: call i64 @llvm.experimental.constrained.llround
+define i64 @f28() #0 {
+entry:
+  %result = call i64 @llvm.experimental.constrained.llround.i64.f64(double 42.1,
+                                               metadata !"fpexcept.strict") #0
+  ret i64 %result
+}
+
+; Verify that llround(42.0) isn't simplified when the rounding mode is unknown.
+; CHECK-LABEL: f29
+; CHECK: call i64 @llvm.experimental.constrained.llround
+define i64 @f29() #0 {
+entry:
+  %result = call i64 @llvm.experimental.constrained.llround.i64.f32(float 42.0,
+                                               metadata !"fpexcept.strict") #0
+  ret i64 %result
+}
+
+; Verify that sitofp(42) isn't simplified when the rounding mode is unknown.
+; CHECK-LABEL: @f30
+; CHECK: call double @llvm.experimental.constrained.sitofp
+define double @f30() #0 {
+entry:
+  %result = call double @llvm.experimental.constrained.sitofp.f64.i32(i32 42,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
   ret double %result
 }
 
+; Verify that uitofp(42) isn't simplified when the rounding mode is unknown.
+; CHECK-LABEL: @f31
+; CHECK: call double @llvm.experimental.constrained.uitofp
+define double @f31() #0 {
+entry:
+  %result = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 42,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret double %result
+}
+
+attributes #0 = { strictfp }
+
 @llvm.fp.env = thread_local global i8 zeroinitializer, section "llvm.metadata"
 declare double @llvm.experimental.constrained.fdiv.f64(double, double, metadata, metadata)
 declare double @llvm.experimental.constrained.fmul.f64(double, double, metadata, metadata)
@@ -284,5 +415,17 @@ declare double @llvm.experimental.constrained.log2.f64(double, metadata, metadat
 declare double @llvm.experimental.constrained.rint.f64(double, metadata, metadata)
 declare double @llvm.experimental.constrained.nearbyint.f64(double, metadata, metadata)
 declare double @llvm.experimental.constrained.fma.f64(double, double, double, metadata, metadata)
+declare i32 @llvm.experimental.constrained.fptosi.i32.f64(double, metadata)
+declare i32 @llvm.experimental.constrained.fptoui.i32.f64(double, metadata)
 declare float @llvm.experimental.constrained.fptrunc.f32.f64(double, metadata, metadata)
 declare double @llvm.experimental.constrained.fpext.f64.f32(float, metadata)
+declare i32 @llvm.experimental.constrained.lrint.i32.f64(double, metadata, metadata)
+declare i32 @llvm.experimental.constrained.lrint.i32.f32(float, metadata, metadata)
+declare i64 @llvm.experimental.constrained.llrint.i64.f64(double, metadata, metadata)
+declare i64 @llvm.experimental.constrained.llrint.i64.f32(float, metadata, metadata)
+declare i32 @llvm.experimental.constrained.lround.i32.f64(double, metadata)
+declare i32 @llvm.experimental.constrained.lround.i32.f32(float, metadata)
+declare i64 @llvm.experimental.constrained.llround.i64.f64(double, metadata)
+declare i64 @llvm.experimental.constrained.llround.i64.f32(float, metadata)
+declare double @llvm.experimental.constrained.sitofp.f64.i32(i32, metadata, metadata)
+declare double @llvm.experimental.constrained.uitofp.f64.i32(i32, metadata, metadata)
diff --git a/llvm/test/Feature/intrinsics.ll b/llvm/test/Feature/intrinsics.ll
index 71bb73cde..0301d1a64 100644
--- a/llvm/test/Feature/intrinsics.ll
+++ b/llvm/test/Feature/intrinsics.ll
@@ -69,5 +69,5 @@ define void @trap() {
   ret void
 }
 
-; CHECK: attributes #0 = { nounwind readnone speculatable }
+; CHECK: attributes #0 = { nounwind readnone speculatable willreturn }
 ; CHECK: attributes #1 = { cold noreturn nounwind }
diff --git a/llvm/test/Feature/load_extension.ll b/llvm/test/Feature/load_extension.ll
new file mode 100644
index 000000000..1838aa4ab
--- /dev/null
+++ b/llvm/test/Feature/load_extension.ll
@@ -0,0 +1,13 @@
+; This is currently failing on multiple platforms. Disable while investigation occurs.
+; UNSUPPORTED: darwin
+
+; RUN: opt %s %loadbye -goodbye -wave-goodbye -disable-output 2>&1 | FileCheck %s
+; REQUIRES: plugins, examples
+; CHECK: Bye
+
+@junk = global i32 0
+
+define i32* @somefunk() {
+  ret i32* @junk
+}
+
diff --git a/llvm/test/Feature/optnone-opt.ll b/llvm/test/Feature/optnone-opt.ll
index ae0e1a48a..f706ade79 100644
--- a/llvm/test/Feature/optnone-opt.ll
+++ b/llvm/test/Feature/optnone-opt.ll
@@ -39,16 +39,10 @@ attributes #0 = { optnone noinline }
 ; IR passes run at -O1 and higher.
 ; OPT-O1-DAG: Skipping pass 'Aggressive Dead Code Elimination'
 ; OPT-O1-DAG: Skipping pass 'Combine redundant instructions'
-; OPT-O1-DAG: Skipping pass 'Dead Store Elimination'
 ; OPT-O1-DAG: Skipping pass 'Early CSE'
-; OPT-O1-DAG: Skipping pass 'Jump Threading'
-; OPT-O1-DAG: Skipping pass 'MemCpy Optimization'
 ; OPT-O1-DAG: Skipping pass 'Reassociate expressions'
 ; OPT-O1-DAG: Skipping pass 'Simplify the CFG'
 ; OPT-O1-DAG: Skipping pass 'Sparse Conditional Constant Propagation'
-; OPT-O1-DAG: Skipping pass 'SROA'
-; OPT-O1-DAG: Skipping pass 'Tail Call Elimination'
-; OPT-O1-DAG: Skipping pass 'Value Propagation'
 
 ; Additional IR passes run at -O2 and higher.
 ; OPT-O2O3-DAG: Skipping pass 'Global Value Numbering'
diff --git a/llvm/test/Feature/strip_names.ll b/llvm/test/Feature/strip_names.ll
index 18c982154..e05671799 100644
--- a/llvm/test/Feature/strip_names.ll
+++ b/llvm/test/Feature/strip_names.ll
@@ -10,7 +10,7 @@
 ; CHECK:  ret i32 %add
 
 ; NONAME: @GlobalValueName
-; NONAME: @foo(i32)
+; NONAME: @foo(i32 %0)
 ; NONAME-NOT: somelabel:
 ; NONAME:  %2 = load i32, i32* @GlobalValueName
 ; NONAME:  %3 = add i32 %0, %2
diff --git a/llvm/test/Feature/undefined.ll b/llvm/test/Feature/undefined.ll
index e63ce41f6..e01ed4c0f 100644
--- a/llvm/test/Feature/undefined.ll
+++ b/llvm/test/Feature/undefined.ll
@@ -1,6 +1,7 @@
 ; RUN: llvm-as < %s | llvm-dis > %t1.ll
 ; RUN: llvm-as %t1.ll -o - | llvm-dis > %t2.ll
 ; RUN: diff %t1.ll %t2.ll
+; RUN: FileCheck %s < %t1.ll
 
 @X = global i32 undef           ; <i32*> [#uses=0]
 
@@ -15,3 +16,15 @@ define i32 @test2() {
         ret i32 %X
 }
 
+
+; Check that there is a newline between functions.
+
+; CHECK: @X = global
+; CHECK-EMPTY:
+; CHECK: declare i32 @atoi(i8*)
+; CHECK-EMPTY:
+; CHECK: define i32 @test() {
+; CHECK: {{^[}]}}
+; CHECK-EMPTY:
+; CHECK: define i32 @test2() {
+; CHECK: {{^[}]}}
diff --git a/llvm/test/Instrumentation/AddressSanitizer/debug-info-alloca.ll b/llvm/test/Instrumentation/AddressSanitizer/debug-info-alloca.ll
new file mode 100644
index 000000000..ba148e8d6
--- /dev/null
+++ b/llvm/test/Instrumentation/AddressSanitizer/debug-info-alloca.ll
@@ -0,0 +1,75 @@
+; Checks that asan prologue does not add debug locations, which would
+; fool findPrologueEndLoc because it sets the end of the prologue to the
+; first instruction.  Breaking on the instrumented function in a debugger
+; would then stop at that instruction, before the prologue is finished.
+
+; RUN: opt < %s -asan -asan-module -S | FileCheck %s
+; 1: void f(int *arg) {
+; 2: }
+; 3: int main(int argc, char **argv) {
+; 4:   f(&argc);
+; 5: }
+; clang 1.cc -g -S -emit-llvm -o - | sed 's/#0 = {/#0 = { sanitize_address/'
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define dso_local i32 @main(i32 %argc, i8** %argv) #0 !dbg !15 {
+entry:
+; No suffix like !dbg !123
+; CHECK: %asan_local_stack_base = alloca i64{{$}}
+; CHECK:     %3 = call i64 @__asan_stack_malloc_0(i64 64){{$}}
+  %argc.addr = alloca i32, align 4
+  %argv.addr = alloca i8**, align 8
+  store i32 %argc, i32* %argc.addr, align 4
+  call void @llvm.dbg.declare(metadata i32* %argc.addr, metadata !21, metadata !DIExpression()), !dbg !22
+  store i8** %argv, i8*** %argv.addr, align 8
+  call void @llvm.dbg.declare(metadata i8*** %argv.addr, metadata !23, metadata !DIExpression()), !dbg !24
+  call void @f(i32* %argc.addr), !dbg !25
+  ret i32 0, !dbg !26
+}
+
+define dso_local void @f(i32* %arg) #0 !dbg !7 {
+entry:
+  %arg.addr = alloca i32*, align 8
+  store i32* %arg, i32** %arg.addr, align 8
+  call void @llvm.dbg.declare(metadata i32** %arg.addr, metadata !12, metadata !DIExpression()), !dbg !13
+  ret void, !dbg !14
+}
+
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
+
+attributes #0 = { sanitize_address noinline nounwind optnone uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone speculatable willreturn }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 10.0.0 (git@github.com:llvm/llvm-project 1ac700cdef787383ad49a0e37d9894491ef19480)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "2.c", directory: "/home/builduser")
+!2 = !{}
+!3 = !{i32 7, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{!"clang version 10.0.0 (git@github.com:llvm/llvm-project 1ac700cdef787383ad49a0e37d9894491ef19480)"}
+!7 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 1, type: !8, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !2)
+!8 = !DISubroutineType(types: !9)
+!9 = !{null, !10}
+!10 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !11, size: 64)
+!11 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!12 = !DILocalVariable(name: "arg", arg: 1, scope: !7, file: !1, line: 1, type: !10)
+!13 = !DILocation(line: 1, column: 13, scope: !7)
+!14 = !DILocation(line: 2, column: 1, scope: !7)
+!15 = distinct !DISubprogram(name: "main", scope: !1, file: !1, line: 3, type: !16, scopeLine: 3, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !2)
+!16 = !DISubroutineType(types: !17)
+!17 = !{!11, !11, !18}
+!18 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !19, size: 64)
+!19 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !20, size: 64)
+!20 = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_signed_char)
+!21 = !DILocalVariable(name: "argc", arg: 1, scope: !15, file: !1, line: 3, type: !11)
+!22 = !DILocation(line: 3, column: 14, scope: !15)
+!23 = !DILocalVariable(name: "argv", arg: 2, scope: !15, file: !1, line: 3, type: !18)
+!24 = !DILocation(line: 3, column: 27, scope: !15)
+!25 = !DILocation(line: 4, column: 3, scope: !15)
+!26 = !DILocation(line: 5, column: 1, scope: !15)
diff --git a/llvm/test/Instrumentation/AddressSanitizer/global_addrspace.ll b/llvm/test/Instrumentation/AddressSanitizer/global_addrspace.ll
new file mode 100644
index 000000000..19b76e651
--- /dev/null
+++ b/llvm/test/Instrumentation/AddressSanitizer/global_addrspace.ll
@@ -0,0 +1,31 @@
+; Only verify that asan don't crash on global variables of different
+; address space. The global variable should be unmodified by asan.
+
+; RUN: opt < %s -asan -asan-module -S | FileCheck %s
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@a = internal addrspace(42) global [1 x i32] zeroinitializer, align 4
+
+; CHECK: @a = internal addrspace(42) global [1 x i32] zeroinitializer, align 4
+
+define void @b(i32 %c) {
+entry:
+  %conv = sext i32 %c to i64
+  %0 = inttoptr i64 %conv to i32 addrspace(42)*
+  %cmp = icmp ugt i32 addrspace(42)* %0, getelementptr inbounds ([1 x i32], [1 x i32] addrspace(42)* @a, i64 0, i64 0)
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  %call = tail call i32 (...) @e()
+  br label %if.end
+
+if.end:
+  ret void
+}
+
+declare i32 @e(...)
+
+!llvm.asan.globals = !{!0}
+!0 = !{[1 x i32] addrspace(42)* @a, null, !"a", i1 false, i1 false}
diff --git a/llvm/test/Instrumentation/AddressSanitizer/global_metadata.ll b/llvm/test/Instrumentation/AddressSanitizer/global_metadata.ll
index ecee77209..ea9f2cf3f 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/global_metadata.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/global_metadata.ll
@@ -55,7 +55,7 @@ entry:
 }
 
 attributes #0 = { nounwind sanitize_address }
-attributes #1 = { nounwind sanitize_address "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind sanitize_address "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.asan.globals = !{!0, !1, !2, !3, !4}
 !llvm.ident = !{!5}
diff --git a/llvm/test/Instrumentation/AddressSanitizer/global_metadata_array.ll b/llvm/test/Instrumentation/AddressSanitizer/global_metadata_array.ll
index fbe001614..b1a600f3c 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/global_metadata_array.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/global_metadata_array.ll
@@ -52,7 +52,7 @@ entry:
 }
 
 attributes #0 = { nounwind sanitize_address }
-attributes #1 = { nounwind sanitize_address "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind sanitize_address "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.asan.globals = !{!0, !1, !2, !3, !4}
 !llvm.ident = !{!5}
diff --git a/llvm/test/Instrumentation/AddressSanitizer/global_metadata_windows.ll b/llvm/test/Instrumentation/AddressSanitizer/global_metadata_windows.ll
index 5602cb26e..937f777f2 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/global_metadata_windows.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/global_metadata_windows.ll
@@ -37,8 +37,8 @@ entry:
 ; Function Attrs: nounwind
 declare i32 @puts(i8* nocapture readonly) local_unnamed_addr #1
 
-attributes #0 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.module.flags = !{!0}
 !llvm.ident = !{!1}
diff --git a/llvm/test/Instrumentation/AddressSanitizer/local_stack_base.ll b/llvm/test/Instrumentation/AddressSanitizer/local_stack_base.ll
index ad3a274c8..67e13e564 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/local_stack_base.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/local_stack_base.ll
@@ -18,8 +18,8 @@ entry:
   ; CHECK: %asan_local_stack_base = alloca i64
   ; CHECK: %[[ALLOCA:.*]] = ptrtoint i8* %MyAlloca to i64
   ; CHECK: %[[PHI:.*]] = phi i64 {{.*}} %[[ALLOCA]],
-  ; CHECK: store i64 %[[PHI]], i64* %asan_local_stack_base, !dbg
-  ; CHECK: call void @llvm.dbg.declare(metadata i64* %asan_local_stack_base, metadata !13, metadata !DIExpression(DW_OP_deref, DW_OP_plus_uconst, 32)), !dbg !14
+  ; CHECK: store i64 %[[PHI]], i64* %asan_local_stack_base
+  ; CHECK: call void @llvm.dbg.declare(metadata i64* %asan_local_stack_base, metadata !12, metadata !DIExpression(DW_OP_deref, DW_OP_plus_uconst, 32)), !dbg !13
   %0 = load i32, i32* %i.addr, align 4, !dbg !14
   %add = add nsw i32 %0, 2, !dbg !15
   ret i32 %add, !dbg !16
diff --git a/llvm/test/Instrumentation/AddressSanitizer/version-mismatch-check.ll b/llvm/test/Instrumentation/AddressSanitizer/version-mismatch-check.ll
new file mode 100644
index 000000000..e8f370e39
--- /dev/null
+++ b/llvm/test/Instrumentation/AddressSanitizer/version-mismatch-check.ll
@@ -0,0 +1,12 @@
+; Check that the ASan module constructor guards against compiler/runtime version
+; mismatch.
+
+; RUN: opt < %s -asan-module                                        -S | FileCheck %s
+; RUN: opt < %s -asan-module -asan-guard-against-version-mismatch=0 -S | FileCheck %s --check-prefix=NOGUARD
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-unknown-linux-gnu"
+
+; CHECK-LABEL: define internal void @asan.module_ctor()
+; CHECK:         call void @__asan_version_mismatch_check_
+; NOGUARD-NOT:   call void @__asan_version_mismatch_check_
diff --git a/llvm/test/Instrumentation/DataFlowSanitizer/abilist.ll b/llvm/test/Instrumentation/DataFlowSanitizer/abilist.ll
index e33237ffe..497297a54 100644
--- a/llvm/test/Instrumentation/DataFlowSanitizer/abilist.ll
+++ b/llvm/test/Instrumentation/DataFlowSanitizer/abilist.ll
@@ -13,7 +13,7 @@ define i32 @functional(i32 %a, i32 %b) {
   ret i32 %c
 }
 
-; CHECK: define i32 (i32, i32)* @discardg(i32)
+; CHECK: define i32 (i32, i32)* @discardg(i32 %0)
 ; CHECK: %[[CALL:.*]] = call { i32 (i32, i32)*, i16 } @"dfs$g"(i32 %0, i16 0)
 ; CHECK: %[[XVAL:.*]] = extractvalue { i32 (i32, i32)*, i16 } %[[CALL]], 0
 ; CHECK: ret {{.*}} %[[XVAL]]
@@ -21,7 +21,7 @@ define i32 @functional(i32 %a, i32 %b) {
 
 declare void @custom1(i32 %a, i32 %b)
 
-; CHECK: define linkonce_odr { i32, i16 } @"dfsw$custom2"(i32, i32, i16, i16)
+; CHECK: define linkonce_odr { i32, i16 } @"dfsw$custom2"(i32 %0, i32 %1, i16 %2, i16 %3)
 ; CHECK: %[[LABELRETURN2:.*]] = alloca i16
 ; CHECK: %[[RV:.*]] = call i32 @__dfsw_custom2
 ; CHECK: %[[RVSHADOW:.*]] = load i16, i16* %[[LABELRETURN2]]
@@ -30,7 +30,7 @@ declare void @custom1(i32 %a, i32 %b)
 ; CHECK: ret { i32, i16 }
 declare i32 @custom2(i32 %a, i32 %b)
 
-; CHECK: define linkonce_odr void @"dfsw$custom3"(i32, i16, i16*, ...)
+; CHECK: define linkonce_odr void @"dfsw$custom3"(i32 %0, i16 %1, i16* %2, ...)
 ; CHECK: call void @__dfsan_vararg_wrapper(i8*
 ; CHECK: unreachable
 declare void @custom3(i32 %a, ...)
@@ -78,7 +78,7 @@ define i32 (i32, i32)* @g(i32) {
   ret i32 (i32, i32)* @custom2
 }
 
-; CHECK: define { i32, i16 } @"dfs$adiscard"(i32, i32, i16, i16)
+; CHECK: define { i32, i16 } @"dfs$adiscard"(i32 %0, i32 %1, i16 %2, i16 %3)
 ; CHECK: %[[CALL:.*]] = call i32 @discard(i32 %0, i32 %1)
 ; CHECK: %[[IVAL0:.*]] = insertvalue { i32, i16 } undef, i32 %[[CALL]], 0
 ; CHECK: %[[IVAL1:.*]] = insertvalue { i32, i16 } %[[IVAL0]], i16 0, 1
@@ -88,7 +88,7 @@ define i32 (i32, i32)* @g(i32) {
 ; CHECK: declare void @__dfsw_custom1(i32, i32, i16, i16)
 ; CHECK: declare i32 @__dfsw_custom2(i32, i32, i16, i16, i16*)
 
-; CHECK-LABEL: define linkonce_odr i32 @"dfst0$customcb"(i32 (i32)*, i32, i16, i16*)
+; CHECK-LABEL: define linkonce_odr i32 @"dfst0$customcb"(i32 (i32)* %0, i32 %1, i16 %2, i16* %3)
 ; CHECK: %[[BC:.*]] = bitcast i32 (i32)* %0 to { i32, i16 } (i32, i16)*
 ; CHECK: %[[CALL:.*]] = call { i32, i16 } %[[BC]](i32 %1, i16 %2)
 ; CHECK: %[[XVAL0:.*]] = extractvalue { i32, i16 } %[[CALL]], 0
diff --git a/llvm/test/Instrumentation/DataFlowSanitizer/debug-nonzero-labels.ll b/llvm/test/Instrumentation/DataFlowSanitizer/debug-nonzero-labels.ll
index 6632eb379..cc481afa0 100644
--- a/llvm/test/Instrumentation/DataFlowSanitizer/debug-nonzero-labels.ll
+++ b/llvm/test/Instrumentation/DataFlowSanitizer/debug-nonzero-labels.ll
@@ -4,7 +4,7 @@ target triple = "x86_64-unknown-linux-gnu"
 
 declare i32 @g()
 
-; CHECK: define { i32, i16 } @"dfs$f"(i32, i32, i16, i16)
+; CHECK: define { i32, i16 } @"dfs$f"(i32 %0, i32 %1, i16 %2, i16 %3)
 define i32 @f(i32, i32) {
   ; CHECK: [[LOCALLABELALLOCA:%.*]] = alloca i16
   %i = alloca i32
diff --git a/llvm/test/Instrumentation/DataFlowSanitizer/debug.ll b/llvm/test/Instrumentation/DataFlowSanitizer/debug.ll
index 325ac94af..d18ea76d1 100644
--- a/llvm/test/Instrumentation/DataFlowSanitizer/debug.ll
+++ b/llvm/test/Instrumentation/DataFlowSanitizer/debug.ll
@@ -16,7 +16,7 @@ entry:
   ret i32 0, !dbg !12
 }
 
-attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!9, !10}
diff --git a/llvm/test/Instrumentation/DataFlowSanitizer/memset.ll b/llvm/test/Instrumentation/DataFlowSanitizer/memset.ll
index 765a4022b..afddead6d 100644
--- a/llvm/test/Instrumentation/DataFlowSanitizer/memset.ll
+++ b/llvm/test/Instrumentation/DataFlowSanitizer/memset.ll
@@ -5,7 +5,7 @@ target triple = "x86_64-unknown-linux-gnu"
 declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i1)
 
 define void @ms(i8* %p, i8 %v) {
-  ; CHECK-LABEL: @"dfs$ms"(i8*, i8, i16, i16)
+  ; CHECK-LABEL: @"dfs$ms"(i8* %0, i8 %1, i16 %2, i16 %3)
   ; CHECK: call void @__dfsan_set_label(i16 %3, i8* %0, i64 1)
   call void @llvm.memset.p0i8.i64(i8* %p, i8 %v, i64 1, i1 1)
   ret void
diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/alloca-compat.ll b/llvm/test/Instrumentation/HWAddressSanitizer/alloca-compat.ll
new file mode 100644
index 000000000..d2acbf6bd
--- /dev/null
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/alloca-compat.ll
@@ -0,0 +1,17 @@
+; Test that alloca instrumentation with old API levels does not use short granules.
+;
+; RUN: opt < %s -hwasan -S | FileCheck %s
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64--linux-android"
+
+declare void @use32(i32*)
+
+define void @test_alloca() sanitize_hwaddress {
+; CHECK-LABEL: @test_alloca(
+; CHECK: %[[X_TAG:[^ ]*]] = trunc i64 {{.*}} to i8
+; CHECK: call void @llvm.memset.p0i8.i64(i8* align 1 {{.*}}, i8 %[[X_TAG]], i64 1, i1 false)
+  %x = alloca i32, align 4
+  call void @use32(i32* nonnull %x)
+  ret void
+}
diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/alloca.ll b/llvm/test/Instrumentation/HWAddressSanitizer/alloca.ll
index ea2b566a7..15a1c6828 100644
--- a/llvm/test/Instrumentation/HWAddressSanitizer/alloca.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/alloca.ll
@@ -5,13 +5,13 @@
 ; RUN: opt < %s -hwasan -hwasan-with-ifunc=1 -hwasan-uar-retag-to-zero=0 -S | FileCheck %s --check-prefixes=CHECK,DYNAMIC-SHADOW,UAR-TAGS
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
-target triple = "aarch64--linux-android"
+target triple = "aarch64--linux-android10000"
 
 declare void @use32(i32*)
 
 define void @test_alloca() sanitize_hwaddress {
 ; CHECK-LABEL: @test_alloca(
-; CHECK: %[[FP:[^ ]*]] = call i8* @llvm.frameaddress(i32 0)
+; CHECK: %[[FP:[^ ]*]] = call i8* @llvm.frameaddress.p0i8(i32 0)
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i8* %[[FP]] to i64
 ; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 20
 ; CHECK: %[[BASE_TAG:[^ ]*]] = xor i64 %[[A]], %[[B]]
diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/basic-compat.ll b/llvm/test/Instrumentation/HWAddressSanitizer/basic-compat.ll
new file mode 100644
index 000000000..22626ed50
--- /dev/null
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/basic-compat.ll
@@ -0,0 +1,13 @@
+; Test that the old outlined check is used with old API levels.
+
+; RUN: opt < %s -hwasan -S | FileCheck %s
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64--linux-android"
+
+define i8 @test_load8(i8* %a) sanitize_hwaddress {
+; CHECK-LABEL: @test_load8(
+; CHECK: call void @llvm.hwasan.check.memaccess(i8* {{.*}}, i8* {{.*}}, i32 0)
+  %b = load i8, i8* %a, align 4
+  ret i8 %b
+}
diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/basic.ll b/llvm/test/Instrumentation/HWAddressSanitizer/basic.ll
index 79e414a5e..d5406117c 100644
--- a/llvm/test/Instrumentation/HWAddressSanitizer/basic.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/basic.ll
@@ -14,7 +14,7 @@
 ; CHECK: @llvm.global_ctors = appending global [1 x { i32, void ()*, i8* }] [{ i32, void ()*, i8* } { i32 0, void ()* @hwasan.module_ctor, i8* bitcast (void ()* @hwasan.module_ctor to i8*) }]
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
-target triple = "aarch64--linux-android"
+target triple = "aarch64--linux-android10000"
 
 define i8 @test_load8(i8* %a) sanitize_hwaddress {
 ; CHECK-LABEL: @test_load8(
@@ -56,8 +56,8 @@ define i8 @test_load8(i8* %a) sanitize_hwaddress {
 
 ; RECOVER: [[CONT]]:
 
-; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess(i8* %.hwasan.shadow, i8* %a, i32 0)
-; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess(i8* null, i8* %a, i32 0)
+; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* %.hwasan.shadow, i8* %a, i32 0)
+; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* null, i8* %a, i32 0)
 
 ; CHECK: %[[G:[^ ]*]] = load i8, i8* %a, align 4
 ; CHECK: ret i8 %[[G]]
@@ -108,8 +108,8 @@ define i16 @test_load16(i16* %a) sanitize_hwaddress {
 ; RECOVER: [[CONT]]:
 
 ; ABORT: %[[A:[^ ]*]] = bitcast i16* %a to i8*
-; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess(i8* %.hwasan.shadow, i8* %[[A]], i32 1)
-; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess(i8* null, i8* %[[A]], i32 1)
+; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* %.hwasan.shadow, i8* %[[A]], i32 1)
+; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* null, i8* %[[A]], i32 1)
 
 ; CHECK: %[[G:[^ ]*]] = load i16, i16* %a, align 4
 ; CHECK: ret i16 %[[G]]
@@ -136,8 +136,8 @@ define i32 @test_load32(i32* %a) sanitize_hwaddress {
 ; RECOVER: br label
 
 ; ABORT: %[[A:[^ ]*]] = bitcast i32* %a to i8*
-; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess(i8* %.hwasan.shadow, i8* %[[A]], i32 2)
-; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess(i8* null, i8* %[[A]], i32 2)
+; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* %.hwasan.shadow, i8* %[[A]], i32 2)
+; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* null, i8* %[[A]], i32 2)
 
 ; CHECK: %[[G:[^ ]*]] = load i32, i32* %a, align 4
 ; CHECK: ret i32 %[[G]]
@@ -164,8 +164,8 @@ define i64 @test_load64(i64* %a) sanitize_hwaddress {
 ; RECOVER: br label
 
 ; ABORT: %[[A:[^ ]*]] = bitcast i64* %a to i8*
-; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess(i8* %.hwasan.shadow, i8* %[[A]], i32 3)
-; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess(i8* null, i8* %[[A]], i32 3)
+; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* %.hwasan.shadow, i8* %[[A]], i32 3)
+; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* null, i8* %[[A]], i32 3)
 
 ; CHECK: %[[G:[^ ]*]] = load i64, i64* %a, align 8
 ; CHECK: ret i64 %[[G]]
@@ -192,8 +192,8 @@ define i128 @test_load128(i128* %a) sanitize_hwaddress {
 ; RECOVER: br label
 
 ; ABORT: %[[A:[^ ]*]] = bitcast i128* %a to i8*
-; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess(i8* %.hwasan.shadow, i8* %[[A]], i32 4)
-; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess(i8* null, i8* %[[A]], i32 4)
+; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* %.hwasan.shadow, i8* %[[A]], i32 4)
+; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* null, i8* %[[A]], i32 4)
 
 ; CHECK: %[[G:[^ ]*]] = load i128, i128* %a, align 16
 ; CHECK: ret i128 %[[G]]
@@ -232,8 +232,8 @@ define void @test_store8(i8* %a, i8 %b) sanitize_hwaddress {
 ; RECOVER: call void asm sideeffect "brk #2352", "{x0}"(i64 %[[A]])
 ; RECOVER: br label
 
-; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess(i8* %.hwasan.shadow, i8* %a, i32 16)
-; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess(i8* null, i8* %a, i32 16)
+; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* %.hwasan.shadow, i8* %a, i32 16)
+; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* null, i8* %a, i32 16)
 
 ; CHECK: store i8 %b, i8* %a, align 4
 ; CHECK: ret void
@@ -260,8 +260,8 @@ define void @test_store16(i16* %a, i16 %b) sanitize_hwaddress {
 ; RECOVER: br label
 
 ; ABORT: %[[A:[^ ]*]] = bitcast i16* %a to i8*
-; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess(i8* %.hwasan.shadow, i8* %[[A]], i32 17)
-; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess(i8* null, i8* %[[A]], i32 17)
+; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* %.hwasan.shadow, i8* %[[A]], i32 17)
+; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* null, i8* %[[A]], i32 17)
 
 ; CHECK: store i16 %b, i16* %a, align 4
 ; CHECK: ret void
@@ -288,8 +288,8 @@ define void @test_store32(i32* %a, i32 %b) sanitize_hwaddress {
 ; RECOVER: br label
 
 ; ABORT: %[[A:[^ ]*]] = bitcast i32* %a to i8*
-; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess(i8* %.hwasan.shadow, i8* %[[A]], i32 18)
-; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess(i8* null, i8* %[[A]], i32 18)
+; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* %.hwasan.shadow, i8* %[[A]], i32 18)
+; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* null, i8* %[[A]], i32 18)
 
 ; CHECK: store i32 %b, i32* %a, align 4
 ; CHECK: ret void
@@ -316,8 +316,8 @@ define void @test_store64(i64* %a, i64 %b) sanitize_hwaddress {
 ; RECOVER: br label
 
 ; ABORT: %[[A:[^ ]*]] = bitcast i64* %a to i8*
-; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess(i8* %.hwasan.shadow, i8* %[[A]], i32 19)
-; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess(i8* null, i8* %[[A]], i32 19)
+; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* %.hwasan.shadow, i8* %[[A]], i32 19)
+; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* null, i8* %[[A]], i32 19)
 
 ; CHECK: store i64 %b, i64* %a, align 8
 ; CHECK: ret void
@@ -344,8 +344,8 @@ define void @test_store128(i128* %a, i128 %b) sanitize_hwaddress {
 ; RECOVER: br label
 
 ; ABORT: %[[A:[^ ]*]] = bitcast i128* %a to i8*
-; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess(i8* %.hwasan.shadow, i8* %[[A]], i32 20)
-; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess(i8* null, i8* %[[A]], i32 20)
+; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* %.hwasan.shadow, i8* %[[A]], i32 20)
+; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* null, i8* %[[A]], i32 20)
 
 ; CHECK: store i128 %b, i128* %a, align 16
 ; CHECK: ret void
diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/dbg-value-tag-offset.ll b/llvm/test/Instrumentation/HWAddressSanitizer/dbg-value-tag-offset.ll
new file mode 100644
index 000000000..49e970b3c
--- /dev/null
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/dbg-value-tag-offset.ll
@@ -0,0 +1,61 @@
+; RUN: opt -hwasan -S -o - %s | FileCheck %s
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64-unknown-linux-android24"
+
+define dso_local void @f() sanitize_hwaddress !dbg !14 {
+  %1 = alloca i32, align 4
+  %2 = alloca i32, align 4
+  %3 = bitcast i32* %1 to i8*, !dbg !21
+  %4 = bitcast i32* %2 to i8*, !dbg !21
+; CHECK: call void @llvm.dbg.value(metadata i32 1, {{.*}}, metadata !DIExpression())
+  call void @llvm.dbg.value(metadata i32 1, metadata !20, metadata !DIExpression()), !dbg !22
+  store i32 1, i32* %2, align 4, !dbg !23, !tbaa !24
+; CHECK: call void @llvm.dbg.value(metadata i32* {{.*}}, metadata !DIExpression(DW_OP_LLVM_tag_offset, 0, DW_OP_deref))
+  call void @llvm.dbg.value(metadata i32* %1, metadata !18, metadata !DIExpression(DW_OP_deref)), !dbg !22
+  call void @use(i8* nonnull %3), !dbg !28
+; CHECK: call void @llvm.dbg.value(metadata i32* {{.*}}, metadata !DIExpression(DW_OP_LLVM_tag_offset, 128, DW_OP_deref))
+  call void @llvm.dbg.value(metadata i32* %2, metadata !20, metadata !DIExpression(DW_OP_deref)), !dbg !22
+  call void @use(i8* nonnull %4), !dbg !29
+  ret void, !dbg !30
+}
+
+declare !dbg !5 void @use(i8*)
+
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!8, !9, !10, !11, !12}
+!llvm.ident = !{!13}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 10.0.0 (git@github.com:llvm/llvm-project.git 5560dd08b99a0e8b0c55116376624e4f967caec5)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !3, nameTableKind: None)
+!1 = !DIFile(filename: "dbg.cc", directory: "/tmp")
+!2 = !{}
+!3 = !{!4, !5}
+!4 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: null, size: 64)
+!5 = !DISubprogram(name: "use", scope: !1, file: !1, line: 2, type: !6, flags: DIFlagPrototyped, spFlags: DISPFlagOptimized, retainedNodes: !2)
+!6 = !DISubroutineType(types: !7)
+!7 = !{null, !4}
+!8 = !{i32 7, !"Dwarf Version", i32 4}
+!9 = !{i32 2, !"Debug Info Version", i32 3}
+!10 = !{i32 1, !"wchar_size", i32 4}
+!11 = !{i32 7, !"PIC Level", i32 2}
+!12 = !{i32 7, !"PIE Level", i32 2}
+!13 = !{!"clang version 10.0.0 (git@github.com:llvm/llvm-project.git 5560dd08b99a0e8b0c55116376624e4f967caec5)"}
+!14 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 4, type: !15, scopeLine: 4, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !17)
+!15 = !DISubroutineType(types: !16)
+!16 = !{null}
+!17 = !{!18, !20}
+!18 = !DILocalVariable(name: "x", scope: !14, file: !1, line: 5, type: !19)
+!19 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!20 = !DILocalVariable(name: "y", scope: !14, file: !1, line: 5, type: !19)
+!21 = !DILocation(line: 5, column: 3, scope: !14)
+!22 = !DILocation(line: 0, scope: !14)
+!23 = !DILocation(line: 5, column: 10, scope: !14)
+!24 = !{!25, !25, i64 0}
+!25 = !{!"int", !26, i64 0}
+!26 = !{!"omnipotent char", !27, i64 0}
+!27 = !{!"Simple C++ TBAA"}
+!28 = !DILocation(line: 6, column: 3, scope: !14)
+!29 = !DILocation(line: 7, column: 3, scope: !14)
+!30 = !DILocation(line: 8, column: 1, scope: !14)
diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/globals.ll b/llvm/test/Instrumentation/HWAddressSanitizer/globals.ll
new file mode 100644
index 000000000..d5516bfae
--- /dev/null
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/globals.ll
@@ -0,0 +1,37 @@
+; RUN: opt < %s -S -hwasan -mtriple=aarch64--linux-android29 | FileCheck --check-prefix=CHECK29 %s
+; RUN: opt < %s -S -hwasan -mtriple=aarch64--linux-android30 | FileCheck --check-prefix=CHECK30 %s
+
+; CHECK29-NOT: @hwasan.note
+; CHECK29: @four = global
+
+; CHECK30: @__start_hwasan_globals = external hidden constant [0 x i8]
+; CHECK30: @__stop_hwasan_globals = external hidden constant [0 x i8]
+
+; CHECK30: @hwasan.note = private constant { i32, i32, i32, [8 x i8], i32, i32 } { i32 8, i32 8, i32 3, [8 x i8] c"LLVM\00\00\00\00", i32 trunc (i64 sub (i64 ptrtoint ([0 x i8]* @__start_hwasan_globals to i64), i64 ptrtoint ({ i32, i32, i32, [8 x i8], i32, i32 }* @hwasan.note to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint ([0 x i8]* @__stop_hwasan_globals to i64), i64 ptrtoint ({ i32, i32, i32, [8 x i8], i32, i32 }* @hwasan.note to i64)) to i32) }, section ".note.hwasan.globals", comdat, align 4
+
+; CHECK30: @hwasan.dummy.global = private constant [0 x i8] zeroinitializer, section "hwasan_globals", comdat($hwasan.note), !associated [[NOTE:![0-9]+]]
+
+; CHECK30: @four.hwasan = private global { i32, [12 x i8] } { i32 1, [12 x i8] c"\00\00\00\00\00\00\00\00\00\00\00\AC" }, align 16
+; CHECK30: @four.hwasan.descriptor = private constant { i32, i32 } { i32 trunc (i64 sub (i64 ptrtoint ({ i32, [12 x i8] }* @four.hwasan to i64), i64 ptrtoint ({ i32, i32 }* @four.hwasan.descriptor to i64)) to i32), i32 -1409286140 }, section "hwasan_globals", !associated [[FOUR:![0-9]+]]
+
+; CHECK30: @sixteen.hwasan = private global [16 x i8] zeroinitializer, align 16
+; CHECK30: @sixteen.hwasan.descriptor = private constant { i32, i32 } { i32 trunc (i64 sub (i64 ptrtoint ([16 x i8]* @sixteen.hwasan to i64), i64 ptrtoint ({ i32, i32 }* @sixteen.hwasan.descriptor to i64)) to i32), i32 -1392508912 }, section "hwasan_globals", !associated [[SIXTEEN:![0-9]+]]
+
+; CHECK30: @huge.hwasan = private global [16777232 x i8] zeroinitializer, align 16
+; CHECK30: @huge.hwasan.descriptor = private constant { i32, i32 } { i32 trunc (i64 sub (i64 ptrtoint ([16777232 x i8]* @huge.hwasan to i64), i64 ptrtoint ({ i32, i32 }* @huge.hwasan.descriptor to i64)) to i32), i32 -1358954512 }, section "hwasan_globals", !associated [[HUGE:![0-9]+]]
+; CHECK30: @huge.hwasan.descriptor.1 = private constant { i32, i32 } { i32 trunc (i64 add (i64 sub (i64 ptrtoint ([16777232 x i8]* @huge.hwasan to i64), i64 ptrtoint ({ i32, i32 }* @huge.hwasan.descriptor.1 to i64)), i64 16777200) to i32), i32 -1375731680 }, section "hwasan_globals", !associated [[HUGE]]
+
+; CHECK30: @four = alias i32, inttoptr (i64 add (i64 ptrtoint ({ i32, [12 x i8] }* @four.hwasan to i64), i64 -6052837899185946624) to i32*)
+; CHECK30: @sixteen = alias [16 x i8], inttoptr (i64 add (i64 ptrtoint ([16 x i8]* @sixteen.hwasan to i64), i64 -5980780305148018688) to [16 x i8]*)
+; CHECK30: @huge = alias [16777232 x i8], inttoptr (i64 add (i64 ptrtoint ([16777232 x i8]* @huge.hwasan to i64), i64 -5908722711110090752) to [16777232 x i8]*)
+
+; CHECK30: [[NOTE]] = !{{{{}} i32, i32, i32, [8 x i8], i32, i32 }* @hwasan.note}
+; CHECK30: [[FOUR]] = !{{{{}} i32, [12 x i8] }* @four.hwasan}
+; CHECK30: [[SIXTEEN]] = !{[16 x i8]* @sixteen.hwasan}
+; CHECK30: [[HUGE]] = !{[16777232 x i8]* @huge.hwasan}
+
+source_filename = "foo"
+
+@four = global i32 1
+@sixteen = global [16 x i8] zeroinitializer
+@huge = global [16777232 x i8] zeroinitializer
diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/kernel-alloca.ll b/llvm/test/Instrumentation/HWAddressSanitizer/kernel-alloca.ll
index ddf81c482..08b3b7fed 100644
--- a/llvm/test/Instrumentation/HWAddressSanitizer/kernel-alloca.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/kernel-alloca.ll
@@ -9,7 +9,7 @@ declare void @use32(i32*)
 
 define void @test_alloca() sanitize_hwaddress {
 ; CHECK-LABEL: @test_alloca(
-; CHECK: %[[FP:[^ ]*]] = call i8* @llvm.frameaddress(i32 0)
+; CHECK: %[[FP:[^ ]*]] = call i8* @llvm.frameaddress.p0i8(i32 0)
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i8* %[[FP]] to i64
 ; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 20
 ; CHECK: %[[BASE_TAG:[^ ]*]] = xor i64 %[[A]], %[[B]]
diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/landingpad.ll b/llvm/test/Instrumentation/HWAddressSanitizer/landingpad.ll
index 15cc0bc95..bb6eb02b4 100644
--- a/llvm/test/Instrumentation/HWAddressSanitizer/landingpad.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/landingpad.ll
@@ -1,5 +1,6 @@
-; RUN: opt < %s -mtriple aarch64-linux-android -hwasan -S | FileCheck %s --check-prefixes=COMMON,ARM
-; RUN: opt < %s -mtriple x86_64-linux          -hwasan -S | FileCheck %s --check-prefixes=COMMON,X86
+; RUN: opt < %s -mtriple aarch64-linux-android29 -hwasan -S | FileCheck %s --check-prefixes=COMMON,LP,ARM
+; RUN: opt < %s -mtriple x86_64-linux -hwasan-instrument-landing-pads -hwasan -S | FileCheck %s --check-prefixes=COMMON,LP,X86
+; RUN: opt < %s -mtriple aarch64-linux-android30 -hwasan -S | FileCheck %s --check-prefixes=COMMON,NOLP
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64-unknown-linux-android"
@@ -15,8 +16,9 @@ lpad:
   %0 = landingpad { i8*, i32 }
           catch i8* null
 
-  ; COMMON-NEXT: %[[X:[^ ]*]] = call i64 @llvm.read_register.i64(metadata ![[META:[^ ]*]])
-  ; COMMON-NEXT: call void @__hwasan_handle_vfork(i64 %[[X]])
+  ; NOLP-NOT: call void @__hwasan_handle_vfork
+  ; LP-NEXT: %[[X:[^ ]*]] = call i64 @llvm.read_register.i64(metadata ![[META:[^ ]*]])
+  ; LP-NEXT: call void @__hwasan_handle_vfork(i64 %[[X]])
 
   %1 = extractvalue { i8*, i32 } %0, 0
   %2 = tail call i8* @__cxa_begin_catch(i8* %1)
diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/personality.ll b/llvm/test/Instrumentation/HWAddressSanitizer/personality.ll
new file mode 100644
index 000000000..ad969e393
--- /dev/null
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/personality.ll
@@ -0,0 +1,90 @@
+; RUN: opt < %s -mtriple aarch64-linux-android29 -hwasan -S | FileCheck %s --check-prefix=NOPERS
+; RUN: opt < %s -mtriple aarch64-linux-android30 -hwasan -S | FileCheck %s --check-prefix=PERS
+
+; NOPERS: define void @nostack() #{{[0-9]+}} {
+; PERS: define void @nostack() #{{[0-9]+}} {
+define void @nostack() sanitize_hwaddress {
+  ret void
+}
+
+; NOPERS: define void @stack1() #{{[0-9]+}} {
+; PERS: personality {{.*}} @__hwasan_personality_thunk
+define void @stack1() sanitize_hwaddress {
+  %p = alloca i8
+  call void @sink(i8* %p)
+  ret void
+}
+
+
+; NOPERS: personality void ()* @global
+; PERS: personality {{.*}} @__hwasan_personality_thunk.global
+define void @stack2() sanitize_hwaddress personality void ()* @global {
+  %p = alloca i8
+  call void @sink(i8* %p)
+  ret void
+}
+
+define internal void @local() {
+  ret void
+}
+
+@local_alias = internal alias void (), void ()* @local
+
+; NOPERS: personality void ()* @local
+; PERS: personality {{.*}} @__hwasan_personality_thunk.local
+define void @stack3() sanitize_hwaddress personality void ()* @local {
+  %p = alloca i8
+  call void @sink(i8* %p)
+  ret void
+}
+
+; NOPERS: personality void ()* @local_alias
+; PERS: personality {{.*}} @__hwasan_personality_thunk.local_alias
+define void @stack4() sanitize_hwaddress personality void ()* @local_alias {
+  %p = alloca i8
+  call void @sink(i8* %p)
+  ret void
+}
+
+; NOPERS: personality void ()* inttoptr (i64 1 to void ()*)
+; PERS: personality i32 (i32, i32, i64, i8*, i8*)* @__hwasan_personality_thunk.
+define void @stack5() sanitize_hwaddress personality void ()* inttoptr (i64 1 to void ()*) {
+  %p = alloca i8
+  call void @sink(i8* %p)
+  ret void
+}
+
+; NOPERS: personality void ()* inttoptr (i64 2 to void ()*)
+; PERS: personality i32 (i32, i32, i64, i8*, i8*)* @__hwasan_personality_thunk..1
+define void @stack6() sanitize_hwaddress personality void ()* inttoptr (i64 2 to void ()*) {
+  %p = alloca i8
+  call void @sink(i8* %p)
+  ret void
+}
+
+declare void @global()
+declare void @sink(i8*)
+
+; PERS: define linkonce_odr hidden i32 @__hwasan_personality_thunk(i32 %0, i32 %1, i64 %2, i8* %3, i8* %4) comdat
+; PERS: %5 = tail call i32 @__hwasan_personality_wrapper(i32 %0, i32 %1, i64 %2, i8* %3, i8* %4, i8* null, i8* bitcast (void ()* @_Unwind_GetGR to i8*), i8* bitcast (void ()* @_Unwind_GetCFA to i8*))
+; PERS: ret i32 %5
+
+; PERS: define linkonce_odr hidden i32 @__hwasan_personality_thunk.global(i32 %0, i32 %1, i64 %2, i8* %3, i8* %4) comdat
+; PERS: %5 = tail call i32 @__hwasan_personality_wrapper(i32 %0, i32 %1, i64 %2, i8* %3, i8* %4, i8* bitcast (void ()* @global to i8*), i8* bitcast (void ()* @_Unwind_GetGR to i8*), i8* bitcast (void ()* @_Unwind_GetCFA to i8*))
+; PERS: ret i32 %5
+
+; PERS: define internal i32 @__hwasan_personality_thunk.local(i32 %0, i32 %1, i64 %2, i8* %3, i8* %4)
+; PERS: %5 = tail call i32 @__hwasan_personality_wrapper(i32 %0, i32 %1, i64 %2, i8* %3, i8* %4, i8* bitcast (void ()* @local to i8*), i8* bitcast (void ()* @_Unwind_GetGR to i8*), i8* bitcast (void ()* @_Unwind_GetCFA to i8*))
+; PERS: ret i32 %5
+
+; PERS: define internal i32 @__hwasan_personality_thunk.local_alias(i32 %0, i32 %1, i64 %2, i8* %3, i8* %4)
+; PERS: %5 = tail call i32 @__hwasan_personality_wrapper(i32 %0, i32 %1, i64 %2, i8* %3, i8* %4, i8* bitcast (void ()* @local_alias to i8*), i8* bitcast (void ()* @_Unwind_GetGR to i8*), i8* bitcast (void ()* @_Unwind_GetCFA to i8*))
+; PERS: ret i32 %5
+
+; PERS: define internal i32 @__hwasan_personality_thunk.(i32 %0, i32 %1, i64 %2, i8* %3, i8* %4) {
+; PERS: %5 = tail call i32 @__hwasan_personality_wrapper(i32 %0, i32 %1, i64 %2, i8* %3, i8* %4, i8* inttoptr (i64 1 to i8*), i8* bitcast (void ()* @_Unwind_GetGR to i8*), i8* bitcast (void ()* @_Unwind_GetCFA to i8*))
+; PERS: ret i32 %5
+
+; PERS: define internal i32 @__hwasan_personality_thunk..1(i32 %0, i32 %1, i64 %2, i8* %3, i8* %4) {
+; PERS: %5 = tail call i32 @__hwasan_personality_wrapper(i32 %0, i32 %1, i64 %2, i8* %3, i8* %4, i8* inttoptr (i64 2 to i8*), i8* bitcast (void ()* @_Unwind_GetGR to i8*), i8* bitcast (void ()* @_Unwind_GetCFA to i8*))
+; PERS: ret i32 %5
diff --git a/llvm/test/Instrumentation/InstrProfiling/PR23499.ll b/llvm/test/Instrumentation/InstrProfiling/PR23499.ll
index af95f9a45..1a4c04947 100644
--- a/llvm/test/Instrumentation/InstrProfiling/PR23499.ll
+++ b/llvm/test/Instrumentation/InstrProfiling/PR23499.ll
@@ -14,14 +14,14 @@ $_Z3barIvEvv = comdat any
 @__profn__Z3barIvEvv = linkonce_odr hidden constant [11 x i8] c"_Z3barIvEvv", align 1
 
 ; CHECK-NOT: __profn__Z3barIvEvv
-; CHECK: @__profc__Z3barIvEvv = linkonce_odr hidden global [1 x i64] zeroinitializer, section "{{.*}}__llvm_prf_cnts", comdat($__profv__Z3barIvEvv), align 8
-; CHECK: @__profd__Z3barIvEvv = linkonce_odr hidden global { i64, i64, i64*, i8*, i8*, i32, [2 x i16] } { i64 4947693190065689389, i64 0, i64* getelementptr inbounds ([1 x i64], [1 x i64]* @__profc__Z3barIvEvv, i32 0, i32 0), i8*{{.*}}, i8* null, i32 1, [2 x i16] zeroinitializer }, section "{{.*}}__llvm_prf_data{{.*}}", comdat($__profv__Z3barIvEvv), align 8
+; CHECK: @__profc__Z3barIvEvv = linkonce_odr hidden global [1 x i64] zeroinitializer, section "{{.*}}__llvm_prf_cnts", comdat, align 8
+; CHECK: @__profd__Z3barIvEvv = linkonce_odr hidden global { i64, i64, i64*, i8*, i8*, i32, [2 x i16] } { i64 4947693190065689389, i64 0, i64* getelementptr inbounds ([1 x i64], [1 x i64]* @__profc__Z3barIvEvv, i32 0, i32 0), i8*{{.*}}, i8* null, i32 1, [2 x i16] zeroinitializer }, section "{{.*}}__llvm_prf_data{{.*}}", comdat, align 8
 ; CHECK: @__llvm_prf_nm = private constant [{{.*}} x i8] c"{{.*}}", section "{{.*}}__llvm_prf_names"
 
 
 ; COFF-NOT: __profn__Z3barIvEvv
-; COFF: @__profc__Z3barIvEvv = linkonce_odr dso_local global [1 x i64] zeroinitializer, section "{{.*}}prfc$M", comdat, align 8
-; COFF: @__profd__Z3barIvEvv = internal global { i64, i64, i64*, i8*, i8*, i32, [2 x i16] } { i64 4947693190065689389, i64 0, i64* getelementptr inbounds ([1 x i64], [1 x i64]* @__profc__Z3barIvEvv, i32 0, i32 0), i8*{{.*}}, i8* null, i32 1, [2 x i16] zeroinitializer }, section "{{.*}}prfd{{.*}}", comdat($__profc__Z3barIvEvv), align 8
+; COFF: @__profc__Z3barIvEvv = linkonce_odr hidden global [1 x i64] zeroinitializer, section "{{.*}}prfc$M", comdat, align 8
+; COFF: @__profd__Z3barIvEvv = linkonce_odr hidden global { i64, i64, i64*, i8*, i8*, i32, [2 x i16] } { i64 4947693190065689389, i64 0, i64* getelementptr inbounds ([1 x i64], [1 x i64]* @__profc__Z3barIvEvv, i32 0, i32 0), i8*{{.*}}, i8* null, i32 1, [2 x i16] zeroinitializer }, section "{{.*}}prfd{{.*}}", comdat, align 8
 
 
 declare void @llvm.instrprof.increment(i8*, i64, i32, i32) #1
diff --git a/llvm/test/Instrumentation/InstrProfiling/comdat.ll b/llvm/test/Instrumentation/InstrProfiling/comdat.ll
index 03ee905fa..3f169c4b7 100644
--- a/llvm/test/Instrumentation/InstrProfiling/comdat.ll
+++ b/llvm/test/Instrumentation/InstrProfiling/comdat.ll
@@ -15,10 +15,10 @@ $foo_inline = comdat any
 
 @__profn_foo_inline = linkonce_odr hidden constant [10 x i8] c"foo_inline"
 
-; ELF: @__profc_foo_inline = linkonce_odr hidden global{{.*}}, section "__llvm_prf_cnts", comdat($__profv_foo_inline), align 8
-; ELF: @__profd_foo_inline = linkonce_odr hidden global{{.*}}, section "__llvm_prf_data", comdat($__profv_foo_inline), align 8
-; COFF: @__profc_foo_inline = linkonce_odr dso_local global{{.*}}, section ".lprfc$M", comdat, align 8
-; COFF: @__profd_foo_inline = internal global{{.*}}, section ".lprfd$M", comdat($__profc_foo_inline), align 8
+; ELF: @__profc_foo_inline = linkonce_odr hidden global{{.*}}, section "__llvm_prf_cnts", comdat, align 8
+; ELF: @__profd_foo_inline = linkonce_odr hidden global{{.*}}, section "__llvm_prf_data", comdat, align 8
+; COFF: @__profc_foo_inline = linkonce_odr hidden global{{.*}}, section ".lprfc$M", comdat, align 8
+; COFF: @__profd_foo_inline = linkonce_odr hidden global{{.*}}, section ".lprfd$M", comdat, align 8
 define weak_odr void @foo_inline() comdat {
   call void @llvm.instrprof.increment(i8* getelementptr inbounds ([10 x i8], [10 x i8]* @__profn_foo_inline, i32 0, i32 0), i64 0, i32 1, i32 0)
   ret void
@@ -28,10 +28,10 @@ $foo_extern = comdat any
 
 @__profn_foo_extern = linkonce_odr hidden constant [10 x i8] c"foo_extern"
 
-; ELF: @__profc_foo_extern = linkonce_odr hidden global{{.*}}, section "__llvm_prf_cnts", comdat($__profv_foo_extern)
-; ELF: @__profd_foo_extern = linkonce_odr hidden global{{.*}}, section "__llvm_prf_data", comdat($__profv_foo_extern)
-; COFF: @__profc_foo_extern = linkonce_odr dso_local global{{.*}}, section ".lprfc$M", comdat, align 8
-; COFF: @__profd_foo_extern = internal global{{.*}}, section ".lprfd$M", comdat($__profc_foo_extern), align 8
+; ELF: @__profc_foo_extern = linkonce_odr hidden global{{.*}}, section "__llvm_prf_cnts", comdat, align 8
+; ELF: @__profd_foo_extern = linkonce_odr hidden global{{.*}}, section "__llvm_prf_data", comdat, align 8
+; COFF: @__profc_foo_extern = linkonce_odr hidden global{{.*}}, section ".lprfc$M", comdat, align 8
+; COFF: @__profd_foo_extern = linkonce_odr hidden global{{.*}}, section ".lprfd$M", comdat, align 8
 define available_externally void @foo_extern() {
   call void @llvm.instrprof.increment(i8* getelementptr inbounds ([10 x i8], [10 x i8]* @__profn_foo_extern, i32 0, i32 0), i64 0, i32 1, i32 0)
   ret void
diff --git a/llvm/test/Instrumentation/InstrProfiling/linkage.ll b/llvm/test/Instrumentation/InstrProfiling/linkage.ll
index 29ae5d6e9..8a4dee2f7 100644
--- a/llvm/test/Instrumentation/InstrProfiling/linkage.ll
+++ b/llvm/test/Instrumentation/InstrProfiling/linkage.ll
@@ -53,12 +53,12 @@ define linkonce_odr void @foo_inline() {
   ret void
 }
 
-; LINUX: @__profc_foo_extern = linkonce_odr hidden global {{.*}}section "__llvm_prf_cnts", comdat($__profv_foo_extern), align 8
-; LINUX: @__profd_foo_extern = linkonce_odr hidden global {{.*}}section "__llvm_prf_data", comdat($__profv_foo_extern), align 8
+; LINUX: @__profc_foo_extern = linkonce_odr hidden global {{.*}}section "__llvm_prf_cnts", comdat, align 8
+; LINUX: @__profd_foo_extern = linkonce_odr hidden global {{.*}}section "__llvm_prf_data", comdat, align 8
 ; MACHO: @__profc_foo_extern = linkonce_odr hidden global
 ; MACHO: @__profd_foo_extern = linkonce_odr hidden global
-; COFF: @__profc_foo_extern = linkonce_odr dso_local global {{.*}}section ".lprfc$M", comdat, align 8
-; COFF: @__profd_foo_extern = internal global {{.*}}section ".lprfd$M", comdat($__profc_foo_extern), align 8
+; COFF: @__profc_foo_extern = linkonce_odr hidden global {{.*}}section ".lprfc$M", comdat, align 8
+; COFF: @__profd_foo_extern = linkonce_odr hidden global {{.*}}section ".lprfd$M", comdat, align 8
 define available_externally void @foo_extern() {
   call void @llvm.instrprof.increment(i8* getelementptr inbounds ([10 x i8], [10 x i8]* @__profn_foo_extern, i32 0, i32 0), i64 0, i32 1, i32 0)
   ret void
diff --git a/llvm/test/Instrumentation/MemorySanitizer/attributes.ll b/llvm/test/Instrumentation/MemorySanitizer/attributes.ll
new file mode 100644
index 000000000..16f497109
--- /dev/null
+++ b/llvm/test/Instrumentation/MemorySanitizer/attributes.ll
@@ -0,0 +1,47 @@
+; RUN: opt < %s -S -passes='module(msan-module),function(msan)' 2>&1 | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+
+declare void @a_() sanitize_memory readnone
+declare void @b_() sanitize_memory readonly
+declare void @c_() sanitize_memory writeonly
+declare void @d_(i32* %p) sanitize_memory writeonly argmemonly
+declare void @e_() sanitize_memory speculatable
+
+define void @a() sanitize_memory readnone {
+entry:
+  call void @a_()
+  ret void
+}
+
+define void @b() sanitize_memory readonly {
+entry:
+  call void @b_()
+  ret void
+}
+
+define void @c() sanitize_memory writeonly {
+entry:
+  call void @c_()
+  ret void
+}
+
+define void @d(i32* %p) sanitize_memory writeonly argmemonly {
+entry:
+  call void @d_(i32* %p)
+  ret void
+}
+
+define void @e() sanitize_memory speculatable {
+entry:
+  call void @e_()
+  ret void
+}
+
+; CHECK-NOT: readnone
+; CHECK-NOT: readonly
+; CHECK-NOT: writeonly
+; CHECK-NOT: argmemonly
+; CHECK-NOT: speculatable
diff --git a/llvm/test/Instrumentation/MemorySanitizer/clmul.ll b/llvm/test/Instrumentation/MemorySanitizer/clmul.ll
new file mode 100644
index 000000000..838eb0603
--- /dev/null
+++ b/llvm/test/Instrumentation/MemorySanitizer/clmul.ll
@@ -0,0 +1,72 @@
+; RUN: opt < %s -msan-check-access-address=0 -S -passes=msan 2>&1 | FileCheck  \
+; RUN: %s
+; RUN: opt < %s -msan -msan-check-access-address=0 -S | FileCheck %s
+; RUN: opt < %s -msan -msan-check-access-address=0 -msan-track-origins=1 -S | FileCheck %s --check-prefixes=CHECK,ORIGIN
+; REQUIRES: x86-registered-target
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare <2 x i64> @llvm.x86.pclmulqdq(<2 x i64>, <2 x i64>, i8 immarg) nounwind readnone
+declare <4 x i64> @llvm.x86.pclmulqdq.256(<4 x i64>, <4 x i64>, i8 immarg) nounwind readnone
+declare <8 x i64> @llvm.x86.pclmulqdq.512(<8 x i64>, <8 x i64>, i8 immarg) nounwind readnone
+
+define <2 x i64> @clmul00(<2 x i64> %a, <2 x i64> %b) sanitize_memory {
+entry:
+  %0 = tail call <2 x i64> @llvm.x86.pclmulqdq(<2 x i64> %a, <2 x i64> %b, i8 0)
+  ret <2 x i64> %0
+}
+
+; CHECK-LABEL: @clmul00
+; CHECK: %[[S0:.*]] = load <2 x i64>, <2 x i64>* {{.*}}@__msan_param_tls
+; CHECK: %[[S1:.*]] = load <2 x i64>, <2 x i64>* {{.*}}@__msan_param_tls
+; CHECK: %[[SHUF1:.*]] = shufflevector <2 x i64> %[[S1]], <2 x i64> undef, <2 x i32> zeroinitializer
+; CHECK: %[[SHUF0:.*]] = shufflevector <2 x i64> %[[S0]], <2 x i64> undef, <2 x i32> zeroinitializer
+; CHECK: %[[SRET:.*]] = or <2 x i64> %[[SHUF1]], %[[SHUF0]]
+; CHECK: store <2 x i64> %[[SRET]], <2 x i64>* {{.*}}@__msan_retval_tls
+
+define <2 x i64> @clmul10(<2 x i64> %a, <2 x i64> %b) sanitize_memory {
+entry:
+  %0 = tail call <2 x i64> @llvm.x86.pclmulqdq(<2 x i64> %a, <2 x i64> %b, i8 16)
+  ret <2 x i64> %0
+}
+
+; CHECK-LABEL: @clmul10
+; CHECK: %[[S0:.*]] = load <2 x i64>, <2 x i64>* {{.*}}@__msan_param_tls
+; CHECK: %[[S1:.*]] = load <2 x i64>, <2 x i64>* {{.*}}@__msan_param_tls
+; CHECK: %[[SHUF1:.*]] = shufflevector <2 x i64> %[[S1]], <2 x i64> undef, <2 x i32> zeroinitializer
+; CHECK: %[[SHUF0:.*]] = shufflevector <2 x i64> %[[S0]], <2 x i64> undef, <2 x i32> <i32 1, i32 1>
+; CHECK: %[[SRET:.*]] = or <2 x i64> %[[SHUF1]], %[[SHUF0]]
+; CHECK: store <2 x i64> %[[SRET]], <2 x i64>* {{.*}}@__msan_retval_tls
+
+define <4 x i64> @clmul11_256(<4 x i64> %a, <4 x i64> %b) sanitize_memory {
+entry:
+  %0 = tail call <4 x i64> @llvm.x86.pclmulqdq.256(<4 x i64> %a, <4 x i64> %b, i8 17)
+  ret <4 x i64> %0
+}
+
+; CHECK-LABEL: @clmul11_256
+; CHECK: %[[S0:.*]] = load <4 x i64>, <4 x i64>* {{.*}}@__msan_param_tls
+; CHECK: %[[S1:.*]] = load <4 x i64>, <4 x i64>* {{.*}}@__msan_param_tls
+; CHECK: %[[SHUF1:.*]] = shufflevector <4 x i64> %[[S1]], <4 x i64> undef, <4 x i32> <i32 1, i32 1, i32 3, i32 3>
+; CHECK: %[[SHUF0:.*]] = shufflevector <4 x i64> %[[S0]], <4 x i64> undef, <4 x i32> <i32 1, i32 1, i32 3, i32 3>
+; CHECK: %[[SRET:.*]] = or <4 x i64> %[[SHUF1]], %[[SHUF0]]
+; CHECK: store <4 x i64> %[[SRET]], <4 x i64>* {{.*}}@__msan_retval_tls
+
+define <8 x i64> @clmul01_512(<8 x i64> %a, <8 x i64> %b) sanitize_memory {
+entry:
+  %0 = tail call <8 x i64> @llvm.x86.pclmulqdq.512(<8 x i64> %a, <8 x i64> %b, i8 16)
+  ret <8 x i64> %0
+}
+
+; CHECK-LABEL: @clmul01_512
+; CHECK: %[[S0:.*]] = load <8 x i64>, <8 x i64>* {{.*}}@__msan_param_tls
+; CHECK: %[[S1:.*]] = load <8 x i64>, <8 x i64>* {{.*}}@__msan_param_tls
+; CHECK: %[[SHUF1:.*]] = shufflevector <8 x i64> %[[S1]], <8 x i64> undef, <8 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6>
+; CHECK: %[[SHUF0:.*]] = shufflevector <8 x i64> %[[S0]], <8 x i64> undef, <8 x i32> <i32 1, i32 1, i32 3, i32 3, i32 5, i32 5, i32 7, i32 7>
+; CHECK: %[[SRET:.*]] = or <8 x i64> %[[SHUF1]], %[[SHUF0]]
+; ORIGIN: %[[FLAT:.*]] = bitcast <8 x i64> %[[SHUF0]] to i512
+; ORIGIN: %[[I:.*]] = icmp ne i512 %[[FLAT]], 0
+; ORIGIN: %[[O:.*]] = select i1 %[[I]],
+; CHECK: store <8 x i64> %[[SRET]], <8 x i64>* {{.*}}@__msan_retval_tls
+; ORIGIN: store i32 %[[O]], i32* @__msan_retval_origin_tls
diff --git a/llvm/test/Instrumentation/MemorySanitizer/msan_basic.ll b/llvm/test/Instrumentation/MemorySanitizer/msan_basic.ll
index 499de14ae..f56233d13 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/msan_basic.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/msan_basic.ll
@@ -1,10 +1,9 @@
-; RUN: opt < %s -msan-check-access-address=0 -S -passes=msan 2>&1 | FileCheck  \
-; RUN: -allow-deprecated-dag-overlap %s
-; RUN: opt < %s -msan -msan-check-access-address=0 -S | FileCheck -allow-deprecated-dag-overlap %s
-; RUN: opt < %s -msan-check-access-address=0 -msan-track-origins=1 -S          \
-; RUN: -passes=msan 2>&1 | FileCheck -allow-deprecated-dag-overlap             \
-; RUN: -check-prefix=CHECK -check-prefix=CHECK-ORIGINS %s
-; RUN: opt < %s -msan -msan-check-access-address=0 -msan-track-origins=1 -S | FileCheck -allow-deprecated-dag-overlap -check-prefix=CHECK -check-prefix=CHECK-ORIGINS %s
+; RUN: opt < %s -msan-check-access-address=0 -S -passes='module(msan-module),function(msan)' 2>&1 | FileCheck -allow-deprecated-dag-overlap %s
+; RUN: opt < %s --passes='module(msan-module),function(msan)' -msan-check-access-address=0 -S | FileCheck -allow-deprecated-dag-overlap %s
+; RUN: opt < %s -msan-check-access-address=0 -msan-track-origins=1 -S -passes='module(msan-module),function(msan)' 2>&1 | \
+; RUN:   FileCheck -allow-deprecated-dag-overlap -check-prefixes=CHECK,CHECK-ORIGINS %s
+; RUN: opt < %s -passes='module(msan-module),function(msan)' -msan-check-access-address=0 -msan-track-origins=1 -S | \
+; RUN:   FileCheck -allow-deprecated-dag-overlap -check-prefixes=CHECK,CHECK-ORIGINS %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/Instrumentation/MemorySanitizer/msan_llvm_launder_invariant.ll b/llvm/test/Instrumentation/MemorySanitizer/msan_llvm_launder_invariant.ll
new file mode 100644
index 000000000..63de8663e
--- /dev/null
+++ b/llvm/test/Instrumentation/MemorySanitizer/msan_llvm_launder_invariant.ll
@@ -0,0 +1,38 @@
+; Make sure MSan handles llvm.launder.invariant.group correctly.
+
+; RUN: opt < %s -msan -msan-kernel=1 -O1 -S | FileCheck -check-prefixes=CHECK %s
+; RUN: opt < %s -msan -O1 -S | FileCheck -check-prefixes=CHECK %s
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+%class.Foo = type { i32 (...)** }
+@flag = dso_local local_unnamed_addr global i8 0, align 1
+
+define dso_local %class.Foo* @_Z1fv() local_unnamed_addr #0 {
+entry:
+  %p = alloca i8*, align 8
+  %0 = bitcast i8** %p to i8*
+  call void @llvm.lifetime.start.p0i8(i64 8, i8* nonnull %0)
+  %1 = load i8, i8* @flag, align 1
+  %tobool = icmp ne i8 %1, 0
+  %call = call zeroext i1 @_Z2f1PPvb(i8** nonnull %p, i1 zeroext %tobool)
+  %2 = load i8*, i8** %p, align 8
+  %3 = call i8* @llvm.launder.invariant.group.p0i8(i8* %2)
+  %4 = bitcast i8* %3 to %class.Foo*
+  %retval.0 = select i1 %call, %class.Foo* %4, %class.Foo* null
+  call void @llvm.lifetime.end.p0i8(i64 8, i8* nonnull %0)
+  ret %class.Foo* %retval.0
+}
+
+; CHECK-NOT: call void @__msan_warning_noreturn
+
+declare dso_local zeroext i1 @_Z2f1PPvb(i8**, i1 zeroext) local_unnamed_addr
+
+declare i8* @llvm.launder.invariant.group.p0i8(i8*)
+
+declare void @llvm.lifetime.start.p0i8(i64 immarg, i8* nocapture)
+
+declare void @llvm.lifetime.end.p0i8(i64 immarg, i8* nocapture)
+
+attributes #0 = { sanitize_memory uwtable }
diff --git a/llvm/test/Instrumentation/MemorySanitizer/msan_llvm_strip_invariant.ll b/llvm/test/Instrumentation/MemorySanitizer/msan_llvm_strip_invariant.ll
new file mode 100644
index 000000000..f3b5c0d72
--- /dev/null
+++ b/llvm/test/Instrumentation/MemorySanitizer/msan_llvm_strip_invariant.ll
@@ -0,0 +1,21 @@
+; Make sure MSan handles llvm.launder.invariant.group correctly.
+
+; RUN: opt < %s -msan -msan-kernel=1 -O1 -S | FileCheck -check-prefixes=CHECK %s
+; RUN: opt < %s -msan -O1 -S | FileCheck -check-prefixes=CHECK %s
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@flag = dso_local local_unnamed_addr global i8 0, align 1
+
+define dso_local i8* @f(i8* %x) local_unnamed_addr #0 {
+entry:
+  %0 = call i8* @llvm.strip.invariant.group.p0i8(i8* %x)
+  ret i8* %0
+}
+
+; CHECK-NOT: call void @__msan_warning_noreturn
+
+declare i8* @llvm.strip.invariant.group.p0i8(i8*)
+
+attributes #0 = { sanitize_memory uwtable }
diff --git a/llvm/test/Instrumentation/MemorySanitizer/store-origin.ll b/llvm/test/Instrumentation/MemorySanitizer/store-origin.ll
index ed8d5362e..695d7c40a 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/store-origin.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/store-origin.ll
@@ -29,7 +29,7 @@ entry:
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.value(metadata, i64, metadata, metadata) #1
 
-attributes #0 = { nounwind sanitize_memory "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind sanitize_memory "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/abort-in-entry-block.ll b/llvm/test/Instrumentation/SanitizerCoverage/abort-in-entry-block.ll
index 9bc8acef4..1a44e9b76 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/abort-in-entry-block.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/abort-in-entry-block.ll
@@ -1,5 +1,6 @@
 ; Checks that a function with no-return in the entry block is not instrumented.
 ; RUN: opt < %s -sancov -sanitizer-coverage-level=3 -sanitizer-coverage-trace-pc-guard -S | FileCheck %s
+; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=3 -sanitizer-coverage-trace-pc-guard -S | FileCheck %s
 ; CHECK-NOT: call void @__sanitizer_cov_trace_pc_guard
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/backedge-pruning.ll b/llvm/test/Instrumentation/SanitizerCoverage/backedge-pruning.ll
index 103198311..c3a13b090 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/backedge-pruning.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/backedge-pruning.ll
@@ -1,6 +1,8 @@
 ; Test -sanitizer-coverage-trace-compares=1 and how it prunes backedge compares.
 ; RUN: opt < %s -sancov -sanitizer-coverage-level=1 -sanitizer-coverage-trace-compares=1  -sanitizer-coverage-prune-blocks=1 -S | FileCheck %s --check-prefix=PRUNE
 ; RUN: opt < %s -sancov -sanitizer-coverage-level=1 -sanitizer-coverage-trace-compares=1  -sanitizer-coverage-prune-blocks=0 -S | FileCheck %s --check-prefix=NOPRUNE
+; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=1 -sanitizer-coverage-trace-compares=1  -sanitizer-coverage-prune-blocks=1 -S | FileCheck %s --check-prefix=PRUNE
+; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=1 -sanitizer-coverage-trace-compares=1  -sanitizer-coverage-prune-blocks=0 -S | FileCheck %s --check-prefix=NOPRUNE
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/chains.ll b/llvm/test/Instrumentation/SanitizerCoverage/chains.ll
index 86b109165..ba83c85c8 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/chains.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/chains.ll
@@ -1,4 +1,5 @@
 ; RUN: opt < %s -sancov -sanitizer-coverage-level=4 -sanitizer-coverage-trace-pc -sanitizer-coverage-prune-blocks=1  -S | FileCheck %s
+; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=4 -sanitizer-coverage-trace-pc -sanitizer-coverage-prune-blocks=1  -S | FileCheck %s
 
 define i32 @blah(i32) #0 {
   %2 = icmp sgt i32 %0, 1
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/cmp-tracing-api-x86_32.ll b/llvm/test/Instrumentation/SanitizerCoverage/cmp-tracing-api-x86_32.ll
index 0f42756fd..a0406f833 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/cmp-tracing-api-x86_32.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/cmp-tracing-api-x86_32.ll
@@ -1,5 +1,6 @@
 ; Test -sanitizer-coverage-trace-compares=1 API declarations on a non-x86_64 arch
 ; RUN: opt < %s -sancov -sanitizer-coverage-level=1 -sanitizer-coverage-trace-compares=1  -S | FileCheck %s
+; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=1 -sanitizer-coverage-trace-compares=1  -S | FileCheck %s
 
 target triple = "i386-unknown-linux-gnu"
 define i32 @foo() #0 {
@@ -7,16 +8,16 @@ entry:
   ret i32 0
 }
 
-; CHECK: declare void @__sanitizer_cov_trace_pc_indir(i64)
-; CHECK: declare void @__sanitizer_cov_trace_cmp1(i8, i8)
-; CHECK: declare void @__sanitizer_cov_trace_cmp2(i16, i16)
-; CHECK: declare void @__sanitizer_cov_trace_cmp4(i32, i32)
-; CHECK: declare void @__sanitizer_cov_trace_cmp8(i64, i64)
-; CHECK: declare void @__sanitizer_cov_trace_div4(i32)
-; CHECK: declare void @__sanitizer_cov_trace_div8(i64)
-; CHECK: declare void @__sanitizer_cov_trace_gep(i64)
-; CHECK: declare void @__sanitizer_cov_trace_switch(i64, i64*)
-; CHECK: declare void @__sanitizer_cov_trace_pc()
-; CHECK: declare void @__sanitizer_cov_trace_pc_guard(i32*)
-; CHECK: declare void @__sanitizer_cov_trace_pc_guard_init(i32*, i32*)
+; CHECK-DAG: declare void @__sanitizer_cov_trace_pc_indir(i64)
+; CHECK-DAG: declare void @__sanitizer_cov_trace_cmp1(i8, i8)
+; CHECK-DAG: declare void @__sanitizer_cov_trace_cmp2(i16, i16)
+; CHECK-DAG: declare void @__sanitizer_cov_trace_cmp4(i32, i32)
+; CHECK-DAG: declare void @__sanitizer_cov_trace_cmp8(i64, i64)
+; CHECK-DAG: declare void @__sanitizer_cov_trace_div4(i32)
+; CHECK-DAG: declare void @__sanitizer_cov_trace_div8(i64)
+; CHECK-DAG: declare void @__sanitizer_cov_trace_gep(i64)
+; CHECK-DAG: declare void @__sanitizer_cov_trace_switch(i64, i64*)
+; CHECK-DAG: declare void @__sanitizer_cov_trace_pc()
+; CHECK-DAG: declare void @__sanitizer_cov_trace_pc_guard(i32*)
+; CHECK-DAG: declare void @__sanitizer_cov_trace_pc_guard_init(i32*, i32*)
 ; CHECK-NOT: declare
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/cmp-tracing-api-x86_64.ll b/llvm/test/Instrumentation/SanitizerCoverage/cmp-tracing-api-x86_64.ll
index 16689f983..e2645d19a 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/cmp-tracing-api-x86_64.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/cmp-tracing-api-x86_64.ll
@@ -1,5 +1,6 @@
 ; Test -sanitizer-coverage-trace-compares=1 API declarations on x86_64
 ; RUN: opt < %s -sancov -sanitizer-coverage-level=1 -sanitizer-coverage-trace-compares=1  -S | FileCheck %s
+; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=1 -sanitizer-coverage-trace-compares=1  -S | FileCheck %s
 
 target triple = "x86_64-unknown-linux-gnu"
 define i32 @foo() #0 {
@@ -7,16 +8,16 @@ entry:
   ret i32 0
 }
 
-; CHECK: declare void @__sanitizer_cov_trace_pc_indir(i64)
-; CHECK: declare void @__sanitizer_cov_trace_cmp1(i8 zeroext, i8 zeroext)
-; CHECK: declare void @__sanitizer_cov_trace_cmp2(i16 zeroext, i16 zeroext)
-; CHECK: declare void @__sanitizer_cov_trace_cmp4(i32 zeroext, i32 zeroext)
-; CHECK: declare void @__sanitizer_cov_trace_cmp8(i64, i64)
-; CHECK: declare void @__sanitizer_cov_trace_div4(i32 zeroext)
-; CHECK: declare void @__sanitizer_cov_trace_div8(i64)
-; CHECK: declare void @__sanitizer_cov_trace_gep(i64)
-; CHECK: declare void @__sanitizer_cov_trace_switch(i64, i64*)
-; CHECK: declare void @__sanitizer_cov_trace_pc()
-; CHECK: declare void @__sanitizer_cov_trace_pc_guard(i32*)
-; CHECK: declare void @__sanitizer_cov_trace_pc_guard_init(i32*, i32*)
+; CHECK-DAG: declare void @__sanitizer_cov_trace_pc_indir(i64)
+; CHECK-DAG: declare void @__sanitizer_cov_trace_cmp1(i8 zeroext, i8 zeroext)
+; CHECK-DAG: declare void @__sanitizer_cov_trace_cmp2(i16 zeroext, i16 zeroext)
+; CHECK-DAG: declare void @__sanitizer_cov_trace_cmp4(i32 zeroext, i32 zeroext)
+; CHECK-DAG: declare void @__sanitizer_cov_trace_cmp8(i64, i64)
+; CHECK-DAG: declare void @__sanitizer_cov_trace_div4(i32 zeroext)
+; CHECK-DAG: declare void @__sanitizer_cov_trace_div8(i64)
+; CHECK-DAG: declare void @__sanitizer_cov_trace_gep(i64)
+; CHECK-DAG: declare void @__sanitizer_cov_trace_switch(i64, i64*)
+; CHECK-DAG: declare void @__sanitizer_cov_trace_pc()
+; CHECK-DAG: declare void @__sanitizer_cov_trace_pc_guard(i32*)
+; CHECK-DAG: declare void @__sanitizer_cov_trace_pc_guard_init(i32*, i32*)
 ; CHECK-NOT: declare
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/cmp-tracing.ll b/llvm/test/Instrumentation/SanitizerCoverage/cmp-tracing.ll
index fda6f251b..df329fbe8 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/cmp-tracing.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/cmp-tracing.ll
@@ -1,5 +1,6 @@
 ; Test -sanitizer-coverage-trace-compares=1
 ; RUN: opt < %s -sancov -sanitizer-coverage-level=1 -sanitizer-coverage-trace-compares=1  -S | FileCheck %s
+; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=1 -sanitizer-coverage-trace-compares=1  -S | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/coff-comdat.ll b/llvm/test/Instrumentation/SanitizerCoverage/coff-comdat.ll
index 61a9dcd92..240c6c50e 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/coff-comdat.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/coff-comdat.ll
@@ -1,4 +1,5 @@
 ; RUN: opt < %s -sancov -sanitizer-coverage-level=1 -sanitizer-coverage-inline-8bit-counters=1 -sanitizer-coverage-pc-table=1 -S | FileCheck %s
+; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=1 -sanitizer-coverage-inline-8bit-counters=1 -sanitizer-coverage-pc-table=1 -S | FileCheck %s
 
 ; Make sure we use the right comdat groups for COFF to avoid relocations
 ; against discarded sections. Internal linkage functions are also different from
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/coff-pc-table-inline-8bit-counters.ll b/llvm/test/Instrumentation/SanitizerCoverage/coff-pc-table-inline-8bit-counters.ll
index d81d48000..3c279ebea 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/coff-pc-table-inline-8bit-counters.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/coff-pc-table-inline-8bit-counters.ll
@@ -1,5 +1,6 @@
 ; Checks that the PC and 8-bit Counter Arrays are placed in their own sections in COFF binaries.
 ; RUN: opt < %s -sancov -sanitizer-coverage-level=1 -sanitizer-coverage-inline-8bit-counters=1 -sanitizer-coverage-pc-table=1 -S | FileCheck %s
+; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=1 -sanitizer-coverage-inline-8bit-counters=1 -sanitizer-coverage-pc-table=1 -S | FileCheck %s
 target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-pc-windows-msvc19.14.26433"
 
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/coff-used-ctor.ll b/llvm/test/Instrumentation/SanitizerCoverage/coff-used-ctor.ll
index fd12eed8e..d89375af9 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/coff-used-ctor.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/coff-used-ctor.ll
@@ -1,5 +1,6 @@
 ; Checks that sancov.module_ctor is marked used.
 ; RUN: opt < %s -sancov -sanitizer-coverage-level=1 -sanitizer-coverage-inline-8bit-counters=1 -sanitizer-coverage-pc-table=1 -S | FileCheck %s
+; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=1 -sanitizer-coverage-inline-8bit-counters=1 -sanitizer-coverage-pc-table=1 -S | FileCheck %s
 target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-pc-windows-msvc19.14.26433"
 
@@ -8,4 +9,4 @@ entry:
   ret void
 }
 
-; CHECK: @llvm.used = appending global {{.*}} @sancov.module_ctor
\ No newline at end of file
+; CHECK: @llvm.used = appending global {{.*}} @sancov.module_ctor
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/const-cmp-tracing.ll b/llvm/test/Instrumentation/SanitizerCoverage/const-cmp-tracing.ll
index b61b4eef5..2a1c90a92 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/const-cmp-tracing.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/const-cmp-tracing.ll
@@ -1,5 +1,6 @@
 ; Test -sanitizer-coverage-trace-compares=1
 ; RUN: opt < %s -sancov -sanitizer-coverage-level=1 -sanitizer-coverage-trace-compares=1  -S | FileCheck %s
+; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=1 -sanitizer-coverage-trace-compares=1  -S | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/coverage-dbg.ll b/llvm/test/Instrumentation/SanitizerCoverage/coverage-dbg.ll
index 09e233725..a743642d5 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/coverage-dbg.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/coverage-dbg.ll
@@ -1,6 +1,7 @@
 ; Test that coverage instrumentation does not lose debug location.
 
 ; RUN: opt < %s -sancov -sanitizer-coverage-level=1 -S | FileCheck %s
+; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=1 -S | FileCheck %s
 
 ; C++ source:
 ; 1: struct A {
@@ -36,7 +37,7 @@ entry:
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.value(metadata, i64, metadata, metadata) #1
 
-attributes #0 = { sanitize_address nounwind readonly uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { sanitize_address nounwind readonly uwtable "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/coverage.ll b/llvm/test/Instrumentation/SanitizerCoverage/coverage.ll
index 7b6b5f004..4947af8f2 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/coverage.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/coverage.ll
@@ -1,7 +1,9 @@
 ; RUN: opt < %s -sancov -sanitizer-coverage-level=4 -sanitizer-coverage-trace-pc  -S | FileCheck %s --check-prefix=CHECK_TRACE_PC
-
 ; RUN: opt < %s -sancov -sanitizer-coverage-level=3 -sanitizer-coverage-prune-blocks=1 -S | FileCheck %s --check-prefix=CHECKPRUNE
 
+; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=4 -sanitizer-coverage-trace-pc  -S | FileCheck %s --check-prefix=CHECK_TRACE_PC
+; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=3 -sanitizer-coverage-prune-blocks=1 -S | FileCheck %s --check-prefix=CHECKPRUNE
+
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 target triple = "x86_64-unknown-linux-gnu"
 define void @foo(i32* %a) sanitize_address {
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/coverage2-dbg.ll b/llvm/test/Instrumentation/SanitizerCoverage/coverage2-dbg.ll
index 508657a59..14462677e 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/coverage2-dbg.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/coverage2-dbg.ll
@@ -1,6 +1,7 @@
 ; Test that coverage instrumentation does not lose debug location.
 
 ; RUN: opt < %s -sancov  -sanitizer-coverage-level=2 -S | FileCheck %s
+; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=2 -S | FileCheck %s
 
 ; C++ source:
 ; 1: void foo(int *a) {
@@ -40,7 +41,7 @@ if.end:                                           ; preds = %entry, %if.then
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.value(metadata, i64, metadata, metadata) #1
 
-attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" sanitize_address}
+attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" sanitize_address}
 attributes #1 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/div-tracing.ll b/llvm/test/Instrumentation/SanitizerCoverage/div-tracing.ll
index 0de2ddf68..35162bc27 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/div-tracing.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/div-tracing.ll
@@ -1,5 +1,6 @@
 ; Test -sanitizer-coverage-trace-divs=1
 ; RUN: opt < %s -sancov -sanitizer-coverage-level=1 -sanitizer-coverage-trace-divs=1  -S | FileCheck %s
+; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=1 -sanitizer-coverage-trace-divs=1  -S | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/gep-tracing.ll b/llvm/test/Instrumentation/SanitizerCoverage/gep-tracing.ll
index ac6af4b37..04ff415c2 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/gep-tracing.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/gep-tracing.ll
@@ -1,5 +1,6 @@
 ; Test -sanitizer-coverage-trace-geps=1
 ; RUN: opt < %s -sancov -sanitizer-coverage-level=1 -sanitizer-coverage-trace-geps=1  -S | FileCheck %s
+; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=1 -sanitizer-coverage-trace-geps=1  -S | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/inline-8bit-counters.ll b/llvm/test/Instrumentation/SanitizerCoverage/inline-8bit-counters.ll
index 88141678b..bdcf0208d 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/inline-8bit-counters.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/inline-8bit-counters.ll
@@ -1,5 +1,6 @@
 ; Test -sanitizer-coverage-inline-8bit-counters=1
 ; RUN: opt < %s -sancov -sanitizer-coverage-level=1 -sanitizer-coverage-inline-8bit-counters=1  -S | FileCheck %s
+; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=1 -sanitizer-coverage-inline-8bit-counters=1  -S | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/interposable-symbol-nocomdat.ll b/llvm/test/Instrumentation/SanitizerCoverage/interposable-symbol-nocomdat.ll
index c79a2fb5f..fcdea5535 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/interposable-symbol-nocomdat.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/interposable-symbol-nocomdat.ll
@@ -1,6 +1,8 @@
 ; Test that interposable symbols do not get put in comdats.
 ; RUN: opt < %s -sancov -sanitizer-coverage-level=3 -sanitizer-coverage-trace-pc-guard -mtriple x86_64-linux-gnu -S | FileCheck %s
 ; RUN: opt < %s -sancov -sanitizer-coverage-level=3 -sanitizer-coverage-trace-pc-guard -mtriple x86_64-windows-msvc -S | FileCheck %s
+; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=3 -sanitizer-coverage-trace-pc-guard -mtriple x86_64-linux-gnu -S | FileCheck %s
+; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=3 -sanitizer-coverage-trace-pc-guard -mtriple x86_64-windows-msvc -S | FileCheck %s
 
 define void @Vanilla() {
 entry:
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/no-func.ll b/llvm/test/Instrumentation/SanitizerCoverage/no-func.ll
index ec9e12143..6dfecb87d 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/no-func.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/no-func.ll
@@ -1,6 +1,7 @@
 ; Tests that we don't insert __sanitizer_cov_trace_pc_guard_init or some such
 ; when there is no instrumentation.
 ; RUN: opt < %s -sancov -sanitizer-coverage-level=3 -sanitizer-coverage-trace-pc-guard  -S | FileCheck %s
+; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=3 -sanitizer-coverage-trace-pc-guard  -S | FileCheck %s
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/pc-table.ll b/llvm/test/Instrumentation/SanitizerCoverage/pc-table.ll
index 888277a4c..dfed75610 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/pc-table.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/pc-table.ll
@@ -1,6 +1,8 @@
 ; Test -sanitizer-coverage-pc-table=1
 ; RUN: opt < %s -sancov -sanitizer-coverage-level=3 -sanitizer-coverage-trace-pc-guard       -sanitizer-coverage-pc-table=1 -S | FileCheck %s
 ; RUN: opt < %s -sancov -sanitizer-coverage-level=3 -sanitizer-coverage-inline-8bit-counters -sanitizer-coverage-pc-table=1 -S | FileCheck %s
+; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=3 -sanitizer-coverage-trace-pc-guard       -sanitizer-coverage-pc-table=1 -S | FileCheck %s
+; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=3 -sanitizer-coverage-inline-8bit-counters -sanitizer-coverage-pc-table=1 -S | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/postdominator_check.ll b/llvm/test/Instrumentation/SanitizerCoverage/postdominator_check.ll
index c50d663ef..50b411de9 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/postdominator_check.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/postdominator_check.ll
@@ -1,5 +1,7 @@
 ; RUN: opt < %s -sancov -sanitizer-coverage-level=4 -sanitizer-coverage-trace-pc -sanitizer-coverage-prune-blocks=1 -S | FileCheck %s
 ; RUN: opt < %s -sancov -sanitizer-coverage-level=4 -sanitizer-coverage-trace-pc -sanitizer-coverage-prune-blocks=0 -S | FileCheck %s --check-prefix=CHECK_NO_PRUNE
+; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=4 -sanitizer-coverage-trace-pc -sanitizer-coverage-prune-blocks=1 -S | FileCheck %s
+; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=4 -sanitizer-coverage-trace-pc -sanitizer-coverage-prune-blocks=0 -S | FileCheck %s --check-prefix=CHECK_NO_PRUNE
 
 define i32 @foo(i32) #0 {
   %2 = icmp sgt i32 %0, 0
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/seh.ll b/llvm/test/Instrumentation/SanitizerCoverage/seh.ll
index 94d1a2e9a..f88ccbe2d 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/seh.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/seh.ll
@@ -1,6 +1,9 @@
 ; RUN: opt < %s -sancov -sanitizer-coverage-level=0 -S | FileCheck %s
 ; RUN: opt < %s -sancov -sanitizer-coverage-level=1 -S | FileCheck %s
 ; RUN: opt < %s -sancov -sanitizer-coverage-level=2 -S | FileCheck %s
+; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=0 -S | FileCheck %s
+; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=1 -S | FileCheck %s
+; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=2 -S | FileCheck %s
 
 target datalayout = "e-m:x-p:32:32-i64:64-f80:32-n8:16:32-a:0:32-S32"
 target triple = "i686-pc-windows-msvc18.0.0"
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/stack-depth-variable-declared-by-user.ll.x b/llvm/test/Instrumentation/SanitizerCoverage/stack-depth-variable-declared-by-user.ll.x
index 1ad96f82a..b93d6e91a 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/stack-depth-variable-declared-by-user.ll.x
+++ b/llvm/test/Instrumentation/SanitizerCoverage/stack-depth-variable-declared-by-user.ll.x
@@ -2,6 +2,8 @@
 ; user declares `__sancov_lowest_stack` with an unexpected type.
 ; RUN: not opt < %s -sancov -sanitizer-coverage-level=1 \
 ; RUN:         -sanitizer-coverage-stack-depth -S 2>&1 | FileCheck %s
+; RUN: not opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=1 \
+; RUN:         -sanitizer-coverage-stack-depth -S 2>&1 | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/stack-depth.ll b/llvm/test/Instrumentation/SanitizerCoverage/stack-depth.ll
index 0c6db1a92..bba3e3b7d 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/stack-depth.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/stack-depth.ll
@@ -4,6 +4,11 @@
 ; RUN: opt < %s -sancov -sanitizer-coverage-level=3 \
 ; RUN:     -sanitizer-coverage-stack-depth -sanitizer-coverage-trace-pc-guard \
 ; RUN:     -S | FileCheck %s
+; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=1 \
+; RUN:     -sanitizer-coverage-stack-depth -S | FileCheck %s
+; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=3 \
+; RUN:     -sanitizer-coverage-stack-depth -sanitizer-coverage-trace-pc-guard \
+; RUN:     -S | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
@@ -14,7 +19,7 @@ target triple = "x86_64-unknown-linux-gnu"
 define i32 @foo() {
 entry:
 ; CHECK-LABEL: define i32 @foo
-; CHECK-NOT: call i8* @llvm.frameaddress(i32 0)
+; CHECK-NOT: call i8* @llvm.frameaddress.p0i8(i32 0)
 ; CHECK-NOT: @__sancov_lowest_stack
 ; CHECK: ret i32 7
 
@@ -24,7 +29,7 @@ entry:
 define i32 @bar() {
 entry:
 ; CHECK-LABEL: define i32 @bar
-; CHECK: [[framePtr:%[^ \t]+]] = call i8* @llvm.frameaddress(i32 0)
+; CHECK: [[framePtr:%[^ \t]+]] = call i8* @llvm.frameaddress.p0i8(i32 0)
 ; CHECK: [[frameInt:%[^ \t]+]] = ptrtoint i8* [[framePtr]] to [[intType:i[0-9]+]]
 ; CHECK: [[lowest:%[^ \t]+]] = load [[intType]], [[intType]]* @__sancov_lowest_stack
 ; CHECK: [[cmp:%[^ \t]+]] = icmp ult [[intType]] [[frameInt]], [[lowest]]
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/switch-tracing.ll b/llvm/test/Instrumentation/SanitizerCoverage/switch-tracing.ll
index debb825db..2d4eb4ca1 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/switch-tracing.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/switch-tracing.ll
@@ -1,5 +1,6 @@
 ; Test -sanitizer-coverage-trace-compares=1 (instrumenting a switch)
 ; RUN: opt < %s -sancov -sanitizer-coverage-level=1 -sanitizer-coverage-trace-compares=1  -S | FileCheck %s
+; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=1 -sanitizer-coverage-trace-compares=1  -S | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/trace-pc-guard-comdat.ll b/llvm/test/Instrumentation/SanitizerCoverage/trace-pc-guard-comdat.ll
index 970ee0d3a..d68eca0fa 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/trace-pc-guard-comdat.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/trace-pc-guard-comdat.ll
@@ -1,4 +1,5 @@
 ; RUN: opt < %s -sancov -sanitizer-coverage-level=4 -sanitizer-coverage-trace-pc-guard  -S | FileCheck %s --check-prefix=CHECK_TRACE_PC_GUARD
+; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=4 -sanitizer-coverage-trace-pc-guard  -S | FileCheck %s --check-prefix=CHECK_TRACE_PC_GUARD
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/trace-pc-guard-inline-8bit-counters.ll b/llvm/test/Instrumentation/SanitizerCoverage/trace-pc-guard-inline-8bit-counters.ll
index d5c9ff451..b3b2cc189 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/trace-pc-guard-inline-8bit-counters.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/trace-pc-guard-inline-8bit-counters.ll
@@ -1,4 +1,5 @@
 ; RUN: opt < %s -sancov -sanitizer-coverage-level=1 -sanitizer-coverage-trace-pc-guard -sanitizer-coverage-inline-8bit-counters -S | FileCheck %s
+; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=1 -sanitizer-coverage-trace-pc-guard -sanitizer-coverage-inline-8bit-counters -S | FileCheck %s
 
 ; Module ctors should have stable names across modules, not something like
 ; @sancov.module_ctor.3 that may cause duplicate ctors after linked together.
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/trace-pc-guard-nocomdat.ll b/llvm/test/Instrumentation/SanitizerCoverage/trace-pc-guard-nocomdat.ll
index 1fe188697..95cb2b96b 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/trace-pc-guard-nocomdat.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/trace-pc-guard-nocomdat.ll
@@ -1,4 +1,5 @@
 ; RUN: opt < %s -sancov -sanitizer-coverage-level=4 -sanitizer-coverage-trace-pc-guard  -S | FileCheck %s --check-prefix=CHECK_TRACE_PC_GUARD
+; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=4 -sanitizer-coverage-trace-pc-guard  -S | FileCheck %s --check-prefix=CHECK_TRACE_PC_GUARD
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 target triple = "x86_64-apple-darwin10.0.0"
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/tracing-comdat.ll b/llvm/test/Instrumentation/SanitizerCoverage/tracing-comdat.ll
index baf4dc1e1..709108434 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/tracing-comdat.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/tracing-comdat.ll
@@ -2,6 +2,9 @@
 ; RUN: opt < %s -sancov                    -sanitizer-coverage-level=3 -sanitizer-coverage-trace-pc-guard  -S  | FileCheck %s
 ; Make sure asan does not instrument __sancov_gen_
 ; RUN: opt < %s -sancov -asan -asan-module -sanitizer-coverage-level=3 -sanitizer-coverage-trace-pc-guard  -S  | FileCheck %s
+
+; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=3 -sanitizer-coverage-trace-pc-guard  -S  | FileCheck %s
+; RUN: opt < %s -passes='module(require<asan-globals-md>,sancov-module,asan-module),function(asan)' -sanitizer-coverage-level=3 -sanitizer-coverage-trace-pc-guard  -S  | FileCheck %s
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 $Foo = comdat any
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/tracing.ll b/llvm/test/Instrumentation/SanitizerCoverage/tracing.ll
index 7bf8cf7e1..ddc0e354e 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/tracing.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/tracing.ll
@@ -3,6 +3,10 @@
 ; RUN: opt < %s -sancov -sanitizer-coverage-level=3 -sanitizer-coverage-trace-pc-guard  -S | FileCheck %s --check-prefix=CHECK_PC_GUARD
 ; RUN: opt < %s -sancov -sanitizer-coverage-level=3 -sanitizer-coverage-trace-pc-guard  -S -mtriple=x86_64-apple-macosx | FileCheck %s --check-prefix=CHECK_PC_GUARD_DARWIN
 
+; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=3 -sanitizer-coverage-trace-pc  -S | FileCheck %s --check-prefix=CHECK_PC
+; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=3 -sanitizer-coverage-trace-pc-guard  -S | FileCheck %s --check-prefix=CHECK_PC_GUARD
+; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=3 -sanitizer-coverage-trace-pc-guard  -S -mtriple=x86_64-apple-macosx | FileCheck %s --check-prefix=CHECK_PC_GUARD_DARWIN
+
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 target triple = "x86_64-unknown-linux-gnu"
 define void @foo(i32* %a) sanitize_address {
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/unreachable-critedge.ll b/llvm/test/Instrumentation/SanitizerCoverage/unreachable-critedge.ll
index ad6cd574d..99e3383db 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/unreachable-critedge.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/unreachable-critedge.ll
@@ -1,4 +1,5 @@
 ; RUN: opt < %s -S -sancov -sanitizer-coverage-level=3 | FileCheck %s
+; RUN: opt < %s -S -passes='module(sancov-module)' -sanitizer-coverage-level=3 | FileCheck %s
 
 ; The critical edges to unreachable_bb should not be split.
 define i32 @foo(i32 %c, i32 %d) {
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/wineh.ll b/llvm/test/Instrumentation/SanitizerCoverage/wineh.ll
index 87b44be55..fbe9d89f0 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/wineh.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/wineh.ll
@@ -1,4 +1,5 @@
 ; RUN: opt < %s -sancov -sanitizer-coverage-level=3 -sanitizer-coverage-trace-pc -S | FileCheck %s --check-prefix=CHECK
+; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=3 -sanitizer-coverage-trace-pc -S | FileCheck %s --check-prefix=CHECK
 
 ; Generated from this C++ source:
 ; $ clang -O2 t.cpp -S -emit-llvm
@@ -98,10 +99,10 @@ declare void @"\01??1Foo@@QEAA@XZ"(%struct.Foo*) unnamed_addr #3
 ; Function Attrs: argmemonly nounwind
 declare void @llvm.lifetime.end(i64, i8* nocapture) #1
 
-attributes #0 = { uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { argmemonly nounwind }
-attributes #2 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #3 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #4 = { nounwind }
 
 !llvm.module.flags = !{!0}
diff --git a/llvm/test/Instrumentation/ThreadSanitizer/tsan_basic.ll b/llvm/test/Instrumentation/ThreadSanitizer/tsan_basic.ll
index 8b85d7b8b..953ab8ed8 100644
--- a/llvm/test/Instrumentation/ThreadSanitizer/tsan_basic.ll
+++ b/llvm/test/Instrumentation/ThreadSanitizer/tsan_basic.ll
@@ -1,5 +1,5 @@
 ; RUN: opt < %s -tsan -S | FileCheck %s
-; RUN: opt < %s -passes=tsan -S | FileCheck %s
+; RUN: opt < %s -passes='function(tsan),module(tsan-module)' -S | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/JitListener/multiple.ll b/llvm/test/JitListener/multiple.ll
index 9c72fb369..d60bf0b79 100644
--- a/llvm/test/JitListener/multiple.ll
+++ b/llvm/test/JitListener/multiple.ll
@@ -118,7 +118,7 @@ return:                                           ; preds = %sw.default, %sw.bb1
   ret i32 %1, !dbg !39
 }
 
-attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/JitListener/simple.ll b/llvm/test/JitListener/simple.ll
index ce98844aa..07085153d 100644
--- a/llvm/test/JitListener/simple.ll
+++ b/llvm/test/JitListener/simple.ll
@@ -28,7 +28,7 @@ entry:
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
 
-attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/LTO/ARM/lto-linking-metadata.ll b/llvm/test/LTO/ARM/lto-linking-metadata.ll
new file mode 100644
index 000000000..ae6f42ff9
--- /dev/null
+++ b/llvm/test/LTO/ARM/lto-linking-metadata.ll
@@ -0,0 +1,19 @@
+; RUN: opt %s -o %t1.bc
+
+; RUN: llvm-lto %t1.bc -o %t1.save.opt -save-merged-module -O1 --exported-symbol=foo
+; RUN: llvm-dis < %t1.save.opt.merged.bc | FileCheck %s
+
+; RUN: llvm-lto2 run %t1.bc -o %t.out.o -save-temps \
+; RUN:     -r=%t1.bc,foo,pxl
+; RUN: llvm-dis < %t.out.o.0.2.internalize.bc | FileCheck  %s
+
+target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
+target triple = "armv7a-unknown-linux"
+
+define void @foo() {
+entry:
+  ret void
+}
+
+; CHECK: !llvm.module.flags = !{[[MD_NUM:![0-9]+]]}
+; CHECK: [[MD_NUM]] = !{i32 1, !"LTOPostLink", i32 1}
diff --git a/llvm/test/LTO/Resolution/X86/Inputs/alias-1.ll b/llvm/test/LTO/Resolution/X86/Inputs/alias-1.ll
index 01c9987fd..bc54680fa 100644
--- a/llvm/test/LTO/Resolution/X86/Inputs/alias-1.ll
+++ b/llvm/test/LTO/Resolution/X86/Inputs/alias-1.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @a = global i32 42
diff --git a/llvm/test/LTO/Resolution/X86/Inputs/alias-alias-1.ll b/llvm/test/LTO/Resolution/X86/Inputs/alias-alias-1.ll
index 956deabcf..4ee753e40 100644
--- a/llvm/test/LTO/Resolution/X86/Inputs/alias-alias-1.ll
+++ b/llvm/test/LTO/Resolution/X86/Inputs/alias-alias-1.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @a = external global i32
diff --git a/llvm/test/LTO/Resolution/X86/Inputs/appending-var-2.ll b/llvm/test/LTO/Resolution/X86/Inputs/appending-var-2.ll
index 29bc8433d..1cd01d789 100644
--- a/llvm/test/LTO/Resolution/X86/Inputs/appending-var-2.ll
+++ b/llvm/test/LTO/Resolution/X86/Inputs/appending-var-2.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 %"foo" = type { i8 }
diff --git a/llvm/test/LTO/Resolution/X86/Inputs/comdat-mixed-lto.ll b/llvm/test/LTO/Resolution/X86/Inputs/comdat-mixed-lto.ll
index 0112b89f9..b2f901cb0 100644
--- a/llvm/test/LTO/Resolution/X86/Inputs/comdat-mixed-lto.ll
+++ b/llvm/test/LTO/Resolution/X86/Inputs/comdat-mixed-lto.ll
@@ -1,6 +1,6 @@
 ; ModuleID = 'comdat-mixed-lto1.o'
 source_filename = "comdat-mixed-lto1.cpp"
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 %"class.Test::ptr" = type { i32 }
diff --git a/llvm/test/LTO/Resolution/X86/Inputs/comdat.ll b/llvm/test/LTO/Resolution/X86/Inputs/comdat.ll
index ca4bbb4bf..52cadf39a 100644
--- a/llvm/test/LTO/Resolution/X86/Inputs/comdat.ll
+++ b/llvm/test/LTO/Resolution/X86/Inputs/comdat.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 $c2 = comdat any
diff --git a/llvm/test/LTO/Resolution/X86/Inputs/common2.ll b/llvm/test/LTO/Resolution/X86/Inputs/common2.ll
index c3a7f7536..3a4c631ad 100644
--- a/llvm/test/LTO/Resolution/X86/Inputs/common2.ll
+++ b/llvm/test/LTO/Resolution/X86/Inputs/common2.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @v = common global i16 0, align 4
diff --git a/llvm/test/LTO/Resolution/X86/Inputs/commons.ll b/llvm/test/LTO/Resolution/X86/Inputs/commons.ll
index 9372600d0..9ff7de26a 100644
--- a/llvm/test/LTO/Resolution/X86/Inputs/commons.ll
+++ b/llvm/test/LTO/Resolution/X86/Inputs/commons.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @x = global i32 42, align 4
diff --git a/llvm/test/LTO/Resolution/X86/Inputs/dead-strip-alias.ll b/llvm/test/LTO/Resolution/X86/Inputs/dead-strip-alias.ll
index 16154d2c4..b32d5b0f2 100644
--- a/llvm/test/LTO/Resolution/X86/Inputs/dead-strip-alias.ll
+++ b/llvm/test/LTO/Resolution/X86/Inputs/dead-strip-alias.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @external = global i8 42
diff --git a/llvm/test/LTO/Resolution/X86/Inputs/dead-strip-fulllto.ll b/llvm/test/LTO/Resolution/X86/Inputs/dead-strip-fulllto.ll
index cc508924c..cfe1544a6 100644
--- a/llvm/test/LTO/Resolution/X86/Inputs/dead-strip-fulllto.ll
+++ b/llvm/test/LTO/Resolution/X86/Inputs/dead-strip-fulllto.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define void @live1() {
diff --git a/llvm/test/LTO/Resolution/X86/Inputs/ifunc2.ll b/llvm/test/LTO/Resolution/X86/Inputs/ifunc2.ll
new file mode 100644
index 000000000..a70325beb
--- /dev/null
+++ b/llvm/test/LTO/Resolution/X86/Inputs/ifunc2.ll
@@ -0,0 +1,6 @@
+target datalayout = "e-p:64:64"
+target triple = "x86_64-unknown-linux-gnu"
+
+define i32 @foo_resolver() {
+  ret i32 2
+}
diff --git a/llvm/test/LTO/Resolution/X86/Inputs/intrinsic.ll b/llvm/test/LTO/Resolution/X86/Inputs/intrinsic.ll
index 52ee296b1..82a7c2e31 100644
--- a/llvm/test/LTO/Resolution/X86/Inputs/intrinsic.ll
+++ b/llvm/test/LTO/Resolution/X86/Inputs/intrinsic.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 %foo = type {  }
 declare <4 x %foo*> @llvm.masked.load.v4p0foo.p0v4p0foo(<4 x %foo*>*, i32, <4 x i1>, <4 x %foo*>)
diff --git a/llvm/test/LTO/Resolution/X86/Inputs/link-odr-availextern-ae.ll b/llvm/test/LTO/Resolution/X86/Inputs/link-odr-availextern-ae.ll
index f2d180afc..1b8064a8d 100644
--- a/llvm/test/LTO/Resolution/X86/Inputs/link-odr-availextern-ae.ll
+++ b/llvm/test/LTO/Resolution/X86/Inputs/link-odr-availextern-ae.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define available_externally i32 @f() {
diff --git a/llvm/test/LTO/Resolution/X86/Inputs/link-odr-availextern-odr.ll b/llvm/test/LTO/Resolution/X86/Inputs/link-odr-availextern-odr.ll
index 76e745a44..312b3ad1e 100644
--- a/llvm/test/LTO/Resolution/X86/Inputs/link-odr-availextern-odr.ll
+++ b/llvm/test/LTO/Resolution/X86/Inputs/link-odr-availextern-odr.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define linkonce_odr i32 @f() {
diff --git a/llvm/test/LTO/Resolution/X86/Inputs/mixed_lto.ll b/llvm/test/LTO/Resolution/X86/Inputs/mixed_lto.ll
index 2393deb32..ec561362a 100644
--- a/llvm/test/LTO/Resolution/X86/Inputs/mixed_lto.ll
+++ b/llvm/test/LTO/Resolution/X86/Inputs/mixed_lto.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 declare i32 @g()
 define i32 @main() {
diff --git a/llvm/test/LTO/Resolution/X86/Inputs/mod-asm-used.ll b/llvm/test/LTO/Resolution/X86/Inputs/mod-asm-used.ll
index 3b1988bb5..c3b08ed60 100644
--- a/llvm/test/LTO/Resolution/X86/Inputs/mod-asm-used.ll
+++ b/llvm/test/LTO/Resolution/X86/Inputs/mod-asm-used.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @foo = global i32 1, align 4
diff --git a/llvm/test/LTO/Resolution/X86/Inputs/not-prevailing.ll b/llvm/test/LTO/Resolution/X86/Inputs/not-prevailing.ll
index 68b3c8362..7986919be 100644
--- a/llvm/test/LTO/Resolution/X86/Inputs/not-prevailing.ll
+++ b/llvm/test/LTO/Resolution/X86/Inputs/not-prevailing.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define void @bar() {
diff --git a/llvm/test/LTO/Resolution/X86/alias-alias.ll b/llvm/test/LTO/Resolution/X86/alias-alias.ll
index eaf2615b0..a8489112a 100644
--- a/llvm/test/LTO/Resolution/X86/alias-alias.ll
+++ b/llvm/test/LTO/Resolution/X86/alias-alias.ll
@@ -19,7 +19,7 @@
 ; RES-NEXT: {{^}}-r={{.*}}2.o,a,{{$}}
 ; RES-NEXT: {{^}}-r={{.*}}2.o,d,px{{$}}
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @a = weak alias i32, i32* @b
diff --git a/llvm/test/LTO/Resolution/X86/alias.ll b/llvm/test/LTO/Resolution/X86/alias.ll
index 886eadcfc..f565e4d23 100644
--- a/llvm/test/LTO/Resolution/X86/alias.ll
+++ b/llvm/test/LTO/Resolution/X86/alias.ll
@@ -15,7 +15,7 @@
 ; RES: {{^}}-r={{.*}}1.o,b,px{{$}}
 ; RES: {{^}}-r={{.*}}1.o,a,{{$}}
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @a = weak alias i32, i32* @b
diff --git a/llvm/test/LTO/Resolution/X86/appending-var.ll b/llvm/test/LTO/Resolution/X86/appending-var.ll
index 4d259d7f8..684e64129 100644
--- a/llvm/test/LTO/Resolution/X86/appending-var.ll
+++ b/llvm/test/LTO/Resolution/X86/appending-var.ll
@@ -7,7 +7,7 @@
 
 ; RUN: llvm-lto2 run -o %t3.o %t1.o %t2.o -r %t1.o,bar, -r %t2.o,bar,px
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 %"foo.1" = type { i8, i8 }
diff --git a/llvm/test/LTO/Resolution/X86/asm-output.ll b/llvm/test/LTO/Resolution/X86/asm-output.ll
index 41d293501..7f15cecc8 100644
--- a/llvm/test/LTO/Resolution/X86/asm-output.ll
+++ b/llvm/test/LTO/Resolution/X86/asm-output.ll
@@ -9,7 +9,7 @@
 ;
 ; ASM: main:
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define i32 @main() {
diff --git a/llvm/test/LTO/Resolution/X86/available-externally.ll b/llvm/test/LTO/Resolution/X86/available-externally.ll
index eb17693c0..ac35b15b3 100644
--- a/llvm/test/LTO/Resolution/X86/available-externally.ll
+++ b/llvm/test/LTO/Resolution/X86/available-externally.ll
@@ -4,7 +4,7 @@
 
 ; CHECK: U bar
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define void ()* @foo() {
diff --git a/llvm/test/LTO/Resolution/X86/cache-dso-local.ll b/llvm/test/LTO/Resolution/X86/cache-dso-local.ll
index 5e5b94cc9..6ed8e056b 100644
--- a/llvm/test/LTO/Resolution/X86/cache-dso-local.ll
+++ b/llvm/test/LTO/Resolution/X86/cache-dso-local.ll
@@ -10,7 +10,7 @@
 ; RUN:   -r %t.bc,bar,px
 ; RUN: ls %t.cache | count 2
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define weak void @foo() {
diff --git a/llvm/test/LTO/Resolution/X86/cache-dso-local2.ll b/llvm/test/LTO/Resolution/X86/cache-dso-local2.ll
index 3a6c15e52..2316cbfdc 100644
--- a/llvm/test/LTO/Resolution/X86/cache-dso-local2.ll
+++ b/llvm/test/LTO/Resolution/X86/cache-dso-local2.ll
@@ -10,7 +10,7 @@
 ; RUN:   -r %t.bc,bar,px
 ; RUN: ls %t.cache | count 2
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define weak void @foo() {
diff --git a/llvm/test/LTO/Resolution/X86/cache-prevailing.ll b/llvm/test/LTO/Resolution/X86/cache-prevailing.ll
index 57687492e..72a1d4e6f 100644
--- a/llvm/test/LTO/Resolution/X86/cache-prevailing.ll
+++ b/llvm/test/LTO/Resolution/X86/cache-prevailing.ll
@@ -7,7 +7,7 @@
 ; RUN:   -r %t.bc,foo, -r %t.bc,bar,px
 ; RUN: ls %t.cache | count 2
 
-target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:w-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-pc-windows-msvc19.11.0"
 
 @foo = linkonce constant i32 1, comdat
diff --git a/llvm/test/LTO/Resolution/X86/comdat-mixed-lto.ll b/llvm/test/LTO/Resolution/X86/comdat-mixed-lto.ll
index d6022c643..d3730f4e9 100644
--- a/llvm/test/LTO/Resolution/X86/comdat-mixed-lto.ll
+++ b/llvm/test/LTO/Resolution/X86/comdat-mixed-lto.ll
@@ -21,7 +21,7 @@
 
 ; ModuleID = 'comdat-mixed-lto.o'
 source_filename = "comdat-mixed-lto.cpp"
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 %"class.Test::ptr" = type { i32 }
diff --git a/llvm/test/LTO/Resolution/X86/comdat.ll b/llvm/test/LTO/Resolution/X86/comdat.ll
index 144016184..06b61b7ed 100644
--- a/llvm/test/LTO/Resolution/X86/comdat.ll
+++ b/llvm/test/LTO/Resolution/X86/comdat.ll
@@ -22,7 +22,7 @@
 ; RUN:  -r=%t2.o,a25,px
 ; RUN: llvm-dis %t3.o.0.2.internalize.bc -o - | FileCheck %s
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 $c1 = comdat any
@@ -70,7 +70,7 @@ bb11:
 ; CHECK-DAG: @a23 = alias i32 (i8*), i32 (i8*)* @f1.2{{$}}
 ; CHECK-DAG: @a24 = alias i16, bitcast (i32 (i8*)* @f1.2 to i16*)
 
-; CHECK:      define weak_odr dso_local i32 @f1(i8*) comdat($c1) {
+; CHECK:      define weak_odr dso_local i32 @f1(i8* %0) comdat($c1) {
 ; CHECK-NEXT: bb10:
 ; CHECK-NEXT:   br label %bb11{{$}}
 ; CHECK:      bb11:
diff --git a/llvm/test/LTO/Resolution/X86/common2.ll b/llvm/test/LTO/Resolution/X86/common2.ll
index de702d057..c0447b75b 100644
--- a/llvm/test/LTO/Resolution/X86/common2.ll
+++ b/llvm/test/LTO/Resolution/X86/common2.ll
@@ -52,7 +52,7 @@
 ; RUN:  -r %t2.bc,bar,px
 ; RUN: llvm-dis < %t.o.0.0.preopt.bc | FileCheck  %s --check-prefix=NONE-PREVAILED2
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @v = common global i8 0, align 8
diff --git a/llvm/test/LTO/Resolution/X86/commons.ll b/llvm/test/LTO/Resolution/X86/commons.ll
index 8adfb87d6..1ccb5ed57 100644
--- a/llvm/test/LTO/Resolution/X86/commons.ll
+++ b/llvm/test/LTO/Resolution/X86/commons.ll
@@ -6,7 +6,7 @@
 ; A strong definition should override the common
 ; CHECK: @x = dso_local global i32 42, align 4
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @x = common global i16 0, align 2
diff --git a/llvm/test/LTO/Resolution/X86/dead-strip-alias.ll b/llvm/test/LTO/Resolution/X86/dead-strip-alias.ll
index fc5554c34..d62621acc 100644
--- a/llvm/test/LTO/Resolution/X86/dead-strip-alias.ll
+++ b/llvm/test/LTO/Resolution/X86/dead-strip-alias.ll
@@ -7,7 +7,7 @@
 
 ; CHECK: D external
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @alias = alias i8*, i8** @internal
diff --git a/llvm/test/LTO/Resolution/X86/dead-strip-fulllto.ll b/llvm/test/LTO/Resolution/X86/dead-strip-fulllto.ll
index 11a8981ac..9a5dd9983 100644
--- a/llvm/test/LTO/Resolution/X86/dead-strip-fulllto.ll
+++ b/llvm/test/LTO/Resolution/X86/dead-strip-fulllto.ll
@@ -23,7 +23,7 @@
 ; THIN: U live2
 ; THIN-NOT: odr
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define void @main() {
diff --git a/llvm/test/LTO/Resolution/X86/diagnostic-handler-remarks-with-hotness.ll b/llvm/test/LTO/Resolution/X86/diagnostic-handler-remarks-with-hotness.ll
index 9932f527d..33e345f65 100644
--- a/llvm/test/LTO/Resolution/X86/diagnostic-handler-remarks-with-hotness.ll
+++ b/llvm/test/LTO/Resolution/X86/diagnostic-handler-remarks-with-hotness.ll
@@ -35,7 +35,7 @@
 
 ; CHECK: tinkywinky inlined into main with (cost=-15000, threshold=337) (hotness: 300)
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-scei-ps4"
 
 declare i32 @patatino()
diff --git a/llvm/test/LTO/Resolution/X86/diagnostic-handler-remarks.ll b/llvm/test/LTO/Resolution/X86/diagnostic-handler-remarks.ll
index beb0f4fa4..bc3227017 100644
--- a/llvm/test/LTO/Resolution/X86/diagnostic-handler-remarks.ll
+++ b/llvm/test/LTO/Resolution/X86/diagnostic-handler-remarks.ll
@@ -36,7 +36,7 @@
 ; YAML-NEXT:   - String:          ')'
 ; YAML-NEXT: ...
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-scei-ps4"
 
 declare i32 @patatino()
diff --git a/llvm/test/LTO/Resolution/X86/export-jumptable.ll b/llvm/test/LTO/Resolution/X86/export-jumptable.ll
index 8ced9d962..ec23d1559 100644
--- a/llvm/test/LTO/Resolution/X86/export-jumptable.ll
+++ b/llvm/test/LTO/Resolution/X86/export-jumptable.ll
@@ -6,7 +6,7 @@
 ; RUN: llvm-lto2 run -o %t2 -r %t,f1,p -r %t,f2,p -r %t,_start,px %t -save-temps
 ; RUN: llvm-dis %t2.1.2.internalize.bc -o - | FileCheck %s
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 ; CHECK: define void @f1()
diff --git a/llvm/test/LTO/Resolution/X86/function-alias-non-prevailing.ll b/llvm/test/LTO/Resolution/X86/function-alias-non-prevailing.ll
index ab2cefebb..cc793f64d 100644
--- a/llvm/test/LTO/Resolution/X86/function-alias-non-prevailing.ll
+++ b/llvm/test/LTO/Resolution/X86/function-alias-non-prevailing.ll
@@ -2,7 +2,7 @@
 ; RUN: llvm-lto2 run %t -r %t,foo, -r %t,baz,p -o %t2 -save-temps
 ; RUN: llvm-dis -o - %t2.0.0.preopt.bc | FileCheck %s
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64--fuchsia"
 
 ; CHECK: declare void @foo()
diff --git a/llvm/test/LTO/Resolution/X86/ifunc.ll b/llvm/test/LTO/Resolution/X86/ifunc.ll
index c77f10682..7192ea2d5 100644
--- a/llvm/test/LTO/Resolution/X86/ifunc.ll
+++ b/llvm/test/LTO/Resolution/X86/ifunc.ll
@@ -1,10 +1,10 @@
 ; RUN: opt -module-summary -o %t.bc %s
 ; RUN: llvm-lto2 run %t.bc -r %t.bc,foo,pl -o %t2
 ; RUN: llvm-nm %t2.1 | FileCheck %s
-; CHECK: T foo
+; CHECK: i foo
 ; CHECK: t foo_ifunc
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @foo = ifunc i32 (i32), i64 ()* @foo_ifunc
diff --git a/llvm/test/LTO/Resolution/X86/ifunc2.ll b/llvm/test/LTO/Resolution/X86/ifunc2.ll
new file mode 100644
index 000000000..6dd5e5983
--- /dev/null
+++ b/llvm/test/LTO/Resolution/X86/ifunc2.ll
@@ -0,0 +1,19 @@
+; RUN: llvm-as -o %t1.o %s
+; RUN: llvm-as -o %t2.o %S/Inputs/ifunc2.ll
+; RUN: llvm-lto2 run %t1.o %t2.o -r %t1.o,foo,p -r %t1.o,foo_resolver, -r %t2.o,foo_resolver,p -save-temps -o %t3.o
+; RUN: llvm-dis -o - %t3.o.0.0.preopt.bc | FileCheck %s
+
+target datalayout = "e-p:64:64"
+target triple = "x86_64-unknown-linux-gnu"
+
+; CHECK: @foo = ifunc i32 (), i32 ()* @foo_resolver.2
+@foo = ifunc i32 (), i32 ()* @foo_resolver
+
+; CHECK: define internal i32 @foo_resolver.2() {
+; CHECK-NEXT: ret i32 1
+define weak i32 @foo_resolver() {
+  ret i32 1
+}
+
+; CHECK: define i32 @foo_resolver() {
+; CHECK-NEXT: ret i32 2
diff --git a/llvm/test/LTO/Resolution/X86/intrinsic.ll b/llvm/test/LTO/Resolution/X86/intrinsic.ll
index dc287ace0..597953155 100644
--- a/llvm/test/LTO/Resolution/X86/intrinsic.ll
+++ b/llvm/test/LTO/Resolution/X86/intrinsic.ll
@@ -2,7 +2,7 @@
 ; RUN: llvm-as %p/Inputs/intrinsic.ll -o %t2.o
 ; RUN: llvm-lto2 run -o %t3.o %t1.o %t2.o -r %t1.o,foo
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 %foo = type {  }
 declare void @foo( %foo*  )
diff --git a/llvm/test/LTO/Resolution/X86/link-odr-availextern.ll b/llvm/test/LTO/Resolution/X86/link-odr-availextern.ll
index cc360338d..cb522bfe7 100644
--- a/llvm/test/LTO/Resolution/X86/link-odr-availextern.ll
+++ b/llvm/test/LTO/Resolution/X86/link-odr-availextern.ll
@@ -26,7 +26,7 @@
 ; RUN: llvm-lto2 run -o %t3 %t2odr %t1 -r %t1,f, -r %t2odr,f, -save-temps
 ; RUN: llvm-dis < %t3.0.0.preopt.bc -o - | FileCheck --check-prefix=NONPREVAILING %s
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 ; PREVAILING: define weak_odr i32 @f()
diff --git a/llvm/test/LTO/Resolution/X86/linker-redef-thin.ll b/llvm/test/LTO/Resolution/X86/linker-redef-thin.ll
index 4625ce3fa..d965452ba 100644
--- a/llvm/test/LTO/Resolution/X86/linker-redef-thin.ll
+++ b/llvm/test/LTO/Resolution/X86/linker-redef-thin.ll
@@ -8,7 +8,7 @@
 ; CHECK-NEXT: Binding: Weak
 ; CHECK-NEXT: Type: Function
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define void @patatino() {
diff --git a/llvm/test/LTO/Resolution/X86/linker-redef.ll b/llvm/test/LTO/Resolution/X86/linker-redef.ll
index b53f811e4..73301e9e6 100644
--- a/llvm/test/LTO/Resolution/X86/linker-redef.ll
+++ b/llvm/test/LTO/Resolution/X86/linker-redef.ll
@@ -8,7 +8,7 @@
 ; CHECK-NEXT: Binding: Weak
 ; CHECK-NEXT: Type: Function
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define void @bar() {
diff --git a/llvm/test/LTO/Resolution/X86/linkonce.ll b/llvm/test/LTO/Resolution/X86/linkonce.ll
index fbbb458b9..a52c4451a 100644
--- a/llvm/test/LTO/Resolution/X86/linkonce.ll
+++ b/llvm/test/LTO/Resolution/X86/linkonce.ll
@@ -2,7 +2,7 @@
 ; RUN: llvm-lto2 run %t -O0 -r %t,foo,px -o %t2
 ; RUN: llvm-nm %t2.1 | FileCheck %s
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 ; CHECK: W foo
diff --git a/llvm/test/LTO/Resolution/X86/load-sample-prof-icp.ll b/llvm/test/LTO/Resolution/X86/load-sample-prof-icp.ll
index 79f9c1e9b..2c8549c38 100644
--- a/llvm/test/LTO/Resolution/X86/load-sample-prof-icp.ll
+++ b/llvm/test/LTO/Resolution/X86/load-sample-prof-icp.ll
@@ -11,7 +11,7 @@
 ; RUN:   -lto-sample-profile-file=%S/Inputs/load-sample-prof-icp.prof
 ; RUN: llvm-dis %t.out.1.4.opt.bc -o - | FileCheck %s
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 ; CHECK-LABEL: @test
diff --git a/llvm/test/LTO/Resolution/X86/load-sample-prof-lto.ll b/llvm/test/LTO/Resolution/X86/load-sample-prof-lto.ll
index bdf4f08db..47287d164 100644
--- a/llvm/test/LTO/Resolution/X86/load-sample-prof-lto.ll
+++ b/llvm/test/LTO/Resolution/X86/load-sample-prof-lto.ll
@@ -17,7 +17,7 @@
 ; Make sure profile information is attached.
 ; CHECK: !prof
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define void @foo() local_unnamed_addr !dbg !7 {
diff --git a/llvm/test/LTO/Resolution/X86/load-sample-prof.ll b/llvm/test/LTO/Resolution/X86/load-sample-prof.ll
index 2f9d66e69..8d09cc732 100644
--- a/llvm/test/LTO/Resolution/X86/load-sample-prof.ll
+++ b/llvm/test/LTO/Resolution/X86/load-sample-prof.ll
@@ -8,7 +8,7 @@
 ;
 ; CHECK: !prof
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define void @foo() local_unnamed_addr !dbg !7 {
diff --git a/llvm/test/LTO/Resolution/X86/local-def-dllimport.ll b/llvm/test/LTO/Resolution/X86/local-def-dllimport.ll
index c97e4b7e1..4c70e7280 100644
--- a/llvm/test/LTO/Resolution/X86/local-def-dllimport.ll
+++ b/llvm/test/LTO/Resolution/X86/local-def-dllimport.ll
@@ -7,7 +7,7 @@
 ; RUN:               -save-temps -o %t1 %t0.bc
 ; RUN: llvm-dis %t1.1.3.import.bc -o - | FileCheck %s
 source_filename = "test.cpp"
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 $g = comdat any
diff --git a/llvm/test/LTO/Resolution/X86/lowertypetests.ll b/llvm/test/LTO/Resolution/X86/lowertypetests.ll
index e5be4b98c..8016a66fb 100644
--- a/llvm/test/LTO/Resolution/X86/lowertypetests.ll
+++ b/llvm/test/LTO/Resolution/X86/lowertypetests.ll
@@ -8,7 +8,7 @@
 ; CHECK: U __typeid_bar_global_addr
 ; CHECK: U __typeid_foo_global_addr
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @foo = global i32 0, !type !0
diff --git a/llvm/test/LTO/Resolution/X86/lto-unit-check.ll b/llvm/test/LTO/Resolution/X86/lto-unit-check.ll
index 1736a5b21..de638c1eb 100644
--- a/llvm/test/LTO/Resolution/X86/lto-unit-check.ll
+++ b/llvm/test/LTO/Resolution/X86/lto-unit-check.ll
@@ -51,5 +51,5 @@
 ; ENABLESPLITFLAG: !{i32 1, !"EnableSplitLTOUnit", i32 1}
 ; NOENABLESPLITFLAG-NOT: !{i32 1, !"EnableSplitLTOUnit", i32 1}
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/LTO/Resolution/X86/mixed_lto.ll b/llvm/test/LTO/Resolution/X86/mixed_lto.ll
index aa686a811..8e5620b70 100644
--- a/llvm/test/LTO/Resolution/X86/mixed_lto.ll
+++ b/llvm/test/LTO/Resolution/X86/mixed_lto.ll
@@ -19,7 +19,7 @@
 ; RUN: llvm-nm %t5.o.0 | FileCheck %s --check-prefix=NM0
 ; RUN: llvm-nm %t5.o.1 | FileCheck %s --check-prefix=NM1
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 define i32 @g() {
   ret i32 0
diff --git a/llvm/test/LTO/Resolution/X86/mod-asm-used.ll b/llvm/test/LTO/Resolution/X86/mod-asm-used.ll
index 8e02b6a44..245ff4292 100644
--- a/llvm/test/LTO/Resolution/X86/mod-asm-used.ll
+++ b/llvm/test/LTO/Resolution/X86/mod-asm-used.ll
@@ -7,7 +7,7 @@
 ; INDEX: ^0 = module: (path: "{{.*}}mod-asm-used.ll.tmp.bc"
 ; INDEX: ^1 = module: (path: "{{.*}}mod-asm-used.ll.tmp2.bc"
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 ; CHECK: D foo
diff --git a/llvm/test/LTO/Resolution/X86/multi-thinlto.ll b/llvm/test/LTO/Resolution/X86/multi-thinlto.ll
index 06150e44a..e782b4fd8 100644
--- a/llvm/test/LTO/Resolution/X86/multi-thinlto.ll
+++ b/llvm/test/LTO/Resolution/X86/multi-thinlto.ll
@@ -3,5 +3,5 @@
 ; RUN: not llvm-lto2 run -o %t3.o %t2.o 2>&1 | FileCheck %s
 ; CHECK: Expected at most one ThinLTO module per bitcode file
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/LTO/Resolution/X86/not-prevailing-alias.ll b/llvm/test/LTO/Resolution/X86/not-prevailing-alias.ll
index 79770c5cf..9d1164725 100644
--- a/llvm/test/LTO/Resolution/X86/not-prevailing-alias.ll
+++ b/llvm/test/LTO/Resolution/X86/not-prevailing-alias.ll
@@ -25,7 +25,7 @@
 ; SYMBOLS: NOTYPE  GLOBAL DEFAULT  UND varAlias
 ; SYMBOLS: FUNC    GLOBAL DEFAULT    2 zed
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @barAlias = alias void(), void()* @bar
diff --git a/llvm/test/LTO/Resolution/X86/not-prevailing-variables.ll b/llvm/test/LTO/Resolution/X86/not-prevailing-variables.ll
index 6d22baf35..9a7131b64 100644
--- a/llvm/test/LTO/Resolution/X86/not-prevailing-variables.ll
+++ b/llvm/test/LTO/Resolution/X86/not-prevailing-variables.ll
@@ -19,7 +19,7 @@
 ; RUN: llvm-readelf --symbols %t2.o.1 | FileCheck %s --check-prefix=UND
 ; UND: NOTYPE  GLOBAL DEFAULT UND var2
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @var1 = global i32 10, align 4
diff --git a/llvm/test/LTO/Resolution/X86/not-prevailing-weak-aliasee.ll b/llvm/test/LTO/Resolution/X86/not-prevailing-weak-aliasee.ll
new file mode 100644
index 000000000..0112a3871
--- /dev/null
+++ b/llvm/test/LTO/Resolution/X86/not-prevailing-weak-aliasee.ll
@@ -0,0 +1,33 @@
+; Test to ensure that non-prevailing weak aliasee is kept as a weak definition
+; when the alias is not dead.
+; RUN: opt -module-summary %s -o %t1.bc
+; RUN: llvm-lto2 run %t1.bc \
+; RUN:	 -r=%t1.bc,__a,lx \
+; RUN:	 -r=%t1.bc,__b,l \
+; RUN:	 -r=%t1.bc,a,plx \
+; RUN:	 -r=%t1.bc,b,pl \
+; RUN:   -o %t2.o -save-temps
+
+; Check that __a is kept as a weak def. __b can be dropped since its alias is
+; not live and will also be dropped.
+; RUN: llvm-dis %t2.o.1.1.promote.bc -o - | FileCheck %s
+; CHECK: define weak hidden void @__a
+; CHECK: declare hidden void @__b
+; CHECK: declare void @b
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@a = hidden alias void (), void ()* @__a
+
+define weak hidden void @__a() {
+entry:
+  ret void
+}
+
+@b = hidden alias void (), void ()* @__b
+
+define weak hidden void @__b() {
+entry:
+  ret void
+}
diff --git a/llvm/test/LTO/Resolution/X86/not-prevailing.ll b/llvm/test/LTO/Resolution/X86/not-prevailing.ll
index 58c772472..fb68b0169 100644
--- a/llvm/test/LTO/Resolution/X86/not-prevailing.ll
+++ b/llvm/test/LTO/Resolution/X86/not-prevailing.ll
@@ -21,7 +21,7 @@
 ; SYMBOLS: NOTYPE  GLOBAL DEFAULT  UND foo
 ; SYMBOLS: FUNC    GLOBAL DEFAULT    2 zed
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 $foo = comdat any
diff --git a/llvm/test/LTO/Resolution/X86/setting-dso-local.ll b/llvm/test/LTO/Resolution/X86/setting-dso-local.ll
index 817a8181f..bd9351e91 100644
--- a/llvm/test/LTO/Resolution/X86/setting-dso-local.ll
+++ b/llvm/test/LTO/Resolution/X86/setting-dso-local.ll
@@ -6,7 +6,7 @@
 
 ; CHECK:  0000000000000000     0 NOTYPE  GLOBAL HIDDEN   UND foobar
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @foobar = external hidden global i32
diff --git a/llvm/test/LTO/Resolution/X86/symtab-elf.ll b/llvm/test/LTO/Resolution/X86/symtab-elf.ll
index 40864a82d..229151179 100644
--- a/llvm/test/LTO/Resolution/X86/symtab-elf.ll
+++ b/llvm/test/LTO/Resolution/X86/symtab-elf.ll
@@ -3,7 +3,7 @@
 
 ; CHECK: target triple: x86_64-unknown-linux-gnu
 target triple = "x86_64-unknown-linux-gnu"
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 
 ; CHECK-NOT: linker opts:
 !0 = !{!"/include:foo"}
diff --git a/llvm/test/LTO/Resolution/X86/type-checked-load.ll b/llvm/test/LTO/Resolution/X86/type-checked-load.ll
index 3ec783bc5..726dc9701 100644
--- a/llvm/test/LTO/Resolution/X86/type-checked-load.ll
+++ b/llvm/test/LTO/Resolution/X86/type-checked-load.ll
@@ -5,7 +5,7 @@
 ; It means that the wholeprogramdevirt pass must have run and lowered
 ; the llvm.type.checked.load call.
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define {i8*, i1} @foo(i8* %ptr) {
diff --git a/llvm/test/LTO/X86/Inputs/codemodel-3.ll b/llvm/test/LTO/X86/Inputs/codemodel-3.ll
index 7b264af6f..a5ec57280 100644
--- a/llvm/test/LTO/X86/Inputs/codemodel-3.ll
+++ b/llvm/test/LTO/X86/Inputs/codemodel-3.ll
@@ -1,5 +1,5 @@
 target triple = "x86_64-unknown-linux-gnu"
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 
 !llvm.module.flags = !{!0, !1}
 
diff --git a/llvm/test/LTO/X86/Inputs/dllimport.ll b/llvm/test/LTO/X86/Inputs/dllimport.ll
index b1914bebf..f3daab1df 100644
--- a/llvm/test/LTO/X86/Inputs/dllimport.ll
+++ b/llvm/test/LTO/X86/Inputs/dllimport.ll
@@ -1,6 +1,6 @@
 ; ModuleID = 'b.obj'
 source_filename = "b.cpp"
-target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:w-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-pc-windows-msvc19.11.0"
 
 ; Function Attrs: norecurse nounwind readnone sspstrong uwtable
diff --git a/llvm/test/LTO/X86/Inputs/remangle_intrinsics_tbaa.ll b/llvm/test/LTO/X86/Inputs/remangle_intrinsics_tbaa.ll
index b216ed04c..e213c2155 100644
--- a/llvm/test/LTO/X86/Inputs/remangle_intrinsics_tbaa.ll
+++ b/llvm/test/LTO/X86/Inputs/remangle_intrinsics_tbaa.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.11.0"
 
 %some_named_struct = type { i8, i8 }
diff --git a/llvm/test/LTO/X86/Inputs/start-lib1.ll b/llvm/test/LTO/X86/Inputs/start-lib1.ll
new file mode 100644
index 000000000..9f42e6aff
--- /dev/null
+++ b/llvm/test/LTO/X86/Inputs/start-lib1.ll
@@ -0,0 +1,8 @@
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare void @bar()
+
+define void @foo() {
+  ret void
+}
diff --git a/llvm/test/LTO/X86/Inputs/start-lib2.ll b/llvm/test/LTO/X86/Inputs/start-lib2.ll
new file mode 100644
index 000000000..68b3c8362
--- /dev/null
+++ b/llvm/test/LTO/X86/Inputs/start-lib2.ll
@@ -0,0 +1,6 @@
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @bar() {
+  ret void
+}
diff --git a/llvm/test/LTO/X86/Inputs/strip-debug-info-bar.ll b/llvm/test/LTO/X86/Inputs/strip-debug-info-bar.ll
index 426988667..be4322caa 100644
--- a/llvm/test/LTO/X86/Inputs/strip-debug-info-bar.ll
+++ b/llvm/test/LTO/X86/Inputs/strip-debug-info-bar.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.12"
 
 define void @bar() !dbg !3 {
diff --git a/llvm/test/LTO/X86/Inputs/type-mapping-bug2.ll b/llvm/test/LTO/X86/Inputs/type-mapping-bug2.ll
index b58a1e953..e866cb712 100644
--- a/llvm/test/LTO/X86/Inputs/type-mapping-bug2.ll
+++ b/llvm/test/LTO/X86/Inputs/type-mapping-bug2.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define i32 @c() !dbg !6 {
diff --git a/llvm/test/LTO/X86/Inputs/type-mapping-bug3.ll b/llvm/test/LTO/X86/Inputs/type-mapping-bug3.ll
new file mode 100644
index 000000000..d6fe8e83a
--- /dev/null
+++ b/llvm/test/LTO/X86/Inputs/type-mapping-bug3.ll
@@ -0,0 +1,26 @@
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; T2 is the non-opaque struct required to trigger the uniqued T2.0 and T3.0 to
+; respectively T2 and T3 in the destination module.
+%"T2" = type { %"T3"* }
+%"T3" = type opaque
+
+; Use/refer to T2 so it gets added as an IdentifiedStructType.  The debug
+; reference to !6 is required to transitively load the metadata !5.
+define void @a(%"T2") !dbg !6 {
+  ret void
+}
+
+!llvm.module.flags = !{!0, !1}
+!llvm.dbg.cu = !{!2}
+!0 = !{i32 1, !"ThinLTO", i32 0}
+!1 = !{i32 2, !"Debug Info Version", i32 3}
+!2 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !3, retainedTypes: !4)
+!3 = !DIFile(filename: "f2", directory: "")
+!4 = !{!5}
+
+; This DICompositeType refers to !5 in type-mapping-bug3.ll
+!5 = !DICompositeType(tag: DW_TAG_structure_type, flags: DIFlagFwdDecl, identifier: "SHARED")
+
+!6 = distinct !DISubprogram(unit: !2)
diff --git a/llvm/test/LTO/X86/codemodel-1.ll b/llvm/test/LTO/X86/codemodel-1.ll
index ca89c9433..2c54ac31c 100644
--- a/llvm/test/LTO/X86/codemodel-1.ll
+++ b/llvm/test/LTO/X86/codemodel-1.ll
@@ -3,7 +3,7 @@
 ; RUN: llvm-objdump -d %t.s.0 | FileCheck %s --check-prefix=CHECK-SMALL
 
 target triple = "x86_64-unknown-linux-gnu"
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 
 !llvm.module.flags = !{!0, !1}
 
diff --git a/llvm/test/LTO/X86/codemodel-2.ll b/llvm/test/LTO/X86/codemodel-2.ll
index 408c02a9e..1b61747c2 100644
--- a/llvm/test/LTO/X86/codemodel-2.ll
+++ b/llvm/test/LTO/X86/codemodel-2.ll
@@ -3,7 +3,7 @@
 ; RUN: llvm-objdump -d %t.s.0 | FileCheck %s --check-prefix=CHECK-LARGE
 
 target triple = "x86_64-unknown-linux-gnu"
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 
 !llvm.module.flags = !{!0, !1}
 
diff --git a/llvm/test/LTO/X86/codemodel-3.ll b/llvm/test/LTO/X86/codemodel-3.ll
index 987d37d22..9e03f84b6 100644
--- a/llvm/test/LTO/X86/codemodel-3.ll
+++ b/llvm/test/LTO/X86/codemodel-3.ll
@@ -3,7 +3,7 @@
 ; RUN: not llvm-lto2 run -r %t0.o,_start,px -r %t1.o,bar,px %t0.o %t1.o -o %t2.s 2>&1 | FileCheck %s 
 
 target triple = "x86_64-unknown-linux-gnu"
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 
 !llvm.module.flags = !{!0, !1}
 
diff --git a/llvm/test/LTO/X86/current-section.ll b/llvm/test/LTO/X86/current-section.ll
index 69f8bfaac..a54f09d85 100644
--- a/llvm/test/LTO/X86/current-section.ll
+++ b/llvm/test/LTO/X86/current-section.ll
@@ -2,7 +2,7 @@
 ; RUN: llvm-lto -o %t2 %t1
 ; REQUIRES: default_triple
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 module asm ".align 4"
diff --git a/llvm/test/LTO/X86/diagnostic-handler-remarks-with-hotness.ll b/llvm/test/LTO/X86/diagnostic-handler-remarks-with-hotness.ll
index d1dddf1a7..56ed420ff 100644
--- a/llvm/test/LTO/X86/diagnostic-handler-remarks-with-hotness.ll
+++ b/llvm/test/LTO/X86/diagnostic-handler-remarks-with-hotness.ll
@@ -25,7 +25,7 @@
 ; YAML-NEXT:   - String:          ')'
 ; YAML-NEXT: ...
 
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-darwin"
 
 declare i32 @bar()
diff --git a/llvm/test/LTO/X86/diagnostic-handler-remarks.ll b/llvm/test/LTO/X86/diagnostic-handler-remarks.ll
index 94f88ec43..6af6d8637 100644
--- a/llvm/test/LTO/X86/diagnostic-handler-remarks.ll
+++ b/llvm/test/LTO/X86/diagnostic-handler-remarks.ll
@@ -61,7 +61,7 @@
 ; YAML-NEXT:   - String:          ')'
 ; YAML-NEXT: ...
 
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-darwin"
 
 declare i32 @bar()
diff --git a/llvm/test/LTO/X86/disable-verify.ll b/llvm/test/LTO/X86/disable-verify.ll
index e66ebac05..68c2c9dd2 100644
--- a/llvm/test/LTO/X86/disable-verify.ll
+++ b/llvm/test/LTO/X86/disable-verify.ll
@@ -2,7 +2,7 @@
 ; RUN: llvm-lto -debug-pass=Arguments -exported-symbol=_f -o /dev/null %t.bc 2>&1 -disable-verify | FileCheck %s
 ; RUN: llvm-lto -debug-pass=Arguments -exported-symbol=_f -o /dev/null %t.bc 2>&1 | FileCheck %s -check-prefix=VERIFY
 
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.10.0"
 
 ; -disable-verify should disable verification from the optimization pipeline.
diff --git a/llvm/test/LTO/X86/dllimport.ll b/llvm/test/LTO/X86/dllimport.ll
index 4e2666817..eb9f7df5c 100644
--- a/llvm/test/LTO/X86/dllimport.ll
+++ b/llvm/test/LTO/X86/dllimport.ll
@@ -9,7 +9,7 @@
 
 ; ModuleID = 'a.obj'
 source_filename = "a.cpp"
-target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:w-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-pc-windows-msvc19.11.0"
 
 ; Function Attrs: norecurse nounwind sspstrong uwtable
diff --git a/llvm/test/LTO/X86/embed-bitcode.ll b/llvm/test/LTO/X86/embed-bitcode.ll
new file mode 100644
index 000000000..f57ea1fa3
--- /dev/null
+++ b/llvm/test/LTO/X86/embed-bitcode.ll
@@ -0,0 +1,28 @@
+; RUN: llvm-as %s -o %t1.o
+; RUN: llvm-as %p/Inputs/start-lib1.ll -o %t2.o
+; RUN: llvm-as %p/Inputs/start-lib2.ll -o %t3.o
+
+; RUN: llvm-lto2 run -r %t1.o,_start,px -r %t2.o,foo,px -r %t3.o,bar,px -r %t2.o,bar,lx -o %t3 %t1.o %t2.o %t3.o
+; RUN: llvm-readelf -S %t3.0 | FileCheck %s --implicit-check-not=.llvmbc
+
+; RUN: llvm-lto2 run -r %t1.o,_start,px -r %t2.o,foo,px -r %t3.o,bar,px -r %t2.o,bar,lx -lto-embed-bitcode=false -o %t3 %t1.o %t2.o %t3.o
+; RUN: llvm-readelf -S %t3.0 | FileCheck %s --implicit-check-not=.llvmbc
+
+; RUN: llvm-lto2 run -r %t1.o,_start,px -r %t2.o,foo,px -r %t3.o,bar,px -r %t2.o,bar,lx -lto-embed-bitcode -o %t3 %t1.o %t2.o %t3.o
+; RUN: llvm-readelf -S %t3.0 | FileCheck %s --check-prefix=CHECK-ELF
+; RUN: llvm-objcopy -O binary -j .llvmbc %t3.0 %t-embedded.bc
+; RUN: llvm-dis %t-embedded.bc -o - | FileCheck %s --check-prefix=CHECK-LL
+
+; CHECK-ELF: .text
+; CHECK-ELF: .llvmbc
+
+; CHECK-LL: @_start
+; CHECK-LL: @foo
+; CHECK-LL: @bar
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @_start() {
+  ret void
+}
diff --git a/llvm/test/LTO/X86/internalize.ll b/llvm/test/LTO/X86/internalize.ll
index 6b18aa7de..aa6aa652c 100644
--- a/llvm/test/LTO/X86/internalize.ll
+++ b/llvm/test/LTO/X86/internalize.ll
@@ -30,7 +30,7 @@
 ; INTERNALIZE2-OPTION-DISABLE: define dso_local void @foo
 ; INTERNALIZE2-OPTION-DISABLE: define dso_local void @bar
 
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.11.0"
 
 define void @foo() {
diff --git a/llvm/test/LTO/X86/no-undefined-puts-when-implemented.ll b/llvm/test/LTO/X86/no-undefined-puts-when-implemented.ll
index 51e5eabd0..34d2544db 100644
--- a/llvm/test/LTO/X86/no-undefined-puts-when-implemented.ll
+++ b/llvm/test/LTO/X86/no-undefined-puts-when-implemented.ll
@@ -5,7 +5,7 @@
 ; rdar://problem/16165191
 ; runtime library implementations should not be renamed
 
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-darwin11"
 
 @str = private unnamed_addr constant [13 x i8] c"hello world\0A\00"
diff --git a/llvm/test/LTO/X86/objc-detection-i386.ll b/llvm/test/LTO/X86/objc-detection-i386.ll
index d5da1469b..18f2bd9ea 100644
--- a/llvm/test/LTO/X86/objc-detection-i386.ll
+++ b/llvm/test/LTO/X86/objc-detection-i386.ll
@@ -45,7 +45,7 @@ define internal void @"\01-[A(foo) foo_myStuff]"(%0*, i8*) #0 {
   ret void
 }
 
-attributes #0 = { nounwind ssp "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+fxsr,+mmx,+sse,+sse2,+sse3,+sse4.1,+ssse3" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind ssp "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+fxsr,+mmx,+sse,+sse2,+sse3,+sse4.1,+ssse3" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.module.flags = !{!0, !1, !2, !3, !4, !5}
 !llvm.ident = !{!6}
diff --git a/llvm/test/LTO/X86/objc-detection.ll b/llvm/test/LTO/X86/objc-detection.ll
index cdae8ab38..6e96b9b43 100644
--- a/llvm/test/LTO/X86/objc-detection.ll
+++ b/llvm/test/LTO/X86/objc-detection.ll
@@ -3,7 +3,7 @@
 
 ; CHECK: contains ObjC
 
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.12.0"
 
 %0 = type opaque
@@ -38,7 +38,7 @@ define internal void @"\01-[A(foo) foo_myStuff]"(%0*, i8*) #0 {
   ret void
 }
 
-attributes #0 = { ssp uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+fxsr,+mmx,+sse,+sse2,+sse3,+sse4.1,+ssse3" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { ssp uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+fxsr,+mmx,+sse,+sse2,+sse3,+sse4.1,+ssse3" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.module.flags = !{!0, !1, !2, !3, !4, !5}
 !llvm.ident = !{!6}
diff --git a/llvm/test/LTO/X86/parallel.ll b/llvm/test/LTO/X86/parallel.ll
index 9a8494c12..b3c128193 100644
--- a/llvm/test/LTO/X86/parallel.ll
+++ b/llvm/test/LTO/X86/parallel.ll
@@ -3,7 +3,7 @@
 ; RUN: llvm-nm %t.o.0 | FileCheck --check-prefix=CHECK0 %s
 ; RUN: llvm-nm %t.o.1 | FileCheck --check-prefix=CHECK1 %s
 
-; FIXME: Investigate test failures on these architecures.
+; FIXME: Investigate test failures on these architectures.
 ; UNSUPPORTED: mips, mipsel, aarch64, powerpc64
 
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/LTO/X86/pr38046.ll b/llvm/test/LTO/X86/pr38046.ll
index 6743121b7..be5b88c59 100644
--- a/llvm/test/LTO/X86/pr38046.ll
+++ b/llvm/test/LTO/X86/pr38046.ll
@@ -7,7 +7,7 @@
 
 ; CHECK-NOT: Global is external, but doesn't have external or weak linkage
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define i32 @foo() {
diff --git a/llvm/test/LTO/X86/private-symbol.ll b/llvm/test/LTO/X86/private-symbol.ll
index e13a39344..3f705fc84 100644
--- a/llvm/test/LTO/X86/private-symbol.ll
+++ b/llvm/test/LTO/X86/private-symbol.ll
@@ -1,7 +1,7 @@
 ; RUN: llvm-as < %s >%t1
 ; RUN: llvm-lto -o %t2 %t1
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @.str = private unnamed_addr constant [7 x i8] c"hello\0A\00", align 1
diff --git a/llvm/test/LTO/X86/remangle_intrinsics_tbaa.ll b/llvm/test/LTO/X86/remangle_intrinsics_tbaa.ll
index cac72f433..30c27dfd5 100644
--- a/llvm/test/LTO/X86/remangle_intrinsics_tbaa.ll
+++ b/llvm/test/LTO/X86/remangle_intrinsics_tbaa.ll
@@ -5,7 +5,7 @@
 ; Verify that we correctly rename the intrinsic and don't crash
 ; CHECK: @llvm.masked.store.v4p0s_some_named_struct.0s.p0v4p0s_some_named_struct.0s
 
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.11.0"
 
 %some_named_struct = type { i8 }
diff --git a/llvm/test/LTO/X86/set-merged.ll b/llvm/test/LTO/X86/set-merged.ll
index e6cc51fcf..56cbb4b2d 100644
--- a/llvm/test/LTO/X86/set-merged.ll
+++ b/llvm/test/LTO/X86/set-merged.ll
@@ -2,7 +2,7 @@
 ; RUN: llvm-lto -exported-symbol=_main -set-merged-module -o %t2 %t1
 ; RUN: llvm-objdump --print-imm-hex=false -d %t2 | FileCheck %s
 
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.10.0"
 
 ; CHECK: _main
diff --git a/llvm/test/LTO/X86/strip-debug-info-no-call-loc.ll b/llvm/test/LTO/X86/strip-debug-info-no-call-loc.ll
index 6ef469744..00508aeaf 100644
--- a/llvm/test/LTO/X86/strip-debug-info-no-call-loc.ll
+++ b/llvm/test/LTO/X86/strip-debug-info-no-call-loc.ll
@@ -9,7 +9,7 @@
 
 ; CHECK-WARN: warning{{.*}} ignoring invalid debug info
 ; CHECK: {{f$}}
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx"
 
 define void @h() #0 !dbg !7 {
diff --git a/llvm/test/LTO/X86/strip-debug-info.ll b/llvm/test/LTO/X86/strip-debug-info.ll
index f8db5a15f..32a721dbe 100644
--- a/llvm/test/LTO/X86/strip-debug-info.ll
+++ b/llvm/test/LTO/X86/strip-debug-info.ll
@@ -29,7 +29,7 @@
 ; CHECK-WARN: warning{{.*}} ignoring invalid debug info
 ; CHECK-WARN-NOT: Broken module found
 ; CHECK: foo
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.12"
 
 declare void @bar()
diff --git a/llvm/test/LTO/X86/symver-asm.ll b/llvm/test/LTO/X86/symver-asm.ll
index 85d032ba1..0cb3d661d 100644
--- a/llvm/test/LTO/X86/symver-asm.ll
+++ b/llvm/test/LTO/X86/symver-asm.ll
@@ -5,7 +5,7 @@
 ; RUN: llvm-nm %t3.0 | FileCheck %s
 ; RUN: llvm-dis %t3.0.2.internalize.bc -o - | FileCheck %s --check-prefix=INTERN
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 module asm ".symver io_cancel_0_4,io_cancel@@LIBAIO_0.4"
diff --git a/llvm/test/LTO/X86/symver-asm2.ll b/llvm/test/LTO/X86/symver-asm2.ll
index 42d6e54bd..5eca39d6f 100644
--- a/llvm/test/LTO/X86/symver-asm2.ll
+++ b/llvm/test/LTO/X86/symver-asm2.ll
@@ -9,7 +9,7 @@
 ; RUN: llvm-lto2 run -r %t1,_start,plx -r %t1,_start3,plx -r %t1,foo@@SOME_VERSION -r %t1,foo@SOME_VERSION3 -o %t3 %t1 -save-temps
 ; RUN: llvm-nm %t3.0 | FileCheck %s
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 module asm ".global _start"
diff --git a/llvm/test/LTO/X86/symver-asm3.ll b/llvm/test/LTO/X86/symver-asm3.ll
index 0ae8f88c4..c64a9beb8 100644
--- a/llvm/test/LTO/X86/symver-asm3.ll
+++ b/llvm/test/LTO/X86/symver-asm3.ll
@@ -3,7 +3,7 @@
 ; RUN: llvm-as < %s >%t1
 ; RUN: llvm-nm %t1 | FileCheck %s
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 module asm "foo1:"
diff --git a/llvm/test/LTO/X86/tailcallelim.ll b/llvm/test/LTO/X86/tailcallelim.ll
index 2c3a9b52f..794e6031e 100644
--- a/llvm/test/LTO/X86/tailcallelim.ll
+++ b/llvm/test/LTO/X86/tailcallelim.ll
@@ -10,7 +10,7 @@
 ; RUN: llvm-lto2 run -r %t1,foo,plx -r %t1,bar,plx -o %t4 %t1 -save-temps -use-new-pm
 ; RUN: llvm-dis < %t4.0.4.opt.bc | FileCheck %s
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define void @foo() {
diff --git a/llvm/test/LTO/X86/triple-init.ll b/llvm/test/LTO/X86/triple-init.ll
index 4c45dd1e7..397587380 100644
--- a/llvm/test/LTO/X86/triple-init.ll
+++ b/llvm/test/LTO/X86/triple-init.ll
@@ -11,7 +11,7 @@
 ; RUN: llvm-lto2 run -r %t1,_pow, -r %t1,_main,plx -o %t2 %t1
 ; RUN: llvm-nm %t2.1 | FileCheck %s
 
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.9"
 
 declare double @pow(double, double)
diff --git a/llvm/test/LTO/X86/triple-init2.ll b/llvm/test/LTO/X86/triple-init2.ll
index a6ff12d11..9e9149847 100644
--- a/llvm/test/LTO/X86/triple-init2.ll
+++ b/llvm/test/LTO/X86/triple-init2.ll
@@ -15,7 +15,7 @@
 ; transformation on Windows.
 ; CHECK: U exp2f
 
-target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:w-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-pc-windows-msvc19.11.0"
 
 define dso_local i32 @main(i32 %argc, i8** nocapture readnone %argv) local_unnamed_addr {
diff --git a/llvm/test/LTO/X86/type-mapping-bug2.ll b/llvm/test/LTO/X86/type-mapping-bug2.ll
index f14183f50..967b68fa3 100644
--- a/llvm/test/LTO/X86/type-mapping-bug2.ll
+++ b/llvm/test/LTO/X86/type-mapping-bug2.ll
@@ -4,7 +4,7 @@
 ;
 ; Test for the issue described in https://bugs.llvm.org/show_bug.cgi?id=37684
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 ; T1 will be linked against T2 because T2 was already loaded in %t0.o due to
diff --git a/llvm/test/LTO/X86/type-mapping-bug3.ll b/llvm/test/LTO/X86/type-mapping-bug3.ll
new file mode 100644
index 000000000..2f845a56a
--- /dev/null
+++ b/llvm/test/LTO/X86/type-mapping-bug3.ll
@@ -0,0 +1,47 @@
+; RUN: opt -module-summary -o %t0.o %S/Inputs/type-mapping-bug3.ll
+; RUN: opt -module-summary -o %t1.o %s
+; RUN: llvm-lto2 run -o %t2 %t0.o %t1.o -r %t0.o,a,px -r %t1.o,b,px -r %t1.o,c,px -r %t1.o,d,
+;
+; Test for the issue described in https://bugs.llvm.org/show_bug.cgi?id=40312
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; T2 is the non-opaque struct required to trigger the uniqued T2.0 and T3.0 to
+; respectively T2 and T3 in the destination module.
+%"T2" = type { %"T3"* }
+%"T3" = type opaque
+
+; Use/refer to T2 so it gets added as an IdentifiedStructType.
+define void @c(%"T2") {
+    unreachable
+}
+
+; The global declaration that causes the assertion when its type is mapped to
+; itself incorrectly.
+declare void @d(%"T3"*)
+
+define void @b() {
+entry:
+  %f.addr = alloca %"T3"*load %"T3"*, %"T3"** %f.addr
+
+  ; The call with the getCalledValue() vs getCalledFunction() mismatch.
+  call void @d(%"T3"* %0)
+  unreachable
+}
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!0, !1}
+!0 = !{i32 1, !"ThinLTO", i32 0}
+!1 = !{i32 2, !"Debug Info Version", i32 3}
+!2 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !3, retainedTypes: !4)
+!3 = !DIFile(filename: "f1", directory: "")
+!4 = !{!5}
+
+; This DICompositeType is referenced by !5 in Inputs/type-mapping-bug3.ll
+; causing the function type in !7 to be added to its module.
+!5 = !DICompositeType(tag: DW_TAG_structure_type, templateParams: !6, identifier: "SHARED")
+!6 = !{!7}
+
+; The reference to d and T3 that gets loaded into %t0.o
+!7 = !DITemplateValueParameter(value: void (%"T3"*)* @d)
diff --git a/llvm/test/Linker/Inputs/module-max-warn.ll b/llvm/test/Linker/Inputs/module-max-warn.ll
new file mode 100644
index 000000000..2d6a1004d
--- /dev/null
+++ b/llvm/test/Linker/Inputs/module-max-warn.ll
@@ -0,0 +1,3 @@
+!llvm.module.flags = !{!0, !1}
+!0 = !{i32 2, !"Combine Max and Warn", i32 4}
+!1 = !{i32 7, !"Combine Warn and Max", i32 5}
diff --git a/llvm/test/Linker/Inputs/thinlto_funcimport_debug.ll b/llvm/test/Linker/Inputs/thinlto_funcimport_debug.ll
index a5fab0cdd..132361aeb 100644
--- a/llvm/test/Linker/Inputs/thinlto_funcimport_debug.ll
+++ b/llvm/test/Linker/Inputs/thinlto_funcimport_debug.ll
@@ -14,8 +14,8 @@ declare i32 @func1(i32) #1
 
 declare i32 @func2(i32) #1
 
-attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/Linker/Inputs/type-unique-inheritance-a.ll b/llvm/test/Linker/Inputs/type-unique-inheritance-a.ll
index 4c8502623..0f6b98072 100644
--- a/llvm/test/Linker/Inputs/type-unique-inheritance-a.ll
+++ b/llvm/test/Linker/Inputs/type-unique-inheritance-a.ll
@@ -60,7 +60,7 @@ entry:
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
 
-attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/Linker/Inputs/type-unique-inheritance-b.ll b/llvm/test/Linker/Inputs/type-unique-inheritance-b.ll
index 1684a2ebc..6ae2da5cd 100644
--- a/llvm/test/Linker/Inputs/type-unique-inheritance-b.ll
+++ b/llvm/test/Linker/Inputs/type-unique-inheritance-b.ll
@@ -32,10 +32,10 @@ entry:
 
 declare void @_Z1fi(i32) #3
 
-attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
-attributes #2 = { ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #3 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { ssp uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!27, !38}
diff --git a/llvm/test/Linker/Inputs/type-unique-simple2-a.ll b/llvm/test/Linker/Inputs/type-unique-simple2-a.ll
index 35f6ea50a..4f9008e42 100644
--- a/llvm/test/Linker/Inputs/type-unique-simple2-a.ll
+++ b/llvm/test/Linker/Inputs/type-unique-simple2-a.ll
@@ -57,7 +57,7 @@ entry:
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
 
-attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/Linker/Inputs/type-unique-simple2-b.ll b/llvm/test/Linker/Inputs/type-unique-simple2-b.ll
index e81466efc..0e8b46c06 100644
--- a/llvm/test/Linker/Inputs/type-unique-simple2-b.ll
+++ b/llvm/test/Linker/Inputs/type-unique-simple2-b.ll
@@ -28,10 +28,10 @@ entry:
 
 declare void @_Z1fi(i32) #3
 
-attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
-attributes #2 = { ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #3 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { ssp uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!19, !28}
diff --git a/llvm/test/Linker/addrspace.ll b/llvm/test/Linker/addrspace.ll
new file mode 100644
index 000000000..713aa01fc
--- /dev/null
+++ b/llvm/test/Linker/addrspace.ll
@@ -0,0 +1,12 @@
+; RUN: llvm-link %s -S | FileCheck %s
+
+@G = addrspace(2) global i32 256 
+; CHECK: @G = addrspace(2) global i32
+
+@GA = alias i32, i32 addrspace(2)* @G
+; CHECK: @GA = alias i32, i32 addrspace(2)* @G
+
+define void @foo() addrspace(3) {
+; CHECK: define void @foo() addrspace(3)
+  ret void
+}
diff --git a/llvm/test/Linker/comdat16.ll b/llvm/test/Linker/comdat16.ll
index 47b142207..f7143228d 100644
--- a/llvm/test/Linker/comdat16.ll
+++ b/llvm/test/Linker/comdat16.ll
@@ -47,7 +47,7 @@ bb11:
 ; CHECK-DAG: @a23 = alias i32 (i8*), i32 (i8*)* @f1.2{{$}}
 ; CHECK-DAG: @a24 = alias i16, bitcast (i32 (i8*)* @f1.2 to i16*)
 
-; CHECK:      define weak_odr protected i32 @f1(i8*) comdat($c1) {
+; CHECK:      define weak_odr protected i32 @f1(i8* %0) comdat($c1) {
 ; CHECK-NEXT: bb10:
 ; CHECK-NEXT:   br label %bb11{{$}}
 ; CHECK:      bb11:
diff --git a/llvm/test/Linker/comdat8.ll b/llvm/test/Linker/comdat8.ll
index 490f8053c..f7aa3117d 100644
--- a/llvm/test/Linker/comdat8.ll
+++ b/llvm/test/Linker/comdat8.ll
@@ -2,7 +2,7 @@
 
 $c1 = comdat largest
 
-@some_name = private unnamed_addr constant i32 42, comdat($c1)
-@c1 = alias i8, inttoptr (i32 ptrtoint (i32* @some_name to i32) to i8*)
+@some_name = unnamed_addr constant i32 42, comdat($c1)
+@c1 = alias i8, inttoptr (i32 1 to i8*)
 
 ; CHECK: COMDAT key involves incomputable alias size.
diff --git a/llvm/test/Linker/datalayout.ll b/llvm/test/Linker/datalayout.ll
index 55e804e85..8e71e3b84 100644
--- a/llvm/test/Linker/datalayout.ll
+++ b/llvm/test/Linker/datalayout.ll
@@ -1,6 +1,7 @@
-; REQUIRES: shell
-; RUN: llvm-link %s %S/Inputs/datalayout-a.ll -S -o - 2>%t.a.err
-; RUN: (echo foo ;cat %t.a.err) | FileCheck --check-prefix=WARN-A %s
+;   Ensure t.a.err is non-empty.
+; RUN: echo foo > %t.a.err
+; RUN: llvm-link %s %S/Inputs/datalayout-a.ll -S -o - 2>>%t.a.err
+; RUN: FileCheck --check-prefix=WARN-A %s < %t.a.err
 
 ; RUN: llvm-link %s %S/Inputs/datalayout-b.ll -S -o - 2>%t.b.err
 ; RUN: cat %t.b.err | FileCheck --check-prefix=WARN-B %s
diff --git a/llvm/test/Linker/module-max-warn.ll b/llvm/test/Linker/module-max-warn.ll
new file mode 100644
index 000000000..a92cd22f1
--- /dev/null
+++ b/llvm/test/Linker/module-max-warn.ll
@@ -0,0 +1,12 @@
+; RUN: llvm-link %s %p/Inputs/module-max-warn.ll -S -o - 2>&1 | FileCheck %s
+
+; CHECK: warning: linking module flags 'Combine Max and Warn': IDs have conflicting values ('i32 4' from {{.*}}/Inputs/module-max-warn.ll with 'i32 2' from llvm-link)
+; CHECK: warning: linking module flags 'Combine Warn and Max': IDs have conflicting values ('i32 5' from {{.*}}/Inputs/module-max-warn.ll with 'i32 3' from llvm-link)
+
+
+; CHECK: !0 = !{i32 7, !"Combine Max and Warn", i32 4}
+; CHECK: !1 = !{i32 7, !"Combine Warn and Max", i32 5}
+
+!llvm.module.flags = !{!0, !1}
+!0 = !{i32 7, !"Combine Max and Warn", i32 2}
+!1 = !{i32 2, !"Combine Warn and Max", i32 3}
diff --git a/llvm/test/Linker/thinlto_funcimport_debug.ll b/llvm/test/Linker/thinlto_funcimport_debug.ll
index 6328f7020..294b3a773 100644
--- a/llvm/test/Linker/thinlto_funcimport_debug.ll
+++ b/llvm/test/Linker/thinlto_funcimport_debug.ll
@@ -45,7 +45,7 @@ entry:
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.value(metadata, i64, metadata, metadata) #1
 
-attributes #0 = { nounwind readnone uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind readnone uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/Linker/type-unique-simple-a.ll b/llvm/test/Linker/type-unique-simple-a.ll
index c6e6c2c3a..bff3325f2 100644
--- a/llvm/test/Linker/type-unique-simple-a.ll
+++ b/llvm/test/Linker/type-unique-simple-a.ll
@@ -62,7 +62,7 @@ entry:
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
 
-attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/Linker/type-unique-simple-b.ll b/llvm/test/Linker/type-unique-simple-b.ll
index 85dcfcaad..217084e17 100644
--- a/llvm/test/Linker/type-unique-simple-b.ll
+++ b/llvm/test/Linker/type-unique-simple-b.ll
@@ -30,10 +30,10 @@ entry:
 
 declare void @_Z1fi(i32) #3
 
-attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
-attributes #2 = { ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #3 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { ssp uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!17, !26}
diff --git a/llvm/test/Linker/type-unique-type-array-a.ll b/llvm/test/Linker/type-unique-type-array-a.ll
index 010f478cd..4f0cc44b9 100644
--- a/llvm/test/Linker/type-unique-type-array-a.ll
+++ b/llvm/test/Linker/type-unique-type-array-a.ll
@@ -83,9 +83,9 @@ entry:
 ; Function Attrs: nounwind
 declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture readonly, i64, i1) #3
 
-attributes #0 = { ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { ssp uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
-attributes #2 = { nounwind ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { nounwind ssp uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #3 = { nounwind }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/Linker/type-unique-type-array-b.ll b/llvm/test/Linker/type-unique-type-array-b.ll
index 18f06b56a..82084fcf6 100644
--- a/llvm/test/Linker/type-unique-type-array-b.ll
+++ b/llvm/test/Linker/type-unique-type-array-b.ll
@@ -62,9 +62,9 @@ entry:
 ; Function Attrs: nounwind
 declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture readonly, i64, i1) #3
 
-attributes #0 = { ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { ssp uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
-attributes #2 = { nounwind ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { nounwind ssp uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #3 = { nounwind }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/MC/AArch64/coff-debug.ll b/llvm/test/MC/AArch64/coff-debug.ll
index f336ea26d..bb7cc4db3 100644
--- a/llvm/test/MC/AArch64/coff-debug.ll
+++ b/llvm/test/MC/AArch64/coff-debug.ll
@@ -14,7 +14,7 @@ entry:
   ret i32 1, !dbg !11
 }
 
-attributes #0 = { noinline nounwind optnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="generic" "target-features"="+neon" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noinline nounwind optnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="generic" "target-features"="+neon" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!3, !4, !5}
diff --git a/llvm/test/MC/ARM/elf-execute-only-section.ll b/llvm/test/MC/ARM/elf-execute-only-section.ll
index 555294b61..73b088fa4 100644
--- a/llvm/test/MC/ARM/elf-execute-only-section.ll
+++ b/llvm/test/MC/ARM/elf-execute-only-section.ll
@@ -5,8 +5,8 @@
 ; RUN: llc < %s -mtriple=thumbv7m-eabi -mattr=+execute-only -filetype=obj %s -o - | \
 ; RUN: llvm-readelf -S | FileCheck %s
 
-; CHECK-NOT: {{.text[ ]+PROGBITS[ ]+[0-9]+ [0-9]+ [0-9]+ [0-9]+ AX[^p]}}
-; CHECK: {{.text[ ]+PROGBITS[ ]+[0-9]+ [0-9]+ [0-9]+ [0-9]+ AXp}}
+; CHECK-NOT: {{.text[ ]+PROGBITS[ ]+[0-9]+ [0-9]+ [0-9]+ [0-9]+ AX[^y]}}
+; CHECK: {{.text[ ]+PROGBITS[ ]+[0-9]+ [0-9]+ [0-9]+ [0-9]+ AXy}}
 define void @test_func() {
 entry:
   ret void
diff --git a/llvm/test/MC/ARM/inline-asm-srcloc.ll b/llvm/test/MC/ARM/inline-asm-srcloc.ll
index 9fb9c5b4e..d78bc3b9b 100644
--- a/llvm/test/MC/ARM/inline-asm-srcloc.ll
+++ b/llvm/test/MC/ARM/inline-asm-srcloc.ll
@@ -23,7 +23,7 @@ entry:
   ret void
 }
 
-attributes #0 = { noinline nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="cortex-a8" "target-features"="+dsp,+neon,+strict-align,+vfp3" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noinline nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="cortex-a8" "target-features"="+dsp,+neon,+strict-align,+vfp3" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind }
 
 !llvm.module.flags = !{!0, !1}
diff --git a/llvm/test/MC/COFF/cv-compiler-info.ll b/llvm/test/MC/COFF/cv-compiler-info.ll
index 8c533a5ca..d8cfe9973 100644
--- a/llvm/test/MC/COFF/cv-compiler-info.ll
+++ b/llvm/test/MC/COFF/cv-compiler-info.ll
@@ -10,7 +10,7 @@ entry:
   ret i32 42, !dbg !14
 }
 
-attributes #0 = { nounwind sspstrong "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind sspstrong "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!7, !8}
diff --git a/llvm/test/MC/ELF/cfi-version.ll b/llvm/test/MC/ELF/cfi-version.ll
index 1beb9eae8..3c3c9ab21 100644
--- a/llvm/test/MC/ELF/cfi-version.ll
+++ b/llvm/test/MC/ELF/cfi-version.ll
@@ -17,8 +17,8 @@ entry:
 
 declare i32 @bar(...) #1
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!9, !10}
diff --git a/llvm/test/MC/ELF/section-relro.ll b/llvm/test/MC/ELF/section-relro.ll
new file mode 100644
index 000000000..27fc109de
--- /dev/null
+++ b/llvm/test/MC/ELF/section-relro.ll
@@ -0,0 +1,65 @@
+; Tests that data and relro are correctly placed in sections
+; specified by "#pragma clang section"
+; RUN: llc -filetype=obj -mtriple x86_64-unknown-linux %s -o - | llvm-readobj -S -t | FileCheck %s
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux"
+
+@funcs_relro = hidden constant [2 x i32 ()*] [i32 ()* bitcast (i32 (...)* @func1 to i32 ()*), i32 ()* bitcast (i32 (...)* @func2 to i32 ()*)], align 16 #0
+@var_data = hidden global i32 33, align 4 #0
+
+declare i32 @func1(...)
+declare i32 @func2(...)
+
+; Function Attrs: noinline nounwind optnone sspstrong uwtable
+define hidden i32 @foo(i32 %i) {
+entry:
+  %i.addr = alloca i32, align 4
+  store i32 %i, i32* %i.addr, align 4
+  %0 = load i32, i32* %i.addr, align 4
+  %idxprom = sext i32 %0 to i64
+  %arrayidx = getelementptr inbounds [2 x i32 ()*], [2 x i32 ()*]* @funcs_relro, i64 0, i64 %idxprom
+  %1 = load i32 ()*, i32 ()** %arrayidx, align 8
+  %call = call i32 %1()
+  %2 = load i32, i32* @var_data, align 4
+  %add = add nsw i32 %call, %2
+  ret i32 %add
+}
+
+attributes #0 = { "data-section"=".my_data" "relro-section"=".my_relro" "rodata-section"=".my_rodata" }
+
+; CHECK:  Section {
+; CHECK:    Index:
+; CHECK:    Name: .my_rodata
+; CHECK:    Type: SHT_PROGBITS (0x1)
+; CHECK:    Flags [ (0x2)
+; CHECK:      SHF_ALLOC (0x2)
+; CHECK:    ]
+; CHECK:    Size: 16
+; CHECK:  }
+; CHECK:  Section {
+; CHECK:    Index:
+; CHECK:    Name: .my_data
+; CHECK:    Type: SHT_PROGBITS (0x1)
+; CHECK:    Flags [ (0x3)
+; CHECK:      SHF_ALLOC (0x2)
+; CHECK:      SHF_WRITE (0x1)
+; CHECK:    ]
+; CHECK:    Size: 4
+; CHECK:  }
+; CHECK:   Symbol {
+; CHECK:    Name: funcs_relro
+; CHECK:    Value: 0x0
+; CHECK:    Size: 16
+; CHECK:    Binding: Global (0x1)
+; CHECK:    Type: Object (0x1)
+; CHECK:    Section: .my_rodata
+; CHECK:  }
+; CHECK:  Symbol {
+; CHECK:    Name: var_data
+; CHECK:    Value: 0x0
+; CHECK:    Size: 4
+; CHECK:    Binding: Global (0x1)
+; CHECK:    Type: Object (0x1)
+; CHECK:    Section: .my_data
+; CHECK:  }
diff --git a/llvm/test/MC/Hexagon/extended_relocations.ll b/llvm/test/MC/Hexagon/extended_relocations.ll
index dde50bf8d..6c79a0ab7 100644
--- a/llvm/test/MC/Hexagon/extended_relocations.ll
+++ b/llvm/test/MC/Hexagon/extended_relocations.ll
@@ -18,6 +18,6 @@ entry:
   ret i32 0
 }
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
diff --git a/llvm/test/MC/MachO/cstexpr-gotpcrel-32.ll b/llvm/test/MC/MachO/cstexpr-gotpcrel-32.ll
index 9b08ee7b8..3d06301b9 100644
--- a/llvm/test/MC/MachO/cstexpr-gotpcrel-32.ll
+++ b/llvm/test/MC/MachO/cstexpr-gotpcrel-32.ll
@@ -73,7 +73,7 @@ define i32 @t0(i32 %a) {
   ret i32 %x
 }
 
-; Text indirect local symbols.
+; Test indirect local symbols.
 ; CHECK-LABEL: _localindirect
 ; CHECK: .long 65603
 @localindirect = internal constant i32  65603
@@ -85,11 +85,38 @@ define i32 @t0(i32 %a) {
   i32 sub (i32 ptrtoint (i32** @got.localindirect to i32),
            i32 ptrtoint (i32* @localindirectuser to i32))
 
+; Test internal indirect local symbols where the user doesn't see the
+; definition of the other symbols yet.
+
+; We used to check if the symbol is defined and not external to guess if it has
+; local linkage, but that doesn't work if the symbol is defined after. The code
+; should check if the GlobalValue itself has local linkage.
+
+; CHECK-LABEL: _undeflocalindirectuser:
+; CHECK: .long L_undeflocalindirect$non_lazy_ptr-_undeflocalindirectuser
+@undeflocalindirectuser = internal constant
+  i32 sub (i32 ptrtoint (i32** @got.undeflocalindirect to i32),
+           i32 ptrtoint (i32* @undeflocalindirectuser to i32)),
+  section "__TEXT,__const"
+
+; CHECK-LABEL: _undeflocalindirect:
+; CHECK: .long 65603
+@undeflocalindirect = internal constant i32  65603
+@got.undeflocalindirect = private unnamed_addr constant i32* @undeflocalindirect
+
+; CHECK-LABEL: .section __IMPORT,__pointers
+
+; CHECK-LABEL: L_localfoo$non_lazy_ptr:
+; CHECK: .indirect_symbol _localfoo
+; CHECK-NOT: .long _localfoo
+; CHECK-NEXT: .long 0
+
 ; CHECK-LABEL: L_localindirect$non_lazy_ptr:
 ; CHECK: .indirect_symbol _localindirect
 ; CHECK-NOT: .long 0
 ; CHECK-NEXT: .long _localindirect
-define i8* @testRelativeIndirectSymbol() {
-  %1 = bitcast i32* @localindirectuser to i8*
-  ret i8* %1
-}
+
+; CHECK-LABEL: L_undeflocalindirect$non_lazy_ptr:
+; CHECK: .indirect_symbol _undeflocalindirect
+; CHECK-NOT: .long 0
+; CHECK-NEXT: .long _undeflocalindirect
diff --git a/llvm/test/MC/MachO/tlv-bss.ll b/llvm/test/MC/MachO/tlv-bss.ll
index 1ffddd2e1..784036727 100644
--- a/llvm/test/MC/MachO/tlv-bss.ll
+++ b/llvm/test/MC/MachO/tlv-bss.ll
@@ -35,4 +35,4 @@ entry:
   ret i8* %0
 }
 
-attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf"="true" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/MC/WebAssembly/assembler-binary.ll b/llvm/test/MC/WebAssembly/assembler-binary.ll
index 3683d63e4..35fa41111 100644
--- a/llvm/test/MC/WebAssembly/assembler-binary.ll
+++ b/llvm/test/MC/WebAssembly/assembler-binary.ll
@@ -38,12 +38,12 @@ entry:
 ; CHECK-NEXT:   - Type:            TYPE
 ; CHECK-NEXT:     Signatures:
 ; CHECK-NEXT:       - Index:           0
-; CHECK-NEXT:         ReturnType:      NORESULT
 ; CHECK-NEXT:         ParamTypes:
 ; CHECK-NEXT:           - I32
+; CHECK-NEXT:         ReturnTypes:     []
 ; CHECK-NEXT:       - Index:           1
-; CHECK-NEXT:         ReturnType:      NORESULT
 ; CHECK-NEXT:         ParamTypes:      []
+; CHECK-NEXT:         ReturnTypes:     []
 ; CHECK-NEXT:   - Type:            IMPORT
 ; CHECK-NEXT:     Imports:
 ; CHECK-NEXT:       - Module:          env
diff --git a/llvm/test/MC/WebAssembly/comdat.ll b/llvm/test/MC/WebAssembly/comdat.ll
index ecc5e5be2..e50f5bf1c 100644
--- a/llvm/test/MC/WebAssembly/comdat.ll
+++ b/llvm/test/MC/WebAssembly/comdat.ll
@@ -28,8 +28,9 @@ define linkonce_odr i32 @sharedFn() #1 comdat($sharedComdat) {
 ; CHECK-NEXT:   - Type:            TYPE
 ; CHECK-NEXT:     Signatures:
 ; CHECK-NEXT:       - Index:           0
-; CHECK-NEXT:         ReturnType:      I32
-; CHECK-NEXT:         ParamTypes:
+; CHECK-NEXT:         ParamTypes:      []
+; CHECK-NEXT:         ReturnTypes:
+; CHECK-NEXT:           - I32
 ; CHECK-NEXT:   - Type:            IMPORT
 ; CHECK-NEXT:     Imports:
 ; CHECK-NEXT:       - Module:          env
diff --git a/llvm/test/MC/WebAssembly/data-symbol-in-text-section.ll b/llvm/test/MC/WebAssembly/data-symbol-in-text-section.ll
new file mode 100644
index 000000000..7e9b28ac2
--- /dev/null
+++ b/llvm/test/MC/WebAssembly/data-symbol-in-text-section.ll
@@ -0,0 +1,13 @@
+; RUN: not llc -filetype=obj %s -o /dev/null 2>&1 | FileCheck %s
+; CHECK: data symbols must live in a data section: data_symbol
+
+target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
+target triple = "wasm32-unknown-unknown"
+
+@data_symbol = constant [1024 x i32] zeroinitializer, section ".text", align 16
+
+define hidden i32 @main() local_unnamed_addr #0 {
+entry:
+  %0 = load i32, i32* getelementptr inbounds ([1024 x i32], [1024 x i32]* @data_symbol, i32 0, i32 10)
+  ret i32 %0
+}
diff --git a/llvm/test/MC/WebAssembly/debug-info.ll b/llvm/test/MC/WebAssembly/debug-info.ll
index 0d78f97fb..0ee06ffcd 100644
--- a/llvm/test/MC/WebAssembly/debug-info.ll
+++ b/llvm/test/MC/WebAssembly/debug-info.ll
@@ -71,68 +71,62 @@
 ; CHECK-NEXT:  }
 ; CHECK-NEXT:  Section {
 ; CHECK-NEXT:    Type: CUSTOM (0x0)
-; CHECK-NEXT:    Size: 1
-; CHECK-NEXT:    Offset: 511
-; CHECK-NEXT:    Name: .debug_macinfo
-; CHECK-NEXT:  }
-; CHECK-NEXT:  Section {
-; CHECK-NEXT:    Type: CUSTOM (0x0)
 ; CHECK-NEXT:    Size: 42
-; CHECK-NEXT:    Offset: 533
+; CHECK-NEXT:    Offset: 511
 ; CHECK-NEXT:    Name: .debug_pubnames
 ; CHECK-NEXT:  }
 ; CHECK-NEXT:  Section {
 ; CHECK-NEXT:    Type: CUSTOM (0x0)
 ; CHECK-NEXT:    Size: 26
-; CHECK-NEXT:    Offset: 597
+; CHECK-NEXT:    Offset: 575
 ; CHECK-NEXT:    Name: .debug_pubtypes
 ; CHECK-NEXT:  }
 ; CHECK-NEXT:  Section {
 ; CHECK-NEXT:    Type: CUSTOM (0x0)
 ; CHECK-NEXT:    Size: 57
-; CHECK-NEXT:    Offset: 645
+; CHECK-NEXT:    Offset: 623
 ; CHECK-NEXT:    Name: .debug_line
 ; CHECK-NEXT:  }
 ; CHECK-NEXT:  Section {
 ; CHECK-NEXT:    Type: CUSTOM (0x0)
 ; CHECK-NEXT:    Size: 88
-; CHECK-NEXT:    Offset: 720
+; CHECK-NEXT:    Offset: 698
 ; CHECK-NEXT:    Name: linking
 ; CHECK-NEXT:  }
 ; CHECK-NEXT:  Section {
 ; CHECK-NEXT:    Type: CUSTOM (0x0)
 ; CHECK-NEXT:    Size: 9
-; CHECK-NEXT:    Offset: 822
+; CHECK-NEXT:    Offset: 800
 ; CHECK-NEXT:    Name: reloc.DATA
 ; CHECK-NEXT:  }
 ; CHECK-NEXT:  Section {
 ; CHECK-NEXT:    Type: CUSTOM (0x0)
 ; CHECK-NEXT:    Size: 58
-; CHECK-NEXT:    Offset: 848
+; CHECK-NEXT:    Offset: 826
 ; CHECK-NEXT:    Name: reloc..debug_info
 ; CHECK-NEXT:  }
 ; CHECK-NEXT:  Section {
 ; CHECK-NEXT:    Type: CUSTOM (0x0)
 ; CHECK-NEXT:    Size: 6
-; CHECK-NEXT:    Offset: 930
+; CHECK-NEXT:    Offset: 908
 ; CHECK-NEXT:    Name: reloc..debug_pubnames
 ; CHECK-NEXT:  }
 ; CHECK-NEXT:  Section {
 ; CHECK-NEXT:    Type: CUSTOM (0x0)
 ; CHECK-NEXT:    Size: 6
-; CHECK-NEXT:    Offset: 964
+; CHECK-NEXT:    Offset: 942
 ; CHECK-NEXT:    Name: reloc..debug_pubtypes
 ; CHECK-NEXT:  }
 ; CHECK-NEXT:  Section {
 ; CHECK-NEXT:    Type: CUSTOM (0x0)
 ; CHECK-NEXT:    Size: 6
-; CHECK-NEXT:    Offset: 998
+; CHECK-NEXT:    Offset: 976
 ; CHECK-NEXT:    Name: reloc..debug_line
 ; CHECK-NEXT:  }
 ; CHECK-NEXT:  Section {
 ; CHECK-NEXT:    Type: CUSTOM (0x0)
 ; CHECK-NEXT:    Size: 77
-; CHECK-NEXT:    Offset: 1028
+; CHECK-NEXT:    Offset: 1006
 ; CHECK-NEXT:    Name: producers
 ; CHECK-NEXT:  }
 ; CHECK-NEXT:]
@@ -156,13 +150,13 @@
 ; CHECK-NEXT:    0x5B R_WASM_FUNCTION_OFFSET_I32 f2 0
 ; CHECK-NEXT:    0x63 R_WASM_SECTION_OFFSET_I32 .debug_str 118
 ; CHECK-NEXT:  }
-; CHECK-NEXT:  Section (12) .debug_pubnames {
+; CHECK-NEXT:  Section (11) .debug_pubnames {
 ; CHECK-NEXT:    0x6 R_WASM_SECTION_OFFSET_I32 .debug_info 0
 ; CHECK-NEXT:  }
-; CHECK-NEXT:  Section (13) .debug_pubtypes {
+; CHECK-NEXT:  Section (12) .debug_pubtypes {
 ; CHECK-NEXT:    0x6 R_WASM_SECTION_OFFSET_I32 .debug_info 0
 ; CHECK-NEXT:  }
-; CHECK-NEXT:  Section (14) .debug_line {
+; CHECK-NEXT:  Section (13) .debug_line {
 ; CHECK-NEXT:    0x2B R_WASM_FUNCTION_OFFSET_I32 f2 0
 ; CHECK-NEXT:  }
 ; CHECK-NEXT:]
@@ -234,7 +228,7 @@
 ; CHECK-NEXT:    Flags [ (0x2)
 ; CHECK-NEXT:      BINDING_LOCAL (0x2)
 ; CHECK-NEXT:    ]
-; CHECK-NEXT:    ElementIndex: 0xD
+; CHECK-NEXT:    ElementIndex: 0xC
 ; CHECK-NEXT:  }
 ; CHECK-NEXT:]
 
@@ -259,7 +253,7 @@ entry:
   ret void, !dbg !18
 }
 
-attributes #0 = { noinline nounwind optnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="generic" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noinline nounwind optnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="generic" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!2}
 !llvm.module.flags = !{!13, !14, !15}
diff --git a/llvm/test/MC/WebAssembly/dwarfdump.ll b/llvm/test/MC/WebAssembly/dwarfdump.ll
index 0873a1a8f..55a721761 100644
--- a/llvm/test/MC/WebAssembly/dwarfdump.ll
+++ b/llvm/test/MC/WebAssembly/dwarfdump.ll
@@ -68,7 +68,7 @@ entry:
   ret void, !dbg !18
 }
 
-attributes #0 = { noinline nounwind optnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="generic" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noinline nounwind optnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="generic" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!2}
 !llvm.module.flags = !{!13, !14, !15}
diff --git a/llvm/test/MC/WebAssembly/event-section.ll b/llvm/test/MC/WebAssembly/event-section.ll
index 2b5b5bba9..b662c290d 100644
--- a/llvm/test/MC/WebAssembly/event-section.ll
+++ b/llvm/test/MC/WebAssembly/event-section.ll
@@ -19,13 +19,14 @@ define i32 @test_throw1(i8* %p) {
 ; CHECK-NEXT:   - Type:            TYPE
 ; CHECK-NEXT:     Signatures:
 ; CHECK-NEXT:       - Index:           0
-; CHECK-NEXT:         ReturnType:      I32
 ; CHECK-NEXT:         ParamTypes:
 ; CHECK-NEXT:           - I32
+; CHECK-NEXT:         ReturnTypes:
+; CHECK-NEXT:           - I32
 ; CHECK-NEXT:       - Index:           1
-; CHECK-NEXT:         ReturnType:      NORESULT
 ; CHECK-NEXT:         ParamTypes:
 ; CHECK-NEXT:           - I32
+; CHECK-NEXT:         ReturnTypes:      []
 
 ; CHECK:        - Type:            EVENT
 ; CHECK-NEXT:     Events:
diff --git a/llvm/test/MC/WebAssembly/external-func-address.ll b/llvm/test/MC/WebAssembly/external-func-address.ll
index 9d3b33592..9c16c02a4 100644
--- a/llvm/test/MC/WebAssembly/external-func-address.ll
+++ b/llvm/test/MC/WebAssembly/external-func-address.ll
@@ -27,15 +27,16 @@ define void @call(i32) {
 ; CHECK-NEXT:   - Type:            TYPE
 ; CHECK-NEXT:     Signatures:
 ; CHECK-NEXT:       - Index:           0
-; CHECK-NEXT:         ReturnType:      NORESULT
 ; CHECK-NEXT:         ParamTypes:
 ; CHECK-NEXT:           - I32
+; CHECK-NEXT:         ReturnTypes:     []
 ; CHECK-NEXT:       - Index:           1
-; CHECK-NEXT:         ReturnType:      I32
 ; CHECK-NEXT:         ParamTypes:
 ; CHECK-NEXT:           - I32
 ; CHECK-NEXT:           - I32
 ; CHECK-NEXT:           - I32
+; CHECK-NEXT:         ReturnTypes:
+; CHECK-NEXT:           - I32
 ; CHECK:        - Type:            IMPORT
 ; CHECK-NEXT:     Imports:
 ; CHECK:            - Module:          env
diff --git a/llvm/test/MC/WebAssembly/libcall.ll b/llvm/test/MC/WebAssembly/libcall.ll
index b22535838..9f8abba70 100644
--- a/llvm/test/MC/WebAssembly/libcall.ll
+++ b/llvm/test/MC/WebAssembly/libcall.ll
@@ -17,14 +17,15 @@ declare void @llvm.memcpy.p0i8.p0i8.i32(i8* nocapture writeonly, i8* nocapture r
 ; CHECK-NEXT:   - Type:            TYPE
 ; CHECK-NEXT:     Signatures:
 ; CHECK-NEXT:       - Index:           0
-; CHECK-NEXT:         ReturnType:      NORESULT
 ; CHECK-NEXT:         ParamTypes:
 ; CHECK-NEXT:           - I32
 ; CHECK-NEXT:           - I32
+; CHECK-NEXT:         ReturnTypes:     []
 ; CHECK-NEXT:       - Index:           1
-; CHECK-NEXT:         ReturnType:      I32
 ; CHECK-NEXT:         ParamTypes:
 ; CHECK-NEXT:           - I32
 ; CHECK-NEXT:           - I32
 ; CHECK-NEXT:           - I32
+; CHECK-NEXT:         ReturnTypes:
+; CHECK-NEXT:           - I32
 ; CHECK-NEXT:   - Type:            IMPORT
diff --git a/llvm/test/MC/WebAssembly/no-dead-strip.ll b/llvm/test/MC/WebAssembly/no-dead-strip.ll
index 1810c49c7..f35be2c7f 100644
--- a/llvm/test/MC/WebAssembly/no-dead-strip.ll
+++ b/llvm/test/MC/WebAssembly/no-dead-strip.ll
@@ -13,8 +13,8 @@ entry:
 ; CHECK-NEXT:   Symbol {
 ; CHECK-NEXT:     Name: foo
 ; CHECK-NEXT:     Type: FUNCTION (0x0)
-; CHECK-NEXT:     Flags [ (0x20)
-; CHECK-NEXT:       EXPORTED (0x20)
+; CHECK-NEXT:     Flags [ (0x80)
+; CHECK-NEXT:       NO_STRIP (0x80)
 ; CHECK-NEXT:     ]
 ; CHECK-NEXT:     ElementIndex: 0x0
 ; CHECK-NEXT:   }
diff --git a/llvm/test/MC/WebAssembly/types.ll b/llvm/test/MC/WebAssembly/types.ll
index 6eeeef2f5..c049d3ce0 100644
--- a/llvm/test/MC/WebAssembly/types.ll
+++ b/llvm/test/MC/WebAssembly/types.ll
@@ -37,32 +37,37 @@ define void @vararg(i32, i32, ...) {
 ; CHECK-LABEL: - Type: TYPE
 ; CHECK-NEXT:    Signatures:
 ; CHECK-NEXT:       - Index: 0
-; CHECK-NEXT:         ReturnType: NORESULT
 ; CHECK-NEXT:         ParamTypes:
+; CHECK-NEXT:         ReturnTypes: []
 ; CHECK-NEXT:       - Index: 1
-; CHECK-NEXT:         ReturnType: I32
 ; CHECK-NEXT:         ParamTypes:
+; CHECK-NEXT:         ReturnTypes:
+; CHECK-NEXT:           - I32
 ; CHECK-NEXT:       - Index: 2
-; CHECK-NEXT:         ReturnType: I64
 ; CHECK-NEXT:         ParamTypes:
+; CHECK-NEXT:         ReturnTypes:
+; CHECK-NEXT:           - I64
 ; CHECK-NEXT:       - Index: 3
-; CHECK-NEXT:         ReturnType: F32
 ; CHECK-NEXT:         ParamTypes:
+; CHECK-NEXT:         ReturnTypes:
+; CHECK-NEXT:           - F32
 ; CHECK-NEXT:       - Index: 4
-; CHECK-NEXT:         ReturnType: F64
 ; CHECK-NEXT:         ParamTypes:
+; CHECK-NEXT:         ReturnTypes:
+; CHECK-NEXT:           - F64
 ; CHECK-NEXT:       - Index: 5
-; CHECK-NEXT:         ReturnType: V128
 ; CHECK-NEXT:         ParamTypes:
+; CHECK-NEXT:         ReturnTypes:
+; CHECK-NEXT:           - V128
 ; CHECK-NEXT:       - Index: 6
-; CHECK-NEXT:         ReturnType: NORESULT
 ; CHECK-NEXT:         ParamTypes:
 ; CHECK-NEXT:           - I32
+; CHECK-NEXT:         ReturnTypes: []
 ; CHECK-NEXT:       - Index: 7
-; CHECK-NEXT:         ReturnType: NORESULT
 ; CHECK-NEXT:         ParamTypes:
 ; CHECK-NEXT:           - I32
 ; CHECK-NEXT:           - I32
 ; CHECK-NEXT:           - I32
+; CHECK-NEXT:         ReturnTypes: []
 ; should be no additional types
-; CHECK-NOT: ReturnType
+; CHECK-NOT: ReturnTypes
diff --git a/llvm/test/MC/X86/stackmap-nops.ll b/llvm/test/MC/X86/stackmap-nops.ll
index 97cef8bf7..26397d90e 100644
--- a/llvm/test/MC/X86/stackmap-nops.ll
+++ b/llvm/test/MC/X86/stackmap-nops.ll
@@ -1,5 +1,5 @@
-; RUN: llc -mtriple=x86_64-apple-darwin -mcpu=corei7 -frame-pointer=all -filetype=obj %s -o - | llvm-objdump -d - | FileCheck %s
-; RUN: llc -mtriple=x86_64-apple-darwin -mcpu=corei7 -frame-pointer=all -filetype=asm %s -o - | llvm-mc -triple=x86_64-apple-darwin -mcpu=corei7 -filetype=obj - | llvm-objdump -d - | FileCheck %s
+; RUN: llc -mtriple=x86_64-apple-darwin -mcpu=znver1 -frame-pointer=all -filetype=obj %s -o - | llvm-objdump -d - | FileCheck %s
+; RUN: llc -mtriple=x86_64-apple-darwin -mcpu=znver1 -frame-pointer=all -filetype=asm %s -o - | llvm-mc -triple=x86_64-apple-darwin -mcpu=znver1 -filetype=obj - | llvm-objdump -d - | FileCheck %s
 
 define void @nop_test() {
 entry:
diff --git a/llvm/test/Other/2010-05-06-Printer.ll b/llvm/test/Other/2010-05-06-Printer.ll
index 9e7c9cb6a..decd977c3 100644
--- a/llvm/test/Other/2010-05-06-Printer.ll
+++ b/llvm/test/Other/2010-05-06-Printer.ll
@@ -16,6 +16,5 @@ define void @foo(){
 ;ALL: ModuleID =
 
 ;FOO: IR Dump After
-;FOO-EMPTY:
 ;FOO-NEXT: define void @foo()
 ;FOO-NOT: define void @tester
diff --git a/llvm/test/Other/attribute-comment.ll b/llvm/test/Other/attribute-comment.ll
index d12b179ef..eab96e8aa 100644
--- a/llvm/test/Other/attribute-comment.ll
+++ b/llvm/test/Other/attribute-comment.ll
@@ -6,4 +6,4 @@ define void @test1() #0 {
   ret void
 }
 
-attributes #0 = { nounwind ssp "less-precise-fpmad"="false" uwtable "no-frame-pointer-elim"="true" readnone "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind ssp "less-precise-fpmad"="false" uwtable "frame-pointer"="all" readnone "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/Other/cgscc-devirt-iteration.ll b/llvm/test/Other/cgscc-devirt-iteration.ll
index 204754e8f..3fcb332b4 100644
--- a/llvm/test/Other/cgscc-devirt-iteration.ll
+++ b/llvm/test/Other/cgscc-devirt-iteration.ll
@@ -96,7 +96,7 @@ entry:
 }
 
 declare i8* @memcpy(i8*, i8*, i64)
-; CHECK-LABEL: declare i8* @memcpy(
+; CHECK-LABEL: i8* @memcpy(
 
 ; The @test3 function checks that when we refine an indirect call to an
 ; intrinsic we still revisit the SCC pass. This also covers cases where the
diff --git a/llvm/test/Other/cgscc-libcall-update.ll b/llvm/test/Other/cgscc-libcall-update.ll
index 5aa61df6a..43de193a6 100644
--- a/llvm/test/Other/cgscc-libcall-update.ll
+++ b/llvm/test/Other/cgscc-libcall-update.ll
@@ -14,11 +14,12 @@ bb:
   %tmp = alloca [1024 x i8], align 16
   %tmp2 = getelementptr inbounds [1024 x i8], [1024 x i8]* %tmp, i64 0, i64 0
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %tmp2, i8* %arg1, i64 1024, i1 false)
-; CHECK:         call void @llvm.memcpy
+; CHECK:         call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 16 dereferenceable(1024) 
   %tmp3 = call i64 @llvm.objectsize.i64.p0i8(i8* %tmp2, i1 false, i1 true, i1 false)
   %tmp4 = call i8* @__strncpy_chk(i8* %arg2, i8* %tmp2, i64 1023, i64 %tmp3)
 ; CHECK-NOT:     call
-; CHECK:         call i8* @strncpy(i8* %arg2, i8* nonnull %tmp2, i64 1023)
+; CHECK:         call i8* @strncpy(i8* nonnull dereferenceable(1) %arg2, i8* nonnull dereferenceable(1) %tmp2, i64 1023)
+
 ; CHECK-NOT:     call
 
   ret i8* %tmp4
@@ -27,6 +28,7 @@ bb:
 
 define i8* @strncpy(i8* %arg1, i8* %arg2, i64 %size) noinline {
 bb:
+; CHECK:         call i8* @my_special_strncpy(i8* %arg1, i8* %arg2, i64 %size)
   %result = call i8* @my_special_strncpy(i8* %arg1, i8* %arg2, i64 %size)
   ret i8* %result
 }
diff --git a/llvm/test/Other/invariant.group.ll b/llvm/test/Other/invariant.group.ll
index fdb3a009f..3b2cacedb 100644
--- a/llvm/test/Other/invariant.group.ll
+++ b/llvm/test/Other/invariant.group.ll
@@ -92,11 +92,11 @@ declare void @use(i8* readonly)
 declare void @useBool(i1)
 
 declare void @clobber(i8*)
-; CHECK: Function Attrs: inaccessiblememonly nounwind speculatable{{$}}
+; CHECK: Function Attrs: inaccessiblememonly nounwind speculatable willreturn{{$}}
 ; CHECK-NEXT: declare i8* @llvm.launder.invariant.group.p0i8(i8*)
 declare i8* @llvm.launder.invariant.group.p0i8(i8*)
 
-; CHECK: Function Attrs: nounwind readnone speculatable{{$}}
+; CHECK: Function Attrs: nounwind readnone speculatable willreturn{{$}}
 ; CHECK-NEXT: declare i8* @llvm.strip.invariant.group.p0i8(i8*)
 declare i8* @llvm.strip.invariant.group.p0i8(i8*)
 
diff --git a/llvm/test/Other/lint.ll b/llvm/test/Other/lint.ll
index ab5df0ec2..415fc29d8 100644
--- a/llvm/test/Other/lint.ll
+++ b/llvm/test/Other/lint.ll
@@ -9,6 +9,7 @@ declare void @has_noaliases(i32* noalias %p, i32* %q)
 declare void @one_arg(i32)
 
 @CG = constant i32 7
+@CG2 = constant i32 7
 @E = external global i8
 
 define i32 @foo() noreturn {
@@ -78,7 +79,9 @@ define i32 @foo() noreturn {
   call void (float) bitcast (void (i32)* @one_arg to void (float)*)(float 0.0)
 
 ; CHECK: Write to read-only memory
-  call void @llvm.memcpy.p0i8.p0i8.i64(i8* bitcast (i32* @CG to i8*), i8* bitcast (i32* @CG to i8*), i64 1, i1 0)
+call void @llvm.memcpy.p0i8.p0i8.i64(i8* bitcast (i32* @CG to i8*), i8* bitcast (i32* @CG2 to i8*), i64 1, i1 0)
+; CHECK: Unusual: noalias argument aliases another argument
+call void @llvm.memcpy.p0i8.p0i8.i64(i8* bitcast (i32* @CG to i8*), i8* bitcast (i32* @CG to i8*), i64 1, i1 0)
 
 ; CHECK: Undefined behavior: Buffer overflow
   %wider = bitcast i8* %buf to i16*
diff --git a/llvm/test/Other/module-pass-printer.ll b/llvm/test/Other/module-pass-printer.ll
new file mode 100644
index 000000000..9c16cf10a
--- /dev/null
+++ b/llvm/test/Other/module-pass-printer.ll
@@ -0,0 +1,18 @@
+; Check pass name is only printed once.
+; RUN: opt < %s 2>&1 -forceattrs -disable-output -print-after-all | FileCheck %s
+; RUN: opt < %s 2>&1 -forceattrs -disable-output -print-after-all -filter-print-funcs=foo,bar | FileCheck %s
+
+; Check pass name is not printed if a module doesn't include any function specified in -filter-print-funcs.
+; RUN: opt < %s 2>&1 -forceattrs -disable-output -print-after-all -filter-print-funcs=baz | FileCheck %s -allow-empty -check-prefix=EMPTY
+
+; CHECK: *** IR Dump After Force set function attributes ***
+; CHECK-NOT: *** IR Dump After Force set function attributes ***
+; EMPTY-NOT: *** IR Dump After Force set function attributes ***
+
+define void @foo() {
+  ret void
+}
+
+define void @bar() {
+  ret void
+}
diff --git a/llvm/test/Other/new-pass-manager.ll b/llvm/test/Other/new-pass-manager.ll
index e836b3210..d769319ea 100644
--- a/llvm/test/Other/new-pass-manager.ll
+++ b/llvm/test/Other/new-pass-manager.ll
@@ -24,6 +24,7 @@
 ; CHECK-CGSCC-PASS-NEXT: Running analysis: InnerAnalysisManagerProxy<{{.*(FunctionAnalysisManager|AnalysisManager<.*Function.*>).*}},{{.*}}Module>
 ; CHECK-CGSCC-PASS-NEXT: Running analysis: LazyCallGraphAnalysis
 ; CHECK-CGSCC-PASS-NEXT: Running analysis: TargetLibraryAnalysis
+; CHECK-CGSCC-PASS-NEXT: Running analysis: PassInstrumentationAnalysis
 ; CHECK-CGSCC-PASS-NEXT: Running analysis: FunctionAnalysisManagerCGSCCProxy
 ; CHECK-CGSCC-PASS-NEXT: Running analysis: PassInstrumentationAnalysis
 ; CHECK-CGSCC-PASS-NEXT: Running analysis: OuterAnalysisManagerProxy<{{.*}}LazyCallGraph::SCC{{.*}}>
@@ -410,6 +411,7 @@
 ; CHECK-REPEAT-CGSCC-PASS-NEXT: Running analysis: InnerAnalysisManagerProxy<{{.*(FunctionAnalysisManager|AnalysisManager<.*Function.*>).*}},{{.*}}Module>
 ; CHECK-REPEAT-CGSCC-PASS-NEXT: Running analysis: LazyCallGraphAnalysis
 ; CHECK-REPEAT-CGSCC-PASS-NEXT: Running analysis: TargetLibraryAnalysis
+; CHECK-REPEAT-CGSCC-PASS-NEXT: Running analysis: PassInstrumentationAnalysis
 ; CHECK-REPEAT-CGSCC-PASS-NEXT: Running analysis: FunctionAnalysisManagerCGSCCProxy
 ; CHECK-REPEAT-CGSCC-PASS-NEXT: Running analysis: PassInstrumentationAnalysis
 ; CHECK-REPEAT-CGSCC-PASS-NEXT: Running analysis: OuterAnalysisManagerProxy<{{.*}}LazyCallGraph::SCC{{.*}}>
diff --git a/llvm/test/Other/new-pm-defaults.ll b/llvm/test/Other/new-pm-defaults.ll
index 4437ed89a..1dc96ef3a 100644
--- a/llvm/test/Other/new-pm-defaults.ll
+++ b/llvm/test/Other/new-pm-defaults.ll
@@ -9,69 +9,84 @@
 
 ; RUN: opt -disable-verify -debug-pass-manager \
 ; RUN:     -passes='default<O1>' -S %s 2>&1 \
-; RUN:     | FileCheck %s --check-prefix=CHECK-O --check-prefix=CHECK-O1
+; RUN:     | FileCheck %s --check-prefix=CHECK-O --check-prefix=CHECK-O1 \
+; RUN:      --check-prefix=%llvmcheckext
 ; RUN: opt -disable-verify -debug-pass-manager \
 ; RUN:     -passes='default<O2>' -S  %s 2>&1 \
-; RUN:     | FileCheck %s --check-prefix=CHECK-O --check-prefix=CHECK-O2
+; RUN:     | FileCheck %s --check-prefix=CHECK-O --check-prefix=CHECK-O2 \
+; RUN:      --check-prefix=CHECK-O23SZ --check-prefix=%llvmcheckext
 ; RUN: opt -disable-verify -debug-pass-manager \
 ; RUN:     -passes='default<O3>' -S  %s 2>&1 \
-; RUN:     | FileCheck %s --check-prefix=CHECK-O --check-prefix=CHECK-O3
+; RUN:     | FileCheck %s --check-prefix=CHECK-O --check-prefix=CHECK-O3 \
+; RUN:      --check-prefix=CHECK-O23SZ --check-prefix=%llvmcheckext
 ; RUN: opt -disable-verify -debug-pass-manager \
 ; RUN:     -passes='default<Os>' -S %s 2>&1 \
-; RUN:     | FileCheck %s --check-prefix=CHECK-O --check-prefix=CHECK-Os
+; RUN:     | FileCheck %s --check-prefix=CHECK-O --check-prefix=CHECK-Os \
+; RUN:      --check-prefix=CHECK-O23SZ --check-prefix=%llvmcheckext
 ; RUN: opt -disable-verify -debug-pass-manager \
 ; RUN:     -passes='default<Oz>' -S %s 2>&1 \
-; RUN:     | FileCheck %s --check-prefix=CHECK-O --check-prefix=CHECK-Oz
+; RUN:     | FileCheck %s --check-prefix=CHECK-O --check-prefix=CHECK-Oz \
+; RUN:     --check-prefix=CHECK-O23SZ --check-prefix=%llvmcheckext
 ; RUN: opt -disable-verify -debug-pass-manager \
 ; RUN:     -passes='lto-pre-link<O2>' -S %s 2>&1 \
 ; RUN:     | FileCheck %s --check-prefix=CHECK-O --check-prefix=CHECK-O2 \
+; RUN:     --check-prefix=CHECK-O23SZ --check-prefix=%llvmcheckext \
 ; RUN:     --check-prefix=CHECK-O2-LTO
 
 ; RUN: opt -disable-verify -debug-pass-manager \
 ; RUN:     -passes-ep-peephole='no-op-function' \
 ; RUN:     -passes='default<O3>' -S  %s 2>&1 \
 ; RUN:     | FileCheck %s --check-prefix=CHECK-O --check-prefix=CHECK-O3 \
-; RUN:     --check-prefix=CHECK-EP-PEEPHOLE
+; RUN:     --check-prefix=%llvmcheckext \
+; RUN:     --check-prefix=CHECK-EP-PEEPHOLE --check-prefix=CHECK-O23SZ
 ; RUN: opt -disable-verify -debug-pass-manager \
 ; RUN:     -passes-ep-late-loop-optimizations='no-op-loop' \
 ; RUN:     -passes='default<O3>' -S  %s 2>&1 \
 ; RUN:     | FileCheck %s --check-prefix=CHECK-O --check-prefix=CHECK-O3 \
-; RUN:     --check-prefix=CHECK-EP-LOOP-LATE
+; RUN:     --check-prefix=%llvmcheckext \
+; RUN:     --check-prefix=CHECK-EP-LOOP-LATE --check-prefix=CHECK-O23SZ
 ; RUN: opt -disable-verify -debug-pass-manager \
 ; RUN:     -passes-ep-loop-optimizer-end='no-op-loop' \
 ; RUN:     -passes='default<O3>' -S  %s 2>&1 \
 ; RUN:     | FileCheck %s --check-prefix=CHECK-O --check-prefix=CHECK-O3 \
-; RUN:     --check-prefix=CHECK-EP-LOOP-END
+; RUN:     --check-prefix=%llvmcheckext \
+; RUN:     --check-prefix=CHECK-EP-LOOP-END --check-prefix=CHECK-O23SZ
 ; RUN: opt -disable-verify -debug-pass-manager \
 ; RUN:     -passes-ep-scalar-optimizer-late='no-op-function' \
 ; RUN:     -passes='default<O3>' -S  %s 2>&1 \
 ; RUN:     | FileCheck %s --check-prefix=CHECK-O --check-prefix=CHECK-O3 \
-; RUN:     --check-prefix=CHECK-EP-SCALAR-LATE
+; RUN:     --check-prefix=%llvmcheckext \
+; RUN:     --check-prefix=CHECK-EP-SCALAR-LATE --check-prefix=CHECK-O23SZ
 ; RUN: opt -disable-verify -debug-pass-manager \
 ; RUN:     -passes-ep-cgscc-optimizer-late='no-op-cgscc' \
 ; RUN:     -passes='default<O3>' -S  %s 2>&1 \
 ; RUN:     | FileCheck %s --check-prefix=CHECK-O --check-prefix=CHECK-O3 \
-; RUN:     --check-prefix=CHECK-EP-CGSCC-LATE
+; RUN:     --check-prefix=%llvmcheckext \
+; RUN:     --check-prefix=CHECK-EP-CGSCC-LATE --check-prefix=CHECK-O23SZ
 ; RUN: opt -disable-verify -debug-pass-manager \
 ; RUN:     -passes-ep-vectorizer-start='no-op-function' \
 ; RUN:     -passes='default<O3>' -S  %s 2>&1 \
 ; RUN:     | FileCheck %s --check-prefix=CHECK-O --check-prefix=CHECK-O3 \
-; RUN:     --check-prefix=CHECK-EP-VECTORIZER-START
+; RUN:     --check-prefix=%llvmcheckext \
+; RUN:     --check-prefix=CHECK-EP-VECTORIZER-START --check-prefix=CHECK-O23SZ
 ; RUN: opt -disable-verify -debug-pass-manager \
 ; RUN:     -passes-ep-pipeline-start='no-op-module' \
 ; RUN:     -passes='default<O3>' -S  %s 2>&1 \
 ; RUN:     | FileCheck %s --check-prefix=CHECK-O --check-prefix=CHECK-O3 \
-; RUN:     --check-prefix=CHECK-EP-PIPELINE-START
+; RUN:     --check-prefix=%llvmcheckext \
+; RUN:     --check-prefix=CHECK-EP-PIPELINE-START --check-prefix=CHECK-O23SZ
 ; RUN: opt -disable-verify -debug-pass-manager \
 ; RUN:     -passes-ep-pipeline-start='no-op-module' \
 ; RUN:     -passes='lto-pre-link<O3>' -S  %s 2>&1 \
 ; RUN:     | FileCheck %s --check-prefix=CHECK-O --check-prefix=CHECK-O3 \
-; RUN:     --check-prefix=CHECK-EP-PIPELINE-START
+; RUN:     --check-prefix=%llvmcheckext \
+; RUN:     --check-prefix=CHECK-EP-PIPELINE-START --check-prefix=CHECK-O23SZ
 ; RUN: opt -disable-verify -debug-pass-manager \
 ; RUN:     -passes-ep-optimizer-last='no-op-function' \
 ; RUN:     -passes='default<O3>' -S  %s 2>&1 \
 ; RUN:     | FileCheck %s --check-prefix=CHECK-O --check-prefix=CHECK-O3 \
-; RUN:     --check-prefix=CHECK-EP-OPTIMIZER-LAST
+; RUN:     --check-prefix=%llvmcheckext \
+; RUN:     --check-prefix=CHECK-EP-OPTIMIZER-LAST --check-prefix=CHECK-O23SZ
 
 ; CHECK-O: Running analysis: PassInstrumentationAnalysis
 ; CHECK-O-NEXT: Starting llvm::Module pass manager run.
@@ -82,9 +97,10 @@
 ; CHECK-O-NEXT: Running pass: PassManager<{{.*}}Module{{.*}}>
 ; CHECK-O-NEXT: Starting llvm::Module pass manager run.
 ; CHECK-O-NEXT: Running pass: InferFunctionAttrsPass
+; CHECK-O-NEXT: Running analysis: InnerAnalysisManagerProxy
 ; CHECK-O-NEXT: Running analysis: TargetLibraryAnalysis
+; CHECK-O-NEXT: Running analysis: PassInstrumentationAnalysis
 ; CHECK-O-NEXT: Running pass: ModuleToFunctionPassAdaptor<{{.*}}PassManager{{.*}}>
-; CHECK-O-NEXT: Running analysis: InnerAnalysisManagerProxy
 ; CHECK-O-NEXT: Running analysis: PassInstrumentationAnalysis
 ; CHECK-O-NEXT: Starting llvm::Function pass manager run.
 ; CHECK-O-NEXT: Running pass: SimplifyCFGPass
@@ -131,10 +147,10 @@
 ; CHECK-O-NEXT: Running pass: SROA
 ; CHECK-O-NEXT: Running pass: EarlyCSEPass
 ; CHECK-O-NEXT: Running analysis: MemorySSAAnalysis
-; CHECK-O-NEXT: Running pass: SpeculativeExecutionPass
-; CHECK-O-NEXT: Running pass: JumpThreadingPass
-; CHECK-O-NEXT: Running analysis: LazyValueAnalysis
-; CHECK-O-NEXT: Running pass: CorrelatedValuePropagationPass
+; CHECK-O23SZ-NEXT: Running pass: SpeculativeExecutionPass
+; CHECK-O23SZ-NEXT: Running pass: JumpThreadingPass
+; CHECK-O23SZ-NEXT: Running analysis: LazyValueAnalysis
+; CHECK-O23SZ-NEXT: Running pass: CorrelatedValuePropagationPass
 ; CHECK-O-NEXT: Running pass: SimplifyCFGPass
 ; CHECK-O3-NEXT: AggressiveInstCombinePass
 ; CHECK-O-NEXT: Running pass: InstCombinePass
@@ -142,7 +158,7 @@
 ; CHECK-O2-NEXT: Running pass: LibCallsShrinkWrapPass
 ; CHECK-O3-NEXT: Running pass: LibCallsShrinkWrapPass
 ; CHECK-EP-PEEPHOLE-NEXT: Running pass: NoOpFunctionPass
-; CHECK-O-NEXT: Running pass: TailCallElimPass
+; CHECK-O23SZ-NEXT: Running pass: TailCallElimPass
 ; CHECK-O-NEXT: Running pass: SimplifyCFGPass
 ; CHECK-O-NEXT: Running pass: ReassociatePass
 ; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}OptimizationRemarkEmitterAnalysis
@@ -178,22 +194,11 @@
 ; CHECK-O-NEXT: Running pass: LoopFullUnrollPass
 ; CHECK-EP-LOOP-END-NEXT: Running pass: NoOpLoopPass
 ; CHECK-O-NEXT: Finished Loop pass manager run.
-; CHECK-Os-NEXT: Running pass: MergedLoadStoreMotionPass
-; CHECK-Os-NEXT: Running pass: GVN
-; CHECK-Os-NEXT: Running analysis: MemoryDependenceAnalysis
-; CHECK-Os-NEXT: Running analysis: PhiValuesAnalysis
-; CHECK-Oz-NEXT: Running pass: MergedLoadStoreMotionPass
-; CHECK-Oz-NEXT: Running pass: GVN
-; CHECK-Oz-NEXT: Running analysis: MemoryDependenceAnalysis
-; CHECK-Oz-NEXT: Running analysis: PhiValuesAnalysis
-; CHECK-O2-NEXT: Running pass: MergedLoadStoreMotionPass
-; CHECK-O2-NEXT: Running pass: GVN
-; CHECK-O2-NEXT: Running analysis: MemoryDependenceAnalysis
-; CHECK-O2-NEXT: Running analysis: PhiValuesAnalysis
-; CHECK-O3-NEXT: Running pass: MergedLoadStoreMotionPass
-; CHECK-O3-NEXT: Running pass: GVN
-; CHECK-O3-NEXT: Running analysis: MemoryDependenceAnalysis
-; CHECK-O3-NEXT: Running analysis: PhiValuesAnalysis
+; CHECK-O-NEXT: Running pass: SROA on foo
+; CHECK-O23SZ-NEXT: Running pass: MergedLoadStoreMotionPass
+; CHECK-O23SZ-NEXT: Running pass: GVN
+; CHECK-O23SZ-NEXT: Running analysis: MemoryDependenceAnalysis
+; CHECK-O23SZ-NEXT: Running analysis: PhiValuesAnalysis
 ; CHECK-O-NEXT: Running pass: MemCpyOptPass
 ; CHECK-O1-NEXT: Running analysis: MemoryDependenceAnalysis
 ; CHECK-O1-NEXT: Running analysis: PhiValuesAnalysis
@@ -202,14 +207,14 @@
 ; CHECK-O-NEXT: Running analysis: DemandedBitsAnalysis
 ; CHECK-O-NEXT: Running pass: InstCombinePass
 ; CHECK-EP-PEEPHOLE-NEXT: Running pass: NoOpFunctionPass
-; CHECK-O-NEXT: Running pass: JumpThreadingPass
-; CHECK-O-NEXT: Running pass: CorrelatedValuePropagationPass
-; CHECK-O-NEXT: Running pass: DSEPass
-; CHECK-O-NEXT: Running pass: FunctionToLoopPassAdaptor<{{.*}}LICMPass{{.*}}>
-; CHECK-O-NEXT: Starting llvm::Function pass manager run.
-; CHECK-O-NEXT: Running pass: LoopSimplifyPass
-; CHECK-O-NEXT: Running pass: LCSSAPass
-; CHECK-O-NEXT: Finished llvm::Function pass manager run.
+; CHECK-O23SZ-NEXT: Running pass: JumpThreadingPass
+; CHECK-O23SZ-NEXT: Running pass: CorrelatedValuePropagationPass
+; CHECK-O23SZ-NEXT: Running pass: DSEPass
+; CHECK-O23SZ-NEXT: Running pass: FunctionToLoopPassAdaptor<{{.*}}LICMPass{{.*}}>
+; CHECK-O23SZ-NEXT: Starting llvm::Function pass manager run.
+; CHECK-O23SZ-NEXT: Running pass: LoopSimplifyPass
+; CHECK-O23SZ-NEXT: Running pass: LCSSAPass
+; CHECK-O23SZ-NEXT: Finished llvm::Function pass manager run.
 ; CHECK-EP-SCALAR-LATE-NEXT: Running pass: NoOpFunctionPass
 ; CHECK-O-NEXT: Running pass: ADCEPass
 ; CHECK-O-NEXT: Running analysis: PostDominatorTreeAnalysis
@@ -230,7 +235,9 @@
 ; CHECK-O-NEXT: Running pass: ModuleToFunctionPassAdaptor<{{.*}}PassManager{{.*}}>
 ; CHECK-O-NEXT: Starting llvm::Function pass manager run.
 ; CHECK-O-NEXT: Running pass: Float2IntPass
+; CHECK-O-NEXT: Running pass: LowerConstantIntrinsicsPass on foo
 ; CHECK-EP-VECTORIZER-START-NEXT: Running pass: NoOpFunctionPass
+; CHECK-EXT: Running pass: {{.*}}::Bye on foo
 ; CHECK-O-NEXT: Running pass: FunctionToLoopPassAdaptor<{{.*}}LoopRotatePass
 ; CHECK-O-NEXT: Starting llvm::Function pass manager run.
 ; CHECK-O-NEXT: Running pass: LoopSimplifyPass
@@ -244,6 +251,9 @@
 ; CHECK-O-NEXT: Running analysis: LoopAccessAnalysis
 ; CHECK-O-NEXT: Running pass: InstCombinePass
 ; CHECK-O-NEXT: Running pass: SimplifyCFGPass
+; CHECK-O2-NEXT: Running pass: SLPVectorizerPass
+; CHECK-O3-NEXT: Running pass: SLPVectorizerPass
+; CHECK-Os-NEXT: Running pass: SLPVectorizerPass
 ; CHECK-O-NEXT: Running pass: InstCombinePass
 ; CHECK-O-NEXT: Running pass: LoopUnrollPass
 ; CHECK-O-NEXT: Running pass: WarnMissedTransformationsPass
diff --git a/llvm/test/Other/new-pm-lto-defaults.ll b/llvm/test/Other/new-pm-lto-defaults.ll
index 1128ae3a8..4bfee7372 100644
--- a/llvm/test/Other/new-pm-lto-defaults.ll
+++ b/llvm/test/Other/new-pm-lto-defaults.ll
@@ -30,9 +30,11 @@
 ; CHECK-O-NEXT: Running pass: GlobalDCEPass
 ; CHECK-O-NEXT: Running pass: ForceFunctionAttrsPass
 ; CHECK-O-NEXT: Running pass: InferFunctionAttrsPass
+; CHECK-O-NEXT: Running analysis: InnerAnalysisManagerProxy<{{.*}}Module
 ; CHECK-O-NEXT: Running analysis: TargetLibraryAnalysis
+; CHECK-O-NEXT: Running analysis: PassInstrumentationAnalysis
+; CHECK-O1-NEXT: Running pass: ModuleToPostOrderCGSCCPassAdaptor<{{.*}}PostOrderFunctionAttrsPass>
 ; CHECK-O2-NEXT: Running pass: ModuleToFunctionPassAdaptor<{{.*}}PassManager{{.*}}>
-; CHECK-O2-NEXT: Running analysis: InnerAnalysisManagerProxy<{{.*}}Module
 ; CHECK-O2-NEXT: Running analysis: PassInstrumentationAnalysis
 ; CHECK-O2-NEXT: Starting llvm::Function pass manager run.
 ; CHECK-O2-NEXT: Running pass: CallSiteSplittingPass on foo
@@ -46,16 +48,15 @@
 ; CHECK-O2-NEXT: Running pass: IPSCCPPass
 ; CHECK-O2-NEXT: Running analysis: AssumptionAnalysis on foo
 ; CHECK-O2-NEXT: Running pass: CalledValuePropagationPass
-; CHECK-O-NEXT: Running pass: ModuleToPostOrderCGSCCPassAdaptor<{{.*}}PostOrderFunctionAttrsPass>
+; CHECK-O2-NEXT: Running pass: ModuleToPostOrderCGSCCPassAdaptor<{{.*}}PostOrderFunctionAttrsPass>
 ; CHECK-O-NEXT: Running analysis: InnerAnalysisManagerProxy<{{.*}}SCC
-; CHECK-O1-NEXT: Running analysis: InnerAnalysisManagerProxy<{{.*}}Function
 ; CHECK-O-NEXT: Running analysis: LazyCallGraphAnalysis
+; CHECK-O1-NEXT: Running analysis: TargetLibraryAnalysis
+; CHECK-O1-NEXT: Running analysis: PassInstrumentationAnalysis
 ; CHECK-O-NEXT: Running analysis: FunctionAnalysisManagerCGSCCProxy
 ; CHECK-O-NEXT: Running analysis: PassInstrumentationAnalysis
 ; CHECK-O-NEXT: Running analysis: OuterAnalysisManagerProxy<{{.*}}LazyCallGraph{{.*}}>
 ; CHECK-O-NEXT: Running analysis: AAManager
-; CHECK-O1-NEXT: Running analysis: PassInstrumentationAnalysis
-; CHECK-O1-NEXT: Running analysis: TargetLibraryAnalysis
 ; CHECK-O-NEXT: Running pass: ReversePostOrderFunctionAttrsPass
 ; CHECK-O-NEXT: Running analysis: CallGraphAnalysis
 ; CHECK-O-NEXT: Running pass: GlobalSplitPass
diff --git a/llvm/test/Other/new-pm-pgo-O0.ll b/llvm/test/Other/new-pm-pgo-O0.ll
new file mode 100644
index 000000000..6a6da67bb
--- /dev/null
+++ b/llvm/test/Other/new-pm-pgo-O0.ll
@@ -0,0 +1,21 @@
+; RUN: opt -debug-pass-manager -passes='default<O0>' -pgo-kind=pgo-instr-gen-pipeline -profile-file='temp' %s 2>&1 |FileCheck %s --check-prefixes=GEN
+; RUN: llvm-profdata merge %S/Inputs/new-pm-pgo.proftext -o %t.profdata
+; RUN: opt -debug-pass-manager -passes='default<O0>' -pgo-kind=pgo-instr-use-pipeline -profile-file='%t.profdata' %s 2>&1 |FileCheck %s --check-prefixes=USE_DEFAULT,USE
+; RUN: opt -debug-pass-manager -passes='thinlto-pre-link<O0>' -pgo-kind=pgo-instr-use-pipeline -profile-file='%t.profdata' %s 2>&1 \
+; RUN:     |FileCheck %s --check-prefixes=USE_PRE_LINK,USE
+; RUN: opt -debug-pass-manager -passes='lto-pre-link<O0>' -pgo-kind=pgo-instr-use-pipeline -profile-file='%t.profdata' %s 2>&1 \
+; RUN:     |FileCheck %s --check-prefixes=USE_PRE_LINK,USE
+; RUN: opt -debug-pass-manager -passes='thinlto<O0>' -pgo-kind=pgo-instr-use-pipeline -profile-file='%t.profdata' %s 2>&1 \
+; RUN:     |FileCheck %s --check-prefixes=USE_POST_LINK,USE
+
+;
+; GEN: Running pass: PGOInstrumentationGen
+; USE_DEFAULT: Running pass: PGOInstrumentationUse
+; USE_PRE_LINK: Running pass: PGOInstrumentationUse
+; USE_POST_LINK-NOT: Running pass: PGOInstrumentationUse
+; USE-NOT: Running pass: PGOIndirectCallPromotion
+; USE-NOT: Running pass: PGOMemOPSizeOpt
+
+define void @foo() {
+  ret void
+}
diff --git a/llvm/test/Other/new-pm-pr42726-cgscc.ll b/llvm/test/Other/new-pm-pr42726-cgscc.ll
new file mode 100644
index 000000000..bd9a47b5a
--- /dev/null
+++ b/llvm/test/Other/new-pm-pr42726-cgscc.ll
@@ -0,0 +1,57 @@
+; RUN: opt -aa-pipeline=default -passes="default<O1>" %s -S | FileCheck %s
+; REQUIRES: asserts
+
+declare void @bar()
+declare void @baz(i32*)
+
+; CHECK-LABEL: @foo1()
+define void @foo1() {
+entry:
+  %tag = alloca i32, align 4
+  call void @baz(i32* %tag)
+  %tmp = load i32, i32* %tag, align 4
+  switch i32 %tmp, label %sw.bb799 [
+    i32 10, label %sw.bb239
+  ]
+
+sw.bb239:
+  call void @foo2()
+  br label %cleanup871
+
+sw.bb799:
+  call void @foo3(i32 undef)
+  br label %cleanup871
+
+cleanup871:
+  call void @bar()
+  unreachable
+}
+
+define void @foo2() {
+  call void @foo4()
+  unreachable
+}
+
+define void @foo3(i32 %ptr) {
+  call void @foo1()
+  unreachable
+}
+
+define void @foo4() {
+entry:
+  %tag = alloca i32, align 4
+  call void @baz(i32* %tag)
+  %tmp = load i32, i32* %tag, align 4
+  switch i32 %tmp, label %sw.bb442 [
+    i32 16, label %sw.bb352
+  ]
+
+sw.bb352:
+  call void @foo3(i32 undef)
+  unreachable
+
+sw.bb442:
+  call void @foo2()
+  unreachable
+}
+
diff --git a/llvm/test/Other/new-pm-thinlto-defaults.ll b/llvm/test/Other/new-pm-thinlto-defaults.ll
index 8dc6bf7a2..48d59dd6a 100644
--- a/llvm/test/Other/new-pm-thinlto-defaults.ll
+++ b/llvm/test/Other/new-pm-thinlto-defaults.ll
@@ -13,39 +13,39 @@
 ; RUN:     | FileCheck %s --check-prefixes=CHECK-O,CHECK-O1,CHECK-PRELINK-O,CHECK-PRELINK-O-NODIS,CHECK-PRELINK-O1
 ; RUN: opt -disable-verify -debug-pass-manager \
 ; RUN:     -passes='thinlto-pre-link<O2>,name-anon-globals' -S  %s 2>&1 \
-; RUN:     | FileCheck %s --check-prefixes=CHECK-O,CHECK-O2,CHECK-PRELINK-O,CHECK-PRELINK-O-NODIS,CHECK-PRELINK-O2
+; RUN:     | FileCheck %s --check-prefixes=CHECK-O,CHECK-O2,CHECK-O23SZ,CHECK-PRELINK-O,CHECK-PRELINK-O-NODIS,CHECK-PRELINK-O2
 ; RUN: opt -disable-verify -debug-pass-manager \
 ; RUN:     -passes='thinlto-pre-link<O3>,name-anon-globals' -S -passes-ep-pipeline-start='no-op-module' %s 2>&1 \
-; RUN:     | FileCheck %s --check-prefixes=CHECK-O,CHECK-O3,CHECK-PRELINK-O,CHECK-PRELINK-O-NODIS,CHECK-PRELINK-O3,CHECK-EP-PIPELINE-START
+; RUN:     | FileCheck %s --check-prefixes=CHECK-O,CHECK-O3,CHECK-O23SZ,CHECK-PRELINK-O,CHECK-PRELINK-O-NODIS,CHECK-PRELINK-O3,CHECK-EP-PIPELINE-START
 ; RUN: opt -disable-verify -debug-pass-manager \
 ; RUN:     -passes='thinlto-pre-link<Os>,name-anon-globals' -S %s 2>&1 \
-; RUN:     | FileCheck %s --check-prefixes=CHECK-O,CHECK-Os,CHECK-PRELINK-O,CHECK-PRELINK-O-NODIS,CHECK-PRELINK-Os
+; RUN:     | FileCheck %s --check-prefixes=CHECK-O,CHECK-Os,CHECK-O23SZ,CHECK-PRELINK-O,CHECK-PRELINK-O-NODIS,CHECK-PRELINK-Os
 ; RUN: opt -disable-verify -debug-pass-manager \
 ; RUN:     -passes='thinlto-pre-link<Oz>,name-anon-globals' -S %s 2>&1 \
-; RUN:     | FileCheck %s --check-prefixes=CHECK-O,CHECK-Oz,CHECK-PRELINK-O,CHECK-PRELINK-O-NODIS,CHECK-PRELINK-Oz
+; RUN:     | FileCheck %s --check-prefixes=CHECK-O,CHECK-Oz,CHECK-O23SZ,CHECK-PRELINK-O,CHECK-PRELINK-O-NODIS,CHECK-PRELINK-Oz
 ; RUN: opt -disable-verify -debug-pass-manager -new-pm-debug-info-for-profiling \
 ; RUN:     -passes='thinlto-pre-link<O2>,name-anon-globals' -S  %s 2>&1 \
-; RUN:     | FileCheck %s --check-prefixes=CHECK-DIS,CHECK-O,CHECK-O2,CHECK-PRELINK-O,CHECK-PRELINK-O2
+; RUN:     | FileCheck %s --check-prefixes=CHECK-DIS,CHECK-O,CHECK-O2,CHECK-O23SZ,CHECK-PRELINK-O,CHECK-PRELINK-O2
 ;
 ; Postlink pipelines:
 ; RUN: opt -disable-verify -debug-pass-manager \
 ; RUN:     -passes='thinlto<O1>' -S %s 2>&1 \
-; RUN:     | FileCheck %s --check-prefixes=CHECK-O,CHECK-O1,CHECK-POSTLINK-O,CHECK-POSTLINK-O1
+; RUN:     | FileCheck %s --check-prefixes=CHECK-O,CHECK-O1,CHECK-POSTLINK-O,%llvmcheckext,CHECK-POSTLINK-O1
 ; RUN: opt -disable-verify -debug-pass-manager \
 ; RUN:     -passes='thinlto<O2>' -S  %s 2>&1 \
-; RUN:     | FileCheck %s --check-prefixes=CHECK-O,CHECK-O2,CHECK-POSTLINK-O,CHECK-POSTLINK-O2
+; RUN:     | FileCheck %s --check-prefixes=CHECK-O,CHECK-O2,CHECK-O23SZ,CHECK-POSTLINK-O,%llvmcheckext,CHECK-POSTLINK-O2
 ; RUN: opt -disable-verify -debug-pass-manager -passes-ep-pipeline-start='no-op-module' \
 ; RUN:     -passes='thinlto<O3>' -S  %s 2>&1 \
-; RUN:     | FileCheck %s --check-prefixes=CHECK-O,CHECK-O3,CHECK-POSTLINK-O,CHECK-POSTLINK-O3
+; RUN:     | FileCheck %s --check-prefixes=CHECK-O,CHECK-O3,CHECK-O23SZ,CHECK-POSTLINK-O,%llvmcheckext,CHECK-POSTLINK-O3
 ; RUN: opt -disable-verify -debug-pass-manager \
 ; RUN:     -passes='thinlto<Os>' -S %s 2>&1 \
-; RUN:     | FileCheck %s --check-prefixes=CHECK-O,CHECK-Os,CHECK-POSTLINK-O,CHECK-POSTLINK-Os
+; RUN:     | FileCheck %s --check-prefixes=CHECK-O,CHECK-Os,CHECK-O23SZ,CHECK-POSTLINK-O,%llvmcheckext,CHECK-POSTLINK-Os
 ; RUN: opt -disable-verify -debug-pass-manager \
 ; RUN:     -passes='thinlto<Oz>' -S %s 2>&1 \
-; RUN:     | FileCheck %s --check-prefixes=CHECK-O,CHECK-Oz,CHECK-POSTLINK-O,CHECK-POSTLINK-Oz
+; RUN:     | FileCheck %s --check-prefixes=CHECK-O,CHECK-Oz,CHECK-O23SZ,CHECK-POSTLINK-O,%llvmcheckext,CHECK-POSTLINK-Oz
 ; RUN: opt -disable-verify -debug-pass-manager -new-pm-debug-info-for-profiling \
 ; RUN:     -passes='thinlto<O2>' -S  %s 2>&1 \
-; RUN:     | FileCheck %s --check-prefixes=CHECK-O,CHECK-O2,CHECK-POSTLINK-O,CHECK-POSTLINK-O2
+; RUN:     | FileCheck %s --check-prefixes=CHECK-O,CHECK-O2,CHECK-O23SZ,CHECK-POSTLINK-O,%llvmcheckext,CHECK-POSTLINK-O2
 ;
 ; CHECK-O: Running analysis: PassInstrumentationAnalysis
 ; CHECK-O-NEXT: Starting llvm::Module pass manager run.
@@ -63,9 +63,10 @@
 ; CHECK-POSTLINK-O-NEXT: Running analysis: OptimizationRemarkEmitterAnalysis
 ; CHECK-POSTLINK-O-NEXT: Running analysis: PassInstrumentationAnalysis
 ; CHECK-O-NEXT: Running pass: InferFunctionAttrsPass
+; CHECK-PRELINK-O-NODIS-NEXT: Running analysis: InnerAnalysisManagerProxy
 ; CHECK-O-NEXT: Running analysis: TargetLibraryAnalysis
+; CHECK-O-NEXT: Running analysis: PassInstrumentationAnalysis
 ; CHECK-O-NEXT: Running pass: ModuleToFunctionPassAdaptor<{{.*}}PassManager{{.*}}>
-; CHECK-PRELINK-O-NODIS-NEXT: Running analysis: InnerAnalysisManagerProxy
 ; CHECK-PRELINK-O-NEXT: Running analysis: PassInstrumentationAnalysis
 ; CHECK-O-NEXT: Starting llvm::Function pass manager run.
 ; CHECK-O-NEXT: Running pass: SimplifyCFGPass
@@ -111,17 +112,17 @@
 ; CHECK-O-NEXT: Running pass: SROA
 ; CHECK-O-NEXT: Running pass: EarlyCSEPass
 ; CHECK-O-NEXT: Running analysis: MemorySSAAnalysis
-; CHECK-O-NEXT: Running pass: SpeculativeExecutionPass
-; CHECK-O-NEXT: Running pass: JumpThreadingPass
-; CHECK-O-NEXT: Running analysis: LazyValueAnalysis
-; CHECK-O-NEXT: Running pass: CorrelatedValuePropagationPass
+; CHECK-O23SZ-NEXT: Running pass: SpeculativeExecutionPass
+; CHECK-O23SZ-NEXT: Running pass: JumpThreadingPass
+; CHECK-O23SZ-NEXT: Running analysis: LazyValueAnalysis
+; CHECK-O23SZ-NEXT: Running pass: CorrelatedValuePropagationPass
 ; CHECK-O-NEXT: Running pass: SimplifyCFGPass
 ; CHECK-O3-NEXT: Running pass: AggressiveInstCombinePass
 ; CHECK-O-NEXT: Running pass: InstCombinePass
 ; CHECK-O1-NEXT: Running pass: LibCallsShrinkWrapPass
 ; CHECK-O2-NEXT: Running pass: LibCallsShrinkWrapPass
 ; CHECK-O3-NEXT: Running pass: LibCallsShrinkWrapPass
-; CHECK-O-NEXT: Running pass: TailCallElimPass
+; CHECK-O23SZ-NEXT: Running pass: TailCallElimPass
 ; CHECK-O-NEXT: Running pass: SimplifyCFGPass
 ; CHECK-O-NEXT: Running pass: ReassociatePass
 ; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}OptimizationRemarkEmitterAnalysis
@@ -155,6 +156,7 @@
 ; CHECK-O-NEXT: Running pass: LoopDeletionPass
 ; CHECK-O-NEXT: Running pass: LoopFullUnrollPass
 ; CHECK-O-NEXT: Finished Loop pass manager run.
+; CHECK-O-NEXT: Running pass: SROA on foo
 ; CHECK-Os-NEXT: Running pass: MergedLoadStoreMotionPass
 ; CHECK-Os-NEXT: Running pass: GVN
 ; CHECK-Os-NEXT: Running analysis: MemoryDependenceAnalysis
@@ -178,14 +180,14 @@
 ; CHECK-O-NEXT: Running pass: BDCEPass
 ; CHECK-O-NEXT: Running analysis: DemandedBitsAnalysis
 ; CHECK-O-NEXT: Running pass: InstCombinePass
-; CHECK-O-NEXT: Running pass: JumpThreadingPass
-; CHECK-O-NEXT: Running pass: CorrelatedValuePropagationPass
-; CHECK-O-NEXT: Running pass: DSEPass
-; CHECK-O-NEXT: Running pass: FunctionToLoopPassAdaptor<{{.*}}LICMPass{{.*}}>
-; CHECK-O-NEXT: Starting llvm::Function pass manager run
-; CHECK-O-NEXT: Running pass: LoopSimplifyPass
-; CHECK-O-NEXT: Running pass: LCSSAPass
-; CHECK-O-NEXT: Finished llvm::Function pass manager run
+; CHECK-O23SZ-NEXT: Running pass: JumpThreadingPass
+; CHECK-O23SZ-NEXT: Running pass: CorrelatedValuePropagationPass
+; CHECK-O23SZ-NEXT: Running pass: DSEPass
+; CHECK-O23SZ-NEXT: Running pass: FunctionToLoopPassAdaptor<{{.*}}LICMPass{{.*}}>
+; CHECK-O23SZ-NEXT: Starting llvm::Function pass manager run
+; CHECK-O23SZ-NEXT: Running pass: LoopSimplifyPass
+; CHECK-O23SZ-NEXT: Running pass: LCSSAPass
+; CHECK-O23SZ-NEXT: Finished llvm::Function pass manager run
 ; CHECK-O-NEXT: Running pass: ADCEPass
 ; CHECK-O-NEXT: Running analysis: PostDominatorTreeAnalysis
 ; CHECK-O-NEXT: Running pass: SimplifyCFGPass
@@ -204,6 +206,8 @@
 ; CHECK-POSTLINK-O-NEXT: Running pass: ModuleToFunctionPassAdaptor<{{.*}}PassManager{{.*}}>
 ; CHECK-POSTLINK-O-NEXT: Starting llvm::Function pass manager run.
 ; CHECK-POSTLINK-O-NEXT: Running pass: Float2IntPass
+; CHECK-POSTLINK-O-NEXT: Running pass: LowerConstantIntrinsicsPass
+; CHECK-EXT: Running pass: {{.*}}::Bye
 ; CHECK-POSTLINK-O-NEXT: Running pass: FunctionToLoopPassAdaptor<{{.*}}LoopRotatePass
 ; CHECK-POSTLINK-O-NEXT: Starting llvm::Function pass manager run
 ; CHECK-POSTLINK-O-NEXT: Running pass: LoopSimplifyPass
@@ -217,6 +221,9 @@
 ; CHECK-POSTLINK-O-NEXT: Running analysis: LoopAccessAnalysis
 ; CHECK-POSTLINK-O-NEXT: Running pass: InstCombinePass
 ; CHECK-POSTLINK-O-NEXT: Running pass: SimplifyCFGPass
+; CHECK-POSTLINK-O2-NEXT: Running pass: SLPVectorizerPass
+; CHECK-POSTLINK-O3-NEXT: Running pass: SLPVectorizerPass
+; CHECK-POSTLINK-Os-NEXT: Running pass: SLPVectorizerPass
 ; CHECK-POSTLINK-O-NEXT: Running pass: InstCombinePass
 ; CHECK-POSTLINK-O-NEXT: Running pass: LoopUnrollPass
 ; CHECK-POSTLINK-O-NEXT: Running pass: WarnMissedTransformationsPass
diff --git a/llvm/test/Other/opt-O0-pipeline.ll b/llvm/test/Other/opt-O0-pipeline.ll
index 090620652..ce431a502 100644
--- a/llvm/test/Other/opt-O0-pipeline.ll
+++ b/llvm/test/Other/opt-O0-pipeline.ll
@@ -1,4 +1,4 @@
-; RUN: opt -mtriple=x86_64-- -O0 -debug-pass=Structure < %s -o /dev/null 2>&1 | FileCheck %s
+; RUN: opt -mtriple=x86_64-- -O0 -debug-pass=Structure < %s -o /dev/null 2>&1 | FileCheck %s --check-prefixes=CHECK,%llvmcheckext
 
 ; REQUIRES: asserts
 
@@ -6,6 +6,7 @@
 ; CHECK-NEXT: Target Transform Information
 ; CHECK-NEXT:   FunctionPass Manager
 ; CHECK-NEXT:     Module Verifier
+; CHECK-EXT:     Good Bye World Pass
 ; CHECK-NEXT:     Instrument function entry/exit with calls to e.g. mcount() (pre inlining)
 ; CHECK-NEXT: Pass Arguments:
 ; CHECK-NEXT: Target Library Information
diff --git a/llvm/test/Other/opt-O2-pipeline.ll b/llvm/test/Other/opt-O2-pipeline.ll
index 8f4d98fe3..da0edf105 100644
--- a/llvm/test/Other/opt-O2-pipeline.ll
+++ b/llvm/test/Other/opt-O2-pipeline.ll
@@ -1,4 +1,4 @@
-; RUN: opt -mtriple=x86_64-- -O2 -debug-pass=Structure < %s -o /dev/null 2>&1 | FileCheck %s
+; RUN: opt -mtriple=x86_64-- -O2 -debug-pass=Structure < %s -o /dev/null 2>&1 | FileCheck --check-prefixes=CHECK,%llvmcheckext %s 
 
 ; REQUIRES: asserts
 
@@ -10,6 +10,8 @@
 ; CHECK-NEXT: Target Library Information
 ; CHECK-NEXT:   FunctionPass Manager
 ; CHECK-NEXT:     Module Verifier
+; CHECK-EXT:      Good Bye World Pass
+; CHECK-NOEXT-NOT:      Good Bye World Pass
 ; CHECK-NEXT:     Instrument function entry/exit with calls to e.g. mcount() (pre inlining)
 ; CHECK-NEXT:     Simplify the CFG
 ; CHECK-NEXT:     Dominator Tree Construction
@@ -28,11 +30,16 @@
 ; CHECK-NEXT:     Force set function attributes
 ; CHECK-NEXT:     Infer set function attributes
 ; CHECK-NEXT:     Interprocedural Sparse Conditional Constant Propagation
-; CHECK-NEXT:       Unnamed pass: implement Pass::getPassName()
+; CHECK-NEXT:       FunctionPass Manager
+; CHECK-NEXT:         Dominator Tree Construction
 ; CHECK-NEXT:     Called Value Propagation
 ; CHECK-NEXT:     Deduce and propagate attributes
 ; CHECK-NEXT:     Global Variable Optimizer
-; CHECK-NEXT:       Unnamed pass: implement Pass::getPassName()
+; CHECK-NEXT:       FunctionPass Manager
+; CHECK-NEXT:         Dominator Tree Construction
+; CHECK-NEXT:         Natural Loop Information
+; CHECK-NEXT:         Branch Probability Analysis
+; CHECK-NEXT:         Block Frequency Analysis
 ; CHECK-NEXT:     FunctionPass Manager
 ; CHECK-NEXT:       Dominator Tree Construction
 ; CHECK-NEXT:       Promote Memory to Register
@@ -102,6 +109,8 @@
 ; CHECK-NEXT:         Scalar Evolution Analysis
 ; CHECK-NEXT:         Loop Pass Manager
 ; CHECK-NEXT:           Rotate Loops
+; CHECK-NEXT:         Memory SSA
+; CHECK-NEXT:         Loop Pass Manager
 ; CHECK-NEXT:           Loop Invariant Code Motion
 ; CHECK-NEXT:           Unswitch loops
 ; CHECK-NEXT:         Simplify the CFG
@@ -141,7 +150,6 @@
 ; CHECK-NEXT:         Bit-Tracking Dead Code Elimination
 ; CHECK-NEXT:         Basic Alias Analysis (stateless AA impl)
 ; CHECK-NEXT:         Function Alias Analysis Results
-; CHECK-NEXT:         Natural Loop Information
 ; CHECK-NEXT:         Lazy Branch Probability Analysis
 ; CHECK-NEXT:         Lazy Block Frequency Analysis
 ; CHECK-NEXT:         Optimization Remark Emitter
@@ -154,12 +162,13 @@
 ; CHECK-NEXT:         Phi Values Analysis
 ; CHECK-NEXT:         Memory Dependence Analysis
 ; CHECK-NEXT:         Dead Store Elimination
+; CHECK-NEXT:         Basic Alias Analysis (stateless AA impl)
+; CHECK-NEXT:         Function Alias Analysis Results
+; CHECK-NEXT:         Memory SSA
 ; CHECK-NEXT:         Natural Loop Information
 ; CHECK-NEXT:         Canonicalize natural loops
 ; CHECK-NEXT:         LCSSA Verifier
 ; CHECK-NEXT:         Loop-Closed SSA Form Pass
-; CHECK-NEXT:         Basic Alias Analysis (stateless AA impl)
-; CHECK-NEXT:         Function Alias Analysis Results
 ; CHECK-NEXT:         Scalar Evolution Analysis
 ; CHECK-NEXT:         Loop Pass Manager
 ; CHECK-NEXT:           Loop Invariant Code Motion
@@ -179,12 +188,18 @@
 ; CHECK-NEXT:     CallGraph Construction
 ; CHECK-NEXT:     Deduce function attributes in RPO
 ; CHECK-NEXT:     Global Variable Optimizer
-; CHECK-NEXT:       Unnamed pass: implement Pass::getPassName()
+; CHECK-NEXT:       FunctionPass Manager
+; CHECK-NEXT:         Dominator Tree Construction
+; CHECK-NEXT:         Natural Loop Information
+; CHECK-NEXT:         Branch Probability Analysis
+; CHECK-NEXT:         Block Frequency Analysis
 ; CHECK-NEXT:     Dead Global Elimination
 ; CHECK-NEXT:     CallGraph Construction
 ; CHECK-NEXT:     Globals Alias Analysis
 ; CHECK-NEXT:     FunctionPass Manager
+; CHECK-NEXT:       Dominator Tree Construction
 ; CHECK-NEXT:       Float to int
+; CHECK-NEXT:       Lower constant intrinsics
 ; CHECK-NEXT:       Dominator Tree Construction
 ; CHECK-NEXT:       Natural Loop Information
 ; CHECK-NEXT:       Canonicalize natural loops
@@ -247,6 +262,7 @@
 ; CHECK-NEXT:       Lazy Block Frequency Analysis
 ; CHECK-NEXT:       Optimization Remark Emitter
 ; CHECK-NEXT:       Combine redundant instructions
+; CHECK-NEXT:       Memory SSA
 ; CHECK-NEXT:       Canonicalize natural loops
 ; CHECK-NEXT:       LCSSA Verifier
 ; CHECK-NEXT:       Loop-Closed SSA Form Pass
diff --git a/llvm/test/Other/opt-O3-pipeline.ll b/llvm/test/Other/opt-O3-pipeline.ll
index 07e20b80f..113137a6f 100644
--- a/llvm/test/Other/opt-O3-pipeline.ll
+++ b/llvm/test/Other/opt-O3-pipeline.ll
@@ -1,4 +1,4 @@
-; RUN: opt -mtriple=x86_64-- -O3 -debug-pass=Structure < %s -o /dev/null 2>&1 | FileCheck %s
+; RUN: opt -mtriple=x86_64-- -O3 -debug-pass=Structure < %s -o /dev/null 2>&1 | FileCheck --check-prefixes=CHECK,%llvmcheckext %s
 
 ; REQUIRES: asserts
 
@@ -10,6 +10,8 @@
 ; CHECK-NEXT: Target Library Information
 ; CHECK-NEXT:   FunctionPass Manager
 ; CHECK-NEXT:     Module Verifier
+; CHECK-EXT:      Good Bye World Pass
+; CHECK-NOEXT-NOT:      Good Bye World Pass
 ; CHECK-NEXT:     Instrument function entry/exit with calls to e.g. mcount() (pre inlining)
 ; CHECK-NEXT:     Simplify the CFG
 ; CHECK-NEXT:     Dominator Tree Construction
@@ -31,11 +33,16 @@
 ; CHECK-NEXT:       Dominator Tree Construction
 ; CHECK-NEXT:       Call-site splitting
 ; CHECK-NEXT:     Interprocedural Sparse Conditional Constant Propagation
-; CHECK-NEXT:       Unnamed pass: implement Pass::getPassName()
+; CHECK-NEXT:       FunctionPass Manager
+; CHECK-NEXT:         Dominator Tree Construction
 ; CHECK-NEXT:     Called Value Propagation
 ; CHECK-NEXT:     Deduce and propagate attributes
 ; CHECK-NEXT:     Global Variable Optimizer
-; CHECK-NEXT:       Unnamed pass: implement Pass::getPassName()
+; CHECK-NEXT:       FunctionPass Manager
+; CHECK-NEXT:         Dominator Tree Construction
+; CHECK-NEXT:         Natural Loop Information
+; CHECK-NEXT:         Branch Probability Analysis
+; CHECK-NEXT:         Block Frequency Analysis
 ; CHECK-NEXT:     FunctionPass Manager
 ; CHECK-NEXT:       Dominator Tree Construction
 ; CHECK-NEXT:       Promote Memory to Register
@@ -107,6 +114,8 @@
 ; CHECK-NEXT:         Scalar Evolution Analysis
 ; CHECK-NEXT:         Loop Pass Manager
 ; CHECK-NEXT:           Rotate Loops
+; CHECK-NEXT:         Memory SSA
+; CHECK-NEXT:         Loop Pass Manager
 ; CHECK-NEXT:           Loop Invariant Code Motion
 ; CHECK-NEXT:           Unswitch loops
 ; CHECK-NEXT:         Simplify the CFG
@@ -146,7 +155,6 @@
 ; CHECK-NEXT:         Bit-Tracking Dead Code Elimination
 ; CHECK-NEXT:         Basic Alias Analysis (stateless AA impl)
 ; CHECK-NEXT:         Function Alias Analysis Results
-; CHECK-NEXT:         Natural Loop Information
 ; CHECK-NEXT:         Lazy Branch Probability Analysis
 ; CHECK-NEXT:         Lazy Block Frequency Analysis
 ; CHECK-NEXT:         Optimization Remark Emitter
@@ -159,12 +167,13 @@
 ; CHECK-NEXT:         Phi Values Analysis
 ; CHECK-NEXT:         Memory Dependence Analysis
 ; CHECK-NEXT:         Dead Store Elimination
+; CHECK-NEXT:         Basic Alias Analysis (stateless AA impl)
+; CHECK-NEXT:         Function Alias Analysis Results
+; CHECK-NEXT:         Memory SSA
 ; CHECK-NEXT:         Natural Loop Information
 ; CHECK-NEXT:         Canonicalize natural loops
 ; CHECK-NEXT:         LCSSA Verifier
 ; CHECK-NEXT:         Loop-Closed SSA Form Pass
-; CHECK-NEXT:         Basic Alias Analysis (stateless AA impl)
-; CHECK-NEXT:         Function Alias Analysis Results
 ; CHECK-NEXT:         Scalar Evolution Analysis
 ; CHECK-NEXT:         Loop Pass Manager
 ; CHECK-NEXT:           Loop Invariant Code Motion
@@ -184,12 +193,18 @@
 ; CHECK-NEXT:     CallGraph Construction
 ; CHECK-NEXT:     Deduce function attributes in RPO
 ; CHECK-NEXT:     Global Variable Optimizer
-; CHECK-NEXT:       Unnamed pass: implement Pass::getPassName()
+; CHECK-NEXT:       FunctionPass Manager
+; CHECK-NEXT:         Dominator Tree Construction
+; CHECK-NEXT:         Natural Loop Information
+; CHECK-NEXT:         Branch Probability Analysis
+; CHECK-NEXT:         Block Frequency Analysis
 ; CHECK-NEXT:     Dead Global Elimination
 ; CHECK-NEXT:     CallGraph Construction
 ; CHECK-NEXT:     Globals Alias Analysis
 ; CHECK-NEXT:     FunctionPass Manager
+; CHECK-NEXT:       Dominator Tree Construction
 ; CHECK-NEXT:       Float to int
+; CHECK-NEXT:       Lower constant intrinsics
 ; CHECK-NEXT:       Dominator Tree Construction
 ; CHECK-NEXT:       Natural Loop Information
 ; CHECK-NEXT:       Canonicalize natural loops
@@ -252,6 +267,7 @@
 ; CHECK-NEXT:       Lazy Block Frequency Analysis
 ; CHECK-NEXT:       Optimization Remark Emitter
 ; CHECK-NEXT:       Combine redundant instructions
+; CHECK-NEXT:       Memory SSA
 ; CHECK-NEXT:       Canonicalize natural loops
 ; CHECK-NEXT:       LCSSA Verifier
 ; CHECK-NEXT:       Loop-Closed SSA Form Pass
diff --git a/llvm/test/Other/opt-Os-pipeline.ll b/llvm/test/Other/opt-Os-pipeline.ll
index bacda22ea..c77f84005 100644
--- a/llvm/test/Other/opt-Os-pipeline.ll
+++ b/llvm/test/Other/opt-Os-pipeline.ll
@@ -1,4 +1,4 @@
-; RUN: opt -mtriple=x86_64-- -Os -debug-pass=Structure < %s -o /dev/null 2>&1 | FileCheck %s
+; RUN: opt -mtriple=x86_64-- -Os -debug-pass=Structure < %s -o /dev/null 2>&1 | FileCheck --check-prefixes=CHECK,%llvmcheckext %s
 
 ; REQUIRES: asserts
 
@@ -10,6 +10,8 @@
 ; CHECK-NEXT: Target Library Information
 ; CHECK-NEXT:   FunctionPass Manager
 ; CHECK-NEXT:     Module Verifier
+; CHECK-EXT:      Good Bye World Pass
+; CHECK-NOEXT-NOT:      Good Bye World Pass
 ; CHECK-NEXT:     Instrument function entry/exit with calls to e.g. mcount() (pre inlining)
 ; CHECK-NEXT:     Simplify the CFG
 ; CHECK-NEXT:     Dominator Tree Construction
@@ -28,11 +30,16 @@
 ; CHECK-NEXT:     Force set function attributes
 ; CHECK-NEXT:     Infer set function attributes
 ; CHECK-NEXT:     Interprocedural Sparse Conditional Constant Propagation
-; CHECK-NEXT:       Unnamed pass: implement Pass::getPassName()
+; CHECK-NEXT:       FunctionPass Manager
+; CHECK-NEXT:         Dominator Tree Construction
 ; CHECK-NEXT:     Called Value Propagation
 ; CHECK-NEXT:     Deduce and propagate attributes
 ; CHECK-NEXT:     Global Variable Optimizer
-; CHECK-NEXT:       Unnamed pass: implement Pass::getPassName()
+; CHECK-NEXT:       FunctionPass Manager
+; CHECK-NEXT:         Dominator Tree Construction
+; CHECK-NEXT:         Natural Loop Information
+; CHECK-NEXT:         Branch Probability Analysis
+; CHECK-NEXT:         Block Frequency Analysis
 ; CHECK-NEXT:     FunctionPass Manager
 ; CHECK-NEXT:       Dominator Tree Construction
 ; CHECK-NEXT:       Promote Memory to Register
@@ -89,6 +96,8 @@
 ; CHECK-NEXT:         Scalar Evolution Analysis
 ; CHECK-NEXT:         Loop Pass Manager
 ; CHECK-NEXT:           Rotate Loops
+; CHECK-NEXT:         Memory SSA
+; CHECK-NEXT:         Loop Pass Manager
 ; CHECK-NEXT:           Loop Invariant Code Motion
 ; CHECK-NEXT:           Unswitch loops
 ; CHECK-NEXT:         Simplify the CFG
@@ -128,7 +137,6 @@
 ; CHECK-NEXT:         Bit-Tracking Dead Code Elimination
 ; CHECK-NEXT:         Basic Alias Analysis (stateless AA impl)
 ; CHECK-NEXT:         Function Alias Analysis Results
-; CHECK-NEXT:         Natural Loop Information
 ; CHECK-NEXT:         Lazy Branch Probability Analysis
 ; CHECK-NEXT:         Lazy Block Frequency Analysis
 ; CHECK-NEXT:         Optimization Remark Emitter
@@ -141,12 +149,13 @@
 ; CHECK-NEXT:         Phi Values Analysis
 ; CHECK-NEXT:         Memory Dependence Analysis
 ; CHECK-NEXT:         Dead Store Elimination
+; CHECK-NEXT:         Basic Alias Analysis (stateless AA impl)
+; CHECK-NEXT:         Function Alias Analysis Results
+; CHECK-NEXT:         Memory SSA
 ; CHECK-NEXT:         Natural Loop Information
 ; CHECK-NEXT:         Canonicalize natural loops
 ; CHECK-NEXT:         LCSSA Verifier
 ; CHECK-NEXT:         Loop-Closed SSA Form Pass
-; CHECK-NEXT:         Basic Alias Analysis (stateless AA impl)
-; CHECK-NEXT:         Function Alias Analysis Results
 ; CHECK-NEXT:         Scalar Evolution Analysis
 ; CHECK-NEXT:         Loop Pass Manager
 ; CHECK-NEXT:           Loop Invariant Code Motion
@@ -166,12 +175,18 @@
 ; CHECK-NEXT:     CallGraph Construction
 ; CHECK-NEXT:     Deduce function attributes in RPO
 ; CHECK-NEXT:     Global Variable Optimizer
-; CHECK-NEXT:       Unnamed pass: implement Pass::getPassName()
+; CHECK-NEXT:       FunctionPass Manager
+; CHECK-NEXT:         Dominator Tree Construction
+; CHECK-NEXT:         Natural Loop Information
+; CHECK-NEXT:         Branch Probability Analysis
+; CHECK-NEXT:         Block Frequency Analysis
 ; CHECK-NEXT:     Dead Global Elimination
 ; CHECK-NEXT:     CallGraph Construction
 ; CHECK-NEXT:     Globals Alias Analysis
 ; CHECK-NEXT:     FunctionPass Manager
+; CHECK-NEXT:       Dominator Tree Construction
 ; CHECK-NEXT:       Float to int
+; CHECK-NEXT:       Lower constant intrinsics
 ; CHECK-NEXT:       Dominator Tree Construction
 ; CHECK-NEXT:       Natural Loop Information
 ; CHECK-NEXT:       Canonicalize natural loops
@@ -234,6 +249,7 @@
 ; CHECK-NEXT:       Lazy Block Frequency Analysis
 ; CHECK-NEXT:       Optimization Remark Emitter
 ; CHECK-NEXT:       Combine redundant instructions
+; CHECK-NEXT:       Memory SSA
 ; CHECK-NEXT:       Canonicalize natural loops
 ; CHECK-NEXT:       LCSSA Verifier
 ; CHECK-NEXT:       Loop-Closed SSA Form Pass
diff --git a/llvm/test/Other/opt-override-mcpu-mattr.ll b/llvm/test/Other/opt-override-mcpu-mattr.ll
index 84d4209a9..967df2c7a 100644
--- a/llvm/test/Other/opt-override-mcpu-mattr.ll
+++ b/llvm/test/Other/opt-override-mcpu-mattr.ll
@@ -1,13 +1,21 @@
 ; RUN: opt < %s -mtriple=x86_64-apple-darwin -mcpu=broadwell -mattr=+avx2 -S | FileCheck %s
 
-; Check that opt can rewrite function attributes target-cpu and target-features
-; using command line options -mcpu and -mattr.
+; Check that opt can add but not rewrite function attributes
+; target-cpu and target-features using command line options -mcpu and
+; -mattr.
 
 ; CHECK: attributes #0 = { nounwind readnone ssp uwtable "target-cpu"="broadwell" "target-features"="+avx2" "use-soft-float"="false" }
+; CHECK: attributes #1 = { nounwind readnone ssp uwtable "target-cpu"="core2" "target-features"="+avx2" "use-soft-float"="false" }
 
-define i32 @foo1() #0 {
+define i32 @no_target_cpu() #0 {
 entry:
   ret i32 0
 }
 
-attributes #0 = { nounwind readnone ssp uwtable "target-cpu"="core2" "target-features"="+ssse3,+cx16,+sse,+sse2,+sse3" "use-soft-float"="false" }
+define i32 @has_targe_cpu() #1 {
+entry:
+  ret i32 0
+}
+
+attributes #0 = { nounwind readnone ssp uwtable "target-features"="+ssse3,+cx16,+sse,+sse2,+sse3" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone ssp uwtable "target-cpu"="core2" "target-features"="+ssse3,+cx16,+sse,+sse2,+sse3" "use-soft-float"="false" }
diff --git a/llvm/test/Other/pass-pipelines.ll b/llvm/test/Other/pass-pipelines.ll
index be14c6cde..0b3b5a960 100644
--- a/llvm/test/Other/pass-pipelines.ll
+++ b/llvm/test/Other/pass-pipelines.ll
@@ -28,7 +28,7 @@
 ; CHECK-O2: ModulePass Manager
 ; CHECK-O2-NOT: Manager
 ; First function pass pipeline just does early opts.
-; CHECK-O2: FunctionPass Manager
+; CHECK-O2-COUNT-3: FunctionPass Manager
 ; CHECK-O2-NOT: Manager
 ; FIXME: It's a bit odd to do dead arg elim in the middle of early opts...
 ; CHECK-O2: Dead Argument Elimination
@@ -52,6 +52,7 @@
 ; CHECK-O2-NEXT: FunctionPass Manager
 ; CHECK-O2-NOT: Manager
 ; CHECK-O2: Loop Pass Manager
+; CHECK-O2: Loop Pass Manager
 ; CHECK-O2-NOT: Manager
 ; FIXME: We shouldn't be pulling out to simplify-cfg and instcombine and
 ; causing new loop pass managers.
diff --git a/llvm/test/Other/print-module-scope.ll b/llvm/test/Other/print-module-scope.ll
index a6f744657..54e087ff2 100644
--- a/llvm/test/Other/print-module-scope.ll
+++ b/llvm/test/Other/print-module-scope.ll
@@ -51,10 +51,10 @@ define void @bar() #0 {
 
 declare void @baz() #1
 
-attributes #0 = { nounwind "no-frame-pointer-elim"="true" }
+attributes #0 = { nounwind "frame-pointer"="all" }
 
 attributes #1 = { nounwind readnone ssp "use-soft-float"="false" }
-; FOO: attributes #{{[0-9]}} = { nounwind "no-frame-pointer-elim"="true" }
+; FOO: attributes #{{[0-9]}} = { nounwind "frame-pointer"="all" }
 
 ; FOO: attributes #{{[0-9]}} = { nounwind readnone ssp "use-soft-float"="false" }
 
diff --git a/llvm/test/Other/print-slotindexes.ll b/llvm/test/Other/print-slotindexes.ll
new file mode 100644
index 000000000..722a3dc5a
--- /dev/null
+++ b/llvm/test/Other/print-slotindexes.ll
@@ -0,0 +1,12 @@
+; RUN: llc -print-after=slotindexes < %s 2>&1 | FileCheck %s --check-prefixes=ALL,SI 
+; RUN: llc -print-after=slotindexes -print-slotindexes=false < %s 2>&1 | FileCheck %s --check-prefixes=ALL,NOSI
+; REQUIRES: default_triple
+define void @foo(){
+  ret void
+}
+
+;CHECK: IR Dump {{.*}}
+;CHECK: # Machine code for function foo{{.*}}
+;SI: {{[0-9]+}}B bb.0 (%ir-block.0)
+;NOSI: {{^}}bb.0 (%ir-block.0)
+
diff --git a/llvm/test/Other/printer.ll b/llvm/test/Other/printer.ll
index 9785a17b2..863376562 100644
--- a/llvm/test/Other/printer.ll
+++ b/llvm/test/Other/printer.ll
@@ -1,5 +1,7 @@
-; RUN: opt -mem2reg -instcombine -print-after-all -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt -passes='mem2reg,instcombine' -print-after-all -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt -mem2reg -instcombine -print-after-all -disable-output < %s 2>&1 | \
+; RUN:   FileCheck --check-prefixes=CHECK,OLDPM %s --implicit-check-not='IR Dump'
+; RUN: opt -passes='mem2reg,instcombine' -print-after-all -disable-output < %s 2>&1 | \
+; RUN:   FileCheck --check-prefixes=CHECK,NEWPM %s --implicit-check-not='IR Dump'
 define void @tester(){
   ret void
 }
@@ -8,21 +10,14 @@ define void @foo(){
   ret void
 }
 
-;CHECK-NOT: IR Dump After PassManager
-;CHECK-NOT: IR Dump After ModuleToFunctionPassAdaptor
-;
-;CHECK:     *** IR Dump After {{Promote Memory to Register|PromotePass}}
-;CHECK:     define void @tester
-;CHECK-NOT: define void @foo
-;CHECK:     *** IR Dump After {{Combine redundant instructions|InstCombinePass}}
-;CHECK:     define void @tester
-;CHECK-NOT: define void @foo
-;CHECK:     *** IR Dump After {{Promote Memory to Register|PromotePass}}
-;CHECK:     define void @foo
-;CHECK-NOT: define void @tester
-;CHECK:     *** IR Dump After {{Combine redundant instructions|InstCombinePass}}
-;CHECK:     define void @foo
-;CHECK-NOT: define void @tester
-;CHECK:     *** IR Dump After {{Module Verifier|VerifierPass}}
-;
-;CHECK-NOT: IR Dump After Print Module IR
+; NEWPM:      *** IR Dump After VerifierPass
+; CHECK:      *** IR Dump After {{Promote Memory to Register|PromotePass}}
+; CHECK-NEXT: define void @tester
+; CHECK:      *** IR Dump After {{Combine redundant instructions|InstCombinePass}}
+; CHECK-NEXT: define void @tester
+; OLDPM:      *** IR Dump After Module Verifier
+; CHECK:      *** IR Dump After {{Promote Memory to Register|PromotePass}}
+; CHECK-NEXT: define void @foo
+; CHECK:      *** IR Dump After {{Combine redundant instructions|InstCombinePass}}
+; CHECK-NEXT: define void @foo
+; CHECK:      *** IR Dump After {{Module Verifier|VerifierPass}}
diff --git a/llvm/test/Other/scalable-vectors-core-ir.ll b/llvm/test/Other/scalable-vectors-core-ir.ll
new file mode 100644
index 000000000..60cbab3cd
--- /dev/null
+++ b/llvm/test/Other/scalable-vectors-core-ir.ll
@@ -0,0 +1,393 @@
+; RUN: opt -S -verify < %s | FileCheck %s
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64--linux-gnu"
+
+;; Check supported instructions are accepted without dropping 'vscale'.
+;; Same order as the LangRef
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Unary Operations
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+
+define <vscale x 2 x double> @fneg(<vscale x 2 x double> %val) {
+; CHECK-LABEL: @fneg
+; CHECK: %r = fneg <vscale x 2 x double> %val
+; CHECK-NEXT: ret <vscale x 2 x double> %r
+  %r = fneg <vscale x 2 x double> %val
+  ret <vscale x 2 x double> %r
+}
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Binary Operations
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+define <vscale x 8 x i16> @add(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: @add
+; CHECK: %r = add <vscale x 8 x i16> %a, %b
+; CHECK-NEXT: ret <vscale x 8 x i16> %r
+  %r = add <vscale x 8 x i16> %a, %b
+  ret <vscale x 8 x i16> %r
+}
+
+define <vscale x 4 x float> @fadd(<vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: @fadd
+; CHECK: %r = fadd <vscale x 4 x float> %a, %b
+; CHECK-NEXT: ret <vscale x 4 x float> %r
+  %r = fadd <vscale x 4 x float> %a, %b
+  ret <vscale x 4 x float> %r
+}
+
+define <vscale x 4 x i32> @sub(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: @sub
+; CHECK: %r = sub <vscale x 4 x i32> %a, %b
+; CHECK-NEXT: ret <vscale x 4 x i32> %r
+  %r = sub <vscale x 4 x i32> %a, %b
+  ret <vscale x 4 x i32> %r
+}
+
+define <vscale x 4 x float> @fsub(<vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: @fsub
+; CHECK: %r = fsub <vscale x 4 x float> %a, %b
+; CHECK-NEXT: ret <vscale x 4 x float> %r
+  %r = fsub <vscale x 4 x float> %a, %b
+  ret <vscale x 4 x float> %r
+}
+
+define <vscale x 4 x i32> @mul(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: @mul
+; CHECK: %r = mul <vscale x 4 x i32> %a, %b
+; CHECK-NEXT: ret <vscale x 4 x i32> %r
+  %r = mul <vscale x 4 x i32> %a, %b
+  ret <vscale x 4 x i32> %r
+}
+
+define <vscale x 4 x float> @fmul(<vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: @fmul
+; CHECK: %r = fmul <vscale x 4 x float> %a, %b
+; CHECK-NEXT: ret <vscale x 4 x float> %r
+  %r = fmul <vscale x 4 x float> %a, %b
+  ret <vscale x 4 x float> %r
+}
+
+define <vscale x 4 x i32> @udiv(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: @udiv
+; CHECK: %r = udiv <vscale x 4 x i32> %a, %b
+; CHECK-NEXT: ret <vscale x 4 x i32> %r
+  %r = udiv <vscale x 4 x i32> %a, %b
+  ret <vscale x 4 x i32> %r
+}
+
+define <vscale x 4 x i32> @sdiv(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: @sdiv
+; CHECK: %r = sdiv <vscale x 4 x i32> %a, %b
+; CHECK-NEXT: ret <vscale x 4 x i32> %r
+  %r = sdiv <vscale x 4 x i32> %a, %b
+  ret <vscale x 4 x i32> %r
+}
+
+define <vscale x 4 x float> @fdiv(<vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: @fdiv
+; CHECK: %r = fdiv <vscale x 4 x float> %a, %b
+; CHECK-NEXT: ret <vscale x 4 x float> %r
+  %r = fdiv <vscale x 4 x float> %a, %b
+  ret <vscale x 4 x float> %r
+}
+
+define <vscale x 4 x i32> @urem(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: @urem
+; CHECK: %r = urem <vscale x 4 x i32> %a, %b
+; CHECK-NEXT: ret <vscale x 4 x i32> %r
+  %r = urem <vscale x 4 x i32> %a, %b
+  ret <vscale x 4 x i32> %r
+}
+
+define <vscale x 4 x i32> @srem(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: @srem
+; CHECK: %r = srem <vscale x 4 x i32> %a, %b
+; CHECK-NEXT: ret <vscale x 4 x i32> %r
+  %r = srem <vscale x 4 x i32> %a, %b
+  ret <vscale x 4 x i32> %r
+}
+
+define <vscale x 4 x float> @frem(<vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: @frem
+; CHECK: %r = frem <vscale x 4 x float> %a, %b
+; CHECK-NEXT: ret <vscale x 4 x float> %r
+  %r = frem <vscale x 4 x float> %a, %b
+  ret <vscale x 4 x float> %r
+}
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Bitwise Binary Operations
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+define <vscale x 4 x i32> @shl(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: @shl
+; CHECK: %r = shl <vscale x 4 x i32> %a, %b
+; CHECK-NEXT: ret <vscale x 4 x i32> %r
+  %r = shl <vscale x 4 x i32> %a, %b
+  ret <vscale x 4 x i32> %r
+}
+
+define <vscale x 4 x i32> @lshr(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: @lshr
+; CHECK: %r = lshr <vscale x 4 x i32> %a, %b
+; CHECK-NEXT: ret <vscale x 4 x i32> %r
+  %r = lshr <vscale x 4 x i32> %a, %b
+  ret <vscale x 4 x i32> %r
+}
+
+define <vscale x 4 x i32> @ashr(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: @ashr
+; CHECK: %r = ashr <vscale x 4 x i32> %a, %b
+; CHECK-NEXT: ret <vscale x 4 x i32> %r
+  %r = ashr <vscale x 4 x i32> %a, %b
+  ret <vscale x 4 x i32> %r
+}
+
+define <vscale x 4 x i32> @and(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: @and
+; CHECK: %r = and <vscale x 4 x i32> %a, %b
+; CHECK-NEXT: ret <vscale x 4 x i32> %r
+  %r = and <vscale x 4 x i32> %a, %b
+  ret <vscale x 4 x i32> %r
+}
+
+define <vscale x 4 x i32> @or(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: @or
+; CHECK: %r = or <vscale x 4 x i32> %a, %b
+; CHECK-NEXT: ret <vscale x 4 x i32> %r
+  %r = or <vscale x 4 x i32> %a, %b
+  ret <vscale x 4 x i32> %r
+}
+
+define <vscale x 4 x i32> @xor(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: @xor
+; CHECK: %r = xor <vscale x 4 x i32> %a, %b
+; CHECK-NEXT: ret <vscale x 4 x i32> %r
+  %r = xor <vscale x 4 x i32> %a, %b
+  ret <vscale x 4 x i32> %r
+}
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Vector Operations
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+define i64 @extractelement(<vscale x 2 x i64> %val) {
+; CHECK-LABEL: @extractelement
+; CHECK: %r = extractelement <vscale x 2 x i64> %val, i32 0
+; CHECK-NEXT: ret i64 %r
+  %r = extractelement <vscale x 2 x i64> %val, i32 0
+  ret i64 %r
+}
+
+define <vscale x 16 x i8> @insertelement(<vscale x 16 x i8> %vec, i8 %ins) {
+; CHECK-LABEL: @insertelement
+; CHECK: %r = insertelement <vscale x 16 x i8> %vec, i8 %ins, i32 0
+; CHECK-NEXT: ret <vscale x 16 x i8> %r
+  %r = insertelement <vscale x 16 x i8> %vec, i8 %ins, i32 0
+  ret <vscale x 16 x i8> %r
+}
+
+define <vscale x 8 x half> @shufflevector(half %val) {
+; CHECK-LABEL: @shufflevector
+; CHECK: %insvec = insertelement <vscale x 8 x half> undef, half %val, i32 0
+; CHECK-NEXT: %r = shufflevector <vscale x 8 x half> %insvec, <vscale x 8 x half> undef, <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT: ret <vscale x 8 x half> %r
+  %insvec = insertelement <vscale x 8 x half> undef, half %val, i32 0
+  %r = shufflevector <vscale x 8 x half> %insvec, <vscale x 8 x half> undef, <vscale x 8 x i32> zeroinitializer
+  ret <vscale x 8 x half> %r
+}
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Memory Access and Addressing Operations
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+define void @alloca() {
+; CHECK-LABEL: @alloca
+; CHECK: %vec = alloca <vscale x 4 x i32>
+; CHECK-NEXT: ret void
+  %vec = alloca <vscale x 4 x i32>
+  ret void
+}
+
+define <vscale x 2 x double> @load(<vscale x 2 x double>* %ptr) {
+; CHECK-LABEL: @load
+; CHECK: %r = load <vscale x 2 x double>, <vscale x 2 x double>* %ptr
+; CHECK-NEXT: ret <vscale x 2 x double> %r
+  %r = load <vscale x 2 x double>, <vscale x 2 x double>* %ptr
+  ret <vscale x 2 x double> %r
+}
+
+define void @store(<vscale x 4 x i32> %data, <vscale x 4 x i32>* %ptr) {
+; CHECK-LABEL: @store
+; CHECK: store <vscale x 4 x i32> %data, <vscale x 4 x i32>* %ptr
+; CHECK-NEXT: ret void
+  store <vscale x 4 x i32> %data, <vscale x 4 x i32>* %ptr
+  ret void
+}
+
+define <vscale x 4 x float>* @getelementptr(<vscale x 4 x float>* %base) {
+; CHECK-LABEL: @getelementptr
+; CHECK: %r = getelementptr <vscale x 4 x float>, <vscale x 4 x float>* %base, i64 0
+; CHECK-NEXT: ret <vscale x 4 x float>* %r
+  %r = getelementptr <vscale x 4 x float>, <vscale x 4 x float>* %base, i64 0
+  ret <vscale x 4 x float>* %r
+}
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Conversion Operations
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+define <vscale x 4 x i32> @truncto(<vscale x 4 x i64> %val) {
+; CHECK-LABEL: @truncto
+; CHECK: %r = trunc <vscale x 4 x i64> %val to <vscale x 4 x i32>
+; CHECK-NEXT: ret <vscale x 4 x i32> %r
+  %r = trunc <vscale x 4 x i64> %val to <vscale x 4 x i32>
+  ret <vscale x 4 x i32> %r
+}
+
+define <vscale x 2 x i64> @zextto(<vscale x 2 x i16> %val) {
+; CHECK-LABEL: @zextto
+; CHECK: %r = zext <vscale x 2 x i16> %val to <vscale x 2 x i64>
+; CHECK-NEXT: ret <vscale x 2 x i64> %r
+  %r = zext <vscale x 2 x i16> %val to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %r
+}
+
+define <vscale x 4 x i32> @sextto(<vscale x 4 x i8> %val) {
+; CHECK-LABEL: @sextto
+; CHECK: %r = sext <vscale x 4 x i8> %val to <vscale x 4 x i32>
+; CHECK-NEXT: ret <vscale x 4 x i32> %r
+  %r = sext <vscale x 4 x i8> %val to <vscale x 4 x i32>
+  ret <vscale x 4 x i32> %r
+}
+
+define <vscale x 4 x half> @fptruncto(<vscale x 4 x float> %val) {
+; CHECK-LABEL: @fptruncto
+; CHECK: %r = fptrunc <vscale x 4 x float> %val to <vscale x 4 x half>
+; CHECK-NEXT: ret <vscale x 4 x half> %r
+  %r = fptrunc <vscale x 4 x float> %val to <vscale x 4 x half>
+  ret <vscale x 4 x half> %r
+}
+
+define <vscale x 2 x double> @fpextto(<vscale x 2 x half> %val) {
+; CHECK-LABEL: @fpextto
+; CHECK: %r = fpext <vscale x 2 x half> %val to <vscale x 2 x double>
+; CHECK-NEXT: ret <vscale x 2 x double> %r
+  %r = fpext <vscale x 2 x half> %val to <vscale x 2 x double>
+  ret <vscale x 2 x double> %r
+}
+
+define <vscale x 4 x i32> @fptouito(<vscale x 4 x float> %val) {
+; CHECK-LABEL: @fptoui
+; CHECK: %r = fptoui <vscale x 4 x float> %val to <vscale x 4 x i32>
+; CHECK-NEXT: ret <vscale x 4 x i32> %r
+  %r = fptoui <vscale x 4 x float> %val to <vscale x 4 x i32>
+  ret <vscale x 4 x i32> %r
+}
+
+define <vscale x 4 x i32> @fptosito(<vscale x 4 x float> %val) {
+; CHECK-LABEL: @fptosi
+; CHECK: %r = fptosi <vscale x 4 x float> %val to <vscale x 4 x i32>
+; CHECK-NEXT: ret <vscale x 4 x i32> %r
+  %r = fptosi <vscale x 4 x float> %val to <vscale x 4 x i32>
+  ret <vscale x 4 x i32> %r
+}
+
+define <vscale x 4 x float> @uitofpto(<vscale x 4 x i32> %val) {
+; CHECK-LABEL: @uitofp
+; CHECK: %r = uitofp <vscale x 4 x i32> %val to <vscale x 4 x float>
+; CHECK-NEXT: ret <vscale x 4 x float> %r
+  %r = uitofp <vscale x 4 x i32> %val to <vscale x 4 x float>
+  ret <vscale x 4 x float> %r
+}
+
+define <vscale x 4 x float> @sitofpto(<vscale x 4 x i32> %val) {
+; CHECK-LABEL: @sitofp
+; CHECK: %r = sitofp <vscale x 4 x i32> %val to <vscale x 4 x float>
+; CHECK-NEXT: ret <vscale x 4 x float> %r
+  %r = sitofp <vscale x 4 x i32> %val to <vscale x 4 x float>
+  ret <vscale x 4 x float> %r
+}
+
+define <vscale x 2 x i64> @ptrtointto(<vscale x 2 x i32*> %val) {
+; CHECK-LABEL: @ptrtointto
+; CHECK: %r = ptrtoint <vscale x 2 x i32*> %val to <vscale x 2 x i64>
+; CHECK-NEXT: ret <vscale x 2 x i64> %r
+  %r = ptrtoint <vscale x 2 x i32*> %val to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %r
+}
+
+define <vscale x 2 x i32*> @inttoptrto(<vscale x 2 x i64> %val) {
+; CHECK-LABEL: @inttoptrto
+; CHECK: %r = inttoptr <vscale x 2 x i64> %val to <vscale x 2 x i32*>
+; CHECK-NEXT: ret <vscale x 2 x i32*> %r
+  %r = inttoptr <vscale x 2 x i64> %val to <vscale x 2 x i32*>
+  ret <vscale x 2 x i32*> %r
+}
+
+define <vscale x 2 x i64> @bitcastto(<vscale x 2 x double> %a) {
+; CHECK-LABEL: @bitcast
+; CHECK: %r = bitcast <vscale x 2 x double> %a to <vscale x 2 x i64>
+; CHECK-NEXT: ret <vscale x 2 x i64> %r
+  %r = bitcast <vscale x 2 x double> %a to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %r
+}
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Other Operations
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+define <vscale x 4 x i1> @icmp(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: @icmp
+; CHECK: %r = icmp eq <vscale x 4 x i32> %a, %b
+; CHECK-NEXT: ret <vscale x 4 x i1> %r
+  %r = icmp eq <vscale x 4 x i32> %a, %b
+  ret <vscale x 4 x i1> %r
+}
+
+define <vscale x 4 x i1> @fcmp(<vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: @fcmp
+; CHECK: %r = fcmp une <vscale x 4 x float> %a, %b
+; CHECK-NEXT: ret <vscale x 4 x i1> %r
+  %r = fcmp une <vscale x 4 x float> %a, %b
+  ret <vscale x 4 x i1> %r
+}
+
+define <vscale x 16 x i8> @phi(<vscale x 16 x i8> %a, i32 %val) {
+; CHECK-LABEL: @phi
+; CHECK: %r = phi <vscale x 16 x i8> [ %a, %entry ], [ %added, %iszero ]
+; CHECK-NEXT: ret <vscale x 16 x i8> %r
+entry:
+  %cmp = icmp eq i32 %val, 0
+  br i1 %cmp, label %iszero, label %end
+
+iszero:
+  %ins = insertelement <vscale x 16 x i8> undef, i8 1, i32 0
+  %splatone = shufflevector <vscale x 16 x i8> %ins, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %added = add <vscale x 16 x i8> %a, %splatone
+  br label %end
+
+end:
+  %r = phi <vscale x 16 x i8> [ %a, %entry ], [ %added, %iszero ]
+  ret <vscale x 16 x i8> %r
+}
+
+define <vscale x 8 x half> @select(<vscale x 8 x half> %a, <vscale x 8 x half> %b, <vscale x 8 x i1> %sval) {
+; CHECK-LABEL: @select
+; CHECK: %r = select <vscale x 8 x i1> %sval, <vscale x 8 x half> %a, <vscale x 8 x half> %b
+; CHECK-NEXT: ret <vscale x 8 x half> %r
+  %r = select <vscale x 8 x i1> %sval, <vscale x 8 x half> %a, <vscale x 8 x half> %b
+  ret <vscale x 8 x half> %r
+}
+
+declare <vscale x 4 x i32> @callee(<vscale x 4 x i32>)
+define <vscale x 4 x i32> @call(<vscale x 4 x i32> %val) {
+; CHECK-LABEL: @call
+; CHECK: %r = call <vscale x 4 x i32> @callee(<vscale x 4 x i32> %val)
+; CHECK-NEXT: ret <vscale x 4 x i32> %r
+  %r = call <vscale x 4 x i32> @callee(<vscale x 4 x i32> %val)
+  ret <vscale x 4 x i32> %r
+}
\ No newline at end of file
diff --git a/llvm/test/Other/unroll-sroa.ll b/llvm/test/Other/unroll-sroa.ll
new file mode 100644
index 000000000..e65756284
--- /dev/null
+++ b/llvm/test/Other/unroll-sroa.ll
@@ -0,0 +1,61 @@
+; RUN: opt -disable-verify -passes='default<O2>' -S < %s | FileCheck %s
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; The local array %tmp can only be optimized away by sroa after loop unroll.
+
+; CHECK-LABEL: define void @foo
+; CHECK-NOT:   alloca
+; CHECK-NOT:   call void @llvm.memcpy.p0i8.p0i8.i64
+
+; Function Attrs: nounwind uwtable
+define void @foo(i32* %a, i32* %b) {
+entry:
+  %a.addr = alloca i32*, align 8
+  %b.addr = alloca i32*, align 8
+  %tmp = alloca [4 x float], align 16
+  %i = alloca i32, align 4
+  store i32* %a, i32** %a.addr, align 8
+  store i32* %b, i32** %b.addr, align 8
+  store i32 0, i32* %i, align 4
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %iter2 = load i32, i32* %i, align 4
+  %cmp = icmp slt i32 %iter2, 4
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.cond
+  br label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %inptr = load i32*, i32** %a.addr, align 8
+  %idx2 = load i32, i32* %i, align 4
+  %idxprom = sext i32 %idx2 to i64
+  %arrayidx = getelementptr inbounds i32, i32* %inptr, i64 %idxprom
+  %val = load i32, i32* %arrayidx, align 4
+  %conv = sitofp i32 %val to float
+  %idx = load i32, i32* %i, align 4
+  %idxprom1 = sext i32 %idx to i64
+  %arrayidx2 = getelementptr inbounds [4 x float], [4 x float]* %tmp, i64 0, i64 %idxprom1
+  store float %conv, float* %arrayidx2, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body
+  %iter = load i32, i32* %i, align 4
+  %inc = add nsw i32 %iter, 1
+  store i32 %inc, i32* %i, align 4
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond.cleanup
+  %dstptr = load i32*, i32** %b.addr, align 8
+  %dst = bitcast i32* %dstptr to i8*
+  %arraydecay = getelementptr inbounds [4 x float], [4 x float]* %tmp, i64 0, i64 0
+  %src = bitcast float* %arraydecay to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %dst, i8* align 16 %src, i64 16, i1 false)
+  ret void
+}
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* noalias nocapture writeonly, i8* noalias nocapture readonly, i64, i1 immarg)
diff --git a/llvm/test/Reduce/remove-args.ll b/llvm/test/Reduce/remove-args.ll
new file mode 100644
index 000000000..8d6130262
--- /dev/null
+++ b/llvm/test/Reduce/remove-args.ll
@@ -0,0 +1,13 @@
+; Test that llvm-reduce can remove uninteresting function arguments from function definitions as well as their calls.
+;
+; RUN: rm -rf %t
+; RUN: llvm-reduce --test %python --test-arg %p/Inputs/remove-args.py %s -o %t
+; RUN: cat %t | FileCheck -implicit-check-not=uninteresting %s
+
+; CHECK: @interesting(i32 %interesting)
+define void @interesting(i32 %uninteresting1, i32 %interesting, i32 %uninteresting2) {
+entry:
+  ; CHECK: call void @interesting(i32 0)
+  call void @interesting(i32 -1, i32 0, i32 -1)
+  ret void
+}
diff --git a/llvm/test/Reduce/remove-bbs.ll b/llvm/test/Reduce/remove-bbs.ll
new file mode 100644
index 000000000..9e49bc531
--- /dev/null
+++ b/llvm/test/Reduce/remove-bbs.ll
@@ -0,0 +1,29 @@
+; Test that llvm-reduce can remove uninteresting Basic Blocks, and remove them from instructions (i.e. SwitchInst, BranchInst and IndirectBrInst)
+; Note: if an uninteresting BB is the default case for a switch, the instruction is removed altogether (since the default case cannot be replaced)
+;
+; RUN: llvm-reduce --test %python --test-arg %p/Inputs/remove-bbs.py %s -o %t
+; RUN: cat %t | FileCheck -implicit-check-not=uninteresting %s
+
+define void @main() {
+interesting:
+  ; CHECK-NOT: switch i32 0, label %uninteresting
+  switch i32 0, label %uninteresting [
+    i32 0, label %uninteresting
+  ]
+
+uninteresting:
+  ret void
+
+interesting2:
+  ; CHECK: switch i32 1, label %interesting3
+  switch i32 1, label %interesting3 [
+    ; CHECK-NOT: i32 0, label %uninteresting
+    i32 0, label %uninteresting
+    ; CHECK: i32 1, label %interesting3
+    i32 1, label %interesting3
+  ]
+
+interesting3:
+  ; CHECK: br label %interesting2
+  br i1 true, label %interesting2, label %uninteresting
+}
diff --git a/llvm/test/Reduce/remove-funcs.ll b/llvm/test/Reduce/remove-funcs.ll
new file mode 100644
index 000000000..8e9b2579a
--- /dev/null
+++ b/llvm/test/Reduce/remove-funcs.ll
@@ -0,0 +1,27 @@
+; Test that llvm-reduce can remove uninteresting functions as well as
+; their InstCalls.
+;
+; RUN: rm -rf %t
+; RUN: llvm-reduce --test %python --test-arg %p/Inputs/remove-funcs.py %s -o %t
+; RUN: cat %t | FileCheck -implicit-check-not=uninteresting %s
+
+define i32 @uninteresting1() {
+entry:
+  ret i32 0
+}
+
+; CHECK: interesting()
+define i32 @interesting() {
+entry:
+  ; CHECK: call i32 @interesting()
+  %call2 = call i32 @interesting()
+  %call = call i32 @uninteresting1()
+  ret i32 5
+}
+
+define i32 @uninteresting2() {
+entry:
+  ret i32 0
+}
+
+declare void @uninteresting3()
diff --git a/llvm/test/Reduce/remove-global-vars.ll b/llvm/test/Reduce/remove-global-vars.ll
new file mode 100644
index 000000000..921083fc9
--- /dev/null
+++ b/llvm/test/Reduce/remove-global-vars.ll
@@ -0,0 +1,25 @@
+; Test that llvm-reduce can remove uninteresting Global Variables as well as
+; their direct uses (which in turn are replaced with 'undef').
+;
+; RUN: rm -rf %t
+; RUN: llvm-reduce --test %python --test-arg %p/Inputs/remove-global-vars.py %s -o %t
+; RUN: cat %t | FileCheck -implicit-check-not=uninteresting %s
+
+; CHECK: @interesting = global
+@interesting = global i32 0, align 4
+@uninteresting = global i32 1, align 4
+
+define i32 @main() {
+entry:
+  %0 = load i32, i32* @uninteresting, align 4
+  ; CHECK: store i32 undef, i32* @interesting, align 4
+  store i32 %0, i32* @interesting, align 4
+
+  ; CHECK: load i32, i32* @interesting, align 4
+  %1 = load i32, i32* @interesting, align 4
+  store i32 %1, i32* @uninteresting, align 4
+
+  ; CHECK: store i32 5, i32* @interesting, align 4
+  store i32 5, i32* @interesting, align 4
+  ret i32 0
+}
diff --git a/llvm/test/Reduce/remove-instructions.ll b/llvm/test/Reduce/remove-instructions.ll
new file mode 100644
index 000000000..101351a89
--- /dev/null
+++ b/llvm/test/Reduce/remove-instructions.ll
@@ -0,0 +1,23 @@
+; Test that llvm-reduce can remove uninteresting instructions.
+;
+; RUN: llvm-reduce --test %python --test-arg %p/Inputs/remove-instructions.py %s -o %t
+; RUN: cat %t | FileCheck -implicit-check-not=uninteresting %s
+; REQUIRES: plugins
+
+; We're testing all direct uses of %interesting are conserved
+; CHECK-COUNT-5: %interesting
+define i32 @main() #0 {
+entry:
+  %uninteresting1 = alloca i32, align 4
+  %interesting = alloca i32, align 4
+  %uninteresting2 = alloca i32, align 4
+  store i32 0, i32* %uninteresting1, align 4
+  store i32 0, i32* %interesting, align 4
+  %0 = load i32, i32* %interesting, align 4
+  %uninteresting3 = add nsw i32 %0, 1
+  store i32 %uninteresting3, i32* %interesting, align 4
+  %1 = load i32, i32* %interesting, align 4
+  store i32 %1, i32* %uninteresting2, align 4
+  ; CHECK-NOT: ret
+  ret i32 0
+}
diff --git a/llvm/test/Reduce/remove-metadata.ll b/llvm/test/Reduce/remove-metadata.ll
new file mode 100644
index 000000000..da7d5a2f1
--- /dev/null
+++ b/llvm/test/Reduce/remove-metadata.ll
@@ -0,0 +1,20 @@
+; Test that llvm-reduce can remove uninteresting metadata from an IR file.
+; The Metadata pass erases named & unnamed metadata nodes.
+;
+; RUN: rm -rf %t
+; RUN: llvm-reduce --test %python --test-arg %p/Inputs/remove-metadata.py %s -o %t
+; RUN: cat %t | FileCheck -implicit-check-not=! %s
+
+@global = global i32 0, !dbg !0
+
+define void @main() !dbg !0 {
+   ret void, !dbg !0
+}
+
+!uninteresting = !{!0}
+; CHECK: !interesting = !{!0}
+!interesting = !{!1}
+
+!0 = !{!"uninteresting"}
+; CHECK: !0 = !{!"interesting"}
+!1 = !{!"interesting"}
diff --git a/llvm/test/ThinLTO/X86/Inputs/alias_internal.ll b/llvm/test/ThinLTO/X86/Inputs/alias_internal.ll
index e55e40b1d..88bc8335f 100644
--- a/llvm/test/ThinLTO/X86/Inputs/alias_internal.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/alias_internal.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define internal i32 @f(i8*) unnamed_addr {
diff --git a/llvm/test/ThinLTO/X86/Inputs/cache-icall.ll b/llvm/test/ThinLTO/X86/Inputs/cache-icall.ll
index 43608d5bf..489db0627 100644
--- a/llvm/test/ThinLTO/X86/Inputs/cache-icall.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/cache-icall.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define void @f() !type !0 {
diff --git a/llvm/test/ThinLTO/X86/Inputs/cache-import-lists1.ll b/llvm/test/ThinLTO/X86/Inputs/cache-import-lists1.ll
index 58bfb39f9..f0ebfe29d 100644
--- a/llvm/test/ThinLTO/X86/Inputs/cache-import-lists1.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/cache-import-lists1.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define void @f1() {
diff --git a/llvm/test/ThinLTO/X86/Inputs/cache-import-lists2.ll b/llvm/test/ThinLTO/X86/Inputs/cache-import-lists2.ll
index 899bbaea1..258a20be1 100644
--- a/llvm/test/ThinLTO/X86/Inputs/cache-import-lists2.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/cache-import-lists2.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define void @f2() {
diff --git a/llvm/test/ThinLTO/X86/Inputs/cache-typeid-resolutions-import.ll b/llvm/test/ThinLTO/X86/Inputs/cache-typeid-resolutions-import.ll
index 95ecd1824..2cf9a2d86 100644
--- a/llvm/test/ThinLTO/X86/Inputs/cache-typeid-resolutions-import.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/cache-typeid-resolutions-import.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define i1 @importf1(i8* %p) {
diff --git a/llvm/test/ThinLTO/X86/Inputs/cache-typeid-resolutions1.ll b/llvm/test/ThinLTO/X86/Inputs/cache-typeid-resolutions1.ll
index e53673bcd..679b6b3c6 100644
--- a/llvm/test/ThinLTO/X86/Inputs/cache-typeid-resolutions1.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/cache-typeid-resolutions1.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @vt1 = constant i32 0, !type !0
diff --git a/llvm/test/ThinLTO/X86/Inputs/cache-typeid-resolutions2.ll b/llvm/test/ThinLTO/X86/Inputs/cache-typeid-resolutions2.ll
index 283badad3..067804293 100644
--- a/llvm/test/ThinLTO/X86/Inputs/cache-typeid-resolutions2.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/cache-typeid-resolutions2.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @vt2 = constant i1 (i8*)* @vf2, !type !0
diff --git a/llvm/test/ThinLTO/X86/Inputs/cache-typeid-resolutions3.ll b/llvm/test/ThinLTO/X86/Inputs/cache-typeid-resolutions3.ll
index 830622e9c..c3de5205e 100644
--- a/llvm/test/ThinLTO/X86/Inputs/cache-typeid-resolutions3.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/cache-typeid-resolutions3.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @vt2a = constant i1 (i8*)* @vf2a, !type !0
diff --git a/llvm/test/ThinLTO/X86/Inputs/cache.ll b/llvm/test/ThinLTO/X86/Inputs/cache.ll
index aab44cb80..62ddd68ef 100644
--- a/llvm/test/ThinLTO/X86/Inputs/cache.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/cache.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.11.0"
 
 
diff --git a/llvm/test/ThinLTO/X86/Inputs/callees-metadata.ll b/llvm/test/ThinLTO/X86/Inputs/callees-metadata.ll
index 1e77b0886..7718347e3 100644
--- a/llvm/test/ThinLTO/X86/Inputs/callees-metadata.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/callees-metadata.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @fptr = internal unnamed_addr global i32 (i32)* @f2, align 8
diff --git a/llvm/test/ThinLTO/X86/Inputs/cfi-distributed.ll b/llvm/test/ThinLTO/X86/Inputs/cfi-distributed.ll
index cc1c4fd78..18554a009 100644
--- a/llvm/test/ThinLTO/X86/Inputs/cfi-distributed.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/cfi-distributed.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-grtev4-linux-gnu"
 
 %struct.B2 = type { %struct.A2 }
diff --git a/llvm/test/ThinLTO/X86/Inputs/cfi-icall-only-bazqux.ll b/llvm/test/ThinLTO/X86/Inputs/cfi-icall-only-bazqux.ll
index 06fb015b2..94186ae16 100644
--- a/llvm/test/ThinLTO/X86/Inputs/cfi-icall-only-bazqux.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/cfi-icall-only-bazqux.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 declare !type !0 i8 @bar(i8*)
diff --git a/llvm/test/ThinLTO/X86/Inputs/crash_debuginfo.ll b/llvm/test/ThinLTO/X86/Inputs/crash_debuginfo.ll
index 4880cd98d..70d3edd7e 100644
--- a/llvm/test/ThinLTO/X86/Inputs/crash_debuginfo.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/crash_debuginfo.ll
@@ -1,6 +1,6 @@
 ; ModuleID = 'test/ThinLTO/X86/Inputs/crash_debuginfo.ll'
 source_filename = "src.bc"
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.7.0"
 
 define void @bar(i32 %arg) {
diff --git a/llvm/test/ThinLTO/X86/Inputs/deadstrip.ll b/llvm/test/ThinLTO/X86/Inputs/deadstrip.ll
index 057305ae6..34211802a 100644
--- a/llvm/test/ThinLTO/X86/Inputs/deadstrip.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/deadstrip.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.11.0"
 
 declare void @dead_func()
diff --git a/llvm/test/ThinLTO/X86/Inputs/debuginfo-compositetype-import.ll b/llvm/test/ThinLTO/X86/Inputs/debuginfo-compositetype-import.ll
index 7e80c4d7c..5b1e16b47 100644
--- a/llvm/test/ThinLTO/X86/Inputs/debuginfo-compositetype-import.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/debuginfo-compositetype-import.ll
@@ -1,6 +1,6 @@
 ; ModuleID = 'debuginfo-compositetype-import2.c'
 source_filename = "debuginfo-compositetype-import2.c"
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 ; Function Attrs: nounwind uwtable
diff --git a/llvm/test/ThinLTO/X86/Inputs/debuginfo-cu-import.ll b/llvm/test/ThinLTO/X86/Inputs/debuginfo-cu-import.ll
index 6b824e940..243707249 100644
--- a/llvm/test/ThinLTO/X86/Inputs/debuginfo-cu-import.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/debuginfo-cu-import.ll
@@ -1,6 +1,6 @@
 ; ModuleID = 'debuginfo-cu-import2.c'
 source_filename = "debuginfo-cu-import2.c"
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 ; Function Attrs: nounwind uwtable
diff --git a/llvm/test/ThinLTO/X86/Inputs/devirt2.ll b/llvm/test/ThinLTO/X86/Inputs/devirt2.ll
new file mode 100644
index 000000000..a67e80316
--- /dev/null
+++ b/llvm/test/ThinLTO/X86/Inputs/devirt2.ll
@@ -0,0 +1,59 @@
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-grtev4-linux-gnu"
+
+%struct.A = type { i32 (...)** }
+%struct.B = type { %struct.A }
+%struct.C = type { %struct.A }
+%struct.D = type { i32 (...)** }
+%struct.E = type { i32 (...)** }
+
+@_ZTV1B = constant { [4 x i8*] } { [4 x i8*] [i8* null, i8* undef, i8* bitcast (i32 (%struct.B*, i32)* @_ZN1B1fEi to i8*), i8* bitcast (i32 (%struct.A*, i32)* @_ZN1A1nEi to i8*)] }, !type !0, !type !1
+@_ZTV1C = constant { [4 x i8*] } { [4 x i8*] [i8* null, i8* undef, i8* bitcast (i32 (%struct.C*, i32)* @_ZN1C1fEi to i8*), i8* bitcast (i32 (%struct.A*, i32)* @_ZN1A1nEi to i8*)] }, !type !0, !type !2
+@_ZTV1D = linkonce_odr constant { [3 x i8*] } { [3 x i8*] [i8* null, i8* undef, i8* bitcast (i32 (%struct.D*, i32)* @_ZN1D1mEi to i8*)] }, !type !3
+@_ZTV1E = constant { [3 x i8*] } { [3 x i8*] [i8* null, i8* undef, i8* bitcast (i32 (%struct.E*, i32)* @_ZN1E1mEi to i8*)] }, !type !4
+
+define i32 @_ZN1B1fEi(%struct.B* %this, i32 %a) #0 {
+   ret i32 0;
+}
+
+define internal i32 @_ZN1A1nEi(%struct.A* %this, i32 %a) #0 {
+   ret i32 0;
+}
+
+define i32 @_ZN1C1fEi(%struct.C* %this, i32 %a) #0 {
+   ret i32 0;
+}
+
+define linkonce_odr i32 @_ZN1D1mEi(%struct.D* %this, i32 %a) #0 {
+   ret i32 0;
+}
+
+define internal i32 @_ZN1E1mEi(%struct.E* %this, i32 %a) #0 {
+   ret i32 0;
+}
+
+define i32 @test2(%struct.E* %obj, i32 %a) {
+entry:
+  %0 = bitcast %struct.E* %obj to i8***
+  %vtable2 = load i8**, i8*** %0
+  %1 = bitcast i8** %vtable2 to i8*
+  %p2 = call i1 @llvm.type.test(i8* %1, metadata !"_ZTS1E")
+  call void @llvm.assume(i1 %p2)
+
+  %2 = bitcast i8** %vtable2 to i32 (%struct.E*, i32)**
+  %fptr33 = load i32 (%struct.E*, i32)*, i32 (%struct.E*, i32)** %2, align 8
+
+  %call4 = tail call i32 %fptr33(%struct.E* nonnull %obj, i32 %a)
+  ret i32 %call4
+}
+
+attributes #0 = { noinline optnone }
+
+declare i1 @llvm.type.test(i8*, metadata)
+declare void @llvm.assume(i1)
+
+!0 = !{i64 16, !"_ZTS1A"}
+!1 = !{i64 16, !"_ZTS1B"}
+!2 = !{i64 16, !"_ZTS1C"}
+!3 = !{i64 16, !"_ZTS1D"}
+!4 = !{i64 16, !"_ZTS1E"}
diff --git a/llvm/test/ThinLTO/X86/Inputs/devirt_alias.ll b/llvm/test/ThinLTO/X86/Inputs/devirt_alias.ll
new file mode 100644
index 000000000..869751b40
--- /dev/null
+++ b/llvm/test/ThinLTO/X86/Inputs/devirt_alias.ll
@@ -0,0 +1,15 @@
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-grtev4-linux-gnu"
+
+%struct.D = type { i32 (...)** }
+
+@some_name = constant { [3 x i8*] } { [3 x i8*] [i8* null, i8* undef, i8* bitcast (i32 (%struct.D*, i32)* @_ZN1D1mEi to i8*)] }, !type !3
+@_ZTV1D = alias { [3 x i8*] }, { [3 x i8*] }* @some_name
+
+define i32 @_ZN1D1mEi(%struct.D* %this, i32 %a) #0 {
+   ret i32 0;
+}
+
+attributes #0 = { noinline optnone }
+
+!3 = !{i64 16, !"_ZTS1D"}
diff --git a/llvm/test/ThinLTO/X86/Inputs/devirt_available_externally.ll b/llvm/test/ThinLTO/X86/Inputs/devirt_available_externally.ll
new file mode 100644
index 000000000..56d6ea5fc
--- /dev/null
+++ b/llvm/test/ThinLTO/X86/Inputs/devirt_available_externally.ll
@@ -0,0 +1,14 @@
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-grtev4-linux-gnu"
+
+%struct.D = type { i32 (...)** }
+
+@_ZTV1D = constant { [3 x i8*] } { [3 x i8*] [i8* null, i8* undef, i8* bitcast (i32 (%struct.D*, i32)* @_ZN1D1mEi to i8*)] }, !type !3
+
+define i32 @_ZN1D1mEi(%struct.D* %this, i32 %a) #0 {
+   ret i32 0;
+}
+
+attributes #0 = { noinline optnone }
+
+!3 = !{i64 16, !"_ZTS1D"}
diff --git a/llvm/test/ThinLTO/X86/Inputs/devirt_external_comdat_same_guid.ll b/llvm/test/ThinLTO/X86/Inputs/devirt_external_comdat_same_guid.ll
new file mode 100644
index 000000000..a25563926
--- /dev/null
+++ b/llvm/test/ThinLTO/X86/Inputs/devirt_external_comdat_same_guid.ll
@@ -0,0 +1,43 @@
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-grtev4-linux-gnu"
+
+source_filename = "-"
+
+%struct.A = type { i32 (...)** }
+%struct.B = type { %struct.A }
+
+$_ZTV1B = comdat any
+
+@_ZTV1B = constant { [4 x i8*] } { [4 x i8*] [i8* null, i8* undef, i8* bitcast (i32 (%struct.B*, i32)* @_ZN1B1fEi to i8*), i8* bitcast (i32 (%struct.A*, i32)* @_ZN1A1nEi to i8*)] }, comdat, !type !0, !type !1
+
+define i32 @_ZN1B1fEi(%struct.B* %this, i32 %a) #0 comdat($_ZTV1B) {
+   ret i32 0;
+}
+
+define i32 @_ZN1A1nEi(%struct.A* %this, i32 %a) #0 comdat($_ZTV1B) {
+   ret i32 0;
+}
+
+define i32 @test2(%struct.B* %obj, i32 %a) {
+entry:
+  %0 = bitcast %struct.B* %obj to i8***
+  %vtable2 = load i8**, i8*** %0
+  %1 = bitcast i8** %vtable2 to i8*
+  %p2 = call i1 @llvm.type.test(i8* %1, metadata !"_ZTS1B")
+  call void @llvm.assume(i1 %p2)
+
+  %fptrptr = getelementptr i8*, i8** %vtable2, i32 1
+  %2 = bitcast i8** %fptrptr to i32 (%struct.B*, i32)**
+  %fptr33 = load i32 (%struct.B*, i32)*, i32 (%struct.B*, i32)** %2, align 8
+
+  %call4 = tail call i32 %fptr33(%struct.B* nonnull %obj, i32 %a)
+  ret i32 %call4
+}
+
+attributes #0 = { noinline optnone }
+
+declare i1 @llvm.type.test(i8*, metadata)
+declare void @llvm.assume(i1)
+
+!0 = !{i64 16, !"_ZTS1A"}
+!1 = !{i64 16, !"_ZTS1B"}
diff --git a/llvm/test/ThinLTO/X86/Inputs/devirt_local_same_guid.ll b/llvm/test/ThinLTO/X86/Inputs/devirt_local_same_guid.ll
new file mode 100644
index 000000000..cf3d024c3
--- /dev/null
+++ b/llvm/test/ThinLTO/X86/Inputs/devirt_local_same_guid.ll
@@ -0,0 +1,41 @@
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-grtev4-linux-gnu"
+
+source_filename = "-"
+
+%struct.A = type { i32 (...)** }
+%struct.B = type { %struct.A }
+
+@_ZTV1B = internal constant { [4 x i8*] } { [4 x i8*] [i8* null, i8* undef, i8* bitcast (i32 (%struct.B*, i32)* @_ZN1B1fEi to i8*), i8* bitcast (i32 (%struct.A*, i32)* @_ZN1A1nEi to i8*)] }, !type !0, !type !1
+
+define internal i32 @_ZN1B1fEi(%struct.B* %this, i32 %a) #0 {
+   ret i32 0;
+}
+
+define internal i32 @_ZN1A1nEi(%struct.A* %this, i32 %a) #0 {
+   ret i32 0;
+}
+
+define i32 @test2(%struct.B* %obj, i32 %a) {
+entry:
+  %0 = bitcast %struct.B* %obj to i8***
+  %vtable2 = load i8**, i8*** %0
+  %1 = bitcast i8** %vtable2 to i8*
+  %p2 = call i1 @llvm.type.test(i8* %1, metadata !"_ZTS1B")
+  call void @llvm.assume(i1 %p2)
+
+  %fptrptr = getelementptr i8*, i8** %vtable2, i32 1
+  %2 = bitcast i8** %fptrptr to i32 (%struct.B*, i32)**
+  %fptr33 = load i32 (%struct.B*, i32)*, i32 (%struct.B*, i32)** %2, align 8
+
+  %call4 = tail call i32 %fptr33(%struct.B* nonnull %obj, i32 %a)
+  ret i32 %call4
+}
+
+attributes #0 = { noinline optnone }
+
+declare i1 @llvm.type.test(i8*, metadata)
+declare void @llvm.assume(i1)
+
+!0 = !{i64 16, !"_ZTS1A"}
+!1 = !{i64 16, !"_ZTS1B"}
diff --git a/llvm/test/ThinLTO/X86/Inputs/devirt_promote.ll b/llvm/test/ThinLTO/X86/Inputs/devirt_promote.ll
new file mode 100644
index 000000000..bcd49aefe
--- /dev/null
+++ b/llvm/test/ThinLTO/X86/Inputs/devirt_promote.ll
@@ -0,0 +1,39 @@
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-grtev4-linux-gnu"
+
+%struct.A = type { i32 (...)** }
+%struct.B = type { %struct.A }
+
+@_ZTV1B = constant { [4 x i8*] } { [4 x i8*] [i8* null, i8* undef, i8* bitcast (i32 (%struct.B*, i32)* @_ZN1B1fEi to i8*), i8* bitcast (i32 (%struct.A*, i32)* @_ZN1A1nEi to i8*)] }, !type !0, !type !1
+
+define i32 @_ZN1B1fEi(%struct.B* %this, i32 %a) #0 {
+   ret i32 0;
+}
+
+define internal i32 @_ZN1A1nEi(%struct.A* %this, i32 %a) #0 {
+   ret i32 0;
+}
+
+define i32 @test2(%struct.B* %obj, i32 %a) {
+entry:
+  %0 = bitcast %struct.B* %obj to i8***
+  %vtable2 = load i8**, i8*** %0
+  %1 = bitcast i8** %vtable2 to i8*
+  %p2 = call i1 @llvm.type.test(i8* %1, metadata !"_ZTS1B")
+  call void @llvm.assume(i1 %p2)
+
+  %fptrptr = getelementptr i8*, i8** %vtable2, i32 1
+  %2 = bitcast i8** %fptrptr to i32 (%struct.B*, i32)**
+  %fptr33 = load i32 (%struct.B*, i32)*, i32 (%struct.B*, i32)** %2, align 8
+
+  %call4 = tail call i32 %fptr33(%struct.B* nonnull %obj, i32 %a)
+  ret i32 %call4
+}
+
+attributes #0 = { noinline optnone }
+
+declare i1 @llvm.type.test(i8*, metadata)
+declare void @llvm.assume(i1)
+
+!0 = !{i64 16, !"_ZTS1A"}
+!1 = !{i64 16, !"_ZTS1B"}
diff --git a/llvm/test/ThinLTO/X86/Inputs/devirt_single_hybrid_bar.ll b/llvm/test/ThinLTO/X86/Inputs/devirt_single_hybrid_bar.ll
new file mode 100644
index 000000000..414e7d5dd
--- /dev/null
+++ b/llvm/test/ThinLTO/X86/Inputs/devirt_single_hybrid_bar.ll
@@ -0,0 +1,58 @@
+; ModuleID = 'bar.cpp'
+source_filename = "bar.cpp"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+%struct.A = type { i32 (...)** }
+
+$_ZNK1A1fEv = comdat any
+
+$_ZTV1A = comdat any
+
+$_ZTS1A = comdat any
+
+$_ZTI1A = comdat any
+
+@_ZTV1A = linkonce_odr hidden unnamed_addr constant { [3 x i8*] } { [3 x i8*] [i8* null, i8* bitcast ({ i8*, i8* }* @_ZTI1A to i8*), i8* bitcast (i32 (%struct.A*)* @_ZNK1A1fEv to i8*)] }, comdat, align 8, !type !0, !type !1
+@_ZTVN10__cxxabiv117__class_type_infoE = external dso_local global i8*
+@_ZTS1A = linkonce_odr hidden constant [3 x i8] c"1A\00", comdat, align 1
+@_ZTI1A = linkonce_odr hidden constant { i8*, i8* } { i8* bitcast (i8** getelementptr inbounds (i8*, i8** @_ZTVN10__cxxabiv117__class_type_infoE, i64 2) to i8*), i8* getelementptr inbounds ([3 x i8], [3 x i8]* @_ZTS1A, i32 0, i32 0) }, comdat, align 8
+
+; Function Attrs: uwtable
+define hidden i32 @_Z3barv() local_unnamed_addr #0 {
+entry:
+  %b = alloca %struct.A, align 8
+  %0 = bitcast %struct.A* %b to i8*
+  call void @llvm.lifetime.start.p0i8(i64 8, i8* nonnull %0)
+  %1 = getelementptr inbounds %struct.A, %struct.A* %b, i64 0, i32 0
+  store i32 (...)** bitcast (i8** getelementptr inbounds ({ [3 x i8*] }, { [3 x i8*] }* @_ZTV1A, i64 0, inrange i32 0, i64 2) to i32 (...)**), i32 (...)*** %1, align 8, !tbaa !4
+  %call = call i32 @_Z3fooP1A(%struct.A* nonnull %b)
+  %add = add nsw i32 %call, 10
+  call void @llvm.lifetime.end.p0i8(i64 8, i8* nonnull %0) #4
+  ret i32 %add
+}
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.lifetime.start.p0i8(i64 immarg, i8* nocapture)
+
+declare dso_local i32 @_Z3fooP1A(%struct.A*) local_unnamed_addr
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.lifetime.end.p0i8(i64 immarg, i8* nocapture)
+
+; Function Attrs: nounwind uwtable
+define linkonce_odr hidden i32 @_ZNK1A1fEv(%struct.A* %this) unnamed_addr comdat align 2 {
+entry:
+  ret i32 3
+}
+
+!llvm.module.flags = !{!2}
+!llvm.ident = !{!3}
+
+!0 = !{i64 16, !"_ZTS1A"}
+!1 = !{i64 16, !"_ZTSM1AKFivE.virtual"}
+!2 = !{i32 1, !"wchar_size", i32 4}
+!3 = !{!"clang version 10.0.0 (trunk 373596)"}
+!4 = !{!5, !5, i64 0}
+!5 = !{!"vtable pointer", !6, i64 0}
+!6 = !{!"Simple C++ TBAA"}
diff --git a/llvm/test/ThinLTO/X86/Inputs/devirt_single_hybrid_foo.ll b/llvm/test/ThinLTO/X86/Inputs/devirt_single_hybrid_foo.ll
new file mode 100644
index 000000000..d29f74b56
--- /dev/null
+++ b/llvm/test/ThinLTO/X86/Inputs/devirt_single_hybrid_foo.ll
@@ -0,0 +1,35 @@
+; ModuleID = 'foo.cpp'
+source_filename = "foo.cpp"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+%struct.A = type { i32 (...)** }
+
+; Function Attrs: uwtable
+define hidden i32 @_Z3fooP1A(%struct.A* %pA) local_unnamed_addr {
+entry:
+  %0 = bitcast %struct.A* %pA to i32 (%struct.A*)***
+  %vtable = load i32 (%struct.A*)**, i32 (%struct.A*)*** %0, align 8, !tbaa !2
+  %1 = bitcast i32 (%struct.A*)** %vtable to i8*
+  %2 = tail call i1 @llvm.type.test(i8* %1, metadata !"_ZTS1A")
+  tail call void @llvm.assume(i1 %2)
+  %3 = load i32 (%struct.A*)*, i32 (%struct.A*)** %vtable, align 8
+  %call = tail call i32 %3(%struct.A* %pA)
+  %add = add nsw i32 %call, 10
+  ret i32 %add
+}
+
+; Function Attrs: nounwind readnone willreturn
+declare i1 @llvm.type.test(i8*, metadata)
+
+; Function Attrs: nounwind willreturn
+declare void @llvm.assume(i1)
+
+!llvm.module.flags = !{!0}
+!llvm.ident = !{!1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{!"clang version 10.0.0 (trunk 373596)"}
+!2 = !{!3, !3, i64 0}
+!3 = !{!"vtable pointer", !4, i64 0}
+!4 = !{!"Simple C++ TBAA"}
diff --git a/llvm/test/ThinLTO/X86/Inputs/diagnostic-handler-remarks.ll b/llvm/test/ThinLTO/X86/Inputs/diagnostic-handler-remarks.ll
index 35154e679..c6ef6d1c7 100644
--- a/llvm/test/ThinLTO/X86/Inputs/diagnostic-handler-remarks.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/diagnostic-handler-remarks.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.11.0"
 
 
diff --git a/llvm/test/ThinLTO/X86/Inputs/dicompositetype-unique-alias.ll b/llvm/test/ThinLTO/X86/Inputs/dicompositetype-unique-alias.ll
index 8fa97d4f0..ee1c272d3 100644
--- a/llvm/test/ThinLTO/X86/Inputs/dicompositetype-unique-alias.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/dicompositetype-unique-alias.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-scei-ps4"
 
 %struct.CFVS = type { %struct.Vec }
diff --git a/llvm/test/ThinLTO/X86/Inputs/dicompositetype-unique.ll b/llvm/test/ThinLTO/X86/Inputs/dicompositetype-unique.ll
index e27a7b162..8d31eae6e 100644
--- a/llvm/test/ThinLTO/X86/Inputs/dicompositetype-unique.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/dicompositetype-unique.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.11.0"
 
 
diff --git a/llvm/test/ThinLTO/X86/Inputs/dicompositetype-unique2.ll b/llvm/test/ThinLTO/X86/Inputs/dicompositetype-unique2.ll
index 507fa13c6..1d2f58497 100644
--- a/llvm/test/ThinLTO/X86/Inputs/dicompositetype-unique2.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/dicompositetype-unique2.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-scei-ps4"
 
 %struct.CFVS = type { %struct.Vec }
diff --git a/llvm/test/ThinLTO/X86/Inputs/dot-dumper.ll b/llvm/test/ThinLTO/X86/Inputs/dot-dumper.ll
index d149de2a9..4e745a459 100644
--- a/llvm/test/ThinLTO/X86/Inputs/dot-dumper.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/dot-dumper.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @A = local_unnamed_addr global i32 10, align 4
diff --git a/llvm/test/ThinLTO/X86/Inputs/dot-dumper2.ll b/llvm/test/ThinLTO/X86/Inputs/dot-dumper2.ll
index 6d9783e92..854c1e4a2 100644
--- a/llvm/test/ThinLTO/X86/Inputs/dot-dumper2.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/dot-dumper2.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @A = local_unnamed_addr global i32 10, align 4
diff --git a/llvm/test/ThinLTO/X86/Inputs/drop-debug-info.ll b/llvm/test/ThinLTO/X86/Inputs/drop-debug-info.ll
index cf3b29515..ad9c48597 100644
--- a/llvm/test/ThinLTO/X86/Inputs/drop-debug-info.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/drop-debug-info.ll
@@ -1,7 +1,7 @@
 ; This file is checked-in as a .bc file, because the debug info version is
 ; intentionally out-of-date and llvm-as will drop it before writing the bitcode
 
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.11.0"
 
 
diff --git a/llvm/test/ThinLTO/X86/Inputs/dsolocal_dllimport.ll b/llvm/test/ThinLTO/X86/Inputs/dsolocal_dllimport.ll
index 5e08d9812..1dfb7c5b9 100644
--- a/llvm/test/ThinLTO/X86/Inputs/dsolocal_dllimport.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/dsolocal_dllimport.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:w-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-pc-windows-msvc19.11.0"
 define dso_local dllexport i32 @foo() {
   ret i32 42
diff --git a/llvm/test/ThinLTO/X86/Inputs/emit_imports.ll b/llvm/test/ThinLTO/X86/Inputs/emit_imports.ll
index 31c72ec46..7cf01bc86 100644
--- a/llvm/test/ThinLTO/X86/Inputs/emit_imports.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/emit_imports.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define void @g() {
diff --git a/llvm/test/ThinLTO/X86/Inputs/emit_imports2.ll b/llvm/test/ThinLTO/X86/Inputs/emit_imports2.ll
index 2136ec347..8776860d7 100644
--- a/llvm/test/ThinLTO/X86/Inputs/emit_imports2.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/emit_imports2.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define void @h() {
diff --git a/llvm/test/ThinLTO/X86/Inputs/empty.ll b/llvm/test/ThinLTO/X86/Inputs/empty.ll
index a3c99cdfe..a4e314042 100644
--- a/llvm/test/ThinLTO/X86/Inputs/empty.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/empty.ll
@@ -1,2 +1,2 @@
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/ThinLTO/X86/Inputs/empty_module_with_cache.ll b/llvm/test/ThinLTO/X86/Inputs/empty_module_with_cache.ll
index 1c55e078e..276cf61d0 100644
--- a/llvm/test/ThinLTO/X86/Inputs/empty_module_with_cache.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/empty_module_with_cache.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.11.0"
 
 
diff --git a/llvm/test/ThinLTO/X86/Inputs/export.ll b/llvm/test/ThinLTO/X86/Inputs/export.ll
index cab8591fd..d99a76c3f 100644
--- a/llvm/test/ThinLTO/X86/Inputs/export.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/export.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.11.0"
 
 define i32 @main() #0 {
diff --git a/llvm/test/ThinLTO/X86/Inputs/funcimport-debug.ll b/llvm/test/ThinLTO/X86/Inputs/funcimport-debug.ll
index b204afe3d..022d10641 100644
--- a/llvm/test/ThinLTO/X86/Inputs/funcimport-debug.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/funcimport-debug.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.11.0"
 
 
diff --git a/llvm/test/ThinLTO/X86/Inputs/funcimport-tbaa.ll b/llvm/test/ThinLTO/X86/Inputs/funcimport-tbaa.ll
index 72aea1e5e..4ecfa7886 100644
--- a/llvm/test/ThinLTO/X86/Inputs/funcimport-tbaa.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/funcimport-tbaa.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.11.0"
 
 
diff --git a/llvm/test/ThinLTO/X86/Inputs/funcimport.ll b/llvm/test/ThinLTO/X86/Inputs/funcimport.ll
index e0df9e0e7..1a79dbe14 100644
--- a/llvm/test/ThinLTO/X86/Inputs/funcimport.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/funcimport.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.11.0"
 
 
diff --git a/llvm/test/ThinLTO/X86/Inputs/funcimport2.ll b/llvm/test/ThinLTO/X86/Inputs/funcimport2.ll
index 216ef0f1d..dcebd1848 100644
--- a/llvm/test/ThinLTO/X86/Inputs/funcimport2.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/funcimport2.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.11.0"
 
 
diff --git a/llvm/test/ThinLTO/X86/Inputs/funcimport_alwaysinline.ll b/llvm/test/ThinLTO/X86/Inputs/funcimport_alwaysinline.ll
new file mode 100644
index 000000000..31ca7f5b3
--- /dev/null
+++ b/llvm/test/ThinLTO/X86/Inputs/funcimport_alwaysinline.ll
@@ -0,0 +1,10 @@
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define i32 @main() {
+entry:
+  call void (...) @foo()
+  ret i32 0
+}
+
+declare void @foo(...)
diff --git a/llvm/test/ThinLTO/X86/Inputs/function_entry_count.ll b/llvm/test/ThinLTO/X86/Inputs/function_entry_count.ll
index ea2c2eda8..87418dd30 100644
--- a/llvm/test/ThinLTO/X86/Inputs/function_entry_count.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/function_entry_count.ll
@@ -1,5 +1,5 @@
 target triple = "x86_64-unknown-linux-gnu"
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 
 declare void @h();
 
diff --git a/llvm/test/ThinLTO/X86/Inputs/globals-import-blockaddr.ll b/llvm/test/ThinLTO/X86/Inputs/globals-import-blockaddr.ll
index fe1fa70ee..3349549f6 100644
--- a/llvm/test/ThinLTO/X86/Inputs/globals-import-blockaddr.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/globals-import-blockaddr.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @label_addr = internal constant [1 x i8*] [i8* blockaddress(@foo, %lb)], align 8
diff --git a/llvm/test/ThinLTO/X86/Inputs/globals-import-cf-baz.ll b/llvm/test/ThinLTO/X86/Inputs/globals-import-cf-baz.ll
index e41163063..5a3e5001f 100644
--- a/llvm/test/ThinLTO/X86/Inputs/globals-import-cf-baz.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/globals-import-cf-baz.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-pc-linux-gnu"
 
 @baz = local_unnamed_addr constant i32 10, align 4
diff --git a/llvm/test/ThinLTO/X86/Inputs/globals-import.ll b/llvm/test/ThinLTO/X86/Inputs/globals-import.ll
index b229f4a4b..ec4ff6eb3 100644
--- a/llvm/test/ThinLTO/X86/Inputs/globals-import.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/globals-import.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-pc-linux-gnu"
 
 @baz = internal constant i32 10, align 4
diff --git a/llvm/test/ThinLTO/X86/Inputs/guid_collision.ll b/llvm/test/ThinLTO/X86/Inputs/guid_collision.ll
new file mode 100644
index 000000000..3d46c7b53
--- /dev/null
+++ b/llvm/test/ThinLTO/X86/Inputs/guid_collision.ll
@@ -0,0 +1,15 @@
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-linux-gnu"
+
+; The source for the GUID for this symbol will be -:F
+source_filename = "-"
+define internal fastcc i64 @F() {
+  ret i64 0
+}
+
+@llvm.global_ctors = appending global [0 x { i32, void ()*, i8* }] zeroinitializer
+
+define i64 @G() {
+  ;%1 = load i32, i32* @dummy2, align 4
+  ret i64 0
+}
diff --git a/llvm/test/ThinLTO/X86/Inputs/import_opaque_type.ll b/llvm/test/ThinLTO/X86/Inputs/import_opaque_type.ll
index fe2b29347..5c452d58e 100644
--- a/llvm/test/ThinLTO/X86/Inputs/import_opaque_type.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/import_opaque_type.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.11.0"
 
 %0 = type { i8 }
diff --git a/llvm/test/ThinLTO/X86/Inputs/index-const-prop-alias.ll b/llvm/test/ThinLTO/X86/Inputs/index-const-prop-alias.ll
index d5aa6ee0b..8d76b450c 100644
--- a/llvm/test/ThinLTO/X86/Inputs/index-const-prop-alias.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/index-const-prop-alias.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @g = global i32 42, align 4
diff --git a/llvm/test/ThinLTO/X86/Inputs/index-const-prop-cache-foo.ll b/llvm/test/ThinLTO/X86/Inputs/index-const-prop-cache-foo.ll
index 5f85f1e38..a306b5a34 100644
--- a/llvm/test/ThinLTO/X86/Inputs/index-const-prop-cache-foo.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/index-const-prop-cache-foo.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @gFoo = internal unnamed_addr global i32 1, align 4
diff --git a/llvm/test/ThinLTO/X86/Inputs/index-const-prop-cache-test1.ll b/llvm/test/ThinLTO/X86/Inputs/index-const-prop-cache-test1.ll
index 6bcf3b32d..a7a21f933 100644
--- a/llvm/test/ThinLTO/X86/Inputs/index-const-prop-cache-test1.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/index-const-prop-cache-test1.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 ; Function Attrs: nounwind ssp uwtable
diff --git a/llvm/test/ThinLTO/X86/Inputs/index-const-prop-cache-test2.ll b/llvm/test/ThinLTO/X86/Inputs/index-const-prop-cache-test2.ll
index 75a8f071f..05a168fc8 100644
--- a/llvm/test/ThinLTO/X86/Inputs/index-const-prop-cache-test2.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/index-const-prop-cache-test2.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 ; Function Attrs: nounwind ssp uwtable
diff --git a/llvm/test/ThinLTO/X86/Inputs/index-const-prop-comdat.ll b/llvm/test/ThinLTO/X86/Inputs/index-const-prop-comdat.ll
index 819ba8caa..44c9b3e26 100644
--- a/llvm/test/ThinLTO/X86/Inputs/index-const-prop-comdat.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/index-const-prop-comdat.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 $comdat.any = comdat any
diff --git a/llvm/test/ThinLTO/X86/Inputs/index-const-prop-define-g.ll b/llvm/test/ThinLTO/X86/Inputs/index-const-prop-define-g.ll
index 555f76f59..68947b4c3 100644
--- a/llvm/test/ThinLTO/X86/Inputs/index-const-prop-define-g.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/index-const-prop-define-g.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @g = global i32 42, align 4
diff --git a/llvm/test/ThinLTO/X86/Inputs/index-const-prop-full-lto.ll b/llvm/test/ThinLTO/X86/Inputs/index-const-prop-full-lto.ll
index e22f91989..df57a897b 100644
--- a/llvm/test/ThinLTO/X86/Inputs/index-const-prop-full-lto.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/index-const-prop-full-lto.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @g = external global i32
diff --git a/llvm/test/ThinLTO/X86/Inputs/index-const-prop-gvref.ll b/llvm/test/ThinLTO/X86/Inputs/index-const-prop-gvref.ll
index ec0de9c91..53020763d 100644
--- a/llvm/test/ThinLTO/X86/Inputs/index-const-prop-gvref.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/index-const-prop-gvref.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @b = global i32* @a, align 8
diff --git a/llvm/test/ThinLTO/X86/Inputs/index-const-prop-linkage.ll b/llvm/test/ThinLTO/X86/Inputs/index-const-prop-linkage.ll
index 24c724743..7f74ca671 100644
--- a/llvm/test/ThinLTO/X86/Inputs/index-const-prop-linkage.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/index-const-prop-linkage.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @g1 = common global i32 0, align 4
diff --git a/llvm/test/ThinLTO/X86/Inputs/index-const-prop.ll b/llvm/test/ThinLTO/X86/Inputs/index-const-prop.ll
index eda62fb4e..90e176334 100644
--- a/llvm/test/ThinLTO/X86/Inputs/index-const-prop.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/index-const-prop.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-pc-linux-gnu"
 
 @gBar = local_unnamed_addr global i32 2, align 4, !dbg !0
diff --git a/llvm/test/ThinLTO/X86/Inputs/internalize.ll b/llvm/test/ThinLTO/X86/Inputs/internalize.ll
new file mode 100644
index 000000000..d95078c34
--- /dev/null
+++ b/llvm/test/ThinLTO/X86/Inputs/internalize.ll
@@ -0,0 +1,6 @@
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.11.0"
+
+define weak void @weak_func_nonprevailing() {
+    ret void
+}
diff --git a/llvm/test/ThinLTO/X86/Inputs/lazyload_metadata.ll b/llvm/test/ThinLTO/X86/Inputs/lazyload_metadata.ll
index f51a08a1a..5ad8652b5 100644
--- a/llvm/test/ThinLTO/X86/Inputs/lazyload_metadata.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/lazyload_metadata.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.11.0"
 
 declare void @globalfunc1()
diff --git a/llvm/test/ThinLTO/X86/Inputs/linkonce_aliasee_ref_import.ll b/llvm/test/ThinLTO/X86/Inputs/linkonce_aliasee_ref_import.ll
index 66847f50b..f832f843d 100644
--- a/llvm/test/ThinLTO/X86/Inputs/linkonce_aliasee_ref_import.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/linkonce_aliasee_ref_import.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-grtev4-linux-gnu"
 
 define i32 @main() #0 {
diff --git a/llvm/test/ThinLTO/X86/Inputs/linkonce_odr_unnamed_addr.ll b/llvm/test/ThinLTO/X86/Inputs/linkonce_odr_unnamed_addr.ll
index 6ec8d0048..e0bb06859 100644
--- a/llvm/test/ThinLTO/X86/Inputs/linkonce_odr_unnamed_addr.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/linkonce_odr_unnamed_addr.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-grtev4-linux-gnu"
 
 @linkonceodrunnamed = linkonce_odr unnamed_addr constant i32 0
diff --git a/llvm/test/ThinLTO/X86/Inputs/linkonce_resolution_comdat.ll b/llvm/test/ThinLTO/X86/Inputs/linkonce_resolution_comdat.ll
index 59434a819..92b518231 100644
--- a/llvm/test/ThinLTO/X86/Inputs/linkonce_resolution_comdat.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/linkonce_resolution_comdat.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 $c2 = comdat any
diff --git a/llvm/test/ThinLTO/X86/Inputs/llvm.used.ll b/llvm/test/ThinLTO/X86/Inputs/llvm.used.ll
index 35f5529c2..8f1370952 100644
--- a/llvm/test/ThinLTO/X86/Inputs/llvm.used.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/llvm.used.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.11.0"
 
 
diff --git a/llvm/test/ThinLTO/X86/Inputs/local_name_conflict1.ll b/llvm/test/ThinLTO/X86/Inputs/local_name_conflict1.ll
index d7b7ea618..de43b7825 100644
--- a/llvm/test/ThinLTO/X86/Inputs/local_name_conflict1.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/local_name_conflict1.ll
@@ -1,6 +1,6 @@
 ; ModuleID = 'local_name_conflict.o'
 source_filename = "local_name_conflict.c"
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @baz = internal constant i32 10, align 4
diff --git a/llvm/test/ThinLTO/X86/Inputs/local_name_conflict2.ll b/llvm/test/ThinLTO/X86/Inputs/local_name_conflict2.ll
index 40a4e8665..7e2412af2 100644
--- a/llvm/test/ThinLTO/X86/Inputs/local_name_conflict2.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/local_name_conflict2.ll
@@ -1,6 +1,6 @@
 ; ModuleID = 'local_name_conflict.o'
 source_filename = "local_name_conflict.c"
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @baz = internal constant i32 10, align 4
diff --git a/llvm/test/ThinLTO/X86/Inputs/local_name_conflict_var1.ll b/llvm/test/ThinLTO/X86/Inputs/local_name_conflict_var1.ll
index fb5306fc3..bec5f67f5 100644
--- a/llvm/test/ThinLTO/X86/Inputs/local_name_conflict_var1.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/local_name_conflict_var1.ll
@@ -1,6 +1,6 @@
 ; ModuleID = 'local_name_conflict_var.o'
 source_filename = "local_name_conflict_var.c"
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @baz = internal global i32 10, align 4
diff --git a/llvm/test/ThinLTO/X86/Inputs/local_name_conflict_var2.ll b/llvm/test/ThinLTO/X86/Inputs/local_name_conflict_var2.ll
index bf3c262f1..51cd0bef0 100644
--- a/llvm/test/ThinLTO/X86/Inputs/local_name_conflict_var2.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/local_name_conflict_var2.ll
@@ -1,6 +1,6 @@
 ; ModuleID = 'local_name_conflict_var.o'
 source_filename = "local_name_conflict_var.c"
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @baz = internal global i32 10, align 4
diff --git a/llvm/test/ThinLTO/X86/Inputs/merge-triple.ll b/llvm/test/ThinLTO/X86/Inputs/merge-triple.ll
index 6ee881573..3be4b8f44 100644
--- a/llvm/test/ThinLTO/X86/Inputs/merge-triple.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/merge-triple.ll
@@ -1,2 +1,2 @@
 target triple = "x86_64-apple-macosx10.11.0"
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/ThinLTO/X86/Inputs/module_asm.ll b/llvm/test/ThinLTO/X86/Inputs/module_asm.ll
index 337e8613c..f713310ea 100644
--- a/llvm/test/ThinLTO/X86/Inputs/module_asm.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/module_asm.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define i32 @main({ i64, { i64, i8* }* } %unnamed) #0 {
diff --git a/llvm/test/ThinLTO/X86/Inputs/module_asm2.ll b/llvm/test/ThinLTO/X86/Inputs/module_asm2.ll
index a8081ccca..f5ff45178 100644
--- a/llvm/test/ThinLTO/X86/Inputs/module_asm2.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/module_asm2.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define i32 @main({ i64, { i64, i8* }* } %unnamed) #0 {
diff --git a/llvm/test/ThinLTO/X86/Inputs/noinline.ll b/llvm/test/ThinLTO/X86/Inputs/noinline.ll
index 73db2912c..f412b332c 100644
--- a/llvm/test/ThinLTO/X86/Inputs/noinline.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/noinline.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-pc-linux-gnu"
 
 define i32 @foo(i32) local_unnamed_addr #0 {
diff --git a/llvm/test/ThinLTO/X86/Inputs/personality-local.ll b/llvm/test/ThinLTO/X86/Inputs/personality-local.ll
index dc27a375e..c610a6974 100644
--- a/llvm/test/ThinLTO/X86/Inputs/personality-local.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/personality-local.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-pc-linux-gnu"
 
 define void @foo() {
diff --git a/llvm/test/ThinLTO/X86/Inputs/personality.ll b/llvm/test/ThinLTO/X86/Inputs/personality.ll
index 37046b5c2..c97b50386 100644
--- a/llvm/test/ThinLTO/X86/Inputs/personality.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/personality.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-pc-linux-gnu"
 
 define void @bar() personality i32 (i32, i32, i64, i8*, i8*)* @personality_routine {
diff --git a/llvm/test/ThinLTO/X86/Inputs/pr35472.ll b/llvm/test/ThinLTO/X86/Inputs/pr35472.ll
index b9c92b3e3..c330193d7 100644
--- a/llvm/test/ThinLTO/X86/Inputs/pr35472.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/pr35472.ll
@@ -1,6 +1,6 @@
 ; ModuleID = 'b.cpp'
 source_filename = "b.cpp"
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 ; Function Attrs: noinline optnone uwtable
diff --git a/llvm/test/ThinLTO/X86/Inputs/reference_non_importable.ll b/llvm/test/ThinLTO/X86/Inputs/reference_non_importable.ll
index dab18539e..66289663a 100644
--- a/llvm/test/ThinLTO/X86/Inputs/reference_non_importable.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/reference_non_importable.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.11.0"
 
 declare i8 **@foo()
diff --git a/llvm/test/ThinLTO/X86/Inputs/referenced_by_constant.ll b/llvm/test/ThinLTO/X86/Inputs/referenced_by_constant.ll
index f5e7bc9d8..205308ba2 100644
--- a/llvm/test/ThinLTO/X86/Inputs/referenced_by_constant.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/referenced_by_constant.ll
@@ -1,5 +1,5 @@
 
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.11.0"
 
 define void @referencedbyglobal() {
diff --git a/llvm/test/ThinLTO/X86/Inputs/strong_non_prevailing.ll b/llvm/test/ThinLTO/X86/Inputs/strong_non_prevailing.ll
index 5473f817f..f2d74a037 100644
--- a/llvm/test/ThinLTO/X86/Inputs/strong_non_prevailing.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/strong_non_prevailing.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 $__llvm_profile_filename = comdat any
diff --git a/llvm/test/ThinLTO/X86/Inputs/thinlto-internalize-doublepromoted.ll b/llvm/test/ThinLTO/X86/Inputs/thinlto-internalize-doublepromoted.ll
new file mode 100644
index 000000000..327ead5ec
--- /dev/null
+++ b/llvm/test/ThinLTO/X86/Inputs/thinlto-internalize-doublepromoted.ll
@@ -0,0 +1,10 @@
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.15.0"
+
+define i32 @main() {
+entry:
+  call void @foo.llvm.123()
+  ret i32 0
+}
+
+declare void @foo.llvm.123()
diff --git a/llvm/test/ThinLTO/X86/Inputs/thinlto-internalize-used2.ll b/llvm/test/ThinLTO/X86/Inputs/thinlto-internalize-used2.ll
index 225685d93..de4167627 100644
--- a/llvm/test/ThinLTO/X86/Inputs/thinlto-internalize-used2.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/thinlto-internalize-used2.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.15.0"
 
 define i32 @main() {
diff --git a/llvm/test/ThinLTO/X86/Inputs/weak_resolution.ll b/llvm/test/ThinLTO/X86/Inputs/weak_resolution.ll
index fe9edc66d..9ef7cbb09 100644
--- a/llvm/test/ThinLTO/X86/Inputs/weak_resolution.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/weak_resolution.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.11.0"
 
 ; Alias are not optimized
diff --git a/llvm/test/ThinLTO/X86/Inputs/writeonly-with-refs.ll b/llvm/test/ThinLTO/X86/Inputs/writeonly-with-refs.ll
new file mode 100644
index 000000000..31ca2ad9f
--- /dev/null
+++ b/llvm/test/ThinLTO/X86/Inputs/writeonly-with-refs.ll
@@ -0,0 +1,17 @@
+; ModuleID = 'foo.o'
+source_filename = "foo.cpp"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+%struct.S = type { i32 }
+%struct.Q = type { %struct.S* }
+
+@_ZL3Obj = internal constant %struct.S { i32 42 }, align 4
+@outer = dso_local local_unnamed_addr global %struct.Q { %struct.S* @_ZL3Obj }, align 8
+
+; Function Attrs: nofree norecurse nounwind uwtable writeonly
+define dso_local void @_Z3foov() local_unnamed_addr {
+entry:
+  store %struct.S* null, %struct.S** getelementptr inbounds (%struct.Q, %struct.Q* @outer, i64 0, i32 0), align 8
+  ret void
+}
diff --git a/llvm/test/ThinLTO/X86/alias_internal.ll b/llvm/test/ThinLTO/X86/alias_internal.ll
index d6433f698..18ee176b8 100644
--- a/llvm/test/ThinLTO/X86/alias_internal.ll
+++ b/llvm/test/ThinLTO/X86/alias_internal.ll
@@ -11,7 +11,7 @@
 ; CHECK-DAG: M0_12511626713252727690 -> M0_{{.*}} // alias
 ; CHECK-DAG: M1_8129049334585965161 -> M1_{{.*}} // alias
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define internal i32 @f(i8*) unnamed_addr {
diff --git a/llvm/test/ThinLTO/X86/autoupgrade.ll b/llvm/test/ThinLTO/X86/autoupgrade.ll
index 77f7eb0e0..3d31dd726 100644
--- a/llvm/test/ThinLTO/X86/autoupgrade.ll
+++ b/llvm/test/ThinLTO/X86/autoupgrade.ll
@@ -19,7 +19,7 @@
 ; SUMMARYNAMES: ^2 = gv: (name: "globalfunc2",
 ; SUMMARYNAMES: ^3 = gv: (name: "globalfunc1"
 
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.11.0"
 
 define i32 @main() #0 {
diff --git a/llvm/test/ThinLTO/X86/builtin-nostrip.ll b/llvm/test/ThinLTO/X86/builtin-nostrip.ll
index 7dc6504a7..690ae322a 100644
--- a/llvm/test/ThinLTO/X86/builtin-nostrip.ll
+++ b/llvm/test/ThinLTO/X86/builtin-nostrip.ll
@@ -46,7 +46,7 @@
 ; eliminated once the old lto interface is fixed.
 ; CHECK-NM2-DAG: t __stack_chk_fail
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define void @bar() {
diff --git a/llvm/test/ThinLTO/X86/cache-config.ll b/llvm/test/ThinLTO/X86/cache-config.ll
index 54328885f..ccc730d35 100644
--- a/llvm/test/ThinLTO/X86/cache-config.ll
+++ b/llvm/test/ThinLTO/X86/cache-config.ll
@@ -18,7 +18,7 @@
 ; RUN: llvm-lto2 run -o %t.o %t.bc -cache-dir %t.cache -r=%t.bc,globalfunc,plx -default-triple=x86_64-unknown-linux-gnu
 ; RUN: ls %t.cache | count 15
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define void @globalfunc() {
diff --git a/llvm/test/ThinLTO/X86/cache-icall.ll b/llvm/test/ThinLTO/X86/cache-icall.ll
index 83b740de4..e3a3c66c1 100644
--- a/llvm/test/ThinLTO/X86/cache-icall.ll
+++ b/llvm/test/ThinLTO/X86/cache-icall.ll
@@ -29,7 +29,7 @@
 ; SYMBOLS-YES-DAG: {{FUNC .* f.cfi$}}
 ; SYMBOLS-YES-DAG: {{NOTYPE .* UND f.cfi$}}
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define i8* @_start(void ()* %p) !type !0 {
diff --git a/llvm/test/ThinLTO/X86/cache-import-lists.ll b/llvm/test/ThinLTO/X86/cache-import-lists.ll
index ba7b437e8..225f7d8bb 100644
--- a/llvm/test/ThinLTO/X86/cache-import-lists.ll
+++ b/llvm/test/ThinLTO/X86/cache-import-lists.ll
@@ -11,7 +11,7 @@
 ; RUN: llvm-lto2 run -cache-dir %t.cache -o %t.o %t.bc %t2.bc %t1.bc -r=%t.bc,main,plx -r=%t.bc,f1,lx -r=%t.bc,f2,lx -r=%t2.bc,f2,plx -r=%t2.bc,linkonce_odr,plx -r=%t1.bc,f1,plx -r=%t1.bc,linkonce_odr,lx
 ; RUN: ls %t.cache | count 6
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define void @main() {
diff --git a/llvm/test/ThinLTO/X86/cache-typeid-resolutions.ll b/llvm/test/ThinLTO/X86/cache-typeid-resolutions.ll
index e5092dd63..6618a6f28 100644
--- a/llvm/test/ThinLTO/X86/cache-typeid-resolutions.ll
+++ b/llvm/test/ThinLTO/X86/cache-typeid-resolutions.ll
@@ -22,7 +22,7 @@
 ; RUN: llvm-lto2 run -o %t.o %t.bc %t-import.bc %t3.bc -cache-dir %t.cache -r=%t.bc,f1,plx -r=%t.bc,f2,plx -r=%t.bc,f1_actual,plx -r=%t3.bc,vt2a,plx -r=%t3.bc,vt2b,plx -r=%t-import.bc,importf1,plx -r=%t-import.bc,f1,lx -r=%t-import.bc,importf2,plx -r=%t-import.bc,f2,lx
 ; RUN: ls %t.cache | count 6
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @f1 = alias i1(i8*), i1 (i8*)* @f1_actual
diff --git a/llvm/test/ThinLTO/X86/cache.ll b/llvm/test/ThinLTO/X86/cache.ll
index 75e6b7fd6..c9285f91b 100644
--- a/llvm/test/ThinLTO/X86/cache.ll
+++ b/llvm/test/ThinLTO/X86/cache.ll
@@ -160,7 +160,7 @@
 ; RUN: not ls %t.cache/llvmcache-foo-1024
 ; RUN: not ls %t.cache/llvmcache-foo-7
 
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.11.0"
 
 define void @globalfunc() #0 {
diff --git a/llvm/test/ThinLTO/X86/callees-metadata.ll b/llvm/test/ThinLTO/X86/callees-metadata.ll
index 457cdb319..3e416dd3d 100644
--- a/llvm/test/ThinLTO/X86/callees-metadata.ll
+++ b/llvm/test/ThinLTO/X86/callees-metadata.ll
@@ -10,7 +10,7 @@
 ; CHECK: define {{.*}} i32 @f1.llvm.0
 ; CHECK: define {{.*}} i32 @f2.llvm.0
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define dso_local i32 @bar(i32 %x) {
diff --git a/llvm/test/ThinLTO/X86/cfi-devirt.ll b/llvm/test/ThinLTO/X86/cfi-devirt.ll
index 760c02a24..dd83024e2 100644
--- a/llvm/test/ThinLTO/X86/cfi-devirt.ll
+++ b/llvm/test/ThinLTO/X86/cfi-devirt.ll
@@ -61,7 +61,7 @@
 ; RUN:   -r=%t.o,_ZTV1C,px 2>&1 | FileCheck %s --check-prefix=ERROR
 ; ERROR: failed: inconsistent LTO Unit splitting (recompile with -fsplit-lto-unit)
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-grtev4-linux-gnu"
 
 %struct.A = type { i32 (...)** }
diff --git a/llvm/test/ThinLTO/X86/cfi-distributed.ll b/llvm/test/ThinLTO/X86/cfi-distributed.ll
index 533922851..5959cc729 100644
--- a/llvm/test/ThinLTO/X86/cfi-distributed.ll
+++ b/llvm/test/ThinLTO/X86/cfi-distributed.ll
@@ -40,7 +40,7 @@
 ; INDEX2: typeid: (name: "_ZTS1A2"
 ; INDEX2: typeid: (name: "_ZTS1A3"
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-grtev4-linux-gnu"
 
 %struct.B = type { %struct.A }
diff --git a/llvm/test/ThinLTO/X86/cfi-icall-only-defuse.ll b/llvm/test/ThinLTO/X86/cfi-icall-only-defuse.ll
index 842468508..1721778d7 100644
--- a/llvm/test/ThinLTO/X86/cfi-icall-only-defuse.ll
+++ b/llvm/test/ThinLTO/X86/cfi-icall-only-defuse.ll
@@ -14,7 +14,7 @@
 ; RUN: llvm-bcanalyzer -dump %t1.bc.thinlto.bc | FileCheck %s --check-prefix=FOOBAZ
 ; RUN: llvm-bcanalyzer -dump %t2.bc.thinlto.bc | FileCheck %s --check-prefix=BARQUX
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 declare !type !0 i8 @baz(i8*)
diff --git a/llvm/test/ThinLTO/X86/cfi-icall.ll b/llvm/test/ThinLTO/X86/cfi-icall.ll
index 42c26f158..96179e256 100644
--- a/llvm/test/ThinLTO/X86/cfi-icall.ll
+++ b/llvm/test/ThinLTO/X86/cfi-icall.ll
@@ -5,7 +5,7 @@
 ; RUN:   -r %t1.bc,addrtaken,px
 ; RUN: llvm-bcanalyzer -dump %t.out.index.bc | FileCheck %s --check-prefix=COMBINED
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define i1 @foo(i8* %p) !type !0 {
diff --git a/llvm/test/ThinLTO/X86/cfi.ll b/llvm/test/ThinLTO/X86/cfi.ll
index 0edddb82b..e40600974 100644
--- a/llvm/test/ThinLTO/X86/cfi.ll
+++ b/llvm/test/ThinLTO/X86/cfi.ll
@@ -22,7 +22,7 @@
 ; RUN:   -r=%t.o,_ZTV1B,px
 ; RUN: llvm-dis %t3.1.4.opt.bc -o - | FileCheck %s --check-prefix=CHECK-IR
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-grtev4-linux-gnu"
 
 %struct.B = type { %struct.A }
diff --git a/llvm/test/ThinLTO/X86/crash_debuginfo.ll b/llvm/test/ThinLTO/X86/crash_debuginfo.ll
index 31403269b..d0d63f25e 100644
--- a/llvm/test/ThinLTO/X86/crash_debuginfo.ll
+++ b/llvm/test/ThinLTO/X86/crash_debuginfo.ll
@@ -11,7 +11,7 @@
 
 ; ModuleID = 'test/ThinLTO/X86/crash_debuginfo.ll'
 source_filename = "test/ThinLTO/X86/crash_debuginfo.ll"
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.7.0"
 
 define void @foo(i32 %arg) {
diff --git a/llvm/test/ThinLTO/X86/deadstrip.ll b/llvm/test/ThinLTO/X86/deadstrip.ll
index 2f389c52c..9409039f4 100644
--- a/llvm/test/ThinLTO/X86/deadstrip.ll
+++ b/llvm/test/ThinLTO/X86/deadstrip.ll
@@ -132,7 +132,7 @@
 ; and called from @dead_func.
 ; CHECK-NM-NOTDEAD: T _baz
 
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.11.0"
 
 
diff --git a/llvm/test/ThinLTO/X86/debuginfo-compositetype-import.ll b/llvm/test/ThinLTO/X86/debuginfo-compositetype-import.ll
index 5859875b1..578b0ced8 100644
--- a/llvm/test/ThinLTO/X86/debuginfo-compositetype-import.ll
+++ b/llvm/test/ThinLTO/X86/debuginfo-compositetype-import.ll
@@ -34,7 +34,7 @@
 
 ; ModuleID = 'debuginfo-compositetype-import.c'
 source_filename = "debuginfo-compositetype-import.c"
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 ; Function Attrs: nounwind uwtable
diff --git a/llvm/test/ThinLTO/X86/debuginfo-cu-import.ll b/llvm/test/ThinLTO/X86/debuginfo-cu-import.ll
index 93fd8dc86..1f07f96f3 100644
--- a/llvm/test/ThinLTO/X86/debuginfo-cu-import.ll
+++ b/llvm/test/ThinLTO/X86/debuginfo-cu-import.ll
@@ -17,7 +17,7 @@
 
 ; ModuleID = 'debuginfo-cu-import.c'
 source_filename = "debuginfo-cu-import.c"
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define void @foo() !dbg !28 {
diff --git a/llvm/test/ThinLTO/X86/devirt-after-icp.ll b/llvm/test/ThinLTO/X86/devirt-after-icp.ll
index 8edb86981..af6eba77b 100644
--- a/llvm/test/ThinLTO/X86/devirt-after-icp.ll
+++ b/llvm/test/ThinLTO/X86/devirt-after-icp.ll
@@ -85,7 +85,7 @@
 ; REMARK: single-impl: devirtualized a call to _ZN1B3barEv
 ; REMARK-NOT: single-impl: devirtualized a call to _ZN1B3fooEv
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-grtev4-linux-gnu"
 
 %class.A = type { i32 (...)** }
diff --git a/llvm/test/ThinLTO/X86/devirt.ll b/llvm/test/ThinLTO/X86/devirt.ll
index 4ea615ee3..eae8c69eb 100644
--- a/llvm/test/ThinLTO/X86/devirt.ll
+++ b/llvm/test/ThinLTO/X86/devirt.ll
@@ -20,10 +20,10 @@
 ; and that we generate summary information needed for index-based WPD.
 ; RUN: llvm-dis -o - %t2.o | FileCheck %s --check-prefix=NOENABLESPLITFLAG
 ; NOENABLESPLITFLAG-DAG: !{i32 1, !"EnableSplitLTOUnit", i32 0}
-; NOENABLESPLITFLAG-DAG: [[An:\^[0-9]+]] = gv: (name: "_ZN1A1nEi")
-; NOENABLESPLITFLAG-DAG: [[Bf:\^[0-9]+]] = gv: (name: "_ZN1B1fEi")
-; NOENABLESPLITFLAG-DAG: [[Cf:\^[0-9]+]] = gv: (name: "_ZN1C1fEi")
-; NOENABLESPLITFLAG-DAG: [[Dm:\^[0-9]+]] = gv: (name: "_ZN1D1mEi")
+; NOENABLESPLITFLAG-DAG: [[An:\^[0-9]+]] = gv: (name: "_ZN1A1nEi"
+; NOENABLESPLITFLAG-DAG: [[Bf:\^[0-9]+]] = gv: (name: "_ZN1B1fEi"
+; NOENABLESPLITFLAG-DAG: [[Cf:\^[0-9]+]] = gv: (name: "_ZN1C1fEi"
+; NOENABLESPLITFLAG-DAG: [[Dm:\^[0-9]+]] = gv: (name: "_ZN1D1mEi"
 ; NOENABLESPLITFLAG-DAG: [[B:\^[0-9]+]] = gv: (name: "_ZTV1B", {{.*}} vTableFuncs: ((virtFunc: [[Bf]], offset: 16), (virtFunc: [[An]], offset: 24)), refs: ([[Bf]], [[An]])
 ; NOENABLESPLITFLAG-DAG: [[C:\^[0-9]+]] = gv: (name: "_ZTV1C", {{.*}} vTableFuncs: ((virtFunc: [[Cf]], offset: 16), (virtFunc: [[An]], offset: 24)), refs: ([[An]], [[Cf]])
 ; NOENABLESPLITFLAG-DAG: [[D:\^[0-9]+]] = gv: (name: "_ZTV1D", {{.*}} vTableFuncs: ((virtFunc: [[Dm]], offset: 16)), refs: ([[Dm]])
@@ -33,7 +33,31 @@
 ; Type Id on _ZTV1D should have been promoted
 ; NOENABLESPLITFLAG-DAG: typeidCompatibleVTable: (name: "1${{.*}}", summary: ((offset: 16, [[D]])))
 
-; TODO: Test index-based WPD one %t2.o once implemented.
+; Legacy PM, Index based WPD
+; RUN: llvm-lto2 run %t2.o -save-temps -pass-remarks=. \
+; RUN:   -o %t3 \
+; RUN:   -r=%t2.o,test,px \
+; RUN:   -r=%t2.o,_ZN1A1nEi,p \
+; RUN:   -r=%t2.o,_ZN1B1fEi,p \
+; RUN:   -r=%t2.o,_ZN1C1fEi,p \
+; RUN:   -r=%t2.o,_ZN1D1mEi,p \
+; RUN:   -r=%t2.o,_ZTV1B,px \
+; RUN:   -r=%t2.o,_ZTV1C,px \
+; RUN:   -r=%t2.o,_ZTV1D,px 2>&1 | FileCheck %s --check-prefix=REMARK
+; RUN: llvm-dis %t3.1.4.opt.bc -o - | FileCheck %s --check-prefix=CHECK-IR
+
+; New PM, Index based WPD
+; RUN: llvm-lto2 run %t2.o -save-temps -use-new-pm -pass-remarks=. \
+; RUN:   -o %t3 \
+; RUN:   -r=%t2.o,test,px \
+; RUN:   -r=%t2.o,_ZN1A1nEi,p \
+; RUN:   -r=%t2.o,_ZN1B1fEi,p \
+; RUN:   -r=%t2.o,_ZN1C1fEi,p \
+; RUN:   -r=%t2.o,_ZN1D1mEi,p \
+; RUN:   -r=%t2.o,_ZTV1B,px \
+; RUN:   -r=%t2.o,_ZTV1C,px \
+; RUN:   -r=%t2.o,_ZTV1D,px 2>&1 | FileCheck %s --check-prefix=REMARK
+; RUN: llvm-dis %t3.1.4.opt.bc -o - | FileCheck %s --check-prefix=CHECK-IR
 
 ; Legacy PM
 ; FIXME: Fix machine verifier issues and remove -verify-machineinstrs=0. PR39436.
@@ -82,7 +106,7 @@
 ; REMARK-DAG: single-impl: devirtualized a call to _ZN1A1nEi
 ; REMARK-DAG: single-impl: devirtualized a call to _ZN1D1mEi
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-grtev4-linux-gnu"
 
 %struct.A = type { i32 (...)** }
@@ -138,10 +162,24 @@ entry:
 declare i1 @llvm.type.test(i8*, metadata)
 declare void @llvm.assume(i1)
 
-declare i32 @_ZN1B1fEi(%struct.B* %this, i32 %a)
-declare i32 @_ZN1A1nEi(%struct.A* %this, i32 %a)
-declare i32 @_ZN1C1fEi(%struct.C* %this, i32 %a)
-declare i32 @_ZN1D1mEi(%struct.D* %this, i32 %a)
+define i32 @_ZN1B1fEi(%struct.B* %this, i32 %a) #0 {
+   ret i32 0;
+}
+
+define i32 @_ZN1A1nEi(%struct.A* %this, i32 %a) #0 {
+   ret i32 0;
+}
+
+define i32 @_ZN1C1fEi(%struct.C* %this, i32 %a) #0 {
+   ret i32 0;
+}
+
+define i32 @_ZN1D1mEi(%struct.D* %this, i32 %a) #0 {
+   ret i32 0;
+}
+
+; Make sure we don't inline or otherwise optimize out the direct calls.
+attributes #0 = { noinline optnone }
 
 !0 = !{i64 16, !"_ZTS1A"}
 !1 = !{i64 16, !"_ZTS1B"}
diff --git a/llvm/test/ThinLTO/X86/devirt2.ll b/llvm/test/ThinLTO/X86/devirt2.ll
new file mode 100644
index 000000000..01eed382f
--- /dev/null
+++ b/llvm/test/ThinLTO/X86/devirt2.ll
@@ -0,0 +1,278 @@
+; REQUIRES: x86-registered-target
+
+; Test devirtualization requiring promotion of local targets.
+
+; Generate split module with summary for hybrid Thin/Regular LTO WPD.
+; RUN: opt -thinlto-bc -thinlto-split-lto-unit -o %t1.o %s
+; RUN: opt -thinlto-bc -thinlto-split-lto-unit -o %t2.o %p/Inputs/devirt2.ll
+
+; Check that we have module flag showing splitting enabled, and that we don't
+; generate summary information needed for index-based WPD.
+; RUN: llvm-modextract -b -n=0 %t2.o -o %t2.o.0
+; RUN: llvm-dis -o - %t2.o.0 | FileCheck %s --check-prefix=ENABLESPLITFLAG --implicit-check-not=vTableFuncs --implicit-check-not=typeidCompatibleVTable
+; RUN: llvm-modextract -b -n=1 %t2.o -o %t2.o.1
+; RUN: llvm-dis -o - %t2.o.1 | FileCheck %s --check-prefix=ENABLESPLITFLAG --implicit-check-not=vTableFuncs --implicit-check-not=typeidCompatibleVTable
+; ENABLESPLITFLAG: !{i32 1, !"EnableSplitLTOUnit", i32 1}
+
+; Generate unsplit module with summary for ThinLTO index-based WPD.
+; RUN: opt -thinlto-bc -o %t3.o %s
+; RUN: opt -thinlto-bc -o %t4.o %p/Inputs/devirt2.ll
+
+; Check that we don't have module flag when splitting not enabled for ThinLTO,
+; and that we generate summary information needed for index-based WPD.
+; RUN: llvm-dis -o - %t4.o | FileCheck %s --check-prefix=NOENABLESPLITFLAG
+; NOENABLESPLITFLAG-DAG: !{i32 1, !"EnableSplitLTOUnit", i32 0}
+; NOENABLESPLITFLAG-DAG: [[An:\^[0-9]+]] = gv: (name: "_ZN1A1nEi"
+; NOENABLESPLITFLAG-DAG: [[Bf:\^[0-9]+]] = gv: (name: "_ZN1B1fEi"
+; NOENABLESPLITFLAG-DAG: [[Cf:\^[0-9]+]] = gv: (name: "_ZN1C1fEi"
+; NOENABLESPLITFLAG-DAG: [[Dm:\^[0-9]+]] = gv: (name: "_ZN1D1mEi"
+; NOENABLESPLITFLAG-DAG: [[B:\^[0-9]+]] = gv: (name: "_ZTV1B", {{.*}} vTableFuncs: ((virtFunc: [[Bf]], offset: 16), (virtFunc: [[An]], offset: 24)), refs: ([[Bf]], [[An]])
+; NOENABLESPLITFLAG-DAG: [[C:\^[0-9]+]] = gv: (name: "_ZTV1C", {{.*}} vTableFuncs: ((virtFunc: [[Cf]], offset: 16), (virtFunc: [[An]], offset: 24)), refs: ([[An]], [[Cf]])
+; NOENABLESPLITFLAG-DAG: [[D:\^[0-9]+]] = gv: (name: "_ZTV1D", {{.*}} vTableFuncs: ((virtFunc: [[Dm]], offset: 16)), refs: ([[Dm]])
+; NOENABLESPLITFLAG-DAG: typeidCompatibleVTable: (name: "_ZTS1A", summary: ((offset: 16, [[B]]), (offset: 16, [[C]])))
+; NOENABLESPLITFLAG-DAG: typeidCompatibleVTable: (name: "_ZTS1B", summary: ((offset: 16, [[B]])))
+; NOENABLESPLITFLAG-DAG: typeidCompatibleVTable: (name: "_ZTS1C", summary: ((offset: 16, [[C]])))
+; NOENABLESPLITFLAG-DAG: typeidCompatibleVTable: (name: "_ZTS1D", summary: ((offset: 16, [[D]])))
+
+; Legacy PM, Index based WPD
+; RUN: llvm-lto2 run %t3.o %t4.o -save-temps -pass-remarks=. \
+; RUN:   -wholeprogramdevirt-print-index-based \
+; RUN:   -o %t5 \
+; RUN:   -r=%t3.o,test,px \
+; RUN:   -r=%t3.o,_ZTV1B, \
+; RUN:   -r=%t3.o,_ZTV1C, \
+; RUN:   -r=%t3.o,_ZTV1D, \
+; RUN:   -r=%t3.o,_ZN1D1mEi, \
+; RUN:   -r=%t3.o,test2, \
+; RUN:   -r=%t4.o,_ZN1B1fEi,p \
+; RUN:   -r=%t4.o,_ZN1C1fEi,p \
+; RUN:   -r=%t4.o,_ZN1D1mEi,p \
+; RUN:   -r=%t4.o,test2,px \
+; RUN:   -r=%t4.o,_ZTV1B,px \
+; RUN:   -r=%t4.o,_ZTV1C,px \
+; RUN:   -r=%t4.o,_ZTV1D,px \
+; RUN:   -r=%t4.o,_ZTV1E,px 2>&1 | FileCheck %s --check-prefix=REMARK --check-prefix=PRINT
+; RUN: llvm-dis %t5.1.4.opt.bc -o - | FileCheck %s --check-prefix=CHECK-IR1
+; RUN: llvm-dis %t5.2.4.opt.bc -o - | FileCheck %s --check-prefix=CHECK-IR2
+; RUN: llvm-nm %t5.1 | FileCheck %s --check-prefix=NM-INDEX1
+; RUN: llvm-nm %t5.2 | FileCheck %s --check-prefix=NM-INDEX2
+
+; New PM, Index based WPD
+; RUN: llvm-lto2 run %t3.o %t4.o -save-temps -use-new-pm -pass-remarks=. \
+; RUN:   -wholeprogramdevirt-print-index-based \
+; RUN:   -o %t5 \
+; RUN:   -r=%t3.o,test,px \
+; RUN:   -r=%t3.o,_ZTV1B, \
+; RUN:   -r=%t3.o,_ZTV1C, \
+; RUN:   -r=%t3.o,_ZTV1D, \
+; RUN:   -r=%t3.o,_ZN1D1mEi, \
+; RUN:   -r=%t3.o,test2, \
+; RUN:   -r=%t4.o,_ZN1B1fEi,p \
+; RUN:   -r=%t4.o,_ZN1C1fEi,p \
+; RUN:   -r=%t4.o,_ZN1D1mEi,p \
+; RUN:   -r=%t4.o,test2,px \
+; RUN:   -r=%t4.o,_ZTV1B,px \
+; RUN:   -r=%t4.o,_ZTV1C,px \
+; RUN:   -r=%t4.o,_ZTV1D,px \
+; RUN:   -r=%t4.o,_ZTV1E,px 2>&1 | FileCheck %s --check-prefix=REMARK --check-prefix=PRINT
+; RUN: llvm-dis %t5.1.4.opt.bc -o - | FileCheck %s --check-prefix=CHECK-IR1
+; RUN: llvm-dis %t5.2.4.opt.bc -o - | FileCheck %s --check-prefix=CHECK-IR2
+; RUN: llvm-nm %t5.1 | FileCheck %s --check-prefix=NM-INDEX1
+; RUN: llvm-nm %t5.2 | FileCheck %s --check-prefix=NM-INDEX2
+
+; NM-INDEX1-DAG: U _ZN1A1nEi.llvm.
+; NM-INDEX1-DAG: U _ZN1E1mEi.llvm.
+; NM-INDEX1-DAG: U _ZN1D1mEi
+
+; NM-INDEX2-DAG: T _ZN1A1nEi.llvm.
+; NM-INDEX2-DAG: T _ZN1E1mEi.llvm.
+; NM-INDEX2-DAG: W _ZN1D1mEi
+; NM-INDEX2-DAG: t _ZN1B1fEi
+; NM-INDEX2-DAG: t _ZN1C1fEi
+
+; Index based WPD, distributed backends
+; RUN: llvm-lto2 run %t3.o %t4.o -save-temps -use-new-pm \
+; RUN:   -thinlto-distributed-indexes -wholeprogramdevirt-print-index-based \
+; RUN:   -o %t5 \
+; RUN:   -r=%t3.o,test,px \
+; RUN:   -r=%t3.o,_ZTV1B, \
+; RUN:   -r=%t3.o,_ZTV1C, \
+; RUN:   -r=%t3.o,_ZTV1D, \
+; RUN:   -r=%t3.o,_ZN1D1mEi, \
+; RUN:   -r=%t3.o,test2, \
+; RUN:   -r=%t4.o,_ZN1B1fEi,p \
+; RUN:   -r=%t4.o,_ZN1C1fEi,p \
+; RUN:   -r=%t4.o,_ZN1D1mEi,p \
+; RUN:   -r=%t4.o,test2,px \
+; RUN:   -r=%t4.o,_ZTV1B,px \
+; RUN:   -r=%t4.o,_ZTV1C,px \
+; RUN:   -r=%t4.o,_ZTV1D,px \
+; RUN:   -r=%t4.o,_ZTV1E,px 2>&1 | FileCheck %s --check-prefix=PRINT
+
+; PRINT-DAG: Devirtualized call to {{.*}} (_ZN1A1nEi)
+; PRINT-DAG: Devirtualized call to {{.*}} (_ZN1E1mEi)
+; PRINT-DAG: Devirtualized call to {{.*}} (_ZN1D1mEi)
+
+; Legacy PM
+; RUN: llvm-lto2 run %t1.o %t2.o -save-temps -pass-remarks=. \
+; RUN:   -o %t5 \
+; RUN:   -r=%t1.o,test,px \
+; RUN:   -r=%t1.o,_ZTV1B, \
+; RUN:   -r=%t1.o,_ZTV1C, \
+; RUN:   -r=%t1.o,_ZTV1D, \
+; RUN:   -r=%t1.o,_ZTV1D, \
+; RUN:   -r=%t1.o,_ZN1D1mEi, \
+; RUN:   -r=%t1.o,_ZN1D1mEi, \
+; RUN:   -r=%t1.o,test2, \
+; RUN:   -r=%t2.o,_ZN1A1nEi,p \
+; RUN:   -r=%t2.o,_ZN1B1fEi,p \
+; RUN:   -r=%t2.o,_ZN1C1fEi,p \
+; RUN:   -r=%t2.o,_ZN1D1mEi,p \
+; RUN:   -r=%t2.o,_ZN1E1mEi,p \
+; RUN:   -r=%t2.o,_ZTV1B, \
+; RUN:   -r=%t2.o,_ZTV1C, \
+; RUN:   -r=%t2.o,_ZTV1D, \
+; RUN:   -r=%t2.o,_ZTV1E, \
+; RUN:   -r=%t2.o,test2,px \
+; RUN:   -r=%t2.o,_ZN1A1nEi, \
+; RUN:   -r=%t2.o,_ZN1B1fEi, \
+; RUN:   -r=%t2.o,_ZN1C1fEi, \
+; RUN:   -r=%t2.o,_ZN1D1mEi, \
+; RUN:   -r=%t2.o,_ZN1E1mEi, \
+; RUN:   -r=%t2.o,_ZTV1B,px \
+; RUN:   -r=%t2.o,_ZTV1C,px \
+; RUN:   -r=%t2.o,_ZTV1D,px \
+; RUN:   -r=%t2.o,_ZTV1E,px 2>&1 | FileCheck %s --check-prefix=REMARK
+; RUN: llvm-dis %t5.1.4.opt.bc -o - | FileCheck %s --check-prefix=CHECK-IR1
+; RUN: llvm-dis %t5.2.4.opt.bc -o - | FileCheck %s --check-prefix=CHECK-IR2
+; RUN: llvm-nm %t5.1 | FileCheck %s --check-prefix=NM-HYBRID1
+; RUN: llvm-nm %t5.2 | FileCheck %s --check-prefix=NM-HYBRID2
+
+; New PM
+; RUN: llvm-lto2 run %t1.o %t2.o -save-temps -use-new-pm -pass-remarks=. \
+; RUN:   -o %t5 \
+; RUN:   -r=%t1.o,test,px \
+; RUN:   -r=%t1.o,_ZTV1B, \
+; RUN:   -r=%t1.o,_ZTV1C, \
+; RUN:   -r=%t1.o,_ZTV1D, \
+; RUN:   -r=%t1.o,_ZTV1D, \
+; RUN:   -r=%t1.o,_ZN1D1mEi, \
+; RUN:   -r=%t1.o,_ZN1D1mEi, \
+; RUN:   -r=%t1.o,test2, \
+; RUN:   -r=%t2.o,_ZN1A1nEi,p \
+; RUN:   -r=%t2.o,_ZN1B1fEi,p \
+; RUN:   -r=%t2.o,_ZN1C1fEi,p \
+; RUN:   -r=%t2.o,_ZN1D1mEi,p \
+; RUN:   -r=%t2.o,_ZN1E1mEi,p \
+; RUN:   -r=%t2.o,_ZTV1B, \
+; RUN:   -r=%t2.o,_ZTV1C, \
+; RUN:   -r=%t2.o,_ZTV1D, \
+; RUN:   -r=%t2.o,_ZTV1E, \
+; RUN:   -r=%t2.o,test2,px \
+; RUN:   -r=%t2.o,_ZN1A1nEi, \
+; RUN:   -r=%t2.o,_ZN1B1fEi, \
+; RUN:   -r=%t2.o,_ZN1C1fEi, \
+; RUN:   -r=%t2.o,_ZN1D1mEi, \
+; RUN:   -r=%t2.o,_ZN1E1mEi, \
+; RUN:   -r=%t2.o,_ZTV1B,px \
+; RUN:   -r=%t2.o,_ZTV1C,px \
+; RUN:   -r=%t2.o,_ZTV1D,px \
+; RUN:   -r=%t2.o,_ZTV1E,px 2>&1 | FileCheck %s --check-prefix=REMARK
+; RUN: llvm-dis %t5.1.4.opt.bc -o - | FileCheck %s --check-prefix=CHECK-IR1
+; RUN: llvm-dis %t5.2.4.opt.bc -o - | FileCheck %s --check-prefix=CHECK-IR2
+; RUN: llvm-nm %t5.1 | FileCheck %s --check-prefix=NM-HYBRID1
+; RUN: llvm-nm %t5.2 | FileCheck %s --check-prefix=NM-HYBRID2
+
+; NM-HYBRID1-DAG: U _ZN1A1nEi$
+; NM-HYBRID1-DAG: U _ZN1E1mEi$
+; NM-HYBRID1-DAG: U _ZN1D1mEi
+
+; NM-HYBRID2-DAG: T _ZN1A1nEi$
+; NM-HYBRID2-DAG: T _ZN1E1mEi$
+; NM-HYBRID2-DAG: W _ZN1D1mEi
+; NM-HYBRID2-DAG: T _ZN1B1fEi
+; NM-HYBRID2-DAG: T _ZN1C1fEi
+
+; REMARK-DAG: single-impl: devirtualized a call to _ZN1A1nEi
+; REMARK-DAG: single-impl: devirtualized a call to _ZN1D1mEi
+; We should devirt call to _ZN1E1mEi once in importing module and once
+; in original (exporting) module.
+; REMARK-DAG: single-impl: devirtualized a call to _ZN1E1mEi
+; REMARK-DAG: single-impl: devirtualized a call to _ZN1E1mEi
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-grtev4-linux-gnu"
+
+%struct.A = type { i32 (...)** }
+%struct.B = type { %struct.A }
+%struct.C = type { %struct.A }
+%struct.D = type { i32 (...)** }
+%struct.E = type { i32 (...)** }
+
+@_ZTV1B = external constant [4 x i8*]
+@_ZTV1C = external constant [4 x i8*]
+;@_ZTV1D = external constant [3 x i8*]
+@_ZTV1D = linkonce_odr constant { [3 x i8*] } { [3 x i8*] [i8* null, i8* undef, i8* bitcast (i32 (%struct.D*, i32)* @_ZN1D1mEi to i8*)] }, !type !3
+
+define linkonce_odr i32 @_ZN1D1mEi(%struct.D* %this, i32 %a) #0 {
+   ret i32 0
+}
+
+; CHECK-IR1-LABEL: define i32 @test
+define i32 @test(%struct.A* %obj, %struct.D* %obj2, %struct.E* %obj3, i32 %a) {
+entry:
+  %0 = bitcast %struct.A* %obj to i8***
+  %vtable = load i8**, i8*** %0
+  %1 = bitcast i8** %vtable to i8*
+  %p = call i1 @llvm.type.test(i8* %1, metadata !"_ZTS1A")
+  call void @llvm.assume(i1 %p)
+  %fptrptr = getelementptr i8*, i8** %vtable, i32 1
+  %2 = bitcast i8** %fptrptr to i32 (%struct.A*, i32)**
+  %fptr1 = load i32 (%struct.A*, i32)*, i32 (%struct.A*, i32)** %2, align 8
+
+  ; Check that the call was devirtualized. Ignore extra character before
+  ; symbol name which would happen if it was promoted during module
+  ; splitting for hybrid WPD.
+  ; CHECK-IR1: %call = tail call i32 bitcast (void ()* @{{.*}}_ZN1A1nEi
+  %call = tail call i32 %fptr1(%struct.A* nonnull %obj, i32 %a)
+
+  %3 = bitcast i8** %vtable to i32 (%struct.A*, i32)**
+  %fptr22 = load i32 (%struct.A*, i32)*, i32 (%struct.A*, i32)** %3, align 8
+
+  ; We still have to call it as virtual.
+  ; CHECK-IR1: %call3 = tail call i32 %fptr22
+  %call3 = tail call i32 %fptr22(%struct.A* nonnull %obj, i32 %call)
+
+  %4 = bitcast %struct.D* %obj2 to i8***
+  %vtable2 = load i8**, i8*** %4
+  %5 = bitcast i8** %vtable2 to i8*
+  %p2 = call i1 @llvm.type.test(i8* %5, metadata !"_ZTS1D")
+  call void @llvm.assume(i1 %p2)
+
+  %6 = bitcast i8** %vtable2 to i32 (%struct.D*, i32)**
+  %fptr33 = load i32 (%struct.D*, i32)*, i32 (%struct.D*, i32)** %6, align 8
+
+  ; Check that the call was devirtualized.
+  ; CHECK-IR1: %call4 = tail call i32 @_ZN1D1mEi
+  %call4 = tail call i32 %fptr33(%struct.D* nonnull %obj2, i32 %call3)
+
+  %call5 = tail call i32 @test2(%struct.E* nonnull %obj3, i32 %call4)
+  ret i32 %call5
+}
+; CHECK-IR1-LABEL: ret i32
+; CHECK-IR1-LABEL: }
+
+; CHECK-IR2: define i32 @test2
+; CHECK-IR2-NEXT: entry:
+; Check that the call was devirtualized. Ignore extra character before
+; symbol name which would happen if it was promoted during module
+; splitting for hybrid WPD.
+; CHECK-IR2-NEXT:   %call4 = tail call i32 @{{.*}}_ZN1E1mEi
+
+declare i1 @llvm.type.test(i8*, metadata)
+declare void @llvm.assume(i1)
+declare i32 @test2(%struct.E* %obj, i32 %a)
+
+attributes #0 = { noinline optnone }
+
+!3 = !{i64 16, !"_ZTS1D"}
diff --git a/llvm/test/ThinLTO/X86/devirt_alias.ll b/llvm/test/ThinLTO/X86/devirt_alias.ll
new file mode 100644
index 000000000..92aa2bcba
--- /dev/null
+++ b/llvm/test/ThinLTO/X86/devirt_alias.ll
@@ -0,0 +1,58 @@
+; REQUIRES: x86-registered-target
+
+; Test index-based devirtualization when one copy is an alias.
+
+; Generate unsplit module with summary for ThinLTO index-based WPD.
+; RUN: opt -thinlto-bc -o %t3.o %s
+; RUN: opt -thinlto-bc -o %t4.o %p/Inputs/devirt_alias.ll
+
+; RUN: llvm-lto2 run %t3.o %t4.o -save-temps -pass-remarks=. \
+; RUN:   -wholeprogramdevirt-print-index-based \
+; RUN:   -o %t5 \
+; RUN:   -r=%t3.o,test,px \
+; RUN:   -r=%t3.o,_ZTV1D, \
+; RUN:   -r=%t3.o,_ZN1D1mEi, \
+; RUN:   -r=%t4.o,_ZN1D1mEi,p \
+; RUN:   -r=%t4.o,_ZTV1D,px \
+; RUN:   -r=%t4.o,some_name,px \
+; RUN:   2>&1 | FileCheck %s --check-prefix=REMARK --check-prefix=PRINT
+; RUN: llvm-dis %t5.1.4.opt.bc -o - | FileCheck %s --check-prefix=CHECK-IR1
+
+; PRINT-DAG: Devirtualized call to {{.*}} (_ZN1D1mEi)
+
+; REMARK-DAG: single-impl: devirtualized a call to _ZN1D1mEi
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-grtev4-linux-gnu"
+
+%struct.D = type { i32 (...)** }
+
+@_ZTV1D = linkonce_odr constant { [3 x i8*] } { [3 x i8*] [i8* null, i8* undef, i8* bitcast (i32 (%struct.D*, i32)* @_ZN1D1mEi to i8*)] }, !type !3
+
+; CHECK-IR1-LABEL: define i32 @test
+define i32 @test(%struct.D* %obj2, i32 %a) {
+entry:
+  %0 = bitcast %struct.D* %obj2 to i8***
+  %vtable2 = load i8**, i8*** %0
+  %1 = bitcast i8** %vtable2 to i8*
+  %p2 = call i1 @llvm.type.test(i8* %1, metadata !"_ZTS1D")
+  call void @llvm.assume(i1 %p2)
+
+  %2 = bitcast i8** %vtable2 to i32 (%struct.D*, i32)**
+  %fptr33 = load i32 (%struct.D*, i32)*, i32 (%struct.D*, i32)** %2, align 8
+
+  ; Check that the call was devirtualized.
+  ; CHECK-IR1: %call4 = tail call i32 @_ZN1D1mEi
+  %call4 = tail call i32 %fptr33(%struct.D* nonnull %obj2, i32 %a)
+  ret i32 %call4
+}
+; CHECK-IR1-LABEL: ret i32
+; CHECK-IR1-LABEL: }
+
+declare i1 @llvm.type.test(i8*, metadata)
+declare void @llvm.assume(i1)
+declare i32 @_ZN1D1mEi(%struct.D* %this, i32 %a)
+
+attributes #0 = { noinline optnone }
+
+!3 = !{i64 16, !"_ZTS1D"}
diff --git a/llvm/test/ThinLTO/X86/devirt_available_externally.ll b/llvm/test/ThinLTO/X86/devirt_available_externally.ll
new file mode 100644
index 000000000..128055f70
--- /dev/null
+++ b/llvm/test/ThinLTO/X86/devirt_available_externally.ll
@@ -0,0 +1,72 @@
+; REQUIRES: x86-registered-target
+
+; Test index-based devirtualization when first copy is available_externally,
+; which doesn't have type metadata. We should use the strong external
+; def in the other module to devirtualize.
+
+; Generate unsplit module with summary for ThinLTO index-based WPD.
+; RUN: opt -thinlto-bc -o %t3.o %s
+; RUN: opt -thinlto-bc -o %t4.o %p/Inputs/devirt_available_externally.ll
+
+; The available_externally copy should not get vTableFuncs information in its
+; summary entry, but the external def should.
+; RUN: llvm-dis -o - %t3.o | FileCheck %s --check-prefix=AVAILEXTERNAL
+; AVAILEXTERNAL: gv: (name: "_ZTV1D"
+; AVAILEXTERNAL-NOT: vTableFuncs
+; AVAILEXTERNAL-SAME: ; guid =
+; RUN: llvm-dis -o - %t4.o | FileCheck %s --check-prefix=EXTERNAL
+; EXTERNAL: gv: (name: "_ZTV1D", {{.*}} vTableFuncs: ((virtFunc:
+
+; RUN: llvm-lto2 run %t3.o %t4.o -save-temps -pass-remarks=. \
+; RUN:   -wholeprogramdevirt-print-index-based \
+; RUN:   -o %t5 \
+; RUN:   -r=%t3.o,test,px \
+; RUN:   -r=%t3.o,_ZTV1D, \
+; RUN:   -r=%t3.o,_ZN1D1mEi, \
+; RUN:   -r=%t4.o,_ZN1D1mEi,p \
+; RUN:   -r=%t4.o,_ZTV1D,px \
+; RUN:   2>&1 | FileCheck %s --check-prefix=REMARK --check-prefix=PRINT
+; RUN: llvm-dis %t5.1.4.opt.bc -o - | FileCheck %s --check-prefix=CHECK-IR1
+; RUN: llvm-nm %t5.1 | FileCheck %s --check-prefix=NM-INDEX1
+; RUN: llvm-nm %t5.2 | FileCheck %s --check-prefix=NM-INDEX2
+
+; NM-INDEX1-DAG: U _ZN1D1mEi
+
+; NM-INDEX2-DAG: T _ZN1D1mEi
+
+; PRINT-DAG: Devirtualized call to {{.*}} (_ZN1D1mEi)
+
+; REMARK-DAG: single-impl: devirtualized a call to _ZN1D1mEi
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-grtev4-linux-gnu"
+
+%struct.D = type { i32 (...)** }
+
+@_ZTV1D = available_externally constant { [3 x i8*] } { [3 x i8*] [i8* null, i8* undef, i8* bitcast (i32 (%struct.D*, i32)* @_ZN1D1mEi to i8*)] }
+
+; CHECK-IR1-LABEL: define i32 @test
+define i32 @test(%struct.D* %obj2, i32 %a) {
+entry:
+  %0 = bitcast %struct.D* %obj2 to i8***
+  %vtable2 = load i8**, i8*** %0
+  %1 = bitcast i8** %vtable2 to i8*
+  %p2 = call i1 @llvm.type.test(i8* %1, metadata !"_ZTS1D")
+  call void @llvm.assume(i1 %p2)
+
+  %2 = bitcast i8** %vtable2 to i32 (%struct.D*, i32)**
+  %fptr33 = load i32 (%struct.D*, i32)*, i32 (%struct.D*, i32)** %2, align 8
+
+  ; Check that the call was devirtualized.
+  ; CHECK-IR1: %call4 = tail call i32 @_ZN1D1mEi
+  %call4 = tail call i32 %fptr33(%struct.D* nonnull %obj2, i32 %a)
+  ret i32 %call4
+}
+; CHECK-IR1-LABEL: ret i32
+; CHECK-IR1-LABEL: }
+
+declare i1 @llvm.type.test(i8*, metadata)
+declare void @llvm.assume(i1)
+declare i32 @_ZN1D1mEi(%struct.D* %this, i32 %a)
+
+attributes #0 = { noinline optnone }
diff --git a/llvm/test/ThinLTO/X86/devirt_external_comdat_same_guid.ll b/llvm/test/ThinLTO/X86/devirt_external_comdat_same_guid.ll
new file mode 100644
index 000000000..18482a051
--- /dev/null
+++ b/llvm/test/ThinLTO/X86/devirt_external_comdat_same_guid.ll
@@ -0,0 +1,87 @@
+; REQUIRES: x86-registered-target
+
+; Test that index-based devirtualization in the presence of external
+; vtables with the same name (with comdat) succeeds.
+
+; Generate unsplit module with summary for ThinLTO index-based WPD.
+; RUN: opt -thinlto-bc -o %t3.o %s
+; RUN: opt -thinlto-bc -o %t4.o %p/Inputs/devirt_external_comdat_same_guid.ll
+
+; RUN: llvm-lto2 run %t3.o %t4.o -save-temps -use-new-pm -pass-remarks=. \
+; RUN:   -wholeprogramdevirt-print-index-based \
+; RUN:   -o %t5 \
+; RUN:   -r=%t3.o,use_B,px \
+; RUN:   -r=%t3.o,test,px \
+; RUN:   -r=%t3.o,_ZTV1B,px \
+; RUN:   -r=%t3.o,_ZN1B1fEi,px \
+; RUN:   -r=%t3.o,_ZN1B1nEi,px \
+; RUN:   -r=%t4.o,_ZTV1B,x \
+; RUN:   -r=%t4.o,_ZN1B1fEi,x \
+; RUN:   -r=%t4.o,_ZN1A1nEi,x \
+; RUN:   -r=%t4.o,test2,px \
+; RUN:   2>&1 | FileCheck %s --check-prefix=REMARK --check-prefix=PRINT
+; RUN: llvm-dis %t5.1.4.opt.bc -o - | FileCheck %s --check-prefix=CHECK-IR1
+; RUN: llvm-dis %t5.2.4.opt.bc -o - | FileCheck %s --check-prefix=CHECK-IR2
+
+; PRINT-DAG: Devirtualized call to {{.*}} (_ZN1A1nEi)
+
+; REMARK-DAG: single-impl: devirtualized a call to _ZN1A1nEi
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-grtev4-linux-gnu"
+
+source_filename = "-"
+
+%struct.A = type { i32 (...)** }
+%struct.B = type { %struct.A }
+
+$_ZTV1B = comdat any
+
+@_ZTV1B = constant { [4 x i8*] } { [4 x i8*] [i8* null, i8* undef, i8* bitcast (i32 (%struct.B*, i32)* @_ZN1B1fEi to i8*), i8* bitcast (i32 (%struct.B*, i32)* @_ZN1B1nEi to i8*)] }, comdat, !type !0, !type !1
+
+define i32 @_ZN1B1fEi(%struct.B* %this, i32 %a) #0 comdat($_ZTV1B) {
+   ret i32 0;
+}
+
+define i32 @_ZN1B1nEi(%struct.B* %this, i32 %a) #0 comdat($_ZTV1B) {
+   ret i32 0;
+}
+
+; Ensures that vtable of B is live so that we will attempt devirt.
+define dso_local i32 @use_B(%struct.B* %a) {
+entry:
+  %0 = bitcast %struct.B* %a to i32 (...)***
+  store i32 (...)** bitcast (i8** getelementptr inbounds ({ [4 x i8*] }, { [4 x i8*] }* @_ZTV1B, i64 0, inrange i32 0, i64 2) to i32 (...)**), i32 (...)*** %0, align 8
+  ret i32 0
+}
+
+; CHECK-IR1: define i32 @test(
+define i32 @test(%struct.A* %obj, i32 %a) {
+entry:
+  %0 = bitcast %struct.A* %obj to i8***
+  %vtable = load i8**, i8*** %0
+  %1 = bitcast i8** %vtable to i8*
+  %p = call i1 @llvm.type.test(i8* %1, metadata !"_ZTS1A")
+  call void @llvm.assume(i1 %p)
+  %fptrptr = getelementptr i8*, i8** %vtable, i32 1
+  %2 = bitcast i8** %fptrptr to i32 (%struct.A*, i32)**
+  %fptr1 = load i32 (%struct.A*, i32)*, i32 (%struct.A*, i32)** %2, align 8
+
+  ; Check that the call was devirtualized.
+  ; CHECK-IR1: tail call i32 {{.*}}@_ZN1A1nEi
+  %call = tail call i32 %fptr1(%struct.A* nonnull %obj, i32 %a)
+
+  ret i32 %call
+}
+
+; CHECK-IR2: define i32 @test2
+; Check that the call was devirtualized.
+; CHECK-IR2:   tail call i32 @_ZN1A1nEi
+
+declare i1 @llvm.type.test(i8*, metadata)
+declare void @llvm.assume(i1)
+
+attributes #0 = { noinline optnone }
+
+!0 = !{i64 16, !"_ZTS1A"}
+!1 = !{i64 16, !"_ZTS1B"}
diff --git a/llvm/test/ThinLTO/X86/devirt_local_same_guid.ll b/llvm/test/ThinLTO/X86/devirt_local_same_guid.ll
new file mode 100644
index 000000000..8354bc191
--- /dev/null
+++ b/llvm/test/ThinLTO/X86/devirt_local_same_guid.ll
@@ -0,0 +1,74 @@
+; REQUIRES: x86-registered-target
+
+; Test that index-based devirtualization in the presence of same-named
+; local vtables in same named source files fails.
+
+; Generate unsplit module with summary for ThinLTO index-based WPD.
+; RUN: opt -thinlto-bc -o %t3.o %s
+; RUN: opt -thinlto-bc -o %t4.o %p/Inputs/devirt_local_same_guid.ll
+
+; RUN: llvm-lto2 run %t3.o %t4.o -save-temps -use-new-pm -pass-remarks=. \
+; RUN:   -wholeprogramdevirt-print-index-based \
+; RUN:   -o %t5 \
+; RUN:   -r=%t3.o,use_B,px \
+; RUN:   -r=%t3.o,test,px \
+; RUN:   -r=%t4.o,test2,px
+; RUN: llvm-dis %t5.1.4.opt.bc -o - | FileCheck %s --check-prefix=CHECK-IR1
+; RUN: llvm-dis %t5.2.4.opt.bc -o - | FileCheck %s --check-prefix=CHECK-IR2
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-grtev4-linux-gnu"
+
+source_filename = "-"
+
+%struct.A = type { i32 (...)** }
+%struct.B = type { %struct.A }
+
+@_ZTV1B = internal constant { [4 x i8*] } { [4 x i8*] [i8* null, i8* undef, i8* bitcast (i32 (%struct.B*, i32)* @_ZN1B1fEi to i8*), i8* bitcast (i32 (%struct.B*, i32)* @_ZN1B1nEi to i8*)] }, !type !0, !type !1
+
+define internal i32 @_ZN1B1fEi(%struct.B* %this, i32 %a) #0 {
+   ret i32 0;
+}
+
+define internal i32 @_ZN1B1nEi(%struct.B* %this, i32 %a) #0 {
+   ret i32 0;
+}
+
+; Ensures that vtable of B is live so that we will attempt devirt.
+define dso_local i32 @use_B(%struct.B* %a) {
+entry:
+  %0 = bitcast %struct.B* %a to i32 (...)***
+  store i32 (...)** bitcast (i8** getelementptr inbounds ({ [4 x i8*] }, { [4 x i8*] }* @_ZTV1B, i64 0, inrange i32 0, i64 2) to i32 (...)**), i32 (...)*** %0, align 8
+  ret i32 0
+}
+
+; CHECK-IR1: define i32 @test(
+define i32 @test(%struct.A* %obj, i32 %a) {
+entry:
+  %0 = bitcast %struct.A* %obj to i8***
+  %vtable = load i8**, i8*** %0
+  %1 = bitcast i8** %vtable to i8*
+  %p = call i1 @llvm.type.test(i8* %1, metadata !"_ZTS1A")
+  call void @llvm.assume(i1 %p)
+  %fptrptr = getelementptr i8*, i8** %vtable, i32 1
+  %2 = bitcast i8** %fptrptr to i32 (%struct.A*, i32)**
+  %fptr1 = load i32 (%struct.A*, i32)*, i32 (%struct.A*, i32)** %2, align 8
+
+  ; Check that the call was not devirtualized.
+  ; CHECK-IR1: %call = tail call i32 %fptr1
+  %call = tail call i32 %fptr1(%struct.A* nonnull %obj, i32 %a)
+
+  ret i32 %call
+}
+
+; CHECK-IR2: define i32 @test2
+; Check that the call was not devirtualized.
+; CHECK-IR2:   %call4 = tail call i32 %fptr
+
+declare i1 @llvm.type.test(i8*, metadata)
+declare void @llvm.assume(i1)
+
+attributes #0 = { noinline optnone }
+
+!0 = !{i64 16, !"_ZTS1A"}
+!1 = !{i64 16, !"_ZTS1B"}
diff --git a/llvm/test/ThinLTO/X86/devirt_promote.ll b/llvm/test/ThinLTO/X86/devirt_promote.ll
new file mode 100644
index 000000000..563ed9941
--- /dev/null
+++ b/llvm/test/ThinLTO/X86/devirt_promote.ll
@@ -0,0 +1,72 @@
+; REQUIRES: x86-registered-target
+
+; Test devirtualization requiring promotion of local targets, where the
+; promotion is required by one devirtualization and needs to be updated
+; for a second devirtualization in the defining module as a post-pass
+; update.
+
+; Generate unsplit module with summary for ThinLTO index-based WPD.
+; RUN: opt -thinlto-bc -o %t3.o %s
+; RUN: opt -thinlto-bc -o %t4.o %p/Inputs/devirt_promote.ll
+
+; RUN: llvm-lto2 run %t3.o %t4.o -save-temps -use-new-pm -pass-remarks=. \
+; RUN:   -wholeprogramdevirt-print-index-based \
+; RUN:   -o %t5 \
+; RUN:   -r=%t3.o,test,px \
+; RUN:   -r=%t4.o,_ZN1B1fEi,p \
+; RUN:   -r=%t4.o,test2,px \
+; RUN:   -r=%t4.o,_ZTV1B,px \
+; RUN:   2>&1 | FileCheck %s --check-prefix=REMARK --check-prefix=PRINT
+; RUN: llvm-dis %t5.1.4.opt.bc -o - | FileCheck %s --check-prefix=CHECK-IR1
+; RUN: llvm-dis %t5.2.4.opt.bc -o - | FileCheck %s --check-prefix=CHECK-IR2
+; RUN: llvm-nm %t5.1 | FileCheck %s --check-prefix=NM-INDEX1
+; RUN: llvm-nm %t5.2 | FileCheck %s --check-prefix=NM-INDEX2
+
+; NM-INDEX1: U _ZN1A1nEi.llvm.
+
+; Make sure that not only did _ZN1A1nEi get promoted (due to the
+; devirtualization in the other module) but the reference due to the
+; devirtualization in its defining module should be to the promoted
+; symbol.
+; NM-INDEX2-NOT: U _ZN1A1nEi
+; NM-INDEX2: T _ZN1A1nEi.llvm.
+; NM-INDEX2-NOT: U _ZN1A1nEi
+
+; We should devirt call to _ZN1A1nEi once in importing module and once
+; in original (exporting) module.
+; REMARK-COUNT-2: single-impl: devirtualized a call to _ZN1A1nEi.llvm.
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-grtev4-linux-gnu"
+
+%struct.A = type { i32 (...)** }
+
+; CHECK-IR1-LABEL: define i32 @test
+define i32 @test(%struct.A* %obj, i32 %a) {
+entry:
+  %0 = bitcast %struct.A* %obj to i8***
+  %vtable = load i8**, i8*** %0
+  %1 = bitcast i8** %vtable to i8*
+  %p = call i1 @llvm.type.test(i8* %1, metadata !"_ZTS1A")
+  call void @llvm.assume(i1 %p)
+  %fptrptr = getelementptr i8*, i8** %vtable, i32 1
+  %2 = bitcast i8** %fptrptr to i32 (%struct.A*, i32)**
+  %fptr1 = load i32 (%struct.A*, i32)*, i32 (%struct.A*, i32)** %2, align 8
+
+  ; Check that the call was devirtualized.
+  ; CHECK-IR1: %call = tail call i32 bitcast (void ()* @_ZN1A1nEi
+  %call = tail call i32 %fptr1(%struct.A* nonnull %obj, i32 %a)
+
+  ret i32 %call
+}
+; CHECK-IR1-LABEL: ret i32
+; CHECK-IR1-LABEL: }
+
+; CHECK-IR2: define i32 @test2
+; Check that the call was devirtualized.
+; CHECK-IR2:   %call4 = tail call i32 @_ZN1A1nEi
+
+declare i1 @llvm.type.test(i8*, metadata)
+declare void @llvm.assume(i1)
+
+attributes #0 = { noinline optnone }
diff --git a/llvm/test/ThinLTO/X86/devirt_promote_legacy.ll b/llvm/test/ThinLTO/X86/devirt_promote_legacy.ll
new file mode 100644
index 000000000..79fde540c
--- /dev/null
+++ b/llvm/test/ThinLTO/X86/devirt_promote_legacy.ll
@@ -0,0 +1,57 @@
+; REQUIRES: x86-registered-target
+
+; Test devirtualization requiring promotion of local targets, where the
+; promotion is required by one devirtualization and needs to be updated
+; for a second devirtualization in the defining module as a post-pass
+; update.
+
+; Generate unsplit module with summary for ThinLTO index-based WPD.
+; RUN: opt -thinlto-bc -o %t3.o %s
+; RUN: opt -thinlto-bc -o %t4.o %p/Inputs/devirt_promote.ll
+
+; RUN: llvm-lto -thinlto-action=run %t3.o %t4.o --thinlto-save-temps=%t5. \
+; RUN:   --pass-remarks=. \
+; RUN:   --exported-symbol=test \
+; RUN:   --exported-symbol=test2 \
+; RUN:   --exported-symbol=_ZTV1B 2>&1 | FileCheck %s --check-prefix=REMARK
+; RUN: llvm-dis %t5.0.4.opt.bc -o - | FileCheck %s --check-prefix=CHECK-IR1
+; RUN: llvm-dis %t5.1.4.opt.bc -o - | FileCheck %s --check-prefix=CHECK-IR2
+
+; We should devirt call to _ZN1A1nEi once in importing module and once
+; in original (exporting) module.
+; REMARK-COUNT-2: single-impl: devirtualized a call to _ZN1A1nEi.llvm.
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-grtev4-linux-gnu"
+
+%struct.A = type { i32 (...)** }
+
+; CHECK-IR1-LABEL: define i32 @test
+define i32 @test(%struct.A* %obj, i32 %a) {
+entry:
+  %0 = bitcast %struct.A* %obj to i8***
+  %vtable = load i8**, i8*** %0
+  %1 = bitcast i8** %vtable to i8*
+  %p = call i1 @llvm.type.test(i8* %1, metadata !"_ZTS1A")
+  call void @llvm.assume(i1 %p)
+  %fptrptr = getelementptr i8*, i8** %vtable, i32 1
+  %2 = bitcast i8** %fptrptr to i32 (%struct.A*, i32)**
+  %fptr1 = load i32 (%struct.A*, i32)*, i32 (%struct.A*, i32)** %2, align 8
+
+  ; Check that the call was devirtualized.
+  ; CHECK-IR1: = tail call i32 bitcast (void ()* @_ZN1A1nEi
+  %call = tail call i32 %fptr1(%struct.A* nonnull %obj, i32 %a)
+
+  ret i32 %call
+}
+; CHECK-IR1-LABEL: ret i32
+; CHECK-IR1-LABEL: }
+
+; CHECK-IR2: define i32 @test2
+; Check that the call was devirtualized.
+; CHECK-IR2: = tail call i32 @_ZN1A1nEi
+
+declare i1 @llvm.type.test(i8*, metadata)
+declare void @llvm.assume(i1)
+
+attributes #0 = { noinline optnone }
diff --git a/llvm/test/ThinLTO/X86/devirt_single_hybrid.ll b/llvm/test/ThinLTO/X86/devirt_single_hybrid.ll
new file mode 100644
index 000000000..5b2df6cf6
--- /dev/null
+++ b/llvm/test/ThinLTO/X86/devirt_single_hybrid.ll
@@ -0,0 +1,55 @@
+; Check that we import and inline virtual method with single implementation
+; when we're running hybrid LTO.
+;
+; RUN: opt -thinlto-bc -thinlto-split-lto-unit %s -o %t-main.bc
+; RUN: opt -thinlto-bc -thinlto-split-lto-unit %p/Inputs/devirt_single_hybrid_foo.ll -o %t-foo.bc
+; RUN: opt -thinlto-bc -thinlto-split-lto-unit %p/Inputs/devirt_single_hybrid_bar.ll -o %t-bar.bc
+; RUN: llvm-lto2 run -save-temps %t-main.bc %t-foo.bc %t-bar.bc -pass-remarks=. -o %t \
+; RUN:    -r=%t-foo.bc,_Z3fooP1A,pl \
+; RUN:    -r=%t-main.bc,main,plx \
+; RUN:    -r=%t-main.bc,_Z3barv,l \
+; RUN:    -r=%t-bar.bc,_Z3barv,pl \
+; RUN:    -r=%t-bar.bc,_Z3fooP1A, \
+; RUN:    -r=%t-bar.bc,_ZNK1A1fEv,pl \
+; RUN:    -r=%t-bar.bc,_ZTV1A,l \
+; RUN:    -r=%t-bar.bc,_ZTVN10__cxxabiv117__class_type_infoE, \
+; RUN:    -r=%t-bar.bc,_ZTS1A,pl \
+; RUN:    -r=%t-bar.bc,_ZTI1A,pl \
+; RUN:    -r=%t-bar.bc,_ZNK1A1fEv, \
+; RUN:    -r=%t-bar.bc,_ZTV1A,pl \
+; RUN:    -r=%t-bar.bc,_ZTI1A, 2>&1 | FileCheck %s --check-prefix=REMARK
+; RUN: llvm-dis %t.1.3.import.bc -o - | FileCheck %s --check-prefix=IMPORT
+; RUN: llvm-dis %t.1.5.precodegen.bc -o - | FileCheck %s --check-prefix=CODEGEN
+
+; REMARK-COUNT-3: single-impl: devirtualized a call to _ZNK1A1fEv
+
+; IMPORT:       define available_externally hidden i32 @_ZNK1A1fEv(%struct.A* %this)
+; IMPORT-NEXT:  entry:
+; IMPORT-NEXT:      ret i32 3
+
+; CODEGEN:        define hidden i32 @main()
+; CODEGEN-NEXT:   entry:
+; CODEGEN-NEXT:     ret i32 23
+
+; Virtual method should have been optimized out
+; CODEGEN-NOT: _ZNK1A1fEv
+
+; ModuleID = 'main.cpp'
+source_filename = "main.cpp"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Function Attrs: norecurse uwtable
+define hidden i32 @main() local_unnamed_addr {
+entry:
+  %call = tail call i32 @_Z3barv()
+  ret i32 %call
+}
+
+declare dso_local i32 @_Z3barv() local_unnamed_addr
+
+!llvm.module.flags = !{!0}
+!llvm.ident = !{!1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{!"clang version 10.0.0 (trunk 373596)"}
diff --git a/llvm/test/ThinLTO/X86/diagnostic-handler-remarks-with-hotness.ll b/llvm/test/ThinLTO/X86/diagnostic-handler-remarks-with-hotness.ll
index 239e6efe3..94bad9b3b 100644
--- a/llvm/test/ThinLTO/X86/diagnostic-handler-remarks-with-hotness.ll
+++ b/llvm/test/ThinLTO/X86/diagnostic-handler-remarks-with-hotness.ll
@@ -53,7 +53,7 @@
 ; YAML2-NEXT: ...
 
 
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.11.0"
 
 define i32 @bar() {
diff --git a/llvm/test/ThinLTO/X86/diagnostic-handler-remarks.ll b/llvm/test/ThinLTO/X86/diagnostic-handler-remarks.ll
index f0ff9542e..a996b8586 100644
--- a/llvm/test/ThinLTO/X86/diagnostic-handler-remarks.ll
+++ b/llvm/test/ThinLTO/X86/diagnostic-handler-remarks.ll
@@ -51,8 +51,21 @@
 ; YAML2-NEXT:   - String:          ')'
 ; YAML2-NEXT: ...
 
+; The file extension depends on the format of the remarks
+; RUN: rm -f %t.bitstream.thin.0.bitstream %t.bitstream.thin.1.bitstream
+; RUN: llvm-lto -thinlto-action=run \
+; RUN:          -lto-pass-remarks-output=%t.bitstream \
+; RUN:          -lto-pass-remarks-filter=inline \
+; RUN:          -lto-pass-remarks-format=bitstream \
+; RUN:          -exported-symbol _func2 \
+; RUN:          -exported-symbol _main %t1.bc %t2.bc 2>&1 | \
+; RUN:     FileCheck %s -allow-empty
+; RUN: llvm-bcanalyzer %t.bitstream.thin.0.bitstream
+; RUN: llvm-bcanalyzer %t.bitstream.thin.1.bitstream
+; CHECK-NOT: remark:
+; CHECK-NOT: llvm-lto:
 
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.11.0"
 
 define i32 @bar() {
diff --git a/llvm/test/ThinLTO/X86/dicompositetype-unique-alias.ll b/llvm/test/ThinLTO/X86/dicompositetype-unique-alias.ll
index 2c4e3ea72..ff1f40d6d 100644
--- a/llvm/test/ThinLTO/X86/dicompositetype-unique-alias.ll
+++ b/llvm/test/ThinLTO/X86/dicompositetype-unique-alias.ll
@@ -20,7 +20,7 @@
 ; CHECK: DICompositeType
 ; CHECK-NOT: DICompositeType
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-scei-ps4"
 
 %class.C = type <{ i32 (...)**, %class.A, %struct.CFVS, [6 x i8] }>
diff --git a/llvm/test/ThinLTO/X86/dicompositetype-unique.ll b/llvm/test/ThinLTO/X86/dicompositetype-unique.ll
index e61b515d4..dbf14911c 100644
--- a/llvm/test/ThinLTO/X86/dicompositetype-unique.ll
+++ b/llvm/test/ThinLTO/X86/dicompositetype-unique.ll
@@ -8,7 +8,7 @@
 ; RUN: llvm-dis %t.1.3.import.bc -o - | FileCheck %s
 
 
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.11.0"
 
 declare void @foo()
diff --git a/llvm/test/ThinLTO/X86/dicompositetype-unique2.ll b/llvm/test/ThinLTO/X86/dicompositetype-unique2.ll
index f275ea1b2..e7ecc97e3 100644
--- a/llvm/test/ThinLTO/X86/dicompositetype-unique2.ll
+++ b/llvm/test/ThinLTO/X86/dicompositetype-unique2.ll
@@ -13,7 +13,7 @@
 ; Only llvm-lto2 adds the dso_local keyword, hence the {{.*}}
 ; CHECK: define available_externally{{.*}} void @_ZN4CFVSD2Ev
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-scei-ps4"
 
 %class.C = type <{ i32 (...)**, %class.A, %struct.CFVS, [6 x i8] }>
diff --git a/llvm/test/ThinLTO/X86/distributed_import.ll b/llvm/test/ThinLTO/X86/distributed_import.ll
index a67fe20bd..b65c3be14 100644
--- a/llvm/test/ThinLTO/X86/distributed_import.ll
+++ b/llvm/test/ThinLTO/X86/distributed_import.ll
@@ -59,7 +59,7 @@
 ; EXPORT: @G.llvm.
 
 target triple = "x86_64-unknown-linux-gnu"
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 
 declare i32 @g(...)
 declare void @analias(...)
diff --git a/llvm/test/ThinLTO/X86/dot-dumper-full-lto.ll b/llvm/test/ThinLTO/X86/dot-dumper-full-lto.ll
index 6d4d1efa9..d8acfaaf0 100644
--- a/llvm/test/ThinLTO/X86/dot-dumper-full-lto.ll
+++ b/llvm/test/ThinLTO/X86/dot-dumper-full-lto.ll
@@ -14,7 +14,7 @@
 ; CHECK: // Cross-module edges:
 ; CHECK:  M4294967295_[[ID]] -> M0_{{[0-9]+}}{{.*}}// ref
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @A = external global i32
diff --git a/llvm/test/ThinLTO/X86/dot-dumper.ll b/llvm/test/ThinLTO/X86/dot-dumper.ll
index d546d90a2..22cbdec28 100644
--- a/llvm/test/ThinLTO/X86/dot-dumper.ll
+++ b/llvm/test/ThinLTO/X86/dot-dumper.ll
@@ -21,7 +21,7 @@
 ; PERMODULE-NEXT:    label = "";
 ; PERMODULE-NEXT:    node [style=filled,fillcolor=lightblue];
 ; PERMODULE-NEXT:    M0_[[MAIN_ALIAS:[0-9]+]] [style="dotted,filled",shape="box",label="main_alias",fillcolor="red"]; // alias, dead
-; PERMODULE-NEXT:    M0_[[MAIN:[0-9]+]] [shape="record",label="main|extern (inst: 4, ffl: 00000)}",fillcolor="red"]; // function, dead
+; PERMODULE-NEXT:    M0_[[MAIN:[0-9]+]] [shape="record",label="main|extern (inst: 4, ffl: 000000)}",fillcolor="red"]; // function, dead
 ; PERMODULE-NEXT:    // Edges:
 ; PERMODULE-NEXT:    M0_[[MAIN_ALIAS]] -> M0_[[MAIN]] [style=dotted]; // alias
 ; PERMODULE-NEXT:  }
@@ -40,7 +40,7 @@
 ; COMBINED-NEXT:    label = "dot-dumper{{.*}}1.bc";
 ; COMBINED-NEXT:    node [style=filled,fillcolor=lightblue];
 ; COMBINED-NEXT:    M0_[[MAIN_ALIAS:[0-9]+]] [style="dotted,filled",shape="box",label="main_alias",fillcolor="red"]; // alias, dead
-; COMBINED-NEXT:    M0_[[MAIN:[0-9]+]] [shape="record",label="main|extern (inst: 4, ffl: 00000)}"]; // function
+; COMBINED-NEXT:    M0_[[MAIN:[0-9]+]] [shape="record",label="main|extern (inst: 4, ffl: 000000)}"]; // function, preserved
 ; COMBINED-NEXT:    // Edges:
 ; COMBINED-NEXT:    M0_[[MAIN_ALIAS]] -> M0_[[MAIN]] [style=dotted]; // alias
 ; COMBINED-NEXT:  }
@@ -50,10 +50,10 @@
 ; COMBINED-NEXT:    color = lightgrey;
 ; COMBINED-NEXT:    label = "dot-dumper{{.*}}2.bc";
 ; COMBINED-NEXT:    node [style=filled,fillcolor=lightblue];
-; COMBINED-NEXT:    M1_[[FOO:[0-9]+]] [shape="record",label="foo|extern (inst: 4, ffl: 00001)}"]; // function
+; COMBINED-NEXT:    M1_[[FOO:[0-9]+]] [shape="record",label="foo|extern (inst: 4, ffl: 000010)}"]; // function
 ; COMBINED-NEXT:    M1_[[A:[0-9]+]] [shape="Mrecord",label="A|extern}"]; // variable, immutable
 ; COMBINED-NEXT:    M1_[[B:[0-9]+]] [shape="Mrecord",label="B|extern}"]; // variable, immutable
-; COMBINED-NEXT:    M1_{{[0-9]+}} [shape="record",label="bar|extern (inst: 1, ffl: 00000)}",fillcolor="red"]; // function, dead
+; COMBINED-NEXT:    M1_{{[0-9]+}} [shape="record",label="bar|extern (inst: 1, ffl: 000000)}",fillcolor="red"]; // function, dead
 ; COMBINED-NEXT:    // Edges:
 ; COMBINED-NEXT:    M1_[[FOO]] -> M1_[[B]] [style=dashed,color=forestgreen]; // const-ref
 ; COMBINED-NEXT:    M1_[[FOO]] -> M1_[[A]] [style=dashed,color=forestgreen]; // const-ref
@@ -63,7 +63,7 @@
 ; COMBINED-NEXT:  M0_[[MAIN]] -> M1_[[FOO]] // call (hotness : Unknown)
 ; COMBINED-NEXT: }
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @A = external local_unnamed_addr global i32, align 4
diff --git a/llvm/test/ThinLTO/X86/dot-dumper2.ll b/llvm/test/ThinLTO/X86/dot-dumper2.ll
index 8cc9822ae..95bfa020f 100644
--- a/llvm/test/ThinLTO/X86/dot-dumper2.ll
+++ b/llvm/test/ThinLTO/X86/dot-dumper2.ll
@@ -15,7 +15,7 @@
 ; COMBINED-NEXT:    color = lightgrey;
 ; COMBINED-NEXT:    label =
 ; COMBINED-NEXT:    node [style=filled,fillcolor=lightblue];
-; COMBINED-NEXT:    M0_[[MAIN:[0-9]+]] [shape="record",label="main|extern (inst: 2, ffl: 00000)}"]; // function
+; COMBINED-NEXT:    M0_[[MAIN:[0-9]+]] [shape="record",label="main|extern (inst: 2, ffl: 000000)}"]; // function
 ; COMBINED-NEXT:    // Edges:
 ; COMBINED-NEXT:  }
 ; COMBINED-NEXT:  // Module:
@@ -31,7 +31,7 @@
 ; COMBINED-NEXT:  M0_[[MAIN]] -> M1_[[A]] [style=dashed,color=violetred]; // writeOnly-ref
 ; COMBINED-NEXT: }
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @A = external local_unnamed_addr global i32, align 4
diff --git a/llvm/test/ThinLTO/X86/drop-debug-info.ll b/llvm/test/ThinLTO/X86/drop-debug-info.ll
index a097d6bac..7bbbb9e59 100644
--- a/llvm/test/ThinLTO/X86/drop-debug-info.ll
+++ b/llvm/test/ThinLTO/X86/drop-debug-info.ll
@@ -7,7 +7,7 @@
 ; CHECK: define available_externally void @globalfunc
 ; CHECK-NOT: llvm.dbg.value
 
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.11.0"
 
 
diff --git a/llvm/test/ThinLTO/X86/dsolocal_dllimport.ll b/llvm/test/ThinLTO/X86/dsolocal_dllimport.ll
index 12f097f39..2c3815a15 100644
--- a/llvm/test/ThinLTO/X86/dsolocal_dllimport.ll
+++ b/llvm/test/ThinLTO/X86/dsolocal_dllimport.ll
@@ -7,7 +7,7 @@
 
 ; CHECK: declare dso_local void @foo()
 
-target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:w-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-pc-windows-msvc19.11.0"
 define void @bar() {
   call void @foo()
diff --git a/llvm/test/ThinLTO/X86/emit_imports.ll b/llvm/test/ThinLTO/X86/emit_imports.ll
index 41dc1482c..424626465 100644
--- a/llvm/test/ThinLTO/X86/emit_imports.ll
+++ b/llvm/test/ThinLTO/X86/emit_imports.ll
@@ -44,7 +44,7 @@
 ; The index file should be created even for the input with an empty summary.
 ; RUN: ls %t3.bc.thinlto.bc
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 declare void @g(...)
diff --git a/llvm/test/ThinLTO/X86/empty-module.ll b/llvm/test/ThinLTO/X86/empty-module.ll
index 285e1bcec..01e708941 100644
--- a/llvm/test/ThinLTO/X86/empty-module.ll
+++ b/llvm/test/ThinLTO/X86/empty-module.ll
@@ -8,7 +8,7 @@
 
 ; CHECK: Format: ELF64-x86-64
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @foo = ifunc i32 (i32), i64 ()* @foo_ifunc
diff --git a/llvm/test/ThinLTO/X86/empty_module_with_cache.ll b/llvm/test/ThinLTO/X86/empty_module_with_cache.ll
index 76fe3e91d..8e58d9f0d 100644
--- a/llvm/test/ThinLTO/X86/empty_module_with_cache.ll
+++ b/llvm/test/ThinLTO/X86/empty_module_with_cache.ll
@@ -31,5 +31,5 @@
 ; RUN: ls %t.cache | count 0
 
 
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.11.0"
diff --git a/llvm/test/ThinLTO/X86/export.ll b/llvm/test/ThinLTO/X86/export.ll
index cb87ddd06..b0db1e746 100644
--- a/llvm/test/ThinLTO/X86/export.ll
+++ b/llvm/test/ThinLTO/X86/export.ll
@@ -8,7 +8,7 @@
 ; CHECK-DAG: @staticvar.llvm.0 = hidden global
 ; CHECK-DAG: define hidden void @staticfunc.llvm.0
 
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.11.0"
 
 @staticvar = internal global i32 1, align 4
diff --git a/llvm/test/ThinLTO/X86/funcimport-debug.ll b/llvm/test/ThinLTO/X86/funcimport-debug.ll
index ca6128a51..4e03813b6 100644
--- a/llvm/test/ThinLTO/X86/funcimport-debug.ll
+++ b/llvm/test/ThinLTO/X86/funcimport-debug.ll
@@ -21,7 +21,7 @@
 ; CHECK-DAG: ref -> 7546896869197086323 (baz)
 ; CHECK-DAG: edge -> 6699318081062747564 (foo) Threshold:100
 
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.11.0"
 
 define i32 @main() #0 {
diff --git a/llvm/test/ThinLTO/X86/funcimport-stats.ll b/llvm/test/ThinLTO/X86/funcimport-stats.ll
new file mode 100644
index 000000000..913b13004
--- /dev/null
+++ b/llvm/test/ThinLTO/X86/funcimport-stats.ll
@@ -0,0 +1,21 @@
+; REQUIRES: asserts
+; RUN: opt -module-summary %p/funcimport.ll -o %t.bc
+; RUN: opt -module-summary %p/Inputs/funcimport.ll -o %t2.bc
+; RUN: llvm-lto -thinlto-action=thinlink -o %t3.bc %t.bc %t2.bc
+
+; RUN: llvm-lto -thinlto-action=import %t2.bc -thinlto-index=%t3.bc -o /dev/null -debug-only=function-import -stats > %t4 2>&1
+; RUN: cat %t4 | grep 'Is importing global' | count 4
+; RUN: cat %t4 | grep 'Is importing function' | count 8
+; RUN: cat %t4 | grep 'Is importing aliasee' | count 1
+; RUN: cat %t4 | FileCheck %s
+
+; CHECK:      - [[NUM_FUNCS:[0-9]+]] functions imported from
+; CHECK-NEXT: - [[NUM_VARS:[0-9]+]] global vars imported from
+
+; CHECK:      [[NUM_FUNCS]] function-import - Number of functions imported in backend
+; CHECK-NEXT: [[NUM_FUNCS]] function-import - Number of functions thin link decided to import
+; CHECK-NEXT: [[NUM_VARS]] function-import - Number of global variables imported in backend
+; CHECK-NEXT: [[NUM_VARS]] function-import - Number of global variables thin link decided to import
+; CHECK-NEXT: 1 function-import - Number of modules imported from
+; CHECK-NEXT: [[NUM_VARS]] module-summary-index - Number of live global variables marked read only
+; CHECK-NEXT: 1 module-summary-index - Number of live global variables marked write only
diff --git a/llvm/test/ThinLTO/X86/funcimport-tbaa.ll b/llvm/test/ThinLTO/X86/funcimport-tbaa.ll
index c3dfd7d90..5334fbecb 100644
--- a/llvm/test/ThinLTO/X86/funcimport-tbaa.ll
+++ b/llvm/test/ThinLTO/X86/funcimport-tbaa.ll
@@ -10,7 +10,7 @@
 ; RUN:  | llvm-dis -o - | FileCheck %s --check-prefix=IMPORTGLOB1
 ; IMPORTGLOB1: define available_externally float @globalfunc1
 
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.11.0"
 
 define float @globalfunc1(i32*, float*) {
diff --git a/llvm/test/ThinLTO/X86/funcimport.ll b/llvm/test/ThinLTO/X86/funcimport.ll
index fa1bdbf9f..cb75d55ed 100644
--- a/llvm/test/ThinLTO/X86/funcimport.ll
+++ b/llvm/test/ThinLTO/X86/funcimport.ll
@@ -46,7 +46,7 @@
 ; ALL: T _callfuncptr
 ; ALL2: T _main
 
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.11.0"
 
 @globalvar_in_section = global i32 1, align 4
diff --git a/llvm/test/ThinLTO/X86/funcimport2.ll b/llvm/test/ThinLTO/X86/funcimport2.ll
index 15087be6b..9792fbc57 100644
--- a/llvm/test/ThinLTO/X86/funcimport2.ll
+++ b/llvm/test/ThinLTO/X86/funcimport2.ll
@@ -33,7 +33,7 @@
 ; RUN: llvm-dis %t.o.2.3.import.bc -o - | FileCheck %s --check-prefix=NOIMPORT
 ; NOIMPORT: declare dso_local void @foo(...)
 
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.11.0"
 
 define void @foo() #0 {
diff --git a/llvm/test/ThinLTO/X86/funcimport_alwaysinline.ll b/llvm/test/ThinLTO/X86/funcimport_alwaysinline.ll
new file mode 100644
index 000000000..aa4947602
--- /dev/null
+++ b/llvm/test/ThinLTO/X86/funcimport_alwaysinline.ll
@@ -0,0 +1,26 @@
+; RUN: opt -module-summary %s -o %t1.bc
+; RUN: opt -module-summary %p/Inputs/funcimport_alwaysinline.ll -o %t2.bc
+
+; RUN: llvm-lto2 run %t1.bc %t2.bc -o %t.o -save-temps \
+; RUN:     -r=%t1.bc,foo,plx \
+; RUN:     -r=%t2.bc,main,plx \
+; RUN:     -r=%t2.bc,foo,l \
+; RUN:     -import-instr-limit=0
+; RUN: llvm-dis %t.o.2.3.import.bc -o - | FileCheck %s --check-prefix=CHECK1
+; RUN: llvm-dis %t.o.index.bc -o - | FileCheck %s --check-prefix=CHECK2
+
+; foo() being always_inline should be imported irrespective of the
+; instruction limit
+; CHECK1: define available_externally dso_local void @foo()
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Function Attrs: alwaysinline nounwind uwtable
+define void @foo() #0 {
+entry:
+  ret void
+}
+
+attributes #0 = { alwaysinline nounwind uwtable }
+; CHECK2: ^2 = gv: (guid: {{.*}}, summaries: (function: (module: ^0, flags: (linkage: external, notEligibleToImport: 0, live: 1, dsoLocal: 1, canAutoHide: 0), insts: 1, funcFlags: (readNone: 0, readOnly: 0, noRecurse: 0, returnDoesNotAlias: 0, noInline: 0, alwaysInline: 1))))
diff --git a/llvm/test/ThinLTO/X86/function_entry_count.ll b/llvm/test/ThinLTO/X86/function_entry_count.ll
index 4ca2ec26a..686c85291 100644
--- a/llvm/test/ThinLTO/X86/function_entry_count.ll
+++ b/llvm/test/ThinLTO/X86/function_entry_count.ll
@@ -21,7 +21,7 @@
 ; CHECK-DAG: ![[PROF2]] = !{!"synthetic_function_entry_count", i64 198}
 
 target triple = "x86_64-unknown-linux-gnu"
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 
 declare void @g();
 
diff --git a/llvm/test/ThinLTO/X86/globals-import-blockaddr.ll b/llvm/test/ThinLTO/X86/globals-import-blockaddr.ll
index d4ed67403..9bbbf76f1 100644
--- a/llvm/test/ThinLTO/X86/globals-import-blockaddr.ll
+++ b/llvm/test/ThinLTO/X86/globals-import-blockaddr.ll
@@ -6,7 +6,7 @@
 ; Verify that we haven't imported GV containing blockaddress
 ; CHECK: @label_addr.llvm.0 = external hidden constant
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 declare dso_local [1 x i8*]* @foo();
diff --git a/llvm/test/ThinLTO/X86/globals-import-const-fold.ll b/llvm/test/ThinLTO/X86/globals-import-const-fold.ll
index a250ed2e9..663b4ee7e 100644
--- a/llvm/test/ThinLTO/X86/globals-import-const-fold.ll
+++ b/llvm/test/ThinLTO/X86/globals-import-const-fold.ll
@@ -12,7 +12,7 @@
 ; OPTIMIZE:       define i32 @main()
 ; OPTIMIZE-NEXT:    ret i32 10
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-pc-linux-gnu"
 
 @baz = external local_unnamed_addr constant i32, align 4
diff --git a/llvm/test/ThinLTO/X86/globals-import.ll b/llvm/test/ThinLTO/X86/globals-import.ll
index 9fe1ebefe..d784398f7 100644
--- a/llvm/test/ThinLTO/X86/globals-import.ll
+++ b/llvm/test/ThinLTO/X86/globals-import.ll
@@ -15,7 +15,7 @@
 ; RUN: llvm-dis %t2.bc.thinlto.promoted.bc -o - | FileCheck --check-prefix=PROMOTE1 %s
 ; RUN: llvm-dis %t2b.bc.thinlto.promoted.bc -o - | FileCheck --check-prefix=PROMOTE2 %s
 
-; IMPORT: @baz.llvm.0 = available_externally hidden constant i32 10, align 4
+; IMPORT: @baz.llvm.0 = internal constant i32 10, align 4
 
 ; PROMOTE1: @baz.llvm.0 = hidden constant i32 10, align 4
 ; PROMOTE1: define weak_odr i32 @foo() {
@@ -24,7 +24,7 @@
 ; PROMOTE2: @baz = internal constant i32 10, align 4
 ; PROMOTE2: define available_externally i32 @foo() {
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-pc-linux-gnu"
 
 declare i32 @foo()
diff --git a/llvm/test/ThinLTO/X86/guid_collision.ll b/llvm/test/ThinLTO/X86/guid_collision.ll
new file mode 100644
index 000000000..5ba39b8c0
--- /dev/null
+++ b/llvm/test/ThinLTO/X86/guid_collision.ll
@@ -0,0 +1,37 @@
+; Make sure LTO succeeds even if %t.bc contains a GlobalVariable F and
+; %t2.bc cointains a Function F with the same GUID.
+;
+; RUN: opt -module-summary %s -o %t.bc
+; RUN: opt -module-summary %p/Inputs/guid_collision.ll -o %t2.bc
+; RUN: llvm-lto2 run %t.bc %t2.bc -o %t.out -save-temps \
+; RUN: -r=%t.bc,H,px -r=%t.bc,G, -r=%t2.bc,G,px
+; RUN: llvm-dis -o - %t.out.1.3.import.bc | FileCheck %s
+
+; RUN: llvm-lto2 run %t.bc %t2.bc -o %t.out -thinlto-distributed-indexes \
+; RUN: -r=%t.bc,H,px -r=%t.bc,G, -r=%t2.bc,G,px
+; RUN: opt -function-import -import-all-index -summary-file %t.bc.thinlto.bc %t.bc -o %t.out
+; RUN: llvm-dis -o - %t.out | FileCheck %s
+
+; Sanity check that G was imported
+; CHECK: define available_externally i64 @G
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-linux-gnu"
+
+; The source for the GUID for this symbol will be -:F
+source_filename = "-"
+
+@F = internal constant i8 0
+
+; Provide a global that has the same name as one from the module we import G
+; from, to test handling of a global variable with an entry in the distributed
+; index but not with a copy in the source module (since we can't import
+; appending linkage globals).
+@llvm.global_ctors = appending global [0 x { i32, void ()*, i8* }] zeroinitializer
+
+define i64 @H() {
+  call i64 @G()
+  ret i64 0
+}
+
+declare i64 @G()
diff --git a/llvm/test/ThinLTO/X86/import_opaque_type.ll b/llvm/test/ThinLTO/X86/import_opaque_type.ll
index bfa251aba..0a49116a6 100644
--- a/llvm/test/ThinLTO/X86/import_opaque_type.ll
+++ b/llvm/test/ThinLTO/X86/import_opaque_type.ll
@@ -7,7 +7,7 @@
 ; RUN: llvm-lto -thinlto-action=import %t.bc -thinlto-index=%t3.bc -o - | llvm-dis -o - | FileCheck %s
 
 
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.11.0"
 
 ; CHECK: %0 = type { i8 }
diff --git a/llvm/test/ThinLTO/X86/index-const-prop-O0.ll b/llvm/test/ThinLTO/X86/index-const-prop-O0.ll
index 6426d7268..c1e945010 100644
--- a/llvm/test/ThinLTO/X86/index-const-prop-O0.ll
+++ b/llvm/test/ThinLTO/X86/index-const-prop-O0.ll
@@ -7,7 +7,7 @@
 ; read-only globals
 ; CHECK: @g = dso_local global i32 42
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @g = external global i32
diff --git a/llvm/test/ThinLTO/X86/index-const-prop-alias.ll b/llvm/test/ThinLTO/X86/index-const-prop-alias.ll
index 592fc9e08..bd15b5a62 100644
--- a/llvm/test/ThinLTO/X86/index-const-prop-alias.ll
+++ b/llvm/test/ThinLTO/X86/index-const-prop-alias.ll
@@ -26,7 +26,7 @@
 ; PRESERVED:      @g.alias = external dso_local global i32
 ; PRESERVED-NEXT: @g = available_externally dso_local global i32 42, align 4
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @g.alias = external global i32
diff --git a/llvm/test/ThinLTO/X86/index-const-prop-cache.ll b/llvm/test/ThinLTO/X86/index-const-prop-cache.ll
index 62a22af0b..4e45d5e8d 100644
--- a/llvm/test/ThinLTO/X86/index-const-prop-cache.ll
+++ b/llvm/test/ThinLTO/X86/index-const-prop-cache.ll
@@ -29,7 +29,7 @@
 ; RUN:  -r=%t2.bc,foo,pl -r=%t2.bc,bar,pl -r=%t2.bc,rand,
 ; RUN: ls %t.cache/llvmcache-* | count 6
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 ; Function Attrs: nounwind ssp uwtable
diff --git a/llvm/test/ThinLTO/X86/index-const-prop-comdat.ll b/llvm/test/ThinLTO/X86/index-const-prop-comdat.ll
index e05aaf98d..2fdc793b3 100644
--- a/llvm/test/ThinLTO/X86/index-const-prop-comdat.ll
+++ b/llvm/test/ThinLTO/X86/index-const-prop-comdat.ll
@@ -6,7 +6,7 @@
 ; Comdats are not internalized even if they are read only.
 ; CHECK: @g = available_externally dso_local global i32 42 
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @g = external global i32
diff --git a/llvm/test/ThinLTO/X86/index-const-prop-dead.ll b/llvm/test/ThinLTO/X86/index-const-prop-dead.ll
index f3ffca8cb..44fc350e9 100644
--- a/llvm/test/ThinLTO/X86/index-const-prop-dead.ll
+++ b/llvm/test/ThinLTO/X86/index-const-prop-dead.ll
@@ -8,7 +8,7 @@
 ; If we try to internalize such we'll get a broken module. 
 ; CHECK: @g = external dso_local global i32
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @g = external global i32
diff --git a/llvm/test/ThinLTO/X86/index-const-prop-full-lto.ll b/llvm/test/ThinLTO/X86/index-const-prop-full-lto.ll
index 26fe4d644..f6260c535 100644
--- a/llvm/test/ThinLTO/X86/index-const-prop-full-lto.ll
+++ b/llvm/test/ThinLTO/X86/index-const-prop-full-lto.ll
@@ -10,7 +10,7 @@
 ; We cannot internalize @g
 ; CHECK: @g = available_externally dso_local global i32 42
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 declare i32 @foo()
diff --git a/llvm/test/ThinLTO/X86/index-const-prop-gvref.ll b/llvm/test/ThinLTO/X86/index-const-prop-gvref.ll
index 87103fee1..972283774 100644
--- a/llvm/test/ThinLTO/X86/index-const-prop-gvref.ll
+++ b/llvm/test/ThinLTO/X86/index-const-prop-gvref.ll
@@ -13,7 +13,7 @@
 ; DEST:      @b = external dso_local global i32*
 ; DEST-NEXT: @a = available_externally dso_local global i32 42, align 4
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @a = external global i32
diff --git a/llvm/test/ThinLTO/X86/index-const-prop-ldst.ll b/llvm/test/ThinLTO/X86/index-const-prop-ldst.ll
index 4646557b6..3c6d03cb7 100644
--- a/llvm/test/ThinLTO/X86/index-const-prop-ldst.ll
+++ b/llvm/test/ThinLTO/X86/index-const-prop-ldst.ll
@@ -7,7 +7,7 @@
 ; even when there is 'load' instruction before it.
 ; CHECK: @g = available_externally dso_local global i32 42
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @g = external global i32
diff --git a/llvm/test/ThinLTO/X86/index-const-prop-linkage.ll b/llvm/test/ThinLTO/X86/index-const-prop-linkage.ll
index aac917721..2c7c79437 100644
--- a/llvm/test/ThinLTO/X86/index-const-prop-linkage.ll
+++ b/llvm/test/ThinLTO/X86/index-const-prop-linkage.ll
@@ -14,7 +14,7 @@
 ; CHECK-NEXT: @g2 = available_externally dso_local global i32 42, align 4
 ; CHECK-NEXT: @g3 = available_externally global i32 42, align 4
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 declare i32 @foo()
diff --git a/llvm/test/ThinLTO/X86/index-const-prop.ll b/llvm/test/ThinLTO/X86/index-const-prop.ll
index 9dac8b81b..9718aec8a 100644
--- a/llvm/test/ThinLTO/X86/index-const-prop.ll
+++ b/llvm/test/ThinLTO/X86/index-const-prop.ll
@@ -28,7 +28,7 @@
 
 ; IMPORT2: @gBar = available_externally local_unnamed_addr global i32 2, align 4, !dbg !5
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-pc-linux-gnu"
 
 @gBar = external global i32
diff --git a/llvm/test/ThinLTO/X86/index-const-prop2.ll b/llvm/test/ThinLTO/X86/index-const-prop2.ll
index a5fbbdf1e..928d00adc 100644
--- a/llvm/test/ThinLTO/X86/index-const-prop2.ll
+++ b/llvm/test/ThinLTO/X86/index-const-prop2.ll
@@ -36,6 +36,8 @@
 ; RUN:  -o %t4
 ; RUN: llvm-dis %t4.1.3.import.bc -o - | FileCheck %s --check-prefix=IMPORT2
 
+; Run again but with main2 exported instead of main to check that write only
+; variables are optimized out.
 ; RUN: llvm-lto2 run %t1.bc %t2.bc -save-temps \
 ; RUN:  -r=%t2.bc,foo,pl \
 ; RUN:  -r=%t2.bc,bar,pl \
@@ -49,7 +51,7 @@
 ; RUN:  -r=%t1.bc,baz, \
 ; RUN:  -r=%t1.bc,gBar, \
 ; RUN:  -o %t5
-; RUN: llvm-dis %t5.1.3.import.bc -o - | FileCheck %s --check-prefix=IMPORT
+; RUN: llvm-dis %t5.1.3.import.bc -o - | FileCheck %s --check-prefix=IMPORT-WRITEONLY
 ; RUN: llvm-dis %t5.1.5.precodegen.bc -o - | FileCheck %s --check-prefix=CODEGEN2
 ; Check that gFoo and gBar were eliminated from source module together
 ; with corresponsing stores
@@ -59,6 +61,10 @@
 ; IMPORT-NEXT:  @gBar = internal local_unnamed_addr global i32 2, align 4
 ; IMPORT:       !DICompileUnit({{.*}})
 
+; Write only variables are imported with a zero initializer.
+; IMPORT-WRITEONLY:  @gFoo.llvm.0 = internal unnamed_addr global i32 0
+; IMPORT-WRITEONLY:  @gBar = internal local_unnamed_addr global i32 0
+
 ; CODEGEN:        i32 @main()
 ; CODEGEN-NEXT:     ret i32 3
 
@@ -74,7 +80,7 @@
 ; CODEGEN2-SRC-NEXT:    %2 = tail call i32 @rand()
 ; CODEGEN2-SRC-NEXT:    ret void
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-pc-linux-gnu"
 
 ; We should be able to link external definition of gBar to its declaration
diff --git a/llvm/test/ThinLTO/X86/internalize.ll b/llvm/test/ThinLTO/X86/internalize.ll
index 49cab144b..edd5abe8a 100644
--- a/llvm/test/ThinLTO/X86/internalize.ll
+++ b/llvm/test/ThinLTO/X86/internalize.ll
@@ -1,5 +1,8 @@
 ; RUN: opt -module-summary %s -o %t1.bc
-; RUN: llvm-lto -thinlto-action=thinlink -o %t.index.bc %t1.bc
+; RUN: opt -module-summary %p/Inputs/internalize.ll -o %t2.bc
+; Link in %t2.bc first to force its copy of @weak_func_nonprevailing as
+; prevailing the %t1.bc copy as non-prevailing.
+; RUN: llvm-lto -thinlto-action=thinlink -o %t.index.bc %t2.bc %t1.bc
 ; RUN: llvm-lto -thinlto-action=internalize -thinlto-index %t.index.bc %t1.bc -o - | llvm-dis -o - | FileCheck %s --check-prefix=REGULAR
 ; RUN: llvm-lto -thinlto-action=internalize -thinlto-index %t.index.bc %t1.bc -o -  --exported-symbol=foo | llvm-dis -o - | FileCheck %s --check-prefix=INTERNALIZE
 
@@ -13,7 +16,10 @@
 ; RUN: llvm-lto2 run %t1.bc -o %t.o -save-temps \
 ; RUN:     -r=%t1.bc,_foo,pxl \
 ; RUN:     -r=%t1.bc,_bar,pl \
-; RUN:     -r=%t1.bc,_linkonce_func,pl
+; RUN:     -r=%t1.bc,_linkonce_func,pl \
+; RUN:     -r=%t1.bc,_weak_func_prevailing,pl \
+; RUN:     -r=%t1.bc,_alias1,plx \
+; RUN:     -r=%t1.bc,_weak_func_nonprevailing,l
 ; RUN: llvm-dis < %t.o.1.2.internalize.bc | FileCheck  %s --check-prefix=INTERNALIZE2
 
 ; Test the enable-lto-internalization option by setting it to false.
@@ -22,7 +28,10 @@
 ; RUN: llvm-lto2 run %t1.bc -o %t.o -save-temps -enable-lto-internalization=false \
 ; RUN:     -r=%t1.bc,_foo,pxl \
 ; RUN:     -r=%t1.bc,_bar,pl \
-; RUN:     -r=%t1.bc,_linkonce_func,pl
+; RUN:     -r=%t1.bc,_linkonce_func,pl \
+; RUN:     -r=%t1.bc,_weak_func_prevailing,pl \
+; RUN:     -r=%t1.bc,_alias1,plx \
+; RUN:     -r=%t1.bc,_weak_func_nonprevailing,l
 ; RUN: llvm-dis < %t.o.1.2.internalize.bc | FileCheck  %s --check-prefix=INTERNALIZE2-OPTION-DISABLE
 
 ; REGULAR: define void @foo
@@ -31,17 +40,25 @@
 ; INTERNALIZE: define void @foo
 ; INTERNALIZE: define internal void @bar
 ; INTERNALIZE: define internal void @linkonce_func()
+; INTERNALIZE: define internal void @weak_func_prevailing()
+; INTERNALIZE: define weak void @weak_func_nonprevailing()
 ; INTERNALIZE-OPTION-DISABLE: define void @foo
 ; INTERNALIZE-OPTION-DISABLE: define void @bar
 ; INTERNALIZE-OPTION-DISABLE: define weak void @linkonce_func()
+; INTERNALIZE-OPTION-DISABLE: define weak void @weak_func_prevailing()
+; INTERNALIZE-OPTION-DISABLE: define weak void @weak_func_nonprevailing()
 ; INTERNALIZE2: define dso_local void @foo
 ; INTERNALIZE2: define internal void @bar
 ; INTERNALIZE2: define internal void @linkonce_func()
+; INTERNALIZE2: define internal void @weak_func_prevailing()
+; INTERNALIZE2: define weak dso_local void @weak_func_nonprevailing()
 ; INTERNALIZE2-OPTION-DISABLE: define dso_local void @foo
 ; INTERNALIZE2-OPTION-DISABLE: define dso_local void @bar
 ; INTERNALIZE2-OPTION-DISABLE: define weak dso_local void @linkonce_func()
+; INTERNALIZE2-OPTION-DISABLE: define weak dso_local void @weak_func_prevailing()
+; INTERNALIZE2-OPTION-DISABLE: define weak dso_local void @weak_func_nonprevailing()
 
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.11.0"
 
 define void @foo() {
@@ -50,8 +67,20 @@ define void @foo() {
 }
 define void @bar() {
     call void @linkonce_func()
+    call void @weak_func_prevailing()
+    call void @weak_func_nonprevailing()
 	ret void
 }
 define linkonce void @linkonce_func() {
     ret void
 }
+define weak void @weak_func_prevailing() {
+    ret void
+}
+; Make @weak_func_nonprevailing an aliasee to ensure it is still marked
+; live and kept as a definition even when non-prevailing. We want to ensure
+; this definition is not internalized.
+@alias1 = hidden alias void (), void ()* @weak_func_nonprevailing
+define weak void @weak_func_nonprevailing() {
+    ret void
+}
diff --git a/llvm/test/ThinLTO/X86/lazyload_metadata.ll b/llvm/test/ThinLTO/X86/lazyload_metadata.ll
index b590ffd41..351201982 100644
--- a/llvm/test/ThinLTO/X86/lazyload_metadata.ll
+++ b/llvm/test/ThinLTO/X86/lazyload_metadata.ll
@@ -7,20 +7,19 @@
 ; Check that importing @globalfunc1 does not trigger loading all the global
 ; metadata for @globalfunc2 and @globalfunc3
 
-; RUN: llvm-lto -thinlto-action=import %t2.bc -thinlto-index=%t3.bc \
-; RUN:          -o /dev/null -stats \
-; RUN:  2>&1 | FileCheck %s -check-prefix=LAZY
-; LAZY: 61 bitcode-reader  - Number of Metadata records loaded
-; LAZY: 2 bitcode-reader  - Number of MDStrings loaded
+; RUN: llvm-lto -thinlto-action=import %t2.bc -thinlto-index=%t3.bc -stats 2> %t4.txt
+; RUN: llvm-lto -thinlto-action=import %t2.bc -thinlto-index=%t3.bc -stats -disable-ondemand-mds-loading 2> %t5.txt
+; RUN: cat %t4.txt %t5.txt | FileCheck %s
 
-; RUN: llvm-lto -thinlto-action=import %t2.bc -thinlto-index=%t3.bc \
-; RUN:          -o /dev/null -disable-ondemand-mds-loading -stats \
-; RUN:  2>&1 | FileCheck %s -check-prefix=NOTLAZY
-; NOTLAZY: 70 bitcode-reader  - Number of Metadata records loaded
-; NOTLAZY: 7 bitcode-reader  - Number of MDStrings loaded
+; Check llvm-lto call with lazy loading enabled
+; CHECK: [[#LAZY_RECORDS:]] bitcode-reader  - Number of Metadata records loaded
+; CHECK: 2 bitcode-reader  - Number of MDStrings loaded
 
+; Check llvm-lto call with lazy loading disabled
+; CHECK: [[#LAZY_RECORDS+9]] bitcode-reader  - Number of Metadata records loaded
+; CHECK: 7 bitcode-reader  - Number of MDStrings loaded
 
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.11.0"
 
 define void @globalfunc1(i32 %arg) {
diff --git a/llvm/test/ThinLTO/X86/linkonce_aliasee_ref_import.ll b/llvm/test/ThinLTO/X86/linkonce_aliasee_ref_import.ll
index 9086d9824..2cff8b4b2 100644
--- a/llvm/test/ThinLTO/X86/linkonce_aliasee_ref_import.ll
+++ b/llvm/test/ThinLTO/X86/linkonce_aliasee_ref_import.ll
@@ -24,7 +24,7 @@
 ; Check that we imported a ref (and not def) to baz.clone
 ; NM2: U baz.clone
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-grtev4-linux-gnu"
 
 $baz.clone = comdat any
diff --git a/llvm/test/ThinLTO/X86/linkonce_odr_unnamed_addr.ll b/llvm/test/ThinLTO/X86/linkonce_odr_unnamed_addr.ll
index 68ba48ebd..8b8e3677c 100644
--- a/llvm/test/ThinLTO/X86/linkonce_odr_unnamed_addr.ll
+++ b/llvm/test/ThinLTO/X86/linkonce_odr_unnamed_addr.ll
@@ -17,7 +17,7 @@
 ; RUN: llvm-lto2 run -save-temps -o %t6.bc %t.bc %t4.bc -r=%t.bc,linkonceodrunnamed,p -r=%t.bc,odrunnamed,p -r=%t4.bc,linkonceodrunnamed, -r=%t4.bc,odrunnamed,
 ; RUN: llvm-dis %t6.bc.1.1.promote.bc -o - | FileCheck %s --check-prefix=NOSUMMARY
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-grtev4-linux-gnu"
 
 ; In this case all copies are linkonce_odr, so it may be hidden.
diff --git a/llvm/test/ThinLTO/X86/linkonce_resolution_comdat.ll b/llvm/test/ThinLTO/X86/linkonce_resolution_comdat.ll
index c1ffdf176..2d8b7e773 100644
--- a/llvm/test/ThinLTO/X86/linkonce_resolution_comdat.ll
+++ b/llvm/test/ThinLTO/X86/linkonce_resolution_comdat.ll
@@ -10,8 +10,8 @@
 ; Copy from first module is prevailing and converted to weak_odr, copy
 ; from second module is preempted and converted to available_externally and
 ; removed from comdat.
-; IMPORT1: define weak_odr i32 @f(i8*) unnamed_addr comdat($c1) {
-; IMPORT2: define available_externally i32 @f(i8*) unnamed_addr {
+; IMPORT1: define weak_odr i32 @f(i8* %0) unnamed_addr comdat($c1) {
+; IMPORT2: define available_externally i32 @f(i8* %0) unnamed_addr {
 
 ; RUN: llvm-nm -o - < %t1.bc.thinlto.o | FileCheck %s --check-prefix=NM1
 ; NM1: W f
@@ -21,7 +21,7 @@
 ; and inlined into g()
 ; NM2-NOT: f
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 $c1 = comdat any
diff --git a/llvm/test/ThinLTO/X86/llvm.used.ll b/llvm/test/ThinLTO/X86/llvm.used.ll
index 4b379f72c..266ab2beb 100644
--- a/llvm/test/ThinLTO/X86/llvm.used.ll
+++ b/llvm/test/ThinLTO/X86/llvm.used.ll
@@ -8,7 +8,7 @@
 ; CHECK: define available_externally void @globalfunc
 
 
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.11.0"
 
 
diff --git a/llvm/test/ThinLTO/X86/load-store-caching.ll b/llvm/test/ThinLTO/X86/load-store-caching.ll
index 9ca9c40fd..cb3a5c0d5 100644
--- a/llvm/test/ThinLTO/X86/load-store-caching.ll
+++ b/llvm/test/ThinLTO/X86/load-store-caching.ll
@@ -5,7 +5,7 @@
 ; RUN: opt -module-summary %s -o %t.bc
 ; RUN: llvm-dis %t.bc -o - | FileCheck %s
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 %struct.S = type { %struct.Derived* }
diff --git a/llvm/test/ThinLTO/X86/local_name_conflict.ll b/llvm/test/ThinLTO/X86/local_name_conflict.ll
index 1912297ce..d5497a469 100644
--- a/llvm/test/ThinLTO/X86/local_name_conflict.ll
+++ b/llvm/test/ThinLTO/X86/local_name_conflict.ll
@@ -12,7 +12,7 @@
 ; that module (%t3.bc) to be imported. Check that the imported reference's
 ; promoted name matches the imported copy.
 ; RUN: llvm-lto -thinlto-action=import %t.bc -thinlto-index=%t4.bc -o - | llvm-dis -o - | FileCheck %s --check-prefix=IMPORT
-; IMPORT: @baz.llvm.[[HASH:[0-9]+]] = available_externally hidden constant i32 10, align 4
+; IMPORT: @baz.llvm.[[HASH:[0-9]+]] = internal constant i32 10, align 4
 ; IMPORT: call i32 @foo.llvm.[[HASH]]
 ; IMPORT: define available_externally hidden i32 @foo.llvm.[[HASH]]()
 
@@ -28,7 +28,7 @@
 
 ; ModuleID = 'local_name_conflict_main.o'
 source_filename = "local_name_conflict_main.c"
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 ; Function Attrs: noinline nounwind uwtable
diff --git a/llvm/test/ThinLTO/X86/local_name_conflict_var.ll b/llvm/test/ThinLTO/X86/local_name_conflict_var.ll
index d203fe08e..ed5c6e4fd 100644
--- a/llvm/test/ThinLTO/X86/local_name_conflict_var.ll
+++ b/llvm/test/ThinLTO/X86/local_name_conflict_var.ll
@@ -17,7 +17,7 @@
 
 ; ModuleID = 'local_name_conflict_var_main.o'
 source_filename = "local_name_conflict_var_main.c"
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 ; Function Attrs: noinline nounwind uwtable
diff --git a/llvm/test/ThinLTO/X86/merge-triple.ll b/llvm/test/ThinLTO/X86/merge-triple.ll
index 11ab50236..6c5eac6cf 100644
--- a/llvm/test/ThinLTO/X86/merge-triple.ll
+++ b/llvm/test/ThinLTO/X86/merge-triple.ll
@@ -4,7 +4,7 @@
 ; RUN: llvm-dis < %t1.bc.thinlto.imported.bc | FileCheck %s --check-prefix=CHECK1
 ; RUN: llvm-dis < %t2.bc.thinlto.imported.bc | FileCheck %s --check-prefix=CHECK2
 
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.12.0"
 
 ; CHECK1: target triple = "x86_64-apple-macosx10.12.0"
diff --git a/llvm/test/ThinLTO/X86/module_asm2.ll b/llvm/test/ThinLTO/X86/module_asm2.ll
index b6e2f2350..3f0033849 100644
--- a/llvm/test/ThinLTO/X86/module_asm2.ll
+++ b/llvm/test/ThinLTO/X86/module_asm2.ll
@@ -64,7 +64,7 @@
 ; NM1-NOT: b
 ; NM1-NOT: x
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @b = internal global i32 1, align 4
diff --git a/llvm/test/ThinLTO/X86/module_asm_glob.ll b/llvm/test/ThinLTO/X86/module_asm_glob.ll
index 3ce4816c9..45b6113cb 100644
--- a/llvm/test/ThinLTO/X86/module_asm_glob.ll
+++ b/llvm/test/ThinLTO/X86/module_asm_glob.ll
@@ -17,7 +17,7 @@
 ; NM0: T foo
 ; NM1-NOT: foo
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 module asm "\09.text"
diff --git a/llvm/test/ThinLTO/X86/module_summary_graph_traits.ll b/llvm/test/ThinLTO/X86/module_summary_graph_traits.ll
index d2f1dfbf7..b6a649546 100644
--- a/llvm/test/ThinLTO/X86/module_summary_graph_traits.ll
+++ b/llvm/test/ThinLTO/X86/module_summary_graph_traits.ll
@@ -31,7 +31,7 @@
 ; CHECK-NEXT: {{^}} 0{{$}}
 ; CHECK-NEXT: }
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 declare void @external()
diff --git a/llvm/test/ThinLTO/X86/newpm-basic.ll b/llvm/test/ThinLTO/X86/newpm-basic.ll
index bfcc60c68..860aa4c01 100644
--- a/llvm/test/ThinLTO/X86/newpm-basic.ll
+++ b/llvm/test/ThinLTO/X86/newpm-basic.ll
@@ -3,7 +3,7 @@
 ; RUN:     -r=%t1.bc,_tinkywinky,pxl \
 ; RUN:     -use-new-pm
 
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.11.0"
 
 define void @tinkywinky() {
diff --git a/llvm/test/ThinLTO/X86/nodevirt-nonpromoted-typeid.ll b/llvm/test/ThinLTO/X86/nodevirt-nonpromoted-typeid.ll
new file mode 100644
index 000000000..611a42414
--- /dev/null
+++ b/llvm/test/ThinLTO/X86/nodevirt-nonpromoted-typeid.ll
@@ -0,0 +1,66 @@
+; REQUIRES: x86-registered-target
+
+; Test that index-only devirtualization handles and ignores any
+; type metadata that could not be summarized (because it was internal
+; and could not be promoted due to the fact that the module has
+; no external symbols and therefore could not be assigned a unique
+; identifier). In this case we should simply not get the type
+; metadata summary entries, and no promotion will occur.
+
+; Generate unsplit module with summary for ThinLTO index-based WPD.
+; RUN: opt -thinlto-bc -thinlto-split-lto-unit=false -o %t2.o %s
+
+; Check that we don't have module flag when splitting not enabled for ThinLTO,
+; and that we generate summary information needed for index-based WPD.
+; RUN: llvm-dis -o - %t2.o | FileCheck %s --check-prefix=DIS
+; DIS-NOT: typeIdInfo
+; DIS-NOT: typeidMetadata
+
+; Legacy PM, Index based WPD
+; RUN: llvm-lto2 run %t2.o -save-temps -pass-remarks=. \
+; RUN:   -o %t3 \
+; RUN:   -r=%t2.o,test,plx \
+; RUN:   -r=%t2.o,_ZN1D1mEi,
+; RUN: llvm-dis %t3.1.4.opt.bc -o - | FileCheck %s --check-prefix=CHECK-IR
+
+; New PM, Index based WPD
+; RUN: llvm-lto2 run %t2.o -save-temps -use-new-pm -pass-remarks=. \
+; RUN:   -o %t3 \
+; RUN:   -r=%t2.o,test,plx \
+; RUN:   -r=%t2.o,_ZN1D1mEi,
+; RUN: llvm-dis %t3.1.4.opt.bc -o - | FileCheck %s --check-prefix=CHECK-IR
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-grtev4-linux-gnu"
+
+%struct.D = type { i32 (...)** }
+
+@_ZTV1D = internal constant { [3 x i8*] } { [3 x i8*] [i8* null, i8* undef, i8* bitcast (i32 (%struct.D*, i32)* @_ZN1D1mEi to i8*)] }, !type !3
+
+; CHECK-IR-LABEL: define weak_odr dso_local i32 @test
+define weak_odr i32 @test(%struct.D* %obj2, i32 %a) {
+entry:
+  %0 = bitcast %struct.D* %obj2 to i8***
+  %vtable2 = load i8**, i8*** %0
+  %1 = bitcast i8** %vtable2 to i8*
+  %p2 = call i1 @llvm.type.test(i8* %1, metadata !4)
+  call void @llvm.assume(i1 %p2)
+
+  %2 = bitcast i8** %vtable2 to i32 (%struct.D*, i32)**
+  %fptr33 = load i32 (%struct.D*, i32)*, i32 (%struct.D*, i32)** %2, align 8
+
+  ; Check that the call was not devirtualized.
+  ; CHECK-IR: %call4 = tail call i32 %fptr33
+  %call4 = tail call i32 %fptr33(%struct.D* nonnull %obj2, i32 0)
+  ret i32 %call4
+}
+; CHECK-IR-LABEL: ret i32
+; CHECK-IR-LABEL: }
+
+declare i1 @llvm.type.test(i8*, metadata)
+declare void @llvm.assume(i1)
+
+declare i32 @_ZN1D1mEi(%struct.D* %this, i32 %a)
+
+!3 = !{i64 16, !4}
+!4 = distinct !{}
diff --git a/llvm/test/ThinLTO/X86/noinline.ll b/llvm/test/ThinLTO/X86/noinline.ll
index 27f59ab90..74461853e 100644
--- a/llvm/test/ThinLTO/X86/noinline.ll
+++ b/llvm/test/ThinLTO/X86/noinline.ll
@@ -12,7 +12,7 @@
 
 ; CHECK-NOT: define available_externally i32 @foo
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-pc-linux-gnu"
 
 ; Function Attrs: nounwind ssp uwtable
diff --git a/llvm/test/ThinLTO/X86/not-internalized.ll b/llvm/test/ThinLTO/X86/not-internalized.ll
new file mode 100644
index 000000000..cf3ae058f
--- /dev/null
+++ b/llvm/test/ThinLTO/X86/not-internalized.ll
@@ -0,0 +1,34 @@
+; ModuleID = 'test.c'
+source_filename = "test.c"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; RUN: opt -module-summary %s -o %t.bc
+; RUN: llvm-lto2 run -save-temps %t.bc -o %t.out \
+; RUN:    -r=%t.bc,foo,plx \
+; RUN:    -r=%t.bc,bar,lx
+
+; Check that we don't internalize `bar` during promotion,
+; because foo and bar are members of the same comdat
+; RUN: llvm-dis %t.out.1.1.promote.bc -o - | FileCheck %s
+
+; Thin LTO internalization shouldn't internalize `bar` as well
+; RUN: llvm-dis %t.out.1.2.internalize.bc -o - | FileCheck %s
+
+; CHECK: define linkonce_odr dso_local i32 @bar() comdat($foo)
+
+$foo = comdat any
+
+; Function Attrs: noinline nounwind optnone uwtable
+define linkonce_odr dso_local i32 @bar() comdat($foo) {
+entry:
+  ret i32 33
+}
+
+; Function Attrs: noinline nounwind optnone uwtable
+define linkonce_odr dso_local i32 @foo() comdat {
+entry:
+  %call = call i32 @bar()
+  %add = add nsw i32 42, %call
+  ret i32 %add
+}
diff --git a/llvm/test/ThinLTO/X86/personality-local.ll b/llvm/test/ThinLTO/X86/personality-local.ll
index 8aa75efaa..af88d2922 100644
--- a/llvm/test/ThinLTO/X86/personality-local.ll
+++ b/llvm/test/ThinLTO/X86/personality-local.ll
@@ -18,7 +18,7 @@
 ; CHECK-NEXT:   Section: .text
 ; CHECK-NEXT: }
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-pc-linux-gnu"
 
 declare void @foo()
diff --git a/llvm/test/ThinLTO/X86/personality.ll b/llvm/test/ThinLTO/X86/personality.ll
index edf91057c..bf773a8ae 100644
--- a/llvm/test/ThinLTO/X86/personality.ll
+++ b/llvm/test/ThinLTO/X86/personality.ll
@@ -42,7 +42,7 @@
 
 ; BINDING-NOT:  Name: personality_routine3
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-pc-linux-gnu"
 
 declare protected i32 @personality_routine(i32, i32, i64, i8*, i8*)
diff --git a/llvm/test/ThinLTO/X86/pr35472.ll b/llvm/test/ThinLTO/X86/pr35472.ll
index 96a95aa46..324cb735e 100644
--- a/llvm/test/ThinLTO/X86/pr35472.ll
+++ b/llvm/test/ThinLTO/X86/pr35472.ll
@@ -13,7 +13,7 @@
 ; ThinLTOa-DAG: W _ZN4EchoD2Ev
 ; ThinLTOb-DAG: T _Z5Alphav
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 %struct.Delta = type { %struct.Charlie }
diff --git a/llvm/test/ThinLTO/X86/printer.ll b/llvm/test/ThinLTO/X86/printer.ll
new file mode 100644
index 000000000..79f4f5645
--- /dev/null
+++ b/llvm/test/ThinLTO/X86/printer.ll
@@ -0,0 +1,12 @@
+; RUN: llvm-as %s -o %t.bc
+; RUN: llvm-lto2 run -r=%t.bc,foo,pxl -use-new-pm --print-before-all %t.bc -o %t2 2>&1 | FileCheck %s --check-prefix=CHECK-BEFORE
+; RUN: llvm-lto2 run -r=%t.bc,foo,pxl -use-new-pm --print-after-all %t.bc -o %t3 2>&1 | FileCheck %s --check-prefix=CHECK-AFTER
+; CHECK-BEFORE: *** IR Dump Before GlobalDCEPass ***
+; CHECK-AFTER: *** IR Dump After GlobalDCEPass ***
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define i32 @foo() {
+    ret i32 42
+}
diff --git a/llvm/test/ThinLTO/X86/reference_non_importable.ll b/llvm/test/ThinLTO/X86/reference_non_importable.ll
index 51e6f38c3..a488b2432 100644
--- a/llvm/test/ThinLTO/X86/reference_non_importable.ll
+++ b/llvm/test/ThinLTO/X86/reference_non_importable.ll
@@ -10,7 +10,7 @@
 
 
 
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.11.0"
 
 ; We shouldn't promote the private because it has a section
diff --git a/llvm/test/ThinLTO/X86/referenced_by_constant.ll b/llvm/test/ThinLTO/X86/referenced_by_constant.ll
index ac72edd59..b352cbb2e 100644
--- a/llvm/test/ThinLTO/X86/referenced_by_constant.ll
+++ b/llvm/test/ThinLTO/X86/referenced_by_constant.ll
@@ -21,7 +21,7 @@
 ; EXPORT: define void @referencedbyglobal()
 ; EXPORT: define internal void @localreferencedbyglobal()
 
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.11.0"
 
 declare void @bar()
diff --git a/llvm/test/ThinLTO/X86/save_objects.ll b/llvm/test/ThinLTO/X86/save_objects.ll
index b9a9c983f..11b7e9da9 100644
--- a/llvm/test/ThinLTO/X86/save_objects.ll
+++ b/llvm/test/ThinLTO/X86/save_objects.ll
@@ -24,7 +24,7 @@
 ; RUN: ls %t.thin.out | grep x86_64.thinlto.o | count 2
 
 
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.11.0"
 
 define void @globalfunc() #0 {
diff --git a/llvm/test/ThinLTO/X86/strong_non_prevailing.ll b/llvm/test/ThinLTO/X86/strong_non_prevailing.ll
index 214af649f..4fd72cf3f 100644
--- a/llvm/test/ThinLTO/X86/strong_non_prevailing.ll
+++ b/llvm/test/ThinLTO/X86/strong_non_prevailing.ll
@@ -5,7 +5,7 @@
 ; RUN: llvm-nm -o - < %t.bc.thinlto.o | FileCheck %s --check-prefix=EXPORTED
 ; RUN: llvm-nm -o - < %t2.bc.thinlto.o 2>&1 | FileCheck %s --check-prefix=NOT_EXPORTED
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 $__llvm_profile_filename = comdat any
diff --git a/llvm/test/ThinLTO/X86/thinlto-internalize-doublepromoted.ll b/llvm/test/ThinLTO/X86/thinlto-internalize-doublepromoted.ll
new file mode 100644
index 000000000..fbdedda97
--- /dev/null
+++ b/llvm/test/ThinLTO/X86/thinlto-internalize-doublepromoted.ll
@@ -0,0 +1,33 @@
+; Test to ensure that we can internalize values produced from two rounds
+; of ThinLTO promotion, so they end up with two ".llvm.${hash}" suffixes.
+; Only the second should be stripped when consulting the index to locate the
+; summary.
+;
+; Note that this cannot happen currently via clang, but in other use cases such
+; as the Rust compiler which does a first round of ThinLTO on library code,
+; producing bitcode, and a second round on the final binary.
+;
+; In this case we assume a prior round of ThinLTO has promoted @foo, and
+; subsequent optimization created an internal switch table expansion variable
+; that is internal and contains the promoted name of the enclosing function.
+; This variable will be promoted in the second round of ThinLTO if @foo is
+; imported again.
+
+; RUN: opt -module-summary -o %t.bc %s
+; RUN: opt -module-summary -o %t-main.bc %S/Inputs/thinlto-internalize-doublepromoted.ll
+; RUN: llvm-lto -thinlto-action=thinlink %t.bc %t-main.bc -o %t-index.bc
+; RUN: llvm-lto -thinlto-action=internalize -exported-symbol=main -thinlto-index %t-index.bc %t.bc -o %t.internalize.bc
+; RUN: llvm-dis %t.internalize.bc -o - | FileCheck %s
+
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.15.0"
+
+; CHECK: @switch.table.foo.llvm.123.llvm.0 = hidden unnamed_addr constant
+@switch.table.foo.llvm.123 = private unnamed_addr constant [10 x i8] c"\00\01\02\03\00\00\00\00\00\09", align 1
+
+; CHECK: define hidden void @foo.llvm.123()
+define hidden void @foo.llvm.123() {
+  %1 = getelementptr inbounds [10 x i8], [10 x i8]* @switch.table.foo.llvm.123, i64 0, i64 0
+  store i8 1, i8* %1, align 8
+  ret void
+}
diff --git a/llvm/test/ThinLTO/X86/thinlto-internalize-used.ll b/llvm/test/ThinLTO/X86/thinlto-internalize-used.ll
index 065f64ec0..e0a2eb7b3 100644
--- a/llvm/test/ThinLTO/X86/thinlto-internalize-used.ll
+++ b/llvm/test/ThinLTO/X86/thinlto-internalize-used.ll
@@ -4,7 +4,7 @@
 ; RUN: llvm-lto -thinlto-action=internalize -thinlto-index %t-index.bc %t.bc -o %t.promote.bc
 ; RUN: llvm-dis %t.promote.bc -o - | FileCheck %s
 
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.15.0"
 
 @llvm.used = appending global [1 x i8*] [i8* bitcast (i32 ()* @foo to i8*)], section "llvm.metadata"
diff --git a/llvm/test/ThinLTO/X86/tli-nobuiltin.ll b/llvm/test/ThinLTO/X86/tli-nobuiltin.ll
index 9a480cba1..bf9cf907a 100644
--- a/llvm/test/ThinLTO/X86/tli-nobuiltin.ll
+++ b/llvm/test/ThinLTO/X86/tli-nobuiltin.ll
@@ -25,7 +25,7 @@
 ; ThinLTO-FREESTANDING: fprintf
 
 
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.11.0"
 
 declare i32 @fprintf(%FILE*, i8*, ...)
diff --git a/llvm/test/ThinLTO/X86/weak_externals.ll b/llvm/test/ThinLTO/X86/weak_externals.ll
index a07b431e9..83e0e0ecd 100644
--- a/llvm/test/ThinLTO/X86/weak_externals.ll
+++ b/llvm/test/ThinLTO/X86/weak_externals.ll
@@ -7,12 +7,14 @@
 ; RUN:               -r=%t.bc,_ZZN9SingletonI1SE11getInstanceEvE8instance,lx \
 ; RUN:               -r=%t.bc,_ZZN9SingletonI1SE11getInstanceEvE13instance_weak,lx
 ; RUN: llvm-dis %t.out.1.1.promote.bc -o - | FileCheck %s
+; RUN: llvm-dis %t.out.1.2.internalize.bc -o - | FileCheck %s --check-prefix=INTERNALIZE
 
 ; CHECK: @_ZZN9SingletonI1SE11getInstanceEvE8instance = available_externally dso_local global %struct.S zeroinitializer
 ; CHECK: @_ZZN9SingletonI1SE11getInstanceEvE13instance_weak = available_externally dso_local global %struct.S* null, align 8
-; CHECK: define internal dereferenceable(16) %struct.S* @_ZN9SingletonI1SE11getInstanceEv() comdat
+; CHECK: define linkonce_odr dso_local dereferenceable(16) %struct.S* @_ZN9SingletonI1SE11getInstanceEv() comdat
+; INTERNALIZE: define internal dereferenceable(16) %struct.S* @_ZN9SingletonI1SE11getInstanceEv()
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 %struct.S = type { i64, i64 }
diff --git a/llvm/test/ThinLTO/X86/weak_resolution.ll b/llvm/test/ThinLTO/X86/weak_resolution.ll
index e39e42285..b9f10afd6 100644
--- a/llvm/test/ThinLTO/X86/weak_resolution.ll
+++ b/llvm/test/ThinLTO/X86/weak_resolution.ll
@@ -12,7 +12,7 @@
 ; When exported, we always preserve a linkonce
 ; RUN: llvm-lto -thinlto-action=promote %t.bc -thinlto-index=%t3.bc -o - --exported-symbol=linkonceodrfuncInSingleModule | llvm-dis -o - | FileCheck %s --check-prefix=EXPORTED
 
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.11.0"
 
 ; Alias are resolved, but can't be turned into "available_externally"
diff --git a/llvm/test/ThinLTO/X86/writeonly-with-refs.ll b/llvm/test/ThinLTO/X86/writeonly-with-refs.ll
new file mode 100644
index 000000000..787d97758
--- /dev/null
+++ b/llvm/test/ThinLTO/X86/writeonly-with-refs.ll
@@ -0,0 +1,38 @@
+; RUN: opt -thinlto-bc %s -o %t1
+; RUN: opt -thinlto-bc %p/Inputs/writeonly-with-refs.ll -o %t2
+; RUN: llvm-lto2 run -save-temps %t1 %t2 -o %t-out \
+; RUN:    -r=%t1,main,plx \
+; RUN:    -r=%t1,_Z3foov,l \
+; RUN:    -r=%t2,_Z3foov,pl \
+; RUN:    -r=%t2,outer,pl
+
+; @outer should have been internalized and converted to zeroinitilizer.
+; RUN: llvm-dis %t-out.1.3.import.bc -o - | FileCheck %s
+; RUN: llvm-dis %t-out.2.3.import.bc -o - | FileCheck %s
+
+; CHECK: @outer = internal local_unnamed_addr global %struct.Q zeroinitializer
+
+; Test again in distributed ThinLTO mode.
+; RUN: llvm-lto2 run -save-temps %t1 %t2 -o %t-out \
+; RUN:    -thinlto-distributed-indexes \
+; RUN:    -r=%t1,main,plx \
+; RUN:    -r=%t1,_Z3foov,l \
+; RUN:    -r=%t2,_Z3foov,pl \
+; RUN:    -r=%t2,outer,pl
+; RUN: opt -function-import -import-all-index -enable-import-metadata -summary-file %t1.thinlto.bc %t1 -o %t1.out
+; RUN: opt -function-import -import-all-index -summary-file %t2.thinlto.bc %t2 -o %t2.out
+; RUN: llvm-dis %t1.out -o - | FileCheck %s
+; RUN: llvm-dis %t2.out -o - | FileCheck %s
+
+source_filename = "main.cpp"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Function Attrs: norecurse uwtable
+define dso_local i32 @main() local_unnamed_addr {
+entry:
+  tail call void @_Z3foov()
+  ret i32 0
+}
+
+declare dso_local void @_Z3foov() local_unnamed_addr
diff --git a/llvm/test/ThinLTO/X86/writeonly.ll b/llvm/test/ThinLTO/X86/writeonly.ll
index 7c2af6d9c..7616f192f 100644
--- a/llvm/test/ThinLTO/X86/writeonly.ll
+++ b/llvm/test/ThinLTO/X86/writeonly.ll
@@ -11,8 +11,8 @@
 ; RUN: llvm-dis %t1.imported.bc -o - | FileCheck %s --check-prefix=IMPORT
 ; RUN: llvm-lto -thinlto-action=optimize %t1.imported.bc -o - | llvm-dis - -o - | FileCheck %s --check-prefix=OPTIMIZE
 
-; IMPORT: @gFoo.llvm.0 = internal unnamed_addr global i32 1, align 4, !dbg !0
-; IMPORT-NEXT: @gBar = internal local_unnamed_addr global i32 2, align 4, !dbg !5
+; IMPORT: @gFoo.llvm.0 = internal unnamed_addr global i32 0, align 4, !dbg !0
+; IMPORT-NEXT: @gBar = internal local_unnamed_addr global i32 0, align 4, !dbg !5
 ; IMPORT: !DICompileUnit({{.*}})
 
 ; STATS:  2 module-summary-index - Number of live global variables marked write only 
@@ -25,7 +25,14 @@
 ; OPTIMIZE-NEXT:   %2 = tail call i32 @rand()
 ; OPTIMIZE-NEXT:   ret i32 0
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+; Confirm that with -propagate-attrs=false we no longer do write-only importing
+; RUN: llvm-lto -propagate-attrs=false -thinlto-action=import -exported-symbol=main  %t1.bc -thinlto-index=%t3.index.bc -o %t1.imported.bc -stats 2>&1 | FileCheck %s --check-prefix=STATS-NOPROP
+; RUN: llvm-dis %t1.imported.bc -o - | FileCheck %s --check-prefix=IMPORT-NOPROP
+; STATS-NOPROP-NOT: Number of live global variables marked write only
+; IMPORT-NOPROP: @gFoo.llvm.0 = available_externally
+; IMPORT-NOPROP-NEXT: @gBar = available_externally
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-pc-linux-gnu"
 
 @gBar = external global i32
diff --git a/llvm/test/ThinLTO/X86/writeonly2.ll b/llvm/test/ThinLTO/X86/writeonly2.ll
index b1e53ba47..2648727f0 100644
--- a/llvm/test/ThinLTO/X86/writeonly2.ll
+++ b/llvm/test/ThinLTO/X86/writeonly2.ll
@@ -19,8 +19,8 @@
 ; with corresponsing stores
 ; RUN: llvm-dis %t3.2.5.precodegen.bc -o - | FileCheck %s --check-prefix=CODEGEN-SRC
 
-; IMPORT:       @gFoo.llvm.0 = internal unnamed_addr global i32 1, align 4
-; IMPORT-NEXT:  @gBar = internal local_unnamed_addr global i32 2, align 4
+; IMPORT:       @gFoo.llvm.0 = internal unnamed_addr global i32 0, align 4
+; IMPORT-NEXT:  @gBar = internal local_unnamed_addr global i32 0, align 4
 ; IMPORT:       !DICompileUnit({{.*}})
 
 ; CODEGEN-NOT:  gFoo
@@ -37,7 +37,7 @@
 ; CODEGEN-SRC-NEXT:    %2 = tail call i32 @rand()
 ; CODEGEN-SRC-NEXT:    ret void
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-pc-linux-gnu"
 
 ; We should be able to link external definition of gBar to its declaration
diff --git a/llvm/test/Transforms/ADCE/2016-09-06.ll b/llvm/test/Transforms/ADCE/2016-09-06.ll
index 6a2d396ec..9c69784f6 100644
--- a/llvm/test/Transforms/ADCE/2016-09-06.ll
+++ b/llvm/test/Transforms/ADCE/2016-09-06.ll
@@ -48,7 +48,7 @@ B21:
   ret i32 %I22
 }
 
-attributes #0 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.ident = !{!0}
 
diff --git a/llvm/test/Transforms/ADCE/blocks-with-dead-term-nondeterministic.ll b/llvm/test/Transforms/ADCE/blocks-with-dead-term-nondeterministic.ll
index dd8a70e49..48cdccaab 100644
--- a/llvm/test/Transforms/ADCE/blocks-with-dead-term-nondeterministic.ll
+++ b/llvm/test/Transforms/ADCE/blocks-with-dead-term-nondeterministic.ll
@@ -64,7 +64,7 @@ bb22:                                             ; preds = %bb21, %bb
   ret void
 }
 
-attributes #0 = { noinline nounwind ssp uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="core2" "target-features"="+cx16,+fxsr,+mmx,+sse,+sse2,+sse3,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noinline nounwind ssp uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="core2" "target-features"="+cx16,+fxsr,+mmx,+sse,+sse2,+sse3,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.module.flags = !{!0}
 
diff --git a/llvm/test/Transforms/AddDiscriminators/basic.ll b/llvm/test/Transforms/AddDiscriminators/basic.ll
index da0d22aa9..3b443b3c5 100644
--- a/llvm/test/Transforms/AddDiscriminators/basic.ll
+++ b/llvm/test/Transforms/AddDiscriminators/basic.ll
@@ -36,7 +36,7 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK:   ret void, !dbg ![[END:[0-9]+]]
 }
 
-attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!7, !8}
diff --git a/llvm/test/Transforms/AddDiscriminators/call-nested.ll b/llvm/test/Transforms/AddDiscriminators/call-nested.ll
index d96ff31c6..905b3c731 100644
--- a/llvm/test/Transforms/AddDiscriminators/call-nested.ll
+++ b/llvm/test/Transforms/AddDiscriminators/call-nested.ll
@@ -24,8 +24,8 @@ declare i32 @_Z3fooii(i32, i32) #1
 
 declare i32 @_Z3barv() #1
 
-attributes #0 = { uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!8, !9}
diff --git a/llvm/test/Transforms/AddDiscriminators/call.ll b/llvm/test/Transforms/AddDiscriminators/call.ll
index d20645d0e..f1d70cb8e 100644
--- a/llvm/test/Transforms/AddDiscriminators/call.ll
+++ b/llvm/test/Transforms/AddDiscriminators/call.ll
@@ -27,8 +27,8 @@ declare void @_Z3barv() #1
 declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture) nounwind argmemonly
 declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture) nounwind argmemonly
 
-attributes #0 = { uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!7, !8}
diff --git a/llvm/test/Transforms/AddDiscriminators/diamond.ll b/llvm/test/Transforms/AddDiscriminators/diamond.ll
index a2f80d234..bbada5f28 100644
--- a/llvm/test/Transforms/AddDiscriminators/diamond.ll
+++ b/llvm/test/Transforms/AddDiscriminators/diamond.ll
@@ -39,9 +39,9 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
 
 declare void @_Z3bari(i32) #2
 
-attributes #0 = { uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
-attributes #2 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!8, !9}
diff --git a/llvm/test/Transforms/AddDiscriminators/first-only.ll b/llvm/test/Transforms/AddDiscriminators/first-only.ll
index 0417a61bb..d7915d87f 100644
--- a/llvm/test/Transforms/AddDiscriminators/first-only.ll
+++ b/llvm/test/Transforms/AddDiscriminators/first-only.ll
@@ -45,7 +45,7 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK:  ret void, !dbg ![[END:[0-9]+]]
 }
 
-attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!7, !8}
diff --git a/llvm/test/Transforms/AddDiscriminators/invoke.ll b/llvm/test/Transforms/AddDiscriminators/invoke.ll
index 0a2f86926..06b7c230f 100644
--- a/llvm/test/Transforms/AddDiscriminators/invoke.ll
+++ b/llvm/test/Transforms/AddDiscriminators/invoke.ll
@@ -91,9 +91,9 @@ define linkonce_odr hidden void @__clang_call_terminate(i8*) #3 {
 
 declare void @_ZSt9terminatev()
 
-attributes #0 = { ssp uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+fxsr,+mmx,+sahf,+sse,+sse2,+sse3,+sse4.1,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+fxsr,+mmx,+sahf,+sse,+sse2,+sse3,+sse4.1,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #2 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+fxsr,+mmx,+sahf,+sse,+sse2,+sse3,+sse4.1,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { ssp uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+fxsr,+mmx,+sahf,+sse,+sse2,+sse3,+sse4.1,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+fxsr,+mmx,+sahf,+sse,+sse2,+sse3,+sse4.1,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+fxsr,+mmx,+sahf,+sse,+sse2,+sse3,+sse4.1,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #3 = { noinline noreturn nounwind }
 attributes #4 = { nounwind }
 attributes #5 = { noreturn }
diff --git a/llvm/test/Transforms/AddDiscriminators/multiple.ll b/llvm/test/Transforms/AddDiscriminators/multiple.ll
index ea75ee189..6b1db7c6f 100644
--- a/llvm/test/Transforms/AddDiscriminators/multiple.ll
+++ b/llvm/test/Transforms/AddDiscriminators/multiple.ll
@@ -46,7 +46,7 @@ if.end:                                           ; preds = %if.else, %if.then
   ret void, !dbg !12
 }
 
-attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!7, !8}
diff --git a/llvm/test/Transforms/AddDiscriminators/no-discriminators.ll b/llvm/test/Transforms/AddDiscriminators/no-discriminators.ll
index 7248ad05f..1bd0d09e9 100644
--- a/llvm/test/Transforms/AddDiscriminators/no-discriminators.ll
+++ b/llvm/test/Transforms/AddDiscriminators/no-discriminators.ll
@@ -42,7 +42,7 @@ return:                                           ; preds = %if.else, %if.then
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
 
-attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 ; We should be able to add discriminators even in the absence of llvm.dbg.cu.
diff --git a/llvm/test/Transforms/AddDiscriminators/oneline.ll b/llvm/test/Transforms/AddDiscriminators/oneline.ll
index 5eeeb9ba0..b32b0b74c 100644
--- a/llvm/test/Transforms/AddDiscriminators/oneline.ll
+++ b/llvm/test/Transforms/AddDiscriminators/oneline.ll
@@ -52,7 +52,7 @@ define i32 @_Z3fooi(i32 %i) #0 !dbg !4 {
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
 
-attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/Transforms/AggressiveInstCombine/popcount.ll b/llvm/test/Transforms/AggressiveInstCombine/popcount.ll
new file mode 100644
index 000000000..2b4c03cd8
--- /dev/null
+++ b/llvm/test/Transforms/AggressiveInstCombine/popcount.ll
@@ -0,0 +1,193 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -aggressive-instcombine -S | FileCheck %s
+
+;int popcount8(unsigned char i) {
+;  i = i - ((i >> 1) & 0x55);
+;  i = (i & 0x33) + ((i >> 2) & 0x33);
+;  i = ((i + (i >> 4)) & 0x0F);
+; return (i * 0x01010101);
+;}
+define signext i32 @popcount8(i8 zeroext %0) {
+; CHECK-LABEL: @popcount8(
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr i8 [[TMP0:%.*]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = and i8 [[TMP2]], 85
+; CHECK-NEXT:    [[TMP4:%.*]] = sub i8 [[TMP0]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = and i8 [[TMP4]], 51
+; CHECK-NEXT:    [[TMP6:%.*]] = lshr i8 [[TMP4]], 2
+; CHECK-NEXT:    [[TMP7:%.*]] = and i8 [[TMP6]], 51
+; CHECK-NEXT:    [[TMP8:%.*]] = add nuw nsw i8 [[TMP7]], [[TMP5]]
+; CHECK-NEXT:    [[TMP9:%.*]] = lshr i8 [[TMP8]], 4
+; CHECK-NEXT:    [[TMP10:%.*]] = add nuw nsw i8 [[TMP9]], [[TMP8]]
+; CHECK-NEXT:    [[TMP11:%.*]] = and i8 [[TMP10]], 15
+; CHECK-NEXT:    [[TMP12:%.*]] = zext i8 [[TMP11]] to i32 
+; CHECK-NEXT:    ret i32 [[TMP12]]
+;
+  %2 = lshr i8 %0, 1
+  %3 = and i8 %2, 85
+  %4 = sub i8 %0, %3
+  %5 = and i8 %4, 51
+  %6 = lshr i8 %4, 2
+  %7 = and i8 %6, 51
+  %8 = add nuw nsw i8 %7, %5
+  %9 = lshr i8 %8, 4
+  %10 = add nuw nsw i8 %9, %8
+  %11 = and i8 %10, 15
+  %12 = zext i8 %11 to i32 
+  ret i32 %12 
+}
+
+;int popcount32(unsigned i) {
+;  i = i - ((i >> 1) & 0x55555555);
+;  i = (i & 0x33333333) + ((i >> 2) & 0x33333333);
+;  i = ((i + (i >> 4)) & 0x0F0F0F0F);
+; return (i * 0x01010101) >> 24;
+;}
+define signext i32 @popcount32(i32 zeroext %0) {
+; CHECK-LABEL: @popcount32(
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.ctpop.i32(i32 [[TMP0:%.*]])
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
+  %2 = lshr i32 %0, 1
+  %3 = and i32 %2, 1431655765
+  %4 = sub i32 %0, %3
+  %5 = and i32 %4, 858993459
+  %6 = lshr i32 %4, 2
+  %7 = and i32 %6, 858993459
+  %8 = add nuw nsw i32 %7, %5
+  %9 = lshr i32 %8, 4
+  %10 = add nuw nsw i32 %9, %8
+  %11 = and i32 %10, 252645135
+  %12 = mul i32 %11, 16843009
+  %13 = lshr i32 %12, 24
+  ret i32 %13
+}
+
+;int popcount64(unsigned long long i) {
+;  i = i - ((i >> 1) & 0x5555555555555555);
+;  i = (i & 0x3333333333333333) + ((i >> 2) & 0x3333333333333333);
+;  i = ((i + (i >> 4)) & 0x0F0F0F0F0F0F0F0F);
+; return (i * 0x0101010101010101) >> 56;
+;}
+define signext i32 @popcount64(i64 %0) {
+; CHECK-LABEL: @popcount64(
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0:%.*]])
+; CHECK-NEXT:    [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %2 = lshr i64 %0, 1
+  %3 = and i64 %2, 6148914691236517205
+  %4 = sub i64 %0, %3
+  %5 = and i64 %4, 3689348814741910323
+  %6 = lshr i64 %4, 2
+  %7 = and i64 %6, 3689348814741910323
+  %8 = add nuw nsw i64 %7, %5
+  %9 = lshr i64 %8, 4
+  %10 = add nuw nsw i64 %9, %8
+  %11 = and i64 %10, 1085102592571150095
+  %12 = mul i64 %11, 72340172838076673
+  %13 = lshr i64 %12, 56
+  %14 = trunc i64 %13 to i32
+  ret i32 %14
+}
+
+;int popcount128(__uint128_t i) {
+;  __uint128_t x = 0x5555555555555555;
+;  x <<= 64;
+;  x |= 0x5555555555555555;
+;  __uint128_t y = 0x3333333333333333;
+;  y <<= 64;
+;  y |= 0x3333333333333333;
+;  __uint128_t z = 0x0f0f0f0f0f0f0f0f;
+;  z <<= 64; 
+;  z |= 0x0f0f0f0f0f0f0f0f;
+;  __uint128_t a = 0x0101010101010101;
+;  a <<= 64;
+;  a |= 0x0101010101010101;
+;  unsigned mask = 120;
+;  i = i - ((i >> 1) & x);
+;  i = (i & y) + ((i >> 2) & y);
+;  i = ((i + (i >> 4)) & z);
+;  return (i * a) >> mask;
+;}
+define signext i32 @popcount128(i128 %0) {
+; CHECK-LABEL: @popcount128(
+; CHECK-NEXT:    [[TMP2:%.*]] = call i128 @llvm.ctpop.i128(i128 [[TMP0:%.*]])
+; CHECK-NEXT:    [[TMP3:%.*]] = trunc i128 [[TMP2]] to i32 
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %2 = lshr i128 %0, 1
+  %3 = and i128 %2, 113427455640312821154458202477256070485
+  %4 = sub i128 %0, %3
+  %5 = and i128 %4, 68056473384187692692674921486353642291
+  %6 = lshr i128 %4, 2
+  %7 = and i128 %6, 68056473384187692692674921486353642291
+  %8 = add nuw nsw i128 %7, %5
+  %9 = lshr i128 %8, 4
+  %10 = add nuw nsw i128 %9, %8
+  %11 = and i128 %10, 20016609818878733144904388672456953615
+  %12 = mul i128 %11, 1334440654591915542993625911497130241
+  %13 = lshr i128 %12, 120 
+  %14 = trunc i128 %13 to i32 
+  ret i32 %14 
+}
+
+;vector unsigned char popcount8vec(vector unsigned char i)
+;{
+;  i = i - ((i>> 1) & 0x55);
+;  i = (i & 0x33) + ((i >> 2) & 0x33);
+;  i = ((i + (i >> 4)) & 0x0F);
+;  return (i * 0x01);
+;}
+define <16 x i8> @popcount8vec(<16 x i8> %0) {
+; CHECK-LABEL: @popcount8vec(
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr <16 x i8> [[TMP0:%.*]], <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
+; CHECK-NEXT:    [[TMP3:%.*]] = and <16 x i8> [[TMP2]], <i8 85, i8 85, i8 85, i8 85, i8 85, i8 85, i8 85, i8 85, i8 85, i8 85, i8 85, i8 85, i8 85, i8 85, i8 85, i8 85>
+; CHECK-NEXT:    [[TMP4:%.*]] = sub <16 x i8> [[TMP0]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = and <16 x i8> [[TMP4]], <i8 51, i8 51, i8 51, i8 51, i8 51, i8 51, i8 51, i8 51, i8 51, i8 51, i8 51, i8 51, i8 51, i8 51, i8 51, i8 51>
+; CHECK-NEXT:    [[TMP6:%.*]] = lshr <16 x i8> [[TMP4]], <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>
+; CHECK-NEXT:    [[TMP7:%.*]] = and <16 x i8> [[TMP6]], <i8 51, i8 51, i8 51, i8 51, i8 51, i8 51, i8 51, i8 51, i8 51, i8 51, i8 51, i8 51, i8 51, i8 51, i8 51, i8 51>
+; CHECK-NEXT:    [[TMP8:%.*]] = add nuw nsw <16 x i8> [[TMP7]], [[TMP5]]
+; CHECK-NEXT:    [[TMP9:%.*]] = lshr <16 x i8> [[TMP8]], <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>
+; CHECK-NEXT:    [[TMP10:%.*]] = add nuw nsw <16 x i8> [[TMP9]], [[TMP8]]
+; CHECK-NEXT:    [[TMP11:%.*]] = and <16 x i8> [[TMP10]], <i8 15, i8 15, i8 15, i8 15, i8 15, i8 15, i8 15, i8 15, i8 15, i8 15, i8 15, i8 15, i8 15, i8 15, i8 15, i8 15>
+; CHECK-NEXT:    ret <16 x i8> [[TMP11]]
+;
+  %2 = lshr <16 x i8> %0, <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
+  %3 = and <16 x i8> %2, <i8 85, i8 85, i8 85, i8 85, i8 85, i8 85, i8 85, i8 85, i8 85, i8 85, i8 85, i8 85, i8 85, i8 85, i8 85, i8 85>
+  %4 = sub <16 x i8> %0, %3
+  %5 = and <16 x i8> %4, <i8 51, i8 51, i8 51, i8 51, i8 51, i8 51, i8 51, i8 51, i8 51, i8 51, i8 51, i8 51, i8 51, i8 51, i8 51, i8 51>
+  %6 = lshr <16 x i8> %4, <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>
+  %7 = and <16 x i8> %6, <i8 51, i8 51, i8 51, i8 51, i8 51, i8 51, i8 51, i8 51, i8 51, i8 51, i8 51, i8 51, i8 51, i8 51, i8 51, i8 51>
+  %8 = add nuw nsw <16 x i8> %7, %5
+  %9 = lshr <16 x i8> %8, <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>
+  %10 = add nuw nsw <16 x i8> %9, %8
+  %11 = and <16 x i8> %10, <i8 15, i8 15, i8 15, i8 15, i8 15, i8 15, i8 15, i8 15, i8 15, i8 15, i8 15, i8 15, i8 15, i8 15, i8 15, i8 15>
+  ret <16 x i8> %11
+}
+
+;vector unsigned int popcount32vec(vector unsigned int i)
+;{
+;  i = i - ((i>> 1) & 0x55555555);
+;  i = (i & 0x33333333) + ((i >> 2) & 0x33333333);
+;  i = ((i + (i >> 4)) & 0x0F0F0F0F);
+;  return (i * 0x01010101) >> 24;
+;}
+define <4 x i32> @popcount32vec(<4 x i32> %0) {
+; CHECK-LABEL: @popcount32vec(
+; CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> [[TMP0:%.*]])
+; CHECK-NEXT:    ret <4 x i32> [[TMP2]]
+;
+  %2 = lshr <4 x i32> %0, <i32 1, i32 1, i32 1, i32 1>
+  %3 = and <4 x i32> %2, <i32 1431655765, i32 1431655765, i32 1431655765, i32 1431655765>
+  %4 = sub <4 x i32> %0, %3
+  %5 = and <4 x i32> %4, <i32 858993459, i32 858993459, i32 858993459, i32 858993459>
+  %6 = lshr <4 x i32> %4, <i32 2, i32 2, i32 2, i32 2>
+  %7 = and <4 x i32> %6, <i32 858993459, i32 858993459, i32 858993459, i32 858993459>
+  %8 = add nuw nsw <4 x i32> %7, %5
+  %9 = lshr <4 x i32> %8, <i32 4, i32 4, i32 4, i32 4>
+  %10 = add nuw nsw <4 x i32> %9, %8
+  %11 = and <4 x i32> %10, <i32 252645135, i32 252645135, i32 252645135, i32 252645135>
+  %12 = mul <4 x i32> %11, <i32 16843009, i32 16843009, i32 16843009, i32 16843009>
+  %13 = lshr <4 x i32> %12, <i32 24, i32 24, i32 24, i32 24>
+  ret <4 x i32> %13
+}
diff --git a/llvm/test/Transforms/AlignmentFromAssumptions/simple.ll b/llvm/test/Transforms/AlignmentFromAssumptions/simple.ll
index 6ee08b81e..14e764f04 100644
--- a/llvm/test/Transforms/AlignmentFromAssumptions/simple.ll
+++ b/llvm/test/Transforms/AlignmentFromAssumptions/simple.ll
@@ -90,6 +90,61 @@ for.end:                                          ; preds = %for.body
 ; CHECK: ret i32 %add.lcssa
 }
 
+; test D66575
+; def hoo2(a, id, num):
+;   for i0 in range(id*64, 4096, num*64):
+;     for i1 in range(0, 4096, 32):
+;       for i2 in range(0, 4096, 32):
+;         load(a, i0+i1+i2+32)
+define void @hoo2(i32* nocapture %a, i64 %id, i64 %num) nounwind uwtable readonly {
+entry:
+  %ptrint = ptrtoint i32* %a to i64
+  %maskedptr = and i64 %ptrint, 31
+  %maskcond = icmp eq i64 %maskedptr, 0
+  tail call void @llvm.assume(i1 %maskcond)
+  %id.mul = shl nsw i64 %id, 6
+  %num.mul = shl nsw i64 %num, 6
+  br label %for0.body
+
+for0.body:
+  %i0 = phi i64 [ %id.mul, %entry ], [ %i0.next, %for0.end ]
+  br label %for1.body
+
+for1.body:
+  %i1 = phi i64 [ 0, %for0.body ], [ %i1.next, %for1.end ]
+  br label %for2.body
+
+for2.body:
+  %i2 = phi i64 [ 0, %for1.body ], [ %i2.next, %for2.body ]
+
+  %t1 = add nuw nsw i64 %i0, %i1
+  %t2 = add nuw nsw i64 %t1, %i2
+  %t3 = add nuw nsw i64 %t2, 32
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %t3
+  %x = load i32, i32* %arrayidx, align 4
+
+  %i2.next = add nuw nsw i64 %i2, 32
+  %cmp2 = icmp ult i64 %i2.next, 4096
+  br i1 %cmp2, label %for2.body, label %for1.end
+
+for1.end:
+  %i1.next = add nuw nsw i64 %i1, 32
+  %cmp1 = icmp ult i64 %i1.next, 4096
+  br i1 %cmp1, label %for1.body, label %for0.end
+
+for0.end:
+  %i0.next = add nuw nsw i64 %i0, %num.mul
+  %cmp0 = icmp ult i64 %i0.next, 4096
+  br i1 %cmp0, label %for0.body, label %return
+
+return:
+  ret void
+
+; CHECK-LABEL: @hoo2
+; CHECK: load i32, i32* %arrayidx, align 32
+; CHECK: ret void
+}
+
 define i32 @joo(i32* nocapture %a) nounwind uwtable readonly {
 entry:
   %ptrint = ptrtoint i32* %a to i64
diff --git a/llvm/test/Transforms/ArgumentPromotion/2008-02-01-ReturnAttrs.ll b/llvm/test/Transforms/ArgumentPromotion/2008-02-01-ReturnAttrs.ll
index c988774da..9b13fee1f 100644
--- a/llvm/test/Transforms/ArgumentPromotion/2008-02-01-ReturnAttrs.ll
+++ b/llvm/test/Transforms/ArgumentPromotion/2008-02-01-ReturnAttrs.ll
@@ -1,19 +1,30 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
 ; RUN: opt < %s -argpromotion -S | FileCheck %s
 
-; CHECK: define internal i32 @deref(i32 %x.val) #0 {
 define internal i32 @deref(i32* %x) nounwind {
+; CHECK-LABEL: define {{[^@]+}}@deref
+; CHECK-SAME: (i32 [[X_VAL:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret i32 [[X_VAL]]
+;
 entry:
   %tmp2 = load i32, i32* %x, align 4
   ret i32 %tmp2
 }
 
 define i32 @f(i32 %x) {
+; CHECK-LABEL: define {{[^@]+}}@f
+; CHECK-SAME: (i32 [[X:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[X_ADDR:%.*]] = alloca i32
+; CHECK-NEXT:    store i32 [[X]], i32* [[X_ADDR]], align 4
+; CHECK-NEXT:    [[X_ADDR_VAL:%.*]] = load i32, i32* [[X_ADDR]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @deref(i32 [[X_ADDR_VAL]])
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
 entry:
   %x_addr = alloca i32
   store i32 %x, i32* %x_addr, align 4
-; CHECK: %tmp1 = call i32 @deref(i32 %x_addr.val) [[NUW:#[0-9]+]]
   %tmp1 = call i32 @deref( i32* %x_addr ) nounwind
   ret i32 %tmp1
 }
-
-; CHECK: attributes [[NUW]] = { nounwind }
diff --git a/llvm/test/Transforms/ArgumentPromotion/2008-07-02-array-indexing.ll b/llvm/test/Transforms/ArgumentPromotion/2008-07-02-array-indexing.ll
index fac84d092..f94393045 100644
--- a/llvm/test/Transforms/ArgumentPromotion/2008-07-02-array-indexing.ll
+++ b/llvm/test/Transforms/ArgumentPromotion/2008-07-02-array-indexing.ll
@@ -1,11 +1,22 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
 ; RUN: opt < %s -argpromotion -S | FileCheck %s
 ; PR2498
 
-; This test tries to convince argpromotion about promoting the load from %A + 2,
+; This test tries to convince CHECK about promoting the load from %A + 2,
 ; because there is a load of %A in the entry block
 define internal i32 @callee(i1 %C, i32* %A) {
-; CHECK-LABEL: define internal i32 @callee(
-; CHECK: i1 %C, i32* %A)
+; CHECK-LABEL: define {{[^@]+}}@callee
+; CHECK-SAME: (i1 [[C:%.*]], i32* [[A:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[A_0:%.*]] = load i32, i32* [[A]]
+; CHECK-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]]
+; CHECK:       T:
+; CHECK-NEXT:    ret i32 [[A_0]]
+; CHECK:       F:
+; CHECK-NEXT:    [[A_2:%.*]] = getelementptr i32, i32* [[A]], i32 2
+; CHECK-NEXT:    [[R:%.*]] = load i32, i32* [[A_2]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
 entry:
   ; Unconditonally load the element at %A
   %A.0 = load i32, i32* %A
@@ -22,9 +33,11 @@ F:
 }
 
 define i32 @foo() {
-; CHECK-LABEL: define i32 @foo
-        %X = call i32 @callee(i1 false, i32* null)             ; <i32> [#uses=1]
-; CHECK: call i32 @callee(i1 false, i32* null)
-        ret i32 %X
+; CHECK-LABEL: define {{[^@]+}}@foo()
+; CHECK-NEXT:    [[X:%.*]] = call i32 @callee(i1 false, i32* null)
+; CHECK-NEXT:    ret i32 [[X]]
+;
+  %X = call i32 @callee(i1 false, i32* null)             ; <i32> [#uses=1]
+  ret i32 %X
 }
 
diff --git a/llvm/test/Transforms/ArgumentPromotion/2008-09-07-CGUpdate.ll b/llvm/test/Transforms/ArgumentPromotion/2008-09-07-CGUpdate.ll
index 7ee6654ea..639457ac6 100644
--- a/llvm/test/Transforms/ArgumentPromotion/2008-09-07-CGUpdate.ll
+++ b/llvm/test/Transforms/ArgumentPromotion/2008-09-07-CGUpdate.ll
@@ -1,12 +1,13 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
 ; RUN: opt < %s -inline -argpromotion -disable-output
 
 define internal fastcc i32 @hash(i32* %ts, i32 %mod) nounwind {
 entry:
-	unreachable
+  unreachable
 }
 
 define void @encode(i32* %m, i32* %ts, i32* %new) nounwind {
 entry:
-	%0 = call fastcc i32 @hash( i32* %ts, i32 0 ) nounwind		; <i32> [#uses=0]
-	unreachable
+  %0 = call fastcc i32 @hash( i32* %ts, i32 0 ) nounwind		; <i32> [#uses=0]
+  unreachable
 }
diff --git a/llvm/test/Transforms/ArgumentPromotion/2008-09-08-CGUpdateSelfEdge.ll b/llvm/test/Transforms/ArgumentPromotion/2008-09-08-CGUpdateSelfEdge.ll
index aff917c6a..4b2245eec 100644
--- a/llvm/test/Transforms/ArgumentPromotion/2008-09-08-CGUpdateSelfEdge.ll
+++ b/llvm/test/Transforms/ArgumentPromotion/2008-09-08-CGUpdateSelfEdge.ll
@@ -1,25 +1,26 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
 ; RUN: opt < %s -argpromotion -disable-output
 
 define internal fastcc i32 @term_SharingList(i32* %Term, i32* %List) nounwind {
 entry:
-	br i1 false, label %bb, label %bb5
+  br i1 false, label %bb, label %bb5
 
 bb:		; preds = %entry
-	%0 = call fastcc i32 @term_SharingList( i32* null, i32* %List ) nounwind		; <i32> [#uses=0]
-	unreachable
+  %0 = call fastcc i32 @term_SharingList( i32* null, i32* %List ) nounwind		; <i32> [#uses=0]
+  unreachable
 
 bb5:		; preds = %entry
-	ret i32 0
+  ret i32 0
 }
 
 define i32 @term_Sharing(i32* %Term) nounwind {
 entry:
-	br i1 false, label %bb.i, label %bb14
+  br i1 false, label %bb.i, label %bb14
 
 bb.i:		; preds = %entry
-	%0 = call fastcc i32 @term_SharingList( i32* null, i32* null ) nounwind		; <i32> [#uses=0]
-	ret i32 1
+  %0 = call fastcc i32 @term_SharingList( i32* null, i32* null ) nounwind		; <i32> [#uses=0]
+  ret i32 1
 
 bb14:		; preds = %entry
-	ret i32 0
+  ret i32 0
 }
diff --git a/llvm/test/Transforms/ArgumentPromotion/X86/attributes.ll b/llvm/test/Transforms/ArgumentPromotion/X86/attributes.ll
index 82a264109..c849ad807 100644
--- a/llvm/test/Transforms/ArgumentPromotion/X86/attributes.ll
+++ b/llvm/test/Transforms/ArgumentPromotion/X86/attributes.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
 ; RUN: opt -S -argpromotion < %s | FileCheck %s
 ; RUN: opt -S -passes=argpromotion < %s | FileCheck %s
 ; Test that we only promote arguments when the caller/callee have compatible
@@ -5,8 +6,14 @@
 
 target triple = "x86_64-unknown-linux-gnu"
 
-; CHECK-LABEL: @no_promote_avx2(<4 x i64>* %arg, <4 x i64>* readonly %arg1)
 define internal fastcc void @no_promote_avx2(<4 x i64>* %arg, <4 x i64>* readonly %arg1) #0 {
+; CHECK-LABEL: define {{[^@]+}}@no_promote_avx2
+; CHECK-SAME: (<4 x i64>* [[ARG:%.*]], <4 x i64>* readonly [[ARG1:%.*]])
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP:%.*]] = load <4 x i64>, <4 x i64>* [[ARG1]]
+; CHECK-NEXT:    store <4 x i64> [[TMP]], <4 x i64>* [[ARG]]
+; CHECK-NEXT:    ret void
+;
 bb:
   %tmp = load <4 x i64>, <4 x i64>* %arg1
   store <4 x i64> %tmp, <4 x i64>* %arg
@@ -14,6 +21,18 @@ bb:
 }
 
 define void @no_promote(<4 x i64>* %arg) #1 {
+; CHECK-LABEL: define {{[^@]+}}@no_promote
+; CHECK-SAME: (<4 x i64>* [[ARG:%.*]])
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP:%.*]] = alloca <4 x i64>, align 32
+; CHECK-NEXT:    [[TMP2:%.*]] = alloca <4 x i64>, align 32
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i64>* [[TMP]] to i8*
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* align 32 [[TMP3]], i8 0, i64 32, i1 false)
+; CHECK-NEXT:    call fastcc void @no_promote_avx2(<4 x i64>* [[TMP2]], <4 x i64>* [[TMP]])
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i64>, <4 x i64>* [[TMP2]], align 32
+; CHECK-NEXT:    store <4 x i64> [[TMP4]], <4 x i64>* [[ARG]], align 2
+; CHECK-NEXT:    ret void
+;
 bb:
   %tmp = alloca <4 x i64>, align 32
   %tmp2 = alloca <4 x i64>, align 32
@@ -25,8 +44,13 @@ bb:
   ret void
 }
 
-; CHECK-LABEL: @promote_avx2(<4 x i64>* %arg, <4 x i64> %
 define internal fastcc void @promote_avx2(<4 x i64>* %arg, <4 x i64>* readonly %arg1) #0 {
+; CHECK-LABEL: define {{[^@]+}}@promote_avx2
+; CHECK-SAME: (<4 x i64>* [[ARG:%.*]], <4 x i64> [[ARG1_VAL:%.*]])
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    store <4 x i64> [[ARG1_VAL]], <4 x i64>* [[ARG]]
+; CHECK-NEXT:    ret void
+;
 bb:
   %tmp = load <4 x i64>, <4 x i64>* %arg1
   store <4 x i64> %tmp, <4 x i64>* %arg
@@ -34,6 +58,19 @@ bb:
 }
 
 define void @promote(<4 x i64>* %arg) #0 {
+; CHECK-LABEL: define {{[^@]+}}@promote
+; CHECK-SAME: (<4 x i64>* [[ARG:%.*]])
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP:%.*]] = alloca <4 x i64>, align 32
+; CHECK-NEXT:    [[TMP2:%.*]] = alloca <4 x i64>, align 32
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i64>* [[TMP]] to i8*
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* align 32 [[TMP3]], i8 0, i64 32, i1 false)
+; CHECK-NEXT:    [[TMP_VAL:%.*]] = load <4 x i64>, <4 x i64>* [[TMP]]
+; CHECK-NEXT:    call fastcc void @promote_avx2(<4 x i64>* [[TMP2]], <4 x i64> [[TMP_VAL]])
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i64>, <4 x i64>* [[TMP2]], align 32
+; CHECK-NEXT:    store <4 x i64> [[TMP4]], <4 x i64>* [[ARG]], align 2
+; CHECK-NEXT:    ret void
+;
 bb:
   %tmp = alloca <4 x i64>, align 32
   %tmp2 = alloca <4 x i64>, align 32
diff --git a/llvm/test/Transforms/ArgumentPromotion/X86/min-legal-vector-width.ll b/llvm/test/Transforms/ArgumentPromotion/X86/min-legal-vector-width.ll
index 59ac7d3c1..6cca81531 100644
--- a/llvm/test/Transforms/ArgumentPromotion/X86/min-legal-vector-width.ll
+++ b/llvm/test/Transforms/ArgumentPromotion/X86/min-legal-vector-width.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
 ; RUN: opt -S -argpromotion < %s | FileCheck %s
 ; RUN: opt -S -passes=argpromotion < %s | FileCheck %s
 ; Test that we only promote arguments when the caller/callee have compatible
@@ -6,8 +7,13 @@
 target triple = "x86_64-unknown-linux-gnu"
 
 ; This should promote
-; CHECK-LABEL: @callee_avx512_legal512_prefer512_call_avx512_legal512_prefer512(<8 x i64>* %arg, <8 x i64> %arg1.val)
 define internal fastcc void @callee_avx512_legal512_prefer512_call_avx512_legal512_prefer512(<8 x i64>* %arg, <8 x i64>* readonly %arg1) #0 {
+; CHECK-LABEL: define {{[^@]+}}@callee_avx512_legal512_prefer512_call_avx512_legal512_prefer512
+; CHECK-SAME: (<8 x i64>* [[ARG:%.*]], <8 x i64> [[ARG1_VAL:%.*]])
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    store <8 x i64> [[ARG1_VAL]], <8 x i64>* [[ARG]]
+; CHECK-NEXT:    ret void
+;
 bb:
   %tmp = load <8 x i64>, <8 x i64>* %arg1
   store <8 x i64> %tmp, <8 x i64>* %arg
@@ -15,6 +21,19 @@ bb:
 }
 
 define void @avx512_legal512_prefer512_call_avx512_legal512_prefer512(<8 x i64>* %arg) #0 {
+; CHECK-LABEL: define {{[^@]+}}@avx512_legal512_prefer512_call_avx512_legal512_prefer512
+; CHECK-SAME: (<8 x i64>* [[ARG:%.*]])
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP:%.*]] = alloca <8 x i64>, align 32
+; CHECK-NEXT:    [[TMP2:%.*]] = alloca <8 x i64>, align 32
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i64>* [[TMP]] to i8*
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* align 32 [[TMP3]], i8 0, i64 32, i1 false)
+; CHECK-NEXT:    [[TMP_VAL:%.*]] = load <8 x i64>, <8 x i64>* [[TMP]]
+; CHECK-NEXT:    call fastcc void @callee_avx512_legal512_prefer512_call_avx512_legal512_prefer512(<8 x i64>* [[TMP2]], <8 x i64> [[TMP_VAL]])
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, <8 x i64>* [[TMP2]], align 32
+; CHECK-NEXT:    store <8 x i64> [[TMP4]], <8 x i64>* [[ARG]], align 2
+; CHECK-NEXT:    ret void
+;
 bb:
   %tmp = alloca <8 x i64>, align 32
   %tmp2 = alloca <8 x i64>, align 32
@@ -27,8 +46,13 @@ bb:
 }
 
 ; This should promote
-; CHECK-LABEL: @callee_avx512_legal512_prefer256_call_avx512_legal512_prefer256(<8 x i64>* %arg, <8 x i64> %arg1.val)
 define internal fastcc void @callee_avx512_legal512_prefer256_call_avx512_legal512_prefer256(<8 x i64>* %arg, <8 x i64>* readonly %arg1) #1 {
+; CHECK-LABEL: define {{[^@]+}}@callee_avx512_legal512_prefer256_call_avx512_legal512_prefer256
+; CHECK-SAME: (<8 x i64>* [[ARG:%.*]], <8 x i64> [[ARG1_VAL:%.*]])
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    store <8 x i64> [[ARG1_VAL]], <8 x i64>* [[ARG]]
+; CHECK-NEXT:    ret void
+;
 bb:
   %tmp = load <8 x i64>, <8 x i64>* %arg1
   store <8 x i64> %tmp, <8 x i64>* %arg
@@ -36,6 +60,19 @@ bb:
 }
 
 define void @avx512_legal512_prefer256_call_avx512_legal512_prefer256(<8 x i64>* %arg) #1 {
+; CHECK-LABEL: define {{[^@]+}}@avx512_legal512_prefer256_call_avx512_legal512_prefer256
+; CHECK-SAME: (<8 x i64>* [[ARG:%.*]])
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP:%.*]] = alloca <8 x i64>, align 32
+; CHECK-NEXT:    [[TMP2:%.*]] = alloca <8 x i64>, align 32
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i64>* [[TMP]] to i8*
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* align 32 [[TMP3]], i8 0, i64 32, i1 false)
+; CHECK-NEXT:    [[TMP_VAL:%.*]] = load <8 x i64>, <8 x i64>* [[TMP]]
+; CHECK-NEXT:    call fastcc void @callee_avx512_legal512_prefer256_call_avx512_legal512_prefer256(<8 x i64>* [[TMP2]], <8 x i64> [[TMP_VAL]])
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, <8 x i64>* [[TMP2]], align 32
+; CHECK-NEXT:    store <8 x i64> [[TMP4]], <8 x i64>* [[ARG]], align 2
+; CHECK-NEXT:    ret void
+;
 bb:
   %tmp = alloca <8 x i64>, align 32
   %tmp2 = alloca <8 x i64>, align 32
@@ -48,8 +85,13 @@ bb:
 }
 
 ; This should promote
-; CHECK-LABEL: @callee_avx512_legal512_prefer512_call_avx512_legal512_prefer256(<8 x i64>* %arg, <8 x i64> %arg1.val)
 define internal fastcc void @callee_avx512_legal512_prefer512_call_avx512_legal512_prefer256(<8 x i64>* %arg, <8 x i64>* readonly %arg1) #1 {
+; CHECK-LABEL: define {{[^@]+}}@callee_avx512_legal512_prefer512_call_avx512_legal512_prefer256
+; CHECK-SAME: (<8 x i64>* [[ARG:%.*]], <8 x i64> [[ARG1_VAL:%.*]])
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    store <8 x i64> [[ARG1_VAL]], <8 x i64>* [[ARG]]
+; CHECK-NEXT:    ret void
+;
 bb:
   %tmp = load <8 x i64>, <8 x i64>* %arg1
   store <8 x i64> %tmp, <8 x i64>* %arg
@@ -57,6 +99,19 @@ bb:
 }
 
 define void @avx512_legal512_prefer512_call_avx512_legal512_prefer256(<8 x i64>* %arg) #0 {
+; CHECK-LABEL: define {{[^@]+}}@avx512_legal512_prefer512_call_avx512_legal512_prefer256
+; CHECK-SAME: (<8 x i64>* [[ARG:%.*]])
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP:%.*]] = alloca <8 x i64>, align 32
+; CHECK-NEXT:    [[TMP2:%.*]] = alloca <8 x i64>, align 32
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i64>* [[TMP]] to i8*
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* align 32 [[TMP3]], i8 0, i64 32, i1 false)
+; CHECK-NEXT:    [[TMP_VAL:%.*]] = load <8 x i64>, <8 x i64>* [[TMP]]
+; CHECK-NEXT:    call fastcc void @callee_avx512_legal512_prefer512_call_avx512_legal512_prefer256(<8 x i64>* [[TMP2]], <8 x i64> [[TMP_VAL]])
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, <8 x i64>* [[TMP2]], align 32
+; CHECK-NEXT:    store <8 x i64> [[TMP4]], <8 x i64>* [[ARG]], align 2
+; CHECK-NEXT:    ret void
+;
 bb:
   %tmp = alloca <8 x i64>, align 32
   %tmp2 = alloca <8 x i64>, align 32
@@ -69,8 +124,13 @@ bb:
 }
 
 ; This should promote
-; CHECK-LABEL: @callee_avx512_legal512_prefer256_call_avx512_legal512_prefer512(<8 x i64>* %arg, <8 x i64> %arg1.val)
 define internal fastcc void @callee_avx512_legal512_prefer256_call_avx512_legal512_prefer512(<8 x i64>* %arg, <8 x i64>* readonly %arg1) #0 {
+; CHECK-LABEL: define {{[^@]+}}@callee_avx512_legal512_prefer256_call_avx512_legal512_prefer512
+; CHECK-SAME: (<8 x i64>* [[ARG:%.*]], <8 x i64> [[ARG1_VAL:%.*]])
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    store <8 x i64> [[ARG1_VAL]], <8 x i64>* [[ARG]]
+; CHECK-NEXT:    ret void
+;
 bb:
   %tmp = load <8 x i64>, <8 x i64>* %arg1
   store <8 x i64> %tmp, <8 x i64>* %arg
@@ -78,6 +138,19 @@ bb:
 }
 
 define void @avx512_legal512_prefer256_call_avx512_legal512_prefer512(<8 x i64>* %arg) #1 {
+; CHECK-LABEL: define {{[^@]+}}@avx512_legal512_prefer256_call_avx512_legal512_prefer512
+; CHECK-SAME: (<8 x i64>* [[ARG:%.*]])
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP:%.*]] = alloca <8 x i64>, align 32
+; CHECK-NEXT:    [[TMP2:%.*]] = alloca <8 x i64>, align 32
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i64>* [[TMP]] to i8*
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* align 32 [[TMP3]], i8 0, i64 32, i1 false)
+; CHECK-NEXT:    [[TMP_VAL:%.*]] = load <8 x i64>, <8 x i64>* [[TMP]]
+; CHECK-NEXT:    call fastcc void @callee_avx512_legal512_prefer256_call_avx512_legal512_prefer512(<8 x i64>* [[TMP2]], <8 x i64> [[TMP_VAL]])
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, <8 x i64>* [[TMP2]], align 32
+; CHECK-NEXT:    store <8 x i64> [[TMP4]], <8 x i64>* [[ARG]], align 2
+; CHECK-NEXT:    ret void
+;
 bb:
   %tmp = alloca <8 x i64>, align 32
   %tmp2 = alloca <8 x i64>, align 32
@@ -90,8 +163,14 @@ bb:
 }
 
 ; This should not promote
-; CHECK-LABEL: @callee_avx512_legal256_prefer256_call_avx512_legal512_prefer256(<8 x i64>* %arg, <8 x i64>* readonly %arg1)
 define internal fastcc void @callee_avx512_legal256_prefer256_call_avx512_legal512_prefer256(<8 x i64>* %arg, <8 x i64>* readonly %arg1) #1 {
+; CHECK-LABEL: define {{[^@]+}}@callee_avx512_legal256_prefer256_call_avx512_legal512_prefer256
+; CHECK-SAME: (<8 x i64>* [[ARG:%.*]], <8 x i64>* readonly [[ARG1:%.*]])
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP:%.*]] = load <8 x i64>, <8 x i64>* [[ARG1]]
+; CHECK-NEXT:    store <8 x i64> [[TMP]], <8 x i64>* [[ARG]]
+; CHECK-NEXT:    ret void
+;
 bb:
   %tmp = load <8 x i64>, <8 x i64>* %arg1
   store <8 x i64> %tmp, <8 x i64>* %arg
@@ -99,6 +178,18 @@ bb:
 }
 
 define void @avx512_legal256_prefer256_call_avx512_legal512_prefer256(<8 x i64>* %arg) #2 {
+; CHECK-LABEL: define {{[^@]+}}@avx512_legal256_prefer256_call_avx512_legal512_prefer256
+; CHECK-SAME: (<8 x i64>* [[ARG:%.*]])
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP:%.*]] = alloca <8 x i64>, align 32
+; CHECK-NEXT:    [[TMP2:%.*]] = alloca <8 x i64>, align 32
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i64>* [[TMP]] to i8*
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* align 32 [[TMP3]], i8 0, i64 32, i1 false)
+; CHECK-NEXT:    call fastcc void @callee_avx512_legal256_prefer256_call_avx512_legal512_prefer256(<8 x i64>* [[TMP2]], <8 x i64>* [[TMP]])
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, <8 x i64>* [[TMP2]], align 32
+; CHECK-NEXT:    store <8 x i64> [[TMP4]], <8 x i64>* [[ARG]], align 2
+; CHECK-NEXT:    ret void
+;
 bb:
   %tmp = alloca <8 x i64>, align 32
   %tmp2 = alloca <8 x i64>, align 32
@@ -111,8 +202,14 @@ bb:
 }
 
 ; This should not promote
-; CHECK-LABEL: @callee_avx512_legal512_prefer256_call_avx512_legal256_prefer256(<8 x i64>* %arg, <8 x i64>* readonly %arg1)
 define internal fastcc void @callee_avx512_legal512_prefer256_call_avx512_legal256_prefer256(<8 x i64>* %arg, <8 x i64>* readonly %arg1) #2 {
+; CHECK-LABEL: define {{[^@]+}}@callee_avx512_legal512_prefer256_call_avx512_legal256_prefer256
+; CHECK-SAME: (<8 x i64>* [[ARG:%.*]], <8 x i64>* readonly [[ARG1:%.*]])
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP:%.*]] = load <8 x i64>, <8 x i64>* [[ARG1]]
+; CHECK-NEXT:    store <8 x i64> [[TMP]], <8 x i64>* [[ARG]]
+; CHECK-NEXT:    ret void
+;
 bb:
   %tmp = load <8 x i64>, <8 x i64>* %arg1
   store <8 x i64> %tmp, <8 x i64>* %arg
@@ -120,6 +217,18 @@ bb:
 }
 
 define void @avx512_legal512_prefer256_call_avx512_legal256_prefer256(<8 x i64>* %arg) #1 {
+; CHECK-LABEL: define {{[^@]+}}@avx512_legal512_prefer256_call_avx512_legal256_prefer256
+; CHECK-SAME: (<8 x i64>* [[ARG:%.*]])
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP:%.*]] = alloca <8 x i64>, align 32
+; CHECK-NEXT:    [[TMP2:%.*]] = alloca <8 x i64>, align 32
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i64>* [[TMP]] to i8*
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* align 32 [[TMP3]], i8 0, i64 32, i1 false)
+; CHECK-NEXT:    call fastcc void @callee_avx512_legal512_prefer256_call_avx512_legal256_prefer256(<8 x i64>* [[TMP2]], <8 x i64>* [[TMP]])
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, <8 x i64>* [[TMP2]], align 32
+; CHECK-NEXT:    store <8 x i64> [[TMP4]], <8 x i64>* [[ARG]], align 2
+; CHECK-NEXT:    ret void
+;
 bb:
   %tmp = alloca <8 x i64>, align 32
   %tmp2 = alloca <8 x i64>, align 32
@@ -132,8 +241,13 @@ bb:
 }
 
 ; This should promote
-; CHECK-LABEL: @callee_avx2_legal256_prefer256_call_avx2_legal512_prefer256(<8 x i64>* %arg, <8 x i64> %arg1.val)
 define internal fastcc void @callee_avx2_legal256_prefer256_call_avx2_legal512_prefer256(<8 x i64>* %arg, <8 x i64>* readonly %arg1) #3 {
+; CHECK-LABEL: define {{[^@]+}}@callee_avx2_legal256_prefer256_call_avx2_legal512_prefer256
+; CHECK-SAME: (<8 x i64>* [[ARG:%.*]], <8 x i64> [[ARG1_VAL:%.*]])
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    store <8 x i64> [[ARG1_VAL]], <8 x i64>* [[ARG]]
+; CHECK-NEXT:    ret void
+;
 bb:
   %tmp = load <8 x i64>, <8 x i64>* %arg1
   store <8 x i64> %tmp, <8 x i64>* %arg
@@ -141,6 +255,19 @@ bb:
 }
 
 define void @avx2_legal256_prefer256_call_avx2_legal512_prefer256(<8 x i64>* %arg) #4 {
+; CHECK-LABEL: define {{[^@]+}}@avx2_legal256_prefer256_call_avx2_legal512_prefer256
+; CHECK-SAME: (<8 x i64>* [[ARG:%.*]])
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP:%.*]] = alloca <8 x i64>, align 32
+; CHECK-NEXT:    [[TMP2:%.*]] = alloca <8 x i64>, align 32
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i64>* [[TMP]] to i8*
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* align 32 [[TMP3]], i8 0, i64 32, i1 false)
+; CHECK-NEXT:    [[TMP_VAL:%.*]] = load <8 x i64>, <8 x i64>* [[TMP]]
+; CHECK-NEXT:    call fastcc void @callee_avx2_legal256_prefer256_call_avx2_legal512_prefer256(<8 x i64>* [[TMP2]], <8 x i64> [[TMP_VAL]])
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, <8 x i64>* [[TMP2]], align 32
+; CHECK-NEXT:    store <8 x i64> [[TMP4]], <8 x i64>* [[ARG]], align 2
+; CHECK-NEXT:    ret void
+;
 bb:
   %tmp = alloca <8 x i64>, align 32
   %tmp2 = alloca <8 x i64>, align 32
@@ -153,8 +280,13 @@ bb:
 }
 
 ; This should promote
-; CHECK-LABEL: @callee_avx2_legal512_prefer256_call_avx2_legal256_prefer256(<8 x i64>* %arg, <8 x i64> %arg1.val)
 define internal fastcc void @callee_avx2_legal512_prefer256_call_avx2_legal256_prefer256(<8 x i64>* %arg, <8 x i64>* readonly %arg1) #4 {
+; CHECK-LABEL: define {{[^@]+}}@callee_avx2_legal512_prefer256_call_avx2_legal256_prefer256
+; CHECK-SAME: (<8 x i64>* [[ARG:%.*]], <8 x i64> [[ARG1_VAL:%.*]])
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    store <8 x i64> [[ARG1_VAL]], <8 x i64>* [[ARG]]
+; CHECK-NEXT:    ret void
+;
 bb:
   %tmp = load <8 x i64>, <8 x i64>* %arg1
   store <8 x i64> %tmp, <8 x i64>* %arg
@@ -162,6 +294,19 @@ bb:
 }
 
 define void @avx2_legal512_prefer256_call_avx2_legal256_prefer256(<8 x i64>* %arg) #3 {
+; CHECK-LABEL: define {{[^@]+}}@avx2_legal512_prefer256_call_avx2_legal256_prefer256
+; CHECK-SAME: (<8 x i64>* [[ARG:%.*]])
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP:%.*]] = alloca <8 x i64>, align 32
+; CHECK-NEXT:    [[TMP2:%.*]] = alloca <8 x i64>, align 32
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i64>* [[TMP]] to i8*
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* align 32 [[TMP3]], i8 0, i64 32, i1 false)
+; CHECK-NEXT:    [[TMP_VAL:%.*]] = load <8 x i64>, <8 x i64>* [[TMP]]
+; CHECK-NEXT:    call fastcc void @callee_avx2_legal512_prefer256_call_avx2_legal256_prefer256(<8 x i64>* [[TMP2]], <8 x i64> [[TMP_VAL]])
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, <8 x i64>* [[TMP2]], align 32
+; CHECK-NEXT:    store <8 x i64> [[TMP4]], <8 x i64>* [[ARG]], align 2
+; CHECK-NEXT:    ret void
+;
 bb:
   %tmp = alloca <8 x i64>, align 32
   %tmp2 = alloca <8 x i64>, align 32
diff --git a/llvm/test/Transforms/ArgumentPromotion/X86/thiscall.ll b/llvm/test/Transforms/ArgumentPromotion/X86/thiscall.ll
index 8e7fd975d..d9f3681ba 100644
--- a/llvm/test/Transforms/ArgumentPromotion/X86/thiscall.ll
+++ b/llvm/test/Transforms/ArgumentPromotion/X86/thiscall.ll
@@ -1,12 +1,11 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
 ; In PR41658, argpromotion put an inalloca in a position that per the
 ; calling convention is passed in a register. This test verifies that
 ; we don't do that anymore. It also verifies that the combination of
 ; globalopt and argpromotion is able to optimize the call safely.
 ;
-; RUN: opt -S -argpromotion %s | FileCheck --check-prefix=THIS %s
-; RUN: opt -S -globalopt -argpromotion %s | FileCheck --check-prefix=OPT %s
-; THIS: define internal x86_thiscallcc void @internalfun(%struct.a* %this, <{ %struct.a
-; OPT: define internal fastcc void @internalfun(<{ %struct.a }>*)
+; RUN: opt -S -argpromotion %s | FileCheck %s --check-prefix=ARGPROMOTION
+; RUN: opt -S -globalopt -argpromotion %s | FileCheck %s --check-prefix=GLOBALOPT_ARGPROMOTION
 
 target datalayout = "e-m:x-p:32:32-i64:64-f80:32-n8:16:32-a:0:32-S32"
 target triple = "i386-pc-windows-msvc19.11.0"
@@ -14,6 +13,26 @@ target triple = "i386-pc-windows-msvc19.11.0"
 %struct.a = type { i8 }
 
 define internal x86_thiscallcc void @internalfun(%struct.a* %this, <{ %struct.a }>* inalloca) {
+; ARGPROMOTION-LABEL: define {{[^@]+}}@internalfun
+; ARGPROMOTION-SAME: (%struct.a* [[THIS:%.*]], <{ [[STRUCT_A:%.*]] }>* inalloca [[TMP0:%.*]])
+; ARGPROMOTION-NEXT:  entry:
+; ARGPROMOTION-NEXT:    [[A:%.*]] = getelementptr inbounds <{ [[STRUCT_A]] }>, <{ [[STRUCT_A]] }>* [[TMP0]], i32 0, i32 0
+; ARGPROMOTION-NEXT:    [[ARGMEM:%.*]] = alloca inalloca <{ [[STRUCT_A]] }>, align 4
+; ARGPROMOTION-NEXT:    [[TMP1:%.*]] = getelementptr inbounds <{ [[STRUCT_A]] }>, <{ [[STRUCT_A]] }>* [[ARGMEM]], i32 0, i32 0
+; ARGPROMOTION-NEXT:    [[CALL:%.*]] = call x86_thiscallcc %struct.a* @copy_ctor(%struct.a* [[TMP1]], %struct.a* dereferenceable(1) [[A]])
+; ARGPROMOTION-NEXT:    call void @ext(<{ [[STRUCT_A]] }>* inalloca [[ARGMEM]])
+; ARGPROMOTION-NEXT:    ret void
+;
+; GLOBALOPT_ARGPROMOTION-LABEL: define {{[^@]+}}@internalfun
+; GLOBALOPT_ARGPROMOTION-SAME: (<{ [[STRUCT_A:%.*]] }>* [[TMP0:%.*]]) unnamed_addr
+; GLOBALOPT_ARGPROMOTION-NEXT:  entry:
+; GLOBALOPT_ARGPROMOTION-NEXT:    [[A:%.*]] = getelementptr inbounds <{ [[STRUCT_A]] }>, <{ [[STRUCT_A]] }>* [[TMP0]], i32 0, i32 0
+; GLOBALOPT_ARGPROMOTION-NEXT:    [[ARGMEM:%.*]] = alloca inalloca <{ [[STRUCT_A]] }>, align 4
+; GLOBALOPT_ARGPROMOTION-NEXT:    [[TMP1:%.*]] = getelementptr inbounds <{ [[STRUCT_A]] }>, <{ [[STRUCT_A]] }>* [[ARGMEM]], i32 0, i32 0
+; GLOBALOPT_ARGPROMOTION-NEXT:    [[CALL:%.*]] = call x86_thiscallcc %struct.a* @copy_ctor(%struct.a* [[TMP1]], %struct.a* dereferenceable(1) [[A]])
+; GLOBALOPT_ARGPROMOTION-NEXT:    call void @ext(<{ [[STRUCT_A]] }>* inalloca [[ARGMEM]])
+; GLOBALOPT_ARGPROMOTION-NEXT:    ret void
+;
 entry:
   %a = getelementptr inbounds <{ %struct.a }>, <{ %struct.a }>* %0, i32 0, i32 0
   %argmem = alloca inalloca <{ %struct.a }>, align 4
@@ -25,6 +44,22 @@ entry:
 
 ; This is here to ensure @internalfun is live.
 define void @exportedfun(%struct.a* %a) {
+; ARGPROMOTION-LABEL: define {{[^@]+}}@exportedfun
+; ARGPROMOTION-SAME: (%struct.a* [[A:%.*]])
+; ARGPROMOTION-NEXT:    [[INALLOCA_SAVE:%.*]] = tail call i8* @llvm.stacksave()
+; ARGPROMOTION-NEXT:    [[ARGMEM:%.*]] = alloca inalloca <{ [[STRUCT_A:%.*]] }>, align 4
+; ARGPROMOTION-NEXT:    call x86_thiscallcc void @internalfun(%struct.a* [[A]], <{ [[STRUCT_A]] }>* inalloca [[ARGMEM]])
+; ARGPROMOTION-NEXT:    call void @llvm.stackrestore(i8* [[INALLOCA_SAVE]])
+; ARGPROMOTION-NEXT:    ret void
+;
+; GLOBALOPT_ARGPROMOTION-LABEL: define {{[^@]+}}@exportedfun
+; GLOBALOPT_ARGPROMOTION-SAME: (%struct.a* [[A:%.*]]) local_unnamed_addr
+; GLOBALOPT_ARGPROMOTION-NEXT:    [[INALLOCA_SAVE:%.*]] = tail call i8* @llvm.stacksave()
+; GLOBALOPT_ARGPROMOTION-NEXT:    [[ARGMEM:%.*]] = alloca inalloca <{ [[STRUCT_A:%.*]] }>, align 4
+; GLOBALOPT_ARGPROMOTION-NEXT:    call fastcc void @internalfun(<{ [[STRUCT_A]] }>* [[ARGMEM]])
+; GLOBALOPT_ARGPROMOTION-NEXT:    call void @llvm.stackrestore(i8* [[INALLOCA_SAVE]])
+; GLOBALOPT_ARGPROMOTION-NEXT:    ret void
+;
   %inalloca.save = tail call i8* @llvm.stacksave()
   %argmem = alloca inalloca <{ %struct.a }>, align 4
   call x86_thiscallcc void @internalfun(%struct.a* %a, <{ %struct.a }>* inalloca %argmem)
diff --git a/llvm/test/Transforms/ArgumentPromotion/aggregate-promote.ll b/llvm/test/Transforms/ArgumentPromotion/aggregate-promote.ll
index b0bab7784..9c8f9f5df 100644
--- a/llvm/test/Transforms/ArgumentPromotion/aggregate-promote.ll
+++ b/llvm/test/Transforms/ArgumentPromotion/aggregate-promote.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
 ; RUN: opt < %s -argpromotion -S | FileCheck %s
 ; RUN: opt < %s -passes=argpromotion -S | FileCheck %s
 
@@ -5,27 +6,32 @@
 @G = constant %T { i32 0, i32 0, i32 17, i32 25 }
 
 define internal i32 @test(%T* %p) {
-; CHECK-LABEL: define internal i32 @test(
-; CHECK: i32 %{{.*}}, i32 %{{.*}})
+; CHECK-LABEL: define {{[^@]+}}@test
+; CHECK-SAME: (i32 [[P_0_2_VAL:%.*]], i32 [[P_0_3_VAL:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[V:%.*]] = add i32 [[P_0_3_VAL]], [[P_0_2_VAL]]
+; CHECK-NEXT:    ret i32 [[V]]
+;
 entry:
   %a.gep = getelementptr %T, %T* %p, i64 0, i32 3
   %b.gep = getelementptr %T, %T* %p, i64 0, i32 2
   %a = load i32, i32* %a.gep
   %b = load i32, i32* %b.gep
-; CHECK-NOT: load
   %v = add i32 %a, %b
   ret i32 %v
-; CHECK: ret i32
 }
 
 define i32 @caller() {
-; CHECK-LABEL: define i32 @caller(
+; CHECK-LABEL: define {{[^@]+}}@caller()
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[G_IDX:%.*]] = getelementptr [[T:%.*]], %T* @G, i64 0, i32 2
+; CHECK-NEXT:    [[G_IDX_VAL:%.*]] = load i32, i32* [[G_IDX]]
+; CHECK-NEXT:    [[G_IDX1:%.*]] = getelementptr [[T]], %T* @G, i64 0, i32 3
+; CHECK-NEXT:    [[G_IDX1_VAL:%.*]] = load i32, i32* [[G_IDX1]]
+; CHECK-NEXT:    [[V:%.*]] = call i32 @test(i32 [[G_IDX_VAL]], i32 [[G_IDX1_VAL]])
+; CHECK-NEXT:    ret i32 [[V]]
+;
 entry:
   %v = call i32 @test(%T* @G)
-; CHECK: %[[B_GEP:.*]] = getelementptr %T, %T* @G, i64 0, i32 2
-; CHECK: %[[B:.*]] = load i32, i32* %[[B_GEP]]
-; CHECK: %[[A_GEP:.*]] = getelementptr %T, %T* @G, i64 0, i32 3
-; CHECK: %[[A:.*]] = load i32, i32* %[[A_GEP]]
-; CHECK: call i32 @test(i32 %[[B]], i32 %[[A]])
   ret i32 %v
 }
diff --git a/llvm/test/Transforms/ArgumentPromotion/attrs.ll b/llvm/test/Transforms/ArgumentPromotion/attrs.ll
index 29cef50fe..7d801ebf6 100644
--- a/llvm/test/Transforms/ArgumentPromotion/attrs.ll
+++ b/llvm/test/Transforms/ArgumentPromotion/attrs.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
 ; RUN: opt < %s -argpromotion -S | FileCheck %s
 ; RUN: opt < %s -passes=argpromotion -S | FileCheck %s
 
@@ -5,48 +6,57 @@
 
 ; Don't drop 'byval' on %X here.
 define internal void @f(%struct.ss* byval %b, i32* byval %X, i32 %i) nounwind {
-; CHECK-LABEL: define internal void @f(
-; CHECK: i32 %[[B0:.*]], i64 %[[B1:.*]], i32* byval %X, i32 %i)
+; CHECK-LABEL: define {{[^@]+}}@f
+; CHECK-SAME: (i32 [[B_0:%.*]], i64 [[B_1:%.*]], i32* byval [[X:%.*]], i32 [[I:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[B:%.*]] = alloca [[STRUCT_SS:%.*]]
+; CHECK-NEXT:    [[DOT0:%.*]] = getelementptr [[STRUCT_SS]], %struct.ss* [[B]], i32 0, i32 0
+; CHECK-NEXT:    store i32 [[B_0]], i32* [[DOT0]]
+; CHECK-NEXT:    [[DOT1:%.*]] = getelementptr [[STRUCT_SS]], %struct.ss* [[B]], i32 0, i32 1
+; CHECK-NEXT:    store i64 [[B_1]], i64* [[DOT1]]
+; CHECK-NEXT:    [[TMP:%.*]] = getelementptr [[STRUCT_SS]], %struct.ss* [[B]], i32 0, i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[TMP]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = add i32 [[TMP1]], 1
+; CHECK-NEXT:    store i32 [[TMP2]], i32* [[TMP]], align 4
+; CHECK-NEXT:    store i32 0, i32* [[X]]
+; CHECK-NEXT:    ret void
+;
 entry:
-; CHECK: %[[B:.*]] = alloca %struct.ss
-; CHECK: %[[B_GEP0:.*]] = getelementptr %struct.ss, %struct.ss* %[[B]], i32 0, i32 0
-; CHECK: store i32 %[[B0]], i32* %[[B_GEP0]]
-; CHECK: %[[B_GEP1:.*]] = getelementptr %struct.ss, %struct.ss* %[[B]], i32 0, i32 1
-; CHECK: store i64 %[[B1]], i64* %[[B_GEP1]]
 
   %tmp = getelementptr %struct.ss, %struct.ss* %b, i32 0, i32 0
-; CHECK: %[[TMP:.*]] = getelementptr %struct.ss, %struct.ss* %[[B]], i32 0, i32 0
   %tmp1 = load i32, i32* %tmp, align 4
-; CHECK: %[[TMP1:.*]] = load i32, i32* %[[TMP]]
   %tmp2 = add i32 %tmp1, 1
-; CHECK: %[[TMP2:.*]] = add i32 %[[TMP1]], 1
   store i32 %tmp2, i32* %tmp, align 4
-; CHECK: store i32 %[[TMP2]], i32* %[[TMP]]
 
   store i32 0, i32* %X
-; CHECK: store i32 0, i32* %X
   ret void
 }
 
 ; Also make sure we don't drop the call zeroext attribute.
 define i32 @test(i32* %X) {
-; CHECK-LABEL: define i32 @test(
+; CHECK-LABEL: define {{[^@]+}}@test
+; CHECK-SAME: (i32* [[X:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[S:%.*]] = alloca [[STRUCT_SS:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr [[STRUCT_SS]], %struct.ss* [[S]], i32 0, i32 0
+; CHECK-NEXT:    store i32 1, i32* [[TMP1]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr [[STRUCT_SS]], %struct.ss* [[S]], i32 0, i32 1
+; CHECK-NEXT:    store i64 2, i64* [[TMP4]], align 4
+; CHECK-NEXT:    [[S_0:%.*]] = getelementptr [[STRUCT_SS]], %struct.ss* [[S]], i32 0, i32 0
+; CHECK-NEXT:    [[S_0_VAL:%.*]] = load i32, i32* [[S_0]]
+; CHECK-NEXT:    [[S_1:%.*]] = getelementptr [[STRUCT_SS]], %struct.ss* [[S]], i32 0, i32 1
+; CHECK-NEXT:    [[S_1_VAL:%.*]] = load i64, i64* [[S_1]]
+; CHECK-NEXT:    call void @f(i32 [[S_0_VAL]], i64 [[S_1_VAL]], i32* byval [[X]], i32 zeroext 0)
+; CHECK-NEXT:    ret i32 0
+;
 entry:
   %S = alloca %struct.ss
-; CHECK: %[[S:.*]] = alloca %struct.ss
   %tmp1 = getelementptr %struct.ss, %struct.ss* %S, i32 0, i32 0
   store i32 1, i32* %tmp1, align 8
-; CHECK: store i32 1
   %tmp4 = getelementptr %struct.ss, %struct.ss* %S, i32 0, i32 1
   store i64 2, i64* %tmp4, align 4
-; CHECK: store i64 2
 
   call void @f( %struct.ss* byval %S, i32* byval %X, i32 zeroext 0)
-; CHECK: %[[S_GEP0:.*]] = getelementptr %struct.ss, %struct.ss* %[[S]], i32 0, i32 0
-; CHECK: %[[S0:.*]] = load i32, i32* %[[S_GEP0]]
-; CHECK: %[[S_GEP1:.*]] = getelementptr %struct.ss, %struct.ss* %[[S]], i32 0, i32 1
-; CHECK: %[[S1:.*]] = load i64, i64* %[[S_GEP1]]
-; CHECK: call void @f(i32 %[[S0]], i64 %[[S1]], i32* byval %X, i32 zeroext 0)
 
   ret i32 0
 }
diff --git a/llvm/test/Transforms/ArgumentPromotion/basictest.ll b/llvm/test/Transforms/ArgumentPromotion/basictest.ll
index 89888bbc9..13cbd7781 100644
--- a/llvm/test/Transforms/ArgumentPromotion/basictest.ll
+++ b/llvm/test/Transforms/ArgumentPromotion/basictest.ll
@@ -1,8 +1,13 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
 ; RUN: opt < %s -basicaa -argpromotion -mem2reg -S | FileCheck %s
 target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
 
 define internal i32 @test(i32* %X, i32* %Y) {
-; CHECK-LABEL: define internal i32 @test(i32 %X.val, i32 %Y.val)
+; CHECK-LABEL: define {{[^@]+}}@test
+; CHECK-SAME: (i32 [[X_VAL:%.*]], i32 [[Y_VAL:%.*]])
+; CHECK-NEXT:    [[C:%.*]] = add i32 [[X_VAL]], [[Y_VAL]]
+; CHECK-NEXT:    ret i32 [[C]]
+;
   %A = load i32, i32* %X
   %B = load i32, i32* %Y
   %C = add i32 %A, %B
@@ -10,20 +15,25 @@ define internal i32 @test(i32* %X, i32* %Y) {
 }
 
 define internal i32 @caller(i32* %B) {
-; CHECK-LABEL: define internal i32 @caller(i32 %B.val1)
+; CHECK-LABEL: define {{[^@]+}}@caller
+; CHECK-SAME: (i32 [[B_VAL1:%.*]])
+; CHECK-NEXT:    [[C:%.*]] = call i32 @test(i32 1, i32 [[B_VAL1]])
+; CHECK-NEXT:    ret i32 [[C]]
+;
   %A = alloca i32
   store i32 1, i32* %A
   %C = call i32 @test(i32* %A, i32* %B)
-; CHECK: call i32 @test(i32 1, i32 %B.val1)
   ret i32 %C
 }
 
 define i32 @callercaller() {
-; CHECK-LABEL: define i32 @callercaller()
+; CHECK-LABEL: define {{[^@]+}}@callercaller()
+; CHECK-NEXT:    [[X:%.*]] = call i32 @caller(i32 2)
+; CHECK-NEXT:    ret i32 [[X]]
+;
   %B = alloca i32
   store i32 2, i32* %B
   %X = call i32 @caller(i32* %B)
-; CHECK: call i32 @caller(i32 2)
   ret i32 %X
 }
 
diff --git a/llvm/test/Transforms/ArgumentPromotion/byval-2.ll b/llvm/test/Transforms/ArgumentPromotion/byval-2.ll
index 3e1fee8ba..f46a1f371 100644
--- a/llvm/test/Transforms/ArgumentPromotion/byval-2.ll
+++ b/llvm/test/Transforms/ArgumentPromotion/byval-2.ll
@@ -1,13 +1,28 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
 ; RUN: opt < %s -argpromotion -S | FileCheck %s
 ; RUN: opt < %s -passes=argpromotion -S | FileCheck %s
 
 ; Arg promotion eliminates the struct argument.
-; FIXME: Should it eliminate the i32* argument?
+; FIXME: We should eliminate the i32* argument.
 
 %struct.ss = type { i32, i64 }
 
 define internal void @f(%struct.ss* byval  %b, i32* byval %X) nounwind  {
-; CHECK-LABEL: define internal void @f(i32 %b.0, i64 %b.1, i32* byval %X)
+; CHECK-LABEL: define {{[^@]+}}@f
+; CHECK-SAME: (i32 [[B_0:%.*]], i64 [[B_1:%.*]], i32* byval [[X:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[B:%.*]] = alloca [[STRUCT_SS:%.*]]
+; CHECK-NEXT:    [[DOT0:%.*]] = getelementptr [[STRUCT_SS]], %struct.ss* [[B]], i32 0, i32 0
+; CHECK-NEXT:    store i32 [[B_0]], i32* [[DOT0]]
+; CHECK-NEXT:    [[DOT1:%.*]] = getelementptr [[STRUCT_SS]], %struct.ss* [[B]], i32 0, i32 1
+; CHECK-NEXT:    store i64 [[B_1]], i64* [[DOT1]]
+; CHECK-NEXT:    [[TMP:%.*]] = getelementptr [[STRUCT_SS]], %struct.ss* [[B]], i32 0, i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[TMP]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = add i32 [[TMP1]], 1
+; CHECK-NEXT:    store i32 [[TMP2]], i32* [[TMP]], align 4
+; CHECK-NEXT:    store i32 0, i32* [[X]]
+; CHECK-NEXT:    ret void
+;
 entry:
   %tmp = getelementptr %struct.ss, %struct.ss* %b, i32 0, i32 0
   %tmp1 = load i32, i32* %tmp, align 4
@@ -19,7 +34,21 @@ entry:
 }
 
 define i32 @test(i32* %X) {
-; CHECK-LABEL: define i32 @test
+; CHECK-LABEL: define {{[^@]+}}@test
+; CHECK-SAME: (i32* [[X:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[S:%.*]] = alloca [[STRUCT_SS:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr [[STRUCT_SS]], %struct.ss* [[S]], i32 0, i32 0
+; CHECK-NEXT:    store i32 1, i32* [[TMP1]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr [[STRUCT_SS]], %struct.ss* [[S]], i32 0, i32 1
+; CHECK-NEXT:    store i64 2, i64* [[TMP4]], align 4
+; CHECK-NEXT:    [[S_0:%.*]] = getelementptr [[STRUCT_SS]], %struct.ss* [[S]], i32 0, i32 0
+; CHECK-NEXT:    [[S_0_VAL:%.*]] = load i32, i32* [[S_0]]
+; CHECK-NEXT:    [[S_1:%.*]] = getelementptr [[STRUCT_SS]], %struct.ss* [[S]], i32 0, i32 1
+; CHECK-NEXT:    [[S_1_VAL:%.*]] = load i64, i64* [[S_1]]
+; CHECK-NEXT:    call void @f(i32 [[S_0_VAL]], i64 [[S_1_VAL]], i32* byval [[X]])
+; CHECK-NEXT:    ret i32 0
+;
 entry:
   %S = alloca %struct.ss
   %tmp1 = getelementptr %struct.ss, %struct.ss* %S, i32 0, i32 0
@@ -27,6 +56,5 @@ entry:
   %tmp4 = getelementptr %struct.ss, %struct.ss* %S, i32 0, i32 1
   store i64 2, i64* %tmp4, align 4
   call void @f( %struct.ss* byval %S, i32* byval %X)
-; CHECK: call void @f(i32 %{{.*}}, i64 %{{.*}}, i32* byval %{{.*}})
   ret i32 0
 }
diff --git a/llvm/test/Transforms/ArgumentPromotion/byval.ll b/llvm/test/Transforms/ArgumentPromotion/byval.ll
index 00542e3ec..020c0f292 100644
--- a/llvm/test/Transforms/ArgumentPromotion/byval.ll
+++ b/llvm/test/Transforms/ArgumentPromotion/byval.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
 ; RUN: opt < %s -argpromotion -S | FileCheck %s
 ; RUN: opt < %s -passes=argpromotion -S | FileCheck %s
 
@@ -6,6 +7,20 @@ target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:1
 %struct.ss = type { i32, i64 }
 
 define internal void @f(%struct.ss* byval  %b) nounwind  {
+; CHECK-LABEL: define {{[^@]+}}@f
+; CHECK-SAME: (i32 [[B_0:%.*]], i64 [[B_1:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[B:%.*]] = alloca [[STRUCT_SS:%.*]]
+; CHECK-NEXT:    [[DOT0:%.*]] = getelementptr [[STRUCT_SS]], %struct.ss* [[B]], i32 0, i32 0
+; CHECK-NEXT:    store i32 [[B_0]], i32* [[DOT0]]
+; CHECK-NEXT:    [[DOT1:%.*]] = getelementptr [[STRUCT_SS]], %struct.ss* [[B]], i32 0, i32 1
+; CHECK-NEXT:    store i64 [[B_1]], i64* [[DOT1]]
+; CHECK-NEXT:    [[TMP:%.*]] = getelementptr [[STRUCT_SS]], %struct.ss* [[B]], i32 0, i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[TMP]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = add i32 [[TMP1]], 1
+; CHECK-NEXT:    store i32 [[TMP2]], i32* [[TMP]], align 4
+; CHECK-NEXT:    ret void
+;
 entry:
   %tmp = getelementptr %struct.ss, %struct.ss* %b, i32 0, i32 0
   %tmp1 = load i32, i32* %tmp, align 4
@@ -14,12 +29,22 @@ entry:
   ret void
 }
 
-; CHECK-LABEL: define internal void @f(i32 %b.0, i64 %b.1)
-; CHECK: alloca %struct.ss{{$}}
-; CHECK: store i32 %b.0
-; CHECK: store i64 %b.1
 
 define internal void @g(%struct.ss* byval align 32 %b) nounwind {
+; CHECK-LABEL: define {{[^@]+}}@g
+; CHECK-SAME: (i32 [[B_0:%.*]], i64 [[B_1:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[B:%.*]] = alloca [[STRUCT_SS:%.*]], align 32
+; CHECK-NEXT:    [[DOT0:%.*]] = getelementptr [[STRUCT_SS]], %struct.ss* [[B]], i32 0, i32 0
+; CHECK-NEXT:    store i32 [[B_0]], i32* [[DOT0]]
+; CHECK-NEXT:    [[DOT1:%.*]] = getelementptr [[STRUCT_SS]], %struct.ss* [[B]], i32 0, i32 1
+; CHECK-NEXT:    store i64 [[B_1]], i64* [[DOT1]]
+; CHECK-NEXT:    [[TMP:%.*]] = getelementptr [[STRUCT_SS]], %struct.ss* [[B]], i32 0, i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[TMP]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = add i32 [[TMP1]], 1
+; CHECK-NEXT:    store i32 [[TMP2]], i32* [[TMP]], align 4
+; CHECK-NEXT:    ret void
+;
 entry:
   %tmp = getelementptr %struct.ss, %struct.ss* %b, i32 0, i32 0
   %tmp1 = load i32, i32* %tmp, align 4
@@ -28,12 +53,27 @@ entry:
   ret void
 }
 
-; CHECK-LABEL: define internal void @g(i32 %b.0, i64 %b.1)
-; CHECK: alloca %struct.ss, align 32
-; CHECK: store i32 %b.0
-; CHECK: store i64 %b.1
 
 define i32 @main() nounwind  {
+; CHECK-LABEL: define {{[^@]+}}@main()
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[S:%.*]] = alloca [[STRUCT_SS:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr [[STRUCT_SS]], %struct.ss* [[S]], i32 0, i32 0
+; CHECK-NEXT:    store i32 1, i32* [[TMP1]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr [[STRUCT_SS]], %struct.ss* [[S]], i32 0, i32 1
+; CHECK-NEXT:    store i64 2, i64* [[TMP4]], align 4
+; CHECK-NEXT:    [[S_0:%.*]] = getelementptr [[STRUCT_SS]], %struct.ss* [[S]], i32 0, i32 0
+; CHECK-NEXT:    [[S_0_VAL:%.*]] = load i32, i32* [[S_0]]
+; CHECK-NEXT:    [[S_1:%.*]] = getelementptr [[STRUCT_SS]], %struct.ss* [[S]], i32 0, i32 1
+; CHECK-NEXT:    [[S_1_VAL:%.*]] = load i64, i64* [[S_1]]
+; CHECK-NEXT:    call void @f(i32 [[S_0_VAL]], i64 [[S_1_VAL]])
+; CHECK-NEXT:    [[S_01:%.*]] = getelementptr [[STRUCT_SS]], %struct.ss* [[S]], i32 0, i32 0
+; CHECK-NEXT:    [[S_01_VAL:%.*]] = load i32, i32* [[S_01]]
+; CHECK-NEXT:    [[S_12:%.*]] = getelementptr [[STRUCT_SS]], %struct.ss* [[S]], i32 0, i32 1
+; CHECK-NEXT:    [[S_12_VAL:%.*]] = load i64, i64* [[S_12]]
+; CHECK-NEXT:    call void @g(i32 [[S_01_VAL]], i64 [[S_12_VAL]])
+; CHECK-NEXT:    ret i32 0
+;
 entry:
   %S = alloca %struct.ss
   %tmp1 = getelementptr %struct.ss, %struct.ss* %S, i32 0, i32 0
@@ -45,6 +85,4 @@ entry:
   ret i32 0
 }
 
-; CHECK-LABEL: define i32 @main
-; CHECK: call void @f(i32 %{{.*}}, i64 %{{.*}})
-; CHECK: call void @g(i32 %{{.*}}, i64 %{{.*}})
+
diff --git a/llvm/test/Transforms/ArgumentPromotion/chained.ll b/llvm/test/Transforms/ArgumentPromotion/chained.ll
index 028c6c426..c7115fed9 100644
--- a/llvm/test/Transforms/ArgumentPromotion/chained.ll
+++ b/llvm/test/Transforms/ArgumentPromotion/chained.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
 ; RUN: opt < %s -argpromotion -S | FileCheck %s
 ; RUN: opt < %s -passes=argpromotion -S | FileCheck %s
 
@@ -5,23 +6,27 @@
 @G2 = constant i32* @G1
 
 define internal i32 @test(i32** %x) {
-; CHECK-LABEL: define internal i32 @test(
-; CHECK: i32 %{{.*}})
+; CHECK-LABEL: define {{[^@]+}}@test
+; CHECK-SAME: (i32 [[X_VAL_VAL:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret i32 [[X_VAL_VAL]]
+;
 entry:
   %y = load i32*, i32** %x
   %z = load i32, i32* %y
-; CHECK-NOT: load
   ret i32 %z
-; CHECK: ret i32
 }
 
 define i32 @caller() {
-; CHECK-LABEL: define i32 @caller()
+; CHECK-LABEL: define {{[^@]+}}@caller()
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[G2_VAL:%.*]] = load i32*, i32** @G2
+; CHECK-NEXT:    [[G2_VAL_VAL:%.*]] = load i32, i32* [[G2_VAL]]
+; CHECK-NEXT:    [[X:%.*]] = call i32 @test(i32 [[G2_VAL_VAL]])
+; CHECK-NEXT:    ret i32 [[X]]
+;
 entry:
   %x = call i32 @test(i32** @G2)
-; CHECK: %[[Y:.*]] = load i32*, i32** @G2
-; CHECK: %[[Z:.*]] = load i32, i32* %[[Y]]
-; CHECK: call i32 @test(i32 %[[Z]])
   ret i32 %x
 }
 
diff --git a/llvm/test/Transforms/ArgumentPromotion/control-flow.ll b/llvm/test/Transforms/ArgumentPromotion/control-flow.ll
index c3fe0c00e..2835788e7 100644
--- a/llvm/test/Transforms/ArgumentPromotion/control-flow.ll
+++ b/llvm/test/Transforms/ArgumentPromotion/control-flow.ll
@@ -1,10 +1,19 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
 ; RUN: opt < %s -argpromotion -S | FileCheck %s
 ; RUN: opt < %s -passes=argpromotion -S | FileCheck %s
 
 ; Don't promote around control flow.
 define internal i32 @callee(i1 %C, i32* %P) {
-; CHECK-LABEL: define internal i32 @callee(
-; CHECK: i1 %C, i32* %P)
+; CHECK-LABEL: define {{[^@]+}}@callee
+; CHECK-SAME: (i1 [[C:%.*]], i32* [[P:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]]
+; CHECK:       T:
+; CHECK-NEXT:    ret i32 17
+; CHECK:       F:
+; CHECK-NEXT:    [[X:%.*]] = load i32, i32* [[P]]
+; CHECK-NEXT:    ret i32 [[X]]
+;
 entry:
   br i1 %C, label %T, label %F
 
@@ -17,11 +26,13 @@ F:
 }
 
 define i32 @foo() {
-; CHECK-LABEL: define i32 @foo(
+; CHECK-LABEL: define {{[^@]+}}@foo()
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[X:%.*]] = call i32 @callee(i1 true, i32* null)
+; CHECK-NEXT:    ret i32 [[X]]
+;
 entry:
-; CHECK-NOT: load i32, i32* null
   %X = call i32 @callee(i1 true, i32* null)
-; CHECK: call i32 @callee(i1 true, i32* null)
   ret i32 %X
 }
 
diff --git a/llvm/test/Transforms/ArgumentPromotion/control-flow2.ll b/llvm/test/Transforms/ArgumentPromotion/control-flow2.ll
index b75a32ddb..3f13b5e23 100644
--- a/llvm/test/Transforms/ArgumentPromotion/control-flow2.ll
+++ b/llvm/test/Transforms/ArgumentPromotion/control-flow2.ll
@@ -1,24 +1,39 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
 ; RUN: opt < %s -argpromotion -S | FileCheck %s
 ; RUN: opt < %s -passes=argpromotion -S | FileCheck %s
 
-; CHECK: load i32, i32* %A
 target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
 
 define internal i32 @callee(i1 %C, i32* %P) {
-        br i1 %C, label %T, label %F
+; CHECK-LABEL: define {{[^@]+}}@callee
+; CHECK-SAME: (i1 [[C:%.*]], i32 [[P_VAL:%.*]])
+; CHECK-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]]
+; CHECK:       T:
+; CHECK-NEXT:    ret i32 17
+; CHECK:       F:
+; CHECK-NEXT:    ret i32 [[P_VAL]]
+;
+  br i1 %C, label %T, label %F
 
 T:              ; preds = %0
-        ret i32 17
+  ret i32 17
 
 F:              ; preds = %0
-        %X = load i32, i32* %P               ; <i32> [#uses=1]
-        ret i32 %X
+  %X = load i32, i32* %P               ; <i32> [#uses=1]
+  ret i32 %X
 }
 
 define i32 @foo() {
-        %A = alloca i32         ; <i32*> [#uses=2]
-        store i32 17, i32* %A
-        %X = call i32 @callee( i1 false, i32* %A )              ; <i32> [#uses=1]
-        ret i32 %X
+; CHECK-LABEL: define {{[^@]+}}@foo()
+; CHECK-NEXT:    [[A:%.*]] = alloca i32
+; CHECK-NEXT:    store i32 17, i32* [[A]]
+; CHECK-NEXT:    [[A_VAL:%.*]] = load i32, i32* [[A]]
+; CHECK-NEXT:    [[X:%.*]] = call i32 @callee(i1 false, i32 [[A_VAL]])
+; CHECK-NEXT:    ret i32 [[X]]
+;
+  %A = alloca i32         ; <i32*> [#uses=2]
+  store i32 17, i32* %A
+  %X = call i32 @callee( i1 false, i32* %A )              ; <i32> [#uses=1]
+  ret i32 %X
 }
 
diff --git a/llvm/test/Transforms/ArgumentPromotion/crash.ll b/llvm/test/Transforms/ArgumentPromotion/crash.ll
index d46a48101..d79d90643 100644
--- a/llvm/test/Transforms/ArgumentPromotion/crash.ll
+++ b/llvm/test/Transforms/ArgumentPromotion/crash.ll
@@ -1,24 +1,34 @@
-; RUN: opt -S < %s -inline -argpromotion | FileCheck %s
-; RUN: opt -S < %s -passes=inline,argpromotion | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; RUN: opt -S < %s -inline -argpromotion | FileCheck %s --check-prefixes=ARGPROMOTION,ALL_OLDPM
+; RUN: opt -S < %s -passes=inline,argpromotion | FileCheck %s --check-prefixes=ARGPROMOTION,ALL_NEWPM
 
 %S = type { %S* }
 
 ; Inlining should nuke the invoke (and any inlined calls) here even with
 ; argument promotion running along with it.
 define void @zot() personality i32 (...)* @wibble {
-; CHECK-LABEL: define void @zot() personality i32 (...)* @wibble
-; CHECK-NOT: call
-; CHECK-NOT: invoke
+; ARGPROMOTION-LABEL: define {{[^@]+}}@zot() personality i32 (...)* @wibble
+; ARGPROMOTION-NEXT:  bb:
+; ARGPROMOTION-NEXT:    unreachable
+; ARGPROMOTION:       hoge.exit:
+; ARGPROMOTION-NEXT:    br label [[BB1:%.*]]
+; ARGPROMOTION:       bb1:
+; ARGPROMOTION-NEXT:    unreachable
+; ARGPROMOTION:       bb2:
+; ARGPROMOTION-NEXT:    [[TMP:%.*]] = landingpad { i8*, i32 }
+; ARGPROMOTION-NEXT:    cleanup
+; ARGPROMOTION-NEXT:    unreachable
+;
 bb:
   invoke void @hoge()
-          to label %bb1 unwind label %bb2
+  to label %bb1 unwind label %bb2
 
 bb1:
   unreachable
 
 bb2:
   %tmp = landingpad { i8*, i32 }
-          cleanup
+  cleanup
   unreachable
 }
 
@@ -35,6 +45,10 @@ bb:
 }
 
 define internal i1 @eggs(i8* %arg) {
+; ALL_NEWPM-LABEL: define {{[^@]+}}@eggs()
+; ALL_NEWPM-NEXT:  bb:
+; ALL_NEWPM-NEXT:    unreachable
+;
 bb:
   %tmp = call zeroext i1 @barney(i8* %arg)
   unreachable
@@ -46,26 +60,39 @@ bb:
 }
 
 define i32 @test_inf_promote_caller(i32 %arg) {
-; CHECK-LABEL: define i32 @test_inf_promote_caller(
+; ARGPROMOTION-LABEL: define {{[^@]+}}@test_inf_promote_caller
+; ARGPROMOTION-SAME: (i32 [[ARG:%.*]])
+; ARGPROMOTION-NEXT:  bb:
+; ARGPROMOTION-NEXT:    [[TMP:%.*]] = alloca [[S:%.*]]
+; ARGPROMOTION-NEXT:    [[TMP1:%.*]] = alloca [[S]]
+; ARGPROMOTION-NEXT:    [[TMP2:%.*]] = call i32 @test_inf_promote_callee(%S* [[TMP]], %S* [[TMP1]])
+; ARGPROMOTION-NEXT:    ret i32 0
+;
 bb:
   %tmp = alloca %S
   %tmp1 = alloca %S
   %tmp2 = call i32 @test_inf_promote_callee(%S* %tmp, %S* %tmp1)
-; CHECK: call i32 @test_inf_promote_callee(%S* %{{.*}}, %S* %{{.*}})
 
   ret i32 0
 }
 
 define internal i32 @test_inf_promote_callee(%S* %arg, %S* %arg1) {
-; CHECK-LABEL: define internal i32 @test_inf_promote_callee(
-; CHECK: %S* %{{.*}}, %S* %{{.*}})
+; ARGPROMOTION-LABEL: define {{[^@]+}}@test_inf_promote_callee
+; ARGPROMOTION-SAME: (%S* [[ARG:%.*]], %S* [[ARG1:%.*]])
+; ARGPROMOTION-NEXT:  bb:
+; ARGPROMOTION-NEXT:    [[TMP:%.*]] = getelementptr [[S:%.*]], %S* [[ARG1]], i32 0, i32 0
+; ARGPROMOTION-NEXT:    [[TMP2:%.*]] = load %S*, %S** [[TMP]]
+; ARGPROMOTION-NEXT:    [[TMP3:%.*]] = getelementptr [[S]], %S* [[ARG]], i32 0, i32 0
+; ARGPROMOTION-NEXT:    [[TMP4:%.*]] = load %S*, %S** [[TMP3]]
+; ARGPROMOTION-NEXT:    [[TMP5:%.*]] = call i32 @test_inf_promote_callee(%S* [[TMP4]], %S* [[TMP2]])
+; ARGPROMOTION-NEXT:    ret i32 0
+;
 bb:
   %tmp = getelementptr %S, %S* %arg1, i32 0, i32 0
   %tmp2 = load %S*, %S** %tmp
   %tmp3 = getelementptr %S, %S* %arg, i32 0, i32 0
   %tmp4 = load %S*, %S** %tmp3
   %tmp5 = call i32 @test_inf_promote_callee(%S* %tmp4, %S* %tmp2)
-; CHECK: call i32 @test_inf_promote_callee(%S* %{{.*}}, %S* %{{.*}})
 
   ret i32 0
 }
diff --git a/llvm/test/Transforms/ArgumentPromotion/dbg.ll b/llvm/test/Transforms/ArgumentPromotion/dbg.ll
index 668276290..b26f5ce04 100644
--- a/llvm/test/Transforms/ArgumentPromotion/dbg.ll
+++ b/llvm/test/Transforms/ArgumentPromotion/dbg.ll
@@ -1,10 +1,15 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
 ; RUN: opt < %s -argpromotion -S | FileCheck %s
 ; RUN: opt < %s -passes=argpromotion -S | FileCheck %s
 
 declare void @sink(i32)
 
-; CHECK: define internal void @test({{.*}} !dbg [[SP:![0-9]+]]
 define internal void @test(i32** %X) !dbg !2 {
+; CHECK-LABEL: define {{[^@]+}}@test
+; CHECK-SAME: (i32 [[X_VAL_VAL:%.*]]) !dbg !3
+; CHECK-NEXT:    call void @sink(i32 [[X_VAL_VAL]])
+; CHECK-NEXT:    ret void
+;
   %1 = load i32*, i32** %X, align 8
   %2 = load i32, i32* %1, align 8
   call void @sink(i32 %2)
@@ -13,30 +18,38 @@ define internal void @test(i32** %X) !dbg !2 {
 
 %struct.pair = type { i32, i32 }
 
-; CHECK: define internal void @test_byval(i32 %{{.*}}, i32 %{{.*}})
 define internal void @test_byval(%struct.pair* byval %P) {
+; CHECK-LABEL: define {{[^@]+}}@test_byval
+; CHECK-SAME: (i32 [[P_0:%.*]], i32 [[P_1:%.*]])
+; CHECK-NEXT:    [[P:%.*]] = alloca [[STRUCT_PAIR:%.*]]
+; CHECK-NEXT:    [[DOT0:%.*]] = getelementptr [[STRUCT_PAIR]], %struct.pair* [[P]], i32 0, i32 0
+; CHECK-NEXT:    store i32 [[P_0]], i32* [[DOT0]]
+; CHECK-NEXT:    [[DOT1:%.*]] = getelementptr [[STRUCT_PAIR]], %struct.pair* [[P]], i32 0, i32 1
+; CHECK-NEXT:    store i32 [[P_1]], i32* [[DOT1]]
+; CHECK-NEXT:    ret void
+;
   ret void
 }
 
-; CHECK-LABEL: define {{.*}} @caller(
 define void @caller(i32** %Y, %struct.pair* %P) {
-; CHECK:  load i32*, {{.*}} !dbg [[LOC_1:![0-9]+]]
-; CHECK-NEXT:  load i32, {{.*}} !dbg [[LOC_1]]
-; CHECK-NEXT: call void @test(i32 %{{.*}}), !dbg [[LOC_1]]
+; CHECK-LABEL: define {{[^@]+}}@caller
+; CHECK-SAME: (i32** [[Y:%.*]], %struct.pair* [[P:%.*]])
+; CHECK-NEXT:    [[Y_VAL:%.*]] = load i32*, i32** [[Y]], align 8, !dbg !4
+; CHECK-NEXT:    [[Y_VAL_VAL:%.*]] = load i32, i32* [[Y_VAL]], align 8, !dbg !4
+; CHECK-NEXT:    call void @test(i32 [[Y_VAL_VAL]]), !dbg !4
+; CHECK-NEXT:    [[P_0:%.*]] = getelementptr [[STRUCT_PAIR:%.*]], %struct.pair* [[P]], i32 0, i32 0, !dbg !5
+; CHECK-NEXT:    [[P_0_VAL:%.*]] = load i32, i32* [[P_0]], !dbg !5
+; CHECK-NEXT:    [[P_1:%.*]] = getelementptr [[STRUCT_PAIR]], %struct.pair* [[P]], i32 0, i32 1, !dbg !5
+; CHECK-NEXT:    [[P_1_VAL:%.*]] = load i32, i32* [[P_1]], !dbg !5
+; CHECK-NEXT:    call void @test_byval(i32 [[P_0_VAL]], i32 [[P_1_VAL]]), !dbg !5
+; CHECK-NEXT:    ret void
+;
   call void @test(i32** %Y), !dbg !1
 
-; CHECK: getelementptr %struct.pair, {{.*}} !dbg [[LOC_2:![0-9]+]]
-; CHECK-NEXT: load i32, i32* {{.*}} !dbg [[LOC_2]]
-; CHECK-NEXT: getelementptr %struct.pair, {{.*}} !dbg [[LOC_2]]
-; CHECK-NEXT: load i32, i32* {{.*}} !dbg [[LOC_2]]
-; CHECK-NEXT: call void @test_byval(i32 %{{.*}}, i32 %{{.*}}), !dbg [[LOC_2]]
   call void @test_byval(%struct.pair* %P), !dbg !6
   ret void
 }
 
-; CHECK: [[SP]] = distinct !DISubprogram(name: "test",
-; CHECK: [[LOC_1]] = !DILocation(line: 8
-; CHECK: [[LOC_2]] = !DILocation(line: 9
 
 !llvm.module.flags = !{!0}
 !llvm.dbg.cu = !{!3}
diff --git a/llvm/test/Transforms/ArgumentPromotion/fp80.ll b/llvm/test/Transforms/ArgumentPromotion/fp80.ll
index bd780fa21..dd7cebf5a 100644
--- a/llvm/test/Transforms/ArgumentPromotion/fp80.ll
+++ b/llvm/test/Transforms/ArgumentPromotion/fp80.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
 ; RUN: opt < %s -argpromotion -S | FileCheck %s
 ; RUN: opt < %s -passes=argpromotion -S | FileCheck %s
 
@@ -13,6 +14,16 @@ target triple = "x86_64-unknown-linux-gnu"
 @a = internal global %struct.Foo { i32 1, i64 2 }, align 8
 
 define void @run() {
+; CHECK-LABEL: define {{[^@]+}}@run()
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = tail call i8 @UseLongDoubleUnsafely(%union.u* byval align 16 bitcast (%struct.s* @b to %union.u*))
+; CHECK-NEXT:    [[DOT0:%.*]] = getelementptr [[UNION_U:%.*]], %union.u* bitcast (%struct.s* @b to %union.u*), i32 0, i32 0
+; CHECK-NEXT:    [[DOT0_VAL:%.*]] = load x86_fp80, x86_fp80* [[DOT0]]
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call x86_fp80 @UseLongDoubleSafely(x86_fp80 [[DOT0_VAL]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @AccessPaddingOfStruct(%struct.Foo* @a)
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @CaptureAStruct(%struct.Foo* @a)
+; CHECK-NEXT:    ret void
+;
 entry:
   tail call i8 @UseLongDoubleUnsafely(%union.u* byval align 16 bitcast (%struct.s* @b to %union.u*))
   tail call x86_fp80 @UseLongDoubleSafely(%union.u* byval align 16 bitcast (%struct.s* @b to %union.u*))
@@ -21,8 +32,15 @@ entry:
   ret void
 }
 
-; CHECK: internal i8 @UseLongDoubleUnsafely(%union.u* byval align 16 %arg) {
 define internal i8 @UseLongDoubleUnsafely(%union.u* byval align 16 %arg) {
+; CHECK-LABEL: define {{[^@]+}}@UseLongDoubleUnsafely
+; CHECK-SAME: (%union.u* byval align 16 [[ARG:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[BITCAST:%.*]] = bitcast %union.u* [[ARG]] to %struct.s*
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds [[STRUCT_S:%.*]], %struct.s* [[BITCAST]], i64 0, i32 2
+; CHECK-NEXT:    [[RESULT:%.*]] = load i8, i8* [[GEP]]
+; CHECK-NEXT:    ret i8 [[RESULT]]
+;
 entry:
   %bitcast = bitcast %union.u* %arg to %struct.s*
   %gep = getelementptr inbounds %struct.s, %struct.s* %bitcast, i64 0, i32 2
@@ -30,22 +48,46 @@ entry:
   ret i8 %result
 }
 
-; CHECK: internal x86_fp80 @UseLongDoubleSafely(x86_fp80 {{%.*}}) {
 define internal x86_fp80 @UseLongDoubleSafely(%union.u* byval align 16 %arg) {
+; CHECK-LABEL: define {{[^@]+}}@UseLongDoubleSafely
+; CHECK-SAME: (x86_fp80 [[ARG_0:%.*]])
+; CHECK-NEXT:    [[ARG:%.*]] = alloca [[UNION_U:%.*]], align 16
+; CHECK-NEXT:    [[DOT0:%.*]] = getelementptr [[UNION_U]], %union.u* [[ARG]], i32 0, i32 0
+; CHECK-NEXT:    store x86_fp80 [[ARG_0]], x86_fp80* [[DOT0]]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds [[UNION_U]], %union.u* [[ARG]], i64 0, i32 0
+; CHECK-NEXT:    [[FP80:%.*]] = load x86_fp80, x86_fp80* [[GEP]]
+; CHECK-NEXT:    ret x86_fp80 [[FP80]]
+;
   %gep = getelementptr inbounds %union.u, %union.u* %arg, i64 0, i32 0
   %fp80 = load x86_fp80, x86_fp80* %gep
   ret x86_fp80 %fp80
 }
 
-; CHECK: define internal i64 @AccessPaddingOfStruct(%struct.Foo* byval %a) {
 define internal i64 @AccessPaddingOfStruct(%struct.Foo* byval %a) {
+; CHECK-LABEL: define {{[^@]+}}@AccessPaddingOfStruct
+; CHECK-SAME: (%struct.Foo* byval [[A:%.*]])
+; CHECK-NEXT:    [[P:%.*]] = bitcast %struct.Foo* [[A]] to i64*
+; CHECK-NEXT:    [[V:%.*]] = load i64, i64* [[P]]
+; CHECK-NEXT:    ret i64 [[V]]
+;
   %p = bitcast %struct.Foo* %a to i64*
   %v = load i64, i64* %p
   ret i64 %v
 }
 
-; CHECK: define internal i64 @CaptureAStruct(%struct.Foo* byval %a) {
 define internal i64 @CaptureAStruct(%struct.Foo* byval %a) {
+; CHECK-LABEL: define {{[^@]+}}@CaptureAStruct
+; CHECK-SAME: (%struct.Foo* byval [[A:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[A_PTR:%.*]] = alloca %struct.Foo*
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[PHI:%.*]] = phi %struct.Foo* [ null, [[ENTRY:%.*]] ], [ [[GEP:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = phi %struct.Foo* [ [[A]], [[ENTRY]] ], [ [[TMP0]], [[LOOP]] ]
+; CHECK-NEXT:    store %struct.Foo* [[PHI]], %struct.Foo** [[A_PTR]]
+; CHECK-NEXT:    [[GEP]] = getelementptr [[STRUCT_FOO:%.*]], %struct.Foo* [[A]], i64 0
+; CHECK-NEXT:    br label [[LOOP]]
+;
 entry:
   %a_ptr = alloca %struct.Foo*
   br label %loop
diff --git a/llvm/test/Transforms/ArgumentPromotion/inalloca.ll b/llvm/test/Transforms/ArgumentPromotion/inalloca.ll
index 02d782b65..ebf3d18f2 100644
--- a/llvm/test/Transforms/ArgumentPromotion/inalloca.ll
+++ b/llvm/test/Transforms/ArgumentPromotion/inalloca.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
 ; RUN: opt %s -globalopt -argpromotion -sroa -S | FileCheck %s
 ; RUN: opt %s -passes='module(globalopt),cgscc(argpromotion),function(sroa)' -S | FileCheck %s
 
@@ -7,6 +8,12 @@ target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:1
 
 ; Argpromote + sroa should change this to passing the two integers by value.
 define internal i32 @f(%struct.ss* inalloca  %s) {
+; CHECK-LABEL: define {{[^@]+}}@f
+; CHECK-SAME: (i32 [[S_0_0_VAL:%.*]], i32 [[S_0_1_VAL:%.*]]) unnamed_addr
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[R:%.*]] = add i32 [[S_0_0_VAL]], [[S_0_1_VAL]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
 entry:
   %f0 = getelementptr %struct.ss, %struct.ss* %s, i32 0, i32 0
   %f1 = getelementptr %struct.ss, %struct.ss* %s, i32 0, i32 1
@@ -15,11 +22,13 @@ entry:
   %r = add i32 %a, %b
   ret i32 %r
 }
-; CHECK-LABEL: define internal fastcc i32 @f
-; CHECK-NOT: load
-; CHECK: ret
 
 define i32 @main() {
+; CHECK-LABEL: define {{[^@]+}}@main() local_unnamed_addr
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[R:%.*]] = call fastcc i32 @f(i32 1, i32 2)
+; CHECK-NEXT:    ret i32 [[R]]
+;
 entry:
   %S = alloca inalloca %struct.ss
   %f0 = getelementptr %struct.ss, %struct.ss* %S, i32 0, i32 0
@@ -29,22 +38,29 @@ entry:
   %r = call i32 @f(%struct.ss* inalloca %S)
   ret i32 %r
 }
-; CHECK-LABEL: define i32 @main
-; CHECK-NOT: load
-; CHECK: ret
 
 ; Argpromote can't promote %a because of the icmp use.
 define internal i1 @g(%struct.ss* %a, %struct.ss* inalloca %b) nounwind  {
-; CHECK: define internal fastcc i1 @g(%struct.ss* %a, %struct.ss* %b)
+; CHECK-LABEL: define {{[^@]+}}@g
+; CHECK-SAME: (%struct.ss* [[A:%.*]], %struct.ss* [[B:%.*]]) unnamed_addr
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[C:%.*]] = icmp eq %struct.ss* [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
 entry:
   %c = icmp eq %struct.ss* %a, %b
   ret i1 %c
 }
 
 define i32 @test() {
+; CHECK-LABEL: define {{[^@]+}}@test() local_unnamed_addr
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[S:%.*]] = alloca inalloca [[STRUCT_SS:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = call fastcc i1 @g(%struct.ss* [[S]], %struct.ss* [[S]])
+; CHECK-NEXT:    ret i32 0
+;
 entry:
   %S = alloca inalloca %struct.ss
   %c = call i1 @g(%struct.ss* %S, %struct.ss* inalloca %S)
-; CHECK: call fastcc i1 @g(%struct.ss* %S, %struct.ss* %S)
   ret i32 0
 }
diff --git a/llvm/test/Transforms/ArgumentPromotion/invalidation.ll b/llvm/test/Transforms/ArgumentPromotion/invalidation.ll
index fd7168e35..d86c93b28 100644
--- a/llvm/test/Transforms/ArgumentPromotion/invalidation.ll
+++ b/llvm/test/Transforms/ArgumentPromotion/invalidation.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
 ; Check that when argument promotion changes a function in some parent node of
 ; the call graph, any analyses that happened to be cached for that function are
 ; actually invalidated. We are using `demanded-bits` here because when printed
@@ -11,37 +12,37 @@
 @G = constant i32 0
 
 define internal i32 @a(i32* %x) {
-; CHECK-LABEL: define internal i32 @a(
-; CHECK-SAME:                         i32 %[[V:.*]]) {
+; CHECK-LABEL: define {{[^@]+}}@a
+; CHECK-SAME: (i32 [[X_VAL:%.*]])
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    ret i32 %[[V]]
-; CHECK-NEXT:  }
+; CHECK-NEXT:    ret i32 [[X_VAL]]
+;
 entry:
   %v = load i32, i32* %x
   ret i32 %v
 }
 
 define i32 @b() {
-; CHECK-LABEL: define i32 @b()
+; CHECK-LABEL: define {{[^@]+}}@b()
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    %[[L:.*]] = load i32, i32* @G
-; CHECK-NEXT:    %[[V:.*]] = call i32 @a(i32 %[[L]])
-; CHECK-NEXT:    ret i32 %[[V]]
-; CHECK-NEXT:  }
+; CHECK-NEXT:    [[G_VAL:%.*]] = load i32, i32* @G
+; CHECK-NEXT:    [[V:%.*]] = call i32 @a(i32 [[G_VAL]])
+; CHECK-NEXT:    ret i32 [[V]]
+;
 entry:
   %v = call i32 @a(i32* @G)
   ret i32 %v
 }
 
 define i32 @c() {
-; CHECK-LABEL: define i32 @c()
+; CHECK-LABEL: define {{[^@]+}}@c()
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    %[[L:.*]] = load i32, i32* @G
-; CHECK-NEXT:    %[[V1:.*]] = call i32 @a(i32 %[[L]])
-; CHECK-NEXT:    %[[V2:.*]] = call i32 @b()
-; CHECK-NEXT:    %[[RESULT:.*]] = add i32 %[[V1]], %[[V2]]
-; CHECK-NEXT:    ret i32 %[[RESULT]]
-; CHECK-NEXT:  }
+; CHECK-NEXT:    [[G_VAL:%.*]] = load i32, i32* @G
+; CHECK-NEXT:    [[V1:%.*]] = call i32 @a(i32 [[G_VAL]])
+; CHECK-NEXT:    [[V2:%.*]] = call i32 @b()
+; CHECK-NEXT:    [[RESULT:%.*]] = add i32 [[V1]], [[V2]]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
 entry:
   %v1 = call i32 @a(i32* @G)
   %v2 = call i32 @b()
diff --git a/llvm/test/Transforms/ArgumentPromotion/musttail.ll b/llvm/test/Transforms/ArgumentPromotion/musttail.ll
index aa1871168..29511d009 100644
--- a/llvm/test/Transforms/ArgumentPromotion/musttail.ll
+++ b/llvm/test/Transforms/ArgumentPromotion/musttail.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
 ; RUN: opt < %s -argpromotion -S | FileCheck %s
 ; PR36543
 
@@ -5,8 +6,16 @@
 
 %T = type { i32, i32, i32, i32 }
 
-; CHECK-LABEL: define internal i32 @test(%T* %p)
 define internal i32 @test(%T* %p) {
+; CHECK-LABEL: define {{[^@]+}}@test
+; CHECK-SAME: (%T* [[P:%.*]])
+; CHECK-NEXT:    [[A_GEP:%.*]] = getelementptr [[T:%.*]], %T* [[P]], i64 0, i32 3
+; CHECK-NEXT:    [[B_GEP:%.*]] = getelementptr [[T]], %T* [[P]], i64 0, i32 2
+; CHECK-NEXT:    [[A:%.*]] = load i32, i32* [[A_GEP]]
+; CHECK-NEXT:    [[B:%.*]] = load i32, i32* [[B_GEP]]
+; CHECK-NEXT:    [[V:%.*]] = add i32 [[A]], [[B]]
+; CHECK-NEXT:    ret i32 [[V]]
+;
   %a.gep = getelementptr %T, %T* %p, i64 0, i32 3
   %b.gep = getelementptr %T, %T* %p, i64 0, i32 2
   %a = load i32, i32* %a.gep
@@ -15,8 +24,12 @@ define internal i32 @test(%T* %p) {
   ret i32 %v
 }
 
-; CHECK-LABEL: define i32 @caller(%T* %p)
 define i32 @caller(%T* %p) {
+; CHECK-LABEL: define {{[^@]+}}@caller
+; CHECK-SAME: (%T* [[P:%.*]])
+; CHECK-NEXT:    [[V:%.*]] = musttail call i32 @test(%T* [[P]])
+; CHECK-NEXT:    ret i32 [[V]]
+;
   %v = musttail call i32 @test(%T* %p)
   ret i32 %v
 }
@@ -24,11 +37,24 @@ define i32 @caller(%T* %p) {
 ; Don't promote arguments of musttail caller
 
 define i32 @foo(%T* %p, i32 %v) {
+; CHECK-LABEL: define {{[^@]+}}@foo
+; CHECK-SAME: (%T* [[P:%.*]], i32 [[V:%.*]])
+; CHECK-NEXT:    ret i32 0
+;
   ret i32 0
 }
 
-; CHECK-LABEL: define internal i32 @test2(%T* %p, i32 %p2)
 define internal i32 @test2(%T* %p, i32 %p2) {
+; CHECK-LABEL: define {{[^@]+}}@test2
+; CHECK-SAME: (%T* [[P:%.*]], i32 [[P2:%.*]])
+; CHECK-NEXT:    [[A_GEP:%.*]] = getelementptr [[T:%.*]], %T* [[P]], i64 0, i32 3
+; CHECK-NEXT:    [[B_GEP:%.*]] = getelementptr [[T]], %T* [[P]], i64 0, i32 2
+; CHECK-NEXT:    [[A:%.*]] = load i32, i32* [[A_GEP]]
+; CHECK-NEXT:    [[B:%.*]] = load i32, i32* [[B_GEP]]
+; CHECK-NEXT:    [[V:%.*]] = add i32 [[A]], [[B]]
+; CHECK-NEXT:    [[CA:%.*]] = musttail call i32 @foo(%T* undef, i32 [[V]])
+; CHECK-NEXT:    ret i32 [[CA]]
+;
   %a.gep = getelementptr %T, %T* %p, i64 0, i32 3
   %b.gep = getelementptr %T, %T* %p, i64 0, i32 2
   %a = load i32, i32* %a.gep
@@ -38,8 +64,12 @@ define internal i32 @test2(%T* %p, i32 %p2) {
   ret i32 %ca
 }
 
-; CHECK-LABEL: define i32 @caller2(%T* %g)
 define i32 @caller2(%T* %g) {
+; CHECK-LABEL: define {{[^@]+}}@caller2
+; CHECK-SAME: (%T* [[G:%.*]])
+; CHECK-NEXT:    [[V:%.*]] = call i32 @test2(%T* [[G]], i32 0)
+; CHECK-NEXT:    ret i32 [[V]]
+;
   %v = call i32 @test2(%T* %g, i32 0)
   ret i32 %v
 }
diff --git a/llvm/test/Transforms/ArgumentPromotion/naked_functions.ll b/llvm/test/Transforms/ArgumentPromotion/naked_functions.ll
index 70a63f4d0..5560bdf56 100644
--- a/llvm/test/Transforms/ArgumentPromotion/naked_functions.ll
+++ b/llvm/test/Transforms/ArgumentPromotion/naked_functions.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
 ; RUN: opt < %s -argpromotion -S | FileCheck %s
 
 ; Don't promote paramaters of/arguments to naked functions
@@ -5,19 +6,29 @@
 @g = common global i32 0, align 4
 
 define i32 @bar() {
+; CHECK-LABEL: define {{[^@]+}}@bar()
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @foo(i32* @g)
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
 entry:
   %call = call i32 @foo(i32* @g)
-; CHECK: %call = call i32 @foo(i32* @g)
   ret i32 %call
 }
 
 define internal i32 @foo(i32*) #0 {
+; CHECK-LABEL: define {{[^@]+}}@foo
+; CHECK-SAME: (i32* [[TMP0:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[RETVAL:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    call void asm sideeffect "ldr r0, [r0] \0Abx lr \0A", ""()
+; CHECK-NEXT:    unreachable
+;
 entry:
   %retval = alloca i32, align 4
   call void asm sideeffect "ldr r0, [r0] \0Abx lr        \0A", ""()
   unreachable
 }
 
-; CHECK: define internal i32 @foo(i32*)
 
 attributes #0 = { naked }
diff --git a/llvm/test/Transforms/ArgumentPromotion/nonzero-address-spaces.ll b/llvm/test/Transforms/ArgumentPromotion/nonzero-address-spaces.ll
index 2ed362b87..083188395 100644
--- a/llvm/test/Transforms/ArgumentPromotion/nonzero-address-spaces.ll
+++ b/llvm/test/Transforms/ArgumentPromotion/nonzero-address-spaces.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
 ; RUN: opt < %s -argpromotion -S | FileCheck %s
 
 ; ArgumentPromotion should preserve the default function address space
@@ -8,14 +9,24 @@ target datalayout = "e-P1-p:16:8-i8:8-i16:8-i32:8-i64:8-f32:8-f64:8-n8-a:8"
 @g = common global i32 0, align 4
 
 define i32 @bar() {
+; CHECK-LABEL: define {{[^@]+}}@bar() addrspace(1)
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CALL:%.*]] = call addrspace(1) i32 @foo()
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+
 entry:
   %call = call i32 @foo(i32* @g)
-; CHECK: %call = call addrspace(1) i32 @foo()
   ret i32 %call
 }
 
-; CHECK: define internal i32 @foo() addrspace(1)
 define internal i32 @foo(i32*) {
+; CHECK-LABEL: define {{[^@]+}}@foo() addrspace(1)
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[RETVAL:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    call addrspace(0) void asm sideeffect "ldr r0, [r0] \0Abx lr \0A", ""()
+; CHECK-NEXT:    unreachable
+;
 entry:
   %retval = alloca i32, align 4
   call void asm sideeffect "ldr r0, [r0] \0Abx lr        \0A", ""()
diff --git a/llvm/test/Transforms/ArgumentPromotion/pr27568.ll b/llvm/test/Transforms/ArgumentPromotion/pr27568.ll
index 711a71fed..1cecbb0ff 100644
--- a/llvm/test/Transforms/ArgumentPromotion/pr27568.ll
+++ b/llvm/test/Transforms/ArgumentPromotion/pr27568.ll
@@ -1,18 +1,35 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
 ; RUN: opt -S -argpromotion < %s | FileCheck %s
 ; RUN: opt -S -passes=argpromotion < %s | FileCheck %s
 ; RUN: opt -S -debugify -o /dev/null < %s
 target triple = "x86_64-pc-windows-msvc"
 
 define internal void @callee(i8*) {
+; CHECK-LABEL: define {{[^@]+}}@callee()
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    call void @thunk()
+; CHECK-NEXT:    ret void
+;
 entry:
   call void @thunk()
   ret void
 }
 
 define void @test1() personality i32 (...)* @__CxxFrameHandler3 {
+; CHECK-LABEL: define {{[^@]+}}@test1() personality i32 (...)* @__CxxFrameHandler3
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    invoke void @thunk()
+; CHECK-NEXT:    to label [[OUT:%.*]] unwind label [[CPAD:%.*]]
+; CHECK:       out:
+; CHECK-NEXT:    ret void
+; CHECK:       cpad:
+; CHECK-NEXT:    [[PAD:%.*]] = cleanuppad within none []
+; CHECK-NEXT:    call void @callee() [ "funclet"(token [[PAD]]) ]
+; CHECK-NEXT:    cleanupret from [[PAD]] unwind to caller
+;
 entry:
   invoke void @thunk()
-          to label %out unwind label %cpad
+  to label %out unwind label %cpad
 
 out:
   ret void
@@ -23,10 +40,6 @@ cpad:
   cleanupret from %pad unwind to caller
 }
 
-; CHECK-LABEL: define void @test1(
-; CHECK:      %[[pad:.*]] = cleanuppad within none []
-; CHECK-NEXT: call void @callee() [ "funclet"(token %[[pad]]) ]
-; CHECK-NEXT: cleanupret from %[[pad]] unwind to caller
 
 declare void @thunk()
 
diff --git a/llvm/test/Transforms/ArgumentPromotion/pr3085.ll b/llvm/test/Transforms/ArgumentPromotion/pr3085.ll
index 3048c603d..d7eefcf7f 100644
--- a/llvm/test/Transforms/ArgumentPromotion/pr3085.ll
+++ b/llvm/test/Transforms/ArgumentPromotion/pr3085.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
 ; RUN: opt < %s -disable-output -loop-extract-single -loop-rotate -loop-reduce -argpromotion
 ; PR 3085
 
@@ -5,1940 +6,1940 @@
 
 define fastcc %struct.Lit* @import_lit(i32 %lit) nounwind {
 entry:
-	br i1 false, label %bb, label %bb1
+  br i1 false, label %bb, label %bb1
 
 bb:		; preds = %entry
-	unreachable
+  unreachable
 
 bb1:		; preds = %entry
-	br label %bb3
+  br label %bb3
 
 bb2:		; preds = %bb3
-	br label %bb3
+  br label %bb3
 
 bb3:		; preds = %bb2, %bb1
-	br i1 false, label %bb2, label %bb6
+  br i1 false, label %bb2, label %bb6
 
 bb6:		; preds = %bb3
-	br i1 false, label %bb.i.i, label %bb1.i.i
+  br i1 false, label %bb.i.i, label %bb1.i.i
 
 bb.i.i:		; preds = %bb6
-	br label %int2lit.exit
+  br label %int2lit.exit
 
 bb1.i.i:		; preds = %bb6
-	br label %int2lit.exit
+  br label %int2lit.exit
 
 int2lit.exit:		; preds = %bb1.i.i, %bb.i.i
-	ret %struct.Lit* null
+  ret %struct.Lit* null
 }
 
 define fastcc i32 @picosat_main(i32 %argc, i8** %argv) nounwind {
 entry:
-	br i1 false, label %bb.i, label %picosat_time_stamp.exit
+  br i1 false, label %bb.i, label %picosat_time_stamp.exit
 
 bb.i:		; preds = %entry
-	br label %picosat_time_stamp.exit
+  br label %picosat_time_stamp.exit
 
 picosat_time_stamp.exit:		; preds = %bb.i, %entry
-	br label %bb108
+  br label %bb108
 
 bb:		; preds = %bb108
-	br i1 false, label %bb1, label %bb2
+  br i1 false, label %bb1, label %bb2
 
 bb1:		; preds = %bb
-	br label %bb106
+  br label %bb106
 
 bb2:		; preds = %bb
-	br i1 false, label %bb3, label %bb4
+  br i1 false, label %bb3, label %bb4
 
 bb3:		; preds = %bb2
-	br label %bb106
+  br label %bb106
 
 bb4:		; preds = %bb2
-	br i1 false, label %bb5, label %bb6
+  br i1 false, label %bb5, label %bb6
 
 bb5:		; preds = %bb4
-	br label %bb106
+  br label %bb106
 
 bb6:		; preds = %bb4
-	br i1 false, label %bb7, label %bb8
+  br i1 false, label %bb7, label %bb8
 
 bb7:		; preds = %bb6
-	br label %bb106
+  br label %bb106
 
 bb8:		; preds = %bb6
-	br i1 false, label %bb106, label %bb10
+  br i1 false, label %bb106, label %bb10
 
 bb10:		; preds = %bb8
-	br i1 false, label %bb106, label %bb12
+  br i1 false, label %bb106, label %bb12
 
 bb12:		; preds = %bb10
-	br i1 false, label %bb106, label %bb14
+  br i1 false, label %bb106, label %bb14
 
 bb14:		; preds = %bb12
-	br i1 false, label %bb15, label %bb19
+  br i1 false, label %bb15, label %bb19
 
 bb15:		; preds = %bb14
-	br i1 false, label %bb16, label %bb17
+  br i1 false, label %bb16, label %bb17
 
 bb16:		; preds = %bb15
-	br label %bb106
+  br label %bb106
 
 bb17:		; preds = %bb15
-	br label %bb106
+  br label %bb106
 
 bb19:		; preds = %bb14
-	br i1 false, label %bb20, label %bb28
+  br i1 false, label %bb20, label %bb28
 
 bb20:		; preds = %bb19
-	br i1 false, label %bb21, label %bb22
+  br i1 false, label %bb21, label %bb22
 
 bb21:		; preds = %bb20
-	br label %bb106
+  br label %bb106
 
 bb22:		; preds = %bb20
-	br i1 false, label %bb106, label %bb24
+  br i1 false, label %bb106, label %bb24
 
 bb24:		; preds = %bb22
-	br i1 false, label %bb106, label %bb26
+  br i1 false, label %bb106, label %bb26
 
 bb26:		; preds = %bb24
-	br label %bb106
+  br label %bb106
 
 bb28:		; preds = %bb19
-	br i1 false, label %bb29, label %bb35
+  br i1 false, label %bb29, label %bb35
 
 bb29:		; preds = %bb28
-	br i1 false, label %bb30, label %bb31
+  br i1 false, label %bb30, label %bb31
 
 bb30:		; preds = %bb29
-	br label %bb106
+  br label %bb106
 
 bb31:		; preds = %bb29
-	br i1 false, label %bb32, label %bb33
+  br i1 false, label %bb32, label %bb33
 
 bb32:		; preds = %bb31
-	br label %bb106
+  br label %bb106
 
 bb33:		; preds = %bb31
-	br label %bb106
+  br label %bb106
 
 bb35:		; preds = %bb28
-	br i1 false, label %bb36, label %bb40
+  br i1 false, label %bb36, label %bb40
 
 bb36:		; preds = %bb35
-	br i1 false, label %bb37, label %bb38
+  br i1 false, label %bb37, label %bb38
 
 bb37:		; preds = %bb36
-	br label %bb106
+  br label %bb106
 
 bb38:		; preds = %bb36
-	br label %bb106
+  br label %bb106
 
 bb40:		; preds = %bb35
-	br i1 false, label %bb41, label %bb49
+  br i1 false, label %bb41, label %bb49
 
 bb41:		; preds = %bb40
-	br i1 false, label %bb43, label %bb42
+  br i1 false, label %bb43, label %bb42
 
 bb42:		; preds = %bb41
-	br label %bb106
+  br label %bb106
 
 bb43:		; preds = %bb41
-	br i1 false, label %bb44, label %bb45
+  br i1 false, label %bb44, label %bb45
 
 bb44:		; preds = %bb43
-	br label %bb106
+  br label %bb106
 
 bb45:		; preds = %bb43
-	br i1 false, label %bb46, label %bb47
+  br i1 false, label %bb46, label %bb47
 
 bb46:		; preds = %bb45
-	br label %bb106
+  br label %bb106
 
 bb47:		; preds = %bb45
-	br label %bb106
+  br label %bb106
 
 bb49:		; preds = %bb40
-	br i1 false, label %bb50, label %bb56
+  br i1 false, label %bb50, label %bb56
 
 bb50:		; preds = %bb49
-	br i1 false, label %bb52, label %bb51
+  br i1 false, label %bb52, label %bb51
 
 bb51:		; preds = %bb50
-	br label %bb106
+  br label %bb106
 
 bb52:		; preds = %bb50
-	br i1 false, label %bb53, label %bb54
+  br i1 false, label %bb53, label %bb54
 
 bb53:		; preds = %bb52
-	br label %bb106
+  br label %bb106
 
 bb54:		; preds = %bb52
-	br label %bb106
+  br label %bb106
 
 bb56:		; preds = %bb49
-	br i1 false, label %bb57, label %bb63
+  br i1 false, label %bb57, label %bb63
 
 bb57:		; preds = %bb56
-	br i1 false, label %bb59, label %bb58
+  br i1 false, label %bb59, label %bb58
 
 bb58:		; preds = %bb57
-	br label %bb106
+  br label %bb106
 
 bb59:		; preds = %bb57
-	br i1 false, label %bb60, label %bb61
+  br i1 false, label %bb60, label %bb61
 
 bb60:		; preds = %bb59
-	br label %bb106
+  br label %bb106
 
 bb61:		; preds = %bb59
-	br label %bb106
+  br label %bb106
 
 bb63:		; preds = %bb56
-	br i1 false, label %bb64, label %bb70
+  br i1 false, label %bb64, label %bb70
 
 bb64:		; preds = %bb63
-	br i1 false, label %bb66, label %bb65
+  br i1 false, label %bb66, label %bb65
 
 bb65:		; preds = %bb64
-	br label %bb106
+  br label %bb106
 
 bb66:		; preds = %bb64
-	br i1 false, label %bb67, label %bb68
+  br i1 false, label %bb67, label %bb68
 
 bb67:		; preds = %bb66
-	br label %bb106
+  br label %bb106
 
 bb68:		; preds = %bb66
-	br label %bb106
+  br label %bb106
 
 bb70:		; preds = %bb63
-	br i1 false, label %bb71, label %bb79
+  br i1 false, label %bb71, label %bb79
 
 bb71:		; preds = %bb70
-	br i1 false, label %bb73, label %bb72
+  br i1 false, label %bb73, label %bb72
 
 bb72:		; preds = %bb71
-	br label %bb106
+  br label %bb106
 
 bb73:		; preds = %bb71
-	br i1 false, label %bb74, label %bb75
+  br i1 false, label %bb74, label %bb75
 
 bb74:		; preds = %bb73
-	br label %bb106
+  br label %bb106
 
 bb75:		; preds = %bb73
-	br i1 false, label %bb76, label %bb77
+  br i1 false, label %bb76, label %bb77
 
 bb76:		; preds = %bb75
-	br label %bb106
+  br label %bb106
 
 bb77:		; preds = %bb75
-	br label %bb106
+  br label %bb106
 
 bb79:		; preds = %bb70
-	br i1 false, label %bb80, label %bb86
+  br i1 false, label %bb80, label %bb86
 
 bb80:		; preds = %bb79
-	br i1 false, label %bb82, label %bb81
+  br i1 false, label %bb82, label %bb81
 
 bb81:		; preds = %bb80
-	br label %bb106
+  br label %bb106
 
 bb82:		; preds = %bb80
-	br i1 false, label %bb83, label %bb84
+  br i1 false, label %bb83, label %bb84
 
 bb83:		; preds = %bb82
-	br label %bb106
+  br label %bb106
 
 bb84:		; preds = %bb82
-	br label %bb106
+  br label %bb106
 
 bb86:		; preds = %bb79
-	br i1 false, label %bb87, label %bb93
+  br i1 false, label %bb87, label %bb93
 
 bb87:		; preds = %bb86
-	br i1 false, label %bb89, label %bb88
+  br i1 false, label %bb89, label %bb88
 
 bb88:		; preds = %bb87
-	br label %bb106
+  br label %bb106
 
 bb89:		; preds = %bb87
-	br i1 false, label %bb90, label %bb91
+  br i1 false, label %bb90, label %bb91
 
 bb90:		; preds = %bb89
-	br label %bb106
+  br label %bb106
 
 bb91:		; preds = %bb89
-	br label %bb106
+  br label %bb106
 
 bb93:		; preds = %bb86
-	br i1 false, label %bb94, label %bb95
+  br i1 false, label %bb94, label %bb95
 
 bb94:		; preds = %bb93
-	br label %bb106
+  br label %bb106
 
 bb95:		; preds = %bb93
-	br i1 false, label %bb98, label %bb97
+  br i1 false, label %bb98, label %bb97
 
 bb97:		; preds = %bb95
-	br label %bb106
+  br label %bb106
 
 bb98:		; preds = %bb95
-	br i1 false, label %bb103, label %bb1.i24
+  br i1 false, label %bb103, label %bb1.i24
 
 bb1.i24:		; preds = %bb98
-	br i1 false, label %bb99, label %bb103
+  br i1 false, label %bb99, label %bb103
 
 bb99:		; preds = %bb1.i24
-	br i1 false, label %bb101, label %bb100
+  br i1 false, label %bb101, label %bb100
 
 bb100:		; preds = %bb99
-	br label %bb102
+  br label %bb102
 
 bb101:		; preds = %bb99
-	br label %bb102
+  br label %bb102
 
 bb102:		; preds = %bb101, %bb100
-	br label %bb106
+  br label %bb106
 
 bb103:		; preds = %bb1.i24, %bb98
-	br i1 false, label %bb104, label %bb105
+  br i1 false, label %bb104, label %bb105
 
 bb104:		; preds = %bb103
-	br label %bb106
+  br label %bb106
 
 bb105:		; preds = %bb103
-	br label %bb106
+  br label %bb106
 
 bb106:		; preds = %bb105, %bb104, %bb102, %bb97, %bb94, %bb91, %bb90, %bb88, %bb84, %bb83, %bb81, %bb77, %bb76, %bb74, %bb72, %bb68, %bb67, %bb65, %bb61, %bb60, %bb58, %bb54, %bb53, %bb51, %bb47, %bb46, %bb44, %bb42, %bb38, %bb37, %bb33, %bb32, %bb30, %bb26, %bb24, %bb22, %bb21, %bb17, %bb16, %bb12, %bb10, %bb8, %bb7, %bb5, %bb3, %bb1
-	br i1 false, label %bb108, label %bb110
+  br i1 false, label %bb108, label %bb110
 
 bb108:		; preds = %bb106, %picosat_time_stamp.exit
-	br i1 false, label %bb, label %bb110
+  br i1 false, label %bb, label %bb110
 
 bb110:		; preds = %bb108, %bb106
-	br i1 false, label %bb112, label %bb171
+  br i1 false, label %bb112, label %bb171
 
 bb112:		; preds = %bb110
-	br i1 false, label %bb114, label %bb113
+  br i1 false, label %bb114, label %bb113
 
 bb113:		; preds = %bb112
-	br label %bb114
+  br label %bb114
 
 bb114:		; preds = %bb113, %bb112
-	br i1 false, label %bb.i.i35, label %bb1.i.i36
+  br i1 false, label %bb.i.i35, label %bb1.i.i36
 
 bb.i.i35:		; preds = %bb114
-	unreachable
+  unreachable
 
 bb1.i.i36:		; preds = %bb114
-	br i1 false, label %bb5.i.i.i41, label %bb6.i.i.i42
+  br i1 false, label %bb5.i.i.i41, label %bb6.i.i.i42
 
 bb5.i.i.i41:		; preds = %bb1.i.i36
-	unreachable
+  unreachable
 
 bb6.i.i.i42:		; preds = %bb1.i.i36
-	br i1 false, label %bb7.i.i.i43, label %bb8.i.i.i44
+  br i1 false, label %bb7.i.i.i43, label %bb8.i.i.i44
 
 bb7.i.i.i43:		; preds = %bb6.i.i.i42
-	br label %bb8.i.i.i44
+  br label %bb8.i.i.i44
 
 bb8.i.i.i44:		; preds = %bb7.i.i.i43, %bb6.i.i.i42
-	br i1 false, label %picosat_init.exit, label %bb14.i.i
+  br i1 false, label %picosat_init.exit, label %bb14.i.i
 
 bb14.i.i:		; preds = %bb8.i.i.i44
-	br label %picosat_init.exit
+  br label %picosat_init.exit
 
 picosat_init.exit:		; preds = %bb14.i.i, %bb8.i.i.i44
-	br i1 false, label %bb116, label %bb115
+  br i1 false, label %bb116, label %bb115
 
 bb115:		; preds = %picosat_init.exit
-	br label %bb116
+  br label %bb116
 
 bb116:		; preds = %bb115, %picosat_init.exit
-	br i1 false, label %bb119, label %bb118
+  br i1 false, label %bb119, label %bb118
 
 bb118:		; preds = %bb116
-	br label %bb119
+  br label %bb119
 
 bb119:		; preds = %bb118, %bb116
-	br i1 false, label %bb121, label %bb120
+  br i1 false, label %bb121, label %bb120
 
 bb120:		; preds = %bb119
-	br label %bb121
+  br label %bb121
 
 bb121:		; preds = %bb120, %bb119
-	br i1 false, label %bb126, label %bb122
+  br i1 false, label %bb126, label %bb122
 
 bb122:		; preds = %bb121
-	br label %bb126
+  br label %bb126
 
 bb126:		; preds = %bb122, %bb121
-	br i1 false, label %bb128, label %bb127
+  br i1 false, label %bb128, label %bb127
 
 bb127:		; preds = %bb126
-	br label %bb128
+  br label %bb128
 
 bb128:		; preds = %bb127, %bb126
-	br label %SKIP_COMMENTS.i
+  br label %SKIP_COMMENTS.i
 
 SKIP_COMMENTS.i.loopexit:		; preds = %bb.i149, %bb.i149
-	br label %SKIP_COMMENTS.i.backedge
+  br label %SKIP_COMMENTS.i.backedge
 
 SKIP_COMMENTS.i:		; preds = %SKIP_COMMENTS.i.backedge, %bb128
-	br i1 false, label %bb.i149.preheader, label %bb3.i152
+  br i1 false, label %bb.i149.preheader, label %bb3.i152
 
 bb.i149.preheader:		; preds = %SKIP_COMMENTS.i
-	br label %bb.i149
+  br label %bb.i149
 
 bb.i149:		; preds = %bb.i149, %bb.i149.preheader
-	switch i32 0, label %bb.i149 [
-		i32 -1, label %SKIP_COMMENTS.i.loopexit
-		i32 10, label %SKIP_COMMENTS.i.loopexit
-	]
+  switch i32 0, label %bb.i149 [
+  i32 -1, label %SKIP_COMMENTS.i.loopexit
+  i32 10, label %SKIP_COMMENTS.i.loopexit
+  ]
 
 bb3.i152:		; preds = %SKIP_COMMENTS.i
-	br i1 false, label %bb4.i153, label %SKIP_COMMENTS.i.backedge
+  br i1 false, label %bb4.i153, label %SKIP_COMMENTS.i.backedge
 
 SKIP_COMMENTS.i.backedge:		; preds = %bb3.i152, %SKIP_COMMENTS.i.loopexit
-	br label %SKIP_COMMENTS.i
+  br label %SKIP_COMMENTS.i
 
 bb4.i153:		; preds = %bb3.i152
-	br i1 false, label %bb5.i154, label %bb129
+  br i1 false, label %bb5.i154, label %bb129
 
 bb5.i154:		; preds = %bb4.i153
-	br i1 false, label %bb129, label %bb6.i155.preheader
+  br i1 false, label %bb129, label %bb6.i155.preheader
 
 bb6.i155.preheader:		; preds = %bb5.i154
-	br label %bb6.i155
+  br label %bb6.i155
 
 bb6.i155:		; preds = %bb6.i155, %bb6.i155.preheader
-	br i1 false, label %bb7.i156, label %bb6.i155
+  br i1 false, label %bb7.i156, label %bb6.i155
 
 bb7.i156:		; preds = %bb6.i155
-	br i1 false, label %bb8.i157, label %bb129
+  br i1 false, label %bb8.i157, label %bb129
 
 bb8.i157:		; preds = %bb7.i156
-	br i1 false, label %bb9.i158, label %bb129
+  br i1 false, label %bb9.i158, label %bb129
 
 bb9.i158:		; preds = %bb8.i157
-	br i1 false, label %bb10.i159, label %bb129
+  br i1 false, label %bb10.i159, label %bb129
 
 bb10.i159:		; preds = %bb9.i158
-	br i1 false, label %bb129, label %bb11.i160.preheader
+  br i1 false, label %bb129, label %bb11.i160.preheader
 
 bb11.i160.preheader:		; preds = %bb10.i159
-	br label %bb11.i160
+  br label %bb11.i160
 
 bb11.i160:		; preds = %bb11.i160, %bb11.i160.preheader
-	br i1 false, label %bb12.i161, label %bb11.i160
+  br i1 false, label %bb12.i161, label %bb11.i160
 
 bb12.i161:		; preds = %bb11.i160
-	br i1 false, label %bb129, label %bb15.i165.preheader
+  br i1 false, label %bb129, label %bb15.i165.preheader
 
 bb15.i165.preheader:		; preds = %bb12.i161
-	br label %bb15.i165
+  br label %bb15.i165
 
 bb14.i163:		; preds = %bb15.i165
-	br label %bb15.i165
+  br label %bb15.i165
 
 bb15.i165:		; preds = %bb14.i163, %bb15.i165.preheader
-	br i1 false, label %bb16.i166, label %bb14.i163
+  br i1 false, label %bb16.i166, label %bb14.i163
 
 bb16.i166:		; preds = %bb15.i165
-	br i1 false, label %bb129, label %bb17.i167.preheader
+  br i1 false, label %bb129, label %bb17.i167.preheader
 
 bb17.i167.preheader:		; preds = %bb16.i166
-	br label %bb17.i167
+  br label %bb17.i167
 
 bb17.i167:		; preds = %bb17.i167, %bb17.i167.preheader
-	br i1 false, label %bb18.i168, label %bb17.i167
+  br i1 false, label %bb18.i168, label %bb17.i167
 
 bb18.i168:		; preds = %bb17.i167
-	br i1 false, label %bb129, label %bb21.i172.preheader
+  br i1 false, label %bb129, label %bb21.i172.preheader
 
 bb21.i172.preheader:		; preds = %bb18.i168
-	br label %bb21.i172
+  br label %bb21.i172
 
 bb20.i170:		; preds = %bb21.i172
-	br label %bb21.i172
+  br label %bb21.i172
 
 bb21.i172:		; preds = %bb20.i170, %bb21.i172.preheader
-	br i1 false, label %bb22.i173, label %bb20.i170
+  br i1 false, label %bb22.i173, label %bb20.i170
 
 bb22.i173:		; preds = %bb21.i172
-	br i1 false, label %bb24.i175, label %bb129
+  br i1 false, label %bb24.i175, label %bb129
 
 bb24.i175:		; preds = %bb22.i173
-	br i1 false, label %bb26.i180, label %bb25.i176
+  br i1 false, label %bb26.i180, label %bb25.i176
 
 bb25.i176:		; preds = %bb24.i175
-	br label %bb26.i180
+  br label %bb26.i180
 
 bb26.i180:		; preds = %bb25.i176, %bb24.i175
-	br i1 false, label %bb.i.i181, label %bb3.i.i184.preheader
+  br i1 false, label %bb.i.i181, label %bb3.i.i184.preheader
 
 bb.i.i181:		; preds = %bb26.i180
-	br label %bb3.i.i184.preheader
+  br label %bb3.i.i184.preheader
 
 bb3.i.i184.preheader:		; preds = %bb.i.i181, %bb26.i180
-	br label %bb3.i.i184
+  br label %bb3.i.i184
 
 bb2.i.i183:		; preds = %bb3.i.i184
-	br label %bb3.i.i184
+  br label %bb3.i.i184
 
 bb3.i.i184:		; preds = %bb2.i.i183, %bb3.i.i184.preheader
-	br i1 false, label %bb2.i.i183, label %bb4.i.i185
+  br i1 false, label %bb2.i.i183, label %bb4.i.i185
 
 bb4.i.i185:		; preds = %bb3.i.i184
-	br i1 false, label %bb.i.i.i186, label %picosat_adjust.exit.i
+  br i1 false, label %bb.i.i.i186, label %picosat_adjust.exit.i
 
 bb.i.i.i186:		; preds = %bb4.i.i185
-	br label %picosat_adjust.exit.i
+  br label %picosat_adjust.exit.i
 
 picosat_adjust.exit.i:		; preds = %bb.i.i.i186, %bb4.i.i185
-	br i1 false, label %bb28.i188, label %bb27.i187
+  br i1 false, label %bb28.i188, label %bb27.i187
 
 bb27.i187:		; preds = %picosat_adjust.exit.i
-	br label %bb28.i188
+  br label %bb28.i188
 
 bb28.i188:		; preds = %bb27.i187, %picosat_adjust.exit.i
-	br label %READ_LITERAL.i.outer
+  br label %READ_LITERAL.i.outer
 
 READ_LITERAL.i.outer:		; preds = %READ_LITERAL.i.outer.backedge, %bb28.i188
-	br label %READ_LITERAL.i
+  br label %READ_LITERAL.i
 
 READ_LITERAL.i.loopexit:		; preds = %bb29.i189, %bb29.i189
-	br label %READ_LITERAL.i.backedge
+  br label %READ_LITERAL.i.backedge
 
 READ_LITERAL.i:		; preds = %READ_LITERAL.i.backedge, %READ_LITERAL.i.outer
-	switch i32 0, label %bb39.i199 [
-		i32 99, label %bb29.i189.preheader
-		i32 -1, label %bb33.i193
-	]
+  switch i32 0, label %bb39.i199 [
+  i32 99, label %bb29.i189.preheader
+  i32 -1, label %bb33.i193
+  ]
 
 bb29.i189.preheader:		; preds = %READ_LITERAL.i
-	br label %bb29.i189
+  br label %bb29.i189
 
 bb29.i189:		; preds = %bb29.i189, %bb29.i189.preheader
-	switch i32 0, label %bb29.i189 [
-		i32 -1, label %READ_LITERAL.i.loopexit
-		i32 10, label %READ_LITERAL.i.loopexit
-	]
+  switch i32 0, label %bb29.i189 [
+  i32 -1, label %READ_LITERAL.i.loopexit
+  i32 10, label %READ_LITERAL.i.loopexit
+  ]
 
 bb33.i193:		; preds = %READ_LITERAL.i
-	br i1 false, label %bb35.i195, label %parse.exit
+  br i1 false, label %bb35.i195, label %parse.exit
 
 bb35.i195:		; preds = %bb33.i193
-	br i1 false, label %bb38.i198, label %parse.exit
+  br i1 false, label %bb38.i198, label %parse.exit
 
 bb38.i198:		; preds = %bb35.i195
-	br label %parse.exit
+  br label %parse.exit
 
 bb39.i199:		; preds = %READ_LITERAL.i
-	br i1 false, label %bb40.i200, label %READ_LITERAL.i.backedge
+  br i1 false, label %bb40.i200, label %READ_LITERAL.i.backedge
 
 READ_LITERAL.i.backedge:		; preds = %bb39.i199, %READ_LITERAL.i.loopexit
-	br label %READ_LITERAL.i
+  br label %READ_LITERAL.i
 
 bb40.i200:		; preds = %bb39.i199
-	br i1 false, label %bb41.i201, label %bb42.i202
+  br i1 false, label %bb41.i201, label %bb42.i202
 
 bb41.i201:		; preds = %bb40.i200
-	br label %bb42.i202
+  br label %bb42.i202
 
 bb42.i202:		; preds = %bb41.i201, %bb40.i200
-	br i1 false, label %parse.exit.loopexit, label %bb46.i.preheader
+  br i1 false, label %parse.exit.loopexit, label %bb46.i.preheader
 
 bb46.i.preheader:		; preds = %bb42.i202
-	br label %bb46.i
+  br label %bb46.i
 
 bb45.i:		; preds = %bb46.i
-	br label %bb46.i
+  br label %bb46.i
 
 bb46.i:		; preds = %bb45.i, %bb46.i.preheader
-	br i1 false, label %bb47.i, label %bb45.i
+  br i1 false, label %bb47.i, label %bb45.i
 
 bb47.i:		; preds = %bb46.i
-	br i1 false, label %parse.exit.loopexit, label %bb50.i
+  br i1 false, label %parse.exit.loopexit, label %bb50.i
 
 bb50.i:		; preds = %bb47.i
-	br i1 false, label %bb55.i, label %bb51.i
+  br i1 false, label %bb55.i, label %bb51.i
 
 bb51.i:		; preds = %bb50.i
-	br i1 false, label %parse.exit.loopexit, label %bb54.i
+  br i1 false, label %parse.exit.loopexit, label %bb54.i
 
 bb54.i:		; preds = %bb51.i
-	br label %bb56.i
+  br label %bb56.i
 
 bb55.i:		; preds = %bb50.i
-	br label %bb56.i
+  br label %bb56.i
 
 bb56.i:		; preds = %bb55.i, %bb54.i
-	br i1 false, label %bb3.i11.i, label %bb.i8.i
+  br i1 false, label %bb3.i11.i, label %bb.i8.i
 
 bb.i8.i:		; preds = %bb56.i
-	br i1 false, label %bb1.i9.i, label %bb3.i11.i
+  br i1 false, label %bb1.i9.i, label %bb3.i11.i
 
 bb1.i9.i:		; preds = %bb.i8.i
-	br i1 false, label %bb3.i11.i, label %bb2.i10.i
+  br i1 false, label %bb3.i11.i, label %bb2.i10.i
 
 bb2.i10.i:		; preds = %bb1.i9.i
-	unreachable
+  unreachable
 
 bb3.i11.i:		; preds = %bb1.i9.i, %bb.i8.i, %bb56.i
-	br i1 false, label %bb7.i.i208, label %bb6.i.i207
+  br i1 false, label %bb7.i.i208, label %bb6.i.i207
 
 bb6.i.i207:		; preds = %bb3.i11.i
-	br label %READ_LITERAL.i.outer.backedge
+  br label %READ_LITERAL.i.outer.backedge
 
 bb7.i.i208:		; preds = %bb3.i11.i
-	br i1 false, label %bb53.i.i.i.i.preheader, label %bb.i.i.i.i210.preheader
+  br i1 false, label %bb53.i.i.i.i.preheader, label %bb.i.i.i.i210.preheader
 
 bb.i.i.i.i210.preheader:		; preds = %bb7.i.i208
-	br label %bb.i.i.i.i210
+  br label %bb.i.i.i.i210
 
 bb.i.i.i.i210:		; preds = %bb.i.i.i.i210.backedge, %bb.i.i.i.i210.preheader
-	br i1 false, label %bb17.i.i.i.i, label %bb18.i.i.i.i
+  br i1 false, label %bb17.i.i.i.i, label %bb18.i.i.i.i
 
 bb17.i.i.i.i:		; preds = %bb.i.i.i.i210
-	br label %bb18.i.i.i.i
+  br label %bb18.i.i.i.i
 
 bb18.i.i.i.i:		; preds = %bb17.i.i.i.i, %bb.i.i.i.i210
-	br i1 false, label %bb19.i.i.i.i, label %bb20.i.i.i.i
+  br i1 false, label %bb19.i.i.i.i, label %bb20.i.i.i.i
 
 bb19.i.i.i.i:		; preds = %bb18.i.i.i.i
-	br label %bb20.i.i.i.i
+  br label %bb20.i.i.i.i
 
 bb20.i.i.i.i:		; preds = %bb19.i.i.i.i, %bb18.i.i.i.i
-	br i1 false, label %bb21.i.i.i.i, label %bb22.i.i.i.i
+  br i1 false, label %bb21.i.i.i.i, label %bb22.i.i.i.i
 
 bb21.i.i.i.i:		; preds = %bb20.i.i.i.i
-	br label %bb22.i.i.i.i
+  br label %bb22.i.i.i.i
 
 bb22.i.i.i.i:		; preds = %bb21.i.i.i.i, %bb20.i.i.i.i
-	br label %bb23.i.i.i.i.outer
+  br label %bb23.i.i.i.i.outer
 
 bb23.i.i.i.i.outer:		; preds = %bb28.i.i.i.i, %bb22.i.i.i.i
-	br label %bb23.i.i.i.i
+  br label %bb23.i.i.i.i
 
 bb23.i.i.i.i:		; preds = %bb23.i.i.i.i, %bb23.i.i.i.i.outer
-	br i1 false, label %bb23.i.i.i.i, label %bb26.i.i.i.i.preheader
+  br i1 false, label %bb23.i.i.i.i, label %bb26.i.i.i.i.preheader
 
 bb26.i.i.i.i.preheader:		; preds = %bb23.i.i.i.i
-	br label %bb26.i.i.i.i
+  br label %bb26.i.i.i.i
 
 bb26.i.i.i.i:		; preds = %bb26.i.i.i.i, %bb26.i.i.i.i.preheader
-	br i1 false, label %bb27.i.i.i.i, label %bb26.i.i.i.i
+  br i1 false, label %bb27.i.i.i.i, label %bb26.i.i.i.i
 
 bb27.i.i.i.i:		; preds = %bb26.i.i.i.i
-	br i1 false, label %bb28.i.i.i.i, label %bb29.i.i.i.i
+  br i1 false, label %bb28.i.i.i.i, label %bb29.i.i.i.i
 
 bb28.i.i.i.i:		; preds = %bb27.i.i.i.i
-	br label %bb23.i.i.i.i.outer
+  br label %bb23.i.i.i.i.outer
 
 bb29.i.i.i.i:		; preds = %bb27.i.i.i.i
-	br i1 false, label %bb33.i.i.i.i, label %bb44.i.i.i.i
+  br i1 false, label %bb33.i.i.i.i, label %bb44.i.i.i.i
 
 bb33.i.i.i.i:		; preds = %bb29.i.i.i.i
-	br i1 false, label %bb34.i.i.i.i, label %bb38.i.i.i.i
+  br i1 false, label %bb34.i.i.i.i, label %bb38.i.i.i.i
 
 bb34.i.i.i.i:		; preds = %bb33.i.i.i.i
-	br i1 false, label %bb37.i.i.i.i, label %bb35.i.i.i.i
+  br i1 false, label %bb37.i.i.i.i, label %bb35.i.i.i.i
 
 bb35.i.i.i.i:		; preds = %bb34.i.i.i.i
-	br label %bb37.i.i.i.i
+  br label %bb37.i.i.i.i
 
 bb37.i.i.i.i:		; preds = %bb35.i.i.i.i, %bb34.i.i.i.i
-	br label %bb38.i.i.i.i
+  br label %bb38.i.i.i.i
 
 bb38.i.i.i.i:		; preds = %bb37.i.i.i.i, %bb33.i.i.i.i
-	br i1 false, label %bb39.i.i.i.i, label %bb43.i.i.i.i
+  br i1 false, label %bb39.i.i.i.i, label %bb43.i.i.i.i
 
 bb39.i.i.i.i:		; preds = %bb38.i.i.i.i
-	br i1 false, label %bb42.i.i.i.i, label %bb40.i.i.i.i
+  br i1 false, label %bb42.i.i.i.i, label %bb40.i.i.i.i
 
 bb40.i.i.i.i:		; preds = %bb39.i.i.i.i
-	br label %bb42.i.i.i.i
+  br label %bb42.i.i.i.i
 
 bb42.i.i.i.i:		; preds = %bb40.i.i.i.i, %bb39.i.i.i.i
-	br label %bb43.i.i.i.i
+  br label %bb43.i.i.i.i
 
 bb43.i.i.i.i:		; preds = %bb42.i.i.i.i, %bb38.i.i.i.i
-	br label %bb.i.i.i.i210.backedge
+  br label %bb.i.i.i.i210.backedge
 
 bb.i.i.i.i210.backedge:		; preds = %bb47.i.i.i.i, %bb44.i.i.i.i, %bb43.i.i.i.i
-	br label %bb.i.i.i.i210
+  br label %bb.i.i.i.i210
 
 bb44.i.i.i.i:		; preds = %bb29.i.i.i.i
-	br i1 false, label %bb.i.i.i.i210.backedge, label %bb46.i.i.i.i
+  br i1 false, label %bb.i.i.i.i210.backedge, label %bb46.i.i.i.i
 
 bb46.i.i.i.i:		; preds = %bb44.i.i.i.i
-	br i1 false, label %bb47.i.i.i.i, label %bb53.i.i.i.i.preheader.loopexit
+  br i1 false, label %bb47.i.i.i.i, label %bb53.i.i.i.i.preheader.loopexit
 
 bb53.i.i.i.i.preheader.loopexit:		; preds = %bb46.i.i.i.i
-	br label %bb53.i.i.i.i.preheader
+  br label %bb53.i.i.i.i.preheader
 
 bb53.i.i.i.i.preheader:		; preds = %bb53.i.i.i.i.preheader.loopexit, %bb7.i.i208
-	br label %bb53.i.i.i.i
+  br label %bb53.i.i.i.i
 
 bb47.i.i.i.i:		; preds = %bb46.i.i.i.i
-	br label %bb.i.i.i.i210.backedge
+  br label %bb.i.i.i.i210.backedge
 
 bb50.i.i.i.i:		; preds = %bb53.i.i.i.i
-	br i1 false, label %bb51.i.i.i.i, label %bb52.i.i.i.i
+  br i1 false, label %bb51.i.i.i.i, label %bb52.i.i.i.i
 
 bb51.i.i.i.i:		; preds = %bb50.i.i.i.i
-	br label %bb52.i.i.i.i
+  br label %bb52.i.i.i.i
 
 bb52.i.i.i.i:		; preds = %bb51.i.i.i.i, %bb50.i.i.i.i
-	br label %bb53.i.i.i.i
+  br label %bb53.i.i.i.i
 
 bb53.i.i.i.i:		; preds = %bb52.i.i.i.i, %bb53.i.i.i.i.preheader
-	br i1 false, label %bb50.i.i.i.i, label %bb59.i.i.i.i.preheader
+  br i1 false, label %bb50.i.i.i.i, label %bb59.i.i.i.i.preheader
 
 bb59.i.i.i.i.preheader:		; preds = %bb53.i.i.i.i
-	br label %bb59.i.i.i.i
+  br label %bb59.i.i.i.i
 
 bb55.i.i.i.i:		; preds = %bb59.i.i.i.i
-	br label %bb57.i.i.i.i
+  br label %bb57.i.i.i.i
 
 bb56.i.i.i.i:		; preds = %bb57.i.i.i.i
-	br label %bb57.i.i.i.i
+  br label %bb57.i.i.i.i
 
 bb57.i.i.i.i:		; preds = %bb56.i.i.i.i, %bb55.i.i.i.i
-	br i1 false, label %bb56.i.i.i.i, label %bb58.i.i.i.i
+  br i1 false, label %bb56.i.i.i.i, label %bb58.i.i.i.i
 
 bb58.i.i.i.i:		; preds = %bb57.i.i.i.i
-	br label %bb59.i.i.i.i
+  br label %bb59.i.i.i.i
 
 bb59.i.i.i.i:		; preds = %bb58.i.i.i.i, %bb59.i.i.i.i.preheader
-	br i1 false, label %bb60.i.i.i.i, label %bb55.i.i.i.i
+  br i1 false, label %bb60.i.i.i.i, label %bb55.i.i.i.i
 
 bb60.i.i.i.i:		; preds = %bb59.i.i.i.i
-	br label %bb69.i.i.i.i
+  br label %bb69.i.i.i.i
 
 bb61.i.i.i.i:		; preds = %bb69.i.i.i.i
-	br i1 false, label %bb68.i.i.i.i, label %bb62.i.i.i.i
+  br i1 false, label %bb68.i.i.i.i, label %bb62.i.i.i.i
 
 bb62.i.i.i.i:		; preds = %bb61.i.i.i.i
-	br i1 false, label %bb63.i.i.i.i, label %bb65.i.i.i.i
+  br i1 false, label %bb63.i.i.i.i, label %bb65.i.i.i.i
 
 bb63.i.i.i.i:		; preds = %bb62.i.i.i.i
-	br i1 false, label %bb.i.i12.i, label %bb65.i.i.i.i
+  br i1 false, label %bb.i.i12.i, label %bb65.i.i.i.i
 
 bb65.i.i.i.i:		; preds = %bb63.i.i.i.i, %bb62.i.i.i.i
-	br i1 false, label %bb.i.i12.i, label %bb67.i.i.i.i
+  br i1 false, label %bb.i.i12.i, label %bb67.i.i.i.i
 
 bb67.i.i.i.i:		; preds = %bb65.i.i.i.i
-	br label %bb68.i.i.i.i
+  br label %bb68.i.i.i.i
 
 bb68.i.i.i.i:		; preds = %bb67.i.i.i.i, %bb61.i.i.i.i
-	br label %bb69.i.i.i.i
+  br label %bb69.i.i.i.i
 
 bb69.i.i.i.i:		; preds = %bb68.i.i.i.i, %bb60.i.i.i.i
-	br i1 false, label %bb61.i.i.i.i, label %bb70.i.i.i.i
+  br i1 false, label %bb61.i.i.i.i, label %bb70.i.i.i.i
 
 bb70.i.i.i.i:		; preds = %bb69.i.i.i.i
-	br label %READ_LITERAL.i.outer.backedge
+  br label %READ_LITERAL.i.outer.backedge
 
 bb.i.i12.i:		; preds = %bb65.i.i.i.i, %bb63.i.i.i.i
-	br i1 false, label %bb1.i.i.i213, label %bb5.i.i.i218
+  br i1 false, label %bb1.i.i.i213, label %bb5.i.i.i218
 
 bb1.i.i.i213:		; preds = %bb.i.i12.i
-	br i1 false, label %bb4.i.i.i217, label %bb2.i.i.i214
+  br i1 false, label %bb4.i.i.i217, label %bb2.i.i.i214
 
 bb2.i.i.i214:		; preds = %bb1.i.i.i213
-	br label %bb4.i.i.i217
+  br label %bb4.i.i.i217
 
 bb4.i.i.i217:		; preds = %bb2.i.i.i214, %bb1.i.i.i213
-	br label %bb5.i.i.i218
+  br label %bb5.i.i.i218
 
 bb5.i.i.i218:		; preds = %bb4.i.i.i217, %bb.i.i12.i
-	br label %READ_LITERAL.i.outer.backedge
+  br label %READ_LITERAL.i.outer.backedge
 
 READ_LITERAL.i.outer.backedge:		; preds = %bb5.i.i.i218, %bb70.i.i.i.i, %bb6.i.i207
-	br label %READ_LITERAL.i.outer
+  br label %READ_LITERAL.i.outer
 
 parse.exit.loopexit:		; preds = %bb51.i, %bb47.i, %bb42.i202
-	br label %parse.exit
+  br label %parse.exit
 
 parse.exit:		; preds = %parse.exit.loopexit, %bb38.i198, %bb35.i195, %bb33.i193
-	br i1 false, label %bb130, label %bb129
+  br i1 false, label %bb130, label %bb129
 
 bb129:		; preds = %parse.exit, %bb22.i173, %bb18.i168, %bb16.i166, %bb12.i161, %bb10.i159, %bb9.i158, %bb8.i157, %bb7.i156, %bb5.i154, %bb4.i153
-	br label %bb170
+  br label %bb170
 
 bb130:		; preds = %parse.exit
-	br i1 false, label %bb143, label %bb142.preheader
+  br i1 false, label %bb143, label %bb142.preheader
 
 bb142.preheader:		; preds = %bb130
-	br label %bb142
+  br label %bb142
 
 bb132:		; preds = %bb142
-	br i1 false, label %bb137, label %bb133
+  br i1 false, label %bb137, label %bb133
 
 bb133:		; preds = %bb132
-	br i1 false, label %bb137, label %bb134
+  br i1 false, label %bb137, label %bb134
 
 bb134:		; preds = %bb133
-	br i1 false, label %bb137, label %bb135
+  br i1 false, label %bb137, label %bb135
 
 bb135:		; preds = %bb134
-	br i1 false, label %bb137, label %bb136
+  br i1 false, label %bb137, label %bb136
 
 bb136:		; preds = %bb135
-	br i1 false, label %bb137, label %bb138
+  br i1 false, label %bb137, label %bb138
 
 bb137:		; preds = %bb136, %bb135, %bb134, %bb133, %bb132
-	br label %bb141
+  br label %bb141
 
 bb138:		; preds = %bb136
-	br i1 false, label %bb139, label %bb141
+  br i1 false, label %bb139, label %bb141
 
 bb139:		; preds = %bb138
-	br i1 false, label %bb2.i126, label %picosat_assume.exit
+  br i1 false, label %bb2.i126, label %picosat_assume.exit
 
 bb2.i126:		; preds = %bb139
-	br i1 false, label %bb5.i130, label %bb3.i127
+  br i1 false, label %bb5.i130, label %bb3.i127
 
 bb3.i127:		; preds = %bb2.i126
-	br label %bb5.i130
+  br label %bb5.i130
 
 bb5.i130:		; preds = %bb3.i127, %bb2.i126
-	br label %picosat_assume.exit
+  br label %picosat_assume.exit
 
 picosat_assume.exit:		; preds = %bb5.i130, %bb139
-	br i1 false, label %bb141, label %bb140
+  br i1 false, label %bb141, label %bb140
 
 bb140:		; preds = %picosat_assume.exit
-	br label %bb141
+  br label %bb141
 
 bb141:		; preds = %bb140, %picosat_assume.exit, %bb138, %bb137
-	br label %bb142
+  br label %bb142
 
 bb142:		; preds = %bb141, %bb142.preheader
-	br i1 false, label %bb132, label %bb143.loopexit
+  br i1 false, label %bb132, label %bb143.loopexit
 
 bb143.loopexit:		; preds = %bb142
-	br label %bb143
+  br label %bb143
 
 bb143:		; preds = %bb143.loopexit, %bb130
-	br i1 false, label %bb145, label %bb144
+  br i1 false, label %bb145, label %bb144
 
 bb144:		; preds = %bb143
-	br label %bb11.i
+  br label %bb11.i
 
 bb5.i114:		; preds = %bb11.i
-	br label %bb11.i
+  br label %bb11.i
 
 bb11.i:		; preds = %bb5.i114, %bb144
-	br i1 false, label %bb12.i, label %bb5.i114
+  br i1 false, label %bb12.i, label %bb5.i114
 
 bb12.i:		; preds = %bb11.i
-	br i1 false, label %bb.i.i.i118, label %bb1.i.i.i119
+  br i1 false, label %bb.i.i.i118, label %bb1.i.i.i119
 
 bb.i.i.i118:		; preds = %bb12.i
-	br label %int2lit.exit.i
+  br label %int2lit.exit.i
 
 bb1.i.i.i119:		; preds = %bb12.i
-	br label %int2lit.exit.i
+  br label %int2lit.exit.i
 
 int2lit.exit.i:		; preds = %bb1.i.i.i119, %bb.i.i.i118
-	br label %bb19.i
+  br label %bb19.i
 
 bb13.i:		; preds = %bb19.i
-	br label %bb17.i
+  br label %bb17.i
 
 bb14.i:		; preds = %bb17.i
-	br label %bb17.i
+  br label %bb17.i
 
 bb17.i:		; preds = %bb14.i, %bb13.i
-	br i1 false, label %bb14.i, label %bb18.i
+  br i1 false, label %bb14.i, label %bb18.i
 
 bb18.i:		; preds = %bb17.i
-	br label %bb19.i
+  br label %bb19.i
 
 bb19.i:		; preds = %bb18.i, %int2lit.exit.i
-	br i1 false, label %bb20.i, label %bb13.i
+  br i1 false, label %bb20.i, label %bb13.i
 
 bb20.i:		; preds = %bb19.i
-	br label %bb33.i
+  br label %bb33.i
 
 bb24.i:		; preds = %bb33.i
-	br i1 false, label %bb29.i, label %bb25.i
+  br i1 false, label %bb29.i, label %bb25.i
 
 bb25.i:		; preds = %bb24.i
-	br label %bb27.i
+  br label %bb27.i
 
 bb26.i:		; preds = %bb27.i
-	br label %bb27.i
+  br label %bb27.i
 
 bb27.i:		; preds = %bb26.i, %bb25.i
-	br i1 false, label %bb26.i, label %bb28.i
+  br i1 false, label %bb26.i, label %bb28.i
 
 bb28.i:		; preds = %bb27.i
-	br label %bb29.i
+  br label %bb29.i
 
 bb29.i:		; preds = %bb28.i, %bb24.i
-	br label %bb33.i
+  br label %bb33.i
 
 bb33.i:		; preds = %bb29.i, %bb20.i
-	br i1 false, label %bb34.i, label %bb24.i
+  br i1 false, label %bb34.i, label %bb24.i
 
 bb34.i:		; preds = %bb33.i
-	br i1 false, label %bb.i.i58.i, label %bb1.i.i59.i
+  br i1 false, label %bb.i.i58.i, label %bb1.i.i59.i
 
 bb.i.i58.i:		; preds = %bb34.i
-	br label %int2lit.exit63.i
+  br label %int2lit.exit63.i
 
 bb1.i.i59.i:		; preds = %bb34.i
-	br label %int2lit.exit63.i
+  br label %int2lit.exit63.i
 
 int2lit.exit63.i:		; preds = %bb1.i.i59.i, %bb.i.i58.i
-	br label %bb41.i
+  br label %bb41.i
 
 bb35.i:		; preds = %bb41.i
-	br label %bb39.i
+  br label %bb39.i
 
 bb36.i:		; preds = %bb39.i
-	br i1 false, label %bb38.i, label %bb37.i
+  br i1 false, label %bb38.i, label %bb37.i
 
 bb37.i:		; preds = %bb36.i
-	br label %bb38.i
+  br label %bb38.i
 
 bb38.i:		; preds = %bb37.i, %bb36.i
-	br label %bb39.i
+  br label %bb39.i
 
 bb39.i:		; preds = %bb38.i, %bb35.i
-	br i1 false, label %bb36.i, label %bb40.i
+  br i1 false, label %bb36.i, label %bb40.i
 
 bb40.i:		; preds = %bb39.i
-	br label %bb41.i
+  br label %bb41.i
 
 bb41.i:		; preds = %bb40.i, %int2lit.exit63.i
-	br i1 false, label %bb42.i, label %bb35.i
+  br i1 false, label %bb42.i, label %bb35.i
 
 bb42.i:		; preds = %bb41.i
-	br label %bb44.i
+  br label %bb44.i
 
 bb43.i:		; preds = %bb44.i
-	br label %bb44.i
+  br label %bb44.i
 
 bb44.i:		; preds = %bb43.i, %bb42.i
-	br i1 false, label %bb43.i, label %picosat_print.exit
+  br i1 false, label %bb43.i, label %picosat_print.exit
 
 picosat_print.exit:		; preds = %bb44.i
-	br label %bb167
+  br label %bb167
 
 bb145:		; preds = %bb143
-	br i1 false, label %bb147, label %bb146
+  br i1 false, label %bb147, label %bb146
 
 bb146:		; preds = %bb145
-	br label %bb147
+  br label %bb147
 
 bb147:		; preds = %bb146, %bb145
-	br i1 false, label %bb149, label %bb148
+  br i1 false, label %bb149, label %bb148
 
 bb148:		; preds = %bb147
-	br label %bb149
+  br label %bb149
 
 bb149:		; preds = %bb148, %bb147
-	br i1 false, label %bb.i54, label %bb1.i55
+  br i1 false, label %bb.i54, label %bb1.i55
 
 bb.i54:		; preds = %bb149
-	unreachable
+  unreachable
 
 bb1.i55:		; preds = %bb149
-	br i1 false, label %bb.i.i56, label %bb1.i.i57
+  br i1 false, label %bb.i.i56, label %bb1.i.i57
 
 bb.i.i56:		; preds = %bb1.i55
-	br label %bb1.i.i57
+  br label %bb1.i.i57
 
 bb1.i.i57:		; preds = %bb.i.i56, %bb1.i55
-	br i1 false, label %bb3.i.i59, label %bb2.i.i58
+  br i1 false, label %bb3.i.i59, label %bb2.i.i58
 
 bb2.i.i58:		; preds = %bb1.i.i57
-	br label %bb3.i.i59
+  br label %bb3.i.i59
 
 bb3.i.i59:		; preds = %bb2.i.i58, %bb1.i.i57
-	br i1 false, label %bb5.i.i61, label %sat.exit.i
+  br i1 false, label %bb5.i.i61, label %sat.exit.i
 
 bb5.i.i61:		; preds = %bb3.i.i59
-	br i1 false, label %bb6.i.i65, label %bb1.i.i.i63
+  br i1 false, label %bb6.i.i65, label %bb1.i.i.i63
 
 bb1.i.i.i63:		; preds = %bb5.i.i61
-	br i1 false, label %sat.exit.i, label %bb6.i.i65
+  br i1 false, label %sat.exit.i, label %bb6.i.i65
 
 bb6.i.i65:		; preds = %bb1.i.i.i63, %bb5.i.i61
-	br i1 false, label %bb8.i.i67, label %bb7.i.i66
+  br i1 false, label %bb8.i.i67, label %bb7.i.i66
 
 bb7.i.i66:		; preds = %bb6.i.i65
-	br label %bb8.i.i67
+  br label %bb8.i.i67
 
 bb8.i.i67:		; preds = %bb7.i.i66, %bb6.i.i65
-	br i1 false, label %bb10.i.i69, label %sat.exit.i
+  br i1 false, label %bb10.i.i69, label %sat.exit.i
 
 bb10.i.i69:		; preds = %bb8.i.i67
-	br i1 false, label %bb11.i.i70, label %bb1.i61.i.i
+  br i1 false, label %bb11.i.i70, label %bb1.i61.i.i
 
 bb1.i61.i.i:		; preds = %bb10.i.i69
-	br i1 false, label %sat.exit.i, label %bb11.i.i70
+  br i1 false, label %sat.exit.i, label %bb11.i.i70
 
 bb11.i.i70:		; preds = %bb1.i61.i.i, %bb10.i.i69
-	br label %bb13.i.i71.outer
+  br label %bb13.i.i71.outer
 
 bb13.i.i71.outer:		; preds = %bb42.i.i, %bb11.i.i70
-	br label %bb13.i.i71
+  br label %bb13.i.i71
 
 bb13.i.i71:		; preds = %bb13.i.i71.backedge, %bb13.i.i71.outer
-	br i1 false, label %bb14.i.i72, label %bb15.i.i73
+  br i1 false, label %bb14.i.i72, label %bb15.i.i73
 
 bb14.i.i72:		; preds = %bb13.i.i71
-	br label %bb15.i.i73
+  br label %bb15.i.i73
 
 bb15.i.i73:		; preds = %bb14.i.i72, %bb13.i.i71
-	br i1 false, label %bb19.i.i, label %bb16.i.i
+  br i1 false, label %bb19.i.i, label %bb16.i.i
 
 bb16.i.i:		; preds = %bb15.i.i73
-	br i1 false, label %bb.i.i79.i.i, label %incincs.exit.i.i
+  br i1 false, label %bb.i.i79.i.i, label %incincs.exit.i.i
 
 bb.i.i79.i.i:		; preds = %bb16.i.i
-	br label %bb4.i.i.i85.i.i
+  br label %bb4.i.i.i85.i.i
 
 bb.i.i.i80.i.i:		; preds = %bb4.i.i.i85.i.i
-	br i1 false, label %bb3.i.i.i83.i.i, label %bb1.i.i.i81.i.i
+  br i1 false, label %bb3.i.i.i83.i.i, label %bb1.i.i.i81.i.i
 
 bb1.i.i.i81.i.i:		; preds = %bb.i.i.i80.i.i
-	br i1 false, label %bb2.i.i.i82.i.i, label %bb3.i.i.i83.i.i
+  br i1 false, label %bb2.i.i.i82.i.i, label %bb3.i.i.i83.i.i
 
 bb2.i.i.i82.i.i:		; preds = %bb1.i.i.i81.i.i
-	br label %bb3.i.i.i83.i.i
+  br label %bb3.i.i.i83.i.i
 
 bb3.i.i.i83.i.i:		; preds = %bb2.i.i.i82.i.i, %bb1.i.i.i81.i.i, %bb.i.i.i80.i.i
-	br label %bb4.i.i.i85.i.i
+  br label %bb4.i.i.i85.i.i
 
 bb4.i.i.i85.i.i:		; preds = %bb3.i.i.i83.i.i, %bb.i.i79.i.i
-	br i1 false, label %crescore.exit.i.i.i.i, label %bb.i.i.i80.i.i
+  br i1 false, label %crescore.exit.i.i.i.i, label %bb.i.i.i80.i.i
 
 crescore.exit.i.i.i.i:		; preds = %bb4.i.i.i85.i.i
-	br label %incincs.exit.i.i
+  br label %incincs.exit.i.i
 
 incincs.exit.i.i:		; preds = %crescore.exit.i.i.i.i, %bb16.i.i
-	br i1 false, label %bb13.i.i71.backedge, label %sat.exit.i.loopexit.loopexit
+  br i1 false, label %bb13.i.i71.backedge, label %sat.exit.i.loopexit.loopexit
 
 bb13.i.i71.backedge:		; preds = %bb1.i55.i.i, %bb28.i.i, %incincs.exit.i.i
-	br label %bb13.i.i71
+  br label %bb13.i.i71
 
 bb19.i.i:		; preds = %bb15.i.i73
-	br i1 false, label %bb20.i.i, label %bb1.i68.i.i
+  br i1 false, label %bb20.i.i, label %bb1.i68.i.i
 
 bb1.i68.i.i:		; preds = %bb19.i.i
-	br i1 false, label %sat.exit.i.loopexit.loopexit, label %bb20.i.i
+  br i1 false, label %sat.exit.i.loopexit.loopexit, label %bb20.i.i
 
 bb20.i.i:		; preds = %bb1.i68.i.i, %bb19.i.i
-	br i1 false, label %bb24.i.i, label %bb21.i.i
+  br i1 false, label %bb24.i.i, label %bb21.i.i
 
 bb21.i.i:		; preds = %bb20.i.i
-	br i1 false, label %bb22.i.i, label %bb24.i.i
+  br i1 false, label %bb22.i.i, label %bb24.i.i
 
 bb22.i.i:		; preds = %bb21.i.i
-	br i1 false, label %bb23.i.i, label %bb24.i.i
+  br i1 false, label %bb23.i.i, label %bb24.i.i
 
 bb23.i.i:		; preds = %bb22.i.i
-	br label %bb24.i.i
+  br label %bb24.i.i
 
 bb24.i.i:		; preds = %bb23.i.i, %bb22.i.i, %bb21.i.i, %bb20.i.i
-	br i1 false, label %bb26.i.i, label %sat.exit.i.loopexit.loopexit
+  br i1 false, label %bb26.i.i, label %sat.exit.i.loopexit.loopexit
 
 bb26.i.i:		; preds = %bb24.i.i
-	br i1 false, label %bb27.i.i, label %bb33.i.i.loopexit
+  br i1 false, label %bb27.i.i, label %bb33.i.i.loopexit
 
 bb27.i.i:		; preds = %bb26.i.i
-	br i1 false, label %bb33.i.i.loopexit, label %bb28.i.i
+  br i1 false, label %bb33.i.i.loopexit, label %bb28.i.i
 
 bb28.i.i:		; preds = %bb27.i.i
-	br i1 false, label %bb1.i55.i.i, label %bb13.i.i71.backedge
+  br i1 false, label %bb1.i55.i.i, label %bb13.i.i71.backedge
 
 bb1.i55.i.i:		; preds = %bb28.i.i
-	br i1 false, label %bb29.i.i, label %bb13.i.i71.backedge
+  br i1 false, label %bb29.i.i, label %bb13.i.i71.backedge
 
 bb29.i.i:		; preds = %bb1.i55.i.i
-	br i1 false, label %bb31.i.i, label %sat.exit.i.loopexit.loopexit2
+  br i1 false, label %bb31.i.i, label %sat.exit.i.loopexit.loopexit2
 
 bb31.i.i:		; preds = %bb29.i.i
-	br i1 false, label %bb33.i.i, label %bb1.i48.i.i
+  br i1 false, label %bb33.i.i, label %bb1.i48.i.i
 
 bb1.i48.i.i:		; preds = %bb31.i.i
-	br i1 false, label %sat.exit.i.loopexit.loopexit2, label %bb33.i.i
+  br i1 false, label %sat.exit.i.loopexit.loopexit2, label %bb33.i.i
 
 bb33.i.i.loopexit:		; preds = %bb27.i.i, %bb26.i.i
-	br label %bb33.i.i
+  br label %bb33.i.i
 
 bb33.i.i:		; preds = %bb33.i.i.loopexit, %bb1.i48.i.i, %bb31.i.i
-	br i1 false, label %bb34.i.i, label %bb35.i.i
+  br i1 false, label %bb34.i.i, label %bb35.i.i
 
 bb34.i.i:		; preds = %bb33.i.i
-	br i1 false, label %bb35.i.i, label %bb2.i44.i.i76
+  br i1 false, label %bb35.i.i, label %bb2.i44.i.i76
 
 bb2.i44.i.i76:		; preds = %bb34.i.i
-	br label %bb35.i.i
+  br label %bb35.i.i
 
 bb35.i.i:		; preds = %bb2.i44.i.i76, %bb34.i.i, %bb33.i.i
-	br i1 false, label %bb1.i37.i.i, label %bb.i35.i.i
+  br i1 false, label %bb1.i37.i.i, label %bb.i35.i.i
 
 bb.i35.i.i:		; preds = %bb35.i.i
-	br label %bb36.i.i
+  br label %bb36.i.i
 
 bb1.i37.i.i:		; preds = %bb35.i.i
-	br i1 false, label %bb37.i.i, label %bb36.i.i
+  br i1 false, label %bb37.i.i, label %bb36.i.i
 
 bb36.i.i:		; preds = %bb1.i37.i.i, %bb.i35.i.i
-	br label %bb25.i23.i.i
+  br label %bb25.i23.i.i
 
 bb.i18.i.i:		; preds = %bb25.i23.i.i
-	br i1 false, label %bb24.i22.i.i, label %bb22.i19.i.i
+  br i1 false, label %bb24.i22.i.i, label %bb22.i19.i.i
 
 bb22.i19.i.i:		; preds = %bb.i18.i.i
-	br label %bb24.i22.i.i
+  br label %bb24.i22.i.i
 
 bb24.i22.i.i:		; preds = %bb22.i19.i.i, %bb.i18.i.i
-	br label %bb25.i23.i.i
+  br label %bb25.i23.i.i
 
 bb25.i23.i.i:		; preds = %bb24.i22.i.i, %bb36.i.i
-	br i1 false, label %bb.i18.i.i, label %bb26.i24.i.i
+  br i1 false, label %bb.i18.i.i, label %bb26.i24.i.i
 
 bb26.i24.i.i:		; preds = %bb25.i23.i.i
-	br i1 false, label %bb27.i25.i.i, label %bb32.i.i.i
+  br i1 false, label %bb27.i25.i.i, label %bb32.i.i.i
 
 bb27.i25.i.i:		; preds = %bb26.i24.i.i
-	br label %bb32.i.i.i
+  br label %bb32.i.i.i
 
 bb32.i.i.i:		; preds = %bb27.i25.i.i, %bb26.i24.i.i
-	br label %bb64.i.i.i
+  br label %bb64.i.i.i
 
 bb33.i.i.i:		; preds = %bb64.i.i.i
-	br i1 false, label %bb60.i.i.i, label %bb34.i.i.i
+  br i1 false, label %bb60.i.i.i, label %bb34.i.i.i
 
 bb34.i.i.i:		; preds = %bb33.i.i.i
-	br i1 false, label %bb38.i.i.i, label %bb60.i.i.i
+  br i1 false, label %bb38.i.i.i, label %bb60.i.i.i
 
 bb38.i.i.i:		; preds = %bb34.i.i.i
-	br i1 false, label %bb39.i.i.i, label %bb48.i.i.i
+  br i1 false, label %bb39.i.i.i, label %bb48.i.i.i
 
 bb39.i.i.i:		; preds = %bb38.i.i.i
-	br i1 false, label %bb48.i.i.i, label %bb40.i.i.i
+  br i1 false, label %bb48.i.i.i, label %bb40.i.i.i
 
 bb40.i.i.i:		; preds = %bb39.i.i.i
-	br i1 false, label %bb60.i.i.i, label %bb45.i.i.i
+  br i1 false, label %bb60.i.i.i, label %bb45.i.i.i
 
 bb45.i.i.i:		; preds = %bb40.i.i.i
-	br label %bb60.i.i.i
+  br label %bb60.i.i.i
 
 bb48.i.i.i:		; preds = %bb39.i.i.i, %bb38.i.i.i
-	br i1 false, label %bb53.i.i.i, label %bb60.i.i.i
+  br i1 false, label %bb53.i.i.i, label %bb60.i.i.i
 
 bb53.i.i.i:		; preds = %bb48.i.i.i
-	br i1 false, label %bb60.i.i.i, label %bb58.i.i.i
+  br i1 false, label %bb60.i.i.i, label %bb58.i.i.i
 
 bb58.i.i.i:		; preds = %bb53.i.i.i
-	br i1 false, label %bb59.i.i.i, label %bb60.i.i.i
+  br i1 false, label %bb59.i.i.i, label %bb60.i.i.i
 
 bb59.i.i.i:		; preds = %bb58.i.i.i
-	br label %bb60.i.i.i
+  br label %bb60.i.i.i
 
 bb60.i.i.i:		; preds = %bb59.i.i.i, %bb58.i.i.i, %bb53.i.i.i, %bb48.i.i.i, %bb45.i.i.i, %bb40.i.i.i, %bb34.i.i.i, %bb33.i.i.i
-	%lcollect.i.i.i.1 = phi i32 [ %lcollect.i.i.i.2, %bb34.i.i.i ], [ %lcollect.i.i.i.2, %bb48.i.i.i ], [ %lcollect.i.i.i.2, %bb58.i.i.i ], [ %lcollect.i.i.i.2, %bb59.i.i.i ], [ %lcollect.i.i.i.2, %bb53.i.i.i ], [ %lcollect.i.i.i.2, %bb33.i.i.i ], [ %lcollect.i.i.i.2, %bb40.i.i.i ], [ 0, %bb45.i.i.i ]		; <i32> [#uses=1]
-	br label %bb64.i.i.i
+  %lcollect.i.i.i.1 = phi i32 [ %lcollect.i.i.i.2, %bb34.i.i.i ], [ %lcollect.i.i.i.2, %bb48.i.i.i ], [ %lcollect.i.i.i.2, %bb58.i.i.i ], [ %lcollect.i.i.i.2, %bb59.i.i.i ], [ %lcollect.i.i.i.2, %bb53.i.i.i ], [ %lcollect.i.i.i.2, %bb33.i.i.i ], [ %lcollect.i.i.i.2, %bb40.i.i.i ], [ 0, %bb45.i.i.i ]		; <i32> [#uses=1]
+  br label %bb64.i.i.i
 
 bb64.i.i.i:		; preds = %bb60.i.i.i, %bb32.i.i.i
-	%lcollect.i.i.i.2 = phi i32 [ 0, %bb32.i.i.i ], [ %lcollect.i.i.i.1, %bb60.i.i.i ]		; <i32> [#uses=8]
-	br i1 false, label %bb65.i.i.i, label %bb33.i.i.i
+  %lcollect.i.i.i.2 = phi i32 [ 0, %bb32.i.i.i ], [ %lcollect.i.i.i.1, %bb60.i.i.i ]		; <i32> [#uses=8]
+  br i1 false, label %bb65.i.i.i, label %bb33.i.i.i
 
 bb65.i.i.i:		; preds = %bb64.i.i.i
-	br i1 false, label %bb103.i.i.i.preheader, label %bb66.i.i.i.preheader
+  br i1 false, label %bb103.i.i.i.preheader, label %bb66.i.i.i.preheader
 
 bb66.i.i.i.preheader:		; preds = %bb65.i.i.i
-	br label %bb66.i.i.i
+  br label %bb66.i.i.i
 
 bb66.i.i.i:		; preds = %bb66.i.i.i.backedge, %bb66.i.i.i.preheader
-	br i1 false, label %bb67.i.i.i, label %bb68.i.i.i
+  br i1 false, label %bb67.i.i.i, label %bb68.i.i.i
 
 bb67.i.i.i:		; preds = %bb66.i.i.i
-	br label %bb68.i.i.i
+  br label %bb68.i.i.i
 
 bb68.i.i.i:		; preds = %bb67.i.i.i, %bb66.i.i.i
-	br i1 false, label %bb69.i.i.i, label %bb70.i.i.i
+  br i1 false, label %bb69.i.i.i, label %bb70.i.i.i
 
 bb69.i.i.i:		; preds = %bb68.i.i.i
-	br label %bb70.i.i.i
+  br label %bb70.i.i.i
 
 bb70.i.i.i:		; preds = %bb69.i.i.i, %bb68.i.i.i
-	br i1 false, label %bb71.i.i.i, label %bb72.i.i.i
+  br i1 false, label %bb71.i.i.i, label %bb72.i.i.i
 
 bb71.i.i.i:		; preds = %bb70.i.i.i
-	br label %bb72.i.i.i
+  br label %bb72.i.i.i
 
 bb72.i.i.i:		; preds = %bb71.i.i.i, %bb70.i.i.i
-	br label %bb73.i.i.i.outer
+  br label %bb73.i.i.i.outer
 
 bb73.i.i.i.outer:		; preds = %bb78.i.i.i, %bb72.i.i.i
-	br label %bb73.i.i.i
+  br label %bb73.i.i.i
 
 bb73.i.i.i:		; preds = %bb73.i.i.i, %bb73.i.i.i.outer
-	br i1 false, label %bb73.i.i.i, label %bb76.i.i.i.preheader
+  br i1 false, label %bb73.i.i.i, label %bb76.i.i.i.preheader
 
 bb76.i.i.i.preheader:		; preds = %bb73.i.i.i
-	br label %bb76.i.i.i
+  br label %bb76.i.i.i
 
 bb76.i.i.i:		; preds = %bb76.i.i.i, %bb76.i.i.i.preheader
-	br i1 false, label %bb77.i.i.i, label %bb76.i.i.i
+  br i1 false, label %bb77.i.i.i, label %bb76.i.i.i
 
 bb77.i.i.i:		; preds = %bb76.i.i.i
-	br i1 false, label %bb78.i.i.i, label %bb79.i.i.i
+  br i1 false, label %bb78.i.i.i, label %bb79.i.i.i
 
 bb78.i.i.i:		; preds = %bb77.i.i.i
-	br label %bb73.i.i.i.outer
+  br label %bb73.i.i.i.outer
 
 bb79.i.i.i:		; preds = %bb77.i.i.i
-	br i1 false, label %bb83.i.i.i, label %bb94.i.i.i
+  br i1 false, label %bb83.i.i.i, label %bb94.i.i.i
 
 bb83.i.i.i:		; preds = %bb79.i.i.i
-	br i1 false, label %bb84.i.i.i, label %bb88.i.i.i
+  br i1 false, label %bb84.i.i.i, label %bb88.i.i.i
 
 bb84.i.i.i:		; preds = %bb83.i.i.i
-	br i1 false, label %bb87.i.i.i, label %bb85.i.i.i
+  br i1 false, label %bb87.i.i.i, label %bb85.i.i.i
 
 bb85.i.i.i:		; preds = %bb84.i.i.i
-	br label %bb87.i.i.i
+  br label %bb87.i.i.i
 
 bb87.i.i.i:		; preds = %bb85.i.i.i, %bb84.i.i.i
-	br label %bb88.i.i.i
+  br label %bb88.i.i.i
 
 bb88.i.i.i:		; preds = %bb87.i.i.i, %bb83.i.i.i
-	br i1 false, label %bb89.i.i.i, label %bb93.i.i.i
+  br i1 false, label %bb89.i.i.i, label %bb93.i.i.i
 
 bb89.i.i.i:		; preds = %bb88.i.i.i
-	br i1 false, label %bb92.i.i.i, label %bb90.i.i.i
+  br i1 false, label %bb92.i.i.i, label %bb90.i.i.i
 
 bb90.i.i.i:		; preds = %bb89.i.i.i
-	br label %bb92.i.i.i
+  br label %bb92.i.i.i
 
 bb92.i.i.i:		; preds = %bb90.i.i.i, %bb89.i.i.i
-	br label %bb93.i.i.i
+  br label %bb93.i.i.i
 
 bb93.i.i.i:		; preds = %bb92.i.i.i, %bb88.i.i.i
-	br label %bb66.i.i.i.backedge
+  br label %bb66.i.i.i.backedge
 
 bb66.i.i.i.backedge:		; preds = %bb97.i.i.i, %bb94.i.i.i, %bb93.i.i.i
-	br label %bb66.i.i.i
+  br label %bb66.i.i.i
 
 bb94.i.i.i:		; preds = %bb79.i.i.i
-	br i1 false, label %bb66.i.i.i.backedge, label %bb96.i.i.i
+  br i1 false, label %bb66.i.i.i.backedge, label %bb96.i.i.i
 
 bb96.i.i.i:		; preds = %bb94.i.i.i
-	br i1 false, label %bb97.i.i.i, label %bb103.i.i.i.preheader.loopexit
+  br i1 false, label %bb97.i.i.i, label %bb103.i.i.i.preheader.loopexit
 
 bb103.i.i.i.preheader.loopexit:		; preds = %bb96.i.i.i
-	br label %bb103.i.i.i.preheader
+  br label %bb103.i.i.i.preheader
 
 bb103.i.i.i.preheader:		; preds = %bb103.i.i.i.preheader.loopexit, %bb65.i.i.i
-	br label %bb103.i.i.i
+  br label %bb103.i.i.i
 
 bb97.i.i.i:		; preds = %bb96.i.i.i
-	br label %bb66.i.i.i.backedge
+  br label %bb66.i.i.i.backedge
 
 bb100.i.i.i:		; preds = %bb103.i.i.i
-	br i1 false, label %bb101.i.i.i, label %bb102.i.i.i
+  br i1 false, label %bb101.i.i.i, label %bb102.i.i.i
 
 bb101.i.i.i:		; preds = %bb100.i.i.i
-	br label %bb102.i.i.i
+  br label %bb102.i.i.i
 
 bb102.i.i.i:		; preds = %bb101.i.i.i, %bb100.i.i.i
-	br label %bb103.i.i.i
+  br label %bb103.i.i.i
 
 bb103.i.i.i:		; preds = %bb102.i.i.i, %bb103.i.i.i.preheader
-	br i1 false, label %bb100.i.i.i, label %bb109.i.i.i.preheader
+  br i1 false, label %bb100.i.i.i, label %bb109.i.i.i.preheader
 
 bb109.i.i.i.preheader:		; preds = %bb103.i.i.i
-	br label %bb109.i.i.i
+  br label %bb109.i.i.i
 
 bb105.i.i.i:		; preds = %bb109.i.i.i
-	br label %bb107.i.i.i
+  br label %bb107.i.i.i
 
 bb106.i.i.i:		; preds = %bb107.i.i.i
-	br label %bb107.i.i.i
+  br label %bb107.i.i.i
 
 bb107.i.i.i:		; preds = %bb106.i.i.i, %bb105.i.i.i
-	br i1 false, label %bb106.i.i.i, label %bb108.i.i.i
+  br i1 false, label %bb106.i.i.i, label %bb108.i.i.i
 
 bb108.i.i.i:		; preds = %bb107.i.i.i
-	br label %bb109.i.i.i
+  br label %bb109.i.i.i
 
 bb109.i.i.i:		; preds = %bb108.i.i.i, %bb109.i.i.i.preheader
-	br i1 false, label %bb110.i.i.i, label %bb105.i.i.i
+  br i1 false, label %bb110.i.i.i, label %bb105.i.i.i
 
 bb110.i.i.i:		; preds = %bb109.i.i.i
-	%0 = sub i32 0, %lcollect.i.i.i.2		; <i32> [#uses=1]
-	%1 = add i32 %0, 1		; <i32> [#uses=1]
-	br label %bb113.i.i.i
+  %0 = sub i32 0, %lcollect.i.i.i.2		; <i32> [#uses=1]
+  %1 = add i32 %0, 1		; <i32> [#uses=1]
+  br label %bb113.i.i.i
 
 bb111.i.i.i:		; preds = %bb113.i.i.i
-	br i1 false, label %bb114.i.i.i, label %bb113.i.i.i
+  br i1 false, label %bb114.i.i.i, label %bb113.i.i.i
 
 bb113.i.i.i:		; preds = %bb111.i.i.i, %bb110.i.i.i
-	br i1 false, label %bb111.i.i.i, label %bb114.i.i.i
+  br i1 false, label %bb111.i.i.i, label %bb114.i.i.i
 
 bb114.i.i.i:		; preds = %bb113.i.i.i, %bb111.i.i.i
-	%2 = lshr i32 %1, 1		; <i32> [#uses=2]
-	br i1 false, label %bb116.i.i.i, label %bb124.i.i.i
+  %2 = lshr i32 %1, 1		; <i32> [#uses=2]
+  br i1 false, label %bb116.i.i.i, label %bb124.i.i.i
 
 bb116.i.i.i:		; preds = %bb114.i.i.i
-	br i1 false, label %bb117.i.i.i.preheader, label %bb122.i.i.i.preheader
+  br i1 false, label %bb117.i.i.i.preheader, label %bb122.i.i.i.preheader
 
 bb122.i.i.i.preheader:		; preds = %bb116.i.i.i
-	br label %bb122.i.i.i
+  br label %bb122.i.i.i
 
 bb117.i.i.i.preheader:		; preds = %bb116.i.i.i
-	br label %bb117.i.i.i
+  br label %bb117.i.i.i
 
 bb117.i.i.i:		; preds = %bb118.i.i.i, %bb117.i.i.i.preheader
-	%target.i.i.i.1 = phi i32 [ %3, %bb118.i.i.i ], [ %2, %bb117.i.i.i.preheader ]		; <i32> [#uses=1]
-	%3 = add i32 %target.i.i.i.1, 1		; <i32> [#uses=2]
-	br i1 false, label %bb118.i.i.i, label %bb124.i.i.i.loopexit
+  %target.i.i.i.1 = phi i32 [ %3, %bb118.i.i.i ], [ %2, %bb117.i.i.i.preheader ]		; <i32> [#uses=1]
+  %3 = add i32 %target.i.i.i.1, 1		; <i32> [#uses=2]
+  br i1 false, label %bb118.i.i.i, label %bb124.i.i.i.loopexit
 
 bb118.i.i.i:		; preds = %bb117.i.i.i
-	br i1 false, label %bb117.i.i.i, label %bb124.i.i.i.loopexit
+  br i1 false, label %bb117.i.i.i, label %bb124.i.i.i.loopexit
 
 bb122.i.i.i:		; preds = %bb123.i.i.i, %bb122.i.i.i.preheader
-	%target.i.i.i.2 = phi i32 [ %4, %bb123.i.i.i ], [ %2, %bb122.i.i.i.preheader ]		; <i32> [#uses=2]
-	br i1 false, label %bb124.i.i.i.loopexit1, label %bb123.i.i.i
+  %target.i.i.i.2 = phi i32 [ %4, %bb123.i.i.i ], [ %2, %bb122.i.i.i.preheader ]		; <i32> [#uses=2]
+  br i1 false, label %bb124.i.i.i.loopexit1, label %bb123.i.i.i
 
 bb123.i.i.i:		; preds = %bb122.i.i.i
-	%4 = add i32 %target.i.i.i.2, -1		; <i32> [#uses=1]
-	br i1 false, label %bb122.i.i.i, label %bb124.i.i.i.loopexit1
+  %4 = add i32 %target.i.i.i.2, -1		; <i32> [#uses=1]
+  br i1 false, label %bb122.i.i.i, label %bb124.i.i.i.loopexit1
 
 bb124.i.i.i.loopexit:		; preds = %bb118.i.i.i, %bb117.i.i.i
-	br label %bb124.i.i.i
+  br label %bb124.i.i.i
 
 bb124.i.i.i.loopexit1:		; preds = %bb123.i.i.i, %bb122.i.i.i
-	br label %bb124.i.i.i
+  br label %bb124.i.i.i
 
 bb124.i.i.i:		; preds = %bb124.i.i.i.loopexit1, %bb124.i.i.i.loopexit, %bb114.i.i.i
-	%target.i.i.i.0 = phi i32 [ 0, %bb114.i.i.i ], [ %3, %bb124.i.i.i.loopexit ], [ %target.i.i.i.2, %bb124.i.i.i.loopexit1 ]		; <i32> [#uses=0]
-	br label %bb132.i.i.i.outer
+  %target.i.i.i.0 = phi i32 [ 0, %bb114.i.i.i ], [ %3, %bb124.i.i.i.loopexit ], [ %target.i.i.i.2, %bb124.i.i.i.loopexit1 ]		; <i32> [#uses=0]
+  br label %bb132.i.i.i.outer
 
 bb125.i.i.i:		; preds = %bb132.i.i.i
-	br i1 false, label %bb132.i.i.i, label %bb130.i.i.i
+  br i1 false, label %bb132.i.i.i, label %bb130.i.i.i
 
 bb130.i.i.i:		; preds = %bb125.i.i.i
-	br label %bb132.i.i.i.outer
+  br label %bb132.i.i.i.outer
 
 bb132.i.i.i.outer:		; preds = %bb130.i.i.i, %bb124.i.i.i
-	br label %bb132.i.i.i
+  br label %bb132.i.i.i
 
 bb132.i.i.i:		; preds = %bb132.i.i.i.outer, %bb125.i.i.i
-	br i1 false, label %bb125.i.i.i, label %bb133.i.i.i
+  br i1 false, label %bb125.i.i.i, label %bb133.i.i.i
 
 bb133.i.i.i:		; preds = %bb132.i.i.i
-	br i1 false, label %bb136.i.i.i, label %bb134.i.i.i
+  br i1 false, label %bb136.i.i.i, label %bb134.i.i.i
 
 bb134.i.i.i:		; preds = %bb133.i.i.i
-	br i1 false, label %bb136.i.i.i, label %bb135.i.i.i
+  br i1 false, label %bb136.i.i.i, label %bb135.i.i.i
 
 bb135.i.i.i:		; preds = %bb134.i.i.i
-	br label %bb136.i.i.i
+  br label %bb136.i.i.i
 
 bb136.i.i.i:		; preds = %bb135.i.i.i, %bb134.i.i.i, %bb133.i.i.i
-	br i1 false, label %bb137.i.i.i, label %bb37.i.i
+  br i1 false, label %bb137.i.i.i, label %bb37.i.i
 
 bb137.i.i.i:		; preds = %bb136.i.i.i
-	br label %bb37.i.i
+  br label %bb37.i.i
 
 bb37.i.i:		; preds = %bb137.i.i.i, %bb136.i.i.i, %bb1.i37.i.i
-	br i1 false, label %bb40.i.i, label %bb38.i.i
+  br i1 false, label %bb40.i.i, label %bb38.i.i
 
 bb38.i.i:		; preds = %bb37.i.i
-	br i1 false, label %bb39.i.i, label %bb40.i.i
+  br i1 false, label %bb39.i.i, label %bb40.i.i
 
 bb39.i.i:		; preds = %bb38.i.i
-	br i1 false, label %bb17.i.i.i, label %bb3.i12.i.i
+  br i1 false, label %bb17.i.i.i, label %bb3.i12.i.i
 
 bb3.i12.i.i:		; preds = %bb39.i.i
-	br label %bb5.i14.i.i
+  br label %bb5.i14.i.i
 
 bb5.i14.i.i:		; preds = %bb8.i.i.i79, %bb3.i12.i.i
-	br i1 false, label %bb6.i15.i.i, label %bb9.i.i.i80
+  br i1 false, label %bb6.i15.i.i, label %bb9.i.i.i80
 
 bb6.i15.i.i:		; preds = %bb5.i14.i.i
-	br i1 false, label %bb7.i.i.i78, label %bb9.i.i.i80
+  br i1 false, label %bb7.i.i.i78, label %bb9.i.i.i80
 
 bb7.i.i.i78:		; preds = %bb6.i15.i.i
-	br i1 false, label %bb9.i.i.i80, label %bb8.i.i.i79
+  br i1 false, label %bb9.i.i.i80, label %bb8.i.i.i79
 
 bb8.i.i.i79:		; preds = %bb7.i.i.i78
-	br i1 false, label %bb9.i.i.i80, label %bb5.i14.i.i
+  br i1 false, label %bb9.i.i.i80, label %bb5.i14.i.i
 
 bb9.i.i.i80:		; preds = %bb8.i.i.i79, %bb7.i.i.i78, %bb6.i15.i.i, %bb5.i14.i.i
-	br i1 false, label %bb16.i.i.i, label %bb10.i.i.i81
+  br i1 false, label %bb16.i.i.i, label %bb10.i.i.i81
 
 bb10.i.i.i81:		; preds = %bb9.i.i.i80
-	br i1 false, label %bb11.i.i.i, label %bb15.i.i.i
+  br i1 false, label %bb11.i.i.i, label %bb15.i.i.i
 
 bb11.i.i.i:		; preds = %bb10.i.i.i81
-	br i1 false, label %bb16.i.i.i, label %bb15.i.i.i
+  br i1 false, label %bb16.i.i.i, label %bb15.i.i.i
 
 bb15.i.i.i:		; preds = %bb11.i.i.i, %bb10.i.i.i81
-	br label %bb16.i.i.i
+  br label %bb16.i.i.i
 
 bb16.i.i.i:		; preds = %bb15.i.i.i, %bb11.i.i.i, %bb9.i.i.i80
-	br label %bb17.i.i.i
+  br label %bb17.i.i.i
 
 bb17.i.i.i:		; preds = %bb16.i.i.i, %bb39.i.i
-	br i1 false, label %bb18.i.i.i, label %bb25.i.i.i
+  br i1 false, label %bb18.i.i.i, label %bb25.i.i.i
 
 bb18.i.i.i:		; preds = %bb17.i.i.i
-	br i1 false, label %bb24.i.i.i, label %bb23.i.i.i
+  br i1 false, label %bb24.i.i.i, label %bb23.i.i.i
 
 bb23.i.i.i:		; preds = %bb18.i.i.i
-	br label %bb24.i.i.i
+  br label %bb24.i.i.i
 
 bb24.i.i.i:		; preds = %bb23.i.i.i, %bb18.i.i.i
-	br label %bb29.i.i.i
+  br label %bb29.i.i.i
 
 bb25.i.i.i:		; preds = %bb17.i.i.i
-	br i1 false, label %bb29.i.i.i, label %bb27.i.i.i
+  br i1 false, label %bb29.i.i.i, label %bb27.i.i.i
 
 bb27.i.i.i:		; preds = %bb25.i.i.i
-	br i1 false, label %bb29.i.i.i, label %bb28.i.i.i
+  br i1 false, label %bb29.i.i.i, label %bb28.i.i.i
 
 bb28.i.i.i:		; preds = %bb27.i.i.i
-	br i1 false, label %bb29.i.i.i, label %bb.i4.i.i.i
+  br i1 false, label %bb29.i.i.i, label %bb.i4.i.i.i
 
 bb.i4.i.i.i:		; preds = %bb28.i.i.i
-	br i1 false, label %bb4.i.i16.i.i, label %bb29.i.i.i
+  br i1 false, label %bb4.i.i16.i.i, label %bb29.i.i.i
 
 bb4.i.i16.i.i:		; preds = %bb.i4.i.i.i
-	br label %bb29.i.i.i
+  br label %bb29.i.i.i
 
 bb29.i.i.i:		; preds = %bb4.i.i16.i.i, %bb.i4.i.i.i, %bb28.i.i.i, %bb27.i.i.i, %bb25.i.i.i, %bb24.i.i.i
-	br label %bb40.i.i
+  br label %bb40.i.i
 
 bb40.i.i:		; preds = %bb29.i.i.i, %bb38.i.i, %bb37.i.i
-	br i1 false, label %bb9.i.i.i.i.preheader, label %bb2.i.i.i87
+  br i1 false, label %bb9.i.i.i.i.preheader, label %bb2.i.i.i87
 
 bb9.i.i.i.i.preheader:		; preds = %bb40.i.i
-	br label %bb9.i.i.i.i
+  br label %bb9.i.i.i.i
 
 bb.i.i.i.i84:		; preds = %bb9.i.i.i.i
-	switch i8 0, label %bb8.i.i.i.i [
-		i8 -1, label %bb1.i.i.i.i85
-		i8 1, label %bb9.i.i.i.i
-	]
+  switch i8 0, label %bb8.i.i.i.i [
+  i8 -1, label %bb1.i.i.i.i85
+  i8 1, label %bb9.i.i.i.i
+  ]
 
 bb1.i.i.i.i85:		; preds = %bb.i.i.i.i84
-	br i1 false, label %bb5.i.i.i.i, label %bb2.i.i.i87
+  br i1 false, label %bb5.i.i.i.i, label %bb2.i.i.i87
 
 bb5.i.i.i.i:		; preds = %bb1.i.i.i.i85
-	br label %bb2.i.i.i87
+  br label %bb2.i.i.i87
 
 bb8.i.i.i.i:		; preds = %bb.i.i.i.i84
-	br i1 false, label %bb2.i.i.i87, label %bb6.i.i.i95
+  br i1 false, label %bb2.i.i.i87, label %bb6.i.i.i95
 
 bb9.i.i.i.i:		; preds = %bb.i.i.i.i84, %bb9.i.i.i.i.preheader
-	br i1 false, label %bb.i.i.i.i84, label %bb10.i.i.i.i
+  br i1 false, label %bb.i.i.i.i84, label %bb10.i.i.i.i
 
 bb10.i.i.i.i:		; preds = %bb9.i.i.i.i
-	br label %bb2.i.i.i87
+  br label %bb2.i.i.i87
 
 bb2.i.i.i87:		; preds = %bb10.i.i.i.i, %bb8.i.i.i.i, %bb5.i.i.i.i, %bb1.i.i.i.i85, %bb40.i.i
-	br i1 false, label %bb3.i.i.i88, label %decide.exit.i.i
+  br i1 false, label %bb3.i.i.i88, label %decide.exit.i.i
 
 bb3.i.i.i88:		; preds = %bb2.i.i.i87
-	br i1 false, label %bb4.i.i.i90, label %bb1.i23.i.i.i
+  br i1 false, label %bb4.i.i.i90, label %bb1.i23.i.i.i
 
 bb1.i23.i.i.i:		; preds = %bb3.i.i.i88
-	br i1 false, label %decide.exit.i.i, label %bb4.i.i.i90
+  br i1 false, label %decide.exit.i.i, label %bb4.i.i.i90
 
 bb4.i.i.i90:		; preds = %bb1.i23.i.i.i, %bb3.i.i.i88
-	br i1 false, label %bb1.i9.i.i.i, label %bb5.i.i.i94
+  br i1 false, label %bb1.i9.i.i.i, label %bb5.i.i.i94
 
 bb1.i9.i.i.i:		; preds = %bb4.i.i.i90
-	br i1 false, label %bb.i.i27.i.i.i.i, label %bb1.i.i28.i.i.i.i
+  br i1 false, label %bb.i.i27.i.i.i.i, label %bb1.i.i28.i.i.i.i
 
 bb.i.i27.i.i.i.i:		; preds = %bb1.i9.i.i.i
-	br label %int2lit.exit32.i.i.i.i
+  br label %int2lit.exit32.i.i.i.i
 
 bb1.i.i28.i.i.i.i:		; preds = %bb1.i9.i.i.i
-	br label %int2lit.exit32.i.i.i.i
+  br label %int2lit.exit32.i.i.i.i
 
 int2lit.exit32.i.i.i.i:		; preds = %bb1.i.i28.i.i.i.i, %bb.i.i27.i.i.i.i
-	br i1 false, label %bb8.i19.i.i.i, label %bb2.i.i.i.i91
+  br i1 false, label %bb8.i19.i.i.i, label %bb2.i.i.i.i91
 
 bb2.i.i.i.i91:		; preds = %int2lit.exit32.i.i.i.i
-	br label %bb4.i.i.i.i
+  br label %bb4.i.i.i.i
 
 bb3.i.i.i.i92:		; preds = %gcd.exit.i.i.i.i
-	br label %bb4.i.i.i.i
+  br label %bb4.i.i.i.i
 
 bb4.i.i.i.i:		; preds = %bb3.i.i.i.i92, %bb2.i.i.i.i91
-	br label %bb3.i.i13.i.i.i
+  br label %bb3.i.i13.i.i.i
 
 bb2.i.i12.i.i.i:		; preds = %bb3.i.i13.i.i.i
-	br label %bb3.i.i13.i.i.i
+  br label %bb3.i.i13.i.i.i
 
 bb3.i.i13.i.i.i:		; preds = %bb2.i.i12.i.i.i, %bb4.i.i.i.i
-	br i1 false, label %gcd.exit.i.i.i.i, label %bb2.i.i12.i.i.i
+  br i1 false, label %gcd.exit.i.i.i.i, label %bb2.i.i12.i.i.i
 
 gcd.exit.i.i.i.i:		; preds = %bb3.i.i13.i.i.i
-	br i1 false, label %bb5.i14.i.i.i.preheader, label %bb3.i.i.i.i92
+  br i1 false, label %bb5.i14.i.i.i.preheader, label %bb3.i.i.i.i92
 
 bb5.i14.i.i.i.preheader:		; preds = %gcd.exit.i.i.i.i
-	br label %bb5.i14.i.i.i
+  br label %bb5.i14.i.i.i
 
 bb5.i14.i.i.i:		; preds = %int2lit.exit.i.i.i.i, %bb5.i14.i.i.i.preheader
-	br i1 false, label %bb.i.i.i17.i.i.i, label %bb1.i.i.i18.i.i.i
+  br i1 false, label %bb.i.i.i17.i.i.i, label %bb1.i.i.i18.i.i.i
 
 bb.i.i.i17.i.i.i:		; preds = %bb5.i14.i.i.i
-	br label %int2lit.exit.i.i.i.i
+  br label %int2lit.exit.i.i.i.i
 
 bb1.i.i.i18.i.i.i:		; preds = %bb5.i14.i.i.i
-	br label %int2lit.exit.i.i.i.i
+  br label %int2lit.exit.i.i.i.i
 
 int2lit.exit.i.i.i.i:		; preds = %bb1.i.i.i18.i.i.i, %bb.i.i.i17.i.i.i
-	br i1 false, label %bb8.i19.i.i.i.loopexit, label %bb5.i14.i.i.i
+  br i1 false, label %bb8.i19.i.i.i.loopexit, label %bb5.i14.i.i.i
 
 bb8.i19.i.i.i.loopexit:		; preds = %int2lit.exit.i.i.i.i
-	br label %bb8.i19.i.i.i
+  br label %bb8.i19.i.i.i
 
 bb8.i19.i.i.i:		; preds = %bb8.i19.i.i.i.loopexit, %int2lit.exit32.i.i.i.i
-	br i1 false, label %bb5.i.i.i94, label %bb6.i.i.i95
+  br i1 false, label %bb5.i.i.i94, label %bb6.i.i.i95
 
 bb5.i.i.i94:		; preds = %bb8.i19.i.i.i, %bb4.i.i.i90
-	br label %bb.i2.i.i.i
+  br label %bb.i2.i.i.i
 
 bb.i2.i.i.i:		; preds = %hpop.exit.i.i.i.i, %bb5.i.i.i94
-	br i1 false, label %hpop.exit.i.i.i.i, label %bb1.i.i.i.i.i
+  br i1 false, label %hpop.exit.i.i.i.i, label %bb1.i.i.i.i.i
 
 bb1.i.i.i.i.i:		; preds = %bb.i2.i.i.i
-	br label %bb2.i.i.i.i.i
+  br label %bb2.i.i.i.i.i
 
 bb2.i.i.i.i.i:		; preds = %bb11.i.i.i.i.i, %bb1.i.i.i.i.i
-	br i1 false, label %bb3.i.i.i.i.i, label %bb12.i.i.i.i.i
+  br i1 false, label %bb3.i.i.i.i.i, label %bb12.i.i.i.i.i
 
 bb3.i.i.i.i.i:		; preds = %bb2.i.i.i.i.i
-	br i1 false, label %bb4.i.i.i.i.i, label %bb1.i.i.i.i.i.i
+  br i1 false, label %bb4.i.i.i.i.i, label %bb1.i.i.i.i.i.i
 
 bb1.i.i.i.i.i.i:		; preds = %bb3.i.i.i.i.i
-	br i1 false, label %bb8.i.i.i.i.i, label %bb3.i.i.i.i.i.i
+  br i1 false, label %bb8.i.i.i.i.i, label %bb3.i.i.i.i.i.i
 
 bb3.i.i.i.i.i.i:		; preds = %bb1.i.i.i.i.i.i
-	br i1 false, label %bb4.i.i.i.i.i, label %bb8.i.i.i.i.i
+  br i1 false, label %bb4.i.i.i.i.i, label %bb8.i.i.i.i.i
 
 bb4.i.i.i.i.i:		; preds = %bb3.i.i.i.i.i.i, %bb3.i.i.i.i.i
-	br i1 false, label %bb5.i.i.i.i.i, label %bb11.i.i.i.i.i
+  br i1 false, label %bb5.i.i.i.i.i, label %bb11.i.i.i.i.i
 
 bb5.i.i.i.i.i:		; preds = %bb4.i.i.i.i.i
-	br i1 false, label %bb6.i.i.i.i.i, label %bb1.i21.i.i.i.i.i
+  br i1 false, label %bb6.i.i.i.i.i, label %bb1.i21.i.i.i.i.i
 
 bb1.i21.i.i.i.i.i:		; preds = %bb5.i.i.i.i.i
-	br i1 false, label %bb11.i.i.i.i.i, label %bb3.i24.i.i.i.i.i
+  br i1 false, label %bb11.i.i.i.i.i, label %bb3.i24.i.i.i.i.i
 
 bb3.i24.i.i.i.i.i:		; preds = %bb1.i21.i.i.i.i.i
-	br i1 false, label %bb6.i.i.i.i.i, label %bb11.i.i.i.i.i
+  br i1 false, label %bb6.i.i.i.i.i, label %bb11.i.i.i.i.i
 
 bb6.i.i.i.i.i:		; preds = %bb3.i24.i.i.i.i.i, %bb5.i.i.i.i.i
-	br label %bb11.i.i.i.i.i
+  br label %bb11.i.i.i.i.i
 
 bb8.i.i.i.i.i:		; preds = %bb3.i.i.i.i.i.i, %bb1.i.i.i.i.i.i
-	br i1 false, label %bb9.i.i.i.i.i, label %bb12.i.i.i.i.i
+  br i1 false, label %bb9.i.i.i.i.i, label %bb12.i.i.i.i.i
 
 bb9.i.i.i.i.i:		; preds = %bb8.i.i.i.i.i
-	br i1 false, label %bb11.i.i.i.i.i, label %bb1.i8.i.i.i.i.i
+  br i1 false, label %bb11.i.i.i.i.i, label %bb1.i8.i.i.i.i.i
 
 bb1.i8.i.i.i.i.i:		; preds = %bb9.i.i.i.i.i
-	br i1 false, label %bb12.i.i.i.i.i, label %bb3.i11.i.i.i.i.i
+  br i1 false, label %bb12.i.i.i.i.i, label %bb3.i11.i.i.i.i.i
 
 bb3.i11.i.i.i.i.i:		; preds = %bb1.i8.i.i.i.i.i
-	br i1 false, label %bb11.i.i.i.i.i, label %bb12.i.i.i.i.i
+  br i1 false, label %bb11.i.i.i.i.i, label %bb12.i.i.i.i.i
 
 bb11.i.i.i.i.i:		; preds = %bb3.i11.i.i.i.i.i, %bb9.i.i.i.i.i, %bb6.i.i.i.i.i, %bb3.i24.i.i.i.i.i, %bb1.i21.i.i.i.i.i, %bb4.i.i.i.i.i
-	br label %bb2.i.i.i.i.i
+  br label %bb2.i.i.i.i.i
 
 bb12.i.i.i.i.i:		; preds = %bb3.i11.i.i.i.i.i, %bb1.i8.i.i.i.i.i, %bb8.i.i.i.i.i, %bb2.i.i.i.i.i
-	br label %hpop.exit.i.i.i.i
+  br label %hpop.exit.i.i.i.i
 
 hpop.exit.i.i.i.i:		; preds = %bb12.i.i.i.i.i, %bb.i2.i.i.i
-	br i1 false, label %sdecide.exit.i.i.i, label %bb.i2.i.i.i
+  br i1 false, label %sdecide.exit.i.i.i, label %bb.i2.i.i.i
 
 sdecide.exit.i.i.i:		; preds = %hpop.exit.i.i.i.i
-	br label %bb6.i.i.i95
+  br label %bb6.i.i.i95
 
 bb6.i.i.i95:		; preds = %sdecide.exit.i.i.i, %bb8.i19.i.i.i, %bb8.i.i.i.i
-	br label %decide.exit.i.i
+  br label %decide.exit.i.i
 
 decide.exit.i.i:		; preds = %bb6.i.i.i95, %bb1.i23.i.i.i, %bb2.i.i.i87
-	br i1 false, label %bb42.i.i, label %sat.exit.i.loopexit.loopexit2
+  br i1 false, label %bb42.i.i, label %sat.exit.i.loopexit.loopexit2
 
 bb42.i.i:		; preds = %decide.exit.i.i
-	br label %bb13.i.i71.outer
+  br label %bb13.i.i71.outer
 
 sat.exit.i.loopexit.loopexit:		; preds = %bb24.i.i, %bb1.i68.i.i, %incincs.exit.i.i
-	br label %sat.exit.i.loopexit
+  br label %sat.exit.i.loopexit
 
 sat.exit.i.loopexit.loopexit2:		; preds = %decide.exit.i.i, %bb1.i48.i.i, %bb29.i.i
-	br label %sat.exit.i.loopexit
+  br label %sat.exit.i.loopexit
 
 sat.exit.i.loopexit:		; preds = %sat.exit.i.loopexit.loopexit2, %sat.exit.i.loopexit.loopexit
-	br label %sat.exit.i
+  br label %sat.exit.i
 
 sat.exit.i:		; preds = %sat.exit.i.loopexit, %bb1.i61.i.i, %bb8.i.i67, %bb1.i.i.i63, %bb3.i.i59
-	br i1 false, label %bb7.i, label %bb2.i96
+  br i1 false, label %bb7.i, label %bb2.i96
 
 bb2.i96:		; preds = %sat.exit.i
-	switch i32 0, label %bb5.i99 [
-		i32 10, label %bb4.i98
-		i32 20, label %bb6.i100
-	]
+  switch i32 0, label %bb5.i99 [
+  i32 10, label %bb4.i98
+  i32 20, label %bb6.i100
+  ]
 
 bb4.i98:		; preds = %bb2.i96
-	br label %bb6.i100
+  br label %bb6.i100
 
 bb5.i99:		; preds = %bb2.i96
-	br label %bb6.i100
+  br label %bb6.i100
 
 bb6.i100:		; preds = %bb5.i99, %bb4.i98, %bb2.i96
-	br label %bb7.i
+  br label %bb7.i
 
 bb7.i:		; preds = %bb6.i100, %sat.exit.i
-	br i1 false, label %bb.i1.i, label %picosat_sat.exit
+  br i1 false, label %bb.i1.i, label %picosat_sat.exit
 
 bb.i1.i:		; preds = %bb7.i
-	br label %picosat_sat.exit
+  br label %picosat_sat.exit
 
 picosat_sat.exit:		; preds = %bb.i1.i, %bb7.i
-	switch i32 0, label %bb166 [
-		i32 20, label %bb150
-		i32 10, label %bb163
-	]
+  switch i32 0, label %bb166 [
+  i32 20, label %bb150
+  i32 10, label %bb163
+  ]
 
 bb150:		; preds = %picosat_sat.exit
-	br i1 false, label %bb152, label %bb151
+  br i1 false, label %bb152, label %bb151
 
 bb151:		; preds = %bb150
-	br label %bb152
+  br label %bb152
 
 bb152:		; preds = %bb151, %bb150
-	br i1 false, label %bb154, label %bb153
+  br i1 false, label %bb154, label %bb153
 
 bb153:		; preds = %bb152
-	br label %bb154
+  br label %bb154
 
 bb154:		; preds = %bb153, %bb152
-	br i1 false, label %bb157, label %bb156
+  br i1 false, label %bb157, label %bb156
 
 bb156:		; preds = %bb154
-	br label %bb157
+  br label %bb157
 
 bb157:		; preds = %bb156, %bb154
-	br i1 false, label %bb159, label %bb158
+  br i1 false, label %bb159, label %bb158
 
 bb158:		; preds = %bb157
-	br label %bb159
+  br label %bb159
 
 bb159:		; preds = %bb158, %bb157
-	br i1 false, label %bb167, label %bb160
+  br i1 false, label %bb167, label %bb160
 
 bb160:		; preds = %bb159
-	br label %bb167
+  br label %bb167
 
 bb163:		; preds = %picosat_sat.exit
-	br i1 false, label %bb167, label %bb164
+  br i1 false, label %bb167, label %bb164
 
 bb164:		; preds = %bb163
-	br label %bb4.i
+  br label %bb4.i
 
 bb.i11:		; preds = %bb4.i
-	br i1 false, label %bb.i.i12, label %bb1.i.i14
+  br i1 false, label %bb.i.i12, label %bb1.i.i14
 
 bb.i.i12:		; preds = %bb.i11
-	unreachable
+  unreachable
 
 bb1.i.i14:		; preds = %bb.i11
-	br i1 false, label %bb3.i.i16, label %bb2.i.i15
+  br i1 false, label %bb3.i.i16, label %bb2.i.i15
 
 bb2.i.i15:		; preds = %bb1.i.i14
-	unreachable
+  unreachable
 
 bb3.i.i16:		; preds = %bb1.i.i14
-	br i1 false, label %bb3.i, label %bb7.i.i
+  br i1 false, label %bb3.i, label %bb7.i.i
 
 bb7.i.i:		; preds = %bb3.i.i16
-	br i1 false, label %bb.i.i.i.i17, label %bb1.i.i.i.i18
+  br i1 false, label %bb.i.i.i.i17, label %bb1.i.i.i.i18
 
 bb.i.i.i.i17:		; preds = %bb7.i.i
-	br label %int2lit.exit.i.i
+  br label %int2lit.exit.i.i
 
 bb1.i.i.i.i18:		; preds = %bb7.i.i
-	br label %int2lit.exit.i.i
+  br label %int2lit.exit.i.i
 
 int2lit.exit.i.i:		; preds = %bb1.i.i.i.i18, %bb.i.i.i.i17
-	br i1 false, label %bb3.i, label %bb9.i.i
+  br i1 false, label %bb3.i, label %bb9.i.i
 
 bb9.i.i:		; preds = %int2lit.exit.i.i
-	br label %bb3.i
+  br label %bb3.i
 
 bb3.i:		; preds = %bb9.i.i, %int2lit.exit.i.i, %bb3.i.i16
-	br label %bb4.i
+  br label %bb4.i
 
 bb4.i:		; preds = %bb3.i, %bb164
-	br i1 false, label %bb5.i, label %bb.i11
+  br i1 false, label %bb5.i, label %bb.i11
 
 bb5.i:		; preds = %bb4.i
-	br i1 false, label %bb6.i, label %bb167
+  br i1 false, label %bb6.i, label %bb167
 
 bb6.i:		; preds = %bb5.i
-	br label %bb167
+  br label %bb167
 
 bb166:		; preds = %picosat_sat.exit
-	br label %bb167
+  br label %bb167
 
 bb167:		; preds = %bb166, %bb6.i, %bb5.i, %bb163, %bb160, %bb159, %picosat_print.exit
-	br i1 false, label %bb168, label %bb170
+  br i1 false, label %bb168, label %bb170
 
 bb168:		; preds = %bb167
-	br i1 false, label %bb170, label %bb169
+  br i1 false, label %bb170, label %bb169
 
 bb169:		; preds = %bb168
-	br i1 false, label %bb.i7, label %picosat_time_stamp.exit9
+  br i1 false, label %bb.i7, label %picosat_time_stamp.exit9
 
 bb.i7:		; preds = %bb169
-	br label %picosat_time_stamp.exit9
+  br label %picosat_time_stamp.exit9
 
 picosat_time_stamp.exit9:		; preds = %bb.i7, %bb169
-	br label %bb170
+  br label %bb170
 
 bb170:		; preds = %picosat_time_stamp.exit9, %bb168, %bb167, %bb129
-	br i1 false, label %bb.i.i3, label %picosat_leave.exit
+  br i1 false, label %bb.i.i3, label %picosat_leave.exit
 
 bb.i.i3:		; preds = %bb170
-	br label %picosat_leave.exit
+  br label %picosat_leave.exit
 
 picosat_leave.exit:		; preds = %bb.i.i3, %bb170
-	br i1 false, label %bb1.i.i, label %bb.i.i
+  br i1 false, label %bb1.i.i, label %bb.i.i
 
 bb.i.i:		; preds = %picosat_leave.exit
-	unreachable
+  unreachable
 
 bb1.i.i:		; preds = %picosat_leave.exit
-	br label %bb9.i.i.i
+  br label %bb9.i.i.i
 
 bb3.i.i.i:		; preds = %bb9.i.i.i
-	br i1 false, label %bb5.i.i.i, label %bb4.i.i.i
+  br i1 false, label %bb5.i.i.i, label %bb4.i.i.i
 
 bb4.i.i.i:		; preds = %bb3.i.i.i
-	br label %bb5.i.i.i
+  br label %bb5.i.i.i
 
 bb5.i.i.i:		; preds = %bb4.i.i.i, %bb3.i.i.i
-	br label %bb9.i.i.i
+  br label %bb9.i.i.i
 
 bb9.i.i.i:		; preds = %bb5.i.i.i, %bb1.i.i
-	br i1 false, label %bb10.i.i.i, label %bb3.i.i.i
+  br i1 false, label %bb10.i.i.i, label %bb3.i.i.i
 
 bb10.i.i.i:		; preds = %bb9.i.i.i
-	br i1 false, label %delete.exit.i.i.i, label %bb1.i.i.i.i
+  br i1 false, label %delete.exit.i.i.i, label %bb1.i.i.i.i
 
 bb1.i.i.i.i:		; preds = %bb10.i.i.i
-	br label %delete.exit.i.i.i
+  br label %delete.exit.i.i.i
 
 delete.exit.i.i.i:		; preds = %bb1.i.i.i.i, %bb10.i.i.i
-	br i1 false, label %delete_clauses.exit.i.i, label %bb1.i7.i.i.i
+  br i1 false, label %delete_clauses.exit.i.i, label %bb1.i7.i.i.i
 
 bb1.i7.i.i.i:		; preds = %delete.exit.i.i.i
-	br label %delete_clauses.exit.i.i
+  br label %delete_clauses.exit.i.i
 
 delete_clauses.exit.i.i:		; preds = %bb1.i7.i.i.i, %delete.exit.i.i.i
-	br label %bb3.i.i
+  br label %bb3.i.i
 
 bb2.i.i:		; preds = %bb3.i.i
-	br i1 false, label %lrelease.exit.i.i, label %bb1.i.i23.i.i
+  br i1 false, label %lrelease.exit.i.i, label %bb1.i.i23.i.i
 
 bb1.i.i23.i.i:		; preds = %bb2.i.i
-	br label %lrelease.exit.i.i
+  br label %lrelease.exit.i.i
 
 lrelease.exit.i.i:		; preds = %bb1.i.i23.i.i, %bb2.i.i
-	br label %bb3.i.i
+  br label %bb3.i.i
 
 bb3.i.i:		; preds = %lrelease.exit.i.i, %delete_clauses.exit.i.i
-	br i1 false, label %bb4.i.i, label %bb2.i.i
+  br i1 false, label %bb4.i.i, label %bb2.i.i
 
 bb4.i.i:		; preds = %bb3.i.i
-	br i1 false, label %delete.exit214.i.i, label %bb1.i208.i.i
+  br i1 false, label %delete.exit214.i.i, label %bb1.i208.i.i
 
 bb1.i208.i.i:		; preds = %bb4.i.i
-	br label %delete.exit214.i.i
+  br label %delete.exit214.i.i
 
 delete.exit214.i.i:		; preds = %bb1.i208.i.i, %bb4.i.i
-	br i1 false, label %delete.exit203.i.i, label %bb1.i197.i.i
+  br i1 false, label %delete.exit203.i.i, label %bb1.i197.i.i
 
 bb1.i197.i.i:		; preds = %delete.exit214.i.i
-	br label %delete.exit203.i.i
+  br label %delete.exit203.i.i
 
 delete.exit203.i.i:		; preds = %bb1.i197.i.i, %delete.exit214.i.i
-	br i1 false, label %delete.exit192.i.i, label %bb1.i186.i.i
+  br i1 false, label %delete.exit192.i.i, label %bb1.i186.i.i
 
 bb1.i186.i.i:		; preds = %delete.exit203.i.i
-	br label %delete.exit192.i.i
+  br label %delete.exit192.i.i
 
 delete.exit192.i.i:		; preds = %bb1.i186.i.i, %delete.exit203.i.i
-	br i1 false, label %delete.exit181.i.i, label %bb1.i175.i.i
+  br i1 false, label %delete.exit181.i.i, label %bb1.i175.i.i
 
 bb1.i175.i.i:		; preds = %delete.exit192.i.i
-	br label %delete.exit181.i.i
+  br label %delete.exit181.i.i
 
 delete.exit181.i.i:		; preds = %bb1.i175.i.i, %delete.exit192.i.i
-	br i1 false, label %delete.exit170.i.i, label %bb1.i164.i.i
+  br i1 false, label %delete.exit170.i.i, label %bb1.i164.i.i
 
 bb1.i164.i.i:		; preds = %delete.exit181.i.i
-	br label %delete.exit170.i.i
+  br label %delete.exit170.i.i
 
 delete.exit170.i.i:		; preds = %bb1.i164.i.i, %delete.exit181.i.i
-	br i1 false, label %delete.exit159.i.i, label %bb1.i153.i.i
+  br i1 false, label %delete.exit159.i.i, label %bb1.i153.i.i
 
 bb1.i153.i.i:		; preds = %delete.exit170.i.i
-	br label %delete.exit159.i.i
+  br label %delete.exit159.i.i
 
 delete.exit159.i.i:		; preds = %bb1.i153.i.i, %delete.exit170.i.i
-	br i1 false, label %delete.exit148.i.i, label %bb1.i142.i.i
+  br i1 false, label %delete.exit148.i.i, label %bb1.i142.i.i
 
 bb1.i142.i.i:		; preds = %delete.exit159.i.i
-	br label %delete.exit148.i.i
+  br label %delete.exit148.i.i
 
 delete.exit148.i.i:		; preds = %bb1.i142.i.i, %delete.exit159.i.i
-	br i1 false, label %delete.exit137.i.i, label %bb1.i131.i.i
+  br i1 false, label %delete.exit137.i.i, label %bb1.i131.i.i
 
 bb1.i131.i.i:		; preds = %delete.exit148.i.i
-	br label %delete.exit137.i.i
+  br label %delete.exit137.i.i
 
 delete.exit137.i.i:		; preds = %bb1.i131.i.i, %delete.exit148.i.i
-	br i1 false, label %delete.exit126.i.i, label %bb1.i120.i.i
+  br i1 false, label %delete.exit126.i.i, label %bb1.i120.i.i
 
 bb1.i120.i.i:		; preds = %delete.exit137.i.i
-	br label %delete.exit126.i.i
+  br label %delete.exit126.i.i
 
 delete.exit126.i.i:		; preds = %bb1.i120.i.i, %delete.exit137.i.i
-	br i1 false, label %delete.exit115.i.i, label %bb1.i109.i.i
+  br i1 false, label %delete.exit115.i.i, label %bb1.i109.i.i
 
 bb1.i109.i.i:		; preds = %delete.exit126.i.i
-	br label %delete.exit115.i.i
+  br label %delete.exit115.i.i
 
 delete.exit115.i.i:		; preds = %bb1.i109.i.i, %delete.exit126.i.i
-	br i1 false, label %delete.exit104.i.i, label %bb1.i98.i.i
+  br i1 false, label %delete.exit104.i.i, label %bb1.i98.i.i
 
 bb1.i98.i.i:		; preds = %delete.exit115.i.i
-	br label %delete.exit104.i.i
+  br label %delete.exit104.i.i
 
 delete.exit104.i.i:		; preds = %bb1.i98.i.i, %delete.exit115.i.i
-	br i1 false, label %delete.exit93.i.i, label %bb1.i87.i.i
+  br i1 false, label %delete.exit93.i.i, label %bb1.i87.i.i
 
 bb1.i87.i.i:		; preds = %delete.exit104.i.i
-	br label %delete.exit93.i.i
+  br label %delete.exit93.i.i
 
 delete.exit93.i.i:		; preds = %bb1.i87.i.i, %delete.exit104.i.i
-	br i1 false, label %delete.exit82.i.i, label %bb1.i76.i.i
+  br i1 false, label %delete.exit82.i.i, label %bb1.i76.i.i
 
 bb1.i76.i.i:		; preds = %delete.exit93.i.i
-	br label %delete.exit82.i.i
+  br label %delete.exit82.i.i
 
 delete.exit82.i.i:		; preds = %bb1.i76.i.i, %delete.exit93.i.i
-	br i1 false, label %delete.exit71.i.i, label %bb1.i65.i.i
+  br i1 false, label %delete.exit71.i.i, label %bb1.i65.i.i
 
 bb1.i65.i.i:		; preds = %delete.exit82.i.i
-	br label %delete.exit71.i.i
+  br label %delete.exit71.i.i
 
 delete.exit71.i.i:		; preds = %bb1.i65.i.i, %delete.exit82.i.i
-	br i1 false, label %delete.exit60.i.i, label %bb1.i54.i.i
+  br i1 false, label %delete.exit60.i.i, label %bb1.i54.i.i
 
 bb1.i54.i.i:		; preds = %delete.exit71.i.i
-	br label %delete.exit60.i.i
+  br label %delete.exit60.i.i
 
 delete.exit60.i.i:		; preds = %bb1.i54.i.i, %delete.exit71.i.i
-	br i1 false, label %delete.exit38.i.i, label %bb1.i32.i.i
+  br i1 false, label %delete.exit38.i.i, label %bb1.i32.i.i
 
 bb1.i32.i.i:		; preds = %delete.exit60.i.i
-	br label %delete.exit38.i.i
+  br label %delete.exit38.i.i
 
 delete.exit38.i.i:		; preds = %bb1.i32.i.i, %delete.exit60.i.i
-	br i1 false, label %delete.exit18.i.i, label %bb1.i12.i.i
+  br i1 false, label %delete.exit18.i.i, label %bb1.i12.i.i
 
 bb1.i12.i.i:		; preds = %delete.exit38.i.i
-	br label %delete.exit18.i.i
+  br label %delete.exit18.i.i
 
 delete.exit18.i.i:		; preds = %bb1.i12.i.i, %delete.exit38.i.i
-	br i1 false, label %picosat_reset.exit, label %bb1.i2.i.i
+  br i1 false, label %picosat_reset.exit, label %bb1.i2.i.i
 
 bb1.i2.i.i:		; preds = %delete.exit18.i.i
-	br label %picosat_reset.exit
+  br label %picosat_reset.exit
 
 picosat_reset.exit:		; preds = %bb1.i2.i.i, %delete.exit18.i.i
-	br label %bb171
+  br label %bb171
 
 bb171:		; preds = %picosat_reset.exit, %bb110
-	br i1 false, label %bb173, label %bb172
+  br i1 false, label %bb173, label %bb172
 
 bb172:		; preds = %bb171
-	br label %bb173
+  br label %bb173
 
 bb173:		; preds = %bb172, %bb171
-	br i1 false, label %bb175, label %bb174
+  br i1 false, label %bb175, label %bb174
 
 bb174:		; preds = %bb173
-	br label %bb175
+  br label %bb175
 
 bb175:		; preds = %bb174, %bb173
-	br i1 false, label %bb177, label %bb176
+  br i1 false, label %bb177, label %bb176
 
 bb176:		; preds = %bb175
-	br label %bb177
+  br label %bb177
 
 bb177:		; preds = %bb176, %bb175
-	br i1 false, label %bb179, label %bb178
+  br i1 false, label %bb179, label %bb178
 
 bb178:		; preds = %bb177
-	ret i32 0
+  ret i32 0
 
 bb179:		; preds = %bb177
-	ret i32 0
+  ret i32 0
 }
 
 define i32 @main(i32 %argc, i8** %argv) nounwind {
 entry:
-	br label %bb2
+  br label %bb2
 
 bb:		; preds = %bb2
-	br i1 false, label %bb3, label %bb2
+  br i1 false, label %bb3, label %bb2
 
 bb2:		; preds = %bb, %entry
-	br i1 false, label %bb5.loopexit, label %bb
+  br i1 false, label %bb5.loopexit, label %bb
 
 bb3:		; preds = %bb
-	br i1 false, label %bb5, label %bb4
+  br i1 false, label %bb5, label %bb4
 
 bb4:		; preds = %bb3
-	br label %bb5
+  br label %bb5
 
 bb5.loopexit:		; preds = %bb2
-	br label %bb5
+  br label %bb5
 
 bb5:		; preds = %bb5.loopexit, %bb4, %bb3
-	%0 = call fastcc i32 @picosat_main(i32 %argc, i8** %argv) nounwind		; <i32> [#uses=2]
-	br i1 false, label %bb7, label %bb6
+  %0 = call fastcc i32 @picosat_main(i32 %argc, i8** %argv) nounwind		; <i32> [#uses=2]
+  br i1 false, label %bb7, label %bb6
 
 bb6:		; preds = %bb5
-	ret i32 %0
+  ret i32 %0
 
 bb7:		; preds = %bb5
-	ret i32 %0
+  ret i32 %0
 }
diff --git a/llvm/test/Transforms/ArgumentPromotion/pr32917.ll b/llvm/test/Transforms/ArgumentPromotion/pr32917.ll
index a2aeac081..34c025072 100644
--- a/llvm/test/Transforms/ArgumentPromotion/pr32917.ll
+++ b/llvm/test/Transforms/ArgumentPromotion/pr32917.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
 ; RUN: opt < %s -argpromotion -S | FileCheck %s
 ; PR 32917
 
@@ -5,6 +6,15 @@
 @a = common local_unnamed_addr global i32 0, align 4
 
 define i32 @fn2() local_unnamed_addr {
+; CHECK-LABEL: define {{[^@]+}}@fn2() local_unnamed_addr
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* @b, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = sext i32 [[TMP1]] to i64
+; CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[TMP2]] to i32*
+; CHECK-NEXT:    [[DOTIDX:%.*]] = getelementptr i32, i32* [[TMP3]], i64 -1
+; CHECK-NEXT:    [[DOTIDX_VAL:%.*]] = load i32, i32* [[DOTIDX]], align 4
+; CHECK-NEXT:    call fastcc void @fn1(i32 [[DOTIDX_VAL]])
+; CHECK-NEXT:    ret i32 undef
+;
   %1 = load i32, i32* @b, align 4
   %2 = sext i32 %1 to i64
   %3 = inttoptr i64 %2 to i32*
@@ -13,11 +23,13 @@ define i32 @fn2() local_unnamed_addr {
 }
 
 define internal fastcc void @fn1(i32* nocapture readonly) unnamed_addr {
+; CHECK-LABEL: define {{[^@]+}}@fn1
+; CHECK-SAME: (i32 [[DOT18446744073709551615_VAL:%.*]]) unnamed_addr
+; CHECK-NEXT:    store i32 [[DOT18446744073709551615_VAL]], i32* @a, align 4
+; CHECK-NEXT:    ret void
+;
   %2 = getelementptr inbounds i32, i32* %0, i64 -1
   %3 = load i32, i32* %2, align 4
   store i32 %3, i32* @a, align 4
   ret void
 }
-
-; CHECK: getelementptr {{.*}} -1
-; CHECK-NOT: getelementptr {{.*}} 4294967295
diff --git a/llvm/test/Transforms/ArgumentPromotion/pr33641_remove_arg_dbgvalue.ll b/llvm/test/Transforms/ArgumentPromotion/pr33641_remove_arg_dbgvalue.ll
index ad22bed03..4c2503d8b 100644
--- a/llvm/test/Transforms/ArgumentPromotion/pr33641_remove_arg_dbgvalue.ll
+++ b/llvm/test/Transforms/ArgumentPromotion/pr33641_remove_arg_dbgvalue.ll
@@ -1,18 +1,29 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
 ; RUN: opt -argpromotion -verify -dse -S %s -o - | FileCheck %s
 
 ; Fix for PR33641. ArgumentPromotion removed the argument to bar but left the call to
 ; dbg.value which still used the removed argument.
 
+; The %p argument should be removed, and the use of it in dbg.value should be
+; changed to undef.
+
 %p_t = type i16*
 %fun_t = type void (%p_t)*
 
 define void @foo() {
+; CHECK-LABEL: define {{[^@]+}}@foo()
+; CHECK-NEXT:    ret void
+;
   %tmp = alloca %fun_t
   store %fun_t @bar, %fun_t* %tmp
   ret void
 }
 
 define internal void @bar(%p_t %p)  {
+; CHECK-LABEL: define {{[^@]+}}@bar()
+; CHECK-NEXT:    call void @llvm.dbg.value(metadata i16* undef, metadata !3, metadata !DIExpression()), !dbg !5
+; CHECK-NEXT:    ret void
+;
   call void @llvm.dbg.value(metadata %p_t %p, metadata !4, metadata !5), !dbg !6
   ret void
 }
@@ -29,10 +40,3 @@ declare void @llvm.dbg.value(metadata, metadata, metadata)
 !4 = !DILocalVariable(name: "p", scope: !3)
 !5 = !DIExpression()
 !6 = !DILocation(line: 1, column: 1, scope: !3)
-
-; The %p argument should be removed, and the use of it in dbg.value should be
-; changed to undef.
-; CHECK:      define internal void @bar() {
-; CHECK-NEXT:   call void @llvm.dbg.value(metadata i16* undef
-; CHECK-NEXT:   ret void
-; CHECK-NEXT: }
diff --git a/llvm/test/Transforms/ArgumentPromotion/profile.ll b/llvm/test/Transforms/ArgumentPromotion/profile.ll
index f667f9ea2..f4bceb3eb 100644
--- a/llvm/test/Transforms/ArgumentPromotion/profile.ll
+++ b/llvm/test/Transforms/ArgumentPromotion/profile.ll
@@ -1,17 +1,26 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
 ; RUN: opt -argpromotion -mem2reg -S < %s | FileCheck %s
 target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
 
 ; Checks if !prof metadata is corret in deadargelim.
 
 define void @caller() #0 {
+; CHECK-LABEL: define {{[^@]+}}@caller()
+; CHECK-NEXT:    call void @promote_i32_ptr(i32 42), !prof !0
+; CHECK-NEXT:    ret void
+;
   %x = alloca i32
   store i32 42, i32* %x
   call void @promote_i32_ptr(i32* %x), !prof !0
-; CHECK: call void @promote_i32_ptr(i32 42), !prof ![[PROF:[0-9]]]
   ret void
 }
 
 define internal void @promote_i32_ptr(i32* %xp) {
+; CHECK-LABEL: define {{[^@]+}}@promote_i32_ptr
+; CHECK-SAME: (i32 [[XP_VAL:%.*]])
+; CHECK-NEXT:    call void @use_i32(i32 [[XP_VAL]])
+; CHECK-NEXT:    ret void
+;
   %x = load i32, i32* %xp
   call void @use_i32(i32 %x)
   ret void
@@ -19,5 +28,4 @@ define internal void @promote_i32_ptr(i32* %xp) {
 
 declare void @use_i32(i32)
 
-; CHECK: ![[PROF]] = !{!"branch_weights", i32 30}
 !0 = !{!"branch_weights", i32 30}
diff --git a/llvm/test/Transforms/ArgumentPromotion/reserve-tbaa.ll b/llvm/test/Transforms/ArgumentPromotion/reserve-tbaa.ll
index 3a3aa44b2..ed76f59cb 100644
--- a/llvm/test/Transforms/ArgumentPromotion/reserve-tbaa.ll
+++ b/llvm/test/Transforms/ArgumentPromotion/reserve-tbaa.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
 ; RUN: opt < %s -argpromotion -S | FileCheck %s
 ; RUN: opt < %s -passes=argpromotion -S | FileCheck %s
 
@@ -14,6 +15,14 @@
 @d = global i8 0, align 1
 
 define internal fastcc void @fn(i32* nocapture readonly %p1, i64* nocapture readonly %p2) {
+; CHECK-LABEL: define {{[^@]+}}@fn
+; CHECK-SAME: (i32 [[P1_VAL:%.*]], i64 [[P2_VAL:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CONV:%.*]] = trunc i64 [[P2_VAL]] to i32
+; CHECK-NEXT:    [[CONV1:%.*]] = trunc i32 [[P1_VAL]] to i8
+; CHECK-NEXT:    store i8 [[CONV1]], i8* @d, align 1, !tbaa !0
+; CHECK-NEXT:    ret void
+;
 entry:
   %0 = load i64, i64* %p2, align 8, !tbaa !1
   %conv = trunc i64 %0 to i32
@@ -24,11 +33,18 @@ entry:
 }
 
 define i32 @main() {
+; CHECK-LABEL: define {{[^@]+}}@main()
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32**, i32*** @e, align 8, !tbaa !3
+; CHECK-NEXT:    store i32* @g, i32** [[TMP0]], align 8, !tbaa !3
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32*, i32** @a, align 8, !tbaa !3
+; CHECK-NEXT:    store i32 1, i32* [[TMP1]], align 4, !tbaa !5
+; CHECK-NEXT:    [[G_VAL:%.*]] = load i32, i32* @g, align 4, !tbaa !5
+; CHECK-NEXT:    [[C_VAL:%.*]] = load i64, i64* @c, align 8, !tbaa !7
+; CHECK-NEXT:    call fastcc void @fn(i32 [[G_VAL]], i64 [[C_VAL]])
+; CHECK-NEXT:    ret i32 0
+;
 entry:
-; CHECK-LABEL: main
-; CHECK: store i32 1, i32* %{{.*}}, align 4, !tbaa ![[I32:[0-9]+]]
-; CHECK: %g.val = load i32, i32* @g, align 4, !tbaa ![[I32]]
-; CHECK: %c.val = load i64, i64* @c, align 8, !tbaa ![[LONG:[0-9]+]]
   %0 = load i32**, i32*** @e, align 8, !tbaa !8
   store i32* @g, i32** %0, align 8, !tbaa !8
   %1 = load i32*, i32** @a, align 8, !tbaa !8
@@ -47,7 +63,4 @@ entry:
 !7 = !{!3, !3, i64 0}
 !8 = !{!9, !9, i64 0}
 !9 = !{!"any pointer", !3, i64 0}
-; CHECK: ![[I32]] = !{![[I32_TYPE:[0-9]+]], ![[I32_TYPE]], i64 0}
-; CHECK: ![[I32_TYPE]] = !{!"int", !{{.*}}, i64 0}
-; CHECK: ![[LONG]] = !{![[LONG_TYPE:[0-9]+]], ![[LONG_TYPE]], i64 0}
-; CHECK: ![[LONG_TYPE]] = !{!"long", !{{.*}}, i64 0}
+
diff --git a/llvm/test/Transforms/ArgumentPromotion/sret.ll b/llvm/test/Transforms/ArgumentPromotion/sret.ll
index 55fc036f1..c4328e41b 100644
--- a/llvm/test/Transforms/ArgumentPromotion/sret.ll
+++ b/llvm/test/Transforms/ArgumentPromotion/sret.ll
@@ -1,29 +1,40 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
 ; RUN: opt < %s -argpromotion -S | FileCheck %s
 ; RUN: opt < %s -passes=argpromotion -S | FileCheck %s
 
 target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-pc-windows-msvc"
 
-; CHECK: define internal void @add(i32 %[[THIS1:.*]], i32 %[[THIS2:.*]], i32* noalias %[[SR:.*]])
 define internal void @add({i32, i32}* %this, i32* sret %r) {
+; CHECK-LABEL: define {{[^@]+}}@add
+; CHECK-SAME: (i32 [[THIS_0_0_VAL:%.*]], i32 [[THIS_0_1_VAL:%.*]], i32* noalias [[R:%.*]])
+; CHECK-NEXT:    [[AB:%.*]] = add i32 [[THIS_0_0_VAL]], [[THIS_0_1_VAL]]
+; CHECK-NEXT:    store i32 [[AB]], i32* [[R]]
+; CHECK-NEXT:    ret void
+;
   %ap = getelementptr {i32, i32}, {i32, i32}* %this, i32 0, i32 0
   %bp = getelementptr {i32, i32}, {i32, i32}* %this, i32 0, i32 1
   %a = load i32, i32* %ap
   %b = load i32, i32* %bp
-  ; CHECK: %[[AB:.*]] = add i32 %[[THIS1]], %[[THIS2]]
   %ab = add i32 %a, %b
-  ; CHECK: store i32 %[[AB]], i32* %[[SR]]
   store i32 %ab, i32* %r
   ret void
 }
 
-; CHECK: define void @f()
 define void @f() {
-  ; CHECK: %[[R:.*]] = alloca i32
+; CHECK-LABEL: define {{[^@]+}}@f()
+; CHECK-NEXT:    [[R:%.*]] = alloca i32
+; CHECK-NEXT:    [[PAIR:%.*]] = alloca { i32, i32 }
+; CHECK-NEXT:    [[PAIR_IDX:%.*]] = getelementptr { i32, i32 }, { i32, i32 }* [[PAIR]], i64 0, i32 0
+; CHECK-NEXT:    [[PAIR_IDX_VAL:%.*]] = load i32, i32* [[PAIR_IDX]]
+; CHECK-NEXT:    [[PAIR_IDX1:%.*]] = getelementptr { i32, i32 }, { i32, i32 }* [[PAIR]], i64 0, i32 1
+; CHECK-NEXT:    [[PAIR_IDX1_VAL:%.*]] = load i32, i32* [[PAIR_IDX1]]
+; CHECK-NEXT:    call void @add(i32 [[PAIR_IDX_VAL]], i32 [[PAIR_IDX1_VAL]], i32* noalias [[R]])
+; CHECK-NEXT:    ret void
+;
   %r = alloca i32
   %pair = alloca {i32, i32}
 
-  ; CHECK: call void @add(i32 %{{.*}}, i32 %{{.*}}, i32* noalias %[[R]])
   call void @add({i32, i32}* %pair, i32* sret %r)
   ret void
 }
diff --git a/llvm/test/Transforms/ArgumentPromotion/tail.ll b/llvm/test/Transforms/ArgumentPromotion/tail.ll
index 93de60afe..f4bb06a6e 100644
--- a/llvm/test/Transforms/ArgumentPromotion/tail.ll
+++ b/llvm/test/Transforms/ArgumentPromotion/tail.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
 ; RUN: opt %s -argpromotion -S -o - | FileCheck %s
 ; RUN: opt %s -passes=argpromotion -S -o - | FileCheck %s
 ; PR14710
@@ -9,15 +10,30 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 declare i8* @foo(%pair*)
 
 define internal void @bar(%pair* byval %Data) {
-; CHECK: define internal void @bar(i32 %Data.0, i32 %Data.1)
-; CHECK: %Data = alloca %pair
-; CHECK-NOT: tail
-; CHECK: call i8* @foo(%pair* %Data)
+; CHECK-LABEL: define {{[^@]+}}@bar
+; CHECK-SAME: (i32 [[DATA_0:%.*]], i32 [[DATA_1:%.*]])
+; CHECK-NEXT:    [[DATA:%.*]] = alloca [[PAIR:%.*]]
+; CHECK-NEXT:    [[DOT0:%.*]] = getelementptr [[PAIR]], %pair* [[DATA]], i32 0, i32 0
+; CHECK-NEXT:    store i32 [[DATA_0]], i32* [[DOT0]]
+; CHECK-NEXT:    [[DOT1:%.*]] = getelementptr [[PAIR]], %pair* [[DATA]], i32 0, i32 1
+; CHECK-NEXT:    store i32 [[DATA_1]], i32* [[DOT1]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8* @foo(%pair* [[DATA]])
+; CHECK-NEXT:    ret void
+;
   tail call i8* @foo(%pair* %Data)
   ret void
 }
 
 define void @zed(%pair* byval %Data) {
+; CHECK-LABEL: define {{[^@]+}}@zed
+; CHECK-SAME: (%pair* byval [[DATA:%.*]])
+; CHECK-NEXT:    [[DATA_0:%.*]] = getelementptr [[PAIR:%.*]], %pair* [[DATA]], i32 0, i32 0
+; CHECK-NEXT:    [[DATA_0_VAL:%.*]] = load i32, i32* [[DATA_0]]
+; CHECK-NEXT:    [[DATA_1:%.*]] = getelementptr [[PAIR]], %pair* [[DATA]], i32 0, i32 1
+; CHECK-NEXT:    [[DATA_1_VAL:%.*]] = load i32, i32* [[DATA_1]]
+; CHECK-NEXT:    call void @bar(i32 [[DATA_0_VAL]], i32 [[DATA_1_VAL]])
+; CHECK-NEXT:    ret void
+;
   call void @bar(%pair* byval %Data)
   ret void
 }
diff --git a/llvm/test/Transforms/ArgumentPromotion/variadic.ll b/llvm/test/Transforms/ArgumentPromotion/variadic.ll
index 034f85388..2b35c5ace 100644
--- a/llvm/test/Transforms/ArgumentPromotion/variadic.ll
+++ b/llvm/test/Transforms/ArgumentPromotion/variadic.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
 ; RUN: opt < %s -argpromotion -S | FileCheck %s
 ; RUN: opt < %s -passes=argpromotion -S | FileCheck %s
 
@@ -15,6 +16,12 @@ target triple = "x86_64-unknown-linux-gnu"
 
 ; Function Attrs: nounwind uwtable
 define i32 @main(i32 %argc, i8** nocapture readnone %argv) #0 {
+; CHECK-LABEL: define {{[^@]+}}@main
+; CHECK-SAME: (i32 [[ARGC:%.*]], i8** nocapture readnone [[ARGV:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    tail call void (i8*, i8*, i8*, i8*, i8*, ...) @callee_t0f(i8* undef, i8* undef, i8* undef, i8* undef, i8* undef, %struct.tt0* byval align 8 @t45)
+; CHECK-NEXT:    ret i32 0
+;
 entry:
   tail call void (i8*, i8*, i8*, i8*, i8*, ...) @callee_t0f(i8* undef, i8* undef, i8* undef, i8* undef, i8* undef, %struct.tt0* byval align 8 @t45)
   ret i32 0
@@ -22,8 +29,11 @@ entry:
 
 ; Function Attrs: nounwind uwtable
 define internal void @callee_t0f(i8* nocapture readnone %tp13, i8* nocapture readnone %tp14, i8* nocapture readnone %tp15, i8* nocapture readnone %tp16, i8* nocapture readnone %tp17, ...) {
+; CHECK-LABEL: define {{[^@]+}}@callee_t0f
+; CHECK-SAME: (i8* nocapture readnone [[TP13:%.*]], i8* nocapture readnone [[TP14:%.*]], i8* nocapture readnone [[TP15:%.*]], i8* nocapture readnone [[TP16:%.*]], i8* nocapture readnone [[TP17:%.*]], ...)
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret void
+;
 entry:
   ret void
 }
-
-; CHECK-LABEL: define internal void @callee_t0f(i8* nocapture readnone %tp13, i8* nocapture readnone %tp14, i8* nocapture readnone %tp15, i8* nocapture readnone %tp16, i8* nocapture readnone %tp17, ...)
diff --git a/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-fadd.ll b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-fadd.ll
index f96cd0b29..42ff7aa32 100644
--- a/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-fadd.ll
+++ b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-fadd.ll
@@ -1,6 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=hawaii -atomic-expand %s | FileCheck -check-prefix=CI %s
 ; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -atomic-expand %s | FileCheck -check-prefix=GFX9 %s
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 -atomic-expand %s | FileCheck -check-prefix=GFX908 %s
 
 define float @test_atomicrmw_fadd_f32_flat(float* %ptr, float %value) {
 ; CI-LABEL: @test_atomicrmw_fadd_f32_flat(
@@ -36,6 +37,23 @@ define float @test_atomicrmw_fadd_f32_flat(float* %ptr, float %value) {
 ; GFX9-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
 ; GFX9:       atomicrmw.end:
 ; GFX9-NEXT:    ret float [[TMP6]]
+;
+; GFX908-LABEL: @test_atomicrmw_fadd_f32_flat(
+; GFX908-NEXT:    [[TMP1:%.*]] = load float, float* [[PTR:%.*]], align 4
+; GFX908-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX908:       atomicrmw.start:
+; GFX908-NEXT:    [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX908-NEXT:    [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]
+; GFX908-NEXT:    [[TMP2:%.*]] = bitcast float* [[PTR]] to i32*
+; GFX908-NEXT:    [[TMP3:%.*]] = bitcast float [[NEW]] to i32
+; GFX908-NEXT:    [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
+; GFX908-NEXT:    [[TMP5:%.*]] = cmpxchg i32* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst
+; GFX908-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; GFX908-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
+; GFX908-NEXT:    [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
+; GFX908-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX908:       atomicrmw.end:
+; GFX908-NEXT:    ret float [[TMP6]]
 ;
   %res = atomicrmw fadd float* %ptr, float %value seq_cst
   ret float %res
@@ -75,11 +93,71 @@ define float @test_atomicrmw_fadd_f32_global(float addrspace(1)* %ptr, float %va
 ; GFX9-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
 ; GFX9:       atomicrmw.end:
 ; GFX9-NEXT:    ret float [[TMP6]]
+;
+; GFX908-LABEL: @test_atomicrmw_fadd_f32_global(
+; GFX908-NEXT:    [[TMP1:%.*]] = load float, float addrspace(1)* [[PTR:%.*]], align 4
+; GFX908-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX908:       atomicrmw.start:
+; GFX908-NEXT:    [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX908-NEXT:    [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]
+; GFX908-NEXT:    [[TMP2:%.*]] = bitcast float addrspace(1)* [[PTR]] to i32 addrspace(1)*
+; GFX908-NEXT:    [[TMP3:%.*]] = bitcast float [[NEW]] to i32
+; GFX908-NEXT:    [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
+; GFX908-NEXT:    [[TMP5:%.*]] = cmpxchg i32 addrspace(1)* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst
+; GFX908-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; GFX908-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
+; GFX908-NEXT:    [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
+; GFX908-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX908:       atomicrmw.end:
+; GFX908-NEXT:    ret float [[TMP6]]
 ;
   %res = atomicrmw fadd float addrspace(1)* %ptr, float %value seq_cst
   ret float %res
 }
 
+define void @test_atomicrmw_fadd_f32_global_no_use(float addrspace(1)* %ptr, float %value) {
+; CI-LABEL: @test_atomicrmw_fadd_f32_global_no_use(
+; CI-NEXT:    [[TMP1:%.*]] = load float, float addrspace(1)* [[PTR:%.*]], align 4
+; CI-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CI:       atomicrmw.start:
+; CI-NEXT:    [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; CI-NEXT:    [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]
+; CI-NEXT:    [[TMP2:%.*]] = bitcast float addrspace(1)* [[PTR]] to i32 addrspace(1)*
+; CI-NEXT:    [[TMP3:%.*]] = bitcast float [[NEW]] to i32
+; CI-NEXT:    [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
+; CI-NEXT:    [[TMP5:%.*]] = cmpxchg i32 addrspace(1)* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst
+; CI-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; CI-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
+; CI-NEXT:    [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
+; CI-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; CI:       atomicrmw.end:
+; CI-NEXT:    ret void
+;
+; GFX9-LABEL: @test_atomicrmw_fadd_f32_global_no_use(
+; GFX9-NEXT:    [[TMP1:%.*]] = load float, float addrspace(1)* [[PTR:%.*]], align 4
+; GFX9-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX9:       atomicrmw.start:
+; GFX9-NEXT:    [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX9-NEXT:    [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]
+; GFX9-NEXT:    [[TMP2:%.*]] = bitcast float addrspace(1)* [[PTR]] to i32 addrspace(1)*
+; GFX9-NEXT:    [[TMP3:%.*]] = bitcast float [[NEW]] to i32
+; GFX9-NEXT:    [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
+; GFX9-NEXT:    [[TMP5:%.*]] = cmpxchg i32 addrspace(1)* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst
+; GFX9-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; GFX9-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
+; GFX9-NEXT:    [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
+; GFX9-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX9:       atomicrmw.end:
+; GFX9-NEXT:    ret void
+;
+; GFX908-LABEL: @test_atomicrmw_fadd_f32_global_no_use(
+; GFX908-NEXT:    [[RES:%.*]] = atomicrmw fadd float addrspace(1)* [[PTR:%.*]], float [[VALUE:%.*]] seq_cst
+; GFX908-NEXT:    ret void
+;
+  %res = atomicrmw fadd float addrspace(1)* %ptr, float %value seq_cst
+  ret void
+}
+
 define float @test_atomicrmw_fadd_f32_local(float addrspace(3)* %ptr, float %value) {
 ; CI-LABEL: @test_atomicrmw_fadd_f32_local(
 ; CI-NEXT:    [[TMP1:%.*]] = load float, float addrspace(3)* [[PTR:%.*]], align 4
@@ -101,6 +179,10 @@ define float @test_atomicrmw_fadd_f32_local(float addrspace(3)* %ptr, float %val
 ; GFX9-LABEL: @test_atomicrmw_fadd_f32_local(
 ; GFX9-NEXT:    [[RES:%.*]] = atomicrmw fadd float addrspace(3)* [[PTR:%.*]], float [[VALUE:%.*]] seq_cst
 ; GFX9-NEXT:    ret float [[RES]]
+;
+; GFX908-LABEL: @test_atomicrmw_fadd_f32_local(
+; GFX908-NEXT:    [[RES:%.*]] = atomicrmw fadd float addrspace(3)* [[PTR:%.*]], float [[VALUE:%.*]] seq_cst
+; GFX908-NEXT:    ret float [[RES]]
 ;
   %res = atomicrmw fadd float addrspace(3)* %ptr, float %value seq_cst
   ret float %res
@@ -114,6 +196,10 @@ define half @test_atomicrmw_fadd_f16_flat(half* %ptr, half %value) {
 ; GFX9-LABEL: @test_atomicrmw_fadd_f16_flat(
 ; GFX9-NEXT:    [[RES:%.*]] = atomicrmw fadd half* [[PTR:%.*]], half [[VALUE:%.*]] seq_cst
 ; GFX9-NEXT:    ret half [[RES]]
+;
+; GFX908-LABEL: @test_atomicrmw_fadd_f16_flat(
+; GFX908-NEXT:    [[RES:%.*]] = atomicrmw fadd half* [[PTR:%.*]], half [[VALUE:%.*]] seq_cst
+; GFX908-NEXT:    ret half [[RES]]
 ;
   %res = atomicrmw fadd half* %ptr, half %value seq_cst
   ret half %res
@@ -127,6 +213,10 @@ define half @test_atomicrmw_fadd_f16_global(half addrspace(1)* %ptr, half %value
 ; GFX9-LABEL: @test_atomicrmw_fadd_f16_global(
 ; GFX9-NEXT:    [[RES:%.*]] = atomicrmw fadd half addrspace(1)* [[PTR:%.*]], half [[VALUE:%.*]] seq_cst
 ; GFX9-NEXT:    ret half [[RES]]
+;
+; GFX908-LABEL: @test_atomicrmw_fadd_f16_global(
+; GFX908-NEXT:    [[RES:%.*]] = atomicrmw fadd half addrspace(1)* [[PTR:%.*]], half [[VALUE:%.*]] seq_cst
+; GFX908-NEXT:    ret half [[RES]]
 ;
   %res = atomicrmw fadd half addrspace(1)* %ptr, half %value seq_cst
   ret half %res
@@ -140,6 +230,10 @@ define half @test_atomicrmw_fadd_f16_local(half addrspace(3)* %ptr, half %value)
 ; GFX9-LABEL: @test_atomicrmw_fadd_f16_local(
 ; GFX9-NEXT:    [[RES:%.*]] = atomicrmw fadd half addrspace(3)* [[PTR:%.*]], half [[VALUE:%.*]] seq_cst
 ; GFX9-NEXT:    ret half [[RES]]
+;
+; GFX908-LABEL: @test_atomicrmw_fadd_f16_local(
+; GFX908-NEXT:    [[RES:%.*]] = atomicrmw fadd half addrspace(3)* [[PTR:%.*]], half [[VALUE:%.*]] seq_cst
+; GFX908-NEXT:    ret half [[RES]]
 ;
   %res = atomicrmw fadd half addrspace(3)* %ptr, half %value seq_cst
   ret half %res
@@ -179,6 +273,23 @@ define double @test_atomicrmw_fadd_f64_flat(double* %ptr, double %value) {
 ; GFX9-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
 ; GFX9:       atomicrmw.end:
 ; GFX9-NEXT:    ret double [[TMP6]]
+;
+; GFX908-LABEL: @test_atomicrmw_fadd_f64_flat(
+; GFX908-NEXT:    [[TMP1:%.*]] = load double, double* [[PTR:%.*]], align 8
+; GFX908-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX908:       atomicrmw.start:
+; GFX908-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX908-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]]
+; GFX908-NEXT:    [[TMP2:%.*]] = bitcast double* [[PTR]] to i64*
+; GFX908-NEXT:    [[TMP3:%.*]] = bitcast double [[NEW]] to i64
+; GFX908-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX908-NEXT:    [[TMP5:%.*]] = cmpxchg i64* [[TMP2]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst
+; GFX908-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX908-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX908-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX908-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX908:       atomicrmw.end:
+; GFX908-NEXT:    ret double [[TMP6]]
 ;
   %res = atomicrmw fadd double* %ptr, double %value seq_cst
   ret double %res
@@ -218,6 +329,23 @@ define double @test_atomicrmw_fadd_f64_global(double addrspace(1)* %ptr, double
 ; GFX9-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
 ; GFX9:       atomicrmw.end:
 ; GFX9-NEXT:    ret double [[TMP6]]
+;
+; GFX908-LABEL: @test_atomicrmw_fadd_f64_global(
+; GFX908-NEXT:    [[TMP1:%.*]] = load double, double addrspace(1)* [[PTR:%.*]], align 8
+; GFX908-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX908:       atomicrmw.start:
+; GFX908-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX908-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]]
+; GFX908-NEXT:    [[TMP2:%.*]] = bitcast double addrspace(1)* [[PTR]] to i64 addrspace(1)*
+; GFX908-NEXT:    [[TMP3:%.*]] = bitcast double [[NEW]] to i64
+; GFX908-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX908-NEXT:    [[TMP5:%.*]] = cmpxchg i64 addrspace(1)* [[TMP2]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst
+; GFX908-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX908-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX908-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX908-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX908:       atomicrmw.end:
+; GFX908-NEXT:    ret double [[TMP6]]
 ;
   %res = atomicrmw fadd double addrspace(1)* %ptr, double %value seq_cst
   ret double %res
@@ -257,6 +385,23 @@ define double @test_atomicrmw_fadd_f64_local(double addrspace(3)* %ptr, double %
 ; GFX9-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
 ; GFX9:       atomicrmw.end:
 ; GFX9-NEXT:    ret double [[TMP6]]
+;
+; GFX908-LABEL: @test_atomicrmw_fadd_f64_local(
+; GFX908-NEXT:    [[TMP1:%.*]] = load double, double addrspace(3)* [[PTR:%.*]], align 8
+; GFX908-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX908:       atomicrmw.start:
+; GFX908-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX908-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]]
+; GFX908-NEXT:    [[TMP2:%.*]] = bitcast double addrspace(3)* [[PTR]] to i64 addrspace(3)*
+; GFX908-NEXT:    [[TMP3:%.*]] = bitcast double [[NEW]] to i64
+; GFX908-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX908-NEXT:    [[TMP5:%.*]] = cmpxchg i64 addrspace(3)* [[TMP2]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst
+; GFX908-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX908-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX908-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX908-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX908:       atomicrmw.end:
+; GFX908-NEXT:    ret double [[TMP6]]
 ;
   %res = atomicrmw fadd double addrspace(3)* %ptr, double %value seq_cst
   ret double %res
diff --git a/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-fsub.ll b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-fsub.ll
index 845489721..8a42e05f5 100644
--- a/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-fsub.ll
+++ b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-fsub.ll
@@ -2,8 +2,8 @@
 ; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=hawaii -atomic-expand %s | FileCheck -check-prefix=GCN %s
 ; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -atomic-expand %s | FileCheck -check-prefix=GCN %s
 
-define float @test_atomicrmw_fadd_f32_flat(float* %ptr, float %value) {
-; GCN-LABEL: @test_atomicrmw_fadd_f32_flat(
+define float @test_atomicrmw_fsub_f32_flat(float* %ptr, float %value) {
+; GCN-LABEL: @test_atomicrmw_fsub_f32_flat(
 ; GCN-NEXT:    [[TMP1:%.*]] = load float, float* [[PTR:%.*]], align 4
 ; GCN-NEXT:    br label [[ATOMICRMW_START:%.*]]
 ; GCN:       atomicrmw.start:
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/2008-02-01-ReturnAttrs.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/2008-02-01-ReturnAttrs.ll
new file mode 100644
index 000000000..d19cb444a
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/2008-02-01-ReturnAttrs.ll
@@ -0,0 +1,30 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; RUN: opt -passes=attributor -aa-pipeline='basic-aa' -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=2 -S < %s | FileCheck %s
+
+define internal i32 @deref(i32* %x) nounwind {
+; CHECK-LABEL: define {{[^@]+}}@deref
+; CHECK-SAME: (i32* noalias nocapture nofree nonnull readonly align 4 dereferenceable(4) [[X:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* [[X]], align 4
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
+entry:
+  %tmp2 = load i32, i32* %x, align 4
+  ret i32 %tmp2
+}
+
+define i32 @f(i32 %x) {
+; CHECK-LABEL: define {{[^@]+}}@f
+; CHECK-SAME: (i32 [[X:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[X_ADDR:%.*]] = alloca i32
+; CHECK-NEXT:    store i32 [[X]], i32* [[X_ADDR]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @deref(i32* noalias nocapture nofree nonnull readonly align 4 dereferenceable(4) [[X_ADDR]])
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+entry:
+  %x_addr = alloca i32
+  store i32 %x, i32* %x_addr, align 4
+  %tmp1 = call i32 @deref( i32* %x_addr ) nounwind
+  ret i32 %tmp1
+}
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/2008-07-02-array-indexing.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/2008-07-02-array-indexing.ll
new file mode 100644
index 000000000..bde1ce855
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/2008-07-02-array-indexing.ll
@@ -0,0 +1,43 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; RUN: opt -passes=attributor -aa-pipeline='basic-aa' -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=3 -S < %s | FileCheck %s
+; PR2498
+
+; This test tries to convince CHECK about promoting the load from %A + 2,
+; because there is a load of %A in the entry block
+define internal i32 @callee(i1 %C, i32* %A) {
+; CHECK-LABEL: define {{[^@]+}}@callee
+; CHECK-SAME: (i1 [[C:%.*]], i32* noalias nocapture nofree nonnull readonly dereferenceable(4) [[A:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[A_0:%.*]] = load i32, i32* null
+; CHECK-NEXT:    br label [[F:%.*]]
+; CHECK:       T:
+; CHECK-NEXT:    unreachable
+; CHECK:       F:
+; CHECK-NEXT:    [[A_2:%.*]] = getelementptr i32, i32* null, i32 2
+; CHECK-NEXT:    [[R:%.*]] = load i32, i32* [[A_2]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+entry:
+  ; Unconditonally load the element at %A
+  %A.0 = load i32, i32* %A
+  br i1 %C, label %T, label %F
+
+T:
+  ret i32 %A.0
+
+F:
+  ; Load the element at offset two from %A. This should not be promoted!
+  %A.2 = getelementptr i32, i32* %A, i32 2
+  %R = load i32, i32* %A.2
+  ret i32 %R
+}
+
+define i32 @foo() {
+; CHECK-LABEL: define {{[^@]+}}@foo()
+; CHECK-NEXT:    [[X:%.*]] = call i32 @callee(i1 false, i32* noalias nofree readonly null)
+; CHECK-NEXT:    ret i32 [[X]]
+;
+  %X = call i32 @callee(i1 false, i32* null)             ; <i32> [#uses=1]
+  ret i32 %X
+}
+
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/2008-09-07-CGUpdate.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/2008-09-07-CGUpdate.ll
new file mode 100644
index 000000000..fd2887ece
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/2008-09-07-CGUpdate.ll
@@ -0,0 +1,13 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; RUN: opt -disable-output -passes=attributor -aa-pipeline='basic-aa' -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=1 < %s
+
+define internal fastcc i32 @hash(i32* %ts, i32 %mod) nounwind {
+entry:
+  unreachable
+}
+
+define void @encode(i32* %m, i32* %ts, i32* %new) nounwind {
+entry:
+  %0 = call fastcc i32 @hash( i32* %ts, i32 0 ) nounwind		; <i32> [#uses=0]
+  unreachable
+}
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/2008-09-08-CGUpdateSelfEdge.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/2008-09-08-CGUpdateSelfEdge.ll
new file mode 100644
index 000000000..fe3f1c697
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/2008-09-08-CGUpdateSelfEdge.ll
@@ -0,0 +1,26 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; RUN: opt -disable-output -passes=attributor -aa-pipeline='basic-aa' -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=1 < %s
+
+define internal fastcc i32 @term_SharingList(i32* %Term, i32* %List) nounwind {
+entry:
+  br i1 false, label %bb, label %bb5
+
+bb:		; preds = %entry
+  %0 = call fastcc i32 @term_SharingList( i32* null, i32* %List ) nounwind		; <i32> [#uses=0]
+  unreachable
+
+bb5:		; preds = %entry
+  ret i32 0
+}
+
+define i32 @term_Sharing(i32* %Term) nounwind {
+entry:
+  br i1 false, label %bb.i, label %bb14
+
+bb.i:		; preds = %entry
+  %0 = call fastcc i32 @term_SharingList( i32* null, i32* null ) nounwind		; <i32> [#uses=0]
+  ret i32 1
+
+bb14:		; preds = %entry
+  ret i32 0
+}
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/X86/attributes.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/X86/attributes.ll
new file mode 100644
index 000000000..b66e09875
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/X86/attributes.ll
@@ -0,0 +1,89 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; RUN: opt -S -passes='attributor' -aa-pipeline='basic-aa' -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=2 < %s | FileCheck %s
+; Test that we only promote arguments when the caller/callee have compatible
+; function attrubtes.
+
+target triple = "x86_64-unknown-linux-gnu"
+
+define internal fastcc void @no_promote_avx2(<4 x i64>* %arg, <4 x i64>* readonly %arg1) #0 {
+; CHECK-LABEL: define {{[^@]+}}@no_promote_avx2
+; CHECK-SAME: (<4 x i64>* noalias nocapture nofree nonnull writeonly align 32 dereferenceable(32) [[ARG:%.*]], <4 x i64>* noalias nocapture nofree nonnull readonly align 32 dereferenceable(32) [[ARG1:%.*]])
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP:%.*]] = load <4 x i64>, <4 x i64>* [[ARG1]], align 32
+; CHECK-NEXT:    store <4 x i64> [[TMP]], <4 x i64>* [[ARG]], align 32
+; CHECK-NEXT:    ret void
+;
+bb:
+  %tmp = load <4 x i64>, <4 x i64>* %arg1
+  store <4 x i64> %tmp, <4 x i64>* %arg
+  ret void
+}
+
+define void @no_promote(<4 x i64>* %arg) #1 {
+; CHECK-LABEL: define {{[^@]+}}@no_promote
+; CHECK-SAME: (<4 x i64>* nocapture writeonly [[ARG:%.*]])
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP:%.*]] = alloca <4 x i64>, align 32
+; CHECK-NEXT:    [[TMP2:%.*]] = alloca <4 x i64>, align 32
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i64>* [[TMP]] to i8*
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* nonnull align 32 dereferenceable(32) [[TMP3]], i8 0, i64 32, i1 false)
+; CHECK-NEXT:    call fastcc void @no_promote_avx2(<4 x i64>* noalias nocapture nofree nonnull writeonly align 32 dereferenceable(32) [[TMP2]], <4 x i64>* noalias nocapture nofree nonnull readonly align 32 dereferenceable(32) [[TMP]])
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i64>, <4 x i64>* [[TMP2]], align 32
+; CHECK-NEXT:    store <4 x i64> [[TMP4]], <4 x i64>* [[ARG]], align 2
+; CHECK-NEXT:    ret void
+;
+bb:
+  %tmp = alloca <4 x i64>, align 32
+  %tmp2 = alloca <4 x i64>, align 32
+  %tmp3 = bitcast <4 x i64>* %tmp to i8*
+  call void @llvm.memset.p0i8.i64(i8* align 32 %tmp3, i8 0, i64 32, i1 false)
+  call fastcc void @no_promote_avx2(<4 x i64>* %tmp2, <4 x i64>* %tmp)
+  %tmp4 = load <4 x i64>, <4 x i64>* %tmp2, align 32
+  store <4 x i64> %tmp4, <4 x i64>* %arg, align 2
+  ret void
+}
+
+define internal fastcc void @promote_avx2(<4 x i64>* %arg, <4 x i64>* readonly %arg1) #0 {
+; CHECK-LABEL: define {{[^@]+}}@promote_avx2
+; CHECK-SAME: (<4 x i64>* noalias nocapture nofree nonnull writeonly align 32 dereferenceable(32) [[ARG:%.*]], <4 x i64>* noalias nocapture nofree nonnull readonly align 32 dereferenceable(32) [[ARG1:%.*]])
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP:%.*]] = load <4 x i64>, <4 x i64>* [[ARG1]], align 32
+; CHECK-NEXT:    store <4 x i64> [[TMP]], <4 x i64>* [[ARG]], align 32
+; CHECK-NEXT:    ret void
+;
+bb:
+  %tmp = load <4 x i64>, <4 x i64>* %arg1
+  store <4 x i64> %tmp, <4 x i64>* %arg
+  ret void
+}
+
+define void @promote(<4 x i64>* %arg) #0 {
+; CHECK-LABEL: define {{[^@]+}}@promote
+; CHECK-SAME: (<4 x i64>* nocapture writeonly [[ARG:%.*]])
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP:%.*]] = alloca <4 x i64>, align 32
+; CHECK-NEXT:    [[TMP2:%.*]] = alloca <4 x i64>, align 32
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i64>* [[TMP]] to i8*
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* nonnull align 32 dereferenceable(32) [[TMP3]], i8 0, i64 32, i1 false)
+; CHECK-NEXT:    call fastcc void @promote_avx2(<4 x i64>* noalias nocapture nofree nonnull writeonly align 32 dereferenceable(32) [[TMP2]], <4 x i64>* noalias nocapture nofree nonnull readonly align 32 dereferenceable(32) [[TMP]])
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i64>, <4 x i64>* [[TMP2]], align 32
+; CHECK-NEXT:    store <4 x i64> [[TMP4]], <4 x i64>* [[ARG]], align 2
+; CHECK-NEXT:    ret void
+;
+bb:
+  %tmp = alloca <4 x i64>, align 32
+  %tmp2 = alloca <4 x i64>, align 32
+  %tmp3 = bitcast <4 x i64>* %tmp to i8*
+  call void @llvm.memset.p0i8.i64(i8* align 32 %tmp3, i8 0, i64 32, i1 false)
+  call fastcc void @promote_avx2(<4 x i64>* %tmp2, <4 x i64>* %tmp)
+  %tmp4 = load <4 x i64>, <4 x i64>* %tmp2, align 32
+  store <4 x i64> %tmp4, <4 x i64>* %arg, align 2
+  ret void
+}
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.memset.p0i8.i64(i8* nocapture writeonly, i8, i64, i1) #2
+
+attributes #0 = { inlinehint norecurse nounwind uwtable "target-features"="+avx2" }
+attributes #1 = { nounwind uwtable }
+attributes #2 = { argmemonly nounwind }
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/X86/min-legal-vector-width.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/X86/min-legal-vector-width.ll
new file mode 100644
index 000000000..158e950ed
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/X86/min-legal-vector-width.ll
@@ -0,0 +1,328 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; RUN: opt -S -passes='attributor' -aa-pipeline='basic-aa' -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=2 < %s | FileCheck %s
+; Test that we only promote arguments when the caller/callee have compatible
+; function attrubtes.
+
+target triple = "x86_64-unknown-linux-gnu"
+
+; This should promote
+define internal fastcc void @callee_avx512_legal512_prefer512_call_avx512_legal512_prefer512(<8 x i64>* %arg, <8 x i64>* readonly %arg1) #0 {
+; CHECK-LABEL: define {{[^@]+}}@callee_avx512_legal512_prefer512_call_avx512_legal512_prefer512
+; CHECK-SAME: (<8 x i64>* noalias nocapture nofree nonnull writeonly align 32 dereferenceable(64) [[ARG:%.*]], <8 x i64>* noalias nocapture nofree nonnull readonly align 32 dereferenceable(64) [[ARG1:%.*]])
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP:%.*]] = load <8 x i64>, <8 x i64>* [[ARG1]], align 32
+; CHECK-NEXT:    store <8 x i64> [[TMP]], <8 x i64>* [[ARG]], align 32
+; CHECK-NEXT:    ret void
+;
+bb:
+  %tmp = load <8 x i64>, <8 x i64>* %arg1
+  store <8 x i64> %tmp, <8 x i64>* %arg
+  ret void
+}
+
+define void @avx512_legal512_prefer512_call_avx512_legal512_prefer512(<8 x i64>* %arg) #0 {
+; CHECK-LABEL: define {{[^@]+}}@avx512_legal512_prefer512_call_avx512_legal512_prefer512
+; CHECK-SAME: (<8 x i64>* nocapture writeonly [[ARG:%.*]])
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP:%.*]] = alloca <8 x i64>, align 32
+; CHECK-NEXT:    [[TMP2:%.*]] = alloca <8 x i64>, align 32
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i64>* [[TMP]] to i8*
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* nonnull align 32 dereferenceable(64) [[TMP3]], i8 0, i64 32, i1 false)
+; CHECK-NEXT:    call fastcc void @callee_avx512_legal512_prefer512_call_avx512_legal512_prefer512(<8 x i64>* noalias nocapture nofree nonnull writeonly align 32 dereferenceable(64) [[TMP2]], <8 x i64>* noalias nocapture nofree nonnull readonly align 32 dereferenceable(64) [[TMP]])
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, <8 x i64>* [[TMP2]], align 32
+; CHECK-NEXT:    store <8 x i64> [[TMP4]], <8 x i64>* [[ARG]], align 2
+; CHECK-NEXT:    ret void
+;
+bb:
+  %tmp = alloca <8 x i64>, align 32
+  %tmp2 = alloca <8 x i64>, align 32
+  %tmp3 = bitcast <8 x i64>* %tmp to i8*
+  call void @llvm.memset.p0i8.i64(i8* align 32 %tmp3, i8 0, i64 32, i1 false)
+  call fastcc void @callee_avx512_legal512_prefer512_call_avx512_legal512_prefer512(<8 x i64>* %tmp2, <8 x i64>* %tmp)
+  %tmp4 = load <8 x i64>, <8 x i64>* %tmp2, align 32
+  store <8 x i64> %tmp4, <8 x i64>* %arg, align 2
+  ret void
+}
+
+; This should promote
+define internal fastcc void @callee_avx512_legal512_prefer256_call_avx512_legal512_prefer256(<8 x i64>* %arg, <8 x i64>* readonly %arg1) #1 {
+; CHECK-LABEL: define {{[^@]+}}@callee_avx512_legal512_prefer256_call_avx512_legal512_prefer256
+; CHECK-SAME: (<8 x i64>* noalias nocapture nofree nonnull writeonly align 32 dereferenceable(64) [[ARG:%.*]], <8 x i64>* noalias nocapture nofree nonnull readonly align 32 dereferenceable(64) [[ARG1:%.*]])
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP:%.*]] = load <8 x i64>, <8 x i64>* [[ARG1]], align 32
+; CHECK-NEXT:    store <8 x i64> [[TMP]], <8 x i64>* [[ARG]], align 32
+; CHECK-NEXT:    ret void
+;
+bb:
+  %tmp = load <8 x i64>, <8 x i64>* %arg1
+  store <8 x i64> %tmp, <8 x i64>* %arg
+  ret void
+}
+
+define void @avx512_legal512_prefer256_call_avx512_legal512_prefer256(<8 x i64>* %arg) #1 {
+; CHECK-LABEL: define {{[^@]+}}@avx512_legal512_prefer256_call_avx512_legal512_prefer256
+; CHECK-SAME: (<8 x i64>* nocapture writeonly [[ARG:%.*]])
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP:%.*]] = alloca <8 x i64>, align 32
+; CHECK-NEXT:    [[TMP2:%.*]] = alloca <8 x i64>, align 32
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i64>* [[TMP]] to i8*
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* nonnull align 32 dereferenceable(64) [[TMP3]], i8 0, i64 32, i1 false)
+; CHECK-NEXT:    call fastcc void @callee_avx512_legal512_prefer256_call_avx512_legal512_prefer256(<8 x i64>* noalias nocapture nofree nonnull writeonly align 32 dereferenceable(64) [[TMP2]], <8 x i64>* noalias nocapture nofree nonnull readonly align 32 dereferenceable(64) [[TMP]])
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, <8 x i64>* [[TMP2]], align 32
+; CHECK-NEXT:    store <8 x i64> [[TMP4]], <8 x i64>* [[ARG]], align 2
+; CHECK-NEXT:    ret void
+;
+bb:
+  %tmp = alloca <8 x i64>, align 32
+  %tmp2 = alloca <8 x i64>, align 32
+  %tmp3 = bitcast <8 x i64>* %tmp to i8*
+  call void @llvm.memset.p0i8.i64(i8* align 32 %tmp3, i8 0, i64 32, i1 false)
+  call fastcc void @callee_avx512_legal512_prefer256_call_avx512_legal512_prefer256(<8 x i64>* %tmp2, <8 x i64>* %tmp)
+  %tmp4 = load <8 x i64>, <8 x i64>* %tmp2, align 32
+  store <8 x i64> %tmp4, <8 x i64>* %arg, align 2
+  ret void
+}
+
+; This should promote
+define internal fastcc void @callee_avx512_legal512_prefer512_call_avx512_legal512_prefer256(<8 x i64>* %arg, <8 x i64>* readonly %arg1) #1 {
+; CHECK-LABEL: define {{[^@]+}}@callee_avx512_legal512_prefer512_call_avx512_legal512_prefer256
+; CHECK-SAME: (<8 x i64>* noalias nocapture nofree nonnull writeonly align 32 dereferenceable(64) [[ARG:%.*]], <8 x i64>* noalias nocapture nofree nonnull readonly align 32 dereferenceable(64) [[ARG1:%.*]])
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP:%.*]] = load <8 x i64>, <8 x i64>* [[ARG1]], align 32
+; CHECK-NEXT:    store <8 x i64> [[TMP]], <8 x i64>* [[ARG]], align 32
+; CHECK-NEXT:    ret void
+;
+bb:
+  %tmp = load <8 x i64>, <8 x i64>* %arg1
+  store <8 x i64> %tmp, <8 x i64>* %arg
+  ret void
+}
+
+define void @avx512_legal512_prefer512_call_avx512_legal512_prefer256(<8 x i64>* %arg) #0 {
+; CHECK-LABEL: define {{[^@]+}}@avx512_legal512_prefer512_call_avx512_legal512_prefer256
+; CHECK-SAME: (<8 x i64>* nocapture writeonly [[ARG:%.*]])
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP:%.*]] = alloca <8 x i64>, align 32
+; CHECK-NEXT:    [[TMP2:%.*]] = alloca <8 x i64>, align 32
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i64>* [[TMP]] to i8*
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* nonnull align 32 dereferenceable(64) [[TMP3]], i8 0, i64 32, i1 false)
+; CHECK-NEXT:    call fastcc void @callee_avx512_legal512_prefer512_call_avx512_legal512_prefer256(<8 x i64>* noalias nocapture nofree nonnull writeonly align 32 dereferenceable(64) [[TMP2]], <8 x i64>* noalias nocapture nofree nonnull readonly align 32 dereferenceable(64) [[TMP]])
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, <8 x i64>* [[TMP2]], align 32
+; CHECK-NEXT:    store <8 x i64> [[TMP4]], <8 x i64>* [[ARG]], align 2
+; CHECK-NEXT:    ret void
+;
+bb:
+  %tmp = alloca <8 x i64>, align 32
+  %tmp2 = alloca <8 x i64>, align 32
+  %tmp3 = bitcast <8 x i64>* %tmp to i8*
+  call void @llvm.memset.p0i8.i64(i8* align 32 %tmp3, i8 0, i64 32, i1 false)
+  call fastcc void @callee_avx512_legal512_prefer512_call_avx512_legal512_prefer256(<8 x i64>* %tmp2, <8 x i64>* %tmp)
+  %tmp4 = load <8 x i64>, <8 x i64>* %tmp2, align 32
+  store <8 x i64> %tmp4, <8 x i64>* %arg, align 2
+  ret void
+}
+
+; This should promote
+define internal fastcc void @callee_avx512_legal512_prefer256_call_avx512_legal512_prefer512(<8 x i64>* %arg, <8 x i64>* readonly %arg1) #0 {
+; CHECK-LABEL: define {{[^@]+}}@callee_avx512_legal512_prefer256_call_avx512_legal512_prefer512
+; CHECK-SAME: (<8 x i64>* noalias nocapture nofree nonnull writeonly align 32 dereferenceable(64) [[ARG:%.*]], <8 x i64>* noalias nocapture nofree nonnull readonly align 32 dereferenceable(64) [[ARG1:%.*]])
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP:%.*]] = load <8 x i64>, <8 x i64>* [[ARG1]], align 32
+; CHECK-NEXT:    store <8 x i64> [[TMP]], <8 x i64>* [[ARG]], align 32
+; CHECK-NEXT:    ret void
+;
+bb:
+  %tmp = load <8 x i64>, <8 x i64>* %arg1
+  store <8 x i64> %tmp, <8 x i64>* %arg
+  ret void
+}
+
+define void @avx512_legal512_prefer256_call_avx512_legal512_prefer512(<8 x i64>* %arg) #1 {
+; CHECK-LABEL: define {{[^@]+}}@avx512_legal512_prefer256_call_avx512_legal512_prefer512
+; CHECK-SAME: (<8 x i64>* nocapture writeonly [[ARG:%.*]])
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP:%.*]] = alloca <8 x i64>, align 32
+; CHECK-NEXT:    [[TMP2:%.*]] = alloca <8 x i64>, align 32
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i64>* [[TMP]] to i8*
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* nonnull align 32 dereferenceable(64) [[TMP3]], i8 0, i64 32, i1 false)
+; CHECK-NEXT:    call fastcc void @callee_avx512_legal512_prefer256_call_avx512_legal512_prefer512(<8 x i64>* noalias nocapture nofree nonnull writeonly align 32 dereferenceable(64) [[TMP2]], <8 x i64>* noalias nocapture nofree nonnull readonly align 32 dereferenceable(64) [[TMP]])
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, <8 x i64>* [[TMP2]], align 32
+; CHECK-NEXT:    store <8 x i64> [[TMP4]], <8 x i64>* [[ARG]], align 2
+; CHECK-NEXT:    ret void
+;
+bb:
+  %tmp = alloca <8 x i64>, align 32
+  %tmp2 = alloca <8 x i64>, align 32
+  %tmp3 = bitcast <8 x i64>* %tmp to i8*
+  call void @llvm.memset.p0i8.i64(i8* align 32 %tmp3, i8 0, i64 32, i1 false)
+  call fastcc void @callee_avx512_legal512_prefer256_call_avx512_legal512_prefer512(<8 x i64>* %tmp2, <8 x i64>* %tmp)
+  %tmp4 = load <8 x i64>, <8 x i64>* %tmp2, align 32
+  store <8 x i64> %tmp4, <8 x i64>* %arg, align 2
+  ret void
+}
+
+; This should not promote
+define internal fastcc void @callee_avx512_legal256_prefer256_call_avx512_legal512_prefer256(<8 x i64>* %arg, <8 x i64>* readonly %arg1) #1 {
+; CHECK-LABEL: define {{[^@]+}}@callee_avx512_legal256_prefer256_call_avx512_legal512_prefer256
+; CHECK-SAME: (<8 x i64>* noalias nocapture nofree nonnull writeonly align 32 dereferenceable(64) [[ARG:%.*]], <8 x i64>* noalias nocapture nofree nonnull readonly align 32 dereferenceable(64) [[ARG1:%.*]])
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP:%.*]] = load <8 x i64>, <8 x i64>* [[ARG1]], align 32
+; CHECK-NEXT:    store <8 x i64> [[TMP]], <8 x i64>* [[ARG]], align 32
+; CHECK-NEXT:    ret void
+;
+bb:
+  %tmp = load <8 x i64>, <8 x i64>* %arg1
+  store <8 x i64> %tmp, <8 x i64>* %arg
+  ret void
+}
+
+define void @avx512_legal256_prefer256_call_avx512_legal512_prefer256(<8 x i64>* %arg) #2 {
+; CHECK-LABEL: define {{[^@]+}}@avx512_legal256_prefer256_call_avx512_legal512_prefer256
+; CHECK-SAME: (<8 x i64>* nocapture writeonly [[ARG:%.*]])
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP:%.*]] = alloca <8 x i64>, align 32
+; CHECK-NEXT:    [[TMP2:%.*]] = alloca <8 x i64>, align 32
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i64>* [[TMP]] to i8*
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* nonnull align 32 dereferenceable(64) [[TMP3]], i8 0, i64 32, i1 false)
+; CHECK-NEXT:    call fastcc void @callee_avx512_legal256_prefer256_call_avx512_legal512_prefer256(<8 x i64>* noalias nocapture nofree nonnull writeonly align 32 dereferenceable(64) [[TMP2]], <8 x i64>* noalias nocapture nofree nonnull readonly align 32 dereferenceable(64) [[TMP]])
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, <8 x i64>* [[TMP2]], align 32
+; CHECK-NEXT:    store <8 x i64> [[TMP4]], <8 x i64>* [[ARG]], align 2
+; CHECK-NEXT:    ret void
+;
+bb:
+  %tmp = alloca <8 x i64>, align 32
+  %tmp2 = alloca <8 x i64>, align 32
+  %tmp3 = bitcast <8 x i64>* %tmp to i8*
+  call void @llvm.memset.p0i8.i64(i8* align 32 %tmp3, i8 0, i64 32, i1 false)
+  call fastcc void @callee_avx512_legal256_prefer256_call_avx512_legal512_prefer256(<8 x i64>* %tmp2, <8 x i64>* %tmp)
+  %tmp4 = load <8 x i64>, <8 x i64>* %tmp2, align 32
+  store <8 x i64> %tmp4, <8 x i64>* %arg, align 2
+  ret void
+}
+
+; This should not promote
+define internal fastcc void @callee_avx512_legal512_prefer256_call_avx512_legal256_prefer256(<8 x i64>* %arg, <8 x i64>* readonly %arg1) #2 {
+; CHECK-LABEL: define {{[^@]+}}@callee_avx512_legal512_prefer256_call_avx512_legal256_prefer256
+; CHECK-SAME: (<8 x i64>* noalias nocapture nofree nonnull writeonly align 32 dereferenceable(64) [[ARG:%.*]], <8 x i64>* noalias nocapture nofree nonnull readonly align 32 dereferenceable(64) [[ARG1:%.*]])
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP:%.*]] = load <8 x i64>, <8 x i64>* [[ARG1]], align 32
+; CHECK-NEXT:    store <8 x i64> [[TMP]], <8 x i64>* [[ARG]], align 32
+; CHECK-NEXT:    ret void
+;
+bb:
+  %tmp = load <8 x i64>, <8 x i64>* %arg1
+  store <8 x i64> %tmp, <8 x i64>* %arg
+  ret void
+}
+
+define void @avx512_legal512_prefer256_call_avx512_legal256_prefer256(<8 x i64>* %arg) #1 {
+; CHECK-LABEL: define {{[^@]+}}@avx512_legal512_prefer256_call_avx512_legal256_prefer256
+; CHECK-SAME: (<8 x i64>* nocapture writeonly [[ARG:%.*]])
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP:%.*]] = alloca <8 x i64>, align 32
+; CHECK-NEXT:    [[TMP2:%.*]] = alloca <8 x i64>, align 32
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i64>* [[TMP]] to i8*
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* nonnull align 32 dereferenceable(64) [[TMP3]], i8 0, i64 32, i1 false)
+; CHECK-NEXT:    call fastcc void @callee_avx512_legal512_prefer256_call_avx512_legal256_prefer256(<8 x i64>* noalias nocapture nofree nonnull writeonly align 32 dereferenceable(64) [[TMP2]], <8 x i64>* noalias nocapture nofree nonnull readonly align 32 dereferenceable(64) [[TMP]])
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, <8 x i64>* [[TMP2]], align 32
+; CHECK-NEXT:    store <8 x i64> [[TMP4]], <8 x i64>* [[ARG]], align 2
+; CHECK-NEXT:    ret void
+;
+bb:
+  %tmp = alloca <8 x i64>, align 32
+  %tmp2 = alloca <8 x i64>, align 32
+  %tmp3 = bitcast <8 x i64>* %tmp to i8*
+  call void @llvm.memset.p0i8.i64(i8* align 32 %tmp3, i8 0, i64 32, i1 false)
+  call fastcc void @callee_avx512_legal512_prefer256_call_avx512_legal256_prefer256(<8 x i64>* %tmp2, <8 x i64>* %tmp)
+  %tmp4 = load <8 x i64>, <8 x i64>* %tmp2, align 32
+  store <8 x i64> %tmp4, <8 x i64>* %arg, align 2
+  ret void
+}
+
+; This should promote
+define internal fastcc void @callee_avx2_legal256_prefer256_call_avx2_legal512_prefer256(<8 x i64>* %arg, <8 x i64>* readonly %arg1) #3 {
+; CHECK-LABEL: define {{[^@]+}}@callee_avx2_legal256_prefer256_call_avx2_legal512_prefer256
+; CHECK-SAME: (<8 x i64>* noalias nocapture nofree nonnull writeonly align 32 dereferenceable(64) [[ARG:%.*]], <8 x i64>* noalias nocapture nofree nonnull readonly align 32 dereferenceable(64) [[ARG1:%.*]])
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP:%.*]] = load <8 x i64>, <8 x i64>* [[ARG1]], align 32
+; CHECK-NEXT:    store <8 x i64> [[TMP]], <8 x i64>* [[ARG]], align 32
+; CHECK-NEXT:    ret void
+;
+bb:
+  %tmp = load <8 x i64>, <8 x i64>* %arg1
+  store <8 x i64> %tmp, <8 x i64>* %arg
+  ret void
+}
+
+define void @avx2_legal256_prefer256_call_avx2_legal512_prefer256(<8 x i64>* %arg) #4 {
+; CHECK-LABEL: define {{[^@]+}}@avx2_legal256_prefer256_call_avx2_legal512_prefer256
+; CHECK-SAME: (<8 x i64>* nocapture writeonly [[ARG:%.*]])
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP:%.*]] = alloca <8 x i64>, align 32
+; CHECK-NEXT:    [[TMP2:%.*]] = alloca <8 x i64>, align 32
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i64>* [[TMP]] to i8*
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* nonnull align 32 dereferenceable(64) [[TMP3]], i8 0, i64 32, i1 false)
+; CHECK-NEXT:    call fastcc void @callee_avx2_legal256_prefer256_call_avx2_legal512_prefer256(<8 x i64>* noalias nocapture nofree nonnull writeonly align 32 dereferenceable(64) [[TMP2]], <8 x i64>* noalias nocapture nofree nonnull readonly align 32 dereferenceable(64) [[TMP]])
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, <8 x i64>* [[TMP2]], align 32
+; CHECK-NEXT:    store <8 x i64> [[TMP4]], <8 x i64>* [[ARG]], align 2
+; CHECK-NEXT:    ret void
+;
+bb:
+  %tmp = alloca <8 x i64>, align 32
+  %tmp2 = alloca <8 x i64>, align 32
+  %tmp3 = bitcast <8 x i64>* %tmp to i8*
+  call void @llvm.memset.p0i8.i64(i8* align 32 %tmp3, i8 0, i64 32, i1 false)
+  call fastcc void @callee_avx2_legal256_prefer256_call_avx2_legal512_prefer256(<8 x i64>* %tmp2, <8 x i64>* %tmp)
+  %tmp4 = load <8 x i64>, <8 x i64>* %tmp2, align 32
+  store <8 x i64> %tmp4, <8 x i64>* %arg, align 2
+  ret void
+}
+
+; This should promote
+define internal fastcc void @callee_avx2_legal512_prefer256_call_avx2_legal256_prefer256(<8 x i64>* %arg, <8 x i64>* readonly %arg1) #4 {
+; CHECK-LABEL: define {{[^@]+}}@callee_avx2_legal512_prefer256_call_avx2_legal256_prefer256
+; CHECK-SAME: (<8 x i64>* noalias nocapture nofree nonnull writeonly align 32 dereferenceable(64) [[ARG:%.*]], <8 x i64>* noalias nocapture nofree nonnull readonly align 32 dereferenceable(64) [[ARG1:%.*]])
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP:%.*]] = load <8 x i64>, <8 x i64>* [[ARG1]], align 32
+; CHECK-NEXT:    store <8 x i64> [[TMP]], <8 x i64>* [[ARG]], align 32
+; CHECK-NEXT:    ret void
+;
+bb:
+  %tmp = load <8 x i64>, <8 x i64>* %arg1
+  store <8 x i64> %tmp, <8 x i64>* %arg
+  ret void
+}
+
+define void @avx2_legal512_prefer256_call_avx2_legal256_prefer256(<8 x i64>* %arg) #3 {
+; CHECK-LABEL: define {{[^@]+}}@avx2_legal512_prefer256_call_avx2_legal256_prefer256
+; CHECK-SAME: (<8 x i64>* nocapture writeonly [[ARG:%.*]])
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP:%.*]] = alloca <8 x i64>, align 32
+; CHECK-NEXT:    [[TMP2:%.*]] = alloca <8 x i64>, align 32
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i64>* [[TMP]] to i8*
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* nonnull align 32 dereferenceable(64) [[TMP3]], i8 0, i64 32, i1 false)
+; CHECK-NEXT:    call fastcc void @callee_avx2_legal512_prefer256_call_avx2_legal256_prefer256(<8 x i64>* noalias nocapture nofree nonnull writeonly align 32 dereferenceable(64) [[TMP2]], <8 x i64>* noalias nocapture nofree nonnull readonly align 32 dereferenceable(64) [[TMP]])
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, <8 x i64>* [[TMP2]], align 32
+; CHECK-NEXT:    store <8 x i64> [[TMP4]], <8 x i64>* [[ARG]], align 2
+; CHECK-NEXT:    ret void
+;
+bb:
+  %tmp = alloca <8 x i64>, align 32
+  %tmp2 = alloca <8 x i64>, align 32
+  %tmp3 = bitcast <8 x i64>* %tmp to i8*
+  call void @llvm.memset.p0i8.i64(i8* align 32 %tmp3, i8 0, i64 32, i1 false)
+  call fastcc void @callee_avx2_legal512_prefer256_call_avx2_legal256_prefer256(<8 x i64>* %tmp2, <8 x i64>* %tmp)
+  %tmp4 = load <8 x i64>, <8 x i64>* %tmp2, align 32
+  store <8 x i64> %tmp4, <8 x i64>* %arg, align 2
+  ret void
+}
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.memset.p0i8.i64(i8* nocapture writeonly, i8, i64, i1) #5
+
+attributes #0 = { inlinehint norecurse nounwind uwtable "target-features"="+avx512vl" "min-legal-vector-width"="512" "prefer-vector-width"="512" }
+attributes #1 = { inlinehint norecurse nounwind uwtable "target-features"="+avx512vl" "min-legal-vector-width"="512" "prefer-vector-width"="256" }
+attributes #2 = { inlinehint norecurse nounwind uwtable "target-features"="+avx512vl" "min-legal-vector-width"="256" "prefer-vector-width"="256" }
+attributes #3 = { inlinehint norecurse nounwind uwtable "target-features"="+avx2" "min-legal-vector-width"="512" "prefer-vector-width"="256" }
+attributes #4 = { inlinehint norecurse nounwind uwtable "target-features"="+avx2" "min-legal-vector-width"="256" "prefer-vector-width"="256" }
+attributes #5 = { argmemonly nounwind }
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/X86/thiscall.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/X86/thiscall.ll
new file mode 100644
index 000000000..d9f3681ba
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/X86/thiscall.ll
@@ -0,0 +1,73 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; In PR41658, argpromotion put an inalloca in a position that per the
+; calling convention is passed in a register. This test verifies that
+; we don't do that anymore. It also verifies that the combination of
+; globalopt and argpromotion is able to optimize the call safely.
+;
+; RUN: opt -S -argpromotion %s | FileCheck %s --check-prefix=ARGPROMOTION
+; RUN: opt -S -globalopt -argpromotion %s | FileCheck %s --check-prefix=GLOBALOPT_ARGPROMOTION
+
+target datalayout = "e-m:x-p:32:32-i64:64-f80:32-n8:16:32-a:0:32-S32"
+target triple = "i386-pc-windows-msvc19.11.0"
+
+%struct.a = type { i8 }
+
+define internal x86_thiscallcc void @internalfun(%struct.a* %this, <{ %struct.a }>* inalloca) {
+; ARGPROMOTION-LABEL: define {{[^@]+}}@internalfun
+; ARGPROMOTION-SAME: (%struct.a* [[THIS:%.*]], <{ [[STRUCT_A:%.*]] }>* inalloca [[TMP0:%.*]])
+; ARGPROMOTION-NEXT:  entry:
+; ARGPROMOTION-NEXT:    [[A:%.*]] = getelementptr inbounds <{ [[STRUCT_A]] }>, <{ [[STRUCT_A]] }>* [[TMP0]], i32 0, i32 0
+; ARGPROMOTION-NEXT:    [[ARGMEM:%.*]] = alloca inalloca <{ [[STRUCT_A]] }>, align 4
+; ARGPROMOTION-NEXT:    [[TMP1:%.*]] = getelementptr inbounds <{ [[STRUCT_A]] }>, <{ [[STRUCT_A]] }>* [[ARGMEM]], i32 0, i32 0
+; ARGPROMOTION-NEXT:    [[CALL:%.*]] = call x86_thiscallcc %struct.a* @copy_ctor(%struct.a* [[TMP1]], %struct.a* dereferenceable(1) [[A]])
+; ARGPROMOTION-NEXT:    call void @ext(<{ [[STRUCT_A]] }>* inalloca [[ARGMEM]])
+; ARGPROMOTION-NEXT:    ret void
+;
+; GLOBALOPT_ARGPROMOTION-LABEL: define {{[^@]+}}@internalfun
+; GLOBALOPT_ARGPROMOTION-SAME: (<{ [[STRUCT_A:%.*]] }>* [[TMP0:%.*]]) unnamed_addr
+; GLOBALOPT_ARGPROMOTION-NEXT:  entry:
+; GLOBALOPT_ARGPROMOTION-NEXT:    [[A:%.*]] = getelementptr inbounds <{ [[STRUCT_A]] }>, <{ [[STRUCT_A]] }>* [[TMP0]], i32 0, i32 0
+; GLOBALOPT_ARGPROMOTION-NEXT:    [[ARGMEM:%.*]] = alloca inalloca <{ [[STRUCT_A]] }>, align 4
+; GLOBALOPT_ARGPROMOTION-NEXT:    [[TMP1:%.*]] = getelementptr inbounds <{ [[STRUCT_A]] }>, <{ [[STRUCT_A]] }>* [[ARGMEM]], i32 0, i32 0
+; GLOBALOPT_ARGPROMOTION-NEXT:    [[CALL:%.*]] = call x86_thiscallcc %struct.a* @copy_ctor(%struct.a* [[TMP1]], %struct.a* dereferenceable(1) [[A]])
+; GLOBALOPT_ARGPROMOTION-NEXT:    call void @ext(<{ [[STRUCT_A]] }>* inalloca [[ARGMEM]])
+; GLOBALOPT_ARGPROMOTION-NEXT:    ret void
+;
+entry:
+  %a = getelementptr inbounds <{ %struct.a }>, <{ %struct.a }>* %0, i32 0, i32 0
+  %argmem = alloca inalloca <{ %struct.a }>, align 4
+  %1 = getelementptr inbounds <{ %struct.a }>, <{ %struct.a }>* %argmem, i32 0, i32 0
+  %call = call x86_thiscallcc %struct.a* @copy_ctor(%struct.a* %1, %struct.a* dereferenceable(1) %a)
+  call void @ext(<{ %struct.a }>* inalloca %argmem)
+  ret void
+}
+
+; This is here to ensure @internalfun is live.
+define void @exportedfun(%struct.a* %a) {
+; ARGPROMOTION-LABEL: define {{[^@]+}}@exportedfun
+; ARGPROMOTION-SAME: (%struct.a* [[A:%.*]])
+; ARGPROMOTION-NEXT:    [[INALLOCA_SAVE:%.*]] = tail call i8* @llvm.stacksave()
+; ARGPROMOTION-NEXT:    [[ARGMEM:%.*]] = alloca inalloca <{ [[STRUCT_A:%.*]] }>, align 4
+; ARGPROMOTION-NEXT:    call x86_thiscallcc void @internalfun(%struct.a* [[A]], <{ [[STRUCT_A]] }>* inalloca [[ARGMEM]])
+; ARGPROMOTION-NEXT:    call void @llvm.stackrestore(i8* [[INALLOCA_SAVE]])
+; ARGPROMOTION-NEXT:    ret void
+;
+; GLOBALOPT_ARGPROMOTION-LABEL: define {{[^@]+}}@exportedfun
+; GLOBALOPT_ARGPROMOTION-SAME: (%struct.a* [[A:%.*]]) local_unnamed_addr
+; GLOBALOPT_ARGPROMOTION-NEXT:    [[INALLOCA_SAVE:%.*]] = tail call i8* @llvm.stacksave()
+; GLOBALOPT_ARGPROMOTION-NEXT:    [[ARGMEM:%.*]] = alloca inalloca <{ [[STRUCT_A:%.*]] }>, align 4
+; GLOBALOPT_ARGPROMOTION-NEXT:    call fastcc void @internalfun(<{ [[STRUCT_A]] }>* [[ARGMEM]])
+; GLOBALOPT_ARGPROMOTION-NEXT:    call void @llvm.stackrestore(i8* [[INALLOCA_SAVE]])
+; GLOBALOPT_ARGPROMOTION-NEXT:    ret void
+;
+  %inalloca.save = tail call i8* @llvm.stacksave()
+  %argmem = alloca inalloca <{ %struct.a }>, align 4
+  call x86_thiscallcc void @internalfun(%struct.a* %a, <{ %struct.a }>* inalloca %argmem)
+  call void @llvm.stackrestore(i8* %inalloca.save)
+  ret void
+}
+
+declare x86_thiscallcc %struct.a* @copy_ctor(%struct.a* returned, %struct.a* dereferenceable(1))
+declare void @ext(<{ %struct.a }>* inalloca)
+declare i8* @llvm.stacksave()
+declare void @llvm.stackrestore(i8*)
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/aggregate-promote.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/aggregate-promote.ll
new file mode 100644
index 000000000..449b05e44
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/aggregate-promote.ll
@@ -0,0 +1,36 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; RUN: opt -disable-output -passes=attributor -aa-pipeline='basic-aa' -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=3 < %s
+
+%T = type { i32, i32, i32, i32 }
+@G = constant %T { i32 0, i32 0, i32 17, i32 25 }
+
+define internal i32 @test(%T* %p) {
+; CHECK-LABEL: define {{[^@]+}}@test
+; CHECK-SAME: (i32 [[P_0_2_VAL:%.*]], i32 [[P_0_3_VAL:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[V:%.*]] = add i32 [[P_0_3_VAL]], [[P_0_2_VAL]]
+; CHECK-NEXT:    ret i32 [[V]]
+;
+entry:
+  %a.gep = getelementptr %T, %T* %p, i64 0, i32 3
+  %b.gep = getelementptr %T, %T* %p, i64 0, i32 2
+  %a = load i32, i32* %a.gep
+  %b = load i32, i32* %b.gep
+  %v = add i32 %a, %b
+  ret i32 %v
+}
+
+define i32 @caller() {
+; CHECK-LABEL: define {{[^@]+}}@caller()
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[G_IDX:%.*]] = getelementptr [[T:%.*]], %T* @G, i64 0, i32 2
+; CHECK-NEXT:    [[G_IDX_VAL:%.*]] = load i32, i32* [[G_IDX]]
+; CHECK-NEXT:    [[G_IDX1:%.*]] = getelementptr [[T]], %T* @G, i64 0, i32 3
+; CHECK-NEXT:    [[G_IDX1_VAL:%.*]] = load i32, i32* [[G_IDX1]]
+; CHECK-NEXT:    [[V:%.*]] = call i32 @test(i32 [[G_IDX_VAL]], i32 [[G_IDX1_VAL]])
+; CHECK-NEXT:    ret i32 [[V]]
+;
+entry:
+  %v = call i32 @test(%T* @G)
+  ret i32 %v
+}
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/attrs.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/attrs.ll
new file mode 100644
index 000000000..8f2e14d7b
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/attrs.ll
@@ -0,0 +1,52 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; RUN: opt -S -passes=attributor -aa-pipeline='basic-aa' -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=3 < %s | FileCheck %s
+
+%struct.ss = type { i32, i64 }
+
+; Don't drop 'byval' on %X here.
+define internal void @f(%struct.ss* byval %b, i32* byval %X, i32 %i) nounwind {
+; CHECK-LABEL: define {{[^@]+}}@f
+; CHECK-SAME: (%struct.ss* noalias nocapture nofree nonnull byval align 8 dereferenceable(12) [[B:%.*]], i32* nocapture nofree nonnull writeonly byval dereferenceable(4) [[X:%.*]], i32 [[I:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP:%.*]] = getelementptr [[STRUCT_SS:%.*]], %struct.ss* [[B]], i32 0, i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[TMP]], align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = add i32 [[TMP1]], 1
+; CHECK-NEXT:    store i32 [[TMP2]], i32* [[TMP]], align 8
+; CHECK-NEXT:    store i32 0, i32* [[X]]
+; CHECK-NEXT:    ret void
+;
+entry:
+
+  %tmp = getelementptr %struct.ss, %struct.ss* %b, i32 0, i32 0
+  %tmp1 = load i32, i32* %tmp, align 4
+  %tmp2 = add i32 %tmp1, 1
+  store i32 %tmp2, i32* %tmp, align 4
+
+  store i32 %i, i32* %X
+  ret void
+}
+
+; Also make sure we don't drop the call zeroext attribute.
+define i32 @test(i32* %X) {
+; CHECK-LABEL: define {{[^@]+}}@test
+; CHECK-SAME: (i32* nocapture nofree readonly [[X:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[S:%.*]] = alloca [[STRUCT_SS:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr [[STRUCT_SS]], %struct.ss* [[S]], i32 0, i32 0
+; CHECK-NEXT:    store i32 1, i32* [[TMP1]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr [[STRUCT_SS]], %struct.ss* [[S]], i32 0, i32 1
+; CHECK-NEXT:    store i64 2, i64* [[TMP4]], align 4
+; CHECK-NEXT:    call void @f(%struct.ss* noalias nocapture nofree nonnull readonly byval align 8 dereferenceable(12) [[S]], i32* nocapture nofree readonly byval [[X]], i32 zeroext 0)
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %S = alloca %struct.ss
+  %tmp1 = getelementptr %struct.ss, %struct.ss* %S, i32 0, i32 0
+  store i32 1, i32* %tmp1, align 8
+  %tmp4 = getelementptr %struct.ss, %struct.ss* %S, i32 0, i32 1
+  store i64 2, i64* %tmp4, align 4
+
+  call void @f( %struct.ss* byval %S, i32* byval %X, i32 zeroext 0)
+
+  ret i32 0
+}
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/basictest.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/basictest.ll
new file mode 100644
index 000000000..ea3feffe6
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/basictest.ll
@@ -0,0 +1,45 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; RUN: opt -S -passes='attributor' -aa-pipeline='basic-aa' -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=6 < %s | FileCheck %s
+target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
+
+define internal i32 @test(i32* %X, i32* %Y) {
+; CHECK-LABEL: define {{[^@]+}}@test
+; CHECK-SAME: (i32* noalias nocapture nofree nonnull readonly align 4 dereferenceable(4) [[X:%.*]], i32* noalias nocapture nofree nonnull readonly align 4 dereferenceable(4) [[Y:%.*]])
+; CHECK-NEXT:    [[A:%.*]] = load i32, i32* [[X]], align 4
+; CHECK-NEXT:    [[B:%.*]] = load i32, i32* [[Y]], align 4
+; CHECK-NEXT:    [[C:%.*]] = add i32 [[A]], [[B]]
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %A = load i32, i32* %X
+  %B = load i32, i32* %Y
+  %C = add i32 %A, %B
+  ret i32 %C
+}
+
+define internal i32 @caller(i32* %B) {
+; CHECK-LABEL: define {{[^@]+}}@caller
+; CHECK-SAME: (i32* noalias nocapture nofree nonnull readonly align 4 dereferenceable(4) [[B:%.*]])
+; CHECK-NEXT:    [[A:%.*]] = alloca i32
+; CHECK-NEXT:    store i32 1, i32* [[A]], align 4
+; CHECK-NEXT:    [[C:%.*]] = call i32 @test(i32* noalias nocapture nofree nonnull readonly align 4 dereferenceable(4) [[A]], i32* noalias nocapture nofree nonnull readonly align 4 dereferenceable(4) [[B]])
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %A = alloca i32
+  store i32 1, i32* %A
+  %C = call i32 @test(i32* %A, i32* %B)
+  ret i32 %C
+}
+
+define i32 @callercaller() {
+; CHECK-LABEL: define {{[^@]+}}@callercaller()
+; CHECK-NEXT:    [[B:%.*]] = alloca i32
+; CHECK-NEXT:    store i32 2, i32* [[B]], align 4
+; CHECK-NEXT:    [[X:%.*]] = call i32 @caller(i32* noalias nocapture nofree nonnull readonly align 4 dereferenceable(4) [[B]])
+; CHECK-NEXT:    ret i32 [[X]]
+;
+  %B = alloca i32
+  store i32 2, i32* %B
+  %X = call i32 @caller(i32* %B)
+  ret i32 %X
+}
+
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/byval-2.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/byval-2.ll
new file mode 100644
index 000000000..2ebf5ec4a
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/byval-2.ll
@@ -0,0 +1,47 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; RUN: opt -S -passes=attributor -aa-pipeline='basic-aa' -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=3 < %s | FileCheck %s
+
+%struct.ss = type { i32, i64 }
+
+define internal void @f(%struct.ss* byval  %b, i32* byval %X) nounwind  {
+; CHECK-LABEL: define {{[^@]+}}@f
+; CHECK-SAME: (%struct.ss* noalias nocapture nofree nonnull byval align 8 dereferenceable(12) [[B:%.*]], i32* nocapture nofree nonnull writeonly byval dereferenceable(4) [[X:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP:%.*]] = getelementptr [[STRUCT_SS:%.*]], %struct.ss* [[B]], i32 0, i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[TMP]], align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = add i32 [[TMP1]], 1
+; CHECK-NEXT:    store i32 [[TMP2]], i32* [[TMP]], align 8
+; CHECK-NEXT:    store i32 0, i32* [[X]]
+; CHECK-NEXT:    ret void
+;
+entry:
+  %tmp = getelementptr %struct.ss, %struct.ss* %b, i32 0, i32 0
+  %tmp1 = load i32, i32* %tmp, align 4
+  %tmp2 = add i32 %tmp1, 1
+  store i32 %tmp2, i32* %tmp, align 4
+
+  store i32 0, i32* %X
+  ret void
+}
+
+define i32 @test(i32* %X) {
+; CHECK-LABEL: define {{[^@]+}}@test
+; CHECK-SAME: (i32* nocapture nofree readonly [[X:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[S:%.*]] = alloca [[STRUCT_SS:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr [[STRUCT_SS]], %struct.ss* [[S]], i32 0, i32 0
+; CHECK-NEXT:    store i32 1, i32* [[TMP1]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr [[STRUCT_SS]], %struct.ss* [[S]], i32 0, i32 1
+; CHECK-NEXT:    store i64 2, i64* [[TMP4]], align 4
+; CHECK-NEXT:    call void @f(%struct.ss* noalias nocapture nofree nonnull readonly byval align 8 dereferenceable(12) [[S]], i32* nocapture nofree readonly byval [[X]])
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %S = alloca %struct.ss
+  %tmp1 = getelementptr %struct.ss, %struct.ss* %S, i32 0, i32 0
+  store i32 1, i32* %tmp1, align 8
+  %tmp4 = getelementptr %struct.ss, %struct.ss* %S, i32 0, i32 1
+  store i64 2, i64* %tmp4, align 4
+  call void @f( %struct.ss* byval %S, i32* byval %X)
+  ret i32 0
+}
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/byval.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/byval.ll
new file mode 100644
index 000000000..bba9f6f20
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/byval.ll
@@ -0,0 +1,69 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; RUN: opt -S -passes='attributor' -aa-pipeline='basic-aa' -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=2 < %s | FileCheck %s
+
+target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
+
+%struct.ss = type { i32, i64 }
+
+define internal void @f(%struct.ss* byval  %b) nounwind  {
+; CHECK-LABEL: define {{[^@]+}}@f
+; CHECK-SAME: (%struct.ss* noalias nocapture nofree nonnull byval align 8 dereferenceable(12) [[B:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP:%.*]] = getelementptr [[STRUCT_SS:%.*]], %struct.ss* [[B]], i32 0, i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[TMP]], align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = add i32 [[TMP1]], 1
+; CHECK-NEXT:    store i32 [[TMP2]], i32* [[TMP]], align 8
+; CHECK-NEXT:    ret void
+;
+entry:
+  %tmp = getelementptr %struct.ss, %struct.ss* %b, i32 0, i32 0
+  %tmp1 = load i32, i32* %tmp, align 4
+  %tmp2 = add i32 %tmp1, 1
+  store i32 %tmp2, i32* %tmp, align 4
+  ret void
+}
+
+
+define internal void @g(%struct.ss* byval align 32 %b) nounwind {
+; CHECK-LABEL: define {{[^@]+}}@g
+; CHECK-SAME: (%struct.ss* noalias nocapture nofree nonnull byval align 32 dereferenceable(12) [[B:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP:%.*]] = getelementptr [[STRUCT_SS:%.*]], %struct.ss* [[B]], i32 0, i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[TMP]], align 32
+; CHECK-NEXT:    [[TMP2:%.*]] = add i32 [[TMP1]], 1
+; CHECK-NEXT:    store i32 [[TMP2]], i32* [[TMP]], align 32
+; CHECK-NEXT:    ret void
+;
+entry:
+  %tmp = getelementptr %struct.ss, %struct.ss* %b, i32 0, i32 0
+  %tmp1 = load i32, i32* %tmp, align 4
+  %tmp2 = add i32 %tmp1, 1
+  store i32 %tmp2, i32* %tmp, align 4
+  ret void
+}
+
+
+define i32 @main() nounwind  {
+; CHECK-LABEL: define {{[^@]+}}@main()
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[S:%.*]] = alloca [[STRUCT_SS:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr [[STRUCT_SS]], %struct.ss* [[S]], i32 0, i32 0
+; CHECK-NEXT:    store i32 1, i32* [[TMP1]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr [[STRUCT_SS]], %struct.ss* [[S]], i32 0, i32 1
+; CHECK-NEXT:    store i64 2, i64* [[TMP4]], align 4
+; CHECK-NEXT:    call void @f(%struct.ss* noalias nocapture nofree nonnull readonly byval align 8 dereferenceable(12) [[S]])
+; CHECK-NEXT:    call void @g(%struct.ss* noalias nocapture nofree nonnull readonly byval align 32 dereferenceable(12) [[S]])
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %S = alloca %struct.ss
+  %tmp1 = getelementptr %struct.ss, %struct.ss* %S, i32 0, i32 0
+  store i32 1, i32* %tmp1, align 8
+  %tmp4 = getelementptr %struct.ss, %struct.ss* %S, i32 0, i32 1
+  store i64 2, i64* %tmp4, align 4
+  call void @f(%struct.ss* byval %S) nounwind
+  call void @g(%struct.ss* byval %S) nounwind
+  ret i32 0
+}
+
+
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/chained.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/chained.ll
new file mode 100644
index 000000000..15ddb2112
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/chained.ll
@@ -0,0 +1,31 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; RUN: opt -S -passes='attributor' -aa-pipeline='basic-aa' -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=3 < %s | FileCheck %s
+
+@G1 = constant i32 0
+@G2 = constant i32* @G1
+
+define internal i32 @test(i32** %x) {
+; CHECK-LABEL: define {{[^@]+}}@test
+; CHECK-SAME: (i32** nocapture nofree nonnull readonly align 8 dereferenceable(8) [[X:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[Y:%.*]] = load i32*, i32** @G2, align 8
+; CHECK-NEXT:    [[Z:%.*]] = load i32, i32* [[Y]]
+; CHECK-NEXT:    ret i32 [[Z]]
+;
+entry:
+  %y = load i32*, i32** %x
+  %z = load i32, i32* %y
+  ret i32 %z
+}
+
+define i32 @caller() {
+; CHECK-LABEL: define {{[^@]+}}@caller()
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[X:%.*]] = call i32 @test(i32** nofree nonnull readonly align 8 dereferenceable(8) @G2)
+; CHECK-NEXT:    ret i32 [[X]]
+;
+entry:
+  %x = call i32 @test(i32** @G2)
+  ret i32 %x
+}
+
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/control-flow.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/control-flow.ll
new file mode 100644
index 000000000..494107d17
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/control-flow.ll
@@ -0,0 +1,38 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; RUN: opt -S -passes='attributor' -aa-pipeline='basic-aa' -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=3 < %s | FileCheck %s
+
+; Don't promote around control flow.
+define internal i32 @callee(i1 %C, i32* %P) {
+; CHECK-LABEL: define {{[^@]+}}@callee
+; CHECK-SAME: (i1 [[C:%.*]], i32* nocapture nofree readonly [[P:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]]
+; CHECK:       T:
+; CHECK-NEXT:    ret i32 17
+; CHECK:       F:
+; CHECK-NEXT:    [[X:%.*]] = load i32, i32* [[P]]
+; CHECK-NEXT:    ret i32 [[X]]
+;
+entry:
+  br i1 %C, label %T, label %F
+
+T:
+  ret i32 17
+
+F:
+  %X = load i32, i32* %P
+  ret i32 %X
+}
+
+define i32 @foo(i1 %C, i32* %P) {
+; CHECK-LABEL: define {{[^@]+}}@foo
+; CHECK-SAME: (i1 [[C:%.*]], i32* nocapture nofree readonly [[P:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[X:%.*]] = call i32 @callee(i1 [[C]], i32* nocapture nofree readonly [[P]])
+; CHECK-NEXT:    ret i32 [[X]]
+;
+entry:
+  %X = call i32 @callee(i1 %C, i32* %P)
+  ret i32 %X
+}
+
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/control-flow2.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/control-flow2.ll
new file mode 100644
index 000000000..92257857f
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/control-flow2.ll
@@ -0,0 +1,38 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; RUN: opt -S -passes='attributor' -aa-pipeline='basic-aa' -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=9 < %s | FileCheck %s
+
+target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
+
+define internal i32 @callee(i1 %C, i32* %P) {
+; CHECK-LABEL: define {{[^@]+}}@callee
+; CHECK-SAME: (i1 [[C:%.*]], i32* noalias nocapture nofree nonnull readonly align 4 dereferenceable(4) [[P:%.*]])
+; CHECK-NEXT:    br label [[F:%.*]]
+; CHECK:       T:
+; CHECK-NEXT:    unreachable
+; CHECK:       F:
+; CHECK-NEXT:    [[X:%.*]] = load i32, i32* [[P]], align 4
+; CHECK-NEXT:    ret i32 [[X]]
+;
+  br i1 %C, label %T, label %F
+
+T:              ; preds = %0
+  ret i32 17
+
+F:              ; preds = %0
+  %X = load i32, i32* %P               ; <i32> [#uses=1]
+  ret i32 %X
+}
+
+define i32 @foo() {
+; CHECK-LABEL: define {{[^@]+}}@foo()
+; CHECK-NEXT:    [[A:%.*]] = alloca i32
+; CHECK-NEXT:    store i32 17, i32* [[A]], align 4
+; CHECK-NEXT:    [[X:%.*]] = call i32 @callee(i1 false, i32* noalias nocapture nofree nonnull readonly align 4 dereferenceable(4) [[A]])
+; CHECK-NEXT:    ret i32 [[X]]
+;
+  %A = alloca i32         ; <i32*> [#uses=2]
+  store i32 17, i32* %A
+  %X = call i32 @callee( i1 false, i32* %A )              ; <i32> [#uses=1]
+  ret i32 %X
+}
+
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/crash.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/crash.ll
new file mode 100644
index 000000000..db3db632e
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/crash.ll
@@ -0,0 +1,94 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; RUN: opt -S -passes='attributor' -aa-pipeline='basic-aa' -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=3 < %s | FileCheck %s --check-prefixes=CHECK,ATTRIBUTOR
+; RUN: opt -S -passes='cgscc(inline),attributor' -aa-pipeline='basic-aa' -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=3 < %s | FileCheck %s --check-prefixes=CHECK,INLINE_ATTRIBUTOR
+
+%S = type { %S* }
+
+; Inlining should nuke the invoke (and any inlined calls) here even with
+; argument promotion running along with it.
+define void @zot() personality i32 (...)* @wibble {
+; ATTRIBUTOR-LABEL: define {{[^@]+}}@zot() #0 personality i32 (...)* @wibble
+; ATTRIBUTOR-NEXT:  bb:
+; ATTRIBUTOR-NEXT:    call void @hoge()
+; ATTRIBUTOR-NEXT:    unreachable
+; ATTRIBUTOR:       bb1:
+; ATTRIBUTOR-NEXT:    unreachable
+; ATTRIBUTOR:       bb2:
+; ATTRIBUTOR-NEXT:    unreachable
+;
+; INLINE_ATTRIBUTOR-LABEL: define {{[^@]+}}@zot() #0 personality i32 (...)* @wibble
+; INLINE_ATTRIBUTOR-NEXT:  bb:
+; INLINE_ATTRIBUTOR-NEXT:    unreachable
+; INLINE_ATTRIBUTOR:       hoge.exit:
+; INLINE_ATTRIBUTOR-NEXT:    unreachable
+; INLINE_ATTRIBUTOR:       bb1:
+; INLINE_ATTRIBUTOR-NEXT:    unreachable
+; INLINE_ATTRIBUTOR:       bb2:
+; INLINE_ATTRIBUTOR-NEXT:    unreachable
+;
+bb:
+  invoke void @hoge()
+  to label %bb1 unwind label %bb2
+
+bb1:
+  unreachable
+
+bb2:
+  %tmp = landingpad { i8*, i32 }
+  cleanup
+  unreachable
+}
+
+define internal void @hoge() {
+; ATTRIBUTOR-LABEL: define {{[^@]+}}@hoge()
+; ATTRIBUTOR-NEXT:  bb:
+; ATTRIBUTOR-NEXT:    unreachable
+;
+bb:
+  %tmp = call fastcc i8* @spam(i1 (i8*)* @eggs)
+  %tmp1 = call fastcc i8* @spam(i1 (i8*)* @barney)
+  unreachable
+}
+
+define internal fastcc i8* @spam(i1 (i8*)* %arg) {
+bb:
+  unreachable
+}
+
+define internal i1 @eggs(i8* %arg) {
+bb:
+  %tmp = call zeroext i1 @barney(i8* %arg)
+  unreachable
+}
+
+define internal i1 @barney(i8* %arg) {
+bb:
+  ret i1 undef
+}
+
+define i32 @test_inf_promote_caller(i32 %arg) {
+; CHECK-LABEL: define {{[^@]+}}@test_inf_promote_caller
+; CHECK-SAME: (i32 [[ARG:%.*]])
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    unreachable
+;
+bb:
+  %tmp = alloca %S
+  %tmp1 = alloca %S
+  %tmp2 = call i32 @test_inf_promote_callee(%S* %tmp, %S* %tmp1)
+
+  ret i32 0
+}
+
+define internal i32 @test_inf_promote_callee(%S* %arg, %S* %arg1) {
+bb:
+  %tmp = getelementptr %S, %S* %arg1, i32 0, i32 0
+  %tmp2 = load %S*, %S** %tmp
+  %tmp3 = getelementptr %S, %S* %arg, i32 0, i32 0
+  %tmp4 = load %S*, %S** %tmp3
+  %tmp5 = call i32 @test_inf_promote_callee(%S* %tmp4, %S* %tmp2)
+
+  ret i32 0
+}
+
+declare i32 @wibble(...)
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/dbg.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/dbg.ll
new file mode 100644
index 000000000..a76a1c957
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/dbg.ll
@@ -0,0 +1,51 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; RUN: opt -S -passes=attributor -aa-pipeline='basic-aa' -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=2 < %s | FileCheck %s
+
+declare void @sink(i32)
+
+define internal void @test(i32** %X) !dbg !2 {
+; CHECK-LABEL: define {{[^@]+}}@test
+; CHECK-SAME: (i32** nocapture nonnull readonly align 8 dereferenceable(8) [[X:%.*]]) !dbg !3
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32*, i32** [[X]], align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* [[TMP1]], align 8
+; CHECK-NEXT:    call void @sink(i32 [[TMP2]])
+; CHECK-NEXT:    ret void
+;
+  %1 = load i32*, i32** %X, align 8
+  %2 = load i32, i32* %1, align 8
+  call void @sink(i32 %2)
+  ret void
+}
+
+%struct.pair = type { i32, i32 }
+
+define internal void @test_byval(%struct.pair* byval %P) {
+; CHECK-LABEL: define {{[^@]+}}@test_byval()
+; CHECK-NEXT:    ret void
+;
+  ret void
+}
+
+define void @caller(i32** %Y, %struct.pair* %P) {
+; CHECK-LABEL: define {{[^@]+}}@caller
+; CHECK-SAME: (i32** nocapture readonly [[Y:%.*]], %struct.pair* nocapture nofree readonly [[P:%.*]])
+; CHECK-NEXT:    call void @test(i32** nocapture readonly align 8 [[Y]]), !dbg !4
+; CHECK-NEXT:    call void @test_byval(), !dbg !5
+; CHECK-NEXT:    ret void
+;
+  call void @test(i32** %Y), !dbg !1
+
+  call void @test_byval(%struct.pair* %P), !dbg !6
+  ret void
+}
+
+
+!llvm.module.flags = !{!0}
+!llvm.dbg.cu = !{!3}
+
+!0 = !{i32 2, !"Debug Info Version", i32 3}
+!1 = !DILocation(line: 8, scope: !2)
+!2 = distinct !DISubprogram(name: "test", file: !5, line: 3, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !3, scopeLine: 3, scope: null)
+!3 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: LineTablesOnly, file: !5)
+!5 = !DIFile(filename: "test.c", directory: "")
+!6 = !DILocation(line: 9, scope: !2)
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/fp80.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/fp80.ll
new file mode 100644
index 000000000..153ce6893
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/fp80.ll
@@ -0,0 +1,72 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; RUN: opt -S -passes=attributor -aa-pipeline='basic-aa' -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=2 < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+%union.u = type { x86_fp80 }
+%struct.s = type { double, i16, i8, [5 x i8] }
+
+@b = internal global %struct.s { double 3.14, i16 9439, i8 25, [5 x i8] undef }, align 16
+
+%struct.Foo = type { i32, i64 }
+@a = internal global %struct.Foo { i32 1, i64 2 }, align 8
+
+define void @run() {
+; CHECK-LABEL: define {{[^@]+}}@run()
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @CaptureAStruct(%struct.Foo* nofree nonnull readonly align 8 dereferenceable(16) @a)
+; CHECK-NEXT:    unreachable
+;
+entry:
+  tail call i8 @UseLongDoubleUnsafely(%union.u* byval align 16 bitcast (%struct.s* @b to %union.u*))
+  tail call x86_fp80 @UseLongDoubleSafely(%union.u* byval align 16 bitcast (%struct.s* @b to %union.u*))
+  call i64 @AccessPaddingOfStruct(%struct.Foo* @a)
+  call i64 @CaptureAStruct(%struct.Foo* @a)
+  ret void
+}
+
+define internal i8 @UseLongDoubleUnsafely(%union.u* byval align 16 %arg) {
+entry:
+  %bitcast = bitcast %union.u* %arg to %struct.s*
+  %gep = getelementptr inbounds %struct.s, %struct.s* %bitcast, i64 0, i32 2
+  %result = load i8, i8* %gep
+  ret i8 %result
+}
+
+define internal x86_fp80 @UseLongDoubleSafely(%union.u* byval align 16 %arg) {
+  %gep = getelementptr inbounds %union.u, %union.u* %arg, i64 0, i32 0
+  %fp80 = load x86_fp80, x86_fp80* %gep
+  ret x86_fp80 %fp80
+}
+
+define internal i64 @AccessPaddingOfStruct(%struct.Foo* byval %a) {
+  %p = bitcast %struct.Foo* %a to i64*
+  %v = load i64, i64* %p
+  ret i64 %v
+}
+
+define internal i64 @CaptureAStruct(%struct.Foo* byval %a) {
+; CHECK-LABEL: define {{[^@]+}}@CaptureAStruct
+; CHECK-SAME: (%struct.Foo* nofree nonnull byval align 8 dereferenceable(16) [[A:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[A_PTR:%.*]] = alloca %struct.Foo*
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[PHI:%.*]] = phi %struct.Foo* [ null, [[ENTRY:%.*]] ], [ [[GEP:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = phi %struct.Foo* [ [[A]], [[ENTRY]] ], [ [[TMP0]], [[LOOP]] ]
+; CHECK-NEXT:    store %struct.Foo* [[PHI]], %struct.Foo** [[A_PTR]], align 8
+; CHECK-NEXT:    [[GEP]] = getelementptr [[STRUCT_FOO:%.*]], %struct.Foo* [[A]], i64 0
+; CHECK-NEXT:    br label [[LOOP]]
+;
+entry:
+  %a_ptr = alloca %struct.Foo*
+  br label %loop
+
+loop:
+  %phi = phi %struct.Foo* [ null, %entry ], [ %gep, %loop ]
+  %0   = phi %struct.Foo* [ %a, %entry ],   [ %0, %loop ]
+  store %struct.Foo* %phi, %struct.Foo** %a_ptr
+  %gep = getelementptr %struct.Foo, %struct.Foo* %a, i64 0
+  br label %loop
+}
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/inalloca.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/inalloca.ll
new file mode 100644
index 000000000..88c13ac98
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/inalloca.ll
@@ -0,0 +1,91 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; RUN: opt -S -passes=attributor -aa-pipeline='basic-aa' -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=2 < %s | FileCheck %s --check-prefixes=ATTRIBUTOR
+; RUN: opt -S -passes='globalopt,attributor' -aa-pipeline='basic-aa' -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=2 < %s | FileCheck %s --check-prefixes=GLOBALOPT_ATTRIBUTOR
+
+target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
+
+%struct.ss = type { i32, i32 }
+
+; Argpromote + sroa should change this to passing the two integers by value.
+define internal i32 @f(%struct.ss* inalloca  %s) {
+; ATTRIBUTOR-LABEL: define {{[^@]+}}@f
+; ATTRIBUTOR-SAME: (%struct.ss* inalloca noalias nocapture nofree nonnull align 4 dereferenceable(8) [[S:%.*]])
+; ATTRIBUTOR-NEXT:  entry:
+; ATTRIBUTOR-NEXT:    [[F0:%.*]] = getelementptr [[STRUCT_SS:%.*]], %struct.ss* [[S]], i32 0, i32 0
+; ATTRIBUTOR-NEXT:    [[F1:%.*]] = getelementptr [[STRUCT_SS]], %struct.ss* [[S]], i32 0, i32 1
+; ATTRIBUTOR-NEXT:    [[A:%.*]] = load i32, i32* [[F0]], align 4
+; ATTRIBUTOR-NEXT:    [[B:%.*]] = load i32, i32* [[F1]], align 4
+; ATTRIBUTOR-NEXT:    [[R:%.*]] = add i32 [[A]], [[B]]
+; ATTRIBUTOR-NEXT:    ret i32 [[R]]
+;
+; GLOBALOPT_ATTRIBUTOR-LABEL: define {{[^@]+}}@f
+; GLOBALOPT_ATTRIBUTOR-SAME: (%struct.ss* noalias nocapture nofree nonnull readonly align 4 dereferenceable(8) [[S:%.*]]) unnamed_addr
+; GLOBALOPT_ATTRIBUTOR-NEXT:  entry:
+; GLOBALOPT_ATTRIBUTOR-NEXT:    [[F0:%.*]] = getelementptr [[STRUCT_SS:%.*]], %struct.ss* [[S]], i32 0, i32 0
+; GLOBALOPT_ATTRIBUTOR-NEXT:    [[F1:%.*]] = getelementptr [[STRUCT_SS]], %struct.ss* [[S]], i32 0, i32 1
+; GLOBALOPT_ATTRIBUTOR-NEXT:    [[A:%.*]] = load i32, i32* [[F0]], align 4
+; GLOBALOPT_ATTRIBUTOR-NEXT:    [[B:%.*]] = load i32, i32* [[F1]], align 4
+; GLOBALOPT_ATTRIBUTOR-NEXT:    [[R:%.*]] = add i32 [[A]], [[B]]
+; GLOBALOPT_ATTRIBUTOR-NEXT:    ret i32 [[R]]
+;
+entry:
+  %f0 = getelementptr %struct.ss, %struct.ss* %s, i32 0, i32 0
+  %f1 = getelementptr %struct.ss, %struct.ss* %s, i32 0, i32 1
+  %a = load i32, i32* %f0, align 4
+  %b = load i32, i32* %f1, align 4
+  %r = add i32 %a, %b
+  ret i32 %r
+}
+
+define i32 @main() {
+; ATTRIBUTOR-LABEL: define {{[^@]+}}@main()
+; ATTRIBUTOR-NEXT:  entry:
+; ATTRIBUTOR-NEXT:    [[S:%.*]] = alloca inalloca [[STRUCT_SS:%.*]]
+; ATTRIBUTOR-NEXT:    [[F0:%.*]] = getelementptr [[STRUCT_SS]], %struct.ss* [[S]], i32 0, i32 0
+; ATTRIBUTOR-NEXT:    [[F1:%.*]] = getelementptr [[STRUCT_SS]], %struct.ss* [[S]], i32 0, i32 1
+; ATTRIBUTOR-NEXT:    store i32 1, i32* [[F0]], align 4
+; ATTRIBUTOR-NEXT:    store i32 2, i32* [[F1]], align 4
+; ATTRIBUTOR-NEXT:    [[R:%.*]] = call i32 @f(%struct.ss* inalloca noalias nocapture nofree nonnull align 4 dereferenceable(8) [[S]])
+; ATTRIBUTOR-NEXT:    ret i32 [[R]]
+;
+; GLOBALOPT_ATTRIBUTOR-LABEL: define {{[^@]+}}@main() local_unnamed_addr
+; GLOBALOPT_ATTRIBUTOR-NEXT:  entry:
+; GLOBALOPT_ATTRIBUTOR-NEXT:    [[S:%.*]] = alloca inalloca [[STRUCT_SS:%.*]]
+; GLOBALOPT_ATTRIBUTOR-NEXT:    [[F0:%.*]] = getelementptr [[STRUCT_SS]], %struct.ss* [[S]], i32 0, i32 0
+; GLOBALOPT_ATTRIBUTOR-NEXT:    [[F1:%.*]] = getelementptr [[STRUCT_SS]], %struct.ss* [[S]], i32 0, i32 1
+; GLOBALOPT_ATTRIBUTOR-NEXT:    store i32 1, i32* [[F0]], align 4
+; GLOBALOPT_ATTRIBUTOR-NEXT:    store i32 2, i32* [[F1]], align 4
+; GLOBALOPT_ATTRIBUTOR-NEXT:    [[R:%.*]] = call fastcc i32 @f(%struct.ss* noalias nocapture nofree nonnull readonly align 4 dereferenceable(8) [[S]])
+; GLOBALOPT_ATTRIBUTOR-NEXT:    ret i32 [[R]]
+;
+entry:
+  %S = alloca inalloca %struct.ss
+  %f0 = getelementptr %struct.ss, %struct.ss* %S, i32 0, i32 0
+  %f1 = getelementptr %struct.ss, %struct.ss* %S, i32 0, i32 1
+  store i32 1, i32* %f0, align 4
+  store i32 2, i32* %f1, align 4
+  %r = call i32 @f(%struct.ss* inalloca %S)
+  ret i32 %r
+}
+
+; Argpromote can't promote %a because of the icmp use.
+define internal i1 @g(%struct.ss* %a, %struct.ss* inalloca %b) nounwind  {
+entry:
+  %c = icmp eq %struct.ss* %a, %b
+  ret i1 %c
+}
+
+define i32 @test() {
+; ATTRIBUTOR-LABEL: define {{[^@]+}}@test()
+; ATTRIBUTOR-NEXT:  entry:
+; ATTRIBUTOR-NEXT:    ret i32 0
+;
+; GLOBALOPT_ATTRIBUTOR-LABEL: define {{[^@]+}}@test() local_unnamed_addr
+; GLOBALOPT_ATTRIBUTOR-NEXT:  entry:
+; GLOBALOPT_ATTRIBUTOR-NEXT:    ret i32 0
+;
+entry:
+  %S = alloca inalloca %struct.ss
+  %c = call i1 @g(%struct.ss* %S, %struct.ss* inalloca %S)
+  ret i32 0
+}
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/invalidation.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/invalidation.ll
new file mode 100644
index 000000000..e49cc7e2e
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/invalidation.ll
@@ -0,0 +1,50 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; Check that when argument promotion changes a function in some parent node of
+; the call graph, any analyses that happened to be cached for that function are
+; actually invalidated. We are using `demanded-bits` here because when printed
+; it will end up caching a value for every instruction, making it easy to
+; detect the instruction-level changes that will fail here. With improper
+; invalidation this will crash in the second printer as it tries to reuse
+; now-invalid demanded bits.
+;
+; RUN: opt < %s -passes='function(print<demanded-bits>),attributor,function(print<demanded-bits>)' -S | FileCheck %s
+
+@G = constant i32 0
+
+define internal i32 @a(i32* %x) {
+; CHECK-LABEL: define {{[^@]+}}@a
+; CHECK-SAME: (i32* [[X:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[V:%.*]] = load i32, i32* [[X]]
+; CHECK-NEXT:    ret i32 [[V]]
+;
+entry:
+  %v = load i32, i32* %x
+  ret i32 %v
+}
+
+define i32 @b() {
+; CHECK-LABEL: define {{[^@]+}}@b()
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[V:%.*]] = call i32 @a(i32* @G)
+; CHECK-NEXT:    ret i32 [[V]]
+;
+entry:
+  %v = call i32 @a(i32* @G)
+  ret i32 %v
+}
+
+define i32 @c() {
+; CHECK-LABEL: define {{[^@]+}}@c()
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[V1:%.*]] = call i32 @a(i32* @G)
+; CHECK-NEXT:    [[V2:%.*]] = call i32 @b()
+; CHECK-NEXT:    [[RESULT:%.*]] = add i32 [[V1]], [[V2]]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %v1 = call i32 @a(i32* @G)
+  %v2 = call i32 @b()
+  %result = add i32 %v1, %v2
+  ret i32 %result
+}
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/live_called_from_dead.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/live_called_from_dead.ll
new file mode 100644
index 000000000..a5ca51e9b
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/live_called_from_dead.ll
@@ -0,0 +1,55 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; RUN: opt -S -basicaa -attributor -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=3 < %s | FileCheck %s --check-prefixes=CHECK,OLDPM_MODULE
+; RUN: opt -S -passes='attributor' -aa-pipeline='basic-aa' -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=3 < %s | FileCheck %s --check-prefixes=CHECK,NEWPM_MODULE
+
+; OLDPM_MODULE-NOT: @dead
+; NEWPM_MODULE-NOT: @dead
+
+define internal void @dead() {
+  call i32 @test(i32* null, i32* null)
+  ret void
+}
+
+define internal i32 @test(i32* %X, i32* %Y) {
+; CHECK-LABEL: define {{[^@]+}}@test()
+; CHECK-NEXT:    br i1 true, label [[LIVE:%.*]], label [[DEAD:%.*]]
+; CHECK:       live:
+; CHECK-NEXT:    ret i32 0
+; CHECK:       dead:
+; CHECK-NEXT:    unreachable
+;
+  br i1 true, label %live, label %dead
+live:
+  ret i32 0
+dead:
+  call i32 @caller(i32* null)
+  call void @dead()
+  ret i32 1
+}
+
+define internal i32 @caller(i32* %B) {
+; CHECK-LABEL: define {{[^@]+}}@caller()
+; CHECK-NEXT:    [[A:%.*]] = alloca i32
+; CHECK-NEXT:    store i32 1, i32* [[A]], align 4
+; CHECK-NEXT:    [[C:%.*]] = call i32 @test()
+; CHECK-NEXT:    ret i32 0
+;
+  %A = alloca i32
+  store i32 1, i32* %A
+  %C = call i32 @test(i32* %A, i32* %B)
+  ret i32 %C
+}
+
+define i32 @callercaller() {
+; CHECK-LABEL: define {{[^@]+}}@callercaller()
+; CHECK-NEXT:    [[B:%.*]] = alloca i32
+; CHECK-NEXT:    store i32 2, i32* [[B]], align 4
+; CHECK-NEXT:    [[X:%.*]] = call i32 @caller()
+; CHECK-NEXT:    ret i32 0
+;
+  %B = alloca i32
+  store i32 2, i32* %B
+  %X = call i32 @caller(i32* %B)
+  ret i32 %X
+}
+
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/musttail.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/musttail.ll
new file mode 100644
index 000000000..2c83cbb4a
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/musttail.ll
@@ -0,0 +1,70 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; RUN: opt -S -passes=attributor -aa-pipeline='basic-aa' -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=3 < %s | FileCheck %s
+; PR36543
+
+; Don't promote arguments of musttail callee
+
+%T = type { i32, i32, i32, i32 }
+
+define internal i32 @test(%T* %p) {
+; CHECK-LABEL: define {{[^@]+}}@test
+; CHECK-SAME: (%T* nocapture nofree readonly [[P:%.*]])
+; CHECK-NEXT:    [[A_GEP:%.*]] = getelementptr [[T:%.*]], %T* [[P]], i64 0, i32 3
+; CHECK-NEXT:    [[B_GEP:%.*]] = getelementptr [[T]], %T* [[P]], i64 0, i32 2
+; CHECK-NEXT:    [[A:%.*]] = load i32, i32* [[A_GEP]]
+; CHECK-NEXT:    [[B:%.*]] = load i32, i32* [[B_GEP]]
+; CHECK-NEXT:    [[V:%.*]] = add i32 [[A]], [[B]]
+; CHECK-NEXT:    ret i32 [[V]]
+;
+  %a.gep = getelementptr %T, %T* %p, i64 0, i32 3
+  %b.gep = getelementptr %T, %T* %p, i64 0, i32 2
+  %a = load i32, i32* %a.gep
+  %b = load i32, i32* %b.gep
+  %v = add i32 %a, %b
+  ret i32 %v
+}
+
+define i32 @caller(%T* %p) {
+; CHECK-LABEL: define {{[^@]+}}@caller
+; CHECK-SAME: (%T* nocapture nofree readonly [[P:%.*]])
+; CHECK-NEXT:    [[V:%.*]] = musttail call i32 @test(%T* nocapture nofree readonly [[P]])
+; CHECK-NEXT:    ret i32 [[V]]
+;
+  %v = musttail call i32 @test(%T* %p)
+  ret i32 %v
+}
+
+; Don't promote arguments of musttail caller
+
+define i32 @foo(%T* %p, i32 %v) {
+; CHECK-LABEL: define {{[^@]+}}@foo
+; CHECK-SAME: (%T* nocapture nofree readnone [[P:%.*]], i32 [[V:%.*]])
+; CHECK-NEXT:    ret i32 0
+;
+  ret i32 0
+}
+
+define internal i32 @test2(%T* %p, i32 %p2) {
+; CHECK-LABEL: define {{[^@]+}}@test2
+; CHECK-SAME: (%T* nocapture nofree readonly [[P:%.*]], i32 [[P2:%.*]])
+; CHECK-NEXT:    [[CA:%.*]] = musttail call i32 @foo(%T* undef, i32 undef)
+; CHECK-NEXT:    ret i32 [[CA]]
+;
+  %a.gep = getelementptr %T, %T* %p, i64 0, i32 3
+  %b.gep = getelementptr %T, %T* %p, i64 0, i32 2
+  %a = load i32, i32* %a.gep
+  %b = load i32, i32* %b.gep
+  %v = add i32 %a, %b
+  %ca = musttail call i32 @foo(%T* undef, i32 %v)
+  ret i32 %ca
+}
+
+define i32 @caller2(%T* %g) {
+; CHECK-LABEL: define {{[^@]+}}@caller2
+; CHECK-SAME: (%T* nocapture nofree readonly [[G:%.*]])
+; CHECK-NEXT:    [[V:%.*]] = call i32 @test2(%T* nocapture nofree readonly undef, i32 undef)
+; CHECK-NEXT:    ret i32 0
+;
+  %v = call i32 @test2(%T* %g, i32 0)
+  ret i32 %v
+}
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/naked_functions.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/naked_functions.ll
new file mode 100644
index 000000000..4bfe138f4
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/naked_functions.ll
@@ -0,0 +1,34 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; RUN: opt -S -passes=attributor -aa-pipeline='basic-aa' -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=1 < %s | FileCheck %s
+
+; Don't promote paramaters of/arguments to naked functions
+
+@g = common global i32 0, align 4
+
+define i32 @bar() {
+; CHECK-LABEL: define {{[^@]+}}@bar()
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @foo(i32* nonnull align 4 dereferenceable(4) @g)
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+entry:
+  %call = call i32 @foo(i32* @g)
+  ret i32 %call
+}
+
+define internal i32 @foo(i32*) #0 {
+; CHECK-LABEL: define {{[^@]+}}@foo
+; CHECK-SAME: (i32* [[TMP0:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[RETVAL:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    call void asm sideeffect "ldr r0, [r0] \0Abx lr \0A", ""()
+; CHECK-NEXT:    unreachable
+;
+entry:
+  %retval = alloca i32, align 4
+  call void asm sideeffect "ldr r0, [r0] \0Abx lr        \0A", ""()
+  unreachable
+}
+
+
+attributes #0 = { naked }
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/nonzero-address-spaces.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/nonzero-address-spaces.ll
new file mode 100644
index 000000000..d08969c0a
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/nonzero-address-spaces.ll
@@ -0,0 +1,35 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; RUN: opt -S -passes=attributor -aa-pipeline='basic-aa' -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=1 < %s | FileCheck %s
+
+; ArgumentPromotion should preserve the default function address space
+; from the data layout.
+
+target datalayout = "e-P1-p:16:8-i8:8-i16:8-i32:8-i64:8-f32:8-f64:8-n8-a:8"
+
+@g = common global i32 0, align 4
+
+define i32 @bar() {
+; CHECK-LABEL: define {{[^@]+}}@bar() addrspace(1)
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CALL:%.*]] = call addrspace(1) i32 @foo()
+; CHECK-NEXT:    unreachable
+;
+
+entry:
+  %call = call i32 @foo(i32* @g)
+  ret i32 %call
+}
+
+define internal i32 @foo(i32*) {
+; CHECK-LABEL: define {{[^@]+}}@foo() addrspace(1)
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[RETVAL:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    call addrspace(0) void asm sideeffect "ldr r0, [r0] \0Abx lr \0A", ""()
+; CHECK-NEXT:    unreachable
+;
+entry:
+  %retval = alloca i32, align 4
+  call void asm sideeffect "ldr r0, [r0] \0Abx lr        \0A", ""()
+  unreachable
+}
+
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/pr27568.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/pr27568.ll
new file mode 100644
index 000000000..f2ff8dc63
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/pr27568.ll
@@ -0,0 +1,44 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; RUN: opt -S -passes=attributor -aa-pipeline='basic-aa' -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=1 < %s | FileCheck %s
+target triple = "x86_64-pc-windows-msvc"
+
+define internal void @callee(i8*) {
+; CHECK-LABEL: define {{[^@]+}}@callee()
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    call void @thunk()
+; CHECK-NEXT:    ret void
+;
+entry:
+  call void @thunk()
+  ret void
+}
+
+define void @test1() personality i32 (...)* @__CxxFrameHandler3 {
+; CHECK-LABEL: define {{[^@]+}}@test1() personality i32 (...)* @__CxxFrameHandler3
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    invoke void @thunk()
+; CHECK-NEXT:    to label [[OUT:%.*]] unwind label [[CPAD:%.*]]
+; CHECK:       out:
+; CHECK-NEXT:    ret void
+; CHECK:       cpad:
+; CHECK-NEXT:    [[PAD:%.*]] = cleanuppad within none []
+; CHECK-NEXT:    call void @callee() [ "funclet"(token [[PAD]]) ]
+; CHECK-NEXT:    cleanupret from [[PAD]] unwind to caller
+;
+entry:
+  invoke void @thunk()
+  to label %out unwind label %cpad
+
+out:
+  ret void
+
+cpad:
+  %pad = cleanuppad within none []
+  call void @callee(i8* null) [ "funclet"(token %pad) ]
+  cleanupret from %pad unwind to caller
+}
+
+
+declare void @thunk()
+
+declare i32 @__CxxFrameHandler3(...)
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/pr3085.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/pr3085.ll
new file mode 100644
index 000000000..5b8b7da46
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/pr3085.ll
@@ -0,0 +1,1945 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; RUN: opt -disable-output -passes=attributor -aa-pipeline='basic-aa' -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=1 < %s
+; PR 3085
+
+	%struct.Lit = type { i8 }
+
+define fastcc %struct.Lit* @import_lit(i32 %lit) nounwind {
+entry:
+  br i1 false, label %bb, label %bb1
+
+bb:		; preds = %entry
+  unreachable
+
+bb1:		; preds = %entry
+  br label %bb3
+
+bb2:		; preds = %bb3
+  br label %bb3
+
+bb3:		; preds = %bb2, %bb1
+  br i1 false, label %bb2, label %bb6
+
+bb6:		; preds = %bb3
+  br i1 false, label %bb.i.i, label %bb1.i.i
+
+bb.i.i:		; preds = %bb6
+  br label %int2lit.exit
+
+bb1.i.i:		; preds = %bb6
+  br label %int2lit.exit
+
+int2lit.exit:		; preds = %bb1.i.i, %bb.i.i
+  ret %struct.Lit* null
+}
+
+define fastcc i32 @picosat_main(i32 %argc, i8** %argv) nounwind {
+entry:
+  br i1 false, label %bb.i, label %picosat_time_stamp.exit
+
+bb.i:		; preds = %entry
+  br label %picosat_time_stamp.exit
+
+picosat_time_stamp.exit:		; preds = %bb.i, %entry
+  br label %bb108
+
+bb:		; preds = %bb108
+  br i1 false, label %bb1, label %bb2
+
+bb1:		; preds = %bb
+  br label %bb106
+
+bb2:		; preds = %bb
+  br i1 false, label %bb3, label %bb4
+
+bb3:		; preds = %bb2
+  br label %bb106
+
+bb4:		; preds = %bb2
+  br i1 false, label %bb5, label %bb6
+
+bb5:		; preds = %bb4
+  br label %bb106
+
+bb6:		; preds = %bb4
+  br i1 false, label %bb7, label %bb8
+
+bb7:		; preds = %bb6
+  br label %bb106
+
+bb8:		; preds = %bb6
+  br i1 false, label %bb106, label %bb10
+
+bb10:		; preds = %bb8
+  br i1 false, label %bb106, label %bb12
+
+bb12:		; preds = %bb10
+  br i1 false, label %bb106, label %bb14
+
+bb14:		; preds = %bb12
+  br i1 false, label %bb15, label %bb19
+
+bb15:		; preds = %bb14
+  br i1 false, label %bb16, label %bb17
+
+bb16:		; preds = %bb15
+  br label %bb106
+
+bb17:		; preds = %bb15
+  br label %bb106
+
+bb19:		; preds = %bb14
+  br i1 false, label %bb20, label %bb28
+
+bb20:		; preds = %bb19
+  br i1 false, label %bb21, label %bb22
+
+bb21:		; preds = %bb20
+  br label %bb106
+
+bb22:		; preds = %bb20
+  br i1 false, label %bb106, label %bb24
+
+bb24:		; preds = %bb22
+  br i1 false, label %bb106, label %bb26
+
+bb26:		; preds = %bb24
+  br label %bb106
+
+bb28:		; preds = %bb19
+  br i1 false, label %bb29, label %bb35
+
+bb29:		; preds = %bb28
+  br i1 false, label %bb30, label %bb31
+
+bb30:		; preds = %bb29
+  br label %bb106
+
+bb31:		; preds = %bb29
+  br i1 false, label %bb32, label %bb33
+
+bb32:		; preds = %bb31
+  br label %bb106
+
+bb33:		; preds = %bb31
+  br label %bb106
+
+bb35:		; preds = %bb28
+  br i1 false, label %bb36, label %bb40
+
+bb36:		; preds = %bb35
+  br i1 false, label %bb37, label %bb38
+
+bb37:		; preds = %bb36
+  br label %bb106
+
+bb38:		; preds = %bb36
+  br label %bb106
+
+bb40:		; preds = %bb35
+  br i1 false, label %bb41, label %bb49
+
+bb41:		; preds = %bb40
+  br i1 false, label %bb43, label %bb42
+
+bb42:		; preds = %bb41
+  br label %bb106
+
+bb43:		; preds = %bb41
+  br i1 false, label %bb44, label %bb45
+
+bb44:		; preds = %bb43
+  br label %bb106
+
+bb45:		; preds = %bb43
+  br i1 false, label %bb46, label %bb47
+
+bb46:		; preds = %bb45
+  br label %bb106
+
+bb47:		; preds = %bb45
+  br label %bb106
+
+bb49:		; preds = %bb40
+  br i1 false, label %bb50, label %bb56
+
+bb50:		; preds = %bb49
+  br i1 false, label %bb52, label %bb51
+
+bb51:		; preds = %bb50
+  br label %bb106
+
+bb52:		; preds = %bb50
+  br i1 false, label %bb53, label %bb54
+
+bb53:		; preds = %bb52
+  br label %bb106
+
+bb54:		; preds = %bb52
+  br label %bb106
+
+bb56:		; preds = %bb49
+  br i1 false, label %bb57, label %bb63
+
+bb57:		; preds = %bb56
+  br i1 false, label %bb59, label %bb58
+
+bb58:		; preds = %bb57
+  br label %bb106
+
+bb59:		; preds = %bb57
+  br i1 false, label %bb60, label %bb61
+
+bb60:		; preds = %bb59
+  br label %bb106
+
+bb61:		; preds = %bb59
+  br label %bb106
+
+bb63:		; preds = %bb56
+  br i1 false, label %bb64, label %bb70
+
+bb64:		; preds = %bb63
+  br i1 false, label %bb66, label %bb65
+
+bb65:		; preds = %bb64
+  br label %bb106
+
+bb66:		; preds = %bb64
+  br i1 false, label %bb67, label %bb68
+
+bb67:		; preds = %bb66
+  br label %bb106
+
+bb68:		; preds = %bb66
+  br label %bb106
+
+bb70:		; preds = %bb63
+  br i1 false, label %bb71, label %bb79
+
+bb71:		; preds = %bb70
+  br i1 false, label %bb73, label %bb72
+
+bb72:		; preds = %bb71
+  br label %bb106
+
+bb73:		; preds = %bb71
+  br i1 false, label %bb74, label %bb75
+
+bb74:		; preds = %bb73
+  br label %bb106
+
+bb75:		; preds = %bb73
+  br i1 false, label %bb76, label %bb77
+
+bb76:		; preds = %bb75
+  br label %bb106
+
+bb77:		; preds = %bb75
+  br label %bb106
+
+bb79:		; preds = %bb70
+  br i1 false, label %bb80, label %bb86
+
+bb80:		; preds = %bb79
+  br i1 false, label %bb82, label %bb81
+
+bb81:		; preds = %bb80
+  br label %bb106
+
+bb82:		; preds = %bb80
+  br i1 false, label %bb83, label %bb84
+
+bb83:		; preds = %bb82
+  br label %bb106
+
+bb84:		; preds = %bb82
+  br label %bb106
+
+bb86:		; preds = %bb79
+  br i1 false, label %bb87, label %bb93
+
+bb87:		; preds = %bb86
+  br i1 false, label %bb89, label %bb88
+
+bb88:		; preds = %bb87
+  br label %bb106
+
+bb89:		; preds = %bb87
+  br i1 false, label %bb90, label %bb91
+
+bb90:		; preds = %bb89
+  br label %bb106
+
+bb91:		; preds = %bb89
+  br label %bb106
+
+bb93:		; preds = %bb86
+  br i1 false, label %bb94, label %bb95
+
+bb94:		; preds = %bb93
+  br label %bb106
+
+bb95:		; preds = %bb93
+  br i1 false, label %bb98, label %bb97
+
+bb97:		; preds = %bb95
+  br label %bb106
+
+bb98:		; preds = %bb95
+  br i1 false, label %bb103, label %bb1.i24
+
+bb1.i24:		; preds = %bb98
+  br i1 false, label %bb99, label %bb103
+
+bb99:		; preds = %bb1.i24
+  br i1 false, label %bb101, label %bb100
+
+bb100:		; preds = %bb99
+  br label %bb102
+
+bb101:		; preds = %bb99
+  br label %bb102
+
+bb102:		; preds = %bb101, %bb100
+  br label %bb106
+
+bb103:		; preds = %bb1.i24, %bb98
+  br i1 false, label %bb104, label %bb105
+
+bb104:		; preds = %bb103
+  br label %bb106
+
+bb105:		; preds = %bb103
+  br label %bb106
+
+bb106:		; preds = %bb105, %bb104, %bb102, %bb97, %bb94, %bb91, %bb90, %bb88, %bb84, %bb83, %bb81, %bb77, %bb76, %bb74, %bb72, %bb68, %bb67, %bb65, %bb61, %bb60, %bb58, %bb54, %bb53, %bb51, %bb47, %bb46, %bb44, %bb42, %bb38, %bb37, %bb33, %bb32, %bb30, %bb26, %bb24, %bb22, %bb21, %bb17, %bb16, %bb12, %bb10, %bb8, %bb7, %bb5, %bb3, %bb1
+  br i1 false, label %bb108, label %bb110
+
+bb108:		; preds = %bb106, %picosat_time_stamp.exit
+  br i1 false, label %bb, label %bb110
+
+bb110:		; preds = %bb108, %bb106
+  br i1 false, label %bb112, label %bb171
+
+bb112:		; preds = %bb110
+  br i1 false, label %bb114, label %bb113
+
+bb113:		; preds = %bb112
+  br label %bb114
+
+bb114:		; preds = %bb113, %bb112
+  br i1 false, label %bb.i.i35, label %bb1.i.i36
+
+bb.i.i35:		; preds = %bb114
+  unreachable
+
+bb1.i.i36:		; preds = %bb114
+  br i1 false, label %bb5.i.i.i41, label %bb6.i.i.i42
+
+bb5.i.i.i41:		; preds = %bb1.i.i36
+  unreachable
+
+bb6.i.i.i42:		; preds = %bb1.i.i36
+  br i1 false, label %bb7.i.i.i43, label %bb8.i.i.i44
+
+bb7.i.i.i43:		; preds = %bb6.i.i.i42
+  br label %bb8.i.i.i44
+
+bb8.i.i.i44:		; preds = %bb7.i.i.i43, %bb6.i.i.i42
+  br i1 false, label %picosat_init.exit, label %bb14.i.i
+
+bb14.i.i:		; preds = %bb8.i.i.i44
+  br label %picosat_init.exit
+
+picosat_init.exit:		; preds = %bb14.i.i, %bb8.i.i.i44
+  br i1 false, label %bb116, label %bb115
+
+bb115:		; preds = %picosat_init.exit
+  br label %bb116
+
+bb116:		; preds = %bb115, %picosat_init.exit
+  br i1 false, label %bb119, label %bb118
+
+bb118:		; preds = %bb116
+  br label %bb119
+
+bb119:		; preds = %bb118, %bb116
+  br i1 false, label %bb121, label %bb120
+
+bb120:		; preds = %bb119
+  br label %bb121
+
+bb121:		; preds = %bb120, %bb119
+  br i1 false, label %bb126, label %bb122
+
+bb122:		; preds = %bb121
+  br label %bb126
+
+bb126:		; preds = %bb122, %bb121
+  br i1 false, label %bb128, label %bb127
+
+bb127:		; preds = %bb126
+  br label %bb128
+
+bb128:		; preds = %bb127, %bb126
+  br label %SKIP_COMMENTS.i
+
+SKIP_COMMENTS.i.loopexit:		; preds = %bb.i149, %bb.i149
+  br label %SKIP_COMMENTS.i.backedge
+
+SKIP_COMMENTS.i:		; preds = %SKIP_COMMENTS.i.backedge, %bb128
+  br i1 false, label %bb.i149.preheader, label %bb3.i152
+
+bb.i149.preheader:		; preds = %SKIP_COMMENTS.i
+  br label %bb.i149
+
+bb.i149:		; preds = %bb.i149, %bb.i149.preheader
+  switch i32 0, label %bb.i149 [
+  i32 -1, label %SKIP_COMMENTS.i.loopexit
+  i32 10, label %SKIP_COMMENTS.i.loopexit
+  ]
+
+bb3.i152:		; preds = %SKIP_COMMENTS.i
+  br i1 false, label %bb4.i153, label %SKIP_COMMENTS.i.backedge
+
+SKIP_COMMENTS.i.backedge:		; preds = %bb3.i152, %SKIP_COMMENTS.i.loopexit
+  br label %SKIP_COMMENTS.i
+
+bb4.i153:		; preds = %bb3.i152
+  br i1 false, label %bb5.i154, label %bb129
+
+bb5.i154:		; preds = %bb4.i153
+  br i1 false, label %bb129, label %bb6.i155.preheader
+
+bb6.i155.preheader:		; preds = %bb5.i154
+  br label %bb6.i155
+
+bb6.i155:		; preds = %bb6.i155, %bb6.i155.preheader
+  br i1 false, label %bb7.i156, label %bb6.i155
+
+bb7.i156:		; preds = %bb6.i155
+  br i1 false, label %bb8.i157, label %bb129
+
+bb8.i157:		; preds = %bb7.i156
+  br i1 false, label %bb9.i158, label %bb129
+
+bb9.i158:		; preds = %bb8.i157
+  br i1 false, label %bb10.i159, label %bb129
+
+bb10.i159:		; preds = %bb9.i158
+  br i1 false, label %bb129, label %bb11.i160.preheader
+
+bb11.i160.preheader:		; preds = %bb10.i159
+  br label %bb11.i160
+
+bb11.i160:		; preds = %bb11.i160, %bb11.i160.preheader
+  br i1 false, label %bb12.i161, label %bb11.i160
+
+bb12.i161:		; preds = %bb11.i160
+  br i1 false, label %bb129, label %bb15.i165.preheader
+
+bb15.i165.preheader:		; preds = %bb12.i161
+  br label %bb15.i165
+
+bb14.i163:		; preds = %bb15.i165
+  br label %bb15.i165
+
+bb15.i165:		; preds = %bb14.i163, %bb15.i165.preheader
+  br i1 false, label %bb16.i166, label %bb14.i163
+
+bb16.i166:		; preds = %bb15.i165
+  br i1 false, label %bb129, label %bb17.i167.preheader
+
+bb17.i167.preheader:		; preds = %bb16.i166
+  br label %bb17.i167
+
+bb17.i167:		; preds = %bb17.i167, %bb17.i167.preheader
+  br i1 false, label %bb18.i168, label %bb17.i167
+
+bb18.i168:		; preds = %bb17.i167
+  br i1 false, label %bb129, label %bb21.i172.preheader
+
+bb21.i172.preheader:		; preds = %bb18.i168
+  br label %bb21.i172
+
+bb20.i170:		; preds = %bb21.i172
+  br label %bb21.i172
+
+bb21.i172:		; preds = %bb20.i170, %bb21.i172.preheader
+  br i1 false, label %bb22.i173, label %bb20.i170
+
+bb22.i173:		; preds = %bb21.i172
+  br i1 false, label %bb24.i175, label %bb129
+
+bb24.i175:		; preds = %bb22.i173
+  br i1 false, label %bb26.i180, label %bb25.i176
+
+bb25.i176:		; preds = %bb24.i175
+  br label %bb26.i180
+
+bb26.i180:		; preds = %bb25.i176, %bb24.i175
+  br i1 false, label %bb.i.i181, label %bb3.i.i184.preheader
+
+bb.i.i181:		; preds = %bb26.i180
+  br label %bb3.i.i184.preheader
+
+bb3.i.i184.preheader:		; preds = %bb.i.i181, %bb26.i180
+  br label %bb3.i.i184
+
+bb2.i.i183:		; preds = %bb3.i.i184
+  br label %bb3.i.i184
+
+bb3.i.i184:		; preds = %bb2.i.i183, %bb3.i.i184.preheader
+  br i1 false, label %bb2.i.i183, label %bb4.i.i185
+
+bb4.i.i185:		; preds = %bb3.i.i184
+  br i1 false, label %bb.i.i.i186, label %picosat_adjust.exit.i
+
+bb.i.i.i186:		; preds = %bb4.i.i185
+  br label %picosat_adjust.exit.i
+
+picosat_adjust.exit.i:		; preds = %bb.i.i.i186, %bb4.i.i185
+  br i1 false, label %bb28.i188, label %bb27.i187
+
+bb27.i187:		; preds = %picosat_adjust.exit.i
+  br label %bb28.i188
+
+bb28.i188:		; preds = %bb27.i187, %picosat_adjust.exit.i
+  br label %READ_LITERAL.i.outer
+
+READ_LITERAL.i.outer:		; preds = %READ_LITERAL.i.outer.backedge, %bb28.i188
+  br label %READ_LITERAL.i
+
+READ_LITERAL.i.loopexit:		; preds = %bb29.i189, %bb29.i189
+  br label %READ_LITERAL.i.backedge
+
+READ_LITERAL.i:		; preds = %READ_LITERAL.i.backedge, %READ_LITERAL.i.outer
+  switch i32 0, label %bb39.i199 [
+  i32 99, label %bb29.i189.preheader
+  i32 -1, label %bb33.i193
+  ]
+
+bb29.i189.preheader:		; preds = %READ_LITERAL.i
+  br label %bb29.i189
+
+bb29.i189:		; preds = %bb29.i189, %bb29.i189.preheader
+  switch i32 0, label %bb29.i189 [
+  i32 -1, label %READ_LITERAL.i.loopexit
+  i32 10, label %READ_LITERAL.i.loopexit
+  ]
+
+bb33.i193:		; preds = %READ_LITERAL.i
+  br i1 false, label %bb35.i195, label %parse.exit
+
+bb35.i195:		; preds = %bb33.i193
+  br i1 false, label %bb38.i198, label %parse.exit
+
+bb38.i198:		; preds = %bb35.i195
+  br label %parse.exit
+
+bb39.i199:		; preds = %READ_LITERAL.i
+  br i1 false, label %bb40.i200, label %READ_LITERAL.i.backedge
+
+READ_LITERAL.i.backedge:		; preds = %bb39.i199, %READ_LITERAL.i.loopexit
+  br label %READ_LITERAL.i
+
+bb40.i200:		; preds = %bb39.i199
+  br i1 false, label %bb41.i201, label %bb42.i202
+
+bb41.i201:		; preds = %bb40.i200
+  br label %bb42.i202
+
+bb42.i202:		; preds = %bb41.i201, %bb40.i200
+  br i1 false, label %parse.exit.loopexit, label %bb46.i.preheader
+
+bb46.i.preheader:		; preds = %bb42.i202
+  br label %bb46.i
+
+bb45.i:		; preds = %bb46.i
+  br label %bb46.i
+
+bb46.i:		; preds = %bb45.i, %bb46.i.preheader
+  br i1 false, label %bb47.i, label %bb45.i
+
+bb47.i:		; preds = %bb46.i
+  br i1 false, label %parse.exit.loopexit, label %bb50.i
+
+bb50.i:		; preds = %bb47.i
+  br i1 false, label %bb55.i, label %bb51.i
+
+bb51.i:		; preds = %bb50.i
+  br i1 false, label %parse.exit.loopexit, label %bb54.i
+
+bb54.i:		; preds = %bb51.i
+  br label %bb56.i
+
+bb55.i:		; preds = %bb50.i
+  br label %bb56.i
+
+bb56.i:		; preds = %bb55.i, %bb54.i
+  br i1 false, label %bb3.i11.i, label %bb.i8.i
+
+bb.i8.i:		; preds = %bb56.i
+  br i1 false, label %bb1.i9.i, label %bb3.i11.i
+
+bb1.i9.i:		; preds = %bb.i8.i
+  br i1 false, label %bb3.i11.i, label %bb2.i10.i
+
+bb2.i10.i:		; preds = %bb1.i9.i
+  unreachable
+
+bb3.i11.i:		; preds = %bb1.i9.i, %bb.i8.i, %bb56.i
+  br i1 false, label %bb7.i.i208, label %bb6.i.i207
+
+bb6.i.i207:		; preds = %bb3.i11.i
+  br label %READ_LITERAL.i.outer.backedge
+
+bb7.i.i208:		; preds = %bb3.i11.i
+  br i1 false, label %bb53.i.i.i.i.preheader, label %bb.i.i.i.i210.preheader
+
+bb.i.i.i.i210.preheader:		; preds = %bb7.i.i208
+  br label %bb.i.i.i.i210
+
+bb.i.i.i.i210:		; preds = %bb.i.i.i.i210.backedge, %bb.i.i.i.i210.preheader
+  br i1 false, label %bb17.i.i.i.i, label %bb18.i.i.i.i
+
+bb17.i.i.i.i:		; preds = %bb.i.i.i.i210
+  br label %bb18.i.i.i.i
+
+bb18.i.i.i.i:		; preds = %bb17.i.i.i.i, %bb.i.i.i.i210
+  br i1 false, label %bb19.i.i.i.i, label %bb20.i.i.i.i
+
+bb19.i.i.i.i:		; preds = %bb18.i.i.i.i
+  br label %bb20.i.i.i.i
+
+bb20.i.i.i.i:		; preds = %bb19.i.i.i.i, %bb18.i.i.i.i
+  br i1 false, label %bb21.i.i.i.i, label %bb22.i.i.i.i
+
+bb21.i.i.i.i:		; preds = %bb20.i.i.i.i
+  br label %bb22.i.i.i.i
+
+bb22.i.i.i.i:		; preds = %bb21.i.i.i.i, %bb20.i.i.i.i
+  br label %bb23.i.i.i.i.outer
+
+bb23.i.i.i.i.outer:		; preds = %bb28.i.i.i.i, %bb22.i.i.i.i
+  br label %bb23.i.i.i.i
+
+bb23.i.i.i.i:		; preds = %bb23.i.i.i.i, %bb23.i.i.i.i.outer
+  br i1 false, label %bb23.i.i.i.i, label %bb26.i.i.i.i.preheader
+
+bb26.i.i.i.i.preheader:		; preds = %bb23.i.i.i.i
+  br label %bb26.i.i.i.i
+
+bb26.i.i.i.i:		; preds = %bb26.i.i.i.i, %bb26.i.i.i.i.preheader
+  br i1 false, label %bb27.i.i.i.i, label %bb26.i.i.i.i
+
+bb27.i.i.i.i:		; preds = %bb26.i.i.i.i
+  br i1 false, label %bb28.i.i.i.i, label %bb29.i.i.i.i
+
+bb28.i.i.i.i:		; preds = %bb27.i.i.i.i
+  br label %bb23.i.i.i.i.outer
+
+bb29.i.i.i.i:		; preds = %bb27.i.i.i.i
+  br i1 false, label %bb33.i.i.i.i, label %bb44.i.i.i.i
+
+bb33.i.i.i.i:		; preds = %bb29.i.i.i.i
+  br i1 false, label %bb34.i.i.i.i, label %bb38.i.i.i.i
+
+bb34.i.i.i.i:		; preds = %bb33.i.i.i.i
+  br i1 false, label %bb37.i.i.i.i, label %bb35.i.i.i.i
+
+bb35.i.i.i.i:		; preds = %bb34.i.i.i.i
+  br label %bb37.i.i.i.i
+
+bb37.i.i.i.i:		; preds = %bb35.i.i.i.i, %bb34.i.i.i.i
+  br label %bb38.i.i.i.i
+
+bb38.i.i.i.i:		; preds = %bb37.i.i.i.i, %bb33.i.i.i.i
+  br i1 false, label %bb39.i.i.i.i, label %bb43.i.i.i.i
+
+bb39.i.i.i.i:		; preds = %bb38.i.i.i.i
+  br i1 false, label %bb42.i.i.i.i, label %bb40.i.i.i.i
+
+bb40.i.i.i.i:		; preds = %bb39.i.i.i.i
+  br label %bb42.i.i.i.i
+
+bb42.i.i.i.i:		; preds = %bb40.i.i.i.i, %bb39.i.i.i.i
+  br label %bb43.i.i.i.i
+
+bb43.i.i.i.i:		; preds = %bb42.i.i.i.i, %bb38.i.i.i.i
+  br label %bb.i.i.i.i210.backedge
+
+bb.i.i.i.i210.backedge:		; preds = %bb47.i.i.i.i, %bb44.i.i.i.i, %bb43.i.i.i.i
+  br label %bb.i.i.i.i210
+
+bb44.i.i.i.i:		; preds = %bb29.i.i.i.i
+  br i1 false, label %bb.i.i.i.i210.backedge, label %bb46.i.i.i.i
+
+bb46.i.i.i.i:		; preds = %bb44.i.i.i.i
+  br i1 false, label %bb47.i.i.i.i, label %bb53.i.i.i.i.preheader.loopexit
+
+bb53.i.i.i.i.preheader.loopexit:		; preds = %bb46.i.i.i.i
+  br label %bb53.i.i.i.i.preheader
+
+bb53.i.i.i.i.preheader:		; preds = %bb53.i.i.i.i.preheader.loopexit, %bb7.i.i208
+  br label %bb53.i.i.i.i
+
+bb47.i.i.i.i:		; preds = %bb46.i.i.i.i
+  br label %bb.i.i.i.i210.backedge
+
+bb50.i.i.i.i:		; preds = %bb53.i.i.i.i
+  br i1 false, label %bb51.i.i.i.i, label %bb52.i.i.i.i
+
+bb51.i.i.i.i:		; preds = %bb50.i.i.i.i
+  br label %bb52.i.i.i.i
+
+bb52.i.i.i.i:		; preds = %bb51.i.i.i.i, %bb50.i.i.i.i
+  br label %bb53.i.i.i.i
+
+bb53.i.i.i.i:		; preds = %bb52.i.i.i.i, %bb53.i.i.i.i.preheader
+  br i1 false, label %bb50.i.i.i.i, label %bb59.i.i.i.i.preheader
+
+bb59.i.i.i.i.preheader:		; preds = %bb53.i.i.i.i
+  br label %bb59.i.i.i.i
+
+bb55.i.i.i.i:		; preds = %bb59.i.i.i.i
+  br label %bb57.i.i.i.i
+
+bb56.i.i.i.i:		; preds = %bb57.i.i.i.i
+  br label %bb57.i.i.i.i
+
+bb57.i.i.i.i:		; preds = %bb56.i.i.i.i, %bb55.i.i.i.i
+  br i1 false, label %bb56.i.i.i.i, label %bb58.i.i.i.i
+
+bb58.i.i.i.i:		; preds = %bb57.i.i.i.i
+  br label %bb59.i.i.i.i
+
+bb59.i.i.i.i:		; preds = %bb58.i.i.i.i, %bb59.i.i.i.i.preheader
+  br i1 false, label %bb60.i.i.i.i, label %bb55.i.i.i.i
+
+bb60.i.i.i.i:		; preds = %bb59.i.i.i.i
+  br label %bb69.i.i.i.i
+
+bb61.i.i.i.i:		; preds = %bb69.i.i.i.i
+  br i1 false, label %bb68.i.i.i.i, label %bb62.i.i.i.i
+
+bb62.i.i.i.i:		; preds = %bb61.i.i.i.i
+  br i1 false, label %bb63.i.i.i.i, label %bb65.i.i.i.i
+
+bb63.i.i.i.i:		; preds = %bb62.i.i.i.i
+  br i1 false, label %bb.i.i12.i, label %bb65.i.i.i.i
+
+bb65.i.i.i.i:		; preds = %bb63.i.i.i.i, %bb62.i.i.i.i
+  br i1 false, label %bb.i.i12.i, label %bb67.i.i.i.i
+
+bb67.i.i.i.i:		; preds = %bb65.i.i.i.i
+  br label %bb68.i.i.i.i
+
+bb68.i.i.i.i:		; preds = %bb67.i.i.i.i, %bb61.i.i.i.i
+  br label %bb69.i.i.i.i
+
+bb69.i.i.i.i:		; preds = %bb68.i.i.i.i, %bb60.i.i.i.i
+  br i1 false, label %bb61.i.i.i.i, label %bb70.i.i.i.i
+
+bb70.i.i.i.i:		; preds = %bb69.i.i.i.i
+  br label %READ_LITERAL.i.outer.backedge
+
+bb.i.i12.i:		; preds = %bb65.i.i.i.i, %bb63.i.i.i.i
+  br i1 false, label %bb1.i.i.i213, label %bb5.i.i.i218
+
+bb1.i.i.i213:		; preds = %bb.i.i12.i
+  br i1 false, label %bb4.i.i.i217, label %bb2.i.i.i214
+
+bb2.i.i.i214:		; preds = %bb1.i.i.i213
+  br label %bb4.i.i.i217
+
+bb4.i.i.i217:		; preds = %bb2.i.i.i214, %bb1.i.i.i213
+  br label %bb5.i.i.i218
+
+bb5.i.i.i218:		; preds = %bb4.i.i.i217, %bb.i.i12.i
+  br label %READ_LITERAL.i.outer.backedge
+
+READ_LITERAL.i.outer.backedge:		; preds = %bb5.i.i.i218, %bb70.i.i.i.i, %bb6.i.i207
+  br label %READ_LITERAL.i.outer
+
+parse.exit.loopexit:		; preds = %bb51.i, %bb47.i, %bb42.i202
+  br label %parse.exit
+
+parse.exit:		; preds = %parse.exit.loopexit, %bb38.i198, %bb35.i195, %bb33.i193
+  br i1 false, label %bb130, label %bb129
+
+bb129:		; preds = %parse.exit, %bb22.i173, %bb18.i168, %bb16.i166, %bb12.i161, %bb10.i159, %bb9.i158, %bb8.i157, %bb7.i156, %bb5.i154, %bb4.i153
+  br label %bb170
+
+bb130:		; preds = %parse.exit
+  br i1 false, label %bb143, label %bb142.preheader
+
+bb142.preheader:		; preds = %bb130
+  br label %bb142
+
+bb132:		; preds = %bb142
+  br i1 false, label %bb137, label %bb133
+
+bb133:		; preds = %bb132
+  br i1 false, label %bb137, label %bb134
+
+bb134:		; preds = %bb133
+  br i1 false, label %bb137, label %bb135
+
+bb135:		; preds = %bb134
+  br i1 false, label %bb137, label %bb136
+
+bb136:		; preds = %bb135
+  br i1 false, label %bb137, label %bb138
+
+bb137:		; preds = %bb136, %bb135, %bb134, %bb133, %bb132
+  br label %bb141
+
+bb138:		; preds = %bb136
+  br i1 false, label %bb139, label %bb141
+
+bb139:		; preds = %bb138
+  br i1 false, label %bb2.i126, label %picosat_assume.exit
+
+bb2.i126:		; preds = %bb139
+  br i1 false, label %bb5.i130, label %bb3.i127
+
+bb3.i127:		; preds = %bb2.i126
+  br label %bb5.i130
+
+bb5.i130:		; preds = %bb3.i127, %bb2.i126
+  br label %picosat_assume.exit
+
+picosat_assume.exit:		; preds = %bb5.i130, %bb139
+  br i1 false, label %bb141, label %bb140
+
+bb140:		; preds = %picosat_assume.exit
+  br label %bb141
+
+bb141:		; preds = %bb140, %picosat_assume.exit, %bb138, %bb137
+  br label %bb142
+
+bb142:		; preds = %bb141, %bb142.preheader
+  br i1 false, label %bb132, label %bb143.loopexit
+
+bb143.loopexit:		; preds = %bb142
+  br label %bb143
+
+bb143:		; preds = %bb143.loopexit, %bb130
+  br i1 false, label %bb145, label %bb144
+
+bb144:		; preds = %bb143
+  br label %bb11.i
+
+bb5.i114:		; preds = %bb11.i
+  br label %bb11.i
+
+bb11.i:		; preds = %bb5.i114, %bb144
+  br i1 false, label %bb12.i, label %bb5.i114
+
+bb12.i:		; preds = %bb11.i
+  br i1 false, label %bb.i.i.i118, label %bb1.i.i.i119
+
+bb.i.i.i118:		; preds = %bb12.i
+  br label %int2lit.exit.i
+
+bb1.i.i.i119:		; preds = %bb12.i
+  br label %int2lit.exit.i
+
+int2lit.exit.i:		; preds = %bb1.i.i.i119, %bb.i.i.i118
+  br label %bb19.i
+
+bb13.i:		; preds = %bb19.i
+  br label %bb17.i
+
+bb14.i:		; preds = %bb17.i
+  br label %bb17.i
+
+bb17.i:		; preds = %bb14.i, %bb13.i
+  br i1 false, label %bb14.i, label %bb18.i
+
+bb18.i:		; preds = %bb17.i
+  br label %bb19.i
+
+bb19.i:		; preds = %bb18.i, %int2lit.exit.i
+  br i1 false, label %bb20.i, label %bb13.i
+
+bb20.i:		; preds = %bb19.i
+  br label %bb33.i
+
+bb24.i:		; preds = %bb33.i
+  br i1 false, label %bb29.i, label %bb25.i
+
+bb25.i:		; preds = %bb24.i
+  br label %bb27.i
+
+bb26.i:		; preds = %bb27.i
+  br label %bb27.i
+
+bb27.i:		; preds = %bb26.i, %bb25.i
+  br i1 false, label %bb26.i, label %bb28.i
+
+bb28.i:		; preds = %bb27.i
+  br label %bb29.i
+
+bb29.i:		; preds = %bb28.i, %bb24.i
+  br label %bb33.i
+
+bb33.i:		; preds = %bb29.i, %bb20.i
+  br i1 false, label %bb34.i, label %bb24.i
+
+bb34.i:		; preds = %bb33.i
+  br i1 false, label %bb.i.i58.i, label %bb1.i.i59.i
+
+bb.i.i58.i:		; preds = %bb34.i
+  br label %int2lit.exit63.i
+
+bb1.i.i59.i:		; preds = %bb34.i
+  br label %int2lit.exit63.i
+
+int2lit.exit63.i:		; preds = %bb1.i.i59.i, %bb.i.i58.i
+  br label %bb41.i
+
+bb35.i:		; preds = %bb41.i
+  br label %bb39.i
+
+bb36.i:		; preds = %bb39.i
+  br i1 false, label %bb38.i, label %bb37.i
+
+bb37.i:		; preds = %bb36.i
+  br label %bb38.i
+
+bb38.i:		; preds = %bb37.i, %bb36.i
+  br label %bb39.i
+
+bb39.i:		; preds = %bb38.i, %bb35.i
+  br i1 false, label %bb36.i, label %bb40.i
+
+bb40.i:		; preds = %bb39.i
+  br label %bb41.i
+
+bb41.i:		; preds = %bb40.i, %int2lit.exit63.i
+  br i1 false, label %bb42.i, label %bb35.i
+
+bb42.i:		; preds = %bb41.i
+  br label %bb44.i
+
+bb43.i:		; preds = %bb44.i
+  br label %bb44.i
+
+bb44.i:		; preds = %bb43.i, %bb42.i
+  br i1 false, label %bb43.i, label %picosat_print.exit
+
+picosat_print.exit:		; preds = %bb44.i
+  br label %bb167
+
+bb145:		; preds = %bb143
+  br i1 false, label %bb147, label %bb146
+
+bb146:		; preds = %bb145
+  br label %bb147
+
+bb147:		; preds = %bb146, %bb145
+  br i1 false, label %bb149, label %bb148
+
+bb148:		; preds = %bb147
+  br label %bb149
+
+bb149:		; preds = %bb148, %bb147
+  br i1 false, label %bb.i54, label %bb1.i55
+
+bb.i54:		; preds = %bb149
+  unreachable
+
+bb1.i55:		; preds = %bb149
+  br i1 false, label %bb.i.i56, label %bb1.i.i57
+
+bb.i.i56:		; preds = %bb1.i55
+  br label %bb1.i.i57
+
+bb1.i.i57:		; preds = %bb.i.i56, %bb1.i55
+  br i1 false, label %bb3.i.i59, label %bb2.i.i58
+
+bb2.i.i58:		; preds = %bb1.i.i57
+  br label %bb3.i.i59
+
+bb3.i.i59:		; preds = %bb2.i.i58, %bb1.i.i57
+  br i1 false, label %bb5.i.i61, label %sat.exit.i
+
+bb5.i.i61:		; preds = %bb3.i.i59
+  br i1 false, label %bb6.i.i65, label %bb1.i.i.i63
+
+bb1.i.i.i63:		; preds = %bb5.i.i61
+  br i1 false, label %sat.exit.i, label %bb6.i.i65
+
+bb6.i.i65:		; preds = %bb1.i.i.i63, %bb5.i.i61
+  br i1 false, label %bb8.i.i67, label %bb7.i.i66
+
+bb7.i.i66:		; preds = %bb6.i.i65
+  br label %bb8.i.i67
+
+bb8.i.i67:		; preds = %bb7.i.i66, %bb6.i.i65
+  br i1 false, label %bb10.i.i69, label %sat.exit.i
+
+bb10.i.i69:		; preds = %bb8.i.i67
+  br i1 false, label %bb11.i.i70, label %bb1.i61.i.i
+
+bb1.i61.i.i:		; preds = %bb10.i.i69
+  br i1 false, label %sat.exit.i, label %bb11.i.i70
+
+bb11.i.i70:		; preds = %bb1.i61.i.i, %bb10.i.i69
+  br label %bb13.i.i71.outer
+
+bb13.i.i71.outer:		; preds = %bb42.i.i, %bb11.i.i70
+  br label %bb13.i.i71
+
+bb13.i.i71:		; preds = %bb13.i.i71.backedge, %bb13.i.i71.outer
+  br i1 false, label %bb14.i.i72, label %bb15.i.i73
+
+bb14.i.i72:		; preds = %bb13.i.i71
+  br label %bb15.i.i73
+
+bb15.i.i73:		; preds = %bb14.i.i72, %bb13.i.i71
+  br i1 false, label %bb19.i.i, label %bb16.i.i
+
+bb16.i.i:		; preds = %bb15.i.i73
+  br i1 false, label %bb.i.i79.i.i, label %incincs.exit.i.i
+
+bb.i.i79.i.i:		; preds = %bb16.i.i
+  br label %bb4.i.i.i85.i.i
+
+bb.i.i.i80.i.i:		; preds = %bb4.i.i.i85.i.i
+  br i1 false, label %bb3.i.i.i83.i.i, label %bb1.i.i.i81.i.i
+
+bb1.i.i.i81.i.i:		; preds = %bb.i.i.i80.i.i
+  br i1 false, label %bb2.i.i.i82.i.i, label %bb3.i.i.i83.i.i
+
+bb2.i.i.i82.i.i:		; preds = %bb1.i.i.i81.i.i
+  br label %bb3.i.i.i83.i.i
+
+bb3.i.i.i83.i.i:		; preds = %bb2.i.i.i82.i.i, %bb1.i.i.i81.i.i, %bb.i.i.i80.i.i
+  br label %bb4.i.i.i85.i.i
+
+bb4.i.i.i85.i.i:		; preds = %bb3.i.i.i83.i.i, %bb.i.i79.i.i
+  br i1 false, label %crescore.exit.i.i.i.i, label %bb.i.i.i80.i.i
+
+crescore.exit.i.i.i.i:		; preds = %bb4.i.i.i85.i.i
+  br label %incincs.exit.i.i
+
+incincs.exit.i.i:		; preds = %crescore.exit.i.i.i.i, %bb16.i.i
+  br i1 false, label %bb13.i.i71.backedge, label %sat.exit.i.loopexit.loopexit
+
+bb13.i.i71.backedge:		; preds = %bb1.i55.i.i, %bb28.i.i, %incincs.exit.i.i
+  br label %bb13.i.i71
+
+bb19.i.i:		; preds = %bb15.i.i73
+  br i1 false, label %bb20.i.i, label %bb1.i68.i.i
+
+bb1.i68.i.i:		; preds = %bb19.i.i
+  br i1 false, label %sat.exit.i.loopexit.loopexit, label %bb20.i.i
+
+bb20.i.i:		; preds = %bb1.i68.i.i, %bb19.i.i
+  br i1 false, label %bb24.i.i, label %bb21.i.i
+
+bb21.i.i:		; preds = %bb20.i.i
+  br i1 false, label %bb22.i.i, label %bb24.i.i
+
+bb22.i.i:		; preds = %bb21.i.i
+  br i1 false, label %bb23.i.i, label %bb24.i.i
+
+bb23.i.i:		; preds = %bb22.i.i
+  br label %bb24.i.i
+
+bb24.i.i:		; preds = %bb23.i.i, %bb22.i.i, %bb21.i.i, %bb20.i.i
+  br i1 false, label %bb26.i.i, label %sat.exit.i.loopexit.loopexit
+
+bb26.i.i:		; preds = %bb24.i.i
+  br i1 false, label %bb27.i.i, label %bb33.i.i.loopexit
+
+bb27.i.i:		; preds = %bb26.i.i
+  br i1 false, label %bb33.i.i.loopexit, label %bb28.i.i
+
+bb28.i.i:		; preds = %bb27.i.i
+  br i1 false, label %bb1.i55.i.i, label %bb13.i.i71.backedge
+
+bb1.i55.i.i:		; preds = %bb28.i.i
+  br i1 false, label %bb29.i.i, label %bb13.i.i71.backedge
+
+bb29.i.i:		; preds = %bb1.i55.i.i
+  br i1 false, label %bb31.i.i, label %sat.exit.i.loopexit.loopexit2
+
+bb31.i.i:		; preds = %bb29.i.i
+  br i1 false, label %bb33.i.i, label %bb1.i48.i.i
+
+bb1.i48.i.i:		; preds = %bb31.i.i
+  br i1 false, label %sat.exit.i.loopexit.loopexit2, label %bb33.i.i
+
+bb33.i.i.loopexit:		; preds = %bb27.i.i, %bb26.i.i
+  br label %bb33.i.i
+
+bb33.i.i:		; preds = %bb33.i.i.loopexit, %bb1.i48.i.i, %bb31.i.i
+  br i1 false, label %bb34.i.i, label %bb35.i.i
+
+bb34.i.i:		; preds = %bb33.i.i
+  br i1 false, label %bb35.i.i, label %bb2.i44.i.i76
+
+bb2.i44.i.i76:		; preds = %bb34.i.i
+  br label %bb35.i.i
+
+bb35.i.i:		; preds = %bb2.i44.i.i76, %bb34.i.i, %bb33.i.i
+  br i1 false, label %bb1.i37.i.i, label %bb.i35.i.i
+
+bb.i35.i.i:		; preds = %bb35.i.i
+  br label %bb36.i.i
+
+bb1.i37.i.i:		; preds = %bb35.i.i
+  br i1 false, label %bb37.i.i, label %bb36.i.i
+
+bb36.i.i:		; preds = %bb1.i37.i.i, %bb.i35.i.i
+  br label %bb25.i23.i.i
+
+bb.i18.i.i:		; preds = %bb25.i23.i.i
+  br i1 false, label %bb24.i22.i.i, label %bb22.i19.i.i
+
+bb22.i19.i.i:		; preds = %bb.i18.i.i
+  br label %bb24.i22.i.i
+
+bb24.i22.i.i:		; preds = %bb22.i19.i.i, %bb.i18.i.i
+  br label %bb25.i23.i.i
+
+bb25.i23.i.i:		; preds = %bb24.i22.i.i, %bb36.i.i
+  br i1 false, label %bb.i18.i.i, label %bb26.i24.i.i
+
+bb26.i24.i.i:		; preds = %bb25.i23.i.i
+  br i1 false, label %bb27.i25.i.i, label %bb32.i.i.i
+
+bb27.i25.i.i:		; preds = %bb26.i24.i.i
+  br label %bb32.i.i.i
+
+bb32.i.i.i:		; preds = %bb27.i25.i.i, %bb26.i24.i.i
+  br label %bb64.i.i.i
+
+bb33.i.i.i:		; preds = %bb64.i.i.i
+  br i1 false, label %bb60.i.i.i, label %bb34.i.i.i
+
+bb34.i.i.i:		; preds = %bb33.i.i.i
+  br i1 false, label %bb38.i.i.i, label %bb60.i.i.i
+
+bb38.i.i.i:		; preds = %bb34.i.i.i
+  br i1 false, label %bb39.i.i.i, label %bb48.i.i.i
+
+bb39.i.i.i:		; preds = %bb38.i.i.i
+  br i1 false, label %bb48.i.i.i, label %bb40.i.i.i
+
+bb40.i.i.i:		; preds = %bb39.i.i.i
+  br i1 false, label %bb60.i.i.i, label %bb45.i.i.i
+
+bb45.i.i.i:		; preds = %bb40.i.i.i
+  br label %bb60.i.i.i
+
+bb48.i.i.i:		; preds = %bb39.i.i.i, %bb38.i.i.i
+  br i1 false, label %bb53.i.i.i, label %bb60.i.i.i
+
+bb53.i.i.i:		; preds = %bb48.i.i.i
+  br i1 false, label %bb60.i.i.i, label %bb58.i.i.i
+
+bb58.i.i.i:		; preds = %bb53.i.i.i
+  br i1 false, label %bb59.i.i.i, label %bb60.i.i.i
+
+bb59.i.i.i:		; preds = %bb58.i.i.i
+  br label %bb60.i.i.i
+
+bb60.i.i.i:		; preds = %bb59.i.i.i, %bb58.i.i.i, %bb53.i.i.i, %bb48.i.i.i, %bb45.i.i.i, %bb40.i.i.i, %bb34.i.i.i, %bb33.i.i.i
+  %lcollect.i.i.i.1 = phi i32 [ %lcollect.i.i.i.2, %bb34.i.i.i ], [ %lcollect.i.i.i.2, %bb48.i.i.i ], [ %lcollect.i.i.i.2, %bb58.i.i.i ], [ %lcollect.i.i.i.2, %bb59.i.i.i ], [ %lcollect.i.i.i.2, %bb53.i.i.i ], [ %lcollect.i.i.i.2, %bb33.i.i.i ], [ %lcollect.i.i.i.2, %bb40.i.i.i ], [ 0, %bb45.i.i.i ]		; <i32> [#uses=1]
+  br label %bb64.i.i.i
+
+bb64.i.i.i:		; preds = %bb60.i.i.i, %bb32.i.i.i
+  %lcollect.i.i.i.2 = phi i32 [ 0, %bb32.i.i.i ], [ %lcollect.i.i.i.1, %bb60.i.i.i ]		; <i32> [#uses=8]
+  br i1 false, label %bb65.i.i.i, label %bb33.i.i.i
+
+bb65.i.i.i:		; preds = %bb64.i.i.i
+  br i1 false, label %bb103.i.i.i.preheader, label %bb66.i.i.i.preheader
+
+bb66.i.i.i.preheader:		; preds = %bb65.i.i.i
+  br label %bb66.i.i.i
+
+bb66.i.i.i:		; preds = %bb66.i.i.i.backedge, %bb66.i.i.i.preheader
+  br i1 false, label %bb67.i.i.i, label %bb68.i.i.i
+
+bb67.i.i.i:		; preds = %bb66.i.i.i
+  br label %bb68.i.i.i
+
+bb68.i.i.i:		; preds = %bb67.i.i.i, %bb66.i.i.i
+  br i1 false, label %bb69.i.i.i, label %bb70.i.i.i
+
+bb69.i.i.i:		; preds = %bb68.i.i.i
+  br label %bb70.i.i.i
+
+bb70.i.i.i:		; preds = %bb69.i.i.i, %bb68.i.i.i
+  br i1 false, label %bb71.i.i.i, label %bb72.i.i.i
+
+bb71.i.i.i:		; preds = %bb70.i.i.i
+  br label %bb72.i.i.i
+
+bb72.i.i.i:		; preds = %bb71.i.i.i, %bb70.i.i.i
+  br label %bb73.i.i.i.outer
+
+bb73.i.i.i.outer:		; preds = %bb78.i.i.i, %bb72.i.i.i
+  br label %bb73.i.i.i
+
+bb73.i.i.i:		; preds = %bb73.i.i.i, %bb73.i.i.i.outer
+  br i1 false, label %bb73.i.i.i, label %bb76.i.i.i.preheader
+
+bb76.i.i.i.preheader:		; preds = %bb73.i.i.i
+  br label %bb76.i.i.i
+
+bb76.i.i.i:		; preds = %bb76.i.i.i, %bb76.i.i.i.preheader
+  br i1 false, label %bb77.i.i.i, label %bb76.i.i.i
+
+bb77.i.i.i:		; preds = %bb76.i.i.i
+  br i1 false, label %bb78.i.i.i, label %bb79.i.i.i
+
+bb78.i.i.i:		; preds = %bb77.i.i.i
+  br label %bb73.i.i.i.outer
+
+bb79.i.i.i:		; preds = %bb77.i.i.i
+  br i1 false, label %bb83.i.i.i, label %bb94.i.i.i
+
+bb83.i.i.i:		; preds = %bb79.i.i.i
+  br i1 false, label %bb84.i.i.i, label %bb88.i.i.i
+
+bb84.i.i.i:		; preds = %bb83.i.i.i
+  br i1 false, label %bb87.i.i.i, label %bb85.i.i.i
+
+bb85.i.i.i:		; preds = %bb84.i.i.i
+  br label %bb87.i.i.i
+
+bb87.i.i.i:		; preds = %bb85.i.i.i, %bb84.i.i.i
+  br label %bb88.i.i.i
+
+bb88.i.i.i:		; preds = %bb87.i.i.i, %bb83.i.i.i
+  br i1 false, label %bb89.i.i.i, label %bb93.i.i.i
+
+bb89.i.i.i:		; preds = %bb88.i.i.i
+  br i1 false, label %bb92.i.i.i, label %bb90.i.i.i
+
+bb90.i.i.i:		; preds = %bb89.i.i.i
+  br label %bb92.i.i.i
+
+bb92.i.i.i:		; preds = %bb90.i.i.i, %bb89.i.i.i
+  br label %bb93.i.i.i
+
+bb93.i.i.i:		; preds = %bb92.i.i.i, %bb88.i.i.i
+  br label %bb66.i.i.i.backedge
+
+bb66.i.i.i.backedge:		; preds = %bb97.i.i.i, %bb94.i.i.i, %bb93.i.i.i
+  br label %bb66.i.i.i
+
+bb94.i.i.i:		; preds = %bb79.i.i.i
+  br i1 false, label %bb66.i.i.i.backedge, label %bb96.i.i.i
+
+bb96.i.i.i:		; preds = %bb94.i.i.i
+  br i1 false, label %bb97.i.i.i, label %bb103.i.i.i.preheader.loopexit
+
+bb103.i.i.i.preheader.loopexit:		; preds = %bb96.i.i.i
+  br label %bb103.i.i.i.preheader
+
+bb103.i.i.i.preheader:		; preds = %bb103.i.i.i.preheader.loopexit, %bb65.i.i.i
+  br label %bb103.i.i.i
+
+bb97.i.i.i:		; preds = %bb96.i.i.i
+  br label %bb66.i.i.i.backedge
+
+bb100.i.i.i:		; preds = %bb103.i.i.i
+  br i1 false, label %bb101.i.i.i, label %bb102.i.i.i
+
+bb101.i.i.i:		; preds = %bb100.i.i.i
+  br label %bb102.i.i.i
+
+bb102.i.i.i:		; preds = %bb101.i.i.i, %bb100.i.i.i
+  br label %bb103.i.i.i
+
+bb103.i.i.i:		; preds = %bb102.i.i.i, %bb103.i.i.i.preheader
+  br i1 false, label %bb100.i.i.i, label %bb109.i.i.i.preheader
+
+bb109.i.i.i.preheader:		; preds = %bb103.i.i.i
+  br label %bb109.i.i.i
+
+bb105.i.i.i:		; preds = %bb109.i.i.i
+  br label %bb107.i.i.i
+
+bb106.i.i.i:		; preds = %bb107.i.i.i
+  br label %bb107.i.i.i
+
+bb107.i.i.i:		; preds = %bb106.i.i.i, %bb105.i.i.i
+  br i1 false, label %bb106.i.i.i, label %bb108.i.i.i
+
+bb108.i.i.i:		; preds = %bb107.i.i.i
+  br label %bb109.i.i.i
+
+bb109.i.i.i:		; preds = %bb108.i.i.i, %bb109.i.i.i.preheader
+  br i1 false, label %bb110.i.i.i, label %bb105.i.i.i
+
+bb110.i.i.i:		; preds = %bb109.i.i.i
+  %0 = sub i32 0, %lcollect.i.i.i.2		; <i32> [#uses=1]
+  %1 = add i32 %0, 1		; <i32> [#uses=1]
+  br label %bb113.i.i.i
+
+bb111.i.i.i:		; preds = %bb113.i.i.i
+  br i1 false, label %bb114.i.i.i, label %bb113.i.i.i
+
+bb113.i.i.i:		; preds = %bb111.i.i.i, %bb110.i.i.i
+  br i1 false, label %bb111.i.i.i, label %bb114.i.i.i
+
+bb114.i.i.i:		; preds = %bb113.i.i.i, %bb111.i.i.i
+  %2 = lshr i32 %1, 1		; <i32> [#uses=2]
+  br i1 false, label %bb116.i.i.i, label %bb124.i.i.i
+
+bb116.i.i.i:		; preds = %bb114.i.i.i
+  br i1 false, label %bb117.i.i.i.preheader, label %bb122.i.i.i.preheader
+
+bb122.i.i.i.preheader:		; preds = %bb116.i.i.i
+  br label %bb122.i.i.i
+
+bb117.i.i.i.preheader:		; preds = %bb116.i.i.i
+  br label %bb117.i.i.i
+
+bb117.i.i.i:		; preds = %bb118.i.i.i, %bb117.i.i.i.preheader
+  %target.i.i.i.1 = phi i32 [ %3, %bb118.i.i.i ], [ %2, %bb117.i.i.i.preheader ]		; <i32> [#uses=1]
+  %3 = add i32 %target.i.i.i.1, 1		; <i32> [#uses=2]
+  br i1 false, label %bb118.i.i.i, label %bb124.i.i.i.loopexit
+
+bb118.i.i.i:		; preds = %bb117.i.i.i
+  br i1 false, label %bb117.i.i.i, label %bb124.i.i.i.loopexit
+
+bb122.i.i.i:		; preds = %bb123.i.i.i, %bb122.i.i.i.preheader
+  %target.i.i.i.2 = phi i32 [ %4, %bb123.i.i.i ], [ %2, %bb122.i.i.i.preheader ]		; <i32> [#uses=2]
+  br i1 false, label %bb124.i.i.i.loopexit1, label %bb123.i.i.i
+
+bb123.i.i.i:		; preds = %bb122.i.i.i
+  %4 = add i32 %target.i.i.i.2, -1		; <i32> [#uses=1]
+  br i1 false, label %bb122.i.i.i, label %bb124.i.i.i.loopexit1
+
+bb124.i.i.i.loopexit:		; preds = %bb118.i.i.i, %bb117.i.i.i
+  br label %bb124.i.i.i
+
+bb124.i.i.i.loopexit1:		; preds = %bb123.i.i.i, %bb122.i.i.i
+  br label %bb124.i.i.i
+
+bb124.i.i.i:		; preds = %bb124.i.i.i.loopexit1, %bb124.i.i.i.loopexit, %bb114.i.i.i
+  %target.i.i.i.0 = phi i32 [ 0, %bb114.i.i.i ], [ %3, %bb124.i.i.i.loopexit ], [ %target.i.i.i.2, %bb124.i.i.i.loopexit1 ]		; <i32> [#uses=0]
+  br label %bb132.i.i.i.outer
+
+bb125.i.i.i:		; preds = %bb132.i.i.i
+  br i1 false, label %bb132.i.i.i, label %bb130.i.i.i
+
+bb130.i.i.i:		; preds = %bb125.i.i.i
+  br label %bb132.i.i.i.outer
+
+bb132.i.i.i.outer:		; preds = %bb130.i.i.i, %bb124.i.i.i
+  br label %bb132.i.i.i
+
+bb132.i.i.i:		; preds = %bb132.i.i.i.outer, %bb125.i.i.i
+  br i1 false, label %bb125.i.i.i, label %bb133.i.i.i
+
+bb133.i.i.i:		; preds = %bb132.i.i.i
+  br i1 false, label %bb136.i.i.i, label %bb134.i.i.i
+
+bb134.i.i.i:		; preds = %bb133.i.i.i
+  br i1 false, label %bb136.i.i.i, label %bb135.i.i.i
+
+bb135.i.i.i:		; preds = %bb134.i.i.i
+  br label %bb136.i.i.i
+
+bb136.i.i.i:		; preds = %bb135.i.i.i, %bb134.i.i.i, %bb133.i.i.i
+  br i1 false, label %bb137.i.i.i, label %bb37.i.i
+
+bb137.i.i.i:		; preds = %bb136.i.i.i
+  br label %bb37.i.i
+
+bb37.i.i:		; preds = %bb137.i.i.i, %bb136.i.i.i, %bb1.i37.i.i
+  br i1 false, label %bb40.i.i, label %bb38.i.i
+
+bb38.i.i:		; preds = %bb37.i.i
+  br i1 false, label %bb39.i.i, label %bb40.i.i
+
+bb39.i.i:		; preds = %bb38.i.i
+  br i1 false, label %bb17.i.i.i, label %bb3.i12.i.i
+
+bb3.i12.i.i:		; preds = %bb39.i.i
+  br label %bb5.i14.i.i
+
+bb5.i14.i.i:		; preds = %bb8.i.i.i79, %bb3.i12.i.i
+  br i1 false, label %bb6.i15.i.i, label %bb9.i.i.i80
+
+bb6.i15.i.i:		; preds = %bb5.i14.i.i
+  br i1 false, label %bb7.i.i.i78, label %bb9.i.i.i80
+
+bb7.i.i.i78:		; preds = %bb6.i15.i.i
+  br i1 false, label %bb9.i.i.i80, label %bb8.i.i.i79
+
+bb8.i.i.i79:		; preds = %bb7.i.i.i78
+  br i1 false, label %bb9.i.i.i80, label %bb5.i14.i.i
+
+bb9.i.i.i80:		; preds = %bb8.i.i.i79, %bb7.i.i.i78, %bb6.i15.i.i, %bb5.i14.i.i
+  br i1 false, label %bb16.i.i.i, label %bb10.i.i.i81
+
+bb10.i.i.i81:		; preds = %bb9.i.i.i80
+  br i1 false, label %bb11.i.i.i, label %bb15.i.i.i
+
+bb11.i.i.i:		; preds = %bb10.i.i.i81
+  br i1 false, label %bb16.i.i.i, label %bb15.i.i.i
+
+bb15.i.i.i:		; preds = %bb11.i.i.i, %bb10.i.i.i81
+  br label %bb16.i.i.i
+
+bb16.i.i.i:		; preds = %bb15.i.i.i, %bb11.i.i.i, %bb9.i.i.i80
+  br label %bb17.i.i.i
+
+bb17.i.i.i:		; preds = %bb16.i.i.i, %bb39.i.i
+  br i1 false, label %bb18.i.i.i, label %bb25.i.i.i
+
+bb18.i.i.i:		; preds = %bb17.i.i.i
+  br i1 false, label %bb24.i.i.i, label %bb23.i.i.i
+
+bb23.i.i.i:		; preds = %bb18.i.i.i
+  br label %bb24.i.i.i
+
+bb24.i.i.i:		; preds = %bb23.i.i.i, %bb18.i.i.i
+  br label %bb29.i.i.i
+
+bb25.i.i.i:		; preds = %bb17.i.i.i
+  br i1 false, label %bb29.i.i.i, label %bb27.i.i.i
+
+bb27.i.i.i:		; preds = %bb25.i.i.i
+  br i1 false, label %bb29.i.i.i, label %bb28.i.i.i
+
+bb28.i.i.i:		; preds = %bb27.i.i.i
+  br i1 false, label %bb29.i.i.i, label %bb.i4.i.i.i
+
+bb.i4.i.i.i:		; preds = %bb28.i.i.i
+  br i1 false, label %bb4.i.i16.i.i, label %bb29.i.i.i
+
+bb4.i.i16.i.i:		; preds = %bb.i4.i.i.i
+  br label %bb29.i.i.i
+
+bb29.i.i.i:		; preds = %bb4.i.i16.i.i, %bb.i4.i.i.i, %bb28.i.i.i, %bb27.i.i.i, %bb25.i.i.i, %bb24.i.i.i
+  br label %bb40.i.i
+
+bb40.i.i:		; preds = %bb29.i.i.i, %bb38.i.i, %bb37.i.i
+  br i1 false, label %bb9.i.i.i.i.preheader, label %bb2.i.i.i87
+
+bb9.i.i.i.i.preheader:		; preds = %bb40.i.i
+  br label %bb9.i.i.i.i
+
+bb.i.i.i.i84:		; preds = %bb9.i.i.i.i
+  switch i8 0, label %bb8.i.i.i.i [
+  i8 -1, label %bb1.i.i.i.i85
+  i8 1, label %bb9.i.i.i.i
+  ]
+
+bb1.i.i.i.i85:		; preds = %bb.i.i.i.i84
+  br i1 false, label %bb5.i.i.i.i, label %bb2.i.i.i87
+
+bb5.i.i.i.i:		; preds = %bb1.i.i.i.i85
+  br label %bb2.i.i.i87
+
+bb8.i.i.i.i:		; preds = %bb.i.i.i.i84
+  br i1 false, label %bb2.i.i.i87, label %bb6.i.i.i95
+
+bb9.i.i.i.i:		; preds = %bb.i.i.i.i84, %bb9.i.i.i.i.preheader
+  br i1 false, label %bb.i.i.i.i84, label %bb10.i.i.i.i
+
+bb10.i.i.i.i:		; preds = %bb9.i.i.i.i
+  br label %bb2.i.i.i87
+
+bb2.i.i.i87:		; preds = %bb10.i.i.i.i, %bb8.i.i.i.i, %bb5.i.i.i.i, %bb1.i.i.i.i85, %bb40.i.i
+  br i1 false, label %bb3.i.i.i88, label %decide.exit.i.i
+
+bb3.i.i.i88:		; preds = %bb2.i.i.i87
+  br i1 false, label %bb4.i.i.i90, label %bb1.i23.i.i.i
+
+bb1.i23.i.i.i:		; preds = %bb3.i.i.i88
+  br i1 false, label %decide.exit.i.i, label %bb4.i.i.i90
+
+bb4.i.i.i90:		; preds = %bb1.i23.i.i.i, %bb3.i.i.i88
+  br i1 false, label %bb1.i9.i.i.i, label %bb5.i.i.i94
+
+bb1.i9.i.i.i:		; preds = %bb4.i.i.i90
+  br i1 false, label %bb.i.i27.i.i.i.i, label %bb1.i.i28.i.i.i.i
+
+bb.i.i27.i.i.i.i:		; preds = %bb1.i9.i.i.i
+  br label %int2lit.exit32.i.i.i.i
+
+bb1.i.i28.i.i.i.i:		; preds = %bb1.i9.i.i.i
+  br label %int2lit.exit32.i.i.i.i
+
+int2lit.exit32.i.i.i.i:		; preds = %bb1.i.i28.i.i.i.i, %bb.i.i27.i.i.i.i
+  br i1 false, label %bb8.i19.i.i.i, label %bb2.i.i.i.i91
+
+bb2.i.i.i.i91:		; preds = %int2lit.exit32.i.i.i.i
+  br label %bb4.i.i.i.i
+
+bb3.i.i.i.i92:		; preds = %gcd.exit.i.i.i.i
+  br label %bb4.i.i.i.i
+
+bb4.i.i.i.i:		; preds = %bb3.i.i.i.i92, %bb2.i.i.i.i91
+  br label %bb3.i.i13.i.i.i
+
+bb2.i.i12.i.i.i:		; preds = %bb3.i.i13.i.i.i
+  br label %bb3.i.i13.i.i.i
+
+bb3.i.i13.i.i.i:		; preds = %bb2.i.i12.i.i.i, %bb4.i.i.i.i
+  br i1 false, label %gcd.exit.i.i.i.i, label %bb2.i.i12.i.i.i
+
+gcd.exit.i.i.i.i:		; preds = %bb3.i.i13.i.i.i
+  br i1 false, label %bb5.i14.i.i.i.preheader, label %bb3.i.i.i.i92
+
+bb5.i14.i.i.i.preheader:		; preds = %gcd.exit.i.i.i.i
+  br label %bb5.i14.i.i.i
+
+bb5.i14.i.i.i:		; preds = %int2lit.exit.i.i.i.i, %bb5.i14.i.i.i.preheader
+  br i1 false, label %bb.i.i.i17.i.i.i, label %bb1.i.i.i18.i.i.i
+
+bb.i.i.i17.i.i.i:		; preds = %bb5.i14.i.i.i
+  br label %int2lit.exit.i.i.i.i
+
+bb1.i.i.i18.i.i.i:		; preds = %bb5.i14.i.i.i
+  br label %int2lit.exit.i.i.i.i
+
+int2lit.exit.i.i.i.i:		; preds = %bb1.i.i.i18.i.i.i, %bb.i.i.i17.i.i.i
+  br i1 false, label %bb8.i19.i.i.i.loopexit, label %bb5.i14.i.i.i
+
+bb8.i19.i.i.i.loopexit:		; preds = %int2lit.exit.i.i.i.i
+  br label %bb8.i19.i.i.i
+
+bb8.i19.i.i.i:		; preds = %bb8.i19.i.i.i.loopexit, %int2lit.exit32.i.i.i.i
+  br i1 false, label %bb5.i.i.i94, label %bb6.i.i.i95
+
+bb5.i.i.i94:		; preds = %bb8.i19.i.i.i, %bb4.i.i.i90
+  br label %bb.i2.i.i.i
+
+bb.i2.i.i.i:		; preds = %hpop.exit.i.i.i.i, %bb5.i.i.i94
+  br i1 false, label %hpop.exit.i.i.i.i, label %bb1.i.i.i.i.i
+
+bb1.i.i.i.i.i:		; preds = %bb.i2.i.i.i
+  br label %bb2.i.i.i.i.i
+
+bb2.i.i.i.i.i:		; preds = %bb11.i.i.i.i.i, %bb1.i.i.i.i.i
+  br i1 false, label %bb3.i.i.i.i.i, label %bb12.i.i.i.i.i
+
+bb3.i.i.i.i.i:		; preds = %bb2.i.i.i.i.i
+  br i1 false, label %bb4.i.i.i.i.i, label %bb1.i.i.i.i.i.i
+
+bb1.i.i.i.i.i.i:		; preds = %bb3.i.i.i.i.i
+  br i1 false, label %bb8.i.i.i.i.i, label %bb3.i.i.i.i.i.i
+
+bb3.i.i.i.i.i.i:		; preds = %bb1.i.i.i.i.i.i
+  br i1 false, label %bb4.i.i.i.i.i, label %bb8.i.i.i.i.i
+
+bb4.i.i.i.i.i:		; preds = %bb3.i.i.i.i.i.i, %bb3.i.i.i.i.i
+  br i1 false, label %bb5.i.i.i.i.i, label %bb11.i.i.i.i.i
+
+bb5.i.i.i.i.i:		; preds = %bb4.i.i.i.i.i
+  br i1 false, label %bb6.i.i.i.i.i, label %bb1.i21.i.i.i.i.i
+
+bb1.i21.i.i.i.i.i:		; preds = %bb5.i.i.i.i.i
+  br i1 false, label %bb11.i.i.i.i.i, label %bb3.i24.i.i.i.i.i
+
+bb3.i24.i.i.i.i.i:		; preds = %bb1.i21.i.i.i.i.i
+  br i1 false, label %bb6.i.i.i.i.i, label %bb11.i.i.i.i.i
+
+bb6.i.i.i.i.i:		; preds = %bb3.i24.i.i.i.i.i, %bb5.i.i.i.i.i
+  br label %bb11.i.i.i.i.i
+
+bb8.i.i.i.i.i:		; preds = %bb3.i.i.i.i.i.i, %bb1.i.i.i.i.i.i
+  br i1 false, label %bb9.i.i.i.i.i, label %bb12.i.i.i.i.i
+
+bb9.i.i.i.i.i:		; preds = %bb8.i.i.i.i.i
+  br i1 false, label %bb11.i.i.i.i.i, label %bb1.i8.i.i.i.i.i
+
+bb1.i8.i.i.i.i.i:		; preds = %bb9.i.i.i.i.i
+  br i1 false, label %bb12.i.i.i.i.i, label %bb3.i11.i.i.i.i.i
+
+bb3.i11.i.i.i.i.i:		; preds = %bb1.i8.i.i.i.i.i
+  br i1 false, label %bb11.i.i.i.i.i, label %bb12.i.i.i.i.i
+
+bb11.i.i.i.i.i:		; preds = %bb3.i11.i.i.i.i.i, %bb9.i.i.i.i.i, %bb6.i.i.i.i.i, %bb3.i24.i.i.i.i.i, %bb1.i21.i.i.i.i.i, %bb4.i.i.i.i.i
+  br label %bb2.i.i.i.i.i
+
+bb12.i.i.i.i.i:		; preds = %bb3.i11.i.i.i.i.i, %bb1.i8.i.i.i.i.i, %bb8.i.i.i.i.i, %bb2.i.i.i.i.i
+  br label %hpop.exit.i.i.i.i
+
+hpop.exit.i.i.i.i:		; preds = %bb12.i.i.i.i.i, %bb.i2.i.i.i
+  br i1 false, label %sdecide.exit.i.i.i, label %bb.i2.i.i.i
+
+sdecide.exit.i.i.i:		; preds = %hpop.exit.i.i.i.i
+  br label %bb6.i.i.i95
+
+bb6.i.i.i95:		; preds = %sdecide.exit.i.i.i, %bb8.i19.i.i.i, %bb8.i.i.i.i
+  br label %decide.exit.i.i
+
+decide.exit.i.i:		; preds = %bb6.i.i.i95, %bb1.i23.i.i.i, %bb2.i.i.i87
+  br i1 false, label %bb42.i.i, label %sat.exit.i.loopexit.loopexit2
+
+bb42.i.i:		; preds = %decide.exit.i.i
+  br label %bb13.i.i71.outer
+
+sat.exit.i.loopexit.loopexit:		; preds = %bb24.i.i, %bb1.i68.i.i, %incincs.exit.i.i
+  br label %sat.exit.i.loopexit
+
+sat.exit.i.loopexit.loopexit2:		; preds = %decide.exit.i.i, %bb1.i48.i.i, %bb29.i.i
+  br label %sat.exit.i.loopexit
+
+sat.exit.i.loopexit:		; preds = %sat.exit.i.loopexit.loopexit2, %sat.exit.i.loopexit.loopexit
+  br label %sat.exit.i
+
+sat.exit.i:		; preds = %sat.exit.i.loopexit, %bb1.i61.i.i, %bb8.i.i67, %bb1.i.i.i63, %bb3.i.i59
+  br i1 false, label %bb7.i, label %bb2.i96
+
+bb2.i96:		; preds = %sat.exit.i
+  switch i32 0, label %bb5.i99 [
+  i32 10, label %bb4.i98
+  i32 20, label %bb6.i100
+  ]
+
+bb4.i98:		; preds = %bb2.i96
+  br label %bb6.i100
+
+bb5.i99:		; preds = %bb2.i96
+  br label %bb6.i100
+
+bb6.i100:		; preds = %bb5.i99, %bb4.i98, %bb2.i96
+  br label %bb7.i
+
+bb7.i:		; preds = %bb6.i100, %sat.exit.i
+  br i1 false, label %bb.i1.i, label %picosat_sat.exit
+
+bb.i1.i:		; preds = %bb7.i
+  br label %picosat_sat.exit
+
+picosat_sat.exit:		; preds = %bb.i1.i, %bb7.i
+  switch i32 0, label %bb166 [
+  i32 20, label %bb150
+  i32 10, label %bb163
+  ]
+
+bb150:		; preds = %picosat_sat.exit
+  br i1 false, label %bb152, label %bb151
+
+bb151:		; preds = %bb150
+  br label %bb152
+
+bb152:		; preds = %bb151, %bb150
+  br i1 false, label %bb154, label %bb153
+
+bb153:		; preds = %bb152
+  br label %bb154
+
+bb154:		; preds = %bb153, %bb152
+  br i1 false, label %bb157, label %bb156
+
+bb156:		; preds = %bb154
+  br label %bb157
+
+bb157:		; preds = %bb156, %bb154
+  br i1 false, label %bb159, label %bb158
+
+bb158:		; preds = %bb157
+  br label %bb159
+
+bb159:		; preds = %bb158, %bb157
+  br i1 false, label %bb167, label %bb160
+
+bb160:		; preds = %bb159
+  br label %bb167
+
+bb163:		; preds = %picosat_sat.exit
+  br i1 false, label %bb167, label %bb164
+
+bb164:		; preds = %bb163
+  br label %bb4.i
+
+bb.i11:		; preds = %bb4.i
+  br i1 false, label %bb.i.i12, label %bb1.i.i14
+
+bb.i.i12:		; preds = %bb.i11
+  unreachable
+
+bb1.i.i14:		; preds = %bb.i11
+  br i1 false, label %bb3.i.i16, label %bb2.i.i15
+
+bb2.i.i15:		; preds = %bb1.i.i14
+  unreachable
+
+bb3.i.i16:		; preds = %bb1.i.i14
+  br i1 false, label %bb3.i, label %bb7.i.i
+
+bb7.i.i:		; preds = %bb3.i.i16
+  br i1 false, label %bb.i.i.i.i17, label %bb1.i.i.i.i18
+
+bb.i.i.i.i17:		; preds = %bb7.i.i
+  br label %int2lit.exit.i.i
+
+bb1.i.i.i.i18:		; preds = %bb7.i.i
+  br label %int2lit.exit.i.i
+
+int2lit.exit.i.i:		; preds = %bb1.i.i.i.i18, %bb.i.i.i.i17
+  br i1 false, label %bb3.i, label %bb9.i.i
+
+bb9.i.i:		; preds = %int2lit.exit.i.i
+  br label %bb3.i
+
+bb3.i:		; preds = %bb9.i.i, %int2lit.exit.i.i, %bb3.i.i16
+  br label %bb4.i
+
+bb4.i:		; preds = %bb3.i, %bb164
+  br i1 false, label %bb5.i, label %bb.i11
+
+bb5.i:		; preds = %bb4.i
+  br i1 false, label %bb6.i, label %bb167
+
+bb6.i:		; preds = %bb5.i
+  br label %bb167
+
+bb166:		; preds = %picosat_sat.exit
+  br label %bb167
+
+bb167:		; preds = %bb166, %bb6.i, %bb5.i, %bb163, %bb160, %bb159, %picosat_print.exit
+  br i1 false, label %bb168, label %bb170
+
+bb168:		; preds = %bb167
+  br i1 false, label %bb170, label %bb169
+
+bb169:		; preds = %bb168
+  br i1 false, label %bb.i7, label %picosat_time_stamp.exit9
+
+bb.i7:		; preds = %bb169
+  br label %picosat_time_stamp.exit9
+
+picosat_time_stamp.exit9:		; preds = %bb.i7, %bb169
+  br label %bb170
+
+bb170:		; preds = %picosat_time_stamp.exit9, %bb168, %bb167, %bb129
+  br i1 false, label %bb.i.i3, label %picosat_leave.exit
+
+bb.i.i3:		; preds = %bb170
+  br label %picosat_leave.exit
+
+picosat_leave.exit:		; preds = %bb.i.i3, %bb170
+  br i1 false, label %bb1.i.i, label %bb.i.i
+
+bb.i.i:		; preds = %picosat_leave.exit
+  unreachable
+
+bb1.i.i:		; preds = %picosat_leave.exit
+  br label %bb9.i.i.i
+
+bb3.i.i.i:		; preds = %bb9.i.i.i
+  br i1 false, label %bb5.i.i.i, label %bb4.i.i.i
+
+bb4.i.i.i:		; preds = %bb3.i.i.i
+  br label %bb5.i.i.i
+
+bb5.i.i.i:		; preds = %bb4.i.i.i, %bb3.i.i.i
+  br label %bb9.i.i.i
+
+bb9.i.i.i:		; preds = %bb5.i.i.i, %bb1.i.i
+  br i1 false, label %bb10.i.i.i, label %bb3.i.i.i
+
+bb10.i.i.i:		; preds = %bb9.i.i.i
+  br i1 false, label %delete.exit.i.i.i, label %bb1.i.i.i.i
+
+bb1.i.i.i.i:		; preds = %bb10.i.i.i
+  br label %delete.exit.i.i.i
+
+delete.exit.i.i.i:		; preds = %bb1.i.i.i.i, %bb10.i.i.i
+  br i1 false, label %delete_clauses.exit.i.i, label %bb1.i7.i.i.i
+
+bb1.i7.i.i.i:		; preds = %delete.exit.i.i.i
+  br label %delete_clauses.exit.i.i
+
+delete_clauses.exit.i.i:		; preds = %bb1.i7.i.i.i, %delete.exit.i.i.i
+  br label %bb3.i.i
+
+bb2.i.i:		; preds = %bb3.i.i
+  br i1 false, label %lrelease.exit.i.i, label %bb1.i.i23.i.i
+
+bb1.i.i23.i.i:		; preds = %bb2.i.i
+  br label %lrelease.exit.i.i
+
+lrelease.exit.i.i:		; preds = %bb1.i.i23.i.i, %bb2.i.i
+  br label %bb3.i.i
+
+bb3.i.i:		; preds = %lrelease.exit.i.i, %delete_clauses.exit.i.i
+  br i1 false, label %bb4.i.i, label %bb2.i.i
+
+bb4.i.i:		; preds = %bb3.i.i
+  br i1 false, label %delete.exit214.i.i, label %bb1.i208.i.i
+
+bb1.i208.i.i:		; preds = %bb4.i.i
+  br label %delete.exit214.i.i
+
+delete.exit214.i.i:		; preds = %bb1.i208.i.i, %bb4.i.i
+  br i1 false, label %delete.exit203.i.i, label %bb1.i197.i.i
+
+bb1.i197.i.i:		; preds = %delete.exit214.i.i
+  br label %delete.exit203.i.i
+
+delete.exit203.i.i:		; preds = %bb1.i197.i.i, %delete.exit214.i.i
+  br i1 false, label %delete.exit192.i.i, label %bb1.i186.i.i
+
+bb1.i186.i.i:		; preds = %delete.exit203.i.i
+  br label %delete.exit192.i.i
+
+delete.exit192.i.i:		; preds = %bb1.i186.i.i, %delete.exit203.i.i
+  br i1 false, label %delete.exit181.i.i, label %bb1.i175.i.i
+
+bb1.i175.i.i:		; preds = %delete.exit192.i.i
+  br label %delete.exit181.i.i
+
+delete.exit181.i.i:		; preds = %bb1.i175.i.i, %delete.exit192.i.i
+  br i1 false, label %delete.exit170.i.i, label %bb1.i164.i.i
+
+bb1.i164.i.i:		; preds = %delete.exit181.i.i
+  br label %delete.exit170.i.i
+
+delete.exit170.i.i:		; preds = %bb1.i164.i.i, %delete.exit181.i.i
+  br i1 false, label %delete.exit159.i.i, label %bb1.i153.i.i
+
+bb1.i153.i.i:		; preds = %delete.exit170.i.i
+  br label %delete.exit159.i.i
+
+delete.exit159.i.i:		; preds = %bb1.i153.i.i, %delete.exit170.i.i
+  br i1 false, label %delete.exit148.i.i, label %bb1.i142.i.i
+
+bb1.i142.i.i:		; preds = %delete.exit159.i.i
+  br label %delete.exit148.i.i
+
+delete.exit148.i.i:		; preds = %bb1.i142.i.i, %delete.exit159.i.i
+  br i1 false, label %delete.exit137.i.i, label %bb1.i131.i.i
+
+bb1.i131.i.i:		; preds = %delete.exit148.i.i
+  br label %delete.exit137.i.i
+
+delete.exit137.i.i:		; preds = %bb1.i131.i.i, %delete.exit148.i.i
+  br i1 false, label %delete.exit126.i.i, label %bb1.i120.i.i
+
+bb1.i120.i.i:		; preds = %delete.exit137.i.i
+  br label %delete.exit126.i.i
+
+delete.exit126.i.i:		; preds = %bb1.i120.i.i, %delete.exit137.i.i
+  br i1 false, label %delete.exit115.i.i, label %bb1.i109.i.i
+
+bb1.i109.i.i:		; preds = %delete.exit126.i.i
+  br label %delete.exit115.i.i
+
+delete.exit115.i.i:		; preds = %bb1.i109.i.i, %delete.exit126.i.i
+  br i1 false, label %delete.exit104.i.i, label %bb1.i98.i.i
+
+bb1.i98.i.i:		; preds = %delete.exit115.i.i
+  br label %delete.exit104.i.i
+
+delete.exit104.i.i:		; preds = %bb1.i98.i.i, %delete.exit115.i.i
+  br i1 false, label %delete.exit93.i.i, label %bb1.i87.i.i
+
+bb1.i87.i.i:		; preds = %delete.exit104.i.i
+  br label %delete.exit93.i.i
+
+delete.exit93.i.i:		; preds = %bb1.i87.i.i, %delete.exit104.i.i
+  br i1 false, label %delete.exit82.i.i, label %bb1.i76.i.i
+
+bb1.i76.i.i:		; preds = %delete.exit93.i.i
+  br label %delete.exit82.i.i
+
+delete.exit82.i.i:		; preds = %bb1.i76.i.i, %delete.exit93.i.i
+  br i1 false, label %delete.exit71.i.i, label %bb1.i65.i.i
+
+bb1.i65.i.i:		; preds = %delete.exit82.i.i
+  br label %delete.exit71.i.i
+
+delete.exit71.i.i:		; preds = %bb1.i65.i.i, %delete.exit82.i.i
+  br i1 false, label %delete.exit60.i.i, label %bb1.i54.i.i
+
+bb1.i54.i.i:		; preds = %delete.exit71.i.i
+  br label %delete.exit60.i.i
+
+delete.exit60.i.i:		; preds = %bb1.i54.i.i, %delete.exit71.i.i
+  br i1 false, label %delete.exit38.i.i, label %bb1.i32.i.i
+
+bb1.i32.i.i:		; preds = %delete.exit60.i.i
+  br label %delete.exit38.i.i
+
+delete.exit38.i.i:		; preds = %bb1.i32.i.i, %delete.exit60.i.i
+  br i1 false, label %delete.exit18.i.i, label %bb1.i12.i.i
+
+bb1.i12.i.i:		; preds = %delete.exit38.i.i
+  br label %delete.exit18.i.i
+
+delete.exit18.i.i:		; preds = %bb1.i12.i.i, %delete.exit38.i.i
+  br i1 false, label %picosat_reset.exit, label %bb1.i2.i.i
+
+bb1.i2.i.i:		; preds = %delete.exit18.i.i
+  br label %picosat_reset.exit
+
+picosat_reset.exit:		; preds = %bb1.i2.i.i, %delete.exit18.i.i
+  br label %bb171
+
+bb171:		; preds = %picosat_reset.exit, %bb110
+  br i1 false, label %bb173, label %bb172
+
+bb172:		; preds = %bb171
+  br label %bb173
+
+bb173:		; preds = %bb172, %bb171
+  br i1 false, label %bb175, label %bb174
+
+bb174:		; preds = %bb173
+  br label %bb175
+
+bb175:		; preds = %bb174, %bb173
+  br i1 false, label %bb177, label %bb176
+
+bb176:		; preds = %bb175
+  br label %bb177
+
+bb177:		; preds = %bb176, %bb175
+  br i1 false, label %bb179, label %bb178
+
+bb178:		; preds = %bb177
+  ret i32 0
+
+bb179:		; preds = %bb177
+  ret i32 0
+}
+
+define i32 @main(i32 %argc, i8** %argv) nounwind {
+entry:
+  br label %bb2
+
+bb:		; preds = %bb2
+  br i1 false, label %bb3, label %bb2
+
+bb2:		; preds = %bb, %entry
+  br i1 false, label %bb5.loopexit, label %bb
+
+bb3:		; preds = %bb
+  br i1 false, label %bb5, label %bb4
+
+bb4:		; preds = %bb3
+  br label %bb5
+
+bb5.loopexit:		; preds = %bb2
+  br label %bb5
+
+bb5:		; preds = %bb5.loopexit, %bb4, %bb3
+  %0 = call fastcc i32 @picosat_main(i32 %argc, i8** %argv) nounwind		; <i32> [#uses=2]
+  br i1 false, label %bb7, label %bb6
+
+bb6:		; preds = %bb5
+  ret i32 %0
+
+bb7:		; preds = %bb5
+  ret i32 %0
+}
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/pr32917.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/pr32917.ll
new file mode 100644
index 000000000..b2bca6206
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/pr32917.ll
@@ -0,0 +1,35 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; RUN: opt -S -passes=attributor -aa-pipeline='basic-aa' -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=3 < %s | FileCheck %s
+; PR 32917
+
+@b = common local_unnamed_addr global i32 0, align 4
+@a = common local_unnamed_addr global i32 0, align 4
+
+define i32 @fn2() local_unnamed_addr {
+; CHECK-LABEL: define {{[^@]+}}@fn2() local_unnamed_addr
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* @b, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = sext i32 [[TMP1]] to i64
+; CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[TMP2]] to i32*
+; CHECK-NEXT:    call fastcc void @fn1(i32* nofree readonly align 4 [[TMP3]])
+; CHECK-NEXT:    ret i32 undef
+;
+  %1 = load i32, i32* @b, align 4
+  %2 = sext i32 %1 to i64
+  %3 = inttoptr i64 %2 to i32*
+  call fastcc void @fn1(i32* %3)
+  ret i32 undef
+}
+
+define internal fastcc void @fn1(i32* nocapture readonly) unnamed_addr {
+; CHECK-LABEL: define {{[^@]+}}@fn1
+; CHECK-SAME: (i32* nocapture nofree nonnull readonly align 4 [[TMP0:%.*]]) unnamed_addr
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 -1
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, i32* [[TMP2]], align 4
+; CHECK-NEXT:    store i32 [[TMP3]], i32* @a, align 4
+; CHECK-NEXT:    ret void
+;
+  %2 = getelementptr inbounds i32, i32* %0, i64 -1
+  %3 = load i32, i32* %2, align 4
+  store i32 %3, i32* @a, align 4
+  ret void
+}
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/pr33641_remove_arg_dbgvalue.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/pr33641_remove_arg_dbgvalue.ll
new file mode 100644
index 000000000..de67b8970
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/pr33641_remove_arg_dbgvalue.ll
@@ -0,0 +1,45 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; RUN: opt -S -passes=attributor -aa-pipeline='basic-aa' -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=1 < %s | FileCheck %s
+
+; Fix for PR33641. ArgumentPromotion removed the argument to bar but left the call to
+; dbg.value which still used the removed argument.
+
+; The %p argument should be removed, and the use of it in dbg.value should be
+; changed to undef.
+
+%p_t = type i16*
+%fun_t = type void (%p_t)*
+
+define void @foo() {
+; CHECK-LABEL: define {{[^@]+}}@foo()
+; CHECK-NEXT:    [[TMP:%.*]] = alloca void (i16*)*
+; CHECK-NEXT:    store void (i16*)* @bar, void (i16*)** [[TMP]], align 8
+; CHECK-NEXT:    ret void
+;
+  %tmp = alloca %fun_t
+  store %fun_t @bar, %fun_t* %tmp
+  ret void
+}
+
+define internal void @bar(%p_t %p)  {
+; CHECK-LABEL: define {{[^@]+}}@bar
+; CHECK-SAME: (i16* nocapture nofree readnone [[P:%.*]])
+; CHECK-NEXT:    call void @llvm.dbg.value(metadata i16* [[P]], metadata !3, metadata !DIExpression()) #3, !dbg !5
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.dbg.value(metadata %p_t %p, metadata !4, metadata !5), !dbg !6
+  ret void
+}
+
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!2}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C, file: !1)
+!1 = !DIFile(filename: "test.c", directory: "")
+!2 = !{i32 2, !"Debug Info Version", i32 3}
+!3 = distinct !DISubprogram(name: "bar", unit: !0)
+!4 = !DILocalVariable(name: "p", scope: !3)
+!5 = !DIExpression()
+!6 = !DILocation(line: 1, column: 1, scope: !3)
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/profile.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/profile.ll
new file mode 100644
index 000000000..5eda92356
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/profile.ll
@@ -0,0 +1,34 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; RUN: opt -S -passes='attributor' -aa-pipeline='basic-aa' -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=2 < %s | FileCheck %s
+target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
+
+; Checks if !prof metadata is corret in deadargelim.
+
+define void @caller() #0 {
+; CHECK-LABEL: define {{[^@]+}}@caller()
+; CHECK-NEXT:    [[X:%.*]] = alloca i32
+; CHECK-NEXT:    store i32 42, i32* [[X]], align 4
+; CHECK-NEXT:    call void @promote_i32_ptr(i32* noalias nocapture nonnull readonly align 4 dereferenceable(4) [[X]]), !prof !0
+; CHECK-NEXT:    ret void
+;
+  %x = alloca i32
+  store i32 42, i32* %x
+  call void @promote_i32_ptr(i32* %x), !prof !0
+  ret void
+}
+
+define internal void @promote_i32_ptr(i32* %xp) {
+; CHECK-LABEL: define {{[^@]+}}@promote_i32_ptr
+; CHECK-SAME: (i32* noalias nocapture nonnull readonly align 4 dereferenceable(4) [[XP:%.*]])
+; CHECK-NEXT:    [[X:%.*]] = load i32, i32* [[XP]], align 4
+; CHECK-NEXT:    call void @use_i32(i32 [[X]])
+; CHECK-NEXT:    ret void
+;
+  %x = load i32, i32* %xp
+  call void @use_i32(i32 %x)
+  ret void
+}
+
+declare void @use_i32(i32)
+
+!0 = !{!"branch_weights", i32 30}
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/reserve-tbaa.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/reserve-tbaa.ll
new file mode 100644
index 000000000..ef5dfb641
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/reserve-tbaa.ll
@@ -0,0 +1,65 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; RUN: opt -S -passes='attributor' -aa-pipeline='basic-aa' -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=2 < %s | FileCheck %s
+
+; PR17906
+; When we promote two arguments in a single function with different types,
+; before the fix, we used the same tag for the newly-created two loads.
+; This testing case makes sure that we correctly transfer the tbaa tags from the
+; original loads to the newly-created loads when promoting pointer arguments.
+
+@a = global i32* null, align 8
+@e = global i32** @a, align 8
+@g = global i32 0, align 4
+@c = global i64 0, align 8
+@d = global i8 0, align 1
+
+define internal fastcc void @fn(i32* nocapture readonly %p1, i64* nocapture readonly %p2) {
+; CHECK-LABEL: define {{[^@]+}}@fn
+; CHECK-SAME: (i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[P1:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, i64* undef, align 8, !tbaa !0
+; CHECK-NEXT:    [[CONV:%.*]] = trunc i64 [[TMP0]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* @g, align 4, !tbaa !4
+; CHECK-NEXT:    [[CONV1:%.*]] = trunc i32 [[TMP1]] to i8
+; CHECK-NEXT:    store i8 [[CONV1]], i8* @d, align 1, !tbaa !6
+; CHECK-NEXT:    ret void
+;
+entry:
+  %0 = load i64, i64* %p2, align 8, !tbaa !1
+  %conv = trunc i64 %0 to i32
+  %1 = load i32, i32* %p1, align 4, !tbaa !5
+  %conv1 = trunc i32 %1 to i8
+  store i8 %conv1, i8* @d, align 1, !tbaa !7
+  ret void
+}
+
+define i32 @main() {
+; CHECK-LABEL: define {{[^@]+}}@main()
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32**, i32*** @e, align 8, !tbaa !7
+; CHECK-NEXT:    store i32* @g, i32** [[TMP0]], align 8, !tbaa !7
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32*, i32** @a, align 8, !tbaa !7
+; CHECK-NEXT:    store i32 1, i32* [[TMP1]], align 4, !tbaa !4
+; CHECK-NEXT:    call fastcc void @fn(i32* nofree nonnull readonly align 4 dereferenceable(4) @g)
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %0 = load i32**, i32*** @e, align 8, !tbaa !8
+  store i32* @g, i32** %0, align 8, !tbaa !8
+  %1 = load i32*, i32** @a, align 8, !tbaa !8
+  store i32 1, i32* %1, align 4, !tbaa !5
+  call fastcc void @fn(i32* @g, i64* @c)
+
+  ret i32 0
+}
+
+!1 = !{!2, !2, i64 0}
+!2 = !{!"long", !3, i64 0}
+!3 = !{!"omnipotent char", !4, i64 0}
+!4 = !{!"Simple C/C++ TBAA"}
+!5 = !{!6, !6, i64 0}
+!6 = !{!"int", !3, i64 0}
+!7 = !{!3, !3, i64 0}
+!8 = !{!9, !9, i64 0}
+!9 = !{!"any pointer", !3, i64 0}
+
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/sret.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/sret.ll
new file mode 100644
index 000000000..33048a31a
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/sret.ll
@@ -0,0 +1,39 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; RUN: opt -S -passes='attributor' -aa-pipeline='basic-aa' -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=2 < %s | FileCheck %s
+
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc"
+
+define internal void @add({i32, i32}* %this, i32* sret %r) {
+; CHECK-LABEL: define {{[^@]+}}@add
+; CHECK-SAME: ({ i32, i32 }* noalias nocapture nofree nonnull readonly align 8 dereferenceable(8) [[THIS:%.*]], i32* noalias nocapture nofree nonnull sret writeonly align 4 dereferenceable(4) [[R:%.*]])
+; CHECK-NEXT:    [[AP:%.*]] = getelementptr { i32, i32 }, { i32, i32 }* [[THIS]], i32 0, i32 0
+; CHECK-NEXT:    [[BP:%.*]] = getelementptr { i32, i32 }, { i32, i32 }* [[THIS]], i32 0, i32 1
+; CHECK-NEXT:    [[A:%.*]] = load i32, i32* [[AP]], align 8
+; CHECK-NEXT:    [[B:%.*]] = load i32, i32* [[BP]]
+; CHECK-NEXT:    [[AB:%.*]] = add i32 [[A]], [[B]]
+; CHECK-NEXT:    store i32 [[AB]], i32* [[R]], align 4
+; CHECK-NEXT:    ret void
+;
+  %ap = getelementptr {i32, i32}, {i32, i32}* %this, i32 0, i32 0
+  %bp = getelementptr {i32, i32}, {i32, i32}* %this, i32 0, i32 1
+  %a = load i32, i32* %ap
+  %b = load i32, i32* %bp
+  %ab = add i32 %a, %b
+  store i32 %ab, i32* %r
+  ret void
+}
+
+define void @f() {
+; CHECK-LABEL: define {{[^@]+}}@f()
+; CHECK-NEXT:    [[R:%.*]] = alloca i32
+; CHECK-NEXT:    [[PAIR:%.*]] = alloca { i32, i32 }
+; CHECK-NEXT:    call void @add({ i32, i32 }* noalias nocapture nofree nonnull readonly align 8 dereferenceable(8) [[PAIR]], i32* noalias nocapture nofree nonnull sret writeonly align 4 dereferenceable(4) [[R]])
+; CHECK-NEXT:    ret void
+;
+  %r = alloca i32
+  %pair = alloca {i32, i32}
+
+  call void @add({i32, i32}* %pair, i32* sret %r)
+  ret void
+}
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/tail.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/tail.ll
new file mode 100644
index 000000000..2a34bde66
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/tail.ll
@@ -0,0 +1,29 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; RUN: opt -S -passes='attributor' -aa-pipeline='basic-aa' -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=4 < %s | FileCheck %s
+; PR14710
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+%pair = type { i32, i32 }
+
+declare i8* @foo(%pair*)
+
+define internal void @bar(%pair* byval %Data) {
+; CHECK-LABEL: define {{[^@]+}}@bar
+; CHECK-SAME: (%pair* byval [[DATA:%.*]])
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call i8* @foo(%pair* [[DATA]])
+; CHECK-NEXT:    ret void
+;
+  tail call i8* @foo(%pair* %Data)
+  ret void
+}
+
+define void @zed(%pair* byval %Data) {
+; CHECK-LABEL: define {{[^@]+}}@zed
+; CHECK-SAME: (%pair* nocapture readonly byval [[DATA:%.*]])
+; CHECK-NEXT:    call void @bar(%pair* nocapture readonly byval [[DATA]])
+; CHECK-NEXT:    ret void
+;
+  call void @bar(%pair* byval %Data)
+  ret void
+}
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/variadic.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/variadic.ll
new file mode 100644
index 000000000..b42d4bc41
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/variadic.ll
@@ -0,0 +1,38 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; RUN: opt -S -passes='attributor' -aa-pipeline='basic-aa' -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=1 < %s | FileCheck %s
+
+; Unused arguments from variadic functions cannot be eliminated as that changes
+; their classiciation according to the SysV amd64 ABI. Clang and other frontends
+; bake in the classification when they use things like byval, as in this test.
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+%struct.tt0 = type { i64, i64 }
+%struct.__va_list_tag = type { i32, i32, i8*, i8* }
+
+@t45 = internal global %struct.tt0 { i64 1335139741, i64 438042995 }, align 8
+
+; Function Attrs: nounwind uwtable
+define i32 @main(i32 %argc, i8** nocapture readnone %argv) #0 {
+; CHECK-LABEL: define {{[^@]+}}@main
+; CHECK-SAME: (i32 [[ARGC:%.*]], i8** nocapture nofree readnone [[ARGV:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    tail call void (i8*, i8*, i8*, i8*, i8*, ...) @callee_t0f(i8* undef, i8* undef, i8* undef, i8* undef, i8* undef, %struct.tt0* nonnull byval align 8 dereferenceable(16) @t45)
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  tail call void (i8*, i8*, i8*, i8*, i8*, ...) @callee_t0f(i8* undef, i8* undef, i8* undef, i8* undef, i8* undef, %struct.tt0* byval align 8 @t45)
+  ret i32 0
+}
+
+; Function Attrs: nounwind uwtable
+define internal void @callee_t0f(i8* nocapture readnone %tp13, i8* nocapture readnone %tp14, i8* nocapture readnone %tp15, i8* nocapture readnone %tp16, i8* nocapture readnone %tp17, ...) {
+; CHECK-LABEL: define {{[^@]+}}@callee_t0f
+; CHECK-SAME: (i8* nocapture nofree nonnull readnone [[TP13:%.*]], i8* nocapture nofree nonnull readnone [[TP14:%.*]], i8* nocapture nofree nonnull readnone [[TP15:%.*]], i8* nocapture nofree nonnull readnone [[TP16:%.*]], i8* nocapture nofree nonnull readnone [[TP17:%.*]], ...)
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret void
+;
+entry:
+  ret void
+}
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/2008-06-09-WeakProp.ll b/llvm/test/Transforms/Attributor/IPConstantProp/2008-06-09-WeakProp.ll
new file mode 100644
index 000000000..d05e3b113
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/2008-06-09-WeakProp.ll
@@ -0,0 +1,25 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; RUN: opt < %s -passes=attributor -S | FileCheck %s
+; Should not propagate the result of a weak function.
+; PR2411
+
+define weak i32 @foo() nounwind  {
+; CHECK-LABEL: define {{[^@]+}}@foo()
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret i32 1
+;
+entry:
+  ret i32 1
+}
+
+define i32 @main() nounwind  {
+; CHECK-LABEL: define {{[^@]+}}@main()
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[R:%.*]] = call i32 @foo()
+; CHECK-NEXT:    ret i32 [[R]]
+;
+entry:
+  %r = call i32 @foo( ) nounwind
+  ret i32 %r
+}
+
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/2009-09-24-byval-ptr.ll b/llvm/test/Transforms/Attributor/IPConstantProp/2009-09-24-byval-ptr.ll
new file mode 100644
index 000000000..8616794fc
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/2009-09-24-byval-ptr.ll
@@ -0,0 +1,102 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; RUN: opt -S -passes=attributor -aa-pipeline='basic-aa' -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=2 < %s | FileCheck %s
+; Don't constant-propagate byval pointers, since they are not pointers!
+; PR5038
+%struct.MYstr = type { i8, i32 }
+@mystr = internal global %struct.MYstr zeroinitializer ; <%struct.MYstr*> [#uses=3]
+define internal void @vfu1(%struct.MYstr* byval align 4 %u) nounwind {
+; CHECK-LABEL: define {{[^@]+}}@vfu1
+; CHECK-SAME: (%struct.MYstr* nocapture nofree nonnull writeonly byval align 8 dereferenceable(8) [[U:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr [[STRUCT_MYSTR:%.*]], %struct.MYstr* [[U]], i32 0, i32 1
+; CHECK-NEXT:    store i32 99, i32* [[TMP0]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr [[STRUCT_MYSTR]], %struct.MYstr* [[U]], i32 0, i32 0
+; CHECK-NEXT:    store i8 97, i8* [[TMP1]], align 8
+; CHECK-NEXT:    br label [[RETURN:%.*]]
+; CHECK:       return:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %0 = getelementptr %struct.MYstr, %struct.MYstr* %u, i32 0, i32 1 ; <i32*> [#uses=1]
+  store i32 99, i32* %0, align 4
+  %1 = getelementptr %struct.MYstr, %struct.MYstr* %u, i32 0, i32 0 ; <i8*> [#uses=1]
+  store i8 97, i8* %1, align 4
+  br label %return
+
+return:                                           ; preds = %entry
+  ret void
+}
+
+define internal i32 @vfu2(%struct.MYstr* byval align 4 %u) nounwind readonly {
+; CHECK-LABEL: define {{[^@]+}}@vfu2
+; CHECK-SAME: (%struct.MYstr* nocapture nofree nonnull readonly byval align 8 dereferenceable(8) [[U:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr [[STRUCT_MYSTR:%.*]], %struct.MYstr* @mystr, i32 0, i32 1
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr [[STRUCT_MYSTR]], %struct.MYstr* @mystr, i32 0, i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, i8* [[TMP2]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i8 [[TMP3]] to i32
+; CHECK-NEXT:    [[TMP5:%.*]] = add i32 [[TMP4]], [[TMP1]]
+; CHECK-NEXT:    ret i32 [[TMP5]]
+;
+entry:
+  %0 = getelementptr %struct.MYstr, %struct.MYstr* %u, i32 0, i32 1 ; <i32*> [#uses=1]
+  %1 = load i32, i32* %0
+  %2 = getelementptr %struct.MYstr, %struct.MYstr* %u, i32 0, i32 0 ; <i8*> [#uses=1]
+  %3 = load i8, i8* %2
+  %4 = zext i8 %3 to i32
+  %5 = add i32 %4, %1
+  ret i32 %5
+}
+
+define i32 @unions() nounwind {
+; CHECK-LABEL: define {{[^@]+}}@unions()
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    call void @vfu1(%struct.MYstr* nofree nonnull readonly byval align 8 dereferenceable(8) @mystr)
+; CHECK-NEXT:    [[RESULT:%.*]] = call i32 @vfu2(%struct.MYstr* nofree nonnull readonly byval align 8 dereferenceable(8) @mystr)
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  call void @vfu1(%struct.MYstr* byval align 4 @mystr) nounwind
+  %result = call i32 @vfu2(%struct.MYstr* byval align 4 @mystr) nounwind
+  ret i32 %result
+}
+
+define internal i32 @vfu2_v2(%struct.MYstr* byval align 4 %u) nounwind readonly {
+; CHECK-LABEL: define {{[^@]+}}@vfu2_v2
+; CHECK-SAME: (%struct.MYstr* nocapture nofree nonnull byval align 8 dereferenceable(8) [[U:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[Z:%.*]] = getelementptr [[STRUCT_MYSTR:%.*]], %struct.MYstr* [[U]], i32 0, i32 1
+; CHECK-NEXT:    store i32 99, i32* [[Z]], align 4
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr [[STRUCT_MYSTR]], %struct.MYstr* [[U]], i32 0, i32 1
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr [[STRUCT_MYSTR]], %struct.MYstr* [[U]], i32 0, i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, i8* [[TMP2]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i8 [[TMP3]] to i32
+; CHECK-NEXT:    [[TMP5:%.*]] = add i32 [[TMP4]], [[TMP1]]
+; CHECK-NEXT:    ret i32 [[TMP5]]
+;
+entry:
+  %z = getelementptr %struct.MYstr, %struct.MYstr* %u, i32 0, i32 1
+  store i32 99, i32* %z, align 4
+  %0 = getelementptr %struct.MYstr, %struct.MYstr* %u, i32 0, i32 1 ; <i32*> [#uses=1]
+  %1 = load i32, i32* %0
+  %2 = getelementptr %struct.MYstr, %struct.MYstr* %u, i32 0, i32 0 ; <i8*> [#uses=1]
+  %3 = load i8, i8* %2
+  %4 = zext i8 %3 to i32
+  %5 = add i32 %4, %1
+  ret i32 %5
+}
+
+define i32 @unions_v2() nounwind {
+; CHECK-LABEL: define {{[^@]+}}@unions_v2()
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    call void @vfu1(%struct.MYstr* nofree nonnull readonly byval align 8 dereferenceable(8) @mystr)
+; CHECK-NEXT:    [[RESULT:%.*]] = call i32 @vfu2_v2(%struct.MYstr* nofree nonnull readonly byval align 8 dereferenceable(8) @mystr)
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  call void @vfu1(%struct.MYstr* byval align 4 @mystr) nounwind
+  %result = call i32 @vfu2_v2(%struct.MYstr* byval align 4 @mystr) nounwind
+  ret i32 %result
+}
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/PR16052.ll b/llvm/test/Transforms/Attributor/IPConstantProp/PR16052.ll
new file mode 100644
index 000000000..6ec7dde13
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/PR16052.ll
@@ -0,0 +1,34 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; RUN: opt -S -passes=attributor -aa-pipeline='basic-aa' -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=3 < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define i64 @fn2() {
+; CHECK-LABEL: define {{[^@]+}}@fn2()
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CONV:%.*]] = sext i32 undef to i64
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv i64 8, [[CONV]]
+; CHECK-NEXT:    [[CALL2:%.*]] = call i64 @fn1(i64 [[DIV]])
+; CHECK-NEXT:    ret i64 [[CALL2]]
+;
+entry:
+  %conv = sext i32 undef to i64
+  %div = sdiv i64 8, %conv
+  %call2 = call i64 @fn1(i64 %div)
+  ret i64 %call2
+}
+
+define internal i64 @fn1(i64 %p1) {
+; CHECK-LABEL: define {{[^@]+}}@fn1
+; CHECK-SAME: (i64 returned [[P1:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne i64 [[P1]], 0
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[TOBOOL]], i64 [[P1]], i64 [[P1]]
+; CHECK-NEXT:    ret i64 [[COND]]
+;
+entry:
+  %tobool = icmp ne i64 %p1, 0
+  %cond = select i1 %tobool, i64 %p1, i64 %p1
+  ret i64 %cond
+}
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/PR26044.ll b/llvm/test/Transforms/Attributor/IPConstantProp/PR26044.ll
new file mode 100644
index 000000000..b7f24b885
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/PR26044.ll
@@ -0,0 +1,88 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; RUN: opt -S -passes=attributor -aa-pipeline='basic-aa' -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=1 < %s | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @fn2(i32* %P) {
+; CHECK-LABEL: define {{[^@]+}}@fn2
+; CHECK-SAME: (i32* nocapture nofree writeonly [[P:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[IF_END:%.*]]
+; CHECK:       for.cond1:
+; CHECK-NEXT:    unreachable
+; CHECK:       if.end:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* undef, align 4
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @fn1(i32 [[TMP0]])
+; CHECK-NEXT:    store i32 [[CALL]], i32* [[P]]
+; CHECK-NEXT:    br label [[FOR_COND1:%.*]]
+;
+entry:
+  br label %if.end
+
+for.cond1:                                        ; preds = %if.end, %for.end
+  br i1 undef, label %if.end, label %if.end
+
+if.end:                                           ; preds = %lbl, %for.cond1
+  %e.2 = phi i32* [ undef, %entry ], [ null, %for.cond1 ], [ null, %for.cond1 ]
+  %0 = load i32, i32* %e.2, align 4
+  %call = call i32 @fn1(i32 %0)
+  store i32 %call, i32* %P
+  br label %for.cond1
+}
+
+define internal i32 @fn1(i32 %p1) {
+; CHECK-LABEL: define {{[^@]+}}@fn1
+; CHECK-SAME: (i32 returned [[P1:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne i32 [[P1]], 0
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[TOBOOL]], i32 [[P1]], i32 [[P1]]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+entry:
+  %tobool = icmp ne i32 %p1, 0
+  %cond = select i1 %tobool, i32 %p1, i32 %p1
+  ret i32 %cond
+}
+
+define void @fn_no_null_opt(i32* %P) #0 {
+; CHECK-LABEL: define {{[^@]+}}@fn_no_null_opt
+; CHECK-SAME: (i32* nocapture nofree writeonly [[P:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[IF_END:%.*]]
+; CHECK:       for.cond1:
+; CHECK-NEXT:    unreachable
+; CHECK:       if.end:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* undef, align 4
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @fn0(i32 [[TMP0]])
+; CHECK-NEXT:    store i32 [[CALL]], i32* [[P]]
+; CHECK-NEXT:    br label [[FOR_COND1:%.*]]
+;
+entry:
+  br label %if.end
+
+for.cond1:                                        ; preds = %if.end, %for.end
+  br i1 undef, label %if.end, label %if.end
+
+if.end:                                           ; preds = %lbl, %for.cond1
+  %e.2 = phi i32* [ undef, %entry ], [ null, %for.cond1 ], [ null, %for.cond1 ]
+  %0 = load i32, i32* %e.2, align 4
+  %call = call i32 @fn0(i32 %0)
+  store i32 %call, i32* %P
+  br label %for.cond1
+}
+
+define internal i32 @fn0(i32 %p1) {
+; CHECK-LABEL: define {{[^@]+}}@fn0
+; CHECK-SAME: (i32 returned [[P1:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne i32 [[P1]], 0
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[TOBOOL]], i32 [[P1]], i32 [[P1]]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+entry:
+  %tobool = icmp ne i32 %p1, 0
+  %cond = select i1 %tobool, i32 %p1, i32 %p1
+  ret i32 %cond
+}
+
+attributes #0 = { "null-pointer-is-valid"="true" }
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/PR43857.ll b/llvm/test/Transforms/Attributor/IPConstantProp/PR43857.ll
new file mode 100644
index 000000000..82c404ea2
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/PR43857.ll
@@ -0,0 +1,30 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; RUN: opt -S -passes=attributor -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=1 < %s | FileCheck %s
+
+%struct.wobble = type { i32 }
+%struct.zot = type { %struct.wobble, %struct.wobble, %struct.wobble }
+
+declare dso_local fastcc float @bar(%struct.wobble* noalias, <8 x i32>) unnamed_addr
+
+define %struct.zot @widget(<8 x i32> %arg) local_unnamed_addr {
+; CHECK-LABEL: define {{[^@]+}}@widget
+; CHECK-SAME: (<8 x i32> [[ARG:%.*]]) local_unnamed_addr
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    ret [[STRUCT_ZOT:%.*]] undef
+;
+bb:
+  ret %struct.zot undef
+}
+
+define void @baz(<8 x i32> %arg) local_unnamed_addr {
+; CHECK-LABEL: define {{[^@]+}}@baz
+; CHECK-SAME: (<8 x i32> [[ARG:%.*]]) local_unnamed_addr
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP1:%.*]] = extractvalue [[STRUCT_ZOT:%.*]] undef, 0, 0
+; CHECK-NEXT:    ret void
+;
+bb:
+  %tmp = call %struct.zot @widget(<8 x i32> %arg)
+  %tmp1 = extractvalue %struct.zot %tmp, 0, 0
+  ret void
+}
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/arg-count-mismatch.ll b/llvm/test/Transforms/Attributor/IPConstantProp/arg-count-mismatch.ll
new file mode 100644
index 000000000..cd8a9b4ba
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/arg-count-mismatch.ll
@@ -0,0 +1,85 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; RUN: opt -S -passes=attributor -aa-pipeline='basic-aa' -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=3 < %s | FileCheck %s
+
+; The original C source looked like this:
+;
+;   long long a101, b101, e101;
+;   volatile long c101;
+;   int d101;
+;
+;   static inline int bar(p1, p2)
+;   {
+;       return 0;
+;   }
+;
+;   void foo(unsigned p1)
+;   {
+;       long long *f = &b101, *g = &e101;
+;       c101 = 0;
+;       (void)((*f |= a101) - (*g = bar(d101)));
+;       c101 = (*f |= a101 &= p1) == d101;
+;   }
+;
+; When compiled with Clang it gives a warning
+;   warning: too few arguments in call to 'bar'
+;
+; This ll reproducer has been reduced to only include tha call.
+;
+; Note that -lint will report this as UB, but it passes -verify.
+
+; This test is just to verify that we do not crash/assert due to mismatch in
+; argument count between the caller and callee.
+
+define dso_local void @foo(i16 %a) {
+; CHECK-LABEL: define {{[^@]+}}@foo
+; CHECK-SAME: (i16 [[A:%.*]])
+; CHECK-NEXT:    [[CALL:%.*]] = call i16 bitcast (i16 (i16, i16)* @bar to i16 (i16)*)(i16 [[A]])
+; CHECK-NEXT:    ret void
+;
+  %call = call i16 bitcast (i16 (i16, i16) * @bar to i16 (i16) *)(i16 %a)
+  ret void
+}
+
+define internal i16 @bar(i16 %p1, i16 %p2) {
+; CHECK-LABEL: define {{[^@]+}}@bar
+; CHECK-SAME: (i16 [[P1:%.*]], i16 [[P2:%.*]])
+; CHECK-NEXT:    ret i16 0
+;
+  ret i16 0
+}
+
+;-------------------------------------------------------------------------------
+; Additional tests to verify that we still optimize when having a mismatch
+; in argument count due to varargs (as long as all non-variadic arguments have
+; been provided),
+
+define dso_local i16 @vararg_tests(i16 %a) {
+; CHECK-LABEL: define {{[^@]+}}@vararg_tests
+; CHECK-SAME: (i16 [[A:%.*]])
+; CHECK-NEXT:    [[CALL1:%.*]] = call i16 (i16, ...) @vararg_prop(i16 7, i16 8, i16 [[A]])
+; CHECK-NEXT:    [[CALL2:%.*]] = call i16 bitcast (i16 (i16, i16, ...)* @vararg_no_prop to i16 (i16)*)(i16 7)
+; CHECK-NEXT:    [[ADD:%.*]] = add i16 [[CALL1]], [[CALL2]]
+; CHECK-NEXT:    ret i16 [[ADD]]
+;
+  %call1 = call i16 (i16, ...) @vararg_prop(i16 7, i16 8, i16 %a)
+  %call2 = call i16 bitcast (i16 (i16, i16, ...) * @vararg_no_prop to i16 (i16) *) (i16 7)
+  %add = add i16 %call1, %call2
+  ret i16 %add
+}
+
+define internal i16 @vararg_prop(i16 %p1, ...) {
+; CHECK-LABEL: define {{[^@]+}}@vararg_prop
+; CHECK-SAME: (i16 returned [[P1:%.*]], ...)
+; CHECK-NEXT:    ret i16 7
+;
+  ret i16 %p1
+}
+
+define internal i16 @vararg_no_prop(i16 %p1, i16 %p2, ...) {
+; CHECK-LABEL: define {{[^@]+}}@vararg_no_prop
+; CHECK-SAME: (i16 returned [[P1:%.*]], i16 [[P2:%.*]], ...)
+; CHECK-NEXT:    ret i16 [[P1]]
+;
+  ret i16 %p1
+}
+
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/arg-type-mismatch.ll b/llvm/test/Transforms/Attributor/IPConstantProp/arg-type-mismatch.ll
new file mode 100644
index 000000000..e7f870567
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/arg-type-mismatch.ll
@@ -0,0 +1,25 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; RUN: opt -S -passes=attributor -aa-pipeline='basic-aa' -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=1 < %s | FileCheck %s
+
+; This test is just to verify that we do not crash/assert due to mismatch in
+; argument type between the caller and callee.
+
+define dso_local void @foo(i16 %a) {
+; CHECK-LABEL: define {{[^@]+}}@foo
+; CHECK-SAME: (i16 [[A:%.*]])
+; CHECK-NEXT:    [[CALL:%.*]] = call i16 bitcast (i16 (i16, i16)* @bar to i16 (i16, i32)*)(i16 [[A]], i32 7)
+; CHECK-NEXT:    ret void
+;
+  %call = call i16 bitcast (i16 (i16, i16) * @bar to i16 (i16, i32) *)(i16 %a, i32 7)
+  ret void
+}
+
+define internal i16 @bar(i16 %p1, i16 %p2) {
+; CHECK-LABEL: define {{[^@]+}}@bar
+; CHECK-SAME: (i16 [[P1:%.*]], i16 returned [[P2:%.*]])
+; CHECK-NEXT:    ret i16 [[P2]]
+;
+  ret i16 %p2
+}
+
+
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/comdat-ipo.ll b/llvm/test/Transforms/Attributor/IPConstantProp/comdat-ipo.ll
new file mode 100644
index 000000000..2f50a9c3d
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/comdat-ipo.ll
@@ -0,0 +1,34 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; RUN: opt -S -passes=attributor -aa-pipeline='basic-aa' -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=1 < %s | FileCheck %s
+
+; See PR26774
+
+define i32 @baz() {
+; CHECK-LABEL: define {{[^@]+}}@baz()
+; CHECK-NEXT:    ret i32 10
+;
+  ret i32 10
+}
+
+; We can const-prop @baz's return value *into* @foo, but cannot
+; constprop @foo's return value into bar.
+
+define linkonce_odr i32 @foo() {
+; CHECK-LABEL: define {{[^@]+}}@foo()
+; CHECK-NEXT:    [[VAL:%.*]] = call i32 @baz()
+; CHECK-NEXT:    ret i32 10
+;
+
+  %val = call i32 @baz()
+  ret i32 %val
+}
+
+define i32 @bar() {
+; CHECK-LABEL: define {{[^@]+}}@bar()
+; CHECK-NEXT:    [[VAL:%.*]] = call i32 @foo()
+; CHECK-NEXT:    ret i32 [[VAL]]
+;
+
+  %val = call i32 @foo()
+  ret i32 %val
+}
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/dangling-block-address.ll b/llvm/test/Transforms/Attributor/IPConstantProp/dangling-block-address.ll
new file mode 100644
index 000000000..679067650
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/dangling-block-address.ll
@@ -0,0 +1,43 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; RUN: opt -S -passes='internalize,attributor' -aa-pipeline='basic-aa' -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=1 < %s | FileCheck %s
+; PR5569
+
+; IPSCCP should prove that the blocks are dead and delete them, and
+; properly handle the dangling blockaddress constants.
+
+; CHECK: @bar.l = internal constant [2 x i8*] [i8* inttoptr (i32 1 to i8*), i8* inttoptr (i32 1 to i8*)]
+
+@code = global [5 x i32] [i32 0, i32 0, i32 0, i32 0, i32 1], align 4 ; <[5 x i32]*> [#uses=0]
+@bar.l = internal constant [2 x i8*] [i8* blockaddress(@bar, %lab0), i8* blockaddress(@bar, %end)] ; <[2 x i8*]*> [#uses=1]
+
+define void @foo(i32 %x) nounwind readnone {
+entry:
+  %b = alloca i32, align 4                        ; <i32*> [#uses=1]
+  store volatile i32 -1, i32* %b
+  ret void
+}
+
+define void @bar(i32* nocapture %pc) nounwind readonly {
+entry:
+  br label %indirectgoto
+
+lab0:                                             ; preds = %indirectgoto
+  %indvar.next = add i32 %indvar, 1               ; <i32> [#uses=1]
+  br label %indirectgoto
+
+end:                                              ; preds = %indirectgoto
+  ret void
+
+indirectgoto:                                     ; preds = %lab0, %entry
+  %indvar = phi i32 [ %indvar.next, %lab0 ], [ 0, %entry ] ; <i32> [#uses=2]
+  %pc.addr.0 = getelementptr i32, i32* %pc, i32 %indvar ; <i32*> [#uses=1]
+  %tmp1.pn = load i32, i32* %pc.addr.0                 ; <i32> [#uses=1]
+  %indirect.goto.dest.in = getelementptr inbounds [2 x i8*], [2 x i8*]* @bar.l, i32 0, i32 %tmp1.pn ; <i8**> [#uses=1]
+  %indirect.goto.dest = load i8*, i8** %indirect.goto.dest.in ; <i8*> [#uses=1]
+  indirectbr i8* %indirect.goto.dest, [label %lab0, label %end]
+}
+
+define i32 @main() nounwind readnone {
+entry:
+  ret i32 0
+}
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/deadarg.ll b/llvm/test/Transforms/Attributor/IPConstantProp/deadarg.ll
new file mode 100644
index 000000000..c84f4dee2
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/deadarg.ll
@@ -0,0 +1,7 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; RUN: opt -passes=attributor -aa-pipeline='basic-aa' -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=1 -disable-output < %s
+define internal void @foo(i32 %X) {
+  call void @foo( i32 %X )
+  ret void
+}
+
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/fp-bc-icmp-const-fold.ll b/llvm/test/Transforms/Attributor/IPConstantProp/fp-bc-icmp-const-fold.ll
new file mode 100644
index 000000000..8d8dd00cf
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/fp-bc-icmp-const-fold.ll
@@ -0,0 +1,78 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; RUN: opt -S -passes=attributor -aa-pipeline='basic-aa' -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=1 < %s | FileCheck %s
+target datalayout = "E-m:e-i64:64-n32:64"
+target triple = "powerpc64-bgq-linux"
+
+define void @test(i32 signext %n) {
+; CHECK-LABEL: define {{[^@]+}}@test
+; CHECK-SAME: (i32 signext [[N:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    unreachable
+; CHECK:       if.then:
+; CHECK-NEXT:    unreachable
+; CHECK:       if.end:
+; CHECK-NEXT:    unreachable
+; CHECK:       if.then2:
+; CHECK-NEXT:    unreachable
+; CHECK:       if.end4:
+; CHECK-NEXT:    unreachable
+; CHECK:       if.then9:
+; CHECK-NEXT:    unreachable
+; CHECK:       if.then12:
+; CHECK-NEXT:    unreachable
+; CHECK:       if.else14:
+; CHECK-NEXT:    unreachable
+; CHECK:       do.body:
+; CHECK-NEXT:    unreachable
+; CHECK:       if.then33:
+; CHECK-NEXT:    unreachable
+; CHECK:       cond.false.i28:
+; CHECK-NEXT:    unreachable
+; CHECK:       _ZN5boost4math4signIgEEiRKT_.exit30:
+; CHECK-NEXT:    unreachable
+;
+
+entry:
+  br i1 undef, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  ret void
+
+if.end:                                           ; preds = %entry
+  br i1 undef, label %if.then2, label %if.end4
+
+if.then2:                                         ; preds = %if.end
+  unreachable
+
+if.end4:                                          ; preds = %if.end
+  %sub.n = select i1 undef, i32 undef, i32 %n
+  switch i32 %sub.n, label %if.else14 [
+  i32 0, label %if.then9
+  i32 1, label %if.then12
+  ]
+
+if.then9:                                         ; preds = %if.end4
+  unreachable
+
+if.then12:                                        ; preds = %if.end4
+  unreachable
+
+if.else14:                                        ; preds = %if.end4
+  br label %do.body
+
+do.body:                                          ; preds = %do.body, %if.else14
+  %scale.0 = phi ppc_fp128 [ 0xM3FF00000000000000000000000000000, %if.else14 ], [ %scale.0, %do.body ]
+  br i1 undef, label %do.body, label %if.then33
+
+if.then33:                                        ; preds = %do.body
+  br i1 undef, label %_ZN5boost4math4signIgEEiRKT_.exit30, label %cond.false.i28
+
+cond.false.i28:                                   ; preds = %if.then33
+  %0 = bitcast ppc_fp128 %scale.0 to i128
+  %tobool.i26 = icmp slt i128 %0, 0
+  br label %_ZN5boost4math4signIgEEiRKT_.exit30
+
+_ZN5boost4math4signIgEEiRKT_.exit30:              ; preds = %cond.false.i28, %if.then33
+  unreachable
+}
+
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/global.ll b/llvm/test/Transforms/Attributor/IPConstantProp/global.ll
new file mode 100644
index 000000000..7b186f059
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/global.ll
@@ -0,0 +1,40 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; RUN: opt -S -passes=attributor -aa-pipeline='basic-aa' -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=1 < %s | FileCheck %s
+
+@_ZL6test1g = internal global i32 42, align 4
+
+define void @_Z7test1f1v() nounwind {
+; CHECK-LABEL: define {{[^@]+}}@_Z7test1f1v()
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP:%.*]] = load i32, i32* @_ZL6test1g, align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    store i32 0, i32* @_ZL6test1g, align 4
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %tmp = load i32, i32* @_ZL6test1g, align 4
+  %cmp = icmp eq i32 %tmp, 0
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  store i32 0, i32* @_ZL6test1g, align 4
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %entry
+  ret void
+}
+
+define i32 @_Z7test1f2v() nounwind {
+; CHECK-LABEL: define {{[^@]+}}@_Z7test1f2v()
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP:%.*]] = load i32, i32* @_ZL6test1g, align 4
+; CHECK-NEXT:    ret i32 [[TMP]]
+;
+entry:
+  %tmp = load i32, i32* @_ZL6test1g, align 4
+  ret i32 %tmp
+}
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/multiple_callbacks.ll b/llvm/test/Transforms/Attributor/IPConstantProp/multiple_callbacks.ll
new file mode 100644
index 000000000..ddb0e4430
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/multiple_callbacks.ll
@@ -0,0 +1,117 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; RUN: opt -S -passes=attributor -aa-pipeline='basic-aa' -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=1 < %s | FileCheck %s
+;
+;
+;                            /---------------------------------------|
+;                            |                /----------------------|----|
+;                            |                |                /-----|    |
+;                            V                V                V     |    |
+;    void broker(int (*cb0)(int), int (*cb1)(int), int (*cb2)(int), int, int);
+;
+;    static int cb0(int zero) {
+;      return zero;
+;    }
+;    static int cb1(int unknown) {
+;      return unknown;
+;    }
+;    static int cb2(int unknown) {
+;      cb0(0);
+;      return unknown;
+;    }
+;    static int cb3(int unknown) {
+;      return unknown;
+;    }
+;    static int cb4(int unknown) {
+;      return unknown;
+;    }
+;
+;    void foo() {
+;      cb0(0);
+;      cb3(1);
+;      broker(cb0, cb1, cb0, 0, 1);
+;      broker(cb1, cb2, cb2, 0, 1);
+;      broker(cb3, cb2, cb3, 0, 1);
+;      broker(cb4, cb4, cb4, 0, 1);
+;    }
+;
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+define internal i32 @cb0(i32 %zero) {
+; CHECK-LABEL: define {{[^@]+}}@cb0
+; CHECK-SAME: (i32 returned [[ZERO:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  ret i32 %zero
+}
+
+define internal i32 @cb1(i32 %unknown) {
+; CHECK-LABEL: define {{[^@]+}}@cb1
+; CHECK-SAME: (i32 returned [[UNKNOWN:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret i32 [[UNKNOWN]]
+;
+entry:
+  ret i32 %unknown
+}
+
+define internal i32 @cb2(i32 %unknown) {
+; CHECK-LABEL: define {{[^@]+}}@cb2
+; CHECK-SAME: (i32 returned [[UNKNOWN:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @cb0(i32 0)
+; CHECK-NEXT:    ret i32 [[UNKNOWN]]
+;
+entry:
+  %call = call i32 @cb0(i32 0)
+  ret i32 %unknown
+}
+
+define internal i32 @cb3(i32 %unknown) {
+; CHECK-LABEL: define {{[^@]+}}@cb3
+; CHECK-SAME: (i32 returned [[UNKNOWN:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret i32 [[UNKNOWN]]
+;
+entry:
+  ret i32 %unknown
+}
+
+define internal i32 @cb4(i32 %unknown) {
+; CHECK-LABEL: define {{[^@]+}}@cb4
+; CHECK-SAME: (i32 returned [[UNKNOWN:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret i32 [[UNKNOWN]]
+;
+entry:
+  ret i32 %unknown
+}
+
+define void @foo() {
+; CHECK-LABEL: define {{[^@]+}}@foo()
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @cb0(i32 0)
+; CHECK-NEXT:    [[CALL1:%.*]] = call i32 @cb3(i32 1)
+; CHECK-NEXT:    call void @broker(i32 (i32)* nonnull @cb0, i32 (i32)* nonnull @cb1, i32 (i32)* nonnull @cb0, i32 0, i32 1)
+; CHECK-NEXT:    call void @broker(i32 (i32)* nonnull @cb1, i32 (i32)* nonnull @cb2, i32 (i32)* nonnull @cb2, i32 0, i32 1)
+; CHECK-NEXT:    call void @broker(i32 (i32)* nonnull @cb3, i32 (i32)* nonnull @cb2, i32 (i32)* nonnull @cb3, i32 0, i32 1)
+; CHECK-NEXT:    call void @broker(i32 (i32)* nonnull @cb4, i32 (i32)* nonnull @cb4, i32 (i32)* nonnull @cb4, i32 0, i32 1)
+; CHECK-NEXT:    ret void
+;
+entry:
+  %call = call i32 @cb0(i32 0)
+  %call1 = call i32 @cb3(i32 1)
+  call void @broker(i32 (i32)* nonnull @cb0, i32 (i32)* nonnull @cb1, i32 (i32)* nonnull @cb0, i32 0, i32 1)
+  call void @broker(i32 (i32)* nonnull @cb1, i32 (i32)* nonnull @cb2, i32 (i32)* nonnull @cb2, i32 0, i32 1)
+  call void @broker(i32 (i32)* nonnull @cb3, i32 (i32)* nonnull @cb2, i32 (i32)* nonnull @cb3, i32 0, i32 1)
+  call void @broker(i32 (i32)* nonnull @cb4, i32 (i32)* nonnull @cb4, i32 (i32)* nonnull @cb4, i32 0, i32 1)
+  ret void
+}
+
+declare !callback !3 void @broker(i32 (i32)*, i32 (i32)*, i32 (i32)*, i32, i32)
+
+!0 = !{i64 0, i64 3, i1 false}
+!1 = !{i64 1, i64 4, i1 false}
+!2 = !{i64 2, i64 3, i1 false}
+!3 = !{!0, !2, !1}
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/musttail-call.ll b/llvm/test/Transforms/Attributor/IPConstantProp/musttail-call.ll
new file mode 100644
index 000000000..19e52931f
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/musttail-call.ll
@@ -0,0 +1,79 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; RUN: opt -S -passes=attributor -aa-pipeline='basic-aa' -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=4 < %s | FileCheck %s
+; PR36485
+; musttail call result can't be replaced with a constant, unless the call can be removed
+
+declare i32 @external()
+
+define i8* @start(i8 %v) {
+; CHECK-LABEL: define {{[^@]+}}@start
+; CHECK-SAME: (i8 [[V:%.*]])
+; CHECK-NEXT:    [[C1:%.*]] = icmp eq i8 [[V]], 0
+; CHECK-NEXT:    br i1 [[C1]], label [[TRUE:%.*]], label [[FALSE:%.*]]
+; CHECK:       true:
+; CHECK-NEXT:    [[CA:%.*]] = musttail call i8* @side_effects(i8 [[V]])
+; CHECK-NEXT:    ret i8* [[CA]]
+; CHECK:       false:
+; CHECK-NEXT:    [[C2:%.*]] = icmp eq i8 [[V]], 1
+; CHECK-NEXT:    br i1 [[C2]], label [[C2_TRUE:%.*]], label [[C2_FALSE:%.*]]
+; CHECK:       c2_true:
+; CHECK-NEXT:    [[CA1:%.*]] = musttail call i8* @no_side_effects(i8 undef)
+; CHECK-NEXT:    ret i8* [[CA1]]
+; CHECK:       c2_false:
+; CHECK-NEXT:    [[CA2:%.*]] = musttail call i8* @dont_zap_me(i8 undef)
+; CHECK-NEXT:    ret i8* [[CA2]]
+;
+  %c1 = icmp eq i8 %v, 0
+  br i1 %c1, label %true, label %false
+true:
+  ; FIXME: propagate the value information for %v
+  %ca = musttail call i8* @side_effects(i8 %v)
+  ret i8* %ca
+false:
+  %c2 = icmp eq i8 %v, 1
+  br i1 %c2, label %c2_true, label %c2_false
+c2_true:
+  %ca1 = musttail call i8* @no_side_effects(i8 %v)
+  ; FIXME: zap this call
+  ret i8* %ca1
+c2_false:
+  %ca2 = musttail call i8* @dont_zap_me(i8 %v)
+  ret i8* %ca2
+}
+
+define internal i8* @side_effects(i8 %v) {
+; CHECK-LABEL: define {{[^@]+}}@side_effects
+; CHECK-SAME: (i8 [[V:%.*]])
+; CHECK-NEXT:    [[I1:%.*]] = call i32 @external()
+; CHECK-NEXT:    [[CA:%.*]] = musttail call i8* @start(i8 [[V]])
+; CHECK-NEXT:    ret i8* [[CA]]
+;
+  %i1 = call i32 @external()
+
+  ; since this goes back to `start` the SCPP should be see that the return value
+  ; is always `null`.
+  ; The call can't be removed due to `external` call above, though.
+
+  %ca = musttail call i8* @start(i8 %v)
+
+  ; Thus the result must be returned anyway
+  ret i8* %ca
+}
+
+define internal i8* @no_side_effects(i8 %v) readonly nounwind {
+; CHECK-LABEL: define {{[^@]+}}@no_side_effects
+; CHECK-SAME: (i8 [[V:%.*]])
+; CHECK-NEXT:    ret i8* null
+;
+  ret i8* null
+}
+
+define internal i8* @dont_zap_me(i8 %v) {
+; CHECK-LABEL: define {{[^@]+}}@dont_zap_me
+; CHECK-SAME: (i8 [[V:%.*]])
+; CHECK-NEXT:    [[I1:%.*]] = call i32 @external()
+; CHECK-NEXT:    ret i8* null
+;
+  %i1 = call i32 @external()
+  ret i8* null
+}
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/naked-return.ll b/llvm/test/Transforms/Attributor/IPConstantProp/naked-return.ll
new file mode 100644
index 000000000..818f29914
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/naked-return.ll
@@ -0,0 +1,47 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; RUN: opt -S -passes=attributor -aa-pipeline='basic-aa' -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=1 < %s | FileCheck %s
+
+target datalayout = "e-m:x-p:32:32-i64:64-f80:32-n8:16:32-a:0:32-S32"
+target triple = "i686-pc-windows-msvc19.0.24215"
+
+define i32 @dipsy(i32, i32) local_unnamed_addr #0 {
+; CHECK-LABEL: define {{[^@]+}}@dipsy
+; CHECK-SAME: (i32 [[TMP0:%.*]], i32 [[TMP1:%.*]]) local_unnamed_addr
+; CHECK-NEXT:  BasicBlock0:
+; CHECK-NEXT:    call void asm "\0D\0Apushl %ebp\0D\0Amovl 8(%esp),%eax\0D\0Amovl 12(%esp), %ebp\0D\0Acalll *%eax\0D\0Apopl %ebp\0D\0Aretl\0D\0A", ""()
+; CHECK-NEXT:    ret i32 0
+;
+BasicBlock0:
+  call void asm "\0D\0Apushl %ebp\0D\0Amovl 8(%esp),%eax\0D\0Amovl 12(%esp), %ebp\0D\0Acalll *%eax\0D\0Apopl %ebp\0D\0Aretl\0D\0A", ""()
+  ret i32 0
+}
+
+define void @tinkywinky(i32, i32, i32) local_unnamed_addr #0 {
+; CHECK-LABEL: define {{[^@]+}}@tinkywinky
+; CHECK-SAME: (i32 [[TMP0:%.*]], i32 [[TMP1:%.*]], i32 [[TMP2:%.*]]) local_unnamed_addr
+; CHECK-NEXT:  BasicBlock1:
+; CHECK-NEXT:    call void asm "\0D\0A movl 12(%esp), %ebp\0D\0A movl 4(%esp), %eax\0D\0A movl 8(%esp), %esp\0D\0A jmpl *%eax\0D\0A", ""()
+; CHECK-NEXT:    ret void
+;
+BasicBlock1:
+  call void asm "\0D\0A    movl 12(%esp), %ebp\0D\0A    movl 4(%esp), %eax\0D\0A    movl 8(%esp), %esp\0D\0A    jmpl *%eax\0D\0A", ""()
+  ret void
+}
+
+define void @patatino(i32, i32, i32) local_unnamed_addr #1 {
+; CHECK-LABEL: define {{[^@]+}}@patatino
+; CHECK-SAME: (i32 [[TMP0:%.*]], i32 [[TMP1:%.*]], i32 [[TMP2:%.*]]) local_unnamed_addr
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP3:%.*]] = tail call i32 @dipsy(i32 [[TMP0]], i32 [[TMP1]])
+; CHECK-NEXT:    tail call void @tinkywinky(i32 [[TMP3]], i32 [[TMP2]], i32 [[TMP1]])
+; CHECK-NEXT:    ret void
+;
+bb:
+  %3 = tail call i32 @dipsy(i32 %0, i32 %1) #0
+; Check that we don't accidentally propagate zero.
+  tail call void @tinkywinky(i32 %3, i32 %2, i32 %1) #0
+  ret void
+}
+
+attributes #0 = { naked }
+attributes #1 = { "frame-pointer"="all" }
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/openmp_parallel_for.ll b/llvm/test/Transforms/Attributor/IPConstantProp/openmp_parallel_for.ll
new file mode 100644
index 000000000..8785eff4b
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/openmp_parallel_for.ll
@@ -0,0 +1,132 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; RUN: opt -S -passes=attributor -aa-pipeline='basic-aa' -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=1 < %s | FileCheck %s
+;
+;    void bar(int, float, double);
+;
+;    void foo(int N) {
+;      float p = 3;
+;      double q = 5;
+;      N = 7;
+;
+;    #pragma omp parallel for firstprivate(q)
+;      for (int i = 2; i < N; i++) {
+;        bar(i, p, q);
+;      }
+;    }
+;
+; Verify the constant value of q is propagated into the outlined function.
+;
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+%struct.ident_t = type { i32, i32, i32, i32, i8* }
+
+@.str = private unnamed_addr constant [23 x i8] c";unknown;unknown;0;0;;\00", align 1
+@0 = private unnamed_addr global %struct.ident_t { i32 0, i32 514, i32 0, i32 0, i8* getelementptr inbounds ([23 x i8], [23 x i8]* @.str, i32 0, i32 0) }, align 8
+@1 = private unnamed_addr global %struct.ident_t { i32 0, i32 2, i32 0, i32 0, i8* getelementptr inbounds ([23 x i8], [23 x i8]* @.str, i32 0, i32 0) }, align 8
+
+define dso_local void @foo(i32 %N) {
+; CHECK-LABEL: define {{[^@]+}}@foo
+; CHECK-SAME: (i32 [[N:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[N_ADDR:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[P:%.*]] = alloca float, align 4
+; CHECK-NEXT:    store i32 [[N]], i32* [[N_ADDR]], align 4
+; CHECK-NEXT:    store float 3.000000e+00, float* [[P]], align 4
+; CHECK-NEXT:    store i32 7, i32* [[N_ADDR]], align 4
+; CHECK-NEXT:    call void (%struct.ident_t*, i32, void (i32*, i32*, ...)*, ...) @__kmpc_fork_call(%struct.ident_t* nonnull align 8 dereferenceable(24) @1, i32 3, void (i32*, i32*, ...)* nonnull bitcast (void (i32*, i32*, i32*, float*, i64)* @.omp_outlined. to void (i32*, i32*, ...)*), i32* noalias nocapture nonnull readonly align 4 dereferenceable(4) [[N_ADDR]], float* noalias nocapture nonnull readonly align 4 dereferenceable(4) [[P]], i64 4617315517961601024)
+; CHECK-NEXT:    ret void
+;
+entry:
+  %N.addr = alloca i32, align 4
+  %p = alloca float, align 4
+  store i32 %N, i32* %N.addr, align 4
+  store float 3.000000e+00, float* %p, align 4
+  store i32 7, i32* %N.addr, align 4
+  call void (%struct.ident_t*, i32, void (i32*, i32*, ...)*, ...) @__kmpc_fork_call(%struct.ident_t* nonnull @1, i32 3, void (i32*, i32*, ...)* bitcast (void (i32*, i32*, i32*, float*, i64)* @.omp_outlined. to void (i32*, i32*, ...)*), i32* nonnull %N.addr, float* nonnull %p, i64 4617315517961601024)
+  ret void
+}
+
+define internal void @.omp_outlined.(i32* noalias %.global_tid., i32* noalias %.bound_tid., i32* dereferenceable(4) %N, float* dereferenceable(4) %p, i64 %q) {
+entry:
+  %q.addr = alloca i64, align 8
+  %.omp.lb = alloca i32, align 4
+  %.omp.ub = alloca i32, align 4
+  %.omp.stride = alloca i32, align 4
+  %.omp.is_last = alloca i32, align 4
+; CHECK: store i64 4617315517961601024, i64* %q.addr, align 8
+  store i64 %q, i64* %q.addr, align 8
+  %conv = bitcast i64* %q.addr to double*
+  %tmp = load i32, i32* %N, align 4
+  %sub3 = add nsw i32 %tmp, -3
+  %cmp = icmp sgt i32 %tmp, 2
+  br i1 %cmp, label %omp.precond.then, label %omp.precond.end
+
+omp.precond.then:                                 ; preds = %entry
+  store i32 0, i32* %.omp.lb, align 4
+  store i32 %sub3, i32* %.omp.ub, align 4
+  store i32 1, i32* %.omp.stride, align 4
+  store i32 0, i32* %.omp.is_last, align 4
+  %tmp5 = load i32, i32* %.global_tid., align 4
+  call void @__kmpc_for_static_init_4(%struct.ident_t* nonnull @0, i32 %tmp5, i32 34, i32* nonnull %.omp.is_last, i32* nonnull %.omp.lb, i32* nonnull %.omp.ub, i32* nonnull %.omp.stride, i32 1, i32 1)
+  %tmp6 = load i32, i32* %.omp.ub, align 4
+  %cmp6 = icmp sgt i32 %tmp6, %sub3
+  br i1 %cmp6, label %cond.true, label %cond.false
+
+cond.true:                                        ; preds = %omp.precond.then
+  br label %cond.end
+
+cond.false:                                       ; preds = %omp.precond.then
+  %tmp7 = load i32, i32* %.omp.ub, align 4
+  br label %cond.end
+
+cond.end:                                         ; preds = %cond.false, %cond.true
+  %cond = phi i32 [ %sub3, %cond.true ], [ %tmp7, %cond.false ]
+  store i32 %cond, i32* %.omp.ub, align 4
+  %tmp8 = load i32, i32* %.omp.lb, align 4
+  br label %omp.inner.for.cond
+
+omp.inner.for.cond:                               ; preds = %omp.inner.for.inc, %cond.end
+  %.omp.iv.0 = phi i32 [ %tmp8, %cond.end ], [ %add11, %omp.inner.for.inc ]
+  %tmp9 = load i32, i32* %.omp.ub, align 4
+  %cmp8 = icmp sgt i32 %.omp.iv.0, %tmp9
+  br i1 %cmp8, label %omp.inner.for.cond.cleanup, label %omp.inner.for.body
+
+omp.inner.for.cond.cleanup:                       ; preds = %omp.inner.for.cond
+  br label %omp.inner.for.end
+
+omp.inner.for.body:                               ; preds = %omp.inner.for.cond
+  %add10 = add nsw i32 %.omp.iv.0, 2
+  %tmp10 = load float, float* %p, align 4
+  %tmp11 = load double, double* %conv, align 8
+  call void @bar(i32 %add10, float %tmp10, double %tmp11)
+  br label %omp.body.continue
+
+omp.body.continue:                                ; preds = %omp.inner.for.body
+  br label %omp.inner.for.inc
+
+omp.inner.for.inc:                                ; preds = %omp.body.continue
+  %add11 = add nsw i32 %.omp.iv.0, 1
+  br label %omp.inner.for.cond
+
+omp.inner.for.end:                                ; preds = %omp.inner.for.cond.cleanup
+  br label %omp.loop.exit
+
+omp.loop.exit:                                    ; preds = %omp.inner.for.end
+  %tmp12 = load i32, i32* %.global_tid., align 4
+  call void @__kmpc_for_static_fini(%struct.ident_t* nonnull @0, i32 %tmp12)
+  br label %omp.precond.end
+
+omp.precond.end:                                  ; preds = %omp.loop.exit, %entry
+  ret void
+}
+
+declare dso_local void @__kmpc_for_static_init_4(%struct.ident_t*, i32, i32, i32*, i32*, i32*, i32*, i32, i32)
+
+declare dso_local void @bar(i32, float, double)
+
+declare dso_local void @__kmpc_for_static_fini(%struct.ident_t*, i32)
+
+declare !callback !0 dso_local void @__kmpc_fork_call(%struct.ident_t*, i32, void (i32*, i32*, ...)*, ...)
+
+!1 = !{i64 2, i64 -1, i64 -1, i1 true}
+!0 = !{!1}
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/pthreads.ll b/llvm/test/Transforms/Attributor/IPConstantProp/pthreads.ll
new file mode 100644
index 000000000..b6b5e1968
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/pthreads.ll
@@ -0,0 +1,96 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; RUN: opt -S -passes=attributor -aa-pipeline='basic-aa' -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=1 < %s | FileCheck %s
+;
+;    #include <pthread.h>
+;
+;    void *GlobalVPtr;
+;
+;    static void *foo(void *arg) { return arg; }
+;    static void *bar(void *arg) { return arg; }
+;
+;    int main() {
+;      pthread_t thread;
+;      pthread_create(&thread, NULL, foo, NULL);
+;      pthread_create(&thread, NULL, bar, &GlobalVPtr);
+;      return 0;
+;    }
+;
+; Verify the constant values NULL and &GlobalVPtr are propagated into foo and
+; bar, respectively.
+;
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+%union.pthread_attr_t = type { i64, [48 x i8] }
+
+@GlobalVPtr = common dso_local global i8* null, align 8
+
+; FIXME: nocapture & noalias for @GlobalVPtr in %call1
+; FIXME: nocapture & noalias for %alloc2 in %call3
+
+define dso_local i32 @main() {
+; CHECK-LABEL: define {{[^@]+}}@main()
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ALLOC1:%.*]] = alloca i8, align 8
+; CHECK-NEXT:    [[ALLOC2:%.*]] = alloca i8, align 8
+; CHECK-NEXT:    [[THREAD:%.*]] = alloca i64, align 8
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @pthread_create(i64* nonnull align 8 dereferenceable(8) [[THREAD]], %union.pthread_attr_t* noalias null, i8* (i8*)* nonnull @foo, i8* noalias nofree readnone null)
+; CHECK-NEXT:    [[CALL1:%.*]] = call i32 @pthread_create(i64* nonnull align 8 dereferenceable(8) [[THREAD]], %union.pthread_attr_t* noalias null, i8* (i8*)* nonnull @bar, i8* nofree nonnull readnone align 8 dereferenceable(8) bitcast (i8** @GlobalVPtr to i8*))
+; CHECK-NEXT:    [[CALL2:%.*]] = call i32 @pthread_create(i64* nonnull align 8 dereferenceable(8) [[THREAD]], %union.pthread_attr_t* noalias null, i8* (i8*)* nonnull @baz, i8* noalias nocapture nofree nonnull readnone align 8 dereferenceable(1) [[ALLOC1]])
+; CHECK-NEXT:    [[CALL3:%.*]] = call i32 @pthread_create(i64* nonnull align 8 dereferenceable(8) [[THREAD]], %union.pthread_attr_t* noalias null, i8* (i8*)* nonnull @buz, i8* nofree nonnull readnone align 8 dereferenceable(1) [[ALLOC2]])
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %alloc1 = alloca i8, align 8
+  %alloc2 = alloca i8, align 8
+  %thread = alloca i64, align 8
+  %call = call i32 @pthread_create(i64* nonnull %thread, %union.pthread_attr_t* null, i8* (i8*)* nonnull @foo, i8* null)
+  %call1 = call i32 @pthread_create(i64* nonnull %thread, %union.pthread_attr_t* null, i8* (i8*)* nonnull @bar, i8* bitcast (i8** @GlobalVPtr to i8*))
+  %call2 = call i32 @pthread_create(i64* nonnull %thread, %union.pthread_attr_t* null, i8* (i8*)* nonnull @baz, i8* nocapture %alloc1)
+  %call3 = call i32 @pthread_create(i64* nonnull %thread, %union.pthread_attr_t* null, i8* (i8*)* nonnull @buz, i8* %alloc2)
+  ret i32 0
+}
+
+declare !callback !0 dso_local i32 @pthread_create(i64*, %union.pthread_attr_t*, i8* (i8*)*, i8*)
+
+define internal i8* @foo(i8* %arg) {
+; CHECK-LABEL: define {{[^@]+}}@foo
+; CHECK-SAME: (i8* noalias nofree readnone returned [[ARG:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret i8* null
+;
+entry:
+  ret i8* %arg
+}
+
+define internal i8* @bar(i8* %arg) {
+; CHECK-LABEL: define {{[^@]+}}@bar
+; CHECK-SAME: (i8* nofree nonnull readnone returned align 8 dereferenceable(8) [[ARG:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret i8* bitcast (i8** @GlobalVPtr to i8*)
+;
+entry:
+  ret i8* %arg
+}
+
+define internal i8* @baz(i8* %arg) {
+; CHECK-LABEL: define {{[^@]+}}@baz
+; CHECK-SAME: (i8* noalias nofree nonnull readnone returned align 8 dereferenceable(1) [[ARG:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret i8* [[ARG]]
+;
+entry:
+  ret i8* %arg
+}
+
+define internal i8* @buz(i8* %arg) {
+; CHECK-LABEL: define {{[^@]+}}@buz
+; CHECK-SAME: (i8* nofree nonnull readnone returned align 8 dereferenceable(1) [[ARG:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret i8* [[ARG]]
+;
+entry:
+  ret i8* %arg
+}
+
+!1 = !{i64 2, i64 3, i1 false}
+!0 = !{!1}
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/recursion.ll b/llvm/test/Transforms/Attributor/IPConstantProp/recursion.ll
new file mode 100644
index 000000000..fc82342a9
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/recursion.ll
@@ -0,0 +1,19 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; RUN: opt -S -passes=attributor -aa-pipeline='basic-aa' -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=2 < %s | FileCheck %s
+
+; CHECK-NOT: %X
+
+define internal i32 @foo(i32 %X) {
+  %Y = call i32 @foo( i32 %X )            ; <i32> [#uses=1]
+  %Z = add i32 %Y, 1              ; <i32> [#uses=1]
+  ret i32 %Z
+}
+
+define void @bar() {
+; CHECK-LABEL: define {{[^@]+}}@bar()
+; CHECK-NEXT:    unreachable
+;
+  call i32 @foo( i32 17 )         ; <i32>:1 [#uses=0]
+  ret void
+}
+
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/remove-call-inst.ll b/llvm/test/Transforms/Attributor/IPConstantProp/remove-call-inst.ll
new file mode 100644
index 000000000..3e0b575f9
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/remove-call-inst.ll
@@ -0,0 +1,45 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; RUN: opt -S -passes=attributor -aa-pipeline='basic-aa' -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=2 < %s | FileCheck %s
+; PR5596
+
+; IPSCCP should propagate the 0 argument, eliminate the switch, and propagate
+; the result.
+
+; FIXME: Remove obsolete calls/instructions
+
+define i32 @main() noreturn nounwind {
+; CHECK-LABEL: define {{[^@]+}}@main()
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CALL2:%.*]] = tail call i32 @wwrite(i64 0)
+; CHECK-NEXT:    ret i32 123
+;
+entry:
+  %call2 = tail call i32 @wwrite(i64 0) nounwind
+  ret i32 %call2
+}
+
+define internal i32 @wwrite(i64 %i) nounwind readnone {
+; CHECK-LABEL: define {{[^@]+}}@wwrite
+; CHECK-SAME: (i64 [[I:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    switch i64 0, label [[SW_DEFAULT:%.*]] [
+; CHECK-NEXT:    i64 3, label [[RETURN:%.*]]
+; CHECK-NEXT:    i64 10, label [[RETURN]]
+; CHECK-NEXT:    ]
+; CHECK:       sw.default:
+; CHECK-NEXT:    ret i32 123
+; CHECK:       return:
+; CHECK-NEXT:    unreachable
+;
+entry:
+  switch i64 %i, label %sw.default [
+  i64 3, label %return
+  i64 10, label %return
+  ]
+
+sw.default:
+  ret i32 123
+
+return:
+  ret i32 0
+}
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/return-argument.ll b/llvm/test/Transforms/Attributor/IPConstantProp/return-argument.ll
new file mode 100644
index 000000000..f89da0304
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/return-argument.ll
@@ -0,0 +1,96 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; RUN: opt -S -passes=attributor -aa-pipeline='basic-aa' -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=9 < %s | FileCheck %s
+
+;; This function returns its second argument on all return statements
+define internal i32* @incdec(i1 %C, i32* %V) {
+; CHECK-LABEL: define {{[^@]+}}@incdec
+; CHECK-SAME: (i1 [[C:%.*]], i32* noalias nofree nonnull returned align 4 dereferenceable(4) [[V:%.*]])
+; CHECK-NEXT:    [[X:%.*]] = load i32, i32* [[V]], align 4
+; CHECK-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]]
+; CHECK:       T:
+; CHECK-NEXT:    [[X1:%.*]] = add i32 [[X]], 1
+; CHECK-NEXT:    store i32 [[X1]], i32* [[V]], align 4
+; CHECK-NEXT:    ret i32* [[V]]
+; CHECK:       F:
+; CHECK-NEXT:    [[X2:%.*]] = sub i32 [[X]], 1
+; CHECK-NEXT:    store i32 [[X2]], i32* [[V]], align 4
+; CHECK-NEXT:    ret i32* [[V]]
+;
+  %X = load i32, i32* %V
+  br i1 %C, label %T, label %F
+
+T:              ; preds = %0
+  %X1 = add i32 %X, 1
+  store i32 %X1, i32* %V
+  ret i32* %V
+
+F:              ; preds = %0
+  %X2 = sub i32 %X, 1
+  store i32 %X2, i32* %V
+  ret i32* %V
+}
+
+;; This function returns its first argument as a part of a multiple return
+;; value
+define internal { i32, i32 } @foo(i32 %A, i32 %B) {
+; CHECK-LABEL: define {{[^@]+}}@foo
+; CHECK-SAME: (i32 [[A:%.*]], i32 [[B:%.*]])
+; CHECK-NEXT:    [[X:%.*]] = add i32 [[A]], [[B]]
+; CHECK-NEXT:    [[Y:%.*]] = insertvalue { i32, i32 } undef, i32 [[A]], 0
+; CHECK-NEXT:    [[Z:%.*]] = insertvalue { i32, i32 } [[Y]], i32 [[X]], 1
+; CHECK-NEXT:    ret { i32, i32 } [[Z]]
+;
+  %X = add i32 %A, %B
+  %Y = insertvalue { i32, i32 } undef, i32 %A, 0
+  %Z = insertvalue { i32, i32 } %Y, i32 %X, 1
+  ret { i32, i32 } %Z
+}
+
+define void @caller(i1 %C) personality i32 (...)* @__gxx_personality_v0 {
+; CHECK-LABEL: define {{[^@]+}}@caller
+; CHECK-SAME: (i1 [[C:%.*]]) #2 personality i32 (...)* @__gxx_personality_v0
+; CHECK-NEXT:    [[Q:%.*]] = alloca i32
+; CHECK-NEXT:    [[W:%.*]] = call align 4 i32* @incdec(i1 [[C]], i32* noalias nofree nonnull align 4 dereferenceable(4) [[Q]])
+; CHECK-NEXT:    [[S1:%.*]] = call { i32, i32 } @foo(i32 1, i32 2)
+; CHECK-NEXT:    [[X1:%.*]] = extractvalue { i32, i32 } [[S1]], 0
+; CHECK-NEXT:    [[S2:%.*]] = invoke { i32, i32 } @foo(i32 3, i32 4)
+; CHECK-NEXT:    to label [[OK:%.*]] unwind label [[LPAD:%.*]]
+; CHECK:       OK:
+; CHECK-NEXT:    [[X2:%.*]] = extractvalue { i32, i32 } [[S2]], 0
+; CHECK-NEXT:    [[Z:%.*]] = add i32 [[X1]], [[X2]]
+; CHECK-NEXT:    store i32 [[Z]], i32* [[W]], align 4
+; CHECK-NEXT:    br label [[RET:%.*]]
+; CHECK:       LPAD:
+; CHECK-NEXT:    [[EXN:%.*]] = landingpad { i8*, i32 }
+; CHECK-NEXT:    cleanup
+; CHECK-NEXT:    br label [[RET]]
+; CHECK:       RET:
+; CHECK-NEXT:    ret void
+;
+  %Q = alloca i32
+  ;; Call incdec to see if %W is properly replaced by %Q
+  %W = call i32* @incdec(i1 %C, i32* %Q )             ; <i32> [#uses=1]
+  ;; Call @foo twice, to prevent the arguments from propagating into the
+  ;; function (so we can check the returned argument is properly
+  ;; propagated per-caller).
+  %S1 = call { i32, i32 } @foo(i32 1, i32 2)
+  %X1 = extractvalue { i32, i32 } %S1, 0
+  %S2 = invoke { i32, i32 } @foo(i32 3, i32 4) to label %OK unwind label %LPAD
+
+OK:
+  %X2 = extractvalue { i32, i32 } %S2, 0
+  ;; Do some stuff with the returned values which we can grep for
+  %Z  = add i32 %X1, %X2
+  store i32 %Z, i32* %W
+  br label %RET
+
+LPAD:
+  %exn = landingpad {i8*, i32}
+  cleanup
+  br label %RET
+
+RET:
+  ret void
+}
+
+declare i32 @__gxx_personality_v0(...)
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/return-constant.ll b/llvm/test/Transforms/Attributor/IPConstantProp/return-constant.ll
new file mode 100644
index 000000000..4bc1b2557
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/return-constant.ll
@@ -0,0 +1,55 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; RUN: opt -S -passes=attributor -aa-pipeline='basic-aa' -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=1 < %s | FileCheck %s
+
+; FIXME: icmp folding is missing
+
+define i1 @invokecaller(i1 %C) personality i32 (...)* @__gxx_personality_v0 {
+; CHECK-LABEL: define {{[^@]+}}@invokecaller
+; CHECK-SAME: (i1 [[C:%.*]]) #0 personality i32 (...)* @__gxx_personality_v0
+; CHECK-NEXT:    [[X:%.*]] = call i32 @foo(i1 [[C]])
+; CHECK-NEXT:    br label [[OK:%.*]]
+; CHECK:       OK:
+; CHECK-NEXT:    ret i1 true
+; CHECK:       FAIL:
+; CHECK-NEXT:    unreachable
+;
+  %X = invoke i32 @foo( i1 %C ) to label %OK unwind label %FAIL             ; <i32> [#uses=1]
+OK:
+  %Y = icmp ne i32 %X, 0          ; <i1> [#uses=1]
+  ret i1 %Y
+FAIL:
+  %exn = landingpad {i8*, i32}
+  cleanup
+  ret i1 false
+}
+
+define internal i32 @foo(i1 %C) {
+; CHECK-LABEL: define {{[^@]+}}@foo
+; CHECK-SAME: (i1 [[C:%.*]])
+; CHECK-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]]
+; CHECK:       T:
+; CHECK-NEXT:    ret i32 52
+; CHECK:       F:
+; CHECK-NEXT:    ret i32 52
+;
+  br i1 %C, label %T, label %F
+
+T:              ; preds = %0
+  ret i32 52
+
+F:              ; preds = %0
+  ret i32 52
+}
+
+define i1 @caller(i1 %C) {
+; CHECK-LABEL: define {{[^@]+}}@caller
+; CHECK-SAME: (i1 [[C:%.*]])
+; CHECK-NEXT:    [[X:%.*]] = call i32 @foo(i1 [[C]])
+; CHECK-NEXT:    ret i1 true
+;
+  %X = call i32 @foo( i1 %C )             ; <i32> [#uses=1]
+  %Y = icmp ne i32 %X, 0          ; <i1> [#uses=1]
+  ret i1 %Y
+}
+
+declare i32 @__gxx_personality_v0(...)
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/return-constants.ll b/llvm/test/Transforms/Attributor/IPConstantProp/return-constants.ll
new file mode 100644
index 000000000..a2be8c073
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/return-constants.ll
@@ -0,0 +1,69 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; RUN: opt -S -passes=attributor -aa-pipeline='basic-aa' -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=7 < %s | FileCheck %s
+
+;; FIXME: support for extractvalue and insertvalue missing.
+
+%0 = type { i32, i32 }
+
+define internal %0 @foo(i1 %Q) {
+; CHECK-LABEL: define {{[^@]+}}@foo
+; CHECK-SAME: (i1 [[Q:%.*]])
+; CHECK-NEXT:    br i1 [[Q]], label [[T:%.*]], label [[F:%.*]]
+; CHECK:       T:
+; CHECK-NEXT:    [[MRV:%.*]] = insertvalue [[TMP0:%.*]] undef, i32 21, 0
+; CHECK-NEXT:    [[MRV1:%.*]] = insertvalue [[TMP0]] %mrv, i32 22, 1
+; CHECK-NEXT:    ret [[TMP0]] %mrv1
+; CHECK:       F:
+; CHECK-NEXT:    [[MRV2:%.*]] = insertvalue [[TMP0]] undef, i32 21, 0
+; CHECK-NEXT:    [[MRV3:%.*]] = insertvalue [[TMP0]] %mrv2, i32 23, 1
+; CHECK-NEXT:    ret [[TMP0]] %mrv3
+;
+  br i1 %Q, label %T, label %F
+
+T:                                                ; preds = %0
+  %mrv = insertvalue %0 undef, i32 21, 0
+  %mrv1 = insertvalue %0 %mrv, i32 22, 1
+  ret %0 %mrv1
+
+F:                                                ; preds = %0
+  %mrv2 = insertvalue %0 undef, i32 21, 0
+  %mrv3 = insertvalue %0 %mrv2, i32 23, 1
+  ret %0 %mrv3
+}
+
+define internal %0 @bar(i1 %Q) {
+  %A = insertvalue %0 undef, i32 21, 0
+  br i1 %Q, label %T, label %F
+
+T:                                                ; preds = %0
+  %B = insertvalue %0 %A, i32 22, 1
+  ret %0 %B
+
+F:                                                ; preds = %0
+  %C = insertvalue %0 %A, i32 23, 1
+  ret %0 %C
+}
+
+define %0 @caller(i1 %Q) {
+; CHECK-LABEL: define {{[^@]+}}@caller
+; CHECK-SAME: (i1 [[Q:%.*]])
+; CHECK-NEXT:    [[X:%.*]] = call [[TMP0:%.*]] @foo(i1 [[Q]])
+; CHECK-NEXT:    [[A:%.*]] = extractvalue [[TMP0]] %X, 0
+; CHECK-NEXT:    [[B:%.*]] = extractvalue [[TMP0]] %X, 1
+; CHECK-NEXT:    [[C:%.*]] = extractvalue [[TMP0]] undef, 0
+; CHECK-NEXT:    [[D:%.*]] = extractvalue [[TMP0]] undef, 1
+; CHECK-NEXT:    [[M:%.*]] = add i32 [[A]], [[C]]
+; CHECK-NEXT:    [[N:%.*]] = add i32 [[B]], [[D]]
+; CHECK-NEXT:    ret [[TMP0]] %X
+;
+  %X = call %0 @foo(i1 %Q)
+  %A = extractvalue %0 %X, 0
+  %B = extractvalue %0 %X, 1
+  %Y = call %0 @bar(i1 %Q)
+  %C = extractvalue %0 %Y, 0
+  %D = extractvalue %0 %Y, 1
+  %M = add i32 %A, %C
+;; Check that the second return values didn't get propagated
+  %N = add i32 %B, %D
+  ret %0 %X
+}
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/solve-after-each-resolving-undefs-for-function.ll b/llvm/test/Transforms/Attributor/IPConstantProp/solve-after-each-resolving-undefs-for-function.ll
new file mode 100644
index 000000000..ff33eccb2
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/solve-after-each-resolving-undefs-for-function.ll
@@ -0,0 +1,65 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; RUN: opt -S -passes=attributor -aa-pipeline='basic-aa' -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=2 < %s | FileCheck %s
+
+define internal i32 @testf(i1 %c) {
+; CHECK-LABEL: define {{[^@]+}}@testf
+; CHECK-SAME: (i1 [[C:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[C]], label [[IF_COND:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.cond:
+; CHECK-NEXT:    unreachable
+; CHECK:       if.then:
+; CHECK-NEXT:    unreachable
+; CHECK:       if.end:
+; CHECK-NEXT:    ret i32 10
+;
+entry:
+  br i1 %c, label %if.cond, label %if.end
+
+if.cond:
+  br i1 undef, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry, %if.then
+  ret i32 11
+
+if.end:                                          ; preds = %if.then1, %entry
+  ret i32 10
+}
+
+define internal i32 @test1(i1 %c) {
+; CHECK-LABEL: define {{[^@]+}}@test1
+; CHECK-SAME: (i1 [[C:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[IF_THEN:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @testf(i1 [[C]])
+; CHECK-NEXT:    br label [[RET1:%.*]]
+; CHECK:       ret1:
+; CHECK-NEXT:    ret i32 99
+; CHECK:       ret2:
+; CHECK-NEXT:    unreachable
+;
+entry:
+  br label %if.then
+
+if.then:                                          ; preds = %entry, %if.then
+  %call = call i32 @testf(i1 %c)
+  %res = icmp eq i32 %call, 10
+  br i1 %res, label %ret1, label %ret2
+
+ret1:                                           ; preds = %if.then, %entry
+  ret i32 99
+
+ret2:                                           ; preds = %if.then, %entry
+  ret i32 0
+}
+
+define i32 @main(i1 %c) {
+; CHECK-LABEL: define {{[^@]+}}@main
+; CHECK-SAME: (i1 [[C:%.*]])
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @test1(i1 [[C]])
+; CHECK-NEXT:    ret i32 99
+;
+  %res = call i32 @test1(i1 %c)
+  ret i32 %res
+}
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/thread_local_acs.ll b/llvm/test/Transforms/Attributor/IPConstantProp/thread_local_acs.ll
new file mode 100644
index 000000000..49a344773
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/thread_local_acs.ll
@@ -0,0 +1,53 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; RUN: opt -S -passes=attributor -aa-pipeline='basic-aa' -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=1 < %s | FileCheck %s
+;
+;    #include <threads.h>
+;    thread_local int gtl = 0;
+;    int gsh = 0;
+;
+;    static int callee(int *thread_local_ptr, int *shared_ptr) {
+;      return *thread_local_ptr + *shared_ptr;
+;    }
+;
+;    void broker(int *, int (*callee)(int *, int *), int *);
+;
+;    void caller() {
+;      broker(&gtl, callee, &gsh);
+;    }
+;
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+@gtl = dso_local thread_local global i32 0, align 4
+@gsh = dso_local global i32 0, align 4
+
+define internal i32 @callee(i32* %thread_local_ptr, i32* %shared_ptr) {
+; CHECK-LABEL: define {{[^@]+}}@callee
+; CHECK-SAME: (i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[THREAD_LOCAL_PTR:%.*]], i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[SHARED_PTR:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP:%.*]] = load i32, i32* [[THREAD_LOCAL_PTR]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* @gsh, align 4
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP]], [[TMP1]]
+; CHECK-NEXT:    ret i32 [[ADD]]
+;
+entry:
+  %tmp = load i32, i32* %thread_local_ptr, align 4
+  %tmp1 = load i32, i32* %shared_ptr, align 4
+  %add = add nsw i32 %tmp, %tmp1
+  ret i32 %add
+}
+
+define dso_local void @caller() {
+; CHECK-LABEL: define {{[^@]+}}@caller()
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    call void @broker(i32* nofree nonnull readonly align 4 dereferenceable(4) @gtl, i32 (i32*, i32*)* nonnull @callee, i32* nofree nonnull readonly align 4 dereferenceable(4) @gsh)
+; CHECK-NEXT:    ret void
+;
+entry:
+  call void @broker(i32* nonnull @gtl, i32 (i32*, i32*)* nonnull @callee, i32* nonnull @gsh)
+  ret void
+}
+
+declare !callback !0 dso_local void @broker(i32*, i32 (i32*, i32*)*, i32*)
+
+!1 = !{i64 1, i64 0, i64 2, i1 false}
+!0 = !{!1}
diff --git a/llvm/test/Transforms/Attributor/align.ll b/llvm/test/Transforms/Attributor/align.ll
new file mode 100644
index 000000000..f01ae2843
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/align.ll
@@ -0,0 +1,412 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --turn off
+; RUN: opt -attributor -attributor-manifest-internal -attributor-disable=false -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=5 -S < %s | FileCheck %s --check-prefix=ATTRIBUTOR
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+; Test cases specifically designed for "align" attribute.
+; We use FIXME's to indicate problems and missing attributes.
+
+
+; TEST 1
+; ATTRIBUTOR: define align 8 i32* @test1(i32* nofree readnone returned align 8 "no-capture-maybe-returned" %0)
+define i32* @test1(i32* align 8 %0) #0 {
+  ret i32* %0
+}
+
+; TEST 2
+; ATTRIBUTOR: define i32* @test2(i32* nofree readnone returned "no-capture-maybe-returned" %0)
+define i32* @test2(i32* %0) #0 {
+  ret i32* %0
+}
+
+; TEST 3
+; ATTRIBUTOR: define align 4 i32* @test3(i32* nofree readnone align 8 "no-capture-maybe-returned" %0, i32* nofree readnone align 4 "no-capture-maybe-returned" %1, i1 %2)
+define i32* @test3(i32* align 8 %0, i32* align 4 %1, i1 %2) #0 {
+  %ret = select i1 %2, i32* %0, i32* %1
+  ret i32* %ret
+}
+
+; TEST 4
+; ATTRIBUTOR: define align 32 i32* @test4(i32* nofree readnone align 32 "no-capture-maybe-returned" %0, i32* nofree readnone align 32 "no-capture-maybe-returned" %1, i1 %2)
+define i32* @test4(i32* align 32 %0, i32* align 32 %1, i1 %2) #0 {
+  %ret = select i1 %2, i32* %0, i32* %1
+  ret i32* %ret
+}
+
+; TEST 5
+declare i32* @unknown()
+declare align 8 i32* @align8()
+
+
+; ATTRIBUTOR: define align 8 i32* @test5_1()
+define i32* @test5_1() {
+  %ret = tail call align 8 i32* @unknown()
+  ret i32* %ret
+}
+
+; ATTRIBUTOR: define align 8 i32* @test5_2()
+define i32* @test5_2() {
+  %ret = tail call i32* @align8()
+  ret i32* %ret
+}
+
+; TEST 6
+; SCC
+; ATTRIBUTOR: define noalias nonnull align 536870912 dereferenceable(4294967295) i32* @test6_1()
+define i32* @test6_1() #0 {
+  %ret = tail call i32* @test6_2()
+  ret i32* %ret
+}
+
+; ATTRIBUTOR: define noalias nonnull align 536870912 dereferenceable(4294967295) i32* @test6_2()
+define i32* @test6_2() #0 {
+  %ret = tail call i32* @test6_1()
+  ret i32* %ret
+}
+
+
+; char a1 __attribute__((aligned(8)));
+; char a2 __attribute__((aligned(16)));
+;
+; char* f1(char* a ){
+;     return a?a:f2(&a1);
+; }
+; char* f2(char* a){
+;     return a?f1(a):f3(&a2);
+; }
+;
+; char* f3(char* a){
+;     return a?&a1: f1(&a2);
+; }
+
+@a1 = common global i8 0, align 8
+@a2 = common global i8 0, align 16
+
+; Function Attrs: nounwind readnone ssp uwtable
+define internal i8* @f1(i8* readnone %0) local_unnamed_addr #0 {
+  %2 = icmp eq i8* %0, null
+  br i1 %2, label %3, label %5
+
+; <label>:3:                                      ; preds = %1
+  %4 = tail call i8* @f2(i8* nonnull @a1)
+  %l = load i8, i8* %4
+  br label %5
+
+; <label>:5:                                      ; preds = %1, %3
+  %6 = phi i8* [ %4, %3 ], [ %0, %1 ]
+  ret i8* %6
+}
+
+; Function Attrs: nounwind readnone ssp uwtable
+define internal i8* @f2(i8* readnone %0) local_unnamed_addr #0 {
+  %2 = icmp eq i8* %0, null
+  br i1 %2, label %5, label %3
+
+; <label>:3:                                      ; preds = %1
+
+  %4 = tail call i8* @f1(i8* nonnull %0)
+  br label %7
+
+; <label>:5:                                      ; preds = %1
+  %6 = tail call i8* @f3(i8* nonnull @a2)
+  br label %7
+
+; <label>:7:                                      ; preds = %5, %3
+  %8 = phi i8* [ %4, %3 ], [ %6, %5 ]
+  ret i8* %8
+}
+
+; Function Attrs: nounwind readnone ssp uwtable
+define internal i8* @f3(i8* readnone %0) local_unnamed_addr #0 {
+  %2 = icmp eq i8* %0, null
+  br i1 %2, label %3, label %5
+
+; <label>:3:                                      ; preds = %1
+  %4 = tail call i8* @f1(i8* nonnull @a2)
+  br label %5
+
+; <label>:5:                                      ; preds = %1, %3
+  %6 = phi i8* [ %4, %3 ], [ @a1, %1 ]
+  ret i8* %6
+}
+
+; UTC_ARGS: --turn=on
+
+; TEST 7
+; Better than IR information
+define align 4 i32* @test7(i32* align 32 %p) #0 {
+; ATTRIBUTOR-LABEL: define {{[^@]+}}@test7
+; ATTRIBUTOR-SAME: (i32* nofree readnone returned align 32 "no-capture-maybe-returned" [[P:%.*]])
+; ATTRIBUTOR-NEXT:    ret i32* [[P]]
+;
+  tail call i8* @f1(i8* align 8 dereferenceable(1) @a1)
+  ret i32* %p
+}
+
+; TEST 7b
+; Function Attrs: nounwind readnone ssp uwtable
+define internal i8* @f1b(i8* readnone %0) local_unnamed_addr #0 {
+; ATTRIBUTOR-LABEL: define {{[^@]+}}@f1b
+; ATTRIBUTOR-SAME: (i8* nofree nonnull readnone align 8 dereferenceable(1) "no-capture-maybe-returned" [[TMP0:%.*]]) local_unnamed_addr
+; ATTRIBUTOR-NEXT:    [[TMP2:%.*]] = icmp eq i8* [[TMP0]], null
+; ATTRIBUTOR-NEXT:    br i1 [[TMP2]], label [[TMP3:%.*]], label [[TMP5:%.*]]
+; ATTRIBUTOR:       3:
+; ATTRIBUTOR-NEXT:    [[TMP4:%.*]] = tail call align 8 i8* @f2b(i8* nofree nonnull readnone align 8 dereferenceable(1) @a1)
+; ATTRIBUTOR-NEXT:    [[L:%.*]] = load i8, i8* [[TMP4]], align 8
+; ATTRIBUTOR-NEXT:    store i8 [[L]], i8* @a1, align 8
+; ATTRIBUTOR-NEXT:    br label [[TMP5]]
+; ATTRIBUTOR:       5:
+; ATTRIBUTOR-NEXT:    [[TMP6:%.*]] = phi i8* [ [[TMP4]], [[TMP3]] ], [ [[TMP0]], [[TMP1:%.*]] ]
+; ATTRIBUTOR-NEXT:    ret i8* [[TMP6]]
+;
+  %2 = icmp eq i8* %0, null
+  br i1 %2, label %3, label %5
+
+; <label>:3:                                      ; preds = %1
+  %4 = tail call i8* @f2b(i8* nonnull @a1)
+  %l = load i8, i8* %4
+  store i8 %l, i8* @a1
+  br label %5
+
+; <label>:5:                                      ; preds = %1, %3
+  %6 = phi i8* [ %4, %3 ], [ %0, %1 ]
+  ret i8* %6
+}
+
+; Function Attrs: nounwind readnone ssp uwtable
+define internal i8* @f2b(i8* readnone %0) local_unnamed_addr #0 {
+;
+; ATTRIBUTOR-LABEL: define {{[^@]+}}@f2b
+; ATTRIBUTOR-SAME: (i8* nofree nonnull readnone align 8 dereferenceable(1) "no-capture-maybe-returned" [[TMP0:%.*]]) local_unnamed_addr
+; ATTRIBUTOR-NEXT:    [[TMP2:%.*]] = icmp eq i8* @a1, null
+; ATTRIBUTOR-NEXT:    br i1 [[TMP2]], label [[TMP5:%.*]], label [[TMP3:%.*]]
+; ATTRIBUTOR:       3:
+; ATTRIBUTOR-NEXT:    [[TMP4:%.*]] = tail call i8* @f1b(i8* nofree nonnull readnone align 8 dereferenceable(1) "no-capture-maybe-returned" @a1)
+; ATTRIBUTOR-NEXT:    br label [[TMP7:%.*]]
+; ATTRIBUTOR:       5:
+; ATTRIBUTOR-NEXT:    [[TMP6:%.*]] = tail call i8* @f3b(i8* nofree nonnull readnone align 16 dereferenceable(1) @a2)
+; ATTRIBUTOR-NEXT:    br label [[TMP7]]
+; ATTRIBUTOR:       7:
+; ATTRIBUTOR-NEXT:    [[TMP8:%.*]] = phi i8* [ [[TMP4]], [[TMP3]] ], [ [[TMP6]], [[TMP5]] ]
+; ATTRIBUTOR-NEXT:    ret i8* [[TMP8]]
+;
+  %2 = icmp eq i8* %0, null
+  br i1 %2, label %5, label %3
+
+; <label>:3:                                      ; preds = %1
+
+  %4 = tail call i8* @f1b(i8* nonnull %0)
+  br label %7
+
+; <label>:5:                                      ; preds = %1
+  %6 = tail call i8* @f3b(i8* nonnull @a2)
+  br label %7
+
+; <label>:7:                                      ; preds = %5, %3
+  %8 = phi i8* [ %4, %3 ], [ %6, %5 ]
+  ret i8* %8
+}
+
+; Function Attrs: nounwind readnone ssp uwtable
+define internal i8* @f3b(i8* readnone %0) local_unnamed_addr #0 {
+;
+; ATTRIBUTOR-LABEL: define {{[^@]+}}@f3b
+; ATTRIBUTOR-SAME: (i8* nocapture nofree nonnull readnone align 16 dereferenceable(1) [[TMP0:%.*]]) local_unnamed_addr
+; ATTRIBUTOR-NEXT:    [[TMP2:%.*]] = icmp eq i8* @a2, null
+; ATTRIBUTOR-NEXT:    br i1 [[TMP2]], label [[TMP3:%.*]], label [[TMP5:%.*]]
+; ATTRIBUTOR:       3:
+; ATTRIBUTOR-NEXT:    [[TMP4:%.*]] = tail call i8* @f1b(i8* nofree nonnull readnone align 16 dereferenceable(1) @a2)
+; ATTRIBUTOR-NEXT:    br label [[TMP5]]
+; ATTRIBUTOR:       5:
+; ATTRIBUTOR-NEXT:    [[TMP6:%.*]] = phi i8* [ [[TMP4]], [[TMP3]] ], [ @a1, [[TMP1:%.*]] ]
+; ATTRIBUTOR-NEXT:    ret i8* [[TMP6]]
+;
+  %2 = icmp eq i8* %0, null
+  br i1 %2, label %3, label %5
+
+; <label>:3:                                      ; preds = %1
+  %4 = tail call i8* @f1b(i8* nonnull @a2)
+  br label %5
+
+; <label>:5:                                      ; preds = %1, %3
+  %6 = phi i8* [ %4, %3 ], [ @a1, %1 ]
+  ret i8* %6
+}
+
+define align 4 i32* @test7b(i32* align 32 %p) #0 {
+; ATTRIBUTOR-LABEL: define {{[^@]+}}@test7b
+; ATTRIBUTOR-SAME: (i32* nofree readnone returned align 32 "no-capture-maybe-returned" [[P:%.*]])
+; ATTRIBUTOR-NEXT:    [[TMP1:%.*]] = tail call i8* @f1b(i8* nofree nonnull readnone align 8 dereferenceable(1) @a1)
+; ATTRIBUTOR-NEXT:    ret i32* [[P]]
+;
+  tail call i8* @f1b(i8* align 8 dereferenceable(1) @a1)
+  ret i32* %p
+}
+
+; UTC_ARGS: --turn=off
+
+; TEST 8
+define void @test8_helper() {
+  %ptr0 = tail call i32* @unknown()
+  %ptr1 = tail call align 4 i32* @unknown()
+  %ptr2 = tail call align 8 i32* @unknown()
+
+  tail call void @test8(i32* %ptr1, i32* %ptr1, i32* %ptr0)
+; ATTRIBUTOR: tail call void @test8(i32* readnone align 4 %ptr1, i32* readnone align 4 %ptr1, i32* readnone %ptr0)
+  tail call void @test8(i32* %ptr2, i32* %ptr1, i32* %ptr1)
+; ATTRIBUTOR: tail call void @test8(i32* readnone align 8 %ptr2, i32* readnone align 4 %ptr1, i32* readnone align 4 %ptr1)
+  tail call void @test8(i32* %ptr2, i32* %ptr1, i32* %ptr1)
+; ATTRIBUTOR: tail call void @test8(i32* readnone align 8 %ptr2, i32* readnone align 4 %ptr1, i32* readnone align 4 %ptr1)
+  ret void
+}
+
+declare void @user_i32_ptr(i32*) readnone nounwind
+define internal void @test8(i32* %a, i32* %b, i32* %c) {
+; ATTRIBUTOR: define internal void @test8(i32* nocapture readnone align 4 %a, i32* nocapture readnone align 4 %b, i32* nocapture readnone %c)
+  call void @user_i32_ptr(i32* %a)
+  call void @user_i32_ptr(i32* %b)
+  call void @user_i32_ptr(i32* %c)
+  ret void
+}
+
+declare void @test9_helper(i32* %A)
+define void @test9_traversal(i1 %c, i32* align 4 %B, i32* align 8 %C) {
+  %sel = select i1 %c, i32* %B, i32* %C
+  call void @test9_helper(i32* %sel)
+  ret void
+}
+
+; FIXME: This will work with an upcoming patch (D66618 or similar)
+;             define align 32 i32* @test10a(i32* align 32 "no-capture-maybe-returned" %p)
+; ATTRIBUTOR: define i32* @test10a(i32* nofree nonnull align 32 dereferenceable(4) "no-capture-maybe-returned" %p)
+define i32* @test10a(i32* align 32 %p) {
+; ATTRIBUTOR: %l = load i32, i32* %p, align 32
+  %l = load i32, i32* %p
+  %c = icmp eq i32 %l, 0
+  br i1 %c, label %t, label %f
+t:
+  %r = call i32* @test10a(i32* %p)
+; FIXME: This will work with an upcoming patch (D66618 or similar)
+;             store i32 1, i32* %r, align 32
+; ATTRIBUTOR: store i32 1, i32* %r
+  store i32 1, i32* %r
+  %g0 = getelementptr i32, i32* %p, i32 8
+  br label %e
+f:
+  %g1 = getelementptr i32, i32* %p, i32 8
+; FIXME: This will work with an upcoming patch (D66618 or similar)
+;             store i32 -1, i32* %g1, align 32
+; ATTRIBUTOR: store i32 -1, i32* %g1
+  store i32 -1, i32* %g1
+  br label %e
+e:
+  %phi = phi i32* [%g0, %t], [%g1, %f]
+  ret i32* %phi
+}
+
+; FIXME: This will work with an upcoming patch (D66618 or similar)
+;             define align 32 i32* @test10b(i32* align 32 "no-capture-maybe-returned" %p)
+; ATTRIBUTOR: define i32* @test10b(i32* nofree nonnull align 32 dereferenceable(4) "no-capture-maybe-returned" %p)
+define i32* @test10b(i32* align 32 %p) {
+; ATTRIBUTOR: %l = load i32, i32* %p, align 32
+  %l = load i32, i32* %p
+  %c = icmp eq i32 %l, 0
+  br i1 %c, label %t, label %f
+t:
+  %r = call i32* @test10b(i32* %p)
+; FIXME: This will work with an upcoming patch (D66618 or similar)
+;             store i32 1, i32* %r, align 32
+; ATTRIBUTOR: store i32 1, i32* %r
+  store i32 1, i32* %r
+  %g0 = getelementptr i32, i32* %p, i32 8
+  br label %e
+f:
+  %g1 = getelementptr i32, i32* %p, i32 -8
+; FIXME: This will work with an upcoming patch (D66618 or similar)
+;             store i32 -1, i32* %g1, align 32
+; ATTRIBUTOR: store i32 -1, i32* %g1
+  store i32 -1, i32* %g1
+  br label %e
+e:
+  %phi = phi i32* [%g0, %t], [%g1, %f]
+  ret i32* %phi
+}
+
+
+; ATTRIBUTOR: define i64 @test11(i32* nocapture nofree nonnull readonly align 8 dereferenceable(8) %p)
+define i64 @test11(i32* %p) {
+  %p-cast = bitcast i32* %p to i64*
+  %ret = load i64, i64* %p-cast, align 8
+  ret i64 %ret
+}
+
+; TEST 12
+; Test for deduction using must-be-executed-context and GEP instruction
+
+; FXIME: %p should have nonnull
+; ATTRIBUTOR: define i64 @test12-1(i32* nocapture nofree readonly align 16 %p)
+define i64 @test12-1(i32* align 4 %p) {
+  %p-cast = bitcast i32* %p to i64*
+  %arrayidx0 = getelementptr i64, i64* %p-cast, i64 1
+  %arrayidx1 = getelementptr i64, i64* %arrayidx0, i64 3
+  %ret = load i64, i64* %arrayidx1, align 16
+  ret i64 %ret
+}
+
+; ATTRIBUTOR: define i64 @test12-2(i32* nocapture nofree nonnull readonly align 16 dereferenceable(8) %p)
+define i64 @test12-2(i32* align 4 %p) {
+  %p-cast = bitcast i32* %p to i64*
+  %arrayidx0 = getelementptr i64, i64* %p-cast, i64 0
+  %ret = load i64, i64* %arrayidx0, align 16
+  ret i64 %ret
+}
+
+; FXIME: %p should have nonnull
+; ATTRIBUTOR: define void @test12-3(i32* nocapture nofree writeonly align 16 %p)
+define void @test12-3(i32* align 4 %p) {
+  %p-cast = bitcast i32* %p to i64*
+  %arrayidx0 = getelementptr i64, i64* %p-cast, i64 1
+  %arrayidx1 = getelementptr i64, i64* %arrayidx0, i64 3
+  store i64 0, i64* %arrayidx1, align 16
+  ret void
+}
+
+; ATTRIBUTOR: define void @test12-4(i32* nocapture nofree nonnull writeonly align 16 dereferenceable(8) %p)
+define void @test12-4(i32* align 4 %p) {
+  %p-cast = bitcast i32* %p to i64*
+  %arrayidx0 = getelementptr i64, i64* %p-cast, i64 0
+  store i64 0, i64* %arrayidx0, align 16
+  ret void
+}
+
+declare void @use(i64*) willreturn nounwind
+
+; ATTRIBUTOR: define void @test12-5(i32* align 16 %p)
+define void @test12-5(i32* align 4 %p) {
+  %p-cast = bitcast i32* %p to i64*
+  %arrayidx0 = getelementptr i64, i64* %p-cast, i64 1
+  %arrayidx1 = getelementptr i64, i64* %arrayidx0, i64 3
+  tail call void @use(i64* align 16 %arrayidx1)
+  ret void
+}
+
+; ATTRIBUTOR: define void @test12-6(i32* align 16 %p)
+define void @test12-6(i32* align 4 %p) {
+  %p-cast = bitcast i32* %p to i64*
+  %arrayidx0 = getelementptr i64, i64* %p-cast, i64 0
+  tail call void @use(i64* align 16 %arrayidx0)
+  ret void
+}
+
+; Don't crash on ptr2int/int2ptr uses.
+define i64 @ptr2int(i32* %p) {
+  %p2i = ptrtoint i32* %p to i64
+  ret i64 %p2i
+}
+define i64* @int2ptr(i64 %i) {
+  %i2p = inttoptr i64 %i to i64*
+  ret i64* %i2p
+}
+
+attributes #0 = { nounwind uwtable noinline }
+attributes #1 = { uwtable noinline }
diff --git a/llvm/test/Transforms/Attributor/callbacks.ll b/llvm/test/Transforms/Attributor/callbacks.ll
new file mode 100644
index 000000000..74c870c7e
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/callbacks.ll
@@ -0,0 +1,66 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; FIXME: Add -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations below.
+;        This flag was removed because max iterations is 2 in most cases, but in windows it is 1.
+; RUN: opt -S -passes=attributor -aa-pipeline='basic-aa' -attributor-disable=false -attributor-annotate-decl-cs < %s | FileCheck %s
+; ModuleID = 'callback_simple.c'
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+
+; Test 0
+;
+; Make sure we propagate information from the caller to the callback callee but
+; only for arguments that are mapped through the callback metadata. Here, the
+; first two arguments of the call and the callback callee do not correspond to
+; each other but argument 3-5 of the transitive call site in the caller match
+; arguments 2-4 of the callback callee. Here we should see information and value
+; transfer in both directions.
+
+define void @t0_caller(i32* %a) {
+; CHECK-LABEL: @t0_caller(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[B:%.*]] = alloca i32, align 32
+; CHECK-NEXT:    [[C:%.*]] = alloca i32*, align 64
+; CHECK-NEXT:    [[PTR:%.*]] = alloca i32, align 128
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32* [[B]] to i8*
+; CHECK-NEXT:    store i32 42, i32* [[B]], align 32
+; CHECK-NEXT:    store i32* [[B]], i32** [[C]], align 64
+; CHECK-NEXT:    call void (i32*, i32*, void (i32*, i32*, ...)*, ...) @t0_callback_broker(i32* noalias null, i32* nonnull align 128 dereferenceable(4) [[PTR]], void (i32*, i32*, ...)* nonnull bitcast (void (i32*, i32*, i32*, i64, i32**)* @t0_callback_callee to void (i32*, i32*, ...)*), i32* align 256 [[A:%.*]], i64 99, i32** noalias nocapture nonnull readonly align 64 dereferenceable(8) [[C]])
+
+; CHECK-NEXT:    ret void
+;
+entry:
+  %b = alloca i32, align 32
+  %c = alloca i32*, align 64
+  %ptr = alloca i32, align 128
+  %0 = bitcast i32* %b to i8*
+  store i32 42, i32* %b, align 4
+  store i32* %b, i32** %c, align 8
+  call void (i32*, i32*, void (i32*, i32*, ...)*, ...) @t0_callback_broker(i32* null, i32* %ptr, void (i32*, i32*, ...)* bitcast (void (i32*, i32*, i32*, i64, i32**)* @t0_callback_callee to void (i32*, i32*, ...)*), i32* %a, i64 99, i32** %c)
+  ret void
+}
+
+; Note that the first two arguments are provided by the callback_broker according to the callback in !1 below!
+; The others are annotated with alignment information, amongst others, or even replaced by the constants passed to the call.
+define internal void @t0_callback_callee(i32* %is_not_null, i32* %ptr, i32* %a, i64 %b, i32** %c) {
+; CHECK-LABEL: define {{[^@]+}}@t0_callback_callee
+; CHECK-SAME: (i32* nocapture nonnull writeonly dereferenceable(4) [[IS_NOT_NULL:%.*]], i32* nocapture nonnull readonly align 8 dereferenceable(4) [[PTR:%.*]], i32* align 256 [[A:%.*]], i64 [[B:%.*]], i32** noalias nocapture nonnull readonly align 64 dereferenceable(8) [[C:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[PTR_VAL:%.*]] = load i32, i32* [[PTR:%.*]], align 8
+; CHECK-NEXT:    store i32 [[PTR_VAL]], i32* [[IS_NOT_NULL:%.*]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32*, i32** [[C:%.*]], align 64
+; CHECK-NEXT:    tail call void @t0_check(i32* align 256 [[A:%.*]], i64 99, i32* [[TMP0]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %ptr_val = load i32, i32* %ptr, align 8
+  store i32 %ptr_val, i32* %is_not_null
+  %0 = load i32*, i32** %c, align 8
+  tail call void @t0_check(i32* %a, i64 %b, i32* %0)
+  ret void
+}
+
+declare void @t0_check(i32* align 256, i64, i32*)
+
+declare !callback !0 void @t0_callback_broker(i32*, i32*, void (i32*, i32*, ...)*, ...)
+
+!0 = !{!1}
+!1 = !{i64 2, i64 -1, i64 -1, i1 true}
diff --git a/llvm/test/Transforms/Attributor/dereferenceable-1.ll b/llvm/test/Transforms/Attributor/dereferenceable-1.ll
new file mode 100644
index 000000000..56220c2cf
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/dereferenceable-1.ll
@@ -0,0 +1,312 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -attributor -attributor-manifest-internal --attributor-disable=false -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=4 -S < %s | FileCheck %s --check-prefix=ATTRIBUTOR
+
+
+declare void @deref_phi_user(i32* %a);
+
+; TEST 1
+; take mininimum of return values
+;
+define i32* @test1(i32* dereferenceable(4) %0, double* dereferenceable(8) %1, i1 zeroext %2) local_unnamed_addr {
+; ATTRIBUTOR: define nonnull dereferenceable(4) i32* @test1(i32* nofree nonnull readnone dereferenceable(4) "no-capture-maybe-returned" %0, double* nofree nonnull readnone dereferenceable(8) "no-capture-maybe-returned" %1, i1 zeroext %2)
+  %4 = bitcast double* %1 to i32*
+  %5 = select i1 %2, i32* %0, i32* %4
+  ret i32* %5
+}
+
+; TEST 2
+define i32* @test2(i32* dereferenceable_or_null(4) %0, double* dereferenceable(8) %1, i1 zeroext %2) local_unnamed_addr {
+; ATTRIBUTOR: define dereferenceable_or_null(4) i32* @test2(i32* nofree readnone dereferenceable_or_null(4) "no-capture-maybe-returned" %0, double* nofree nonnull readnone dereferenceable(8) "no-capture-maybe-returned" %1, i1 zeroext %2)
+  %4 = bitcast double* %1 to i32*
+  %5 = select i1 %2, i32* %0, i32* %4
+  ret i32* %5
+}
+
+; TEST 3
+; GEP inbounds
+define i32* @test3_1(i32* dereferenceable(8) %0) local_unnamed_addr {
+; ATTRIBUTOR: define nonnull dereferenceable(4) i32* @test3_1(i32* nofree nonnull readnone dereferenceable(8) "no-capture-maybe-returned" %0)
+  %ret = getelementptr inbounds i32, i32* %0, i64 1
+  ret i32* %ret
+}
+
+define i32* @test3_2(i32* dereferenceable_or_null(32) %0) local_unnamed_addr {
+; ATTRIBUTOR: define nonnull dereferenceable(16) i32* @test3_2(i32* nofree readnone dereferenceable_or_null(32) "no-capture-maybe-returned" %0)
+  %ret = getelementptr inbounds i32, i32* %0, i64 4
+  ret i32* %ret
+}
+
+define i32* @test3_3(i32* dereferenceable(8) %0, i32* dereferenceable(16) %1, i1 %2) local_unnamed_addr {
+; ATTRIBUTOR: define nonnull dereferenceable(4) i32* @test3_3(i32* nofree nonnull readnone dereferenceable(8) "no-capture-maybe-returned" %0, i32* nofree nonnull readnone dereferenceable(16) "no-capture-maybe-returned" %1, i1 %2) local_unnamed_addr
+  %ret1 = getelementptr inbounds i32, i32* %0, i64 1
+  %ret2 = getelementptr inbounds i32, i32* %1, i64 2
+  %ret = select i1 %2, i32* %ret1, i32* %ret2
+  ret i32* %ret
+}
+
+; TEST 4
+; Better than known in IR.
+
+define dereferenceable(4) i32* @test4(i32* dereferenceable(8) %0) local_unnamed_addr {
+; ATTRIBUTOR: define nonnull dereferenceable(8) i32* @test4(i32* nofree nonnull readnone returned dereferenceable(8) "no-capture-maybe-returned" %0)
+  ret i32* %0
+}
+
+; TEST 5
+; loop in which dereferenceabily "grows"
+define void @deref_phi_growing(i32* dereferenceable(4000) %a) {
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
+  %a.addr.0 = phi i32* [ %a, %entry ], [ %incdec.ptr, %for.inc ]
+; ATTRIBUTOR: call void @deref_phi_user(i32* nonnull dereferenceable(4000) %a.addr.0)
+  call void @deref_phi_user(i32* %a.addr.0)
+  %tmp = load i32, i32* %a.addr.0, align 4
+  %cmp = icmp slt i32 %i.0, %tmp
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.cond
+  br label %for.end
+
+for.body:                                         ; preds = %for.cond
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body
+  %incdec.ptr = getelementptr inbounds i32, i32* %a.addr.0, i64 -1
+  %inc = add nuw nsw i32 %i.0, 1
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond.cleanup
+  ret void
+}
+
+; TEST 6
+; loop in which dereferenceabily "shrinks"
+define void @deref_phi_shrinking(i32* dereferenceable(4000) %a) {
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
+  %a.addr.0 = phi i32* [ %a, %entry ], [ %incdec.ptr, %for.inc ]
+; ATTRIBUTOR: call void @deref_phi_user(i32* nonnull %a.addr.0)
+  call void @deref_phi_user(i32* %a.addr.0)
+  %tmp = load i32, i32* %a.addr.0, align 4
+  %cmp = icmp slt i32 %i.0, %tmp
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.cond
+  br label %for.end
+
+for.body:                                         ; preds = %for.cond
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body
+  %incdec.ptr = getelementptr inbounds i32, i32* %a.addr.0, i64 1
+  %inc = add nuw nsw i32 %i.0, 1
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond.cleanup
+  ret void
+}
+
+; TEST 7
+; share known infomation in must-be-executed-context
+declare i32* @unkown_ptr() willreturn nounwind
+declare i32 @unkown_f(i32*) willreturn nounwind
+define i32* @f7_0(i32* %ptr) {
+; ATTRIBUTOR: define nonnull dereferenceable(8) i32* @f7_0(i32* nonnull returned dereferenceable(8) %ptr)
+  %T = tail call i32 @unkown_f(i32* dereferenceable(8) %ptr)
+  ret i32* %ptr
+}	
+
+; ATTRIBUTOR: define void @f7_1(i32* nonnull dereferenceable(4) %ptr, i1 %c) 
+define void @f7_1(i32* %ptr, i1 %c) {
+
+; ATTRIBUTOR:   %A = tail call i32 @unkown_f(i32* nonnull dereferenceable(4) %ptr) 
+  %A = tail call i32 @unkown_f(i32* %ptr)
+
+  %ptr.0 = load i32, i32* %ptr
+  ; deref 4 hold
+
+; FIXME: this should be %B = tail call i32 @unkown_f(i32* nonnull dereferenceable(4) %ptr) 
+; ATTRIBUTOR:   %B = tail call i32 @unkown_f(i32* nonnull dereferenceable(4) %ptr) 
+  %B = tail call i32 @unkown_f(i32* dereferenceable(1) %ptr)
+
+  br i1%c, label %if.true, label %if.false
+if.true:
+; ATTRIBUTOR:   %C = tail call i32 @unkown_f(i32* nonnull dereferenceable(8) %ptr) 
+  %C = tail call i32 @unkown_f(i32* %ptr)
+
+; ATTRIBUTOR:   %D = tail call i32 @unkown_f(i32* nonnull dereferenceable(8) %ptr) 
+  %D = tail call i32 @unkown_f(i32* dereferenceable(8) %ptr)
+
+; FIXME: This should be tail call i32 @unkown_f(i32* nonnull dereferenceable(8) %ptr) 
+;        Making must-be-executed-context backward exploration will fix this.
+; ATTRIBUTOR:   %E = tail call i32 @unkown_f(i32* nonnull dereferenceable(4) %ptr) 
+  %E = tail call i32 @unkown_f(i32* %ptr)
+
+  ret void
+
+if.false:
+  ret void
+}
+
+; ATTRIBUTOR: define void @f7_2(i1 %c) 
+define void @f7_2(i1 %c) {
+
+  %ptr =  tail call i32* @unkown_ptr()
+
+; ATTRIBUTOR:   %A = tail call i32 @unkown_f(i32* nonnull dereferenceable(4) %ptr) 
+  %A = tail call i32 @unkown_f(i32* %ptr)
+
+  %arg_a.0 = load i32, i32* %ptr
+  ; deref 4 hold
+
+; ATTRIBUTOR:   %B = tail call i32 @unkown_f(i32* nonnull dereferenceable(4) %ptr)
+  %B = tail call i32 @unkown_f(i32* dereferenceable(1) %ptr)
+
+  br i1%c, label %if.true, label %if.false
+if.true:
+
+; ATTRIBUTOR:   %C = tail call i32 @unkown_f(i32* nonnull dereferenceable(8) %ptr) 
+  %C = tail call i32 @unkown_f(i32* %ptr)
+
+; ATTRIBUTOR:   %D = tail call i32 @unkown_f(i32* nonnull dereferenceable(8) %ptr) 
+  %D = tail call i32 @unkown_f(i32* dereferenceable(8) %ptr)
+
+  %E = tail call i32 @unkown_f(i32* %ptr)
+; FIXME: This should be @unkown_f(i32* nonnull dereferenceable(8) %ptr) 
+;        Making must-be-executed-context backward exploration will fix this.
+; ATTRIBUTOR:   %E = tail call i32 @unkown_f(i32* nonnull dereferenceable(4) %ptr)
+
+  ret void
+
+if.false:
+  ret void
+}
+
+define i32* @f7_3() {
+; ATTRIBUTOR: define nonnull align 16 dereferenceable(4) i32* @f7_3()
+  %ptr = tail call i32* @unkown_ptr()
+  store i32 10, i32* %ptr, align 16
+  ret i32* %ptr
+}
+
+define i32* @test_for_minus_index(i32* %p) {
+; FIXME: This should have a return dereferenceable(8) but we need to make sure it will work in loops as well.
+; ATTRIBUTOR: define nonnull i32* @test_for_minus_index(i32* nofree nonnull writeonly "no-capture-maybe-returned" %p)
+  %q = getelementptr inbounds i32, i32* %p, i32 -2
+  store i32 1, i32* %q
+  ret i32* %q
+}
+
+define void @deref_or_null_and_nonnull(i32* dereferenceable_or_null(100) %0) {
+; ATTRIBUTOR: define void @deref_or_null_and_nonnull(i32* nocapture nofree nonnull writeonly dereferenceable(100) %0)
+  store i32 1, i32* %0
+  ret void
+}
+
+; TEST 8
+; Use Constant range in deereferenceable
+; void g(int *p, long long int *range){
+;   int r = *range ; // [10, 99]
+;   fill_range(p, *range);
+; }
+
+; void fill_range(int* p, long long int start){
+;   for(long long int i = start;i<start+10;i++){
+;     // If p[i] is inbounds, p is dereferenceable(40) at least.
+;     p[i] = i;
+;   }
+; }
+
+define internal void @fill_range_not_inbounds(i32* %p, i64 %start){
+; ATTRIBUTOR-LABEL: define {{[^@]+}}@fill_range_not_inbounds
+; NOTE: %p should not be dereferenceable
+; ATTRIBUTOR-SAME: (i32* nocapture nofree writeonly [[P:%.*]], i64 [[START:%.*]])
+; ATTRIBUTOR-NEXT:  entry:
+; ATTRIBUTOR-NEXT:    [[TMP0:%.*]] = add nsw i64 [[START:%.*]], 9
+; ATTRIBUTOR-NEXT:    br label [[FOR_BODY:%.*]]
+; ATTRIBUTOR:       for.cond.cleanup:
+; ATTRIBUTOR-NEXT:    ret void
+; ATTRIBUTOR:       for.body:
+; ATTRIBUTOR-NEXT:    [[I_06:%.*]] = phi i64 [ [[START]], [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
+; ATTRIBUTOR-NEXT:    [[CONV:%.*]] = trunc i64 [[I_06]] to i32
+; ATTRIBUTOR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr i32, i32* [[P:%.*]], i64 [[I_06]]
+; ATTRIBUTOR-NEXT:    store i32 [[CONV]], i32* [[ARRAYIDX]], align 4
+; ATTRIBUTOR-NEXT:    [[INC]] = add nsw i64 [[I_06]], 1
+; ATTRIBUTOR-NEXT:    [[CMP:%.*]] = icmp slt i64 [[I_06]], [[TMP0]]
+; ATTRIBUTOR-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_COND_CLEANUP:%.*]]
+;
+entry:
+  %0 = add nsw i64 %start, 9
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body
+  ret void
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.06 = phi i64 [ %start, %entry ], [ %inc, %for.body ]
+  %conv = trunc i64 %i.06 to i32
+  %arrayidx = getelementptr i32, i32* %p, i64 %i.06
+  store i32 %conv, i32* %arrayidx, align 4
+  %inc = add nsw i64 %i.06, 1
+  %cmp = icmp slt i64 %i.06, %0
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+}
+define internal void @fill_range_inbounds(i32* %p, i64 %start){
+; ATTRIBUTOR-LABEL: define {{[^@]+}}@fill_range_inbounds
+; FIXME: %p should be dereferenceable(40)
+; ATTRIBUTOR-SAME: (i32* nocapture nofree writeonly [[P:%.*]], i64 [[START:%.*]])
+; ATTRIBUTOR-NEXT:  entry:
+; ATTRIBUTOR-NEXT:    [[TMP0:%.*]] = add nsw i64 [[START:%.*]], 9
+; ATTRIBUTOR-NEXT:    br label [[FOR_BODY:%.*]]
+; ATTRIBUTOR:       for.cond.cleanup:
+; ATTRIBUTOR-NEXT:    ret void
+; ATTRIBUTOR:       for.body:
+; ATTRIBUTOR-NEXT:    [[I_06:%.*]] = phi i64 [ [[START]], [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
+; ATTRIBUTOR-NEXT:    [[CONV:%.*]] = trunc i64 [[I_06]] to i32
+; ATTRIBUTOR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[P:%.*]], i64 [[I_06]]
+; ATTRIBUTOR-NEXT:    store i32 [[CONV]], i32* [[ARRAYIDX]], align 4
+; ATTRIBUTOR-NEXT:    [[INC]] = add nsw i64 [[I_06]], 1
+; ATTRIBUTOR-NEXT:    [[CMP:%.*]] = icmp slt i64 [[I_06]], [[TMP0]]
+; ATTRIBUTOR-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_COND_CLEANUP:%.*]]
+;
+entry:
+  %0 = add nsw i64 %start, 9
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body
+  ret void
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.06 = phi i64 [ %start, %entry ], [ %inc, %for.body ]
+  %conv = trunc i64 %i.06 to i32
+  %arrayidx = getelementptr inbounds i32, i32* %p, i64 %i.06
+  store i32 %conv, i32* %arrayidx, align 4
+  %inc = add nsw i64 %i.06, 1
+  %cmp = icmp slt i64 %i.06, %0
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+}
+
+define void @call_fill_range(i32* nocapture %p, i64* nocapture readonly %range) {
+; ATTRIBUTOR-LABEL: define {{[^@]+}}@call_fill_range
+; ATTRIBUTOR-SAME: (i32* nocapture nofree writeonly [[P:%.*]], i64* nocapture nofree nonnull readonly align 8 dereferenceable(8) [[RANGE:%.*]])
+; ATTRIBUTOR-NEXT:  entry:
+; ATTRIBUTOR-NEXT:    [[TMP0:%.*]] = load i64, i64* [[RANGE:%.*]], align 8, !range !0
+; ATTRIBUTOR-NEXT:    tail call void @fill_range_inbounds(i32* nocapture nofree writeonly [[P:%.*]], i64 [[TMP0]])
+; ATTRIBUTOR-NEXT:    tail call void @fill_range_not_inbounds(i32* nocapture nofree writeonly [[P]], i64 [[TMP0]])
+; ATTRIBUTOR-NEXT:    ret void
+;
+entry:
+  %0 = load i64, i64* %range, align 8, !range !0
+  tail call void @fill_range_inbounds(i32* %p, i64 %0)
+  tail call void @fill_range_not_inbounds(i32* %p, i64 %0)
+  ret void
+}
+
+!0 = !{i64 10, i64 100}
+
diff --git a/llvm/test/Transforms/Attributor/dereferenceable-2.ll b/llvm/test/Transforms/Attributor/dereferenceable-2.ll
new file mode 100644
index 000000000..b3c0440f9
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/dereferenceable-2.ll
@@ -0,0 +1,356 @@
+; RUN: opt < %s -attributor --attributor-disable=false -S | FileCheck %s --check-prefix=ATTRIBUTOR
+; Copied from Transforms/InferFunctionAttrs/dereferenceable.ll
+
+; Determine dereference-ability before unused loads get deleted:
+; https://bugs.llvm.org/show_bug.cgi?id=21780
+
+define <4 x double> @PR21780(double* %ptr) {
+; ATTRIBUTOR-LABEL: @PR21780(double* nocapture nofree nonnull readonly align 8 dereferenceable(32) %ptr)
+
+  ; GEP of index 0 is simplified away.
+  %arrayidx1 = getelementptr inbounds double, double* %ptr, i64 1
+  %arrayidx2 = getelementptr inbounds double, double* %ptr, i64 2
+  %arrayidx3 = getelementptr inbounds double, double* %ptr, i64 3
+
+  %t0 = load double, double* %ptr, align 8
+  %t1 = load double, double* %arrayidx1, align 8
+  %t2 = load double, double* %arrayidx2, align 8
+  %t3 = load double, double* %arrayidx3, align 8
+
+  %vecinit0 = insertelement <4 x double> undef, double %t0, i32 0
+  %vecinit1 = insertelement <4 x double> %vecinit0, double %t1, i32 1
+  %vecinit2 = insertelement <4 x double> %vecinit1, double %t2, i32 2
+  %vecinit3 = insertelement <4 x double> %vecinit2, double %t3, i32 3
+  %shuffle = shufflevector <4 x double> %vecinit3, <4 x double> %vecinit3, <4 x i32> <i32 0, i32 0, i32 2, i32 2>
+  ret <4 x double> %shuffle
+}
+
+
+define double @PR21780_only_access3_with_inbounds(double* %ptr) {
+; ATTRIBUTOR-LABEL: @PR21780_only_access3_with_inbounds(double* nocapture nofree nonnull readonly align 8 dereferenceable(32) %ptr)
+
+  %arrayidx3 = getelementptr inbounds double, double* %ptr, i64 3
+  %t3 = load double, double* %arrayidx3, align 8
+  ret double %t3
+}
+
+define double @PR21780_only_access3_without_inbounds(double* %ptr) {
+; ATTRIBUTOR-LABEL: @PR21780_only_access3_without_inbounds(double* nocapture nofree readonly align 8 %ptr)
+  %arrayidx3 = getelementptr double, double* %ptr, i64 3
+  %t3 = load double, double* %arrayidx3, align 8
+  ret double %t3
+}
+
+define double @PR21780_without_inbounds(double* %ptr) {
+; ATTRIBUTOR-LABEL: @PR21780_without_inbounds(double* nocapture nofree nonnull readonly align 8 dereferenceable(32) %ptr)
+
+  %arrayidx1 = getelementptr double, double* %ptr, i64 1
+  %arrayidx2 = getelementptr double, double* %ptr, i64 2
+  %arrayidx3 = getelementptr double, double* %ptr, i64 3
+
+  %t0 = load double, double* %ptr, align 8
+  %t1 = load double, double* %arrayidx1, align 8
+  %t2 = load double, double* %arrayidx2, align 8
+  %t3 = load double, double* %arrayidx3, align 8
+
+  ret double %t3
+}
+
+; Unsimplified, but still valid. Also, throw in some bogus arguments.
+
+define void @gep0(i8* %unused, i8* %other, i8* %ptr) {
+; ATTRIBUTOR-LABEL: @gep0(i8* nocapture nofree readnone %unused, i8* nocapture nofree nonnull writeonly dereferenceable(1) %other, i8* nocapture nofree nonnull readonly dereferenceable(3) %ptr)
+  %arrayidx0 = getelementptr i8, i8* %ptr, i64 0
+  %arrayidx1 = getelementptr i8, i8* %ptr, i64 1
+  %arrayidx2 = getelementptr i8, i8* %ptr, i64 2
+  %t0 = load i8, i8* %arrayidx0
+  %t1 = load i8, i8* %arrayidx1
+  %t2 = load i8, i8* %arrayidx2
+  store i8 %t2, i8* %other
+  ret void
+}
+
+; Order of accesses does not change computation.
+; Multiple arguments may be dereferenceable.
+
+define void @ordering(i8* %ptr1, i32* %ptr2) {
+; ATTRIBUTOR-LABEL: @ordering(i8* nocapture nofree nonnull readonly dereferenceable(3) %ptr1, i32* nocapture nofree nonnull readonly dereferenceable(8) %ptr2)
+  %a20 = getelementptr i32, i32* %ptr2, i64 0
+  %a12 = getelementptr i8, i8* %ptr1, i64 2
+  %t12 = load i8, i8* %a12
+  %a11 = getelementptr i8, i8* %ptr1, i64 1
+  %t20 = load i32, i32* %a20
+  %a10 = getelementptr i8, i8* %ptr1, i64 0
+  %t10 = load i8, i8* %a10
+  %t11 = load i8, i8* %a11
+  %a21 = getelementptr i32, i32* %ptr2, i64 1
+  %t21 = load i32, i32* %a21
+  ret void
+}
+
+; Not in entry block.
+
+define void @not_entry_but_guaranteed_to_execute(i8* %ptr) {
+; ATTRIBUTOR-LABEL: @not_entry_but_guaranteed_to_execute(i8* nocapture nofree nonnull readonly dereferenceable(3) %ptr)
+entry:
+  br label %exit
+exit:
+  %arrayidx0 = getelementptr i8, i8* %ptr, i64 0
+  %arrayidx1 = getelementptr i8, i8* %ptr, i64 1
+  %arrayidx2 = getelementptr i8, i8* %ptr, i64 2
+  %t0 = load i8, i8* %arrayidx0
+  %t1 = load i8, i8* %arrayidx1
+  %t2 = load i8, i8* %arrayidx2
+  ret void
+}
+
+; Not in entry block and not guaranteed to execute.
+
+define void @not_entry_not_guaranteed_to_execute(i8* %ptr, i1 %cond) {
+; ATTRIBUTOR-LABEL: @not_entry_not_guaranteed_to_execute(i8* nocapture nofree readonly %ptr, i1 %cond)
+entry:
+  br i1 %cond, label %loads, label %exit
+loads:
+  %arrayidx0 = getelementptr i8, i8* %ptr, i64 0
+  %arrayidx1 = getelementptr i8, i8* %ptr, i64 1
+  %arrayidx2 = getelementptr i8, i8* %ptr, i64 2
+  %t0 = load i8, i8* %arrayidx0
+  %t1 = load i8, i8* %arrayidx1
+  %t2 = load i8, i8* %arrayidx2
+  ret void
+exit:
+  ret void
+}
+
+; The last load may not execute, so derefenceable bytes only covers the 1st two loads.
+
+define void @partial_in_entry(i16* %ptr, i1 %cond) {
+; ATTRIBUTOR-LABEL: @partial_in_entry(i16* nocapture nofree nonnull readonly dereferenceable(4) %ptr, i1 %cond)
+entry:
+  %arrayidx0 = getelementptr i16, i16* %ptr, i64 0
+  %arrayidx1 = getelementptr i16, i16* %ptr, i64 1
+  %arrayidx2 = getelementptr i16, i16* %ptr, i64 2
+  %t0 = load i16, i16* %arrayidx0
+  %t1 = load i16, i16* %arrayidx1
+  br i1 %cond, label %loads, label %exit
+loads:
+  %t2 = load i16, i16* %arrayidx2
+  ret void
+exit:
+  ret void
+}
+
+; The volatile load can't be used to prove a non-volatile access is allowed.
+; The 2nd and 3rd loads may never execute.
+
+define void @volatile_is_not_dereferenceable(i16* %ptr) {
+; ATTRIBUTOR-LABEL: @volatile_is_not_dereferenceable(i16* nofree %ptr)
+  %arrayidx0 = getelementptr i16, i16* %ptr, i64 0
+  %arrayidx1 = getelementptr i16, i16* %ptr, i64 1
+  %arrayidx2 = getelementptr i16, i16* %ptr, i64 2
+  %t0 = load volatile i16, i16* %arrayidx0
+  %t1 = load i16, i16* %arrayidx1
+  %t2 = load i16, i16* %arrayidx2
+  ret void
+}
+
+; TODO: We should allow inference for atomic (but not volatile) ops.
+
+define void @atomic_is_alright(i16* %ptr) {
+; ATTRIBUTOR-LABEL: @atomic_is_alright(i16* nocapture nofree nonnull readonly align 2 dereferenceable(6) %ptr)
+  %arrayidx0 = getelementptr i16, i16* %ptr, i64 0
+  %arrayidx1 = getelementptr i16, i16* %ptr, i64 1
+  %arrayidx2 = getelementptr i16, i16* %ptr, i64 2
+  %t0 = load atomic i16, i16* %arrayidx0 unordered, align 2
+  %t1 = load i16, i16* %arrayidx1
+  %t2 = load i16, i16* %arrayidx2
+  ret void
+}
+
+declare void @may_not_return()
+
+define void @not_guaranteed_to_transfer_execution(i16* %ptr) {
+; ATTRIBUTOR-LABEL: @not_guaranteed_to_transfer_execution(i16* nocapture nonnull readonly dereferenceable(2) %ptr)
+  %arrayidx0 = getelementptr i16, i16* %ptr, i64 0
+  %arrayidx1 = getelementptr i16, i16* %ptr, i64 1
+  %arrayidx2 = getelementptr i16, i16* %ptr, i64 2
+  %t0 = load i16, i16* %arrayidx0
+  call void @may_not_return()
+  %t1 = load i16, i16* %arrayidx1
+  %t2 = load i16, i16* %arrayidx2
+  ret void
+}
+
+; We must have consecutive accesses.
+
+define void @variable_gep_index(i8* %unused, i8* %ptr, i64 %variable_index) {
+; ATTRIBUTOR-LABEL: @variable_gep_index(i8* nocapture nofree readnone %unused, i8* nocapture nofree nonnull readonly dereferenceable(1) %ptr, i64 %variable_index)
+  %arrayidx1 = getelementptr i8, i8* %ptr, i64 %variable_index
+  %arrayidx2 = getelementptr i8, i8* %ptr, i64 2
+  %t0 = load i8, i8* %ptr
+  %t1 = load i8, i8* %arrayidx1
+  %t2 = load i8, i8* %arrayidx2
+  ret void
+}
+
+; Deal with >1 GEP index.
+
+define void @multi_index_gep(<4 x i8>* %ptr) {
+; FIXME: %ptr should be dereferenceable(4)
+; ATTRIBUTOR-LABEL: @multi_index_gep(<4 x i8>* nocapture nofree nonnull readonly dereferenceable(1) %ptr)
+  %arrayidx00 = getelementptr <4 x i8>, <4 x i8>* %ptr, i64 0, i64 0
+  %t0 = load i8, i8* %arrayidx00
+  ret void
+}
+
+; Could round weird bitwidths down?
+
+define void @not_byte_multiple(i9* %ptr) {
+; ATTRIBUTOR-LABEL: @not_byte_multiple(i9* nocapture nofree nonnull readonly dereferenceable(2) %ptr) 
+  %arrayidx0 = getelementptr i9, i9* %ptr, i64 0
+  %t0 = load i9, i9* %arrayidx0
+  ret void
+}
+
+; Missing direct access from the pointer.
+
+define void @no_pointer_deref(i16* %ptr) {
+; ATTRIBUTOR-LABEL: @no_pointer_deref(i16* nocapture nofree readonly %ptr)
+  %arrayidx1 = getelementptr i16, i16* %ptr, i64 1
+  %arrayidx2 = getelementptr i16, i16* %ptr, i64 2
+  %t1 = load i16, i16* %arrayidx1
+  %t2 = load i16, i16* %arrayidx2
+  ret void
+}
+
+; Out-of-order is ok, but missing access concludes dereferenceable range.
+
+define void @non_consecutive(i32* %ptr) {
+; ATTRIBUTOR-LABEL: @non_consecutive(i32* nocapture nofree nonnull readonly dereferenceable(8) %ptr)
+  %arrayidx1 = getelementptr i32, i32* %ptr, i64 1
+  %arrayidx0 = getelementptr i32, i32* %ptr, i64 0
+  %arrayidx3 = getelementptr i32, i32* %ptr, i64 3
+  %t1 = load i32, i32* %arrayidx1
+  %t0 = load i32, i32* %arrayidx0
+  %t3 = load i32, i32* %arrayidx3
+  ret void
+}
+
+; Improve on existing dereferenceable attribute.
+
+define void @more_bytes(i32* dereferenceable(8) %ptr) {
+; ATTRIBUTOR-LABEL: @more_bytes(i32* nocapture nofree nonnull readonly dereferenceable(16) %ptr)
+  %arrayidx3 = getelementptr i32, i32* %ptr, i64 3
+  %arrayidx1 = getelementptr i32, i32* %ptr, i64 1
+  %arrayidx0 = getelementptr i32, i32* %ptr, i64 0
+  %arrayidx2 = getelementptr i32, i32* %ptr, i64 2
+  %t3 = load i32, i32* %arrayidx3
+  %t1 = load i32, i32* %arrayidx1
+  %t2 = load i32, i32* %arrayidx2
+  %t0 = load i32, i32* %arrayidx0
+  ret void
+}
+
+; Improve on existing dereferenceable_or_null attribute.
+
+define void @more_bytes_and_not_null(i32* dereferenceable_or_null(8) %ptr) {
+; ATTRIBUTOR-LABEL: @more_bytes_and_not_null(i32* nocapture nofree nonnull readonly dereferenceable(16) %ptr)
+  %arrayidx3 = getelementptr i32, i32* %ptr, i64 3
+  %arrayidx1 = getelementptr i32, i32* %ptr, i64 1
+  %arrayidx0 = getelementptr i32, i32* %ptr, i64 0
+  %arrayidx2 = getelementptr i32, i32* %ptr, i64 2
+  %t3 = load i32, i32* %arrayidx3
+  %t1 = load i32, i32* %arrayidx1
+  %t2 = load i32, i32* %arrayidx2
+  %t0 = load i32, i32* %arrayidx0
+  ret void
+}
+
+; But don't pessimize existing dereferenceable attribute.
+
+define void @better_bytes(i32* dereferenceable(100) %ptr) {
+; ATTRIBUTOR-LABEL: @better_bytes(i32* nocapture nofree nonnull readonly dereferenceable(100) %ptr)
+  %arrayidx3 = getelementptr i32, i32* %ptr, i64 3
+  %arrayidx1 = getelementptr i32, i32* %ptr, i64 1
+  %arrayidx0 = getelementptr i32, i32* %ptr, i64 0
+  %arrayidx2 = getelementptr i32, i32* %ptr, i64 2
+  %t3 = load i32, i32* %arrayidx3
+  %t1 = load i32, i32* %arrayidx1
+  %t2 = load i32, i32* %arrayidx2
+  %t0 = load i32, i32* %arrayidx0
+  ret void
+}
+
+define void @bitcast(i32* %arg) {
+; ATTRIBUTOR-LABEL: @bitcast(i32* nocapture nofree nonnull readonly dereferenceable(8) %arg)
+  %ptr = bitcast i32* %arg to float*
+  %arrayidx0 = getelementptr float, float* %ptr, i64 0
+  %arrayidx1 = getelementptr float, float* %ptr, i64 1
+  %t0 = load float, float* %arrayidx0
+  %t1 = load float, float* %arrayidx1
+  ret void
+}
+
+define void @bitcast_different_sizes(double* %arg1, i8* %arg2) {
+; ATTRIBUTOR-LABEL: @bitcast_different_sizes(double* nocapture nofree nonnull readonly dereferenceable(12) %arg1, i8* nocapture nofree nonnull readonly dereferenceable(16) %arg2)
+  %ptr1 = bitcast double* %arg1 to float*
+  %a10 = getelementptr float, float* %ptr1, i64 0
+  %a11 = getelementptr float, float* %ptr1, i64 1
+  %a12 = getelementptr float, float* %ptr1, i64 2
+  %ld10 = load float, float* %a10
+  %ld11 = load float, float* %a11
+  %ld12 = load float, float* %a12
+
+  %ptr2 = bitcast i8* %arg2 to i64*
+  %a20 = getelementptr i64, i64* %ptr2, i64 0
+  %a21 = getelementptr i64, i64* %ptr2, i64 1
+  %ld20 = load i64, i64* %a20
+  %ld21 = load i64, i64* %a21
+  ret void
+}
+
+define void @negative_offset(i32* %arg) {
+; ATTRIBUTOR-LABEL: @negative_offset(i32* nocapture nofree nonnull readonly dereferenceable(4) %arg)
+  %ptr = bitcast i32* %arg to float*
+  %arrayidx0 = getelementptr float, float* %ptr, i64 0
+  %arrayidx1 = getelementptr float, float* %ptr, i64 -1
+  %t0 = load float, float* %arrayidx0
+  %t1 = load float, float* %arrayidx1
+  ret void
+}
+
+define void @stores(i32* %arg) {
+; ATTRIBUTOR-LABEL: @stores(i32* nocapture nofree nonnull writeonly dereferenceable(8) %arg)
+  %ptr = bitcast i32* %arg to float*
+  %arrayidx0 = getelementptr float, float* %ptr, i64 0
+  %arrayidx1 = getelementptr float, float* %ptr, i64 1
+  store float 1.0, float* %arrayidx0
+  store float 2.0, float* %arrayidx1
+  ret void
+}
+
+define void @load_store(i32* %arg) {
+; ATTRIBUTOR-LABEL: @load_store(i32* nocapture nofree nonnull dereferenceable(8) %arg)
+  %ptr = bitcast i32* %arg to float*
+  %arrayidx0 = getelementptr float, float* %ptr, i64 0
+  %arrayidx1 = getelementptr float, float* %ptr, i64 1
+  %t1 = load float, float* %arrayidx0
+  store float 2.0, float* %arrayidx1
+  ret void
+}
+
+define void @different_size1(i32* %arg) {
+; ATTRIBUTOR-LABEL: @different_size1(i32* nocapture nofree nonnull writeonly dereferenceable(8) %arg)
+  %arg-cast = bitcast i32* %arg to double*
+  store double 0.000000e+00, double* %arg-cast
+  store i32 0, i32* %arg
+  ret void
+}
+
+define void @different_size2(i32* %arg) {
+; ATTRIBUTOR-LABEL: @different_size2(i32* nocapture nofree nonnull writeonly dereferenceable(8) %arg)
+  store i32 0, i32* %arg
+  %arg-cast = bitcast i32* %arg to double*
+  store double 0.000000e+00, double* %arg-cast
+  ret void
+}
diff --git a/llvm/test/Transforms/Attributor/heap_to_stack.ll b/llvm/test/Transforms/Attributor/heap_to_stack.ll
new file mode 100644
index 000000000..3d8b1b3aa
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/heap_to_stack.ll
@@ -0,0 +1,402 @@
+; RUN: opt -passes=attributor --attributor-disable=false -attributor-annotate-decl-cs -S < %s | FileCheck %s
+
+declare noalias i8* @malloc(i64)
+
+declare void @nocapture_func_frees_pointer(i8* nocapture)
+
+declare void @func_throws(...)
+
+declare void @sync_func(i8* %p)
+
+declare void @sync_will_return(i8* %p) willreturn nounwind
+
+declare void @no_sync_func(i8* nocapture %p) nofree nosync willreturn
+
+declare void @nofree_func(i8* nocapture %p) nofree  nosync willreturn
+
+declare void @foo(i32* %p)
+
+declare void @foo_nounw(i32* %p) nounwind nofree
+
+declare i32 @no_return_call() noreturn
+
+declare void @free(i8* nocapture)
+
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture) nounwind
+
+; CHECK: @nofree_arg_only(i8* nocapture nofree %p1, i8* nocapture %p2)
+define void @nofree_arg_only(i8* %p1, i8* %p2) {
+  tail call void @free(i8* %p2)
+  tail call void @nofree_func(i8* %p1)
+  ret void
+}
+
+; TEST 1 - negative, pointer freed in another function.
+
+define void @test1() {
+  %1 = tail call noalias i8* @malloc(i64 4)
+  ; CHECK: @malloc(i64 4)
+  ; CHECK-NEXT: @nocapture_func_frees_pointer(i8* noalias nocapture %1)
+  tail call void @nocapture_func_frees_pointer(i8* %1)
+  tail call void (...) @func_throws()
+  tail call void @free(i8* %1)
+  ret void
+}
+
+; TEST 2 - negative, call to a sync function.
+
+define void @test2() {
+  %1 = tail call noalias i8* @malloc(i64 4)
+  ; CHECK: @malloc(i64 4)
+  ; CHECK-NEXT: @sync_func(i8* %1)
+  tail call void @sync_func(i8* %1)
+  tail call void @free(i8* %1)
+  ret void
+}
+
+; TEST 3 - 1 malloc, 1 free
+
+define void @test3() {
+  %1 = tail call noalias i8* @malloc(i64 4)
+  ; CHECK: %1 = alloca i8, i64 4
+  ; CHECK-NEXT: @no_sync_func(i8* noalias nocapture nofree %1)
+  tail call void @no_sync_func(i8* %1)
+  ; CHECK-NOT: @free(i8* %1)
+  tail call void @free(i8* %1)
+  ret void
+}
+
+define void @test3a(i8* %p) {
+  %1 = tail call noalias i8* @malloc(i64 4)
+  ; CHECK: %1 = alloca i8, i64 4
+  ; CHECK-NEXT: tail call void @nofree_arg_only
+  tail call void @nofree_arg_only(i8* %1, i8* %p)
+  ; CHECK-NOT: @free(i8* %1)
+  tail call void @free(i8* %1)
+  ret void
+}
+
+declare noalias i8* @calloc(i64, i64)
+
+define void @test0() {
+  %1 = tail call noalias i8* @calloc(i64 2, i64 4)
+  ; CHECK: %1 = alloca i8, i64 8
+  ; CHECK-NEXT: %calloc_bc = bitcast i8* %1 to i8*
+  ; CHECK-NEXT: call void @llvm.memset.p0i8.i64(i8* %calloc_bc, i8 0, i64 8, i1 false)
+  ; CHECK-NEXT: @no_sync_func(i8* noalias nocapture nofree %1)
+  tail call void @no_sync_func(i8* %1)
+  ; CHECK-NOT: @free(i8* %1)
+  tail call void @free(i8* %1)
+  ret void
+}
+
+; TEST 4 
+define void @test4() {
+  %1 = tail call noalias i8* @malloc(i64 4)
+  ; CHECK: %1 = alloca i8, i64 4
+  ; CHECK-NEXT: @nofree_func(i8* noalias nocapture nofree %1)
+  tail call void @nofree_func(i8* %1)
+  ret void
+}
+
+; TEST 5 - not all exit paths have a call to free, but all uses of malloc
+; are in nofree functions and are not captured
+
+define void @test5(i32, i8* %p) {
+  %2 = tail call noalias i8* @malloc(i64 4)
+  ; CHECK: %2 = alloca i8, i64 4
+  ; CHECK-NEXT: icmp eq i32 %0, 0
+  %3 = icmp eq i32 %0, 0
+  br i1 %3, label %5, label %4
+
+4:                                                ; preds = %1
+  tail call void @nofree_func(i8* %2)
+  br label %6
+
+5:                                                ; preds = %1
+  tail call void @nofree_arg_only(i8* %2, i8* %p)
+  tail call void @free(i8* %2)
+  ; CHECK-NOT: @free(i8* %2)
+  br label %6
+
+6:                                                ; preds = %5, %4
+  ret void
+}
+
+; TEST 6 - all exit paths have a call to free
+
+define void @test6(i32) {
+  %2 = tail call noalias i8* @malloc(i64 4)
+  ; CHECK: %2 = alloca i8, i64 4
+  ; CHECK-NEXT: icmp eq i32 %0, 0
+  %3 = icmp eq i32 %0, 0
+  br i1 %3, label %5, label %4
+
+4:                                                ; preds = %1
+  tail call void @nofree_func(i8* %2)
+  tail call void @free(i8* %2)
+  ; CHECK-NOT: @free(i8* %2)
+  br label %6
+
+5:                                                ; preds = %1
+  tail call void @free(i8* %2)
+  ; CHECK-NOT: @free(i8* %2)
+  br label %6
+
+6:                                                ; preds = %5, %4
+  ret void
+}
+
+; TEST 7 - free is dead.
+
+define void @test7() {
+  %1 = tail call noalias i8* @malloc(i64 4)
+  ; CHECK: alloca i8, i64 4
+  ; CHECK-NEXT: tail call i32 @no_return_call()
+  tail call i32 @no_return_call()
+  ; CHECK-NOT: @free(i8* %1)
+  tail call void @free(i8* %1)
+  ret void
+}
+
+; TEST 8 - Negative: bitcast pointer used in capture function
+
+define void @test8() {
+  %1 = tail call noalias i8* @malloc(i64 4)
+  ; CHECK: %1 = tail call noalias i8* @malloc(i64 4)
+  ; CHECK-NEXT: @no_sync_func(i8* nocapture nofree %1)
+  tail call void @no_sync_func(i8* %1)
+  %2 = bitcast i8* %1 to i32*
+  store i32 10, i32* %2
+  %3 = load i32, i32* %2
+  tail call void @foo(i32* %2)
+  ; CHECK: @free(i8* %1)
+  tail call void @free(i8* %1)
+  ret void
+}
+
+; TEST 9 - FIXME: malloc should be converted.
+define void @test9() {
+  %1 = tail call noalias i8* @malloc(i64 4)
+  ; CHECK: %1 = tail call noalias i8* @malloc(i64 4)
+  ; CHECK-NEXT: @no_sync_func(i8* nocapture nofree %1)
+  tail call void @no_sync_func(i8* %1)
+  %2 = bitcast i8* %1 to i32*
+  store i32 10, i32* %2
+  %3 = load i32, i32* %2
+  tail call void @foo_nounw(i32* %2)
+  ; CHECK: @free(i8* %1)
+  tail call void @free(i8* %1)
+  ret void
+}
+
+; TEST 10 - 1 malloc, 1 free
+
+define i32 @test10() {
+  %1 = tail call noalias i8* @malloc(i64 4)
+  ; CHECK: %1 = alloca i8, i64 4
+  ; CHECK-NEXT: @no_sync_func(i8* noalias nocapture nofree %1)
+  tail call void @no_sync_func(i8* %1)
+  %2 = bitcast i8* %1 to i32*
+  store i32 10, i32* %2
+  %3 = load i32, i32* %2
+  ; CHECK-NOT: @free(i8* %1)
+  tail call void @free(i8* %1)
+  ret i32 %3
+}
+
+define i32 @test_lifetime() {
+  %1 = tail call noalias i8* @malloc(i64 4)
+  ; CHECK: %1 = alloca i8, i64 4
+  ; CHECK-NEXT: @no_sync_func(i8* noalias nocapture nofree %1)
+  tail call void @no_sync_func(i8* %1)
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* %1)
+  %2 = bitcast i8* %1 to i32*
+  store i32 10, i32* %2
+  %3 = load i32, i32* %2
+  ; CHECK-NOT: @free(i8* %1)
+  tail call void @free(i8* %1)
+  ret i32 %3
+}
+
+; TEST 11 
+
+define void @test11() {
+  %1 = tail call noalias i8* @malloc(i64 4)
+  ; CHECK: test11
+  ; CHECK-NEXT: alloc
+  ; CHECK-NEXT: @sync_will_return(i8* %1)
+  tail call void @sync_will_return(i8* %1)
+  tail call void @free(i8* %1)
+  ret void
+}
+
+; TEST 12
+define i32 @irreducible_cfg(i32 %0) {
+  ; CHECK: alloca i8, i64 4
+  ; CHECK-NEXT: %3 = bitcast
+  %2 = call noalias i8* @malloc(i64 4)
+  %3 = bitcast i8* %2 to i32*
+  store i32 10, i32* %3, align 4
+  %4 = icmp eq i32 %0, 1
+  br i1 %4, label %5, label %7
+
+5:                                                ; preds = %1
+  %6 = add nsw i32 %0, 5
+  br label %13
+
+7:                                                ; preds = %1
+  br label %8
+
+8:                                                ; preds = %13, %7
+  %.0 = phi i32 [ %14, %13 ], [ 1, %7 ]
+  %9 = load i32, i32* %3, align 4
+  %10 = add nsw i32 %9, -1
+  store i32 %10, i32* %3, align 4
+  %11 = icmp ne i32 %9, 0
+  br i1 %11, label %12, label %15
+
+12:                                               ; preds = %8
+  br label %13
+
+13:                                               ; preds = %12, %5
+  %.1 = phi i32 [ %6, %5 ], [ %.0, %12 ]
+  %14 = add nsw i32 %.1, 1
+  br label %8
+
+15:                                               ; preds = %8
+  %16 = load i32, i32* %3, align 4
+  %17 = bitcast i32* %3 to i8*
+  call void @free(i8* %17)
+  %18 = load i32, i32* %3, align 4
+  ret i32 %18
+}
+
+
+define i32 @malloc_in_loop(i32 %0) {
+  %2 = alloca i32, align 4
+  %3 = alloca i32*, align 8
+  store i32 %0, i32* %2, align 4
+  br label %4
+
+4:                                                ; preds = %8, %1
+  %5 = load i32, i32* %2, align 4
+  %6 = add nsw i32 %5, -1
+  store i32 %6, i32* %2, align 4
+  %7 = icmp sgt i32 %6, 0
+  br i1 %7, label %8, label %11
+
+8:                                                ; preds = %4
+  %9 = call noalias i8* @malloc(i64 4)
+  ; CHECK: alloca i8, i64 4
+  %10 = bitcast i8* %9 to i32*
+  store i32 1, i32* %10, align 8
+  br label %4
+
+11:                                               ; preds = %4
+  ret i32 5
+}
+
+; Malloc/Calloc too large
+define i32 @test13() {
+  %1 = tail call noalias i8* @malloc(i64 256)
+  ; CHECK: %1 = tail call noalias i8* @malloc(i64 256)
+  ; CHECK-NEXT: @no_sync_func(i8* noalias nofree %1)
+  tail call void @no_sync_func(i8* %1)
+  %2 = bitcast i8* %1 to i32*
+  store i32 10, i32* %2
+  %3 = load i32, i32* %2
+  tail call void @free(i8* %1)
+  ; CHECK: tail call void @free(i8* noalias %1)
+  ret i32 %3
+}
+
+define i32 @test_sle() {
+  %1 = tail call noalias i8* @malloc(i64 -1)
+  ; CHECK: %1 = tail call noalias i8* @malloc(i64 -1)
+  ; CHECK-NEXT: @no_sync_func(i8* noalias nofree %1)
+  tail call void @no_sync_func(i8* %1)
+  %2 = bitcast i8* %1 to i32*
+  store i32 10, i32* %2
+  %3 = load i32, i32* %2
+  tail call void @free(i8* %1)
+  ; CHECK: tail call void @free(i8* noalias %1)
+  ret i32 %3
+}
+
+define i32 @test_overflow() {
+  %1 = tail call noalias i8* @calloc(i64 65537, i64 65537)
+  ; CHECK: %1 = tail call noalias i8* @calloc(i64 65537, i64 65537)
+  ; CHECK-NEXT: @no_sync_func(i8* noalias nofree %1)
+  tail call void @no_sync_func(i8* %1)
+  %2 = bitcast i8* %1 to i32*
+  store i32 10, i32* %2
+  %3 = load i32, i32* %2
+  tail call void @free(i8* %1)
+  ; CHECK: tail call void @free(i8* noalias %1)
+  ret i32 %3
+}
+
+define void @test14() {
+  %1 = tail call noalias i8* @calloc(i64 64, i64 4)
+  ; CHECK: %1 = tail call noalias i8* @calloc(i64 64, i64 4)
+  ; CHECK-NEXT: @no_sync_func(i8* noalias nofree %1)
+  tail call void @no_sync_func(i8* %1)
+  tail call void @free(i8* %1)
+  ; CHECK: tail call void @free(i8* noalias %1)
+  ret void
+}
+
+define void @test15(i64 %S) {
+  ; CHECK: %1 = tail call noalias i8* @malloc(i64 %S)
+  %1 = tail call noalias i8* @malloc(i64 %S)
+  ; CHECK-NEXT: @no_sync_func(i8* noalias nofree %1)
+  tail call void @no_sync_func(i8* %1)
+  ; CHECK-NEXT: @free(i8* noalias %1)
+  tail call void @free(i8* %1)
+  ret void
+}
+
+define void @test16a(i8 %v, i8** %P) {
+  ; CHECK: %1 = alloca
+  %1 = tail call noalias i8* @malloc(i64 4)
+  ; CHECK-NEXT: store i8 %v, i8* %1
+  store i8 %v, i8* %1
+  ; CHECK-NEXT: @no_sync_func(i8* noalias nocapture nofree %1)
+  tail call void @no_sync_func(i8* %1)
+  ; CHECK-NOT: @free(i8* %1)
+  tail call void @free(i8* %1)
+  ret void
+}
+
+define void @test16b(i8 %v, i8** %P) {
+  ; CHECK: %1 = tail call noalias i8* @malloc(i64 4)
+  %1 = tail call noalias i8* @malloc(i64 4)
+  ; CHECK-NEXT: store i8* %1, i8** %P
+  store i8* %1, i8** %P
+  ; CHECK-NEXT: @no_sync_func(i8* nocapture nofree %1)
+  tail call void @no_sync_func(i8* %1)
+  ; CHECK-NEXT: @free(i8* %1)
+  tail call void @free(i8* %1)
+  ret void
+}
+
+define void @test16c(i8 %v, i8** %P) {
+  ; CHECK: %1 = alloca
+  %1 = tail call noalias i8* @malloc(i64 4)
+  ; CHECK-NEXT: store i8* %1, i8** %P
+  store i8* %1, i8** %P
+  ; CHECK-NEXT: @no_sync_func(i8* nocapture nofree %1)
+  tail call void @no_sync_func(i8* %1) nounwind
+  ; CHECK-NOT: @free
+  tail call void @free(i8* %1)
+  ret void
+}
+
+define void @test16d(i8 %v, i8** %P) {
+  ; CHECK: %1 = tail call noalias i8* @malloc(i64 4)
+  %1 = tail call noalias i8* @malloc(i64 4)
+  ; CHECK-NEXT: store i8* %1, i8** %P
+  store i8* %1, i8** %P
+  ret void
+}
diff --git a/llvm/test/Transforms/Attributor/internal-noalias.ll b/llvm/test/Transforms/Attributor/internal-noalias.ll
new file mode 100644
index 000000000..fd6e17bc7
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/internal-noalias.ll
@@ -0,0 +1,48 @@
+; RUN: opt -S -passes=attributor -aa-pipeline='basic-aa' -attributor-disable=false -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=5 < %s | FileCheck %s
+
+define dso_local i32 @visible(i32* noalias %A, i32* noalias %B) #0 {
+entry:
+  %call1 = call i32 @noalias_args(i32* %A, i32* %B)
+  %call2 = call i32 @noalias_args_argmem(i32* %A, i32* %B)
+  %add = add nsw i32 %call1, %call2
+  ret i32 %add
+}
+
+; CHECK: define private i32 @noalias_args(i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) %A, i32* noalias nocapture nofree nonnull readonly align 4 dereferenceable(4) %B)
+
+define private i32 @noalias_args(i32* %A, i32* %B) #0 {
+entry:
+  %0 = load i32, i32* %A, align 4
+  %1 = load i32, i32* %B, align 4
+  %add = add nsw i32 %0, %1
+  %call = call i32 @noalias_args_argmem(i32* %A, i32* %B)
+  %add2 = add nsw i32 %add, %call
+  ret i32 %add2
+}
+
+
+; FIXME: Should be something like this.
+; define internal i32 @noalias_args_argmem(i32* noalias nocapture readonly %A, i32* noalias nocapture readonly %B)
+; CHECK: define internal i32 @noalias_args_argmem(i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) %A, i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) %B)
+
+;
+define internal i32 @noalias_args_argmem(i32* %A, i32* %B) #1 {
+entry:
+  %0 = load i32, i32* %A, align 4
+  %1 = load i32, i32* %B, align 4
+  %add = add nsw i32 %0, %1
+  ret i32 %add
+}
+
+define dso_local i32 @visible_local(i32* %A) #0 {
+entry:
+  %B = alloca i32, align 4
+  store i32 5, i32* %B, align 4
+  %call1 = call i32 @noalias_args(i32* %A, i32* nonnull %B)
+  %call2 = call i32 @noalias_args_argmem(i32* %A, i32* nonnull %B)
+  %add = add nsw i32 %call1, %call2
+  ret i32 %add
+}
+
+attributes #0 = { noinline nounwind uwtable willreturn }
+attributes #1 = { argmemonly noinline nounwind uwtable willreturn}
diff --git a/llvm/test/Transforms/Attributor/liveness.ll b/llvm/test/Transforms/Attributor/liveness.ll
new file mode 100644
index 000000000..4dd37865f
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/liveness.ll
@@ -0,0 +1,948 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; RUN: opt -attributor --attributor-disable=false -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=4 -S < %s | FileCheck %s --check-prefixes=CHECK,OLDPM
+; RUN: opt -passes=attributor --attributor-disable=false -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=4 -S < %s | FileCheck %s --check-prefixes=CHECK,NEWPM
+; UTC_ARGS: --turn off
+
+; CHECK: @dead_with_blockaddress_users.l = constant [2 x i8*] [i8* inttoptr (i32 1 to i8*), i8* inttoptr (i32 1 to i8*)]
+@dead_with_blockaddress_users.l = constant [2 x i8*] [i8* blockaddress(@dead_with_blockaddress_users, %lab0), i8* blockaddress(@dead_with_blockaddress_users, %end)]
+
+declare void @no_return_call() nofree noreturn nounwind readnone
+
+declare void @normal_call() readnone
+
+declare i32 @foo()
+
+declare i32 @foo_nounwind() nounwind
+
+declare i32 @foo_noreturn_nounwind() noreturn nounwind
+
+declare i32 @foo_noreturn() noreturn
+
+declare i32 @bar() nosync readnone
+
+; This internal function has no live call sites, so all its BBs are considered dead,
+; and nothing should be deduced for it.
+
+; CHECK-NOT: define internal i32 @dead_internal_func(i32 %0)
+define internal i32 @dead_internal_func(i32 %0) {
+  %2 = icmp slt i32 %0, 1
+  br i1 %2, label %3, label %5
+
+; <label>:3:                                      ; preds = %5, %1
+  %4 = phi i32 [ 1, %1 ], [ %8, %5 ]
+  ret i32 %4
+
+; <label>:5:                                      ; preds = %1, %5
+  %6 = phi i32 [ %9, %5 ], [ 1, %1 ]
+  %7 = phi i32 [ %8, %5 ], [ 1, %1 ]
+  %8 = mul nsw i32 %6, %7
+  %9 = add nuw nsw i32 %6, 1
+  %10 = icmp eq i32 %6, %0
+  br i1 %10, label %3, label %5
+}
+
+; CHECK: Function Attrs: nofree norecurse nounwind uwtable willreturn
+define i32 @volatile_load(i32*) norecurse nounwind uwtable {
+  %2 = load volatile i32, i32* %0, align 4
+  ret i32 %2
+}
+
+; CHECK-NOT: internal_load
+define internal i32 @internal_load(i32*) norecurse nounwind uwtable {
+  %2 = load i32, i32* %0, align 4
+  ret i32 %2
+}
+; TEST 1: Only first block is live.
+
+; CHECK: Function Attrs: nofree noreturn nosync nounwind
+; CHECK-NEXT: define i32 @first_block_no_return(i32 %a, i32* nocapture nofree nonnull readonly %ptr1, i32* nocapture nofree readnone %ptr2)
+define i32 @first_block_no_return(i32 %a, i32* nonnull %ptr1, i32* %ptr2) #0 {
+entry:
+  call i32 @internal_load(i32* %ptr1)
+  call void @no_return_call()
+  ; CHECK: call void @no_return_call()
+  ; CHECK-NEXT: unreachable
+  call i32 @dead_internal_func(i32 10)
+  %cmp = icmp eq i32 %a, 0
+  br i1 %cmp, label %cond.true, label %cond.false
+
+cond.true:                                        ; preds = %entry
+  call i32 @internal_load(i32* %ptr2)
+  %load = call i32 @volatile_load(i32* %ptr1)
+  call void @normal_call()
+  %call = call i32 @foo()
+  br label %cond.end
+
+cond.false:                                       ; preds = %entry
+  call void @normal_call()
+  %call1 = call i32 @bar()
+  br label %cond.end
+
+cond.end:                                         ; preds = %cond.false, %cond.true
+  %cond = phi i32 [ %call, %cond.true ], [ %call1, %cond.false ]
+  ret i32 %cond
+}
+
+; TEST 2: cond.true is dead, but cond.end is not, since cond.false is live
+
+; This is just an example. For example we can put a sync call in a
+; dead block and check if it is deduced.
+
+; CHECK: Function Attrs: nosync
+; CHECK-NEXT: define i32 @dead_block_present(i32 %a, i32* nocapture nofree readnone %ptr1)
+define i32 @dead_block_present(i32 %a, i32* %ptr1) #0 {
+entry:
+  %cmp = icmp eq i32 %a, 0
+  br i1 %cmp, label %cond.true, label %cond.false
+
+cond.true:                                        ; preds = %entry
+  call void @no_return_call()
+  ; CHECK: call void @no_return_call()
+  ; CHECK-NEXT: unreachable
+  %call = call i32 @volatile_load(i32* %ptr1)
+  br label %cond.end
+
+cond.false:                                       ; preds = %entry
+  call void @normal_call()
+  %call1 = call i32 @bar()
+  br label %cond.end
+
+cond.end:                                         ; preds = %cond.false, %cond.true
+; CHECK:      cond.end:
+; CHECK-NEXT:   ret i32 %call1
+  %cond = phi i32 [ %call, %cond.true ], [ %call1, %cond.false ]
+  ret i32 %cond
+}
+
+; TEST 3: both cond.true and cond.false are dead, therfore cond.end is dead as well.
+
+define i32 @all_dead(i32 %a) #0 {
+entry:
+  %cmp = icmp eq i32 %a, 0
+  br i1 %cmp, label %cond.true, label %cond.false
+
+cond.true:                                        ; preds = %entry
+  call void @no_return_call()
+  ; CHECK: call void @no_return_call()
+  ; CHECK-NEXT: unreachable
+  call i32 @dead_internal_func(i32 10)
+  ; CHECK-NOT: call
+  %call = call i32 @foo()
+  br label %cond.end
+
+cond.false:                                       ; preds = %entry
+  call void @no_return_call()
+  ; CHECK: call void @no_return_call()
+  ; CHECK-NEXT: unreachable
+  call i32 @dead_internal_func(i32 10)
+  %call1 = call i32 @bar()
+  br label %cond.end
+
+cond.end:                                         ; preds = %cond.false, %cond.true
+  %cond = phi i32 [ %call, %cond.true ], [ %call1, %cond.false ]
+  ret i32 %cond
+}
+
+declare i32 @__gxx_personality_v0(...)
+
+; TEST 4: All blocks are live.
+
+; CHECK: define i32 @all_live(i32 %a)
+define i32 @all_live(i32 %a) #0 {
+entry:
+  %cmp = icmp eq i32 %a, 0
+  br i1 %cmp, label %cond.true, label %cond.false
+
+cond.true:                                        ; preds = %entry
+  call void @normal_call()
+  %call = call i32 @foo_noreturn()
+  br label %cond.end
+
+cond.false:                                       ; preds = %entry
+  call void @normal_call()
+  %call1 = call i32 @bar()
+  br label %cond.end
+
+cond.end:                                         ; preds = %cond.false, %cond.true
+  %cond = phi i32 [ %call, %cond.true ], [ %call1, %cond.false ]
+  ret i32 %cond
+}
+
+; TEST 5.1 noreturn invoke instruction with a unreachable normal successor block.
+
+; CHECK: define i32 @invoke_noreturn(i32 %a)
+define i32 @invoke_noreturn(i32 %a) personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+entry:
+  %cmp = icmp eq i32 %a, 0
+  br i1 %cmp, label %cond.true, label %cond.false
+
+cond.true:                                        ; preds = %entry
+  call void @normal_call()
+  %call = invoke i32 @foo_noreturn() to label %continue
+            unwind label %cleanup
+  ; CHECK:      %call = invoke i32 @foo_noreturn()
+  ; CHECK-NEXT:         to label %continue unwind label %cleanup
+
+cond.false:                                       ; preds = %entry
+  call void @normal_call()
+  %call1 = call i32 @bar()
+  br label %cond.end
+
+cond.end:                                         ; preds = %cond.false, %continue
+  %cond = phi i32 [ %call, %continue ], [ %call1, %cond.false ]
+  ret i32 %cond
+
+continue:
+  ; CHECK:      continue:
+  ; CHECK-NEXT: unreachable
+  br label %cond.end
+
+cleanup:
+  %res = landingpad { i8*, i32 }
+  catch i8* null
+  ret i32 0
+}
+
+; TEST 5.2 noreturn invoke instruction replaced by a call and an unreachable instruction
+; put after it.
+
+; CHECK: define i32 @invoke_noreturn_nounwind(i32 %a)
+define i32 @invoke_noreturn_nounwind(i32 %a) personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+entry:
+  %cmp = icmp eq i32 %a, 0
+  br i1 %cmp, label %cond.true, label %cond.false
+
+cond.true:                                        ; preds = %entry
+  call void @normal_call()
+  %call = invoke i32 @foo_noreturn_nounwind() to label %continue
+            unwind label %cleanup
+  ; CHECK:      call void @normal_call()
+  ; CHECK-NEXT: call i32 @foo_noreturn_nounwind()
+  ; CHECK-NEXT: unreachable
+
+  ; CHECK-NOT:      @foo_noreturn_nounwind()
+
+cond.false:                                       ; preds = %entry
+  call void @normal_call()
+  %call1 = call i32 @bar()
+  br label %cond.end
+
+cond.end:                                         ; preds = %cond.false, %continue
+  %cond = phi i32 [ %call, %continue ], [ %call1, %cond.false ]
+  ret i32 %cond
+
+continue:
+  br label %cond.end
+
+cleanup:
+  %res = landingpad { i8*, i32 }
+  catch i8* null
+  ret i32 0
+}
+
+; TEST 5.3 unounwind invoke instruction replaced by a call and a branch instruction put after it.
+define i32 @invoke_nounwind(i32 %a) personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+; CHECK-LABEL: define {{[^@]+}}@invoke_nounwind
+; CHECK:       cond.true:
+; CHECK-NEXT:    call void @normal_call()
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @foo_nounwind()
+; CHECK-NEXT:    br label [[CONTINUE:%.*]]
+; CHECK:       continue:
+; CHECK-NEXT:    br label [[COND_END:%.*]]
+;
+entry:
+  %cmp = icmp eq i32 %a, 0
+  br i1 %cmp, label %cond.true, label %cond.false
+
+cond.true:                                        ; preds = %entry
+  call void @normal_call()
+  %call = invoke i32 @foo_nounwind() to label %continue
+  unwind label %cleanup
+
+cond.false:                                       ; preds = %entry
+  call void @normal_call()
+  %call1 = call i32 @bar()
+  br label %cond.end
+
+cond.end:                                         ; preds = %cond.false, %continue
+  %cond = phi i32 [ %call, %continue ], [ %call1, %cond.false ]
+  ret i32 %cond
+
+continue:
+  br label %cond.end
+
+cleanup:
+  %res = landingpad { i8*, i32 }
+  catch i8* null
+  ret i32 0
+}
+
+; UTC_ARGS: --turn on
+
+; TEST 5.4 unounwind invoke instruction replaced by a call and a branch instruction put after it.
+define i32 @invoke_nounwind_phi(i32 %a) personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+; CHECK-LABEL: define {{[^@]+}}@invoke_nounwind_phi
+; CHECK-SAME: (i32 [[A:%.*]]) personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[A]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+; CHECK:       cond.true:
+; CHECK-NEXT:    call void @normal_call()
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @foo_nounwind()
+; CHECK-NEXT:    br label [[CONTINUE:%.*]]
+; CHECK:       cond.false:
+; CHECK-NEXT:    call void @normal_call()
+; CHECK-NEXT:    [[CALL1:%.*]] = call i32 @bar()
+; CHECK-NEXT:    br label [[CONTINUE]]
+; CHECK:       continue:
+; CHECK-NEXT:    [[P:%.*]] = phi i32 [ 0, [[COND_TRUE]] ], [ 1, [[COND_FALSE]] ]
+; CHECK-NEXT:    ret i32 [[P]]
+; CHECK:       cleanup:
+; CHECK-NEXT:    unreachable
+;
+entry:
+  %cmp = icmp eq i32 %a, 0
+  br i1 %cmp, label %cond.true, label %cond.false
+
+cond.true:                                        ; preds = %entry
+  call void @normal_call()
+  %call = invoke i32 @foo_nounwind() to label %continue
+  unwind label %cleanup
+
+cond.false:                                       ; preds = %entry
+  call void @normal_call()
+  %call1 = call i32 @bar()
+  br label %continue
+
+continue:
+  %p = phi i32 [ 0, %cond.true ], [ 1, %cond.false ]
+  ret i32 %p
+
+cleanup:
+  %res = landingpad { i8*, i32 } catch i8* null
+  ret i32 0
+}
+
+; TEST 5.5 unounwind invoke instruction replaced by a call and a branch instruction put after it.
+define i32 @invoke_nounwind_phi_dom(i32 %a) personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+; CHECK-LABEL: define {{[^@]+}}@invoke_nounwind_phi_dom
+; CHECK-SAME: (i32 [[A:%.*]]) personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[A]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+; CHECK:       cond.true:
+; CHECK-NEXT:    call void @normal_call()
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @foo_nounwind()
+; CHECK-NEXT:    br label [[CONTINUE:%.*]]
+; CHECK:       cond.false:
+; CHECK-NEXT:    call void @normal_call()
+; CHECK-NEXT:    [[CALL1:%.*]] = call i32 @bar()
+; CHECK-NEXT:    br label [[CONTINUE]]
+; CHECK:       continue:
+; CHECK-NEXT:    [[P:%.*]] = phi i32 [ [[CALL]], [[COND_TRUE]] ], [ [[CALL1]], [[COND_FALSE]] ]
+; CHECK-NEXT:    ret i32 [[P]]
+; CHECK:       cleanup:
+; CHECK-NEXT:    unreachable
+;
+entry:
+  %cmp = icmp eq i32 %a, 0
+  br i1 %cmp, label %cond.true, label %cond.false
+
+cond.true:                                        ; preds = %entry
+  call void @normal_call()
+  %call = invoke i32 @foo_nounwind() to label %continue
+  unwind label %cleanup
+
+cond.false:                                       ; preds = %entry
+  call void @normal_call()
+  %call1 = call i32 @bar()
+  br label %continue
+
+continue:
+  %p = phi i32 [ %call, %cond.true ], [ %call1, %cond.false ]
+  ret i32 %p
+
+cleanup:
+  %res = landingpad { i8*, i32 } catch i8* null
+  ret i32 0
+}
+
+; UTC_ARGS: --turn off
+
+; TEST 6: Undefined behvior, taken from LangRef.
+; FIXME: Should be able to detect undefined behavior.
+
+; CHECK: define void @ub(i32* nocapture nofree writeonly %0)
+define void @ub(i32* %0) {
+  %poison = sub nuw i32 0, 1           ; Results in a poison value.
+  %still_poison = and i32 %poison, 0   ; 0, but also poison.
+  %poison_yet_again = getelementptr i32, i32* %0, i32 %still_poison
+  store i32 0, i32* %poison_yet_again  ; Undefined behavior due to store to poison.
+  ret void
+}
+
+define void @inf_loop() #0 {
+entry:
+  br label %while.body
+
+while.body:                                       ; preds = %entry, %while.body
+  br label %while.body
+}
+
+; TEST 7: Infinite loop.
+; FIXME: Detect infloops, and mark affected blocks dead.
+
+define i32 @test5(i32, i32) #0 {
+  %3 = icmp sgt i32 %0, %1
+  br i1 %3, label %cond.if, label %cond.elseif
+
+cond.if:                                                ; preds = %2
+  %4 = tail call i32 @bar()
+  br label %cond.end
+
+cond.elseif:                                                ; preds = %2
+  call void @inf_loop()
+  %5 = icmp slt i32 %0, %1
+  br i1 %5, label %cond.end, label %cond.else
+
+cond.else:                                                ; preds = %cond.elseif
+  %6 = tail call i32 @foo()
+  br label %cond.end
+
+cond.end:                                               ; preds = %cond.if, %cond.else, %cond.elseif
+  %7 = phi i32 [ %1, %cond.elseif ], [ 0, %cond.else ], [ 0, %cond.if ]
+  ret i32 %7
+}
+
+define void @rec() #0 {
+entry:
+  call void @rec()
+  ret void
+}
+
+; TEST 8: Recursion
+; FIXME: everything after first block should be marked dead
+; and unreachable should be put after call to @rec().
+
+define i32 @test6(i32, i32) #0 {
+  call void @rec()
+  %3 = icmp sgt i32 %0, %1
+  br i1 %3, label %cond.if, label %cond.elseif
+
+cond.if:                                                ; preds = %2
+  %4 = tail call i32 @bar()
+  br label %cond.end
+
+cond.elseif:                                                ; preds = %2
+  call void @rec()
+  %5 = icmp slt i32 %0, %1
+  br i1 %5, label %cond.end, label %cond.else
+
+cond.else:                                                ; preds = %cond.elseif
+  %6 = tail call i32 @foo()
+  br label %cond.end
+
+cond.end:                                               ; preds = %cond.if, %cond.else, %cond.elseif
+  %7 = phi i32 [ %1, %cond.elseif ], [ 0, %cond.else ], [ 0, %cond.if ]
+  ret i32 %7
+}
+; TEST 9: Recursion
+; FIXME: contains recursive call to itself in cond.elseif block
+
+define i32 @test7(i32, i32) #0 {
+  %3 = icmp sgt i32 %0, %1
+  br i1 %3, label %cond.if, label %cond.elseif
+
+cond.if:                                                ; preds = %2
+  %4 = tail call i32 @bar()
+  br label %cond.end
+
+cond.elseif:                                                ; preds = %2
+  %5 = tail call i32 @test7(i32 %0, i32 %1)
+  %6 = icmp slt i32 %0, %1
+  br i1 %6, label %cond.end, label %cond.else
+
+cond.else:                                                ; preds = %cond.elseif
+  %7 = tail call i32 @foo()
+  br label %cond.end
+
+cond.end:                                               ; preds = %cond.if, %cond.else, %cond.elseif
+  %8 = phi i32 [ %1, %cond.elseif ], [ 0, %cond.else ], [ 0, %cond.if ]
+  ret i32 %8
+}
+
+; SCC test
+;
+; char a1 __attribute__((aligned(8)));
+; char a2 __attribute__((aligned(16)));
+;
+; char* f1(char* a ){
+;     return a?a:f2(&a1);
+; }
+; char* f2(char* a){
+;     return a?f1(a):f3(&a2);
+; }
+;
+; char* f3(char* a){
+;     return a?&a1: f1(&a2);
+; }
+
+@a1 = common global i8 0, align 8
+@a2 = common global i8 0, align 16
+
+define internal i8* @f1(i8* readnone %0) local_unnamed_addr #0 {
+; ATTRIBUTOR: define internal i8* @f1(i8* readnone %0)
+  %2 = icmp eq i8* %0, null
+  br i1 %2, label %3, label %5
+
+; <label>:3:                                      ; preds = %1
+; ATTRIBUTOR: %4 = tail call i8* undef(i8* nonnull align 8 @a1)
+  %4 = tail call i8* @f2(i8* nonnull @a1)
+  br label %5
+
+; <label>:5:                                      ; preds = %1, %3
+  %6 = phi i8* [ %4, %3 ], [ %0, %1 ]
+  ret i8* %6
+}
+
+define internal i8* @f2(i8* readnone %0) local_unnamed_addr #0 {
+; ATTRIBUTOR: define internal i8* @f2(i8* readnone %0)
+  %2 = icmp eq i8* %0, null
+  br i1 %2, label %5, label %3
+
+; <label>:3:                                      ; preds = %1
+
+; ATTRIBUTOR: %4 = tail call i8* undef(i8* nonnull align 8 %0)
+  %4 = tail call i8* @f1(i8* nonnull %0)
+  br label %7
+
+; <label>:5:                                      ; preds = %1
+; ATTRIBUTOR: %6 = tail call i8* undef(i8* nonnull align 16 @a2)
+  %6 = tail call i8* @f3(i8* nonnull @a2)
+  br label %7
+
+; <label>:7:                                      ; preds = %5, %3
+  %8 = phi i8* [ %4, %3 ], [ %6, %5 ]
+  ret i8* %8
+}
+
+define internal i8* @f3(i8* readnone %0) local_unnamed_addr #0 {
+; ATTRIBUTOR: define internal i8* @f3(i8* readnone %0)
+  %2 = icmp eq i8* %0, null
+  br i1 %2, label %3, label %5
+
+; <label>:3:                                      ; preds = %1
+; ATTRIBUTOR: %4 = tail call i8* undef(i8* nonnull align 16 @a2)
+  %4 = tail call i8* @f1(i8* nonnull @a2)
+  br label %5
+
+; <label>:5:                                      ; preds = %1, %3
+  %6 = phi i8* [ %4, %3 ], [ @a1, %1 ]
+  ret i8* %6
+}
+
+define void @test_unreachable() {
+; CHECK:       define void @test_unreachable()
+; CHECK-NEXT:    call void @test_unreachable()
+; CHECK-NEXT:    unreachable
+; CHECK-NEXT:  }
+  call void @test_unreachable()
+  unreachable
+}
+
+define linkonce_odr void @non_exact1() {
+  call void @non_dead_a0()
+  call void @non_dead_a1()
+  call void @non_dead_a2()
+  call void @non_dead_a3()
+  call void @non_dead_a4()
+  call void @non_dead_a5()
+  call void @non_dead_a6()
+  call void @non_dead_a7()
+  call void @non_dead_a8()
+  call void @non_dead_a9()
+  call void @non_dead_a10()
+  call void @non_dead_a11()
+  call void @non_dead_a12()
+  call void @non_dead_a13()
+  call void @non_dead_a14()
+  call void @non_dead_a15()
+  call void @middle()
+  ret void
+}
+define internal void @middle() {
+bb0:
+  call void @non_dead_b0()
+  call void @non_dead_b1()
+  call void @non_dead_b2()
+  call void @non_dead_b3()
+br label %bb1
+bb1:
+  call void @non_dead_b4()
+  call void @non_dead_b5()
+  call void @non_dead_b6()
+  call void @non_dead_b7()
+br label %bb2
+bb2:
+  call void @non_dead_b8()
+  call void @non_dead_b9()
+  call void @non_dead_b10()
+  call void @non_dead_b11()
+br label %bb3
+bb3:
+  call void @non_dead_b12()
+  call void @non_dead_b13()
+  call void @non_dead_b14()
+  call void @non_dead_b15()
+br label %bb4
+bb4:
+  call void @non_exact2()
+  ret void
+}
+define linkonce_odr void @non_exact2() {
+  call void @non_dead_c0()
+  call void @non_dead_c1()
+  call void @non_dead_c2()
+  call void @non_dead_c3()
+  call void @non_dead_c4()
+  call void @non_dead_c5()
+  call void @non_dead_c6()
+  call void @non_dead_c7()
+  call void @non_dead_c8()
+  call void @non_dead_c9()
+  call void @non_dead_c10()
+  call void @non_dead_c11()
+  call void @non_dead_c12()
+  call void @non_dead_c13()
+  call void @non_dead_c14()
+  call void @non_dead_c15()
+  call void @non_exact3()
+  ret void
+}
+define linkonce_odr void @non_exact3() {
+  call void @non_dead_d0()
+  call void @non_dead_d1()
+  call void @non_dead_d2()
+  call void @non_dead_d3()
+  call void @non_dead_d4()
+  call void @non_dead_d5()
+  call void @non_dead_d6()
+  call void @non_dead_d7()
+  call void @non_dead_d8()
+  call void @non_dead_d9()
+  call void @non_dead_d10()
+  call void @non_dead_d11()
+  call void @non_dead_d12()
+  call void @non_dead_d13()
+  call void @non_dead_d14()
+  call void @non_dead_d15()
+  %nr = call i32 @foo_noreturn()
+  call void @dead_e1()
+  ret void
+}
+; CHECK:       define linkonce_odr void @non_exact3() {
+; CHECK-NEXT:   call void @non_dead_d0()
+; CHECK-NEXT:   call void @non_dead_d1()
+; CHECK-NEXT:   call void @non_dead_d2()
+; CHECK-NEXT:   call void @non_dead_d3()
+; CHECK-NEXT:   call void @non_dead_d4()
+; CHECK-NEXT:   call void @non_dead_d5()
+; CHECK-NEXT:   call void @non_dead_d6()
+; CHECK-NEXT:   call void @non_dead_d7()
+; CHECK-NEXT:   call void @non_dead_d8()
+; CHECK-NEXT:   call void @non_dead_d9()
+; CHECK-NEXT:   call void @non_dead_d10()
+; CHECK-NEXT:   call void @non_dead_d11()
+; CHECK-NEXT:   call void @non_dead_d12()
+; CHECK-NEXT:   call void @non_dead_d13()
+; CHECK-NEXT:   call void @non_dead_d14()
+; CHECK-NEXT:   call void @non_dead_d15()
+; CHECK-NEXT:   %nr = call i32 @foo_noreturn()
+; CHECK-NEXT:   unreachable
+
+define internal void @non_dead_a0() { ret void }
+define internal void @non_dead_a1() { ret void }
+define internal void @non_dead_a2() { ret void }
+define internal void @non_dead_a3() { ret void }
+define internal void @non_dead_a4() { ret void }
+define internal void @non_dead_a5() { ret void }
+define internal void @non_dead_a6() { ret void }
+define internal void @non_dead_a7() { ret void }
+define internal void @non_dead_a8() { ret void }
+define internal void @non_dead_a9() { ret void }
+define internal void @non_dead_a10() { ret void }
+define internal void @non_dead_a11() { ret void }
+define internal void @non_dead_a12() { ret void }
+define internal void @non_dead_a13() { ret void }
+define internal void @non_dead_a14() { ret void }
+define internal void @non_dead_a15() { ret void }
+define internal void @non_dead_b0() { ret void }
+define internal void @non_dead_b1() { ret void }
+define internal void @non_dead_b2() { ret void }
+define internal void @non_dead_b3() { ret void }
+define internal void @non_dead_b4() { ret void }
+define internal void @non_dead_b5() { ret void }
+define internal void @non_dead_b6() { ret void }
+define internal void @non_dead_b7() { ret void }
+define internal void @non_dead_b8() { ret void }
+define internal void @non_dead_b9() { ret void }
+define internal void @non_dead_b10() { ret void }
+define internal void @non_dead_b11() { ret void }
+define internal void @non_dead_b12() { ret void }
+define internal void @non_dead_b13() { ret void }
+define internal void @non_dead_b14() { ret void }
+define internal void @non_dead_b15() { ret void }
+define internal void @non_dead_c0() { ret void }
+define internal void @non_dead_c1() { ret void }
+define internal void @non_dead_c2() { ret void }
+define internal void @non_dead_c3() { ret void }
+define internal void @non_dead_c4() { ret void }
+define internal void @non_dead_c5() { ret void }
+define internal void @non_dead_c6() { ret void }
+define internal void @non_dead_c7() { ret void }
+define internal void @non_dead_c8() { ret void }
+define internal void @non_dead_c9() { ret void }
+define internal void @non_dead_c10() { ret void }
+define internal void @non_dead_c11() { ret void }
+define internal void @non_dead_c12() { ret void }
+define internal void @non_dead_c13() { ret void }
+define internal void @non_dead_c14() { ret void }
+define internal void @non_dead_c15() { ret void }
+define internal void @non_dead_d0() { ret void }
+define internal void @non_dead_d1() { ret void }
+define internal void @non_dead_d2() { ret void }
+define internal void @non_dead_d3() { ret void }
+define internal void @non_dead_d4() { ret void }
+define internal void @non_dead_d5() { ret void }
+define internal void @non_dead_d6() { ret void }
+define internal void @non_dead_d7() { ret void }
+define internal void @non_dead_d8() { ret void }
+define internal void @non_dead_d9() { ret void }
+define internal void @non_dead_d10() { ret void }
+define internal void @non_dead_d11() { ret void }
+define internal void @non_dead_d12() { ret void }
+define internal void @non_dead_d13() { ret void }
+define internal void @non_dead_d14() { ret void }
+define internal void @non_dead_d15() { ret void }
+define internal void @dead_e0() { call void @dead_e1() ret void }
+define internal void @dead_e1() { call void @dead_e2() ret void }
+define internal void @dead_e2() { ret void }
+
+; CHECK: define internal void @non_dead_a0()
+; CHECK: define internal void @non_dead_a1()
+; CHECK: define internal void @non_dead_a2()
+; CHECK: define internal void @non_dead_a3()
+; CHECK: define internal void @non_dead_a4()
+; CHECK: define internal void @non_dead_a5()
+; CHECK: define internal void @non_dead_a6()
+; CHECK: define internal void @non_dead_a7()
+; CHECK: define internal void @non_dead_a8()
+; CHECK: define internal void @non_dead_a9()
+; CHECK: define internal void @non_dead_a10()
+; CHECK: define internal void @non_dead_a11()
+; CHECK: define internal void @non_dead_a12()
+; CHECK: define internal void @non_dead_a13()
+; CHECK: define internal void @non_dead_a14()
+; CHECK: define internal void @non_dead_a15()
+; CHECK: define internal void @non_dead_b0()
+; CHECK: define internal void @non_dead_b1()
+; CHECK: define internal void @non_dead_b2()
+; CHECK: define internal void @non_dead_b3()
+; CHECK: define internal void @non_dead_b4()
+; CHECK: define internal void @non_dead_b5()
+; CHECK: define internal void @non_dead_b6()
+; CHECK: define internal void @non_dead_b7()
+; CHECK: define internal void @non_dead_b8()
+; CHECK: define internal void @non_dead_b9()
+; CHECK: define internal void @non_dead_b10()
+; CHECK: define internal void @non_dead_b11()
+; CHECK: define internal void @non_dead_b12()
+; CHECK: define internal void @non_dead_b13()
+; CHECK: define internal void @non_dead_b14()
+; CHECK: define internal void @non_dead_b15()
+; CHECK: define internal void @non_dead_c0()
+; CHECK: define internal void @non_dead_c1()
+; CHECK: define internal void @non_dead_c2()
+; CHECK: define internal void @non_dead_c3()
+; CHECK: define internal void @non_dead_c4()
+; CHECK: define internal void @non_dead_c5()
+; CHECK: define internal void @non_dead_c6()
+; CHECK: define internal void @non_dead_c7()
+; CHECK: define internal void @non_dead_c8()
+; CHECK: define internal void @non_dead_c9()
+; CHECK: define internal void @non_dead_c10()
+; CHECK: define internal void @non_dead_c11()
+; CHECK: define internal void @non_dead_c12()
+; CHECK: define internal void @non_dead_c13()
+; CHECK: define internal void @non_dead_c14()
+; CHECK: define internal void @non_dead_c15()
+; CHECK: define internal void @non_dead_d0()
+; CHECK: define internal void @non_dead_d1()
+; CHECK: define internal void @non_dead_d2()
+; CHECK: define internal void @non_dead_d3()
+; CHECK: define internal void @non_dead_d4()
+; CHECK: define internal void @non_dead_d5()
+; CHECK: define internal void @non_dead_d6()
+; CHECK: define internal void @non_dead_d7()
+; CHECK: define internal void @non_dead_d8()
+; CHECK: define internal void @non_dead_d9()
+; CHECK: define internal void @non_dead_d10()
+; CHECK: define internal void @non_dead_d11()
+; CHECK: define internal void @non_dead_d12()
+; CHECK: define internal void @non_dead_d13()
+; CHECK: define internal void @non_dead_d14()
+; Verify we actually deduce information for these functions.
+; OLDPM: Function Attrs: nofree nosync nounwind readnone willreturn
+; NEWPM: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; CHECK-NEXT: define internal void @non_dead_d15()
+; CHECK-NOT: define internal void @dead_e
+
+declare void @blowup() noreturn
+define void @live_with_dead_entry() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+; CHECK:      define void @live_with_dead_entry(
+; CHECK-NEXT: entry:
+; CHECK-NEXT:   invoke void @blowup()
+; CHECK-NEXT:           to label %live_with_dead_entry.dead unwind label %lpad
+; CHECK:      lpad:                                             ; preds = %entry
+; CHECK-NEXT:   %0 = landingpad { i8*, i32 }
+; CHECK-NEXT:           catch i8* null
+; CHECK-NEXT:   br label %live_with_dead_entry
+; CHECK:      live_with_dead_entry.dead:                        ; preds = %entry
+; CHECK-NEXT:   unreachable
+; CHECK:      live_with_dead_entry:                             ; preds = %lpad
+; CHECK-NEXT:   ret void
+entry:
+  invoke void @blowup() to label %live_with_dead_entry unwind label %lpad
+lpad:
+  %0 = landingpad { i8*, i32 } catch i8* null
+  br label %live_with_dead_entry
+live_with_dead_entry:
+  ret void
+}
+
+define void @live_with_dead_entry_lp() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+; CHECK:      define void @live_with_dead_entry_lp(
+; CHECK-NEXT: entry:
+; CHECK-NEXT:   invoke void @blowup()
+; CHECK-NEXT:    to label %[[LIVE_WITH_DEAD_ENTRY_DEAD1:.*]] unwind label %[[LP1:.*]]
+; CHECK:      [[LP1]]:                                              ; preds = %entry
+; CHECK-NEXT:   %lp = landingpad { i8*, i32 }
+; CHECK-NEXT:           catch i8* null
+; CHECK-NEXT:   invoke void @blowup()
+; CHECK-NEXT:    to label %[[LIVE_WITH_DEAD_ENTRY_DEAD2:.*]] unwind label %[[LP2:.*]]
+; CHECK:      [[LP2]]:                                              ; preds = %lp1
+; CHECK-NEXT:   %0 = landingpad { i8*, i32 }
+; CHECK-NEXT:           catch i8* null
+; CHECK-NEXT:   br label %live_with_dead_entry
+; CHECK:      [[LIVE_WITH_DEAD_ENTRY_DEAD1]]:
+; CHECK-NEXT:   unreachable
+; CHECK:      [[LIVE_WITH_DEAD_ENTRY_DEAD2]]:
+; CHECK-NEXT:   unreachable
+; CHECK:      live_with_dead_entry:                             ; preds = %lp2
+; CHECK-NEXT:   ret void
+entry:
+  invoke void @blowup() to label %live_with_dead_entry unwind label %lp1
+lp1:
+  %lp = landingpad { i8*, i32 } catch i8* null
+  invoke void @blowup() to label %live_with_dead_entry unwind label %lp2
+lp2:
+  %0 = landingpad { i8*, i32 } catch i8* null
+  br label %live_with_dead_entry
+live_with_dead_entry:
+  ret void
+}
+
+; CHECK: define internal void @useless_arg_sink()
+define internal void @useless_arg_sink(i32* %a) {
+  ret void
+}
+
+; CHECK: define internal void @useless_arg_almost_sink()
+define internal void @useless_arg_almost_sink(i32* %a) {
+; CHECK: call void @useless_arg_sink()
+  call void @useless_arg_sink(i32* %a)
+  ret void
+}
+
+; Check we do not annotate the function interface of this weak function.
+; CHECK: define weak_odr void @useless_arg_ext(i32* %a)
+define weak_odr void @useless_arg_ext(i32* %a) {
+; CHECK: call void @useless_arg_almost_sink()
+  call void @useless_arg_almost_sink(i32* %a)
+  ret void
+}
+
+; CHECK: define internal void @useless_arg_ext_int(i32* %a)
+define internal void @useless_arg_ext_int(i32* %a) {
+; CHECK: call void @useless_arg_ext(i32* %a)
+  call void @useless_arg_ext(i32* %a)
+  ret void
+}
+
+define void @useless_arg_ext_int_ext(i32* %a) {
+; CHECK: call void @useless_arg_ext_int(i32* %a)
+  call void @useless_arg_ext_int(i32* %a)
+  ret void
+}
+
+; UTC_ARGS: --turn on
+
+; FIXME: We should fold terminators.
+
+define internal i32 @switch_default(i64 %i) nounwind {
+; CHECK-LABEL: define {{[^@]+}}@switch_default
+; CHECK-SAME: (i64 [[I:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    switch i64 0, label [[SW_DEFAULT:%.*]] [
+; CHECK-NEXT:    i64 3, label [[RETURN:%.*]]
+; CHECK-NEXT:    i64 10, label [[RETURN]]
+; CHECK-NEXT:    ]
+; CHECK:       sw.default:
+; CHECK-NEXT:    ret i32 123
+; CHECK:       return:
+; CHECK-NEXT:    unreachable
+;
+entry:
+  switch i64 %i, label %sw.default [
+  i64 3, label %return
+  i64 10, label %return
+  ]
+
+sw.default:
+  ret i32 123
+
+return:
+  ret i32 0
+}
+
+define i32 @switch_default_caller() {
+; CHECK-LABEL: define {{[^@]+}}@switch_default_caller()
+; CHECK-NEXT:    [[CALL2:%.*]] = tail call i32 @switch_default(i64 0)
+; CHECK-NEXT:    ret i32 123
+;
+  %call2 = tail call i32 @switch_default(i64 0)
+  ret i32 %call2
+}
+; UTC_ARGS: --turn off
+
+; Allow blockaddress users
+; CHECK-NOT @dead_with_blockaddress_users
+define internal void @dead_with_blockaddress_users(i32* nocapture %pc) nounwind readonly {
+entry:
+  br label %indirectgoto
+
+lab0:                                             ; preds = %indirectgoto
+  %indvar.next = add i32 %indvar, 1               ; <i32> [#uses=1]
+  br label %indirectgoto
+
+end:                                              ; preds = %indirectgoto
+  ret void
+
+indirectgoto:                                     ; preds = %lab0, %entry
+  %indvar = phi i32 [ %indvar.next, %lab0 ], [ 0, %entry ] ; <i32> [#uses=2]
+  %pc.addr.0 = getelementptr i32, i32* %pc, i32 %indvar ; <i32*> [#uses=1]
+  %tmp1.pn = load i32, i32* %pc.addr.0                 ; <i32> [#uses=1]
+  %indirect.goto.dest.in = getelementptr inbounds [2 x i8*], [2 x i8*]* @dead_with_blockaddress_users.l, i32 0, i32 %tmp1.pn ; <i8**> [#uses=1]
+  %indirect.goto.dest = load i8*, i8** %indirect.goto.dest.in ; <i8*> [#uses=1]
+  indirectbr i8* %indirect.goto.dest, [label %lab0, label %end]
+}
diff --git a/llvm/test/Transforms/Attributor/lvi-after-jumpthreading.ll b/llvm/test/Transforms/Attributor/lvi-after-jumpthreading.ll
new file mode 100644
index 000000000..7dbf8d073
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/lvi-after-jumpthreading.ll
@@ -0,0 +1,181 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -passes=attributor -attributor-disable=false -S < %s | FileCheck %s
+
+define i8 @test1(i32 %a, i32 %length) {
+; CHECK-LABEL: define {{[^@]+}}@test1
+; CHECK-SAME: (i32 [[A:%.*]], i32 [[LENGTH:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
+; CHECK-NEXT:    br label [[BACKEDGE]]
+; CHECK:       backedge:
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i32 [[IV]], 1
+; CHECK-NEXT:    [[CONT:%.*]] = icmp slt i32 [[IV_NEXT]], 400
+; CHECK-NEXT:    br i1 [[CONT]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i8 0
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [0, %entry], [%iv.next, %backedge]
+  %cnd = icmp sge i32 %iv, 0
+  br i1 %cnd, label %backedge, label %exit
+
+backedge:
+  %iv.next = add nsw i32 %iv, 1
+  %cont = icmp slt i32 %iv.next, 400
+  br i1 %cont, label %loop, label %exit
+
+exit:
+  ret i8 0
+}
+
+define i8 @test2(i32 %n) {
+; CHECK-LABEL: define {{[^@]+}}@test2
+; CHECK-SAME: (i32 [[N:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
+; CHECK-NEXT:    [[IV2:%.*]] = phi i32 [ [[N:%.*]], [[ENTRY]] ], [ [[IV2_NEXT:%.*]], [[BACKEDGE]] ]
+; CHECK-NEXT:    [[CND1:%.*]] = icmp sge i32 [[IV]], 0
+; CHECK-NEXT:    [[CND2:%.*]] = icmp sgt i32 [[IV2]], 0
+; CHECK-NEXT:    [[CND:%.*]] = and i1 [[CND1]], [[CND2]]
+; CHECK-NEXT:    br i1 [[CND]], label [[BACKEDGE]], label [[EXIT:%.*]]
+; CHECK:       backedge:
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i32 [[IV]], 1
+; CHECK-NEXT:    [[IV2_NEXT]] = sub nsw i32 [[IV2]], 1
+; CHECK-NEXT:    [[CONT1:%.*]] = icmp slt i32 [[IV_NEXT]], 400
+; CHECK-NEXT:    [[CONT2:%.*]] = icmp sgt i32 [[IV2_NEXT]], 0
+; CHECK-NEXT:    [[CONT:%.*]] = and i1 [[CONT1]], [[CONT2]]
+; CHECK-NEXT:    br i1 [[CONT]], label [[LOOP]], label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i8 0
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [0, %entry], [%iv.next, %backedge]
+  %iv2 = phi i32 [%n, %entry], [%iv2.next, %backedge]
+
+  %cnd1 = icmp sge i32 %iv, 0
+  %cnd2 = icmp sgt i32 %iv2, 0
+  %cnd = and i1 %cnd1, %cnd2
+  br i1 %cnd, label %backedge, label %exit
+
+backedge:
+  %iv.next = add nsw i32 %iv, 1
+  %iv2.next = sub nsw i32 %iv2, 1
+  %cont1 = icmp slt i32 %iv.next, 400
+  %cont2 = icmp sgt i32 %iv2.next, 0
+  %cont = and i1 %cont1, %cont2
+  br i1 %cont, label %loop, label %exit
+
+exit:
+  ret i8 0
+}
+
+; Merging cont block into do block.
+define i32 @test3(i32 %i, i1 %f, i32 %n) {
+; CHECK-LABEL: define {{[^@]+}}@test3
+; CHECK-SAME: (i32 [[I:%.*]], i1 [[F:%.*]], i32 [[N:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[C:%.*]] = icmp ne i32 [[I:%.*]], -2134
+; CHECK-NEXT:    br i1 [[C]], label [[DO:%.*]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[C1:%.*]] = icmp ne i32 [[I]], -42
+; CHECK-NEXT:    br i1 [[C1]], label [[EXIT2:%.*]], label [[EXIT]]
+; CHECK:       cont:
+; CHECK-NEXT:    [[COND_3:%.*]] = icmp sgt i32 [[I]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[COND_3]], label [[EXIT2]], label [[EXIT]]
+; CHECK:       do:
+; CHECK-NEXT:    [[COND_0:%.*]] = icmp sgt i32 [[I]], 0
+; CHECK-NEXT:    [[CONSUME:%.*]] = call i32 @consume(i1 [[COND_0]])
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i32 [[I]], 0
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[COND]]) [ "deopt"() ]
+; CHECK-NEXT:    br label [[CONT:%.*]]
+; CHECK:       exit2:
+; CHECK-NEXT:    ret i32 30
+;
+entry:
+  %c = icmp ne i32 %i, -2134
+  br i1 %c, label %do, label %exit
+
+exit:
+  %c1 = icmp ne i32 %i, -42
+  br i1 %c1, label %exit2, label %exit
+
+; Here cont is merged to do and i is any value except -2134.
+; i is not the single value: zero.
+cont:
+  %cond.3 = icmp sgt i32 %i, %n
+  br i1 %cond.3, label %exit2, label %exit
+
+do:
+  %cond.0 = icmp sgt i32 %i, 0
+  %consume = call i32 @consume(i1 %cond.0)
+  %cond = icmp eq i32 %i, 0
+  call void (i1, ...) @llvm.experimental.guard(i1 %cond) [ "deopt"() ]
+  %cond.2 = icmp sgt i32 %i, 0
+  br i1 %cond.2, label %exit, label %cont
+
+exit2:
+; LatticeVal for: 'i32 %i' is: constantrange<-2134, 1>
+  ret i32 30
+}
+
+; FIXME: We should be able to merge cont into do.
+define i32 @test4(i32 %i, i1 %f, i32 %n) {
+; CHECK-LABEL: define {{[^@]+}}@test4
+; CHECK-SAME: (i32 [[I:%.*]], i1 [[F:%.*]], i32 [[N:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[C:%.*]] = icmp ne i32 [[I:%.*]], -2134
+; CHECK-NEXT:    br i1 [[C]], label [[DO:%.*]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[C1:%.*]] = icmp ne i32 [[I]], -42
+; CHECK-NEXT:    br i1 [[C1]], label [[EXIT2:%.*]], label [[EXIT]]
+; CHECK:       cont:
+; CHECK-NEXT:    call void @dummy(i1 [[F:%.*]])
+; CHECK-NEXT:    br label [[EXIT2]]
+; CHECK:       do:
+; CHECK-NEXT:    call void @dummy(i1 [[F]])
+; CHECK-NEXT:    [[CONSUME:%.*]] = call i32 @exit()
+; CHECK-NEXT:    call void @llvm.assume(i1 [[F]])
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i1 [[F]], false
+; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[CONT:%.*]]
+; CHECK:       exit2:
+; CHECK-NEXT:    ret i32 30
+;
+; FIXME: COND should be replaced with false. This will be fixed by improving LVI.
+entry:
+  %c = icmp ne i32 %i, -2134
+  br i1 %c, label %do, label %exit
+
+exit:                                             ; preds = %do, %cont, %exit, %entry
+  %c1 = icmp ne i32 %i, -42
+  br i1 %c1, label %exit2, label %exit
+
+cont:                                             ; preds = %do
+  call void @dummy(i1 %f)
+  br label %exit2
+
+do:                                               ; preds = %entry
+  call void @dummy(i1 %f)
+  %consume = call i32 @exit()
+  call void @llvm.assume(i1 %f)
+  %cond = icmp eq i1 %f, false
+  br i1 %cond, label %exit, label %cont
+
+exit2:                                            ; preds = %cont, %exit
+  ret i32 30
+}
+
+declare i32 @exit()
+declare i32 @consume(i1)
+declare void @llvm.assume(i1) nounwind
+declare void @dummy(i1) nounwind
+declare void @llvm.experimental.guard(i1, ...)
diff --git a/llvm/test/Transforms/Attributor/lvi-for-ashr.ll b/llvm/test/Transforms/Attributor/lvi-for-ashr.ll
new file mode 100644
index 000000000..ccc5aba33
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/lvi-for-ashr.ll
@@ -0,0 +1,46 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -passes=attributor -attributor-disable=false -S < %s | FileCheck %s
+define i32 @test-ashr(i32 %c) {
+; CHECK-LABEL: define {{[^@]+}}@test-ashr
+; CHECK-SAME: (i32 [[C:%.*]])
+; CHECK-NEXT:  chk65:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[C:%.*]], 65
+; CHECK-NEXT:    br i1 [[CMP]], label [[RETURN:%.*]], label [[CHK0:%.*]]
+; CHECK:       chk0:
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[C]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[RETURN]], label [[BB_IF:%.*]]
+; CHECK:       bb_if:
+; CHECK-NEXT:    [[ASHR_VAL:%.*]] = ashr exact i32 [[C]], 2
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i32 [[ASHR_VAL]], 15
+; CHECK-NEXT:    br i1 [[CMP2]], label [[BB_THEN:%.*]], label [[RETURN]]
+; CHECK:       bb_then:
+; CHECK-NEXT:    [[CMP3:%.*]] = icmp eq i32 [[ASHR_VAL]], 16
+; CHECK-NEXT:    [[DOT:%.*]] = select i1 [[CMP3]], i32 3, i32 2
+; CHECK-NEXT:    br label [[RETURN]]
+; CHECK:       return:
+; CHECK-NEXT:    [[RETVAL:%.*]] = phi i32 [ 0, [[CHK65:%.*]] ], [ 1, [[CHK0]] ], [ [[DOT]], [[BB_THEN]] ], [ 4, [[BB_IF]] ]
+; CHECK-NEXT:    ret i32 [[RETVAL]]
+;
+; FIXME: DOT should be replaced with 3
+chk65:
+  %cmp = icmp sgt i32 %c, 65
+  br i1 %cmp, label %return, label %chk0
+
+chk0:
+  %cmp1 = icmp slt i32 %c, 0
+  br i1 %cmp, label %return, label %bb_if
+
+bb_if:
+  %ashr.val = ashr exact i32 %c, 2
+  %cmp2 = icmp sgt i32 %ashr.val, 15
+  br i1 %cmp2, label %bb_then, label %return
+
+bb_then:
+  %cmp3 = icmp eq i32 %ashr.val, 16
+  %. = select i1 %cmp3, i32 3, i32 2
+  br label %return
+
+return:
+  %retval = phi i32 [0, %chk65], [1, %chk0], [%., %bb_then], [4, %bb_if]
+  ret i32 %retval
+}
diff --git a/llvm/test/Transforms/Attributor/misc.ll b/llvm/test/Transforms/Attributor/misc.ll
new file mode 100644
index 000000000..3ca99c0c2
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/misc.ll
@@ -0,0 +1,105 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; RUN: opt -S -attributor -attributor-disable=false < %s | FileCheck %s --check-prefixes=ALL,CHECK
+; RUN: opt -S -aa-pipeline='basic-aa' -passes=attributor -attributor-disable=false -attributor-annotate-decl-cs < %s | FileCheck %s --check-prefixes=ALL,DECL_CS
+;
+; Mostly check we do not crash on these uses
+
+define internal void @internal(void (i8*)* %fp) {
+; CHECK-LABEL: define {{[^@]+}}@internal
+; CHECK-SAME: (void (i8*)* [[FP:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[A:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[TMP:%.*]] = bitcast i32* [[A]] to i8*
+; CHECK-NEXT:    call void @foo(i32* nocapture nofree nonnull readnone align 4 dereferenceable(4) undef)
+; CHECK-NEXT:    call void [[FP]](i8* bitcast (void (i32*)* @foo to i8*))
+; CHECK-NEXT:    call void @callback1(void (i32*)* nonnull @foo)
+; CHECK-NEXT:    call void @callback2(void (i8*)* bitcast (void (i32*)* @foo to void (i8*)*))
+; CHECK-NEXT:    call void @callback2(void (i8*)* [[FP]])
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32* [[A]] to i8*
+; CHECK-NEXT:    call void [[FP]](i8* [[TMP1]])
+; CHECK-NEXT:    ret void
+;
+; DECL_CS-LABEL: define {{[^@]+}}@internal
+; DECL_CS-SAME: (void (i8*)* [[FP:%.*]])
+; DECL_CS-NEXT:  entry:
+; DECL_CS-NEXT:    [[A:%.*]] = alloca i32, align 4
+; DECL_CS-NEXT:    [[TMP:%.*]] = bitcast i32* [[A]] to i8*
+; DECL_CS-NEXT:    call void @foo(i32* nocapture nofree nonnull readnone align 4 dereferenceable(4) undef)
+; DECL_CS-NEXT:    call void [[FP]](i8* bitcast (void (i32*)* @foo to i8*))
+; DECL_CS-NEXT:    call void @callback1(void (i32*)* nonnull @foo)
+; DECL_CS-NEXT:    call void @callback2(void (i8*)* nonnull bitcast (void (i32*)* @foo to void (i8*)*))
+; DECL_CS-NEXT:    call void @callback2(void (i8*)* [[FP]])
+; DECL_CS-NEXT:    [[TMP1:%.*]] = bitcast i32* [[A]] to i8*
+; DECL_CS-NEXT:    call void [[FP]](i8* [[TMP1]])
+; DECL_CS-NEXT:    ret void
+;
+entry:
+  %a = alloca i32, align 4
+  %tmp = bitcast i32* %a to i8*
+  call void @foo(i32* nonnull %a)
+  call void %fp(i8* bitcast (void (i32*)* @foo to i8*))
+  call void @callback1(void (i32*)* nonnull @foo)
+  call void @callback2(void (i8*)* bitcast (void (i32*)* @foo to void (i8*)*))
+  call void @callback2(void (i8*)* %fp)
+  %tmp1 = bitcast i32* %a to i8*
+  call void %fp(i8* %tmp1)
+  ret void
+}
+
+define void @external(void (i8*)* %fp) {
+; CHECK-LABEL: define {{[^@]+}}@external
+; CHECK-SAME: (void (i8*)* [[FP:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[A:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[TMP:%.*]] = bitcast i32* [[A]] to i8*
+; CHECK-NEXT:    call void @foo(i32* nocapture nofree nonnull readnone align 4 dereferenceable(4) undef)
+; CHECK-NEXT:    call void @callback1(void (i32*)* nonnull @foo)
+; CHECK-NEXT:    call void @callback2(void (i8*)* bitcast (void (i32*)* @foo to void (i8*)*))
+; CHECK-NEXT:    call void @callback2(void (i8*)* [[FP]])
+; CHECK-NEXT:    call void [[FP]](i8* bitcast (void (i32*)* @foo to i8*))
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32* [[A]] to i8*
+; CHECK-NEXT:    call void [[FP]](i8* [[TMP1]])
+; CHECK-NEXT:    call void @internal(void (i8*)* [[FP]])
+; CHECK-NEXT:    ret void
+;
+; DECL_CS-LABEL: define {{[^@]+}}@external
+; DECL_CS-SAME: (void (i8*)* [[FP:%.*]])
+; DECL_CS-NEXT:  entry:
+; DECL_CS-NEXT:    [[A:%.*]] = alloca i32, align 4
+; DECL_CS-NEXT:    [[TMP:%.*]] = bitcast i32* [[A]] to i8*
+; DECL_CS-NEXT:    call void @foo(i32* nocapture nofree nonnull readnone align 4 dereferenceable(4) undef)
+; DECL_CS-NEXT:    call void @callback1(void (i32*)* nonnull @foo)
+; DECL_CS-NEXT:    call void @callback2(void (i8*)* nonnull bitcast (void (i32*)* @foo to void (i8*)*))
+; DECL_CS-NEXT:    call void @callback2(void (i8*)* [[FP]])
+; DECL_CS-NEXT:    call void [[FP]](i8* bitcast (void (i32*)* @foo to i8*))
+; DECL_CS-NEXT:    [[TMP1:%.*]] = bitcast i32* [[A]] to i8*
+; DECL_CS-NEXT:    call void [[FP]](i8* [[TMP1]])
+; DECL_CS-NEXT:    call void @internal(void (i8*)* [[FP]])
+; DECL_CS-NEXT:    ret void
+;
+entry:
+  %a = alloca i32, align 4
+  %tmp = bitcast i32* %a to i8*
+  call void @foo(i32* nonnull %a)
+  call void @callback1(void (i32*)* nonnull @foo)
+  call void @callback2(void (i8*)* bitcast (void (i32*)* @foo to void (i8*)*))
+  call void @callback2(void (i8*)* %fp)
+  call void %fp(i8* bitcast (void (i32*)* @foo to i8*))
+  %tmp1 = bitcast i32* %a to i8*
+  call void %fp(i8* %tmp1)
+  call void @internal(void (i8*)* %fp)
+  ret void
+}
+
+define internal void @foo(i32* %a) {
+; ALL-LABEL: define {{[^@]+}}@foo
+; ALL-SAME: (i32* nocapture nofree readnone [[A:%.*]])
+; ALL-NEXT:  entry:
+; ALL-NEXT:    ret void
+;
+entry:
+  ret void
+}
+
+declare void @callback1(void (i32*)*)
+declare void @callback2(void (i8*)*)
diff --git a/llvm/test/Transforms/Attributor/new_attributes.ll b/llvm/test/Transforms/Attributor/new_attributes.ll
new file mode 100644
index 000000000..6e87cffeb
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/new_attributes.ll
@@ -0,0 +1,43 @@
+; RUN: opt < %s -attributor -attributor-annotate-decl-cs -attributor-disable=false -attributor-max-iterations=0 -S | FileCheck %s
+; RUN: opt < %s -attributor -attributor-annotate-decl-cs -attributor-disable=false -attributor-max-iterations=1 -S | FileCheck %s
+; RUN: opt < %s -attributor -attributor-annotate-decl-cs -attributor-disable=false -attributor-max-iterations=2 -S | FileCheck %s
+; RUN: opt < %s -attributor -attributor-annotate-decl-cs -attributor-disable=false -attributor-max-iterations=3 -S | FileCheck %s
+; RUN: opt < %s -attributor -attributor-annotate-decl-cs -attributor-disable=false -attributor-max-iterations=4 -S | FileCheck %s
+; RUN: opt < %s -attributor -attributor-annotate-decl-cs -attributor-disable=false -attributor-max-iterations=2147483647 -S | FileCheck %s
+
+; CHECK-NOT: Function
+; CHECK: declare i32 @foo1()
+; CHECK-NOT: Function
+; CHECK: declare i32 @foo2()
+; CHECK-NOT: Function
+; CHECK: declare i32 @foo3()
+declare i32 @foo1()
+declare i32 @foo2()
+declare i32 @foo3()
+
+; CHECK-NOT: Function
+; CHECK:      define internal i32 @bar() {
+; CHECK-NEXT:   %1 = call i32 @foo1()
+; CHECK-NEXT:   %2 = call i32 @foo2()
+; CHECK-NEXT:   %3 = call i32 @foo3()
+; CHECK-NEXT:   ret i32 undef
+; CHECK-NEXT: }
+define internal i32 @bar() {
+  %1 = call i32 @foo1()
+  %2 = call i32 @foo2()
+  %3 = call i32 @foo3()
+  ret i32 1
+}
+
+; CHECK-NOT: Function
+; CHECK:      define i32 @baz() {
+; CHECK-NEXT:   %1 = call i32 @bar()
+; CHECK-NEXT:   ret i32 0
+; CHECK-NEXT: }
+define i32 @baz() {
+  %1 = call i32 @bar()
+  ret i32 0
+}
+
+; We should never derive anything here
+; CHECK-NOT: attributes
diff --git a/llvm/test/Transforms/Attributor/noalias.ll b/llvm/test/Transforms/Attributor/noalias.ll
new file mode 100644
index 000000000..f7f6297f6
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/noalias.ll
@@ -0,0 +1,284 @@
+; RUN: opt -S -passes=attributor -aa-pipeline='basic-aa' -attributor-disable=false -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=3 < %s | FileCheck %s
+
+; TEST 1 - negative.
+
+; void *G;
+; void *foo(){
+;   void *V = malloc(4);
+;   G = V;
+;   return V;
+; }
+
+@G = external global i8*
+
+; CHECK: define i8* @foo()
+define i8* @foo() {
+  %1 = tail call noalias i8* @malloc(i64 4)
+  store i8* %1, i8** @G, align 8
+  ret i8* %1
+}
+
+declare noalias i8* @malloc(i64)
+
+; TEST 2
+; call noalias function in return instruction.
+
+; CHECK: define noalias i8* @return_noalias()
+define i8* @return_noalias(){
+  %1 = tail call noalias i8* @malloc(i64 4)
+  ret i8* %1
+}
+
+define void @nocapture(i8* %a){
+  ret void
+}
+
+; CHECK: define noalias i8* @return_noalias_looks_like_capture()
+define i8* @return_noalias_looks_like_capture(){
+  %1 = tail call noalias i8* @malloc(i64 4)
+  call void @nocapture(i8* %1)
+  ret i8* %1
+}
+
+declare i8* @alias()
+
+; TEST 3
+; CHECK: define i8* @call_alias()
+; CHECK-NOT: noalias
+define i8* @call_alias(){
+  %1 = tail call i8* @alias()
+  ret i8* %1
+}
+
+; TEST 4
+; void *baz();
+; void *foo(int a);
+;
+; void *bar()  {
+;   foo(0);
+;    return baz();
+; }
+;
+; void *foo(int a)  {
+;   if (a)
+;   bar();
+;   return malloc(4);
+; }
+
+; CHECK: define i8* @bar()
+define i8* @bar() nounwind uwtable {
+  %1 = tail call i8* (...) @baz()
+  ret i8* %1
+}
+
+; CHECK: define noalias i8* @foo1(i32 %0)
+define i8* @foo1(i32 %0) nounwind uwtable {
+  %2 = icmp eq i32 %0, 0
+  br i1 %2, label %5, label %3
+
+3:                                                ; preds = %1
+  %4 = tail call i8* (...) @baz()
+  br label %5
+
+5:                                                ; preds = %1, %3
+  %6 = tail call noalias i8* @malloc(i64 4)
+  ret i8* %6
+}
+
+declare i8* @baz(...) nounwind uwtable
+
+; TEST 5
+
+; Returning global pointer. Should not be noalias.
+; CHECK: define nonnull align 8 dereferenceable(8) i8** @getter()
+define i8** @getter() {
+  ret i8** @G
+}
+
+; Returning global pointer. Should not be noalias.
+; CHECK: define nonnull align 8 dereferenceable(8) i8** @calle1()
+define i8** @calle1(){
+  %1 = call i8** @getter()
+  ret i8** %1
+}
+
+; TEST 6
+declare noalias i8* @strdup(i8* nocapture) nounwind
+
+; CHECK: define noalias i8* @test6()
+define i8* @test6() nounwind uwtable ssp {
+  %x = alloca [2 x i8], align 1
+  %arrayidx = getelementptr inbounds [2 x i8], [2 x i8]* %x, i64 0, i64 0
+  store i8 97, i8* %arrayidx, align 1
+  %arrayidx1 = getelementptr inbounds [2 x i8], [2 x i8]* %x, i64 0, i64 1
+  store i8 0, i8* %arrayidx1, align 1
+  %call = call noalias i8* @strdup(i8* %arrayidx) nounwind
+  ret i8* %call
+}
+
+; TEST 7
+
+; CHECK: define noalias i8* @test7()
+define i8* @test7() nounwind {
+entry:
+  %A = call noalias i8* @malloc(i64 4) nounwind
+  %tobool = icmp eq i8* %A, null
+  br i1 %tobool, label %return, label %if.end
+
+if.end:
+  store i8 7, i8* %A
+  br label %return
+
+return:
+  %retval.0 = phi i8* [ %A, %if.end ], [ null, %entry ]
+  ret i8* %retval.0
+}
+
+; TEST 8
+
+; CHECK: define noalias i8* @test8(i32* %0)
+define i8* @test8(i32* %0) nounwind uwtable {
+  %2 = tail call noalias i8* @malloc(i64 4)
+  %3 = icmp ne i32* %0, null
+  br i1 %3, label %4, label %5
+
+4:                                                ; preds = %1
+  store i8 10, i8* %2
+  br label %5
+
+5:                                                ; preds = %1, %4
+  ret i8* %2
+}
+
+; TEST 9
+; Simple Argument Test
+declare void @use_i8(i8* nocapture) readnone
+define internal void @test9a(i8* %a, i8* %b) {
+; CHECK: define internal void @test9a()
+  ret void
+}
+define internal void @test9b(i8* %a, i8* %b) {
+; CHECK: define internal void @test9b(i8* noalias nocapture readnone %a, i8* nocapture readnone %b)
+  call void @use_i8(i8* %a)
+  call void @use_i8(i8* %b)
+  ret void
+}
+define void @test9_helper(i8* %a, i8* %b) {
+; CHECK: define void @test9_helper(i8* nocapture readnone %a, i8* nocapture readnone %b)
+; CHECK:  tail call void @test9a()
+; CHECK:  tail call void @test9a()
+; CHECK:  tail call void @test9b(i8* noalias nocapture readnone %a, i8* nocapture readnone %b)
+; CHECK:  tail call void @test9b(i8* noalias nocapture readnone %b, i8* noalias nocapture readnone %a)
+  tail call void @test9a(i8* noalias %a, i8* %b)
+  tail call void @test9a(i8* noalias %b, i8* noalias %a)
+  tail call void @test9b(i8* noalias %a, i8* %b)
+  tail call void @test9b(i8* noalias %b, i8* noalias %a)
+  ret void
+}
+
+
+; TEST 10
+; Simple CallSite Test
+
+declare void @test10_helper_1(i8* %a)
+define void @test10_helper_2(i8* noalias %a) {
+; CHECK:   tail call void @test10_helper_1(i8* %a)
+  tail call void @test10_helper_1(i8* %a)
+  ret void
+}
+define void @test10(i8* noalias %a) {
+; CHECK: define void @test10(i8* noalias %a)
+; FIXME: missing noalias
+; CHECK-NEXT:   tail call void @test10_helper_1(i8* %a)
+  tail call void @test10_helper_1(i8* %a)
+
+; CHECK-NEXT:   tail call void @test10_helper_2(i8* noalias %a)
+  tail call void @test10_helper_2(i8* %a)
+  ret void
+}
+
+; TEST 11
+; CallSite Test
+
+declare void @test11_helper(i8* %a, i8 *%b)
+define void @test11(i8* noalias %a) {
+; CHECK: define void @test11(i8* noalias %a)
+; CHECK-NEXT:   tail call void @test11_helper(i8* %a, i8* %a)
+  tail call void @test11_helper(i8* %a, i8* %a)
+  ret void
+}
+
+
+; TEST 12
+; CallSite Argument
+declare void @use_nocapture(i8* nocapture)
+declare void @use(i8*)
+define void @test12_1() {
+; CHECK-LABEL: @test12_1(
+; CHECK-NEXT:    [[A:%.*]] = alloca i8, align 4
+; CHECK-NEXT:    [[B:%.*]] = tail call noalias i8* @malloc(i64 4)
+; CHECK-NEXT:    tail call void @use_nocapture(i8* noalias nonnull align 4 dereferenceable(1) [[A]])
+; CHECK-NEXT:    tail call void @use_nocapture(i8* noalias nonnull align 4 dereferenceable(1) [[A]])
+; CHECK-NEXT:    tail call void @use_nocapture(i8* noalias nocapture [[B]])
+; CHECK-NEXT:    tail call void @use_nocapture(i8* noalias nocapture [[B]])
+; CHECK-NEXT:    ret void
+;
+  %A = alloca i8, align 4
+  %B = tail call noalias i8* @malloc(i64 4)
+  tail call void @use_nocapture(i8* %A)
+  tail call void @use_nocapture(i8* %A)
+  tail call void @use_nocapture(i8* %B)
+  tail call void @use_nocapture(i8* %B)
+  ret void
+}
+
+define void @test12_2(){
+; CHECK-LABEL: @test12_2(
+; CHECK-NEXT:    [[A:%.*]] = tail call noalias i8* @malloc(i64 4)
+; FIXME: This should be @use_nocapture(i8* noalias [[A]])
+; CHECK-NEXT:    tail call void @use_nocapture(i8* nocapture [[A]])
+; FIXME: This should be @use_nocapture(i8* noalias nocapture [[A]])
+; CHECK-NEXT:    tail call void @use_nocapture(i8* nocapture [[A]])
+; CHECK-NEXT:    tail call void @use(i8* [[A]])
+; CHECK-NEXT:    tail call void @use_nocapture(i8* nocapture [[A]])
+; CHECK-NEXT:    ret void
+;
+  %A = tail call noalias i8* @malloc(i64 4)
+  tail call void @use_nocapture(i8* %A)
+  tail call void @use_nocapture(i8* %A)
+  tail call void @use(i8* %A)
+  tail call void @use_nocapture(i8* %A)
+  ret void
+}
+
+declare void @two_args(i8* nocapture , i8* nocapture)
+define void @test12_3(){
+; CHECK-LABEL: @test12_3(
+  %A = tail call noalias i8* @malloc(i64 4)
+; CHECK: tail call void @two_args(i8* nocapture %A, i8* nocapture %A)
+  tail call void @two_args(i8* %A, i8* %A)
+  ret void
+}
+
+define void @test12_4(){
+; CHECK-LABEL: @test12_4(
+  %A = tail call noalias i8* @malloc(i64 4)
+  %B = tail call noalias i8* @malloc(i64 4)
+  %A_0 = getelementptr i8, i8* %A, i64 0
+  %A_1 = getelementptr i8, i8* %A, i64 1
+  %B_0 = getelementptr i8, i8* %B, i64 0
+
+; CHECK: tail call void @two_args(i8* noalias nocapture %A, i8* noalias nocapture %B)
+  tail call void @two_args(i8* %A, i8* %B)
+
+; CHECK: tail call void @two_args(i8* nocapture %A, i8* nocapture %A_0)
+  tail call void @two_args(i8* %A, i8* %A_0)
+
+; CHECK: tail call void @two_args(i8* nocapture %A, i8* nocapture %A_1)
+  tail call void @two_args(i8* %A, i8* %A_1)
+
+; FIXME: This should be @two_args(i8* noalias nocapture %A_0, i8* noalias nocapture %B_0)
+; CHECK: tail call void @two_args(i8* nocapture %A_0, i8* nocapture %B_0)
+  tail call void @two_args(i8* %A_0, i8* %B_0)
+  ret void
+}
diff --git a/llvm/test/Transforms/Attributor/nocapture-1.ll b/llvm/test/Transforms/Attributor/nocapture-1.ll
new file mode 100644
index 000000000..34bdb1c83
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/nocapture-1.ll
@@ -0,0 +1,346 @@
+; RUN: opt -attributor -attributor-manifest-internal -attributor-disable=false -S -attributor-annotate-decl-cs < %s | FileCheck %s --check-prefixes=ATTRIBUTOR
+; RUN: opt -passes=attributor -attributor-manifest-internal -attributor-disable=false -S -attributor-annotate-decl-cs < %s | FileCheck %s --check-prefixes=ATTRIBUTOR
+; Copied from Transforms/FunctoinAttrs/nocapture.ll
+
+@g = global i32* null		; <i32**> [#uses=1]
+
+; ATTRIBUTOR: define i32* @c1(i32* nofree readnone returned "no-capture-maybe-returned" %q)
+define i32* @c1(i32* %q) {
+	ret i32* %q
+}
+
+; ATTRIBUTOR: define void @c2(i32* nofree writeonly %q)
+; It would also be acceptable to mark %q as readnone. Update @c3 too.
+define void @c2(i32* %q) {
+	store i32* %q, i32** @g
+	ret void
+}
+
+; ATTRIBUTOR: define void @c3(i32* nofree writeonly %q)
+define void @c3(i32* %q) {
+	call void @c2(i32* %q)
+	ret void
+}
+
+; ATTRIBUTOR: define i1 @c4(i32* nofree readnone %q, i32 %bitno)
+define i1 @c4(i32* %q, i32 %bitno) {
+	%tmp = ptrtoint i32* %q to i32
+	%tmp2 = lshr i32 %tmp, %bitno
+	%bit = trunc i32 %tmp2 to i1
+	br i1 %bit, label %l1, label %l0
+l0:
+	ret i1 0 ; escaping value not caught by def-use chaining.
+l1:
+	ret i1 1 ; escaping value not caught by def-use chaining.
+}
+
+; c4b is c4 but without the escaping part
+; ATTRIBUTOR: define i1 @c4b(i32* nocapture nofree readnone %q, i32 %bitno)
+define i1 @c4b(i32* %q, i32 %bitno) {
+	%tmp = ptrtoint i32* %q to i32
+	%tmp2 = lshr i32 %tmp, %bitno
+	%bit = trunc i32 %tmp2 to i1
+	br i1 %bit, label %l1, label %l0
+l0:
+	ret i1 0 ; not escaping!
+l1:
+	ret i1 0 ; not escaping!
+}
+
+@lookup_table = global [2 x i1] [ i1 0, i1 1 ]
+
+; ATTRIBUTOR: define i1 @c5(i32* nofree readonly %q, i32 %bitno)
+define i1 @c5(i32* %q, i32 %bitno) {
+	%tmp = ptrtoint i32* %q to i32
+	%tmp2 = lshr i32 %tmp, %bitno
+	%bit = and i32 %tmp2, 1
+        ; subtle escape mechanism follows
+	%lookup = getelementptr [2 x i1], [2 x i1]* @lookup_table, i32 0, i32 %bit
+	%val = load i1, i1* %lookup
+	ret i1 %val
+}
+
+declare void @throw_if_bit_set(i8*, i8) readonly
+
+; ATTRIBUTOR: define i1 @c6(i8* readonly %q, i8 %bit)
+define i1 @c6(i8* %q, i8 %bit) personality i32 (...)* @__gxx_personality_v0 {
+	invoke void @throw_if_bit_set(i8* %q, i8 %bit)
+		to label %ret0 unwind label %ret1
+ret0:
+	ret i1 0
+ret1:
+        %exn = landingpad {i8*, i32}
+                 cleanup
+	ret i1 1
+}
+
+declare i32 @__gxx_personality_v0(...)
+
+define i1* @lookup_bit(i32* %q, i32 %bitno) readnone nounwind {
+	%tmp = ptrtoint i32* %q to i32
+	%tmp2 = lshr i32 %tmp, %bitno
+	%bit = and i32 %tmp2, 1
+	%lookup = getelementptr [2 x i1], [2 x i1]* @lookup_table, i32 0, i32 %bit
+	ret i1* %lookup
+}
+
+; ATTRIBUTOR: define i1 @c7(i32* nofree readonly %q, i32 %bitno)
+define i1 @c7(i32* %q, i32 %bitno) {
+	%ptr = call i1* @lookup_bit(i32* %q, i32 %bitno)
+	%val = load i1, i1* %ptr
+	ret i1 %val
+}
+
+
+; ATTRIBUTOR: define i32 @nc1(i32* nofree %q, i32* nocapture nofree %p, i1 %b)
+define i32 @nc1(i32* %q, i32* %p, i1 %b) {
+e:
+	br label %l
+l:
+	%x = phi i32* [ %p, %e ]
+	%y = phi i32* [ %q, %e ]
+	%tmp = bitcast i32* %x to i32*		; <i32*> [#uses=2]
+	%tmp2 = select i1 %b, i32* %tmp, i32* %y
+	%val = load i32, i32* %tmp2		; <i32> [#uses=1]
+	store i32 0, i32* %tmp
+	store i32* %y, i32** @g
+	ret i32 %val
+}
+
+; ATTRIBUTOR: define i32 @nc1_addrspace(i32* nofree %q, i32 addrspace(1)* nocapture nofree %p, i1 %b)
+define i32 @nc1_addrspace(i32* %q, i32 addrspace(1)* %p, i1 %b) {
+e:
+	br label %l
+l:
+	%x = phi i32 addrspace(1)* [ %p, %e ]
+	%y = phi i32* [ %q, %e ]
+	%tmp = addrspacecast i32 addrspace(1)* %x to i32*		; <i32*> [#uses=2]
+	%tmp2 = select i1 %b, i32* %tmp, i32* %y
+	%val = load i32, i32* %tmp2		; <i32> [#uses=1]
+	store i32 0, i32* %tmp
+	store i32* %y, i32** @g
+	ret i32 %val
+}
+
+; ATTRIBUTOR: define void @nc2(i32* nocapture nofree %p, i32* nofree %q)
+define void @nc2(i32* %p, i32* %q) {
+	%1 = call i32 @nc1(i32* %q, i32* %p, i1 0)		; <i32> [#uses=0]
+	ret void
+}
+
+
+; ATTRIBUTOR: define void @nc3(void ()* nocapture nofree nonnull %p)
+define void @nc3(void ()* %p) {
+	call void %p()
+	ret void
+}
+
+declare void @external(i8*) readonly nounwind
+; ATTRIBUTOR: define void @nc4(i8* nocapture readonly %p)
+define void @nc4(i8* %p) {
+	call void @external(i8* %p)
+	ret void
+}
+
+; ATTRIBUTOR: define void @nc5(void (i8*)* nocapture nofree nonnull %f, i8* nocapture %p)
+define void @nc5(void (i8*)* %f, i8* %p) {
+	call void %f(i8* %p) readonly nounwind
+	call void %f(i8* nocapture %p)
+	ret void
+}
+
+; ATTRIBUTOR: define void @test1_1(i8* nocapture nofree readnone %x1_1, i8* nocapture nofree readnone %y1_1, i1 %c)
+; It would be acceptable to add readnone to %y1_1 and %y1_2.
+define void @test1_1(i8* %x1_1, i8* %y1_1, i1 %c) {
+  call i8* @test1_2(i8* %x1_1, i8* %y1_1, i1 %c)
+  store i32* null, i32** @g
+  ret void
+}
+
+; ATTRIBUTOR: define i8* @test1_2(i8* nocapture nofree readnone %x1_2, i8* nofree readnone returned "no-capture-maybe-returned" %y1_2, i1 %c)
+define i8* @test1_2(i8* %x1_2, i8* %y1_2, i1 %c) {
+  br i1 %c, label %t, label %f
+t:
+  call void @test1_1(i8* %x1_2, i8* %y1_2, i1 %c)
+  store i32* null, i32** @g
+  br label %f
+f:
+  ret i8* %y1_2
+}
+
+; ATTRIBUTOR: define void @test2(i8* nocapture nofree readnone %x2)
+define void @test2(i8* %x2) {
+  call void @test2(i8* %x2)
+  store i32* null, i32** @g
+  ret void
+}
+
+; ATTRIBUTOR: define void @test3(i8* nocapture nofree readnone %x3, i8* nocapture nofree readnone %y3, i8* nocapture nofree readnone %z3)
+define void @test3(i8* %x3, i8* %y3, i8* %z3) {
+  call void @test3(i8* %z3, i8* %y3, i8* %x3)
+  store i32* null, i32** @g
+  ret void
+}
+
+; ATTRIBUTOR: define void @test4_1(i8* nocapture nofree readnone %x4_1, i1 %c)
+define void @test4_1(i8* %x4_1, i1 %c) {
+  call i8* @test4_2(i8* %x4_1, i8* %x4_1, i8* %x4_1, i1 %c)
+  store i32* null, i32** @g
+  ret void
+}
+
+; ATTRIBUTOR: define i8* @test4_2(i8* nocapture nofree readnone %x4_2, i8* nofree readnone returned "no-capture-maybe-returned" %y4_2, i8* nocapture nofree readnone %z4_2, i1 %c)
+define i8* @test4_2(i8* %x4_2, i8* %y4_2, i8* %z4_2, i1 %c) {
+  br i1 %c, label %t, label %f
+t:
+  call void @test4_1(i8* null, i1 %c)
+  store i32* null, i32** @g
+  br label %f
+f:
+  ret i8* %y4_2
+}
+
+declare i8* @test5_1(i8* %x5_1)
+
+; ATTRIBUTOR: define void @test5_2(i8* %x5_2)
+define void @test5_2(i8* %x5_2) {
+  call i8* @test5_1(i8* %x5_2)
+  store i32* null, i32** @g
+  ret void
+}
+
+declare void @test6_1(i8* %x6_1, i8* nocapture %y6_1, ...)
+
+; ATTRIBUTOR: define void @test6_2(i8* %x6_2, i8* nocapture %y6_2, i8* %z6_2)
+define void @test6_2(i8* %x6_2, i8* %y6_2, i8* %z6_2) {
+  call void (i8*, i8*, ...) @test6_1(i8* %x6_2, i8* %y6_2, i8* %z6_2)
+  store i32* null, i32** @g
+  ret void
+}
+
+; ATTRIBUTOR: define void @test_cmpxchg(i32* nocapture nofree nonnull dereferenceable(4) %p)
+define void @test_cmpxchg(i32* %p) {
+  cmpxchg i32* %p, i32 0, i32 1 acquire monotonic
+  ret void
+}
+
+; ATTRIBUTOR: define void @test_cmpxchg_ptr(i32** nocapture nofree nonnull dereferenceable(8) %p, i32* nofree %q)
+define void @test_cmpxchg_ptr(i32** %p, i32* %q) {
+  cmpxchg i32** %p, i32* null, i32* %q acquire monotonic
+  ret void
+}
+
+; ATTRIBUTOR: define void @test_atomicrmw(i32* nocapture nofree nonnull dereferenceable(4) %p)
+define void @test_atomicrmw(i32* %p) {
+  atomicrmw add i32* %p, i32 1 seq_cst
+  ret void
+}
+
+; ATTRIBUTOR: define void @test_volatile(i32* nofree align 4 %x)
+define void @test_volatile(i32* %x) {
+entry:
+  %gep = getelementptr i32, i32* %x, i64 1
+  store volatile i32 0, i32* %gep, align 4
+  ret void
+}
+
+; ATTRIBUTOR: nocaptureLaunder(i8* nocapture %p)
+define void @nocaptureLaunder(i8* %p) {
+entry:
+  %b = call i8* @llvm.launder.invariant.group.p0i8(i8* %p)
+  store i8 42, i8* %b
+  ret void
+}
+
+@g2 = global i8* null
+; ATTRIBUTOR: define void @captureLaunder(i8* %p)
+define void @captureLaunder(i8* %p) {
+  %b = call i8* @llvm.launder.invariant.group.p0i8(i8* %p)
+  store i8* %b, i8** @g2
+  ret void
+}
+
+; ATTRIBUTOR: @nocaptureStrip(i8* nocapture writeonly %p)
+define void @nocaptureStrip(i8* %p) {
+entry:
+  %b = call i8* @llvm.strip.invariant.group.p0i8(i8* %p)
+  store i8 42, i8* %b
+  ret void
+}
+
+@g3 = global i8* null
+; ATTRIBUTOR: define void @captureStrip(i8* writeonly %p)
+define void @captureStrip(i8* %p) {
+  %b = call i8* @llvm.strip.invariant.group.p0i8(i8* %p)
+  store i8* %b, i8** @g3
+  ret void
+}
+
+; ATTRIBUTOR: define i1 @captureICmp(i32* nofree readnone %x)
+define i1 @captureICmp(i32* %x) {
+  %1 = icmp eq i32* %x, null
+  ret i1 %1
+}
+
+; ATTRIBUTOR: define i1 @captureICmpRev(i32* nofree readnone %x)
+define i1 @captureICmpRev(i32* %x) {
+  %1 = icmp eq i32* null, %x
+  ret i1 %1
+}
+
+; ATTRIBUTOR: define i1 @nocaptureInboundsGEPICmp(i32* nocapture nofree readnone %x)
+define i1 @nocaptureInboundsGEPICmp(i32* %x) {
+  %1 = getelementptr inbounds i32, i32* %x, i32 5
+  %2 = bitcast i32* %1 to i8*
+  %3 = icmp eq i8* %2, null
+  ret i1 %3
+}
+
+; ATTRIBUTOR: define i1 @nocaptureInboundsGEPICmpRev(i32* nocapture nofree readnone %x)
+define i1 @nocaptureInboundsGEPICmpRev(i32* %x) {
+  %1 = getelementptr inbounds i32, i32* %x, i32 5
+  %2 = bitcast i32* %1 to i8*
+  %3 = icmp eq i8* null, %2
+  ret i1 %3
+}
+
+; ATTRIBUTOR: define i1 @nocaptureDereferenceableOrNullICmp(i32* nocapture nofree readnone dereferenceable_or_null(4) %x)
+define i1 @nocaptureDereferenceableOrNullICmp(i32* dereferenceable_or_null(4) %x) {
+  %1 = bitcast i32* %x to i8*
+  %2 = icmp eq i8* %1, null
+  ret i1 %2
+}
+
+; ATTRIBUTOR: define i1 @captureDereferenceableOrNullICmp(i32* nofree readnone dereferenceable_or_null(4) %x)
+define i1 @captureDereferenceableOrNullICmp(i32* dereferenceable_or_null(4) %x) "null-pointer-is-valid"="true" {
+  %1 = bitcast i32* %x to i8*
+  %2 = icmp eq i8* %1, null
+  ret i1 %2
+}
+
+declare void @unknown(i8*)
+define void @test_callsite() {
+entry:
+; We know that 'null' in AS 0 does not alias anything and cannot be captured. Though the latter is not qurried -> derived atm.
+; ATTRIBUTOR: call void @unknown(i8* noalias null)
+  call void @unknown(i8* null)
+  ret void
+}
+
+declare i8* @unknownpi8pi8(i8*,i8* returned)
+define i8* @test_returned1(i8* %A, i8* returned %B) nounwind readonly {
+; ATTRIBUTOR: define i8* @test_returned1(i8* nocapture readonly %A, i8* readonly returned %B)
+entry:
+  %p = call i8* @unknownpi8pi8(i8* %A, i8* %B)
+  ret i8* %p
+}
+
+define i8* @test_returned2(i8* %A, i8* %B) {
+; ATTRIBUTOR: define i8* @test_returned2(i8* nocapture readonly %A, i8* readonly returned %B)
+entry:
+  %p = call i8* @unknownpi8pi8(i8* %A, i8* %B) nounwind readonly
+  ret i8* %p
+}
+
+declare i8* @llvm.launder.invariant.group.p0i8(i8*)
+declare i8* @llvm.strip.invariant.group.p0i8(i8*)
diff --git a/llvm/test/Transforms/Attributor/nocapture-2.ll b/llvm/test/Transforms/Attributor/nocapture-2.ll
new file mode 100644
index 000000000..ddc466ee7
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/nocapture-2.ll
@@ -0,0 +1,471 @@
+; RUN: opt -functionattrs -attributor -attributor-manifest-internal -attributor-disable=false -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=6 -S < %s | FileCheck %s
+;
+; Test cases specifically designed for the "no-capture" argument attribute.
+; We use FIXME's to indicate problems and missing attributes.
+;
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+; TEST comparison against NULL
+;
+; int is_null_return(int *p) {
+;   return p == 0;
+; }
+;
+; FIXME: no-capture missing for %p
+; CHECK: define i32 @is_null_return(i32* nofree readnone %p)
+define i32 @is_null_return(i32* %p) #0 {
+entry:
+  %cmp = icmp eq i32* %p, null
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; TEST comparison against NULL in control flow
+;
+; int is_null_control(int *p) {
+;   if (p == 0)
+;     return 1;
+;   if (0 == p)
+;     return 1;
+;   return 0;
+; }
+;
+; FIXME: no-capture missing for %p
+; CHECK: define i32 @is_null_control(i32* nofree readnone %p)
+define i32 @is_null_control(i32* %p) #0 {
+entry:
+  %retval = alloca i32, align 4
+  %cmp = icmp eq i32* %p, null
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  store i32 1, i32* %retval, align 4
+  br label %return
+
+if.end:                                           ; preds = %entry
+  %cmp1 = icmp eq i32* null, %p
+  br i1 %cmp1, label %if.then2, label %if.end3
+
+if.then2:                                         ; preds = %if.end
+  store i32 1, i32* %retval, align 4
+  br label %return
+
+if.end3:                                          ; preds = %if.end
+  store i32 0, i32* %retval, align 4
+  br label %return
+
+return:                                           ; preds = %if.end3, %if.then2, %if.then
+  %0 = load i32, i32* %retval, align 4
+  ret i32 %0
+}
+
+; TEST singleton SCC
+;
+; double *srec0(double *a) {
+;   srec0(a);
+;   return 0;
+; }
+;
+; CHECK: define noalias nonnull align 536870912 dereferenceable(4294967295) double* @srec0(double* nocapture nofree readnone %a)
+define double* @srec0(double* %a) #0 {
+entry:
+  %call = call double* @srec0(double* %a)
+  ret double* null
+}
+
+; TEST singleton SCC with lots of nested recursive calls
+;
+; int* srec16(int* a) {
+;   return srec16(srec16(srec16(srec16(
+;          srec16(srec16(srec16(srec16(
+;          srec16(srec16(srec16(srec16(
+;          srec16(srec16(srec16(srec16(
+;                        a
+;          ))))))))))))))));
+; }
+;
+; Other arguments are possible here due to the no-return behavior.
+;
+; CHECK: define noalias nonnull align 536870912 dereferenceable(4294967295) i32* @srec16(i32* nocapture nofree readnone %a)
+define i32* @srec16(i32* %a) #0 {
+entry:
+  %call = call i32* @srec16(i32* %a)
+; CHECK-NOT:      %call
+; CHECK: unreachable
+  %call1 = call i32* @srec16(i32* %call)
+  %call2 = call i32* @srec16(i32* %call1)
+  %call3 = call i32* @srec16(i32* %call2)
+  %call4 = call i32* @srec16(i32* %call3)
+  %call5 = call i32* @srec16(i32* %call4)
+  %call6 = call i32* @srec16(i32* %call5)
+  %call7 = call i32* @srec16(i32* %call6)
+  %call8 = call i32* @srec16(i32* %call7)
+  %call9 = call i32* @srec16(i32* %call8)
+  %call10 = call i32* @srec16(i32* %call9)
+  %call11 = call i32* @srec16(i32* %call10)
+  %call12 = call i32* @srec16(i32* %call11)
+  %call13 = call i32* @srec16(i32* %call12)
+  %call14 = call i32* @srec16(i32* %call13)
+  %call15 = call i32* @srec16(i32* %call14)
+  ret i32* %call15
+}
+
+; TEST SCC with various calls, casts, and comparisons agains NULL
+;
+; CHECK: define dereferenceable_or_null(4) float* @scc_A(i32* nofree readnone returned dereferenceable_or_null(4) "no-capture-maybe-returned" %a)
+;
+; CHECK: define dereferenceable_or_null(8) i64* @scc_B(double* nofree readnone returned dereferenceable_or_null(8) "no-capture-maybe-returned" %a)
+;
+; CHECK: define dereferenceable_or_null(4) i8* @scc_C(i16* nofree readnone returned dereferenceable_or_null(4) "no-capture-maybe-returned" %a)
+;
+; float *scc_A(int *a) {
+;   return (float*)(a ? (int*)scc_A((int*)scc_B((double*)scc_C((short*)a))) : a);
+; }
+;
+; long *scc_B(double *a) {
+;   return (long*)(a ? scc_C((short*)scc_B((double*)scc_A((int*)a))) : a);
+; }
+;
+; void *scc_C(short *a) {
+;   return scc_A((int*)(scc_A(a) ? scc_B((double*)a) : scc_C(a)));
+; }
+define float* @scc_A(i32* dereferenceable_or_null(4) %a) {
+entry:
+  %tobool = icmp ne i32* %a, null
+  br i1 %tobool, label %cond.true, label %cond.false
+
+cond.true:                                        ; preds = %entry
+  %0 = bitcast i32* %a to i16*
+  %call = call i8* @scc_C(i16* %0)
+  %1 = bitcast i8* %call to double*
+  %call1 = call i64* @scc_B(double* %1)
+  %2 = bitcast i64* %call1 to i32*
+  %call2 = call float* @scc_A(i32* %2)
+  %3 = bitcast float* %call2 to i32*
+  br label %cond.end
+
+cond.false:                                       ; preds = %entry
+  br label %cond.end
+
+cond.end:                                         ; preds = %cond.false, %cond.true
+  %cond = phi i32* [ %3, %cond.true ], [ %a, %cond.false ]
+  %4 = bitcast i32* %cond to float*
+  ret float* %4
+}
+
+define i64* @scc_B(double* dereferenceable_or_null(8) %a) {
+entry:
+  %tobool = icmp ne double* %a, null
+  br i1 %tobool, label %cond.true, label %cond.false
+
+cond.true:                                        ; preds = %entry
+  %0 = bitcast double* %a to i32*
+  %call = call float* @scc_A(i32* %0)
+  %1 = bitcast float* %call to double*
+  %call1 = call i64* @scc_B(double* %1)
+  %2 = bitcast i64* %call1 to i16*
+  %call2 = call i8* @scc_C(i16* %2)
+  br label %cond.end
+
+cond.false:                                       ; preds = %entry
+  %3 = bitcast double* %a to i8*
+  br label %cond.end
+
+cond.end:                                         ; preds = %cond.false, %cond.true
+  %cond = phi i8* [ %call2, %cond.true ], [ %3, %cond.false ]
+  %4 = bitcast i8* %cond to i64*
+  ret i64* %4
+}
+
+define i8* @scc_C(i16* dereferenceable_or_null(2) %a) {
+entry:
+  %bc = bitcast i16* %a to i32*
+  %call = call float* @scc_A(i32* %bc)
+  %bc2 = bitcast float* %call to i8*
+  %tobool = icmp ne i8* %bc2, null
+  br i1 %tobool, label %cond.true, label %cond.false
+
+cond.true:                                        ; preds = %entry
+  %0 = bitcast i16* %a to double*
+  %call1 = call i64* @scc_B(double* %0)
+  %1 = bitcast i64* %call1 to i8*
+  br label %cond.end
+
+cond.false:                                       ; preds = %entry
+  %call2 = call i8* @scc_C(i16* %a)
+  br label %cond.end
+
+cond.end:                                         ; preds = %cond.false, %cond.true
+  %cond = phi i8* [ %1, %cond.true ], [ %call2, %cond.false ]
+  %2 = bitcast i8* %cond to i32*
+  %call3 = call float* @scc_A(i32* %2)
+  %3 = bitcast float* %call3 to i8*
+  ret i8* %3
+}
+
+
+; TEST call to external function, marked no-capture
+;
+; void external_no_capture(int /* no-capture */ *p);
+; void test_external_no_capture(int *p) {
+;   external_no_capture(p);
+; }
+;
+; CHECK: define void @test_external_no_capture(i32* nocapture %p)
+declare void @external_no_capture(i32* nocapture)
+
+define void @test_external_no_capture(i32* %p) #0 {
+entry:
+  call void @external_no_capture(i32* %p)
+  ret void
+}
+
+; TEST call to external var-args function, marked no-capture
+;
+; void test_var_arg_call(char *p, int a) {
+;   printf(p, a);
+; }
+;
+; CHECK: define void @test_var_arg_call(i8* nocapture %p, i32 %a)
+define void @test_var_arg_call(i8* %p, i32 %a) #0 {
+entry:
+  %call = call i32 (i8*, ...) @printf(i8* %p, i32 %a)
+  ret void
+}
+
+declare i32 @printf(i8* nocapture, ...)
+
+
+; TEST "captured" only through return
+;
+; long *not_captured_but_returned_0(long *a) {
+;   *a1 = 0;
+;   return a;
+; }
+;
+; There should *not* be a no-capture attribute on %a
+; CHECK: define nonnull align 8 dereferenceable(8) i64* @not_captured_but_returned_0(i64* nofree nonnull returned writeonly align 8 dereferenceable(8) "no-capture-maybe-returned" %a)
+
+define i64* @not_captured_but_returned_0(i64* %a) #0 {
+entry:
+  store i64 0, i64* %a, align 8
+  ret i64* %a
+}
+
+; TEST "captured" only through return
+;
+; long *not_captured_but_returned_1(long *a) {
+;   *(a+1) = 1;
+;   return a + 1;
+; }
+;
+; There should *not* be a no-capture attribute on %a
+; CHECK: define nonnull align 8 dereferenceable(8) i64* @not_captured_but_returned_1(i64* nofree nonnull writeonly align 8 dereferenceable(16) "no-capture-maybe-returned" %a)
+define i64* @not_captured_but_returned_1(i64* %a) #0 {
+entry:
+  %add.ptr = getelementptr inbounds i64, i64* %a, i64 1
+  store i64 1, i64* %add.ptr, align 8
+  ret i64* %add.ptr
+}
+
+; TEST calls to "captured" only through return functions
+;
+; void test_not_captured_but_returned_calls(long *a) {
+;   not_captured_but_returned_0(a);
+;   not_captured_but_returned_1(a);
+; }
+;
+; CHECK: define void @test_not_captured_but_returned_calls(i64* nocapture nofree writeonly align 8 %a)
+define void @test_not_captured_but_returned_calls(i64* %a) #0 {
+entry:
+  %call = call i64* @not_captured_but_returned_0(i64* %a)
+  %call1 = call i64* @not_captured_but_returned_1(i64* %a)
+  ret void
+}
+
+; TEST "captured" only through transitive return
+;
+; long* negative_test_not_captured_but_returned_call_0a(long *a) {
+;   return not_captured_but_returned_0(a);
+; }
+;
+; There should *not* be a no-capture attribute on %a
+; CHECK: define align 8 i64* @negative_test_not_captured_but_returned_call_0a(i64* nofree returned writeonly align 8 "no-capture-maybe-returned" %a)
+define i64* @negative_test_not_captured_but_returned_call_0a(i64* %a) #0 {
+entry:
+  %call = call i64* @not_captured_but_returned_0(i64* %a)
+  ret i64* %call
+}
+
+; TEST captured through write
+;
+; void negative_test_not_captured_but_returned_call_0b(long *a) {
+;   *a = (long)not_captured_but_returned_0(a);
+; }
+;
+; There should *not* be a no-capture attribute on %a
+; CHECK: define void @negative_test_not_captured_but_returned_call_0b(i64* nofree writeonly align 8 %a)
+define void @negative_test_not_captured_but_returned_call_0b(i64* %a) #0 {
+entry:
+  %call = call i64* @not_captured_but_returned_0(i64* %a)
+  %0 = ptrtoint i64* %call to i64
+  store i64 %0, i64* %a, align 8
+  ret void
+}
+
+; TEST "captured" only through transitive return
+;
+; long* negative_test_not_captured_but_returned_call_1a(long *a) {
+;   return not_captured_but_returned_1(a);
+; }
+;
+; There should *not* be a no-capture attribute on %a
+; CHECK: define nonnull align 8 dereferenceable(8) i64* @negative_test_not_captured_but_returned_call_1a(i64* nofree writeonly align 8 "no-capture-maybe-returned" %a)
+define i64* @negative_test_not_captured_but_returned_call_1a(i64* %a) #0 {
+entry:
+  %call = call i64* @not_captured_but_returned_1(i64* %a)
+  ret i64* %call
+}
+
+; TEST captured through write
+;
+; void negative_test_not_captured_but_returned_call_1b(long *a) {
+;   *a = (long)not_captured_but_returned_1(a);
+; }
+;
+; There should *not* be a no-capture attribute on %a
+; CHECK: define void @negative_test_not_captured_but_returned_call_1b(i64* nofree writeonly align 8 %a)
+define void @negative_test_not_captured_but_returned_call_1b(i64* %a) #0 {
+entry:
+  %call = call i64* @not_captured_but_returned_1(i64* %a)
+  %0 = ptrtoint i64* %call to i64
+  store i64 %0, i64* %call, align 8
+  ret void
+}
+
+; TEST return argument or unknown call result
+;
+; int* ret_arg_or_unknown(int* b) {
+;   if (b == 0)
+;     return b;
+;   return unknown();
+; }
+;
+; Verify we do *not* assume b is returned or not captured.
+;
+; CHECK:     define i32* @ret_arg_or_unknown(i32* readnone %b)
+; CHECK:     define i32* @ret_arg_or_unknown_through_phi(i32* readnone %b)
+
+declare i32* @unknown()
+
+define i32* @ret_arg_or_unknown(i32* %b) #0 {
+entry:
+  %cmp = icmp eq i32* %b, null
+  br i1 %cmp, label %ret_arg, label %ret_unknown
+
+ret_arg:
+  ret i32* %b
+
+ret_unknown:
+  %call = call i32* @unknown()
+  ret i32* %call
+}
+
+define i32* @ret_arg_or_unknown_through_phi(i32* %b) #0 {
+entry:
+  %cmp = icmp eq i32* %b, null
+  br i1 %cmp, label %ret_arg, label %ret_unknown
+
+ret_arg:
+  br label %r
+
+ret_unknown:
+  %call = call i32* @unknown()
+  br label %r
+
+r:
+  %phi = phi i32* [ %b, %ret_arg ], [ %call, %ret_unknown ]
+  ret i32* %phi
+}
+
+
+; TEST not captured by readonly external function
+;
+; CHECK: define void @not_captured_by_readonly_call(i32* nocapture readonly %b)
+declare i32* @readonly_unknown(i32*, i32*) readonly
+
+define void @not_captured_by_readonly_call(i32* %b) #0 {
+entry:
+  %call = call i32* @readonly_unknown(i32* %b, i32* %b)
+  ret void
+}
+
+
+; TEST not captured by readonly external function if return chain is known
+;
+; Make sure the returned flag on %r is strong enough to justify nocapture on %b but **not** on %r.
+;
+; CHECK: define i32* @not_captured_by_readonly_call_not_returned_either1(i32* nocapture readonly %b, i32* readonly returned %r)
+;
+; CHECK: define i32* @not_captured_by_readonly_call_not_returned_either2(i32* nocapture readonly %b, i32* readonly returned %r)
+; CHECK: define i32* @not_captured_by_readonly_call_not_returned_either3(i32* nocapture readonly %b, i32* readonly returned %r)
+;
+; CHECK: define i32* @not_captured_by_readonly_call_not_returned_either4(i32* nocapture readonly %b, i32* readonly returned %r)
+define i32* @not_captured_by_readonly_call_not_returned_either1(i32* %b, i32* returned %r) {
+entry:
+  %call = call i32* @readonly_unknown(i32* %b, i32* %r) nounwind
+  ret i32* %call
+}
+
+declare i32* @readonly_unknown_r1a(i32*, i32* returned) readonly
+define i32* @not_captured_by_readonly_call_not_returned_either2(i32* %b, i32* %r) {
+entry:
+  %call = call i32* @readonly_unknown_r1a(i32* %b, i32* %r) nounwind
+  ret i32* %call
+}
+
+declare i32* @readonly_unknown_r1b(i32*, i32* returned) readonly nounwind
+define i32* @not_captured_by_readonly_call_not_returned_either3(i32* %b, i32* %r) {
+entry:
+  %call = call i32* @readonly_unknown_r1b(i32* %b, i32* %r)
+  ret i32* %call
+}
+
+define i32* @not_captured_by_readonly_call_not_returned_either4(i32* %b, i32* %r) nounwind {
+entry:
+  %call = call i32* @readonly_unknown_r1a(i32* %b, i32* %r)
+  ret i32* %call
+}
+
+
+declare i32* @unknown_i32p(i32*)
+define void @nocapture_is_not_subsumed_1(i32* nocapture %b) {
+; CHECK-LABEL: define {{[^@]+}}@nocapture_is_not_subsumed_1
+; CHECK-SAME: (i32* nocapture [[B:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CALL:%.*]] = call i32* @unknown_i32p(i32* [[B:%.*]])
+; CHECK-NEXT:    store i32 0, i32* [[CALL]]
+; CHECK-NEXT:    ret void
+;
+entry:
+  %call = call i32* @unknown_i32p(i32* %b)
+  store i32 0, i32* %call
+  ret void
+}
+
+declare i32* @readonly_i32p(i32*) readonly
+define void @nocapture_is_not_subsumed_2(i32* nocapture %b) {
+; CHECK-LABEL: define {{[^@]+}}@nocapture_is_not_subsumed_2
+; CHECK-SAME: (i32* nocapture nofree [[B:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CALL:%.*]] = call i32* @readonly_i32p(i32* readonly [[B:%.*]])
+; CHECK-NEXT:    store i32 0, i32* [[CALL]]
+; CHECK-NEXT:    ret void
+;
+entry:
+  %call = call i32* @readonly_i32p(i32* %b)
+  store i32 0, i32* %call
+  ret void
+}
+
+attributes #0 = { noinline nounwind uwtable }
diff --git a/llvm/test/Transforms/Attributor/nofree.ll b/llvm/test/Transforms/Attributor/nofree.ll
new file mode 100644
index 000000000..d06a0ea1e
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/nofree.ll
@@ -0,0 +1,243 @@
+; RUN: opt -attributor --attributor-disable=false -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=2 -S < %s | FileCheck %s --check-prefix=ATTRIBUTOR
+; Copied from Transforms/FunctoinAttrs/nofree-attributor.ll
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+; Test cases specifically designed for the "nofree" function attribute.
+; We use FIXME's to indicate problems and missing attributes.
+
+; Free functions
+declare void @free(i8* nocapture) local_unnamed_addr #1
+declare noalias i8* @realloc(i8* nocapture, i64) local_unnamed_addr #0
+declare void @_ZdaPv(i8*) local_unnamed_addr #2
+
+
+; TEST 1 (positive case)
+; ATTRIBUTOR: Function Attrs: nofree noinline nosync nounwind readnone uwtable
+; ATTRIBUTOR-NEXT: define void @only_return()
+define void @only_return() #0 {
+    ret void
+}
+
+
+; TEST 2 (negative case)
+; Only free
+; void only_free(char* p) {
+;    free(p);
+; }
+
+; ATTRIBUTOR: Function Attrs: noinline nounwind uwtable
+; ATTRIBUTOR-NOT: nofree
+; ATTRIBUTOR-NEXT: define void @only_free(i8* nocapture %0) local_unnamed_addr #1
+define void @only_free(i8* nocapture %0) local_unnamed_addr #0 {
+    tail call void @free(i8* %0) #1
+    ret void
+}
+
+
+; TEST 3 (negative case)
+; Free occurs in same scc.
+; void free_in_scc1(char*p){
+;    free_in_scc2(p);
+; }
+; void free_in_scc2(char*p){
+;    free_in_scc1(p);
+;    free(p);
+; }
+
+
+; ATTRIBUTOR: Function Attrs: noinline nounwind uwtable
+; ATTRIBUTOR-NOT: nofree
+; ATTRIBUTOR-NEXT :define void @free_in_scc1(i8* nocapture %0) local_unnamed_addr
+define void @free_in_scc1(i8* nocapture %0) local_unnamed_addr #0 {
+  tail call void @free_in_scc2(i8* %0) #1
+  ret void
+}
+
+
+; ATTRIBUTOR: Function Attrs: noinline nounwind uwtable
+; ATTRIBUTOR-NOT: nofree
+; ATTRIBUTOR: define void @free_in_scc2(i8* nocapture %0) local_unnamed_addr
+define void @free_in_scc2(i8* nocapture %0) local_unnamed_addr #0 {
+  %cmp = icmp eq i8* %0, null
+  br i1 %cmp, label %rec, label %call
+call:
+  tail call void @free(i8* %0) #1
+  br label %end
+rec:
+  tail call void @free_in_scc1(i8* %0)
+  br label %end
+end:
+  ret void
+}
+
+
+; TEST 4 (positive case)
+; Free doesn't occur.
+; void mutual_recursion1(){
+;    mutual_recursion2();
+; }
+; void mutual_recursion2(){
+;     mutual_recursion1();
+; }
+
+
+; ATTRIBUTOR: Function Attrs: nofree noinline noreturn nosync nounwind readnone uwtable
+; ATTRIBUTOR-NEXT: define void @mutual_recursion1()
+define void @mutual_recursion1() #0 {
+  call void @mutual_recursion2()
+  ret void
+}
+
+; ATTRIBUTOR: Function Attrs: nofree noinline noreturn nosync nounwind readnone uwtable
+; ATTRIBUTOR-NEXT: define void @mutual_recursion2()
+define void @mutual_recursion2() #0 {
+  call void @mutual_recursion1()
+  ret void
+}
+
+
+; TEST 5
+; C++ delete operation (negative case)
+; void delete_op (char p[]){
+;     delete [] p;
+; }
+
+; ATTRIBUTOR: Function Attrs: noinline nounwind uwtable
+; ATTRIBUTOR-NOT: nofree
+; ATTRIBUTOR-NEXT: define void @_Z9delete_opPc(i8* %0) local_unnamed_addr #1
+define void @_Z9delete_opPc(i8* %0) local_unnamed_addr #0 {
+  %2 = icmp eq i8* %0, null
+  br i1 %2, label %4, label %3
+
+; <label>:3:                                      ; preds = %1
+  tail call void @_ZdaPv(i8* nonnull %0) #2
+  br label %4
+
+; <label>:4:                                      ; preds = %3, %1
+  ret void
+}
+
+
+; TEST 6 (negative case)
+; Call realloc
+; ATTRIBUTOR: Function Attrs: noinline nounwind uwtable
+; ATTRIBUTOR-NOT: nofree
+; ATTRIBUTOR-NEXT: define noalias i8* @call_realloc(i8* nocapture %0, i64 %1) local_unnamed_addr
+define noalias i8* @call_realloc(i8* nocapture %0, i64 %1) local_unnamed_addr #0 {
+    %ret = tail call i8* @realloc(i8* %0, i64 %1) #2
+    ret i8* %ret
+}
+
+
+; TEST 7 (positive case)
+; Call function declaration with "nofree"
+
+
+; ATTRIBUTOR: Function Attrs:  nofree noinline nounwind readnone uwtable 
+; ATTRIBUTOR-NEXT: declare void @nofree_function()
+declare void @nofree_function() nofree readnone #0
+
+; ATTRIBUTOR: Function Attrs: nofree noinline nosync nounwind readnone uwtable
+; ATTRIBUTOR-NEXT: define void @call_nofree_function()
+define void @call_nofree_function() #0 {
+    tail call void @nofree_function()
+    ret void
+}
+
+; TEST 8 (negative case)
+; Call function declaration without "nofree"
+
+
+; ATTRIBUTOR: Function Attrs: noinline nounwind uwtable
+; ATTRIBUTOR-NEXT: declare void @maybe_free()
+declare void @maybe_free() #0
+
+
+; ATTRIBUTOR: Function Attrs: noinline nounwind uwtable
+; ATTRIBUTOR-NOT: nofree
+; ATTRIBUTOR-NEXT: define void @call_maybe_free()
+define void @call_maybe_free() #0 {
+    tail call void @maybe_free()
+    ret void
+}
+
+
+; TEST 9 (negative case)
+; Call both of above functions
+
+; ATTRIBUTOR: Function Attrs: noinline nounwind uwtable
+; ATTRIBUTOR-NOT: nofree
+; ATTRIBUTOR-NEXT: define void @call_both()
+define void @call_both() #0 {
+    tail call void @maybe_free()
+    tail call void @nofree_function()
+    ret void
+}
+
+
+; TEST 10 (positive case)
+; Call intrinsic function
+; ATTRIBUTOR: Function Attrs: nounwind readnone speculatable
+; ATTRIBUTOR-NEXT: declare float @llvm.floor.f32(float)
+declare float @llvm.floor.f32(float)
+
+; FIXME: missing nofree
+; ATTRIBUTOR: Function Attrs: noinline nosync nounwind readnone uwtable
+; ATTRIBUTOR-NEXT: define void @call_floor(float %a)
+
+define void @call_floor(float %a) #0 {
+    tail call float @llvm.floor.f32(float %a)
+    ret void
+}
+
+; TEST 11 (positive case)
+; Check propagation.
+
+; ATTRIBUTOR: Function Attrs: nofree noinline nosync nounwind readnone uwtable
+; ATTRIBUTOR-NEXT: define void @f1()
+define void @f1() #0 {
+    tail call void @nofree_function()
+    ret void
+}
+
+; ATTRIBUTOR: Function Attrs: nofree noinline nosync nounwind readnone uwtable
+; ATTRIBUTOR-NEXT: define void @f2()
+define void @f2() #0 {
+    tail call void @f1()
+    ret void
+}
+
+; TEST 12 NoFree argument - positive.
+; ATTRIBUTOR: define double @test12(double* nocapture nofree nonnull readonly align 8 dereferenceable(8) %a)
+define double @test12(double* nocapture readonly %a) {
+entry:
+	%0 = load double, double* %a, align 8
+	%call = tail call double @cos(double %0) #2
+	ret double %call
+}
+
+declare double @cos(double) nobuiltin nounwind nofree
+
+; FIXME: %a should be nofree.
+; TEST 13 NoFree argument - positive.
+; ATTRIBUTOR: define noalias i32* @test13(i64* nocapture nonnull readonly align 8 dereferenceable(8) %a)
+define noalias i32* @test13(i64* nocapture readonly %a) {
+entry:
+	%0 = load i64, i64* %a, align 8
+	%call = tail call noalias i8* @malloc(i64 %0) #2
+	%1 = bitcast i8* %call to i32*
+	ret i32* %1
+}
+
+; ATTRIBUTOR: define void @test14(i8* nocapture %0, i8* nocapture nofree readnone %1)
+define void @test14(i8* nocapture %0, i8* nocapture %1) {
+	tail call void @free(i8* %0) #1
+	ret void
+}
+
+declare noalias i8* @malloc(i64)
+
+attributes #0 = { nounwind uwtable noinline }
+attributes #1 = { nounwind }
+attributes #2 = { nobuiltin nounwind }
diff --git a/llvm/test/Transforms/Attributor/nonnull.ll b/llvm/test/Transforms/Attributor/nonnull.ll
new file mode 100644
index 000000000..519d48bd7
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/nonnull.ll
@@ -0,0 +1,840 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -attributor --attributor-disable=false -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=5 -S < %s | FileCheck %s --check-prefixes=ATTRIBUTOR,ATTRIBUTOR_OPM
+; RUN: opt -passes=attributor --attributor-disable=false -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=5 -S < %s | FileCheck %s --check-prefixes=ATTRIBUTOR,ATTRIBUTOR_NPM
+; Copied from Transforms/FunctoinAttrs/nonnull.ll
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+declare nonnull i8* @ret_nonnull()
+
+; Return a pointer trivially nonnull (call return attribute)
+define i8* @test1() {
+; ATTRIBUTOR: define nonnull i8* @test1
+  %ret = call i8* @ret_nonnull()
+  ret i8* %ret
+}
+
+; Return a pointer trivially nonnull (argument attribute)
+define i8* @test2(i8* nonnull %p) {
+; ATTRIBUTOR: define nonnull i8* @test2
+  ret i8* %p
+}
+
+; Given an SCC where one of the functions can not be marked nonnull,
+; can we still mark the other one which is trivially nonnull
+define i8* @scc_binder(i1 %c) {
+; ATTRIBUTOR: define noalias i8* @scc_binder
+  br i1 %c, label %rec, label %end
+rec:
+  call i8* @test3(i1 %c)
+  br label %end
+end:
+  ret i8* null
+}
+
+define i8* @test3(i1 %c) {
+; ATTRIBUTOR: define nonnull i8* @test3
+  call i8* @scc_binder(i1 %c)
+  %ret = call i8* @ret_nonnull()
+  ret i8* %ret
+}
+
+; Given a mutual recursive set of functions, we can mark them
+; nonnull if neither can ever return null.  (In this case, they
+; just never return period.)
+define i8* @test4_helper() {
+; ATTRIBUTOR: define noalias nonnull align 536870912 dereferenceable(4294967295) i8* @test4_helper
+  %ret = call i8* @test4()
+  ret i8* %ret
+}
+
+define i8* @test4() {
+; ATTRIBUTOR: define noalias nonnull align 536870912 dereferenceable(4294967295) i8* @test4
+  %ret = call i8* @test4_helper()
+  ret i8* %ret
+}
+
+; Given a mutual recursive set of functions which *can* return null
+; make sure we haven't marked them as nonnull.
+define i8* @test5_helper(i1 %c) {
+; ATTRIBUTOR: define noalias i8* @test5_helper
+  br i1 %c, label %rec, label %end
+rec:
+  %ret = call i8* @test5(i1 %c)
+  br label %end
+end:
+  ret i8* null
+}
+
+define i8* @test5(i1 %c) {
+; ATTRIBUTOR: define noalias i8* @test5
+  %ret = call i8* @test5_helper(i1 %c)
+  ret i8* %ret
+}
+
+; Local analysis, but going through a self recursive phi
+; ATTRIBUTOR: Function Attrs: noreturn
+; ATTRIBUTOR: define noalias nonnull align 536870912 dereferenceable(4294967295) i8* @test6a()
+define i8* @test6a() {
+entry:
+  %ret = call i8* @ret_nonnull()
+  br label %loop
+loop:
+  %phi = phi i8* [%ret, %entry], [%phi, %loop]
+  br i1 undef, label %loop, label %exit
+exit:
+  ret i8* %phi
+}
+
+; ATTRIBUTOR: define nonnull i8* @test6b(i1 %c)
+define i8* @test6b(i1 %c) {
+entry:
+  %ret = call i8* @ret_nonnull()
+  br label %loop
+loop:
+  %phi = phi i8* [%ret, %entry], [%phi, %loop]
+  br i1 %c, label %loop, label %exit
+exit:
+  ret i8* %phi
+}
+
+; ATTRIBUTOR: define i8* @test7
+define i8* @test7(i8* %a) {
+  %b = getelementptr inbounds i8, i8* %a, i64 0
+  ret i8* %b
+}
+
+; ATTRIBUTOR: define nonnull i8* @test8
+define i8* @test8(i8* %a) {
+  %b = getelementptr inbounds i8, i8* %a, i64 1
+  ret i8* %b
+}
+
+; ATTRIBUTOR: define i8* @test9
+define i8* @test9(i8* %a, i64 %n) {
+  %b = getelementptr inbounds i8, i8* %a, i64 %n
+  ret i8* %b
+}
+
+declare void @llvm.assume(i1)
+; FIXME: missing nonnull
+; ATTRIBUTOR: define i8* @test10
+define i8* @test10(i8* %a, i64 %n) {
+  %cmp = icmp ne i64 %n, 0
+  call void @llvm.assume(i1 %cmp)
+  %b = getelementptr inbounds i8, i8* %a, i64 %n
+  ret i8* %b
+}
+
+; TEST 11
+; char* test11(char *p) {
+;   return p? p: nonnull();
+; }
+; FIXME: missing nonnull
+; ATTRIBUTOR: define i8* @test11
+define i8* @test11(i8*) local_unnamed_addr {
+  %2 = icmp eq i8* %0, null
+  br i1 %2, label %3, label %5
+
+; <label>:3:                                      ; preds = %1
+  %4 = tail call i8* @ret_nonnull()
+  br label %5
+
+; <label>:5:                                      ; preds = %3, %1
+  %6 = phi i8* [ %4, %3 ], [ %0, %1 ]
+  ret i8* %6
+}
+
+; TEST 12
+; Simple CallSite Test
+declare void @test12_helper(i8*)
+define void @test12(i8* nonnull %a) {
+; ATTRIBUTOR: define void @test12(i8* nonnull %a)
+; ATTRIBUTOR-NEXT: tail call void @test12_helper(i8* nonnull %a)
+  tail call void @test12_helper(i8* %a)
+  ret void
+}
+
+; TEST 13
+; Simple Argument Tests
+declare i8* @unknown()
+define void @test13_helper() {
+  %nonnullptr = tail call i8* @ret_nonnull()
+  %maybenullptr = tail call i8* @unknown()
+  tail call void @test13(i8* %nonnullptr, i8* %nonnullptr, i8* %maybenullptr)
+  tail call void @test13(i8* %nonnullptr, i8* %maybenullptr, i8* %nonnullptr)
+  ret void
+}
+declare void @use_i8_ptr(i8* nofree) readnone nounwind
+define internal void @test13(i8* %a, i8* %b, i8* %c) {
+; ATTRIBUTOR: define internal void @test13(i8* nocapture nofree nonnull readnone %a, i8* nocapture nofree readnone %b, i8* nocapture nofree readnone %c)
+  call void @use_i8_ptr(i8* %a)
+  call void @use_i8_ptr(i8* %b)
+  call void @use_i8_ptr(i8* %c)
+  ret void
+}
+
+declare nonnull i8* @nonnull()
+
+; TEST 14
+; Complex propagation
+; Argument of f1, f2, f3 can be marked with nonnull.
+
+; * Argument
+; 1. In f1:bb6, %arg can be marked with nonnull because of the comparison in bb1
+; 2. Because f2 is internal function, f2(i32* %arg) -> @f2(i32* nonnull %arg)
+; 3. In f1:bb4 %tmp5 is nonnull and f3 is internal function.
+;    Then, f3(i32* %arg) -> @f3(i32* nonnull %arg)
+; 4. We get nonnull in whole f1 call sites so f1(i32* %arg) -> @f1(i32* nonnull %arg)
+
+
+define internal i32* @f1(i32* %arg) {
+; FIXME: missing nonnull It should be nonnull @f1(i32* nonnull readonly %arg)
+; ATTRIBUTOR: define internal nonnull i32* @f1(i32* nofree readonly %arg)
+
+bb:
+  %tmp = icmp eq i32* %arg, null
+  br i1 %tmp, label %bb9, label %bb1
+
+bb1:                                              ; preds = %bb
+  %tmp2 = load i32, i32* %arg, align 4
+  %tmp3 = icmp eq i32 %tmp2, 0
+  br i1 %tmp3, label %bb6, label %bb4
+
+bb4:                                              ; preds = %bb1
+  %tmp5 = getelementptr inbounds i32, i32* %arg, i64 1
+; ATTRIBUTOR: %tmp5b = tail call nonnull i32* @f3(i32* nofree nonnull readonly %tmp5)
+  %tmp5b = tail call i32* @f3(i32* %tmp5)
+  %tmp5c = getelementptr inbounds i32, i32* %tmp5b, i64 -1
+  br label %bb9
+
+bb6:                                              ; preds = %bb1
+; FIXME: missing nonnull. It should be @f2(i32* nonnull %arg)
+; ATTRIBUTOR: %tmp7 = tail call nonnull i32* @f2(i32* nofree readonly %arg)
+  %tmp7 = tail call i32* @f2(i32* %arg)
+  ret i32* %tmp7
+
+bb9:                                              ; preds = %bb4, %bb
+  %tmp10 = phi i32* [ %tmp5c, %bb4 ], [ inttoptr (i64 4 to i32*), %bb ]
+  ret i32* %tmp10
+}
+
+define internal i32* @f2(i32* %arg) {
+; FIXME: missing nonnull. It should be nonnull @f2(i32* nonnull %arg)
+; ATTRIBUTOR: define internal nonnull i32* @f2(i32* nofree readonly %arg)
+bb:
+
+; FIXME: missing nonnull. It should be @f1(i32* nonnull readonly %arg)
+; ATTRIBUTOR:   %tmp = tail call nonnull i32* @f1(i32* nofree readonly %arg)
+  %tmp = tail call i32* @f1(i32* %arg)
+  ret i32* %tmp
+}
+
+define dso_local noalias i32* @f3(i32* %arg) {
+; FIXME: missing nonnull. It should be nonnull @f3(i32* nonnull readonly %arg)
+; ATTRIBUTOR: define dso_local noalias nonnull i32* @f3(i32* nofree readonly %arg)
+bb:
+; FIXME: missing nonnull. It should be @f1(i32* nonnull readonly %arg)
+; ATTRIBUTOR:   %tmp = call nonnull i32* @f1(i32* nofree readonly %arg)
+  %tmp = call i32* @f1(i32* %arg)
+  ret i32* %tmp
+}
+
+; TEST 15
+define void @f15(i8* %arg) {
+; ATTRIBUTOR:   tail call void @use1(i8* nonnull dereferenceable(4) %arg)
+
+  tail call void @use1(i8* dereferenceable(4) %arg)
+  ret void
+}
+
+declare void @fun0() #1
+declare void @fun1(i8*) #1
+declare void @fun2(i8*, i8*) #1
+declare void @fun3(i8*, i8*, i8*) #1
+; TEST 16 simple path test
+; if(..)
+;   fun2(nonnull %a, nonnull %b)
+; else
+;   fun2(nonnull %a, %b)
+; We can say that %a is nonnull but %b is not.
+define void @f16(i8* %a, i8 * %b, i8 %c) {
+; FIXME: missing nonnull on %a
+; ATTRIBUTOR: define void @f16(i8* %a, i8* %b, i8 %c)
+  %cmp = icmp eq i8 %c, 0
+  br i1 %cmp, label %if.then, label %if.else
+if.then:
+  tail call void @fun2(i8* nonnull %a, i8* nonnull %b)
+  ret void
+if.else:
+  tail call void @fun2(i8* nonnull %a, i8* %b)
+  ret void
+}
+; TEST 17 explore child BB test
+; if(..)
+;    ... (willreturn & nounwind)
+; else
+;    ... (willreturn & nounwind)
+; fun1(nonnull %a)
+; We can say that %a is nonnull
+define void @f17(i8* %a, i8 %c) {
+; ATTRIBUTOR: define void @f17(i8* nonnull %a, i8 %c)
+  %cmp = icmp eq i8 %c, 0
+  br i1 %cmp, label %if.then, label %if.else
+if.then:
+  tail call void @fun0()
+  br label %cont
+if.else:
+  tail call void @fun0()
+  br label %cont
+cont:
+  tail call void @fun1(i8* nonnull %a)
+  ret void
+}
+; TEST 18 More complex test
+; if(..)
+;    ... (willreturn & nounwind)
+; else
+;    ... (willreturn & nounwind)
+; if(..)
+;    ... (willreturn & nounwind)
+; else
+;    ... (willreturn & nounwind)
+; fun1(nonnull %a)
+
+define void @f18(i8* %a, i8* %b, i8 %c) {
+; ATTRIBUTOR: define void @f18(i8* nonnull %a, i8* %b, i8 %c)
+  %cmp1 = icmp eq i8 %c, 0
+  br i1 %cmp1, label %if.then, label %if.else
+if.then:
+  tail call void @fun0()
+  br label %cont
+if.else:
+  tail call void @fun0()
+  br label %cont
+cont:
+  %cmp2 = icmp eq i8 %c, 1
+  br i1 %cmp2, label %cont.then, label %cont.else
+cont.then:
+  tail call void @fun1(i8* nonnull %b)
+  br label %cont2
+cont.else:
+  tail call void @fun0()
+  br label %cont2
+cont2:
+  tail call void @fun1(i8* nonnull %a)
+  ret void
+}
+
+; TEST 19: Loop
+
+define void @f19(i8* %a, i8* %b, i8 %c) {
+; FIXME: missing nonnull on %b
+; ATTRIBUTOR: define void @f19(i8* %a, i8* %b, i8 %c)
+  br label %loop.header
+loop.header:
+  %cmp2 = icmp eq i8 %c, 0
+  br i1 %cmp2, label %loop.body, label %loop.exit
+loop.body:
+  tail call void @fun1(i8* nonnull %b)
+  tail call void @fun1(i8* nonnull %a)
+  br label %loop.header
+loop.exit:
+  tail call void @fun1(i8* nonnull %b)
+  ret void
+}
+
+; Test propagation of nonnull callsite args back to caller.
+
+declare void @use1(i8* %x)
+declare void @use2(i8* %x, i8* %y);
+declare void @use3(i8* %x, i8* %y, i8* %z);
+
+declare void @use1nonnull(i8* nonnull %x);
+declare void @use2nonnull(i8* nonnull %x, i8* nonnull %y);
+declare void @use3nonnull(i8* nonnull %x, i8* nonnull %y, i8* nonnull %z);
+
+declare i8 @use1safecall(i8* %x) readonly nounwind ; readonly+nounwind guarantees that execution continues to successor
+
+; Can't extend non-null to parent for any argument because the 2nd call is not guaranteed to execute.
+
+define void @parent1(i8* %a, i8* %b, i8* %c) {
+; ATTRIBUTOR-LABEL: @parent1(i8* %a, i8* %b, i8* %c)
+; ATTRIBUTOR-NEXT:    call void @use3(i8* %c, i8* %a, i8* %b)
+; ATTRIBUTOR-NEXT:    call void @use3nonnull(i8* nonnull %b, i8* nonnull %c, i8* nonnull %a)
+; ATTRIBUTOR-NEXT:    ret void
+  call void @use3(i8* %c, i8* %a, i8* %b)
+  call void @use3nonnull(i8* %b, i8* %c, i8* %a)
+  ret void
+}
+
+; Extend non-null to parent for all arguments.
+
+define void @parent2(i8* %a, i8* %b, i8* %c) {
+
+; ATTRIBUTOR-LABEL: @parent2(i8* nonnull %a, i8* nonnull %b, i8* nonnull %c)
+; ATTRIBUTOR-NEXT:    call void @use3nonnull(i8* nonnull %b, i8* nonnull %c, i8* nonnull %a)
+; ATTRIBUTOR-NEXT:    call void @use3(i8* nonnull %c, i8* nonnull %a, i8* nonnull %b)
+
+; ATTRIBUTOR-NEXT:    ret void
+  call void @use3nonnull(i8* %b, i8* %c, i8* %a)
+  call void @use3(i8* %c, i8* %a, i8* %b)
+  ret void
+}
+
+; Extend non-null to parent for 1st argument.
+
+define void @parent3(i8* %a, i8* %b, i8* %c) {
+
+; ATTRIBUTOR-LABEL: @parent3(i8* nonnull %a, i8* %b, i8* %c)
+; ATTRIBUTOR-NEXT:    call void @use1nonnull(i8* nonnull %a)
+; ATTRIBUTOR-NEXT:    call void @use3(i8* %c, i8* %b, i8* nonnull %a)
+
+; ATTRIBUTOR-NEXT:  ret void
+
+  call void @use1nonnull(i8* %a)
+  call void @use3(i8* %c, i8* %b, i8* %a)
+  ret void
+}
+
+; Extend non-null to parent for last 2 arguments.
+
+define void @parent4(i8* %a, i8* %b, i8* %c) {
+; CHECK-LABEL: @parent4(i8* %a, i8* nonnull %b, i8* nonnull %c)
+; CHECK-NEXT:    call void @use2nonnull(i8* %c, i8* %b)
+; CHECK-NEXT:    call void @use2(i8* %a, i8* %c)
+; CHECK-NEXT:    call void @use1(i8* %b)
+
+; ATTRIBUTOR-LABEL: @parent4(i8* %a, i8* nonnull %b, i8* nonnull %c)
+; ATTRIBUTOR-NEXT:    call void @use2nonnull(i8* nonnull %c, i8* nonnull %b)
+; ATTRIBUTOR-NEXT:    call void @use2(i8* %a, i8* nonnull %c)
+; ATTRIBUTOR-NEXT:    call void @use1(i8* nonnull %b)
+
+; ATTRIBUTOR: ret void
+
+  call void @use2nonnull(i8* %c, i8* %b)
+  call void @use2(i8* %a, i8* %c)
+  call void @use1(i8* %b)
+  ret void
+}
+
+; The callsite must execute in order for the attribute to transfer to the parent.
+; It appears benign to extend non-null to the parent in this case, but we can't do that
+; because it would incorrectly propagate the wrong information to its callers.
+
+define void @parent5(i8* %a, i1 %a_is_notnull) {
+; ATTRIBUTOR: @parent5(i8* %a, i1 %a_is_notnull)
+; ATTRIBUTOR-NEXT:    br i1 %a_is_notnull, label %t, label %f
+; ATTRIBUTOR:       t:
+; ATTRIBUTOR-NEXT:    call void @use1nonnull(i8* nonnull %a)
+; ATTRIBUTOR-NEXT:    ret void
+; ATTRIBUTOR:       f:
+; ATTRIBUTOR-NEXT:    ret void
+
+  br i1 %a_is_notnull, label %t, label %f
+t:
+  call void @use1nonnull(i8* %a)
+  ret void
+f:
+  ret void
+}
+
+; The callsite must execute in order for the attribute to transfer to the parent.
+; The volatile load can't trap, so we can guarantee that we'll get to the call.
+
+define i8 @parent6(i8* %a, i8* %b) {
+; ATTRIBUTOR-LABEL: @parent6(i8* nonnull %a, i8* %b)
+; ATTRIBUTOR-NEXT:    [[C:%.*]] = load volatile i8, i8* %b
+; ATTRIBUTOR-NEXT:    call void @use1nonnull(i8* nonnull %a)
+; ATTRIBUTOR-NEXT:    ret i8 [[C]]
+
+  %c = load volatile i8, i8* %b
+  call void @use1nonnull(i8* %a)
+  ret i8 %c
+}
+
+; The nonnull callsite is guaranteed to execute, so the argument must be nonnull throughout the parent.
+
+define i8 @parent7(i8* %a) {
+
+
+; ATTRIBUTOR-LABEL: @parent7(i8* nonnull %a)
+; ATTRIBUTOR-NEXT:    [[RET:%.*]] = call i8 @use1safecall(i8* nonnull readonly %a)
+; ATTRIBUTOR-NEXT:    call void @use1nonnull(i8* nonnull %a)
+
+; ATTRIBUTOR-NEXT: ret i8 [[RET]]
+
+  %ret = call i8 @use1safecall(i8* %a)
+  call void @use1nonnull(i8* %a)
+  ret i8 %ret
+}
+
+; Make sure that an invoke works similarly to a call.
+
+declare i32 @esfp(...)
+
+define i1 @parent8(i8* %a, i8* %bogus1, i8* %b) personality i8* bitcast (i32 (...)* @esfp to i8*){
+; ATTRIBUTOR-LABEL: @parent8(i8* nonnull %a, i8* nocapture nofree readnone %bogus1, i8* nonnull %b)
+; ATTRIBUTOR-NEXT:  entry:
+; ATTRIBUTOR-NEXT:    invoke void @use2nonnull(i8* nonnull %a, i8* nonnull %b)
+; ATTRIBUTOR-NEXT:    to label %cont unwind label %exc
+; ATTRIBUTOR:       cont:
+; ATTRIBUTOR-NEXT:    [[NULL_CHECK:%.*]] = icmp eq i8* %b, null
+; ATTRIBUTOR-NEXT:    ret i1 [[NULL_CHECK]]
+; ATTRIBUTOR:       exc:
+; ATTRIBUTOR-NEXT:    [[LP:%.*]] = landingpad { i8*, i32 }
+; ATTRIBUTOR-NEXT:    filter [0 x i8*] zeroinitializer
+; ATTRIBUTOR-NEXT:    unreachable
+
+entry:
+  invoke void @use2nonnull(i8* %a, i8* %b)
+  to label %cont unwind label %exc
+
+cont:
+  %null_check = icmp eq i8* %b, null
+  ret i1 %null_check
+
+exc:
+  %lp = landingpad { i8*, i32 }
+  filter [0 x i8*] zeroinitializer
+  unreachable
+}
+
+; ATTRIBUTOR: define nonnull i32* @gep1(
+define i32* @gep1(i32* %p) {
+  %q = getelementptr inbounds i32, i32* %p, i32 1
+  ret i32* %q
+}
+
+define i32* @gep1_no_null_opt(i32* %p) #0 {
+; Should't be able to derive nonnull based on gep.
+; ATTRIBUTOR: define i32* @gep1_no_null_opt(
+  %q = getelementptr inbounds i32, i32* %p, i32 1
+  ret i32* %q
+}
+
+; ATTRIBUTOR: define i32 addrspace(3)* @gep2(
+define i32 addrspace(3)* @gep2(i32 addrspace(3)* %p) {
+  %q = getelementptr inbounds i32, i32 addrspace(3)* %p, i32 1
+  ret i32 addrspace(3)* %q
+}
+
+; FIXME: We should propagate dereferenceable here but *not* nonnull
+; ATTRIBUTOR: define dereferenceable_or_null(4) i32 addrspace(3)* @as(i32 addrspace(3)* nofree readnone returned dereferenceable(4) dereferenceable_or_null(4) %p)
+define i32 addrspace(3)* @as(i32 addrspace(3)* dereferenceable(4) %p) {
+  ret i32 addrspace(3)* %p
+}
+
+; ATTRIBUTOR: define internal nonnull i32* @g2()
+define internal i32* @g2() {
+  ret i32* inttoptr (i64 4 to i32*)
+}
+
+define  i32* @g1() {
+ %c = call i32* @g2()
+  ret i32* %c
+}
+
+declare void @use_i32_ptr(i32*) readnone nounwind
+; ATTRIBUTOR: define internal void @called_by_weak(i32* nocapture nonnull readnone %a)
+define internal void @called_by_weak(i32* %a) {
+  call void @use_i32_ptr(i32* %a)
+  ret void
+}
+
+; Check we do not annotate the function interface of this weak function.
+; ATTRIBUTOR: define weak_odr void @weak_caller(i32* nonnull %a)
+define weak_odr void @weak_caller(i32* nonnull %a) {
+  call void @called_by_weak(i32* %a)
+  ret void
+}
+
+; Expect nonnull
+; ATTRIBUTOR: define internal void @control(i32* nocapture nonnull readnone align 16 dereferenceable(8) %a)
+define internal void @control(i32* dereferenceable(4) %a) {
+  call void @use_i32_ptr(i32* %a)
+  ret void
+}
+; Avoid nonnull as we do not touch naked functions
+; ATTRIBUTOR: define internal void @naked(i32* dereferenceable(4) %a)
+define internal void @naked(i32* dereferenceable(4) %a) naked {
+  call void @use_i32_ptr(i32* %a)
+  ret void
+}
+; Avoid nonnull as we do not touch optnone
+; ATTRIBUTOR: define internal void @optnone(i32* dereferenceable(4) %a)
+define internal void @optnone(i32* dereferenceable(4) %a) optnone noinline {
+  call void @use_i32_ptr(i32* %a)
+  ret void
+}
+define void @make_live(i32* nonnull dereferenceable(8) %a) {
+  call void @naked(i32* nonnull dereferenceable(8) align 16 %a)
+  call void @control(i32* nonnull dereferenceable(8) align 16 %a)
+  call void @optnone(i32* nonnull dereferenceable(8) align 16 %a)
+  ret void
+}
+
+;int f(int *u, int n){
+;  for(int i = 0;i<n;i++){
+;    h(u);
+;  }
+;  return g(nonnull u);
+;}
+declare void @h(i32*) willreturn nounwind
+declare i32 @g(i32*) willreturn nounwind
+define i32 @nonnull_exec_ctx_1(i32* %a, i32 %b) {
+;
+; ATTRIBUTOR-LABEL: define {{[^@]+}}@nonnull_exec_ctx_1
+; ATTRIBUTOR-SAME: (i32* [[A:%.*]], i32 [[B:%.*]])
+; ATTRIBUTOR-NEXT:  en:
+; ATTRIBUTOR-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[B:%.*]], 0
+; ATTRIBUTOR-NEXT:    br i1 [[TMP3]], label [[EX:%.*]], label [[HD:%.*]]
+; ATTRIBUTOR:       ex:
+; ATTRIBUTOR-NEXT:    [[TMP5:%.*]] = tail call i32 @g(i32* nonnull [[A:%.*]])
+; ATTRIBUTOR-NEXT:    ret i32 [[TMP5]]
+; ATTRIBUTOR:       hd:
+; ATTRIBUTOR-NEXT:    [[TMP7:%.*]] = phi i32 [ [[TMP8:%.*]], [[HD]] ], [ 0, [[EN:%.*]] ]
+; ATTRIBUTOR-NEXT:    tail call void @h(i32* [[A]])
+; ATTRIBUTOR-NEXT:    [[TMP8]] = add nuw i32 [[TMP7]], 1
+; ATTRIBUTOR-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[TMP8]], [[B]]
+; ATTRIBUTOR-NEXT:    br i1 [[TMP9]], label [[EX]], label [[HD]]
+;
+en:
+  %tmp3 = icmp eq i32 %b, 0
+  br i1 %tmp3, label %ex, label %hd
+
+ex:
+  %tmp5 = tail call i32 @g(i32* nonnull %a)
+  ret i32 %tmp5
+
+hd:
+  %tmp7 = phi i32 [ %tmp8, %hd ], [ 0, %en ]
+  tail call void @h(i32* %a)
+  %tmp8 = add nuw i32 %tmp7, 1
+  %tmp9 = icmp eq i32 %tmp8, %b
+  br i1 %tmp9, label %ex, label %hd
+}
+
+define i32 @nonnull_exec_ctx_1b(i32* %a, i32 %b) {
+;
+; ATTRIBUTOR-LABEL: define {{[^@]+}}@nonnull_exec_ctx_1b
+; ATTRIBUTOR-SAME: (i32* [[A:%.*]], i32 [[B:%.*]])
+; ATTRIBUTOR-NEXT:  en:
+; ATTRIBUTOR-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[B:%.*]], 0
+; ATTRIBUTOR-NEXT:    br i1 [[TMP3]], label [[EX:%.*]], label [[HD:%.*]]
+; ATTRIBUTOR:       ex:
+; ATTRIBUTOR-NEXT:    [[TMP5:%.*]] = tail call i32 @g(i32* nonnull [[A:%.*]])
+; ATTRIBUTOR-NEXT:    ret i32 [[TMP5]]
+; ATTRIBUTOR:       hd:
+; ATTRIBUTOR-NEXT:    [[TMP7:%.*]] = phi i32 [ [[TMP8:%.*]], [[HD2:%.*]] ], [ 0, [[EN:%.*]] ]
+; ATTRIBUTOR-NEXT:    tail call void @h(i32* [[A]])
+; ATTRIBUTOR-NEXT:    br label [[HD2]]
+; ATTRIBUTOR:       hd2:
+; ATTRIBUTOR-NEXT:    [[TMP8]] = add nuw i32 [[TMP7]], 1
+; ATTRIBUTOR-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[TMP8]], [[B]]
+; ATTRIBUTOR-NEXT:    br i1 [[TMP9]], label [[EX]], label [[HD]]
+;
+en:
+  %tmp3 = icmp eq i32 %b, 0
+  br i1 %tmp3, label %ex, label %hd
+
+ex:
+  %tmp5 = tail call i32 @g(i32* nonnull %a)
+  ret i32 %tmp5
+
+hd:
+  %tmp7 = phi i32 [ %tmp8, %hd2 ], [ 0, %en ]
+  tail call void @h(i32* %a)
+  br label %hd2
+
+hd2:
+  %tmp8 = add nuw i32 %tmp7, 1
+  %tmp9 = icmp eq i32 %tmp8, %b
+  br i1 %tmp9, label %ex, label %hd
+}
+
+define i32 @nonnull_exec_ctx_2(i32* %a, i32 %b) willreturn nounwind {
+;
+; ATTRIBUTOR-LABEL: define {{[^@]+}}@nonnull_exec_ctx_2
+; ATTRIBUTOR-SAME: (i32* [[A:%.*]], i32 [[B:%.*]])
+; ATTRIBUTOR-NEXT:  en:
+; ATTRIBUTOR-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[B:%.*]], 0
+; ATTRIBUTOR-NEXT:    br i1 [[TMP3]], label [[EX:%.*]], label [[HD:%.*]]
+; ATTRIBUTOR:       ex:
+; ATTRIBUTOR-NEXT:    [[TMP5:%.*]] = tail call i32 @g(i32* nonnull [[A:%.*]])
+; ATTRIBUTOR-NEXT:    ret i32 [[TMP5]]
+; ATTRIBUTOR:       hd:
+; ATTRIBUTOR-NEXT:    [[TMP7:%.*]] = phi i32 [ [[TMP8:%.*]], [[HD]] ], [ 0, [[EN:%.*]] ]
+; ATTRIBUTOR-NEXT:    tail call void @h(i32* nonnull [[A]])
+; ATTRIBUTOR-NEXT:    [[TMP8]] = add nuw i32 [[TMP7]], 1
+; ATTRIBUTOR-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[TMP8]], [[B]]
+; ATTRIBUTOR-NEXT:    br i1 [[TMP9]], label [[EX]], label [[HD]]
+;
+en:
+  %tmp3 = icmp eq i32 %b, 0
+  br i1 %tmp3, label %ex, label %hd
+
+ex:
+  %tmp5 = tail call i32 @g(i32* nonnull %a)
+  ret i32 %tmp5
+
+hd:
+  %tmp7 = phi i32 [ %tmp8, %hd ], [ 0, %en ]
+  tail call void @h(i32* %a)
+  %tmp8 = add nuw i32 %tmp7, 1
+  %tmp9 = icmp eq i32 %tmp8, %b
+  br i1 %tmp9, label %ex, label %hd
+}
+
+define i32 @nonnull_exec_ctx_2b(i32* %a, i32 %b) willreturn nounwind {
+;
+; ATTRIBUTOR-LABEL: define {{[^@]+}}@nonnull_exec_ctx_2b
+; ATTRIBUTOR-SAME: (i32* [[A:%.*]], i32 [[B:%.*]])
+; ATTRIBUTOR-NEXT:  en:
+; ATTRIBUTOR-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[B:%.*]], 0
+; ATTRIBUTOR-NEXT:    br i1 [[TMP3]], label [[EX:%.*]], label [[HD:%.*]]
+; ATTRIBUTOR:       ex:
+; ATTRIBUTOR-NEXT:    [[TMP5:%.*]] = tail call i32 @g(i32* nonnull [[A:%.*]])
+; ATTRIBUTOR-NEXT:    ret i32 [[TMP5]]
+; ATTRIBUTOR:       hd:
+; ATTRIBUTOR-NEXT:    [[TMP7:%.*]] = phi i32 [ [[TMP8:%.*]], [[HD2:%.*]] ], [ 0, [[EN:%.*]] ]
+; ATTRIBUTOR-NEXT:    tail call void @h(i32* nonnull [[A]])
+; ATTRIBUTOR-NEXT:    br label [[HD2]]
+; ATTRIBUTOR:       hd2:
+; ATTRIBUTOR-NEXT:    [[TMP8]] = add nuw i32 [[TMP7]], 1
+; ATTRIBUTOR-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[TMP8]], [[B]]
+; ATTRIBUTOR-NEXT:    br i1 [[TMP9]], label [[EX]], label [[HD]]
+;
+en:
+  %tmp3 = icmp eq i32 %b, 0
+  br i1 %tmp3, label %ex, label %hd
+
+ex:
+  %tmp5 = tail call i32 @g(i32* nonnull %a)
+  ret i32 %tmp5
+
+hd:
+  %tmp7 = phi i32 [ %tmp8, %hd2 ], [ 0, %en ]
+  tail call void @h(i32* %a)
+  br label %hd2
+
+hd2:
+  %tmp8 = add nuw i32 %tmp7, 1
+  %tmp9 = icmp eq i32 %tmp8, %b
+  br i1 %tmp9, label %ex, label %hd
+}
+
+; Original from PR43833
+declare void @sink(i32*)
+
+; FIXME: the sink argument should be marked nonnull as in @PR43833_simple.
+define void @PR43833(i32* %0, i32 %1) {
+; ATTRIBUTOR-LABEL: @PR43833(
+; ATTRIBUTOR-NEXT:    [[TMP3:%.*]] = icmp sgt i32 [[TMP1:%.*]], 1
+; ATTRIBUTOR-NEXT:    br i1 [[TMP3]], label [[TMP4:%.*]], label [[TMP7:%.*]]
+; ATTRIBUTOR:       4:
+; ATTRIBUTOR-NEXT:    [[TMP5:%.*]] = zext i32 [[TMP1]] to i64
+; ATTRIBUTOR-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[TMP0:%.*]], i64 [[TMP5]]
+; ATTRIBUTOR-NEXT:    br label [[TMP8:%.*]]
+; ATTRIBUTOR:       7:
+; ATTRIBUTOR-NEXT:    ret void
+; ATTRIBUTOR:       8:
+; ATTRIBUTOR-NEXT:    [[TMP9:%.*]] = phi i32 [ 1, [[TMP4]] ], [ [[TMP10:%.*]], [[TMP8]] ]
+; ATTRIBUTOR-NEXT:    tail call void @sink(i32* [[TMP6]])
+; ATTRIBUTOR-NEXT:    [[TMP10]] = add nuw nsw i32 [[TMP9]], 1
+; ATTRIBUTOR-NEXT:    [[TMP11:%.*]] = icmp eq i32 [[TMP10]], [[TMP1]]
+; ATTRIBUTOR-NEXT:    br i1 [[TMP11]], label [[TMP7]], label [[TMP8]]
+;
+  %3 = icmp sgt i32 %1, 1
+  br i1 %3, label %4, label %7
+
+4:                                                ; preds = %2
+  %5 = zext i32 %1 to i64
+  %6 = getelementptr inbounds i32, i32* %0, i64 %5
+  br label %8
+
+7:                                                ; preds = %8, %2
+  ret void
+
+8:                                                ; preds = %8, %4
+  %9 = phi i32 [ 1, %4 ], [ %10, %8 ]
+  tail call void @sink(i32* %6)
+  %10 = add nuw nsw i32 %9, 1
+  %11 = icmp eq i32 %10, %1
+  br i1 %11, label %7, label %8
+}
+
+; Adjusted from PR43833
+define void @PR43833_simple(i32* %0, i32 %1) {
+; ATTRIBUTOR_OPM-LABEL: @PR43833_simple(
+; ATTRIBUTOR_OPM-NEXT:    [[TMP3:%.*]] = icmp ne i32 [[TMP1:%.*]], 0
+; ATTRIBUTOR_OPM-NEXT:    br i1 [[TMP3]], label [[TMP4:%.*]], label [[TMP7:%.*]]
+; ATTRIBUTOR_OPM:       4:
+; ATTRIBUTOR_OPM-NEXT:    [[TMP5:%.*]] = zext i32 [[TMP1]] to i64
+; ATTRIBUTOR_OPM-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[TMP0:%.*]], i64 [[TMP5]]
+; ATTRIBUTOR_OPM-NEXT:    br label [[TMP8:%.*]]
+; ATTRIBUTOR_OPM:       7:
+; ATTRIBUTOR_OPM-NEXT:    ret void
+; ATTRIBUTOR_OPM:       8:
+; ATTRIBUTOR_OPM-NEXT:    [[TMP9:%.*]] = phi i32 [ 1, [[TMP4]] ], [ [[TMP10:%.*]], [[TMP8]] ]
+; ATTRIBUTOR_OPM-NEXT:    tail call void @sink(i32* [[TMP6]])
+; ATTRIBUTOR_OPM-NEXT:    [[TMP10]] = add nuw nsw i32 [[TMP9]], 1
+; ATTRIBUTOR_OPM-NEXT:    [[TMP11:%.*]] = icmp eq i32 [[TMP10]], [[TMP1]]
+; ATTRIBUTOR_OPM-NEXT:    br i1 [[TMP11]], label [[TMP7]], label [[TMP8]]
+;
+; ATTRIBUTOR_NPM-LABEL: @PR43833_simple(
+; ATTRIBUTOR_NPM-NEXT:    [[TMP3:%.*]] = icmp ne i32 [[TMP1:%.*]], 0
+; ATTRIBUTOR_NPM-NEXT:    br i1 [[TMP3]], label [[TMP4:%.*]], label [[TMP7:%.*]]
+; ATTRIBUTOR_NPM:       4:
+; ATTRIBUTOR_NPM-NEXT:    [[TMP5:%.*]] = zext i32 [[TMP1]] to i64
+; ATTRIBUTOR_NPM-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[TMP0:%.*]], i64 [[TMP5]]
+; ATTRIBUTOR_NPM-NEXT:    br label [[TMP8:%.*]]
+; ATTRIBUTOR_NPM:       7:
+; ATTRIBUTOR_NPM-NEXT:    ret void
+; ATTRIBUTOR_NPM:       8:
+; ATTRIBUTOR_NPM-NEXT:    [[TMP9:%.*]] = phi i32 [ 1, [[TMP4]] ], [ [[TMP10:%.*]], [[TMP8]] ]
+; ATTRIBUTOR_NPM-NEXT:    tail call void @sink(i32* nonnull [[TMP6]])
+; ATTRIBUTOR_NPM-NEXT:    [[TMP10]] = add nuw nsw i32 [[TMP9]], 1
+; ATTRIBUTOR_NPM-NEXT:    [[TMP11:%.*]] = icmp eq i32 [[TMP10]], [[TMP1]]
+; ATTRIBUTOR_NPM-NEXT:    br i1 [[TMP11]], label [[TMP7]], label [[TMP8]]
+;
+  %3 = icmp ne i32 %1, 0
+  br i1 %3, label %4, label %7
+
+4:                                                ; preds = %2
+  %5 = zext i32 %1 to i64
+  %6 = getelementptr inbounds i32, i32* %0, i64 %5
+  br label %8
+
+7:                                                ; preds = %8, %2
+  ret void
+
+8:                                                ; preds = %8, %4
+  %9 = phi i32 [ 1, %4 ], [ %10, %8 ]
+  tail call void @sink(i32* %6)
+  %10 = add nuw nsw i32 %9, 1
+  %11 = icmp eq i32 %10, %1
+  br i1 %11, label %7, label %8
+}
+
+declare i8* @strrchr(i8* %0, i32 %1) nofree nounwind readonly
+
+; We should not mark the return of @strrchr as `nonnull`, it may well be NULL!
+define i8* @mybasename(i8* nofree readonly %str) {
+; ATTRIBUTOR-LABEL: define {{[^@]+}}@mybasename
+; ATTRIBUTOR-SAME: (i8* nofree readonly [[STR:%.*]])
+; ATTRIBUTOR-NEXT:    [[CALL:%.*]] = call i8* @strrchr(i8* nofree readonly [[STR]], i32 47)
+; ATTRIBUTOR-NEXT:    [[TOBOOL:%.*]] = icmp ne i8* [[CALL]], null
+; ATTRIBUTOR-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds i8, i8* [[CALL]], i64 1
+; ATTRIBUTOR-NEXT:    [[COND:%.*]] = select i1 [[TOBOOL]], i8* [[ADD_PTR]], i8* [[STR]]
+; ATTRIBUTOR-NEXT:    ret i8* [[COND]]
+;
+  %call = call i8* @strrchr(i8* %str, i32 47)
+  %tobool = icmp ne i8* %call, null
+  %add.ptr = getelementptr inbounds i8, i8* %call, i64 1
+  %cond = select i1 %tobool, i8* %add.ptr, i8* %str
+  ret i8* %cond
+}
+
+attributes #0 = { "null-pointer-is-valid"="true" }
+attributes #1 = { nounwind willreturn}
diff --git a/llvm/test/Transforms/Attributor/norecurse.ll b/llvm/test/Transforms/Attributor/norecurse.ll
new file mode 100644
index 000000000..25f7fdee5
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/norecurse.ll
@@ -0,0 +1,147 @@
+; RUN: opt -passes=attributor --attributor-disable=false -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=2 -S < %s | FileCheck %s --check-prefixes=ATTRIBUTOR
+; Copied from Transforms/FunctoinAttrs/norecurse.ll
+
+; ATTRIBUTOR: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; ATTRIBUTOR-NEXT: define i32 @leaf()
+define i32 @leaf() {
+  ret i32 1
+}
+
+; ATTRIBUTOR: Function Attrs
+; ATTRIBUTOR-SAME: readnone
+; ATTRIBUTOR-NOT: norecurse
+; ATTRIBUTOR-NEXT: define i32 @self_rec()
+define i32 @self_rec() {
+  %a = call i32 @self_rec()
+  ret i32 4
+}
+
+; ATTRIBUTOR: Function Attrs
+; ATTRIBUTOR-SAME: readnone
+; ATTRIBUTOR-NOT: norecurse
+; ATTRIBUTOR-NEXT: define i32 @indirect_rec()
+define i32 @indirect_rec() {
+  %a = call i32 @indirect_rec2()
+  ret i32 %a
+}
+; ATTRIBUTOR: Function Attrs
+; ATTRIBUTOR-SAME: readnone
+; ATTRIBUTOR-NOT: norecurse
+; ATTRIBUTOR-NEXT: define i32 @indirect_rec2()
+define i32 @indirect_rec2() {
+  %a = call i32 @indirect_rec()
+  ret i32 %a
+}
+
+; ATTRIBUTOR: Function Attrs
+; ATTRIBUTOR-SAME: readnone
+; ATTRIBUTOR-NOT: norecurse
+; ATTRIBUTOR-NEXT: define i32 @extern()
+define i32 @extern() {
+  %a = call i32 @k()
+  ret i32 %a
+}
+
+; ATTRIBUTOR: Function Attrs
+; ATTRIBUTOR-NEXT: declare i32 @k()
+declare i32 @k() readnone
+
+; ATTRIBUTOR: Function Attrs
+; ATTRIBUTOR-NOT: norecurse
+; ATTRIBUTOR-NEXT: define void @intrinsic(i8* nocapture writeonly %dest, i8* nocapture readonly %src, i32 %len)
+define void @intrinsic(i8* %dest, i8* %src, i32 %len) {
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %dest, i8* %src, i32 %len, i1 false)
+  ret void
+}
+
+; ATTRIBUTOR: Function Attrs
+; ATTRIBUTOR-NEXT: declare void @llvm.memcpy.p0i8.p0i8.i32
+declare void @llvm.memcpy.p0i8.p0i8.i32(i8*, i8*, i32, i1)
+
+; ATTRIBUTOR: Function Attrs
+; FIXME: missing "norecurse"
+; ATTRIBUTOR-SAME: nosync readnone
+define internal i32 @called_by_norecurse() {
+  %a = call i32 @k()
+  ret i32 %a
+}
+; ATTRIBUTOR: Function Attrs
+; ATTRIBUTOR-NEXT: define void @m()
+define void @m() norecurse {
+  %a = call i32 @called_by_norecurse()
+  ret void
+}
+
+; ATTRIBUTOR: Function Attrs
+; FIXME: missing "norecurse"
+; ATTRIBUTOR-SAME: nosync
+define internal i32 @called_by_norecurse_indirectly() {
+  %a = call i32 @k()
+  ret i32 %a
+}
+define internal void @o() {
+  %a = call i32 @called_by_norecurse_indirectly()
+  ret void
+}
+define void @p() norecurse {
+  call void @o()
+  ret void
+}
+
+; ATTRIBUTOR: Function Attrs: nofree nosync nounwind
+; ATTRIBUTOR-NEXT: define void @f(i32 %x)
+define void @f(i32 %x)  {
+entry:
+  %x.addr = alloca i32, align 4
+  store i32 %x, i32* %x.addr, align 4
+  %0 = load i32, i32* %x.addr, align 4
+  %tobool = icmp ne i32 %0, 0
+  br i1 %tobool, label %if.then, label %if.end
+
+if.then:
+  call void @g() norecurse
+  br label %if.end
+
+if.end:
+  ret void
+}
+
+; ATTRIBUTOR: define void @g()
+define void @g() norecurse {
+entry:
+  call void @f(i32 0)
+  ret void
+}
+
+; ATTRIBUTOR-NOT: Function Attrs
+; ATTRIBUTOR: define linkonce_odr i32 @leaf_redefinable()
+define linkonce_odr i32 @leaf_redefinable() {
+  ret i32 1
+}
+
+; Call through a function pointer
+; ATTRIBUTOR-NOT: Function Attrs
+; ATTRIBUTOR: define i32 @eval_func1(i32 (i32)* nocapture nofree nonnull %0, i32 %1)
+define i32 @eval_func1(i32 (i32)* , i32) local_unnamed_addr {
+  %3 = tail call i32 %0(i32 %1) #2
+  ret i32 %3
+}
+
+; ATTRIBUTOR-NOT: Function Attrs
+; ATTRIBUTOR: define i32 @eval_func2(i32 (i32)* nocapture nofree %0, i32 %1)
+define i32 @eval_func2(i32 (i32)* , i32) local_unnamed_addr "null-pointer-is-valid"="true"{
+  %3 = tail call i32 %0(i32 %1) #2
+  ret i32 %3
+}
+
+declare void @unknown()
+; Call an unknown function in a dead block.
+; ATTRIBUTOR: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; ATTRIBUTOR: define i32 @call_unknown_in_dead_block()
+define i32 @call_unknown_in_dead_block() local_unnamed_addr {
+  ret i32 0
+Dead:
+  tail call void @unknown()
+  ret i32 1
+}
+
diff --git a/llvm/test/Transforms/Attributor/noreturn.ll b/llvm/test/Transforms/Attributor/noreturn.ll
new file mode 100644
index 000000000..2b15e0780
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/noreturn.ll
@@ -0,0 +1,149 @@
+; RUN: opt -functionattrs -attributor -attributor-disable=false -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=2 -S < %s | FileCheck %s
+;
+; Test cases specifically designed for the "no-return" function attribute.
+; We use FIXME's to indicate problems and missing attributes.
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+
+; TEST 1, singleton SCC void return type
+;
+; void srec0() {
+;   return srec0();
+; }
+;
+; CHECK: Function Attrs: nofree noinline noreturn nosync nounwind readnone uwtable
+; CHECK: define void @srec0()
+;
+define void @srec0() #0 {
+entry:
+  call void @srec0()
+  ret void
+}
+
+
+; TEST 2: singleton SCC int return type with a lot of recursive calls
+;
+; int srec16(int a) {
+;   return srec16(srec16(srec16(srec16(srec16(srec16(srec16(srec16(srec16(srec16(srec16(srec16(srec16(srec16(srec16(srec16(a))))))))))))))));
+; }
+;
+; CHECK: Function Attrs: nofree noinline noreturn nosync nounwind readnone uwtable
+; CHECK: define i32 @srec16(i32 %a)
+;
+define i32 @srec16(i32 %a) #0 {
+entry:
+  %call = call i32 @srec16(i32 %a)
+  %call1 = call i32 @srec16(i32 %call)
+  %call2 = call i32 @srec16(i32 %call1)
+  %call3 = call i32 @srec16(i32 %call2)
+  %call4 = call i32 @srec16(i32 %call3)
+  %call5 = call i32 @srec16(i32 %call4)
+  %call6 = call i32 @srec16(i32 %call5)
+  %call7 = call i32 @srec16(i32 %call6)
+  %call8 = call i32 @srec16(i32 %call7)
+  %call9 = call i32 @srec16(i32 %call8)
+  %call10 = call i32 @srec16(i32 %call9)
+  %call11 = call i32 @srec16(i32 %call10)
+  %call12 = call i32 @srec16(i32 %call11)
+  %call13 = call i32 @srec16(i32 %call12)
+  %call14 = call i32 @srec16(i32 %call13)
+  %call15 = call i32 @srec16(i32 %call14)
+  br label %exit
+
+exit:
+  ret i32 %call15
+}
+
+
+; TEST 3: endless loop, no return instruction
+;
+; int endless_loop(int a) {
+;   while (1);
+; }
+;
+; CHECK: Function Attrs: nofree noinline norecurse noreturn nosync nounwind readnone uwtable
+; CHECK: define i32 @endless_loop(i32 %a)
+;
+define i32 @endless_loop(i32 %a) #0 {
+entry:
+  br label %while.body
+
+while.body:                                       ; preds = %entry, %while.body
+  br label %while.body
+}
+
+
+; TEST 4: endless loop, dead return instruction
+;
+; int endless_loop(int a) {
+;   while (1);
+;   return a;
+; }
+;
+; FIXME: no-return missing (D65243 should fix this)
+; CHECK: Function Attrs: nofree noinline norecurse noreturn nosync nounwind readnone uwtable
+; CHECK: define i32 @dead_return(i32 returned %a)
+;
+define i32 @dead_return(i32 %a) #0 {
+entry:
+  br label %while.body
+
+while.body:                                       ; preds = %entry, %while.body
+  br label %while.body
+
+return:                                           ; No predecessors!
+  ret i32 %a
+}
+
+
+; TEST 5: all paths contain a no-return function call
+;
+; int multiple_noreturn_calls(int a) {
+;   return a == 0 ? endless_loop(a) : srec16(a);
+; }
+;
+; CHECK: Function Attrs: nofree noinline noreturn nosync nounwind readnone uwtable
+; CHECK: define i32 @multiple_noreturn_calls(i32 %a)
+;
+define i32 @multiple_noreturn_calls(i32 %a) #0 {
+entry:
+  %cmp = icmp eq i32 %a, 0
+  br i1 %cmp, label %cond.true, label %cond.false
+
+cond.true:                                        ; preds = %entry
+  %call = call i32 @endless_loop(i32 %a)
+  br label %cond.end
+
+cond.false:                                       ; preds = %entry
+  %call1 = call i32 @srec16(i32 %a)
+  br label %cond.end
+
+cond.end:                                         ; preds = %cond.false, %cond.true
+  %cond = phi i32 [ %call, %cond.true ], [ %call1, %cond.false ]
+  ret i32 %cond
+}
+
+
+; TEST 6a: willreturn means *not* no-return or UB
+; FIXME: we should derive "UB" as an argument and report it to the user on request.
+
+; CHECK:      Function Attrs: nofree norecurse noreturn nosync nounwind readnone willreturn
+; CHECK-NEXT: define i32 @endless_loop_but_willreturn
+define i32 @endless_loop_but_willreturn() willreturn {
+entry:
+  br label %while.body
+
+while.body:                                       ; preds = %entry, %while.body
+  br label %while.body
+}
+
+; TEST 6b: willreturn means *not* no-return or UB
+; CHECK:      Function Attrs: nofree norecurse noreturn nosync nounwind readnone willreturn
+; CHECK-NEXT: define i32 @UB_and_willreturn
+define i32 @UB_and_willreturn() willreturn {
+entry:
+  unreachable
+}
+
+attributes #0 = { noinline nounwind uwtable }
diff --git a/llvm/test/Transforms/Attributor/noreturn_async.ll b/llvm/test/Transforms/Attributor/noreturn_async.ll
new file mode 100644
index 000000000..9fb99159a
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/noreturn_async.ll
@@ -0,0 +1,142 @@
+; RUN: opt -attributor -attributor-disable=false -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=2 -S < %s | FileCheck %s
+;
+; This file is the same as noreturn_sync.ll but with a personality which
+; indicates that the exception handler *can* catch asynchronous exceptions. As
+; a consequence, invokes to noreturn and nounwind functions are not translated
+; to calls followed by an unreachable but the unwind edge is considered live.
+;
+; https://reviews.llvm.org/D59978#inline-586873
+;
+; Make sure we handle invoke of a noreturn function correctly.
+;
+; This test is also a reminder of how we handle (=ignore) stackoverflow exception handling.
+;
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc19.16.27032"
+
+@"??_C@_0BG@CMNEKHOP@Exception?5NOT?5caught?6?$AA@" = linkonce_odr dso_local unnamed_addr constant [22 x i8] c"Exception NOT caught\0A\00", align 1
+@"??_C@_0BC@NKPAGFFJ@Exception?5caught?6?$AA@" = linkonce_odr dso_local unnamed_addr constant [18 x i8] c"Exception caught\0A\00", align 1
+@"??_C@_0BK@JHJLGDKL@Done?5execution?5result?$DN?$CFi?6?$AA@" = linkonce_odr dso_local unnamed_addr constant [26 x i8] c"Done execution result=%i\0A\00", align 1
+@"?_OptionsStorage@?1??__local_stdio_printf_options@@9@4_KA" = linkonce_odr dso_local global i64 0, align 8
+
+
+define dso_local void @"?overflow@@YAXXZ"() {
+entry:
+; CHECK:      Function Attrs: nofree noreturn nosync nounwind
+; CHECK-NEXT: define
+; CHECK-NEXT:   entry:
+; CHECK-NEXT:   call void @"?overflow@@YAXXZ"()
+; CHECK-NEXT:   unreachable
+  call void @"?overflow@@YAXXZ"()
+  %call3 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([18 x i8], [18 x i8]* @"??_C@_0BC@NKPAGFFJ@Exception?5caught?6?$AA@", i64 0, i64 0))
+  ret void
+}
+
+
+; CHECK-NOT:    nounwind
+; CHECK-NOT:    noreturn
+; CHECK:        define
+; CHECK-SAME:   @"?catchoverflow@@YAHXZ"()
+define dso_local i32 @"?catchoverflow@@YAHXZ"()  personality i8* bitcast (i32 (...)* @__C_specific_handler to i8*) {
+entry:
+  %retval = alloca i32, align 4
+  %__exception_code = alloca i32, align 4
+; CHECK: invoke void @"?overflow@@YAXXZ"()
+; CHECK:          to label %invoke.cont unwind label %catch.dispatch
+  invoke void @"?overflow@@YAXXZ"()
+          to label %invoke.cont unwind label %catch.dispatch
+
+invoke.cont:                                      ; preds = %entry
+; CHECK:      invoke.cont:
+; CHECK-NEXT: unreachable
+  br label %invoke.cont1
+
+catch.dispatch:                                   ; preds = %invoke.cont, %entry
+  %0 = catchswitch within none [label %__except] unwind to caller
+
+__except:                                         ; preds = %catch.dispatch
+  %1 = catchpad within %0 [i8* null]
+  catchret from %1 to label %__except2
+
+__except2:                                        ; preds = %__except
+  %2 = call i32 @llvm.eh.exceptioncode(token %1)
+  store i32 1, i32* %retval, align 4
+  br label %return
+
+invoke.cont1:                                     ; preds = %invoke.cont
+  store i32 0, i32* %retval, align 4
+  br label %return
+
+__try.cont:                                       ; No predecessors!
+  store i32 2, i32* %retval, align 4
+  br label %return
+
+return:                                           ; preds = %__try.cont, %__except2, %invoke.cont1
+  %3 = load i32, i32* %retval, align 4
+  ret i32 %3
+}
+
+
+define dso_local void @"?overflow@@YAXXZ_may_throw"()  {
+entry:
+; CHECK:      Function Attrs: noreturn
+; CHECK-NOT:  nounwind
+; CHECK-NEXT: define
+; CHECK-NEXT:   entry:
+; CHECK-NEXT:   %call3 = call i32 (i8*, ...) @printf(i8* nonnull dereferenceable(18) getelementptr inbounds ([18 x i8], [18 x i8]* @"??_C@_0BC@NKPAGFFJ@Exception?5caught?6?$AA@", i64 0, i64 0))
+; CHECK-NEXT:   call void @"?overflow@@YAXXZ_may_throw"()
+; CHECK-NEXT:   unreachable
+  %call3 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([18 x i8], [18 x i8]* @"??_C@_0BC@NKPAGFFJ@Exception?5caught?6?$AA@", i64 0, i64 0))
+  call void @"?overflow@@YAXXZ_may_throw"()
+  ret void
+}
+
+
+; CHECK-NOT:    nounwind
+; CHECK-NOT:    noreturn
+; CHECK:        define
+; CHECK-SAME:   @"?catchoverflow@@YAHXZ_may_throw"()
+define dso_local i32 @"?catchoverflow@@YAHXZ_may_throw"()  personality i8* bitcast (i32 (...)* @__C_specific_handler to i8*) {
+entry:
+  %retval = alloca i32, align 4
+  %__exception_code = alloca i32, align 4
+; CHECK: invoke void @"?overflow@@YAXXZ_may_throw"()
+; CHECK:          to label %invoke.cont unwind label %catch.dispatch
+  invoke void @"?overflow@@YAXXZ_may_throw"()
+          to label %invoke.cont unwind label %catch.dispatch
+
+invoke.cont:                                      ; preds = %entry
+; CHECK:      invoke.cont:
+; CHECK-NEXT: unreachable
+  br label %invoke.cont1
+
+catch.dispatch:                                   ; preds = %invoke.cont, %entry
+  %0 = catchswitch within none [label %__except] unwind to caller
+
+__except:                                         ; preds = %catch.dispatch
+  %1 = catchpad within %0 [i8* null]
+  catchret from %1 to label %__except2
+
+__except2:                                        ; preds = %__except
+  %2 = call i32 @llvm.eh.exceptioncode(token %1)
+  store i32 1, i32* %retval, align 4
+  br label %return
+
+invoke.cont1:                                     ; preds = %invoke.cont
+  store i32 0, i32* %retval, align 4
+  br label %return
+
+__try.cont:                                       ; No predecessors!
+  store i32 2, i32* %retval, align 4
+  br label %return
+
+return:                                           ; preds = %__try.cont, %__except2, %invoke.cont1
+  %3 = load i32, i32* %retval, align 4
+  ret i32 %3
+}
+
+declare dso_local i32 @__C_specific_handler(...)
+
+declare dso_local i32 @printf(i8* %_Format, ...)
+
+declare i32 @llvm.eh.exceptioncode(token)
diff --git a/llvm/test/Transforms/Attributor/noreturn_sync.ll b/llvm/test/Transforms/Attributor/noreturn_sync.ll
new file mode 100644
index 000000000..4e6f13737
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/noreturn_sync.ll
@@ -0,0 +1,138 @@
+; RUN: opt -attributor -attributor-disable=false -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=2 -S < %s | FileCheck %s
+;
+; This file is the same as noreturn_async.ll but with a personality which
+; indicates that the exception handler *cannot* catch asynchronous exceptions.
+; As a consequence, invokes to noreturn and nounwind functions are translated
+; to calls followed by an unreachable.
+;
+; https://reviews.llvm.org/D59978#inline-586873
+;
+; Make sure we handle invoke of a noreturn function correctly.
+;
+; This test is also a reminder of how we handle (=ignore) stackoverflow exception handling.
+;
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-linux-gnu"
+
+@"??_C@_0BG@CMNEKHOP@Exception?5NOT?5caught?6?$AA@" = linkonce_odr dso_local unnamed_addr constant [22 x i8] c"Exception NOT caught\0A\00", align 1
+@"??_C@_0BC@NKPAGFFJ@Exception?5caught?6?$AA@" = linkonce_odr dso_local unnamed_addr constant [18 x i8] c"Exception caught\0A\00", align 1
+@"??_C@_0BK@JHJLGDKL@Done?5execution?5result?$DN?$CFi?6?$AA@" = linkonce_odr dso_local unnamed_addr constant [26 x i8] c"Done execution result=%i\0A\00", align 1
+@"?_OptionsStorage@?1??__local_stdio_printf_options@@9@4_KA" = linkonce_odr dso_local global i64 0, align 8
+
+
+define dso_local void @"?overflow@@YAXXZ"() {
+entry:
+; CHECK:      Function Attrs: nofree noreturn nosync nounwind
+; CHECK-NEXT: define
+; CHECK-NEXT:   entry:
+; CHECK-NEXT:   call void @"?overflow@@YAXXZ"()
+; CHECK-NEXT:   unreachable
+  call void @"?overflow@@YAXXZ"()
+  %call3 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([18 x i8], [18 x i8]* @"??_C@_0BC@NKPAGFFJ@Exception?5caught?6?$AA@", i64 0, i64 0))
+  ret void
+}
+
+
+; CHECK:       Function Attrs: nofree noreturn nosync nounwind
+; CHECK-NEXT:   @"?catchoverflow@@YAHXZ"()
+define dso_local i32 @"?catchoverflow@@YAHXZ"()  personality i8* bitcast (i32 (...)* @__gcc_personality_v0 to i8*) {
+entry:
+  %retval = alloca i32, align 4
+  %__exception_code = alloca i32, align 4
+  invoke void @"?overflow@@YAXXZ"() 
+          to label %invoke.cont unwind label %catch.dispatch
+; CHECK:      call void @"?overflow@@YAXXZ"()
+; CHECK-NEXT: unreachable
+
+invoke.cont:                                      ; preds = %entry
+  br label %invoke.cont1
+
+catch.dispatch:                                   ; preds = %invoke.cont, %entry
+  %0 = catchswitch within none [label %__except] unwind to caller
+
+__except:                                         ; preds = %catch.dispatch
+  %1 = catchpad within %0 [i8* null]
+  catchret from %1 to label %__except2
+
+__except2:                                        ; preds = %__except
+  %2 = call i32 @llvm.eh.exceptioncode(token %1)
+  store i32 1, i32* %retval, align 4
+  br label %return
+
+invoke.cont1:                                     ; preds = %invoke.cont
+  store i32 0, i32* %retval, align 4
+  br label %return
+
+__try.cont:                                       ; No predecessors!
+  store i32 2, i32* %retval, align 4
+  br label %return
+
+return:                                           ; preds = %__try.cont, %__except2, %invoke.cont1
+  %3 = load i32, i32* %retval, align 4
+  ret i32 %3
+}
+
+
+define dso_local void @"?overflow@@YAXXZ_may_throw"()  {
+entry:
+; CHECK:      Function Attrs: noreturn
+; CHECK-NOT:  nounwind
+; CHECK-NEXT: define
+; CHECK-NEXT:   entry:
+; CHECK-NEXT:   %call3 = call i32 (i8*, ...) @printf(i8* nonnull dereferenceable(18) getelementptr inbounds ([18 x i8], [18 x i8]* @"??_C@_0BC@NKPAGFFJ@Exception?5caught?6?$AA@", i64 0, i64 0))
+; CHECK-NEXT:   call void @"?overflow@@YAXXZ_may_throw"()
+; CHECK-NEXT:   unreachable
+  %call3 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([18 x i8], [18 x i8]* @"??_C@_0BC@NKPAGFFJ@Exception?5caught?6?$AA@", i64 0, i64 0))
+  call void @"?overflow@@YAXXZ_may_throw"()
+  ret void
+}
+
+
+; CHECK-NOT:    nounwind
+; CHECK-NOT:    noreturn
+; CHECK:        define
+; CHECK-SAME:   @"?catchoverflow@@YAHXZ_may_throw"()
+define dso_local i32 @"?catchoverflow@@YAHXZ_may_throw"()  personality i8* bitcast (i32 (...)* @__gcc_personality_v0 to i8*) {
+entry:
+  %retval = alloca i32, align 4
+  %__exception_code = alloca i32, align 4
+; CHECK: invoke void @"?overflow@@YAXXZ_may_throw"() 
+; CHECK:          to label %invoke.cont unwind label %catch.dispatch
+  invoke void @"?overflow@@YAXXZ_may_throw"() 
+          to label %invoke.cont unwind label %catch.dispatch
+
+invoke.cont:                                      ; preds = %entry
+; CHECK:      invoke.cont:
+; CHECK-NEXT: unreachable
+  br label %invoke.cont1
+
+catch.dispatch:                                   ; preds = %invoke.cont, %entry
+  %0 = catchswitch within none [label %__except] unwind to caller
+
+__except:                                         ; preds = %catch.dispatch
+  %1 = catchpad within %0 [i8* null]
+  catchret from %1 to label %__except2
+
+__except2:                                        ; preds = %__except
+  %2 = call i32 @llvm.eh.exceptioncode(token %1)
+  store i32 1, i32* %retval, align 4
+  br label %return
+
+invoke.cont1:                                     ; preds = %invoke.cont
+  store i32 0, i32* %retval, align 4
+  br label %return
+
+__try.cont:                                       ; No predecessors!
+  store i32 2, i32* %retval, align 4
+  br label %return
+
+return:                                           ; preds = %__try.cont, %__except2, %invoke.cont1
+  %3 = load i32, i32* %retval, align 4
+  ret i32 %3
+}
+
+declare dso_local i32 @__gcc_personality_v0(...)
+
+declare dso_local i32 @printf(i8* %_Format, ...)
+
+declare i32 @llvm.eh.exceptioncode(token)
diff --git a/llvm/test/Transforms/Attributor/nosync.ll b/llvm/test/Transforms/Attributor/nosync.ll
new file mode 100644
index 000000000..67125fce4
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/nosync.ll
@@ -0,0 +1,319 @@
+; RUN: opt -attributor -attributor-manifest-internal -attributor-disable=false -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=2 -S < %s | FileCheck %s --check-prefix=ATTRIBUTOR
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+; Test cases designed for the nosync function attribute.
+; FIXME's are used to indicate problems and missing attributes.
+
+; struct RT {
+;   char A;
+;   int B[10][20];
+;   char C;
+; };
+; struct ST {
+;   int X;
+;   double Y;
+;   struct RT Z;
+; };
+;
+; int *foo(struct ST *s) {
+;   return &s[1].Z.B[5][13];
+; }
+
+; TEST 1
+; non-convergent and readnone implies nosync
+%struct.RT = type { i8, [10 x [20 x i32]], i8 }
+%struct.ST = type { i32, double, %struct.RT }
+
+; ATTRIBUTOR: Function Attrs: nofree nosync nounwind optsize readnone ssp uwtable
+; ATTRIBUTOR-NEXT: define nonnull i32* @foo(%struct.ST* nofree readnone "no-capture-maybe-returned" %s)
+define i32* @foo(%struct.ST* %s) nounwind uwtable readnone optsize ssp {
+entry:
+  %arrayidx = getelementptr inbounds %struct.ST, %struct.ST* %s, i64 1, i32 2, i32 1, i64 5, i64 13
+  ret i32* %arrayidx
+}
+
+; TEST 2
+; atomic load with monotonic ordering
+; int load_monotonic(_Atomic int *num) {
+;   int n = atomic_load_explicit(num, memory_order_relaxed);
+;   return n;
+; }
+
+; ATTRIBUTOR: Function Attrs: nofree norecurse nosync nounwind uwtable
+; ATTRIBUTOR-NEXT: define i32 @load_monotonic(i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) %0)
+define i32 @load_monotonic(i32* nocapture readonly %0) norecurse nounwind uwtable {
+  %2 = load atomic i32, i32* %0 monotonic, align 4
+  ret i32 %2
+}
+
+
+; TEST 3
+; atomic store with monotonic ordering.
+; void store_monotonic(_Atomic int *num) {
+;   atomic_load_explicit(num, memory_order_relaxed);
+; }
+
+; ATTRIBUTOR: Function Attrs: nofree norecurse nosync nounwind uwtable
+; ATTRIBUTOR-NEXT: define void @store_monotonic(i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) %0)
+define void @store_monotonic(i32* nocapture %0) norecurse nounwind uwtable {
+  store atomic i32 10, i32* %0 monotonic, align 4
+  ret void
+}
+
+; TEST 4 - negative, should not deduce nosync
+; atomic load with acquire ordering.
+; int load_acquire(_Atomic int *num) {
+;   int n = atomic_load_explicit(num, memory_order_acquire);
+;   return n;
+; }
+
+; ATTRIBUTOR: Function Attrs: nofree norecurse nounwind uwtable
+; ATTRIBUTOR-NOT: nosync
+; ATTRIBUTOR-NEXT: define i32 @load_acquire(i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) %0)
+define i32 @load_acquire(i32* nocapture readonly %0) norecurse nounwind uwtable {
+  %2 = load atomic i32, i32* %0 acquire, align 4
+  ret i32 %2
+}
+
+; TEST 5 - negative, should not deduce nosync
+; atomic load with release ordering
+; void load_release(_Atomic int *num) {
+;   atomic_store_explicit(num, 10, memory_order_release);
+; }
+
+; ATTRIBUTOR: Function Attrs: nofree norecurse nounwind uwtable
+; ATTRIBUTOR-NOT: nosync
+; ATTRIBUTOR-NEXT: define void @load_release(i32* nocapture nofree writeonly align 4 %0)
+define void @load_release(i32* nocapture %0) norecurse nounwind uwtable {
+  store atomic volatile i32 10, i32* %0 release, align 4
+  ret void
+}
+
+; TEST 6 - negative volatile, relaxed atomic
+
+; ATTRIBUTOR: Function Attrs: nofree norecurse nounwind uwtable
+; ATTRIBUTOR-NOT: nosync
+; ATTRIBUTOR-NEXT: define void @load_volatile_release(i32* nocapture nofree writeonly align 4 %0)
+define void @load_volatile_release(i32* nocapture %0) norecurse nounwind uwtable {
+  store atomic volatile i32 10, i32* %0 release, align 4
+  ret void
+}
+
+; TEST 7 - negative, should not deduce nosync
+; volatile store.
+; void volatile_store(volatile int *num) {
+;   *num = 14;
+; }
+
+; ATTRIBUTOR: Function Attrs: nofree norecurse nounwind uwtable
+; ATTRIBUTOR-NOT: nosync
+; ATTRIBUTOR-NEXT: define void @volatile_store(i32* nofree align 4 %0)
+define void @volatile_store(i32* %0) norecurse nounwind uwtable {
+  store volatile i32 14, i32* %0, align 4
+  ret void
+}
+
+; TEST 8 - negative, should not deduce nosync
+; volatile load.
+; int volatile_load(volatile int *num) {
+;   int n = *num;
+;   return n;
+; }
+
+; ATTRIBUTOR: Function Attrs: nofree norecurse nounwind uwtable
+; ATTRIBUTOR-NOT: nosync
+; ATTRIBUTOR-NEXT: define i32 @volatile_load(i32* nofree align 4 %0)
+define i32 @volatile_load(i32* %0) norecurse nounwind uwtable {
+  %2 = load volatile i32, i32* %0, align 4
+  ret i32 %2
+}
+
+; TEST 9
+
+; ATTRIBUTOR: Function Attrs: noinline nosync nounwind uwtable
+; ATTRIBUTOR-NEXT: declare void @nosync_function()
+declare void @nosync_function() noinline nounwind uwtable nosync
+
+; ATTRIBUTOR: Function Attrs: noinline nosync nounwind uwtable
+; ATTRIBUTOR-next: define void @call_nosync_function()
+define void @call_nosync_function() nounwind uwtable noinline {
+  tail call void @nosync_function() noinline nounwind uwtable
+  ret void
+}
+
+; TEST 10 - negative, should not deduce nosync
+
+; ATTRIBUTOR: Function Attrs: noinline nounwind uwtable
+; ATTRIBUTOR-NEXT: declare void @might_sync()
+declare void @might_sync() noinline nounwind uwtable
+
+; ATTRIBUTOR: Function Attrs: noinline nounwind uwtable
+; ATTRIBUTOR-NOT: nosync
+; ATTRIBUTOR-NEXT: define void @call_might_sync()
+define void @call_might_sync() nounwind uwtable noinline {
+  tail call void @might_sync() noinline nounwind uwtable
+  ret void
+}
+
+; TEST 11 - positive, should deduce nosync
+; volatile operation in same scc but dead. Call volatile_load defined in TEST 8.
+
+; ATTRIBUTOR: Function Attrs: nofree noinline noreturn nosync nounwind readnone uwtable
+; ATTRIBUTOR-NEXT: define i32 @scc1(i32* nocapture nofree readnone %0)
+define i32 @scc1(i32* %0) noinline nounwind uwtable {
+  tail call void @scc2(i32* %0);
+  %val = tail call i32 @volatile_load(i32* %0);
+  ret i32 %val;
+}
+
+; ATTRIBUTOR: Function Attrs: nofree noinline noreturn nosync nounwind readnone uwtable
+; ATTRIBUTOR-NEXT: define void @scc2(i32* nocapture nofree readnone %0)
+define void @scc2(i32* %0) noinline nounwind uwtable {
+  tail call i32 @scc1(i32* %0);
+  ret void;
+}
+
+; TEST 12 - fences, negative
+;
+; void foo1(int *a, std::atomic<bool> flag){
+;   *a = 100;
+;   atomic_thread_fence(std::memory_order_release);
+;   flag.store(true, std::memory_order_relaxed);
+; }
+;
+; void bar(int *a, std::atomic<bool> flag){
+;   while(!flag.load(std::memory_order_relaxed))
+;     ;
+;
+;   atomic_thread_fence(std::memory_order_acquire);
+;   int b = *a;
+; }
+
+%"struct.std::atomic" = type { %"struct.std::__atomic_base" }
+%"struct.std::__atomic_base" = type { i8 }
+
+; ATTRIBUTOR-NOT: nosync
+; ATTRIBUTOR: define void @foo1(i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) %0, %"struct.std::atomic"* nocapture nofree nonnull writeonly dereferenceable(1) %1)
+
+define void @foo1(i32* %0, %"struct.std::atomic"* %1) {
+  store i32 100, i32* %0, align 4
+  fence release
+  %3 = getelementptr inbounds %"struct.std::atomic", %"struct.std::atomic"* %1, i64 0, i32 0, i32 0
+  store atomic i8 1, i8* %3 monotonic, align 1
+  ret void
+}
+
+; ATTRIBUTOR-NOT: nosync
+; ATTRIBUTOR: define void @bar(i32* nocapture nofree readnone %0, %"struct.std::atomic"* nocapture nofree nonnull readonly dereferenceable(1) %1)
+define void @bar(i32* %0, %"struct.std::atomic"* %1) {
+  %3 = getelementptr inbounds %"struct.std::atomic", %"struct.std::atomic"* %1, i64 0, i32 0, i32 0
+  br label %4
+
+4:                                                ; preds = %4, %2
+  %5 = load atomic i8, i8* %3  monotonic, align 1
+  %6 = and i8 %5, 1
+  %7 = icmp eq i8 %6, 0
+  br i1 %7, label %4, label %8
+
+8:                                                ; preds = %4
+  fence acquire
+  ret void
+}
+
+; TEST 13 - Fence syncscope("singlethread") seq_cst
+; ATTRIBUTOR: Function Attrs: nofree nosync nounwind willreturn
+; ATTRIBUTOR: define void @foo1_singlethread(i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) %0, %"struct.std::atomic"* nocapture nofree nonnull writeonly dereferenceable(1) %1)
+
+define void @foo1_singlethread(i32* %0, %"struct.std::atomic"* %1) {
+  store i32 100, i32* %0, align 4
+  fence syncscope("singlethread") release
+  %3 = getelementptr inbounds %"struct.std::atomic", %"struct.std::atomic"* %1, i64 0, i32 0, i32 0
+  store atomic i8 1, i8* %3 monotonic, align 1
+  ret void
+}
+
+; ATTRIBUTOR: Function Attrs: nofree nosync nounwind
+; ATTRIBUTOR: define void @bar_singlethread(i32* nocapture nofree readnone %0, %"struct.std::atomic"* nocapture nofree nonnull readonly dereferenceable(1) %1)
+define void @bar_singlethread(i32* %0, %"struct.std::atomic"* %1) {
+  %3 = getelementptr inbounds %"struct.std::atomic", %"struct.std::atomic"* %1, i64 0, i32 0, i32 0
+  br label %4
+
+4:                                                ; preds = %4, %2
+  %5 = load atomic i8, i8* %3  monotonic, align 1
+  %6 = and i8 %5, 1
+  %7 = icmp eq i8 %6, 0
+  br i1 %7, label %4, label %8
+
+8:                                                ; preds = %4
+  fence syncscope("singlethread") acquire
+  ret void
+}
+
+declare void @llvm.memcpy(i8* %dest, i8* %src, i32 %len, i1 %isvolatile)
+declare void @llvm.memset(i8* %dest, i8 %val, i32 %len, i1 %isvolatile)
+
+; TEST 14 - negative, checking volatile intrinsics.
+
+; It is odd to add nocapture but a result of the llvm.memcpy nocapture.
+;
+; ATTRIBUTOR: Function Attrs: nounwind
+; ATTRIBUTOR-NOT: nosync
+; ATTRIBUTOR-NEXT: define i32 @memcpy_volatile(i8* nocapture writeonly %ptr1, i8* nocapture readonly %ptr2)
+define i32 @memcpy_volatile(i8* %ptr1, i8* %ptr2) {
+  call void @llvm.memcpy(i8* %ptr1, i8* %ptr2, i32 8, i1 1)
+  ret i32 4
+}
+
+; TEST 15 - positive, non-volatile intrinsic.
+
+; It is odd to add nocapture but a result of the llvm.memset nocapture.
+;
+; ATTRIBUTOR: Function Attrs: nosync
+; ATTRIBUTOR-NEXT: define i32 @memset_non_volatile(i8* nocapture writeonly %ptr1, i8 %val)
+define i32 @memset_non_volatile(i8* %ptr1, i8 %val) {
+  call void @llvm.memset(i8* %ptr1, i8 %val, i32 8, i1 0)
+  ret i32 4
+}
+
+; TEST 16 - negative, inline assembly.
+
+; ATTRIBUTOR: define i32 @inline_asm_test(i32 %x)
+define i32 @inline_asm_test(i32 %x) {
+  call i32 asm "bswap $0", "=r,r"(i32 %x)
+  ret i32 4
+}
+
+declare void @readnone_test() convergent readnone
+
+; ATTRIBUTOR: define void @convergent_readnone()
+; TEST 17 - negative. Convergent
+define void @convergent_readnone(){
+    call void @readnone_test()
+    ret void
+}
+
+; ATTRIBUTOR: Function Attrs: nounwind
+; ATTRIBUTOR-NEXT: declare void @llvm.x86.sse2.clflush(i8*)
+declare void @llvm.x86.sse2.clflush(i8*)
+@a = common global i32 0, align 4
+
+; TEST 18 - negative. Synchronizing intrinsic
+
+; ATTRIBUTOR: Function Attrs: nounwind
+; ATTRIBUTOR-NOT: nosync
+; ATTRIBUTOR-NEXT: define void @i_totally_sync()
+define void @i_totally_sync() {
+  tail call void @llvm.x86.sse2.clflush(i8* bitcast (i32* @a to i8*))
+  ret void
+}
+
+declare float @llvm.cos(float %val) readnone
+
+; TEST 19 - positive, readnone & non-convergent intrinsic.
+
+; ATTRIBUTOR: Function Attrs: nosync nounwind
+; ATTRIBUTOR-NEXT: define i32 @cos_test(float %x)
+define i32 @cos_test(float %x) {
+  call float @llvm.cos(float %x)
+  ret i32 4
+}
diff --git a/llvm/test/Transforms/Attributor/nounwind.ll b/llvm/test/Transforms/Attributor/nounwind.ll
new file mode 100644
index 000000000..e569095c8
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/nounwind.ll
@@ -0,0 +1,98 @@
+; RUN: opt < %s -attributor -attributor-disable=false -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=3 -S | FileCheck %s --check-prefix=ATTRIBUTOR
+; Copied from Transforms/FunctoinAttrs/nounwind.ll
+
+; TEST 1
+; ATTRIBUTOR: Function Attrs: nofree nosync nounwind
+; ATTRIBUTOR-NEXT: define i32 @foo1()
+define i32 @foo1() {
+  ret i32 1
+}
+
+; TEST 2
+; ATTRIBUTOR: Function Attrs: nofree noreturn nosync nounwind
+; ATTRIBUTOR-NEXT: define i32 @scc1_foo()
+define i32 @scc1_foo() {
+  %1 = call i32 @scc1_bar()
+  ret i32 1
+}
+
+
+; TEST 3
+; ATTRIBUTOR: Function Attrs: nofree noreturn nosync nounwind
+; ATTRIBUTOR-NEXT: define i32 @scc1_bar()
+define i32 @scc1_bar() {
+  %1 = call i32 @scc1_foo()
+  ret i32 1
+}
+
+declare i32 @non_nounwind()
+
+; TEST 4
+; ATTRIBUTOR: define void @call_non_nounwind() {
+define void @call_non_nounwind(){
+    tail call i32 @non_nounwind()
+    ret void
+}
+
+; TEST 5 - throw
+; int maybe_throw(bool canThrow) {
+;   if (canThrow)
+;     throw;
+;   else
+;     return -1;
+; }
+
+; ATTRIBUTOR: define i32 @maybe_throw(i1 zeroext %0)
+define i32 @maybe_throw(i1 zeroext %0) {
+  br i1 %0, label %2, label %3
+
+2:                                                ; preds = %1
+  tail call void @__cxa_rethrow() #1
+  unreachable
+
+3:                                                ; preds = %1
+  ret i32 -1
+}
+
+declare void @__cxa_rethrow()
+
+; TEST 6 - catch
+; int catch_thing() {
+;   try {
+;       int a = doThing(true);
+;   }
+;   catch(...) { return -1; }
+;   return 1;
+; }
+
+; ATTRIBUTOR: define i32 @catch_thing()
+define i32 @catch_thing() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+  invoke void @__cxa_rethrow() #1
+          to label %1 unwind label %2
+
+1:                                                ; preds = %0
+  unreachable
+
+2:                                                ; preds = %0
+  %3 = landingpad { i8*, i32 }
+          catch i8* null
+  %4 = extractvalue { i8*, i32 } %3, 0
+  %5 = tail call i8* @__cxa_begin_catch(i8* %4) #2
+  tail call void @__cxa_end_catch()
+  ret i32 -1
+}
+
+define i32 @catch_thing_user() {
+; ATTRIBUTOR:     define i32 @catch_thing_user
+; ATTRIBUTOR-NEXT: %catch_thing_call = call
+; ATTRIBUTOR-NEXT: ret i32 -1
+  %catch_thing_call = call i32 @catch_thing()
+  ret i32 %catch_thing_call
+}
+
+
+declare i32 @__gxx_personality_v0(...)
+
+declare i8* @__cxa_begin_catch(i8*)
+
+declare void @__cxa_end_catch()
diff --git a/llvm/test/Transforms/Attributor/range.ll b/llvm/test/Transforms/Attributor/range.ll
new file mode 100644
index 000000000..a73a1ea23
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/range.ll
@@ -0,0 +1,504 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -passes=attributor -attributor-disable=false -S < %s | FileCheck %s
+
+define i32 @test0(i32* %p) {
+; CHECK-LABEL: define {{[^@]+}}@test0
+; CHECK-SAME: (i32* nocapture nofree nonnull readonly dereferenceable(4) [[P:%.*]])
+; CHECK-NEXT:    [[A:%.*]] = load i32, i32* [[P]], !range !0
+; CHECK-NEXT:    ret i32 [[A]]
+;
+  %a = load i32, i32* %p, !range !0
+  ret i32 %a
+}
+
+define i32 @test0-range-check(i32* %p) {
+; CHECK-LABEL: define {{[^@]+}}@test0-range-check
+; CHECK-SAME: (i32* nocapture nofree readonly [[P:%.*]])
+; CHECK-NEXT:    [[A:%.*]] = tail call i32 @test0(i32* nocapture nofree readonly [[P]])
+; CHECK-SAME: !range !0
+; CHECK-NEXT:    ret i32 [[A]]
+;
+  %a = tail call i32 @test0(i32* %p)
+  ret i32 %a
+}
+
+declare void @use3-dummy(i1, i1, i1)
+define void @use3(i1, i1, i1) {
+; CHECK-LABEL: define {{[^@]+}}@use3
+; CHECK-SAME: (i1 [[TMP0:%.*]], i1 [[TMP1:%.*]], i1 [[TMP2:%.*]])
+; CHECK-NEXT:    tail call void @use3-dummy(i1 [[TMP0]], i1 [[TMP1]], i1 [[TMP2]])
+; CHECK-NEXT:    ret void
+;
+  tail call void @use3-dummy(i1 %0, i1 %1, i1 %2)
+  ret void
+}
+
+; TEST0 icmp test
+define void @test0-icmp-check(i32* %p){
+; CHECK-LABEL: define {{[^@]+}}@test0-icmp-check
+; CHECK-SAME: (i32* nocapture nofree readonly [[P:%.*]])
+; CHECK-NEXT:    [[RET:%.*]] = tail call i32 @test0(i32* nocapture nofree readonly [[P]])
+; CHECK-SAME: !range !0
+; CHECK-NEXT:    [[CMP_EQ_2:%.*]] = icmp eq i32 [[RET]], 9
+; CHECK-NEXT:    [[CMP_EQ_3:%.*]] = icmp eq i32 [[RET]], 8
+; CHECK-NEXT:    [[CMP_EQ_4:%.*]] = icmp eq i32 [[RET]], 1
+; CHECK-NEXT:    [[CMP_EQ_5:%.*]] = icmp eq i32 [[RET]], 0
+; CHECK-NEXT:    tail call void @use3(i1 false, i1 [[CMP_EQ_2]], i1 [[CMP_EQ_3]])
+; CHECK-NEXT:    tail call void @use3(i1 [[CMP_EQ_4]], i1 [[CMP_EQ_5]], i1 false)
+; CHECK-NEXT:    [[CMP_NE_2:%.*]] = icmp ne i32 [[RET]], 9
+; CHECK-NEXT:    [[CMP_NE_3:%.*]] = icmp ne i32 [[RET]], 8
+; CHECK-NEXT:    [[CMP_NE_4:%.*]] = icmp ne i32 [[RET]], 1
+; CHECK-NEXT:    [[CMP_NE_5:%.*]] = icmp ne i32 [[RET]], 0
+; CHECK-NEXT:    tail call void @use3(i1 true, i1 [[CMP_NE_2]], i1 [[CMP_NE_3]])
+; CHECK-NEXT:    tail call void @use3(i1 [[CMP_NE_4]], i1 [[CMP_NE_5]], i1 true)
+; CHECK-NEXT:    [[CMP_UGT_3:%.*]] = icmp ugt i32 [[RET]], 8
+; CHECK-NEXT:    [[CMP_UGT_4:%.*]] = icmp ugt i32 [[RET]], 1
+; CHECK-NEXT:    [[CMP_UGT_5:%.*]] = icmp ugt i32 [[RET]], 0
+; CHECK-NEXT:    tail call void @use3(i1 false, i1 false, i1 [[CMP_UGT_3]])
+; CHECK-NEXT:    tail call void @use3(i1 [[CMP_UGT_4]], i1 [[CMP_UGT_5]], i1 false)
+; CHECK-NEXT:    [[CMP_UGE_2:%.*]] = icmp uge i32 [[RET]], 9
+; CHECK-NEXT:    [[CMP_UGE_3:%.*]] = icmp uge i32 [[RET]], 8
+; CHECK-NEXT:    [[CMP_UGE_4:%.*]] = icmp uge i32 [[RET]], 1
+; CHECK-NEXT:    tail call void @use3(i1 false, i1 [[CMP_UGE_2]], i1 [[CMP_UGE_3]])
+; CHECK-NEXT:    tail call void @use3(i1 [[CMP_UGE_4]], i1 true, i1 false)
+; CHECK-NEXT:    [[CMP_SGT_3:%.*]] = icmp sgt i32 [[RET]], 8
+; CHECK-NEXT:    [[CMP_SGT_4:%.*]] = icmp sgt i32 [[RET]], 1
+; CHECK-NEXT:    [[CMP_SGT_5:%.*]] = icmp sgt i32 [[RET]], 0
+; CHECK-NEXT:    tail call void @use3(i1 false, i1 false, i1 [[CMP_SGT_3]])
+; CHECK-NEXT:    tail call void @use3(i1 [[CMP_SGT_4]], i1 [[CMP_SGT_5]], i1 true)
+; CHECK-NEXT:    [[CMP_GTE_2:%.*]] = icmp sge i32 [[RET]], 9
+; CHECK-NEXT:    [[CMP_GTE_3:%.*]] = icmp sge i32 [[RET]], 8
+; CHECK-NEXT:    [[CMP_GTE_4:%.*]] = icmp sge i32 [[RET]], 1
+; CHECK-NEXT:    tail call void @use3(i1 false, i1 [[CMP_GTE_2]], i1 [[CMP_GTE_3]])
+; CHECK-NEXT:    tail call void @use3(i1 [[CMP_GTE_4]], i1 true, i1 true)
+; CHECK-NEXT:    [[CMP_SLT_2:%.*]] = icmp slt i32 [[RET]], 9
+; CHECK-NEXT:    [[CMP_SLT_3:%.*]] = icmp slt i32 [[RET]], 8
+; CHECK-NEXT:    [[CMP_SLT_4:%.*]] = icmp slt i32 [[RET]], 1
+; CHECK-NEXT:    tail call void @use3(i1 true, i1 [[CMP_SLT_2]], i1 [[CMP_SLT_3]])
+; CHECK-NEXT:    tail call void @use3(i1 [[CMP_SLT_4]], i1 false, i1 false)
+; CHECK-NEXT:    [[CMP_LTE_3:%.*]] = icmp sle i32 [[RET]], 8
+; CHECK-NEXT:    [[CMP_LTE_4:%.*]] = icmp sle i32 [[RET]], 1
+; CHECK-NEXT:    [[CMP_LTE_5:%.*]] = icmp sle i32 [[RET]], 0
+; CHECK-NEXT:    tail call void @use3(i1 true, i1 true, i1 [[CMP_LTE_3]])
+; CHECK-NEXT:    tail call void @use3(i1 [[CMP_LTE_4]], i1 [[CMP_LTE_5]], i1 false)
+; CHECK-NEXT:    ret void
+;
+  ; ret = [0, 10)
+  %ret = tail call i32 @test0(i32 *%p)
+
+  ; ret = [0, 10), eq
+  %cmp-eq-1 = icmp eq i32 %ret, 10
+  %cmp-eq-2 = icmp eq i32 %ret, 9
+  %cmp-eq-3 = icmp eq i32 %ret, 8
+  %cmp-eq-4 = icmp eq i32 %ret, 1
+  %cmp-eq-5 = icmp eq i32 %ret, 0
+  %cmp-eq-6 = icmp eq i32 %ret, -1
+  tail call void @use3(i1 %cmp-eq-1, i1 %cmp-eq-2, i1 %cmp-eq-3)
+  tail call void @use3(i1 %cmp-eq-4, i1 %cmp-eq-5, i1 %cmp-eq-6)
+
+  ; ret = [0, 10), ne
+  %cmp-ne-1 = icmp ne i32 %ret, 10
+  %cmp-ne-2 = icmp ne i32 %ret, 9
+  %cmp-ne-3 = icmp ne i32 %ret, 8
+  %cmp-ne-4 = icmp ne i32 %ret, 1
+  %cmp-ne-5 = icmp ne i32 %ret, 0
+  %cmp-ne-6 = icmp ne i32 %ret, -1
+  tail call void @use3(i1 %cmp-ne-1, i1 %cmp-ne-2, i1 %cmp-ne-3)
+  tail call void @use3(i1 %cmp-ne-4, i1 %cmp-ne-5, i1 %cmp-ne-6)
+
+  ; ret = [0, 10), ugt
+  %cmp-ugt-1 = icmp ugt i32 %ret, 10
+  %cmp-ugt-2 = icmp ugt i32 %ret, 9
+  %cmp-ugt-3 = icmp ugt i32 %ret, 8
+  %cmp-ugt-4 = icmp ugt i32 %ret, 1
+  %cmp-ugt-5 = icmp ugt i32 %ret, 0
+  %cmp-ugt-6 = icmp ugt i32 %ret, -1
+  tail call void @use3(i1 %cmp-ugt-1, i1 %cmp-ugt-2, i1 %cmp-ugt-3)
+  tail call void @use3(i1 %cmp-ugt-4, i1 %cmp-ugt-5, i1 %cmp-ugt-6)
+
+  ; ret = [0, 10), uge
+  %cmp-uge-1 = icmp uge i32 %ret, 10
+  %cmp-uge-2 = icmp uge i32 %ret, 9
+  %cmp-uge-3 = icmp uge i32 %ret, 8
+  %cmp-uge-4 = icmp uge i32 %ret, 1
+  %cmp-uge-5 = icmp uge i32 %ret, 0
+  %cmp-uge-6 = icmp uge i32 %ret, -1
+  tail call void @use3(i1 %cmp-uge-1, i1 %cmp-uge-2, i1 %cmp-uge-3)
+  tail call void @use3(i1 %cmp-uge-4, i1 %cmp-uge-5, i1 %cmp-uge-6)
+
+  ; ret = [0, 10), sgt
+  %cmp-sgt-1 = icmp sgt i32 %ret, 10
+  %cmp-sgt-2 = icmp sgt i32 %ret, 9
+  %cmp-sgt-3 = icmp sgt i32 %ret, 8
+  %cmp-sgt-4 = icmp sgt i32 %ret, 1
+  %cmp-sgt-5 = icmp sgt i32 %ret, 0
+  %cmp-sgt-6 = icmp sgt i32 %ret, -1
+  tail call void @use3(i1 %cmp-sgt-1, i1 %cmp-sgt-2, i1 %cmp-sgt-3)
+  tail call void @use3(i1 %cmp-sgt-4, i1 %cmp-sgt-5, i1 %cmp-sgt-6)
+
+  ; ret = [0, 10), sge
+  %cmp-gte-1 = icmp sge i32 %ret, 10
+  %cmp-gte-2 = icmp sge i32 %ret, 9
+  %cmp-gte-3 = icmp sge i32 %ret, 8
+  %cmp-gte-4 = icmp sge i32 %ret, 1
+  %cmp-gte-5 = icmp sge i32 %ret, 0
+  %cmp-gte-6 = icmp sge i32 %ret, -1
+  tail call void @use3(i1 %cmp-gte-1, i1 %cmp-gte-2, i1 %cmp-gte-3)
+  tail call void @use3(i1 %cmp-gte-4, i1 %cmp-gte-5, i1 %cmp-gte-6)
+
+  ; ret = [0, 10), slt
+  %cmp-slt-1 = icmp slt i32 %ret, 10
+  %cmp-slt-2 = icmp slt i32 %ret, 9
+  %cmp-slt-3 = icmp slt i32 %ret, 8
+  %cmp-slt-4 = icmp slt i32 %ret, 1
+  %cmp-slt-5 = icmp slt i32 %ret, 0
+  %cmp-slt-6 = icmp slt i32 %ret, -1
+  tail call void @use3(i1 %cmp-slt-1, i1 %cmp-slt-2, i1 %cmp-slt-3)
+  tail call void @use3(i1 %cmp-slt-4, i1 %cmp-slt-5, i1 %cmp-slt-6)
+
+  ; ret = [0, 10), sle
+  %cmp-lte-1 = icmp sle i32 %ret, 10
+  %cmp-lte-2 = icmp sle i32 %ret, 9
+  %cmp-lte-3 = icmp sle i32 %ret, 8
+  %cmp-lte-4 = icmp sle i32 %ret, 1
+  %cmp-lte-5 = icmp sle i32 %ret, 0
+  %cmp-lte-6 = icmp sle i32 %ret, -1
+  tail call void @use3(i1 %cmp-lte-1, i1 %cmp-lte-2, i1 %cmp-lte-3)
+  tail call void @use3(i1 %cmp-lte-4, i1 %cmp-lte-5, i1 %cmp-lte-6)
+
+  ret void
+}
+define i32 @test1(i32* %p) {
+; CHECK-LABEL: define {{[^@]+}}@test1
+; CHECK-SAME: (i32* nocapture nofree nonnull readonly dereferenceable(4) [[P:%.*]])
+; CHECK-NEXT:    [[LOAD_10_100:%.*]] = load i32, i32* [[P]], !range !1
+; CHECK-NEXT:    [[ADD_10_THEN_20_110:%.*]] = add i32 [[LOAD_10_100]], 10
+; CHECK-NEXT:    [[MUL_10_THEN_200_1091:%.*]] = mul i32 [[ADD_10_THEN_20_110]], 10
+; CHECK-NEXT:    ret i32 [[MUL_10_THEN_200_1091]]
+;
+  %load-10-100 = load i32, i32* %p, !range !1
+  %add-10-then-20-110 = add i32 %load-10-100, 10
+  %mul-10-then-200-1091 = mul i32 %add-10-then-20-110, 10
+  ret i32 %mul-10-then-200-1091
+}
+
+define i1 @test1-check(i32* %p) {
+; CHECK-LABEL: define {{[^@]+}}@test1-check
+; CHECK-SAME: (i32* nocapture nofree readonly [[P:%.*]])
+; CHECK-NEXT:    [[RES:%.*]] = tail call i32 @test1(i32* nocapture nofree readonly [[P]])
+; CHECK-SANME: !range !2
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[RES]], 500
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %res = tail call i32 @test1(i32* %p)
+  %cmp = icmp eq i32 %res, 500
+  ret i1 %cmp
+}
+
+;  TEST2
+;  int test2(int *p) { return *p == 0 ? 4 : 3; }
+;  int test2_check(int *p) {
+;    int call = test2(p);
+;    if (call == 5) {
+;      // dead block
+;      return 2;
+;    } else {
+;      return 3;
+;    }
+;  }
+
+define i32 @test2(i32* %p) {
+; CHECK-LABEL: define {{[^@]+}}@test2
+; CHECK-SAME: (i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[P:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[P]], align 4
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[TMP0]], 0
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[TOBOOL]], i32 4, i32 3
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+entry:
+  %0 = load i32, i32* %p, align 4
+  %tobool = icmp eq i32 %0, 0
+  %cond = select i1 %tobool, i32 4, i32 3
+  ret i32 %cond
+}
+
+define i32 @test2_check(i32* %p) {
+; CHECK-LABEL: define {{[^@]+}}@test2_check
+; CHECK-SAME: (i32* nocapture nofree readonly align 4 [[P:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[IF_THEN:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    br label [[RETURN:%.*]]
+; CHECK:       if.end:
+; CHECK-NEXT:    unreachable
+; CHECK:       return:
+; CHECK-NEXT:    ret i32 2
+;
+entry:
+  %call = tail call i32 @test2(i32* %p)
+  %cmp = icmp slt i32 %call, 5
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  br label %return
+
+if.end:                                           ; preds = %entry
+  br label %return
+
+return:                                           ; preds = %if.end, %if.then
+  %retval.0 = phi i32 [ 2, %if.then ], [ 3, %if.end ]
+  ret i32 %retval.0
+}
+
+; TEST 3 SECV test
+
+; void unkown();
+; int r1(unsigned int u){
+;   int sum = 0;
+;   for(int i = 0; i<100;i++){
+;     sum += i;
+;   }
+;   // sum = 50 * 49 / 2
+;   if(sum > 10000){
+;   // dead block
+;     return 20;
+;   }else {
+;     return 10;
+;   }
+; }
+; void f1(int u){
+;   if(r1(u) > 15){
+;   // deadblock
+;     unkown();
+;   }else {
+;     return;
+;   }
+; }
+
+declare dso_local void @unkown()
+
+define internal i32 @r1(i32) local_unnamed_addr {
+; CHECK-LABEL: define {{[^@]+}}@r1() local_unnamed_addr
+; CHECK-NEXT:    br label [[TMP3:%.*]]
+; CHECK:       1:
+; CHECK-NEXT:    br label [[F:%.*]]
+; CHECK:       2:
+; CHECK-NEXT:    unreachable
+; CHECK:       f:
+; CHECK-NEXT:    ret i32 10
+; CHECK:       3:
+; CHECK-NEXT:    [[TMP4:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP7:%.*]], [[TMP3]] ]
+; CHECK-NEXT:    [[TMP5:%.*]] = phi i32 [ 0, [[TMP0]] ], [ [[TMP6:%.*]], [[TMP3]] ]
+; CHECK-NEXT:    [[TMP6]] = add nuw nsw i32 [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[TMP7]] = add nuw nsw i32 [[TMP4]], 1
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i32 [[TMP7]], 100
+; CHECK-NEXT:    br i1 [[TMP8]], label [[TMP1:%.*]], label [[TMP3]]
+;
+  br label %5
+
+2:                                                ; preds = %5
+  %3 = icmp sgt i32 %8, 10000
+  br i1 %3, label %4, label %f
+4:
+  ret i32 20
+f:
+  ret i32 10
+5:                                                ; preds = %5, %1
+  %6 = phi i32 [ 0, %1 ], [ %9, %5 ]
+  %7 = phi i32 [ 0, %1 ], [ %8, %5 ]
+  %8 = add nuw nsw i32 %6, %7
+  %9 = add nuw nsw i32 %6, 1
+  %10 = icmp eq i32 %9, 100
+  br i1 %10, label %2, label %5
+}
+
+define void @f1(i32){
+; CHECK-LABEL: define {{[^@]+}}@f1
+; CHECK-SAME: (i32 [[TMP0:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = tail call i32 @r1()
+; CHECK-NEXT:    br label [[TMP4:%.*]]
+; CHECK:       3:
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    ret void
+;
+  %2 = tail call i32 @r1(i32 %0)
+  %3 = icmp sgt i32 %2, 15
+  br i1 %3, label %4, label %5
+
+4:                                                ; preds = %1
+  tail call void @unkown()
+  br label %5
+
+5:                                                ; preds = %1, %4
+  ret void
+}
+
+; TEST4 LVI test
+
+; f1
+; int test4-f1(int u){
+;   if(u>=0) {
+;     return u;
+;   }else{
+;     return 0;
+;   }
+; }
+define dso_local i32 @test4-f1(i32 %u) {
+; CHECK-LABEL: define {{[^@]+}}@test4-f1
+; CHECK-SAME: (i32 [[U:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[U]], -1
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[RETURN:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    br label [[RETURN]]
+; CHECK:       return:
+; CHECK-NEXT:    [[RETVAL_0:%.*]] = phi i32 [ [[U]], [[IF_THEN]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    ret i32 [[RETVAL_0]]
+;
+; FIXME: RETVAL_0 >= 0
+entry:
+  %cmp = icmp sgt i32 %u, -1
+  br i1 %cmp, label %if.then, label %return
+
+if.then:                                          ; preds = %entry
+  br label %return
+
+return:                                           ; preds = %entry, %if.then
+  %retval.0 = phi i32 [ %u, %if.then ], [ 0, %entry ]
+  ret i32 %retval.0
+}
+
+
+define dso_local i32 @test4-g1(i32 %u) {
+; CHECK-LABEL: define {{[^@]+}}@test4-g1
+; CHECK-SAME: (i32 [[U:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 @test4-f1(i32 [[U]])
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+; FIXME: %call should have range [0, inf]
+
+entry:
+  %call = tail call i32 @test4-f1(i32 %u)
+  ret i32 %call
+}
+
+; f2
+; int test4-f1(int u){
+;   if(u>-1) {
+;     return u+1;
+;   }else{
+;     return 1;
+;   }
+; }
+define dso_local i32 @test4-f2(i32 %u) {
+; CHECK-LABEL: define {{[^@]+}}@test4-f2
+; CHECK-SAME: (i32 [[U:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[U]], -1
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[ADD:%.*]] = add nuw nsw i32 [[U]], 1
+; CHECK-NEXT:    br label [[RETURN:%.*]]
+; CHECK:       if.else:
+; CHECK-NEXT:    br label [[RETURN]]
+; CHECK:       return:
+; CHECK-NEXT:    [[RETVAL_0:%.*]] = phi i32 [ [[ADD]], [[IF_THEN]] ], [ 1, [[IF_ELSE]] ]
+; CHECK-NEXT:    ret i32 [[RETVAL_0]]
+;
+entry:
+  %cmp = icmp sgt i32 %u, -1
+  br i1 %cmp, label %if.then, label %if.else
+
+if.then:                                          ; preds = %entry
+  %add = add nuw nsw i32 %u, 1
+  br label %return
+
+if.else:                                          ; preds = %entry
+  br label %return
+
+return:                                           ; preds = %if.else, %if.then
+  %retval.0 = phi i32 [ %add, %if.then ], [ 1, %if.else ]
+  ret i32 %retval.0
+}
+
+
+define dso_local i32 @test4-g2(i32 %u) {
+; CHECK-LABEL: define {{[^@]+}}@test4-g2
+; CHECK-SAME: (i32 [[U:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 @test4-f2(i32 [[U]])
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+; FIXME: %call should have range [1, inf]
+entry:
+  %call = tail call i32 @test4-f2(i32 %u)
+  ret i32 %call
+}
+
+define dso_local i32 @test-5() {
+; CHECK-LABEL: define {{[^@]+}}@test-5()
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @rec(i32 0), !range !3
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+entry:
+  %call = call i32 @rec(i32 0)
+  ret i32 %call
+}
+define internal i32 @rec(i32 %depth) {
+; CHECK-LABEL: define {{[^@]+}}@rec
+; CHECK-SAME: (i32 [[DEPTH:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @foo(i32 [[DEPTH]])
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne i32 [[CALL]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    br label [[RETURN:%.*]]
+; CHECK:       if.end:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[DEPTH]], 10
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN1:%.*]], label [[IF_END3:%.*]]
+; CHECK:       if.then1:
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[DEPTH]], 1
+; CHECK-NEXT:    [[CALL2:%.*]] = call i32 @rec(i32 [[ADD]])
+; CHECK-NEXT:    br label [[IF_END3]]
+; CHECK:       if.end3:
+; CHECK-NEXT:    br label [[RETURN]]
+; CHECK:       return:
+; CHECK-NEXT:    [[RETVAL_0:%.*]] = phi i32 [ 0, [[IF_THEN]] ], [ 1, [[IF_END3]] ]
+; CHECK-NEXT:    ret i32 [[RETVAL_0]]
+;
+entry:
+  %call = call i32 @foo(i32 %depth)
+  %tobool = icmp ne i32 %call, 0
+  br i1 %tobool, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  br label %return
+
+if.end:                                           ; preds = %entry
+  %cmp = icmp slt i32 %depth, 10
+  br i1 %cmp, label %if.then1, label %if.end3
+
+if.then1:                                         ; preds = %if.end
+  %add = add nsw i32 %depth, 1
+  %call2 = call i32 @rec(i32 %add)
+  br label %if.end3
+
+if.end3:                                          ; preds = %if.then1, %if.end
+  br label %return
+
+return:                                           ; preds = %if.end3, %if.then
+  %retval.0 = phi i32 [ 0, %if.then ], [ 1, %if.end3 ]
+  ret i32 %retval.0
+}
+declare dso_local i32 @foo(i32)
+
+!0 = !{i32 0, i32 10}
+!1 = !{i32 10, i32 100}
+;CHECK: !0 = !{i32 0, i32 10}
+;CHECK-NEXT: !1 = !{i32 10, i32 100}
+;CHECK-NEXT: !2 = !{i32 200, i32 1091}
+
diff --git a/llvm/test/Transforms/Attributor/read_write_returned_arguments_scc.ll b/llvm/test/Transforms/Attributor/read_write_returned_arguments_scc.ll
new file mode 100644
index 000000000..72d062b76
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/read_write_returned_arguments_scc.ll
@@ -0,0 +1,165 @@
+; RUN: opt -functionattrs -enable-nonnull-arg-prop -attributor -attributor-manifest-internal -attributor-disable=false -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=6 -S < %s | FileCheck %s
+;
+; This is an evolved example to stress test SCC parameter attribute propagation.
+; The SCC in this test is made up of the following six function, three of which
+; are internal and three externally visible:
+;
+; static int *internal_ret0_nw(int *n0, int *w0);
+; static int *internal_ret1_rw(int *r0, int *w0);
+; static int *internal_ret1_rrw(int *r0, int *r1, int *w0);
+;        int *external_ret2_nrw(int *n0, int *r0, int *w0);
+;        int *external_sink_ret2_nrw(int *n0, int *r0, int *w0);
+;        int *external_source_ret2_nrw(int *n0, int *r0, int *w0);
+;
+; The top four functions call each other while the "sink" function will not
+; call anything and the "source" function will not be called in this module.
+; The names of the functions define the returned parameter (X for "_retX_"),
+; as well as how the parameters are (transitively) used (n = readnone,
+; r = readonly, w = writeonly).
+;
+; What we should see is something along the lines of:
+;   1 - Number of functions marked as norecurse
+;   6 - Number of functions marked argmemonly
+;   6 - Number of functions marked as nounwind
+;  16 - Number of arguments marked nocapture
+;   4 - Number of arguments marked readnone
+;   6 - Number of arguments marked writeonly
+;   6 - Number of arguments marked readonly
+;   6 - Number of arguments marked returned
+;
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+; CHECK: Function Attrs: nofree nosync nounwind
+; CHECK-NEXT: define i32* @external_ret2_nrw(i32* nofree %n0, i32* nofree %r0, i32* nofree returned %w0)
+define i32* @external_ret2_nrw(i32* %n0, i32* %r0, i32* %w0) {
+entry:
+  %call = call i32* @internal_ret0_nw(i32* %n0, i32* %w0)
+  %call1 = call i32* @internal_ret1_rrw(i32* %r0, i32* %r0, i32* %w0)
+  %call2 = call i32* @external_sink_ret2_nrw(i32* %n0, i32* %r0, i32* %w0)
+  %call3 = call i32* @internal_ret1_rw(i32* %r0, i32* %w0)
+  ret i32* %call3
+}
+
+; CHECK: Function Attrs: nofree nosync nounwind
+; CHECK-NEXT: define internal i32* @internal_ret0_nw(i32* nofree returned %n0, i32* nofree %w0)
+define internal i32* @internal_ret0_nw(i32* %n0, i32* %w0) {
+entry:
+  %r0 = alloca i32, align 4
+  %r1 = alloca i32, align 4
+  %tobool = icmp ne i32* %n0, null
+  br i1 %tobool, label %if.end, label %if.then
+
+if.then:                                          ; preds = %entry
+  br label %return
+
+if.end:                                           ; preds = %entry
+  store i32 3, i32* %r0, align 4
+  store i32 5, i32* %r1, align 4
+  store i32 1, i32* %w0, align 4
+  %call = call i32* @internal_ret1_rrw(i32* %r0, i32* %r1, i32* %w0)
+  %call1 = call i32* @external_ret2_nrw(i32* %n0, i32* %r0, i32* %w0)
+  %call2 = call i32* @external_ret2_nrw(i32* %n0, i32* %r1, i32* %w0)
+  %call3 = call i32* @external_sink_ret2_nrw(i32* %n0, i32* %r0, i32* %w0)
+  %call4 = call i32* @external_sink_ret2_nrw(i32* %n0, i32* %r1, i32* %w0)
+  %call5 = call i32* @internal_ret0_nw(i32* %n0, i32* %w0)
+  br label %return
+
+return:                                           ; preds = %if.end, %if.then
+  %retval.0 = phi i32* [ %call5, %if.end ], [ %n0, %if.then ]
+  ret i32* %retval.0
+}
+
+; CHECK: Function Attrs: nofree nosync nounwind
+; CHECK-NEXT: define internal i32* @internal_ret1_rrw(i32* nofree nonnull align 4 dereferenceable(4) %r0, i32* nofree returned %r1, i32* nofree %w0)
+define internal i32* @internal_ret1_rrw(i32* %r0, i32* %r1, i32* %w0) {
+entry:
+  %0 = load i32, i32* %r0, align 4
+  %tobool = icmp ne i32 %0, 0
+  br i1 %tobool, label %if.end, label %if.then
+
+if.then:                                          ; preds = %entry
+  br label %return
+
+if.end:                                           ; preds = %entry
+  %call = call i32* @internal_ret1_rw(i32* %r0, i32* %w0)
+  %1 = load i32, i32* %r0, align 4
+  %2 = load i32, i32* %r1, align 4
+  %add = add nsw i32 %1, %2
+  store i32 %add, i32* %w0, align 4
+  %call1 = call i32* @internal_ret1_rw(i32* %r1, i32* %w0)
+  %call2 = call i32* @internal_ret0_nw(i32* %r0, i32* %w0)
+  %call3 = call i32* @internal_ret0_nw(i32* %w0, i32* %w0)
+  %call4 = call i32* @external_ret2_nrw(i32* %r0, i32* %r1, i32* %w0)
+  %call5 = call i32* @external_ret2_nrw(i32* %r1, i32* %r0, i32* %w0)
+  %call6 = call i32* @external_sink_ret2_nrw(i32* %r0, i32* %r1, i32* %w0)
+  %call7 = call i32* @external_sink_ret2_nrw(i32* %r1, i32* %r0, i32* %w0)
+  %call8 = call i32* @internal_ret0_nw(i32* %r1, i32* %w0)
+  br label %return
+
+return:                                           ; preds = %if.end, %if.then
+  %retval.0 = phi i32* [ %call8, %if.end ], [ %r1, %if.then ]
+  ret i32* %retval.0
+}
+
+; CHECK: Function Attrs: nofree norecurse nosync nounwind
+; CHECK-NEXT: define i32* @external_sink_ret2_nrw(i32* nofree readnone %n0, i32* nocapture nofree readonly %r0, i32* nofree returned writeonly "no-capture-maybe-returned" %w0)
+define i32* @external_sink_ret2_nrw(i32* %n0, i32* %r0, i32* %w0) {
+entry:
+  %tobool = icmp ne i32* %n0, null
+  br i1 %tobool, label %if.end, label %if.then
+
+if.then:                                          ; preds = %entry
+  br label %return
+
+if.end:                                           ; preds = %entry
+  %0 = load i32, i32* %r0, align 4
+  store i32 %0, i32* %w0, align 4
+  br label %return
+
+return:                                           ; preds = %if.end, %if.then
+  ret i32* %w0
+}
+
+; CHECK: Function Attrs: nofree nosync nounwind
+; CHECK-NEXT: define internal i32* @internal_ret1_rw(i32* nofree nonnull align 4 dereferenceable(4) %r0, i32* nofree returned %w0)
+define internal i32* @internal_ret1_rw(i32* %r0, i32* %w0) {
+entry:
+  %0 = load i32, i32* %r0, align 4
+  %tobool = icmp ne i32 %0, 0
+  br i1 %tobool, label %if.end, label %if.then
+
+if.then:                                          ; preds = %entry
+  br label %return
+
+if.end:                                           ; preds = %entry
+  %call = call i32* @internal_ret1_rrw(i32* %r0, i32* %r0, i32* %w0)
+  %1 = load i32, i32* %r0, align 4
+  store i32 %1, i32* %w0, align 4
+  %call1 = call i32* @internal_ret0_nw(i32* %r0, i32* %w0)
+  %call2 = call i32* @internal_ret0_nw(i32* %w0, i32* %w0)
+  %call3 = call i32* @external_sink_ret2_nrw(i32* %r0, i32* %r0, i32* %w0)
+  %call4 = call i32* @external_ret2_nrw(i32* %r0, i32* %r0, i32* %w0)
+  br label %return
+
+return:                                           ; preds = %if.end, %if.then
+  %retval.0 = phi i32* [ %call4, %if.end ], [ %w0, %if.then ]
+  ret i32* %retval.0
+}
+
+; CHECK: Function Attrs: nofree nosync nounwind
+; CHECK-NEXT: define i32* @external_source_ret2_nrw(i32* nofree %n0, i32* nofree %r0, i32* nofree returned %w0)
+define i32* @external_source_ret2_nrw(i32* %n0, i32* %r0, i32* %w0) {
+entry:
+  %call = call i32* @external_sink_ret2_nrw(i32* %n0, i32* %r0, i32* %w0)
+  %call1 = call i32* @external_ret2_nrw(i32* %n0, i32* %r0, i32* %w0)
+  ret i32* %call1
+}
+
+; Verify that we see only expected attribute sets, the above lines only check
+; for a subset relation.
+;
+; CHECK-NOT: attributes #
+; CHECK: attributes #{{.*}} = { nofree nosync nounwind }
+; CHECK: attributes #{{.*}} = { nofree norecurse nosync nounwind }
+; CHECK: attributes #{{.*}} = { nosync }
+; CHECK-NOT: attributes #
diff --git a/llvm/test/Transforms/Attributor/readattrs.ll b/llvm/test/Transforms/Attributor/readattrs.ll
new file mode 100644
index 000000000..cfb4f71ce
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/readattrs.ll
@@ -0,0 +1,197 @@
+; RUN: opt < %s -attributor -attributor-disable=false -S -attributor-annotate-decl-cs | FileCheck %s --check-prefixes=ATTRIBUTOR
+; RUN: opt < %s -aa-pipeline=basic-aa -passes='attributor' -attributor-disable=false -S -attributor-annotate-decl-cs | FileCheck %s --check-prefixes=ATTRIBUTOR
+; Copied from Transforms/FunctionAttrs/readattrs.ll
+
+@x = global i32 0
+
+declare void @test1_1(i8* %x1_1, i8* readonly %y1_1, ...)
+
+; NOTE: readonly for %y1_2 would be OK here but not for the similar situation in test13.
+;
+; ATTRIBUTOR: define void @test1_2(i8* %x1_2, i8* %y1_2, i8* %z1_2)
+define void @test1_2(i8* %x1_2, i8* %y1_2, i8* %z1_2) {
+  call void (i8*, i8*, ...) @test1_1(i8* %x1_2, i8* %y1_2, i8* %z1_2)
+  store i32 0, i32* @x
+  ret void
+}
+
+; ATTRIBUTOR: define i8* @test2(i8* nofree readnone returned %p)
+define i8* @test2(i8* %p) {
+  store i32 0, i32* @x
+  ret i8* %p
+}
+
+; ATTRIBUTOR: define i1 @test3(i8* nofree readnone %p, i8* nofree readnone %q)
+define i1 @test3(i8* %p, i8* %q) {
+  %A = icmp ult i8* %p, %q
+  ret i1 %A
+}
+
+declare void @test4_1(i8* nocapture) readonly
+
+; ATTRIBUTOR: define void @test4_2(i8* nocapture readonly %p)
+define void @test4_2(i8* %p) {
+  call void @test4_1(i8* %p)
+  ret void
+}
+
+; ATTRIBUTOR: define void @test5(i8** nocapture nofree nonnull writeonly dereferenceable(8) %p, i8* nofree writeonly %q)
+; Missed optz'n: we could make %q readnone, but don't break test6!
+define void @test5(i8** %p, i8* %q) {
+  store i8* %q, i8** %p
+  ret void
+}
+
+declare void @test6_1()
+; ATTRIBUTOR: define void @test6_2(i8** nocapture nonnull writeonly dereferenceable(8) %p, i8* %q)
+; This is not a missed optz'n.
+define void @test6_2(i8** %p, i8* %q) {
+  store i8* %q, i8** %p
+  call void @test6_1()
+  ret void
+}
+
+; ATTRIBUTOR: define void @test7_1(i32* inalloca nocapture nofree writeonly %a)
+; inalloca parameters are always considered written
+define void @test7_1(i32* inalloca %a) {
+  ret void
+}
+
+; ATTRIBUTOR: define i32* @test8_1(i32* nofree readnone returned %p)
+define i32* @test8_1(i32* %p) {
+entry:
+  ret i32* %p
+}
+
+; ATTRIBUTOR: define void @test8_2(i32* nocapture nofree writeonly %p)
+define void @test8_2(i32* %p) {
+entry:
+  %call = call i32* @test8_1(i32* %p)
+  store i32 10, i32* %call, align 4
+  ret void
+}
+
+; ATTRIBUTOR: declare void @llvm.masked.scatter
+declare void @llvm.masked.scatter.v4i32.v4p0i32(<4 x i32>%val, <4 x i32*>, i32, <4 x i1>)
+
+; ATTRIBUTOR-NOT: readnone
+; ATTRIBUTOR-NOT: readonly
+; ATTRIBUTOR: define void @test9
+define void @test9(<4 x i32*> %ptrs, <4 x i32>%val) {
+  call void @llvm.masked.scatter.v4i32.v4p0i32(<4 x i32>%val, <4 x i32*> %ptrs, i32 4, <4 x i1><i1 true, i1 false, i1 true, i1 false>)
+  ret void
+}
+
+; ATTRIBUTOR: declare <4 x i32> @llvm.masked.gather
+declare <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*>, i32, <4 x i1>, <4 x i32>)
+; ATTRIBUTOR: readonly
+; ATTRIBUTOR: define <4 x i32> @test10
+define <4 x i32> @test10(<4 x i32*> %ptrs) {
+  %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %ptrs, i32 4, <4 x i1><i1 true, i1 false, i1 true, i1 false>, <4 x i32>undef)
+  ret <4 x i32> %res
+}
+
+; ATTRIBUTOR: declare <4 x i32> @test11_1
+declare <4 x i32> @test11_1(<4 x i32*>) argmemonly nounwind readonly
+; ATTRIBUTOR: readonly
+; ATTRIBUTOR-NOT: readnone
+; ATTRIBUTOR: define <4 x i32> @test11_2
+define <4 x i32> @test11_2(<4 x i32*> %ptrs) {
+  %res = call <4 x i32> @test11_1(<4 x i32*> %ptrs)
+  ret <4 x i32> %res
+}
+
+declare <4 x i32> @test12_1(<4 x i32*>) argmemonly nounwind
+; ATTRIBUTOR-NOT: readnone
+; ATTRIBUTOR: define <4 x i32> @test12_2
+define <4 x i32> @test12_2(<4 x i32*> %ptrs) {
+  %res = call <4 x i32> @test12_1(<4 x i32*> %ptrs)
+  ret <4 x i32> %res
+}
+
+; ATTRIBUTOR: define i32 @volatile_load(
+; ATTRIBUTOR-NOT: readonly
+; ATTRIBUTOR: ret
+define i32 @volatile_load(i32* %p) {
+  %load = load volatile i32, i32* %p
+  ret i32 %load
+}
+
+declare void @escape_readnone_ptr(i8** %addr, i8* readnone %ptr)
+declare void @escape_readonly_ptr(i8** %addr, i8* readonly %ptr)
+
+; The argument pointer %escaped_then_written cannot be marked readnone/only even
+; though the only direct use, in @escape_readnone_ptr/@escape_readonly_ptr,
+; is marked as readnone/only. However, the functions can write the pointer into
+; %addr, causing the store to write to %escaped_then_written.
+;
+;
+; ATTRIBUTOR: define void @unsound_readnone(i8* nocapture nofree readnone %ignored, i8* %escaped_then_written)
+; ATTRIBUTOR: define void @unsound_readonly(i8* nocapture nofree readnone %ignored, i8* %escaped_then_written)
+define void @unsound_readnone(i8* %ignored, i8* %escaped_then_written) {
+  %addr = alloca i8*
+  call void @escape_readnone_ptr(i8** %addr, i8* %escaped_then_written)
+  %addr.ld = load i8*, i8** %addr
+  store i8 0, i8* %addr.ld
+  ret void
+}
+
+define void @unsound_readonly(i8* %ignored, i8* %escaped_then_written) {
+  %addr = alloca i8*
+  call void @escape_readonly_ptr(i8** %addr, i8* %escaped_then_written)
+  %addr.ld = load i8*, i8** %addr
+  store i8 0, i8* %addr.ld
+  ret void
+}
+
+; Byval but not readonly/none tests
+;
+;{
+declare void @escape_i8(i8* %ptr)
+
+; ATTRIBUTOR:      @byval_not_readonly_1
+; ATTRIBUTOR-SAME: i8* byval %written
+define void @byval_not_readonly_1(i8* byval %written) readonly {
+  call void @escape_i8(i8* %written)
+  ret void
+}
+
+; ATTRIBUTOR:      @byval_not_readonly_2
+; ATTRIBUTOR-SAME: i8* nocapture nofree nonnull writeonly byval dereferenceable(1) %written
+define void @byval_not_readonly_2(i8* byval %written) readonly {
+  store i8 0, i8* %written
+  ret void
+}
+
+; ATTRIBUTOR:      @byval_not_readnone_1
+; ATTRIBUTOR-SAME: i8* byval %written
+define void @byval_not_readnone_1(i8* byval %written) readnone {
+  call void @escape_i8(i8* %written)
+  ret void
+}
+
+; ATTRIBUTOR:      @byval_not_readnone_2
+; ATTRIBUTOR-SAME: i8* nocapture nofree nonnull writeonly byval dereferenceable(1) %written
+define void @byval_not_readnone_2(i8* byval %written) readnone {
+  store i8 0, i8* %written
+  ret void
+}
+
+; ATTRIBUTOR:      @byval_no_fnarg
+; ATTRIBUTOR-SAME: i8* nocapture nofree nonnull writeonly byval dereferenceable(1) %written
+define void @byval_no_fnarg(i8* byval %written) {
+  store i8 0, i8* %written
+  ret void
+}
+
+; ATTRIBUTOR: @testbyval
+; ATTRIBUTOR-SAME: i8* nocapture readonly %read_only
+define void @testbyval(i8* %read_only) {
+  call void @byval_not_readonly_1(i8* %read_only)
+  call void @byval_not_readonly_2(i8* %read_only)
+  call void @byval_not_readnone_1(i8* %read_only)
+  call void @byval_not_readnone_2(i8* %read_only)
+  call void @byval_no_fnarg(i8* %read_only)
+  ret void
+}
+;}
diff --git a/llvm/test/Transforms/Attributor/returned.ll b/llvm/test/Transforms/Attributor/returned.ll
new file mode 100644
index 000000000..85ab69a0d
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/returned.ll
@@ -0,0 +1,812 @@
+; RUN: opt -attributor -attributor-manifest-internal -attributor-disable=false -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=5 -S < %s | FileCheck %s --check-prefix=ATTRIBUTOR
+; RUN: opt -attributor -attributor-manifest-internal -attributor-disable=false -attributor-annotate-decl-cs -functionattrs -S < %s | FileCheck %s --check-prefix=BOTH
+;
+; Copied from Transforms/FunctoinAttrs/read_write_returned_arguments_scc.ll
+; 
+; Test cases specifically designed for the "returned" argument attribute.
+; We use FIXME's to indicate problems and missing attributes.
+;
+
+; TEST SCC test returning an integer value argument
+;
+; BOTH: Function Attrs: nofree noinline norecurse nosync nounwind readnone uwtable
+; BOTH-NEXT: define i32 @sink_r0(i32 returned %r)
+; BOTH: Function Attrs: nofree noinline nosync nounwind readnone uwtable
+; BOTH-NEXT: define i32 @scc_r1(i32 %a, i32 returned %r, i32 %b)
+; BOTH: Function Attrs: nofree noinline nosync nounwind readnone uwtable
+; BOTH-NEXT: define i32 @scc_r2(i32 %a, i32 %b, i32 returned %r)
+; BOTH: Function Attrs: nofree noinline nosync nounwind readnone uwtable
+; BOTH-NEXT: define i32 @scc_rX(i32 %a, i32 %b, i32 %r)
+;
+;
+; ATTRIBUTOR: define i32 @sink_r0(i32 returned %r)
+; ATTRIBUTOR: define i32 @scc_r1(i32 %a, i32 returned %r, i32 %b)
+; ATTRIBUTOR: define i32 @scc_r2(i32 %a, i32 %b, i32 returned %r)
+; ATTRIBUTOR: define i32 @scc_rX(i32 %a, i32 %b, i32 %r)
+;
+; int scc_r1(int a, int b, int r);
+; int scc_r2(int a, int b, int r);
+;
+; __attribute__((noinline)) int sink_r0(int r) {
+;   return r;
+; }
+;
+; __attribute__((noinline)) int scc_r1(int a, int r, int b) {
+;   return scc_r2(r, a, sink_r0(r));
+; }
+;
+; __attribute__((noinline)) int scc_r2(int a, int b, int r) {
+;   if (a > b)
+;     return scc_r2(b, a, sink_r0(r));
+;   if (a < b)
+;     return scc_r1(sink_r0(b), scc_r2(scc_r1(a, b, r), scc_r1(a, scc_r2(r, r, r), r), scc_r2(a, b, r)), scc_r1(a, b, r));
+;   return a == b ? r : scc_r2(a, b, r);
+; }
+; __attribute__((noinline)) int scc_rX(int a, int b, int r) {
+;   if (a > b)
+;     return scc_r2(b, a, sink_r0(r));
+;   if (a < b)                                                                         // V Diff to scc_r2
+;     return scc_r1(sink_r0(b), scc_r2(scc_r1(a, b, r), scc_r1(a, scc_r2(r, r, r), r), scc_r1(a, b, r)), scc_r1(a, b, r));
+;   return a == b ? r : scc_r2(a, b, r);
+; }
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+define i32 @sink_r0(i32 %r) #0 {
+entry:
+  ret i32 %r
+}
+
+define i32 @scc_r1(i32 %a, i32 %r, i32 %b) #0 {
+entry:
+  %call = call i32 @sink_r0(i32 %r)
+  %call1 = call i32 @scc_r2(i32 %r, i32 %a, i32 %call)
+  ret i32 %call1
+}
+
+define i32 @scc_r2(i32 %a, i32 %b, i32 %r) #0 {
+entry:
+  %cmp = icmp sgt i32 %a, %b
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %call = call i32 @sink_r0(i32 %r)
+  %call1 = call i32 @scc_r2(i32 %b, i32 %a, i32 %call)
+  br label %return
+
+if.end:                                           ; preds = %entry
+  %cmp2 = icmp slt i32 %a, %b
+  br i1 %cmp2, label %if.then3, label %if.end12
+
+if.then3:                                         ; preds = %if.end
+  %call4 = call i32 @sink_r0(i32 %b)
+  %call5 = call i32 @scc_r1(i32 %a, i32 %b, i32 %r)
+  %call6 = call i32 @scc_r2(i32 %r, i32 %r, i32 %r)
+  %call7 = call i32 @scc_r1(i32 %a, i32 %call6, i32 %r)
+  %call8 = call i32 @scc_r2(i32 %a, i32 %b, i32 %r)
+  %call9 = call i32 @scc_r2(i32 %call5, i32 %call7, i32 %call8)
+  %call10 = call i32 @scc_r1(i32 %a, i32 %b, i32 %r)
+  %call11 = call i32 @scc_r1(i32 %call4, i32 %call9, i32 %call10)
+  br label %return
+
+if.end12:                                         ; preds = %if.end
+  %cmp13 = icmp eq i32 %a, %b
+  br i1 %cmp13, label %cond.true, label %cond.false
+
+cond.true:                                        ; preds = %if.end12
+  br label %cond.end
+
+cond.false:                                       ; preds = %if.end12
+  %call14 = call i32 @scc_r2(i32 %a, i32 %b, i32 %r)
+  br label %cond.end
+
+cond.end:                                         ; preds = %cond.false, %cond.true
+  %cond = phi i32 [ %r, %cond.true ], [ %call14, %cond.false ]
+  br label %return
+
+return:                                           ; preds = %cond.end, %if.then3, %if.then
+  %retval.0 = phi i32 [ %call1, %if.then ], [ %call11, %if.then3 ], [ %cond, %cond.end ]
+  ret i32 %retval.0
+}
+
+define i32 @scc_rX(i32 %a, i32 %b, i32 %r) #0 {
+entry:
+  %cmp = icmp sgt i32 %a, %b
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %call = call i32 @sink_r0(i32 %r)
+  %call1 = call i32 @scc_r2(i32 %b, i32 %a, i32 %call)
+  br label %return
+
+if.end:                                           ; preds = %entry
+  %cmp2 = icmp slt i32 %a, %b
+  br i1 %cmp2, label %if.then3, label %if.end12
+
+if.then3:                                         ; preds = %if.end
+  %call4 = call i32 @sink_r0(i32 %b)
+  %call5 = call i32 @scc_r1(i32 %a, i32 %b, i32 %r)
+  %call6 = call i32 @scc_r2(i32 %r, i32 %r, i32 %r)
+  %call7 = call i32 @scc_r1(i32 %a, i32 %call6, i32 %r)
+  %call8 = call i32 @scc_r1(i32 %a, i32 %b, i32 %r)
+  %call9 = call i32 @scc_r2(i32 %call5, i32 %call7, i32 %call8)
+  %call10 = call i32 @scc_r1(i32 %a, i32 %b, i32 %r)
+  %call11 = call i32 @scc_r1(i32 %call4, i32 %call9, i32 %call10)
+  br label %return
+
+if.end12:                                         ; preds = %if.end
+  %cmp13 = icmp eq i32 %a, %b
+  br i1 %cmp13, label %cond.true, label %cond.false
+
+cond.true:                                        ; preds = %if.end12
+  br label %cond.end
+
+cond.false:                                       ; preds = %if.end12
+  %call14 = call i32 @scc_r2(i32 %a, i32 %b, i32 %r)
+  br label %cond.end
+
+cond.end:                                         ; preds = %cond.false, %cond.true
+  %cond = phi i32 [ %r, %cond.true ], [ %call14, %cond.false ]
+  br label %return
+
+return:                                           ; preds = %cond.end, %if.then3, %if.then
+  %retval.0 = phi i32 [ %call1, %if.then ], [ %call11, %if.then3 ], [ %cond, %cond.end ]
+  ret i32 %retval.0
+}
+
+
+; TEST SCC test returning a pointer value argument
+;
+;
+; ATTRIBUTOR: Function Attrs: nofree noinline nosync nounwind readnone uwtable
+; ATTRIBUTOR-NEXT: define double* @ptr_sink_r0(double* nofree readnone returned "no-capture-maybe-returned" %r)
+; ATTRIBUTOR: Function Attrs: nofree noinline nosync nounwind readnone uwtable
+; ATTRIBUTOR-NEXT: define double* @ptr_scc_r1(double* nofree readnone %a, double* nofree readnone returned %r, double* nocapture nofree readnone %b)
+; ATTRIBUTOR: Function Attrs: nofree noinline nosync nounwind readnone uwtable
+; ATTRIBUTOR-NEXT: define double* @ptr_scc_r2(double* nofree readnone %a, double* nofree readnone %b, double* nofree readnone returned %r)
+;
+; double* ptr_scc_r1(double* a, double* b, double* r);
+; double* ptr_scc_r2(double* a, double* b, double* r);
+;
+; __attribute__((noinline)) double* ptr_sink_r0(double* r) {
+;   return r;
+; }
+;
+; __attribute__((noinline)) double* ptr_scc_r1(double* a, double* r, double* b) {
+;   return ptr_scc_r2(r, a, ptr_sink_r0(r));
+; }
+;
+; __attribute__((noinline)) double* ptr_scc_r2(double* a, double* b, double* r) {
+;   if (a > b)
+;     return ptr_scc_r2(b, a, ptr_sink_r0(r));
+;   if (a < b)
+;     return ptr_scc_r1(ptr_sink_r0(b), ptr_scc_r2(ptr_scc_r1(a, b, r), ptr_scc_r1(a, ptr_scc_r2(r, r, r), r), ptr_scc_r2(a, b, r)), ptr_scc_r1(a, b, r));
+;   return a == b ? r : ptr_scc_r2(a, b, r);
+; }
+define double* @ptr_sink_r0(double* %r) #0 {
+entry:
+  ret double* %r
+}
+
+define double* @ptr_scc_r1(double* %a, double* %r, double* %b) #0 {
+entry:
+  %call = call double* @ptr_sink_r0(double* %r)
+  %call1 = call double* @ptr_scc_r2(double* %r, double* %a, double* %call)
+  ret double* %call1
+}
+
+define double* @ptr_scc_r2(double* %a, double* %b, double* %r) #0 {
+entry:
+  %cmp = icmp ugt double* %a, %b
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %call = call double* @ptr_sink_r0(double* %r)
+  %call1 = call double* @ptr_scc_r2(double* %b, double* %a, double* %call)
+  br label %return
+
+if.end:                                           ; preds = %entry
+  %cmp2 = icmp ult double* %a, %b
+  br i1 %cmp2, label %if.then3, label %if.end12
+
+if.then3:                                         ; preds = %if.end
+  %call4 = call double* @ptr_sink_r0(double* %b)
+  %call5 = call double* @ptr_scc_r1(double* %a, double* %b, double* %r)
+  %call6 = call double* @ptr_scc_r2(double* %r, double* %r, double* %r)
+  %call7 = call double* @ptr_scc_r1(double* %a, double* %call6, double* %r)
+  %call8 = call double* @ptr_scc_r2(double* %a, double* %b, double* %r)
+  %call9 = call double* @ptr_scc_r2(double* %call5, double* %call7, double* %call8)
+  %call10 = call double* @ptr_scc_r1(double* %a, double* %b, double* %r)
+  %call11 = call double* @ptr_scc_r1(double* %call4, double* %call9, double* %call10)
+  br label %return
+
+if.end12:                                         ; preds = %if.end
+  %cmp13 = icmp eq double* %a, %b
+  br i1 %cmp13, label %cond.true, label %cond.false
+
+cond.true:                                        ; preds = %if.end12
+  br label %cond.end
+
+cond.false:                                       ; preds = %if.end12
+  %call14 = call double* @ptr_scc_r2(double* %a, double* %b, double* %r)
+  br label %cond.end
+
+cond.end:                                         ; preds = %cond.false, %cond.true
+  %cond = phi double* [ %r, %cond.true ], [ %call14, %cond.false ]
+  br label %return
+
+return:                                           ; preds = %cond.end, %if.then3, %if.then
+  %retval.0 = phi double* [ %call1, %if.then ], [ %call11, %if.then3 ], [ %cond, %cond.end ]
+  ret double* %retval.0
+}
+
+
+; TEST a no-return singleton SCC
+;
+; int* rt0(int *a) {
+;   return *a ? a : rt0(a);
+; }
+;
+; BOTH:      Function Attrs: nofree noinline norecurse noreturn nosync nounwind readonly uwtable
+; BOTH-NEXT: define noalias nonnull align 536870912 dereferenceable(4294967295) i32* @rt0(i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) %a)
+define i32* @rt0(i32* %a) #0 {
+entry:
+  %v = load i32, i32* %a, align 4
+  %tobool = icmp ne i32 %v, 0
+  %call = call i32* @rt0(i32* %a)
+  %sel = select i1 %tobool, i32* %a, i32* %call
+  ret i32* %sel
+}
+
+; TEST a no-return singleton SCC
+;
+; int* rt1(int *a) {
+;   return *a ? undef : rt1(a);
+; }
+;
+; BOTH: Function Attrs: nofree noinline norecurse noreturn nosync nounwind readonly uwtable
+; BOTH-NEXT:    define noalias nonnull align 536870912 dereferenceable(4294967295) i32* @rt1(i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) %a)
+define i32* @rt1(i32* %a) #0 {
+entry:
+  %v = load i32, i32* %a, align 4
+  %tobool = icmp ne i32 %v, 0
+  %call = call i32* @rt1(i32* %a)
+  %sel = select i1 %tobool, i32* undef, i32* %call
+  ret i32* %sel
+}
+
+; TEST another SCC test
+;
+; BOTH:    define i32* @rt2_helper(i32* nofree readnone returned %a)
+; BOTH:    define i32* @rt2(i32* nofree readnone %a, i32* nofree readnone "no-capture-maybe-returned" %b)
+define i32* @rt2_helper(i32* %a) #0 {
+entry:
+  %call = call i32* @rt2(i32* %a, i32* %a)
+  ret i32* %call
+}
+
+define i32* @rt2(i32* %a, i32 *%b) #0 {
+entry:
+  %cmp = icmp eq i32* %a, null
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  %call = call i32* @rt2_helper(i32* %a)
+  br label %if.end
+
+if.end:
+  %sel = phi i32* [ %b, %entry], [%call, %if.then]
+  ret i32* %sel
+}
+
+; TEST another SCC test
+;
+; BOTH:    define i32* @rt3_helper(i32* nofree readnone %a, i32* nofree readnone returned "no-capture-maybe-returned" %b)
+; BOTH:    define i32* @rt3(i32* nofree readnone %a, i32* nofree readnone returned "no-capture-maybe-returned" %b)
+define i32* @rt3_helper(i32* %a, i32* %b) #0 {
+entry:
+  %call = call i32* @rt3(i32* %a, i32* %b)
+  ret i32* %call
+}
+
+define i32* @rt3(i32* %a, i32 *%b) #0 {
+entry:
+  %cmp = icmp eq i32* %a, null
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  %call = call i32* @rt3_helper(i32* %a, i32* %b)
+  br label %if.end
+
+if.end:
+  %sel = phi i32* [ %b, %entry], [%call, %if.then]
+  ret i32* %sel
+}
+
+; TEST address taken function with call to an external functions
+;
+;  void unknown_fn(void *);
+;
+;  int* calls_unknown_fn(int *r) {
+;    unknown_fn(&calls_unknown_fn);
+;    return r;
+;  }
+;
+; BOTH: declare void @unknown_fn(i32* (i32*)*)
+;
+; BOTH:       Function Attrs: noinline nounwind uwtable
+; BOTH-NEXT:  define i32* @calls_unknown_fn(i32* readnone returned "no-capture-maybe-returned" %r)
+; ATTRIBUTOR: define i32* @calls_unknown_fn(i32* readnone returned "no-capture-maybe-returned" %r)
+declare void @unknown_fn(i32* (i32*)*) #0
+
+define i32* @calls_unknown_fn(i32* %r) #0 {
+  tail call void @unknown_fn(i32* (i32*)* nonnull @calls_unknown_fn)
+  ret i32* %r
+}
+
+
+; TEST call to a function that might be redifined at link time
+;
+;  int *maybe_redefined_fn(int *r) {
+;    return r;
+;  }
+;
+;  int *calls_maybe_redefined_fn(int *r) {
+;    maybe_redefined_fn(r);
+;    return r;
+;  }
+;
+; Verify the maybe-redefined function is not annotated:
+;
+; ATTRIBUTOR: Function Attrs: noinline nounwind uwtable
+; ATTRIBUTOR: define linkonce_odr i32* @maybe_redefined_fn(i32* %r)
+;
+; ATTRIBUTOR: Function Attrs: noinline nounwind uwtable
+; ATTRIBUTOR: define i32* @calls_maybe_redefined_fn(i32* returned %r)
+;
+; BOTH: Function Attrs: noinline nounwind uwtable
+; BOTH-NEXT: define linkonce_odr i32* @maybe_redefined_fn(i32* %r)
+;
+; BOTH: Function Attrs: noinline nounwind uwtable
+; BOTH-NEXT: define i32* @calls_maybe_redefined_fn(i32* returned %r)
+define linkonce_odr i32* @maybe_redefined_fn(i32* %r) #0 {
+entry:
+  ret i32* %r
+}
+
+define i32* @calls_maybe_redefined_fn(i32* %r) #0 {
+entry:
+  %call = call i32* @maybe_redefined_fn(i32* %r)
+  ret i32* %r
+}
+
+; TEST return call to a function that might be redifined at link time
+;
+;  int *maybe_redefined_fn2(int *r) {
+;    return r;
+;  }
+;
+;  int *calls_maybe_redefined_fn2(int *r) {
+;    return maybe_redefined_fn2(r);
+;  }
+;
+; Verify the maybe-redefined function is not annotated:
+;
+; BOTH: Function Attrs: noinline nounwind uwtable
+; BOTH-NEXT: define linkonce_odr i32* @maybe_redefined_fn2(i32* %r)
+; BOTH: Function Attrs: noinline nounwind uwtable
+; BOTH-NEXT: define i32* @calls_maybe_redefined_fn2(i32* %r)
+;
+; ATTRIBUTOR: define i32* @calls_maybe_redefined_fn2(i32* %r)
+define linkonce_odr i32* @maybe_redefined_fn2(i32* %r) #0 {
+entry:
+  ret i32* %r
+}
+
+define i32* @calls_maybe_redefined_fn2(i32* %r) #0 {
+entry:
+  %call = call i32* @maybe_redefined_fn2(i32* %r)
+  ret i32* %call
+}
+
+
+; TEST returned argument goes through select and phi
+;
+; double select_and_phi(double b) {
+;   double x = b;
+;   if (b > 0)
+;     x = b;
+;   return b == 0? b : x;
+; }
+;
+; BOTH: Function Attrs: nofree noinline norecurse nosync nounwind readnone uwtable
+; BOTH-NEXT: define double @select_and_phi(double returned %b)
+;
+; ATTRIBUTOR: Function Attrs: nofree noinline nosync nounwind readnone uwtable
+; ATTRIBUTOR-NEXT: define double @select_and_phi(double returned %b)
+define double @select_and_phi(double %b) #0 {
+entry:
+  %cmp = fcmp ogt double %b, 0.000000e+00
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %entry
+  %phi = phi double [ %b, %if.then ], [ %b, %entry ]
+  %cmp1 = fcmp oeq double %b, 0.000000e+00
+  %sel = select i1 %cmp1, double %b, double %phi
+  ret double %sel
+}
+
+
+; TEST returned argument goes through recursion, select, and phi
+;
+; double recursion_select_and_phi(int a, double b) {
+;   double x = b;
+;   if (a-- > 0)
+;     x = recursion_select_and_phi(a, b);
+;   return b == 0? b : x;
+; }
+;
+; BOTH: Function Attrs: nofree noinline nosync nounwind readnone uwtable
+; BOTH-NEXT: define double @recursion_select_and_phi(i32 %a, double returned %b)
+;
+;
+; ATTRIBUTOR: Function Attrs: nofree noinline nosync nounwind readnone uwtable
+; ATTRIBUTOR-NEXT: define double @recursion_select_and_phi(i32 %a, double returned %b)
+define double @recursion_select_and_phi(i32 %a, double %b) #0 {
+entry:
+  %dec = add nsw i32 %a, -1
+  %cmp = icmp sgt i32 %a, 0
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %call = call double @recursion_select_and_phi(i32 %dec, double %b)
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %entry
+  %phi = phi double [ %call, %if.then ], [ %b, %entry ]
+  %cmp1 = fcmp oeq double %b, 0.000000e+00
+  %sel = select i1 %cmp1, double %b, double %phi
+  ret double %sel
+}
+
+
+; TEST returned argument goes through bitcasts
+;
+; double* bitcast(int* b) {
+;   return (double*)b;
+; }
+;
+; BOTH: Function Attrs: nofree noinline norecurse nosync nounwind readnone uwtable
+; BOTH-NEXT:  define double* @bitcast(i32* nofree readnone returned "no-capture-maybe-returned" %b)
+;
+;
+; ATTRIBUTOR: Function Attrs: nofree noinline nosync nounwind readnone uwtable
+; ATTRIBUTOR-NEXT: define double* @bitcast(i32* nofree readnone returned "no-capture-maybe-returned" %b)
+define double* @bitcast(i32* %b) #0 {
+entry:
+  %bc0 = bitcast i32* %b to double*
+  ret double* %bc0
+}
+
+
+; TEST returned argument goes through select and phi interleaved with bitcasts
+;
+; double* bitcasts_select_and_phi(int* b) {
+;   double* x = b;
+;   if (b == 0)
+;     x = b;
+;   return b != 0 ? b : x;
+; }
+;
+; BOTH: Function Attrs: nofree noinline norecurse nosync nounwind readnone uwtable
+; BOTH-NEXT: define double* @bitcasts_select_and_phi(i32* nofree readnone returned %b)
+;
+;
+; ATTRIBUTOR: Function Attrs: nofree noinline nosync nounwind readnone uwtable
+; ATTRIBUTOR-NEXT: define double* @bitcasts_select_and_phi(i32* nofree readnone returned %b)
+define double* @bitcasts_select_and_phi(i32* %b) #0 {
+entry:
+  %bc0 = bitcast i32* %b to double*
+  %cmp = icmp eq double* %bc0, null
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %bc1 = bitcast i32* %b to double*
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %entry
+  %phi = phi double* [ %bc1, %if.then ], [ %bc0, %entry ]
+  %bc2 = bitcast double* %phi to i8*
+  %bc3 = bitcast i32* %b to i8*
+  %cmp2 = icmp ne double* %bc0, null
+  %sel = select i1 %cmp2, i8* %bc2, i8* %bc3
+  %bc4 = bitcast i8* %sel to double*
+  ret double* %bc4
+}
+
+
+; TEST return argument or argument or undef
+;
+; double* ret_arg_arg_undef(int* b) {
+;   if (b == 0)
+;     return (double*)b;
+;   if (b == 0)
+;     return (double*)b;
+;   /* return undef */
+; }
+;
+; BOTH: Function Attrs: nofree noinline norecurse nosync nounwind readnone uwtable
+; BOTH-NEXT:  define double* @ret_arg_arg_undef(i32* nofree readnone returned %b)
+;
+;
+; ATTRIBUTOR: Function Attrs: nofree noinline nosync nounwind readnone uwtable
+; ATTRIBUTOR-NEXT: define double* @ret_arg_arg_undef(i32* nofree readnone returned %b)
+define double* @ret_arg_arg_undef(i32* %b) #0 {
+entry:
+  %bc0 = bitcast i32* %b to double*
+  %cmp = icmp eq double* %bc0, null
+  br i1 %cmp, label %ret_arg0, label %if.end
+
+ret_arg0:
+  %bc1 = bitcast i32* %b to double*
+  ret double* %bc1
+
+if.end:
+  br i1 %cmp, label %ret_arg1, label %ret_undef
+
+ret_arg1:
+  ret double* %bc0
+
+ret_undef:
+  ret double *undef
+}
+
+
+; TEST return undef or argument or argument
+;
+; double* ret_undef_arg_arg(int* b) {
+;   if (b == 0)
+;     return (double*)b;
+;   if (b == 0)
+;     return (double*)b;
+;   /* return undef */
+; }
+;
+; BOTH: Function Attrs: nofree noinline norecurse nosync nounwind readnone uwtable
+; BOTH-NEXT:  define double* @ret_undef_arg_arg(i32* nofree readnone returned %b)
+;
+;
+; ATTRIBUTOR: Function Attrs: nofree noinline nosync nounwind readnone uwtable
+; ATTRIBUTOR-NEXT: define double* @ret_undef_arg_arg(i32* nofree readnone returned %b)
+define double* @ret_undef_arg_arg(i32* %b) #0 {
+entry:
+  %bc0 = bitcast i32* %b to double*
+  %cmp = icmp eq double* %bc0, null
+  br i1 %cmp, label %ret_undef, label %if.end
+
+ret_undef:
+  ret double *undef
+
+if.end:
+  br i1 %cmp, label %ret_arg0, label %ret_arg1
+
+ret_arg0:
+  ret double* %bc0
+
+ret_arg1:
+  %bc1 = bitcast i32* %b to double*
+  ret double* %bc1
+}
+
+
+; TEST return undef or argument or undef
+;
+; double* ret_undef_arg_undef(int* b) {
+;   if (b == 0)
+;     /* return undef */
+;   if (b == 0)
+;     return (double*)b;
+;   /* return undef */
+; }
+;
+; BOTH: Function Attrs: nofree noinline norecurse nosync nounwind readnone uwtable
+; BOTH-NEXT:  define double* @ret_undef_arg_undef(i32* nofree readnone returned %b)
+;
+; ATTRIBUTOR: define double* @ret_undef_arg_undef(i32* nofree readnone returned %b)
+define double* @ret_undef_arg_undef(i32* %b) #0 {
+entry:
+  %bc0 = bitcast i32* %b to double*
+  %cmp = icmp eq double* %bc0, null
+  br i1 %cmp, label %ret_undef0, label %if.end
+
+ret_undef0:
+  ret double *undef
+
+if.end:
+  br i1 %cmp, label %ret_arg, label %ret_undef1
+
+ret_arg:
+  ret double* %bc0
+
+ret_undef1:
+  ret double *undef
+}
+
+; TEST return argument or unknown call result
+;
+; int* ret_arg_or_unknown(int* b) {
+;   if (b == 0)
+;     return b;
+;   return unknown();
+; }
+;
+; Verify we do not assume b is returned
+;
+; ATTRIBUTOR: define i32* @ret_arg_or_unknown(i32* %b)
+; ATTRIBUTOR: define i32* @ret_arg_or_unknown_through_phi(i32* %b)
+; BOTH:       define i32* @ret_arg_or_unknown(i32* %b)
+; BOTH:       define i32* @ret_arg_or_unknown_through_phi(i32* %b)
+declare i32* @unknown(i32*)
+
+define i32* @ret_arg_or_unknown(i32* %b) #0 {
+entry:
+  %cmp = icmp eq i32* %b, null
+  br i1 %cmp, label %ret_arg, label %ret_unknown
+
+ret_arg:
+  ret i32* %b
+
+ret_unknown:
+  %call = call i32* @unknown(i32* %b)
+  ret i32* %call
+}
+
+define i32* @ret_arg_or_unknown_through_phi(i32* %b) #0 {
+entry:
+  %cmp = icmp eq i32* %b, null
+  br i1 %cmp, label %ret_arg, label %ret_unknown
+
+ret_arg:
+  br label %r
+
+ret_unknown:
+  %call = call i32* @unknown(i32* %b)
+  br label %r
+
+r:
+  %phi = phi i32* [ %b, %ret_arg ], [ %call, %ret_unknown ]
+  ret i32* %phi
+}
+
+; TEST inconsistent IR in dead code.
+;
+; ATTRIBUTOR: define i32 @deadblockcall1(i32 returned %A)
+; ATTRIBUTOR: define i32 @deadblockcall2(i32 returned %A)
+; ATTRIBUTOR: define i32 @deadblockphi1(i32 returned %A)
+; ATTRIBUTOR: define i32 @deadblockphi2(i32 returned %A)
+; BOTH:       define i32 @deadblockcall1(i32 returned %A)
+; BOTH:       define i32 @deadblockcall2(i32 returned %A)
+; BOTH:       define i32 @deadblockphi1(i32 returned %A)
+; BOTH:       define i32 @deadblockphi2(i32 returned %A)
+define i32 @deadblockcall1(i32 %A) #0 {
+entry:
+  ret i32 %A
+unreachableblock:
+  %B = call i32 @deadblockcall1(i32 %B)
+  ret i32 %B
+}
+
+declare i32 @deadblockcall_helper(i32 returned %A);
+
+define i32 @deadblockcall2(i32 %A) #0 {
+entry:
+  ret i32 %A
+unreachableblock1:
+  %B = call i32 @deadblockcall_helper(i32 %B)
+  ret i32 %B
+unreachableblock2:
+  %C = call i32 @deadblockcall1(i32 %C)
+  ret i32 %C
+}
+
+define i32 @deadblockphi1(i32 %A) #0 {
+entry:
+  br label %r
+unreachableblock1:
+  %B = call i32 @deadblockcall_helper(i32 %B)
+  ret i32 %B
+unreachableblock2:
+  %C = call i32 @deadblockcall1(i32 %C)
+  br label %r
+r:
+  %PHI = phi i32 [%A, %entry], [%C, %unreachableblock2]
+  ret i32 %PHI
+}
+
+define i32 @deadblockphi2(i32 %A) #0 {
+entry:
+  br label %r
+unreachableblock1:
+  %B = call i32 @deadblockcall_helper(i32 %B)
+  br label %unreachableblock3
+unreachableblock2:
+  %C = call i32 @deadblockcall1(i32 %C)
+  br label %unreachableblock3
+unreachableblock3:
+  %PHI1 = phi i32 [%B, %unreachableblock1], [%C, %unreachableblock2]
+  br label %r
+r:
+  %PHI2 = phi i32 [%A, %entry], [%PHI1, %unreachableblock3]
+  ret i32 %PHI2
+}
+
+declare void @noreturn() noreturn;
+
+define i32 @deadblockphi3(i32 %A, i1 %c) #0 {
+entry:
+  br i1 %c, label %r, label %unreachablecall
+unreachablecall:
+  call void @noreturn();
+  %B = call i32 @deadblockcall_helper(i32 0)
+  br label %unreachableblock3
+unreachableblock2:
+  %C = call i32 @deadblockcall1(i32 %C)
+  br label %unreachableblock3
+unreachableblock3:
+  %PHI1 = phi i32 [%B, %unreachablecall], [%C, %unreachableblock2]
+  br label %r
+r:
+  %PHI2 = phi i32 [%A, %entry], [%PHI1, %unreachableblock3]
+  ret i32 %PHI2
+}
+
+define weak_odr i32 @non_exact_0() {
+  ret i32 0
+}
+define weak_odr i32 @non_exact_1(i32 %a) {
+  ret i32 %a
+}
+define weak_odr i32 @non_exact_2(i32 returned %a) {
+  ret i32 %a
+}
+define weak_odr i32* @non_exact_3(i32* align 32 returned %a) {
+  ret i32* %a
+}
+define i32 @exact(i32* %a) {
+  %c0 = call i32 @non_exact_0()
+  %c1 = call i32 @non_exact_1(i32 1)
+  %c2 = call i32 @non_exact_2(i32 2)
+  %c3 = call i32* @non_exact_3(i32* %a)
+; We can use the information of the weak function non_exact_3 because it was
+; given to us and not derived (the alignment of the returned argument).
+; ATTRIBUTOR:  %c4 = load i32, i32* %c3, align 32
+  %c4 = load i32, i32* %c3
+; FIXME: %c2 and %c3 should be replaced but not %c0 or %c1!
+; ATTRIBUTOR:  %add1 = add i32 %c0, %c1
+; ATTRIBUTOR:  %add2 = add i32 %add1, %c2
+; ATTRIBUTOR:  %add3 = add i32 %add2, %c4
+  %add1 = add i32 %c0, %c1
+  %add2 = add i32 %add1, %c2
+  %add3 = add i32 %add2, %c4
+  ret i32 %add3
+}
+
+@G = external global i8
+define i32* @ret_const() #0 {
+  %bc = bitcast i8* @G to i32*
+  ret i32* %bc
+}
+define i32* @use_const() #0 {
+  %c = call i32* @ret_const()
+  ; ATTRIBUTOR: ret i32* bitcast (i8* @G to i32*)
+  ret i32* %c
+}
+define i32* @dont_use_const() #0 {
+  %c = musttail call i32* @ret_const()
+  ; ATTRIBUTOR: ret i32* %c
+  ret i32* %c
+}
+
+attributes #0 = { noinline nounwind uwtable }
diff --git a/llvm/test/Transforms/Attributor/undefined_behavior.ll b/llvm/test/Transforms/Attributor/undefined_behavior.ll
new file mode 100644
index 000000000..fd0ddb1eb
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/undefined_behavior.ll
@@ -0,0 +1,305 @@
+; RUN: opt --attributor --attributor-disable=false -S < %s | FileCheck %s --check-prefix=ATTRIBUTOR
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+; Test cases specifically designed for the "undefined behavior" abstract function attribute.
+; We want to verify that whenever undefined behavior is assumed, the code becomes unreachable.
+; We use FIXME's to indicate problems and missing attributes.
+
+; -- Load tests --
+
+define void @load_wholly_unreachable() {
+; ATTRIBUTOR-LABEL: @load_wholly_unreachable(
+; ATTRIBUTOR-NEXT:    unreachable
+;
+  %a = load i32, i32* null
+  ret void
+}
+
+define void @loads_wholly_unreachable() {
+; ATTRIBUTOR-LABEL: @loads_wholly_unreachable(
+; ATTRIBUTOR-NEXT:    unreachable
+;
+  %a = load i32, i32* null
+  %b = load i32, i32* null
+  ret void
+}
+
+
+define void @load_single_bb_unreachable(i1 %cond) {
+; ATTRIBUTOR-LABEL: @load_single_bb_unreachable(
+; ATTRIBUTOR-NEXT:    br i1 [[COND:%.*]], label [[T:%.*]], label [[E:%.*]]
+; ATTRIBUTOR:       t:
+; ATTRIBUTOR-NEXT:    unreachable
+; ATTRIBUTOR:       e:
+; ATTRIBUTOR-NEXT:    ret void
+;
+  br i1 %cond, label %t, label %e
+t:
+  %b = load i32, i32* null
+  br label %e
+e:
+  ret void
+}
+
+define void @load_null_pointer_is_defined() "null-pointer-is-valid"="true" {
+; ATTRIBUTOR-LABEL: @load_null_pointer_is_defined(
+; ATTRIBUTOR-NEXT:    [[A:%.*]] = load i32, i32* null
+; ATTRIBUTOR-NEXT:    ret void
+;
+  %a = load i32, i32* null
+  ret void
+}
+
+define internal i32* @ret_null() {
+  ret i32* null
+}
+
+; FIXME: null is propagated but the instruction
+; is not changed to unreachable.
+define void @load_null_propagated() {
+; ATTRIBUTOR-LABEL: @load_null_propagated(
+; ATTRIBUTOR-NEXT:    [[A:%.*]] = load i32, i32* null
+; ATTRIBUTOR-NEXT:    ret void
+;
+  %ptr = call i32* @ret_null()
+  %a = load i32, i32* %ptr
+  ret void
+}
+
+; -- Store tests --
+
+define void @store_wholly_unreachable() {
+; ATTRIBUTOR-LABEL: @store_wholly_unreachable(
+; ATTRIBUTOR-NEXT:    unreachable
+;
+  store i32 5, i32* null
+  ret void
+}
+
+define void @store_single_bb_unreachable(i1 %cond) {
+; ATTRIBUTOR-LABEL: @store_single_bb_unreachable(
+; ATTRIBUTOR-NEXT:    br i1 [[COND:%.*]], label [[T:%.*]], label [[E:%.*]]
+; ATTRIBUTOR:       t:
+; ATTRIBUTOR-NEXT:    unreachable
+; ATTRIBUTOR:       e:
+; ATTRIBUTOR-NEXT:    ret void
+;
+  br i1 %cond, label %t, label %e
+t:
+  store i32 5, i32* null
+  br label %e
+e:
+  ret void
+}
+
+define void @store_null_pointer_is_defined() "null-pointer-is-valid"="true" {
+; ATTRIBUTOR-LABEL: @store_null_pointer_is_defined(
+; ATTRIBUTOR-NEXT:    store i32 5, i32* null
+; ATTRIBUTOR-NEXT:    ret void
+;
+  store i32 5, i32* null
+  ret void
+}
+
+; -- AtomicRMW tests --
+
+define void @atomicrmw_wholly_unreachable() {
+; ATTRIBUTOR-LABEL: @atomicrmw_wholly_unreachable(
+; ATTRIBUTOR-NEXT:    unreachable
+;
+  %a = atomicrmw add i32* null, i32 1 acquire
+  ret void
+}
+
+define void @atomicrmw_single_bb_unreachable(i1 %cond) {
+; ATTRIBUTOR-LABEL: @atomicrmw_single_bb_unreachable(
+; ATTRIBUTOR-NEXT:    br i1 [[COND:%.*]], label [[T:%.*]], label [[E:%.*]]
+; ATTRIBUTOR:       t:
+; ATTRIBUTOR-NEXT:    unreachable
+; ATTRIBUTOR:       e:
+; ATTRIBUTOR-NEXT:    ret void
+;
+  br i1 %cond, label %t, label %e
+t:
+  %a = atomicrmw add i32* null, i32 1 acquire
+  br label %e
+e:
+  ret void
+}
+
+define void @atomicrmw_null_pointer_is_defined() "null-pointer-is-valid"="true" {
+; ATTRIBUTOR-LABEL: @atomicrmw_null_pointer_is_defined(
+; ATTRIBUTOR-NEXT:    [[A:%.*]] = atomicrmw add i32* null, i32 1 acquire
+; ATTRIBUTOR-NEXT:    ret void
+;
+  %a = atomicrmw add i32* null, i32 1 acquire
+  ret void
+}
+
+; -- AtomicCmpXchg tests --
+
+define void @atomiccmpxchg_wholly_unreachable() {
+; ATTRIBUTOR-LABEL: @atomiccmpxchg_wholly_unreachable(
+; ATTRIBUTOR-NEXT:    unreachable
+;
+  %a = cmpxchg i32* null, i32 2, i32 3 acq_rel monotonic
+  ret void
+}
+
+define void @atomiccmpxchg_single_bb_unreachable(i1 %cond) {
+; ATTRIBUTOR-LABEL: @atomiccmpxchg_single_bb_unreachable(
+; ATTRIBUTOR-NEXT:    br i1 [[COND:%.*]], label [[T:%.*]], label [[E:%.*]]
+; ATTRIBUTOR:       t:
+; ATTRIBUTOR-NEXT:    unreachable
+; ATTRIBUTOR:       e:
+; ATTRIBUTOR-NEXT:    ret void
+;
+  br i1 %cond, label %t, label %e
+t:
+  %a = cmpxchg i32* null, i32 2, i32 3 acq_rel monotonic
+  br label %e
+e:
+  ret void
+}
+
+define void @atomiccmpxchg_null_pointer_is_defined() "null-pointer-is-valid"="true" {
+; ATTRIBUTOR-LABEL: @atomiccmpxchg_null_pointer_is_defined(
+; ATTRIBUTOR-NEXT:    [[A:%.*]] = cmpxchg i32* null, i32 2, i32 3 acq_rel monotonic
+; ATTRIBUTOR-NEXT:    ret void
+;
+  %a = cmpxchg i32* null, i32 2, i32 3 acq_rel monotonic
+  ret void
+}
+
+; Note: The unreachable on %t and %e is _not_ from AAUndefinedBehavior
+
+define i32 @cond_br_on_undef() {
+; ATTRIBUTOR-LABEL: @cond_br_on_undef(
+; ATTRIBUTOR-NEXT:    unreachable
+; ATTRIBUTOR:       t:
+; ATTRIBUTOR-NEXT:    unreachable
+; ATTRIBUTOR:       e:
+; ATTRIBUTOR-NEXT:    unreachable
+;
+
+  br i1 undef, label %t, label %e
+t:
+  ret i32 1
+e:
+  ret i32 2
+}
+
+; More complicated branching
+define void @cond_br_on_undef2(i1 %cond) {
+; ATTRIBUTOR-LABEL: @cond_br_on_undef2(
+; ATTRIBUTOR-NEXT:    br i1 [[COND:%.*]], label [[T1:%.*]], label [[E1:%.*]]
+; ATTRIBUTOR:       t1:
+; ATTRIBUTOR-NEXT:    unreachable
+; ATTRIBUTOR:       t2:
+; ATTRIBUTOR-NEXT:    unreachable
+; ATTRIBUTOR:       e2:
+; ATTRIBUTOR-NEXT:    unreachable
+; ATTRIBUTOR:       e1:
+; ATTRIBUTOR-NEXT:    ret void
+;
+
+  ; Valid branch - verify that this is not converted
+  ; to unreachable.
+  br i1 %cond, label %t1, label %e1
+t1:
+  br i1 undef, label %t2, label %e2
+t2:
+  ret void
+e2:
+  ret void
+e1:
+  ret void
+}
+
+define i1 @ret_undef() {
+  ret i1 undef
+}
+
+define void @cond_br_on_undef_interproc() {
+; ATTRIBUTOR-LABEL: @cond_br_on_undef_interproc(
+; ATTRIBUTOR-NEXT:    %cond = call i1 @ret_undef()
+; ATTRIBUTOR-NEXT:    unreachable
+; ATTRIBUTOR:       t:
+; ATTRIBUTOR-NEXT:    unreachable
+; ATTRIBUTOR:       e:
+; ATTRIBUTOR-NEXT:    unreachable
+  
+  %cond = call i1 @ret_undef()
+  br i1 %cond, label %t, label %e
+t:
+  ret void
+e:
+  ret void
+}
+
+define i1 @ret_undef2() {
+  br i1 true, label %t, label %e
+t:
+  ret i1 undef
+e:
+  ret i1 undef
+}
+
+; More complicated interproc deduction of undef
+define void @cond_br_on_undef_interproc2() {
+; ATTRIBUTOR-LABEL: @cond_br_on_undef_interproc2(
+; ATTRIBUTOR-NEXT:    %cond = call i1 @ret_undef2()
+; ATTRIBUTOR-NEXT:    unreachable
+; ATTRIBUTOR:       t:
+; ATTRIBUTOR-NEXT:    unreachable
+; ATTRIBUTOR:       e:
+; ATTRIBUTOR-NEXT:    unreachable
+  %cond = call i1 @ret_undef2()
+  br i1 %cond, label %t, label %e
+t:
+  ret void
+e:
+  ret void
+}
+
+; Branch on undef that depends on propagation of
+; undef of a previous instruction.
+; FIXME: Currently it doesn't propagate the undef.
+define i32 @cond_br_on_undef3() {
+; ATTRIBUTOR-LABEL: @cond_br_on_undef3(
+; ATTRIBUTOR-NEXT:    %cond = icmp ne i32 1, undef
+; ATTRIBUTOR-NEXT:    br i1 %cond, label %t, label %e
+; ATTRIBUTOR:       t:
+; ATTRIBUTOR-NEXT:    ret i32 1
+; ATTRIBUTOR:       e:
+; ATTRIBUTOR-NEXT:    ret i32 2
+
+  %cond = icmp ne i32 1, undef
+  br i1 %cond, label %t, label %e
+t:
+  ret i32 1
+e:
+  ret i32 2
+}
+
+; Branch on undef because of uninitialized value.
+; FIXME: Currently it doesn't propagate the undef.
+define i32 @cond_br_on_undef_uninit() {
+; ATTRIBUTOR-LABEL: @cond_br_on_undef_uninit(
+; ATTRIBUTOR-NEXT:    %alloc = alloca i1
+; ATTRIBUTOR-NEXT:    %cond = load i1, i1* %alloc
+; ATTRIBUTOR-NEXT:    br i1 %cond, label %t, label %e
+; ATTRIBUTOR:       t:
+; ATTRIBUTOR-NEXT:    ret i32 1
+; ATTRIBUTOR:       e:
+; ATTRIBUTOR-NEXT:    ret i32 2
+  
+  %alloc = alloca i1
+  %cond = load i1, i1* %alloc
+  br i1 %cond, label %t, label %e
+t:
+  ret i32 1
+e:
+  ret i32 2
+}
diff --git a/llvm/test/Transforms/Attributor/value-simplify.ll b/llvm/test/Transforms/Attributor/value-simplify.ll
new file mode 100644
index 000000000..05d4a50af
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/value-simplify.ll
@@ -0,0 +1,316 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; RUN: opt -attributor --attributor-disable=false -attributor-annotate-decl-cs -S < %s | FileCheck %s
+; TODO: Add max-iteration check
+
+; Disable update test checks and enable it where required.
+; UTC_ARGS: --turn off
+
+; ModuleID = 'value-simplify.ll'
+source_filename = "value-simplify.ll"
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+declare void @f(i32)
+
+; Test1: Replace argument with constant
+define internal void @test1(i32 %a) {
+; CHECK: tail call void @f(i32 1)
+  tail call void @f(i32 %a)
+  ret void
+}
+
+define void @test1_helper() {
+  tail call void @test1(i32 1)
+  ret void
+}
+
+; TEST 2 : Simplify return value
+define i32 @return0() {
+  ret i32 0
+}
+
+define i32 @return1() {
+  ret i32 1
+}
+
+; CHECK: define i32 @test2_1(i1 %c)
+define i32 @test2_1(i1 %c) {
+  br i1 %c, label %if.true, label %if.false
+if.true:
+  %call = tail call i32 @return0()
+  %ret0 = add i32 %call, 1
+  br label %end
+if.false:
+  %ret1 = tail call i32 @return1()
+  br label %end
+end:
+
+; CHECK: %ret = phi i32 [ %ret0, %if.true ], [ 1, %if.false ]
+  %ret = phi i32 [ %ret0, %if.true ], [ %ret1, %if.false ]
+
+; CHECK: ret i32 1
+  ret i32 1
+}
+
+
+
+; CHECK: define i32 @test2_2(i1 %c)
+define i32 @test2_2(i1 %c) {
+  %ret = tail call i32 @test2_1(i1 %c)
+; CHECK: ret i32 1
+  ret i32 %ret
+}
+
+declare void @use(i32)
+; CHECK: define void @test3(i1 %c)
+define void @test3(i1 %c) {
+  br i1 %c, label %if.true, label %if.false
+if.true:
+  br label %end
+if.false:
+  %ret1 = tail call i32 @return1()
+  br label %end
+end:
+
+; CHECK: %r = phi i32 [ 1, %if.true ], [ 1, %if.false ]
+  %r = phi i32 [ 1, %if.true ], [ %ret1, %if.false ]
+
+; CHECK: tail call void @use(i32 1)
+  tail call void @use(i32 %r)
+  ret void
+}
+
+define void @test-select-phi(i1 %c) {
+  %select-same = select i1 %c, i32 1, i32 1
+  ; CHECK: tail call void @use(i32 1)
+  tail call void @use(i32 %select-same)
+
+  %select-not-same = select i1 %c, i32 1, i32 0
+  ; CHECK: tail call void @use(i32 %select-not-same)
+  tail call void @use(i32 %select-not-same)
+  br i1 %c, label %if-true, label %if-false
+if-true:
+  br label %end
+if-false:
+  br label %end
+end:
+  %phi-same = phi i32 [ 1, %if-true ], [ 1, %if-false ]
+  %phi-not-same = phi i32 [ 0, %if-true ], [ 1, %if-false ]
+  %phi-same-prop = phi i32 [ 1, %if-true ], [ %select-same, %if-false ]
+  %phi-same-undef = phi i32 [ 1, %if-true ], [ undef, %if-false ]
+  %select-not-same-undef = select i1 %c, i32 %phi-not-same, i32 undef
+
+
+  ; CHECK: tail call void @use(i32 1)
+  tail call void @use(i32 %phi-same)
+
+  ; CHECK: tail call void @use(i32 %phi-not-same)
+  tail call void @use(i32 %phi-not-same)
+
+  ; CHECK: tail call void @use(i32 1)
+  tail call void @use(i32 %phi-same-prop)
+
+  ; CHECK: tail call void @use(i32 1)
+  tail call void @use(i32 %phi-same-undef)
+
+  ; CHECK: tail call void @use(i32 %select-not-same-undef)
+  tail call void @use(i32 %select-not-same-undef)
+
+  ret void
+
+}
+
+define i32 @ipccp1(i32 %a) {
+; CHECK-LABEL: define {{[^@]+}}@ipccp1
+; CHECK-SAME: (i32 returned [[A:%.*]])
+; CHECK-NEXT:    br i1 true, label [[T:%.*]], label [[F:%.*]]
+; CHECK:       t:
+; CHECK-NEXT:    ret i32 [[A:%.*]]
+; CHECK:       f:
+; CHECK-NEXT:    unreachable
+;
+  br i1 true, label %t, label %f
+t:
+  ret i32 %a
+f:
+  %r = call i32 @ipccp1(i32 5)
+  ret i32 %r
+}
+
+define internal i1 @ipccp2i(i1 %a) {
+; CHECK-LABEL: define {{[^@]+}}@ipccp2i
+; CHECK-SAME: (i1 returned [[A:%.*]])
+; CHECK-NEXT:    br label %t
+; CHECK:       t:
+; CHECK-NEXT:    ret i1 true
+; CHECK:       f:
+; CHECK-NEXT:    unreachable
+;
+  br i1 %a, label %t, label %f
+t:
+  ret i1 %a
+f:
+  %r = call i1 @ipccp2i(i1 false)
+  ret i1 %r
+}
+
+define i1 @ipccp2() {
+; CHECK-LABEL: define {{[^@]+}}@ipccp2()
+; CHECK-NEXT:    [[R:%.*]] = call i1 @ipccp2i(i1 true)
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %r = call i1 @ipccp2i(i1 true)
+  ret i1 %r
+}
+
+define internal i32 @ipccp3i(i32 %a) {
+; CHECK-LABEL: define {{[^@]+}}@ipccp3i
+; CHECK-SAME: (i32 returned [[A:%.*]])
+; CHECK-NEXT:    br label [[T:%.*]]
+; CHECK:       t:
+; CHECK-NEXT:    ret i32 7
+; CHECK:       f:
+; CHECK-NEXT:    unreachable
+;
+  %c = icmp eq i32 %a, 7
+  br i1 %c, label %t, label %f
+t:
+  ret i32 %a
+f:
+  %r = call i32 @ipccp3i(i32 5)
+  ret i32 %r
+}
+
+define i32 @ipccp3() {
+; CHECK-LABEL: define {{[^@]+}}@ipccp3()
+; CHECK-NEXT:    [[R:%.*]] = call i32 @ipccp3i(i32 7)
+; CHECK-NEXT:    ret i32 [[R]]
+; FIXME: R should be replaced with 7
+  %r = call i32 @ipccp3i(i32 7)
+  ret i32 %r
+}
+
+; UTC_ARGS: --turn on
+
+; Do not touch complicated arguments (for now)
+%struct.X = type { i8* }
+define internal i32* @test_inalloca(i32* inalloca %a) {
+; CHECK-LABEL: define {{[^@]+}}@test_inalloca
+; CHECK-SAME: (i32* inalloca noalias nofree returned writeonly [[A:%.*]])
+; CHECK-NEXT:    ret i32* [[A]]
+;
+  ret i32* %a
+}
+define i32* @complicated_args_inalloca() {
+; CHECK-LABEL: define {{[^@]+}}@complicated_args_inalloca()
+; CHECK-NEXT:    [[CALL:%.*]] = call i32* @test_inalloca(i32* noalias nofree writeonly null)
+; CHECK-NEXT:    ret i32* [[CALL]]
+;
+  %call = call i32* @test_inalloca(i32* null)
+  ret i32* %call
+}
+
+define internal void @test_sret(%struct.X* sret %a, %struct.X** %b) {
+; CHECK-LABEL: define {{[^@]+}}@test_sret
+; CHECK-SAME: (%struct.X* nofree sret writeonly [[A:%.*]], %struct.X** nocapture nofree nonnull writeonly dereferenceable(8) [[B:%.*]])
+; CHECK-NEXT:    store %struct.X* [[A]], %struct.X** [[B]]
+; CHECK-NEXT:    ret void
+;
+  store %struct.X* %a, %struct.X** %b
+  ret void
+}
+define void @complicated_args_sret(%struct.X** %b) {
+; CHECK-LABEL: define {{[^@]+}}@complicated_args_sret
+; CHECK-SAME: (%struct.X** nocapture nofree writeonly [[B:%.*]])
+; CHECK-NEXT:    call void @test_sret(%struct.X* nofree writeonly null, %struct.X** nocapture nofree writeonly [[B]])
+; CHECK-NEXT:    ret void
+;
+  call void @test_sret(%struct.X* null, %struct.X** %b)
+  ret void
+}
+
+define internal %struct.X* @test_nest(%struct.X* nest %a) {
+; CHECK-LABEL: define {{[^@]+}}@test_nest
+; CHECK-SAME: (%struct.X* nest noalias nofree readnone returned [[A:%.*]])
+; CHECK-NEXT:    ret %struct.X* [[A]]
+;
+  ret %struct.X* %a
+}
+define %struct.X* @complicated_args_nest() {
+; CHECK-LABEL: define {{[^@]+}}@complicated_args_nest()
+; CHECK-NEXT:    [[CALL:%.*]] = call %struct.X* @test_nest(%struct.X* noalias nofree readnone null)
+; CHECK-NEXT:    ret %struct.X* [[CALL]]
+;
+  %call = call %struct.X* @test_nest(%struct.X* null)
+  ret %struct.X* %call
+}
+
+@S = external global %struct.X
+define internal void @test_byval(%struct.X* byval %a) {
+; CHECK-LABEL: define {{[^@]+}}@test_byval
+; CHECK-SAME: (%struct.X* nocapture nofree nonnull writeonly byval align 8 dereferenceable(8) [[A:%.*]])
+; CHECK-NEXT:    [[G0:%.*]] = getelementptr [[STRUCT_X:%.*]], %struct.X* [[A]], i32 0, i32 0
+; CHECK-NEXT:    store i8* null, i8** [[G0]], align 8
+; CHECK-NEXT:    ret void
+;
+  %g0 = getelementptr %struct.X, %struct.X* %a, i32 0, i32 0
+  store i8* null, i8** %g0
+  ret void
+}
+define void @complicated_args_byval() {
+; CHECK-LABEL: define {{[^@]+}}@complicated_args_byval()
+; CHECK-NEXT:    call void @test_byval(%struct.X* nofree nonnull readonly align 8 dereferenceable(8) @S)
+; CHECK-NEXT:    ret void
+;
+  call void @test_byval(%struct.X* @S)
+  ret void
+}
+
+define void @fixpoint_changed(i32* %p) {
+; CHECK-LABEL: define {{[^@]+}}@fixpoint_changed
+; CHECK-SAME: (i32* nocapture nofree writeonly [[P:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_COND:%.*]]
+; CHECK:       for.cond:
+; CHECK-NEXT:    [[J_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[SW_EPILOG:%.*]] ]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[J_0]], 30
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    switch i32 [[J_0]], label [[SW_EPILOG]] [
+; CHECK-NEXT:    i32 1, label [[SW_BB:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       sw.bb:
+; CHECK-NEXT:    br label [[SW_EPILOG]]
+; CHECK:       sw.epilog:
+; CHECK-NEXT:    [[X_0:%.*]] = phi i32 [ 255, [[FOR_BODY]] ], [ 253, [[SW_BB]] ]
+; CHECK-NEXT:    store i32 [[X_0]], i32* [[P]]
+; CHECK-NEXT:    [[INC]] = add nsw i32 [[J_0]], 1
+; CHECK-NEXT:    br label [[FOR_COND]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %for.cond
+
+for.cond:
+  %j.0 = phi i32 [ 0, %entry ], [ %inc, %sw.epilog ]
+  %cmp = icmp slt i32 %j.0, 30
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:
+  switch i32 %j.0, label %sw.epilog [
+  i32 1, label %sw.bb
+  ]
+
+sw.bb:
+  br label %sw.epilog
+
+sw.epilog:
+  %x.0 = phi i32 [ 255, %for.body ], [ 253, %sw.bb ]
+  store i32 %x.0, i32* %p
+  %inc = add nsw i32 %j.0, 1
+  br label %for.cond
+
+for.end:
+  ret void
+}
+
+; UTC_ARGS: --turn off
diff --git a/llvm/test/Transforms/Attributor/willreturn.ll b/llvm/test/Transforms/Attributor/willreturn.ll
new file mode 100644
index 000000000..90fef48b2
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/willreturn.ll
@@ -0,0 +1,482 @@
+; RUN: opt -passes=attributor --attributor-disable=false -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=2 -S < %s | FileCheck %s --check-prefix=ATTRIBUTOR
+
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+; Test cases specifically designed for the "willreturn" function attribute.
+; We use FIXME's to indicate problems and missing attributes.
+
+
+; TEST 1 (positive case)
+; ATTRIBUTOR: Function Attrs: nofree noinline norecurse nosync nounwind readnone uwtable willreturn
+; ATTRIBUTOR-NEXT: define void @only_return()
+define void @only_return() #0 {
+    ret void
+}
+
+
+; TEST 2 (positive & negative case)
+; 2.1 (positive case)
+; recursive function which will halt
+; int fib(int n){
+;    return n<=1? n : fib(n-1) + fib(n-2);
+; }
+
+; FIXME: missing willreturn
+; ATTRIBUTOR: Function Attrs: nofree noinline nosync nounwind readnone uwtable
+; ATTRIBUTOR-NEXT: define i32 @fib(i32 %0) local_unnamed_addr
+define i32 @fib(i32 %0) local_unnamed_addr #0 {
+  %2 = icmp slt i32 %0, 2
+  br i1 %2, label %9, label %3
+
+; <label>:3:                                      ; preds = %1
+  %4 = add nsw i32 %0, -1
+  %5 = tail call i32 @fib(i32 %4)
+  %6 = add nsw i32 %0, -2
+  %7 = tail call i32 @fib(i32 %6)
+  %8 = add nsw i32 %7, %5
+  ret i32 %8
+
+; <label>:9:                                      ; preds = %1
+  ret i32 %0
+}
+
+; 2.2 (negative case)
+; recursive function which doesn't stop for some input.
+; int fact_maybe_not_halt(int n) {
+;   if (n==0) {
+;     return 1;
+;   }
+;   return fact_maybe_not_halt( n > 0 ? n-1 : n) * n;
+; }
+; fact_maybe_not(-1) doesn't stop.
+
+; ATTRIBUTOR: Function Attrs: nofree noinline norecurse nosync nounwind readnone uwtable
+; ATTRIBUTOR-NOT: willreturn
+; ATTRIBUTOR-NEXT: define i32 @fact_maybe_not_halt(i32 %0) local_unnamed_addr
+define i32 @fact_maybe_not_halt(i32 %0) local_unnamed_addr #0 {
+  %2 = icmp eq i32 %0, 0
+  br i1 %2, label %11, label %3
+
+; <label>:3:                                      ; preds = %1, %3
+  %4 = phi i32 [ %8, %3 ], [ %0, %1 ]
+  %5 = phi i32 [ %9, %3 ], [ 1, %1 ]
+  %6 = icmp sgt i32 %4, 0
+  %7 = sext i1 %6 to i32
+  %8 = add nsw i32 %4, %7
+  %9 = mul nsw i32 %4, %5
+  %10 = icmp eq i32 %8, 0
+  br i1 %10, label %11, label %3
+
+; <label>:11:                                     ; preds = %3, %1
+  %12 = phi i32 [ 1, %1 ], [ %9, %3 ]
+  ret i32 %12
+}
+
+
+; TEST 3 (positive case)
+; loop
+; int fact_loop(int n ){
+;   int ans = 1;
+;   for(int i = 1;i<=n;i++){
+;     ans *= i;
+;   }
+;   return ans;
+; }
+
+; FIXME: missing willreturn
+; ATTRIBUTOR: Function Attrs: nofree noinline norecurse nosync nounwind readnone uwtable
+; ATTRIBUTOR-NEXT: define i32 @fact_loop(i32 %0) local_unnamed_addr
+define i32 @fact_loop(i32 %0) local_unnamed_addr #0 {
+  %2 = icmp slt i32 %0, 1
+  br i1 %2, label %3, label %5
+
+; <label>:3:                                      ; preds = %5, %1
+  %4 = phi i32 [ 1, %1 ], [ %8, %5 ]
+  ret i32 %4
+
+; <label>:5:                                      ; preds = %1, %5
+  %6 = phi i32 [ %9, %5 ], [ 1, %1 ]
+  %7 = phi i32 [ %8, %5 ], [ 1, %1 ]
+  %8 = mul nsw i32 %6, %7
+  %9 = add nuw nsw i32 %6, 1
+  %10 = icmp eq i32 %6, %0
+  br i1 %10, label %3, label %5
+}
+
+; TEST 4 (negative case)
+; mutual recursion
+; void mutual_recursion1(){
+;    mutual_recursion2();
+; }
+; void mutual_recursion2(){
+;     mutual_recursion1();
+; }
+
+; ATTRIBUTOR: Function Attrs: nofree noinline nosync nounwind readnone uwtable
+; ATTRIBUTOR-NOT: willreturn
+; ATTRIBUTOR-NEXT: define void @mutual_recursion1(i1 %c)
+define void @mutual_recursion1(i1 %c) #0 {
+  br i1 %c, label %rec, label %end
+rec:
+  call void @mutual_recursion2(i1 %c)
+  br label %end
+end:
+  ret void
+}
+
+
+; ATTRIBUTOR: Function Attrs: nofree noinline nosync nounwind readnone uwtable
+; ATTRIBUTOR-NOT: willreturn
+; ATTRIBUTOR-NEXT: define void @mutual_recursion2(i1 %c)
+define void @mutual_recursion2(i1 %c) #0 {
+  call void @mutual_recursion1(i1 %c)
+  ret void
+}
+
+
+; TEST 5 (negative case)
+; call exit/abort (has noreturn attribute)
+; ATTRIBUTOR: Function Attrs: noreturn
+; ATTRIBUTOR-NEXT: declare void @exit(i32) local_unnamed_add
+declare void @exit(i32 %0) local_unnamed_addr noreturn
+
+; ATTRIBUTOR: Function Attrs: noinline noreturn nounwind uwtable
+; ATTRIBUTOR-NOT: willreturn
+; ATTRIBUTOR-NEXT: define void @only_exit() local_unnamed_addr
+define void @only_exit() local_unnamed_addr #0 {
+  tail call void @exit(i32 0)
+  unreachable
+}
+
+; conditional exit
+; void conditional_exit(int cond, int *p){
+;     if(cond){
+;       exit(0);
+;     }
+;     if(*p){
+;       exit(1);
+;     }
+;     return;
+; }
+; ATTRIBUTOR: Function Attrs: noinline nounwind uwtable
+; ATTRIBUTOR-NOT: willreturn
+; ATTRIBUTOR-NEXT: define void @conditional_exit(i32 %0, i32* nocapture readonly %1) local_unnamed_addr
+define void @conditional_exit(i32 %0, i32* nocapture readonly %1) local_unnamed_addr #0 {
+  %3 = icmp eq i32 %0, 0
+  br i1 %3, label %5, label %4
+
+; <label>:4:                                      ; preds = %2
+  tail call void @exit(i32 0)
+  unreachable
+
+; <label>:5:                                      ; preds = %2
+  %6 = load i32, i32* %1, align 4
+  %7 = icmp eq i32 %6, 0
+  br i1 %7, label %9, label %8
+
+; <label>:8:                                      ; preds = %5
+  tail call void @exit(i32 1)
+  unreachable
+
+; <label>:9:                                      ; preds = %5
+  ret void
+}
+
+; TEST 6 (positive case)
+; Call intrinsic function
+; FIXME: missing willreturn
+; ATTRIBUTOR: Function Attrs: nounwind readnone speculatable
+; ATTRIBUTOR-NEXT: declare float @llvm.floor.f32(float)
+declare float @llvm.floor.f32(float)
+
+; FIXME: missing willreturn
+; ATTRIBUTOR: Function Attrs: noinline nosync nounwind readnone uwtable
+; ATTRIBUTOR-NEXT: define void @call_floor(float %a)
+define void @call_floor(float %a) #0 {
+    tail call float @llvm.floor.f32(float %a)
+    ret void
+}
+
+
+; TEST 7 (negative case)
+; Call function declaration without willreturn
+
+; ATTRIBUTOR: Function Attrs: noinline nounwind uwtable
+; ATTRIBUTOR-NOT: willreturn
+; ATTRIBUTOR-NEXT: declare void @maybe_noreturn()
+declare void @maybe_noreturn() #0
+
+; ATTRIBUTOR: Function Attrs: noinline nounwind uwtable
+; ATTRIBUTOR-NOT: willreturn
+; ATTRIBUTOR-NEXT: define void @call_maybe_noreturn()
+define void @call_maybe_noreturn() #0 {
+    tail call void @maybe_noreturn()
+    ret void
+}
+
+
+; TEST 8 (positive case)
+; Check propagation.
+
+; ATTRIBUTOR: Function Attrs: norecurse willreturn
+; ATTRIBUTOR-NEXT: declare void @will_return()
+declare void @will_return() willreturn norecurse
+
+; ATTRIBUTOR: Function Attrs: noinline norecurse nounwind uwtable willreturn
+; ATTRIBUTOR-NEXT: define void @f1()
+define void @f1() #0 {
+    tail call void @will_return()
+    ret void
+}
+
+; ATTRIBUTOR: Function Attrs: noinline norecurse nounwind uwtable willreturn
+; ATTRIBUTOR-NEXT: define void @f2()
+define void @f2() #0 {
+    tail call void @f1()
+    ret void
+}
+
+
+; TEST 9 (negative case)
+; call willreturn function in endless loop.
+
+; ATTRIBUTOR: Function Attrs: noinline norecurse noreturn nounwind uwtable
+; ATTRIBUTOR-NOT: willreturn
+; ATTRIBUTOR-NEXT: define void @call_will_return_but_has_loop()
+define void @call_will_return_but_has_loop() #0 {
+  br label %label1
+label1:
+  tail call void @will_return()
+  br label %label2
+label2:
+  br label %label1
+}
+
+
+; TEST 10 (positive case)
+; invoke a function with willreturn
+
+; ATTRIBUTOR: Function Attrs: noinline uwtable willreturn
+; ATTRIBUTOR-NEXT: declare i1 @maybe_raise_exception()
+declare i1 @maybe_raise_exception() #1 willreturn
+
+; ATTRIBUTOR: Function Attrs: nounwind willreturn
+; ATTRIBUTOR-NEXT: define void @invoke_test()
+define void @invoke_test() personality i32 (...)* @__gxx_personality_v0 {
+  invoke i1 @maybe_raise_exception()
+      to label %N unwind label %F
+  N:
+    ret void
+  F:
+    %val = landingpad { i8*, i32 }
+                  catch i8* null
+    ret void
+}
+
+declare i32 @__gxx_personality_v0(...)
+
+
+; TEST 11 (positive case)
+; counstant trip count
+; int loop_constant_trip_count(int*p){
+;    int ans = 0;
+;    for(int i = 0;i<10;i++){
+;        ans += p[i];
+;    }
+;    return ans;
+; }
+
+; FIXME: missing willreturn
+; ATTRIBUTOR: Function Attrs: nofree noinline norecurse nosync nounwind readonly uwtable
+; ATTRIBUTOR-NEXT: define i32 @loop_constant_trip_count(i32* nocapture nofree readonly %0)
+define i32 @loop_constant_trip_count(i32* nocapture readonly %0) #0 {
+  br label %3
+
+; <label>:2:                                      ; preds = %3
+  ret i32 %8
+
+; <label>:3:                                      ; preds = %3, %1
+  %4 = phi i64 [ 0, %1 ], [ %9, %3 ]
+  %5 = phi i32 [ 0, %1 ], [ %8, %3 ]
+  %6 = getelementptr inbounds i32, i32* %0, i64 %4
+  %7 = load i32, i32* %6, align 4
+  %8 = add nsw i32 %7, %5
+  %9 = add nuw nsw i64 %4, 1
+  %10 = icmp eq i64 %9, 10
+  br i1 %10, label %2, label %3
+}
+
+
+; TEST 12 (negative case)
+; unbounded trip count
+
+; int loop_trip_count_unbound(unsigned s,unsigned e, int *p, int offset){
+;     int ans = 0;
+;     for(unsigned i = s;i != e;i+=offset){
+;         ans += p[i];
+;     }
+;     return ans;
+; }
+; ATTRIBUTOR: Function Attrs: nofree noinline norecurse nosync nounwind readonly uwtable
+; ATTRIBUTOR-NOT: willreturn
+; ATTRIBUTOR-NEXT: define i32 @loop_trip_count_unbound(i32 %0, i32 %1, i32* nocapture nofree readonly %2, i32 %3) local_unnamed_addr
+define i32 @loop_trip_count_unbound(i32 %0, i32 %1, i32* nocapture readonly %2, i32 %3) local_unnamed_addr #0 {
+  %5 = icmp eq i32 %0, %1
+  br i1 %5, label %6, label %8
+
+; <label>:6:                                      ; preds = %8, %4
+  %7 = phi i32 [ 0, %4 ], [ %14, %8 ]
+  ret i32 %7
+
+; <label>:8:                                      ; preds = %4, %8
+  %9 = phi i32 [ %15, %8 ], [ %0, %4 ]
+  %10 = phi i32 [ %14, %8 ], [ 0, %4 ]
+  %11 = zext i32 %9 to i64
+  %12 = getelementptr inbounds i32, i32* %2, i64 %11
+  %13 = load i32, i32* %12, align 4
+  %14 = add nsw i32 %13, %10
+  %15 = add i32 %9, %3
+  %16 = icmp eq i32 %15, %1
+  br i1 %16, label %6, label %8
+}
+
+
+; TEST 13 (positive case)
+; Function Attrs: norecurse nounwind readonly uwtable
+;  int loop_trip_dec(int n, int *p){
+;    int ans = 0;
+;    for(;n >= 0;n--){
+;        ans += p[n];
+;    }
+;    return ans;
+;  }
+
+
+; FIXME: missing willreturn
+; ATTRIBUTOR: Function Attrs: nofree noinline norecurse nosync nounwind readonly uwtable
+; ATTRIBUTOR-NEXT: define i32 @loop_trip_dec(i32 %0, i32* nocapture nofree readonly %1) local_unnamed_addr
+
+define i32 @loop_trip_dec(i32 %0, i32* nocapture readonly %1) local_unnamed_addr #0 {
+  %3 = icmp sgt i32 %0, -1
+  br i1 %3, label %4, label %14
+
+; <label>:4:                                      ; preds = %2
+  %5 = sext i32 %0 to i64
+  br label %6
+
+; <label>:6:                                      ; preds = %4, %6
+  %7 = phi i64 [ %5, %4 ], [ %12, %6 ]
+  %8 = phi i32 [ 0, %4 ], [ %11, %6 ]
+  %9 = getelementptr inbounds i32, i32* %1, i64 %7
+  %10 = load i32, i32* %9, align 4
+  %11 = add nsw i32 %10, %8
+  %12 = add nsw i64 %7, -1
+  %13 = icmp sgt i64 %7, 0
+  br i1 %13, label %6, label %14
+
+; <label>:14:                                     ; preds = %6, %2
+  %15 = phi i32 [ 0, %2 ], [ %11, %6 ]
+  ret i32 %15
+}
+
+; TEST 14 (positive case)
+; multiple return
+
+; ATTRIBUTOR: Function Attrs: nofree noinline norecurse nosync nounwind readnone uwtable willreturn
+; ATTRIBUTOR-NEXT: define i32 @multiple_return(i32 %a)
+define i32 @multiple_return(i32 %a) #0 {
+  %b =  icmp eq i32 %a, 0
+  br i1 %b, label %t, label %f
+
+t:
+  ret i32 1
+f:
+  ret i32 0
+}
+
+; TEST 15 (positive & negative case)
+; unreachable exit
+
+; 15.1 (positive case)
+; ATTRIBUTOR: Function Attrs: noinline norecurse nounwind uwtable willreturn
+; ATTRIBUTOR-NEXT: define void @unreachable_exit_positive1()
+define void @unreachable_exit_positive1() #0 {
+  tail call void @will_return()
+  ret void
+
+unreachable_label:
+  tail call void @exit(i32 0)
+  unreachable
+}
+
+; FIXME: missing willreturn
+; ATTRIBUTOR: Function Attrs: nofree noinline norecurse nosync nounwind readnone uwtable
+; ATTRIBUTOR-NEXT: define i32 @unreachable_exit_positive2(i32 %0)
+define i32 @unreachable_exit_positive2(i32) local_unnamed_addr #0 {
+  %2 = icmp slt i32 %0, 1
+  br i1 %2, label %3, label %5
+
+; <label>:3:                                      ; preds = %5, %1
+  %4 = phi i32 [ 1, %1 ], [ %8, %5 ]
+  ret i32 %4
+
+; <label>:5:                                      ; preds = %1, %5
+  %6 = phi i32 [ %9, %5 ], [ 1, %1 ]
+  %7 = phi i32 [ %8, %5 ], [ 1, %1 ]
+  %8 = mul nsw i32 %6, %7
+  %9 = add nuw nsw i32 %6, 1
+  %10 = icmp eq i32 %6, %0
+  br i1 %10, label %3, label %5
+
+unreachable_label:
+  tail call void @exit(i32 0)
+  unreachable
+}
+
+
+;15.2
+
+; ATTRIBUTOR: Function Attrs: noinline noreturn nounwind uwtable
+; ATTRIBUTOR-NOT: willreturn
+; ATTRIBUTOR-NEXT: define void @unreachable_exit_negative1()
+define void @unreachable_exit_negative1() #0 {
+  tail call void @exit(i32 0)
+  ret void
+
+unreachable_label:
+  tail call void @exit(i32 0)
+  unreachable
+}
+
+; ATTRIBUTOR: Function Attrs: nofree noinline norecurse noreturn nosync nounwind readnone uwtable
+; ATTRIBUTOR-NOT: willreturn
+; ATTRIBUTOR-NEXT: define void @unreachable_exit_negative2()
+define void @unreachable_exit_negative2() #0 {
+
+  br label %L1
+L1:
+  br label %L2
+L2:
+  br label %L1
+
+unreachable_label:
+  tail call void @exit(i32 0)
+  unreachable
+}
+
+; ATTRIBUTOR: Function Attrs: noreturn nounwind
+; ATTRIBUTOR-NEXT: declare void @llvm.eh.sjlj.longjmp(i8*)
+declare void @llvm.eh.sjlj.longjmp(i8*)
+
+; ATTRIBUTOR: Function Attrs: noinline noreturn nounwind uwtable
+; ATTRIBUTOR-NOT: willreturn
+; ATTRIBUTOR-NEXT: define void @call_longjmp(i8* nocapture readnone %0) local_unnamed_addr
+define void @call_longjmp(i8* nocapture readnone %0) local_unnamed_addr #0 {
+  tail call void @llvm.eh.sjlj.longjmp(i8* %0)
+  ret void
+}
+
+
+attributes #0 = { nounwind uwtable noinline }
+attributes #1 = { uwtable noinline }
diff --git a/llvm/test/Transforms/BDCE/pr41925.ll b/llvm/test/Transforms/BDCE/pr41925.ll
new file mode 100644
index 000000000..0b7d03df6
--- /dev/null
+++ b/llvm/test/Transforms/BDCE/pr41925.ll
@@ -0,0 +1,126 @@
+; Check for setting dbg.value undef which depends on instruction which is going to be dead
+; RUN: opt -bdce %s -S | FileCheck %s
+
+; Generated from this
+; char a, b;
+; void optimize_me_not();
+; int main() {
+;  char l_177 = 2;
+;  for (; b <= 0; b++)
+;    for (; a >= 0; a--) {
+;      ++l_177;
+;      optimize_me_not();
+;    }
+; }
+
+@b = common dso_local local_unnamed_addr global i8 0, align 1, !dbg !0
+@a = common dso_local local_unnamed_addr global i8 0, align 1, !dbg !6
+
+define dso_local i32 @main() local_unnamed_addr !dbg !13 {
+entry:
+;CHECK: call void @llvm.dbg.value(metadata i8 2
+;CHECK: call void @llvm.dbg.value(metadata i8 2
+  call void @llvm.dbg.value(metadata i8 2, metadata !17, metadata !DIExpression()), !dbg !18
+  %.pr = load i8, i8* @b, align 1, !dbg !19
+  call void @llvm.dbg.value(metadata i8 2, metadata !17, metadata !DIExpression()), !dbg !18
+  %cmp5 = icmp slt i8 %.pr, 1, !dbg !22
+  br i1 %cmp5, label %for.cond2thread-pre-split.preheader, label %for.end9, !dbg !23
+
+for.cond2thread-pre-split.preheader:              ; preds = %entry
+  br label %for.cond2thread-pre-split, !dbg !23
+for.cond2thread-pre-split:                        ; preds = %for.cond2thread-pre-split.preheader, %for.inc7
+;CHECK: call void @llvm.dbg.value(metadata i8 undef
+  %l_177.06 = phi i8 [ %l_177.1.lcssa, %for.inc7 ], [ 2, %for.cond2thread-pre-split.preheader ]
+  call void @llvm.dbg.value(metadata i8 %l_177.06, metadata !17, metadata !DIExpression()), !dbg !18
+;CHECK: call void @llvm.dbg.value(metadata i8 undef
+  %.pr1 = load i8, i8* @a, align 1, !dbg !24
+  call void @llvm.dbg.value(metadata i8 %l_177.06, metadata !17, metadata !DIExpression()), !dbg !18
+  %cmp42 = icmp sgt i8 %.pr1, -1, !dbg !27
+  br i1 %cmp42, label %for.body6.preheader, label %for.inc7, !dbg !28
+
+for.body6.preheader:                              ; preds = %for.cond2thread-pre-split
+  br label %for.body6, !dbg !28
+
+for.body6:                                        ; preds = %for.body6.preheader, %for.body6
+;CHECK: call void @llvm.dbg.value(metadata i8 undef
+;CHECK: call void @llvm.dbg.value(metadata i8 undef
+  %l_177.13 = phi i8 [ %inc, %for.body6 ], [ %l_177.06, %for.body6.preheader ]
+  call void @llvm.dbg.value(metadata i8 %l_177.13, metadata !17, metadata !DIExpression()), !dbg !18
+  call void @llvm.dbg.value(metadata i8 %l_177.13, metadata !17, metadata !DIExpression(DW_OP_plus_uconst, 1, DW_OP_stack_value)), !dbg !18
+  tail call void (...) @optimize_me_not(),  !dbg !29
+  %inc = add i8 %l_177.13, 1, !dbg !31
+  %0 = load i8, i8* @a, align 1, !dbg !32
+  %dec = add i8 %0, -1, !dbg !32
+  store i8 %dec, i8* @a, align 1, !dbg !32
+  call void @llvm.dbg.value(metadata i8 %inc, metadata !17, metadata !DIExpression()), !dbg !18
+  %cmp4 = icmp sgt i8 %dec, -1, !dbg !27
+  br i1 %cmp4, label %for.body6, label %for.inc7.loopexit, !dbg !28, !llvm.loop !33
+
+for.inc7.loopexit:                                ; preds = %for.body6
+  %inc.lcssa = phi i8 [ %inc, %for.body6 ], !dbg !31
+  br label %for.inc7, !dbg !35
+
+for.inc7:                                         ; preds = %for.inc7.loopexit, %for.cond2thread-pre-split
+;CHECK: call void @llvm.dbg.value(metadata i8 undef
+  %l_177.1.lcssa = phi i8 [ %l_177.06, %for.cond2thread-pre-split ], [ %inc.lcssa, %for.inc7.loopexit ], !dbg !18
+  %1 = load i8, i8* @b, align 1, !dbg !35
+  %inc8 = add i8 %1, 1, !dbg !35
+  store i8 %inc8, i8* @b, align 1, !dbg !35
+  call void @llvm.dbg.value(metadata i8 %l_177.1.lcssa, metadata !17, metadata !DIExpression()), !dbg !18
+  %cmp = icmp slt i8 %inc8, 1, !dbg !22
+  br i1 %cmp, label %for.cond2thread-pre-split, label %for.end9.loopexit, !dbg !23, !llvm.loop !36
+
+for.end9.loopexit:                                ; preds = %for.inc7
+  br label %for.end9, !dbg !38
+
+for.end9:                                         ; preds = %for.end9.loopexit, %entry
+  ret i32 0, !dbg !38
+}
+
+declare dso_local void @optimize_me_not(...) local_unnamed_addr 
+
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!9, !10, !11}
+!llvm.ident = !{!12}
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "b", scope: !2, file: !3, line: 3, type: !8, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 10.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5, nameTableKind: None)
+!3 = !DIFile(filename: "test", directory: "test")
+!4 = !{}
+!5 = !{!6, !0}
+!6 = !DIGlobalVariableExpression(var: !7, expr: !DIExpression())
+!7 = distinct !DIGlobalVariable(name: "a", scope: !2, file: !3, line: 3, type: !8, isLocal: false, isDefinition: true)
+!8 = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_signed_char)
+!9 = !{i32 2, !"Dwarf Version", i32 4}
+!10 = !{i32 2, !"Debug Info Version", i32 3}
+!11 = !{i32 1, !"wchar_size", i32 4}
+!12 = !{!"clang version 10.0.0"}
+!13 = distinct !DISubprogram(name: "main", scope: !3, file: !3, line: 4, type: !14, scopeLine: 4, spFlags: DISPFlagDefinition, unit: !2, retainedNodes: !4)
+!14 = !DISubroutineType(types: !15)
+!15 = !{!16}
+!16 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!17 = !DILocalVariable(name: "l_177", scope: !13, file: !3, line: 5, type: !8)
+!18 = !DILocation(line: 0, scope: !13)
+!19 = !DILocation(line: 6, column: 10, scope: !20)
+!20 = distinct !DILexicalBlock(scope: !21, file: !3, line: 6, column: 3)
+!21 = distinct !DILexicalBlock(scope: !13, file: !3, line: 6, column: 3)
+!22 = !DILocation(line: 6, column: 12, scope: !20)
+!23 = !DILocation(line: 6, column: 3, scope: !21)
+!24 = !DILocation(line: 7, column: 12, scope: !25)
+!25 = distinct !DILexicalBlock(scope: !26, file: !3, line: 7, column: 5)
+!26 = distinct !DILexicalBlock(scope: !20, file: !3, line: 7, column: 5)
+!27 = !DILocation(line: 7, column: 14, scope: !25)
+!28 = !DILocation(line: 7, column: 5, scope: !26)
+!29 = !DILocation(line: 9, column: 7, scope: !30)
+!30 = distinct !DILexicalBlock(scope: !25, file: !3, line: 7, column: 25)
+!31 = !DILocation(line: 8, column: 7, scope: !30)
+!32 = !DILocation(line: 7, column: 21, scope: !25)
+!33 = distinct !{!33, !28, !34}
+!34 = !DILocation(line: 10, column: 5, scope: !26)
+!35 = !DILocation(line: 6, column: 19, scope: !20)
+!36 = distinct !{!36, !23, !37}
+!37 = !DILocation(line: 10, column: 5, scope: !21)
+!38 = !DILocation(line: 11, column: 1, scope: !13)
diff --git a/llvm/test/Transforms/BlockExtractor/invalid-line.ll b/llvm/test/Transforms/BlockExtractor/invalid-line.ll
new file mode 100644
index 000000000..7e409d359
--- /dev/null
+++ b/llvm/test/Transforms/BlockExtractor/invalid-line.ll
@@ -0,0 +1,9 @@
+; RUN: echo 'foo' > %t
+; RUN: not opt -S -extract-blocks -extract-blocks-file=%t %s 2>&1 | FileCheck %s
+
+; CHECK: Invalid line
+define void @bar() {
+bb:
+  ret void
+}
+
diff --git a/llvm/test/Transforms/CodeGenPrepare/ARM/bitreverse-recognize.ll b/llvm/test/Transforms/CodeGenPrepare/ARM/bitreverse-recognize.ll
index f5644e4ad..d272fef54 100644
--- a/llvm/test/Transforms/CodeGenPrepare/ARM/bitreverse-recognize.ll
+++ b/llvm/test/Transforms/CodeGenPrepare/ARM/bitreverse-recognize.ll
@@ -25,7 +25,7 @@ for.body:                                         ; preds = %for.body, %entry
   br i1 %exitcond, label %for.cond.cleanup, label %for.body, !llvm.loop !3
 }
 
-attributes #0 = { norecurse nounwind readnone "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="cortex-a8" "target-features"="+dsp,+neon,+vfp3" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { norecurse nounwind readnone "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="cortex-a8" "target-features"="+dsp,+neon,+vfp3" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.module.flags = !{!0, !1}
 !llvm.ident = !{!2}
diff --git a/llvm/test/Transforms/CodeGenPrepare/ARM/sink-add-mul-shufflevector.ll b/llvm/test/Transforms/CodeGenPrepare/ARM/sink-add-mul-shufflevector.ll
new file mode 100644
index 000000000..cb0737ffa
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/ARM/sink-add-mul-shufflevector.ll
@@ -0,0 +1,219 @@
+; RUN: opt -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve.fp < %s -codegenprepare -S | FileCheck -check-prefix=CHECK %s
+
+define void @sink_add_mul(i32* %s1, i32 %x, i32* %d, i32 %n) {
+; CHECK-LABEL: @sink_add_mul(
+; CHECK:    vector.ph:
+; CHECK-NOT:  [[BROADCAST_SPLATINSERT8:%.*]] = insertelement <4 x i32> undef, i32 [[X:%.*]], i32 0
+; CHECK-NOT:  [[BROADCAST_SPLAT9:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT8]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK:    vector.body:
+; CHECK:      [[TMP2:%.*]] = insertelement <4 x i32> undef, i32 [[X:%.*]], i32 0
+; CHECK:      [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> undef, <4 x i32> zeroinitializer
+;
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %vector.ph, label %for.cond.cleanup
+
+vector.ph:                                        ; preds = %for.body.preheader
+  %n.vec = and i32 %n, -4
+  %broadcast.splatinsert8 = insertelement <4 x i32> undef, i32 %x, i32 0
+  %broadcast.splat9 = shufflevector <4 x i32> %broadcast.splatinsert8, <4 x i32> undef, <4 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %0 = getelementptr inbounds i32, i32* %s1, i32 %index
+  %1 = bitcast i32* %0 to <4 x i32>*
+  %wide.load = load <4 x i32>, <4 x i32>* %1, align 4
+  %2 = mul nsw <4 x i32> %wide.load, %broadcast.splat9
+  %3 = getelementptr inbounds i32, i32* %d, i32 %index
+  %4 = bitcast i32* %3 to <4 x i32>*
+  %wide.load10 = load <4 x i32>, <4 x i32>* %4, align 4
+  %5 = add nsw <4 x i32> %wide.load10, %2
+  %6 = bitcast i32* %3 to <4 x i32>*
+  store <4 x i32> %5, <4 x i32>* %6, align 4
+  %index.next = add i32 %index, 4
+  %7 = icmp eq i32 %index.next, %n.vec
+  br i1 %7, label %for.cond.cleanup, label %vector.body
+
+for.cond.cleanup:                                 ; preds = %for.body, %middle.block, %entry
+  ret void
+}
+
+define void @sink_add_mul_multiple(i32* %s1, i32* %s2, i32 %x, i32* %d, i32* %d2, i32 %n) {
+; CHECK-LABEL: @sink_add_mul_multiple(
+; CHECK:    vector.ph:
+; CHECK-NOT:  [[BROADCAST_SPLATINSERT8:%.*]] = insertelement <4 x i32> undef, i32 [[X:%.*]], i32 0
+; CHECK-NOT:  [[BROADCAST_SPLAT9:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT8]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK:    vector.body:
+; CHECK:      [[TMP2:%.*]] = insertelement <4 x i32> undef, i32 %x, i32 0
+; CHECK:      [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK:      mul nsw <4 x i32> %wide.load, [[TMP3]]
+; CHECK:      [[TMP2b:%.*]] = insertelement <4 x i32> undef, i32 %x, i32 0
+; CHECK:      [[TMP3b:%.*]] = shufflevector <4 x i32> [[TMP2b]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK:      mul nsw <4 x i32> %wide.load18, [[TMP3b]]
+;
+entry:
+  %cmp13 = icmp sgt i32 %n, 0
+  br i1 %cmp13, label %vector.ph, label %for.cond.cleanup
+
+vector.ph:                                        ; preds = %for.body.preheader
+  %n.vec = and i32 %n, -4
+  %broadcast.splatinsert15 = insertelement <4 x i32> undef, i32 %x, i32 0
+  %broadcast.splat16 = shufflevector <4 x i32> %broadcast.splatinsert15, <4 x i32> undef, <4 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %0 = getelementptr inbounds i32, i32* %s1, i32 %index
+  %1 = bitcast i32* %0 to <4 x i32>*
+  %wide.load = load <4 x i32>, <4 x i32>* %1, align 4
+  %2 = mul nsw <4 x i32> %wide.load, %broadcast.splat16
+  %3 = getelementptr inbounds i32, i32* %d, i32 %index
+  %4 = bitcast i32* %3 to <4 x i32>*
+  %wide.load17 = load <4 x i32>, <4 x i32>* %4, align 4
+  %5 = add nsw <4 x i32> %wide.load17, %2
+  %6 = bitcast i32* %3 to <4 x i32>*
+  store <4 x i32> %5, <4 x i32>* %6, align 4
+  %7 = getelementptr inbounds i32, i32* %s2, i32 %index
+  %8 = bitcast i32* %7 to <4 x i32>*
+  %wide.load18 = load <4 x i32>, <4 x i32>* %8, align 4
+  %9 = mul nsw <4 x i32> %wide.load18, %broadcast.splat16
+  %10 = getelementptr inbounds i32, i32* %d2, i32 %index
+  %11 = bitcast i32* %10 to <4 x i32>*
+  %wide.load19 = load <4 x i32>, <4 x i32>* %11, align 4
+  %12 = add nsw <4 x i32> %wide.load19, %9
+  %13 = bitcast i32* %10 to <4 x i32>*
+  store <4 x i32> %12, <4 x i32>* %13, align 4
+  %index.next = add i32 %index, 4
+  %14 = icmp eq i32 %index.next, %n.vec
+  br i1 %14, label %for.cond.cleanup, label %vector.body
+
+for.cond.cleanup:                                 ; preds = %for.body, %middle.block, %entry
+  ret void
+}
+
+
+define void @sink_add_sub_unsinkable(i32* %s1, i32* %s2, i32 %x, i32* %d, i32* %d2, i32 %n) {
+; CHECK-LABEL: @sink_add_sub_unsinkable(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP13:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP13]], label [[VECTOR_PH:%.*]], label [[FOR_COND_CLEANUP:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[N_VEC:%.*]] = and i32 [[N]], -4
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT15:%.*]] = insertelement <4 x i32> undef, i32 [[X:%.*]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT16:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT15]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+;
+entry:
+  %cmp13 = icmp sgt i32 %n, 0
+  br i1 %cmp13, label %vector.ph, label %for.cond.cleanup
+
+vector.ph:                                        ; preds = %for.body.preheader
+  %n.vec = and i32 %n, -4
+  %broadcast.splatinsert15 = insertelement <4 x i32> undef, i32 %x, i32 0
+  %broadcast.splat16 = shufflevector <4 x i32> %broadcast.splatinsert15, <4 x i32> undef, <4 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %0 = getelementptr inbounds i32, i32* %s1, i32 %index
+  %1 = bitcast i32* %0 to <4 x i32>*
+  %wide.load = load <4 x i32>, <4 x i32>* %1, align 4
+  %2 = mul nsw <4 x i32> %wide.load, %broadcast.splat16
+  %3 = getelementptr inbounds i32, i32* %d, i32 %index
+  %4 = bitcast i32* %3 to <4 x i32>*
+  %wide.load17 = load <4 x i32>, <4 x i32>* %4, align 4
+  %5 = add nsw <4 x i32> %wide.load17, %2
+  %6 = bitcast i32* %3 to <4 x i32>*
+  store <4 x i32> %5, <4 x i32>* %6, align 4
+  %7 = getelementptr inbounds i32, i32* %s2, i32 %index
+  %8 = bitcast i32* %7 to <4 x i32>*
+  %wide.load18 = load <4 x i32>, <4 x i32>* %8, align 4
+  %9 = sub nsw <4 x i32> %broadcast.splat16, %wide.load18
+  %10 = getelementptr inbounds i32, i32* %d2, i32 %index
+  %11 = bitcast i32* %10 to <4 x i32>*
+  %wide.load19 = load <4 x i32>, <4 x i32>* %11, align 4
+  %12 = add nsw <4 x i32> %wide.load19, %9
+  %13 = bitcast i32* %10 to <4 x i32>*
+  store <4 x i32> %12, <4 x i32>* %13, align 4
+  %index.next = add i32 %index, 4
+  %14 = icmp eq i32 %index.next, %n.vec
+  br i1 %14, label %for.cond.cleanup, label %vector.body
+
+for.cond.cleanup:                                 ; preds = %for.body, %middle.block, %entry
+  ret void
+}
+
+define void @sink_sub(i32* %s1, i32 %x, i32* %d, i32 %n) {
+; CHECK-LABEL: @sink_sub(
+; CHECK:    vector.ph:
+; CHECK-NOT:  [[BROADCAST_SPLATINSERT8:%.*]] = insertelement <4 x i32> undef, i32 [[X:%.*]], i32 0
+; CHECK-NOT:  [[BROADCAST_SPLAT9:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT8]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK:    vector.body:
+; CHECK:      [[TMP2:%.*]] = insertelement <4 x i32> undef, i32 [[X:%.*]], i32 0
+; CHECK:      [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> undef, <4 x i32> zeroinitializer
+;
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %vector.ph, label %for.cond.cleanup
+
+vector.ph:                                        ; preds = %for.body.preheader
+  %n.vec = and i32 %n, -4
+  %broadcast.splatinsert8 = insertelement <4 x i32> undef, i32 %x, i32 0
+  %broadcast.splat9 = shufflevector <4 x i32> %broadcast.splatinsert8, <4 x i32> undef, <4 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %0 = getelementptr inbounds i32, i32* %s1, i32 %index
+  %1 = bitcast i32* %0 to <4 x i32>*
+  %wide.load = load <4 x i32>, <4 x i32>* %1, align 4
+  %2 = sub nsw <4 x i32> %wide.load, %broadcast.splat9
+  %3 = getelementptr inbounds i32, i32* %d, i32 %index
+  %4 = bitcast i32* %3 to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %4, align 4
+  %index.next = add i32 %index, 4
+  %5 = icmp eq i32 %index.next, %n.vec
+  br i1 %5, label %for.cond.cleanup, label %vector.body
+
+for.cond.cleanup:                                 ; preds = %for.body, %middle.block, %entry
+  ret void
+}
+
+define void @sink_sub_unsinkable(i32* %s1, i32 %x, i32* %d, i32 %n) {
+entry:
+; CHECK-LABEL: @sink_sub_unsinkable(
+; CHECK:      vector.ph:
+; CHECK-NEXT:   [[N_VEC:%.*]] = and i32 [[N]], -4
+; CHECK-NEXT:   [[BROADCAST_SPLATINSERT15:%.*]] = insertelement <4 x i32> undef, i32 [[X:%.*]], i32 0
+; CHECK-NEXT:   [[BROADCAST_SPLAT16:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT15]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:   br label [[VECTOR_BODY:%.*]]
+; CHECK:      vector.body:
+; CHECK-NOT:    [[TMP2:%.*]] = insertelement <4 x i32> undef, i32 [[X:%.*]], i32 0
+; CHECK-NOT:    [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> undef, <4 x i32> zeroinitializer
+;
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %vector.ph, label %for.cond.cleanup
+
+vector.ph:                                        ; preds = %for.body.preheader
+  %n.vec = and i32 %n, -4
+  %broadcast.splatinsert8 = insertelement <4 x i32> undef, i32 %x, i32 0
+  %broadcast.splat9 = shufflevector <4 x i32> %broadcast.splatinsert8, <4 x i32> undef, <4 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %0 = getelementptr inbounds i32, i32* %s1, i32 %index
+  %1 = bitcast i32* %0 to <4 x i32>*
+  %wide.load = load <4 x i32>, <4 x i32>* %1, align 4
+  %2 = sub nsw <4 x i32> %broadcast.splat9, %wide.load
+  %3 = getelementptr inbounds i32, i32* %d, i32 %index
+  %4 = bitcast i32* %3 to <4 x i32>*
+  store <4 x i32> %2, <4 x i32>* %4, align 4
+  %index.next = add i32 %index, 4
+  %5 = icmp eq i32 %index.next, %n.vec
+  br i1 %5, label %for.cond.cleanup, label %vector.body
+
+for.cond.cleanup:                                 ; preds = %for.body, %middle.block, %entry
+  ret void
+}
diff --git a/llvm/test/Transforms/CodeGenPrepare/ARM/sinkchain.ll b/llvm/test/Transforms/CodeGenPrepare/ARM/sinkchain.ll
new file mode 100644
index 000000000..c0da3eb56
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/ARM/sinkchain.ll
@@ -0,0 +1,107 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve.fp < %s -codegenprepare -S | FileCheck -check-prefix=CHECK %s
+
+; Sink the shufflevector/insertelement pair, followed by the trunc. The sunk instruction end up dead.
+define signext i8 @dead(i16* noalias nocapture readonly %s1, i16 zeroext %x, i8* noalias nocapture %d, i32 %n) {
+; CHECK-LABEL: @dead(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[N_VEC:%.*]] = and i32 [[N:%.*]], -8
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = trunc i16 [[X:%.*]] to i8
+; CHECK-NEXT:    [[L6:%.*]] = getelementptr inbounds i16, i16* [[S1:%.*]], i32 [[INDEX]]
+; CHECK-NEXT:    [[L7:%.*]] = bitcast i16* [[L6]] to <8 x i16>*
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x i16>, <8 x i16>* [[L7]], align 2
+; CHECK-NEXT:    [[L8:%.*]] = trunc <8 x i16> [[WIDE_LOAD]] to <8 x i8>
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <8 x i8> undef, i8 [[TMP0]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i8> [[TMP1]], <8 x i8> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[L9:%.*]] = mul <8 x i8> [[TMP2]], [[L8]]
+; CHECK-NEXT:    [[L13:%.*]] = getelementptr inbounds i8, i8* [[D:%.*]], i32 [[INDEX]]
+; CHECK-NEXT:    [[L14:%.*]] = bitcast i8* [[L13]] to <8 x i8>*
+; CHECK-NEXT:    store <8 x i8> [[L9]], <8 x i8>* [[L14]], align 1
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 8
+; CHECK-NEXT:    [[L15:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[L15]], label [[EXIT:%.*]], label [[VECTOR_BODY]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i8 0
+;
+entry:
+  %n.vec = and i32 %n, -8
+  %l0 = trunc i16 %x to i8
+  %l1 = insertelement <8 x i8> undef, i8 %l0, i32 0
+  %broadcast.splat26 = shufflevector <8 x i8> %l1, <8 x i8> undef, <8 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %entry
+  %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]
+  %l6 = getelementptr inbounds i16, i16* %s1, i32 %index
+  %l7 = bitcast i16* %l6 to <8 x i16>*
+  %wide.load = load <8 x i16>, <8 x i16>* %l7, align 2
+  %l8 = trunc <8 x i16> %wide.load to <8 x i8>
+  %l9 = mul <8 x i8> %broadcast.splat26, %l8
+  %l13 = getelementptr inbounds i8, i8* %d, i32 %index
+  %l14 = bitcast i8* %l13 to <8 x i8>*
+  store <8 x i8> %l9, <8 x i8>* %l14, align 1
+  %index.next = add i32 %index, 8
+  %l15 = icmp eq i32 %index.next, %n.vec
+  br i1 %l15, label %exit, label %vector.body
+
+exit:                                     ; preds = %vector.body
+  ret i8 0
+}
+
+; Same as above, but the shuffle has an extra use meaning it shouldnt be deleted
+define signext i8 @alive(i16* noalias nocapture readonly %s1, i16 zeroext %x, i8* noalias nocapture %d, i32 %n) {
+; CHECK-LABEL: @alive(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[N_VEC:%.*]] = and i32 [[N:%.*]], -8
+; CHECK-NEXT:    [[L0:%.*]] = trunc i16 [[X:%.*]] to i8
+; CHECK-NEXT:    [[L1:%.*]] = insertelement <8 x i8> undef, i8 [[L0]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT26:%.*]] = shufflevector <8 x i8> [[L1]], <8 x i8> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[L2:%.*]] = sub <8 x i8> zeroinitializer, [[BROADCAST_SPLAT26]]
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = trunc i16 [[X]] to i8
+; CHECK-NEXT:    [[L6:%.*]] = getelementptr inbounds i16, i16* [[S1:%.*]], i32 [[INDEX]]
+; CHECK-NEXT:    [[L7:%.*]] = bitcast i16* [[L6]] to <8 x i16>*
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x i16>, <8 x i16>* [[L7]], align 2
+; CHECK-NEXT:    [[L8:%.*]] = trunc <8 x i16> [[WIDE_LOAD]] to <8 x i8>
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <8 x i8> undef, i8 [[TMP0]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i8> [[TMP1]], <8 x i8> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[L9:%.*]] = mul <8 x i8> [[TMP2]], [[L8]]
+; CHECK-NEXT:    [[L13:%.*]] = getelementptr inbounds i8, i8* [[D:%.*]], i32 [[INDEX]]
+; CHECK-NEXT:    [[L14:%.*]] = bitcast i8* [[L13]] to <8 x i8>*
+; CHECK-NEXT:    store <8 x i8> [[L9]], <8 x i8>* [[L14]], align 1
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 8
+; CHECK-NEXT:    [[L15:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[L15]], label [[EXIT:%.*]], label [[VECTOR_BODY]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i8 0
+;
+entry:
+  %n.vec = and i32 %n, -8
+  %l0 = trunc i16 %x to i8
+  %l1 = insertelement <8 x i8> undef, i8 %l0, i32 0
+  %broadcast.splat26 = shufflevector <8 x i8> %l1, <8 x i8> undef, <8 x i32> zeroinitializer
+  %l2 = sub <8 x i8> zeroinitializer, %broadcast.splat26
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %entry
+  %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]
+  %l6 = getelementptr inbounds i16, i16* %s1, i32 %index
+  %l7 = bitcast i16* %l6 to <8 x i16>*
+  %wide.load = load <8 x i16>, <8 x i16>* %l7, align 2
+  %l8 = trunc <8 x i16> %wide.load to <8 x i8>
+  %l9 = mul <8 x i8> %broadcast.splat26, %l8
+  %l13 = getelementptr inbounds i8, i8* %d, i32 %index
+  %l14 = bitcast i8* %l13 to <8 x i8>*
+  store <8 x i8> %l9, <8 x i8>* %l14, align 1
+  %index.next = add i32 %index, 8
+  %l15 = icmp eq i32 %index.next, %n.vec
+  br i1 %l15, label %exit, label %vector.body
+
+exit:                                     ; preds = %vector.body
+  ret i8 0
+}
diff --git a/llvm/test/Transforms/CodeGenPrepare/PowerPC/split-store-alignment.ll b/llvm/test/Transforms/CodeGenPrepare/PowerPC/split-store-alignment.ll
new file mode 100644
index 000000000..5bc7d3a3b
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/PowerPC/split-store-alignment.ll
@@ -0,0 +1,111 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -codegenprepare -mtriple=powerpc64-unknown-linux-gnu -data-layout="E-m:e-i64:64-n32:64" -force-split-store < %s  | FileCheck --check-prefixes=ALL,BE %s
+; RUN: opt -S -codegenprepare -mtriple=powerpc64le-unknown-linux-gnu -data-layout="e-m:e-i64:64-n32:64" -force-split-store < %s  | FileCheck --check-prefixes=ALL,LE %s
+
+define void @split_store_align1(float %x, i64* %p) {
+; BE-LABEL: @split_store_align1(
+; BE-NEXT:    [[B:%.*]] = bitcast float [[X:%.*]] to i32
+; BE-NEXT:    [[Z:%.*]] = zext i32 0 to i64
+; BE-NEXT:    [[S:%.*]] = shl nuw nsw i64 [[Z]], 32
+; BE-NEXT:    [[Z2:%.*]] = zext i32 [[B]] to i64
+; BE-NEXT:    [[O:%.*]] = or i64 [[S]], [[Z2]]
+; BE-NEXT:    [[TMP1:%.*]] = bitcast i64* [[P:%.*]] to i32*
+; BE-NEXT:    [[TMP2:%.*]] = getelementptr i32, i32* [[TMP1]], i32 1
+; BE-NEXT:    store i32 [[B]], i32* [[TMP2]], align 1
+; BE-NEXT:    [[TMP3:%.*]] = bitcast i64* [[P]] to i32*
+; BE-NEXT:    store i32 0, i32* [[TMP3]], align 1
+; BE-NEXT:    ret void
+;
+; LE-LABEL: @split_store_align1(
+; LE-NEXT:    [[B:%.*]] = bitcast float [[X:%.*]] to i32
+; LE-NEXT:    [[Z:%.*]] = zext i32 0 to i64
+; LE-NEXT:    [[S:%.*]] = shl nuw nsw i64 [[Z]], 32
+; LE-NEXT:    [[Z2:%.*]] = zext i32 [[B]] to i64
+; LE-NEXT:    [[O:%.*]] = or i64 [[S]], [[Z2]]
+; LE-NEXT:    [[TMP1:%.*]] = bitcast i64* [[P:%.*]] to i32*
+; LE-NEXT:    store i32 [[B]], i32* [[TMP1]], align 1
+; LE-NEXT:    [[TMP2:%.*]] = bitcast i64* [[P]] to i32*
+; LE-NEXT:    [[TMP3:%.*]] = getelementptr i32, i32* [[TMP2]], i32 1
+; LE-NEXT:    store i32 0, i32* [[TMP3]], align 1
+; LE-NEXT:    ret void
+;
+  %b = bitcast float %x to i32
+  %z = zext i32 0 to i64
+  %s = shl nuw nsw i64 %z, 32
+  %z2 = zext i32 %b to i64
+  %o = or i64 %s, %z2
+  store i64 %o, i64* %p, align 1
+  ret void
+}
+
+define void @split_store_align2(float %x, i64* %p) {
+; BE-LABEL: @split_store_align2(
+; BE-NEXT:    [[B:%.*]] = bitcast float [[X:%.*]] to i32
+; BE-NEXT:    [[Z:%.*]] = zext i32 0 to i64
+; BE-NEXT:    [[S:%.*]] = shl nuw nsw i64 [[Z]], 32
+; BE-NEXT:    [[Z2:%.*]] = zext i32 [[B]] to i64
+; BE-NEXT:    [[O:%.*]] = or i64 [[S]], [[Z2]]
+; BE-NEXT:    [[TMP1:%.*]] = bitcast i64* [[P:%.*]] to i32*
+; BE-NEXT:    [[TMP2:%.*]] = getelementptr i32, i32* [[TMP1]], i32 1
+; BE-NEXT:    store i32 [[B]], i32* [[TMP2]], align 2
+; BE-NEXT:    [[TMP3:%.*]] = bitcast i64* [[P]] to i32*
+; BE-NEXT:    store i32 0, i32* [[TMP3]], align 2
+; BE-NEXT:    ret void
+;
+; LE-LABEL: @split_store_align2(
+; LE-NEXT:    [[B:%.*]] = bitcast float [[X:%.*]] to i32
+; LE-NEXT:    [[Z:%.*]] = zext i32 0 to i64
+; LE-NEXT:    [[S:%.*]] = shl nuw nsw i64 [[Z]], 32
+; LE-NEXT:    [[Z2:%.*]] = zext i32 [[B]] to i64
+; LE-NEXT:    [[O:%.*]] = or i64 [[S]], [[Z2]]
+; LE-NEXT:    [[TMP1:%.*]] = bitcast i64* [[P:%.*]] to i32*
+; LE-NEXT:    store i32 [[B]], i32* [[TMP1]], align 2
+; LE-NEXT:    [[TMP2:%.*]] = bitcast i64* [[P]] to i32*
+; LE-NEXT:    [[TMP3:%.*]] = getelementptr i32, i32* [[TMP2]], i32 1
+; LE-NEXT:    store i32 0, i32* [[TMP3]], align 2
+; LE-NEXT:    ret void
+;
+  %b = bitcast float %x to i32
+  %z = zext i32 0 to i64
+  %s = shl nuw nsw i64 %z, 32
+  %z2 = zext i32 %b to i64
+  %o = or i64 %s, %z2
+  store i64 %o, i64* %p, align 2
+  ret void
+}
+
+define void @split_store_align8(float %x, i64* %p) {
+; BE-LABEL: @split_store_align8(
+; BE-NEXT:    [[B:%.*]] = bitcast float [[X:%.*]] to i32
+; BE-NEXT:    [[Z:%.*]] = zext i32 0 to i64
+; BE-NEXT:    [[S:%.*]] = shl nuw nsw i64 [[Z]], 32
+; BE-NEXT:    [[Z2:%.*]] = zext i32 [[B]] to i64
+; BE-NEXT:    [[O:%.*]] = or i64 [[S]], [[Z2]]
+; BE-NEXT:    [[TMP1:%.*]] = bitcast i64* [[P:%.*]] to i32*
+; BE-NEXT:    [[TMP2:%.*]] = getelementptr i32, i32* [[TMP1]], i32 1
+; BE-NEXT:    store i32 [[B]], i32* [[TMP2]], align 4
+; BE-NEXT:    [[TMP3:%.*]] = bitcast i64* [[P]] to i32*
+; BE-NEXT:    store i32 0, i32* [[TMP3]], align 8
+; BE-NEXT:    ret void
+;
+; LE-LABEL: @split_store_align8(
+; LE-NEXT:    [[B:%.*]] = bitcast float [[X:%.*]] to i32
+; LE-NEXT:    [[Z:%.*]] = zext i32 0 to i64
+; LE-NEXT:    [[S:%.*]] = shl nuw nsw i64 [[Z]], 32
+; LE-NEXT:    [[Z2:%.*]] = zext i32 [[B]] to i64
+; LE-NEXT:    [[O:%.*]] = or i64 [[S]], [[Z2]]
+; LE-NEXT:    [[TMP1:%.*]] = bitcast i64* [[P:%.*]] to i32*
+; LE-NEXT:    store i32 [[B]], i32* [[TMP1]], align 8
+; LE-NEXT:    [[TMP2:%.*]] = bitcast i64* [[P]] to i32*
+; LE-NEXT:    [[TMP3:%.*]] = getelementptr i32, i32* [[TMP2]], i32 1
+; LE-NEXT:    store i32 0, i32* [[TMP3]], align 4
+; LE-NEXT:    ret void
+;
+  %b = bitcast float %x to i32
+  %z = zext i32 0 to i64
+  %s = shl nuw nsw i64 %z, 32
+  %z2 = zext i32 %b to i64
+  %o = or i64 %s, %z2
+  store i64 %o, i64* %p, align 8
+  ret void
+}
diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/memset_chk-simplify-nobuiltin.ll b/llvm/test/Transforms/CodeGenPrepare/X86/memset_chk-simplify-nobuiltin.ll
index f4c1af5ed..09545e268 100644
--- a/llvm/test/Transforms/CodeGenPrepare/X86/memset_chk-simplify-nobuiltin.ll
+++ b/llvm/test/Transforms/CodeGenPrepare/X86/memset_chk-simplify-nobuiltin.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -S -disable-simplify-libcalls -codegenprepare < %s | FileCheck %s
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 
@@ -8,9 +9,11 @@ target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 ; - attribute nobuiltin
 ; - TLI::has (always returns false thanks to -disable-simplify-libcalls)
 
-; CHECK-NOT: _chk
-; CHECK: call void @llvm.memset.p0i8.i64(i8* align 1 %dst, i8 0, i64 %len, i1 false)
 define void @test_nobuiltin(i8* %dst, i64 %len) {
+; CHECK-LABEL: @test_nobuiltin(
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[DST:%.*]], i8 0, i64 [[LEN:%.*]], i1 false) #1
+; CHECK-NEXT:    ret void
+;
   call i8* @__memset_chk(i8* %dst, i32 0, i64 %len, i64 -1) nobuiltin
   ret void
 }
diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/overflow-intrinsics.ll b/llvm/test/Transforms/CodeGenPrepare/X86/overflow-intrinsics.ll
index fc6d66a60..9ba1d7db5 100644
--- a/llvm/test/Transforms/CodeGenPrepare/X86/overflow-intrinsics.ll
+++ b/llvm/test/Transforms/CodeGenPrepare/X86/overflow-intrinsics.ll
@@ -514,26 +514,6 @@ exit:
   ret void
 }
 
-; This was crashing when trying to delay instruction removal/deletion.
-
-declare i64 @llvm.objectsize.i64.p0i8(i8*, i1 immarg, i1 immarg, i1 immarg) #0
-
-define hidden fastcc void @crash() {
-; CHECK-LABEL: @crash(
-; CHECK-NEXT:    [[TMP1:%.*]] = call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 undef, i64 undef)
-; CHECK-NEXT:    [[MATH:%.*]] = extractvalue { i64, i1 } [[TMP1]], 0
-; CHECK-NEXT:    [[OV:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1
-; CHECK-NEXT:    [[T2:%.*]] = select i1 undef, i1 undef, i1 [[OV]]
-; CHECK-NEXT:    unreachable
-;
-  %t0 = add i64 undef, undef
-  %t1 = icmp ult i64 %t0, undef
-  %t2 = select i1 undef, i1 undef, i1 %t1
-  %t3 = call i64 @llvm.objectsize.i64.p0i8(i8* nonnull undef, i1 false, i1 false, i1 false)
-  %t4 = icmp ugt i64 %t3, 7
-  unreachable
-}
-
 ; Check that every instruction inserted by -codegenprepare has a debug location.
 ; DEBUG: CheckModuleDebugify: PASS
 
diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode.ll b/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode.ll
index 4d28e06f2..7a5a2ebd6 100644
--- a/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode.ll
+++ b/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode.ll
@@ -152,6 +152,30 @@ rare.1:
   br label %fallthrough
 }
 
+; Negative test - opt for size
+define void @test6_pgso(i1 %cond, i64* %base) !prof !14 {
+; CHECK-LABEL: @test6
+entry:
+; CHECK: %addr = getelementptr
+  %addr = getelementptr inbounds i64, i64* %base, i64 5
+  %casted = bitcast i64* %addr to i32*
+  br i1 %cond, label %if.then, label %fallthrough
+
+if.then:
+; CHECK-LABEL: if.then:
+; CHECK-NOT: getelementptr inbounds i8, {{.+}} 40
+  %v1 = load i32, i32* %casted, align 4
+  call void @foo(i32 %v1)
+  %cmp = icmp eq i32 %v1, 0
+  br i1 %cmp, label %rare.1, label %fallthrough
+
+fallthrough:
+  ret void
+
+rare.1:
+  call void @slowpath(i32 %v1, i32* %casted) cold
+  br label %fallthrough
+}
 
 ; Make sure sinking two copies of addressing mode into different blocks works
 ; when there are cold paths for each.
@@ -278,3 +302,20 @@ BB:
   store i1 false, i1* %G23
   ret void
 }
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 1, !"ProfileSummary", !1}
+!1 = !{!2, !3, !4, !5, !6, !7, !8, !9}
+!2 = !{!"ProfileFormat", !"InstrProf"}
+!3 = !{!"TotalCount", i64 10000}
+!4 = !{!"MaxCount", i64 10}
+!5 = !{!"MaxInternalCount", i64 1}
+!6 = !{!"MaxFunctionCount", i64 1000}
+!7 = !{!"NumCounts", i64 3}
+!8 = !{!"NumFunctions", i64 3}
+!9 = !{!"DetailedSummary", !10}
+!10 = !{!11, !12, !13}
+!11 = !{i32 10000, i64 100, i32 1}
+!12 = !{i32 999000, i64 100, i32 1}
+!13 = !{i32 999999, i64 1, i32 2}
+!14 = !{!"function_entry_count", i64 0}
diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/split-store-alignment.ll b/llvm/test/Transforms/CodeGenPrepare/X86/split-store-alignment.ll
new file mode 100644
index 000000000..7eb8cb8eb
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/X86/split-store-alignment.ll
@@ -0,0 +1,74 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -codegenprepare -mtriple=x86_64-unknown-unknown -force-split-store -S < %s | FileCheck %s
+
+target datalayout = "e-m:x-p:32:32-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:32-n8:16:32-a:0:32-S32"
+target triple = "i686-w64-windows-gnu"
+
+define void @split_store_align1(float %x, i64* %p) {
+; CHECK-LABEL: @split_store_align1(
+; CHECK-NEXT:    [[B:%.*]] = bitcast float [[X:%.*]] to i32
+; CHECK-NEXT:    [[Z:%.*]] = zext i32 0 to i64
+; CHECK-NEXT:    [[S:%.*]] = shl nuw nsw i64 [[Z]], 32
+; CHECK-NEXT:    [[Z2:%.*]] = zext i32 [[B]] to i64
+; CHECK-NEXT:    [[O:%.*]] = or i64 [[S]], [[Z2]]
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i64* [[P:%.*]] to i32*
+; CHECK-NEXT:    store i32 [[B]], i32* [[TMP1]], align 1
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i64* [[P]] to i32*
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr i32, i32* [[TMP2]], i32 1
+; CHECK-NEXT:    store i32 0, i32* [[TMP3]], align 1
+; CHECK-NEXT:    ret void
+;
+  %b = bitcast float %x to i32
+  %z = zext i32 0 to i64
+  %s = shl nuw nsw i64 %z, 32
+  %z2 = zext i32 %b to i64
+  %o = or i64 %s, %z2
+  store i64 %o, i64* %p, align 1
+  ret void
+}
+
+define void @split_store_align2(float %x, i64* %p) {
+; CHECK-LABEL: @split_store_align2(
+; CHECK-NEXT:    [[B:%.*]] = bitcast float [[X:%.*]] to i32
+; CHECK-NEXT:    [[Z:%.*]] = zext i32 0 to i64
+; CHECK-NEXT:    [[S:%.*]] = shl nuw nsw i64 [[Z]], 32
+; CHECK-NEXT:    [[Z2:%.*]] = zext i32 [[B]] to i64
+; CHECK-NEXT:    [[O:%.*]] = or i64 [[S]], [[Z2]]
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i64* [[P:%.*]] to i32*
+; CHECK-NEXT:    store i32 [[B]], i32* [[TMP1]], align 2
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i64* [[P]] to i32*
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr i32, i32* [[TMP2]], i32 1
+; CHECK-NEXT:    store i32 0, i32* [[TMP3]], align 2
+; CHECK-NEXT:    ret void
+;
+  %b = bitcast float %x to i32
+  %z = zext i32 0 to i64
+  %s = shl nuw nsw i64 %z, 32
+  %z2 = zext i32 %b to i64
+  %o = or i64 %s, %z2
+  store i64 %o, i64* %p, align 2
+  ret void
+}
+
+define void @split_store_align8(float %x, i64* %p) {
+; CHECK-LABEL: @split_store_align8(
+; CHECK-NEXT:    [[B:%.*]] = bitcast float [[X:%.*]] to i32
+; CHECK-NEXT:    [[Z:%.*]] = zext i32 0 to i64
+; CHECK-NEXT:    [[S:%.*]] = shl nuw nsw i64 [[Z]], 32
+; CHECK-NEXT:    [[Z2:%.*]] = zext i32 [[B]] to i64
+; CHECK-NEXT:    [[O:%.*]] = or i64 [[S]], [[Z2]]
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i64* [[P:%.*]] to i32*
+; CHECK-NEXT:    store i32 [[B]], i32* [[TMP1]], align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i64* [[P]] to i32*
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr i32, i32* [[TMP2]], i32 1
+; CHECK-NEXT:    store i32 0, i32* [[TMP3]], align 4
+; CHECK-NEXT:    ret void
+;
+  %b = bitcast float %x to i32
+  %z = zext i32 0 to i64
+  %s = shl nuw nsw i64 %z, 32
+  %z2 = zext i32 %b to i64
+  %o = or i64 %s, %z2
+  store i64 %o, i64* %p, align 8
+  ret void
+}
diff --git a/llvm/test/Transforms/CodeGenPrepare/bitreverse-hang.ll b/llvm/test/Transforms/CodeGenPrepare/bitreverse-hang.ll
index 340440556..4abc57d84 100644
--- a/llvm/test/Transforms/CodeGenPrepare/bitreverse-hang.ll
+++ b/llvm/test/Transforms/CodeGenPrepare/bitreverse-hang.ll
@@ -40,7 +40,7 @@ for.end:                                          ; preds = %for.body
   ret i32 undef
 }
 
-attributes #0 = { norecurse nounwind ssp uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="core2" "target-features"="+cx16,+fxsr,+mmx,+sse,+sse2,+sse3,+ssse3" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { norecurse nounwind ssp uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="core2" "target-features"="+cx16,+fxsr,+mmx,+sse,+sse2,+sse3,+ssse3" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.module.flags = !{!0}
 !llvm.ident = !{!1}
diff --git a/llvm/test/Transforms/CodeGenPrepare/dom-tree.ll b/llvm/test/Transforms/CodeGenPrepare/dom-tree.ll
index 28c3b94dc..cd90debcc 100644
--- a/llvm/test/Transforms/CodeGenPrepare/dom-tree.ll
+++ b/llvm/test/Transforms/CodeGenPrepare/dom-tree.ll
@@ -29,7 +29,7 @@ for.body:
   br i1 %exitcond, label %for.cond.cleanup, label %for.body, !llvm.loop !3
 }
 
-attributes #0 = { norecurse nounwind readnone "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="cortex-a8" "target-features"="+dsp,+neon,+vfp3" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { norecurse nounwind readnone "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="cortex-a8" "target-features"="+dsp,+neon,+vfp3" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.module.flags = !{!0, !1}
 !llvm.ident = !{!2}
diff --git a/llvm/test/Transforms/CodeGenPrepare/sink-shift-and-trunc.ll b/llvm/test/Transforms/CodeGenPrepare/sink-shift-and-trunc.ll
index 609b6bf67..929a4f43e 100644
--- a/llvm/test/Transforms/CodeGenPrepare/sink-shift-and-trunc.ll
+++ b/llvm/test/Transforms/CodeGenPrepare/sink-shift-and-trunc.ll
@@ -1,5 +1,5 @@
 ; REQUIRES: aarch64-registered-target
-; RUN: opt -codegenprepare -mtriple=arm64-apple=ios -S -o - %s | FileCheck %s
+; RUN: opt < %s -codegenprepare -mtriple=arm64-apple-ios -S | FileCheck %s
 
 @first_ones = external global [65536 x i8]
 
diff --git a/llvm/test/Transforms/ConstProp/InsertElement.ll b/llvm/test/Transforms/ConstProp/InsertElement.ll
index 011ad3f57..1048b5752 100644
--- a/llvm/test/Transforms/ConstProp/InsertElement.ll
+++ b/llvm/test/Transforms/ConstProp/InsertElement.ll
@@ -1,32 +1,53 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -constprop -S | FileCheck %s
 
-; CHECK-LABEL: @test1
 define i32 @test1() {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    ret i32 2139171423
+;
   %A = bitcast i32 2139171423 to float
   %B = insertelement <1 x float> undef, float %A, i32 0
   %C = extractelement <1 x float> %B, i32 0
   %D = bitcast float %C to i32
   ret i32 %D
-; CHECK: ret i32 2139171423
 }
 
-; CHECK-LABEL: @insertelement
 define <4 x i64> @insertelement() {
+; CHECK-LABEL: @insertelement(
+; CHECK-NEXT:    ret <4 x i64> <i64 -1, i64 -2, i64 -3, i64 -4>
+;
   %vec1 = insertelement <4 x i64> undef, i64 -1, i32 0
   %vec2 = insertelement <4 x i64> %vec1, i64 -2, i32 1
   %vec3 = insertelement <4 x i64> %vec2, i64 -3, i32 2
   %vec4 = insertelement <4 x i64> %vec3, i64 -4, i32 3
-  ; CHECK: ret <4 x i64> <i64 -1, i64 -2, i64 -3, i64 -4>
   ret <4 x i64> %vec4
 }
 
-; CHECK-LABEL: @insertelement_undef
 define <4 x i64> @insertelement_undef() {
+; CHECK-LABEL: @insertelement_undef(
+; CHECK-NEXT:    [[VEC4:%.*]] = insertelement <4 x i64> <i64 -1, i64 -2, i64 -3, i64 undef>, i64 -4, i32 3
+; CHECK-NEXT:    ret <4 x i64> undef
+;
   %vec1 = insertelement <4 x i64> undef, i64 -1, i32 0
   %vec2 = insertelement <4 x i64> %vec1, i64 -2, i32 1
   %vec3 = insertelement <4 x i64> %vec2, i64 -3, i32 2
   %vec4 = insertelement <4 x i64> %vec3, i64 -4, i32 3
   %vec5 = insertelement <4 x i64> %vec3, i64 -5, i32 4
-  ; CHECK: ret <4 x i64> undef
   ret <4 x i64> %vec5
 }
+
+define i64 @extract_undef_index_from_zero_vec() {
+; CHECK-LABEL: @extract_undef_index_from_zero_vec(
+; CHECK-NEXT:    ret i64 undef
+;
+  %E = extractelement <2 x i64> zeroinitializer, i64 undef
+  ret i64 %E
+}
+
+define i64 @extract_undef_index_from_nonzero_vec() {
+; CHECK-LABEL: @extract_undef_index_from_nonzero_vec(
+; CHECK-NEXT:    ret i64 undef
+;
+  %E = extractelement <2 x i64> <i64 -1, i64 -1>, i64 undef
+  ret i64 %E
+}
diff --git a/llvm/test/Transforms/ConstProp/fma.ll b/llvm/test/Transforms/ConstProp/fma.ll
new file mode 100644
index 000000000..9119548a9
--- /dev/null
+++ b/llvm/test/Transforms/ConstProp/fma.ll
@@ -0,0 +1,246 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -constprop -S < %s | FileCheck %s
+
+; Fixes PR20832
+; Make sure that we correctly fold a fused multiply-add where operands
+; are all finite constants and addend is zero.
+
+declare double @llvm.fma.f64(double, double, double)
+
+
+define double @PR20832()  {
+; CHECK-LABEL: @PR20832(
+; CHECK-NEXT:    ret double 5.600000e+01
+;
+  %1 = call double @llvm.fma.f64(double 7.0, double 8.0, double 0.0)
+  ret double %1
+}
+
+; Test builtin fma with all finite non-zero constants.
+define double @test_all_finite()  {
+; CHECK-LABEL: @test_all_finite(
+; CHECK-NEXT:    ret double 6.100000e+01
+;
+  %1 = call double @llvm.fma.f64(double 7.0, double 8.0, double 5.0)
+  ret double %1
+}
+
+; Test builtin fma with a +/-NaN addend.
+define double @test_NaN_addend()  {
+; CHECK-LABEL: @test_NaN_addend(
+; CHECK-NEXT:    ret double 0x7FF8000000000000
+;
+  %1 = call double @llvm.fma.f64(double 7.0, double 8.0, double 0x7FF8000000000000)
+  ret double %1
+}
+
+define double @test_NaN_addend_2()  {
+; CHECK-LABEL: @test_NaN_addend_2(
+; CHECK-NEXT:    ret double 0xFFF8000000000000
+;
+  %1 = call double @llvm.fma.f64(double 7.0, double 8.0, double 0xFFF8000000000000)
+  ret double %1
+}
+
+; Test builtin fma with a +/-Inf addend.
+define double @test_Inf_addend()  {
+; CHECK-LABEL: @test_Inf_addend(
+; CHECK-NEXT:    ret double 0x7FF0000000000000
+;
+  %1 = call double @llvm.fma.f64(double 7.0, double 8.0, double 0x7FF0000000000000)
+  ret double %1
+}
+
+define double @test_Inf_addend_2()  {
+; CHECK-LABEL: @test_Inf_addend_2(
+; CHECK-NEXT:    ret double 0xFFF0000000000000
+;
+  %1 = call double @llvm.fma.f64(double 7.0, double 8.0, double 0xFFF0000000000000)
+  ret double %1
+}
+
+; Test builtin fma with one of the operands to the multiply being +/-NaN.
+define double @test_NaN_1()  {
+; CHECK-LABEL: @test_NaN_1(
+; CHECK-NEXT:    ret double 0x7FF8000000000000
+;
+  %1 = call double @llvm.fma.f64(double 0x7FF8000000000000, double 8.0, double 0.0)
+  ret double %1
+}
+
+define double @test_NaN_2()  {
+; CHECK-LABEL: @test_NaN_2(
+; CHECK-NEXT:    ret double 0x7FF8000000000000
+;
+  %1 = call double @llvm.fma.f64(double 7.0, double 0x7FF8000000000000, double 0.0)
+  ret double %1
+}
+
+define double @test_NaN_3()  {
+; CHECK-LABEL: @test_NaN_3(
+; CHECK-NEXT:    ret double 0x7FF8000000000000
+;
+  %1 = call double @llvm.fma.f64(double 0xFFF8000000000000, double 8.0, double 0.0)
+  ret double %1
+}
+
+define double @test_NaN_4()  {
+; CHECK-LABEL: @test_NaN_4(
+; CHECK-NEXT:    ret double 0x7FF8000000000000
+;
+  %1 = call double @llvm.fma.f64(double 7.0, double 0xFFF8000000000000, double 0.0)
+  ret double %1
+}
+
+; Test builtin fma with one of the operands to the multiply being +/-Inf.
+define double @test_Inf_1()  {
+; CHECK-LABEL: @test_Inf_1(
+; CHECK-NEXT:    ret double 0x7FF0000000000000
+;
+  %1 = call double @llvm.fma.f64(double 0x7FF0000000000000, double 8.0, double 0.0)
+  ret double %1
+}
+
+define double @test_Inf_2()  {
+; CHECK-LABEL: @test_Inf_2(
+; CHECK-NEXT:    ret double 0x7FF0000000000000
+;
+  %1 = call double @llvm.fma.f64(double 7.0, double 0x7FF0000000000000, double 0.0)
+  ret double %1
+}
+
+define double @test_Inf_3()  {
+; CHECK-LABEL: @test_Inf_3(
+; CHECK-NEXT:    ret double 0xFFF0000000000000
+;
+  %1 = call double @llvm.fma.f64(double 0xFFF0000000000000, double 8.0, double 0.0)
+  ret double %1
+}
+
+define double @test_Inf_4()  {
+; CHECK-LABEL: @test_Inf_4(
+; CHECK-NEXT:    ret double 0xFFF0000000000000
+;
+  %1 = call double @llvm.fma.f64(double 7.0, double 0xFFF0000000000000, double 0.0)
+  ret double %1
+}
+
+; -inf + inf --> NaN
+
+define double @inf_product_opposite_inf_addend_1()  {
+; CHECK-LABEL: @inf_product_opposite_inf_addend_1(
+; CHECK-NEXT:    ret double 0x7FF8000000000000
+;
+  %1 = call double @llvm.fma.f64(double 7.0, double 0xFFF0000000000000, double 0x7FF0000000000000)
+  ret double %1
+}
+
+; inf + -inf --> NaN
+
+define double @inf_product_opposite_inf_addend_2()  {
+; CHECK-LABEL: @inf_product_opposite_inf_addend_2(
+; CHECK-NEXT:    ret double 0x7FF8000000000000
+;
+  %1 = call double @llvm.fma.f64(double 7.0, double 0x7FF0000000000000, double 0xFFF0000000000000)
+  ret double %1
+}
+
+; -inf + inf --> NaN
+
+define double @inf_product_opposite_inf_addend_3()  {
+; CHECK-LABEL: @inf_product_opposite_inf_addend_3(
+; CHECK-NEXT:    ret double 0x7FF8000000000000
+;
+  %1 = call double @llvm.fma.f64(double 0xFFF0000000000000, double 42.0, double 0x7FF0000000000000)
+  ret double %1
+}
+
+; inf + -inf --> NaN
+
+define double @inf_product_opposite_inf_addend_4()  {
+; CHECK-LABEL: @inf_product_opposite_inf_addend_4(
+; CHECK-NEXT:    ret double 0x7FF8000000000000
+;
+  %1 = call double @llvm.fma.f64(double 0x7FF0000000000000, double 42.0, double 0xFFF0000000000000)
+  ret double %1
+}
+
+; 0 * -inf --> NaN
+
+define double @inf_times_zero_1()  {
+; CHECK-LABEL: @inf_times_zero_1(
+; CHECK-NEXT:    ret double 0x7FF8000000000000
+;
+  %1 = call double @llvm.fma.f64(double 0.0, double 0xFFF0000000000000, double 42.0)
+  ret double %1
+}
+
+; 0 * inf --> NaN
+
+define double @inf_times_zero_2()  {
+; CHECK-LABEL: @inf_times_zero_2(
+; CHECK-NEXT:    ret double 0x7FF8000000000000
+;
+  %1 = call double @llvm.fma.f64(double 0.0, double 0x7FF0000000000000, double 42.0)
+  ret double %1
+}
+
+; -inf * 0 --> NaN
+
+define double @inf_times_zero_3()  {
+; CHECK-LABEL: @inf_times_zero_3(
+; CHECK-NEXT:    ret double 0x7FF8000000000000
+;
+  %1 = call double @llvm.fma.f64(double 0xFFF0000000000000, double 0.0, double 42.0)
+  ret double %1
+}
+
+; inf * 0 --> NaN
+
+define double @inf_times_zero_4()  {
+; CHECK-LABEL: @inf_times_zero_4(
+; CHECK-NEXT:    ret double 0x7FF8000000000000
+;
+  %1 = call double @llvm.fma.f64(double 0x7FF0000000000000, double 0.0, double 42.0)
+  ret double %1
+}
+
+; -0 * -inf --> NaN
+
+define double @inf_times_zero_5()  {
+; CHECK-LABEL: @inf_times_zero_5(
+; CHECK-NEXT:    ret double 0x7FF8000000000000
+;
+  %1 = call double @llvm.fma.f64(double -0.0, double 0xFFF0000000000000, double 42.0)
+  ret double %1
+}
+
+; -0 * inf --> NaN
+
+define double @inf_times_zero_6()  {
+; CHECK-LABEL: @inf_times_zero_6(
+; CHECK-NEXT:    ret double 0x7FF8000000000000
+;
+  %1 = call double @llvm.fma.f64(double -0.0, double 0x7FF0000000000000, double 42.0)
+  ret double %1
+}
+
+; -inf * -0 --> NaN
+
+define double @inf_times_zero_7()  {
+; CHECK-LABEL: @inf_times_zero_7(
+; CHECK-NEXT:    ret double 0x7FF8000000000000
+;
+  %1 = call double @llvm.fma.f64(double 0xFFF0000000000000, double -0.0, double 42.0)
+  ret double %1
+}
+
+; inf * -0 --> NaN
+
+define double @inf_times_zero_8()  {
+; CHECK-LABEL: @inf_times_zero_8(
+; CHECK-NEXT:    ret double 0x7FF8000000000000
+;
+  %1 = call double @llvm.fma.f64(double 0x7FF0000000000000, double -0.0, double 42.0)
+  ret double %1
+}
diff --git a/llvm/test/Transforms/ConstProp/overflow-ops.ll b/llvm/test/Transforms/ConstProp/overflow-ops.ll
index 303b3b90a..1d090d5d0 100644
--- a/llvm/test/Transforms/ConstProp/overflow-ops.ll
+++ b/llvm/test/Transforms/ConstProp/overflow-ops.ll
@@ -1,5 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -constprop -S | FileCheck %s
+; RUN: opt < %s -constprop -S | FileCheck %s --check-prefixes=CHECK,CONSTPROP
+; RUN: opt < %s -instsimplify -S | FileCheck %s --check-prefixes=CHECK,INSTSIMPLIFY
+; We must *NOT* have any check-lines with prefixes other than CHECK here.
+; If we do, that means the rules are different between the passes.
 
 declare {i8, i1} @llvm.uadd.with.overflow.i8(i8, i8)
 declare {i8, i1} @llvm.usub.with.overflow.i8(i8, i8)
@@ -31,7 +34,7 @@ define {i8, i1} @uadd_2() nounwind {
 
 define {i8, i1} @uadd_undef() nounwind {
 ; CHECK-LABEL: @uadd_undef(
-; CHECK-NEXT:    ret { i8, i1 } undef
+; CHECK-NEXT:    ret { i8, i1 } { i8 undef, i1 false }
 ;
   %t = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 142, i8 undef)
   ret {i8, i1} %t
@@ -59,7 +62,7 @@ define {i8, i1} @usub_2() nounwind {
 
 define {i8, i1} @usub_undef() nounwind {
 ; CHECK-LABEL: @usub_undef(
-; CHECK-NEXT:    ret { i8, i1 } undef
+; CHECK-NEXT:    ret { i8, i1 } { i8 undef, i1 false }
 ;
   %t = call {i8, i1} @llvm.usub.with.overflow.i8(i8 4, i8 undef)
   ret {i8, i1} %t
@@ -147,7 +150,7 @@ define {i8, i1} @sadd_5() nounwind {
 
 define {i8, i1} @sadd_undef() nounwind {
 ; CHECK-LABEL: @sadd_undef(
-; CHECK-NEXT:    ret { i8, i1 } undef
+; CHECK-NEXT:    ret { i8, i1 } { i8 undef, i1 false }
 ;
   %t = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 undef, i8 -10)
   ret {i8, i1} %t
@@ -215,7 +218,7 @@ define {i8, i1} @ssub_5() nounwind {
 
 define {i8, i1} @ssub_undef() nounwind {
 ; CHECK-LABEL: @ssub_undef(
-; CHECK-NEXT:    ret { i8, i1 } undef
+; CHECK-NEXT:    ret { i8, i1 } { i8 undef, i1 false }
 ;
   %t = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 undef, i8 -10)
   ret {i8, i1} %t
diff --git a/llvm/test/Transforms/ConstantHoisting/AArch64/const-hoist-intrinsics.ll b/llvm/test/Transforms/ConstantHoisting/AArch64/const-hoist-intrinsics.ll
new file mode 100644
index 000000000..92007ef42
--- /dev/null
+++ b/llvm/test/Transforms/ConstantHoisting/AArch64/const-hoist-intrinsics.ll
@@ -0,0 +1,89 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -mtriple=arm64-darwin-unknown -S -consthoist < %s | FileCheck %s
+
+; Make sure we hoist constants out of intrinsics.
+
+define void @test_stxr(i64* %ptr) {
+; CHECK-LABEL: @test_stxr(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CONST:%.*]] = bitcast i64 -9223372036317904832 to i64
+; CHECK-NEXT:    [[PTR_0:%.*]] = getelementptr i64, i64* [[PTR:%.*]], i64 0
+; CHECK-NEXT:    [[CONST_MAT:%.*]] = add i64 [[CONST]], -64
+; CHECK-NEXT:    [[BAR_0:%.*]] = call i32 @llvm.aarch64.stxr.p0i64(i64 [[CONST_MAT]], i64* [[PTR_0]])
+; CHECK-NEXT:    [[PTR_1:%.*]] = getelementptr i64, i64* [[PTR]], i64 1
+; CHECK-NEXT:    [[BAR_1:%.*]] = call i32 @llvm.aarch64.stxr.p0i64(i64 [[CONST]], i64* [[PTR_1]])
+; CHECK-NEXT:    [[PTR_2:%.*]] = getelementptr i64, i64* [[PTR]], i64 2
+; CHECK-NEXT:    [[CONST_MAT1:%.*]] = add i64 [[CONST]], 64
+; CHECK-NEXT:    [[BAR_2:%.*]] = call i32 @llvm.aarch64.stxr.p0i64(i64 [[CONST_MAT1]], i64* [[PTR_2]])
+; CHECK-NEXT:    [[PTR_3:%.*]] = getelementptr i64, i64* [[PTR]], i64 3
+; CHECK-NEXT:    [[CONST_MAT2:%.*]] = add i64 [[CONST]], 128
+; CHECK-NEXT:    [[BAR_3:%.*]] = call i32 @llvm.aarch64.stxr.p0i64(i64 [[CONST_MAT2]], i64* [[PTR_3]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %ptr.0 = getelementptr i64, i64* %ptr, i64 0
+  %bar.0 = call i32 @llvm.aarch64.stxr.p0i64(i64 -9223372036317904896, i64* %ptr.0)
+  %ptr.1 = getelementptr i64, i64* %ptr, i64 1
+  %bar.1 = call i32 @llvm.aarch64.stxr.p0i64(i64 -9223372036317904832,  i64* %ptr.1)
+  %ptr.2 = getelementptr i64, i64* %ptr, i64 2
+  %bar.2 = call i32 @llvm.aarch64.stxr.p0i64(i64 -9223372036317904768, i64* %ptr.2)
+  %ptr.3 = getelementptr i64, i64* %ptr, i64 3
+  %bar.3 = call i32 @llvm.aarch64.stxr.p0i64(i64 -9223372036317904704, i64* %ptr.3)
+  ret void
+}
+
+declare i32 @llvm.aarch64.stxr.p0i64(i64 , i64*)
+
+define i64 @test_udiv(i64 %x) {
+; CHECK-LABEL: @test_udiv(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CONST:%.*]] = bitcast i64 -9223372036317904832 to i64
+; CHECK-NEXT:    [[CONST_MAT:%.*]] = add i64 [[CONST]], -64
+; CHECK-NEXT:    [[BAR_0:%.*]] = call i64 @llvm.aarch64.udiv.i64(i64 [[CONST_MAT]], i64 [[X:%.*]])
+; CHECK-NEXT:    [[BAR_1:%.*]] = call i64 @llvm.aarch64.udiv.i64(i64 [[CONST]], i64 [[X]])
+; CHECK-NEXT:    [[CONST_MAT1:%.*]] = add i64 [[CONST]], 64
+; CHECK-NEXT:    [[BAR_2:%.*]] = call i64 @llvm.aarch64.udiv.i64(i64 [[CONST_MAT1]], i64 [[X]])
+; CHECK-NEXT:    [[CONST_MAT2:%.*]] = add i64 [[CONST]], 128
+; CHECK-NEXT:    [[BAR_3:%.*]] = call i64 @llvm.aarch64.udiv.i64(i64 [[CONST_MAT2]], i64 [[X]])
+; CHECK-NEXT:    [[RES_1:%.*]] = add i64 [[BAR_0]], [[BAR_1]]
+; CHECK-NEXT:    [[RES_2:%.*]] = add i64 [[RES_1]], [[BAR_2]]
+; CHECK-NEXT:    [[RES_3:%.*]] = add i64 [[RES_2]], [[BAR_3]]
+; CHECK-NEXT:    ret i64 [[RES_3]]
+;
+entry:
+  %bar.0 = call i64 @llvm.aarch64.udiv.i64.i64(i64 -9223372036317904896, i64 %x)
+  %bar.1 = call i64 @llvm.aarch64.udiv.i64.i64(i64 -9223372036317904832,  i64 %x)
+  %bar.2 = call i64 @llvm.aarch64.udiv.i64.i64(i64 -9223372036317904768, i64 %x)
+  %bar.3 = call i64 @llvm.aarch64.udiv.i64.i64(i64 -9223372036317904704, i64 %x)
+  %res.1 = add i64 %bar.0, %bar.1
+  %res.2 = add i64 %res.1, %bar.2
+  %res.3 = add i64 %res.2, %bar.3
+  ret i64 %res.3
+}
+
+declare i64 @llvm.aarch64.udiv.i64.i64(i64, i64)
+
+define void @test_free_intrinsics(i64 %x, i8* %ptr) {
+; CHECK-LABEL: @test_free_intrinsics(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0i8(i64 100000000032, i8* [[PTR:%.*]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0i8(i64 100000000064, i8* [[PTR]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0i8(i64 100000000128, i8* [[PTR]])
+; CHECK-NEXT:    [[I:%.*]] = call {}* @llvm.invariant.start.p0i8(i64 100000000256, i8* [[PTR]])
+; CHECK-NEXT:    call void @llvm.invariant.end.p0i8({}* [[I]], i64 100000000256, i8* [[PTR]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  call void @llvm.lifetime.start.p0i8(i64 100000000032, i8* %ptr)
+  call void @llvm.lifetime.start.p0i8(i64 100000000064, i8* %ptr)
+  call void @llvm.lifetime.end.p0i8(i64 100000000128, i8* %ptr)
+  %i = call {}* @llvm.invariant.start.p0i8(i64 100000000256, i8* %ptr)
+  call void @llvm.invariant.end.p0i8({}* %i, i64 100000000256, i8* %ptr)
+  ret void
+}
+
+declare void @llvm.lifetime.start.p0i8(i64, i8*)
+declare void @llvm.lifetime.end.p0i8(i64, i8*)
+
+declare {}* @llvm.invariant.start.p0i8(i64, i8* nocapture)
+declare void @llvm.invariant.end.p0i8({}*, i64, i8* nocapture)
diff --git a/llvm/test/Transforms/ConstantHoisting/AArch64/consthoist-unreachable.ll b/llvm/test/Transforms/ConstantHoisting/AArch64/consthoist-unreachable.ll
new file mode 100755
index 000000000..6f8d00f66
--- /dev/null
+++ b/llvm/test/Transforms/ConstantHoisting/AArch64/consthoist-unreachable.ll
@@ -0,0 +1,64 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mtriple aarch64-- -consthoist -S | FileCheck %s
+
+; This used to trigger an assertion failure:
+;
+;    ../lib/Transforms/Scalar/ConstantHoisting.cpp:779: void llvm::ConstantHoistingPass::emitBaseConstants(llvm::Instruction *, llvm::Constant *, llvm::Type *, const llvm::consthoist::ConstantUser &): Assertion `CastInst->isCast() && "Expected an cast instruction!"' failed.
+
+@c.a = external global i32, align 1
+
+define void @c() {
+; CHECK-LABEL: @c(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne i16 0, 0
+; CHECK-NEXT:    br i1 undef, label [[LBL1_US:%.*]], label [[ENTRY_ENTRY_SPLIT_CRIT_EDGE:%.*]]
+; CHECK:       entry.entry.split_crit_edge:
+; CHECK-NEXT:    [[CONST:%.*]] = bitcast i32 1232131 to i32
+; CHECK-NEXT:    br label [[LBL1:%.*]]
+; CHECK:       lbl1.us:
+; CHECK-NEXT:    [[CONST1:%.*]] = bitcast i32 1232131 to i32
+; CHECK-NEXT:    store i32 [[CONST1]], i32* @c.a, align 1
+; CHECK-NEXT:    br label [[FOR_COND4:%.*]]
+; CHECK:       lbl1:
+; CHECK-NEXT:    store i32 [[CONST]], i32* @c.a, align 1
+; CHECK-NEXT:    br i1 undef, label [[IF_THEN:%.*]], label [[FOR_END12:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    br i1 undef, label [[LBL1]], label [[FOR_COND4]]
+; CHECK:       for.cond4:
+; CHECK-NEXT:    br label [[FOR_COND4]]
+; CHECK:       for.body9:
+; CHECK-NEXT:    store i32 1232131, i32* undef, align 1
+; CHECK-NEXT:    store i32 1232132, i32* undef, align 1
+; CHECK-NEXT:    br label [[FOR_BODY9:%.*]]
+; CHECK:       for.end12:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %tobool = icmp ne i16 0, 0
+  br i1 undef, label %lbl1.us, label %entry.entry.split_crit_edge
+
+entry.entry.split_crit_edge:                      ; preds = %entry
+  br label %lbl1
+
+lbl1.us:                                          ; preds = %entry
+  store i32 1232131, i32* @c.a, align 1
+  br label %for.cond4
+
+lbl1:                                             ; preds = %if.then, %entry.entry.split_crit_edge
+  store i32 1232131, i32* @c.a, align 1
+  br i1 undef, label %if.then, label %for.end12
+
+if.then:                                          ; preds = %lbl1
+  br i1 undef, label %lbl1, label %for.cond4
+
+for.cond4:                                        ; preds = %for.cond4, %if.then, %lbl1.us
+  br label %for.cond4
+
+for.body9:                                        ; preds = %for.body9
+  store i32 1232131, i32* undef, align 1
+  store i32 1232132, i32* undef, align 1
+  br label %for.body9
+
+for.end12:                                        ; preds = %lbl1
+  ret void
+}
diff --git a/llvm/test/Transforms/ConstantHoisting/X86/ehpad.ll b/llvm/test/Transforms/ConstantHoisting/X86/ehpad.ll
index 5e345c451..5be600497 100644
--- a/llvm/test/Transforms/ConstantHoisting/X86/ehpad.ll
+++ b/llvm/test/Transforms/ConstantHoisting/X86/ehpad.ll
@@ -66,5 +66,5 @@ declare i64 @fn(i64) local_unnamed_addr #1
 
 declare i32 @__CxxFrameHandler3(...)
 
-attributes #0 = { norecurse "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "stack-protector-buffer-size"="8" "target-features"="+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "stack-protector-buffer-size"="8" "target-features"="+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { norecurse "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "stack-protector-buffer-size"="8" "target-features"="+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "stack-protector-buffer-size"="8" "target-features"="+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/Transforms/ConstantHoisting/X86/pr43903-not-all-uses-rebased.ll b/llvm/test/Transforms/ConstantHoisting/X86/pr43903-not-all-uses-rebased.ll
new file mode 100644
index 000000000..a5ca9b4b8
--- /dev/null
+++ b/llvm/test/Transforms/ConstantHoisting/X86/pr43903-not-all-uses-rebased.ll
@@ -0,0 +1,51 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -consthoist -consthoist-gep=1 -S | FileCheck %s
+
+; This is a reproducer for PR43903 where we hit an assertion:
+; opt: ../lib/Transforms/Scalar/ConstantHoisting.cpp:903: bool llvm::ConstantHoistingPass::emitBaseConstants(llvm::GlobalVariable *): Assertion `UsesNum == (ReBasesNum + NotRebasedNum) && "Not all uses are rebased"' failed.
+
+target triple = "x86_64-unknown-linux-gnu"
+
+@a = external global [2 x i16], align 1
+
+define void @c() {
+; CHECK-LABEL: @c(
+; CHECK-NEXT:  for.cond:
+; CHECK-NEXT:    br i1 undef, label [[FOR_BODY2:%.*]], label [[FOR_END4:%.*]]
+; CHECK:       for.body2:
+; CHECK-NEXT:    br i1 undef, label [[LAND_RHS:%.*]], label [[LAND_END:%.*]]
+; CHECK:       land.rhs:
+; CHECK-NEXT:    unreachable
+; CHECK:       land.end:
+; CHECK-NEXT:    [[CONST1:%.*]] = bitcast i16* getelementptr inbounds ([2 x i16], [2 x i16]* @a, i32 0, i32 0) to i16*
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ule i16* undef, [[CONST1]]
+; CHECK-NEXT:    unreachable
+; CHECK:       for.cond3:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i16, i16* getelementptr inbounds ([2 x i16], [2 x i16]* @a, i32 0, i32 1), align 1
+; CHECK-NEXT:    br label [[FOR_COND3:%.*]]
+; CHECK:       for.end4:
+; CHECK-NEXT:    [[CONST:%.*]] = bitcast i16* getelementptr inbounds ([2 x i16], [2 x i16]* @a, i32 0, i32 0) to i16*
+; CHECK-NEXT:    [[TMP1:%.*]] = load i16, i16* [[CONST]], align 1
+; CHECK-NEXT:    ret void
+;
+for.cond:
+  br i1 undef, label %for.body2, label %for.end4
+
+for.body2:                                        ; preds = %for.cond
+  br i1 undef, label %land.rhs, label %land.end
+
+land.rhs:                                         ; preds = %for.body2
+  unreachable
+
+land.end:                                         ; preds = %for.body2
+  %cmp = icmp ule i16* undef, getelementptr inbounds ([2 x i16], [2 x i16]* @a, i32 0, i32 0)
+  unreachable
+
+for.cond3:                                        ; preds = %for.cond3
+  %tmp0 = load i16, i16* getelementptr inbounds ([2 x i16], [2 x i16]* @a, i32 0, i32 1), align 1
+  br label %for.cond3
+
+for.end4:                                         ; preds = %for.cond
+  %tmp1 = load i16, i16* getelementptr inbounds ([2 x i16], [2 x i16]* @a, i32 0, i32 0), align 1
+  ret void
+}
diff --git a/llvm/test/Transforms/Coroutines/coro-alloc-with-param.ll b/llvm/test/Transforms/Coroutines/coro-alloc-with-param.ll
new file mode 100644
index 000000000..ce0975f10
--- /dev/null
+++ b/llvm/test/Transforms/Coroutines/coro-alloc-with-param.ll
@@ -0,0 +1,96 @@
+; Check that we can handle the case when both alloc function and
+; the user body consume the same argument.
+; RUN: opt < %s -coro-split -S | FileCheck %s
+
+; using this directly (as it would happen under -O2)
+define i8* @f_direct(i64 %this) "coroutine.presplit"="1" {
+entry:
+  %id = call token @llvm.coro.id(i32 0, i8* null, i8* null, i8* null)
+  %size = call i32 @llvm.coro.size.i32()
+  %alloc = call i8* @myAlloc(i64 %this, i32 %size)
+  %hdl = call i8* @llvm.coro.begin(token %id, i8* %alloc)
+  %0 = call i8 @llvm.coro.suspend(token none, i1 false)
+  switch i8 %0, label %suspend [i8 0, label %resume
+                                i8 1, label %cleanup]
+resume:
+  call void @print2(i64 %this)
+  br label %cleanup
+
+cleanup:
+  %mem = call i8* @llvm.coro.free(token %id, i8* %hdl)
+  call void @free(i8* %mem)
+  br label %suspend
+suspend:
+  call i1 @llvm.coro.end(i8* %hdl, i1 0)
+  ret i8* %hdl
+}
+
+; using copy of this (as it would happen under -O0)
+define i8* @f_copy(i64 %this_arg) "coroutine.presplit"="1" {
+entry:
+  %this.addr = alloca i64
+  store i64 %this_arg, i64* %this.addr
+  %this = load i64, i64* %this.addr
+  %id = call token @llvm.coro.id(i32 0, i8* null, i8* null, i8* null)
+  %size = call i32 @llvm.coro.size.i32()
+  %alloc = call i8* @myAlloc(i64 %this, i32 %size)
+  %hdl = call i8* @llvm.coro.begin(token %id, i8* %alloc)
+  %0 = call i8 @llvm.coro.suspend(token none, i1 false)
+  switch i8 %0, label %suspend [i8 0, label %resume
+                                i8 1, label %cleanup]
+resume:
+  call void @print2(i64 %this)
+  br label %cleanup
+
+cleanup:
+  %mem = call i8* @llvm.coro.free(token %id, i8* %hdl)
+  call void @free(i8* %mem)
+  br label %suspend
+suspend:
+  call i1 @llvm.coro.end(i8* %hdl, i1 0)
+  ret i8* %hdl
+}
+
+; See if %this was added to the frame
+; CHECK: %f_direct.Frame = type { void (%f_direct.Frame*)*, void (%f_direct.Frame*)*, i1, i1, i64 }
+; CHECK: %f_copy.Frame = type { void (%f_copy.Frame*)*, void (%f_copy.Frame*)*, i1, i1, i64 }
+
+; See that %this is spilled into the frame
+; CHECK-LABEL: define i8* @f_direct(i64 %this)
+; CHECK: %this.spill.addr = getelementptr inbounds %f_direct.Frame, %f_direct.Frame* %FramePtr, i32 0, i32 4
+; CHECK: store i64 %this, i64* %this.spill.addr
+; CHECK: ret i8* %hdl
+
+; See that %this is spilled into the frame
+; CHECK-LABEL: define i8* @f_copy(i64 %this_arg)
+; CHECK:  %this.spill.addr = getelementptr inbounds %f_copy.Frame, %f_copy.Frame* %FramePtr, i32 0, i32 4
+; CHECK:  store i64 %this_arg, i64* %this.spill.addr
+; CHECK: ret i8* %hdl
+
+; See that %this was loaded from the frame
+; CHECK-LABEL: @f_direct.resume(
+; CHECK:  %this.reload = load i64, i64* %this.reload.addr
+; CHECK:  call void @print2(i64 %this.reload)
+; CHECK: ret void
+
+; See that %this was loaded from the frame
+; CHECK-LABEL: @f_copy.resume(
+; CHECK:  %this.reload = load i64, i64* %this.reload.addr
+; CHECK:  call void @print2(i64 %this.reload)
+; CHECK: ret void
+
+declare i8* @llvm.coro.free(token, i8*)
+declare i32 @llvm.coro.size.i32()
+declare i8  @llvm.coro.suspend(token, i1)
+declare void @llvm.coro.resume(i8*)
+declare void @llvm.coro.destroy(i8*)
+
+declare token @llvm.coro.id(i32, i8*, i8*, i8*)
+declare i1 @llvm.coro.alloc(token)
+declare i8* @llvm.coro.begin(token, i8*)
+declare i1 @llvm.coro.end(i8*, i1)
+
+declare noalias i8* @myAlloc(i64, i32)
+declare double @print(double)
+declare void @print2(i64)
+declare void @free(i8*)
diff --git a/llvm/test/Transforms/Coroutines/coro-debug.ll b/llvm/test/Transforms/Coroutines/coro-debug.ll
index e9e61ed64..1d40ddd67 100644
--- a/llvm/test/Transforms/Coroutines/coro-debug.ll
+++ b/llvm/test/Transforms/Coroutines/coro-debug.ll
@@ -87,10 +87,10 @@ entry:
 ; Function Attrs: argmemonly nounwind readonly
 declare i8* @llvm.coro.subfn.addr(i8* nocapture readonly, i8) #2
 
-attributes #0 = { noinline nounwind "coroutine.presplit"="1" "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-features"="+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noinline nounwind "coroutine.presplit"="1" "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-features"="+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone speculatable }
 attributes #2 = { argmemonly nounwind readonly }
-attributes #3 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-features"="+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-features"="+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #4 = { nounwind readnone }
 attributes #5 = { nounwind }
 attributes #6 = { alwaysinline }
@@ -127,9 +127,9 @@ attributes #7 = { noduplicate }
 !24 = !DILocation(line: 62, column: 3, scope: !6)
 
 ; CHECK: define i8* @f(i32 %x) #0 !dbg ![[ORIG:[0-9]+]]
-; CHECK: define internal fastcc void @f.resume(%f.Frame* %FramePtr) #0 !dbg ![[RESUME:[0-9]+]]
-; CHECK: define internal fastcc void @f.destroy(%f.Frame* %FramePtr) #0 !dbg ![[DESTROY:[0-9]+]]
-; CHECK: define internal fastcc void @f.cleanup(%f.Frame* %FramePtr) #0 !dbg ![[CLEANUP:[0-9]+]]
+; CHECK: define internal fastcc void @f.resume(%f.Frame* noalias nonnull %FramePtr) #0 !dbg ![[RESUME:[0-9]+]]
+; CHECK: define internal fastcc void @f.destroy(%f.Frame* noalias nonnull %FramePtr) #0 !dbg ![[DESTROY:[0-9]+]]
+; CHECK: define internal fastcc void @f.cleanup(%f.Frame* noalias nonnull %FramePtr) #0 !dbg ![[CLEANUP:[0-9]+]]
 
 ; CHECK: ![[ORIG]] = distinct !DISubprogram(name: "f", linkageName: "flink"
 ; CHECK: !DILocalVariable(name: "x", arg: 1, scope: ![[ORIG]]
diff --git a/llvm/test/Transforms/Coroutines/coro-early.ll b/llvm/test/Transforms/Coroutines/coro-early.ll
index 44e716925..0453ecf75 100644
--- a/llvm/test/Transforms/Coroutines/coro-early.ll
+++ b/llvm/test/Transforms/Coroutines/coro-early.ll
@@ -50,7 +50,7 @@ entry:
   ret i8* %n
 }
 
-; CHECK-LABEL: define private fastcc void @NoopCoro.ResumeDestroy(%NoopCoro.Frame*) {
+; CHECK-LABEL: define private fastcc void @NoopCoro.ResumeDestroy(%NoopCoro.Frame* %0) {
 ; CHECK-NEXT: entry
 ; CHECK-NEXT:    ret void
 
diff --git a/llvm/test/Transforms/Coroutines/coro-param-copy.ll b/llvm/test/Transforms/Coroutines/coro-param-copy.ll
new file mode 100644
index 000000000..6f4d0f3b2
--- /dev/null
+++ b/llvm/test/Transforms/Coroutines/coro-param-copy.ll
@@ -0,0 +1,69 @@
+; Check that we create copy the data from the alloca into the coroutine
+; frame slot if it was written to.
+; RUN: opt < %s -coro-split -S | FileCheck %s
+
+define i8* @f() "coroutine.presplit"="1" {
+entry:
+  %x.addr = alloca i64
+  call void @use(i64* %x.addr) ; might write to %x
+  %y.addr = alloca i64
+  %y = load i64, i64* %y.addr ; cannot modify the value, don't need to copy
+  call void @print(i64 %y)
+
+  %id = call token @llvm.coro.id(i32 0, i8* null, i8* null, i8* null)
+  %size = call i32 @llvm.coro.size.i32()
+  %alloc = call i8* @myAlloc(i64 %y, i32 %size)
+  %hdl = call i8* @llvm.coro.begin(token %id, i8* %alloc)
+  %0 = call i8 @llvm.coro.suspend(token none, i1 false)
+  switch i8 %0, label %suspend [i8 0, label %resume
+                                i8 1, label %cleanup]
+resume:
+  call void @use(i64* %x.addr)
+  call void @use(i64* %y.addr)
+  br label %cleanup
+
+cleanup:
+  %mem = call i8* @llvm.coro.free(token %id, i8* %hdl)
+  call void @free(i8* %mem)
+  br label %suspend
+suspend:
+  call i1 @llvm.coro.end(i8* %hdl, i1 0)
+  ret i8* %hdl
+}
+
+; See that we added both x and y to the frame.
+; CHECK: %f.Frame = type { void (%f.Frame*)*, void (%f.Frame*)*, i1, i1, i64, i64 }
+
+; See that all of the uses prior to coro-begin stays put.
+; CHECK-LABEL: define i8* @f() {
+; CHECK-NEXT: entry:
+; CHECK-NEXT:   %x.addr = alloca i64
+; CHECK-NEXT:   call void @use(i64* %x.addr)
+; CHECK-NEXT:   %y.addr = alloca i64
+; CHECK-NEXT:   %y = load i64, i64* %y.addr
+; CHECK-NEXT:   call void @print(i64 %y)
+
+; See that we only copy the x as y was not modified prior to coro.begin.
+; CHECK:  store void (%f.Frame*)* @f.destroy, void (%f.Frame*)** %destroy.addr
+; CHECK-NEXT:  %0 = getelementptr inbounds %f.Frame, %f.Frame* %FramePtr, i32 0, i32 4
+; CHECK-NEXT:  %1 = load i64, i64* %x.addr
+; CHECK-NEXT:  store i64 %1, i64* %0
+; CHECK-NEXT:  %index.addr1 = getelementptr inbounds %f.Frame, %f.Frame* %FramePtr, i32 0, i32 3
+; CHECK-NEXT:  store i1 false, i1* %index.addr1
+; CHECK-NEXT:  ret i8* %hdl
+
+declare i8* @llvm.coro.free(token, i8*)
+declare i32 @llvm.coro.size.i32()
+declare i8  @llvm.coro.suspend(token, i1)
+declare void @llvm.coro.resume(i8*)
+declare void @llvm.coro.destroy(i8*)
+
+declare token @llvm.coro.id(i32, i8*, i8*, i8*)
+declare i1 @llvm.coro.alloc(token)
+declare i8* @llvm.coro.begin(token, i8*)
+declare i1 @llvm.coro.end(i8*, i1)
+
+declare noalias i8* @myAlloc(i64, i32)
+declare void @print(i64)
+declare void @use(i64*)
+declare void @free(i8*)
diff --git a/llvm/test/Transforms/Coroutines/coro-retcon-alloca.ll b/llvm/test/Transforms/Coroutines/coro-retcon-alloca.ll
new file mode 100644
index 000000000..17aec4eed
--- /dev/null
+++ b/llvm/test/Transforms/Coroutines/coro-retcon-alloca.ll
@@ -0,0 +1,247 @@
+; RUN: opt < %s -enable-coroutines -O2 -S | FileCheck %s
+
+target datalayout = "p:64:64:64"
+
+declare {i8*, i8*, i32} @prototype_f(i8*, i1)
+define {i8*, i8*, i32} @f(i8* %buffer, i32 %n) {
+entry:
+  %id = call token @llvm.coro.id.retcon(i32 1024, i32 8, i8* %buffer, i8* bitcast ({i8*, i8*, i32} (i8*, i1)* @prototype_f to i8*), i8* bitcast (i8* (i32)* @allocate to i8*), i8* bitcast (void (i8*)* @deallocate to i8*))
+  %hdl = call i8* @llvm.coro.begin(token %id, i8* null)
+  br label %loop
+
+loop:
+  %n.val = phi i32 [ %n, %entry ], [ %inc, %resume ]
+  %alloca = call token @llvm.coro.alloca.alloc.i32(i32 %n.val, i32 8)
+  %ptr = call i8* @llvm.coro.alloca.get(token %alloca)
+  %unwind = call i1 (...) @llvm.coro.suspend.retcon.i1(i8* %ptr, i32 %n.val)
+  call void @llvm.coro.alloca.free(token %alloca)
+  br i1 %unwind, label %cleanup, label %resume
+
+resume:
+  %inc = add i32 %n.val, 1
+  br label %loop
+
+cleanup:
+  call i1 @llvm.coro.end(i8* %hdl, i1 0)
+  unreachable
+}
+
+; CHECK-LABEL: define { i8*, i8*, i32 } @f(i8* %buffer, i32 %n)
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[T0:%.*]] = bitcast i8* %buffer to i32*
+; CHECK-NEXT:    store i32 %n, i32* [[T0]], align 4
+; CHECK-NEXT:    [[ALLOC:%.*]] = tail call i8* @allocate(i32 %n)
+; CHECK-NEXT:    [[T0:%.*]] = getelementptr inbounds i8, i8* %buffer, i64 8
+; CHECK-NEXT:    [[T1:%.*]] = bitcast i8* [[T0]] to i8**
+; CHECK-NEXT:    store i8* [[ALLOC]], i8** [[T1]], align 8
+; CHECK-NEXT:    [[T0:%.*]] = insertvalue { i8*, i8*, i32 } { i8* bitcast ({ i8*, i8*, i32 } (i8*, i1)* @f.resume.0 to i8*), i8* undef, i32 undef }, i8* [[ALLOC]], 1
+; CHECK-NEXT:    [[RET:%.*]] = insertvalue { i8*, i8*, i32 } [[T0]], i32 %n, 2
+; CHECK-NEXT:    ret { i8*, i8*, i32 } [[RET]]
+; CHECK-NEXT:  }
+
+; CHECK-LABEL: define internal { i8*, i8*, i32 } @f.resume.0(i8* noalias nonnull %0, i1 %1)
+; CHECK-NEXT:  :
+; CHECK-NEXT:    [[T0:%.*]] = getelementptr inbounds i8, i8* %0, i64 8
+; CHECK-NEXT:    [[T1:%.*]] = bitcast i8* [[T0]] to i8**
+; CHECK-NEXT:    [[ALLOC:%.*]] = load i8*, i8** [[T1]], align 8
+; CHECK-NEXT:    tail call void @deallocate(i8* [[ALLOC]])
+; CHECK-NEXT:    br i1 %1,
+
+declare {i8*, i32} @prototype_g(i8*, i1)
+define {i8*, i32} @g(i8* %buffer, i32 %n) {
+entry:
+  %id = call token @llvm.coro.id.retcon(i32 1024, i32 8, i8* %buffer, i8* bitcast ({i8*, i32} (i8*, i1)* @prototype_g to i8*), i8* bitcast (i8* (i32)* @allocate to i8*), i8* bitcast (void (i8*)* @deallocate to i8*))
+  %hdl = call i8* @llvm.coro.begin(token %id, i8* null)
+  br label %loop
+
+loop:
+  %n.val = phi i32 [ %n, %entry ], [ %inc, %resume ]
+  %alloca = call token @llvm.coro.alloca.alloc.i32(i32 %n.val, i32 8)
+  %ptr = call i8* @llvm.coro.alloca.get(token %alloca)
+  call void @use(i8* %ptr)
+  call void @llvm.coro.alloca.free(token %alloca)
+  %unwind = call i1 (...) @llvm.coro.suspend.retcon.i1(i32 %n.val)
+  br i1 %unwind, label %cleanup, label %resume
+
+resume:
+  %inc = add i32 %n.val, 1
+  br label %loop
+
+cleanup:
+  call i1 @llvm.coro.end(i8* %hdl, i1 0)
+  unreachable
+}
+
+; CHECK-LABEL: define { i8*, i32 } @g(i8* %buffer, i32 %n)
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[T0:%.*]] = bitcast i8* %buffer to i32*
+; CHECK-NEXT:    store i32 %n, i32* [[T0]], align 4
+; CHECK-NEXT:    [[T0:%.*]] = zext i32 %n to i64
+; CHECK-NEXT:    [[ALLOC:%.*]] = alloca i8, i64 [[T0]], align 8
+; CHECK-NEXT:    call void @use(i8* nonnull [[ALLOC]])
+; CHECK-NEXT:    [[RET:%.*]] = insertvalue { i8*, i32 } { i8* bitcast ({ i8*, i32 } (i8*, i1)* @g.resume.0 to i8*), i32 undef }, i32 %n, 1
+; CHECK-NEXT:    ret { i8*, i32 } [[RET]]
+; CHECK-NEXT:  }
+
+; CHECK-LABEL: define internal { i8*, i32 } @g.resume.0(i8* noalias nonnull %0, i1 %1)
+; CHECK-NEXT:  :
+; CHECK-NEXT:    br i1 %1,
+; CHECK:       :
+; CHECK-NEXT:    [[T0:%.*]] = bitcast i8* %0 to i32*
+; CHECK-NEXT:    [[T1:%.*]] = load i32, i32* [[T0]], align 4
+; CHECK-NEXT:    %inc = add i32 [[T1]], 1
+; CHECK-NEXT:    store i32 %inc, i32* [[T0]], align 4
+; CHECK-NEXT:    [[T0:%.*]] = zext i32 %inc to i64
+; CHECK-NEXT:    [[ALLOC:%.*]] = alloca i8, i64 [[T0]], align 8
+; CHECK-NEXT:    call void @use(i8* nonnull [[ALLOC]])
+; CHECK-NEXT:    [[RET:%.*]] = insertvalue { i8*, i32 } { i8* bitcast ({ i8*, i32 } (i8*, i1)* @g.resume.0 to i8*), i32 undef }, i32 %inc, 1
+; CHECK-NEXT:    ret { i8*, i32 } [[RET]]
+; CHECK:       :
+; CHECK-NEXT:    ret { i8*, i32 } { i8* null, i32 undef }
+
+declare {i8*, i32} @prototype_h(i8*, i1)
+define {i8*, i32} @h(i8* %buffer, i32 %n) {
+entry:
+  %id = call token @llvm.coro.id.retcon(i32 1024, i32 8, i8* %buffer, i8* bitcast ({i8*, i32} (i8*, i1)* @prototype_h to i8*), i8* bitcast (i8* (i32)* @allocate to i8*), i8* bitcast (void (i8*)* @deallocate to i8*))
+  %hdl = call i8* @llvm.coro.begin(token %id, i8* null)
+  br label %loop
+
+loop:
+  %n.val = phi i32 [ %n, %entry ], [ %inc, %resume ]
+  %unwind = call i1 (...) @llvm.coro.suspend.retcon.i1(i32 %n.val)
+  br i1 %unwind, label %cleanup, label %resume
+
+resume:
+  %inc = add i32 %n.val, 1
+  %alloca = call token @llvm.coro.alloca.alloc.i32(i32 %inc, i32 8)
+  %ptr = call i8* @llvm.coro.alloca.get(token %alloca)
+  call void @use(i8* %ptr)
+  call void @llvm.coro.alloca.free(token %alloca)
+  br label %loop
+
+cleanup:
+  call i1 @llvm.coro.end(i8* %hdl, i1 0)
+  unreachable
+}
+
+; CHECK-LABEL: define { i8*, i32 } @h(i8* %buffer, i32 %n)
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[T0:%.*]] = bitcast i8* %buffer to i32*
+; CHECK-NEXT:    store i32 %n, i32* [[T0]], align 4
+; CHECK-NEXT:    [[RET:%.*]] = insertvalue { i8*, i32 } { i8* bitcast ({ i8*, i32 } (i8*, i1)* @h.resume.0 to i8*), i32 undef }, i32 %n, 1
+; CHECK-NEXT:    ret { i8*, i32 } [[RET]]
+; CHECK-NEXT:  }
+
+; CHECK-LABEL: define internal { i8*, i32 } @h.resume.0(i8* noalias nonnull %0, i1 %1)
+; CHECK-NEXT:  :
+; CHECK-NEXT:    br i1 %1,
+; CHECK:       :
+; CHECK-NEXT:    [[NSLOT:%.*]] = bitcast i8* %0 to i32*
+; CHECK-NEXT:    [[T1:%.*]] = load i32, i32* [[NSLOT]], align 4
+; CHECK-NEXT:    %inc = add i32 [[T1]], 1
+; CHECK-NEXT:    [[T0:%.*]] = zext i32 %inc to i64
+; CHECK-NEXT:    [[ALLOC:%.*]] = alloca i8, i64 [[T0]], align 8
+; CHECK-NEXT:    call void @use(i8* nonnull [[ALLOC]])
+; CHECK-NEXT:    store i32 %inc, i32* [[NSLOT]], align 4
+; CHECK-NEXT:    [[RET:%.*]] = insertvalue { i8*, i32 } { i8* bitcast ({ i8*, i32 } (i8*, i1)* @h.resume.0 to i8*), i32 undef }, i32 %inc, 1
+; CHECK-NEXT:    ret { i8*, i32 } [[RET]]
+; CHECK:       :
+; CHECK-NEXT:    ret { i8*, i32 } { i8* null, i32 undef }
+
+declare {i8*, i32} @prototype_i(i8*)
+define {i8*, i32} @i(i8* %buffer, i32 %n) {
+entry:
+  %id = call token @llvm.coro.id.retcon(i32 1024, i32 8, i8* %buffer, i8* bitcast ({i8*, i32} (i8*)* @prototype_i to i8*), i8* bitcast (i8* (i32)* @allocate to i8*), i8* bitcast (void (i8*)* @deallocate to i8*))
+  %hdl = call i8* @llvm.coro.begin(token %id, i8* null)
+  br label %loop
+
+loop:
+  %n.val = phi i32 [ %n, %entry ], [ %k, %loop2 ]
+  call void (...) @llvm.coro.suspend.retcon.isVoid(i32 %n.val)
+  %inc = add i32 %n.val, 1
+  br label %loop2
+
+loop2:
+  %k = phi i32 [ %inc, %loop ], [ %k2, %loop2 ]
+  %alloca = call token @llvm.coro.alloca.alloc.i32(i32 %k, i32 8)
+  %ptr = call i8* @llvm.coro.alloca.get(token %alloca)
+  call void @use(i8* %ptr)
+  call void @llvm.coro.alloca.free(token %alloca)
+  %k2 = lshr i32 %k, 1
+  %cmp = icmp ugt i32 %k, 128
+  br i1 %cmp, label %loop2, label %loop
+}
+
+; CHECK-LABEL: define { i8*, i32 } @i(i8* %buffer, i32 %n)
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[T0:%.*]] = bitcast i8* %buffer to i32*
+; CHECK-NEXT:    store i32 %n, i32* [[T0]], align 4
+; CHECK-NEXT:    [[RET:%.*]] = insertvalue { i8*, i32 } { i8* bitcast ({ i8*, i32 } (i8*)* @i.resume.0 to i8*), i32 undef }, i32 %n, 1
+; CHECK-NEXT:    ret { i8*, i32 } [[RET]]
+; CHECK-NEXT:  }
+
+; CHECK-LABEL: define internal { i8*, i32 } @i.resume.0(i8* noalias nonnull %0)
+; CHECK-NEXT:  :
+; CHECK-NEXT:    [[NSLOT:%.*]] = bitcast i8* %0 to i32*
+; CHECK-NEXT:    [[T1:%.*]] = load i32, i32* [[NSLOT]], align 4
+; CHECK-NEXT:    %inc = add i32 [[T1]], 1
+; CHECK-NEXT:    br label %loop2
+; CHECK:       :
+; CHECK-NEXT:    store i32 %k, i32* [[NSLOT]], align 4
+; CHECK-NEXT:    [[RET:%.*]] = insertvalue { i8*, i32 } { i8* bitcast ({ i8*, i32 } (i8*)* @i.resume.0 to i8*), i32 undef }, i32 %k, 1
+; CHECK-NEXT:    ret { i8*, i32 } [[RET]]
+; CHECK:       loop2:
+; CHECK-NEXT:    %k = phi i32 [ %inc, {{.*}} ], [ %k2, %loop2 ]
+; CHECK-NEXT:    [[SAVE:%.*]] = call i8* @llvm.stacksave()
+; CHECK-NEXT:    [[T0:%.*]] = zext i32 %k to i64
+; CHECK-NEXT:    [[ALLOC:%.*]] = alloca i8, i64 [[T0]], align 8
+; CHECK-NEXT:    call void @use(i8* nonnull [[ALLOC]])
+; CHECK-NEXT:    call void @llvm.stackrestore(i8* [[SAVE]])
+; CHECK-NEXT:    %k2 = lshr i32 %k, 1
+; CHECK-NEXT:    %cmp = icmp ugt i32 %k, 128
+; CHECK-NEXT:    br i1 %cmp, label %loop2,
+; CHECK-NEXT:  }
+
+declare {i8*, i32} @prototype_j(i8*)
+define {i8*, i32} @j(i8* %buffer, i32 %n) {
+entry:
+  %id = call token @llvm.coro.id.retcon(i32 1024, i32 8, i8* %buffer, i8* bitcast ({i8*, i32} (i8*)* @prototype_j to i8*), i8* bitcast (i8* (i32)* @allocate to i8*), i8* bitcast (void (i8*)* @deallocate to i8*))
+  %hdl = call i8* @llvm.coro.begin(token %id, i8* null)
+  br label %forward
+
+back:
+  ; We should encounter this 'get' before we encounter the 'alloc'.
+  %ptr = call i8* @llvm.coro.alloca.get(token %alloca)
+  call void @use(i8* %ptr)
+  call void @llvm.coro.alloca.free(token %alloca)
+  %k = add i32 %n.val, 1
+  %cmp = icmp ugt i32 %k, 128
+  br i1 %cmp, label %forward, label %end
+
+forward:
+  %n.val = phi i32 [ %n, %entry ], [ %k, %back ]
+  call void (...) @llvm.coro.suspend.retcon.isVoid(i32 %n.val)
+  %alloca = call token @llvm.coro.alloca.alloc.i32(i32 %n.val, i32 8)
+  %inc = add i32 %n.val, 1
+  br label %back
+
+end:
+  call i1 @llvm.coro.end(i8* %hdl, i1 0)
+  unreachable
+}
+
+declare token @llvm.coro.id.retcon(i32, i32, i8*, i8*, i8*, i8*)
+declare i8* @llvm.coro.begin(token, i8*)
+declare i1 @llvm.coro.suspend.retcon.i1(...)
+declare void @llvm.coro.suspend.retcon.isVoid(...)
+declare i1 @llvm.coro.end(i8*, i1)
+declare i8* @llvm.coro.prepare.retcon(i8*)
+declare token @llvm.coro.alloca.alloc.i32(i32, i32)
+declare i8* @llvm.coro.alloca.get(token)
+declare void @llvm.coro.alloca.free(token)
+
+declare noalias i8* @allocate(i32 %size)
+declare void @deallocate(i8* %ptr)
+
+declare void @print(i32)
+declare void @use(i8*)
diff --git a/llvm/test/Transforms/Coroutines/coro-retcon-once-value.ll b/llvm/test/Transforms/Coroutines/coro-retcon-once-value.ll
new file mode 100644
index 000000000..ac49b22ee
--- /dev/null
+++ b/llvm/test/Transforms/Coroutines/coro-retcon-once-value.ll
@@ -0,0 +1,114 @@
+; RUN: opt < %s -enable-coroutines -O2 -S | FileCheck %s
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.12.0"
+
+define {i8*, i32} @f(i8* %buffer, i32* %array) {
+entry:
+  %id = call token @llvm.coro.id.retcon.once(i32 8, i32 8, i8* %buffer, i8* bitcast (void (i8*, i1)* @prototype to i8*), i8* bitcast (i8* (i32)* @allocate to i8*), i8* bitcast (void (i8*)* @deallocate to i8*))
+  %hdl = call i8* @llvm.coro.begin(token %id, i8* null)
+  %load = load i32, i32* %array
+  %load.pos = icmp sgt i32 %load, 0
+  br i1 %load.pos, label %pos, label %neg
+
+pos:
+  %unwind0 = call i1 (...) @llvm.coro.suspend.retcon.i1(i32 %load)
+  br i1 %unwind0, label %cleanup, label %pos.cont
+
+pos.cont:
+  store i32 0, i32* %array, align 4
+  br label %cleanup
+
+neg:
+  %unwind1 = call i1 (...) @llvm.coro.suspend.retcon.i1(i32 0)
+  br i1 %unwind1, label %cleanup, label %neg.cont
+
+neg.cont:
+  store i32 10, i32* %array, align 4
+  br label %cleanup
+
+cleanup:
+  call i1 @llvm.coro.end(i8* %hdl, i1 0)
+  unreachable
+}
+
+; CHECK-LABEL: define { i8*, i32 } @f(i8* %buffer, i32* %array)
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[T0:%.*]] = bitcast i8* %buffer to i32**
+; CHECK-NEXT:    store i32* %array, i32** [[T0]], align 8
+; CHECK-NEXT:    %load = load i32, i32* %array, align 4
+; CHECK-NEXT:    %load.pos = icmp sgt i32 %load, 0
+; CHECK-NEXT:    [[CONT:%.*]] = select i1 %load.pos, void (i8*, i1)* @f.resume.0, void (i8*, i1)* @f.resume.1
+; CHECK-NEXT:    [[VAL:%.*]] = select i1 %load.pos, i32 %load, i32 0
+; CHECK-NEXT:    [[CONT_CAST:%.*]] = bitcast void (i8*, i1)* [[CONT]] to i8*
+; CHECK-NEXT:    [[T0:%.*]] = insertvalue { i8*, i32 } undef, i8* [[CONT_CAST]], 0
+; CHECK-NEXT:    [[T1:%.*]] = insertvalue { i8*, i32 } [[T0]], i32 [[VAL]], 1
+; CHECK-NEXT:    ret { i8*, i32 } [[T1]]
+; CHECK-NEXT:  }
+
+; CHECK-LABEL: define internal void @f.resume.0(i8* noalias nonnull %0, i1 zeroext %1)
+; CHECK-NEXT:  :
+; CHECK-NEXT:    br i1 %1,
+; CHECK:       :
+; CHECK-NEXT:    [[T0:%.*]] = bitcast i8* %0 to i32**
+; CHECK-NEXT:    [[RELOAD:%.*]] = load i32*, i32** [[T0]], align 8
+; CHECK-NEXT:    store i32 0, i32* [[RELOAD]], align 4
+; CHECK-NEXT:    br label
+; CHECK:       :
+; CHECK-NEXT:    ret void
+; CHECK-NEXT:  }
+
+; CHECK-LABEL: define internal void @f.resume.1(i8* noalias nonnull %0, i1 zeroext %1)
+; CHECK-NEXT:  :
+; CHECK-NEXT:    br i1 %1,
+; CHECK:       :
+; CHECK-NEXT:    [[T0:%.*]] = bitcast i8* %0 to i32**
+; CHECK-NEXT:    [[RELOAD:%.*]] = load i32*, i32** [[T0]], align 8
+; CHECK-NEXT:    store i32 10, i32* [[RELOAD]], align 4
+; CHECK-NEXT:    br label
+; CHECK:       :
+; CHECK-NEXT:    ret void
+; CHECK-NEXT:  }
+
+define void @test(i32* %array) {
+entry:
+  %0 = alloca [8 x i8], align 8
+  %buffer = bitcast [8 x i8]* %0 to i8*
+  %prepare = call i8* @llvm.coro.prepare.retcon(i8* bitcast ({i8*, i32} (i8*, i32*)* @f to i8*))
+  %f = bitcast i8* %prepare to {i8*, i32} (i8*, i32*)*
+  %result = call {i8*, i32} %f(i8* %buffer, i32* %array)
+  %value = extractvalue {i8*, i32} %result, 1
+  call void @print(i32 %value)
+  %cont = extractvalue {i8*, i32} %result, 0
+  %cont.cast = bitcast i8* %cont to void (i8*, i1)*
+  call void %cont.cast(i8* %buffer, i1 zeroext 0)
+  ret void
+}
+
+;   Unfortunately, we don't seem to fully optimize this right now due
+;   to some sort of phase-ordering thing.
+; CHECK-LABEL: define void @test(i32* %array)
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[BUFFER:%.*]] = alloca i32*, align 8
+; CHECK-NEXT:    [[BUFFER_CAST:%.*]] = bitcast i32** [[BUFFER]] to i8*
+; CHECK-NEXT:    store i32* %array, i32** [[BUFFER]], align 8
+; CHECK-NEXT:    [[LOAD:%.*]] = load i32, i32* %array, align 4
+; CHECK-NEXT:    [[LOAD_POS:%.*]] = icmp sgt i32 [[LOAD]], 0
+; CHECK-NEXT:    [[CONT:%.*]] = select i1 [[LOAD_POS]], void (i8*, i1)* @f.resume.0, void (i8*, i1)* @f.resume.1
+; CHECK-NEXT:    [[VAL:%.*]] = select i1 [[LOAD_POS]], i32 [[LOAD]], i32 0
+; CHECK-NEXT:    call void @print(i32 [[VAL]])
+; CHECK-NEXT:    call void [[CONT]](i8* nonnull [[BUFFER_CAST]], i1 zeroext false)
+; CHECK-NEXT:    ret void
+
+declare token @llvm.coro.id.retcon.once(i32, i32, i8*, i8*, i8*, i8*)
+declare i8* @llvm.coro.begin(token, i8*)
+declare i1 @llvm.coro.suspend.retcon.i1(...)
+declare i1 @llvm.coro.end(i8*, i1)
+declare i8* @llvm.coro.prepare.retcon(i8*)
+
+declare void @prototype(i8*, i1 zeroext)
+
+declare noalias i8* @allocate(i32 %size)
+declare void @deallocate(i8* %ptr)
+
+declare void @print(i32)
+
diff --git a/llvm/test/Transforms/Coroutines/coro-retcon-once-value2.ll b/llvm/test/Transforms/Coroutines/coro-retcon-once-value2.ll
new file mode 100644
index 000000000..ad49f24dc
--- /dev/null
+++ b/llvm/test/Transforms/Coroutines/coro-retcon-once-value2.ll
@@ -0,0 +1,71 @@
+; RUN: opt < %s -coro-split -coro-cleanup -S | FileCheck %s
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.12.0"
+
+define {i8*, i32*} @f(i8* %buffer, i32* %ptr) "coroutine.presplit"="1" {
+entry:
+  %temp = alloca i32, align 4
+  %id = call token @llvm.coro.id.retcon.once(i32 8, i32 8, i8* %buffer, i8* bitcast (void (i8*, i1)* @prototype to i8*), i8* bitcast (i8* (i32)* @allocate to i8*), i8* bitcast (void (i8*)* @deallocate to i8*))
+  %hdl = call i8* @llvm.coro.begin(token %id, i8* null)
+  %oldvalue = load i32, i32* %ptr
+  store i32 %oldvalue, i32* %temp
+  %unwind = call i1 (...) @llvm.coro.suspend.retcon.i1(i32* %temp)
+  br i1 %unwind, label %cleanup, label %cont
+
+cont:
+  %newvalue = load i32, i32* %temp
+  store i32 %newvalue, i32* %ptr
+  br label %cleanup
+
+cleanup:
+  call i1 @llvm.coro.end(i8* %hdl, i1 0)
+  unreachable
+}
+
+; CHECK-LABEL: define { i8*, i32* } @f(i8* %buffer, i32* %ptr)
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ALLOC:%.*]] = call i8* @allocate(i32 16)
+; CHECK-NEXT:    [[T0:%.*]] = bitcast i8* %buffer to i8**
+; CHECK-NEXT:    store i8* [[ALLOC]], i8** [[T0]]
+; CHECK-NEXT:    [[FRAME:%.*]] = bitcast i8* [[ALLOC]] to [[FRAME_T:%.*]]*
+; CHECK-NEXT:    %temp = getelementptr inbounds [[FRAME_T]], [[FRAME_T]]* [[FRAME]], i32 0, i32 1
+; CHECK-NEXT:    [[SPILL:%.*]] = getelementptr inbounds [[FRAME_T]], [[FRAME_T]]* [[FRAME]], i32 0, i32 0
+; CHECK-NEXT:    store i32* %ptr, i32** [[SPILL]]
+; CHECK-NEXT:    %oldvalue = load i32, i32* %ptr
+; CHECK-NEXT:    store i32 %oldvalue, i32* %temp
+; CHECK-NEXT:    [[T0:%.*]] = insertvalue { i8*, i32* } { i8* bitcast (void (i8*, i1)* @f.resume.0 to i8*), i32* undef }, i32* %temp, 1
+; CHECK-NEXT:    ret { i8*, i32* } [[T0]]
+; CHECK-NEXT:  }
+
+; CHECK-LABEL: define internal void @f.resume.0(i8* noalias nonnull %0, i1 zeroext %1)
+; CHECK-NEXT:  :
+; CHECK-NEXT:    [[T0:%.*]] = bitcast i8* %0 to [[FRAME_T:%.*]]**
+; CHECK-NEXT:    [[FRAME:%.*]] = load [[FRAME_T]]*, [[FRAME_T]]** [[T0]]
+; CHECK-NEXT:    bitcast [[FRAME_T]]* [[FRAME]] to i8*
+; CHECK-NEXT:    %temp = getelementptr inbounds [[FRAME_T]], [[FRAME_T]]* [[FRAME]], i32 0, i32 1
+; CHECK-NEXT:    br i1 %1,
+; CHECK:       :
+; CHECK-NEXT:    [[TEMP_SLOT:%.*]] = getelementptr inbounds [[FRAME_T]], [[FRAME_T]]* [[FRAME]], i32 0, i32 1
+; CHECK-NEXT:    [[PTR_SLOT:%.*]] = getelementptr inbounds [[FRAME_T]], [[FRAME_T]]* [[FRAME]], i32 0, i32 0
+; CHECK-NEXT:    [[PTR_RELOAD:%.*]] = load i32*, i32** [[PTR_SLOT]]
+; CHECK-NEXT:    %newvalue = load i32, i32* [[TEMP_SLOT]]
+; CHECK-NEXT:    store i32 %newvalue, i32* [[PTR_RELOAD]]
+; CHECK-NEXT:    br label
+; CHECK:       :
+; CHECK-NEXT:    [[T0:%.*]] = bitcast [[FRAME_T]]* [[FRAME]] to i8*
+; CHECK-NEXT:    call fastcc void @deallocate(i8* [[T0]])
+; CHECK-NEXT:    ret void
+; CHECK-NEXT:  }
+
+declare token @llvm.coro.id.retcon.once(i32, i32, i8*, i8*, i8*, i8*)
+declare i8* @llvm.coro.begin(token, i8*)
+declare i1 @llvm.coro.suspend.retcon.i1(...)
+declare i1 @llvm.coro.end(i8*, i1)
+
+declare void @prototype(i8*, i1 zeroext)
+
+declare noalias i8* @allocate(i32 %size)
+declare fastcc void @deallocate(i8* %ptr)
+
+declare void @print(i32)
+
diff --git a/llvm/test/Transforms/Coroutines/coro-retcon-resume-values.ll b/llvm/test/Transforms/Coroutines/coro-retcon-resume-values.ll
new file mode 100644
index 000000000..ac99dd15b
--- /dev/null
+++ b/llvm/test/Transforms/Coroutines/coro-retcon-resume-values.ll
@@ -0,0 +1,85 @@
+; RUN: opt < %s -enable-coroutines -O2 -S | FileCheck %s
+
+define i8* @f(i8* %buffer, i32 %n) {
+entry:
+  %id = call token @llvm.coro.id.retcon(i32 8, i32 4, i8* %buffer, i8* bitcast (i8* (i8*, i32, i1)* @prototype to i8*), i8* bitcast (i8* (i32)* @allocate to i8*), i8* bitcast (void (i8*)* @deallocate to i8*))
+  %hdl = call i8* @llvm.coro.begin(token %id, i8* null)
+  br label %loop
+
+loop:
+  %n.val = phi i32 [ %n, %entry ], [ %sum, %resume ]
+  %values = call { i32, i1 } (...) @llvm.coro.suspend.retcon.sl_i32i1s()
+  %finished = extractvalue { i32, i1 } %values, 1
+  br i1 %finished, label %cleanup, label %resume
+
+resume:
+  %input = extractvalue { i32, i1 } %values, 0
+  %sum = add i32 %n.val, %input
+  br label %loop
+
+cleanup:
+  call void @print(i32 %n.val)
+  call i1 @llvm.coro.end(i8* %hdl, i1 0)
+  unreachable
+}
+
+; CHECK-LABEL: define i8* @f(i8* %buffer, i32 %n)
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[T0:%.*]] = bitcast i8* %buffer to i32*
+; CHECK-NEXT:    store i32 %n, i32* [[T0]], align 4
+; CHECK-NEXT:    ret i8* bitcast (i8* (i8*, i32, i1)* @f.resume.0 to i8*)
+; CHECK-NEXT:  }
+
+; CHECK-LABEL: define internal i8* @f.resume.0(i8* noalias nonnull %0, i32 %1, i1 zeroext %2)
+; CHECK-NEXT:  :
+; CHECK-NEXT:    [[T0:%.*]] = bitcast i8* %0 to i32*
+; CHECK-NEXT:    [[T1:%.*]] = load i32, i32* [[T0]], align 4
+; CHECK-NEXT:    br i1 %2,
+; CHECK:       :
+; CHECK-NEXT:    %sum = add i32 [[T1]], %1
+; CHECK-NEXT:    store i32 %sum, i32* [[T0]], align 4
+; CHECK-NEXT:    ret i8* bitcast (i8* (i8*, i32, i1)* @f.resume.0 to i8*)
+; CHECK:       :
+; CHECK-NEXT:    call void @print(i32 [[T1]])
+; CHECK-NEXT:    ret i8* null
+; CHECK-NEXT:  }
+
+define i32 @main() {
+entry:
+  %0 = alloca [8 x i8], align 4
+  %buffer = bitcast [8 x i8]* %0 to i8*
+  %prepare = call i8* @llvm.coro.prepare.retcon(i8* bitcast (i8* (i8*, i32)* @f to i8*))
+  %f = bitcast i8* %prepare to i8* (i8*, i32)*
+  %cont0 = call i8* %f(i8* %buffer, i32 1)
+  %cont0.cast = bitcast i8* %cont0 to i8* (i8*, i32, i1)*
+  %cont1 = call i8* %cont0.cast(i8* %buffer, i32 2, i1 zeroext false)
+  %cont1.cast = bitcast i8* %cont1 to i8* (i8*, i32, i1)*
+  %cont2 = call i8* %cont1.cast(i8* %buffer, i32 4, i1 zeroext false)
+  %cont2.cast = bitcast i8* %cont2 to i8* (i8*, i32, i1)*
+  call i8* %cont2.cast(i8* %buffer, i32 100, i1 zeroext true)
+  ret i32 0
+}
+
+;   Unfortunately, we don't seem to fully optimize this right now due
+;   to some sort of phase-ordering thing.
+; CHECK-LABEL: define i32 @main
+; CHECK-NEXT:  entry:
+; CHECK:         [[BUFFER:%.*]] = alloca [8 x i8], align 4
+; CHECK:         [[SLOT:%.*]] = bitcast [8 x i8]* [[BUFFER]] to i32*
+; CHECK-NEXT:    store i32 7, i32* [[SLOT]], align 4
+; CHECK-NEXT:    call void @print(i32 7)
+; CHECK-NEXT:    ret i32 0
+
+declare token @llvm.coro.id.retcon(i32, i32, i8*, i8*, i8*, i8*)
+declare i8* @llvm.coro.begin(token, i8*)
+declare { i32, i1 } @llvm.coro.suspend.retcon.sl_i32i1s(...)
+declare i1 @llvm.coro.end(i8*, i1)
+declare i8* @llvm.coro.prepare.retcon(i8*)
+
+declare i8* @prototype(i8*, i32, i1 zeroext)
+
+declare noalias i8* @allocate(i32 %size)
+declare void @deallocate(i8* %ptr)
+
+declare void @print(i32)
+
diff --git a/llvm/test/Transforms/Coroutines/coro-retcon-resume-values2.ll b/llvm/test/Transforms/Coroutines/coro-retcon-resume-values2.ll
new file mode 100644
index 000000000..43f98e958
--- /dev/null
+++ b/llvm/test/Transforms/Coroutines/coro-retcon-resume-values2.ll
@@ -0,0 +1,100 @@
+; RUN: opt < %s -coro-split -coro-cleanup -S | FileCheck %s
+
+define i8* @f(i8* %buffer, i32 %n) "coroutine.presplit"="1" {
+entry:
+  %id = call token @llvm.coro.id.retcon(i32 8, i32 4, i8* %buffer, i8* bitcast (i8* (i8*, i32)* @prototype to i8*), i8* bitcast (i8* (i32)* @allocate to i8*), i8* bitcast (void (i8*)* @deallocate to i8*))
+  %hdl = call i8* @llvm.coro.begin(token %id, i8* null)
+  %value0 = call i32 (...) @llvm.coro.suspend.retcon.i32()
+  %sum0 = call i32 @add(i32 %n, i32 %value0)
+  %value1 = call i32 (...) @llvm.coro.suspend.retcon.i32()
+  %sum1 = call i32 @add(i32 %sum0, i32 %value0)
+  %sum2 = call i32 @add(i32 %sum1, i32 %value1)
+  %value2 = call i32 (...) @llvm.coro.suspend.retcon.i32()
+  %sum3 = call i32 @add(i32 %sum2, i32 %value0)
+  %sum4 = call i32 @add(i32 %sum3, i32 %value1)
+  %sum5 = call i32 @add(i32 %sum4, i32 %value2)
+  call void @print(i32 %sum5)
+  call i1 @llvm.coro.end(i8* %hdl, i1 0)
+  unreachable
+}
+
+; CHECK-LABEL: define i8* @f(i8* %buffer, i32 %n)
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ALLOC:%.*]] = call i8* @allocate(i32 20)
+; CHECK-NEXT:    [[T0:%.*]] = bitcast i8* %buffer to i8**
+; CHECK-NEXT:    store i8* [[ALLOC]], i8** [[T0]]
+; CHECK-NEXT:    [[FRAME:%.*]] = bitcast i8* [[ALLOC]] to [[FRAME_T:%.*]]*
+; CHECK-NEXT:    [[T0:%.*]] = getelementptr inbounds [[FRAME_T]], [[FRAME_T]]* [[FRAME]], i32 0, i32 0
+; CHECK-NEXT:    store i32 %n, i32* [[T0]]
+; CHECK-NEXT:    ret i8* bitcast (i8* (i8*, i32)* @f.resume.0 to i8*)
+; CHECK-NEXT:  }
+
+; CHECK-LABEL: define internal i8* @f.resume.0(i8* noalias nonnull %0, i32 %1)
+; CHECK-NEXT:  :
+; CHECK-NEXT:    [[T0:%.*]] = bitcast i8* %0 to [[FRAME_T:%.*]]**
+; CHECK-NEXT:    [[FRAME:%.*]] = load [[FRAME_T]]*, [[FRAME_T]]** [[T0]]
+; CHECK-NEXT:    [[VFRAME:%.*]] = bitcast [[FRAME_T]]* [[FRAME]] to i8*
+; CHECK-NEXT:    [[T0:%.*]] = getelementptr inbounds [[FRAME_T]], [[FRAME_T]]* [[FRAME]], i32 0, i32 1
+; CHECK-NEXT:    store i32 %1, i32* [[T0]]
+; CHECK-NEXT:    [[T0:%.*]] = getelementptr inbounds [[FRAME_T]], [[FRAME_T]]* [[FRAME]], i32 0, i32 0
+; CHECK-NEXT:    [[N:%.*]] = load i32, i32* [[T0]]
+; CHECK-NEXT:    %sum0 = call i32 @add(i32 [[N]], i32 %1)
+; CHECK-NEXT:    [[T0:%.*]] = getelementptr inbounds [[FRAME_T]], [[FRAME_T]]* [[FRAME]], i32 0, i32 2
+; CHECK-NEXT:    store i32 %sum0, i32* [[T0]]
+; CHECK-NEXT:    [[CONT:%.*]] = bitcast i8* (i8*, i32)* @f.resume.1 to i8*
+; CHECK-NEXT:    ret i8* [[CONT]]
+; CHECK-NEXT:  }
+
+; CHECK-LABEL: define internal i8* @f.resume.1(i8* noalias nonnull %0, i32 %1)
+; CHECK-NEXT:  :
+; CHECK-NEXT:    [[T0:%.*]] = bitcast i8* %0 to [[FRAME_T:%.*]]**
+; CHECK-NEXT:    [[FRAME:%.*]] = load [[FRAME_T]]*, [[FRAME_T]]** [[T0]]
+; CHECK-NEXT:    [[VFRAME:%.*]] = bitcast [[FRAME_T]]* [[FRAME]] to i8*
+; CHECK-NEXT:    [[T0:%.*]] = getelementptr inbounds [[FRAME_T]], [[FRAME_T]]* [[FRAME]], i32 0, i32 3
+; CHECK-NEXT:    store i32 %1, i32* [[T0]]
+; CHECK-NEXT:    [[T0:%.*]] = getelementptr inbounds [[FRAME_T]], [[FRAME_T]]* [[FRAME]], i32 0, i32 2
+; CHECK-NEXT:    [[SUM0:%.*]] = load i32, i32* [[T0]]
+; CHECK-NEXT:    [[T0:%.*]] = getelementptr inbounds [[FRAME_T]], [[FRAME_T]]* [[FRAME]], i32 0, i32 1
+; CHECK-NEXT:    [[VALUE0:%.*]] = load i32, i32* [[T0]]
+; CHECK-NEXT:    %sum1 = call i32 @add(i32 [[SUM0]], i32 [[VALUE0]])
+; CHECK-NEXT:    %sum2 = call i32 @add(i32 %sum1, i32 %1)
+; CHECK-NEXT:    [[T0:%.*]] = getelementptr inbounds [[FRAME_T]], [[FRAME_T]]* [[FRAME]], i32 0, i32 4
+; CHECK-NEXT:    store i32 %sum2, i32* [[T0]]
+; CHECK-NEXT:    [[CONT:%.*]] = bitcast i8* (i8*, i32)* @f.resume.2 to i8*
+; CHECK-NEXT:    ret i8* [[CONT]]
+; CHECK-NEXT:  }
+
+; CHECK-LABEL: define internal i8* @f.resume.2(i8* noalias nonnull %0, i32 %1)
+; CHECK-NEXT:  :
+; CHECK-NEXT:    [[T0:%.*]] = bitcast i8* %0 to [[FRAME_T:%.*]]**
+; CHECK-NEXT:    [[FRAME:%.*]] = load [[FRAME_T]]*, [[FRAME_T]]** [[T0]]
+; CHECK-NEXT:    [[VFRAME:%.*]] = bitcast [[FRAME_T]]* [[FRAME]] to i8*
+; CHECK-NEXT:    [[T0:%.*]] = getelementptr inbounds [[FRAME_T]], [[FRAME_T]]* [[FRAME]], i32 0, i32 4
+; CHECK-NEXT:    [[SUM2:%.*]] = load i32, i32* [[T0]]
+; CHECK-NEXT:    [[T0:%.*]] = getelementptr inbounds [[FRAME_T]], [[FRAME_T]]* [[FRAME]], i32 0, i32 3
+; CHECK-NEXT:    [[VALUE1:%.*]] = load i32, i32* [[T0]]
+; CHECK-NEXT:    [[T0:%.*]] = getelementptr inbounds [[FRAME_T]], [[FRAME_T]]* [[FRAME]], i32 0, i32 1
+; CHECK-NEXT:    [[VALUE0:%.*]] = load i32, i32* [[T0]]
+; CHECK-NEXT:    %sum3 = call i32 @add(i32 [[SUM2]], i32 [[VALUE0]])
+; CHECK-NEXT:    %sum4 = call i32 @add(i32 %sum3, i32 [[VALUE1]])
+; CHECK-NEXT:    %sum5 = call i32 @add(i32 %sum4, i32 %1)
+; CHECK-NEXT:    call void @print(i32 %sum5)
+; CHECK-NEXT:    [[CONT:%.*]] = bitcast [[FRAME_T]]* [[FRAME]] to i8*
+; CHECK-NEXT:    call void @deallocate(i8* [[CONT]])
+; CHECK-NEXT:    ret i8* null
+; CHECK-NEXT:  }
+
+declare token @llvm.coro.id.retcon(i32, i32, i8*, i8*, i8*, i8*)
+declare i8* @llvm.coro.begin(token, i8*)
+declare i32 @llvm.coro.suspend.retcon.i32(...)
+declare i1 @llvm.coro.end(i8*, i1)
+declare i8* @llvm.coro.prepare.retcon(i8*)
+
+declare i8* @prototype(i8*, i32)
+
+declare noalias i8* @allocate(i32 %size)
+declare void @deallocate(i8* %ptr)
+
+declare i32 @add(i32, i32)
+declare void @print(i32)
+
diff --git a/llvm/test/Transforms/Coroutines/coro-retcon-value.ll b/llvm/test/Transforms/Coroutines/coro-retcon-value.ll
new file mode 100644
index 000000000..cfda73bbe
--- /dev/null
+++ b/llvm/test/Transforms/Coroutines/coro-retcon-value.ll
@@ -0,0 +1,103 @@
+; First example from Doc/Coroutines.rst (two block loop) converted to retcon
+; RUN: opt < %s -enable-coroutines -O2 -S | FileCheck %s
+
+define {i8*, i32} @f(i8* %buffer, i32 %n) {
+entry:
+  %id = call token @llvm.coro.id.retcon(i32 8, i32 4, i8* %buffer, i8* bitcast ({i8*, i32} (i8*, i8)* @prototype to i8*), i8* bitcast (i8* (i32)* @allocate to i8*), i8* bitcast (void (i8*)* @deallocate to i8*))
+  %hdl = call i8* @llvm.coro.begin(token %id, i8* null)
+  br label %loop
+
+loop:
+  %n.val = phi i32 [ %n, %entry ], [ %inc, %resume ]
+  %unwind = call i8 (...) @llvm.coro.suspend.retcon.i8(i32 %n.val)
+  %unwind0 = icmp ne i8 %unwind, 0
+  br i1 %unwind0, label %cleanup, label %resume
+
+resume:
+  %inc = add i32 %n.val, 1
+  br label %loop
+
+cleanup:
+  call i1 @llvm.coro.end(i8* %hdl, i1 0)
+  unreachable
+}
+
+; CHECK-LABEL: define { i8*, i32 } @f(i8* %buffer, i32 %n)
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[T0:%.*]] = bitcast i8* %buffer to i32*
+; CHECK-NEXT:    store i32 %n, i32* [[T0]], align 4
+; CHECK-NEXT:    [[RET:%.*]] = insertvalue { i8*, i32 } { i8* bitcast ({ i8*, i32 } (i8*, i8)* @f.resume.0 to i8*), i32 undef }, i32 %n, 1
+; CHECK-NEXT:    ret { i8*, i32 } [[RET]]
+; CHECK-NEXT:  }
+
+; CHECK-LABEL: define internal { i8*, i32 } @f.resume.0(i8* noalias nonnull %0, i8 zeroext %1)
+; CHECK-NEXT:  :
+; CHECK-NEXT:    [[T0:%.*]] = icmp eq i8 %1, 0
+; CHECK-NEXT:    br i1 [[T0]],
+; CHECK:       :
+; CHECK-NEXT:    [[T0:%.*]] = bitcast i8* %0 to i32*
+; CHECK-NEXT:    [[T1:%.*]] = load i32, i32* [[T0]], align 4
+; CHECK-NEXT:    %inc = add i32 [[T1]], 1
+; CHECK-NEXT:    store i32 %inc, i32* [[T0]], align 4
+; CHECK-NEXT:    [[RET:%.*]] = insertvalue { i8*, i32 } { i8* bitcast ({ i8*, i32 } (i8*, i8)* @f.resume.0 to i8*), i32 undef }, i32 %inc, 1
+; CHECK-NEXT:    ret { i8*, i32 } [[RET]]
+; CHECK:       :
+; CHECK-NEXT:    ret { i8*, i32 } { i8* null, i32 undef }
+; CHECK-NEXT:  }
+
+define i32 @main() {
+entry:
+  %0 = alloca [8 x i8], align 4
+  %buffer = bitcast [8 x i8]* %0 to i8*
+  %prepare = call i8* @llvm.coro.prepare.retcon(i8* bitcast ({i8*, i32} (i8*, i32)* @f to i8*))
+  %f = bitcast i8* %prepare to {i8*, i32} (i8*, i32)*
+  %result0 = call {i8*, i32} %f(i8* %buffer, i32 4)
+  %value0 = extractvalue {i8*, i32} %result0, 1
+  call void @print(i32 %value0)
+  %cont0 = extractvalue {i8*, i32} %result0, 0
+  %cont0.cast = bitcast i8* %cont0 to {i8*, i32} (i8*, i8)*
+  %result1 = call {i8*, i32} %cont0.cast(i8* %buffer, i8 zeroext 0)
+  %value1 = extractvalue {i8*, i32} %result1, 1
+  call void @print(i32 %value1)
+  %cont1 = extractvalue {i8*, i32} %result1, 0
+  %cont1.cast = bitcast i8* %cont1 to {i8*, i32} (i8*, i8)*
+  %result2 = call {i8*, i32} %cont1.cast(i8* %buffer, i8 zeroext 0)
+  %value2 = extractvalue {i8*, i32} %result2, 1
+  call void @print(i32 %value2)
+  %cont2 = extractvalue {i8*, i32} %result2, 0
+  %cont2.cast = bitcast i8* %cont2 to {i8*, i32} (i8*, i8)*
+  call {i8*, i32} %cont2.cast(i8* %buffer, i8 zeroext 1)
+  ret i32 0
+}
+
+;   Unfortunately, we don't seem to fully optimize this right now due
+;   to some sort of phase-ordering thing.
+; CHECK-LABEL: define i32 @main
+; CHECK-NEXT:  entry:
+; CHECK:         [[BUFFER:%.*]] = alloca [8 x i8], align 4
+; CHECK:         [[SLOT:%.*]] = bitcast [8 x i8]* [[BUFFER]] to i32*
+; CHECK-NEXT:    store i32 4, i32* [[SLOT]], align 4
+; CHECK-NEXT:    call void @print(i32 4)
+; CHECK-NEXT:    [[LOAD:%.*]] = load i32, i32* [[SLOT]], align 4
+; CHECK-NEXT:    [[INC:%.*]] = add i32 [[LOAD]], 1
+; CHECK-NEXT:    store i32 [[INC]], i32* [[SLOT]], align 4
+; CHECK-NEXT:    call void @print(i32 [[INC]])
+; CHECK-NEXT:    [[LOAD:%.*]] = load i32, i32* [[SLOT]], align 4
+; CHECK-NEXT:    [[INC:%.*]] = add i32 [[LOAD]], 1
+; CHECK-NEXT:    store i32 [[INC]], i32* [[SLOT]], align 4
+; CHECK-NEXT:    call void @print(i32 [[INC]])
+; CHECK-NEXT:    ret i32 0
+
+declare token @llvm.coro.id.retcon(i32, i32, i8*, i8*, i8*, i8*)
+declare i8* @llvm.coro.begin(token, i8*)
+declare i8 @llvm.coro.suspend.retcon.i8(...)
+declare i1 @llvm.coro.end(i8*, i1)
+declare i8* @llvm.coro.prepare.retcon(i8*)
+
+declare {i8*, i32} @prototype(i8*, i8 zeroext)
+
+declare noalias i8* @allocate(i32 %size)
+declare void @deallocate(i8* %ptr)
+
+declare void @print(i32)
+
diff --git a/llvm/test/Transforms/Coroutines/coro-retcon.ll b/llvm/test/Transforms/Coroutines/coro-retcon.ll
new file mode 100644
index 000000000..5cd4cb61d
--- /dev/null
+++ b/llvm/test/Transforms/Coroutines/coro-retcon.ll
@@ -0,0 +1,113 @@
+; First example from Doc/Coroutines.rst (two block loop) converted to retcon
+; RUN: opt < %s -enable-coroutines -O2 -S | FileCheck %s
+
+define i8* @f(i8* %buffer, i32 %n) {
+entry:
+  %id = call token @llvm.coro.id.retcon(i32 8, i32 4, i8* %buffer, i8* bitcast (i8* (i8*, i1)* @prototype to i8*), i8* bitcast (i8* (i32)* @allocate to i8*), i8* bitcast (void (i8*)* @deallocate to i8*))
+  %hdl = call i8* @llvm.coro.begin(token %id, i8* null)
+  br label %loop
+
+loop:
+  %n.val = phi i32 [ %n, %entry ], [ %inc, %resume ]
+  call void @print(i32 %n.val)
+  %unwind0 = call i1 (...) @llvm.coro.suspend.retcon.i1()
+  br i1 %unwind0, label %cleanup, label %resume
+
+resume:
+  %inc = add i32 %n.val, 1
+  br label %loop
+
+cleanup:
+  call i1 @llvm.coro.end(i8* %hdl, i1 0)
+  unreachable
+}
+
+; CHECK-LABEL: define i8* @f(i8* %buffer, i32 %n)
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[T0:%.*]] = bitcast i8* %buffer to i32*
+; CHECK-NEXT:    store i32 %n, i32* [[T0]], align 4
+; CHECK-NEXT:    call void @print(i32 %n)
+; CHECK-NEXT:    ret i8* bitcast (i8* (i8*, i1)* @f.resume.0 to i8*)
+; CHECK-NEXT:  }
+
+; CHECK-LABEL: define internal i8* @f.resume.0(i8* noalias nonnull %0, i1 zeroext %1)
+; CHECK-NEXT:  :
+; CHECK-NEXT:    br i1 %1,
+; CHECK:       :
+; CHECK-NEXT:    [[T0:%.*]] = bitcast i8* %0 to i32*
+; CHECK-NEXT:    [[T1:%.*]] = load i32, i32* [[T0]], align 4
+; CHECK-NEXT:    %inc = add i32 [[T1]], 1
+; CHECK-NEXT:    store i32 %inc, i32* [[T0]], align 4
+; CHECK-NEXT:    call void @print(i32 %inc)
+; CHECK-NEXT:    ret i8* bitcast (i8* (i8*, i1)* @f.resume.0 to i8*)
+; CHECK:       :
+; CHECK-NEXT:    ret i8* null
+; CHECK-NEXT:  }
+
+define i32 @main() {
+entry:
+  %0 = alloca [8 x i8], align 4
+  %buffer = bitcast [8 x i8]* %0 to i8*
+  %prepare = call i8* @llvm.coro.prepare.retcon(i8* bitcast (i8* (i8*, i32)* @f to i8*))
+  %f = bitcast i8* %prepare to i8* (i8*, i32)*
+  %cont0 = call i8* %f(i8* %buffer, i32 4)
+  %cont0.cast = bitcast i8* %cont0 to i8* (i8*, i1)*
+  %cont1 = call i8* %cont0.cast(i8* %buffer, i1 zeroext false)
+  %cont1.cast = bitcast i8* %cont1 to i8* (i8*, i1)*
+  %cont2 = call i8* %cont1.cast(i8* %buffer, i1 zeroext false)
+  %cont2.cast = bitcast i8* %cont2 to i8* (i8*, i1)*
+  call i8* %cont2.cast(i8* %buffer, i1 zeroext true)
+  ret i32 0
+}
+
+;   Unfortunately, we don't seem to fully optimize this right now due
+;   to some sort of phase-ordering thing.
+; CHECK-LABEL: define i32 @main
+; CHECK-NEXT:  entry:
+; CHECK:         [[BUFFER:%.*]] = alloca [8 x i8], align 4
+; CHECK:         [[SLOT:%.*]] = bitcast [8 x i8]* [[BUFFER]] to i32*
+; CHECK-NEXT:    store i32 4, i32* [[SLOT]], align 4
+; CHECK-NEXT:    call void @print(i32 4)
+; CHECK-NEXT:    [[LOAD:%.*]] = load i32, i32* [[SLOT]], align 4
+; CHECK-NEXT:    [[INC:%.*]] = add i32 [[LOAD]], 1
+; CHECK-NEXT:    store i32 [[INC]], i32* [[SLOT]], align 4
+; CHECK-NEXT:    call void @print(i32 [[INC]])
+; CHECK-NEXT:    [[LOAD:%.*]] = load i32, i32* [[SLOT]], align 4
+; CHECK-NEXT:    [[INC:%.*]] = add i32 [[LOAD]], 1
+; CHECK-NEXT:    store i32 [[INC]], i32* [[SLOT]], align 4
+; CHECK-NEXT:    call void @print(i32 [[INC]])
+; CHECK-NEXT:    ret i32 0
+
+define hidden { i8*, i8* } @g(i8* %buffer, i16* %ptr) {
+entry:
+  %id = call token @llvm.coro.id.retcon(i32 8, i32 4, i8* %buffer, i8* bitcast ({ i8*, i8* } (i8*, i1)* @g_prototype to i8*), i8* bitcast (i8* (i32)* @allocate to i8*), i8* bitcast (void (i8*)* @deallocate to i8*))
+  %hdl = call i8* @llvm.coro.begin(token %id, i8* null)
+  br label %loop
+
+loop:
+  %ptr2 = bitcast i16* %ptr to i8*
+  %unwind0 = call i1 (...) @llvm.coro.suspend.retcon.i1(i8* %ptr2)
+  br i1 %unwind0, label %cleanup, label %resume
+
+resume:
+  br label %loop
+
+cleanup:
+  call i1 @llvm.coro.end(i8* %hdl, i1 0)
+  unreachable
+}
+
+declare token @llvm.coro.id.retcon(i32, i32, i8*, i8*, i8*, i8*)
+declare i8* @llvm.coro.begin(token, i8*)
+declare i1 @llvm.coro.suspend.retcon.i1(...)
+declare i1 @llvm.coro.end(i8*, i1)
+declare i8* @llvm.coro.prepare.retcon(i8*)
+
+declare i8* @prototype(i8*, i1 zeroext)
+declare {i8*,i8*} @g_prototype(i8*, i1 zeroext)
+
+declare noalias i8* @allocate(i32 %size)
+declare void @deallocate(i8* %ptr)
+
+declare void @print(i32)
+
diff --git a/llvm/test/Transforms/Coroutines/coro-split-dbg.ll b/llvm/test/Transforms/Coroutines/coro-split-dbg.ll
index e79d87170..5f767935e 100644
--- a/llvm/test/Transforms/Coroutines/coro-split-dbg.ll
+++ b/llvm/test/Transforms/Coroutines/coro-split-dbg.ll
@@ -62,13 +62,13 @@ declare i8* @llvm.coro.subfn.addr(i8* nocapture readonly, i8) #5
 
 declare void @llvm.dbg.value(metadata, metadata, metadata) #1
 
-attributes #0 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
-attributes #2 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #3 = { nounwind uwtable "coroutine.presplit"="1" "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { nounwind uwtable "coroutine.presplit"="1" "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #4 = { argmemonly nounwind }
 attributes #5 = { argmemonly nounwind readonly }
-attributes #6 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #6 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #7 = { nounwind }
 attributes #8 = { alwaysinline nounwind }
 attributes #9 = { noduplicate }
diff --git a/llvm/test/Transforms/Coroutines/coro-split-musttail.ll b/llvm/test/Transforms/Coroutines/coro-split-musttail.ll
index 6a94d09af..d5af91b0f 100644
--- a/llvm/test/Transforms/Coroutines/coro-split-musttail.ll
+++ b/llvm/test/Transforms/Coroutines/coro-split-musttail.ll
@@ -2,7 +2,7 @@
 ; musttail call.
 ; RUN: opt < %s -coro-split -S | FileCheck %s
 
-define void @f() "coroutine.presplit"="1" {
+define void @f() #0 {
 entry:
   %id = call token @llvm.coro.id(i32 0, i8* null, i8* null, i8* null)
   %alloc = call i8* @malloc(i64 16) #3
@@ -47,14 +47,19 @@ exit:
 ; CHECK-NEXT: musttail call fastcc void %[[pv2]](i8* null)
 ; CHECK-NEXT: ret void
 
-declare token @llvm.coro.id(i32, i8* readnone, i8* nocapture readonly, i8*)
-declare i1 @llvm.coro.alloc(token) #3
-declare i64 @llvm.coro.size.i64() #5
-declare i8* @llvm.coro.begin(token, i8* writeonly) #3
-declare token @llvm.coro.save(i8*) #3
-declare i8* @llvm.coro.frame() #5
-declare i8 @llvm.coro.suspend(token, i1) #3
-declare i8* @llvm.coro.free(token, i8* nocapture readonly) #2
-declare i1 @llvm.coro.end(i8*, i1) #3
-declare i8* @llvm.coro.subfn.addr(i8* nocapture readonly, i8) #5
+declare token @llvm.coro.id(i32, i8* readnone, i8* nocapture readonly, i8*) #1
+declare i1 @llvm.coro.alloc(token) #2
+declare i64 @llvm.coro.size.i64() #3
+declare i8* @llvm.coro.begin(token, i8* writeonly) #2
+declare token @llvm.coro.save(i8*) #2
+declare i8* @llvm.coro.frame() #3
+declare i8 @llvm.coro.suspend(token, i1) #2
+declare i8* @llvm.coro.free(token, i8* nocapture readonly) #1
+declare i1 @llvm.coro.end(i8*, i1) #2
+declare i8* @llvm.coro.subfn.addr(i8* nocapture readonly, i8) #1
 declare i8* @malloc(i64)
+
+attributes #0 = { "coroutine.presplit"="1" }
+attributes #1 = { argmemonly nounwind readonly }
+attributes #2 = { nounwind }
+attributes #3 = { nounwind readnone }
diff --git a/llvm/test/Transforms/Coroutines/coro-split-musttail1.ll b/llvm/test/Transforms/Coroutines/coro-split-musttail1.ll
new file mode 100644
index 000000000..f831041e8
--- /dev/null
+++ b/llvm/test/Transforms/Coroutines/coro-split-musttail1.ll
@@ -0,0 +1,104 @@
+; Tests that coro-split will convert coro.resume followed by a suspend to a
+; musttail call.
+; RUN: opt < %s -coro-split -S | FileCheck %s
+
+define void @f() #0 {
+entry:
+  %id = call token @llvm.coro.id(i32 0, i8* null, i8* null, i8* null)
+  %alloc = call i8* @malloc(i64 16) #3
+  %vFrame = call noalias nonnull i8* @llvm.coro.begin(token %id, i8* %alloc)
+
+  %save = call token @llvm.coro.save(i8* null)
+  %addr1 = call i8* @llvm.coro.subfn.addr(i8* null, i8 0)
+  %pv1 = bitcast i8* %addr1 to void (i8*)*
+  call fastcc void %pv1(i8* null)
+
+  %suspend = call i8 @llvm.coro.suspend(token %save, i1 false)
+  switch i8 %suspend, label %exit [
+    i8 0, label %await.suspend
+    i8 1, label %exit
+  ]
+await.suspend:
+  %save2 = call token @llvm.coro.save(i8* null)
+  %br0 = call i8 @switch_result()
+  switch i8 %br0, label %unreach [
+    i8 0, label %await.resume3
+    i8 1, label %await.resume1
+    i8 2, label %await.resume2
+  ]
+await.resume1:
+  %hdl = call i8* @g()
+  %addr2 = call i8* @llvm.coro.subfn.addr(i8* %hdl, i8 0)
+  %pv2 = bitcast i8* %addr2 to void (i8*)*
+  call fastcc void %pv2(i8* %hdl)
+  br label %final.suspend
+await.resume2:
+  %hdl2 = call i8* @h()
+  %addr3 = call i8* @llvm.coro.subfn.addr(i8* %hdl2, i8 0)
+  %pv3 = bitcast i8* %addr3 to void (i8*)*
+  call fastcc void %pv3(i8* %hdl2)
+  br label %final.suspend
+await.resume3:
+  %addr4 = call i8* @llvm.coro.subfn.addr(i8* null, i8 0)
+  %pv4 = bitcast i8* %addr4 to void (i8*)*
+  call fastcc void %pv4(i8* null)
+  br label %final.suspend
+final.suspend:
+  %suspend2 = call i8 @llvm.coro.suspend(token %save2, i1 false)
+  switch i8 %suspend2, label %exit [
+    i8 0, label %pre.exit
+    i8 1, label %exit
+  ]
+pre.exit:
+  br label %exit
+exit:
+  call i1 @llvm.coro.end(i8* null, i1 false)
+  ret void
+unreach:
+  unreachable
+}
+
+; Verify that in the initial function resume is not marked with musttail.
+; CHECK-LABEL: @f(
+; CHECK: %[[addr1:.+]] = call i8* @llvm.coro.subfn.addr(i8* null, i8 0)
+; CHECK-NEXT: %[[pv1:.+]] = bitcast i8* %[[addr1]] to void (i8*)*
+; CHECK-NOT: musttail call fastcc void %[[pv1]](i8* null)
+
+; Verify that in the resume part resume call is marked with musttail.
+; CHECK-LABEL: @f.resume(
+; CHECK: %[[hdl:.+]] = call i8* @g()
+; CHECK-NEXT: %[[addr2:.+]] = call i8* @llvm.coro.subfn.addr(i8* %[[hdl]], i8 0)
+; CHECK-NEXT: %[[pv2:.+]] = bitcast i8* %[[addr2]] to void (i8*)*
+; CHECK-NEXT: musttail call fastcc void %[[pv2]](i8* %[[hdl]])
+; CHECK-NEXT: ret void
+; CHECK: %[[hdl2:.+]] = call i8* @h()
+; CHECK-NEXT: %[[addr3:.+]] = call i8* @llvm.coro.subfn.addr(i8* %[[hdl2]], i8 0)
+; CHECK-NEXT: %[[pv3:.+]] = bitcast i8* %[[addr3]] to void (i8*)*
+; CHECK-NEXT: musttail call fastcc void %[[pv3]](i8* %[[hdl2]])
+; CHECK-NEXT: ret void
+; CHECK: %[[addr4:.+]] = call i8* @llvm.coro.subfn.addr(i8* null, i8 0)
+; CHECK-NEXT: %[[pv4:.+]] = bitcast i8* %[[addr4]] to void (i8*)*
+; CHECK-NEXT: musttail call fastcc void %[[pv4]](i8* null)
+; CHECK-NEXT: ret void
+
+
+
+declare token @llvm.coro.id(i32, i8* readnone, i8* nocapture readonly, i8*) #1
+declare i1 @llvm.coro.alloc(token) #2
+declare i64 @llvm.coro.size.i64() #3
+declare i8* @llvm.coro.begin(token, i8* writeonly) #2
+declare token @llvm.coro.save(i8*) #2
+declare i8* @llvm.coro.frame() #3
+declare i8 @llvm.coro.suspend(token, i1) #2
+declare i8* @llvm.coro.free(token, i8* nocapture readonly) #1
+declare i1 @llvm.coro.end(i8*, i1) #2
+declare i8* @llvm.coro.subfn.addr(i8* nocapture readonly, i8) #1
+declare i8* @malloc(i64)
+declare i8 @switch_result()
+declare i8* @g()
+declare i8* @h()
+
+attributes #0 = { "coroutine.presplit"="1" }
+attributes #1 = { argmemonly nounwind readonly }
+attributes #2 = { nounwind }
+attributes #3 = { nounwind readnone }
diff --git a/llvm/test/Transforms/Coroutines/coro-swifterror.ll b/llvm/test/Transforms/Coroutines/coro-swifterror.ll
new file mode 100644
index 000000000..932e448a5
--- /dev/null
+++ b/llvm/test/Transforms/Coroutines/coro-swifterror.ll
@@ -0,0 +1,143 @@
+; RUN: opt < %s -enable-coroutines -O2 -S | FileCheck %s
+target datalayout = "E-p:32:32"
+
+define i8* @f(i8* %buffer, i32 %n, i8** swifterror %errorslot) {
+entry:
+  %id = call token @llvm.coro.id.retcon(i32 8, i32 4, i8* %buffer, i8* bitcast (i8* (i8*, i1, i8**)* @f_prototype to i8*), i8* bitcast (i8* (i32)* @allocate to i8*), i8* bitcast (void (i8*)* @deallocate to i8*))
+  %hdl = call i8* @llvm.coro.begin(token %id, i8* null)
+  br label %loop
+
+loop:
+  %n.val = phi i32 [ %n, %entry ], [ %inc, %resume ]
+  call void @print(i32 %n.val)
+  call void @maybeThrow(i8** swifterror %errorslot)
+  %errorload1 = load i8*, i8** %errorslot
+  call void @logError(i8* %errorload1)
+  %suspend_result = call { i1, i8** } (...) @llvm.coro.suspend.retcon.i1p0p0i8()
+  %unwind0 = extractvalue { i1, i8** } %suspend_result, 0
+  br i1 %unwind0, label %cleanup, label %resume
+
+resume:
+  %inc = add i32 %n.val, 1
+  br label %loop
+
+cleanup:
+  call i1 @llvm.coro.end(i8* %hdl, i1 0)
+  unreachable
+}
+
+; CHECK-LABEL: define i8* @f(i8* %buffer, i32 %n, i8** swifterror %errorslot)
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[T0:%.*]] = bitcast i8* %buffer to i32*
+; CHECK-NEXT:    store i32 %n, i32* [[T0]], align 4
+; CHECK-NEXT:    call void @print(i32 %n)
+;   TODO: figure out a way to eliminate this
+; CHECK-NEXT:    store i8* null, i8** %errorslot
+; CHECK-NEXT:    call void @maybeThrow(i8** nonnull swifterror %errorslot)
+; CHECK-NEXT:    [[T1:%.*]] = load i8*, i8** %errorslot
+; CHECK-NEXT:    call void @logError(i8* [[T1]])
+; CHECK-NEXT:    store i8* [[T1]], i8** %errorslot
+; CHECK-NEXT:    ret i8* bitcast (i8* (i8*, i1, i8**)* @f.resume.0 to i8*)
+; CHECK-NEXT:  }
+
+; CHECK-LABEL: define internal i8* @f.resume.0(i8* noalias nonnull %0, i1 zeroext %1, i8** swifterror %2)
+; CHECK-NEXT:  :
+; CHECK-NEXT:    br i1 %1,
+; CHECK:       :
+; CHECK-NEXT:    [[ERROR:%.*]] = load i8*, i8** %2, align 4
+; CHECK-NEXT:    [[T0:%.*]] = bitcast i8* %0 to i32*
+; CHECK-NEXT:    [[T1:%.*]] = load i32, i32* [[T0]], align 4
+; CHECK-NEXT:    %inc = add i32 [[T1]], 1
+; CHECK-NEXT:    store i32 %inc, i32* [[T0]], align 4
+; CHECK-NEXT:    call void @print(i32 %inc)
+; CHECK-NEXT:    store i8* [[ERROR]], i8** %2
+; CHECK-NEXT:    call void @maybeThrow(i8** nonnull swifterror %2)
+; CHECK-NEXT:    [[T2:%.*]] = load i8*, i8** %2
+; CHECK-NEXT:    call void @logError(i8* [[T2]])
+; CHECK-NEXT:    store i8* [[T2]], i8** %2
+; CHECK-NEXT:    ret i8* bitcast (i8* (i8*, i1, i8**)* @f.resume.0 to i8*)
+; CHECK:       :
+; CHECK-NEXT:    ret i8* null
+; CHECK-NEXT:  }
+
+define i8* @g(i8* %buffer, i32 %n) {
+entry:
+  %errorslot = alloca swifterror i8*, align 4
+  store i8* null, i8** %errorslot
+  %id = call token @llvm.coro.id.retcon(i32 8, i32 4, i8* %buffer, i8* bitcast (i8* (i8*, i1)* @g_prototype to i8*), i8* bitcast (i8* (i32)* @allocate to i8*), i8* bitcast (void (i8*)* @deallocate to i8*))
+  %hdl = call i8* @llvm.coro.begin(token %id, i8* null)
+  br label %loop
+
+loop:
+  %n.val = phi i32 [ %n, %entry ], [ %inc, %resume ]
+  call void @print(i32 %n.val)
+  call void @maybeThrow(i8** swifterror %errorslot)
+  %errorload1 = load i8*, i8** %errorslot
+  call void @logError(i8* %errorload1)
+  %unwind0 = call i1 (...) @llvm.coro.suspend.retcon.i1()
+  br i1 %unwind0, label %cleanup, label %resume
+
+resume:
+  %inc = add i32 %n.val, 1
+  br label %loop
+
+cleanup:
+  call i1 @llvm.coro.end(i8* %hdl, i1 0)
+  unreachable
+}
+
+; CHECK-LABEL: define i8* @g(i8* %buffer, i32 %n)
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ERRORSLOT:%.*]] = alloca swifterror i8*, align 4
+; CHECK-NEXT:    [[T0:%.*]] = bitcast i8* %buffer to i32*
+; CHECK-NEXT:    store i32 %n, i32* [[T0]], align 4
+; CHECK-NEXT:    call void @print(i32 %n)
+; CHECK-NEXT:    store i8* null, i8** [[ERRORSLOT]], align 4
+; CHECK-NEXT:    call void @maybeThrow(i8** nonnull swifterror [[ERRORSLOT]])
+; CHECK-NEXT:    [[T1:%.*]] = load i8*, i8** [[ERRORSLOT]], align 4
+; CHECK-NEXT:    [[T2:%.*]] = getelementptr inbounds i8, i8* %buffer, i32 4
+; CHECK-NEXT:    [[T3:%.*]] = bitcast i8* [[T2]] to i8**
+; CHECK-NEXT:    store i8* [[T1]], i8** [[T3]], align 4
+; CHECK-NEXT:    call void @logError(i8* [[T1]])
+; CHECK-NEXT:    ret i8* bitcast (i8* (i8*, i1)* @g.resume.0 to i8*)
+; CHECK-NEXT:  }
+
+; CHECK-LABEL: define internal i8* @g.resume.0(i8* noalias nonnull %0, i1 zeroext %1)
+; CHECK-NEXT:  :
+; CHECK-NEXT:    [[ERRORSLOT:%.*]] = alloca swifterror i8*, align 4
+; CHECK-NEXT:    br i1 %1,
+; CHECK:       :
+; CHECK-NEXT:    [[T0:%.*]] = bitcast i8* %0 to i32*
+; CHECK-NEXT:    [[T1:%.*]] = load i32, i32* [[T0]], align 4
+; CHECK-NEXT:    %inc = add i32 [[T1]], 1
+; CHECK-NEXT:    [[T2:%.*]] = getelementptr inbounds i8, i8* %0, i32 4
+; CHECK-NEXT:    [[T3:%.*]] = bitcast i8* [[T2]] to i8**
+; CHECK-NEXT:    [[T4:%.*]] = load i8*, i8** [[T3]]
+; CHECK-NEXT:    store i32 %inc, i32* [[T0]], align 4
+; CHECK-NEXT:    call void @print(i32 %inc)
+; CHECK-NEXT:    store i8* [[T4]], i8** [[ERRORSLOT]]
+; CHECK-NEXT:    call void @maybeThrow(i8** nonnull swifterror [[ERRORSLOT]])
+; CHECK-NEXT:    [[T5:%.*]] = load i8*, i8** [[ERRORSLOT]]
+; CHECK-NEXT:    store i8* [[T5]], i8** [[T3]], align 4
+; CHECK-NEXT:    call void @logError(i8* [[T5]])
+; CHECK-NEXT:    ret i8* bitcast (i8* (i8*, i1)* @g.resume.0 to i8*)
+; CHECK:       :
+; CHECK-NEXT:    ret i8* null
+; CHECK-NEXT:  }
+
+declare token @llvm.coro.id.retcon(i32, i32, i8*, i8*, i8*, i8*)
+declare i8* @llvm.coro.begin(token, i8*)
+declare { i1, i8** } @llvm.coro.suspend.retcon.i1p0p0i8(...)
+declare i1 @llvm.coro.suspend.retcon.i1(...)
+declare i1 @llvm.coro.end(i8*, i1)
+declare i8* @llvm.coro.prepare.retcon(i8*)
+
+declare i8* @f_prototype(i8*, i1 zeroext, i8** swifterror)
+declare i8* @g_prototype(i8*, i1 zeroext)
+
+declare noalias i8* @allocate(i32 %size)
+declare void @deallocate(i8* %ptr)
+
+declare void @print(i32)
+declare void @maybeThrow(i8** swifterror)
+declare void @logError(i8*)
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/and.ll b/llvm/test/Transforms/CorrelatedValuePropagation/and.ll
new file mode 100644
index 000000000..475824420
--- /dev/null
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/and.ll
@@ -0,0 +1,127 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -correlated-propagation -S | FileCheck %s
+
+define i32 @test(i32 %a) {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[A:%.*]], 128
+; CHECK-NEXT:    br i1 [[CMP]], label [[CONTINUE:%.*]], label [[EXIT:%.*]]
+; CHECK:       continue:
+; CHECK-NEXT:    ret i32 [[A]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i32 -1
+;
+entry:
+  %cmp = icmp ult i32 %a, 128
+  br i1 %cmp, label %continue, label %exit
+continue:
+  %and = and i32 %a, 255
+  ret i32 %and
+exit:
+  ret i32 -1
+}
+
+define i32 @test2(i32 %a) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[A:%.*]], 256
+; CHECK-NEXT:    br i1 [[CMP]], label [[CONTINUE:%.*]], label [[EXIT:%.*]]
+; CHECK:       continue:
+; CHECK-NEXT:    ret i32 [[A]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i32 -1
+;
+entry:
+  %cmp = icmp ult i32 %a, 256
+  br i1 %cmp, label %continue, label %exit
+continue:
+  %and = and i32 %a, 255
+  ret i32 %and
+exit:
+  ret i32 -1
+}
+
+define i32 @test3(i32 %a) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[A:%.*]], 256
+; CHECK-NEXT:    br i1 [[CMP]], label [[CONTINUE:%.*]], label [[EXIT:%.*]]
+; CHECK:       continue:
+; CHECK-NEXT:    ret i32 [[A]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i32 -1
+;
+entry:
+  %cmp = icmp ult i32 %a, 256
+  br i1 %cmp, label %continue, label %exit
+continue:
+  %and = and i32 %a, 1023
+  ret i32 %and
+exit:
+  ret i32 -1
+}
+
+
+define i32 @neg1(i32 %a) {
+; CHECK-LABEL: @neg1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ule i32 [[A:%.*]], 256
+; CHECK-NEXT:    br i1 [[CMP]], label [[CONTINUE:%.*]], label [[EXIT:%.*]]
+; CHECK:       continue:
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[A]], 255
+; CHECK-NEXT:    ret i32 [[AND]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i32 -1
+;
+entry:
+  %cmp = icmp ule i32 %a, 256
+  br i1 %cmp, label %continue, label %exit
+continue:
+  %and = and i32 %a, 255
+  ret i32 %and
+exit:
+  ret i32 -1
+}
+
+define i32 @neg2(i32 %a) {
+; CHECK-LABEL: @neg2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[A:%.*]], 513
+; CHECK-NEXT:    br i1 [[CMP]], label [[CONTINUE:%.*]], label [[EXIT:%.*]]
+; CHECK:       continue:
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[A]], 255
+; CHECK-NEXT:    ret i32 [[AND]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i32 -1
+;
+entry:
+  %cmp = icmp ult i32 %a, 513
+  br i1 %cmp, label %continue, label %exit
+continue:
+  %and = and i32 %a, 255
+  ret i32 %and
+exit:
+  ret i32 -1
+}
+
+define i32 @neg3(i32 %a) {
+; CHECK-LABEL: @neg3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[A:%.*]], 256
+; CHECK-NEXT:    br i1 [[CMP]], label [[CONTINUE:%.*]], label [[EXIT:%.*]]
+; CHECK:       continue:
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[A]], 254
+; CHECK-NEXT:    ret i32 [[AND]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i32 -1
+;
+entry:
+  %cmp = icmp ult i32 %a, 256
+  br i1 %cmp, label %continue, label %exit
+continue:
+  %and = and i32 %a, 254
+  ret i32 %and
+exit:
+  ret i32 -1
+}
+
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/icmp.ll b/llvm/test/Transforms/CorrelatedValuePropagation/icmp.ll
index 3b0f33a51..8f783f828 100644
--- a/llvm/test/Transforms/CorrelatedValuePropagation/icmp.ll
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/icmp.ll
@@ -173,7 +173,7 @@ define i1 @test5(i32 %x, i32 %y) #0 {
 ; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult i32 [[Y:%.*]], 5
 ; CHECK-NEXT:    br i1 [[CMP2]], label [[CONT2:%.*]], label [[OUT]]
 ; CHECK:       cont2:
-; CHECK-NEXT:    [[SHIFTED:%.*]] = shl i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[SHIFTED:%.*]] = shl nuw nsw i32 [[X]], [[Y]]
 ; CHECK-NEXT:    br label [[CONT3:%.*]]
 ; CHECK:       cont3:
 ; CHECK-NEXT:    br label [[OUT]]
@@ -212,7 +212,7 @@ define i1 @test6(i32 %x, i32 %y) #0 {
 ; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult i32 [[Y:%.*]], 15
 ; CHECK-NEXT:    br i1 [[CMP2]], label [[CONT2:%.*]], label [[OUT]]
 ; CHECK:       cont2:
-; CHECK-NEXT:    [[SHIFTED:%.*]] = shl i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[SHIFTED:%.*]] = shl nuw nsw i32 [[X]], [[Y]]
 ; CHECK-NEXT:    br label [[CONT3:%.*]]
 ; CHECK:       cont3:
 ; CHECK-NEXT:    [[CMP3:%.*]] = icmp ult i32 [[SHIFTED]], 65536
@@ -242,4 +242,381 @@ out:
   ret i1 %ret
 }
 
+define i1 @test7(i32 %a, i32 %b) {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:  begin:
+; CHECK-NEXT:    [[CMP0:%.*]] = icmp sge i32 [[A:%.*]], 0
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sge i32 [[B:%.*]], 0
+; CHECK-NEXT:    [[BR:%.*]] = and i1 [[CMP0]], [[CMP1]]
+; CHECK-NEXT:    br i1 [[BR]], label [[BB:%.*]], label [[EXIT:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[ADD:%.*]] = add nuw i32 [[A]], [[B]]
+; CHECK-NEXT:    br label [[CONT:%.*]]
+; CHECK:       cont:
+; CHECK-NEXT:    [[RES:%.*]] = icmp sge i32 [[ADD]], 0
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[IV:%.*]] = phi i1 [ true, [[BEGIN:%.*]] ], [ [[RES]], [[CONT]] ]
+; CHECK-NEXT:    ret i1 [[IV]]
+;
+begin:
+  %cmp0 = icmp sge i32 %a, 0
+  %cmp1 = icmp sge i32 %b, 0
+  %br = and i1 %cmp0, %cmp1
+  br i1 %br, label %bb, label %exit
+
+bb:
+  %add = add i32 %a, %b
+  br label %cont
+
+cont:
+  %res = icmp sge i32 %add, 0
+  br label %exit
+
+exit:
+  %iv = phi i1 [ true, %begin ], [ %res, %cont ]
+  ret i1 %iv
+}
+
+define i1 @test8(i32 %a, i32 %b) {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT:  begin:
+; CHECK-NEXT:    [[CMP0:%.*]] = icmp sge i32 [[A:%.*]], 0
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sge i32 [[B:%.*]], 0
+; CHECK-NEXT:    [[BR:%.*]] = and i1 [[CMP0]], [[CMP1]]
+; CHECK-NEXT:    br i1 [[BR]], label [[BB:%.*]], label [[EXIT:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[ADD:%.*]] = add nuw nsw i32 [[A]], [[B]]
+; CHECK-NEXT:    br label [[CONT:%.*]]
+; CHECK:       cont:
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i1 true
+;
+begin:
+  %cmp0 = icmp sge i32 %a, 0
+  %cmp1 = icmp sge i32 %b, 0
+  %br = and i1 %cmp0, %cmp1
+  br i1 %br, label %bb, label %exit
+
+bb:
+  %add = add nsw i32 %a, %b
+  br label %cont
+
+cont:
+  %res = icmp sge i32 %add, 0
+  br label %exit
+
+exit:
+  %iv = phi i1 [ true, %begin ], [ %res, %cont ]
+  ret i1 %iv
+}
+
+define i1 @test10(i32 %a, i32 %b) {
+; CHECK-LABEL: @test10(
+; CHECK-NEXT:  begin:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp uge i32 [[A:%.*]], -256
+; CHECK-NEXT:    br i1 [[CMP]], label [[BB:%.*]], label [[EXIT:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[A]], [[B:%.*]]
+; CHECK-NEXT:    br label [[CONT:%.*]]
+; CHECK:       cont:
+; CHECK-NEXT:    [[RES:%.*]] = icmp uge i32 [[ADD]], -256
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[IV:%.*]] = phi i1 [ true, [[BEGIN:%.*]] ], [ [[RES]], [[CONT]] ]
+; CHECK-NEXT:    ret i1 [[IV]]
+;
+begin:
+  %cmp = icmp uge i32 %a, 4294967040
+  br i1 %cmp, label %bb, label %exit
+
+bb:
+  %add = add i32 %a, %b
+  br label %cont
+
+cont:
+  %res = icmp uge i32 %add, 4294967040
+  br label %exit
+
+exit:
+  %iv = phi i1 [ true, %begin ], [ %res, %cont ]
+  ret i1 %iv
+}
+
+define i1 @test11(i32 %a, i32 %b) {
+; CHECK-LABEL: @test11(
+; CHECK-NEXT:  begin:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp uge i32 [[A:%.*]], -256
+; CHECK-NEXT:    br i1 [[CMP]], label [[BB:%.*]], label [[EXIT:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[ADD:%.*]] = add nuw i32 [[A]], [[B:%.*]]
+; CHECK-NEXT:    br label [[CONT:%.*]]
+; CHECK:       cont:
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i1 true
+;
+begin:
+  %cmp = icmp uge i32 %a, 4294967040
+  br i1 %cmp, label %bb, label %exit
+
+bb:
+  %add = add nuw i32 %a, %b
+  br label %cont
+
+cont:
+  %res = icmp uge i32 %add, 4294967040
+  br label %exit
+
+exit:
+  %iv = phi i1 [ true, %begin ], [ %res, %cont ]
+  ret i1 %iv
+}
+
+define i1 @test12(i32 %x) {
+; CHECK-LABEL: @test12(
+; CHECK-NEXT:    [[ZEXT:%.*]] = zext i32 [[X:%.*]] to i64
+; CHECK-NEXT:    [[MUL:%.*]] = mul nuw nsw i64 [[ZEXT]], 7
+; CHECK-NEXT:    [[SHR:%.*]] = lshr i64 [[MUL]], 32
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc i64 [[SHR]] to i32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[TRUNC]], 7
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %zext = zext i32 %x to i64
+  %mul = mul nuw i64 %zext, 7
+  %shr = lshr i64 %mul, 32
+  %trunc = trunc i64 %shr to i32
+  %cmp = icmp ult i32 %trunc, 7
+  ret i1 %cmp
+}
+
+define i1 @test13(i8 %x, i64* %p) {
+; CHECK-LABEL: @test13(
+; CHECK-NEXT:    [[ZEXT:%.*]] = zext i8 [[X:%.*]] to i64
+; CHECK-NEXT:    [[ADD:%.*]] = add nuw nsw i64 [[ZEXT]], 128
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i64 [[ADD]], 384
+; CHECK-NEXT:    store i64 [[ADD]], i64* [[P:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %zext = zext i8 %x to i64
+  %add = add nuw nsw i64 %zext, 128
+  %cmp = icmp ult i64 %add, 384
+  ; Without this extra use, InstSimplify could handle this
+  store i64 %add, i64* %p
+  ret i1 %cmp
+}
+
+define i1 @test14(i32 %a, i32 %b) {
+; CHECK-LABEL: @test14(
+; CHECK-NEXT:  begin:
+; CHECK-NEXT:    [[CMP0:%.*]] = icmp sge i32 [[A:%.*]], 0
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sge i32 [[B:%.*]], 0
+; CHECK-NEXT:    [[BR:%.*]] = and i1 [[CMP0]], [[CMP1]]
+; CHECK-NEXT:    br i1 [[BR]], label [[BB:%.*]], label [[EXIT:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 [[A]], [[B]]
+; CHECK-NEXT:    br label [[CONT:%.*]]
+; CHECK:       cont:
+; CHECK-NEXT:    [[RES:%.*]] = icmp sge i32 [[SUB]], 0
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[IV:%.*]] = phi i1 [ true, [[BEGIN:%.*]] ], [ [[RES]], [[CONT]] ]
+; CHECK-NEXT:    ret i1 [[IV]]
+;
+begin:
+  %cmp0 = icmp sge i32 %a, 0
+  %cmp1 = icmp sge i32 %b, 0
+  %br = and i1 %cmp0, %cmp1
+  br i1 %br, label %bb, label %exit
+
+bb:
+  %sub = sub i32 %a, %b
+  br label %cont
+
+cont:
+  %res = icmp sge i32 %sub, 0
+  br label %exit
+
+exit:
+  %iv = phi i1 [ true, %begin ], [ %res, %cont ]
+  ret i1 %iv
+}
+
+define i1 @test15(i32 %a, i32 %b) {
+; CHECK-LABEL: @test15(
+; CHECK-NEXT:  begin:
+; CHECK-NEXT:    [[CMP0:%.*]] = icmp sge i32 [[A:%.*]], 0
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sge i32 [[B:%.*]], 0
+; CHECK-NEXT:    [[BR:%.*]] = and i1 [[CMP0]], [[CMP1]]
+; CHECK-NEXT:    br i1 [[BR]], label [[BB:%.*]], label [[EXIT:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 [[A]], [[B]]
+; CHECK-NEXT:    br label [[CONT:%.*]]
+; CHECK:       cont:
+; CHECK-NEXT:    [[RES:%.*]] = icmp sge i32 [[SUB]], 0
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[IV:%.*]] = phi i1 [ true, [[BEGIN:%.*]] ], [ [[RES]], [[CONT]] ]
+; CHECK-NEXT:    ret i1 [[IV]]
+;
+begin:
+  %cmp0 = icmp sge i32 %a, 0
+  %cmp1 = icmp sge i32 %b, 0
+  %br = and i1 %cmp0, %cmp1
+  br i1 %br, label %bb, label %exit
+
+bb:
+  %sub = sub nsw i32 %a, %b
+  br label %cont
+
+cont:
+  %res = icmp sge i32 %sub, 0
+  br label %exit
+
+exit:
+  %iv = phi i1 [ true, %begin ], [ %res, %cont ]
+  ret i1 %iv
+}
+
+define i1 @test16(i32 %a, i32 %b) {
+; CHECK-LABEL: @test16(
+; CHECK-NEXT:  begin:
+; CHECK-NEXT:    [[CMP0:%.*]] = icmp sge i32 [[A:%.*]], 0
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sge i32 [[B:%.*]], 0
+; CHECK-NEXT:    [[BR:%.*]] = and i1 [[CMP0]], [[CMP1]]
+; CHECK-NEXT:    br i1 [[BR]], label [[BB:%.*]], label [[EXIT:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[SUB:%.*]] = sub nuw nsw i32 [[A]], [[B]]
+; CHECK-NEXT:    br label [[CONT:%.*]]
+; CHECK:       cont:
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i1 true
+;
+begin:
+  %cmp0 = icmp sge i32 %a, 0
+  %cmp1 = icmp sge i32 %b, 0
+  %br = and i1 %cmp0, %cmp1
+  br i1 %br, label %bb, label %exit
+
+bb:
+  %sub = sub nuw i32 %a, %b
+  br label %cont
+
+cont:
+  %res = icmp sge i32 %sub, 0
+  br label %exit
+
+exit:
+  %iv = phi i1 [ true, %begin ], [ %res, %cont ]
+  ret i1 %iv
+}
+
+define i1 @test17(i32 %a, i32 %b) {
+; CHECK-LABEL: @test17(
+; CHECK-NEXT:  begin:
+; CHECK-NEXT:    [[CMP0:%.*]] = icmp sle i32 [[A:%.*]], 0
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sge i32 [[B:%.*]], 0
+; CHECK-NEXT:    [[BR:%.*]] = and i1 [[CMP0]], [[CMP1]]
+; CHECK-NEXT:    br i1 [[BR]], label [[BB:%.*]], label [[EXIT:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 [[A]], [[B]]
+; CHECK-NEXT:    br label [[CONT:%.*]]
+; CHECK:       cont:
+; CHECK-NEXT:    [[RES:%.*]] = icmp sle i32 [[SUB]], 0
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[IV:%.*]] = phi i1 [ true, [[BEGIN:%.*]] ], [ [[RES]], [[CONT]] ]
+; CHECK-NEXT:    ret i1 [[IV]]
+;
+begin:
+  %cmp0 = icmp sle i32 %a, 0
+  %cmp1 = icmp sge i32 %b, 0
+  %br = and i1 %cmp0, %cmp1
+  br i1 %br, label %bb, label %exit
+
+bb:
+  %sub = sub i32 %a, %b
+  br label %cont
+
+cont:
+  %res = icmp sle i32 %sub, 0
+  br label %exit
+
+exit:
+  %iv = phi i1 [ true, %begin ], [ %res, %cont ]
+  ret i1 %iv
+}
+
+define i1 @test18(i32 %a, i32 %b) {
+; CHECK-LABEL: @test18(
+; CHECK-NEXT:  begin:
+; CHECK-NEXT:    [[CMP0:%.*]] = icmp sle i32 [[A:%.*]], 0
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sge i32 [[B:%.*]], 0
+; CHECK-NEXT:    [[BR:%.*]] = and i1 [[CMP0]], [[CMP1]]
+; CHECK-NEXT:    br i1 [[BR]], label [[BB:%.*]], label [[EXIT:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[SUB:%.*]] = sub nuw i32 [[A]], [[B]]
+; CHECK-NEXT:    br label [[CONT:%.*]]
+; CHECK:       cont:
+; CHECK-NEXT:    [[RES:%.*]] = icmp sle i32 [[SUB]], 0
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[IV:%.*]] = phi i1 [ true, [[BEGIN:%.*]] ], [ [[RES]], [[CONT]] ]
+; CHECK-NEXT:    ret i1 [[IV]]
+;
+begin:
+  %cmp0 = icmp sle i32 %a, 0
+  %cmp1 = icmp sge i32 %b, 0
+  %br = and i1 %cmp0, %cmp1
+  br i1 %br, label %bb, label %exit
+
+bb:
+  %sub = sub nuw i32 %a, %b
+  br label %cont
+
+cont:
+  %res = icmp sle i32 %sub, 0
+  br label %exit
+
+exit:
+  %iv = phi i1 [ true, %begin ], [ %res, %cont ]
+  ret i1 %iv
+}
+
+define i1 @test19(i32 %a, i32 %b) {
+; CHECK-LABEL: @test19(
+; CHECK-NEXT:  begin:
+; CHECK-NEXT:    [[CMP0:%.*]] = icmp sle i32 [[A:%.*]], 0
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sge i32 [[B:%.*]], 0
+; CHECK-NEXT:    [[BR:%.*]] = and i1 [[CMP0]], [[CMP1]]
+; CHECK-NEXT:    br i1 [[BR]], label [[BB:%.*]], label [[EXIT:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 [[A]], [[B]]
+; CHECK-NEXT:    br label [[CONT:%.*]]
+; CHECK:       cont:
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i1 true
+;
+begin:
+  %cmp0 = icmp sle i32 %a, 0
+  %cmp1 = icmp sge i32 %b, 0
+  %br = and i1 %cmp0, %cmp1
+  br i1 %br, label %bb, label %exit
+
+bb:
+  %sub = sub nsw i32 %a, %b
+  br label %cont
+
+cont:
+  %res = icmp sle i32 %sub, 0
+  br label %exit
+
+exit:
+  %iv = phi i1 [ true, %begin ], [ %res, %cont ]
+  ret i1 %iv
+}
+
 attributes #4 = { noreturn }
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/mul.ll b/llvm/test/Transforms/CorrelatedValuePropagation/mul.ll
new file mode 100644
index 000000000..288d114cb
--- /dev/null
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/mul.ll
@@ -0,0 +1,175 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -correlated-propagation -cvp-dont-add-nowrap-flags=false -S | FileCheck %s
+
+define i8 @test0(i8 %a) {
+; CHECK-LABEL: @test0(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i8 [[A:%.*]], 3
+; CHECK-NEXT:    br i1 [[CMP]], label [[BB:%.*]], label [[EXIT:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[MUL:%.*]] = mul nuw nsw i8 [[A]], 50
+; CHECK-NEXT:    ret i8 [[MUL]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i8 0
+;
+entry:
+  %cmp = icmp ult i8 %a, 3
+  br i1 %cmp, label %bb, label %exit
+
+bb:
+  %mul = mul i8 %a, 50
+  ret i8 %mul
+
+exit:
+  ret i8 0
+}
+
+define i8 @test1(i8 %a) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i8 [[A:%.*]], 4
+; CHECK-NEXT:    br i1 [[CMP]], label [[BB:%.*]], label [[EXIT:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[MUL:%.*]] = mul nuw i8 [[A]], 50
+; CHECK-NEXT:    ret i8 [[MUL]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i8 0
+;
+entry:
+  %cmp = icmp ult i8 %a, 4
+  br i1 %cmp, label %bb, label %exit
+
+bb:
+  %mul = mul i8 %a, 50
+  ret i8 %mul
+
+exit:
+  ret i8 0
+}
+
+define i8 @test2(i8 %a) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i8 [[A:%.*]], 6
+; CHECK-NEXT:    br i1 [[CMP]], label [[BB:%.*]], label [[EXIT:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[MUL:%.*]] = mul nuw i8 [[A]], 50
+; CHECK-NEXT:    ret i8 [[MUL]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i8 0
+;
+entry:
+  %cmp = icmp ult i8 %a, 6
+  br i1 %cmp, label %bb, label %exit
+
+bb:
+  %mul = mul i8 %a, 50
+  ret i8 %mul
+
+exit:
+  ret i8 0
+}
+
+define i8 @test3(i8 %a) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i8 [[A:%.*]], 7
+; CHECK-NEXT:    br i1 [[CMP]], label [[BB:%.*]], label [[EXIT:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[MUL:%.*]] = mul i8 [[A]], 50
+; CHECK-NEXT:    ret i8 [[MUL]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i8 0
+;
+entry:
+  %cmp = icmp ult i8 %a, 7
+  br i1 %cmp, label %bb, label %exit
+
+bb:
+  %mul = mul i8 %a, 50
+  ret i8 %mul
+
+exit:
+  ret i8 0
+}
+
+define i8 @test4(i8 %a) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i8 [[A:%.*]], 3
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i8 [[A]], -3
+; CHECK-NEXT:    [[COND:%.*]] = and i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    br i1 [[COND]], label [[BB:%.*]], label [[EXIT:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i8 [[A]], 50
+; CHECK-NEXT:    ret i8 [[MUL]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i8 0
+;
+entry:
+  %cmp1 = icmp slt i8 %a, 3
+  %cmp2 = icmp sgt i8 %a, -3
+  %cond = and i1 %cmp1, %cmp2
+  br i1 %cond, label %bb, label %exit
+
+bb:
+  %mul = mul i8 %a, 50
+  ret i8 %mul
+
+exit:
+  ret i8 0
+}
+
+define i8 @test5(i8 %a) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i8 [[A:%.*]], 3
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i8 [[A]], -4
+; CHECK-NEXT:    [[COND:%.*]] = and i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    br i1 [[COND]], label [[BB:%.*]], label [[EXIT:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[MUL:%.*]] = mul i8 [[A]], 50
+; CHECK-NEXT:    ret i8 [[MUL]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i8 0
+;
+entry:
+  %cmp1 = icmp slt i8 %a, 3
+  %cmp2 = icmp sgt i8 %a, -4
+  %cond = and i1 %cmp1, %cmp2
+  br i1 %cond, label %bb, label %exit
+
+bb:
+  %mul = mul i8 %a, 50
+  ret i8 %mul
+
+exit:
+  ret i8 0
+}
+
+define i8 @test6(i8 %a) {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i8 [[A:%.*]], 4
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i8 [[A]], -3
+; CHECK-NEXT:    [[COND:%.*]] = and i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    br i1 [[COND]], label [[BB:%.*]], label [[EXIT:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[MUL:%.*]] = mul i8 [[A]], 50
+; CHECK-NEXT:    ret i8 [[MUL]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i8 0
+;
+entry:
+  %cmp1 = icmp slt i8 %a, 4
+  %cmp2 = icmp sgt i8 %a, -3
+  %cond = and i1 %cmp1, %cmp2
+  br i1 %cond, label %bb, label %exit
+
+bb:
+  %mul = mul i8 %a, 50
+  ret i8 %mul
+
+exit:
+  ret i8 0
+}
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/overflows.ll b/llvm/test/Transforms/CorrelatedValuePropagation/overflows.ll
index 04b1471eb..211e63aaf 100644
--- a/llvm/test/Transforms/CorrelatedValuePropagation/overflows.ll
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/overflows.ll
@@ -38,38 +38,36 @@ define i32 @signed_add(i32 %x, i32 %y) {
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[Y:%.*]], 0
 ; CHECK-NEXT:    br i1 [[CMP]], label [[LAND_LHS_TRUE:%.*]], label [[LOR_LHS_FALSE:%.*]]
 ; CHECK:       land.lhs.true:
-; CHECK-NEXT:    [[TMP0:%.*]] = sub nsw i32 2147483647, [[Y]]
-; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue { i32, i1 } undef, i32 [[TMP0]], 0
-; CHECK-NEXT:    [[TMP2:%.*]] = insertvalue { i32, i1 } [[TMP1]], i1 false, 1
-; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1
-; CHECK-NEXT:    br i1 [[TMP3]], label [[TRAP:%.*]], label [[CONT:%.*]]
+; CHECK-NEXT:    [[TMP0:%.*]] = sub nuw nsw i32 2147483647, [[Y]]
+; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue { i32, i1 } { i32 undef, i1 false }, i32 [[TMP0]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1
+; CHECK-NEXT:    br i1 [[TMP2]], label [[TRAP:%.*]], label [[CONT:%.*]]
 ; CHECK:       trap:
 ; CHECK-NEXT:    tail call void @llvm.trap()
 ; CHECK-NEXT:    unreachable
 ; CHECK:       cont:
-; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { i32, i1 } [[TMP2]], 0
-; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[TMP4]], [[X:%.*]]
+; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { i32, i1 } [[TMP1]], 0
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[TMP3]], [[X:%.*]]
 ; CHECK-NEXT:    br i1 [[CMP1]], label [[COND_END:%.*]], label [[COND_FALSE:%.*]]
 ; CHECK:       lor.lhs.false:
 ; CHECK-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[Y]], 0
 ; CHECK-NEXT:    br i1 [[CMP2]], label [[LAND_LHS_TRUE3:%.*]], label [[COND_FALSE]]
 ; CHECK:       land.lhs.true3:
-; CHECK-NEXT:    [[TMP5:%.*]] = sub nsw i32 -2147483648, [[Y]]
-; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { i32, i1 } undef, i32 [[TMP5]], 0
-; CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { i32, i1 } [[TMP6]], i1 false, 1
-; CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { i32, i1 } [[TMP7]], 1
-; CHECK-NEXT:    br i1 [[TMP8]], label [[TRAP]], label [[CONT4:%.*]]
+; CHECK-NEXT:    [[TMP4:%.*]] = sub nsw i32 -2147483648, [[Y]]
+; CHECK-NEXT:    [[TMP5:%.*]] = insertvalue { i32, i1 } { i32 undef, i1 false }, i32 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; CHECK-NEXT:    br i1 [[TMP6]], label [[TRAP]], label [[CONT4:%.*]]
 ; CHECK:       cont4:
-; CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { i32, i1 } [[TMP7]], 0
-; CHECK-NEXT:    [[CMP5:%.*]] = icmp sgt i32 [[TMP9]], [[X]]
+; CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
+; CHECK-NEXT:    [[CMP5:%.*]] = icmp sgt i32 [[TMP7]], [[X]]
 ; CHECK-NEXT:    br i1 [[CMP5]], label [[COND_END]], label [[COND_FALSE]]
 ; CHECK:       cond.false:
-; CHECK-NEXT:    [[TMP10:%.*]] = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[X]], i32 [[Y]])
-; CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { i32, i1 } [[TMP10]], 0
-; CHECK-NEXT:    [[TMP12:%.*]] = extractvalue { i32, i1 } [[TMP10]], 1
-; CHECK-NEXT:    br i1 [[TMP12]], label [[TRAP]], label [[COND_END]]
+; CHECK-NEXT:    [[TMP8:%.*]] = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[X]], i32 [[Y]])
+; CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { i32, i1 } [[TMP8]], 0
+; CHECK-NEXT:    [[TMP10:%.*]] = extractvalue { i32, i1 } [[TMP8]], 1
+; CHECK-NEXT:    br i1 [[TMP10]], label [[TRAP]], label [[COND_END]]
 ; CHECK:       cond.end:
-; CHECK-NEXT:    [[COND:%.*]] = phi i32 [ 0, [[CONT4]] ], [ 0, [[CONT]] ], [ [[TMP11]], [[COND_FALSE]] ]
+; CHECK-NEXT:    [[COND:%.*]] = phi i32 [ 0, [[CONT4]] ], [ 0, [[CONT]] ], [ [[TMP9]], [[COND_FALSE]] ]
 ; CHECK-NEXT:    ret i32 [[COND]]
 ;
 entry:
@@ -118,25 +116,24 @@ cond.end:                                         ; preds = %cond.false, %cont,
 define i32 @unsigned_add(i32 %x, i32 %y) {
 ; CHECK-LABEL: @unsigned_add(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = sub nuw i32 -1, [[Y:%.*]]
-; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue { i32, i1 } undef, i32 [[TMP0]], 0
-; CHECK-NEXT:    [[TMP2:%.*]] = insertvalue { i32, i1 } [[TMP1]], i1 false, 1
-; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1
-; CHECK-NEXT:    br i1 [[TMP3]], label [[TRAP:%.*]], label [[CONT:%.*]]
+; CHECK-NEXT:    [[TMP0:%.*]] = sub nuw nsw i32 -1, [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue { i32, i1 } { i32 undef, i1 false }, i32 [[TMP0]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1
+; CHECK-NEXT:    br i1 [[TMP2]], label [[TRAP:%.*]], label [[CONT:%.*]]
 ; CHECK:       trap:
 ; CHECK-NEXT:    tail call void @llvm.trap()
 ; CHECK-NEXT:    unreachable
 ; CHECK:       cont:
-; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { i32, i1 } [[TMP2]], 0
-; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult i32 [[TMP4]], [[X:%.*]]
+; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { i32, i1 } [[TMP1]], 0
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult i32 [[TMP3]], [[X:%.*]]
 ; CHECK-NEXT:    br i1 [[CMP1]], label [[COND_END:%.*]], label [[COND_FALSE:%.*]]
 ; CHECK:       cond.false:
-; CHECK-NEXT:    [[TMP5:%.*]] = tail call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 [[X]], i32 [[Y]])
-; CHECK-NEXT:    [[TMP6:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
-; CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
-; CHECK-NEXT:    br i1 [[TMP7]], label [[TRAP]], label [[COND_END]]
+; CHECK-NEXT:    [[TMP4:%.*]] = tail call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 [[X]], i32 [[Y]])
+; CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1
+; CHECK-NEXT:    br i1 [[TMP6]], label [[TRAP]], label [[COND_END]]
 ; CHECK:       cond.end:
-; CHECK-NEXT:    [[COND:%.*]] = phi i32 [ 0, [[CONT]] ], [ [[TMP6]], [[COND_FALSE]] ]
+; CHECK-NEXT:    [[COND:%.*]] = phi i32 [ 0, [[CONT]] ], [ [[TMP5]], [[COND_FALSE]] ]
 ; CHECK-NEXT:    ret i32 [[COND]]
 ;
 entry:
@@ -171,37 +168,35 @@ define i32 @signed_sub(i32 %x, i32 %y) {
 ; CHECK-NEXT:    br i1 [[CMP]], label [[LAND_LHS_TRUE:%.*]], label [[LOR_LHS_FALSE:%.*]]
 ; CHECK:       land.lhs.true:
 ; CHECK-NEXT:    [[TMP0:%.*]] = add nsw i32 [[Y]], 2147483647
-; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue { i32, i1 } undef, i32 [[TMP0]], 0
-; CHECK-NEXT:    [[TMP2:%.*]] = insertvalue { i32, i1 } [[TMP1]], i1 false, 1
-; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1
-; CHECK-NEXT:    br i1 [[TMP3]], label [[TRAP:%.*]], label [[CONT:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue { i32, i1 } { i32 undef, i1 false }, i32 [[TMP0]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1
+; CHECK-NEXT:    br i1 [[TMP2]], label [[TRAP:%.*]], label [[CONT:%.*]]
 ; CHECK:       trap:
 ; CHECK-NEXT:    tail call void @llvm.trap()
 ; CHECK-NEXT:    unreachable
 ; CHECK:       cont:
-; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { i32, i1 } [[TMP2]], 0
-; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[TMP4]], [[X:%.*]]
+; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { i32, i1 } [[TMP1]], 0
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[TMP3]], [[X:%.*]]
 ; CHECK-NEXT:    br i1 [[CMP1]], label [[COND_END:%.*]], label [[COND_FALSE:%.*]]
 ; CHECK:       lor.lhs.false:
 ; CHECK-NEXT:    [[CMP2:%.*]] = icmp eq i32 [[Y]], 0
 ; CHECK-NEXT:    br i1 [[CMP2]], label [[COND_FALSE]], label [[LAND_LHS_TRUE3:%.*]]
 ; CHECK:       land.lhs.true3:
-; CHECK-NEXT:    [[TMP5:%.*]] = add nsw i32 [[Y]], -2147483648
-; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { i32, i1 } undef, i32 [[TMP5]], 0
-; CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { i32, i1 } [[TMP6]], i1 false, 1
-; CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { i32, i1 } [[TMP7]], 1
-; CHECK-NEXT:    br i1 [[TMP8]], label [[TRAP]], label [[CONT4:%.*]]
+; CHECK-NEXT:    [[TMP4:%.*]] = add nuw nsw i32 [[Y]], -2147483648
+; CHECK-NEXT:    [[TMP5:%.*]] = insertvalue { i32, i1 } { i32 undef, i1 false }, i32 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; CHECK-NEXT:    br i1 [[TMP6]], label [[TRAP]], label [[CONT4:%.*]]
 ; CHECK:       cont4:
-; CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { i32, i1 } [[TMP7]], 0
-; CHECK-NEXT:    [[CMP5:%.*]] = icmp sgt i32 [[TMP9]], [[X]]
+; CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
+; CHECK-NEXT:    [[CMP5:%.*]] = icmp sgt i32 [[TMP7]], [[X]]
 ; CHECK-NEXT:    br i1 [[CMP5]], label [[COND_END]], label [[COND_FALSE]]
 ; CHECK:       cond.false:
-; CHECK-NEXT:    [[TMP10:%.*]] = tail call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 [[X]], i32 [[Y]])
-; CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { i32, i1 } [[TMP10]], 0
-; CHECK-NEXT:    [[TMP12:%.*]] = extractvalue { i32, i1 } [[TMP10]], 1
-; CHECK-NEXT:    br i1 [[TMP12]], label [[TRAP]], label [[COND_END]]
+; CHECK-NEXT:    [[TMP8:%.*]] = tail call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 [[X]], i32 [[Y]])
+; CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { i32, i1 } [[TMP8]], 0
+; CHECK-NEXT:    [[TMP10:%.*]] = extractvalue { i32, i1 } [[TMP8]], 1
+; CHECK-NEXT:    br i1 [[TMP10]], label [[TRAP]], label [[COND_END]]
 ; CHECK:       cond.end:
-; CHECK-NEXT:    [[COND:%.*]] = phi i32 [ 0, [[CONT4]] ], [ 0, [[CONT]] ], [ [[TMP11]], [[COND_FALSE]] ]
+; CHECK-NEXT:    [[COND:%.*]] = phi i32 [ 0, [[CONT4]] ], [ 0, [[CONT]] ], [ [[TMP9]], [[COND_FALSE]] ]
 ; CHECK-NEXT:    ret i32 [[COND]]
 ;
 entry:
@@ -290,16 +285,15 @@ define i32 @signed_add_r1(i32 %x) {
 ; CHECK-NEXT:    br i1 [[CMP]], label [[COND_END:%.*]], label [[COND_FALSE:%.*]]
 ; CHECK:       cond.false:
 ; CHECK-NEXT:    [[TMP0:%.*]] = add nsw i32 [[X]], 1
-; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue { i32, i1 } undef, i32 [[TMP0]], 0
-; CHECK-NEXT:    [[TMP2:%.*]] = insertvalue { i32, i1 } [[TMP1]], i1 false, 1
-; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { i32, i1 } [[TMP2]], 0
-; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1
-; CHECK-NEXT:    br i1 [[TMP4]], label [[TRAP:%.*]], label [[COND_END]]
+; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue { i32, i1 } { i32 undef, i1 false }, i32 [[TMP0]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { i32, i1 } [[TMP1]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1
+; CHECK-NEXT:    br i1 [[TMP3]], label [[TRAP:%.*]], label [[COND_END]]
 ; CHECK:       trap:
 ; CHECK-NEXT:    tail call void @llvm.trap()
 ; CHECK-NEXT:    unreachable
 ; CHECK:       cond.end:
-; CHECK-NEXT:    [[COND:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP3]], [[COND_FALSE]] ]
+; CHECK-NEXT:    [[COND:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP2]], [[COND_FALSE]] ]
 ; CHECK-NEXT:    ret i32 [[COND]]
 ;
 entry:
@@ -328,16 +322,15 @@ define i32 @unsigned_add_r1(i32 %x) {
 ; CHECK-NEXT:    br i1 [[CMP]], label [[COND_END:%.*]], label [[COND_FALSE:%.*]]
 ; CHECK:       cond.false:
 ; CHECK-NEXT:    [[TMP0:%.*]] = add nuw i32 [[X]], 1
-; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue { i32, i1 } undef, i32 [[TMP0]], 0
-; CHECK-NEXT:    [[TMP2:%.*]] = insertvalue { i32, i1 } [[TMP1]], i1 false, 1
-; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { i32, i1 } [[TMP2]], 0
-; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1
-; CHECK-NEXT:    br i1 [[TMP4]], label [[TRAP:%.*]], label [[COND_END]]
+; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue { i32, i1 } { i32 undef, i1 false }, i32 [[TMP0]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { i32, i1 } [[TMP1]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1
+; CHECK-NEXT:    br i1 [[TMP3]], label [[TRAP:%.*]], label [[COND_END]]
 ; CHECK:       trap:
 ; CHECK-NEXT:    tail call void @llvm.trap()
 ; CHECK-NEXT:    unreachable
 ; CHECK:       cond.end:
-; CHECK-NEXT:    [[COND:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP3]], [[COND_FALSE]] ]
+; CHECK-NEXT:    [[COND:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP2]], [[COND_FALSE]] ]
 ; CHECK-NEXT:    ret i32 [[COND]]
 ;
 entry:
@@ -366,16 +359,15 @@ define i32 @signed_sub_r1(i32 %x) {
 ; CHECK-NEXT:    br i1 [[CMP]], label [[COND_END:%.*]], label [[COND_FALSE:%.*]]
 ; CHECK:       cond.false:
 ; CHECK-NEXT:    [[TMP0:%.*]] = sub nsw i32 [[X]], 1
-; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue { i32, i1 } undef, i32 [[TMP0]], 0
-; CHECK-NEXT:    [[TMP2:%.*]] = insertvalue { i32, i1 } [[TMP1]], i1 false, 1
-; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { i32, i1 } [[TMP2]], 0
-; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1
-; CHECK-NEXT:    br i1 [[TMP4]], label [[TRAP:%.*]], label [[COND_END]]
+; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue { i32, i1 } { i32 undef, i1 false }, i32 [[TMP0]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { i32, i1 } [[TMP1]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1
+; CHECK-NEXT:    br i1 [[TMP3]], label [[TRAP:%.*]], label [[COND_END]]
 ; CHECK:       trap:
 ; CHECK-NEXT:    tail call void @llvm.trap()
 ; CHECK-NEXT:    unreachable
 ; CHECK:       cond.end:
-; CHECK-NEXT:    [[COND:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP3]], [[COND_FALSE]] ]
+; CHECK-NEXT:    [[COND:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP2]], [[COND_FALSE]] ]
 ; CHECK-NEXT:    ret i32 [[COND]]
 ;
 entry:
@@ -404,16 +396,15 @@ define i32 @unsigned_sub_r1(i32 %x) {
 ; CHECK-NEXT:    br i1 [[CMP]], label [[COND_END:%.*]], label [[COND_FALSE:%.*]]
 ; CHECK:       cond.false:
 ; CHECK-NEXT:    [[TMP0:%.*]] = sub nuw i32 [[X]], 1
-; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue { i32, i1 } undef, i32 [[TMP0]], 0
-; CHECK-NEXT:    [[TMP2:%.*]] = insertvalue { i32, i1 } [[TMP1]], i1 false, 1
-; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { i32, i1 } [[TMP2]], 0
-; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1
-; CHECK-NEXT:    br i1 [[TMP4]], label [[TRAP:%.*]], label [[COND_END]]
+; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue { i32, i1 } { i32 undef, i1 false }, i32 [[TMP0]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { i32, i1 } [[TMP1]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1
+; CHECK-NEXT:    br i1 [[TMP3]], label [[TRAP:%.*]], label [[COND_END]]
 ; CHECK:       trap:
 ; CHECK-NEXT:    tail call void @llvm.trap()
 ; CHECK-NEXT:    unreachable
 ; CHECK:       cond.end:
-; CHECK-NEXT:    [[COND:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP3]], [[COND_FALSE]] ]
+; CHECK-NEXT:    [[COND:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP2]], [[COND_FALSE]] ]
 ; CHECK-NEXT:    ret i32 [[COND]]
 ;
 entry:
@@ -442,16 +433,15 @@ define i32 @signed_add_rn1(i32 %x) {
 ; CHECK-NEXT:    br i1 [[CMP]], label [[COND_END:%.*]], label [[COND_FALSE:%.*]]
 ; CHECK:       cond.false:
 ; CHECK-NEXT:    [[TMP0:%.*]] = add nsw i32 [[X]], -1
-; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue { i32, i1 } undef, i32 [[TMP0]], 0
-; CHECK-NEXT:    [[TMP2:%.*]] = insertvalue { i32, i1 } [[TMP1]], i1 false, 1
-; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { i32, i1 } [[TMP2]], 0
-; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1
-; CHECK-NEXT:    br i1 [[TMP4]], label [[TRAP:%.*]], label [[COND_END]]
+; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue { i32, i1 } { i32 undef, i1 false }, i32 [[TMP0]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { i32, i1 } [[TMP1]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1
+; CHECK-NEXT:    br i1 [[TMP3]], label [[TRAP:%.*]], label [[COND_END]]
 ; CHECK:       trap:
 ; CHECK-NEXT:    tail call void @llvm.trap()
 ; CHECK-NEXT:    unreachable
 ; CHECK:       cond.end:
-; CHECK-NEXT:    [[COND:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP3]], [[COND_FALSE]] ]
+; CHECK-NEXT:    [[COND:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP2]], [[COND_FALSE]] ]
 ; CHECK-NEXT:    ret i32 [[COND]]
 ;
 entry:
@@ -480,16 +470,15 @@ define i32 @signed_sub_rn1(i32 %x) {
 ; CHECK-NEXT:    br i1 [[CMP]], label [[COND_END:%.*]], label [[COND_FALSE:%.*]]
 ; CHECK:       cond.false:
 ; CHECK-NEXT:    [[TMP0:%.*]] = sub nsw i32 [[X]], -1
-; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue { i32, i1 } undef, i32 [[TMP0]], 0
-; CHECK-NEXT:    [[TMP2:%.*]] = insertvalue { i32, i1 } [[TMP1]], i1 false, 1
-; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { i32, i1 } [[TMP2]], 0
-; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1
-; CHECK-NEXT:    br i1 [[TMP4]], label [[TRAP:%.*]], label [[COND_END]]
+; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue { i32, i1 } { i32 undef, i1 false }, i32 [[TMP0]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { i32, i1 } [[TMP1]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1
+; CHECK-NEXT:    br i1 [[TMP3]], label [[TRAP:%.*]], label [[COND_END]]
 ; CHECK:       trap:
 ; CHECK-NEXT:    tail call void @llvm.trap()
 ; CHECK-NEXT:    unreachable
 ; CHECK:       cond.end:
-; CHECK-NEXT:    [[COND:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP3]], [[COND_FALSE]] ]
+; CHECK-NEXT:    [[COND:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP2]], [[COND_FALSE]] ]
 ; CHECK-NEXT:    ret i32 [[COND]]
 ;
 entry:
@@ -517,11 +506,10 @@ define i32 @unsigned_mul(i32 %x) {
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[X:%.*]], 10000
 ; CHECK-NEXT:    br i1 [[CMP]], label [[COND_END:%.*]], label [[COND_FALSE:%.*]]
 ; CHECK:       cond.false:
-; CHECK-NEXT:    [[MULO1:%.*]] = mul nuw i32 [[X]], 100
-; CHECK-NEXT:    [[TMP0:%.*]] = insertvalue { i32, i1 } undef, i32 [[MULO1]], 0
-; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue { i32, i1 } [[TMP0]], i1 false, 1
-; CHECK-NEXT:    [[RES:%.*]] = extractvalue { i32, i1 } [[TMP1]], 0
-; CHECK-NEXT:    [[OV:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1
+; CHECK-NEXT:    [[MULO1:%.*]] = mul nuw nsw i32 [[X]], 100
+; CHECK-NEXT:    [[TMP0:%.*]] = insertvalue { i32, i1 } { i32 undef, i1 false }, i32 [[MULO1]], 0
+; CHECK-NEXT:    [[RES:%.*]] = extractvalue { i32, i1 } [[TMP0]], 0
+; CHECK-NEXT:    [[OV:%.*]] = extractvalue { i32, i1 } [[TMP0]], 1
 ; CHECK-NEXT:    br i1 [[OV]], label [[TRAP:%.*]], label [[COND_END]]
 ; CHECK:       trap:
 ; CHECK-NEXT:    tail call void @llvm.trap()
@@ -558,10 +546,9 @@ define i32 @signed_mul(i32 %x) {
 ; CHECK-NEXT:    br i1 [[CMP3]], label [[COND_END:%.*]], label [[COND_FALSE:%.*]]
 ; CHECK:       cond.false:
 ; CHECK-NEXT:    [[MULO1:%.*]] = mul nsw i32 [[X]], 100
-; CHECK-NEXT:    [[TMP0:%.*]] = insertvalue { i32, i1 } undef, i32 [[MULO1]], 0
-; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue { i32, i1 } [[TMP0]], i1 false, 1
-; CHECK-NEXT:    [[RES:%.*]] = extractvalue { i32, i1 } [[TMP1]], 0
-; CHECK-NEXT:    [[OV:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1
+; CHECK-NEXT:    [[TMP0:%.*]] = insertvalue { i32, i1 } { i32 undef, i1 false }, i32 [[MULO1]], 0
+; CHECK-NEXT:    [[RES:%.*]] = extractvalue { i32, i1 } [[TMP0]], 0
+; CHECK-NEXT:    [[OV:%.*]] = extractvalue { i32, i1 } [[TMP0]], 1
 ; CHECK-NEXT:    br i1 [[OV]], label [[TRAP:%.*]], label [[COND_END]]
 ; CHECK:       trap:
 ; CHECK-NEXT:    tail call void @llvm.trap()
@@ -601,19 +588,18 @@ define void @unsigned_loop(i32 %i) {
 ; CHECK:       while.body.preheader:
 ; CHECK-NEXT:    br label [[WHILE_BODY:%.*]]
 ; CHECK:       while.body:
-; CHECK-NEXT:    [[I_ADDR_04:%.*]] = phi i32 [ [[TMP4:%.*]], [[CONT:%.*]] ], [ [[I]], [[WHILE_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[I_ADDR_04:%.*]] = phi i32 [ [[TMP3:%.*]], [[CONT:%.*]] ], [ [[I]], [[WHILE_BODY_PREHEADER]] ]
 ; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 @bar(i32 [[I_ADDR_04]])
 ; CHECK-NEXT:    [[TMP0:%.*]] = sub nuw i32 [[I_ADDR_04]], 1
-; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue { i32, i1 } undef, i32 [[TMP0]], 0
-; CHECK-NEXT:    [[TMP2:%.*]] = insertvalue { i32, i1 } [[TMP1]], i1 false, 1
-; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1
-; CHECK-NEXT:    br i1 [[TMP3]], label [[TRAP:%.*]], label [[CONT]]
+; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue { i32, i1 } { i32 undef, i1 false }, i32 [[TMP0]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1
+; CHECK-NEXT:    br i1 [[TMP2]], label [[TRAP:%.*]], label [[CONT]]
 ; CHECK:       trap:
 ; CHECK-NEXT:    tail call void @llvm.trap()
 ; CHECK-NEXT:    unreachable
 ; CHECK:       cont:
-; CHECK-NEXT:    [[TMP4]] = extractvalue { i32, i1 } [[TMP2]], 0
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP3]] = extractvalue { i32, i1 } [[TMP1]], 0
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP3]], 0
 ; CHECK-NEXT:    br i1 [[CMP]], label [[WHILE_END]], label [[WHILE_BODY]]
 ; CHECK:       while.end:
 ; CHECK-NEXT:    ret void
@@ -651,33 +637,32 @@ define void @intrinsic_into_phi(i32 %n) {
 ; CHECK-NEXT:    br label [[CONT:%.*]]
 ; CHECK:       for.cond:
 ; CHECK-NEXT:    [[TMP0:%.*]] = add nsw i32 [[DOTLCSSA:%.*]], 1
-; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue { i32, i1 } undef, i32 [[TMP0]], 0
-; CHECK-NEXT:    [[TMP2:%.*]] = insertvalue { i32, i1 } [[TMP1]], i1 false, 1
-; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1
-; CHECK-NEXT:    br i1 [[TMP3]], label [[TRAP:%.*]], label [[CONT]]
+; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue { i32, i1 } { i32 undef, i1 false }, i32 [[TMP0]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1
+; CHECK-NEXT:    br i1 [[TMP2]], label [[TRAP:%.*]], label [[CONT]]
 ; CHECK:       trap:
 ; CHECK-NEXT:    tail call void @llvm.trap()
 ; CHECK-NEXT:    unreachable
 ; CHECK:       cont:
-; CHECK-NEXT:    [[TMP4:%.*]] = phi { i32, i1 } [ zeroinitializer, [[ENTRY:%.*]] ], [ [[TMP2]], [[FOR_COND:%.*]] ]
-; CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0
-; CHECK-NEXT:    [[CALL9:%.*]] = tail call i32 @bar(i32 [[TMP5]])
+; CHECK-NEXT:    [[TMP3:%.*]] = phi { i32, i1 } [ zeroinitializer, [[ENTRY:%.*]] ], [ [[TMP1]], [[FOR_COND:%.*]] ]
+; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { i32, i1 } [[TMP3]], 0
+; CHECK-NEXT:    [[CALL9:%.*]] = tail call i32 @bar(i32 [[TMP4]])
 ; CHECK-NEXT:    [[TOBOOL10:%.*]] = icmp eq i32 [[CALL9]], 0
 ; CHECK-NEXT:    br i1 [[TOBOOL10]], label [[WHILE_END:%.*]], label [[WHILE_BODY_PREHEADER:%.*]]
 ; CHECK:       while.body.preheader:
 ; CHECK-NEXT:    br label [[WHILE_BODY:%.*]]
 ; CHECK:       while.cond:
-; CHECK-NEXT:    [[TMP6:%.*]] = extractvalue { i32, i1 } [[TMP8:%.*]], 0
-; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 @bar(i32 [[TMP6]])
+; CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { i32, i1 } [[TMP7:%.*]], 0
+; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 @bar(i32 [[TMP5]])
 ; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[CALL]], 0
 ; CHECK-NEXT:    br i1 [[TOBOOL]], label [[WHILE_END]], label [[WHILE_BODY]]
 ; CHECK:       while.body:
-; CHECK-NEXT:    [[TMP7:%.*]] = phi i32 [ [[TMP6]], [[WHILE_COND:%.*]] ], [ [[TMP5]], [[WHILE_BODY_PREHEADER]] ]
-; CHECK-NEXT:    [[TMP8]] = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[TMP7]], i32 1)
-; CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { i32, i1 } [[TMP8]], 1
-; CHECK-NEXT:    br i1 [[TMP9]], label [[TRAP]], label [[WHILE_COND]]
+; CHECK-NEXT:    [[TMP6:%.*]] = phi i32 [ [[TMP5]], [[WHILE_COND:%.*]] ], [ [[TMP4]], [[WHILE_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[TMP7]] = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[TMP6]], i32 1)
+; CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { i32, i1 } [[TMP7]], 1
+; CHECK-NEXT:    br i1 [[TMP8]], label [[TRAP]], label [[WHILE_COND]]
 ; CHECK:       while.end:
-; CHECK-NEXT:    [[DOTLCSSA]] = phi i32 [ [[TMP5]], [[CONT]] ], [ [[TMP6]], [[WHILE_COND]] ]
+; CHECK-NEXT:    [[DOTLCSSA]] = phi i32 [ [[TMP4]], [[CONT]] ], [ [[TMP5]], [[WHILE_COND]] ]
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[DOTLCSSA]], [[N:%.*]]
 ; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_COND]], label [[CLEANUP2:%.*]]
 ; CHECK:       cleanup2:
@@ -744,8 +729,8 @@ define { <2 x i32>, <2 x i1> } @uaddo_vec(<2 x i32> %a) {
 }
 
 
-define i8 @uadd_sat_no_overflow(i8 %x) {
-; CHECK-LABEL: @uadd_sat_no_overflow(
+define i8 @uadd_sat_no_unsigned_overflow(i8 %x) {
+; CHECK-LABEL: @uadd_sat_no_unsigned_overflow(
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i8 [[X:%.*]], 100
 ; CHECK-NEXT:    br i1 [[CMP]], label [[TRAP:%.*]], label [[CONT:%.*]]
 ; CHECK:       trap:
@@ -767,8 +752,31 @@ cont:
   ret i8 %res
 }
 
-define i8 @sadd_sat_no_overflow(i8 %x) {
-; CHECK-LABEL: @sadd_sat_no_overflow(
+define i8 @uadd_sat_no_overflow(i8 %x) {
+; CHECK-LABEL: @uadd_sat_no_overflow(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i8 [[X:%.*]], 27
+; CHECK-NEXT:    br i1 [[CMP]], label [[TRAP:%.*]], label [[CONT:%.*]]
+; CHECK:       trap:
+; CHECK-NEXT:    call void @llvm.trap()
+; CHECK-NEXT:    unreachable
+; CHECK:       cont:
+; CHECK-NEXT:    [[RES1:%.*]] = add nuw nsw i8 [[X]], 100
+; CHECK-NEXT:    ret i8 [[RES1]]
+;
+  %cmp = icmp ugt i8 %x, 27
+  br i1 %cmp, label %trap, label %cont
+
+trap:
+  call void @llvm.trap()
+  unreachable
+
+cont:
+  %res = call i8 @llvm.uadd.sat.i8(i8 %x, i8 100)
+  ret i8 %res
+}
+
+define i8 @sadd_sat_no_signed_overflow(i8 %x) {
+; CHECK-LABEL: @sadd_sat_no_signed_overflow(
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i8 [[X:%.*]], 100
 ; CHECK-NEXT:    br i1 [[CMP]], label [[TRAP:%.*]], label [[CONT:%.*]]
 ; CHECK:       trap:
@@ -790,8 +798,31 @@ cont:
   ret i8 %res
 }
 
-define i8 @usub_sat_no_overflow(i8 %x) {
-; CHECK-LABEL: @usub_sat_no_overflow(
+define i8 @sadd_sat_no_overflow(i8 %x) {
+; CHECK-LABEL: @sadd_sat_no_overflow(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i8 [[X:%.*]], 107
+; CHECK-NEXT:    br i1 [[CMP]], label [[TRAP:%.*]], label [[CONT:%.*]]
+; CHECK:       trap:
+; CHECK-NEXT:    call void @llvm.trap()
+; CHECK-NEXT:    unreachable
+; CHECK:       cont:
+; CHECK-NEXT:    [[RES1:%.*]] = add nuw nsw i8 [[X]], 20
+; CHECK-NEXT:    ret i8 [[RES1]]
+;
+  %cmp = icmp ugt i8 %x, 107
+  br i1 %cmp, label %trap, label %cont
+
+trap:
+  call void @llvm.trap()
+  unreachable
+
+cont:
+  %res = call i8 @llvm.sadd.sat.i8(i8 %x, i8 20)
+  ret i8 %res
+}
+
+define i8 @usub_sat_no_unsigned_overflow(i8 %x) {
+; CHECK-LABEL: @usub_sat_no_unsigned_overflow(
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i8 [[X:%.*]], 100
 ; CHECK-NEXT:    br i1 [[CMP]], label [[TRAP:%.*]], label [[CONT:%.*]]
 ; CHECK:       trap:
@@ -813,8 +844,31 @@ cont:
   ret i8 %res
 }
 
-define i8 @ssub_sat_no_overflow(i8 %x) {
-; CHECK-LABEL: @ssub_sat_no_overflow(
+define i8 @usub_sat_no_overflow(i8 %x) {
+; CHECK-LABEL: @usub_sat_no_overflow(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i8 [[X:%.*]], -28
+; CHECK-NEXT:    br i1 [[CMP]], label [[TRAP:%.*]], label [[CONT:%.*]]
+; CHECK:       trap:
+; CHECK-NEXT:    call void @llvm.trap()
+; CHECK-NEXT:    unreachable
+; CHECK:       cont:
+; CHECK-NEXT:    [[RES1:%.*]] = sub nuw nsw i8 [[X]], 100
+; CHECK-NEXT:    ret i8 [[RES1]]
+;
+  %cmp = icmp ult i8 %x, 228
+  br i1 %cmp, label %trap, label %cont
+
+trap:
+  call void @llvm.trap()
+  unreachable
+
+cont:
+  %res = call i8 @llvm.usub.sat.i8(i8 %x, i8 100)
+  ret i8 %res
+}
+
+define i8 @ssub_sat_no_signed_overflow(i8 %x) {
+; CHECK-LABEL: @ssub_sat_no_signed_overflow(
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[X:%.*]], -100
 ; CHECK-NEXT:    br i1 [[CMP]], label [[TRAP:%.*]], label [[CONT:%.*]]
 ; CHECK:       trap:
@@ -836,6 +890,29 @@ cont:
   ret i8 %res
 }
 
+define i8 @ssub_sat_no_overflow(i8 %x) {
+; CHECK-LABEL: @ssub_sat_no_overflow(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i8 [[X:%.*]], -108
+; CHECK-NEXT:    br i1 [[CMP]], label [[TRAP:%.*]], label [[CONT:%.*]]
+; CHECK:       trap:
+; CHECK-NEXT:    call void @llvm.trap()
+; CHECK-NEXT:    unreachable
+; CHECK:       cont:
+; CHECK-NEXT:    [[RES1:%.*]] = sub nuw nsw i8 [[X]], 20
+; CHECK-NEXT:    ret i8 [[RES1]]
+;
+  %cmp = icmp ult i8 %x, 148
+  br i1 %cmp, label %trap, label %cont
+
+trap:
+  call void @llvm.trap()
+  unreachable
+
+cont:
+  %res = call i8 @llvm.ssub.sat.i8(i8 %x, i8 20)
+  ret i8 %res
+}
+
 define <2 x i8> @uadd_sat_vec(<2 x i8> %a) {
 ; CHECK-LABEL: @uadd_sat_vec(
 ; CHECK-NEXT:    [[ADD:%.*]] = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> [[A:%.*]], <2 x i8> <i8 1, i8 1>)
@@ -844,3 +921,267 @@ define <2 x i8> @uadd_sat_vec(<2 x i8> %a) {
   %add = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> %a, <2 x i8> <i8 1, i8 1>)
   ret <2 x i8> %add
 }
+
+; In the following tests, we should first simplify a with.overflow intrinsic
+; to a simple arithmetic operation and insertvalue, but also simplify the
+; subsequent comparison that is based on it.
+
+define i1 @sadd_and_cmp(i32 %x, i32 %y) #0 {
+; CHECK-LABEL: @sadd_and_cmp(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[X_OFFSET:%.*]] = add i32 [[X:%.*]], 9
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult i32 [[X_OFFSET]], 19
+; CHECK-NEXT:    br i1 [[CMP1]], label [[CONT1:%.*]], label [[OUT:%.*]]
+; CHECK:       cont1:
+; CHECK-NEXT:    [[Y_OFFSET:%.*]] = add i32 [[Y:%.*]], 9
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult i32 [[Y_OFFSET]], 19
+; CHECK-NEXT:    br i1 [[CMP2]], label [[CONT2:%.*]], label [[OUT]]
+; CHECK:       cont2:
+; CHECK-NEXT:    [[RES1:%.*]] = add nsw i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[TMP0:%.*]] = insertvalue { i32, i1 } { i32 undef, i1 false }, i32 [[RES1]], 0
+; CHECK-NEXT:    [[ADD:%.*]] = extractvalue { i32, i1 } [[TMP0]], 0
+; CHECK-NEXT:    br label [[CONT3:%.*]]
+; CHECK:       cont3:
+; CHECK-NEXT:    br label [[OUT]]
+; CHECK:       out:
+; CHECK-NEXT:    ret i1 true
+;
+entry:
+  %x.offset = add i32 %x, 9
+  %cmp1 = icmp ult i32 %x.offset, 19
+  br i1 %cmp1, label %cont1, label %out
+
+cont1:
+  %y.offset = add i32 %y, 9
+  %cmp2 = icmp ult i32 %y.offset, 19
+  br i1 %cmp2, label %cont2, label %out
+
+cont2:
+  ; x = [-9,10), y = [-9,10)
+  %res = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 %x, i32 %y)
+  %add = extractvalue { i32, i1 } %res, 0
+  br label %cont3
+
+cont3:
+  ; add = [-18,19)
+  %cmp3 = icmp slt i32 %add, 19
+  br label %out
+
+out:
+  %ret = phi i1 [ true, %entry], [ true, %cont1 ], [ %cmp3, %cont3 ]
+  ret i1 %ret
+}
+
+
+define i1 @uadd_and_cmp(i32 %x, i32 %y) #0 {
+; CHECK-LABEL: @uadd_and_cmp(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult i32 [[X:%.*]], 10
+; CHECK-NEXT:    br i1 [[CMP1]], label [[CONT1:%.*]], label [[OUT:%.*]]
+; CHECK:       cont1:
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult i32 [[Y:%.*]], 10
+; CHECK-NEXT:    br i1 [[CMP2]], label [[CONT2:%.*]], label [[OUT]]
+; CHECK:       cont2:
+; CHECK-NEXT:    [[RES1:%.*]] = add nuw nsw i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[TMP0:%.*]] = insertvalue { i32, i1 } { i32 undef, i1 false }, i32 [[RES1]], 0
+; CHECK-NEXT:    [[ADD:%.*]] = extractvalue { i32, i1 } [[TMP0]], 0
+; CHECK-NEXT:    br label [[CONT3:%.*]]
+; CHECK:       cont3:
+; CHECK-NEXT:    br label [[OUT]]
+; CHECK:       out:
+; CHECK-NEXT:    ret i1 true
+;
+entry:
+  %cmp1 = icmp ult i32 %x, 10
+  br i1 %cmp1, label %cont1, label %out
+
+cont1:
+  %cmp2 = icmp ult i32 %y, 10
+  br i1 %cmp2, label %cont2, label %out
+
+cont2:
+  %res = tail call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 %x, i32 %y)
+  %add = extractvalue { i32, i1 } %res, 0
+  br label %cont3
+
+cont3:
+  %cmp3 = icmp ult i32 %add, 19
+  br label %out
+
+out:
+  %ret = phi i1 [ true, %entry], [ true, %cont1 ], [ %cmp3, %cont3 ]
+  ret i1 %ret
+}
+
+define i1 @ssub_and_cmp(i32 %x, i32 %y) #0 {
+; CHECK-LABEL: @ssub_and_cmp(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult i32 [[X:%.*]], 10
+; CHECK-NEXT:    br i1 [[CMP1]], label [[CONT1:%.*]], label [[OUT:%.*]]
+; CHECK:       cont1:
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult i32 [[Y:%.*]], 10
+; CHECK-NEXT:    br i1 [[CMP2]], label [[CONT2:%.*]], label [[OUT]]
+; CHECK:       cont2:
+; CHECK-NEXT:    [[OFFSET:%.*]] = add nuw nsw i32 [[X]], 9
+; CHECK-NEXT:    [[RES1:%.*]] = sub nuw nsw i32 [[OFFSET]], [[Y]]
+; CHECK-NEXT:    [[TMP0:%.*]] = insertvalue { i32, i1 } { i32 undef, i1 false }, i32 [[RES1]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = extractvalue { i32, i1 } [[TMP0]], 0
+; CHECK-NEXT:    br label [[CONT3:%.*]]
+; CHECK:       cont3:
+; CHECK-NEXT:    br label [[OUT]]
+; CHECK:       out:
+; CHECK-NEXT:    ret i1 true
+;
+entry:
+  %cmp1 = icmp ult i32 %x, 10
+  br i1 %cmp1, label %cont1, label %out
+
+cont1:
+  %cmp2 = icmp ult i32 %y, 10
+  br i1 %cmp2, label %cont2, label %out
+
+cont2:
+  %offset = add i32 %x, 9
+  ; x = [0,10), y = [0,10), offset = [9,19)
+  %res = tail call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 %offset, i32 %y)
+  %sub = extractvalue { i32, i1 } %res, 0
+  br label %cont3
+
+cont3:
+  %cmp3 = icmp ult i32 %sub, 19
+  br label %out
+
+out:
+  %ret = phi i1 [ true, %entry], [ true, %cont1 ], [ %cmp3, %cont3 ]
+  ret i1 %ret
+}
+
+define i1 @usub_and_cmp(i32 %x, i32 %y) #0 {
+; CHECK-LABEL: @usub_and_cmp(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult i32 [[X:%.*]], 10
+; CHECK-NEXT:    br i1 [[CMP1]], label [[CONT1:%.*]], label [[OUT:%.*]]
+; CHECK:       cont1:
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult i32 [[Y:%.*]], 10
+; CHECK-NEXT:    br i1 [[CMP2]], label [[CONT2:%.*]], label [[OUT]]
+; CHECK:       cont2:
+; CHECK-NEXT:    [[OFFSET:%.*]] = add nuw nsw i32 [[X]], 9
+; CHECK-NEXT:    [[RES1:%.*]] = sub nuw nsw i32 [[OFFSET]], [[Y]]
+; CHECK-NEXT:    [[TMP0:%.*]] = insertvalue { i32, i1 } { i32 undef, i1 false }, i32 [[RES1]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = extractvalue { i32, i1 } [[TMP0]], 0
+; CHECK-NEXT:    br label [[CONT3:%.*]]
+; CHECK:       cont3:
+; CHECK-NEXT:    br label [[OUT]]
+; CHECK:       out:
+; CHECK-NEXT:    ret i1 true
+;
+entry:
+  %cmp1 = icmp ult i32 %x, 10
+  br i1 %cmp1, label %cont1, label %out
+
+cont1:
+  %cmp2 = icmp ult i32 %y, 10
+  br i1 %cmp2, label %cont2, label %out
+
+cont2:
+  %offset = add i32 %x, 9
+  ; x = [0,10), y = [0,10), offset = [9,19)
+  %res = tail call { i32, i1 } @llvm.usub.with.overflow.i32(i32 %offset, i32 %y)
+  %sub = extractvalue { i32, i1 } %res, 0
+  br label %cont3
+
+cont3:
+  %cmp3 = icmp ult i32 %sub, 19
+  br label %out
+
+out:
+  %ret = phi i1 [ true, %entry], [ true, %cont1 ], [ %cmp3, %cont3 ]
+  ret i1 %ret
+}
+
+define i1 @smul_and_cmp(i32 %x, i32 %y) #0 {
+; CHECK-LABEL: @smul_and_cmp(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[X_OFFSET:%.*]] = add i32 [[X:%.*]], 9
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult i32 [[X_OFFSET]], 19
+; CHECK-NEXT:    br i1 [[CMP1]], label [[CONT1:%.*]], label [[OUT:%.*]]
+; CHECK:       cont1:
+; CHECK-NEXT:    [[Y_OFFSET:%.*]] = add i32 [[Y:%.*]], 9
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult i32 [[Y_OFFSET]], 19
+; CHECK-NEXT:    br i1 [[CMP2]], label [[CONT2:%.*]], label [[OUT]]
+; CHECK:       cont2:
+; CHECK-NEXT:    [[RES1:%.*]] = mul nsw i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[TMP0:%.*]] = insertvalue { i32, i1 } { i32 undef, i1 false }, i32 [[RES1]], 0
+; CHECK-NEXT:    [[MUL:%.*]] = extractvalue { i32, i1 } [[TMP0]], 0
+; CHECK-NEXT:    br label [[CONT3:%.*]]
+; CHECK:       cont3:
+; CHECK-NEXT:    br label [[OUT]]
+; CHECK:       out:
+; CHECK-NEXT:    ret i1 true
+;
+entry:
+  %x.offset = add i32 %x, 9
+  %cmp1 = icmp ult i32 %x.offset, 19
+  br i1 %cmp1, label %cont1, label %out
+
+cont1:
+  %y.offset = add i32 %y, 9
+  %cmp2 = icmp ult i32 %y.offset, 19
+  br i1 %cmp2, label %cont2, label %out
+
+cont2:
+  ; x = [-9,10), y = [-9,10)
+  %res = tail call { i32, i1 } @llvm.smul.with.overflow.i32(i32 %x, i32 %y)
+  %mul = extractvalue { i32, i1 } %res, 0
+  br label %cont3
+
+cont3:
+  %cmp3 = icmp sle i32 %mul, 81
+  %cmp4 = icmp sge i32 %mul, -81
+  %cmp5 = and i1 %cmp3, %cmp4
+  br label %out
+
+out:
+  %ret = phi i1 [ true, %entry], [ true, %cont1 ], [ %cmp5, %cont3 ]
+  ret i1 %ret
+}
+
+define i1 @umul_and_cmp(i32 %x, i32 %y) #0 {
+; CHECK-LABEL: @umul_and_cmp(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult i32 [[X:%.*]], 100
+; CHECK-NEXT:    br i1 [[CMP1]], label [[CONT1:%.*]], label [[OUT:%.*]]
+; CHECK:       cont1:
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult i32 [[Y:%.*]], 100
+; CHECK-NEXT:    br i1 [[CMP2]], label [[CONT2:%.*]], label [[OUT]]
+; CHECK:       cont2:
+; CHECK-NEXT:    [[RES1:%.*]] = mul nuw nsw i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[TMP0:%.*]] = insertvalue { i32, i1 } { i32 undef, i1 false }, i32 [[RES1]], 0
+; CHECK-NEXT:    [[MUL:%.*]] = extractvalue { i32, i1 } [[TMP0]], 0
+; CHECK-NEXT:    br label [[CONT3:%.*]]
+; CHECK:       cont3:
+; CHECK-NEXT:    br label [[OUT]]
+; CHECK:       out:
+; CHECK-NEXT:    ret i1 true
+;
+entry:
+  %cmp1 = icmp ult i32 %x, 100
+  br i1 %cmp1, label %cont1, label %out
+
+cont1:
+  %cmp2 = icmp ult i32 %y, 100
+  br i1 %cmp2, label %cont2, label %out
+
+cont2:
+  %res = tail call { i32, i1 } @llvm.umul.with.overflow.i32(i32 %x, i32 %y)
+  %mul = extractvalue { i32, i1 } %res, 0
+  br label %cont3
+
+cont3:
+  %cmp3 = icmp ule i32 %mul, 9801
+  br label %out
+
+out:
+  %ret = phi i1 [ true, %entry], [ true, %cont1 ], [ %cmp3, %cont3 ]
+  ret i1 %ret
+}
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/phi-common-val.ll b/llvm/test/Transforms/CorrelatedValuePropagation/phi-common-val.ll
index b761fff00..d6afcc863 100644
--- a/llvm/test/Transforms/CorrelatedValuePropagation/phi-common-val.ll
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/phi-common-val.ll
@@ -123,3 +123,29 @@ return:
   ret i8* %r
 }
 
+; The sub has 'nsw', so it is not safe to propagate that value along
+; the bb2 edge because that would propagate poison to the return.
+
+define i32 @PR43802(i32 %arg) {
+; CHECK-LABEL: @PR43802(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 0, [[ARG:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[ARG]], -2147483648
+; CHECK-NEXT:    br i1 [[CMP]], label [[BB2:%.*]], label [[BB3:%.*]]
+; CHECK:       bb2:
+; CHECK-NEXT:    br label [[BB3]]
+; CHECK:       bb3:
+; CHECK-NEXT:    ret i32 [[SUB]]
+;
+entry:
+  %sub = sub nsw i32 0, %arg
+  %cmp = icmp eq i32 %arg, -2147483648
+  br i1 %cmp, label %bb2, label %bb3
+
+bb2:
+  br label %bb3
+
+bb3:
+  %r = phi i32 [ -2147483648, %bb2 ], [ %sub, %entry ]
+  ret i32 %r
+}
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/range.ll b/llvm/test/Transforms/CorrelatedValuePropagation/range.ll
index 49d4e1f48..6315e3bd7 100644
--- a/llvm/test/Transforms/CorrelatedValuePropagation/range.ll
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/range.ll
@@ -745,10 +745,9 @@ target93:
 define i1 @test17_i1(i1 %a) {
 ; CHECK-LABEL: @test17_i1(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[C:%.*]] = and i1 [[A:%.*]], true
 ; CHECK-NEXT:    br label [[DISPATCH:%.*]]
 ; CHECK:       dispatch:
-; CHECK-NEXT:    br i1 [[A]], label [[TRUE:%.*]], label [[DISPATCH]]
+; CHECK-NEXT:    br i1 [[A:%.*]], label [[TRUE:%.*]], label [[DISPATCH]]
 ; CHECK:       true:
 ; CHECK-NEXT:    ret i1 true
 ;
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/select.ll b/llvm/test/Transforms/CorrelatedValuePropagation/select.ll
index d237521f3..c22a3bfab 100644
--- a/llvm/test/Transforms/CorrelatedValuePropagation/select.ll
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/select.ll
@@ -1,30 +1,47 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -correlated-propagation -S | FileCheck %s
 
-; CHECK-LABEL: @simple(
 define i8 @simple(i1) {
+; CHECK-LABEL: @simple(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[TMP0:%.*]], i8 0, i8 1
+; CHECK-NEXT:    br i1 [[TMP0]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    ret i8 0
+; CHECK:       else:
+; CHECK-NEXT:    ret i8 1
+;
 entry:
   %s = select i1 %0, i8 0, i8 1
   br i1 %0, label %then, label %else
 
 then:
-; CHECK: ret i8 0
   %a = phi i8 [ %s, %entry ]
   ret i8 %a
 
 else:
-; CHECK: ret i8 1
   %b = phi i8 [ %s, %entry ]
   ret i8 %b
 }
 
-; CHECK-LABEL: @loop(
 define void @loop(i32) {
+; CHECK-LABEL: @loop(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IDX:%.*]] = phi i32 [ [[TMP0:%.*]], [[ENTRY:%.*]] ], [ [[TMP2:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[IDX]], 0
+; CHECK-NEXT:    [[TMP2]] = add i32 [[IDX]], -1
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[TMP1]], i32 0, i32 [[TMP2]]
+; CHECK-NEXT:    br i1 [[TMP1]], label [[OUT:%.*]], label [[LOOP]]
+; CHECK:       out:
+; CHECK-NEXT:    ret void
+;
 entry:
   br label %loop
 
 loop:
   %idx = phi i32 [ %0, %entry ], [ %sel, %loop ]
-; CHECK: %idx = phi i32 [ %0, %entry ], [ %2, %loop ]
   %1 = icmp eq i32 %idx, 0
   %2 = add i32 %idx, -1
   %sel = select i1 %1, i32 0, i32 %2
@@ -34,19 +51,25 @@ out:
   ret void
 }
 
-; CHECK-LABEL: @not_correlated(
 define i8 @not_correlated(i1, i1) {
+; CHECK-LABEL: @not_correlated(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[TMP0:%.*]], i8 0, i8 1
+; CHECK-NEXT:    br i1 [[TMP1:%.*]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    ret i8 [[S]]
+; CHECK:       else:
+; CHECK-NEXT:    ret i8 [[S]]
+;
 entry:
   %s = select i1 %0, i8 0, i8 1
   br i1 %1, label %then, label %else
 
 then:
-; CHECK: ret i8 %s
   %a = phi i8 [ %s, %entry ]
   ret i8 %a
 
 else:
-; CHECK: ret i8 %s
   %b = phi i8 [ %s, %entry ]
   ret i8 %b
 }
@@ -54,8 +77,17 @@ else:
 @c = global i32 0, align 4
 @b = global i32 0, align 4
 
-; CHECK-LABEL: @PR23752(
 define i32 @PR23752() {
+; CHECK-LABEL: @PR23752(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 icmp sgt (i32* @b, i32* @c), i32 0, i32 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[SEL]], 1
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[IF_END:%.*]]
+; CHECK:       if.end:
+; CHECK-NEXT:    ret i32 1
+;
 entry:
   br label %for.body
 
@@ -65,17 +97,24 @@ for.body:
   %cmp = icmp ne i32 %sel, 1
   br i1 %cmp, label %for.body, label %if.end
 
-; CHECK:      %[[sel:.*]] = select i1 icmp sgt (i32* @b, i32* @c), i32 0, i32 1
-; CHECK-NEXT: %[[cmp:.*]] = icmp ne i32 %[[sel]], 1
-; CHECK-NEXT: br i1 %[[cmp]]
 
 if.end:
   ret i32 %sel
-; CHECK: ret i32 1
 }
 
 define i1 @test1(i32* %p, i1 %unknown) {
-; CHECK-LABEL: @test1
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[PVAL:%.*]] = load i32, i32* [[P:%.*]]
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[PVAL]], 255
+; CHECK-NEXT:    br i1 [[CMP1]], label [[NEXT:%.*]], label [[EXIT:%.*]]
+; CHECK:       next:
+; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[UNKNOWN:%.*]], i32 [[PVAL]], i32 5
+; CHECK-NEXT:    br label [[NEXT2:%.*]]
+; CHECK:       next2:
+; CHECK-NEXT:    ret i1 false
+; CHECK:       exit:
+; CHECK-NEXT:    ret i1 true
+;
   %pval = load i32, i32* %p
   %cmp1 = icmp slt i32 %pval, 255
   br i1 %cmp1, label %next, label %exit
@@ -85,20 +124,28 @@ next:
   ;; TODO: This pointless branch shouldn't be neccessary
   br label %next2
 next2:
-; CHECK-LABEL: next2:
-; CHECK: ret i1 false
   %res = icmp eq i32 %min, 255
   ret i1 %res
 
 exit:
-; CHECK-LABEL: exit:
-; CHECK: ret i1 true
   ret i1 true
 }
 
 ; Check that we take a conservative meet
 define i1 @test2(i32* %p, i32 %qval, i1 %unknown) {
-; CHECK-LABEL: test2
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[PVAL:%.*]] = load i32, i32* [[P:%.*]]
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[PVAL]], 255
+; CHECK-NEXT:    br i1 [[CMP1]], label [[NEXT:%.*]], label [[EXIT:%.*]]
+; CHECK:       next:
+; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[UNKNOWN:%.*]], i32 [[PVAL]], i32 [[QVAL:%.*]]
+; CHECK-NEXT:    br label [[NEXT2:%.*]]
+; CHECK:       next2:
+; CHECK-NEXT:    [[RES:%.*]] = icmp eq i32 [[MIN]], 255
+; CHECK-NEXT:    ret i1 [[RES]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i1 true
+;
   %pval = load i32, i32* %p
   %cmp1 = icmp slt i32 %pval, 255
   br i1 %cmp1, label %next, label %exit
@@ -108,20 +155,28 @@ next:
   ;; TODO: This pointless branch shouldn't be neccessary
   br label %next2
 next2:
-; CHECK-LABEL: next2
-; CHECK: ret i1 %res
   %res = icmp eq i32 %min, 255
   ret i1 %res
 
 exit:
-; CHECK-LABEL: exit:
-; CHECK: ret i1 true
   ret i1 true
 }
 
 ; Same as @test2, but for the opposite select input
 define i1 @test3(i32* %p, i32 %qval, i1 %unknown) {
-; CHECK-LABEL: test3
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    [[PVAL:%.*]] = load i32, i32* [[P:%.*]]
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[PVAL]], 255
+; CHECK-NEXT:    br i1 [[CMP1]], label [[NEXT:%.*]], label [[EXIT:%.*]]
+; CHECK:       next:
+; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[UNKNOWN:%.*]], i32 [[QVAL:%.*]], i32 [[PVAL]]
+; CHECK-NEXT:    br label [[NEXT2:%.*]]
+; CHECK:       next2:
+; CHECK-NEXT:    [[RES:%.*]] = icmp eq i32 [[MIN]], 255
+; CHECK-NEXT:    ret i1 [[RES]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i1 true
+;
   %pval = load i32, i32* %p
   %cmp1 = icmp slt i32 %pval, 255
   br i1 %cmp1, label %next, label %exit
@@ -131,14 +186,10 @@ next:
   ;; TODO: This pointless branch shouldn't be neccessary
   br label %next2
 next2:
-; CHECK-LABEL: next2
-; CHECK: ret i1 %res
   %res = icmp eq i32 %min, 255
   ret i1 %res
 
 exit:
-; CHECK-LABEL: exit:
-; CHECK: ret i1 true
   ret i1 true
 }
 
@@ -147,7 +198,19 @@ exit:
 ; is to get around the fact that all integers (including constants
 ; and non-constants) are actually represented as constant-ranges.
 define i1 @test4(i32* %p, i32 %qval, i1 %unknown) {
-; CHECK-LABEL: test4
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    [[PVAL:%.*]] = load i32, i32* [[P:%.*]]
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[PVAL]], 255
+; CHECK-NEXT:    br i1 [[CMP1]], label [[NEXT:%.*]], label [[EXIT:%.*]]
+; CHECK:       next:
+; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[UNKNOWN:%.*]], double 1.000000e+00, double 0.000000e+00
+; CHECK-NEXT:    br label [[NEXT2:%.*]]
+; CHECK:       next2:
+; CHECK-NEXT:    [[RES:%.*]] = fcmp oeq double [[MIN]], 3.000000e+02
+; CHECK-NEXT:    ret i1 [[RES]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i1 true
+;
   %pval = load i32, i32* %p
   %cmp1 = icmp slt i32 %pval, 255
   br i1 %cmp1, label %next, label %exit
@@ -157,22 +220,30 @@ next:
   ;; TODO: This pointless branch shouldn't be neccessary
   br label %next2
 next2:
-; CHECK-LABEL: next2
-; CHECK: ret i1 %res
   %res = fcmp oeq double %min, 300.0
   ret i1 %res
 
 exit:
-; CHECK-LABEL: exit:
-; CHECK: ret i1 true
   ret i1 true
 }
 
 ;; Using the condition to clamp the result
-;; 
+;;
 
 define i1 @test5(i32* %p, i1 %unknown) {
-; CHECK-LABEL: @test5
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    [[PVAL:%.*]] = load i32, i32* [[P:%.*]]
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[PVAL]], 255
+; CHECK-NEXT:    br i1 [[CMP1]], label [[NEXT:%.*]], label [[EXIT:%.*]]
+; CHECK:       next:
+; CHECK-NEXT:    [[COND:%.*]] = icmp sgt i32 [[PVAL]], 0
+; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[COND]], i32 [[PVAL]], i32 5
+; CHECK-NEXT:    br label [[NEXT2:%.*]]
+; CHECK:       next2:
+; CHECK-NEXT:    ret i1 false
+; CHECK:       exit:
+; CHECK-NEXT:    ret i1 true
+;
   %pval = load i32, i32* %p
   %cmp1 = icmp slt i32 %pval, 255
   br i1 %cmp1, label %next, label %exit
@@ -183,19 +254,27 @@ next:
   ;; TODO: This pointless branch shouldn't be neccessary
   br label %next2
 next2:
-; CHECK-LABEL: next2:
-; CHECK: ret i1 false
   %res = icmp eq i32 %min, -1
   ret i1 %res
 
 exit:
-; CHECK-LABEL: exit:
-; CHECK: ret i1 true
   ret i1 true
 }
 
 define i1 @test6(i32* %p, i1 %unknown) {
-; CHECK-LABEL: @test6
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:    [[PVAL:%.*]] = load i32, i32* [[P:%.*]]
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult i32 [[PVAL]], 255
+; CHECK-NEXT:    br i1 [[CMP1]], label [[NEXT:%.*]], label [[EXIT:%.*]]
+; CHECK:       next:
+; CHECK-NEXT:    [[COND:%.*]] = icmp ne i32 [[PVAL]], 254
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[COND]], i32 [[PVAL]], i32 1
+; CHECK-NEXT:    br label [[NEXT2:%.*]]
+; CHECK:       next2:
+; CHECK-NEXT:    ret i1 true
+; CHECK:       exit:
+; CHECK-NEXT:    ret i1 true
+;
   %pval = load i32, i32* %p
   %cmp1 = icmp ult i32 %pval, 255
   br i1 %cmp1, label %next, label %exit
@@ -206,13 +285,9 @@ next:
   ;; TODO: This pointless branch shouldn't be neccessary
   br label %next2
 next2:
-; CHECK-LABEL: next2:
-; CHECK: ret i1 true
   %res = icmp slt i32 %sel, 254
   ret i1 %res
 
 exit:
-; CHECK-LABEL: exit:
-; CHECK: ret i1 true
   ret i1 true
 }
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/sext.ll b/llvm/test/Transforms/CorrelatedValuePropagation/sext.ll
new file mode 100644
index 000000000..dff1c1206
--- /dev/null
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/sext.ll
@@ -0,0 +1,135 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -correlated-propagation -S | FileCheck %s
+
+; Check that debug locations are preserved. For more info see:
+;   https://llvm.org/docs/SourceLevelDebugging.html#fixing-errors
+; RUN: opt < %s -enable-debugify -correlated-propagation -S 2>&1 | \
+; RUN:   FileCheck %s -check-prefix=DEBUG
+; DEBUG: CheckModuleDebugify: PASS
+
+declare void @use64(i64)
+
+define void @test1(i32 %n) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_COND:%.*]]
+; CHECK:       for.cond:
+; CHECK-NEXT:    [[A:%.*]] = phi i32 [ [[N:%.*]], [[ENTRY:%.*]] ], [ [[EXT:%.*]], [[FOR_BODY:%.*]] ]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[A]], -1
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[EXT_WIDE1:%.*]] = zext i32 [[A]] to i64
+; CHECK-NEXT:    call void @use64(i64 [[EXT_WIDE1]])
+; CHECK-NEXT:    [[EXT]] = trunc i64 [[EXT_WIDE1]] to i32
+; CHECK-NEXT:    br label [[FOR_COND]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.body, %entry
+  %a = phi i32 [ %n, %entry ], [ %ext, %for.body ]
+  %cmp = icmp sgt i32 %a, -1
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %ext.wide = sext i32 %a to i64
+  call void @use64(i64 %ext.wide)
+  %ext = trunc i64 %ext.wide to i32
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  ret void
+}
+
+;; Negative test to show transform doesn't happen unless n >= 0.
+define void @test2(i32 %n) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_COND:%.*]]
+; CHECK:       for.cond:
+; CHECK-NEXT:    [[A:%.*]] = phi i32 [ [[N:%.*]], [[ENTRY:%.*]] ], [ [[EXT:%.*]], [[FOR_BODY:%.*]] ]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[A]], -2
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[EXT_WIDE:%.*]] = sext i32 [[A]] to i64
+; CHECK-NEXT:    call void @use64(i64 [[EXT_WIDE]])
+; CHECK-NEXT:    [[EXT]] = trunc i64 [[EXT_WIDE]] to i32
+; CHECK-NEXT:    br label [[FOR_COND]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.body, %entry
+  %a = phi i32 [ %n, %entry ], [ %ext, %for.body ]
+  %cmp = icmp sgt i32 %a, -2
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %ext.wide = sext i32 %a to i64
+  call void @use64(i64 %ext.wide)
+  %ext = trunc i64 %ext.wide to i32
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  ret void
+}
+
+;; Non looping test case.
+define void @test3(i32 %n) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[N:%.*]], -1
+; CHECK-NEXT:    br i1 [[CMP]], label [[BB:%.*]], label [[EXIT:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[EXT_WIDE1:%.*]] = zext i32 [[N]] to i64
+; CHECK-NEXT:    call void @use64(i64 [[EXT_WIDE1]])
+; CHECK-NEXT:    [[EXT:%.*]] = trunc i64 [[EXT_WIDE1]] to i32
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cmp = icmp sgt i32 %n, -1
+  br i1 %cmp, label %bb, label %exit
+
+bb:
+  %ext.wide = sext i32 %n to i64
+  call void @use64(i64 %ext.wide)
+  %ext = trunc i64 %ext.wide to i32
+  br label %exit
+
+exit:
+  ret void
+}
+
+;; Non looping negative test case.
+define void @test4(i32 %n) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[N:%.*]], -2
+; CHECK-NEXT:    br i1 [[CMP]], label [[BB:%.*]], label [[EXIT:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[EXT_WIDE:%.*]] = sext i32 [[N]] to i64
+; CHECK-NEXT:    call void @use64(i64 [[EXT_WIDE]])
+; CHECK-NEXT:    [[EXT:%.*]] = trunc i64 [[EXT_WIDE]] to i32
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cmp = icmp sgt i32 %n, -2
+  br i1 %cmp, label %bb, label %exit
+
+bb:
+  %ext.wide = sext i32 %n to i64
+  call void @use64(i64 %ext.wide)
+  %ext = trunc i64 %ext.wide to i32
+  br label %exit
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/shl.ll b/llvm/test/Transforms/CorrelatedValuePropagation/shl.ll
new file mode 100644
index 000000000..0514ec95b
--- /dev/null
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/shl.ll
@@ -0,0 +1,378 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -correlated-propagation -cvp-dont-add-nowrap-flags=false -S | FileCheck %s
+
+define i8 @test0(i8 %a, i8 %b) {
+; CHECK-LABEL: @test0(
+; CHECK-NEXT:    [[SHL:%.*]] = shl i8 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i8 [[SHL]]
+;
+  %shl = shl i8 %a, %b
+  ret i8 %shl
+}
+
+define i8 @test1(i8 %a, i8 %b) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i8 [[B:%.*]], 8
+; CHECK-NEXT:    br i1 [[CMP]], label [[BB:%.*]], label [[EXIT:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[SHL:%.*]] = shl i8 [[A:%.*]], [[B]]
+; CHECK-NEXT:    ret i8 [[SHL]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i8 0
+;
+entry:
+  %cmp = icmp ult i8 %b, 8
+  br i1 %cmp, label %bb, label %exit
+
+bb:
+  %shl = shl i8 %a, %b
+  ret i8 %shl
+
+exit:
+  ret i8 0
+}
+
+define i8 @test2(i8 %a, i8 %b) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i8 [[B:%.*]], 9
+; CHECK-NEXT:    br i1 [[CMP]], label [[BB:%.*]], label [[EXIT:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[SHL:%.*]] = shl i8 [[A:%.*]], [[B]]
+; CHECK-NEXT:    ret i8 [[SHL]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i8 0
+;
+entry:
+  %cmp = icmp ult i8 %b, 9
+  br i1 %cmp, label %bb, label %exit
+
+bb:
+  %shl = shl i8 %a, %b
+  ret i8 %shl
+
+exit:
+  ret i8 0
+}
+
+define i8 @test3(i8 %a, i8 %b) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i8 [[B:%.*]], 6
+; CHECK-NEXT:    br i1 [[CMP]], label [[BB:%.*]], label [[EXIT:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[SHL:%.*]] = shl i8 [[A:%.*]], [[B]]
+; CHECK-NEXT:    ret i8 [[SHL]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i8 0
+;
+entry:
+  %cmp = icmp ugt i8 %b, 6
+  br i1 %cmp, label %bb, label %exit
+
+bb:
+  %shl = shl i8 %a, %b
+  ret i8 %shl
+
+exit:
+  ret i8 0
+}
+
+define i8 @test4(i8 %a, i8 %b) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i8 [[B:%.*]], 7
+; CHECK-NEXT:    br i1 [[CMP]], label [[BB:%.*]], label [[EXIT:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[SHL:%.*]] = shl nuw nsw i8 [[A:%.*]], [[B]]
+; CHECK-NEXT:    ret i8 [[SHL]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i8 0
+;
+entry:
+  %cmp = icmp ugt i8 %b, 7
+  br i1 %cmp, label %bb, label %exit
+
+bb:
+  %shl = shl i8 %a, %b
+  ret i8 %shl
+
+exit:
+  ret i8 0
+}
+
+define i8 @test5(i8 %b) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    [[SHL:%.*]] = shl nuw nsw i8 0, [[B:%.*]]
+; CHECK-NEXT:    ret i8 [[SHL]]
+;
+  %shl = shl i8 0, %b
+  ret i8 %shl
+}
+
+define i8 @test6(i8 %b) {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:    [[SHL:%.*]] = shl nuw i8 1, [[B:%.*]]
+; CHECK-NEXT:    ret i8 [[SHL]]
+;
+  %shl = shl i8 1, %b
+  ret i8 %shl
+}
+
+define i8 @test7(i8 %b) {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i8 [[B:%.*]], 7
+; CHECK-NEXT:    br i1 [[CMP]], label [[BB:%.*]], label [[EXIT:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[SHL:%.*]] = shl nuw nsw i8 1, [[B]]
+; CHECK-NEXT:    ret i8 [[SHL]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i8 0
+;
+entry:
+  %cmp = icmp ult i8 %b, 7
+  br i1 %cmp, label %bb, label %exit
+
+bb:
+  %shl = shl i8 1, %b
+  ret i8 %shl
+
+exit:
+  ret i8 0
+}
+
+define i8 @test8(i8 %b) {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT:    [[SHL:%.*]] = shl nsw i8 -1, [[B:%.*]]
+; CHECK-NEXT:    ret i8 [[SHL]]
+;
+  %shl = shl i8 -1, %b
+  ret i8 %shl
+}
+
+define i8 @test9(i8 %b) {
+; CHECK-LABEL: @test9(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[B:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[BB:%.*]], label [[EXIT:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[SHL:%.*]] = shl nuw nsw i8 -1, [[B]]
+; CHECK-NEXT:    ret i8 -1
+; CHECK:       exit:
+; CHECK-NEXT:    ret i8 0
+;
+entry:
+  %cmp = icmp eq i8 %b, 0
+  br i1 %cmp, label %bb, label %exit
+
+bb:
+  %shl = shl i8 -1, %b
+  ret i8 %shl
+
+exit:
+  ret i8 0
+}
+
+define i8 @test10(i8 %b) {
+; CHECK-LABEL: @test10(
+; CHECK-NEXT:    [[SHL:%.*]] = shl i8 42, [[B:%.*]]
+; CHECK-NEXT:    ret i8 [[SHL]]
+;
+  %shl = shl i8 42, %b
+  ret i8 %shl
+}
+
+define i8 @test11(i8 %b) {
+; CHECK-LABEL: @test11(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i8 [[B:%.*]], 2
+; CHECK-NEXT:    br i1 [[CMP]], label [[BB:%.*]], label [[EXIT:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[SHL:%.*]] = shl nuw nsw i8 42, [[B]]
+; CHECK-NEXT:    ret i8 [[SHL]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i8 0
+;
+entry:
+  %cmp = icmp ult i8 %b, 2
+  br i1 %cmp, label %bb, label %exit
+
+bb:
+  %shl = shl i8 42, %b
+  ret i8 %shl
+
+exit:
+  ret i8 0
+}
+
+define i8 @test12(i8 %b) {
+; CHECK-LABEL: @test12(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i8 [[B:%.*]], 3
+; CHECK-NEXT:    br i1 [[CMP]], label [[BB:%.*]], label [[EXIT:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[SHL:%.*]] = shl nuw i8 42, [[B]]
+; CHECK-NEXT:    ret i8 [[SHL]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i8 0
+;
+entry:
+  %cmp = icmp ult i8 %b, 3
+  br i1 %cmp, label %bb, label %exit
+
+bb:
+  %shl = shl i8 42, %b
+  ret i8 %shl
+
+exit:
+  ret i8 0
+}
+
+define i8 @test13(i8 %b) {
+; CHECK-LABEL: @test13(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i8 [[B:%.*]], 4
+; CHECK-NEXT:    br i1 [[CMP]], label [[BB:%.*]], label [[EXIT:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[SHL:%.*]] = shl i8 42, [[B]]
+; CHECK-NEXT:    ret i8 [[SHL]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i8 0
+;
+entry:
+  %cmp = icmp ult i8 %b, 4
+  br i1 %cmp, label %bb, label %exit
+
+bb:
+  %shl = shl i8 42, %b
+  ret i8 %shl
+
+exit:
+  ret i8 0
+}
+
+define i8 @test14(i8 %b) {
+; CHECK-LABEL: @test14(
+; CHECK-NEXT:    [[SHL:%.*]] = shl i8 -42, [[B:%.*]]
+; CHECK-NEXT:    ret i8 [[SHL]]
+;
+  %shl = shl i8 -42, %b
+  ret i8 %shl
+}
+
+define i8 @test15(i8 %b) {
+; CHECK-LABEL: @test15(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i8 [[B:%.*]], 2
+; CHECK-NEXT:    br i1 [[CMP]], label [[BB:%.*]], label [[EXIT:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[SHL:%.*]] = shl nsw i8 -42, [[B]]
+; CHECK-NEXT:    ret i8 [[SHL]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i8 0
+;
+entry:
+  %cmp = icmp ult i8 %b, 2
+  br i1 %cmp, label %bb, label %exit
+
+bb:
+  %shl = shl i8 -42, %b
+  ret i8 %shl
+
+exit:
+  ret i8 0
+}
+
+define i8 @test16(i8 %b) {
+; CHECK-LABEL: @test16(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i8 [[B:%.*]], 3
+; CHECK-NEXT:    br i1 [[CMP]], label [[BB:%.*]], label [[EXIT:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[SHL:%.*]] = shl i8 -42, [[B]]
+; CHECK-NEXT:    ret i8 [[SHL]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i8 0
+;
+entry:
+  %cmp = icmp ult i8 %b, 3
+  br i1 %cmp, label %bb, label %exit
+
+bb:
+  %shl = shl i8 -42, %b
+  ret i8 %shl
+
+exit:
+  ret i8 0
+}
+
+define i8 @test17(i8 %b) {
+; CHECK-LABEL: @test17(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[B:%.*]], 2
+; CHECK-NEXT:    br i1 [[CMP]], label [[BB:%.*]], label [[EXIT:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[SHL:%.*]] = shl nuw nsw i8 42, [[B]]
+; CHECK-NEXT:    ret i8 [[SHL]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i8 0
+;
+entry:
+  %cmp = icmp slt i8 %b, 2
+  br i1 %cmp, label %bb, label %exit
+
+bb:
+  %shl = shl i8 42, %b
+  ret i8 %shl
+
+exit:
+  ret i8 0
+}
+
+define i8 @test18(i8 %b) {
+; CHECK-LABEL: @test18(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[B:%.*]], 3
+; CHECK-NEXT:    br i1 [[CMP]], label [[BB:%.*]], label [[EXIT:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[SHL:%.*]] = shl nuw i8 42, [[B]]
+; CHECK-NEXT:    ret i8 [[SHL]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i8 0
+;
+entry:
+  %cmp = icmp slt i8 %b, 3
+  br i1 %cmp, label %bb, label %exit
+
+bb:
+  %shl = shl i8 42, %b
+  ret i8 %shl
+
+exit:
+  ret i8 0
+}
+
+define i8 @test19(i8 %b) {
+; CHECK-LABEL: @test19(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[B:%.*]], 4
+; CHECK-NEXT:    br i1 [[CMP]], label [[BB:%.*]], label [[EXIT:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[SHL:%.*]] = shl i8 42, [[B]]
+; CHECK-NEXT:    ret i8 [[SHL]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i8 0
+;
+entry:
+  %cmp = icmp slt i8 %b, 4
+  br i1 %cmp, label %bb, label %exit
+
+bb:
+  %shl = shl i8 42, %b
+  ret i8 %shl
+
+exit:
+  ret i8 0
+}
diff --git a/llvm/test/Transforms/DCE/calls-errno.ll b/llvm/test/Transforms/DCE/calls-errno.ll
index 20ee0d06d..376b8d1e5 100644
--- a/llvm/test/Transforms/DCE/calls-errno.ll
+++ b/llvm/test/Transforms/DCE/calls-errno.ll
@@ -76,10 +76,6 @@ entry:
 ; CHECK-NEXT: %cos3 = call double @cos(double 0.000000e+00)
   %cos3 = call double @cos(double 0.000000e+00) nobuiltin
 
-; cos(1) strictfp sets FP status flags
-; CHECK-NEXT: %cos4 = call double @cos(double 1.000000e+00)
-  %cos4 = call double @cos(double 1.000000e+00) strictfp
-
 ; pow(0, 1) is 0
   %pow1 = call double @pow(double 0x7FF0000000000000, double 1.000000e+00)
 
@@ -97,3 +93,16 @@ entry:
 ; CHECK-NEXT: ret void
   ret void
 }
+
+define void @Tstrict() strictfp {
+entry:
+; CHECK-LABEL: @Tstrict(
+; CHECK-NEXT: entry:
+
+; cos(1) strictfp sets FP status flags
+; CHECK-NEXT: %cos4 = call double @cos(double 1.000000e+00)
+  %cos4 = call double @cos(double 1.000000e+00) strictfp
+
+; CHECK-NEXT: ret void
+  ret void
+}
diff --git a/llvm/test/Transforms/DCE/dbg-value-removal.ll b/llvm/test/Transforms/DCE/dbg-value-removal.ll
new file mode 100644
index 000000000..3231cf62e
--- /dev/null
+++ b/llvm/test/Transforms/DCE/dbg-value-removal.ll
@@ -0,0 +1,112 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -S -redundant-dbg-inst-elim | FileCheck %s
+
+; All dbg.value with location "!dbg !19" are redundant in the input.
+; FIXME: We do not handle non-overlapping/overlapping fragments perfectly yet.
+
+define dso_local i16 @main(i16 %a1, i16 %a2) local_unnamed_addr #0 !dbg !7 {
+; CHECK-LABEL: @main(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[BB0:%.*]]
+; CHECK:       bb0:
+; CHECK-NEXT:    call void @llvm.dbg.value(metadata i16 13, metadata !13, metadata !DIExpression()), !dbg !16
+; CHECK-NEXT:    call void @llvm.dbg.value(metadata i16 14, metadata !14, metadata !DIExpression()), !dbg !18
+; CHECK-NEXT:    call void @llvm.dbg.value(metadata i16 13, metadata !13, metadata !DIExpression()), !dbg !18
+; CHECK-NEXT:    call void @llvm.dbg.value(metadata i16 12, metadata !12, metadata !DIExpression()), !dbg !18
+; CHECK-NEXT:    br label [[BB1:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    call void @llvm.dbg.value(metadata i16 [[A1:%.*]], metadata !14, metadata !DIExpression()), !dbg !18
+; CHECK-NEXT:    call void @llvm.dbg.value(metadata i16 888, metadata !13, metadata !DIExpression()), !dbg !18
+; CHECK-NEXT:    call void @llvm.dbg.value(metadata i16 [[A2:%.*]], metadata !12, metadata !DIExpression()), !dbg !18
+; CHECK-NEXT:    [[T1:%.*]] = call i16 @bar(i16 0)
+; CHECK-NEXT:    call void @llvm.dbg.value(metadata i16 [[T1]], metadata !13, metadata !DIExpression()), !dbg !18
+; CHECK-NEXT:    call void @llvm.dbg.value(metadata i16 [[A2]], metadata !12, metadata !DIExpression(DW_OP_constu, 2, DW_OP_shr, DW_OP_stack_value)), !dbg !18
+; CHECK-NEXT:    br label [[BB2:%.*]]
+; CHECK:       bb2:
+; CHECK-NEXT:    call void @llvm.dbg.value(metadata i16 [[A1]], metadata !13, metadata !DIExpression(DW_OP_LLVM_fragment, 0, 8)), !dbg !18
+; CHECK-NEXT:    call void @llvm.dbg.value(metadata i16 [[A1]], metadata !13, metadata !DIExpression(DW_OP_LLVM_fragment, 8, 8)), !dbg !18
+; CHECK-NEXT:    [[T2:%.*]] = call i16 @bar(i16 [[T1]])
+; CHECK-NEXT:    call void @llvm.dbg.value(metadata i16 [[T2]], metadata !13, metadata !DIExpression(DW_OP_LLVM_fragment, 0, 8)), !dbg !18
+; CHECK-NEXT:    call void @llvm.dbg.value(metadata i16 [[A1]], metadata !13, metadata !DIExpression(DW_OP_LLVM_fragment, 8, 8)), !dbg !19
+; CHECK-NEXT:    br label [[BB3:%.*]]
+; CHECK:       bb3:
+; CHECK-NEXT:    call void @llvm.dbg.value(metadata i16 [[A1]], metadata !13, metadata !DIExpression(DW_OP_LLVM_fragment, 0, 8)), !dbg !19
+; CHECK-NEXT:    call void @llvm.dbg.value(metadata i16 [[A1]], metadata !13, metadata !DIExpression()), !dbg !18
+; CHECK-NEXT:    br label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i16 [[T2]]
+;
+entry:
+  br label %bb0
+
+bb0:
+  call void @llvm.dbg.value(metadata i16 999, metadata !12, metadata !DIExpression()), !dbg !19
+  call void @llvm.dbg.value(metadata i16 996, metadata !13, metadata !DIExpression()), !dbg !19
+  call void @llvm.dbg.value(metadata i16 13, metadata !13, metadata !DIExpression()), !dbg !17
+  call void @llvm.dbg.value(metadata i16 998, metadata !12, metadata !DIExpression(DW_OP_constu, 2, DW_OP_shr, DW_OP_stack_value)), !dbg !19
+  call void @llvm.dbg.value(metadata i16 14, metadata !14, metadata !DIExpression()), !dbg !16
+  call void @llvm.dbg.value(metadata i16 997, metadata !12, metadata !DIExpression()), !dbg !19
+  call void @llvm.dbg.value(metadata i16 13, metadata !13, metadata !DIExpression()), !dbg !16
+  call void @llvm.dbg.value(metadata i16 12, metadata !12, metadata !DIExpression()), !dbg !16
+  br label %bb1
+
+bb1:
+  call void @llvm.dbg.value(metadata i16 %a1, metadata !14, metadata !DIExpression()), !dbg !16
+  call void @llvm.dbg.value(metadata i16 888, metadata !13, metadata !DIExpression()), !dbg !16
+  call void @llvm.dbg.value(metadata i16 %a2, metadata !12, metadata !DIExpression()), !dbg !16
+  %t1 = call i16 @bar(i16 0)
+  call void @llvm.dbg.value(metadata i16 %a1, metadata !14, metadata !DIExpression()), !dbg !19
+  call void @llvm.dbg.value(metadata i16 %t1, metadata !13, metadata !DIExpression()), !dbg !16
+  call void @llvm.dbg.value(metadata i16 %a2, metadata !12, metadata !DIExpression(DW_OP_constu, 2, DW_OP_shr, DW_OP_stack_value)), !dbg !16
+  br label %bb2
+
+bb2:
+  call void @llvm.dbg.value(metadata i16 %a1, metadata !13, metadata !DIExpression(DW_OP_LLVM_fragment, 0, 8)), !dbg !19
+  call void @llvm.dbg.value(metadata i16 %a1, metadata !13, metadata !DIExpression(DW_OP_LLVM_fragment, 8, 8)), !dbg !19
+  call void @llvm.dbg.value(metadata i16 %a1, metadata !13, metadata !DIExpression(DW_OP_LLVM_fragment, 0, 8)), !dbg !16
+  call void @llvm.dbg.value(metadata i16 %a1, metadata !13, metadata !DIExpression(DW_OP_LLVM_fragment, 8, 8)), !dbg !16
+  %t2 = call i16 @bar(i16 %t1)
+  call void @llvm.dbg.value(metadata i16 %t2, metadata !13, metadata !DIExpression(DW_OP_LLVM_fragment, 0, 8)), !dbg !16
+  call void @llvm.dbg.value(metadata i16 %a1, metadata !13, metadata !DIExpression(DW_OP_LLVM_fragment, 8, 8)), !dbg !19
+  br label %bb3
+
+bb3:
+  call void @llvm.dbg.value(metadata i16 %a1, metadata !13, metadata !DIExpression(DW_OP_LLVM_fragment, 0, 8)), !dbg !19
+  call void @llvm.dbg.value(metadata i16 %a1, metadata !13, metadata !DIExpression()), !dbg !16
+  br label %exit
+
+exit:
+  ret i16 %t2
+}
+
+declare void @llvm.dbg.value(metadata, metadata, metadata) #1
+declare i16 @bar(i16) #2
+
+attributes #0 = { noinline nounwind }
+attributes #1 = { nounwind readnone speculatable willreturn }
+attributes #2 = { noinline nounwind readnone }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 10.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, globals: !2, nameTableKind: None)
+!1 = !DIFile(filename: "foo.c", directory: "")
+!2 = !{}
+!3 = !{i32 7, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 1}
+!6 = !{!"clang version 10.0.0"}
+!7 = distinct !DISubprogram(name: "main", scope: !1, file: !1, line: 8, type: !8, scopeLine: 8, flags: DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !11)
+!8 = !DISubroutineType(types: !9)
+!9 = !{!10}
+!10 = !DIBasicType(name: "int", size: 16, encoding: DW_ATE_signed)
+!11 = !{!12, !13, !14}
+!12 = !DILocalVariable(name: "x", scope: !7, file: !1, line: 9, type: !10)
+!13 = !DILocalVariable(name: "y", scope: !7, file: !1, line: 10, type: !10)
+!14 = !DILocalVariable(name: "u", scope: !15, file: !1, line: 11, type: !10)
+!15 = distinct !DILexicalBlock(scope: !7, file: !1, line: 11, column: 3)
+!16 = !DILocation(line: 0, scope: !7)
+!17 = !DILocation(line: 0, scope: !7, inlinedAt: !18)
+!18 = !DILocation(line: 1, scope: !7)
+!19 = !DILocation(line: 77, scope: !7)
diff --git a/llvm/test/Transforms/DeadArgElim/2010-04-30-DbgInfo.ll b/llvm/test/Transforms/DeadArgElim/2010-04-30-DbgInfo.ll
index 2bd10a577..418688438 100644
--- a/llvm/test/Transforms/DeadArgElim/2010-04-30-DbgInfo.ll
+++ b/llvm/test/Transforms/DeadArgElim/2010-04-30-DbgInfo.ll
@@ -39,7 +39,7 @@ bb2:                                              ; preds = %bb1, %bb
 declare void @llvm.dbg.value(metadata, metadata, metadata) nounwind readnone
 
 ; CHECK: attributes #0 = { nounwind ssp }
-; CHECK: attributes #1 = { nounwind readnone speculatable }
+; CHECK: attributes #1 = { nounwind readnone speculatable willreturn }
 ; CHECK: attributes #2 = { noinline nounwind ssp }
 ; CHECK: attributes [[NUW]] = { nounwind }
 
diff --git a/llvm/test/Transforms/DeadArgElim/dbginfo.ll b/llvm/test/Transforms/DeadArgElim/dbginfo.ll
index 2bfd9139a..0ef4bfa42 100644
--- a/llvm/test/Transforms/DeadArgElim/dbginfo.ll
+++ b/llvm/test/Transforms/DeadArgElim/dbginfo.ll
@@ -40,8 +40,8 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata) #2
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.value(metadata, metadata, metadata) #2
 
-attributes #0 = { uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/Transforms/DeadArgElim/naked_functions.ll b/llvm/test/Transforms/DeadArgElim/naked_functions.ll
index b7955a101..ed125d45e 100644
--- a/llvm/test/Transforms/DeadArgElim/naked_functions.ll
+++ b/llvm/test/Transforms/DeadArgElim/naked_functions.ll
@@ -1,6 +1,6 @@
 ; RUN: opt -S -deadargelim %s | FileCheck %s
 
-; Don't eliminate dead arugments from naked functions.
+; Don't eliminate dead arguments from naked functions.
 ; CHECK: define internal i32 @naked(i32 %x)
 
 define internal i32 @naked(i32 %x) #0 {
diff --git a/llvm/test/Transforms/DeadStoreElimination/DeleteThrowableInst.ll b/llvm/test/Transforms/DeadStoreElimination/DeleteThrowableInst.ll
new file mode 100644
index 000000000..f392b2e94
--- /dev/null
+++ b/llvm/test/Transforms/DeadStoreElimination/DeleteThrowableInst.ll
@@ -0,0 +1,41 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -dse -S | FileCheck %s
+
+declare i8* @_Znwj(i32) local_unnamed_addr
+declare void @foo() readnone
+
+define void @test1(i8** %ptr) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[VAL:%.*]] = inttoptr i64 23452 to i8*
+; CHECK-NEXT:    store i8* [[VAL]], i8** [[PTR:%.*]]
+; CHECK-NEXT:    ret void
+;
+  %val = inttoptr i64 23452 to i8*
+  store i8* %val, i8** %ptr
+  %call = call i8* @_Znwj(i32 1)
+  store i8* %call, i8** %ptr
+  store i8* %val, i8** %ptr
+  ret void
+}
+
+define void @test2(i8** %ptr, i8* %p1, i8* %p2) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[VAL:%.*]] = inttoptr i64 23452 to i8*
+; CHECK-NEXT:    store i8* [[VAL]], i8** [[PTR:%.*]]
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    store i8* [[P1:%.*]], i8** [[PTR]]
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    store i8* [[VAL]], i8** [[PTR]]
+; CHECK-NEXT:    ret void
+;
+  %val = inttoptr i64 23452 to i8*
+  store i8* %val, i8** %ptr
+  call void @foo()
+  store i8* %p1, i8** %ptr
+  call void @foo()
+  store i8* %p2, i8** %ptr
+  %call = call i8* @_Znwj(i32 1)
+  store i8* %call, i8** %ptr
+  store i8* %val, i8** %ptr
+  ret void
+}
diff --git a/llvm/test/Transforms/DeadStoreElimination/libcalls.ll b/llvm/test/Transforms/DeadStoreElimination/libcalls.ll
index 8afa148a5..eac772460 100644
--- a/llvm/test/Transforms/DeadStoreElimination/libcalls.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/libcalls.ll
@@ -1,5 +1,7 @@
 ; RUN: opt -S -basicaa -dse < %s | FileCheck %s
 
+target triple = "x86_64-unknown-linux-gnu"
+
 declare i8* @strcpy(i8* %dest, i8* %src) nounwind
 define void @test1(i8* %src) {
 ; CHECK-LABEL: @test1(
@@ -11,13 +13,13 @@ define void @test1(i8* %src) {
   ret void
 }
 
-declare i8* @strncpy(i8* %dest, i8* %src, i32 %n) nounwind
+declare i8* @strncpy(i8* %dest, i8* %src, i64 %n) nounwind
 define void @test2(i8* %src) {
 ; CHECK-LABEL: @test2(
   %B = alloca [16 x i8]
   %dest = getelementptr inbounds [16 x i8], [16 x i8]* %B, i64 0, i64 0
 ; CHECK-NOT: @strncpy
-  %call = call i8* @strncpy(i8* %dest, i8* %src, i32 12)
+  %call = call i8* @strncpy(i8* %dest, i8* %src, i64 12)
 ; CHECK: ret void
   ret void
 }
@@ -33,13 +35,13 @@ define void @test3(i8* %src) {
   ret void
 }
 
-declare i8* @strncat(i8* %dest, i8* %src, i32 %n) nounwind
+declare i8* @strncat(i8* %dest, i8* %src, i64 %n) nounwind
 define void @test4(i8* %src) {
 ; CHECK-LABEL: @test4(
   %B = alloca [16 x i8]
   %dest = getelementptr inbounds [16 x i8], [16 x i8]* %B, i64 0, i64 0
 ; CHECK-NOT: @strncat
-  %call = call i8* @strncat(i8* %dest, i8* %src, i32 12)
+  %call = call i8* @strncat(i8* %dest, i8* %src, i64 12)
 ; CHECK: ret void
   ret void
 }
diff --git a/llvm/test/Transforms/DeadStoreElimination/libcalls2.ll b/llvm/test/Transforms/DeadStoreElimination/libcalls2.ll
new file mode 100644
index 000000000..0e8d9eade
--- /dev/null
+++ b/llvm/test/Transforms/DeadStoreElimination/libcalls2.ll
@@ -0,0 +1,14 @@
+; RUN: opt -S -basicaa -dse < %s | FileCheck %s
+
+target triple = "x86_64-unknown-linux-gnu"
+
+declare i8* @strncpy(i8* %dest, i8* %src, i32 %n) nounwind
+define void @test2(i8* %src) {
+; CHECK-LABEL: @test2(
+  %B = alloca [16 x i8]
+  %dest = getelementptr inbounds [16 x i8], [16 x i8]* %B, i64 0, i64 0
+; CHECK: @strncpy
+  %call = call i8* @strncpy(i8* %dest, i8* %src, i32 12)
+; CHECK: ret void
+  ret void
+}
diff --git a/llvm/test/Transforms/DeadStoreElimination/mda-with-dbg-values.ll b/llvm/test/Transforms/DeadStoreElimination/mda-with-dbg-values.ll
index 9cffd7b08..4c6776b37 100644
--- a/llvm/test/Transforms/DeadStoreElimination/mda-with-dbg-values.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/mda-with-dbg-values.ll
@@ -34,7 +34,7 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata) #1
 ; Function Attrs: argmemonly nounwind
 declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture writeonly, i8* nocapture readonly, i64, i1) #2
 
-attributes #0 = { noinline nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noinline nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone speculatable }
 attributes #2 = { argmemonly nounwind }
 
diff --git a/llvm/test/Transforms/DeadStoreElimination/simple.ll b/llvm/test/Transforms/DeadStoreElimination/simple.ll
index 412b563ff..755f17357 100644
--- a/llvm/test/Transforms/DeadStoreElimination/simple.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/simple.ll
@@ -508,7 +508,7 @@ define noalias i8* @test23() nounwind uwtable ssp {
 ; CHECK-NEXT:    store i8 97, i8* [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds [2 x i8], [2 x i8]* [[X]], i64 0, i64 1
 ; CHECK-NEXT:    store i8 0, i8* [[ARRAYIDX1]], align 1
-; CHECK-NEXT:    [[CALL:%.*]] = call i8* @strdup(i8* [[ARRAYIDX]]) #1
+; CHECK-NEXT:    [[CALL:%.*]] = call i8* @strdup(i8* [[ARRAYIDX]]) #2
 ; CHECK-NEXT:    ret i8* [[CALL]]
 ;
   %x = alloca [2 x i8], align 1
@@ -546,7 +546,7 @@ define i8* @test25(i8* %p) nounwind {
 ; CHECK-NEXT:    [[P_4:%.*]] = getelementptr i8, i8* [[P:%.*]], i64 4
 ; CHECK-NEXT:    [[TMP:%.*]] = load i8, i8* [[P_4]], align 1
 ; CHECK-NEXT:    store i8 0, i8* [[P_4]], align 1
-; CHECK-NEXT:    [[Q:%.*]] = call i8* @strdup(i8* [[P]]) #4
+; CHECK-NEXT:    [[Q:%.*]] = call i8* @strdup(i8* [[P]]) #5
 ; CHECK-NEXT:    store i8 [[TMP]], i8* [[P_4]], align 1
 ; CHECK-NEXT:    ret i8* [[Q]]
 ;
diff --git a/llvm/test/Transforms/DivRemPairs/X86/div-expanded-rem-pair.ll b/llvm/test/Transforms/DivRemPairs/X86/div-expanded-rem-pair.ll
index b0cb92a39..7c43d87ff 100644
--- a/llvm/test/Transforms/DivRemPairs/X86/div-expanded-rem-pair.ll
+++ b/llvm/test/Transforms/DivRemPairs/X86/div-expanded-rem-pair.ll
@@ -7,8 +7,8 @@ define void @decompose_illegal_srem_same_block(i32 %a, i32 %b) {
 ; CHECK-LABEL: @decompose_illegal_srem_same_block(
 ; CHECK-NEXT:    [[DIV:%.*]] = sdiv i32 [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    [[T0:%.*]] = mul i32 [[DIV]], [[B]]
-; CHECK-NEXT:    [[REM:%.*]] = sub i32 [[A]], [[T0]]
-; CHECK-NEXT:    call void @foo(i32 [[REM]], i32 [[DIV]])
+; CHECK-NEXT:    [[REM_RECOMPOSED:%.*]] = srem i32 [[A]], [[B]]
+; CHECK-NEXT:    call void @foo(i32 [[REM_RECOMPOSED]], i32 [[DIV]])
 ; CHECK-NEXT:    ret void
 ;
   %div = sdiv i32 %a, %b
@@ -22,8 +22,8 @@ define void @decompose_illegal_urem_same_block(i32 %a, i32 %b) {
 ; CHECK-LABEL: @decompose_illegal_urem_same_block(
 ; CHECK-NEXT:    [[DIV:%.*]] = udiv i32 [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    [[T0:%.*]] = mul i32 [[DIV]], [[B]]
-; CHECK-NEXT:    [[REM:%.*]] = sub i32 [[A]], [[T0]]
-; CHECK-NEXT:    call void @foo(i32 [[REM]], i32 [[DIV]])
+; CHECK-NEXT:    [[REM_RECOMPOSED:%.*]] = urem i32 [[A]], [[B]]
+; CHECK-NEXT:    call void @foo(i32 [[REM_RECOMPOSED]], i32 [[DIV]])
 ; CHECK-NEXT:    ret void
 ;
   %div = udiv i32 %a, %b
@@ -39,14 +39,14 @@ define i16 @hoist_srem(i16 %a, i16 %b) {
 ; CHECK-LABEL: @hoist_srem(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[DIV:%.*]] = sdiv i16 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[REM_RECOMPOSED:%.*]] = srem i16 [[A]], [[B]]
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i16 [[DIV]], 42
 ; CHECK-NEXT:    br i1 [[CMP]], label [[IF:%.*]], label [[END:%.*]]
 ; CHECK:       if:
 ; CHECK-NEXT:    [[T0:%.*]] = mul i16 [[DIV]], [[B]]
-; CHECK-NEXT:    [[REM:%.*]] = sub i16 [[A]], [[T0]]
 ; CHECK-NEXT:    br label [[END]]
 ; CHECK:       end:
-; CHECK-NEXT:    [[RET:%.*]] = phi i16 [ [[REM]], [[IF]] ], [ 3, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[RET:%.*]] = phi i16 [ [[REM_RECOMPOSED]], [[IF]] ], [ 3, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    ret i16 [[RET]]
 ;
 entry:
@@ -70,14 +70,14 @@ define i8 @hoist_urem(i8 %a, i8 %b) {
 ; CHECK-LABEL: @hoist_urem(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[DIV:%.*]] = udiv i8 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[REM_RECOMPOSED:%.*]] = urem i8 [[A]], [[B]]
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[DIV]], 42
 ; CHECK-NEXT:    br i1 [[CMP]], label [[IF:%.*]], label [[END:%.*]]
 ; CHECK:       if:
 ; CHECK-NEXT:    [[T0:%.*]] = mul i8 [[DIV]], [[B]]
-; CHECK-NEXT:    [[REM:%.*]] = sub i8 [[A]], [[T0]]
 ; CHECK-NEXT:    br label [[END]]
 ; CHECK:       end:
-; CHECK-NEXT:    [[RET:%.*]] = phi i8 [ [[REM]], [[IF]] ], [ 3, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[RET:%.*]] = phi i8 [ [[REM_RECOMPOSED]], [[IF]] ], [ 3, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    ret i8 [[RET]]
 ;
 entry:
@@ -122,11 +122,11 @@ define i32 @srem_of_srem_expanded(i32 %X, i32 %Y, i32 %Z) {
 ; CHECK-NEXT:    [[T0:%.*]] = mul nsw i32 [[Z:%.*]], [[Y:%.*]]
 ; CHECK-NEXT:    [[T1:%.*]] = sdiv i32 [[X:%.*]], [[T0]]
 ; CHECK-NEXT:    [[T2:%.*]] = mul nsw i32 [[T0]], [[T1]]
-; CHECK-NEXT:    [[T3:%.*]] = sub nsw i32 [[X]], [[T2]]
-; CHECK-NEXT:    [[T4:%.*]] = sdiv i32 [[T3]], [[Y]]
+; CHECK-NEXT:    [[T3_RECOMPOSED:%.*]] = srem i32 [[X]], [[T0]]
+; CHECK-NEXT:    [[T4:%.*]] = sdiv i32 [[T3_RECOMPOSED]], [[Y]]
 ; CHECK-NEXT:    [[T5:%.*]] = mul nsw i32 [[T4]], [[Y]]
-; CHECK-NEXT:    [[T6:%.*]] = sub nsw i32 [[T3]], [[T5]]
-; CHECK-NEXT:    ret i32 [[T6]]
+; CHECK-NEXT:    [[T6_RECOMPOSED:%.*]] = srem i32 [[T3_RECOMPOSED]], [[Y]]
+; CHECK-NEXT:    ret i32 [[T6_RECOMPOSED]]
 ;
   %t0 = mul nsw i32 %Z, %Y
   %t1 = sdiv i32 %X, %t0
@@ -168,3 +168,39 @@ end:
   %ret = phi i128 [ %rem, %if ], [ 3, %entry ]
   ret i128 %ret
 }
+
+; Even in expanded form, we can end up with div and rem in different basic
+; blocks neither of which dominates each another.
+define i32 @can_have_divrem_in_mutually_nondominating_bbs(i1 %cmp, i32 %a, i32 %b) {
+; CHECK-LABEL: @can_have_divrem_in_mutually_nondominating_bbs(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[CMP:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[T0:%.*]] = udiv i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = mul nuw i32 [[T0]], [[B]]
+; CHECK-NEXT:    [[T2_RECOMPOSED:%.*]] = urem i32 [[A]], [[B]]
+; CHECK-NEXT:    br label [[END:%.*]]
+; CHECK:       if.else:
+; CHECK-NEXT:    [[T3:%.*]] = udiv i32 [[A]], [[B]]
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    [[RET:%.*]] = phi i32 [ [[T2_RECOMPOSED]], [[IF_THEN]] ], [ [[T3]], [[IF_ELSE]] ]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+entry:
+  br i1 %cmp, label %if.then, label %if.else
+
+if.then:
+  %t0 = udiv i32 %a, %b
+  %t1 = mul nuw i32 %t0, %b
+  %t2 = sub i32 %a, %t1
+  br label %end
+
+if.else:
+  %t3 = udiv i32 %a, %b
+  br label %end
+
+end:
+  %ret = phi i32 [ %t2, %if.then ], [ %t3, %if.else ]
+  ret i32 %ret
+}
diff --git a/llvm/test/Transforms/DivRemPairs/X86/div-rem-pairs.ll b/llvm/test/Transforms/DivRemPairs/X86/div-rem-pairs.ll
index 711790004..5a5deb5e4 100644
--- a/llvm/test/Transforms/DivRemPairs/X86/div-rem-pairs.ll
+++ b/llvm/test/Transforms/DivRemPairs/X86/div-rem-pairs.ll
@@ -173,11 +173,11 @@ define i32 @srem_of_srem_expanded(i32 %X, i32 %Y, i32 %Z) {
 ; CHECK-NEXT:    [[T0:%.*]] = mul nsw i32 [[Z:%.*]], [[Y:%.*]]
 ; CHECK-NEXT:    [[T1:%.*]] = sdiv i32 [[X:%.*]], [[T0]]
 ; CHECK-NEXT:    [[T2:%.*]] = mul nsw i32 [[T0]], [[T1]]
-; CHECK-NEXT:    [[T3:%.*]] = sub nsw i32 [[X]], [[T2]]
-; CHECK-NEXT:    [[T4:%.*]] = sdiv i32 [[T3]], [[Y]]
+; CHECK-NEXT:    [[T3_RECOMPOSED:%.*]] = srem i32 [[X]], [[T0]]
+; CHECK-NEXT:    [[T4:%.*]] = sdiv i32 [[T3_RECOMPOSED]], [[Y]]
 ; CHECK-NEXT:    [[T5:%.*]] = mul nsw i32 [[T4]], [[Y]]
-; CHECK-NEXT:    [[T6:%.*]] = sub nsw i32 [[T3]], [[T5]]
-; CHECK-NEXT:    ret i32 [[T6]]
+; CHECK-NEXT:    [[T6_RECOMPOSED:%.*]] = srem i32 [[T3_RECOMPOSED]], [[Y]]
+; CHECK-NEXT:    ret i32 [[T6_RECOMPOSED]]
 ;
   %t0 = mul nsw i32 %Z, %Y
   %t1 = sdiv i32 %X, %t0
diff --git a/llvm/test/Transforms/EarlyCSE/commute.ll b/llvm/test/Transforms/EarlyCSE/commute.ll
index 0c8639b5e..6fdb0af0d 100644
--- a/llvm/test/Transforms/EarlyCSE/commute.ll
+++ b/llvm/test/Transforms/EarlyCSE/commute.ll
@@ -279,6 +279,8 @@ define i1 @umax_inverted(i8 %a, i8 %b) {
 }
 
 ; Min/max may exist with non-canonical operands. Value tracking can match those.
+; But we do not use value tracking, so we expect instcombine will canonicalize
+; this code to a form that allows CSE.
 
 define i8 @smax_nsw(i8 %a, i8 %b) {
 ; CHECK-LABEL: @smax_nsw(
@@ -286,7 +288,9 @@ define i8 @smax_nsw(i8 %a, i8 %b) {
 ; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i8 [[A]], [[B]]
 ; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i8 [[SUB]], 0
 ; CHECK-NEXT:    [[M1:%.*]] = select i1 [[CMP1]], i8 0, i8 [[SUB]]
-; CHECK-NEXT:    ret i8 0
+; CHECK-NEXT:    [[M2:%.*]] = select i1 [[CMP2]], i8 [[SUB]], i8 0
+; CHECK-NEXT:    [[R:%.*]] = sub i8 [[M2]], [[M1]]
+; CHECK-NEXT:    ret i8 [[R]]
 ;
   %sub = sub nsw i8 %a, %b
   %cmp1 = icmp slt i8 %a, %b
@@ -297,13 +301,19 @@ define i8 @smax_nsw(i8 %a, i8 %b) {
   ret i8 %r
 }
 
+; Abs/nabs may exist with non-canonical operands. Value tracking can match those.
+; But we do not use value tracking, so we expect instcombine will canonicalize
+; this code to a form that allows CSE.
+
 define i8 @abs_swapped(i8 %a) {
 ; CHECK-LABEL: @abs_swapped(
 ; CHECK-NEXT:    [[NEG:%.*]] = sub i8 0, [[A:%.*]]
 ; CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i8 [[A]], 0
 ; CHECK-NEXT:    [[CMP2:%.*]] = icmp slt i8 [[A]], 0
 ; CHECK-NEXT:    [[M1:%.*]] = select i1 [[CMP1]], i8 [[A]], i8 [[NEG]]
-; CHECK-NEXT:    ret i8 [[M1]]
+; CHECK-NEXT:    [[M2:%.*]] = select i1 [[CMP2]], i8 [[NEG]], i8 [[A]]
+; CHECK-NEXT:    [[R:%.*]] = or i8 [[M2]], [[M1]]
+; CHECK-NEXT:    ret i8 [[R]]
 ;
   %neg = sub i8 0, %a
   %cmp1 = icmp sgt i8 %a, 0
@@ -331,13 +341,19 @@ define i8 @abs_inverted(i8 %a) {
   ret i8 %r
 }
 
+; Abs/nabs may exist with non-canonical operands. Value tracking can match those.
+; But we do not use value tracking, so we expect instcombine will canonicalize
+; this code to a form that allows CSE.
+
 define i8 @nabs_swapped(i8 %a) {
 ; CHECK-LABEL: @nabs_swapped(
 ; CHECK-NEXT:    [[NEG:%.*]] = sub i8 0, [[A:%.*]]
 ; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i8 [[A]], 0
 ; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i8 [[A]], 0
 ; CHECK-NEXT:    [[M1:%.*]] = select i1 [[CMP1]], i8 [[A]], i8 [[NEG]]
-; CHECK-NEXT:    ret i8 0
+; CHECK-NEXT:    [[M2:%.*]] = select i1 [[CMP2]], i8 [[NEG]], i8 [[A]]
+; CHECK-NEXT:    [[R:%.*]] = xor i8 [[M2]], [[M1]]
+; CHECK-NEXT:    ret i8 [[R]]
 ;
   %neg = sub i8 0, %a
   %cmp1 = icmp slt i8 %a, 0
@@ -365,7 +381,10 @@ define i8 @nabs_inverted(i8 %a) {
   ret i8 %r
 }
 
-; These two tests make sure we still consider it a match when the RHS of the
+; Abs/nabs may exist with non-canonical operands. Value tracking can match those.
+; But we do not use value tracking, so we expect instcombine will canonicalize
+; this code to a form that allows CSE.
+
 ; compares are different.
 define i8 @abs_different_constants(i8 %a) {
 ; CHECK-LABEL: @abs_different_constants(
@@ -373,7 +392,9 @@ define i8 @abs_different_constants(i8 %a) {
 ; CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i8 [[A]], -1
 ; CHECK-NEXT:    [[CMP2:%.*]] = icmp slt i8 [[A]], 0
 ; CHECK-NEXT:    [[M1:%.*]] = select i1 [[CMP1]], i8 [[A]], i8 [[NEG]]
-; CHECK-NEXT:    ret i8 [[M1]]
+; CHECK-NEXT:    [[M2:%.*]] = select i1 [[CMP2]], i8 [[NEG]], i8 [[A]]
+; CHECK-NEXT:    [[R:%.*]] = or i8 [[M2]], [[M1]]
+; CHECK-NEXT:    ret i8 [[R]]
 ;
   %neg = sub i8 0, %a
   %cmp1 = icmp sgt i8 %a, -1
@@ -384,13 +405,19 @@ define i8 @abs_different_constants(i8 %a) {
   ret i8 %r
 }
 
+; Abs/nabs may exist with non-canonical operands. Value tracking can match those.
+; But we do not use value tracking, so we expect instcombine will canonicalize
+; this code to a form that allows CSE.
+
 define i8 @nabs_different_constants(i8 %a) {
 ; CHECK-LABEL: @nabs_different_constants(
 ; CHECK-NEXT:    [[NEG:%.*]] = sub i8 0, [[A:%.*]]
 ; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i8 [[A]], 0
 ; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i8 [[A]], -1
 ; CHECK-NEXT:    [[M1:%.*]] = select i1 [[CMP1]], i8 [[A]], i8 [[NEG]]
-; CHECK-NEXT:    ret i8 0
+; CHECK-NEXT:    [[M2:%.*]] = select i1 [[CMP2]], i8 [[NEG]], i8 [[A]]
+; CHECK-NEXT:    [[R:%.*]] = xor i8 [[M2]], [[M1]]
+; CHECK-NEXT:    ret i8 [[R]]
 ;
   %neg = sub i8 0, %a
   %cmp1 = icmp slt i8 %a, 0
@@ -689,3 +716,49 @@ define void @not_not_min(i32* %px, i32* %py, i32* %pout) {
 
   ret void
 }
+
+; This would cause an assert/crash because we matched
+; a ValueTracking select pattern that required 'nsw'
+; on an operand, but we remove that flag as part of
+; CSE matching/hashing.
+
+define void @PR41083_1(i32 %span_left, i32 %clip_left) {
+; CHECK-LABEL: @PR41083_1(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[CLIP_LEFT:%.*]], [[SPAN_LEFT:%.*]]
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 [[CLIP_LEFT]], [[SPAN_LEFT]]
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i32 [[SUB]], i32 0
+; CHECK-NEXT:    ret void
+;
+  %cmp = icmp sgt i32 %clip_left, %span_left
+  %sub = sub nsw i32 %clip_left, %span_left
+  %cond = select i1 %cmp, i32 %sub, i32 0
+  %cmp83292 = icmp slt i32 %cond, undef
+  %sub2 = sub i32 %clip_left, %span_left
+  %sel2 = select i1 %cmp, i32 %sub2, i32 0
+  ret void
+}
+
+; This would cause an assert/crash because we matched
+; a ValueTracking select pattern that required 'nsw'
+; on an operand, but we remove that flag as part of
+; CSE matching/hashing.
+
+define i32 @PR41083_2(i32 %p) {
+; CHECK-LABEL: @PR41083_2(
+; CHECK-NEXT:    [[S:%.*]] = sub i32 0, [[P:%.*]]
+; CHECK-NEXT:    [[A:%.*]] = ashr exact i32 [[S]], 2
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 0, [[A]]
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 0, [[A]]
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 [[SUB]], i32 0
+; CHECK-NEXT:    [[M:%.*]] = mul i32 [[SEL]], [[SUB]]
+; CHECK-NEXT:    ret i32 [[M]]
+;
+  %s = sub i32 0, %p
+  %a = ashr exact i32 %s, 2
+  %cmp = icmp sgt i32 0, %a
+  %sub = sub nsw i32 0, %a
+  %sel = select i1 %cmp, i32 %sub, i32 0
+  %s2 = sub i32 0, %a
+  %m = mul i32 %sel, %s2
+  ret i32 %m
+}
diff --git a/llvm/test/Transforms/EarlyCSE/debug-info-undef.ll b/llvm/test/Transforms/EarlyCSE/debug-info-undef.ll
index 4615aa264..b0fb8ff75 100644
--- a/llvm/test/Transforms/EarlyCSE/debug-info-undef.ll
+++ b/llvm/test/Transforms/EarlyCSE/debug-info-undef.ll
@@ -10,7 +10,7 @@ entry:
   %0 = load i8, i8* @a, align 1, !dbg !19, !tbaa !20
   %conv = sext i8 %0 to i16, !dbg !19
 
-; CHECK: call void @llvm.dbg.value(metadata i16 undef, metadata !17, metadata !DIExpression()), !dbg !18
+; CHECK: call void @llvm.dbg.value(metadata i8 %0, metadata !17, metadata !DIExpression(DW_OP_LLVM_convert, 8, DW_ATE_signed, DW_OP_LLVM_convert, 16, DW_ATE_signed, DW_OP_stack_value)), !dbg !18
 ; CHECK-NEXT:  call i32 (...) @optimize_me_not()
 
   call void @llvm.dbg.value(metadata i16 %conv, metadata !17, metadata !DIExpression()), !dbg !18
diff --git a/llvm/test/Transforms/EarlyCSE/floatingpoint.ll b/llvm/test/Transforms/EarlyCSE/floatingpoint.ll
index d6811a3d6..998f86dff 100644
--- a/llvm/test/Transforms/EarlyCSE/floatingpoint.ll
+++ b/llvm/test/Transforms/EarlyCSE/floatingpoint.ll
@@ -13,3 +13,15 @@ define <4 x float> @fW( <4 x float> %a) {
        %b = fadd  <4 x float> %a, <float -0.0,float -0.0,float -0.0,float -0.0>
        ret <4 x float> %b
 }
+
+; CSE unary fnegs.
+define void @fX(<4 x float> *%p, <4 x float> %a) {
+       ; CHECK: %x = fneg <4 x float> %a
+       ; CHECK-NEXT: store volatile <4 x float> %x, <4 x float>* %p
+       ; CHECK-NEXT: store volatile <4 x float> %x, <4 x float>* %p
+       %x = fneg <4 x float> %a
+       %y = fneg <4 x float> %a
+       store volatile <4 x float> %x, <4 x float>* %p
+       store volatile <4 x float> %y, <4 x float>* %p
+       ret void
+}
diff --git a/llvm/test/Transforms/EarlyCSE/writeonly.ll b/llvm/test/Transforms/EarlyCSE/writeonly.ll
new file mode 100644
index 000000000..0a3cd1c74
--- /dev/null
+++ b/llvm/test/Transforms/EarlyCSE/writeonly.ll
@@ -0,0 +1,15 @@
+; RUN: opt -S -early-cse < %s | FileCheck %s
+
+@var = global i32 undef
+declare void @foo() nounwind
+
+define void @test() {
+; CHECK-LABEL: @test(
+; CHECK-NOT: store
+  store i32 1, i32* @var
+; CHECK: call void @foo()
+  call void @foo() writeonly
+; CHECK: store i32 2, i32* @var
+  store i32 2, i32* @var
+  ret void
+}
diff --git a/llvm/test/Transforms/EntryExitInstrumenter/mcount.ll b/llvm/test/Transforms/EntryExitInstrumenter/mcount.ll
index 30483606c..56c8b53aa 100644
--- a/llvm/test/Transforms/EntryExitInstrumenter/mcount.ll
+++ b/llvm/test/Transforms/EntryExitInstrumenter/mcount.ll
@@ -54,7 +54,7 @@ define void @f1() #1 { entry: ret void }
 
 define void @f2() #2 { entry: ret void }
 ; CHECK-LABEL: define void @f2
-; CHECK: call void @"\01__gnu_mcount_nc"
+; CHECK: call void @llvm.arm.gnu.eabi.mcount
 
 define void @f3() #3 { entry: ret void }
 ; CHECK-LABEL: define void @f3
@@ -105,7 +105,7 @@ define i8* @tailcaller2() #8 {
 
 attributes #0 = { "instrument-function-entry-inlined"="mcount" "instrument-function-entry"="__cyg_profile_func_enter" "instrument-function-exit"="__cyg_profile_func_exit" }
 attributes #1 = { "instrument-function-entry-inlined"=".mcount" }
-attributes #2 = { "instrument-function-entry-inlined"="\01__gnu_mcount_nc" }
+attributes #2 = { "instrument-function-entry-inlined"="llvm.arm.gnu.eabi.mcount" }
 attributes #3 = { "instrument-function-entry-inlined"="\01_mcount" }
 attributes #4 = { "instrument-function-entry-inlined"="\01mcount" }
 attributes #5 = { "instrument-function-entry-inlined"="__mcount" }
diff --git a/llvm/test/Transforms/ExpandMemCmp/X86/memcmp.ll b/llvm/test/Transforms/ExpandMemCmp/X86/memcmp.ll
index c1cbcc327..6926029d2 100644
--- a/llvm/test/Transforms/ExpandMemCmp/X86/memcmp.ll
+++ b/llvm/test/Transforms/ExpandMemCmp/X86/memcmp.ll
@@ -41,8 +41,8 @@ define i32 @cmp3(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 ; ALL-NEXT:    [[TMP9:%.*]] = icmp eq i16 [[TMP7]], [[TMP8]]
 ; ALL-NEXT:    br i1 [[TMP9]], label [[LOADBB1:%.*]], label [[RES_BLOCK:%.*]]
 ; ALL:       loadbb1:
-; ALL-NEXT:    [[TMP10:%.*]] = getelementptr i8, i8* [[X]], i8 2
-; ALL-NEXT:    [[TMP11:%.*]] = getelementptr i8, i8* [[Y]], i8 2
+; ALL-NEXT:    [[TMP10:%.*]] = getelementptr i8, i8* [[X]], i64 2
+; ALL-NEXT:    [[TMP11:%.*]] = getelementptr i8, i8* [[Y]], i64 2
 ; ALL-NEXT:    [[TMP12:%.*]] = load i8, i8* [[TMP10]]
 ; ALL-NEXT:    [[TMP13:%.*]] = load i8, i8* [[TMP11]]
 ; ALL-NEXT:    [[TMP14:%.*]] = zext i8 [[TMP12]] to i32
@@ -95,8 +95,8 @@ define i32 @cmp5(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 ; ALL-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[TMP7]], [[TMP8]]
 ; ALL-NEXT:    br i1 [[TMP9]], label [[LOADBB1:%.*]], label [[RES_BLOCK:%.*]]
 ; ALL:       loadbb1:
-; ALL-NEXT:    [[TMP10:%.*]] = getelementptr i8, i8* [[X]], i8 4
-; ALL-NEXT:    [[TMP11:%.*]] = getelementptr i8, i8* [[Y]], i8 4
+; ALL-NEXT:    [[TMP10:%.*]] = getelementptr i8, i8* [[X]], i64 4
+; ALL-NEXT:    [[TMP11:%.*]] = getelementptr i8, i8* [[Y]], i64 4
 ; ALL-NEXT:    [[TMP12:%.*]] = load i8, i8* [[TMP10]]
 ; ALL-NEXT:    [[TMP13:%.*]] = load i8, i8* [[TMP11]]
 ; ALL-NEXT:    [[TMP14:%.*]] = zext i8 [[TMP12]] to i32
@@ -130,9 +130,9 @@ define i32 @cmp6(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 ; ALL-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[TMP7]], [[TMP8]]
 ; ALL-NEXT:    br i1 [[TMP9]], label [[LOADBB1]], label [[RES_BLOCK:%.*]]
 ; ALL:       loadbb1:
-; ALL-NEXT:    [[TMP10:%.*]] = getelementptr i8, i8* [[X]], i8 4
+; ALL-NEXT:    [[TMP10:%.*]] = getelementptr i8, i8* [[X]], i64 4
 ; ALL-NEXT:    [[TMP11:%.*]] = bitcast i8* [[TMP10]] to i16*
-; ALL-NEXT:    [[TMP12:%.*]] = getelementptr i8, i8* [[Y]], i8 4
+; ALL-NEXT:    [[TMP12:%.*]] = getelementptr i8, i8* [[Y]], i64 4
 ; ALL-NEXT:    [[TMP13:%.*]] = bitcast i8* [[TMP12]] to i16*
 ; ALL-NEXT:    [[TMP14:%.*]] = load i16, i16* [[TMP11]]
 ; ALL-NEXT:    [[TMP15:%.*]] = load i16, i16* [[TMP13]]
@@ -178,9 +178,9 @@ define i32 @cmp8(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 ; X32-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[TMP7]], [[TMP8]]
 ; X32-NEXT:    br i1 [[TMP9]], label [[LOADBB1]], label [[RES_BLOCK:%.*]]
 ; X32:       loadbb1:
-; X32-NEXT:    [[TMP10:%.*]] = getelementptr i8, i8* [[X]], i8 4
+; X32-NEXT:    [[TMP10:%.*]] = getelementptr i8, i8* [[X]], i64 4
 ; X32-NEXT:    [[TMP11:%.*]] = bitcast i8* [[TMP10]] to i32*
-; X32-NEXT:    [[TMP12:%.*]] = getelementptr i8, i8* [[Y]], i8 4
+; X32-NEXT:    [[TMP12:%.*]] = getelementptr i8, i8* [[Y]], i64 4
 ; X32-NEXT:    [[TMP13:%.*]] = bitcast i8* [[TMP12]] to i32*
 ; X32-NEXT:    [[TMP14:%.*]] = load i32, i32* [[TMP11]]
 ; X32-NEXT:    [[TMP15:%.*]] = load i32, i32* [[TMP13]]
@@ -233,8 +233,8 @@ define i32 @cmp9(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 ; X64-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[TMP7]], [[TMP8]]
 ; X64-NEXT:    br i1 [[TMP9]], label [[LOADBB1:%.*]], label [[RES_BLOCK:%.*]]
 ; X64:       loadbb1:
-; X64-NEXT:    [[TMP10:%.*]] = getelementptr i8, i8* [[X]], i8 8
-; X64-NEXT:    [[TMP11:%.*]] = getelementptr i8, i8* [[Y]], i8 8
+; X64-NEXT:    [[TMP10:%.*]] = getelementptr i8, i8* [[X]], i64 8
+; X64-NEXT:    [[TMP11:%.*]] = getelementptr i8, i8* [[Y]], i64 8
 ; X64-NEXT:    [[TMP12:%.*]] = load i8, i8* [[TMP10]]
 ; X64-NEXT:    [[TMP13:%.*]] = load i8, i8* [[TMP11]]
 ; X64-NEXT:    [[TMP14:%.*]] = zext i8 [[TMP12]] to i32
@@ -272,9 +272,9 @@ define i32 @cmp10(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 ; X64-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[TMP7]], [[TMP8]]
 ; X64-NEXT:    br i1 [[TMP9]], label [[LOADBB1]], label [[RES_BLOCK:%.*]]
 ; X64:       loadbb1:
-; X64-NEXT:    [[TMP10:%.*]] = getelementptr i8, i8* [[X]], i8 8
+; X64-NEXT:    [[TMP10:%.*]] = getelementptr i8, i8* [[X]], i64 8
 ; X64-NEXT:    [[TMP11:%.*]] = bitcast i8* [[TMP10]] to i16*
-; X64-NEXT:    [[TMP12:%.*]] = getelementptr i8, i8* [[Y]], i8 8
+; X64-NEXT:    [[TMP12:%.*]] = getelementptr i8, i8* [[Y]], i64 8
 ; X64-NEXT:    [[TMP13:%.*]] = bitcast i8* [[TMP12]] to i16*
 ; X64-NEXT:    [[TMP14:%.*]] = load i16, i16* [[TMP11]]
 ; X64-NEXT:    [[TMP15:%.*]] = load i16, i16* [[TMP13]]
@@ -324,9 +324,9 @@ define i32 @cmp12(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 ; X64-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[TMP7]], [[TMP8]]
 ; X64-NEXT:    br i1 [[TMP9]], label [[LOADBB1]], label [[RES_BLOCK:%.*]]
 ; X64:       loadbb1:
-; X64-NEXT:    [[TMP10:%.*]] = getelementptr i8, i8* [[X]], i8 8
+; X64-NEXT:    [[TMP10:%.*]] = getelementptr i8, i8* [[X]], i64 8
 ; X64-NEXT:    [[TMP11:%.*]] = bitcast i8* [[TMP10]] to i32*
-; X64-NEXT:    [[TMP12:%.*]] = getelementptr i8, i8* [[Y]], i8 8
+; X64-NEXT:    [[TMP12:%.*]] = getelementptr i8, i8* [[Y]], i64 8
 ; X64-NEXT:    [[TMP13:%.*]] = bitcast i8* [[TMP12]] to i32*
 ; X64-NEXT:    [[TMP14:%.*]] = load i32, i32* [[TMP11]]
 ; X64-NEXT:    [[TMP15:%.*]] = load i32, i32* [[TMP13]]
@@ -394,9 +394,9 @@ define i32 @cmp16(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 ; X64-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[TMP7]], [[TMP8]]
 ; X64-NEXT:    br i1 [[TMP9]], label [[LOADBB1]], label [[RES_BLOCK:%.*]]
 ; X64:       loadbb1:
-; X64-NEXT:    [[TMP10:%.*]] = getelementptr i8, i8* [[X]], i8 8
+; X64-NEXT:    [[TMP10:%.*]] = getelementptr i8, i8* [[X]], i64 8
 ; X64-NEXT:    [[TMP11:%.*]] = bitcast i8* [[TMP10]] to i64*
-; X64-NEXT:    [[TMP12:%.*]] = getelementptr i8, i8* [[Y]], i8 8
+; X64-NEXT:    [[TMP12:%.*]] = getelementptr i8, i8* [[Y]], i64 8
 ; X64-NEXT:    [[TMP13:%.*]] = bitcast i8* [[TMP12]] to i64*
 ; X64-NEXT:    [[TMP14:%.*]] = load i64, i64* [[TMP11]]
 ; X64-NEXT:    [[TMP15:%.*]] = load i64, i64* [[TMP13]]
@@ -437,8 +437,8 @@ define i32 @cmp_eq3(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 ; X32-NEXT:    [[TMP3:%.*]] = load i16, i16* [[TMP1]]
 ; X32-NEXT:    [[TMP4:%.*]] = load i16, i16* [[TMP2]]
 ; X32-NEXT:    [[TMP5:%.*]] = xor i16 [[TMP3]], [[TMP4]]
-; X32-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 2
-; X32-NEXT:    [[TMP7:%.*]] = getelementptr i8, i8* [[Y]], i8 2
+; X32-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i64 2
+; X32-NEXT:    [[TMP7:%.*]] = getelementptr i8, i8* [[Y]], i64 2
 ; X32-NEXT:    [[TMP8:%.*]] = load i8, i8* [[TMP6]]
 ; X32-NEXT:    [[TMP9:%.*]] = load i8, i8* [[TMP7]]
 ; X32-NEXT:    [[TMP10:%.*]] = zext i8 [[TMP8]] to i16
@@ -463,8 +463,8 @@ define i32 @cmp_eq3(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 ; X64_1LD-NEXT:    [[TMP5:%.*]] = icmp ne i16 [[TMP3]], [[TMP4]]
 ; X64_1LD-NEXT:    br i1 [[TMP5]], label [[RES_BLOCK:%.*]], label [[LOADBB1:%.*]]
 ; X64_1LD:       loadbb1:
-; X64_1LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 2
-; X64_1LD-NEXT:    [[TMP7:%.*]] = getelementptr i8, i8* [[Y]], i8 2
+; X64_1LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i64 2
+; X64_1LD-NEXT:    [[TMP7:%.*]] = getelementptr i8, i8* [[Y]], i64 2
 ; X64_1LD-NEXT:    [[TMP8:%.*]] = load i8, i8* [[TMP6]]
 ; X64_1LD-NEXT:    [[TMP9:%.*]] = load i8, i8* [[TMP7]]
 ; X64_1LD-NEXT:    [[TMP10:%.*]] = icmp ne i8 [[TMP8]], [[TMP9]]
@@ -481,8 +481,8 @@ define i32 @cmp_eq3(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 ; X64_2LD-NEXT:    [[TMP3:%.*]] = load i16, i16* [[TMP1]]
 ; X64_2LD-NEXT:    [[TMP4:%.*]] = load i16, i16* [[TMP2]]
 ; X64_2LD-NEXT:    [[TMP5:%.*]] = xor i16 [[TMP3]], [[TMP4]]
-; X64_2LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 2
-; X64_2LD-NEXT:    [[TMP7:%.*]] = getelementptr i8, i8* [[Y]], i8 2
+; X64_2LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i64 2
+; X64_2LD-NEXT:    [[TMP7:%.*]] = getelementptr i8, i8* [[Y]], i64 2
 ; X64_2LD-NEXT:    [[TMP8:%.*]] = load i8, i8* [[TMP6]]
 ; X64_2LD-NEXT:    [[TMP9:%.*]] = load i8, i8* [[TMP7]]
 ; X64_2LD-NEXT:    [[TMP10:%.*]] = zext i8 [[TMP8]] to i16
@@ -526,8 +526,8 @@ define i32 @cmp_eq5(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 ; X32-NEXT:    [[TMP3:%.*]] = load i32, i32* [[TMP1]]
 ; X32-NEXT:    [[TMP4:%.*]] = load i32, i32* [[TMP2]]
 ; X32-NEXT:    [[TMP5:%.*]] = xor i32 [[TMP3]], [[TMP4]]
-; X32-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 4
-; X32-NEXT:    [[TMP7:%.*]] = getelementptr i8, i8* [[Y]], i8 4
+; X32-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i64 4
+; X32-NEXT:    [[TMP7:%.*]] = getelementptr i8, i8* [[Y]], i64 4
 ; X32-NEXT:    [[TMP8:%.*]] = load i8, i8* [[TMP6]]
 ; X32-NEXT:    [[TMP9:%.*]] = load i8, i8* [[TMP7]]
 ; X32-NEXT:    [[TMP10:%.*]] = zext i8 [[TMP8]] to i32
@@ -552,8 +552,8 @@ define i32 @cmp_eq5(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 ; X64_1LD-NEXT:    [[TMP5:%.*]] = icmp ne i32 [[TMP3]], [[TMP4]]
 ; X64_1LD-NEXT:    br i1 [[TMP5]], label [[RES_BLOCK:%.*]], label [[LOADBB1:%.*]]
 ; X64_1LD:       loadbb1:
-; X64_1LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 4
-; X64_1LD-NEXT:    [[TMP7:%.*]] = getelementptr i8, i8* [[Y]], i8 4
+; X64_1LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i64 4
+; X64_1LD-NEXT:    [[TMP7:%.*]] = getelementptr i8, i8* [[Y]], i64 4
 ; X64_1LD-NEXT:    [[TMP8:%.*]] = load i8, i8* [[TMP6]]
 ; X64_1LD-NEXT:    [[TMP9:%.*]] = load i8, i8* [[TMP7]]
 ; X64_1LD-NEXT:    [[TMP10:%.*]] = icmp ne i8 [[TMP8]], [[TMP9]]
@@ -570,8 +570,8 @@ define i32 @cmp_eq5(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 ; X64_2LD-NEXT:    [[TMP3:%.*]] = load i32, i32* [[TMP1]]
 ; X64_2LD-NEXT:    [[TMP4:%.*]] = load i32, i32* [[TMP2]]
 ; X64_2LD-NEXT:    [[TMP5:%.*]] = xor i32 [[TMP3]], [[TMP4]]
-; X64_2LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 4
-; X64_2LD-NEXT:    [[TMP7:%.*]] = getelementptr i8, i8* [[Y]], i8 4
+; X64_2LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i64 4
+; X64_2LD-NEXT:    [[TMP7:%.*]] = getelementptr i8, i8* [[Y]], i64 4
 ; X64_2LD-NEXT:    [[TMP8:%.*]] = load i8, i8* [[TMP6]]
 ; X64_2LD-NEXT:    [[TMP9:%.*]] = load i8, i8* [[TMP7]]
 ; X64_2LD-NEXT:    [[TMP10:%.*]] = zext i8 [[TMP8]] to i32
@@ -597,9 +597,9 @@ define i32 @cmp_eq6(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 ; X32-NEXT:    [[TMP3:%.*]] = load i32, i32* [[TMP1]]
 ; X32-NEXT:    [[TMP4:%.*]] = load i32, i32* [[TMP2]]
 ; X32-NEXT:    [[TMP5:%.*]] = xor i32 [[TMP3]], [[TMP4]]
-; X32-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 4
+; X32-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i64 4
 ; X32-NEXT:    [[TMP7:%.*]] = bitcast i8* [[TMP6]] to i16*
-; X32-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i8 4
+; X32-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i64 4
 ; X32-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP8]] to i16*
 ; X32-NEXT:    [[TMP10:%.*]] = load i16, i16* [[TMP7]]
 ; X32-NEXT:    [[TMP11:%.*]] = load i16, i16* [[TMP9]]
@@ -625,9 +625,9 @@ define i32 @cmp_eq6(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 ; X64_1LD-NEXT:    [[TMP5:%.*]] = icmp ne i32 [[TMP3]], [[TMP4]]
 ; X64_1LD-NEXT:    br i1 [[TMP5]], label [[RES_BLOCK:%.*]], label [[LOADBB1:%.*]]
 ; X64_1LD:       loadbb1:
-; X64_1LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 4
+; X64_1LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i64 4
 ; X64_1LD-NEXT:    [[TMP7:%.*]] = bitcast i8* [[TMP6]] to i16*
-; X64_1LD-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i8 4
+; X64_1LD-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i64 4
 ; X64_1LD-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP8]] to i16*
 ; X64_1LD-NEXT:    [[TMP10:%.*]] = load i16, i16* [[TMP7]]
 ; X64_1LD-NEXT:    [[TMP11:%.*]] = load i16, i16* [[TMP9]]
@@ -645,9 +645,9 @@ define i32 @cmp_eq6(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 ; X64_2LD-NEXT:    [[TMP3:%.*]] = load i32, i32* [[TMP1]]
 ; X64_2LD-NEXT:    [[TMP4:%.*]] = load i32, i32* [[TMP2]]
 ; X64_2LD-NEXT:    [[TMP5:%.*]] = xor i32 [[TMP3]], [[TMP4]]
-; X64_2LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 4
+; X64_2LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i64 4
 ; X64_2LD-NEXT:    [[TMP7:%.*]] = bitcast i8* [[TMP6]] to i16*
-; X64_2LD-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i8 4
+; X64_2LD-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i64 4
 ; X64_2LD-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP8]] to i16*
 ; X64_2LD-NEXT:    [[TMP10:%.*]] = load i16, i16* [[TMP7]]
 ; X64_2LD-NEXT:    [[TMP11:%.*]] = load i16, i16* [[TMP9]]
@@ -674,9 +674,9 @@ define i32 @cmp_eq7(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 ; X32-NEXT:    [[TMP3:%.*]] = load i32, i32* [[TMP1]]
 ; X32-NEXT:    [[TMP4:%.*]] = load i32, i32* [[TMP2]]
 ; X32-NEXT:    [[TMP5:%.*]] = xor i32 [[TMP3]], [[TMP4]]
-; X32-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 3
+; X32-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i64 3
 ; X32-NEXT:    [[TMP7:%.*]] = bitcast i8* [[TMP6]] to i32*
-; X32-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i8 3
+; X32-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i64 3
 ; X32-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP8]] to i32*
 ; X32-NEXT:    [[TMP10:%.*]] = load i32, i32* [[TMP7]]
 ; X32-NEXT:    [[TMP11:%.*]] = load i32, i32* [[TMP9]]
@@ -700,9 +700,9 @@ define i32 @cmp_eq7(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 ; X64_1LD-NEXT:    [[TMP5:%.*]] = icmp ne i32 [[TMP3]], [[TMP4]]
 ; X64_1LD-NEXT:    br i1 [[TMP5]], label [[RES_BLOCK:%.*]], label [[LOADBB1:%.*]]
 ; X64_1LD:       loadbb1:
-; X64_1LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 3
+; X64_1LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i64 3
 ; X64_1LD-NEXT:    [[TMP7:%.*]] = bitcast i8* [[TMP6]] to i32*
-; X64_1LD-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i8 3
+; X64_1LD-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i64 3
 ; X64_1LD-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP8]] to i32*
 ; X64_1LD-NEXT:    [[TMP10:%.*]] = load i32, i32* [[TMP7]]
 ; X64_1LD-NEXT:    [[TMP11:%.*]] = load i32, i32* [[TMP9]]
@@ -720,9 +720,9 @@ define i32 @cmp_eq7(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 ; X64_2LD-NEXT:    [[TMP3:%.*]] = load i32, i32* [[TMP1]]
 ; X64_2LD-NEXT:    [[TMP4:%.*]] = load i32, i32* [[TMP2]]
 ; X64_2LD-NEXT:    [[TMP5:%.*]] = xor i32 [[TMP3]], [[TMP4]]
-; X64_2LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 3
+; X64_2LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i64 3
 ; X64_2LD-NEXT:    [[TMP7:%.*]] = bitcast i8* [[TMP6]] to i32*
-; X64_2LD-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i8 3
+; X64_2LD-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i64 3
 ; X64_2LD-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP8]] to i32*
 ; X64_2LD-NEXT:    [[TMP10:%.*]] = load i32, i32* [[TMP7]]
 ; X64_2LD-NEXT:    [[TMP11:%.*]] = load i32, i32* [[TMP9]]
@@ -747,9 +747,9 @@ define i32 @cmp_eq8(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 ; X32-NEXT:    [[TMP3:%.*]] = load i32, i32* [[TMP1]]
 ; X32-NEXT:    [[TMP4:%.*]] = load i32, i32* [[TMP2]]
 ; X32-NEXT:    [[TMP5:%.*]] = xor i32 [[TMP3]], [[TMP4]]
-; X32-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 4
+; X32-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i64 4
 ; X32-NEXT:    [[TMP7:%.*]] = bitcast i8* [[TMP6]] to i32*
-; X32-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i8 4
+; X32-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i64 4
 ; X32-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP8]] to i32*
 ; X32-NEXT:    [[TMP10:%.*]] = load i32, i32* [[TMP7]]
 ; X32-NEXT:    [[TMP11:%.*]] = load i32, i32* [[TMP9]]
@@ -797,8 +797,8 @@ define i32 @cmp_eq9(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 ; X64_1LD-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP3]], [[TMP4]]
 ; X64_1LD-NEXT:    br i1 [[TMP5]], label [[RES_BLOCK:%.*]], label [[LOADBB1:%.*]]
 ; X64_1LD:       loadbb1:
-; X64_1LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 8
-; X64_1LD-NEXT:    [[TMP7:%.*]] = getelementptr i8, i8* [[Y]], i8 8
+; X64_1LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i64 8
+; X64_1LD-NEXT:    [[TMP7:%.*]] = getelementptr i8, i8* [[Y]], i64 8
 ; X64_1LD-NEXT:    [[TMP8:%.*]] = load i8, i8* [[TMP6]]
 ; X64_1LD-NEXT:    [[TMP9:%.*]] = load i8, i8* [[TMP7]]
 ; X64_1LD-NEXT:    [[TMP10:%.*]] = icmp ne i8 [[TMP8]], [[TMP9]]
@@ -815,8 +815,8 @@ define i32 @cmp_eq9(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 ; X64_2LD-NEXT:    [[TMP3:%.*]] = load i64, i64* [[TMP1]]
 ; X64_2LD-NEXT:    [[TMP4:%.*]] = load i64, i64* [[TMP2]]
 ; X64_2LD-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP3]], [[TMP4]]
-; X64_2LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 8
-; X64_2LD-NEXT:    [[TMP7:%.*]] = getelementptr i8, i8* [[Y]], i8 8
+; X64_2LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i64 8
+; X64_2LD-NEXT:    [[TMP7:%.*]] = getelementptr i8, i8* [[Y]], i64 8
 ; X64_2LD-NEXT:    [[TMP8:%.*]] = load i8, i8* [[TMP6]]
 ; X64_2LD-NEXT:    [[TMP9:%.*]] = load i8, i8* [[TMP7]]
 ; X64_2LD-NEXT:    [[TMP10:%.*]] = zext i8 [[TMP8]] to i64
@@ -854,9 +854,9 @@ define i32 @cmp_eq10(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 ; X64_1LD-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP3]], [[TMP4]]
 ; X64_1LD-NEXT:    br i1 [[TMP5]], label [[RES_BLOCK:%.*]], label [[LOADBB1:%.*]]
 ; X64_1LD:       loadbb1:
-; X64_1LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 8
+; X64_1LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i64 8
 ; X64_1LD-NEXT:    [[TMP7:%.*]] = bitcast i8* [[TMP6]] to i16*
-; X64_1LD-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i8 8
+; X64_1LD-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i64 8
 ; X64_1LD-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP8]] to i16*
 ; X64_1LD-NEXT:    [[TMP10:%.*]] = load i16, i16* [[TMP7]]
 ; X64_1LD-NEXT:    [[TMP11:%.*]] = load i16, i16* [[TMP9]]
@@ -874,9 +874,9 @@ define i32 @cmp_eq10(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 ; X64_2LD-NEXT:    [[TMP3:%.*]] = load i64, i64* [[TMP1]]
 ; X64_2LD-NEXT:    [[TMP4:%.*]] = load i64, i64* [[TMP2]]
 ; X64_2LD-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP3]], [[TMP4]]
-; X64_2LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 8
+; X64_2LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i64 8
 ; X64_2LD-NEXT:    [[TMP7:%.*]] = bitcast i8* [[TMP6]] to i16*
-; X64_2LD-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i8 8
+; X64_2LD-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i64 8
 ; X64_2LD-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP8]] to i16*
 ; X64_2LD-NEXT:    [[TMP10:%.*]] = load i16, i16* [[TMP7]]
 ; X64_2LD-NEXT:    [[TMP11:%.*]] = load i16, i16* [[TMP9]]
@@ -915,9 +915,9 @@ define i32 @cmp_eq11(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 ; X64_1LD-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP3]], [[TMP4]]
 ; X64_1LD-NEXT:    br i1 [[TMP5]], label [[RES_BLOCK:%.*]], label [[LOADBB1:%.*]]
 ; X64_1LD:       loadbb1:
-; X64_1LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 3
+; X64_1LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i64 3
 ; X64_1LD-NEXT:    [[TMP7:%.*]] = bitcast i8* [[TMP6]] to i64*
-; X64_1LD-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i8 3
+; X64_1LD-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i64 3
 ; X64_1LD-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP8]] to i64*
 ; X64_1LD-NEXT:    [[TMP10:%.*]] = load i64, i64* [[TMP7]]
 ; X64_1LD-NEXT:    [[TMP11:%.*]] = load i64, i64* [[TMP9]]
@@ -935,9 +935,9 @@ define i32 @cmp_eq11(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 ; X64_2LD-NEXT:    [[TMP3:%.*]] = load i64, i64* [[TMP1]]
 ; X64_2LD-NEXT:    [[TMP4:%.*]] = load i64, i64* [[TMP2]]
 ; X64_2LD-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP3]], [[TMP4]]
-; X64_2LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 3
+; X64_2LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i64 3
 ; X64_2LD-NEXT:    [[TMP7:%.*]] = bitcast i8* [[TMP6]] to i64*
-; X64_2LD-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i8 3
+; X64_2LD-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i64 3
 ; X64_2LD-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP8]] to i64*
 ; X64_2LD-NEXT:    [[TMP10:%.*]] = load i64, i64* [[TMP7]]
 ; X64_2LD-NEXT:    [[TMP11:%.*]] = load i64, i64* [[TMP9]]
@@ -974,9 +974,9 @@ define i32 @cmp_eq12(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 ; X64_1LD-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP3]], [[TMP4]]
 ; X64_1LD-NEXT:    br i1 [[TMP5]], label [[RES_BLOCK:%.*]], label [[LOADBB1:%.*]]
 ; X64_1LD:       loadbb1:
-; X64_1LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 8
+; X64_1LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i64 8
 ; X64_1LD-NEXT:    [[TMP7:%.*]] = bitcast i8* [[TMP6]] to i32*
-; X64_1LD-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i8 8
+; X64_1LD-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i64 8
 ; X64_1LD-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP8]] to i32*
 ; X64_1LD-NEXT:    [[TMP10:%.*]] = load i32, i32* [[TMP7]]
 ; X64_1LD-NEXT:    [[TMP11:%.*]] = load i32, i32* [[TMP9]]
@@ -994,9 +994,9 @@ define i32 @cmp_eq12(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 ; X64_2LD-NEXT:    [[TMP3:%.*]] = load i64, i64* [[TMP1]]
 ; X64_2LD-NEXT:    [[TMP4:%.*]] = load i64, i64* [[TMP2]]
 ; X64_2LD-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP3]], [[TMP4]]
-; X64_2LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 8
+; X64_2LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i64 8
 ; X64_2LD-NEXT:    [[TMP7:%.*]] = bitcast i8* [[TMP6]] to i32*
-; X64_2LD-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i8 8
+; X64_2LD-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i64 8
 ; X64_2LD-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP8]] to i32*
 ; X64_2LD-NEXT:    [[TMP10:%.*]] = load i32, i32* [[TMP7]]
 ; X64_2LD-NEXT:    [[TMP11:%.*]] = load i32, i32* [[TMP9]]
@@ -1035,9 +1035,9 @@ define i32 @cmp_eq13(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 ; X64_1LD-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP3]], [[TMP4]]
 ; X64_1LD-NEXT:    br i1 [[TMP5]], label [[RES_BLOCK:%.*]], label [[LOADBB1:%.*]]
 ; X64_1LD:       loadbb1:
-; X64_1LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 5
+; X64_1LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i64 5
 ; X64_1LD-NEXT:    [[TMP7:%.*]] = bitcast i8* [[TMP6]] to i64*
-; X64_1LD-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i8 5
+; X64_1LD-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i64 5
 ; X64_1LD-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP8]] to i64*
 ; X64_1LD-NEXT:    [[TMP10:%.*]] = load i64, i64* [[TMP7]]
 ; X64_1LD-NEXT:    [[TMP11:%.*]] = load i64, i64* [[TMP9]]
@@ -1055,9 +1055,9 @@ define i32 @cmp_eq13(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 ; X64_2LD-NEXT:    [[TMP3:%.*]] = load i64, i64* [[TMP1]]
 ; X64_2LD-NEXT:    [[TMP4:%.*]] = load i64, i64* [[TMP2]]
 ; X64_2LD-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP3]], [[TMP4]]
-; X64_2LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 5
+; X64_2LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i64 5
 ; X64_2LD-NEXT:    [[TMP7:%.*]] = bitcast i8* [[TMP6]] to i64*
-; X64_2LD-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i8 5
+; X64_2LD-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i64 5
 ; X64_2LD-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP8]] to i64*
 ; X64_2LD-NEXT:    [[TMP10:%.*]] = load i64, i64* [[TMP7]]
 ; X64_2LD-NEXT:    [[TMP11:%.*]] = load i64, i64* [[TMP9]]
@@ -1094,9 +1094,9 @@ define i32 @cmp_eq14(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 ; X64_1LD-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP3]], [[TMP4]]
 ; X64_1LD-NEXT:    br i1 [[TMP5]], label [[RES_BLOCK:%.*]], label [[LOADBB1:%.*]]
 ; X64_1LD:       loadbb1:
-; X64_1LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 6
+; X64_1LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i64 6
 ; X64_1LD-NEXT:    [[TMP7:%.*]] = bitcast i8* [[TMP6]] to i64*
-; X64_1LD-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i8 6
+; X64_1LD-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i64 6
 ; X64_1LD-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP8]] to i64*
 ; X64_1LD-NEXT:    [[TMP10:%.*]] = load i64, i64* [[TMP7]]
 ; X64_1LD-NEXT:    [[TMP11:%.*]] = load i64, i64* [[TMP9]]
@@ -1114,9 +1114,9 @@ define i32 @cmp_eq14(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 ; X64_2LD-NEXT:    [[TMP3:%.*]] = load i64, i64* [[TMP1]]
 ; X64_2LD-NEXT:    [[TMP4:%.*]] = load i64, i64* [[TMP2]]
 ; X64_2LD-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP3]], [[TMP4]]
-; X64_2LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 6
+; X64_2LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i64 6
 ; X64_2LD-NEXT:    [[TMP7:%.*]] = bitcast i8* [[TMP6]] to i64*
-; X64_2LD-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i8 6
+; X64_2LD-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i64 6
 ; X64_2LD-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP8]] to i64*
 ; X64_2LD-NEXT:    [[TMP10:%.*]] = load i64, i64* [[TMP7]]
 ; X64_2LD-NEXT:    [[TMP11:%.*]] = load i64, i64* [[TMP9]]
@@ -1153,9 +1153,9 @@ define i32 @cmp_eq15(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 ; X64_1LD-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP3]], [[TMP4]]
 ; X64_1LD-NEXT:    br i1 [[TMP5]], label [[RES_BLOCK:%.*]], label [[LOADBB1:%.*]]
 ; X64_1LD:       loadbb1:
-; X64_1LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 7
+; X64_1LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i64 7
 ; X64_1LD-NEXT:    [[TMP7:%.*]] = bitcast i8* [[TMP6]] to i64*
-; X64_1LD-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i8 7
+; X64_1LD-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i64 7
 ; X64_1LD-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP8]] to i64*
 ; X64_1LD-NEXT:    [[TMP10:%.*]] = load i64, i64* [[TMP7]]
 ; X64_1LD-NEXT:    [[TMP11:%.*]] = load i64, i64* [[TMP9]]
@@ -1173,9 +1173,9 @@ define i32 @cmp_eq15(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 ; X64_2LD-NEXT:    [[TMP3:%.*]] = load i64, i64* [[TMP1]]
 ; X64_2LD-NEXT:    [[TMP4:%.*]] = load i64, i64* [[TMP2]]
 ; X64_2LD-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP3]], [[TMP4]]
-; X64_2LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 7
+; X64_2LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i64 7
 ; X64_2LD-NEXT:    [[TMP7:%.*]] = bitcast i8* [[TMP6]] to i64*
-; X64_2LD-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i8 7
+; X64_2LD-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i64 7
 ; X64_2LD-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP8]] to i64*
 ; X64_2LD-NEXT:    [[TMP10:%.*]] = load i64, i64* [[TMP7]]
 ; X64_2LD-NEXT:    [[TMP11:%.*]] = load i64, i64* [[TMP9]]
diff --git a/llvm/test/Transforms/Float2Int/basic.ll b/llvm/test/Transforms/Float2Int/basic.ll
index e1eddc668..615cc601a 100644
--- a/llvm/test/Transforms/Float2Int/basic.ll
+++ b/llvm/test/Transforms/Float2Int/basic.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -float2int -S | FileCheck %s
 ; RUN: opt < %s -passes='float2int' -S | FileCheck %s
 
@@ -5,108 +6,116 @@
 ; Positive tests
 ;
 
-; CHECK-LABEL: @simple1
-; CHECK:  %1 = zext i8 %a to i32
-; CHECK:  %2 = add i32 %1, 1
-; CHECK:  %3 = trunc i32 %2 to i16
-; CHECK:  ret i16 %3
 define i16 @simple1(i8 %a) {
-  %1 = uitofp i8 %a to float
-  %2 = fadd float %1, 1.0
-  %3 = fptoui float %2 to i16
-  ret i16 %3
+; CHECK-LABEL: @simple1(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i32
+; CHECK-NEXT:    [[T21:%.*]] = add i32 [[TMP1]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc i32 [[T21]] to i16
+; CHECK-NEXT:    ret i16 [[TMP2]]
+;
+  %t1 = uitofp i8 %a to float
+  %t2 = fadd float %t1, 1.0
+  %t3 = fptoui float %t2 to i16
+  ret i16 %t3
 }
 
-; CHECK-LABEL: @simple2
-; CHECK:  %1 = zext i8 %a to i32
-; CHECK:  %2 = sub i32 %1, 1
-; CHECK:  %3 = trunc i32 %2 to i8
-; CHECK:  ret i8 %3
 define i8 @simple2(i8 %a) {
-  %1 = uitofp i8 %a to float
-  %2 = fsub float %1, 1.0
-  %3 = fptoui float %2 to i8
-  ret i8 %3
+; CHECK-LABEL: @simple2(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i32
+; CHECK-NEXT:    [[T21:%.*]] = sub i32 [[TMP1]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc i32 [[T21]] to i8
+; CHECK-NEXT:    ret i8 [[TMP2]]
+;
+  %t1 = uitofp i8 %a to float
+  %t2 = fsub float %t1, 1.0
+  %t3 = fptoui float %t2 to i8
+  ret i8 %t3
 }
 
-; CHECK-LABEL: @simple3
-; CHECK:  %1 = zext i8 %a to i32
-; CHECK:  %2 = sub i32 %1, 1
-; CHECK:  ret i32 %2
 define i32 @simple3(i8 %a) {
-  %1 = uitofp i8 %a to float
-  %2 = fsub float %1, 1.0
-  %3 = fptoui float %2 to i32
-  ret i32 %3
+; CHECK-LABEL: @simple3(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i32
+; CHECK-NEXT:    [[T21:%.*]] = sub i32 [[TMP1]], 1
+; CHECK-NEXT:    ret i32 [[T21]]
+;
+  %t1 = uitofp i8 %a to float
+  %t2 = fsub float %t1, 1.0
+  %t3 = fptoui float %t2 to i32
+  ret i32 %t3
 }
 
-; CHECK-LABEL: @cmp
-; CHECK:  %1 = zext i8 %a to i32
-; CHECK:  %2 = zext i8 %b to i32
-; CHECK:  %3 = icmp slt i32 %1, %2
-; CHECK:  ret i1 %3
 define i1 @cmp(i8 %a, i8 %b) {
-  %1 = uitofp i8 %a to float
-  %2 = uitofp i8 %b to float
-  %3 = fcmp ult float %1, %2
-  ret i1 %3
+; CHECK-LABEL: @cmp(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i8 [[B:%.*]] to i32
+; CHECK-NEXT:    [[T31:%.*]] = icmp slt i32 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret i1 [[T31]]
+;
+  %t1 = uitofp i8 %a to float
+  %t2 = uitofp i8 %b to float
+  %t3 = fcmp ult float %t1, %t2
+  ret i1 %t3
 }
 
-; CHECK-LABEL: @simple4
-; CHECK:  %1 = zext i32 %a to i64
-; CHECK:  %2 = add i64 %1, 1
-; CHECK:  %3 = trunc i64 %2 to i32
-; CHECK:  ret i32 %3
 define i32 @simple4(i32 %a) {
-  %1 = uitofp i32 %a to double
-  %2 = fadd double %1, 1.0
-  %3 = fptoui double %2 to i32
-  ret i32 %3
+; CHECK-LABEL: @simple4(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[A:%.*]] to i64
+; CHECK-NEXT:    [[T21:%.*]] = add i64 [[TMP1]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc i64 [[T21]] to i32
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
+  %t1 = uitofp i32 %a to double
+  %t2 = fadd double %t1, 1.0
+  %t3 = fptoui double %t2 to i32
+  ret i32 %t3
 }
 
-; CHECK-LABEL: @simple5
-; CHECK:  %1 = zext i8 %a to i32
-; CHECK:  %2 = zext i8 %b to i32
-; CHECK:  %3 = add i32 %1, 1
-; CHECK:  %4 = mul i32 %3, %2
-; CHECK:  ret i32 %4
 define i32 @simple5(i8 %a, i8 %b) {
-  %1 = uitofp i8 %a to float
-  %2 = uitofp i8 %b to float
-  %3 = fadd float %1, 1.0
-  %4 = fmul float %3, %2
-  %5 = fptoui float %4 to i32
-  ret i32 %5
+; CHECK-LABEL: @simple5(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i8 [[B:%.*]] to i32
+; CHECK-NEXT:    [[T31:%.*]] = add i32 [[TMP1]], 1
+; CHECK-NEXT:    [[T42:%.*]] = mul i32 [[T31]], [[TMP2]]
+; CHECK-NEXT:    ret i32 [[T42]]
+;
+  %t1 = uitofp i8 %a to float
+  %t2 = uitofp i8 %b to float
+  %t3 = fadd float %t1, 1.0
+  %t4 = fmul float %t3, %t2
+  %t5 = fptoui float %t4 to i32
+  ret i32 %t5
 }
 
-; CHECK-LABEL: @simple6
-; CHECK:  %1 = zext i8 %a to i32
-; CHECK:  %2 = zext i8 %b to i32
-; CHECK:  %3 = sub i32 0, %1
-; CHECK:  %4 = mul i32 %3, %2
-; CHECK:  ret i32 %4
 define i32 @simple6(i8 %a, i8 %b) {
-  %1 = uitofp i8 %a to float
-  %2 = uitofp i8 %b to float
-  %3 = fneg float %1
-  %4 = fmul float %3, %2
-  %5 = fptoui float %4 to i32
-  ret i32 %5
+; CHECK-LABEL: @simple6(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i8 [[B:%.*]] to i32
+; CHECK-NEXT:    [[T31:%.*]] = sub i32 0, [[TMP1]]
+; CHECK-NEXT:    [[T42:%.*]] = mul i32 [[T31]], [[TMP2]]
+; CHECK-NEXT:    ret i32 [[T42]]
+;
+  %t1 = uitofp i8 %a to float
+  %t2 = uitofp i8 %b to float
+  %t3 = fneg float %t1
+  %t4 = fmul float %t3, %t2
+  %t5 = fptoui float %t4 to i32
+  ret i32 %t5
 }
 
 ; The two chains don't interact - failure of one shouldn't
 ; cause failure of the other.
 
-; CHECK-LABEL: @multi1
-; CHECK:  %1 = zext i8 %a to i32
-; CHECK:  %2 = zext i8 %b to i32
-; CHECK:  %fc = uitofp i8 %c to float
-; CHECK:  %x1 = add i32 %1, %2
-; CHECK:  %z = fadd float %fc, %d
-; CHECK:  %w = fptoui float %z to i32
-; CHECK:  %r = add i32 %x1, %w
-; CHECK:  ret i32 %r
 define i32 @multi1(i8 %a, i8 %b, i8 %c, float %d) {
+; CHECK-LABEL: @multi1(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i8 [[B:%.*]] to i32
+; CHECK-NEXT:    [[FC:%.*]] = uitofp i8 [[C:%.*]] to float
+; CHECK-NEXT:    [[X1:%.*]] = add i32 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[Z:%.*]] = fadd float [[FC]], [[D:%.*]]
+; CHECK-NEXT:    [[W:%.*]] = fptoui float [[Z]] to i32
+; CHECK-NEXT:    [[R:%.*]] = add i32 [[X1]], [[W]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
   %fa = uitofp i8 %a to float
   %fb = uitofp i8 %b to float
   %fc = uitofp i8 %c to float
@@ -118,25 +127,27 @@ define i32 @multi1(i8 %a, i8 %b, i8 %c, float %d) {
   ret i32 %r
 }
 
-; CHECK-LABEL: @simple_negzero
-; CHECK:  %1 = zext i8 %a to i32
-; CHECK:  %2 = add i32 %1, 0
-; CHECK:  %3 = trunc i32 %2 to i16
-; CHECK:  ret i16 %3
 define i16 @simple_negzero(i8 %a) {
-  %1 = uitofp i8 %a to float
-  %2 = fadd fast float %1, -0.0
-  %3 = fptoui float %2 to i16
-  ret i16 %3
+; CHECK-LABEL: @simple_negzero(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i32
+; CHECK-NEXT:    [[T21:%.*]] = add i32 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc i32 [[T21]] to i16
+; CHECK-NEXT:    ret i16 [[TMP2]]
+;
+  %t1 = uitofp i8 %a to float
+  %t2 = fadd fast float %t1, -0.0
+  %t3 = fptoui float %t2 to i16
+  ret i16 %t3
 }
 
-; CHECK-LABEL: @simple_negative
-; CHECK: %1 = sext i8 %call to i32
-; CHECK: %mul1 = mul i32 %1, -3
-; CHECK: %2 = trunc i32 %mul1 to i8
-; CHECK: %conv3 = sext i8 %2 to i32
-; CHECK: ret i32 %conv3
 define i32 @simple_negative(i8 %call) {
+; CHECK-LABEL: @simple_negative(
+; CHECK-NEXT:    [[TMP1:%.*]] = sext i8 [[CALL:%.*]] to i32
+; CHECK-NEXT:    [[MUL1:%.*]] = mul i32 [[TMP1]], -3
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc i32 [[MUL1]] to i8
+; CHECK-NEXT:    [[CONV3:%.*]] = sext i8 [[TMP2]] to i32
+; CHECK-NEXT:    ret i32 [[CONV3]]
+;
   %conv1 = sitofp i8 %call to float
   %mul = fmul float %conv1, -3.000000e+00
   %conv2 = fptosi float %mul to i8
@@ -144,34 +155,37 @@ define i32 @simple_negative(i8 %call) {
   ret i32 %conv3
 }
 
-; CHECK-LABEL: @simple_fneg
-; CHECK:  %1 = zext i8 %a to i32
-; CHECK:  %2 = sub i32 0, %1
-; CHECK:  %3 = trunc i32 %2 to i16
-; CHECK:  ret i16 %3
 define i16 @simple_fneg(i8 %a) {
-  %1 = uitofp i8 %a to float
-  %2 = fneg fast float %1
-  %3 = fptoui float %2 to i16
-  ret i16 %3
+; CHECK-LABEL: @simple_fneg(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i32
+; CHECK-NEXT:    [[T21:%.*]] = sub i32 0, [[TMP1]]
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc i32 [[T21]] to i16
+; CHECK-NEXT:    ret i16 [[TMP2]]
+;
+  %t1 = uitofp i8 %a to float
+  %t2 = fneg fast float %t1
+  %t3 = fptoui float %t2 to i16
+  ret i16 %t3
 }
 
 ;
 ; Negative tests
 ;
 
-; CHECK-LABEL: @neg_multi1
-; CHECK:  %fa = uitofp i8 %a to float
-; CHECK:  %fc = uitofp i8 %c to float
-; CHECK:  %x = fadd float %fa, %fc
-; CHECK:  %y = fptoui float %x to i32
-; CHECK:  %z = fadd float %fc, %d
-; CHECK:  %w = fptoui float %z to i32
-; CHECK:  %r = add i32 %y, %w
-; CHECK:  ret i32 %r
 ; The two chains intersect, which means because one fails, no
 ; transform can occur.
+
 define i32 @neg_multi1(i8 %a, i8 %b, i8 %c, float %d) {
+; CHECK-LABEL: @neg_multi1(
+; CHECK-NEXT:    [[FA:%.*]] = uitofp i8 [[A:%.*]] to float
+; CHECK-NEXT:    [[FC:%.*]] = uitofp i8 [[C:%.*]] to float
+; CHECK-NEXT:    [[X:%.*]] = fadd float [[FA]], [[FC]]
+; CHECK-NEXT:    [[Y:%.*]] = fptoui float [[X]] to i32
+; CHECK-NEXT:    [[Z:%.*]] = fadd float [[FC]], [[D:%.*]]
+; CHECK-NEXT:    [[W:%.*]] = fptoui float [[Z]] to i32
+; CHECK-NEXT:    [[R:%.*]] = add i32 [[Y]], [[W]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
   %fa = uitofp i8 %a to float
   %fc = uitofp i8 %c to float
   %x = fadd float %fa, %fc
@@ -182,15 +196,17 @@ define i32 @neg_multi1(i8 %a, i8 %b, i8 %c, float %d) {
   ret i32 %r
 }
 
-; CHECK-LABEL: @neg_muld
-; CHECK:  %fa = uitofp i32 %a to double
-; CHECK:  %fb = uitofp i32 %b to double
-; CHECK:  %mul = fmul double %fa, %fb
-; CHECK:  %r = fptoui double %mul to i64
-; CHECK:  ret i64 %r
 ; The i32 * i32 = i64, which has 64 bits, which is greater than the 52 bits
 ; that can be exactly represented in a double.
+
 define i64 @neg_muld(i32 %a, i32 %b) {
+; CHECK-LABEL: @neg_muld(
+; CHECK-NEXT:    [[FA:%.*]] = uitofp i32 [[A:%.*]] to double
+; CHECK-NEXT:    [[FB:%.*]] = uitofp i32 [[B:%.*]] to double
+; CHECK-NEXT:    [[MUL:%.*]] = fmul double [[FA]], [[FB]]
+; CHECK-NEXT:    [[R:%.*]] = fptoui double [[MUL]] to i64
+; CHECK-NEXT:    ret i64 [[R]]
+;
   %fa = uitofp i32 %a to double
   %fb = uitofp i32 %b to double
   %mul = fmul double %fa, %fb
@@ -198,16 +214,18 @@ define i64 @neg_muld(i32 %a, i32 %b) {
   ret i64 %r
 }
 
-; CHECK-LABEL: @neg_mulf
-; CHECK:  %fa = uitofp i16 %a to float
-; CHECK:  %fb = uitofp i16 %b to float
-; CHECK:  %mul = fmul float %fa, %fb
-; CHECK:  %r = fptoui float %mul to i32
-; CHECK:  ret i32 %r
 ; The i16 * i16 = i32, which can't be represented in a float, but can in a
 ; double. This should fail, as the written code uses floats, not doubles so
 ; the original result may be inaccurate.
+
 define i32 @neg_mulf(i16 %a, i16 %b) {
+; CHECK-LABEL: @neg_mulf(
+; CHECK-NEXT:    [[FA:%.*]] = uitofp i16 [[A:%.*]] to float
+; CHECK-NEXT:    [[FB:%.*]] = uitofp i16 [[B:%.*]] to float
+; CHECK-NEXT:    [[MUL:%.*]] = fmul float [[FA]], [[FB]]
+; CHECK-NEXT:    [[R:%.*]] = fptoui float [[MUL]] to i32
+; CHECK-NEXT:    ret i32 [[R]]
+;
   %fa = uitofp i16 %a to float
   %fb = uitofp i16 %b to float
   %mul = fmul float %fa, %fb
@@ -215,80 +233,120 @@ define i32 @neg_mulf(i16 %a, i16 %b) {
   ret i32 %r
 }
 
-; CHECK-LABEL: @neg_cmp
-; CHECK:  %1 = uitofp i8 %a to float
-; CHECK:  %2 = uitofp i8 %b to float
-; CHECK:  %3 = fcmp false float %1, %2
-; CHECK:  ret i1 %3
 ; "false" doesn't have an icmp equivalent.
+
 define i1 @neg_cmp(i8 %a, i8 %b) {
-  %1 = uitofp i8 %a to float
-  %2 = uitofp i8 %b to float
-  %3 = fcmp false float %1, %2
-  ret i1 %3
+; CHECK-LABEL: @neg_cmp(
+; CHECK-NEXT:    [[T1:%.*]] = uitofp i8 [[A:%.*]] to float
+; CHECK-NEXT:    [[T2:%.*]] = uitofp i8 [[B:%.*]] to float
+; CHECK-NEXT:    [[T3:%.*]] = fcmp false float [[T1]], [[T2]]
+; CHECK-NEXT:    ret i1 [[T3]]
+;
+  %t1 = uitofp i8 %a to float
+  %t2 = uitofp i8 %b to float
+  %t3 = fcmp false float %t1, %t2
+  ret i1 %t3
 }
 
-; CHECK-LABEL: @neg_div
-; CHECK:  %1 = uitofp i8 %a to float
-; CHECK:  %2 = fdiv float %1, 1.0
-; CHECK:  %3 = fptoui float %2 to i16
-; CHECK:  ret i16 %3
 ; Division isn't a supported operator.
+
 define i16 @neg_div(i8 %a) {
-  %1 = uitofp i8 %a to float
-  %2 = fdiv float %1, 1.0
-  %3 = fptoui float %2 to i16
-  ret i16 %3
+; CHECK-LABEL: @neg_div(
+; CHECK-NEXT:    [[T1:%.*]] = uitofp i8 [[A:%.*]] to float
+; CHECK-NEXT:    [[T2:%.*]] = fdiv float [[T1]], 1.000000e+00
+; CHECK-NEXT:    [[T3:%.*]] = fptoui float [[T2]] to i16
+; CHECK-NEXT:    ret i16 [[T3]]
+;
+  %t1 = uitofp i8 %a to float
+  %t2 = fdiv float %t1, 1.0
+  %t3 = fptoui float %t2 to i16
+  ret i16 %t3
 }
 
-; CHECK-LABEL: @neg_remainder
-; CHECK:  %1 = uitofp i8 %a to float
-; CHECK:  %2 = fadd float %1, 1.2
-; CHECK:  %3 = fptoui float %2 to i16
-; CHECK:  ret i16 %3
 ; 1.2 is not an integer.
+
 define i16 @neg_remainder(i8 %a) {
-  %1 = uitofp i8 %a to float
-  %2 = fadd float %1, 1.25
-  %3 = fptoui float %2 to i16
-  ret i16 %3
+; CHECK-LABEL: @neg_remainder(
+; CHECK-NEXT:    [[T1:%.*]] = uitofp i8 [[A:%.*]] to float
+; CHECK-NEXT:    [[T2:%.*]] = fadd float [[T1]], 1.250000e+00
+; CHECK-NEXT:    [[T3:%.*]] = fptoui float [[T2]] to i16
+; CHECK-NEXT:    ret i16 [[T3]]
+;
+  %t1 = uitofp i8 %a to float
+  %t2 = fadd float %t1, 1.25
+  %t3 = fptoui float %t2 to i16
+  ret i16 %t3
 }
 
-; CHECK-LABEL: @neg_toolarge
-; CHECK:  %1 = uitofp i80 %a to fp128
-; CHECK:  %2 = fadd fp128 %1, %1
-; CHECK:  %3 = fptoui fp128 %2 to i80
-; CHECK:  ret i80 %3
 ; i80 > i64, which is the largest bitwidth handleable by default.
+
 define i80 @neg_toolarge(i80 %a) {
-  %1 = uitofp i80 %a to fp128
-  %2 = fadd fp128 %1, %1
-  %3 = fptoui fp128 %2 to i80
-  ret i80 %3
+; CHECK-LABEL: @neg_toolarge(
+; CHECK-NEXT:    [[T1:%.*]] = uitofp i80 [[A:%.*]] to fp128
+; CHECK-NEXT:    [[T2:%.*]] = fadd fp128 [[T1]], [[T1]]
+; CHECK-NEXT:    [[T3:%.*]] = fptoui fp128 [[T2]] to i80
+; CHECK-NEXT:    ret i80 [[T3]]
+;
+  %t1 = uitofp i80 %a to fp128
+  %t2 = fadd fp128 %t1, %t1
+  %t3 = fptoui fp128 %t2 to i80
+  ret i80 %t3
 }
 
-; CHECK-LABEL: @neg_calluser
-; CHECK: sitofp
-; CHECK: fcmp
-; The sequence %1..%3 cannot be converted because %4 uses %2.
+; The sequence %t1..%t3 cannot be converted because %t4 uses %t2.
+
 define i32 @neg_calluser(i32 %value) {
-  %1 = sitofp i32 %value to double
-  %2 = fadd double %1, 1.0
-  %3 = fcmp olt double %2, 0.000000e+00
-  %4 = tail call double @g(double %2)
-  %5 = fptosi double %4 to i32
-  %6 = zext i1 %3 to i32
-  %7 = add i32 %6, %5
-  ret i32 %7
+; CHECK-LABEL: @neg_calluser(
+; CHECK-NEXT:    [[T1:%.*]] = sitofp i32 [[VALUE:%.*]] to double
+; CHECK-NEXT:    [[T2:%.*]] = fadd double [[T1]], 1.000000e+00
+; CHECK-NEXT:    [[T3:%.*]] = fcmp olt double [[T2]], 0.000000e+00
+; CHECK-NEXT:    [[T4:%.*]] = tail call double @g(double [[T2]])
+; CHECK-NEXT:    [[T5:%.*]] = fptosi double [[T4]] to i32
+; CHECK-NEXT:    [[T6:%.*]] = zext i1 [[T3]] to i32
+; CHECK-NEXT:    [[T7:%.*]] = add i32 [[T6]], [[T5]]
+; CHECK-NEXT:    ret i32 [[T7]]
+;
+  %t1 = sitofp i32 %value to double
+  %t2 = fadd double %t1, 1.0
+  %t3 = fcmp olt double %t2, 0.000000e+00
+  %t4 = tail call double @g(double %t2)
+  %t5 = fptosi double %t4 to i32
+  %t6 = zext i1 %t3 to i32
+  %t7 = add i32 %t6, %t5
+  ret i32 %t7
 }
+
 declare double @g(double)
 
-; CHECK-LABEL: @neg_vector
-; CHECK:  %1 = uitofp <4 x i8> %a to <4 x float>
-; CHECK:  %2 = fptoui <4 x float> %1 to <4 x i16>
-; CHECK:  ret <4 x i16> %2
 define <4 x i16> @neg_vector(<4 x i8> %a) {
-  %1 = uitofp <4 x i8> %a to <4 x float>
-  %2 = fptoui <4 x float> %1 to <4 x i16>
-  ret <4 x i16> %2
+; CHECK-LABEL: @neg_vector(
+; CHECK-NEXT:    [[T1:%.*]] = uitofp <4 x i8> [[A:%.*]] to <4 x float>
+; CHECK-NEXT:    [[T2:%.*]] = fptoui <4 x float> [[T1]] to <4 x i16>
+; CHECK-NEXT:    ret <4 x i16> [[T2]]
+;
+  %t1 = uitofp <4 x i8> %a to <4 x float>
+  %t2 = fptoui <4 x float> %t1 to <4 x i16>
+  ret <4 x i16> %t2
+}
+
+; Don't crash while processing unreachable (non-standard) IR.
+
+define void @PR38502() {
+; CHECK-LABEL: @PR38502(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret void
+; CHECK:       bogusBB:
+; CHECK-NEXT:    [[INC1:%.*]] = fadd double [[INC:%.*]], 1.000000e+00
+; CHECK-NEXT:    [[INC]] = fadd double [[INC1]], 1.000000e+00
+; CHECK-NEXT:    [[TOBOOL:%.*]] = fcmp une double [[INC]], 0.000000e+00
+; CHECK-NEXT:    br label [[BOGUSBB:%.*]]
+;
+entry:
+  ret void
+
+bogusBB:                                          ; preds = %bogusBB
+  %inc1 = fadd double %inc, 1.000000e+00
+  %inc = fadd double %inc1, 1.000000e+00
+  %tobool = fcmp une double %inc, 0.000000e+00
+  br label %bogusBB
 }
diff --git a/llvm/test/Transforms/FunctionAttrs/arg_returned.ll b/llvm/test/Transforms/FunctionAttrs/arg_returned.ll
index a5699fc1f..0adf91cd9 100644
--- a/llvm/test/Transforms/FunctionAttrs/arg_returned.ll
+++ b/llvm/test/Transforms/FunctionAttrs/arg_returned.ll
@@ -1,6 +1,4 @@
 ; RUN: opt -functionattrs -S < %s | FileCheck %s --check-prefix=FNATTR
-; RUN: opt -attributor -attributor-disable=false -S < %s | FileCheck %s --check-prefix=ATTRIBUTOR
-; RUN: opt -attributor -attributor-disable=false -functionattrs -S < %s | FileCheck %s --check-prefix=BOTH
 ;
 ; Test cases specifically designed for the "returned" argument attribute.
 ; We use FIXME's to indicate problems and missing attributes.
@@ -8,24 +6,12 @@
 
 ; TEST SCC test returning an integer value argument
 ;
-; BOTH: Function Attrs: nofree noinline norecurse nosync nounwind readnone uwtable
-; BOTH-NEXT: define i32 @sink_r0(i32 returned %r)
-; BOTH: Function Attrs: nofree noinline nosync nounwind readnone uwtable
-; BOTH-NEXT: define i32 @scc_r1(i32 %a, i32 returned %r, i32 %b)
-; BOTH: Function Attrs: nofree noinline nosync nounwind readnone uwtable
-; BOTH-NEXT: define i32 @scc_r2(i32 %a, i32 %b, i32 returned %r)
-; BOTH: Function Attrs: nofree noinline nosync nounwind readnone uwtable
-; BOTH-NEXT: define i32 @scc_rX(i32 %a, i32 %b, i32 %r)
 ;
 ; FNATTR: define i32 @sink_r0(i32 returned %r)
 ; FNATTR: define i32 @scc_r1(i32 %a, i32 %r, i32 %b)
 ; FNATTR: define i32 @scc_r2(i32 %a, i32 %b, i32 %r)
 ; FNATTR: define i32 @scc_rX(i32 %a, i32 %b, i32 %r)
 ;
-; ATTRIBUTOR: define i32 @sink_r0(i32 returned %r)
-; ATTRIBUTOR: define i32 @scc_r1(i32 %a, i32 returned %r, i32 %b)
-; ATTRIBUTOR: define i32 @scc_r2(i32 %a, i32 %b, i32 returned %r)
-; ATTRIBUTOR: define i32 @scc_rX(i32 %a, i32 %b, i32 %r)
 ;
 ; int scc_r1(int a, int b, int r);
 ; int scc_r2(int a, int b, int r);
@@ -159,23 +145,10 @@ return:                                           ; preds = %cond.end, %if.then3
 
 ; TEST SCC test returning a pointer value argument
 ;
-; BOTH: Function Attrs: nofree noinline norecurse nosync nounwind readnone uwtable
-; BOTH-NEXT: define double* @ptr_sink_r0(double* readnone returned %r)
-; BOTH: Function Attrs: nofree noinline nosync nounwind readnone uwtable
-; BOTH-NEXT: define double* @ptr_scc_r1(double* %a, double* readnone returned %r, double* nocapture readnone %b)
-; BOTH: Function Attrs: nofree noinline nosync nounwind readnone uwtable
-; BOTH-NEXT: define double* @ptr_scc_r2(double* readnone %a, double* readnone %b, double* readnone returned %r)
-;
 ; FNATTR: define double* @ptr_sink_r0(double* readnone returned %r)
 ; FNATTR: define double* @ptr_scc_r1(double* %a, double* readnone %r, double* nocapture readnone %b)
 ; FNATTR: define double* @ptr_scc_r2(double* readnone %a, double* readnone %b, double* readnone %r)
 ;
-; ATTRIBUTOR: Function Attrs: nofree noinline nosync nounwind uwtable
-; ATTRIBUTOR-NEXT: define double* @ptr_sink_r0(double* returned %r)
-; ATTRIBUTOR: Function Attrs: nofree noinline nosync nounwind uwtable
-; ATTRIBUTOR-NEXT: define double* @ptr_scc_r1(double* %a, double* returned %r, double* %b)
-; ATTRIBUTOR: Function Attrs: nofree noinline nosync nounwind uwtable
-; ATTRIBUTOR-NEXT: define double* @ptr_scc_r2(double* %a, double* %b, double* returned %r)
 ;
 ; double* ptr_scc_r1(double* a, double* b, double* r);
 ; double* ptr_scc_r2(double* a, double* b, double* r);
@@ -259,10 +232,7 @@ return:                                           ; preds = %cond.end, %if.then3
 ;   return *a ? a : rt0(a);
 ; }
 ;
-; FIXME: no-return missing
 ; FNATTR:  define i32* @rt0(i32* readonly %a)
-; BOTH: Function Attrs: nofree noinline nosync nounwind readonly uwtable
-; BOTH-NEXT:    define i32* @rt0(i32* readonly returned %a)
 define i32* @rt0(i32* %a) #0 {
 entry:
   %v = load i32, i32* %a, align 4
@@ -278,10 +248,7 @@ entry:
 ;   return *a ? undef : rt1(a);
 ; }
 ;
-; FIXME: no-return missing
 ; FNATTR:  define noalias i32* @rt1(i32* nocapture readonly %a)
-; BOTH: Function Attrs: nofree noinline nosync nounwind readonly uwtable
-; BOTH-NEXT:    define noalias i32* @rt1(i32* nocapture readonly %a)
 define i32* @rt1(i32* %a) #0 {
 entry:
   %v = load i32, i32* %a, align 4
@@ -295,8 +262,6 @@ entry:
 ;
 ; FNATTR:  define i32* @rt2_helper(i32* %a)
 ; FNATTR:  define i32* @rt2(i32* readnone %a, i32* readnone %b)
-; BOTH:    define i32* @rt2_helper(i32* %a)
-; BOTH:    define i32* @rt2(i32* readnone %a, i32* readnone %b)
 define i32* @rt2_helper(i32* %a) #0 {
 entry:
   %call = call i32* @rt2(i32* %a, i32* %a)
@@ -321,8 +286,6 @@ if.end:
 ;
 ; FNATTR:  define i32* @rt3_helper(i32* %a, i32* %b)
 ; FNATTR:  define i32* @rt3(i32* readnone %a, i32* readnone %b)
-; BOTH:    define i32* @rt3_helper(i32* %a, i32* returned %b)
-; BOTH:    define i32* @rt3(i32* readnone %a, i32* readnone returned %b)
 define i32* @rt3_helper(i32* %a, i32* %b) #0 {
 entry:
   %call = call i32* @rt3(i32* %a, i32* %b)
@@ -352,12 +315,8 @@ if.end:
 ;    return r;
 ;  }
 ;
-; BOTH: declare void @unknown_fn(i32* (i32*)*)
 ;
-; BOTH:       Function Attrs: noinline nounwind uwtable
-; BOTH-NEXT:  define i32* @calls_unknown_fn(i32* readnone returned %r)
 ; FNATTR:     define i32* @calls_unknown_fn(i32* readnone returned %r)
-; ATTRIBUTOR: define i32* @calls_unknown_fn(i32* returned %r)
 declare void @unknown_fn(i32* (i32*)*) #0
 
 define i32* @calls_unknown_fn(i32* %r) #0 {
@@ -366,41 +325,6 @@ define i32* @calls_unknown_fn(i32* %r) #0 {
 }
 
 
-; TEST call to a function that might be redifined at link time
-;
-;  int *maybe_redefined_fn(int *r) {
-;    return r;
-;  }
-;
-;  int *calls_maybe_redefined_fn(int *r) {
-;    maybe_redefined_fn(r);
-;    return r;
-;  }
-;
-; Verify the maybe-redefined function is not annotated:
-;
-; CHECK: Function Attrs: noinline nounwind uwtable
-; CHECK: define linkonce_odr i32* @maybe_redefined_fn(i32* %r)
-;
-; CHECK: Function Attrs: noinline nounwind uwtable
-; CHECK: define i32* @calls_maybe_redefined_fn(i32* returned %r)
-;
-; BOTH: Function Attrs: noinline nounwind uwtable
-; BOTH-NEXT: define linkonce_odr i32* @maybe_redefined_fn(i32* %r)
-;
-; BOTH: Function Attrs: noinline nounwind uwtable
-; BOTH-NEXT: define i32* @calls_maybe_redefined_fn(i32* returned %r)
-define linkonce_odr i32* @maybe_redefined_fn(i32* %r) #0 {
-entry:
-  ret i32* %r
-}
-
-define i32* @calls_maybe_redefined_fn(i32* %r) #0 {
-entry:
-  %call = call i32* @maybe_redefined_fn(i32* %r)
-  ret i32* %r
-}
-
 ; TEST return call to a function that might be redifined at link time
 ;
 ;  int *maybe_redefined_fn2(int *r) {
@@ -413,13 +337,8 @@ entry:
 ;
 ; Verify the maybe-redefined function is not annotated:
 ;
-; BOTH: Function Attrs: noinline nounwind uwtable
-; BOTH-NEXT: define linkonce_odr i32* @maybe_redefined_fn2(i32* %r)
-; BOTH: Function Attrs: noinline nounwind uwtable
-; BOTH-NEXT: define i32* @calls_maybe_redefined_fn2(i32* %r)
 ;
 ; FNATTR:     define i32* @calls_maybe_redefined_fn2(i32* %r)
-; ATTRIBUTOR: define i32* @calls_maybe_redefined_fn2(i32* %r)
 define linkonce_odr i32* @maybe_redefined_fn2(i32* %r) #0 {
 entry:
   ret i32* %r
@@ -441,12 +360,8 @@ entry:
 ;   return b == 0? b : x;
 ; }
 ;
-; BOTH: Function Attrs: nofree noinline norecurse nosync nounwind readnone uwtable
-; BOTH-NEXT: define double @select_and_phi(double returned %b)
 ;
 ; FNATTR:     define double @select_and_phi(double %b)
-; ATTRIBUTOR: Function Attrs: nofree noinline nosync nounwind uwtable
-; ATTRIBUTOR-NEXT: define double @select_and_phi(double returned %b)
 define double @select_and_phi(double %b) #0 {
 entry:
   %cmp = fcmp ogt double %b, 0.000000e+00
@@ -472,13 +387,9 @@ if.end:                                           ; preds = %if.then, %entry
 ;   return b == 0? b : x;
 ; }
 ;
-; BOTH: Function Attrs: nofree noinline nosync nounwind readnone uwtable
-; BOTH-NEXT: define double @recursion_select_and_phi(i32 %a, double returned %b)
 ;
 ; FNATTR:     define double @recursion_select_and_phi(i32 %a, double %b)
 ;
-; ATTRIBUTOR: Function Attrs: nofree noinline nosync nounwind uwtable
-; ATTRIBUTOR-NEXT: define double @recursion_select_and_phi(i32 %a, double returned %b)
 define double @recursion_select_and_phi(i32 %a, double %b) #0 {
 entry:
   %dec = add nsw i32 %a, -1
@@ -503,13 +414,9 @@ if.end:                                           ; preds = %if.then, %entry
 ;   return (double*)b;
 ; }
 ;
-; BOTH: Function Attrs: nofree noinline norecurse nosync nounwind readnone uwtable
-; BOTH-NEXT:  define double* @bitcast(i32* readnone returned %b)
 ;
 ; FNATTR:     define double* @bitcast(i32* readnone %b)
 ;
-; ATTRIBUTOR: Function Attrs: nofree noinline nosync nounwind uwtable
-; ATTRIBUTOR-NEXT: define double* @bitcast(i32* returned %b)
 define double* @bitcast(i32* %b) #0 {
 entry:
   %bc0 = bitcast i32* %b to double*
@@ -526,13 +433,9 @@ entry:
 ;   return b != 0 ? b : x;
 ; }
 ;
-; BOTH: Function Attrs: nofree noinline norecurse nosync nounwind readnone uwtable
-; BOTH-NEXT: define double* @bitcasts_select_and_phi(i32* readnone returned %b)
 ;
 ; FNATTR:     define double* @bitcasts_select_and_phi(i32* readnone %b)
 ;
-; ATTRIBUTOR: Function Attrs: nofree noinline nosync nounwind uwtable
-; ATTRIBUTOR-NEXT: define double* @bitcasts_select_and_phi(i32* returned %b)
 define double* @bitcasts_select_and_phi(i32* %b) #0 {
 entry:
   %bc0 = bitcast i32* %b to double*
@@ -564,13 +467,9 @@ if.end:                                           ; preds = %if.then, %entry
 ;   /* return undef */
 ; }
 ;
-; BOTH: Function Attrs: nofree noinline norecurse nosync nounwind readnone uwtable
-; BOTH-NEXT:  define double* @ret_arg_arg_undef(i32* readnone returned %b)
 ;
 ; FNATTR:     define double* @ret_arg_arg_undef(i32* readnone %b)
 ;
-; ATTRIBUTOR: Function Attrs: nofree noinline nosync nounwind uwtable
-; ATTRIBUTOR-NEXT: define double* @ret_arg_arg_undef(i32* returned %b)
 define double* @ret_arg_arg_undef(i32* %b) #0 {
 entry:
   %bc0 = bitcast i32* %b to double*
@@ -602,13 +501,9 @@ ret_undef:
 ;   /* return undef */
 ; }
 ;
-; BOTH: Function Attrs: nofree noinline norecurse nosync nounwind readnone uwtable
-; BOTH-NEXT:  define double* @ret_undef_arg_arg(i32* readnone returned %b)
 ;
 ; FNATTR:     define double* @ret_undef_arg_arg(i32* readnone %b)
 ;
-; ATTRIBUTOR: Function Attrs: nofree noinline nosync nounwind uwtable
-; ATTRIBUTOR-NEXT: define double* @ret_undef_arg_arg(i32* returned %b)
 define double* @ret_undef_arg_arg(i32* %b) #0 {
 entry:
   %bc0 = bitcast i32* %b to double*
@@ -640,11 +535,8 @@ ret_arg1:
 ;   /* return undef */
 ; }
 ;
-; BOTH: Function Attrs: nofree noinline norecurse nosync nounwind readnone uwtable
-; BOTH-NEXT:  define double* @ret_undef_arg_undef(i32* readnone returned %b)
 ;
 ; FNATTR:     define double* @ret_undef_arg_undef(i32* readnone %b)
-; ATTRIBUTOR: define double* @ret_undef_arg_undef(i32* returned %b)
 define double* @ret_undef_arg_undef(i32* %b) #0 {
 entry:
   %bc0 = bitcast i32* %b to double*
@@ -676,10 +568,6 @@ ret_undef1:
 ;
 ; FNATTR:     define i32* @ret_arg_or_unknown(i32* %b)
 ; FNATTR:     define i32* @ret_arg_or_unknown_through_phi(i32* %b)
-; ATTRIBUTOR: define i32* @ret_arg_or_unknown(i32* %b)
-; ATTRIBUTOR: define i32* @ret_arg_or_unknown_through_phi(i32* %b)
-; BOTH:       define i32* @ret_arg_or_unknown(i32* %b)
-; BOTH:       define i32* @ret_arg_or_unknown_through_phi(i32* %b)
 declare i32* @unknown(i32*)
 
 define i32* @ret_arg_or_unknown(i32* %b) #0 {
@@ -716,10 +604,8 @@ r:
 ;
 ; FNATTR:     define i32 @deadblockcall1(i32 %A)
 ; FNATTR:     define i32 @deadblockcall2(i32 %A)
-; ATTRIBUTOR: define i32 @deadblockcall1(i32 returned %A)
-; ATTRIBUTOR: define i32 @deadblockcall2(i32 returned %A)
-; BOTH:       define i32 @deadblockcall1(i32 returned %A)
-; BOTH:       define i32 @deadblockcall2(i32 returned %A)
+; FNATTR:     define i32 @deadblockphi1(i32 %A)
+; FNATTR:     define i32 @deadblockphi2(i32 %A)
 define i32 @deadblockcall1(i32 %A) #0 {
 entry:
   ret i32 %A
@@ -741,11 +627,55 @@ unreachableblock2:
   ret i32 %C
 }
 
-attributes #0 = { noinline nounwind uwtable }
+define i32 @deadblockphi1(i32 %A) #0 {
+entry:
+  br label %r
+unreachableblock1:
+  %B = call i32 @deadblockcall_helper(i32 %B)
+  ret i32 %B
+unreachableblock2:
+  %C = call i32 @deadblockcall1(i32 %C)
+  br label %r
+r:
+  %PHI = phi i32 [%A, %entry], [%C, %unreachableblock2]
+  ret i32 %PHI
+}
 
-; BOTH-NOT: attributes #
-; BOTH-DAG: attributes #{{[0-9]*}} = { nofree noinline norecurse nosync nounwind readnone uwtable willreturn }
-; BOTH-DAG: attributes #{{[0-9]*}} = { nofree noinline nosync nounwind readnone uwtable }
-; BOTH-DAG: attributes #{{[0-9]*}} = { nofree noinline nosync nounwind readonly uwtable }
-; BOTH-DAG: attributes #{{[0-9]*}} = { noinline nounwind uwtable }
-; BOTH-NOT: attributes #
+define i32 @deadblockphi2(i32 %A) #0 {
+entry:
+  br label %r
+unreachableblock1:
+  %B = call i32 @deadblockcall_helper(i32 %B)
+  br label %unreachableblock3
+unreachableblock2:
+  %C = call i32 @deadblockcall1(i32 %C)
+  br label %unreachableblock3
+unreachableblock3:
+  %PHI1 = phi i32 [%B, %unreachableblock1], [%C, %unreachableblock2]
+  br label %r
+r:
+  %PHI2 = phi i32 [%A, %entry], [%PHI1, %unreachableblock3]
+  ret i32 %PHI2
+}
+
+declare void @noreturn() noreturn;
+
+define i32 @deadblockphi3(i32 %A, i1 %c) #0 {
+entry:
+  br i1 %c, label %r, label %unreachablecall
+unreachablecall:
+  call void @noreturn();
+  %B = call i32 @deadblockcall_helper(i32 0)
+  br label %unreachableblock3
+unreachableblock2:
+  %C = call i32 @deadblockcall1(i32 %C)
+  br label %unreachableblock3
+unreachableblock3:
+  %PHI1 = phi i32 [%B, %unreachablecall], [%C, %unreachableblock2]
+  br label %r
+r:
+  %PHI2 = phi i32 [%A, %entry], [%PHI1, %unreachableblock3]
+  ret i32 %PHI2
+}
+
+attributes #0 = { noinline nounwind uwtable }
diff --git a/llvm/test/Transforms/FunctionAttrs/comdat-ipo.ll b/llvm/test/Transforms/FunctionAttrs/comdat-ipo.ll
index 2a149e436..d2f194fac 100644
--- a/llvm/test/Transforms/FunctionAttrs/comdat-ipo.ll
+++ b/llvm/test/Transforms/FunctionAttrs/comdat-ipo.ll
@@ -3,14 +3,14 @@
 
 ; See PR26774
 
-; CHECK-LABEL: define void @bar(i8* readonly) {
+; CHECK-LABEL: define void @bar(i8* readonly %0) {
 define void @bar(i8* readonly) {
   call void @foo(i8* %0)
   ret void
 }
 
 
-; CHECK-LABEL: define linkonce_odr void @foo(i8* readonly) {
+; CHECK-LABEL: define linkonce_odr void @foo(i8* readonly %0) {
 define linkonce_odr void @foo(i8* readonly) {
   call void @bar(i8* %0)
   ret void
diff --git a/llvm/test/Transforms/FunctionAttrs/naked_functions.ll b/llvm/test/Transforms/FunctionAttrs/naked_functions.ll
index d34dc0c20..c4996d4e7 100644
--- a/llvm/test/Transforms/FunctionAttrs/naked_functions.ll
+++ b/llvm/test/Transforms/FunctionAttrs/naked_functions.ll
@@ -20,6 +20,6 @@ entry:
   unreachable
 }
 
-; CHECK: define internal i32 @foo(i32*)
+; CHECK: define internal i32 @foo(i32* %0)
 
 attributes #0 = { naked }
diff --git a/llvm/test/Transforms/FunctionAttrs/nocapture.ll b/llvm/test/Transforms/FunctionAttrs/nocapture.ll
index b10924967..25759cffe 100644
--- a/llvm/test/Transforms/FunctionAttrs/nocapture.ll
+++ b/llvm/test/Transforms/FunctionAttrs/nocapture.ll
@@ -1,27 +1,27 @@
-; RUN: opt < %s -functionattrs -S | FileCheck %s
-; RUN: opt < %s -passes=function-attrs -S | FileCheck %s
+; RUN: opt -functionattrs -S < %s | FileCheck %s --check-prefixes=FNATTR
+; RUN: opt -passes=function-attrs -S < %s | FileCheck %s --check-prefixes=FNATTR
 
 @g = global i32* null		; <i32**> [#uses=1]
 
-; CHECK: define i32* @c1(i32* readnone returned %q)
+; FNATTR: define i32* @c1(i32* readnone returned %q)
 define i32* @c1(i32* %q) {
 	ret i32* %q
 }
 
-; CHECK: define void @c2(i32* %q)
+; FNATTR: define void @c2(i32* %q)
 ; It would also be acceptable to mark %q as readnone. Update @c3 too.
 define void @c2(i32* %q) {
 	store i32* %q, i32** @g
 	ret void
 }
 
-; CHECK: define void @c3(i32* %q)
+; FNATTR: define void @c3(i32* %q)
 define void @c3(i32* %q) {
 	call void @c2(i32* %q)
 	ret void
 }
 
-; CHECK: define i1 @c4(i32* %q, i32 %bitno)
+; FNATTR: define i1 @c4(i32* %q, i32 %bitno)
 define i1 @c4(i32* %q, i32 %bitno) {
 	%tmp = ptrtoint i32* %q to i32
 	%tmp2 = lshr i32 %tmp, %bitno
@@ -33,9 +33,22 @@ l1:
 	ret i1 1 ; escaping value not caught by def-use chaining.
 }
 
+; c4b is c4 but without the escaping part
+; FNATTR: define i1 @c4b(i32* %q, i32 %bitno)
+define i1 @c4b(i32* %q, i32 %bitno) {
+	%tmp = ptrtoint i32* %q to i32
+	%tmp2 = lshr i32 %tmp, %bitno
+	%bit = trunc i32 %tmp2 to i1
+	br i1 %bit, label %l1, label %l0
+l0:
+	ret i1 0 ; not escaping!
+l1:
+	ret i1 0 ; not escaping!
+}
+
 @lookup_table = global [2 x i1] [ i1 0, i1 1 ]
 
-; CHECK: define i1 @c5(i32* %q, i32 %bitno)
+; FNATTR: define i1 @c5(i32* %q, i32 %bitno)
 define i1 @c5(i32* %q, i32 %bitno) {
 	%tmp = ptrtoint i32* %q to i32
 	%tmp2 = lshr i32 %tmp, %bitno
@@ -48,7 +61,7 @@ define i1 @c5(i32* %q, i32 %bitno) {
 
 declare void @throw_if_bit_set(i8*, i8) readonly
 
-; CHECK: define i1 @c6(i8* readonly %q, i8 %bit)
+; FNATTR: define i1 @c6(i8* readonly %q, i8 %bit)
 define i1 @c6(i8* %q, i8 %bit) personality i32 (...)* @__gxx_personality_v0 {
 	invoke void @throw_if_bit_set(i8* %q, i8 %bit)
 		to label %ret0 unwind label %ret1
@@ -70,7 +83,7 @@ define i1* @lookup_bit(i32* %q, i32 %bitno) readnone nounwind {
 	ret i1* %lookup
 }
 
-; CHECK: define i1 @c7(i32* readonly %q, i32 %bitno)
+; FNATTR: define i1 @c7(i32* readonly %q, i32 %bitno)
 define i1 @c7(i32* %q, i32 %bitno) {
 	%ptr = call i1* @lookup_bit(i32* %q, i32 %bitno)
 	%val = load i1, i1* %ptr
@@ -78,7 +91,7 @@ define i1 @c7(i32* %q, i32 %bitno) {
 }
 
 
-; CHECK: define i32 @nc1(i32* %q, i32* nocapture %p, i1 %b)
+; FNATTR: define i32 @nc1(i32* %q, i32* nocapture %p, i1 %b)
 define i32 @nc1(i32* %q, i32* %p, i1 %b) {
 e:
 	br label %l
@@ -93,7 +106,7 @@ l:
 	ret i32 %val
 }
 
-; CHECK: define i32 @nc1_addrspace(i32* %q, i32 addrspace(1)* nocapture %p, i1 %b)
+; FNATTR: define i32 @nc1_addrspace(i32* %q, i32 addrspace(1)* nocapture %p, i1 %b)
 define i32 @nc1_addrspace(i32* %q, i32 addrspace(1)* %p, i1 %b) {
 e:
 	br label %l
@@ -108,78 +121,87 @@ l:
 	ret i32 %val
 }
 
-; CHECK: define void @nc2(i32* nocapture %p, i32* %q)
+; FNATTR: define void @nc2(i32* nocapture %p, i32* %q)
 define void @nc2(i32* %p, i32* %q) {
 	%1 = call i32 @nc1(i32* %q, i32* %p, i1 0)		; <i32> [#uses=0]
 	ret void
 }
 
-; CHECK: define void @nc3(void ()* nocapture %p)
+
+; FNATTR: define void @nc3(void ()* nocapture %p)
 define void @nc3(void ()* %p) {
 	call void %p()
 	ret void
 }
 
 declare void @external(i8*) readonly nounwind
-; CHECK: define void @nc4(i8* nocapture readonly %p)
+; FNATTR: define void @nc4(i8* nocapture readonly %p)
 define void @nc4(i8* %p) {
 	call void @external(i8* %p)
 	ret void
 }
 
-; CHECK: define void @nc5(void (i8*)* nocapture %f, i8* nocapture %p)
+; FNATTR: define void @nc5(void (i8*)* nocapture %f, i8* nocapture %p)
 define void @nc5(void (i8*)* %f, i8* %p) {
 	call void %f(i8* %p) readonly nounwind
 	call void %f(i8* nocapture %p)
 	ret void
 }
 
-; CHECK: define void @test1_1(i8* nocapture readnone %x1_1, i8* %y1_1)
+; FNATTR:     define void @test1_1(i8* nocapture readnone %x1_1, i8* %y1_1, i1 %c)
 ; It would be acceptable to add readnone to %y1_1 and %y1_2.
-define void @test1_1(i8* %x1_1, i8* %y1_1) {
-  call i8* @test1_2(i8* %x1_1, i8* %y1_1)
+define void @test1_1(i8* %x1_1, i8* %y1_1, i1 %c) {
+  call i8* @test1_2(i8* %x1_1, i8* %y1_1, i1 %c)
   store i32* null, i32** @g
   ret void
 }
 
-; CHECK: define i8* @test1_2(i8* nocapture readnone %x1_2, i8* returned %y1_2)
-define i8* @test1_2(i8* %x1_2, i8* %y1_2) {
-  call void @test1_1(i8* %x1_2, i8* %y1_2)
+; FNATTR: define i8* @test1_2(i8* nocapture readnone %x1_2, i8* returned %y1_2, i1 %c)
+define i8* @test1_2(i8* %x1_2, i8* %y1_2, i1 %c) {
+  br i1 %c, label %t, label %f
+t:
+  call void @test1_1(i8* %x1_2, i8* %y1_2, i1 %c)
   store i32* null, i32** @g
+  br label %f
+f:
   ret i8* %y1_2
 }
 
-; CHECK: define void @test2(i8* nocapture readnone %x2)
+; FNATTR: define void @test2(i8* nocapture readnone %x2)
 define void @test2(i8* %x2) {
   call void @test2(i8* %x2)
   store i32* null, i32** @g
   ret void
 }
 
-; CHECK: define void @test3(i8* nocapture readnone %x3, i8* nocapture readnone %y3, i8* nocapture readnone %z3)
+; FNATTR: define void @test3(i8* nocapture readnone %x3, i8* nocapture readnone %y3, i8* nocapture readnone %z3)
 define void @test3(i8* %x3, i8* %y3, i8* %z3) {
   call void @test3(i8* %z3, i8* %y3, i8* %x3)
   store i32* null, i32** @g
   ret void
 }
 
-; CHECK: define void @test4_1(i8* %x4_1)
-define void @test4_1(i8* %x4_1) {
-  call i8* @test4_2(i8* %x4_1, i8* %x4_1, i8* %x4_1)
+; FNATTR: define void @test4_1(i8* %x4_1, i1 %c)
+define void @test4_1(i8* %x4_1, i1 %c) {
+  call i8* @test4_2(i8* %x4_1, i8* %x4_1, i8* %x4_1, i1 %c)
   store i32* null, i32** @g
   ret void
 }
 
-; CHECK: define i8* @test4_2(i8* nocapture readnone %x4_2, i8* readnone returned %y4_2, i8* nocapture readnone %z4_2)
-define i8* @test4_2(i8* %x4_2, i8* %y4_2, i8* %z4_2) {
-  call void @test4_1(i8* null)
+; FNATTR: define i8* @test4_2(i8* nocapture readnone %x4_2, i8* readnone returned %y4_2, i8* nocapture readnone %z4_2, i1 %c)
+define i8* @test4_2(i8* %x4_2, i8* %y4_2, i8* %z4_2, i1 %c) {
+  br i1 %c, label %t, label %f
+t:
+  call void @test4_1(i8* null, i1 %c)
   store i32* null, i32** @g
+  br label %f
+f:
   ret i8* %y4_2
 }
 
 declare i8* @test5_1(i8* %x5_1)
 
-; CHECK: define void @test5_2(i8* %x5_2)
+; FNATTR: define void @test5_2(i8* %x5_2)
 define void @test5_2(i8* %x5_2) {
   call i8* @test5_1(i8* %x5_2)
   store i32* null, i32** @g
@@ -188,32 +210,32 @@ define void @test5_2(i8* %x5_2) {
 
 declare void @test6_1(i8* %x6_1, i8* nocapture %y6_1, ...)
 
-; CHECK: define void @test6_2(i8* %x6_2, i8* nocapture %y6_2, i8* %z6_2)
+; FNATTR: define void @test6_2(i8* %x6_2, i8* nocapture %y6_2, i8* %z6_2)
 define void @test6_2(i8* %x6_2, i8* %y6_2, i8* %z6_2) {
   call void (i8*, i8*, ...) @test6_1(i8* %x6_2, i8* %y6_2, i8* %z6_2)
   store i32* null, i32** @g
   ret void
 }
 
-; CHECK: define void @test_cmpxchg(i32* nocapture %p)
+; FNATTR: define void @test_cmpxchg(i32* nocapture %p)
 define void @test_cmpxchg(i32* %p) {
   cmpxchg i32* %p, i32 0, i32 1 acquire monotonic
   ret void
 }
 
-; CHECK: define void @test_cmpxchg_ptr(i32** nocapture %p, i32* %q)
+; FNATTR: define void @test_cmpxchg_ptr(i32** nocapture %p, i32* %q)
 define void @test_cmpxchg_ptr(i32** %p, i32* %q) {
   cmpxchg i32** %p, i32* null, i32* %q acquire monotonic
   ret void
 }
 
-; CHECK: define void @test_atomicrmw(i32* nocapture %p)
+; FNATTR: define void @test_atomicrmw(i32* nocapture %p)
 define void @test_atomicrmw(i32* %p) {
   atomicrmw add i32* %p, i32 1 seq_cst
   ret void
 }
 
-; CHECK: define void @test_volatile(i32* %x)
+; FNATTR: define void @test_volatile(i32* %x)
 define void @test_volatile(i32* %x) {
 entry:
   %gep = getelementptr i32, i32* %x, i64 1
@@ -221,7 +243,7 @@ entry:
   ret void
 }
 
-; CHECK: nocaptureLaunder(i8* nocapture %p)
+; FNATTR: nocaptureLaunder(i8* nocapture %p)
 define void @nocaptureLaunder(i8* %p) {
 entry:
   %b = call i8* @llvm.launder.invariant.group.p0i8(i8* %p)
@@ -230,14 +252,14 @@ entry:
 }
 
 @g2 = global i8* null
-; CHECK: define void @captureLaunder(i8* %p)
+; FNATTR: define void @captureLaunder(i8* %p)
 define void @captureLaunder(i8* %p) {
   %b = call i8* @llvm.launder.invariant.group.p0i8(i8* %p)
   store i8* %b, i8** @g2
   ret void
 }
 
-; CHECK: @nocaptureStrip(i8* nocapture %p)
+; FNATTR: @nocaptureStrip(i8* nocapture %p)
 define void @nocaptureStrip(i8* %p) {
 entry:
   %b = call i8* @llvm.strip.invariant.group.p0i8(i8* %p)
@@ -246,20 +268,26 @@ entry:
 }
 
 @g3 = global i8* null
-; CHECK: define void @captureStrip(i8* %p)
+; FNATTR: define void @captureStrip(i8* %p)
 define void @captureStrip(i8* %p) {
   %b = call i8* @llvm.strip.invariant.group.p0i8(i8* %p)
   store i8* %b, i8** @g3
   ret void
 }
 
-; CHECK: define i1 @captureICmp(i32* readnone %x)
+; FNATTR: define i1 @captureICmp(i32* readnone %x)
 define i1 @captureICmp(i32* %x) {
   %1 = icmp eq i32* %x, null
   ret i1 %1
 }
 
-; CHECK: define i1 @nocaptureInboundsGEPICmp(i32* nocapture readnone %x)
+; FNATTR: define i1 @captureICmpRev(i32* readnone %x)
+define i1 @captureICmpRev(i32* %x) {
+  %1 = icmp eq i32* null, %x
+  ret i1 %1
+}
+
+; FNATTR: define i1 @nocaptureInboundsGEPICmp(i32* nocapture readnone %x)
 define i1 @nocaptureInboundsGEPICmp(i32* %x) {
   %1 = getelementptr inbounds i32, i32* %x, i32 5
   %2 = bitcast i32* %1 to i8*
@@ -267,14 +295,22 @@ define i1 @nocaptureInboundsGEPICmp(i32* %x) {
   ret i1 %3
 }
 
-; CHECK: define i1 @nocaptureDereferenceableOrNullICmp(i32* nocapture readnone dereferenceable_or_null(4) %x)
+; FNATTR: define i1 @nocaptureInboundsGEPICmpRev(i32* nocapture readnone %x)
+define i1 @nocaptureInboundsGEPICmpRev(i32* %x) {
+  %1 = getelementptr inbounds i32, i32* %x, i32 5
+  %2 = bitcast i32* %1 to i8*
+  %3 = icmp eq i8* null, %2
+  ret i1 %3
+}
+
+; FNATTR: define i1 @nocaptureDereferenceableOrNullICmp(i32* nocapture readnone dereferenceable_or_null(4) %x)
 define i1 @nocaptureDereferenceableOrNullICmp(i32* dereferenceable_or_null(4) %x) {
   %1 = bitcast i32* %x to i8*
   %2 = icmp eq i8* %1, null
   ret i1 %2
 }
 
-; CHECK: define i1 @captureDereferenceableOrNullICmp(i32* readnone dereferenceable_or_null(4) %x)
+; FNATTR: define i1 @captureDereferenceableOrNullICmp(i32* readnone dereferenceable_or_null(4) %x)
 define i1 @captureDereferenceableOrNullICmp(i32* dereferenceable_or_null(4) %x) "null-pointer-is-valid"="true" {
   %1 = bitcast i32* %x to i8*
   %2 = icmp eq i8* %1, null
diff --git a/llvm/test/Transforms/FunctionAttrs/nofree-attributor.ll b/llvm/test/Transforms/FunctionAttrs/nofree-attributor.ll
index 16364b5a5..8ac037e5c 100644
--- a/llvm/test/Transforms/FunctionAttrs/nofree-attributor.ll
+++ b/llvm/test/Transforms/FunctionAttrs/nofree-attributor.ll
@@ -1,5 +1,4 @@
 ; RUN: opt -functionattrs --disable-nofree-inference=false -S < %s | FileCheck %s --check-prefix=FNATTR
-; RUN: opt -attributor --attributor-disable=false -S < %s | FileCheck %s --check-prefix=ATTRIBUTOR
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
@@ -15,8 +14,6 @@ declare void @_ZdaPv(i8*) local_unnamed_addr #2
 ; TEST 1 (positive case)
 ; FNATTR: Function Attrs: noinline norecurse nounwind readnone uwtable
 ; FNATTR-NEXT: define void @only_return()
-; ATTRIBUTOR: Function Attrs: nofree noinline nosync nounwind uwtable
-; ATTRIBUTOR-NEXT: define void @only_return()
 define void @only_return() #0 {
     ret void
 }
@@ -29,11 +26,8 @@ define void @only_return() #0 {
 ; }
 
 ; FNATTR: Function Attrs: noinline nounwind uwtable
-; FNATTR-NEXT: define void @only_free(i8* nocapture) local_unnamed_addr
-; ATTRIBUTOR: Function Attrs: noinline nounwind uwtable
-; ATTRIBUTOR-NOT: nofree
-; ATTRIBUTOR-NEXT: define void @only_free(i8* nocapture) local_unnamed_addr #1
-define void @only_free(i8* nocapture) local_unnamed_addr #0 {
+; FNATTR-NEXT: define void @only_free(i8* nocapture %0) local_unnamed_addr
+define void @only_free(i8* nocapture %0) local_unnamed_addr #0 {
     tail call void @free(i8* %0) #1
     ret void
 }
@@ -51,24 +45,25 @@ define void @only_free(i8* nocapture) local_unnamed_addr #0 {
 
 
 ; FNATTR: Function Attrs: noinline nounwind uwtable
-; FNATTR-NEXT: define void @free_in_scc1(i8* nocapture) local_unnamed_addr
-; ATTRIBUTOR: Function Attrs: noinline nounwind uwtable
-; ATTRIBUTOR-NOT: nofree
-; ATTRIBUTOR-NEXT :define void @free_in_scc1(i8* nocapture) local_unnamed_addr
-define void @free_in_scc1(i8* nocapture) local_unnamed_addr #0 {
+; FNATTR-NEXT: define void @free_in_scc1(i8* nocapture %0) local_unnamed_addr
+define void @free_in_scc1(i8* nocapture %0) local_unnamed_addr #0 {
   tail call void @free_in_scc2(i8* %0) #1
   ret void
 }
 
 
 ; FNATTR: Function Attrs: noinline nounwind uwtable
-; FNATTR-NEXT: define void @free_in_scc2(i8* nocapture) local_unnamed_addr
-; ATTRIBUTOR: Function Attrs: noinline nounwind uwtable
-; ATTRIBUTOR-NOT: nofree
-; ATTRIBUTOR: define void @free_in_scc2(i8* nocapture) local_unnamed_addr
-define void @free_in_scc2(i8* nocapture) local_unnamed_addr #0 {
-  tail call void @free_in_scc1(i8* %0)
+; FNATTR-NEXT: define void @free_in_scc2(i8* nocapture %0) local_unnamed_addr
+define void @free_in_scc2(i8* nocapture %0) local_unnamed_addr #0 {
+  %cmp = icmp eq i8* %0, null
+  br i1 %cmp, label %rec, label %call
+call:
   tail call void @free(i8* %0) #1
+  br label %end
+rec:
+  tail call void @free_in_scc1(i8* %0)
+  br label %end
+end:
   ret void
 }
 
@@ -85,8 +80,6 @@ define void @free_in_scc2(i8* nocapture) local_unnamed_addr #0 {
 
 ; FNATTR: Function Attrs: noinline nounwind readnone uwtable
 ; FNATTR-NEXT: define void @mutual_recursion1()
-; ATTRIBUTOR: Function Attrs: nofree noinline nosync nounwind uwtable
-; ATTRIBUTOR-NEXT: define void @mutual_recursion1()
 define void @mutual_recursion1() #0 {
   call void @mutual_recursion2()
   ret void
@@ -94,8 +87,6 @@ define void @mutual_recursion1() #0 {
 
 ; FNATTR: Function Attrs: noinline nounwind readnone uwtable
 ; FNATTR-NEXT: define void @mutual_recursion2()
-; ATTRIBUTOR: Function Attrs: nofree noinline nosync nounwind uwtable
-; ATTRIBUTOR-NEXT: define void @mutual_recursion2()
 define void @mutual_recursion2() #0 {
   call void @mutual_recursion1()
   ret void
@@ -109,11 +100,8 @@ define void @mutual_recursion2() #0 {
 ; }
 
 ; FNATTR: Function Attrs: noinline nounwind uwtable
-; FNATTR-NEXT: define void @_Z9delete_opPc(i8*) local_unnamed_addr
-; ATTRIBUTOR: Function Attrs: noinline nounwind uwtable
-; ATTRIBUTOR-NOT: nofree
-; ATTRIBUTOR-NEXT: define void @_Z9delete_opPc(i8*) local_unnamed_addr #1
-define void @_Z9delete_opPc(i8*) local_unnamed_addr #0 {
+; FNATTR-NEXT: define void @_Z9delete_opPc(i8* %0) local_unnamed_addr
+define void @_Z9delete_opPc(i8* %0) local_unnamed_addr #0 {
   %2 = icmp eq i8* %0, null
   br i1 %2, label %4, label %3
 
@@ -129,11 +117,8 @@ define void @_Z9delete_opPc(i8*) local_unnamed_addr #0 {
 ; TEST 6 (negative case)
 ; Call realloc
 ; FNATTR: Function Attrs: noinline nounwind uwtable
-; FNATTR-NEXT: define noalias i8* @call_realloc(i8* nocapture, i64) local_unnamed_addr
-; ATTRIBUTOR: Function Attrs: noinline nounwind uwtable
-; ATTRIBUTOR-NOT: nofree
-; ATTRIBUTOR-NEXT: define noalias i8* @call_realloc(i8* nocapture, i64) local_unnamed_addr
-define noalias i8* @call_realloc(i8* nocapture, i64) local_unnamed_addr #0 {
+; FNATTR-NEXT: define noalias i8* @call_realloc(i8* nocapture %0, i64 %1) local_unnamed_addr
+define noalias i8* @call_realloc(i8* nocapture %0, i64 %1) local_unnamed_addr #0 {
     %ret = tail call i8* @realloc(i8* %0, i64 %1) #2
     ret i8* %ret
 }
@@ -145,14 +130,10 @@ define noalias i8* @call_realloc(i8* nocapture, i64) local_unnamed_addr #0 {
 
 ; FNATTR: Function Attrs: nofree noinline nounwind readnone uwtable
 ; FNATTR-NEXT: declare void @nofree_function()
-; ATTRIBUTOR: Function Attrs:  nofree noinline nounwind readnone uwtable 
-; ATTRIBUTOR-NEXT: declare void @nofree_function()
 declare void @nofree_function() nofree readnone #0
 
 ; FNATTR: Function Attrs: noinline nounwind readnone uwtable
 ; FNATTR-NEXT: define void @call_nofree_function()
-; ATTRIBUTOR: Function Attrs: nofree noinline nosync nounwind uwtable
-; ATTRIBUTOR-NEXT: define void @call_nofree_function()
 define void @call_nofree_function() #0 {
     tail call void @nofree_function()
     ret void
@@ -162,16 +143,11 @@ define void @call_nofree_function() #0 {
 ; Call function declaration without "nofree"
 
 
-; ATTRIBUTOR: Function Attrs: noinline nounwind uwtable
-; ATTRIBUTOR-NEXT: declare void @maybe_free()
 declare void @maybe_free() #0
 
 
 ; FNATTR: Function Attrs: noinline nounwind uwtable
 ; FNATTR: define void @call_maybe_free()
-; ATTRIBUTOR: Function Attrs: noinline nounwind uwtable
-; ATTRIBUTOR-NOT: nofree
-; ATTRIBUTOR-NEXT: define void @call_maybe_free()
 define void @call_maybe_free() #0 {
     tail call void @maybe_free()
     ret void
@@ -183,9 +159,6 @@ define void @call_maybe_free() #0 {
 
 ; FNATTR: Function Attrs: noinline nounwind uwtable
 ; FNATTR-NEXT: define void @call_both()
-; ATTRIBUTOR: Function Attrs: noinline nounwind uwtable
-; ATTRIBUTOR-NOT: nofree
-; ATTRIBUTOR-NEXT: define void @call_both()
 define void @call_both() #0 {
     tail call void @maybe_free()
     tail call void @nofree_function()
@@ -196,16 +169,12 @@ define void @call_both() #0 {
 ; TEST 10 (positive case)
 ; Call intrinsic function
 ; FNATTRS: Function Attrs: noinline readnone speculatable
-; FNATTRS-NEXT: declare float @llvm.floor.f32(float)
-; ATTRIBUTOR: Function Attrs: nounwind readnone speculatable
-; ATTRIBUTOR-NEXT: declare float @llvm.floor.f32(float)
+; FNATTRS-NEXT: declare float @llvm.floor.f32(float %0)
 declare float @llvm.floor.f32(float)
 
 ; FNATTRS: Function Attrs: noinline nounwind uwtable
 ; FNATTRS-NEXT: define void @call_floor(float %a)
 ; FIXME: missing nofree
-; ATTRIBUTOR: Function Attrs: noinline nosync nounwind uwtable
-; ATTRIBUTOR-NEXT: define void @call_floor(float %a)
 
 define void @call_floor(float %a) #0 {
     tail call float @llvm.floor.f32(float %a)
@@ -217,8 +186,6 @@ define void @call_floor(float %a) #0 {
 
 ; FNATTRS: Function Attrs: noinline nounwind uwtable
 ; FNATTRS-NEXT: define void @f1()
-; ATTRIBUTOR: Function Attrs: nofree noinline nosync nounwind uwtable
-; ATTRIBUTOR-NEXT: define void @f1()
 define void @f1() #0 {
     tail call void @nofree_function()
     ret void
@@ -226,14 +193,14 @@ define void @f1() #0 {
 
 ; FNATTRS: Function Attrs: noinline nounwind uwtable
 ; FNATTRS-NEXT: define void @f2()
-; ATTRIBUTOR: Function Attrs: nofree noinline nosync nounwind uwtable
-; ATTRIBUTOR-NEXT: define void @f2()
 define void @f2() #0 {
     tail call void @f1()
     ret void
 }
 
 
+declare noalias i8* @malloc(i64)
+
 attributes #0 = { nounwind uwtable noinline }
 attributes #1 = { nounwind }
 attributes #2 = { nobuiltin nounwind }
diff --git a/llvm/test/Transforms/FunctionAttrs/nonnull.ll b/llvm/test/Transforms/FunctionAttrs/nonnull.ll
index c37defc20..128a2bdbe 100644
--- a/llvm/test/Transforms/FunctionAttrs/nonnull.ll
+++ b/llvm/test/Transforms/FunctionAttrs/nonnull.ll
@@ -1,6 +1,6 @@
-; RUN: opt -S -functionattrs -enable-nonnull-arg-prop %s | FileCheck %s --check-prefixes=BOTH,FNATTR
-; RUN: opt -S -passes=function-attrs -enable-nonnull-arg-prop %s | FileCheck %s --check-prefixes=BOTH,FNATTR
-; RUN: opt -attributor --attributor-disable=false -S < %s | FileCheck %s --check-prefixes=BOTH,ATTRIBUTOR
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -functionattrs -enable-nonnull-arg-prop %s | FileCheck %s --check-prefixes=FNATTR
+; RUN: opt -S -passes=function-attrs -enable-nonnull-arg-prop %s | FileCheck %s --check-prefixes=FNATTR
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
@@ -8,28 +8,32 @@ declare nonnull i8* @ret_nonnull()
 
 ; Return a pointer trivially nonnull (call return attribute)
 define i8* @test1() {
-; BOTH: define nonnull i8* @test1
+; FNATTR: define nonnull i8* @test1
   %ret = call i8* @ret_nonnull()
   ret i8* %ret
 }
 
 ; Return a pointer trivially nonnull (argument attribute)
 define i8* @test2(i8* nonnull %p) {
-; BOTH: define nonnull i8* @test2
+; FNATTR: define nonnull i8* @test2
   ret i8* %p
 }
 
 ; Given an SCC where one of the functions can not be marked nonnull,
 ; can we still mark the other one which is trivially nonnull
-define i8* @scc_binder() {
-; BOTH: define i8* @scc_binder
-  call i8* @test3()
+define i8* @scc_binder(i1 %c) {
+; FNATTR: define i8* @scc_binder
+  br i1 %c, label %rec, label %end
+rec:
+  call i8* @test3(i1 %c)
+  br label %end
+end:
   ret i8* null
 }
 
-define i8* @test3() {
-; BOTH: define nonnull i8* @test3
-  call i8* @scc_binder()
+define i8* @test3(i1 %c) {
+; FNATTR: define nonnull i8* @test3
+  call i8* @scc_binder(i1 %c)
   %ret = call i8* @ret_nonnull()
   ret i8* %ret
 }
@@ -39,38 +43,37 @@ define i8* @test3() {
 ; just never return period.)
 define i8* @test4_helper() {
 ; FNATTR: define noalias nonnull i8* @test4_helper
-; ATTRIBUTOR: define nonnull i8* @test4_helper
   %ret = call i8* @test4()
   ret i8* %ret
 }
 
 define i8* @test4() {
 ; FNATTR: define noalias nonnull i8* @test4
-; ATTRIBUTOR: define nonnull i8* @test4
   %ret = call i8* @test4_helper()
   ret i8* %ret
 }
 
 ; Given a mutual recursive set of functions which *can* return null
 ; make sure we haven't marked them as nonnull.
-define i8* @test5_helper() {
+define i8* @test5_helper(i1 %c) {
 ; FNATTR: define noalias i8* @test5_helper
-; ATTRIBUTOR: define i8* @test5_helper
-  %ret = call i8* @test5()
+  br i1 %c, label %rec, label %end
+rec:
+  %ret = call i8* @test5(i1 %c)
+  br label %end
+end:
   ret i8* null
 }
 
-define i8* @test5() {
+define i8* @test5(i1 %c) {
 ; FNATTR: define noalias i8* @test5
-; ATTRIBUTOR: define i8* @test5
-  %ret = call i8* @test5_helper()
+  %ret = call i8* @test5_helper(i1 %c)
   ret i8* %ret
 }
 
 ; Local analysis, but going through a self recursive phi
-define i8* @test6() {
+define i8* @test6a() {
 entry:
-; BOTH: define nonnull i8* @test6
   %ret = call i8* @ret_nonnull()
   br label %loop
 loop:
@@ -80,19 +83,30 @@ exit:
   ret i8* %phi
 }
 
-; BOTH: define i8* @test7
+define i8* @test6b(i1 %c) {
+entry:
+  %ret = call i8* @ret_nonnull()
+  br label %loop
+loop:
+  %phi = phi i8* [%ret, %entry], [%phi, %loop]
+  br i1 %c, label %loop, label %exit
+exit:
+  ret i8* %phi
+}
+
+; FNATTR: define i8* @test7
 define i8* @test7(i8* %a) {
   %b = getelementptr inbounds i8, i8* %a, i64 0
   ret i8* %b
 }
 
-; BOTH: define nonnull i8* @test8
+; FNATTR: define nonnull i8* @test8
 define i8* @test8(i8* %a) {
   %b = getelementptr inbounds i8, i8* %a, i64 1
   ret i8* %b
 }
 
-; BOTH: define i8* @test9
+; FNATTR: define i8* @test9
 define i8* @test9(i8* %a, i64 %n) {
   %b = getelementptr inbounds i8, i8* %a, i64 %n
   ret i8* %b
@@ -101,7 +115,6 @@ define i8* @test9(i8* %a, i64 %n) {
 declare void @llvm.assume(i1)
 ; FNATTR: define i8* @test10
 ; FIXME: missing nonnull
-; ATTRIBUTOR: define i8* @test10
 define i8* @test10(i8* %a, i64 %n) {
   %cmp = icmp ne i64 %n, 0
   call void @llvm.assume(i1 %cmp)
@@ -115,7 +128,6 @@ define i8* @test10(i8* %a, i64 %n) {
 ; }
 ; FNATTR: define i8* @test11
 ; FIXME: missing nonnull
-; ATTRIBUTOR: define i8* @test11
 define i8* @test11(i8*) local_unnamed_addr {
   %2 = icmp eq i8* %0, null
   br i1 %2, label %3, label %5
@@ -133,8 +145,6 @@ define i8* @test11(i8*) local_unnamed_addr {
 ; Simple CallSite Test
 declare void @test12_helper(i8*)
 define void @test12(i8* nonnull %a) {
-; ATTRIBUTOR: define void @test12(i8* nonnull %a)
-; ATTRIBUTOR-NEXT: tail call void @test12_helper(i8* nonnull %a)
   tail call void @test12_helper(i8* %a)
   ret void
 }
@@ -150,7 +160,6 @@ define void @test13_helper() {
   ret void
 }
 define internal void @test13(i8* %a, i8* %b, i8* %c) {
-; ATTRIBUTOR: define internal void @test13(i8* nonnull %a, i8* %b, i8* %c) 
   ret void
 }
 
@@ -163,14 +172,13 @@ declare nonnull i8* @nonnull()
 ; * Argument
 ; 1. In f1:bb6, %arg can be marked with nonnull because of the comparison in bb1
 ; 2. Because f2 is internal function, f2(i32* %arg) -> @f2(i32* nonnull %arg)
-; 3. In f1:bb4 %tmp5 is nonnull and f3 is internal function. 
+; 3. In f1:bb4 %tmp5 is nonnull and f3 is internal function.
 ;    Then, f3(i32* %arg) -> @f3(i32* nonnull %arg)
 ; 4. We get nonnull in whole f1 call sites so f1(i32* %arg) -> @f1(i32* nonnull %arg)
 
 
 define internal i32* @f1(i32* %arg) {
-; FIXME: missing nonnull It should be nonnull @f1(i32* nonnull %arg)
-; ATTRIBUTOR: define internal nonnull i32* @f1(i32* %arg)
+; FIXME: missing nonnull It should be nonnull @f1(i32* nonnull readonly %arg)
 
 bb:
   %tmp = icmp eq i32* %arg, null
@@ -183,41 +191,136 @@ bb1:                                              ; preds = %bb
 
 bb4:                                              ; preds = %bb1
   %tmp5 = getelementptr inbounds i32, i32* %arg, i64 1
-; ATTRIBUTOR: %tmp5b = tail call i32* @f3(i32* nonnull %tmp5)
   %tmp5b = tail call i32* @f3(i32* %tmp5)
+  %tmp5c = getelementptr inbounds i32, i32* %tmp5b, i64 -1
   br label %bb9
 
 bb6:                                              ; preds = %bb1
 ; FIXME: missing nonnull. It should be @f2(i32* nonnull %arg)
-; ATTRIBUTOR: %tmp7 = tail call i32* @f2(i32* %arg)
   %tmp7 = tail call i32* @f2(i32* %arg)
   ret i32* %tmp7
 
 bb9:                                              ; preds = %bb4, %bb
-  %tmp10 = phi i32* [ %tmp5, %bb4 ], [ inttoptr (i64 4 to i32*), %bb ]
+  %tmp10 = phi i32* [ %tmp5c, %bb4 ], [ inttoptr (i64 4 to i32*), %bb ]
   ret i32* %tmp10
 }
 
 define internal i32* @f2(i32* %arg) {
-; FIXME: missing nonnull. It should be nonnull @f2(i32* nonnull %arg) 
-; ATTRIBUTOR: define internal nonnull i32* @f2(i32* %arg)
+; FIXME: missing nonnull. It should be nonnull @f2(i32* nonnull %arg)
 bb:
 
-; FIXME: missing nonnull. It should be @f1(i32* nonnull %arg) 
-; ATTRIBUTOR:   %tmp = tail call i32* @f1(i32* %arg)
+; FIXME: missing nonnull. It should be @f1(i32* nonnull readonly %arg)
   %tmp = tail call i32* @f1(i32* %arg)
   ret i32* %tmp
 }
 
 define dso_local noalias i32* @f3(i32* %arg) {
-; FIXME: missing nonnull. It should be nonnull @f3(i32* nonnull %arg) 
-; ATTRIBUTOR: define dso_local noalias i32* @f3(i32* %arg)
+; FIXME: missing nonnull. It should be nonnull @f3(i32* nonnull readonly %arg)
 bb:
-; FIXME: missing nonnull. It should be @f1(i32* nonnull %arg) 
-; ATTRIBUTOR:   %tmp = call i32* @f1(i32* %arg)
+; FIXME: missing nonnull. It should be @f1(i32* nonnull readonly %arg)
   %tmp = call i32* @f1(i32* %arg)
-  ret i32* null
+  ret i32* %tmp
+}
+
+; TEST 15
+define void @f15(i8* %arg) {
+
+  tail call void @use1(i8* dereferenceable(4) %arg)
+  ret void
+}
+
+declare void @fun0() #1
+declare void @fun1(i8*) #1
+declare void @fun2(i8*, i8*) #1
+declare void @fun3(i8*, i8*, i8*) #1
+; TEST 16 simple path test
+; if(..)
+;   fun2(nonnull %a, nonnull %b)
+; else
+;   fun2(nonnull %a, %b)
+; We can say that %a is nonnull but %b is not.
+define void @f16(i8* %a, i8 * %b, i8 %c) {
+; FIXME: missing nonnull on %a
+  %cmp = icmp eq i8 %c, 0
+  br i1 %cmp, label %if.then, label %if.else
+if.then:
+  tail call void @fun2(i8* nonnull %a, i8* nonnull %b)
+  ret void
+if.else:
+  tail call void @fun2(i8* nonnull %a, i8* %b)
+  ret void
+}
+; TEST 17 explore child BB test
+; if(..)
+;    ... (willreturn & nounwind)
+; else
+;    ... (willreturn & nounwind)
+; fun1(nonnull %a)
+; We can say that %a is nonnull
+define void @f17(i8* %a, i8 %c) {
+  %cmp = icmp eq i8 %c, 0
+  br i1 %cmp, label %if.then, label %if.else
+if.then:
+  tail call void @fun0()
+  br label %cont
+if.else:
+  tail call void @fun0()
+  br label %cont
+cont:
+  tail call void @fun1(i8* nonnull %a)
+  ret void
+}
+; TEST 18 More complex test
+; if(..)
+;    ... (willreturn & nounwind)
+; else
+;    ... (willreturn & nounwind)
+; if(..)
+;    ... (willreturn & nounwind)
+; else
+;    ... (willreturn & nounwind)
+; fun1(nonnull %a)
+
+define void @f18(i8* %a, i8* %b, i8 %c) {
+  %cmp1 = icmp eq i8 %c, 0
+  br i1 %cmp1, label %if.then, label %if.else
+if.then:
+  tail call void @fun0()
+  br label %cont
+if.else:
+  tail call void @fun0()
+  br label %cont
+cont:
+  %cmp2 = icmp eq i8 %c, 1
+  br i1 %cmp2, label %cont.then, label %cont.else
+cont.then:
+  tail call void @fun1(i8* nonnull %b)
+  br label %cont2
+cont.else:
+  tail call void @fun0()
+  br label %cont2
+cont2:
+  tail call void @fun1(i8* nonnull %a)
+  ret void
+}
+
+; TEST 19: Loop
+
+define void @f19(i8* %a, i8* %b, i8 %c) {
+; FIXME: missing nonnull on %b
+  br label %loop.header
+loop.header:
+  %cmp2 = icmp eq i8 %c, 0
+  br i1 %cmp2, label %loop.body, label %loop.exit
+loop.body:
+  tail call void @fun1(i8* nonnull %b)
+  tail call void @fun1(i8* nonnull %a)
+  br label %loop.header
+loop.exit:
+  tail call void @fun1(i8* nonnull %b)
+  ret void
 }
+
 ; Test propagation of nonnull callsite args back to caller.
 
 declare void @use1(i8* %x)
@@ -233,11 +336,10 @@ declare i8 @use1safecall(i8* %x) readonly nounwind ; readonly+nounwind guarantee
 ; Can't extend non-null to parent for any argument because the 2nd call is not guaranteed to execute.
 
 define void @parent1(i8* %a, i8* %b, i8* %c) {
-; BOTH-LABEL: @parent1(i8* %a, i8* %b, i8* %c)
-; BOTH-NEXT:    call void @use3(i8* %c, i8* %a, i8* %b)
+; FNATTR-LABEL: @parent1(i8* %a, i8* %b, i8* %c)
+; FNATTR-NEXT:    call void @use3(i8* %c, i8* %a, i8* %b)
 ; FNATTR-NEXT:    call void @use3nonnull(i8* %b, i8* %c, i8* %a)
-; ATTRIBUTOR-NEXT:    call void @use3nonnull(i8* nonnull %b, i8* nonnull %c, i8* nonnull %a)
-; BOTH-NEXT:    ret void
+; FNATTR-NEXT:    ret void
   call void @use3(i8* %c, i8* %a, i8* %b)
   call void @use3nonnull(i8* %b, i8* %c, i8* %a)
   ret void
@@ -250,16 +352,8 @@ define void @parent2(i8* %a, i8* %b, i8* %c) {
 ; FNATTR-NEXT:    call void @use3nonnull(i8* %b, i8* %c, i8* %a)
 ; FNATTR-NEXT:    call void @use3(i8* %c, i8* %a, i8* %b)
 
-; FIXME: missing "nonnull", it should be
-; @parent2(i8* nonnull %a, i8* nonnull %b, i8* nonnull %c)
-;     call void @use3nonnull(i8* nonnull %b, i8* nonnull %c, i8* nonnull %a)
-;     call void @use3(i8* nonnull %c, i8* nonnull %a, i8* nonnull %b)
 
-; ATTRIBUTOR-LABEL: @parent2(i8* %a, i8* %b, i8* %c)
-; ATTRIBUTOR-NEXT:    call void @use3nonnull(i8* nonnull %b, i8* nonnull %c, i8* nonnull %a)
-; ATTRIBUTOR-NEXT:    call void @use3(i8* %c, i8* %a, i8* %b)
-
-; BOTH-NEXT:    ret void
+; FNATTR-NEXT:    ret void
   call void @use3nonnull(i8* %b, i8* %c, i8* %a)
   call void @use3(i8* %c, i8* %a, i8* %b)
   ret void
@@ -272,15 +366,8 @@ define void @parent3(i8* %a, i8* %b, i8* %c) {
 ; FNATTR-NEXT:    call void @use1nonnull(i8* %a)
 ; FNATTR-NEXT:    call void @use3(i8* %c, i8* %b, i8* %a)
 
-; FIXME: missing "nonnull", it should be,
-; @parent3(i8* nonnull %a, i8* %b, i8* %c)
-;    call void @use1nonnull(i8* nonnull %a)
-;    call void @use3(i8* %c, i8* %b, i8* nonnull %a)
-; ATTRIBUTOR-LABEL: @parent3(i8* %a, i8* %b, i8* %c)
-; ATTRIBUTOR-NEXT:    call void @use1nonnull(i8* nonnull %a)
-; ATTRIBUTOR-NEXT:    call void @use3(i8* %c, i8* %b, i8* %a)
 
-; BOTH-NEXT:  ret void
+; FNATTR-NEXT:  ret void
 
   call void @use1nonnull(i8* %a)
   call void @use3(i8* %c, i8* %b, i8* %a)
@@ -295,18 +382,8 @@ define void @parent4(i8* %a, i8* %b, i8* %c) {
 ; CHECK-NEXT:    call void @use2(i8* %a, i8* %c)
 ; CHECK-NEXT:    call void @use1(i8* %b)
 
-; FIXME : missing "nonnull", it should be
-; @parent4(i8* %a, i8* nonnull %b, i8* nonnull %c)
-;   call void @use2nonnull(i8* nonnull %c, i8* nonull %b)
-;   call void @use2(i8* %a, i8* nonnull %c)
-;   call void @use1(i8* nonnull %b)
-
-; ATTRIBUTOR-LABEL: @parent4(i8* %a, i8* %b, i8* %c)
-; ATTRIBUTOR-NEXT:    call void @use2nonnull(i8* nonnull %c, i8* nonnull %b)
-; ATTRIBUTOR-NEXT:    call void @use2(i8* %a, i8* %c)
-; ATTRIBUTOR-NEXT:    call void @use1(i8* %b)
 
-; BOTH: ret void
+; FNATTR: ret void
 
   call void @use2nonnull(i8* %c, i8* %b)
   call void @use2(i8* %a, i8* %c)
@@ -319,14 +396,13 @@ define void @parent4(i8* %a, i8* %b, i8* %c) {
 ; because it would incorrectly propagate the wrong information to its callers.
 
 define void @parent5(i8* %a, i1 %a_is_notnull) {
-; BOTH: @parent5(i8* %a, i1 %a_is_notnull)
-; BOTH-NEXT:    br i1 %a_is_notnull, label %t, label %f
-; BOTH:       t:
+; FNATTR: @parent5(i8* %a, i1 %a_is_notnull)
+; FNATTR-NEXT:    br i1 %a_is_notnull, label %t, label %f
+; FNATTR:       t:
 ; FNATTR-NEXT:    call void @use1nonnull(i8* %a)
-; ATTRIBUTOR-NEXT:    call void @use1nonnull(i8* nonnull %a)
-; BOTH-NEXT:    ret void
-; BOTH:       f:
-; BOTH-NEXT:    ret void
+; FNATTR-NEXT:    ret void
+; FNATTR:       f:
+; FNATTR-NEXT:    ret void
 
   br i1 %a_is_notnull, label %t, label %f
 t:
@@ -337,14 +413,13 @@ f:
 }
 
 ; The callsite must execute in order for the attribute to transfer to the parent.
-; The volatile load might trap, so there's no guarantee that we'll ever get to the call.
+; The volatile load can't trap, so we can guarantee that we'll get to the call.
 
 define i8 @parent6(i8* %a, i8* %b) {
-; BOTH-LABEL: @parent6(i8* %a, i8* %b)
-; BOTH-NEXT:    [[C:%.*]] = load volatile i8, i8* %b
+; FNATTR-LABEL: @parent6(i8* nonnull %a, i8* %b)
+; FNATTR-NEXT:    [[C:%.*]] = load volatile i8, i8* %b
 ; FNATTR-NEXT:    call void @use1nonnull(i8* %a)
-; ATTRIBUTOR-NEXT:    call void @use1nonnull(i8* nonnull %a)
-; BOTH-NEXT:    ret i8 [[C]]
+; FNATTR-NEXT:    ret i8 [[C]]
 
   %c = load volatile i8, i8* %b
   call void @use1nonnull(i8* %a)
@@ -358,17 +433,9 @@ define i8 @parent7(i8* %a) {
 ; FNATTR-NEXT:    [[RET:%.*]] = call i8 @use1safecall(i8* %a)
 ; FNATTR-NEXT:    call void @use1nonnull(i8* %a)
 
-; FIXME : missing "nonnull", it should be
-; @parent7(i8* nonnull %a)
-;   [[RET:%.*]] = call i8 @use1safecall(i8* nonnull %a)
-;   call void @use1nonnull(i8* nonnull %a)
-;   ret i8 [[RET]]
 
-; ATTRIBUTOR-LABEL: @parent7(i8* %a)
-; ATTRIBUTOR-NEXT:    [[RET:%.*]] = call i8 @use1safecall(i8* %a)
-; ATTRIBUTOR-NEXT:    call void @use1nonnull(i8* nonnull %a)
 
-; BOTH-NEXT: ret i8 [[RET]]
+; FNATTR-NEXT: ret i8 [[RET]]
 
   %ret = call i8 @use1safecall(i8* %a)
   call void @use1nonnull(i8* %a)
@@ -381,19 +448,16 @@ declare i32 @esfp(...)
 
 define i1 @parent8(i8* %a, i8* %bogus1, i8* %b) personality i8* bitcast (i32 (...)* @esfp to i8*){
 ; FNATTR-LABEL: @parent8(i8* nonnull %a, i8* nocapture readnone %bogus1, i8* nonnull %b)
-; FIXME : missing "nonnull", it should be @parent8(i8* nonnull %a, i8* %bogus1, i8* nonnull %b)
-; ATTRIBUTOR-LABEL: @parent8(i8* %a, i8* %bogus1, i8* %b)
-; BOTH-NEXT:  entry:
+; FNATTR-NEXT:  entry:
 ; FNATTR-NEXT:    invoke void @use2nonnull(i8* %a, i8* %b)
-; ATTRIBUTOR-NEXT:    invoke void @use2nonnull(i8* nonnull %a, i8* nonnull %b)
-; BOTH-NEXT:    to label %cont unwind label %exc
-; BOTH:       cont:
-; BOTH-NEXT:    [[NULL_CHECK:%.*]] = icmp eq i8* %b, null
-; BOTH-NEXT:    ret i1 [[NULL_CHECK]]
-; BOTH:       exc:
-; BOTH-NEXT:    [[LP:%.*]] = landingpad { i8*, i32 }
-; BOTH-NEXT:    filter [0 x i8*] zeroinitializer
-; BOTH-NEXT:    unreachable
+; FNATTR-NEXT:    to label %cont unwind label %exc
+; FNATTR:       cont:
+; FNATTR-NEXT:    [[NULL_CHECK:%.*]] = icmp eq i8* %b, null
+; FNATTR-NEXT:    ret i1 [[NULL_CHECK]]
+; FNATTR:       exc:
+; FNATTR-NEXT:    [[LP:%.*]] = landingpad { i8*, i32 }
+; FNATTR-NEXT:    filter [0 x i8*] zeroinitializer
+; FNATTR-NEXT:    unreachable
 
 entry:
   invoke void @use2nonnull(i8* %a, i8* %b)
@@ -409,7 +473,7 @@ exc:
   unreachable
 }
 
-; BOTH: define nonnull i32* @gep1(
+; FNATTR: define nonnull i32* @gep1(
 define i32* @gep1(i32* %p) {
   %q = getelementptr inbounds i32, i32* %p, i32 1
   ret i32* %q
@@ -417,18 +481,24 @@ define i32* @gep1(i32* %p) {
 
 define i32* @gep1_no_null_opt(i32* %p) #0 {
 ; Should't be able to derive nonnull based on gep.
-; BOTH: define i32* @gep1_no_null_opt(
+; FNATTR: define i32* @gep1_no_null_opt(
   %q = getelementptr inbounds i32, i32* %p, i32 1
   ret i32* %q
 }
 
-; BOTH: define i32 addrspace(3)* @gep2(
+; FNATTR: define i32 addrspace(3)* @gep2(
 define i32 addrspace(3)* @gep2(i32 addrspace(3)* %p) {
   %q = getelementptr inbounds i32, i32 addrspace(3)* %p, i32 1
   ret i32 addrspace(3)* %q
 }
 
-; BOTH: define internal nonnull i32* @g2()
+; FNATTR:     define i32 addrspace(3)* @as(i32 addrspace(3)* readnone returned dereferenceable(4) %p)
+; FIXME: We should propagate dereferenceable here but *not* nonnull
+define i32 addrspace(3)* @as(i32 addrspace(3)* dereferenceable(4) %p) {
+  ret i32 addrspace(3)* %p
+}
+
+; FNATTR: define internal nonnull i32* @g2()
 define internal i32* @g2() {
   ret i32* inttoptr (i64 4 to i32*)
 }
@@ -438,4 +508,267 @@ define  i32* @g1() {
   ret i32* %c
 }
 
+declare void @use_i32_ptr(i32*) readnone nounwind
+define internal void @called_by_weak(i32* %a) {
+  call void @use_i32_ptr(i32* %a)
+  ret void
+}
+
+; Check we do not annotate the function interface of this weak function.
+define weak_odr void @weak_caller(i32* nonnull %a) {
+  call void @called_by_weak(i32* %a)
+  ret void
+}
+
+; Expect nonnull
+define internal void @control(i32* dereferenceable(4) %a) {
+  call void @use_i32_ptr(i32* %a)
+  ret void
+}
+; Avoid nonnull as we do not touch naked functions
+define internal void @naked(i32* dereferenceable(4) %a) naked {
+  call void @use_i32_ptr(i32* %a)
+  ret void
+}
+; Avoid nonnull as we do not touch optnone
+define internal void @optnone(i32* dereferenceable(4) %a) optnone noinline {
+  call void @use_i32_ptr(i32* %a)
+  ret void
+}
+define void @make_live(i32* nonnull dereferenceable(8) %a) {
+  call void @naked(i32* nonnull dereferenceable(8) align 16 %a)
+  call void @control(i32* nonnull dereferenceable(8) align 16 %a)
+  call void @optnone(i32* nonnull dereferenceable(8) align 16 %a)
+  ret void
+}
+
+;int f(int *u, int n){
+;  for(int i = 0;i<n;i++){
+;    h(u);
+;  }
+;  return g(nonnull u);
+;}
+declare void @h(i32*) willreturn nounwind
+declare i32 @g(i32*) willreturn nounwind
+define i32 @nonnull_exec_ctx_1(i32* %a, i32 %b) {
+; FNATTR-LABEL: define {{[^@]+}}@nonnull_exec_ctx_1
+; FNATTR-SAME: (i32* [[A:%.*]], i32 [[B:%.*]])
+; FNATTR-NEXT:  en:
+; FNATTR-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[B:%.*]], 0
+; FNATTR-NEXT:    br i1 [[TMP3]], label [[EX:%.*]], label [[HD:%.*]]
+; FNATTR:       ex:
+; FNATTR-NEXT:    [[TMP5:%.*]] = tail call i32 @g(i32* nonnull [[A:%.*]])
+; FNATTR-NEXT:    ret i32 [[TMP5]]
+; FNATTR:       hd:
+; FNATTR-NEXT:    [[TMP7:%.*]] = phi i32 [ [[TMP8:%.*]], [[HD]] ], [ 0, [[EN:%.*]] ]
+; FNATTR-NEXT:    tail call void @h(i32* [[A]])
+; FNATTR-NEXT:    [[TMP8]] = add nuw i32 [[TMP7]], 1
+; FNATTR-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[TMP8]], [[B]]
+; FNATTR-NEXT:    br i1 [[TMP9]], label [[EX]], label [[HD]]
+;
+;
+en:
+  %tmp3 = icmp eq i32 %b, 0
+  br i1 %tmp3, label %ex, label %hd
+
+ex:
+  %tmp5 = tail call i32 @g(i32* nonnull %a)
+  ret i32 %tmp5
+
+hd:
+  %tmp7 = phi i32 [ %tmp8, %hd ], [ 0, %en ]
+  tail call void @h(i32* %a)
+  %tmp8 = add nuw i32 %tmp7, 1
+  %tmp9 = icmp eq i32 %tmp8, %b
+  br i1 %tmp9, label %ex, label %hd
+}
+
+define i32 @nonnull_exec_ctx_1b(i32* %a, i32 %b) {
+; FNATTR-LABEL: define {{[^@]+}}@nonnull_exec_ctx_1b
+; FNATTR-SAME: (i32* [[A:%.*]], i32 [[B:%.*]])
+; FNATTR-NEXT:  en:
+; FNATTR-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[B:%.*]], 0
+; FNATTR-NEXT:    br i1 [[TMP3]], label [[EX:%.*]], label [[HD:%.*]]
+; FNATTR:       ex:
+; FNATTR-NEXT:    [[TMP5:%.*]] = tail call i32 @g(i32* nonnull [[A:%.*]])
+; FNATTR-NEXT:    ret i32 [[TMP5]]
+; FNATTR:       hd:
+; FNATTR-NEXT:    [[TMP7:%.*]] = phi i32 [ [[TMP8:%.*]], [[HD2:%.*]] ], [ 0, [[EN:%.*]] ]
+; FNATTR-NEXT:    tail call void @h(i32* [[A]])
+; FNATTR-NEXT:    br label [[HD2]]
+; FNATTR:       hd2:
+; FNATTR-NEXT:    [[TMP8]] = add nuw i32 [[TMP7]], 1
+; FNATTR-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[TMP8]], [[B]]
+; FNATTR-NEXT:    br i1 [[TMP9]], label [[EX]], label [[HD]]
+;
+;
+en:
+  %tmp3 = icmp eq i32 %b, 0
+  br i1 %tmp3, label %ex, label %hd
+
+ex:
+  %tmp5 = tail call i32 @g(i32* nonnull %a)
+  ret i32 %tmp5
+
+hd:
+  %tmp7 = phi i32 [ %tmp8, %hd2 ], [ 0, %en ]
+  tail call void @h(i32* %a)
+  br label %hd2
+
+hd2:
+  %tmp8 = add nuw i32 %tmp7, 1
+  %tmp9 = icmp eq i32 %tmp8, %b
+  br i1 %tmp9, label %ex, label %hd
+}
+
+define i32 @nonnull_exec_ctx_2(i32* %a, i32 %b) willreturn nounwind {
+; FNATTR-LABEL: define {{[^@]+}}@nonnull_exec_ctx_2
+; FNATTR-SAME: (i32* [[A:%.*]], i32 [[B:%.*]])
+; FNATTR-NEXT:  en:
+; FNATTR-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[B:%.*]], 0
+; FNATTR-NEXT:    br i1 [[TMP3]], label [[EX:%.*]], label [[HD:%.*]]
+; FNATTR:       ex:
+; FNATTR-NEXT:    [[TMP5:%.*]] = tail call i32 @g(i32* nonnull [[A:%.*]])
+; FNATTR-NEXT:    ret i32 [[TMP5]]
+; FNATTR:       hd:
+; FNATTR-NEXT:    [[TMP7:%.*]] = phi i32 [ [[TMP8:%.*]], [[HD]] ], [ 0, [[EN:%.*]] ]
+; FNATTR-NEXT:    tail call void @h(i32* [[A]])
+; FNATTR-NEXT:    [[TMP8]] = add nuw i32 [[TMP7]], 1
+; FNATTR-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[TMP8]], [[B]]
+; FNATTR-NEXT:    br i1 [[TMP9]], label [[EX]], label [[HD]]
+;
+;
+en:
+  %tmp3 = icmp eq i32 %b, 0
+  br i1 %tmp3, label %ex, label %hd
+
+ex:
+  %tmp5 = tail call i32 @g(i32* nonnull %a)
+  ret i32 %tmp5
+
+hd:
+  %tmp7 = phi i32 [ %tmp8, %hd ], [ 0, %en ]
+  tail call void @h(i32* %a)
+  %tmp8 = add nuw i32 %tmp7, 1
+  %tmp9 = icmp eq i32 %tmp8, %b
+  br i1 %tmp9, label %ex, label %hd
+}
+
+define i32 @nonnull_exec_ctx_2b(i32* %a, i32 %b) willreturn nounwind {
+; FNATTR-LABEL: define {{[^@]+}}@nonnull_exec_ctx_2b
+; FNATTR-SAME: (i32* [[A:%.*]], i32 [[B:%.*]])
+; FNATTR-NEXT:  en:
+; FNATTR-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[B:%.*]], 0
+; FNATTR-NEXT:    br i1 [[TMP3]], label [[EX:%.*]], label [[HD:%.*]]
+; FNATTR:       ex:
+; FNATTR-NEXT:    [[TMP5:%.*]] = tail call i32 @g(i32* nonnull [[A:%.*]])
+; FNATTR-NEXT:    ret i32 [[TMP5]]
+; FNATTR:       hd:
+; FNATTR-NEXT:    [[TMP7:%.*]] = phi i32 [ [[TMP8:%.*]], [[HD2:%.*]] ], [ 0, [[EN:%.*]] ]
+; FNATTR-NEXT:    tail call void @h(i32* [[A]])
+; FNATTR-NEXT:    br label [[HD2]]
+; FNATTR:       hd2:
+; FNATTR-NEXT:    [[TMP8]] = add nuw i32 [[TMP7]], 1
+; FNATTR-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[TMP8]], [[B]]
+; FNATTR-NEXT:    br i1 [[TMP9]], label [[EX]], label [[HD]]
+;
+;
+en:
+  %tmp3 = icmp eq i32 %b, 0
+  br i1 %tmp3, label %ex, label %hd
+
+ex:
+  %tmp5 = tail call i32 @g(i32* nonnull %a)
+  ret i32 %tmp5
+
+hd:
+  %tmp7 = phi i32 [ %tmp8, %hd2 ], [ 0, %en ]
+  tail call void @h(i32* %a)
+  br label %hd2
+
+hd2:
+  %tmp8 = add nuw i32 %tmp7, 1
+  %tmp9 = icmp eq i32 %tmp8, %b
+  br i1 %tmp9, label %ex, label %hd
+}
+
+; Original from PR43833
+declare void @sink(i32*)
+
+; FIXME: the sink argument should be marked nonnull as in @PR43833_simple.
+define void @PR43833(i32* %0, i32 %1) {
+; FNATTR-LABEL: @PR43833(
+; FNATTR-NEXT:    [[TMP3:%.*]] = icmp sgt i32 [[TMP1:%.*]], 1
+; FNATTR-NEXT:    br i1 [[TMP3]], label [[TMP4:%.*]], label [[TMP7:%.*]]
+; FNATTR:       4:
+; FNATTR-NEXT:    [[TMP5:%.*]] = zext i32 [[TMP1]] to i64
+; FNATTR-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[TMP0:%.*]], i64 [[TMP5]]
+; FNATTR-NEXT:    br label [[TMP8:%.*]]
+; FNATTR:       7:
+; FNATTR-NEXT:    ret void
+; FNATTR:       8:
+; FNATTR-NEXT:    [[TMP9:%.*]] = phi i32 [ 1, [[TMP4]] ], [ [[TMP10:%.*]], [[TMP8]] ]
+; FNATTR-NEXT:    tail call void @sink(i32* [[TMP6]])
+; FNATTR-NEXT:    [[TMP10]] = add nuw nsw i32 [[TMP9]], 1
+; FNATTR-NEXT:    [[TMP11:%.*]] = icmp eq i32 [[TMP10]], [[TMP1]]
+; FNATTR-NEXT:    br i1 [[TMP11]], label [[TMP7]], label [[TMP8]]
+;
+  %3 = icmp sgt i32 %1, 1
+  br i1 %3, label %4, label %7
+
+4:                                                ; preds = %2
+  %5 = zext i32 %1 to i64
+  %6 = getelementptr inbounds i32, i32* %0, i64 %5
+  br label %8
+
+7:                                                ; preds = %8, %2
+  ret void
+
+8:                                                ; preds = %8, %4
+  %9 = phi i32 [ 1, %4 ], [ %10, %8 ]
+  tail call void @sink(i32* %6)
+  %10 = add nuw nsw i32 %9, 1
+  %11 = icmp eq i32 %10, %1
+  br i1 %11, label %7, label %8
+}
+
+; Adjusted from PR43833
+define void @PR43833_simple(i32* %0, i32 %1) {
+; FNATTR-LABEL: @PR43833_simple(
+; FNATTR-NEXT:    [[TMP3:%.*]] = icmp ne i32 [[TMP1:%.*]], 0
+; FNATTR-NEXT:    br i1 [[TMP3]], label [[TMP4:%.*]], label [[TMP7:%.*]]
+; FNATTR:       4:
+; FNATTR-NEXT:    [[TMP5:%.*]] = zext i32 [[TMP1]] to i64
+; FNATTR-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[TMP0:%.*]], i64 [[TMP5]]
+; FNATTR-NEXT:    br label [[TMP8:%.*]]
+; FNATTR:       7:
+; FNATTR-NEXT:    ret void
+; FNATTR:       8:
+; FNATTR-NEXT:    [[TMP9:%.*]] = phi i32 [ 1, [[TMP4]] ], [ [[TMP10:%.*]], [[TMP8]] ]
+; FNATTR-NEXT:    tail call void @sink(i32* [[TMP6]])
+; FNATTR-NEXT:    [[TMP10]] = add nuw nsw i32 [[TMP9]], 1
+; FNATTR-NEXT:    [[TMP11:%.*]] = icmp eq i32 [[TMP10]], [[TMP1]]
+; FNATTR-NEXT:    br i1 [[TMP11]], label [[TMP7]], label [[TMP8]]
+;
+;
+  %3 = icmp ne i32 %1, 0
+  br i1 %3, label %4, label %7
+
+4:                                                ; preds = %2
+  %5 = zext i32 %1 to i64
+  %6 = getelementptr inbounds i32, i32* %0, i64 %5
+  br label %8
+
+7:                                                ; preds = %8, %2
+  ret void
+
+8:                                                ; preds = %8, %4
+  %9 = phi i32 [ 1, %4 ], [ %10, %8 ]
+  tail call void @sink(i32* %6)
+  %10 = add nuw nsw i32 %9, 1
+  %11 = icmp eq i32 %10, %1
+  br i1 %11, label %7, label %8
+}
+
 attributes #0 = { "null-pointer-is-valid"="true" }
+attributes #1 = { nounwind willreturn}
diff --git a/llvm/test/Transforms/FunctionAttrs/norecurse.ll b/llvm/test/Transforms/FunctionAttrs/norecurse.ll
index 0293938e4..b780bb7b7 100644
--- a/llvm/test/Transforms/FunctionAttrs/norecurse.ll
+++ b/llvm/test/Transforms/FunctionAttrs/norecurse.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -functionattrs -rpo-functionattrs -S | FileCheck %s
+; RUN: opt < %s -basicaa -functionattrs -rpo-functionattrs -S | FileCheck %s 
 ; RUN: opt < %s -aa-pipeline=basic-aa -passes='cgscc(function-attrs),rpo-functionattrs' -S | FileCheck %s
 
 ; CHECK: Function Attrs
@@ -62,6 +62,7 @@ declare void @llvm.memcpy.p0i8.p0i8.i32(i8*, i8*, i32, i1)
 
 ; CHECK: Function Attrs
 ; CHECK-SAME: norecurse readnone
+; FIXME: missing "norecurse"
 ; CHECK-NEXT: define internal i32 @called_by_norecurse()
 define internal i32 @called_by_norecurse() {
   %a = call i32 @k()
@@ -76,6 +77,7 @@ define void @m() norecurse {
 
 ; CHECK: Function Attrs
 ; CHECK-SAME: norecurse readnone
+; FIXME: missing "norecurse"
 ; CHECK-NEXT: define internal i32 @called_by_norecurse_indirectly()
 define internal i32 @called_by_norecurse_indirectly() {
   %a = call i32 @k()
diff --git a/llvm/test/Transforms/FunctionAttrs/nounwind.ll b/llvm/test/Transforms/FunctionAttrs/nounwind.ll
index 1625d12e9..6d5e3a2ea 100644
--- a/llvm/test/Transforms/FunctionAttrs/nounwind.ll
+++ b/llvm/test/Transforms/FunctionAttrs/nounwind.ll
@@ -1,11 +1,8 @@
 ; RUN: opt < %s -functionattrs -S | FileCheck %s
-; RUN: opt < %s -attributor -attributor-disable=false -S | FileCheck %s --check-prefix=ATTRIBUTOR
 
 ; TEST 1
 ; CHECK: Function Attrs: norecurse nounwind readnone
 ; CHECK-NEXT: define i32 @foo1()
-; ATTRIBUTOR: Function Attrs: nofree nosync nounwind
-; ATTRIBUTOR-NEXT: define i32 @foo1()
 define i32 @foo1() {
   ret i32 1
 }
@@ -13,8 +10,6 @@ define i32 @foo1() {
 ; TEST 2
 ; CHECK: Function Attrs: nounwind readnone
 ; CHECK-NEXT: define i32 @scc1_foo()
-; ATTRIBUTOR: Function Attrs: nofree nosync nounwind
-; ATTRIBUTOR-NEXT: define i32 @scc1_foo()
 define i32 @scc1_foo() {
   %1 = call i32 @scc1_bar()
   ret i32 1
@@ -24,8 +19,6 @@ define i32 @scc1_foo() {
 ; TEST 3
 ; CHECK: Function Attrs: nounwind readnone
 ; CHECK-NEXT: define i32 @scc1_bar()
-; ATTRIBUTOR: Function Attrs: nofree nosync nounwind
-; ATTRIBUTOR-NEXT: define i32 @scc1_bar()
 define i32 @scc1_bar() {
   %1 = call i32 @scc1_foo()
   ret i32 1
@@ -36,7 +29,6 @@ declare i32 @non_nounwind()
 
 ; TEST 4
 ; CHECK: define void @call_non_nounwind() {
-; ATTRIBUTOR: define void @call_non_nounwind() {
 define void @call_non_nounwind(){
     tail call i32 @non_nounwind()
     ret void
@@ -50,9 +42,8 @@ define void @call_non_nounwind(){
 ;     return -1;
 ; }
 
-; CHECK: define i32 @maybe_throw(i1 zeroext)
-; ATTRIBUTOR: define i32 @maybe_throw(i1 zeroext)
-define i32 @maybe_throw(i1 zeroext) {
+; CHECK: define i32 @maybe_throw(i1 zeroext %0)
+define i32 @maybe_throw(i1 zeroext %0) {
   br i1 %0, label %2, label %3
 
 2:                                                ; preds = %1
@@ -75,7 +66,6 @@ declare void @__cxa_rethrow()
 ; }
 
 ; CHECK: define i32 @catch_thing()
-; ATTRIBUTOR: define i32 @catch_thing()
 define i32 @catch_thing() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
   invoke void @__cxa_rethrow() #1
           to label %1 unwind label %2
@@ -92,6 +82,12 @@ define i32 @catch_thing() personality i8* bitcast (i32 (...)* @__gxx_personality
   ret i32 -1
 }
 
+define i32 @catch_thing_user() {
+  %catch_thing_call = call i32 @catch_thing()
+  ret i32 %catch_thing_call
+}
+
+
 declare i32 @__gxx_personality_v0(...)
 
 declare i8* @__cxa_begin_catch(i8*)
diff --git a/llvm/test/Transforms/FunctionAttrs/readattrs.ll b/llvm/test/Transforms/FunctionAttrs/readattrs.ll
index 0521986bd..b11b3edce 100644
--- a/llvm/test/Transforms/FunctionAttrs/readattrs.ll
+++ b/llvm/test/Transforms/FunctionAttrs/readattrs.ll
@@ -1,5 +1,6 @@
 ; RUN: opt < %s -functionattrs -S | FileCheck %s
 ; RUN: opt < %s -aa-pipeline=basic-aa -passes='cgscc(function-attrs)' -S | FileCheck %s
+
 @x = global i32 0
 
 declare void @test1_1(i8* %x1_1, i8* readonly %y1_1, ...)
@@ -115,18 +116,19 @@ define i32 @volatile_load(i32* %p) {
   ret i32 %load
 }
 
-declare void @escape_readonly_ptr(i8** %addr, i8* readnone %ptr)
-declare void @escape_readnone_ptr(i8** %addr, i8* readonly %ptr)
+declare void @escape_readnone_ptr(i8** %addr, i8* readnone %ptr)
+declare void @escape_readonly_ptr(i8** %addr, i8* readonly %ptr)
 
 ; The argument pointer %escaped_then_written cannot be marked readnone/only even
 ; though the only direct use, in @escape_readnone_ptr/@escape_readonly_ptr,
 ; is marked as readnone/only. However, the functions can write the pointer into
 ; %addr, causing the store to write to %escaped_then_written.
 ;
-; FIXME: This test currently exposes a bug!
+; FIXME: This test currently exposes a bug in functionattrs!
+;
+; CHECK: define void @unsound_readnone(i8* nocapture readnone %ignored, i8* readnone %escaped_then_written)
+; CHECK: define void @unsound_readonly(i8* nocapture readnone %ignored, i8* readonly %escaped_then_written)
 ;
-; BUG: define void @unsound_readnone(i8* %ignored, i8* readnone %escaped_then_written)
-; BUG: define void @unsound_readonly(i8* %ignored, i8* readonly %escaped_then_written)
 define void @unsound_readnone(i8* %ignored, i8* %escaped_then_written) {
   %addr = alloca i8*
   call void @escape_readnone_ptr(i8** %addr, i8* %escaped_then_written)
diff --git a/llvm/test/Transforms/FunctionAttrs/readnone.ll b/llvm/test/Transforms/FunctionAttrs/readnone.ll
index b5a5b30ae..b18aab539 100644
--- a/llvm/test/Transforms/FunctionAttrs/readnone.ll
+++ b/llvm/test/Transforms/FunctionAttrs/readnone.ll
@@ -1,14 +1,14 @@
 ; RUN: opt < %s -functionattrs -S | FileCheck %s
 ; RUN: opt < %s -passes=function-attrs -S | FileCheck %s
 
-; CHECK: define void @bar(i8* nocapture readnone)
-define void @bar(i8* readonly) {
+; CHECK: define void @bar(i8* nocapture readnone %0)
+define void @bar(i8* readonly %0) {
   call void @foo(i8* %0)
     ret void
 }
 
-; CHECK: define void @foo(i8* nocapture readnone)
-define void @foo(i8* readonly) {
+; CHECK: define void @foo(i8* nocapture readnone %0)
+define void @foo(i8* readonly %0) {
   call void @bar(i8* %0)
   ret void
 }
diff --git a/llvm/test/Transforms/FunctionAttrs/writeonly.ll b/llvm/test/Transforms/FunctionAttrs/writeonly.ll
new file mode 100644
index 000000000..6514cd9d1
--- /dev/null
+++ b/llvm/test/Transforms/FunctionAttrs/writeonly.ll
@@ -0,0 +1,30 @@
+; RUN: opt < %s -functionattrs         -S | FileCheck %s
+; RUN: opt < %s -passes=function-attrs -S | FileCheck %s
+
+; CHECK: define void @nouses-argworn-funrn(i32* nocapture readnone %.aaa) #0 {
+define void @nouses-argworn-funrn(i32* writeonly %.aaa) {
+nouses-argworn-funrn_entry:
+  ret void
+}
+
+; CHECK: define void @nouses-argworn-funro(i32* nocapture readnone %.aaa, i32* nocapture readonly %.bbb) #1 {
+define void @nouses-argworn-funro(i32* writeonly %.aaa, i32* %.bbb) {
+nouses-argworn-funro_entry:
+  %val = load i32 , i32* %.bbb
+  ret void
+}
+
+%_type_of_d-ccc = type <{ i8*, i8, i8, i8, i8 }>
+
+@d-ccc = internal global %_type_of_d-ccc <{ i8* null, i8 1, i8 13, i8 0, i8 -127 }>, align 8
+
+; CHECK: define void @nouses-argworn-funwo(i32* nocapture readnone %.aaa) #2 {
+define void @nouses-argworn-funwo(i32* writeonly %.aaa) {
+nouses-argworn-funwo_entry:
+  store i8 0, i8* getelementptr inbounds (%_type_of_d-ccc, %_type_of_d-ccc* @d-ccc, i32 0, i32 3)
+  ret void
+}
+
+; CHECK: attributes #0 = { {{.*}} readnone }
+; CHECK: attributes #1 = { {{.*}} readonly }
+; CHECK: attributes #2 = { {{.*}} writeonly }
diff --git a/llvm/test/Transforms/FunctionImport/Inputs/comdat.ll b/llvm/test/Transforms/FunctionImport/Inputs/comdat.ll
index 1df6f2535..3eac3e319 100644
--- a/llvm/test/Transforms/FunctionImport/Inputs/comdat.ll
+++ b/llvm/test/Transforms/FunctionImport/Inputs/comdat.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:w-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-pc-windows-msvc19.0.24215"
 
 define void @main() {
diff --git a/llvm/test/Transforms/FunctionImport/Inputs/funcimport_debug.ll b/llvm/test/Transforms/FunctionImport/Inputs/funcimport_debug.ll
index f553af418..4d3a24135 100644
--- a/llvm/test/Transforms/FunctionImport/Inputs/funcimport_debug.ll
+++ b/llvm/test/Transforms/FunctionImport/Inputs/funcimport_debug.ll
@@ -1,5 +1,5 @@
 ; ModuleID = 'funcimport_debug.o'
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 ; Function Attrs: nounwind uwtable
@@ -8,7 +8,7 @@ entry:
     ret void, !dbg !10
 }
 
-attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!7, !8}
diff --git a/llvm/test/Transforms/FunctionImport/Inputs/funcimport_resolved1.ll b/llvm/test/Transforms/FunctionImport/Inputs/funcimport_resolved1.ll
index 2b2443c96..a2ea145f2 100644
--- a/llvm/test/Transforms/FunctionImport/Inputs/funcimport_resolved1.ll
+++ b/llvm/test/Transforms/FunctionImport/Inputs/funcimport_resolved1.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.11.0"
 
 define void @foo() {
diff --git a/llvm/test/Transforms/FunctionImport/Inputs/funcimport_resolved2.ll b/llvm/test/Transforms/FunctionImport/Inputs/funcimport_resolved2.ll
index 278a7f455..78da8b599 100644
--- a/llvm/test/Transforms/FunctionImport/Inputs/funcimport_resolved2.ll
+++ b/llvm/test/Transforms/FunctionImport/Inputs/funcimport_resolved2.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.11.0"
 
 define linkonce_odr void @linkonceodrfunc() {
diff --git a/llvm/test/Transforms/FunctionImport/Inputs/funcimport_var2.ll b/llvm/test/Transforms/FunctionImport/Inputs/funcimport_var2.ll
index 95abe658e..5a6240aa4 100644
--- a/llvm/test/Transforms/FunctionImport/Inputs/funcimport_var2.ll
+++ b/llvm/test/Transforms/FunctionImport/Inputs/funcimport_var2.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @link = internal global i32 0, align 4
diff --git a/llvm/test/Transforms/FunctionImport/Inputs/hotness_based_import.ll b/llvm/test/Transforms/FunctionImport/Inputs/hotness_based_import.ll
index 6951b6581..3ee74a25e 100644
--- a/llvm/test/Transforms/FunctionImport/Inputs/hotness_based_import.ll
+++ b/llvm/test/Transforms/FunctionImport/Inputs/hotness_based_import.ll
@@ -1,5 +1,5 @@
 ; ModuleID = 'thinlto-function-summary-callgraph-profile-summary2.ll'
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 
diff --git a/llvm/test/Transforms/FunctionImport/Inputs/import_stats.ll b/llvm/test/Transforms/FunctionImport/Inputs/import_stats.ll
index 818fbf20d..413f6cea5 100644
--- a/llvm/test/Transforms/FunctionImport/Inputs/import_stats.ll
+++ b/llvm/test/Transforms/FunctionImport/Inputs/import_stats.ll
@@ -1,5 +1,5 @@
 ; ModuleID = 'import_stats2.ll'
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @globalvar = global i32 1, align 4
diff --git a/llvm/test/Transforms/FunctionImport/Inputs/not-prevailing.ll b/llvm/test/Transforms/FunctionImport/Inputs/not-prevailing.ll
index ca17d7f37..a3f1f070f 100644
--- a/llvm/test/Transforms/FunctionImport/Inputs/not-prevailing.ll
+++ b/llvm/test/Transforms/FunctionImport/Inputs/not-prevailing.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define weak i32 @foo() {
diff --git a/llvm/test/Transforms/FunctionImport/comdat.ll b/llvm/test/Transforms/FunctionImport/comdat.ll
index 29e8cb538..5e6643f64 100644
--- a/llvm/test/Transforms/FunctionImport/comdat.ll
+++ b/llvm/test/Transforms/FunctionImport/comdat.ll
@@ -11,7 +11,7 @@
 ; RUN:          -r %t2.bc,lwt_fun,
 ; RUN: llvm-dis -o - %t3.1.3.import.bc | FileCheck %s
 
-target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:w-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-pc-windows-msvc19.0.24215"
 
 ; CHECK: $lwt.llvm.[[HASH:[0-9]+]] = comdat any
diff --git a/llvm/test/Transforms/FunctionImport/funcimport_debug.ll b/llvm/test/Transforms/FunctionImport/funcimport_debug.ll
index e764d78c3..1cf766607 100644
--- a/llvm/test/Transforms/FunctionImport/funcimport_debug.ll
+++ b/llvm/test/Transforms/FunctionImport/funcimport_debug.ll
@@ -20,7 +20,7 @@
 ; CHECK-NOT: distinct !DISubprogram
 
 ; ModuleID = 'funcimport_debug.o'
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 ; Function Attrs: nounwind uwtable
@@ -32,8 +32,8 @@ entry:
 
 declare void @func(...) #1
 
-attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!8, !9}
diff --git a/llvm/test/Transforms/FunctionImport/funcimport_resolved.ll b/llvm/test/Transforms/FunctionImport/funcimport_resolved.ll
index b256a6136..2a06be4db 100644
--- a/llvm/test/Transforms/FunctionImport/funcimport_resolved.ll
+++ b/llvm/test/Transforms/FunctionImport/funcimport_resolved.ll
@@ -38,7 +38,7 @@
 ; INSTLIM8: Is importing function {{.*}} f from {{.*}}funcimport_resolved1.ll
 ; INSTLIM8-NOT: Is importing function {{.*}} linkonceodrfunc from {{.*}}funcimport_resolved2.ll
 
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.11.0"
 
 define i32 @main() #0 {
diff --git a/llvm/test/Transforms/FunctionImport/funcimport_var.ll b/llvm/test/Transforms/FunctionImport/funcimport_var.ll
index edd874e62..0ed644242 100644
--- a/llvm/test/Transforms/FunctionImport/funcimport_var.ll
+++ b/llvm/test/Transforms/FunctionImport/funcimport_var.ll
@@ -15,7 +15,7 @@
 
 ; REQUIRES: x86-registered-target
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define i32 @_Z4LinkPKcS0_(i8*, i8*) local_unnamed_addr  {
diff --git a/llvm/test/Transforms/FunctionImport/hotness_based_import.ll b/llvm/test/Transforms/FunctionImport/hotness_based_import.ll
index 9de871407..bd90360dc 100644
--- a/llvm/test/Transforms/FunctionImport/hotness_based_import.ll
+++ b/llvm/test/Transforms/FunctionImport/hotness_based_import.ll
@@ -74,7 +74,7 @@
 
 
 ; ModuleID = 'thinlto-function-summary-callgraph.ll'
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 ; This function have high profile count, so entry block is hot.
diff --git a/llvm/test/Transforms/FunctionImport/import_stats.ll b/llvm/test/Transforms/FunctionImport/import_stats.ll
index 2cb415d1e..50d2167c6 100644
--- a/llvm/test/Transforms/FunctionImport/import_stats.ll
+++ b/llvm/test/Transforms/FunctionImport/import_stats.ll
@@ -28,7 +28,7 @@
 ; THINLINKSTATS-DAG: 1 function-import  - Number of hot functions thin link decided to import
 
 ; ModuleID = 'import_stats.ll'
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 ; This function has a high profile count, so entry block is hot.
diff --git a/llvm/test/Transforms/FunctionImport/not-prevailing.ll b/llvm/test/Transforms/FunctionImport/not-prevailing.ll
index 4412715e0..0e0dd5395 100644
--- a/llvm/test/Transforms/FunctionImport/not-prevailing.ll
+++ b/llvm/test/Transforms/FunctionImport/not-prevailing.ll
@@ -5,7 +5,7 @@
 
 ; CHECK: Interposable and available_externally/linkonce_odr/weak_odr symbol
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define available_externally i32 @foo() {
diff --git a/llvm/test/Transforms/GCOVProfiling/global-ctor.ll.x b/llvm/test/Transforms/GCOVProfiling/global-ctor.ll.x
index 4f9d2e894..7df4f6a51 100644
--- a/llvm/test/Transforms/GCOVProfiling/global-ctor.ll.x
+++ b/llvm/test/Transforms/GCOVProfiling/global-ctor.ll.x
@@ -36,7 +36,7 @@ entry:
 }
 
 attributes #0 = { nounwind }
-attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!10, !11}
diff --git a/llvm/test/Transforms/GCOVProfiling/linezero.ll b/llvm/test/Transforms/GCOVProfiling/linezero.ll
index 7c084a3ec..e609e8800 100644
--- a/llvm/test/Transforms/GCOVProfiling/linezero.ll
+++ b/llvm/test/Transforms/GCOVProfiling/linezero.ll
@@ -88,9 +88,9 @@ entry:
   ret void, !dbg !45
 }
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
-attributes #2 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #3 = { noreturn nounwind }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/Transforms/GCOVProfiling/return-block.ll.x b/llvm/test/Transforms/GCOVProfiling/return-block.ll.x
index 74812d14c..ce6651488 100644
--- a/llvm/test/Transforms/GCOVProfiling/return-block.ll.x
+++ b/llvm/test/Transforms/GCOVProfiling/return-block.ll.x
@@ -45,8 +45,8 @@ declare void @f(...) #1
 
 declare void @g(...) #1
 
-attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind }
 
 !llvm.gcov = !{!19}
diff --git a/llvm/test/Transforms/GVN/PRE/2011-06-01-NonLocalMemdepMiscompile.ll b/llvm/test/Transforms/GVN/PRE/2011-06-01-NonLocalMemdepMiscompile.ll
index 05dc79db9..6592c69b0 100644
--- a/llvm/test/Transforms/GVN/PRE/2011-06-01-NonLocalMemdepMiscompile.ll
+++ b/llvm/test/Transforms/GVN/PRE/2011-06-01-NonLocalMemdepMiscompile.ll
@@ -50,8 +50,14 @@ bb15:
   %tmp18 = icmp eq i8 %tmp17, 0
   br label %bb19
 
-; CHECK: bb15:
-; CHECK: %tmp17 = phi i8 [ %tmp17.pre, %bb1.bb15_crit_edge ], [ %tmp8, %bb6 ]
+; CHECK-LABEL: bb6:
+; CHECK:         br i1 undef, label %bb15split, label %bb10
+
+; CHECK-LABEL: bb15split:                                        ; preds = %bb6
+; CHECK-NEXT:    br label %bb15
+
+; CHECK-LABEL: bb15:
+; CHECK:         %tmp17 = phi i8 [ %tmp8, %bb15split ], [ %tmp17.pre, %bb1.bb15_crit_edge ]
 
 bb19:                                             ; preds = %bb15
   ret i1 %tmp18
diff --git a/llvm/test/Transforms/GVN/PRE/rle.ll b/llvm/test/Transforms/GVN/PRE/rle.ll
index 5ff2927ed..a461be6f4 100644
--- a/llvm/test/Transforms/GVN/PRE/rle.ll
+++ b/llvm/test/Transforms/GVN/PRE/rle.ll
@@ -546,6 +546,130 @@ out:
   ret i8 %R
 }
 
+declare void @use_i32(i32) readonly
+
+; indirectbr currently prevents MergeBlockIntoPredecessor from merging latch
+; into header. Make sure we translate the address for %l1 correctly where
+; parts of the address computations are in different basic blocks.
+define i32 @phi_trans6(i32* noalias nocapture readonly %x, i1 %cond) {
+; CHECK-LABEL: define i32 @phi_trans6(
+; CHECK-LABEL: entry:
+; CHECK-NEXT:   %l0 = load i32, i32* %x
+;
+; CHECK-LABEL: header:
+; CHECK-NEXT:    %l1 = phi i32 [ %l0, %entry ], [ %l1.pre, %latch.header_crit_edge ]
+; CHECK-NEXT:    %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch.header_crit_edge ]
+; CHECK-NEXT:    indirectbr i8* blockaddress(@phi_trans6, %latch), [label %latch]
+;
+; CHECK-LABEL: latch:
+; CHECK-NEXT:    %iv.next = add i32 %iv, 1
+; CHECK-NEXT:    br i1 %cond, label %exit, label %latch.header_crit_edge
+;
+; CHECK-LABEL: latch.header_crit_edge:
+; CHECK-NEXT:    %gep.1.phi.trans.insert.phi.trans.insert = getelementptr i32, i32* %x, i32 %iv.next
+; CHECK-NEXT:    %l1.pre = load i32, i32* %gep.1.phi.trans.insert.phi.trans.insert
+; CHECK-LABEL:   br label %header
+;
+entry:
+  %l0 = load i32, i32* %x
+  call void @use_i32(i32 %l0)
+  br label %header
+
+header:
+  %iv = phi i32 [0, %entry], [ %iv.next, %latch]
+  indirectbr i8* blockaddress(@phi_trans6, %latch), [label %latch]
+
+latch:
+  %gep.1 = getelementptr i32, i32* %x, i32 %iv
+  %l1 = load i32, i32* %gep.1
+  %iv.next = add i32 %iv, 1
+  br i1 %cond, label %exit, label %header
+
+exit:
+  ret i32 %l1
+}
+
+; FIXME: Currently we fail to translate the PHI in this case.
+define i32 @phi_trans7(i32* noalias nocapture readonly %x, i1 %cond) {
+; CHECK-LABEL: define i32 @phi_trans7(
+; CHECK-LABEL: entry:
+; CHECK-NEXT:   %l0 = load i32, i32* %x
+;
+; CHECK-LABEL: header:
+; CHECK-NEXT:    %iv = phi i32 [ 2, %entry ], [ %iv.next, %latch.header_crit_edge ]
+; CHECK-NEXT:    %offset = add i32 %iv, -2
+; CHECK-NEXT:    indirectbr i8* blockaddress(@phi_trans7, %latch), [label %latch]
+;
+; CHECK-LABEL: latch:
+; CHECK-NEXT:    %gep.1 = getelementptr i32, i32* %x, i32 %offset
+; CHECK-NEXT:    %l1 = load i32, i32* %gep.1
+; CHECK-NEXT:    %iv.next = add i32 %iv, 1
+; CHECK-NEXT:    br i1 %cond, label %exit, label %latch.header_crit_edge
+;
+; CHECK-LABEL: latch.header_crit_edge:
+; CHECK-LABEL:   br label %header
+;
+entry:
+  %l0 = load i32, i32* %x
+  call void @use_i32(i32 %l0)
+  br label %header
+
+header:
+  %iv = phi i32 [2, %entry], [ %iv.next, %latch]
+  %offset = add i32 %iv, -2
+  indirectbr i8* blockaddress(@phi_trans7, %latch), [label %latch]
+
+latch:
+  %gep.1 = getelementptr i32, i32* %x, i32 %offset
+  %l1 = load i32, i32* %gep.1
+  %iv.next = add i32 %iv, 1
+  br i1 %cond, label %exit, label %header
+
+exit:
+  ret i32 %l1
+}
+
+; FIXME: Currently we fail to translate the PHI in this case.
+define i32 @phi_trans8(i32* noalias nocapture readonly %x, i1 %cond) {
+; CHECK-LABEL: define i32 @phi_trans8(
+; CHECK-LABEL: entry:
+; CHECK-NEXT:   %l0 = load i32, i32* %x
+;
+; CHECK-LABEL: header:
+; CHECK-NEXT:    %iv = phi i32 [ 2, %entry ], [ %iv.next, %latch.header_crit_edge ]
+; CHECK-NEXT:    indirectbr i8* blockaddress(@phi_trans8, %latch), [label %latch]
+;
+; CHECK-LABEL: latch:
+; CHECK-NEXT:    %offset = add i32 %iv, -2
+; CHECK-NEXT:    %gep.1 = getelementptr i32, i32* %x, i32 %offset
+; CHECK-NEXT:    %l1 = load i32, i32* %gep.1
+; CHECK-NEXT:    %iv.next = add i32 %iv, 1
+; CHECK-NEXT:    br i1 %cond, label %exit, label %latch.header_crit_edge
+;
+; CHECK-LABEL: latch.header_crit_edge:
+; CHECK-LABEL:   br label %header
+;
+entry:
+  %l0 = load i32, i32* %x
+  call void @use_i32(i32 %l0)
+  br label %header
+
+header:
+  %iv = phi i32 [2, %entry], [ %iv.next, %latch]
+  indirectbr i8* blockaddress(@phi_trans8, %latch), [label %latch]
+
+latch:
+  %offset = add i32 %iv, -2
+  %gep.1 = getelementptr i32, i32* %x, i32 %offset
+  %l1 = load i32, i32* %gep.1
+  %iv.next = add i32 %iv, 1
+  br i1 %cond, label %exit, label %header
+
+exit:
+  ret i32 %l1
+}
+
+
 
 ; PR6642
 define i32 @memset_to_load() nounwind readnone {
@@ -661,6 +785,7 @@ entry:
 ; CHECK: ret i32
 }
 
+
 declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i1) nounwind
 
 declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i1) nounwind
diff --git a/llvm/test/Transforms/GVN/cond_br2.ll b/llvm/test/Transforms/GVN/cond_br2.ll
index a3749510c..557ae5363 100644
--- a/llvm/test/Transforms/GVN/cond_br2.ll
+++ b/llvm/test/Transforms/GVN/cond_br2.ll
@@ -127,10 +127,10 @@ declare void @_ZN4llvm15SmallVectorBase8grow_podEmm(%"class.llvm::SmallVectorBas
 ; Function Attrs: nounwind
 declare void @free(i8* nocapture) #3
 
-attributes #0 = { ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { ssp uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind }
-attributes #2 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #3 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !0 = !{!"any pointer", !1}
 !1 = !{!"omnipotent char", !2}
diff --git a/llvm/test/Transforms/GVN/edge.ll b/llvm/test/Transforms/GVN/edge.ll
index 0c1a3fbec..1d9407b78 100644
--- a/llvm/test/Transforms/GVN/edge.ll
+++ b/llvm/test/Transforms/GVN/edge.ll
@@ -93,6 +93,40 @@ return:
 ; CHECK: %div = fdiv double %x, 2.0
 }
 
+define double @fcmp_one_possibly_nan(double %x, double %y) {
+entry:
+  %cmp = fcmp one double %y, 2.0
+  br i1 %cmp, label %return, label %else
+
+else:
+  %div = fdiv double %x, %y
+  br label %return
+
+return:
+  %retval = phi double [ %div, %else ], [ %x, %entry ]
+  ret double %retval
+
+; CHECK-LABEL: define double @fcmp_one_possibly_nan(
+; CHECK: %div = fdiv double %x, %y
+}
+
+define double @fcmp_one_not_zero_or_nan(double %x, double %y) {
+entry:
+  %cmp = fcmp nnan one double %y, 2.0
+  br i1 %cmp, label %return, label %else
+
+else:
+  %div = fdiv double %x, %y
+  br label %return
+
+return:
+  %retval = phi double [ %div, %else ], [ %x, %entry ]
+  ret double %retval
+
+; CHECK-LABEL: define double @fcmp_one_not_zero_or_nan(
+; CHECK: %div = fdiv double %x, 2.0
+}
+
 ; PR22376 - We can't propagate zero constants because -0.0 
 ; compares equal to 0.0. If %y is -0.0 in this test case,
 ; we would produce the wrong sign on the infinity return value.
@@ -168,3 +202,38 @@ return:
 ; CHECK-LABEL: define double @fcmp_une_maybe_zero(
 ; CHECK: %div = fdiv double %x, %z
 }
+
+
+define double @fcmp_ueq_possibly_nan(double %x, double %y) {
+entry:
+  %cmp = fcmp ueq double %y, 2.0
+  br i1 %cmp, label %do_div, label %return
+
+do_div:
+  %div = fdiv double %x, %y
+  br label %return
+
+return:
+  %retval = phi double [ %div, %do_div ], [ %x, %entry ]
+  ret double %retval
+
+; CHECK-LABEL: define double @fcmp_ueq_possibly_nan(
+; CHECK: %div = fdiv double %x, %y
+}
+
+define double @fcmp_ueq_not_zero_or_nan(double %x, double %y) {
+entry:
+  %cmp = fcmp nnan ueq double %y, 2.0
+  br i1 %cmp, label %do_div, label %return
+
+do_div:
+  %div = fdiv double %x, %y
+  br label %return
+
+return:
+  %retval = phi double [ %div, %do_div ], [ %x, %entry ]
+  ret double %retval
+
+; CHECK-LABEL: define double @fcmp_ueq_not_zero_or_nan(
+; CHECK: %div = fdiv double %x, 2.0
+}
diff --git a/llvm/test/Transforms/GVN/equality-assume.ll b/llvm/test/Transforms/GVN/equality-assume.ll
new file mode 100644
index 000000000..ee2cb06c1
--- /dev/null
+++ b/llvm/test/Transforms/GVN/equality-assume.ll
@@ -0,0 +1,168 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -gvn < %s | FileCheck %s
+
+define i32 @test(i32* %p, i32 %v) {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:    [[LOAD:%.*]] = load i32, i32* [[P:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i32 [[LOAD]], [[V:%.*]]
+; CHECK-NEXT:    call void @llvm.assume(i1 [[C]])
+; CHECK-NEXT:    ret i32 [[V]]
+;
+  %load = load i32, i32* %p
+  %c = icmp eq i32 %load, %v
+  call void @llvm.assume(i1 %c)
+  ret i32 %load
+}
+
+define i32 @reverse(i32* %p, i32 %v) {
+; CHECK-LABEL: @reverse(
+; CHECK-NEXT:    [[LOAD:%.*]] = load i32, i32* [[P:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i32 [[LOAD]], [[V:%.*]]
+; CHECK-NEXT:    call void @llvm.assume(i1 [[C]])
+; CHECK-NEXT:    ret i32 [[V]]
+;
+  %load = load i32, i32* %p
+  %c = icmp eq i32 %load, %v
+  call void @llvm.assume(i1 %c)
+  ret i32 %v
+}
+
+; Lack of equivalance due to +0.0 vs -0.0
+define float @neg_float_oeq(float* %p, float %v) {
+; CHECK-LABEL: @neg_float_oeq(
+; CHECK-NEXT:    [[LOAD:%.*]] = load float, float* [[P:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = fcmp oeq float [[LOAD]], [[V:%.*]]
+; CHECK-NEXT:    call void @llvm.assume(i1 [[C]])
+; CHECK-NEXT:    ret float [[LOAD]]
+;
+  %load = load float, float* %p
+  %c = fcmp oeq float %load, %v
+  call void @llvm.assume(i1 %c)
+  ret float %load
+}
+
+; Lack of equivalance due to +0.0 vs -0.0
+define float @neg_float_ueq(float* %p, float %v) {
+; CHECK-LABEL: @neg_float_ueq(
+; CHECK-NEXT:    [[LOAD:%.*]] = load float, float* [[P:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = fcmp ueq float [[LOAD]], [[V:%.*]]
+; CHECK-NEXT:    call void @llvm.assume(i1 [[C]])
+; CHECK-NEXT:    ret float [[LOAD]]
+;
+  %load = load float, float* %p
+  %c = fcmp ueq float %load, %v
+  call void @llvm.assume(i1 %c)
+  ret float %load
+}
+
+define float @float_oeq_constant(float* %p) {
+; CHECK-LABEL: @float_oeq_constant(
+; CHECK-NEXT:    [[LOAD:%.*]] = load float, float* [[P:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = fcmp oeq float [[LOAD]], 5.000000e+00
+; CHECK-NEXT:    call void @llvm.assume(i1 [[C]])
+; CHECK-NEXT:    ret float 5.000000e+00
+;
+  %load = load float, float* %p
+  %c = fcmp oeq float %load, 5.0
+  call void @llvm.assume(i1 %c)
+  ret float %load
+}
+
+; Lack of equivalance due to Nan
+define float @neq_float_ueq_constant(float* %p) {
+; CHECK-LABEL: @neq_float_ueq_constant(
+; CHECK-NEXT:    [[LOAD:%.*]] = load float, float* [[P:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = fcmp ueq float [[LOAD]], 5.000000e+00
+; CHECK-NEXT:    call void @llvm.assume(i1 [[C]])
+; CHECK-NEXT:    ret float [[LOAD]]
+;
+  %load = load float, float* %p
+  %c = fcmp ueq float %load, 5.0
+  call void @llvm.assume(i1 %c)
+  ret float %load
+}
+
+define float @float_ueq_constant_nnas(float* %p) {
+; CHECK-LABEL: @float_ueq_constant_nnas(
+; CHECK-NEXT:    [[LOAD:%.*]] = load float, float* [[P:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = fcmp nnan ueq float [[LOAD]], 5.000000e+00
+; CHECK-NEXT:    call void @llvm.assume(i1 [[C]])
+; CHECK-NEXT:    ret float 5.000000e+00
+;
+  %load = load float, float* %p
+  %c = fcmp nnan ueq float %load, 5.0
+  call void @llvm.assume(i1 %c)
+  ret float %load
+}
+
+define i32 @test2(i32* %p, i32 %v) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[LOAD:%.*]] = load i32, i32* [[P:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i32 [[LOAD]], [[V:%.*]]
+; CHECK-NEXT:    call void @llvm.assume(i1 [[C]])
+; CHECK-NEXT:    ret i32 [[V]]
+;
+  %load = load i32, i32* %p
+  %c = icmp eq i32 %load, %v
+  call void @llvm.assume(i1 %c)
+  %load2 = load i32, i32* %p
+  ret i32 %load2
+}
+
+define i32 @test3(i32* %p, i32 %v) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    [[LOAD:%.*]] = load i32, i32* [[P:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i32 [[LOAD]], [[V:%.*]]
+; CHECK-NEXT:    call void @llvm.assume(i1 [[C]])
+; CHECK-NEXT:    br i1 undef, label [[TAKEN:%.*]], label [[MERGE:%.*]]
+; CHECK:       taken:
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       merge:
+; CHECK-NEXT:    ret i32 [[V]]
+;
+  %load = load i32, i32* %p
+  %c = icmp eq i32 %load, %v
+  call void @llvm.assume(i1 %c)
+  br i1 undef, label %taken, label %merge
+taken:
+  br label %merge
+merge:
+  ret i32 %load
+}
+
+define i32 @trivial_constants(i32* %p) {
+; CHECK-LABEL: @trivial_constants(
+; CHECK-NEXT:    br i1 undef, label [[TAKEN:%.*]], label [[MERGE:%.*]]
+; CHECK:       taken:
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       merge:
+; CHECK-NEXT:    ret i32 0
+;
+  %c = icmp eq i32 0, 0
+  call void @llvm.assume(i1 %c)
+  br i1 undef, label %taken, label %merge
+taken:
+  br label %merge
+merge:
+  ret i32 0
+}
+
+define i32 @conflicting_constants(i32* %p) {
+; CHECK-LABEL: @conflicting_constants(
+; CHECK-NEXT:    store i8 undef, i8* null
+; CHECK-NEXT:    br i1 undef, label [[TAKEN:%.*]], label [[MERGE:%.*]]
+; CHECK:       taken:
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       merge:
+; CHECK-NEXT:    ret i32 1
+;
+  %c = icmp eq i32 0, 5
+  call void @llvm.assume(i1 %c)
+  br i1 undef, label %taken, label %merge
+taken:
+  br label %merge
+merge:
+  ret i32 1
+}
+
+declare void @llvm.assume(i1)
diff --git a/llvm/test/Transforms/GVN/pr42605.ll b/llvm/test/Transforms/GVN/pr42605.ll
new file mode 100644
index 000000000..542f6f9c0
--- /dev/null
+++ b/llvm/test/Transforms/GVN/pr42605.ll
@@ -0,0 +1,87 @@
+; RUN: opt -gvn %s -S | FileCheck %s
+; PR42605. Check phi-translate won't translate the value number of a call
+; to the value of another call with clobber in between.
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@global = dso_local local_unnamed_addr global i32 0, align 4
+@.str = private unnamed_addr constant [8 x i8] c"%d, %d\0A\00", align 1
+
+; Function Attrs: nofree nounwind
+declare dso_local i32 @printf(i8* nocapture readonly, ...) local_unnamed_addr
+
+; Function Attrs: noinline norecurse nounwind readonly uwtable
+define dso_local i32 @_Z3gooi(i32 %i) local_unnamed_addr #0 {
+entry:
+  %t0 = load i32, i32* @global, align 4, !tbaa !2
+  %add = add nsw i32 %t0, %i
+  ret i32 %add
+}
+
+; Function Attrs: nofree nounwind uwtable
+define dso_local void @noclobber() local_unnamed_addr {
+entry:
+  %call = tail call i32 @_Z3gooi(i32 2)
+  %add = add nsw i32 %call, 5
+  %cmp = icmp sgt i32 %add, 2
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %call1 = tail call i32 @_Z3gooi(i32 3)
+  %add2 = add nsw i32 %call1, 5
+  br label %if.end
+
+; Check pre happens after phitranslate.
+; CHECK-LABEL: @noclobber
+; CHECK: %add4.pre-phi = phi i32 [ %add2, %if.then ], [ %add, %entry ]
+; CHECK: printf(i8* getelementptr inbounds {{.*}}, i32 %add4.pre-phi)
+
+if.end:                                           ; preds = %if.then, %entry
+  %i.0 = phi i32 [ 3, %if.then ], [ 2, %entry ]
+  %global2.0 = phi i32 [ %add2, %if.then ], [ %add, %entry ]
+  %call3 = tail call i32 @_Z3gooi(i32 %i.0)
+  %add4 = add nsw i32 %call3, 5
+  %call5 = tail call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([8 x i8], [8 x i8]* @.str, i64 0, i64 0), i32 %global2.0, i32 %add4)
+  ret void
+}
+
+; Function Attrs: nofree nounwind uwtable
+define dso_local void @hasclobber() local_unnamed_addr {
+entry:
+  %call = tail call i32 @_Z3gooi(i32 2)
+  %add = add nsw i32 %call, 5
+  %cmp = icmp sgt i32 %add, 2
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %call1 = tail call i32 @_Z3gooi(i32 3)
+  %add2 = add nsw i32 %call1, 5
+  br label %if.end
+
+; Check no pre happens.
+; CHECK-LABEL: @hasclobber
+; CHECK: %call3 = tail call i32 @_Z3gooi(i32 %i.0)
+; CHECK-NEXT: %add4 = add nsw i32 %call3, 5
+; CHECK-NEXT: printf(i8* getelementptr inbounds ({{.*}}, i32 %global2.0, i32 %add4)
+
+if.end:                                           ; preds = %if.then, %entry
+  %i.0 = phi i32 [ 3, %if.then ], [ 2, %entry ]
+  %global2.0 = phi i32 [ %add2, %if.then ], [ %add, %entry ]
+  store i32 5, i32* @global, align 4, !tbaa !2
+  %call3 = tail call i32 @_Z3gooi(i32 %i.0)
+  %add4 = add nsw i32 %call3, 5
+  %call5 = tail call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([8 x i8], [8 x i8]* @.str, i64 0, i64 0), i32 %global2.0, i32 %add4)
+  ret void
+}
+
+attributes #0 = { noinline norecurse nounwind readonly uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.module.flags = !{!0}
+!llvm.ident = !{!1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{!"clang version 10.0.0 (trunk 369798)"}
+!2 = !{!3, !3, i64 0}
+!3 = !{!"int", !4, i64 0}
+!4 = !{!"omnipotent char", !5, i64 0}
+!5 = !{!"Simple C++ TBAA"}
diff --git a/llvm/test/Transforms/GVN/preserve-analysis.ll b/llvm/test/Transforms/GVN/preserve-analysis.ll
new file mode 100644
index 000000000..2454bb1a6
--- /dev/null
+++ b/llvm/test/Transforms/GVN/preserve-analysis.ll
@@ -0,0 +1,56 @@
+; RUN: opt < %s -debug-pass=Structure -indvars -gvn -indvars 2>&1 -S | FileCheck --check-prefix=CHECK --check-prefix=IR %s
+; RUN: opt < %s -debug-pass-manager -passes='require<domtree>,loop(simplify-cfg),gvn,loop(indvars)' 2>&1 -S | FileCheck --check-prefix=NEW-PM --check-prefix=IR %s
+
+; Check CFG-only analysis are preserved by SCCP by running it between 2
+; loop-vectorize runs.
+
+; CHECK: Dominator Tree Construction
+; CHECK: Natural Loop Information
+; CHECK: Canonicalize natural loops
+; CHECK: LCSSA Verifier
+; CHECK: Loop-Closed SSA Form Pass
+; CHECK: Global Value Numbering
+; CHECK-NOT: Dominator Tree Construction
+; CHECK-NOT: Natural Loop Information
+; CHECK-NOT: Canonicalize natural loops
+
+; NEW-PM-DAG: Running analysis: LoopAnalysis on test
+; NEW-PM-DAG: Running analysis: DominatorTreeAnalysis on test
+; NEW-PM: Running pass: GVN on test
+; NEW-PM-NOT: Running analysis: LoopAnalysis on test
+; NEW-PM-NOT: Running analysis: DominatorTreeAnalysis on test
+
+declare i1 @cond()
+declare void @dostuff()
+
+define i32 @test() {
+; IR-LABEL: define i32 @test()
+; IR-LABEL: header:
+; IR:         br i1 false, label %then, label %latch
+; IR-LABEL: then:
+; IR-NEXT:   call void @dostuff()
+; IR-NEXT:   br label %latch
+entry:
+  %res = add i32 1, 10
+  br label %header
+
+header:
+  %iv = phi i32 [ %res, %entry ], [ 0, %latch ]
+  %ic = icmp eq i32 %res, 99
+  br i1 %ic, label %then, label %latch
+
+then:
+  br label %then.2
+
+then.2:
+  call void @dostuff()
+  br label %latch
+
+
+latch:
+  %ec = call i1 @cond()
+  br i1 %ec, label %exit, label %header
+
+exit:
+  ret i32 %iv
+}
diff --git a/llvm/test/Transforms/GVNHoist/hoist-unsafe-pr31729.ll b/llvm/test/Transforms/GVNHoist/hoist-unsafe-pr31729.ll
index 654d5b6a5..40a3f4957 100644
--- a/llvm/test/Transforms/GVNHoist/hoist-unsafe-pr31729.ll
+++ b/llvm/test/Transforms/GVNHoist/hoist-unsafe-pr31729.ll
@@ -78,4 +78,4 @@ cleanup:
   ret i32 %retval.0
 }
 
-attributes #0 = { minsize noinline nounwind optsize uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { minsize noinline nounwind optsize uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/Transforms/GVNHoist/pr30499.ll b/llvm/test/Transforms/GVNHoist/pr30499.ll
index 591e0e4bd..deae28647 100644
--- a/llvm/test/Transforms/GVNHoist/pr30499.ll
+++ b/llvm/test/Transforms/GVNHoist/pr30499.ll
@@ -25,6 +25,6 @@ if.end:                                           ; preds = %if.then, %entry
 ; Function Attrs: nounwind readonly
 declare i64 @_Z3fn1v() #1
 
-attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { nounwind readonly "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readonly "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind readonly }
diff --git a/llvm/test/Transforms/GlobalDCE/virtual-functions-base-call.ll b/llvm/test/Transforms/GlobalDCE/virtual-functions-base-call.ll
new file mode 100644
index 000000000..84d95f607
--- /dev/null
+++ b/llvm/test/Transforms/GlobalDCE/virtual-functions-base-call.ll
@@ -0,0 +1,78 @@
+; RUN: opt < %s -globaldce -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+; struct A {
+;   A();
+;   virtual int foo();
+; };
+; 
+; struct B : A {
+;   B();
+;   virtual int foo();
+; };
+; 
+; A::A() {}
+; B::B() {}
+; int A::foo() { return 42; }
+; int B::foo() { return 1337; }
+; 
+; extern "C" int test(A *p) { return p->foo(); }
+
+; The virtual call in test could be dispatched to either A::foo or B::foo, so
+; both must be retained.
+
+%struct.A = type { i32 (...)** }
+%struct.B = type { %struct.A }
+
+; CHECK: @_ZTV1A = internal unnamed_addr constant { [3 x i8*] } { [3 x i8*] [i8* null, i8* null, i8* bitcast (i32 (%struct.A*)* @_ZN1A3fooEv to i8*)] }
+@_ZTV1A = internal unnamed_addr constant { [3 x i8*] } { [3 x i8*] [i8* null, i8* null, i8* bitcast (i32 (%struct.A*)* @_ZN1A3fooEv to i8*)] }, align 8, !type !0, !type !1, !vcall_visibility !2
+
+; CHECK: @_ZTV1B = internal unnamed_addr constant { [3 x i8*] } { [3 x i8*] [i8* null, i8* null, i8* bitcast (i32 (%struct.B*)* @_ZN1B3fooEv to i8*)] }
+@_ZTV1B = internal unnamed_addr constant { [3 x i8*] } { [3 x i8*] [i8* null, i8* null, i8* bitcast (i32 (%struct.B*)* @_ZN1B3fooEv to i8*)] }, align 8, !type !0, !type !1, !type !3, !type !4, !vcall_visibility !2
+
+; CHECK: define internal i32 @_ZN1A3fooEv(
+define internal i32 @_ZN1A3fooEv(%struct.A* nocapture readnone %this) {
+entry:
+  ret i32 42
+}
+
+; CHECK: define internal i32 @_ZN1B3fooEv(
+define internal i32 @_ZN1B3fooEv(%struct.B* nocapture readnone %this) {
+entry:
+  ret i32 1337
+}
+
+define hidden void @_ZN1AC2Ev(%struct.A* nocapture %this) {
+entry:
+  %0 = getelementptr inbounds %struct.A, %struct.A* %this, i64 0, i32 0
+  store i32 (...)** bitcast (i8** getelementptr inbounds ({ [3 x i8*] }, { [3 x i8*] }* @_ZTV1A, i64 0, inrange i32 0, i64 2) to i32 (...)**), i32 (...)*** %0, align 8
+  ret void
+}
+
+define hidden void @_ZN1BC2Ev(%struct.B* nocapture %this) {
+entry:
+  %0 = getelementptr inbounds %struct.B, %struct.B* %this, i64 0, i32 0, i32 0
+  store i32 (...)** bitcast (i8** getelementptr inbounds ({ [3 x i8*] }, { [3 x i8*] }* @_ZTV1B, i64 0, inrange i32 0, i64 2) to i32 (...)**), i32 (...)*** %0, align 8
+  ret void
+}
+
+define hidden i32 @test(%struct.A* %p) {
+entry:
+  %0 = bitcast %struct.A* %p to i8**
+  %vtable1 = load i8*, i8** %0, align 8
+  %1 = tail call { i8*, i1 } @llvm.type.checked.load(i8* %vtable1, i32 0, metadata !"_ZTS1A"), !nosanitize !10
+  %2 = extractvalue { i8*, i1 } %1, 0, !nosanitize !10
+  %3 = bitcast i8* %2 to i32 (%struct.A*)*, !nosanitize !10
+  %call = tail call i32 %3(%struct.A* %p)
+  ret i32 %call
+}
+
+declare { i8*, i1 } @llvm.type.checked.load(i8*, i32, metadata) #2
+
+!0 = !{i64 16, !"_ZTS1A"}
+!1 = !{i64 16, !"_ZTSM1AFivE.virtual"}
+!2 = !{i64 2}
+!3 = !{i64 16, !"_ZTS1B"}
+!4 = !{i64 16, !"_ZTSM1BFivE.virtual"}
+!10 = !{}
diff --git a/llvm/test/Transforms/GlobalDCE/virtual-functions-base-pointer-call.ll b/llvm/test/Transforms/GlobalDCE/virtual-functions-base-pointer-call.ll
new file mode 100644
index 000000000..d498a336a
--- /dev/null
+++ b/llvm/test/Transforms/GlobalDCE/virtual-functions-base-pointer-call.ll
@@ -0,0 +1,118 @@
+; RUN: opt < %s -globaldce -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+; struct A {
+;   A();
+;   virtual int foo(int);
+;   virtual int bar(float);
+; };
+; 
+; struct B : A {
+;   B();
+;   virtual int foo(int);
+;   virtual int bar(float);
+; };
+; 
+; A::A() {}
+; B::B() {}
+; int A::foo(int)   { return 1; }
+; int A::bar(float) { return 2; }
+; int B::foo(int)   { return 3; }
+; int B::bar(float) { return 4; }
+; 
+; extern "C" int test(A *p, int (A::*q)(int)) { return (p->*q)(42); }
+
+; Member function pointers are tracked by the combination of their object type
+; and function type, which must both be compatible. Here, the call is through a
+; pointer of type "int (A::*q)(int)", so the call could be dispatched to A::foo
+; or B::foo. It can't be dispatched to A::bar or B::bar as the function pointer
+; does not match, so those can be removed.
+
+%struct.A = type { i32 (...)** }
+%struct.B = type { %struct.A }
+
+; CHECK: @_ZTV1A = internal unnamed_addr constant { [4 x i8*] } { [4 x i8*] [i8* null, i8* null, i8* bitcast (i32 (%struct.A*, i32)* @_ZN1A3fooEi to i8*), i8* null] }
+@_ZTV1A = internal unnamed_addr constant { [4 x i8*] } { [4 x i8*] [i8* null, i8* null, i8* bitcast (i32 (%struct.A*, i32)* @_ZN1A3fooEi to i8*), i8* bitcast (i32 (%struct.A*, float)* @_ZN1A3barEf to i8*)] }, align 8, !type !0, !type !1, !type !2, !vcall_visibility !3
+; CHECK: @_ZTV1B = internal unnamed_addr constant { [4 x i8*] } { [4 x i8*] [i8* null, i8* null, i8* bitcast (i32 (%struct.B*, i32)* @_ZN1B3fooEi to i8*), i8* null] }
+@_ZTV1B = internal unnamed_addr constant { [4 x i8*] } { [4 x i8*] [i8* null, i8* null, i8* bitcast (i32 (%struct.B*, i32)* @_ZN1B3fooEi to i8*), i8* bitcast (i32 (%struct.B*, float)* @_ZN1B3barEf to i8*)] }, align 8, !type !0, !type !1, !type !2, !type !4, !type !5, !type !6, !vcall_visibility !3
+
+
+; CHECK: define internal i32 @_ZN1A3fooEi(
+define internal i32 @_ZN1A3fooEi(%struct.A* nocapture readnone %this, i32) unnamed_addr #1 align 2 {
+entry:
+  ret i32 1
+}
+
+; CHECK-NOT: define internal i32 @_ZN1A3barEf(
+define internal i32 @_ZN1A3barEf(%struct.A* nocapture readnone %this, float) unnamed_addr #1 align 2 {
+entry:
+  ret i32 2
+}
+
+; CHECK: define internal i32 @_ZN1B3fooEi(
+define internal i32 @_ZN1B3fooEi(%struct.B* nocapture readnone %this, i32) unnamed_addr #1 align 2 {
+entry:
+  ret i32 3
+}
+
+; CHECK-NOT: define internal i32 @_ZN1B3barEf(
+define internal i32 @_ZN1B3barEf(%struct.B* nocapture readnone %this, float) unnamed_addr #1 align 2 {
+entry:
+  ret i32 4
+}
+
+
+define hidden void @_ZN1AC2Ev(%struct.A* nocapture %this) {
+entry:
+  %0 = getelementptr inbounds %struct.A, %struct.A* %this, i64 0, i32 0
+  store i32 (...)** bitcast (i8** getelementptr inbounds ({ [4 x i8*] }, { [4 x i8*] }* @_ZTV1A, i64 0, inrange i32 0, i64 2) to i32 (...)**), i32 (...)*** %0, align 8
+  ret void
+}
+
+define hidden void @_ZN1BC2Ev(%struct.B* nocapture %this) {
+entry:
+  %0 = getelementptr inbounds %struct.B, %struct.B* %this, i64 0, i32 0, i32 0
+  store i32 (...)** bitcast (i8** getelementptr inbounds ({ [4 x i8*] }, { [4 x i8*] }* @_ZTV1B, i64 0, inrange i32 0, i64 2) to i32 (...)**), i32 (...)*** %0, align 8
+  ret void
+}
+
+define hidden i32 @test(%struct.A* %p, i64 %q.coerce0, i64 %q.coerce1) {
+entry:
+  %0 = bitcast %struct.A* %p to i8*
+  %1 = getelementptr inbounds i8, i8* %0, i64 %q.coerce1
+  %this.adjusted = bitcast i8* %1 to %struct.A*
+  %2 = and i64 %q.coerce0, 1
+  %memptr.isvirtual = icmp eq i64 %2, 0
+  br i1 %memptr.isvirtual, label %memptr.nonvirtual, label %memptr.virtual
+
+memptr.virtual:                                   ; preds = %entry
+  %3 = bitcast i8* %1 to i8**
+  %vtable = load i8*, i8** %3, align 8
+  %4 = add i64 %q.coerce0, -1
+  %5 = getelementptr i8, i8* %vtable, i64 %4, !nosanitize !12
+  %6 = tail call { i8*, i1 } @llvm.type.checked.load(i8* %5, i32 0, metadata !"_ZTSM1AFiiE.virtual"), !nosanitize !12
+  %7 = extractvalue { i8*, i1 } %6, 0, !nosanitize !12
+  %memptr.virtualfn = bitcast i8* %7 to i32 (%struct.A*, i32)*, !nosanitize !12
+  br label %memptr.end
+
+memptr.nonvirtual:                                ; preds = %entry
+  %memptr.nonvirtualfn = inttoptr i64 %q.coerce0 to i32 (%struct.A*, i32)*
+  br label %memptr.end
+
+memptr.end:                                       ; preds = %memptr.nonvirtual, %memptr.virtual
+  %8 = phi i32 (%struct.A*, i32)* [ %memptr.virtualfn, %memptr.virtual ], [ %memptr.nonvirtualfn, %memptr.nonvirtual ]
+  %call = tail call i32 %8(%struct.A* %this.adjusted, i32 42)
+  ret i32 %call
+}
+
+declare { i8*, i1 } @llvm.type.checked.load(i8*, i32, metadata)
+
+!0 = !{i64 16, !"_ZTS1A"}
+!1 = !{i64 16, !"_ZTSM1AFiiE.virtual"}
+!2 = !{i64 24, !"_ZTSM1AFifE.virtual"}
+!3 = !{i64 2}
+!4 = !{i64 16, !"_ZTS1B"}
+!5 = !{i64 16, !"_ZTSM1BFiiE.virtual"}
+!6 = !{i64 24, !"_ZTSM1BFifE.virtual"}
+!12 = !{}
diff --git a/llvm/test/Transforms/GlobalDCE/virtual-functions-derived-call.ll b/llvm/test/Transforms/GlobalDCE/virtual-functions-derived-call.ll
new file mode 100644
index 000000000..fb39f649b
--- /dev/null
+++ b/llvm/test/Transforms/GlobalDCE/virtual-functions-derived-call.ll
@@ -0,0 +1,78 @@
+; RUN: opt < %s -globaldce -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+; struct A {
+;   A();
+;   virtual int foo();
+; };
+; 
+; struct B : A {
+;   B();
+;   virtual int foo();
+; };
+; 
+; A::A() {}
+; B::B() {}
+; int A::foo() { return 42; }
+; int B::foo() { return 1337; }
+; 
+; extern "C" int test(B *p) { return p->foo(); }
+
+; The virtual call in test can only be dispatched to B::foo (or a more-derived
+; class, if there was one), so A::foo can be removed.
+
+%struct.A = type { i32 (...)** }
+%struct.B = type { %struct.A }
+
+; CHECK: @_ZTV1A = internal unnamed_addr constant { [3 x i8*] } zeroinitializer
+@_ZTV1A = internal unnamed_addr constant { [3 x i8*] } { [3 x i8*] [i8* null, i8* null, i8* bitcast (i32 (%struct.A*)* @_ZN1A3fooEv to i8*)] }, align 8, !type !0, !type !1, !vcall_visibility !2
+
+; CHECK: @_ZTV1B = internal unnamed_addr constant { [3 x i8*] } { [3 x i8*] [i8* null, i8* null, i8* bitcast (i32 (%struct.B*)* @_ZN1B3fooEv to i8*)] }
+@_ZTV1B = internal unnamed_addr constant { [3 x i8*] } { [3 x i8*] [i8* null, i8* null, i8* bitcast (i32 (%struct.B*)* @_ZN1B3fooEv to i8*)] }, align 8, !type !0, !type !1, !type !3, !type !4, !vcall_visibility !2
+
+; CHECK-NOT: define internal i32 @_ZN1A3fooEv(
+define internal i32 @_ZN1A3fooEv(%struct.A* nocapture readnone %this) {
+entry:
+  ret i32 42
+}
+
+; CHECK: define internal i32 @_ZN1B3fooEv(
+define internal i32 @_ZN1B3fooEv(%struct.B* nocapture readnone %this) {
+entry:
+  ret i32 1337
+}
+
+define hidden void @_ZN1AC2Ev(%struct.A* nocapture %this) {
+entry:
+  %0 = getelementptr inbounds %struct.A, %struct.A* %this, i64 0, i32 0
+  store i32 (...)** bitcast (i8** getelementptr inbounds ({ [3 x i8*] }, { [3 x i8*] }* @_ZTV1A, i64 0, inrange i32 0, i64 2) to i32 (...)**), i32 (...)*** %0, align 8
+  ret void
+}
+
+define hidden void @_ZN1BC2Ev(%struct.B* nocapture %this) {
+entry:
+  %0 = getelementptr inbounds %struct.B, %struct.B* %this, i64 0, i32 0, i32 0
+  store i32 (...)** bitcast (i8** getelementptr inbounds ({ [3 x i8*] }, { [3 x i8*] }* @_ZTV1B, i64 0, inrange i32 0, i64 2) to i32 (...)**), i32 (...)*** %0, align 8
+  ret void
+}
+
+define hidden i32 @test(%struct.B* %p) {
+entry:
+  %0 = bitcast %struct.B* %p to i8**
+  %vtable1 = load i8*, i8** %0, align 8
+  %1 = tail call { i8*, i1 } @llvm.type.checked.load(i8* %vtable1, i32 0, metadata !"_ZTS1B"), !nosanitize !10
+  %2 = extractvalue { i8*, i1 } %1, 0, !nosanitize !10
+  %3 = bitcast i8* %2 to i32 (%struct.B*)*, !nosanitize !10
+  %call = tail call i32 %3(%struct.B* %p)
+  ret i32 %call
+}
+
+declare { i8*, i1 } @llvm.type.checked.load(i8*, i32, metadata) #2
+
+!0 = !{i64 16, !"_ZTS1A"}
+!1 = !{i64 16, !"_ZTSM1AFivE.virtual"}
+!2 = !{i64 2}
+!3 = !{i64 16, !"_ZTS1B"}
+!4 = !{i64 16, !"_ZTSM1BFivE.virtual"}
+!10 = !{}
diff --git a/llvm/test/Transforms/GlobalDCE/virtual-functions-derived-pointer-call.ll b/llvm/test/Transforms/GlobalDCE/virtual-functions-derived-pointer-call.ll
new file mode 100644
index 000000000..62b5b8d37
--- /dev/null
+++ b/llvm/test/Transforms/GlobalDCE/virtual-functions-derived-pointer-call.ll
@@ -0,0 +1,120 @@
+
+; RUN: opt < %s -globaldce -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+; struct A {
+;   A();
+;   virtual int foo(int);
+;   virtual int bar(float);
+; };
+; 
+; struct B : A {
+;   B();
+;   virtual int foo(int);
+;   virtual int bar(float);
+; };
+; 
+; A::A() {}
+; B::B() {}
+; int A::foo(int)   { return 1; }
+; int A::bar(float) { return 2; }
+; int B::foo(int)   { return 3; }
+; int B::bar(float) { return 4; }
+; 
+; extern "C" int test(B *p, int (B::*q)(int)) { return (p->*q)(42); }
+
+; Member function pointers are tracked by the combination of their object type
+; and function type, which must both be compatible. Here, the call is through a
+; pointer of type "int (B::*q)(int)", so the call could only be dispatched to
+; B::foo. It can't be dispatched to A::bar or B::bar as the function pointer
+; does not match, and it can't be dispatched to A::foo as the object type
+; doesn't match, so those can be removed.
+
+%struct.A = type { i32 (...)** }
+%struct.B = type { %struct.A }
+
+; CHECK: @_ZTV1A = internal unnamed_addr constant { [4 x i8*] } zeroinitializer
+@_ZTV1A = internal unnamed_addr constant { [4 x i8*] } { [4 x i8*] [i8* null, i8* null, i8* bitcast (i32 (%struct.A*, i32)* @_ZN1A3fooEi to i8*), i8* bitcast (i32 (%struct.A*, float)* @_ZN1A3barEf to i8*)] }, align 8, !type !0, !type !1, !type !2, !vcall_visibility !3
+; CHECK: @_ZTV1B = internal unnamed_addr constant { [4 x i8*] } { [4 x i8*] [i8* null, i8* null, i8* bitcast (i32 (%struct.B*, i32)* @_ZN1B3fooEi to i8*), i8* null] }
+@_ZTV1B = internal unnamed_addr constant { [4 x i8*] } { [4 x i8*] [i8* null, i8* null, i8* bitcast (i32 (%struct.B*, i32)* @_ZN1B3fooEi to i8*), i8* bitcast (i32 (%struct.B*, float)* @_ZN1B3barEf to i8*)] }, align 8, !type !0, !type !1, !type !2, !type !4, !type !5, !type !6, !vcall_visibility !3
+
+
+; CHECK-NOT: define internal i32 @_ZN1A3fooEi(
+define internal i32 @_ZN1A3fooEi(%struct.A* nocapture readnone %this, i32) unnamed_addr #1 align 2 {
+entry:
+  ret i32 1
+}
+
+; CHECK-NOT: define internal i32 @_ZN1A3barEf(
+define internal i32 @_ZN1A3barEf(%struct.A* nocapture readnone %this, float) unnamed_addr #1 align 2 {
+entry:
+  ret i32 2
+}
+
+; CHECK: define internal i32 @_ZN1B3fooEi(
+define internal i32 @_ZN1B3fooEi(%struct.B* nocapture readnone %this, i32) unnamed_addr #1 align 2 {
+entry:
+  ret i32 3
+}
+
+; CHECK-NOT: define internal i32 @_ZN1B3barEf(
+define internal i32 @_ZN1B3barEf(%struct.B* nocapture readnone %this, float) unnamed_addr #1 align 2 {
+entry:
+  ret i32 4
+}
+
+
+define hidden void @_ZN1AC2Ev(%struct.A* nocapture %this) {
+entry:
+  %0 = getelementptr inbounds %struct.A, %struct.A* %this, i64 0, i32 0
+  store i32 (...)** bitcast (i8** getelementptr inbounds ({ [4 x i8*] }, { [4 x i8*] }* @_ZTV1A, i64 0, inrange i32 0, i64 2) to i32 (...)**), i32 (...)*** %0, align 8
+  ret void
+}
+
+define hidden void @_ZN1BC2Ev(%struct.B* nocapture %this) {
+entry:
+  %0 = getelementptr inbounds %struct.B, %struct.B* %this, i64 0, i32 0, i32 0
+  store i32 (...)** bitcast (i8** getelementptr inbounds ({ [4 x i8*] }, { [4 x i8*] }* @_ZTV1B, i64 0, inrange i32 0, i64 2) to i32 (...)**), i32 (...)*** %0, align 8
+  ret void
+}
+
+define hidden i32 @test(%struct.B* %p, i64 %q.coerce0, i64 %q.coerce1) {
+entry:
+  %0 = bitcast %struct.B* %p to i8*
+  %1 = getelementptr inbounds i8, i8* %0, i64 %q.coerce1
+  %this.adjusted = bitcast i8* %1 to %struct.B*
+  %2 = and i64 %q.coerce0, 1
+  %memptr.isvirtual = icmp eq i64 %2, 0
+  br i1 %memptr.isvirtual, label %memptr.nonvirtual, label %memptr.virtual
+
+memptr.virtual:                                   ; preds = %entry
+  %3 = bitcast i8* %1 to i8**
+  %vtable = load i8*, i8** %3, align 8
+  %4 = add i64 %q.coerce0, -1
+  %5 = getelementptr i8, i8* %vtable, i64 %4, !nosanitize !12
+  %6 = tail call { i8*, i1 } @llvm.type.checked.load(i8* %5, i32 0, metadata !"_ZTSM1BFiiE.virtual"), !nosanitize !12
+  %7 = extractvalue { i8*, i1 } %6, 0, !nosanitize !12
+  %memptr.virtualfn = bitcast i8* %7 to i32 (%struct.B*, i32)*, !nosanitize !12
+  br label %memptr.end
+
+memptr.nonvirtual:                                ; preds = %entry
+  %memptr.nonvirtualfn = inttoptr i64 %q.coerce0 to i32 (%struct.B*, i32)*
+  br label %memptr.end
+
+memptr.end:                                       ; preds = %memptr.nonvirtual, %memptr.virtual
+  %8 = phi i32 (%struct.B*, i32)* [ %memptr.virtualfn, %memptr.virtual ], [ %memptr.nonvirtualfn, %memptr.nonvirtual ]
+  %call = tail call i32 %8(%struct.B* %this.adjusted, i32 42)
+  ret i32 %call
+}
+
+declare { i8*, i1 } @llvm.type.checked.load(i8*, i32, metadata)
+
+!0 = !{i64 16, !"_ZTS1A"}
+!1 = !{i64 16, !"_ZTSM1AFiiE.virtual"}
+!2 = !{i64 24, !"_ZTSM1AFifE.virtual"}
+!3 = !{i64 2}
+!4 = !{i64 16, !"_ZTS1B"}
+!5 = !{i64 16, !"_ZTSM1BFiiE.virtual"}
+!6 = !{i64 24, !"_ZTSM1BFifE.virtual"}
+!12 = !{}
diff --git a/llvm/test/Transforms/GlobalDCE/virtual-functions-visibility-post-lto.ll b/llvm/test/Transforms/GlobalDCE/virtual-functions-visibility-post-lto.ll
new file mode 100644
index 000000000..d636b5a3d
--- /dev/null
+++ b/llvm/test/Transforms/GlobalDCE/virtual-functions-visibility-post-lto.ll
@@ -0,0 +1,95 @@
+; RUN: opt < %s -globaldce -S | FileCheck %s
+
+; structs A, B and C have vcall_visibility of public, linkage-unit and
+; translation-unit respectively. This test is run after LTO linking (the
+; LTOPostLink metadata is present), so B and C can be VFE'd.
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+
+%struct.A = type { i32 (...)** }
+
+@_ZTV1A = hidden unnamed_addr constant { [3 x i8*] } { [3 x i8*] [i8* null, i8* null, i8* bitcast (void (%struct.A*)* @_ZN1A3fooEv to i8*)] }, align 8, !type !0, !type !1, !vcall_visibility !2
+
+define internal void @_ZN1AC2Ev(%struct.A* %this) {
+entry:
+  %0 = getelementptr inbounds %struct.A, %struct.A* %this, i64 0, i32 0
+  store i32 (...)** bitcast (i8** getelementptr inbounds ({ [3 x i8*] }, { [3 x i8*] }* @_ZTV1A, i64 0, inrange i32 0, i64 2) to i32 (...)**), i32 (...)*** %0, align 8
+  ret void
+}
+
+; CHECK: define {{.*}} @_ZN1A3fooEv(
+define internal void @_ZN1A3fooEv(%struct.A* nocapture %this) {
+entry:
+  ret void
+}
+
+define dso_local i8* @_Z6make_Av() {
+entry:
+  %call = tail call i8* @_Znwm(i64 8)
+  %0 = bitcast i8* %call to %struct.A*
+  tail call void @_ZN1AC2Ev(%struct.A* %0)
+  ret i8* %call
+}
+
+
+%struct.B = type { i32 (...)** }
+
+@_ZTV1B = hidden unnamed_addr constant { [3 x i8*] } { [3 x i8*] [i8* null, i8* null, i8* bitcast (void (%struct.B*)* @_ZN1B3fooEv to i8*)] }, align 8, !type !0, !type !1, !vcall_visibility !3
+
+define internal void @_ZN1BC2Ev(%struct.B* %this) {
+entry:
+  %0 = getelementptr inbounds %struct.B, %struct.B* %this, i64 0, i32 0
+  store i32 (...)** bitcast (i8** getelementptr inbounds ({ [3 x i8*] }, { [3 x i8*] }* @_ZTV1B, i64 0, inrange i32 0, i64 2) to i32 (...)**), i32 (...)*** %0, align 8
+  ret void
+}
+
+; CHECK-NOT: define {{.*}} @_ZN1B3fooEv(
+define internal void @_ZN1B3fooEv(%struct.B* nocapture %this) {
+entry:
+  ret void
+}
+
+define dso_local i8* @_Z6make_Bv() {
+entry:
+  %call = tail call i8* @_Znwm(i64 8)
+  %0 = bitcast i8* %call to %struct.B*
+  tail call void @_ZN1BC2Ev(%struct.B* %0)
+  ret i8* %call
+}
+
+
+%struct.C = type { i32 (...)** }
+
+@_ZTV1C = hidden unnamed_addr constant { [3 x i8*] } { [3 x i8*] [i8* null, i8* null, i8* bitcast (void (%struct.C*)* @_ZN1C3fooEv to i8*)] }, align 8, !type !0, !type !1, !vcall_visibility !4
+
+define internal void @_ZN1CC2Ev(%struct.C* %this) {
+entry:
+  %0 = getelementptr inbounds %struct.C, %struct.C* %this, i64 0, i32 0
+  store i32 (...)** bitcast (i8** getelementptr inbounds ({ [3 x i8*] }, { [3 x i8*] }* @_ZTV1C, i64 0, inrange i32 0, i64 2) to i32 (...)**), i32 (...)*** %0, align 8
+  ret void
+}
+
+; CHECK-NOT: define {{.*}} @_ZN1C3fooEv(
+define internal void @_ZN1C3fooEv(%struct.C* nocapture %this) {
+entry:
+  ret void
+}
+
+define dso_local i8* @_Z6make_Cv() {
+entry:
+  %call = tail call i8* @_Znwm(i64 8)
+  %0 = bitcast i8* %call to %struct.C*
+  tail call void @_ZN1CC2Ev(%struct.C* %0)
+  ret i8* %call
+}
+
+declare dso_local noalias nonnull i8* @_Znwm(i64)
+
+!llvm.module.flags = !{!5}
+
+!0 = !{i64 16, !"_ZTS1A"}
+!1 = !{i64 16, !"_ZTSM1AFvvE.virtual"}
+!2 = !{i64 0} ; public vcall visibility
+!3 = !{i64 1} ; linkage-unit vcall visibility
+!4 = !{i64 2} ; translation-unit vcall visibility
+!5 = !{i32 1, !"LTOPostLink", i32 1}
diff --git a/llvm/test/Transforms/GlobalDCE/virtual-functions-visibility-pre-lto.ll b/llvm/test/Transforms/GlobalDCE/virtual-functions-visibility-pre-lto.ll
new file mode 100644
index 000000000..b0b34c0bb
--- /dev/null
+++ b/llvm/test/Transforms/GlobalDCE/virtual-functions-visibility-pre-lto.ll
@@ -0,0 +1,94 @@
+; RUN: opt < %s -globaldce -S | FileCheck %s
+
+; structs A, B and C have vcall_visibility of public, linkage-unit and
+; translation-unit respectively. This test is run before LTO linking occurs
+; (the LTOPostLink metadata is not present), so only C can be VFE'd.
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+
+%struct.A = type { i32 (...)** }
+
+@_ZTV1A = hidden unnamed_addr constant { [3 x i8*] } { [3 x i8*] [i8* null, i8* null, i8* bitcast (void (%struct.A*)* @_ZN1A3fooEv to i8*)] }, align 8, !type !0, !type !1, !vcall_visibility !2
+
+define internal void @_ZN1AC2Ev(%struct.A* %this) {
+entry:
+  %0 = getelementptr inbounds %struct.A, %struct.A* %this, i64 0, i32 0
+  store i32 (...)** bitcast (i8** getelementptr inbounds ({ [3 x i8*] }, { [3 x i8*] }* @_ZTV1A, i64 0, inrange i32 0, i64 2) to i32 (...)**), i32 (...)*** %0, align 8
+  ret void
+}
+
+; CHECK: define {{.*}} @_ZN1A3fooEv(
+define internal void @_ZN1A3fooEv(%struct.A* nocapture %this) {
+entry:
+  ret void
+}
+
+define dso_local i8* @_Z6make_Av() {
+entry:
+  %call = tail call i8* @_Znwm(i64 8)
+  %0 = bitcast i8* %call to %struct.A*
+  tail call void @_ZN1AC2Ev(%struct.A* %0)
+  ret i8* %call
+}
+
+
+%struct.B = type { i32 (...)** }
+
+@_ZTV1B = hidden unnamed_addr constant { [3 x i8*] } { [3 x i8*] [i8* null, i8* null, i8* bitcast (void (%struct.B*)* @_ZN1B3fooEv to i8*)] }, align 8, !type !0, !type !1, !vcall_visibility !3
+
+define internal void @_ZN1BC2Ev(%struct.B* %this) {
+entry:
+  %0 = getelementptr inbounds %struct.B, %struct.B* %this, i64 0, i32 0
+  store i32 (...)** bitcast (i8** getelementptr inbounds ({ [3 x i8*] }, { [3 x i8*] }* @_ZTV1B, i64 0, inrange i32 0, i64 2) to i32 (...)**), i32 (...)*** %0, align 8
+  ret void
+}
+
+; CHECK: define {{.*}} @_ZN1B3fooEv(
+define internal void @_ZN1B3fooEv(%struct.B* nocapture %this) {
+entry:
+  ret void
+}
+
+define dso_local i8* @_Z6make_Bv() {
+entry:
+  %call = tail call i8* @_Znwm(i64 8)
+  %0 = bitcast i8* %call to %struct.B*
+  tail call void @_ZN1BC2Ev(%struct.B* %0)
+  ret i8* %call
+}
+
+
+%struct.C = type { i32 (...)** }
+
+@_ZTV1C = hidden unnamed_addr constant { [3 x i8*] } { [3 x i8*] [i8* null, i8* null, i8* bitcast (void (%struct.C*)* @_ZN1C3fooEv to i8*)] }, align 8, !type !0, !type !1, !vcall_visibility !4
+
+define internal void @_ZN1CC2Ev(%struct.C* %this) {
+entry:
+  %0 = getelementptr inbounds %struct.C, %struct.C* %this, i64 0, i32 0
+  store i32 (...)** bitcast (i8** getelementptr inbounds ({ [3 x i8*] }, { [3 x i8*] }* @_ZTV1C, i64 0, inrange i32 0, i64 2) to i32 (...)**), i32 (...)*** %0, align 8
+  ret void
+}
+
+; CHECK-NOT: define {{.*}} @_ZN1C3fooEv(
+define internal void @_ZN1C3fooEv(%struct.C* nocapture %this) {
+entry:
+  ret void
+}
+
+define dso_local i8* @_Z6make_Cv() {
+entry:
+  %call = tail call i8* @_Znwm(i64 8)
+  %0 = bitcast i8* %call to %struct.C*
+  tail call void @_ZN1CC2Ev(%struct.C* %0)
+  ret i8* %call
+}
+
+declare dso_local noalias nonnull i8* @_Znwm(i64)
+
+!llvm.module.flags = !{}
+
+!0 = !{i64 16, !"_ZTS1A"}
+!1 = !{i64 16, !"_ZTSM1AFvvE.virtual"}
+!2 = !{i64 0} ; public vcall visibility
+!3 = !{i64 1} ; linkage-unit vcall visibility
+!4 = !{i64 2} ; translation-unit vcall visibility
diff --git a/llvm/test/Transforms/GlobalDCE/virtual-functions.ll b/llvm/test/Transforms/GlobalDCE/virtual-functions.ll
new file mode 100644
index 000000000..614907197
--- /dev/null
+++ b/llvm/test/Transforms/GlobalDCE/virtual-functions.ll
@@ -0,0 +1,55 @@
+; RUN: opt < %s -globaldce -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+declare dso_local noalias nonnull i8* @_Znwm(i64)
+declare { i8*, i1 } @llvm.type.checked.load(i8*, i32, metadata)
+
+; %struct.A is a C++ struct with two virtual functions, A::foo and A::bar. The
+; !vcall_visibility metadata is set on the vtable, so we know that all virtual
+; calls through this vtable are visible and use the @llvm.type.checked.load
+; intrinsic. Function test_A makes a call to A::foo, but there is no call to
+; A::bar anywhere, so A::bar can be deleted, and its vtable slot replaced with
+; null.
+
+%struct.A = type { i32 (...)** }
+
+; The pointer to A::bar in the vtable can be removed, because it will never be
+; loaded. We replace it with null to keep the layout the same. Because it is at
+; the end of the vtable we could potentially shrink the vtable, but don't
+; currently do that.
+; CHECK: @_ZTV1A = internal unnamed_addr constant { [4 x i8*] } { [4 x i8*] [i8* null, i8* null, i8* bitcast (i32 (%struct.A*)* @_ZN1A3fooEv to i8*), i8* null] }
+@_ZTV1A = internal unnamed_addr constant { [4 x i8*] } { [4 x i8*] [i8* null, i8* null, i8* bitcast (i32 (%struct.A*)* @_ZN1A3fooEv to i8*), i8* bitcast (i32 (%struct.A*)* @_ZN1A3barEv to i8*)] }, align 8, !type !0, !type !1, !type !2, !vcall_visibility !3
+
+; A::foo is called, so must be retained.
+; CHECK: define internal i32 @_ZN1A3fooEv(
+define internal i32 @_ZN1A3fooEv(%struct.A* nocapture readnone %this) {
+entry:
+  ret i32 42
+}
+
+; A::bar is not used, so can be deleted.
+; CHECK-NOT: define internal i32 @_ZN1A3barEv(
+define internal i32 @_ZN1A3barEv(%struct.A* nocapture readnone %this) {
+entry:
+  ret i32 1337
+}
+
+define dso_local i32 @test_A() {
+entry:
+  %call = tail call i8* @_Znwm(i64 8)
+  %0 = bitcast i8* %call to %struct.A*
+  %1 = bitcast i8* %call to i32 (...)***
+  store i32 (...)** bitcast (i8** getelementptr inbounds ({ [4 x i8*] }, { [4 x i8*] }* @_ZTV1A, i64 0, inrange i32 0, i64 2) to i32 (...)**), i32 (...)*** %1, align 8
+  %2 = tail call { i8*, i1 } @llvm.type.checked.load(i8* bitcast (i8** getelementptr inbounds ({ [4 x i8*] }, { [4 x i8*] }* @_ZTV1A, i64 0, inrange i32 0, i64 2) to i8*), i32 0, metadata !"_ZTS1A"), !nosanitize !9
+  %3 = extractvalue { i8*, i1 } %2, 0, !nosanitize !9
+  %4 = bitcast i8* %3 to i32 (%struct.A*)*, !nosanitize !9
+  %call1 = tail call i32 %4(%struct.A* nonnull %0)
+  ret i32 %call1
+}
+
+!0 = !{i64 16, !"_ZTS1A"}
+!1 = !{i64 16, !"_ZTSM1AFivE.virtual"}
+!2 = !{i64 24, !"_ZTSM1AFivE.virtual"}
+!3 = !{i64 2}
+!9 = !{}
diff --git a/llvm/test/Transforms/GlobalDCE/vtable-rtti.ll b/llvm/test/Transforms/GlobalDCE/vtable-rtti.ll
new file mode 100644
index 000000000..dd6119340
--- /dev/null
+++ b/llvm/test/Transforms/GlobalDCE/vtable-rtti.ll
@@ -0,0 +1,47 @@
+; RUN: opt < %s -globaldce -S | FileCheck %s
+
+; We currently only use llvm.type.checked.load for virtual function pointers,
+; not any other part of the vtable, so we can't remove the RTTI pointer even if
+; it's never going to be loaded from.
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+
+%struct.A = type { i32 (...)** }
+
+; CHECK: @_ZTV1A = hidden unnamed_addr constant { [3 x i8*] } { [3 x i8*] [i8* null, i8* bitcast ({ i8*, i8* }* @_ZTI1A to i8*), i8* null] }, align 8, !type !0, !type !1, !vcall_visibility !2
+
+@_ZTV1A = hidden unnamed_addr constant { [3 x i8*] } { [3 x i8*] [i8* null, i8* bitcast ({ i8*, i8* }* @_ZTI1A to i8*), i8* bitcast (void (%struct.A*)* @_ZN1A3fooEv to i8*)] }, align 8, !type !0, !type !1, !vcall_visibility !2
+@_ZTS1A = hidden constant [3 x i8] c"1A\00", align 1
+@_ZTI1A = hidden constant { i8*, i8* } { i8* bitcast (i8** getelementptr inbounds (i8*, i8** @_ZTVN10__cxxabiv117__class_type_infoE, i64 2) to i8*), i8* getelementptr inbounds ([3 x i8], [3 x i8]* @_ZTS1A, i32 0, i32 0) }, align 8
+
+define internal void @_ZN1AC2Ev(%struct.A* %this) {
+entry:
+  %0 = getelementptr inbounds %struct.A, %struct.A* %this, i64 0, i32 0
+  store i32 (...)** bitcast (i8** getelementptr inbounds ({ [3 x i8*] }, { [3 x i8*] }* @_ZTV1A, i64 0, inrange i32 0, i64 2) to i32 (...)**), i32 (...)*** %0, align 8
+  ret void
+}
+
+; CHECK-NOT: define {{.*}} @_ZN1A3fooEv(
+define internal void @_ZN1A3fooEv(%struct.A* nocapture %this) {
+entry:
+  ret void
+}
+
+define dso_local i8* @_Z6make_Av() {
+entry:
+  %call = tail call i8* @_Znwm(i64 8)
+  %0 = bitcast i8* %call to %struct.A*
+  tail call void @_ZN1AC2Ev(%struct.A* %0)
+  ret i8* %call
+}
+
+
+declare dso_local noalias nonnull i8* @_Znwm(i64)
+@_ZTVN10__cxxabiv117__class_type_infoE = external dso_local global i8*
+
+!llvm.module.flags = !{!3}
+
+!0 = !{i64 16, !"_ZTS1A"}
+!1 = !{i64 16, !"_ZTSM1AFvvE.virtual"}
+!2 = !{i64 2} ; translation-unit vcall visibility
+!3 = !{i32 1, !"LTOPostLink", i32 1}
diff --git a/llvm/test/Transforms/GlobalOpt/atomic.ll b/llvm/test/Transforms/GlobalOpt/atomic.ll
index 7597e0f03..f49ab5270 100644
--- a/llvm/test/Transforms/GlobalOpt/atomic.ll
+++ b/llvm/test/Transforms/GlobalOpt/atomic.ll
@@ -1,10 +1,10 @@
 ; RUN: opt -globalopt < %s -S -o - | FileCheck %s
 
-@GV1 = internal global i64 1
-@GV2 = internal global i32 0
+@GV1 = internal global i64 1, align 8
+@GV2 = internal global i32 0, align 4
 
-; CHECK: @GV1 = internal unnamed_addr global i64 1
-; CHECK: @GV2 = internal unnamed_addr global i32 0
+; CHECK: @GV1 = internal unnamed_addr global i64 1, align 8
+; CHECK: @GV2 = internal unnamed_addr global i32 0, align 4
 
 define void @test1() {
 entry:
diff --git a/llvm/test/Transforms/GlobalOpt/large-int-crash.ll b/llvm/test/Transforms/GlobalOpt/large-int-crash.ll
new file mode 100644
index 000000000..7584554b9
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/large-int-crash.ll
@@ -0,0 +1,23 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -globalopt -S | FileCheck %s
+
+@X = internal global i128 0
+
+define void @foo() {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:    [[T0_B:%.*]] = load i1, i1* @X
+; CHECK-NEXT:    [[T0:%.*]] = select i1 [[T0_B]], i128 18446744073709551616, i128 0
+; CHECK-NEXT:    ret void
+;
+  %t0 = load i128, i128* @X, align 8
+  ret void
+}
+
+define void @store() {
+; CHECK-LABEL: @store(
+; CHECK-NEXT:    store i1 true, i1* @X
+; CHECK-NEXT:    ret void
+;
+  store i128 18446744073709551616, i128* @X, align 8
+  ret void
+}
diff --git a/llvm/test/Transforms/GlobalOpt/long-compilation-global-sra.ll b/llvm/test/Transforms/GlobalOpt/long-compilation-global-sra.ll
new file mode 100644
index 000000000..69a358edc
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/long-compilation-global-sra.ll
@@ -0,0 +1,61 @@
+; RUN: opt %s --O0 -globalopt -S -o -
+
+; This is a regression test against very slow execution...
+; In bad case it should fail by timeout.
+
+; Hand-reduced from this example.
+; clang++ -mllvm -disable-llvm-optzns
+
+;#include <stdio.h>
+;
+;namespace {
+;  char LargeBuffer[64 * 1024 * 1024];
+;}
+;
+;int main ( void ) {
+;
+;    LargeBuffer[0] = 0;
+;
+;    printf("");
+;
+;    return LargeBuffer[0] == 0;
+;}
+
+; check that global array LargeBufferE was optimized out
+; and local variable LargeBufferE.0 was used instead.
+
+; CHECK-NOT: global
+; CHECK: main()
+; CHECK-NEXT: LargeBufferE.0
+; CHECK-NOT: global
+
+; ModuleID = 'test.cpp'
+source_filename = "test.cpp"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@LargeBufferE = internal global [67108864 x i8] zeroinitializer, align 16
+@.str = private unnamed_addr constant [1 x i8] c"\00", align 1
+
+; Function Attrs: norecurse uwtable
+define dso_local i32 @main() #0 {
+  %1 = alloca i32, align 4
+  store i32 0, i32* %1, align 4
+  store i8 0, i8* getelementptr inbounds ([67108864 x i8], [67108864 x i8]* @LargeBufferE, i64 0, i64 0), align 16
+  %2 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([1 x i8], [1 x i8]* @.str, i64 0, i64 0))
+  %3 = load i8, i8* getelementptr inbounds ([67108864 x i8], [67108864 x i8]* @LargeBufferE, i64 0, i64 0), align 16
+  %4 = sext i8 %3 to i32
+  %5 = icmp eq i32 %4, 0
+  %6 = zext i1 %5 to i32
+  ret i32 %6
+}
+
+declare dso_local i32 @printf(i8*, ...) #0
+
+attributes #0 = { norecurse uwtable }
+
+!llvm.module.flags = !{!0}
+!llvm.ident = !{!1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{!"clang version 10.0.0 "}
diff --git a/llvm/test/Transforms/GlobalOpt/naked_functions.ll b/llvm/test/Transforms/GlobalOpt/naked_functions.ll
index 80c3aa8c3..8dd63699a 100644
--- a/llvm/test/Transforms/GlobalOpt/naked_functions.ll
+++ b/llvm/test/Transforms/GlobalOpt/naked_functions.ll
@@ -18,6 +18,6 @@ entry:
   unreachable
 }
 
-; CHECK: define internal i32 @foo(i32*)
+; CHECK: define internal i32 @foo(i32* %0)
 
 attributes #0 = { naked }
diff --git a/llvm/test/Transforms/GuardWidening/basic_widenable_condition_guards.ll b/llvm/test/Transforms/GuardWidening/basic_widenable_condition_guards.ll
index 605178a78..40d22e07e 100644
--- a/llvm/test/Transforms/GuardWidening/basic_widenable_condition_guards.ll
+++ b/llvm/test/Transforms/GuardWidening/basic_widenable_condition_guards.ll
@@ -8,10 +8,9 @@ define void @f_0(i1 %cond_0, i1 %cond_1) {
 ; CHECK-LABEL: @f_0(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
-; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_0:%.*]], [[WIDENABLE_COND]]
-; CHECK-NEXT:    [[WIDE_CHK:%.*]] = and i1 [[COND_0]], [[COND_1:%.*]]
-; CHECK-NEXT:    [[GUARD_CHK:%.*]] = and i1 [[WIDE_CHK]], [[WIDENABLE_COND]]
-; CHECK-NEXT:    br i1 [[GUARD_CHK]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK-NEXT:    [[WIDE_CHK:%.*]] = and i1 [[COND_0:%.*]], [[COND_1:%.*]]
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[WIDE_CHK]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
 ; CHECK:       deopt:
 ; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
 ; CHECK-NEXT:    ret void
@@ -52,10 +51,9 @@ define void @f_1(i1 %cond_0, i1 %cond_1) {
 ; CHECK-LABEL: @f_1(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
-; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_0:%.*]], [[WIDENABLE_COND]]
-; CHECK-NEXT:    [[WIDE_CHK:%.*]] = and i1 [[COND_0]], [[COND_1:%.*]]
-; CHECK-NEXT:    [[GUARD_CHK:%.*]] = and i1 [[WIDE_CHK]], [[WIDENABLE_COND]]
-; CHECK-NEXT:    br i1 [[GUARD_CHK]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK-NEXT:    [[WIDE_CHK:%.*]] = and i1 [[COND_0:%.*]], [[COND_1:%.*]]
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[WIDE_CHK]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
 ; CHECK:       deopt:
 ; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
 ; CHECK-NEXT:    ret void
@@ -113,11 +111,10 @@ define void @f_2(i32 %a, i32 %b) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[COND_0:%.*]] = icmp ult i32 [[A:%.*]], 10
 ; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
-; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_0]], [[WIDENABLE_COND]]
 ; CHECK-NEXT:    [[COND_1:%.*]] = icmp ult i32 [[B:%.*]], 10
 ; CHECK-NEXT:    [[WIDE_CHK:%.*]] = and i1 [[COND_0]], [[COND_1]]
-; CHECK-NEXT:    [[GUARD_CHK:%.*]] = and i1 [[WIDE_CHK]], [[WIDENABLE_COND]]
-; CHECK-NEXT:    br i1 [[GUARD_CHK]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[WIDE_CHK]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
 ; CHECK:       deopt:
 ; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
 ; CHECK-NEXT:    ret void
@@ -235,11 +232,10 @@ define void @f_4(i32 %a, i32 %b) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[COND_0:%.*]] = icmp ult i32 [[A:%.*]], 10
 ; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
-; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_0]], [[WIDENABLE_COND]]
 ; CHECK-NEXT:    [[COND_1:%.*]] = icmp ult i32 [[B:%.*]], 10
 ; CHECK-NEXT:    [[WIDE_CHK:%.*]] = and i1 [[COND_0]], [[COND_1]]
-; CHECK-NEXT:    [[GUARD_CHK:%.*]] = and i1 [[WIDE_CHK]], [[WIDENABLE_COND]]
-; CHECK-NEXT:    br i1 [[GUARD_CHK]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[WIDE_CHK]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
 ; CHECK:       deopt:
 ; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
 ; CHECK-NEXT:    ret void
@@ -294,10 +290,9 @@ define void @f_5(i32 %a) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[COND_0:%.*]] = icmp ugt i32 [[A:%.*]], 7
 ; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
-; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_0]], [[WIDENABLE_COND]]
 ; CHECK-NEXT:    [[WIDE_CHK:%.*]] = icmp uge i32 [[A]], 11
-; CHECK-NEXT:    [[GUARD_CHK:%.*]] = and i1 [[WIDE_CHK]], [[WIDENABLE_COND]]
-; CHECK-NEXT:    br i1 [[GUARD_CHK]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[WIDE_CHK]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
 ; CHECK:       deopt:
 ; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
 ; CHECK-NEXT:    ret void
@@ -403,11 +398,10 @@ define void @f_7(i32 %a, i1* %cond_buf) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[COND_1:%.*]] = load volatile i1, i1* [[COND_BUF:%.*]]
 ; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
-; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_1]], [[WIDENABLE_COND]]
 ; CHECK-NEXT:    [[COND_3:%.*]] = icmp ult i32 [[A:%.*]], 7
 ; CHECK-NEXT:    [[WIDE_CHK:%.*]] = and i1 [[COND_1]], [[COND_3]]
-; CHECK-NEXT:    [[GUARD_CHK:%.*]] = and i1 [[WIDE_CHK]], [[WIDENABLE_COND]]
-; CHECK-NEXT:    br i1 [[GUARD_CHK]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[WIDE_CHK]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
 ; CHECK:       deopt:
 ; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
 ; CHECK-NEXT:    ret void
@@ -491,11 +485,10 @@ define void @f_8(i32 %a, i1 %cond_1, i1 %cond_2) {
 ; CHECK-NEXT:    br i1 undef, label [[LOOP]], label [[LEAVE:%.*]]
 ; CHECK:       leave:
 ; CHECK-NEXT:    [[WIDENABLE_COND3:%.*]] = call i1 @llvm.experimental.widenable.condition()
-; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND4:%.*]] = and i1 [[COND_2:%.*]], [[WIDENABLE_COND3]]
 ; CHECK-NEXT:    [[COND_3:%.*]] = icmp ult i32 [[A:%.*]], 7
-; CHECK-NEXT:    [[WIDE_CHK:%.*]] = and i1 [[COND_2]], [[COND_3]]
-; CHECK-NEXT:    [[GUARD_CHK:%.*]] = and i1 [[WIDE_CHK]], [[WIDENABLE_COND3]]
-; CHECK-NEXT:    br i1 [[GUARD_CHK]], label [[GUARDED1:%.*]], label [[DEOPT2:%.*]], !prof !0
+; CHECK-NEXT:    [[WIDE_CHK:%.*]] = and i1 [[COND_2:%.*]], [[COND_3]]
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND4:%.*]] = and i1 [[WIDE_CHK]], [[WIDENABLE_COND3]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND4]], label [[GUARDED1:%.*]], label [[DEOPT2:%.*]], !prof !0
 ; CHECK:       deopt2:
 ; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
 ; CHECK-NEXT:    ret void
@@ -671,10 +664,9 @@ define void @f_11(i32 %a, i1 %cond_0, i1 %cond_1) {
 ; CHECK-NEXT:    br label [[OUTER_HEADER:%.*]]
 ; CHECK:       outer_header:
 ; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
-; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_0:%.*]], [[WIDENABLE_COND]]
-; CHECK-NEXT:    [[WIDE_CHK:%.*]] = and i1 [[COND_0]], [[COND_1:%.*]]
-; CHECK-NEXT:    [[GUARD_CHK:%.*]] = and i1 [[WIDE_CHK]], [[WIDENABLE_COND]]
-; CHECK-NEXT:    br i1 [[GUARD_CHK]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK-NEXT:    [[WIDE_CHK:%.*]] = and i1 [[COND_0:%.*]], [[COND_1:%.*]]
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[WIDE_CHK]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
 ; CHECK:       deopt:
 ; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
 ; CHECK-NEXT:    ret void
@@ -734,7 +726,6 @@ define void @f_12(i32 %a0) {
 ; CHECK-LABEL: @f_12(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
-; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 true, [[WIDENABLE_COND]]
 ; CHECK-NEXT:    [[A1:%.*]] = mul i32 [[A0:%.*]], [[A0]]
 ; CHECK-NEXT:    [[A2:%.*]] = mul i32 [[A1]], [[A1]]
 ; CHECK-NEXT:    [[A3:%.*]] = mul i32 [[A2]], [[A2]]
@@ -767,8 +758,8 @@ define void @f_12(i32 %a0) {
 ; CHECK-NEXT:    [[A30:%.*]] = mul i32 [[A29]], [[A29]]
 ; CHECK-NEXT:    [[COND:%.*]] = trunc i32 [[A30]] to i1
 ; CHECK-NEXT:    [[WIDE_CHK:%.*]] = and i1 true, [[COND]]
-; CHECK-NEXT:    [[GUARD_CHK:%.*]] = and i1 [[WIDE_CHK]], [[WIDENABLE_COND]]
-; CHECK-NEXT:    br i1 [[GUARD_CHK]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[WIDE_CHK]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
 ; CHECK:       deopt:
 ; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
 ; CHECK-NEXT:    ret void
@@ -840,10 +831,9 @@ define void @f_13(i32 %a) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[COND_0:%.*]] = icmp ult i32 [[A:%.*]], 14
 ; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
-; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_0]], [[WIDENABLE_COND]]
 ; CHECK-NEXT:    [[WIDE_CHK:%.*]] = icmp ult i32 [[A]], 10
-; CHECK-NEXT:    [[GUARD_CHK:%.*]] = and i1 [[WIDE_CHK]], [[WIDENABLE_COND]]
-; CHECK-NEXT:    br i1 [[GUARD_CHK]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[WIDE_CHK]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
 ; CHECK:       deopt:
 ; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
 ; CHECK-NEXT:    ret void
@@ -1031,6 +1021,91 @@ guarded1:                                         ; preds = %guarded
   ret void
 }
 
+
+define void @swapped_wb(i1 %cond_0, i1 %cond_1) {
+; CHECK-LABEL: @swapped_wb(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[WIDE_CHK:%.*]] = and i1 [[COND_0:%.*]], [[COND_1:%.*]]
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[WIDENABLE_COND]], [[WIDE_CHK]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded:
+; CHECK-NEXT:    [[WIDENABLE_COND3:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND4:%.*]] = and i1 [[COND_1]], [[WIDENABLE_COND3]]
+; CHECK-NEXT:    br i1 true, label [[GUARDED1:%.*]], label [[DEOPT2:%.*]], !prof !0
+; CHECK:       deopt2:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded1:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %widenable_cond, %cond_0
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %entry
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded:                                          ; preds = %entry
+  %widenable_cond3 = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond4 = and i1 %cond_1, %widenable_cond3
+  br i1 %exiplicit_guard_cond4, label %guarded1, label %deopt2, !prof !0
+
+deopt2:                                           ; preds = %guarded
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded1:                                         ; preds = %guarded
+  ret void
+}
+
+define void @trivial_wb(i1 %cond_0) {
+; CHECK-LABEL: @trivial_wb(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[WIDE_CHK:%.*]] = and i1 true, [[COND_0:%.*]]
+; CHECK-NEXT:    [[TMP0:%.*]] = and i1 [[WIDE_CHK]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[TMP0]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded:
+; CHECK-NEXT:    [[WIDENABLE_COND3:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND4:%.*]] = and i1 [[COND_0]], [[WIDENABLE_COND3]]
+; CHECK-NEXT:    br i1 true, label [[GUARDED1:%.*]], label [[DEOPT2:%.*]], !prof !0
+; CHECK:       deopt2:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded1:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  br i1 %widenable_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %entry
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded:                                          ; preds = %entry
+  %widenable_cond3 = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond4 = and i1 %cond_0, %widenable_cond3
+  br i1 %exiplicit_guard_cond4, label %guarded1, label %deopt2, !prof !0
+
+deopt2:                                           ; preds = %guarded
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded1:                                         ; preds = %guarded
+  ret void
+}
+
+
 declare void @llvm.experimental.deoptimize.isVoid(...)
 
 ; Function Attrs: inaccessiblememonly nounwind
diff --git a/llvm/test/Transforms/GuardWidening/mixed_guards.ll b/llvm/test/Transforms/GuardWidening/mixed_guards.ll
index 58908c0ff..f136f98da 100644
--- a/llvm/test/Transforms/GuardWidening/mixed_guards.ll
+++ b/llvm/test/Transforms/GuardWidening/mixed_guards.ll
@@ -45,8 +45,7 @@ define void @test_02(i1 %cond_0, i1 %cond_1) {
 ; CHECK-LABEL: @test_02(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
-; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_0:%.*]], [[WIDENABLE_COND]]
-; CHECK-NEXT:    [[WIDE_CHK:%.*]] = and i1 [[COND_0]], [[COND_1:%.*]]
+; CHECK-NEXT:    [[WIDE_CHK:%.*]] = and i1 [[COND_0:%.*]], [[COND_1:%.*]]
 ; CHECK-NEXT:    [[GUARD_CHK:%.*]] = and i1 [[WIDE_CHK]], [[WIDENABLE_COND]]
 ; CHECK-NEXT:    br i1 [[GUARD_CHK]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
 ; CHECK:       deopt:
diff --git a/llvm/test/Transforms/HardwareLoops/ARM/calls-codegen.ll b/llvm/test/Transforms/HardwareLoops/ARM/calls-codegen.ll
index 85160ced3..2771235d5 100644
--- a/llvm/test/Transforms/HardwareLoops/ARM/calls-codegen.ll
+++ b/llvm/test/Transforms/HardwareLoops/ARM/calls-codegen.ll
@@ -1,4 +1,7 @@
-; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+lob,+mve.fp -disable-arm-loloops=false %s -o - | FileCheck %s
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+lob,+mve.fp -disable-arm-loloops=true %s -o - | FileCheck %s --check-prefix=DISABLED
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+lob,+mve.fp %s -o - | FileCheck %s
+
+; DISABLED-NOT: dls lr,
 
 ; CHECK-LABEL: test_target_specific:
 ; CHECK:        mov.w lr, #50
diff --git a/llvm/test/Transforms/HardwareLoops/ARM/calls.ll b/llvm/test/Transforms/HardwareLoops/ARM/calls.ll
index d2cce5c38..43c6d06a1 100644
--- a/llvm/test/Transforms/HardwareLoops/ARM/calls.ll
+++ b/llvm/test/Transforms/HardwareLoops/ARM/calls.ll
@@ -1,8 +1,11 @@
-; RUN: opt -mtriple=thumbv8.1m.main-arm-none-eabi -hardware-loops -disable-arm-loloops=false %s -S -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-MAIN
-; RUN: opt -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+fullfp16 -hardware-loops -disable-arm-loloops=false %s -S -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-FP
-; RUN: opt -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+fp-armv8,+fullfp16 -hardware-loops -disable-arm-loloops=false %s -S -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-FP64
-; RUN: opt -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve -hardware-loops -disable-arm-loloops=false %s -S -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-MVE
-; RUN: opt -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve.fp -hardware-loops -disable-arm-loloops=false %s -S -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-MVEFP
+; RUN: opt -mtriple=thumbv8.1m.main-arm-none-eabi -hardware-loops %s -S -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-MAIN
+; RUN: opt -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+fullfp16 -hardware-loops %s -S -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-FP
+; RUN: opt -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+fp-armv8,+fullfp16 -hardware-loops %s -S -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-FP64
+; RUN: opt -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve -hardware-loops %s -S -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-MVE
+; RUN: opt -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve.fp -hardware-loops %s -S -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-MVEFP
+; RUN: opt -mtriple=thumbv8.1m.main-arm-none-eabi -hardware-loops -disable-arm-loloops=true %s -S -o - | FileCheck %s --check-prefix=DISABLED
+
+; DISABLED-NOT: call i32 @llvm.loop.decrement
 
 ; CHECK-LABEL: skip_call
 ; CHECK-NOT: call void @llvm.set.loop.iterations
diff --git a/llvm/test/Transforms/HardwareLoops/ARM/counter.ll b/llvm/test/Transforms/HardwareLoops/ARM/counter.ll
index bdd83d123..edb635464 100644
--- a/llvm/test/Transforms/HardwareLoops/ARM/counter.ll
+++ b/llvm/test/Transforms/HardwareLoops/ARM/counter.ll
@@ -1,4 +1,4 @@
-; RUN: opt -mtriple=thumbv8.1m.main-arm-none-eabi -hardware-loops -disable-arm-loloops=false %s -o - | FileCheck %s
+; RUN: opt -mtriple=thumbv8.1m.main-arm-none-eabi -hardware-loops %s -o - | FileCheck %s
 
 @g = common local_unnamed_addr global i32* null, align 4
 
diff --git a/llvm/test/Transforms/HardwareLoops/ARM/do-rem.ll b/llvm/test/Transforms/HardwareLoops/ARM/do-rem.ll
index 144600fe5..c3e0d88d8 100644
--- a/llvm/test/Transforms/HardwareLoops/ARM/do-rem.ll
+++ b/llvm/test/Transforms/HardwareLoops/ARM/do-rem.ll
@@ -1,4 +1,4 @@
-; RUN: opt -mtriple=thumbv8.1m.main-arm-none-eabi -hardware-loops -disable-arm-loloops=false %s -S -o - | FileCheck %s
+; RUN: opt -mtriple=thumbv8.1m.main-arm-none-eabi -hardware-loops %s -S -o - | FileCheck %s
 
 @g = common local_unnamed_addr global i32* null, align 4
 
diff --git a/llvm/test/Transforms/HardwareLoops/ARM/fp-emulation.ll b/llvm/test/Transforms/HardwareLoops/ARM/fp-emulation.ll
index ddb4c6a4c..4bf76b6cd 100644
--- a/llvm/test/Transforms/HardwareLoops/ARM/fp-emulation.ll
+++ b/llvm/test/Transforms/HardwareLoops/ARM/fp-emulation.ll
@@ -1,5 +1,5 @@
-; RUN: opt -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+fp-armv8 -hardware-loops -disable-arm-loloops=false %s -S -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-FP
-; RUN: opt -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+soft-float -hardware-loops -disable-arm-loloops=false %s -S -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-SOFT
+; RUN: opt -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+fp-armv8 -hardware-loops %s -S -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-FP
+; RUN: opt -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+soft-float -hardware-loops %s -S -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-SOFT
 
 ; CHECK-LABEL: test_fptosi
 ; CHECK-SOFT-NOT: call void @llvm.set.loop.iterations
diff --git a/llvm/test/Transforms/HardwareLoops/ARM/simple-do.ll b/llvm/test/Transforms/HardwareLoops/ARM/simple-do.ll
index da169eebf..8d5cdc1a3 100644
--- a/llvm/test/Transforms/HardwareLoops/ARM/simple-do.ll
+++ b/llvm/test/Transforms/HardwareLoops/ARM/simple-do.ll
@@ -1,7 +1,6 @@
-; RUN: opt -mtriple=thumbv8.1m.main-arm-none-eabi -hardware-loops -disable-arm-loloops=false %s -S -o - | FileCheck %s
+; RUN: opt -mtriple=thumbv8.1m.main-arm-none-eabi -hardware-loops %s -S -o - | FileCheck %s
 ; RUN: opt -mtriple=thumbv8.1m.main-arm-none-eabi -hardware-loops -disable-arm-loloops=true %s -S -o - | FileCheck %s --check-prefix=DISABLED
-; RUN: opt -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=-lob -hardware-loops %s -S -o - | FileCheck %s --check-prefix=DISABLED
-; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -disable-arm-loloops=false %s -o - | FileCheck %s --check-prefix=CHECK-LLC
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi %s -o - | FileCheck %s --check-prefix=CHECK-LLC
 
 ; DISABLED-NOT: llvm.{{.*}}.loop.iterations
 ; DISABLED-NOT: llvm.loop.decrement
diff --git a/llvm/test/Transforms/HardwareLoops/ARM/structure.ll b/llvm/test/Transforms/HardwareLoops/ARM/structure.ll
index 198cbef46..fbc09a175 100644
--- a/llvm/test/Transforms/HardwareLoops/ARM/structure.ll
+++ b/llvm/test/Transforms/HardwareLoops/ARM/structure.ll
@@ -1,6 +1,25 @@
-; RUN: opt -mtriple=thumbv8.1m.main-arm-none-eabi -hardware-loops -disable-arm-loloops=false %s -S -o - | FileCheck %s
-; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -disable-arm-loloops=false %s -o - | FileCheck %s --check-prefix=CHECK-LLC
-; RUN: opt -mtriple=thumbv8.1m.main -loop-unroll -unroll-remainder=false -S < %s | llc -mtriple=thumbv8.1m.main -disable-arm-loloops=false | FileCheck %s --check-prefix=CHECK-UNROLL
+; RUN: opt -mtriple=thumbv8.1m.main-arm-none-eabi -hardware-loops %s -S -o - | \
+; RUN:     FileCheck %s
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi %s -o - | \
+; RUN:     FileCheck %s --check-prefix=CHECK-LLC
+; RUN: opt -mtriple=thumbv8.1m.main -loop-unroll -unroll-remainder=false -S < %s | \
+; RUN:     llc -mtriple=thumbv8.1m.main | FileCheck %s --check-prefix=CHECK-UNROLL
+; RUN: opt -mtriple=thumbv8.1m.main-arm-none-eabi -hardware-loops \
+; RUN:     -pass-remarks-analysis=hardware-loops  %s -S -o - 2>&1 | \
+; RUN:     FileCheck %s --check-prefix=CHECK-REMARKS
+
+
+; CHECK-REMARKS: remark: <unknown>:0:0: hardware-loop not created: it's not profitable to create a hardware-loop
+; CHECK-REMARKS: remark: <unknown>:0:0: hardware-loop not created: nested hardware-loops not supported
+; CHECK-REMARKS: remark: <unknown>:0:0: hardware-loop not created: it's not profitable to create a hardware-loop
+; CHECK-REMARKS: remark: <unknown>:0:0: hardware-loop not created: it's not profitable to create a hardware-loop
+; CHECK-REMARKS: remark: <unknown>:0:0: hardware-loop not created: it's not profitable to create a hardware-loop
+; CHECK-REMARKS: remark: <unknown>:0:0: hardware-loop not created: it's not profitable to create a hardware-loop
+; CHECK-REMARKS: remark: <unknown>:0:0: hardware-loop not created: loop is not a candidate
+; CHECK-REMARKS: remark: <unknown>:0:0: hardware-loop not created: nested hardware-loops not supported
+; CHECK-REMARKS: remark: <unknown>:0:0: hardware-loop not created: it's not profitable to create a hardware-loop
+; CHECK-REMARKS: remark: <unknown>:0:0: hardware-loop not created: it's not profitable to create a hardware-loop
+
 
 ; CHECK-LABEL: early_exit
 ; CHECK-NOT: llvm.set.loop.iterations
diff --git a/llvm/test/Transforms/HotColdSplit/assumption-cache-invalidation.ll b/llvm/test/Transforms/HotColdSplit/assumption-cache-invalidation.ll
new file mode 100644
index 000000000..fbf2061ff
--- /dev/null
+++ b/llvm/test/Transforms/HotColdSplit/assumption-cache-invalidation.ll
@@ -0,0 +1,50 @@
+; REQUIRES: asserts
+; RUN: opt -S -instsimplify -hotcoldsplit -debug < %s 2>&1 | FileCheck %s
+; RUN: opt -instcombine -hotcoldsplit -instsimplify %s -o /dev/null
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64"
+
+%a = type { i64, i64 }
+%b = type { i64 }
+
+; CHECK: @f
+; CHECK-LABEL: codeRepl:
+; CHECK-NOT: @llvm.assume
+; CHECK: }
+; CHECK: declare {{.*}}@llvm.assume
+; CHECK: define {{.*}}@f.cold.1(i64 %0)
+; CHECK-LABEL: newFuncRoot:
+; CHECK: %1 = icmp eq i64 %0, 0
+; CHECK: call void @llvm.assume(i1 %1)
+
+define void @f() {
+entry:
+  %0 = getelementptr inbounds %a, %a* null, i64 0, i32 1
+  br label %label
+
+label:
+  %1 = bitcast i64* %0 to %b**
+  %2 = load %b*, %b** %1, align 8
+  %3 = getelementptr inbounds %b, %b* %2, i64 undef, i32 0
+  %4 = load i64, i64* %3, align 8
+  %5 = icmp ugt i64 %4, 1
+  br i1 %5, label %if.then, label %if.else
+
+if.then:
+  unreachable
+
+if.else:
+  call void @g(i8* undef)
+  %6 = load i64, i64* undef, align 8
+  %7 = and i64 %6, -16
+  %8 = inttoptr i64 %7 to i8*
+  %9 = icmp eq i64 %4, 0
+  call void @llvm.assume(i1 %9)
+  unreachable
+}
+
+declare void @g(i8*)
+
+declare void @llvm.assume(i1)
+
diff --git a/llvm/test/Transforms/HotColdSplit/noreturn.ll b/llvm/test/Transforms/HotColdSplit/noreturn.ll
index ca0f58815..74f4cb18c 100644
--- a/llvm/test/Transforms/HotColdSplit/noreturn.ll
+++ b/llvm/test/Transforms/HotColdSplit/noreturn.ll
@@ -23,6 +23,24 @@ define void @foo(i32, %struct.__jmp_buf_tag*) {
   ret void
 }
 
+; Don't outline within a noreturn function.
+
+; CHECK: define {{.*}}@xpc_objc_main(i32 {{.*}}) [[XPC_OBJC_MAIN_ATTRS:#[0-9]+]]
+; CHECK-NOT: xpc_objc_main.cold.1
+define void @xpc_objc_main(i32) noreturn {
+  %2 = icmp eq i32 %0, 0
+  tail call void @_Z10sideeffectv()
+  br i1 %2, label %4, label %3
+
+; <label>:3:                                      ; preds = %2
+  call void @_Z10sideeffectv()
+  unreachable
+
+; <label>:4:                                      ; preds = %2
+  ; Crash with an error message, "not supposed to return".
+  unreachable
+}
+
 ; Do outline noreturn calls marked cold.
 
 ; CHECK-LABEL: define {{.*}}@bar(
@@ -62,6 +80,8 @@ define void @baz(i32, %struct.__jmp_buf_tag*) {
 ; CHECK-LABEL: define {{.*}}@bar.cold.1(
 ; CHECK: call {{.*}}@llvm.trap(
 
+; CHECK: attributes [[XPC_OBJC_MAIN_ATTRS]] = { noreturn }
+
 declare void @sink() cold
 
 declare void @llvm.trap() noreturn cold
diff --git a/llvm/test/Transforms/HotColdSplit/retain-section.ll b/llvm/test/Transforms/HotColdSplit/retain-section.ll
new file mode 100644
index 000000000..af611a758
--- /dev/null
+++ b/llvm/test/Transforms/HotColdSplit/retain-section.ll
@@ -0,0 +1,28 @@
+; RUN: opt -hotcoldsplit -hotcoldsplit-threshold=0 -S < %s | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.14.0"
+
+; Retain section after cold code is split off.
+
+; CHECK-LABEL: @fun
+; CHECK: call void @fun.cold.1{{.*}}
+define void @fun() section ".text.cold" {
+entry:
+  br i1 undef, label %if.then, label %if.else
+
+if.then:
+  ret void
+
+if.else:
+  call void @sink()
+  ret void
+}
+
+define void @foo() cold {
+  ret void
+}
+
+declare void @sink() cold
+
+; CHECK: define {{.*}} @fun.cold.1{{.*}} section ".text.cold"
diff --git a/llvm/test/Transforms/IPConstantProp/2008-06-09-WeakProp.ll b/llvm/test/Transforms/IPConstantProp/2008-06-09-WeakProp.ll
index 54a65d61d..6471ed925 100644
--- a/llvm/test/Transforms/IPConstantProp/2008-06-09-WeakProp.ll
+++ b/llvm/test/Transforms/IPConstantProp/2008-06-09-WeakProp.ll
@@ -1,7 +1,9 @@
-; RUN: opt < %s -ipconstprop -S | grep "ret i32 %r"
+; RUN: opt < %s -ipconstprop -S | FileCheck %s
 ; Should not propagate the result of a weak function.
 ; PR2411
 
+; CHECK: ret i32 %r
+
 define weak i32 @foo() nounwind  {
 entry:
         ret i32 1
diff --git a/llvm/test/Transforms/IPConstantProp/2009-09-24-byval-ptr.ll b/llvm/test/Transforms/IPConstantProp/2009-09-24-byval-ptr.ll
index 61f4bf6d9..9a0276680 100644
--- a/llvm/test/Transforms/IPConstantProp/2009-09-24-byval-ptr.ll
+++ b/llvm/test/Transforms/IPConstantProp/2009-09-24-byval-ptr.ll
@@ -21,10 +21,10 @@ define internal i32 @vfu2(%struct.MYstr* byval align 4 %u) nounwind readonly {
 entry:
   %0 = getelementptr %struct.MYstr, %struct.MYstr* %u, i32 0, i32 1 ; <i32*> [#uses=1]
   %1 = load i32, i32* %0
-; CHECK: load i32, i32* getelementptr inbounds (%struct.MYstr, %struct.MYstr* @mystr, i32 0, i32 1)
+; CHECK: %struct.MYstr* @mystr, i{{..}} 0, i32 1
   %2 = getelementptr %struct.MYstr, %struct.MYstr* %u, i32 0, i32 0 ; <i8*> [#uses=1]
   %3 = load i8, i8* %2
-; CHECK: load i8, i8* getelementptr inbounds (%struct.MYstr, %struct.MYstr* @mystr, i32 0, i32 0)
+; CHECK: %struct.MYstr* @mystr, i{{..}} 0, i32 0
   %4 = zext i8 %3 to i32
   %5 = add i32 %4, %1
   ret i32 %5
@@ -34,7 +34,7 @@ define i32 @unions() nounwind {
 entry:
   call void @vfu1(%struct.MYstr* byval align 4 @mystr) nounwind
   %result = call i32 @vfu2(%struct.MYstr* byval align 4 @mystr) nounwind
-
+; CHECK: ret i32 %result
   ret i32 %result
 }
 
diff --git a/llvm/test/Transforms/IPConstantProp/PR16052.ll b/llvm/test/Transforms/IPConstantProp/PR16052.ll
index 959074d77..451693f1c 100644
--- a/llvm/test/Transforms/IPConstantProp/PR16052.ll
+++ b/llvm/test/Transforms/IPConstantProp/PR16052.ll
@@ -1,9 +1,15 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes 
 ; RUN: opt < %s -S -ipsccp | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define i64 @fn2() {
+; CHECK-LABEL: define {{[^@]+}}@fn2()
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CALL2:%.*]] = call i64 @fn1(i64 undef)
+; CHECK-NEXT:    ret i64 [[CALL2]]
+;
 entry:
   %conv = sext i32 undef to i64
   %div = sdiv i64 8, %conv
@@ -11,16 +17,15 @@ entry:
   ret i64 %call2
 }
 
-; CHECK-DAG: define i64 @fn2(
-; CHECK: %[[CALL:.*]] = call i64 @fn1(i64 undef)
-
 define internal i64 @fn1(i64 %p1) {
+; CHECK-LABEL: define {{[^@]+}}@fn1
+; CHECK-SAME: (i64 [[P1:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[COND:%.*]] = select i1 undef, i64 undef, i64 undef
+; CHECK-NEXT:    ret i64 [[COND]]
+;
 entry:
   %tobool = icmp ne i64 %p1, 0
   %cond = select i1 %tobool, i64 %p1, i64 %p1
   ret i64 %cond
 }
-
-; CHECK-DAG: define internal i64 @fn1(
-; CHECK: %[[SEL:.*]] = select i1 undef, i64 undef, i64 undef
-; CHECK: ret i64 %[[SEL]]
diff --git a/llvm/test/Transforms/IPConstantProp/PR26044.ll b/llvm/test/Transforms/IPConstantProp/PR26044.ll
index 6c6088682..eeb5b87e0 100644
--- a/llvm/test/Transforms/IPConstantProp/PR26044.ll
+++ b/llvm/test/Transforms/IPConstantProp/PR26044.ll
@@ -1,8 +1,20 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
 ; RUN: opt < %s -S -ipsccp | FileCheck %s
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
-define void @fn2() {
+define void @fn2(i32* %P) {
+; CHECK-LABEL: define {{[^@]+}}@fn2
+; CHECK-SAME: (i32* [[P:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[IF_END:%.*]]
+; CHECK:       for.cond1:
+; CHECK-NEXT:    br i1 false, label [[IF_END]], label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @fn1(i32 undef)
+; CHECK-NEXT:    store i32 [[CALL]], i32* [[P]]
+; CHECK-NEXT:    br label [[FOR_COND1:%.*]]
+;
 entry:
   br label %if.end
 
@@ -13,17 +25,36 @@ if.end:                                           ; preds = %lbl, %for.cond1
   %e.2 = phi i32* [ undef, %entry ], [ null, %for.cond1 ], [ null, %for.cond1 ]
   %0 = load i32, i32* %e.2, align 4
   %call = call i32 @fn1(i32 %0)
+  store i32 %call, i32* %P
   br label %for.cond1
 }
 
 define internal i32 @fn1(i32 %p1) {
+; CHECK-LABEL: define {{[^@]+}}@fn1
+; CHECK-SAME: (i32 [[P1:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[COND:%.*]] = select i1 undef, i32 undef, i32 undef
+; CHECK-NEXT:    ret i32 [[COND]]
+;
 entry:
   %tobool = icmp ne i32 %p1, 0
   %cond = select i1 %tobool, i32 %p1, i32 %p1
   ret i32 %cond
 }
 
-define void @fn_no_null_opt() #0 {
+define void @fn_no_null_opt(i32* %P) #0 {
+; CHECK-LABEL: define {{[^@]+}}@fn_no_null_opt
+; CHECK-SAME: (i32* [[P:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[IF_END:%.*]]
+; CHECK:       for.cond1:
+; CHECK-NEXT:    br i1 false, label [[IF_END]], label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* null, align 4
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @fn0(i32 [[TMP0]])
+; CHECK-NEXT:    store i32 [[CALL]], i32* [[P]]
+; CHECK-NEXT:    br label [[FOR_COND1:%.*]]
+;
 entry:
   br label %if.end
 
@@ -34,10 +65,18 @@ if.end:                                           ; preds = %lbl, %for.cond1
   %e.2 = phi i32* [ undef, %entry ], [ null, %for.cond1 ], [ null, %for.cond1 ]
   %0 = load i32, i32* %e.2, align 4
   %call = call i32 @fn0(i32 %0)
+  store i32 %call, i32* %P
   br label %for.cond1
 }
 
 define internal i32 @fn0(i32 %p1) {
+; CHECK-LABEL: define {{[^@]+}}@fn0
+; CHECK-SAME: (i32 [[P1:%.*]])
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne i32 [[P1]], 0
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[TOBOOL]], i32 [[P1]], i32 [[P1]]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
 entry:
   %tobool = icmp ne i32 %p1, 0
   %cond = select i1 %tobool, i32 %p1, i32 %p1
@@ -45,18 +84,3 @@ entry:
 }
 
 attributes #0 = { "null-pointer-is-valid"="true" }
-
-; CHECK-LABEL: define void @fn2(
-; CHECK: call i32 @fn1(i32 undef)
-
-; CHECK-LABEL: define internal i32 @fn1(
-; CHECK:%[[COND:.*]] = select i1 undef, i32 undef, i32 undef
-; CHECK: ret i32 %[[COND]]
-
-; CHECK-LABEL: define void @fn_no_null_opt(
-; CHECK: call i32 @fn0(i32 %0)
-
-; CHECK-LABEL: define internal i32 @fn0(
-; CHECK:%[[TOBOOL:.*]] = icmp ne i32 %p1, 0
-; CHECK:%[[COND:.*]] = select i1 %[[TOBOOL]], i32 %p1, i32 %p1
-; CHECK: ret i32 %[[COND]]
diff --git a/llvm/test/Transforms/IPConstantProp/PR43857.ll b/llvm/test/Transforms/IPConstantProp/PR43857.ll
new file mode 100644
index 000000000..0d0d14d39
--- /dev/null
+++ b/llvm/test/Transforms/IPConstantProp/PR43857.ll
@@ -0,0 +1,29 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -S -ipconstprop | FileCheck %s
+
+%struct.wobble = type { i32 }
+%struct.zot = type { %struct.wobble, %struct.wobble, %struct.wobble }
+
+declare dso_local fastcc float @bar(%struct.wobble* noalias, <8 x i32>) unnamed_addr
+
+define %struct.zot @widget(<8 x i32> %arg) local_unnamed_addr {
+; CHECK-LABEL: @widget(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    ret [[STRUCT_ZOT:%.*]] undef
+;
+bb:
+  ret %struct.zot undef
+}
+
+define void @baz(<8 x i32> %arg) local_unnamed_addr {
+; CHECK-LABEL: @baz(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP:%.*]] = call [[STRUCT_ZOT:%.*]] @widget(<8 x i32> [[ARG:%.*]])
+; CHECK-NEXT:    [[TMP1:%.*]] = extractvalue [[STRUCT_ZOT]] %tmp, 0, 0
+; CHECK-NEXT:    ret void
+;
+bb:
+  %tmp = call %struct.zot @widget(<8 x i32> %arg)
+  %tmp1 = extractvalue %struct.zot %tmp, 0, 0
+  ret void
+}
diff --git a/llvm/test/Transforms/IPConstantProp/arg-count-mismatch.ll b/llvm/test/Transforms/IPConstantProp/arg-count-mismatch.ll
index 1f62f6465..2a7d92d3f 100644
--- a/llvm/test/Transforms/IPConstantProp/arg-count-mismatch.ll
+++ b/llvm/test/Transforms/IPConstantProp/arg-count-mismatch.ll
@@ -50,10 +50,11 @@ define internal i16 @bar(i16 %p1, i16 %p2) {
 ; in argument count due to varargs (as long as all non-variadic arguments have
 ; been provided),
 
-define dso_local void @vararg_tests(i16 %a) {
+define dso_local i16 @vararg_tests(i16 %a) {
   %call1 = call i16 (i16, ...) @vararg_prop(i16 7, i16 8, i16 %a)
   %call2 = call i16 bitcast (i16 (i16, i16, ...) * @vararg_no_prop to i16 (i16) *) (i16 7)
-  ret void
+  %add = add i16 %call1, %call2
+  ret i16 %add
 }
 
 define internal i16 @vararg_prop(i16 %p1, ...) {
diff --git a/llvm/test/Transforms/IPConstantProp/multiple_callbacks.ll b/llvm/test/Transforms/IPConstantProp/multiple_callbacks.ll
index 3288b5b94..0d1df8dff 100644
--- a/llvm/test/Transforms/IPConstantProp/multiple_callbacks.ll
+++ b/llvm/test/Transforms/IPConstantProp/multiple_callbacks.ll
@@ -1,4 +1,5 @@
 ; RUN: opt -ipconstprop -S < %s | FileCheck %s
+; RUN: opt -S -passes=attributor -aa-pipeline='basic-aa' -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=1 < %s | FileCheck %s
 ;
 ;
 ;                            /---------------------------------------|
diff --git a/llvm/test/Transforms/IPConstantProp/naked-return.ll b/llvm/test/Transforms/IPConstantProp/naked-return.ll
index 3a2dedafc..ac3dc062a 100644
--- a/llvm/test/Transforms/IPConstantProp/naked-return.ll
+++ b/llvm/test/Transforms/IPConstantProp/naked-return.ll
@@ -26,4 +26,4 @@ bb:
 }
 
 attributes #0 = { naked }
-attributes #1 = { "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" }
+attributes #1 = { "frame-pointer"="all" }
diff --git a/llvm/test/Transforms/IPConstantProp/recursion.ll b/llvm/test/Transforms/IPConstantProp/recursion.ll
index b25a6c081..13aea8fc7 100644
--- a/llvm/test/Transforms/IPConstantProp/recursion.ll
+++ b/llvm/test/Transforms/IPConstantProp/recursion.ll
@@ -1,4 +1,7 @@
-; RUN: opt < %s -ipconstprop -deadargelim -S | not grep %X
+; RUN: opt < %s -ipconstprop -deadargelim -S | FileCheck %s
+
+; CHECK-NOT: %X
+
 define internal i32 @foo(i32 %X) {
         %Y = call i32 @foo( i32 %X )            ; <i32> [#uses=1]
         %Z = add i32 %Y, 1              ; <i32> [#uses=1]
diff --git a/llvm/test/Transforms/IPConstantProp/return-argument.ll b/llvm/test/Transforms/IPConstantProp/return-argument.ll
index 0290adc4b..830a6547b 100644
--- a/llvm/test/Transforms/IPConstantProp/return-argument.ll
+++ b/llvm/test/Transforms/IPConstantProp/return-argument.ll
@@ -1,6 +1,7 @@
-; RUN: opt < %s -ipconstprop -S > %t
-; RUN: cat %t | grep "store i32 %Z, i32\* %Q"
-; RUN: cat %t | grep "add i32 1, 3"
+; RUN: opt < %s -ipconstprop -S | FileCheck %s
+
+; CHECK: add i32 1, 3
+; CHECK: store i32 %Z, i32* %Q
 
 ;; This function returns its second argument on all return statements
 define internal i32* @incdec(i1 %C, i32* %V) {
diff --git a/llvm/test/Transforms/IPConstantProp/return-constant.ll b/llvm/test/Transforms/IPConstantProp/return-constant.ll
index 195420d6a..155a3011d 100644
--- a/llvm/test/Transforms/IPConstantProp/return-constant.ll
+++ b/llvm/test/Transforms/IPConstantProp/return-constant.ll
@@ -1,4 +1,9 @@
-; RUN: opt < %s -ipconstprop -instcombine -S | grep "ret i1 true" | count 2
+; RUN: opt < %s -ipconstprop -instcombine -S | FileCheck %s
+
+; CHECK: ret i1 true
+; CHECK: ret i1 true
+; CHECK-NOT: ret i1 true
+
 define internal i32 @foo(i1 %C) {
         br i1 %C, label %T, label %F
 
diff --git a/llvm/test/Transforms/IPConstantProp/return-constants.ll b/llvm/test/Transforms/IPConstantProp/return-constants.ll
index be2ca71c5..40b97ad04 100644
--- a/llvm/test/Transforms/IPConstantProp/return-constants.ll
+++ b/llvm/test/Transforms/IPConstantProp/return-constants.ll
@@ -1,8 +1,9 @@
-; RUN: opt < %s -ipconstprop -S > %t
+; RUN: opt < %s -ipconstprop -S | FileCheck %s
+
 ;; Check that the 21 constants got propagated properly
-; RUN: cat %t | grep "%M = add i32 21, 21"
+; CHECK: %M = add i32 21, 21
 ;; Check that the second return values didn't get propagated
-; RUN: cat %t | grep "%N = add i32 %B, %D"
+; CHECK: %N = add i32 %B, %D
 
 %0 = type { i32, i32 }
 
diff --git a/llvm/test/Transforms/IPConstantProp/solve-after-each-resolving-undefs-for-function.ll b/llvm/test/Transforms/IPConstantProp/solve-after-each-resolving-undefs-for-function.ll
index 5e7c32364..b7b3e7191 100644
--- a/llvm/test/Transforms/IPConstantProp/solve-after-each-resolving-undefs-for-function.ll
+++ b/llvm/test/Transforms/IPConstantProp/solve-after-each-resolving-undefs-for-function.ll
@@ -3,12 +3,15 @@
 ; CHECK-LABEL: @testf(
 ; CHECK:         ret i32 undef
 ;
-define internal i32 @testf() {
+define internal i32 @testf(i1 %c) {
 entry:
+  br i1 %c, label %if.cond, label %if.end
+
+if.cond:
   br i1 undef, label %if.then, label %if.end
 
 if.then:                                          ; preds = %entry, %if.then
-  br label %if.end
+  ret i32 11
 
 if.end:                                          ; preds = %if.then1, %entry
   ret i32 10
@@ -17,12 +20,12 @@ if.end:                                          ; preds = %if.then1, %entry
 ; CHECK-LABEL: @test1(
 ; CHECK:         ret i32 undef
 ;
-define internal i32 @test1() {
+define internal i32 @test1(i1 %c) {
 entry:
   br label %if.then
 
 if.then:                                          ; preds = %entry, %if.then
-  %call = call i32 @testf()
+  %call = call i32 @testf(i1 %c)
   %res = icmp eq i32 %call, 10
   br i1 %res, label %ret1, label %ret2
 
@@ -34,10 +37,10 @@ ret2:                                           ; preds = %if.then, %entry
 }
 
 ; CHECK-LABEL: @main(
-; CHECK-NEXT:    %res = call i32 @test1()
+; CHECK-NEXT:    %res = call i32 @test1(
 ; CHECK-NEXT:    ret i32 99
 ;
-define i32 @main() {
-  %res = call i32 @test1()
+define i32 @main(i1 %c) {
+  %res = call i32 @test1(i1 %c)
   ret i32 %res
 }
diff --git a/llvm/test/Transforms/IPConstantProp/user-with-multiple-uses.ll b/llvm/test/Transforms/IPConstantProp/user-with-multiple-uses.ll
index 3146709ae..92e1f3423 100644
--- a/llvm/test/Transforms/IPConstantProp/user-with-multiple-uses.ll
+++ b/llvm/test/Transforms/IPConstantProp/user-with-multiple-uses.ll
@@ -28,6 +28,3 @@ sw.default:
 return:
   ret i32 0
 }
-
-; CHECK: attributes #0 = { noreturn nounwind }
-; CHECK: attributes #1 = { nounwind }
diff --git a/llvm/test/Transforms/IRCE/ranges_of_different_types.ll b/llvm/test/Transforms/IRCE/ranges_of_different_types.ll
index d5c694b05..9903cb147 100644
--- a/llvm/test/Transforms/IRCE/ranges_of_different_types.ll
+++ b/llvm/test/Transforms/IRCE/ranges_of_different_types.ll
@@ -84,9 +84,7 @@ define void @test_02(i32* %arr, i32* %a_len_ptr) #0 {
 ; CHECK-NEXT:      [[SMAX1:%[^ ]+]] = select i1 [[CMP1]], i32 [[LEN_MINUS_SMAX]], i32 -13
 ; CHECK-NEXT:      [[SUB1:%[^ ]+]] = sub i32 %len, [[SMAX1]]
 ; CHECK-NEXT:      [[CMP2:%[^ ]+]] = icmp slt i32 [[SUB1]], 101
-; CHECK-NEXT:      [[SMAX2:%[^ ]+]] = select i1 [[CMP2]], i32 [[SUB1]], i32 101
-; CHECK-NEXT:      [[CMP3:%[^ ]+]] = icmp sgt i32 [[SMAX2]], 0
-; CHECK-NEXT:      %exit.mainloop.at = select i1 [[CMP3]], i32 [[SMAX2]], i32 0
+; CHECK-NEXT:      %exit.mainloop.at = select i1 [[CMP2]], i32 [[SUB1]], i32 101
 ; CHECK-NEXT:      br i1 true, label %loop.preloop.preheader
 ; CHECK:         loop.preloop:
 ; CHECK-NEXT:      %idx.preloop = phi i32 [ %idx.next.preloop, %in.bounds.preloop ], [ 0, %loop.preloop.preheader ]
@@ -291,9 +289,7 @@ define void @test_06(i32* %arr, i32* %a_len_ptr) #0 {
 ; CHECK-NEXT:      [[SMAX1:%[^ ]+]] = select i1 [[CMP1]], i32 [[LEN_MINUS_SMAX]], i32 -13
 ; CHECK-NEXT:      [[SUB1:%[^ ]+]] = sub i32 %len, [[SMAX1]]
 ; CHECK-NEXT:      [[CMP2:%[^ ]+]] = icmp slt i32 [[SUB1]], 101
-; CHECK-NEXT:      [[SMAX2:%[^ ]+]] = select i1 [[CMP2]], i32 [[SUB1]], i32 101
-; CHECK-NEXT:      [[CMP3:%[^ ]+]] = icmp sgt i32 [[SMAX2]], 0
-; CHECK-NEXT:      %exit.mainloop.at = select i1 [[CMP3]], i32 [[SMAX2]], i32 0
+; CHECK-NEXT:      %exit.mainloop.at = select i1 [[CMP2]], i32 [[SUB1]], i32 101
 ; CHECK-NEXT:      br i1 true, label %loop.preloop.preheader
 ; CHECK:         in.bounds.preloop:
 ; CHECK-NEXT:      %addr.preloop = getelementptr i32, i32* %arr, i32 %idx.preloop
diff --git a/llvm/test/Transforms/IRCE/rc-negative-bound.ll b/llvm/test/Transforms/IRCE/rc-negative-bound.ll
index d226bffea..e0af34bdf 100644
--- a/llvm/test/Transforms/IRCE/rc-negative-bound.ll
+++ b/llvm/test/Transforms/IRCE/rc-negative-bound.ll
@@ -114,18 +114,18 @@ define void @test_03(i32 *%arr, i32 %n, i32 %bound) {
 ; CHECK:       loop.preheader:
 ; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[BOUND:%.*]], -2147483647
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[TMP0]], 0
-; CHECK-NEXT:    [[SMIN:%.*]] = select i1 [[TMP1]], i32 [[TMP0]], i32 0
-; CHECK-NEXT:    [[TMP2:%.*]] = sub i32 [[BOUND]], [[SMIN]]
+; CHECK-NEXT:    [[SMAX:%.*]] = select i1 [[TMP1]], i32 [[TMP0]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = sub i32 [[BOUND]], [[SMAX]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp slt i32 [[BOUND]], 0
-; CHECK-NEXT:    [[SMAX:%.*]] = select i1 [[TMP3]], i32 [[BOUND]], i32 0
-; CHECK-NEXT:    [[TMP4:%.*]] = icmp sgt i32 [[SMAX]], -1
-; CHECK-NEXT:    [[SMIN1:%.*]] = select i1 [[TMP4]], i32 [[SMAX]], i32 -1
-; CHECK-NEXT:    [[TMP5:%.*]] = add i32 [[SMIN1]], 1
+; CHECK-NEXT:    [[SMIN:%.*]] = select i1 [[TMP3]], i32 [[BOUND]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp sgt i32 [[SMIN]], -1
+; CHECK-NEXT:    [[SMAX1:%.*]] = select i1 [[TMP4]], i32 [[SMIN]], i32 -1
+; CHECK-NEXT:    [[TMP5:%.*]] = add nsw i32 [[SMAX1]], 1
 ; CHECK-NEXT:    [[TMP6:%.*]] = mul i32 [[TMP2]], [[TMP5]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = icmp slt i32 [[N]], [[TMP6]]
-; CHECK-NEXT:    [[SMAX2:%.*]] = select i1 [[TMP7]], i32 [[N]], i32 [[TMP6]]
-; CHECK-NEXT:    [[TMP8:%.*]] = icmp sgt i32 [[SMAX2]], 0
-; CHECK-NEXT:    [[EXIT_MAINLOOP_AT:%.*]] = select i1 [[TMP8]], i32 [[SMAX2]], i32 0
+; CHECK-NEXT:    [[SMIN2:%.*]] = select i1 [[TMP7]], i32 [[N]], i32 [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp sgt i32 [[SMIN2]], 0
+; CHECK-NEXT:    [[EXIT_MAINLOOP_AT:%.*]] = select i1 [[TMP8]], i32 [[SMIN2]], i32 0
 ; CHECK-NEXT:    [[TMP9:%.*]] = icmp slt i32 0, [[EXIT_MAINLOOP_AT]]
 ; CHECK-NEXT:    br i1 [[TMP9]], label [[LOOP_PREHEADER4:%.*]], label [[MAIN_PSEUDO_EXIT:%.*]]
 ; CHECK:       loop.preheader4:
@@ -207,11 +207,11 @@ define void @test_04(i32 *%arr, i32 %n, i32 %bound) {
 ; CHECK-NEXT:    br i1 [[FIRST_ITR_CHECK]], label [[LOOP_PREHEADER:%.*]], label [[EXIT:%.*]]
 ; CHECK:       loop.preheader:
 ; CHECK-NEXT:    [[TMP0:%.*]] = icmp slt i32 [[BOUND:%.*]], 0
-; CHECK-NEXT:    [[SMAX:%.*]] = select i1 [[TMP0]], i32 [[BOUND]], i32 0
-; CHECK-NEXT:    [[TMP1:%.*]] = sub i32 [[BOUND]], [[SMAX]]
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp sgt i32 [[SMAX]], -1
-; CHECK-NEXT:    [[SMIN:%.*]] = select i1 [[TMP2]], i32 [[SMAX]], i32 -1
-; CHECK-NEXT:    [[TMP3:%.*]] = add i32 [[SMIN]], 1
+; CHECK-NEXT:    [[SMIN:%.*]] = select i1 [[TMP0]], i32 [[BOUND]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i32 [[BOUND]], [[SMIN]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp sgt i32 [[SMIN]], -1
+; CHECK-NEXT:    [[SMAX:%.*]] = select i1 [[TMP2]], i32 [[SMIN]], i32 -1
+; CHECK-NEXT:    [[TMP3:%.*]] = add nsw i32 [[SMAX]], 1
 ; CHECK-NEXT:    [[TMP4:%.*]] = mul i32 [[TMP1]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp ult i32 [[N]], [[TMP4]]
 ; CHECK-NEXT:    [[EXIT_MAINLOOP_AT:%.*]] = select i1 [[TMP5]], i32 [[N]], i32 [[TMP4]]
@@ -402,18 +402,18 @@ define void @test_07(i32 *%arr, i32 %n, i32 %bound) {
 ; CHECK:       loop.preheader:
 ; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[BOUND:%.*]], -2147483647
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[TMP0]], 0
-; CHECK-NEXT:    [[SMIN:%.*]] = select i1 [[TMP1]], i32 [[TMP0]], i32 0
-; CHECK-NEXT:    [[TMP2:%.*]] = sub i32 [[BOUND]], [[SMIN]]
+; CHECK-NEXT:    [[SMAX:%.*]] = select i1 [[TMP1]], i32 [[TMP0]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = sub i32 [[BOUND]], [[SMAX]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp slt i32 [[BOUND]], 0
-; CHECK-NEXT:    [[SMAX:%.*]] = select i1 [[TMP3]], i32 [[BOUND]], i32 0
-; CHECK-NEXT:    [[TMP4:%.*]] = icmp sgt i32 [[SMAX]], -1
-; CHECK-NEXT:    [[SMIN1:%.*]] = select i1 [[TMP4]], i32 [[SMAX]], i32 -1
-; CHECK-NEXT:    [[TMP5:%.*]] = add i32 [[SMIN1]], 1
+; CHECK-NEXT:    [[SMIN:%.*]] = select i1 [[TMP3]], i32 [[BOUND]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp sgt i32 [[SMIN]], -1
+; CHECK-NEXT:    [[SMAX1:%.*]] = select i1 [[TMP4]], i32 [[SMIN]], i32 -1
+; CHECK-NEXT:    [[TMP5:%.*]] = add nsw i32 [[SMAX1]], 1
 ; CHECK-NEXT:    [[TMP6:%.*]] = mul i32 [[TMP2]], [[TMP5]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = icmp slt i32 [[N]], [[TMP6]]
-; CHECK-NEXT:    [[SMAX2:%.*]] = select i1 [[TMP7]], i32 [[N]], i32 [[TMP6]]
-; CHECK-NEXT:    [[TMP8:%.*]] = icmp sgt i32 [[SMAX2]], 0
-; CHECK-NEXT:    [[EXIT_MAINLOOP_AT:%.*]] = select i1 [[TMP8]], i32 [[SMAX2]], i32 0
+; CHECK-NEXT:    [[SMIN2:%.*]] = select i1 [[TMP7]], i32 [[N]], i32 [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp sgt i32 [[SMIN2]], 0
+; CHECK-NEXT:    [[EXIT_MAINLOOP_AT:%.*]] = select i1 [[TMP8]], i32 [[SMIN2]], i32 0
 ; CHECK-NEXT:    [[TMP9:%.*]] = icmp slt i32 0, [[EXIT_MAINLOOP_AT]]
 ; CHECK-NEXT:    br i1 [[TMP9]], label [[LOOP_PREHEADER4:%.*]], label [[MAIN_PSEUDO_EXIT:%.*]]
 ; CHECK:       loop.preheader4:
@@ -497,11 +497,11 @@ define void @test_08(i32 *%arr, i32 %n, i32 %bound) {
 ; CHECK-NEXT:    br i1 [[FIRST_ITR_CHECK]], label [[LOOP_PREHEADER:%.*]], label [[EXIT:%.*]]
 ; CHECK:       loop.preheader:
 ; CHECK-NEXT:    [[TMP0:%.*]] = icmp slt i32 [[BOUND:%.*]], 0
-; CHECK-NEXT:    [[SMAX:%.*]] = select i1 [[TMP0]], i32 [[BOUND]], i32 0
-; CHECK-NEXT:    [[TMP1:%.*]] = sub i32 [[BOUND]], [[SMAX]]
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp sgt i32 [[SMAX]], -1
-; CHECK-NEXT:    [[SMIN:%.*]] = select i1 [[TMP2]], i32 [[SMAX]], i32 -1
-; CHECK-NEXT:    [[TMP3:%.*]] = add i32 [[SMIN]], 1
+; CHECK-NEXT:    [[SMIN:%.*]] = select i1 [[TMP0]], i32 [[BOUND]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i32 [[BOUND]], [[SMIN]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp sgt i32 [[SMIN]], -1
+; CHECK-NEXT:    [[SMAX:%.*]] = select i1 [[TMP2]], i32 [[SMIN]], i32 -1
+; CHECK-NEXT:    [[TMP3:%.*]] = add nsw i32 [[SMAX]], 1
 ; CHECK-NEXT:    [[TMP4:%.*]] = mul i32 [[TMP1]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp ult i32 [[N]], [[TMP4]]
 ; CHECK-NEXT:    [[EXIT_MAINLOOP_AT:%.*]] = select i1 [[TMP5]], i32 [[N]], i32 [[TMP4]]
diff --git a/llvm/test/Transforms/IndVarSimplify/eliminate-exit-no-dl.ll b/llvm/test/Transforms/IndVarSimplify/eliminate-exit-no-dl.ll
new file mode 100644
index 000000000..0b1609b30
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/eliminate-exit-no-dl.ll
@@ -0,0 +1,45 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -indvars -S < %s | FileCheck %s
+
+; Check the case where one exit has a pointer EC, and the other doesn't.
+; Note that this test case is really really fragile.  Removing any
+; instruction in the below causes the result to differ.  Note that the lack
+; of a data layout (with pointer size info) is critical to getting a pointer
+; EC returned by SCEV.
+
+@global = external global [0 x i8], align 1
+
+define void @foo() {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    br label [[BB3:%.*]]
+; CHECK:       bb3:
+; CHECK-NEXT:    [[TMP:%.*]] = phi i8* [ [[TMP4:%.*]], [[BB7:%.*]] ], [ getelementptr inbounds ([0 x i8], [0 x i8]* @global, i64 0, i64 2), [[BB:%.*]] ]
+; CHECK-NEXT:    [[TMP4]] = getelementptr inbounds i8, i8* [[TMP]], i64 -1
+; CHECK-NEXT:    [[TMP6:%.*]] = load i8, i8* [[TMP4]], align 1
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ugt i8* [[TMP4]], getelementptr inbounds ([0 x i8], [0 x i8]* @global, i64 0, i64 500)
+; CHECK-NEXT:    br i1 [[TMP5]], label [[BB7]], label [[BB11:%.*]]
+; CHECK:       bb7:
+; CHECK-NEXT:    [[TMP8:%.*]] = zext i8 [[TMP6]] to i64
+; CHECK-NEXT:    br i1 true, label [[BB11]], label [[BB3]]
+; CHECK:       bb11:
+; CHECK-NEXT:    ret void
+;
+bb:
+  br label %bb3
+
+bb3:                                              ; preds = %bb7, %bb2
+  %tmp = phi i8* [ %tmp4, %bb7 ], [ getelementptr inbounds ([0 x i8], [0 x i8]* @global, i64 0, i64 2), %bb ]
+  %tmp4 = getelementptr inbounds i8, i8* %tmp, i64 -1
+  %tmp6 = load i8, i8* %tmp4, align 1
+  %tmp5 = icmp ugt i8* %tmp4, getelementptr inbounds ([0 x i8], [0 x i8]* @global, i64 0, i64 500)
+  br i1 %tmp5, label %bb7, label %bb11
+
+bb7:                                              ; preds = %bb3
+  %tmp8 = zext i8 %tmp6 to i64
+  %tmp10 = icmp eq i16 0, 0
+  br i1 %tmp10, label %bb11, label %bb3
+
+bb11:                                             ; preds = %bb7, %bb3
+  ret void
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/eliminate-exit.ll b/llvm/test/Transforms/IndVarSimplify/eliminate-exit.ll
index b1c6c4d2a..726f3cd7f 100644
--- a/llvm/test/Transforms/IndVarSimplify/eliminate-exit.ll
+++ b/llvm/test/Transforms/IndVarSimplify/eliminate-exit.ll
@@ -185,5 +185,39 @@ exit:
   ret void
 }
 
+define void @mixed_width(i32 %len) {
+; CHECK-LABEL: @mixed_width(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LEN_ZEXT:%.*]] = zext i32 [[LEN:%.*]] to i64
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult i64 [[IV]], [[LEN_ZEXT]]
+; CHECK-NEXT:    br i1 [[CMP1]], label [[BACKEDGE]], label [[EXIT:%.*]]
+; CHECK:       backedge:
+; CHECK-NEXT:    call void @side_effect()
+; CHECK-NEXT:    br i1 true, label [[LOOP]], label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %len.zext = zext i32 %len to i64
+  br label %loop
+loop:
+  %iv = phi i64 [0, %entry], [%iv.next, %backedge]
+  %iv2 = phi i32 [0, %entry], [%iv2.next, %backedge]
+  %iv.next = add i64 %iv, 1
+  %iv2.next = add i32 %iv2, 1
+  %cmp1 = icmp ult i64 %iv, %len.zext
+  br i1 %cmp1, label %backedge, label %exit
+
+backedge:
+  call void @side_effect()
+  %cmp2 = icmp ult i32 %iv2, %len
+  br i1 %cmp2, label %loop, label %exit
+exit:
+  ret void
+}
 
 declare void @side_effect()
diff --git a/llvm/test/Transforms/IndVarSimplify/eliminate-trunc.ll b/llvm/test/Transforms/IndVarSimplify/eliminate-trunc.ll
index 74e422b79..8258445d4 100644
--- a/llvm/test/Transforms/IndVarSimplify/eliminate-trunc.ll
+++ b/llvm/test/Transforms/IndVarSimplify/eliminate-trunc.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -indvars -S < %s | FileCheck %s
+; RUN: opt -indvars -S -indvars-predicate-loops=0 < %s | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/Transforms/IndVarSimplify/exit_value_tests.ll b/llvm/test/Transforms/IndVarSimplify/exit_value_tests.ll
index 619ee4d08..f06164e4a 100644
--- a/llvm/test/Transforms/IndVarSimplify/exit_value_tests.ll
+++ b/llvm/test/Transforms/IndVarSimplify/exit_value_tests.ll
@@ -226,7 +226,6 @@ loopexit:
 define i32 @unroll_phi_select(i32 %arg1, i32 %arg2, i16 %len) {
 ; CHECK-LABEL: @unroll_phi_select(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[LENGTH:%.*]] = zext i16 [[LEN:%.*]] to i32
 ; CHECK-NEXT:    ret i32 [[ARG2:%.*]]
 ;
 entry:
diff --git a/llvm/test/Transforms/IndVarSimplify/floating-point-iv.ll b/llvm/test/Transforms/IndVarSimplify/floating-point-iv.ll
index d8a215891..97fe20c05 100644
--- a/llvm/test/Transforms/IndVarSimplify/floating-point-iv.ll
+++ b/llvm/test/Transforms/IndVarSimplify/floating-point-iv.ll
@@ -139,3 +139,234 @@ define void @test5() nounwind {
 exit:
   ret void
 }
+
+define double @test_max_be() {
+; CHECK-LABEL: @test_max_be(
+; CHECK-NEXT:  bb4:
+; CHECK-NEXT:    br label [[BB8:%.*]]
+; CHECK:       bb8:
+; CHECK-NEXT:    [[TMP10:%.*]] = phi double [ 0.000000e+00, [[BB4:%.*]] ], [ [[TMP12:%.*]], [[BB22:%.*]] ]
+; CHECK-NEXT:    [[TMP11_INT:%.*]] = phi i32 [ 0, [[BB4]] ], [ [[TMP13_INT:%.*]], [[BB22]] ]
+; CHECK-NEXT:    [[INDVAR_CONV:%.*]] = sitofp i32 [[TMP11_INT]] to double
+; CHECK-NEXT:    [[TMP12]] = fadd double [[TMP10]], [[INDVAR_CONV]]
+; CHECK-NEXT:    [[TMP13_INT]] = add nuw nsw i32 [[TMP11_INT]], 1
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp slt i32 [[TMP13_INT]], 99999
+; CHECK-NEXT:    br i1 [[TMP14]], label [[BB22]], label [[BB6:%.*]]
+; CHECK:       bb22:
+; CHECK-NEXT:    br i1 true, label [[BB8]], label [[BB6]]
+; CHECK:       bb6:
+; CHECK-NEXT:    [[TMP12_LCSSA:%.*]] = phi double [ [[TMP12]], [[BB22]] ], [ [[TMP12]], [[BB8]] ]
+; CHECK-NEXT:    ret double [[TMP12_LCSSA]]
+;
+bb4:
+  br label %bb8
+
+bb8:
+  %tmp9 = phi i64 [ 1, %bb4 ], [ %tmp23, %bb22 ]
+  %tmp10 = phi double [ 0.000000e+00, %bb4 ], [ %tmp12, %bb22 ]
+  %tmp11 = phi double [ 0.000000e+00, %bb4 ], [ %tmp13, %bb22 ]
+  %tmp12 = fadd double %tmp10, %tmp11
+  %tmp13 = fadd double %tmp11, 1.000000e+00
+  %tmp14 = fcmp olt double %tmp13, 9.999900e+04
+  br i1 %tmp14, label %bb22, label %bb6
+
+bb22:
+  %tmp23 = add nuw nsw i64 %tmp9, 1
+  %tmp24 = icmp ult i64 %tmp9, 1048576
+  br i1 %tmp24, label %bb8, label %bb6
+
+bb6:
+  ret double %tmp12
+}
+
+define float @test_max_be2() {
+; CHECK-LABEL: @test_max_be2(
+; CHECK-NEXT:  bb4:
+; CHECK-NEXT:    br label [[BB8:%.*]]
+; CHECK:       bb8:
+; CHECK-NEXT:    [[TMP10:%.*]] = phi float [ 0.000000e+00, [[BB4:%.*]] ], [ [[TMP12:%.*]], [[BB22:%.*]] ]
+; CHECK-NEXT:    [[TMP11_INT:%.*]] = phi i32 [ 0, [[BB4]] ], [ [[TMP13_INT:%.*]], [[BB22]] ]
+; CHECK-NEXT:    [[INDVAR_CONV:%.*]] = sitofp i32 [[TMP11_INT]] to float
+; CHECK-NEXT:    [[TMP12]] = fadd float [[TMP10]], [[INDVAR_CONV]]
+; CHECK-NEXT:    [[TMP13_INT]] = add nuw nsw i32 [[TMP11_INT]], 1
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp slt i32 [[TMP13_INT]], 99999
+; CHECK-NEXT:    br i1 [[TMP14]], label [[BB22]], label [[BB6:%.*]]
+; CHECK:       bb22:
+; CHECK-NEXT:    br i1 true, label [[BB8]], label [[BB6]]
+; CHECK:       bb6:
+; CHECK-NEXT:    [[TMP12_LCSSA:%.*]] = phi float [ [[TMP12]], [[BB22]] ], [ [[TMP12]], [[BB8]] ]
+; CHECK-NEXT:    ret float [[TMP12_LCSSA]]
+;
+bb4:
+  br label %bb8
+
+bb8:
+  %tmp9 = phi i64 [ 1, %bb4 ], [ %tmp23, %bb22 ]
+  %tmp10 = phi float [ 0.000000e+00, %bb4 ], [ %tmp12, %bb22 ]
+  %tmp11 = phi float [ 0.000000e+00, %bb4 ], [ %tmp13, %bb22 ]
+  %tmp12 = fadd float %tmp10, %tmp11
+  %tmp13 = fadd float %tmp11, 1.000000e+00
+  %tmp14 = fcmp olt float %tmp13, 9.999900e+04
+  br i1 %tmp14, label %bb22, label %bb6
+
+bb22:
+  %tmp23 = add nuw nsw i64 %tmp9, 1
+  %tmp24 = icmp ult i64 %tmp9, 1048576
+  br i1 %tmp24, label %bb8, label %bb6
+
+bb6:
+  ret float %tmp12
+}
+
+; Bounds check
+define float @test_max_be3() {
+; CHECK-LABEL: @test_max_be3(
+; CHECK-NEXT:  bb4:
+; CHECK-NEXT:    br label [[BB8:%.*]]
+; CHECK:       bb8:
+; CHECK-NEXT:    [[TMP10:%.*]] = phi float [ 0.000000e+00, [[BB4:%.*]] ], [ [[TMP12:%.*]], [[BB22:%.*]] ]
+; CHECK-NEXT:    [[TMP11_INT:%.*]] = phi i32 [ 0, [[BB4]] ], [ [[TMP13_INT:%.*]], [[BB22]] ]
+; CHECK-NEXT:    [[INDVAR_CONV:%.*]] = sitofp i32 [[TMP11_INT]] to float
+; CHECK-NEXT:    [[TMP12]] = fadd float [[TMP10]], [[INDVAR_CONV]]
+; CHECK-NEXT:    [[TMP13_INT]] = add nuw nsw i32 [[TMP11_INT]], 1
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp slt i32 [[TMP13_INT]], 99999
+; CHECK-NEXT:    br i1 [[TMP14]], label [[BB22]], label [[BB6:%.*]]
+; CHECK:       bb22:
+; CHECK-NEXT:    br i1 true, label [[BB8]], label [[BB6]]
+; CHECK:       bb6:
+; CHECK-NEXT:    [[TMP12_LCSSA:%.*]] = phi float [ [[TMP12]], [[BB22]] ], [ [[TMP12]], [[BB8]] ]
+; CHECK-NEXT:    ret float [[TMP12_LCSSA]]
+;
+bb4:
+  br label %bb8
+
+bb8:
+  %tmp9 = phi i64 [ 1, %bb4 ], [ %tmp23, %bb22 ]
+  %tmp10 = phi float [ 0.000000e+00, %bb4 ], [ %tmp12, %bb22 ]
+  %tmp11 = phi float [ 0.000000e+00, %bb4 ], [ %tmp13, %bb22 ]
+  %tmp12 = fadd float %tmp10, %tmp11
+  %tmp13 = fadd float %tmp11, 1.000000e+00
+  %tmp14 = fcmp olt float %tmp13, 9.999900e+04
+  br i1 %tmp14, label %bb22, label %bb6
+
+bb22:
+  %tmp23 = add nuw nsw i64 %tmp9, 1
+  ;; 2^23 = 16777215
+  %tmp24 = icmp ult i64 %tmp9, 16777215
+  br i1 %tmp24, label %bb8, label %bb6
+
+bb6:
+  ret float %tmp12
+}
+
+
+; Show that given a computeable exit count, we can remove an
+; fcmp of a casted integer IV. (TODO)
+define void @fcmp1() nounwind {
+; CHECK-LABEL: @fcmp1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[BB:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult i64 [[IV]], 20000
+; CHECK-NEXT:    br i1 [[CMP1]], label [[BACKEDGE]], label [[RETURN:%.*]]
+; CHECK:       backedge:
+; CHECK-NEXT:    [[IV_FP:%.*]] = sitofp i64 [[IV]] to double
+; CHECK-NEXT:    [[TMP0:%.*]] = tail call i32 @foo(double [[IV_FP]]) #0
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[CMP2:%.*]] = fcmp olt double [[IV_FP]], 1.000000e+04
+; CHECK-NEXT:    br i1 [[CMP2]], label [[BB]], label [[RETURN]]
+; CHECK:       return:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %bb
+
+bb:		; preds = %bb, %entry
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %backedge ]
+  %cmp1 = icmp slt i64 %iv, 20000
+  br i1 %cmp1, label %backedge, label %return
+
+backedge:
+  %iv.fp = sitofp i64 %iv to double
+  tail call i32 @foo(double %iv.fp) nounwind
+  %iv.next = add nsw nuw i64 %iv, 1
+  %cmp2 = fcmp olt double %iv.fp, 1.000000e+04
+  br i1 %cmp2, label %bb, label %return
+
+return:		; preds = %bb
+  ret void
+}
+
+define void @fcmp2() nounwind {
+; CHECK-LABEL: @fcmp2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[BB:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult i64 [[IV]], 2000
+; CHECK-NEXT:    br i1 [[CMP1]], label [[BACKEDGE]], label [[RETURN:%.*]]
+; CHECK:       backedge:
+; CHECK-NEXT:    [[IV_FP:%.*]] = sitofp i64 [[IV]] to double
+; CHECK-NEXT:    [[TMP0:%.*]] = tail call i32 @foo(double [[IV_FP]]) #0
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[CMP2:%.*]] = fcmp olt double [[IV_FP]], 1.000000e+04
+; CHECK-NEXT:    br i1 [[CMP2]], label [[BB]], label [[RETURN]]
+; CHECK:       return:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %bb
+
+bb:		; preds = %bb, %entry
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %backedge ]
+  %cmp1 = icmp slt i64 %iv, 2000
+  br i1 %cmp1, label %backedge, label %return
+
+backedge:
+  %iv.fp = sitofp i64 %iv to double
+  tail call i32 @foo(double %iv.fp) nounwind
+  %iv.next = add nsw nuw i64 %iv, 1
+  %cmp2 = fcmp olt double %iv.fp, 1.000000e+04
+  br i1 %cmp2, label %bb, label %return
+
+return:		; preds = %bb
+  ret void
+}
+
+define void @fcmp_neg1() nounwind {
+; CHECK-LABEL: @fcmp_neg1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[BB:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult i64 [[IV]], -20
+; CHECK-NEXT:    br i1 [[CMP1]], label [[BACKEDGE]], label [[RETURN:%.*]]
+; CHECK:       backedge:
+; CHECK-NEXT:    [[IV_FP:%.*]] = sitofp i64 [[IV]] to double
+; CHECK-NEXT:    [[TMP0:%.*]] = tail call i32 @foo(double [[IV_FP]]) #0
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw i64 [[IV]], 1
+; CHECK-NEXT:    [[CMP2:%.*]] = fcmp olt double [[IV_FP]], 1.000000e+04
+; CHECK-NEXT:    br i1 [[CMP2]], label [[BB]], label [[RETURN]]
+; CHECK:       return:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %bb
+
+bb:		; preds = %bb, %entry
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %backedge ]
+  ;; Range fact outside precise integer region
+  %cmp1 = icmp ult i64 %iv, -20
+  br i1 %cmp1, label %backedge, label %return
+
+backedge:
+  %iv.fp = sitofp i64 %iv to double
+  tail call i32 @foo(double %iv.fp) nounwind
+  %iv.next = add nuw i64 %iv, 1
+  %cmp2 = fcmp olt double %iv.fp, 1.000000e+04
+  br i1 %cmp2, label %bb, label %return
+
+return:		; preds = %bb
+  ret void
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/indvar-debug-value.ll b/llvm/test/Transforms/IndVarSimplify/indvar-debug-value.ll
new file mode 100644
index 000000000..06d5f719d
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/indvar-debug-value.ll
@@ -0,0 +1,92 @@
+; RUN: opt %s -indvars -verify -S -o - | FileCheck %s
+
+; Hand-reduced from this example:
+;
+; #include <stdio.h>
+
+; int main( int argc, char **argv )
+; {
+;    for( int ArgIndex = 1; ArgIndex < argc; ArgIndex += 1 )
+;    {
+;       printf("\n Argument %d:  %s\n", ArgIndex, argv[ArgIndex] );
+;    }
+; }
+
+; clang++ -g -O -mllvm -disable-llvm-optzns -gno-column-info
+; opt  -mem2reg -loop-rotate -scalar-evolution
+
+; CHECK: @main
+; CHECK: llvm.dbg.value(metadata i32 1, metadata [[METADATA_IDX1:![0-9]+]]
+; CHECK: %[[VAR_NAME:.*]] = add nuw nsw i64
+; CHECK: llvm.dbg.value(metadata i64 %[[VAR_NAME]], metadata [[METADATA_IDX1]], metadata !DIExpression())
+; CHECK: DICompileUnit
+; CHECK: [[METADATA_IDX1]] = !DILocalVariable(name: "ArgIndex"
+
+source_filename = "test.cpp"
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@.str = private unnamed_addr constant [20 x i8] c"\0A Argument %d:  %s\0A\00", align 1
+
+define dso_local i32 @main(i32 %argc, i8** %argv) !dbg !7 {
+entry:
+  call void @llvm.dbg.value(metadata i32 1, metadata !17, metadata !DIExpression()), !dbg !19
+  %cmp1 = icmp slt i32 1, %argc, !dbg !19
+  br i1 %cmp1, label %for.body.lr.ph, label %for.cond.cleanup, !dbg !19
+
+for.body.lr.ph:                                   ; preds = %entry
+  br label %for.body, !dbg !19
+
+for.cond.for.cond.cleanup_crit_edge:              ; preds = %for.inc
+  br label %for.cond.cleanup, !dbg !19
+
+for.cond.cleanup:                                 ; preds = %for.cond.for.cond.cleanup_crit_edge, %entry
+  br label %for.end
+
+for.body:                                         ; preds = %for.body.lr.ph, %for.inc
+  %ArgIndex.02 = phi i32 [ 1, %for.body.lr.ph ], [ %add, %for.inc ]
+  call void @llvm.dbg.value(metadata i32 %ArgIndex.02, metadata !17, metadata !DIExpression()), !dbg !19
+  %idxprom = sext i32 %ArgIndex.02 to i64, !dbg !19
+  %arrayidx = getelementptr inbounds i8*, i8** %argv, i64 %idxprom, !dbg !19
+  %0 = load i8*, i8** %arrayidx, align 8, !dbg !19
+  %call = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([20 x i8], [20 x i8]* @.str, i64 0, i64 0), i32 %ArgIndex.02, i8* %0), !dbg !19
+  br label %for.inc, !dbg !19
+
+for.inc:                                          ; preds = %for.body
+  %add = add nsw i32 %ArgIndex.02, 1, !dbg !19
+  call void @llvm.dbg.value(metadata i32 %add, metadata !17, metadata !DIExpression()), !dbg !19
+  %cmp = icmp slt i32 %add, %argc, !dbg !19
+  br i1 %cmp, label %for.body, label %for.cond.for.cond.cleanup_crit_edge, !dbg !19, !llvm.loop !19
+
+for.end:                                          ; preds = %for.cond.cleanup
+  ret i32 0, !dbg !19
+}
+
+declare dso_local i32 @printf(i8*, ...)
+
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "test.cpp", directory: "")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{!"clang"}
+!7 = distinct !DISubprogram(name: "main", scope: !1, file: !1, line: 4, type: !8, scopeLine: 5, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !14)
+!8 = !DISubroutineType(types: !9)
+!9 = !{!10, !10, !11}
+!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!11 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !12, size: 64)
+!12 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !13, size: 64)
+!13 = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_signed_char)
+!14 = !{!15, !16, !17}
+!15 = !DILocalVariable(name: "argc", arg: 1, scope: !7, file: !1, line: 4, type: !10)
+!16 = !DILocalVariable(name: "argv", arg: 2, scope: !7, file: !1, line: 4, type: !11)
+!17 = !DILocalVariable(name: "ArgIndex", scope: !18, file: !1, line: 6, type: !10)
+!18 = distinct !DILexicalBlock(scope: !7, file: !1, line: 6)
+!19 = !DILocation(line: 0, scope: !7)
diff --git a/llvm/test/Transforms/IndVarSimplify/indvar-debug-value2.ll b/llvm/test/Transforms/IndVarSimplify/indvar-debug-value2.ll
new file mode 100644
index 000000000..3859111eb
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/indvar-debug-value2.ll
@@ -0,0 +1,90 @@
+; RUN: opt %s -indvars -verify -S -o - | FileCheck %s
+
+; Hand-reduced from this example:
+;
+; #include <stdio.h>
+
+; int main( int argc, char **argv )
+; {
+;    for( int ArgIndex = 1; ArgIndex < argc; ArgIndex += 1 )
+;    {
+;       printf("\n Argument %d:  %s\n", ArgIndex, argv[ArgIndex] );
+;    }
+; }
+
+; clang++ -g -O -mllvm -disable-llvm-optzns -gno-column-info
+; opt  -mem2reg -scalar-evolution
+
+; CHECK: @main
+; CHECK: llvm.dbg.value(metadata i32 1, metadata [[METADATA_IDX1:![0-9]+]]
+; CHECK: %[[VAR_NAME:.*]] = add nuw nsw i64
+; CHECK: llvm.dbg.value(metadata i64 %[[VAR_NAME]], metadata [[METADATA_IDX1]], metadata !DIExpression())
+; CHECK: DICompileUnit
+; CHECK: [[METADATA_IDX1]] = !DILocalVariable(name: "ArgIndex"
+
+source_filename = "test.cpp"
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@.str = private unnamed_addr constant [20 x i8] c"\0A Argument %d:  %s\0A\00", align 1
+
+define dso_local i32 @main(i32 %argc, i8** %argv) !dbg !7 {
+entry:
+  call void @llvm.dbg.value(metadata i32 %argc, metadata !15, metadata !DIExpression()), !dbg !19
+  call void @llvm.dbg.value(metadata i8** %argv, metadata !16, metadata !DIExpression()), !dbg !19
+  call void @llvm.dbg.value(metadata i32 1, metadata !17, metadata !DIExpression()), !dbg !19
+  br label %for.cond, !dbg !19
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %ArgIndex.0 = phi i32 [ 1, %entry ], [ %add, %for.inc ], !dbg !19
+  call void @llvm.dbg.value(metadata i32 %ArgIndex.0, metadata !17, metadata !DIExpression()), !dbg !19
+  %cmp = icmp slt i32 %ArgIndex.0, %argc, !dbg !19
+  br i1 %cmp, label %for.body, label %for.cond.cleanup, !dbg !19
+
+for.cond.cleanup:                                 ; preds = %for.cond
+  br label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %idxprom = sext i32 %ArgIndex.0 to i64, !dbg !19
+  %arrayidx = getelementptr inbounds i8*, i8** %argv, i64 %idxprom, !dbg !19
+  %0 = load i8*, i8** %arrayidx, align 8, !dbg !19
+  %call = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([20 x i8], [20 x i8]* @.str, i64 0, i64 0), i32 %ArgIndex.0, i8* %0), !dbg !19
+  br label %for.inc, !dbg !19
+
+for.inc:                                          ; preds = %for.body
+  %add = add nsw i32 %ArgIndex.0, 1, !dbg !19
+  call void @llvm.dbg.value(metadata i32 %add, metadata !17, metadata !DIExpression()), !dbg !19
+  br label %for.cond, !dbg !19, !llvm.loop !19
+
+for.end:                                          ; preds = %for.cond.cleanup
+  ret i32 0, !dbg !19
+}
+
+declare dso_local i32 @printf(i8*, ...)
+
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "test.cpp", directory: "")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{!"clang"}
+!7 = distinct !DISubprogram(name: "main", scope: !1, file: !1, line: 4, type: !8, scopeLine: 5, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !14)
+!8 = !DISubroutineType(types: !9)
+!9 = !{!10, !10, !11}
+!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!11 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !12, size: 64)
+!12 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !13, size: 64)
+!13 = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_signed_char)
+!14 = !{!15, !16, !17}
+!15 = !DILocalVariable(name: "argc", arg: 1, scope: !7, file: !1, line: 4, type: !10)
+!16 = !DILocalVariable(name: "argv", arg: 2, scope: !7, file: !1, line: 4, type: !11)
+!17 = !DILocalVariable(name: "ArgIndex", scope: !18, file: !1, line: 6, type: !10)
+!18 = distinct !DILexicalBlock(scope: !7, file: !1, line: 6)
+!19 = !DILocation(line: 0, scope: !7)
diff --git a/llvm/test/Transforms/IndVarSimplify/lftr-dead-ivs.ll b/llvm/test/Transforms/IndVarSimplify/lftr-dead-ivs.ll
index 5ef6b4a6e..0ec469fea 100644
--- a/llvm/test/Transforms/IndVarSimplify/lftr-dead-ivs.ll
+++ b/llvm/test/Transforms/IndVarSimplify/lftr-dead-ivs.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S -indvars < %s | FileCheck %s
+; RUN: opt -S -indvars -indvars-predicate-loops=0 < %s | FileCheck %s
 
 ; Tests in this file are specifically about correctly handling possibly poison
 ; producing flags when converting from one IV to another.  In particular, there
diff --git a/llvm/test/Transforms/IndVarSimplify/lftr-pr20680.ll b/llvm/test/Transforms/IndVarSimplify/lftr-pr20680.ll
index 096501e86..1bb13832c 100644
--- a/llvm/test/Transforms/IndVarSimplify/lftr-pr20680.ll
+++ b/llvm/test/Transforms/IndVarSimplify/lftr-pr20680.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -indvars -S | FileCheck %s
+; RUN: opt < %s -indvars -S -indvars-predicate-loops=0 | FileCheck %s
 
 ; Provide legal integer types.
 target datalayout = "n8:16:32:64"
@@ -62,7 +62,6 @@ define void @f() {
 ; CHECK:       cond.false.us:
 ; CHECK-NEXT:    br label [[COND_END_US]]
 ; CHECK:       cond.end.us:
-; CHECK-NEXT:    [[COND_US:%.*]] = phi i32 [ [[DIV]], [[COND_FALSE_US]] ], [ [[INDVARS_IV]], [[FOR_BODY3_US]] ]
 ; CHECK-NEXT:    [[TMP6:%.*]] = load i32, i32* @b, align 4
 ; CHECK-NEXT:    [[CMP91_US:%.*]] = icmp slt i32 [[TMP6]], 1
 ; CHECK-NEXT:    br i1 [[CMP91_US]], label [[FOR_INC_LR_PH_US:%.*]], label [[FOR_COND2_LOOPEXIT_US:%.*]]
@@ -79,10 +78,9 @@ define void @f() {
 ; CHECK-NEXT:    store i32 1, i32* @b, align 4
 ; CHECK-NEXT:    br label [[FOR_COND2_LOOPEXIT_US]]
 ; CHECK:       for.cond2.for.inc13_crit_edge.us-lcssa.us.us-lcssa:
-; CHECK-NEXT:    [[COND_LCSSA_PH_US_PH:%.*]] = phi i32 [ [[COND_US]], [[FOR_COND2_LOOPEXIT_US]] ]
 ; CHECK-NEXT:    br label [[FOR_COND2_FOR_INC13_CRIT_EDGE_US_LCSSA_US]]
 ; CHECK:       for.cond2.for.inc13_crit_edge.us-lcssa.us:
-; CHECK-NEXT:    [[COND_LCSSA_PH_US:%.*]] = phi i32 [ [[COND_LCSSA_PH_US_PH]], [[FOR_COND2_FOR_INC13_CRIT_EDGE_US_LCSSA_US_US_LCSSA]] ], [ [[DIV]], [[FOR_COND2_FOR_INC13_CRIT_EDGE_US_LCSSA_US_US_LCSSA_US]] ]
+; CHECK-NEXT:    [[COND_LCSSA_PH_US:%.*]] = phi i32 [ [[DIV]], [[FOR_COND2_FOR_INC13_CRIT_EDGE_US_LCSSA_US_US_LCSSA]] ], [ [[DIV]], [[FOR_COND2_FOR_INC13_CRIT_EDGE_US_LCSSA_US_US_LCSSA_US]] ]
 ; CHECK-NEXT:    br label [[FOR_COND2_FOR_INC13_CRIT_EDGE:%.*]]
 ; CHECK:       for.body3.lr.ph.split:
 ; CHECK-NEXT:    br i1 [[TOBOOL]], label [[FOR_BODY3_LR_PH_SPLIT_SPLIT_US:%.*]], label [[FOR_BODY3_LR_PH_SPLIT_FOR_BODY3_LR_PH_SPLIT_SPLIT_CRIT_EDGE:%.*]]
diff --git a/llvm/test/Transforms/IndVarSimplify/lftr-reuse.ll b/llvm/test/Transforms/IndVarSimplify/lftr-reuse.ll
index 14ae97386..2053d70fb 100644
--- a/llvm/test/Transforms/IndVarSimplify/lftr-reuse.ll
+++ b/llvm/test/Transforms/IndVarSimplify/lftr-reuse.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -indvars -S | FileCheck %s
+; RUN: opt < %s -indvars -S -indvars-predicate-loops=0 | FileCheck %s
 
 ; Make sure that indvars can perform LFTR without a canonical IV.
 
diff --git a/llvm/test/Transforms/IndVarSimplify/lftr.ll b/llvm/test/Transforms/IndVarSimplify/lftr.ll
index b72778b2e..abeeb5b05 100644
--- a/llvm/test/Transforms/IndVarSimplify/lftr.ll
+++ b/llvm/test/Transforms/IndVarSimplify/lftr.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -indvars -dce -S | FileCheck %s
+; RUN: opt < %s -indvars -dce -S -indvars-predicate-loops=0 | FileCheck %s
 
 ; Provide legal integer types.
 target datalayout = "n8:16:32:64"
@@ -153,6 +153,34 @@ loopexit:
   ret i32 %i
 }
 
+define i32 @quadratic_sgt_loopdec() {
+; CHECK-LABEL: @quadratic_sgt_loopdec(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 10, [[ENTRY:%.*]] ], [ [[I_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[I_NEXT]] = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 [[I]], i32 1)
+; CHECK-NEXT:    store i32 [[I]], i32* @A
+; CHECK-NEXT:    [[I2:%.*]] = mul i32 [[I]], [[I]]
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp sgt i32 [[I2]], 0
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOPEXIT:%.*]]
+; CHECK:       loopexit:
+; CHECK-NEXT:    ret i32 0
+
+entry:
+  br label %loop
+
+loop:
+  %i = phi i32 [ 10, %entry ], [ %i.next, %loop ]
+  %i.next = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %i, i32 1)
+  store i32 %i, i32* @A
+  %i2 = mul i32 %i, %i
+  %c = icmp sgt i32 %i2, 0
+  br i1 %c, label %loop, label %loopexit
+
+loopexit:
+  ret i32 %i
+}
 
 @data = common global [240 x i8] zeroinitializer, align 16
 
@@ -629,4 +657,5 @@ exit:
 }
 
 
+declare i32 @llvm.loop.decrement.reg.i32.i32.i32(i32, i32)
 
diff --git a/llvm/test/Transforms/IndVarSimplify/loop-invariant-conditions.ll b/llvm/test/Transforms/IndVarSimplify/loop-invariant-conditions.ll
index 320f0b30c..f5b80d368 100644
--- a/llvm/test/Transforms/IndVarSimplify/loop-invariant-conditions.ll
+++ b/llvm/test/Transforms/IndVarSimplify/loop-invariant-conditions.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -indvars %s | FileCheck %s
+; RUN: opt -S -indvars -indvars-predicate-loops=0 %s | FileCheck %s
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
diff --git a/llvm/test/Transforms/IndVarSimplify/loop-predication.ll b/llvm/test/Transforms/IndVarSimplify/loop-predication.ll
new file mode 100644
index 000000000..77c18ef23
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/loop-predication.ll
@@ -0,0 +1,858 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -indvars -indvars-predicate-loops=1 -S | FileCheck %s
+
+declare void @prevent_merging()
+
+; Base case
+define i32 @test1(i32* %array, i32 %length, i32 %n) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ugt i32 [[N:%.*]], 1
+; CHECK-NEXT:    [[UMAX:%.*]] = select i1 [[TMP0]], i32 [[N]], i32 1
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[UMAX]], -1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i32 [[LENGTH:%.*]], [[TMP1]]
+; CHECK-NEXT:    [[UMIN:%.*]] = select i1 [[TMP2]], i32 [[LENGTH]], i32 [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne i32 [[LENGTH]], [[UMIN]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER:%.*]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    br i1 [[TMP3]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void @prevent_merging()
+; CHECK-NEXT:    ret i32 -1
+; CHECK:       guarded:
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ult i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  %within.bounds = icmp ult i32 %i, %length
+  br i1 %within.bounds, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  call void @prevent_merging()
+  ret i32 -1
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp ult i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:                                             ; preds = %guarded, %entry
+  %result = phi i32 [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+; Has side effect which must be reflected
+define i32 @neg_store(i32* %array, i32 %length, i32 %n) {
+; CHECK-LABEL: @neg_store(
+; CHECK-NEXT:  loop.preheader:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER:%.*]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[I]], [[LENGTH:%.*]]
+; CHECK-NEXT:    br i1 [[WITHIN_BOUNDS]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void @prevent_merging()
+; CHECK-NEXT:    ret i32 -1
+; CHECK:       guarded:
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    store i32 0, i32* [[ARRAY_I_PTR]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ult i32 [[I_NEXT]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  %within.bounds = icmp ult i32 %i, %length
+  br i1 %within.bounds, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  call void @prevent_merging()
+  ret i32 -1
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  store i32 0, i32* %array.i.ptr
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp ult i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:                                             ; preds = %guarded, %entry
+  %result = phi i32 [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+declare void @maythrow()
+
+; May exit through implicit exception edge
+define i32 @neg_implicit_exit(i32* %array, i32 %length, i32 %n) {
+; CHECK-LABEL: @neg_implicit_exit(
+; CHECK-NEXT:  loop.preheader:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER:%.*]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    call void @maythrow()
+; CHECK-NEXT:    [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[I]], [[LENGTH:%.*]]
+; CHECK-NEXT:    br i1 [[WITHIN_BOUNDS]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void @prevent_merging()
+; CHECK-NEXT:    ret i32 -1
+; CHECK:       guarded:
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ult i32 [[I_NEXT]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  call void @maythrow()
+  %within.bounds = icmp ult i32 %i, %length
+  br i1 %within.bounds, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  call void @prevent_merging()
+  ret i32 -1
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp ult i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:                                             ; preds = %guarded, %entry
+  %result = phi i32 [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+
+
+; Base case, but in LFTR form (just for sanity checking)
+define i32 @test2(i32* %array, i32 %length, i32 %n) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:  loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[N:%.*]], -1
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[LENGTH:%.*]], [[TMP0]]
+; CHECK-NEXT:    [[UMIN:%.*]] = select i1 [[TMP1]], i32 [[LENGTH]], i32 [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i32 [[LENGTH]], [[UMIN]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER:%.*]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    br i1 [[TMP2]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void @prevent_merging()
+; CHECK-NEXT:    ret i32 -1
+; CHECK:       guarded:
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ne i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  %within.bounds = icmp ne i32 %i, %length
+  br i1 %within.bounds, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  call void @prevent_merging()
+  ret i32 -1
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp ne i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:                                             ; preds = %guarded, %entry
+  %result = phi i32 [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+; br (and rcheck1, rcheck2)
+define i32 @two_range_checks(i32* %array.1, i32 %length.1, i32* %array.2, i32 %length.2, i32 %n) {
+; CHECK-LABEL: @two_range_checks(
+; CHECK-NEXT:  loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[LENGTH_2:%.*]], [[LENGTH_1:%.*]]
+; CHECK-NEXT:    [[UMIN:%.*]] = select i1 [[TMP0]], i32 [[LENGTH_2]], i32 [[LENGTH_1]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[LENGTH_2]], [[LENGTH_1]]
+; CHECK-NEXT:    [[UMIN1:%.*]] = select i1 [[TMP1]], i32 [[LENGTH_2]], i32 [[LENGTH_1]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ugt i32 [[N:%.*]], 1
+; CHECK-NEXT:    [[UMAX:%.*]] = select i1 [[TMP2]], i32 [[N]], i32 1
+; CHECK-NEXT:    [[TMP3:%.*]] = add i32 [[UMAX]], -1
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ult i32 [[UMIN1]], [[TMP3]]
+; CHECK-NEXT:    [[UMIN2:%.*]] = select i1 [[TMP4]], i32 [[UMIN1]], i32 [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i32 [[UMIN]], [[UMIN2]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER:%.*]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    br i1 [[TMP5]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void @prevent_merging()
+; CHECK-NEXT:    ret i32 -1
+; CHECK:       guarded:
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_1_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY_1:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_1_I:%.*]] = load i32, i32* [[ARRAY_1_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_1:%.*]] = add i32 [[LOOP_ACC]], [[ARRAY_1_I]]
+; CHECK-NEXT:    [[ARRAY_2_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY_2:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_2_I:%.*]] = load i32, i32* [[ARRAY_2_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC_1]], [[ARRAY_2_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ult i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  %within.bounds.1 = icmp ult i32 %i, %length.1
+  %within.bounds.2 = icmp ult i32 %i, %length.2
+  %within.bounds = and i1 %within.bounds.1, %within.bounds.2
+  br i1 %within.bounds, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  call void @prevent_merging()
+  ret i32 -1
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.1.i.ptr = getelementptr inbounds i32, i32* %array.1, i64 %i.i64
+  %array.1.i = load i32, i32* %array.1.i.ptr, align 4
+  %loop.acc.1 = add i32 %loop.acc, %array.1.i
+  %array.2.i.ptr = getelementptr inbounds i32, i32* %array.2, i64 %i.i64
+  %array.2.i = load i32, i32* %array.2.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc.1, %array.2.i
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp ult i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:                                             ; preds = %guarded, %entry
+  %result = phi i32 [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+define i32 @three_range_checks(i32* %array.1, i32 %length.1, i32* %array.2, i32 %length.2, i32* %array.3, i32 %length.3, i32 %n) {
+; CHECK-LABEL: @three_range_checks(
+; CHECK-NEXT:  loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[LENGTH_3:%.*]], [[LENGTH_2:%.*]]
+; CHECK-NEXT:    [[UMIN:%.*]] = select i1 [[TMP0]], i32 [[LENGTH_3]], i32 [[LENGTH_2]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[UMIN]], [[LENGTH_1:%.*]]
+; CHECK-NEXT:    [[UMIN1:%.*]] = select i1 [[TMP1]], i32 [[UMIN]], i32 [[LENGTH_1]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i32 [[LENGTH_3]], [[LENGTH_2]]
+; CHECK-NEXT:    [[UMIN2:%.*]] = select i1 [[TMP2]], i32 [[LENGTH_3]], i32 [[LENGTH_2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ult i32 [[UMIN2]], [[LENGTH_1]]
+; CHECK-NEXT:    [[UMIN3:%.*]] = select i1 [[TMP3]], i32 [[UMIN2]], i32 [[LENGTH_1]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ugt i32 [[N:%.*]], 1
+; CHECK-NEXT:    [[UMAX:%.*]] = select i1 [[TMP4]], i32 [[N]], i32 1
+; CHECK-NEXT:    [[TMP5:%.*]] = add i32 [[UMAX]], -1
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ult i32 [[UMIN3]], [[TMP5]]
+; CHECK-NEXT:    [[UMIN4:%.*]] = select i1 [[TMP6]], i32 [[UMIN3]], i32 [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp ne i32 [[UMIN1]], [[UMIN4]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER:%.*]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    br i1 [[TMP7]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void @prevent_merging()
+; CHECK-NEXT:    ret i32 -1
+; CHECK:       guarded:
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_1_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY_1:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_1_I:%.*]] = load i32, i32* [[ARRAY_1_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_1:%.*]] = add i32 [[LOOP_ACC]], [[ARRAY_1_I]]
+; CHECK-NEXT:    [[ARRAY_2_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY_2:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_2_I:%.*]] = load i32, i32* [[ARRAY_2_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_2:%.*]] = add i32 [[LOOP_ACC_1]], [[ARRAY_2_I]]
+; CHECK-NEXT:    [[ARRAY_3_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY_3:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_3_I:%.*]] = load i32, i32* [[ARRAY_3_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC_2]], [[ARRAY_3_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ult i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  %within.bounds.1 = icmp ult i32 %i, %length.1
+  %within.bounds.2 = icmp ult i32 %i, %length.2
+  %within.bounds.3 = icmp ult i32 %i, %length.3
+  %within.bounds.1.and.2 = and i1 %within.bounds.1, %within.bounds.2
+  %within.bounds = and i1 %within.bounds.1.and.2, %within.bounds.3
+  br i1 %within.bounds, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  call void @prevent_merging()
+  ret i32 -1
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.1.i.ptr = getelementptr inbounds i32, i32* %array.1, i64 %i.i64
+  %array.1.i = load i32, i32* %array.1.i.ptr, align 4
+  %loop.acc.1 = add i32 %loop.acc, %array.1.i
+  %array.2.i.ptr = getelementptr inbounds i32, i32* %array.2, i64 %i.i64
+  %array.2.i = load i32, i32* %array.2.i.ptr, align 4
+  %loop.acc.2 = add i32 %loop.acc.1, %array.2.i
+  %array.3.i.ptr = getelementptr inbounds i32, i32* %array.3, i64 %i.i64
+  %array.3.i = load i32, i32* %array.3.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc.2, %array.3.i
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp ult i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:                                             ; preds = %guarded, %entry
+  %result = phi i32 [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+; Analogous to the above, but with two distinct branches (on different conditions)
+define i32 @distinct_checks(i32* %array.1, i32 %length.1, i32* %array.2, i32 %length.2, i32* %array.3, i32 %length.3, i32 %n) {
+; CHECK-LABEL: @distinct_checks(
+; CHECK-NEXT:  loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[LENGTH_2:%.*]], [[LENGTH_1:%.*]]
+; CHECK-NEXT:    [[UMIN:%.*]] = select i1 [[TMP0]], i32 [[LENGTH_2]], i32 [[LENGTH_1]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i32 [[N:%.*]], 1
+; CHECK-NEXT:    [[UMAX:%.*]] = select i1 [[TMP1]], i32 [[N]], i32 1
+; CHECK-NEXT:    [[TMP2:%.*]] = add i32 [[UMAX]], -1
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ult i32 [[UMIN]], [[TMP2]]
+; CHECK-NEXT:    [[UMIN1:%.*]] = select i1 [[TMP3]], i32 [[UMIN]], i32 [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne i32 [[LENGTH_1]], [[UMIN1]]
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i32 [[LENGTH_2]], [[UMIN1]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED1:%.*]] ], [ 0, [[LOOP_PREHEADER:%.*]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED1]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    br i1 [[TMP4]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void @prevent_merging()
+; CHECK-NEXT:    ret i32 -1
+; CHECK:       guarded:
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_1_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY_1:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_1_I:%.*]] = load i32, i32* [[ARRAY_1_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_1:%.*]] = add i32 [[LOOP_ACC]], [[ARRAY_1_I]]
+; CHECK-NEXT:    br i1 [[TMP5]], label [[GUARDED1]], label [[DEOPT2:%.*]], !prof !0
+; CHECK:       deopt2:
+; CHECK-NEXT:    call void @prevent_merging()
+; CHECK-NEXT:    ret i32 -1
+; CHECK:       guarded1:
+; CHECK-NEXT:    [[ARRAY_3_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY_3:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_3_I:%.*]] = load i32, i32* [[ARRAY_3_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC_1]], [[ARRAY_3_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ult i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED1]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded4, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded1 ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded1 ], [ 0, %loop.preheader ]
+  %within.bounds.1 = icmp ult i32 %i, %length.1
+  br i1 %within.bounds.1, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  call void @prevent_merging()
+  ret i32 -1
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.1.i.ptr = getelementptr inbounds i32, i32* %array.1, i64 %i.i64
+  %array.1.i = load i32, i32* %array.1.i.ptr, align 4
+  %loop.acc.1 = add i32 %loop.acc, %array.1.i
+  %within.bounds.2 = icmp ult i32 %i, %length.2
+  br i1 %within.bounds.2, label %guarded1, label %deopt2, !prof !0
+
+deopt2:                                           ; preds = %guarded
+  call void @prevent_merging()
+  ret i32 -1
+
+guarded1:                                         ; preds = %guarded1
+  %array.3.i.ptr = getelementptr inbounds i32, i32* %array.3, i64 %i.i64
+  %array.3.i = load i32, i32* %array.3.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc.1, %array.3.i
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp ult i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:
+  %result = phi i32 [ %loop.acc.next, %guarded1 ]
+  ret i32 %result
+}
+
+define i32 @duplicate_checks(i32* %array.1, i32* %array.2, i32* %array.3, i32 %length, i32 %n) {
+; CHECK-LABEL: @duplicate_checks(
+; CHECK-NEXT:  loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ugt i32 [[N:%.*]], 1
+; CHECK-NEXT:    [[UMAX:%.*]] = select i1 [[TMP0]], i32 [[N]], i32 1
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[UMAX]], -1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i32 [[LENGTH:%.*]], [[TMP1]]
+; CHECK-NEXT:    [[UMIN:%.*]] = select i1 [[TMP2]], i32 [[LENGTH]], i32 [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne i32 [[LENGTH]], [[UMIN]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED1:%.*]] ], [ 0, [[LOOP_PREHEADER:%.*]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED1]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    br i1 [[TMP3]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void @prevent_merging()
+; CHECK-NEXT:    ret i32 -1
+; CHECK:       guarded:
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_1_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY_1:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_1_I:%.*]] = load i32, i32* [[ARRAY_1_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_1:%.*]] = add i32 [[LOOP_ACC]], [[ARRAY_1_I]]
+; CHECK-NEXT:    br i1 true, label [[GUARDED1]], label [[DEOPT2:%.*]], !prof !0
+; CHECK:       deopt2:
+; CHECK-NEXT:    call void @prevent_merging()
+; CHECK-NEXT:    ret i32 -1
+; CHECK:       guarded1:
+; CHECK-NEXT:    [[ARRAY_3_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY_3:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_3_I:%.*]] = load i32, i32* [[ARRAY_3_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC_1]], [[ARRAY_3_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ult i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED1]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded4, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded1 ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded1 ], [ 0, %loop.preheader ]
+  %within.bounds.1 = icmp ult i32 %i, %length
+  br i1 %within.bounds.1, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  call void @prevent_merging()
+  ret i32 -1
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.1.i.ptr = getelementptr inbounds i32, i32* %array.1, i64 %i.i64
+  %array.1.i = load i32, i32* %array.1.i.ptr, align 4
+  %loop.acc.1 = add i32 %loop.acc, %array.1.i
+  %within.bounds.2 = icmp ult i32 %i, %length
+  br i1 %within.bounds.2, label %guarded1, label %deopt2, !prof !0
+
+deopt2:                                           ; preds = %guarded
+  call void @prevent_merging()
+  ret i32 -1
+
+guarded1:                                         ; preds = %guarded1
+  %array.3.i.ptr = getelementptr inbounds i32, i32* %array.3, i64 %i.i64
+  %array.3.i = load i32, i32* %array.3.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc.1, %array.3.i
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp ult i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:
+  %result = phi i32 [ %loop.acc.next, %guarded1 ]
+  ret i32 %result
+}
+
+
+define i32 @provably_taken(i32* %array, i32* %length.ptr) {
+; CHECK-LABEL: @provably_taken(
+; CHECK-NEXT:  loop.preheader:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER:%.*]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    br i1 false, label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void @prevent_merging()
+; CHECK-NEXT:    ret i32 -1
+; CHECK:       guarded:
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    br i1 true, label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+loop.preheader:
+  %length = load i32, i32* %length.ptr, !range !2
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  %within.bounds = icmp ult i32 %i, %length
+  br i1 %within.bounds, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  call void @prevent_merging()
+  ret i32 -1
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp slt i32 %i.next, 200
+  br i1 %continue, label %loop, label %exit
+
+exit:                                             ; preds = %guarded
+  %result = phi i32 [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+; Non-latch exits can still be predicated
+define i32 @unconditional_latch(i32* %a, i32 %length) {
+; CHECK-LABEL: @unconditional_latch(
+; CHECK-NEXT:  loop.preheader:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    br i1 false, label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void @prevent_merging()
+; CHECK-NEXT:    ret i32 -1
+; CHECK:       guarded:
+; CHECK-NEXT:    br label [[LOOP]]
+;
+loop.preheader:
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %i = phi i32 [ %i.next, %guarded ], [ 400, %loop.preheader ]
+  %within.bounds = icmp ult i32 %i, %length
+  br i1 %within.bounds, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  call void @prevent_merging()
+  ret i32 -1
+
+guarded:                                          ; preds = %loop
+  %i.next = add i32 %i, 1
+  br label %loop
+}
+
+; Side effect in loop must run proper number of times
+define i32 @unconditional_latch_with_side_effect(i32* %a, i32 %length) {
+; CHECK-LABEL: @unconditional_latch_with_side_effect(
+; CHECK-NEXT:  loop.preheader:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED:%.*]] ], [ 400, [[LOOP_PREHEADER:%.*]] ]
+; CHECK-NEXT:    [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[I]], [[LENGTH:%.*]]
+; CHECK-NEXT:    br i1 [[WITHIN_BOUNDS]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void @prevent_merging()
+; CHECK-NEXT:    ret i32 -1
+; CHECK:       guarded:
+; CHECK-NEXT:    store volatile i32 0, i32* [[A:%.*]]
+; CHECK-NEXT:    [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT:    br label [[LOOP]]
+;
+loop.preheader:
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %i = phi i32 [ %i.next, %guarded ], [ 400, %loop.preheader ]
+  %within.bounds = icmp ult i32 %i, %length
+  br i1 %within.bounds, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  call void @prevent_merging()
+  ret i32 -1
+
+guarded:                                          ; preds = %loop
+  store volatile i32 0, i32* %a
+  %i.next = add i32 %i, 1
+  br label %loop
+}
+
+; Demonstrate that this approach works with IVs of different steps, and types
+; This version uses a manually lftred exit condition to work around an issue described
+; in detail on next test.
+define i32 @different_ivs(i32* %array, i32 %length, i32 %n) {
+; CHECK-LABEL: @different_ivs(
+; CHECK-NEXT:  loop.preheader:
+; CHECK-NEXT:    [[N64:%.*]] = zext i32 [[N:%.*]] to i64
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ugt i64 [[N64]], 1
+; CHECK-NEXT:    [[UMAX:%.*]] = select i1 [[TMP0]], i64 [[N64]], i64 1
+; CHECK-NEXT:    [[TMP1:%.*]] = add nsw i64 [[UMAX]], -1
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i32 [[LENGTH:%.*]] to i64
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[UMIN:%.*]] = select i1 [[TMP3]], i64 [[TMP1]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i32 [[LENGTH]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP4]], [[UMIN]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER:%.*]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    br i1 [[TMP5]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void @prevent_merging()
+; CHECK-NEXT:    ret i32 -1
+; CHECK:       guarded:
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i64 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ult i64 [[I_NEXT]], [[N64]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+loop.preheader:
+  %j.start = sub nuw nsw i32 %length, 1
+  %n64 = zext i32 %n to i64
+  br label %loop
+
+loop:
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i64 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  %j = phi i32 [ %j.next, %guarded ], [ %j.start, %loop.preheader ]
+  %within.bounds = icmp ne i32 %j, -1
+  br i1 %within.bounds, label %guarded, label %deopt, !prof !0
+
+deopt:
+  call void @prevent_merging()
+  ret i32 -1
+
+guarded:
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %i.next = add nuw i64 %i, 1
+  %j.next = sub nuw i32 %j, 1
+  %continue = icmp ult i64 %i.next, %n64
+  br i1 %continue, label %loop, label %exit
+
+exit:
+  %result = phi i32 [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+; TODO: We're failing to compute an exit count for the bounds check.
+; From some quick analysis, it looks like we don't handle -1 step
+; in howManyLessThans.  Should be a simple fix.
+define i32 @different_ivs2(i32* %array, i32 %length, i32 %n) {
+; CHECK-LABEL: @different_ivs2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[POS_LENGTH:%.*]] = icmp sgt i32 [[LENGTH:%.*]], 0
+; CHECK-NEXT:    br i1 [[POS_LENGTH]], label [[LOOP_PREHEADER:%.*]], label [[EXIT:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[J_START:%.*]] = sub nuw nsw i32 [[LENGTH]], 1
+; CHECK-NEXT:    [[N64:%.*]] = zext i32 [[N:%.*]] to i64
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[J:%.*]] = phi i32 [ [[J_NEXT:%.*]], [[GUARDED]] ], [ [[J_START]], [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[J]], [[LENGTH]]
+; CHECK-NEXT:    br i1 [[WITHIN_BOUNDS]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void @prevent_merging()
+; CHECK-NEXT:    ret i32 -1
+; CHECK:       guarded:
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i64 [[I]], 1
+; CHECK-NEXT:    [[J_NEXT]] = sub nuw i32 [[J]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ult i64 [[I_NEXT]], [[N64]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]]
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %pos_length = icmp sgt i32 %length, 0
+  br i1 %pos_length, label %loop.preheader, label %exit
+
+loop.preheader:
+  %j.start = sub nuw nsw i32 %length, 1
+  %n64 = zext i32 %n to i64
+  br label %loop
+
+loop:
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i64 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  %j = phi i32 [ %j.next, %guarded ], [ %j.start, %loop.preheader ]
+  %within.bounds = icmp ult i32 %j, %length
+  br i1 %within.bounds, label %guarded, label %deopt, !prof !0
+
+deopt:
+  call void @prevent_merging()
+  ret i32 -1
+
+guarded:
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %i.next = add nuw i64 %i, 1
+  %j.next = sub nuw i32 %j, 1
+  %continue = icmp ult i64 %i.next, %n64
+  br i1 %continue, label %loop, label %exit
+
+exit:
+  %result = phi i32 [ %loop.acc.next, %guarded ], [0, %entry]
+  ret i32 %result
+}
+
+; If we have a dominating exit (exit1) which can't be itself rewritten, we
+; can't rewrite a later exit (exit2).  Doing so would cause the loop to exit
+; from the exit2 when it should have exited from exit1.
+define i32 @neg_dominating_exit(i32* %array, i32 %length, i32 %length2, i32 %n) {
+; CHECK-LABEL: @neg_dominating_exit(
+; CHECK-NEXT:  loop.preheader:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED2:%.*]] ], [ 0, [[LOOP_PREHEADER:%.*]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED2]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[I]], [[LENGTH:%.*]]
+; CHECK-NEXT:    br i1 [[WITHIN_BOUNDS]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ [[LOOP_ACC]], [[LOOP]] ]
+; CHECK-NEXT:    call void @prevent_merging()
+; CHECK-NEXT:    ret i32 [[RESULT]]
+; CHECK:       guarded:
+; CHECK-NEXT:    [[WITHIN_BOUNDS2:%.*]] = icmp ult i32 [[I]], [[LENGTH2:%.*]]
+; CHECK-NEXT:    br i1 [[WITHIN_BOUNDS2]], label [[GUARDED2]], label [[DEOPT2:%.*]], !prof !0
+; CHECK:       deopt2:
+; CHECK-NEXT:    call void @prevent_merging()
+; CHECK-NEXT:    ret i32 -1
+; CHECK:       guarded2:
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ult i32 [[I_NEXT]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT2:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED2]] ]
+; CHECK-NEXT:    ret i32 [[RESULT2]]
+;
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded2 ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded2 ], [ 0, %loop.preheader ]
+  %within.bounds = icmp ult i32 %i, %length
+  br i1 %within.bounds, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  %result = phi i32 [ %loop.acc, %loop ]
+  call void @prevent_merging()
+  ret i32 %result
+
+guarded:                                          ; preds = %loop
+  %within.bounds2 = icmp ult i32 %i, %length2
+  br i1 %within.bounds2, label %guarded2, label %deopt2, !prof !0
+
+deopt2:                                            ; preds = %loop
+  call void @prevent_merging()
+  ret i32 -1
+
+guarded2:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp ult i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:                                             ; preds = %guarded, %entry
+  %result2 = phi i32 [ %loop.acc.next, %guarded2 ]
+  ret i32 %result2
+}
+
+
+declare i32 @llvm.experimental.deoptimize.i32(...)
+
+!0 = !{!"branch_weights", i32 1048576, i32 1}
+!1 = !{i32 1, i32 -2147483648}
+!2 = !{i32 0, i32 50}
diff --git a/llvm/test/Transforms/IndVarSimplify/loop_evaluate_1.ll b/llvm/test/Transforms/IndVarSimplify/loop_evaluate_1.ll
index 3e2e04513..6edacc175 100644
--- a/llvm/test/Transforms/IndVarSimplify/loop_evaluate_1.ll
+++ b/llvm/test/Transforms/IndVarSimplify/loop_evaluate_1.ll
@@ -1,34 +1,39 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -indvars -loop-deletion -simplifycfg -S | FileCheck %s
 
 ; Testcase distilled from 256.bzip2
-; CHECK-LABEL: @test1
-; CHECK-NOT: br
 define i32 @test1() {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret i32 364
+;
 entry:
-        br label %loopentry
+  br label %loopentry
 
 loopentry:              ; preds = %loopentry, %entry
-        %indvar1 = phi i32 [ 0, %entry ], [ %indvar.next2, %loopentry ]         ; <i32> [#uses=1]
-        %h.0 = phi i32 [ %tmp.2, %loopentry ], [ 4, %entry ]            ; <i32> [#uses=1]
-        %tmp.1 = mul i32 %h.0, 3                ; <i32> [#uses=1]
-        %tmp.2 = add i32 %tmp.1, 1              ; <i32> [#uses=2]
-        %indvar.next2 = add i32 %indvar1, 1             ; <i32> [#uses=2]
-        %exitcond3 = icmp ne i32 %indvar.next2, 4               ; <i1> [#uses=1]
-        br i1 %exitcond3, label %loopentry, label %loopexit
+  %indvar1 = phi i32 [ 0, %entry ], [ %indvar.next2, %loopentry ]         ; <i32> [#uses=1]
+  %h.0 = phi i32 [ %tmp.2, %loopentry ], [ 4, %entry ]            ; <i32> [#uses=1]
+  %tmp.1 = mul i32 %h.0, 3                ; <i32> [#uses=1]
+  %tmp.2 = add i32 %tmp.1, 1              ; <i32> [#uses=2]
+  %indvar.next2 = add i32 %indvar1, 1             ; <i32> [#uses=2]
+  %exitcond3 = icmp ne i32 %indvar.next2, 4               ; <i1> [#uses=1]
+  br i1 %exitcond3, label %loopentry, label %loopexit
 
 loopexit:               ; preds = %loopentry
-        ret i32 %tmp.2
+  ret i32 %tmp.2
 }
 
-
 ; PR12377
-; CHECK-LABEL: @test2
-; CHECK: [[VAR1:%.+]] = add i32 %arg, -11
-; CHECK: [[VAR2:%.+]] = lshr i32 [[VAR1]], 1
-; CHECK: [[VAR3:%.+]] = add nuw i32 [[VAR2]], 1
-; CHECK: [[VAR4:%.+]] = phi i32 [ 0, %bb ], [ [[VAR3]], %bb1.preheader ]
-; CHECK: ret i32 [[VAR4]]
 define i32 @test2(i32 %arg) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP:%.*]] = icmp ugt i32 [[ARG:%.*]], 10
+; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[ARG]], -11
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i32 [[TMP0]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = add nuw i32 [[TMP1]], 1
+; CHECK-NEXT:    [[TMP8:%.*]] = select i1 [[TMP]], i32 [[TMP2]], i32 0
+; CHECK-NEXT:    ret i32 [[TMP8]]
+;
 bb:
   %tmp = icmp ugt i32 %arg, 10
   br i1 %tmp, label %bb1, label %bb7
diff --git a/llvm/test/Transforms/IndVarSimplify/no-iv-rewrite.ll b/llvm/test/Transforms/IndVarSimplify/no-iv-rewrite.ll
index 7411b16bb..53c08ec5d 100644
--- a/llvm/test/Transforms/IndVarSimplify/no-iv-rewrite.ll
+++ b/llvm/test/Transforms/IndVarSimplify/no-iv-rewrite.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -indvars -S | FileCheck %s
+; RUN: opt < %s -indvars -S -indvars-predicate-loops=0  | FileCheck %s
 ;
 ; Make sure that indvars isn't inserting canonical IVs.
 ; This is kinda hard to do until linear function test replacement is removed.
diff --git a/llvm/test/Transforms/IndVarSimplify/pr24356.ll b/llvm/test/Transforms/IndVarSimplify/pr24356.ll
index eac4204c0..2718a6c24 100644
--- a/llvm/test/Transforms/IndVarSimplify/pr24356.ll
+++ b/llvm/test/Transforms/IndVarSimplify/pr24356.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -indvars < %s | FileCheck %s
+; RUN: opt -S -indvars -indvars-predicate-loops=0  < %s | FileCheck %s
 
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.10.0"
diff --git a/llvm/test/Transforms/IndVarSimplify/pr38674.ll b/llvm/test/Transforms/IndVarSimplify/pr38674.ll
index 78d59a372..bd4a23dc3 100644
--- a/llvm/test/Transforms/IndVarSimplify/pr38674.ll
+++ b/llvm/test/Transforms/IndVarSimplify/pr38674.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S -indvars < %s | FileCheck %s
+; RUN: opt -S -indvars -indvars-predicate-loops=0  < %s | FileCheck %s
 
 ; Check that we don't reuse %zext instead of %inc11 for LCSSA Phi node. Case
 ; with constants SCEV.
@@ -14,10 +14,9 @@ define i32 @test_01() {
 ; CHECK-NEXT:    [[ZEXT:%.*]] = zext i16 1 to i32
 ; CHECK-NEXT:    br label [[FOR_BODY6:%.*]]
 ; CHECK:       for.cond4:
-; CHECK-NEXT:    [[CMP5:%.*]] = icmp ult i32 [[INC:%.*]], 2
-; CHECK-NEXT:    br i1 [[CMP5]], label [[FOR_BODY6]], label [[FOR_END:%.*]]
+; CHECK-NEXT:    br i1 true, label [[FOR_BODY6]], label [[FOR_END:%.*]]
 ; CHECK:       for.body6:
-; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[FOR_COND4_PREHEADER]] ], [ [[INC]], [[FOR_COND4:%.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[FOR_COND4_PREHEADER]] ], [ [[INC:%.*]], [[FOR_COND4:%.*]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = icmp eq i32 [[IV]], [[ZEXT]]
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[IV]], 1
 ; CHECK-NEXT:    br i1 [[TMP0]], label [[RETURN_LOOPEXIT:%.*]], label [[FOR_COND4]]
@@ -92,15 +91,13 @@ define i32 @test_02(i32 %x) {
 ; CHECK:       for.end:
 ; CHECK-NEXT:    br i1 false, label [[FOR_COND4_PREHEADER]], label [[FOR_END9:%.*]]
 ; CHECK:       for.end9:
-; CHECK-NEXT:    [[INC11:%.*]] = add nuw nsw i32 0, [[X]]
 ; CHECK-NEXT:    br i1 false, label [[FOR_COND1_PREHEADER]], label [[RETURN_LOOPEXIT3:%.*]]
 ; CHECK:       return.loopexit:
 ; CHECK-NEXT:    unreachable
 ; CHECK:       return.loopexit3:
-; CHECK-NEXT:    [[INC11_LCSSA:%.*]] = phi i32 [ [[INC11]], [[FOR_END9]] ]
 ; CHECK-NEXT:    br label [[RETURN:%.*]]
 ; CHECK:       return:
-; CHECK-NEXT:    ret i32 [[INC11_LCSSA]]
+; CHECK-NEXT:    ret i32 [[X]]
 ;
 entry:
   br label %for.cond1.preheader
diff --git a/llvm/test/Transforms/IndVarSimplify/pr39673.ll b/llvm/test/Transforms/IndVarSimplify/pr39673.ll
index 4c1bfea0b..7fb90a900 100644
--- a/llvm/test/Transforms/IndVarSimplify/pr39673.ll
+++ b/llvm/test/Transforms/IndVarSimplify/pr39673.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S -indvars < %s | FileCheck %s
+; RUN: opt -S -indvars -indvars-predicate-loops=0  < %s | FileCheck %s
 
 define i16 @constant() {
 ; CHECK-LABEL: @constant(
diff --git a/llvm/test/Transforms/IndVarSimplify/replace-loop-exit-folds.ll b/llvm/test/Transforms/IndVarSimplify/replace-loop-exit-folds.ll
index 77bc2b6ee..9cc911cc5 100644
--- a/llvm/test/Transforms/IndVarSimplify/replace-loop-exit-folds.ll
+++ b/llvm/test/Transforms/IndVarSimplify/replace-loop-exit-folds.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -indvars -S < %s | FileCheck %s
+; RUN: opt -indvars -S -indvars-predicate-loops=0 < %s | FileCheck %s
 
 target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
 
diff --git a/llvm/test/Transforms/IndVarSimplify/rlev-add-me.ll b/llvm/test/Transforms/IndVarSimplify/rlev-add-me.ll
new file mode 100644
index 000000000..9520f86b3
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/rlev-add-me.ll
@@ -0,0 +1,228 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -indvars < %s | FileCheck %s
+target datalayout = "n8:16:32:64"
+@G = external global i32
+
+; Basic case where we know the value of an induction variable along one
+; exit edge, but not another.
+define i32 @test(i32 %n) {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[N:%.*]], 1
+; CHECK-NEXT:    br label [[HEADER:%.*]]
+; CHECK:       header:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
+; CHECK-NEXT:    [[V:%.*]] = load volatile i32, i32* @G
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i32 [[V]], 0
+; CHECK-NEXT:    br i1 [[CMP1]], label [[LATCH]], label [[EXIT1:%.*]]
+; CHECK:       latch:
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i32 [[IV_NEXT]], [[TMP0]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[HEADER]], label [[EXIT2:%.*]]
+; CHECK:       exit1:
+; CHECK-NEXT:    [[IV_LCSSA:%.*]] = phi i32 [ [[IV]], [[HEADER]] ]
+; CHECK-NEXT:    ret i32 [[IV_LCSSA]]
+; CHECK:       exit2:
+; CHECK-NEXT:    ret i32 [[N]]
+;
+entry:
+  br label %header
+header:
+  %iv = phi i32 [0, %entry], [%iv.next, %latch]
+  %v = load volatile i32, i32* @G
+  %cmp1 = icmp eq i32 %v, 0
+  br i1 %cmp1, label %latch, label %exit1
+
+latch:
+  %iv.next = add i32 %iv, 1
+  %cmp2 = icmp ult i32 %iv, %n
+  br i1 %cmp2, label %header, label %exit2
+exit1:
+  ret i32 %iv
+exit2:
+  ret i32 %iv
+}
+
+define i32 @test2(i32 %n) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[N:%.*]], 1
+; CHECK-NEXT:    br label [[HEADER:%.*]]
+; CHECK:       header:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
+; CHECK-NEXT:    [[V:%.*]] = load volatile i32, i32* @G
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i32 [[V]], 0
+; CHECK-NEXT:    br i1 [[CMP1]], label [[LATCH]], label [[EXIT1:%.*]]
+; CHECK:       latch:
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i32 [[IV_NEXT]], [[TMP0]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[HEADER]], label [[EXIT2:%.*]]
+; CHECK:       exit1:
+; CHECK-NEXT:    [[IV_LCSSA:%.*]] = phi i32 [ [[IV]], [[HEADER]] ]
+; CHECK-NEXT:    ret i32 [[IV_LCSSA]]
+; CHECK:       exit2:
+; CHECK-NEXT:    ret i32 [[TMP0]]
+;
+entry:
+  br label %header
+header:
+  %iv = phi i32 [0, %entry], [%iv.next, %latch]
+  %v = load volatile i32, i32* @G
+  %cmp1 = icmp eq i32 %v, 0
+  br i1 %cmp1, label %latch, label %exit1
+
+latch:
+  %iv.next = add i32 %iv, 1
+  %cmp2 = icmp ult i32 %iv, %n
+  br i1 %cmp2, label %header, label %exit2
+exit1:
+  ret i32 %iv
+exit2:
+  ret i32 %iv.next
+}
+
+define i32 @neg_wrong_loop(i32 %n) {
+; CHECK-LABEL: @neg_wrong_loop(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[WRONG_LOOP:%.*]]
+; CHECK:       wrong_loop:
+; CHECK-NEXT:    [[IV2:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV2_NEXT:%.*]], [[WRONG_LOOP]] ]
+; CHECK-NEXT:    [[IV2_NEXT]] = add i32 [[IV2]], 1
+; CHECK-NEXT:    [[UNKNOWN:%.*]] = load volatile i32, i32* @G
+; CHECK-NEXT:    [[CMP_UNK:%.*]] = icmp eq i32 [[UNKNOWN]], 0
+; CHECK-NEXT:    br i1 [[CMP_UNK]], label [[HEADER_PREHEADER:%.*]], label [[WRONG_LOOP]]
+; CHECK:       header.preheader:
+; CHECK-NEXT:    [[IV2_LCSSA:%.*]] = phi i32 [ [[IV2]], [[WRONG_LOOP]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[N:%.*]], 1
+; CHECK-NEXT:    br label [[HEADER:%.*]]
+; CHECK:       header:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[HEADER_PREHEADER]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
+; CHECK-NEXT:    [[V:%.*]] = load volatile i32, i32* @G
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i32 [[V]], 0
+; CHECK-NEXT:    br i1 [[CMP1]], label [[LATCH]], label [[EXIT1:%.*]]
+; CHECK:       latch:
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i32 [[IV_NEXT]], [[TMP0]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[HEADER]], label [[EXIT2:%.*]]
+; CHECK:       exit1:
+; CHECK-NEXT:    [[IV_LCSSA:%.*]] = phi i32 [ [[IV]], [[HEADER]] ]
+; CHECK-NEXT:    ret i32 [[IV_LCSSA]]
+; CHECK:       exit2:
+; CHECK-NEXT:    [[EXITVAL:%.*]] = phi i32 [ [[IV2_LCSSA]], [[LATCH]] ]
+; CHECK-NEXT:    ret i32 [[EXITVAL]]
+;
+entry:
+  br label %wrong_loop
+
+wrong_loop:
+  %iv2 = phi i32 [0, %entry], [%iv2.next, %wrong_loop]
+  %iv2.next = add i32 %iv2, 1
+  %unknown = load volatile i32, i32* @G
+  %cmp_unk = icmp eq i32 %unknown, 0
+  br i1 %cmp_unk, label %header.preheader, label %wrong_loop
+
+header.preheader:
+  %iv2.lcssa = phi i32 [%iv2, %wrong_loop]
+  br label %header
+
+header:
+  %iv = phi i32 [0, %header.preheader], [%iv.next, %latch]
+  %v = load volatile i32, i32* @G
+  %cmp1 = icmp eq i32 %v, 0
+  br i1 %cmp1, label %latch, label %exit1
+
+latch:
+  %iv.next = add i32 %iv, 1
+  %cmp2 = icmp ult i32 %iv, %n
+  br i1 %cmp2, label %header, label %exit2
+exit1:
+  ret i32 %iv
+exit2:
+  %exitval = phi i32 [%iv2.lcssa, %latch]
+  ret i32 %exitval
+}
+
+; TODO: Generalize the code to handle other SCEV expressions
+define i32 @test3(i32 %n) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[N:%.*]], 1
+; CHECK-NEXT:    br label [[HEADER:%.*]]
+; CHECK:       header:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
+; CHECK-NEXT:    [[EXPR:%.*]] = udiv i32 [[IV]], 5
+; CHECK-NEXT:    [[V:%.*]] = load volatile i32, i32* @G
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i32 [[V]], 0
+; CHECK-NEXT:    br i1 [[CMP1]], label [[LATCH]], label [[EXIT1:%.*]]
+; CHECK:       latch:
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i32 [[IV_NEXT]], [[TMP0]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[HEADER]], label [[EXIT2:%.*]]
+; CHECK:       exit1:
+; CHECK-NEXT:    [[EXPR_LCSSA:%.*]] = phi i32 [ [[EXPR]], [[HEADER]] ]
+; CHECK-NEXT:    ret i32 [[EXPR_LCSSA]]
+; CHECK:       exit2:
+; CHECK-NEXT:    [[EXPR_LCSSA1:%.*]] = phi i32 [ [[EXPR]], [[LATCH]] ]
+; CHECK-NEXT:    ret i32 [[EXPR_LCSSA1]]
+;
+entry:
+  br label %header
+header:
+  %iv = phi i32 [0, %entry], [%iv.next, %latch]
+  %expr = udiv i32 %iv, 5
+  %v = load volatile i32, i32* @G
+  %cmp1 = icmp eq i32 %v, 0
+  br i1 %cmp1, label %latch, label %exit1
+
+latch:
+  %iv.next = add i32 %iv, 1
+  %cmp2 = icmp ult i32 %iv, %n
+  br i1 %cmp2, label %header, label %exit2
+exit1:
+  ret i32 %expr
+exit2:
+  ret i32 %expr
+}
+
+
+; A slightly more real example where we're searching for either a) the first
+; non-zero element, or b) the end of a memory region.
+define i32 @bounded_find(i32 %n) {
+; CHECK-LABEL: @bounded_find(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[N:%.*]], 1
+; CHECK-NEXT:    br label [[HEADER:%.*]]
+; CHECK:       header:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
+; CHECK-NEXT:    [[ADDR:%.*]] = getelementptr i32, i32* @G, i32 [[IV]]
+; CHECK-NEXT:    [[V:%.*]] = load i32, i32* [[ADDR]]
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i32 [[V]], 0
+; CHECK-NEXT:    br i1 [[CMP1]], label [[LATCH]], label [[EXIT1:%.*]]
+; CHECK:       latch:
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i32 [[IV_NEXT]], [[TMP0]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[HEADER]], label [[EXIT2:%.*]]
+; CHECK:       exit1:
+; CHECK-NEXT:    [[IV_LCSSA:%.*]] = phi i32 [ [[IV]], [[HEADER]] ]
+; CHECK-NEXT:    ret i32 [[IV_LCSSA]]
+; CHECK:       exit2:
+; CHECK-NEXT:    ret i32 [[N]]
+;
+entry:
+  br label %header
+header:
+  %iv = phi i32 [0, %entry], [%iv.next, %latch]
+  %addr = getelementptr i32, i32* @G, i32 %iv
+  %v = load i32, i32* %addr
+  %cmp1 = icmp eq i32 %v, 0
+  br i1 %cmp1, label %latch, label %exit1
+
+latch:
+  %iv.next = add i32 %iv, 1
+  %cmp2 = icmp ult i32 %iv, %n
+  br i1 %cmp2, label %header, label %exit2
+exit1:
+  ret i32 %iv
+exit2:
+  ret i32 %iv
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/sink-from-preheader.ll b/llvm/test/Transforms/IndVarSimplify/sink-from-preheader.ll
index 29e8b6382..00067fc97 100644
--- a/llvm/test/Transforms/IndVarSimplify/sink-from-preheader.ll
+++ b/llvm/test/Transforms/IndVarSimplify/sink-from-preheader.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -indvars -S | FileCheck %s
+; RUN: opt < %s -indvars -indvars-predicate-loops=0 -S | FileCheck %s
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
 target triple = "i386-apple-darwin10.0"
 
diff --git a/llvm/test/Transforms/IndVarSimplify/widen-nsw.ll b/llvm/test/Transforms/IndVarSimplify/widen-nsw.ll
index 8dbbb51ee..9365d6816 100644
--- a/llvm/test/Transforms/IndVarSimplify/widen-nsw.ll
+++ b/llvm/test/Transforms/IndVarSimplify/widen-nsw.ll
@@ -26,4 +26,4 @@ for.end:                                          ; preds = %for.cond
   ret i32 %sum.0
 }
 
-attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf"="true" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/Transforms/InferAddressSpaces/AMDGPU/address-space-id-funcs.ll b/llvm/test/Transforms/InferAddressSpaces/AMDGPU/address-space-id-funcs.ll
new file mode 100644
index 000000000..63cabb97c
--- /dev/null
+++ b/llvm/test/Transforms/InferAddressSpaces/AMDGPU/address-space-id-funcs.ll
@@ -0,0 +1,55 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -infer-address-spaces -instsimplify %s | FileCheck %s
+
+define amdgpu_kernel void @is_local_true(i8 addrspace(3)* %lptr) {
+; CHECK-LABEL: @is_local_true(
+; CHECK-NEXT:    store i32 1, i32 addrspace(1)* undef
+; CHECK-NEXT:    ret void
+;
+  %cast = addrspacecast i8 addrspace(3)* %lptr to i8*
+  %is.shared = call i1 @llvm.amdgcn.is.shared(i8* %cast)
+  %ext = zext i1 %is.shared to i32
+  store i32 %ext, i32 addrspace(1)* undef
+  ret void
+}
+
+define amdgpu_kernel void @is_local_false(i8 addrspace(1)* %gptr) {
+; CHECK-LABEL: @is_local_false(
+; CHECK-NEXT:    store i32 0, i32 addrspace(1)* undef
+; CHECK-NEXT:    ret void
+;
+  %cast = addrspacecast i8 addrspace(1)* %gptr to i8*
+  %is.shared = call i1 @llvm.amdgcn.is.shared(i8* %cast)
+  %ext = zext i1 %is.shared to i32
+  store i32 %ext, i32 addrspace(1)* undef
+  ret void
+}
+
+define void @is_private_true(i8 addrspace(5)* %lptr) {
+; CHECK-LABEL: @is_private_true(
+; CHECK-NEXT:    store i32 1, i32 addrspace(1)* undef
+; CHECK-NEXT:    ret void
+;
+  %cast = addrspacecast i8 addrspace(5)* %lptr to i8*
+  %is.private = call i1 @llvm.amdgcn.is.private(i8* %cast)
+  %ext = zext i1 %is.private to i32
+  store i32 %ext, i32 addrspace(1)* undef
+  ret void
+}
+
+define void @is_private_false(i8 addrspace(1)* %gptr) {
+; CHECK-LABEL: @is_private_false(
+; CHECK-NEXT:    store i32 0, i32 addrspace(1)* undef
+; CHECK-NEXT:    ret void
+;
+  %cast = addrspacecast i8 addrspace(1)* %gptr to i8*
+  %is.private = call i1 @llvm.amdgcn.is.private(i8* %cast)
+  %ext = zext i1 %is.private to i32
+  store i32 %ext, i32 addrspace(1)* undef
+  ret void
+}
+
+declare i1 @llvm.amdgcn.is.shared(i8* nocapture) #0
+declare i1 @llvm.amdgcn.is.private(i8* nocapture) #0
+
+attributes #0 = { nounwind readnone speculatable }
diff --git a/llvm/test/Transforms/InferFunctionAttrs/annotate.ll b/llvm/test/Transforms/InferFunctionAttrs/annotate.ll
index cb4464764..8054802ff 100644
--- a/llvm/test/Transforms/InferFunctionAttrs/annotate.ll
+++ b/llvm/test/Transforms/InferFunctionAttrs/annotate.ll
@@ -241,8 +241,8 @@ declare i64 @atol(i8*)
 ; CHECK: declare i64 @atoll(i8* nocapture) [[G2]]
 declare i64 @atoll(i8*)
 
-; CHECK-DARWIN: declare i32 @bcmp(i8* nocapture, i8* nocapture, i64) [[G2]]
 ; CHECK-LINUX: declare i32 @bcmp(i8* nocapture, i8* nocapture, i64) [[G2]]
+; CHECK-DARWIN-NOT: declare i32 @bcmp(i8* nocapture, i8* nocapture, i64) [[G2]]
 ; CHECK-UNKNOWN-NOT: declare i32 @bcmp(i8* nocapture, i8* nocapture, i64) [[G2]]
 ; CHECK-NVPTX-NOT: declare i32 @bcmp(i8* nocapture, i8* nocapture, i64) [[G2]]
 declare i32 @bcmp(i8*, i8*, i64)
@@ -619,7 +619,7 @@ declare i8* @malloc(i64)
 ; CHECK-LINUX: declare noalias i8* @memalign(i64, i64) [[G0]]
 declare i8* @memalign(i64, i64)
 
-; CHECK: declare i8* @memccpy(i8*, i8* nocapture readonly, i32, i64) [[G1]]
+; CHECK: declare i8* @memccpy(i8* noalias, i8* noalias nocapture readonly, i32, i64) [[G1]]
 declare i8* @memccpy(i8*, i8*, i32, i64)
 
 ; CHECK: declare i8* @memchr(i8*, i32, i64) [[G2]]
@@ -628,10 +628,10 @@ declare i8* @memchr(i8*, i32, i64)
 ; CHECK: declare i32 @memcmp(i8* nocapture, i8* nocapture, i64) [[G2]]
 declare i32 @memcmp(i8*, i8*, i64)
 
-; CHECK: declare i8* @memcpy(i8* returned, i8* nocapture readonly, i64) [[G1]]
+; CHECK: declare i8* @memcpy(i8* noalias returned, i8* noalias nocapture readonly, i64) [[G1]]
 declare i8* @memcpy(i8*, i8*, i64)
 
-; CHECK: declare i8* @mempcpy(i8*, i8* nocapture readonly, i64) [[G1]]
+; CHECK: declare i8* @mempcpy(i8* noalias, i8* noalias nocapture readonly, i64) [[G1]]
 declare i8* @mempcpy(i8*, i8*, i64)
 
 ; CHECK: declare i8* @memmove(i8* returned, i8* nocapture readonly, i64) [[G1]]
@@ -793,10 +793,10 @@ declare x86_fp80 @sinhl(x86_fp80)
 ; CHECK: declare x86_fp80 @sinl(x86_fp80) [[G0]]
 declare x86_fp80 @sinl(x86_fp80)
 
-; CHECK: declare i32 @snprintf(i8* nocapture, i64, i8* nocapture readonly, ...) [[G1]]
+; CHECK: declare i32 @snprintf(i8* noalias nocapture, i64, i8* nocapture readonly, ...) [[G1]]
 declare i32 @snprintf(i8*, i64, i8*, ...)
 
-; CHECK: declare i32 @sprintf(i8* nocapture, i8* nocapture readonly, ...) [[G1]]
+; CHECK: declare i32 @sprintf(i8* noalias nocapture, i8* nocapture readonly, ...) [[G1]]
 declare i32 @sprintf(i8*, i8*, ...)
 
 ; CHECK: declare double @sqrt(double) [[G0]]
@@ -844,7 +844,7 @@ declare i32 @strcmp(i8*, i8*)
 ; CHECK: declare i32 @strcoll(i8* nocapture, i8* nocapture) [[G2]]
 declare i32 @strcoll(i8*, i8*)
 
-; CHECK: declare i8* @strcpy(i8* returned, i8* nocapture readonly) [[G1]]
+; CHECK: declare i8* @strcpy(i8* noalias returned, i8* noalias nocapture readonly) [[G1]]
 declare i8* @strcpy(i8*, i8*)
 
 ; CHECK: declare i64 @strcspn(i8* nocapture, i8* nocapture) [[G2]]
@@ -865,7 +865,7 @@ declare i8* @strncat(i8*, i8*, i64)
 ; CHECK: declare i32 @strncmp(i8* nocapture, i8* nocapture, i64) [[G2]]
 declare i32 @strncmp(i8*, i8*, i64)
 
-; CHECK: declare i8* @strncpy(i8* returned, i8* nocapture readonly, i64) [[G1]]
+; CHECK: declare i8* @strncpy(i8* noalias returned, i8* noalias nocapture readonly, i64) [[G1]]
 declare i8* @strncpy(i8*, i8*, i64)
 
 ; CHECK: declare noalias i8* @strndup(i8* nocapture readonly, i64) [[G1]]
diff --git a/llvm/test/Transforms/InferFunctionAttrs/dereferenceable.ll b/llvm/test/Transforms/InferFunctionAttrs/dereferenceable.ll
index f6580826b..e707b88b5 100644
--- a/llvm/test/Transforms/InferFunctionAttrs/dereferenceable.ll
+++ b/llvm/test/Transforms/InferFunctionAttrs/dereferenceable.ll
@@ -1,10 +1,13 @@
 ; RUN: opt < %s -inferattrs -S | FileCheck %s
 
+
+
 ; Determine dereference-ability before unused loads get deleted:
 ; https://bugs.llvm.org/show_bug.cgi?id=21780
 
 define <4 x double> @PR21780(double* %ptr) {
 ; CHECK-LABEL: @PR21780(double* %ptr)
+
   ; GEP of index 0 is simplified away.
   %arrayidx1 = getelementptr inbounds double, double* %ptr, i64 1
   %arrayidx2 = getelementptr inbounds double, double* %ptr, i64 2
@@ -23,6 +26,37 @@ define <4 x double> @PR21780(double* %ptr) {
   ret <4 x double> %shuffle
 }
 
+
+define double @PR21780_only_access3_with_inbounds(double* %ptr) {
+; CHECK-LABEL: @PR21780_only_access3_with_inbounds(double* %ptr)
+
+  %arrayidx3 = getelementptr inbounds double, double* %ptr, i64 3
+  %t3 = load double, double* %arrayidx3, align 8
+  ret double %t3
+}
+
+define double @PR21780_only_access3_without_inbounds(double* %ptr) {
+; CHECK-LABEL: @PR21780_only_access3_without_inbounds(double* %ptr)
+  %arrayidx3 = getelementptr double, double* %ptr, i64 3
+  %t3 = load double, double* %arrayidx3, align 8
+  ret double %t3
+}
+
+define double @PR21780_without_inbounds(double* %ptr) {
+; CHECK-LABEL: @PR21780_without_inbounds(double* %ptr)
+
+  %arrayidx1 = getelementptr double, double* %ptr, i64 1
+  %arrayidx2 = getelementptr double, double* %ptr, i64 2
+  %arrayidx3 = getelementptr double, double* %ptr, i64 3
+
+  %t0 = load double, double* %ptr, align 8
+  %t1 = load double, double* %arrayidx1, align 8
+  %t2 = load double, double* %arrayidx2, align 8
+  %t3 = load double, double* %arrayidx3, align 8
+
+  ret double %t3
+}
+
 ; Unsimplified, but still valid. Also, throw in some bogus arguments.
 
 define void @gep0(i8* %unused, i8* %other, i8* %ptr) {
@@ -121,6 +155,19 @@ define void @volatile_is_not_dereferenceable(i16* %ptr) {
   ret void
 }
 
+; TODO: We should allow inference for atomic (but not volatile) ops.
+
+define void @atomic_is_alright(i16* %ptr) {
+; CHECK-LABEL: @atomic_is_alright(i16* %ptr)
+  %arrayidx0 = getelementptr i16, i16* %ptr, i64 0
+  %arrayidx1 = getelementptr i16, i16* %ptr, i64 1
+  %arrayidx2 = getelementptr i16, i16* %ptr, i64 2
+  %t0 = load atomic i16, i16* %arrayidx0 unordered, align 2
+  %t1 = load i16, i16* %arrayidx1
+  %t2 = load i16, i16* %arrayidx2
+  ret void
+}
+
 declare void @may_not_return()
 
 define void @not_guaranteed_to_transfer_execution(i16* %ptr) {
@@ -151,6 +198,7 @@ define void @variable_gep_index(i8* %unused, i8* %ptr, i64 %variable_index) {
 
 define void @multi_index_gep(<4 x i8>* %ptr) {
 ; CHECK-LABEL: @multi_index_gep(<4 x i8>* %ptr)
+; FIXME: %ptr should be dereferenceable(4)
   %arrayidx00 = getelementptr <4 x i8>, <4 x i8>* %ptr, i64 0, i64 0
   %t0 = load i8, i8* %arrayidx00
   ret void
@@ -204,6 +252,21 @@ define void @more_bytes(i32* dereferenceable(8) %ptr) {
   ret void
 }
 
+; Improve on existing dereferenceable_or_null attribute.
+
+define void @more_bytes_and_not_null(i32* dereferenceable_or_null(8) %ptr) {
+; CHECK-LABEL: @more_bytes_and_not_null(i32* dereferenceable_or_null(8) %ptr)
+  %arrayidx3 = getelementptr i32, i32* %ptr, i64 3
+  %arrayidx1 = getelementptr i32, i32* %ptr, i64 1
+  %arrayidx0 = getelementptr i32, i32* %ptr, i64 0
+  %arrayidx2 = getelementptr i32, i32* %ptr, i64 2
+  %t3 = load i32, i32* %arrayidx3
+  %t1 = load i32, i32* %arrayidx1
+  %t2 = load i32, i32* %arrayidx2
+  %t0 = load i32, i32* %arrayidx0
+  ret void
+}
+
 ; But don't pessimize existing dereferenceable attribute.
 
 define void @better_bytes(i32* dereferenceable(100) %ptr) {
@@ -276,3 +339,19 @@ define void @load_store(i32* %arg) {
   store float 2.0, float* %arrayidx1
   ret void
 }
+
+define void @different_size1(i32* %arg) {
+; CHECK-LABEL: @different_size1(i32* %arg)
+  %arg-cast = bitcast i32* %arg to double*
+  store double 0.000000e+00, double* %arg-cast
+  store i32 0, i32* %arg
+  ret void
+}
+
+define void @different_size2(i32* %arg) {
+; CHECK-LABEL: @different_size2(i32* %arg)
+  store i32 0, i32* %arg
+  %arg-cast = bitcast i32* %arg to double*
+  store double 0.000000e+00, double* %arg-cast
+  ret void
+}
diff --git a/llvm/test/Transforms/Inline/X86/switch.ll b/llvm/test/Transforms/Inline/X86/switch.ll
new file mode 100644
index 000000000..5565c7aba
--- /dev/null
+++ b/llvm/test/Transforms/Inline/X86/switch.ll
@@ -0,0 +1,160 @@
+; RUN: opt < %s -inline -inline-threshold=1 -S -mtriple=x86_64-unknown-linux-gnu  | FileCheck %s
+; RUN: opt < %s -passes='cgscc(inline)' -inline-threshold=1 -S -mtriple=x86_64-unknown-linux-gnu | FileCheck %s
+
+define i32 @callee_range(i32 %a, i32* %P) {
+  switch i32 %a, label %sw.default [
+    i32 0, label %sw.bb0
+    i32 1000, label %sw.bb1
+    i32 2000, label %sw.bb1
+    i32 3000, label %sw.bb1
+    i32 4000, label %sw.bb1
+    i32 5000, label %sw.bb1
+    i32 6000, label %sw.bb1
+    i32 7000, label %sw.bb1
+    i32 8000, label %sw.bb1
+    i32 9000, label %sw.bb1
+  ]
+
+sw.default:
+  store volatile i32 %a, i32* %P
+  br label %return
+sw.bb0:
+  store volatile i32 %a, i32* %P
+  br label %return
+sw.bb1:
+  store volatile i32 %a, i32* %P
+  br label %return
+return:
+  ret i32 42
+}
+
+define i32 @caller_range(i32 %a, i32* %P) {
+; CHECK-LABEL: @caller_range(
+; CHECK: call i32 @callee_range
+  %r = call i32 @callee_range(i32 %a, i32* %P)
+  ret i32 %r
+}
+
+define i32 @callee_bittest(i32 %a, i32* %P) {
+  switch i32 %a, label %sw.default [
+    i32 0, label %sw.bb0
+    i32 1, label %sw.bb1
+    i32 2, label %sw.bb2
+    i32 3, label %sw.bb0
+    i32 4, label %sw.bb1
+    i32 5, label %sw.bb2
+    i32 6, label %sw.bb0
+    i32 7, label %sw.bb1
+    i32 8, label %sw.bb2
+  ]
+
+sw.default:
+  store volatile i32 %a, i32* %P
+  br label %return
+
+sw.bb0:
+  store volatile i32 %a, i32* %P
+  br label %return
+
+sw.bb1:
+  store volatile i32 %a, i32* %P
+  br label %return
+
+sw.bb2:
+  br label %return
+
+return:
+  ret i32 42
+}
+
+
+define i32 @caller_bittest(i32 %a, i32* %P) {
+; CHECK-LABEL: @caller_bittest(
+; CHECK-NOT: call i32 @callee_bittest
+  %r= call i32 @callee_bittest(i32 %a, i32* %P)
+  ret i32 %r
+}
+
+define i32 @callee_jumptable(i32 %a, i32* %P) {
+  switch i32 %a, label %sw.default [
+    i32 1001, label %sw.bb101
+    i32 1002, label %sw.bb102
+    i32 1003, label %sw.bb103
+    i32 1004, label %sw.bb104
+    i32 1005, label %sw.bb101
+    i32 1006, label %sw.bb102
+    i32 1007, label %sw.bb103
+    i32 1008, label %sw.bb104
+    i32 1009, label %sw.bb101
+    i32 1010, label %sw.bb102
+    i32 1011, label %sw.bb103
+    i32 1012, label %sw.bb104
+ ]
+
+sw.default:
+  br label %return
+
+sw.bb101:
+  store volatile i32 %a, i32* %P
+  br label %return
+
+sw.bb102:
+  store volatile i32 %a, i32* %P
+  br label %return
+
+sw.bb103:
+  store volatile i32 %a, i32* %P
+  br label %return
+
+sw.bb104:
+  store volatile i32 %a, i32* %P
+  br label %return
+
+return:
+  ret i32 42
+}
+
+define i32 @caller_jumptable(i32 %a, i32 %b, i32* %P) {
+; CHECK-LABEL: @caller_jumptable(
+; CHECK: call i32 @callee_jumptable
+  %r = call i32 @callee_jumptable(i32 %b, i32* %P)
+  ret i32 %r
+}
+
+
+define internal i32 @callee_negativeCost(i32 %t)  {
+entry:
+  switch i32 %t, label %sw.default [
+    i32 1, label %sw.bb
+    i32 0, label %sw.bb1
+    i32 42, label %sw.bb2
+    i32 43, label %sw.bb3
+  ]
+
+sw.bb:                                            ; preds = %entry
+  br label %cleanup
+
+sw.bb1:                                           ; preds = %entry
+  br label %cleanup
+
+sw.bb2:                                           ; preds = %entry
+  br label %cleanup
+
+sw.bb3:                                           ; preds = %entry
+  br label %cleanup
+
+sw.default:                                       ; preds = %entry
+  br label %cleanup
+
+cleanup:                                          ; preds = %sw.default, %sw.bb3, %sw.bb2, %sw.bb1, %sw.bb
+  %retval.0 = phi i32 [ 1, %sw.default ], [ 3, %sw.bb3 ], [ 2, %sw.bb2 ], [ 0, %sw.bb1 ], [ 0, %sw.bb ]
+  ret i32 %retval.0
+}
+
+define i32 @caller_negativeCost(i32 %t) {
+; CHECK-LABEL: @caller_negativeCost(
+; CHECK-NOT: call i32 @callee_negativeCost
+entry:
+  %call = call i32 @callee_negativeCost(i32 %t)
+  ret i32 %call
+}
diff --git a/llvm/test/Transforms/Inline/alloca-dbgdeclare-merge.ll b/llvm/test/Transforms/Inline/alloca-dbgdeclare-merge.ll
index 261313422..e0e51b268 100644
--- a/llvm/test/Transforms/Inline/alloca-dbgdeclare-merge.ll
+++ b/llvm/test/Transforms/Inline/alloca-dbgdeclare-merge.ll
@@ -62,9 +62,9 @@ entry:
   ret void, !dbg !29
 }
 
-attributes #0 = { alwaysinline nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { alwaysinline nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
-attributes #2 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!9, !10}
diff --git a/llvm/test/Transforms/Inline/debug-info-duplicate-calls.ll b/llvm/test/Transforms/Inline/debug-info-duplicate-calls.ll
index 6e0dbc194..ae845e71f 100644
--- a/llvm/test/Transforms/Inline/debug-info-duplicate-calls.ll
+++ b/llvm/test/Transforms/Inline/debug-info-duplicate-calls.ll
@@ -91,9 +91,9 @@ entry:
 
 declare void @_Z2f1v() #2
 
-attributes #0 = { uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { alwaysinline inlinehint uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #2 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { alwaysinline inlinehint uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!10, !11}
diff --git a/llvm/test/Transforms/Inline/inline-indirect-chain.ll b/llvm/test/Transforms/Inline/inline-indirect-chain.ll
new file mode 100644
index 000000000..bf73ad35d
--- /dev/null
+++ b/llvm/test/Transforms/Inline/inline-indirect-chain.ll
@@ -0,0 +1,55 @@
+; RUN: opt -inline -early-cse < %s
+; This test used to crash (PR35469).
+
+define void @func1() {
+  %t = bitcast void ()* @func2 to void ()*
+  tail call void %t()
+  ret void
+}
+
+define void @func2() {
+  %t = bitcast void ()* @func3 to void ()*
+  tail call void %t()
+  ret void
+}
+
+define void @func3() {
+  %t = bitcast void ()* @func4 to void ()*
+  tail call void %t()
+  ret void
+}
+
+define void @func4() {
+  br i1 undef, label %left, label %right
+
+left:
+  %t = bitcast void ()* @func5 to void ()*
+  tail call void %t()
+  ret void
+
+right:
+  ret void
+}
+
+define void @func5() {
+  %t = bitcast void ()* @func6 to void ()*
+  tail call void %t()
+  ret void
+}
+
+define void @func6() {
+  %t = bitcast void ()* @func2 to void ()*
+  tail call void %t()
+  ret void
+}
+
+define void @func7() {
+  %t = bitcast void ()* @func3 to void ()*
+  tail call void @func8(void()* %t)
+  ret void
+}
+
+define void @func8(void()* %f) {
+  tail call void %f()
+  ret void
+}
diff --git a/llvm/test/Transforms/Inline/inline-skip-use-empty-alloca.ll b/llvm/test/Transforms/Inline/inline-skip-use-empty-alloca.ll
new file mode 100644
index 000000000..48545f10c
--- /dev/null
+++ b/llvm/test/Transforms/Inline/inline-skip-use-empty-alloca.ll
@@ -0,0 +1,71 @@
+; RUN: opt < %s -S -inline | FileCheck %s
+; RUN: opt < %s -S -strip-debug -inline | FileCheck %s
+
+; https://bugs.llvm.org/show_bug.cgi?id=43291
+; The purpose of this test is to check if there is use_empty in the inner loop when scanning
+; blocks of allocas, if the alloca is use_empty, skip it when allocas inlining.
+
+define void @foo(i16 %k) !dbg !6 {
+  call void @llvm.dbg.value(metadata i16 %k, metadata !14, metadata !DIExpression()), !dbg !15
+  %alloca_a = alloca i32
+  call void @llvm.dbg.value(metadata i16 %k, metadata !14, metadata !DIExpression()), !dbg !15
+  %alloca_b = alloca [3 x i32]
+  call void @llvm.dbg.value(metadata i16 %k, metadata !14, metadata !DIExpression()), !dbg !15
+  %alloca_c = alloca i32
+  call void @llvm.dbg.value(metadata i16 %k, metadata !14, metadata !DIExpression()), !dbg !15
+  %alloca_a..1 = bitcast i32* %alloca_a to i8**, !dbg !16
+  %alloca_c..1 = bitcast i32* %alloca_c to i8**, !dbg !16
+  %_tmp23 = icmp ne i16 %k, 0
+  br i1 %_tmp23, label %bb1, label %bb2
+
+bb1:                                              ; preds = %0
+  %_tmp28 = getelementptr [3 x i32], [3 x i32]* %alloca_b, i16 0, i64 0
+  store i32 0, i32* %_tmp28
+  br label %bb2
+
+bb2:                                              ; preds = %bb1, %0
+  ret void
+}
+
+define i16 @bar() !dbg !17 {
+  ; CHECK-LABEL: bar
+  ; CHECK: %alloca_a.i = alloca i32
+  ; CHECK-NOT: %alloca_b.i = alloca [3 x i32]
+  ; CHECK: %alloca_c.i = alloca i32
+  call void @foo(i16 0), !dbg !20
+  br label %bb1
+
+bb1:                                              ; preds = %0
+  ret i16 0
+}
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.value(metadata, metadata, metadata) #0
+
+attributes #0 = { nounwind readnone speculatable willreturn }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4}
+!llvm.ident = !{!5}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C, file: !1, producer: "Foo", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !2, globals: !2)
+!1 = !DIFile(filename: "foo2.c", directory: "/tmp")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{!"Foo"}
+!6 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 17, type: !7, scopeLine: 17, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !2)
+!7 = !DISubroutineType(types: !8)
+!8 = !{!9, !13}
+!9 = !DIDerivedType(tag: DW_TAG_typedef, name: "int64_t", file: !10, line: 76, baseType: !11)
+!10 = !DIFile(filename: "stdint.h", directory: "/tmp")
+!11 = !DIDerivedType(tag: DW_TAG_typedef, name: "__i64_t", file: !1, baseType: !12)
+!12 = !DIBasicType(name: "signed long long", size: 64, encoding: DW_ATE_signed)
+!13 = !DIBasicType(name: "int", size: 16, encoding: DW_ATE_signed)
+!14 = !DILocalVariable(name: "k", arg: 1, scope: !6, file: !1, line: 17, type: !13)
+!15 = !DILocation(line: 0, scope: !6)
+!16 = !DILocation(line: 19, column: 3, scope: !6)
+!17 = distinct !DISubprogram(name: "g", scope: !1, file: !1, line: 8, type: !18, scopeLine: 8, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !2)
+!18 = !DISubroutineType(types: !19)
+!19 = !{!13}
+!20 = !DILocation(line: 8, column: 7, scope: !17)
diff --git a/llvm/test/Transforms/Inline/inline-vla.ll b/llvm/test/Transforms/Inline/inline-vla.ll
index 88dfc2be8..fcdbd9353 100644
--- a/llvm/test/Transforms/Inline/inline-vla.ll
+++ b/llvm/test/Transforms/Inline/inline-vla.ll
@@ -30,8 +30,8 @@ entry:
 ; Function Attrs: nounwind
 declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture readonly, i64, i1) #2
 
-attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { inlinehint nounwind ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { inlinehint nounwind ssp uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind }
 
 !llvm.ident = !{!0}
diff --git a/llvm/test/Transforms/Inline/inline_invoke.ll b/llvm/test/Transforms/Inline/inline_invoke.ll
index 2b34140aa..838093613 100644
--- a/llvm/test/Transforms/Inline/inline_invoke.ll
+++ b/llvm/test/Transforms/Inline/inline_invoke.ll
@@ -345,5 +345,5 @@ terminate:
 ; CHECK: attributes [[NUW]] = { nounwind }
 ; CHECK: attributes #1 = { nounwind readnone }
 ; CHECK: attributes #2 = { ssp uwtable }
-; CHECK: attributes #3 = { argmemonly nounwind }
+; CHECK: attributes #3 = { argmemonly nounwind willreturn }
 ; CHECK: attributes #4 = { noreturn nounwind }
diff --git a/llvm/test/Transforms/Inline/inline_minisize.ll b/llvm/test/Transforms/Inline/inline_minisize.ll
index 0bf75d72b..e4bc9bcd7 100644
--- a/llvm/test/Transforms/Inline/inline_minisize.ll
+++ b/llvm/test/Transforms/Inline/inline_minisize.ll
@@ -1,4 +1,4 @@
-; RUN: opt -O2 -S < %s | FileCheck %s
+; RUN: opt -inline -inline-threshold=225 -inlinehint-threshold=360 -S < %s | FileCheck %s
 
 @data = common global i32* null, align 8
 
diff --git a/llvm/test/Transforms/Inline/no-inline-line-tables.ll b/llvm/test/Transforms/Inline/no-inline-line-tables.ll
new file mode 100644
index 000000000..b20f6b758
--- /dev/null
+++ b/llvm/test/Transforms/Inline/no-inline-line-tables.ll
@@ -0,0 +1,99 @@
+; RUN: opt < %s -inline -S | FileCheck %s
+
+; This tests that functions with the attribute `no-inline-line-tables` have the
+; correct debug information when they are inlined.
+
+; ModuleID = 't.c'
+source_filename = "t.c"
+target datalayout = "e-m:w-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-windows-msvc"
+
+; Function Attrs: alwaysinline nounwind
+define dso_local i32 @f(i32 %x) #0 !dbg !7 {
+entry:
+  %x.addr = alloca i32, align 4
+  store i32 %x, i32* %x.addr, align 4
+  call void @llvm.dbg.declare(metadata i32* %x.addr, metadata !12, metadata !DIExpression()), !dbg !13
+  %0 = load i32, i32* %x.addr, align 4, !dbg !14
+  ret i32 %0, !dbg !14
+}
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
+
+; Function Attrs: alwaysinline nounwind
+define i32 @g(i32 %x) #0 !dbg !15 {
+entry:
+  %x.addr = alloca i32, align 4
+  store i32 %x, i32* %x.addr, align 4
+  call void @llvm.dbg.declare(metadata i32* %x.addr, metadata !16, metadata !DIExpression()), !dbg !17
+  br label %L, !dbg !17
+
+L:                                                ; preds = %entry
+  call void @llvm.dbg.label(metadata !18), !dbg !19
+  store i32 42, i32* %x.addr, align 4, !dbg !20
+  %0 = load i32, i32* %x.addr, align 4, !dbg !21
+  ret i32 %0, !dbg !21
+}
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.label(metadata) #1
+
+; Check that debug info for inlined code uses the call location and that debug
+; intrinsics are removed.
+; Function Attrs: noinline nounwind optnone
+define i32 @main() #2 !dbg !22 {
+entry:
+; CHECK-LABEL: @main()
+; CHECK-NOT: @f
+; CHECK-NOT: @llvm.dbg.declare
+; CHECK: %{{[0-9]+}} = load i32, i32* %x.addr.i, align 4, !dbg ![[VAR1:[0-9]+]]
+  %call = call i32 @f(i32 3), !dbg !25
+
+; Another test for inlining debug intrinsics where the intrinsic appears at the
+; start of the basic block.
+; CHECK-NOT: @g
+; CHECK-NOT: @llvm.dbg.label
+; CHECK: %{{[0-9]+}} = load i32, i32* %x.addr.i1, align 4, !dbg ![[VAR2:[0-9]+]]
+  %call1 = call i32 @g(i32 340), !dbg !26
+  ret i32 0, !dbg !27
+}
+
+; CHECK: ![[VAR1]] = !DILocation(line: 10, scope: ![[SCOPE:[0-9]+]])
+; CHECK: ![[VAR2]] = !DILocation(line: 11, scope: ![[SCOPE]])
+
+attributes #0 = { alwaysinline nounwind "no-inline-line-tables" }
+attributes #2 = { noinline nounwind optnone "no-inline-line-tables"}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 10.0.0 (https://github.com/llvm/llvm-project.git 03ec3a12a94bbbaa11999b6da3a43221a5aa54a5)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "<stdin>", directory: "/usr/local/google/home/akhuang/testing/inline-line-tables", checksumkind: CSK_MD5, checksum: "38a4785b48742d3ea655b8f3461436a4")
+!2 = !{}
+!3 = !{i32 2, !"CodeView", i32 1}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{!"clang version 10.0.0 (https://github.com/llvm/llvm-project.git 03ec3a12a94bbbaa11999b6da3a43221a5aa54a5)"}
+!7 = distinct !DISubprogram(name: "f", scope: !8, file: !8, line: 1, type: !9, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !2)
+!8 = !DIFile(filename: "t.c", directory: "/usr/local/google/home/akhuang/testing/inline-line-tables", checksumkind: CSK_MD5, checksum: "38a4785b48742d3ea655b8f3461436a4")
+!9 = !DISubroutineType(types: !10)
+!10 = !{!11, !11}
+!11 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!12 = !DILocalVariable(name: "x", arg: 1, scope: !7, file: !8, line: 1, type: !11)
+!13 = !DILocation(line: 1, scope: !7)
+!14 = !DILocation(line: 2, scope: !7)
+!15 = distinct !DISubprogram(name: "g", scope: !8, file: !8, line: 4, type: !9, scopeLine: 4, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !2)
+!16 = !DILocalVariable(name: "x", arg: 1, scope: !15, file: !8, line: 4, type: !11)
+!17 = !DILocation(line: 4, scope: !15)
+!18 = !DILabel(scope: !15, name: "L", file: !8, line: 5)
+!19 = !DILocation(line: 5, scope: !15)
+!20 = !DILocation(line: 6, scope: !15)
+!21 = !DILocation(line: 7, scope: !15)
+!22 = distinct !DISubprogram(name: "main", scope: !8, file: !8, line: 9, type: !23, scopeLine: 9, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !2)
+!23 = !DISubroutineType(types: !24)
+!24 = !{!11}
+!25 = !DILocation(line: 10, scope: !22)
+!26 = !DILocation(line: 11, scope: !22)
+!27 = !DILocation(line: 12, scope: !22)
diff --git a/llvm/test/Transforms/Inline/noalias-calls.ll b/llvm/test/Transforms/Inline/noalias-calls.ll
index f4fe6fe9f..a5643cd2b 100644
--- a/llvm/test/Transforms/Inline/noalias-calls.ll
+++ b/llvm/test/Transforms/Inline/noalias-calls.ll
@@ -32,7 +32,7 @@ entry:
 ; CHECK:   ret void
 ; CHECK: }
 
-attributes #0 = { nounwind argmemonly }
+attributes #0 = { nounwind argmemonly willreturn }
 attributes #1 = { nounwind }
 attributes #2 = { nounwind uwtable }
 
diff --git a/llvm/test/Transforms/Inline/optimization-remarks-hotness-threshold.ll b/llvm/test/Transforms/Inline/optimization-remarks-hotness-threshold.ll
index 16d7db334..14b413b74 100644
--- a/llvm/test/Transforms/Inline/optimization-remarks-hotness-threshold.ll
+++ b/llvm/test/Transforms/Inline/optimization-remarks-hotness-threshold.ll
@@ -35,7 +35,7 @@ entry:
   ret i32 %call, !dbg !12
 }
 
-attributes #0 = { nounwind ssp uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="core2" "target-features"="+cx16,+fxsr,+mmx,+sse,+sse2,+sse3,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind ssp uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="core2" "target-features"="+cx16,+fxsr,+mmx,+sse,+sse2,+sse3,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!3, !4, !5}
diff --git a/llvm/test/Transforms/Inline/optimization-remarks-passed-yaml.ll b/llvm/test/Transforms/Inline/optimization-remarks-passed-yaml.ll
index 8692abfaf..3d572219a 100644
--- a/llvm/test/Transforms/Inline/optimization-remarks-passed-yaml.ll
+++ b/llvm/test/Transforms/Inline/optimization-remarks-passed-yaml.ll
@@ -57,7 +57,7 @@ entry:
   ret i32 %call, !dbg !12
 }
 
-attributes #0 = { nounwind ssp uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="core2" "target-features"="+cx16,+fxsr,+mmx,+sse,+sse2,+sse3,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind ssp uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="core2" "target-features"="+cx16,+fxsr,+mmx,+sse,+sse2,+sse3,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!3, !4, !5}
diff --git a/llvm/test/Transforms/Inline/optimization-remarks.ll b/llvm/test/Transforms/Inline/optimization-remarks.ll
index 72e90aefa..a3a21a848 100644
--- a/llvm/test/Transforms/Inline/optimization-remarks.ll
+++ b/llvm/test/Transforms/Inline/optimization-remarks.ll
@@ -67,9 +67,9 @@ entry:
   ret i32 %add
 }
 
-attributes #0 = { alwaysinline nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { noinline nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #2 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { alwaysinline nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { noinline nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.ident = !{!0}
 
diff --git a/llvm/test/Transforms/InstCombine/2004-11-27-SetCCForCastLargerAndConstant.ll b/llvm/test/Transforms/InstCombine/2004-11-27-SetCCForCastLargerAndConstant.ll
index 6a95c8237..170114fb4 100644
--- a/llvm/test/Transforms/InstCombine/2004-11-27-SetCCForCastLargerAndConstant.ll
+++ b/llvm/test/Transforms/InstCombine/2004-11-27-SetCCForCastLargerAndConstant.ll
@@ -1,6 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; This test case tests the InstructionCombining optimization that
-; reduces things like:
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; This tests the InstructionCombining optimization that reduces things like:
 ;   %Y = sext i8 %X to i32
 ;   %C = icmp ult i32 %Y, 1024
 ; to
@@ -10,12 +11,11 @@
 ; be eliminated. In many cases the setCC is also eliminated based on the
 ; constant value and the range of the casted value.
 ;
-; RUN: opt < %s -instcombine -S | FileCheck %s
 
 define i1 @lt_signed_to_large_unsigned(i8 %SB) {
 ; CHECK-LABEL: @lt_signed_to_large_unsigned(
-; CHECK-NEXT:    [[C1:%.*]] = icmp sgt i8 %SB, -1
-; CHECK-NEXT:    ret i1 [[C1]]
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i8 [[SB:%.*]], -1
+; CHECK-NEXT:    ret i1 [[C]]
 ;
   %Y = sext i8 %SB to i32
   %C = icmp ult i32 %Y, 1024
@@ -30,7 +30,7 @@ define i1 @lt_signed_to_large_unsigned(i8 %SB) {
 
 define i1 @PR28011(i16 %a) {
 ; CHECK-LABEL: @PR28011(
-; CHECK-NEXT:    [[CONV:%.*]] = sext i16 %a to i32
+; CHECK-NEXT:    [[CONV:%.*]] = sext i16 [[A:%.*]] to i32
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[CONV]], or (i32 zext (i1 icmp ne (i32*** bitcast ([1 x i32]* @b to i32***), i32*** @a) to i32), i32 1)
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -41,7 +41,7 @@ define i1 @PR28011(i16 %a) {
 
 define <2 x i1> @lt_signed_to_large_unsigned_vec(<2 x i8> %SB) {
 ; CHECK-LABEL: @lt_signed_to_large_unsigned_vec(
-; CHECK-NEXT:    [[Y:%.*]] = sext <2 x i8> %SB to <2 x i32>
+; CHECK-NEXT:    [[Y:%.*]] = sext <2 x i8> [[SB:%.*]] to <2 x i32>
 ; CHECK-NEXT:    [[C:%.*]] = icmp ult <2 x i32> [[Y]], <i32 1024, i32 2>
 ; CHECK-NEXT:    ret <2 x i1> [[C]]
 ;
@@ -70,7 +70,7 @@ define i1 @lt_signed_to_large_negative(i8 %SB) {
 
 define i1 @lt_signed_to_small_unsigned(i8 %SB) {
 ; CHECK-LABEL: @lt_signed_to_small_unsigned(
-; CHECK-NEXT:    [[C:%.*]] = icmp ult i8 %SB, 17
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i8 [[SB:%.*]], 17
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %Y = sext i8 %SB to i32
@@ -80,7 +80,7 @@ define i1 @lt_signed_to_small_unsigned(i8 %SB) {
 
 define i1 @lt_signed_to_small_signed(i8 %SB) {
 ; CHECK-LABEL: @lt_signed_to_small_signed(
-; CHECK-NEXT:    [[C:%.*]] = icmp slt i8 %SB, 17
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i8 [[SB:%.*]], 17
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %Y = sext i8 %SB to i32
@@ -89,7 +89,7 @@ define i1 @lt_signed_to_small_signed(i8 %SB) {
 }
 define i1 @lt_signed_to_small_negative(i8 %SB) {
 ; CHECK-LABEL: @lt_signed_to_small_negative(
-; CHECK-NEXT:    [[C:%.*]] = icmp slt i8 %SB, -17
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i8 [[SB:%.*]], -17
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %Y = sext i8 %SB to i32
@@ -126,7 +126,7 @@ define i1 @lt_unsigned_to_large_negative(i8 %SB) {
 
 define i1 @lt_unsigned_to_small_unsigned(i8 %SB) {
 ; CHECK-LABEL: @lt_unsigned_to_small_unsigned(
-; CHECK-NEXT:    [[C:%.*]] = icmp ult i8 %SB, 17
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i8 [[SB:%.*]], 17
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %Y = zext i8 %SB to i32
@@ -136,7 +136,7 @@ define i1 @lt_unsigned_to_small_unsigned(i8 %SB) {
 
 define i1 @lt_unsigned_to_small_signed(i8 %SB) {
 ; CHECK-LABEL: @lt_unsigned_to_small_signed(
-; CHECK-NEXT:    [[C:%.*]] = icmp ult i8 %SB, 17
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i8 [[SB:%.*]], 17
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %Y = zext i8 %SB to i32
@@ -155,7 +155,7 @@ define i1 @lt_unsigned_to_small_negative(i8 %SB) {
 
 define i1 @gt_signed_to_large_unsigned(i8 %SB) {
 ; CHECK-LABEL: @gt_signed_to_large_unsigned(
-; CHECK-NEXT:    [[C:%.*]] = icmp slt i8 %SB, 0
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i8 [[SB:%.*]], 0
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %Y = sext i8 %SB to i32
@@ -183,7 +183,7 @@ define i1 @gt_signed_to_large_negative(i8 %SB) {
 
 define i1 @gt_signed_to_small_unsigned(i8 %SB) {
 ; CHECK-LABEL: @gt_signed_to_small_unsigned(
-; CHECK-NEXT:    [[C:%.*]] = icmp ugt i8 %SB, 17
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i8 [[SB:%.*]], 17
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %Y = sext i8 %SB to i32
@@ -193,7 +193,7 @@ define i1 @gt_signed_to_small_unsigned(i8 %SB) {
 
 define i1 @gt_signed_to_small_signed(i8 %SB) {
 ; CHECK-LABEL: @gt_signed_to_small_signed(
-; CHECK-NEXT:    [[C:%.*]] = icmp sgt i8 %SB, 17
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i8 [[SB:%.*]], 17
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %Y = sext i8 %SB to i32
@@ -203,7 +203,7 @@ define i1 @gt_signed_to_small_signed(i8 %SB) {
 
 define i1 @gt_signed_to_small_negative(i8 %SB) {
 ; CHECK-LABEL: @gt_signed_to_small_negative(
-; CHECK-NEXT:    [[C:%.*]] = icmp sgt i8 %SB, -17
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i8 [[SB:%.*]], -17
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %Y = sext i8 %SB to i32
@@ -240,7 +240,7 @@ define i1 @gt_unsigned_to_large_negative(i8 %SB) {
 
 define i1 @gt_unsigned_to_small_unsigned(i8 %SB) {
 ; CHECK-LABEL: @gt_unsigned_to_small_unsigned(
-; CHECK-NEXT:    [[C:%.*]] = icmp ugt i8 %SB, 17
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i8 [[SB:%.*]], 17
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %Y = zext i8 %SB to i32
@@ -250,7 +250,7 @@ define i1 @gt_unsigned_to_small_unsigned(i8 %SB) {
 
 define i1 @gt_unsigned_to_small_signed(i8 %SB) {
 ; CHECK-LABEL: @gt_unsigned_to_small_signed(
-; CHECK-NEXT:    [[C:%.*]] = icmp ugt i8 %SB, 17
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i8 [[SB:%.*]], 17
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %Y = zext i8 %SB to i32
@@ -267,3 +267,200 @@ define i1 @gt_unsigned_to_small_negative(i8 %SB) {
   ret i1 %C
 }
 
+define i1 @different_size_zext_zext_ugt(i7 %x, i4 %y) {
+; CHECK-LABEL: @different_size_zext_zext_ugt(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i4 [[Y:%.*]] to i7
+; CHECK-NEXT:    [[R:%.*]] = icmp ult i7 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %zx = zext i7 %x to i25
+  %zy = zext i4 %y to i25
+  %r = icmp ugt i25 %zx, %zy
+  ret i1 %r
+}
+
+define <2 x i1> @different_size_zext_zext_ugt_commute(<2 x i4> %x, <2 x i7> %y) {
+; CHECK-LABEL: @different_size_zext_zext_ugt_commute(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext <2 x i4> [[X:%.*]] to <2 x i7>
+; CHECK-NEXT:    [[R:%.*]] = icmp ugt <2 x i7> [[TMP1]], [[Y:%.*]]
+; CHECK-NEXT:    ret <2 x i1> [[R]]
+;
+  %zx = zext <2 x i4> %x to <2 x i25>
+  %zy = zext <2 x i7> %y to <2 x i25>
+  %r = icmp ugt <2 x i25> %zx, %zy
+  ret <2 x i1> %r
+}
+
+define i1 @different_size_zext_zext_ult(i4 %x, i7 %y) {
+; CHECK-LABEL: @different_size_zext_zext_ult(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i4 [[X:%.*]] to i7
+; CHECK-NEXT:    [[R:%.*]] = icmp ult i7 [[TMP1]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %zx = zext i4 %x to i25
+  %zy = zext i7 %y to i25
+  %r = icmp ult i25 %zx, %zy
+  ret i1 %r
+}
+
+define i1 @different_size_zext_zext_eq(i4 %x, i7 %y) {
+; CHECK-LABEL: @different_size_zext_zext_eq(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i4 [[X:%.*]] to i7
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i7 [[TMP1]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %zx = zext i4 %x to i25
+  %zy = zext i7 %y to i25
+  %r = icmp eq i25 %zx, %zy
+  ret i1 %r
+}
+
+define i1 @different_size_zext_zext_ne_commute(i7 %x, i4 %y) {
+; CHECK-LABEL: @different_size_zext_zext_ne_commute(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i4 [[Y:%.*]] to i7
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i7 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %zx = zext i7 %x to i25
+  %zy = zext i4 %y to i25
+  %r = icmp ne i25 %zx, %zy
+  ret i1 %r
+}
+
+define i1 @different_size_zext_zext_slt(i7 %x, i4 %y) {
+; CHECK-LABEL: @different_size_zext_zext_slt(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i4 [[Y:%.*]] to i7
+; CHECK-NEXT:    [[R:%.*]] = icmp ugt i7 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %zx = zext i7 %x to i25
+  %zy = zext i4 %y to i25
+  %r = icmp slt i25 %zx, %zy
+  ret i1 %r
+}
+
+define i1 @different_size_zext_zext_sgt(i7 %x, i4 %y) {
+; CHECK-LABEL: @different_size_zext_zext_sgt(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i4 [[Y:%.*]] to i7
+; CHECK-NEXT:    [[R:%.*]] = icmp ult i7 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %zx = zext i7 %x to i25
+  %zy = zext i4 %y to i25
+  %r = icmp sgt i25 %zx, %zy
+  ret i1 %r
+}
+
+define i1 @different_size_sext_sext_sgt(i7 %x, i4 %y) {
+; CHECK-LABEL: @different_size_sext_sext_sgt(
+; CHECK-NEXT:    [[TMP1:%.*]] = sext i4 [[Y:%.*]] to i7
+; CHECK-NEXT:    [[R:%.*]] = icmp slt i7 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %sx = sext i7 %x to i25
+  %sy = sext i4 %y to i25
+  %r = icmp sgt i25 %sx, %sy
+  ret i1 %r
+}
+
+define i1 @different_size_sext_sext_sle(i7 %x, i4 %y) {
+; CHECK-LABEL: @different_size_sext_sext_sle(
+; CHECK-NEXT:    [[TMP1:%.*]] = sext i4 [[Y:%.*]] to i7
+; CHECK-NEXT:    [[R:%.*]] = icmp sge i7 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %sx = sext i7 %x to i25
+  %sy = sext i4 %y to i25
+  %r = icmp sle i25 %sx, %sy
+  ret i1 %r
+}
+
+define i1 @different_size_sext_sext_eq(i7 %x, i4 %y) {
+; CHECK-LABEL: @different_size_sext_sext_eq(
+; CHECK-NEXT:    [[TMP1:%.*]] = sext i4 [[Y:%.*]] to i7
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i7 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %sx = sext i7 %x to i25
+  %sy = sext i4 %y to i25
+  %r = icmp eq i25 %sx, %sy
+  ret i1 %r
+}
+
+define i1 @different_size_sext_sext_ule(i7 %x, i4 %y) {
+; CHECK-LABEL: @different_size_sext_sext_ule(
+; CHECK-NEXT:    [[TMP1:%.*]] = sext i4 [[Y:%.*]] to i7
+; CHECK-NEXT:    [[R:%.*]] = icmp uge i7 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %sx = sext i7 %x to i25
+  %sy = sext i4 %y to i25
+  %r = icmp ule i25 %sx, %sy
+  ret i1 %r
+}
+
+; TODO: This can be reduced.
+
+define i1 @different_size_sext_zext_ne(i7 %x, i4 %y) {
+; CHECK-LABEL: @different_size_sext_zext_ne(
+; CHECK-NEXT:    [[SX:%.*]] = sext i7 [[X:%.*]] to i25
+; CHECK-NEXT:    [[ZY:%.*]] = zext i4 [[Y:%.*]] to i25
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i25 [[SX]], [[ZY]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %sx = sext i7 %x to i25
+  %zy = zext i4 %y to i25
+  %r = icmp ne i25 %sx, %zy
+  ret i1 %r
+}
+
+declare void @use(i25)
+
+define i1 @different_size_sext_sext_ule_extra_use1(i7 %x, i4 %y) {
+; CHECK-LABEL: @different_size_sext_sext_ule_extra_use1(
+; CHECK-NEXT:    [[SY:%.*]] = sext i4 [[Y:%.*]] to i25
+; CHECK-NEXT:    call void @use(i25 [[SY]])
+; CHECK-NEXT:    [[TMP1:%.*]] = sext i4 [[Y]] to i7
+; CHECK-NEXT:    [[R:%.*]] = icmp uge i7 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %sx = sext i7 %x to i25
+  %sy = sext i4 %y to i25
+  call void @use(i25 %sy)
+  %r = icmp ule i25 %sx, %sy
+  ret i1 %r
+}
+
+define i1 @different_size_sext_sext_ule_extra_use2(i7 %x, i4 %y) {
+; CHECK-LABEL: @different_size_sext_sext_ule_extra_use2(
+; CHECK-NEXT:    [[SX:%.*]] = sext i7 [[X:%.*]] to i25
+; CHECK-NEXT:    call void @use(i25 [[SX]])
+; CHECK-NEXT:    [[TMP1:%.*]] = sext i4 [[Y:%.*]] to i7
+; CHECK-NEXT:    [[R:%.*]] = icmp uge i7 [[TMP1]], [[X]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %sx = sext i7 %x to i25
+  call void @use(i25 %sx)
+  %sy = sext i4 %y to i25
+  %r = icmp ule i25 %sx, %sy
+  ret i1 %r
+}
+
+; Negative test - extra uses on both casts is too much.
+
+define i1 @different_size_sext_sext_ule_extra_use3(i7 %x, i4 %y) {
+; CHECK-LABEL: @different_size_sext_sext_ule_extra_use3(
+; CHECK-NEXT:    [[SX:%.*]] = sext i7 [[X:%.*]] to i25
+; CHECK-NEXT:    call void @use(i25 [[SX]])
+; CHECK-NEXT:    [[SY:%.*]] = sext i4 [[Y:%.*]] to i25
+; CHECK-NEXT:    call void @use(i25 [[SY]])
+; CHECK-NEXT:    [[R:%.*]] = icmp ule i25 [[SX]], [[SY]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %sx = sext i7 %x to i25
+  call void @use(i25 %sx)
+  %sy = sext i4 %y to i25
+  call void @use(i25 %sy)
+  %r = icmp ule i25 %sx, %sy
+  ret i1 %r
+}
diff --git a/llvm/test/Transforms/InstCombine/2008-12-17-SRemNegConstVec.ll b/llvm/test/Transforms/InstCombine/2008-12-17-SRemNegConstVec.ll
index 75bd5e017..cc8737dbd 100644
--- a/llvm/test/Transforms/InstCombine/2008-12-17-SRemNegConstVec.ll
+++ b/llvm/test/Transforms/InstCombine/2008-12-17-SRemNegConstVec.ll
@@ -1,7 +1,12 @@
-; RUN: opt < %s -instcombine -S | grep "i8 2, i8 2"
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
 ; PR2756
 
 define <2 x i8> @foo(<2 x i8> %x) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:    [[A:%.*]] = srem <2 x i8> [[X:%.*]], <i8 2, i8 2>
+; CHECK-NEXT:    ret <2 x i8> [[A]]
+;
   %A = srem <2 x i8> %x, <i8 2, i8 -2>
   ret <2 x i8> %A
 }
diff --git a/llvm/test/Transforms/InstCombine/2009-06-16-SRemDemandedBits.ll b/llvm/test/Transforms/InstCombine/2009-06-16-SRemDemandedBits.ll
index 6beedf83c..8730b4e08 100644
--- a/llvm/test/Transforms/InstCombine/2009-06-16-SRemDemandedBits.ll
+++ b/llvm/test/Transforms/InstCombine/2009-06-16-SRemDemandedBits.ll
@@ -1,9 +1,16 @@
-; RUN: opt < %s -instcombine -S | grep srem
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
 ; PR3439
 
 define i32 @a(i32 %x) nounwind {
+; CHECK-LABEL: @a(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[REM:%.*]] = srem i32 [[X:%.*]], 2
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[REM]], 2
+; CHECK-NEXT:    ret i32 [[AND]]
+;
 entry:
-	%rem = srem i32 %x, 2
-	%and = and i32 %rem, 2
-	ret i32 %and
+  %rem = srem i32 %x, 2
+  %and = and i32 %rem, 2
+  ret i32 %and
 }
diff --git a/llvm/test/Transforms/InstCombine/2010-01-28-NegativeSRem.ll b/llvm/test/Transforms/InstCombine/2010-01-28-NegativeSRem.ll
index 4ab9bf0c3..077419027 100644
--- a/llvm/test/Transforms/InstCombine/2010-01-28-NegativeSRem.ll
+++ b/llvm/test/Transforms/InstCombine/2010-01-28-NegativeSRem.ll
@@ -1,19 +1,28 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -instcombine -S | FileCheck %s
 ; PR6165
 
 define i32 @f() {
+; CHECK-LABEL: @f(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[BB1:%.*]]
+; CHECK:       BB1:
+; CHECK-NEXT:    [[X:%.*]] = phi i32 [ -29, [[ENTRY:%.*]] ], [ 0, [[BB1]] ]
+; CHECK-NEXT:    [[REM:%.*]] = srem i32 [[X]], 2
+; CHECK-NEXT:    [[T:%.*]] = icmp eq i32 [[REM]], -1
+; CHECK-NEXT:    br i1 [[T]], label [[BB2:%.*]], label [[BB1]]
+; CHECK:       BB2:
+; CHECK-NEXT:    ret i32 [[X]]
+;
 entry:
   br label %BB1
 
 BB1:                                              ; preds = %BB1, %entry
-; CHECK: BB1:
   %x = phi i32 [ -29, %entry ], [ 0, %BB1 ]       ; <i32> [#uses=2]
   %rem = srem i32 %x, 2                           ; <i32> [#uses=1]
   %t = icmp eq i32 %rem, -1                       ; <i1> [#uses=1]
   br i1 %t, label %BB2, label %BB1
-; CHECK-NOT: br i1 false
 
 BB2:                                              ; preds = %BB1
-; CHECK: BB2:
   ret i32 %x
 }
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts.ll b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts.ll
index 9c45cf5aa..389ca6010 100644
--- a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts.ll
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts.ll
@@ -152,11 +152,10 @@ define amdgpu_ps <3 x float> @extract_elt0_elt2_elt3_buffer_load_v4f32(<4 x i32>
   ret <3 x float> %shuf
 }
 
-; FIXME: Not handled even though only 2 elts used
 ; CHECK-LABEL: @extract_elt0_elt1_buffer_load_v4f32_2(
-; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
-; CHECK-NEXT: %elt0 = extractelement <4 x float> %data, i32 0
-; CHECK-NEXT: %elt1 = extractelement <4 x float> %data, i32 1
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.buffer.load.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+; CHECK-NEXT: %elt0 = extractelement <2 x float> %data, i32 0
+; CHECK-NEXT: %elt1 = extractelement <2 x float> %data, i32 1
 ; CHECK-NEXT: %ins0 = insertvalue { float, float } undef, float %elt0, 0
 ; CHECK-NEXT: %ins1 = insertvalue { float, float } %ins0, float %elt1, 1
 ; CHECK-NEXT: ret { float, float } %ins1
@@ -169,6 +168,74 @@ define amdgpu_ps { float, float } @extract_elt0_elt1_buffer_load_v4f32_2(<4 x i3
   ret { float, float } %ins1
 }
 
+; CHECK-LABEL: @extract_elt0_elt1_elt2_buffer_load_v4f32_2(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+; CHECK-NEXT: %elt0 = extractelement <3 x float> %data, i32 0
+; CHECK-NEXT: %elt1 = extractelement <3 x float> %data, i32 1
+; CHECK-NEXT: %elt2 = extractelement <3 x float> %data, i32 2
+; CHECK-NEXT: %ins0 = insertvalue { float, float, float } undef, float %elt0, 0
+; CHECK-NEXT: %ins1 = insertvalue { float, float, float } %ins0, float %elt1, 1
+; CHECK-NEXT: %ins2 = insertvalue { float, float, float } %ins1, float %elt2, 2
+; CHECK-NEXT: ret { float, float, float } %ins2
+define amdgpu_ps { float, float, float } @extract_elt0_elt1_elt2_buffer_load_v4f32_2(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
+  %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  %elt1 = extractelement <4 x float> %data, i32 1
+  %elt2 = extractelement <4 x float> %data, i32 2
+  %ins0 = insertvalue { float, float, float } undef, float %elt0, 0
+  %ins1 = insertvalue { float, float, float } %ins0, float %elt1, 1
+  %ins2 = insertvalue { float, float, float } %ins1, float %elt2, 2
+  ret { float, float, float } %ins2
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_elt2_buffer_load_v4f32_3(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+; CHECK-NEXT: %ins1 = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT: %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 undef, i32 1>
+; CHECK-NEXT: %ret = fadd <2 x float> %ins1, %shuf
+define amdgpu_ps <2 x float> @extract_elt0_elt1_elt2_buffer_load_v4f32_3(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
+  %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  %elt2 = extractelement <4 x float> %data, i32 2
+  %ins0 = insertelement <2 x float> undef, float %elt0, i32 0
+  %ins1 = insertelement <2 x float> %ins0, float %elt2, i32 1
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 4, i32 1>
+  %ret = fadd <2 x float> %ins1, %shuf
+  ret <2 x float> %ret
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_elt2_buffer_load_v4f32_4(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+; CHECK-NEXT: %ins1 = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT: %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 1, i32 undef>
+; CHECK-NEXT: %ret = fadd <2 x float> %ins1, %shuf
+; CHECK-NEXT: ret <2 x float> %ret
+define amdgpu_ps <2 x float> @extract_elt0_elt1_elt2_buffer_load_v4f32_4(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
+  %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  %elt2 = extractelement <4 x float> %data, i32 2
+  %ins0 = insertelement <2 x float> undef, float %elt0, i32 0
+  %ins1 = insertelement <2 x float> %ins0, float %elt2, i32 1
+  %shuf = shufflevector <4 x float> undef, <4 x float> %data, <2 x i32> <i32 5, i32 1>
+  %ret = fadd <2 x float> %ins1, %shuf
+  ret <2 x float> %ret
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_elt2_buffer_load_v4f32_5(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+; CHECK-NEXT: %ins1 = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 2, i32 2>
+; CHECK-NEXT: %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: %ret = fadd <2 x float> %ins1, %shuf
+define amdgpu_ps <2 x float> @extract_elt0_elt1_elt2_buffer_load_v4f32_5(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
+  %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+  %elt2 = extractelement <4 x float> %data, i32 2
+  %ins0 = insertelement <2 x float> undef, float %elt2, i32 0
+  %ins1 = insertelement <2 x float> %ins0, float %elt2, i32 1
+  %shuf = shufflevector <4 x float> %data, <4 x float> %data, <2 x i32> <i32 0, i32 5>
+  %ret = fadd <2 x float> %ins1, %shuf
+  ret <2 x float> %ret
+}
+
 ; CHECK-LABEL: @extract_elt0_buffer_load_v3f32(
 ; CHECK-NEXT: %data = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
 ; CHECK-NEXT: ret float %data
@@ -765,14 +832,12 @@ define i32 @extract0_bitcast_raw_buffer_load_format_v4f32(<4 x i32> inreg %rsrc,
 }
 
 ; CHECK-LABEL: @extract0_bitcast_raw_buffer_load_format_v4i32(
-; CHECK-NEXT: %tmp = call i32 @llvm.amdgcn.raw.buffer.load.format.i32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
-; CHECK-NEXT: %tmp2 = bitcast i32 %tmp to float
-; CHECK-NEXT: ret float %tmp2
+; CHECK-NEXT: %tmp = call float @llvm.amdgcn.raw.buffer.load.format.f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret float %tmp
 define float @extract0_bitcast_raw_buffer_load_format_v4i32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
-  %tmp = call <4 x i32> @llvm.amdgcn.raw.buffer.load.format.v4i32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
-  %tmp1 = bitcast <4 x i32> %tmp to <4 x float>
-  %tmp2 = extractelement <4 x float> %tmp1, i32 0
-  ret float %tmp2
+  %tmp = call <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+  %tmp1 = extractelement <4 x float> %tmp, i32 0
+  ret float %tmp1
 }
 
 ; CHECK-LABEL: @preserve_metadata_extract_elt0_raw_buffer_load_format_v2f32(
@@ -790,8 +855,6 @@ declare <2 x float> @llvm.amdgcn.raw.buffer.load.format.v2f32(<4 x i32>, i32, i3
 declare <3 x float> @llvm.amdgcn.raw.buffer.load.format.v3f32(<4 x i32>, i32, i32, i32) #1
 declare <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32>, i32, i32, i32) #1
 
-declare <4 x i32> @llvm.amdgcn.raw.buffer.load.format.v4i32(<4 x i32>, i32, i32, i32) #1
-
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.struct.buffer.load
 ; --------------------------------------------------------------------
@@ -1242,17 +1305,6 @@ define i32 @extract0_bitcast_struct_buffer_load_format_v4f32(<4 x i32> inreg %rs
   ret i32 %tmp2
 }
 
-; CHECK-LABEL: @extract0_bitcast_struct_buffer_load_format_v4i32(
-; CHECK-NEXT: %tmp = call i32 @llvm.amdgcn.struct.buffer.load.format.i32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
-; CHECK-NEXT: %tmp2 = bitcast i32 %tmp to float
-; CHECK-NEXT: ret float %tmp2
-define float @extract0_bitcast_struct_buffer_load_format_v4i32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
-  %tmp = call <4 x i32> @llvm.amdgcn.struct.buffer.load.format.v4i32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
-  %tmp1 = bitcast <4 x i32> %tmp to <4 x float>
-  %tmp2 = extractelement <4 x float> %tmp1, i32 0
-  ret float %tmp2
-}
-
 ; CHECK-LABEL: @preserve_metadata_extract_elt0_struct_buffer_load_format_v2f32(
 ; CHECK-NEXT: %data = call float @llvm.amdgcn.struct.buffer.load.format.f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0), !fpmath !0
 ; CHECK-NEXT: ret float %data
@@ -1270,6 +1322,709 @@ declare <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32>, i32,
 
 declare <4 x i32> @llvm.amdgcn.struct.buffer.load.format.v4i32(<4 x i32>, i32, i32, i32, i32) #1
 
+; --------------------------------------------------------------------
+; llvm.amdgcn.raw.tbuffer.load
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @raw_tbuffer_load_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.raw.tbuffer.load.f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @raw_tbuffer_load_f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 inreg %arg1) #0 {
+  %data = call float @llvm.amdgcn.raw.tbuffer.load.f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+  ret float %data
+}
+
+; CHECK-LABEL: @raw_tbuffer_load_v2f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.raw.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+; CHECK-NEXT: ret <2 x float> %data
+define amdgpu_ps <2 x float> @raw_tbuffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 inreg %arg1) #0 {
+  %data = call <2 x float> @llvm.amdgcn.raw.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+  ret <2 x float> %data
+}
+
+; CHECK-LABEL: @raw_tbuffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.raw.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+; CHECK-NEXT: ret <4 x float> %data
+define amdgpu_ps <4 x float> @raw_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 inreg %arg1) #0 {
+  %data = call <4 x float> @llvm.amdgcn.raw.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+  ret <4 x float> %data
+}
+
+; CHECK-LABEL: @extract_elt0_raw_tbuffer_load_v2f32(
+; CHECK: %data = call float @llvm.amdgcn.raw.tbuffer.load.f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_raw_tbuffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 inreg %arg1) #0 {
+  %data = call <2 x float> @llvm.amdgcn.raw.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+  %elt0 = extractelement <2 x float> %data, i32 0
+  ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_raw_tbuffer_load_v2f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.raw.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+; CHECK-NEXT: %elt1 = extractelement <2 x float> %data, i32 1
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt1_raw_tbuffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 inreg %arg1) #0 {
+  %data = call <2 x float> @llvm.amdgcn.raw.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+  %elt1 = extractelement <2 x float> %data, i32 1
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_raw_tbuffer_load_v4f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.raw.tbuffer.load.f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_raw_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 inreg %arg1) #0 {
+  %data = call <4 x float> @llvm.amdgcn.raw.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_raw_tbuffer_load_v4f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.raw.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+; CHECK-NEXT: %elt1 = extractelement <2 x float> %data, i32 1
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt1_raw_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 inreg %arg1) #0 {
+  %data = call <4 x float> @llvm.amdgcn.raw.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+  %elt1 = extractelement <4 x float> %data, i32 1
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt2_raw_tbuffer_load_v4f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.raw.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+; CHECK-NEXT: %elt1 = extractelement <3 x float> %data, i32 2
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt2_raw_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 inreg %arg1) #0 {
+  %data = call <4 x float> @llvm.amdgcn.raw.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+  %elt1 = extractelement <4 x float> %data, i32 2
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt3_raw_tbuffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.raw.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+; CHECK-NEXT: %elt1 = extractelement <4 x float> %data, i32 3
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt3_raw_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 inreg %arg1) #0 {
+  %data = call <4 x float> @llvm.amdgcn.raw.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+  %elt1 = extractelement <4 x float> %data, i32 3
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_raw_tbuffer_load_v4f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.raw.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+; CHECK-NEXT: ret <2 x float>
+define amdgpu_ps <2 x float> @extract_elt0_elt1_raw_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 inreg %arg1) #0 {
+  %data = call <4 x float> @llvm.amdgcn.raw.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 0, i32 1>
+  ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt1_elt2_raw_tbuffer_load_v4f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.raw.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+; CHECK-NEXT: %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 1, i32 2>
+; CHECK-NEXT: ret <2 x float> %shuf
+define amdgpu_ps <2 x float> @extract_elt1_elt2_raw_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 inreg %arg1) #0 {
+  %data = call <4 x float> @llvm.amdgcn.raw.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 1, i32 2>
+  ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt2_elt3_raw_tbuffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.raw.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+; CHECK-NEXT: %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT: ret <2 x float> %shuf
+define amdgpu_ps <2 x float> @extract_elt2_elt3_raw_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 inreg %arg1) #0 {
+  %data = call <4 x float> @llvm.amdgcn.raw.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 2, i32 3>
+  ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_elt2_raw_tbuffer_load_v4f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.raw.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+; CHECK-NEXT: ret <3 x float> %data
+define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_raw_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 inreg %arg1) #0 {
+  %data = call <4 x float> @llvm.amdgcn.raw.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 1, i32 2>
+  ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt1_elt2_elt3_raw_tbuffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.raw.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+; CHECK-NEXT: %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 1, i32 2, i32 3>
+; CHECK-NEXT: ret <3 x float> %shuf
+define amdgpu_ps <3 x float> @extract_elt1_elt2_elt3_raw_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 inreg %arg1) #0 {
+  %data = call <4 x float> @llvm.amdgcn.raw.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 1, i32 2, i32 3>
+  ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_elt2_elt3_raw_tbuffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.raw.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+; CHECK-NEXT: %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 2, i32 3>
+; CHECK-NEXT: ret <3 x float> %shuf
+define amdgpu_ps <3 x float> @extract_elt0_elt2_elt3_raw_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 inreg %arg1) #0 {
+  %data = call <4 x float> @llvm.amdgcn.raw.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 2, i32 3>
+  ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_raw_tbuffer_load_v3f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.raw.tbuffer.load.f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_raw_tbuffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 inreg %arg1) #0 {
+  %data = call <3 x float> @llvm.amdgcn.raw.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+  %elt0 = extractelement <3 x float> %data, i32 0
+  ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_raw_tbuffer_load_v3f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.raw.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+; CHECK-NEXT: %elt1 = extractelement <2 x float> %data, i32 1
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt1_raw_tbuffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 inreg %arg1) #0 {
+  %data = call <3 x float> @llvm.amdgcn.raw.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+  %elt1 = extractelement <3 x float> %data, i32 1
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt2_raw_tbuffer_load_v3f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.raw.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+; CHECK-NEXT: %elt1 = extractelement <3 x float> %data, i32 2
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt2_raw_tbuffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 inreg %arg1) #0 {
+  %data = call <3 x float> @llvm.amdgcn.raw.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+  %elt1 = extractelement <3 x float> %data, i32 2
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_raw_tbuffer_load_v3f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.raw.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+; CHECK-NEXT: ret <2 x float>
+define amdgpu_ps <2 x float> @extract_elt0_elt1_raw_tbuffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 inreg %arg1) #0 {
+  %data = call <3 x float> @llvm.amdgcn.raw.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+  %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 0, i32 1>
+  ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt1_elt2_raw_tbuffer_load_v3f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.raw.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+; CHECK-NEXT: %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 1, i32 2>
+; CHECK-NEXT: ret <2 x float> %shuf
+define amdgpu_ps <2 x float> @extract_elt1_elt2_raw_tbuffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 inreg %arg1) #0 {
+  %data = call <3 x float> @llvm.amdgcn.raw.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+  %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 1, i32 2>
+  ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract0_bitcast_raw_tbuffer_load_v4f32(
+; CHECK-NEXT: %tmp = call float @llvm.amdgcn.raw.tbuffer.load.f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+; CHECK-NEXT: %tmp2 = bitcast float %tmp to i32
+; CHECK-NEXT: ret i32 %tmp2
+define i32 @extract0_bitcast_raw_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 inreg %arg1) #0 {
+  %tmp = call <4 x float> @llvm.amdgcn.raw.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+  %tmp1 = bitcast <4 x float> %tmp to <4 x i32>
+  %tmp2 = extractelement <4 x i32> %tmp1, i32 0
+  ret i32 %tmp2
+}
+
+; CHECK-LABEL: @preserve_metadata_extract_elt0_raw_tbuffer_load_v2f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.raw.tbuffer.load.f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0), !fpmath !0
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @preserve_metadata_extract_elt0_raw_tbuffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 inreg %arg1) #0 {
+  %data = call <2 x float> @llvm.amdgcn.raw.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0), !fpmath !0
+  %elt0 = extractelement <2 x float> %data, i32 0
+  ret float %elt0
+}
+
+declare float @llvm.amdgcn.raw.tbuffer.load.f32(<4 x i32>, i32, i32, i32, i32) #1
+declare <2 x float> @llvm.amdgcn.raw.tbuffer.load.v2f32(<4 x i32>, i32, i32, i32, i32) #1
+declare <3 x float> @llvm.amdgcn.raw.tbuffer.load.v3f32(<4 x i32>, i32, i32, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.raw.tbuffer.load.v4f32(<4 x i32>, i32, i32, i32, i32) #1
+
+declare <4 x i32> @llvm.amdgcn.raw.tbuffer.load.v4i32(<4 x i32>, i32, i32, i32, i32) #1
+
+; CHECK-LABEL: @extract_elt3_raw_tbuffer_load_v4f16(
+; CHECK-NEXT: %data = call <4 x half> @llvm.amdgcn.raw.tbuffer.load.v4f16(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+; CHECK-NEXT: %elt1 = extractelement <4 x half> %data, i32 3
+; CHECK-NEXT: ret half %elt1
+define amdgpu_ps half @extract_elt3_raw_tbuffer_load_v4f16(<4 x i32> inreg %rsrc, i32 %arg0, i32 inreg %arg1) #0 {
+  %data = call <4 x half> @llvm.amdgcn.raw.tbuffer.load.v4f16(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+  %elt1 = extractelement <4 x half> %data, i32 3
+  ret half %elt1
+}
+
+; FIXME: Enable load shortening when full support for v3f16 has been added (should expect call <3 x half> @llvm.amdgcn.raw.tbuffer.load.v3f16).
+; CHECK-LABEL: @extract_elt2_raw_tbuffer_load_v4f16(
+; CHECK-NEXT: %data = call <4 x half> @llvm.amdgcn.raw.tbuffer.load.v4f16(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+; CHECK-NEXT: %elt1 = extractelement <4 x half> %data, i32 2
+; CHECK-NEXT: ret half %elt1
+define amdgpu_ps half @extract_elt2_raw_tbuffer_load_v4f16(<4 x i32> inreg %rsrc, i32 %arg0, i32 inreg %arg1) #0 {
+  %data = call <4 x half> @llvm.amdgcn.raw.tbuffer.load.v4f16(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+  %elt1 = extractelement <4 x half> %data, i32 2
+  ret half %elt1
+}
+
+; CHECK-LABEL: @extract_elt1_raw_tbuffer_load_v4f16(
+; CHECK-NEXT: %data = call <2 x half> @llvm.amdgcn.raw.tbuffer.load.v2f16(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+; CHECK-NEXT: %elt1 = extractelement <2 x half> %data, i32 1
+; CHECK-NEXT: ret half %elt1
+define amdgpu_ps half @extract_elt1_raw_tbuffer_load_v4f16(<4 x i32> inreg %rsrc, i32 %arg0, i32 inreg %arg1) #0 {
+  %data = call <4 x half> @llvm.amdgcn.raw.tbuffer.load.v4f16(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+  %elt1 = extractelement <4 x half> %data, i32 1
+  ret half %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_raw_tbuffer_load_v4f16(
+; CHECK-NEXT: %data = call half @llvm.amdgcn.raw.tbuffer.load.f16(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+; CHECK-NEXT: ret half %data
+define amdgpu_ps half @extract_elt0_raw_tbuffer_load_v4f16(<4 x i32> inreg %rsrc, i32 %arg0, i32 inreg %arg1) #0 {
+  %data = call <4 x half> @llvm.amdgcn.raw.tbuffer.load.v4f16(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+  %elt1 = extractelement <4 x half> %data, i32 0
+  ret half %elt1
+}
+
+declare half @llvm.amdgcn.raw.tbuffer.load.f16(<4 x i32>, i32, i32, i32, i32) #1
+declare <2 x half> @llvm.amdgcn.raw.tbuffer.load.v2f16(<4 x i32>, i32, i32, i32, i32) #1
+declare <3 x half> @llvm.amdgcn.raw.tbuffer.load.v3f16(<4 x i32>, i32, i32, i32, i32) #1
+declare <4 x half> @llvm.amdgcn.raw.tbuffer.load.v4f16(<4 x i32>, i32, i32, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.struct.tbuffer.load
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @struct_tbuffer_load_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.struct.tbuffer.load.f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @struct_tbuffer_load_f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1, i32 inreg %arg2) #0 {
+  %data = call float @llvm.amdgcn.struct.tbuffer.load.f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+  ret float %data
+}
+
+; CHECK-LABEL: @struct_tbuffer_load_v2f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.struct.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+; CHECK-NEXT: ret <2 x float> %data
+define amdgpu_ps <2 x float> @struct_tbuffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1, i32 inreg %arg2) #0 {
+  %data = call <2 x float> @llvm.amdgcn.struct.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+  ret <2 x float> %data
+}
+
+; CHECK-LABEL: @struct_tbuffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.struct.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+; CHECK-NEXT: ret <4 x float> %data
+define amdgpu_ps <4 x float> @struct_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1, i32 inreg %arg2) #0 {
+  %data = call <4 x float> @llvm.amdgcn.struct.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+  ret <4 x float> %data
+}
+
+; CHECK-LABEL: @extract_elt0_struct_tbuffer_load_v2f32(
+; CHECK: %data = call float @llvm.amdgcn.struct.tbuffer.load.f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_struct_tbuffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1, i32 inreg %arg2) #0 {
+  %data = call <2 x float> @llvm.amdgcn.struct.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+  %elt0 = extractelement <2 x float> %data, i32 0
+  ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_struct_tbuffer_load_v2f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.struct.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+; CHECK-NEXT: %elt1 = extractelement <2 x float> %data, i32 1
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt1_struct_tbuffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1, i32 inreg %arg2) #0 {
+  %data = call <2 x float> @llvm.amdgcn.struct.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+  %elt1 = extractelement <2 x float> %data, i32 1
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_struct_tbuffer_load_v4f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.struct.tbuffer.load.f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_struct_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1, i32 inreg %arg2) #0 {
+  %data = call <4 x float> @llvm.amdgcn.struct.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_struct_tbuffer_load_v4f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.struct.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+; CHECK-NEXT: %elt1 = extractelement <2 x float> %data, i32 1
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt1_struct_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1, i32 inreg %arg2) #0 {
+  %data = call <4 x float> @llvm.amdgcn.struct.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+  %elt1 = extractelement <4 x float> %data, i32 1
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt2_struct_tbuffer_load_v4f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.struct.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+; CHECK-NEXT: %elt1 = extractelement <3 x float> %data, i32 2
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt2_struct_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1, i32 inreg %arg2) #0 {
+  %data = call <4 x float> @llvm.amdgcn.struct.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+  %elt1 = extractelement <4 x float> %data, i32 2
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt3_struct_tbuffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.struct.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+; CHECK-NEXT: %elt1 = extractelement <4 x float> %data, i32 3
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt3_struct_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1, i32 inreg %arg2) #0 {
+  %data = call <4 x float> @llvm.amdgcn.struct.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+  %elt1 = extractelement <4 x float> %data, i32 3
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_struct_tbuffer_load_v4f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.struct.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+; CHECK-NEXT: ret <2 x float>
+define amdgpu_ps <2 x float> @extract_elt0_elt1_struct_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1, i32 inreg %arg2) #0 {
+  %data = call <4 x float> @llvm.amdgcn.struct.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 0, i32 1>
+  ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt1_elt2_struct_tbuffer_load_v4f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.struct.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+; CHECK-NEXT: %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 1, i32 2>
+; CHECK-NEXT: ret <2 x float> %shuf
+define amdgpu_ps <2 x float> @extract_elt1_elt2_struct_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1, i32 inreg %arg2) #0 {
+  %data = call <4 x float> @llvm.amdgcn.struct.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 1, i32 2>
+  ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt2_elt3_struct_tbuffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.struct.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+; CHECK-NEXT: %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT: ret <2 x float> %shuf
+define amdgpu_ps <2 x float> @extract_elt2_elt3_struct_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1, i32 inreg %arg2) #0 {
+  %data = call <4 x float> @llvm.amdgcn.struct.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 2, i32 3>
+  ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_elt2_struct_tbuffer_load_v4f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.struct.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+; CHECK-NEXT: ret <3 x float> %data
+define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_struct_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1, i32 inreg %arg2) #0 {
+  %data = call <4 x float> @llvm.amdgcn.struct.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 1, i32 2>
+  ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt1_elt2_elt3_struct_tbuffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.struct.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+; CHECK-NEXT: %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 1, i32 2, i32 3>
+; CHECK-NEXT: ret <3 x float> %shuf
+define amdgpu_ps <3 x float> @extract_elt1_elt2_elt3_struct_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1, i32 inreg %arg2) #0 {
+  %data = call <4 x float> @llvm.amdgcn.struct.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 1, i32 2, i32 3>
+  ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_elt2_elt3_struct_tbuffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.struct.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+; CHECK-NEXT: %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 2, i32 3>
+; CHECK-NEXT: ret <3 x float> %shuf
+define amdgpu_ps <3 x float> @extract_elt0_elt2_elt3_struct_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1, i32 inreg %arg2) #0 {
+  %data = call <4 x float> @llvm.amdgcn.struct.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 2, i32 3>
+  ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_struct_tbuffer_load_v3f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.struct.tbuffer.load.f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_struct_tbuffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1, i32 inreg %arg2) #0 {
+  %data = call <3 x float> @llvm.amdgcn.struct.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+  %elt0 = extractelement <3 x float> %data, i32 0
+  ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_struct_tbuffer_load_v3f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.struct.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+; CHECK-NEXT: %elt1 = extractelement <2 x float> %data, i32 1
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt1_struct_tbuffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1, i32 inreg %arg2) #0 {
+  %data = call <3 x float> @llvm.amdgcn.struct.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+  %elt1 = extractelement <3 x float> %data, i32 1
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt2_struct_tbuffer_load_v3f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.struct.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+; CHECK-NEXT: %elt1 = extractelement <3 x float> %data, i32 2
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt2_struct_tbuffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1, i32 inreg %arg2) #0 {
+  %data = call <3 x float> @llvm.amdgcn.struct.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+  %elt1 = extractelement <3 x float> %data, i32 2
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_struct_tbuffer_load_v3f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.struct.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+; CHECK-NEXT: ret <2 x float>
+define amdgpu_ps <2 x float> @extract_elt0_elt1_struct_tbuffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1, i32 inreg %arg2) #0 {
+  %data = call <3 x float> @llvm.amdgcn.struct.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+  %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 0, i32 1>
+  ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt1_elt2_struct_tbuffer_load_v3f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.struct.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+; CHECK-NEXT: %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 1, i32 2>
+; CHECK-NEXT: ret <2 x float> %shuf
+define amdgpu_ps <2 x float> @extract_elt1_elt2_struct_tbuffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1, i32 inreg %arg2) #0 {
+  %data = call <3 x float> @llvm.amdgcn.struct.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+  %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 1, i32 2>
+  ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract0_bitcast_struct_tbuffer_load_v4f32(
+; CHECK-NEXT: %tmp = call float @llvm.amdgcn.struct.tbuffer.load.f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+; CHECK-NEXT: %tmp2 = bitcast float %tmp to i32
+; CHECK-NEXT: ret i32 %tmp2
+define i32 @extract0_bitcast_struct_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1, i32 inreg %arg2) #0 {
+  %tmp = call <4 x float> @llvm.amdgcn.struct.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+  %tmp1 = bitcast <4 x float> %tmp to <4 x i32>
+  %tmp2 = extractelement <4 x i32> %tmp1, i32 0
+  ret i32 %tmp2
+}
+
+; CHECK-LABEL: @preserve_metadata_extract_elt0_struct_tbuffer_load_v2f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.struct.tbuffer.load.f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0), !fpmath !0
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @preserve_metadata_extract_elt0_struct_tbuffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1, i32 inreg %arg2) #0 {
+  %data = call <2 x float> @llvm.amdgcn.struct.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0), !fpmath !0
+  %elt0 = extractelement <2 x float> %data, i32 0
+  ret float %elt0
+}
+
+declare float @llvm.amdgcn.struct.tbuffer.load.f32(<4 x i32>, i32, i32, i32, i32, i32) #1
+declare <2 x float> @llvm.amdgcn.struct.tbuffer.load.v2f32(<4 x i32>, i32, i32, i32, i32, i32) #1
+declare <3 x float> @llvm.amdgcn.struct.tbuffer.load.v3f32(<4 x i32>, i32, i32, i32, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.struct.tbuffer.load.v4f32(<4 x i32>, i32, i32, i32, i32, i32) #1
+
+declare <4 x i32> @llvm.amdgcn.struct.tbuffer.load.v4i32(<4 x i32>, i32, i32, i32, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.tbuffer.load
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @tbuffer_load_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.tbuffer.load.f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @tbuffer_load_f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
+  %data = call float @llvm.amdgcn.tbuffer.load.f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+  ret float %data
+}
+
+; CHECK-LABEL: @tbuffer_load_v2f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+; CHECK-NEXT: ret <2 x float> %data
+define amdgpu_ps <2 x float> @tbuffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
+  %data = call <2 x float> @llvm.amdgcn.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+  ret <2 x float> %data
+}
+
+; CHECK-LABEL: @tbuffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+; CHECK-NEXT: ret <4 x float> %data
+define amdgpu_ps <4 x float> @tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
+  %data = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+  ret <4 x float> %data
+}
+
+; CHECK-LABEL: @extract_elt0_tbuffer_load_v2f32(
+; CHECK: %data = call float @llvm.amdgcn.tbuffer.load.f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_tbuffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
+  %data = call <2 x float> @llvm.amdgcn.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+  %elt0 = extractelement <2 x float> %data, i32 0
+  ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_tbuffer_load_v2f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+; CHECK-NEXT: %elt1 = extractelement <2 x float> %data, i32 1
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt1_tbuffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
+  %data = call <2 x float> @llvm.amdgcn.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+  %elt1 = extractelement <2 x float> %data, i32 1
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_tbuffer_load_v4f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.tbuffer.load.f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
+  %data = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_tbuffer_load_v4f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+; CHECK-NEXT: %elt1 = extractelement <2 x float> %data, i32 1
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt1_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
+  %data = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+  %elt1 = extractelement <4 x float> %data, i32 1
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt2_tbuffer_load_v4f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+; CHECK-NEXT: %elt1 = extractelement <3 x float> %data, i32 2
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt2_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
+  %data = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+  %elt1 = extractelement <4 x float> %data, i32 2
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt3_tbuffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+; CHECK-NEXT: %elt1 = extractelement <4 x float> %data, i32 3
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt3_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
+  %data = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+  %elt1 = extractelement <4 x float> %data, i32 3
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_tbuffer_load_v4f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+; CHECK-NEXT: ret <2 x float>
+define amdgpu_ps <2 x float> @extract_elt0_elt1_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
+  %data = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 0, i32 1>
+  ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt1_elt2_tbuffer_load_v4f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+; CHECK-NEXT: %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 1, i32 2>
+; CHECK-NEXT: ret <2 x float> %shuf
+define amdgpu_ps <2 x float> @extract_elt1_elt2_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
+  %data = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 1, i32 2>
+  ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt2_elt3_tbuffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+; CHECK-NEXT: %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT: ret <2 x float> %shuf
+define amdgpu_ps <2 x float> @extract_elt2_elt3_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
+  %data = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 2, i32 3>
+  ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_elt2_tbuffer_load_v4f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+; CHECK-NEXT: ret <3 x float> %data
+define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
+  %data = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 1, i32 2>
+  ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt1_elt2_elt3_tbuffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+; CHECK-NEXT: %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 1, i32 2, i32 3>
+; CHECK-NEXT: ret <3 x float> %shuf
+define amdgpu_ps <3 x float> @extract_elt1_elt2_elt3_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
+  %data = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 1, i32 2, i32 3>
+  ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_elt2_elt3_tbuffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+; CHECK-NEXT: %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 2, i32 3>
+; CHECK-NEXT: ret <3 x float> %shuf
+define amdgpu_ps <3 x float> @extract_elt0_elt2_elt3_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
+  %data = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 2, i32 3>
+  ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_tbuffer_load_v3f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.tbuffer.load.f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_tbuffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
+  %data = call <3 x float> @llvm.amdgcn.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+  %elt0 = extractelement <3 x float> %data, i32 0
+  ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_tbuffer_load_v3f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+; CHECK-NEXT: %elt1 = extractelement <2 x float> %data, i32 1
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt1_tbuffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
+  %data = call <3 x float> @llvm.amdgcn.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+  %elt1 = extractelement <3 x float> %data, i32 1
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt2_tbuffer_load_v3f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+; CHECK-NEXT: %elt1 = extractelement <3 x float> %data, i32 2
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt2_tbuffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
+  %data = call <3 x float> @llvm.amdgcn.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+  %elt1 = extractelement <3 x float> %data, i32 2
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_tbuffer_load_v3f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+; CHECK-NEXT: ret <2 x float>
+define amdgpu_ps <2 x float> @extract_elt0_elt1_tbuffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
+  %data = call <3 x float> @llvm.amdgcn.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+  %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 0, i32 1>
+  ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt1_elt2_tbuffer_load_v3f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+; CHECK-NEXT: %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 1, i32 2>
+; CHECK-NEXT: ret <2 x float> %shuf
+define amdgpu_ps <2 x float> @extract_elt1_elt2_tbuffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
+  %data = call <3 x float> @llvm.amdgcn.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+  %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 1, i32 2>
+  ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract0_bitcast_tbuffer_load_v4f32(
+; CHECK-NEXT: %tmp = call float @llvm.amdgcn.tbuffer.load.f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+; CHECK-NEXT: %tmp2 = bitcast float %tmp to i32
+; CHECK-NEXT: ret i32 %tmp2
+define i32 @extract0_bitcast_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
+  %tmp = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+  %tmp1 = bitcast <4 x float> %tmp to <4 x i32>
+  %tmp2 = extractelement <4 x i32> %tmp1, i32 0
+  ret i32 %tmp2
+}
+
+; CHECK-LABEL: @preserve_metadata_extract_elt0_tbuffer_load_v2f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.tbuffer.load.f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false), !fpmath !0
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @preserve_metadata_extract_elt0_tbuffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
+  %data = call <2 x float> @llvm.amdgcn.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false), !fpmath !0
+  %elt0 = extractelement <2 x float> %data, i32 0
+  ret float %elt0
+}
+
+declare float @llvm.amdgcn.tbuffer.load.f32(<4 x i32>, i32, i32, i32, i32, i32, i32, i1, i1) #1
+declare <1 x float> @llvm.amdgcn.tbuffer.load.v1f32(<4 x i32>, i32, i32, i32, i32, i32, i32, i1, i1) #1
+declare <2 x float> @llvm.amdgcn.tbuffer.load.v2f32(<4 x i32>, i32, i32, i32, i32, i32, i32, i1, i1) #1
+declare <3 x float> @llvm.amdgcn.tbuffer.load.v3f32(<4 x i32>, i32, i32, i32, i32, i32, i32, i1, i1) #1
+declare <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32>, i32, i32, i32, i32, i32, i32, i1, i1) #1
+
+declare <4 x i32> @llvm.amdgcn.tbuffer.load.v4i32(<4 x i32>, i32, i32, i32, i32, i32, i32, i1, i1) #1
+
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample
 ; --------------------------------------------------------------------
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-intrinsics.ll b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-intrinsics.ll
index 4406269e0..7c11a748a 100644
--- a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-intrinsics.ll
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-intrinsics.ll
@@ -50,7 +50,7 @@ define double @test_constant_fold_rcp_f64_half() nounwind {
 
 define float @test_constant_fold_rcp_f32_43() nounwind {
 ; CHECK-LABEL: @test_constant_fold_rcp_f32_43(
-; CHECK-NEXT:    [[VAL:%.*]] = call float @llvm.amdgcn.rcp.f32(float 4.300000e+01) #5
+; CHECK-NEXT:    [[VAL:%.*]] = call float @llvm.amdgcn.rcp.f32(float 4.300000e+01)
 ; CHECK-NEXT:    ret float [[VAL]]
 ;
   %val = call float @llvm.amdgcn.rcp.f32(float 4.300000e+01) nounwind readnone
@@ -59,7 +59,7 @@ define float @test_constant_fold_rcp_f32_43() nounwind {
 
 define double @test_constant_fold_rcp_f64_43() nounwind {
 ; CHECK-LABEL: @test_constant_fold_rcp_f64_43(
-; CHECK-NEXT:    [[VAL:%.*]] = call double @llvm.amdgcn.rcp.f64(double 4.300000e+01) #5
+; CHECK-NEXT:    [[VAL:%.*]] = call double @llvm.amdgcn.rcp.f64(double 4.300000e+01)
 ; CHECK-NEXT:    ret double [[VAL]]
 ;
   %val = call double @llvm.amdgcn.rcp.f64(double 4.300000e+01) nounwind readnone
@@ -1655,7 +1655,7 @@ define i64 @icmp_constant_inputs_false() {
 
 define i64 @icmp_constant_inputs_true() {
 ; CHECK-LABEL: @icmp_constant_inputs_true(
-; CHECK-NEXT:    [[RESULT:%.*]] = call i64 @llvm.read_register.i64(metadata !0) #6
+; CHECK-NEXT:    [[RESULT:%.*]] = call i64 @llvm.read_register.i64(metadata !0) [[CONVERGENT:#[0-9]*]]
 ; CHECK-NEXT:    ret i64 [[RESULT]]
 ;
   %result = call i64 @llvm.amdgcn.icmp.i64.i32(i32 9, i32 8, i32 34)
@@ -2362,7 +2362,7 @@ define i64 @fcmp_constant_inputs_false() {
 
 define i64 @fcmp_constant_inputs_true() {
 ; CHECK-LABEL: @fcmp_constant_inputs_true(
-; CHECK-NEXT:    [[RESULT:%.*]] = call i64 @llvm.read_register.i64(metadata !0) #6
+; CHECK-NEXT:    [[RESULT:%.*]] = call i64 @llvm.read_register.i64(metadata !0) [[CONVERGENT]]
 ; CHECK-NEXT:    ret i64 [[RESULT]]
 ;
   %result = call i64 @llvm.amdgcn.fcmp.i64.f32(float 2.0, float 4.0, i32 4)
@@ -2655,5 +2655,5 @@ define amdgpu_kernel void @update_dpp_undef_old(i32 addrspace(1)* %out, i32 %in1
   ret void
 }
 
-; CHECK: attributes #6 = { convergent }
+; CHECK: attributes [[CONVERGENT]] = { convergent }
 
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/tan.ll b/llvm/test/Transforms/InstCombine/AMDGPU/tan.ll
new file mode 100644
index 000000000..4c3a9f1fb
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/tan.ll
@@ -0,0 +1,21 @@
+; RUN: opt -mtriple=amdgcn--amdpal -S -instcombine <%s | FileCheck --check-prefixes=GCN %s
+
+; Check that sin/cos is not folded to tan on amdgcn.
+
+; GCN-LABEL: define amdgpu_ps float @llpc.shader.FS.main
+; GCN: call float @llvm.sin.f32
+; GCN: call float @llvm.cos.f32
+
+declare float @llvm.sin.f32(float) #0
+declare float @llvm.cos.f32(float) #0
+
+define amdgpu_ps float @llpc.shader.FS.main(float %arg) {
+.entry:
+  %tmp32 = call float @llvm.sin.f32(float %arg)
+  %tmp33 = call float @llvm.cos.f32(float %arg)
+  %tmp34 = fdiv reassoc nnan nsz arcp contract float 1.000000e+00, %tmp33
+  %tmp35 = fmul reassoc nnan nsz arcp contract float %tmp32, %tmp34
+  ret float %tmp35
+}
+
+attributes #0 = { nounwind readnone speculatable willreturn }
diff --git a/llvm/test/Transforms/InstCombine/ARM/mve-v2i2v.ll b/llvm/test/Transforms/InstCombine/ARM/mve-v2i2v.ll
new file mode 100644
index 000000000..7c511135b
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/ARM/mve-v2i2v.ll
@@ -0,0 +1,330 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -instcombine -S -o - %s | FileCheck %s
+
+target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
+
+declare i32 @llvm.arm.mve.pred.v2i.v4i1(<4 x i1>)
+declare i32 @llvm.arm.mve.pred.v2i.v8i1(<8 x i1>)
+declare i32 @llvm.arm.mve.pred.v2i.v16i1(<16 x i1>)
+
+declare <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32)
+declare <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32)
+declare <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32)
+
+; Round-trip conversions from predicate vector to i32 back to the same
+; size of vector should be eliminated.
+
+define <4 x i1> @v2i2v_4(<4 x i1> %vin) {
+; CHECK-LABEL: @v2i2v_4(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <4 x i1> [[VIN:%.*]]
+;
+entry:
+  %int = call i32 @llvm.arm.mve.pred.v2i.v4i1(<4 x i1> %vin)
+  %vout = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %int)
+  ret <4 x i1> %vout
+}
+
+define <8 x i1> @v2i2v_8(<8 x i1> %vin) {
+; CHECK-LABEL: @v2i2v_8(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <8 x i1> [[VIN:%.*]]
+;
+entry:
+  %int = call i32 @llvm.arm.mve.pred.v2i.v8i1(<8 x i1> %vin)
+  %vout = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %int)
+  ret <8 x i1> %vout
+}
+
+define <16 x i1> @v2i2v_16(<16 x i1> %vin) {
+; CHECK-LABEL: @v2i2v_16(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <16 x i1> [[VIN:%.*]]
+;
+entry:
+  %int = call i32 @llvm.arm.mve.pred.v2i.v16i1(<16 x i1> %vin)
+  %vout = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %int)
+  ret <16 x i1> %vout
+}
+
+; Conversions from a predicate vector to i32 and then to a _different_
+; size of predicate vector should be left alone.
+
+define <16 x i1> @v2i2v_4_16(<4 x i1> %vin) {
+; CHECK-LABEL: @v2i2v_4_16(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[INT:%.*]] = call i32 @llvm.arm.mve.pred.v2i.v4i1(<4 x i1> [[VIN:%.*]]), !range !0
+; CHECK-NEXT:    [[VOUT:%.*]] = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 [[INT]])
+; CHECK-NEXT:    ret <16 x i1> [[VOUT]]
+;
+entry:
+  %int = call i32 @llvm.arm.mve.pred.v2i.v4i1(<4 x i1> %vin)
+  %vout = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %int)
+  ret <16 x i1> %vout
+}
+
+define <4 x i1> @v2i2v_8_4(<8 x i1> %vin) {
+; CHECK-LABEL: @v2i2v_8_4(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[INT:%.*]] = call i32 @llvm.arm.mve.pred.v2i.v8i1(<8 x i1> [[VIN:%.*]]), !range !0
+; CHECK-NEXT:    [[VOUT:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[INT]])
+; CHECK-NEXT:    ret <4 x i1> [[VOUT]]
+;
+entry:
+  %int = call i32 @llvm.arm.mve.pred.v2i.v8i1(<8 x i1> %vin)
+  %vout = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %int)
+  ret <4 x i1> %vout
+}
+
+define <8 x i1> @v2i2v_16_8(<16 x i1> %vin) {
+; CHECK-LABEL: @v2i2v_16_8(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[INT:%.*]] = call i32 @llvm.arm.mve.pred.v2i.v16i1(<16 x i1> [[VIN:%.*]]), !range !0
+; CHECK-NEXT:    [[VOUT:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[INT]])
+; CHECK-NEXT:    ret <8 x i1> [[VOUT]]
+;
+entry:
+  %int = call i32 @llvm.arm.mve.pred.v2i.v16i1(<16 x i1> %vin)
+  %vout = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %int)
+  ret <8 x i1> %vout
+}
+
+; Round-trip conversions from i32 to predicate vector back to i32
+; should be eliminated.
+
+define i32 @i2v2i_4(i32 %iin) {
+; CHECK-LABEL: @i2v2i_4(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret i32 [[IIN:%.*]]
+;
+entry:
+  %vec = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %iin)
+  %iout = call i32 @llvm.arm.mve.pred.v2i.v4i1(<4 x i1> %vec)
+  ret i32 %iout
+}
+
+define i32 @i2v2i_8(i32 %iin) {
+; CHECK-LABEL: @i2v2i_8(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret i32 [[IIN:%.*]]
+;
+entry:
+  %vec = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %iin)
+  %iout = call i32 @llvm.arm.mve.pred.v2i.v8i1(<8 x i1> %vec)
+  ret i32 %iout
+}
+
+define i32 @i2v2i_16(i32 %iin) {
+; CHECK-LABEL: @i2v2i_16(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret i32 [[IIN:%.*]]
+;
+entry:
+  %vec = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %iin)
+  %iout = call i32 @llvm.arm.mve.pred.v2i.v16i1(<16 x i1> %vec)
+  ret i32 %iout
+}
+
+; v2i leaves the top 16 bits clear. So a trunc/zext pair applied to
+; its output, going via i16, can be completely eliminated - but not
+; one going via i8. Similarly with other methods of clearing the top
+; bits, like bitwise and.
+
+define i32 @v2i_truncext_i16(<4 x i1> %vin) {
+; CHECK-LABEL: @v2i_truncext_i16(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[WIDE1:%.*]] = call i32 @llvm.arm.mve.pred.v2i.v4i1(<4 x i1> [[VIN:%.*]]), !range !0
+; CHECK-NEXT:    ret i32 [[WIDE1]]
+;
+entry:
+  %wide1 = call i32 @llvm.arm.mve.pred.v2i.v4i1(<4 x i1> %vin)
+  %narrow = trunc i32 %wide1 to i16
+  %wide2 = zext i16 %narrow to i32
+  ret i32 %wide2
+}
+
+define i32 @v2i_truncext_i8(<4 x i1> %vin) {
+; CHECK-LABEL: @v2i_truncext_i8(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[WIDE1:%.*]] = call i32 @llvm.arm.mve.pred.v2i.v4i1(<4 x i1> [[VIN:%.*]]), !range !0
+; CHECK-NEXT:    [[WIDE2:%.*]] = and i32 [[WIDE1]], 255
+; CHECK-NEXT:    ret i32 [[WIDE2]]
+;
+entry:
+  %wide1 = call i32 @llvm.arm.mve.pred.v2i.v4i1(<4 x i1> %vin)
+  %narrow = trunc i32 %wide1 to i8
+  %wide2 = zext i8 %narrow to i32
+  ret i32 %wide2
+}
+
+define i32 @v2i_and_16(<4 x i1> %vin) {
+; CHECK-LABEL: @v2i_and_16(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[WIDE1:%.*]] = call i32 @llvm.arm.mve.pred.v2i.v4i1(<4 x i1> [[VIN:%.*]]), !range !0
+; CHECK-NEXT:    ret i32 [[WIDE1]]
+;
+entry:
+  %wide1 = call i32 @llvm.arm.mve.pred.v2i.v4i1(<4 x i1> %vin)
+  %wide2 = and i32 %wide1, 65535
+  ret i32 %wide2
+}
+
+define i32 @v2i_and_15(<4 x i1> %vin) {
+; CHECK-LABEL: @v2i_and_15(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[WIDE1:%.*]] = call i32 @llvm.arm.mve.pred.v2i.v4i1(<4 x i1> [[VIN:%.*]]), !range !0
+; CHECK-NEXT:    [[WIDE2:%.*]] = and i32 [[WIDE1]], 32767
+; CHECK-NEXT:    ret i32 [[WIDE2]]
+;
+entry:
+  %wide1 = call i32 @llvm.arm.mve.pred.v2i.v4i1(<4 x i1> %vin)
+  %wide2 = and i32 %wide1, 32767
+  ret i32 %wide2
+}
+
+; i2v doesn't use the top bits of its input. So the same operations
+; on a value that's about to be passed to i2v can be eliminated.
+
+define <4 x i1> @i2v_truncext_i16(i32 %wide1) {
+; CHECK-LABEL: @i2v_truncext_i16(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[VOUT:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[WIDE1:%.*]])
+; CHECK-NEXT:    ret <4 x i1> [[VOUT]]
+;
+entry:
+  %narrow = trunc i32 %wide1 to i16
+  %wide2 = zext i16 %narrow to i32
+  %vout = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %wide2)
+  ret <4 x i1> %vout
+}
+
+define <4 x i1> @i2v_truncext_i8(i32 %wide1) {
+; CHECK-LABEL: @i2v_truncext_i8(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[WIDE2:%.*]] = and i32 [[WIDE1:%.*]], 255
+; CHECK-NEXT:    [[VOUT:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[WIDE2]])
+; CHECK-NEXT:    ret <4 x i1> [[VOUT]]
+;
+entry:
+  %narrow = trunc i32 %wide1 to i8
+  %wide2 = zext i8 %narrow to i32
+  %vout = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %wide2)
+  ret <4 x i1> %vout
+}
+
+define <4 x i1> @i2v_and_16(i32 %wide1) {
+; CHECK-LABEL: @i2v_and_16(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[VOUT:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[WIDE1:%.*]])
+; CHECK-NEXT:    ret <4 x i1> [[VOUT]]
+;
+entry:
+  %wide2 = and i32 %wide1, 65535
+  %vout = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %wide2)
+  ret <4 x i1> %vout
+}
+
+define <4 x i1> @i2v_and_15(i32 %wide1) {
+; CHECK-LABEL: @i2v_and_15(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[WIDE2:%.*]] = and i32 [[WIDE1:%.*]], 32767
+; CHECK-NEXT:    [[VOUT:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[WIDE2]])
+; CHECK-NEXT:    ret <4 x i1> [[VOUT]]
+;
+entry:
+  %wide2 = and i32 %wide1, 32767
+  %vout = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %wide2)
+  ret <4 x i1> %vout
+}
+
+; If a predicate vector is round-tripped to an integer and back, and
+; complemented while it's in integer form, we should collapse that to
+; a complement of the vector itself. (Rationale: this is likely to
+; allow it to be code-generated as MVE VPNOT.)
+
+define <4 x i1> @vpnot_4(<4 x i1> %vin) {
+; CHECK-LABEL: @vpnot_4(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[VOUT:%.*]] = xor <4 x i1> [[VIN:%.*]], <i1 true, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    ret <4 x i1> [[VOUT]]
+;
+entry:
+  %int = call i32 @llvm.arm.mve.pred.v2i.v4i1(<4 x i1> %vin)
+  %flipped = xor i32 %int, 65535
+  %vout = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %flipped)
+  ret <4 x i1> %vout
+}
+
+define <8 x i1> @vpnot_8(<8 x i1> %vin) {
+; CHECK-LABEL: @vpnot_8(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[VOUT:%.*]] = xor <8 x i1> [[VIN:%.*]], <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    ret <8 x i1> [[VOUT]]
+;
+entry:
+  %int = call i32 @llvm.arm.mve.pred.v2i.v8i1(<8 x i1> %vin)
+  %flipped = xor i32 %int, 65535
+  %vout = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %flipped)
+  ret <8 x i1> %vout
+}
+
+define <16 x i1> @vpnot_16(<16 x i1> %vin) {
+; CHECK-LABEL: @vpnot_16(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[VOUT:%.*]] = xor <16 x i1> [[VIN:%.*]], <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    ret <16 x i1> [[VOUT]]
+;
+entry:
+  %int = call i32 @llvm.arm.mve.pred.v2i.v16i1(<16 x i1> %vin)
+  %flipped = xor i32 %int, 65535
+  %vout = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %flipped)
+  ret <16 x i1> %vout
+}
+
+; And this still works even if the i32 is narrowed to i16 and back on
+; opposite sides of the xor.
+
+define <4 x i1> @vpnot_narrow_4(<4 x i1> %vin) {
+; CHECK-LABEL: @vpnot_narrow_4(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[VOUT:%.*]] = xor <4 x i1> [[VIN:%.*]], <i1 true, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    ret <4 x i1> [[VOUT]]
+;
+entry:
+  %int = call i32 @llvm.arm.mve.pred.v2i.v4i1(<4 x i1> %vin)
+  %narrow = trunc i32 %int to i16
+  %flipped_narrow = xor i16 %narrow, -1
+  %flipped = zext i16 %flipped_narrow to i32
+  %vout = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %flipped)
+  ret <4 x i1> %vout
+}
+
+define <8 x i1> @vpnot_narrow_8(<8 x i1> %vin) {
+; CHECK-LABEL: @vpnot_narrow_8(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[VOUT:%.*]] = xor <8 x i1> [[VIN:%.*]], <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    ret <8 x i1> [[VOUT]]
+;
+entry:
+  %int = call i32 @llvm.arm.mve.pred.v2i.v8i1(<8 x i1> %vin)
+  %narrow = trunc i32 %int to i16
+  %flipped_narrow = xor i16 %narrow, -1
+  %flipped = zext i16 %flipped_narrow to i32
+  %vout = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %flipped)
+  ret <8 x i1> %vout
+}
+
+define <16 x i1> @vpnot_narrow_16(<16 x i1> %vin) {
+; CHECK-LABEL: @vpnot_narrow_16(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[VOUT:%.*]] = xor <16 x i1> [[VIN:%.*]], <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    ret <16 x i1> [[VOUT]]
+;
+entry:
+  %int = call i32 @llvm.arm.mve.pred.v2i.v16i1(<16 x i1> %vin)
+  %narrow = trunc i32 %int to i16
+  %flipped_narrow = xor i16 %narrow, -1
+  %flipped = zext i16 %flipped_narrow to i32
+  %vout = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %flipped)
+  ret <16 x i1> %vout
+}
diff --git a/llvm/test/Transforms/InstCombine/ARM/strcmp.ll b/llvm/test/Transforms/InstCombine/ARM/strcmp.ll
index 571a0f950..a7f3d21a0 100644
--- a/llvm/test/Transforms/InstCombine/ARM/strcmp.ll
+++ b/llvm/test/Transforms/InstCombine/ARM/strcmp.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; Test that the strcmp library call simplifier works correctly.
 ; RUN: opt < %s -instcombine -S | FileCheck %s
 
@@ -13,10 +14,11 @@ declare i32 @strcmp(i8*, i8*)
 ; strcmp("", x) -> -*x
 define arm_aapcscc i32 @test1(i8* %str2) {
 ; CHECK-LABEL: @test1(
-; CHECK: %strcmpload = load i8, i8* %str
-; CHECK: %1 = zext i8 %strcmpload to i32
-; CHECK: %2 = sub nsw i32 0, %1
-; CHECK: ret i32 %2
+; CHECK-NEXT:    [[STRCMPLOAD:%.*]] = load i8, i8* [[STR2:%.*]], align 1
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[STRCMPLOAD]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = sub nsw i32 0, [[TMP1]]
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
 
   %str1 = getelementptr inbounds [1 x i8], [1 x i8]* @null, i32 0, i32 0
   %temp1 = call arm_apcscc i32 @strcmp(i8* %str1, i8* %str2)
@@ -27,9 +29,10 @@ define arm_aapcscc i32 @test1(i8* %str2) {
 ; strcmp(x, "") -> *x
 define arm_aapcscc i32 @test2(i8* %str1) {
 ; CHECK-LABEL: @test2(
-; CHECK: %strcmpload = load i8, i8* %str
-; CHECK: %1 = zext i8 %strcmpload to i32
-; CHECK: ret i32 %1
+; CHECK-NEXT:    [[STRCMPLOAD:%.*]] = load i8, i8* [[STR1:%.*]], align 1
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[STRCMPLOAD]] to i32
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
 
   %str2 = getelementptr inbounds [1 x i8], [1 x i8]* @null, i32 0, i32 0
   %temp1 = call arm_aapcscc i32 @strcmp(i8* %str1, i8* %str2)
@@ -39,7 +42,8 @@ define arm_aapcscc i32 @test2(i8* %str1) {
 ; strcmp(x, y)  -> cnst
 define arm_aapcscc i32 @test3() {
 ; CHECK-LABEL: @test3(
-; CHECK: ret i32 -1
+; CHECK-NEXT:    ret i32 -1
+;
 
   %str1 = getelementptr inbounds [5 x i8], [5 x i8]* @hell, i32 0, i32 0
   %str2 = getelementptr inbounds [6 x i8], [6 x i8]* @hello, i32 0, i32 0
@@ -49,7 +53,8 @@ define arm_aapcscc i32 @test3() {
 
 define arm_aapcscc i32 @test4() {
 ; CHECK-LABEL: @test4(
-; CHECK: ret i32 1
+; CHECK-NEXT:    ret i32 1
+;
 
   %str1 = getelementptr inbounds [5 x i8], [5 x i8]* @hell, i32 0, i32 0
   %str2 = getelementptr inbounds [1 x i8], [1 x i8]* @null, i32 0, i32 0
@@ -61,8 +66,10 @@ define arm_aapcscc i32 @test4() {
 ; (This transform is rather difficult to trigger in a useful manner)
 define arm_aapcscc i32 @test5(i1 %b) {
 ; CHECK-LABEL: @test5(
-; CHECK: %memcmp = call i32 @memcmp(i8* getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i32 0, i32 0), i8* %str2, i32 5)
-; CHECK: ret i32 %memcmp
+; CHECK-NEXT:    [[STR2:%.*]] = select i1 [[B:%.*]], i8* getelementptr inbounds ([5 x i8], [5 x i8]* @hell, i32 0, i32 0), i8* getelementptr inbounds ([5 x i8], [5 x i8]* @bell, i32 0, i32 0)
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(5) getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i32 0, i32 0), i8* nonnull dereferenceable(5) [[STR2]], i32 5)
+; CHECK-NEXT:    ret i32 [[MEMCMP]]
+;
 
   %str1 = getelementptr inbounds [6 x i8], [6 x i8]* @hello, i32 0, i32 0
   %temp1 = getelementptr inbounds [5 x i8], [5 x i8]* @hell, i32 0, i32 0
@@ -75,7 +82,8 @@ define arm_aapcscc i32 @test5(i1 %b) {
 ; strcmp(x,x)  -> 0
 define arm_aapcscc i32 @test6(i8* %str) {
 ; CHECK-LABEL: @test6(
-; CHECK: ret i32 0
+; CHECK-NEXT:    ret i32 0
+;
 
   %temp1 = call arm_aapcscc i32 @strcmp(i8* %str, i8* %str)
   ret i32 %temp1
@@ -84,10 +92,11 @@ define arm_aapcscc i32 @test6(i8* %str) {
 ; strcmp("", x) -> -*x
 define arm_aapcs_vfpcc i32 @test1_vfp(i8* %str2) {
 ; CHECK-LABEL: @test1_vfp(
-; CHECK: %strcmpload = load i8, i8* %str
-; CHECK: %1 = zext i8 %strcmpload to i32
-; CHECK: %2 = sub nsw i32 0, %1
-; CHECK: ret i32 %2
+; CHECK-NEXT:    [[STRCMPLOAD:%.*]] = load i8, i8* [[STR2:%.*]], align 1
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[STRCMPLOAD]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = sub nsw i32 0, [[TMP1]]
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
 
   %str1 = getelementptr inbounds [1 x i8], [1 x i8]* @null, i32 0, i32 0
   %temp1 = call arm_aapcs_vfpcc i32 @strcmp(i8* %str1, i8* %str2)
@@ -98,9 +107,10 @@ define arm_aapcs_vfpcc i32 @test1_vfp(i8* %str2) {
 ; strcmp(x, "") -> *x
 define arm_aapcs_vfpcc i32 @test2_vfp(i8* %str1) {
 ; CHECK-LABEL: @test2_vfp(
-; CHECK: %strcmpload = load i8, i8* %str
-; CHECK: %1 = zext i8 %strcmpload to i32
-; CHECK: ret i32 %1
+; CHECK-NEXT:    [[STRCMPLOAD:%.*]] = load i8, i8* [[STR1:%.*]], align 1
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[STRCMPLOAD]] to i32
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
 
   %str2 = getelementptr inbounds [1 x i8], [1 x i8]* @null, i32 0, i32 0
   %temp1 = call arm_aapcs_vfpcc i32 @strcmp(i8* %str1, i8* %str2)
@@ -110,7 +120,8 @@ define arm_aapcs_vfpcc i32 @test2_vfp(i8* %str1) {
 ; strcmp(x, y)  -> cnst
 define arm_aapcs_vfpcc i32 @test3_vfp() {
 ; CHECK-LABEL: @test3_vfp(
-; CHECK: ret i32 -1
+; CHECK-NEXT:    ret i32 -1
+;
 
   %str1 = getelementptr inbounds [5 x i8], [5 x i8]* @hell, i32 0, i32 0
   %str2 = getelementptr inbounds [6 x i8], [6 x i8]* @hello, i32 0, i32 0
@@ -120,7 +131,8 @@ define arm_aapcs_vfpcc i32 @test3_vfp() {
 
 define arm_aapcs_vfpcc i32 @test4_vfp() {
 ; CHECK-LABEL: @test4_vfp(
-; CHECK: ret i32 1
+; CHECK-NEXT:    ret i32 1
+;
 
   %str1 = getelementptr inbounds [5 x i8], [5 x i8]* @hell, i32 0, i32 0
   %str2 = getelementptr inbounds [1 x i8], [1 x i8]* @null, i32 0, i32 0
@@ -132,8 +144,10 @@ define arm_aapcs_vfpcc i32 @test4_vfp() {
 ; (This transform is rather difficult to trigger in a useful manner)
 define arm_aapcs_vfpcc i32 @test5_vfp(i1 %b) {
 ; CHECK-LABEL: @test5_vfp(
-; CHECK: %memcmp = call i32 @memcmp(i8* getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i32 0, i32 0), i8* %str2, i32 5)
-; CHECK: ret i32 %memcmp
+; CHECK-NEXT:    [[STR2:%.*]] = select i1 [[B:%.*]], i8* getelementptr inbounds ([5 x i8], [5 x i8]* @hell, i32 0, i32 0), i8* getelementptr inbounds ([5 x i8], [5 x i8]* @bell, i32 0, i32 0)
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(5) getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i32 0, i32 0), i8* nonnull dereferenceable(5) [[STR2]], i32 5)
+; CHECK-NEXT:    ret i32 [[MEMCMP]]
+;
 
   %str1 = getelementptr inbounds [6 x i8], [6 x i8]* @hello, i32 0, i32 0
   %temp1 = getelementptr inbounds [5 x i8], [5 x i8]* @hell, i32 0, i32 0
@@ -146,7 +160,8 @@ define arm_aapcs_vfpcc i32 @test5_vfp(i1 %b) {
 ; strcmp(x,x)  -> 0
 define arm_aapcs_vfpcc i32 @test6_vfp(i8* %str) {
 ; CHECK-LABEL: @test6_vfp(
-; CHECK: ret i32 0
+; CHECK-NEXT:    ret i32 0
+;
 
   %temp1 = call arm_aapcs_vfpcc i32 @strcmp(i8* %str, i8* %str)
   ret i32 %temp1
diff --git a/llvm/test/Transforms/InstCombine/PR37526.ll b/llvm/test/Transforms/InstCombine/PR37526.ll
index 651c25ae0..25812c075 100644
--- a/llvm/test/Transforms/InstCombine/PR37526.ll
+++ b/llvm/test/Transforms/InstCombine/PR37526.ll
@@ -3,11 +3,14 @@
 
 define void @PR37526(i32* %pz, i32* %px, i32* %py) {
 ; CHECK-LABEL: @PR37526(
+; CHECK-NEXT:    [[T1:%.*]] = bitcast i32* [[PZ:%.*]] to i64*
 ; CHECK-NEXT:    [[T2:%.*]] = load i32, i32* [[PY:%.*]], align 4
 ; CHECK-NEXT:    [[T3:%.*]] = load i32, i32* [[PX:%.*]], align 4
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[T2]], [[T3]]
-; CHECK-NEXT:    [[R1:%.*]] = select i1 [[CMP]], i32 [[T3]], i32 [[T2]]
-; CHECK-NEXT:    store i32 [[R1]], i32* [[PZ:%.*]], align 4
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32* [[PX]], i32* [[PY]]
+; CHECK-NEXT:    [[BC:%.*]] = bitcast i32* [[SELECT]] to i64*
+; CHECK-NEXT:    [[R:%.*]] = load i64, i64* [[BC]], align 4
+; CHECK-NEXT:    store i64 [[R]], i64* [[T1]], align 4
 ; CHECK-NEXT:    ret void
 ;
   %t1 = bitcast i32* %pz to i64*
diff --git a/llvm/test/Transforms/InstCombine/X86/2009-03-23-i80-fp80.ll b/llvm/test/Transforms/InstCombine/X86/2009-03-23-i80-fp80.ll
new file mode 100644
index 000000000..f093c435a
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/X86/2009-03-23-i80-fp80.ll
@@ -0,0 +1,20 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+target triple = "i686-apple-darwin9"
+
+define i80 @from() {
+; CHECK-LABEL: @from(
+; CHECK-NEXT:    ret i80 302245289961712575840256
+;
+  %tmp = bitcast x86_fp80 0xK4000C000000000000000 to i80
+  ret i80 %tmp
+}
+
+define x86_fp80 @to() {
+; CHECK-LABEL: @to(
+; CHECK-NEXT:    ret x86_fp80 0xK40018000000000000000
+;
+  %tmp = bitcast i80 302259125019767858003968 to x86_fp80
+  ret x86_fp80 %tmp
+}
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-avx2.ll b/llvm/test/Transforms/InstCombine/X86/x86-avx2.ll
index f4045f788..9a6a0a5a4 100644
--- a/llvm/test/Transforms/InstCombine/X86/x86-avx2.ll
+++ b/llvm/test/Transforms/InstCombine/X86/x86-avx2.ll
@@ -6,7 +6,7 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
 define <8 x i32> @identity_test_vpermd(<8 x i32> %a0) {
 ; CHECK-LABEL: @identity_test_vpermd(
-; CHECK-NEXT:    ret <8 x i32> %a0
+; CHECK-NEXT:    ret <8 x i32> [[A0:%.*]]
 ;
   %a = tail call <8 x i32> @llvm.x86.avx2.permd(<8 x i32> %a0, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>)
   ret <8 x i32> %a
@@ -14,7 +14,7 @@ define <8 x i32> @identity_test_vpermd(<8 x i32> %a0) {
 
 define <8 x float> @identity_test_vpermps(<8 x float> %a0) {
 ; CHECK-LABEL: @identity_test_vpermps(
-; CHECK-NEXT:    ret <8 x float> %a0
+; CHECK-NEXT:    ret <8 x float> [[A0:%.*]]
 ;
   %a = tail call <8 x float> @llvm.x86.avx2.permps(<8 x float> %a0, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>)
   ret <8 x float> %a
@@ -25,7 +25,7 @@ define <8 x float> @identity_test_vpermps(<8 x float> %a0) {
 
 define <8 x i32> @zero_test_vpermd(<8 x i32> %a0) {
 ; CHECK-LABEL: @zero_test_vpermd(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> %a0, <8 x i32> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A0:%.*]], <8 x i32> undef, <8 x i32> zeroinitializer
 ; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
 ;
   %a = tail call <8 x i32> @llvm.x86.avx2.permd(<8 x i32> %a0, <8 x i32> zeroinitializer)
@@ -34,7 +34,7 @@ define <8 x i32> @zero_test_vpermd(<8 x i32> %a0) {
 
 define <8 x float> @zero_test_vpermps(<8 x float> %a0) {
 ; CHECK-LABEL: @zero_test_vpermps(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> %a0, <8 x float> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A0:%.*]], <8 x float> undef, <8 x i32> zeroinitializer
 ; CHECK-NEXT:    ret <8 x float> [[TMP1]]
 ;
   %a = tail call <8 x float> @llvm.x86.avx2.permps(<8 x float> %a0, <8 x i32> zeroinitializer)
@@ -45,7 +45,7 @@ define <8 x float> @zero_test_vpermps(<8 x float> %a0) {
 
 define <8 x i32> @shuffle_test_vpermd(<8 x i32> %a0) {
 ; CHECK-LABEL: @shuffle_test_vpermd(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> %a0, <8 x i32> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A0:%.*]], <8 x i32> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
 ;
   %a = tail call <8 x i32> @llvm.x86.avx2.permd(<8 x i32> %a0, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
@@ -54,7 +54,7 @@ define <8 x i32> @shuffle_test_vpermd(<8 x i32> %a0) {
 
 define <8 x float> @shuffle_test_vpermps(<8 x float> %a0) {
 ; CHECK-LABEL: @shuffle_test_vpermps(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> %a0, <8 x float> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A0:%.*]], <8 x float> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    ret <8 x float> [[TMP1]]
 ;
   %a = tail call <8 x float> @llvm.x86.avx2.permps(<8 x float> %a0, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
@@ -65,7 +65,7 @@ define <8 x float> @shuffle_test_vpermps(<8 x float> %a0) {
 
 define <8 x i32> @undef_test_vpermd(<8 x i32> %a0) {
 ; CHECK-LABEL: @undef_test_vpermd(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> %a0, <8 x i32> undef, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A0:%.*]], <8 x i32> undef, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
 ;
   %a = tail call <8 x i32> @llvm.x86.avx2.permd(<8 x i32> %a0, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
@@ -74,7 +74,7 @@ define <8 x i32> @undef_test_vpermd(<8 x i32> %a0) {
 
 define <8 x float> @undef_test_vpermps(<8 x float> %a0) {
 ; CHECK-LABEL: @undef_test_vpermps(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> %a0, <8 x float> undef, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A0:%.*]], <8 x float> undef, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    ret <8 x float> [[TMP1]]
 ;
   %a = tail call <8 x float> @llvm.x86.avx2.permps(<8 x float> %a0, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
@@ -85,7 +85,8 @@ define <8 x float> @undef_test_vpermps(<8 x float> %a0) {
 
 define <8 x i32> @elts_test_vpermd(<8 x i32> %a0, i32 %a1) {
 ; CHECK-LABEL: @elts_test_vpermd(
-; CHECK-NEXT:    ret <8 x i32> %a0
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A0:%.*]], <8 x i32> undef, <8 x i32> <i32 undef, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
 ;
   %1 = insertelement <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, i32 %a1, i32 0
   %2 = tail call <8 x i32> @llvm.x86.avx2.permd(<8 x i32> %a0, <8 x i32> %1)
@@ -95,7 +96,7 @@ define <8 x i32> @elts_test_vpermd(<8 x i32> %a0, i32 %a1) {
 
 define <8 x float> @elts_test_vpermps(<8 x float> %a0, <8 x i32> %a1) {
 ; CHECK-LABEL: @elts_test_vpermps(
-; CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x float> @llvm.x86.avx2.permps(<8 x float> %a0, <8 x i32> %a1)
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x float> @llvm.x86.avx2.permps(<8 x float> [[A0:%.*]], <8 x i32> [[A1:%.*]])
 ; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> undef, <8 x i32> zeroinitializer
 ; CHECK-NEXT:    ret <8 x float> [[TMP2]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-avx512.ll b/llvm/test/Transforms/InstCombine/X86/x86-avx512.ll
index 8491dec37..127260d0e 100644
--- a/llvm/test/Transforms/InstCombine/X86/x86-avx512.ll
+++ b/llvm/test/Transforms/InstCombine/X86/x86-avx512.ll
@@ -759,72 +759,72 @@ define i8 @test_cmp_sd(<2 x double> %a, <2 x double> %b, i8 %mask) {
 define i64 @test(float %f, double %d) {
 ; CHECK-LABEL: @test(
 ; CHECK-NEXT:    [[V03:%.*]] = insertelement <4 x float> undef, float [[F:%.*]], i32 0
-; CHECK-NEXT:    [[TMP0:%.*]] = tail call i32 @llvm.x86.avx512.vcvtss2si32(<4 x float> [[V03]], i32 4)
+; CHECK-NEXT:    [[T0:%.*]] = tail call i32 @llvm.x86.avx512.vcvtss2si32(<4 x float> [[V03]], i32 4)
 ; CHECK-NEXT:    [[V13:%.*]] = insertelement <4 x float> undef, float [[F]], i32 0
-; CHECK-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.x86.avx512.vcvtss2si64(<4 x float> [[V13]], i32 4)
+; CHECK-NEXT:    [[T1:%.*]] = tail call i64 @llvm.x86.avx512.vcvtss2si64(<4 x float> [[V13]], i32 4)
 ; CHECK-NEXT:    [[V23:%.*]] = insertelement <4 x float> undef, float [[F]], i32 0
-; CHECK-NEXT:    [[TMP2:%.*]] = tail call i32 @llvm.x86.avx512.cvttss2si(<4 x float> [[V23]], i32 4)
+; CHECK-NEXT:    [[T2:%.*]] = tail call i32 @llvm.x86.avx512.cvttss2si(<4 x float> [[V23]], i32 4)
 ; CHECK-NEXT:    [[V33:%.*]] = insertelement <4 x float> undef, float [[F]], i32 0
-; CHECK-NEXT:    [[TMP3:%.*]] = tail call i64 @llvm.x86.avx512.cvttss2si64(<4 x float> [[V33]], i32 4)
+; CHECK-NEXT:    [[T3:%.*]] = tail call i64 @llvm.x86.avx512.cvttss2si64(<4 x float> [[V33]], i32 4)
 ; CHECK-NEXT:    [[V41:%.*]] = insertelement <2 x double> undef, double [[D:%.*]], i32 0
-; CHECK-NEXT:    [[TMP4:%.*]] = tail call i32 @llvm.x86.avx512.vcvtsd2si32(<2 x double> [[V41]], i32 4)
+; CHECK-NEXT:    [[T4:%.*]] = tail call i32 @llvm.x86.avx512.vcvtsd2si32(<2 x double> [[V41]], i32 4)
 ; CHECK-NEXT:    [[V51:%.*]] = insertelement <2 x double> undef, double [[D]], i32 0
-; CHECK-NEXT:    [[TMP5:%.*]] = tail call i64 @llvm.x86.avx512.vcvtsd2si64(<2 x double> [[V51]], i32 4)
+; CHECK-NEXT:    [[T5:%.*]] = tail call i64 @llvm.x86.avx512.vcvtsd2si64(<2 x double> [[V51]], i32 4)
 ; CHECK-NEXT:    [[V61:%.*]] = insertelement <2 x double> undef, double [[D]], i32 0
-; CHECK-NEXT:    [[TMP6:%.*]] = tail call i32 @llvm.x86.avx512.cvttsd2si(<2 x double> [[V61]], i32 4)
+; CHECK-NEXT:    [[T6:%.*]] = tail call i32 @llvm.x86.avx512.cvttsd2si(<2 x double> [[V61]], i32 4)
 ; CHECK-NEXT:    [[V71:%.*]] = insertelement <2 x double> undef, double [[D]], i32 0
-; CHECK-NEXT:    [[TMP7:%.*]] = tail call i64 @llvm.x86.avx512.cvttsd2si64(<2 x double> [[V71]], i32 4)
-; CHECK-NEXT:    [[TMP8:%.*]] = add i32 [[TMP0]], [[TMP2]]
-; CHECK-NEXT:    [[TMP9:%.*]] = add i32 [[TMP4]], [[TMP6]]
-; CHECK-NEXT:    [[TMP10:%.*]] = add i32 [[TMP8]], [[TMP9]]
-; CHECK-NEXT:    [[TMP11:%.*]] = sext i32 [[TMP10]] to i64
-; CHECK-NEXT:    [[TMP12:%.*]] = add i64 [[TMP1]], [[TMP3]]
-; CHECK-NEXT:    [[TMP13:%.*]] = add i64 [[TMP5]], [[TMP7]]
-; CHECK-NEXT:    [[TMP14:%.*]] = add i64 [[TMP12]], [[TMP13]]
-; CHECK-NEXT:    [[TMP15:%.*]] = add i64 [[TMP14]], [[TMP11]]
-; CHECK-NEXT:    ret i64 [[TMP15]]
+; CHECK-NEXT:    [[T7:%.*]] = tail call i64 @llvm.x86.avx512.cvttsd2si64(<2 x double> [[V71]], i32 4)
+; CHECK-NEXT:    [[T8:%.*]] = add i32 [[T0]], [[T2]]
+; CHECK-NEXT:    [[T9:%.*]] = add i32 [[T4]], [[T6]]
+; CHECK-NEXT:    [[T10:%.*]] = add i32 [[T8]], [[T9]]
+; CHECK-NEXT:    [[T11:%.*]] = sext i32 [[T10]] to i64
+; CHECK-NEXT:    [[T12:%.*]] = add i64 [[T1]], [[T3]]
+; CHECK-NEXT:    [[T13:%.*]] = add i64 [[T5]], [[T7]]
+; CHECK-NEXT:    [[T14:%.*]] = add i64 [[T12]], [[T13]]
+; CHECK-NEXT:    [[T15:%.*]] = add i64 [[T14]], [[T11]]
+; CHECK-NEXT:    ret i64 [[T15]]
 ;
   %v00 = insertelement <4 x float> undef, float %f, i32 0
   %v01 = insertelement <4 x float> %v00, float 0.000000e+00, i32 1
   %v02 = insertelement <4 x float> %v01, float 0.000000e+00, i32 2
   %v03 = insertelement <4 x float> %v02, float 0.000000e+00, i32 3
-  %tmp0 = tail call i32 @llvm.x86.avx512.vcvtss2si32(<4 x float> %v03, i32 4)
+  %t0 = tail call i32 @llvm.x86.avx512.vcvtss2si32(<4 x float> %v03, i32 4)
   %v10 = insertelement <4 x float> undef, float %f, i32 0
   %v11 = insertelement <4 x float> %v10, float 0.000000e+00, i32 1
   %v12 = insertelement <4 x float> %v11, float 0.000000e+00, i32 2
   %v13 = insertelement <4 x float> %v12, float 0.000000e+00, i32 3
-  %tmp1 = tail call i64 @llvm.x86.avx512.vcvtss2si64(<4 x float> %v13, i32 4)
+  %t1 = tail call i64 @llvm.x86.avx512.vcvtss2si64(<4 x float> %v13, i32 4)
   %v20 = insertelement <4 x float> undef, float %f, i32 0
   %v21 = insertelement <4 x float> %v20, float 0.000000e+00, i32 1
   %v22 = insertelement <4 x float> %v21, float 0.000000e+00, i32 2
   %v23 = insertelement <4 x float> %v22, float 0.000000e+00, i32 3
-  %tmp2 = tail call i32 @llvm.x86.avx512.cvttss2si(<4 x float> %v23, i32 4)
+  %t2 = tail call i32 @llvm.x86.avx512.cvttss2si(<4 x float> %v23, i32 4)
   %v30 = insertelement <4 x float> undef, float %f, i32 0
   %v31 = insertelement <4 x float> %v30, float 0.000000e+00, i32 1
   %v32 = insertelement <4 x float> %v31, float 0.000000e+00, i32 2
   %v33 = insertelement <4 x float> %v32, float 0.000000e+00, i32 3
-  %tmp3 = tail call i64 @llvm.x86.avx512.cvttss2si64(<4 x float> %v33, i32 4)
+  %t3 = tail call i64 @llvm.x86.avx512.cvttss2si64(<4 x float> %v33, i32 4)
   %v40 = insertelement <2 x double> undef, double %d, i32 0
   %v41 = insertelement <2 x double> %v40, double 0.000000e+00, i32 1
-  %tmp4 = tail call i32 @llvm.x86.avx512.vcvtsd2si32(<2 x double> %v41, i32 4)
+  %t4 = tail call i32 @llvm.x86.avx512.vcvtsd2si32(<2 x double> %v41, i32 4)
   %v50 = insertelement <2 x double> undef, double %d, i32 0
   %v51 = insertelement <2 x double> %v50, double 0.000000e+00, i32 1
-  %tmp5 = tail call i64 @llvm.x86.avx512.vcvtsd2si64(<2 x double> %v51, i32 4)
+  %t5 = tail call i64 @llvm.x86.avx512.vcvtsd2si64(<2 x double> %v51, i32 4)
   %v60 = insertelement <2 x double> undef, double %d, i32 0
   %v61 = insertelement <2 x double> %v60, double 0.000000e+00, i32 1
-  %tmp6 = tail call i32 @llvm.x86.avx512.cvttsd2si(<2 x double> %v61, i32 4)
+  %t6 = tail call i32 @llvm.x86.avx512.cvttsd2si(<2 x double> %v61, i32 4)
   %v70 = insertelement <2 x double> undef, double %d, i32 0
   %v71 = insertelement <2 x double> %v70, double 0.000000e+00, i32 1
-  %tmp7 = tail call i64 @llvm.x86.avx512.cvttsd2si64(<2 x double> %v71, i32 4)
-  %tmp8 = add i32 %tmp0, %tmp2
-  %tmp9 = add i32 %tmp4, %tmp6
-  %tmp10 = add i32 %tmp8, %tmp9
-  %tmp11 = sext i32 %tmp10 to i64
-  %tmp12 = add i64 %tmp1, %tmp3
-  %tmp13 = add i64 %tmp5, %tmp7
-  %tmp14 = add i64 %tmp12, %tmp13
-  %tmp15 = add i64 %tmp11, %tmp14
-  ret i64 %tmp15
+  %t7 = tail call i64 @llvm.x86.avx512.cvttsd2si64(<2 x double> %v71, i32 4)
+  %t8 = add i32 %t0, %t2
+  %t9 = add i32 %t4, %t6
+  %t10 = add i32 %t8, %t9
+  %t11 = sext i32 %t10 to i64
+  %t12 = add i64 %t1, %t3
+  %t13 = add i64 %t5, %t7
+  %t14 = add i64 %t12, %t13
+  %t15 = add i64 %t11, %t14
+  ret i64 %t15
 }
 
 declare i32 @llvm.x86.avx512.vcvtss2si32(<4 x float>, i32)
@@ -839,72 +839,72 @@ declare i64 @llvm.x86.avx512.cvttsd2si64(<2 x double>, i32)
 define i64 @test2(float %f, double %d) {
 ; CHECK-LABEL: @test2(
 ; CHECK-NEXT:    [[V03:%.*]] = insertelement <4 x float> undef, float [[F:%.*]], i32 0
-; CHECK-NEXT:    [[TMP0:%.*]] = tail call i32 @llvm.x86.avx512.vcvtss2usi32(<4 x float> [[V03]], i32 4)
+; CHECK-NEXT:    [[T0:%.*]] = tail call i32 @llvm.x86.avx512.vcvtss2usi32(<4 x float> [[V03]], i32 4)
 ; CHECK-NEXT:    [[V13:%.*]] = insertelement <4 x float> undef, float [[F]], i32 0
-; CHECK-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.x86.avx512.vcvtss2usi64(<4 x float> [[V13]], i32 4)
+; CHECK-NEXT:    [[T1:%.*]] = tail call i64 @llvm.x86.avx512.vcvtss2usi64(<4 x float> [[V13]], i32 4)
 ; CHECK-NEXT:    [[V23:%.*]] = insertelement <4 x float> undef, float [[F]], i32 0
-; CHECK-NEXT:    [[TMP2:%.*]] = tail call i32 @llvm.x86.avx512.cvttss2usi(<4 x float> [[V23]], i32 4)
+; CHECK-NEXT:    [[T2:%.*]] = tail call i32 @llvm.x86.avx512.cvttss2usi(<4 x float> [[V23]], i32 4)
 ; CHECK-NEXT:    [[V33:%.*]] = insertelement <4 x float> undef, float [[F]], i32 0
-; CHECK-NEXT:    [[TMP3:%.*]] = tail call i64 @llvm.x86.avx512.cvttss2usi64(<4 x float> [[V33]], i32 4)
+; CHECK-NEXT:    [[T3:%.*]] = tail call i64 @llvm.x86.avx512.cvttss2usi64(<4 x float> [[V33]], i32 4)
 ; CHECK-NEXT:    [[V41:%.*]] = insertelement <2 x double> undef, double [[D:%.*]], i32 0
-; CHECK-NEXT:    [[TMP4:%.*]] = tail call i32 @llvm.x86.avx512.vcvtsd2usi32(<2 x double> [[V41]], i32 4)
+; CHECK-NEXT:    [[T4:%.*]] = tail call i32 @llvm.x86.avx512.vcvtsd2usi32(<2 x double> [[V41]], i32 4)
 ; CHECK-NEXT:    [[V51:%.*]] = insertelement <2 x double> undef, double [[D]], i32 0
-; CHECK-NEXT:    [[TMP5:%.*]] = tail call i64 @llvm.x86.avx512.vcvtsd2usi64(<2 x double> [[V51]], i32 4)
+; CHECK-NEXT:    [[T5:%.*]] = tail call i64 @llvm.x86.avx512.vcvtsd2usi64(<2 x double> [[V51]], i32 4)
 ; CHECK-NEXT:    [[V61:%.*]] = insertelement <2 x double> undef, double [[D]], i32 0
-; CHECK-NEXT:    [[TMP6:%.*]] = tail call i32 @llvm.x86.avx512.cvttsd2usi(<2 x double> [[V61]], i32 4)
+; CHECK-NEXT:    [[T6:%.*]] = tail call i32 @llvm.x86.avx512.cvttsd2usi(<2 x double> [[V61]], i32 4)
 ; CHECK-NEXT:    [[V71:%.*]] = insertelement <2 x double> undef, double [[D]], i32 0
-; CHECK-NEXT:    [[TMP7:%.*]] = tail call i64 @llvm.x86.avx512.cvttsd2usi64(<2 x double> [[V71]], i32 4)
-; CHECK-NEXT:    [[TMP8:%.*]] = add i32 [[TMP0]], [[TMP2]]
-; CHECK-NEXT:    [[TMP9:%.*]] = add i32 [[TMP4]], [[TMP6]]
-; CHECK-NEXT:    [[TMP10:%.*]] = add i32 [[TMP8]], [[TMP9]]
-; CHECK-NEXT:    [[TMP11:%.*]] = sext i32 [[TMP10]] to i64
-; CHECK-NEXT:    [[TMP12:%.*]] = add i64 [[TMP1]], [[TMP3]]
-; CHECK-NEXT:    [[TMP13:%.*]] = add i64 [[TMP5]], [[TMP7]]
-; CHECK-NEXT:    [[TMP14:%.*]] = add i64 [[TMP12]], [[TMP13]]
-; CHECK-NEXT:    [[TMP15:%.*]] = add i64 [[TMP14]], [[TMP11]]
-; CHECK-NEXT:    ret i64 [[TMP15]]
+; CHECK-NEXT:    [[T7:%.*]] = tail call i64 @llvm.x86.avx512.cvttsd2usi64(<2 x double> [[V71]], i32 4)
+; CHECK-NEXT:    [[T8:%.*]] = add i32 [[T0]], [[T2]]
+; CHECK-NEXT:    [[T9:%.*]] = add i32 [[T4]], [[T6]]
+; CHECK-NEXT:    [[T10:%.*]] = add i32 [[T8]], [[T9]]
+; CHECK-NEXT:    [[T11:%.*]] = sext i32 [[T10]] to i64
+; CHECK-NEXT:    [[T12:%.*]] = add i64 [[T1]], [[T3]]
+; CHECK-NEXT:    [[T13:%.*]] = add i64 [[T5]], [[T7]]
+; CHECK-NEXT:    [[T14:%.*]] = add i64 [[T12]], [[T13]]
+; CHECK-NEXT:    [[T15:%.*]] = add i64 [[T14]], [[T11]]
+; CHECK-NEXT:    ret i64 [[T15]]
 ;
   %v00 = insertelement <4 x float> undef, float %f, i32 0
   %v01 = insertelement <4 x float> %v00, float 0.000000e+00, i32 1
   %v02 = insertelement <4 x float> %v01, float 0.000000e+00, i32 2
   %v03 = insertelement <4 x float> %v02, float 0.000000e+00, i32 3
-  %tmp0 = tail call i32 @llvm.x86.avx512.vcvtss2usi32(<4 x float> %v03, i32 4)
+  %t0 = tail call i32 @llvm.x86.avx512.vcvtss2usi32(<4 x float> %v03, i32 4)
   %v10 = insertelement <4 x float> undef, float %f, i32 0
   %v11 = insertelement <4 x float> %v10, float 0.000000e+00, i32 1
   %v12 = insertelement <4 x float> %v11, float 0.000000e+00, i32 2
   %v13 = insertelement <4 x float> %v12, float 0.000000e+00, i32 3
-  %tmp1 = tail call i64 @llvm.x86.avx512.vcvtss2usi64(<4 x float> %v13, i32 4)
+  %t1 = tail call i64 @llvm.x86.avx512.vcvtss2usi64(<4 x float> %v13, i32 4)
   %v20 = insertelement <4 x float> undef, float %f, i32 0
   %v21 = insertelement <4 x float> %v20, float 0.000000e+00, i32 1
   %v22 = insertelement <4 x float> %v21, float 0.000000e+00, i32 2
   %v23 = insertelement <4 x float> %v22, float 0.000000e+00, i32 3
-  %tmp2 = tail call i32 @llvm.x86.avx512.cvttss2usi(<4 x float> %v23, i32 4)
+  %t2 = tail call i32 @llvm.x86.avx512.cvttss2usi(<4 x float> %v23, i32 4)
   %v30 = insertelement <4 x float> undef, float %f, i32 0
   %v31 = insertelement <4 x float> %v30, float 0.000000e+00, i32 1
   %v32 = insertelement <4 x float> %v31, float 0.000000e+00, i32 2
   %v33 = insertelement <4 x float> %v32, float 0.000000e+00, i32 3
-  %tmp3 = tail call i64 @llvm.x86.avx512.cvttss2usi64(<4 x float> %v33, i32 4)
+  %t3 = tail call i64 @llvm.x86.avx512.cvttss2usi64(<4 x float> %v33, i32 4)
   %v40 = insertelement <2 x double> undef, double %d, i32 0
   %v41 = insertelement <2 x double> %v40, double 0.000000e+00, i32 1
-  %tmp4 = tail call i32 @llvm.x86.avx512.vcvtsd2usi32(<2 x double> %v41, i32 4)
+  %t4 = tail call i32 @llvm.x86.avx512.vcvtsd2usi32(<2 x double> %v41, i32 4)
   %v50 = insertelement <2 x double> undef, double %d, i32 0
   %v51 = insertelement <2 x double> %v50, double 0.000000e+00, i32 1
-  %tmp5 = tail call i64 @llvm.x86.avx512.vcvtsd2usi64(<2 x double> %v51, i32 4)
+  %t5 = tail call i64 @llvm.x86.avx512.vcvtsd2usi64(<2 x double> %v51, i32 4)
   %v60 = insertelement <2 x double> undef, double %d, i32 0
   %v61 = insertelement <2 x double> %v60, double 0.000000e+00, i32 1
-  %tmp6 = tail call i32 @llvm.x86.avx512.cvttsd2usi(<2 x double> %v61, i32 4)
+  %t6 = tail call i32 @llvm.x86.avx512.cvttsd2usi(<2 x double> %v61, i32 4)
   %v70 = insertelement <2 x double> undef, double %d, i32 0
   %v71 = insertelement <2 x double> %v70, double 0.000000e+00, i32 1
-  %tmp7 = tail call i64 @llvm.x86.avx512.cvttsd2usi64(<2 x double> %v71, i32 4)
-  %tmp8 = add i32 %tmp0, %tmp2
-  %tmp9 = add i32 %tmp4, %tmp6
-  %tmp10 = add i32 %tmp8, %tmp9
-  %tmp11 = sext i32 %tmp10 to i64
-  %tmp12 = add i64 %tmp1, %tmp3
-  %tmp13 = add i64 %tmp5, %tmp7
-  %tmp14 = add i64 %tmp12, %tmp13
-  %tmp15 = add i64 %tmp11, %tmp14
-  ret i64 %tmp15
+  %t7 = tail call i64 @llvm.x86.avx512.cvttsd2usi64(<2 x double> %v71, i32 4)
+  %t8 = add i32 %t0, %t2
+  %t9 = add i32 %t4, %t6
+  %t10 = add i32 %t8, %t9
+  %t11 = sext i32 %t10 to i64
+  %t12 = add i64 %t1, %t3
+  %t13 = add i64 %t5, %t7
+  %t14 = add i64 %t12, %t13
+  %t15 = add i64 %t11, %t14
+  ret i64 %t15
 }
 
 declare i32 @llvm.x86.avx512.vcvtss2usi32(<4 x float>, i32)
@@ -921,8 +921,8 @@ declare float @llvm.fma.f32(float, float, float) #1
 define <4 x float> @test_mask_vfmadd_ss(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
 ; CHECK-LABEL: @test_mask_vfmadd_ss(
 ; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x float> [[A:%.*]], i64 0
-; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[B:%.*]], i32 0
-; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[C:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[B:%.*]], i64 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[C:%.*]], i64 0
 ; CHECK-NEXT:    [[TMP4:%.*]] = call float @llvm.fma.f32(float [[TMP1]], float [[TMP2]], float [[TMP3]])
 ; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <8 x i1> [[TMP5]], i64 0
@@ -1063,8 +1063,8 @@ define double @test_mask_vfmadd_sd_1(<2 x double> %a, <2 x double> %b, <2 x doub
 define <4 x float> @test_maskz_vfmadd_ss(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
 ; CHECK-LABEL: @test_maskz_vfmadd_ss(
 ; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x float> [[A:%.*]], i64 0
-; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[B:%.*]], i32 0
-; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[C:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[B:%.*]], i64 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[C:%.*]], i64 0
 ; CHECK-NEXT:    [[TMP4:%.*]] = call float @llvm.fma.f32(float [[TMP1]], float [[TMP2]], float [[TMP3]])
 ; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <8 x i1> [[TMP5]], i64 0
@@ -1202,8 +1202,8 @@ define double @test_maskz_vfmadd_sd_1(<2 x double> %a, <2 x double> %b, <2 x dou
 
 define <4 x float> @test_mask3_vfmadd_ss(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
 ; CHECK-LABEL: @test_mask3_vfmadd_ss(
-; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x float> [[A:%.*]], i32 0
-; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[B:%.*]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x float> [[A:%.*]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[B:%.*]], i64 0
 ; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[C:%.*]], i64 0
 ; CHECK-NEXT:    [[TMP4:%.*]] = call float @llvm.fma.f32(float [[TMP1]], float [[TMP2]], float [[TMP3]])
 ; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
@@ -1342,8 +1342,8 @@ define double @test_mask3_vfmadd_sd_1(<2 x double> %a, <2 x double> %b, <2 x dou
 
 define <4 x float> @test_mask3_vfmsub_ss(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
 ; CHECK-LABEL: @test_mask3_vfmsub_ss(
-; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x float> [[A:%.*]], i32 0
-; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[B:%.*]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x float> [[A:%.*]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[B:%.*]], i64 0
 ; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[C:%.*]], i64 0
 ; CHECK-NEXT:    [[TMP4:%.*]] = fsub float -0.000000e+00, [[TMP3]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = call float @llvm.fma.f32(float [[TMP1]], float [[TMP2]], float [[TMP4]])
@@ -1544,7 +1544,7 @@ define <4 x float> @test_mask3_vfnmsub_ss(<4 x float> %a, <4 x float> %b, <4 x f
 ; CHECK-LABEL: @test_mask3_vfnmsub_ss(
 ; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x float> [[A:%.*]], i64 0
 ; CHECK-NEXT:    [[TMP2:%.*]] = fsub float -0.000000e+00, [[TMP1]]
-; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[B:%.*]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[B:%.*]], i64 0
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[C:%.*]], i64 0
 ; CHECK-NEXT:    [[TMP5:%.*]] = fsub float -0.000000e+00, [[TMP4]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = call float @llvm.fma.f32(float [[TMP2]], float [[TMP3]], float [[TMP5]])
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-bmi-tbm.ll b/llvm/test/Transforms/InstCombine/X86/x86-bmi-tbm.ll
index 2b472cad2..669f81330 100644
--- a/llvm/test/Transforms/InstCombine/X86/x86-bmi-tbm.ll
+++ b/llvm/test/Transforms/InstCombine/X86/x86-bmi-tbm.ll
@@ -7,6 +7,10 @@ declare i32 @llvm.x86.bmi.bextr.32(i32, i32) nounwind readnone
 declare i64 @llvm.x86.bmi.bextr.64(i64, i64) nounwind readnone
 declare i32 @llvm.x86.bmi.bzhi.32(i32, i32) nounwind readnone
 declare i64 @llvm.x86.bmi.bzhi.64(i64, i64) nounwind readnone
+declare i32 @llvm.x86.bmi.pext.32(i32, i32) nounwind readnone
+declare i64 @llvm.x86.bmi.pext.64(i64, i64) nounwind readnone
+declare i32 @llvm.x86.bmi.pdep.32(i32, i32) nounwind readnone
+declare i64 @llvm.x86.bmi.pdep.64(i64, i64) nounwind readnone
 
 define i32 @test_x86_tbm_bextri_u32(i32 %a) nounwind readnone {
 ; CHECK-LABEL: @test_x86_tbm_bextri_u32(
@@ -269,3 +273,131 @@ define i64 @test_x86_bmi_bzhi_64_constfold() nounwind readnone {
   %1 = tail call i64 @llvm.x86.bmi.bzhi.64(i64 5, i64 1)
   ret i64 %1
 }
+
+define i32 @test_x86_pext_32_zero_mask(i32 %x) nounwind readnone {
+; CHECK-LABEL: @test_x86_pext_32_zero_mask(
+; CHECK-NEXT:    ret i32 0
+;
+  %1 = tail call i32 @llvm.x86.bmi.pext.32(i32 %x, i32 0)
+  ret i32 %1
+}
+
+define i64 @test_x86_pext_64_zero_mask(i64 %x) nounwind readnone {
+; CHECK-LABEL: @test_x86_pext_64_zero_mask(
+; CHECK-NEXT:    ret i64 0
+;
+  %1 = tail call i64 @llvm.x86.bmi.pext.64(i64 %x, i64 0)
+  ret i64 %1
+}
+
+define i32 @test_x86_pext_32_allones_mask(i32 %x) nounwind readnone {
+; CHECK-LABEL: @test_x86_pext_32_allones_mask(
+; CHECK-NEXT:    ret i32 %x
+;
+  %1 = tail call i32 @llvm.x86.bmi.pext.32(i32 %x, i32 -1)
+  ret i32 %1
+}
+
+define i64 @test_x86_pext_64_allones_mask(i64 %x) nounwind readnone {
+; CHECK-LABEL: @test_x86_pext_64_allones_mask(
+; CHECK-NEXT:    ret i64 %x
+;
+  %1 = tail call i64 @llvm.x86.bmi.pext.64(i64 %x, i64 -1)
+  ret i64 %1
+}
+
+define i32 @test_x86_pext_32_constant_fold() nounwind readnone {
+; CHECK-LABEL: @test_x86_pext_32_constant_fold(
+; CHECK-NEXT:    ret i32 30001
+;
+  %1 = tail call i32 @llvm.x86.bmi.pext.32(i32 1985229328, i32 4042322160)
+  ret i32 %1
+}
+
+define i64 @test_x86_pext_64_constant_fold() nounwind readnone {
+; CHECK-LABEL: @test_x86_pext_64_constant_fold(
+; CHECK-NEXT:    ret i64 1966210489
+;
+  %1 = tail call i64 @llvm.x86.bmi.pext.64(i64 8526495043095935640, i64 -1085102592571150096)
+  ret i64 %1
+}
+
+define i32 @test_x86_pext_32_constant_fold_2() nounwind readnone {
+; CHECK-LABEL: @test_x86_pext_32_constant_fold_2(
+; CHECK-NEXT:    ret i32 30224
+;
+  %1 = tail call i32 @llvm.x86.bmi.pext.32(i32 1985229328, i32 4278190335)
+  ret i32 %1
+}
+
+define i64 @test_x86_pext_64_constant_fold_2() nounwind readnone {
+; CHECK-LABEL: @test_x86_pext_64_constant_fold_2(
+; CHECK-NEXT:    ret i64 1980816570
+;
+  %1 = tail call i64 @llvm.x86.bmi.pext.64(i64 8526495043095935640, i64 -72056498804490496)
+  ret i64 %1
+}
+
+define i32 @test_x86_pdep_32_zero_mask(i32 %x) nounwind readnone {
+; CHECK-LABEL: @test_x86_pdep_32_zero_mask(
+; CHECK-NEXT:    ret i32 0
+;
+  %1 = tail call i32 @llvm.x86.bmi.pdep.32(i32 %x, i32 0)
+  ret i32 %1
+}
+
+define i64 @test_x86_pdep_64_zero_mask(i64 %x) nounwind readnone {
+; CHECK-LABEL: @test_x86_pdep_64_zero_mask(
+; CHECK-NEXT:    ret i64 0
+;
+  %1 = tail call i64 @llvm.x86.bmi.pdep.64(i64 %x, i64 0)
+  ret i64 %1
+}
+
+define i32 @test_x86_pdep_32_allones_mask(i32 %x) nounwind readnone {
+; CHECK-LABEL: @test_x86_pdep_32_allones_mask(
+; CHECK-NEXT:    ret i32 %x
+;
+  %1 = tail call i32 @llvm.x86.bmi.pdep.32(i32 %x, i32 -1)
+  ret i32 %1
+}
+
+define i64 @test_x86_pdep_64_allones_mask(i64 %x) nounwind readnone {
+; CHECK-LABEL: @test_x86_pdep_64_allones_mask(
+; CHECK-NEXT:    ret i64 %x
+;
+  %1 = tail call i64 @llvm.x86.bmi.pdep.64(i64 %x, i64 -1)
+  ret i64 %1
+}
+
+define i32 @test_x86_pdep_32_constant_fold() nounwind readnone {
+; CHECK-LABEL: @test_x86_pdep_32_constant_fold(
+; CHECK-NEXT:    ret i32 807407616
+;
+  %1 = tail call i32 @llvm.x86.bmi.pdep.32(i32 1985229328, i32 4042322160)
+  ret i32 %1
+}
+
+define i64 @test_x86_pdep_64_constant_fold() nounwind readnone {
+; CHECK-LABEL: @test_x86_pdep_64_constant_fold(
+; CHECK-NEXT:    ret i64 -1089641583808049024
+;
+  %1 = tail call i64 @llvm.x86.bmi.pdep.64(i64 8526495043095935640, i64 -1085102592571150096)
+  ret i64 %1
+}
+
+define i32 @test_x86_pdep_32_constant_fold_2() nounwind readnone {
+; CHECK-LABEL: @test_x86_pdep_32_constant_fold_2(
+; CHECK-NEXT:    ret i32 838860816
+;
+  %1 = tail call i32 @llvm.x86.bmi.pdep.32(i32 1985229328, i32 4278190335)
+  ret i32 %1
+}
+
+define i64 @test_x86_pdep_64_constant_fold_2() nounwind readnone {
+; CHECK-LABEL: @test_x86_pdep_64_constant_fold_2(
+; CHECK-NEXT:    ret i64 -144114243170822144
+;
+  %1 = tail call i64 @llvm.x86.bmi.pdep.64(i64 8526495043095935640, i64 -72056498804490496)
+  ret i64 %1
+}
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-f16c.ll b/llvm/test/Transforms/InstCombine/X86/x86-f16c.ll
index 6b5b6cb26..dc0f3e454 100644
--- a/llvm/test/Transforms/InstCombine/X86/x86-f16c.ll
+++ b/llvm/test/Transforms/InstCombine/X86/x86-f16c.ll
@@ -11,7 +11,7 @@ declare <8 x float> @llvm.x86.vcvtph2ps.256(<8 x i16>)
 ; Only bottom 4 elements required.
 define <4 x float> @demand_vcvtph2ps_128(<8 x i16> %A) {
 ; CHECK-LABEL: @demand_vcvtph2ps_128(
-; CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.vcvtph2ps.128(<8 x i16> %A)
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.vcvtph2ps.128(<8 x i16> [[A:%.*]])
 ; CHECK-NEXT:    ret <4 x float> [[TMP1]]
 ;
   %1 = shufflevector <8 x i16> %A, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
@@ -22,7 +22,7 @@ define <4 x float> @demand_vcvtph2ps_128(<8 x i16> %A) {
 ; All 8 elements required.
 define <8 x float> @demand_vcvtph2ps_256(<8 x i16> %A) {
 ; CHECK-LABEL: @demand_vcvtph2ps_256(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i16> %A, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i16> [[A:%.*]], <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:    [[TMP2:%.*]] = tail call <8 x float> @llvm.x86.vcvtph2ps.256(<8 x i16> [[TMP1]])
 ; CHECK-NEXT:    ret <8 x float> [[TMP2]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-fma.ll b/llvm/test/Transforms/InstCombine/X86/x86-fma.ll
index cddb1bf9c..4893131ba 100644
--- a/llvm/test/Transforms/InstCombine/X86/x86-fma.ll
+++ b/llvm/test/Transforms/InstCombine/X86/x86-fma.ll
@@ -5,8 +5,8 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 define <4 x float> @test_vfmadd_ss(<4 x float> %a, <4 x float> %b, <4 x float> %c) {
 ; CHECK-LABEL: @test_vfmadd_ss(
 ; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x float> [[A:%.*]], i64 0
-; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[B:%.*]], i32 0
-; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[C:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[B:%.*]], i64 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[C:%.*]], i64 0
 ; CHECK-NEXT:    [[TMP4:%.*]] = call float @llvm.fma.f32(float [[TMP1]], float [[TMP2]], float [[TMP3]])
 ; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <4 x float> [[A]], float [[TMP4]], i64 0
 ; CHECK-NEXT:    ret <4 x float> [[TMP5]]
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-pack.ll b/llvm/test/Transforms/InstCombine/X86/x86-pack.ll
index 5611d6d7a..5f643bcc0 100644
--- a/llvm/test/Transforms/InstCombine/X86/x86-pack.ll
+++ b/llvm/test/Transforms/InstCombine/X86/x86-pack.ll
@@ -221,7 +221,8 @@ define <8 x i16> @elts_packssdw_128(<4 x i32> %a0, <4 x i32> %a1) {
 define <8 x i16> @elts_packusdw_128(<4 x i32> %a0, <4 x i32> %a1) {
 ; CHECK-LABEL: @elts_packusdw_128(
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i16> @llvm.x86.sse41.packusdw(<4 x i32> [[A0:%.*]], <4 x i32> [[A1:%.*]])
-; CHECK-NEXT:    ret <8 x i16> [[TMP1]]
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> undef, <8 x i32> <i32 undef, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 undef>
+; CHECK-NEXT:    ret <8 x i16> [[TMP2]]
 ;
   %1 = insertelement <4 x i32> %a0, i32 0, i32 0
   %2 = insertelement <4 x i32> %a1, i32 0, i32 3
@@ -255,7 +256,8 @@ define <16 x i8> @elts_packuswb_128(<8 x i16> %a0, <8 x i16> %a1) {
 define <16 x i16> @elts_packssdw_256(<8 x i32> %a0, <8 x i32> %a1) {
 ; CHECK-LABEL: @elts_packssdw_256(
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <16 x i16> @llvm.x86.avx2.packssdw(<8 x i32> [[A0:%.*]], <8 x i32> undef)
-; CHECK-NEXT:    ret <16 x i16> [[TMP1]]
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i16> [[TMP1]], <16 x i16> undef, <16 x i32> <i32 undef, i32 undef, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 8, i32 undef, i32 undef, i32 11, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    ret <16 x i16> [[TMP2]]
 ;
   %1 = shufflevector <8 x i32> %a0, <8 x i32> undef, <8 x i32> <i32 1, i32 0, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   %2 = shufflevector <8 x i32> %a1, <8 x i32> undef, <8 x i32> <i32 undef, i32 2, i32 1, i32 undef, i32 undef, i32 6, i32 5, i32 undef>
@@ -303,7 +305,8 @@ define <32 x i8> @elts_packuswb_256(<16 x i16> %a0, <16 x i16> %a1) {
 define <32 x i16> @elts_packssdw_512(<16 x i32> %a0, <16 x i32> %a1) {
 ; CHECK-LABEL: @elts_packssdw_512(
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <32 x i16> @llvm.x86.avx512.packssdw.512(<16 x i32> [[A0:%.*]], <16 x i32> undef)
-; CHECK-NEXT:    ret <32 x i16> [[TMP1]]
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <32 x i16> [[TMP1]], <32 x i16> undef, <32 x i32> <i32 undef, i32 undef, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 8, i32 undef, i32 undef, i32 11, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 18, i32 19, i32 undef, i32 undef, i32 undef, i32 undef, i32 24, i32 undef, i32 undef, i32 27, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    ret <32 x i16> [[TMP2]]
 ;
   %1 = shufflevector <16 x i32> %a0, <16 x i32> undef, <16 x i32> <i32 1, i32 0, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 9, i32 8, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
   %2 = shufflevector <16 x i32> %a1, <16 x i32> undef, <16 x i32> <i32 undef, i32 2, i32 1, i32 undef, i32 undef, i32 6, i32 5, i32 undef, i32 undef, i32 10, i32 9, i32 undef, i32 undef, i32 14, i32 13, i32 undef>
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-pshufb.ll b/llvm/test/Transforms/InstCombine/X86/x86-pshufb.ll
index d3ffd1780..92122a10d 100644
--- a/llvm/test/Transforms/InstCombine/X86/x86-pshufb.ll
+++ b/llvm/test/Transforms/InstCombine/X86/x86-pshufb.ll
@@ -5,7 +5,7 @@
 
 define <16 x i8> @identity_test(<16 x i8> %InVec) {
 ; CHECK-LABEL: @identity_test(
-; CHECK-NEXT:    ret <16 x i8> %InVec
+; CHECK-NEXT:    ret <16 x i8> [[INVEC:%.*]]
 ;
   %1 = tail call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %InVec, <16 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15>)
   ret <16 x i8> %1
@@ -13,7 +13,7 @@ define <16 x i8> @identity_test(<16 x i8> %InVec) {
 
 define <32 x i8> @identity_test_avx2(<32 x i8> %InVec) {
 ; CHECK-LABEL: @identity_test_avx2(
-; CHECK-NEXT:    ret <32 x i8> %InVec
+; CHECK-NEXT:    ret <32 x i8> [[INVEC:%.*]]
 ;
   %1 = tail call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> %InVec, <32 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15>)
   ret <32 x i8> %1
@@ -21,7 +21,7 @@ define <32 x i8> @identity_test_avx2(<32 x i8> %InVec) {
 
 define <64 x i8> @identity_test_avx512(<64 x i8> %InVec) {
 ; CHECK-LABEL: @identity_test_avx512(
-; CHECK-NEXT:    ret <64 x i8> %InVec
+; CHECK-NEXT:    ret <64 x i8> [[INVEC:%.*]]
 ;
   %1 = tail call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> %InVec, <64 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15>)
   ret <64 x i8> %1
@@ -58,7 +58,7 @@ define <64 x i8> @fold_to_zero_vector_avx512(<64 x i8> %InVec) {
 
 define <16 x i8> @splat_test(<16 x i8> %InVec) {
 ; CHECK-LABEL: @splat_test(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> %InVec, <16 x i8> undef, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> [[INVEC:%.*]], <16 x i8> undef, <16 x i32> zeroinitializer
 ; CHECK-NEXT:    ret <16 x i8> [[TMP1]]
 ;
   %1 = tail call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %InVec, <16 x i8> zeroinitializer)
@@ -72,7 +72,7 @@ define <16 x i8> @splat_test(<16 x i8> %InVec) {
 
 define <32 x i8> @splat_test_avx2(<32 x i8> %InVec) {
 ; CHECK-LABEL: @splat_test_avx2(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i8> %InVec, <32 x i8> undef, <32 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i8> [[INVEC:%.*]], <32 x i8> undef, <32 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
 ; CHECK-NEXT:    ret <32 x i8> [[TMP1]]
 ;
   %1 = tail call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> %InVec, <32 x i8> zeroinitializer)
@@ -81,7 +81,7 @@ define <32 x i8> @splat_test_avx2(<32 x i8> %InVec) {
 
 define <64 x i8> @splat_test_avx512(<64 x i8> %InVec) {
 ; CHECK-LABEL: @splat_test_avx512(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <64 x i8> %InVec, <64 x i8> undef, <64 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <64 x i8> [[INVEC:%.*]], <64 x i8> undef, <64 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48>
 ; CHECK-NEXT:    ret <64 x i8> [[TMP1]]
 ;
   %1 = tail call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> %InVec, <64 x i8> zeroinitializer)
@@ -93,7 +93,7 @@ define <64 x i8> @splat_test_avx512(<64 x i8> %InVec) {
 
 define <16 x i8> @blend1(<16 x i8> %InVec) {
 ; CHECK-LABEL: @blend1(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> %InVec, <16 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <16 x i32> <i32 16, i32 1, i32 16, i32 3, i32 16, i32 5, i32 16, i32 7, i32 16, i32 9, i32 16, i32 11, i32 16, i32 13, i32 16, i32 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> [[INVEC:%.*]], <16 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <16 x i32> <i32 16, i32 1, i32 16, i32 3, i32 16, i32 5, i32 16, i32 7, i32 16, i32 9, i32 16, i32 11, i32 16, i32 13, i32 16, i32 15>
 ; CHECK-NEXT:    ret <16 x i8> [[TMP1]]
 ;
   %1 = tail call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %InVec, <16 x i8> <i8 -128, i8 1, i8 -128, i8 3, i8 -128, i8 5, i8 -128, i8 7, i8 -128, i8 9, i8 -128, i8 11, i8 -128, i8 13, i8 -128, i8 15>)
@@ -102,7 +102,7 @@ define <16 x i8> @blend1(<16 x i8> %InVec) {
 
 define <16 x i8> @blend2(<16 x i8> %InVec) {
 ; CHECK-LABEL: @blend2(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> %InVec, <16 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <16 x i32> <i32 16, i32 16, i32 2, i32 3, i32 16, i32 16, i32 6, i32 7, i32 16, i32 16, i32 10, i32 11, i32 16, i32 16, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> [[INVEC:%.*]], <16 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <16 x i32> <i32 16, i32 16, i32 2, i32 3, i32 16, i32 16, i32 6, i32 7, i32 16, i32 16, i32 10, i32 11, i32 16, i32 16, i32 14, i32 15>
 ; CHECK-NEXT:    ret <16 x i8> [[TMP1]]
 ;
   %1 = tail call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %InVec, <16 x i8> <i8 -128, i8 -128, i8 2, i8 3, i8 -128, i8 -128, i8 6, i8 7, i8 -128, i8 -128, i8 10, i8 11, i8 -128, i8 -128, i8 14, i8 15>)
@@ -111,7 +111,7 @@ define <16 x i8> @blend2(<16 x i8> %InVec) {
 
 define <16 x i8> @blend3(<16 x i8> %InVec) {
 ; CHECK-LABEL: @blend3(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> %InVec, <16 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <16 x i32> <i32 16, i32 16, i32 16, i32 16, i32 4, i32 5, i32 6, i32 7, i32 16, i32 16, i32 16, i32 16, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> [[INVEC:%.*]], <16 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <16 x i32> <i32 16, i32 16, i32 16, i32 16, i32 4, i32 5, i32 6, i32 7, i32 16, i32 16, i32 16, i32 16, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:    ret <16 x i8> [[TMP1]]
 ;
   %1 = tail call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %InVec, <16 x i8> <i8 -128, i8 -128, i8 -128, i8 -128, i8 4, i8 5, i8 6, i8 7, i8 -128, i8 -128, i8 -128, i8 -128, i8 12, i8 13, i8 14, i8 15>)
@@ -120,7 +120,7 @@ define <16 x i8> @blend3(<16 x i8> %InVec) {
 
 define <16 x i8> @blend4(<16 x i8> %InVec) {
 ; CHECK-LABEL: @blend4(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> %InVec, <16 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <16 x i32> <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> [[INVEC:%.*]], <16 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <16 x i32> <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:    ret <16 x i8> [[TMP1]]
 ;
   %1 = tail call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %InVec, <16 x i8> <i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15>)
@@ -129,7 +129,7 @@ define <16 x i8> @blend4(<16 x i8> %InVec) {
 
 define <16 x i8> @blend5(<16 x i8> %InVec) {
 ; CHECK-LABEL: @blend5(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> %InVec, <16 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> [[INVEC:%.*]], <16 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
 ; CHECK-NEXT:    ret <16 x i8> [[TMP1]]
 ;
   %1 = tail call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %InVec, <16 x i8> <i8 0, i8 1, i8 2, i8 3, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128>)
@@ -138,7 +138,7 @@ define <16 x i8> @blend5(<16 x i8> %InVec) {
 
 define <16 x i8> @blend6(<16 x i8> %InVec) {
 ; CHECK-LABEL: @blend6(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> %InVec, <16 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <16 x i32> <i32 0, i32 1, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> [[INVEC:%.*]], <16 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <16 x i32> <i32 0, i32 1, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
 ; CHECK-NEXT:    ret <16 x i8> [[TMP1]]
 ;
   %1 = tail call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %InVec, <16 x i8> <i8 0, i8 1, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128>)
@@ -147,7 +147,7 @@ define <16 x i8> @blend6(<16 x i8> %InVec) {
 
 define <32 x i8> @blend1_avx2(<32 x i8> %InVec) {
 ; CHECK-LABEL: @blend1_avx2(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i8> %InVec, <32 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <32 x i32> <i32 32, i32 1, i32 32, i32 3, i32 32, i32 5, i32 32, i32 7, i32 32, i32 9, i32 32, i32 11, i32 32, i32 13, i32 32, i32 15, i32 48, i32 17, i32 48, i32 19, i32 48, i32 21, i32 48, i32 23, i32 48, i32 25, i32 48, i32 27, i32 48, i32 29, i32 48, i32 31>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i8> [[INVEC:%.*]], <32 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <32 x i32> <i32 32, i32 1, i32 32, i32 3, i32 32, i32 5, i32 32, i32 7, i32 32, i32 9, i32 32, i32 11, i32 32, i32 13, i32 32, i32 15, i32 48, i32 17, i32 48, i32 19, i32 48, i32 21, i32 48, i32 23, i32 48, i32 25, i32 48, i32 27, i32 48, i32 29, i32 48, i32 31>
 ; CHECK-NEXT:    ret <32 x i8> [[TMP1]]
 ;
   %1 = tail call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> %InVec, <32 x i8> <i8 -128, i8 1, i8 -128, i8 3, i8 -128, i8 5, i8 -128, i8 7, i8 -128, i8 9, i8 -128, i8 11, i8 -128, i8 13, i8 -128, i8 15, i8 -128, i8 1, i8 -128, i8 3, i8 -128, i8 5, i8 -128, i8 7, i8 -128, i8 9, i8 -128, i8 11, i8 -128, i8 13, i8 -128, i8 15>)
@@ -156,7 +156,7 @@ define <32 x i8> @blend1_avx2(<32 x i8> %InVec) {
 
 define <32 x i8> @blend2_avx2(<32 x i8> %InVec) {
 ; CHECK-LABEL: @blend2_avx2(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i8> %InVec, <32 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <32 x i32> <i32 32, i32 32, i32 2, i32 3, i32 32, i32 32, i32 6, i32 7, i32 32, i32 32, i32 10, i32 11, i32 32, i32 32, i32 14, i32 15, i32 48, i32 48, i32 18, i32 19, i32 48, i32 48, i32 22, i32 23, i32 48, i32 48, i32 26, i32 27, i32 48, i32 48, i32 30, i32 31>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i8> [[INVEC:%.*]], <32 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <32 x i32> <i32 32, i32 32, i32 2, i32 3, i32 32, i32 32, i32 6, i32 7, i32 32, i32 32, i32 10, i32 11, i32 32, i32 32, i32 14, i32 15, i32 48, i32 48, i32 18, i32 19, i32 48, i32 48, i32 22, i32 23, i32 48, i32 48, i32 26, i32 27, i32 48, i32 48, i32 30, i32 31>
 ; CHECK-NEXT:    ret <32 x i8> [[TMP1]]
 ;
   %1 = tail call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> %InVec, <32 x i8> <i8 -128, i8 -128, i8 2, i8 3, i8 -128, i8 -128, i8 6, i8 7, i8 -128, i8 -128, i8 10, i8 11, i8 -128, i8 -128, i8 14, i8 15, i8 -128, i8 -128, i8 2, i8 3, i8 -128, i8 -128, i8 6, i8 7, i8 -128, i8 -128, i8 10, i8 11, i8 -128, i8 -128, i8 14, i8 15>)
@@ -165,7 +165,7 @@ define <32 x i8> @blend2_avx2(<32 x i8> %InVec) {
 
 define <32 x i8> @blend3_avx2(<32 x i8> %InVec) {
 ; CHECK-LABEL: @blend3_avx2(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i8> %InVec, <32 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <32 x i32> <i32 32, i32 32, i32 32, i32 32, i32 4, i32 5, i32 6, i32 7, i32 32, i32 32, i32 32, i32 32, i32 12, i32 13, i32 14, i32 15, i32 48, i32 48, i32 48, i32 48, i32 20, i32 21, i32 22, i32 23, i32 48, i32 48, i32 48, i32 48, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i8> [[INVEC:%.*]], <32 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <32 x i32> <i32 32, i32 32, i32 32, i32 32, i32 4, i32 5, i32 6, i32 7, i32 32, i32 32, i32 32, i32 32, i32 12, i32 13, i32 14, i32 15, i32 48, i32 48, i32 48, i32 48, i32 20, i32 21, i32 22, i32 23, i32 48, i32 48, i32 48, i32 48, i32 28, i32 29, i32 30, i32 31>
 ; CHECK-NEXT:    ret <32 x i8> [[TMP1]]
 ;
   %1 = tail call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> %InVec, <32 x i8> <i8 -128, i8 -128, i8 -128, i8 -128, i8 4, i8 5, i8 6, i8 7, i8 -128, i8 -128, i8 -128, i8 -128, i8 12, i8 13, i8 14, i8 15, i8 -128, i8 -128, i8 -128, i8 -128, i8 4, i8 5, i8 6, i8 7, i8 -128, i8 -128, i8 -128, i8 -128, i8 12, i8 13, i8 14, i8 15>)
@@ -174,7 +174,7 @@ define <32 x i8> @blend3_avx2(<32 x i8> %InVec) {
 
 define <32 x i8> @blend4_avx2(<32 x i8> %InVec) {
 ; CHECK-LABEL: @blend4_avx2(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i8> %InVec, <32 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <32 x i32> <i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i8> [[INVEC:%.*]], <32 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <32 x i32> <i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; CHECK-NEXT:    ret <32 x i8> [[TMP1]]
 ;
   %1 = tail call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> %InVec, <32 x i8> <i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15>)
@@ -183,7 +183,7 @@ define <32 x i8> @blend4_avx2(<32 x i8> %InVec) {
 
 define <32 x i8> @blend5_avx2(<32 x i8> %InVec) {
 ; CHECK-LABEL: @blend5_avx2(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i8> %InVec, <32 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 16, i32 17, i32 18, i32 19, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i8> [[INVEC:%.*]], <32 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 16, i32 17, i32 18, i32 19, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48>
 ; CHECK-NEXT:    ret <32 x i8> [[TMP1]]
 ;
   %1 = tail call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> %InVec, <32 x i8> <i8 0, i8 1, i8 2, i8 3, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 0, i8 1, i8 2, i8 3, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128>)
@@ -192,7 +192,7 @@ define <32 x i8> @blend5_avx2(<32 x i8> %InVec) {
 
 define <32 x i8> @blend6_avx2(<32 x i8> %InVec) {
 ; CHECK-LABEL: @blend6_avx2(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i8> %InVec, <32 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <32 x i32> <i32 0, i32 1, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 16, i32 17, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i8> [[INVEC:%.*]], <32 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <32 x i32> <i32 0, i32 1, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 16, i32 17, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48>
 ; CHECK-NEXT:    ret <32 x i8> [[TMP1]]
 ;
   %1 = tail call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> %InVec, <32 x i8> <i8 0, i8 1, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 0, i8 1, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128>)
@@ -201,7 +201,7 @@ define <32 x i8> @blend6_avx2(<32 x i8> %InVec) {
 
 define <64 x i8> @blend1_avx512(<64 x i8> %InVec) {
 ; CHECK-LABEL: @blend1_avx512(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <64 x i8> %InVec, <64 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <64 x i32> <i32 64, i32 1, i32 64, i32 3, i32 64, i32 5, i32 64, i32 7, i32 64, i32 9, i32 64, i32 11, i32 64, i32 13, i32 64, i32 15, i32 80, i32 17, i32 80, i32 19, i32 80, i32 21, i32 80, i32 23, i32 80, i32 25, i32 80, i32 27, i32 80, i32 29, i32 80, i32 31, i32 96, i32 33, i32 96, i32 35, i32 96, i32 37, i32 96, i32 39, i32 96, i32 41, i32 96, i32 43, i32 96, i32 45, i32 96, i32 47, i32 112, i32 49, i32 112, i32 51, i32 112, i32 53, i32 112, i32 55, i32 112, i32 57, i32 112, i32 59, i32 112, i32 61, i32 112, i32 63>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <64 x i8> [[INVEC:%.*]], <64 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <64 x i32> <i32 64, i32 1, i32 64, i32 3, i32 64, i32 5, i32 64, i32 7, i32 64, i32 9, i32 64, i32 11, i32 64, i32 13, i32 64, i32 15, i32 80, i32 17, i32 80, i32 19, i32 80, i32 21, i32 80, i32 23, i32 80, i32 25, i32 80, i32 27, i32 80, i32 29, i32 80, i32 31, i32 96, i32 33, i32 96, i32 35, i32 96, i32 37, i32 96, i32 39, i32 96, i32 41, i32 96, i32 43, i32 96, i32 45, i32 96, i32 47, i32 112, i32 49, i32 112, i32 51, i32 112, i32 53, i32 112, i32 55, i32 112, i32 57, i32 112, i32 59, i32 112, i32 61, i32 112, i32 63>
 ; CHECK-NEXT:    ret <64 x i8> [[TMP1]]
 ;
   %1 = tail call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> %InVec, <64 x i8> <i8 -128, i8 1, i8 -128, i8 3, i8 -128, i8 5, i8 -128, i8 7, i8 -128, i8 9, i8 -128, i8 11, i8 -128, i8 13, i8 -128, i8 15, i8 -128, i8 1, i8 -128, i8 3, i8 -128, i8 5, i8 -128, i8 7, i8 -128, i8 9, i8 -128, i8 11, i8 -128, i8 13, i8 -128, i8 15, i8 -128, i8 1, i8 -128, i8 3, i8 -128, i8 5, i8 -128, i8 7, i8 -128, i8 9, i8 -128, i8 11, i8 -128, i8 13, i8 -128, i8 15, i8 -128, i8 1, i8 -128, i8 3, i8 -128, i8 5, i8 -128, i8 7, i8 -128, i8 9, i8 -128, i8 11, i8 -128, i8 13, i8 -128, i8 15>)
@@ -210,7 +210,7 @@ define <64 x i8> @blend1_avx512(<64 x i8> %InVec) {
 
 define <64 x i8> @blend2_avx512(<64 x i8> %InVec) {
 ; CHECK-LABEL: @blend2_avx512(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <64 x i8> %InVec, <64 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <64 x i32> <i32 64, i32 64, i32 2, i32 3, i32 64, i32 64, i32 6, i32 7, i32 64, i32 64, i32 10, i32 11, i32 64, i32 64, i32 14, i32 15, i32 80, i32 80, i32 18, i32 19, i32 80, i32 80, i32 22, i32 23, i32 80, i32 80, i32 26, i32 27, i32 80, i32 80, i32 30, i32 31, i32 96, i32 96, i32 34, i32 35, i32 96, i32 96, i32 38, i32 39, i32 96, i32 96, i32 42, i32 43, i32 96, i32 96, i32 46, i32 47, i32 112, i32 112, i32 50, i32 51, i32 112, i32 112, i32 54, i32 55, i32 112, i32 112, i32 58, i32 59, i32 112, i32 112, i32 62, i32 63>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <64 x i8> [[INVEC:%.*]], <64 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <64 x i32> <i32 64, i32 64, i32 2, i32 3, i32 64, i32 64, i32 6, i32 7, i32 64, i32 64, i32 10, i32 11, i32 64, i32 64, i32 14, i32 15, i32 80, i32 80, i32 18, i32 19, i32 80, i32 80, i32 22, i32 23, i32 80, i32 80, i32 26, i32 27, i32 80, i32 80, i32 30, i32 31, i32 96, i32 96, i32 34, i32 35, i32 96, i32 96, i32 38, i32 39, i32 96, i32 96, i32 42, i32 43, i32 96, i32 96, i32 46, i32 47, i32 112, i32 112, i32 50, i32 51, i32 112, i32 112, i32 54, i32 55, i32 112, i32 112, i32 58, i32 59, i32 112, i32 112, i32 62, i32 63>
 ; CHECK-NEXT:    ret <64 x i8> [[TMP1]]
 ;
   %1 = tail call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> %InVec, <64 x i8> <i8 -128, i8 -128, i8 2, i8 3, i8 -128, i8 -128, i8 6, i8 7, i8 -128, i8 -128, i8 10, i8 11, i8 -128, i8 -128, i8 14, i8 15, i8 -128, i8 -128, i8 2, i8 3, i8 -128, i8 -128, i8 6, i8 7, i8 -128, i8 -128, i8 10, i8 11, i8 -128, i8 -128, i8 14, i8 15, i8 -128, i8 -128, i8 2, i8 3, i8 -128, i8 -128, i8 6, i8 7, i8 -128, i8 -128, i8 10, i8 11, i8 -128, i8 -128, i8 14, i8 15, i8 -128, i8 -128, i8 2, i8 3, i8 -128, i8 -128, i8 6, i8 7, i8 -128, i8 -128, i8 10, i8 11, i8 -128, i8 -128, i8 14, i8 15>)
@@ -219,7 +219,7 @@ define <64 x i8> @blend2_avx512(<64 x i8> %InVec) {
 
 define <64 x i8> @blend3_avx512(<64 x i8> %InVec) {
 ; CHECK-LABEL: @blend3_avx512(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <64 x i8> %InVec, <64 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <64 x i32> <i32 64, i32 64, i32 64, i32 64, i32 4, i32 5, i32 6, i32 7, i32 64, i32 64, i32 64, i32 64, i32 12, i32 13, i32 14, i32 15, i32 80, i32 80, i32 80, i32 80, i32 20, i32 21, i32 22, i32 23, i32 80, i32 80, i32 80, i32 80, i32 28, i32 29, i32 30, i32 31, i32 96, i32 96, i32 96, i32 96, i32 36, i32 37, i32 38, i32 39, i32 96, i32 96, i32 96, i32 96, i32 44, i32 45, i32 46, i32 47, i32 112, i32 112, i32 112, i32 112, i32 52, i32 53, i32 54, i32 55, i32 112, i32 112, i32 112, i32 112, i32 60, i32 61, i32 62, i32 63>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <64 x i8> [[INVEC:%.*]], <64 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <64 x i32> <i32 64, i32 64, i32 64, i32 64, i32 4, i32 5, i32 6, i32 7, i32 64, i32 64, i32 64, i32 64, i32 12, i32 13, i32 14, i32 15, i32 80, i32 80, i32 80, i32 80, i32 20, i32 21, i32 22, i32 23, i32 80, i32 80, i32 80, i32 80, i32 28, i32 29, i32 30, i32 31, i32 96, i32 96, i32 96, i32 96, i32 36, i32 37, i32 38, i32 39, i32 96, i32 96, i32 96, i32 96, i32 44, i32 45, i32 46, i32 47, i32 112, i32 112, i32 112, i32 112, i32 52, i32 53, i32 54, i32 55, i32 112, i32 112, i32 112, i32 112, i32 60, i32 61, i32 62, i32 63>
 ; CHECK-NEXT:    ret <64 x i8> [[TMP1]]
 ;
   %1 = tail call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> %InVec, <64 x i8> <i8 -128, i8 -128, i8 -128, i8 -128, i8 4, i8 5, i8 6, i8 7, i8 -128, i8 -128, i8 -128, i8 -128, i8 12, i8 13, i8 14, i8 15, i8 -128, i8 -128, i8 -128, i8 -128, i8 4, i8 5, i8 6, i8 7, i8 -128, i8 -128, i8 -128, i8 -128, i8 12, i8 13, i8 14, i8 15, i8 -128, i8 -128, i8 -128, i8 -128, i8 4, i8 5, i8 6, i8 7, i8 -128, i8 -128, i8 -128, i8 -128, i8 12, i8 13, i8 14, i8 15, i8 -128, i8 -128, i8 -128, i8 -128, i8 4, i8 5, i8 6, i8 7, i8 -128, i8 -128, i8 -128, i8 -128, i8 12, i8 13, i8 14, i8 15>)
@@ -228,7 +228,7 @@ define <64 x i8> @blend3_avx512(<64 x i8> %InVec) {
 
 define <64 x i8> @blend4_avx512(<64 x i8> %InVec) {
 ; CHECK-LABEL: @blend4_avx512(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <64 x i8> %InVec, <64 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <64 x i32> <i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <64 x i8> [[INVEC:%.*]], <64 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <64 x i32> <i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
 ; CHECK-NEXT:    ret <64 x i8> [[TMP1]]
 ;
   %1 = tail call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> %InVec, <64 x i8> <i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15>)
@@ -237,7 +237,7 @@ define <64 x i8> @blend4_avx512(<64 x i8> %InVec) {
 
 define <64 x i8> @blend5_avx512(<64 x i8> %InVec) {
 ; CHECK-LABEL: @blend5_avx512(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <64 x i8> %InVec, <64 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 16, i32 17, i32 18, i32 19, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 32, i32 33, i32 34, i32 35, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 48, i32 49, i32 50, i32 51, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <64 x i8> [[INVEC:%.*]], <64 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 16, i32 17, i32 18, i32 19, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 32, i32 33, i32 34, i32 35, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 48, i32 49, i32 50, i32 51, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112>
 ; CHECK-NEXT:    ret <64 x i8> [[TMP1]]
 ;
   %1 = tail call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> %InVec, <64 x i8> <i8 0, i8 1, i8 2, i8 3, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 0, i8 1, i8 2, i8 3, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 0, i8 1, i8 2, i8 3, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 0, i8 1, i8 2, i8 3, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128>)
@@ -246,7 +246,7 @@ define <64 x i8> @blend5_avx512(<64 x i8> %InVec) {
 
 define <64 x i8> @blend6_avx512(<64 x i8> %InVec) {
 ; CHECK-LABEL: @blend6_avx512(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <64 x i8> %InVec, <64 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <64 x i32> <i32 0, i32 1, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 16, i32 17, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 32, i32 33, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 48, i32 49, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <64 x i8> [[INVEC:%.*]], <64 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <64 x i32> <i32 0, i32 1, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 16, i32 17, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 32, i32 33, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 48, i32 49, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112>
 ; CHECK-NEXT:    ret <64 x i8> [[TMP1]]
 ;
   %1 = tail call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> %InVec, <64 x i8> <i8 0, i8 1, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128,i8 0, i8 1, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 0, i8 1, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 0, i8 1, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128>)
@@ -256,7 +256,7 @@ define <64 x i8> @blend6_avx512(<64 x i8> %InVec) {
 ; movq idiom.
 define <16 x i8> @movq_idiom(<16 x i8> %InVec) {
 ; CHECK-LABEL: @movq_idiom(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> %InVec, <16 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> [[INVEC:%.*]], <16 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
 ; CHECK-NEXT:    ret <16 x i8> [[TMP1]]
 ;
   %1 = tail call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %InVec, <16 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128>)
@@ -265,7 +265,7 @@ define <16 x i8> @movq_idiom(<16 x i8> %InVec) {
 
 define <32 x i8> @movq_idiom_avx2(<32 x i8> %InVec) {
 ; CHECK-LABEL: @movq_idiom_avx2(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i8> %InVec, <32 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i8> [[INVEC:%.*]], <32 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48>
 ; CHECK-NEXT:    ret <32 x i8> [[TMP1]]
 ;
   %1 = tail call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> %InVec, <32 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128>)
@@ -274,7 +274,7 @@ define <32 x i8> @movq_idiom_avx2(<32 x i8> %InVec) {
 
 define <64 x i8> @movq_idiom_avx512(<64 x i8> %InVec) {
 ; CHECK-LABEL: @movq_idiom_avx512(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <64 x i8> %InVec, <64 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <64 x i8> [[INVEC:%.*]], <64 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112>
 ; CHECK-NEXT:    ret <64 x i8> [[TMP1]]
 ;
   %1 = tail call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> %InVec, <64 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128>)
@@ -285,7 +285,7 @@ define <64 x i8> @movq_idiom_avx512(<64 x i8> %InVec) {
 
 define <16 x i8> @permute1(<16 x i8> %InVec) {
 ; CHECK-LABEL: @permute1(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> %InVec, <16 x i8> undef, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 12, i32 13, i32 14, i32 15, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> [[INVEC:%.*]], <16 x i8> undef, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 12, i32 13, i32 14, i32 15, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:    ret <16 x i8> [[TMP1]]
 ;
   %1 = tail call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %InVec, <16 x i8> <i8 4, i8 5, i8 6, i8 7, i8 4, i8 5, i8 6, i8 7, i8 12, i8 13, i8 14, i8 15, i8 12, i8 13, i8 14, i8 15>)
@@ -294,7 +294,7 @@ define <16 x i8> @permute1(<16 x i8> %InVec) {
 
 define <16 x i8> @permute2(<16 x i8> %InVec) {
 ; CHECK-LABEL: @permute2(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> %InVec, <16 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> [[INVEC:%.*]], <16 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:    ret <16 x i8> [[TMP1]]
 ;
   %1 = tail call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %InVec, <16 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7>)
@@ -303,7 +303,7 @@ define <16 x i8> @permute2(<16 x i8> %InVec) {
 
 define <32 x i8> @permute1_avx2(<32 x i8> %InVec) {
 ; CHECK-LABEL: @permute1_avx2(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i8> %InVec, <32 x i8> undef, <32 x i32> <i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 12, i32 13, i32 14, i32 15, i32 12, i32 13, i32 14, i32 15, i32 20, i32 21, i32 22, i32 23, i32 20, i32 21, i32 22, i32 23, i32 28, i32 29, i32 30, i32 31, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i8> [[INVEC:%.*]], <32 x i8> undef, <32 x i32> <i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 12, i32 13, i32 14, i32 15, i32 12, i32 13, i32 14, i32 15, i32 20, i32 21, i32 22, i32 23, i32 20, i32 21, i32 22, i32 23, i32 28, i32 29, i32 30, i32 31, i32 28, i32 29, i32 30, i32 31>
 ; CHECK-NEXT:    ret <32 x i8> [[TMP1]]
 ;
   %1 = tail call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> %InVec, <32 x i8> <i8 4, i8 5, i8 6, i8 7, i8 4, i8 5, i8 6, i8 7, i8 12, i8 13, i8 14, i8 15, i8 12, i8 13, i8 14, i8 15, i8 4, i8 5, i8 6, i8 7, i8 4, i8 5, i8 6, i8 7, i8 12, i8 13, i8 14, i8 15, i8 12, i8 13, i8 14, i8 15>)
@@ -312,7 +312,7 @@ define <32 x i8> @permute1_avx2(<32 x i8> %InVec) {
 
 define <32 x i8> @permute2_avx2(<32 x i8> %InVec) {
 ; CHECK-LABEL: @permute2_avx2(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i8> %InVec, <32 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i8> [[INVEC:%.*]], <32 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; CHECK-NEXT:    ret <32 x i8> [[TMP1]]
 ;
   %1 = tail call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> %InVec, <32 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7>)
@@ -321,7 +321,7 @@ define <32 x i8> @permute2_avx2(<32 x i8> %InVec) {
 
 define <64 x i8> @permute1_avx512(<64 x i8> %InVec) {
 ; CHECK-LABEL: @permute1_avx512(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <64 x i8> %InVec, <64 x i8> undef, <64 x i32> <i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 12, i32 13, i32 14, i32 15, i32 12, i32 13, i32 14, i32 15, i32 20, i32 21, i32 22, i32 23, i32 20, i32 21, i32 22, i32 23, i32 28, i32 29, i32 30, i32 31, i32 28, i32 29, i32 30, i32 31, i32 36, i32 37, i32 38, i32 39, i32 36, i32 37, i32 38, i32 39, i32 44, i32 45, i32 46, i32 47, i32 44, i32 45, i32 46, i32 47, i32 52, i32 53, i32 54, i32 55, i32 52, i32 53, i32 54, i32 55, i32 60, i32 61, i32 62, i32 63, i32 60, i32 61, i32 62, i32 63>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <64 x i8> [[INVEC:%.*]], <64 x i8> undef, <64 x i32> <i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 12, i32 13, i32 14, i32 15, i32 12, i32 13, i32 14, i32 15, i32 20, i32 21, i32 22, i32 23, i32 20, i32 21, i32 22, i32 23, i32 28, i32 29, i32 30, i32 31, i32 28, i32 29, i32 30, i32 31, i32 36, i32 37, i32 38, i32 39, i32 36, i32 37, i32 38, i32 39, i32 44, i32 45, i32 46, i32 47, i32 44, i32 45, i32 46, i32 47, i32 52, i32 53, i32 54, i32 55, i32 52, i32 53, i32 54, i32 55, i32 60, i32 61, i32 62, i32 63, i32 60, i32 61, i32 62, i32 63>
 ; CHECK-NEXT:    ret <64 x i8> [[TMP1]]
 ;
   %1 = tail call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> %InVec, <64 x i8> <i8 4, i8 5, i8 6, i8 7, i8 4, i8 5, i8 6, i8 7, i8 12, i8 13, i8 14, i8 15, i8 12, i8 13, i8 14, i8 15, i8 4, i8 5, i8 6, i8 7, i8 4, i8 5, i8 6, i8 7, i8 12, i8 13, i8 14, i8 15, i8 12, i8 13, i8 14, i8 15, i8 4, i8 5, i8 6, i8 7, i8 4, i8 5, i8 6, i8 7, i8 12, i8 13, i8 14, i8 15, i8 12, i8 13, i8 14, i8 15, i8 4, i8 5, i8 6, i8 7, i8 4, i8 5, i8 6, i8 7, i8 12, i8 13, i8 14, i8 15, i8 12, i8 13, i8 14, i8 15>)
@@ -330,7 +330,7 @@ define <64 x i8> @permute1_avx512(<64 x i8> %InVec) {
 
 define <64 x i8> @permute2_avx512(<64 x i8> %InVec) {
 ; CHECK-LABEL: @permute2_avx512(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <64 x i8> %InVec, <64 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <64 x i8> [[INVEC:%.*]], <64 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55>
 ; CHECK-NEXT:    ret <64 x i8> [[TMP1]]
 ;
   %1 = tail call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> %InVec, <64 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7>)
@@ -342,7 +342,7 @@ define <64 x i8> @permute2_avx512(<64 x i8> %InVec) {
 
 define <16 x i8> @identity_test2_2(<16 x i8> %InVec) {
 ; CHECK-LABEL: @identity_test2_2(
-; CHECK-NEXT:    ret <16 x i8> %InVec
+; CHECK-NEXT:    ret <16 x i8> [[INVEC:%.*]]
 ;
   %1 = tail call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %InVec, <16 x i8> <i8 16, i8 17, i8 18, i8 19, i8 20, i8 21, i8 22, i8 23, i8 24, i8 25, i8 26, i8 27, i8 28, i8 29, i8 30, i8 31>)
   ret <16 x i8> %1
@@ -350,7 +350,7 @@ define <16 x i8> @identity_test2_2(<16 x i8> %InVec) {
 
 define <32 x i8> @identity_test_avx2_2(<32 x i8> %InVec) {
 ; CHECK-LABEL: @identity_test_avx2_2(
-; CHECK-NEXT:    ret <32 x i8> %InVec
+; CHECK-NEXT:    ret <32 x i8> [[INVEC:%.*]]
 ;
   %1 = tail call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> %InVec, <32 x i8> <i8 16, i8 33, i8 66, i8 19, i8 36, i8 69, i8 22, i8 39, i8 72, i8 25, i8 42, i8 75, i8 28, i8 45, i8 78, i8 31, i8 48, i8 81, i8 34, i8 51, i8 84, i8 37, i8 54, i8 87, i8 40, i8 57, i8 90, i8 43, i8 60, i8 93, i8 46, i8 63>)
   ret <32 x i8> %1
@@ -358,7 +358,7 @@ define <32 x i8> @identity_test_avx2_2(<32 x i8> %InVec) {
 
 define <64 x i8> @identity_test_avx512_2(<64 x i8> %InVec) {
 ; CHECK-LABEL: @identity_test_avx512_2(
-; CHECK-NEXT:    ret <64 x i8> %InVec
+; CHECK-NEXT:    ret <64 x i8> [[INVEC:%.*]]
 ;
   %1 = tail call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> %InVec, <64 x i8> <i8 16, i8 33, i8 66, i8 19, i8 36, i8 69, i8 22, i8 39, i8 72, i8 25, i8 42, i8 75, i8 28, i8 45, i8 78, i8 31, i8 48, i8 81, i8 34, i8 51, i8 84, i8 37, i8 54, i8 87, i8 40, i8 57, i8 90, i8 43, i8 60, i8 93, i8 46, i8 63, i8 96, i8 49, i8 66, i8 99, i8 52, i8 69, i8 102, i8 55, i8 72, i8 105, i8 58, i8 75, i8 108, i8 61, i8 78, i8 111, i8 64, i8 81, i8 114, i8 67, i8 84, i8 117, i8 70, i8 87, i8 120, i8 73, i8 90, i8 123, i8 76, i8 93, i8 126, i8 79>)
   ret <64 x i8> %1
@@ -390,7 +390,7 @@ define <64 x i8> @fold_to_zero_vector_avx512_2(<64 x i8> %InVec) {
 
 define <16 x i8> @permute3(<16 x i8> %InVec) {
 ; CHECK-LABEL: @permute3(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> %InVec, <16 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> [[INVEC:%.*]], <16 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:    ret <16 x i8> [[TMP1]]
 ;
   %1 = tail call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %InVec, <16 x i8> <i8 48, i8 17, i8 34, i8 51, i8 20, i8 37, i8 54, i8 23, i8 16, i8 49, i8 66, i8 19, i8 52, i8 69, i8 22, i8 55>)
@@ -399,7 +399,7 @@ define <16 x i8> @permute3(<16 x i8> %InVec) {
 
 define <32 x i8> @permute3_avx2(<32 x i8> %InVec) {
 ; CHECK-LABEL: @permute3_avx2(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i8> %InVec, <32 x i8> undef, <32 x i32> <i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 12, i32 13, i32 14, i32 15, i32 12, i32 13, i32 14, i32 15, i32 20, i32 21, i32 22, i32 23, i32 20, i32 21, i32 22, i32 23, i32 28, i32 29, i32 30, i32 31, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i8> [[INVEC:%.*]], <32 x i8> undef, <32 x i32> <i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 12, i32 13, i32 14, i32 15, i32 12, i32 13, i32 14, i32 15, i32 20, i32 21, i32 22, i32 23, i32 20, i32 21, i32 22, i32 23, i32 28, i32 29, i32 30, i32 31, i32 28, i32 29, i32 30, i32 31>
 ; CHECK-NEXT:    ret <32 x i8> [[TMP1]]
 ;
   %1 = tail call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> %InVec, <32 x i8> <i8 52, i8 21, i8 38, i8 55, i8 20, i8 37, i8 54, i8 23, i8 28, i8 61, i8 78, i8 31, i8 60, i8 29, i8 30, i8 79, i8 52, i8 21, i8 38, i8 55, i8 20, i8 53, i8 102, i8 23, i8 92, i8 93, i8 94, i8 95, i8 108, i8 109, i8 110, i8 111>)
@@ -408,7 +408,7 @@ define <32 x i8> @permute3_avx2(<32 x i8> %InVec) {
 
 define <64 x i8> @permute3_avx512(<64 x i8> %InVec) {
 ; CHECK-LABEL: @permute3_avx512(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <64 x i8> %InVec, <64 x i8> undef, <64 x i32> <i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 12, i32 13, i32 14, i32 15, i32 12, i32 13, i32 14, i32 15, i32 20, i32 21, i32 22, i32 23, i32 20, i32 21, i32 22, i32 23, i32 28, i32 29, i32 30, i32 31, i32 28, i32 29, i32 30, i32 31, i32 36, i32 37, i32 38, i32 39, i32 36, i32 37, i32 38, i32 39, i32 44, i32 45, i32 46, i32 47, i32 44, i32 45, i32 46, i32 47, i32 52, i32 53, i32 54, i32 55, i32 52, i32 53, i32 54, i32 55, i32 60, i32 61, i32 62, i32 63, i32 60, i32 61, i32 62, i32 63>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <64 x i8> [[INVEC:%.*]], <64 x i8> undef, <64 x i32> <i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 12, i32 13, i32 14, i32 15, i32 12, i32 13, i32 14, i32 15, i32 20, i32 21, i32 22, i32 23, i32 20, i32 21, i32 22, i32 23, i32 28, i32 29, i32 30, i32 31, i32 28, i32 29, i32 30, i32 31, i32 36, i32 37, i32 38, i32 39, i32 36, i32 37, i32 38, i32 39, i32 44, i32 45, i32 46, i32 47, i32 44, i32 45, i32 46, i32 47, i32 52, i32 53, i32 54, i32 55, i32 52, i32 53, i32 54, i32 55, i32 60, i32 61, i32 62, i32 63, i32 60, i32 61, i32 62, i32 63>
 ; CHECK-NEXT:    ret <64 x i8> [[TMP1]]
 ;
   %1 = tail call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> %InVec, <64 x i8> <i8 52, i8 21, i8 38, i8 55, i8 20, i8 37, i8 54, i8 23, i8 28, i8 61, i8 78, i8 31, i8 60, i8 29, i8 30, i8 79, i8 52, i8 21, i8 38, i8 55, i8 20, i8 53, i8 102, i8 23, i8 92, i8 93, i8 94, i8 95, i8 108, i8 109, i8 110, i8 111, i8 52, i8 21, i8 38, i8 55, i8 20, i8 37, i8 54, i8 23, i8 28, i8 61, i8 78, i8 31, i8 60, i8 29, i8 30, i8 79, i8 52, i8 21, i8 38, i8 55, i8 20, i8 53, i8 102, i8 23, i8 108, i8 109, i8 110, i8 111, i8 124, i8 125, i8 126, i8 127>)
@@ -419,7 +419,7 @@ define <64 x i8> @permute3_avx512(<64 x i8> %InVec) {
 
 define <16 x i8> @fold_with_undef_elts(<16 x i8> %InVec) {
 ; CHECK-LABEL: @fold_with_undef_elts(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> %InVec, <16 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <16 x i32> <i32 0, i32 16, i32 undef, i32 16, i32 1, i32 16, i32 undef, i32 16, i32 2, i32 16, i32 undef, i32 16, i32 3, i32 16, i32 undef, i32 16>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> [[INVEC:%.*]], <16 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <16 x i32> <i32 0, i32 16, i32 undef, i32 16, i32 1, i32 16, i32 undef, i32 16, i32 2, i32 16, i32 undef, i32 16, i32 3, i32 16, i32 undef, i32 16>
 ; CHECK-NEXT:    ret <16 x i8> [[TMP1]]
 ;
   %1 = tail call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %InVec, <16 x i8> <i8 0, i8 -128, i8 undef, i8 -128, i8 1, i8 -128, i8 undef, i8 -128, i8 2, i8 -128, i8 undef, i8 -128, i8 3, i8 -128, i8 undef, i8 -128>)
@@ -428,7 +428,7 @@ define <16 x i8> @fold_with_undef_elts(<16 x i8> %InVec) {
 
 define <32 x i8> @fold_with_undef_elts_avx2(<32 x i8> %InVec) {
 ; CHECK-LABEL: @fold_with_undef_elts_avx2(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i8> %InVec, <32 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <32 x i32> <i32 0, i32 32, i32 undef, i32 32, i32 1, i32 32, i32 undef, i32 32, i32 2, i32 32, i32 undef, i32 32, i32 3, i32 32, i32 undef, i32 32, i32 16, i32 48, i32 undef, i32 48, i32 17, i32 48, i32 undef, i32 48, i32 18, i32 48, i32 undef, i32 48, i32 19, i32 48, i32 undef, i32 48>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i8> [[INVEC:%.*]], <32 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <32 x i32> <i32 0, i32 32, i32 undef, i32 32, i32 1, i32 32, i32 undef, i32 32, i32 2, i32 32, i32 undef, i32 32, i32 3, i32 32, i32 undef, i32 32, i32 16, i32 48, i32 undef, i32 48, i32 17, i32 48, i32 undef, i32 48, i32 18, i32 48, i32 undef, i32 48, i32 19, i32 48, i32 undef, i32 48>
 ; CHECK-NEXT:    ret <32 x i8> [[TMP1]]
 ;
   %1 = tail call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> %InVec, <32 x i8> <i8 0, i8 -128, i8 undef, i8 -128, i8 1, i8 -128, i8 undef, i8 -128, i8 2, i8 -128, i8 undef, i8 -128, i8 3, i8 -128, i8 undef, i8 -128, i8 0, i8 -128, i8 undef, i8 -128, i8 1, i8 -128, i8 undef, i8 -128, i8 2, i8 -128, i8 undef, i8 -128, i8 3, i8 -128, i8 undef, i8 -128>)
@@ -437,7 +437,7 @@ define <32 x i8> @fold_with_undef_elts_avx2(<32 x i8> %InVec) {
 
 define <64 x i8> @fold_with_undef_elts_avx512(<64 x i8> %InVec) {
 ; CHECK-LABEL: @fold_with_undef_elts_avx512(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <64 x i8> %InVec, <64 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <64 x i32> <i32 0, i32 64, i32 undef, i32 64, i32 1, i32 64, i32 undef, i32 64, i32 2, i32 64, i32 undef, i32 64, i32 3, i32 64, i32 undef, i32 64, i32 16, i32 80, i32 undef, i32 80, i32 17, i32 80, i32 undef, i32 80, i32 18, i32 80, i32 undef, i32 80, i32 19, i32 80, i32 undef, i32 80, i32 32, i32 96, i32 undef, i32 96, i32 33, i32 96, i32 undef, i32 96, i32 34, i32 96, i32 undef, i32 96, i32 35, i32 96, i32 undef, i32 96, i32 48, i32 112, i32 undef, i32 112, i32 49, i32 112, i32 undef, i32 112, i32 50, i32 112, i32 undef, i32 112, i32 51, i32 112, i32 undef, i32 112>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <64 x i8> [[INVEC:%.*]], <64 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <64 x i32> <i32 0, i32 64, i32 undef, i32 64, i32 1, i32 64, i32 undef, i32 64, i32 2, i32 64, i32 undef, i32 64, i32 3, i32 64, i32 undef, i32 64, i32 16, i32 80, i32 undef, i32 80, i32 17, i32 80, i32 undef, i32 80, i32 18, i32 80, i32 undef, i32 80, i32 19, i32 80, i32 undef, i32 80, i32 32, i32 96, i32 undef, i32 96, i32 33, i32 96, i32 undef, i32 96, i32 34, i32 96, i32 undef, i32 96, i32 35, i32 96, i32 undef, i32 96, i32 48, i32 112, i32 undef, i32 112, i32 49, i32 112, i32 undef, i32 112, i32 50, i32 112, i32 undef, i32 112, i32 51, i32 112, i32 undef, i32 112>
 ; CHECK-NEXT:    ret <64 x i8> [[TMP1]]
 ;
   %1 = tail call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> %InVec, <64 x i8> <i8 0, i8 -128, i8 undef, i8 -128, i8 1, i8 -128, i8 undef, i8 -128, i8 2, i8 -128, i8 undef, i8 -128, i8 3, i8 -128, i8 undef, i8 -128, i8 0, i8 -128, i8 undef, i8 -128, i8 1, i8 -128, i8 undef, i8 -128, i8 2, i8 -128, i8 undef, i8 -128, i8 3, i8 -128, i8 undef, i8 -128, i8 0, i8 -128, i8 undef, i8 -128, i8 1, i8 -128, i8 undef, i8 -128, i8 2, i8 -128, i8 undef, i8 -128, i8 3, i8 -128, i8 undef, i8 -128, i8 0, i8 -128, i8 undef, i8 -128, i8 1, i8 -128, i8 undef, i8 -128, i8 2, i8 -128, i8 undef, i8 -128, i8 3, i8 -128, i8 undef, i8 -128>)
@@ -472,7 +472,7 @@ define <64 x i8> @fold_with_allundef_elts_avx512(<64 x i8> %InVec) {
 
 define <16 x i8> @demanded_elts_insertion(<16 x i8> %InVec, <16 x i8> %BaseMask, i8 %M0, i8 %M15) {
 ; CHECK-LABEL: @demanded_elts_insertion(
-; CHECK-NEXT:    [[TMP1:%.*]] = tail call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %InVec, <16 x i8> %BaseMask)
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> [[INVEC:%.*]], <16 x i8> [[BASEMASK:%.*]])
 ; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> undef, <16 x i32> <i32 undef, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 undef>
 ; CHECK-NEXT:    ret <16 x i8> [[TMP2]]
 ;
@@ -485,8 +485,9 @@ define <16 x i8> @demanded_elts_insertion(<16 x i8> %InVec, <16 x i8> %BaseMask,
 
 define <32 x i8> @demanded_elts_insertion_avx2(<32 x i8> %InVec, <32 x i8> %BaseMask, i8 %M0, i8 %M22) {
 ; CHECK-LABEL: @demanded_elts_insertion_avx2(
-; CHECK-NEXT:    [[TMP1:%.*]] = tail call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> %InVec, <32 x i8> %BaseMask)
-; CHECK-NEXT:    ret <32 x i8> [[TMP1]]
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> [[INVEC:%.*]], <32 x i8> [[BASEMASK:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <32 x i8> [[TMP1]], <32 x i8> undef, <32 x i32> <i32 undef, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 undef, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    ret <32 x i8> [[TMP2]]
 ;
   %1 = insertelement <32 x i8> %BaseMask, i8 %M0, i32 0
   %2 = insertelement <32 x i8> %1, i8 %M22, i32 22
@@ -497,8 +498,8 @@ define <32 x i8> @demanded_elts_insertion_avx2(<32 x i8> %InVec, <32 x i8> %Base
 
 define <64 x i8> @demanded_elts_insertion_avx512(<64 x i8> %InVec, <64 x i8> %BaseMask, i8 %M0, i8 %M30) {
 ; CHECK-LABEL: @demanded_elts_insertion_avx512(
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <64 x i8> undef, i8 %M0, i32 0
-; CHECK-NEXT:    [[TMP2:%.*]] = tail call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> %InVec, <64 x i8> [[TMP1]])
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <64 x i8> undef, i8 [[M0:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = tail call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> [[INVEC:%.*]], <64 x i8> [[TMP1]])
 ; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <64 x i8> [[TMP2]], <64 x i8> undef, <64 x i32> zeroinitializer
 ; CHECK-NEXT:    ret <64 x i8> [[TMP3]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-sse.ll b/llvm/test/Transforms/InstCombine/X86/x86-sse.ll
index 830782b3b..d58b083df 100644
--- a/llvm/test/Transforms/InstCombine/X86/x86-sse.ll
+++ b/llvm/test/Transforms/InstCombine/X86/x86-sse.ll
@@ -4,7 +4,7 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
 define float @test_rcp_ss_0(float %a) {
 ; CHECK-LABEL: @test_rcp_ss_0(
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> undef, float %a, i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> undef, float [[A:%.*]], i32 0
 ; CHECK-NEXT:    [[TMP2:%.*]] = tail call <4 x float> @llvm.x86.sse.rcp.ss(<4 x float> [[TMP1]])
 ; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[TMP2]], i32 0
 ; CHECK-NEXT:    ret float [[TMP3]]
@@ -33,7 +33,7 @@ define float @test_rcp_ss_1(float %a) {
 
 define float @test_sqrt_ss_0(float %a) {
 ; CHECK-LABEL: @test_sqrt_ss_0(
-; CHECK-NEXT:    [[TMP1:%.*]] = call float @llvm.sqrt.f32(float %a)
+; CHECK-NEXT:    [[TMP1:%.*]] = call float @llvm.sqrt.f32(float [[A:%.*]])
 ; CHECK-NEXT:    ret float [[TMP1]]
 ;
   %1 = insertelement <4 x float> undef, float %a, i32 0
@@ -60,7 +60,7 @@ define float @test_sqrt_ss_2(float %a) {
 
 define float @test_rsqrt_ss_0(float %a) {
 ; CHECK-LABEL: @test_rsqrt_ss_0(
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> undef, float %a, i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> undef, float [[A:%.*]], i32 0
 ; CHECK-NEXT:    [[TMP2:%.*]] = tail call <4 x float> @llvm.x86.sse.rsqrt.ss(<4 x float> [[TMP1]])
 ; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[TMP2]], i32 0
 ; CHECK-NEXT:    ret float [[TMP3]]
@@ -89,7 +89,7 @@ define float @test_rsqrt_ss_3(float %a) {
 
 define float @test_add_ss_0(float %a, float %b) {
 ; CHECK-LABEL: @test_add_ss_0(
-; CHECK-NEXT:    [[TMP1:%.*]] = fadd float %a, %b
+; CHECK-NEXT:    [[TMP1:%.*]] = fadd float [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    ret float [[TMP1]]
 ;
   %1 = insertelement <4 x float> undef, float %a, i32 0
@@ -121,7 +121,7 @@ define float @test_add_ss_1(float %a, float %b) {
 
 define float @test_sub_ss_0(float %a, float %b) {
 ; CHECK-LABEL: @test_sub_ss_0(
-; CHECK-NEXT:    [[TMP1:%.*]] = fsub float %a, %b
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub float [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    ret float [[TMP1]]
 ;
   %1 = insertelement <4 x float> undef, float %a, i32 0
@@ -153,7 +153,7 @@ define float @test_sub_ss_2(float %a, float %b) {
 
 define float @test_mul_ss_0(float %a, float %b) {
 ; CHECK-LABEL: @test_mul_ss_0(
-; CHECK-NEXT:    [[TMP1:%.*]] = fmul float %a, %b
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul float [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    ret float [[TMP1]]
 ;
   %1 = insertelement <4 x float> undef, float %a, i32 0
@@ -185,7 +185,7 @@ define float @test_mul_ss_3(float %a, float %b) {
 
 define float @test_div_ss_0(float %a, float %b) {
 ; CHECK-LABEL: @test_div_ss_0(
-; CHECK-NEXT:    [[TMP1:%.*]] = fdiv float %a, %b
+; CHECK-NEXT:    [[TMP1:%.*]] = fdiv float [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    ret float [[TMP1]]
 ;
   %1 = insertelement <4 x float> undef, float %a, i32 0
@@ -217,7 +217,7 @@ define float @test_div_ss_1(float %a, float %b) {
 
 define <4 x float> @test_min_ss(<4 x float> %a, <4 x float> %b) {
 ; CHECK-LABEL: @test_min_ss(
-; CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.sse.min.ss(<4 x float> %a, <4 x float> %b)
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.sse.min.ss(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]])
 ; CHECK-NEXT:    ret <4 x float> [[TMP1]]
 ;
   %1 = insertelement <4 x float> %b, float 1.000000e+00, i32 1
@@ -229,8 +229,8 @@ define <4 x float> @test_min_ss(<4 x float> %a, <4 x float> %b) {
 
 define float @test_min_ss_0(float %a, float %b) {
 ; CHECK-LABEL: @test_min_ss_0(
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> undef, float %a, i32 0
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x float> undef, float %b, i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> undef, float [[A:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x float> undef, float [[B:%.*]], i32 0
 ; CHECK-NEXT:    [[TMP3:%.*]] = tail call <4 x float> @llvm.x86.sse.min.ss(<4 x float> [[TMP1]], <4 x float> [[TMP2]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[TMP3]], i32 0
 ; CHECK-NEXT:    ret float [[TMP4]]
@@ -264,7 +264,7 @@ define float @test_min_ss_2(float %a, float %b) {
 
 define <4 x float> @test_max_ss(<4 x float> %a, <4 x float> %b) {
 ; CHECK-LABEL: @test_max_ss(
-; CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.sse.max.ss(<4 x float> %a, <4 x float> %b)
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.sse.max.ss(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]])
 ; CHECK-NEXT:    ret <4 x float> [[TMP1]]
 ;
   %1 = insertelement <4 x float> %b, float 1.000000e+00, i32 1
@@ -276,8 +276,8 @@ define <4 x float> @test_max_ss(<4 x float> %a, <4 x float> %b) {
 
 define float @test_max_ss_0(float %a, float %b) {
 ; CHECK-LABEL: @test_max_ss_0(
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> undef, float %a, i32 0
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x float> undef, float %b, i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> undef, float [[A:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x float> undef, float [[B:%.*]], i32 0
 ; CHECK-NEXT:    [[TMP3:%.*]] = tail call <4 x float> @llvm.x86.sse.max.ss(<4 x float> [[TMP1]], <4 x float> [[TMP2]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[TMP3]], i32 0
 ; CHECK-NEXT:    ret float [[TMP4]]
@@ -311,7 +311,7 @@ define float @test_max_ss_3(float %a, float %b) {
 
 define <4 x float> @test_cmp_ss(<4 x float> %a, <4 x float> %b) {
 ; CHECK-LABEL: @test_cmp_ss(
-; CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.sse.cmp.ss(<4 x float> %a, <4 x float> %b, i8 0)
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.sse.cmp.ss(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], i8 0)
 ; CHECK-NEXT:    ret <4 x float> [[TMP1]]
 ;
   %1 = insertelement <4 x float> %b, float 1.000000e+00, i32 1
@@ -323,8 +323,8 @@ define <4 x float> @test_cmp_ss(<4 x float> %a, <4 x float> %b) {
 
 define float @test_cmp_ss_0(float %a, float %b) {
 ; CHECK-LABEL: @test_cmp_ss_0(
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> undef, float %a, i32 0
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x float> undef, float %b, i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> undef, float [[A:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x float> undef, float [[B:%.*]], i32 0
 ; CHECK-NEXT:    [[TMP3:%.*]] = tail call <4 x float> @llvm.x86.sse.cmp.ss(<4 x float> [[TMP1]], <4 x float> [[TMP2]], i8 0)
 ; CHECK-NEXT:    [[R:%.*]] = extractelement <4 x float> [[TMP3]], i32 0
 ; CHECK-NEXT:    ret float [[R]]
@@ -358,8 +358,8 @@ define float @test_cmp_ss_1(float %a, float %b) {
 
 define i32 @test_comieq_ss_0(float %a, float %b) {
 ; CHECK-LABEL: @test_comieq_ss_0(
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> undef, float %a, i32 0
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x float> undef, float %b, i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> undef, float [[A:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x float> undef, float [[B:%.*]], i32 0
 ; CHECK-NEXT:    [[TMP3:%.*]] = tail call i32 @llvm.x86.sse.comieq.ss(<4 x float> [[TMP1]], <4 x float> [[TMP2]])
 ; CHECK-NEXT:    ret i32 [[TMP3]]
 ;
@@ -377,8 +377,8 @@ define i32 @test_comieq_ss_0(float %a, float %b) {
 
 define i32 @test_comige_ss_0(float %a, float %b) {
 ; CHECK-LABEL: @test_comige_ss_0(
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> undef, float %a, i32 0
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x float> undef, float %b, i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> undef, float [[A:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x float> undef, float [[B:%.*]], i32 0
 ; CHECK-NEXT:    [[TMP3:%.*]] = tail call i32 @llvm.x86.sse.comige.ss(<4 x float> [[TMP1]], <4 x float> [[TMP2]])
 ; CHECK-NEXT:    ret i32 [[TMP3]]
 ;
@@ -396,8 +396,8 @@ define i32 @test_comige_ss_0(float %a, float %b) {
 
 define i32 @test_comigt_ss_0(float %a, float %b) {
 ; CHECK-LABEL: @test_comigt_ss_0(
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> undef, float %a, i32 0
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x float> undef, float %b, i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> undef, float [[A:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x float> undef, float [[B:%.*]], i32 0
 ; CHECK-NEXT:    [[TMP3:%.*]] = tail call i32 @llvm.x86.sse.comigt.ss(<4 x float> [[TMP1]], <4 x float> [[TMP2]])
 ; CHECK-NEXT:    ret i32 [[TMP3]]
 ;
@@ -415,8 +415,8 @@ define i32 @test_comigt_ss_0(float %a, float %b) {
 
 define i32 @test_comile_ss_0(float %a, float %b) {
 ; CHECK-LABEL: @test_comile_ss_0(
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> undef, float %a, i32 0
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x float> undef, float %b, i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> undef, float [[A:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x float> undef, float [[B:%.*]], i32 0
 ; CHECK-NEXT:    [[TMP3:%.*]] = tail call i32 @llvm.x86.sse.comile.ss(<4 x float> [[TMP1]], <4 x float> [[TMP2]])
 ; CHECK-NEXT:    ret i32 [[TMP3]]
 ;
@@ -434,8 +434,8 @@ define i32 @test_comile_ss_0(float %a, float %b) {
 
 define i32 @test_comilt_ss_0(float %a, float %b) {
 ; CHECK-LABEL: @test_comilt_ss_0(
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> undef, float %a, i32 0
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x float> undef, float %b, i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> undef, float [[A:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x float> undef, float [[B:%.*]], i32 0
 ; CHECK-NEXT:    [[TMP3:%.*]] = tail call i32 @llvm.x86.sse.comilt.ss(<4 x float> [[TMP1]], <4 x float> [[TMP2]])
 ; CHECK-NEXT:    ret i32 [[TMP3]]
 ;
@@ -453,8 +453,8 @@ define i32 @test_comilt_ss_0(float %a, float %b) {
 
 define i32 @test_comineq_ss_0(float %a, float %b) {
 ; CHECK-LABEL: @test_comineq_ss_0(
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> undef, float %a, i32 0
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x float> undef, float %b, i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> undef, float [[A:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x float> undef, float [[B:%.*]], i32 0
 ; CHECK-NEXT:    [[TMP3:%.*]] = tail call i32 @llvm.x86.sse.comineq.ss(<4 x float> [[TMP1]], <4 x float> [[TMP2]])
 ; CHECK-NEXT:    ret i32 [[TMP3]]
 ;
@@ -472,8 +472,8 @@ define i32 @test_comineq_ss_0(float %a, float %b) {
 
 define i32 @test_ucomieq_ss_0(float %a, float %b) {
 ; CHECK-LABEL: @test_ucomieq_ss_0(
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> undef, float %a, i32 0
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x float> undef, float %b, i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> undef, float [[A:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x float> undef, float [[B:%.*]], i32 0
 ; CHECK-NEXT:    [[TMP3:%.*]] = tail call i32 @llvm.x86.sse.ucomieq.ss(<4 x float> [[TMP1]], <4 x float> [[TMP2]])
 ; CHECK-NEXT:    ret i32 [[TMP3]]
 ;
@@ -491,8 +491,8 @@ define i32 @test_ucomieq_ss_0(float %a, float %b) {
 
 define i32 @test_ucomige_ss_0(float %a, float %b) {
 ; CHECK-LABEL: @test_ucomige_ss_0(
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> undef, float %a, i32 0
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x float> undef, float %b, i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> undef, float [[A:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x float> undef, float [[B:%.*]], i32 0
 ; CHECK-NEXT:    [[TMP3:%.*]] = tail call i32 @llvm.x86.sse.ucomige.ss(<4 x float> [[TMP1]], <4 x float> [[TMP2]])
 ; CHECK-NEXT:    ret i32 [[TMP3]]
 ;
@@ -510,8 +510,8 @@ define i32 @test_ucomige_ss_0(float %a, float %b) {
 
 define i32 @test_ucomigt_ss_0(float %a, float %b) {
 ; CHECK-LABEL: @test_ucomigt_ss_0(
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> undef, float %a, i32 0
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x float> undef, float %b, i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> undef, float [[A:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x float> undef, float [[B:%.*]], i32 0
 ; CHECK-NEXT:    [[TMP3:%.*]] = tail call i32 @llvm.x86.sse.ucomigt.ss(<4 x float> [[TMP1]], <4 x float> [[TMP2]])
 ; CHECK-NEXT:    ret i32 [[TMP3]]
 ;
@@ -529,8 +529,8 @@ define i32 @test_ucomigt_ss_0(float %a, float %b) {
 
 define i32 @test_ucomile_ss_0(float %a, float %b) {
 ; CHECK-LABEL: @test_ucomile_ss_0(
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> undef, float %a, i32 0
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x float> undef, float %b, i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> undef, float [[A:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x float> undef, float [[B:%.*]], i32 0
 ; CHECK-NEXT:    [[TMP3:%.*]] = tail call i32 @llvm.x86.sse.ucomile.ss(<4 x float> [[TMP1]], <4 x float> [[TMP2]])
 ; CHECK-NEXT:    ret i32 [[TMP3]]
 ;
@@ -548,8 +548,8 @@ define i32 @test_ucomile_ss_0(float %a, float %b) {
 
 define i32 @test_ucomilt_ss_0(float %a, float %b) {
 ; CHECK-LABEL: @test_ucomilt_ss_0(
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> undef, float %a, i32 0
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x float> undef, float %b, i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> undef, float [[A:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x float> undef, float [[B:%.*]], i32 0
 ; CHECK-NEXT:    [[TMP3:%.*]] = tail call i32 @llvm.x86.sse.ucomilt.ss(<4 x float> [[TMP1]], <4 x float> [[TMP2]])
 ; CHECK-NEXT:    ret i32 [[TMP3]]
 ;
@@ -567,8 +567,8 @@ define i32 @test_ucomilt_ss_0(float %a, float %b) {
 
 define i32 @test_ucomineq_ss_0(float %a, float %b) {
 ; CHECK-LABEL: @test_ucomineq_ss_0(
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> undef, float %a, i32 0
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x float> undef, float %b, i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> undef, float [[A:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x float> undef, float [[B:%.*]], i32 0
 ; CHECK-NEXT:    [[TMP3:%.*]] = tail call i32 @llvm.x86.sse.ucomineq.ss(<4 x float> [[TMP1]], <4 x float> [[TMP2]])
 ; CHECK-NEXT:    ret i32 [[TMP3]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-sse4a.ll b/llvm/test/Transforms/InstCombine/X86/x86-sse4a.ll
index e33a382b7..95b02b224 100644
--- a/llvm/test/Transforms/InstCombine/X86/x86-sse4a.ll
+++ b/llvm/test/Transforms/InstCombine/X86/x86-sse4a.ll
@@ -24,7 +24,10 @@ define <2 x i64> @test_extrq_zero_arg0(<2 x i64> %x, <16 x i8> %y) {
 
 define <2 x i64> @test_extrq_zero_arg1(<2 x i64> %x, <16 x i8> %y) {
 ; CHECK-LABEL: @test_extrq_zero_arg1(
-; CHECK-NEXT:    ret <2 x i64> [[X:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64> [[X:%.*]] to <16 x i8>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i8> [[TMP2]] to <2 x i64>
+; CHECK-NEXT:    ret <2 x i64> [[TMP3]]
 ;
   %1 = tail call <2 x i64> @llvm.x86.sse4a.extrq(<2 x i64> %x, <16 x i8> zeroinitializer) nounwind
   ret <2 x i64> %1
@@ -57,7 +60,10 @@ define <2 x i64> @test_extrq_constant_undef(<2 x i64> %x, <16 x i8> %y) {
 
 define <2 x i64> @test_extrq_call_constexpr(<2 x i64> %x) {
 ; CHECK-LABEL: @test_extrq_call_constexpr(
-; CHECK-NEXT:    ret <2 x i64> [[X:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64> [[X:%.*]] to <16 x i8>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i8> [[TMP2]] to <2 x i64>
+; CHECK-NEXT:    ret <2 x i64> [[TMP3]]
 ;
   %1 = call <2 x i64> @llvm.x86.sse4a.extrq(<2 x i64> %x, <16 x i8> bitcast (<2 x i64> <i64 0, i64 undef> to <16 x i8>))
   ret <2 x i64> %1
@@ -235,7 +241,10 @@ define <2 x i64> @test_insertqi_call_constexpr(<2 x i64> %x) {
 ; second arg
 define <2 x i64> @testInsert64Bits(<2 x i64> %v, <2 x i64> %i) {
 ; CHECK-LABEL: @testInsert64Bits(
-; CHECK-NEXT:    ret <2 x i64> [[I:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64> [[I:%.*]] to <16 x i8>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i8> [[TMP2]] to <2 x i64>
+; CHECK-NEXT:    ret <2 x i64> [[TMP3]]
 ;
   %1 = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %v, <2 x i64> %i, i8 64, i8 0)
   ret <2 x i64> %1
@@ -243,7 +252,10 @@ define <2 x i64> @testInsert64Bits(<2 x i64> %v, <2 x i64> %i) {
 
 define <2 x i64> @testZeroLength(<2 x i64> %v, <2 x i64> %i) {
 ; CHECK-LABEL: @testZeroLength(
-; CHECK-NEXT:    ret <2 x i64> [[I:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64> [[I:%.*]] to <16 x i8>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i8> [[TMP2]] to <2 x i64>
+; CHECK-NEXT:    ret <2 x i64> [[TMP3]]
 ;
   %1 = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %v, <2 x i64> %i, i8 0, i8 0)
   ret <2 x i64> %1
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-vector-shifts.ll b/llvm/test/Transforms/InstCombine/X86/x86-vector-shifts.ll
index 306577fae..1fc6b55b6 100644
--- a/llvm/test/Transforms/InstCombine/X86/x86-vector-shifts.ll
+++ b/llvm/test/Transforms/InstCombine/X86/x86-vector-shifts.ll
@@ -8,7 +8,7 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
 define <8 x i16> @sse2_psrai_w_0(<8 x i16> %v) {
 ; CHECK-LABEL: @sse2_psrai_w_0(
-; CHECK-NEXT:    ret <8 x i16> %v
+; CHECK-NEXT:    ret <8 x i16> [[V:%.*]]
 ;
   %1 = tail call <8 x i16> @llvm.x86.sse2.psrai.w(<8 x i16> %v, i32 0)
   ret <8 x i16> %1
@@ -16,7 +16,7 @@ define <8 x i16> @sse2_psrai_w_0(<8 x i16> %v) {
 
 define <8 x i16> @sse2_psrai_w_15(<8 x i16> %v) {
 ; CHECK-LABEL: @sse2_psrai_w_15(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i16> %v, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i16> [[V:%.*]], <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
 ; CHECK-NEXT:    ret <8 x i16> [[TMP1]]
 ;
   %1 = tail call <8 x i16> @llvm.x86.sse2.psrai.w(<8 x i16> %v, i32 15)
@@ -25,7 +25,7 @@ define <8 x i16> @sse2_psrai_w_15(<8 x i16> %v) {
 
 define <8 x i16> @sse2_psrai_w_64(<8 x i16> %v) {
 ; CHECK-LABEL: @sse2_psrai_w_64(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i16> %v, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i16> [[V:%.*]], <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
 ; CHECK-NEXT:    ret <8 x i16> [[TMP1]]
 ;
   %1 = tail call <8 x i16> @llvm.x86.sse2.psrai.w(<8 x i16> %v, i32 64)
@@ -34,7 +34,7 @@ define <8 x i16> @sse2_psrai_w_64(<8 x i16> %v) {
 
 define <4 x i32> @sse2_psrai_d_0(<4 x i32> %v) {
 ; CHECK-LABEL: @sse2_psrai_d_0(
-; CHECK-NEXT:    ret <4 x i32> %v
+; CHECK-NEXT:    ret <4 x i32> [[V:%.*]]
 ;
   %1 = tail call <4 x i32> @llvm.x86.sse2.psrai.d(<4 x i32> %v, i32 0)
   ret <4 x i32> %1
@@ -42,7 +42,7 @@ define <4 x i32> @sse2_psrai_d_0(<4 x i32> %v) {
 
 define <4 x i32> @sse2_psrai_d_15(<4 x i32> %v) {
 ; CHECK-LABEL: @sse2_psrai_d_15(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <4 x i32> %v, <i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <4 x i32> [[V:%.*]], <i32 15, i32 15, i32 15, i32 15>
 ; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
 ;
   %1 = tail call <4 x i32> @llvm.x86.sse2.psrai.d(<4 x i32> %v, i32 15)
@@ -51,7 +51,7 @@ define <4 x i32> @sse2_psrai_d_15(<4 x i32> %v) {
 
 define <4 x i32> @sse2_psrai_d_64(<4 x i32> %v) {
 ; CHECK-LABEL: @sse2_psrai_d_64(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <4 x i32> %v, <i32 31, i32 31, i32 31, i32 31>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <4 x i32> [[V:%.*]], <i32 31, i32 31, i32 31, i32 31>
 ; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
 ;
   %1 = tail call <4 x i32> @llvm.x86.sse2.psrai.d(<4 x i32> %v, i32 64)
@@ -60,7 +60,7 @@ define <4 x i32> @sse2_psrai_d_64(<4 x i32> %v) {
 
 define <16 x i16> @avx2_psrai_w_0(<16 x i16> %v) {
 ; CHECK-LABEL: @avx2_psrai_w_0(
-; CHECK-NEXT:    ret <16 x i16> %v
+; CHECK-NEXT:    ret <16 x i16> [[V:%.*]]
 ;
   %1 = tail call <16 x i16> @llvm.x86.avx2.psrai.w(<16 x i16> %v, i32 0)
   ret <16 x i16> %1
@@ -68,7 +68,7 @@ define <16 x i16> @avx2_psrai_w_0(<16 x i16> %v) {
 
 define <16 x i16> @avx2_psrai_w_15(<16 x i16> %v) {
 ; CHECK-LABEL: @avx2_psrai_w_15(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <16 x i16> %v, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <16 x i16> [[V:%.*]], <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
 ; CHECK-NEXT:    ret <16 x i16> [[TMP1]]
 ;
   %1 = tail call <16 x i16> @llvm.x86.avx2.psrai.w(<16 x i16> %v, i32 15)
@@ -77,7 +77,7 @@ define <16 x i16> @avx2_psrai_w_15(<16 x i16> %v) {
 
 define <16 x i16> @avx2_psrai_w_64(<16 x i16> %v) {
 ; CHECK-LABEL: @avx2_psrai_w_64(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <16 x i16> %v, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <16 x i16> [[V:%.*]], <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
 ; CHECK-NEXT:    ret <16 x i16> [[TMP1]]
 ;
   %1 = tail call <16 x i16> @llvm.x86.avx2.psrai.w(<16 x i16> %v, i32 64)
@@ -86,7 +86,7 @@ define <16 x i16> @avx2_psrai_w_64(<16 x i16> %v) {
 
 define <8 x i32> @avx2_psrai_d_0(<8 x i32> %v) {
 ; CHECK-LABEL: @avx2_psrai_d_0(
-; CHECK-NEXT:    ret <8 x i32> %v
+; CHECK-NEXT:    ret <8 x i32> [[V:%.*]]
 ;
   %1 = tail call <8 x i32> @llvm.x86.avx2.psrai.d(<8 x i32> %v, i32 0)
   ret <8 x i32> %1
@@ -94,7 +94,7 @@ define <8 x i32> @avx2_psrai_d_0(<8 x i32> %v) {
 
 define <8 x i32> @avx2_psrai_d_15(<8 x i32> %v) {
 ; CHECK-LABEL: @avx2_psrai_d_15(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i32> %v, <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i32> [[V:%.*]], <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
 ; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
 ;
   %1 = tail call <8 x i32> @llvm.x86.avx2.psrai.d(<8 x i32> %v, i32 15)
@@ -103,7 +103,7 @@ define <8 x i32> @avx2_psrai_d_15(<8 x i32> %v) {
 
 define <8 x i32> @avx2_psrai_d_64(<8 x i32> %v) {
 ; CHECK-LABEL: @avx2_psrai_d_64(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i32> %v, <i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i32> [[V:%.*]], <i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31>
 ; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
 ;
   %1 = tail call <8 x i32> @llvm.x86.avx2.psrai.d(<8 x i32> %v, i32 64)
@@ -112,7 +112,7 @@ define <8 x i32> @avx2_psrai_d_64(<8 x i32> %v) {
 
 define <2 x i64> @avx512_psrai_q_128_0(<2 x i64> %v) {
 ; CHECK-LABEL: @avx512_psrai_q_128_0(
-; CHECK-NEXT:    ret <2 x i64> %v
+; CHECK-NEXT:    ret <2 x i64> [[V:%.*]]
 ;
   %1 = tail call <2 x i64> @llvm.x86.avx512.psrai.q.128(<2 x i64> %v, i32 0)
   ret <2 x i64> %1
@@ -120,7 +120,7 @@ define <2 x i64> @avx512_psrai_q_128_0(<2 x i64> %v) {
 
 define <2 x i64> @avx512_psrai_q_128_15(<2 x i64> %v) {
 ; CHECK-LABEL: @avx512_psrai_q_128_15(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <2 x i64> %v, <i64 15, i64 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <2 x i64> [[V:%.*]], <i64 15, i64 15>
 ; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
 ;
   %1 = tail call <2 x i64> @llvm.x86.avx512.psrai.q.128(<2 x i64> %v, i32 15)
@@ -129,7 +129,7 @@ define <2 x i64> @avx512_psrai_q_128_15(<2 x i64> %v) {
 
 define <2 x i64> @avx512_psrai_q_128_64(<2 x i64> %v) {
 ; CHECK-LABEL: @avx512_psrai_q_128_64(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <2 x i64> %v, <i64 63, i64 63>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <2 x i64> [[V:%.*]], <i64 63, i64 63>
 ; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
 ;
   %1 = tail call <2 x i64> @llvm.x86.avx512.psrai.q.128(<2 x i64> %v, i32 64)
@@ -138,7 +138,7 @@ define <2 x i64> @avx512_psrai_q_128_64(<2 x i64> %v) {
 
 define <4 x i64> @avx512_psrai_q_256_0(<4 x i64> %v) {
 ; CHECK-LABEL: @avx512_psrai_q_256_0(
-; CHECK-NEXT:    ret <4 x i64> %v
+; CHECK-NEXT:    ret <4 x i64> [[V:%.*]]
 ;
   %1 = tail call <4 x i64> @llvm.x86.avx512.psrai.q.256(<4 x i64> %v, i32 0)
   ret <4 x i64> %1
@@ -146,7 +146,7 @@ define <4 x i64> @avx512_psrai_q_256_0(<4 x i64> %v) {
 
 define <4 x i64> @avx512_psrai_q_256_15(<4 x i64> %v) {
 ; CHECK-LABEL: @avx512_psrai_q_256_15(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <4 x i64> %v, <i64 15, i64 15, i64 15, i64 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <4 x i64> [[V:%.*]], <i64 15, i64 15, i64 15, i64 15>
 ; CHECK-NEXT:    ret <4 x i64> [[TMP1]]
 ;
   %1 = tail call <4 x i64> @llvm.x86.avx512.psrai.q.256(<4 x i64> %v, i32 15)
@@ -155,7 +155,7 @@ define <4 x i64> @avx512_psrai_q_256_15(<4 x i64> %v) {
 
 define <4 x i64> @avx512_psrai_q_256_64(<4 x i64> %v) {
 ; CHECK-LABEL: @avx512_psrai_q_256_64(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <4 x i64> %v, <i64 63, i64 63, i64 63, i64 63>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <4 x i64> [[V:%.*]], <i64 63, i64 63, i64 63, i64 63>
 ; CHECK-NEXT:    ret <4 x i64> [[TMP1]]
 ;
   %1 = tail call <4 x i64> @llvm.x86.avx512.psrai.q.256(<4 x i64> %v, i32 64)
@@ -164,7 +164,7 @@ define <4 x i64> @avx512_psrai_q_256_64(<4 x i64> %v) {
 
 define <32 x i16> @avx512_psrai_w_512_0(<32 x i16> %v) {
 ; CHECK-LABEL: @avx512_psrai_w_512_0(
-; CHECK-NEXT:    ret <32 x i16> %v
+; CHECK-NEXT:    ret <32 x i16> [[V:%.*]]
 ;
   %1 = tail call <32 x i16> @llvm.x86.avx512.psrai.w.512(<32 x i16> %v, i32 0)
   ret <32 x i16> %1
@@ -172,7 +172,7 @@ define <32 x i16> @avx512_psrai_w_512_0(<32 x i16> %v) {
 
 define <32 x i16> @avx512_psrai_w_512_15(<32 x i16> %v) {
 ; CHECK-LABEL: @avx512_psrai_w_512_15(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <32 x i16> %v, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <32 x i16> [[V:%.*]], <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
 ; CHECK-NEXT:    ret <32 x i16> [[TMP1]]
 ;
   %1 = tail call <32 x i16> @llvm.x86.avx512.psrai.w.512(<32 x i16> %v, i32 15)
@@ -181,7 +181,7 @@ define <32 x i16> @avx512_psrai_w_512_15(<32 x i16> %v) {
 
 define <32 x i16> @avx512_psrai_w_512_64(<32 x i16> %v) {
 ; CHECK-LABEL: @avx512_psrai_w_512_64(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <32 x i16> %v, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <32 x i16> [[V:%.*]], <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
 ; CHECK-NEXT:    ret <32 x i16> [[TMP1]]
 ;
   %1 = tail call <32 x i16> @llvm.x86.avx512.psrai.w.512(<32 x i16> %v, i32 64)
@@ -190,7 +190,7 @@ define <32 x i16> @avx512_psrai_w_512_64(<32 x i16> %v) {
 
 define <16 x i32> @avx512_psrai_d_512_0(<16 x i32> %v) {
 ; CHECK-LABEL: @avx512_psrai_d_512_0(
-; CHECK-NEXT:    ret <16 x i32> %v
+; CHECK-NEXT:    ret <16 x i32> [[V:%.*]]
 ;
   %1 = tail call <16 x i32> @llvm.x86.avx512.psrai.d.512(<16 x i32> %v, i32 0)
   ret <16 x i32> %1
@@ -198,7 +198,7 @@ define <16 x i32> @avx512_psrai_d_512_0(<16 x i32> %v) {
 
 define <16 x i32> @avx512_psrai_d_512_15(<16 x i32> %v) {
 ; CHECK-LABEL: @avx512_psrai_d_512_15(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <16 x i32> %v, <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <16 x i32> [[V:%.*]], <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
 ; CHECK-NEXT:    ret <16 x i32> [[TMP1]]
 ;
   %1 = tail call <16 x i32> @llvm.x86.avx512.psrai.d.512(<16 x i32> %v, i32 15)
@@ -207,7 +207,7 @@ define <16 x i32> @avx512_psrai_d_512_15(<16 x i32> %v) {
 
 define <16 x i32> @avx512_psrai_d_512_64(<16 x i32> %v) {
 ; CHECK-LABEL: @avx512_psrai_d_512_64(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <16 x i32> %v, <i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <16 x i32> [[V:%.*]], <i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31>
 ; CHECK-NEXT:    ret <16 x i32> [[TMP1]]
 ;
   %1 = tail call <16 x i32> @llvm.x86.avx512.psrai.d.512(<16 x i32> %v, i32 64)
@@ -216,7 +216,7 @@ define <16 x i32> @avx512_psrai_d_512_64(<16 x i32> %v) {
 
 define <8 x i64> @avx512_psrai_q_512_0(<8 x i64> %v) {
 ; CHECK-LABEL: @avx512_psrai_q_512_0(
-; CHECK-NEXT:    ret <8 x i64> %v
+; CHECK-NEXT:    ret <8 x i64> [[V:%.*]]
 ;
   %1 = tail call <8 x i64> @llvm.x86.avx512.psrai.q.512(<8 x i64> %v, i32 0)
   ret <8 x i64> %1
@@ -224,7 +224,7 @@ define <8 x i64> @avx512_psrai_q_512_0(<8 x i64> %v) {
 
 define <8 x i64> @avx512_psrai_q_512_15(<8 x i64> %v) {
 ; CHECK-LABEL: @avx512_psrai_q_512_15(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i64> %v, <i64 15, i64 15, i64 15, i64 15, i64 15, i64 15, i64 15, i64 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i64> [[V:%.*]], <i64 15, i64 15, i64 15, i64 15, i64 15, i64 15, i64 15, i64 15>
 ; CHECK-NEXT:    ret <8 x i64> [[TMP1]]
 ;
   %1 = tail call <8 x i64> @llvm.x86.avx512.psrai.q.512(<8 x i64> %v, i32 15)
@@ -233,7 +233,7 @@ define <8 x i64> @avx512_psrai_q_512_15(<8 x i64> %v) {
 
 define <8 x i64> @avx512_psrai_q_512_64(<8 x i64> %v) {
 ; CHECK-LABEL: @avx512_psrai_q_512_64(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i64> %v, <i64 63, i64 63, i64 63, i64 63, i64 63, i64 63, i64 63, i64 63>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i64> [[V:%.*]], <i64 63, i64 63, i64 63, i64 63, i64 63, i64 63, i64 63, i64 63>
 ; CHECK-NEXT:    ret <8 x i64> [[TMP1]]
 ;
   %1 = tail call <8 x i64> @llvm.x86.avx512.psrai.q.512(<8 x i64> %v, i32 64)
@@ -246,7 +246,7 @@ define <8 x i64> @avx512_psrai_q_512_64(<8 x i64> %v) {
 
 define <8 x i16> @sse2_psrli_w_0(<8 x i16> %v) {
 ; CHECK-LABEL: @sse2_psrli_w_0(
-; CHECK-NEXT:    ret <8 x i16> %v
+; CHECK-NEXT:    ret <8 x i16> [[V:%.*]]
 ;
   %1 = tail call <8 x i16> @llvm.x86.sse2.psrli.w(<8 x i16> %v, i32 0)
   ret <8 x i16> %1
@@ -254,7 +254,7 @@ define <8 x i16> @sse2_psrli_w_0(<8 x i16> %v) {
 
 define <8 x i16> @sse2_psrli_w_15(<8 x i16> %v) {
 ; CHECK-LABEL: @sse2_psrli_w_15(
-; CHECK-NEXT:    [[TMP1:%.*]] = lshr <8 x i16> %v, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <8 x i16> [[V:%.*]], <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
 ; CHECK-NEXT:    ret <8 x i16> [[TMP1]]
 ;
   %1 = tail call <8 x i16> @llvm.x86.sse2.psrli.w(<8 x i16> %v, i32 15)
@@ -271,7 +271,7 @@ define <8 x i16> @sse2_psrli_w_64(<8 x i16> %v) {
 
 define <4 x i32> @sse2_psrli_d_0(<4 x i32> %v) {
 ; CHECK-LABEL: @sse2_psrli_d_0(
-; CHECK-NEXT:    ret <4 x i32> %v
+; CHECK-NEXT:    ret <4 x i32> [[V:%.*]]
 ;
   %1 = tail call <4 x i32> @llvm.x86.sse2.psrli.d(<4 x i32> %v, i32 0)
   ret <4 x i32> %1
@@ -279,7 +279,7 @@ define <4 x i32> @sse2_psrli_d_0(<4 x i32> %v) {
 
 define <4 x i32> @sse2_psrli_d_15(<4 x i32> %v) {
 ; CHECK-LABEL: @sse2_psrli_d_15(
-; CHECK-NEXT:    [[TMP1:%.*]] = lshr <4 x i32> %v, <i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <4 x i32> [[V:%.*]], <i32 15, i32 15, i32 15, i32 15>
 ; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
 ;
   %1 = tail call <4 x i32> @llvm.x86.sse2.psrli.d(<4 x i32> %v, i32 15)
@@ -296,7 +296,7 @@ define <4 x i32> @sse2_psrli_d_64(<4 x i32> %v) {
 
 define <2 x i64> @sse2_psrli_q_0(<2 x i64> %v) {
 ; CHECK-LABEL: @sse2_psrli_q_0(
-; CHECK-NEXT:    ret <2 x i64> %v
+; CHECK-NEXT:    ret <2 x i64> [[V:%.*]]
 ;
   %1 = tail call <2 x i64> @llvm.x86.sse2.psrli.q(<2 x i64> %v, i32 0)
   ret <2 x i64> %1
@@ -304,7 +304,7 @@ define <2 x i64> @sse2_psrli_q_0(<2 x i64> %v) {
 
 define <2 x i64> @sse2_psrli_q_15(<2 x i64> %v) {
 ; CHECK-LABEL: @sse2_psrli_q_15(
-; CHECK-NEXT:    [[TMP1:%.*]] = lshr <2 x i64> %v, <i64 15, i64 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <2 x i64> [[V:%.*]], <i64 15, i64 15>
 ; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
 ;
   %1 = tail call <2 x i64> @llvm.x86.sse2.psrli.q(<2 x i64> %v, i32 15)
@@ -321,7 +321,7 @@ define <2 x i64> @sse2_psrli_q_64(<2 x i64> %v) {
 
 define <16 x i16> @avx2_psrli_w_0(<16 x i16> %v) {
 ; CHECK-LABEL: @avx2_psrli_w_0(
-; CHECK-NEXT:    ret <16 x i16> %v
+; CHECK-NEXT:    ret <16 x i16> [[V:%.*]]
 ;
   %1 = tail call <16 x i16> @llvm.x86.avx2.psrli.w(<16 x i16> %v, i32 0)
   ret <16 x i16> %1
@@ -329,7 +329,7 @@ define <16 x i16> @avx2_psrli_w_0(<16 x i16> %v) {
 
 define <16 x i16> @avx2_psrli_w_15(<16 x i16> %v) {
 ; CHECK-LABEL: @avx2_psrli_w_15(
-; CHECK-NEXT:    [[TMP1:%.*]] = lshr <16 x i16> %v, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <16 x i16> [[V:%.*]], <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
 ; CHECK-NEXT:    ret <16 x i16> [[TMP1]]
 ;
   %1 = tail call <16 x i16> @llvm.x86.avx2.psrli.w(<16 x i16> %v, i32 15)
@@ -346,7 +346,7 @@ define <16 x i16> @avx2_psrli_w_64(<16 x i16> %v) {
 
 define <8 x i32> @avx2_psrli_d_0(<8 x i32> %v) {
 ; CHECK-LABEL: @avx2_psrli_d_0(
-; CHECK-NEXT:    ret <8 x i32> %v
+; CHECK-NEXT:    ret <8 x i32> [[V:%.*]]
 ;
   %1 = tail call <8 x i32> @llvm.x86.avx2.psrli.d(<8 x i32> %v, i32 0)
   ret <8 x i32> %1
@@ -354,7 +354,7 @@ define <8 x i32> @avx2_psrli_d_0(<8 x i32> %v) {
 
 define <8 x i32> @avx2_psrli_d_15(<8 x i32> %v) {
 ; CHECK-LABEL: @avx2_psrli_d_15(
-; CHECK-NEXT:    [[TMP1:%.*]] = lshr <8 x i32> %v, <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <8 x i32> [[V:%.*]], <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
 ; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
 ;
   %1 = tail call <8 x i32> @llvm.x86.avx2.psrli.d(<8 x i32> %v, i32 15)
@@ -371,7 +371,7 @@ define <8 x i32> @avx2_psrli_d_64(<8 x i32> %v) {
 
 define <4 x i64> @avx2_psrli_q_0(<4 x i64> %v) {
 ; CHECK-LABEL: @avx2_psrli_q_0(
-; CHECK-NEXT:    ret <4 x i64> %v
+; CHECK-NEXT:    ret <4 x i64> [[V:%.*]]
 ;
   %1 = tail call <4 x i64> @llvm.x86.avx2.psrli.q(<4 x i64> %v, i32 0)
   ret <4 x i64> %1
@@ -379,7 +379,7 @@ define <4 x i64> @avx2_psrli_q_0(<4 x i64> %v) {
 
 define <4 x i64> @avx2_psrli_q_15(<4 x i64> %v) {
 ; CHECK-LABEL: @avx2_psrli_q_15(
-; CHECK-NEXT:    [[TMP1:%.*]] = lshr <4 x i64> %v, <i64 15, i64 15, i64 15, i64 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <4 x i64> [[V:%.*]], <i64 15, i64 15, i64 15, i64 15>
 ; CHECK-NEXT:    ret <4 x i64> [[TMP1]]
 ;
   %1 = tail call <4 x i64> @llvm.x86.avx2.psrli.q(<4 x i64> %v, i32 15)
@@ -396,7 +396,7 @@ define <4 x i64> @avx2_psrli_q_64(<4 x i64> %v) {
 
 define <32 x i16> @avx512_psrli_w_512_0(<32 x i16> %v) {
 ; CHECK-LABEL: @avx512_psrli_w_512_0(
-; CHECK-NEXT:    ret <32 x i16> %v
+; CHECK-NEXT:    ret <32 x i16> [[V:%.*]]
 ;
   %1 = tail call <32 x i16> @llvm.x86.avx512.psrli.w.512(<32 x i16> %v, i32 0)
   ret <32 x i16> %1
@@ -404,7 +404,7 @@ define <32 x i16> @avx512_psrli_w_512_0(<32 x i16> %v) {
 
 define <32 x i16> @avx512_psrli_w_512_15(<32 x i16> %v) {
 ; CHECK-LABEL: @avx512_psrli_w_512_15(
-; CHECK-NEXT:    [[TMP1:%.*]] = lshr <32 x i16> %v, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <32 x i16> [[V:%.*]], <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
 ; CHECK-NEXT:    ret <32 x i16> [[TMP1]]
 ;
   %1 = tail call <32 x i16> @llvm.x86.avx512.psrli.w.512(<32 x i16> %v, i32 15)
@@ -421,7 +421,7 @@ define <32 x i16> @avx512_psrli_w_512_64(<32 x i16> %v) {
 
 define <16 x i32> @avx512_psrli_d_512_0(<16 x i32> %v) {
 ; CHECK-LABEL: @avx512_psrli_d_512_0(
-; CHECK-NEXT:    ret <16 x i32> %v
+; CHECK-NEXT:    ret <16 x i32> [[V:%.*]]
 ;
   %1 = tail call <16 x i32> @llvm.x86.avx512.psrli.d.512(<16 x i32> %v, i32 0)
   ret <16 x i32> %1
@@ -429,7 +429,7 @@ define <16 x i32> @avx512_psrli_d_512_0(<16 x i32> %v) {
 
 define <16 x i32> @avx512_psrli_d_512_15(<16 x i32> %v) {
 ; CHECK-LABEL: @avx512_psrli_d_512_15(
-; CHECK-NEXT:    [[TMP1:%.*]] = lshr <16 x i32> %v, <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <16 x i32> [[V:%.*]], <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
 ; CHECK-NEXT:    ret <16 x i32> [[TMP1]]
 ;
   %1 = tail call <16 x i32> @llvm.x86.avx512.psrli.d.512(<16 x i32> %v, i32 15)
@@ -446,7 +446,7 @@ define <16 x i32> @avx512_psrli_d_512_64(<16 x i32> %v) {
 
 define <8 x i64> @avx512_psrli_q_512_0(<8 x i64> %v) {
 ; CHECK-LABEL: @avx512_psrli_q_512_0(
-; CHECK-NEXT:    ret <8 x i64> %v
+; CHECK-NEXT:    ret <8 x i64> [[V:%.*]]
 ;
   %1 = tail call <8 x i64> @llvm.x86.avx512.psrli.q.512(<8 x i64> %v, i32 0)
   ret <8 x i64> %1
@@ -454,7 +454,7 @@ define <8 x i64> @avx512_psrli_q_512_0(<8 x i64> %v) {
 
 define <8 x i64> @avx512_psrli_q_512_15(<8 x i64> %v) {
 ; CHECK-LABEL: @avx512_psrli_q_512_15(
-; CHECK-NEXT:    [[TMP1:%.*]] = lshr <8 x i64> %v, <i64 15, i64 15, i64 15, i64 15, i64 15, i64 15, i64 15, i64 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <8 x i64> [[V:%.*]], <i64 15, i64 15, i64 15, i64 15, i64 15, i64 15, i64 15, i64 15>
 ; CHECK-NEXT:    ret <8 x i64> [[TMP1]]
 ;
   %1 = tail call <8 x i64> @llvm.x86.avx512.psrli.q.512(<8 x i64> %v, i32 15)
@@ -475,7 +475,7 @@ define <8 x i64> @avx512_psrli_q_512_64(<8 x i64> %v) {
 
 define <8 x i16> @sse2_pslli_w_0(<8 x i16> %v) {
 ; CHECK-LABEL: @sse2_pslli_w_0(
-; CHECK-NEXT:    ret <8 x i16> %v
+; CHECK-NEXT:    ret <8 x i16> [[V:%.*]]
 ;
   %1 = tail call <8 x i16> @llvm.x86.sse2.pslli.w(<8 x i16> %v, i32 0)
   ret <8 x i16> %1
@@ -483,7 +483,7 @@ define <8 x i16> @sse2_pslli_w_0(<8 x i16> %v) {
 
 define <8 x i16> @sse2_pslli_w_15(<8 x i16> %v) {
 ; CHECK-LABEL: @sse2_pslli_w_15(
-; CHECK-NEXT:    [[TMP1:%.*]] = shl <8 x i16> %v, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <8 x i16> [[V:%.*]], <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
 ; CHECK-NEXT:    ret <8 x i16> [[TMP1]]
 ;
   %1 = tail call <8 x i16> @llvm.x86.sse2.pslli.w(<8 x i16> %v, i32 15)
@@ -500,7 +500,7 @@ define <8 x i16> @sse2_pslli_w_64(<8 x i16> %v) {
 
 define <4 x i32> @sse2_pslli_d_0(<4 x i32> %v) {
 ; CHECK-LABEL: @sse2_pslli_d_0(
-; CHECK-NEXT:    ret <4 x i32> %v
+; CHECK-NEXT:    ret <4 x i32> [[V:%.*]]
 ;
   %1 = tail call <4 x i32> @llvm.x86.sse2.pslli.d(<4 x i32> %v, i32 0)
   ret <4 x i32> %1
@@ -508,7 +508,7 @@ define <4 x i32> @sse2_pslli_d_0(<4 x i32> %v) {
 
 define <4 x i32> @sse2_pslli_d_15(<4 x i32> %v) {
 ; CHECK-LABEL: @sse2_pslli_d_15(
-; CHECK-NEXT:    [[TMP1:%.*]] = shl <4 x i32> %v, <i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <4 x i32> [[V:%.*]], <i32 15, i32 15, i32 15, i32 15>
 ; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
 ;
   %1 = tail call <4 x i32> @llvm.x86.sse2.pslli.d(<4 x i32> %v, i32 15)
@@ -525,7 +525,7 @@ define <4 x i32> @sse2_pslli_d_64(<4 x i32> %v) {
 
 define <2 x i64> @sse2_pslli_q_0(<2 x i64> %v) {
 ; CHECK-LABEL: @sse2_pslli_q_0(
-; CHECK-NEXT:    ret <2 x i64> %v
+; CHECK-NEXT:    ret <2 x i64> [[V:%.*]]
 ;
   %1 = tail call <2 x i64> @llvm.x86.sse2.pslli.q(<2 x i64> %v, i32 0)
   ret <2 x i64> %1
@@ -533,7 +533,7 @@ define <2 x i64> @sse2_pslli_q_0(<2 x i64> %v) {
 
 define <2 x i64> @sse2_pslli_q_15(<2 x i64> %v) {
 ; CHECK-LABEL: @sse2_pslli_q_15(
-; CHECK-NEXT:    [[TMP1:%.*]] = shl <2 x i64> %v, <i64 15, i64 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <2 x i64> [[V:%.*]], <i64 15, i64 15>
 ; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
 ;
   %1 = tail call <2 x i64> @llvm.x86.sse2.pslli.q(<2 x i64> %v, i32 15)
@@ -550,7 +550,7 @@ define <2 x i64> @sse2_pslli_q_64(<2 x i64> %v) {
 
 define <16 x i16> @avx2_pslli_w_0(<16 x i16> %v) {
 ; CHECK-LABEL: @avx2_pslli_w_0(
-; CHECK-NEXT:    ret <16 x i16> %v
+; CHECK-NEXT:    ret <16 x i16> [[V:%.*]]
 ;
   %1 = tail call <16 x i16> @llvm.x86.avx2.pslli.w(<16 x i16> %v, i32 0)
   ret <16 x i16> %1
@@ -558,7 +558,7 @@ define <16 x i16> @avx2_pslli_w_0(<16 x i16> %v) {
 
 define <16 x i16> @avx2_pslli_w_15(<16 x i16> %v) {
 ; CHECK-LABEL: @avx2_pslli_w_15(
-; CHECK-NEXT:    [[TMP1:%.*]] = shl <16 x i16> %v, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <16 x i16> [[V:%.*]], <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
 ; CHECK-NEXT:    ret <16 x i16> [[TMP1]]
 ;
   %1 = tail call <16 x i16> @llvm.x86.avx2.pslli.w(<16 x i16> %v, i32 15)
@@ -575,7 +575,7 @@ define <16 x i16> @avx2_pslli_w_64(<16 x i16> %v) {
 
 define <8 x i32> @avx2_pslli_d_0(<8 x i32> %v) {
 ; CHECK-LABEL: @avx2_pslli_d_0(
-; CHECK-NEXT:    ret <8 x i32> %v
+; CHECK-NEXT:    ret <8 x i32> [[V:%.*]]
 ;
   %1 = tail call <8 x i32> @llvm.x86.avx2.pslli.d(<8 x i32> %v, i32 0)
   ret <8 x i32> %1
@@ -583,7 +583,7 @@ define <8 x i32> @avx2_pslli_d_0(<8 x i32> %v) {
 
 define <8 x i32> @avx2_pslli_d_15(<8 x i32> %v) {
 ; CHECK-LABEL: @avx2_pslli_d_15(
-; CHECK-NEXT:    [[TMP1:%.*]] = shl <8 x i32> %v, <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <8 x i32> [[V:%.*]], <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
 ; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
 ;
   %1 = tail call <8 x i32> @llvm.x86.avx2.pslli.d(<8 x i32> %v, i32 15)
@@ -600,7 +600,7 @@ define <8 x i32> @avx2_pslli_d_64(<8 x i32> %v) {
 
 define <4 x i64> @avx2_pslli_q_0(<4 x i64> %v) {
 ; CHECK-LABEL: @avx2_pslli_q_0(
-; CHECK-NEXT:    ret <4 x i64> %v
+; CHECK-NEXT:    ret <4 x i64> [[V:%.*]]
 ;
   %1 = tail call <4 x i64> @llvm.x86.avx2.pslli.q(<4 x i64> %v, i32 0)
   ret <4 x i64> %1
@@ -608,7 +608,7 @@ define <4 x i64> @avx2_pslli_q_0(<4 x i64> %v) {
 
 define <4 x i64> @avx2_pslli_q_15(<4 x i64> %v) {
 ; CHECK-LABEL: @avx2_pslli_q_15(
-; CHECK-NEXT:    [[TMP1:%.*]] = shl <4 x i64> %v, <i64 15, i64 15, i64 15, i64 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <4 x i64> [[V:%.*]], <i64 15, i64 15, i64 15, i64 15>
 ; CHECK-NEXT:    ret <4 x i64> [[TMP1]]
 ;
   %1 = tail call <4 x i64> @llvm.x86.avx2.pslli.q(<4 x i64> %v, i32 15)
@@ -625,7 +625,7 @@ define <4 x i64> @avx2_pslli_q_64(<4 x i64> %v) {
 
 define <32 x i16> @avx512_pslli_w_512_0(<32 x i16> %v) {
 ; CHECK-LABEL: @avx512_pslli_w_512_0(
-; CHECK-NEXT:    ret <32 x i16> %v
+; CHECK-NEXT:    ret <32 x i16> [[V:%.*]]
 ;
   %1 = tail call <32 x i16> @llvm.x86.avx512.pslli.w.512(<32 x i16> %v, i32 0)
   ret <32 x i16> %1
@@ -633,7 +633,7 @@ define <32 x i16> @avx512_pslli_w_512_0(<32 x i16> %v) {
 
 define <32 x i16> @avx512_pslli_w_512_15(<32 x i16> %v) {
 ; CHECK-LABEL: @avx512_pslli_w_512_15(
-; CHECK-NEXT:    [[TMP1:%.*]] = shl <32 x i16> %v, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <32 x i16> [[V:%.*]], <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
 ; CHECK-NEXT:    ret <32 x i16> [[TMP1]]
 ;
   %1 = tail call <32 x i16> @llvm.x86.avx512.pslli.w.512(<32 x i16> %v, i32 15)
@@ -650,7 +650,7 @@ define <32 x i16> @avx512_pslli_w_512_64(<32 x i16> %v) {
 
 define <16 x i32> @avx512_pslli_d_512_0(<16 x i32> %v) {
 ; CHECK-LABEL: @avx512_pslli_d_512_0(
-; CHECK-NEXT:    ret <16 x i32> %v
+; CHECK-NEXT:    ret <16 x i32> [[V:%.*]]
 ;
   %1 = tail call <16 x i32> @llvm.x86.avx512.pslli.d.512(<16 x i32> %v, i32 0)
   ret <16 x i32> %1
@@ -658,7 +658,7 @@ define <16 x i32> @avx512_pslli_d_512_0(<16 x i32> %v) {
 
 define <16 x i32> @avx512_pslli_d_512_15(<16 x i32> %v) {
 ; CHECK-LABEL: @avx512_pslli_d_512_15(
-; CHECK-NEXT:    [[TMP1:%.*]] = shl <16 x i32> %v, <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <16 x i32> [[V:%.*]], <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
 ; CHECK-NEXT:    ret <16 x i32> [[TMP1]]
 ;
   %1 = tail call <16 x i32> @llvm.x86.avx512.pslli.d.512(<16 x i32> %v, i32 15)
@@ -675,7 +675,7 @@ define <16 x i32> @avx512_pslli_d_512_64(<16 x i32> %v) {
 
 define <8 x i64> @avx512_pslli_q_512_0(<8 x i64> %v) {
 ; CHECK-LABEL: @avx512_pslli_q_512_0(
-; CHECK-NEXT:    ret <8 x i64> %v
+; CHECK-NEXT:    ret <8 x i64> [[V:%.*]]
 ;
   %1 = tail call <8 x i64> @llvm.x86.avx512.pslli.q.512(<8 x i64> %v, i32 0)
   ret <8 x i64> %1
@@ -683,7 +683,7 @@ define <8 x i64> @avx512_pslli_q_512_0(<8 x i64> %v) {
 
 define <8 x i64> @avx512_pslli_q_512_15(<8 x i64> %v) {
 ; CHECK-LABEL: @avx512_pslli_q_512_15(
-; CHECK-NEXT:    [[TMP1:%.*]] = shl <8 x i64> %v, <i64 15, i64 15, i64 15, i64 15, i64 15, i64 15, i64 15, i64 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <8 x i64> [[V:%.*]], <i64 15, i64 15, i64 15, i64 15, i64 15, i64 15, i64 15, i64 15>
 ; CHECK-NEXT:    ret <8 x i64> [[TMP1]]
 ;
   %1 = tail call <8 x i64> @llvm.x86.avx512.pslli.q.512(<8 x i64> %v, i32 15)
@@ -704,7 +704,7 @@ define <8 x i64> @avx512_pslli_q_512_64(<8 x i64> %v) {
 
 define <8 x i16> @sse2_psra_w_0(<8 x i16> %v) {
 ; CHECK-LABEL: @sse2_psra_w_0(
-; CHECK-NEXT:    ret <8 x i16> %v
+; CHECK-NEXT:    ret <8 x i16> [[V:%.*]]
 ;
   %1 = tail call <8 x i16> @llvm.x86.sse2.psra.w(<8 x i16> %v, <8 x i16> zeroinitializer)
   ret <8 x i16> %1
@@ -712,7 +712,7 @@ define <8 x i16> @sse2_psra_w_0(<8 x i16> %v) {
 
 define <8 x i16> @sse2_psra_w_15(<8 x i16> %v) {
 ; CHECK-LABEL: @sse2_psra_w_15(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i16> %v, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i16> [[V:%.*]], <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
 ; CHECK-NEXT:    ret <8 x i16> [[TMP1]]
 ;
   %1 = tail call <8 x i16> @llvm.x86.sse2.psra.w(<8 x i16> %v, <8 x i16> <i16 15, i16 0, i16 0, i16 0, i16 9999, i16 9999, i16 9999, i16 9999>)
@@ -721,7 +721,7 @@ define <8 x i16> @sse2_psra_w_15(<8 x i16> %v) {
 
 define <8 x i16> @sse2_psra_w_15_splat(<8 x i16> %v) {
 ; CHECK-LABEL: @sse2_psra_w_15_splat(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i16> %v, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i16> [[V:%.*]], <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
 ; CHECK-NEXT:    ret <8 x i16> [[TMP1]]
 ;
   %1 = tail call <8 x i16> @llvm.x86.sse2.psra.w(<8 x i16> %v, <8 x i16> <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>)
@@ -730,7 +730,7 @@ define <8 x i16> @sse2_psra_w_15_splat(<8 x i16> %v) {
 
 define <8 x i16> @sse2_psra_w_64(<8 x i16> %v) {
 ; CHECK-LABEL: @sse2_psra_w_64(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i16> %v, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i16> [[V:%.*]], <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
 ; CHECK-NEXT:    ret <8 x i16> [[TMP1]]
 ;
   %1 = tail call <8 x i16> @llvm.x86.sse2.psra.w(<8 x i16> %v, <8 x i16> <i16 64, i16 0, i16 0, i16 0, i16 9999, i16 9999, i16 9999, i16 9999>)
@@ -739,7 +739,7 @@ define <8 x i16> @sse2_psra_w_64(<8 x i16> %v) {
 
 define <4 x i32> @sse2_psra_d_0(<4 x i32> %v) {
 ; CHECK-LABEL: @sse2_psra_d_0(
-; CHECK-NEXT:    ret <4 x i32> %v
+; CHECK-NEXT:    ret <4 x i32> [[V:%.*]]
 ;
   %1 = tail call <4 x i32> @llvm.x86.sse2.psra.d(<4 x i32> %v, <4 x i32> zeroinitializer)
   ret <4 x i32> %1
@@ -747,7 +747,7 @@ define <4 x i32> @sse2_psra_d_0(<4 x i32> %v) {
 
 define <4 x i32> @sse2_psra_d_15(<4 x i32> %v) {
 ; CHECK-LABEL: @sse2_psra_d_15(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <4 x i32> %v, <i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <4 x i32> [[V:%.*]], <i32 15, i32 15, i32 15, i32 15>
 ; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
 ;
   %1 = tail call <4 x i32> @llvm.x86.sse2.psra.d(<4 x i32> %v, <4 x i32> <i32 15, i32 0, i32 9999, i32 9999>)
@@ -756,7 +756,7 @@ define <4 x i32> @sse2_psra_d_15(<4 x i32> %v) {
 
 define <4 x i32> @sse2_psra_d_15_splat(<4 x i32> %v) {
 ; CHECK-LABEL: @sse2_psra_d_15_splat(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <4 x i32> %v, <i32 31, i32 31, i32 31, i32 31>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <4 x i32> [[V:%.*]], <i32 31, i32 31, i32 31, i32 31>
 ; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
 ;
   %1 = tail call <4 x i32> @llvm.x86.sse2.psra.d(<4 x i32> %v, <4 x i32> <i32 15, i32 15, i32 15, i32 15>)
@@ -765,7 +765,7 @@ define <4 x i32> @sse2_psra_d_15_splat(<4 x i32> %v) {
 
 define <4 x i32> @sse2_psra_d_64(<4 x i32> %v) {
 ; CHECK-LABEL: @sse2_psra_d_64(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <4 x i32> %v, <i32 31, i32 31, i32 31, i32 31>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <4 x i32> [[V:%.*]], <i32 31, i32 31, i32 31, i32 31>
 ; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
 ;
   %1 = tail call <4 x i32> @llvm.x86.sse2.psra.d(<4 x i32> %v, <4 x i32> <i32 64, i32 0, i32 9999, i32 9999>)
@@ -774,7 +774,7 @@ define <4 x i32> @sse2_psra_d_64(<4 x i32> %v) {
 
 define <16 x i16> @avx2_psra_w_0(<16 x i16> %v) {
 ; CHECK-LABEL: @avx2_psra_w_0(
-; CHECK-NEXT:    ret <16 x i16> %v
+; CHECK-NEXT:    ret <16 x i16> [[V:%.*]]
 ;
   %1 = tail call <16 x i16> @llvm.x86.avx2.psra.w(<16 x i16> %v, <8 x i16> zeroinitializer)
   ret <16 x i16> %1
@@ -782,7 +782,7 @@ define <16 x i16> @avx2_psra_w_0(<16 x i16> %v) {
 
 define <16 x i16> @avx2_psra_w_15(<16 x i16> %v) {
 ; CHECK-LABEL: @avx2_psra_w_15(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <16 x i16> %v, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <16 x i16> [[V:%.*]], <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
 ; CHECK-NEXT:    ret <16 x i16> [[TMP1]]
 ;
   %1 = tail call <16 x i16> @llvm.x86.avx2.psra.w(<16 x i16> %v, <8 x i16> <i16 15, i16 0, i16 0, i16 0, i16 9999, i16 9999, i16 9999, i16 9999>)
@@ -791,7 +791,7 @@ define <16 x i16> @avx2_psra_w_15(<16 x i16> %v) {
 
 define <16 x i16> @avx2_psra_w_15_splat(<16 x i16> %v) {
 ; CHECK-LABEL: @avx2_psra_w_15_splat(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <16 x i16> %v, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <16 x i16> [[V:%.*]], <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
 ; CHECK-NEXT:    ret <16 x i16> [[TMP1]]
 ;
   %1 = tail call <16 x i16> @llvm.x86.avx2.psra.w(<16 x i16> %v, <8 x i16> <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>)
@@ -800,7 +800,7 @@ define <16 x i16> @avx2_psra_w_15_splat(<16 x i16> %v) {
 
 define <16 x i16> @avx2_psra_w_64(<16 x i16> %v) {
 ; CHECK-LABEL: @avx2_psra_w_64(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <16 x i16> %v, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <16 x i16> [[V:%.*]], <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
 ; CHECK-NEXT:    ret <16 x i16> [[TMP1]]
 ;
   %1 = tail call <16 x i16> @llvm.x86.avx2.psra.w(<16 x i16> %v, <8 x i16> <i16 64, i16 0, i16 0, i16 0, i16 9999, i16 9999, i16 9999, i16 9999>)
@@ -809,7 +809,7 @@ define <16 x i16> @avx2_psra_w_64(<16 x i16> %v) {
 
 define <8 x i32> @avx2_psra_d_0(<8 x i32> %v) {
 ; CHECK-LABEL: @avx2_psra_d_0(
-; CHECK-NEXT:    ret <8 x i32> %v
+; CHECK-NEXT:    ret <8 x i32> [[V:%.*]]
 ;
   %1 = tail call <8 x i32> @llvm.x86.avx2.psra.d(<8 x i32> %v, <4 x i32> zeroinitializer)
   ret <8 x i32> %1
@@ -817,7 +817,7 @@ define <8 x i32> @avx2_psra_d_0(<8 x i32> %v) {
 
 define <8 x i32> @avx2_psra_d_15(<8 x i32> %v) {
 ; CHECK-LABEL: @avx2_psra_d_15(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i32> %v, <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i32> [[V:%.*]], <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
 ; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
 ;
   %1 = tail call <8 x i32> @llvm.x86.avx2.psra.d(<8 x i32> %v, <4 x i32> <i32 15, i32 0, i32 9999, i32 9999>)
@@ -826,7 +826,7 @@ define <8 x i32> @avx2_psra_d_15(<8 x i32> %v) {
 
 define <8 x i32> @avx2_psra_d_15_splat(<8 x i32> %v) {
 ; CHECK-LABEL: @avx2_psra_d_15_splat(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i32> %v, <i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i32> [[V:%.*]], <i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31>
 ; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
 ;
   %1 = tail call <8 x i32> @llvm.x86.avx2.psra.d(<8 x i32> %v, <4 x i32> <i32 15, i32 15, i32 15, i32 15>)
@@ -835,7 +835,7 @@ define <8 x i32> @avx2_psra_d_15_splat(<8 x i32> %v) {
 
 define <8 x i32> @avx2_psra_d_64(<8 x i32> %v) {
 ; CHECK-LABEL: @avx2_psra_d_64(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i32> %v, <i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i32> [[V:%.*]], <i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31>
 ; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
 ;
   %1 = tail call <8 x i32> @llvm.x86.avx2.psra.d(<8 x i32> %v, <4 x i32> <i32 64, i32 0, i32 9999, i32 9999>)
@@ -844,7 +844,7 @@ define <8 x i32> @avx2_psra_d_64(<8 x i32> %v) {
 
 define <2 x i64> @avx512_psra_q_128_0(<2 x i64> %v) {
 ; CHECK-LABEL: @avx512_psra_q_128_0(
-; CHECK-NEXT:    ret <2 x i64> %v
+; CHECK-NEXT:    ret <2 x i64> [[V:%.*]]
 ;
   %1 = tail call <2 x i64> @llvm.x86.avx512.psra.q.128(<2 x i64> %v, <2 x i64> zeroinitializer)
   ret <2 x i64> %1
@@ -852,7 +852,7 @@ define <2 x i64> @avx512_psra_q_128_0(<2 x i64> %v) {
 
 define <2 x i64> @avx512_psra_q_128_15(<2 x i64> %v) {
 ; CHECK-LABEL: @avx512_psra_q_128_15(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <2 x i64> %v, <i64 15, i64 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <2 x i64> [[V:%.*]], <i64 15, i64 15>
 ; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
 ;
   %1 = tail call <2 x i64> @llvm.x86.avx512.psra.q.128(<2 x i64> %v, <2 x i64> <i64 15, i64 9999>)
@@ -861,7 +861,7 @@ define <2 x i64> @avx512_psra_q_128_15(<2 x i64> %v) {
 
 define <2 x i64> @avx512_psra_q_128_64(<2 x i64> %v) {
 ; CHECK-LABEL: @avx512_psra_q_128_64(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <2 x i64> %v, <i64 63, i64 63>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <2 x i64> [[V:%.*]], <i64 63, i64 63>
 ; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
 ;
   %1 = tail call <2 x i64> @llvm.x86.avx512.psra.q.128(<2 x i64> %v, <2 x i64> <i64 64, i64 9999>)
@@ -870,7 +870,7 @@ define <2 x i64> @avx512_psra_q_128_64(<2 x i64> %v) {
 
 define <4 x i64> @avx512_psra_q_256_0(<4 x i64> %v) {
 ; CHECK-LABEL: @avx512_psra_q_256_0(
-; CHECK-NEXT:    ret <4 x i64> %v
+; CHECK-NEXT:    ret <4 x i64> [[V:%.*]]
 ;
   %1 = tail call <4 x i64> @llvm.x86.avx512.psra.q.256(<4 x i64> %v, <2 x i64> zeroinitializer)
   ret <4 x i64> %1
@@ -878,7 +878,7 @@ define <4 x i64> @avx512_psra_q_256_0(<4 x i64> %v) {
 
 define <4 x i64> @avx512_psra_q_256_15(<4 x i64> %v) {
 ; CHECK-LABEL: @avx512_psra_q_256_15(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <4 x i64> %v, <i64 15, i64 15, i64 15, i64 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <4 x i64> [[V:%.*]], <i64 15, i64 15, i64 15, i64 15>
 ; CHECK-NEXT:    ret <4 x i64> [[TMP1]]
 ;
   %1 = tail call <4 x i64> @llvm.x86.avx512.psra.q.256(<4 x i64> %v, <2 x i64> <i64 15, i64 9999>)
@@ -887,7 +887,7 @@ define <4 x i64> @avx512_psra_q_256_15(<4 x i64> %v) {
 
 define <4 x i64> @avx512_psra_q_256_64(<4 x i64> %v) {
 ; CHECK-LABEL: @avx512_psra_q_256_64(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <4 x i64> %v, <i64 63, i64 63, i64 63, i64 63>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <4 x i64> [[V:%.*]], <i64 63, i64 63, i64 63, i64 63>
 ; CHECK-NEXT:    ret <4 x i64> [[TMP1]]
 ;
   %1 = tail call <4 x i64> @llvm.x86.avx512.psra.q.256(<4 x i64> %v, <2 x i64> <i64 64, i64 9999>)
@@ -896,7 +896,7 @@ define <4 x i64> @avx512_psra_q_256_64(<4 x i64> %v) {
 
 define <32 x i16> @avx512_psra_w_512_0(<32 x i16> %v) {
 ; CHECK-LABEL: @avx512_psra_w_512_0(
-; CHECK-NEXT:    ret <32 x i16> %v
+; CHECK-NEXT:    ret <32 x i16> [[V:%.*]]
 ;
   %1 = tail call <32 x i16> @llvm.x86.avx512.psra.w.512(<32 x i16> %v, <8 x i16> zeroinitializer)
   ret <32 x i16> %1
@@ -904,7 +904,7 @@ define <32 x i16> @avx512_psra_w_512_0(<32 x i16> %v) {
 
 define <32 x i16> @avx512_psra_w_512_15(<32 x i16> %v) {
 ; CHECK-LABEL: @avx512_psra_w_512_15(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <32 x i16> %v, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <32 x i16> [[V:%.*]], <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
 ; CHECK-NEXT:    ret <32 x i16> [[TMP1]]
 ;
   %1 = tail call <32 x i16> @llvm.x86.avx512.psra.w.512(<32 x i16> %v, <8 x i16> <i16 15, i16 0, i16 0, i16 0, i16 9999, i16 9999, i16 9999, i16 9999>)
@@ -913,7 +913,7 @@ define <32 x i16> @avx512_psra_w_512_15(<32 x i16> %v) {
 
 define <32 x i16> @avx512_psra_w_512_15_splat(<32 x i16> %v) {
 ; CHECK-LABEL: @avx512_psra_w_512_15_splat(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <32 x i16> %v, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <32 x i16> [[V:%.*]], <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
 ; CHECK-NEXT:    ret <32 x i16> [[TMP1]]
 ;
   %1 = tail call <32 x i16> @llvm.x86.avx512.psra.w.512(<32 x i16> %v, <8 x i16> <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>)
@@ -922,7 +922,7 @@ define <32 x i16> @avx512_psra_w_512_15_splat(<32 x i16> %v) {
 
 define <32 x i16> @avx512_psra_w_512_64(<32 x i16> %v) {
 ; CHECK-LABEL: @avx512_psra_w_512_64(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <32 x i16> %v, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <32 x i16> [[V:%.*]], <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
 ; CHECK-NEXT:    ret <32 x i16> [[TMP1]]
 ;
   %1 = tail call <32 x i16> @llvm.x86.avx512.psra.w.512(<32 x i16> %v, <8 x i16> <i16 64, i16 0, i16 0, i16 0, i16 9999, i16 9999, i16 9999, i16 9999>)
@@ -931,7 +931,7 @@ define <32 x i16> @avx512_psra_w_512_64(<32 x i16> %v) {
 
 define <16 x i32> @avx512_psra_d_512_0(<16 x i32> %v) {
 ; CHECK-LABEL: @avx512_psra_d_512_0(
-; CHECK-NEXT:    ret <16 x i32> %v
+; CHECK-NEXT:    ret <16 x i32> [[V:%.*]]
 ;
   %1 = tail call <16 x i32> @llvm.x86.avx512.psra.d.512(<16 x i32> %v, <4 x i32> zeroinitializer)
   ret <16 x i32> %1
@@ -939,7 +939,7 @@ define <16 x i32> @avx512_psra_d_512_0(<16 x i32> %v) {
 
 define <16 x i32> @avx512_psra_d_512_15(<16 x i32> %v) {
 ; CHECK-LABEL: @avx512_psra_d_512_15(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <16 x i32> %v, <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <16 x i32> [[V:%.*]], <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
 ; CHECK-NEXT:    ret <16 x i32> [[TMP1]]
 ;
   %1 = tail call <16 x i32> @llvm.x86.avx512.psra.d.512(<16 x i32> %v, <4 x i32> <i32 15, i32 0, i32 9999, i32 9999>)
@@ -948,7 +948,7 @@ define <16 x i32> @avx512_psra_d_512_15(<16 x i32> %v) {
 
 define <16 x i32> @avx512_psra_d_512_15_splat(<16 x i32> %v) {
 ; CHECK-LABEL: @avx512_psra_d_512_15_splat(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <16 x i32> %v, <i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <16 x i32> [[V:%.*]], <i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31>
 ; CHECK-NEXT:    ret <16 x i32> [[TMP1]]
 ;
   %1 = tail call <16 x i32> @llvm.x86.avx512.psra.d.512(<16 x i32> %v, <4 x i32> <i32 15, i32 15, i32 15, i32 15>)
@@ -957,7 +957,7 @@ define <16 x i32> @avx512_psra_d_512_15_splat(<16 x i32> %v) {
 
 define <16 x i32> @avx512_psra_d_512_64(<16 x i32> %v) {
 ; CHECK-LABEL: @avx512_psra_d_512_64(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <16 x i32> %v, <i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <16 x i32> [[V:%.*]], <i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31>
 ; CHECK-NEXT:    ret <16 x i32> [[TMP1]]
 ;
   %1 = tail call <16 x i32> @llvm.x86.avx512.psra.d.512(<16 x i32> %v, <4 x i32> <i32 64, i32 0, i32 9999, i32 9999>)
@@ -966,7 +966,7 @@ define <16 x i32> @avx512_psra_d_512_64(<16 x i32> %v) {
 
 define <8 x i64> @avx512_psra_q_512_0(<8 x i64> %v) {
 ; CHECK-LABEL: @avx512_psra_q_512_0(
-; CHECK-NEXT:    ret <8 x i64> %v
+; CHECK-NEXT:    ret <8 x i64> [[V:%.*]]
 ;
   %1 = tail call <8 x i64> @llvm.x86.avx512.psra.q.512(<8 x i64> %v, <2 x i64> zeroinitializer)
   ret <8 x i64> %1
@@ -974,7 +974,7 @@ define <8 x i64> @avx512_psra_q_512_0(<8 x i64> %v) {
 
 define <8 x i64> @avx512_psra_q_512_15(<8 x i64> %v) {
 ; CHECK-LABEL: @avx512_psra_q_512_15(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i64> %v, <i64 15, i64 15, i64 15, i64 15, i64 15, i64 15, i64 15, i64 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i64> [[V:%.*]], <i64 15, i64 15, i64 15, i64 15, i64 15, i64 15, i64 15, i64 15>
 ; CHECK-NEXT:    ret <8 x i64> [[TMP1]]
 ;
   %1 = tail call <8 x i64> @llvm.x86.avx512.psra.q.512(<8 x i64> %v, <2 x i64> <i64 15, i64 9999>)
@@ -983,7 +983,7 @@ define <8 x i64> @avx512_psra_q_512_15(<8 x i64> %v) {
 
 define <8 x i64> @avx512_psra_q_512_64(<8 x i64> %v) {
 ; CHECK-LABEL: @avx512_psra_q_512_64(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i64> %v, <i64 63, i64 63, i64 63, i64 63, i64 63, i64 63, i64 63, i64 63>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i64> [[V:%.*]], <i64 63, i64 63, i64 63, i64 63, i64 63, i64 63, i64 63, i64 63>
 ; CHECK-NEXT:    ret <8 x i64> [[TMP1]]
 ;
   %1 = tail call <8 x i64> @llvm.x86.avx512.psra.q.512(<8 x i64> %v, <2 x i64> <i64 64, i64 9999>)
@@ -996,7 +996,7 @@ define <8 x i64> @avx512_psra_q_512_64(<8 x i64> %v) {
 
 define <8 x i16> @sse2_psrl_w_0(<8 x i16> %v) {
 ; CHECK-LABEL: @sse2_psrl_w_0(
-; CHECK-NEXT:    ret <8 x i16> %v
+; CHECK-NEXT:    ret <8 x i16> [[V:%.*]]
 ;
   %1 = tail call <8 x i16> @llvm.x86.sse2.psrl.w(<8 x i16> %v, <8 x i16> zeroinitializer)
   ret <8 x i16> %1
@@ -1004,7 +1004,7 @@ define <8 x i16> @sse2_psrl_w_0(<8 x i16> %v) {
 
 define <8 x i16> @sse2_psrl_w_15(<8 x i16> %v) {
 ; CHECK-LABEL: @sse2_psrl_w_15(
-; CHECK-NEXT:    [[TMP1:%.*]] = lshr <8 x i16> %v, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <8 x i16> [[V:%.*]], <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
 ; CHECK-NEXT:    ret <8 x i16> [[TMP1]]
 ;
   %1 = tail call <8 x i16> @llvm.x86.sse2.psrl.w(<8 x i16> %v, <8 x i16> <i16 15, i16 0, i16 0, i16 0, i16 9999, i16 9999, i16 9999, i16 9999>)
@@ -1029,7 +1029,7 @@ define <8 x i16> @sse2_psrl_w_64(<8 x i16> %v) {
 
 define <4 x i32> @sse2_psrl_d_0(<4 x i32> %v) {
 ; CHECK-LABEL: @sse2_psrl_d_0(
-; CHECK-NEXT:    ret <4 x i32> %v
+; CHECK-NEXT:    ret <4 x i32> [[V:%.*]]
 ;
   %1 = tail call <4 x i32> @llvm.x86.sse2.psrl.d(<4 x i32> %v, <4 x i32> zeroinitializer)
   ret <4 x i32> %1
@@ -1037,7 +1037,7 @@ define <4 x i32> @sse2_psrl_d_0(<4 x i32> %v) {
 
 define <4 x i32> @sse2_psrl_d_15(<4 x i32> %v) {
 ; CHECK-LABEL: @sse2_psrl_d_15(
-; CHECK-NEXT:    [[TMP1:%.*]] = lshr <4 x i32> %v, <i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <4 x i32> [[V:%.*]], <i32 15, i32 15, i32 15, i32 15>
 ; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
 ;
   %1 = tail call <4 x i32> @llvm.x86.sse2.psrl.d(<4 x i32> %v, <4 x i32> <i32 15, i32 0, i32 9999, i32 9999>)
@@ -1062,7 +1062,7 @@ define <4 x i32> @sse2_psrl_d_64(<4 x i32> %v) {
 
 define <2 x i64> @sse2_psrl_q_0(<2 x i64> %v) {
 ; CHECK-LABEL: @sse2_psrl_q_0(
-; CHECK-NEXT:    ret <2 x i64> %v
+; CHECK-NEXT:    ret <2 x i64> [[V:%.*]]
 ;
   %1 = tail call <2 x i64> @llvm.x86.sse2.psrl.q(<2 x i64> %v, <2 x i64> zeroinitializer)
   ret <2 x i64> %1
@@ -1070,7 +1070,7 @@ define <2 x i64> @sse2_psrl_q_0(<2 x i64> %v) {
 
 define <2 x i64> @sse2_psrl_q_15(<2 x i64> %v) {
 ; CHECK-LABEL: @sse2_psrl_q_15(
-; CHECK-NEXT:    [[TMP1:%.*]] = lshr <2 x i64> %v, <i64 15, i64 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <2 x i64> [[V:%.*]], <i64 15, i64 15>
 ; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
 ;
   %1 = tail call <2 x i64> @llvm.x86.sse2.psrl.q(<2 x i64> %v, <2 x i64> <i64 15, i64 9999>)
@@ -1087,7 +1087,7 @@ define <2 x i64> @sse2_psrl_q_64(<2 x i64> %v) {
 
 define <16 x i16> @avx2_psrl_w_0(<16 x i16> %v) {
 ; CHECK-LABEL: @avx2_psrl_w_0(
-; CHECK-NEXT:    ret <16 x i16> %v
+; CHECK-NEXT:    ret <16 x i16> [[V:%.*]]
 ;
   %1 = tail call <16 x i16> @llvm.x86.avx2.psrl.w(<16 x i16> %v, <8 x i16> zeroinitializer)
   ret <16 x i16> %1
@@ -1095,7 +1095,7 @@ define <16 x i16> @avx2_psrl_w_0(<16 x i16> %v) {
 
 define <16 x i16> @avx2_psrl_w_15(<16 x i16> %v) {
 ; CHECK-LABEL: @avx2_psrl_w_15(
-; CHECK-NEXT:    [[TMP1:%.*]] = lshr <16 x i16> %v, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <16 x i16> [[V:%.*]], <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
 ; CHECK-NEXT:    ret <16 x i16> [[TMP1]]
 ;
   %1 = tail call <16 x i16> @llvm.x86.avx2.psrl.w(<16 x i16> %v, <8 x i16> <i16 15, i16 0, i16 0, i16 0, i16 9999, i16 9999, i16 9999, i16 9999>)
@@ -1120,7 +1120,7 @@ define <16 x i16> @avx2_psrl_w_64(<16 x i16> %v) {
 
 define <8 x i32> @avx2_psrl_d_0(<8 x i32> %v) {
 ; CHECK-LABEL: @avx2_psrl_d_0(
-; CHECK-NEXT:    ret <8 x i32> %v
+; CHECK-NEXT:    ret <8 x i32> [[V:%.*]]
 ;
   %1 = tail call <8 x i32> @llvm.x86.avx2.psrl.d(<8 x i32> %v, <4 x i32> zeroinitializer)
   ret <8 x i32> %1
@@ -1128,7 +1128,7 @@ define <8 x i32> @avx2_psrl_d_0(<8 x i32> %v) {
 
 define <8 x i32> @avx2_psrl_d_15(<8 x i32> %v) {
 ; CHECK-LABEL: @avx2_psrl_d_15(
-; CHECK-NEXT:    [[TMP1:%.*]] = lshr <8 x i32> %v, <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <8 x i32> [[V:%.*]], <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
 ; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
 ;
   %1 = tail call <8 x i32> @llvm.x86.avx2.psrl.d(<8 x i32> %v, <4 x i32> <i32 15, i32 0, i32 9999, i32 9999>)
@@ -1153,7 +1153,7 @@ define <8 x i32> @avx2_psrl_d_64(<8 x i32> %v) {
 
 define <4 x i64> @avx2_psrl_q_0(<4 x i64> %v) {
 ; CHECK-LABEL: @avx2_psrl_q_0(
-; CHECK-NEXT:    ret <4 x i64> %v
+; CHECK-NEXT:    ret <4 x i64> [[V:%.*]]
 ;
   %1 = tail call <4 x i64> @llvm.x86.avx2.psrl.q(<4 x i64> %v, <2 x i64> zeroinitializer)
   ret <4 x i64> %1
@@ -1161,7 +1161,7 @@ define <4 x i64> @avx2_psrl_q_0(<4 x i64> %v) {
 
 define <4 x i64> @avx2_psrl_q_15(<4 x i64> %v) {
 ; CHECK-LABEL: @avx2_psrl_q_15(
-; CHECK-NEXT:    [[TMP1:%.*]] = lshr <4 x i64> %v, <i64 15, i64 15, i64 15, i64 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <4 x i64> [[V:%.*]], <i64 15, i64 15, i64 15, i64 15>
 ; CHECK-NEXT:    ret <4 x i64> [[TMP1]]
 ;
   %1 = tail call <4 x i64> @llvm.x86.avx2.psrl.q(<4 x i64> %v, <2 x i64> <i64 15, i64 9999>)
@@ -1178,7 +1178,7 @@ define <4 x i64> @avx2_psrl_q_64(<4 x i64> %v) {
 
 define <32 x i16> @avx512_psrl_w_512_0(<32 x i16> %v) {
 ; CHECK-LABEL: @avx512_psrl_w_512_0(
-; CHECK-NEXT:    ret <32 x i16> %v
+; CHECK-NEXT:    ret <32 x i16> [[V:%.*]]
 ;
   %1 = tail call <32 x i16> @llvm.x86.avx512.psrl.w.512(<32 x i16> %v, <8 x i16> zeroinitializer)
   ret <32 x i16> %1
@@ -1186,7 +1186,7 @@ define <32 x i16> @avx512_psrl_w_512_0(<32 x i16> %v) {
 
 define <32 x i16> @avx512_psrl_w_512_15(<32 x i16> %v) {
 ; CHECK-LABEL: @avx512_psrl_w_512_15(
-; CHECK-NEXT:    [[TMP1:%.*]] = lshr <32 x i16> %v, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <32 x i16> [[V:%.*]], <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
 ; CHECK-NEXT:    ret <32 x i16> [[TMP1]]
 ;
   %1 = tail call <32 x i16> @llvm.x86.avx512.psrl.w.512(<32 x i16> %v, <8 x i16> <i16 15, i16 0, i16 0, i16 0, i16 9999, i16 9999, i16 9999, i16 9999>)
@@ -1211,7 +1211,7 @@ define <32 x i16> @avx512_psrl_w_512_64(<32 x i16> %v) {
 
 define <16 x i32> @avx512_psrl_d_512_0(<16 x i32> %v) {
 ; CHECK-LABEL: @avx512_psrl_d_512_0(
-; CHECK-NEXT:    ret <16 x i32> %v
+; CHECK-NEXT:    ret <16 x i32> [[V:%.*]]
 ;
   %1 = tail call <16 x i32> @llvm.x86.avx512.psrl.d.512(<16 x i32> %v, <4 x i32> zeroinitializer)
   ret <16 x i32> %1
@@ -1219,7 +1219,7 @@ define <16 x i32> @avx512_psrl_d_512_0(<16 x i32> %v) {
 
 define <16 x i32> @avx512_psrl_d_512_15(<16 x i32> %v) {
 ; CHECK-LABEL: @avx512_psrl_d_512_15(
-; CHECK-NEXT:    [[TMP1:%.*]] = lshr <16 x i32> %v, <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <16 x i32> [[V:%.*]], <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
 ; CHECK-NEXT:    ret <16 x i32> [[TMP1]]
 ;
   %1 = tail call <16 x i32> @llvm.x86.avx512.psrl.d.512(<16 x i32> %v, <4 x i32> <i32 15, i32 0, i32 9999, i32 9999>)
@@ -1244,7 +1244,7 @@ define <16 x i32> @avx512_psrl_d_512_64(<16 x i32> %v) {
 
 define <8 x i64> @avx512_psrl_q_512_0(<8 x i64> %v) {
 ; CHECK-LABEL: @avx512_psrl_q_512_0(
-; CHECK-NEXT:    ret <8 x i64> %v
+; CHECK-NEXT:    ret <8 x i64> [[V:%.*]]
 ;
   %1 = tail call <8 x i64> @llvm.x86.avx512.psrl.q.512(<8 x i64> %v, <2 x i64> zeroinitializer)
   ret <8 x i64> %1
@@ -1252,7 +1252,7 @@ define <8 x i64> @avx512_psrl_q_512_0(<8 x i64> %v) {
 
 define <8 x i64> @avx512_psrl_q_512_15(<8 x i64> %v) {
 ; CHECK-LABEL: @avx512_psrl_q_512_15(
-; CHECK-NEXT:    [[TMP1:%.*]] = lshr <8 x i64> %v, <i64 15, i64 15, i64 15, i64 15, i64 15, i64 15, i64 15, i64 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <8 x i64> [[V:%.*]], <i64 15, i64 15, i64 15, i64 15, i64 15, i64 15, i64 15, i64 15>
 ; CHECK-NEXT:    ret <8 x i64> [[TMP1]]
 ;
   %1 = tail call <8 x i64> @llvm.x86.avx512.psrl.q.512(<8 x i64> %v, <2 x i64> <i64 15, i64 9999>)
@@ -1273,7 +1273,7 @@ define <8 x i64> @avx512_psrl_q_512_64(<8 x i64> %v) {
 
 define <8 x i16> @sse2_psll_w_0(<8 x i16> %v) {
 ; CHECK-LABEL: @sse2_psll_w_0(
-; CHECK-NEXT:    ret <8 x i16> %v
+; CHECK-NEXT:    ret <8 x i16> [[V:%.*]]
 ;
   %1 = tail call <8 x i16> @llvm.x86.sse2.psll.w(<8 x i16> %v, <8 x i16> zeroinitializer)
   ret <8 x i16> %1
@@ -1281,7 +1281,7 @@ define <8 x i16> @sse2_psll_w_0(<8 x i16> %v) {
 
 define <8 x i16> @sse2_psll_w_15(<8 x i16> %v) {
 ; CHECK-LABEL: @sse2_psll_w_15(
-; CHECK-NEXT:    [[TMP1:%.*]] = shl <8 x i16> %v, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <8 x i16> [[V:%.*]], <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
 ; CHECK-NEXT:    ret <8 x i16> [[TMP1]]
 ;
   %1 = tail call <8 x i16> @llvm.x86.sse2.psll.w(<8 x i16> %v, <8 x i16> <i16 15, i16 0, i16 0, i16 0, i16 9999, i16 9999, i16 9999, i16 9999>)
@@ -1306,7 +1306,7 @@ define <8 x i16> @sse2_psll_w_64(<8 x i16> %v) {
 
 define <4 x i32> @sse2_psll_d_0(<4 x i32> %v) {
 ; CHECK-LABEL: @sse2_psll_d_0(
-; CHECK-NEXT:    ret <4 x i32> %v
+; CHECK-NEXT:    ret <4 x i32> [[V:%.*]]
 ;
   %1 = tail call <4 x i32> @llvm.x86.sse2.psll.d(<4 x i32> %v, <4 x i32> zeroinitializer)
   ret <4 x i32> %1
@@ -1314,7 +1314,7 @@ define <4 x i32> @sse2_psll_d_0(<4 x i32> %v) {
 
 define <4 x i32> @sse2_psll_d_15(<4 x i32> %v) {
 ; CHECK-LABEL: @sse2_psll_d_15(
-; CHECK-NEXT:    [[TMP1:%.*]] = shl <4 x i32> %v, <i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <4 x i32> [[V:%.*]], <i32 15, i32 15, i32 15, i32 15>
 ; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
 ;
   %1 = tail call <4 x i32> @llvm.x86.sse2.psll.d(<4 x i32> %v, <4 x i32> <i32 15, i32 0, i32 9999, i32 9999>)
@@ -1339,7 +1339,7 @@ define <4 x i32> @sse2_psll_d_64(<4 x i32> %v) {
 
 define <2 x i64> @sse2_psll_q_0(<2 x i64> %v) {
 ; CHECK-LABEL: @sse2_psll_q_0(
-; CHECK-NEXT:    ret <2 x i64> %v
+; CHECK-NEXT:    ret <2 x i64> [[V:%.*]]
 ;
   %1 = tail call <2 x i64> @llvm.x86.sse2.psll.q(<2 x i64> %v, <2 x i64> zeroinitializer)
   ret <2 x i64> %1
@@ -1347,7 +1347,7 @@ define <2 x i64> @sse2_psll_q_0(<2 x i64> %v) {
 
 define <2 x i64> @sse2_psll_q_15(<2 x i64> %v) {
 ; CHECK-LABEL: @sse2_psll_q_15(
-; CHECK-NEXT:    [[TMP1:%.*]] = shl <2 x i64> %v, <i64 15, i64 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <2 x i64> [[V:%.*]], <i64 15, i64 15>
 ; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
 ;
   %1 = tail call <2 x i64> @llvm.x86.sse2.psll.q(<2 x i64> %v, <2 x i64> <i64 15, i64 9999>)
@@ -1364,7 +1364,7 @@ define <2 x i64> @sse2_psll_q_64(<2 x i64> %v) {
 
 define <16 x i16> @avx2_psll_w_0(<16 x i16> %v) {
 ; CHECK-LABEL: @avx2_psll_w_0(
-; CHECK-NEXT:    ret <16 x i16> %v
+; CHECK-NEXT:    ret <16 x i16> [[V:%.*]]
 ;
   %1 = tail call <16 x i16> @llvm.x86.avx2.psll.w(<16 x i16> %v, <8 x i16> zeroinitializer)
   ret <16 x i16> %1
@@ -1372,7 +1372,7 @@ define <16 x i16> @avx2_psll_w_0(<16 x i16> %v) {
 
 define <16 x i16> @avx2_psll_w_15(<16 x i16> %v) {
 ; CHECK-LABEL: @avx2_psll_w_15(
-; CHECK-NEXT:    [[TMP1:%.*]] = shl <16 x i16> %v, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <16 x i16> [[V:%.*]], <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
 ; CHECK-NEXT:    ret <16 x i16> [[TMP1]]
 ;
   %1 = tail call <16 x i16> @llvm.x86.avx2.psll.w(<16 x i16> %v, <8 x i16> <i16 15, i16 0, i16 0, i16 0, i16 9999, i16 9999, i16 9999, i16 9999>)
@@ -1397,7 +1397,7 @@ define <16 x i16> @avx2_psll_w_64(<16 x i16> %v) {
 
 define <8 x i32> @avx2_psll_d_0(<8 x i32> %v) {
 ; CHECK-LABEL: @avx2_psll_d_0(
-; CHECK-NEXT:    ret <8 x i32> %v
+; CHECK-NEXT:    ret <8 x i32> [[V:%.*]]
 ;
   %1 = tail call <8 x i32> @llvm.x86.avx2.psll.d(<8 x i32> %v, <4 x i32> zeroinitializer)
   ret <8 x i32> %1
@@ -1405,7 +1405,7 @@ define <8 x i32> @avx2_psll_d_0(<8 x i32> %v) {
 
 define <8 x i32> @avx2_psll_d_15(<8 x i32> %v) {
 ; CHECK-LABEL: @avx2_psll_d_15(
-; CHECK-NEXT:    [[TMP1:%.*]] = shl <8 x i32> %v, <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <8 x i32> [[V:%.*]], <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
 ; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
 ;
   %1 = tail call <8 x i32> @llvm.x86.avx2.psll.d(<8 x i32> %v, <4 x i32> <i32 15, i32 0, i32 9999, i32 9999>)
@@ -1430,7 +1430,7 @@ define <8 x i32> @avx2_psll_d_64(<8 x i32> %v) {
 
 define <4 x i64> @avx2_psll_q_0(<4 x i64> %v) {
 ; CHECK-LABEL: @avx2_psll_q_0(
-; CHECK-NEXT:    ret <4 x i64> %v
+; CHECK-NEXT:    ret <4 x i64> [[V:%.*]]
 ;
   %1 = tail call <4 x i64> @llvm.x86.avx2.psll.q(<4 x i64> %v, <2 x i64> zeroinitializer)
   ret <4 x i64> %1
@@ -1438,7 +1438,7 @@ define <4 x i64> @avx2_psll_q_0(<4 x i64> %v) {
 
 define <4 x i64> @avx2_psll_q_15(<4 x i64> %v) {
 ; CHECK-LABEL: @avx2_psll_q_15(
-; CHECK-NEXT:    [[TMP1:%.*]] = shl <4 x i64> %v, <i64 15, i64 15, i64 15, i64 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <4 x i64> [[V:%.*]], <i64 15, i64 15, i64 15, i64 15>
 ; CHECK-NEXT:    ret <4 x i64> [[TMP1]]
 ;
   %1 = tail call <4 x i64> @llvm.x86.avx2.psll.q(<4 x i64> %v, <2 x i64> <i64 15, i64 9999>)
@@ -1455,7 +1455,7 @@ define <4 x i64> @avx2_psll_q_64(<4 x i64> %v) {
 
 define <32 x i16> @avx512_psll_w_512_0(<32 x i16> %v) {
 ; CHECK-LABEL: @avx512_psll_w_512_0(
-; CHECK-NEXT:    ret <32 x i16> %v
+; CHECK-NEXT:    ret <32 x i16> [[V:%.*]]
 ;
   %1 = tail call <32 x i16> @llvm.x86.avx512.psll.w.512(<32 x i16> %v, <8 x i16> zeroinitializer)
   ret <32 x i16> %1
@@ -1463,7 +1463,7 @@ define <32 x i16> @avx512_psll_w_512_0(<32 x i16> %v) {
 
 define <32 x i16> @avx512_psll_w_512_15(<32 x i16> %v) {
 ; CHECK-LABEL: @avx512_psll_w_512_15(
-; CHECK-NEXT:    [[TMP1:%.*]] = shl <32 x i16> %v, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <32 x i16> [[V:%.*]], <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
 ; CHECK-NEXT:    ret <32 x i16> [[TMP1]]
 ;
   %1 = tail call <32 x i16> @llvm.x86.avx512.psll.w.512(<32 x i16> %v, <8 x i16> <i16 15, i16 0, i16 0, i16 0, i16 9999, i16 9999, i16 9999, i16 9999>)
@@ -1488,7 +1488,7 @@ define <32 x i16> @avx512_psll_w_512_64(<32 x i16> %v) {
 
 define <16 x i32> @avx512_psll_d_512_0(<16 x i32> %v) {
 ; CHECK-LABEL: @avx512_psll_d_512_0(
-; CHECK-NEXT:    ret <16 x i32> %v
+; CHECK-NEXT:    ret <16 x i32> [[V:%.*]]
 ;
   %1 = tail call <16 x i32> @llvm.x86.avx512.psll.d.512(<16 x i32> %v, <4 x i32> zeroinitializer)
   ret <16 x i32> %1
@@ -1496,7 +1496,7 @@ define <16 x i32> @avx512_psll_d_512_0(<16 x i32> %v) {
 
 define <16 x i32> @avx512_psll_d_512_15(<16 x i32> %v) {
 ; CHECK-LABEL: @avx512_psll_d_512_15(
-; CHECK-NEXT:    [[TMP1:%.*]] = shl <16 x i32> %v, <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <16 x i32> [[V:%.*]], <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
 ; CHECK-NEXT:    ret <16 x i32> [[TMP1]]
 ;
   %1 = tail call <16 x i32> @llvm.x86.avx512.psll.d.512(<16 x i32> %v, <4 x i32> <i32 15, i32 0, i32 9999, i32 9999>)
@@ -1521,7 +1521,7 @@ define <16 x i32> @avx512_psll_d_512_64(<16 x i32> %v) {
 
 define <8 x i64> @avx512_psll_q_512_0(<8 x i64> %v) {
 ; CHECK-LABEL: @avx512_psll_q_512_0(
-; CHECK-NEXT:    ret <8 x i64> %v
+; CHECK-NEXT:    ret <8 x i64> [[V:%.*]]
 ;
   %1 = tail call <8 x i64> @llvm.x86.avx512.psll.q.512(<8 x i64> %v, <2 x i64> zeroinitializer)
   ret <8 x i64> %1
@@ -1529,7 +1529,7 @@ define <8 x i64> @avx512_psll_q_512_0(<8 x i64> %v) {
 
 define <8 x i64> @avx512_psll_q_512_15(<8 x i64> %v) {
 ; CHECK-LABEL: @avx512_psll_q_512_15(
-; CHECK-NEXT:    [[TMP1:%.*]] = shl <8 x i64> %v, <i64 15, i64 15, i64 15, i64 15, i64 15, i64 15, i64 15, i64 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <8 x i64> [[V:%.*]], <i64 15, i64 15, i64 15, i64 15, i64 15, i64 15, i64 15, i64 15>
 ; CHECK-NEXT:    ret <8 x i64> [[TMP1]]
 ;
   %1 = tail call <8 x i64> @llvm.x86.avx512.psll.q.512(<8 x i64> %v, <2 x i64> <i64 15, i64 9999>)
@@ -1550,7 +1550,7 @@ define <8 x i64> @avx512_psll_q_512_64(<8 x i64> %v) {
 
 define <4 x i32> @avx2_psrav_d_128_0(<4 x i32> %v) {
 ; CHECK-LABEL: @avx2_psrav_d_128_0(
-; CHECK-NEXT:    ret <4 x i32> %v
+; CHECK-NEXT:    ret <4 x i32> [[V:%.*]]
 ;
   %1 = tail call <4 x i32> @llvm.x86.avx2.psrav.d(<4 x i32> %v, <4 x i32> zeroinitializer)
   ret <4 x i32> %1
@@ -1558,7 +1558,7 @@ define <4 x i32> @avx2_psrav_d_128_0(<4 x i32> %v) {
 
 define <8 x i32> @avx2_psrav_d_256_0(<8 x i32> %v) {
 ; CHECK-LABEL: @avx2_psrav_d_256_0(
-; CHECK-NEXT:    ret <8 x i32> %v
+; CHECK-NEXT:    ret <8 x i32> [[V:%.*]]
 ;
   %1 = tail call <8 x i32> @llvm.x86.avx2.psrav.d.256(<8 x i32> %v, <8 x i32> zeroinitializer)
   ret <8 x i32> %1
@@ -1566,7 +1566,7 @@ define <8 x i32> @avx2_psrav_d_256_0(<8 x i32> %v) {
 
 define <16 x i32> @avx512_psrav_d_512_0(<16 x i32> %v) {
 ; CHECK-LABEL: @avx512_psrav_d_512_0(
-; CHECK-NEXT:    ret <16 x i32> %v
+; CHECK-NEXT:    ret <16 x i32> [[V:%.*]]
 ;
   %1 = tail call <16 x i32> @llvm.x86.avx512.psrav.d.512(<16 x i32> %v, <16 x i32> zeroinitializer)
   ret <16 x i32> %1
@@ -1574,7 +1574,7 @@ define <16 x i32> @avx512_psrav_d_512_0(<16 x i32> %v) {
 
 define <4 x i32> @avx2_psrav_d_128_var(<4 x i32> %v) {
 ; CHECK-LABEL: @avx2_psrav_d_128_var(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <4 x i32> %v, <i32 0, i32 8, i32 16, i32 31>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <4 x i32> [[V:%.*]], <i32 0, i32 8, i32 16, i32 31>
 ; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
 ;
   %1 = tail call <4 x i32> @llvm.x86.avx2.psrav.d(<4 x i32> %v, <4 x i32> <i32 0, i32 8, i32 16, i32 64>)
@@ -1583,7 +1583,7 @@ define <4 x i32> @avx2_psrav_d_128_var(<4 x i32> %v) {
 
 define <8 x i32> @avx2_psrav_d_256_var(<8 x i32> %v) {
 ; CHECK-LABEL: @avx2_psrav_d_256_var(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i32> %v, <i32 0, i32 8, i32 16, i32 24, i32 31, i32 24, i32 8, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i32> [[V:%.*]], <i32 0, i32 8, i32 16, i32 24, i32 31, i32 24, i32 8, i32 0>
 ; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
 ;
   %1 = tail call <8 x i32> @llvm.x86.avx2.psrav.d.256(<8 x i32> %v, <8 x i32> <i32 0, i32 8, i32 16, i32 24, i32 32, i32 24, i32 8, i32 0>)
@@ -1592,7 +1592,7 @@ define <8 x i32> @avx2_psrav_d_256_var(<8 x i32> %v) {
 
 define <16 x i32> @avx512_psrav_d_512_var(<16 x i32> %v) {
 ; CHECK-LABEL: @avx512_psrav_d_512_var(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <16 x i32> %v, <i32 0, i32 8, i32 16, i32 24, i32 31, i32 24, i32 8, i32 0, i32 0, i32 8, i32 16, i32 24, i32 31, i32 24, i32 8, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <16 x i32> [[V:%.*]], <i32 0, i32 8, i32 16, i32 24, i32 31, i32 24, i32 8, i32 0, i32 0, i32 8, i32 16, i32 24, i32 31, i32 24, i32 8, i32 0>
 ; CHECK-NEXT:    ret <16 x i32> [[TMP1]]
 ;
   %1 = tail call <16 x i32> @llvm.x86.avx512.psrav.d.512(<16 x i32> %v, <16 x i32> <i32 0, i32 8, i32 16, i32 24, i32 32, i32 24, i32 8, i32 0, i32 0, i32 8, i32 16, i32 24, i32 32, i32 24, i32 8, i32 0>)
@@ -1601,7 +1601,7 @@ define <16 x i32> @avx512_psrav_d_512_var(<16 x i32> %v) {
 
 define <4 x i32> @avx2_psrav_d_128_allbig(<4 x i32> %v) {
 ; CHECK-LABEL: @avx2_psrav_d_128_allbig(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <4 x i32> %v, <i32 31, i32 31, i32 31, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <4 x i32> [[V:%.*]], <i32 31, i32 31, i32 31, i32 undef>
 ; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
 ;
   %1 = tail call <4 x i32> @llvm.x86.avx2.psrav.d(<4 x i32> %v, <4 x i32> <i32 32, i32 100, i32 -255, i32 undef>)
@@ -1610,7 +1610,7 @@ define <4 x i32> @avx2_psrav_d_128_allbig(<4 x i32> %v) {
 
 define <8 x i32> @avx2_psrav_d_256_allbig(<8 x i32> %v) {
 ; CHECK-LABEL: @avx2_psrav_d_256_allbig(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i32> %v, <i32 undef, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i32> [[V:%.*]], <i32 undef, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31>
 ; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
 ;
   %1 = tail call <8 x i32> @llvm.x86.avx2.psrav.d.256(<8 x i32> %v, <8 x i32> <i32 undef, i32 100, i32 255, i32 55555, i32 -32, i32 -100, i32 -255, i32 -55555>)
@@ -1619,7 +1619,7 @@ define <8 x i32> @avx2_psrav_d_256_allbig(<8 x i32> %v) {
 
 define <16 x i32> @avx512_psrav_d_512_allbig(<16 x i32> %v) {
 ; CHECK-LABEL: @avx512_psrav_d_512_allbig(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <16 x i32> %v, <i32 undef, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 undef, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <16 x i32> [[V:%.*]], <i32 undef, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 undef, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31>
 ; CHECK-NEXT:    ret <16 x i32> [[TMP1]]
 ;
   %1 = tail call <16 x i32> @llvm.x86.avx512.psrav.d.512(<16 x i32> %v, <16 x i32> <i32 undef, i32 100, i32 255, i32 55555, i32 -32, i32 -100, i32 -255, i32 -55555, i32 undef, i32 100, i32 255, i32 55555, i32 -32, i32 -100, i32 -255, i32 -55555>)
@@ -1628,7 +1628,7 @@ define <16 x i32> @avx512_psrav_d_512_allbig(<16 x i32> %v) {
 
 define <4 x i32> @avx2_psrav_d_128_undef(<4 x i32> %v) {
 ; CHECK-LABEL: @avx2_psrav_d_128_undef(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <4 x i32> %v, <i32 undef, i32 8, i32 16, i32 31>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <4 x i32> [[V:%.*]], <i32 undef, i32 8, i32 16, i32 31>
 ; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
 ;
   %1 = insertelement <4 x i32> <i32 0, i32 8, i32 16, i32 64>, i32 undef, i32 0
@@ -1638,7 +1638,7 @@ define <4 x i32> @avx2_psrav_d_128_undef(<4 x i32> %v) {
 
 define <8 x i32> @avx2_psrav_d_256_undef(<8 x i32> %v) {
 ; CHECK-LABEL: @avx2_psrav_d_256_undef(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i32> %v, <i32 0, i32 undef, i32 16, i32 24, i32 31, i32 24, i32 8, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i32> [[V:%.*]], <i32 0, i32 undef, i32 16, i32 24, i32 31, i32 24, i32 8, i32 0>
 ; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
 ;
   %1 = insertelement <8 x i32> <i32 0, i32 8, i32 16, i32 24, i32 32, i32 24, i32 8, i32 0>, i32 undef, i32 1
@@ -1648,7 +1648,7 @@ define <8 x i32> @avx2_psrav_d_256_undef(<8 x i32> %v) {
 
 define <16 x i32> @avx512_psrav_d_512_undef(<16 x i32> %v) {
 ; CHECK-LABEL: @avx512_psrav_d_512_undef(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <16 x i32> %v, <i32 0, i32 undef, i32 16, i32 24, i32 31, i32 24, i32 8, i32 0, i32 0, i32 8, i32 16, i32 24, i32 31, i32 24, i32 8, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <16 x i32> [[V:%.*]], <i32 0, i32 undef, i32 16, i32 24, i32 31, i32 24, i32 8, i32 0, i32 0, i32 8, i32 16, i32 24, i32 31, i32 24, i32 8, i32 0>
 ; CHECK-NEXT:    ret <16 x i32> [[TMP1]]
 ;
   %1 = insertelement <16 x i32> <i32 0, i32 8, i32 16, i32 24, i32 32, i32 24, i32 8, i32 0, i32 0, i32 8, i32 16, i32 24, i32 32, i32 24, i32 8, i32 0>, i32 undef, i32 1
@@ -1658,7 +1658,7 @@ define <16 x i32> @avx512_psrav_d_512_undef(<16 x i32> %v) {
 
 define <2 x i64> @avx512_psrav_q_128_0(<2 x i64> %v) {
 ; CHECK-LABEL: @avx512_psrav_q_128_0(
-; CHECK-NEXT:    ret <2 x i64> %v
+; CHECK-NEXT:    ret <2 x i64> [[V:%.*]]
 ;
   %1 = tail call <2 x i64> @llvm.x86.avx512.psrav.q.128(<2 x i64> %v, <2 x i64> zeroinitializer)
   ret <2 x i64> %1
@@ -1666,7 +1666,7 @@ define <2 x i64> @avx512_psrav_q_128_0(<2 x i64> %v) {
 
 define <4 x i64> @avx512_psrav_q_256_0(<4 x i64> %v) {
 ; CHECK-LABEL: @avx512_psrav_q_256_0(
-; CHECK-NEXT:    ret <4 x i64> %v
+; CHECK-NEXT:    ret <4 x i64> [[V:%.*]]
 ;
   %1 = tail call <4 x i64> @llvm.x86.avx512.psrav.q.256(<4 x i64> %v, <4 x i64> zeroinitializer)
   ret <4 x i64> %1
@@ -1674,14 +1674,16 @@ define <4 x i64> @avx512_psrav_q_256_0(<4 x i64> %v) {
 
 define <2 x i64> @avx512_psrav_q_128_var(<2 x i64> %v) {
 ; CHECK-LABEL: @avx512_psrav_q_128_var(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <2 x i64> %v, <i64 0, i64 8>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <2 x i64> [[V:%.*]], <i64 0, i64 8>
+; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+;
   %1 = tail call <2 x i64> @llvm.x86.avx512.psrav.q.128(<2 x i64> %v, <2 x i64> <i64 0, i64 8>)
   ret <2 x i64> %1
 }
 
 define <4 x i64> @avx512_psrav_q_256_var(<4 x i64> %v) {
 ; CHECK-LABEL: @avx512_psrav_q_256_var(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <4 x i64> %v, <i64 0, i64 8, i64 16, i64 31>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <4 x i64> [[V:%.*]], <i64 0, i64 8, i64 16, i64 31>
 ; CHECK-NEXT:    ret <4 x i64> [[TMP1]]
 ;
   %1 = tail call <4 x i64> @llvm.x86.avx512.psrav.q.256(<4 x i64> %v, <4 x i64> <i64 0, i64 8, i64 16, i64 31>)
@@ -1690,7 +1692,7 @@ define <4 x i64> @avx512_psrav_q_256_var(<4 x i64> %v) {
 
 define <2 x i64> @avx512_psrav_q_128_allbig(<2 x i64> %v) {
 ; CHECK-LABEL: @avx512_psrav_q_128_allbig(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <2 x i64> %v, <i64 63, i64 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <2 x i64> [[V:%.*]], <i64 63, i64 undef>
 ; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
 ;
   %1 = tail call <2 x i64> @llvm.x86.avx512.psrav.q.128(<2 x i64> %v, <2 x i64> <i64 64, i64 undef>)
@@ -1699,7 +1701,7 @@ define <2 x i64> @avx512_psrav_q_128_allbig(<2 x i64> %v) {
 
 define <4 x i64> @avx512_psrav_q_256_allbig(<4 x i64> %v) {
 ; CHECK-LABEL: @avx512_psrav_q_256_allbig(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <4 x i64> %v, <i64 63, i64 undef, i64 63, i64 63>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <4 x i64> [[V:%.*]], <i64 63, i64 undef, i64 63, i64 63>
 ; CHECK-NEXT:    ret <4 x i64> [[TMP1]]
 ;
   %1 = tail call <4 x i64> @llvm.x86.avx512.psrav.q.256(<4 x i64> %v, <4 x i64> <i64 64, i64 undef, i64 -128, i64 -60>)
@@ -1708,7 +1710,7 @@ define <4 x i64> @avx512_psrav_q_256_allbig(<4 x i64> %v) {
 
 define <2 x i64> @avx512_psrav_q_128_undef(<2 x i64> %v) {
 ; CHECK-LABEL: @avx512_psrav_q_128_undef(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <2 x i64> %v, <i64 undef, i64 8>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <2 x i64> [[V:%.*]], <i64 undef, i64 8>
 ; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
 ;
   %1 = insertelement <2 x i64> <i64 0, i64 8>, i64 undef, i64 0
@@ -1718,7 +1720,7 @@ define <2 x i64> @avx512_psrav_q_128_undef(<2 x i64> %v) {
 
 define <4 x i64> @avx512_psrav_q_256_undef(<4 x i64> %v) {
 ; CHECK-LABEL: @avx512_psrav_q_256_undef(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <4 x i64> %v, <i64 undef, i64 8, i64 16, i64 31>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <4 x i64> [[V:%.*]], <i64 undef, i64 8, i64 16, i64 31>
 ; CHECK-NEXT:    ret <4 x i64> [[TMP1]]
 ;
   %1 = insertelement <4 x i64> <i64 0, i64 8, i64 16, i64 31>, i64 undef, i64 0
@@ -1728,7 +1730,7 @@ define <4 x i64> @avx512_psrav_q_256_undef(<4 x i64> %v) {
 
 define <8 x i64> @avx512_psrav_q_512_0(<8 x i64> %v) {
 ; CHECK-LABEL: @avx512_psrav_q_512_0(
-; CHECK-NEXT:    ret <8 x i64> %v
+; CHECK-NEXT:    ret <8 x i64> [[V:%.*]]
 ;
   %1 = tail call <8 x i64> @llvm.x86.avx512.psrav.q.512(<8 x i64> %v, <8 x i64> zeroinitializer)
   ret <8 x i64> %1
@@ -1736,7 +1738,7 @@ define <8 x i64> @avx512_psrav_q_512_0(<8 x i64> %v) {
 
 define <8 x i64> @avx512_psrav_q_512_var(<8 x i64> %v) {
 ; CHECK-LABEL: @avx512_psrav_q_512_var(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i64> %v, <i64 0, i64 8, i64 16, i64 31, i64 0, i64 8, i64 16, i64 31>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i64> [[V:%.*]], <i64 0, i64 8, i64 16, i64 31, i64 0, i64 8, i64 16, i64 31>
 ; CHECK-NEXT:    ret <8 x i64> [[TMP1]]
 ;
   %1 = tail call <8 x i64> @llvm.x86.avx512.psrav.q.512(<8 x i64> %v, <8 x i64> <i64 0, i64 8, i64 16, i64 31, i64 0, i64 8, i64 16, i64 31>)
@@ -1745,7 +1747,7 @@ define <8 x i64> @avx512_psrav_q_512_var(<8 x i64> %v) {
 
 define <8 x i64> @avx512_psrav_q_512_allbig(<8 x i64> %v) {
 ; CHECK-LABEL: @avx512_psrav_q_512_allbig(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i64> %v, <i64 63, i64 undef, i64 63, i64 63, i64 63, i64 undef, i64 63, i64 63>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i64> [[V:%.*]], <i64 63, i64 undef, i64 63, i64 63, i64 63, i64 undef, i64 63, i64 63>
 ; CHECK-NEXT:    ret <8 x i64> [[TMP1]]
 ;
   %1 = tail call <8 x i64> @llvm.x86.avx512.psrav.q.512(<8 x i64> %v, <8 x i64> <i64 64, i64 undef, i64 -128, i64 -60, i64 64, i64 undef, i64 -128, i64 -60>)
@@ -1754,7 +1756,7 @@ define <8 x i64> @avx512_psrav_q_512_allbig(<8 x i64> %v) {
 
 define <8 x i64> @avx512_psrav_q_512_undef(<8 x i64> %v) {
 ; CHECK-LABEL: @avx512_psrav_q_512_undef(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i64> %v, <i64 undef, i64 8, i64 16, i64 31, i64 0, i64 8, i64 16, i64 31>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i64> [[V:%.*]], <i64 undef, i64 8, i64 16, i64 31, i64 0, i64 8, i64 16, i64 31>
 ; CHECK-NEXT:    ret <8 x i64> [[TMP1]]
 ;
   %1 = insertelement <8 x i64> <i64 0, i64 8, i64 16, i64 31, i64 0, i64 8, i64 16, i64 31>, i64 undef, i64 0
@@ -1764,7 +1766,7 @@ define <8 x i64> @avx512_psrav_q_512_undef(<8 x i64> %v) {
 
 define <8 x i16> @avx512_psrav_w_128_0(<8 x i16> %v) {
 ; CHECK-LABEL: @avx512_psrav_w_128_0(
-; CHECK-NEXT:    ret <8 x i16> %v
+; CHECK-NEXT:    ret <8 x i16> [[V:%.*]]
 ;
   %1 = tail call <8 x i16> @llvm.x86.avx512.psrav.w.128(<8 x i16> %v, <8 x i16> zeroinitializer)
   ret <8 x i16> %1
@@ -1772,7 +1774,7 @@ define <8 x i16> @avx512_psrav_w_128_0(<8 x i16> %v) {
 
 define <8 x i16> @avx512_psrav_w_128_var(<8 x i16> %v) {
 ; CHECK-LABEL: @avx512_psrav_w_128_var(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i16> %v, <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i16> [[V:%.*]], <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>
 ; CHECK-NEXT:    ret <8 x i16> [[TMP1]]
 ;
   %1 = tail call <8 x i16> @llvm.x86.avx512.psrav.w.128(<8 x i16> %v, <8 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>)
@@ -1781,9 +1783,8 @@ define <8 x i16> @avx512_psrav_w_128_var(<8 x i16> %v) {
 
 define <8 x i16> @avx512_psrav_w_128_allbig(<8 x i16> %v) {
 ; CHECK-LABEL: @avx512_psrav_w_128_allbig(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i16> %v, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i16> [[V:%.*]], <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 undef>
 ; CHECK-NEXT:    ret <8 x i16> [[TMP1]]
-;
 ;
   %1 = tail call <8 x i16> @llvm.x86.avx512.psrav.w.128(<8 x i16> %v, <8 x i16> <i16 20, i16 -1, i16 -2, i16 33, i16 44, i16 55, i16 66, i16 undef>)
   ret <8 x i16> %1
@@ -1791,7 +1792,7 @@ define <8 x i16> @avx512_psrav_w_128_allbig(<8 x i16> %v) {
 
 define <8 x i16> @avx512_psrav_w_128_undef(<8 x i16> %v) {
 ; CHECK-LABEL: @avx512_psrav_w_128_undef(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i16> %v, <i16 undef, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i16> [[V:%.*]], <i16 undef, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>
 ; CHECK-NEXT:    ret <8 x i16> [[TMP1]]
 ;
   %1 = insertelement <8 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>, i16 undef, i64 0
@@ -1801,7 +1802,7 @@ define <8 x i16> @avx512_psrav_w_128_undef(<8 x i16> %v) {
 
 define <16 x i16> @avx512_psrav_w_256_0(<16 x i16> %v) {
 ; CHECK-LABEL: @avx512_psrav_w_256_0(
-; CHECK-NEXT:    ret <16 x i16> %v
+; CHECK-NEXT:    ret <16 x i16> [[V:%.*]]
 ;
   %1 = tail call <16 x i16> @llvm.x86.avx512.psrav.w.256(<16 x i16> %v, <16 x i16> zeroinitializer)
   ret <16 x i16> %1
@@ -1809,7 +1810,7 @@ define <16 x i16> @avx512_psrav_w_256_0(<16 x i16> %v) {
 
 define <16 x i16> @avx512_psrav_w_256_var(<16 x i16> %v) {
 ; CHECK-LABEL: @avx512_psrav_w_256_var(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <16 x i16> %v, <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <16 x i16> [[V:%.*]], <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>
 ; CHECK-NEXT:    ret <16 x i16> [[TMP1]]
 ;
   %1 = tail call <16 x i16> @llvm.x86.avx512.psrav.w.256(<16 x i16> %v, <16 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>)
@@ -1818,9 +1819,8 @@ define <16 x i16> @avx512_psrav_w_256_var(<16 x i16> %v) {
 
 define <16 x i16> @avx512_psrav_w_256_allbig(<16 x i16> %v) {
 ; CHECK-LABEL: @avx512_psrav_w_256_allbig(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <16 x i16> %v, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 undef, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <16 x i16> [[V:%.*]], <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 undef, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
 ; CHECK-NEXT:    ret <16 x i16> [[TMP1]]
-;
 ;
   %1 = tail call <16 x i16> @llvm.x86.avx512.psrav.w.256(<16 x i16> %v, <16 x i16> <i16 20, i16 -1, i16 -2, i16 33, i16 44, i16 55, i16 66, i16 -7, i16 undef, i16 64, i16 -10, i16 256, i16 16, i16 28, i16 65535, i16 32767>)
   ret <16 x i16> %1
@@ -1828,7 +1828,7 @@ define <16 x i16> @avx512_psrav_w_256_allbig(<16 x i16> %v) {
 
 define <16 x i16> @avx512_psrav_w_256_undef(<16 x i16> %v) {
 ; CHECK-LABEL: @avx512_psrav_w_256_undef(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <16 x i16> %v, <i16 undef, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <16 x i16> [[V:%.*]], <i16 undef, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>
 ; CHECK-NEXT:    ret <16 x i16> [[TMP1]]
 ;
   %1 = insertelement <16 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>, i16 undef, i64 0
@@ -1838,7 +1838,7 @@ define <16 x i16> @avx512_psrav_w_256_undef(<16 x i16> %v) {
 
 define <32 x i16> @avx512_psrav_w_512_0(<32 x i16> %v) {
 ; CHECK-LABEL: @avx512_psrav_w_512_0(
-; CHECK-NEXT:    ret <32 x i16> %v
+; CHECK-NEXT:    ret <32 x i16> [[V:%.*]]
 ;
   %1 = tail call <32 x i16> @llvm.x86.avx512.psrav.w.512(<32 x i16> %v, <32 x i16> zeroinitializer)
   ret <32 x i16> %1
@@ -1846,7 +1846,7 @@ define <32 x i16> @avx512_psrav_w_512_0(<32 x i16> %v) {
 
 define <32 x i16> @avx512_psrav_w_512_var(<32 x i16> %v) {
 ; CHECK-LABEL: @avx512_psrav_w_512_var(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <32 x i16> %v, <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 15, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <32 x i16> [[V:%.*]], <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 15, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>
 ; CHECK-NEXT:    ret <32 x i16> [[TMP1]]
 ;
   %1 = tail call <32 x i16> @llvm.x86.avx512.psrav.w.512(<32 x i16> %v, <32 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 15, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>)
@@ -1855,7 +1855,7 @@ define <32 x i16> @avx512_psrav_w_512_var(<32 x i16> %v) {
 
 define <32 x i16> @avx512_psrav_w_512_allbig(<32 x i16> %v) {
 ; CHECK-LABEL: @avx512_psrav_w_512_allbig(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <32 x i16> %v, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 undef, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 undef, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 undef, i16 15, i16 15, i16 undef, i16 15, i16 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <32 x i16> [[V:%.*]], <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 undef, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 undef, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 undef, i16 15, i16 15, i16 undef, i16 15, i16 15>
 ; CHECK-NEXT:    ret <32 x i16> [[TMP1]]
 ;
   %1 = tail call <32 x i16> @llvm.x86.avx512.psrav.w.512(<32 x i16> %v, <32 x i16> <i16 20, i16 -1, i16 -2, i16 33, i16 44, i16 55, i16 66, i16 -7, i16 undef, i16 64, i16 -10, i16 128, i16 16, i16 28, i16 65535, i16 32767, i16 56, i16 -14, i16 undef, i16 16, i16 67, i16 567, i16 -32768, i16 4096, i16 8192, i16 -12345, i16 undef, i16 345, i16 123, i16 undef, i16 1024, i16 54321>)
@@ -1864,7 +1864,7 @@ define <32 x i16> @avx512_psrav_w_512_allbig(<32 x i16> %v) {
 
 define <32 x i16> @avx512_psrav_w_512_undef(<32 x i16> %v) {
 ; CHECK-LABEL: @avx512_psrav_w_512_undef(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <32 x i16> %v, <i16 undef, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 15, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <32 x i16> [[V:%.*]], <i16 undef, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 15, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>
 ; CHECK-NEXT:    ret <32 x i16> [[TMP1]]
 ;
   %1 = insertelement <32 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 15, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>, i16 undef, i64 0
@@ -1878,7 +1878,7 @@ define <32 x i16> @avx512_psrav_w_512_undef(<32 x i16> %v) {
 
 define <4 x i32> @avx2_psrlv_d_128_0(<4 x i32> %v) {
 ; CHECK-LABEL: @avx2_psrlv_d_128_0(
-; CHECK-NEXT:    ret <4 x i32> %v
+; CHECK-NEXT:    ret <4 x i32> [[V:%.*]]
 ;
   %1 = tail call <4 x i32> @llvm.x86.avx2.psrlv.d(<4 x i32> %v, <4 x i32> zeroinitializer)
   ret <4 x i32> %1
@@ -1886,7 +1886,7 @@ define <4 x i32> @avx2_psrlv_d_128_0(<4 x i32> %v) {
 
 define <8 x i32> @avx2_psrlv_d_256_0(<8 x i32> %v) {
 ; CHECK-LABEL: @avx2_psrlv_d_256_0(
-; CHECK-NEXT:    ret <8 x i32> %v
+; CHECK-NEXT:    ret <8 x i32> [[V:%.*]]
 ;
   %1 = tail call <8 x i32> @llvm.x86.avx2.psrlv.d.256(<8 x i32> %v, <8 x i32> zeroinitializer)
   ret <8 x i32> %1
@@ -1894,7 +1894,7 @@ define <8 x i32> @avx2_psrlv_d_256_0(<8 x i32> %v) {
 
 define <4 x i32> @avx2_psrlv_d_128_var(<4 x i32> %v) {
 ; CHECK-LABEL: @avx2_psrlv_d_128_var(
-; CHECK-NEXT:    [[TMP1:%.*]] = lshr <4 x i32> %v, <i32 0, i32 8, i32 16, i32 31>
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <4 x i32> [[V:%.*]], <i32 0, i32 8, i32 16, i32 31>
 ; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
 ;
   %1 = tail call <4 x i32> @llvm.x86.avx2.psrlv.d(<4 x i32> %v, <4 x i32> <i32 0, i32 8, i32 16, i32 31>)
@@ -1903,7 +1903,7 @@ define <4 x i32> @avx2_psrlv_d_128_var(<4 x i32> %v) {
 
 define <8 x i32> @avx2_psrlv_d_256_var(<8 x i32> %v) {
 ; CHECK-LABEL: @avx2_psrlv_d_256_var(
-; CHECK-NEXT:    [[TMP1:%.*]] = lshr <8 x i32> %v, <i32 0, i32 8, i32 16, i32 24, i32 31, i32 24, i32 8, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <8 x i32> [[V:%.*]], <i32 0, i32 8, i32 16, i32 24, i32 31, i32 24, i32 8, i32 0>
 ; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
 ;
   %1 = tail call <8 x i32> @llvm.x86.avx2.psrlv.d.256(<8 x i32> %v, <8 x i32> <i32 0, i32 8, i32 16, i32 24, i32 31, i32 24, i32 8, i32 0>)
@@ -1912,7 +1912,7 @@ define <8 x i32> @avx2_psrlv_d_256_var(<8 x i32> %v) {
 
 define <4 x i32> @avx2_psrlv_d_128_big(<4 x i32> %v) {
 ; CHECK-LABEL: @avx2_psrlv_d_128_big(
-; CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x i32> @llvm.x86.avx2.psrlv.d(<4 x i32> %v, <4 x i32> <i32 0, i32 8, i32 16, i32 64>)
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x i32> @llvm.x86.avx2.psrlv.d(<4 x i32> [[V:%.*]], <4 x i32> <i32 0, i32 8, i32 16, i32 64>)
 ; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
 ;
   %1 = tail call <4 x i32> @llvm.x86.avx2.psrlv.d(<4 x i32> %v, <4 x i32> <i32 0, i32 8, i32 16, i32 64>)
@@ -1921,7 +1921,7 @@ define <4 x i32> @avx2_psrlv_d_128_big(<4 x i32> %v) {
 
 define <8 x i32> @avx2_psrlv_d_256_big(<8 x i32> %v) {
 ; CHECK-LABEL: @avx2_psrlv_d_256_big(
-; CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i32> @llvm.x86.avx2.psrlv.d.256(<8 x i32> %v, <8 x i32> <i32 0, i32 8, i32 16, i32 64, i32 31, i32 24, i32 8, i32 0>)
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i32> @llvm.x86.avx2.psrlv.d.256(<8 x i32> [[V:%.*]], <8 x i32> <i32 0, i32 8, i32 16, i32 64, i32 31, i32 24, i32 8, i32 0>)
 ; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
 ;
   %1 = tail call <8 x i32> @llvm.x86.avx2.psrlv.d.256(<8 x i32> %v, <8 x i32> <i32 0, i32 8, i32 16, i32 64, i32 31, i32 24, i32 8, i32 0>)
@@ -1946,7 +1946,7 @@ define <8 x i32> @avx2_psrlv_d_256_allbig(<8 x i32> %v) {
 
 define <4 x i32> @avx2_psrlv_d_128_undef(<4 x i32> %v) {
 ; CHECK-LABEL: @avx2_psrlv_d_128_undef(
-; CHECK-NEXT:    [[TMP1:%.*]] = lshr <4 x i32> %v, <i32 undef, i32 8, i32 16, i32 31>
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <4 x i32> [[V:%.*]], <i32 undef, i32 8, i32 16, i32 31>
 ; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
 ;
   %1 = insertelement <4 x i32> <i32 0, i32 8, i32 16, i32 31>, i32 undef, i32 0
@@ -1956,7 +1956,7 @@ define <4 x i32> @avx2_psrlv_d_128_undef(<4 x i32> %v) {
 
 define <8 x i32> @avx2_psrlv_d_256_undef(<8 x i32> %v) {
 ; CHECK-LABEL: @avx2_psrlv_d_256_undef(
-; CHECK-NEXT:    [[TMP1:%.*]] = lshr <8 x i32> %v, <i32 0, i32 undef, i32 16, i32 31, i32 31, i32 24, i32 8, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <8 x i32> [[V:%.*]], <i32 0, i32 undef, i32 16, i32 31, i32 31, i32 24, i32 8, i32 0>
 ; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
 ;
   %1 = insertelement <8 x i32> <i32 0, i32 8, i32 16, i32 31, i32 31, i32 24, i32 8, i32 0>, i32 undef, i32 1
@@ -1966,7 +1966,7 @@ define <8 x i32> @avx2_psrlv_d_256_undef(<8 x i32> %v) {
 
 define <2 x i64> @avx2_psrlv_q_128_0(<2 x i64> %v) {
 ; CHECK-LABEL: @avx2_psrlv_q_128_0(
-; CHECK-NEXT:    ret <2 x i64> %v
+; CHECK-NEXT:    ret <2 x i64> [[V:%.*]]
 ;
   %1 = tail call <2 x i64> @llvm.x86.avx2.psrlv.q(<2 x i64> %v, <2 x i64> zeroinitializer)
   ret <2 x i64> %1
@@ -1974,7 +1974,7 @@ define <2 x i64> @avx2_psrlv_q_128_0(<2 x i64> %v) {
 
 define <4 x i64> @avx2_psrlv_q_256_0(<4 x i64> %v) {
 ; CHECK-LABEL: @avx2_psrlv_q_256_0(
-; CHECK-NEXT:    ret <4 x i64> %v
+; CHECK-NEXT:    ret <4 x i64> [[V:%.*]]
 ;
   %1 = tail call <4 x i64> @llvm.x86.avx2.psrlv.q.256(<4 x i64> %v, <4 x i64> zeroinitializer)
   ret <4 x i64> %1
@@ -1982,7 +1982,7 @@ define <4 x i64> @avx2_psrlv_q_256_0(<4 x i64> %v) {
 
 define <2 x i64> @avx2_psrlv_q_128_var(<2 x i64> %v) {
 ; CHECK-LABEL: @avx2_psrlv_q_128_var(
-; CHECK-NEXT:    [[TMP1:%.*]] = lshr <2 x i64> %v, <i64 0, i64 8>
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <2 x i64> [[V:%.*]], <i64 0, i64 8>
 ; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
 ;
   %1 = tail call <2 x i64> @llvm.x86.avx2.psrlv.q(<2 x i64> %v, <2 x i64> <i64 0, i64 8>)
@@ -1991,7 +1991,7 @@ define <2 x i64> @avx2_psrlv_q_128_var(<2 x i64> %v) {
 
 define <4 x i64> @avx2_psrlv_q_256_var(<4 x i64> %v) {
 ; CHECK-LABEL: @avx2_psrlv_q_256_var(
-; CHECK-NEXT:    [[TMP1:%.*]] = lshr <4 x i64> %v, <i64 0, i64 8, i64 16, i64 31>
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <4 x i64> [[V:%.*]], <i64 0, i64 8, i64 16, i64 31>
 ; CHECK-NEXT:    ret <4 x i64> [[TMP1]]
 ;
   %1 = tail call <4 x i64> @llvm.x86.avx2.psrlv.q.256(<4 x i64> %v, <4 x i64> <i64 0, i64 8, i64 16, i64 31>)
@@ -2000,7 +2000,7 @@ define <4 x i64> @avx2_psrlv_q_256_var(<4 x i64> %v) {
 
 define <2 x i64> @avx2_psrlv_q_128_big(<2 x i64> %v) {
 ; CHECK-LABEL: @avx2_psrlv_q_128_big(
-; CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x i64> @llvm.x86.avx2.psrlv.q(<2 x i64> %v, <2 x i64> <i64 0, i64 128>)
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x i64> @llvm.x86.avx2.psrlv.q(<2 x i64> [[V:%.*]], <2 x i64> <i64 0, i64 128>)
 ; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
 ;
   %1 = tail call <2 x i64> @llvm.x86.avx2.psrlv.q(<2 x i64> %v, <2 x i64> <i64 0, i64 128>)
@@ -2009,7 +2009,7 @@ define <2 x i64> @avx2_psrlv_q_128_big(<2 x i64> %v) {
 
 define <4 x i64> @avx2_psrlv_q_256_big(<4 x i64> %v) {
 ; CHECK-LABEL: @avx2_psrlv_q_256_big(
-; CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x i64> @llvm.x86.avx2.psrlv.q.256(<4 x i64> %v, <4 x i64> <i64 0, i64 8, i64 16, i64 64>)
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x i64> @llvm.x86.avx2.psrlv.q.256(<4 x i64> [[V:%.*]], <4 x i64> <i64 0, i64 8, i64 16, i64 64>)
 ; CHECK-NEXT:    ret <4 x i64> [[TMP1]]
 ;
   %1 = tail call <4 x i64> @llvm.x86.avx2.psrlv.q.256(<4 x i64> %v, <4 x i64> <i64 0, i64 8, i64 16, i64 64>)
@@ -2045,7 +2045,7 @@ define <2 x i64> @avx2_psrlv_q_128_undef(<2 x i64> %v) {
 
 define <4 x i64> @avx2_psrlv_q_256_undef(<4 x i64> %v) {
 ; CHECK-LABEL: @avx2_psrlv_q_256_undef(
-; CHECK-NEXT:    [[TMP1:%.*]] = lshr <4 x i64> %v, <i64 undef, i64 8, i64 16, i64 31>
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <4 x i64> [[V:%.*]], <i64 undef, i64 8, i64 16, i64 31>
 ; CHECK-NEXT:    ret <4 x i64> [[TMP1]]
 ;
   %1 = insertelement <4 x i64> <i64 0, i64 8, i64 16, i64 31>, i64 undef, i64 0
@@ -2055,7 +2055,7 @@ define <4 x i64> @avx2_psrlv_q_256_undef(<4 x i64> %v) {
 
 define <16 x i32> @avx2_psrlv_d_512_0(<16 x i32> %v) {
 ; CHECK-LABEL: @avx2_psrlv_d_512_0(
-; CHECK-NEXT:    ret <16 x i32> %v
+; CHECK-NEXT:    ret <16 x i32> [[V:%.*]]
 ;
   %1 = tail call <16 x i32> @llvm.x86.avx512.psrlv.d.512(<16 x i32> %v, <16 x i32> zeroinitializer)
   ret <16 x i32> %1
@@ -2063,7 +2063,7 @@ define <16 x i32> @avx2_psrlv_d_512_0(<16 x i32> %v) {
 
 define <16 x i32> @avx512_psrlv_d_512_var(<16 x i32> %v) {
 ; CHECK-LABEL: @avx512_psrlv_d_512_var(
-; CHECK-NEXT:    [[TMP1:%.*]] = lshr <16 x i32> %v, <i32 0, i32 8, i32 16, i32 24, i32 31, i32 24, i32 8, i32 0, i32 0, i32 8, i32 16, i32 24, i32 31, i32 24, i32 8, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <16 x i32> [[V:%.*]], <i32 0, i32 8, i32 16, i32 24, i32 31, i32 24, i32 8, i32 0, i32 0, i32 8, i32 16, i32 24, i32 31, i32 24, i32 8, i32 0>
 ; CHECK-NEXT:    ret <16 x i32> [[TMP1]]
 ;
   %1 = tail call <16 x i32> @llvm.x86.avx512.psrlv.d.512(<16 x i32> %v, <16 x i32> <i32 0, i32 8, i32 16, i32 24, i32 31, i32 24, i32 8, i32 0, i32 0, i32 8, i32 16, i32 24, i32 31, i32 24, i32 8, i32 0>)
@@ -2072,7 +2072,7 @@ define <16 x i32> @avx512_psrlv_d_512_var(<16 x i32> %v) {
 
 define <16 x i32> @avx512_psrlv_d_512_big(<16 x i32> %v) {
 ; CHECK-LABEL: @avx512_psrlv_d_512_big(
-; CHECK-NEXT:    [[TMP1:%.*]] = tail call <16 x i32> @llvm.x86.avx512.psrlv.d.512(<16 x i32> %v, <16 x i32> <i32 0, i32 8, i32 16, i32 64, i32 31, i32 24, i32 8, i32 0, i32 0, i32 8, i32 16, i32 64, i32 31, i32 24, i32 8, i32 0>)
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <16 x i32> @llvm.x86.avx512.psrlv.d.512(<16 x i32> [[V:%.*]], <16 x i32> <i32 0, i32 8, i32 16, i32 64, i32 31, i32 24, i32 8, i32 0, i32 0, i32 8, i32 16, i32 64, i32 31, i32 24, i32 8, i32 0>)
 ; CHECK-NEXT:    ret <16 x i32> [[TMP1]]
 ;
   %1 = tail call <16 x i32> @llvm.x86.avx512.psrlv.d.512(<16 x i32> %v, <16 x i32> <i32 0, i32 8, i32 16, i32 64, i32 31, i32 24, i32 8, i32 0, i32 0, i32 8, i32 16, i32 64, i32 31, i32 24, i32 8, i32 0>)
@@ -2089,7 +2089,7 @@ define <16 x i32> @avx512_psrlv_d_512_allbig(<16 x i32> %v) {
 
 define <16 x i32> @avx512_psrlv_d_512_undef(<16 x i32> %v) {
 ; CHECK-LABEL: @avx512_psrlv_d_512_undef(
-; CHECK-NEXT:    [[TMP1:%.*]] = lshr <16 x i32> %v, <i32 0, i32 undef, i32 16, i32 31, i32 31, i32 24, i32 8, i32 0, i32 0, i32 8, i32 16, i32 31, i32 31, i32 24, i32 8, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <16 x i32> [[V:%.*]], <i32 0, i32 undef, i32 16, i32 31, i32 31, i32 24, i32 8, i32 0, i32 0, i32 8, i32 16, i32 31, i32 31, i32 24, i32 8, i32 0>
 ; CHECK-NEXT:    ret <16 x i32> [[TMP1]]
 ;
   %1 = insertelement <16 x i32> <i32 0, i32 8, i32 16, i32 31, i32 31, i32 24, i32 8, i32 0, i32 0, i32 8, i32 16, i32 31, i32 31, i32 24, i32 8, i32 0>, i32 undef, i32 1
@@ -2099,7 +2099,7 @@ define <16 x i32> @avx512_psrlv_d_512_undef(<16 x i32> %v) {
 
 define <8 x i64> @avx512_psrlv_q_512_0(<8 x i64> %v) {
 ; CHECK-LABEL: @avx512_psrlv_q_512_0(
-; CHECK-NEXT:    ret <8 x i64> %v
+; CHECK-NEXT:    ret <8 x i64> [[V:%.*]]
 ;
   %1 = tail call <8 x i64> @llvm.x86.avx512.psrlv.q.512(<8 x i64> %v, <8 x i64> zeroinitializer)
   ret <8 x i64> %1
@@ -2107,7 +2107,7 @@ define <8 x i64> @avx512_psrlv_q_512_0(<8 x i64> %v) {
 
 define <8 x i64> @avx512_psrlv_q_512_var(<8 x i64> %v) {
 ; CHECK-LABEL: @avx512_psrlv_q_512_var(
-; CHECK-NEXT:    [[TMP1:%.*]] = lshr <8 x i64> %v, <i64 0, i64 8, i64 16, i64 31, i64 0, i64 8, i64 16, i64 31>
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <8 x i64> [[V:%.*]], <i64 0, i64 8, i64 16, i64 31, i64 0, i64 8, i64 16, i64 31>
 ; CHECK-NEXT:    ret <8 x i64> [[TMP1]]
 ;
   %1 = tail call <8 x i64> @llvm.x86.avx512.psrlv.q.512(<8 x i64> %v, <8 x i64> <i64 0, i64 8, i64 16, i64 31, i64 0, i64 8, i64 16, i64 31>)
@@ -2116,7 +2116,7 @@ define <8 x i64> @avx512_psrlv_q_512_var(<8 x i64> %v) {
 
 define <8 x i64> @avx512_psrlv_q_512_big(<8 x i64> %v) {
 ; CHECK-LABEL: @avx512_psrlv_q_512_big(
-; CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i64> @llvm.x86.avx512.psrlv.q.512(<8 x i64> %v, <8 x i64> <i64 0, i64 8, i64 16, i64 64, i64 0, i64 8, i64 16, i64 64>)
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i64> @llvm.x86.avx512.psrlv.q.512(<8 x i64> [[V:%.*]], <8 x i64> <i64 0, i64 8, i64 16, i64 64, i64 0, i64 8, i64 16, i64 64>)
 ; CHECK-NEXT:    ret <8 x i64> [[TMP1]]
 ;
   %1 = tail call <8 x i64> @llvm.x86.avx512.psrlv.q.512(<8 x i64> %v, <8 x i64> <i64 0, i64 8, i64 16, i64 64, i64 0, i64 8, i64 16, i64 64>)
@@ -2133,7 +2133,7 @@ define <8 x i64> @avx512_psrlv_q_512_allbig(<8 x i64> %v) {
 
 define <8 x i64> @avx512_psrlv_q_512_undef(<8 x i64> %v) {
 ; CHECK-LABEL: @avx512_psrlv_q_512_undef(
-; CHECK-NEXT:    [[TMP1:%.*]] = lshr <8 x i64> %v, <i64 undef, i64 8, i64 16, i64 31, i64 0, i64 8, i64 16, i64 31>
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <8 x i64> [[V:%.*]], <i64 undef, i64 8, i64 16, i64 31, i64 0, i64 8, i64 16, i64 31>
 ; CHECK-NEXT:    ret <8 x i64> [[TMP1]]
 ;
   %1 = insertelement <8 x i64> <i64 0, i64 8, i64 16, i64 31, i64 0, i64 8, i64 16, i64 31>, i64 undef, i64 0
@@ -2143,7 +2143,7 @@ define <8 x i64> @avx512_psrlv_q_512_undef(<8 x i64> %v) {
 
 define <8 x i16> @avx512_psrlv_w_128_0(<8 x i16> %v) {
 ; CHECK-LABEL: @avx512_psrlv_w_128_0(
-; CHECK-NEXT:    ret <8 x i16> %v
+; CHECK-NEXT:    ret <8 x i16> [[V:%.*]]
 ;
   %1 = tail call <8 x i16> @llvm.x86.avx512.psrlv.w.128(<8 x i16> %v, <8 x i16> zeroinitializer)
   ret <8 x i16> %1
@@ -2151,7 +2151,7 @@ define <8 x i16> @avx512_psrlv_w_128_0(<8 x i16> %v) {
 
 define <8 x i16> @avx512_psrlv_w_128_var(<8 x i16> %v) {
 ; CHECK-LABEL: @avx512_psrlv_w_128_var(
-; CHECK-NEXT:    [[TMP1:%.*]] = lshr <8 x i16> %v, <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <8 x i16> [[V:%.*]], <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>
 ; CHECK-NEXT:    ret <8 x i16> [[TMP1]]
 ;
   %1 = tail call <8 x i16> @llvm.x86.avx512.psrlv.w.128(<8 x i16> %v, <8 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>)
@@ -2160,7 +2160,7 @@ define <8 x i16> @avx512_psrlv_w_128_var(<8 x i16> %v) {
 
 define <8 x i16> @avx512_psrlv_w_128_big(<8 x i16> %v) {
 ; CHECK-LABEL: @avx512_psrlv_w_128_big(
-; CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i16> @llvm.x86.avx512.psrlv.w.128(<8 x i16> %v, <8 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 16>)
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i16> @llvm.x86.avx512.psrlv.w.128(<8 x i16> [[V:%.*]], <8 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 16>)
 ; CHECK-NEXT:    ret <8 x i16> [[TMP1]]
 ;
   %1 = tail call <8 x i16> @llvm.x86.avx512.psrlv.w.128(<8 x i16> %v, <8 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 16>)
@@ -2177,7 +2177,7 @@ define <8 x i16> @avx512_psrlv_w_128_allbig(<8 x i16> %v) {
 
 define <8 x i16> @avx512_psrlv_w_128_undef(<8 x i16> %v) {
 ; CHECK-LABEL: @avx512_psrlv_w_128_undef(
-; CHECK-NEXT:    [[TMP1:%.*]] = lshr <8 x i16> %v, <i16 undef, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <8 x i16> [[V:%.*]], <i16 undef, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>
 ; CHECK-NEXT:    ret <8 x i16> [[TMP1]]
 ;
   %1 = insertelement <8 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>, i16 undef, i64 0
@@ -2187,7 +2187,7 @@ define <8 x i16> @avx512_psrlv_w_128_undef(<8 x i16> %v) {
 
 define <16 x i16> @avx512_psrlv_w_256_0(<16 x i16> %v) {
 ; CHECK-LABEL: @avx512_psrlv_w_256_0(
-; CHECK-NEXT:    ret <16 x i16> %v
+; CHECK-NEXT:    ret <16 x i16> [[V:%.*]]
 ;
   %1 = tail call <16 x i16> @llvm.x86.avx512.psrlv.w.256(<16 x i16> %v, <16 x i16> zeroinitializer)
   ret <16 x i16> %1
@@ -2195,7 +2195,7 @@ define <16 x i16> @avx512_psrlv_w_256_0(<16 x i16> %v) {
 
 define <16 x i16> @avx512_psrlv_w_256_var(<16 x i16> %v) {
 ; CHECK-LABEL: @avx512_psrlv_w_256_var(
-; CHECK-NEXT:    [[TMP1:%.*]] = lshr <16 x i16> %v, <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <16 x i16> [[V:%.*]], <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>
 ; CHECK-NEXT:    ret <16 x i16> [[TMP1]]
 ;
   %1 = tail call <16 x i16> @llvm.x86.avx512.psrlv.w.256(<16 x i16> %v, <16 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>)
@@ -2204,7 +2204,7 @@ define <16 x i16> @avx512_psrlv_w_256_var(<16 x i16> %v) {
 
 define <16 x i16> @avx512_psrlv_w_256_big(<16 x i16> %v) {
 ; CHECK-LABEL: @avx512_psrlv_w_256_big(
-; CHECK-NEXT:    [[TMP1:%.*]] = tail call <16 x i16> @llvm.x86.avx512.psrlv.w.256(<16 x i16> %v, <16 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 16>)
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <16 x i16> @llvm.x86.avx512.psrlv.w.256(<16 x i16> [[V:%.*]], <16 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 16>)
 ; CHECK-NEXT:    ret <16 x i16> [[TMP1]]
 ;
   %1 = tail call <16 x i16> @llvm.x86.avx512.psrlv.w.256(<16 x i16> %v, <16 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 16>)
@@ -2221,7 +2221,7 @@ define <16 x i16> @avx512_psrlv_w_256_allbig(<16 x i16> %v) {
 
 define <16 x i16> @avx512_psrlv_w_256_undef(<16 x i16> %v) {
 ; CHECK-LABEL: @avx512_psrlv_w_256_undef(
-; CHECK-NEXT:    [[TMP1:%.*]] = lshr <16 x i16> %v, <i16 undef, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <16 x i16> [[V:%.*]], <i16 undef, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>
 ; CHECK-NEXT:    ret <16 x i16> [[TMP1]]
 ;
   %1 = insertelement <16 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>, i16 undef, i64 0
@@ -2231,7 +2231,7 @@ define <16 x i16> @avx512_psrlv_w_256_undef(<16 x i16> %v) {
 
 define <32 x i16> @avx512_psrlv_w_512_0(<32 x i16> %v) {
 ; CHECK-LABEL: @avx512_psrlv_w_512_0(
-; CHECK-NEXT:    ret <32 x i16> %v
+; CHECK-NEXT:    ret <32 x i16> [[V:%.*]]
 ;
   %1 = tail call <32 x i16> @llvm.x86.avx512.psrlv.w.512(<32 x i16> %v, <32 x i16> zeroinitializer)
   ret <32 x i16> %1
@@ -2239,7 +2239,7 @@ define <32 x i16> @avx512_psrlv_w_512_0(<32 x i16> %v) {
 
 define <32 x i16> @avx512_psrlv_w_512_var(<32 x i16> %v) {
 ; CHECK-LABEL: @avx512_psrlv_w_512_var(
-; CHECK-NEXT:    [[TMP1:%.*]] = lshr <32 x i16> %v, <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 15, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <32 x i16> [[V:%.*]], <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 15, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>
 ; CHECK-NEXT:    ret <32 x i16> [[TMP1]]
 ;
   %1 = tail call <32 x i16> @llvm.x86.avx512.psrlv.w.512(<32 x i16> %v, <32 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 15, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>)
@@ -2248,7 +2248,7 @@ define <32 x i16> @avx512_psrlv_w_512_var(<32 x i16> %v) {
 
 define <32 x i16> @avx512_psrlv_w_512_big(<32 x i16> %v) {
 ; CHECK-LABEL: @avx512_psrlv_w_512_big(
-; CHECK-NEXT:    [[TMP1:%.*]] = tail call <32 x i16> @llvm.x86.avx512.psrlv.w.512(<32 x i16> %v, <32 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 16, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>)
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <32 x i16> @llvm.x86.avx512.psrlv.w.512(<32 x i16> [[V:%.*]], <32 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 16, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>)
 ; CHECK-NEXT:    ret <32 x i16> [[TMP1]]
 ;
   %1 = tail call <32 x i16> @llvm.x86.avx512.psrlv.w.512(<32 x i16> %v, <32 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 16, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>)
@@ -2265,7 +2265,7 @@ define <32 x i16> @avx512_psrlv_w_512_allbig(<32 x i16> %v) {
 
 define <32 x i16> @avx512_psrlv_w_512_undef(<32 x i16> %v) {
 ; CHECK-LABEL: @avx512_psrlv_w_512_undef(
-; CHECK-NEXT:    [[TMP1:%.*]] = lshr <32 x i16> %v, <i16 undef, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 15, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <32 x i16> [[V:%.*]], <i16 undef, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 15, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>
 ; CHECK-NEXT:    ret <32 x i16> [[TMP1]]
 ;
   %1 = insertelement <32 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 15, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>, i16 undef, i64 0
@@ -2279,7 +2279,7 @@ define <32 x i16> @avx512_psrlv_w_512_undef(<32 x i16> %v) {
 
 define <4 x i32> @avx2_psllv_d_128_0(<4 x i32> %v) {
 ; CHECK-LABEL: @avx2_psllv_d_128_0(
-; CHECK-NEXT:    ret <4 x i32> %v
+; CHECK-NEXT:    ret <4 x i32> [[V:%.*]]
 ;
   %1 = tail call <4 x i32> @llvm.x86.avx2.psllv.d(<4 x i32> %v, <4 x i32> zeroinitializer)
   ret <4 x i32> %1
@@ -2287,7 +2287,7 @@ define <4 x i32> @avx2_psllv_d_128_0(<4 x i32> %v) {
 
 define <8 x i32> @avx2_psllv_d_256_0(<8 x i32> %v) {
 ; CHECK-LABEL: @avx2_psllv_d_256_0(
-; CHECK-NEXT:    ret <8 x i32> %v
+; CHECK-NEXT:    ret <8 x i32> [[V:%.*]]
 ;
   %1 = tail call <8 x i32> @llvm.x86.avx2.psllv.d.256(<8 x i32> %v, <8 x i32> zeroinitializer)
   ret <8 x i32> %1
@@ -2295,7 +2295,7 @@ define <8 x i32> @avx2_psllv_d_256_0(<8 x i32> %v) {
 
 define <4 x i32> @avx2_psllv_d_128_var(<4 x i32> %v) {
 ; CHECK-LABEL: @avx2_psllv_d_128_var(
-; CHECK-NEXT:    [[TMP1:%.*]] = shl <4 x i32> %v, <i32 0, i32 8, i32 16, i32 31>
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <4 x i32> [[V:%.*]], <i32 0, i32 8, i32 16, i32 31>
 ; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
 ;
   %1 = tail call <4 x i32> @llvm.x86.avx2.psllv.d(<4 x i32> %v, <4 x i32> <i32 0, i32 8, i32 16, i32 31>)
@@ -2304,7 +2304,7 @@ define <4 x i32> @avx2_psllv_d_128_var(<4 x i32> %v) {
 
 define <8 x i32> @avx2_psllv_d_256_var(<8 x i32> %v) {
 ; CHECK-LABEL: @avx2_psllv_d_256_var(
-; CHECK-NEXT:    [[TMP1:%.*]] = shl <8 x i32> %v, <i32 0, i32 8, i32 16, i32 24, i32 31, i32 24, i32 8, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <8 x i32> [[V:%.*]], <i32 0, i32 8, i32 16, i32 24, i32 31, i32 24, i32 8, i32 0>
 ; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
 ;
   %1 = tail call <8 x i32> @llvm.x86.avx2.psllv.d.256(<8 x i32> %v, <8 x i32> <i32 0, i32 8, i32 16, i32 24, i32 31, i32 24, i32 8, i32 0>)
@@ -2313,7 +2313,7 @@ define <8 x i32> @avx2_psllv_d_256_var(<8 x i32> %v) {
 
 define <4 x i32> @avx2_psllv_d_128_big(<4 x i32> %v) {
 ; CHECK-LABEL: @avx2_psllv_d_128_big(
-; CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x i32> @llvm.x86.avx2.psllv.d(<4 x i32> %v, <4 x i32> <i32 0, i32 8, i32 16, i32 64>)
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x i32> @llvm.x86.avx2.psllv.d(<4 x i32> [[V:%.*]], <4 x i32> <i32 0, i32 8, i32 16, i32 64>)
 ; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
 ;
   %1 = tail call <4 x i32> @llvm.x86.avx2.psllv.d(<4 x i32> %v, <4 x i32> <i32 0, i32 8, i32 16, i32 64>)
@@ -2322,7 +2322,7 @@ define <4 x i32> @avx2_psllv_d_128_big(<4 x i32> %v) {
 
 define <8 x i32> @avx2_psllv_d_256_big(<8 x i32> %v) {
 ; CHECK-LABEL: @avx2_psllv_d_256_big(
-; CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i32> @llvm.x86.avx2.psllv.d.256(<8 x i32> %v, <8 x i32> <i32 0, i32 8, i32 16, i32 64, i32 31, i32 24, i32 8, i32 0>)
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i32> @llvm.x86.avx2.psllv.d.256(<8 x i32> [[V:%.*]], <8 x i32> <i32 0, i32 8, i32 16, i32 64, i32 31, i32 24, i32 8, i32 0>)
 ; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
 ;
   %1 = tail call <8 x i32> @llvm.x86.avx2.psllv.d.256(<8 x i32> %v, <8 x i32> <i32 0, i32 8, i32 16, i32 64, i32 31, i32 24, i32 8, i32 0>)
@@ -2347,7 +2347,7 @@ define <8 x i32> @avx2_psllv_d_256_allbig(<8 x i32> %v) {
 
 define <4 x i32> @avx2_psllv_d_128_undef(<4 x i32> %v) {
 ; CHECK-LABEL: @avx2_psllv_d_128_undef(
-; CHECK-NEXT:    [[TMP1:%.*]] = shl <4 x i32> %v, <i32 undef, i32 8, i32 16, i32 31>
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <4 x i32> [[V:%.*]], <i32 undef, i32 8, i32 16, i32 31>
 ; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
 ;
   %1 = insertelement <4 x i32> <i32 0, i32 8, i32 16, i32 31>, i32 undef, i32 0
@@ -2357,7 +2357,7 @@ define <4 x i32> @avx2_psllv_d_128_undef(<4 x i32> %v) {
 
 define <8 x i32> @avx2_psllv_d_256_undef(<8 x i32> %v) {
 ; CHECK-LABEL: @avx2_psllv_d_256_undef(
-; CHECK-NEXT:    [[TMP1:%.*]] = shl <8 x i32> %v, <i32 0, i32 undef, i32 16, i32 31, i32 31, i32 24, i32 8, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <8 x i32> [[V:%.*]], <i32 0, i32 undef, i32 16, i32 31, i32 31, i32 24, i32 8, i32 0>
 ; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
 ;
   %1 = insertelement <8 x i32> <i32 0, i32 8, i32 16, i32 31, i32 31, i32 24, i32 8, i32 0>, i32 undef, i32 1
@@ -2367,7 +2367,7 @@ define <8 x i32> @avx2_psllv_d_256_undef(<8 x i32> %v) {
 
 define <2 x i64> @avx2_psllv_q_128_0(<2 x i64> %v) {
 ; CHECK-LABEL: @avx2_psllv_q_128_0(
-; CHECK-NEXT:    ret <2 x i64> %v
+; CHECK-NEXT:    ret <2 x i64> [[V:%.*]]
 ;
   %1 = tail call <2 x i64> @llvm.x86.avx2.psllv.q(<2 x i64> %v, <2 x i64> zeroinitializer)
   ret <2 x i64> %1
@@ -2375,7 +2375,7 @@ define <2 x i64> @avx2_psllv_q_128_0(<2 x i64> %v) {
 
 define <4 x i64> @avx2_psllv_q_256_0(<4 x i64> %v) {
 ; CHECK-LABEL: @avx2_psllv_q_256_0(
-; CHECK-NEXT:    ret <4 x i64> %v
+; CHECK-NEXT:    ret <4 x i64> [[V:%.*]]
 ;
   %1 = tail call <4 x i64> @llvm.x86.avx2.psllv.q.256(<4 x i64> %v, <4 x i64> zeroinitializer)
   ret <4 x i64> %1
@@ -2383,7 +2383,7 @@ define <4 x i64> @avx2_psllv_q_256_0(<4 x i64> %v) {
 
 define <2 x i64> @avx2_psllv_q_128_var(<2 x i64> %v) {
 ; CHECK-LABEL: @avx2_psllv_q_128_var(
-; CHECK-NEXT:    [[TMP1:%.*]] = shl <2 x i64> %v, <i64 0, i64 8>
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <2 x i64> [[V:%.*]], <i64 0, i64 8>
 ; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
 ;
   %1 = tail call <2 x i64> @llvm.x86.avx2.psllv.q(<2 x i64> %v, <2 x i64> <i64 0, i64 8>)
@@ -2392,7 +2392,7 @@ define <2 x i64> @avx2_psllv_q_128_var(<2 x i64> %v) {
 
 define <4 x i64> @avx2_psllv_q_256_var(<4 x i64> %v) {
 ; CHECK-LABEL: @avx2_psllv_q_256_var(
-; CHECK-NEXT:    [[TMP1:%.*]] = shl <4 x i64> %v, <i64 0, i64 8, i64 16, i64 31>
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <4 x i64> [[V:%.*]], <i64 0, i64 8, i64 16, i64 31>
 ; CHECK-NEXT:    ret <4 x i64> [[TMP1]]
 ;
   %1 = tail call <4 x i64> @llvm.x86.avx2.psllv.q.256(<4 x i64> %v, <4 x i64> <i64 0, i64 8, i64 16, i64 31>)
@@ -2401,7 +2401,7 @@ define <4 x i64> @avx2_psllv_q_256_var(<4 x i64> %v) {
 
 define <2 x i64> @avx2_psllv_q_128_big(<2 x i64> %v) {
 ; CHECK-LABEL: @avx2_psllv_q_128_big(
-; CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x i64> @llvm.x86.avx2.psllv.q(<2 x i64> %v, <2 x i64> <i64 0, i64 128>)
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x i64> @llvm.x86.avx2.psllv.q(<2 x i64> [[V:%.*]], <2 x i64> <i64 0, i64 128>)
 ; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
 ;
   %1 = tail call <2 x i64> @llvm.x86.avx2.psllv.q(<2 x i64> %v, <2 x i64> <i64 0, i64 128>)
@@ -2410,7 +2410,7 @@ define <2 x i64> @avx2_psllv_q_128_big(<2 x i64> %v) {
 
 define <4 x i64> @avx2_psllv_q_256_big(<4 x i64> %v) {
 ; CHECK-LABEL: @avx2_psllv_q_256_big(
-; CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x i64> @llvm.x86.avx2.psllv.q.256(<4 x i64> %v, <4 x i64> <i64 0, i64 8, i64 16, i64 64>)
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x i64> @llvm.x86.avx2.psllv.q.256(<4 x i64> [[V:%.*]], <4 x i64> <i64 0, i64 8, i64 16, i64 64>)
 ; CHECK-NEXT:    ret <4 x i64> [[TMP1]]
 ;
   %1 = tail call <4 x i64> @llvm.x86.avx2.psllv.q.256(<4 x i64> %v, <4 x i64> <i64 0, i64 8, i64 16, i64 64>)
@@ -2446,7 +2446,7 @@ define <2 x i64> @avx2_psllv_q_128_undef(<2 x i64> %v) {
 
 define <4 x i64> @avx2_psllv_q_256_undef(<4 x i64> %v) {
 ; CHECK-LABEL: @avx2_psllv_q_256_undef(
-; CHECK-NEXT:    [[TMP1:%.*]] = shl <4 x i64> %v, <i64 undef, i64 8, i64 16, i64 31>
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <4 x i64> [[V:%.*]], <i64 undef, i64 8, i64 16, i64 31>
 ; CHECK-NEXT:    ret <4 x i64> [[TMP1]]
 ;
   %1 = insertelement <4 x i64> <i64 0, i64 8, i64 16, i64 31>, i64 undef, i64 0
@@ -2456,7 +2456,7 @@ define <4 x i64> @avx2_psllv_q_256_undef(<4 x i64> %v) {
 
 define <16 x i32> @avx512_psllv_d_512_0(<16 x i32> %v) {
 ; CHECK-LABEL: @avx512_psllv_d_512_0(
-; CHECK-NEXT:    ret <16 x i32> %v
+; CHECK-NEXT:    ret <16 x i32> [[V:%.*]]
 ;
   %1 = tail call <16 x i32> @llvm.x86.avx512.psllv.d.512(<16 x i32> %v, <16 x i32> zeroinitializer)
   ret <16 x i32> %1
@@ -2464,7 +2464,7 @@ define <16 x i32> @avx512_psllv_d_512_0(<16 x i32> %v) {
 
 define <16 x i32> @avx512_psllv_d_512_var(<16 x i32> %v) {
 ; CHECK-LABEL: @avx512_psllv_d_512_var(
-; CHECK-NEXT:    [[TMP1:%.*]] = shl <16 x i32> %v, <i32 0, i32 8, i32 16, i32 24, i32 31, i32 24, i32 8, i32 0, i32 0, i32 8, i32 16, i32 24, i32 31, i32 24, i32 8, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <16 x i32> [[V:%.*]], <i32 0, i32 8, i32 16, i32 24, i32 31, i32 24, i32 8, i32 0, i32 0, i32 8, i32 16, i32 24, i32 31, i32 24, i32 8, i32 0>
 ; CHECK-NEXT:    ret <16 x i32> [[TMP1]]
 ;
   %1 = tail call <16 x i32> @llvm.x86.avx512.psllv.d.512(<16 x i32> %v, <16 x i32> <i32 0, i32 8, i32 16, i32 24, i32 31, i32 24, i32 8, i32 0, i32 0, i32 8, i32 16, i32 24, i32 31, i32 24, i32 8, i32 0>)
@@ -2473,7 +2473,7 @@ define <16 x i32> @avx512_psllv_d_512_var(<16 x i32> %v) {
 
 define <16 x i32> @avx512_psllv_d_512_big(<16 x i32> %v) {
 ; CHECK-LABEL: @avx512_psllv_d_512_big(
-; CHECK-NEXT:    [[TMP1:%.*]] = tail call <16 x i32> @llvm.x86.avx512.psllv.d.512(<16 x i32> %v, <16 x i32> <i32 0, i32 8, i32 16, i32 64, i32 31, i32 24, i32 8, i32 0, i32 0, i32 8, i32 16, i32 64, i32 31, i32 24, i32 8, i32 0>)
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <16 x i32> @llvm.x86.avx512.psllv.d.512(<16 x i32> [[V:%.*]], <16 x i32> <i32 0, i32 8, i32 16, i32 64, i32 31, i32 24, i32 8, i32 0, i32 0, i32 8, i32 16, i32 64, i32 31, i32 24, i32 8, i32 0>)
 ; CHECK-NEXT:    ret <16 x i32> [[TMP1]]
 ;
   %1 = tail call <16 x i32> @llvm.x86.avx512.psllv.d.512(<16 x i32> %v, <16 x i32> <i32 0, i32 8, i32 16, i32 64, i32 31, i32 24, i32 8, i32 0, i32 0, i32 8, i32 16, i32 64, i32 31, i32 24, i32 8, i32 0>)
@@ -2490,7 +2490,7 @@ define <16 x i32> @avx512_psllv_d_512_allbig(<16 x i32> %v) {
 
 define <16 x i32> @avx512_psllv_d_512_undef(<16 x i32> %v) {
 ; CHECK-LABEL: @avx512_psllv_d_512_undef(
-; CHECK-NEXT:    [[TMP1:%.*]] = shl <16 x i32> %v, <i32 0, i32 undef, i32 16, i32 31, i32 31, i32 24, i32 8, i32 0, i32 0, i32 8, i32 16, i32 31, i32 31, i32 24, i32 8, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <16 x i32> [[V:%.*]], <i32 0, i32 undef, i32 16, i32 31, i32 31, i32 24, i32 8, i32 0, i32 0, i32 8, i32 16, i32 31, i32 31, i32 24, i32 8, i32 0>
 ; CHECK-NEXT:    ret <16 x i32> [[TMP1]]
 ;
   %1 = insertelement <16 x i32> <i32 0, i32 8, i32 16, i32 31, i32 31, i32 24, i32 8, i32 0, i32 0, i32 8, i32 16, i32 31, i32 31, i32 24, i32 8, i32 0>, i32 undef, i32 1
@@ -2500,7 +2500,7 @@ define <16 x i32> @avx512_psllv_d_512_undef(<16 x i32> %v) {
 
 define <8 x i64> @avx512_psllv_q_512_0(<8 x i64> %v) {
 ; CHECK-LABEL: @avx512_psllv_q_512_0(
-; CHECK-NEXT:    ret <8 x i64> %v
+; CHECK-NEXT:    ret <8 x i64> [[V:%.*]]
 ;
   %1 = tail call <8 x i64> @llvm.x86.avx512.psllv.q.512(<8 x i64> %v, <8 x i64> zeroinitializer)
   ret <8 x i64> %1
@@ -2508,7 +2508,7 @@ define <8 x i64> @avx512_psllv_q_512_0(<8 x i64> %v) {
 
 define <8 x i64> @avx512_psllv_q_512_var(<8 x i64> %v) {
 ; CHECK-LABEL: @avx512_psllv_q_512_var(
-; CHECK-NEXT:    [[TMP1:%.*]] = shl <8 x i64> %v, <i64 0, i64 8, i64 16, i64 31, i64 0, i64 8, i64 16, i64 31>
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <8 x i64> [[V:%.*]], <i64 0, i64 8, i64 16, i64 31, i64 0, i64 8, i64 16, i64 31>
 ; CHECK-NEXT:    ret <8 x i64> [[TMP1]]
 ;
   %1 = tail call <8 x i64> @llvm.x86.avx512.psllv.q.512(<8 x i64> %v, <8 x i64> <i64 0, i64 8, i64 16, i64 31, i64 0, i64 8, i64 16, i64 31>)
@@ -2517,7 +2517,7 @@ define <8 x i64> @avx512_psllv_q_512_var(<8 x i64> %v) {
 
 define <8 x i64> @avx512_psllv_q_512_big(<8 x i64> %v) {
 ; CHECK-LABEL: @avx512_psllv_q_512_big(
-; CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i64> @llvm.x86.avx512.psllv.q.512(<8 x i64> %v, <8 x i64> <i64 0, i64 8, i64 16, i64 64, i64 0, i64 8, i64 16, i64 64>)
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i64> @llvm.x86.avx512.psllv.q.512(<8 x i64> [[V:%.*]], <8 x i64> <i64 0, i64 8, i64 16, i64 64, i64 0, i64 8, i64 16, i64 64>)
 ; CHECK-NEXT:    ret <8 x i64> [[TMP1]]
 ;
   %1 = tail call <8 x i64> @llvm.x86.avx512.psllv.q.512(<8 x i64> %v, <8 x i64> <i64 0, i64 8, i64 16, i64 64, i64 0, i64 8, i64 16, i64 64>)
@@ -2534,7 +2534,7 @@ define <8 x i64> @avx512_psllv_q_512_allbig(<8 x i64> %v) {
 
 define <8 x i64> @avx512_psllv_q_512_undef(<8 x i64> %v) {
 ; CHECK-LABEL: @avx512_psllv_q_512_undef(
-; CHECK-NEXT:    [[TMP1:%.*]] = shl <8 x i64> %v, <i64 undef, i64 8, i64 16, i64 31, i64 0, i64 8, i64 16, i64 31>
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <8 x i64> [[V:%.*]], <i64 undef, i64 8, i64 16, i64 31, i64 0, i64 8, i64 16, i64 31>
 ; CHECK-NEXT:    ret <8 x i64> [[TMP1]]
 ;
   %1 = insertelement <8 x i64> <i64 0, i64 8, i64 16, i64 31, i64 0, i64 8, i64 16, i64 31>, i64 undef, i64 0
@@ -2544,7 +2544,7 @@ define <8 x i64> @avx512_psllv_q_512_undef(<8 x i64> %v) {
 
 define <8 x i16> @avx512_psllv_w_128_0(<8 x i16> %v) {
 ; CHECK-LABEL: @avx512_psllv_w_128_0(
-; CHECK-NEXT:    ret <8 x i16> %v
+; CHECK-NEXT:    ret <8 x i16> [[V:%.*]]
 ;
   %1 = tail call <8 x i16> @llvm.x86.avx512.psllv.w.128(<8 x i16> %v, <8 x i16> zeroinitializer)
   ret <8 x i16> %1
@@ -2552,7 +2552,7 @@ define <8 x i16> @avx512_psllv_w_128_0(<8 x i16> %v) {
 
 define <8 x i16> @avx512_psllv_w_128_var(<8 x i16> %v) {
 ; CHECK-LABEL: @avx512_psllv_w_128_var(
-; CHECK-NEXT:    [[TMP1:%.*]] = shl <8 x i16> %v, <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <8 x i16> [[V:%.*]], <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>
 ; CHECK-NEXT:    ret <8 x i16> [[TMP1]]
 ;
   %1 = tail call <8 x i16> @llvm.x86.avx512.psllv.w.128(<8 x i16> %v, <8 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>)
@@ -2561,7 +2561,7 @@ define <8 x i16> @avx512_psllv_w_128_var(<8 x i16> %v) {
 
 define <8 x i16> @avx512_psllv_w_128_big(<8 x i16> %v) {
 ; CHECK-LABEL: @avx512_psllv_w_128_big(
-; CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i16> @llvm.x86.avx512.psllv.w.128(<8 x i16> %v, <8 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 16>)
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i16> @llvm.x86.avx512.psllv.w.128(<8 x i16> [[V:%.*]], <8 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 16>)
 ; CHECK-NEXT:    ret <8 x i16> [[TMP1]]
 ;
   %1 = tail call <8 x i16> @llvm.x86.avx512.psllv.w.128(<8 x i16> %v, <8 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 16>)
@@ -2578,7 +2578,7 @@ define <8 x i16> @avx512_psllv_w_128_allbig(<8 x i16> %v) {
 
 define <8 x i16> @avx512_psllv_w_128_undef(<8 x i16> %v) {
 ; CHECK-LABEL: @avx512_psllv_w_128_undef(
-; CHECK-NEXT:    [[TMP1:%.*]] = shl <8 x i16> %v, <i16 undef, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <8 x i16> [[V:%.*]], <i16 undef, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>
 ; CHECK-NEXT:    ret <8 x i16> [[TMP1]]
 ;
   %1 = insertelement <8 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>, i16 undef, i64 0
@@ -2588,7 +2588,7 @@ define <8 x i16> @avx512_psllv_w_128_undef(<8 x i16> %v) {
 
 define <16 x i16> @avx512_psllv_w_256_0(<16 x i16> %v) {
 ; CHECK-LABEL: @avx512_psllv_w_256_0(
-; CHECK-NEXT:    ret <16 x i16> %v
+; CHECK-NEXT:    ret <16 x i16> [[V:%.*]]
 ;
   %1 = tail call <16 x i16> @llvm.x86.avx512.psllv.w.256(<16 x i16> %v, <16 x i16> zeroinitializer)
   ret <16 x i16> %1
@@ -2596,7 +2596,7 @@ define <16 x i16> @avx512_psllv_w_256_0(<16 x i16> %v) {
 
 define <16 x i16> @avx512_psllv_w_256_var(<16 x i16> %v) {
 ; CHECK-LABEL: @avx512_psllv_w_256_var(
-; CHECK-NEXT:    [[TMP1:%.*]] = shl <16 x i16> %v, <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <16 x i16> [[V:%.*]], <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>
 ; CHECK-NEXT:    ret <16 x i16> [[TMP1]]
 ;
   %1 = tail call <16 x i16> @llvm.x86.avx512.psllv.w.256(<16 x i16> %v, <16 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>)
@@ -2605,7 +2605,7 @@ define <16 x i16> @avx512_psllv_w_256_var(<16 x i16> %v) {
 
 define <16 x i16> @avx512_psllv_w_256_big(<16 x i16> %v) {
 ; CHECK-LABEL: @avx512_psllv_w_256_big(
-; CHECK-NEXT:    [[TMP1:%.*]] = tail call <16 x i16> @llvm.x86.avx512.psllv.w.256(<16 x i16> %v, <16 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 16>)
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <16 x i16> @llvm.x86.avx512.psllv.w.256(<16 x i16> [[V:%.*]], <16 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 16>)
 ; CHECK-NEXT:    ret <16 x i16> [[TMP1]]
 ;
   %1 = tail call <16 x i16> @llvm.x86.avx512.psllv.w.256(<16 x i16> %v, <16 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 16>)
@@ -2622,7 +2622,7 @@ define <16 x i16> @avx512_psllv_w_256_allbig(<16 x i16> %v) {
 
 define <16 x i16> @avx512_psllv_w_256_undef(<16 x i16> %v) {
 ; CHECK-LABEL: @avx512_psllv_w_256_undef(
-; CHECK-NEXT:    [[TMP1:%.*]] = shl <16 x i16> %v, <i16 undef, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <16 x i16> [[V:%.*]], <i16 undef, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>
 ; CHECK-NEXT:    ret <16 x i16> [[TMP1]]
 ;
   %1 = insertelement <16 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>, i16 undef, i64 0
@@ -2632,7 +2632,7 @@ define <16 x i16> @avx512_psllv_w_256_undef(<16 x i16> %v) {
 
 define <32 x i16> @avx512_psllv_w_512_0(<32 x i16> %v) {
 ; CHECK-LABEL: @avx512_psllv_w_512_0(
-; CHECK-NEXT:    ret <32 x i16> %v
+; CHECK-NEXT:    ret <32 x i16> [[V:%.*]]
 ;
   %1 = tail call <32 x i16> @llvm.x86.avx512.psllv.w.512(<32 x i16> %v, <32 x i16> zeroinitializer)
   ret <32 x i16> %1
@@ -2640,7 +2640,7 @@ define <32 x i16> @avx512_psllv_w_512_0(<32 x i16> %v) {
 
 define <32 x i16> @avx512_psllv_w_512_var(<32 x i16> %v) {
 ; CHECK-LABEL: @avx512_psllv_w_512_var(
-; CHECK-NEXT:    [[TMP1:%.*]] = shl <32 x i16> %v, <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 15, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <32 x i16> [[V:%.*]], <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 15, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>
 ; CHECK-NEXT:    ret <32 x i16> [[TMP1]]
 ;
   %1 = tail call <32 x i16> @llvm.x86.avx512.psllv.w.512(<32 x i16> %v, <32 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 15, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>)
@@ -2649,7 +2649,7 @@ define <32 x i16> @avx512_psllv_w_512_var(<32 x i16> %v) {
 
 define <32 x i16> @avx512_psllv_w_512_big(<32 x i16> %v) {
 ; CHECK-LABEL: @avx512_psllv_w_512_big(
-; CHECK-NEXT:    [[TMP1:%.*]] = tail call <32 x i16> @llvm.x86.avx512.psllv.w.512(<32 x i16> %v, <32 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 16, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>)
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <32 x i16> @llvm.x86.avx512.psllv.w.512(<32 x i16> [[V:%.*]], <32 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 16, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>)
 ; CHECK-NEXT:    ret <32 x i16> [[TMP1]]
 ;
   %1 = tail call <32 x i16> @llvm.x86.avx512.psllv.w.512(<32 x i16> %v, <32 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 16, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>)
@@ -2666,7 +2666,7 @@ define <32 x i16> @avx512_psllv_w_512_allbig(<32 x i16> %v) {
 
 define <32 x i16> @avx512_psllv_w_512_undef(<32 x i16> %v) {
 ; CHECK-LABEL: @avx512_psllv_w_512_undef(
-; CHECK-NEXT:    [[TMP1:%.*]] = shl <32 x i16> %v, <i16 undef, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 15, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <32 x i16> [[V:%.*]], <i16 undef, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 15, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>
 ; CHECK-NEXT:    ret <32 x i16> [[TMP1]]
 ;
   %1 = insertelement <32 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 15, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>, i16 undef, i64 0
@@ -2680,7 +2680,7 @@ define <32 x i16> @avx512_psllv_w_512_undef(<32 x i16> %v) {
 
 define <8 x i16> @sse2_psra_w_var(<8 x i16> %v, <8 x i16> %a) {
 ; CHECK-LABEL: @sse2_psra_w_var(
-; CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i16> @llvm.x86.sse2.psra.w(<8 x i16> %v, <8 x i16> %a)
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i16> @llvm.x86.sse2.psra.w(<8 x i16> [[V:%.*]], <8 x i16> [[A:%.*]])
 ; CHECK-NEXT:    ret <8 x i16> [[TMP1]]
 ;
   %1 = shufflevector <8 x i16> %a, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
@@ -2690,8 +2690,8 @@ define <8 x i16> @sse2_psra_w_var(<8 x i16> %v, <8 x i16> %a) {
 
 define <8 x i16> @sse2_psra_w_var_bc(<8 x i16> %v, <2 x i64> %a) {
 ; CHECK-LABEL: @sse2_psra_w_var_bc(
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64> %a to <8 x i16>
-; CHECK-NEXT:    [[TMP2:%.*]] = tail call <8 x i16> @llvm.x86.sse2.psra.w(<8 x i16> %v, <8 x i16> [[TMP1]])
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64> [[A:%.*]] to <8 x i16>
+; CHECK-NEXT:    [[TMP2:%.*]] = tail call <8 x i16> @llvm.x86.sse2.psra.w(<8 x i16> [[V:%.*]], <8 x i16> [[TMP1]])
 ; CHECK-NEXT:    ret <8 x i16> [[TMP2]]
 ;
   %1 = shufflevector <2 x i64> %a, <2 x i64> undef, <2 x i32> <i32 0, i32 0>
@@ -2702,7 +2702,7 @@ define <8 x i16> @sse2_psra_w_var_bc(<8 x i16> %v, <2 x i64> %a) {
 
 define <4 x i32> @sse2_psra_d_var(<4 x i32> %v, <4 x i32> %a) {
 ; CHECK-LABEL: @sse2_psra_d_var(
-; CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x i32> @llvm.x86.sse2.psra.d(<4 x i32> %v, <4 x i32> %a)
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x i32> @llvm.x86.sse2.psra.d(<4 x i32> [[V:%.*]], <4 x i32> [[A:%.*]])
 ; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
 ;
   %1 = shufflevector <4 x i32> %a, <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
@@ -2712,8 +2712,8 @@ define <4 x i32> @sse2_psra_d_var(<4 x i32> %v, <4 x i32> %a) {
 
 define <4 x i32> @sse2_psra_d_var_bc(<4 x i32> %v, <8 x i16> %a) {
 ; CHECK-LABEL: @sse2_psra_d_var_bc(
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <8 x i16> %a to <4 x i32>
-; CHECK-NEXT:    [[TMP2:%.*]] = tail call <4 x i32> @llvm.x86.sse2.psra.d(<4 x i32> %v, <4 x i32> [[TMP1]])
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <8 x i16> [[A:%.*]] to <4 x i32>
+; CHECK-NEXT:    [[TMP2:%.*]] = tail call <4 x i32> @llvm.x86.sse2.psra.d(<4 x i32> [[V:%.*]], <4 x i32> [[TMP1]])
 ; CHECK-NEXT:    ret <4 x i32> [[TMP2]]
 ;
   %1 = shufflevector <8 x i16> %a, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
@@ -2724,7 +2724,7 @@ define <4 x i32> @sse2_psra_d_var_bc(<4 x i32> %v, <8 x i16> %a) {
 
 define <16 x i16> @avx2_psra_w_var(<16 x i16> %v, <8 x i16> %a) {
 ; CHECK-LABEL: @avx2_psra_w_var(
-; CHECK-NEXT:    [[TMP1:%.*]] = tail call <16 x i16> @llvm.x86.avx2.psra.w(<16 x i16> %v, <8 x i16> %a)
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <16 x i16> @llvm.x86.avx2.psra.w(<16 x i16> [[V:%.*]], <8 x i16> [[A:%.*]])
 ; CHECK-NEXT:    ret <16 x i16> [[TMP1]]
 ;
   %1 = shufflevector <8 x i16> %a, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
@@ -2734,7 +2734,7 @@ define <16 x i16> @avx2_psra_w_var(<16 x i16> %v, <8 x i16> %a) {
 
 define <8 x i32> @avx2_psra_d_var(<8 x i32> %v, <4 x i32> %a) {
 ; CHECK-LABEL: @avx2_psra_d_var(
-; CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i32> @llvm.x86.avx2.psra.d(<8 x i32> %v, <4 x i32> %a)
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i32> @llvm.x86.avx2.psra.d(<8 x i32> [[V:%.*]], <4 x i32> [[A:%.*]])
 ; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
 ;
   %1 = shufflevector <4 x i32> %a, <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
@@ -2744,7 +2744,7 @@ define <8 x i32> @avx2_psra_d_var(<8 x i32> %v, <4 x i32> %a) {
 
 define <2 x i64> @avx512_psra_q_128_var(<2 x i64> %v, <2 x i64> %a) {
 ; CHECK-LABEL: @avx512_psra_q_128_var(
-; CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x i64> @llvm.x86.avx512.psra.q.128(<2 x i64> %v, <2 x i64> %a)
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x i64> @llvm.x86.avx512.psra.q.128(<2 x i64> [[V:%.*]], <2 x i64> [[A:%.*]])
 ; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
 ;
   %1 = shufflevector <2 x i64> %a, <2 x i64> undef, <2 x i32> <i32 0, i32 0>
@@ -2754,7 +2754,7 @@ define <2 x i64> @avx512_psra_q_128_var(<2 x i64> %v, <2 x i64> %a) {
 
 define <4 x i64> @avx512_psra_q_256_var(<4 x i64> %v, <2 x i64> %a) {
 ; CHECK-LABEL: @avx512_psra_q_256_var(
-; CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x i64> @llvm.x86.avx512.psra.q.256(<4 x i64> %v, <2 x i64> %a)
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x i64> @llvm.x86.avx512.psra.q.256(<4 x i64> [[V:%.*]], <2 x i64> [[A:%.*]])
 ; CHECK-NEXT:    ret <4 x i64> [[TMP1]]
 ;
   %1 = shufflevector <2 x i64> %a, <2 x i64> undef, <2 x i32> <i32 0, i32 0>
@@ -2764,7 +2764,7 @@ define <4 x i64> @avx512_psra_q_256_var(<4 x i64> %v, <2 x i64> %a) {
 
 define <32 x i16> @avx512_psra_w_512_var(<32 x i16> %v, <8 x i16> %a) {
 ; CHECK-LABEL: @avx512_psra_w_512_var(
-; CHECK-NEXT:    [[TMP1:%.*]] = tail call <32 x i16> @llvm.x86.avx512.psra.w.512(<32 x i16> %v, <8 x i16> %a)
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <32 x i16> @llvm.x86.avx512.psra.w.512(<32 x i16> [[V:%.*]], <8 x i16> [[A:%.*]])
 ; CHECK-NEXT:    ret <32 x i16> [[TMP1]]
 ;
   %1 = shufflevector <8 x i16> %a, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
@@ -2774,7 +2774,7 @@ define <32 x i16> @avx512_psra_w_512_var(<32 x i16> %v, <8 x i16> %a) {
 
 define <16 x i32> @avx512_psra_d_512_var(<16 x i32> %v, <4 x i32> %a) {
 ; CHECK-LABEL: @avx512_psra_d_512_var(
-; CHECK-NEXT:    [[TMP1:%.*]] = tail call <16 x i32> @llvm.x86.avx512.psra.d.512(<16 x i32> %v, <4 x i32> %a)
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <16 x i32> @llvm.x86.avx512.psra.d.512(<16 x i32> [[V:%.*]], <4 x i32> [[A:%.*]])
 ; CHECK-NEXT:    ret <16 x i32> [[TMP1]]
 ;
   %1 = shufflevector <4 x i32> %a, <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
@@ -2784,7 +2784,7 @@ define <16 x i32> @avx512_psra_d_512_var(<16 x i32> %v, <4 x i32> %a) {
 
 define <8 x i64> @avx512_psra_q_512_var(<8 x i64> %v, <2 x i64> %a) {
 ; CHECK-LABEL: @avx512_psra_q_512_var(
-; CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i64> @llvm.x86.avx512.psra.q.512(<8 x i64> %v, <2 x i64> %a)
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i64> @llvm.x86.avx512.psra.q.512(<8 x i64> [[V:%.*]], <2 x i64> [[A:%.*]])
 ; CHECK-NEXT:    ret <8 x i64> [[TMP1]]
 ;
   %1 = shufflevector <2 x i64> %a, <2 x i64> undef, <2 x i32> <i32 0, i32 0>
@@ -2794,7 +2794,7 @@ define <8 x i64> @avx512_psra_q_512_var(<8 x i64> %v, <2 x i64> %a) {
 
 define <8 x i16> @sse2_psrl_w_var(<8 x i16> %v, <8 x i16> %a) {
 ; CHECK-LABEL: @sse2_psrl_w_var(
-; CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i16> @llvm.x86.sse2.psrl.w(<8 x i16> %v, <8 x i16> %a)
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i16> @llvm.x86.sse2.psrl.w(<8 x i16> [[V:%.*]], <8 x i16> [[A:%.*]])
 ; CHECK-NEXT:    ret <8 x i16> [[TMP1]]
 ;
   %1 = shufflevector <8 x i16> %a, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
@@ -2804,7 +2804,7 @@ define <8 x i16> @sse2_psrl_w_var(<8 x i16> %v, <8 x i16> %a) {
 
 define <4 x i32> @sse2_psrl_d_var(<4 x i32> %v, <4 x i32> %a) {
 ; CHECK-LABEL: @sse2_psrl_d_var(
-; CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x i32> @llvm.x86.sse2.psrl.d(<4 x i32> %v, <4 x i32> %a)
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x i32> @llvm.x86.sse2.psrl.d(<4 x i32> [[V:%.*]], <4 x i32> [[A:%.*]])
 ; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
 ;
   %1 = shufflevector <4 x i32> %a, <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
@@ -2814,7 +2814,7 @@ define <4 x i32> @sse2_psrl_d_var(<4 x i32> %v, <4 x i32> %a) {
 
 define <2 x i64> @sse2_psrl_q_var(<2 x i64> %v, <2 x i64> %a) {
 ; CHECK-LABEL: @sse2_psrl_q_var(
-; CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x i64> @llvm.x86.sse2.psrl.q(<2 x i64> %v, <2 x i64> %a)
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x i64> @llvm.x86.sse2.psrl.q(<2 x i64> [[V:%.*]], <2 x i64> [[A:%.*]])
 ; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
 ;
   %1 = shufflevector <2 x i64> %a, <2 x i64> undef, <2 x i32> <i32 0, i32 0>
@@ -2824,7 +2824,7 @@ define <2 x i64> @sse2_psrl_q_var(<2 x i64> %v, <2 x i64> %a) {
 
 define <16 x i16> @avx2_psrl_w_var(<16 x i16> %v, <8 x i16> %a) {
 ; CHECK-LABEL: @avx2_psrl_w_var(
-; CHECK-NEXT:    [[TMP1:%.*]] = tail call <16 x i16> @llvm.x86.avx2.psrl.w(<16 x i16> %v, <8 x i16> %a)
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <16 x i16> @llvm.x86.avx2.psrl.w(<16 x i16> [[V:%.*]], <8 x i16> [[A:%.*]])
 ; CHECK-NEXT:    ret <16 x i16> [[TMP1]]
 ;
   %1 = shufflevector <8 x i16> %a, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
@@ -2834,8 +2834,8 @@ define <16 x i16> @avx2_psrl_w_var(<16 x i16> %v, <8 x i16> %a) {
 
 define <16 x i16> @avx2_psrl_w_var_bc(<16 x i16> %v, <16 x i8> %a) {
 ; CHECK-LABEL: @avx2_psrl_w_var_bc(
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <16 x i8> %a to <8 x i16>
-; CHECK-NEXT:    [[TMP2:%.*]] = tail call <16 x i16> @llvm.x86.avx2.psrl.w(<16 x i16> %v, <8 x i16> [[TMP1]])
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <16 x i8> [[A:%.*]] to <8 x i16>
+; CHECK-NEXT:    [[TMP2:%.*]] = tail call <16 x i16> @llvm.x86.avx2.psrl.w(<16 x i16> [[V:%.*]], <8 x i16> [[TMP1]])
 ; CHECK-NEXT:    ret <16 x i16> [[TMP2]]
 ;
   %1 = shufflevector <16 x i8> %a, <16 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -2846,7 +2846,7 @@ define <16 x i16> @avx2_psrl_w_var_bc(<16 x i16> %v, <16 x i8> %a) {
 
 define <8 x i32> @avx2_psrl_d_var(<8 x i32> %v, <4 x i32> %a) {
 ; CHECK-LABEL: @avx2_psrl_d_var(
-; CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i32> @llvm.x86.avx2.psrl.d(<8 x i32> %v, <4 x i32> %a)
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i32> @llvm.x86.avx2.psrl.d(<8 x i32> [[V:%.*]], <4 x i32> [[A:%.*]])
 ; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
 ;
   %1 = shufflevector <4 x i32> %a, <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
@@ -2856,8 +2856,8 @@ define <8 x i32> @avx2_psrl_d_var(<8 x i32> %v, <4 x i32> %a) {
 
 define <8 x i32> @avx2_psrl_d_var_bc(<8 x i32> %v, <2 x i64> %a) {
 ; CHECK-LABEL: @avx2_psrl_d_var_bc(
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64> %a to <4 x i32>
-; CHECK-NEXT:    [[TMP2:%.*]] = tail call <8 x i32> @llvm.x86.avx2.psrl.d(<8 x i32> %v, <4 x i32> [[TMP1]])
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64> [[A:%.*]] to <4 x i32>
+; CHECK-NEXT:    [[TMP2:%.*]] = tail call <8 x i32> @llvm.x86.avx2.psrl.d(<8 x i32> [[V:%.*]], <4 x i32> [[TMP1]])
 ; CHECK-NEXT:    ret <8 x i32> [[TMP2]]
 ;
   %1 = shufflevector <2 x i64> %a, <2 x i64> undef, <2 x i32> <i32 0, i32 0>
@@ -2868,7 +2868,7 @@ define <8 x i32> @avx2_psrl_d_var_bc(<8 x i32> %v, <2 x i64> %a) {
 
 define <4 x i64> @avx2_psrl_q_var(<4 x i64> %v, <2 x i64> %a) {
 ; CHECK-LABEL: @avx2_psrl_q_var(
-; CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x i64> @llvm.x86.avx2.psrl.q(<4 x i64> %v, <2 x i64> %a)
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x i64> @llvm.x86.avx2.psrl.q(<4 x i64> [[V:%.*]], <2 x i64> [[A:%.*]])
 ; CHECK-NEXT:    ret <4 x i64> [[TMP1]]
 ;
   %1 = shufflevector <2 x i64> %a, <2 x i64> undef, <2 x i32> <i32 0, i32 0>
@@ -2878,7 +2878,7 @@ define <4 x i64> @avx2_psrl_q_var(<4 x i64> %v, <2 x i64> %a) {
 
 define <32 x i16> @avx512_psrl_w_512_var(<32 x i16> %v, <8 x i16> %a) {
 ; CHECK-LABEL: @avx512_psrl_w_512_var(
-; CHECK-NEXT:    [[TMP1:%.*]] = tail call <32 x i16> @llvm.x86.avx512.psrl.w.512(<32 x i16> %v, <8 x i16> %a)
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <32 x i16> @llvm.x86.avx512.psrl.w.512(<32 x i16> [[V:%.*]], <8 x i16> [[A:%.*]])
 ; CHECK-NEXT:    ret <32 x i16> [[TMP1]]
 ;
   %1 = shufflevector <8 x i16> %a, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
@@ -2888,8 +2888,8 @@ define <32 x i16> @avx512_psrl_w_512_var(<32 x i16> %v, <8 x i16> %a) {
 
 define <32 x i16> @avx512_psrl_w_512_var_bc(<32 x i16> %v, <16 x i8> %a) {
 ; CHECK-LABEL: @avx512_psrl_w_512_var_bc(
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <16 x i8> %a to <8 x i16>
-; CHECK-NEXT:    [[TMP2:%.*]] = tail call <32 x i16> @llvm.x86.avx512.psrl.w.512(<32 x i16> %v, <8 x i16> [[TMP1]])
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <16 x i8> [[A:%.*]] to <8 x i16>
+; CHECK-NEXT:    [[TMP2:%.*]] = tail call <32 x i16> @llvm.x86.avx512.psrl.w.512(<32 x i16> [[V:%.*]], <8 x i16> [[TMP1]])
 ; CHECK-NEXT:    ret <32 x i16> [[TMP2]]
 ;
   %1 = shufflevector <16 x i8> %a, <16 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -2900,7 +2900,7 @@ define <32 x i16> @avx512_psrl_w_512_var_bc(<32 x i16> %v, <16 x i8> %a) {
 
 define <16 x i32> @avx512_psrl_d_512_var(<16 x i32> %v, <4 x i32> %a) {
 ; CHECK-LABEL: @avx512_psrl_d_512_var(
-; CHECK-NEXT:    [[TMP1:%.*]] = tail call <16 x i32> @llvm.x86.avx512.psrl.d.512(<16 x i32> %v, <4 x i32> %a)
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <16 x i32> @llvm.x86.avx512.psrl.d.512(<16 x i32> [[V:%.*]], <4 x i32> [[A:%.*]])
 ; CHECK-NEXT:    ret <16 x i32> [[TMP1]]
 ;
   %1 = shufflevector <4 x i32> %a, <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
@@ -2910,8 +2910,8 @@ define <16 x i32> @avx512_psrl_d_512_var(<16 x i32> %v, <4 x i32> %a) {
 
 define <16 x i32> @avx512_psrl_d_512_var_bc(<16 x i32> %v, <2 x i64> %a) {
 ; CHECK-LABEL: @avx512_psrl_d_512_var_bc(
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64> %a to <4 x i32>
-; CHECK-NEXT:    [[TMP2:%.*]] = tail call <16 x i32> @llvm.x86.avx512.psrl.d.512(<16 x i32> %v, <4 x i32> [[TMP1]])
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64> [[A:%.*]] to <4 x i32>
+; CHECK-NEXT:    [[TMP2:%.*]] = tail call <16 x i32> @llvm.x86.avx512.psrl.d.512(<16 x i32> [[V:%.*]], <4 x i32> [[TMP1]])
 ; CHECK-NEXT:    ret <16 x i32> [[TMP2]]
 ;
   %1 = shufflevector <2 x i64> %a, <2 x i64> undef, <2 x i32> <i32 0, i32 0>
@@ -2922,7 +2922,7 @@ define <16 x i32> @avx512_psrl_d_512_var_bc(<16 x i32> %v, <2 x i64> %a) {
 
 define <8 x i64> @avx512_psrl_q_512_var(<8 x i64> %v, <2 x i64> %a) {
 ; CHECK-LABEL: @avx512_psrl_q_512_var(
-; CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i64> @llvm.x86.avx512.psrl.q.512(<8 x i64> %v, <2 x i64> %a)
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i64> @llvm.x86.avx512.psrl.q.512(<8 x i64> [[V:%.*]], <2 x i64> [[A:%.*]])
 ; CHECK-NEXT:    ret <8 x i64> [[TMP1]]
 ;
   %1 = shufflevector <2 x i64> %a, <2 x i64> undef, <2 x i32> <i32 0, i32 0>
@@ -2932,7 +2932,7 @@ define <8 x i64> @avx512_psrl_q_512_var(<8 x i64> %v, <2 x i64> %a) {
 
 define <8 x i16> @sse2_psll_w_var(<8 x i16> %v, <8 x i16> %a) {
 ; CHECK-LABEL: @sse2_psll_w_var(
-; CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i16> @llvm.x86.sse2.psll.w(<8 x i16> %v, <8 x i16> %a)
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i16> @llvm.x86.sse2.psll.w(<8 x i16> [[V:%.*]], <8 x i16> [[A:%.*]])
 ; CHECK-NEXT:    ret <8 x i16> [[TMP1]]
 ;
   %1 = shufflevector <8 x i16> %a, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
@@ -2942,7 +2942,7 @@ define <8 x i16> @sse2_psll_w_var(<8 x i16> %v, <8 x i16> %a) {
 
 define <4 x i32> @sse2_psll_d_var(<4 x i32> %v, <4 x i32> %a) {
 ; CHECK-LABEL: @sse2_psll_d_var(
-; CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x i32> @llvm.x86.sse2.psll.d(<4 x i32> %v, <4 x i32> %a)
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x i32> @llvm.x86.sse2.psll.d(<4 x i32> [[V:%.*]], <4 x i32> [[A:%.*]])
 ; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
 ;
   %1 = shufflevector <4 x i32> %a, <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
@@ -2952,7 +2952,7 @@ define <4 x i32> @sse2_psll_d_var(<4 x i32> %v, <4 x i32> %a) {
 
 define <2 x i64> @sse2_psll_q_var(<2 x i64> %v, <2 x i64> %a) {
 ; CHECK-LABEL: @sse2_psll_q_var(
-; CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x i64> @llvm.x86.sse2.psll.q(<2 x i64> %v, <2 x i64> %a)
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x i64> @llvm.x86.sse2.psll.q(<2 x i64> [[V:%.*]], <2 x i64> [[A:%.*]])
 ; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
 ;
   %1 = shufflevector <2 x i64> %a, <2 x i64> undef, <2 x i32> <i32 0, i32 0>
@@ -2962,7 +2962,7 @@ define <2 x i64> @sse2_psll_q_var(<2 x i64> %v, <2 x i64> %a) {
 
 define <16 x i16> @avx2_psll_w_var(<16 x i16> %v, <8 x i16> %a) {
 ; CHECK-LABEL: @avx2_psll_w_var(
-; CHECK-NEXT:    [[TMP1:%.*]] = tail call <16 x i16> @llvm.x86.avx2.psll.w(<16 x i16> %v, <8 x i16> %a)
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <16 x i16> @llvm.x86.avx2.psll.w(<16 x i16> [[V:%.*]], <8 x i16> [[A:%.*]])
 ; CHECK-NEXT:    ret <16 x i16> [[TMP1]]
 ;
   %1 = shufflevector <8 x i16> %a, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
@@ -2972,7 +2972,7 @@ define <16 x i16> @avx2_psll_w_var(<16 x i16> %v, <8 x i16> %a) {
 
 define <8 x i32> @avx2_psll_d_var(<8 x i32> %v, <4 x i32> %a) {
 ; CHECK-LABEL: @avx2_psll_d_var(
-; CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i32> @llvm.x86.avx2.psll.d(<8 x i32> %v, <4 x i32> %a)
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i32> @llvm.x86.avx2.psll.d(<8 x i32> [[V:%.*]], <4 x i32> [[A:%.*]])
 ; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
 ;
   %1 = shufflevector <4 x i32> %a, <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
@@ -2982,7 +2982,7 @@ define <8 x i32> @avx2_psll_d_var(<8 x i32> %v, <4 x i32> %a) {
 
 define <4 x i64> @avx2_psll_q_var(<4 x i64> %v, <2 x i64> %a) {
 ; CHECK-LABEL: @avx2_psll_q_var(
-; CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x i64> @llvm.x86.avx2.psll.q(<4 x i64> %v, <2 x i64> %a)
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x i64> @llvm.x86.avx2.psll.q(<4 x i64> [[V:%.*]], <2 x i64> [[A:%.*]])
 ; CHECK-NEXT:    ret <4 x i64> [[TMP1]]
 ;
   %1 = shufflevector <2 x i64> %a, <2 x i64> undef, <2 x i32> <i32 0, i32 0>
@@ -2992,7 +2992,7 @@ define <4 x i64> @avx2_psll_q_var(<4 x i64> %v, <2 x i64> %a) {
 
 define <32 x i16> @avx512_psll_w_512_var(<32 x i16> %v, <8 x i16> %a) {
 ; CHECK-LABEL: @avx512_psll_w_512_var(
-; CHECK-NEXT:    [[TMP1:%.*]] = tail call <32 x i16> @llvm.x86.avx512.psll.w.512(<32 x i16> %v, <8 x i16> %a)
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <32 x i16> @llvm.x86.avx512.psll.w.512(<32 x i16> [[V:%.*]], <8 x i16> [[A:%.*]])
 ; CHECK-NEXT:    ret <32 x i16> [[TMP1]]
 ;
   %1 = shufflevector <8 x i16> %a, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
@@ -3002,7 +3002,7 @@ define <32 x i16> @avx512_psll_w_512_var(<32 x i16> %v, <8 x i16> %a) {
 
 define <16 x i32> @avx512_psll_d_512_var(<16 x i32> %v, <4 x i32> %a) {
 ; CHECK-LABEL: @avx512_psll_d_512_var(
-; CHECK-NEXT:    [[TMP1:%.*]] = tail call <16 x i32> @llvm.x86.avx512.psll.d.512(<16 x i32> %v, <4 x i32> %a)
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <16 x i32> @llvm.x86.avx512.psll.d.512(<16 x i32> [[V:%.*]], <4 x i32> [[A:%.*]])
 ; CHECK-NEXT:    ret <16 x i32> [[TMP1]]
 ;
   %1 = shufflevector <4 x i32> %a, <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
@@ -3012,7 +3012,7 @@ define <16 x i32> @avx512_psll_d_512_var(<16 x i32> %v, <4 x i32> %a) {
 
 define <8 x i64> @avx512_psll_q_512_var(<8 x i64> %v, <2 x i64> %a) {
 ; CHECK-LABEL: @avx512_psll_q_512_var(
-; CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i64> @llvm.x86.avx512.psll.q.512(<8 x i64> %v, <2 x i64> %a)
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i64> @llvm.x86.avx512.psll.q.512(<8 x i64> [[V:%.*]], <2 x i64> [[A:%.*]])
 ; CHECK-NEXT:    ret <8 x i64> [[TMP1]]
 ;
   %1 = shufflevector <2 x i64> %a, <2 x i64> undef, <2 x i32> <i32 0, i32 0>
@@ -3026,7 +3026,7 @@ define <8 x i64> @avx512_psll_q_512_var(<8 x i64> %v, <2 x i64> %a) {
 
 define <8 x i16> @test_sse2_psra_w_0(<8 x i16> %A) {
 ; CHECK-LABEL: @test_sse2_psra_w_0(
-; CHECK-NEXT:    ret <8 x i16> %A
+; CHECK-NEXT:    ret <8 x i16> [[A:%.*]]
 ;
   %1 = tail call <8 x i16> @llvm.x86.sse2.psrai.w(<8 x i16> %A, i32 0)
   %2 = tail call <8 x i16> @llvm.x86.sse2.psra.w(<8 x i16> %1, <8 x i16> <i16 0, i16 0, i16 0, i16 0, i16 7, i16 0, i16 0, i16 0>)
@@ -3047,7 +3047,7 @@ define <8 x i16> @test_sse2_psra_w_8() {
 
 define <4 x i32> @test_sse2_psra_d_0(<4 x i32> %A) {
 ; CHECK-LABEL: @test_sse2_psra_d_0(
-; CHECK-NEXT:    ret <4 x i32> %A
+; CHECK-NEXT:    ret <4 x i32> [[A:%.*]]
 ;
   %1 = tail call <4 x i32> @llvm.x86.sse2.psrai.d(<4 x i32> %A, i32 0)
   %2 = tail call <4 x i32> @llvm.x86.sse2.psra.d(<4 x i32> %1, <4 x i32> <i32 0, i32 0, i32 7, i32 0>)
@@ -3068,7 +3068,7 @@ define <4 x i32> @sse2_psra_d_8() {
 
 define <16 x i16> @test_avx2_psra_w_0(<16 x i16> %A) {
 ; CHECK-LABEL: @test_avx2_psra_w_0(
-; CHECK-NEXT:    ret <16 x i16> %A
+; CHECK-NEXT:    ret <16 x i16> [[A:%.*]]
 ;
   %1 = tail call <16 x i16> @llvm.x86.avx2.psrai.w(<16 x i16> %A, i32 0)
   %2 = tail call <16 x i16> @llvm.x86.avx2.psra.w(<16 x i16> %1, <8 x i16> <i16 0, i16 0, i16 0, i16 0, i16 7, i16 0, i16 0, i16 0>)
@@ -3089,7 +3089,7 @@ define <16 x i16> @test_avx2_psra_w_8(<16 x i16> %A) {
 
 define <8 x i32> @test_avx2_psra_d_0(<8 x i32> %A) {
 ; CHECK-LABEL: @test_avx2_psra_d_0(
-; CHECK-NEXT:    ret <8 x i32> %A
+; CHECK-NEXT:    ret <8 x i32> [[A:%.*]]
 ;
   %1 = tail call <8 x i32> @llvm.x86.avx2.psrai.d(<8 x i32> %A, i32 0)
   %2 = tail call <8 x i32> @llvm.x86.avx2.psra.d(<8 x i32> %1, <4 x i32> <i32 0, i32 0, i32 7, i32 0>)
@@ -3110,7 +3110,7 @@ define <8 x i32> @test_avx2_psra_d_8() {
 
 define <32 x i16> @test_avx512_psra_w_512_0(<32 x i16> %A) {
 ; CHECK-LABEL: @test_avx512_psra_w_512_0(
-; CHECK-NEXT:    ret <32 x i16> %A
+; CHECK-NEXT:    ret <32 x i16> [[A:%.*]]
 ;
   %1 = tail call <32 x i16> @llvm.x86.avx512.psrai.w.512(<32 x i16> %A, i32 0)
   %2 = tail call <32 x i16> @llvm.x86.avx512.psra.w.512(<32 x i16> %1, <8 x i16> <i16 0, i16 0, i16 0, i16 0, i16 7, i16 0, i16 0, i16 0>)
@@ -3131,7 +3131,7 @@ define <32 x i16> @test_avx512_psra_w_512_8(<32 x i16> %A) {
 
 define <16 x i32> @test_avx512_psra_d_512_0(<16 x i32> %A) {
 ; CHECK-LABEL: @test_avx512_psra_d_512_0(
-; CHECK-NEXT:    ret <16 x i32> %A
+; CHECK-NEXT:    ret <16 x i32> [[A:%.*]]
 ;
   %1 = tail call <16 x i32> @llvm.x86.avx512.psrai.d.512(<16 x i32> %A, i32 0)
   %2 = tail call <16 x i32> @llvm.x86.avx512.psra.d.512(<16 x i32> %1, <4 x i32> <i32 0, i32 0, i32 7, i32 0>)
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-vpermil.ll b/llvm/test/Transforms/InstCombine/X86/x86-vpermil.ll
index f68eb36c4..c367321b6 100644
--- a/llvm/test/Transforms/InstCombine/X86/x86-vpermil.ll
+++ b/llvm/test/Transforms/InstCombine/X86/x86-vpermil.ll
@@ -6,7 +6,7 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
 define <4 x float> @identity_test_vpermilvar_ps(<4 x float> %v) {
 ; CHECK-LABEL: @identity_test_vpermilvar_ps(
-; CHECK-NEXT:    ret <4 x float> %v
+; CHECK-NEXT:    ret <4 x float> [[V:%.*]]
 ;
   %a = tail call <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float> %v, <4 x i32> <i32 0, i32 1, i32 2, i32 3>)
   ret <4 x float> %a
@@ -14,7 +14,7 @@ define <4 x float> @identity_test_vpermilvar_ps(<4 x float> %v) {
 
 define <8 x float> @identity_test_vpermilvar_ps_256(<8 x float> %v) {
 ; CHECK-LABEL: @identity_test_vpermilvar_ps_256(
-; CHECK-NEXT:    ret <8 x float> %v
+; CHECK-NEXT:    ret <8 x float> [[V:%.*]]
 ;
   %a = tail call <8 x float> @llvm.x86.avx.vpermilvar.ps.256(<8 x float> %v, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>)
   ret <8 x float> %a
@@ -22,7 +22,7 @@ define <8 x float> @identity_test_vpermilvar_ps_256(<8 x float> %v) {
 
 define <16 x float> @identity_test_vpermilvar_ps_512(<16 x float> %v) {
 ; CHECK-LABEL: @identity_test_vpermilvar_ps_512(
-; CHECK-NEXT:    ret <16 x float> %v
+; CHECK-NEXT:    ret <16 x float> [[V:%.*]]
 ;
   %a = tail call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> %v, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>)
   ret <16 x float> %a
@@ -30,7 +30,7 @@ define <16 x float> @identity_test_vpermilvar_ps_512(<16 x float> %v) {
 
 define <2 x double> @identity_test_vpermilvar_pd(<2 x double> %v) {
 ; CHECK-LABEL: @identity_test_vpermilvar_pd(
-; CHECK-NEXT:    ret <2 x double> %v
+; CHECK-NEXT:    ret <2 x double> [[V:%.*]]
 ;
   %a = tail call <2 x double> @llvm.x86.avx.vpermilvar.pd(<2 x double> %v, <2 x i64> <i64 0, i64 2>)
   ret <2 x double> %a
@@ -38,7 +38,7 @@ define <2 x double> @identity_test_vpermilvar_pd(<2 x double> %v) {
 
 define <4 x double> @identity_test_vpermilvar_pd_256(<4 x double> %v) {
 ; CHECK-LABEL: @identity_test_vpermilvar_pd_256(
-; CHECK-NEXT:    ret <4 x double> %v
+; CHECK-NEXT:    ret <4 x double> [[V:%.*]]
 ;
   %a = tail call <4 x double> @llvm.x86.avx.vpermilvar.pd.256(<4 x double> %v, <4 x i64> <i64 0, i64 2, i64 0, i64 2>)
   ret <4 x double> %a
@@ -46,7 +46,7 @@ define <4 x double> @identity_test_vpermilvar_pd_256(<4 x double> %v) {
 
 define <8 x double> @identity_test_vpermilvar_pd_512(<8 x double> %v) {
 ; CHECK-LABEL: @identity_test_vpermilvar_pd_512(
-; CHECK-NEXT:    ret <8 x double> %v
+; CHECK-NEXT:    ret <8 x double> [[V:%.*]]
 ;
   %a = tail call <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double> %v, <8 x i64> <i64 0, i64 2, i64 0, i64 2, i64 0, i64 2, i64 0, i64 2>)
   ret <8 x double> %a
@@ -57,7 +57,7 @@ define <8 x double> @identity_test_vpermilvar_pd_512(<8 x double> %v) {
 
 define <4 x float> @zero_test_vpermilvar_ps_zero(<4 x float> %v) {
 ; CHECK-LABEL: @zero_test_vpermilvar_ps_zero(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> %v, <4 x float> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[V:%.*]], <4 x float> undef, <4 x i32> zeroinitializer
 ; CHECK-NEXT:    ret <4 x float> [[TMP1]]
 ;
   %a = tail call <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float> %v, <4 x i32> zeroinitializer)
@@ -66,7 +66,7 @@ define <4 x float> @zero_test_vpermilvar_ps_zero(<4 x float> %v) {
 
 define <8 x float> @zero_test_vpermilvar_ps_256_zero(<8 x float> %v) {
 ; CHECK-LABEL: @zero_test_vpermilvar_ps_256_zero(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> %v, <8 x float> undef, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 4, i32 4, i32 4, i32 4>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[V:%.*]], <8 x float> undef, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 4, i32 4, i32 4, i32 4>
 ; CHECK-NEXT:    ret <8 x float> [[TMP1]]
 ;
   %a = tail call <8 x float> @llvm.x86.avx.vpermilvar.ps.256(<8 x float> %v, <8 x i32> zeroinitializer)
@@ -75,7 +75,7 @@ define <8 x float> @zero_test_vpermilvar_ps_256_zero(<8 x float> %v) {
 
 define <16 x float> @zero_test_vpermilvar_ps_512_zero(<16 x float> %v) {
 ; CHECK-LABEL: @zero_test_vpermilvar_ps_512_zero(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x float> %v, <16 x float> undef, <16 x i32> <i32 0, i32 0, i32 0, i32 0, i32 4, i32 4, i32 4, i32 4, i32 8, i32 8, i32 8, i32 8, i32 12, i32 12, i32 12, i32 12>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x float> [[V:%.*]], <16 x float> undef, <16 x i32> <i32 0, i32 0, i32 0, i32 0, i32 4, i32 4, i32 4, i32 4, i32 8, i32 8, i32 8, i32 8, i32 12, i32 12, i32 12, i32 12>
 ; CHECK-NEXT:    ret <16 x float> [[TMP1]]
 ;
   %a = tail call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> %v, <16 x i32> zeroinitializer)
@@ -84,7 +84,7 @@ define <16 x float> @zero_test_vpermilvar_ps_512_zero(<16 x float> %v) {
 
 define <2 x double> @zero_test_vpermilvar_pd_zero(<2 x double> %v) {
 ; CHECK-LABEL: @zero_test_vpermilvar_pd_zero(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> %v, <2 x double> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[V:%.*]], <2 x double> undef, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    ret <2 x double> [[TMP1]]
 ;
   %a = tail call <2 x double> @llvm.x86.avx.vpermilvar.pd(<2 x double> %v, <2 x i64> zeroinitializer)
@@ -93,7 +93,7 @@ define <2 x double> @zero_test_vpermilvar_pd_zero(<2 x double> %v) {
 
 define <4 x double> @zero_test_vpermilvar_pd_256_zero(<4 x double> %v) {
 ; CHECK-LABEL: @zero_test_vpermilvar_pd_256_zero(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> %v, <4 x double> undef, <4 x i32> <i32 0, i32 0, i32 2, i32 2>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[V:%.*]], <4 x double> undef, <4 x i32> <i32 0, i32 0, i32 2, i32 2>
 ; CHECK-NEXT:    ret <4 x double> [[TMP1]]
 ;
   %a = tail call <4 x double> @llvm.x86.avx.vpermilvar.pd.256(<4 x double> %v, <4 x i64> zeroinitializer)
@@ -102,7 +102,7 @@ define <4 x double> @zero_test_vpermilvar_pd_256_zero(<4 x double> %v) {
 
 define <8 x double> @zero_test_vpermilvar_pd_512_zero(<8 x double> %v) {
 ; CHECK-LABEL: @zero_test_vpermilvar_pd_512_zero(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x double> %v, <8 x double> undef, <8 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x double> [[V:%.*]], <8 x double> undef, <8 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6>
 ; CHECK-NEXT:    ret <8 x double> [[TMP1]]
 ;
   %a = tail call <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double> %v, <8 x i64> zeroinitializer)
@@ -113,7 +113,7 @@ define <8 x double> @zero_test_vpermilvar_pd_512_zero(<8 x double> %v) {
 
 define <4 x float> @test_vpermilvar_ps(<4 x float> %v) {
 ; CHECK-LABEL: @test_vpermilvar_ps(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> %v, <4 x float> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[V:%.*]], <4 x float> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    ret <4 x float> [[TMP1]]
 ;
   %a = tail call <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float> %v, <4 x i32> <i32 3, i32 2, i32 1, i32 0>)
@@ -122,7 +122,7 @@ define <4 x float> @test_vpermilvar_ps(<4 x float> %v) {
 
 define <8 x float> @test_vpermilvar_ps_256(<8 x float> %v) {
 ; CHECK-LABEL: @test_vpermilvar_ps_256(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> %v, <8 x float> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[V:%.*]], <8 x float> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
 ; CHECK-NEXT:    ret <8 x float> [[TMP1]]
 ;
   %a = tail call <8 x float> @llvm.x86.avx.vpermilvar.ps.256(<8 x float> %v, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
@@ -131,7 +131,7 @@ define <8 x float> @test_vpermilvar_ps_256(<8 x float> %v) {
 
 define <16 x float> @test_vpermilvar_ps_512(<16 x float> %v) {
 ; CHECK-LABEL: @test_vpermilvar_ps_512(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x float> %v, <16 x float> undef, <16 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 11, i32 10, i32 9, i32 8, i32 15, i32 14, i32 13, i32 12>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x float> [[V:%.*]], <16 x float> undef, <16 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 11, i32 10, i32 9, i32 8, i32 15, i32 14, i32 13, i32 12>
 ; CHECK-NEXT:    ret <16 x float> [[TMP1]]
 ;
   %a = tail call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> %v, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
@@ -140,7 +140,7 @@ define <16 x float> @test_vpermilvar_ps_512(<16 x float> %v) {
 
 define <2 x double> @test_vpermilvar_pd(<2 x double> %v) {
 ; CHECK-LABEL: @test_vpermilvar_pd(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> %v, <2 x double> undef, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[V:%.*]], <2 x double> undef, <2 x i32> <i32 1, i32 0>
 ; CHECK-NEXT:    ret <2 x double> [[TMP1]]
 ;
   %a = tail call <2 x double> @llvm.x86.avx.vpermilvar.pd(<2 x double> %v, <2 x i64> <i64 2, i64 0>)
@@ -149,7 +149,7 @@ define <2 x double> @test_vpermilvar_pd(<2 x double> %v) {
 
 define <4 x double> @test_vpermilvar_pd_256(<4 x double> %v) {
 ; CHECK-LABEL: @test_vpermilvar_pd_256(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> %v, <4 x double> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[V:%.*]], <4 x double> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
 ; CHECK-NEXT:    ret <4 x double> [[TMP1]]
 ;
   %a = tail call <4 x double> @llvm.x86.avx.vpermilvar.pd.256(<4 x double> %v, <4 x i64> <i64 3, i64 1, i64 2, i64 0>)
@@ -158,7 +158,7 @@ define <4 x double> @test_vpermilvar_pd_256(<4 x double> %v) {
 
 define <8 x double> @test_vpermilvar_pd_512(<8 x double> %v) {
 ; CHECK-LABEL: @test_vpermilvar_pd_512(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x double> %v, <8 x double> undef, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x double> [[V:%.*]], <8 x double> undef, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
 ; CHECK-NEXT:    ret <8 x double> [[TMP1]]
 ;
   %a = tail call <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double> %v, <8 x i64> <i64 3, i64 1, i64 2, i64 0, i64 7, i64 5, i64 6, i64 4>)
@@ -169,7 +169,7 @@ define <8 x double> @test_vpermilvar_pd_512(<8 x double> %v) {
 
 define <4 x float> @undef_test_vpermilvar_ps(<4 x float> %v) {
 ; CHECK-LABEL: @undef_test_vpermilvar_ps(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> %v, <4 x float> undef, <4 x i32> <i32 undef, i32 2, i32 1, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[V:%.*]], <4 x float> undef, <4 x i32> <i32 undef, i32 2, i32 1, i32 undef>
 ; CHECK-NEXT:    ret <4 x float> [[TMP1]]
 ;
   %a = tail call <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float> %v, <4 x i32> <i32 undef, i32 2, i32 1, i32 undef>)
@@ -178,7 +178,7 @@ define <4 x float> @undef_test_vpermilvar_ps(<4 x float> %v) {
 
 define <8 x float> @undef_test_vpermilvar_ps_256(<8 x float> %v) {
 ; CHECK-LABEL: @undef_test_vpermilvar_ps_256(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> %v, <8 x float> undef, <8 x i32> <i32 undef, i32 2, i32 1, i32 undef, i32 7, i32 6, i32 5, i32 4>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[V:%.*]], <8 x float> undef, <8 x i32> <i32 undef, i32 2, i32 1, i32 undef, i32 7, i32 6, i32 5, i32 4>
 ; CHECK-NEXT:    ret <8 x float> [[TMP1]]
 ;
   %a = tail call <8 x float> @llvm.x86.avx.vpermilvar.ps.256(<8 x float> %v, <8 x i32> <i32 undef, i32 6, i32 5, i32 undef, i32 3, i32 2, i32 1, i32 0>)
@@ -187,7 +187,7 @@ define <8 x float> @undef_test_vpermilvar_ps_256(<8 x float> %v) {
 
 define <16 x float> @undef_test_vpermilvar_ps_512(<16 x float> %v) {
 ; CHECK-LABEL: @undef_test_vpermilvar_ps_512(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x float> %v, <16 x float> undef, <16 x i32> <i32 undef, i32 2, i32 1, i32 undef, i32 7, i32 6, i32 5, i32 4, i32 undef, i32 10, i32 9, i32 undef, i32 15, i32 14, i32 13, i32 12>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x float> [[V:%.*]], <16 x float> undef, <16 x i32> <i32 undef, i32 2, i32 1, i32 undef, i32 7, i32 6, i32 5, i32 4, i32 undef, i32 10, i32 9, i32 undef, i32 15, i32 14, i32 13, i32 12>
 ; CHECK-NEXT:    ret <16 x float> [[TMP1]]
 ;
   %a = tail call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> %v, <16 x i32> <i32 undef, i32 6, i32 5, i32 undef, i32 3, i32 2, i32 1, i32 0, i32 undef, i32 6, i32 5, i32 undef, i32 3, i32 2, i32 1, i32 0>)
@@ -196,7 +196,7 @@ define <16 x float> @undef_test_vpermilvar_ps_512(<16 x float> %v) {
 
 define <2 x double> @undef_test_vpermilvar_pd(<2 x double> %v) {
 ; CHECK-LABEL: @undef_test_vpermilvar_pd(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> %v, <2 x double> undef, <2 x i32> <i32 undef, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[V:%.*]], <2 x double> undef, <2 x i32> <i32 undef, i32 0>
 ; CHECK-NEXT:    ret <2 x double> [[TMP1]]
 ;
   %a = tail call <2 x double> @llvm.x86.avx.vpermilvar.pd(<2 x double> %v, <2 x i64> <i64 undef, i64 0>)
@@ -205,7 +205,7 @@ define <2 x double> @undef_test_vpermilvar_pd(<2 x double> %v) {
 
 define <4 x double> @undef_test_vpermilvar_pd_256(<4 x double> %v) {
 ; CHECK-LABEL: @undef_test_vpermilvar_pd_256(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> %v, <4 x double> undef, <4 x i32> <i32 undef, i32 0, i32 3, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[V:%.*]], <4 x double> undef, <4 x i32> <i32 undef, i32 0, i32 3, i32 undef>
 ; CHECK-NEXT:    ret <4 x double> [[TMP1]]
 ;
   %a = tail call <4 x double> @llvm.x86.avx.vpermilvar.pd.256(<4 x double> %v, <4 x i64> <i64 undef, i64 1, i64 2, i64 undef>)
@@ -214,7 +214,7 @@ define <4 x double> @undef_test_vpermilvar_pd_256(<4 x double> %v) {
 
 define <8 x double> @undef_test_vpermilvar_pd_512(<8 x double> %v) {
 ; CHECK-LABEL: @undef_test_vpermilvar_pd_512(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x double> %v, <8 x double> undef, <8 x i32> <i32 undef, i32 0, i32 3, i32 undef, i32 undef, i32 4, i32 7, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x double> [[V:%.*]], <8 x double> undef, <8 x i32> <i32 undef, i32 0, i32 3, i32 undef, i32 undef, i32 4, i32 7, i32 undef>
 ; CHECK-NEXT:    ret <8 x double> [[TMP1]]
 ;
   %a = tail call <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double> %v, <8 x i64> <i64 undef, i64 1, i64 2, i64 undef, i64 undef, i64 1, i64 2, i64 undef>)
@@ -225,7 +225,8 @@ define <8 x double> @undef_test_vpermilvar_pd_512(<8 x double> %v) {
 
 define <4 x float> @elts_test_vpermilvar_ps(<4 x float> %a0, i32 %a1) {
 ; CHECK-LABEL: @elts_test_vpermilvar_ps(
-; CHECK-NEXT:    ret <4 x float> %a0
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[A0:%.*]], <4 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
+; CHECK-NEXT:    ret <4 x float> [[TMP1]]
 ;
   %1 = insertelement <4 x i32> <i32 0, i32 1, i32 2, i32 3>, i32 %a1, i32 3
   %2 = tail call <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float> %a0, <4 x i32> %1)
@@ -235,7 +236,7 @@ define <4 x float> @elts_test_vpermilvar_ps(<4 x float> %a0, i32 %a1) {
 
 define <8 x float> @elts_test_vpermilvar_ps_256(<8 x float> %a0, <8 x i32> %a1) {
 ; CHECK-LABEL: @elts_test_vpermilvar_ps_256(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> %a0, <8 x float> undef, <8 x i32> <i32 undef, i32 0, i32 undef, i32 1, i32 undef, i32 6, i32 undef, i32 7>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A0:%.*]], <8 x float> undef, <8 x i32> <i32 undef, i32 0, i32 undef, i32 1, i32 undef, i32 6, i32 undef, i32 7>
 ; CHECK-NEXT:    ret <8 x float> [[TMP1]]
 ;
   %1 = shufflevector <8 x i32> %a1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 3, i32 2, i32 1, i32 0>, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11>
@@ -246,8 +247,9 @@ define <8 x float> @elts_test_vpermilvar_ps_256(<8 x float> %a0, <8 x i32> %a1)
 
 define <16 x float> @elts_test_vpermilvar_ps_512(<16 x float> %a0, <16 x i32> %a1, i32 %a2) {
 ; CHECK-LABEL: @elts_test_vpermilvar_ps_512(
-; CHECK-NEXT:    [[TMP1:%.*]] = tail call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> %a0, <16 x i32> %a1)
-; CHECK-NEXT:    ret <16 x float> [[TMP1]]
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> [[A0:%.*]], <16 x i32> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x float> [[TMP1]], <16 x float> undef, <16 x i32> <i32 undef, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    ret <16 x float> [[TMP2]]
 ;
   %1 = insertelement <16 x i32> %a1, i32 %a2, i32 0
   %2 = tail call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> %a0, <16 x i32> %1)
@@ -257,7 +259,8 @@ define <16 x float> @elts_test_vpermilvar_ps_512(<16 x float> %a0, <16 x i32> %a
 
 define <2 x double> @elts_test_vpermilvar_pd(<2 x double> %a0, i64 %a1) {
 ; CHECK-LABEL: @elts_test_vpermilvar_pd(
-; CHECK-NEXT:    ret <2 x double> %a0
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[A0:%.*]], <2 x double> undef, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    ret <2 x double> [[TMP1]]
 ;
   %1 = insertelement <2 x i64> <i64 0, i64 2>, i64 %a1, i32 1
   %2 = tail call <2 x double> @llvm.x86.avx.vpermilvar.pd(<2 x double> %a0, <2 x i64> %1)
@@ -267,7 +270,7 @@ define <2 x double> @elts_test_vpermilvar_pd(<2 x double> %a0, i64 %a1) {
 
 define <4 x double> @elts_test_vpermilvar_pd_256(<4 x double> %a0, <4 x i64> %a1) {
 ; CHECK-LABEL: @elts_test_vpermilvar_pd_256(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> %a0, <4 x double> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[A0:%.*]], <4 x double> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 undef>
 ; CHECK-NEXT:    ret <4 x double> [[TMP1]]
 ;
   %1 = shufflevector <4 x i64> <i64 0, i64 2, i64 0, i64 2>, <4 x i64> %a1, <4 x i32> <i32 1, i32 2, i32 3, i32 4>
@@ -278,8 +281,8 @@ define <4 x double> @elts_test_vpermilvar_pd_256(<4 x double> %a0, <4 x i64> %a1
 
 define <8 x double> @elts_test_vpermilvar_pd_512(<8 x double> %a0, <8 x i64> %a1, i64 %a2) {
 ; CHECK-LABEL: @elts_test_vpermilvar_pd_512(
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <8 x i64> undef, i64 %a2, i32 0
-; CHECK-NEXT:    [[TMP2:%.*]] = tail call <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double> %a0, <8 x i64> [[TMP1]])
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <8 x i64> undef, i64 [[A2:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = tail call <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double> [[A0:%.*]], <8 x i64> [[TMP1]])
 ; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <8 x double> [[TMP2]], <8 x double> undef, <8 x i32> zeroinitializer
 ; CHECK-NEXT:    ret <8 x double> [[TMP3]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-xop.ll b/llvm/test/Transforms/InstCombine/X86/x86-xop.ll
index 03a3f921a..6cc93698f 100644
--- a/llvm/test/Transforms/InstCombine/X86/x86-xop.ll
+++ b/llvm/test/Transforms/InstCombine/X86/x86-xop.ll
@@ -3,7 +3,7 @@
 
 define <2 x double> @test_vfrcz_sd(<2 x double> %a) {
 ; CHECK-LABEL: @test_vfrcz_sd(
-; CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.xop.vfrcz.sd(<2 x double> %a)
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.xop.vfrcz.sd(<2 x double> [[A:%.*]])
 ; CHECK-NEXT:    ret <2 x double> [[TMP1]]
 ;
   %1 = insertelement <2 x double> %a, double 1.000000e+00, i32 1
@@ -13,7 +13,7 @@ define <2 x double> @test_vfrcz_sd(<2 x double> %a) {
 
 define double @test_vfrcz_sd_0(double %a) {
 ; CHECK-LABEL: @test_vfrcz_sd_0(
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> undef, double %a, i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> undef, double [[A:%.*]], i32 0
 ; CHECK-NEXT:    [[TMP2:%.*]] = tail call <2 x double> @llvm.x86.xop.vfrcz.sd(<2 x double> [[TMP1]])
 ; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x double> [[TMP2]], i32 0
 ; CHECK-NEXT:    ret double [[TMP3]]
@@ -38,7 +38,7 @@ define double @test_vfrcz_sd_1(double %a) {
 
 define <4 x float> @test_vfrcz_ss(<4 x float> %a) {
 ; CHECK-LABEL: @test_vfrcz_ss(
-; CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.xop.vfrcz.ss(<4 x float> %a)
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.xop.vfrcz.ss(<4 x float> [[A:%.*]])
 ; CHECK-NEXT:    ret <4 x float> [[TMP1]]
 ;
   %1 = insertelement <4 x float> %a, float 1.000000e+00, i32 1
@@ -50,7 +50,7 @@ define <4 x float> @test_vfrcz_ss(<4 x float> %a) {
 
 define float @test_vfrcz_ss_0(float %a) {
 ; CHECK-LABEL: @test_vfrcz_ss_0(
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> undef, float %a, i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> undef, float [[A:%.*]], i32 0
 ; CHECK-NEXT:    [[TMP2:%.*]] = tail call <4 x float> @llvm.x86.xop.vfrcz.ss(<4 x float> [[TMP1]])
 ; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[TMP2]], i32 0
 ; CHECK-NEXT:    ret float [[TMP3]]
@@ -79,7 +79,7 @@ define float @test_vfrcz_ss_3(float %a) {
 
 define <2 x i64> @cmp_slt_v2i64(<2 x i64> %a, <2 x i64> %b) {
 ; CHECK-LABEL: @cmp_slt_v2i64(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt <2 x i64> %a, %b
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt <2 x i64> [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = sext <2 x i1> [[TMP1]] to <2 x i64>
 ; CHECK-NEXT:    ret <2 x i64> [[TMP2]]
 ;
@@ -89,7 +89,7 @@ define <2 x i64> @cmp_slt_v2i64(<2 x i64> %a, <2 x i64> %b) {
 
 define <2 x i64> @cmp_ult_v2i64(<2 x i64> %a, <2 x i64> %b) {
 ; CHECK-LABEL: @cmp_ult_v2i64(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult <2 x i64> %a, %b
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult <2 x i64> [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = sext <2 x i1> [[TMP1]] to <2 x i64>
 ; CHECK-NEXT:    ret <2 x i64> [[TMP2]]
 ;
@@ -99,7 +99,7 @@ define <2 x i64> @cmp_ult_v2i64(<2 x i64> %a, <2 x i64> %b) {
 
 define <2 x i64> @cmp_sle_v2i64(<2 x i64> %a, <2 x i64> %b) {
 ; CHECK-LABEL: @cmp_sle_v2i64(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp sle <2 x i64> %a, %b
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sle <2 x i64> [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = sext <2 x i1> [[TMP1]] to <2 x i64>
 ; CHECK-NEXT:    ret <2 x i64> [[TMP2]]
 ;
@@ -109,7 +109,7 @@ define <2 x i64> @cmp_sle_v2i64(<2 x i64> %a, <2 x i64> %b) {
 
 define <2 x i64> @cmp_ule_v2i64(<2 x i64> %a, <2 x i64> %b) {
 ; CHECK-LABEL: @cmp_ule_v2i64(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp ule <2 x i64> %a, %b
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ule <2 x i64> [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = sext <2 x i1> [[TMP1]] to <2 x i64>
 ; CHECK-NEXT:    ret <2 x i64> [[TMP2]]
 ;
@@ -119,7 +119,7 @@ define <2 x i64> @cmp_ule_v2i64(<2 x i64> %a, <2 x i64> %b) {
 
 define <4 x i32> @cmp_sgt_v4i32(<4 x i32> %a, <4 x i32> %b) {
 ; CHECK-LABEL: @cmp_sgt_v4i32(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <4 x i32> %a, %b
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <4 x i32> [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = sext <4 x i1> [[TMP1]] to <4 x i32>
 ; CHECK-NEXT:    ret <4 x i32> [[TMP2]]
 ;
@@ -129,7 +129,7 @@ define <4 x i32> @cmp_sgt_v4i32(<4 x i32> %a, <4 x i32> %b) {
 
 define <4 x i32> @cmp_ugt_v4i32(<4 x i32> %a, <4 x i32> %b) {
 ; CHECK-LABEL: @cmp_ugt_v4i32(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt <4 x i32> %a, %b
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt <4 x i32> [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = sext <4 x i1> [[TMP1]] to <4 x i32>
 ; CHECK-NEXT:    ret <4 x i32> [[TMP2]]
 ;
@@ -139,7 +139,7 @@ define <4 x i32> @cmp_ugt_v4i32(<4 x i32> %a, <4 x i32> %b) {
 
 define <4 x i32> @cmp_sge_v4i32(<4 x i32> %a, <4 x i32> %b) {
 ; CHECK-LABEL: @cmp_sge_v4i32(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp sge <4 x i32> %a, %b
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sge <4 x i32> [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = sext <4 x i1> [[TMP1]] to <4 x i32>
 ; CHECK-NEXT:    ret <4 x i32> [[TMP2]]
 ;
@@ -149,7 +149,7 @@ define <4 x i32> @cmp_sge_v4i32(<4 x i32> %a, <4 x i32> %b) {
 
 define <4 x i32> @cmp_uge_v4i32(<4 x i32> %a, <4 x i32> %b) {
 ; CHECK-LABEL: @cmp_uge_v4i32(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp uge <4 x i32> %a, %b
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp uge <4 x i32> [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = sext <4 x i1> [[TMP1]] to <4 x i32>
 ; CHECK-NEXT:    ret <4 x i32> [[TMP2]]
 ;
@@ -159,7 +159,7 @@ define <4 x i32> @cmp_uge_v4i32(<4 x i32> %a, <4 x i32> %b) {
 
 define <8 x i16> @cmp_seq_v8i16(<8 x i16> %a, <8 x i16> %b) {
 ; CHECK-LABEL: @cmp_seq_v8i16(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq <8 x i16> %a, %b
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq <8 x i16> [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = sext <8 x i1> [[TMP1]] to <8 x i16>
 ; CHECK-NEXT:    ret <8 x i16> [[TMP2]]
 ;
@@ -169,7 +169,7 @@ define <8 x i16> @cmp_seq_v8i16(<8 x i16> %a, <8 x i16> %b) {
 
 define <8 x i16> @cmp_ueq_v8i16(<8 x i16> %a, <8 x i16> %b) {
 ; CHECK-LABEL: @cmp_ueq_v8i16(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq <8 x i16> %a, %b
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq <8 x i16> [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = sext <8 x i1> [[TMP1]] to <8 x i16>
 ; CHECK-NEXT:    ret <8 x i16> [[TMP2]]
 ;
@@ -179,7 +179,7 @@ define <8 x i16> @cmp_ueq_v8i16(<8 x i16> %a, <8 x i16> %b) {
 
 define <8 x i16> @cmp_sne_v8i16(<8 x i16> %a, <8 x i16> %b) {
 ; CHECK-LABEL: @cmp_sne_v8i16(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne <8 x i16> %a, %b
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne <8 x i16> [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = sext <8 x i1> [[TMP1]] to <8 x i16>
 ; CHECK-NEXT:    ret <8 x i16> [[TMP2]]
 ;
@@ -189,7 +189,7 @@ define <8 x i16> @cmp_sne_v8i16(<8 x i16> %a, <8 x i16> %b) {
 
 define <8 x i16> @cmp_une_v8i16(<8 x i16> %a, <8 x i16> %b) {
 ; CHECK-LABEL: @cmp_une_v8i16(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne <8 x i16> %a, %b
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne <8 x i16> [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = sext <8 x i1> [[TMP1]] to <8 x i16>
 ; CHECK-NEXT:    ret <8 x i16> [[TMP2]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/abs-1.ll b/llvm/test/Transforms/InstCombine/abs-1.ll
index 8bbc833d5..2b2395349 100644
--- a/llvm/test/Transforms/InstCombine/abs-1.ll
+++ b/llvm/test/Transforms/InstCombine/abs-1.ll
@@ -133,77 +133,77 @@ define i32 @abs_canonical_5(i8 %x) {
 
 define i32 @abs_canonical_6(i32 %a, i32 %b) {
 ; CHECK-LABEL: @abs_canonical_6(
-; CHECK-NEXT:    [[TMP1:%.*]] = sub i32 [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP1]], 0
-; CHECK-NEXT:    [[TMP2:%.*]] = sub i32 0, [[TMP1]]
-; CHECK-NEXT:    [[ABS:%.*]] = select i1 [[CMP]], i32 [[TMP2]], i32 [[TMP1]]
+; CHECK-NEXT:    [[T1:%.*]] = sub i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[T1]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i32 0, [[T1]]
+; CHECK-NEXT:    [[ABS:%.*]] = select i1 [[CMP]], i32 [[TMP1]], i32 [[T1]]
 ; CHECK-NEXT:    ret i32 [[ABS]]
 ;
-  %tmp1 = sub i32 %a, %b
-  %cmp = icmp sgt i32 %tmp1, -1
-  %tmp2 = sub i32 %b, %a
-  %abs = select i1 %cmp, i32 %tmp1, i32 %tmp2
+  %t1 = sub i32 %a, %b
+  %cmp = icmp sgt i32 %t1, -1
+  %t2 = sub i32 %b, %a
+  %abs = select i1 %cmp, i32 %t1, i32 %t2
   ret i32 %abs
 }
 
 define <2 x i8> @abs_canonical_7(<2 x i8> %a, <2 x i8 > %b) {
 ; CHECK-LABEL: @abs_canonical_7(
-; CHECK-NEXT:    [[TMP1:%.*]] = sub <2 x i8> [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i8> [[TMP1]], zeroinitializer
-; CHECK-NEXT:    [[TMP2:%.*]] = sub <2 x i8> zeroinitializer, [[TMP1]]
-; CHECK-NEXT:    [[ABS:%.*]] = select <2 x i1> [[CMP]], <2 x i8> [[TMP2]], <2 x i8> [[TMP1]]
+; CHECK-NEXT:    [[T1:%.*]] = sub <2 x i8> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i8> [[T1]], zeroinitializer
+; CHECK-NEXT:    [[TMP1:%.*]] = sub <2 x i8> zeroinitializer, [[T1]]
+; CHECK-NEXT:    [[ABS:%.*]] = select <2 x i1> [[CMP]], <2 x i8> [[TMP1]], <2 x i8> [[T1]]
 ; CHECK-NEXT:    ret <2 x i8> [[ABS]]
 ;
 
-  %tmp1 = sub <2 x i8> %a, %b
-  %cmp = icmp sgt <2 x i8> %tmp1, <i8 -1, i8 -1>
-  %tmp2 = sub <2 x i8> %b, %a
-  %abs = select <2 x i1> %cmp, <2 x i8> %tmp1, <2 x i8> %tmp2
+  %t1 = sub <2 x i8> %a, %b
+  %cmp = icmp sgt <2 x i8> %t1, <i8 -1, i8 -1>
+  %t2 = sub <2 x i8> %b, %a
+  %abs = select <2 x i1> %cmp, <2 x i8> %t1, <2 x i8> %t2
   ret <2 x i8> %abs
 }
 
 define i32 @abs_canonical_8(i32 %a) {
 ; CHECK-LABEL: @abs_canonical_8(
-; CHECK-NEXT:    [[TMP:%.*]] = sub i32 0, [[A:%.*]]
+; CHECK-NEXT:    [[T:%.*]] = sub i32 0, [[A:%.*]]
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[A]], 0
-; CHECK-NEXT:    [[ABS:%.*]] = select i1 [[CMP]], i32 [[TMP]], i32 [[A]]
+; CHECK-NEXT:    [[ABS:%.*]] = select i1 [[CMP]], i32 [[T]], i32 [[A]]
 ; CHECK-NEXT:    ret i32 [[ABS]]
 ;
-  %tmp = sub i32 0, %a
-  %cmp = icmp slt i32 %tmp, 0
-  %abs = select i1 %cmp, i32 %a, i32 %tmp
+  %t = sub i32 0, %a
+  %cmp = icmp slt i32 %t, 0
+  %abs = select i1 %cmp, i32 %a, i32 %t
   ret i32 %abs
 }
 
 define i32 @abs_canonical_9(i32 %a, i32 %b) {
 ; CHECK-LABEL: @abs_canonical_9(
-; CHECK-NEXT:    [[TMP1:%.*]] = sub i32 [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP1]], -1
-; CHECK-NEXT:    [[TMP2:%.*]] = sub i32 [[B]], [[A]]
-; CHECK-NEXT:    [[ABS:%.*]] = select i1 [[CMP]], i32 [[TMP1]], i32 [[TMP2]]
-; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[ABS]], [[TMP2]]
+; CHECK-NEXT:    [[T1:%.*]] = sub i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[T1]], -1
+; CHECK-NEXT:    [[T2:%.*]] = sub i32 [[B]], [[A]]
+; CHECK-NEXT:    [[ABS:%.*]] = select i1 [[CMP]], i32 [[T1]], i32 [[T2]]
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[ABS]], [[T2]]
 ; CHECK-NEXT:    ret i32 [[ADD]]
 ;
-  %tmp1 = sub i32 %a, %b
-  %cmp = icmp sgt i32 %tmp1, -1
-  %tmp2 = sub i32 %b, %a
-  %abs = select i1 %cmp, i32 %tmp1, i32 %tmp2
-  %add = add i32 %abs, %tmp2 ; increase use count for %tmp2.
+  %t1 = sub i32 %a, %b
+  %cmp = icmp sgt i32 %t1, -1
+  %t2 = sub i32 %b, %a
+  %abs = select i1 %cmp, i32 %t1, i32 %t2
+  %add = add i32 %abs, %t2 ; increase use count for %t2.
   ret i32 %add
 }
 
 define i32 @abs_canonical_10(i32 %a, i32 %b) {
 ; CHECK-LABEL: @abs_canonical_10(
-; CHECK-NEXT:    [[TMP1:%.*]] = sub i32 [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP1]], 0
-; CHECK-NEXT:    [[NEGTMP:%.*]] = sub i32 0, [[TMP1]]
-; CHECK-NEXT:    [[ABS:%.*]] = select i1 [[CMP]], i32 [[NEGTMP]], i32 [[TMP1]]
+; CHECK-NEXT:    [[T1:%.*]] = sub i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[T1]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i32 0, [[T1]]
+; CHECK-NEXT:    [[ABS:%.*]] = select i1 [[CMP]], i32 [[TMP1]], i32 [[T1]]
 ; CHECK-NEXT:    ret i32 [[ABS]]
 ;
-  %tmp2 = sub i32 %b, %a
-  %tmp1 = sub i32 %a, %b
-  %cmp = icmp sgt i32 %tmp1, -1
-  %abs = select i1 %cmp, i32 %tmp1, i32 %tmp2
+  %t2 = sub i32 %b, %a
+  %t1 = sub i32 %a, %b
+  %cmp = icmp sgt i32 %t1, -1
+  %abs = select i1 %cmp, i32 %t1, i32 %t2
   ret i32 %abs
 }
 
@@ -297,76 +297,76 @@ define i32 @nabs_canonical_5(i8 %x) {
 
 define i32 @nabs_canonical_6(i32 %a, i32 %b) {
 ; CHECK-LABEL: @nabs_canonical_6(
-; CHECK-NEXT:    [[TMP1:%.*]] = sub i32 [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP1]], 0
-; CHECK-NEXT:    [[TMP2:%.*]] = sub i32 0, [[TMP1]]
-; CHECK-NEXT:    [[ABS:%.*]] = select i1 [[CMP]], i32 [[TMP1]], i32 [[TMP2]]
+; CHECK-NEXT:    [[T1:%.*]] = sub i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[T1]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i32 0, [[T1]]
+; CHECK-NEXT:    [[ABS:%.*]] = select i1 [[CMP]], i32 [[T1]], i32 [[TMP1]]
 ; CHECK-NEXT:    ret i32 [[ABS]]
 ;
-  %tmp1 = sub i32 %a, %b
-  %cmp = icmp sgt i32 %tmp1, -1
-  %tmp2 = sub i32 %b, %a
-  %abs = select i1 %cmp, i32 %tmp2, i32 %tmp1
+  %t1 = sub i32 %a, %b
+  %cmp = icmp sgt i32 %t1, -1
+  %t2 = sub i32 %b, %a
+  %abs = select i1 %cmp, i32 %t2, i32 %t1
   ret i32 %abs
 }
 
 define <2 x i8> @nabs_canonical_7(<2 x i8> %a, <2 x i8 > %b) {
 ; CHECK-LABEL: @nabs_canonical_7(
-; CHECK-NEXT:    [[TMP1:%.*]] = sub <2 x i8> [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i8> [[TMP1]], zeroinitializer
-; CHECK-NEXT:    [[TMP2:%.*]] = sub <2 x i8> zeroinitializer, [[TMP1]]
-; CHECK-NEXT:    [[ABS:%.*]] = select <2 x i1> [[CMP]], <2 x i8> [[TMP1]], <2 x i8> [[TMP2]]
+; CHECK-NEXT:    [[T1:%.*]] = sub <2 x i8> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i8> [[T1]], zeroinitializer
+; CHECK-NEXT:    [[TMP1:%.*]] = sub <2 x i8> zeroinitializer, [[T1]]
+; CHECK-NEXT:    [[ABS:%.*]] = select <2 x i1> [[CMP]], <2 x i8> [[T1]], <2 x i8> [[TMP1]]
 ; CHECK-NEXT:    ret <2 x i8> [[ABS]]
 ;
-  %tmp1 = sub <2 x i8> %a, %b
-  %cmp = icmp sgt <2 x i8> %tmp1, <i8 -1, i8 -1>
-  %tmp2 = sub <2 x i8> %b, %a
-  %abs = select <2 x i1> %cmp, <2 x i8> %tmp2, <2 x i8> %tmp1
+  %t1 = sub <2 x i8> %a, %b
+  %cmp = icmp sgt <2 x i8> %t1, <i8 -1, i8 -1>
+  %t2 = sub <2 x i8> %b, %a
+  %abs = select <2 x i1> %cmp, <2 x i8> %t2, <2 x i8> %t1
   ret <2 x i8> %abs
 }
 
 define i32 @nabs_canonical_8(i32 %a) {
 ; CHECK-LABEL: @nabs_canonical_8(
-; CHECK-NEXT:    [[TMP:%.*]] = sub i32 0, [[A:%.*]]
+; CHECK-NEXT:    [[T:%.*]] = sub i32 0, [[A:%.*]]
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[A]], 0
-; CHECK-NEXT:    [[ABS:%.*]] = select i1 [[CMP]], i32 [[A]], i32 [[TMP]]
+; CHECK-NEXT:    [[ABS:%.*]] = select i1 [[CMP]], i32 [[A]], i32 [[T]]
 ; CHECK-NEXT:    ret i32 [[ABS]]
 ;
-  %tmp = sub i32 0, %a
-  %cmp = icmp slt i32 %tmp, 0
-  %abs = select i1 %cmp, i32 %tmp, i32 %a
+  %t = sub i32 0, %a
+  %cmp = icmp slt i32 %t, 0
+  %abs = select i1 %cmp, i32 %t, i32 %a
   ret i32 %abs
 }
 
 define i32 @nabs_canonical_9(i32 %a, i32 %b) {
 ; CHECK-LABEL: @nabs_canonical_9(
-; CHECK-NEXT:    [[TMP1:%.*]] = sub i32 [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP1]], -1
-; CHECK-NEXT:    [[TMP2:%.*]] = sub i32 [[B]], [[A]]
-; CHECK-NEXT:    [[ABS:%.*]] = select i1 [[CMP]], i32 [[TMP2]], i32 [[TMP1]]
-; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[TMP2]], [[ABS]]
+; CHECK-NEXT:    [[T1:%.*]] = sub i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[T1]], -1
+; CHECK-NEXT:    [[T2:%.*]] = sub i32 [[B]], [[A]]
+; CHECK-NEXT:    [[ABS:%.*]] = select i1 [[CMP]], i32 [[T2]], i32 [[T1]]
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[T2]], [[ABS]]
 ; CHECK-NEXT:    ret i32 [[ADD]]
 ;
-  %tmp1 = sub i32 %a, %b
-  %cmp = icmp sgt i32 %tmp1, -1
-  %tmp2 = sub i32 %b, %a
-  %abs = select i1 %cmp, i32 %tmp2, i32 %tmp1
-  %add = add i32 %tmp2, %abs ; increase use count for %tmp2
+  %t1 = sub i32 %a, %b
+  %cmp = icmp sgt i32 %t1, -1
+  %t2 = sub i32 %b, %a
+  %abs = select i1 %cmp, i32 %t2, i32 %t1
+  %add = add i32 %t2, %abs ; increase use count for %t2
   ret i32 %add
 }
 
 define i32 @nabs_canonical_10(i32 %a, i32 %b) {
 ; CHECK-LABEL: @nabs_canonical_10(
-; CHECK-NEXT:    [[TMP1:%.*]] = sub i32 [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP1]], 0
-; CHECK-NEXT:    [[NEGTMP:%.*]] = sub i32 0, [[TMP1]]
-; CHECK-NEXT:    [[ABS:%.*]] = select i1 [[CMP]], i32 [[TMP1]], i32 [[NEGTMP]]
+; CHECK-NEXT:    [[T1:%.*]] = sub i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[T1]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i32 0, [[T1]]
+; CHECK-NEXT:    [[ABS:%.*]] = select i1 [[CMP]], i32 [[T1]], i32 [[TMP1]]
 ; CHECK-NEXT:    ret i32 [[ABS]]
 ;
-  %tmp2 = sub i32 %b, %a
-  %tmp1 = sub i32 %a, %b
-  %cmp = icmp slt i32 %tmp1, 1
-  %abs = select i1 %cmp, i32 %tmp1, i32 %tmp2
+  %t2 = sub i32 %b, %a
+  %t1 = sub i32 %a, %b
+  %cmp = icmp slt i32 %t1, 1
+  %abs = select i1 %cmp, i32 %t1, i32 %t2
   ret i32 %abs
 }
 
diff --git a/llvm/test/Transforms/InstCombine/add4.ll b/llvm/test/Transforms/InstCombine/add4.ll
index 79f3fa08f..a61f579c8 100644
--- a/llvm/test/Transforms/InstCombine/add4.ll
+++ b/llvm/test/Transforms/InstCombine/add4.ll
@@ -1,10 +1,11 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -instcombine -S | FileCheck %s
 ; ModuleID = 'test/Transforms/InstCombine/add4.ll'
 source_filename = "test/Transforms/InstCombine/add4.ll"
 
 define i64 @match_unsigned(i64 %x) {
 ; CHECK-LABEL: @match_unsigned(
-; CHECK-NEXT:    bb:
+; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[UREM:%.*]] = urem i64 [[X:%.*]], 19136
 ; CHECK-NEXT:    ret i64 [[UREM]]
 ;
@@ -19,7 +20,7 @@ bb:
 
 define i64 @match_andAsRem_lshrAsDiv_shlAsMul(i64 %x) {
 ; CHECK-LABEL: @match_andAsRem_lshrAsDiv_shlAsMul(
-; CHECK-NEXT:    bb:
+; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[UREM:%.*]] = urem i64 [[X:%.*]], 576
 ; CHECK-NEXT:    ret i64 [[UREM]]
 ;
@@ -34,7 +35,7 @@ bb:
 
 define i64 @match_signed(i64 %x) {
 ; CHECK-LABEL: @match_signed(
-; CHECK-NEXT:    bb:
+; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[SREM1:%.*]] = srem i64 [[X:%.*]], 172224
 ; CHECK-NEXT:    ret i64 [[SREM1]]
 ;
@@ -53,8 +54,13 @@ bb:
 
 define i64 @not_match_inconsistent_signs(i64 %x) {
 ; CHECK-LABEL: @not_match_inconsistent_signs(
-; CHECK:         [[TMP:%.*]] = add
-; CHECK-NEXT:    ret i64 [[TMP]]
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP:%.*]] = urem i64 [[X:%.*]], 299
+; CHECK-NEXT:    [[TMP1:%.*]] = sdiv i64 [[X]], 299
+; CHECK-NEXT:    [[TMP2:%.*]] = and i64 [[TMP1]], 63
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw nsw i64 [[TMP2]], 299
+; CHECK-NEXT:    [[TMP4:%.*]] = add nuw nsw i64 [[TMP]], [[TMP3]]
+; CHECK-NEXT:    ret i64 [[TMP4]]
 ;
 bb:
   %tmp = urem i64 %x, 299
@@ -67,8 +73,13 @@ bb:
 
 define i64 @not_match_inconsistent_values(i64 %x) {
 ; CHECK-LABEL: @not_match_inconsistent_values(
-; CHECK:         [[TMP:%.*]] = add
-; CHECK-NEXT:    ret i64 [[TMP]]
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP:%.*]] = urem i64 [[X:%.*]], 299
+; CHECK-NEXT:    [[TMP1:%.*]] = udiv i64 [[X]], 29
+; CHECK-NEXT:    [[TMP2:%.*]] = and i64 [[TMP1]], 63
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw nsw i64 [[TMP2]], 299
+; CHECK-NEXT:    [[TMP4:%.*]] = add nuw nsw i64 [[TMP]], [[TMP3]]
+; CHECK-NEXT:    ret i64 [[TMP4]]
 ;
 bb:
   %tmp = urem i64 %x, 299
@@ -81,8 +92,12 @@ bb:
 
 define i32 @not_match_overflow(i32 %x) {
 ; CHECK-LABEL: @not_match_overflow(
-; CHECK:         [[TMP:%.*]] = add
-; CHECK-NEXT:    ret i32 [[TMP]]
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP:%.*]] = urem i32 [[X:%.*]], 299
+; CHECK-NEXT:    [[TMP0:%.*]] = urem i32 [[X]], 299
+; CHECK-NEXT:    [[TMP3:%.*]] = sub i32 [[X]], [[TMP0]]
+; CHECK-NEXT:    [[TMP4:%.*]] = add i32 [[TMP]], [[TMP3]]
+; CHECK-NEXT:    ret i32 [[TMP4]]
 ;
 bb:
   %tmp = urem i32 %x, 299
diff --git a/llvm/test/Transforms/InstCombine/addrspacecast.ll b/llvm/test/Transforms/InstCombine/addrspacecast.ll
index 6caefb166..2e34f61a6 100644
--- a/llvm/test/Transforms/InstCombine/addrspacecast.ll
+++ b/llvm/test/Transforms/InstCombine/addrspacecast.ll
@@ -104,7 +104,7 @@ define <4 x float addrspace(2)*> @combine_addrspacecast_types_vector(<4 x i32 ad
 
 define i32 @canonicalize_addrspacecast([16 x i32] addrspace(1)* %arr) {
 ; CHECK-LABEL: @canonicalize_addrspacecast(
-; CHECK-NEXT: getelementptr inbounds [16 x i32], [16 x i32] addrspace(1)* %arr, i32 0, i32 0
+; CHECK-NEXT: getelementptr [16 x i32], [16 x i32] addrspace(1)* %arr, i32 0, i32 0
 ; CHECK-NEXT: addrspacecast i32 addrspace(1)* %{{[a-zA-Z0-9]+}} to i32*
 ; CHECK-NEXT: load i32, i32*
 ; CHECK-NEXT: ret i32
diff --git a/llvm/test/Transforms/InstCombine/adjust-for-minmax.ll b/llvm/test/Transforms/InstCombine/adjust-for-minmax.ll
index 5b5ba21c4..9a5b1f5d5 100644
--- a/llvm/test/Transforms/InstCombine/adjust-for-minmax.ll
+++ b/llvm/test/Transforms/InstCombine/adjust-for-minmax.ll
@@ -7,8 +7,8 @@
 
 define i32 @smax1(i32 %n) {
 ; CHECK-LABEL: @smax1(
-; CHECK-NEXT:    [[T:%.*]] = icmp sgt i32 %n, 0
-; CHECK-NEXT:    [[M:%.*]] = select i1 [[T]], i32 %n, i32 0
+; CHECK-NEXT:    [[T:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; CHECK-NEXT:    [[M:%.*]] = select i1 [[T]], i32 [[N]], i32 0
 ; CHECK-NEXT:    ret i32 [[M]]
 ;
   %t = icmp sgt i32 %n, 0
@@ -20,8 +20,8 @@ define i32 @smax1(i32 %n) {
 
 define i32 @smin1(i32 %n) {
 ; CHECK-LABEL: @smin1(
-; CHECK-NEXT:    [[T:%.*]] = icmp slt i32 %n, 0
-; CHECK-NEXT:    [[M:%.*]] = select i1 [[T]], i32 %n, i32 0
+; CHECK-NEXT:    [[T:%.*]] = icmp slt i32 [[N:%.*]], 0
+; CHECK-NEXT:    [[M:%.*]] = select i1 [[T]], i32 [[N]], i32 0
 ; CHECK-NEXT:    ret i32 [[M]]
 ;
   %t = icmp slt i32 %n, 0
@@ -33,8 +33,8 @@ define i32 @smin1(i32 %n) {
 
 define i32 @smax2(i32 %n) {
 ; CHECK-LABEL: @smax2(
-; CHECK-NEXT:    [[T:%.*]] = icmp sgt i32 %n, 0
-; CHECK-NEXT:    [[M:%.*]] = select i1 [[T]], i32 %n, i32 0
+; CHECK-NEXT:    [[T:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; CHECK-NEXT:    [[M:%.*]] = select i1 [[T]], i32 [[N]], i32 0
 ; CHECK-NEXT:    ret i32 [[M]]
 ;
   %t = icmp sge i32 %n, 1
@@ -46,8 +46,8 @@ define i32 @smax2(i32 %n) {
 
 define i32 @smin2(i32 %n) {
 ; CHECK-LABEL: @smin2(
-; CHECK-NEXT:    [[T:%.*]] = icmp slt i32 %n, 0
-; CHECK-NEXT:    [[M:%.*]] = select i1 [[T]], i32 %n, i32 0
+; CHECK-NEXT:    [[T:%.*]] = icmp slt i32 [[N:%.*]], 0
+; CHECK-NEXT:    [[M:%.*]] = select i1 [[T]], i32 [[N]], i32 0
 ; CHECK-NEXT:    ret i32 [[M]]
 ;
   %t = icmp sle i32 %n, -1
@@ -59,8 +59,8 @@ define i32 @smin2(i32 %n) {
 
 define i32 @smax3(i32 %n) {
 ; CHECK-LABEL: @smax3(
-; CHECK-NEXT:    [[T:%.*]] = icmp sgt i32 %n, 0
-; CHECK-NEXT:    [[M:%.*]] = select i1 [[T]], i32 %n, i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; CHECK-NEXT:    [[M:%.*]] = select i1 [[TMP1]], i32 [[N]], i32 0
 ; CHECK-NEXT:    ret i32 [[M]]
 ;
   %t = icmp sgt i32 %n, -1
@@ -72,8 +72,8 @@ define i32 @smax3(i32 %n) {
 
 define <2 x i32> @smax3_vec(<2 x i32> %n) {
 ; CHECK-LABEL: @smax3_vec(
-; CHECK-NEXT:    [[T:%.*]] = icmp sgt <2 x i32> %n, zeroinitializer
-; CHECK-NEXT:    [[M:%.*]] = select <2 x i1> [[T]], <2 x i32> %n, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <2 x i32> [[N:%.*]], zeroinitializer
+; CHECK-NEXT:    [[M:%.*]] = select <2 x i1> [[TMP1]], <2 x i32> [[N]], <2 x i32> zeroinitializer
 ; CHECK-NEXT:    ret <2 x i32> [[M]]
 ;
   %t = icmp sgt <2 x i32> %n, <i32 -1, i32 -1>
@@ -85,8 +85,8 @@ define <2 x i32> @smax3_vec(<2 x i32> %n) {
 
 define i32 @smin3(i32 %n) {
 ; CHECK-LABEL: @smin3(
-; CHECK-NEXT:    [[T:%.*]] = icmp slt i32 %n, 0
-; CHECK-NEXT:    [[M:%.*]] = select i1 [[T]], i32 %n, i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[N:%.*]], 0
+; CHECK-NEXT:    [[M:%.*]] = select i1 [[TMP1]], i32 [[N]], i32 0
 ; CHECK-NEXT:    ret i32 [[M]]
 ;
   %t = icmp slt i32 %n, 1
@@ -98,8 +98,8 @@ define i32 @smin3(i32 %n) {
 
 define <2 x i32> @smin3_vec(<2 x i32> %n) {
 ; CHECK-LABEL: @smin3_vec(
-; CHECK-NEXT:    [[T:%.*]] = icmp slt <2 x i32> %n, zeroinitializer
-; CHECK-NEXT:    [[M:%.*]] = select <2 x i1> [[T]], <2 x i32> %n, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt <2 x i32> [[N:%.*]], zeroinitializer
+; CHECK-NEXT:    [[M:%.*]] = select <2 x i1> [[TMP1]], <2 x i32> [[N]], <2 x i32> zeroinitializer
 ; CHECK-NEXT:    ret <2 x i32> [[M]]
 ;
   %t = icmp slt <2 x i32> %n, <i32 1, i32 1>
@@ -111,8 +111,8 @@ define <2 x i32> @smin3_vec(<2 x i32> %n) {
 
 define i32 @umax3(i32 %n) {
 ; CHECK-LABEL: @umax3(
-; CHECK-NEXT:    [[T:%.*]] = icmp ugt i32 %n, 5
-; CHECK-NEXT:    [[M:%.*]] = select i1 [[T]], i32 %n, i32 5
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i32 [[N:%.*]], 5
+; CHECK-NEXT:    [[M:%.*]] = select i1 [[TMP1]], i32 [[N]], i32 5
 ; CHECK-NEXT:    ret i32 [[M]]
 ;
   %t = icmp ugt i32 %n, 4
@@ -124,8 +124,8 @@ define i32 @umax3(i32 %n) {
 
 define <2 x i32> @umax3_vec(<2 x i32> %n) {
 ; CHECK-LABEL: @umax3_vec(
-; CHECK-NEXT:    [[T:%.*]] = icmp ugt <2 x i32> %n, <i32 5, i32 5>
-; CHECK-NEXT:    [[M:%.*]] = select <2 x i1> [[T]], <2 x i32> %n, <2 x i32> <i32 5, i32 5>
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt <2 x i32> [[N:%.*]], <i32 5, i32 5>
+; CHECK-NEXT:    [[M:%.*]] = select <2 x i1> [[TMP1]], <2 x i32> [[N]], <2 x i32> <i32 5, i32 5>
 ; CHECK-NEXT:    ret <2 x i32> [[M]]
 ;
   %t = icmp ugt <2 x i32> %n, <i32 4, i32 4>
@@ -137,8 +137,8 @@ define <2 x i32> @umax3_vec(<2 x i32> %n) {
 
 define i32 @umin3(i32 %n) {
 ; CHECK-LABEL: @umin3(
-; CHECK-NEXT:    [[T:%.*]] = icmp ult i32 %n, 6
-; CHECK-NEXT:    [[M:%.*]] = select i1 [[T]], i32 %n, i32 6
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[N:%.*]], 6
+; CHECK-NEXT:    [[M:%.*]] = select i1 [[TMP1]], i32 [[N]], i32 6
 ; CHECK-NEXT:    ret i32 [[M]]
 ;
   %t = icmp ult i32 %n, 7
@@ -150,8 +150,8 @@ define i32 @umin3(i32 %n) {
 
 define <2 x i32> @umin3_vec(<2 x i32> %n) {
 ; CHECK-LABEL: @umin3_vec(
-; CHECK-NEXT:    [[T:%.*]] = icmp ult <2 x i32> %n, <i32 6, i32 6>
-; CHECK-NEXT:    [[M:%.*]] = select <2 x i1> [[T]], <2 x i32> %n, <2 x i32> <i32 6, i32 6>
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult <2 x i32> [[N:%.*]], <i32 6, i32 6>
+; CHECK-NEXT:    [[M:%.*]] = select <2 x i1> [[TMP1]], <2 x i32> [[N]], <2 x i32> <i32 6, i32 6>
 ; CHECK-NEXT:    ret <2 x i32> [[M]]
 ;
   %t = icmp ult <2 x i32> %n, <i32 7, i32 7>
@@ -163,8 +163,8 @@ define <2 x i32> @umin3_vec(<2 x i32> %n) {
 
 define i32 @smax4(i32 %n) {
 ; CHECK-LABEL: @smax4(
-; CHECK-NEXT:    [[T:%.*]] = icmp sgt i32 %n, 0
-; CHECK-NEXT:    [[M:%.*]] = select i1 [[T]], i32 %n, i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; CHECK-NEXT:    [[M:%.*]] = select i1 [[TMP1]], i32 [[N]], i32 0
 ; CHECK-NEXT:    ret i32 [[M]]
 ;
   %t = icmp sge i32 %n, 0
@@ -176,8 +176,8 @@ define i32 @smax4(i32 %n) {
 
 define <2 x i32> @smax4_vec(<2 x i32> %n) {
 ; CHECK-LABEL: @smax4_vec(
-; CHECK-NEXT:    [[T:%.*]] = icmp sgt <2 x i32> %n, zeroinitializer
-; CHECK-NEXT:    [[M:%.*]] = select <2 x i1> [[T]], <2 x i32> %n, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <2 x i32> [[N:%.*]], zeroinitializer
+; CHECK-NEXT:    [[M:%.*]] = select <2 x i1> [[TMP1]], <2 x i32> [[N]], <2 x i32> zeroinitializer
 ; CHECK-NEXT:    ret <2 x i32> [[M]]
 ;
   %t = icmp sge <2 x i32> %n, zeroinitializer
@@ -189,8 +189,8 @@ define <2 x i32> @smax4_vec(<2 x i32> %n) {
 
 define i32 @smin4(i32 %n) {
 ; CHECK-LABEL: @smin4(
-; CHECK-NEXT:    [[T:%.*]] = icmp slt i32 %n, 0
-; CHECK-NEXT:    [[M:%.*]] = select i1 [[T]], i32 %n, i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[N:%.*]], 0
+; CHECK-NEXT:    [[M:%.*]] = select i1 [[TMP1]], i32 [[N]], i32 0
 ; CHECK-NEXT:    ret i32 [[M]]
 ;
   %t = icmp sle i32 %n, 0
@@ -202,8 +202,8 @@ define i32 @smin4(i32 %n) {
 
 define <2 x i32> @smin4_vec(<2 x i32> %n) {
 ; CHECK-LABEL: @smin4_vec(
-; CHECK-NEXT:    [[T:%.*]] = icmp slt <2 x i32> %n, zeroinitializer
-; CHECK-NEXT:    [[M:%.*]] = select <2 x i1> [[T]], <2 x i32> %n, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt <2 x i32> [[N:%.*]], zeroinitializer
+; CHECK-NEXT:    [[M:%.*]] = select <2 x i1> [[TMP1]], <2 x i32> [[N]], <2 x i32> zeroinitializer
 ; CHECK-NEXT:    ret <2 x i32> [[M]]
 ;
   %t = icmp sle <2 x i32> %n, zeroinitializer
@@ -215,8 +215,8 @@ define <2 x i32> @smin4_vec(<2 x i32> %n) {
 
 define i32 @umax4(i32 %n) {
 ; CHECK-LABEL: @umax4(
-; CHECK-NEXT:    [[T:%.*]] = icmp ugt i32 %n, 8
-; CHECK-NEXT:    [[M:%.*]] = select i1 [[T]], i32 %n, i32 8
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i32 [[N:%.*]], 8
+; CHECK-NEXT:    [[M:%.*]] = select i1 [[TMP1]], i32 [[N]], i32 8
 ; CHECK-NEXT:    ret i32 [[M]]
 ;
   %t = icmp uge i32 %n, 8
@@ -228,8 +228,8 @@ define i32 @umax4(i32 %n) {
 
 define <2 x i32> @umax4_vec(<2 x i32> %n) {
 ; CHECK-LABEL: @umax4_vec(
-; CHECK-NEXT:    [[T:%.*]] = icmp ugt <2 x i32> %n, <i32 8, i32 8>
-; CHECK-NEXT:    [[M:%.*]] = select <2 x i1> [[T]], <2 x i32> %n, <2 x i32> <i32 8, i32 8>
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt <2 x i32> [[N:%.*]], <i32 8, i32 8>
+; CHECK-NEXT:    [[M:%.*]] = select <2 x i1> [[TMP1]], <2 x i32> [[N]], <2 x i32> <i32 8, i32 8>
 ; CHECK-NEXT:    ret <2 x i32> [[M]]
 ;
   %t = icmp uge <2 x i32> %n, <i32 8, i32 8>
@@ -241,8 +241,8 @@ define <2 x i32> @umax4_vec(<2 x i32> %n) {
 
 define i32 @umin4(i32 %n) {
 ; CHECK-LABEL: @umin4(
-; CHECK-NEXT:    [[T:%.*]] = icmp ult i32 %n, 9
-; CHECK-NEXT:    [[M:%.*]] = select i1 [[T]], i32 %n, i32 9
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[N:%.*]], 9
+; CHECK-NEXT:    [[M:%.*]] = select i1 [[TMP1]], i32 [[N]], i32 9
 ; CHECK-NEXT:    ret i32 [[M]]
 ;
   %t = icmp ule i32 %n, 9
@@ -254,8 +254,8 @@ define i32 @umin4(i32 %n) {
 
 define <2 x i32> @umin4_vec(<2 x i32> %n) {
 ; CHECK-LABEL: @umin4_vec(
-; CHECK-NEXT:    [[T:%.*]] = icmp ult <2 x i32> %n, <i32 9, i32 9>
-; CHECK-NEXT:    [[M:%.*]] = select <2 x i1> [[T]], <2 x i32> %n, <2 x i32> <i32 9, i32 9>
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult <2 x i32> [[N:%.*]], <i32 9, i32 9>
+; CHECK-NEXT:    [[M:%.*]] = select <2 x i1> [[TMP1]], <2 x i32> [[N]], <2 x i32> <i32 9, i32 9>
 ; CHECK-NEXT:    ret <2 x i32> [[M]]
 ;
   %t = icmp ule <2 x i32> %n, <i32 9, i32 9>
@@ -265,9 +265,9 @@ define <2 x i32> @umin4_vec(<2 x i32> %n) {
 
 define i64 @smax_sext(i32 %a) {
 ; CHECK-LABEL: @smax_sext(
-; CHECK-NEXT:    [[A_EXT:%.*]] = sext i32 %a to i64
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i64 [[A_EXT]], 0
-; CHECK-NEXT:    [[MAX:%.*]] = select i1 [[CMP]], i64 [[A_EXT]], i64 0
+; CHECK-NEXT:    [[A_EXT:%.*]] = sext i32 [[A:%.*]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i64 [[A_EXT]], 0
+; CHECK-NEXT:    [[MAX:%.*]] = select i1 [[TMP1]], i64 [[A_EXT]], i64 0
 ; CHECK-NEXT:    ret i64 [[MAX]]
 ;
   %a_ext = sext i32 %a to i64
@@ -278,9 +278,9 @@ define i64 @smax_sext(i32 %a) {
 
 define <2 x i64> @smax_sext_vec(<2 x i32> %a) {
 ; CHECK-LABEL: @smax_sext_vec(
-; CHECK-NEXT:    [[A_EXT:%.*]] = sext <2 x i32> %a to <2 x i64>
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt <2 x i64> [[A_EXT]], zeroinitializer
-; CHECK-NEXT:    [[MAX:%.*]] = select <2 x i1> [[CMP]], <2 x i64> [[A_EXT]], <2 x i64> zeroinitializer
+; CHECK-NEXT:    [[A_EXT:%.*]] = sext <2 x i32> [[A:%.*]] to <2 x i64>
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <2 x i64> [[A_EXT]], zeroinitializer
+; CHECK-NEXT:    [[MAX:%.*]] = select <2 x i1> [[TMP1]], <2 x i64> [[A_EXT]], <2 x i64> zeroinitializer
 ; CHECK-NEXT:    ret <2 x i64> [[MAX]]
 ;
   %a_ext = sext <2 x i32> %a to <2 x i64>
@@ -291,9 +291,9 @@ define <2 x i64> @smax_sext_vec(<2 x i32> %a) {
 
 define i64 @smin_sext(i32 %a) {
 ; CHECK-LABEL: @smin_sext(
-; CHECK-NEXT:    [[A_EXT:%.*]] = sext i32 %a to i64
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i64 [[A_EXT]], 0
-; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[CMP]], i64 [[A_EXT]], i64 0
+; CHECK-NEXT:    [[A_EXT:%.*]] = sext i32 [[A:%.*]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i64 [[A_EXT]], 0
+; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[TMP1]], i64 [[A_EXT]], i64 0
 ; CHECK-NEXT:    ret i64 [[MIN]]
 ;
   %a_ext = sext i32 %a to i64
@@ -304,9 +304,9 @@ define i64 @smin_sext(i32 %a) {
 
 define <2 x i64>@smin_sext_vec(<2 x i32> %a) {
 ; CHECK-LABEL: @smin_sext_vec(
-; CHECK-NEXT:    [[A_EXT:%.*]] = sext <2 x i32> %a to <2 x i64>
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i64> [[A_EXT]], zeroinitializer
-; CHECK-NEXT:    [[MIN:%.*]] = select <2 x i1> [[CMP]], <2 x i64> [[A_EXT]], <2 x i64> zeroinitializer
+; CHECK-NEXT:    [[A_EXT:%.*]] = sext <2 x i32> [[A:%.*]] to <2 x i64>
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt <2 x i64> [[A_EXT]], zeroinitializer
+; CHECK-NEXT:    [[MIN:%.*]] = select <2 x i1> [[TMP1]], <2 x i64> [[A_EXT]], <2 x i64> zeroinitializer
 ; CHECK-NEXT:    ret <2 x i64> [[MIN]]
 ;
   %a_ext = sext <2 x i32> %a to <2 x i64>
@@ -317,9 +317,9 @@ define <2 x i64>@smin_sext_vec(<2 x i32> %a) {
 
 define i64 @umax_sext(i32 %a) {
 ; CHECK-LABEL: @umax_sext(
-; CHECK-NEXT:    [[A_EXT:%.*]] = sext i32 %a to i64
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[A_EXT]], 3
-; CHECK-NEXT:    [[MAX:%.*]] = select i1 [[CMP]], i64 [[A_EXT]], i64 3
+; CHECK-NEXT:    [[A_EXT:%.*]] = sext i32 [[A:%.*]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i64 [[A_EXT]], 3
+; CHECK-NEXT:    [[MAX:%.*]] = select i1 [[TMP1]], i64 [[A_EXT]], i64 3
 ; CHECK-NEXT:    ret i64 [[MAX]]
 ;
   %a_ext = sext i32 %a to i64
@@ -330,9 +330,9 @@ define i64 @umax_sext(i32 %a) {
 
 define <2 x i64> @umax_sext_vec(<2 x i32> %a) {
 ; CHECK-LABEL: @umax_sext_vec(
-; CHECK-NEXT:    [[A_EXT:%.*]] = sext <2 x i32> %a to <2 x i64>
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt <2 x i64> [[A_EXT]], <i64 3, i64 3>
-; CHECK-NEXT:    [[MAX:%.*]] = select <2 x i1> [[CMP]], <2 x i64> [[A_EXT]], <2 x i64> <i64 3, i64 3>
+; CHECK-NEXT:    [[A_EXT:%.*]] = sext <2 x i32> [[A:%.*]] to <2 x i64>
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt <2 x i64> [[A_EXT]], <i64 3, i64 3>
+; CHECK-NEXT:    [[MAX:%.*]] = select <2 x i1> [[TMP1]], <2 x i64> [[A_EXT]], <2 x i64> <i64 3, i64 3>
 ; CHECK-NEXT:    ret <2 x i64> [[MAX]]
 ;
   %a_ext = sext <2 x i32> %a to <2 x i64>
@@ -343,9 +343,9 @@ define <2 x i64> @umax_sext_vec(<2 x i32> %a) {
 
 define i64 @umin_sext(i32 %a) {
 ; CHECK-LABEL: @umin_sext(
-; CHECK-NEXT:    [[A_EXT:%.*]] = sext i32 %a to i64
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i64 [[A_EXT]], 2
-; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[CMP]], i64 [[A_EXT]], i64 2
+; CHECK-NEXT:    [[A_EXT:%.*]] = sext i32 [[A:%.*]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i64 [[A_EXT]], 2
+; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[TMP1]], i64 [[A_EXT]], i64 2
 ; CHECK-NEXT:    ret i64 [[MIN]]
 ;
   %a_ext = sext i32 %a to i64
@@ -356,9 +356,9 @@ define i64 @umin_sext(i32 %a) {
 
 define <2 x i64> @umin_sext_vec(<2 x i32> %a) {
 ; CHECK-LABEL: @umin_sext_vec(
-; CHECK-NEXT:    [[A_EXT:%.*]] = sext <2 x i32> %a to <2 x i64>
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult <2 x i64> [[A_EXT]], <i64 2, i64 2>
-; CHECK-NEXT:    [[MIN:%.*]] = select <2 x i1> [[CMP]], <2 x i64> [[A_EXT]], <2 x i64> <i64 2, i64 2>
+; CHECK-NEXT:    [[A_EXT:%.*]] = sext <2 x i32> [[A:%.*]] to <2 x i64>
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult <2 x i64> [[A_EXT]], <i64 2, i64 2>
+; CHECK-NEXT:    [[MIN:%.*]] = select <2 x i1> [[TMP1]], <2 x i64> [[A_EXT]], <2 x i64> <i64 2, i64 2>
 ; CHECK-NEXT:    ret <2 x i64> [[MIN]]
 ;
   %a_ext = sext <2 x i32> %a to <2 x i64>
@@ -369,7 +369,7 @@ define <2 x i64> @umin_sext_vec(<2 x i32> %a) {
 
 define i64 @umax_sext2(i32 %a) {
 ; CHECK-LABEL: @umax_sext2(
-; CHECK-NEXT:    [[A_EXT:%.*]] = sext i32 %a to i64
+; CHECK-NEXT:    [[A_EXT:%.*]] = sext i32 [[A:%.*]] to i64
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[A_EXT]], 2
 ; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[CMP]], i64 [[A_EXT]], i64 2
 ; CHECK-NEXT:    ret i64 [[MIN]]
@@ -382,7 +382,7 @@ define i64 @umax_sext2(i32 %a) {
 
 define <2 x i64> @umax_sext2_vec(<2 x i32> %a) {
 ; CHECK-LABEL: @umax_sext2_vec(
-; CHECK-NEXT:    [[A_EXT:%.*]] = sext <2 x i32> %a to <2 x i64>
+; CHECK-NEXT:    [[A_EXT:%.*]] = sext <2 x i32> [[A:%.*]] to <2 x i64>
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt <2 x i64> [[A_EXT]], <i64 2, i64 2>
 ; CHECK-NEXT:    [[MIN:%.*]] = select <2 x i1> [[CMP]], <2 x i64> [[A_EXT]], <2 x i64> <i64 2, i64 2>
 ; CHECK-NEXT:    ret <2 x i64> [[MIN]]
@@ -395,7 +395,7 @@ define <2 x i64> @umax_sext2_vec(<2 x i32> %a) {
 
 define i64 @umin_sext2(i32 %a) {
 ; CHECK-LABEL: @umin_sext2(
-; CHECK-NEXT:    [[A_EXT:%.*]] = sext i32 %a to i64
+; CHECK-NEXT:    [[A_EXT:%.*]] = sext i32 [[A:%.*]] to i64
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i64 [[A_EXT]], 3
 ; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[CMP]], i64 [[A_EXT]], i64 3
 ; CHECK-NEXT:    ret i64 [[MIN]]
@@ -408,7 +408,7 @@ define i64 @umin_sext2(i32 %a) {
 
 define <2 x i64> @umin_sext2_vec(<2 x i32> %a) {
 ; CHECK-LABEL: @umin_sext2_vec(
-; CHECK-NEXT:    [[A_EXT:%.*]] = sext <2 x i32> %a to <2 x i64>
+; CHECK-NEXT:    [[A_EXT:%.*]] = sext <2 x i32> [[A:%.*]] to <2 x i64>
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ult <2 x i64> [[A_EXT]], <i64 3, i64 3>
 ; CHECK-NEXT:    [[MIN:%.*]] = select <2 x i1> [[CMP]], <2 x i64> [[A_EXT]], <2 x i64> <i64 3, i64 3>
 ; CHECK-NEXT:    ret <2 x i64> [[MIN]]
@@ -421,9 +421,9 @@ define <2 x i64> @umin_sext2_vec(<2 x i32> %a) {
 
 define i64 @umax_zext(i32 %a) {
 ; CHECK-LABEL: @umax_zext(
-; CHECK-NEXT:    [[A_EXT:%.*]] = zext i32 %a to i64
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[A_EXT]], 3
-; CHECK-NEXT:    [[MAX:%.*]] = select i1 [[CMP]], i64 [[A_EXT]], i64 3
+; CHECK-NEXT:    [[A_EXT:%.*]] = zext i32 [[A:%.*]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i64 [[A_EXT]], 3
+; CHECK-NEXT:    [[MAX:%.*]] = select i1 [[TMP1]], i64 [[A_EXT]], i64 3
 ; CHECK-NEXT:    ret i64 [[MAX]]
 ;
   %a_ext = zext i32 %a to i64
@@ -434,9 +434,9 @@ define i64 @umax_zext(i32 %a) {
 
 define <2 x i64> @umax_zext_vec(<2 x i32> %a) {
 ; CHECK-LABEL: @umax_zext_vec(
-; CHECK-NEXT:    [[A_EXT:%.*]] = zext <2 x i32> %a to <2 x i64>
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt <2 x i64> [[A_EXT]], <i64 3, i64 3>
-; CHECK-NEXT:    [[MAX:%.*]] = select <2 x i1> [[CMP]], <2 x i64> [[A_EXT]], <2 x i64> <i64 3, i64 3>
+; CHECK-NEXT:    [[A_EXT:%.*]] = zext <2 x i32> [[A:%.*]] to <2 x i64>
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt <2 x i64> [[A_EXT]], <i64 3, i64 3>
+; CHECK-NEXT:    [[MAX:%.*]] = select <2 x i1> [[TMP1]], <2 x i64> [[A_EXT]], <2 x i64> <i64 3, i64 3>
 ; CHECK-NEXT:    ret <2 x i64> [[MAX]]
 ;
   %a_ext = zext <2 x i32> %a to <2 x i64>
@@ -447,9 +447,9 @@ define <2 x i64> @umax_zext_vec(<2 x i32> %a) {
 
 define i64 @umin_zext(i32 %a) {
 ; CHECK-LABEL: @umin_zext(
-; CHECK-NEXT:    [[A_EXT:%.*]] = zext i32 %a to i64
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i64 [[A_EXT]], 2
-; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[CMP]], i64 [[A_EXT]], i64 2
+; CHECK-NEXT:    [[A_EXT:%.*]] = zext i32 [[A:%.*]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i64 [[A_EXT]], 2
+; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[TMP1]], i64 [[A_EXT]], i64 2
 ; CHECK-NEXT:    ret i64 [[MIN]]
 ;
   %a_ext = zext i32 %a to i64
@@ -460,9 +460,9 @@ define i64 @umin_zext(i32 %a) {
 
 define <2 x i64> @umin_zext_vec(<2 x i32> %a) {
 ; CHECK-LABEL: @umin_zext_vec(
-; CHECK-NEXT:    [[A_EXT:%.*]] = zext <2 x i32> %a to <2 x i64>
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult <2 x i64> [[A_EXT]], <i64 2, i64 2>
-; CHECK-NEXT:    [[MIN:%.*]] = select <2 x i1> [[CMP]], <2 x i64> [[A_EXT]], <2 x i64> <i64 2, i64 2>
+; CHECK-NEXT:    [[A_EXT:%.*]] = zext <2 x i32> [[A:%.*]] to <2 x i64>
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult <2 x i64> [[A_EXT]], <i64 2, i64 2>
+; CHECK-NEXT:    [[MIN:%.*]] = select <2 x i1> [[TMP1]], <2 x i64> [[A_EXT]], <2 x i64> <i64 2, i64 2>
 ; CHECK-NEXT:    ret <2 x i64> [[MIN]]
 ;
   %a_ext = zext <2 x i32> %a to <2 x i64>
@@ -475,8 +475,8 @@ define <2 x i64> @umin_zext_vec(<2 x i32> %a) {
 
 define <2 x i16> @scalar_select_of_vectors(<2 x i16> %a, <2 x i16> %b, i8 %x) {
 ; CHECK-LABEL: @scalar_select_of_vectors(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 %x, 0
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], <2 x i16> %a, <2 x i16> %b
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[X:%.*]], 0
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], <2 x i16> [[A:%.*]], <2 x i16> [[B:%.*]]
 ; CHECK-NEXT:    ret <2 x i16> [[SEL]]
 ;
   %cmp = icmp slt i8 %x, 0
diff --git a/llvm/test/Transforms/InstCombine/align-addr.ll b/llvm/test/Transforms/InstCombine/align-addr.ll
index d92daddd7..d6dc5e91d 100644
--- a/llvm/test/Transforms/InstCombine/align-addr.ll
+++ b/llvm/test/Transforms/InstCombine/align-addr.ll
@@ -1,13 +1,33 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -instcombine -S | FileCheck %s
 target datalayout = "E-p:64:64:64-p1:32:32:32-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
 
 ; Instcombine should be able to prove vector alignment in the
 ; presence of a few mild address computation tricks.
 
-; CHECK-LABEL: @test0(
-; CHECK: align 16
-
 define void @test0(i8* %b, i64 %n, i64 %u, i64 %y) nounwind  {
+; CHECK-LABEL: @test0(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[C:%.*]] = ptrtoint i8* [[B:%.*]] to i64
+; CHECK-NEXT:    [[D:%.*]] = and i64 [[C]], -16
+; CHECK-NEXT:    [[E:%.*]] = inttoptr i64 [[D]] to double*
+; CHECK-NEXT:    [[V:%.*]] = shl i64 [[U:%.*]], 1
+; CHECK-NEXT:    [[Z:%.*]] = and i64 [[Y:%.*]], -2
+; CHECK-NEXT:    [[T1421:%.*]] = icmp eq i64 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[T1421]], label [[RETURN:%.*]], label [[BB:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[INDVAR_NEXT:%.*]], [[BB]] ], [ 20, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[J:%.*]] = mul i64 [[I]], [[V]]
+; CHECK-NEXT:    [[H:%.*]] = add i64 [[J]], [[Z]]
+; CHECK-NEXT:    [[T8:%.*]] = getelementptr double, double* [[E]], i64 [[H]]
+; CHECK-NEXT:    [[P:%.*]] = bitcast double* [[T8]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> zeroinitializer, <2 x double>* [[P]], align 16
+; CHECK-NEXT:    [[INDVAR_NEXT]] = add i64 [[I]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVAR_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[RETURN]], label [[BB]]
+; CHECK:       return:
+; CHECK-NEXT:    ret void
+;
 entry:
   %c = ptrtoint i8* %b to i64
   %d = and i64 %c, -16
@@ -35,24 +55,26 @@ return:
 ; When we see a unaligned load from an insufficiently aligned global or
 ; alloca, increase the alignment of the load, turning it into an aligned load.
 
-; CHECK-LABEL: @test1(
-; CHECK: tmp = load
-; CHECK: GLOBAL{{.*}}align 16
-
 @GLOBAL = internal global [4 x i32] zeroinitializer
 
 define <16 x i8> @test1(<2 x i64> %x) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([4 x i32]* @GLOBAL to <16 x i8>*), align 16
+; CHECK-NEXT:    ret <16 x i8> [[TMP]]
+;
 entry:
-	%tmp = load <16 x i8>, <16 x i8>* bitcast ([4 x i32]* @GLOBAL to <16 x i8>*), align 1
-	ret <16 x i8> %tmp
+  %tmp = load <16 x i8>, <16 x i8>* bitcast ([4 x i32]* @GLOBAL to <16 x i8>*), align 1
+  ret <16 x i8> %tmp
 }
 
 @GLOBAL_as1 = internal addrspace(1) global [4 x i32] zeroinitializer
 
 define <16 x i8> @test1_as1(<2 x i64> %x) {
 ; CHECK-LABEL: @test1_as1(
-; CHECK: tmp = load
-; CHECK: GLOBAL_as1{{.*}}align 16
+; CHECK-NEXT:    [[TMP:%.*]] = load <16 x i8>, <16 x i8> addrspace(1)* bitcast ([4 x i32] addrspace(1)* @GLOBAL_as1 to <16 x i8> addrspace(1)*), align 16
+; CHECK-NEXT:    ret <16 x i8> [[TMP]]
+;
   %tmp = load <16 x i8>, <16 x i8> addrspace(1)* bitcast ([4 x i32] addrspace(1)* @GLOBAL_as1 to <16 x i8> addrspace(1)*), align 1
   ret <16 x i8> %tmp
 }
@@ -61,8 +83,9 @@ define <16 x i8> @test1_as1(<2 x i64> %x) {
 
 define <16 x i8> @test1_as1_gep(<2 x i64> %x) {
 ; CHECK-LABEL: @test1_as1_gep(
-; CHECK: tmp = load
-; CHECK: GLOBAL_as1_gep{{.*}}align 16
+; CHECK-NEXT:    [[TMP:%.*]] = load <16 x i8>, <16 x i8> addrspace(1)* bitcast (i32 addrspace(1)* getelementptr inbounds ([8 x i32], [8 x i32] addrspace(1)* @GLOBAL_as1_gep, i32 0, i32 4) to <16 x i8> addrspace(1)*), align 16
+; CHECK-NEXT:    ret <16 x i8> [[TMP]]
+;
   %tmp = load <16 x i8>, <16 x i8> addrspace(1)* bitcast (i32 addrspace(1)* getelementptr ([8 x i32], [8 x i32] addrspace(1)* @GLOBAL_as1_gep, i16 0, i16 4) to <16 x i8> addrspace(1)*), align 1
   ret <16 x i8> %tmp
 }
@@ -70,11 +93,12 @@ define <16 x i8> @test1_as1_gep(<2 x i64> %x) {
 
 ; When a load or store lacks an explicit alignment, add one.
 
-; CHECK-LABEL: @test2(
-; CHECK: load double, double* %p, align 8
-; CHECK: store double %n, double* %p, align 8
-
 define double @test2(double* %p, double %n) nounwind {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[T:%.*]] = load double, double* [[P:%.*]], align 8
+; CHECK-NEXT:    store double [[N:%.*]], double* [[P]], align 8
+; CHECK-NEXT:    ret double [[T]]
+;
   %t = load double, double* %p
   store double %n, double* %p
   ret double %t
@@ -89,9 +113,13 @@ declare void @use(i8*)
 define void @test3(%struct.s* sret %a4) {
 ; Check that the alignment is bumped up the alignment of the sret type.
 ; CHECK-LABEL: @test3(
+; CHECK-NEXT:    [[A4_CAST:%.*]] = bitcast %struct.s* [[A4:%.*]] to i8*
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* nonnull align 4 dereferenceable(16) [[A4_CAST]], i8 0, i64 16, i1 false)
+; CHECK-NEXT:    call void @use(i8* [[A4_CAST]])
+; CHECK-NEXT:    ret void
+;
   %a4.cast = bitcast %struct.s* %a4 to i8*
   call void @llvm.memset.p0i8.i64(i8* %a4.cast, i8 0, i64 16, i1 false)
-; CHECK: call void @llvm.memset.p0i8.i64(i8* align 4 %a4.cast, i8 0, i64 16, i1 false)
   call void @use(i8* %a4.cast)
   ret void
 }
diff --git a/llvm/test/Transforms/InstCombine/and-or-icmps.ll b/llvm/test/Transforms/InstCombine/and-or-icmps.ll
index 516235f94..8d5471790 100644
--- a/llvm/test/Transforms/InstCombine/and-or-icmps.ll
+++ b/llvm/test/Transforms/InstCombine/and-or-icmps.ll
@@ -3,7 +3,7 @@
 
 define i1 @PR1817_1(i32 %X) {
 ; CHECK-LABEL: @PR1817_1(
-; CHECK-NEXT:    [[B:%.*]] = icmp ult i32 %X, 10
+; CHECK-NEXT:    [[B:%.*]] = icmp ult i32 [[X:%.*]], 10
 ; CHECK-NEXT:    ret i1 [[B]]
 ;
   %A = icmp slt i32 %X, 10
@@ -14,7 +14,7 @@ define i1 @PR1817_1(i32 %X) {
 
 define i1 @PR1817_2(i32 %X) {
 ; CHECK-LABEL: @PR1817_2(
-; CHECK-NEXT:    [[A:%.*]] = icmp slt i32 %X, 10
+; CHECK-NEXT:    [[A:%.*]] = icmp slt i32 [[X:%.*]], 10
 ; CHECK-NEXT:    ret i1 [[A]]
 ;
   %A = icmp slt i32 %X, 10
@@ -25,7 +25,7 @@ define i1 @PR1817_2(i32 %X) {
 
 define i1 @PR2330(i32 %a, i32 %b) {
 ; CHECK-LABEL: @PR2330(
-; CHECK-NEXT:    [[TMP1:%.*]] = or i32 %b, %a
+; CHECK-NEXT:    [[TMP1:%.*]] = or i32 [[B:%.*]], [[A:%.*]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i32 [[TMP1]], 8
 ; CHECK-NEXT:    ret i1 [[TMP2]]
 ;
@@ -41,7 +41,7 @@ define i1 @PR2330(i32 %a, i32 %b) {
 
 define i1 @or_eq_with_one_bit_diff_constants1(i32 %x) {
 ; CHECK-LABEL: @or_eq_with_one_bit_diff_constants1(
-; CHECK-NEXT:    [[TMP1:%.*]] = or i32 %x, 1
+; CHECK-NEXT:    [[TMP1:%.*]] = or i32 [[X:%.*]], 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], 51
 ; CHECK-NEXT:    ret i1 [[TMP2]]
 ;
@@ -55,7 +55,7 @@ define i1 @or_eq_with_one_bit_diff_constants1(i32 %x) {
 
 define i1 @and_ne_with_one_bit_diff_constants1(i32 %x) {
 ; CHECK-LABEL: @and_ne_with_one_bit_diff_constants1(
-; CHECK-NEXT:    [[TMP1:%.*]] = or i32 %x, 1
+; CHECK-NEXT:    [[TMP1:%.*]] = or i32 [[X:%.*]], 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i32 [[TMP1]], 51
 ; CHECK-NEXT:    ret i1 [[TMP2]]
 ;
@@ -69,7 +69,7 @@ define i1 @and_ne_with_one_bit_diff_constants1(i32 %x) {
 
 define i1 @or_eq_with_one_bit_diff_constants2(i32 %x) {
 ; CHECK-LABEL: @or_eq_with_one_bit_diff_constants2(
-; CHECK-NEXT:    [[TMP1:%.*]] = or i32 %x, 32
+; CHECK-NEXT:    [[TMP1:%.*]] = or i32 [[X:%.*]], 32
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], 97
 ; CHECK-NEXT:    ret i1 [[TMP2]]
 ;
@@ -81,7 +81,7 @@ define i1 @or_eq_with_one_bit_diff_constants2(i32 %x) {
 
 define i1 @and_ne_with_one_bit_diff_constants2(i19 %x) {
 ; CHECK-LABEL: @and_ne_with_one_bit_diff_constants2(
-; CHECK-NEXT:    [[TMP1:%.*]] = or i19 %x, 128
+; CHECK-NEXT:    [[TMP1:%.*]] = or i19 [[X:%.*]], 128
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i19 [[TMP1]], 193
 ; CHECK-NEXT:    ret i1 [[TMP2]]
 ;
@@ -95,7 +95,7 @@ define i1 @and_ne_with_one_bit_diff_constants2(i19 %x) {
 
 define i1 @or_eq_with_one_bit_diff_constants3(i8 %x) {
 ; CHECK-LABEL: @or_eq_with_one_bit_diff_constants3(
-; CHECK-NEXT:    [[TMP1:%.*]] = or i8 %x, -128
+; CHECK-NEXT:    [[TMP1:%.*]] = or i8 [[X:%.*]], -128
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i8 [[TMP1]], -2
 ; CHECK-NEXT:    ret i1 [[TMP2]]
 ;
@@ -107,7 +107,7 @@ define i1 @or_eq_with_one_bit_diff_constants3(i8 %x) {
 
 define i1 @and_ne_with_one_bit_diff_constants3(i8 %x) {
 ; CHECK-LABEL: @and_ne_with_one_bit_diff_constants3(
-; CHECK-NEXT:    [[TMP1:%.*]] = or i8 %x, -128
+; CHECK-NEXT:    [[TMP1:%.*]] = or i8 [[X:%.*]], -128
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i8 [[TMP1]], -63
 ; CHECK-NEXT:    ret i1 [[TMP2]]
 ;
@@ -122,7 +122,7 @@ define i1 @and_ne_with_one_bit_diff_constants3(i8 %x) {
 
 define i1 @or_eq_with_diff_one(i8 %x) {
 ; CHECK-LABEL: @or_eq_with_diff_one(
-; CHECK-NEXT:    [[TMP1:%.*]] = add i8 %x, -13
+; CHECK-NEXT:    [[TMP1:%.*]] = add i8 [[X:%.*]], -13
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i8 [[TMP1]], 2
 ; CHECK-NEXT:    ret i1 [[TMP2]]
 ;
@@ -136,7 +136,7 @@ define i1 @or_eq_with_diff_one(i8 %x) {
 
 define i1 @and_ne_with_diff_one(i32 %x) {
 ; CHECK-LABEL: @and_ne_with_diff_one(
-; CHECK-NEXT:    [[TMP1:%.*]] = add i32 %x, -39
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[X:%.*]], -39
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp ugt i32 [[TMP1]], 1
 ; CHECK-NEXT:    ret i1 [[TMP2]]
 ;
@@ -151,7 +151,7 @@ define i1 @and_ne_with_diff_one(i32 %x) {
 
 define i1 @or_eq_with_diff_one_signed(i32 %x) {
 ; CHECK-LABEL: @or_eq_with_diff_one_signed(
-; CHECK-NEXT:    [[TMP1:%.*]] = add i32 %x, 1
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[X:%.*]], 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i32 [[TMP1]], 2
 ; CHECK-NEXT:    ret i1 [[TMP2]]
 ;
@@ -163,7 +163,7 @@ define i1 @or_eq_with_diff_one_signed(i32 %x) {
 
 define i1 @and_ne_with_diff_one_signed(i64 %x) {
 ; CHECK-LABEL: @and_ne_with_diff_one_signed(
-; CHECK-NEXT:    [[TMP1:%.*]] = add i64 %x, 1
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[X:%.*]], 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp ugt i64 [[TMP1]], 1
 ; CHECK-NEXT:    ret i1 [[TMP2]]
 ;
@@ -177,7 +177,7 @@ define i1 @and_ne_with_diff_one_signed(i64 %x) {
 
 define <2 x i1> @or_eq_with_one_bit_diff_constants2_splatvec(<2 x i32> %x) {
 ; CHECK-LABEL: @or_eq_with_one_bit_diff_constants2_splatvec(
-; CHECK-NEXT:    [[TMP1:%.*]] = or <2 x i32> %x, <i32 32, i32 32>
+; CHECK-NEXT:    [[TMP1:%.*]] = or <2 x i32> [[X:%.*]], <i32 32, i32 32>
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq <2 x i32> [[TMP1]], <i32 97, i32 97>
 ; CHECK-NEXT:    ret <2 x i1> [[TMP2]]
 ;
@@ -189,7 +189,7 @@ define <2 x i1> @or_eq_with_one_bit_diff_constants2_splatvec(<2 x i32> %x) {
 
 define <2 x i1> @and_ne_with_diff_one_splatvec(<2 x i32> %x) {
 ; CHECK-LABEL: @and_ne_with_diff_one_splatvec(
-; CHECK-NEXT:    [[TMP1:%.*]] = add <2 x i32> %x, <i32 -39, i32 -39>
+; CHECK-NEXT:    [[TMP1:%.*]] = add <2 x i32> [[X:%.*]], <i32 -39, i32 -39>
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp ugt <2 x i32> [[TMP1]], <i32 1, i32 1>
 ; CHECK-NEXT:    ret <2 x i1> [[TMP2]]
 ;
@@ -253,3 +253,118 @@ define void @simplify_before_foldAndOfICmps() {
   ret void
 }
 
+define i1 @PR42691_1(i32 %x) {
+; CHECK-LABEL: @PR42691_1(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i32 [[X:%.*]], 2147483646
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %c1 = icmp slt i32 %x, 0
+  %c2 = icmp eq i32 %x, 2147483647
+  %c = or i1 %c1, %c2
+  ret i1 %c
+}
+
+define i1 @PR42691_2(i32 %x) {
+; CHECK-LABEL: @PR42691_2(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[X:%.*]], -2
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %c1 = icmp ult i32 %x, 2147483648
+  %c2 = icmp eq i32 %x, 4294967295
+  %c = or i1 %c1, %c2
+  ret i1 %c
+}
+
+define i1 @PR42691_3(i32 %x) {
+; CHECK-LABEL: @PR42691_3(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[X:%.*]], -2147483647
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %c1 = icmp sge i32 %x, 0
+  %c2 = icmp eq i32 %x, -2147483648
+  %c = or i1 %c1, %c2
+  ret i1 %c
+}
+
+define i1 @PR42691_4(i32 %x) {
+; CHECK-LABEL: @PR42691_4(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[X:%.*]], 1
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %c1 = icmp uge i32 %x, 2147483648
+  %c2 = icmp eq i32 %x, 0
+  %c = or i1 %c1, %c2
+  ret i1 %c
+}
+
+define i1 @PR42691_5(i32 %x) {
+; CHECK-LABEL: @PR42691_5(
+; CHECK-NEXT:    [[X_OFF:%.*]] = add i32 [[X:%.*]], -1
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i32 [[X_OFF]], 2147483645
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %c1 = icmp slt i32 %x, 1
+  %c2 = icmp eq i32 %x, 2147483647
+  %c = or i1 %c1, %c2
+  ret i1 %c
+}
+
+define i1 @PR42691_6(i32 %x) {
+; CHECK-LABEL: @PR42691_6(
+; CHECK-NEXT:    [[X_OFF:%.*]] = add i32 [[X:%.*]], 2147483647
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i32 [[X_OFF]], 2147483645
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %c1 = icmp ult i32 %x, 2147483649
+  %c2 = icmp eq i32 %x, 4294967295
+  %c = or i1 %c1, %c2
+  ret i1 %c
+}
+
+define i1 @PR42691_7(i32 %x) {
+; CHECK-LABEL: @PR42691_7(
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[X:%.*]], -1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i32 [[TMP1]], 0
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %c1 = icmp uge i32 %x, 2147483649
+  %c2 = icmp eq i32 %x, 0
+  %c = or i1 %c1, %c2
+  ret i1 %c
+}
+
+define i1 @PR42691_8(i32 %x) {
+; CHECK-LABEL: @PR42691_8(
+; CHECK-NEXT:    [[X_OFF:%.*]] = add i32 [[X:%.*]], 2147483647
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[X_OFF]], -2147483635
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %c1 = icmp slt i32 %x, 14
+  %c2 = icmp ne i32 %x, -2147483648
+  %c = and i1 %c1, %c2
+  ret i1 %c
+}
+
+define i1 @PR42691_9(i32 %x) {
+; CHECK-LABEL: @PR42691_9(
+; CHECK-NEXT:    [[X_OFF:%.*]] = add i32 [[X:%.*]], -14
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[X_OFF]], 2147483633
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %c1 = icmp sgt i32 %x, 13
+  %c2 = icmp ne i32 %x, 2147483647
+  %c = and i1 %c1, %c2
+  ret i1 %c
+}
+
+define i1 @PR42691_10(i32 %x) {
+; CHECK-LABEL: @PR42691_10(
+; CHECK-NEXT:    [[X_OFF:%.*]] = add i32 [[X:%.*]], -14
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[X_OFF]], -15
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %c1 = icmp ugt i32 %x, 13
+  %c2 = icmp ne i32 %x, 4294967295
+  %c = and i1 %c1, %c2
+  ret i1 %c
+}
diff --git a/llvm/test/Transforms/InstCombine/and.ll b/llvm/test/Transforms/InstCombine/and.ll
index 4925013b1..025f05793 100644
--- a/llvm/test/Transforms/InstCombine/and.ll
+++ b/llvm/test/Transforms/InstCombine/and.ll
@@ -346,8 +346,7 @@ define i32 @test30(i1 %X) {
 
 define i32 @test31(i1 %X) {
 ; CHECK-LABEL: @test31(
-; CHECK-NEXT:    [[Y:%.*]] = zext i1 %X to i32
-; CHECK-NEXT:    [[Z:%.*]] = shl nuw nsw i32 [[Y]], 4
+; CHECK-NEXT:    [[Z:%.*]] = select i1 [[X:%.*]], i32 16, i32 0
 ; CHECK-NEXT:    ret i32 [[Z]]
 ;
   %Y = zext i1 %X to i32
diff --git a/llvm/test/Transforms/InstCombine/and2.ll b/llvm/test/Transforms/InstCombine/and2.ll
index 7d0562664..74b120458 100644
--- a/llvm/test/Transforms/InstCombine/and2.ll
+++ b/llvm/test/Transforms/InstCombine/and2.ll
@@ -3,8 +3,8 @@
 
 define i1 @test2(i1 %X, i1 %Y) {
 ; CHECK-LABEL: @test2(
-; CHECK-NEXT:    [[B:%.*]] = and i1 %X, %Y
-; CHECK-NEXT:    ret i1 [[B]]
+; CHECK-NEXT:    [[A:%.*]] = and i1 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[A]]
 ;
   %a = and i1 %X, %Y
   %b = and i1 %a, %X
@@ -13,8 +13,8 @@ define i1 @test2(i1 %X, i1 %Y) {
 
 define i32 @test3(i32 %X, i32 %Y) {
 ; CHECK-LABEL: @test3(
-; CHECK-NEXT:    [[B:%.*]] = and i32 %X, %Y
-; CHECK-NEXT:    ret i32 [[B]]
+; CHECK-NEXT:    [[A:%.*]] = and i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[A]]
 ;
   %a = and i32 %X, %Y
   %b = and i32 %Y, %a
@@ -23,8 +23,8 @@ define i32 @test3(i32 %X, i32 %Y) {
 
 define i1 @test7(i32 %i, i1 %b) {
 ; CHECK-LABEL: @test7(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i32 %i, 0
-; CHECK-NEXT:    [[TMP2:%.*]] = and i1 [[TMP1]], %b
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[I:%.*]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = and i1 [[TMP1]], [[B:%.*]]
 ; CHECK-NEXT:    ret i1 [[TMP2]]
 ;
   %cmp1 = icmp slt i32 %i, 1
@@ -36,7 +36,7 @@ define i1 @test7(i32 %i, i1 %b) {
 
 define i1 @test8(i32 %i) {
 ; CHECK-LABEL: @test8(
-; CHECK-NEXT:    [[I_OFF:%.*]] = add i32 %i, -1
+; CHECK-NEXT:    [[I_OFF:%.*]] = add i32 [[I:%.*]], -1
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[I_OFF]], 13
 ; CHECK-NEXT:    ret i1 [[TMP1]]
 ;
@@ -49,8 +49,8 @@ define i1 @test8(i32 %i) {
 ; FIXME: Vectors should fold too.
 define <2 x i1> @test8vec(<2 x i32> %i) {
 ; CHECK-LABEL: @test8vec(
-; CHECK-NEXT:    [[CMP1:%.*]] = icmp ne <2 x i32> %i, zeroinitializer
-; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult <2 x i32> %i, <i32 14, i32 14>
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ne <2 x i32> [[I:%.*]], zeroinitializer
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult <2 x i32> [[I]], <i32 14, i32 14>
 ; CHECK-NEXT:    [[COND:%.*]] = and <2 x i1> [[CMP1]], [[CMP2]]
 ; CHECK-NEXT:    ret <2 x i1> [[COND]]
 ;
@@ -63,7 +63,7 @@ define <2 x i1> @test8vec(<2 x i32> %i) {
 ; combine -x & 1 into x & 1
 define i64 @test9(i64 %x) {
 ; CHECK-LABEL: @test9(
-; CHECK-NEXT:    [[AND:%.*]] = and i64 %x, 1
+; CHECK-NEXT:    [[AND:%.*]] = and i64 [[X:%.*]], 1
 ; CHECK-NEXT:    ret i64 [[AND]]
 ;
   %sub = sub nsw i64 0, %x
@@ -74,7 +74,7 @@ define i64 @test9(i64 %x) {
 ; combine -x & 1 into x & 1
 define <2 x i64> @test9vec(<2 x i64> %x) {
 ; CHECK-LABEL: @test9vec(
-; CHECK-NEXT:    [[AND:%.*]] = and <2 x i64> %x, <i64 1, i64 1>
+; CHECK-NEXT:    [[AND:%.*]] = and <2 x i64> [[X:%.*]], <i64 1, i64 1>
 ; CHECK-NEXT:    ret <2 x i64> [[AND]]
 ;
   %sub = sub nsw <2 x i64> <i64 0, i64 0>, %x
@@ -84,8 +84,8 @@ define <2 x i64> @test9vec(<2 x i64> %x) {
 
 define i64 @test10(i64 %x) {
 ; CHECK-LABEL: @test10(
-; CHECK-NEXT:    [[AND:%.*]] = and i64 %x, 1
-; CHECK-NEXT:    [[ADD:%.*]] = sub i64 [[AND]], %x
+; CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[X:%.*]], -2
+; CHECK-NEXT:    [[ADD:%.*]] = sub i64 0, [[TMP1]]
 ; CHECK-NEXT:    ret i64 [[ADD]]
 ;
   %sub = sub nsw i64 0, %x
@@ -98,7 +98,7 @@ define i64 @test10(i64 %x) {
 
 define i8 @and1_shl1_is_cmp_eq_0(i8 %x) {
 ; CHECK-LABEL: @and1_shl1_is_cmp_eq_0(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i8 %x, 0
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i8 [[X:%.*]], 0
 ; CHECK-NEXT:    [[AND:%.*]] = zext i1 [[TMP1]] to i8
 ; CHECK-NEXT:    ret i8 [[AND]]
 ;
@@ -111,7 +111,7 @@ define i8 @and1_shl1_is_cmp_eq_0(i8 %x) {
 
 define i8 @and1_shl1_is_cmp_eq_0_multiuse(i8 %x) {
 ; CHECK-LABEL: @and1_shl1_is_cmp_eq_0_multiuse(
-; CHECK-NEXT:    [[SH:%.*]] = shl i8 1, %x
+; CHECK-NEXT:    [[SH:%.*]] = shl i8 1, [[X:%.*]]
 ; CHECK-NEXT:    [[AND:%.*]] = and i8 [[SH]], 1
 ; CHECK-NEXT:    [[ADD:%.*]] = add i8 [[SH]], [[AND]]
 ; CHECK-NEXT:    ret i8 [[ADD]]
@@ -126,7 +126,7 @@ define i8 @and1_shl1_is_cmp_eq_0_multiuse(i8 %x) {
 
 define <2 x i8> @and1_shl1_is_cmp_eq_0_vec(<2 x i8> %x) {
 ; CHECK-LABEL: @and1_shl1_is_cmp_eq_0_vec(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq <2 x i8> %x, zeroinitializer
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq <2 x i8> [[X:%.*]], zeroinitializer
 ; CHECK-NEXT:    [[AND:%.*]] = zext <2 x i1> [[TMP1]] to <2 x i8>
 ; CHECK-NEXT:    ret <2 x i8> [[AND]]
 ;
@@ -139,7 +139,7 @@ define <2 x i8> @and1_shl1_is_cmp_eq_0_vec(<2 x i8> %x) {
 
 define i8 @and1_lshr1_is_cmp_eq_0(i8 %x) {
 ; CHECK-LABEL: @and1_lshr1_is_cmp_eq_0(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i8 %x, 0
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i8 [[X:%.*]], 0
 ; CHECK-NEXT:    [[AND:%.*]] = zext i1 [[TMP1]] to i8
 ; CHECK-NEXT:    ret i8 [[AND]]
 ;
@@ -152,7 +152,7 @@ define i8 @and1_lshr1_is_cmp_eq_0(i8 %x) {
 
 define i8 @and1_lshr1_is_cmp_eq_0_multiuse(i8 %x) {
 ; CHECK-LABEL: @and1_lshr1_is_cmp_eq_0_multiuse(
-; CHECK-NEXT:    [[SH:%.*]] = lshr i8 1, %x
+; CHECK-NEXT:    [[SH:%.*]] = lshr i8 1, [[X:%.*]]
 ; CHECK-NEXT:    [[AND:%.*]] = and i8 [[SH]], 1
 ; CHECK-NEXT:    [[ADD:%.*]] = add nuw nsw i8 [[SH]], [[AND]]
 ; CHECK-NEXT:    ret i8 [[ADD]]
@@ -167,7 +167,7 @@ define i8 @and1_lshr1_is_cmp_eq_0_multiuse(i8 %x) {
 
 define <2 x i8> @and1_lshr1_is_cmp_eq_0_vec(<2 x i8> %x) {
 ; CHECK-LABEL: @and1_lshr1_is_cmp_eq_0_vec(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq <2 x i8> %x, zeroinitializer
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq <2 x i8> [[X:%.*]], zeroinitializer
 ; CHECK-NEXT:    [[AND:%.*]] = zext <2 x i1> [[TMP1]] to <2 x i8>
 ; CHECK-NEXT:    ret <2 x i8> [[AND]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/apint-shift.ll b/llvm/test/Transforms/InstCombine/apint-shift.ll
index 495d9d6d8..8f7e36ca6 100644
--- a/llvm/test/Transforms/InstCombine/apint-shift.ll
+++ b/llvm/test/Transforms/InstCombine/apint-shift.ll
@@ -533,8 +533,11 @@ define i177 @ossfuzz_9880(i177 %X) {
 ; CHECK-LABEL: @ossfuzz_9880(
 ; CHECK-NEXT:    [[A:%.*]] = alloca i177, align 8
 ; CHECK-NEXT:    [[L1:%.*]] = load i177, i177* [[A]], align 8
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i177 [[L1]], 0
-; CHECK-NEXT:    [[B1:%.*]] = zext i1 [[TMP1]] to i177
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i177 [[L1]], -1
+; CHECK-NEXT:    [[TMP2:%.*]] = sext i1 [[TMP1]] to i177
+; CHECK-NEXT:    [[B14:%.*]] = add i177 [[L1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i177 [[B14]], -1
+; CHECK-NEXT:    [[B1:%.*]] = zext i1 [[TMP3]] to i177
 ; CHECK-NEXT:    ret i177 [[B1]]
 ;
   %A = alloca i177
diff --git a/llvm/test/Transforms/InstCombine/apint-sub.ll b/llvm/test/Transforms/InstCombine/apint-sub.ll
index 8d80f2c84..e4094f1d4 100644
--- a/llvm/test/Transforms/InstCombine/apint-sub.ll
+++ b/llvm/test/Transforms/InstCombine/apint-sub.ll
@@ -11,7 +11,7 @@ define i23 @test1(i23 %A) {
 
 define i47 @test2(i47 %A) {
 ; CHECK-LABEL: @test2(
-; CHECK-NEXT:    ret i47 %A
+; CHECK-NEXT:    ret i47 [[A:%.*]]
 ;
   %B = sub i47 %A, 0
   ret i47 %B
@@ -19,7 +19,7 @@ define i47 @test2(i47 %A) {
 
 define i97 @test3(i97 %A) {
 ; CHECK-LABEL: @test3(
-; CHECK-NEXT:    ret i97 %A
+; CHECK-NEXT:    ret i97 [[A:%.*]]
 ;
   %B = sub i97 0, %A
   %C = sub i97 0, %B
@@ -28,7 +28,7 @@ define i97 @test3(i97 %A) {
 
 define i108 @test4(i108 %A, i108 %x) {
 ; CHECK-LABEL: @test4(
-; CHECK-NEXT:    [[C:%.*]] = add i108 %x, %A
+; CHECK-NEXT:    [[C:%.*]] = add i108 [[X:%.*]], [[A:%.*]]
 ; CHECK-NEXT:    ret i108 [[C]]
 ;
   %B = sub i108 0, %A
@@ -38,8 +38,8 @@ define i108 @test4(i108 %A, i108 %x) {
 
 define i19 @test5(i19 %A, i19 %Bok, i19 %Cok) {
 ; CHECK-LABEL: @test5(
-; CHECK-NEXT:    [[D1:%.*]] = sub i19 %Cok, %Bok
-; CHECK-NEXT:    [[E:%.*]] = add i19 [[D1]], %A
+; CHECK-NEXT:    [[D1:%.*]] = sub i19 [[COK:%.*]], [[BOK:%.*]]
+; CHECK-NEXT:    [[E:%.*]] = add i19 [[D1]], [[A:%.*]]
 ; CHECK-NEXT:    ret i19 [[E]]
 ;
   %D = sub i19 %Bok, %Cok
@@ -49,8 +49,8 @@ define i19 @test5(i19 %A, i19 %Bok, i19 %Cok) {
 
 define i57 @test6(i57 %A, i57 %B) {
 ; CHECK-LABEL: @test6(
-; CHECK-NEXT:    [[B_NOT:%.*]] = xor i57 %B, -1
-; CHECK-NEXT:    [[D:%.*]] = and i57 [[B_NOT]], %A
+; CHECK-NEXT:    [[B_NOT:%.*]] = xor i57 [[B:%.*]], -1
+; CHECK-NEXT:    [[D:%.*]] = and i57 [[B_NOT]], [[A:%.*]]
 ; CHECK-NEXT:    ret i57 [[D]]
 ;
   %C = and i57 %A, %B
@@ -60,7 +60,7 @@ define i57 @test6(i57 %A, i57 %B) {
 
 define i77 @test7(i77 %A) {
 ; CHECK-LABEL: @test7(
-; CHECK-NEXT:    [[B:%.*]] = xor i77 %A, -1
+; CHECK-NEXT:    [[B:%.*]] = xor i77 [[A:%.*]], -1
 ; CHECK-NEXT:    ret i77 [[B]]
 ;
   %B = sub i77 -1, %A
@@ -69,7 +69,7 @@ define i77 @test7(i77 %A) {
 
 define i27 @test8(i27 %A) {
 ; CHECK-LABEL: @test8(
-; CHECK-NEXT:    [[C:%.*]] = shl i27 %A, 3
+; CHECK-NEXT:    [[C:%.*]] = shl i27 [[A:%.*]], 3
 ; CHECK-NEXT:    ret i27 [[C]]
 ;
   %B = mul i27 9, %A
@@ -79,7 +79,7 @@ define i27 @test8(i27 %A) {
 
 define i42 @test9(i42 %A) {
 ; CHECK-LABEL: @test9(
-; CHECK-NEXT:    [[C:%.*]] = mul i42 %A, -2
+; CHECK-NEXT:    [[C:%.*]] = mul i42 [[A:%.*]], -2
 ; CHECK-NEXT:    ret i42 [[C]]
 ;
   %B = mul i42 3, %A
@@ -89,7 +89,7 @@ define i42 @test9(i42 %A) {
 
 define i1 @test11(i9 %A, i9 %B) {
 ; CHECK-LABEL: @test11(
-; CHECK-NEXT:    [[CD:%.*]] = icmp ne i9 %A, %B
+; CHECK-NEXT:    [[CD:%.*]] = icmp ne i9 [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    ret i1 [[CD]]
 ;
   %C = sub i9 %A, %B
@@ -99,7 +99,7 @@ define i1 @test11(i9 %A, i9 %B) {
 
 define i43 @test12(i43 %A) {
 ; CHECK-LABEL: @test12(
-; CHECK-NEXT:    [[C:%.*]] = lshr i43 %A, 42
+; CHECK-NEXT:    [[C:%.*]] = lshr i43 [[A:%.*]], 42
 ; CHECK-NEXT:    ret i43 [[C]]
 ;
   %B = ashr i43 %A, 42
@@ -109,7 +109,7 @@ define i43 @test12(i43 %A) {
 
 define i79 @test13(i79 %A) {
 ; CHECK-LABEL: @test13(
-; CHECK-NEXT:    [[C:%.*]] = ashr i79 %A, 78
+; CHECK-NEXT:    [[C:%.*]] = ashr i79 [[A:%.*]], 78
 ; CHECK-NEXT:    ret i79 [[C]]
 ;
   %B = lshr i79 %A, 78
@@ -119,7 +119,7 @@ define i79 @test13(i79 %A) {
 
 define i1024 @test14(i1024 %A) {
 ; CHECK-LABEL: @test14(
-; CHECK-NEXT:    [[D:%.*]] = ashr i1024 %A, 1023
+; CHECK-NEXT:    [[D:%.*]] = ashr i1024 [[A:%.*]], 1023
 ; CHECK-NEXT:    ret i1024 [[D]]
 ;
   %B = lshr i1024 %A, 1023
@@ -130,7 +130,7 @@ define i1024 @test14(i1024 %A) {
 
 define i51 @test16(i51 %A) {
 ; CHECK-LABEL: @test16(
-; CHECK-NEXT:    [[Y:%.*]] = sdiv i51 %A, -1123
+; CHECK-NEXT:    [[Y:%.*]] = sdiv i51 [[A:%.*]], -1123
 ; CHECK-NEXT:    ret i51 [[Y]]
 ;
   %X = sdiv i51 %A, 1123
@@ -142,7 +142,7 @@ define i51 @test16(i51 %A) {
 ; PR3142
 define i25 @test17(i25 %Aok) {
 ; CHECK-LABEL: @test17(
-; CHECK-NEXT:    [[B:%.*]] = sub i25 0, %Aok
+; CHECK-NEXT:    [[B:%.*]] = sub i25 0, [[AOK:%.*]]
 ; CHECK-NEXT:    [[C:%.*]] = sdiv i25 [[B]], 1234
 ; CHECK-NEXT:    ret i25 [[C]]
 ;
@@ -163,7 +163,7 @@ define i128 @test18(i128 %Y) {
 
 define i39 @test19(i39 %X, i39 %Y) {
 ; CHECK-LABEL: @test19(
-; CHECK-NEXT:    ret i39 %X
+; CHECK-NEXT:    ret i39 [[X:%.*]]
 ;
   %Z = sub i39 %X, %Y
   %Q = add i39 %Z, %Y
@@ -172,7 +172,7 @@ define i39 @test19(i39 %X, i39 %Y) {
 
 define i1 @test20(i33 %g, i33 %h) {
 ; CHECK-LABEL: @test20(
-; CHECK-NEXT:    [[T4:%.*]] = icmp ne i33 %h, 0
+; CHECK-NEXT:    [[T4:%.*]] = icmp ne i33 [[H:%.*]], 0
 ; CHECK-NEXT:    ret i1 [[T4]]
 ;
   %t2 = sub i33 %g, %h
@@ -182,7 +182,7 @@ define i1 @test20(i33 %g, i33 %h) {
 
 define i1 @test21(i256 %g, i256 %h) {
 ; CHECK-LABEL: @test21(
-; CHECK-NEXT:    [[T4:%.*]] = icmp ne i256 %h, 0
+; CHECK-NEXT:    [[T4:%.*]] = icmp ne i256 [[H:%.*]], 0
 ; CHECK-NEXT:    ret i1 [[T4]]
 ;
   %t2 = sub i256 %g, %h
diff --git a/llvm/test/Transforms/InstCombine/assume.ll b/llvm/test/Transforms/InstCombine/assume.ll
index bc11a1f17..8fabded56 100644
--- a/llvm/test/Transforms/InstCombine/assume.ll
+++ b/llvm/test/Transforms/InstCombine/assume.ll
@@ -1,17 +1,23 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -instcombine -S | FileCheck %s
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define i32 @foo1(i32* %a) #0 {
+; CHECK-LABEL: @foo1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 32
+; CHECK-NEXT:    [[PTRINT:%.*]] = ptrtoint i32* [[A]] to i64
+; CHECK-NEXT:    [[MASKEDPTR:%.*]] = and i64 [[PTRINT]], 31
+; CHECK-NEXT:    [[MASKCOND:%.*]] = icmp eq i64 [[MASKEDPTR]], 0
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[MASKCOND]])
+; CHECK-NEXT:    ret i32 [[TMP0]]
+;
 entry:
   %0 = load i32, i32* %a, align 4
 
 ; Check that the alignment has been upgraded and that the assume has not
 ; been removed:
-; CHECK-LABEL: @foo1
-; CHECK-DAG: load i32, i32* %a, align 32
-; CHECK-DAG: call void @llvm.assume
-; CHECK: ret i32
 
   %ptrint = ptrtoint i32* %a to i64
   %maskedptr = and i64 %ptrint, 31
@@ -22,12 +28,17 @@ entry:
 }
 
 define i32 @foo2(i32* %a) #0 {
+; CHECK-LABEL: @foo2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[PTRINT:%.*]] = ptrtoint i32* [[A:%.*]] to i64
+; CHECK-NEXT:    [[MASKEDPTR:%.*]] = and i64 [[PTRINT]], 31
+; CHECK-NEXT:    [[MASKCOND:%.*]] = icmp eq i64 [[MASKEDPTR]], 0
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[MASKCOND]])
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A]], align 32
+; CHECK-NEXT:    ret i32 [[TMP0]]
+;
 entry:
 ; Same check as in @foo1, but make sure it works if the assume is first too.
-; CHECK-LABEL: @foo2
-; CHECK-DAG: load i32, i32* %a, align 32
-; CHECK-DAG: call void @llvm.assume
-; CHECK: ret i32
 
   %ptrint = ptrtoint i32* %a to i64
   %maskedptr = and i64 %ptrint, 31
@@ -41,11 +52,14 @@ entry:
 declare void @llvm.assume(i1) #1
 
 define i32 @simple(i32 %a) #1 {
+; CHECK-LABEL: @simple(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[A:%.*]], 4
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    ret i32 4
+;
 entry:
 
-; CHECK-LABEL: @simple
-; CHECK: call void @llvm.assume
-; CHECK: ret i32 4
 
   %cmp = icmp eq i32 %a, 4
   tail call void @llvm.assume(i1 %cmp)
@@ -53,43 +67,51 @@ entry:
 }
 
 define i32 @can1(i1 %a, i1 %b, i1 %c) {
+; CHECK-LABEL: @can1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    call void @llvm.assume(i1 [[A:%.*]])
+; CHECK-NEXT:    call void @llvm.assume(i1 [[B:%.*]])
+; CHECK-NEXT:    call void @llvm.assume(i1 [[C:%.*]])
+; CHECK-NEXT:    ret i32 5
+;
 entry:
   %and1 = and i1 %a, %b
   %and  = and i1 %and1, %c
   tail call void @llvm.assume(i1 %and)
 
-; CHECK-LABEL: @can1
-; CHECK: call void @llvm.assume(i1 %a)
-; CHECK: call void @llvm.assume(i1 %b)
-; CHECK: call void @llvm.assume(i1 %c)
-; CHECK: ret i32
 
   ret i32 5
 }
 
 define i32 @can2(i1 %a, i1 %b, i1 %c) {
+; CHECK-LABEL: @can2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = xor i1 [[A:%.*]], true
+; CHECK-NEXT:    call void @llvm.assume(i1 [[TMP0]])
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i1 [[B:%.*]], true
+; CHECK-NEXT:    call void @llvm.assume(i1 [[TMP1]])
+; CHECK-NEXT:    ret i32 5
+;
 entry:
   %v = or i1 %a, %b
   %w = xor i1 %v, 1
   tail call void @llvm.assume(i1 %w)
 
-; CHECK-LABEL: @can2
-; CHECK: %[[V1:[^ ]+]] = xor i1 %a, true
-; CHECK: call void @llvm.assume(i1 %[[V1]])
-; CHECK: %[[V2:[^ ]+]] = xor i1 %b, true
-; CHECK: call void @llvm.assume(i1 %[[V2]])
-; CHECK: ret i32
 
   ret i32 5
 }
 
 define i32 @bar1(i32 %a) #0 {
+; CHECK-LABEL: @bar1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[A:%.*]], 7
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 1
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    ret i32 1
+;
 entry:
   %and1 = and i32 %a, 3
 
-; CHECK-LABEL: @bar1
-; CHECK: call void @llvm.assume
-; CHECK: ret i32 1
 
   %and = and i32 %a, 7
   %cmp = icmp eq i32 %and, 1
@@ -99,10 +121,14 @@ entry:
 }
 
 define i32 @bar2(i32 %a) #0 {
+; CHECK-LABEL: @bar2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[A:%.*]], 7
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 1
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    ret i32 1
+;
 entry:
-; CHECK-LABEL: @bar2
-; CHECK: call void @llvm.assume
-; CHECK: ret i32 1
 
   %and = and i32 %a, 7
   %cmp = icmp eq i32 %and, 1
@@ -113,13 +139,19 @@ entry:
 }
 
 define i32 @bar3(i32 %a, i1 %x, i1 %y) #0 {
+; CHECK-LABEL: @bar3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[X:%.*]])
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[A:%.*]], 7
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 1
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[Y:%.*]])
+; CHECK-NEXT:    ret i32 1
+;
 entry:
   %and1 = and i32 %a, 3
 
 ; Don't be fooled by other assumes around.
-; CHECK-LABEL: @bar3
-; CHECK: call void @llvm.assume
-; CHECK: ret i32 1
 
   tail call void @llvm.assume(i1 %x)
 
@@ -133,13 +165,18 @@ entry:
 }
 
 define i32 @bar4(i32 %a, i32 %b) {
+; CHECK-LABEL: @bar4(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[A:%.*]], 7
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 1
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp eq i32 [[A]], [[B:%.*]]
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[CMP2]])
+; CHECK-NEXT:    ret i32 1
+;
 entry:
   %and1 = and i32 %b, 3
 
-; CHECK-LABEL: @bar4
-; CHECK: call void @llvm.assume
-; CHECK: call void @llvm.assume
-; CHECK: ret i32 1
 
   %and = and i32 %a, 7
   %cmp = icmp eq i32 %and, 1
@@ -195,7 +232,7 @@ declare void @escape(i32* %a)
 
 define i1 @nonnull1(i32** %a) {
 ; CHECK-LABEL: @nonnull1(
-; CHECK-NEXT:    [[LOAD:%.*]] = load i32*, i32** %a, align 8, !nonnull !6
+; CHECK-NEXT:    [[LOAD:%.*]] = load i32*, i32** [[A:%.*]], align 8, !nonnull !6
 ; CHECK-NEXT:    tail call void @escape(i32* nonnull [[LOAD]])
 ; CHECK-NEXT:    ret i1 false
 ;
@@ -212,11 +249,10 @@ define i1 @nonnull1(i32** %a) {
 
 define i1 @nonnull2(i32* %a) {
 ; CHECK-LABEL: @nonnull2(
-; CHECK-NEXT:    [[LOAD:%.*]] = load i32, i32* %a, align 4
+; CHECK-NEXT:    [[LOAD:%.*]] = load i32, i32* [[A:%.*]], align 4
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[LOAD]], 0
 ; CHECK-NEXT:    tail call void @llvm.assume(i1 [[CMP]])
-; CHECK-NEXT:    [[RVAL:%.*]] = icmp eq i32 [[LOAD]], 0
-; CHECK-NEXT:    ret i1 [[RVAL]]
+; CHECK-NEXT:    ret i1 false
 ;
   %load = load i32, i32* %a
   %cmp = icmp ne i32 %load, 0
@@ -231,15 +267,15 @@ define i1 @nonnull2(i32* %a) {
 define i1 @nonnull3(i32** %a, i1 %control) {
 ; CHECK-LABEL: @nonnull3(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[LOAD:%.*]] = load i32*, i32** %a, align 8
-; CHECK-NEXT:    br i1 %control, label %taken, label %not_taken
+; CHECK-NEXT:    [[LOAD:%.*]] = load i32*, i32** [[A:%.*]], align 8
+; CHECK-NEXT:    br i1 [[CONTROL:%.*]], label [[TAKEN:%.*]], label [[NOT_TAKEN:%.*]]
 ; CHECK:       taken:
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32* [[LOAD]], null
 ; CHECK-NEXT:    tail call void @llvm.assume(i1 [[CMP]])
-; CHECK-NEXT:    [[RVAL:%.*]] = icmp eq i32* [[LOAD]], null
-; CHECK-NEXT:    ret i1 [[RVAL]]
+; CHECK-NEXT:    ret i1 false
 ; CHECK:       not_taken:
-; CHECK-NEXT:    ret i1 true
+; CHECK-NEXT:    [[RVAL_2:%.*]] = icmp sgt i32* [[LOAD]], null
+; CHECK-NEXT:    ret i1 [[RVAL_2]]
 ;
 entry:
   %load = load i32*, i32** %a
@@ -250,7 +286,8 @@ taken:
   %rval = icmp eq i32* %load, null
   ret i1 %rval
 not_taken:
-  ret i1 true
+  %rval.2 = icmp sgt i32* %load, null
+  ret i1 %rval.2
 }
 
 ; Make sure the above canonicalization does not trigger
@@ -259,12 +296,11 @@ not_taken:
 
 define i1 @nonnull4(i32** %a) {
 ; CHECK-LABEL: @nonnull4(
-; CHECK-NEXT:    [[LOAD:%.*]] = load i32*, i32** %a, align 8
+; CHECK-NEXT:    [[LOAD:%.*]] = load i32*, i32** [[A:%.*]], align 8
 ; CHECK-NEXT:    tail call void @escape(i32* [[LOAD]])
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32* [[LOAD]], null
 ; CHECK-NEXT:    tail call void @llvm.assume(i1 [[CMP]])
-; CHECK-NEXT:    [[RVAL:%.*]] = icmp eq i32* [[LOAD]], null
-; CHECK-NEXT:    ret i1 [[RVAL]]
+; CHECK-NEXT:    ret i1 false
 ;
   %load = load i32*, i32** %a
   ;; This call may throw!
@@ -274,6 +310,23 @@ define i1 @nonnull4(i32** %a) {
   %rval = icmp eq i32* %load, null
   ret i1 %rval
 }
+define i1 @nonnull5(i32** %a) {
+; CHECK-LABEL: @nonnull5(
+; CHECK-NEXT:    [[LOAD:%.*]] = load i32*, i32** [[A:%.*]], align 8
+; CHECK-NEXT:    tail call void @escape(i32* [[LOAD]])
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32* [[LOAD]], null
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    ret i1 false
+;
+  %load = load i32*, i32** %a
+  ;; This call may throw!
+  tail call void @escape(i32* %load)
+  %integral = ptrtoint i32* %load to i64
+  %cmp = icmp slt i64 %integral, 0
+  tail call void @llvm.assume(i1 %cmp) ; %load has at least highest bit set
+  %rval = icmp eq i32* %load, null
+  ret i1 %rval
+}
 
 ; PR35846 - https://bugs.llvm.org/show_bug.cgi?id=35846
 
@@ -299,12 +352,12 @@ define i32 @assumption_conflicts_with_known_bits(i32 %a, i32 %b) {
 
 define void @debug_interference(i8 %x) {
 ; CHECK-LABEL: @debug_interference(
-; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i8 [[X:%.*]], 0
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ne i8 [[X:%.*]], 0
 ; CHECK-NEXT:    tail call void @llvm.dbg.value(metadata i32 5, metadata !7, metadata !DIExpression()), !dbg !9
-; CHECK-NEXT:    tail call void @llvm.assume(i1 [[CMP1]])
+; CHECK-NEXT:    tail call void @llvm.assume(i1 false)
 ; CHECK-NEXT:    tail call void @llvm.dbg.value(metadata i32 5, metadata !7, metadata !DIExpression()), !dbg !9
 ; CHECK-NEXT:    tail call void @llvm.dbg.value(metadata i32 5, metadata !7, metadata !DIExpression()), !dbg !9
-; CHECK-NEXT:    tail call void @llvm.assume(i1 false)
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[CMP2]])
 ; CHECK-NEXT:    ret void
 ;
   %cmp1 = icmp eq i8 %x, 0
diff --git a/llvm/test/Transforms/InstCombine/assume_inevitable.ll b/llvm/test/Transforms/InstCombine/assume_inevitable.ll
index a063775b0..d04897774 100644
--- a/llvm/test/Transforms/InstCombine/assume_inevitable.ll
+++ b/llvm/test/Transforms/InstCombine/assume_inevitable.ll
@@ -8,10 +8,10 @@ define i32 @assume_inevitable(i32* %a, i32* %b, i8* %c) {
 ; CHECK-LABEL: @assume_inevitable(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[M:%.*]] = alloca i64, align 8
-; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 32
 ; CHECK-NEXT:    [[LOADRES:%.*]] = load i32, i32* [[B:%.*]], align 4
 ; CHECK-NEXT:    [[LOADRES2:%.*]] = call i32 @llvm.annotation.i32(i32 [[LOADRES]], i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str, i64 0, i64 0), i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str1, i64 0, i64 0), i32 2)
-; CHECK-NEXT:    store i32 [[LOADRES2]], i32* [[A]], align 4
+; CHECK-NEXT:    store i32 [[LOADRES2]], i32* [[A]], align 32
 ; CHECK-NEXT:    [[DUMMY_EQ:%.*]] = icmp ugt i32 [[LOADRES]], 42
 ; CHECK-NEXT:    tail call void @llvm.assume(i1 [[DUMMY_EQ]])
 ; CHECK-NEXT:    [[M_I8:%.*]] = bitcast i64* [[M]] to i8*
diff --git a/llvm/test/Transforms/InstCombine/bcopy.ll b/llvm/test/Transforms/InstCombine/bcopy.ll
new file mode 100644
index 000000000..6a53bad7e
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/bcopy.ll
@@ -0,0 +1,25 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+declare void @bcopy(i8* nocapture readonly, i8* nocapture, i32)
+
+define void @bcopy_memmove(i8* nocapture readonly %a, i8* nocapture %b) {
+; CHECK-LABEL: @bcopy_memmove(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8* [[A:%.*]] to i64*
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8* [[B:%.*]] to i64*
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, i64* [[TMP1]], align 1
+; CHECK-NEXT:    store i64 [[TMP3]], i64* [[TMP2]], align 1
+; CHECK-NEXT:    ret void
+;
+  tail call void @bcopy(i8* %a, i8* %b, i32 8)
+  ret void
+}
+
+define void @bcopy_memmove2(i8* nocapture readonly %a, i8* nocapture %b, i32 %len) {
+; CHECK-LABEL: @bcopy_memmove2(
+; CHECK-NEXT:    call void @llvm.memmove.p0i8.p0i8.i32(i8* align 1 [[B:%.*]], i8* align 1 [[A:%.*]], i32 [[LEN:%.*]], i1 false)
+; CHECK-NEXT:    ret void
+;
+  tail call void @bcopy(i8* %a, i8* %b, i32 %len)
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/bitcast-function.ll b/llvm/test/Transforms/InstCombine/bitcast-function.ll
new file mode 100644
index 000000000..ca82165a4
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/bitcast-function.ll
@@ -0,0 +1,206 @@
+; RUN: opt -S -instcombine -o - %s | FileCheck %s
+target datalayout = "e-p:32:32:32-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v16:16:16-v24:32:32-v32:32:32-v64:64:64-v128:128:128-a0:0:64"
+
+define internal <2 x i32> @func_v2i32(<2 x i32> %v) noinline nounwind {
+entry:
+  ret <2 x i32> %v
+}
+
+define internal <2 x float> @func_v2f32(<2 x float> %v) noinline nounwind {
+entry:
+  ret <2 x float> %v
+}
+
+define internal <4 x float> @func_v4f32(<4 x float> %v) noinline nounwind {
+entry:
+  ret <4 x float> %v
+}
+
+define internal i32 @func_i32(i32 %v) noinline nounwind {
+entry:
+  ret i32 %v
+}
+
+define internal i64 @func_i64(i64 %v) noinline nounwind {
+entry:
+  ret i64 %v
+}
+
+define internal <2 x i64> @func_v2i64(<2 x i64> %v) noinline nounwind {
+entry:
+  ret <2 x i64> %v
+}
+
+define internal <2 x i32*> @func_v2i32p(<2 x i32*> %v) noinline nounwind {
+entry:
+  ret <2 x i32*> %v
+}
+
+; Valid cases, only bitcast for argument / return type and call underlying function
+
+; Test cast between scalars with same bit sizes
+; Sizes match, should only bitcast
+define void @bitcast_scalar(float* noalias %source, float* noalias %dest) nounwind {
+entry:
+; CHECK-LABEL: @bitcast_scalar
+; CHECK: bitcast float* %source to i32*
+; CHECK: load i32, i32*
+; CHECK-NOT: fptoui
+; CHECK-NOT: uitofp
+; CHECK: bitcast float* %dest to i32*
+; CHECK: store i32
+  %tmp = load float, float* %source, align 8
+  %call = call float bitcast (i32 (i32)* @func_i32 to float (float)*)(float %tmp) nounwind
+  store float %call, float* %dest, align 8
+  ret void
+}
+
+; Test cast between vectors with same number of elements and bit sizes
+; Sizes match, should only bitcast
+define void @bitcast_vector(<2 x float>* noalias %source, <2 x float>* noalias %dest) nounwind {
+entry:
+; CHECK-LABEL: @bitcast_vector
+; CHECK: bitcast <2 x float>* %source to <2 x i32>*
+; CHECK: load <2 x i32>, <2 x i32>*
+; CHECK-NOT: fptoui
+; CHECK-NOT: uitofp
+; CHECK: bitcast <2 x float>* %dest to <2 x i32>*
+; CHECK: store <2 x i32>
+  %tmp = load <2 x float>, <2 x float>* %source, align 8
+  %call = call <2 x float> bitcast (<2 x i32> (<2 x i32>)* @func_v2i32 to <2 x float> (<2 x float>)*)(<2 x float> %tmp) nounwind
+  store <2 x float> %call, <2 x float>* %dest, align 8
+  ret void
+}
+
+; Test cast from vector to scalar with same number of bits
+; Sizes match, should only bitcast
+define void @bitcast_vector_scalar_same_size(<2 x float>* noalias %source, <2 x float>* noalias %dest) nounwind {
+entry:
+; CHECK-LABEL: @bitcast_vector_scalar_same_size
+; CHECK: bitcast <2 x float>* %source to i64*
+; CHECK: load i64, i64*
+; CHECK: %call = call i64 @func_i64
+; CHECK: bitcast <2 x float>* %dest to i64*
+; CHECK: store i64
+  %tmp = load <2 x float>, <2 x float>* %source, align 8
+  %call = call <2 x float> bitcast (i64 (i64)* @func_i64 to <2 x float> (<2 x float>)*)(<2 x float> %tmp) nounwind
+  store <2 x float> %call, <2 x float>* %dest, align 8
+  ret void
+}
+
+; Test cast from scalar to vector with same number of bits
+define void @bitcast_scalar_vector_same_size(i64* noalias %source, i64* noalias %dest) nounwind {
+entry:
+; CHECK-LABEL: @bitcast_scalar_vector_same_size
+; CHECK: bitcast i64* %source to <2 x float>*
+; CHECK: load <2 x float>, <2 x float>*
+; CHECK: call <2 x float> @func_v2f32
+; CHECK: bitcast i64* %dest to <2 x float>*
+; CHECK: store <2 x float>
+  %tmp = load i64, i64* %source, align 8
+  %call = call i64 bitcast (<2 x float> (<2 x float>)* @func_v2f32 to i64 (i64)*)(i64 %tmp) nounwind
+  store i64 %call, i64* %dest, align 8
+  ret void
+}
+
+; Test cast between vectors of pointers
+define void @bitcast_vector_ptrs_same_size(<2 x i64*>* noalias %source, <2 x i64*>* noalias %dest) nounwind {
+entry:
+; CHECK-LABEL: @bitcast_vector_ptrs_same_size
+; CHECK: bitcast <2 x i64*>* %source to <2 x i32*>*
+; CHECK: load <2 x i32*>, <2 x i32*>*
+; CHECK: call <2 x i32*> @func_v2i32p
+; CHECK: bitcast <2 x i64*>* %dest to <2 x i32*>*
+; CHECK: store <2 x i32*>
+  %tmp = load <2 x i64*>, <2 x i64*>* %source, align 8
+  %call = call <2 x i64*> bitcast (<2 x i32*> (<2 x i32*>)* @func_v2i32p to <2 x i64*> (<2 x i64*>)*)(<2 x i64*> %tmp) nounwind
+  store <2 x i64*> %call, <2 x i64*>* %dest, align 8
+  ret void
+}
+
+; Invalid cases:
+
+; Test cast between scalars with different bit sizes
+define void @bitcast_mismatch_scalar_size(float* noalias %source, float* noalias %dest) nounwind {
+entry:
+; CHECK-LABEL: @bitcast_mismatch_scalar_size
+; CHECK-NOT: fptoui
+; CHECK: call float bitcast
+; CHECK-NOT: uitofp
+  %tmp = load float, float* %source, align 8
+  %call = call float bitcast (i64 (i64)* @func_i64 to float (float)*)(float %tmp) nounwind
+  store float %call, float* %dest, align 8
+  ret void
+}
+
+; Test cast between vectors with different bit sizes but the
+; same number of elements
+define void @bitcast_mismatch_vector_element_and_bit_size(<2 x float>* noalias %source, <2 x float>* noalias %dest) nounwind {
+entry:
+; CHECK-LABEL: @bitcast_mismatch_vector_element_and_bit_size
+; CHECK-NOT: fptoui <2 x float> %tmp to <2 x i64>
+; CHECK: call <2 x float> bitcast
+; CHECK-NOT: uitofp <2 x i64> %call to <2 x float>
+  %tmp = load <2 x float>, <2 x float>* %source, align 8
+  %call = call <2 x float> bitcast (<2 x i64> (<2 x i64>)* @func_v2i64 to <2 x float> (<2 x float>)*)(<2 x float> %tmp) nounwind
+  store <2 x float> %call, <2 x float>* %dest, align 8
+  ret void
+}
+
+; Test cast between vectors with same number of bits and different
+; numbers of elements
+define void @bitcast_vector_mismatched_number_elements(<4 x float>* noalias %source, <4 x float>* noalias %dest) nounwind {
+entry:
+; CHECK-LABEL: @bitcast_vector_mismatched_number_elements
+; CHECK:  %call = call <4 x float> bitcast
+  %tmp = load <4 x float>, <4 x float>* %source, align 8
+  %call = call <4 x float> bitcast (<2 x i32> (<2 x i32>)* @func_v2i32 to <4 x float> (<4 x float>)*)(<4 x float> %tmp) nounwind
+  store <4 x float> %call, <4 x float>* %dest, align 8
+  ret void
+}
+
+; Test cast between vector and scalar with different number of bits
+define void @bitcast_vector_scalar_mismatched_bit_size(<4 x float>* noalias %source, <4 x float>* noalias %dest) nounwind {
+entry:
+; CHECK-LABEL: @bitcast_vector_scalar_mismatched_bit_size
+; CHECK:  %call = call <4 x float> bitcast
+  %tmp = load <4 x float>, <4 x float>* %source, align 8
+  %call = call <4 x float> bitcast (i64 (i64)* @func_i64 to <4 x float> (<4 x float>)*)(<4 x float> %tmp) nounwind
+  store <4 x float> %call, <4 x float>* %dest, align 8
+  ret void
+}
+
+; Test cast between vector of pointers and scalar with different number of bits
+define void @bitcast_vector_ptrs_scalar_mismatched_bit_size(<4 x i32*>* noalias %source, <4 x i32*>* noalias %dest) nounwind {
+entry:
+; CHECK-LABEL: @bitcast_vector_ptrs_scalar_mismatched_bit_size
+; CHECK: call <4 x i32*> bitcast
+  %tmp = load <4 x i32*>, <4 x i32*>* %source, align 8
+  %call = call <4 x i32*> bitcast (i64 (i64)* @func_i64 to <4 x i32*> (<4 x i32*>)*)(<4 x i32*> %tmp) nounwind
+  store <4 x i32*> %call, <4 x i32*>* %dest, align 8
+  ret void
+}
+
+; Test cast from scalar to vector of pointers with same number of bits
+; We don't know the pointer size at this point, so this can't be done
+define void @bitcast_scalar_vector_ptrs_same_size(i64* noalias %source, i64* noalias %dest) nounwind {
+entry:
+; CHECK-LABEL: @bitcast_scalar_vector_ptrs_same_size
+; CHECK: call i64 bitcast
+  %tmp = load i64, i64* %source, align 8
+  %call = call i64 bitcast (<2 x i32*> (<2 x i32*>)* @func_v2i32p to i64 (i64)*)(i64 %tmp) nounwind
+  store i64 %call, i64* %dest, align 8
+  ret void
+}
+
+; Test cast between scalar and vector with different number of bits
+define void @bitcast_scalar_vector_mismatched_bit_size(i64* noalias %source, i64* noalias %dest) nounwind {
+entry:
+; CHECK-LABEL: @bitcast_scalar_vector_mismatched_bit_size
+; CHECK: call i64 bitcast
+  %tmp = load i64, i64* %source, align 8
+  %call = call i64 bitcast (<4 x float> (<4 x float>)* @func_v4f32 to i64 (i64)*)(i64 %tmp) nounwind
+  store i64 %call, i64* %dest, align 8
+  ret void
+}
+
diff --git a/llvm/test/Transforms/InstCombine/bitcast-phi-uselistorder.ll b/llvm/test/Transforms/InstCombine/bitcast-phi-uselistorder.ll
new file mode 100644
index 000000000..d5489484b
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/bitcast-phi-uselistorder.ll
@@ -0,0 +1,33 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+@Q = internal unnamed_addr global double 1.000000e+00, align 8
+
+define double @test(i1 %c, i64* %p) {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[IF:%.*]], label [[END:%.*]]
+; CHECK:       if:
+; CHECK-NEXT:    [[LOAD1:%.*]] = load double, double* @Q, align 8
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    [[TMP0:%.*]] = phi double [ 0.000000e+00, [[ENTRY:%.*]] ], [ [[LOAD1]], [[IF]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i64* [[P:%.*]] to double*
+; CHECK-NEXT:    store double [[TMP0]], double* [[TMP1]], align 8
+; CHECK-NEXT:    ret double [[TMP0]]
+;
+entry:
+  br i1 %c, label %if, label %end
+
+if:
+  %load = load i64, i64* bitcast (double* @Q to i64*), align 8
+  br label %end
+
+end:
+  %phi = phi i64 [ 0, %entry ], [ %load, %if ]
+  store i64 %phi, i64* %p, align 8
+  %cast = bitcast i64 %phi to double
+  ret double %cast
+
+  uselistorder i64 %phi, { 1, 0 }
+}
diff --git a/llvm/test/Transforms/InstCombine/bitcast-vec-canon.ll b/llvm/test/Transforms/InstCombine/bitcast-vec-canon.ll
index a92a7b73f..50cbd7639 100644
--- a/llvm/test/Transforms/InstCombine/bitcast-vec-canon.ll
+++ b/llvm/test/Transforms/InstCombine/bitcast-vec-canon.ll
@@ -38,3 +38,34 @@ define <1 x i64> @d(i64 %y) {
   ret <1 x i64> %c
 }
 
+define x86_mmx @e(<1 x i64> %y) {
+; CHECK-LABEL: @e(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <1 x i64> %y, i32 0
+; CHECK-NEXT:    [[C:%.*]] = bitcast i64 [[TMP1]] to x86_mmx
+; CHECK-NEXT:    ret x86_mmx [[C]]
+;
+  %c = bitcast <1 x i64> %y to x86_mmx
+  ret x86_mmx %c
+}
+
+define <1 x i64> @f(x86_mmx %y) {
+; CHECK-LABEL: @f(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast x86_mmx %y to i64
+; CHECK-NEXT:    [[C:%.*]] = insertelement <1 x i64> undef, i64 [[TMP1]], i32 0
+; CHECK-NEXT:    ret <1 x i64> [[C]]
+;
+  %c = bitcast x86_mmx %y to <1 x i64>
+  ret <1 x i64> %c
+}
+
+define double @g(x86_mmx %x) {
+; CHECK-LABEL: @g(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast x86_mmx %x to double
+; CHECK-NEXT:    ret double [[TMP0]]
+;
+entry:
+  %0 = bitcast x86_mmx %x to <1 x i64>
+  %1 = bitcast <1 x i64> %0 to double
+  ret double %1
+}
diff --git a/llvm/test/Transforms/InstCombine/bitreverse-hang.ll b/llvm/test/Transforms/InstCombine/bitreverse-hang.ll
index 8e6585e99..0c175759b 100644
--- a/llvm/test/Transforms/InstCombine/bitreverse-hang.ll
+++ b/llvm/test/Transforms/InstCombine/bitreverse-hang.ll
@@ -40,7 +40,7 @@ for.end:                                          ; preds = %for.body
   ret i32 undef
 }
 
-attributes #0 = { norecurse nounwind ssp uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="core2" "target-features"="+cx16,+fxsr,+mmx,+sse,+sse2,+sse3,+ssse3" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { norecurse nounwind ssp uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="core2" "target-features"="+cx16,+fxsr,+mmx,+sse,+sse2,+sse3,+ssse3" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.module.flags = !{!0}
 !llvm.ident = !{!1}
diff --git a/llvm/test/Transforms/InstCombine/bswap.ll b/llvm/test/Transforms/InstCombine/bswap.ll
index 69be38df4..540b1752c 100644
--- a/llvm/test/Transforms/InstCombine/bswap.ll
+++ b/llvm/test/Transforms/InstCombine/bswap.ll
@@ -1,10 +1,11 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -instcombine -S | FileCheck %s
 
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128-n8:16:32"
 
 define i32 @test1(i32 %i) {
 ; CHECK-LABEL: @test1(
-; CHECK-NEXT:    [[TMP12:%.*]] = call i32 @llvm.bswap.i32(i32 %i)
+; CHECK-NEXT:    [[TMP12:%.*]] = call i32 @llvm.bswap.i32(i32 [[I:%.*]])
 ; CHECK-NEXT:    ret i32 [[TMP12]]
 ;
   %tmp1 = lshr i32 %i, 24
@@ -21,7 +22,7 @@ define i32 @test1(i32 %i) {
 
 define i32 @test2(i32 %arg) {
 ; CHECK-LABEL: @test2(
-; CHECK-NEXT:    [[TMP14:%.*]] = call i32 @llvm.bswap.i32(i32 %arg)
+; CHECK-NEXT:    [[TMP14:%.*]] = call i32 @llvm.bswap.i32(i32 [[ARG:%.*]])
 ; CHECK-NEXT:    ret i32 [[TMP14]]
 ;
   %tmp2 = shl i32 %arg, 24
@@ -38,7 +39,7 @@ define i32 @test2(i32 %arg) {
 
 define i16 @test3(i16 %s) {
 ; CHECK-LABEL: @test3(
-; CHECK-NEXT:    [[TMP5:%.*]] = call i16 @llvm.bswap.i16(i16 %s)
+; CHECK-NEXT:    [[TMP5:%.*]] = call i16 @llvm.bswap.i16(i16 [[S:%.*]])
 ; CHECK-NEXT:    ret i16 [[TMP5]]
 ;
   %tmp2 = lshr i16 %s, 8
@@ -49,7 +50,7 @@ define i16 @test3(i16 %s) {
 
 define i16 @test4(i16 %s) {
 ; CHECK-LABEL: @test4(
-; CHECK-NEXT:    [[TMP5:%.*]] = call i16 @llvm.bswap.i16(i16 %s)
+; CHECK-NEXT:    [[TMP5:%.*]] = call i16 @llvm.bswap.i16(i16 [[S:%.*]])
 ; CHECK-NEXT:    ret i16 [[TMP5]]
 ;
   %tmp2 = lshr i16 %s, 8
@@ -60,7 +61,7 @@ define i16 @test4(i16 %s) {
 
 define i16 @test5(i16 %a) {
 ; CHECK-LABEL: @test5(
-; CHECK-NEXT:    [[TMP_UPGRD_3:%.*]] = call i16 @llvm.bswap.i16(i16 %a)
+; CHECK-NEXT:    [[TMP_UPGRD_3:%.*]] = call i16 @llvm.bswap.i16(i16 [[A:%.*]])
 ; CHECK-NEXT:    ret i16 [[TMP_UPGRD_3]]
 ;
   %tmp = zext i16 %a to i32
@@ -80,7 +81,7 @@ define i16 @test5(i16 %a) {
 ; PR2842
 define i32 @test6(i32 %x) nounwind readnone {
 ; CHECK-LABEL: @test6(
-; CHECK-NEXT:    [[TMP7:%.*]] = call i32 @llvm.bswap.i32(i32 %x)
+; CHECK-NEXT:    [[TMP7:%.*]] = call i32 @llvm.bswap.i32(i32 [[X:%.*]])
 ; CHECK-NEXT:    ret i32 [[TMP7]]
 ;
   %tmp = shl i32 %x, 16
@@ -102,7 +103,7 @@ declare void @extra_use(i32)
 
 define i32 @bswap32_and_first(i32 %x) {
 ; CHECK-LABEL: @bswap32_and_first(
-; CHECK-NEXT:    [[BSWAP:%.*]] = call i32 @llvm.bswap.i32(i32 %x)
+; CHECK-NEXT:    [[BSWAP:%.*]] = call i32 @llvm.bswap.i32(i32 [[X:%.*]])
 ; CHECK-NEXT:    ret i32 [[BSWAP]]
 ;
   %shl = shl i32 %x, 16
@@ -122,11 +123,11 @@ define i32 @bswap32_and_first(i32 %x) {
 
 define i32 @bswap32_and_first_extra_use(i32 %x) {
 ; CHECK-LABEL: @bswap32_and_first_extra_use(
-; CHECK-NEXT:    [[SHL:%.*]] = shl i32 %x, 16
-; CHECK-NEXT:    [[SHR:%.*]] = lshr i32 %x, 16
+; CHECK-NEXT:    [[SHL:%.*]] = shl i32 [[X:%.*]], 16
+; CHECK-NEXT:    [[SHR:%.*]] = lshr i32 [[X]], 16
 ; CHECK-NEXT:    [[SWAPHALF:%.*]] = or i32 [[SHL]], [[SHR]]
 ; CHECK-NEXT:    [[T:%.*]] = and i32 [[SWAPHALF]], 16711935
-; CHECK-NEXT:    [[BSWAP:%.*]] = call i32 @llvm.bswap.i32(i32 %x)
+; CHECK-NEXT:    [[BSWAP:%.*]] = call i32 @llvm.bswap.i32(i32 [[X]])
 ; CHECK-NEXT:    call void @extra_use(i32 [[T]])
 ; CHECK-NEXT:    ret i32 [[BSWAP]]
 ;
@@ -148,7 +149,7 @@ define i32 @bswap32_and_first_extra_use(i32 %x) {
 ; PR23863
 define i32 @bswap32_shl_first(i32 %x) {
 ; CHECK-LABEL: @bswap32_shl_first(
-; CHECK-NEXT:    [[BSWAP:%.*]] = call i32 @llvm.bswap.i32(i32 %x)
+; CHECK-NEXT:    [[BSWAP:%.*]] = call i32 @llvm.bswap.i32(i32 [[X:%.*]])
 ; CHECK-NEXT:    ret i32 [[BSWAP]]
 ;
   %shl = shl i32 %x, 16
@@ -168,11 +169,11 @@ define i32 @bswap32_shl_first(i32 %x) {
 
 define i32 @bswap32_shl_first_extra_use(i32 %x) {
 ; CHECK-LABEL: @bswap32_shl_first_extra_use(
-; CHECK-NEXT:    [[SHL:%.*]] = shl i32 %x, 16
-; CHECK-NEXT:    [[SHR:%.*]] = lshr i32 %x, 16
-; CHECK-NEXT:    [[SWAPHALF:%.*]] = or i32 [[SHL]], [[SHR]]
-; CHECK-NEXT:    [[T:%.*]] = shl i32 [[SWAPHALF]], 8
-; CHECK-NEXT:    [[BSWAP:%.*]] = call i32 @llvm.bswap.i32(i32 %x)
+; CHECK-NEXT:    [[SHR:%.*]] = lshr i32 [[X:%.*]], 16
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 [[X]], 24
+; CHECK-NEXT:    [[TMP2:%.*]] = shl nuw nsw i32 [[SHR]], 8
+; CHECK-NEXT:    [[T:%.*]] = or i32 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[BSWAP:%.*]] = call i32 @llvm.bswap.i32(i32 [[X]])
 ; CHECK-NEXT:    call void @extra_use(i32 [[T]])
 ; CHECK-NEXT:    ret i32 [[BSWAP]]
 ;
@@ -190,7 +191,7 @@ define i32 @bswap32_shl_first_extra_use(i32 %x) {
 
 define i16 @test8(i16 %a) {
 ; CHECK-LABEL: @test8(
-; CHECK-NEXT:    [[REV:%.*]] = call i16 @llvm.bswap.i16(i16 %a)
+; CHECK-NEXT:    [[REV:%.*]] = call i16 @llvm.bswap.i16(i16 [[A:%.*]])
 ; CHECK-NEXT:    ret i16 [[REV]]
 ;
   %conv = zext i16 %a to i32
@@ -204,7 +205,7 @@ define i16 @test8(i16 %a) {
 
 define i16 @test9(i16 %a) {
 ; CHECK-LABEL: @test9(
-; CHECK-NEXT:    [[REV:%.*]] = call i16 @llvm.bswap.i16(i16 %a)
+; CHECK-NEXT:    [[REV:%.*]] = call i16 @llvm.bswap.i16(i16 [[A:%.*]])
 ; CHECK-NEXT:    ret i16 [[REV]]
 ;
   %conv = zext i16 %a to i32
@@ -217,7 +218,7 @@ define i16 @test9(i16 %a) {
 
 define i16 @test10(i32 %a) {
 ; CHECK-LABEL: @test10(
-; CHECK-NEXT:    [[TRUNC:%.*]] = trunc i32 %a to i16
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc i32 [[A:%.*]] to i16
 ; CHECK-NEXT:    [[REV:%.*]] = call i16 @llvm.bswap.i16(i16 [[TRUNC]])
 ; CHECK-NEXT:    ret i16 [[REV]]
 ;
@@ -230,3 +231,79 @@ define i16 @test10(i32 %a) {
   ret i16 %conv
 }
 
+define i32 @shuf_4bytes(<4 x i8> %x) {
+; CHECK-LABEL: @shuf_4bytes(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i8> [[X:%.*]] to i32
+; CHECK-NEXT:    [[CAST:%.*]] = call i32 @llvm.bswap.i32(i32 [[TMP1]])
+; CHECK-NEXT:    ret i32 [[CAST]]
+;
+  %bswap = shufflevector <4 x i8> %x, <4 x i8> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %cast = bitcast <4 x i8> %bswap to i32
+  ret i32 %cast
+}
+
+define i32 @shuf_load_4bytes(<4 x i8>* %p) {
+; CHECK-LABEL: @shuf_load_4bytes(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i8>* [[P:%.*]] to i32*
+; CHECK-NEXT:    [[X1:%.*]] = load i32, i32* [[TMP1]], align 4
+; CHECK-NEXT:    [[CAST:%.*]] = call i32 @llvm.bswap.i32(i32 [[X1]])
+; CHECK-NEXT:    ret i32 [[CAST]]
+;
+  %x = load <4 x i8>, <4 x i8>* %p
+  %bswap = shufflevector <4 x i8> %x, <4 x i8> undef, <4 x i32> <i32 3, i32 2, i32 undef, i32 0>
+  %cast = bitcast <4 x i8> %bswap to i32
+  ret i32 %cast
+}
+
+define i32 @shuf_bitcast_twice_4bytes(i32 %x) {
+; CHECK-LABEL: @shuf_bitcast_twice_4bytes(
+; CHECK-NEXT:    [[CAST2:%.*]] = call i32 @llvm.bswap.i32(i32 [[X:%.*]])
+; CHECK-NEXT:    ret i32 [[CAST2]]
+;
+  %cast1 = bitcast i32 %x to <4 x i8>
+  %bswap = shufflevector <4 x i8> %cast1, <4 x i8> undef, <4 x i32> <i32 undef, i32 2, i32 1, i32 0>
+  %cast2 = bitcast <4 x i8> %bswap to i32
+  ret i32 %cast2
+}
+
+; Negative test - extra use
+declare void @use(<4 x i8>)
+
+define i32 @shuf_4bytes_extra_use(<4 x i8> %x) {
+; CHECK-LABEL: @shuf_4bytes_extra_use(
+; CHECK-NEXT:    [[BSWAP:%.*]] = shufflevector <4 x i8> [[X:%.*]], <4 x i8> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    call void @use(<4 x i8> [[BSWAP]])
+; CHECK-NEXT:    [[CAST:%.*]] = bitcast <4 x i8> [[BSWAP]] to i32
+; CHECK-NEXT:    ret i32 [[CAST]]
+;
+  %bswap = shufflevector <4 x i8> %x, <4 x i8> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  call void @use(<4 x i8> %bswap)
+  %cast = bitcast <4 x i8> %bswap to i32
+  ret i32 %cast
+}
+
+; Negative test - scalar type is not in the data layout
+
+define i128 @shuf_16bytes(<16 x i8> %x) {
+; CHECK-LABEL: @shuf_16bytes(
+; CHECK-NEXT:    [[BSWAP:%.*]] = shufflevector <16 x i8> [[X:%.*]], <16 x i8> undef, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[CAST:%.*]] = bitcast <16 x i8> [[BSWAP]] to i128
+; CHECK-NEXT:    ret i128 [[CAST]]
+;
+  %bswap = shufflevector <16 x i8> %x, <16 x i8> undef, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  %cast = bitcast <16 x i8> %bswap to i128
+  ret i128 %cast
+}
+
+; Negative test - don't touch widening shuffles (for now)
+
+define i32 @shuf_2bytes_widening(<2 x i8> %x) {
+; CHECK-LABEL: @shuf_2bytes_widening(
+; CHECK-NEXT:    [[BSWAP:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> undef, <4 x i32> <i32 1, i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[CAST:%.*]] = bitcast <4 x i8> [[BSWAP]] to i32
+; CHECK-NEXT:    ret i32 [[CAST]]
+;
+  %bswap = shufflevector <2 x i8> %x, <2 x i8> undef, <4 x i32> <i32 1, i32 0, i32 undef, i32 undef>
+  %cast = bitcast <4 x i8> %bswap to i32
+  ret i32 %cast
+}
diff --git a/llvm/test/Transforms/InstCombine/builtin-dynamic-object-size.ll b/llvm/test/Transforms/InstCombine/builtin-dynamic-object-size.ll
index eabe3a4c4..4093a1210 100644
--- a/llvm/test/Transforms/InstCombine/builtin-dynamic-object-size.ll
+++ b/llvm/test/Transforms/InstCombine/builtin-dynamic-object-size.ll
@@ -1,4 +1,4 @@
-; RUN: opt -instcombine -S < %s | FileCheck %s --dump-input-on-failure
+; RUN: opt -instcombine -S < %s | FileCheck %s
 
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.14.0"
@@ -48,10 +48,8 @@ entry:
 
 ; CHECK:      define i64 @internal_pointer(i64 %sz)
 ; CHECK-NEXT: entry:
-; CHECK-NEXT:   %0 = add i64 %sz, -2
-; CHECK-NEXT:   %1 = icmp ult i64 %sz, 2
-; CHECK-NEXT:   %2 = select i1 %1, i64 0, i64 %0
-; CHECK-NEXT:   ret i64 %2
+; CHECK-NEXT:   %0 = call i64 @llvm.usub.sat.i64(i64 %sz, i64 2)
+; CHECK-NEXT:   ret i64 %0
 ; CHECK-NEXT: }
 
 define i64 @uses_nullptr_no_fold() {
diff --git a/llvm/test/Transforms/InstCombine/builtin-object-size-custom-dl.ll b/llvm/test/Transforms/InstCombine/builtin-object-size-custom-dl.ll
new file mode 100644
index 000000000..eec0f7684
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/builtin-object-size-custom-dl.ll
@@ -0,0 +1,32 @@
+; RUN: opt -instcombine -S < %s | FileCheck %s
+target datalayout = "e-m:o-p:40:64:64:32-i64:64-f80:128-n8:16:32:64-S128"
+
+; check that memory builtins can be handled.
+define i64 @objsize1_custom_idx(i64 %sz) {
+entry:
+  %ptr = call i8* @malloc(i64 %sz)
+  %ptr2 = getelementptr inbounds i8, i8* %ptr, i32 2
+  %calc_size = call i64 @llvm.objectsize.i64.p0i8(i8* %ptr2, i1 false, i1 true, i1 true)
+  ret i64 %calc_size
+}
+
+%struct.V = type { [10 x i8], i32, [10 x i8] }
+
+define i32 @objsize2_custom_idx() #0 {
+entry:
+  %var = alloca %struct.V, align 4
+  %0 = bitcast %struct.V* %var to i8*
+  call void @llvm.lifetime.start.p0i8(i64 28, i8* %0) #3
+  %buf1 = getelementptr inbounds %struct.V, %struct.V* %var, i32 0, i32 0
+  %arrayidx = getelementptr inbounds [10 x i8], [10 x i8]* %buf1, i64 0, i64 1
+  %1 = call i64 @llvm.objectsize.i64.p0i8(i8* %arrayidx, i1 false, i1 false, i1 false)
+  %conv = trunc i64 %1 to i32
+  call void @llvm.lifetime.end.p0i8(i64 28, i8* %0) #3
+  ret i32 %conv
+; CHECK: ret i32 27
+}
+
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture) #1
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture) #1
+declare i8* @malloc(i64)
+declare i64 @llvm.objectsize.i64.p0i8(i8*, i1, i1, i1)
diff --git a/llvm/test/Transforms/InstCombine/builtin-object-size-ptr.ll b/llvm/test/Transforms/InstCombine/builtin-object-size-ptr.ll
index ada3fc167..7747aea11 100644
--- a/llvm/test/Transforms/InstCombine/builtin-object-size-ptr.ll
+++ b/llvm/test/Transforms/InstCombine/builtin-object-size-ptr.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -instcombine -S < %s | FileCheck %s
 
 ; int foo() {
@@ -13,22 +14,62 @@
 %struct.V = type { [10 x i8], i32, [10 x i8] }
 
 define i32 @foo() #0 {
-entry:
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:    ret i32 27
+;
   %var = alloca %struct.V, align 4
-  %0 = bitcast %struct.V* %var to i8*
-  call void @llvm.lifetime.start.p0i8(i64 28, i8* %0) #3
+  %t0 = bitcast %struct.V* %var to i8*
+  call void @llvm.lifetime.start.p0i8(i64 28, i8* %t0) #3
   %buf1 = getelementptr inbounds %struct.V, %struct.V* %var, i32 0, i32 0
   %arrayidx = getelementptr inbounds [10 x i8], [10 x i8]* %buf1, i64 0, i64 1
-  %1 = call i64 @llvm.objectsize.i64.p0i8(i8* %arrayidx, i1 false)
-  %conv = trunc i64 %1 to i32
-  call void @llvm.lifetime.end.p0i8(i64 28, i8* %0) #3
+  %t1 = call i64 @llvm.objectsize.i64.p0i8(i8* %arrayidx, i1 false)
+  %conv = trunc i64 %t1 to i32
+  call void @llvm.lifetime.end.p0i8(i64 28, i8* %t0) #3
   ret i32 %conv
-; CHECK: ret i32 27
-; CHECK-NOT: ret i32 -1
 }
 
-declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture) #1
+; This used to crash while erasing instructions:
+; https://bugs.llvm.org/show_bug.cgi?id=43723
 
-declare i64 @llvm.objectsize.i64.p0i8(i8*, i1) #2
+define void @PR43723() {
+; CHECK-LABEL: @PR43723(
+; CHECK-NEXT:    ret void
+;
+  %tab = alloca [10 x i8], align 16
+  %t0 = bitcast [10 x i8]* %tab to i8*
+  call void @llvm.memset.p0i8.i64(i8* align 16 %t0, i8 9, i64 10, i1 false)
+  %t1 = call {}* @llvm.invariant.start.p0i8(i64 10, i8* align 16 %t0)
+  call void @llvm.invariant.end.p0i8({}* %t1, i64 10, i8* align 16 %t0)
+  ret void
+
+  uselistorder i8* %t0, { 1, 0, 2 }
+}
+
+define void @unknown_use_of_invariant_start({}** %p) {
+; CHECK-LABEL: @unknown_use_of_invariant_start(
+; CHECK-NEXT:    ret void
+;
+  %tab = alloca [10 x i8], align 16
+  %t0 = bitcast [10 x i8]* %tab to i8*
+  call void @llvm.memset.p0i8.i64(i8* align 16 %t0, i8 9, i64 10, i1 false)
+  %t1 = call {}* @llvm.invariant.start.p0i8(i64 10, i8* align 16 %t0)
+  call void @llvm.invariant.end.p0i8({}* %t1, i64 10, i8* align 16 %t0)
+  store {}* %t1, {}** %p
+  ret void
+}
 
+define {}* @minimal_invariant_start_use(i8 %x) {
+; CHECK-LABEL: @minimal_invariant_start_use(
+; CHECK-NEXT:    ret {}* undef
+;
+  %a = alloca i8
+  %i = call {}* @llvm.invariant.start.p0i8(i64 1, i8* %a)
+  ret {}* %i
+}
+
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture) #1
+declare i64 @llvm.objectsize.i64.p0i8(i8*, i1) #2
 declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture) #1
+declare void @llvm.memset.p0i8.i64(i8* nocapture writeonly, i8, i64, i1 immarg) #0
+declare {}* @llvm.invariant.start.p0i8(i64 immarg, i8* nocapture) #0
+declare void @llvm.invariant.end.p0i8({}*, i64 immarg, i8* nocapture) #0
diff --git a/llvm/test/Transforms/InstCombine/call-guard.ll b/llvm/test/Transforms/InstCombine/call-guard.ll
index 8101f4571..241b0788a 100644
--- a/llvm/test/Transforms/InstCombine/call-guard.ll
+++ b/llvm/test/Transforms/InstCombine/call-guard.ll
@@ -1,4 +1,5 @@
-; RUN: opt < %s -instcombine -S | FileCheck %s
+; RUN: opt < %s -instcombine -instcombine-infinite-loop-threshold=2 -S | FileCheck %s
+; RUN: opt < %s -instcombine -S -debugify-each | FileCheck %s
 
 declare void @llvm.experimental.guard(i1, ...)
 
diff --git a/llvm/test/Transforms/InstCombine/canonicalize-clamp-like-pattern-between-negative-and-positive-thresholds.ll b/llvm/test/Transforms/InstCombine/canonicalize-clamp-like-pattern-between-negative-and-positive-thresholds.ll
new file mode 100644
index 000000000..7daac62d5
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/canonicalize-clamp-like-pattern-between-negative-and-positive-thresholds.ll
@@ -0,0 +1,479 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; Given a pattern like:
+;   %old_cmp1 = icmp slt i32 %x, C2
+;   %old_replacement = select i1 %old_cmp1, i32 %target_low, i32 %target_high
+;   %old_x_offseted = add i32 %x, C1
+;   %old_cmp0 = icmp ult i32 %old_x_offseted, C0
+;   %r = select i1 %old_cmp0, i32 %x, i32 %old_replacement
+; it can be rewriten as more canonical pattern:
+;   %new_cmp1 = icmp slt i32 %x, -C1
+;   %new_cmp2 = icmp sge i32 %x, C0-C1
+;   %new_clamped_low = select i1 %new_cmp1, i32 %target_low, i32 %x
+;   %r = select i1 %new_cmp2, i32 %target_high, i32 %new_clamped_low
+; Iff -C1 s<= C2 s<= C0-C1
+; Also, ULT predicate can also be UGE; or UGT iff C0 != -1 (+invert result)
+; Also, SLT predicate can also be SGE; or SGT iff C2 != INT_MAX (+invert res.)
+
+;-------------------------------------------------------------------------------
+
+; Basic pattern. There is no 'and', so lower threshold is 0 (inclusive).
+; The upper threshold is 127 (inclusive).
+; There are 2 icmp's so for scalars there are 4 possible combinations.
+; The constant in %t0 has to be between the thresholds, i.e 128 <= Ct0 <= 0.
+
+define i32 @t0_ult_slt_128(i32 %x, i32 %replacement_low, i32 %replacement_high) {
+; CHECK-LABEL: @t0_ult_slt_128(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[X:%.*]], -16
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i32 [[X]], 128
+; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[TMP1]], i32 [[REPLACEMENT_LOW:%.*]], i32 [[X]]
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[TMP2]], i32 [[TMP3]], i32 [[REPLACEMENT_HIGH:%.*]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t0 = icmp slt i32 %x, 128
+  %t1 = select i1 %t0, i32 %replacement_low, i32 %replacement_high
+  %t2 = add i32 %x, 16
+  %t3 = icmp ult i32 %t2, 144
+  %r = select i1 %t3, i32 %x, i32 %t1
+  ret i32 %r
+}
+define i32 @t1_ult_slt_0(i32 %x, i32 %replacement_low, i32 %replacement_high) {
+; CHECK-LABEL: @t1_ult_slt_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[X:%.*]], -16
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i32 [[X]], 128
+; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[TMP1]], i32 [[REPLACEMENT_LOW:%.*]], i32 [[X]]
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[TMP2]], i32 [[TMP3]], i32 [[REPLACEMENT_HIGH:%.*]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t0 = icmp slt i32 %x, -16
+  %t1 = select i1 %t0, i32 %replacement_low, i32 %replacement_high
+  %t2 = add i32 %x, 16
+  %t3 = icmp ult i32 %t2, 144
+  %r = select i1 %t3, i32 %x, i32 %t1
+  ret i32 %r
+}
+
+define i32 @t2_ult_sgt_128(i32 %x, i32 %replacement_low, i32 %replacement_high) {
+; CHECK-LABEL: @t2_ult_sgt_128(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[X:%.*]], -16
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i32 [[X]], 128
+; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[TMP1]], i32 [[REPLACEMENT_LOW:%.*]], i32 [[X]]
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[TMP2]], i32 [[TMP3]], i32 [[REPLACEMENT_HIGH:%.*]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t0 = icmp sgt i32 %x, 127
+  %t1 = select i1 %t0, i32 %replacement_high, i32 %replacement_low
+  %t2 = add i32 %x, 16
+  %t3 = icmp ult i32 %t2, 144
+  %r = select i1 %t3, i32 %x, i32 %t1
+  ret i32 %r
+}
+define i32 @t3_ult_sgt_neg1(i32 %x, i32 %replacement_low, i32 %replacement_high) {
+; CHECK-LABEL: @t3_ult_sgt_neg1(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[X:%.*]], -16
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i32 [[X]], 128
+; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[TMP1]], i32 [[REPLACEMENT_LOW:%.*]], i32 [[X]]
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[TMP2]], i32 [[TMP3]], i32 [[REPLACEMENT_HIGH:%.*]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t0 = icmp sgt i32 %x, -17
+  %t1 = select i1 %t0, i32 %replacement_high, i32 %replacement_low
+  %t2 = add i32 %x, 16
+  %t3 = icmp ult i32 %t2, 144
+  %r = select i1 %t3, i32 %x, i32 %t1
+  ret i32 %r
+}
+
+define i32 @t4_ugt_slt_128(i32 %x, i32 %replacement_low, i32 %replacement_high) {
+; CHECK-LABEL: @t4_ugt_slt_128(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[X:%.*]], -16
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i32 [[X]], 128
+; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[TMP1]], i32 [[REPLACEMENT_LOW:%.*]], i32 [[X]]
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[TMP2]], i32 [[TMP3]], i32 [[REPLACEMENT_HIGH:%.*]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t0 = icmp slt i32 %x, 128
+  %t1 = select i1 %t0, i32 %replacement_low, i32 %replacement_high
+  %t2 = add i32 %x, 16
+  %t3 = icmp ugt i32 %t2, 143
+  %r = select i1 %t3, i32 %t1, i32 %x
+  ret i32 %r
+}
+define i32 @t5_ugt_slt_0(i32 %x, i32 %replacement_low, i32 %replacement_high) {
+; CHECK-LABEL: @t5_ugt_slt_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[X:%.*]], -16
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i32 [[X]], 128
+; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[TMP1]], i32 [[REPLACEMENT_LOW:%.*]], i32 [[X]]
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[TMP2]], i32 [[TMP3]], i32 [[REPLACEMENT_HIGH:%.*]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t0 = icmp slt i32 %x, -16
+  %t1 = select i1 %t0, i32 %replacement_low, i32 %replacement_high
+  %t2 = add i32 %x, 16
+  %t3 = icmp ugt i32 %t2, 143
+  %r = select i1 %t3, i32 %t1, i32 %x
+  ret i32 %r
+}
+
+define i32 @t6_ugt_sgt_128(i32 %x, i32 %replacement_low, i32 %replacement_high) {
+; CHECK-LABEL: @t6_ugt_sgt_128(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[X:%.*]], -16
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i32 [[X]], 128
+; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[TMP1]], i32 [[REPLACEMENT_LOW:%.*]], i32 [[X]]
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[TMP2]], i32 [[TMP3]], i32 [[REPLACEMENT_HIGH:%.*]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t0 = icmp sgt i32 %x, 127
+  %t1 = select i1 %t0, i32 %replacement_high, i32 %replacement_low
+  %t2 = add i32 %x, 16
+  %t3 = icmp ugt i32 %t2, 143
+  %r = select i1 %t3, i32 %t1, i32 %x
+  ret i32 %r
+}
+define i32 @t7_ugt_sgt_neg1(i32 %x, i32 %replacement_low, i32 %replacement_high) {
+; CHECK-LABEL: @t7_ugt_sgt_neg1(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[X:%.*]], -16
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i32 [[X]], 128
+; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[TMP1]], i32 [[REPLACEMENT_LOW:%.*]], i32 [[X]]
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[TMP2]], i32 [[TMP3]], i32 [[REPLACEMENT_HIGH:%.*]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t0 = icmp sgt i32 %x, -17
+  %t1 = select i1 %t0, i32 %replacement_high, i32 %replacement_low
+  %t2 = add i32 %x, 16
+  %t3 = icmp ugt i32 %t2, 143
+  %r = select i1 %t3, i32 %t1, i32 %x
+  ret i32 %r
+}
+
+;-------------------------------------------------------------------------------
+
+; So Ct0 can not be s> 128, or s< -16
+
+define i32 @n8_ult_slt_129(i32 %x, i32 %replacement_low, i32 %replacement_high) {
+; CHECK-LABEL: @n8_ult_slt_129(
+; CHECK-NEXT:    [[T0:%.*]] = icmp slt i32 [[X:%.*]], 129
+; CHECK-NEXT:    [[T1:%.*]] = select i1 [[T0]], i32 [[REPLACEMENT_LOW:%.*]], i32 [[REPLACEMENT_HIGH:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[X]], 16
+; CHECK-NEXT:    [[T3:%.*]] = icmp ult i32 [[T2]], 144
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[T3]], i32 [[X]], i32 [[T1]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t0 = icmp slt i32 %x, 129
+  %t1 = select i1 %t0, i32 %replacement_low, i32 %replacement_high
+  %t2 = add i32 %x, 16
+  %t3 = icmp ult i32 %t2, 144
+  %r = select i1 %t3, i32 %x, i32 %t1
+  ret i32 %r
+}
+define i32 @n9_ult_slt_neg17(i32 %x, i32 %replacement_low, i32 %replacement_high) {
+; CHECK-LABEL: @n9_ult_slt_neg17(
+; CHECK-NEXT:    [[T0:%.*]] = icmp slt i32 [[X:%.*]], -17
+; CHECK-NEXT:    [[T1:%.*]] = select i1 [[T0]], i32 [[REPLACEMENT_LOW:%.*]], i32 [[REPLACEMENT_HIGH:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[X]], 16
+; CHECK-NEXT:    [[T3:%.*]] = icmp ult i32 [[T2]], 144
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[T3]], i32 [[X]], i32 [[T1]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t0 = icmp slt i32 %x, -17
+  %t1 = select i1 %t0, i32 %replacement_low, i32 %replacement_high
+  %t2 = add i32 %x, 16
+  %t3 = icmp ult i32 %t2, 144
+  %r = select i1 %t3, i32 %x, i32 %t1
+  ret i32 %r
+}
+
+;-------------------------------------------------------------------------------
+
+declare void @use32(i32)
+declare void @use1(i1)
+
+; One-use restrictions: here the entire pattern needs to be one-use.
+; FIXME: if %t0 could be reused then it's less restrictive.
+
+; This one is ok.
+define i32 @t10_oneuse0(i32 %x, i32 %replacement_low, i32 %replacement_high) {
+; CHECK-LABEL: @t10_oneuse0(
+; CHECK-NEXT:    [[T0:%.*]] = icmp slt i32 [[X:%.*]], 64
+; CHECK-NEXT:    call void @use1(i1 [[T0]])
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[X]], -16
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i32 [[X]], 128
+; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[TMP1]], i32 [[REPLACEMENT_LOW:%.*]], i32 [[X]]
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[TMP2]], i32 [[TMP3]], i32 [[REPLACEMENT_HIGH:%.*]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t0 = icmp slt i32 %x, 64
+  call void @use1(i1 %t0)
+  %t1 = select i1 %t0, i32 %replacement_low, i32 %replacement_high
+  %t2 = add i32 %x, 16
+  %t3 = icmp ult i32 %t2, 144
+  %r = select i1 %t3, i32 %x, i32 %t1
+  ret i32 %r
+}
+define i32 @n11_oneuse1(i32 %x, i32 %replacement_low, i32 %replacement_high) {
+; CHECK-LABEL: @n11_oneuse1(
+; CHECK-NEXT:    [[T0:%.*]] = icmp slt i32 [[X:%.*]], 64
+; CHECK-NEXT:    [[T1:%.*]] = select i1 [[T0]], i32 [[REPLACEMENT_LOW:%.*]], i32 [[REPLACEMENT_HIGH:%.*]]
+; CHECK-NEXT:    call void @use32(i32 [[T1]])
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[X]], 16
+; CHECK-NEXT:    [[T3:%.*]] = icmp ult i32 [[T2]], 144
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[T3]], i32 [[X]], i32 [[T1]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t0 = icmp slt i32 %x, 64
+  %t1 = select i1 %t0, i32 %replacement_low, i32 %replacement_high
+  call void @use32(i32 %t1)
+  %t2 = add i32 %x, 16
+  %t3 = icmp ult i32 %t2, 144
+  %r = select i1 %t3, i32 %x, i32 %t1
+  ret i32 %r
+}
+
+; This one is ok.
+define i32 @t12_oneuse2(i32 %x, i32 %replacement_low, i32 %replacement_high) {
+; CHECK-LABEL: @t12_oneuse2(
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[X:%.*]], 16
+; CHECK-NEXT:    call void @use32(i32 [[T2]])
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[X]], -16
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i32 [[X]], 128
+; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[TMP1]], i32 [[REPLACEMENT_LOW:%.*]], i32 [[X]]
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[TMP2]], i32 [[TMP3]], i32 [[REPLACEMENT_HIGH:%.*]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t0 = icmp slt i32 %x, 64
+  %t1 = select i1 %t0, i32 %replacement_low, i32 %replacement_high
+  %t2 = add i32 %x, 16
+  call void @use32(i32 %t2)
+  %t3 = icmp ult i32 %t2, 144
+  %r = select i1 %t3, i32 %x, i32 %t1
+  ret i32 %r
+}
+
+define i32 @n13_oneuse3(i32 %x, i32 %replacement_low, i32 %replacement_high) {
+; CHECK-LABEL: @n13_oneuse3(
+; CHECK-NEXT:    [[T0:%.*]] = icmp slt i32 [[X:%.*]], 64
+; CHECK-NEXT:    [[T1:%.*]] = select i1 [[T0]], i32 [[REPLACEMENT_LOW:%.*]], i32 [[REPLACEMENT_HIGH:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[X]], 16
+; CHECK-NEXT:    [[T3:%.*]] = icmp ult i32 [[T2]], 144
+; CHECK-NEXT:    call void @use1(i1 [[T3]])
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[T3]], i32 [[X]], i32 [[T1]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t0 = icmp slt i32 %x, 64
+  %t1 = select i1 %t0, i32 %replacement_low, i32 %replacement_high
+  %t2 = add i32 %x, 16
+  %t3 = icmp ult i32 %t2, 144
+  call void @use1(i1 %t3)
+  %r = select i1 %t3, i32 %x, i32 %t1
+  ret i32 %r
+}
+
+define i32 @n14_oneuse4(i32 %x, i32 %replacement_low, i32 %replacement_high) {
+; CHECK-LABEL: @n14_oneuse4(
+; CHECK-NEXT:    [[T0:%.*]] = icmp slt i32 [[X:%.*]], 64
+; CHECK-NEXT:    call void @use1(i1 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = select i1 [[T0]], i32 [[REPLACEMENT_LOW:%.*]], i32 [[REPLACEMENT_HIGH:%.*]]
+; CHECK-NEXT:    call void @use32(i32 [[T1]])
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[X]], 16
+; CHECK-NEXT:    [[T3:%.*]] = icmp ult i32 [[T2]], 144
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[T3]], i32 [[X]], i32 [[T1]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t0 = icmp slt i32 %x, 64
+  call void @use1(i1 %t0)
+  %t1 = select i1 %t0, i32 %replacement_low, i32 %replacement_high
+  call void @use32(i32 %t1)
+  %t2 = add i32 %x, 16
+  %t3 = icmp ult i32 %t2, 144
+  %r = select i1 %t3, i32 %x, i32 %t1
+  ret i32 %r
+}
+define i32 @n15_oneuse5(i32 %x, i32 %replacement_low, i32 %replacement_high) {
+; CHECK-LABEL: @n15_oneuse5(
+; CHECK-NEXT:    [[T0:%.*]] = icmp slt i32 [[X:%.*]], 64
+; CHECK-NEXT:    call void @use1(i1 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = select i1 [[T0]], i32 [[REPLACEMENT_LOW:%.*]], i32 [[REPLACEMENT_HIGH:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[X]], 16
+; CHECK-NEXT:    call void @use32(i32 [[T2]])
+; CHECK-NEXT:    [[T3:%.*]] = icmp ult i32 [[T2]], 144
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[T3]], i32 [[X]], i32 [[T1]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t0 = icmp slt i32 %x, 64
+  call void @use1(i1 %t0)
+  %t1 = select i1 %t0, i32 %replacement_low, i32 %replacement_high
+  %t2 = add i32 %x, 16
+  call void @use32(i32 %t2)
+  %t3 = icmp ult i32 %t2, 144
+  %r = select i1 %t3, i32 %x, i32 %t1
+  ret i32 %r
+}
+define i32 @n16_oneuse6(i32 %x, i32 %replacement_low, i32 %replacement_high) {
+; CHECK-LABEL: @n16_oneuse6(
+; CHECK-NEXT:    [[T0:%.*]] = icmp slt i32 [[X:%.*]], 64
+; CHECK-NEXT:    call void @use1(i1 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = select i1 [[T0]], i32 [[REPLACEMENT_LOW:%.*]], i32 [[REPLACEMENT_HIGH:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[X]], 16
+; CHECK-NEXT:    [[T3:%.*]] = icmp ult i32 [[T2]], 144
+; CHECK-NEXT:    call void @use1(i1 [[T3]])
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[T3]], i32 [[X]], i32 [[T1]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t0 = icmp slt i32 %x, 64
+  call void @use1(i1 %t0)
+  %t1 = select i1 %t0, i32 %replacement_low, i32 %replacement_high
+  %t2 = add i32 %x, 16
+  %t3 = icmp ult i32 %t2, 144
+  call void @use1(i1 %t3)
+  %r = select i1 %t3, i32 %x, i32 %t1
+  ret i32 %r
+}
+
+define i32 @n17_oneuse7(i32 %x, i32 %replacement_low, i32 %replacement_high) {
+; CHECK-LABEL: @n17_oneuse7(
+; CHECK-NEXT:    [[T0:%.*]] = icmp slt i32 [[X:%.*]], 64
+; CHECK-NEXT:    call void @use1(i1 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = select i1 [[T0]], i32 [[REPLACEMENT_LOW:%.*]], i32 [[REPLACEMENT_HIGH:%.*]]
+; CHECK-NEXT:    call void @use32(i32 [[T1]])
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[X]], 16
+; CHECK-NEXT:    call void @use32(i32 [[T2]])
+; CHECK-NEXT:    [[T3:%.*]] = icmp ult i32 [[T2]], 144
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[T3]], i32 [[X]], i32 [[T1]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t0 = icmp slt i32 %x, 64
+  call void @use1(i1 %t0)
+  %t1 = select i1 %t0, i32 %replacement_low, i32 %replacement_high
+  call void @use32(i32 %t1)
+  %t2 = add i32 %x, 16
+  call void @use32(i32 %t2)
+  %t3 = icmp ult i32 %t2, 144
+  %r = select i1 %t3, i32 %x, i32 %t1
+  ret i32 %r
+}
+define i32 @n18_oneuse8(i32 %x, i32 %replacement_low, i32 %replacement_high) {
+; CHECK-LABEL: @n18_oneuse8(
+; CHECK-NEXT:    [[T0:%.*]] = icmp slt i32 [[X:%.*]], 64
+; CHECK-NEXT:    call void @use1(i1 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = select i1 [[T0]], i32 [[REPLACEMENT_LOW:%.*]], i32 [[REPLACEMENT_HIGH:%.*]]
+; CHECK-NEXT:    call void @use32(i32 [[T1]])
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[X]], 16
+; CHECK-NEXT:    [[T3:%.*]] = icmp ult i32 [[T2]], 144
+; CHECK-NEXT:    call void @use1(i1 [[T3]])
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[T3]], i32 [[X]], i32 [[T1]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t0 = icmp slt i32 %x, 64
+  call void @use1(i1 %t0)
+  %t1 = select i1 %t0, i32 %replacement_low, i32 %replacement_high
+  call void @use32(i32 %t1)
+  %t2 = add i32 %x, 16
+  %t3 = icmp ult i32 %t2, 144
+  call void @use1(i1 %t3)
+  %r = select i1 %t3, i32 %x, i32 %t1
+  ret i32 %r
+}
+
+define i32 @n19_oneuse9(i32 %x, i32 %replacement_low, i32 %replacement_high) {
+; CHECK-LABEL: @n19_oneuse9(
+; CHECK-NEXT:    [[T0:%.*]] = icmp slt i32 [[X:%.*]], 64
+; CHECK-NEXT:    call void @use1(i1 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = select i1 [[T0]], i32 [[REPLACEMENT_LOW:%.*]], i32 [[REPLACEMENT_HIGH:%.*]]
+; CHECK-NEXT:    call void @use32(i32 [[T1]])
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[X]], 16
+; CHECK-NEXT:    call void @use32(i32 [[T2]])
+; CHECK-NEXT:    [[T3:%.*]] = icmp ult i32 [[T2]], 144
+; CHECK-NEXT:    call void @use1(i1 [[T3]])
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[T3]], i32 [[X]], i32 [[T1]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t0 = icmp slt i32 %x, 64
+  call void @use1(i1 %t0)
+  %t1 = select i1 %t0, i32 %replacement_low, i32 %replacement_high
+  call void @use32(i32 %t1)
+  %t2 = add i32 %x, 16
+  call void @use32(i32 %t2)
+  %t3 = icmp ult i32 %t2, 144
+  call void @use1(i1 %t3)
+  %r = select i1 %t3, i32 %x, i32 %t1
+  ret i32 %r
+}
+
+;-------------------------------------------------------------------------------
+
+; Vectors
+
+define <2 x i32> @t20_ult_slt_vec_splat(<2 x i32> %x, <2 x i32> %replacement_low, <2 x i32> %replacement_high) {
+; CHECK-LABEL: @t20_ult_slt_vec_splat(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt <2 x i32> [[X:%.*]], <i32 -16, i32 -16>
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt <2 x i32> [[X]], <i32 128, i32 128>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <2 x i1> [[TMP1]], <2 x i32> [[REPLACEMENT_LOW:%.*]], <2 x i32> [[X]]
+; CHECK-NEXT:    [[R:%.*]] = select <2 x i1> [[TMP2]], <2 x i32> [[TMP3]], <2 x i32> [[REPLACEMENT_HIGH:%.*]]
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %t0 = icmp slt <2 x i32> %x, <i32 128, i32 128>
+  %t1 = select <2 x i1> %t0, <2 x i32> %replacement_low, <2 x i32> %replacement_high
+  %t2 = add <2 x i32> %x, <i32 16, i32 16>
+  %t3 = icmp ult <2 x i32> %t2, <i32 144, i32 144>
+  %r = select <2 x i1> %t3, <2 x i32> %x, <2 x i32> %t1
+  ret <2 x i32> %r
+}
+define <2 x i32> @t21_ult_slt_vec_nonsplat(<2 x i32> %x, <2 x i32> %replacement_low, <2 x i32> %replacement_high) {
+; CHECK-LABEL: @t21_ult_slt_vec_nonsplat(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt <2 x i32> [[X:%.*]], <i32 -16, i32 -8>
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt <2 x i32> [[X]], <i32 128, i32 256>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <2 x i1> [[TMP1]], <2 x i32> [[REPLACEMENT_LOW:%.*]], <2 x i32> [[X]]
+; CHECK-NEXT:    [[R:%.*]] = select <2 x i1> [[TMP2]], <2 x i32> [[TMP3]], <2 x i32> [[REPLACEMENT_HIGH:%.*]]
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %t0 = icmp slt <2 x i32> %x, <i32 128, i32 64>
+  %t1 = select <2 x i1> %t0, <2 x i32> %replacement_low, <2 x i32> %replacement_high
+  %t2 = add <2 x i32> %x, <i32 16, i32 8>
+  %t3 = icmp ult <2 x i32> %t2, <i32 144, i32 264>
+  %r = select <2 x i1> %t3, <2 x i32> %x, <2 x i32> %t1
+  ret <2 x i32> %r
+}
+
+; Non-canonical predicates
+
+declare void @use2xi1(<2 x i1>)
+
+declare void @use(<2 x i1>)
+define <2 x i32> @t22_uge_slt(<2 x i32> %x, <2 x i32> %replacement_low, <2 x i32> %replacement_high) {
+; CHECK-LABEL: @t22_uge_slt(
+; CHECK-NEXT:    [[T0:%.*]] = icmp slt <2 x i32> [[X:%.*]], <i32 128, i32 128>
+; CHECK-NEXT:    [[T1:%.*]] = select <2 x i1> [[T0]], <2 x i32> [[REPLACEMENT_LOW:%.*]], <2 x i32> [[REPLACEMENT_HIGH:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = add <2 x i32> [[X]], <i32 16, i32 16>
+; CHECK-NEXT:    [[T3:%.*]] = icmp uge <2 x i32> [[T2]], <i32 144, i32 0>
+; CHECK-NEXT:    call void @use2xi1(<2 x i1> [[T3]])
+; CHECK-NEXT:    [[R:%.*]] = select <2 x i1> [[T3]], <2 x i32> [[T1]], <2 x i32> [[X]]
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %t0 = icmp slt <2 x i32> %x, <i32 128, i32 128>
+  %t1 = select <2 x i1> %t0, <2 x i32> %replacement_low, <2 x i32> %replacement_high
+  %t2 = add <2 x i32> %x, <i32 16, i32 16>
+  %t3 = icmp uge <2 x i32> %t2, <i32 144, i32 0>
+  call void @use2xi1(<2 x i1> %t3)
+  %r = select <2 x i1> %t3, <2 x i32> %t1, <2 x i32> %x
+  ret <2 x i32> %r
+}
+
+define <2 x i32> @t23_ult_sge(<2 x i32> %x, <2 x i32> %replacement_low, <2 x i32> %replacement_high) {
+; CHECK-LABEL: @t23_ult_sge(
+; CHECK-NEXT:    [[T0:%.*]] = icmp sge <2 x i32> [[X:%.*]], <i32 128, i32 -2147483648>
+; CHECK-NEXT:    call void @use2xi1(<2 x i1> [[T0]])
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt <2 x i32> [[X]], <i32 -16, i32 -2147483648>
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt <2 x i32> [[X]], <i32 128, i32 2147483647>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <2 x i1> [[TMP1]], <2 x i32> [[REPLACEMENT_LOW:%.*]], <2 x i32> [[X]]
+; CHECK-NEXT:    [[R:%.*]] = select <2 x i1> [[TMP2]], <2 x i32> [[TMP3]], <2 x i32> [[REPLACEMENT_HIGH:%.*]]
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %t0 = icmp sge <2 x i32> %x, <i32 128, i32 -2147483648>
+  call void @use2xi1(<2 x i1> %t0)
+  %t1 = select <2 x i1> %t0, <2 x i32> %replacement_high, <2 x i32> %replacement_low
+  %t2 = add <2 x i32> %x, <i32 16, i32 -2147483648>
+  %t3 = icmp ult <2 x i32> %t2, <i32 144, i32 -1>
+  %r = select <2 x i1> %t3, <2 x i32> %x, <2 x i32> %t1
+  ret <2 x i32> %r
+}
diff --git a/llvm/test/Transforms/InstCombine/canonicalize-clamp-like-pattern-between-zero-and-positive-threshold.ll b/llvm/test/Transforms/InstCombine/canonicalize-clamp-like-pattern-between-zero-and-positive-threshold.ll
new file mode 100644
index 000000000..3a35e4c48
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/canonicalize-clamp-like-pattern-between-zero-and-positive-threshold.ll
@@ -0,0 +1,399 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; Given a pattern like:
+;   %old_cmp1 = icmp slt i32 %x, C2
+;   %old_replacement = select i1 %old_cmp1, i32 %target_low, i32 %target_high
+;   %old_x_offseted = add i32 %x, C1
+;   %old_cmp0 = icmp ult i32 %old_x_offseted, C0
+;   %r = select i1 %old_cmp0, i32 %x, i32 %old_replacement
+; it can be rewriten as more canonical pattern:
+;   %new_cmp1 = icmp slt i32 %x, -C1
+;   %new_cmp2 = icmp sge i32 %x, C0-C1
+;   %new_clamped_low = select i1 %new_cmp1, i32 %target_low, i32 %x
+;   %r = select i1 %new_cmp2, i32 %target_high, i32 %new_clamped_low
+; Iff -C1 s<= C2 s<= C0-C1
+; Also, ULT predicate can also be UGE; or UGT iff C0 != -1 (+invert result)
+; Also, SLT predicate can also be SGE; or SGT iff C2 != INT_MAX (+invert res.)
+
+;-------------------------------------------------------------------------------
+
+; Basic pattern. There is no 'and', so lower threshold is 0 (inclusive).
+; The upper threshold is 65535 (inclusive).
+; There are 2 icmp's so for scalars there are 4 possible combinations.
+; The constant in %t0 has to be between the thresholds, i.e 65536 <= Ct0 <= 0.
+
+define i32 @t0_ult_slt_65536(i32 %x, i32 %replacement_low, i32 %replacement_high) {
+; CHECK-LABEL: @t0_ult_slt_65536(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i32 [[X]], 65536
+; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[TMP1]], i32 [[REPLACEMENT_LOW:%.*]], i32 [[X]]
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[TMP2]], i32 [[TMP3]], i32 [[REPLACEMENT_HIGH:%.*]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t0 = icmp slt i32 %x, 65536
+  %t1 = select i1 %t0, i32 %replacement_low, i32 %replacement_high
+  %t2 = icmp ult i32 %x, 65536
+  %r = select i1 %t2, i32 %x, i32 %t1
+  ret i32 %r
+}
+define i32 @t1_ult_slt_0(i32 %x, i32 %replacement_low, i32 %replacement_high) {
+; CHECK-LABEL: @t1_ult_slt_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i32 [[X]], 65536
+; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[TMP1]], i32 [[REPLACEMENT_LOW:%.*]], i32 [[X]]
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[TMP2]], i32 [[TMP3]], i32 [[REPLACEMENT_HIGH:%.*]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t0 = icmp slt i32 %x, 0
+  %t1 = select i1 %t0, i32 %replacement_low, i32 %replacement_high
+  %t2 = icmp ult i32 %x, 65536
+  %r = select i1 %t2, i32 %x, i32 %t1
+  ret i32 %r
+}
+
+define i32 @t2_ult_sgt_65536(i32 %x, i32 %replacement_low, i32 %replacement_high) {
+; CHECK-LABEL: @t2_ult_sgt_65536(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i32 [[X]], 65536
+; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[TMP1]], i32 [[REPLACEMENT_LOW:%.*]], i32 [[X]]
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[TMP2]], i32 [[TMP3]], i32 [[REPLACEMENT_HIGH:%.*]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t0 = icmp sgt i32 %x, 65535
+  %t1 = select i1 %t0, i32 %replacement_high, i32 %replacement_low
+  %t2 = icmp ult i32 %x, 65536
+  %r = select i1 %t2, i32 %x, i32 %t1
+  ret i32 %r
+}
+define i32 @t3_ult_sgt_neg1(i32 %x, i32 %replacement_low, i32 %replacement_high) {
+; CHECK-LABEL: @t3_ult_sgt_neg1(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i32 [[X]], 65536
+; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[TMP1]], i32 [[REPLACEMENT_LOW:%.*]], i32 [[X]]
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[TMP2]], i32 [[TMP3]], i32 [[REPLACEMENT_HIGH:%.*]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t0 = icmp sgt i32 %x, -1
+  %t1 = select i1 %t0, i32 %replacement_high, i32 %replacement_low
+  %t2 = icmp ult i32 %x, 65536
+  %r = select i1 %t2, i32 %x, i32 %t1
+  ret i32 %r
+}
+
+define i32 @t4_ugt_slt_65536(i32 %x, i32 %replacement_low, i32 %replacement_high) {
+; CHECK-LABEL: @t4_ugt_slt_65536(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i32 [[X]], 65536
+; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[TMP1]], i32 [[REPLACEMENT_LOW:%.*]], i32 [[X]]
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[TMP2]], i32 [[TMP3]], i32 [[REPLACEMENT_HIGH:%.*]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t0 = icmp slt i32 %x, 65536
+  %t1 = select i1 %t0, i32 %replacement_low, i32 %replacement_high
+  %t2 = icmp ugt i32 %x, 65535
+  %r = select i1 %t2, i32 %t1, i32 %x
+  ret i32 %r
+}
+define i32 @t5_ugt_slt_0(i32 %x, i32 %replacement_low, i32 %replacement_high) {
+; CHECK-LABEL: @t5_ugt_slt_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i32 [[X]], 65536
+; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[TMP1]], i32 [[REPLACEMENT_LOW:%.*]], i32 [[X]]
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[TMP2]], i32 [[TMP3]], i32 [[REPLACEMENT_HIGH:%.*]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t0 = icmp slt i32 %x, 0
+  %t1 = select i1 %t0, i32 %replacement_low, i32 %replacement_high
+  %t2 = icmp ugt i32 %x, 65535
+  %r = select i1 %t2, i32 %t1, i32 %x
+  ret i32 %r
+}
+
+define i32 @t6_ugt_sgt_65536(i32 %x, i32 %replacement_low, i32 %replacement_high) {
+; CHECK-LABEL: @t6_ugt_sgt_65536(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i32 [[X]], 65536
+; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[TMP1]], i32 [[REPLACEMENT_LOW:%.*]], i32 [[X]]
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[TMP2]], i32 [[TMP3]], i32 [[REPLACEMENT_HIGH:%.*]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t0 = icmp sgt i32 %x, 65535
+  %t1 = select i1 %t0, i32 %replacement_high, i32 %replacement_low
+  %t2 = icmp ugt i32 %x, 65535
+  %r = select i1 %t2, i32 %t1, i32 %x
+  ret i32 %r
+}
+define i32 @t7_ugt_sgt_neg1(i32 %x, i32 %replacement_low, i32 %replacement_high) {
+; CHECK-LABEL: @t7_ugt_sgt_neg1(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i32 [[X]], 65536
+; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[TMP1]], i32 [[REPLACEMENT_LOW:%.*]], i32 [[X]]
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[TMP2]], i32 [[TMP3]], i32 [[REPLACEMENT_HIGH:%.*]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t0 = icmp sgt i32 %x, -1
+  %t1 = select i1 %t0, i32 %replacement_high, i32 %replacement_low
+  %t2 = icmp ugt i32 %x, 65535
+  %r = select i1 %t2, i32 %t1, i32 %x
+  ret i32 %r
+}
+
+;-------------------------------------------------------------------------------
+
+; So Ct0 can not be s> 65536, or s< 0
+
+define i32 @n8_ult_slt_65537(i32 %x, i32 %replacement_low, i32 %replacement_high) {
+; CHECK-LABEL: @n8_ult_slt_65537(
+; CHECK-NEXT:    [[T0:%.*]] = icmp slt i32 [[X:%.*]], 65537
+; CHECK-NEXT:    [[T1:%.*]] = select i1 [[T0]], i32 [[REPLACEMENT_LOW:%.*]], i32 [[REPLACEMENT_HIGH:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = icmp ult i32 [[X]], 65536
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[T2]], i32 [[X]], i32 [[T1]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t0 = icmp slt i32 %x, 65537
+  %t1 = select i1 %t0, i32 %replacement_low, i32 %replacement_high
+  %t2 = icmp ult i32 %x, 65536
+  %r = select i1 %t2, i32 %x, i32 %t1
+  ret i32 %r
+}
+define i32 @n9_ult_slt_neg1(i32 %x, i32 %replacement_low, i32 %replacement_high) {
+; CHECK-LABEL: @n9_ult_slt_neg1(
+; CHECK-NEXT:    [[T0:%.*]] = icmp slt i32 [[X:%.*]], -1
+; CHECK-NEXT:    [[T1:%.*]] = select i1 [[T0]], i32 [[REPLACEMENT_LOW:%.*]], i32 [[REPLACEMENT_HIGH:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = icmp ult i32 [[X]], 65536
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[T2]], i32 [[X]], i32 [[T1]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t0 = icmp slt i32 %x, -1
+  %t1 = select i1 %t0, i32 %replacement_low, i32 %replacement_high
+  %t2 = icmp ult i32 %x, 65536
+  %r = select i1 %t2, i32 %x, i32 %t1
+  ret i32 %r
+}
+
+;-------------------------------------------------------------------------------
+
+declare void @use32(i32)
+declare void @use1(i1)
+
+; One-use restrictions: here the entire pattern needs to be one-use.
+; FIXME: if %t0 could be reused then it's less restrictive.
+
+define i32 @n10_oneuse0(i32 %x, i32 %replacement_low, i32 %replacement_high) {
+; CHECK-LABEL: @n10_oneuse0(
+; CHECK-NEXT:    [[T0:%.*]] = icmp slt i32 [[X:%.*]], 32768
+; CHECK-NEXT:    [[T1:%.*]] = select i1 [[T0]], i32 [[REPLACEMENT_LOW:%.*]], i32 [[REPLACEMENT_HIGH:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = icmp ult i32 [[X]], 65536
+; CHECK-NEXT:    call void @use1(i1 [[T2]])
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[T2]], i32 [[X]], i32 [[T1]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t0 = icmp slt i32 %x, 32768
+  %t1 = select i1 %t0, i32 %replacement_low, i32 %replacement_high
+  %t2 = icmp ult i32 %x, 65536
+  call void @use1(i1 %t2)
+  %r = select i1 %t2, i32 %x, i32 %t1
+  ret i32 %r
+}
+define i32 @n11_oneuse1(i32 %x, i32 %replacement_low, i32 %replacement_high) {
+; CHECK-LABEL: @n11_oneuse1(
+; CHECK-NEXT:    [[T0:%.*]] = icmp slt i32 [[X:%.*]], 32768
+; CHECK-NEXT:    call void @use1(i1 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = select i1 [[T0]], i32 [[REPLACEMENT_LOW:%.*]], i32 [[REPLACEMENT_HIGH:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = icmp ult i32 [[X]], 65536
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[T2]], i32 [[X]], i32 [[T1]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t0 = icmp slt i32 %x, 32768
+  call void @use1(i1 %t0)
+  %t1 = select i1 %t0, i32 %replacement_low, i32 %replacement_high
+  %t2 = icmp ult i32 %x, 65536
+  %r = select i1 %t2, i32 %x, i32 %t1
+  ret i32 %r
+}
+define i32 @n12_oneuse2(i32 %x, i32 %replacement_low, i32 %replacement_high) {
+; CHECK-LABEL: @n12_oneuse2(
+; CHECK-NEXT:    [[T0:%.*]] = icmp slt i32 [[X:%.*]], 32768
+; CHECK-NEXT:    [[T1:%.*]] = select i1 [[T0]], i32 [[REPLACEMENT_LOW:%.*]], i32 [[REPLACEMENT_HIGH:%.*]]
+; CHECK-NEXT:    call void @use32(i32 [[T1]])
+; CHECK-NEXT:    [[T2:%.*]] = icmp ult i32 [[X]], 65536
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[T2]], i32 [[X]], i32 [[T1]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t0 = icmp slt i32 %x, 32768
+  %t1 = select i1 %t0, i32 %replacement_low, i32 %replacement_high
+  call void @use32(i32 %t1)
+  %t2 = icmp ult i32 %x, 65536
+  %r = select i1 %t2, i32 %x, i32 %t1
+  ret i32 %r
+}
+
+define i32 @n13_oneuse3(i32 %x, i32 %replacement_low, i32 %replacement_high) {
+; CHECK-LABEL: @n13_oneuse3(
+; CHECK-NEXT:    [[T0:%.*]] = icmp slt i32 [[X:%.*]], 32768
+; CHECK-NEXT:    call void @use1(i1 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = select i1 [[T0]], i32 [[REPLACEMENT_LOW:%.*]], i32 [[REPLACEMENT_HIGH:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = icmp ult i32 [[X]], 65536
+; CHECK-NEXT:    call void @use1(i1 [[T2]])
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[T2]], i32 [[X]], i32 [[T1]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t0 = icmp slt i32 %x, 32768
+  call void @use1(i1 %t0)
+  %t1 = select i1 %t0, i32 %replacement_low, i32 %replacement_high
+  %t2 = icmp ult i32 %x, 65536
+  call void @use1(i1 %t2)
+  %r = select i1 %t2, i32 %x, i32 %t1
+  ret i32 %r
+}
+define i32 @n14_oneuse4(i32 %x, i32 %replacement_low, i32 %replacement_high) {
+; CHECK-LABEL: @n14_oneuse4(
+; CHECK-NEXT:    [[T0:%.*]] = icmp slt i32 [[X:%.*]], 32768
+; CHECK-NEXT:    call void @use1(i1 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = select i1 [[T0]], i32 [[REPLACEMENT_LOW:%.*]], i32 [[REPLACEMENT_HIGH:%.*]]
+; CHECK-NEXT:    call void @use32(i32 [[T1]])
+; CHECK-NEXT:    [[T2:%.*]] = icmp ult i32 [[X]], 65536
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[T2]], i32 [[X]], i32 [[T1]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t0 = icmp slt i32 %x, 32768
+  call void @use1(i1 %t0)
+  %t1 = select i1 %t0, i32 %replacement_low, i32 %replacement_high
+  call void @use32(i32 %t1)
+  %t2 = icmp ult i32 %x, 65536
+  %r = select i1 %t2, i32 %x, i32 %t1
+  ret i32 %r
+}
+define i32 @n15_oneuse5(i32 %x, i32 %replacement_low, i32 %replacement_high) {
+; CHECK-LABEL: @n15_oneuse5(
+; CHECK-NEXT:    [[T0:%.*]] = icmp slt i32 [[X:%.*]], 32768
+; CHECK-NEXT:    [[T1:%.*]] = select i1 [[T0]], i32 [[REPLACEMENT_LOW:%.*]], i32 [[REPLACEMENT_HIGH:%.*]]
+; CHECK-NEXT:    call void @use32(i32 [[T1]])
+; CHECK-NEXT:    [[T2:%.*]] = icmp ult i32 [[X]], 65536
+; CHECK-NEXT:    call void @use1(i1 [[T2]])
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[T2]], i32 [[X]], i32 [[T1]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t0 = icmp slt i32 %x, 32768
+  %t1 = select i1 %t0, i32 %replacement_low, i32 %replacement_high
+  call void @use32(i32 %t1)
+  %t2 = icmp ult i32 %x, 65536
+  call void @use1(i1 %t2)
+  %r = select i1 %t2, i32 %x, i32 %t1
+  ret i32 %r
+}
+
+define i32 @n16_oneuse6(i32 %x, i32 %replacement_low, i32 %replacement_high) {
+; CHECK-LABEL: @n16_oneuse6(
+; CHECK-NEXT:    [[T0:%.*]] = icmp slt i32 [[X:%.*]], 32768
+; CHECK-NEXT:    call void @use1(i1 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = select i1 [[T0]], i32 [[REPLACEMENT_LOW:%.*]], i32 [[REPLACEMENT_HIGH:%.*]]
+; CHECK-NEXT:    call void @use32(i32 [[T1]])
+; CHECK-NEXT:    [[T2:%.*]] = icmp ult i32 [[X]], 65536
+; CHECK-NEXT:    call void @use1(i1 [[T2]])
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[T2]], i32 [[X]], i32 [[T1]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t0 = icmp slt i32 %x, 32768
+  call void @use1(i1 %t0)
+  %t1 = select i1 %t0, i32 %replacement_low, i32 %replacement_high
+  call void @use32(i32 %t1)
+  %t2 = icmp ult i32 %x, 65536
+  call void @use1(i1 %t2)
+  %r = select i1 %t2, i32 %x, i32 %t1
+  ret i32 %r
+}
+
+;-------------------------------------------------------------------------------
+
+; Vectors
+
+define <2 x i32> @t17_ult_slt_vec_splat(<2 x i32> %x, <2 x i32> %replacement_low, <2 x i32> %replacement_high) {
+; CHECK-LABEL: @t17_ult_slt_vec_splat(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt <2 x i32> [[X:%.*]], zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt <2 x i32> [[X]], <i32 65536, i32 65536>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <2 x i1> [[TMP1]], <2 x i32> [[REPLACEMENT_LOW:%.*]], <2 x i32> [[X]]
+; CHECK-NEXT:    [[R:%.*]] = select <2 x i1> [[TMP2]], <2 x i32> [[TMP3]], <2 x i32> [[REPLACEMENT_HIGH:%.*]]
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %t0 = icmp slt <2 x i32> %x, <i32 65536, i32 65536>
+  %t1 = select <2 x i1> %t0, <2 x i32> %replacement_low, <2 x i32> %replacement_high
+  %t2 = icmp ult <2 x i32> %x, <i32 65536, i32 65536>
+  %r = select <2 x i1> %t2, <2 x i32> %x, <2 x i32> %t1
+  ret <2 x i32> %r
+}
+define <2 x i32> @t18_ult_slt_vec_nonsplat(<2 x i32> %x, <2 x i32> %replacement_low, <2 x i32> %replacement_high) {
+; CHECK-LABEL: @t18_ult_slt_vec_nonsplat(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt <2 x i32> [[X:%.*]], zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt <2 x i32> [[X]], <i32 65536, i32 32768>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <2 x i1> [[TMP1]], <2 x i32> [[REPLACEMENT_LOW:%.*]], <2 x i32> [[X]]
+; CHECK-NEXT:    [[R:%.*]] = select <2 x i1> [[TMP2]], <2 x i32> [[TMP3]], <2 x i32> [[REPLACEMENT_HIGH:%.*]]
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %t0 = icmp slt <2 x i32> %x, <i32 65536, i32 32768>
+  %t1 = select <2 x i1> %t0, <2 x i32> %replacement_low, <2 x i32> %replacement_high
+  %t2 = icmp ult <2 x i32> %x, <i32 65536, i32 32768>
+  %r = select <2 x i1> %t2, <2 x i32> %x, <2 x i32> %t1
+  ret <2 x i32> %r
+}
+
+define <3 x i32> @t19_ult_slt_vec_undef0(<3 x i32> %x, <3 x i32> %replacement_low, <3 x i32> %replacement_high) {
+; CHECK-LABEL: @t19_ult_slt_vec_undef0(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt <3 x i32> [[X:%.*]], zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt <3 x i32> [[X]], <i32 65536, i32 65536, i32 65536>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <3 x i1> [[TMP1]], <3 x i32> [[REPLACEMENT_LOW:%.*]], <3 x i32> [[X]]
+; CHECK-NEXT:    [[R:%.*]] = select <3 x i1> [[TMP2]], <3 x i32> [[TMP3]], <3 x i32> [[REPLACEMENT_HIGH:%.*]]
+; CHECK-NEXT:    ret <3 x i32> [[R]]
+;
+  %t0 = icmp slt <3 x i32> %x, <i32 65536, i32 undef, i32 65536>
+  %t1 = select <3 x i1> %t0, <3 x i32> %replacement_low, <3 x i32> %replacement_high
+  %t2 = icmp ult <3 x i32> %x, <i32 65536, i32 65536, i32 65536>
+  %r = select <3 x i1> %t2, <3 x i32> %x, <3 x i32> %t1
+  ret <3 x i32> %r
+}
+define <3 x i32> @t20_ult_slt_vec_undef1(<3 x i32> %x, <3 x i32> %replacement_low, <3 x i32> %replacement_high) {
+; CHECK-LABEL: @t20_ult_slt_vec_undef1(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt <3 x i32> [[X:%.*]], zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt <3 x i32> [[X]], <i32 65536, i32 undef, i32 65536>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <3 x i1> [[TMP1]], <3 x i32> [[REPLACEMENT_LOW:%.*]], <3 x i32> [[X]]
+; CHECK-NEXT:    [[R:%.*]] = select <3 x i1> [[TMP2]], <3 x i32> [[TMP3]], <3 x i32> [[REPLACEMENT_HIGH:%.*]]
+; CHECK-NEXT:    ret <3 x i32> [[R]]
+;
+  %t0 = icmp slt <3 x i32> %x, <i32 65536, i32 65537, i32 65536>
+  %t1 = select <3 x i1> %t0, <3 x i32> %replacement_low, <3 x i32> %replacement_high
+  %t2 = icmp ult <3 x i32> %x, <i32 65536, i32 undef, i32 65536>
+  %r = select <3 x i1> %t2, <3 x i32> %x, <3 x i32> %t1
+  ret <3 x i32> %r
+}
+define <3 x i32> @t21_ult_slt_vec_undef2(<3 x i32> %x, <3 x i32> %replacement_low, <3 x i32> %replacement_high) {
+; CHECK-LABEL: @t21_ult_slt_vec_undef2(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt <3 x i32> [[X:%.*]], zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt <3 x i32> [[X]], <i32 65536, i32 undef, i32 65536>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <3 x i1> [[TMP1]], <3 x i32> [[REPLACEMENT_LOW:%.*]], <3 x i32> [[X]]
+; CHECK-NEXT:    [[R:%.*]] = select <3 x i1> [[TMP2]], <3 x i32> [[TMP3]], <3 x i32> [[REPLACEMENT_HIGH:%.*]]
+; CHECK-NEXT:    ret <3 x i32> [[R]]
+;
+  %t0 = icmp slt <3 x i32> %x, <i32 65536, i32 undef, i32 65536>
+  %t1 = select <3 x i1> %t0, <3 x i32> %replacement_low, <3 x i32> %replacement_high
+  %t2 = icmp ult <3 x i32> %x, <i32 65536, i32 undef, i32 65536>
+  %r = select <3 x i1> %t2, <3 x i32> %x, <3 x i32> %t1
+  ret <3 x i32> %r
+}
+
+;-------------------------------------------------------------------------------
+
+define i32* @t22_pointers(i32* %x, i32* %replacement_low, i32* %replacement_high) {
+; CHECK-LABEL: @t22_pointers(
+; CHECK-NEXT:    [[T0:%.*]] = icmp slt i32* [[X:%.*]], inttoptr (i64 65536 to i32*)
+; CHECK-NEXT:    [[T1:%.*]] = select i1 [[T0]], i32* [[REPLACEMENT_LOW:%.*]], i32* [[REPLACEMENT_HIGH:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = icmp ult i32* [[X]], inttoptr (i64 65536 to i32*)
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[T2]], i32* [[X]], i32* [[T1]]
+; CHECK-NEXT:    ret i32* [[R]]
+;
+  %t0 = icmp slt i32* %x, inttoptr (i64 65536 to i32*)
+  %t1 = select i1 %t0, i32* %replacement_low, i32* %replacement_high
+  %t2 = icmp ult i32* %x, inttoptr (i64 65536 to i32*)
+  %r = select i1 %t2, i32* %x, i32* %t1
+  ret i32* %r
+}
diff --git a/llvm/test/Transforms/InstCombine/canonicalize-clamp-with-select-of-constant-threshold-pattern.ll b/llvm/test/Transforms/InstCombine/canonicalize-clamp-with-select-of-constant-threshold-pattern.ll
new file mode 100644
index 000000000..0156c9071
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/canonicalize-clamp-with-select-of-constant-threshold-pattern.ll
@@ -0,0 +1,134 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; These patterns are all just traditional clamp pattern.
+; But they are not canonical, the and/or/xor is more canonically represented
+; as an add+icmp.
+
+define i32 @t0_select_cond_and_v0(i32 %X) {
+; CHECK-LABEL: @t0_select_cond_and_v0(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[X:%.*]], -32768
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 -32768
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp slt i32 [[TMP2]], 32767
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[TMP3]], i32 [[TMP2]], i32 32767
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %dont_need_to_clamp_positive = icmp sle i32 %X, 32767
+  %dont_need_to_clamp_negative = icmp sge i32 %X, -32768
+  %clamp_limit = select i1 %dont_need_to_clamp_positive, i32 -32768, i32 32767
+  %dont_need_to_clamp = and i1 %dont_need_to_clamp_positive, %dont_need_to_clamp_negative
+  %R = select i1 %dont_need_to_clamp, i32 %X, i32 %clamp_limit
+  ret i32 %R
+}
+define i32 @t1_select_cond_and_v1(i32 %X) {
+; CHECK-LABEL: @t1_select_cond_and_v1(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[X:%.*]], -32768
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 -32768
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp slt i32 [[TMP2]], 32767
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[TMP3]], i32 [[TMP2]], i32 32767
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %dont_need_to_clamp_positive = icmp sle i32 %X, 32767
+  %dont_need_to_clamp_negative = icmp sge i32 %X, -32768
+  %clamp_limit = select i1 %dont_need_to_clamp_negative, i32 32767, i32 -32768
+  %dont_need_to_clamp = and i1 %dont_need_to_clamp_positive, %dont_need_to_clamp_negative
+  %R = select i1 %dont_need_to_clamp, i32 %X, i32 %clamp_limit
+  ret i32 %R
+}
+
+;-------------------------------------------------------------------------------
+
+define i32 @t2_select_cond_or_v0(i32 %X) {
+; CHECK-LABEL: @t2_select_cond_or_v0(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[X:%.*]], -32768
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 -32768
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp slt i32 [[TMP2]], 32767
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[TMP3]], i32 [[TMP2]], i32 32767
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %need_to_clamp_positive = icmp sgt i32 %X, 32767
+  %need_to_clamp_negative = icmp slt i32 %X, -32768
+  %clamp_limit = select i1 %need_to_clamp_positive, i32 32767, i32 -32768
+  %need_to_clamp = or i1 %need_to_clamp_positive, %need_to_clamp_negative
+  %R = select i1 %need_to_clamp, i32 %clamp_limit, i32 %X
+  ret i32 %R
+}
+define i32 @t3_select_cond_or_v1(i32 %X) {
+; CHECK-LABEL: @t3_select_cond_or_v1(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[X:%.*]], -32768
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 -32768
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp slt i32 [[TMP2]], 32767
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[TMP3]], i32 [[TMP2]], i32 32767
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %need_to_clamp_positive = icmp sgt i32 %X, 32767
+  %need_to_clamp_negative = icmp slt i32 %X, -32768
+  %clamp_limit = select i1 %need_to_clamp_negative, i32 -32768, i32 32767
+  %need_to_clamp = or i1 %need_to_clamp_positive, %need_to_clamp_negative
+  %R = select i1 %need_to_clamp, i32 %clamp_limit, i32 %X
+  ret i32 %R
+}
+
+;-------------------------------------------------------------------------------
+
+define i32 @t4_select_cond_xor_v0(i32 %X) {
+; CHECK-LABEL: @t4_select_cond_xor_v0(
+; CHECK-NEXT:    [[DOTINV1:%.*]] = icmp sgt i32 [[X:%.*]], -32768
+; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[DOTINV1]], i32 [[X]], i32 -32768
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i32 [[TMP1]], 32767
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[TMP2]], i32 [[TMP1]], i32 32767
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %need_to_clamp_positive = icmp sgt i32 %X, 32767
+  %dont_need_to_clamp_negative = icmp sgt i32 %X, -32768
+  %clamp_limit = select i1 %need_to_clamp_positive, i32 32767, i32 -32768
+  %dont_need_to_clamp = xor i1 %need_to_clamp_positive, %dont_need_to_clamp_negative
+  %R = select i1 %dont_need_to_clamp, i32 %X, i32 %clamp_limit
+  ret i32 %R
+}
+define i32 @t4_select_cond_xor_v1(i32 %X) {
+; CHECK-LABEL: @t4_select_cond_xor_v1(
+; CHECK-NEXT:    [[DOTINV1:%.*]] = icmp sgt i32 [[X:%.*]], -32768
+; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[DOTINV1]], i32 [[X]], i32 -32768
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i32 [[TMP1]], 32767
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[TMP2]], i32 [[TMP1]], i32 32767
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %need_to_clamp_positive = icmp sgt i32 %X, 32767
+  %dont_need_to_clamp_negative = icmp sgt i32 %X, -32768
+  %clamp_limit = select i1 %dont_need_to_clamp_negative, i32 32767, i32 -32768
+  %dont_need_to_clamp = xor i1 %need_to_clamp_positive, %dont_need_to_clamp_negative
+  %R = select i1 %dont_need_to_clamp, i32 %X, i32 %clamp_limit
+  ret i32 %R
+}
+
+define i32 @t5_select_cond_xor_v2(i32 %X) {
+; CHECK-LABEL: @t5_select_cond_xor_v2(
+; CHECK-NEXT:    [[DOTINV1:%.*]] = icmp sgt i32 [[X:%.*]], -32768
+; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[DOTINV1]], i32 [[X]], i32 -32768
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i32 [[TMP1]], 32767
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[TMP2]], i32 [[TMP1]], i32 32767
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %dont_need_to_clamp_positive = icmp sle i32 %X, 32767
+  %need_to_clamp_negative = icmp sle i32 %X, -32768
+  %clamp_limit = select i1 %need_to_clamp_negative, i32 -32768, i32 32767
+  %dont_need_to_clamp = xor i1 %dont_need_to_clamp_positive, %need_to_clamp_negative
+  %R = select i1 %dont_need_to_clamp, i32 %X, i32 %clamp_limit
+  ret i32 %R
+}
+define i32 @t5_select_cond_xor_v3(i32 %X) {
+; CHECK-LABEL: @t5_select_cond_xor_v3(
+; CHECK-NEXT:    [[DOTINV1:%.*]] = icmp sgt i32 [[X:%.*]], -32768
+; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[DOTINV1]], i32 [[X]], i32 -32768
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i32 [[TMP1]], 32767
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[TMP2]], i32 [[TMP1]], i32 32767
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %dont_need_to_clamp_positive = icmp sle i32 %X, 32767
+  %need_to_clamp_negative = icmp sle i32 %X, -32768
+  %clamp_limit = select i1 %dont_need_to_clamp_positive, i32 -32768, i32 32767
+  %dont_need_to_clamp = xor i1 %dont_need_to_clamp_positive, %need_to_clamp_negative
+  %R = select i1 %dont_need_to_clamp, i32 %X, i32 %clamp_limit
+  ret i32 %R
+}
diff --git a/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-eq-to-icmp-ule.ll b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-eq-to-icmp-ule.ll
index bb304d7fa..6a3564eb7 100644
--- a/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-eq-to-icmp-ule.ll
+++ b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-eq-to-icmp-ule.ll
@@ -82,7 +82,7 @@ define <2 x i1> @p2_vec_nonsplat_edgecase1(<2 x i8> %x) {
 
 define <3 x i1> @p3_vec_splat_undef(<3 x i8> %x) {
 ; CHECK-LABEL: @p3_vec_splat_undef(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult <3 x i8> [[X:%.*]], <i8 4, i8 undef, i8 4>
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult <3 x i8> [[X:%.*]], <i8 4, i8 4, i8 4>
 ; CHECK-NEXT:    ret <3 x i1> [[TMP1]]
 ;
   %tmp0 = and <3 x i8> %x, <i8 3, i8 undef, i8 3>
diff --git a/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-ne-to-icmp-ugt.ll b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-ne-to-icmp-ugt.ll
index 35ae2600c..49f76fd9f 100644
--- a/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-ne-to-icmp-ugt.ll
+++ b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-ne-to-icmp-ugt.ll
@@ -82,7 +82,7 @@ define <2 x i1> @p2_vec_nonsplat_edgecase1(<2 x i8> %x) {
 
 define <3 x i1> @p3_vec_splat_undef(<3 x i8> %x) {
 ; CHECK-LABEL: @p3_vec_splat_undef(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt <3 x i8> [[X:%.*]], <i8 3, i8 undef, i8 3>
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt <3 x i8> [[X:%.*]], <i8 3, i8 3, i8 3>
 ; CHECK-NEXT:    ret <3 x i1> [[TMP1]]
 ;
   %tmp0 = and <3 x i8> %x, <i8 3, i8 undef, i8 3>
@@ -90,6 +90,16 @@ define <3 x i1> @p3_vec_splat_undef(<3 x i8> %x) {
   ret <3 x i1> %ret
 }
 
+define <3 x i1> @p3_vec_nonsplat_undef(<3 x i8> %x) {
+; CHECK-LABEL: @p3_vec_nonsplat_undef(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt <3 x i8> [[X:%.*]], <i8 -1, i8 -1, i8 3>
+; CHECK-NEXT:    ret <3 x i1> [[TMP1]]
+;
+  %tmp0 = and <3 x i8> %x, <i8 -1, i8 undef, i8 3>
+  %ret = icmp ne <3 x i8> %tmp0, %x
+  ret <3 x i1> %ret
+}
+
 ; ============================================================================ ;
 ; Commutativity tests.
 ; ============================================================================ ;
diff --git a/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-sge-to-icmp-sle.ll b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-sge-to-icmp-sle.ll
index 3ed42e415..3bfb48f41 100644
--- a/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-sge-to-icmp-sle.ll
+++ b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-sge-to-icmp-sle.ll
@@ -61,7 +61,7 @@ define <2 x i1> @p2_vec_nonsplat_edgecase(<2 x i8> %x) {
 
 define <3 x i1> @p3_vec_splat_undef(<3 x i8> %x) {
 ; CHECK-LABEL: @p3_vec_splat_undef(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt <3 x i8> [[X:%.*]], <i8 4, i8 undef, i8 4>
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt <3 x i8> [[X:%.*]], <i8 4, i8 4, i8 4>
 ; CHECK-NEXT:    ret <3 x i1> [[TMP1]]
 ;
   %tmp0 = and <3 x i8> %x, <i8 3, i8 undef, i8 3>
diff --git a/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-sgt-to-icmp-sgt.ll b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-sgt-to-icmp-sgt.ll
index b6d1c99cc..16d063ee0 100644
--- a/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-sgt-to-icmp-sgt.ll
+++ b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-sgt-to-icmp-sgt.ll
@@ -76,7 +76,7 @@ define <2 x i1> @p2_vec_nonsplat_edgecase() {
 define <3 x i1> @p3_vec_splat_undef() {
 ; CHECK-LABEL: @p3_vec_splat_undef(
 ; CHECK-NEXT:    [[X:%.*]] = call <3 x i8> @gen3x8()
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <3 x i8> [[X]], <i8 3, i8 undef, i8 3>
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <3 x i8> [[X]], <i8 3, i8 3, i8 3>
 ; CHECK-NEXT:    ret <3 x i1> [[TMP1]]
 ;
   %x = call <3 x i8> @gen3x8()
@@ -85,6 +85,18 @@ define <3 x i1> @p3_vec_splat_undef() {
   ret <3 x i1> %ret
 }
 
+define <3 x i1> @p3_vec_nonsplat_undef() {
+; CHECK-LABEL: @p3_vec_nonsplat_undef(
+; CHECK-NEXT:    [[X:%.*]] = call <3 x i8> @gen3x8()
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <3 x i8> [[X]], <i8 15, i8 3, i8 15>
+; CHECK-NEXT:    ret <3 x i1> [[TMP1]]
+;
+  %x = call <3 x i8> @gen3x8()
+  %tmp0 = and <3 x i8> %x, <i8 15, i8 3, i8 undef>
+  %ret = icmp sgt <3 x i8> %x, %tmp0
+  ret <3 x i1> %ret
+}
+
 ; ============================================================================ ;
 ; One-use tests. We don't care about multi-uses here.
 ; ============================================================================ ;
diff --git a/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-sle-to-icmp-sle.ll b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-sle-to-icmp-sle.ll
index 64be961e5..eec8bdc09 100644
--- a/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-sle-to-icmp-sle.ll
+++ b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-sle-to-icmp-sle.ll
@@ -76,7 +76,7 @@ define <2 x i1> @p2_vec_nonsplat_edgecase() {
 define <3 x i1> @p3_vec_splat_undef() {
 ; CHECK-LABEL: @p3_vec_splat_undef(
 ; CHECK-NEXT:    [[X:%.*]] = call <3 x i8> @gen3x8()
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt <3 x i8> [[X]], <i8 4, i8 undef, i8 4>
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt <3 x i8> [[X]], <i8 4, i8 4, i8 4>
 ; CHECK-NEXT:    ret <3 x i1> [[TMP1]]
 ;
   %x = call <3 x i8> @gen3x8()
diff --git a/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-slt-to-icmp-sgt.ll b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-slt-to-icmp-sgt.ll
index 7cda57272..b71172b42 100644
--- a/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-slt-to-icmp-sgt.ll
+++ b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-slt-to-icmp-sgt.ll
@@ -61,7 +61,7 @@ define <2 x i1> @p2_vec_nonsplat_edgecase(<2 x i8> %x) {
 
 define <3 x i1> @p3_vec_splat_undef(<3 x i8> %x) {
 ; CHECK-LABEL: @p3_vec_splat_undef(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <3 x i8> [[X:%.*]], <i8 3, i8 undef, i8 3>
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <3 x i8> [[X:%.*]], <i8 3, i8 3, i8 3>
 ; CHECK-NEXT:    ret <3 x i1> [[TMP1]]
 ;
   %tmp0 = and <3 x i8> %x, <i8 3, i8 undef, i8 3>
@@ -69,6 +69,16 @@ define <3 x i1> @p3_vec_splat_undef(<3 x i8> %x) {
   ret <3 x i1> %ret
 }
 
+define <3 x i1> @p3_vec_nonsplat_undef(<3 x i8> %x) {
+; CHECK-LABEL: @p3_vec_nonsplat_undef(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <3 x i8> [[X:%.*]], <i8 15, i8 15, i8 3>
+; CHECK-NEXT:    ret <3 x i1> [[TMP1]]
+;
+  %tmp0 = and <3 x i8> %x, <i8 undef, i8 15, i8 3>
+  %ret = icmp slt <3 x i8> %tmp0, %x
+  ret <3 x i1> %ret
+}
+
 ; ============================================================================ ;
 ; One-use tests. We don't care about multi-uses here.
 ; ============================================================================ ;
diff --git a/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-uge-to-icmp-ule.ll b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-uge-to-icmp-ule.ll
index 18325ed17..1ae84cf81 100644
--- a/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-uge-to-icmp-ule.ll
+++ b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-uge-to-icmp-ule.ll
@@ -82,7 +82,7 @@ define <2 x i1> @p2_vec_nonsplat_edgecase1(<2 x i8> %x) {
 
 define <3 x i1> @p3_vec_splat_undef(<3 x i8> %x) {
 ; CHECK-LABEL: @p3_vec_splat_undef(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult <3 x i8> [[X:%.*]], <i8 4, i8 undef, i8 4>
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult <3 x i8> [[X:%.*]], <i8 4, i8 4, i8 4>
 ; CHECK-NEXT:    ret <3 x i1> [[TMP1]]
 ;
   %tmp0 = and <3 x i8> %x, <i8 3, i8 undef, i8 3>
diff --git a/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-ugt-to-icmp-ugt.ll b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-ugt-to-icmp-ugt.ll
index 38a57d6d2..9b1d40593 100644
--- a/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-ugt-to-icmp-ugt.ll
+++ b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-ugt-to-icmp-ugt.ll
@@ -99,7 +99,7 @@ define <2 x i1> @p2_vec_nonsplat_edgecase1() {
 define <3 x i1> @p3_vec_splat_undef() {
 ; CHECK-LABEL: @p3_vec_splat_undef(
 ; CHECK-NEXT:    [[X:%.*]] = call <3 x i8> @gen3x8()
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt <3 x i8> [[X]], <i8 3, i8 undef, i8 3>
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt <3 x i8> [[X]], <i8 3, i8 3, i8 3>
 ; CHECK-NEXT:    ret <3 x i1> [[TMP1]]
 ;
   %x = call <3 x i8> @gen3x8()
@@ -108,6 +108,18 @@ define <3 x i1> @p3_vec_splat_undef() {
   ret <3 x i1> %ret
 }
 
+define <3 x i1> @p3_vec_nonsplat_undef() {
+; CHECK-LABEL: @p3_vec_nonsplat_undef(
+; CHECK-NEXT:    [[X:%.*]] = call <3 x i8> @gen3x8()
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt <3 x i8> [[X]], <i8 3, i8 3, i8 15>
+; CHECK-NEXT:    ret <3 x i1> [[TMP1]]
+;
+  %x = call <3 x i8> @gen3x8()
+  %tmp0 = and <3 x i8> %x, <i8 3, i8 undef, i8 15>
+  %ret = icmp ugt <3 x i8> %x, %tmp0
+  ret <3 x i1> %ret
+}
+
 ; ============================================================================ ;
 ; Commutativity tests.
 ; ============================================================================ ;
diff --git a/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-ule-to-icmp-ule.ll b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-ule-to-icmp-ule.ll
index 6f267206d..2d2bfce73 100644
--- a/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-ule-to-icmp-ule.ll
+++ b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-ule-to-icmp-ule.ll
@@ -99,7 +99,7 @@ define <2 x i1> @p2_vec_nonsplat_edgecase1() {
 define <3 x i1> @p3_vec_splat_undef() {
 ; CHECK-LABEL: @p3_vec_splat_undef(
 ; CHECK-NEXT:    [[X:%.*]] = call <3 x i8> @gen3x8()
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult <3 x i8> [[X]], <i8 4, i8 undef, i8 4>
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult <3 x i8> [[X]], <i8 4, i8 4, i8 4>
 ; CHECK-NEXT:    ret <3 x i1> [[TMP1]]
 ;
   %x = call <3 x i8> @gen3x8()
diff --git a/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-ult-to-icmp-ugt.ll b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-ult-to-icmp-ugt.ll
index ad0f0cf3c..728f9e0dd 100644
--- a/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-ult-to-icmp-ugt.ll
+++ b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-ult-to-icmp-ugt.ll
@@ -83,7 +83,7 @@ define <2 x i1> @p2_vec_nonsplat_edgecase1(<2 x i8> %x) {
 
 define <3 x i1> @p3_vec_splat_undef(<3 x i8> %x) {
 ; CHECK-LABEL: @p3_vec_splat_undef(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt <3 x i8> [[X:%.*]], <i8 3, i8 undef, i8 3>
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt <3 x i8> [[X:%.*]], <i8 3, i8 3, i8 3>
 ; CHECK-NEXT:    ret <3 x i1> [[TMP1]]
 ;
   %tmp0 = and <3 x i8> %x, <i8 3, i8 undef, i8 3>
@@ -91,6 +91,16 @@ define <3 x i1> @p3_vec_splat_undef(<3 x i8> %x) {
   ret <3 x i1> %ret
 }
 
+define <3 x i1> @p3_vec_nonsplat_undef(<3 x i8> %x) {
+; CHECK-LABEL: @p3_vec_nonsplat_undef(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt <3 x i8> [[X:%.*]], <i8 7, i8 31, i8 7>
+; CHECK-NEXT:    ret <3 x i1> [[TMP1]]
+;
+  %tmp0 = and <3 x i8> %x, <i8 7, i8 31, i8 undef>
+  %ret = icmp ult <3 x i8> %tmp0, %x
+  ret <3 x i1> %ret
+}
+
 ; ============================================================================ ;
 ; Commutativity tests.
 ; ============================================================================ ;
diff --git a/llvm/test/Transforms/InstCombine/cast-mul-select.ll b/llvm/test/Transforms/InstCombine/cast-mul-select.ll
index c501fd8d0..f82d2fd28 100644
--- a/llvm/test/Transforms/InstCombine/cast-mul-select.ll
+++ b/llvm/test/Transforms/InstCombine/cast-mul-select.ll
@@ -13,6 +13,8 @@ define i32 @mul(i32 %x, i32 %y) {
 ; we preserve the debug information in the resulting
 ; instruction.
 ; DBGINFO-LABEL: @mul(
+; DBGINFO-NEXT:    call void @llvm.dbg.value(metadata i32 %x
+; DBGINFO-NEXT:    call void @llvm.dbg.value(metadata i32 %y
 ; DBGINFO-NEXT:    [[C:%.*]] = mul i32 {{.*}}
 ; DBGINFO-NEXT:    [[D:%.*]] = and i32 {{.*}}
 ; DBGINFO-NEXT:    call void @llvm.dbg.value(metadata i32 [[C]]
diff --git a/llvm/test/Transforms/InstCombine/cast.ll b/llvm/test/Transforms/InstCombine/cast.ll
index b6d1eda06..d85286b46 100644
--- a/llvm/test/Transforms/InstCombine/cast.ll
+++ b/llvm/test/Transforms/InstCombine/cast.ll
@@ -1,13 +1,13 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; Tests to make sure elimination of casts is working correctly
-; RUN: opt < %s -instcombine -S | FileCheck %s
-target datalayout = "E-p:64:64:64-p1:32:32:32-p2:64:64:64-p3:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128-n8:16:32:64"
+; RUN: opt < %s -instcombine -S -data-layout="E-p:64:64:64-p1:32:32:32-p2:64:64:64-p3:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128-n8:16:32:64" | FileCheck %s --check-prefixes=ALL,BE
+; RUN: opt < %s -instcombine -S -data-layout="e-p:64:64:64-p1:32:32:32-p2:64:64:64-p3:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128-n8:16:32:64" | FileCheck %s --check-prefixes=ALL,LE
 
 @inbuf = external global [32832 x i8]
 
 define i32 @test1(i32 %A) {
-; CHECK-LABEL: @test1(
-; CHECK-NEXT:    ret i32 [[A:%.*]]
+; ALL-LABEL: @test1(
+; ALL-NEXT:    ret i32 [[A:%.*]]
 ;
   %c1 = bitcast i32 %A to i32
   %c2 = bitcast i32 %c1 to i32
@@ -15,9 +15,9 @@ define i32 @test1(i32 %A) {
 }
 
 define i64 @test2(i8 %A) {
-; CHECK-LABEL: @test2(
-; CHECK-NEXT:    [[RET:%.*]] = zext i8 [[A:%.*]] to i64
-; CHECK-NEXT:    ret i64 [[RET]]
+; ALL-LABEL: @test2(
+; ALL-NEXT:    [[RET:%.*]] = zext i8 [[A:%.*]] to i64
+; ALL-NEXT:    ret i64 [[RET]]
 ;
   %c1 = zext i8 %A to i16
   %c2 = zext i16 %c1 to i32
@@ -26,9 +26,9 @@ define i64 @test2(i8 %A) {
 }
 
 define i64 @test3(i64 %A) {
-; CHECK-LABEL: @test3(
-; CHECK-NEXT:    [[C2:%.*]] = and i64 [[A:%.*]], 255
-; CHECK-NEXT:    ret i64 [[C2]]
+; ALL-LABEL: @test3(
+; ALL-NEXT:    [[C2:%.*]] = and i64 [[A:%.*]], 255
+; ALL-NEXT:    ret i64 [[C2]]
 ;
   %c1 = trunc i64 %A to i8
   %c2 = zext i8 %c1 to i64
@@ -36,10 +36,10 @@ define i64 @test3(i64 %A) {
 }
 
 define i32 @test4(i32 %A, i32 %B) {
-; CHECK-LABEL: @test4(
-; CHECK-NEXT:    [[COND:%.*]] = icmp slt i32 [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[RESULT:%.*]] = zext i1 [[COND]] to i32
-; CHECK-NEXT:    ret i32 [[RESULT]]
+; ALL-LABEL: @test4(
+; ALL-NEXT:    [[COND:%.*]] = icmp slt i32 [[A:%.*]], [[B:%.*]]
+; ALL-NEXT:    [[RESULT:%.*]] = zext i1 [[COND]] to i32
+; ALL-NEXT:    ret i32 [[RESULT]]
 ;
   %COND = icmp slt i32 %A, %B
   %c = zext i1 %COND to i8
@@ -48,9 +48,9 @@ define i32 @test4(i32 %A, i32 %B) {
 }
 
 define i32 @test5(i1 %B) {
-; CHECK-LABEL: @test5(
-; CHECK-NEXT:    [[RESULT:%.*]] = zext i1 [[B:%.*]] to i32
-; CHECK-NEXT:    ret i32 [[RESULT]]
+; ALL-LABEL: @test5(
+; ALL-NEXT:    [[RESULT:%.*]] = zext i1 [[B:%.*]] to i32
+; ALL-NEXT:    ret i32 [[RESULT]]
 ;
   %c = zext i1 %B to i8
   %result = zext i8 %c to i32
@@ -58,9 +58,9 @@ define i32 @test5(i1 %B) {
 }
 
 define i32 @test6(i64 %A) {
-; CHECK-LABEL: @test6(
-; CHECK-NEXT:    [[C1:%.*]] = trunc i64 [[A:%.*]] to i32
-; CHECK-NEXT:    ret i32 [[C1]]
+; ALL-LABEL: @test6(
+; ALL-NEXT:    [[C1:%.*]] = trunc i64 [[A:%.*]] to i32
+; ALL-NEXT:    ret i32 [[C1]]
 ;
   %c1 = trunc i64 %A to i32
   %res = bitcast i32 %c1 to i32
@@ -68,9 +68,9 @@ define i32 @test6(i64 %A) {
 }
 
 define i64 @test7(i1 %A) {
-; CHECK-LABEL: @test7(
-; CHECK-NEXT:    [[RES:%.*]] = zext i1 [[A:%.*]] to i64
-; CHECK-NEXT:    ret i64 [[RES]]
+; ALL-LABEL: @test7(
+; ALL-NEXT:    [[RES:%.*]] = zext i1 [[A:%.*]] to i64
+; ALL-NEXT:    ret i64 [[RES]]
 ;
   %c1 = zext i1 %A to i32
   %res = sext i32 %c1 to i64
@@ -78,9 +78,9 @@ define i64 @test7(i1 %A) {
 }
 
 define i64 @test8(i8 %A) {
-; CHECK-LABEL: @test8(
-; CHECK-NEXT:    [[C1:%.*]] = sext i8 [[A:%.*]] to i64
-; CHECK-NEXT:    ret i64 [[C1]]
+; ALL-LABEL: @test8(
+; ALL-NEXT:    [[C1:%.*]] = sext i8 [[A:%.*]] to i64
+; ALL-NEXT:    ret i64 [[C1]]
 ;
   %c1 = sext i8 %A to i64
   %res = bitcast i64 %c1 to i64
@@ -88,8 +88,8 @@ define i64 @test8(i8 %A) {
 }
 
 define i16 @test9(i16 %A) {
-; CHECK-LABEL: @test9(
-; CHECK-NEXT:    ret i16 [[A:%.*]]
+; ALL-LABEL: @test9(
+; ALL-NEXT:    ret i16 [[A:%.*]]
 ;
   %c1 = sext i16 %A to i32
   %c2 = trunc i32 %c1 to i16
@@ -97,8 +97,8 @@ define i16 @test9(i16 %A) {
 }
 
 define i16 @test10(i16 %A) {
-; CHECK-LABEL: @test10(
-; CHECK-NEXT:    ret i16 [[A:%.*]]
+; ALL-LABEL: @test10(
+; ALL-NEXT:    ret i16 [[A:%.*]]
 ;
   %c1 = sext i16 %A to i32
   %c2 = trunc i32 %c1 to i16
@@ -108,9 +108,9 @@ define i16 @test10(i16 %A) {
 declare void @varargs(i32, ...)
 
 define void @test11(i32* %P) {
-; CHECK-LABEL: @test11(
-; CHECK-NEXT:    call void (i32, ...) @varargs(i32 5, i32* [[P:%.*]])
-; CHECK-NEXT:    ret void
+; ALL-LABEL: @test11(
+; ALL-NEXT:    call void (i32, ...) @varargs(i32 5, i32* [[P:%.*]])
+; ALL-NEXT:    ret void
 ;
   %c = bitcast i32* %P to i16*
   call void (i32, ...) @varargs( i32 5, i16* %c )
@@ -119,16 +119,16 @@ define void @test11(i32* %P) {
 
 declare i32 @__gxx_personality_v0(...)
 define void @test_invoke_vararg_cast(i32* %a, i32* %b) personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
-; CHECK-LABEL: @test_invoke_vararg_cast(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    invoke void (i32, ...) @varargs(i32 1, i32* [[B:%.*]], i32* [[A:%.*]])
-; CHECK-NEXT:    to label [[INVOKE_CONT:%.*]] unwind label [[LPAD:%.*]]
-; CHECK:       invoke.cont:
-; CHECK-NEXT:    ret void
-; CHECK:       lpad:
-; CHECK-NEXT:    [[TMP0:%.*]] = landingpad { i8*, i32 }
-; CHECK-NEXT:    cleanup
-; CHECK-NEXT:    ret void
+; ALL-LABEL: @test_invoke_vararg_cast(
+; ALL-NEXT:  entry:
+; ALL-NEXT:    invoke void (i32, ...) @varargs(i32 1, i32* [[B:%.*]], i32* [[A:%.*]])
+; ALL-NEXT:    to label [[INVOKE_CONT:%.*]] unwind label [[LPAD:%.*]]
+; ALL:       invoke.cont:
+; ALL-NEXT:    ret void
+; ALL:       lpad:
+; ALL-NEXT:    [[TMP0:%.*]] = landingpad { i8*, i32 }
+; ALL-NEXT:    cleanup
+; ALL-NEXT:    ret void
 ;
 entry:
   %0 = bitcast i32* %b to i8*
@@ -146,18 +146,18 @@ lpad:
 }
 
 define i8* @test13(i64 %A) {
-; CHECK-LABEL: @test13(
-; CHECK-NEXT:    [[C:%.*]] = getelementptr [32832 x i8], [32832 x i8]* @inbuf, i64 0, i64 [[A:%.*]]
-; CHECK-NEXT:    ret i8* [[C]]
+; ALL-LABEL: @test13(
+; ALL-NEXT:    [[C:%.*]] = getelementptr [32832 x i8], [32832 x i8]* @inbuf, i64 0, i64 [[A:%.*]]
+; ALL-NEXT:    ret i8* [[C]]
 ;
   %c = getelementptr [0 x i8], [0 x i8]* bitcast ([32832 x i8]* @inbuf to [0 x i8]*), i64 0, i64 %A
   ret i8* %c
 }
 
 define i1 @test14(i8 %A) {
-; CHECK-LABEL: @test14(
-; CHECK-NEXT:    [[X:%.*]] = icmp sgt i8 [[A:%.*]], -1
-; CHECK-NEXT:    ret i1 [[X]]
+; ALL-LABEL: @test14(
+; ALL-NEXT:    [[X:%.*]] = icmp sgt i8 [[A:%.*]], -1
+; ALL-NEXT:    ret i1 [[X]]
 ;
   %c = bitcast i8 %A to i8
   %X = icmp ult i8 %c, -128
@@ -173,18 +173,18 @@ define i1 @test14(i8 %A) {
 ;}
 
 define i1 @test16(i32* %P) {
-; CHECK-LABEL: @test16(
-; CHECK-NEXT:    [[C:%.*]] = icmp ne i32* [[P:%.*]], null
-; CHECK-NEXT:    ret i1 [[C]]
+; ALL-LABEL: @test16(
+; ALL-NEXT:    [[C:%.*]] = icmp ne i32* [[P:%.*]], null
+; ALL-NEXT:    ret i1 [[C]]
 ;
   %c = icmp ne i32* %P, null
   ret i1 %c
 }
 
 define i16 @test17(i1 %x) {
-; CHECK-LABEL: @test17(
-; CHECK-NEXT:    [[T86:%.*]] = zext i1 [[X:%.*]] to i16
-; CHECK-NEXT:    ret i16 [[T86]]
+; ALL-LABEL: @test17(
+; ALL-NEXT:    [[T86:%.*]] = zext i1 [[X:%.*]] to i16
+; ALL-NEXT:    ret i16 [[T86]]
 ;
   %c = zext i1 %x to i32
   %t86 = trunc i32 %c to i16
@@ -192,9 +192,9 @@ define i16 @test17(i1 %x) {
 }
 
 define i16 @test18(i8 %x) {
-; CHECK-LABEL: @test18(
-; CHECK-NEXT:    [[T86:%.*]] = sext i8 [[X:%.*]] to i16
-; CHECK-NEXT:    ret i16 [[T86]]
+; ALL-LABEL: @test18(
+; ALL-NEXT:    [[T86:%.*]] = sext i8 [[X:%.*]] to i16
+; ALL-NEXT:    ret i16 [[T86]]
 ;
   %c = sext i8 %x to i32
   %t86 = trunc i32 %c to i16
@@ -202,9 +202,9 @@ define i16 @test18(i8 %x) {
 }
 
 define i1 @test19(i32 %X) {
-; CHECK-LABEL: @test19(
-; CHECK-NEXT:    [[Z:%.*]] = icmp slt i32 [[X:%.*]], 12345
-; CHECK-NEXT:    ret i1 [[Z]]
+; ALL-LABEL: @test19(
+; ALL-NEXT:    [[Z:%.*]] = icmp slt i32 [[X:%.*]], 12345
+; ALL-NEXT:    ret i1 [[Z]]
 ;
   %c = sext i32 %X to i64
   %Z = icmp slt i64 %c, 12345
@@ -212,9 +212,9 @@ define i1 @test19(i32 %X) {
 }
 
 define <2 x i1> @test19vec(<2 x i32> %X) {
-; CHECK-LABEL: @test19vec(
-; CHECK-NEXT:    [[Z:%.*]] = icmp slt <2 x i32> [[X:%.*]], <i32 12345, i32 2147483647>
-; CHECK-NEXT:    ret <2 x i1> [[Z]]
+; ALL-LABEL: @test19vec(
+; ALL-NEXT:    [[Z:%.*]] = icmp slt <2 x i32> [[X:%.*]], <i32 12345, i32 2147483647>
+; ALL-NEXT:    ret <2 x i1> [[Z]]
 ;
   %c = sext <2 x i32> %X to <2 x i64>
   %Z = icmp slt <2 x i64> %c, <i64 12345, i64 2147483647>
@@ -222,9 +222,9 @@ define <2 x i1> @test19vec(<2 x i32> %X) {
 }
 
 define <3 x i1> @test19vec2(<3 x i1> %X) {
-; CHECK-LABEL: @test19vec2(
-; CHECK-NEXT:    [[CMPEQ:%.*]] = xor <3 x i1> [[X:%.*]], <i1 true, i1 true, i1 true>
-; CHECK-NEXT:    ret <3 x i1> [[CMPEQ]]
+; ALL-LABEL: @test19vec2(
+; ALL-NEXT:    [[CMPEQ:%.*]] = xor <3 x i1> [[X:%.*]], <i1 true, i1 true, i1 true>
+; ALL-NEXT:    ret <3 x i1> [[CMPEQ]]
 ;
   %sext = sext <3 x i1> %X to <3 x i32>
   %cmpeq = icmp eq <3 x i32> %sext, zeroinitializer
@@ -232,8 +232,8 @@ define <3 x i1> @test19vec2(<3 x i1> %X) {
 }
 
 define i1 @test20(i1 %B) {
-; CHECK-LABEL: @test20(
-; CHECK-NEXT:    ret i1 false
+; ALL-LABEL: @test20(
+; ALL-NEXT:    ret i1 false
 ;
   %c = zext i1 %B to i32
   %D = icmp slt i32 %c, -1
@@ -241,9 +241,9 @@ define i1 @test20(i1 %B) {
 }
 
 define i32 @test21(i32 %X) {
-; CHECK-LABEL: @test21(
-; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 255
-; CHECK-NEXT:    ret i32 [[TMP1]]
+; ALL-LABEL: @test21(
+; ALL-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 255
+; ALL-NEXT:    ret i32 [[TMP1]]
 ;
   %c1 = trunc i32 %X to i8
   %c2 = sext i8 %c1 to i32
@@ -252,9 +252,9 @@ define i32 @test21(i32 %X) {
 }
 
 define i32 @test22(i32 %X) {
-; CHECK-LABEL: @test22(
-; CHECK-NEXT:    [[SEXT:%.*]] = shl i32 [[X:%.*]], 24
-; CHECK-NEXT:    ret i32 [[SEXT]]
+; ALL-LABEL: @test22(
+; ALL-NEXT:    [[SEXT:%.*]] = shl i32 [[X:%.*]], 24
+; ALL-NEXT:    ret i32 [[SEXT]]
 ;
   %c1 = trunc i32 %X to i8
   %c2 = sext i8 %c1 to i32
@@ -263,9 +263,9 @@ define i32 @test22(i32 %X) {
 }
 
 define i32 @test23(i32 %X) {
-; CHECK-LABEL: @test23(
-; CHECK-NEXT:    [[C2:%.*]] = and i32 [[X:%.*]], 65535
-; CHECK-NEXT:    ret i32 [[C2]]
+; ALL-LABEL: @test23(
+; ALL-NEXT:    [[C2:%.*]] = and i32 [[X:%.*]], 65535
+; ALL-NEXT:    ret i32 [[C2]]
 ;
   %c1 = trunc i32 %X to i16
   %c2 = zext i16 %c1 to i32
@@ -273,8 +273,8 @@ define i32 @test23(i32 %X) {
 }
 
 define i1 @test24(i1 %C) {
-; CHECK-LABEL: @test24(
-; CHECK-NEXT:    ret i1 true
+; ALL-LABEL: @test24(
+; ALL-NEXT:    ret i1 true
 ;
   %X = select i1 %C, i32 14, i32 1234
   %c = icmp ne i32 %X, 0
@@ -282,9 +282,9 @@ define i1 @test24(i1 %C) {
 }
 
 define i32 @test26(float %F) {
-; CHECK-LABEL: @test26(
-; CHECK-NEXT:    [[D:%.*]] = fptosi float [[F:%.*]] to i32
-; CHECK-NEXT:    ret i32 [[D]]
+; ALL-LABEL: @test26(
+; ALL-NEXT:    [[D:%.*]] = fptosi float [[F:%.*]] to i32
+; ALL-NEXT:    ret i32 [[D]]
 ;
   %c = fpext float %F to double
   %D = fptosi double %c to i32
@@ -292,28 +292,28 @@ define i32 @test26(float %F) {
 }
 
 define [4 x float]* @test27([9 x [4 x float]]* %A) {
-; CHECK-LABEL: @test27(
-; CHECK-NEXT:    [[C:%.*]] = getelementptr inbounds [9 x [4 x float]], [9 x [4 x float]]* [[A:%.*]], i64 0, i64 0
-; CHECK-NEXT:    ret [4 x float]* [[C]]
+; ALL-LABEL: @test27(
+; ALL-NEXT:    [[C:%.*]] = getelementptr [9 x [4 x float]], [9 x [4 x float]]* [[A:%.*]], i64 0, i64 0
+; ALL-NEXT:    ret [4 x float]* [[C]]
 ;
   %c = bitcast [9 x [4 x float]]* %A to [4 x float]*
   ret [4 x float]* %c
 }
 
 define float* @test28([4 x float]* %A) {
-; CHECK-LABEL: @test28(
-; CHECK-NEXT:    [[C:%.*]] = getelementptr inbounds [4 x float], [4 x float]* [[A:%.*]], i64 0, i64 0
-; CHECK-NEXT:    ret float* [[C]]
+; ALL-LABEL: @test28(
+; ALL-NEXT:    [[C:%.*]] = getelementptr [4 x float], [4 x float]* [[A:%.*]], i64 0, i64 0
+; ALL-NEXT:    ret float* [[C]]
 ;
   %c = bitcast [4 x float]* %A to float*
   ret float* %c
 }
 
 define i32 @test29(i32 %c1, i32 %c2) {
-; CHECK-LABEL: @test29(
-; CHECK-NEXT:    [[T21:%.*]] = or i32 [[C2:%.*]], [[C1:%.*]]
-; CHECK-NEXT:    [[T10:%.*]] = and i32 [[T21]], 255
-; CHECK-NEXT:    ret i32 [[T10]]
+; ALL-LABEL: @test29(
+; ALL-NEXT:    [[T21:%.*]] = or i32 [[C2:%.*]], [[C1:%.*]]
+; ALL-NEXT:    [[T10:%.*]] = and i32 [[T21]], 255
+; ALL-NEXT:    ret i32 [[T10]]
 ;
   %t1 = trunc i32 %c1 to i8
   %tmask = trunc i32 %c2 to i8
@@ -323,10 +323,10 @@ define i32 @test29(i32 %c1, i32 %c2) {
 }
 
 define i32 @test30(i32 %c1) {
-; CHECK-LABEL: @test30(
-; CHECK-NEXT:    [[C3:%.*]] = and i32 [[C1:%.*]], 255
-; CHECK-NEXT:    [[C4:%.*]] = xor i32 [[C3]], 1
-; CHECK-NEXT:    ret i32 [[C4]]
+; ALL-LABEL: @test30(
+; ALL-NEXT:    [[C3:%.*]] = and i32 [[C1:%.*]], 255
+; ALL-NEXT:    [[C4:%.*]] = xor i32 [[C3]], 1
+; ALL-NEXT:    ret i32 [[C4]]
 ;
   %c2 = trunc i32 %c1 to i8
   %c3 = xor i8 %c2, 1
@@ -335,10 +335,10 @@ define i32 @test30(i32 %c1) {
 }
 
 define i1 @test31(i64 %A) {
-; CHECK-LABEL: @test31(
-; CHECK-NEXT:    [[C1:%.*]] = and i64 [[A:%.*]], 42
-; CHECK-NEXT:    [[D:%.*]] = icmp eq i64 [[C1]], 10
-; CHECK-NEXT:    ret i1 [[D]]
+; ALL-LABEL: @test31(
+; ALL-NEXT:    [[C1:%.*]] = and i64 [[A:%.*]], 42
+; ALL-NEXT:    [[D:%.*]] = icmp eq i64 [[C1]], 10
+; ALL-NEXT:    ret i1 [[D]]
 ;
   %B = trunc i64 %A to i32
   %C = and i32 %B, 42
@@ -349,11 +349,11 @@ define i1 @test31(i64 %A) {
 ; FIXME: Vectors should fold too...or not?
 ; Does this depend on the whether the source/dest types of the trunc are legal in the data layout?
 define <2 x i1> @test31vec(<2 x i64> %A) {
-; CHECK-LABEL: @test31vec(
-; CHECK-NEXT:    [[B:%.*]] = trunc <2 x i64> [[A:%.*]] to <2 x i32>
-; CHECK-NEXT:    [[C:%.*]] = and <2 x i32> [[B]], <i32 42, i32 42>
-; CHECK-NEXT:    [[D:%.*]] = icmp eq <2 x i32> [[C]], <i32 10, i32 10>
-; CHECK-NEXT:    ret <2 x i1> [[D]]
+; ALL-LABEL: @test31vec(
+; ALL-NEXT:    [[B:%.*]] = trunc <2 x i64> [[A:%.*]] to <2 x i32>
+; ALL-NEXT:    [[C:%.*]] = and <2 x i32> [[B]], <i32 42, i32 42>
+; ALL-NEXT:    [[D:%.*]] = icmp eq <2 x i32> [[C]], <i32 10, i32 10>
+; ALL-NEXT:    ret <2 x i1> [[D]]
 ;
   %B = trunc <2 x i64> %A to <2 x i32>
   %C = and <2 x i32> %B, <i32 42, i32 42>
@@ -365,10 +365,10 @@ define <2 x i1> @test31vec(<2 x i64> %A) {
 ; even for vectors. Earlier folds should ensure that the icmp(and(zext)) pattern never occurs.
 
 define <2 x i1> @test32vec(<2 x i8> %A) {
-; CHECK-LABEL: @test32vec(
-; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i8> [[A:%.*]], <i8 42, i8 42>
-; CHECK-NEXT:    [[D:%.*]] = icmp eq <2 x i8> [[TMP1]], <i8 10, i8 10>
-; CHECK-NEXT:    ret <2 x i1> [[D]]
+; ALL-LABEL: @test32vec(
+; ALL-NEXT:    [[TMP1:%.*]] = and <2 x i8> [[A:%.*]], <i8 42, i8 42>
+; ALL-NEXT:    [[D:%.*]] = icmp eq <2 x i8> [[TMP1]], <i8 10, i8 10>
+; ALL-NEXT:    ret <2 x i1> [[D]]
 ;
   %B = zext <2 x i8> %A to <2 x i16>
   %C = and <2 x i16> %B, <i16 42, i16 42>
@@ -377,8 +377,8 @@ define <2 x i1> @test32vec(<2 x i8> %A) {
 }
 
 define i32 @test33(i32 %c1) {
-; CHECK-LABEL: @test33(
-; CHECK-NEXT:    ret i32 [[C1:%.*]]
+; ALL-LABEL: @test33(
+; ALL-NEXT:    ret i32 [[C1:%.*]]
 ;
   %x = bitcast i32 %c1 to float
   %y = bitcast float %x to i32
@@ -386,9 +386,9 @@ define i32 @test33(i32 %c1) {
 }
 
 define i16 @test34(i16 %a) {
-; CHECK-LABEL: @test34(
-; CHECK-NEXT:    [[TMP1:%.*]] = lshr i16 [[A:%.*]], 8
-; CHECK-NEXT:    ret i16 [[TMP1]]
+; ALL-LABEL: @test34(
+; ALL-NEXT:    [[TMP1:%.*]] = lshr i16 [[A:%.*]], 8
+; ALL-NEXT:    ret i16 [[TMP1]]
 ;
   %c1 = zext i16 %a to i32
   %t21 = lshr i32 %c1, 8
@@ -397,9 +397,9 @@ define i16 @test34(i16 %a) {
 }
 
 define i16 @test35(i16 %a) {
-; CHECK-LABEL: @test35(
-; CHECK-NEXT:    [[T2:%.*]] = lshr i16 [[A:%.*]], 8
-; CHECK-NEXT:    ret i16 [[T2]]
+; ALL-LABEL: @test35(
+; ALL-NEXT:    [[T2:%.*]] = lshr i16 [[A:%.*]], 8
+; ALL-NEXT:    ret i16 [[T2]]
 ;
   %c1 = bitcast i16 %a to i16
   %t2 = lshr i16 %c1, 8
@@ -409,9 +409,9 @@ define i16 @test35(i16 %a) {
 
 ; rdar://6480391
 define i1 @test36(i32 %a) {
-; CHECK-LABEL: @test36(
-; CHECK-NEXT:    [[D:%.*]] = icmp sgt i32 [[A:%.*]], -1
-; CHECK-NEXT:    ret i1 [[D]]
+; ALL-LABEL: @test36(
+; ALL-NEXT:    [[D:%.*]] = icmp sgt i32 [[A:%.*]], -1
+; ALL-NEXT:    ret i1 [[D]]
 ;
   %b = lshr i32 %a, 31
   %c = trunc i32 %b to i8
@@ -420,9 +420,9 @@ define i1 @test36(i32 %a) {
 }
 
 define <2 x i1> @test36vec(<2 x i32> %a) {
-; CHECK-LABEL: @test36vec(
-; CHECK-NEXT:    [[D:%.*]] = icmp sgt <2 x i32> [[A:%.*]], <i32 -1, i32 -1>
-; CHECK-NEXT:    ret <2 x i1> [[D]]
+; ALL-LABEL: @test36vec(
+; ALL-NEXT:    [[D:%.*]] = icmp sgt <2 x i32> [[A:%.*]], <i32 -1, i32 -1>
+; ALL-NEXT:    ret <2 x i1> [[D]]
 ;
   %b = lshr <2 x i32> %a, <i32 31, i32 31>
   %c = trunc <2 x i32> %b to <2 x i8>
@@ -431,8 +431,8 @@ define <2 x i1> @test36vec(<2 x i32> %a) {
 }
 
 define i1 @test37(i32 %a) {
-; CHECK-LABEL: @test37(
-; CHECK-NEXT:    ret i1 false
+; ALL-LABEL: @test37(
+; ALL-NEXT:    ret i1 false
 ;
   %b = lshr i32 %a, 31
   %c = or i32 %b, 512
@@ -442,10 +442,10 @@ define i1 @test37(i32 %a) {
 }
 
 define i64 @test38(i32 %a) {
-; CHECK-LABEL: @test38(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne i32 [[A:%.*]], -2
-; CHECK-NEXT:    [[TMP2:%.*]] = zext i1 [[TMP1]] to i64
-; CHECK-NEXT:    ret i64 [[TMP2]]
+; ALL-LABEL: @test38(
+; ALL-NEXT:    [[TMP1:%.*]] = icmp ne i32 [[A:%.*]], -2
+; ALL-NEXT:    [[TMP2:%.*]] = zext i1 [[TMP1]] to i64
+; ALL-NEXT:    ret i64 [[TMP2]]
 ;
   %1 = icmp eq i32 %a, -2
   %2 = zext i1 %1 to i8
@@ -455,9 +455,9 @@ define i64 @test38(i32 %a) {
 }
 
 define i16 @test39(i16 %a) {
-; CHECK-LABEL: @test39(
-; CHECK-NEXT:    [[REV:%.*]] = call i16 @llvm.bswap.i16(i16 [[A:%.*]])
-; CHECK-NEXT:    ret i16 [[REV]]
+; ALL-LABEL: @test39(
+; ALL-NEXT:    [[REV:%.*]] = call i16 @llvm.bswap.i16(i16 [[A:%.*]])
+; ALL-NEXT:    ret i16 [[REV]]
 ;
   %t = zext i16 %a to i32
   %t21 = lshr i32 %t, 8
@@ -468,11 +468,11 @@ define i16 @test39(i16 %a) {
 }
 
 define i16 @test40(i16 %a) {
-; CHECK-LABEL: @test40(
-; CHECK-NEXT:    [[T21:%.*]] = lshr i16 [[A:%.*]], 9
-; CHECK-NEXT:    [[T5:%.*]] = shl i16 [[A]], 8
-; CHECK-NEXT:    [[T32:%.*]] = or i16 [[T21]], [[T5]]
-; CHECK-NEXT:    ret i16 [[T32]]
+; ALL-LABEL: @test40(
+; ALL-NEXT:    [[T21:%.*]] = lshr i16 [[A:%.*]], 9
+; ALL-NEXT:    [[T5:%.*]] = shl i16 [[A]], 8
+; ALL-NEXT:    [[T32:%.*]] = or i16 [[T21]], [[T5]]
+; ALL-NEXT:    ret i16 [[T32]]
 ;
   %t = zext i16 %a to i32
   %t21 = lshr i32 %t, 9
@@ -483,11 +483,11 @@ define i16 @test40(i16 %a) {
 }
 
 define <2 x i16> @test40vec(<2 x i16> %a) {
-; CHECK-LABEL: @test40vec(
-; CHECK-NEXT:    [[T21:%.*]] = lshr <2 x i16> [[A:%.*]], <i16 9, i16 9>
-; CHECK-NEXT:    [[T5:%.*]] = shl <2 x i16> [[A]], <i16 8, i16 8>
-; CHECK-NEXT:    [[T32:%.*]] = or <2 x i16> [[T21]], [[T5]]
-; CHECK-NEXT:    ret <2 x i16> [[T32]]
+; ALL-LABEL: @test40vec(
+; ALL-NEXT:    [[T21:%.*]] = lshr <2 x i16> [[A:%.*]], <i16 9, i16 9>
+; ALL-NEXT:    [[T5:%.*]] = shl <2 x i16> [[A]], <i16 8, i16 8>
+; ALL-NEXT:    [[T32:%.*]] = or <2 x i16> [[T21]], [[T5]]
+; ALL-NEXT:    ret <2 x i16> [[T32]]
 ;
   %t = zext <2 x i16> %a to <2 x i32>
   %t21 = lshr <2 x i32> %t, <i32 9, i32 9>
@@ -499,8 +499,8 @@ define <2 x i16> @test40vec(<2 x i16> %a) {
 
 ; PR1263
 define i32* @test41(i32* %t1) {
-; CHECK-LABEL: @test41(
-; CHECK-NEXT:    ret i32* [[T1:%.*]]
+; ALL-LABEL: @test41(
+; ALL-NEXT:    ret i32* [[T1:%.*]]
 ;
   %t64 = bitcast i32* %t1 to { i32 }*
   %t65 = getelementptr { i32 }, { i32 }* %t64, i32 0, i32 0
@@ -508,9 +508,9 @@ define i32* @test41(i32* %t1) {
 }
 
 define i32 addrspace(1)* @test41_addrspacecast_smaller(i32* %t1) {
-; CHECK-LABEL: @test41_addrspacecast_smaller(
-; CHECK-NEXT:    [[T65:%.*]] = addrspacecast i32* [[T1:%.*]] to i32 addrspace(1)*
-; CHECK-NEXT:    ret i32 addrspace(1)* [[T65]]
+; ALL-LABEL: @test41_addrspacecast_smaller(
+; ALL-NEXT:    [[T65:%.*]] = addrspacecast i32* [[T1:%.*]] to i32 addrspace(1)*
+; ALL-NEXT:    ret i32 addrspace(1)* [[T65]]
 ;
   %t64 = addrspacecast i32* %t1 to { i32 } addrspace(1)*
   %t65 = getelementptr { i32 }, { i32 } addrspace(1)* %t64, i32 0, i32 0
@@ -518,9 +518,9 @@ define i32 addrspace(1)* @test41_addrspacecast_smaller(i32* %t1) {
 }
 
 define i32* @test41_addrspacecast_larger(i32 addrspace(1)* %t1) {
-; CHECK-LABEL: @test41_addrspacecast_larger(
-; CHECK-NEXT:    [[T65:%.*]] = addrspacecast i32 addrspace(1)* [[T1:%.*]] to i32*
-; CHECK-NEXT:    ret i32* [[T65]]
+; ALL-LABEL: @test41_addrspacecast_larger(
+; ALL-NEXT:    [[T65:%.*]] = addrspacecast i32 addrspace(1)* [[T1:%.*]] to i32*
+; ALL-NEXT:    ret i32* [[T65]]
 ;
   %t64 = addrspacecast i32 addrspace(1)* %t1 to { i32 }*
   %t65 = getelementptr { i32 }, { i32 }* %t64, i32 0, i32 0
@@ -528,9 +528,9 @@ define i32* @test41_addrspacecast_larger(i32 addrspace(1)* %t1) {
 }
 
 define i32 @test42(i32 %X) {
-; CHECK-LABEL: @test42(
-; CHECK-NEXT:    [[Z:%.*]] = and i32 [[X:%.*]], 255
-; CHECK-NEXT:    ret i32 [[Z]]
+; ALL-LABEL: @test42(
+; ALL-NEXT:    [[Z:%.*]] = and i32 [[X:%.*]], 255
+; ALL-NEXT:    ret i32 [[Z]]
 ;
   %Y = trunc i32 %X to i8
   %Z = zext i8 %Y to i32
@@ -539,10 +539,10 @@ define i32 @test42(i32 %X) {
 
 ; rdar://6598839
 define zeroext i64 @test43(i8 zeroext %on_off) {
-; CHECK-LABEL: @test43(
-; CHECK-NEXT:    [[A:%.*]] = zext i8 [[ON_OFF:%.*]] to i64
-; CHECK-NEXT:    [[B:%.*]] = add nsw i64 [[A]], -1
-; CHECK-NEXT:    ret i64 [[B]]
+; ALL-LABEL: @test43(
+; ALL-NEXT:    [[A:%.*]] = zext i8 [[ON_OFF:%.*]] to i64
+; ALL-NEXT:    [[B:%.*]] = add nsw i64 [[A]], -1
+; ALL-NEXT:    ret i64 [[B]]
 ;
   %A = zext i8 %on_off to i32
   %B = add i32 %A, -1
@@ -551,10 +551,10 @@ define zeroext i64 @test43(i8 zeroext %on_off) {
 }
 
 define i64 @test44(i8 %T) {
-; CHECK-LABEL: @test44(
-; CHECK-NEXT:    [[A:%.*]] = zext i8 [[T:%.*]] to i64
-; CHECK-NEXT:    [[B:%.*]] = or i64 [[A]], 1234
-; CHECK-NEXT:    ret i64 [[B]]
+; ALL-LABEL: @test44(
+; ALL-NEXT:    [[A:%.*]] = zext i8 [[T:%.*]] to i64
+; ALL-NEXT:    [[B:%.*]] = or i64 [[A]], 1234
+; ALL-NEXT:    ret i64 [[B]]
 ;
   %A = zext i8 %T to i16
   %B = or i16 %A, 1234
@@ -563,11 +563,11 @@ define i64 @test44(i8 %T) {
 }
 
 define i64 @test45(i8 %A, i64 %Q) {
-; CHECK-LABEL: @test45(
-; CHECK-NEXT:    [[B:%.*]] = sext i8 [[A:%.*]] to i64
-; CHECK-NEXT:    [[C:%.*]] = or i64 [[B]], [[Q:%.*]]
-; CHECK-NEXT:    [[E:%.*]] = and i64 [[C]], 4294967295
-; CHECK-NEXT:    ret i64 [[E]]
+; ALL-LABEL: @test45(
+; ALL-NEXT:    [[B:%.*]] = sext i8 [[A:%.*]] to i64
+; ALL-NEXT:    [[C:%.*]] = or i64 [[B]], [[Q:%.*]]
+; ALL-NEXT:    [[E:%.*]] = and i64 [[C]], 4294967295
+; ALL-NEXT:    ret i64 [[E]]
 ;
   %D = trunc i64 %Q to i32  ;; should be removed
   %B = sext i8 %A to i32
@@ -578,10 +578,10 @@ define i64 @test45(i8 %A, i64 %Q) {
 
 
 define i64 @test46(i64 %A) {
-; CHECK-LABEL: @test46(
-; CHECK-NEXT:    [[C:%.*]] = shl i64 [[A:%.*]], 8
-; CHECK-NEXT:    [[D:%.*]] = and i64 [[C]], 10752
-; CHECK-NEXT:    ret i64 [[D]]
+; ALL-LABEL: @test46(
+; ALL-NEXT:    [[C:%.*]] = shl i64 [[A:%.*]], 8
+; ALL-NEXT:    [[D:%.*]] = and i64 [[C]], 10752
+; ALL-NEXT:    ret i64 [[D]]
 ;
   %B = trunc i64 %A to i32
   %C = and i32 %B, 42
@@ -591,12 +591,12 @@ define i64 @test46(i64 %A) {
 }
 
 define <2 x i64> @test46vec(<2 x i64> %A) {
-; CHECK-LABEL: @test46vec(
-; CHECK-NEXT:    [[B:%.*]] = trunc <2 x i64> [[A:%.*]] to <2 x i32>
-; CHECK-NEXT:    [[C:%.*]] = shl <2 x i32> [[B]], <i32 8, i32 8>
-; CHECK-NEXT:    [[D:%.*]] = and <2 x i32> [[C]], <i32 10752, i32 10752>
-; CHECK-NEXT:    [[E:%.*]] = zext <2 x i32> [[D]] to <2 x i64>
-; CHECK-NEXT:    ret <2 x i64> [[E]]
+; ALL-LABEL: @test46vec(
+; ALL-NEXT:    [[B:%.*]] = trunc <2 x i64> [[A:%.*]] to <2 x i32>
+; ALL-NEXT:    [[C:%.*]] = shl <2 x i32> [[B]], <i32 8, i32 8>
+; ALL-NEXT:    [[D:%.*]] = and <2 x i32> [[C]], <i32 10752, i32 10752>
+; ALL-NEXT:    [[E:%.*]] = zext <2 x i32> [[D]] to <2 x i64>
+; ALL-NEXT:    ret <2 x i64> [[E]]
 ;
   %B = trunc <2 x i64> %A to <2 x i32>
   %C = and <2 x i32> %B, <i32 42, i32 42>
@@ -606,11 +606,11 @@ define <2 x i64> @test46vec(<2 x i64> %A) {
 }
 
 define i64 @test47(i8 %A) {
-; CHECK-LABEL: @test47(
-; CHECK-NEXT:    [[TMP1:%.*]] = or i8 [[A:%.*]], 42
-; CHECK-NEXT:    [[C:%.*]] = sext i8 [[TMP1]] to i64
-; CHECK-NEXT:    [[E:%.*]] = and i64 [[C]], 4294967295
-; CHECK-NEXT:    ret i64 [[E]]
+; ALL-LABEL: @test47(
+; ALL-NEXT:    [[TMP1:%.*]] = or i8 [[A:%.*]], 42
+; ALL-NEXT:    [[C:%.*]] = sext i8 [[TMP1]] to i64
+; ALL-NEXT:    [[E:%.*]] = and i64 [[C]], 4294967295
+; ALL-NEXT:    ret i64 [[E]]
 ;
   %B = sext i8 %A to i32
   %C = or i32 %B, 42
@@ -619,12 +619,12 @@ define i64 @test47(i8 %A) {
 }
 
 define i64 @test48(i8 %A1, i8 %a2) {
-; CHECK-LABEL: @test48(
-; CHECK-NEXT:    [[Z2:%.*]] = zext i8 [[A1:%.*]] to i32
-; CHECK-NEXT:    [[C:%.*]] = shl nuw nsw i32 [[Z2]], 8
-; CHECK-NEXT:    [[D:%.*]] = or i32 [[C]], [[Z2]]
-; CHECK-NEXT:    [[E:%.*]] = zext i32 [[D]] to i64
-; CHECK-NEXT:    ret i64 [[E]]
+; ALL-LABEL: @test48(
+; ALL-NEXT:    [[Z2:%.*]] = zext i8 [[A1:%.*]] to i32
+; ALL-NEXT:    [[C:%.*]] = shl nuw nsw i32 [[Z2]], 8
+; ALL-NEXT:    [[D:%.*]] = or i32 [[C]], [[Z2]]
+; ALL-NEXT:    [[E:%.*]] = zext i32 [[D]] to i64
+; ALL-NEXT:    ret i64 [[E]]
 ;
   %Z1 = zext i8 %a2 to i32
   %Z2 = zext i8 %A1 to i32
@@ -635,11 +635,11 @@ define i64 @test48(i8 %A1, i8 %a2) {
 }
 
 define i64 @test49(i64 %A) {
-; CHECK-LABEL: @test49(
-; CHECK-NEXT:    [[C:%.*]] = shl i64 [[A:%.*]], 32
-; CHECK-NEXT:    [[SEXT:%.*]] = ashr exact i64 [[C]], 32
-; CHECK-NEXT:    [[D:%.*]] = or i64 [[SEXT]], 1
-; CHECK-NEXT:    ret i64 [[D]]
+; ALL-LABEL: @test49(
+; ALL-NEXT:    [[C:%.*]] = shl i64 [[A:%.*]], 32
+; ALL-NEXT:    [[SEXT:%.*]] = ashr exact i64 [[C]], 32
+; ALL-NEXT:    [[D:%.*]] = or i64 [[SEXT]], 1
+; ALL-NEXT:    ret i64 [[D]]
 ;
   %B = trunc i64 %A to i32
   %C = or i32 %B, 1
@@ -648,12 +648,12 @@ define i64 @test49(i64 %A) {
 }
 
 define i64 @test50(i64 %x) {
-; CHECK-LABEL: @test50(
-; CHECK-NEXT:    [[A:%.*]] = lshr i64 [[X:%.*]], 2
-; CHECK-NEXT:    [[D:%.*]] = shl i64 [[A]], 32
-; CHECK-NEXT:    [[SEXT:%.*]] = add i64 [[D]], -4294967296
-; CHECK-NEXT:    [[E:%.*]] = ashr exact i64 [[SEXT]], 32
-; CHECK-NEXT:    ret i64 [[E]]
+; ALL-LABEL: @test50(
+; ALL-NEXT:    [[A:%.*]] = lshr i64 [[X:%.*]], 2
+; ALL-NEXT:    [[D:%.*]] = shl i64 [[A]], 32
+; ALL-NEXT:    [[SEXT:%.*]] = add i64 [[D]], -4294967296
+; ALL-NEXT:    [[E:%.*]] = ashr exact i64 [[SEXT]], 32
+; ALL-NEXT:    ret i64 [[E]]
 ;
   %a = lshr i64 %x, 2
   %B = trunc i64 %a to i32
@@ -664,13 +664,13 @@ define i64 @test50(i64 %x) {
 }
 
 define i64 @test51(i64 %A, i1 %cond) {
-; CHECK-LABEL: @test51(
-; CHECK-NEXT:    [[C:%.*]] = and i64 [[A:%.*]], 4294967294
-; CHECK-NEXT:    [[D:%.*]] = or i64 [[A]], 1
-; CHECK-NEXT:    [[E:%.*]] = select i1 [[COND:%.*]], i64 [[C]], i64 [[D]]
-; CHECK-NEXT:    [[SEXT:%.*]] = shl i64 [[E]], 32
-; CHECK-NEXT:    [[F:%.*]] = ashr exact i64 [[SEXT]], 32
-; CHECK-NEXT:    ret i64 [[F]]
+; ALL-LABEL: @test51(
+; ALL-NEXT:    [[C:%.*]] = and i64 [[A:%.*]], 4294967294
+; ALL-NEXT:    [[D:%.*]] = or i64 [[A]], 1
+; ALL-NEXT:    [[E:%.*]] = select i1 [[COND:%.*]], i64 [[C]], i64 [[D]]
+; ALL-NEXT:    [[SEXT:%.*]] = shl i64 [[E]], 32
+; ALL-NEXT:    [[F:%.*]] = ashr exact i64 [[SEXT]], 32
+; ALL-NEXT:    ret i64 [[F]]
 ;
   %B = trunc i64 %A to i32
   %C = and i32 %B, -2
@@ -681,11 +681,11 @@ define i64 @test51(i64 %A, i1 %cond) {
 }
 
 define i32 @test52(i64 %A) {
-; CHECK-LABEL: @test52(
-; CHECK-NEXT:    [[B:%.*]] = trunc i64 [[A:%.*]] to i32
-; CHECK-NEXT:    [[C:%.*]] = and i32 [[B]], 7224
-; CHECK-NEXT:    [[D:%.*]] = or i32 [[C]], 32962
-; CHECK-NEXT:    ret i32 [[D]]
+; ALL-LABEL: @test52(
+; ALL-NEXT:    [[B:%.*]] = trunc i64 [[A:%.*]] to i32
+; ALL-NEXT:    [[C:%.*]] = and i32 [[B]], 7224
+; ALL-NEXT:    [[D:%.*]] = or i32 [[C]], 32962
+; ALL-NEXT:    ret i32 [[D]]
 ;
   %B = trunc i64 %A to i16
   %C = or i16 %B, -32574
@@ -695,11 +695,11 @@ define i32 @test52(i64 %A) {
 }
 
 define i64 @test53(i32 %A) {
-; CHECK-LABEL: @test53(
-; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[A:%.*]], 7224
-; CHECK-NEXT:    [[TMP2:%.*]] = or i32 [[TMP1]], 32962
-; CHECK-NEXT:    [[D:%.*]] = zext i32 [[TMP2]] to i64
-; CHECK-NEXT:    ret i64 [[D]]
+; ALL-LABEL: @test53(
+; ALL-NEXT:    [[TMP1:%.*]] = and i32 [[A:%.*]], 7224
+; ALL-NEXT:    [[TMP2:%.*]] = or i32 [[TMP1]], 32962
+; ALL-NEXT:    [[D:%.*]] = zext i32 [[TMP2]] to i64
+; ALL-NEXT:    ret i64 [[D]]
 ;
   %B = trunc i32 %A to i16
   %C = or i16 %B, -32574
@@ -709,11 +709,11 @@ define i64 @test53(i32 %A) {
 }
 
 define i32 @test54(i64 %A) {
-; CHECK-LABEL: @test54(
-; CHECK-NEXT:    [[B:%.*]] = trunc i64 [[A:%.*]] to i32
-; CHECK-NEXT:    [[C:%.*]] = and i32 [[B]], 7224
-; CHECK-NEXT:    [[D:%.*]] = or i32 [[C]], -32574
-; CHECK-NEXT:    ret i32 [[D]]
+; ALL-LABEL: @test54(
+; ALL-NEXT:    [[B:%.*]] = trunc i64 [[A:%.*]] to i32
+; ALL-NEXT:    [[C:%.*]] = and i32 [[B]], 7224
+; ALL-NEXT:    [[D:%.*]] = or i32 [[C]], -32574
+; ALL-NEXT:    ret i32 [[D]]
 ;
   %B = trunc i64 %A to i16
   %C = or i16 %B, -32574
@@ -723,11 +723,11 @@ define i32 @test54(i64 %A) {
 }
 
 define i64 @test55(i32 %A) {
-; CHECK-LABEL: @test55(
-; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[A:%.*]], 7224
-; CHECK-NEXT:    [[C:%.*]] = zext i32 [[TMP1]] to i64
-; CHECK-NEXT:    [[D:%.*]] = or i64 [[C]], -32574
-; CHECK-NEXT:    ret i64 [[D]]
+; ALL-LABEL: @test55(
+; ALL-NEXT:    [[TMP1:%.*]] = and i32 [[A:%.*]], 7224
+; ALL-NEXT:    [[C:%.*]] = zext i32 [[TMP1]] to i64
+; ALL-NEXT:    [[D:%.*]] = or i64 [[C]], -32574
+; ALL-NEXT:    ret i64 [[D]]
 ;
   %B = trunc i32 %A to i16
   %C = or i16 %B, -32574
@@ -737,11 +737,11 @@ define i64 @test55(i32 %A) {
 }
 
 define i64 @test56(i16 %A) {
-; CHECK-LABEL: @test56(
-; CHECK-NEXT:    [[P353:%.*]] = sext i16 [[A:%.*]] to i64
-; CHECK-NEXT:    [[P354:%.*]] = lshr i64 [[P353]], 5
-; CHECK-NEXT:    [[P355:%.*]] = and i64 [[P354]], 134217727
-; CHECK-NEXT:    ret i64 [[P355]]
+; ALL-LABEL: @test56(
+; ALL-NEXT:    [[P353:%.*]] = sext i16 [[A:%.*]] to i64
+; ALL-NEXT:    [[P354:%.*]] = lshr i64 [[P353]], 5
+; ALL-NEXT:    [[P355:%.*]] = and i64 [[P354]], 134217727
+; ALL-NEXT:    ret i64 [[P355]]
 ;
   %p353 = sext i16 %A to i32
   %p354 = lshr i32 %p353, 5
@@ -750,11 +750,11 @@ define i64 @test56(i16 %A) {
 }
 
 define <2 x i64> @test56vec(<2 x i16> %A) {
-; CHECK-LABEL: @test56vec(
-; CHECK-NEXT:    [[P353:%.*]] = sext <2 x i16> [[A:%.*]] to <2 x i32>
-; CHECK-NEXT:    [[P354:%.*]] = lshr <2 x i32> [[P353]], <i32 5, i32 5>
-; CHECK-NEXT:    [[P355:%.*]] = zext <2 x i32> [[P354]] to <2 x i64>
-; CHECK-NEXT:    ret <2 x i64> [[P355]]
+; ALL-LABEL: @test56vec(
+; ALL-NEXT:    [[P353:%.*]] = sext <2 x i16> [[A:%.*]] to <2 x i32>
+; ALL-NEXT:    [[P354:%.*]] = lshr <2 x i32> [[P353]], <i32 5, i32 5>
+; ALL-NEXT:    [[P355:%.*]] = zext <2 x i32> [[P354]] to <2 x i64>
+; ALL-NEXT:    ret <2 x i64> [[P355]]
 ;
   %p353 = sext <2 x i16> %A to <2 x i32>
   %p354 = lshr <2 x i32> %p353, <i32 5, i32 5>
@@ -763,10 +763,10 @@ define <2 x i64> @test56vec(<2 x i16> %A) {
 }
 
 define i64 @test57(i64 %A) {
-; CHECK-LABEL: @test57(
-; CHECK-NEXT:    [[C:%.*]] = lshr i64 [[A:%.*]], 8
-; CHECK-NEXT:    [[E:%.*]] = and i64 [[C]], 16777215
-; CHECK-NEXT:    ret i64 [[E]]
+; ALL-LABEL: @test57(
+; ALL-NEXT:    [[C:%.*]] = lshr i64 [[A:%.*]], 8
+; ALL-NEXT:    [[E:%.*]] = and i64 [[C]], 16777215
+; ALL-NEXT:    ret i64 [[E]]
 ;
   %B = trunc i64 %A to i32
   %C = lshr i32 %B, 8
@@ -775,11 +775,11 @@ define i64 @test57(i64 %A) {
 }
 
 define <2 x i64> @test57vec(<2 x i64> %A) {
-; CHECK-LABEL: @test57vec(
-; CHECK-NEXT:    [[B:%.*]] = trunc <2 x i64> [[A:%.*]] to <2 x i32>
-; CHECK-NEXT:    [[C:%.*]] = lshr <2 x i32> [[B]], <i32 8, i32 8>
-; CHECK-NEXT:    [[E:%.*]] = zext <2 x i32> [[C]] to <2 x i64>
-; CHECK-NEXT:    ret <2 x i64> [[E]]
+; ALL-LABEL: @test57vec(
+; ALL-NEXT:    [[B:%.*]] = trunc <2 x i64> [[A:%.*]] to <2 x i32>
+; ALL-NEXT:    [[C:%.*]] = lshr <2 x i32> [[B]], <i32 8, i32 8>
+; ALL-NEXT:    [[E:%.*]] = zext <2 x i32> [[C]] to <2 x i64>
+; ALL-NEXT:    ret <2 x i64> [[E]]
 ;
   %B = trunc <2 x i64> %A to <2 x i32>
   %C = lshr <2 x i32> %B, <i32 8, i32 8>
@@ -788,11 +788,11 @@ define <2 x i64> @test57vec(<2 x i64> %A) {
 }
 
 define i64 @test58(i64 %A) {
-; CHECK-LABEL: @test58(
-; CHECK-NEXT:    [[C:%.*]] = lshr i64 [[A:%.*]], 8
-; CHECK-NEXT:    [[D:%.*]] = and i64 [[C]], 16777087
-; CHECK-NEXT:    [[E:%.*]] = or i64 [[D]], 128
-; CHECK-NEXT:    ret i64 [[E]]
+; ALL-LABEL: @test58(
+; ALL-NEXT:    [[C:%.*]] = lshr i64 [[A:%.*]], 8
+; ALL-NEXT:    [[D:%.*]] = and i64 [[C]], 16777087
+; ALL-NEXT:    [[E:%.*]] = or i64 [[D]], 128
+; ALL-NEXT:    ret i64 [[E]]
 ;
   %B = trunc i64 %A to i32
   %C = lshr i32 %B, 8
@@ -803,14 +803,14 @@ define i64 @test58(i64 %A) {
 }
 
 define i64 @test59(i8 %A, i8 %B) {
-; CHECK-LABEL: @test59(
-; CHECK-NEXT:    [[C:%.*]] = zext i8 [[A:%.*]] to i64
-; CHECK-NEXT:    [[D:%.*]] = shl nuw nsw i64 [[C]], 4
-; CHECK-NEXT:    [[E:%.*]] = and i64 [[D]], 48
-; CHECK-NEXT:    [[TMP1:%.*]] = lshr i8 [[B:%.*]], 4
-; CHECK-NEXT:    [[G:%.*]] = zext i8 [[TMP1]] to i64
-; CHECK-NEXT:    [[H:%.*]] = or i64 [[E]], [[G]]
-; CHECK-NEXT:    ret i64 [[H]]
+; ALL-LABEL: @test59(
+; ALL-NEXT:    [[C:%.*]] = zext i8 [[A:%.*]] to i64
+; ALL-NEXT:    [[D:%.*]] = shl nuw nsw i64 [[C]], 4
+; ALL-NEXT:    [[E:%.*]] = and i64 [[D]], 48
+; ALL-NEXT:    [[TMP1:%.*]] = lshr i8 [[B:%.*]], 4
+; ALL-NEXT:    [[G:%.*]] = zext i8 [[TMP1]] to i64
+; ALL-NEXT:    [[H:%.*]] = or i64 [[E]], [[G]]
+; ALL-NEXT:    ret i64 [[H]]
 ;
   %C = zext i8 %A to i32
   %D = shl i32 %C, 4
@@ -823,9 +823,13 @@ define i64 @test59(i8 %A, i8 %B) {
 }
 
 define <3 x i32> @test60(<4 x i32> %call4) {
-; CHECK-LABEL: @test60(
-; CHECK-NEXT:    [[P10:%.*]] = shufflevector <4 x i32> [[CALL4:%.*]], <4 x i32> undef, <3 x i32> <i32 0, i32 1, i32 2>
-; CHECK-NEXT:    ret <3 x i32> [[P10]]
+; BE-LABEL: @test60(
+; BE-NEXT:    [[P10:%.*]] = shufflevector <4 x i32> [[CALL4:%.*]], <4 x i32> undef, <3 x i32> <i32 1, i32 2, i32 3>
+; BE-NEXT:    ret <3 x i32> [[P10]]
+;
+; LE-LABEL: @test60(
+; LE-NEXT:    [[P10:%.*]] = shufflevector <4 x i32> [[CALL4:%.*]], <4 x i32> undef, <3 x i32> <i32 0, i32 1, i32 2>
+; LE-NEXT:    ret <3 x i32> [[P10]]
 ;
   %p11 = bitcast <4 x i32> %call4 to i128
   %p9 = trunc i128 %p11 to i96
@@ -835,9 +839,13 @@ define <3 x i32> @test60(<4 x i32> %call4) {
 }
 
 define <4 x i32> @test61(<3 x i32> %call4) {
-; CHECK-LABEL: @test61(
-; CHECK-NEXT:    [[P10:%.*]] = shufflevector <3 x i32> [[CALL4:%.*]], <3 x i32> <i32 0, i32 undef, i32 undef>, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT:    ret <4 x i32> [[P10]]
+; BE-LABEL: @test61(
+; BE-NEXT:    [[P10:%.*]] = shufflevector <3 x i32> [[CALL4:%.*]], <3 x i32> <i32 0, i32 undef, i32 undef>, <4 x i32> <i32 3, i32 0, i32 1, i32 2>
+; BE-NEXT:    ret <4 x i32> [[P10]]
+;
+; LE-LABEL: @test61(
+; LE-NEXT:    [[P10:%.*]] = shufflevector <3 x i32> [[CALL4:%.*]], <3 x i32> <i32 0, i32 undef, i32 undef>, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; LE-NEXT:    ret <4 x i32> [[P10]]
 ;
   %p11 = bitcast <3 x i32> %call4 to i96
   %p9 = zext i96 %p11 to i128
@@ -846,10 +854,15 @@ define <4 x i32> @test61(<3 x i32> %call4) {
 }
 
 define <4 x i32> @test62(<3 x float> %call4) {
-; CHECK-LABEL: @test62(
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <3 x float> [[CALL4:%.*]] to <3 x i32>
-; CHECK-NEXT:    [[P10:%.*]] = shufflevector <3 x i32> [[TMP1]], <3 x i32> <i32 0, i32 undef, i32 undef>, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT:    ret <4 x i32> [[P10]]
+; BE-LABEL: @test62(
+; BE-NEXT:    [[TMP1:%.*]] = bitcast <3 x float> [[CALL4:%.*]] to <3 x i32>
+; BE-NEXT:    [[P10:%.*]] = shufflevector <3 x i32> [[TMP1]], <3 x i32> <i32 0, i32 undef, i32 undef>, <4 x i32> <i32 3, i32 0, i32 1, i32 2>
+; BE-NEXT:    ret <4 x i32> [[P10]]
+;
+; LE-LABEL: @test62(
+; LE-NEXT:    [[TMP1:%.*]] = bitcast <3 x float> [[CALL4:%.*]] to <3 x i32>
+; LE-NEXT:    [[P10:%.*]] = shufflevector <3 x i32> [[TMP1]], <3 x i32> <i32 0, i32 undef, i32 undef>, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; LE-NEXT:    ret <4 x i32> [[P10]]
 ;
   %p11 = bitcast <3 x float> %call4 to i96
   %p9 = zext i96 %p11 to i128
@@ -859,10 +872,10 @@ define <4 x i32> @test62(<3 x float> %call4) {
 
 ; PR7311 - Don't create invalid IR on scalar->vector cast.
 define <2 x float> @test63(i64 %t8) {
-; CHECK-LABEL: @test63(
-; CHECK-NEXT:    [[A:%.*]] = bitcast i64 [[T8:%.*]] to <2 x i32>
-; CHECK-NEXT:    [[VCVT_I:%.*]] = uitofp <2 x i32> [[A]] to <2 x float>
-; CHECK-NEXT:    ret <2 x float> [[VCVT_I]]
+; ALL-LABEL: @test63(
+; ALL-NEXT:    [[A:%.*]] = bitcast i64 [[T8:%.*]] to <2 x i32>
+; ALL-NEXT:    [[VCVT_I:%.*]] = uitofp <2 x i32> [[A]] to <2 x float>
+; ALL-NEXT:    ret <2 x float> [[VCVT_I]]
 ;
   %a = bitcast i64 %t8 to <2 x i32>
   %vcvt.i = uitofp <2 x i32> %a to <2 x float>
@@ -870,8 +883,8 @@ define <2 x float> @test63(i64 %t8) {
 }
 
 define <4 x float> @test64(<4 x float> %c) {
-; CHECK-LABEL: @test64(
-; CHECK-NEXT:    ret <4 x float> [[C:%.*]]
+; ALL-LABEL: @test64(
+; ALL-NEXT:    ret <4 x float> [[C:%.*]]
 ;
   %t0 = bitcast <4 x float> %c to <4 x i32>
   %t1 = bitcast <4 x i32> %t0 to <4 x float>
@@ -879,8 +892,8 @@ define <4 x float> @test64(<4 x float> %c) {
 }
 
 define <4 x float> @test65(<4 x float> %c) {
-; CHECK-LABEL: @test65(
-; CHECK-NEXT:    ret <4 x float> [[C:%.*]]
+; ALL-LABEL: @test65(
+; ALL-NEXT:    ret <4 x float> [[C:%.*]]
 ;
   %t0 = bitcast <4 x float> %c to <2 x double>
   %t1 = bitcast <2 x double> %t0 to <4 x float>
@@ -888,8 +901,8 @@ define <4 x float> @test65(<4 x float> %c) {
 }
 
 define <2 x float> @test66(<2 x float> %c) {
-; CHECK-LABEL: @test66(
-; CHECK-NEXT:    ret <2 x float> [[C:%.*]]
+; ALL-LABEL: @test66(
+; ALL-NEXT:    ret <2 x float> [[C:%.*]]
 ;
   %t0 = bitcast <2 x float> %c to double
   %t1 = bitcast double %t0 to <2 x float>
@@ -897,16 +910,16 @@ define <2 x float> @test66(<2 x float> %c) {
 }
 
 define float @test2c() {
-; CHECK-LABEL: @test2c(
-; CHECK-NEXT:    ret float -1.000000e+00
+; ALL-LABEL: @test2c(
+; ALL-NEXT:    ret float -1.000000e+00
 ;
   ret float extractelement (<2 x float> bitcast (double bitcast (<2 x float> <float -1.000000e+00, float -1.000000e+00> to double) to <2 x float>), i32 0)
 }
 
 define i64 @test_mmx(<2 x i32> %x) {
-; CHECK-LABEL: @test_mmx(
-; CHECK-NEXT:    [[C:%.*]] = bitcast <2 x i32> [[X:%.*]] to i64
-; CHECK-NEXT:    ret i64 [[C]]
+; ALL-LABEL: @test_mmx(
+; ALL-NEXT:    [[C:%.*]] = bitcast <2 x i32> [[X:%.*]] to i64
+; ALL-NEXT:    ret i64 [[C]]
 ;
   %A = bitcast <2 x i32> %x to x86_mmx
   %B = bitcast x86_mmx %A to <2 x i32>
@@ -915,8 +928,8 @@ define i64 @test_mmx(<2 x i32> %x) {
 }
 
 define i64 @test_mmx_const(<2 x i32> %c) {
-; CHECK-LABEL: @test_mmx_const(
-; CHECK-NEXT:    ret i64 0
+; ALL-LABEL: @test_mmx_const(
+; ALL-NEXT:    ret i64 0
 ;
   %A = bitcast <2 x i32> zeroinitializer to x86_mmx
   %B = bitcast x86_mmx %A to <2 x i32>
@@ -926,8 +939,8 @@ define i64 @test_mmx_const(<2 x i32> %c) {
 
 ; PR12514
 define i1 @test67(i1 %a, i32 %b) {
-; CHECK-LABEL: @test67(
-; CHECK-NEXT:    ret i1 false
+; ALL-LABEL: @test67(
+; ALL-NEXT:    ret i1 false
 ;
   %t2 = zext i1 %a to i32
   %conv6 = xor i32 %t2, 1
@@ -943,10 +956,10 @@ define i1 @test67(i1 %a, i32 %b) {
 %s = type { i32, i32, i16 }
 
 define %s @test68(%s *%p, i64 %i) {
-; CHECK-LABEL: @test68(
-; CHECK-NEXT:    [[PP1:%.*]] = getelementptr [[S:%.*]], %s* [[P:%.*]], i64 [[I:%.*]]
-; CHECK-NEXT:    [[L:%.*]] = load [[S]], %s* [[PP1]], align 4
-; CHECK-NEXT:    ret [[S]] %l
+; ALL-LABEL: @test68(
+; ALL-NEXT:    [[PP1:%.*]] = getelementptr [[S:%.*]], %s* [[P:%.*]], i64 [[I:%.*]]
+; ALL-NEXT:    [[L:%.*]] = load [[S]], %s* [[PP1]], align 4
+; ALL-NEXT:    ret [[S]] %l
 ;
   %o = mul i64 %i, 12
   %q = bitcast %s* %p to i8*
@@ -958,10 +971,10 @@ define %s @test68(%s *%p, i64 %i) {
 
 ; addrspacecasts should be eliminated.
 define %s @test68_addrspacecast(%s* %p, i64 %i) {
-; CHECK-LABEL: @test68_addrspacecast(
-; CHECK-NEXT:    [[PP1:%.*]] = getelementptr [[S:%.*]], %s* [[P:%.*]], i64 [[I:%.*]]
-; CHECK-NEXT:    [[L:%.*]] = load [[S]], %s* [[PP1]], align 4
-; CHECK-NEXT:    ret [[S]] %l
+; ALL-LABEL: @test68_addrspacecast(
+; ALL-NEXT:    [[PP1:%.*]] = getelementptr [[S:%.*]], %s* [[P:%.*]], i64 [[I:%.*]]
+; ALL-NEXT:    [[L:%.*]] = load [[S]], %s* [[PP1]], align 4
+; ALL-NEXT:    ret [[S]] %l
 ;
   %o = mul i64 %i, 12
   %q = addrspacecast %s* %p to i8 addrspace(2)*
@@ -972,11 +985,11 @@ define %s @test68_addrspacecast(%s* %p, i64 %i) {
 }
 
 define %s @test68_addrspacecast_2(%s* %p, i64 %i) {
-; CHECK-LABEL: @test68_addrspacecast_2(
-; CHECK-NEXT:    [[PP1:%.*]] = getelementptr [[S:%.*]], %s* [[P:%.*]], i64 [[I:%.*]]
-; CHECK-NEXT:    [[R:%.*]] = addrspacecast %s* [[PP1]] to [[S]] addrspace(1)*
-; CHECK-NEXT:    [[L:%.*]] = load [[S]], [[S]] addrspace(1)* [[R]], align 4
-; CHECK-NEXT:    ret [[S]] %l
+; ALL-LABEL: @test68_addrspacecast_2(
+; ALL-NEXT:    [[PP1:%.*]] = getelementptr [[S:%.*]], %s* [[P:%.*]], i64 [[I:%.*]]
+; ALL-NEXT:    [[R:%.*]] = addrspacecast %s* [[PP1]] to [[S]] addrspace(1)*
+; ALL-NEXT:    [[L:%.*]] = load [[S]], [[S]] addrspace(1)* [[R]], align 4
+; ALL-NEXT:    ret [[S]] %l
 ;
   %o = mul i64 %i, 12
   %q = addrspacecast %s* %p to i8 addrspace(2)*
@@ -987,10 +1000,10 @@ define %s @test68_addrspacecast_2(%s* %p, i64 %i) {
 }
 
 define %s @test68_as1(%s addrspace(1)* %p, i32 %i) {
-; CHECK-LABEL: @test68_as1(
-; CHECK-NEXT:    [[PP1:%.*]] = getelementptr [[S:%.*]], [[S]] addrspace(1)* [[P:%.*]], i32 [[I:%.*]]
-; CHECK-NEXT:    [[L:%.*]] = load [[S]], [[S]] addrspace(1)* [[PP1]], align 4
-; CHECK-NEXT:    ret [[S]] %l
+; ALL-LABEL: @test68_as1(
+; ALL-NEXT:    [[PP1:%.*]] = getelementptr [[S:%.*]], [[S]] addrspace(1)* [[P:%.*]], i32 [[I:%.*]]
+; ALL-NEXT:    [[L:%.*]] = load [[S]], [[S]] addrspace(1)* [[PP1]], align 4
+; ALL-NEXT:    ret [[S]] %l
 ;
   %o = mul i32 %i, 12
   %q = bitcast %s addrspace(1)* %p to i8 addrspace(1)*
@@ -1001,10 +1014,10 @@ define %s @test68_as1(%s addrspace(1)* %p, i32 %i) {
 }
 
 define double @test69(double *%p, i64 %i) {
-; CHECK-LABEL: @test69(
-; CHECK-NEXT:    [[PP1:%.*]] = getelementptr inbounds double, double* [[P:%.*]], i64 [[I:%.*]]
-; CHECK-NEXT:    [[L:%.*]] = load double, double* [[PP1]], align 8
-; CHECK-NEXT:    ret double [[L]]
+; ALL-LABEL: @test69(
+; ALL-NEXT:    [[PP1:%.*]] = getelementptr inbounds double, double* [[P:%.*]], i64 [[I:%.*]]
+; ALL-NEXT:    [[L:%.*]] = load double, double* [[PP1]], align 8
+; ALL-NEXT:    ret double [[L]]
 ;
   %o = shl nsw i64 %i, 3
   %q = bitcast double* %p to i8*
@@ -1015,11 +1028,11 @@ define double @test69(double *%p, i64 %i) {
 }
 
 define %s @test70(%s *%p, i64 %i) {
-; CHECK-LABEL: @test70(
-; CHECK-NEXT:    [[O:%.*]] = mul nsw i64 [[I:%.*]], 3
-; CHECK-NEXT:    [[PP1:%.*]] = getelementptr inbounds [[S:%.*]], %s* [[P:%.*]], i64 [[O]]
-; CHECK-NEXT:    [[L:%.*]] = load [[S]], %s* [[PP1]], align 4
-; CHECK-NEXT:    ret [[S]] %l
+; ALL-LABEL: @test70(
+; ALL-NEXT:    [[O:%.*]] = mul nsw i64 [[I:%.*]], 3
+; ALL-NEXT:    [[PP1:%.*]] = getelementptr inbounds [[S:%.*]], %s* [[P:%.*]], i64 [[O]]
+; ALL-NEXT:    [[L:%.*]] = load [[S]], %s* [[PP1]], align 4
+; ALL-NEXT:    ret [[S]] %l
 ;
   %o = mul nsw i64 %i, 36
   %q = bitcast %s* %p to i8*
@@ -1030,11 +1043,11 @@ define %s @test70(%s *%p, i64 %i) {
 }
 
 define double @test71(double *%p, i64 %i) {
-; CHECK-LABEL: @test71(
-; CHECK-NEXT:    [[O:%.*]] = shl i64 [[I:%.*]], 2
-; CHECK-NEXT:    [[PP1:%.*]] = getelementptr double, double* [[P:%.*]], i64 [[O]]
-; CHECK-NEXT:    [[L:%.*]] = load double, double* [[PP1]], align 8
-; CHECK-NEXT:    ret double [[L]]
+; ALL-LABEL: @test71(
+; ALL-NEXT:    [[O:%.*]] = shl i64 [[I:%.*]], 2
+; ALL-NEXT:    [[PP1:%.*]] = getelementptr double, double* [[P:%.*]], i64 [[O]]
+; ALL-NEXT:    [[L:%.*]] = load double, double* [[PP1]], align 8
+; ALL-NEXT:    ret double [[L]]
 ;
   %o = shl i64 %i, 5
   %q = bitcast double* %p to i8*
@@ -1045,11 +1058,11 @@ define double @test71(double *%p, i64 %i) {
 }
 
 define double @test72(double *%p, i32 %i) {
-; CHECK-LABEL: @test72(
-; CHECK-NEXT:    [[O:%.*]] = sext i32 [[I:%.*]] to i64
-; CHECK-NEXT:    [[PP1:%.*]] = getelementptr inbounds double, double* [[P:%.*]], i64 [[O]]
-; CHECK-NEXT:    [[L:%.*]] = load double, double* [[PP1]], align 8
-; CHECK-NEXT:    ret double [[L]]
+; ALL-LABEL: @test72(
+; ALL-NEXT:    [[O:%.*]] = sext i32 [[I:%.*]] to i64
+; ALL-NEXT:    [[PP1:%.*]] = getelementptr inbounds double, double* [[P:%.*]], i64 [[O]]
+; ALL-NEXT:    [[L:%.*]] = load double, double* [[PP1]], align 8
+; ALL-NEXT:    ret double [[L]]
 ;
   %so = shl nsw i32 %i, 3
   %o = sext i32 %so to i64
@@ -1061,11 +1074,11 @@ define double @test72(double *%p, i32 %i) {
 }
 
 define double @test73(double *%p, i128 %i) {
-; CHECK-LABEL: @test73(
-; CHECK-NEXT:    [[I_TR:%.*]] = trunc i128 [[I:%.*]] to i64
-; CHECK-NEXT:    [[PP1:%.*]] = getelementptr double, double* [[P:%.*]], i64 [[I_TR]]
-; CHECK-NEXT:    [[L:%.*]] = load double, double* [[PP1]], align 8
-; CHECK-NEXT:    ret double [[L]]
+; ALL-LABEL: @test73(
+; ALL-NEXT:    [[I_TR:%.*]] = trunc i128 [[I:%.*]] to i64
+; ALL-NEXT:    [[PP1:%.*]] = getelementptr double, double* [[P:%.*]], i64 [[I_TR]]
+; ALL-NEXT:    [[L:%.*]] = load double, double* [[PP1]], align 8
+; ALL-NEXT:    ret double [[L]]
 ;
   %lo = shl nsw i128 %i, 3
   %o = trunc i128 %lo to i64
@@ -1077,10 +1090,10 @@ define double @test73(double *%p, i128 %i) {
 }
 
 define double @test74(double *%p, i64 %i) {
-; CHECK-LABEL: @test74(
-; CHECK-NEXT:    [[PP1:%.*]] = getelementptr inbounds double, double* [[P:%.*]], i64 [[I:%.*]]
-; CHECK-NEXT:    [[L:%.*]] = load double, double* [[PP1]], align 8
-; CHECK-NEXT:    ret double [[L]]
+; ALL-LABEL: @test74(
+; ALL-NEXT:    [[PP1:%.*]] = getelementptr inbounds double, double* [[P:%.*]], i64 [[I:%.*]]
+; ALL-NEXT:    [[L:%.*]] = load double, double* [[PP1]], align 8
+; ALL-NEXT:    ret double [[L]]
 ;
   %q = bitcast double* %p to i64*
   %pp = getelementptr inbounds i64, i64* %q, i64 %i
@@ -1090,13 +1103,13 @@ define double @test74(double *%p, i64 %i) {
 }
 
 define i32* @test75(i32* %p, i32 %x) {
-; CHECK-LABEL: @test75(
-; CHECK-NEXT:    [[Y:%.*]] = shl i32 [[X:%.*]], 3
-; CHECK-NEXT:    [[Z:%.*]] = sext i32 [[Y]] to i64
-; CHECK-NEXT:    [[Q:%.*]] = bitcast i32* [[P:%.*]] to i8*
-; CHECK-NEXT:    [[R:%.*]] = getelementptr i8, i8* [[Q]], i64 [[Z]]
-; CHECK-NEXT:    [[S:%.*]] = bitcast i8* [[R]] to i32*
-; CHECK-NEXT:    ret i32* [[S]]
+; ALL-LABEL: @test75(
+; ALL-NEXT:    [[Y:%.*]] = shl i32 [[X:%.*]], 3
+; ALL-NEXT:    [[Z:%.*]] = sext i32 [[Y]] to i64
+; ALL-NEXT:    [[Q:%.*]] = bitcast i32* [[P:%.*]] to i8*
+; ALL-NEXT:    [[R:%.*]] = getelementptr i8, i8* [[Q]], i64 [[Z]]
+; ALL-NEXT:    [[S:%.*]] = bitcast i8* [[R]] to i32*
+; ALL-NEXT:    ret i32* [[S]]
 ;
   %y = shl i32 %x, 3
   %z = sext i32 %y to i64
@@ -1107,11 +1120,11 @@ define i32* @test75(i32* %p, i32 %x) {
 }
 
 define %s @test76(%s *%p, i64 %i, i64 %j) {
-; CHECK-LABEL: @test76(
-; CHECK-NEXT:    [[O2:%.*]] = mul i64 [[I:%.*]], [[J:%.*]]
-; CHECK-NEXT:    [[PP1:%.*]] = getelementptr [[S:%.*]], %s* [[P:%.*]], i64 [[O2]]
-; CHECK-NEXT:    [[L:%.*]] = load [[S]], %s* [[PP1]], align 4
-; CHECK-NEXT:    ret [[S]] %l
+; ALL-LABEL: @test76(
+; ALL-NEXT:    [[O2:%.*]] = mul i64 [[I:%.*]], [[J:%.*]]
+; ALL-NEXT:    [[PP1:%.*]] = getelementptr [[S:%.*]], %s* [[P:%.*]], i64 [[O2]]
+; ALL-NEXT:    [[L:%.*]] = load [[S]], %s* [[PP1]], align 4
+; ALL-NEXT:    ret [[S]] %l
 ;
   %o = mul i64 %i, 12
   %o2 = mul nsw i64 %o, %j
@@ -1123,12 +1136,12 @@ define %s @test76(%s *%p, i64 %i, i64 %j) {
 }
 
 define %s @test77(%s *%p, i64 %i, i64 %j) {
-; CHECK-LABEL: @test77(
-; CHECK-NEXT:    [[O:%.*]] = mul nsw i64 [[I:%.*]], 3
-; CHECK-NEXT:    [[O2:%.*]] = mul nsw i64 [[O]], [[J:%.*]]
-; CHECK-NEXT:    [[PP1:%.*]] = getelementptr inbounds [[S:%.*]], %s* [[P:%.*]], i64 [[O2]]
-; CHECK-NEXT:    [[L:%.*]] = load [[S]], %s* [[PP1]], align 4
-; CHECK-NEXT:    ret [[S]] %l
+; ALL-LABEL: @test77(
+; ALL-NEXT:    [[O:%.*]] = mul nsw i64 [[I:%.*]], 3
+; ALL-NEXT:    [[O2:%.*]] = mul nsw i64 [[O]], [[J:%.*]]
+; ALL-NEXT:    [[PP1:%.*]] = getelementptr inbounds [[S:%.*]], %s* [[P:%.*]], i64 [[O2]]
+; ALL-NEXT:    [[L:%.*]] = load [[S]], %s* [[PP1]], align 4
+; ALL-NEXT:    ret [[S]] %l
 ;
   %o = mul nsw i64 %i, 36
   %o2 = mul nsw i64 %o, %j
@@ -1140,18 +1153,18 @@ define %s @test77(%s *%p, i64 %i, i64 %j) {
 }
 
 define %s @test78(%s *%p, i64 %i, i64 %j, i32 %k, i32 %l, i128 %m, i128 %n) {
-; CHECK-LABEL: @test78(
-; CHECK-NEXT:    [[A:%.*]] = mul nsw i32 [[K:%.*]], 3
-; CHECK-NEXT:    [[B:%.*]] = mul nsw i32 [[A]], [[L:%.*]]
-; CHECK-NEXT:    [[C:%.*]] = sext i32 [[B]] to i128
-; CHECK-NEXT:    [[D:%.*]] = mul nsw i128 [[C]], [[M:%.*]]
-; CHECK-NEXT:    [[E:%.*]] = mul i128 [[D]], [[N:%.*]]
-; CHECK-NEXT:    [[F:%.*]] = trunc i128 [[E]] to i64
-; CHECK-NEXT:    [[G:%.*]] = mul i64 [[F]], [[I:%.*]]
-; CHECK-NEXT:    [[H:%.*]] = mul i64 [[G]], [[J:%.*]]
-; CHECK-NEXT:    [[PP1:%.*]] = getelementptr [[S:%.*]], %s* [[P:%.*]], i64 [[H]]
-; CHECK-NEXT:    [[LOAD:%.*]] = load [[S]], %s* [[PP1]], align 4
-; CHECK-NEXT:    ret [[S]] %load
+; ALL-LABEL: @test78(
+; ALL-NEXT:    [[A:%.*]] = mul nsw i32 [[K:%.*]], 3
+; ALL-NEXT:    [[B:%.*]] = mul nsw i32 [[A]], [[L:%.*]]
+; ALL-NEXT:    [[C:%.*]] = sext i32 [[B]] to i128
+; ALL-NEXT:    [[D:%.*]] = mul nsw i128 [[C]], [[M:%.*]]
+; ALL-NEXT:    [[E:%.*]] = mul i128 [[D]], [[N:%.*]]
+; ALL-NEXT:    [[F:%.*]] = trunc i128 [[E]] to i64
+; ALL-NEXT:    [[G:%.*]] = mul i64 [[F]], [[I:%.*]]
+; ALL-NEXT:    [[H:%.*]] = mul i64 [[G]], [[J:%.*]]
+; ALL-NEXT:    [[PP1:%.*]] = getelementptr [[S:%.*]], %s* [[P:%.*]], i64 [[H]]
+; ALL-NEXT:    [[LOAD:%.*]] = load [[S]], %s* [[PP1]], align 4
+; ALL-NEXT:    ret [[S]] %load
 ;
   %a = mul nsw i32 %k, 36
   %b = mul nsw i32 %a, %l
@@ -1169,16 +1182,16 @@ define %s @test78(%s *%p, i64 %i, i64 %j, i32 %k, i32 %l, i128 %m, i128 %n) {
 }
 
 define %s @test79(%s *%p, i64 %i, i32 %j) {
-; CHECK-LABEL: @test79(
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[I:%.*]] to i32
-; CHECK-NEXT:    [[B:%.*]] = mul i32 [[TMP1]], 36
-; CHECK-NEXT:    [[C:%.*]] = mul i32 [[B]], [[J:%.*]]
-; CHECK-NEXT:    [[Q:%.*]] = bitcast %s* [[P:%.*]] to i8*
-; CHECK-NEXT:    [[TMP2:%.*]] = sext i32 [[C]] to i64
-; CHECK-NEXT:    [[PP:%.*]] = getelementptr inbounds i8, i8* [[Q]], i64 [[TMP2]]
-; CHECK-NEXT:    [[R:%.*]] = bitcast i8* [[PP]] to %s*
-; CHECK-NEXT:    [[L:%.*]] = load [[S:%.*]], %s* [[R]], align 4
-; CHECK-NEXT:    ret [[S]] %l
+; ALL-LABEL: @test79(
+; ALL-NEXT:    [[TMP1:%.*]] = trunc i64 [[I:%.*]] to i32
+; ALL-NEXT:    [[B:%.*]] = mul i32 [[TMP1]], 36
+; ALL-NEXT:    [[C:%.*]] = mul i32 [[B]], [[J:%.*]]
+; ALL-NEXT:    [[Q:%.*]] = bitcast %s* [[P:%.*]] to i8*
+; ALL-NEXT:    [[TMP2:%.*]] = sext i32 [[C]] to i64
+; ALL-NEXT:    [[PP:%.*]] = getelementptr inbounds i8, i8* [[Q]], i64 [[TMP2]]
+; ALL-NEXT:    [[R:%.*]] = bitcast i8* [[PP]] to %s*
+; ALL-NEXT:    [[L:%.*]] = load [[S:%.*]], %s* [[R]], align 4
+; ALL-NEXT:    ret [[S]] %l
 ;
   %a = mul nsw i64 %i, 36
   %b = trunc i64 %a to i32
@@ -1191,11 +1204,11 @@ define %s @test79(%s *%p, i64 %i, i32 %j) {
 }
 
 define double @test80([100 x double]* %p, i32 %i) {
-; CHECK-LABEL: @test80(
-; CHECK-NEXT:    [[TMP1:%.*]] = sext i32 [[I:%.*]] to i64
-; CHECK-NEXT:    [[PP1:%.*]] = getelementptr [100 x double], [100 x double]* [[P:%.*]], i64 0, i64 [[TMP1]]
-; CHECK-NEXT:    [[L:%.*]] = load double, double* [[PP1]], align 8
-; CHECK-NEXT:    ret double [[L]]
+; ALL-LABEL: @test80(
+; ALL-NEXT:    [[TMP1:%.*]] = sext i32 [[I:%.*]] to i64
+; ALL-NEXT:    [[PP1:%.*]] = getelementptr [100 x double], [100 x double]* [[P:%.*]], i64 0, i64 [[TMP1]]
+; ALL-NEXT:    [[L:%.*]] = load double, double* [[PP1]], align 8
+; ALL-NEXT:    ret double [[L]]
 ;
   %t = shl nsw i32 %i, 3
   %q = bitcast [100 x double]* %p to i8*
@@ -1206,10 +1219,10 @@ define double @test80([100 x double]* %p, i32 %i) {
 }
 
 define double @test80_addrspacecast([100 x double] addrspace(1)* %p, i32 %i) {
-; CHECK-LABEL: @test80_addrspacecast(
-; CHECK-NEXT:    [[PP1:%.*]] = getelementptr [100 x double], [100 x double] addrspace(1)* [[P:%.*]], i32 0, i32 [[I:%.*]]
-; CHECK-NEXT:    [[L:%.*]] = load double, double addrspace(1)* [[PP1]], align 8
-; CHECK-NEXT:    ret double [[L]]
+; ALL-LABEL: @test80_addrspacecast(
+; ALL-NEXT:    [[PP1:%.*]] = getelementptr [100 x double], [100 x double] addrspace(1)* [[P:%.*]], i32 0, i32 [[I:%.*]]
+; ALL-NEXT:    [[L:%.*]] = load double, double addrspace(1)* [[PP1]], align 8
+; ALL-NEXT:    ret double [[L]]
 ;
   %t = shl nsw i32 %i, 3
   %q = addrspacecast [100 x double] addrspace(1)* %p to i8 addrspace(2)*
@@ -1220,11 +1233,11 @@ define double @test80_addrspacecast([100 x double] addrspace(1)* %p, i32 %i) {
 }
 
 define double @test80_addrspacecast_2([100 x double] addrspace(1)* %p, i32 %i) {
-; CHECK-LABEL: @test80_addrspacecast_2(
-; CHECK-NEXT:    [[PP1:%.*]] = getelementptr [100 x double], [100 x double] addrspace(1)* [[P:%.*]], i32 0, i32 [[I:%.*]]
-; CHECK-NEXT:    [[R:%.*]] = addrspacecast double addrspace(1)* [[PP1]] to double addrspace(3)*
-; CHECK-NEXT:    [[L:%.*]] = load double, double addrspace(3)* [[R]], align 8
-; CHECK-NEXT:    ret double [[L]]
+; ALL-LABEL: @test80_addrspacecast_2(
+; ALL-NEXT:    [[PP1:%.*]] = getelementptr [100 x double], [100 x double] addrspace(1)* [[P:%.*]], i32 0, i32 [[I:%.*]]
+; ALL-NEXT:    [[R:%.*]] = addrspacecast double addrspace(1)* [[PP1]] to double addrspace(3)*
+; ALL-NEXT:    [[L:%.*]] = load double, double addrspace(3)* [[R]], align 8
+; ALL-NEXT:    ret double [[L]]
 ;
   %t = shl nsw i32 %i, 3
   %q = addrspacecast [100 x double] addrspace(1)* %p to i8 addrspace(2)*
@@ -1235,11 +1248,11 @@ define double @test80_addrspacecast_2([100 x double] addrspace(1)* %p, i32 %i) {
 }
 
 define double @test80_as1([100 x double] addrspace(1)* %p, i16 %i) {
-; CHECK-LABEL: @test80_as1(
-; CHECK-NEXT:    [[TMP1:%.*]] = sext i16 [[I:%.*]] to i32
-; CHECK-NEXT:    [[PP1:%.*]] = getelementptr [100 x double], [100 x double] addrspace(1)* [[P:%.*]], i32 0, i32 [[TMP1]]
-; CHECK-NEXT:    [[L:%.*]] = load double, double addrspace(1)* [[PP1]], align 8
-; CHECK-NEXT:    ret double [[L]]
+; ALL-LABEL: @test80_as1(
+; ALL-NEXT:    [[TMP1:%.*]] = sext i16 [[I:%.*]] to i32
+; ALL-NEXT:    [[PP1:%.*]] = getelementptr [100 x double], [100 x double] addrspace(1)* [[P:%.*]], i32 0, i32 [[TMP1]]
+; ALL-NEXT:    [[L:%.*]] = load double, double addrspace(1)* [[PP1]], align 8
+; ALL-NEXT:    ret double [[L]]
 ;
   %t = shl nsw i16 %i, 3
   %q = bitcast [100 x double] addrspace(1)* %p to i8 addrspace(1)*
@@ -1250,13 +1263,13 @@ define double @test80_as1([100 x double] addrspace(1)* %p, i16 %i) {
 }
 
 define double @test81(double *%p, float %f) {
-; CHECK-LABEL: @test81(
-; CHECK-NEXT:    [[I:%.*]] = fptosi float [[F:%.*]] to i64
-; CHECK-NEXT:    [[Q:%.*]] = bitcast double* [[P:%.*]] to i8*
-; CHECK-NEXT:    [[PP:%.*]] = getelementptr i8, i8* [[Q]], i64 [[I]]
-; CHECK-NEXT:    [[R:%.*]] = bitcast i8* [[PP]] to double*
-; CHECK-NEXT:    [[L:%.*]] = load double, double* [[R]], align 8
-; CHECK-NEXT:    ret double [[L]]
+; ALL-LABEL: @test81(
+; ALL-NEXT:    [[I:%.*]] = fptosi float [[F:%.*]] to i64
+; ALL-NEXT:    [[Q:%.*]] = bitcast double* [[P:%.*]] to i8*
+; ALL-NEXT:    [[PP:%.*]] = getelementptr i8, i8* [[Q]], i64 [[I]]
+; ALL-NEXT:    [[R:%.*]] = bitcast i8* [[PP]] to double*
+; ALL-NEXT:    [[L:%.*]] = load double, double* [[R]], align 8
+; ALL-NEXT:    ret double [[L]]
 ;
   %i = fptosi float %f to i64
   %q = bitcast double* %p to i8*
@@ -1267,10 +1280,10 @@ define double @test81(double *%p, float %f) {
 }
 
 define i64 @test82(i64 %A) {
-; CHECK-LABEL: @test82(
-; CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[A:%.*]], 1
-; CHECK-NEXT:    [[E:%.*]] = and i64 [[TMP1]], 4294966784
-; CHECK-NEXT:    ret i64 [[E]]
+; ALL-LABEL: @test82(
+; ALL-NEXT:    [[TMP1:%.*]] = shl i64 [[A:%.*]], 1
+; ALL-NEXT:    [[E:%.*]] = and i64 [[TMP1]], 4294966784
+; ALL-NEXT:    ret i64 [[E]]
 ;
   %B = trunc i64 %A to i32
   %C = lshr i32 %B, 8
@@ -1281,13 +1294,13 @@ define i64 @test82(i64 %A) {
 
 ; PR15959
 define i64 @test83(i16 %a, i64 %k) {
-; CHECK-LABEL: @test83(
-; CHECK-NEXT:    [[CONV:%.*]] = sext i16 [[A:%.*]] to i32
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[K:%.*]] to i32
-; CHECK-NEXT:    [[SH_PROM:%.*]] = add i32 [[TMP1]], -1
-; CHECK-NEXT:    [[SHL:%.*]] = shl i32 [[CONV]], [[SH_PROM]]
-; CHECK-NEXT:    [[SH_PROM1:%.*]] = zext i32 [[SHL]] to i64
-; CHECK-NEXT:    ret i64 [[SH_PROM1]]
+; ALL-LABEL: @test83(
+; ALL-NEXT:    [[CONV:%.*]] = sext i16 [[A:%.*]] to i32
+; ALL-NEXT:    [[TMP1:%.*]] = trunc i64 [[K:%.*]] to i32
+; ALL-NEXT:    [[SH_PROM:%.*]] = add i32 [[TMP1]], -1
+; ALL-NEXT:    [[SHL:%.*]] = shl i32 [[CONV]], [[SH_PROM]]
+; ALL-NEXT:    [[SH_PROM1:%.*]] = zext i32 [[SHL]] to i64
+; ALL-NEXT:    ret i64 [[SH_PROM1]]
 ;
   %conv = sext i16 %a to i32
   %sub = add nsw i64 %k, -1
@@ -1298,11 +1311,11 @@ define i64 @test83(i16 %a, i64 %k) {
 }
 
 define i8 @test84(i32 %a) {
-; CHECK-LABEL: @test84(
-; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[A:%.*]], 2130706432
-; CHECK-NEXT:    [[SHR:%.*]] = lshr exact i32 [[ADD]], 23
-; CHECK-NEXT:    [[TRUNC:%.*]] = trunc i32 [[SHR]] to i8
-; CHECK-NEXT:    ret i8 [[TRUNC]]
+; ALL-LABEL: @test84(
+; ALL-NEXT:    [[ADD:%.*]] = add i32 [[A:%.*]], 2130706432
+; ALL-NEXT:    [[SHR:%.*]] = lshr exact i32 [[ADD]], 23
+; ALL-NEXT:    [[TRUNC:%.*]] = trunc i32 [[SHR]] to i8
+; ALL-NEXT:    ret i8 [[TRUNC]]
 ;
   %add = add nsw i32 %a, -16777216
   %shr = lshr exact i32 %add, 23
@@ -1311,11 +1324,11 @@ define i8 @test84(i32 %a) {
 }
 
 define i8 @test85(i32 %a) {
-; CHECK-LABEL: @test85(
-; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[A:%.*]], 2130706432
-; CHECK-NEXT:    [[SHR:%.*]] = lshr exact i32 [[ADD]], 23
-; CHECK-NEXT:    [[TRUNC:%.*]] = trunc i32 [[SHR]] to i8
-; CHECK-NEXT:    ret i8 [[TRUNC]]
+; ALL-LABEL: @test85(
+; ALL-NEXT:    [[ADD:%.*]] = add i32 [[A:%.*]], 2130706432
+; ALL-NEXT:    [[SHR:%.*]] = lshr exact i32 [[ADD]], 23
+; ALL-NEXT:    [[TRUNC:%.*]] = trunc i32 [[SHR]] to i8
+; ALL-NEXT:    ret i8 [[TRUNC]]
 ;
   %add = add nuw i32 %a, -16777216
   %shr = lshr exact i32 %add, 23
@@ -1324,9 +1337,9 @@ define i8 @test85(i32 %a) {
 }
 
 define i16 @test86(i16 %v) {
-; CHECK-LABEL: @test86(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr i16 [[V:%.*]], 4
-; CHECK-NEXT:    ret i16 [[TMP1]]
+; ALL-LABEL: @test86(
+; ALL-NEXT:    [[TMP1:%.*]] = ashr i16 [[V:%.*]], 4
+; ALL-NEXT:    ret i16 [[TMP1]]
 ;
   %a = sext i16 %v to i32
   %s = ashr i32 %a, 4
@@ -1335,9 +1348,9 @@ define i16 @test86(i16 %v) {
 }
 
 define i16 @test87(i16 %v) {
-; CHECK-LABEL: @test87(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr i16 [[V:%.*]], 12
-; CHECK-NEXT:    ret i16 [[TMP1]]
+; ALL-LABEL: @test87(
+; ALL-NEXT:    [[TMP1:%.*]] = ashr i16 [[V:%.*]], 12
+; ALL-NEXT:    ret i16 [[TMP1]]
 ;
   %c = sext i16 %v to i32
   %m = mul nsw i32 %c, 16
@@ -1347,9 +1360,9 @@ define i16 @test87(i16 %v) {
 }
 
 define i16 @test88(i16 %v) {
-; CHECK-LABEL: @test88(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr i16 [[V:%.*]], 15
-; CHECK-NEXT:    ret i16 [[TMP1]]
+; ALL-LABEL: @test88(
+; ALL-NEXT:    [[TMP1:%.*]] = ashr i16 [[V:%.*]], 15
+; ALL-NEXT:    ret i16 [[TMP1]]
 ;
   %a = sext i16 %v to i32
   %s = ashr i32 %a, 18
@@ -1358,10 +1371,10 @@ define i16 @test88(i16 %v) {
 }
 
 define i32 @PR21388(i32* %v) {
-; CHECK-LABEL: @PR21388(
-; CHECK-NEXT:    [[ICMP:%.*]] = icmp slt i32* [[V:%.*]], null
-; CHECK-NEXT:    [[SEXT:%.*]] = sext i1 [[ICMP]] to i32
-; CHECK-NEXT:    ret i32 [[SEXT]]
+; ALL-LABEL: @PR21388(
+; ALL-NEXT:    [[ICMP:%.*]] = icmp slt i32* [[V:%.*]], null
+; ALL-NEXT:    [[SEXT:%.*]] = sext i1 [[ICMP]] to i32
+; ALL-NEXT:    ret i32 [[SEXT]]
 ;
   %icmp = icmp slt i32* %v, null
   %sext = sext i1 %icmp to i32
@@ -1369,9 +1382,9 @@ define i32 @PR21388(i32* %v) {
 }
 
 define float @sitofp_zext(i16 %a) {
-; CHECK-LABEL: @sitofp_zext(
-; CHECK-NEXT:    [[SITOFP:%.*]] = uitofp i16 [[A:%.*]] to float
-; CHECK-NEXT:    ret float [[SITOFP]]
+; ALL-LABEL: @sitofp_zext(
+; ALL-NEXT:    [[SITOFP:%.*]] = uitofp i16 [[A:%.*]] to float
+; ALL-NEXT:    ret float [[SITOFP]]
 ;
   %zext = zext i16 %a to i32
   %sitofp = sitofp i32 %zext to float
@@ -1379,11 +1392,11 @@ define float @sitofp_zext(i16 %a) {
 }
 
 define i1 @PR23309(i32 %A, i32 %B) {
-; CHECK-LABEL: @PR23309(
-; CHECK-NEXT:    [[SUB:%.*]] = sub i32 [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[SUB]], 1
-; CHECK-NEXT:    [[TRUNC:%.*]] = icmp ne i32 [[TMP1]], 0
-; CHECK-NEXT:    ret i1 [[TRUNC]]
+; ALL-LABEL: @PR23309(
+; ALL-NEXT:    [[SUB:%.*]] = sub i32 [[A:%.*]], [[B:%.*]]
+; ALL-NEXT:    [[TMP1:%.*]] = and i32 [[SUB]], 1
+; ALL-NEXT:    [[TRUNC:%.*]] = icmp ne i32 [[TMP1]], 0
+; ALL-NEXT:    ret i1 [[TRUNC]]
 ;
   %add = add i32 %A, -4
   %sub = sub nsw i32 %add, %B
@@ -1392,11 +1405,11 @@ define i1 @PR23309(i32 %A, i32 %B) {
 }
 
 define i1 @PR23309v2(i32 %A, i32 %B) {
-; CHECK-LABEL: @PR23309v2(
-; CHECK-NEXT:    [[SUB:%.*]] = add i32 [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[SUB]], 1
-; CHECK-NEXT:    [[TRUNC:%.*]] = icmp ne i32 [[TMP1]], 0
-; CHECK-NEXT:    ret i1 [[TRUNC]]
+; ALL-LABEL: @PR23309v2(
+; ALL-NEXT:    [[SUB:%.*]] = add i32 [[A:%.*]], [[B:%.*]]
+; ALL-NEXT:    [[TMP1:%.*]] = and i32 [[SUB]], 1
+; ALL-NEXT:    [[TRUNC:%.*]] = icmp ne i32 [[TMP1]], 0
+; ALL-NEXT:    ret i1 [[TRUNC]]
 ;
   %add = add i32 %A, -4
   %sub = add nuw i32 %add, %B
@@ -1405,10 +1418,10 @@ define i1 @PR23309v2(i32 %A, i32 %B) {
 }
 
 define i16 @PR24763(i8 %V) {
-; CHECK-LABEL: @PR24763(
-; CHECK-NEXT:    [[L:%.*]] = ashr i8 [[V:%.*]], 1
-; CHECK-NEXT:    [[T:%.*]] = sext i8 [[L]] to i16
-; CHECK-NEXT:    ret i16 [[T]]
+; ALL-LABEL: @PR24763(
+; ALL-NEXT:    [[L:%.*]] = ashr i8 [[V:%.*]], 1
+; ALL-NEXT:    [[T:%.*]] = sext i8 [[L]] to i16
+; ALL-NEXT:    ret i16 [[T]]
 ;
   %conv = sext i8 %V to i32
   %l = lshr i32 %conv, 1
@@ -1417,23 +1430,32 @@ define i16 @PR24763(i8 %V) {
 }
 
 define i64 @PR28745() {
-; CHECK-LABEL: @PR28745(
-; CHECK-NEXT:    ret i64 1
+; BE-LABEL: @PR28745(
+; BE-NEXT:    ret i64 1
+;
+; LE-LABEL: @PR28745(
+; LE-NEXT:    ret i64 0
 ;
   %b = zext i32 extractvalue ({ i32 } select (i1 icmp eq (i16 extractelement (<2 x i16> bitcast (<1 x i32> <i32 1> to <2 x i16>), i32 0), i16 0), { i32 } { i32 1 }, { i32 } zeroinitializer), 0) to i64
   ret i64 %b
 }
 
 define i32 @test89() {
-; CHECK-LABEL: @test89(
-; CHECK-NEXT:    ret i32 393216
+; BE-LABEL: @test89(
+; BE-NEXT:    ret i32 393216
+;
+; LE-LABEL: @test89(
+; LE-NEXT:    ret i32 6
 ;
   ret i32 bitcast (<2 x i16> <i16 6, i16 undef> to i32)
 }
 
 define <2 x i32> @test90() {
-; CHECK-LABEL: @test90(
-; CHECK-NEXT:    ret <2 x i32> <i32 0, i32 15360>
+; BE-LABEL: @test90(
+; BE-NEXT:    ret <2 x i32> <i32 0, i32 15360>
+;
+; LE-LABEL: @test90(
+; LE-NEXT:    ret <2 x i32> <i32 0, i32 1006632960>
 ;
   %t6 = bitcast <4 x half> <half undef, half undef, half undef, half 0xH3C00> to <2 x i32>
   ret <2 x i32> %t6
@@ -1441,11 +1463,11 @@ define <2 x i32> @test90() {
 
 ; Do not optimize to ashr i64 (shift by 48 > 96 - 64)
 define i64 @test91(i64 %A) {
-; CHECK-LABEL: @test91(
-; CHECK-NEXT:    [[B:%.*]] = sext i64 [[A:%.*]] to i96
-; CHECK-NEXT:    [[C:%.*]] = lshr i96 [[B]], 48
-; CHECK-NEXT:    [[D:%.*]] = trunc i96 [[C]] to i64
-; CHECK-NEXT:    ret i64 [[D]]
+; ALL-LABEL: @test91(
+; ALL-NEXT:    [[B:%.*]] = sext i64 [[A:%.*]] to i96
+; ALL-NEXT:    [[C:%.*]] = lshr i96 [[B]], 48
+; ALL-NEXT:    [[D:%.*]] = trunc i96 [[C]] to i64
+; ALL-NEXT:    ret i64 [[D]]
 ;
   %B = sext i64 %A to i96
   %C = lshr i96 %B, 48
@@ -1455,9 +1477,9 @@ define i64 @test91(i64 %A) {
 
 ; Do optimize to ashr i64 (shift by 32 <= 96 - 64)
 define i64 @test92(i64 %A) {
-; CHECK-LABEL: @test92(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr i64 [[A:%.*]], 32
-; CHECK-NEXT:    ret i64 [[TMP1]]
+; ALL-LABEL: @test92(
+; ALL-NEXT:    [[TMP1:%.*]] = ashr i64 [[A:%.*]], 32
+; ALL-NEXT:    ret i64 [[TMP1]]
 ;
   %B = sext i64 %A to i96
   %C = lshr i96 %B, 32
@@ -1467,9 +1489,9 @@ define i64 @test92(i64 %A) {
 
 ; When optimizing to ashr i32, don't shift by more than 31.
 define i32 @test93(i32 %A) {
-; CHECK-LABEL: @test93(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr i32 [[A:%.*]], 31
-; CHECK-NEXT:    ret i32 [[TMP1]]
+; ALL-LABEL: @test93(
+; ALL-NEXT:    [[TMP1:%.*]] = ashr i32 [[A:%.*]], 31
+; ALL-NEXT:    ret i32 [[TMP1]]
 ;
   %B = sext i32 %A to i96
   %C = lshr i96 %B, 64
@@ -1481,9 +1503,9 @@ define i32 @test93(i32 %A) {
 ; PR33078
 
 define i8 @pr33078_1(i8 %A) {
-; CHECK-LABEL: @pr33078_1(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr i8 [[A:%.*]], 7
-; CHECK-NEXT:    ret i8 [[TMP1]]
+; ALL-LABEL: @pr33078_1(
+; ALL-NEXT:    [[TMP1:%.*]] = ashr i8 [[A:%.*]], 7
+; ALL-NEXT:    ret i8 [[TMP1]]
 ;
   %B = sext i8 %A to i16
   %C = lshr i16 %B, 8
@@ -1492,10 +1514,10 @@ define i8 @pr33078_1(i8 %A) {
 }
 
 define i12 @pr33078_2(i8 %A) {
-; CHECK-LABEL: @pr33078_2(
-; CHECK-NEXT:    [[C:%.*]] = ashr i8 [[A:%.*]], 4
-; CHECK-NEXT:    [[D:%.*]] = sext i8 [[C]] to i12
-; CHECK-NEXT:    ret i12 [[D]]
+; ALL-LABEL: @pr33078_2(
+; ALL-NEXT:    [[C:%.*]] = ashr i8 [[A:%.*]], 4
+; ALL-NEXT:    [[D:%.*]] = sext i8 [[C]] to i12
+; ALL-NEXT:    ret i12 [[D]]
 ;
   %B = sext i8 %A to i16
   %C = lshr i16 %B, 4
@@ -1504,11 +1526,11 @@ define i12 @pr33078_2(i8 %A) {
 }
 
 define i4 @pr33078_3(i8 %A) {
-; CHECK-LABEL: @pr33078_3(
-; CHECK-NEXT:    [[B:%.*]] = sext i8 [[A:%.*]] to i16
-; CHECK-NEXT:    [[C:%.*]] = lshr i16 [[B]], 12
-; CHECK-NEXT:    [[D:%.*]] = trunc i16 [[C]] to i4
-; CHECK-NEXT:    ret i4 [[D]]
+; ALL-LABEL: @pr33078_3(
+; ALL-NEXT:    [[B:%.*]] = sext i8 [[A:%.*]] to i16
+; ALL-NEXT:    [[C:%.*]] = lshr i16 [[B]], 12
+; ALL-NEXT:    [[D:%.*]] = trunc i16 [[C]] to i4
+; ALL-NEXT:    ret i4 [[D]]
 ;
   %B = sext i8 %A to i16
   %C = lshr i16 %B, 12
@@ -1518,11 +1540,11 @@ define i4 @pr33078_3(i8 %A) {
 
 define i8 @pr33078_4(i3 %x) {
 ; Don't turn this in an `ashr`. This was getting miscompiled
-; CHECK-LABEL: @pr33078_4(
-; CHECK-NEXT:    [[B:%.*]] = sext i3 [[X:%.*]] to i16
-; CHECK-NEXT:    [[C:%.*]] = lshr i16 [[B]], 13
-; CHECK-NEXT:    [[D:%.*]] = trunc i16 [[C]] to i8
-; CHECK-NEXT:    ret i8 [[D]]
+; ALL-LABEL: @pr33078_4(
+; ALL-NEXT:    [[B:%.*]] = sext i3 [[X:%.*]] to i16
+; ALL-NEXT:    [[C:%.*]] = lshr i16 [[B]], 13
+; ALL-NEXT:    [[D:%.*]] = trunc i16 [[C]] to i8
+; ALL-NEXT:    ret i8 [[D]]
 ;
   %B = sext i3 %x to i16
   %C = lshr i16 %B, 13
@@ -1532,10 +1554,10 @@ define i8 @pr33078_4(i3 %x) {
 
 ; (sext (xor (cmp), -1)) -> (sext (!cmp))
 define i64 @test94(i32 %a) {
-; CHECK-LABEL: @test94(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne i32 [[A:%.*]], -2
-; CHECK-NEXT:    [[TMP2:%.*]] = sext i1 [[TMP1]] to i64
-; CHECK-NEXT:    ret i64 [[TMP2]]
+; ALL-LABEL: @test94(
+; ALL-NEXT:    [[TMP1:%.*]] = icmp ne i32 [[A:%.*]], -2
+; ALL-NEXT:    [[TMP2:%.*]] = sext i1 [[TMP1]] to i64
+; ALL-NEXT:    ret i64 [[TMP2]]
 ;
   %1 = icmp eq i32 %a, -2
   %2 = sext i1 %1 to i8
@@ -1546,11 +1568,11 @@ define i64 @test94(i32 %a) {
 
 ; We should be able to remove the zext and trunc here.
 define i32 @test95(i32 %x) {
-; CHECK-LABEL: @test95(
-; CHECK-NEXT:    [[TMP1:%.*]] = lshr i32 [[X:%.*]], 6
-; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], 2
-; CHECK-NEXT:    [[TMP3:%.*]] = or i32 [[TMP2]], 40
-; CHECK-NEXT:    ret i32 [[TMP3]]
+; ALL-LABEL: @test95(
+; ALL-NEXT:    [[TMP1:%.*]] = lshr i32 [[X:%.*]], 6
+; ALL-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], 2
+; ALL-NEXT:    [[TMP3:%.*]] = or i32 [[TMP2]], 40
+; ALL-NEXT:    ret i32 [[TMP3]]
 ;
   %1 = trunc i32 %x to i8
   %2 = lshr i8 %1, 6
diff --git a/llvm/test/Transforms/InstCombine/clamp-to-minmax.ll b/llvm/test/Transforms/InstCombine/clamp-to-minmax.ll
index 49b5160ba..f23819cd5 100644
--- a/llvm/test/Transforms/InstCombine/clamp-to-minmax.ll
+++ b/llvm/test/Transforms/InstCombine/clamp-to-minmax.ll
@@ -3,13 +3,11 @@
 
 ; (X < C1) ? C1 : MIN(X, C2)
 define float @clamp_float_fast_ordered_strict_maxmin(float %x) {
-;
-; CHECK-LABEL: @clamp_float_fast_ordered_strict_maxmin(
+; CHECK-LABEL: define {{[^@]+}}@clamp_float_fast_ordered_strict_maxmin(
 ; CHECK-NEXT:    [[CMP2:%.*]] = fcmp fast olt float [[X:%.*]], 2.550000e+02
 ; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[CMP2]], float [[X]], float 2.550000e+02
-; CHECK-NEXT:    [[DOTINV:%.*]] = fcmp fast oge float [[MIN]], 1.000000e+00
-; CHECK-NEXT:    [[R1:%.*]] = select i1 [[DOTINV]], float [[MIN]], float 1.000000e+00
-; CHECK-NEXT:    ret float [[R1]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast float @llvm.maxnum.f32(float [[MIN]], float 1.000000e+00)
+; CHECK-NEXT:    ret float [[TMP1]]
 ;
   %cmp2 = fcmp fast olt float %x, 255.0
   %min = select i1 %cmp2, float %x, float 255.0
@@ -20,13 +18,11 @@ define float @clamp_float_fast_ordered_strict_maxmin(float %x) {
 
 ; (X <= C1) ? C1 : MIN(X, C2)
 define float @clamp_float_fast_ordered_nonstrict_maxmin(float %x) {
-;
-; CHECK-LABEL: @clamp_float_fast_ordered_nonstrict_maxmin(
+; CHECK-LABEL: define {{[^@]+}}@clamp_float_fast_ordered_nonstrict_maxmin(
 ; CHECK-NEXT:    [[CMP2:%.*]] = fcmp fast olt float [[X:%.*]], 2.550000e+02
 ; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[CMP2]], float [[X]], float 2.550000e+02
-; CHECK-NEXT:    [[DOTINV:%.*]] = fcmp fast oge float [[MIN]], 1.000000e+00
-; CHECK-NEXT:    [[R1:%.*]] = select i1 [[DOTINV]], float [[MIN]], float 1.000000e+00
-; CHECK-NEXT:    ret float [[R1]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast float @llvm.maxnum.f32(float [[MIN]], float 1.000000e+00)
+; CHECK-NEXT:    ret float [[TMP1]]
 ;
   %cmp2 = fcmp fast olt float %x, 255.0
   %min = select i1 %cmp2, float %x, float 255.0
@@ -37,13 +33,11 @@ define float @clamp_float_fast_ordered_nonstrict_maxmin(float %x) {
 
 ; (X > C1) ? C1 : MAX(X, C2)
 define float @clamp_float_fast_ordered_strict_minmax(float %x) {
-;
-; CHECK-LABEL: @clamp_float_fast_ordered_strict_minmax(
+; CHECK-LABEL: define {{[^@]+}}@clamp_float_fast_ordered_strict_minmax(
 ; CHECK-NEXT:    [[CMP2:%.*]] = fcmp fast ogt float [[X:%.*]], 1.000000e+00
 ; CHECK-NEXT:    [[MAX:%.*]] = select i1 [[CMP2]], float [[X]], float 1.000000e+00
-; CHECK-NEXT:    [[DOTINV:%.*]] = fcmp fast ole float [[MAX]], 2.550000e+02
-; CHECK-NEXT:    [[R1:%.*]] = select i1 [[DOTINV]], float [[MAX]], float 2.550000e+02
-; CHECK-NEXT:    ret float [[R1]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast float @llvm.minnum.f32(float [[MAX]], float 2.550000e+02)
+; CHECK-NEXT:    ret float [[TMP1]]
 ;
   %cmp2 = fcmp fast ogt float %x, 1.0
   %max = select i1 %cmp2, float %x, float 1.0
@@ -54,13 +48,11 @@ define float @clamp_float_fast_ordered_strict_minmax(float %x) {
 
 ; (X >= C1) ? C1 : MAX(X, C2)
 define float @clamp_float_fast_ordered_nonstrict_minmax(float %x) {
-;
-; CHECK-LABEL: @clamp_float_fast_ordered_nonstrict_minmax(
+; CHECK-LABEL: define {{[^@]+}}@clamp_float_fast_ordered_nonstrict_minmax(
 ; CHECK-NEXT:    [[CMP2:%.*]] = fcmp fast ogt float [[X:%.*]], 1.000000e+00
 ; CHECK-NEXT:    [[MAX:%.*]] = select i1 [[CMP2]], float [[X]], float 1.000000e+00
-; CHECK-NEXT:    [[DOTINV:%.*]] = fcmp fast ole float [[MAX]], 2.550000e+02
-; CHECK-NEXT:    [[R1:%.*]] = select i1 [[DOTINV]], float [[MAX]], float 2.550000e+02
-; CHECK-NEXT:    ret float [[R1]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast float @llvm.minnum.f32(float [[MAX]], float 2.550000e+02)
+; CHECK-NEXT:    ret float [[TMP1]]
 ;
   %cmp2 = fcmp fast ogt float %x, 1.0
   %max = select i1 %cmp2, float %x, float 1.0
@@ -74,13 +66,11 @@ define float @clamp_float_fast_ordered_nonstrict_minmax(float %x) {
 
 ; (X < C1) ? C1 : MIN(X, C2)
 define float @clamp_float_fast_unordered_strict_maxmin(float %x) {
-;
-; CHECK-LABEL: @clamp_float_fast_unordered_strict_maxmin(
+; CHECK-LABEL: define {{[^@]+}}@clamp_float_fast_unordered_strict_maxmin(
 ; CHECK-NEXT:    [[CMP2_INV:%.*]] = fcmp fast oge float [[X:%.*]], 2.550000e+02
-; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[CMP2_INV]], float 2.550000e+02, float [[X]]
-; CHECK-NEXT:    [[DOTINV:%.*]] = fcmp fast oge float [[MIN]], 1.000000e+00
-; CHECK-NEXT:    [[R1:%.*]] = select i1 [[DOTINV]], float [[MIN]], float 1.000000e+00
-; CHECK-NEXT:    ret float [[R1]]
+; CHECK-NEXT:    [[TMP1:%.*]] = select fast i1 [[CMP2_INV]], float 2.550000e+02, float [[X]]
+; CHECK-NEXT:    [[TMP2:%.*]] = call fast float @llvm.maxnum.f32(float [[TMP1]], float 1.000000e+00)
+; CHECK-NEXT:    ret float [[TMP2]]
 ;
   %cmp2 = fcmp fast ult float %x, 255.0
   %min = select i1 %cmp2, float %x, float 255.0
@@ -91,13 +81,11 @@ define float @clamp_float_fast_unordered_strict_maxmin(float %x) {
 
 ; (X <= C1) ? C1 : MIN(X, C2)
 define float @clamp_float_fast_unordered_nonstrict_maxmin(float %x) {
-;
-; CHECK-LABEL: @clamp_float_fast_unordered_nonstrict_maxmin(
+; CHECK-LABEL: define {{[^@]+}}@clamp_float_fast_unordered_nonstrict_maxmin(
 ; CHECK-NEXT:    [[CMP2_INV:%.*]] = fcmp fast oge float [[X:%.*]], 2.550000e+02
-; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[CMP2_INV]], float 2.550000e+02, float [[X]]
-; CHECK-NEXT:    [[DOTINV:%.*]] = fcmp fast oge float [[MIN]], 1.000000e+00
-; CHECK-NEXT:    [[R1:%.*]] = select i1 [[DOTINV]], float [[MIN]], float 1.000000e+00
-; CHECK-NEXT:    ret float [[R1]]
+; CHECK-NEXT:    [[TMP1:%.*]] = select fast i1 [[CMP2_INV]], float 2.550000e+02, float [[X]]
+; CHECK-NEXT:    [[TMP2:%.*]] = call fast float @llvm.maxnum.f32(float [[TMP1]], float 1.000000e+00)
+; CHECK-NEXT:    ret float [[TMP2]]
 ;
   %cmp2 = fcmp fast ult float %x, 255.0
   %min = select i1 %cmp2, float %x, float 255.0
@@ -108,13 +96,11 @@ define float @clamp_float_fast_unordered_nonstrict_maxmin(float %x) {
 
 ; (X > C1) ? C1 : MAX(X, C2)
 define float @clamp_float_fast_unordered_strict_minmax(float %x) {
-;
-; CHECK-LABEL: @clamp_float_fast_unordered_strict_minmax(
+; CHECK-LABEL: define {{[^@]+}}@clamp_float_fast_unordered_strict_minmax(
 ; CHECK-NEXT:    [[CMP2_INV:%.*]] = fcmp fast ole float [[X:%.*]], 1.000000e+00
-; CHECK-NEXT:    [[MAX:%.*]] = select i1 [[CMP2_INV]], float 1.000000e+00, float [[X]]
-; CHECK-NEXT:    [[DOTINV:%.*]] = fcmp fast ole float [[MAX]], 2.550000e+02
-; CHECK-NEXT:    [[R1:%.*]] = select i1 [[DOTINV]], float [[MAX]], float 2.550000e+02
-; CHECK-NEXT:    ret float [[R1]]
+; CHECK-NEXT:    [[TMP1:%.*]] = select fast i1 [[CMP2_INV]], float 1.000000e+00, float [[X]]
+; CHECK-NEXT:    [[TMP2:%.*]] = call fast float @llvm.minnum.f32(float [[TMP1]], float 2.550000e+02)
+; CHECK-NEXT:    ret float [[TMP2]]
 ;
   %cmp2 = fcmp fast ugt float %x, 1.0
   %max = select i1 %cmp2, float %x, float 1.0
@@ -125,13 +111,11 @@ define float @clamp_float_fast_unordered_strict_minmax(float %x) {
 
 ; (X >= C1) ? C1 : MAX(X, C2)
 define float @clamp_float_fast_unordered_nonstrict_minmax(float %x) {
-;
-; CHECK-LABEL: @clamp_float_fast_unordered_nonstrict_minmax(
+; CHECK-LABEL: define {{[^@]+}}@clamp_float_fast_unordered_nonstrict_minmax(
 ; CHECK-NEXT:    [[CMP2_INV:%.*]] = fcmp fast ole float [[X:%.*]], 1.000000e+00
-; CHECK-NEXT:    [[MAX:%.*]] = select i1 [[CMP2_INV]], float 1.000000e+00, float [[X]]
-; CHECK-NEXT:    [[DOTINV:%.*]] = fcmp fast ole float [[MAX]], 2.550000e+02
-; CHECK-NEXT:    [[R1:%.*]] = select i1 [[DOTINV]], float [[MAX]], float 2.550000e+02
-; CHECK-NEXT:    ret float [[R1]]
+; CHECK-NEXT:    [[TMP1:%.*]] = select fast i1 [[CMP2_INV]], float 1.000000e+00, float [[X]]
+; CHECK-NEXT:    [[TMP2:%.*]] = call fast float @llvm.minnum.f32(float [[TMP1]], float 2.550000e+02)
+; CHECK-NEXT:    ret float [[TMP2]]
 ;
   %cmp2 = fcmp fast ugt float %x, 1.0
   %max = select i1 %cmp2, float %x, float 1.0
@@ -145,12 +129,11 @@ define float @clamp_float_fast_unordered_nonstrict_minmax(float %x) {
 ; (X > 1.0) ? min(x, 255.0) : 1.0
 ; That did not match because select was in inverse order.
 define float @clamp_test_1(float %x) {
-; CHECK-LABEL: @clamp_test_1(
+; CHECK-LABEL: define {{[^@]+}}@clamp_test_1(
 ; CHECK-NEXT:    [[INNER_CMP_INV:%.*]] = fcmp fast oge float [[X:%.*]], 2.550000e+02
-; CHECK-NEXT:    [[INNER_SEL:%.*]] = select i1 [[INNER_CMP_INV]], float 2.550000e+02, float [[X]]
-; CHECK-NEXT:    [[DOTINV:%.*]] = fcmp fast oge float [[INNER_SEL]], 1.000000e+00
-; CHECK-NEXT:    [[R1:%.*]] = select i1 [[DOTINV]], float [[INNER_SEL]], float 1.000000e+00
-; CHECK-NEXT:    ret float [[R1]]
+; CHECK-NEXT:    [[TMP1:%.*]] = select fast i1 [[INNER_CMP_INV]], float 2.550000e+02, float [[X]]
+; CHECK-NEXT:    [[TMP2:%.*]] = call fast float @llvm.maxnum.f32(float [[TMP1]], float 1.000000e+00)
+; CHECK-NEXT:    ret float [[TMP2]]
 ;
   %inner_cmp = fcmp fast ult float %x, 255.0
   %inner_sel = select i1 %inner_cmp, float %x, float 255.0
@@ -163,11 +146,11 @@ define float @clamp_test_1(float %x) {
 
 ; Like @clamp_test_1 but HighConst < LowConst
 define float @clamp_negative_wrong_const(float %x) {
-; CHECK-LABEL: @clamp_negative_wrong_const(
+; CHECK-LABEL: define {{[^@]+}}@clamp_negative_wrong_const(
 ; CHECK-NEXT:    [[INNER_CMP_INV:%.*]] = fcmp fast oge float [[X:%.*]], 2.550000e+02
-; CHECK-NEXT:    [[INNER_SEL:%.*]] = select i1 [[INNER_CMP_INV]], float 2.550000e+02, float [[X]]
+; CHECK-NEXT:    [[TMP1:%.*]] = select fast i1 [[INNER_CMP_INV]], float 2.550000e+02, float [[X]]
 ; CHECK-NEXT:    [[OUTER_CMP:%.*]] = fcmp fast ugt float [[X]], 5.120000e+02
-; CHECK-NEXT:    [[R:%.*]] = select i1 [[OUTER_CMP]], float [[INNER_SEL]], float 5.120000e+02
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[OUTER_CMP]], float [[TMP1]], float 5.120000e+02
 ; CHECK-NEXT:    ret float [[R]]
 ;
   %inner_cmp = fcmp fast ult float %x, 255.0
@@ -179,11 +162,11 @@ define float @clamp_negative_wrong_const(float %x) {
 
 ; Like @clamp_test_1 but both are min
 define float @clamp_negative_same_op(float %x) {
-; CHECK-LABEL: @clamp_negative_same_op(
+; CHECK-LABEL: define {{[^@]+}}@clamp_negative_same_op(
 ; CHECK-NEXT:    [[INNER_CMP_INV:%.*]] = fcmp fast oge float [[X:%.*]], 2.550000e+02
-; CHECK-NEXT:    [[INNER_SEL:%.*]] = select i1 [[INNER_CMP_INV]], float 2.550000e+02, float [[X]]
+; CHECK-NEXT:    [[TMP1:%.*]] = select fast i1 [[INNER_CMP_INV]], float 2.550000e+02, float [[X]]
 ; CHECK-NEXT:    [[OUTER_CMP:%.*]] = fcmp fast ult float [[X]], 1.000000e+00
-; CHECK-NEXT:    [[R:%.*]] = select i1 [[OUTER_CMP]], float [[INNER_SEL]], float 1.000000e+00
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[OUTER_CMP]], float [[TMP1]], float 1.000000e+00
 ; CHECK-NEXT:    ret float [[R]]
 ;
   %inner_cmp = fcmp fast ult float %x, 255.0
@@ -198,7 +181,7 @@ define float @clamp_negative_same_op(float %x) {
 
 ; First, check that we don't do bad things in the presence of signed zeros
 define float @clamp_float_with_zero1(float %x) {
-; CHECK-LABEL: @clamp_float_with_zero1(
+; CHECK-LABEL: define {{[^@]+}}@clamp_float_with_zero1(
 ; CHECK-NEXT:    [[CMP2:%.*]] = fcmp fast olt float [[X:%.*]], 2.550000e+02
 ; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[CMP2]], float [[X]], float 2.550000e+02
 ; CHECK-NEXT:    [[CMP1:%.*]] = fcmp ole float [[X]], 0.000000e+00
@@ -213,7 +196,7 @@ define float @clamp_float_with_zero1(float %x) {
 }
 
 define float @clamp_float_with_zero2(float %x) {
-; CHECK-LABEL: @clamp_float_with_zero2(
+; CHECK-LABEL: define {{[^@]+}}@clamp_float_with_zero2(
 ; CHECK-NEXT:    [[CMP2:%.*]] = fcmp fast olt float [[X:%.*]], 2.550000e+02
 ; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[CMP2]], float [[X]], float 2.550000e+02
 ; CHECK-NEXT:    [[CMP1:%.*]] = fcmp olt float [[X]], 0.000000e+00
@@ -235,8 +218,7 @@ define float @clamp_float_with_zero2(float %x) {
 
 ; (X < C1) ? C1 : MIN(X, C2)
 define float @clamp_float_ordered_strict_maxmin1(float %x) {
-;
-; CHECK-LABEL: @clamp_float_ordered_strict_maxmin1(
+; CHECK-LABEL: define {{[^@]+}}@clamp_float_ordered_strict_maxmin1(
 ; CHECK-NEXT:    [[CMP2:%.*]] = fcmp olt float [[X:%.*]], 2.550000e+02
 ; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[CMP2]], float [[X]], float 2.550000e+02
 ; CHECK-NEXT:    [[CMP1:%.*]] = fcmp olt float [[X]], 1.000000e+00
@@ -251,8 +233,7 @@ define float @clamp_float_ordered_strict_maxmin1(float %x) {
 }
 
 define float @clamp_float_ordered_strict_maxmin2(float %x) {
-;
-; CHECK-LABEL: @clamp_float_ordered_strict_maxmin2(
+; CHECK-LABEL: define {{[^@]+}}@clamp_float_ordered_strict_maxmin2(
 ; CHECK-NEXT:    [[CMP2_INV:%.*]] = fcmp oge float [[X:%.*]], 2.550000e+02
 ; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[CMP2_INV]], float 2.550000e+02, float [[X]]
 ; CHECK-NEXT:    [[CMP1:%.*]] = fcmp olt float [[X]], 1.000000e+00
@@ -268,8 +249,7 @@ define float @clamp_float_ordered_strict_maxmin2(float %x) {
 
 ; (X <= C1) ? C1 : MIN(X, C2)
 define float @clamp_float_ordered_nonstrict_maxmin1(float %x) {
-;
-; CHECK-LABEL: @clamp_float_ordered_nonstrict_maxmin1(
+; CHECK-LABEL: define {{[^@]+}}@clamp_float_ordered_nonstrict_maxmin1(
 ; CHECK-NEXT:    [[CMP2:%.*]] = fcmp olt float [[X:%.*]], 2.550000e+02
 ; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[CMP2]], float [[X]], float 2.550000e+02
 ; CHECK-NEXT:    [[CMP1:%.*]] = fcmp ole float [[X]], 1.000000e+00
@@ -284,8 +264,7 @@ define float @clamp_float_ordered_nonstrict_maxmin1(float %x) {
 }
 
 define float @clamp_float_ordered_nonstrict_maxmin2(float %x) {
-;
-; CHECK-LABEL: @clamp_float_ordered_nonstrict_maxmin2(
+; CHECK-LABEL: define {{[^@]+}}@clamp_float_ordered_nonstrict_maxmin2(
 ; CHECK-NEXT:    [[CMP2_INV:%.*]] = fcmp oge float [[X:%.*]], 2.550000e+02
 ; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[CMP2_INV]], float 2.550000e+02, float [[X]]
 ; CHECK-NEXT:    [[CMP1:%.*]] = fcmp ole float [[X]], 1.000000e+00
@@ -301,8 +280,7 @@ define float @clamp_float_ordered_nonstrict_maxmin2(float %x) {
 
 ; (X > C1) ? C1 : MAX(X, C2)
 define float @clamp_float_ordered_strict_minmax1(float %x) {
-;
-; CHECK-LABEL: @clamp_float_ordered_strict_minmax1(
+; CHECK-LABEL: define {{[^@]+}}@clamp_float_ordered_strict_minmax1(
 ; CHECK-NEXT:    [[CMP2:%.*]] = fcmp ogt float [[X:%.*]], 1.000000e+00
 ; CHECK-NEXT:    [[MAX:%.*]] = select i1 [[CMP2]], float [[X]], float 1.000000e+00
 ; CHECK-NEXT:    [[CMP1:%.*]] = fcmp ogt float [[X]], 2.550000e+02
@@ -317,8 +295,7 @@ define float @clamp_float_ordered_strict_minmax1(float %x) {
 }
 
 define float @clamp_float_ordered_strict_minmax2(float %x) {
-;
-; CHECK-LABEL: @clamp_float_ordered_strict_minmax2(
+; CHECK-LABEL: define {{[^@]+}}@clamp_float_ordered_strict_minmax2(
 ; CHECK-NEXT:    [[CMP2_INV:%.*]] = fcmp ole float [[X:%.*]], 1.000000e+00
 ; CHECK-NEXT:    [[MAX:%.*]] = select i1 [[CMP2_INV]], float 1.000000e+00, float [[X]]
 ; CHECK-NEXT:    [[CMP1:%.*]] = fcmp ogt float [[X]], 2.550000e+02
@@ -334,8 +311,7 @@ define float @clamp_float_ordered_strict_minmax2(float %x) {
 
 ; (X >= C1) ? C1 : MAX(X, C2)
 define float @clamp_float_ordered_nonstrict_minmax1(float %x) {
-;
-; CHECK-LABEL: @clamp_float_ordered_nonstrict_minmax1(
+; CHECK-LABEL: define {{[^@]+}}@clamp_float_ordered_nonstrict_minmax1(
 ; CHECK-NEXT:    [[CMP2:%.*]] = fcmp ogt float [[X:%.*]], 1.000000e+00
 ; CHECK-NEXT:    [[MAX:%.*]] = select i1 [[CMP2]], float [[X]], float 1.000000e+00
 ; CHECK-NEXT:    [[CMP1:%.*]] = fcmp oge float [[X]], 2.550000e+02
@@ -350,8 +326,7 @@ define float @clamp_float_ordered_nonstrict_minmax1(float %x) {
 }
 
 define float @clamp_float_ordered_nonstrict_minmax2(float %x) {
-;
-; CHECK-LABEL: @clamp_float_ordered_nonstrict_minmax2(
+; CHECK-LABEL: define {{[^@]+}}@clamp_float_ordered_nonstrict_minmax2(
 ; CHECK-NEXT:    [[CMP2_INV:%.*]] = fcmp ole float [[X:%.*]], 1.000000e+00
 ; CHECK-NEXT:    [[MAX:%.*]] = select i1 [[CMP2_INV]], float 1.000000e+00, float [[X]]
 ; CHECK-NEXT:    [[CMP1:%.*]] = fcmp oge float [[X]], 2.550000e+02
@@ -370,8 +345,7 @@ define float @clamp_float_ordered_nonstrict_minmax2(float %x) {
 
 ; (X < C1) ? C1 : MIN(X, C2)
 define float @clamp_float_unordered_strict_maxmin1(float %x) {
-;
-; CHECK-LABEL: @clamp_float_unordered_strict_maxmin1(
+; CHECK-LABEL: define {{[^@]+}}@clamp_float_unordered_strict_maxmin1(
 ; CHECK-NEXT:    [[CMP2:%.*]] = fcmp olt float [[X:%.*]], 2.550000e+02
 ; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[CMP2]], float [[X]], float 2.550000e+02
 ; CHECK-NEXT:    [[CMP1:%.*]] = fcmp ult float [[X]], 1.000000e+00
@@ -386,8 +360,7 @@ define float @clamp_float_unordered_strict_maxmin1(float %x) {
 }
 
 define float @clamp_float_unordered_strict_maxmin2(float %x) {
-;
-; CHECK-LABEL: @clamp_float_unordered_strict_maxmin2(
+; CHECK-LABEL: define {{[^@]+}}@clamp_float_unordered_strict_maxmin2(
 ; CHECK-NEXT:    [[CMP2_INV:%.*]] = fcmp oge float [[X:%.*]], 2.550000e+02
 ; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[CMP2_INV]], float 2.550000e+02, float [[X]]
 ; CHECK-NEXT:    [[CMP1:%.*]] = fcmp ult float [[X]], 1.000000e+00
@@ -403,8 +376,7 @@ define float @clamp_float_unordered_strict_maxmin2(float %x) {
 
 ; (X <= C1) ? C1 : MIN(X, C2)
 define float @clamp_float_unordered_nonstrict_maxmin1(float %x) {
-;
-; CHECK-LABEL: @clamp_float_unordered_nonstrict_maxmin1(
+; CHECK-LABEL: define {{[^@]+}}@clamp_float_unordered_nonstrict_maxmin1(
 ; CHECK-NEXT:    [[CMP2:%.*]] = fcmp olt float [[X:%.*]], 2.550000e+02
 ; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[CMP2]], float [[X]], float 2.550000e+02
 ; CHECK-NEXT:    [[CMP1:%.*]] = fcmp ule float [[X]], 1.000000e+00
@@ -419,8 +391,7 @@ define float @clamp_float_unordered_nonstrict_maxmin1(float %x) {
 }
 
 define float @clamp_float_unordered_nonstrict_maxmin2(float %x) {
-;
-; CHECK-LABEL: @clamp_float_unordered_nonstrict_maxmin2(
+; CHECK-LABEL: define {{[^@]+}}@clamp_float_unordered_nonstrict_maxmin2(
 ; CHECK-NEXT:    [[CMP2_INV:%.*]] = fcmp oge float [[X:%.*]], 2.550000e+02
 ; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[CMP2_INV]], float 2.550000e+02, float [[X]]
 ; CHECK-NEXT:    [[CMP1:%.*]] = fcmp ule float [[X]], 1.000000e+00
@@ -436,8 +407,7 @@ define float @clamp_float_unordered_nonstrict_maxmin2(float %x) {
 
 ; (X > C1) ? C1 : MAX(X, C2)
 define float @clamp_float_unordered_strict_minmax1(float %x) {
-;
-; CHECK-LABEL: @clamp_float_unordered_strict_minmax1(
+; CHECK-LABEL: define {{[^@]+}}@clamp_float_unordered_strict_minmax1(
 ; CHECK-NEXT:    [[CMP2:%.*]] = fcmp ogt float [[X:%.*]], 1.000000e+00
 ; CHECK-NEXT:    [[MAX:%.*]] = select i1 [[CMP2]], float [[X]], float 1.000000e+00
 ; CHECK-NEXT:    [[CMP1:%.*]] = fcmp ugt float [[X]], 2.550000e+02
@@ -452,8 +422,7 @@ define float @clamp_float_unordered_strict_minmax1(float %x) {
 }
 
 define float @clamp_float_unordered_strict_minmax2(float %x) {
-;
-; CHECK-LABEL: @clamp_float_unordered_strict_minmax2(
+; CHECK-LABEL: define {{[^@]+}}@clamp_float_unordered_strict_minmax2(
 ; CHECK-NEXT:    [[CMP2_INV:%.*]] = fcmp ole float [[X:%.*]], 1.000000e+00
 ; CHECK-NEXT:    [[MAX:%.*]] = select i1 [[CMP2_INV]], float 1.000000e+00, float [[X]]
 ; CHECK-NEXT:    [[CMP1:%.*]] = fcmp ugt float [[X]], 2.550000e+02
@@ -469,8 +438,7 @@ define float @clamp_float_unordered_strict_minmax2(float %x) {
 
 ; (X >= C1) ? C1 : MAX(X, C2)
 define float @clamp_float_unordered_nonstrict_minmax1(float %x) {
-;
-; CHECK-LABEL: @clamp_float_unordered_nonstrict_minmax1(
+; CHECK-LABEL: define {{[^@]+}}@clamp_float_unordered_nonstrict_minmax1(
 ; CHECK-NEXT:    [[CMP2:%.*]] = fcmp ogt float [[X:%.*]], 1.000000e+00
 ; CHECK-NEXT:    [[MAX:%.*]] = select i1 [[CMP2]], float [[X]], float 1.000000e+00
 ; CHECK-NEXT:    [[CMP1:%.*]] = fcmp uge float [[X]], 2.550000e+02
@@ -485,8 +453,7 @@ define float @clamp_float_unordered_nonstrict_minmax1(float %x) {
 }
 
 define float @clamp_float_unordered_nonstrict_minmax2(float %x) {
-;
-; CHECK-LABEL: @clamp_float_unordered_nonstrict_minmax2(
+; CHECK-LABEL: define {{[^@]+}}@clamp_float_unordered_nonstrict_minmax2(
 ; CHECK-NEXT:    [[CMP2_INV:%.*]] = fcmp ole float [[X:%.*]], 1.000000e+00
 ; CHECK-NEXT:    [[MAX:%.*]] = select i1 [[CMP2_INV]], float 1.000000e+00, float [[X]]
 ; CHECK-NEXT:    [[CMP1:%.*]] = fcmp uge float [[X]], 2.550000e+02
@@ -502,7 +469,7 @@ define float @clamp_float_unordered_nonstrict_minmax2(float %x) {
 
 ;; Check casts behavior
 define float @ui32_clamp_and_cast_to_float(i32 %x) {
-; CHECK-LABEL: @ui32_clamp_and_cast_to_float(
+; CHECK-LABEL: define {{[^@]+}}@ui32_clamp_and_cast_to_float(
 ; CHECK-NEXT:    [[LO_CMP:%.*]] = icmp eq i32 [[X:%.*]], 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[X]], 255
 ; CHECK-NEXT:    [[MIN1:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 255
@@ -519,7 +486,7 @@ define float @ui32_clamp_and_cast_to_float(i32 %x) {
 }
 
 define float @ui64_clamp_and_cast_to_float(i64 %x) {
-; CHECK-LABEL: @ui64_clamp_and_cast_to_float(
+; CHECK-LABEL: define {{[^@]+}}@ui64_clamp_and_cast_to_float(
 ; CHECK-NEXT:    [[LO_CMP:%.*]] = icmp eq i64 [[X:%.*]], 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i64 [[X]], 255
 ; CHECK-NEXT:    [[MIN1:%.*]] = select i1 [[TMP1]], i64 [[X]], i64 255
@@ -536,7 +503,7 @@ define float @ui64_clamp_and_cast_to_float(i64 %x) {
 }
 
 define float @mixed_clamp_to_float_1(i32 %x) {
-; CHECK-LABEL: @mixed_clamp_to_float_1(
+; CHECK-LABEL: define {{[^@]+}}@mixed_clamp_to_float_1(
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[X:%.*]], 255
 ; CHECK-NEXT:    [[SI_MIN:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 255
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp sgt i32 [[SI_MIN]], 1
@@ -554,7 +521,7 @@ define float @mixed_clamp_to_float_1(i32 %x) {
 }
 
 define i32 @mixed_clamp_to_i32_1(float %x) {
-; CHECK-LABEL: @mixed_clamp_to_i32_1(
+; CHECK-LABEL: define {{[^@]+}}@mixed_clamp_to_i32_1(
 ; CHECK-NEXT:    [[FLOAT_MIN_CMP:%.*]] = fcmp ogt float [[X:%.*]], 2.550000e+02
 ; CHECK-NEXT:    [[FLOAT_MIN:%.*]] = select i1 [[FLOAT_MIN_CMP]], float 2.550000e+02, float [[X]]
 ; CHECK-NEXT:    [[I32_MIN:%.*]] = fptosi float [[FLOAT_MIN]] to i32
@@ -573,7 +540,7 @@ define i32 @mixed_clamp_to_i32_1(float %x) {
 }
 
 define float @mixed_clamp_to_float_2(i32 %x) {
-; CHECK-LABEL: @mixed_clamp_to_float_2(
+; CHECK-LABEL: define {{[^@]+}}@mixed_clamp_to_float_2(
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[X:%.*]], 255
 ; CHECK-NEXT:    [[SI_MIN:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 255
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp sgt i32 [[SI_MIN]], 1
@@ -590,7 +557,7 @@ define float @mixed_clamp_to_float_2(i32 %x) {
 }
 
 define i32 @mixed_clamp_to_i32_2(float %x) {
-; CHECK-LABEL: @mixed_clamp_to_i32_2(
+; CHECK-LABEL: define {{[^@]+}}@mixed_clamp_to_i32_2(
 ; CHECK-NEXT:    [[FLOAT_MIN_CMP:%.*]] = fcmp ogt float [[X:%.*]], 2.550000e+02
 ; CHECK-NEXT:    [[FLOAT_MIN:%.*]] = select i1 [[FLOAT_MIN_CMP]], float 2.550000e+02, float [[X]]
 ; CHECK-NEXT:    [[I32_MIN:%.*]] = fptosi float [[FLOAT_MIN]] to i32
diff --git a/llvm/test/Transforms/InstCombine/compare-unescaped.ll b/llvm/test/Transforms/InstCombine/compare-unescaped.ll
index d15fc2fd4..a19f80396 100644
--- a/llvm/test/Transforms/InstCombine/compare-unescaped.ll
+++ b/llvm/test/Transforms/InstCombine/compare-unescaped.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -instcombine -S < %s | FileCheck %s
 
 @gp = global i32* null, align 8
@@ -5,23 +6,25 @@
 declare i8* @malloc(i64) #1
 
 define i1 @compare_global_trivialeq() {
+; CHECK-LABEL: @compare_global_trivialeq(
+; CHECK-NEXT:    ret i1 false
+;
   %m = call i8* @malloc(i64 4)
   %bc = bitcast i8* %m to i32*
   %lgp = load i32*, i32** @gp, align 8
   %cmp = icmp eq i32* %bc, %lgp
   ret i1 %cmp
-; CHECK-LABEL: compare_global_trivialeq
-; CHECK: ret i1 false
 }
 
 define i1 @compare_global_trivialne() {
+; CHECK-LABEL: @compare_global_trivialne(
+; CHECK-NEXT:    ret i1 true
+;
   %m = call i8* @malloc(i64 4)
   %bc = bitcast i8* %m to i32*
   %lgp = load i32*, i32** @gp, align 8
   %cmp = icmp ne i32* %bc, %lgp
   ret i1 %cmp
-; CHECK-LABEL: compare_global_trivialne
-; CHECK: ret i1 true
 }
 
 
@@ -30,45 +33,68 @@ define i1 @compare_global_trivialne() {
 ; The comparison should fold to false irrespective of whether the call to malloc can be elided or not
 declare void @f()
 define i1 @compare_and_call_with_deopt() {
-; CHECK-LABEL: compare_and_call_with_deopt
+; CHECK-LABEL: @compare_and_call_with_deopt(
+; CHECK-NEXT:    [[M:%.*]] = call dereferenceable_or_null(24) i8* @malloc(i64 24)
+; CHECK-NEXT:    tail call void @f() [ "deopt"(i8* [[M]]) ]
+; CHECK-NEXT:    ret i1 false
+;
   %m = call i8* @malloc(i64 24)
   %bc = bitcast i8* %m to i32*
   %lgp = load i32*, i32** @gp, align 8, !nonnull !0
   %cmp = icmp eq i32* %lgp, %bc
   tail call void @f() [ "deopt"(i8* %m) ]
   ret i1 %cmp
-; CHECK: ret i1 false
 }
 
 ; Same functon as above with deopt operand in function f, but comparison is NE
 define i1 @compare_ne_and_call_with_deopt() {
-; CHECK-LABEL: compare_ne_and_call_with_deopt
+; CHECK-LABEL: @compare_ne_and_call_with_deopt(
+; CHECK-NEXT:    [[M:%.*]] = call dereferenceable_or_null(24) i8* @malloc(i64 24)
+; CHECK-NEXT:    tail call void @f() [ "deopt"(i8* [[M]]) ]
+; CHECK-NEXT:    ret i1 true
+;
   %m = call i8* @malloc(i64 24)
   %bc = bitcast i8* %m to i32*
   %lgp = load i32*, i32** @gp, align 8, !nonnull !0
   %cmp = icmp ne i32* %lgp, %bc
   tail call void @f() [ "deopt"(i8* %m) ]
   ret i1 %cmp
-; CHECK: ret i1 true
 }
 
 ; Same function as above, but global not marked nonnull, and we cannot fold the comparison
 define i1 @compare_ne_global_maybe_null() {
-; CHECK-LABEL: compare_ne_global_maybe_null
+; CHECK-LABEL: @compare_ne_global_maybe_null(
+; CHECK-NEXT:    [[M:%.*]] = call dereferenceable_or_null(24) i8* @malloc(i64 24)
+; CHECK-NEXT:    [[BC:%.*]] = bitcast i8* [[M]] to i32*
+; CHECK-NEXT:    [[LGP:%.*]] = load i32*, i32** @gp, align 8
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32* [[LGP]], [[BC]]
+; CHECK-NEXT:    tail call void @f() [ "deopt"(i8* [[M]]) ]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
   %m = call i8* @malloc(i64 24)
   %bc = bitcast i8* %m to i32*
   %lgp = load i32*, i32** @gp
   %cmp = icmp ne i32* %lgp, %bc
   tail call void @f() [ "deopt"(i8* %m) ]
   ret i1 %cmp
-; CHECK: ret i1 %cmp
 }
 
 ; FIXME: The comparison should fold to false since %m escapes (call to function escape)
 ; after the comparison.
 declare void @escape(i8*)
 define i1 @compare_and_call_after() {
-; CHECK-LABEL: compare_and_call_after
+; CHECK-LABEL: @compare_and_call_after(
+; CHECK-NEXT:    [[M:%.*]] = call dereferenceable_or_null(24) i8* @malloc(i64 24)
+; CHECK-NEXT:    [[BC:%.*]] = bitcast i8* [[M]] to i32*
+; CHECK-NEXT:    [[LGP:%.*]] = load i32*, i32** @gp, align 8, !nonnull !0
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32* [[LGP]], [[BC]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[ESCAPE_CALL:%.*]], label [[JUST_RETURN:%.*]]
+; CHECK:       escape_call:
+; CHECK-NEXT:    call void @escape(i8* [[M]])
+; CHECK-NEXT:    ret i1 true
+; CHECK:       just_return:
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
   %m = call i8* @malloc(i64 24)
   %bc = bitcast i8* %m to i32*
   %lgp = load i32*, i32** @gp, align 8, !nonnull !0
@@ -76,56 +102,74 @@ define i1 @compare_and_call_after() {
   br i1 %cmp, label %escape_call, label %just_return
 
 escape_call:
- call void @escape(i8* %m)
- ret i1 true
+  call void @escape(i8* %m)
+  ret i1 true
 
 just_return:
- ret i1 %cmp
+  ret i1 %cmp
 }
 
 define i1 @compare_distinct_mallocs() {
+; CHECK-LABEL: @compare_distinct_mallocs(
+; CHECK-NEXT:    ret i1 false
+;
   %m = call i8* @malloc(i64 4)
   %n = call i8* @malloc(i64 4)
   %cmp = icmp eq i8* %m, %n
   ret i1 %cmp
-  ; CHECK-LABEL: compare_distinct_mallocs
-  ; CHECK: ret i1 false
 }
 
-; the compare is folded to true since the folding compare looks through bitcasts. 
-; call to malloc and the bitcast instructions are elided after that since there are no uses of the malloc 
+; the compare is folded to true since the folding compare looks through bitcasts.
+; call to malloc and the bitcast instructions are elided after that since there are no uses of the malloc
 define i1 @compare_samepointer_under_bitcast() {
+; CHECK-LABEL: @compare_samepointer_under_bitcast(
+; CHECK-NEXT:    ret i1 true
+;
   %m = call i8* @malloc(i64 4)
   %bc = bitcast i8* %m to i32*
   %bcback = bitcast i32* %bc to i8*
   %cmp = icmp eq i8* %m, %bcback
   ret i1 %cmp
-; CHECK-LABEL: compare_samepointer_under_bitcast
-; CHECK: ret i1 true 
 }
 
-; the compare is folded to true since the folding compare looks through bitcasts. 
+; the compare is folded to true since the folding compare looks through bitcasts.
 ; The malloc call for %m cannot be elided since it is used in the call to function f.
 define i1 @compare_samepointer_escaped() {
+; CHECK-LABEL: @compare_samepointer_escaped(
+; CHECK-NEXT:    [[M:%.*]] = call dereferenceable_or_null(4) i8* @malloc(i64 4)
+; CHECK-NEXT:    call void @f() [ "deopt"(i8* [[M]]) ]
+; CHECK-NEXT:    ret i1 true
+;
   %m = call i8* @malloc(i64 4)
   %bc = bitcast i8* %m to i32*
   %bcback = bitcast i32* %bc to i8*
   %cmp = icmp eq i8* %m, %bcback
   call void @f() [ "deopt"(i8* %m) ]
   ret i1 %cmp
-; CHECK-LABEL: compare_samepointer_escaped
-; CHECK-NEXT: %m = call i8* @malloc(i64 4)
-; CHECK-NEXT: call void @f() [ "deopt"(i8* %m) ]
-; CHECK: ret i1 true 
 }
 
 ; Technically, we can fold the %cmp2 comparison, even though %m escapes through
 ; the ret statement since `ret` terminates the function and we cannot reach from
-; the ret to cmp. 
+; the ret to cmp.
 ; FIXME: Folding this %cmp2 when %m escapes through ret could be an issue with
 ; cross-threading data dependencies since we do not make the distinction between
 ; atomic and non-atomic loads in capture tracking.
 define i8* @compare_ret_escape(i8* %c) {
+; CHECK-LABEL: @compare_ret_escape(
+; CHECK-NEXT:    [[M:%.*]] = call dereferenceable_or_null(4) i8* @malloc(i64 4)
+; CHECK-NEXT:    [[N:%.*]] = call dereferenceable_or_null(4) i8* @malloc(i64 4)
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8* [[N]], [[C:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[RETST:%.*]], label [[CHK:%.*]]
+; CHECK:       retst:
+; CHECK-NEXT:    ret i8* [[M]]
+; CHECK:       chk:
+; CHECK-NEXT:    [[BC:%.*]] = bitcast i8* [[M]] to i32*
+; CHECK-NEXT:    [[LGP:%.*]] = load i32*, i32** @gp, align 8, !nonnull !0
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp eq i32* [[LGP]], [[BC]]
+; CHECK-NEXT:    br i1 [[CMP2]], label [[RETST]], label [[CHK2:%.*]]
+; CHECK:       chk2:
+; CHECK-NEXT:    ret i8* [[N]]
+;
   %m = call i8* @malloc(i64 4)
   %n = call i8* @malloc(i64 4)
   %cmp = icmp eq i8* %n, %c
@@ -142,23 +186,21 @@ chk:
 
 chk2:
   ret i8* %n
-; CHECK-LABEL: compare_ret_escape
-; CHECK: %cmp = icmp eq i8* %n, %c
-; CHECK: %cmp2 = icmp eq i32* %lgp, %bc
 }
 
 ; The malloc call for %m cannot be elided since it is used in the call to function f.
 ; However, the cmp can be folded to true as %n doesnt escape and %m, %n are distinct allocations
 define i1 @compare_distinct_pointer_escape() {
+; CHECK-LABEL: @compare_distinct_pointer_escape(
+; CHECK-NEXT:    [[M:%.*]] = call dereferenceable_or_null(4) i8* @malloc(i64 4)
+; CHECK-NEXT:    tail call void @f() [ "deopt"(i8* [[M]]) ]
+; CHECK-NEXT:    ret i1 true
+;
   %m = call i8* @malloc(i64 4)
   %n = call i8* @malloc(i64 4)
   tail call void @f() [ "deopt"(i8* %m) ]
   %cmp = icmp ne i8* %m, %n
   ret i1 %cmp
-; CHECK-LABEL: compare_distinct_pointer_escape
-; CHECK-NEXT: %m = call i8* @malloc(i64 4)
-; CHECK-NEXT: tail call void @f() [ "deopt"(i8* %m) ]
-; CHECK-NEXT: ret i1 true
 }
 
 !0 = !{}
diff --git a/llvm/test/Transforms/InstCombine/conditional-variable-length-signext-after-high-bit-extract.ll b/llvm/test/Transforms/InstCombine/conditional-variable-length-signext-after-high-bit-extract.ll
new file mode 100644
index 000000000..cb4d38d66
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/conditional-variable-length-signext-after-high-bit-extract.ll
@@ -0,0 +1,1139 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instcombine -S | FileCheck %s
+
+; If we extract (via lshr) some high bits, and then perform their sign-extension
+; conditionally depending on whether the extracted value is negative or not
+; (i.e. interpreting the highest extracted bit, which was the original signbit
+; of the value from which we extracted as a signbit), then we should just
+; perform extraction via `ashr`.
+
+; Base patterns.
+
+declare void @use1(i1)
+declare void @use16(i16)
+declare void @use32(i32)
+declare void @use64(i64)
+
+define i32 @t0_notrunc_add(i32 %data, i32 %nbits) {
+; CHECK-LABEL: @t0_notrunc_add(
+; CHECK-NEXT:    [[LOW_BITS_TO_SKIP:%.*]] = sub i32 32, [[NBITS:%.*]]
+; CHECK-NEXT:    [[HIGH_BITS_EXTRACTED:%.*]] = lshr i32 [[DATA:%.*]], [[LOW_BITS_TO_SKIP]]
+; CHECK-NEXT:    [[SHOULD_SIGNEXT:%.*]] = icmp slt i32 [[DATA]], 0
+; CHECK-NEXT:    [[ALL_BITS_EXCEPT_LOW_NBITS:%.*]] = shl i32 -1, [[NBITS]]
+; CHECK-NEXT:    [[MAGIC:%.*]] = select i1 [[SHOULD_SIGNEXT]], i32 [[ALL_BITS_EXCEPT_LOW_NBITS]], i32 0
+; CHECK-NEXT:    call void @use32(i32 [[LOW_BITS_TO_SKIP]])
+; CHECK-NEXT:    call void @use32(i32 [[HIGH_BITS_EXTRACTED]])
+; CHECK-NEXT:    call void @use1(i1 [[SHOULD_SIGNEXT]])
+; CHECK-NEXT:    call void @use32(i32 [[ALL_BITS_EXCEPT_LOW_NBITS]])
+; CHECK-NEXT:    call void @use32(i32 [[MAGIC]])
+; CHECK-NEXT:    [[SIGNEXTENDED:%.*]] = ashr i32 [[DATA]], [[LOW_BITS_TO_SKIP]]
+; CHECK-NEXT:    ret i32 [[SIGNEXTENDED]]
+;
+  %low_bits_to_skip = sub i32 32, %nbits
+  %high_bits_extracted = lshr i32 %data, %low_bits_to_skip
+  %should_signext = icmp slt i32 %data, 0
+  %all_bits_except_low_nbits = shl i32 -1, %nbits
+  %magic = select i1 %should_signext, i32 %all_bits_except_low_nbits, i32 0
+
+  call void @use32(i32 %low_bits_to_skip)
+  call void @use32(i32 %high_bits_extracted)
+  call void @use1(i1 %should_signext)
+  call void @use32(i32 %all_bits_except_low_nbits)
+  call void @use32(i32 %magic)
+
+  %signextended = add i32 %high_bits_extracted, %magic
+  ret i32 %signextended
+}
+
+define i32 @t0_notrunc_or(i32 %data, i32 %nbits) {
+; CHECK-LABEL: @t0_notrunc_or(
+; CHECK-NEXT:    [[LOW_BITS_TO_SKIP:%.*]] = sub i32 32, [[NBITS:%.*]]
+; CHECK-NEXT:    [[HIGH_BITS_EXTRACTED:%.*]] = lshr i32 [[DATA:%.*]], [[LOW_BITS_TO_SKIP]]
+; CHECK-NEXT:    [[SHOULD_SIGNEXT:%.*]] = icmp slt i32 [[DATA]], 0
+; CHECK-NEXT:    [[ALL_BITS_EXCEPT_LOW_NBITS:%.*]] = shl i32 -1, [[NBITS]]
+; CHECK-NEXT:    [[MAGIC:%.*]] = select i1 [[SHOULD_SIGNEXT]], i32 [[ALL_BITS_EXCEPT_LOW_NBITS]], i32 0
+; CHECK-NEXT:    call void @use32(i32 [[LOW_BITS_TO_SKIP]])
+; CHECK-NEXT:    call void @use32(i32 [[HIGH_BITS_EXTRACTED]])
+; CHECK-NEXT:    call void @use1(i1 [[SHOULD_SIGNEXT]])
+; CHECK-NEXT:    call void @use32(i32 [[ALL_BITS_EXCEPT_LOW_NBITS]])
+; CHECK-NEXT:    call void @use32(i32 [[MAGIC]])
+; CHECK-NEXT:    [[SIGNEXTENDED:%.*]] = ashr i32 [[DATA]], [[LOW_BITS_TO_SKIP]]
+; CHECK-NEXT:    ret i32 [[SIGNEXTENDED]]
+;
+  %low_bits_to_skip = sub i32 32, %nbits
+  %high_bits_extracted = lshr i32 %data, %low_bits_to_skip
+  %should_signext = icmp slt i32 %data, 0
+  %all_bits_except_low_nbits = shl i32 -1, %nbits
+  %magic = select i1 %should_signext, i32 %all_bits_except_low_nbits, i32 0
+
+  call void @use32(i32 %low_bits_to_skip)
+  call void @use32(i32 %high_bits_extracted)
+  call void @use1(i1 %should_signext)
+  call void @use32(i32 %all_bits_except_low_nbits)
+  call void @use32(i32 %magic)
+
+  %signextended = or i32 %high_bits_extracted, %magic
+  ret i32 %signextended
+}
+
+define i32 @t1_notrunc_sub(i32 %data, i32 %nbits) {
+; CHECK-LABEL: @t1_notrunc_sub(
+; CHECK-NEXT:    [[LOW_BITS_TO_SKIP:%.*]] = sub i32 32, [[NBITS:%.*]]
+; CHECK-NEXT:    [[HIGH_BITS_EXTRACTED:%.*]] = lshr i32 [[DATA:%.*]], [[LOW_BITS_TO_SKIP]]
+; CHECK-NEXT:    [[SHOULD_SIGNEXT:%.*]] = icmp slt i32 [[DATA]], 0
+; CHECK-NEXT:    [[HIGHER_BIT_AFTER_SIGNBIT:%.*]] = shl i32 1, [[NBITS]]
+; CHECK-NEXT:    [[MAGIC:%.*]] = select i1 [[SHOULD_SIGNEXT]], i32 [[HIGHER_BIT_AFTER_SIGNBIT]], i32 0
+; CHECK-NEXT:    call void @use32(i32 [[LOW_BITS_TO_SKIP]])
+; CHECK-NEXT:    call void @use32(i32 [[HIGH_BITS_EXTRACTED]])
+; CHECK-NEXT:    call void @use1(i1 [[SHOULD_SIGNEXT]])
+; CHECK-NEXT:    call void @use32(i32 [[HIGHER_BIT_AFTER_SIGNBIT]])
+; CHECK-NEXT:    call void @use32(i32 [[MAGIC]])
+; CHECK-NEXT:    [[SIGNEXTENDED:%.*]] = ashr i32 [[DATA]], [[LOW_BITS_TO_SKIP]]
+; CHECK-NEXT:    ret i32 [[SIGNEXTENDED]]
+;
+  %low_bits_to_skip = sub i32 32, %nbits
+  %high_bits_extracted = lshr i32 %data, %low_bits_to_skip
+  %should_signext = icmp slt i32 %data, 0
+  %higher_bit_after_signbit = shl i32 1, %nbits
+  %magic = select i1 %should_signext, i32 %higher_bit_after_signbit, i32 0
+
+  call void @use32(i32 %low_bits_to_skip)
+  call void @use32(i32 %high_bits_extracted)
+  call void @use1(i1 %should_signext)
+  call void @use32(i32 %higher_bit_after_signbit)
+  call void @use32(i32 %magic)
+
+  %signextended = sub i32 %high_bits_extracted, %magic
+  ret i32 %signextended
+}
+
+define i32 @t2_trunc_add(i64 %data, i32 %nbits) {
+; CHECK-LABEL: @t2_trunc_add(
+; CHECK-NEXT:    [[LOW_BITS_TO_SKIP:%.*]] = sub i32 64, [[NBITS:%.*]]
+; CHECK-NEXT:    [[LOW_BITS_TO_SKIP_WIDE:%.*]] = zext i32 [[LOW_BITS_TO_SKIP]] to i64
+; CHECK-NEXT:    [[HIGH_BITS_EXTRACTED_WIDE:%.*]] = lshr i64 [[DATA:%.*]], [[LOW_BITS_TO_SKIP_WIDE]]
+; CHECK-NEXT:    [[HIGH_BITS_EXTRACTED:%.*]] = trunc i64 [[HIGH_BITS_EXTRACTED_WIDE]] to i32
+; CHECK-NEXT:    [[SHOULD_SIGNEXT:%.*]] = icmp slt i64 [[DATA]], 0
+; CHECK-NEXT:    [[ALL_BITS_EXCEPT_LOW_NBITS:%.*]] = shl i32 -1, [[NBITS]]
+; CHECK-NEXT:    call void @use32(i32 [[LOW_BITS_TO_SKIP]])
+; CHECK-NEXT:    call void @use64(i64 [[LOW_BITS_TO_SKIP_WIDE]])
+; CHECK-NEXT:    call void @use64(i64 [[HIGH_BITS_EXTRACTED_WIDE]])
+; CHECK-NEXT:    call void @use32(i32 [[HIGH_BITS_EXTRACTED]])
+; CHECK-NEXT:    call void @use1(i1 [[SHOULD_SIGNEXT]])
+; CHECK-NEXT:    call void @use32(i32 [[ALL_BITS_EXCEPT_LOW_NBITS]])
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr i64 [[DATA]], [[LOW_BITS_TO_SKIP_WIDE]]
+; CHECK-NEXT:    [[SIGNEXTENDED:%.*]] = trunc i64 [[TMP1]] to i32
+; CHECK-NEXT:    ret i32 [[SIGNEXTENDED]]
+;
+  %low_bits_to_skip = sub i32 64, %nbits
+  %low_bits_to_skip_wide = zext i32 %low_bits_to_skip to i64
+  %high_bits_extracted_wide = lshr i64 %data, %low_bits_to_skip_wide
+  %high_bits_extracted = trunc i64 %high_bits_extracted_wide to i32
+  %should_signext = icmp slt i64 %data, 0
+  %all_bits_except_low_nbits = shl i32 -1, %nbits
+  %magic = select i1 %should_signext, i32 %all_bits_except_low_nbits, i32 0 ; one-use
+
+  call void @use32(i32 %low_bits_to_skip)
+  call void @use64(i64 %low_bits_to_skip_wide)
+  call void @use64(i64 %high_bits_extracted_wide)
+  call void @use32(i32 %high_bits_extracted)
+  call void @use1(i1 %should_signext)
+  call void @use32(i32 %all_bits_except_low_nbits)
+
+  %signextended = add i32 %magic, %high_bits_extracted
+  ret i32 %signextended
+}
+
+define i32 @t2_trunc_or(i64 %data, i32 %nbits) {
+; CHECK-LABEL: @t2_trunc_or(
+; CHECK-NEXT:    [[LOW_BITS_TO_SKIP:%.*]] = sub i32 64, [[NBITS:%.*]]
+; CHECK-NEXT:    [[LOW_BITS_TO_SKIP_WIDE:%.*]] = zext i32 [[LOW_BITS_TO_SKIP]] to i64
+; CHECK-NEXT:    [[HIGH_BITS_EXTRACTED_WIDE:%.*]] = lshr i64 [[DATA:%.*]], [[LOW_BITS_TO_SKIP_WIDE]]
+; CHECK-NEXT:    [[HIGH_BITS_EXTRACTED:%.*]] = trunc i64 [[HIGH_BITS_EXTRACTED_WIDE]] to i32
+; CHECK-NEXT:    [[SHOULD_SIGNEXT:%.*]] = icmp slt i64 [[DATA]], 0
+; CHECK-NEXT:    [[ALL_BITS_EXCEPT_LOW_NBITS:%.*]] = shl i32 -1, [[NBITS]]
+; CHECK-NEXT:    call void @use32(i32 [[LOW_BITS_TO_SKIP]])
+; CHECK-NEXT:    call void @use64(i64 [[LOW_BITS_TO_SKIP_WIDE]])
+; CHECK-NEXT:    call void @use64(i64 [[HIGH_BITS_EXTRACTED_WIDE]])
+; CHECK-NEXT:    call void @use32(i32 [[HIGH_BITS_EXTRACTED]])
+; CHECK-NEXT:    call void @use1(i1 [[SHOULD_SIGNEXT]])
+; CHECK-NEXT:    call void @use32(i32 [[ALL_BITS_EXCEPT_LOW_NBITS]])
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr i64 [[DATA]], [[LOW_BITS_TO_SKIP_WIDE]]
+; CHECK-NEXT:    [[SIGNEXTENDED:%.*]] = trunc i64 [[TMP1]] to i32
+; CHECK-NEXT:    ret i32 [[SIGNEXTENDED]]
+;
+  %low_bits_to_skip = sub i32 64, %nbits
+  %low_bits_to_skip_wide = zext i32 %low_bits_to_skip to i64
+  %high_bits_extracted_wide = lshr i64 %data, %low_bits_to_skip_wide
+  %high_bits_extracted = trunc i64 %high_bits_extracted_wide to i32
+  %should_signext = icmp slt i64 %data, 0
+  %all_bits_except_low_nbits = shl i32 -1, %nbits
+  %magic = select i1 %should_signext, i32 %all_bits_except_low_nbits, i32 0 ; one-use
+
+  call void @use32(i32 %low_bits_to_skip)
+  call void @use64(i64 %low_bits_to_skip_wide)
+  call void @use64(i64 %high_bits_extracted_wide)
+  call void @use32(i32 %high_bits_extracted)
+  call void @use1(i1 %should_signext)
+  call void @use32(i32 %all_bits_except_low_nbits)
+
+  %signextended = or i32 %magic, %high_bits_extracted
+  ret i32 %signextended
+}
+
+define i32 @t3_trunc_sub(i64 %data, i32 %nbits) {
+; CHECK-LABEL: @t3_trunc_sub(
+; CHECK-NEXT:    [[LOW_BITS_TO_SKIP:%.*]] = sub i32 64, [[NBITS:%.*]]
+; CHECK-NEXT:    [[LOW_BITS_TO_SKIP_WIDE:%.*]] = zext i32 [[LOW_BITS_TO_SKIP]] to i64
+; CHECK-NEXT:    [[HIGH_BITS_EXTRACTED_WIDE:%.*]] = lshr i64 [[DATA:%.*]], [[LOW_BITS_TO_SKIP_WIDE]]
+; CHECK-NEXT:    [[HIGH_BITS_EXTRACTED:%.*]] = trunc i64 [[HIGH_BITS_EXTRACTED_WIDE]] to i32
+; CHECK-NEXT:    [[SHOULD_SIGNEXT:%.*]] = icmp slt i64 [[DATA]], 0
+; CHECK-NEXT:    [[HIGHER_BIT_AFTER_SIGNBIT:%.*]] = shl i32 1, [[NBITS]]
+; CHECK-NEXT:    call void @use32(i32 [[LOW_BITS_TO_SKIP]])
+; CHECK-NEXT:    call void @use64(i64 [[LOW_BITS_TO_SKIP_WIDE]])
+; CHECK-NEXT:    call void @use64(i64 [[HIGH_BITS_EXTRACTED_WIDE]])
+; CHECK-NEXT:    call void @use32(i32 [[HIGH_BITS_EXTRACTED]])
+; CHECK-NEXT:    call void @use1(i1 [[SHOULD_SIGNEXT]])
+; CHECK-NEXT:    call void @use32(i32 [[HIGHER_BIT_AFTER_SIGNBIT]])
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr i64 [[DATA]], [[LOW_BITS_TO_SKIP_WIDE]]
+; CHECK-NEXT:    [[SIGNEXTENDED:%.*]] = trunc i64 [[TMP1]] to i32
+; CHECK-NEXT:    ret i32 [[SIGNEXTENDED]]
+;
+  %low_bits_to_skip = sub i32 64, %nbits
+  %low_bits_to_skip_wide = zext i32 %low_bits_to_skip to i64
+  %high_bits_extracted_wide = lshr i64 %data, %low_bits_to_skip_wide
+  %high_bits_extracted = trunc i64 %high_bits_extracted_wide to i32
+  %should_signext = icmp slt i64 %data, 0
+  %higher_bit_after_signbit = shl i32 1, %nbits
+  %magic = select i1 %should_signext, i32 %higher_bit_after_signbit, i32 0 ; one-use
+
+  call void @use32(i32 %low_bits_to_skip)
+  call void @use64(i64 %low_bits_to_skip_wide)
+  call void @use64(i64 %high_bits_extracted_wide)
+  call void @use32(i32 %high_bits_extracted)
+  call void @use1(i1 %should_signext)
+  call void @use32(i32 %higher_bit_after_signbit)
+
+  %signextended = sub i32 %high_bits_extracted, %magic
+  ret i32 %signextended
+}
+
+; Commutativity
+
+define i32 @t4_commutativity0(i32 %data, i32 %nbits) {
+; CHECK-LABEL: @t4_commutativity0(
+; CHECK-NEXT:    [[LOW_BITS_TO_SKIP:%.*]] = sub i32 32, [[NBITS:%.*]]
+; CHECK-NEXT:    [[HIGH_BITS_EXTRACTED:%.*]] = lshr i32 [[DATA:%.*]], [[LOW_BITS_TO_SKIP]]
+; CHECK-NEXT:    [[SHOULD_SIGNEXT:%.*]] = icmp slt i32 [[DATA]], 0
+; CHECK-NEXT:    [[ALL_BITS_EXCEPT_LOW_NBITS:%.*]] = shl i32 -1, [[NBITS]]
+; CHECK-NEXT:    [[MAGIC:%.*]] = select i1 [[SHOULD_SIGNEXT]], i32 [[ALL_BITS_EXCEPT_LOW_NBITS]], i32 0
+; CHECK-NEXT:    call void @use32(i32 [[LOW_BITS_TO_SKIP]])
+; CHECK-NEXT:    call void @use32(i32 [[HIGH_BITS_EXTRACTED]])
+; CHECK-NEXT:    call void @use1(i1 [[SHOULD_SIGNEXT]])
+; CHECK-NEXT:    call void @use32(i32 [[ALL_BITS_EXCEPT_LOW_NBITS]])
+; CHECK-NEXT:    call void @use32(i32 [[MAGIC]])
+; CHECK-NEXT:    [[SIGNEXTENDED:%.*]] = ashr i32 [[DATA]], [[LOW_BITS_TO_SKIP]]
+; CHECK-NEXT:    ret i32 [[SIGNEXTENDED]]
+;
+  %low_bits_to_skip = sub i32 32, %nbits
+  %high_bits_extracted = lshr i32 %data, %low_bits_to_skip
+  %should_signext = icmp slt i32 %data, 0
+  %all_bits_except_low_nbits = shl i32 -1, %nbits
+  %magic = select i1 %should_signext, i32 %all_bits_except_low_nbits, i32 0
+
+  call void @use32(i32 %low_bits_to_skip)
+  call void @use32(i32 %high_bits_extracted)
+  call void @use1(i1 %should_signext)
+  call void @use32(i32 %all_bits_except_low_nbits)
+  call void @use32(i32 %magic)
+
+  %signextended = add i32 %high_bits_extracted, %magic
+  ret i32 %signextended
+}
+define i32 @t5_commutativity1(i32 %data, i32 %nbits) {
+; CHECK-LABEL: @t5_commutativity1(
+; CHECK-NEXT:    [[LOW_BITS_TO_SKIP:%.*]] = sub i32 32, [[NBITS:%.*]]
+; CHECK-NEXT:    [[HIGH_BITS_EXTRACTED:%.*]] = lshr i32 [[DATA:%.*]], [[LOW_BITS_TO_SKIP]]
+; CHECK-NEXT:    [[SHOULD_SIGNEXT:%.*]] = icmp sgt i32 [[DATA]], -1
+; CHECK-NEXT:    [[ALL_BITS_EXCEPT_LOW_NBITS:%.*]] = shl i32 -1, [[NBITS]]
+; CHECK-NEXT:    [[MAGIC:%.*]] = select i1 [[SHOULD_SIGNEXT]], i32 0, i32 [[ALL_BITS_EXCEPT_LOW_NBITS]]
+; CHECK-NEXT:    call void @use32(i32 [[LOW_BITS_TO_SKIP]])
+; CHECK-NEXT:    call void @use32(i32 [[HIGH_BITS_EXTRACTED]])
+; CHECK-NEXT:    call void @use1(i1 [[SHOULD_SIGNEXT]])
+; CHECK-NEXT:    call void @use32(i32 [[ALL_BITS_EXCEPT_LOW_NBITS]])
+; CHECK-NEXT:    call void @use32(i32 [[MAGIC]])
+; CHECK-NEXT:    [[SIGNEXTENDED:%.*]] = ashr i32 [[DATA]], [[LOW_BITS_TO_SKIP]]
+; CHECK-NEXT:    ret i32 [[SIGNEXTENDED]]
+;
+  %low_bits_to_skip = sub i32 32, %nbits
+  %high_bits_extracted = lshr i32 %data, %low_bits_to_skip
+  %should_signext = icmp sgt i32 %data, -1 ; swapped
+  %all_bits_except_low_nbits = shl i32 -1, %nbits
+  %magic = select i1 %should_signext, i32 0, i32 %all_bits_except_low_nbits ; swapped
+
+  call void @use32(i32 %low_bits_to_skip)
+  call void @use32(i32 %high_bits_extracted)
+  call void @use1(i1 %should_signext)
+  call void @use32(i32 %all_bits_except_low_nbits)
+  call void @use32(i32 %magic)
+
+  %signextended = add i32 %high_bits_extracted, %magic
+  ret i32 %signextended
+}
+define i32 @t6_commutativity2(i32 %data, i32 %nbits) {
+; CHECK-LABEL: @t6_commutativity2(
+; CHECK-NEXT:    [[LOW_BITS_TO_SKIP:%.*]] = sub i32 32, [[NBITS:%.*]]
+; CHECK-NEXT:    [[HIGH_BITS_EXTRACTED:%.*]] = lshr i32 [[DATA:%.*]], [[LOW_BITS_TO_SKIP]]
+; CHECK-NEXT:    [[SHOULD_SIGNEXT:%.*]] = icmp slt i32 [[DATA]], 0
+; CHECK-NEXT:    [[ALL_BITS_EXCEPT_LOW_NBITS:%.*]] = shl i32 -1, [[NBITS]]
+; CHECK-NEXT:    [[MAGIC:%.*]] = select i1 [[SHOULD_SIGNEXT]], i32 [[ALL_BITS_EXCEPT_LOW_NBITS]], i32 0
+; CHECK-NEXT:    call void @use32(i32 [[LOW_BITS_TO_SKIP]])
+; CHECK-NEXT:    call void @use32(i32 [[HIGH_BITS_EXTRACTED]])
+; CHECK-NEXT:    call void @use1(i1 [[SHOULD_SIGNEXT]])
+; CHECK-NEXT:    call void @use32(i32 [[ALL_BITS_EXCEPT_LOW_NBITS]])
+; CHECK-NEXT:    call void @use32(i32 [[MAGIC]])
+; CHECK-NEXT:    [[SIGNEXTENDED:%.*]] = ashr i32 [[DATA]], [[LOW_BITS_TO_SKIP]]
+; CHECK-NEXT:    ret i32 [[SIGNEXTENDED]]
+;
+  %low_bits_to_skip = sub i32 32, %nbits
+  %high_bits_extracted = lshr i32 %data, %low_bits_to_skip
+  %should_signext = icmp slt i32 %data, 0
+  %all_bits_except_low_nbits = shl i32 -1, %nbits
+  %magic = select i1 %should_signext, i32 %all_bits_except_low_nbits, i32 0
+
+  call void @use32(i32 %low_bits_to_skip)
+  call void @use32(i32 %high_bits_extracted)
+  call void @use1(i1 %should_signext)
+  call void @use32(i32 %all_bits_except_low_nbits)
+  call void @use32(i32 %magic)
+
+  %signextended = add i32 %magic, %high_bits_extracted ; swapped
+  ret i32 %signextended
+}
+
+; Extra uses
+
+define i32 @t7_trunc_extrause0(i64 %data, i32 %nbits) {
+; CHECK-LABEL: @t7_trunc_extrause0(
+; CHECK-NEXT:    [[LOW_BITS_TO_SKIP:%.*]] = sub i32 64, [[NBITS:%.*]]
+; CHECK-NEXT:    [[LOW_BITS_TO_SKIP_WIDE:%.*]] = zext i32 [[LOW_BITS_TO_SKIP]] to i64
+; CHECK-NEXT:    [[HIGH_BITS_EXTRACTED_WIDE:%.*]] = lshr i64 [[DATA:%.*]], [[LOW_BITS_TO_SKIP_WIDE]]
+; CHECK-NEXT:    [[HIGH_BITS_EXTRACTED:%.*]] = trunc i64 [[HIGH_BITS_EXTRACTED_WIDE]] to i32
+; CHECK-NEXT:    [[SHOULD_SIGNEXT:%.*]] = icmp slt i64 [[DATA]], 0
+; CHECK-NEXT:    [[ALL_BITS_EXCEPT_LOW_NBITS:%.*]] = shl i32 -1, [[NBITS]]
+; CHECK-NEXT:    call void @use32(i32 [[LOW_BITS_TO_SKIP]])
+; CHECK-NEXT:    call void @use64(i64 [[LOW_BITS_TO_SKIP_WIDE]])
+; CHECK-NEXT:    call void @use64(i64 [[HIGH_BITS_EXTRACTED_WIDE]])
+; CHECK-NEXT:    call void @use32(i32 [[HIGH_BITS_EXTRACTED]])
+; CHECK-NEXT:    call void @use1(i1 [[SHOULD_SIGNEXT]])
+; CHECK-NEXT:    call void @use32(i32 [[ALL_BITS_EXCEPT_LOW_NBITS]])
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr i64 [[DATA]], [[LOW_BITS_TO_SKIP_WIDE]]
+; CHECK-NEXT:    [[SIGNEXTENDED:%.*]] = trunc i64 [[TMP1]] to i32
+; CHECK-NEXT:    ret i32 [[SIGNEXTENDED]]
+;
+  %low_bits_to_skip = sub i32 64, %nbits
+  %low_bits_to_skip_wide = zext i32 %low_bits_to_skip to i64
+  %high_bits_extracted_wide = lshr i64 %data, %low_bits_to_skip_wide
+  %high_bits_extracted = trunc i64 %high_bits_extracted_wide to i32 ; has extra use
+  %should_signext = icmp slt i64 %data, 0
+  %all_bits_except_low_nbits = shl i32 -1, %nbits
+  %magic = select i1 %should_signext, i32 %all_bits_except_low_nbits, i32 0 ; one-use
+
+  call void @use32(i32 %low_bits_to_skip)
+  call void @use64(i64 %low_bits_to_skip_wide)
+  call void @use64(i64 %high_bits_extracted_wide)
+  call void @use32(i32 %high_bits_extracted)
+  call void @use1(i1 %should_signext)
+  call void @use32(i32 %all_bits_except_low_nbits)
+
+  %signextended = add i32 %magic, %high_bits_extracted
+  ret i32 %signextended
+}
+define i32 @t8_trunc_extrause1(i64 %data, i32 %nbits) {
+; CHECK-LABEL: @t8_trunc_extrause1(
+; CHECK-NEXT:    [[LOW_BITS_TO_SKIP:%.*]] = sub i32 64, [[NBITS:%.*]]
+; CHECK-NEXT:    [[LOW_BITS_TO_SKIP_WIDE:%.*]] = zext i32 [[LOW_BITS_TO_SKIP]] to i64
+; CHECK-NEXT:    [[HIGH_BITS_EXTRACTED_WIDE:%.*]] = lshr i64 [[DATA:%.*]], [[LOW_BITS_TO_SKIP_WIDE]]
+; CHECK-NEXT:    [[SHOULD_SIGNEXT:%.*]] = icmp slt i64 [[DATA]], 0
+; CHECK-NEXT:    [[ALL_BITS_EXCEPT_LOW_NBITS:%.*]] = shl i32 -1, [[NBITS]]
+; CHECK-NEXT:    [[MAGIC:%.*]] = select i1 [[SHOULD_SIGNEXT]], i32 [[ALL_BITS_EXCEPT_LOW_NBITS]], i32 0
+; CHECK-NEXT:    call void @use32(i32 [[LOW_BITS_TO_SKIP]])
+; CHECK-NEXT:    call void @use64(i64 [[LOW_BITS_TO_SKIP_WIDE]])
+; CHECK-NEXT:    call void @use64(i64 [[HIGH_BITS_EXTRACTED_WIDE]])
+; CHECK-NEXT:    call void @use1(i1 [[SHOULD_SIGNEXT]])
+; CHECK-NEXT:    call void @use32(i32 [[ALL_BITS_EXCEPT_LOW_NBITS]])
+; CHECK-NEXT:    call void @use32(i32 [[MAGIC]])
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr i64 [[DATA]], [[LOW_BITS_TO_SKIP_WIDE]]
+; CHECK-NEXT:    [[SIGNEXTENDED:%.*]] = trunc i64 [[TMP1]] to i32
+; CHECK-NEXT:    ret i32 [[SIGNEXTENDED]]
+;
+  %low_bits_to_skip = sub i32 64, %nbits
+  %low_bits_to_skip_wide = zext i32 %low_bits_to_skip to i64
+  %high_bits_extracted_wide = lshr i64 %data, %low_bits_to_skip_wide
+  %high_bits_extracted = trunc i64 %high_bits_extracted_wide to i32 ; one-use
+  %should_signext = icmp slt i64 %data, 0
+  %all_bits_except_low_nbits = shl i32 -1, %nbits
+  %magic = select i1 %should_signext, i32 %all_bits_except_low_nbits, i32 0 ; has extra use
+
+  call void @use32(i32 %low_bits_to_skip)
+  call void @use64(i64 %low_bits_to_skip_wide)
+  call void @use64(i64 %high_bits_extracted_wide)
+  call void @use1(i1 %should_signext)
+  call void @use32(i32 %all_bits_except_low_nbits)
+  call void @use32(i32 %magic)
+
+  %signextended = add i32 %magic, %high_bits_extracted
+  ret i32 %signextended
+}
+define i32 @n9_trunc_extrause2(i64 %data, i32 %nbits) {
+; CHECK-LABEL: @n9_trunc_extrause2(
+; CHECK-NEXT:    [[LOW_BITS_TO_SKIP:%.*]] = sub i32 64, [[NBITS:%.*]]
+; CHECK-NEXT:    [[LOW_BITS_TO_SKIP_WIDE:%.*]] = zext i32 [[LOW_BITS_TO_SKIP]] to i64
+; CHECK-NEXT:    [[HIGH_BITS_EXTRACTED_WIDE:%.*]] = lshr i64 [[DATA:%.*]], [[LOW_BITS_TO_SKIP_WIDE]]
+; CHECK-NEXT:    [[HIGH_BITS_EXTRACTED:%.*]] = trunc i64 [[HIGH_BITS_EXTRACTED_WIDE]] to i32
+; CHECK-NEXT:    [[SHOULD_SIGNEXT:%.*]] = icmp slt i64 [[DATA]], 0
+; CHECK-NEXT:    [[ALL_BITS_EXCEPT_LOW_NBITS:%.*]] = shl i32 -1, [[NBITS]]
+; CHECK-NEXT:    [[MAGIC:%.*]] = select i1 [[SHOULD_SIGNEXT]], i32 [[ALL_BITS_EXCEPT_LOW_NBITS]], i32 0
+; CHECK-NEXT:    call void @use32(i32 [[LOW_BITS_TO_SKIP]])
+; CHECK-NEXT:    call void @use64(i64 [[LOW_BITS_TO_SKIP_WIDE]])
+; CHECK-NEXT:    call void @use64(i64 [[HIGH_BITS_EXTRACTED_WIDE]])
+; CHECK-NEXT:    call void @use32(i32 [[HIGH_BITS_EXTRACTED]])
+; CHECK-NEXT:    call void @use1(i1 [[SHOULD_SIGNEXT]])
+; CHECK-NEXT:    call void @use32(i32 [[ALL_BITS_EXCEPT_LOW_NBITS]])
+; CHECK-NEXT:    call void @use32(i32 [[MAGIC]])
+; CHECK-NEXT:    [[SIGNEXTENDED:%.*]] = add i32 [[MAGIC]], [[HIGH_BITS_EXTRACTED]]
+; CHECK-NEXT:    ret i32 [[SIGNEXTENDED]]
+;
+  %low_bits_to_skip = sub i32 64, %nbits
+  %low_bits_to_skip_wide = zext i32 %low_bits_to_skip to i64
+  %high_bits_extracted_wide = lshr i64 %data, %low_bits_to_skip_wide
+  %high_bits_extracted = trunc i64 %high_bits_extracted_wide to i32 ; has extra use
+  %should_signext = icmp slt i64 %data, 0
+  %all_bits_except_low_nbits = shl i32 -1, %nbits
+  %magic = select i1 %should_signext, i32 %all_bits_except_low_nbits, i32 0 ; has extra use
+
+  call void @use32(i32 %low_bits_to_skip)
+  call void @use64(i64 %low_bits_to_skip_wide)
+  call void @use64(i64 %high_bits_extracted_wide)
+  call void @use32(i32 %high_bits_extracted)
+  call void @use1(i1 %should_signext)
+  call void @use32(i32 %all_bits_except_low_nbits)
+  call void @use32(i32 %magic)
+
+  %signextended = add i32 %magic, %high_bits_extracted
+  ret i32 %signextended
+}
+
+define i32 @t10_preserve_exact(i32 %data, i32 %nbits) {
+; CHECK-LABEL: @t10_preserve_exact(
+; CHECK-NEXT:    [[LOW_BITS_TO_SKIP:%.*]] = sub i32 32, [[NBITS:%.*]]
+; CHECK-NEXT:    [[HIGH_BITS_EXTRACTED:%.*]] = lshr exact i32 [[DATA:%.*]], [[LOW_BITS_TO_SKIP]]
+; CHECK-NEXT:    [[SHOULD_SIGNEXT:%.*]] = icmp slt i32 [[DATA]], 0
+; CHECK-NEXT:    [[ALL_BITS_EXCEPT_LOW_NBITS:%.*]] = shl i32 -1, [[NBITS]]
+; CHECK-NEXT:    [[MAGIC:%.*]] = select i1 [[SHOULD_SIGNEXT]], i32 [[ALL_BITS_EXCEPT_LOW_NBITS]], i32 0
+; CHECK-NEXT:    call void @use32(i32 [[LOW_BITS_TO_SKIP]])
+; CHECK-NEXT:    call void @use32(i32 [[HIGH_BITS_EXTRACTED]])
+; CHECK-NEXT:    call void @use1(i1 [[SHOULD_SIGNEXT]])
+; CHECK-NEXT:    call void @use32(i32 [[ALL_BITS_EXCEPT_LOW_NBITS]])
+; CHECK-NEXT:    call void @use32(i32 [[MAGIC]])
+; CHECK-NEXT:    [[SIGNEXTENDED:%.*]] = ashr exact i32 [[DATA]], [[LOW_BITS_TO_SKIP]]
+; CHECK-NEXT:    ret i32 [[SIGNEXTENDED]]
+;
+  %low_bits_to_skip = sub i32 32, %nbits
+  %high_bits_extracted = lshr exact i32 %data, %low_bits_to_skip
+  %should_signext = icmp slt i32 %data, 0
+  %all_bits_except_low_nbits = shl i32 -1, %nbits
+  %magic = select i1 %should_signext, i32 %all_bits_except_low_nbits, i32 0
+
+  call void @use32(i32 %low_bits_to_skip)
+  call void @use32(i32 %high_bits_extracted)
+  call void @use1(i1 %should_signext)
+  call void @use32(i32 %all_bits_except_low_nbits)
+  call void @use32(i32 %magic)
+
+  %signextended = add i32 %high_bits_extracted, %magic
+  ret i32 %signextended
+}
+
+define i32 @t11_different_zext_of_shamt(i32 %data, i8 %nbits) {
+; CHECK-LABEL: @t11_different_zext_of_shamt(
+; CHECK-NEXT:    [[NBITS_16BIT:%.*]] = zext i8 [[NBITS:%.*]] to i16
+; CHECK-NEXT:    [[LOW_BITS_TO_SKIP:%.*]] = sub nsw i16 32, [[NBITS_16BIT]]
+; CHECK-NEXT:    [[LOW_BITS_TO_SKIP_32:%.*]] = zext i16 [[LOW_BITS_TO_SKIP]] to i32
+; CHECK-NEXT:    [[HIGH_BITS_EXTRACTED:%.*]] = lshr i32 [[DATA:%.*]], [[LOW_BITS_TO_SKIP_32]]
+; CHECK-NEXT:    [[SHOULD_SIGNEXT:%.*]] = icmp slt i32 [[DATA]], 0
+; CHECK-NEXT:    [[NBITS_32BIT:%.*]] = zext i8 [[NBITS]] to i32
+; CHECK-NEXT:    [[ALL_BITS_EXCEPT_LOW_NBITS:%.*]] = shl i32 -1, [[NBITS_32BIT]]
+; CHECK-NEXT:    [[MAGIC:%.*]] = select i1 [[SHOULD_SIGNEXT]], i32 [[ALL_BITS_EXCEPT_LOW_NBITS]], i32 0
+; CHECK-NEXT:    call void @use16(i16 [[NBITS_16BIT]])
+; CHECK-NEXT:    call void @use16(i16 [[LOW_BITS_TO_SKIP]])
+; CHECK-NEXT:    call void @use32(i32 [[LOW_BITS_TO_SKIP_32]])
+; CHECK-NEXT:    call void @use32(i32 [[HIGH_BITS_EXTRACTED]])
+; CHECK-NEXT:    call void @use1(i1 [[SHOULD_SIGNEXT]])
+; CHECK-NEXT:    call void @use32(i32 [[NBITS_32BIT]])
+; CHECK-NEXT:    call void @use32(i32 [[ALL_BITS_EXCEPT_LOW_NBITS]])
+; CHECK-NEXT:    call void @use32(i32 [[MAGIC]])
+; CHECK-NEXT:    [[SIGNEXTENDED:%.*]] = ashr i32 [[DATA]], [[LOW_BITS_TO_SKIP_32]]
+; CHECK-NEXT:    ret i32 [[SIGNEXTENDED]]
+;
+  %nbits_16bit = zext i8 %nbits to i16
+  %low_bits_to_skip = sub i16 32, %nbits_16bit
+  %low_bits_to_skip_32 = zext i16 %low_bits_to_skip to i32
+  %high_bits_extracted = lshr i32 %data, %low_bits_to_skip_32
+  %should_signext = icmp slt i32 %data, 0
+  %nbits_32bit = zext i8 %nbits to i32
+  %all_bits_except_low_nbits = shl i32 -1, %nbits_32bit
+  %magic = select i1 %should_signext, i32 %all_bits_except_low_nbits, i32 0
+
+  call void @use16(i16 %nbits_16bit)
+  call void @use16(i16 %low_bits_to_skip)
+  call void @use32(i32 %low_bits_to_skip_32)
+  call void @use32(i32 %high_bits_extracted)
+  call void @use1(i1 %should_signext)
+  call void @use32(i32 %nbits_32bit)
+  call void @use32(i32 %all_bits_except_low_nbits)
+  call void @use32(i32 %magic)
+
+  %signextended = add i32 %high_bits_extracted, %magic
+  ret i32 %signextended
+}
+
+define i32 @t12_add_sext_of_magic(i32 %data, i8 %nbits) {
+; CHECK-LABEL: @t12_add_sext_of_magic(
+; CHECK-NEXT:    [[NBITS_32BIT:%.*]] = zext i8 [[NBITS:%.*]] to i32
+; CHECK-NEXT:    [[LOW_BITS_TO_SKIP:%.*]] = sub nsw i32 32, [[NBITS_32BIT]]
+; CHECK-NEXT:    [[HIGH_BITS_EXTRACTED:%.*]] = lshr i32 [[DATA:%.*]], [[LOW_BITS_TO_SKIP]]
+; CHECK-NEXT:    [[SHOULD_SIGNEXT:%.*]] = icmp slt i32 [[DATA]], 0
+; CHECK-NEXT:    [[NBITS_16BIT:%.*]] = zext i8 [[NBITS]] to i16
+; CHECK-NEXT:    [[ALL_BITS_EXCEPT_LOW_NBITS:%.*]] = shl i16 -1, [[NBITS_16BIT]]
+; CHECK-NEXT:    [[MAGIC:%.*]] = select i1 [[SHOULD_SIGNEXT]], i16 [[ALL_BITS_EXCEPT_LOW_NBITS]], i16 0
+; CHECK-NEXT:    [[MAGIC_WIDE:%.*]] = sext i16 [[MAGIC]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[NBITS_32BIT]])
+; CHECK-NEXT:    call void @use32(i32 [[LOW_BITS_TO_SKIP]])
+; CHECK-NEXT:    call void @use32(i32 [[HIGH_BITS_EXTRACTED]])
+; CHECK-NEXT:    call void @use1(i1 [[SHOULD_SIGNEXT]])
+; CHECK-NEXT:    call void @use16(i16 [[NBITS_16BIT]])
+; CHECK-NEXT:    call void @use16(i16 [[ALL_BITS_EXCEPT_LOW_NBITS]])
+; CHECK-NEXT:    call void @use16(i16 [[MAGIC]])
+; CHECK-NEXT:    call void @use32(i32 [[MAGIC_WIDE]])
+; CHECK-NEXT:    [[SIGNEXTENDED:%.*]] = ashr i32 [[DATA]], [[LOW_BITS_TO_SKIP]]
+; CHECK-NEXT:    ret i32 [[SIGNEXTENDED]]
+;
+  %nbits_32bit = zext i8 %nbits to i32
+  %low_bits_to_skip = sub i32 32, %nbits_32bit
+  %high_bits_extracted = lshr i32 %data, %low_bits_to_skip
+  %should_signext = icmp slt i32 %data, 0
+  %nbits_16bit = zext i8 %nbits to i16
+  %all_bits_except_low_nbits = shl i16 -1, %nbits_16bit
+  %magic = select i1 %should_signext, i16 %all_bits_except_low_nbits, i16 0
+  %magic_wide = sext i16 %magic to i32
+
+  call void @use32(i32 %nbits_32bit)
+  call void @use32(i32 %low_bits_to_skip)
+  call void @use32(i32 %high_bits_extracted)
+  call void @use1(i1 %should_signext)
+  call void @use16(i16 %nbits_16bit)
+  call void @use16(i16 %all_bits_except_low_nbits)
+  call void @use16(i16 %magic)
+  call void @use32(i32 %magic_wide)
+
+  %signextended = add i32 %high_bits_extracted, %magic_wide
+  ret i32 %signextended
+}
+
+define i32 @t13_sub_zext_of_magic(i32 %data, i8 %nbits) {
+; CHECK-LABEL: @t13_sub_zext_of_magic(
+; CHECK-NEXT:    [[NBITS_32BIT:%.*]] = zext i8 [[NBITS:%.*]] to i32
+; CHECK-NEXT:    [[LOW_BITS_TO_SKIP:%.*]] = sub nsw i32 32, [[NBITS_32BIT]]
+; CHECK-NEXT:    [[HIGH_BITS_EXTRACTED:%.*]] = lshr i32 [[DATA:%.*]], [[LOW_BITS_TO_SKIP]]
+; CHECK-NEXT:    [[SHOULD_SIGNEXT:%.*]] = icmp slt i32 [[DATA]], 0
+; CHECK-NEXT:    [[NBITS_16BIT:%.*]] = zext i8 [[NBITS]] to i16
+; CHECK-NEXT:    [[ALL_BITS_EXCEPT_LOW_NBITS:%.*]] = shl i16 1, [[NBITS_16BIT]]
+; CHECK-NEXT:    [[MAGIC:%.*]] = select i1 [[SHOULD_SIGNEXT]], i16 [[ALL_BITS_EXCEPT_LOW_NBITS]], i16 0
+; CHECK-NEXT:    [[MAGIC_WIDE:%.*]] = zext i16 [[MAGIC]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[NBITS_32BIT]])
+; CHECK-NEXT:    call void @use32(i32 [[LOW_BITS_TO_SKIP]])
+; CHECK-NEXT:    call void @use32(i32 [[HIGH_BITS_EXTRACTED]])
+; CHECK-NEXT:    call void @use1(i1 [[SHOULD_SIGNEXT]])
+; CHECK-NEXT:    call void @use16(i16 [[NBITS_16BIT]])
+; CHECK-NEXT:    call void @use16(i16 [[ALL_BITS_EXCEPT_LOW_NBITS]])
+; CHECK-NEXT:    call void @use16(i16 [[MAGIC]])
+; CHECK-NEXT:    call void @use32(i32 [[MAGIC_WIDE]])
+; CHECK-NEXT:    [[SIGNEXTENDED:%.*]] = ashr i32 [[DATA]], [[LOW_BITS_TO_SKIP]]
+; CHECK-NEXT:    ret i32 [[SIGNEXTENDED]]
+;
+  %nbits_32bit = zext i8 %nbits to i32
+  %low_bits_to_skip = sub i32 32, %nbits_32bit
+  %high_bits_extracted = lshr i32 %data, %low_bits_to_skip
+  %should_signext = icmp slt i32 %data, 0
+  %nbits_16bit = zext i8 %nbits to i16
+  %all_bits_except_low_nbits = shl i16 1, %nbits_16bit
+  %magic = select i1 %should_signext, i16 %all_bits_except_low_nbits, i16 0
+  %magic_wide = zext i16 %magic to i32
+
+  call void @use32(i32 %nbits_32bit)
+  call void @use32(i32 %low_bits_to_skip)
+  call void @use32(i32 %high_bits_extracted)
+  call void @use1(i1 %should_signext)
+  call void @use16(i16 %nbits_16bit)
+  call void @use16(i16 %all_bits_except_low_nbits)
+  call void @use16(i16 %magic)
+  call void @use32(i32 %magic_wide)
+
+  %signextended = sub i32 %high_bits_extracted, %magic_wide
+  ret i32 %signextended
+}
+
+define i32 @t14_add_sext_of_shl(i32 %data, i8 %nbits) {
+; CHECK-LABEL: @t14_add_sext_of_shl(
+; CHECK-NEXT:    [[NBITS_32BIT:%.*]] = zext i8 [[NBITS:%.*]] to i32
+; CHECK-NEXT:    [[LOW_BITS_TO_SKIP:%.*]] = sub nsw i32 32, [[NBITS_32BIT]]
+; CHECK-NEXT:    [[HIGH_BITS_EXTRACTED:%.*]] = lshr i32 [[DATA:%.*]], [[LOW_BITS_TO_SKIP]]
+; CHECK-NEXT:    [[SHOULD_SIGNEXT:%.*]] = icmp slt i32 [[DATA]], 0
+; CHECK-NEXT:    [[NBITS_16BIT:%.*]] = zext i8 [[NBITS]] to i16
+; CHECK-NEXT:    [[ALL_BITS_EXCEPT_LOW_NBITS:%.*]] = shl i16 -1, [[NBITS_16BIT]]
+; CHECK-NEXT:    [[ALL_BITS_EXCEPT_LOW_NBITS_WIDE:%.*]] = sext i16 [[ALL_BITS_EXCEPT_LOW_NBITS]] to i32
+; CHECK-NEXT:    [[MAGIC:%.*]] = select i1 [[SHOULD_SIGNEXT]], i32 [[ALL_BITS_EXCEPT_LOW_NBITS_WIDE]], i32 0
+; CHECK-NEXT:    call void @use32(i32 [[NBITS_32BIT]])
+; CHECK-NEXT:    call void @use32(i32 [[LOW_BITS_TO_SKIP]])
+; CHECK-NEXT:    call void @use32(i32 [[HIGH_BITS_EXTRACTED]])
+; CHECK-NEXT:    call void @use1(i1 [[SHOULD_SIGNEXT]])
+; CHECK-NEXT:    call void @use16(i16 [[NBITS_16BIT]])
+; CHECK-NEXT:    call void @use16(i16 [[ALL_BITS_EXCEPT_LOW_NBITS]])
+; CHECK-NEXT:    call void @use32(i32 [[ALL_BITS_EXCEPT_LOW_NBITS_WIDE]])
+; CHECK-NEXT:    call void @use32(i32 [[MAGIC]])
+; CHECK-NEXT:    [[SIGNEXTENDED:%.*]] = ashr i32 [[DATA]], [[LOW_BITS_TO_SKIP]]
+; CHECK-NEXT:    ret i32 [[SIGNEXTENDED]]
+;
+  %nbits_32bit = zext i8 %nbits to i32
+  %low_bits_to_skip = sub i32 32, %nbits_32bit
+  %high_bits_extracted = lshr i32 %data, %low_bits_to_skip
+  %should_signext = icmp slt i32 %data, 0
+  %nbits_16bit = zext i8 %nbits to i16
+  %all_bits_except_low_nbits = shl i16 -1, %nbits_16bit
+  %all_bits_except_low_nbits_wide = sext i16 %all_bits_except_low_nbits to i32
+  %magic = select i1 %should_signext, i32 %all_bits_except_low_nbits_wide, i32 0
+
+  call void @use32(i32 %nbits_32bit)
+  call void @use32(i32 %low_bits_to_skip)
+  call void @use32(i32 %high_bits_extracted)
+  call void @use1(i1 %should_signext)
+  call void @use16(i16 %nbits_16bit)
+  call void @use16(i16 %all_bits_except_low_nbits)
+  call void @use32(i32 %all_bits_except_low_nbits_wide)
+  call void @use32(i32 %magic)
+
+  %signextended = add i32 %high_bits_extracted, %magic
+  ret i32 %signextended
+}
+
+define i32 @t15_sub_zext_of_shl(i32 %data, i8 %nbits) {
+; CHECK-LABEL: @t15_sub_zext_of_shl(
+; CHECK-NEXT:    [[NBITS_32BIT:%.*]] = zext i8 [[NBITS:%.*]] to i32
+; CHECK-NEXT:    [[LOW_BITS_TO_SKIP:%.*]] = sub nsw i32 32, [[NBITS_32BIT]]
+; CHECK-NEXT:    [[HIGH_BITS_EXTRACTED:%.*]] = lshr i32 [[DATA:%.*]], [[LOW_BITS_TO_SKIP]]
+; CHECK-NEXT:    [[SHOULD_SIGNEXT:%.*]] = icmp slt i32 [[DATA]], 0
+; CHECK-NEXT:    [[NBITS_16BIT:%.*]] = zext i8 [[NBITS]] to i16
+; CHECK-NEXT:    [[ALL_BITS_EXCEPT_LOW_NBITS:%.*]] = shl i16 1, [[NBITS_16BIT]]
+; CHECK-NEXT:    [[ALL_BITS_EXCEPT_LOW_NBITS_WIDE:%.*]] = zext i16 [[ALL_BITS_EXCEPT_LOW_NBITS]] to i32
+; CHECK-NEXT:    [[MAGIC:%.*]] = select i1 [[SHOULD_SIGNEXT]], i32 [[ALL_BITS_EXCEPT_LOW_NBITS_WIDE]], i32 0
+; CHECK-NEXT:    call void @use32(i32 [[NBITS_32BIT]])
+; CHECK-NEXT:    call void @use32(i32 [[LOW_BITS_TO_SKIP]])
+; CHECK-NEXT:    call void @use32(i32 [[HIGH_BITS_EXTRACTED]])
+; CHECK-NEXT:    call void @use1(i1 [[SHOULD_SIGNEXT]])
+; CHECK-NEXT:    call void @use16(i16 [[NBITS_16BIT]])
+; CHECK-NEXT:    call void @use16(i16 [[ALL_BITS_EXCEPT_LOW_NBITS]])
+; CHECK-NEXT:    call void @use32(i32 [[ALL_BITS_EXCEPT_LOW_NBITS_WIDE]])
+; CHECK-NEXT:    call void @use32(i32 [[MAGIC]])
+; CHECK-NEXT:    [[SIGNEXTENDED:%.*]] = ashr i32 [[DATA]], [[LOW_BITS_TO_SKIP]]
+; CHECK-NEXT:    ret i32 [[SIGNEXTENDED]]
+;
+  %nbits_32bit = zext i8 %nbits to i32
+  %low_bits_to_skip = sub i32 32, %nbits_32bit
+  %high_bits_extracted = lshr i32 %data, %low_bits_to_skip
+  %should_signext = icmp slt i32 %data, 0
+  %nbits_16bit = zext i8 %nbits to i16
+  %all_bits_except_low_nbits = shl i16 1, %nbits_16bit
+  %all_bits_except_low_nbits_wide = zext i16 %all_bits_except_low_nbits to i32
+  %magic = select i1 %should_signext, i32 %all_bits_except_low_nbits_wide, i32 0
+
+  call void @use32(i32 %nbits_32bit)
+  call void @use32(i32 %low_bits_to_skip)
+  call void @use32(i32 %high_bits_extracted)
+  call void @use1(i1 %should_signext)
+  call void @use16(i16 %nbits_16bit)
+  call void @use16(i16 %all_bits_except_low_nbits)
+  call void @use32(i32 %all_bits_except_low_nbits_wide)
+  call void @use32(i32 %magic)
+
+  %signextended = sub i32 %high_bits_extracted, %magic
+  ret i32 %signextended
+}
+
+; Negative tests.
+
+define i32 @n16(i32 %data, i32 %nbits) {
+; CHECK-LABEL: @n16(
+; CHECK-NEXT:    [[LOW_BITS_TO_SKIP:%.*]] = sub i32 31, [[NBITS:%.*]]
+; CHECK-NEXT:    [[HIGH_BITS_EXTRACTED:%.*]] = lshr i32 [[DATA:%.*]], [[LOW_BITS_TO_SKIP]]
+; CHECK-NEXT:    [[SHOULD_SIGNEXT:%.*]] = icmp slt i32 [[DATA]], 0
+; CHECK-NEXT:    [[ALL_BITS_EXCEPT_LOW_NBITS:%.*]] = shl i32 -1, [[NBITS]]
+; CHECK-NEXT:    [[MAGIC:%.*]] = select i1 [[SHOULD_SIGNEXT]], i32 [[ALL_BITS_EXCEPT_LOW_NBITS]], i32 0
+; CHECK-NEXT:    call void @use32(i32 [[LOW_BITS_TO_SKIP]])
+; CHECK-NEXT:    call void @use32(i32 [[HIGH_BITS_EXTRACTED]])
+; CHECK-NEXT:    call void @use1(i1 [[SHOULD_SIGNEXT]])
+; CHECK-NEXT:    call void @use32(i32 [[ALL_BITS_EXCEPT_LOW_NBITS]])
+; CHECK-NEXT:    call void @use32(i32 [[MAGIC]])
+; CHECK-NEXT:    [[SIGNEXTENDED:%.*]] = add i32 [[HIGH_BITS_EXTRACTED]], [[MAGIC]]
+; CHECK-NEXT:    ret i32 [[SIGNEXTENDED]]
+;
+  %low_bits_to_skip = sub i32 31, %nbits ; not 32
+  %high_bits_extracted = lshr i32 %data, %low_bits_to_skip
+  %should_signext = icmp slt i32 %data, 0
+  %all_bits_except_low_nbits = shl i32 -1, %nbits
+  %magic = select i1 %should_signext, i32 %all_bits_except_low_nbits, i32 0
+
+  call void @use32(i32 %low_bits_to_skip)
+  call void @use32(i32 %high_bits_extracted)
+  call void @use1(i1 %should_signext)
+  call void @use32(i32 %all_bits_except_low_nbits)
+  call void @use32(i32 %magic)
+
+  %signextended = add i32 %high_bits_extracted, %magic
+  ret i32 %signextended
+}
+
+define i32 @n17_add(i32 %data, i32 %nbits) {
+; CHECK-LABEL: @n17_add(
+; CHECK-NEXT:    [[LOW_BITS_TO_SKIP:%.*]] = sub i32 32, [[NBITS:%.*]]
+; CHECK-NEXT:    [[HIGH_BITS_EXTRACTED:%.*]] = lshr i32 [[DATA:%.*]], [[LOW_BITS_TO_SKIP]]
+; CHECK-NEXT:    [[SHOULD_SIGNEXT:%.*]] = icmp slt i32 [[DATA]], 0
+; CHECK-NEXT:    [[ALL_BITS_EXCEPT_LOW_NBITS:%.*]] = shl i32 1, [[NBITS]]
+; CHECK-NEXT:    [[MAGIC:%.*]] = select i1 [[SHOULD_SIGNEXT]], i32 [[ALL_BITS_EXCEPT_LOW_NBITS]], i32 0
+; CHECK-NEXT:    call void @use32(i32 [[LOW_BITS_TO_SKIP]])
+; CHECK-NEXT:    call void @use32(i32 [[HIGH_BITS_EXTRACTED]])
+; CHECK-NEXT:    call void @use1(i1 [[SHOULD_SIGNEXT]])
+; CHECK-NEXT:    call void @use32(i32 [[ALL_BITS_EXCEPT_LOW_NBITS]])
+; CHECK-NEXT:    call void @use32(i32 [[MAGIC]])
+; CHECK-NEXT:    [[SIGNEXTENDED:%.*]] = add i32 [[HIGH_BITS_EXTRACTED]], [[MAGIC]]
+; CHECK-NEXT:    ret i32 [[SIGNEXTENDED]]
+;
+  %low_bits_to_skip = sub i32 32, %nbits
+  %high_bits_extracted = lshr i32 %data, %low_bits_to_skip
+  %should_signext = icmp slt i32 %data, 0
+  %all_bits_except_low_nbits = shl i32 1, %nbits ; not -1
+  %magic = select i1 %should_signext, i32 %all_bits_except_low_nbits, i32 0
+
+  call void @use32(i32 %low_bits_to_skip)
+  call void @use32(i32 %high_bits_extracted)
+  call void @use1(i1 %should_signext)
+  call void @use32(i32 %all_bits_except_low_nbits)
+  call void @use32(i32 %magic)
+
+  %signextended = add i32 %high_bits_extracted, %magic
+  ret i32 %signextended
+}
+
+define i32 @n18(i32 %data, i32 %nbits) {
+; CHECK-LABEL: @n18(
+; CHECK-NEXT:    [[LOW_BITS_TO_SKIP:%.*]] = sub i32 32, [[NBITS:%.*]]
+; CHECK-NEXT:    [[HIGH_BITS_EXTRACTED:%.*]] = lshr i32 [[DATA:%.*]], [[LOW_BITS_TO_SKIP]]
+; CHECK-NEXT:    [[SHOULD_SIGNEXT:%.*]] = icmp slt i32 [[DATA]], 0
+; CHECK-NEXT:    [[ALL_BITS_EXCEPT_LOW_NBITS:%.*]] = shl i32 -1, [[NBITS]]
+; CHECK-NEXT:    [[MAGIC:%.*]] = select i1 [[SHOULD_SIGNEXT]], i32 0, i32 [[ALL_BITS_EXCEPT_LOW_NBITS]]
+; CHECK-NEXT:    call void @use32(i32 [[LOW_BITS_TO_SKIP]])
+; CHECK-NEXT:    call void @use32(i32 [[HIGH_BITS_EXTRACTED]])
+; CHECK-NEXT:    call void @use1(i1 [[SHOULD_SIGNEXT]])
+; CHECK-NEXT:    call void @use32(i32 [[ALL_BITS_EXCEPT_LOW_NBITS]])
+; CHECK-NEXT:    call void @use32(i32 [[MAGIC]])
+; CHECK-NEXT:    [[SIGNEXTENDED:%.*]] = add i32 [[HIGH_BITS_EXTRACTED]], [[MAGIC]]
+; CHECK-NEXT:    ret i32 [[SIGNEXTENDED]]
+;
+  %low_bits_to_skip = sub i32 32, %nbits
+  %high_bits_extracted = lshr i32 %data, %low_bits_to_skip
+  %should_signext = icmp slt i32 %data, 0
+  %all_bits_except_low_nbits = shl i32 -1, %nbits
+  %magic = select i1 %should_signext, i32 0, i32 %all_bits_except_low_nbits ; wrong order
+
+  call void @use32(i32 %low_bits_to_skip)
+  call void @use32(i32 %high_bits_extracted)
+  call void @use1(i1 %should_signext)
+  call void @use32(i32 %all_bits_except_low_nbits)
+  call void @use32(i32 %magic)
+
+  %signextended = add i32 %high_bits_extracted, %magic
+  ret i32 %signextended
+}
+
+define i32 @n19(i32 %data1, i32 %data2, i32 %nbits) {
+; CHECK-LABEL: @n19(
+; CHECK-NEXT:    [[LOW_BITS_TO_SKIP:%.*]] = sub i32 32, [[NBITS:%.*]]
+; CHECK-NEXT:    [[HIGH_BITS_EXTRACTED:%.*]] = lshr i32 [[DATA1:%.*]], [[LOW_BITS_TO_SKIP]]
+; CHECK-NEXT:    [[SHOULD_SIGNEXT:%.*]] = icmp slt i32 [[DATA2:%.*]], 0
+; CHECK-NEXT:    [[ALL_BITS_EXCEPT_LOW_NBITS:%.*]] = shl i32 -1, [[NBITS]]
+; CHECK-NEXT:    [[MAGIC:%.*]] = select i1 [[SHOULD_SIGNEXT]], i32 [[ALL_BITS_EXCEPT_LOW_NBITS]], i32 0
+; CHECK-NEXT:    call void @use32(i32 [[LOW_BITS_TO_SKIP]])
+; CHECK-NEXT:    call void @use32(i32 [[HIGH_BITS_EXTRACTED]])
+; CHECK-NEXT:    call void @use1(i1 [[SHOULD_SIGNEXT]])
+; CHECK-NEXT:    call void @use32(i32 [[ALL_BITS_EXCEPT_LOW_NBITS]])
+; CHECK-NEXT:    call void @use32(i32 [[MAGIC]])
+; CHECK-NEXT:    [[SIGNEXTENDED:%.*]] = add i32 [[HIGH_BITS_EXTRACTED]], [[MAGIC]]
+; CHECK-NEXT:    ret i32 [[SIGNEXTENDED]]
+;
+  %low_bits_to_skip = sub i32 32, %nbits
+  %high_bits_extracted = lshr i32 %data1, %low_bits_to_skip ; not %data2
+  %should_signext = icmp slt i32 %data2, 0 ; not %data1
+  %all_bits_except_low_nbits = shl i32 -1, %nbits
+  %magic = select i1 %should_signext, i32 %all_bits_except_low_nbits, i32 0
+
+  call void @use32(i32 %low_bits_to_skip)
+  call void @use32(i32 %high_bits_extracted)
+  call void @use1(i1 %should_signext)
+  call void @use32(i32 %all_bits_except_low_nbits)
+  call void @use32(i32 %magic)
+
+  %signextended = add i32 %high_bits_extracted, %magic
+  ret i32 %signextended
+}
+
+define i32 @n20(i32 %data, i32 %nbits1, i32 %nbits2) {
+; CHECK-LABEL: @n20(
+; CHECK-NEXT:    [[LOW_BITS_TO_SKIP:%.*]] = sub i32 32, [[NBITS1:%.*]]
+; CHECK-NEXT:    [[HIGH_BITS_EXTRACTED:%.*]] = lshr i32 [[DATA:%.*]], [[LOW_BITS_TO_SKIP]]
+; CHECK-NEXT:    [[SHOULD_SIGNEXT:%.*]] = icmp slt i32 [[DATA]], 0
+; CHECK-NEXT:    [[ALL_BITS_EXCEPT_LOW_NBITS:%.*]] = shl i32 -1, [[NBITS2:%.*]]
+; CHECK-NEXT:    [[MAGIC:%.*]] = select i1 [[SHOULD_SIGNEXT]], i32 [[ALL_BITS_EXCEPT_LOW_NBITS]], i32 0
+; CHECK-NEXT:    call void @use32(i32 [[LOW_BITS_TO_SKIP]])
+; CHECK-NEXT:    call void @use32(i32 [[HIGH_BITS_EXTRACTED]])
+; CHECK-NEXT:    call void @use1(i1 [[SHOULD_SIGNEXT]])
+; CHECK-NEXT:    call void @use32(i32 [[ALL_BITS_EXCEPT_LOW_NBITS]])
+; CHECK-NEXT:    call void @use32(i32 [[MAGIC]])
+; CHECK-NEXT:    [[SIGNEXTENDED:%.*]] = add i32 [[HIGH_BITS_EXTRACTED]], [[MAGIC]]
+; CHECK-NEXT:    ret i32 [[SIGNEXTENDED]]
+;
+  %low_bits_to_skip = sub i32 32, %nbits1 ; not %nbits2
+  %high_bits_extracted = lshr i32 %data, %low_bits_to_skip
+  %should_signext = icmp slt i32 %data, 0
+  %all_bits_except_low_nbits = shl i32 -1, %nbits2 ; not %nbits1
+  %magic = select i1 %should_signext, i32 %all_bits_except_low_nbits, i32 0
+
+  call void @use32(i32 %low_bits_to_skip)
+  call void @use32(i32 %high_bits_extracted)
+  call void @use1(i1 %should_signext)
+  call void @use32(i32 %all_bits_except_low_nbits)
+  call void @use32(i32 %magic)
+
+  %signextended = add i32 %high_bits_extracted, %magic
+  ret i32 %signextended
+}
+
+define i32 @n21(i32 %data, i32 %nbits) {
+; CHECK-LABEL: @n21(
+; CHECK-NEXT:    [[LOW_BITS_TO_SKIP:%.*]] = sub i32 32, [[NBITS:%.*]]
+; CHECK-NEXT:    [[HIGH_BITS_EXTRACTED:%.*]] = lshr i32 [[DATA:%.*]], [[LOW_BITS_TO_SKIP]]
+; CHECK-NEXT:    [[SHOULD_SIGNEXT:%.*]] = icmp sgt i32 [[DATA]], 0
+; CHECK-NEXT:    [[ALL_BITS_EXCEPT_LOW_NBITS:%.*]] = shl i32 -1, [[NBITS]]
+; CHECK-NEXT:    [[MAGIC:%.*]] = select i1 [[SHOULD_SIGNEXT]], i32 [[ALL_BITS_EXCEPT_LOW_NBITS]], i32 0
+; CHECK-NEXT:    call void @use32(i32 [[LOW_BITS_TO_SKIP]])
+; CHECK-NEXT:    call void @use32(i32 [[HIGH_BITS_EXTRACTED]])
+; CHECK-NEXT:    call void @use1(i1 [[SHOULD_SIGNEXT]])
+; CHECK-NEXT:    call void @use32(i32 [[ALL_BITS_EXCEPT_LOW_NBITS]])
+; CHECK-NEXT:    call void @use32(i32 [[MAGIC]])
+; CHECK-NEXT:    [[SIGNEXTENDED:%.*]] = add i32 [[HIGH_BITS_EXTRACTED]], [[MAGIC]]
+; CHECK-NEXT:    ret i32 [[SIGNEXTENDED]]
+;
+  %low_bits_to_skip = sub i32 32, %nbits
+  %high_bits_extracted = lshr i32 %data, %low_bits_to_skip
+  %should_signext = icmp sgt i32 %data, 0 ; this isn't a sign bit test
+  %all_bits_except_low_nbits = shl i32 -1, %nbits
+  %magic = select i1 %should_signext, i32 %all_bits_except_low_nbits, i32 0
+
+  call void @use32(i32 %low_bits_to_skip)
+  call void @use32(i32 %high_bits_extracted)
+  call void @use1(i1 %should_signext)
+  call void @use32(i32 %all_bits_except_low_nbits)
+  call void @use32(i32 %magic)
+
+  %signextended = add i32 %high_bits_extracted, %magic
+  ret i32 %signextended
+}
+
+define i32 @n22(i64 %data, i32 %nbits) {
+; CHECK-LABEL: @n22(
+; CHECK-NEXT:    [[LOW_BITS_TO_SKIP:%.*]] = sub i32 63, [[NBITS:%.*]]
+; CHECK-NEXT:    [[LOW_BITS_TO_SKIP_WIDE:%.*]] = zext i32 [[LOW_BITS_TO_SKIP]] to i64
+; CHECK-NEXT:    [[HIGH_BITS_EXTRACTED_WIDE:%.*]] = lshr i64 [[DATA:%.*]], [[LOW_BITS_TO_SKIP_WIDE]]
+; CHECK-NEXT:    [[HIGH_BITS_EXTRACTED:%.*]] = trunc i64 [[HIGH_BITS_EXTRACTED_WIDE]] to i32
+; CHECK-NEXT:    [[SHOULD_SIGNEXT:%.*]] = icmp slt i64 [[DATA]], 0
+; CHECK-NEXT:    [[ALL_BITS_EXCEPT_LOW_NBITS:%.*]] = shl i32 -1, [[NBITS]]
+; CHECK-NEXT:    [[MAGIC:%.*]] = select i1 [[SHOULD_SIGNEXT]], i32 [[ALL_BITS_EXCEPT_LOW_NBITS]], i32 0
+; CHECK-NEXT:    call void @use32(i32 [[LOW_BITS_TO_SKIP]])
+; CHECK-NEXT:    call void @use64(i64 [[LOW_BITS_TO_SKIP_WIDE]])
+; CHECK-NEXT:    call void @use64(i64 [[HIGH_BITS_EXTRACTED_WIDE]])
+; CHECK-NEXT:    call void @use32(i32 [[HIGH_BITS_EXTRACTED]])
+; CHECK-NEXT:    call void @use1(i1 [[SHOULD_SIGNEXT]])
+; CHECK-NEXT:    call void @use32(i32 [[ALL_BITS_EXCEPT_LOW_NBITS]])
+; CHECK-NEXT:    call void @use32(i32 [[MAGIC]])
+; CHECK-NEXT:    [[SIGNEXTENDED:%.*]] = add i32 [[MAGIC]], [[HIGH_BITS_EXTRACTED]]
+; CHECK-NEXT:    ret i32 [[SIGNEXTENDED]]
+;
+  %low_bits_to_skip = sub i32 63, %nbits ; not 64
+  %low_bits_to_skip_wide = zext i32 %low_bits_to_skip to i64
+  %high_bits_extracted_wide = lshr i64 %data, %low_bits_to_skip_wide
+  %high_bits_extracted = trunc i64 %high_bits_extracted_wide to i32
+  %should_signext = icmp slt i64 %data, 0
+  %all_bits_except_low_nbits = shl i32 -1, %nbits
+  %magic = select i1 %should_signext, i32 %all_bits_except_low_nbits, i32 0
+
+  call void @use32(i32 %low_bits_to_skip)
+  call void @use64(i64 %low_bits_to_skip_wide)
+  call void @use64(i64 %high_bits_extracted_wide)
+  call void @use32(i32 %high_bits_extracted)
+  call void @use1(i1 %should_signext)
+  call void @use32(i32 %all_bits_except_low_nbits)
+  call void @use32(i32 %magic)
+
+  %signextended = add i32 %magic, %high_bits_extracted
+  ret i32 %signextended
+}
+
+define i32 @n23(i32 %data, i32 %nbits) {
+; CHECK-LABEL: @n23(
+; CHECK-NEXT:    [[LOW_BITS_TO_SKIP:%.*]] = sub i32 32, [[NBITS:%.*]]
+; CHECK-NEXT:    [[HIGH_BITS_EXTRACTED:%.*]] = ashr i32 [[DATA:%.*]], [[LOW_BITS_TO_SKIP]]
+; CHECK-NEXT:    [[SHOULD_SIGNEXT:%.*]] = icmp slt i32 [[DATA]], 0
+; CHECK-NEXT:    [[ALL_BITS_EXCEPT_LOW_NBITS:%.*]] = shl i32 -1, [[NBITS]]
+; CHECK-NEXT:    [[MAGIC:%.*]] = select i1 [[SHOULD_SIGNEXT]], i32 [[ALL_BITS_EXCEPT_LOW_NBITS]], i32 0
+; CHECK-NEXT:    call void @use32(i32 [[LOW_BITS_TO_SKIP]])
+; CHECK-NEXT:    call void @use32(i32 [[HIGH_BITS_EXTRACTED]])
+; CHECK-NEXT:    call void @use1(i1 [[SHOULD_SIGNEXT]])
+; CHECK-NEXT:    call void @use32(i32 [[ALL_BITS_EXCEPT_LOW_NBITS]])
+; CHECK-NEXT:    call void @use32(i32 [[MAGIC]])
+; CHECK-NEXT:    [[SIGNEXTENDED:%.*]] = add i32 [[HIGH_BITS_EXTRACTED]], [[MAGIC]]
+; CHECK-NEXT:    ret i32 [[SIGNEXTENDED]]
+;
+  %low_bits_to_skip = sub i32 32, %nbits
+  %high_bits_extracted = ashr i32 %data, %low_bits_to_skip ; not `lshr`
+  %should_signext = icmp slt i32 %data, 0
+  %all_bits_except_low_nbits = shl i32 -1, %nbits
+  %magic = select i1 %should_signext, i32 %all_bits_except_low_nbits, i32 0
+
+  call void @use32(i32 %low_bits_to_skip)
+  call void @use32(i32 %high_bits_extracted)
+  call void @use1(i1 %should_signext)
+  call void @use32(i32 %all_bits_except_low_nbits)
+  call void @use32(i32 %magic)
+
+  %signextended = add i32 %high_bits_extracted, %magic
+  ret i32 %signextended
+}
+
+define i32 @n24(i32 %data, i32 %nbits) {
+; CHECK-LABEL: @n24(
+; CHECK-NEXT:    [[LOW_BITS_TO_SKIP:%.*]] = sub i32 32, [[NBITS:%.*]]
+; CHECK-NEXT:    [[HIGH_BITS_EXTRACTED:%.*]] = lshr i32 [[DATA:%.*]], [[LOW_BITS_TO_SKIP]]
+; CHECK-NEXT:    [[SHOULD_SIGNEXT:%.*]] = icmp slt i32 [[DATA]], 0
+; CHECK-NEXT:    [[HIGHER_BIT_AFTER_SIGNBIT:%.*]] = shl i32 1, [[NBITS]]
+; CHECK-NEXT:    [[MAGIC:%.*]] = select i1 [[SHOULD_SIGNEXT]], i32 [[HIGHER_BIT_AFTER_SIGNBIT]], i32 0
+; CHECK-NEXT:    call void @use32(i32 [[LOW_BITS_TO_SKIP]])
+; CHECK-NEXT:    call void @use32(i32 [[HIGH_BITS_EXTRACTED]])
+; CHECK-NEXT:    call void @use1(i1 [[SHOULD_SIGNEXT]])
+; CHECK-NEXT:    call void @use32(i32 [[HIGHER_BIT_AFTER_SIGNBIT]])
+; CHECK-NEXT:    call void @use32(i32 [[MAGIC]])
+; CHECK-NEXT:    [[SIGNEXTENDED:%.*]] = sub i32 [[MAGIC]], [[HIGH_BITS_EXTRACTED]]
+; CHECK-NEXT:    ret i32 [[SIGNEXTENDED]]
+;
+  %low_bits_to_skip = sub i32 32, %nbits
+  %high_bits_extracted = lshr i32 %data, %low_bits_to_skip
+  %should_signext = icmp slt i32 %data, 0
+  %higher_bit_after_signbit = shl i32 1, %nbits
+  %magic = select i1 %should_signext, i32 %higher_bit_after_signbit, i32 0
+
+  call void @use32(i32 %low_bits_to_skip)
+  call void @use32(i32 %high_bits_extracted)
+  call void @use1(i1 %should_signext)
+  call void @use32(i32 %higher_bit_after_signbit)
+  call void @use32(i32 %magic)
+
+  %signextended = sub i32 %magic, %high_bits_extracted ; wrong order; `sub` is not commutative
+  ret i32 %signextended
+}
+
+define i32 @n25_sub(i32 %data, i32 %nbits) {
+; CHECK-LABEL: @n25_sub(
+; CHECK-NEXT:    [[LOW_BITS_TO_SKIP:%.*]] = sub i32 32, [[NBITS:%.*]]
+; CHECK-NEXT:    [[HIGH_BITS_EXTRACTED:%.*]] = lshr i32 [[DATA:%.*]], [[LOW_BITS_TO_SKIP]]
+; CHECK-NEXT:    [[SHOULD_SIGNEXT:%.*]] = icmp slt i32 [[DATA]], 0
+; CHECK-NEXT:    [[HIGHER_BIT_AFTER_SIGNBIT:%.*]] = shl i32 -1, [[NBITS]]
+; CHECK-NEXT:    [[MAGIC:%.*]] = select i1 [[SHOULD_SIGNEXT]], i32 [[HIGHER_BIT_AFTER_SIGNBIT]], i32 0
+; CHECK-NEXT:    call void @use32(i32 [[LOW_BITS_TO_SKIP]])
+; CHECK-NEXT:    call void @use32(i32 [[HIGH_BITS_EXTRACTED]])
+; CHECK-NEXT:    call void @use1(i1 [[SHOULD_SIGNEXT]])
+; CHECK-NEXT:    call void @use32(i32 [[HIGHER_BIT_AFTER_SIGNBIT]])
+; CHECK-NEXT:    call void @use32(i32 [[MAGIC]])
+; CHECK-NEXT:    [[SIGNEXTENDED:%.*]] = sub i32 [[HIGH_BITS_EXTRACTED]], [[MAGIC]]
+; CHECK-NEXT:    ret i32 [[SIGNEXTENDED]]
+;
+  %low_bits_to_skip = sub i32 32, %nbits
+  %high_bits_extracted = lshr i32 %data, %low_bits_to_skip
+  %should_signext = icmp slt i32 %data, 0
+  %higher_bit_after_signbit = shl i32 -1, %nbits ; not 1
+  %magic = select i1 %should_signext, i32 %higher_bit_after_signbit, i32 0
+
+  call void @use32(i32 %low_bits_to_skip)
+  call void @use32(i32 %high_bits_extracted)
+  call void @use1(i1 %should_signext)
+  call void @use32(i32 %higher_bit_after_signbit)
+  call void @use32(i32 %magic)
+
+  %signextended = sub i32 %high_bits_extracted, %magic
+  ret i32 %signextended
+}
+
+define i32 @n26(i32 %data, i32 %nbits) {
+; CHECK-LABEL: @n26(
+; CHECK-NEXT:    [[LOW_BITS_TO_SKIP:%.*]] = sub i32 32, [[NBITS:%.*]]
+; CHECK-NEXT:    [[HIGH_BITS_EXTRACTED:%.*]] = lshr i32 [[DATA:%.*]], [[LOW_BITS_TO_SKIP]]
+; CHECK-NEXT:    [[SHOULD_SIGNEXT:%.*]] = icmp slt i32 [[DATA]], 0
+; CHECK-NEXT:    [[ALL_BITS_EXCEPT_LOW_NBITS:%.*]] = shl i32 -1, [[NBITS]]
+; CHECK-NEXT:    [[MAGIC:%.*]] = select i1 [[SHOULD_SIGNEXT]], i32 [[ALL_BITS_EXCEPT_LOW_NBITS]], i32 -1
+; CHECK-NEXT:    call void @use32(i32 [[LOW_BITS_TO_SKIP]])
+; CHECK-NEXT:    call void @use32(i32 [[HIGH_BITS_EXTRACTED]])
+; CHECK-NEXT:    call void @use1(i1 [[SHOULD_SIGNEXT]])
+; CHECK-NEXT:    call void @use32(i32 [[ALL_BITS_EXCEPT_LOW_NBITS]])
+; CHECK-NEXT:    call void @use32(i32 [[MAGIC]])
+; CHECK-NEXT:    [[SIGNEXTENDED:%.*]] = add i32 [[HIGH_BITS_EXTRACTED]], [[MAGIC]]
+; CHECK-NEXT:    ret i32 [[SIGNEXTENDED]]
+;
+  %low_bits_to_skip = sub i32 32, %nbits
+  %high_bits_extracted = lshr i32 %data, %low_bits_to_skip
+  %should_signext = icmp slt i32 %data, 0
+  %all_bits_except_low_nbits = shl i32 -1, %nbits
+  %magic = select i1 %should_signext, i32 %all_bits_except_low_nbits, i32 -1 ; not 0
+
+  call void @use32(i32 %low_bits_to_skip)
+  call void @use32(i32 %high_bits_extracted)
+  call void @use1(i1 %should_signext)
+  call void @use32(i32 %all_bits_except_low_nbits)
+  call void @use32(i32 %magic)
+
+  %signextended = add i32 %high_bits_extracted, %magic
+  ret i32 %signextended
+}
+
+define i32 @n27_add_zext_of_magic(i32 %data, i8 %nbits) {
+; CHECK-LABEL: @n27_add_zext_of_magic(
+; CHECK-NEXT:    [[NBITS_32BIT:%.*]] = zext i8 [[NBITS:%.*]] to i32
+; CHECK-NEXT:    [[LOW_BITS_TO_SKIP:%.*]] = sub nsw i32 32, [[NBITS_32BIT]]
+; CHECK-NEXT:    [[HIGH_BITS_EXTRACTED:%.*]] = lshr i32 [[DATA:%.*]], [[LOW_BITS_TO_SKIP]]
+; CHECK-NEXT:    [[SHOULD_SIGNEXT:%.*]] = icmp slt i32 [[DATA]], 0
+; CHECK-NEXT:    [[NBITS_16BIT:%.*]] = zext i8 [[NBITS]] to i16
+; CHECK-NEXT:    [[ALL_BITS_EXCEPT_LOW_NBITS:%.*]] = shl i16 -1, [[NBITS_16BIT]]
+; CHECK-NEXT:    [[MAGIC:%.*]] = select i1 [[SHOULD_SIGNEXT]], i16 [[ALL_BITS_EXCEPT_LOW_NBITS]], i16 0
+; CHECK-NEXT:    [[MAGIC_WIDE:%.*]] = zext i16 [[MAGIC]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[NBITS_32BIT]])
+; CHECK-NEXT:    call void @use32(i32 [[LOW_BITS_TO_SKIP]])
+; CHECK-NEXT:    call void @use32(i32 [[HIGH_BITS_EXTRACTED]])
+; CHECK-NEXT:    call void @use1(i1 [[SHOULD_SIGNEXT]])
+; CHECK-NEXT:    call void @use16(i16 [[NBITS_16BIT]])
+; CHECK-NEXT:    call void @use16(i16 [[ALL_BITS_EXCEPT_LOW_NBITS]])
+; CHECK-NEXT:    call void @use16(i16 [[MAGIC]])
+; CHECK-NEXT:    call void @use32(i32 [[MAGIC_WIDE]])
+; CHECK-NEXT:    [[SIGNEXTENDED:%.*]] = add i32 [[HIGH_BITS_EXTRACTED]], [[MAGIC_WIDE]]
+; CHECK-NEXT:    ret i32 [[SIGNEXTENDED]]
+;
+  %nbits_32bit = zext i8 %nbits to i32
+  %low_bits_to_skip = sub i32 32, %nbits_32bit
+  %high_bits_extracted = lshr i32 %data, %low_bits_to_skip
+  %should_signext = icmp slt i32 %data, 0
+  %nbits_16bit = zext i8 %nbits to i16
+  %all_bits_except_low_nbits = shl i16 -1, %nbits_16bit
+  %magic = select i1 %should_signext, i16 %all_bits_except_low_nbits, i16 0
+  %magic_wide = zext i16 %magic to i32 ; not sext
+
+  call void @use32(i32 %nbits_32bit)
+  call void @use32(i32 %low_bits_to_skip)
+  call void @use32(i32 %high_bits_extracted)
+  call void @use1(i1 %should_signext)
+  call void @use16(i16 %nbits_16bit)
+  call void @use16(i16 %all_bits_except_low_nbits)
+  call void @use16(i16 %magic)
+  call void @use32(i32 %magic_wide)
+
+  %signextended = add i32 %high_bits_extracted, %magic_wide
+  ret i32 %signextended
+}
+
+define i32 @n28_sub_sext_of_magic(i32 %data, i8 %nbits) {
+; CHECK-LABEL: @n28_sub_sext_of_magic(
+; CHECK-NEXT:    [[NBITS_32BIT:%.*]] = zext i8 [[NBITS:%.*]] to i32
+; CHECK-NEXT:    [[LOW_BITS_TO_SKIP:%.*]] = sub nsw i32 32, [[NBITS_32BIT]]
+; CHECK-NEXT:    [[HIGH_BITS_EXTRACTED:%.*]] = lshr i32 [[DATA:%.*]], [[LOW_BITS_TO_SKIP]]
+; CHECK-NEXT:    [[SHOULD_SIGNEXT:%.*]] = icmp slt i32 [[DATA]], 0
+; CHECK-NEXT:    [[NBITS_16BIT:%.*]] = zext i8 [[NBITS]] to i16
+; CHECK-NEXT:    [[ALL_BITS_EXCEPT_LOW_NBITS:%.*]] = shl i16 1, [[NBITS_16BIT]]
+; CHECK-NEXT:    [[MAGIC:%.*]] = select i1 [[SHOULD_SIGNEXT]], i16 [[ALL_BITS_EXCEPT_LOW_NBITS]], i16 0
+; CHECK-NEXT:    [[MAGIC_WIDE:%.*]] = sext i16 [[MAGIC]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[NBITS_32BIT]])
+; CHECK-NEXT:    call void @use32(i32 [[LOW_BITS_TO_SKIP]])
+; CHECK-NEXT:    call void @use32(i32 [[HIGH_BITS_EXTRACTED]])
+; CHECK-NEXT:    call void @use1(i1 [[SHOULD_SIGNEXT]])
+; CHECK-NEXT:    call void @use16(i16 [[NBITS_16BIT]])
+; CHECK-NEXT:    call void @use16(i16 [[ALL_BITS_EXCEPT_LOW_NBITS]])
+; CHECK-NEXT:    call void @use16(i16 [[MAGIC]])
+; CHECK-NEXT:    call void @use32(i32 [[MAGIC_WIDE]])
+; CHECK-NEXT:    [[SIGNEXTENDED:%.*]] = sub i32 [[HIGH_BITS_EXTRACTED]], [[MAGIC_WIDE]]
+; CHECK-NEXT:    ret i32 [[SIGNEXTENDED]]
+;
+  %nbits_32bit = zext i8 %nbits to i32
+  %low_bits_to_skip = sub i32 32, %nbits_32bit
+  %high_bits_extracted = lshr i32 %data, %low_bits_to_skip
+  %should_signext = icmp slt i32 %data, 0
+  %nbits_16bit = zext i8 %nbits to i16
+  %all_bits_except_low_nbits = shl i16 1, %nbits_16bit
+  %magic = select i1 %should_signext, i16 %all_bits_except_low_nbits, i16 0
+  %magic_wide = sext i16 %magic to i32 ; not zext
+
+  call void @use32(i32 %nbits_32bit)
+  call void @use32(i32 %low_bits_to_skip)
+  call void @use32(i32 %high_bits_extracted)
+  call void @use1(i1 %should_signext)
+  call void @use16(i16 %nbits_16bit)
+  call void @use16(i16 %all_bits_except_low_nbits)
+  call void @use16(i16 %magic)
+  call void @use32(i32 %magic_wide)
+
+  %signextended = sub i32 %high_bits_extracted, %magic_wide
+  ret i32 %signextended
+}
+
+define i32 @n290_or_with_wrong_magic(i32 %data, i32 %nbits) {
+; CHECK-LABEL: @n290_or_with_wrong_magic(
+; CHECK-NEXT:    [[LOW_BITS_TO_SKIP:%.*]] = sub i32 32, [[NBITS:%.*]]
+; CHECK-NEXT:    [[HIGH_BITS_EXTRACTED:%.*]] = lshr i32 [[DATA:%.*]], [[LOW_BITS_TO_SKIP]]
+; CHECK-NEXT:    [[SHOULD_SIGNEXT:%.*]] = icmp slt i32 [[DATA]], 0
+; CHECK-NEXT:    [[ALL_BITS_EXCEPT_LOW_NBITS:%.*]] = shl i32 1, [[NBITS]]
+; CHECK-NEXT:    [[MAGIC:%.*]] = select i1 [[SHOULD_SIGNEXT]], i32 [[ALL_BITS_EXCEPT_LOW_NBITS]], i32 0
+; CHECK-NEXT:    call void @use32(i32 [[LOW_BITS_TO_SKIP]])
+; CHECK-NEXT:    call void @use32(i32 [[HIGH_BITS_EXTRACTED]])
+; CHECK-NEXT:    call void @use1(i1 [[SHOULD_SIGNEXT]])
+; CHECK-NEXT:    call void @use32(i32 [[ALL_BITS_EXCEPT_LOW_NBITS]])
+; CHECK-NEXT:    call void @use32(i32 [[MAGIC]])
+; CHECK-NEXT:    [[SIGNEXTENDED:%.*]] = or i32 [[HIGH_BITS_EXTRACTED]], [[MAGIC]]
+; CHECK-NEXT:    ret i32 [[SIGNEXTENDED]]
+;
+  %low_bits_to_skip = sub i32 32, %nbits
+  %high_bits_extracted = lshr i32 %data, %low_bits_to_skip
+  %should_signext = icmp slt i32 %data, 0
+  %all_bits_except_low_nbits = shl i32 1, %nbits ; not -1
+  %magic = select i1 %should_signext, i32 %all_bits_except_low_nbits, i32 0
+
+  call void @use32(i32 %low_bits_to_skip)
+  call void @use32(i32 %high_bits_extracted)
+  call void @use1(i1 %should_signext)
+  call void @use32(i32 %all_bits_except_low_nbits)
+  call void @use32(i32 %magic)
+
+  %signextended = or i32 %high_bits_extracted, %magic
+  ret i32 %signextended
+}
diff --git a/llvm/test/Transforms/InstCombine/constant-fold-libfunc.ll b/llvm/test/Transforms/InstCombine/constant-fold-libfunc.ll
index 5d1aa821e..af33d9895 100644
--- a/llvm/test/Transforms/InstCombine/constant-fold-libfunc.ll
+++ b/llvm/test/Transforms/InstCombine/constant-fold-libfunc.ll
@@ -23,7 +23,7 @@ define double @test_acos_nobuiltin() {
 
 ; Check that we don't constant fold strictfp results that require rounding.
 
-define double @test_acos_strictfp() {
+define double @test_acos_strictfp() strictfp {
 ; CHECK-LABEL: @test_acos_strictfp
   %pi = call double @acos(double -1.000000e+00) strictfp 
 ; CHECK: call double @acos(double -1.000000e+00)
diff --git a/llvm/test/Transforms/InstCombine/copysign.ll b/llvm/test/Transforms/InstCombine/copysign.ll
index 556b79999..f7e387fb6 100644
--- a/llvm/test/Transforms/InstCombine/copysign.ll
+++ b/llvm/test/Transforms/InstCombine/copysign.ll
@@ -1,49 +1,77 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -S -instcombine < %s | FileCheck %s
 
-declare float @llvm.copysign.f32(float, float) #0
-declare double @llvm.copysign.f64(double, double) #0
+declare float @llvm.fabs.f32(float)
+declare float @llvm.copysign.f32(float, float)
+declare <3 x double> @llvm.copysign.v3f64(<3 x double>, <3 x double>)
 
-; CHECK-LABEL: @constant_fold_copysign_f32_01
-; CHECK-NEXT: ret float -1.000000e+00
-define float @constant_fold_copysign_f32_01() #0 {
-  %x = call float @llvm.copysign.f32(float 1.0, float -2.0) #0
-  ret float %x
+define float @positive_sign_arg(float %x) {
+; CHECK-LABEL: @positive_sign_arg(
+; CHECK-NEXT:    [[TMP1:%.*]] = call arcp float @llvm.fabs.f32(float [[X:%.*]])
+; CHECK-NEXT:    ret float [[TMP1]]
+;
+  %r = call arcp float @llvm.copysign.f32(float %x, float 0.0)
+  ret float %r
 }
 
-; CHECK-LABEL: @constant_fold_copysign_f32_02
-; CHECK-NEXT: ret float 2.000000e+00
-define float @constant_fold_copysign_f32_02() #0 {
-  %x = call float @llvm.copysign.f32(float -2.0, float 1.0) #0
-  ret float %x
+define <3 x double> @positive_sign_arg_vec_splat(<3 x double> %x) {
+; CHECK-LABEL: @positive_sign_arg_vec_splat(
+; CHECK-NEXT:    [[TMP1:%.*]] = call ninf <3 x double> @llvm.fabs.v3f64(<3 x double> [[X:%.*]])
+; CHECK-NEXT:    ret <3 x double> [[TMP1]]
+;
+  %r = call ninf <3 x double> @llvm.copysign.v3f64(<3 x double> %x, <3 x double> <double 42.0, double 42.0, double 42.0>)
+  ret <3 x double> %r
 }
 
-; CHECK-LABEL: @constant_fold_copysign_f32_03
-; CHECK-NEXT: ret float -2.000000e+00
-define float @constant_fold_copysign_f32_03() #0 {
-  %x = call float @llvm.copysign.f32(float -2.0, float -1.0) #0
-  ret float %x
+define float @negative_sign_arg(float %x) {
+; CHECK-LABEL: @negative_sign_arg(
+; CHECK-NEXT:    [[TMP1:%.*]] = call nnan float @llvm.fabs.f32(float [[X:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = fneg nnan float [[TMP1]]
+; CHECK-NEXT:    ret float [[TMP2]]
+;
+  %r = call nnan float @llvm.copysign.f32(float %x, float -0.0)
+  ret float %r
 }
 
-; CHECK-LABEL: @constant_fold_copysign_f64_01
-; CHECK-NEXT: ret double -1.000000e+00
-define double @constant_fold_copysign_f64_01() #0 {
-  %x = call double @llvm.copysign.f64(double 1.0, double -2.0) #0
-  ret double %x
+define <3 x double> @negative_sign_arg_vec_splat(<3 x double> %x) {
+; CHECK-LABEL: @negative_sign_arg_vec_splat(
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <3 x double> @llvm.fabs.v3f64(<3 x double> [[X:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = fneg fast <3 x double> [[TMP1]]
+; CHECK-NEXT:    ret <3 x double> [[TMP2]]
+;
+  %r = call fast <3 x double> @llvm.copysign.v3f64(<3 x double> %x, <3 x double> <double -42.0, double -42.0, double -42.0>)
+  ret <3 x double> %r
 }
 
-; CHECK-LABEL: @constant_fold_copysign_f64_02
-; CHECK-NEXT: ret double 1.000000e+00
-define double @constant_fold_copysign_f64_02() #0 {
-  %x = call double @llvm.copysign.f64(double -1.0, double 2.0) #0
-  ret double %x
+define float @known_positive_sign_arg(float %x, float %y) {
+; CHECK-LABEL: @known_positive_sign_arg(
+; CHECK-NEXT:    [[TMP1:%.*]] = call ninf float @llvm.fabs.f32(float [[X:%.*]])
+; CHECK-NEXT:    ret float [[TMP1]]
+;
+  %fabs = call float @llvm.fabs.f32(float %y)
+  %r = call ninf float @llvm.copysign.f32(float %x, float %fabs)
+  ret float %r
 }
 
-; CHECK-LABEL: @constant_fold_copysign_f64_03
-; CHECK-NEXT: ret double -1.000000e+00
-define double @constant_fold_copysign_f64_03() #0 {
-  %x = call double @llvm.copysign.f64(double -1.0, double -2.0) #0
-  ret double %x
+define <3 x double> @known_positive_sign_arg_vec(<3 x double> %x, <3 x i32> %y) {
+; CHECK-LABEL: @known_positive_sign_arg_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = call arcp <3 x double> @llvm.fabs.v3f64(<3 x double> [[X:%.*]])
+; CHECK-NEXT:    ret <3 x double> [[TMP1]]
+;
+  %yf = uitofp <3 x i32> %y to <3 x double>
+  %r = call arcp <3 x double> @llvm.copysign.v3f64(<3 x double> %x, <3 x double> %yf)
+  ret <3 x double> %r
 }
 
+; The magnitude operand of the 1st copysign is irrelevant.
+; copysign(x, copysign(y, z)) --> copysign(x, z)
 
-attributes #0 = { nounwind readnone }
+define float @copysign_sign_arg(float %x, float %y, float %z) {
+; CHECK-LABEL: @copysign_sign_arg(
+; CHECK-NEXT:    [[R:%.*]] = call ninf float @llvm.copysign.f32(float [[X:%.*]], float [[Z:%.*]])
+; CHECK-NEXT:    ret float [[R]]
+;
+  %s = call reassoc float @llvm.copysign.f32(float %y, float %z)
+  %r = call ninf float @llvm.copysign.f32(float %x, float %s)
+  ret float %r
+}
diff --git a/llvm/test/Transforms/InstCombine/cos-1.ll b/llvm/test/Transforms/InstCombine/cos-1.ll
index 12a64403e..cf668f926 100644
--- a/llvm/test/Transforms/InstCombine/cos-1.ll
+++ b/llvm/test/Transforms/InstCombine/cos-1.ll
@@ -84,7 +84,7 @@ define float @cosf_unary_negated_arg_FMF(float %x) {
 define double @sin_negated_arg(double %x) {
 ; ANY-LABEL: @sin_negated_arg(
 ; ANY-NEXT:    [[TMP1:%.*]] = call double @sin(double [[X:%.*]])
-; ANY-NEXT:    [[TMP2:%.*]] = fsub double -0.000000e+00, [[TMP1]]
+; ANY-NEXT:    [[TMP2:%.*]] = fneg double [[TMP1]]
 ; ANY-NEXT:    ret double [[TMP2]]
 ;
   %neg = fsub double -0.0, %x
@@ -95,7 +95,7 @@ define double @sin_negated_arg(double %x) {
 define double @sin_unary_negated_arg(double %x) {
 ; ANY-LABEL: @sin_unary_negated_arg(
 ; ANY-NEXT:    [[TMP1:%.*]] = call double @sin(double [[X:%.*]])
-; ANY-NEXT:    [[TMP2:%.*]] = fsub double -0.000000e+00, [[TMP1]]
+; ANY-NEXT:    [[TMP2:%.*]] = fneg double [[TMP1]]
 ; ANY-NEXT:    ret double [[TMP2]]
 ;
   %neg = fneg double %x
@@ -106,7 +106,7 @@ define double @sin_unary_negated_arg(double %x) {
 define float @sinf_negated_arg(float %x) {
 ; ANY-LABEL: @sinf_negated_arg(
 ; ANY-NEXT:    [[TMP1:%.*]] = call float @sinf(float [[X:%.*]])
-; ANY-NEXT:    [[TMP2:%.*]] = fsub float -0.000000e+00, [[TMP1]]
+; ANY-NEXT:    [[TMP2:%.*]] = fneg float [[TMP1]]
 ; ANY-NEXT:    ret float [[TMP2]]
 ;
   %neg = fsub float -0.0, %x
@@ -117,7 +117,7 @@ define float @sinf_negated_arg(float %x) {
 define float @sinf_unary_negated_arg(float %x) {
 ; ANY-LABEL: @sinf_unary_negated_arg(
 ; ANY-NEXT:    [[TMP1:%.*]] = call float @sinf(float [[X:%.*]])
-; ANY-NEXT:    [[TMP2:%.*]] = fsub float -0.000000e+00, [[TMP1]]
+; ANY-NEXT:    [[TMP2:%.*]] = fneg float [[TMP1]]
 ; ANY-NEXT:    ret float [[TMP2]]
 ;
   %neg = fneg float %x
@@ -128,7 +128,7 @@ define float @sinf_unary_negated_arg(float %x) {
 define float @sinf_negated_arg_FMF(float %x) {
 ; ANY-LABEL: @sinf_negated_arg_FMF(
 ; ANY-NEXT:    [[TMP1:%.*]] = call nnan afn float @sinf(float [[X:%.*]])
-; ANY-NEXT:    [[TMP2:%.*]] = fsub nnan afn float -0.000000e+00, [[TMP1]]
+; ANY-NEXT:    [[TMP2:%.*]] = fneg nnan afn float [[TMP1]]
 ; ANY-NEXT:    ret float [[TMP2]]
 ;
   %neg = fsub ninf float -0.0, %x
@@ -139,7 +139,7 @@ define float @sinf_negated_arg_FMF(float %x) {
 define float @sinf_unary_negated_arg_FMF(float %x) {
 ; ANY-LABEL: @sinf_unary_negated_arg_FMF(
 ; ANY-NEXT:    [[TMP1:%.*]] = call nnan afn float @sinf(float [[X:%.*]])
-; ANY-NEXT:    [[TMP2:%.*]] = fsub nnan afn float -0.000000e+00, [[TMP1]]
+; ANY-NEXT:    [[TMP2:%.*]] = fneg nnan afn float [[TMP1]]
 ; ANY-NEXT:    ret float [[TMP2]]
 ;
   %neg = fneg ninf float %x
@@ -227,7 +227,7 @@ define double @unary_neg_sin_negated_arg(double %x) {
 define double @tan_negated_arg(double %x) {
 ; ANY-LABEL: @tan_negated_arg(
 ; ANY-NEXT:    [[TMP1:%.*]] = call double @tan(double [[X:%.*]])
-; ANY-NEXT:    [[TMP2:%.*]] = fsub double -0.000000e+00, [[TMP1]]
+; ANY-NEXT:    [[TMP2:%.*]] = fneg double [[TMP1]]
 ; ANY-NEXT:    ret double [[TMP2]]
 ;
   %neg = fsub double -0.0, %x
@@ -238,7 +238,7 @@ define double @tan_negated_arg(double %x) {
 define double @tan_unary_negated_arg(double %x) {
 ; ANY-LABEL: @tan_unary_negated_arg(
 ; ANY-NEXT:    [[TMP1:%.*]] = call double @tan(double [[X:%.*]])
-; ANY-NEXT:    [[TMP2:%.*]] = fsub double -0.000000e+00, [[TMP1]]
+; ANY-NEXT:    [[TMP2:%.*]] = fneg double [[TMP1]]
 ; ANY-NEXT:    ret double [[TMP2]]
 ;
   %neg = fneg double %x
@@ -251,7 +251,7 @@ define double @tan_unary_negated_arg(double %x) {
 define fp128 @tanl_negated_arg(fp128 %x) {
 ; ANY-LABEL: @tanl_negated_arg(
 ; ANY-NEXT:    [[TMP1:%.*]] = call fp128 @tanl(fp128 [[X:%.*]])
-; ANY-NEXT:    [[TMP2:%.*]] = fsub fp128 0xL00000000000000008000000000000000, [[TMP1]]
+; ANY-NEXT:    [[TMP2:%.*]] = fneg fp128 [[TMP1]]
 ; ANY-NEXT:    ret fp128 [[TMP2]]
 ;
   %neg = fsub fp128 0xL00000000000000008000000000000000, %x
@@ -262,7 +262,7 @@ define fp128 @tanl_negated_arg(fp128 %x) {
 define fp128 @tanl_unary_negated_arg(fp128 %x) {
 ; ANY-LABEL: @tanl_unary_negated_arg(
 ; ANY-NEXT:    [[TMP1:%.*]] = call fp128 @tanl(fp128 [[X:%.*]])
-; ANY-NEXT:    [[TMP2:%.*]] = fsub fp128 0xL00000000000000008000000000000000, [[TMP1]]
+; ANY-NEXT:    [[TMP2:%.*]] = fneg fp128 [[TMP1]]
 ; ANY-NEXT:    ret fp128 [[TMP2]]
 ;
   %neg = fneg fp128 %x
diff --git a/llvm/test/Transforms/InstCombine/debuginfo-dce.ll b/llvm/test/Transforms/InstCombine/debuginfo-dce.ll
index 200ea26cd..5b10f7ba4 100644
--- a/llvm/test/Transforms/InstCombine/debuginfo-dce.ll
+++ b/llvm/test/Transforms/InstCombine/debuginfo-dce.ll
@@ -34,7 +34,7 @@ entry:
   call void @llvm.dbg.value(metadata %struct.entry* %1, metadata !18, metadata !20), !dbg !19
 ; CHECK: define void @salvage_load
 ; CHECK-NEXT: entry:
-; CHECK-NOT: dbg.value
+; CHECK-NEXT: call void @llvm.dbg.value(metadata %struct.entry* undef
   store %struct.entry* %1, %struct.entry** %im_not_dead, align 8
   ret void, !dbg !21
 }
diff --git a/llvm/test/Transforms/InstCombine/deref-alloc-fns.ll b/llvm/test/Transforms/InstCombine/deref-alloc-fns.ll
new file mode 100644
index 000000000..efaf1cd16
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/deref-alloc-fns.ll
@@ -0,0 +1,229 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -instcombine  -S < %s | FileCheck %s
+
+declare noalias i8* @malloc(i64)
+declare noalias i8* @calloc(i64, i64)
+declare noalias i8* @realloc(i8* nocapture, i64)
+declare noalias nonnull i8* @_Znam(i64) ; throwing version of 'new'
+declare noalias nonnull i8* @_Znwm(i64) ; throwing version of 'new'
+declare noalias i8* @strdup(i8*)
+
+@.str = private unnamed_addr constant [6 x i8] c"hello\00", align 1
+
+define noalias i8* @malloc_nonconstant_size(i64 %n) {
+; CHECK-LABEL: @malloc_nonconstant_size(
+; CHECK-NEXT:    [[CALL:%.*]] = tail call noalias i8* @malloc(i64 [[N:%.*]])
+; CHECK-NEXT:    ret i8* [[CALL]]
+;
+  %call = tail call noalias i8* @malloc(i64 %n)
+  ret i8* %call
+}
+
+define noalias i8* @malloc_constant_size() {
+; CHECK-LABEL: @malloc_constant_size(
+; CHECK-NEXT:    [[CALL:%.*]] = tail call noalias dereferenceable_or_null(40) i8* @malloc(i64 40)
+; CHECK-NEXT:    ret i8* [[CALL]]
+;
+  %call = tail call noalias i8* @malloc(i64 40)
+  ret i8* %call
+}
+
+define noalias i8* @malloc_constant_size2() {
+; CHECK-LABEL: @malloc_constant_size2(
+; CHECK-NEXT:    [[CALL:%.*]] = tail call noalias dereferenceable_or_null(80) i8* @malloc(i64 40)
+; CHECK-NEXT:    ret i8* [[CALL]]
+;
+  %call = tail call noalias dereferenceable_or_null(80) i8* @malloc(i64 40)
+  ret i8* %call
+}
+
+define noalias i8* @malloc_constant_size3() {
+; CHECK-LABEL: @malloc_constant_size3(
+; CHECK-NEXT:    [[CALL:%.*]] = tail call noalias dereferenceable(80) dereferenceable_or_null(40) i8* @malloc(i64 40)
+; CHECK-NEXT:    ret i8* [[CALL]]
+;
+  %call = tail call noalias dereferenceable(80) i8* @malloc(i64 40)
+  ret i8* %call
+}
+
+
+define noalias i8* @malloc_constant_zero_size() {
+; CHECK-LABEL: @malloc_constant_zero_size(
+; CHECK-NEXT:    [[CALL:%.*]] = tail call noalias i8* @malloc(i64 0)
+; CHECK-NEXT:    ret i8* [[CALL]]
+;
+  %call = tail call noalias i8* @malloc(i64 0)
+  ret i8* %call
+}
+
+define noalias i8* @realloc_nonconstant_size(i8* %p, i64 %n) {
+; CHECK-LABEL: @realloc_nonconstant_size(
+; CHECK-NEXT:    [[CALL:%.*]] = tail call noalias i8* @realloc(i8* [[P:%.*]], i64 [[N:%.*]])
+; CHECK-NEXT:    ret i8* [[CALL]]
+;
+  %call = tail call noalias i8* @realloc(i8* %p, i64 %n)
+  ret i8* %call
+}
+
+define noalias i8* @realloc_constant_zero_size(i8* %p) {
+; CHECK-LABEL: @realloc_constant_zero_size(
+; CHECK-NEXT:    [[CALL:%.*]] = tail call noalias i8* @realloc(i8* [[P:%.*]], i64 0)
+; CHECK-NEXT:    ret i8* [[CALL]]
+;
+  %call = tail call noalias i8* @realloc(i8* %p, i64 0)
+  ret i8* %call
+}
+
+define noalias i8* @realloc_constant_size(i8* %p) {
+; CHECK-LABEL: @realloc_constant_size(
+; CHECK-NEXT:    [[CALL:%.*]] = tail call noalias dereferenceable_or_null(40) i8* @realloc(i8* [[P:%.*]], i64 40)
+; CHECK-NEXT:    ret i8* [[CALL]]
+;
+  %call = tail call noalias i8* @realloc(i8* %p, i64 40)
+  ret i8* %call
+}
+
+define noalias i8* @calloc_nonconstant_size(i64 %n) {
+; CHECK-LABEL: @calloc_nonconstant_size(
+; CHECK-NEXT:    [[CALL:%.*]] = tail call noalias i8* @calloc(i64 1, i64 [[N:%.*]])
+; CHECK-NEXT:    ret i8* [[CALL]]
+;
+  %call = tail call noalias i8* @calloc(i64 1, i64 %n)
+  ret i8* %call
+}
+
+define noalias i8* @calloc_nonconstant_size2(i64 %n) {
+; CHECK-LABEL: @calloc_nonconstant_size2(
+; CHECK-NEXT:    [[CALL:%.*]] = tail call noalias i8* @calloc(i64 [[N:%.*]], i64 0)
+; CHECK-NEXT:    ret i8* [[CALL]]
+;
+  %call = tail call noalias i8* @calloc(i64 %n, i64 0)
+  ret i8* %call
+}
+
+define noalias i8* @calloc_nonconstant_size3(i64 %n) {
+; CHECK-LABEL: @calloc_nonconstant_size3(
+; CHECK-NEXT:    [[CALL:%.*]] = tail call noalias i8* @calloc(i64 [[N:%.*]], i64 [[N]])
+; CHECK-NEXT:    ret i8* [[CALL]]
+;
+  %call = tail call noalias i8* @calloc(i64 %n, i64 %n)
+  ret i8* %call
+}
+
+define noalias i8* @calloc_constant_zero_size() {
+; CHECK-LABEL: @calloc_constant_zero_size(
+; CHECK-NEXT:    [[CALL:%.*]] = tail call noalias i8* @calloc(i64 0, i64 0)
+; CHECK-NEXT:    ret i8* [[CALL]]
+;
+  %call = tail call noalias i8* @calloc(i64 0, i64 0)
+  ret i8* %call
+}
+
+define noalias i8* @calloc_constant_zero_size2(i64 %n) {
+; CHECK-LABEL: @calloc_constant_zero_size2(
+; CHECK-NEXT:    [[CALL:%.*]] = tail call noalias i8* @calloc(i64 [[N:%.*]], i64 0)
+; CHECK-NEXT:    ret i8* [[CALL]]
+;
+  %call = tail call noalias i8* @calloc(i64 %n, i64 0)
+  ret i8* %call
+}
+
+
+define noalias i8* @calloc_constant_zero_size3(i64 %n) {
+; CHECK-LABEL: @calloc_constant_zero_size3(
+; CHECK-NEXT:    [[CALL:%.*]] = tail call noalias i8* @calloc(i64 0, i64 [[N:%.*]])
+; CHECK-NEXT:    ret i8* [[CALL]]
+;
+  %call = tail call noalias i8* @calloc(i64 0, i64 %n)
+  ret i8* %call
+}
+
+define noalias i8* @calloc_constant_zero_size4(i64 %n) {
+; CHECK-LABEL: @calloc_constant_zero_size4(
+; CHECK-NEXT:    [[CALL:%.*]] = tail call noalias i8* @calloc(i64 0, i64 1)
+; CHECK-NEXT:    ret i8* [[CALL]]
+;
+  %call = tail call noalias i8* @calloc(i64 0, i64 1)
+  ret i8* %call
+}
+
+define noalias i8* @calloc_constant_zero_size5(i64 %n) {
+; CHECK-LABEL: @calloc_constant_zero_size5(
+; CHECK-NEXT:    [[CALL:%.*]] = tail call noalias i8* @calloc(i64 1, i64 0)
+; CHECK-NEXT:    ret i8* [[CALL]]
+;
+  %call = tail call noalias i8* @calloc(i64 1, i64 0)
+  ret i8* %call
+}
+
+define noalias i8* @calloc_constant_size() {
+; CHECK-LABEL: @calloc_constant_size(
+; CHECK-NEXT:    [[CALL:%.*]] = tail call noalias dereferenceable_or_null(128) i8* @calloc(i64 16, i64 8)
+; CHECK-NEXT:    ret i8* [[CALL]]
+;
+  %call = tail call noalias i8* @calloc(i64 16, i64 8)
+  ret i8* %call
+}
+
+define noalias i8* @calloc_constant_size_overflow() {
+; CHECK-LABEL: @calloc_constant_size_overflow(
+; CHECK-NEXT:    [[CALL:%.*]] = tail call noalias i8* @calloc(i64 2000000000000, i64 80000000000)
+; CHECK-NEXT:    ret i8* [[CALL]]
+;
+  %call = tail call noalias i8* @calloc(i64 2000000000000, i64 80000000000)
+  ret i8* %call
+}
+
+define noalias i8* @op_new_nonconstant_size(i64 %n) {
+; CHECK-LABEL: @op_new_nonconstant_size(
+; CHECK-NEXT:    [[CALL:%.*]] = tail call i8* @_Znam(i64 [[N:%.*]])
+; CHECK-NEXT:    ret i8* [[CALL]]
+;
+  %call = tail call i8* @_Znam(i64 %n)
+  ret i8* %call
+}
+
+define noalias i8* @op_new_constant_size() {
+; CHECK-LABEL: @op_new_constant_size(
+; CHECK-NEXT:    [[CALL:%.*]] = tail call dereferenceable(40) i8* @_Znam(i64 40)
+; CHECK-NEXT:    ret i8* [[CALL]]
+;
+  %call = tail call i8* @_Znam(i64 40)
+  ret i8* %call
+}
+
+define noalias i8* @op_new_constant_size2() {
+; CHECK-LABEL: @op_new_constant_size2(
+; CHECK-NEXT:    [[CALL:%.*]] = tail call dereferenceable(40) i8* @_Znwm(i64 40)
+; CHECK-NEXT:    ret i8* [[CALL]]
+;
+  %call = tail call i8* @_Znwm(i64 40)
+  ret i8* %call
+}
+
+define noalias i8* @op_new_constant_zero_size() {
+; CHECK-LABEL: @op_new_constant_zero_size(
+; CHECK-NEXT:    [[CALL:%.*]] = tail call i8* @_Znam(i64 0)
+; CHECK-NEXT:    ret i8* [[CALL]]
+;
+  %call = tail call i8* @_Znam(i64 0)
+  ret i8* %call
+}
+
+define noalias i8* @strdup_constant_str() {
+; CHECK-LABEL: @strdup_constant_str(
+; CHECK-NEXT:    [[CALL:%.*]] = tail call noalias dereferenceable_or_null(6) i8* @strdup(i8* getelementptr inbounds ([6 x i8], [6 x i8]* @.str, i64 0, i64 0))
+; CHECK-NEXT:    ret i8* [[CALL]]
+;
+  %call = tail call noalias i8* @strdup(i8* getelementptr inbounds ([6 x i8], [6 x i8]* @.str, i64 0, i64 0))
+  ret i8* %call
+}
+
+define noalias i8* @strdup_notconstant_str(i8 * %str) {
+; CHECK-LABEL: @strdup_notconstant_str(
+; CHECK-NEXT:    [[CALL:%.*]] = tail call noalias i8* @strdup(i8* [[STR:%.*]])
+; CHECK-NEXT:    ret i8* [[CALL]]
+;
+  %call = tail call noalias i8* @strdup(i8* %str)
+  ret i8* %call
+}
diff --git a/llvm/test/Transforms/InstCombine/disable-builtin.ll b/llvm/test/Transforms/InstCombine/disable-builtin.ll
new file mode 100644
index 000000000..a301b914b
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/disable-builtin.ll
@@ -0,0 +1,21 @@
+; Test that -disable-builtin works correctly.
+;
+; RUN: opt < %s -instcombine -disable-builtin strcat -S | FileCheck %s
+;
+; RUN: not opt < %s -instcombine -disable-builtin foobar -S 2>&1 | FileCheck --check-prefix=FOOBAR %s
+; FOOBAR: cannot disable nonexistent builtin function foobar
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+@empty = constant [1 x i8] c"\00", align 1
+
+declare i8* @strcat(i8*, i8*)
+
+define i8* @test_strcat(i8* %x) {
+; CHECK-LABEL: @test_strcat(
+  %empty = getelementptr [1 x i8], [1 x i8]* @empty, i32 0, i32 0
+  %ret = call i8* @strcat(i8* %x, i8* %empty)
+  ret i8* %ret
+; CHECK: call i8* @strcat
+}
+
diff --git a/llvm/test/Transforms/InstCombine/div-shift-crash.ll b/llvm/test/Transforms/InstCombine/div-shift-crash.ll
index 936173cd6..33fc558f3 100644
--- a/llvm/test/Transforms/InstCombine/div-shift-crash.ll
+++ b/llvm/test/Transforms/InstCombine/div-shift-crash.ll
@@ -98,4 +98,4 @@ for.end45:                                        ; preds = %for.cond15
   unreachable
 }
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf"="true" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/Transforms/InstCombine/div.ll b/llvm/test/Transforms/InstCombine/div.ll
index 4c4308151..a4c6139e2 100644
--- a/llvm/test/Transforms/InstCombine/div.ll
+++ b/llvm/test/Transforms/InstCombine/div.ll
@@ -522,8 +522,7 @@ define <2 x i8> @sdiv_negated_dividend_constant_divisor_vec_undef(<2 x i8> %x) {
 
 define <2 x i64> @sdiv_negated_dividend_constant_divisor_vec(<2 x i64> %x) {
 ; CHECK-LABEL: @sdiv_negated_dividend_constant_divisor_vec(
-; CHECK-NEXT:    [[DIV1:%.*]] = sdiv <2 x i64> [[X:%.*]], <i64 3, i64 4>
-; CHECK-NEXT:    [[DIV:%.*]] = sub nsw <2 x i64> zeroinitializer, [[DIV1]]
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv <2 x i64> [[X:%.*]], <i64 -3, i64 -4>
 ; CHECK-NEXT:    ret <2 x i64> [[DIV]]
 ;
   %neg = sub nsw <2 x i64> zeroinitializer, %x
@@ -533,8 +532,7 @@ define <2 x i64> @sdiv_negated_dividend_constant_divisor_vec(<2 x i64> %x) {
 
 define <2 x i64> @sdiv_exact_negated_dividend_constant_divisor_vec(<2 x i64> %x) {
 ; CHECK-LABEL: @sdiv_exact_negated_dividend_constant_divisor_vec(
-; CHECK-NEXT:    [[DIV1:%.*]] = sdiv exact <2 x i64> [[X:%.*]], <i64 3, i64 4>
-; CHECK-NEXT:    [[DIV:%.*]] = sub nsw <2 x i64> zeroinitializer, [[DIV1]]
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv exact <2 x i64> [[X:%.*]], <i64 -3, i64 -4>
 ; CHECK-NEXT:    ret <2 x i64> [[DIV]]
 ;
   %neg = sub nsw <2 x i64> zeroinitializer, %x
@@ -860,8 +858,7 @@ define i32 @test_exact_nsw_exact(i32 %x) {
 
 define <2 x i64> @test_exact_vec(<2 x i64> %x) {
 ; CHECK-LABEL: @test_exact_vec(
-; CHECK-NEXT:    [[DIV:%.*]] = sdiv exact <2 x i64> [[X:%.*]], <i64 3, i64 4>
-; CHECK-NEXT:    [[NEG:%.*]] = sub nsw <2 x i64> zeroinitializer, [[DIV]]
+; CHECK-NEXT:    [[NEG:%.*]] = sdiv exact <2 x i64> [[X:%.*]], <i64 -3, i64 -4>
 ; CHECK-NEXT:    ret <2 x i64> [[NEG]]
 ;
   %div = sdiv exact <2 x i64> %x, <i64 3, i64 4>
diff --git a/llvm/test/Transforms/InstCombine/do-not-clone-dbg-declare.ll b/llvm/test/Transforms/InstCombine/do-not-clone-dbg-declare.ll
new file mode 100644
index 000000000..0d9a0aa32
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/do-not-clone-dbg-declare.ll
@@ -0,0 +1,144 @@
+
+; RUN: opt %s -instcombine -verify -S -o - | FileCheck %s
+
+; Hand-reduced from this example.
+; -g -O -mllvm -disable-llvm-optzns -gno-column-info
+; plus opt -sroa -instcombine -inline
+
+; #include <stdio.h>
+;
+; struct S1 {
+;     int p1;
+;     int p2;
+;
+;     bool IsNull (  ) {
+;         return p1 == 0;
+;     }
+; };
+;
+; S1 foo ( void );
+;
+; int bar (  ) {
+;
+;     S1 result = foo();
+;
+;     if ( result.IsNull() )
+;         return 0;
+;
+;     result.p1 = 2;
+;     result.p2 = 3;
+;
+;     int* ptr = &result.p1;
+;
+;     printf("%d", *ptr);
+;     printf("%d", *(ptr+1));
+;
+;     return result.p1 + 1;
+; }
+
+; CHECK: _Z3barv
+; CHECK: llvm.dbg.declare(metadata i64* %{{.*}}, metadata [[METADATA_IDX1:![0-9]+]]
+; CHECK-NOT: llvm.dbg.declare(metadata %struct.S1* %{{.*}}, metadata [[METADATA_IDX1]]
+; CHECK: ret
+; CHECK: DICompileUnit
+; CHECK: [[METADATA_IDX1]] = !DILocalVariable(name: "result"
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+%struct.S1 = type { i32, i32 }
+
+@.str = private unnamed_addr constant [3 x i8] c"%d\00", align 1
+
+define dso_local i32 @_Z3barv() !dbg !7 {
+entry:
+  %result = alloca i64, align 8
+  %tmpcast = bitcast i64* %result to %struct.S1*
+  %0 = bitcast i64* %result to i8*, !dbg !24
+  call void @llvm.lifetime.start.p0i8(i64 8, i8* nonnull %0) #4, !dbg !24
+  call void @llvm.dbg.declare(metadata %struct.S1* %tmpcast, metadata !12, metadata !DIExpression()), !dbg !24
+  %call = call i64 @_Z3foov(), !dbg !24
+  store i64 %call, i64* %result, align 8, !dbg !24
+  call void @llvm.dbg.value(metadata %struct.S1* %tmpcast, metadata !25, metadata !DIExpression()), !dbg !29
+  %p1.i = getelementptr inbounds %struct.S1, %struct.S1* %tmpcast, i64 0, i32 0, !dbg !32
+  %1 = load i32, i32* %p1.i, align 4, !dbg !32
+  %cmp.i = icmp eq i32 %1, 0, !dbg !32
+  br i1 %cmp.i, label %if.then, label %if.end, !dbg !38
+
+if.then:                                          ; preds = %entry
+  br label %cleanup, !dbg !38
+
+if.end:                                           ; preds = %entry
+
+  %p1 = bitcast i64* %result to i32*, !dbg !38
+  store i32 2, i32* %p1, align 8, !dbg !38
+  %p2 = getelementptr inbounds %struct.S1, %struct.S1* %tmpcast, i64 0, i32 1, !dbg !38
+  store i32 3, i32* %p2, align 4, !dbg !38
+  %p12 = bitcast i64* %result to i32*, !dbg !38
+  call void @llvm.dbg.value(metadata i32* %p12, metadata !22, metadata !DIExpression()), !dbg !38
+  %call3 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @.str, i64 0, i64 0), i32 2), !dbg !38
+  %add.ptr = getelementptr inbounds i32, i32* %p12, i64 1, !dbg !38
+  %2 = load i32, i32* %add.ptr, align 4, !dbg !38
+  %call4 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @.str, i64 0, i64 0), i32 %2), !dbg !38
+  %p15 = bitcast i64* %result to i32*, !dbg !38
+  %3 = load i32, i32* %p15, align 8, !dbg !38
+  %add = add nsw i32 %3, 1, !dbg !38
+  br label %cleanup
+
+cleanup:                                          ; preds = %if.end, %if.then
+  %retval.0 = phi i32 [ 0, %if.then ], [ %add, %if.end ], !dbg !38
+  %4 = bitcast i64* %result to i8*, !dbg !38
+  call void @llvm.lifetime.end.p0i8(i64 8, i8* nonnull %4) #4, !dbg !38
+  ret i32 %retval.0, !dbg !38
+}
+
+declare void @llvm.lifetime.start.p0i8(i64 immarg, i8* nocapture) #1
+
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #2
+
+declare dso_local i64 @_Z3foov() #3
+
+declare dso_local i32 @printf(i8*, ...) #3
+
+declare void @llvm.lifetime.end.p0i8(i64 immarg, i8* nocapture) #1
+
+declare void @llvm.dbg.value(metadata, metadata, metadata) #2
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "test.cpp", directory: "")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{!"clang"}
+!7 = distinct !DISubprogram(name: "bar", linkageName: "_Z3barv", scope: !1, file: !1, line: 15, type: !8, scopeLine: 15, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !11)
+!8 = !DISubroutineType(types: !9)
+!9 = !{!10}
+!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!11 = !{!12, !22}
+!12 = !DILocalVariable(name: "result", scope: !7, file: !1, line: 17, type: !13)
+!13 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "S1", file: !1, line: 4, size: 64, flags: DIFlagTypePassByValue, elements: !14, identifier: "_ZTS2S1")
+!14 = !{!15, !16, !17}
+!15 = !DIDerivedType(tag: DW_TAG_member, name: "p1", scope: !13, file: !1, line: 5, baseType: !10, size: 32)
+!16 = !DIDerivedType(tag: DW_TAG_member, name: "p2", scope: !13, file: !1, line: 6, baseType: !10, size: 32, offset: 32)
+!17 = !DISubprogram(name: "IsNull", linkageName: "_ZN2S16IsNullEv", scope: !13, file: !1, line: 8, type: !18, scopeLine: 8, flags: DIFlagPrototyped, spFlags: DISPFlagOptimized)
+!18 = !DISubroutineType(types: !19)
+!19 = !{!20, !21}
+!20 = !DIBasicType(name: "bool", size: 8, encoding: DW_ATE_boolean)
+!21 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !13, size: 64, flags: DIFlagArtificial | DIFlagObjectPointer)
+!22 = !DILocalVariable(name: "ptr", scope: !7, file: !1, line: 25, type: !23)
+!23 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !10, size: 64)
+!24 = !DILocation(line: 17, scope: !7)
+!25 = !DILocalVariable(name: "this", arg: 1, scope: !26, type: !28, flags: DIFlagArtificial | DIFlagObjectPointer)
+!26 = distinct !DISubprogram(name: "IsNull", linkageName: "_ZN2S16IsNullEv", scope: !13, file: !1, line: 8, type: !18, scopeLine: 8, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, declaration: !17, retainedNodes: !27)
+!27 = !{!25}
+!28 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !13, size: 64)
+!29 = !DILocation(line: 0, scope: !26, inlinedAt: !30)
+!30 = distinct !DILocation(line: 19, scope: !31)
+!31 = distinct !DILexicalBlock(scope: !7, file: !1, line: 19)
+!32 = !DILocation(line: 9, scope: !26, inlinedAt: !30)
+!38 = !DILocation(line: 0, scope: !7)
diff --git a/llvm/test/Transforms/InstCombine/element-atomic-memintrins.ll b/llvm/test/Transforms/InstCombine/element-atomic-memintrins.ll
index 6bc62c94e..a49b4c7af 100644
--- a/llvm/test/Transforms/InstCombine/element-atomic-memintrins.ll
+++ b/llvm/test/Transforms/InstCombine/element-atomic-memintrins.ll
@@ -15,10 +15,10 @@ define void @test_memset_zero_length(i8* %dest) {
 define void @test_memset_to_store(i8* %dest) {
 ; CHECK-LABEL: @test_memset_to_store(
 ; CHECK-NEXT:    store atomic i8 1, i8* [[DEST:%.*]] unordered, align 1
-; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 1 [[DEST]], i8 1, i32 2, i32 1)
-; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 1 [[DEST]], i8 1, i32 4, i32 1)
-; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 1 [[DEST]], i8 1, i32 8, i32 1)
-; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 1 [[DEST]], i8 1, i32 16, i32 1)
+; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* nonnull align 1 [[DEST]], i8 1, i32 2, i32 1)
+; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* nonnull align 1 [[DEST]], i8 1, i32 4, i32 1)
+; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* nonnull align 1 [[DEST]], i8 1, i32 8, i32 1)
+; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* nonnull align 1 [[DEST]], i8 1, i32 16, i32 1)
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 1 %dest, i8 1, i32 1, i32 1)
@@ -34,9 +34,9 @@ define void @test_memset_to_store_2(i8* %dest) {
 ; CHECK-NEXT:    store atomic i8 1, i8* [[DEST:%.*]] unordered, align 2
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8* [[DEST]] to i16*
 ; CHECK-NEXT:    store atomic i16 257, i16* [[TMP1]] unordered, align 2
-; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 2 [[DEST]], i8 1, i32 4, i32 2)
-; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 2 [[DEST]], i8 1, i32 8, i32 2)
-; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 2 [[DEST]], i8 1, i32 16, i32 2)
+; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* nonnull align 2 [[DEST]], i8 1, i32 4, i32 2)
+; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* nonnull align 2 [[DEST]], i8 1, i32 8, i32 2)
+; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* nonnull align 2 [[DEST]], i8 1, i32 16, i32 2)
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 2 %dest, i8 1, i32 1, i32 1)
@@ -54,8 +54,8 @@ define void @test_memset_to_store_4(i8* %dest) {
 ; CHECK-NEXT:    store atomic i16 257, i16* [[TMP1]] unordered, align 4
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8* [[DEST]] to i32*
 ; CHECK-NEXT:    store atomic i32 16843009, i32* [[TMP2]] unordered, align 4
-; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 4 [[DEST]], i8 1, i32 8, i32 4)
-; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 4 [[DEST]], i8 1, i32 16, i32 4)
+; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* nonnull align 4 [[DEST]], i8 1, i32 8, i32 4)
+; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* nonnull align 4 [[DEST]], i8 1, i32 16, i32 4)
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 4 %dest, i8 1, i32 1, i32 1)
@@ -75,7 +75,7 @@ define void @test_memset_to_store_8(i8* %dest) {
 ; CHECK-NEXT:    store atomic i32 16843009, i32* [[TMP2]] unordered, align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8* [[DEST]] to i64*
 ; CHECK-NEXT:    store atomic i64 72340172838076673, i64* [[TMP3]] unordered, align 8
-; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 8 [[DEST]], i8 1, i32 16, i32 8)
+; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* nonnull align 8 [[DEST]], i8 1, i32 16, i32 8)
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 8 %dest, i8 1, i32 1, i32 1)
@@ -95,7 +95,7 @@ define void @test_memset_to_store_16(i8* %dest) {
 ; CHECK-NEXT:    store atomic i32 16843009, i32* [[TMP2]] unordered, align 16
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8* [[DEST]] to i64*
 ; CHECK-NEXT:    store atomic i64 72340172838076673, i64* [[TMP3]] unordered, align 16
-; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 16 [[DEST]], i8 1, i32 16, i32 16)
+; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* nonnull align 16 [[DEST]], i8 1, i32 16, i32 16)
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 16 %dest, i8 1, i32 1, i32 1)
@@ -154,10 +154,10 @@ define void @test_memmove_loadstore(i8* %dest, i8* %src) {
 ; CHECK-LABEL: @test_memmove_loadstore(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load atomic i8, i8* [[SRC:%.*]] unordered, align 1
 ; CHECK-NEXT:    store atomic i8 [[TMP1]], i8* [[DEST:%.*]] unordered, align 1
-; CHECK-NEXT:    call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 1 [[DEST]], i8* align 1 [[SRC]], i32 2, i32 1)
-; CHECK-NEXT:    call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 1 [[DEST]], i8* align 1 [[SRC]], i32 4, i32 1)
-; CHECK-NEXT:    call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 1 [[DEST]], i8* align 1 [[SRC]], i32 8, i32 1)
-; CHECK-NEXT:    call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 1 [[DEST]], i8* align 1 [[SRC]], i32 16, i32 1)
+; CHECK-NEXT:    call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* nonnull align 1 [[DEST]], i8* nonnull align 1 [[SRC]], i32 2, i32 1)
+; CHECK-NEXT:    call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* nonnull align 1 [[DEST]], i8* nonnull align 1 [[SRC]], i32 4, i32 1)
+; CHECK-NEXT:    call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* nonnull align 1 [[DEST]], i8* nonnull align 1 [[SRC]], i32 8, i32 1)
+; CHECK-NEXT:    call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* nonnull align 1 [[DEST]], i8* nonnull align 1 [[SRC]], i32 16, i32 1)
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 1 %dest, i8* align 1 %src, i32 1, i32 1)
@@ -176,9 +176,9 @@ define void @test_memmove_loadstore_2(i8* %dest, i8* %src) {
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8* [[DEST]] to i16*
 ; CHECK-NEXT:    [[TMP4:%.*]] = load atomic i16, i16* [[TMP2]] unordered, align 2
 ; CHECK-NEXT:    store atomic i16 [[TMP4]], i16* [[TMP3]] unordered, align 2
-; CHECK-NEXT:    call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 2 [[DEST]], i8* align 2 [[SRC]], i32 4, i32 2)
-; CHECK-NEXT:    call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 2 [[DEST]], i8* align 2 [[SRC]], i32 8, i32 2)
-; CHECK-NEXT:    call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 2 [[DEST]], i8* align 2 [[SRC]], i32 16, i32 2)
+; CHECK-NEXT:    call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* nonnull align 2 [[DEST]], i8* nonnull align 2 [[SRC]], i32 4, i32 2)
+; CHECK-NEXT:    call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* nonnull align 2 [[DEST]], i8* nonnull align 2 [[SRC]], i32 8, i32 2)
+; CHECK-NEXT:    call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* nonnull align 2 [[DEST]], i8* nonnull align 2 [[SRC]], i32 16, i32 2)
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 2 %dest, i8* align 2 %src, i32 1, i32 1)
@@ -201,8 +201,8 @@ define void @test_memmove_loadstore_4(i8* %dest, i8* %src) {
 ; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8* [[DEST]] to i32*
 ; CHECK-NEXT:    [[TMP7:%.*]] = load atomic i32, i32* [[TMP5]] unordered, align 4
 ; CHECK-NEXT:    store atomic i32 [[TMP7]], i32* [[TMP6]] unordered, align 4
-; CHECK-NEXT:    call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 4 [[DEST]], i8* align 4 [[SRC]], i32 8, i32 4)
-; CHECK-NEXT:    call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 4 [[DEST]], i8* align 4 [[SRC]], i32 16, i32 4)
+; CHECK-NEXT:    call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* nonnull align 4 [[DEST]], i8* nonnull align 4 [[SRC]], i32 8, i32 4)
+; CHECK-NEXT:    call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* nonnull align 4 [[DEST]], i8* nonnull align 4 [[SRC]], i32 16, i32 4)
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 4 %dest, i8* align 4 %src, i32 1, i32 1)
@@ -229,7 +229,7 @@ define void @test_memmove_loadstore_8(i8* %dest, i8* %src) {
 ; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i8* [[DEST]] to i64*
 ; CHECK-NEXT:    [[TMP10:%.*]] = load atomic i64, i64* [[TMP8]] unordered, align 8
 ; CHECK-NEXT:    store atomic i64 [[TMP10]], i64* [[TMP9]] unordered, align 8
-; CHECK-NEXT:    call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 8 [[DEST]], i8* align 8 [[SRC]], i32 16, i32 8)
+; CHECK-NEXT:    call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* nonnull align 8 [[DEST]], i8* nonnull align 8 [[SRC]], i32 16, i32 8)
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 8 %dest, i8* align 8 %src, i32 1, i32 1)
@@ -256,7 +256,7 @@ define void @test_memmove_loadstore_16(i8* %dest, i8* %src) {
 ; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i8* [[DEST]] to i64*
 ; CHECK-NEXT:    [[TMP10:%.*]] = load atomic i64, i64* [[TMP8]] unordered, align 16
 ; CHECK-NEXT:    store atomic i64 [[TMP10]], i64* [[TMP9]] unordered, align 16
-; CHECK-NEXT:    call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 16 [[DEST:%.*]], i8* align 16 [[SRC:%.*]], i32 16, i32 16)
+; CHECK-NEXT:    call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* nonnull align 16 [[DEST]], i8* nonnull align 16 [[SRC]], i32 16, i32 16)
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 16 %dest, i8* align 16 %src, i32 1, i32 1)
@@ -302,10 +302,10 @@ define void @test_memcpy_loadstore(i8* %dest, i8* %src) {
 ; CHECK-LABEL: @test_memcpy_loadstore(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load atomic i8, i8* [[SRC:%.*]] unordered, align 1
 ; CHECK-NEXT:    store atomic i8 [[TMP1]], i8* [[DEST:%.*]] unordered, align 1
-; CHECK-NEXT:    call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 1 [[DEST]], i8* align 1 [[SRC]], i32 2, i32 1)
-; CHECK-NEXT:    call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 1 [[DEST]], i8* align 1 [[SRC]], i32 4, i32 1)
-; CHECK-NEXT:    call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 1 [[DEST]], i8* align 1 [[SRC]], i32 8, i32 1)
-; CHECK-NEXT:    call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 1 [[DEST]], i8* align 1 [[SRC]], i32 16, i32 1)
+; CHECK-NEXT:    call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* nonnull align 1 [[DEST]], i8* nonnull align 1 [[SRC]], i32 2, i32 1)
+; CHECK-NEXT:    call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* nonnull align 1 [[DEST]], i8* nonnull align 1 [[SRC]], i32 4, i32 1)
+; CHECK-NEXT:    call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* nonnull align 1 [[DEST]], i8* nonnull align 1 [[SRC]], i32 8, i32 1)
+; CHECK-NEXT:    call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* nonnull align 1 [[DEST]], i8* nonnull align 1 [[SRC]], i32 16, i32 1)
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 1 %dest, i8* align 1 %src, i32 1, i32 1)
@@ -324,9 +324,9 @@ define void @test_memcpy_loadstore_2(i8* %dest, i8* %src) {
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8* [[DEST]] to i16*
 ; CHECK-NEXT:    [[TMP4:%.*]] = load atomic i16, i16* [[TMP2]] unordered, align 2
 ; CHECK-NEXT:    store atomic i16 [[TMP4]], i16* [[TMP3]] unordered, align 2
-; CHECK-NEXT:    call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 2 [[DEST]], i8* align 2 [[SRC]], i32 4, i32 2)
-; CHECK-NEXT:    call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 2 [[DEST]], i8* align 2 [[SRC]], i32 8, i32 2)
-; CHECK-NEXT:    call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 2 [[DEST]], i8* align 2 [[SRC]], i32 16, i32 2)
+; CHECK-NEXT:    call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* nonnull align 2 [[DEST]], i8* nonnull align 2 [[SRC]], i32 4, i32 2)
+; CHECK-NEXT:    call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* nonnull align 2 [[DEST]], i8* nonnull align 2 [[SRC]], i32 8, i32 2)
+; CHECK-NEXT:    call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* nonnull align 2 [[DEST]], i8* nonnull align 2 [[SRC]], i32 16, i32 2)
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 2 %dest, i8* align 2 %src, i32 1, i32 1)
@@ -349,8 +349,8 @@ define void @test_memcpy_loadstore_4(i8* %dest, i8* %src) {
 ; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8* [[DEST]] to i32*
 ; CHECK-NEXT:    [[TMP7:%.*]] = load atomic i32, i32* [[TMP5]] unordered, align 4
 ; CHECK-NEXT:    store atomic i32 [[TMP7]], i32* [[TMP6]] unordered, align 4
-; CHECK-NEXT:    call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 4 [[DEST]], i8* align 4 [[SRC]], i32 8, i32 4)
-; CHECK-NEXT:    call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 4 [[DEST]], i8* align 4 [[SRC]], i32 16, i32 4)
+; CHECK-NEXT:    call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* nonnull align 4 [[DEST]], i8* nonnull align 4 [[SRC]], i32 8, i32 4)
+; CHECK-NEXT:    call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* nonnull align 4 [[DEST]], i8* nonnull align 4 [[SRC]], i32 16, i32 4)
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 4 %dest, i8* align 4 %src, i32 1, i32 1)
@@ -377,7 +377,7 @@ define void @test_memcpy_loadstore_8(i8* %dest, i8* %src) {
 ; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i8* [[DEST]] to i64*
 ; CHECK-NEXT:    [[TMP10:%.*]] = load atomic i64, i64* [[TMP8]] unordered, align 8
 ; CHECK-NEXT:    store atomic i64 [[TMP10]], i64* [[TMP9]] unordered, align 8
-; CHECK-NEXT:    call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 8 [[DEST]], i8* align 8 [[SRC]], i32 16, i32 8)
+; CHECK-NEXT:    call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* nonnull align 8 [[DEST]], i8* nonnull align 8 [[SRC]], i32 16, i32 8)
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 8 %dest, i8* align 8 %src, i32 1, i32 1)
@@ -404,7 +404,7 @@ define void @test_memcpy_loadstore_16(i8* %dest, i8* %src) {
 ; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i8* [[DEST]] to i64*
 ; CHECK-NEXT:    [[TMP10:%.*]] = load atomic i64, i64* [[TMP8]] unordered, align 16
 ; CHECK-NEXT:    store atomic i64 [[TMP10]], i64* [[TMP9]] unordered, align 16
-; CHECK-NEXT:    call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 16 [[DEST:%.*]], i8* align 16 [[SRC:%.*]], i32 16, i32 16)
+; CHECK-NEXT:    call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* nonnull align 16 [[DEST]], i8* nonnull align 16 [[SRC]], i32 16, i32 16)
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 16 %dest, i8* align 16 %src, i32 1, i32 1)
diff --git a/llvm/test/Transforms/InstCombine/expensive-combines.ll b/llvm/test/Transforms/InstCombine/expensive-combines.ll
new file mode 100644
index 000000000..28acb773b
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/expensive-combines.ll
@@ -0,0 +1,28 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -instcombine < %s | FileCheck %s --check-prefix=DEFAULT
+; RUN: opt -S -instcombine -expensive-combines=1 < %s | FileCheck %s --check-prefix=EXPENSIVE-ON
+; RUN: opt -S -instcombine -expensive-combines=0 < %s | FileCheck %s --check-prefix=EXPENSIVE-OFF
+
+define void @test() {
+; DEFAULT-LABEL: @test(
+; DEFAULT-NEXT:    [[CALL:%.*]] = call i32 @passthru(i32 0)
+; DEFAULT-NEXT:    call void @sink(i32 0)
+; DEFAULT-NEXT:    ret void
+;
+; EXPENSIVE-ON-LABEL: @test(
+; EXPENSIVE-ON-NEXT:    [[CALL:%.*]] = call i32 @passthru(i32 0)
+; EXPENSIVE-ON-NEXT:    call void @sink(i32 0)
+; EXPENSIVE-ON-NEXT:    ret void
+;
+; EXPENSIVE-OFF-LABEL: @test(
+; EXPENSIVE-OFF-NEXT:    [[CALL:%.*]] = call i32 @passthru(i32 0)
+; EXPENSIVE-OFF-NEXT:    call void @sink(i32 [[CALL]])
+; EXPENSIVE-OFF-NEXT:    ret void
+;
+  %call = call i32 @passthru(i32 0)
+  call void @sink(i32 %call)
+  ret void
+}
+
+declare i32 @passthru(i32 returned)
+declare void @sink(i32)
diff --git a/llvm/test/Transforms/InstCombine/fabs-copysign.ll b/llvm/test/Transforms/InstCombine/fabs-copysign.ll
new file mode 100644
index 000000000..b95969ed5
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/fabs-copysign.ll
@@ -0,0 +1,119 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+declare double @llvm.fabs.f64(double)
+declare float @llvm.fabs.f32(float)
+declare <4 x double> @llvm.fabs.v4f64(<4 x double>)
+declare void @use(double)
+
+declare double @llvm.copysign.f64(double, double)
+declare float @llvm.copysign.f32(float, float)
+
+define double @fabs_copysign(double %x) {
+; CHECK-LABEL: @fabs_copysign(
+; CHECK-NEXT:    [[TMP1:%.*]] = call nnan ninf double @llvm.copysign.f64(double 1.000000e+00, double [[X:%.*]])
+; CHECK-NEXT:    ret double [[TMP1]]
+;
+  %f = tail call double @llvm.fabs.f64(double %x)
+  %div = fdiv nnan ninf double %x, %f
+  ret double %div
+}
+
+define double @fabs_copysign_commuted(double %x) {
+; CHECK-LABEL: @fabs_copysign_commuted(
+; CHECK-NEXT:    [[TMP1:%.*]] = call nnan ninf double @llvm.copysign.f64(double 1.000000e+00, double [[X:%.*]])
+; CHECK-NEXT:    ret double [[TMP1]]
+;
+  %f = tail call double @llvm.fabs.f64(double %x)
+  %div = fdiv nnan ninf double %f, %x
+  ret double %div
+}
+
+define <4 x double> @fabs_copysign_vec(<4 x double> %x) {
+; CHECK-LABEL: @fabs_copysign_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = call nnan ninf <4 x double> @llvm.copysign.v4f64(<4 x double> <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>, <4 x double> [[X:%.*]])
+; CHECK-NEXT:    ret <4 x double> [[TMP1]]
+;
+  %f = call <4 x double> @llvm.fabs.v4f64(<4 x double> %x)
+  %div = fdiv nnan ninf <4 x double> %x, %f
+  ret <4 x double> %div
+}
+
+define <4 x double> @fabs_copysign_vec_commuted(<4 x double> %x) {
+; CHECK-LABEL: @fabs_copysign_vec_commuted(
+; CHECK-NEXT:    [[TMP1:%.*]] = call nnan ninf <4 x double> @llvm.copysign.v4f64(<4 x double> <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>, <4 x double> [[X:%.*]])
+; CHECK-NEXT:    ret <4 x double> [[TMP1]]
+;
+  %f = call <4 x double> @llvm.fabs.v4f64(<4 x double> %x)
+  %div = fdiv nnan ninf <4 x double> %f, %x
+  ret <4 x double> %div
+}
+
+define float @fabs_copysignf(float %x) {
+; CHECK-LABEL: @fabs_copysignf(
+; CHECK-NEXT:    [[TMP1:%.*]] = call nnan ninf float @llvm.copysign.f32(float 1.000000e+00, float [[X:%.*]])
+; CHECK-NEXT:    ret float [[TMP1]]
+;
+  %f = tail call float @llvm.fabs.f32(float %x)
+  %div = fdiv nnan ninf float %x, %f
+  ret float %div
+}
+
+define double @fabs_copysign_use(double %x) {
+; CHECK-LABEL: @fabs_copysign_use(
+; CHECK-NEXT:    [[F:%.*]] = tail call double @llvm.fabs.f64(double [[X:%.*]])
+; CHECK-NEXT:    call void @use(double [[F]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call nnan ninf double @llvm.copysign.f64(double 1.000000e+00, double [[X]])
+; CHECK-NEXT:    ret double [[TMP1]]
+;
+  %f = tail call double @llvm.fabs.f64(double %x)
+  call void @use(double %f)
+  %div = fdiv nnan ninf double %x, %f
+  ret double %div
+}
+
+; Negative tests
+
+define double @fabs_copysign_mismatch(double %x, double %y) {
+; CHECK-LABEL: @fabs_copysign_mismatch(
+; CHECK-NEXT:    [[F:%.*]] = tail call double @llvm.fabs.f64(double [[Y:%.*]])
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv double [[X:%.*]], [[F]]
+; CHECK-NEXT:    ret double [[DIV]]
+;
+  %f = tail call double @llvm.fabs.f64(double %y)
+  %div = fdiv double %x, %f
+  ret double %div
+}
+
+define double @fabs_copysign_commuted_mismatch(double %x, double %y) {
+; CHECK-LABEL: @fabs_copysign_commuted_mismatch(
+; CHECK-NEXT:    [[F:%.*]] = tail call double @llvm.fabs.f64(double [[Y:%.*]])
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv double [[F]], [[X:%.*]]
+; CHECK-NEXT:    ret double [[DIV]]
+;
+  %f = tail call double @llvm.fabs.f64(double %y)
+  %div = fdiv double %f, %x
+  ret double %div
+}
+
+define double @fabs_copysign_no_nnan(double %x) {
+; CHECK-LABEL: @fabs_copysign_no_nnan(
+; CHECK-NEXT:    [[F:%.*]] = tail call double @llvm.fabs.f64(double [[X:%.*]])
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv ninf double [[X]], [[F]]
+; CHECK-NEXT:    ret double [[DIV]]
+;
+  %f = tail call double @llvm.fabs.f64(double %x)
+  %div = fdiv ninf double %x, %f
+  ret double %div
+}
+
+define double @fabs_copysign_no_ninf(double %x) {
+; CHECK-LABEL: @fabs_copysign_no_ninf(
+; CHECK-NEXT:    [[F:%.*]] = tail call double @llvm.fabs.f64(double [[X:%.*]])
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv nnan double [[X]], [[F]]
+; CHECK-NEXT:    ret double [[DIV]]
+;
+  %f = tail call double @llvm.fabs.f64(double %x)
+  %div = fdiv nnan double %x, %f
+  ret double %div
+}
diff --git a/llvm/test/Transforms/InstCombine/fadd-fsub-factor.ll b/llvm/test/Transforms/InstCombine/fadd-fsub-factor.ll
index 09104e53e..b5e1e8452 100644
--- a/llvm/test/Transforms/InstCombine/fadd-fsub-factor.ll
+++ b/llvm/test/Transforms/InstCombine/fadd-fsub-factor.ll
@@ -471,3 +471,165 @@ define float @fdiv_fsub_denorm(float %x) {
   ret float %r
 }
 
+define float @lerp_commute0(float %a, float %b, float %c) {
+; CHECK-LABEL: @lerp_commute0(
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub fast float [[B:%.*]], [[A:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fmul fast float [[TMP1]], [[C:%.*]]
+; CHECK-NEXT:    [[ADD:%.*]] = fadd fast float [[TMP2]], [[A]]
+; CHECK-NEXT:    ret float [[ADD]]
+;
+  %sub = fsub fast float 1.0, %c
+  %mul = fmul fast float %sub, %a
+  %bc = fmul fast float %c, %b
+  %add = fadd fast float %mul, %bc
+  ret float %add
+}
+
+define <2 x float> @lerp_commute1(<2 x float> %a, <2 x float> %b, <2 x float> %c) {
+; CHECK-LABEL: @lerp_commute1(
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub fast <2 x float> [[B:%.*]], [[A:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fmul fast <2 x float> [[TMP1]], [[C:%.*]]
+; CHECK-NEXT:    [[ADD:%.*]] = fadd fast <2 x float> [[TMP2]], [[A]]
+; CHECK-NEXT:    ret <2 x float> [[ADD]]
+;
+  %sub = fsub <2 x float> <float 1.0, float 1.0>, %c
+  %mul = fmul <2 x float> %sub, %a
+  %bc = fmul <2 x float> %c, %b
+  %add = fadd fast <2 x float> %bc, %mul
+  ret <2 x float> %add
+}
+
+define float @lerp_commute2(float %a, float %b, float %c) {
+; CHECK-LABEL: @lerp_commute2(
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub reassoc nsz float [[B:%.*]], [[A:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fmul reassoc nsz float [[TMP1]], [[C:%.*]]
+; CHECK-NEXT:    [[ADD:%.*]] = fadd reassoc nsz float [[TMP2]], [[A]]
+; CHECK-NEXT:    ret float [[ADD]]
+;
+  %sub = fsub float 1.0, %c
+  %mul = fmul float %sub, %a
+  %bc = fmul float %b, %c
+  %add = fadd reassoc nsz float %mul, %bc
+  ret float %add
+}
+
+define float @lerp_commute3(float %a, float %b, float %c) {
+; CHECK-LABEL: @lerp_commute3(
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub reassoc ninf nsz float [[B:%.*]], [[A:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fmul reassoc ninf nsz float [[TMP1]], [[C:%.*]]
+; CHECK-NEXT:    [[ADD:%.*]] = fadd reassoc ninf nsz float [[TMP2]], [[A]]
+; CHECK-NEXT:    ret float [[ADD]]
+;
+  %sub = fsub fast float 1.0, %c
+  %mul = fmul float %sub, %a
+  %bc = fmul float %b, %c
+  %add = fadd reassoc nsz ninf float %bc, %mul
+  ret float %add
+}
+
+define double @lerp_commute4(double %a, double %b, double %c) {
+; CHECK-LABEL: @lerp_commute4(
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub fast double [[B:%.*]], [[A:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fmul fast double [[TMP1]], [[C:%.*]]
+; CHECK-NEXT:    [[ADD:%.*]] = fadd fast double [[TMP2]], [[A]]
+; CHECK-NEXT:    ret double [[ADD]]
+;
+  %sub = fsub fast double 1.0, %c
+  %mul = fmul fast double %a, %sub
+  %bc = fmul fast double %c, %b
+  %add = fadd fast double %mul, %bc
+  ret double %add
+}
+
+define double @lerp_commute5(double %a, double %b, double %c) {
+; CHECK-LABEL: @lerp_commute5(
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub fast double [[B:%.*]], [[A:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fmul fast double [[TMP1]], [[C:%.*]]
+; CHECK-NEXT:    [[ADD:%.*]] = fadd fast double [[TMP2]], [[A]]
+; CHECK-NEXT:    ret double [[ADD]]
+;
+  %sub = fsub fast double 1.0, %c
+  %mul = fmul fast double %a, %sub
+  %bc = fmul fast double %c, %b
+  %add = fadd fast double %bc, %mul
+  ret double %add
+}
+
+define half @lerp_commute6(half %a, half %b, half %c) {
+; CHECK-LABEL: @lerp_commute6(
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub fast half [[B:%.*]], [[A:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fmul fast half [[TMP1]], [[C:%.*]]
+; CHECK-NEXT:    [[ADD:%.*]] = fadd fast half [[TMP2]], [[A]]
+; CHECK-NEXT:    ret half [[ADD]]
+;
+  %sub = fsub fast half 1.0, %c
+  %mul = fmul fast half %a, %sub
+  %bc = fmul fast half %b, %c
+  %add = fadd fast half %mul, %bc
+  ret half %add
+}
+
+define half @lerp_commute7(half %a, half %b, half %c) {
+; CHECK-LABEL: @lerp_commute7(
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub fast half [[B:%.*]], [[A:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fmul fast half [[TMP1]], [[C:%.*]]
+; CHECK-NEXT:    [[ADD:%.*]] = fadd fast half [[TMP2]], [[A]]
+; CHECK-NEXT:    ret half [[ADD]]
+;
+  %sub = fsub fast half 1.0, %c
+  %mul = fmul fast half %a, %sub
+  %bc = fmul fast half %b, %c
+  %add = fadd fast half %bc, %mul
+  ret half %add
+}
+
+define float @lerp_extra_use1(float %a, float %b, float %c) {
+; CHECK-LABEL: @lerp_extra_use1(
+; CHECK-NEXT:    [[SUB:%.*]] = fsub fast float 1.000000e+00, [[C:%.*]]
+; CHECK-NEXT:    [[MUL:%.*]] = fmul fast float [[SUB]], [[A:%.*]]
+; CHECK-NEXT:    [[BC:%.*]] = fmul fast float [[B:%.*]], [[C]]
+; CHECK-NEXT:    call void @use(float [[BC]])
+; CHECK-NEXT:    [[ADD:%.*]] = fadd fast float [[BC]], [[MUL]]
+; CHECK-NEXT:    ret float [[ADD]]
+;
+  %sub = fsub fast float 1.0, %c
+  %mul = fmul fast float %a, %sub
+  %bc = fmul fast float %b, %c
+  call void @use(float %bc)
+  %add = fadd fast float %bc, %mul
+  ret float %add
+}
+
+define float @lerp_extra_use2(float %a, float %b, float %c) {
+; CHECK-LABEL: @lerp_extra_use2(
+; CHECK-NEXT:    [[SUB:%.*]] = fsub fast float 1.000000e+00, [[C:%.*]]
+; CHECK-NEXT:    [[MUL:%.*]] = fmul fast float [[SUB]], [[A:%.*]]
+; CHECK-NEXT:    call void @use(float [[MUL]])
+; CHECK-NEXT:    [[BC:%.*]] = fmul fast float [[B:%.*]], [[C]]
+; CHECK-NEXT:    [[ADD:%.*]] = fadd fast float [[BC]], [[MUL]]
+; CHECK-NEXT:    ret float [[ADD]]
+;
+  %sub = fsub fast float 1.0, %c
+  %mul = fmul fast float %a, %sub
+  call void @use(float %mul)
+  %bc = fmul fast float %b, %c
+  %add = fadd fast float %bc, %mul
+  ret float %add
+}
+
+define float @lerp_extra_use3(float %a, float %b, float %c) {
+; CHECK-LABEL: @lerp_extra_use3(
+; CHECK-NEXT:    [[SUB:%.*]] = fsub fast float 1.000000e+00, [[C:%.*]]
+; CHECK-NEXT:    call void @use(float [[SUB]])
+; CHECK-NEXT:    [[MUL:%.*]] = fmul fast float [[SUB]], [[A:%.*]]
+; CHECK-NEXT:    [[BC:%.*]] = fmul fast float [[B:%.*]], [[C]]
+; CHECK-NEXT:    [[ADD:%.*]] = fadd fast float [[BC]], [[MUL]]
+; CHECK-NEXT:    ret float [[ADD]]
+;
+  %sub = fsub fast float 1.0, %c
+  call void @use(float %sub)
+  %mul = fmul fast float %a, %sub
+  %bc = fmul fast float %b, %c
+  %add = fadd fast float %bc, %mul
+  ret float %add
+}
diff --git a/llvm/test/Transforms/InstCombine/fadd.ll b/llvm/test/Transforms/InstCombine/fadd.ll
index f7eac8e14..24a22dc52 100644
--- a/llvm/test/Transforms/InstCombine/fadd.ll
+++ b/llvm/test/Transforms/InstCombine/fadd.ll
@@ -1,6 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -instcombine -S | FileCheck %s
 
+declare void @use(float)
+declare void @use_vec(<2 x float>)
+
 ; -x + y => y - x
 
 define float @fneg_op0(float %x, float %y) {
@@ -25,3 +28,362 @@ define float @fneg_op1(float %x, float %y) {
   ret float %add
 }
 
+; Z + (-X / Y) --> Z - (X / Y)
+
+define double @fdiv_fneg1(double %x, double %y, double %pz) {
+; CHECK-LABEL: @fdiv_fneg1(
+; CHECK-NEXT:    [[Z:%.*]] = frem double 4.200000e+01, [[PZ:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fdiv double [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = fsub double [[Z]], [[TMP1]]
+; CHECK-NEXT:    ret double [[R]]
+;
+  %z = frem double 42.0, %pz ; thwart complexity-based canonicalization
+  %neg = fsub double -0.000000e+00, %x
+  %div = fdiv double %neg, %y
+  %r = fadd double %z, %div
+  ret double %r
+}
+
+; Z + (Y / -X) --> Z - (Y / X)
+
+define <2 x double> @fdiv_fneg2(<2 x double> %x, <2 x double> %y, <2 x double> %pz) {
+; CHECK-LABEL: @fdiv_fneg2(
+; CHECK-NEXT:    [[Z:%.*]] = frem <2 x double> <double 4.200000e+01, double 8.000000e+00>, [[PZ:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fdiv <2 x double> [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = fsub <2 x double> [[Z]], [[TMP1]]
+; CHECK-NEXT:    ret <2 x double> [[R]]
+;
+  %z = frem <2 x double> <double 42.0, double 8.0>, %pz ; thwart complexity-based canonicalization
+  %neg = fsub <2 x double> <double -0.0, double -0.0>, %x
+  %div = fdiv <2 x double> %y, %neg
+  %r = fadd <2 x double> %z, %div
+  ret <2 x double> %r
+}
+
+; Z + (-X * Y) --> Z - (X * Y)
+
+define double @fmul_fneg1(double %x, double %y, double %pz) {
+; CHECK-LABEL: @fmul_fneg1(
+; CHECK-NEXT:    [[Z:%.*]] = frem double 4.200000e+01, [[PZ:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul double [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = fsub double [[Z]], [[TMP1]]
+; CHECK-NEXT:    ret double [[R]]
+;
+  %z = frem double 42.0, %pz ; thwart complexity-based canonicalization
+  %neg = fsub double -0.000000e+00, %x
+  %mul = fmul double %neg, %y
+  %r = fadd double %z, %mul
+  ret double %r
+}
+
+; Z + (Y * -X) --> Z - (Y * X)
+
+define double @fmul_fneg2(double %x, double %py, double %pz) {
+; CHECK-LABEL: @fmul_fneg2(
+; CHECK-NEXT:    [[Y:%.*]] = frem double -4.200000e+01, [[PY:%.*]]
+; CHECK-NEXT:    [[Z:%.*]] = frem double 4.200000e+01, [[PZ:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul double [[Y]], [[X:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = fsub double [[Z]], [[TMP1]]
+; CHECK-NEXT:    ret double [[R]]
+;
+  %y = frem double -42.0, %py ; thwart complexity-based canonicalization
+  %z = frem double 42.0, %pz ; thwart complexity-based canonicalization
+  %neg = fsub double -0.000000e+00, %x
+  %mul = fmul double %y, %neg
+  %r = fadd double %z, %mul
+  ret double %r
+}
+
+; (-X / Y) + Z --> Z - (X / Y)
+
+define double @fdiv_fneg1_commute(double %x, double %y, double %pz) {
+; CHECK-LABEL: @fdiv_fneg1_commute(
+; CHECK-NEXT:    [[Z:%.*]] = frem double 4.200000e+01, [[PZ:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fdiv double [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = fsub double [[Z]], [[TMP1]]
+; CHECK-NEXT:    ret double [[R]]
+;
+  %z = frem double 42.0, %pz ; thwart complexity-based canonicalization
+  %neg = fsub double -0.000000e+00, %x
+  %div = fdiv double %neg, %y
+  %r = fadd double %div, %z
+  ret double %r
+}
+
+; (Y / -X) + Z --> Z - (Y / X)
+
+define <2 x double> @fdiv_fneg2_commute(<2 x double> %x, <2 x double> %y, <2 x double> %pz) {
+; CHECK-LABEL: @fdiv_fneg2_commute(
+; CHECK-NEXT:    [[Z:%.*]] = frem <2 x double> <double 4.200000e+01, double 8.000000e+00>, [[PZ:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fdiv <2 x double> [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = fsub <2 x double> [[Z]], [[TMP1]]
+; CHECK-NEXT:    ret <2 x double> [[R]]
+;
+  %z = frem <2 x double> <double 42.0, double 8.0>, %pz ; thwart complexity-based canonicalization
+  %neg = fsub <2 x double> <double -0.0, double -0.0>, %x
+  %div = fdiv <2 x double> %y, %neg
+  %r = fadd <2 x double> %div, %z
+  ret <2 x double> %r
+}
+
+; (-X * Y) + Z --> Z - (X * Y)
+
+define double @fmul_fneg1_commute(double %x, double %y, double %pz) {
+; CHECK-LABEL: @fmul_fneg1_commute(
+; CHECK-NEXT:    [[Z:%.*]] = frem double 4.200000e+01, [[PZ:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul double [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = fsub double [[Z]], [[TMP1]]
+; CHECK-NEXT:    ret double [[R]]
+;
+  %z = frem double 42.0, %pz ; thwart complexity-based canonicalization
+  %neg = fsub double -0.000000e+00, %x
+  %mul = fmul double %neg, %y
+  %r = fadd double %mul, %z
+  ret double %r
+}
+
+; (Y * -X) + Z --> Z - (Y * X)
+
+define double @fmul_fneg2_commute(double %x, double %py, double %pz) {
+; CHECK-LABEL: @fmul_fneg2_commute(
+; CHECK-NEXT:    [[Y:%.*]] = frem double 4.100000e+01, [[PY:%.*]]
+; CHECK-NEXT:    [[Z:%.*]] = frem double 4.200000e+01, [[PZ:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul double [[Y]], [[X:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = fsub double [[Z]], [[TMP1]]
+; CHECK-NEXT:    ret double [[R]]
+;
+  %y = frem double 41.0, %py ; thwart complexity-based canonicalization
+  %z = frem double 42.0, %pz ; thwart complexity-based canonicalization
+  %neg = fsub double -0.000000e+00, %x
+  %mul = fmul double %y, %neg
+  %r = fadd double %mul, %z
+  ret double %r
+}
+
+; Z + (-X / Y) - extra use means we can't transform to fsub without an extra instruction
+
+define float @fdiv_fneg1_extra_use(float %x, float %y, float %pz) {
+; CHECK-LABEL: @fdiv_fneg1_extra_use(
+; CHECK-NEXT:    [[Z:%.*]] = frem float 4.200000e+01, [[PZ:%.*]]
+; CHECK-NEXT:    [[NEG:%.*]] = fsub float -0.000000e+00, [[X:%.*]]
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv float [[NEG]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use(float [[DIV]])
+; CHECK-NEXT:    [[R:%.*]] = fadd float [[Z]], [[DIV]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %z = frem float 42.0, %pz ; thwart complexity-based canonicalization
+  %neg = fsub float -0.000000e+00, %x
+  %div = fdiv float %neg, %y
+  call void @use(float %div)
+  %r = fadd float %z, %div
+  ret float %r
+}
+
+; Z + (Y / -X) - extra use means we can't transform to fsub without an extra instruction
+
+define float @fdiv_fneg2_extra_use(float %x, float %py, float %pz) {
+; CHECK-LABEL: @fdiv_fneg2_extra_use(
+; CHECK-NEXT:    [[Y:%.*]] = frem float -4.200000e+01, [[PY:%.*]]
+; CHECK-NEXT:    [[Z:%.*]] = frem float 4.200000e+01, [[PZ:%.*]]
+; CHECK-NEXT:    [[NEG:%.*]] = fsub float -0.000000e+00, [[X:%.*]]
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv float [[Y]], [[NEG]]
+; CHECK-NEXT:    call void @use(float [[DIV]])
+; CHECK-NEXT:    [[R:%.*]] = fadd float [[Z]], [[DIV]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %y = frem float -42.0, %py ; thwart complexity-based canonicalization
+  %z = frem float 42.0, %pz ; thwart complexity-based canonicalization
+  %neg = fsub float -0.000000e+00, %x
+  %div = fdiv float %y, %neg
+  call void @use(float %div)
+  %r = fadd float %z, %div
+  ret float %r
+}
+
+; Z + (-X * Y) - extra use means we can't transform to fsub without an extra instruction
+
+define <2 x float> @fmul_fneg1_extra_use(<2 x float> %x, <2 x float> %y, <2 x float> %pz) {
+; CHECK-LABEL: @fmul_fneg1_extra_use(
+; CHECK-NEXT:    [[Z:%.*]] = frem <2 x float> <float 4.200000e+01, float -1.000000e+00>, [[PZ:%.*]]
+; CHECK-NEXT:    [[NEG:%.*]] = fsub <2 x float> <float -0.000000e+00, float -0.000000e+00>, [[X:%.*]]
+; CHECK-NEXT:    [[MUL:%.*]] = fmul <2 x float> [[NEG]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use_vec(<2 x float> [[MUL]])
+; CHECK-NEXT:    [[R:%.*]] = fadd <2 x float> [[Z]], [[MUL]]
+; CHECK-NEXT:    ret <2 x float> [[R]]
+;
+  %z = frem <2 x float> <float 42.0, float -1.0>, %pz ; thwart complexity-based canonicalization
+  %neg = fsub <2 x float> <float -0.0, float -0.0>, %x
+  %mul = fmul <2 x float> %neg, %y
+  call void @use_vec(<2 x float> %mul)
+  %r = fadd <2 x float> %z, %mul
+  ret <2 x float> %r
+}
+
+; Z + (Y * -X) - extra use means we can't transform to fsub without an extra instruction
+
+define float @fmul_fneg2_extra_use(float %x, float %py, float %pz) {
+; CHECK-LABEL: @fmul_fneg2_extra_use(
+; CHECK-NEXT:    [[Y:%.*]] = frem float -4.200000e+01, [[PY:%.*]]
+; CHECK-NEXT:    [[Z:%.*]] = frem float 4.200000e+01, [[PZ:%.*]]
+; CHECK-NEXT:    [[NEG:%.*]] = fsub float -0.000000e+00, [[X:%.*]]
+; CHECK-NEXT:    [[MUL:%.*]] = fmul float [[Y]], [[NEG]]
+; CHECK-NEXT:    call void @use(float [[MUL]])
+; CHECK-NEXT:    [[R:%.*]] = fadd float [[Z]], [[MUL]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %y = frem float -42.0, %py ; thwart complexity-based canonicalization
+  %z = frem float 42.0, %pz ; thwart complexity-based canonicalization
+  %neg = fsub float -0.000000e+00, %x
+  %mul = fmul float %y, %neg
+  call void @use(float %mul)
+  %r = fadd float %z, %mul
+  ret float %r
+}
+
+; (-X / Y) + Z --> Z - (X / Y)
+
+define float @fdiv_fneg1_extra_use2(float %x, float %y, float %z) {
+; CHECK-LABEL: @fdiv_fneg1_extra_use2(
+; CHECK-NEXT:    [[NEG:%.*]] = fsub float -0.000000e+00, [[X:%.*]]
+; CHECK-NEXT:    call void @use(float [[NEG]])
+; CHECK-NEXT:    [[TMP1:%.*]] = fdiv float [[X]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = fsub float [[Z:%.*]], [[TMP1]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %neg = fsub float -0.000000e+00, %x
+  call void @use(float %neg)
+  %div = fdiv float %neg, %y
+  %r = fadd float %div, %z
+  ret float %r
+}
+
+; (Y / -X) + Z --> Z - (Y / X)
+
+define float @fdiv_fneg2_extra_use2(float %x, float %y, float %z) {
+; CHECK-LABEL: @fdiv_fneg2_extra_use2(
+; CHECK-NEXT:    [[NEG:%.*]] = fsub float -0.000000e+00, [[X:%.*]]
+; CHECK-NEXT:    call void @use(float [[NEG]])
+; CHECK-NEXT:    [[TMP1:%.*]] = fdiv float [[Y:%.*]], [[X]]
+; CHECK-NEXT:    [[R:%.*]] = fsub float [[Z:%.*]], [[TMP1]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %neg = fsub float -0.000000e+00, %x
+  call void @use(float %neg)
+  %div = fdiv float %y, %neg
+  %r = fadd float %div, %z
+  ret float %r
+}
+
+; (-X * Y) + Z --> Z - (X * Y)
+
+define <2 x float> @fmul_fneg1_extra_use2(<2 x float> %x, <2 x float> %y, <2 x float> %z) {
+; CHECK-LABEL: @fmul_fneg1_extra_use2(
+; CHECK-NEXT:    [[NEG:%.*]] = fsub <2 x float> <float -0.000000e+00, float -0.000000e+00>, [[X:%.*]]
+; CHECK-NEXT:    call void @use_vec(<2 x float> [[NEG]])
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul <2 x float> [[X]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = fsub <2 x float> [[Z:%.*]], [[TMP1]]
+; CHECK-NEXT:    ret <2 x float> [[R]]
+;
+  %neg = fsub <2 x float> <float -0.0, float -0.0>, %x
+  call void @use_vec(<2 x float> %neg)
+  %mul = fmul <2 x float> %neg, %y
+  %r = fadd <2 x float> %mul, %z
+  ret <2 x float> %r
+}
+
+; (Y * -X) + Z --> Z - (Y * X)
+
+define float @fmul_fneg2_extra_use2(float %x, float %py, float %z) {
+; CHECK-LABEL: @fmul_fneg2_extra_use2(
+; CHECK-NEXT:    [[Y:%.*]] = frem float -4.200000e+01, [[PY:%.*]]
+; CHECK-NEXT:    [[NEG:%.*]] = fsub float -0.000000e+00, [[X:%.*]]
+; CHECK-NEXT:    call void @use(float [[NEG]])
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul float [[Y]], [[X]]
+; CHECK-NEXT:    [[R:%.*]] = fsub float [[Z:%.*]], [[TMP1]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %y = frem float -42.0, %py ; thwart complexity-based canonicalization
+  %neg = fsub float -0.000000e+00, %x
+  call void @use(float %neg)
+  %mul = fmul float %y, %neg
+  %r = fadd float %mul, %z
+  ret float %r
+}
+
+; (-X / Y) + Z --> Z - (X / Y)
+
+define float @fdiv_fneg1_extra_use3(float %x, float %y, float %z) {
+; CHECK-LABEL: @fdiv_fneg1_extra_use3(
+; CHECK-NEXT:    [[NEG:%.*]] = fsub float -0.000000e+00, [[X:%.*]]
+; CHECK-NEXT:    call void @use(float [[NEG]])
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv float [[NEG]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use(float [[DIV]])
+; CHECK-NEXT:    [[R:%.*]] = fadd float [[DIV]], [[Z:%.*]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %neg = fsub float -0.000000e+00, %x
+  call void @use(float %neg)
+  %div = fdiv float %neg, %y
+  call void @use(float %div)
+  %r = fadd float %div, %z
+  ret float %r
+}
+
+; (Y / -X) + Z --> Z - (Y / X)
+
+define float @fdiv_fneg2_extra_use3(float %x, float %y, float %z) {
+; CHECK-LABEL: @fdiv_fneg2_extra_use3(
+; CHECK-NEXT:    [[NEG:%.*]] = fsub float -0.000000e+00, [[X:%.*]]
+; CHECK-NEXT:    call void @use(float [[NEG]])
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv float [[Y:%.*]], [[NEG]]
+; CHECK-NEXT:    call void @use(float [[DIV]])
+; CHECK-NEXT:    [[R:%.*]] = fadd float [[DIV]], [[Z:%.*]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %neg = fsub float -0.000000e+00, %x
+  call void @use(float %neg)
+  %div = fdiv float %y, %neg
+  call void @use(float %div)
+  %r = fadd float %div, %z
+  ret float %r
+}
+
+; (-X * Y) + Z --> Z - (X * Y)
+
+define <2 x float> @fmul_fneg1_extra_use3(<2 x float> %x, <2 x float> %y, <2 x float> %z) {
+; CHECK-LABEL: @fmul_fneg1_extra_use3(
+; CHECK-NEXT:    [[NEG:%.*]] = fsub <2 x float> <float -0.000000e+00, float -0.000000e+00>, [[X:%.*]]
+; CHECK-NEXT:    call void @use_vec(<2 x float> [[NEG]])
+; CHECK-NEXT:    [[MUL:%.*]] = fmul <2 x float> [[NEG]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use_vec(<2 x float> [[MUL]])
+; CHECK-NEXT:    [[R:%.*]] = fadd <2 x float> [[MUL]], [[Z:%.*]]
+; CHECK-NEXT:    ret <2 x float> [[R]]
+;
+  %neg = fsub <2 x float> <float -0.0, float -0.0>, %x
+  call void @use_vec(<2 x float> %neg)
+  %mul = fmul <2 x float> %neg, %y
+  call void @use_vec(<2 x float> %mul)
+  %r = fadd <2 x float> %mul, %z
+  ret <2 x float> %r
+}
+
+; (Y * -X) + Z --> Z - (Y * X)
+
+define float @fmul_fneg2_extra_use3(float %x, float %py, float %z) {
+; CHECK-LABEL: @fmul_fneg2_extra_use3(
+; CHECK-NEXT:    [[Y:%.*]] = frem float -4.200000e+01, [[PY:%.*]]
+; CHECK-NEXT:    [[NEG:%.*]] = fsub float -0.000000e+00, [[X:%.*]]
+; CHECK-NEXT:    call void @use(float [[NEG]])
+; CHECK-NEXT:    [[MUL:%.*]] = fmul float [[Y]], [[NEG]]
+; CHECK-NEXT:    call void @use(float [[MUL]])
+; CHECK-NEXT:    [[R:%.*]] = fadd float [[MUL]], [[Z:%.*]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %y = frem float -42.0, %py ; thwart complexity-based canonicalization
+  %neg = fsub float -0.000000e+00, %x
+  call void @use(float %neg)
+  %mul = fmul float %y, %neg
+  call void @use(float %mul)
+  %r = fadd float %mul, %z
+  ret float %r
+}
diff --git a/llvm/test/Transforms/InstCombine/fast-math.ll b/llvm/test/Transforms/InstCombine/fast-math.ll
index 9563c377d..41c14aad9 100644
--- a/llvm/test/Transforms/InstCombine/fast-math.ll
+++ b/llvm/test/Transforms/InstCombine/fast-math.ll
@@ -504,7 +504,7 @@ define float @fsub_op0_fmul_const_wrong_FMF(float %x) {
 define float @fold16(float %x, float %y) {
 ; CHECK-LABEL: @fold16(
 ; CHECK-NEXT:    [[CMP:%.*]] = fcmp ogt float [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[TMP1:%.*]] = fsub float -0.000000e+00, [[Y]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fneg float [[Y]]
 ; CHECK-NEXT:    [[R_P:%.*]] = select i1 [[CMP]], float [[Y]], float [[TMP1]]
 ; CHECK-NEXT:    [[R:%.*]] = fadd float [[R_P]], [[X]]
 ; CHECK-NEXT:    ret float [[R]]
diff --git a/llvm/test/Transforms/InstCombine/fdiv.ll b/llvm/test/Transforms/InstCombine/fdiv.ll
index 619554e62..ec1119cb2 100644
--- a/llvm/test/Transforms/InstCombine/fdiv.ll
+++ b/llvm/test/Transforms/InstCombine/fdiv.ll
@@ -187,6 +187,21 @@ define float @div_with_div_denominator_extra_use(float %x, float %y, float %z) {
   ret float %div2
 }
 
+; Z / (1.0 / Y) ==> Y * Z
+
+define float @div_with_div_denominator_with_one_as_numerator_extra_use(float %x, float %y, float %z) {
+; CHECK-LABEL: @div_with_div_denominator_with_one_as_numerator_extra_use(
+; CHECK-NEXT:    [[DIV1:%.*]] = fdiv float 1.000000e+00, [[Y:%.*]]
+; CHECK-NEXT:    [[DIV2:%.*]] = fmul reassoc arcp float [[Y]], [[Z:%.*]]
+; CHECK-NEXT:    call void @use_f32(float [[DIV1]])
+; CHECK-NEXT:    ret float [[DIV2]]
+;
+  %div1 = fdiv float 1.0, %y
+  %div2 = fdiv reassoc arcp float %z, %div1
+  call void @use_f32(float %div1)
+  ret float %div2
+}
+
 define float @fneg_fneg(float %x, float %y) {
 ; CHECK-LABEL: @fneg_fneg(
 ; CHECK-NEXT:    [[DIV:%.*]] = fdiv float [[X:%.*]], [[Y:%.*]]
@@ -499,3 +514,59 @@ define <2 x float> @div_constant_dividend3(<2 x float> %x) {
   ret <2 x float> %t2
 }
 
+define double @fdiv_fneg1(double %x, double %y) {
+; CHECK-LABEL: @fdiv_fneg1(
+; CHECK-NEXT:    [[NEG:%.*]] = fsub double -0.000000e+00, [[X:%.*]]
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv double [[NEG]], [[Y:%.*]]
+; CHECK-NEXT:    ret double [[DIV]]
+;
+  %neg = fsub double -0.0, %x
+  %div = fdiv double %neg, %y
+  ret double %div
+}
+
+define double @fdiv_unary_fneg1(double %x, double %y) {
+; CHECK-LABEL: @fdiv_unary_fneg1(
+; CHECK-NEXT:    [[NEG:%.*]] = fneg double [[X:%.*]]
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv double [[NEG]], [[Y:%.*]]
+; CHECK-NEXT:    ret double [[DIV]]
+;
+  %neg = fneg double %x
+  %div = fdiv double %neg, %y
+  ret double %div
+}
+
+define <2 x float> @fdiv_fneg2(<2 x float> %x, <2 x float> %y) {
+; CHECK-LABEL: @fdiv_fneg2(
+; CHECK-NEXT:    [[NEG:%.*]] = fsub <2 x float> <float -0.000000e+00, float -0.000000e+00>, [[X:%.*]]
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv <2 x float> [[Y:%.*]], [[NEG]]
+; CHECK-NEXT:    ret <2 x float> [[DIV]]
+;
+  %neg = fsub <2 x float> <float -0.0, float -0.0>, %x
+  %div = fdiv <2 x float> %y, %neg
+  ret <2 x float> %div
+}
+
+define <2 x float> @fdiv_unary_fneg2(<2 x float> %x, <2 x float> %y) {
+; CHECK-LABEL: @fdiv_unary_fneg2(
+; CHECK-NEXT:    [[NEG:%.*]] = fneg <2 x float> [[X:%.*]]
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv <2 x float> [[Y:%.*]], [[NEG]]
+; CHECK-NEXT:    ret <2 x float> [[DIV]]
+;
+  %neg = fneg <2 x float> %x
+  %div = fdiv <2 x float> %y, %neg
+  ret <2 x float> %div
+}
+
+define float @fdiv_fneg1_extra_use(float %x, float %y) {
+; CHECK-LABEL: @fdiv_fneg1_extra_use(
+; CHECK-NEXT:    [[NEG:%.*]] = fsub float -0.000000e+00, [[X:%.*]]
+; CHECK-NEXT:    call void @use_f32(float [[NEG]])
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv float [[NEG]], [[Y:%.*]]
+; CHECK-NEXT:    ret float [[DIV]]
+;
+  %neg = fsub float -0.0, %x
+  call void @use_f32(float %neg)
+  %div = fdiv float %neg, %y
+  ret float %div
+}
diff --git a/llvm/test/Transforms/InstCombine/fma.ll b/llvm/test/Transforms/InstCombine/fma.ll
index f3314c622..f123f5d56 100644
--- a/llvm/test/Transforms/InstCombine/fma.ll
+++ b/llvm/test/Transforms/InstCombine/fma.ll
@@ -182,7 +182,7 @@ define float @fmuladd_unary_fneg_x_unary_fneg_y(float %x, float %y, float %z) {
 
 define float @fmuladd_fneg_x_fneg_y_fast(float %x, float %y, float %z) {
 ; CHECK-LABEL: @fmuladd_fneg_x_fneg_y_fast(
-; CHECK-NEXT:    [[TMP1:%.*]] = fmul fast float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul fast float [[Y:%.*]], [[X:%.*]]
 ; CHECK-NEXT:    [[FMULADD:%.*]] = fadd fast float [[TMP1]], [[Z:%.*]]
 ; CHECK-NEXT:    ret float [[FMULADD]]
 ;
@@ -194,7 +194,7 @@ define float @fmuladd_fneg_x_fneg_y_fast(float %x, float %y, float %z) {
 
 define float @fmuladd_unary_fneg_x_unary_fneg_y_fast(float %x, float %y, float %z) {
 ; CHECK-LABEL: @fmuladd_unary_fneg_x_unary_fneg_y_fast(
-; CHECK-NEXT:    [[TMP1:%.*]] = fmul fast float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul fast float [[Y:%.*]], [[X:%.*]]
 ; CHECK-NEXT:    [[FMULADD:%.*]] = fadd fast float [[TMP1]], [[Z:%.*]]
 ; CHECK-NEXT:    ret float [[FMULADD]]
 ;
@@ -369,5 +369,352 @@ define float @fmuladd_x_1_z_fast(float %x, float %z) {
   ret float %fmuladd
 }
 
+define <2 x double> @fmuladd_a_0_b(<2 x double> %a, <2 x double> %b) {
+; CHECK-LABEL: @fmuladd_a_0_b(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <2 x double> [[B:%.*]]
+;
+entry:
+  %res = call nnan nsz <2 x double> @llvm.fmuladd.v2f64(<2 x double> %a, <2 x double> zeroinitializer, <2 x double> %b)
+  ret <2 x double> %res
+}
+
+define <2 x double> @fmuladd_0_a_b(<2 x double> %a, <2 x double> %b) {
+; CHECK-LABEL: @fmuladd_0_a_b(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <2 x double> [[B:%.*]]
+;
+entry:
+  %res = call nnan nsz <2 x double> @llvm.fmuladd.v2f64(<2 x double> zeroinitializer, <2 x double> %a, <2 x double> %b)
+  ret <2 x double> %res
+}
+
+define <2 x double> @fmuladd_a_0_b_missing_flags(<2 x double> %a, <2 x double> %b) {
+; CHECK-LABEL: @fmuladd_a_0_b_missing_flags(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[RES:%.*]] = call nnan <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[A:%.*]], <2 x double> zeroinitializer, <2 x double> [[B:%.*]])
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+entry:
+  %res = call nnan <2 x double> @llvm.fmuladd.v2f64(<2 x double> %a, <2 x double> zeroinitializer, <2 x double> %b)
+  ret <2 x double> %res
+}
+
+declare <2 x double> @llvm.fmuladd.v2f64(<2 x double>, <2 x double>, <2 x double>)
+
+define <2 x double> @fma_a_0_b(<2 x double> %a, <2 x double> %b) {
+; CHECK-LABEL: @fma_a_0_b(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <2 x double> [[B:%.*]]
+;
+entry:
+  %res = call nnan nsz <2 x double> @llvm.fma.v2f64(<2 x double> %a, <2 x double> zeroinitializer, <2 x double> %b)
+  ret <2 x double> %res
+}
+
+define <2 x double> @fma_0_a_b(<2 x double> %a, <2 x double> %b) {
+; CHECK-LABEL: @fma_0_a_b(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <2 x double> [[B:%.*]]
+;
+entry:
+  %res = call nnan nsz <2 x double> @llvm.fma.v2f64(<2 x double> zeroinitializer, <2 x double> %a, <2 x double> %b)
+  ret <2 x double> %res
+}
+
+define <2 x double> @fma_0_a_b_missing_flags(<2 x double> %a, <2 x double> %b) {
+; CHECK-LABEL: @fma_0_a_b_missing_flags(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[RES:%.*]] = call nsz <2 x double> @llvm.fma.v2f64(<2 x double> [[A:%.*]], <2 x double> zeroinitializer, <2 x double> [[B:%.*]])
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+entry:
+  %res = call nsz <2 x double> @llvm.fma.v2f64(<2 x double> zeroinitializer, <2 x double> %a, <2 x double> %b)
+  ret <2 x double> %res
+}
+
+define <2 x double> @fma_sqrt(<2 x double> %a, <2 x double> %b) {
+; CHECK-LABEL: @fma_sqrt(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[RES:%.*]] = fadd fast <2 x double> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+entry:
+  %sqrt = call fast <2 x double> @llvm.sqrt.v2f64(<2 x double> %a)
+  %res = call fast <2 x double> @llvm.fma.v2f64(<2 x double> %sqrt, <2 x double> %sqrt, <2 x double> %b)
+  ret <2 x double> %res
+}
+
+; We do not fold constant multiplies in FMAs, as they could require rounding, unless either constant is 0.0 or 1.0.
+define <2 x double> @fma_const_fmul(<2 x double> %b) {
+; CHECK-LABEL: @fma_const_fmul(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[RES:%.*]] = call nnan nsz <2 x double> @llvm.fma.v2f64(<2 x double> <double 0x4131233302898702, double 0x40C387800000D6C0>, <2 x double> <double 1.291820e-08, double 9.123000e-06>, <2 x double> [[B:%.*]])
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+entry:
+  %res = call nnan nsz <2 x double> @llvm.fma.v2f64(<2 x double> <double 1123123.0099110012314, double 9999.0000001>, <2 x double> <double 0.0000000129182, double 0.000009123>, <2 x double> %b)
+  ret <2 x double> %res
+}
+
+define <2 x double> @fma_const_fmul_zero(<2 x double> %b) {
+; CHECK-LABEL: @fma_const_fmul_zero(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <2 x double> [[B:%.*]]
+;
+entry:
+  %res = call nnan nsz <2 x double> @llvm.fma.v2f64(<2 x double> <double 0.0, double 0.0>, <2 x double> <double 1123123.0099110012314, double 9999.0000001>, <2 x double> %b)
+  ret <2 x double> %res
+}
+
+define <2 x double> @fma_const_fmul_zero2(<2 x double> %b) {
+; CHECK-LABEL: @fma_const_fmul_zero2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <2 x double> [[B:%.*]]
+;
+entry:
+  %res = call nnan nsz <2 x double> @llvm.fma.v2f64(<2 x double> <double 1123123.0099110012314, double 9999.0000001>, <2 x double> <double 0.0, double 0.0>, <2 x double> %b)
+  ret <2 x double> %res
+}
+
+define <2 x double> @fma_const_fmul_one(<2 x double> %b) {
+; CHECK-LABEL: @fma_const_fmul_one(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[RES:%.*]] = fadd nnan nsz <2 x double> [[B:%.*]], <double 0x4131233302898702, double 0x40C387800000D6C0>
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+entry:
+  %res = call nnan nsz <2 x double> @llvm.fma.v2f64(<2 x double> <double 1.0, double 1.0>, <2 x double> <double 1123123.0099110012314, double 9999.0000001>, <2 x double> %b)
+  ret <2 x double> %res
+}
+
+define <2 x double> @fma_const_fmul_one2(<2 x double> %b) {
+; CHECK-LABEL: @fma_const_fmul_one2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[RES:%.*]] = fadd nnan nsz <2 x double> [[B:%.*]], <double 0x4131233302898702, double 0x40C387800000D6C0>
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+entry:
+  %res = call nnan nsz <2 x double> @llvm.fma.v2f64(<2 x double> <double 1123123.0099110012314, double 9999.0000001>, <2 x double> <double 1.0, double 1.0>, <2 x double> %b)
+  ret <2 x double> %res
+}
+
+define <2 x double> @fma_nan_and_const_0(<2 x double> %b) {
+; CHECK-LABEL: @fma_nan_and_const_0(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <2 x double> <double 0x7FF8000000000000, double 0x7FF8000000000000>
+;
+entry:
+  %res = call nnan nsz <2 x double> @llvm.fma.v2f64(<2 x double> <double 0x7FF8000000000000, double 0x7FF8000000000000>, <2 x double> <double 0.0000000129182, double 0.000009123>, <2 x double> %b)
+  ret <2 x double> %res
+}
+
+define <2 x double> @fma_nan_and_const_1(<2 x double> %b) {
+; CHECK-LABEL: @fma_nan_and_const_1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <2 x double> <double 0x7FF8000000000000, double 0x7FF8000000000000>
+;
+entry:
+  %res = call nnan nsz <2 x double> @llvm.fma.v2f64(<2 x double> <double 0.0000000129182, double 0.000009123>, <2 x double> <double 0x7FF8000000000000, double 0x7FF8000000000000>, <2 x double> %b)
+  ret <2 x double> %res
+}
+
+define <2 x double> @fma_nan_and_const_2(<2 x double> %b) {
+; CHECK-LABEL: @fma_nan_and_const_2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <2 x double> <double 0x7FF8000000000000, double 0x7FF8000000000000>
+;
+entry:
+  %res = call nnan nsz <2 x double> @llvm.fma.v2f64(<2 x double> <double 0.0000000129182, double 0.000009123>, <2 x double> %b, <2 x double> <double 0x7FF8000000000000, double 0x7FF8000000000000>)
+  ret <2 x double> %res
+}
+
+define <2 x double> @fma_undef_0(<2 x double> %b, <2 x double> %c) {
+; CHECK-LABEL: @fma_undef_0(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <2 x double> <double 0x7FF8000000000000, double 0x7FF8000000000000>
+;
+entry:
+  %res = call nnan nsz <2 x double> @llvm.fma.v2f64(<2 x double> <double undef, double undef>, <2 x double> %b, <2 x double> %c)
+  ret <2 x double> %res
+}
+
+define <2 x double> @fma_undef_1(<2 x double> %b, <2 x double> %c) {
+; CHECK-LABEL: @fma_undef_1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <2 x double> <double 0x7FF8000000000000, double 0x7FF8000000000000>
+;
+entry:
+  %res = call nnan nsz <2 x double> @llvm.fma.v2f64(<2 x double> %b, <2 x double> <double undef, double undef>, <2 x double> %c)
+  ret <2 x double> %res
+}
+
+define <2 x double> @fma_undef_2(<2 x double> %b, <2 x double> %c) {
+; CHECK-LABEL: @fma_undef_2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <2 x double> <double 0x7FF8000000000000, double 0x7FF8000000000000>
+;
+entry:
+  %res = call nnan nsz <2 x double> @llvm.fma.v2f64(<2 x double> %b, <2 x double> %c, <2 x double> <double undef, double undef>)
+  ret <2 x double> %res
+}
+
+define <2 x double> @fma_partial_undef_0(<2 x double> %b, <2 x double> %c) {
+; CHECK-LABEL: @fma_partial_undef_0(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[RES:%.*]] = call nnan nsz <2 x double> @llvm.fma.v2f64(<2 x double> [[B:%.*]], <2 x double> <double undef, double 0x4068E00A137F38C5>, <2 x double> [[C:%.*]])
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+entry:
+  %res = call nnan nsz <2 x double> @llvm.fma.v2f64(<2 x double> <double undef, double 199.00123>, <2 x double> %b, <2 x double> %c)
+  ret <2 x double> %res
+}
+
+define <2 x double> @fma_partial_undef_1(<2 x double> %b, <2 x double> %c) {
+; CHECK-LABEL: @fma_partial_undef_1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[RES:%.*]] = call nnan nsz <2 x double> @llvm.fma.v2f64(<2 x double> [[B:%.*]], <2 x double> <double 0x4068E00A137F38C5, double undef>, <2 x double> [[C:%.*]])
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+entry:
+  %res = call nnan nsz <2 x double> @llvm.fma.v2f64(<2 x double> %b, <2 x double> <double 199.00123, double undef>, <2 x double> %c)
+  ret <2 x double> %res
+}
+
+define <2 x double> @fma_partial_undef_2(<2 x double> %b, <2 x double> %c) {
+; CHECK-LABEL: @fma_partial_undef_2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[RES:%.*]] = call nnan nsz <2 x double> @llvm.fma.v2f64(<2 x double> [[B:%.*]], <2 x double> [[C:%.*]], <2 x double> <double 0x4068E00A137F38C5, double undef>)
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+entry:
+  %res = call nnan nsz <2 x double> @llvm.fma.v2f64(<2 x double> %b, <2 x double> %c, <2 x double> <double 199.00123, double undef>)
+  ret <2 x double> %res
+}
+
+
+define <2 x double> @fma_nan_0(<2 x double> %b, <2 x double> %c) {
+; CHECK-LABEL: @fma_nan_0(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <2 x double> <double 0x7FF8000000000000, double 0x7FF8000000000000>
+;
+entry:
+  %res = call nnan nsz <2 x double> @llvm.fma.v2f64(<2 x double> <double 0x7FF8000000000000, double 0x7FF8000000000000>, <2 x double> %b, <2 x double> %c)
+  ret <2 x double> %res
+}
+define <2 x double> @fma_nan_1(<2 x double> %b, <2 x double> %c) {
+; CHECK-LABEL: @fma_nan_1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <2 x double> <double 0x7FF8000000000000, double 0x7FF8000000000000>
+;
+entry:
+  %res = call nnan nsz <2 x double> @llvm.fma.v2f64(<2 x double> %b, <2 x double> <double 0x7FF8000000000000, double 0x7FF8000000000000>, <2 x double> %c)
+  ret <2 x double> %res
+}
+
+define <2 x double> @fma_nan_2(<2 x double> %b, <2 x double> %c) {
+; CHECK-LABEL: @fma_nan_2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <2 x double> <double 0x7FF8000000000000, double 0x7FF8000000000000>
+;
+entry:
+  %res = call nnan nsz <2 x double> @llvm.fma.v2f64(<2 x double> %b, <2 x double> %c, <2 x double> <double 0x7FF8000000000000, double 0x7FF8000000000000>)
+  ret <2 x double> %res
+}
+
+define <2 x double> @fmuladd_const_fmul(<2 x double> %b) {
+; CHECK-LABEL: @fmuladd_const_fmul(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[RES:%.*]] = fadd nnan nsz <2 x double> [[B:%.*]], <double 0x3F8DB6C076AD949B, double 0x3FB75A405B6E6D69>
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+entry:
+  %res = call nnan nsz <2 x double> @llvm.fmuladd.v2f64(<2 x double> <double 1123123.0099110012314, double 9999.0000001>, <2 x double> <double 0.0000000129182, double 0.000009123>, <2 x double> %b)
+  ret <2 x double> %res
+}
+
+define <2 x double> @fmuladd_nan_and_const_0(<2 x double> %b) {
+; CHECK-LABEL: @fmuladd_nan_and_const_0(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <2 x double> <double 0x7FF8000000000000, double 0x7FF8000000000000>
+;
+entry:
+  %res = call nnan nsz <2 x double> @llvm.fmuladd.v2f64(<2 x double> <double 0x7FF8000000000000, double 0x7FF8000000000000>, <2 x double> <double 0.0000000129182, double 0.000009123>, <2 x double> %b)
+  ret <2 x double> %res
+}
+
+define <2 x double> @fmuladd_nan_and_const_1(<2 x double> %b) {
+; CHECK-LABEL: @fmuladd_nan_and_const_1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <2 x double> <double 0x7FF8000000000000, double 0x7FF8000000000000>
+;
+entry:
+  %res = call nnan nsz <2 x double> @llvm.fmuladd.v2f64(<2 x double> <double 0.0000000129182, double 0.000009123>, <2 x double> <double 0x7FF8000000000000, double 0x7FF8000000000000>, <2 x double> %b)
+  ret <2 x double> %res
+}
+
+define <2 x double> @fmuladd_nan_and_const_2(<2 x double> %b) {
+; CHECK-LABEL: @fmuladd_nan_and_const_2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <2 x double> <double 0x7FF8000000000000, double 0x7FF8000000000000>
+;
+entry:
+  %res = call nnan nsz <2 x double> @llvm.fmuladd.v2f64(<2 x double> <double 0.0000000129182, double 0.000009123>, <2 x double> %b, <2 x double> <double 0x7FF8000000000000, double 0x7FF8000000000000>)
+  ret <2 x double> %res
+}
+
+define <2 x double> @fmuladd_nan_0(<2 x double> %b, <2 x double> %c) {
+; CHECK-LABEL: @fmuladd_nan_0(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <2 x double> <double 0x7FF8000000000000, double 0x7FF8000000000000>
+;
+entry:
+  %res = call nnan nsz <2 x double> @llvm.fmuladd.v2f64(<2 x double> <double 0x7FF8000000000000, double 0x7FF8000000000000>, <2 x double> %b, <2 x double> %c)
+  ret <2 x double> %res
+}
+
+define <2 x double> @fmuladd_nan_1(<2 x double> %b, <2 x double> %c) {
+; CHECK-LABEL: @fmuladd_nan_1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <2 x double> <double 0x7FF8000000000000, double 0x7FF8000000000000>
+;
+entry:
+  %res = call nnan nsz <2 x double> @llvm.fmuladd.v2f64(<2 x double> %b, <2 x double> <double 0x7FF8000000000000, double 0x7FF8000000000000>, <2 x double> %c)
+  ret <2 x double> %res
+}
+
+define <2 x double> @fmuladd_undef_0(<2 x double> %b, <2 x double> %c) {
+; CHECK-LABEL: @fmuladd_undef_0(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <2 x double> <double 0x7FF8000000000000, double 0x7FF8000000000000>
+;
+entry:
+  %res = call nnan nsz <2 x double> @llvm.fmuladd.v2f64(<2 x double> <double undef, double undef>, <2 x double> %b, <2 x double> %c)
+  ret <2 x double> %res
+}
+
+define <2 x double> @fmuladd_undef_1(<2 x double> %b, <2 x double> %c) {
+; CHECK-LABEL: @fmuladd_undef_1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <2 x double> <double 0x7FF8000000000000, double 0x7FF8000000000000>
+;
+entry:
+  %res = call nnan nsz <2 x double> @llvm.fmuladd.v2f64(<2 x double> %b, <2 x double> <double undef, double undef>, <2 x double> %c)
+  ret <2 x double> %res
+}
+
+define <2 x double> @fmuladd_undef_2(<2 x double> %b, <2 x double> %c) {
+; CHECK-LABEL: @fmuladd_undef_2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <2 x double> <double 0x7FF8000000000000, double 0x7FF8000000000000>
+;
+entry:
+  %res = call nnan nsz <2 x double> @llvm.fmuladd.v2f64(<2 x double> %b, <2 x double> %c, <2 x double> <double 0x7FF8000000000000, double 0x7FF8000000000000>)
+  ret <2 x double> %res
+}
+declare <2 x double> @llvm.fma.v2f64(<2 x double>, <2 x double>, <2 x double>)
+declare <2 x double> @llvm.sqrt.v2f64(<2 x double>)
+
+
 attributes #0 = { nounwind }
 attributes #1 = { nounwind readnone }
diff --git a/llvm/test/Transforms/InstCombine/fmul.ll b/llvm/test/Transforms/InstCombine/fmul.ll
index 99b67dfa8..b1ee74e1a 100644
--- a/llvm/test/Transforms/InstCombine/fmul.ll
+++ b/llvm/test/Transforms/InstCombine/fmul.ll
@@ -277,11 +277,11 @@ define float @neg_unary_neg_multi_use(float %x, float %y) {
   ret float %mul
 }
 
-; (-0.0 - X) * Y => -0.0 - (X * Y)
-define float @neg_sink(float %x, float %y) {
-; CHECK-LABEL: @neg_sink(
-; CHECK-NEXT:    [[TMP1:%.*]] = fmul float [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[MUL:%.*]] = fsub float -0.000000e+00, [[TMP1]]
+; (-0.0 - X) * Y
+define float @neg_mul(float %x, float %y) {
+; CHECK-LABEL: @neg_mul(
+; CHECK-NEXT:    [[SUB:%.*]] = fsub float -0.000000e+00, [[X:%.*]]
+; CHECK-NEXT:    [[MUL:%.*]] = fmul float [[SUB]], [[Y:%.*]]
 ; CHECK-NEXT:    ret float [[MUL]]
 ;
   %sub = fsub float -0.0, %x
@@ -289,10 +289,10 @@ define float @neg_sink(float %x, float %y) {
   ret float %mul
 }
 
-define float @unary_neg_sink(float %x, float %y) {
-; CHECK-LABEL: @unary_neg_sink(
-; CHECK-NEXT:    [[TMP1:%.*]] = fmul float [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[MUL:%.*]] = fsub float -0.000000e+00, [[TMP1]]
+define float @unary_neg_mul(float %x, float %y) {
+; CHECK-LABEL: @unary_neg_mul(
+; CHECK-NEXT:    [[NEG:%.*]] = fneg float [[X:%.*]]
+; CHECK-NEXT:    [[MUL:%.*]] = fmul float [[NEG]], [[Y:%.*]]
 ; CHECK-NEXT:    ret float [[MUL]]
 ;
   %neg = fneg float %x
@@ -300,10 +300,10 @@ define float @unary_neg_sink(float %x, float %y) {
   ret float %mul
 }
 
-define <2 x float> @neg_sink_vec(<2 x float> %x, <2 x float> %y) {
-; CHECK-LABEL: @neg_sink_vec(
-; CHECK-NEXT:    [[TMP1:%.*]] = fmul <2 x float> [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[MUL:%.*]] = fsub <2 x float> <float -0.000000e+00, float -0.000000e+00>, [[TMP1]]
+define <2 x float> @neg_mul_vec(<2 x float> %x, <2 x float> %y) {
+; CHECK-LABEL: @neg_mul_vec(
+; CHECK-NEXT:    [[SUB:%.*]] = fsub <2 x float> <float -0.000000e+00, float -0.000000e+00>, [[X:%.*]]
+; CHECK-NEXT:    [[MUL:%.*]] = fmul <2 x float> [[SUB]], [[Y:%.*]]
 ; CHECK-NEXT:    ret <2 x float> [[MUL]]
 ;
   %sub = fsub <2 x float> <float -0.0, float -0.0>, %x
@@ -311,11 +311,10 @@ define <2 x float> @neg_sink_vec(<2 x float> %x, <2 x float> %y) {
   ret <2 x float> %mul
 }
 
-; FIXME: Should generate a unary FNeg.
-define <2 x float> @unary_neg_sink_vec(<2 x float> %x, <2 x float> %y) {
-; CHECK-LABEL: @unary_neg_sink_vec(
-; CHECK-NEXT:    [[TMP1:%.*]] = fmul <2 x float> [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[MUL:%.*]] = fsub <2 x float> <float -0.000000e+00, float -0.000000e+00>, [[TMP1]]
+define <2 x float> @unary_neg_mul_vec(<2 x float> %x, <2 x float> %y) {
+; CHECK-LABEL: @unary_neg_mul_vec(
+; CHECK-NEXT:    [[SUB:%.*]] = fneg <2 x float> [[X:%.*]]
+; CHECK-NEXT:    [[MUL:%.*]] = fmul <2 x float> [[SUB]], [[Y:%.*]]
 ; CHECK-NEXT:    ret <2 x float> [[MUL]]
 ;
   %sub = fneg <2 x float> %x
@@ -323,10 +322,10 @@ define <2 x float> @unary_neg_sink_vec(<2 x float> %x, <2 x float> %y) {
   ret <2 x float> %mul
 }
 
-define <2 x float> @neg_sink_vec_undef(<2 x float> %x, <2 x float> %y) {
-; CHECK-LABEL: @neg_sink_vec_undef(
-; CHECK-NEXT:    [[TMP1:%.*]] = fmul <2 x float> [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[MUL:%.*]] = fsub <2 x float> <float -0.000000e+00, float -0.000000e+00>, [[TMP1]]
+define <2 x float> @neg_mul_vec_undef(<2 x float> %x, <2 x float> %y) {
+; CHECK-LABEL: @neg_mul_vec_undef(
+; CHECK-NEXT:    [[SUB:%.*]] = fsub <2 x float> <float undef, float -0.000000e+00>, [[X:%.*]]
+; CHECK-NEXT:    [[MUL:%.*]] = fmul <2 x float> [[SUB]], [[Y:%.*]]
 ; CHECK-NEXT:    ret <2 x float> [[MUL]]
 ;
   %sub = fsub <2 x float> <float undef, float -0.0>, %x
@@ -334,11 +333,11 @@ define <2 x float> @neg_sink_vec_undef(<2 x float> %x, <2 x float> %y) {
   ret <2 x float> %mul
 }
 
-; (0.0 - X) * Y => 0.0 - (X * Y)
+; (0.0 - X) * Y
 define float @neg_sink_nsz(float %x, float %y) {
 ; CHECK-LABEL: @neg_sink_nsz(
-; CHECK-NEXT:    [[TMP1:%.*]] = fmul float [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[MUL:%.*]] = fsub float -0.000000e+00, [[TMP1]]
+; CHECK-NEXT:    [[SUB1:%.*]] = fsub nsz float -0.000000e+00, [[X:%.*]]
+; CHECK-NEXT:    [[MUL:%.*]] = fmul float [[SUB1]], [[Y:%.*]]
 ; CHECK-NEXT:    ret float [[MUL]]
 ;
   %sub1 = fsub nsz float 0.0, %x
@@ -346,8 +345,6 @@ define float @neg_sink_nsz(float %x, float %y) {
   ret float %mul
 }
 
-; "(-0.0 - X) * Y => -0.0 - (X * Y)" is disabled if expression "-0.0 - X"
-; has multiple uses.
 define float @neg_sink_multi_use(float %x, float %y) {
 ; CHECK-LABEL: @neg_sink_multi_use(
 ; CHECK-NEXT:    [[SUB1:%.*]] = fsub float -0.000000e+00, [[X:%.*]]
@@ -361,8 +358,8 @@ define float @neg_sink_multi_use(float %x, float %y) {
   ret float %mul2
 }
 
-define float @unary_neg_sink_multi_use(float %x, float %y) {
-; CHECK-LABEL: @unary_neg_sink_multi_use(
+define float @unary_neg_mul_multi_use(float %x, float %y) {
+; CHECK-LABEL: @unary_neg_mul_multi_use(
 ; CHECK-NEXT:    [[SUB1:%.*]] = fneg float [[X:%.*]]
 ; CHECK-NEXT:    [[MUL:%.*]] = fmul float [[SUB1]], [[Y:%.*]]
 ; CHECK-NEXT:    [[MUL2:%.*]] = fmul float [[MUL]], [[SUB1]]
@@ -994,3 +991,128 @@ define double @fmul_negated_constant_expression(double %x) {
   %r = fmul double %x, fsub (double -0.000000e+00, double bitcast (i64 ptrtoint (i8** getelementptr inbounds ({ [2 x i8*] }, { [2 x i8*] }* @g, i64 0, inrange i32 0, i64 2) to i64) to double))
   ret double %r
 }
+
+define float @negate_if_true(float %x, i1 %cond) {
+; CHECK-LABEL: @negate_if_true(
+; CHECK-NEXT:    [[TMP1:%.*]] = fneg float [[X:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[COND:%.*]], float [[TMP1]], float [[X]]
+; CHECK-NEXT:    ret float [[TMP2]]
+;
+  %sel = select i1 %cond, float -1.0, float 1.0
+  %r = fmul float %sel, %x
+  ret float %r
+}
+
+define float @negate_if_false(float %x, i1 %cond) {
+; CHECK-LABEL: @negate_if_false(
+; CHECK-NEXT:    [[TMP1:%.*]] = fneg arcp float [[X:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = select arcp i1 [[COND:%.*]], float [[X]], float [[TMP1]]
+; CHECK-NEXT:    ret float [[TMP2]]
+;
+  %sel = select i1 %cond, float 1.0, float -1.0
+  %r = fmul arcp float %sel, %x
+  ret float %r
+}
+
+define <2 x double> @negate_if_true_commute(<2 x double> %px, i1 %cond) {
+; CHECK-LABEL: @negate_if_true_commute(
+; CHECK-NEXT:    [[X:%.*]] = fdiv <2 x double> <double 4.200000e+01, double 4.200000e+01>, [[PX:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fneg ninf <2 x double> [[X]]
+; CHECK-NEXT:    [[TMP2:%.*]] = select ninf i1 [[COND:%.*]], <2 x double> [[TMP1]], <2 x double> [[X]]
+; CHECK-NEXT:    ret <2 x double> [[TMP2]]
+;
+  %x = fdiv <2 x double> <double 42.0, double 42.0>, %px  ; thwart complexity-based canonicalization
+  %sel = select i1 %cond, <2 x double> <double -1.0, double -1.0>, <2 x double> <double 1.0, double 1.0>
+  %r = fmul ninf <2 x double> %x, %sel
+  ret <2 x double> %r
+}
+
+define <2 x double> @negate_if_false_commute(<2 x double> %px, <2 x i1> %cond) {
+; CHECK-LABEL: @negate_if_false_commute(
+; CHECK-NEXT:    [[X:%.*]] = fdiv <2 x double> <double 4.200000e+01, double 5.100000e+00>, [[PX:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fneg <2 x double> [[X]]
+; CHECK-NEXT:    [[TMP2:%.*]] = select <2 x i1> [[COND:%.*]], <2 x double> [[X]], <2 x double> [[TMP1]]
+; CHECK-NEXT:    ret <2 x double> [[TMP2]]
+;
+  %x = fdiv <2 x double> <double 42.0, double 5.1>, %px  ; thwart complexity-based canonicalization
+  %sel = select <2 x i1> %cond, <2 x double> <double 1.0, double 1.0>, <2 x double> <double -1.0, double -1.0>
+  %r = fmul <2 x double> %x, %sel
+  ret <2 x double> %r
+}
+
+; Negative test
+
+define float @negate_if_true_extra_use(float %x, i1 %cond) {
+; CHECK-LABEL: @negate_if_true_extra_use(
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[COND:%.*]], float -1.000000e+00, float 1.000000e+00
+; CHECK-NEXT:    call void @use_f32(float [[SEL]])
+; CHECK-NEXT:    [[R:%.*]] = fmul float [[SEL]], [[X:%.*]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %sel = select i1 %cond, float -1.0, float 1.0
+  call void @use_f32(float %sel)
+  %r = fmul float %sel, %x
+  ret float %r
+}
+
+; Negative test
+
+define <2 x double> @negate_if_true_wrong_constant(<2 x double> %px, i1 %cond) {
+; CHECK-LABEL: @negate_if_true_wrong_constant(
+; CHECK-NEXT:    [[X:%.*]] = fdiv <2 x double> <double 4.200000e+01, double 4.200000e+01>, [[PX:%.*]]
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[COND:%.*]], <2 x double> <double -1.000000e+00, double 0.000000e+00>, <2 x double> <double 1.000000e+00, double 1.000000e+00>
+; CHECK-NEXT:    [[R:%.*]] = fmul <2 x double> [[X]], [[SEL]]
+; CHECK-NEXT:    ret <2 x double> [[R]]
+;
+  %x = fdiv <2 x double> <double 42.0, double 42.0>, %px  ; thwart complexity-based canonicalization
+  %sel = select i1 %cond, <2 x double> <double -1.0, double 0.0>, <2 x double> <double 1.0, double 1.0>
+  %r = fmul <2 x double> %x, %sel
+  ret <2 x double> %r
+}
+
+; X *fast (C ? 1.0 : 0.0) -> C ? X : 0.0
+define float @fmul_select(float %x, i1 %c) {
+; CHECK-LABEL: @fmul_select(
+; CHECK-NEXT:    [[MUL:%.*]] = select fast i1 [[C:%.*]], float [[X:%.*]], float 0.000000e+00
+; CHECK-NEXT:    ret float [[MUL]]
+;
+  %sel = select i1 %c, float 1.0, float 0.0
+  %mul = fmul fast float %sel, %x
+  ret float %mul
+}
+
+; X *fast (C ? 1.0 : 0.0) -> C ? X : 0.0
+define <2 x float> @fmul_select_vec(<2 x float> %x, i1 %c) {
+; CHECK-LABEL: @fmul_select_vec(
+; CHECK-NEXT:    [[MUL:%.*]] = select fast i1 [[C:%.*]], <2 x float> [[X:%.*]], <2 x float> zeroinitializer
+; CHECK-NEXT:    ret <2 x float> [[MUL]]
+;
+  %sel = select i1 %c, <2 x float> <float 1.0, float 1.0>, <2 x float> zeroinitializer
+  %mul = fmul fast <2 x float> %sel, %x
+  ret <2 x float> %mul
+}
+
+; Without fast math flags we can't optimize X * (C ? 1.0 : 0.0) -> C ? X : 0.0
+define float @fmul_select_strict(float %x, i1 %c) {
+; CHECK-LABEL: @fmul_select_strict(
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[C:%.*]], float 1.000000e+00, float 0.000000e+00
+; CHECK-NEXT:    [[MUL:%.*]] = fmul float [[SEL]], [[X:%.*]]
+; CHECK-NEXT:    ret float [[MUL]]
+;
+  %sel = select i1 %c, float 1.0, float 0.0
+  %mul = fmul float %sel, %x
+  ret float %mul
+}
+
+; sqrt(X) *fast (C ? sqrt(X) : 1.0) -> C ? X : sqrt(X)
+define double @fmul_sqrt_select(double %x, i1 %c) {
+; CHECK-LABEL: @fmul_sqrt_select(
+; CHECK-NEXT:    [[SQR:%.*]] = call double @llvm.sqrt.f64(double [[X:%.*]])
+; CHECK-NEXT:    [[MUL:%.*]] = select fast i1 [[C:%.*]], double [[X]], double [[SQR]]
+; CHECK-NEXT:    ret double [[MUL]]
+;
+  %sqr = call double @llvm.sqrt.f64(double %x)
+  %sel = select i1 %c, double %sqr, double 1.0
+  %mul = fmul fast double %sqr, %sel
+  ret double %mul
+}
diff --git a/llvm/test/Transforms/InstCombine/fortify-folding.ll b/llvm/test/Transforms/InstCombine/fortify-folding.ll
index 68be219ed..ee8155761 100644
--- a/llvm/test/Transforms/InstCombine/fortify-folding.ll
+++ b/llvm/test/Transforms/InstCombine/fortify-folding.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -instcombine -S | FileCheck %s --dump-input-on-failure
 
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
@@ -9,9 +10,10 @@ target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 %struct.__va_list_tag = type { i32, i32, i8*, i8* }
 
 define i8* @test_memccpy() {
-  ; CHECK-LABEL: define i8* @test_memccpy
-  ; CHECK-NEXT: call i8* @memccpy(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0), i32 0, i64 60)
-  ; CHECK-NEXT: ret i8*
+; CHECK-LABEL: @test_memccpy(
+; CHECK-NEXT:    [[MEMCCPY:%.*]] = call i8* @memccpy(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0), i32 0, i64 60)
+; CHECK-NEXT:    ret i8* [[MEMCCPY]]
+;
   %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
   %src = getelementptr inbounds [60 x i8], [60 x i8]* @b, i32 0, i32 0
   %ret = call i8* @__memccpy_chk(i8* %dst, i8* %src, i32 0, i64 60, i64 -1)
@@ -19,9 +21,10 @@ define i8* @test_memccpy() {
 }
 
 define i8* @test_not_memccpy() {
-  ; CHECK-LABEL: define i8* @test_not_memccpy
-  ; CHECK-NEXT: call i8* @__memccpy_chk
-  ; CHECK-NEXT: ret i8*
+; CHECK-LABEL: @test_not_memccpy(
+; CHECK-NEXT:    [[RET:%.*]] = call i8* @__memccpy_chk(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0), i32 0, i64 60, i64 59)
+; CHECK-NEXT:    ret i8* [[RET]]
+;
   %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
   %src = getelementptr inbounds [60 x i8], [60 x i8]* @b, i32 0, i32 0
   %ret = call i8* @__memccpy_chk(i8* %dst, i8* %src, i32 0, i64 60, i64 59)
@@ -29,9 +32,10 @@ define i8* @test_not_memccpy() {
 }
 
 define i32 @test_snprintf() {
-  ; CHECK-LABEL: define i32 @test_snprintf
-  ; CHECK-NEXT: call i32 (i8*, i64, i8*, ...) @snprintf(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i64 60, i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0))
-  ; CHECK-NEXT: ret i32
+; CHECK-LABEL: @test_snprintf(
+; CHECK-NEXT:    [[SNPRINTF:%.*]] = call i32 (i8*, i64, i8*, ...) @snprintf(i8* nonnull dereferenceable(1) getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i64 60, i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0))
+; CHECK-NEXT:    ret i32 [[SNPRINTF]]
+;
   %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
   %fmt = getelementptr inbounds [60 x i8], [60 x i8]* @b, i32 0, i32 0
   %ret = call i32 (i8*, i64, i32, i64, i8*, ...) @__snprintf_chk(i8* %dst, i64 60, i32 0, i64 -1, i8* %fmt)
@@ -39,10 +43,11 @@ define i32 @test_snprintf() {
 }
 
 define i32 @test_not_snprintf() {
-  ; CHECK-LABEL: define i32 @test_not_snprintf
-  ; CHECK-NEXT: call i32 (i8*, i64, i32, i64, i8*, ...) @__snprintf_chk
-  ; CHECK-NEXT: call i32 (i8*, i64, i32, i64, i8*, ...) @__snprintf_chk
-  ; CHECK-NEXT: ret i32
+; CHECK-LABEL: @test_not_snprintf(
+; CHECK-NEXT:    [[RET:%.*]] = call i32 (i8*, i64, i32, i64, i8*, ...) @__snprintf_chk(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i64 60, i32 0, i64 59, i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0))
+; CHECK-NEXT:    [[IGN:%.*]] = call i32 (i8*, i64, i32, i64, i8*, ...) @__snprintf_chk(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i64 60, i32 1, i64 -1, i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0))
+; CHECK-NEXT:    ret i32 [[RET]]
+;
   %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
   %fmt = getelementptr inbounds [60 x i8], [60 x i8]* @b, i32 0, i32 0
   %ret = call i32 (i8*, i64, i32, i64, i8*, ...) @__snprintf_chk(i8* %dst, i64 60, i32 0, i64 59, i8* %fmt)
@@ -51,9 +56,10 @@ define i32 @test_not_snprintf() {
 }
 
 define i32 @test_sprintf() {
-  ; CHECK-LABEL: define i32 @test_sprintf
-  ; CHECK-NEXT: call i32 (i8*, i8*, ...) @sprintf(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0))
-  ; CHECK-NEXT: ret i32
+; CHECK-LABEL: @test_sprintf(
+; CHECK-NEXT:    [[SPRINTF:%.*]] = call i32 (i8*, i8*, ...) @sprintf(i8* nonnull dereferenceable(1) getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i8* nonnull dereferenceable(1) getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0))
+; CHECK-NEXT:    ret i32 [[SPRINTF]]
+;
   %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
   %fmt = getelementptr inbounds [60 x i8], [60 x i8]* @b, i32 0, i32 0
   %ret = call i32 (i8*, i32, i64, i8*, ...) @__sprintf_chk(i8* %dst, i32 0, i64 -1, i8* %fmt)
@@ -61,10 +67,11 @@ define i32 @test_sprintf() {
 }
 
 define i32 @test_not_sprintf() {
-  ; CHECK-LABEL: define i32 @test_not_sprintf
-  ; CHECK-NEXT: call i32 (i8*, i32, i64, i8*, ...) @__sprintf_chk
-  ; CHECK-NEXT: call i32 (i8*, i32, i64, i8*, ...) @__sprintf_chk
-  ; CHECK-NEXT: ret i32
+; CHECK-LABEL: @test_not_sprintf(
+; CHECK-NEXT:    [[RET:%.*]] = call i32 (i8*, i32, i64, i8*, ...) @__sprintf_chk(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i32 0, i64 59, i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0))
+; CHECK-NEXT:    [[IGNORED:%.*]] = call i32 (i8*, i32, i64, i8*, ...) @__sprintf_chk(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i32 1, i64 -1, i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0))
+; CHECK-NEXT:    ret i32 [[RET]]
+;
   %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
   %fmt = getelementptr inbounds [60 x i8], [60 x i8]* @b, i32 0, i32 0
   %ret = call i32 (i8*, i32, i64, i8*, ...) @__sprintf_chk(i8* %dst, i32 0, i64 59, i8* %fmt)
@@ -73,9 +80,10 @@ define i32 @test_not_sprintf() {
 }
 
 define i8* @test_strcat() {
-  ; CHECK-LABEL: define i8* @test_strcat
-  ; CHECK-NEXT: call i8* @strcat(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0))
-  ; CHECK-NEXT: ret i8*
+; CHECK-LABEL: @test_strcat(
+; CHECK-NEXT:    [[STRCAT:%.*]] = call i8* @strcat(i8* nonnull dereferenceable(1) getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i8* nonnull dereferenceable(1) getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0))
+; CHECK-NEXT:    ret i8* [[STRCAT]]
+;
   %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
   %src = getelementptr inbounds [60 x i8], [60 x i8]* @b, i32 0, i32 0
   %ret = call i8* @__strcat_chk(i8* %dst, i8* %src, i64 -1)
@@ -83,9 +91,10 @@ define i8* @test_strcat() {
 }
 
 define i8* @test_not_strcat() {
-  ; CHECK-LABEL: define i8* @test_not_strcat
-  ; CHECK-NEXT: call i8* @__strcat_chk
-  ; CHECK-NEXT: ret i8*
+; CHECK-LABEL: @test_not_strcat(
+; CHECK-NEXT:    [[RET:%.*]] = call i8* @__strcat_chk(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0), i64 0)
+; CHECK-NEXT:    ret i8* [[RET]]
+;
   %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
   %src = getelementptr inbounds [60 x i8], [60 x i8]* @b, i32 0, i32 0
   %ret = call i8* @__strcat_chk(i8* %dst, i8* %src, i64 0)
@@ -93,9 +102,10 @@ define i8* @test_not_strcat() {
 }
 
 define i64 @test_strlcat() {
-  ; CHECK-LABEL: define i64 @test_strlcat
-  ; CHECK-NEXT: call i64 @strlcat(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0), i64 22)
-  ; CHECK-NEXT: ret i64
+; CHECK-LABEL: @test_strlcat(
+; CHECK-NEXT:    [[STRLCAT:%.*]] = call i64 @strlcat(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0), i64 22)
+; CHECK-NEXT:    ret i64 [[STRLCAT]]
+;
   %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
   %src = getelementptr inbounds [60 x i8], [60 x i8]* @b, i32 0, i32 0
   %ret = call i64 @__strlcat_chk(i8* %dst, i8* %src, i64 22, i64 -1)
@@ -103,9 +113,10 @@ define i64 @test_strlcat() {
 }
 
 define i64 @test_not_strlcat() {
-  ; CHECK-LABEL: define i64 @test_not_strlcat
-  ; CHECK-NEXT: call i64 @__strlcat_chk
-  ; CHECK-NEXT: ret i64
+; CHECK-LABEL: @test_not_strlcat(
+; CHECK-NEXT:    [[RET:%.*]] = call i64 @__strlcat_chk(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0), i64 22, i64 0)
+; CHECK-NEXT:    ret i64 [[RET]]
+;
   %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
   %src = getelementptr inbounds [60 x i8], [60 x i8]* @b, i32 0, i32 0
   %ret = call i64 @__strlcat_chk(i8* %dst, i8* %src, i64 22, i64 0)
@@ -113,9 +124,10 @@ define i64 @test_not_strlcat() {
 }
 
 define i8* @test_strncat() {
-  ; CHECK-LABEL: define i8* @test_strncat
-  ; CHECK-NEXT: call i8* @strncat(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0), i64 22)
-  ; CHECK-NEXT: ret i8*
+; CHECK-LABEL: @test_strncat(
+; CHECK-NEXT:    [[STRNCAT:%.*]] = call i8* @strncat(i8* nonnull dereferenceable(1) getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i8* nonnull dereferenceable(1) getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0), i64 22)
+; CHECK-NEXT:    ret i8* [[STRNCAT]]
+;
   %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
   %src = getelementptr inbounds [60 x i8], [60 x i8]* @b, i32 0, i32 0
   %ret = call i8* @__strncat_chk(i8* %dst, i8* %src, i64 22, i64 -1)
@@ -123,9 +135,10 @@ define i8* @test_strncat() {
 }
 
 define i8* @test_not_strncat() {
-  ; CHECK-LABEL: define i8* @test_not_strncat
-  ; CHECK-NEXT: call i8* @__strncat_chk(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0), i64 22, i64 3)
-  ; CHECK-NEXT: ret i8*
+; CHECK-LABEL: @test_not_strncat(
+; CHECK-NEXT:    [[RET:%.*]] = call i8* @__strncat_chk(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0), i64 22, i64 3)
+; CHECK-NEXT:    ret i8* [[RET]]
+;
   %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
   %src = getelementptr inbounds [60 x i8], [60 x i8]* @b, i32 0, i32 0
   %ret = call i8* @__strncat_chk(i8* %dst, i8* %src, i64 22, i64 3)
@@ -133,9 +146,10 @@ define i8* @test_not_strncat() {
 }
 
 define i64 @test_strlcpy() {
-  ; CHECK-LABEL: define i64 @test_strlcpy
-  ; CHECK-NEXT: call i64 @strlcpy(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0), i64 22)
-  ; CHECK-NEXT: ret i64
+; CHECK-LABEL: @test_strlcpy(
+; CHECK-NEXT:    [[STRLCPY:%.*]] = call i64 @strlcpy(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0), i64 22)
+; CHECK-NEXT:    ret i64 [[STRLCPY]]
+;
   %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
   %src = getelementptr inbounds [60 x i8], [60 x i8]* @b, i32 0, i32 0
   %ret = call i64 @__strlcpy_chk(i8* %dst, i8* %src, i64 22, i64 -1)
@@ -143,9 +157,10 @@ define i64 @test_strlcpy() {
 }
 
 define i64 @test_not_strlcpy() {
-  ; CHECK-LABEL: define i64 @test_not_strlcpy
-  ; CHECK-NEXT: call i64 @__strlcpy_chk(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0), i64 22, i64 2)
-  ; CHECK-NEXT: ret i64
+; CHECK-LABEL: @test_not_strlcpy(
+; CHECK-NEXT:    [[RET:%.*]] = call i64 @__strlcpy_chk(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0), i64 22, i64 2)
+; CHECK-NEXT:    ret i64 [[RET]]
+;
   %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
   %src = getelementptr inbounds [60 x i8], [60 x i8]* @b, i32 0, i32 0
   %ret = call i64 @__strlcpy_chk(i8* %dst, i8* %src, i64 22, i64 2)
@@ -153,8 +168,10 @@ define i64 @test_not_strlcpy() {
 }
 
 define i32 @test_vsnprintf() {
-  ; CHECK-LABEL: define i32 @test_vsnprintf
-  ; CHECK-NEXT: call i32 @vsnprintf(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i64 4, i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0), %struct.__va_list_tag* null)
+; CHECK-LABEL: @test_vsnprintf(
+; CHECK-NEXT:    [[VSNPRINTF:%.*]] = call i32 @vsnprintf(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i64 4, i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0), %struct.__va_list_tag* null)
+; CHECK-NEXT:    ret i32 [[VSNPRINTF]]
+;
   ; ret i32
   %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
   %src = getelementptr inbounds [60 x i8], [60 x i8]* @b, i32 0, i32 0
@@ -163,9 +180,11 @@ define i32 @test_vsnprintf() {
 }
 
 define i32 @test_not_vsnprintf() {
-  ; CHECK-LABEL: define i32 @test_not_vsnprintf
-  ; CHECK-NEXT: call i32 @__vsnprintf_chk(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i64 4, i32 0, i64 3, i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0), %struct.__va_list_tag* null)
-  ; CHECK-NEXT: call i32 @__vsnprintf_chk(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i64 4, i32 1, i64 -1, i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0), %struct.__va_list_tag* null)
+; CHECK-LABEL: @test_not_vsnprintf(
+; CHECK-NEXT:    [[RET:%.*]] = call i32 @__vsnprintf_chk(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i64 4, i32 0, i64 3, i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0), %struct.__va_list_tag* null)
+; CHECK-NEXT:    [[IGN:%.*]] = call i32 @__vsnprintf_chk(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i64 4, i32 1, i64 -1, i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0), %struct.__va_list_tag* null)
+; CHECK-NEXT:    ret i32 [[RET]]
+;
   ; ret i32
   %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
   %src = getelementptr inbounds [60 x i8], [60 x i8]* @b, i32 0, i32 0
@@ -175,8 +194,10 @@ define i32 @test_not_vsnprintf() {
 }
 
 define i32 @test_vsprintf() {
-  ; CHECK-LABEL: define i32 @test_vsprintf
-  ; CHECK-NEXT: call i32 @vsprintf(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0), %struct.__va_list_tag* null)
+; CHECK-LABEL: @test_vsprintf(
+; CHECK-NEXT:    [[VSPRINTF:%.*]] = call i32 @vsprintf(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0), %struct.__va_list_tag* null)
+; CHECK-NEXT:    ret i32 [[VSPRINTF]]
+;
   ; ret i32
   %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
   %src = getelementptr inbounds [60 x i8], [60 x i8]* @b, i32 0, i32 0
@@ -185,9 +206,11 @@ define i32 @test_vsprintf() {
 }
 
 define i32 @test_not_vsprintf() {
-  ; CHECK-LABEL: define i32 @test_not_vsprintf
-  ; CHECK-NEXT: call i32 @__vsprintf_chk(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i32 0, i64 3, i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0), %struct.__va_list_tag* null)
-  ; CHECK-NEXT: call i32 @__vsprintf_chk(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i32 1, i64 -1, i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0), %struct.__va_list_tag* null)
+; CHECK-LABEL: @test_not_vsprintf(
+; CHECK-NEXT:    [[RET:%.*]] = call i32 @__vsprintf_chk(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i32 0, i64 3, i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0), %struct.__va_list_tag* null)
+; CHECK-NEXT:    [[IGN:%.*]] = call i32 @__vsprintf_chk(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i32 1, i64 -1, i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0), %struct.__va_list_tag* null)
+; CHECK-NEXT:    ret i32 [[RET]]
+;
   ; ret i32
   %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
   %src = getelementptr inbounds [60 x i8], [60 x i8]* @b, i32 0, i32 0
diff --git a/llvm/test/Transforms/InstCombine/fpextend.ll b/llvm/test/Transforms/InstCombine/fpextend.ll
index 9b5986c2f..0b2d985bf 100644
--- a/llvm/test/Transforms/InstCombine/fpextend.ll
+++ b/llvm/test/Transforms/InstCombine/fpextend.ll
@@ -3,114 +3,98 @@
 
 define float @test(float %x) nounwind  {
 ; CHECK-LABEL: @test(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP34:%.*]] = fadd float [[X:%.*]], 0.000000e+00
-; CHECK-NEXT:    ret float [[TMP34]]
+; CHECK-NEXT:    [[T34:%.*]] = fadd float [[X:%.*]], 0.000000e+00
+; CHECK-NEXT:    ret float [[T34]]
 ;
-entry:
-  %tmp1 = fpext float %x to double
-  %tmp3 = fadd double %tmp1, 0.000000e+00
-  %tmp34 = fptrunc double %tmp3 to float
-  ret float %tmp34
+  %t1 = fpext float %x to double
+  %t3 = fadd double %t1, 0.000000e+00
+  %t34 = fptrunc double %t3 to float
+  ret float %t34
 }
 
 define float @test2(float %x, float %y) nounwind  {
 ; CHECK-LABEL: @test2(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP56:%.*]] = fmul float [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    ret float [[TMP56]]
+; CHECK-NEXT:    [[T56:%.*]] = fmul float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret float [[T56]]
 ;
-entry:
-  %tmp1 = fpext float %x to double
-  %tmp23 = fpext float %y to double
-  %tmp5 = fmul double %tmp1, %tmp23
-  %tmp56 = fptrunc double %tmp5 to float
-  ret float %tmp56
+  %t1 = fpext float %x to double
+  %t23 = fpext float %y to double
+  %t5 = fmul double %t1, %t23
+  %t56 = fptrunc double %t5 to float
+  ret float %t56
 }
 
 define float @test3(float %x, float %y) nounwind  {
 ; CHECK-LABEL: @test3(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP56:%.*]] = fdiv float [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    ret float [[TMP56]]
+; CHECK-NEXT:    [[T56:%.*]] = fdiv float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret float [[T56]]
 ;
-entry:
-  %tmp1 = fpext float %x to double
-  %tmp23 = fpext float %y to double
-  %tmp5 = fdiv double %tmp1, %tmp23
-  %tmp56 = fptrunc double %tmp5 to float
-  ret float %tmp56
+  %t1 = fpext float %x to double
+  %t23 = fpext float %y to double
+  %t5 = fdiv double %t1, %t23
+  %t56 = fptrunc double %t5 to float
+  ret float %t56
 }
 
 define float @test4(float %x) nounwind  {
 ; CHECK-LABEL: @test4(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP34:%.*]] = fsub float -0.000000e+00, [[X:%.*]]
-; CHECK-NEXT:    ret float [[TMP34]]
+; CHECK-NEXT:    [[T34:%.*]] = fsub float -0.000000e+00, [[X:%.*]]
+; CHECK-NEXT:    ret float [[T34]]
 ;
-entry:
-  %tmp1 = fpext float %x to double
-  %tmp2 = fsub double -0.000000e+00, %tmp1
-  %tmp34 = fptrunc double %tmp2 to float
-  ret float %tmp34
+  %t1 = fpext float %x to double
+  %t2 = fsub double -0.000000e+00, %t1
+  %t34 = fptrunc double %t2 to float
+  ret float %t34
 }
 
 define float @test4_unary_fneg(float %x) nounwind  {
 ; CHECK-LABEL: @test4_unary_fneg(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP34:%.*]] = fneg float [[X:%.*]]
-; CHECK-NEXT:    ret float [[TMP34]]
+; CHECK-NEXT:    [[T34:%.*]] = fneg float [[X:%.*]]
+; CHECK-NEXT:    ret float [[T34]]
 ;
-entry:
-  %tmp1 = fpext float %x to double
-  %tmp2 = fneg double %tmp1
-  %tmp34 = fptrunc double %tmp2 to float
-  ret float %tmp34
+  %t1 = fpext float %x to double
+  %t2 = fneg double %t1
+  %t34 = fptrunc double %t2 to float
+  ret float %t34
 }
 
 ; Test with vector splat constant
 define <2 x float> @test5(<2 x float> %x) nounwind  {
 ; CHECK-LABEL: @test5(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP34:%.*]] = fadd <2 x float> [[X:%.*]], zeroinitializer
-; CHECK-NEXT:    ret <2 x float> [[TMP34]]
+; CHECK-NEXT:    [[T34:%.*]] = fadd <2 x float> [[X:%.*]], zeroinitializer
+; CHECK-NEXT:    ret <2 x float> [[T34]]
 ;
-entry:
-  %tmp1 = fpext <2 x float> %x to <2 x double>
-  %tmp3 = fadd <2 x double> %tmp1, <double 0.000000e+00, double 0.000000e+00>
-  %tmp34 = fptrunc <2 x double> %tmp3 to <2 x float>
-  ret <2 x float> %tmp34
+  %t1 = fpext <2 x float> %x to <2 x double>
+  %t3 = fadd <2 x double> %t1, <double 0.000000e+00, double 0.000000e+00>
+  %t34 = fptrunc <2 x double> %t3 to <2 x float>
+  ret <2 x float> %t34
 }
 
 ; Test with a non-splat constant
 define <2 x float> @test6(<2 x float> %x) nounwind  {
 ; CHECK-LABEL: @test6(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP34:%.*]] = fadd <2 x float> [[X:%.*]], <float 0.000000e+00, float -0.000000e+00>
-; CHECK-NEXT:    ret <2 x float> [[TMP34]]
+; CHECK-NEXT:    [[T34:%.*]] = fadd <2 x float> [[X:%.*]], <float 0.000000e+00, float -0.000000e+00>
+; CHECK-NEXT:    ret <2 x float> [[T34]]
 ;
-entry:
-  %tmp1 = fpext <2 x float> %x to <2 x double>
-  %tmp3 = fadd <2 x double> %tmp1, <double 0.000000e+00, double -0.000000e+00>
-  %tmp34 = fptrunc <2 x double> %tmp3 to <2 x float>
-  ret <2 x float> %tmp34
+  %t1 = fpext <2 x float> %x to <2 x double>
+  %t3 = fadd <2 x double> %t1, <double 0.000000e+00, double -0.000000e+00>
+  %t34 = fptrunc <2 x double> %t3 to <2 x float>
+  ret <2 x float> %t34
 }
 
 ; Test with an undef element
 ; TODO: Support undef elements.
 define <2 x float> @test6_undef(<2 x float> %x) nounwind  {
 ; CHECK-LABEL: @test6_undef(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP1:%.*]] = fpext <2 x float> [[X:%.*]] to <2 x double>
-; CHECK-NEXT:    [[TMP3:%.*]] = fadd <2 x double> [[TMP1]], <double 0.000000e+00, double undef>
-; CHECK-NEXT:    [[TMP34:%.*]] = fptrunc <2 x double> [[TMP3]] to <2 x float>
-; CHECK-NEXT:    ret <2 x float> [[TMP34]]
+; CHECK-NEXT:    [[T1:%.*]] = fpext <2 x float> [[X:%.*]] to <2 x double>
+; CHECK-NEXT:    [[T3:%.*]] = fadd <2 x double> [[T1]], <double 0.000000e+00, double undef>
+; CHECK-NEXT:    [[T34:%.*]] = fptrunc <2 x double> [[T3]] to <2 x float>
+; CHECK-NEXT:    ret <2 x float> [[T34]]
 ;
-entry:
-  %tmp1 = fpext <2 x float> %x to <2 x double>
-  %tmp3 = fadd <2 x double> %tmp1, <double 0.000000e+00, double undef>
-  %tmp34 = fptrunc <2 x double> %tmp3 to <2 x float>
-  ret <2 x float> %tmp34
+  %t1 = fpext <2 x float> %x to <2 x double>
+  %t3 = fadd <2 x double> %t1, <double 0.000000e+00, double undef>
+  %t34 = fptrunc <2 x double> %t3 to <2 x float>
+  ret <2 x float> %t34
 }
 
 define <2 x float> @not_half_shrinkable(<2 x float> %x) {
@@ -146,151 +130,131 @@ define float @test8(half %a) nounwind {
 
 define float @test9(half %x, half %y) nounwind  {
 ; CHECK-LABEL: @test9(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = fpext half [[X:%.*]] to float
-; CHECK-NEXT:    [[TMP1:%.*]] = fpext half [[Y:%.*]] to float
-; CHECK-NEXT:    [[TMP56:%.*]] = fmul float [[TMP0]], [[TMP1]]
-; CHECK-NEXT:    ret float [[TMP56]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fpext half [[X:%.*]] to float
+; CHECK-NEXT:    [[TMP2:%.*]] = fpext half [[Y:%.*]] to float
+; CHECK-NEXT:    [[T56:%.*]] = fmul float [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret float [[T56]]
 ;
-entry:
-  %tmp1 = fpext half %x to double
-  %tmp23 = fpext half %y to double
-  %tmp5 = fmul double %tmp1, %tmp23
-  %tmp56 = fptrunc double %tmp5 to float
-  ret float %tmp56
+  %t1 = fpext half %x to double
+  %t23 = fpext half %y to double
+  %t5 = fmul double %t1, %t23
+  %t56 = fptrunc double %t5 to float
+  ret float %t56
 }
 
 define float @test10(half %x, float %y) nounwind  {
 ; CHECK-LABEL: @test10(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = fpext half [[X:%.*]] to float
-; CHECK-NEXT:    [[TMP56:%.*]] = fmul float [[TMP0]], [[Y:%.*]]
-; CHECK-NEXT:    ret float [[TMP56]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fpext half [[X:%.*]] to float
+; CHECK-NEXT:    [[T56:%.*]] = fmul float [[TMP1]], [[Y:%.*]]
+; CHECK-NEXT:    ret float [[T56]]
 ;
-entry:
-  %tmp1 = fpext half %x to double
-  %tmp23 = fpext float %y to double
-  %tmp5 = fmul double %tmp1, %tmp23
-  %tmp56 = fptrunc double %tmp5 to float
-  ret float %tmp56
+  %t1 = fpext half %x to double
+  %t23 = fpext float %y to double
+  %t5 = fmul double %t1, %t23
+  %t56 = fptrunc double %t5 to float
+  ret float %t56
 }
 
 define float @test11(half %x) nounwind  {
 ; CHECK-LABEL: @test11(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = fpext half [[X:%.*]] to float
-; CHECK-NEXT:    [[TMP34:%.*]] = fadd float [[TMP0]], 0.000000e+00
-; CHECK-NEXT:    ret float [[TMP34]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fpext half [[X:%.*]] to float
+; CHECK-NEXT:    [[T34:%.*]] = fadd float [[TMP1]], 0.000000e+00
+; CHECK-NEXT:    ret float [[T34]]
 ;
-entry:
-  %tmp1 = fpext half %x to double
-  %tmp3 = fadd double %tmp1, 0.000000e+00
-  %tmp34 = fptrunc double %tmp3 to float
-  ret float %tmp34
+  %t1 = fpext half %x to double
+  %t3 = fadd double %t1, 0.000000e+00
+  %t34 = fptrunc double %t3 to float
+  ret float %t34
 }
 
 define float @test12(float %x, half %y) nounwind  {
 ; CHECK-LABEL: @test12(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = fpext half [[Y:%.*]] to float
-; CHECK-NEXT:    [[TMP34:%.*]] = fadd float [[TMP0]], [[X:%.*]]
-; CHECK-NEXT:    ret float [[TMP34]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fpext half [[Y:%.*]] to float
+; CHECK-NEXT:    [[T34:%.*]] = fadd float [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    ret float [[T34]]
 ;
-entry:
-  %tmp1 = fpext float %x to double
-  %tmp2 = fpext half %y to double
-  %tmp3 = fadd double %tmp1, %tmp2
-  %tmp34 = fptrunc double %tmp3 to float
-  ret float %tmp34
+  %t1 = fpext float %x to double
+  %t2 = fpext half %y to double
+  %t3 = fadd double %t1, %t2
+  %t34 = fptrunc double %t3 to float
+  ret float %t34
 }
 
 define float @test13(half %x, float %y) nounwind  {
 ; CHECK-LABEL: @test13(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = fpext half [[X:%.*]] to float
-; CHECK-NEXT:    [[TMP56:%.*]] = fdiv float [[TMP0]], [[Y:%.*]]
-; CHECK-NEXT:    ret float [[TMP56]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fpext half [[X:%.*]] to float
+; CHECK-NEXT:    [[T56:%.*]] = fdiv float [[TMP1]], [[Y:%.*]]
+; CHECK-NEXT:    ret float [[T56]]
 ;
-entry:
-  %tmp1 = fpext half %x to double
-  %tmp23 = fpext float %y to double
-  %tmp5 = fdiv double %tmp1, %tmp23
-  %tmp56 = fptrunc double %tmp5 to float
-  ret float %tmp56
+  %t1 = fpext half %x to double
+  %t23 = fpext float %y to double
+  %t5 = fdiv double %t1, %t23
+  %t56 = fptrunc double %t5 to float
+  ret float %t56
 }
 
 define float @test14(float %x, half %y) nounwind  {
 ; CHECK-LABEL: @test14(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = fpext half [[Y:%.*]] to float
-; CHECK-NEXT:    [[TMP56:%.*]] = fdiv float [[X:%.*]], [[TMP0]]
-; CHECK-NEXT:    ret float [[TMP56]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fpext half [[Y:%.*]] to float
+; CHECK-NEXT:    [[T56:%.*]] = fdiv float [[X:%.*]], [[TMP1]]
+; CHECK-NEXT:    ret float [[T56]]
 ;
-entry:
-  %tmp1 = fpext float %x to double
-  %tmp23 = fpext half %y to double
-  %tmp5 = fdiv double %tmp1, %tmp23
-  %tmp56 = fptrunc double %tmp5 to float
-  ret float %tmp56
+  %t1 = fpext float %x to double
+  %t23 = fpext half %y to double
+  %t5 = fdiv double %t1, %t23
+  %t56 = fptrunc double %t5 to float
+  ret float %t56
 }
 
 define float @test15(half %x, half %y) nounwind  {
 ; CHECK-LABEL: @test15(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = fpext half [[X:%.*]] to float
-; CHECK-NEXT:    [[TMP1:%.*]] = fpext half [[Y:%.*]] to float
-; CHECK-NEXT:    [[TMP56:%.*]] = fdiv float [[TMP0]], [[TMP1]]
-; CHECK-NEXT:    ret float [[TMP56]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fpext half [[X:%.*]] to float
+; CHECK-NEXT:    [[TMP2:%.*]] = fpext half [[Y:%.*]] to float
+; CHECK-NEXT:    [[T56:%.*]] = fdiv float [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret float [[T56]]
 ;
-entry:
-  %tmp1 = fpext half %x to double
-  %tmp23 = fpext half %y to double
-  %tmp5 = fdiv double %tmp1, %tmp23
-  %tmp56 = fptrunc double %tmp5 to float
-  ret float %tmp56
+  %t1 = fpext half %x to double
+  %t23 = fpext half %y to double
+  %t5 = fdiv double %t1, %t23
+  %t56 = fptrunc double %t5 to float
+  ret float %t56
 }
 
 define float @test16(half %x, float %y) nounwind  {
 ; CHECK-LABEL: @test16(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = fpext half [[X:%.*]] to float
-; CHECK-NEXT:    [[TMP1:%.*]] = frem float [[TMP0]], [[Y:%.*]]
-; CHECK-NEXT:    ret float [[TMP1]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fpext half [[X:%.*]] to float
+; CHECK-NEXT:    [[TMP2:%.*]] = frem float [[TMP1]], [[Y:%.*]]
+; CHECK-NEXT:    ret float [[TMP2]]
 ;
-entry:
-  %tmp1 = fpext half %x to double
-  %tmp23 = fpext float %y to double
-  %tmp5 = frem double %tmp1, %tmp23
-  %tmp56 = fptrunc double %tmp5 to float
-  ret float %tmp56
+  %t1 = fpext half %x to double
+  %t23 = fpext float %y to double
+  %t5 = frem double %t1, %t23
+  %t56 = fptrunc double %t5 to float
+  ret float %t56
 }
 
 define float @test17(float %x, half %y) nounwind  {
 ; CHECK-LABEL: @test17(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = fpext half [[Y:%.*]] to float
-; CHECK-NEXT:    [[TMP1:%.*]] = frem float [[X:%.*]], [[TMP0]]
-; CHECK-NEXT:    ret float [[TMP1]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fpext half [[Y:%.*]] to float
+; CHECK-NEXT:    [[TMP2:%.*]] = frem float [[X:%.*]], [[TMP1]]
+; CHECK-NEXT:    ret float [[TMP2]]
 ;
-entry:
-  %tmp1 = fpext float %x to double
-  %tmp23 = fpext half %y to double
-  %tmp5 = frem double %tmp1, %tmp23
-  %tmp56 = fptrunc double %tmp5 to float
-  ret float %tmp56
+  %t1 = fpext float %x to double
+  %t23 = fpext half %y to double
+  %t5 = frem double %t1, %t23
+  %t56 = fptrunc double %t5 to float
+  ret float %t56
 }
 
 define float @test18(half %x, half %y) nounwind  {
 ; CHECK-LABEL: @test18(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = frem half [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[TMP56:%.*]] = fpext half [[TMP0]] to float
-; CHECK-NEXT:    ret float [[TMP56]]
+; CHECK-NEXT:    [[TMP1:%.*]] = frem half [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[T56:%.*]] = fpext half [[TMP1]] to float
+; CHECK-NEXT:    ret float [[T56]]
 ;
-entry:
-  %tmp1 = fpext half %x to double
-  %tmp23 = fpext half %y to double
-  %tmp5 = frem double %tmp1, %tmp23
-  %tmp56 = fptrunc double %tmp5 to float
-  ret float %tmp56
+  %t1 = fpext half %x to double
+  %t23 = fpext half %y to double
+  %t5 = frem double %t1, %t23
+  %t56 = fptrunc double %t5 to float
+  ret float %t56
 }
diff --git a/llvm/test/Transforms/InstCombine/fptrunc.ll b/llvm/test/Transforms/InstCombine/fptrunc.ll
new file mode 100644
index 000000000..e8038f4b8
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/fptrunc.ll
@@ -0,0 +1,138 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define float @fadd_fpext_op0(float %x, double %y) {
+; CHECK-LABEL: @fadd_fpext_op0(
+; CHECK-NEXT:    [[EXT:%.*]] = fpext float [[X:%.*]] to double
+; CHECK-NEXT:    [[BO:%.*]] = fadd reassoc double [[EXT]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = fptrunc double [[BO]] to float
+; CHECK-NEXT:    ret float [[R]]
+;
+  %ext = fpext float %x to double
+  %bo = fadd reassoc double %ext, %y
+  %r = fptrunc double %bo to float
+  ret float %r
+}
+
+define half @fsub_fpext_op1(half %x, double %y) {
+; CHECK-LABEL: @fsub_fpext_op1(
+; CHECK-NEXT:    [[EXT:%.*]] = fpext half [[X:%.*]] to double
+; CHECK-NEXT:    [[BO:%.*]] = fsub reassoc double [[Y:%.*]], [[EXT]]
+; CHECK-NEXT:    [[R:%.*]] = fptrunc double [[BO]] to half
+; CHECK-NEXT:    ret half [[R]]
+;
+  %ext = fpext half %x to double
+  %bo = fsub reassoc double %y, %ext
+  %r = fptrunc double %bo to half
+  ret half %r
+}
+
+define <2 x float> @fdiv_constant_op0(<2 x double> %x) {
+; CHECK-LABEL: @fdiv_constant_op0(
+; CHECK-NEXT:    [[BO:%.*]] = fdiv reassoc <2 x double> <double 4.210000e+01, double -1.000000e-01>, [[X:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = fptrunc <2 x double> [[BO]] to <2 x float>
+; CHECK-NEXT:    ret <2 x float> [[R]]
+;
+  %bo = fdiv reassoc <2 x double> <double 42.1, double -0.1>, %x
+  %r = fptrunc <2 x double> %bo to <2 x float>
+  ret <2 x float> %r
+}
+
+define <2 x half> @fmul_constant_op1(<2 x float> %x) {
+; CHECK-LABEL: @fmul_constant_op1(
+; CHECK-NEXT:    [[BO:%.*]] = fmul reassoc <2 x float> [[X:%.*]], <float 0x47EFFFFFE0000000, float 5.000000e-01>
+; CHECK-NEXT:    [[R:%.*]] = fptrunc <2 x float> [[BO]] to <2 x half>
+; CHECK-NEXT:    ret <2 x half> [[R]]
+;
+  %bo = fmul reassoc <2 x float> %x, <float 0x47efffffe0000000, float 0.5>
+  %r = fptrunc <2 x float> %bo to <2 x half>
+  ret <2 x half> %r
+}
+
+define float @fptrunc_select_true_val(float %x, double %y, i1 %cond) {
+; CHECK-LABEL: @fptrunc_select_true_val(
+; CHECK-NEXT:    [[TMP1:%.*]] = fptrunc double [[Y:%.*]] to float
+; CHECK-NEXT:    [[NARROW_SEL:%.*]] = select fast i1 [[COND:%.*]], float [[TMP1]], float [[X:%.*]]
+; CHECK-NEXT:    ret float [[NARROW_SEL]]
+;
+  %e = fpext float %x to double
+  %sel = select fast i1 %cond, double %y, double %e
+  %r = fptrunc double %sel to float
+  ret float %r
+}
+
+define <2 x float> @fptrunc_select_false_val(<2 x float> %x, <2 x double> %y, <2 x i1> %cond) {
+; CHECK-LABEL: @fptrunc_select_false_val(
+; CHECK-NEXT:    [[TMP1:%.*]] = fptrunc <2 x double> [[Y:%.*]] to <2 x float>
+; CHECK-NEXT:    [[NARROW_SEL:%.*]] = select nnan <2 x i1> [[COND:%.*]], <2 x float> [[X:%.*]], <2 x float> [[TMP1]]
+; CHECK-NEXT:    ret <2 x float> [[NARROW_SEL]]
+;
+  %e = fpext <2 x float> %x to <2 x double>
+  %sel = select nnan <2 x i1> %cond, <2 x double> %e, <2 x double> %y
+  %r = fptrunc <2 x double> %sel to <2 x float>
+  ret <2 x float> %r
+}
+
+declare void @use(float)
+
+define half @fptrunc_select_true_val_extra_use(half %x, float %y, i1 %cond) {
+; CHECK-LABEL: @fptrunc_select_true_val_extra_use(
+; CHECK-NEXT:    [[E:%.*]] = fpext half [[X:%.*]] to float
+; CHECK-NEXT:    call void @use(float [[E]])
+; CHECK-NEXT:    [[TMP1:%.*]] = fptrunc float [[Y:%.*]] to half
+; CHECK-NEXT:    [[NARROW_SEL:%.*]] = select ninf i1 [[COND:%.*]], half [[TMP1]], half [[X]]
+; CHECK-NEXT:    ret half [[NARROW_SEL]]
+;
+  %e = fpext half %x to float
+  call void @use(float %e)
+  %sel = select ninf i1 %cond, float %y, float %e
+  %r = fptrunc float %sel to half
+  ret half %r
+}
+
+; Negative test - this would require an extra instruction.
+
+define half @fptrunc_select_true_val_extra_use_2(half %x, float %y, i1 %cond) {
+; CHECK-LABEL: @fptrunc_select_true_val_extra_use_2(
+; CHECK-NEXT:    [[E:%.*]] = fpext half [[X:%.*]] to float
+; CHECK-NEXT:    [[SEL:%.*]] = select ninf i1 [[COND:%.*]], float [[Y:%.*]], float [[E]]
+; CHECK-NEXT:    call void @use(float [[SEL]])
+; CHECK-NEXT:    [[R:%.*]] = fptrunc float [[SEL]] to half
+; CHECK-NEXT:    ret half [[R]]
+;
+  %e = fpext half %x to float
+  %sel = select ninf i1 %cond, float %y, float %e
+  call void @use(float %sel)
+  %r = fptrunc float %sel to half
+  ret half %r
+}
+
+; Negative test - the extend must be from the same source type as the result of the trunc.
+
+define float @fptrunc_select_true_val_type_mismatch(half %x, double %y, i1 %cond) {
+; CHECK-LABEL: @fptrunc_select_true_val_type_mismatch(
+; CHECK-NEXT:    [[E:%.*]] = fpext half [[X:%.*]] to double
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[COND:%.*]], double [[Y:%.*]], double [[E]]
+; CHECK-NEXT:    [[R:%.*]] = fptrunc double [[SEL]] to float
+; CHECK-NEXT:    ret float [[R]]
+;
+  %e = fpext half %x to double
+  %sel = select i1 %cond, double %y, double %e
+  %r = fptrunc double %sel to float
+  ret float %r
+}
+
+; Negative test - but given enough FMF, should this be folded?
+
+define float @fptrunc_select_true_val_type_mismatch_fast(half %x, double %y, i1 %cond) {
+; CHECK-LABEL: @fptrunc_select_true_val_type_mismatch_fast(
+; CHECK-NEXT:    [[E:%.*]] = fpext half [[X:%.*]] to double
+; CHECK-NEXT:    [[SEL:%.*]] = select fast i1 [[COND:%.*]], double [[Y:%.*]], double [[E]]
+; CHECK-NEXT:    [[R:%.*]] = fptrunc double [[SEL]] to float
+; CHECK-NEXT:    ret float [[R]]
+;
+  %e = fpext half %x to double
+  %sel = select fast i1 %cond, double %y, double %e
+  %r = fptrunc double %sel to float
+  ret float %r
+}
diff --git a/llvm/test/Transforms/InstCombine/freeze.ll b/llvm/test/Transforms/InstCombine/freeze.ll
new file mode 100644
index 000000000..bbd3f1201
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/freeze.ll
@@ -0,0 +1,20 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define i32 @fold(i32 %x) {
+; CHECK-LABEL: @fold(
+; CHECK-NEXT:    [[Y:%.*]] = freeze i32 [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[Y]]
+;
+  %y = freeze i32 %x
+  %z = freeze i32 %y
+  ret i32 %z
+}
+
+define i32 @make_const() {
+; CHECK-LABEL: @make_const(
+; CHECK-NEXT:    ret i32 10
+;
+  %x = freeze i32 10
+  ret i32 %x
+}
diff --git a/llvm/test/Transforms/InstCombine/fsub.ll b/llvm/test/Transforms/InstCombine/fsub.ll
index 903f9d979..f2266dd17 100644
--- a/llvm/test/Transforms/InstCombine/fsub.ll
+++ b/llvm/test/Transforms/InstCombine/fsub.ll
@@ -404,3 +404,375 @@ define float @PR37605(float %conv) {
   ret float %sub
 }
 
+define double @fsub_fdiv_fneg1(double %x, double %y, double %z) {
+; CHECK-LABEL: @fsub_fdiv_fneg1(
+; CHECK-NEXT:    [[TMP1:%.*]] = fdiv double [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = fadd double [[TMP1]], [[Z:%.*]]
+; CHECK-NEXT:    ret double [[R]]
+;
+  %neg = fsub double -0.000000e+00, %x
+  %div = fdiv double %neg, %y
+  %r = fsub double %z, %div
+  ret double %r
+}
+
+define <2 x double> @fsub_fdiv_fneg2(<2 x double> %x, <2 x double> %y, <2 x double> %z) {
+; CHECK-LABEL: @fsub_fdiv_fneg2(
+; CHECK-NEXT:    [[TMP1:%.*]] = fdiv <2 x double> [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = fadd <2 x double> [[TMP1]], [[Z:%.*]]
+; CHECK-NEXT:    ret <2 x double> [[R]]
+;
+  %neg = fsub <2 x double> <double -0.0, double -0.0>, %x
+  %div = fdiv <2 x double> %y, %neg
+  %r = fsub <2 x double> %z, %div
+  ret <2 x double> %r
+}
+
+define double @fsub_fmul_fneg1(double %x, double %y, double %z) {
+; CHECK-LABEL: @fsub_fmul_fneg1(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul double [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = fadd double [[TMP1]], [[Z:%.*]]
+; CHECK-NEXT:    ret double [[R]]
+;
+  %neg = fsub double -0.000000e+00, %x
+  %mul = fmul double %neg, %y
+  %r = fsub double %z, %mul
+  ret double %r
+}
+
+define double @fsub_fmul_fneg2(double %x, double %y, double %z) {
+; CHECK-LABEL: @fsub_fmul_fneg2(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul double [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = fadd double [[TMP1]], [[Z:%.*]]
+; CHECK-NEXT:    ret double [[R]]
+;
+  %neg = fsub double -0.000000e+00, %x
+  %mul = fmul double %y, %neg
+  %r = fsub double %z, %mul
+  ret double %r
+}
+
+define float @fsub_fdiv_fneg1_extra_use(float %x, float %y, float %z) {
+; CHECK-LABEL: @fsub_fdiv_fneg1_extra_use(
+; CHECK-NEXT:    [[NEG:%.*]] = fsub float -0.000000e+00, [[X:%.*]]
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv float [[NEG]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use(float [[DIV]])
+; CHECK-NEXT:    [[R:%.*]] = fsub float [[Z:%.*]], [[DIV]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %neg = fsub float -0.000000e+00, %x
+  %div = fdiv float %neg, %y
+  call void @use(float %div)
+  %r = fsub float %z, %div
+  ret float %r
+}
+
+define float @fsub_fdiv_fneg2_extra_use(float %x, float %y, float %z) {
+; CHECK-LABEL: @fsub_fdiv_fneg2_extra_use(
+; CHECK-NEXT:    [[NEG:%.*]] = fsub float -0.000000e+00, [[X:%.*]]
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv float [[Y:%.*]], [[NEG]]
+; CHECK-NEXT:    call void @use(float [[DIV]])
+; CHECK-NEXT:    [[R:%.*]] = fsub float [[Z:%.*]], [[DIV]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %neg = fsub float -0.000000e+00, %x
+  %div = fdiv float %y, %neg
+  call void @use(float %div)
+  %r = fsub float %z, %div
+  ret float %r
+}
+
+declare void @use_vec(<2 x float>)
+
+define <2 x float> @fsub_fmul_fneg1_extra_use(<2 x float> %x, <2 x float> %y, <2 x float> %z) {
+; CHECK-LABEL: @fsub_fmul_fneg1_extra_use(
+; CHECK-NEXT:    [[NEG:%.*]] = fsub <2 x float> <float -0.000000e+00, float -0.000000e+00>, [[X:%.*]]
+; CHECK-NEXT:    [[MUL:%.*]] = fmul <2 x float> [[NEG]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use_vec(<2 x float> [[MUL]])
+; CHECK-NEXT:    [[R:%.*]] = fsub <2 x float> [[Z:%.*]], [[MUL]]
+; CHECK-NEXT:    ret <2 x float> [[R]]
+;
+  %neg = fsub <2 x float> <float -0.0, float -0.0>, %x
+  %mul = fmul <2 x float> %neg, %y
+  call void @use_vec(<2 x float> %mul)
+  %r = fsub <2 x float> %z, %mul
+  ret <2 x float> %r
+}
+
+define float @fsub_fmul_fneg2_extra_use(float %x, float %y, float %z) {
+; CHECK-LABEL: @fsub_fmul_fneg2_extra_use(
+; CHECK-NEXT:    [[NEG:%.*]] = fsub float -0.000000e+00, [[X:%.*]]
+; CHECK-NEXT:    [[MUL:%.*]] = fmul float [[NEG]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use(float [[MUL]])
+; CHECK-NEXT:    [[R:%.*]] = fsub float [[Z:%.*]], [[MUL]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %neg = fsub float -0.000000e+00, %x
+  %mul = fmul float %y, %neg
+  call void @use(float %mul)
+  %r = fsub float %z, %mul
+  ret float %r
+}
+
+define float @fsub_fdiv_fneg1_extra_use2(float %x, float %y, float %z) {
+; CHECK-LABEL: @fsub_fdiv_fneg1_extra_use2(
+; CHECK-NEXT:    [[NEG:%.*]] = fsub float -0.000000e+00, [[X:%.*]]
+; CHECK-NEXT:    call void @use(float [[NEG]])
+; CHECK-NEXT:    [[TMP1:%.*]] = fdiv float [[X]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = fadd float [[TMP1]], [[Z:%.*]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %neg = fsub float -0.000000e+00, %x
+  call void @use(float %neg)
+  %div = fdiv float %neg, %y
+  %r = fsub float %z, %div
+  ret float %r
+}
+
+define float @fsub_fdiv_fneg2_extra_use2(float %x, float %y, float %z) {
+; CHECK-LABEL: @fsub_fdiv_fneg2_extra_use2(
+; CHECK-NEXT:    [[NEG:%.*]] = fsub float -0.000000e+00, [[X:%.*]]
+; CHECK-NEXT:    call void @use(float [[NEG]])
+; CHECK-NEXT:    [[TMP1:%.*]] = fdiv float [[Y:%.*]], [[X]]
+; CHECK-NEXT:    [[R:%.*]] = fadd float [[TMP1]], [[Z:%.*]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %neg = fsub float -0.000000e+00, %x
+  call void @use(float %neg)
+  %div = fdiv float %y, %neg
+  %r = fsub float %z, %div
+  ret float %r
+}
+
+define <2 x float> @fsub_fmul_fneg1_extra_use2(<2 x float> %x, <2 x float> %y, <2 x float> %z) {
+; CHECK-LABEL: @fsub_fmul_fneg1_extra_use2(
+; CHECK-NEXT:    [[NEG:%.*]] = fsub <2 x float> <float -0.000000e+00, float -0.000000e+00>, [[X:%.*]]
+; CHECK-NEXT:    call void @use_vec(<2 x float> [[NEG]])
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul <2 x float> [[X]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = fadd <2 x float> [[TMP1]], [[Z:%.*]]
+; CHECK-NEXT:    ret <2 x float> [[R]]
+;
+  %neg = fsub <2 x float> <float -0.0, float -0.0>, %x
+  call void @use_vec(<2 x float> %neg)
+  %mul = fmul <2 x float> %neg, %y
+  %r = fsub <2 x float> %z, %mul
+  ret <2 x float> %r
+}
+
+define float @fsub_fmul_fneg2_extra_use2(float %x, float %y, float %z) {
+; CHECK-LABEL: @fsub_fmul_fneg2_extra_use2(
+; CHECK-NEXT:    [[NEG:%.*]] = fsub float -0.000000e+00, [[X:%.*]]
+; CHECK-NEXT:    call void @use(float [[NEG]])
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul float [[X]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = fadd float [[TMP1]], [[Z:%.*]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %neg = fsub float -0.000000e+00, %x
+  call void @use(float %neg)
+  %mul = fmul float %y, %neg
+  %r = fsub float %z, %mul
+  ret float %r
+}
+
+define float @fsub_fdiv_fneg1_extra_use3(float %x, float %y, float %z) {
+; CHECK-LABEL: @fsub_fdiv_fneg1_extra_use3(
+; CHECK-NEXT:    [[NEG:%.*]] = fsub float -0.000000e+00, [[X:%.*]]
+; CHECK-NEXT:    call void @use(float [[NEG]])
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv float [[NEG]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use(float [[DIV]])
+; CHECK-NEXT:    [[R:%.*]] = fsub float [[Z:%.*]], [[DIV]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %neg = fsub float -0.000000e+00, %x
+  call void @use(float %neg)
+  %div = fdiv float %neg, %y
+  call void @use(float %div)
+  %r = fsub float %z, %div
+  ret float %r
+}
+
+define float @fsub_fdiv_fneg2_extra_use3(float %x, float %y, float %z) {
+; CHECK-LABEL: @fsub_fdiv_fneg2_extra_use3(
+; CHECK-NEXT:    [[NEG:%.*]] = fsub float -0.000000e+00, [[X:%.*]]
+; CHECK-NEXT:    call void @use(float [[NEG]])
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv float [[Y:%.*]], [[NEG]]
+; CHECK-NEXT:    call void @use(float [[DIV]])
+; CHECK-NEXT:    [[R:%.*]] = fsub float [[Z:%.*]], [[DIV]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %neg = fsub float -0.000000e+00, %x
+  call void @use(float %neg)
+  %div = fdiv float %y, %neg
+  call void @use(float %div)
+  %r = fsub float %z, %div
+  ret float %r
+}
+
+define <2 x float> @fsub_fmul_fneg1_extra_use3(<2 x float> %x, <2 x float> %y, <2 x float> %z) {
+; CHECK-LABEL: @fsub_fmul_fneg1_extra_use3(
+; CHECK-NEXT:    [[NEG:%.*]] = fsub <2 x float> <float -0.000000e+00, float -0.000000e+00>, [[X:%.*]]
+; CHECK-NEXT:    call void @use_vec(<2 x float> [[NEG]])
+; CHECK-NEXT:    [[MUL:%.*]] = fmul <2 x float> [[NEG]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use_vec(<2 x float> [[MUL]])
+; CHECK-NEXT:    [[R:%.*]] = fsub <2 x float> [[Z:%.*]], [[MUL]]
+; CHECK-NEXT:    ret <2 x float> [[R]]
+;
+  %neg = fsub <2 x float> <float -0.0, float -0.0>, %x
+  call void @use_vec(<2 x float> %neg)
+  %mul = fmul <2 x float> %neg, %y
+  call void @use_vec(<2 x float> %mul)
+  %r = fsub <2 x float> %z, %mul
+  ret <2 x float> %r
+}
+
+define float @fsub_fmul_fneg2_extra_use3(float %x, float %y, float %z) {
+; CHECK-LABEL: @fsub_fmul_fneg2_extra_use3(
+; CHECK-NEXT:    [[NEG:%.*]] = fsub float -0.000000e+00, [[X:%.*]]
+; CHECK-NEXT:    call void @use(float [[NEG]])
+; CHECK-NEXT:    [[MUL:%.*]] = fmul float [[NEG]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use(float [[MUL]])
+; CHECK-NEXT:    [[R:%.*]] = fsub float [[Z:%.*]], [[MUL]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %neg = fsub float -0.000000e+00, %x
+  call void @use(float %neg)
+  %mul = fmul float %y, %neg
+  call void @use(float %mul)
+  %r = fsub float %z, %mul
+  ret float %r
+}
+
+define float @fsub_fsub(float %x, float %y, float %z) {
+; CHECK-LABEL: @fsub_fsub(
+; CHECK-NEXT:    [[XY:%.*]] = fsub float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[XYZ:%.*]] = fsub float [[XY]], [[Z:%.*]]
+; CHECK-NEXT:    ret float [[XYZ]]
+;
+  %xy = fsub float %x, %y
+  %xyz = fsub float %xy, %z
+  ret float %xyz
+}
+
+define float @fsub_fsub_nsz(float %x, float %y, float %z) {
+; CHECK-LABEL: @fsub_fsub_nsz(
+; CHECK-NEXT:    [[XY:%.*]] = fsub float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[XYZ:%.*]] = fsub nsz float [[XY]], [[Z:%.*]]
+; CHECK-NEXT:    ret float [[XYZ]]
+;
+  %xy = fsub float %x, %y
+  %xyz = fsub nsz float %xy, %z
+  ret float %xyz
+}
+
+define float @fsub_fsub_reassoc(float %x, float %y, float %z) {
+; CHECK-LABEL: @fsub_fsub_reassoc(
+; CHECK-NEXT:    [[XY:%.*]] = fsub float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[XYZ:%.*]] = fsub reassoc float [[XY]], [[Z:%.*]]
+; CHECK-NEXT:    ret float [[XYZ]]
+;
+  %xy = fsub float %x, %y
+  %xyz = fsub reassoc float %xy, %z
+  ret float %xyz
+}
+
+define float @fsub_fsub_nsz_reassoc(float %x, float %y, float %z) {
+; CHECK-LABEL: @fsub_fsub_nsz_reassoc(
+; CHECK-NEXT:    [[XY:%.*]] = fsub float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[XYZ:%.*]] = fsub reassoc nsz float [[XY]], [[Z:%.*]]
+; CHECK-NEXT:    ret float [[XYZ]]
+;
+  %xy = fsub float %x, %y
+  %xyz = fsub nsz reassoc float %xy, %z
+  ret float %xyz
+}
+
+define <2 x double> @fsub_fsub_fast_vec(<2 x double> %x, <2 x double> %y, <2 x double> %z) {
+; CHECK-LABEL: @fsub_fsub_fast_vec(
+; CHECK-NEXT:    [[XY:%.*]] = fsub fast <2 x double> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[XYZ:%.*]] = fsub fast <2 x double> [[XY]], [[Z:%.*]]
+; CHECK-NEXT:    ret <2 x double> [[XYZ]]
+;
+  %xy = fsub fast <2 x double> %x, %y
+  %xyz = fsub fast reassoc <2 x double> %xy, %z
+  ret <2 x double> %xyz
+}
+
+define float @fsub_fsub_nsz_reassoc_extra_use(float %x, float %y, float %z) {
+; CHECK-LABEL: @fsub_fsub_nsz_reassoc_extra_use(
+; CHECK-NEXT:    [[XY:%.*]] = fsub float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use(float [[XY]])
+; CHECK-NEXT:    [[XYZ:%.*]] = fsub reassoc nsz float [[XY]], [[Z:%.*]]
+; CHECK-NEXT:    ret float [[XYZ]]
+;
+  %xy = fsub float %x, %y
+  call void @use(float %xy)
+  %xyz = fsub nsz reassoc float %xy, %z
+  ret float %xyz
+}
+
+define float @fneg_fsub(float %x, float %y) {
+; CHECK-LABEL: @fneg_fsub(
+; CHECK-NEXT:    [[NEGX:%.*]] = fneg float [[X:%.*]]
+; CHECK-NEXT:    [[SUB:%.*]] = fsub float [[NEGX]], [[Y:%.*]]
+; CHECK-NEXT:    ret float [[SUB]]
+;
+  %negx = fneg float %x
+  %sub = fsub float %negx, %y
+  ret float %sub
+}
+
+define float @fneg_fsub_nsz(float %x, float %y) {
+; CHECK-LABEL: @fneg_fsub_nsz(
+; CHECK-NEXT:    [[NEGX:%.*]] = fneg float [[X:%.*]]
+; CHECK-NEXT:    [[SUB:%.*]] = fsub nsz float [[NEGX]], [[Y:%.*]]
+; CHECK-NEXT:    ret float [[SUB]]
+;
+  %negx = fneg float %x
+  %sub = fsub nsz float %negx, %y
+  ret float %sub
+}
+
+define float @fake_fneg_fsub_fast(float %x, float %y) {
+; CHECK-LABEL: @fake_fneg_fsub_fast(
+; CHECK-NEXT:    [[NEGX:%.*]] = fsub float -0.000000e+00, [[X:%.*]]
+; CHECK-NEXT:    [[SUB:%.*]] = fsub fast float [[NEGX]], [[Y:%.*]]
+; CHECK-NEXT:    ret float [[SUB]]
+;
+  %negx = fsub float -0.0, %x
+  %sub = fsub fast float %negx, %y
+  ret float %sub
+}
+
+define float @fake_fneg_fsub_fast_extra_use(float %x, float %y) {
+; CHECK-LABEL: @fake_fneg_fsub_fast_extra_use(
+; CHECK-NEXT:    [[NEGX:%.*]] = fsub float -0.000000e+00, [[X:%.*]]
+; CHECK-NEXT:    call void @use(float [[NEGX]])
+; CHECK-NEXT:    [[SUB:%.*]] = fsub fast float [[NEGX]], [[Y:%.*]]
+; CHECK-NEXT:    ret float [[SUB]]
+;
+  %negx = fsub float -0.0, %x
+  call void @use(float %negx)
+  %sub = fsub fast float %negx, %y
+  ret float %sub
+}
+
+define <2 x float> @fake_fneg_fsub_vec(<2 x float> %x, <2 x float> %y) {
+; CHECK-LABEL: @fake_fneg_fsub_vec(
+; CHECK-NEXT:    [[NEGX:%.*]] = fsub <2 x float> <float -0.000000e+00, float -0.000000e+00>, [[X:%.*]]
+; CHECK-NEXT:    [[SUB:%.*]] = fsub nsz <2 x float> [[NEGX]], [[Y:%.*]]
+; CHECK-NEXT:    ret <2 x float> [[SUB]]
+;
+  %negx = fsub <2 x float> <float -0.0, float -0.0>, %x
+  %sub = fsub nsz <2 x float> %negx, %y
+  ret <2 x float> %sub
+}
+
+define float @fneg_fsub_constant(float %x) {
+; CHECK-LABEL: @fneg_fsub_constant(
+; CHECK-NEXT:    [[SUB:%.*]] = fsub nsz float -4.200000e+01, [[X:%.*]]
+; CHECK-NEXT:    ret float [[SUB]]
+;
+  %negx = fneg float %x
+  %sub = fsub nsz float %negx, 42.0
+  ret float %sub
+}
diff --git a/llvm/test/Transforms/InstCombine/gc.relocate.ll b/llvm/test/Transforms/InstCombine/gc.relocate.ll
index 78b3b5f42..f8eade6f1 100644
--- a/llvm/test/Transforms/InstCombine/gc.relocate.ll
+++ b/llvm/test/Transforms/InstCombine/gc.relocate.ll
@@ -52,6 +52,17 @@ entry:
   ret <2 x i8 addrspace(1)*> %obj.relocated
 }
 
+define i32 addrspace(1)* @canonical_base(i32 addrspace(1)* %dparam) gc "statepoint-example" {
+; Checks that a nonnull pointer
+; CHECK-LABEL: @canonical_base
+; CHECK: (token %tok, i32 7, i32 7) ; (%dparam, %dparam)
+entry:
+  %tok = tail call token (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 0, i32 addrspace(1)* %dparam, i32 addrspace(1)* %dparam)
+  %relocate = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %tok,  i32 7, i32 8)
+  ret i32 addrspace(1)* %relocate
+}
+
+
 declare void @do_safepoint()
 
 declare token @llvm.experimental.gc.statepoint.p0f_isVoidf(i64, i32, void ()*, i32, i32, ...)
diff --git a/llvm/test/Transforms/InstCombine/gep-alias.ll b/llvm/test/Transforms/InstCombine/gep-alias.ll
new file mode 100644
index 000000000..890b61688
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/gep-alias.ll
@@ -0,0 +1,15 @@
+; RUN: opt -S -instcombine -o - %s | FileCheck %s
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64-unknown-linux-android10000"
+
+@x.hwasan = private global { [3 x i32], [4 x i8] } { [3 x i32] [i32 42, i32 57, i32 10], [4 x i8] c"\00\00\00\87" }, align 16
+@x = alias [3 x i32], inttoptr (i64 add (i64 ptrtoint ({ [3 x i32], [4 x i8] }* @x.hwasan to i64), i64 -8718968878589280256) to [3 x i32]*)
+
+define i32 @f(i64 %i) {
+entry:
+  ; CHECK: getelementptr inbounds [3 x i32], [3 x i32]* @x
+  %arrayidx = getelementptr inbounds [3 x i32], [3 x i32]* @x, i64 0, i64 %i
+  %0 = load i32, i32* %arrayidx
+  ret i32 %0
+}
diff --git a/llvm/test/Transforms/InstCombine/gep-custom-dl.ll b/llvm/test/Transforms/InstCombine/gep-custom-dl.ll
index e22653042..fdadc80c6 100644
--- a/llvm/test/Transforms/InstCombine/gep-custom-dl.ll
+++ b/llvm/test/Transforms/InstCombine/gep-custom-dl.ll
@@ -101,7 +101,7 @@ define i1 @test5({ i32, i32 }* %x, { i32, i32 }* %y) {
 
 define <2 x i1> @test6(<2 x i32> %X, <2 x %S*> %P) nounwind {
 ; CHECK-LABEL: @test6(
-; CHECK-NEXT:    [[C:%.*]] = icmp eq <2 x i32> [[X:%.*]], <i32 1073741823, i32 1073741823>
+; CHECK-NEXT:    [[C:%.*]] = icmp eq <2 x i32> %X, <i32 -1, i32 -1>
 ; CHECK-NEXT:    ret <2 x i1> [[C]]
 ;
   %A = getelementptr inbounds %S, <2 x %S*> %P, <2 x i32> zeroinitializer, <2 x i32> <i32 1, i32 1>, <2 x i32> %X
@@ -110,6 +110,18 @@ define <2 x i1> @test6(<2 x i32> %X, <2 x %S*> %P) nounwind {
   ret <2 x i1> %C
 }
 
+; Same as above, but indices scalarized.
+define <2 x i1> @test6b(<2 x i32> %X, <2 x %S*> %P) nounwind {
+; CHECK-LABEL: @test6b(
+; CHECK-NEXT:    [[C:%.*]] = icmp eq <2 x i32> %X, <i32 -1, i32 -1>
+; CHECK-NEXT:    ret <2 x i1> [[C]]
+;
+  %A = getelementptr inbounds %S, <2 x %S*> %P, i32 0, i32 1, <2 x i32> %X
+  %B = getelementptr inbounds %S, <2 x %S*> %P, i32 0, i32 0
+  %C = icmp eq <2 x i32*> %A, %B
+  ret <2 x i1> %C
+}
+
 @G = external global [3 x i8]
 define i8* @test7(i16 %Idx) {
 ; CHECK-LABEL: @test7(
@@ -152,3 +164,22 @@ define i32 @test10() {
   %B = ptrtoint double* %A to i32
   ret i32 %B
 }
+
+@X_as1 = addrspace(1) global [1000 x i8] zeroinitializer, align 16
+
+define i16 @constant_fold_custom_dl() {
+; CHECK-LABEL: @constant_fold_custom_dl(
+  ; CHECK: ret i16 ptrtoint
+
+entry:
+  %A = bitcast i8 addrspace(1)* getelementptr inbounds ([1000 x i8], [1000 x i8] addrspace(1)* @X_as1, i64 1, i64 0) to i8 addrspace(1)*
+  %B = bitcast i8 addrspace(1)* getelementptr inbounds ([1000 x i8], [1000 x i8] addrspace(1)* @X_as1, i64 0, i64 0) to i8 addrspace(1)*
+
+  %B2 = ptrtoint i8 addrspace(1)* %B to i16
+  %C = sub i16 0, %B2
+  %D = getelementptr i8, i8 addrspace(1)* %A, i16 %C
+  %E = ptrtoint i8 addrspace(1)* %D to i16
+
+  ret i16 %E
+}
+
diff --git a/llvm/test/Transforms/InstCombine/gep-inbounds-null.ll b/llvm/test/Transforms/InstCombine/gep-inbounds-null.ll
new file mode 100644
index 000000000..ba577cc68
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/gep-inbounds-null.ll
@@ -0,0 +1,236 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S < %s -instcombine | FileCheck %s
+
+;; Start by showing the results of constant folding (which doesn't use
+;; the poison implied by gep for the nonnull cases).
+
+define i1 @test_ne_constants_null() {
+; CHECK-LABEL: @test_ne_constants_null(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret i1 false
+;
+entry:
+  %gep = getelementptr inbounds i8, i8* null, i64 0
+  %cnd = icmp ne i8* %gep, null
+  ret i1 %cnd
+}
+
+define i1 @test_ne_constants_nonnull() {
+; CHECK-LABEL: @test_ne_constants_nonnull(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret i1 true
+;
+entry:
+  %gep = getelementptr inbounds i8, i8* null, i64 1
+  %cnd = icmp ne i8* %gep, null
+  ret i1 %cnd
+}
+
+define i1 @test_eq_constants_null() {
+; CHECK-LABEL: @test_eq_constants_null(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret i1 true
+;
+entry:
+  %gep = getelementptr inbounds i8, i8* null, i64 0
+  %cnd = icmp eq i8* %gep, null
+  ret i1 %cnd
+}
+
+define i1 @test_eq_constants_nonnull() {
+; CHECK-LABEL: @test_eq_constants_nonnull(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret i1 false
+;
+entry:
+  %gep = getelementptr inbounds i8, i8* null, i64 1
+  %cnd = icmp eq i8* %gep, null
+  ret i1 %cnd
+}
+
+;; Then show the results for non-constants.  These use the inbounds provided
+;; UB fact to ignore the possible overflow cases.
+
+define i1 @test_ne(i8* %base, i64 %idx) {
+; CHECK-LABEL: @test_ne(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CND:%.*]] = icmp ne i8* [[BASE:%.*]], null
+; CHECK-NEXT:    ret i1 [[CND]]
+;
+entry:
+  %gep = getelementptr inbounds i8, i8* %base, i64 %idx
+  %cnd = icmp ne i8* %gep, null
+  ret i1 %cnd
+}
+
+define i1 @test_eq(i8* %base, i64 %idx) {
+; CHECK-LABEL: @test_eq(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CND:%.*]] = icmp eq i8* [[BASE:%.*]], null
+; CHECK-NEXT:    ret i1 [[CND]]
+;
+entry:
+  %gep = getelementptr inbounds i8, i8* %base, i64 %idx
+  %cnd = icmp eq i8* %gep, null
+  ret i1 %cnd
+}
+
+define <2 x i1> @test_vector_base(<2 x i8*> %base, i64 %idx) {
+; CHECK-LABEL: @test_vector_base(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CND:%.*]] = icmp eq <2 x i8*> [[BASE:%.*]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[CND]]
+;
+entry:
+  %gep = getelementptr inbounds i8, <2 x i8*> %base, i64 %idx
+  %cnd = icmp eq <2 x i8*> %gep, zeroinitializer
+  ret <2 x i1> %cnd
+}
+
+define <2 x i1> @test_vector_index(i8* %base, <2 x i64> %idx) {
+; CHECK-LABEL: @test_vector_index(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <2 x i8*> undef, i8* [[BASE:%.*]], i32 0
+; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <2 x i8*> [[DOTSPLATINSERT]], <2 x i8*> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[CND:%.*]] = icmp eq <2 x i8*> [[DOTSPLAT]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[CND]]
+;
+entry:
+  %gep = getelementptr inbounds i8, i8* %base, <2 x i64> %idx
+  %cnd = icmp eq <2 x i8*> %gep, zeroinitializer
+  ret <2 x i1> %cnd
+}
+
+define <2 x i1> @test_vector_both(<2 x i8*> %base, <2 x i64> %idx) {
+; CHECK-LABEL: @test_vector_both(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CND:%.*]] = icmp eq <2 x i8*> [[BASE:%.*]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[CND]]
+;
+entry:
+  %gep = getelementptr inbounds i8, <2 x i8*> %base, <2 x i64> %idx
+  %cnd = icmp eq <2 x i8*> %gep, zeroinitializer
+  ret <2 x i1> %cnd
+}
+
+;; These two show instsimplify's reasoning getting to the non-zero offsets
+;; before instcombine does.
+
+define i1 @test_eq_pos_idx(i8* %base) {
+; CHECK-LABEL: @test_eq_pos_idx(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret i1 false
+;
+entry:
+  %gep = getelementptr inbounds i8, i8* %base, i64 1
+  %cnd = icmp eq i8* %gep, null
+  ret i1 %cnd
+}
+
+define i1 @test_eq_neg_idx(i8* %base) {
+; CHECK-LABEL: @test_eq_neg_idx(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret i1 false
+;
+entry:
+  %gep = getelementptr inbounds i8, i8* %base, i64 -1
+  %cnd = icmp eq i8* %gep, null
+  ret i1 %cnd
+}
+
+;; Show an example with a zero sized type since that's
+;; a cornercase which keeps getting mentioned.  The GEP
+;; produces %base regardless of the value of the index
+;; expression.
+define i1 @test_size0({}* %base, i64 %idx) {
+; CHECK-LABEL: @test_size0(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CND:%.*]] = icmp ne {}* [[BASE:%.*]], null
+; CHECK-NEXT:    ret i1 [[CND]]
+;
+entry:
+  %gep = getelementptr inbounds {}, {}* %base, i64 %idx
+  %cnd = icmp ne {}* %gep, null
+  ret i1 %cnd
+}
+define i1 @test_size0_nonzero_offset({}* %base) {
+; CHECK-LABEL: @test_size0_nonzero_offset(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CND:%.*]] = icmp ne {}* [[BASE:%.*]], null
+; CHECK-NEXT:    ret i1 [[CND]]
+;
+entry:
+  %gep = getelementptr inbounds {}, {}* %base, i64 15
+  %cnd = icmp ne {}* %gep, null
+  ret i1 %cnd
+}
+
+
+define i1 @test_index_type([10 x i8]* %base, i64 %idx) {
+; CHECK-LABEL: @test_index_type(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CND:%.*]] = icmp eq [10 x i8]* [[BASE:%.*]], null
+; CHECK-NEXT:    ret i1 [[CND]]
+;
+entry:
+  %gep = getelementptr inbounds [10 x i8], [10 x i8]* %base, i64 %idx, i64 %idx
+  %cnd = icmp eq i8* %gep, null
+  ret i1 %cnd
+}
+
+
+;; Finally, some negative tests for sanity checking.
+
+define i1 @neq_noinbounds(i8* %base, i64 %idx) {
+; CHECK-LABEL: @neq_noinbounds(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i8, i8* [[BASE:%.*]], i64 [[IDX:%.*]]
+; CHECK-NEXT:    [[CND:%.*]] = icmp ne i8* [[GEP]], null
+; CHECK-NEXT:    ret i1 [[CND]]
+;
+entry:
+  %gep = getelementptr i8, i8* %base, i64 %idx
+  %cnd = icmp ne i8* %gep, null
+  ret i1 %cnd
+}
+
+define i1 @neg_objectatnull(i8 addrspace(2)* %base, i64 %idx) {
+; CHECK-LABEL: @neg_objectatnull(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i8, i8 addrspace(2)* [[BASE:%.*]], i64 [[IDX:%.*]]
+; CHECK-NEXT:    [[CND:%.*]] = icmp eq i8 addrspace(2)* [[GEP]], null
+; CHECK-NEXT:    ret i1 [[CND]]
+;
+entry:
+  %gep = getelementptr inbounds i8, i8 addrspace(2)* %base, i64 %idx
+  %cnd = icmp eq i8 addrspace(2)* %gep, null
+  ret i1 %cnd
+}
+
+; Test for an assert from trying to create an invalid constantexpr
+; bitcast between different address spaces. The addrspacecast is
+; stripped off and the addrspace(0) null can be treated as invalid.
+; FIXME: This should be able to fold to ret i1 false
+define i1 @invalid_bitcast_icmp_addrspacecast_as0_null(i32 addrspace(5)* %ptr) {
+; CHECK-LABEL: @invalid_bitcast_icmp_addrspacecast_as0_null(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 addrspace(5)* [[PTR:%.*]], addrspacecast (i32* null to i32 addrspace(5)*)
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+bb:
+  %tmp1 = getelementptr inbounds i32, i32 addrspace(5)* %ptr, i32 1
+  %tmp2 = icmp eq i32 addrspace(5)* %tmp1, addrspacecast (i32* null to i32 addrspace(5)*)
+  ret i1 %tmp2
+}
+
+define i1 @invalid_bitcast_icmp_addrspacecast_as0_null_var(i32 addrspace(5)* %ptr, i32 %idx) {
+; CHECK-LABEL: @invalid_bitcast_icmp_addrspacecast_as0_null_var(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 addrspace(5)* [[PTR:%.*]], addrspacecast (i32* null to i32 addrspace(5)*)
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+bb:
+  %tmp1 = getelementptr inbounds i32, i32 addrspace(5)* %ptr, i32 %idx
+  %tmp2 = icmp eq i32 addrspace(5)* %tmp1, addrspacecast (i32* null to i32 addrspace(5)*)
+  ret i1 %tmp2
+}
diff --git a/llvm/test/Transforms/InstCombine/gep-vector.ll b/llvm/test/Transforms/InstCombine/gep-vector.ll
index c0db01eab..432da0aed 100644
--- a/llvm/test/Transforms/InstCombine/gep-vector.ll
+++ b/llvm/test/Transforms/InstCombine/gep-vector.ll
@@ -27,9 +27,12 @@ define <2 x i8*> @vectorindex3() {
   ret <2 x i8*> %1
 }
 
+; Negative test - datalayout's alloc size for the 2 types must match.
+
 define i32* @bitcast_vec_to_array_gep(<7 x i32>* %x, i64 %y, i64 %z) {
 ; CHECK-LABEL: @bitcast_vec_to_array_gep(
-; CHECK-NEXT:    [[GEP:%.*]] = getelementptr <7 x i32>, <7 x i32>* [[X:%.*]], i64 [[Y:%.*]], i64 [[Z:%.*]]
+; CHECK-NEXT:    [[ARR_PTR:%.*]] = bitcast <7 x i32>* [[X:%.*]] to [7 x i32]*
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr [7 x i32], [7 x i32]* [[ARR_PTR]], i64 [[Y:%.*]], i64 [[Z:%.*]]
 ; CHECK-NEXT:    ret i32* [[GEP]]
 ;
   %arr_ptr = bitcast <7 x i32>* %x to [7 x i32]*
@@ -37,9 +40,12 @@ define i32* @bitcast_vec_to_array_gep(<7 x i32>* %x, i64 %y, i64 %z) {
   ret i32* %gep
 }
 
+; Negative test - datalayout's alloc size for the 2 types must match.
+
 define i32* @bitcast_array_to_vec_gep([3 x i32]* %x, i64 %y, i64 %z) {
 ; CHECK-LABEL: @bitcast_array_to_vec_gep(
-; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds [3 x i32], [3 x i32]* [[X:%.*]], i64 [[Y:%.*]], i64 [[Z:%.*]]
+; CHECK-NEXT:    [[VEC_PTR:%.*]] = bitcast [3 x i32]* [[X:%.*]] to <3 x i32>*
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds <3 x i32>, <3 x i32>* [[VEC_PTR]], i64 [[Y:%.*]], i64 [[Z:%.*]]
 ; CHECK-NEXT:    ret i32* [[GEP]]
 ;
   %vec_ptr = bitcast [3 x i32]* %x to <3 x i32>*
@@ -47,11 +53,38 @@ define i32* @bitcast_array_to_vec_gep([3 x i32]* %x, i64 %y, i64 %z) {
   ret i32* %gep
 }
 
+; Sizes and types match - safe to remove bitcast.
+
+define i32* @bitcast_vec_to_array_gep_matching_alloc_size(<4 x i32>* %x, i64 %y, i64 %z) {
+; CHECK-LABEL: @bitcast_vec_to_array_gep_matching_alloc_size(
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr <4 x i32>, <4 x i32>* [[X:%.*]], i64 [[Y:%.*]], i64 [[Z:%.*]]
+; CHECK-NEXT:    ret i32* [[GEP]]
+;
+  %arr_ptr = bitcast <4 x i32>* %x to [4 x i32]*
+  %gep = getelementptr [4 x i32], [4 x i32]* %arr_ptr, i64 %y, i64 %z
+  ret i32* %gep
+}
+
+; Sizes and types match - safe to remove bitcast.
+
+define i32* @bitcast_array_to_vec_gep_matching_alloc_size([4 x i32]* %x, i64 %y, i64 %z) {
+; CHECK-LABEL: @bitcast_array_to_vec_gep_matching_alloc_size(
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds [4 x i32], [4 x i32]* [[X:%.*]], i64 [[Y:%.*]], i64 [[Z:%.*]]
+; CHECK-NEXT:    ret i32* [[GEP]]
+;
+  %vec_ptr = bitcast [4 x i32]* %x to <4 x i32>*
+  %gep = getelementptr inbounds <4 x i32>, <4 x i32>* %vec_ptr, i64 %y, i64 %z
+  ret i32* %gep
+}
+
+; Negative test - datalayout's alloc size for the 2 types must match.
+
 define i32 addrspace(3)* @bitcast_vec_to_array_addrspace(<7 x i32>* %x, i64 %y, i64 %z) {
 ; CHECK-LABEL: @bitcast_vec_to_array_addrspace(
-; CHECK-NEXT:    [[GEP:%.*]] = getelementptr <7 x i32>, <7 x i32>* [[X:%.*]], i64 [[Y:%.*]], i64 [[Z:%.*]]
-; CHECK-NEXT:    [[TMP1:%.*]] = addrspacecast i32* [[GEP]] to i32 addrspace(3)*
-; CHECK-NEXT:    ret i32 addrspace(3)* [[TMP1]]
+; CHECK-NEXT:    [[ARR_PTR:%.*]] = bitcast <7 x i32>* [[X:%.*]] to [7 x i32]*
+; CHECK-NEXT:    [[ASC:%.*]] = addrspacecast [7 x i32]* [[ARR_PTR]] to [7 x i32] addrspace(3)*
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr [7 x i32], [7 x i32] addrspace(3)* [[ASC]], i64 [[Y:%.*]], i64 [[Z:%.*]]
+; CHECK-NEXT:    ret i32 addrspace(3)* [[GEP]]
 ;
   %arr_ptr = bitcast <7 x i32>* %x to [7 x i32]*
   %asc = addrspacecast [7 x i32]* %arr_ptr to [7 x i32] addrspace(3)*
@@ -59,14 +92,45 @@ define i32 addrspace(3)* @bitcast_vec_to_array_addrspace(<7 x i32>* %x, i64 %y,
   ret i32 addrspace(3)* %gep
 }
 
+; Negative test - datalayout's alloc size for the 2 types must match.
+
 define i32 addrspace(3)* @inbounds_bitcast_vec_to_array_addrspace(<7 x i32>* %x, i64 %y, i64 %z) {
 ; CHECK-LABEL: @inbounds_bitcast_vec_to_array_addrspace(
-; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds <7 x i32>, <7 x i32>* [[X:%.*]], i64 [[Y:%.*]], i64 [[Z:%.*]]
-; CHECK-NEXT:    [[TMP1:%.*]] = addrspacecast i32* [[GEP]] to i32 addrspace(3)*
-; CHECK-NEXT:    ret i32 addrspace(3)* [[TMP1]]
+; CHECK-NEXT:    [[ARR_PTR:%.*]] = bitcast <7 x i32>* [[X:%.*]] to [7 x i32]*
+; CHECK-NEXT:    [[ASC:%.*]] = addrspacecast [7 x i32]* [[ARR_PTR]] to [7 x i32] addrspace(3)*
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds [7 x i32], [7 x i32] addrspace(3)* [[ASC]], i64 [[Y:%.*]], i64 [[Z:%.*]]
+; CHECK-NEXT:    ret i32 addrspace(3)* [[GEP]]
 ;
   %arr_ptr = bitcast <7 x i32>* %x to [7 x i32]*
   %asc = addrspacecast [7 x i32]* %arr_ptr to [7 x i32] addrspace(3)*
   %gep = getelementptr inbounds [7 x i32], [7 x i32] addrspace(3)* %asc, i64 %y, i64 %z
   ret i32 addrspace(3)* %gep
 }
+
+; Sizes and types match - safe to remove bitcast.
+
+define i32 addrspace(3)* @bitcast_vec_to_array_addrspace_matching_alloc_size(<4 x i32>* %x, i64 %y, i64 %z) {
+; CHECK-LABEL: @bitcast_vec_to_array_addrspace_matching_alloc_size(
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr <4 x i32>, <4 x i32>* [[X:%.*]], i64 [[Y:%.*]], i64 [[Z:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = addrspacecast i32* [[GEP]] to i32 addrspace(3)*
+; CHECK-NEXT:    ret i32 addrspace(3)* [[TMP1]]
+;
+  %arr_ptr = bitcast <4 x i32>* %x to [4 x i32]*
+  %asc = addrspacecast [4 x i32]* %arr_ptr to [4 x i32] addrspace(3)*
+  %gep = getelementptr [4 x i32], [4 x i32] addrspace(3)* %asc, i64 %y, i64 %z
+  ret i32 addrspace(3)* %gep
+}
+
+; Sizes and types match - safe to remove bitcast.
+
+define i32 addrspace(3)* @inbounds_bitcast_vec_to_array_addrspace_matching_alloc_size(<4 x i32>* %x, i64 %y, i64 %z) {
+; CHECK-LABEL: @inbounds_bitcast_vec_to_array_addrspace_matching_alloc_size(
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds <4 x i32>, <4 x i32>* [[X:%.*]], i64 [[Y:%.*]], i64 [[Z:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = addrspacecast i32* [[GEP]] to i32 addrspace(3)*
+; CHECK-NEXT:    ret i32 addrspace(3)* [[TMP1]]
+;
+  %arr_ptr = bitcast <4 x i32>* %x to [4 x i32]*
+  %asc = addrspacecast [4 x i32]* %arr_ptr to [4 x i32] addrspace(3)*
+  %gep = getelementptr inbounds [4 x i32], [4 x i32] addrspace(3)* %asc, i64 %y, i64 %z
+  ret i32 addrspace(3)* %gep
+}
diff --git a/llvm/test/Transforms/InstCombine/getelementptr.ll b/llvm/test/Transforms/InstCombine/getelementptr.ll
index 566e15f28..36e09cd0a 100644
--- a/llvm/test/Transforms/InstCombine/getelementptr.ll
+++ b/llvm/test/Transforms/InstCombine/getelementptr.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -instcombine -S | FileCheck %s
 
 target datalayout = "e-p:64:64-p1:16:16-p2:32:32:32-p3:64:64:64"
@@ -14,61 +15,72 @@ target datalayout = "e-p:64:64-p1:16:16-p2:32:32:32-p3:64:64:64"
 
 ; Test noop elimination
 define i32* @test1(i32* %I) {
-        %A = getelementptr i32, i32* %I, i64 0
-        ret i32* %A
 ; CHECK-LABEL: @test1(
-; CHECK: ret i32* %I
+; CHECK-NEXT:    ret i32* [[I:%.*]]
+;
+  %A = getelementptr i32, i32* %I, i64 0
+  ret i32* %A
 }
 
 define i32 addrspace(1)* @test1_as1(i32 addrspace(1)* %I) {
+; CHECK-LABEL: @test1_as1(
+; CHECK-NEXT:    ret i32 addrspace(1)* [[I:%.*]]
+;
   %A = getelementptr i32, i32 addrspace(1)* %I, i64 0
   ret i32 addrspace(1)* %A
-; CHECK-LABEL: @test1_as1(
-; CHECK: ret i32 addrspace(1)* %I
 }
 
 ; Test noop elimination
 define i32* @test2(i32* %I) {
-        %A = getelementptr i32, i32* %I
-        ret i32* %A
 ; CHECK-LABEL: @test2(
-; CHECK: ret i32* %I
+; CHECK-NEXT:    ret i32* [[I:%.*]]
+;
+  %A = getelementptr i32, i32* %I
+  ret i32* %A
 }
 
 ; Test that two array indexing geps fold
 define i32* @test3(i32* %I) {
-        %A = getelementptr i32, i32* %I, i64 17
-        %B = getelementptr i32, i32* %A, i64 4
-        ret i32* %B
 ; CHECK-LABEL: @test3(
-; CHECK: getelementptr i32, i32* %I, i64 21
+; CHECK-NEXT:    [[B:%.*]] = getelementptr i32, i32* [[I:%.*]], i64 21
+; CHECK-NEXT:    ret i32* [[B]]
+;
+  %A = getelementptr i32, i32* %I, i64 17
+  %B = getelementptr i32, i32* %A, i64 4
+  ret i32* %B
 }
 
 ; Test that two getelementptr insts fold
 define i32* @test4({ i32 }* %I) {
-        %A = getelementptr { i32 }, { i32 }* %I, i64 1
-        %B = getelementptr { i32 }, { i32 }* %A, i64 0, i32 0
-        ret i32* %B
 ; CHECK-LABEL: @test4(
-; CHECK: getelementptr { i32 }, { i32 }* %I, i64 1, i32 0
+; CHECK-NEXT:    [[B:%.*]] = getelementptr { i32 }, { i32 }* [[I:%.*]], i64 1, i32 0
+; CHECK-NEXT:    ret i32* [[B]]
+;
+  %A = getelementptr { i32 }, { i32 }* %I, i64 1
+  %B = getelementptr { i32 }, { i32 }* %A, i64 0, i32 0
+  ret i32* %B
 }
 
 define void @test5(i8 %B) {
         ; This should be turned into a constexpr instead of being an instruction
-        %A = getelementptr [10 x i8], [10 x i8]* @Global, i64 0, i64 4
-        store i8 %B, i8* %A
-        ret void
 ; CHECK-LABEL: @test5(
-; CHECK: store i8 %B, i8* getelementptr inbounds ([10 x i8], [10 x i8]* @Global, i64 0, i64 4)
+; CHECK-NEXT:    store i8 [[B:%.*]], i8* getelementptr inbounds ([10 x i8], [10 x i8]* @Global, i64 0, i64 4), align 1
+; CHECK-NEXT:    ret void
+;
+  %A = getelementptr [10 x i8], [10 x i8]* @Global, i64 0, i64 4
+  store i8 %B, i8* %A
+  ret void
 }
 
 define void @test5_as1(i8 %B) {
         ; This should be turned into a constexpr instead of being an instruction
-        %A = getelementptr [10 x i8], [10 x i8] addrspace(1)* @Global_as1, i16 0, i16 4
-        store i8 %B, i8 addrspace(1)* %A
-        ret void
 ; CHECK-LABEL: @test5_as1(
-; CHECK: store i8 %B, i8 addrspace(1)* getelementptr inbounds ([10 x i8], [10 x i8] addrspace(1)* @Global_as1, i16 0, i16 4)
+; CHECK-NEXT:    store i8 [[B:%.*]], i8 addrspace(1)* getelementptr inbounds ([10 x i8], [10 x i8] addrspace(1)* @Global_as1, i16 0, i16 4), align 1
+; CHECK-NEXT:    ret void
+;
+  %A = getelementptr [10 x i8], [10 x i8] addrspace(1)* @Global_as1, i16 0, i16 4
+  store i8 %B, i8 addrspace(1)* %A
+  ret void
 }
 
 %as1_ptr_struct = type { i32 addrspace(1)* }
@@ -80,8 +92,9 @@ define void @test5_as1(i8 %B) {
 ; This should be turned into a constexpr instead of being an instruction
 define void @test_evaluate_gep_nested_as_ptrs(i32 addrspace(2)* %B) {
 ; CHECK-LABEL: @test_evaluate_gep_nested_as_ptrs(
-; CHECK-NEXT: store i32 addrspace(2)* %B, i32 addrspace(2)* addrspace(1)* getelementptr inbounds (%as2_ptr_struct, %as2_ptr_struct addrspace(1)* @global_as1_as2_ptr, i16 0, i32 0), align 8
-; CHECK-NEXT: ret void
+; CHECK-NEXT:    store i32 addrspace(2)* [[B:%.*]], i32 addrspace(2)* addrspace(1)* getelementptr inbounds (%as2_ptr_struct, [[AS2_PTR_STRUCT:%.*]] addrspace(1)* @global_as1_as2_ptr, i16 0, i32 0), align 8
+; CHECK-NEXT:    ret void
+;
   %A = getelementptr %as2_ptr_struct, %as2_ptr_struct addrspace(1)* @global_as1_as2_ptr, i16 0, i32 0
   store i32 addrspace(2)* %B, i32 addrspace(2)* addrspace(1)* %A
   ret void
@@ -91,61 +104,77 @@ define void @test_evaluate_gep_nested_as_ptrs(i32 addrspace(2)* %B) {
 
 define void @test_evaluate_gep_as_ptrs_array(i8 addrspace(2)* %B) {
 ; CHECK-LABEL: @test_evaluate_gep_as_ptrs_array(
-; CHECK-NEXT: store i8 addrspace(2)* %B, i8 addrspace(2)* addrspace(1)* getelementptr inbounds ([4 x i8 addrspace(2)*], [4 x i8 addrspace(2)*] addrspace(1)* @arst, i16 0, i16 2), align 4
+; CHECK-NEXT:    store i8 addrspace(2)* [[B:%.*]], i8 addrspace(2)* addrspace(1)* getelementptr inbounds ([4 x i8 addrspace(2)*], [4 x i8 addrspace(2)*] addrspace(1)* @arst, i16 0, i16 2), align 4
+; CHECK-NEXT:    ret void
+;
 
-; CHECK-NEXT: ret void
   %A = getelementptr [4 x i8 addrspace(2)*], [4 x i8 addrspace(2)*] addrspace(1)* @arst, i16 0, i16 2
   store i8 addrspace(2)* %B, i8 addrspace(2)* addrspace(1)* %A
   ret void
 }
 
 define i32* @test7(i32* %I, i64 %C, i64 %D) {
-        %A = getelementptr i32, i32* %I, i64 %C
-        %B = getelementptr i32, i32* %A, i64 %D
-        ret i32* %B
 ; CHECK-LABEL: @test7(
-; CHECK: %A = getelementptr i32, i32* %I, i64 %C
-; CHECK: %B = getelementptr i32, i32* %A, i64 %D
+; CHECK-NEXT:    [[A:%.*]] = getelementptr i32, i32* [[I:%.*]], i64 [[C:%.*]]
+; CHECK-NEXT:    [[B:%.*]] = getelementptr i32, i32* [[A]], i64 [[D:%.*]]
+; CHECK-NEXT:    ret i32* [[B]]
+;
+  %A = getelementptr i32, i32* %I, i64 %C
+  %B = getelementptr i32, i32* %A, i64 %D
+  ret i32* %B
 }
 
 define i8* @test8([10 x i32]* %X) {
         ;; Fold into the cast.
-        %A = getelementptr [10 x i32], [10 x i32]* %X, i64 0, i64 0
-        %B = bitcast i32* %A to i8*
-        ret i8* %B
 ; CHECK-LABEL: @test8(
-; CHECK: bitcast [10 x i32]* %X to i8*
+; CHECK-NEXT:    [[B:%.*]] = bitcast [10 x i32]* [[X:%.*]] to i8*
+; CHECK-NEXT:    ret i8* [[B]]
+;
+  %A = getelementptr [10 x i32], [10 x i32]* %X, i64 0, i64 0
+  %B = bitcast i32* %A to i8*
+  ret i8* %B
 }
 
 define i32 @test9() {
-        %A = getelementptr { i32, double }, { i32, double }* null, i32 0, i32 1
-        %B = ptrtoint double* %A to i32
-        ret i32 %B
 ; CHECK-LABEL: @test9(
-; CHECK: ret i32 8
+; CHECK-NEXT:    ret i32 8
+;
+  %A = getelementptr { i32, double }, { i32, double }* null, i32 0, i32 1
+  %B = ptrtoint double* %A to i32
+  ret i32 %B
 }
 
 define i1 @test10({ i32, i32 }* %x, { i32, i32 }* %y) {
-        %tmp.1 = getelementptr { i32, i32 }, { i32, i32 }* %x, i32 0, i32 1
-        %tmp.3 = getelementptr { i32, i32 }, { i32, i32 }* %y, i32 0, i32 1
-        ;; seteq x, y
-        %tmp.4 = icmp eq i32* %tmp.1, %tmp.3
-        ret i1 %tmp.4
 ; CHECK-LABEL: @test10(
-; CHECK: icmp eq { i32, i32 }* %x, %y
+; CHECK-NEXT:    [[TMP_4:%.*]] = icmp eq { i32, i32 }* [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP_4]]
+;
+  %tmp.1 = getelementptr { i32, i32 }, { i32, i32 }* %x, i32 0, i32 1
+  %tmp.3 = getelementptr { i32, i32 }, { i32, i32 }* %y, i32 0, i32 1
+  ;; seteq x, y
+  %tmp.4 = icmp eq i32* %tmp.1, %tmp.3
+  ret i1 %tmp.4
 }
 
 define i1 @test11({ i32, i32 }* %X) {
-        %P = getelementptr { i32, i32 }, { i32, i32 }* %X, i32 0, i32 0
-        %Q = icmp eq i32* %P, null
-        ret i1 %Q
 ; CHECK-LABEL: @test11(
-; CHECK: icmp eq { i32, i32 }* %X, null
+; CHECK-NEXT:    [[Q:%.*]] = icmp eq { i32, i32 }* [[X:%.*]], null
+; CHECK-NEXT:    ret i1 [[Q]]
+;
+  %P = getelementptr { i32, i32 }, { i32, i32 }* %X, i32 0, i32 0
+  %Q = icmp eq i32* %P, null
+  ret i1 %Q
 }
 
 
 ; PR4748
 define i32 @test12(%struct.A* %a) {
+; CHECK-LABEL: @test12(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[G3:%.*]] = getelementptr [[STRUCT_A:%.*]], %struct.A* [[A:%.*]], i64 0, i32 1
+; CHECK-NEXT:    store i32 10, i32* [[G3]], align 4
+; CHECK-NEXT:    ret i32 10
+;
 entry:
   %g3 = getelementptr %struct.A, %struct.A* %a, i32 0, i32 1
   store i32 10, i32* %g3, align 4
@@ -157,28 +186,25 @@ entry:
   %g5 = getelementptr %struct.A, %struct.A* %new_a, i32 0, i32 1
   %a_a = load i32, i32* %g5, align 4
   ret i32 %a_a
-; CHECK-LABEL:      @test12(
-; CHECK:      getelementptr %struct.A, %struct.A* %a, i64 0, i32 1
-; CHECK-NEXT: store i32 10, i32* %g3
-; CHECK-NEXT: ret i32 10
 }
 
 
 ; PR2235
 %S = type { i32, [ 100 x i32] }
 define i1 @test13(i64 %X, %S* %P) {
-        %A = getelementptr inbounds %S, %S* %P, i32 0, i32 1, i64 %X
-        %B = getelementptr inbounds %S, %S* %P, i32 0, i32 0
-	%C = icmp eq i32* %A, %B
-	ret i1 %C
 ; CHECK-LABEL: @test13(
-; CHECK:    %C = icmp eq i64 %X, -1
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i64 [[X:%.*]], -1
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %A = getelementptr inbounds %S, %S* %P, i32 0, i32 1, i64 %X
+  %B = getelementptr inbounds %S, %S* %P, i32 0, i32 0
+  %C = icmp eq i32* %A, %B
+  ret i1 %C
 }
 
-; This is a test of icmp + shl nuw in disguise - 4611... is 0x3fff...
 define <2 x i1> @test13_vector(<2 x i64> %X, <2 x %S*> %P) nounwind {
 ; CHECK-LABEL: @test13_vector(
-; CHECK-NEXT:    [[C:%.*]] = icmp eq <2 x i64> %X, <i64 4611686018427387903, i64 4611686018427387903>
+; CHECK-NEXT:    [[C:%.*]] = icmp eq <2 x i64> [[X:%.*]], <i64 -1, i64 -1>
 ; CHECK-NEXT:    ret <2 x i1> [[C]]
 ;
   %A = getelementptr inbounds %S, <2 x %S*> %P, <2 x i64> zeroinitializer, <2 x i32> <i32 1, i32 1>, <2 x i64> %X
@@ -187,20 +213,49 @@ define <2 x i1> @test13_vector(<2 x i64> %X, <2 x %S*> %P) nounwind {
   ret <2 x i1> %C
 }
 
+define <2 x i1> @test13_vector2(i64 %X, <2 x %S*> %P) nounwind {
+; CHECK-LABEL: @test13_vector2(
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <2 x i64> undef, i64 [[X:%.*]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <2 x i64> [[DOTSPLATINSERT]], <i64 2, i64 undef>
+; CHECK-NEXT:    [[A_IDX:%.*]] = shufflevector <2 x i64> [[TMP1]], <2 x i64> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[C:%.*]] = icmp eq <2 x i64> [[A_IDX]], <i64 -4, i64 -4>
+; CHECK-NEXT:    ret <2 x i1> [[C]]
+;
+  %A = getelementptr inbounds %S, <2 x %S*> %P, <2 x i64> zeroinitializer, <2 x i32> <i32 1, i32 1>, i64 %X
+  %B = getelementptr inbounds %S, <2 x %S*> %P, <2 x i64> <i64 0, i64 0>, <2 x i32> <i32 0, i32 0>
+  %C = icmp eq <2 x i32*> %A, %B
+  ret <2 x i1> %C
+}
+
+; This is a test of icmp + shl nuw in disguise - 4611... is 0x3fff...
+define <2 x i1> @test13_vector3(i64 %X, <2 x %S*> %P) nounwind {
+; CHECK-LABEL: @test13_vector3(
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <2 x i64> undef, i64 [[X:%.*]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <2 x i64> [[DOTSPLATINSERT]], <i64 2, i64 undef>
+; CHECK-NEXT:    [[A_IDX:%.*]] = shufflevector <2 x i64> [[TMP1]], <2 x i64> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[C:%.*]] = icmp eq <2 x i64> [[A_IDX]], <i64 4, i64 4>
+; CHECK-NEXT:    ret <2 x i1> [[C]]
+;
+  %A = getelementptr inbounds %S, <2 x %S*> %P, <2 x i64> zeroinitializer, <2 x i32> <i32 1, i32 1>, i64 %X
+  %B = getelementptr inbounds %S, <2 x %S*> %P, <2 x i64> <i64 0, i64 0>, <2 x i32> <i32 1, i32 1>, i64 1
+  %C = icmp eq <2 x i32*> %A, %B
+  ret <2 x i1> %C
+}
+
 define i1 @test13_as1(i16 %X, %S addrspace(1)* %P) {
 ; CHECK-LABEL: @test13_as1(
-; CHECK-NEXT:  %C = icmp eq i16 %X, -1
-; CHECK-NEXT: ret i1 %C
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i16 [[X:%.*]], -1
+; CHECK-NEXT:    ret i1 [[C]]
+;
   %A = getelementptr inbounds %S, %S addrspace(1)* %P, i16 0, i32 1, i16 %X
   %B = getelementptr inbounds %S, %S addrspace(1)* %P, i16 0, i32 0
   %C = icmp eq i32 addrspace(1)* %A, %B
   ret i1 %C
 }
 
-; This is a test of icmp + shl nuw in disguise - 16383 is 0x3fff.
 define <2 x i1> @test13_vector_as1(<2 x i16> %X, <2 x %S addrspace(1)*> %P) {
 ; CHECK-LABEL: @test13_vector_as1(
-; CHECK-NEXT:    [[C:%.*]] = icmp eq <2 x i16> %X, <i16 16383, i16 16383>
+; CHECK-NEXT:    [[C:%.*]] = icmp eq <2 x i16> [[X:%.*]], <i16 -1, i16 -1>
 ; CHECK-NEXT:    ret <2 x i1> [[C]]
 ;
   %A = getelementptr inbounds %S, <2 x %S addrspace(1)*> %P, <2 x i16> <i16 0, i16 0>, <2 x i32> <i32 1, i32 1>, <2 x i16> %X
@@ -211,7 +266,9 @@ define <2 x i1> @test13_vector_as1(<2 x i16> %X, <2 x %S addrspace(1)*> %P) {
 
 define i1 @test13_i32(i32 %X, %S* %P) {
 ; CHECK-LABEL: @test13_i32(
-; CHECK: %C = icmp eq i32 %X, -1
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i32 [[X:%.*]], -1
+; CHECK-NEXT:    ret i1 [[C]]
+;
   %A = getelementptr inbounds %S, %S* %P, i32 0, i32 1, i32 %X
   %B = getelementptr inbounds %S, %S* %P, i32 0, i32 0
   %C = icmp eq i32* %A, %B
@@ -220,7 +277,9 @@ define i1 @test13_i32(i32 %X, %S* %P) {
 
 define i1 @test13_i16(i16 %X, %S* %P) {
 ; CHECK-LABEL: @test13_i16(
-; CHECK: %C = icmp eq i16 %X, -1
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i16 [[X:%.*]], -1
+; CHECK-NEXT:    ret i1 [[C]]
+;
   %A = getelementptr inbounds %S, %S* %P, i16 0, i32 1, i16 %X
   %B = getelementptr inbounds %S, %S* %P, i16 0, i32 0
   %C = icmp eq i32* %A, %B
@@ -229,7 +288,10 @@ define i1 @test13_i16(i16 %X, %S* %P) {
 
 define i1 @test13_i128(i128 %X, %S* %P) {
 ; CHECK-LABEL: @test13_i128(
-; CHECK: %C = icmp eq i64 %1, -1
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i128 [[X:%.*]] to i64
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i64 [[TMP1]], -1
+; CHECK-NEXT:    ret i1 [[C]]
+;
   %A = getelementptr inbounds %S, %S* %P, i128 0, i32 1, i128 %X
   %B = getelementptr inbounds %S, %S* %P, i128 0, i32 0
   %C = icmp eq i32* %A, %B
@@ -238,56 +300,69 @@ define i1 @test13_i128(i128 %X, %S* %P) {
 
 
 @G = external global [3 x i8]
-define i8* @test14(i32 %Idx) {
-        %idx = zext i32 %Idx to i64
-        %tmp = getelementptr i8, i8* getelementptr ([3 x i8], [3 x i8]* @G, i32 0, i32 0), i64 %idx
-        ret i8* %tmp
+define i8* @test14(i32 %idx) {
 ; CHECK-LABEL: @test14(
-; CHECK: getelementptr [3 x i8], [3 x i8]* @G, i64 0, i64 %idx
+; CHECK-NEXT:    [[ZEXT:%.*]] = zext i32 [[IDX:%.*]] to i64
+; CHECK-NEXT:    [[TMP:%.*]] = getelementptr [3 x i8], [3 x i8]* @G, i64 0, i64 [[ZEXT]]
+; CHECK-NEXT:    ret i8* [[TMP]]
+;
+  %zext = zext i32 %idx to i64
+  %tmp = getelementptr i8, i8* getelementptr ([3 x i8], [3 x i8]* @G, i32 0, i32 0), i64 %zext
+  ret i8* %tmp
 }
 
 
 ; Test folding of constantexpr geps into normal geps.
 @Array = external global [40 x i32]
 define i32 *@test15(i64 %X) {
-        %A = getelementptr i32, i32* getelementptr ([40 x i32], [40 x i32]* @Array, i64 0, i64 0), i64 %X
-        ret i32* %A
 ; CHECK-LABEL: @test15(
-; CHECK: getelementptr [40 x i32], [40 x i32]* @Array, i64 0, i64 %X
+; CHECK-NEXT:    [[A:%.*]] = getelementptr [40 x i32], [40 x i32]* @Array, i64 0, i64 [[X:%.*]]
+; CHECK-NEXT:    ret i32* [[A]]
+;
+  %A = getelementptr i32, i32* getelementptr ([40 x i32], [40 x i32]* @Array, i64 0, i64 0), i64 %X
+  ret i32* %A
 }
 
 
 define i32* @test16(i32* %X, i32 %Idx) {
-        %R = getelementptr i32, i32* %X, i32 %Idx
-        ret i32* %R
 ; CHECK-LABEL: @test16(
-; CHECK: sext i32 %Idx to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = sext i32 [[IDX:%.*]] to i64
+; CHECK-NEXT:    [[R:%.*]] = getelementptr i32, i32* [[X:%.*]], i64 [[TMP1]]
+; CHECK-NEXT:    ret i32* [[R]]
+;
+  %R = getelementptr i32, i32* %X, i32 %Idx
+  ret i32* %R
 }
 
 
 define i1 @test17(i16* %P, i32 %I, i32 %J) {
-        %X = getelementptr inbounds i16, i16* %P, i32 %I
-        %Y = getelementptr inbounds i16, i16* %P, i32 %J
-        %C = icmp ult i16* %X, %Y
-        ret i1 %C
 ; CHECK-LABEL: @test17(
-; CHECK: %C = icmp slt i32 %I, %J
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i32 [[I:%.*]], [[J:%.*]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %X = getelementptr inbounds i16, i16* %P, i32 %I
+  %Y = getelementptr inbounds i16, i16* %P, i32 %J
+  %C = icmp ult i16* %X, %Y
+  ret i1 %C
 }
 
 define i1 @test18(i16* %P, i32 %I) {
-        %X = getelementptr inbounds i16, i16* %P, i32 %I
-        %C = icmp ult i16* %X, %P
-        ret i1 %C
 ; CHECK-LABEL: @test18(
-; CHECK: %C = icmp slt i32 %I, 0
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i32 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %X = getelementptr inbounds i16, i16* %P, i32 %I
+  %C = icmp ult i16* %X, %P
+  ret i1 %C
 }
 
 ; Larger than the pointer size for a non-zero address space
 define i1 @test18_as1(i16 addrspace(1)* %P, i32 %I) {
 ; CHECK-LABEL: @test18_as1(
-; CHECK-NEXT: %1 = trunc i32 %I to i16
-; CHECK-NEXT: %C = icmp slt i16 %1, 0
-; CHECK-NEXT: ret i1 %C
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[I:%.*]] to i16
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i16 [[TMP1]], 0
+; CHECK-NEXT:    ret i1 [[C]]
+;
   %X = getelementptr inbounds i16, i16 addrspace(1)* %P, i32 %I
   %C = icmp ult i16 addrspace(1)* %X, %P
   ret i1 %C
@@ -296,9 +371,10 @@ define i1 @test18_as1(i16 addrspace(1)* %P, i32 %I) {
 ; Smaller than the pointer size for a non-zero address space
 define i1 @test18_as1_i32(i16 addrspace(1)* %P, i32 %I) {
 ; CHECK-LABEL: @test18_as1_i32(
-; CHECK-NEXT: %1 = trunc i32 %I to i16
-; CHECK-NEXT: %C = icmp slt i16 %1, 0
-; CHECK-NEXT: ret i1 %C
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[I:%.*]] to i16
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i16 [[TMP1]], 0
+; CHECK-NEXT:    ret i1 [[C]]
+;
   %X = getelementptr inbounds i16, i16 addrspace(1)* %P, i32 %I
   %C = icmp ult i16 addrspace(1)* %X, %P
   ret i1 %C
@@ -307,7 +383,9 @@ define i1 @test18_as1_i32(i16 addrspace(1)* %P, i32 %I) {
 ; Smaller than pointer size
 define i1 @test18_i16(i16* %P, i16 %I) {
 ; CHECK-LABEL: @test18_i16(
-; CHECK: %C = icmp slt i16 %I, 0
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i16 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[C]]
+;
   %X = getelementptr inbounds i16, i16* %P, i16 %I
   %C = icmp ult i16* %X, %P
   ret i1 %C
@@ -316,7 +394,9 @@ define i1 @test18_i16(i16* %P, i16 %I) {
 ; Same as pointer size
 define i1 @test18_i64(i16* %P, i64 %I) {
 ; CHECK-LABEL: @test18_i64(
-; CHECK: %C = icmp slt i64 %I, 0
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i64 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[C]]
+;
   %X = getelementptr inbounds i16, i16* %P, i64 %I
   %C = icmp ult i16* %X, %P
   ret i1 %C
@@ -325,49 +405,66 @@ define i1 @test18_i64(i16* %P, i64 %I) {
 ; Larger than the pointer size
 define i1 @test18_i128(i16* %P, i128 %I) {
 ; CHECK-LABEL: @test18_i128(
-; CHECK: %C = icmp slt i64 %1, 0
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i128 [[I:%.*]] to i64
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i64 [[TMP1]], 0
+; CHECK-NEXT:    ret i1 [[C]]
+;
   %X = getelementptr inbounds i16, i16* %P, i128 %I
   %C = icmp ult i16* %X, %P
   ret i1 %C
 }
 
 define i32 @test19(i32* %P, i32 %A, i32 %B) {
-        %tmp.4 = getelementptr inbounds i32, i32* %P, i32 %A
-        %tmp.9 = getelementptr inbounds i32, i32* %P, i32 %B
-        %tmp.10 = icmp eq i32* %tmp.4, %tmp.9
-        %tmp.11 = zext i1 %tmp.10 to i32
-        ret i32 %tmp.11
 ; CHECK-LABEL: @test19(
-; CHECK: icmp eq i32 %A, %B
+; CHECK-NEXT:    [[TMP_10:%.*]] = icmp eq i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP_11:%.*]] = zext i1 [[TMP_10]] to i32
+; CHECK-NEXT:    ret i32 [[TMP_11]]
+;
+  %tmp.4 = getelementptr inbounds i32, i32* %P, i32 %A
+  %tmp.9 = getelementptr inbounds i32, i32* %P, i32 %B
+  %tmp.10 = icmp eq i32* %tmp.4, %tmp.9
+  %tmp.11 = zext i1 %tmp.10 to i32
+  ret i32 %tmp.11
 }
 
 define i32 @test20(i32* %P, i32 %A, i32 %B) {
-        %tmp.4 = getelementptr inbounds i32, i32* %P, i32 %A
-        %tmp.6 = icmp eq i32* %tmp.4, %P
-        %tmp.7 = zext i1 %tmp.6 to i32
-        ret i32 %tmp.7
 ; CHECK-LABEL: @test20(
-; CHECK: icmp eq i32 %A, 0
+; CHECK-NEXT:    [[TMP_6:%.*]] = icmp eq i32 [[A:%.*]], 0
+; CHECK-NEXT:    [[TMP_7:%.*]] = zext i1 [[TMP_6]] to i32
+; CHECK-NEXT:    ret i32 [[TMP_7]]
+;
+  %tmp.4 = getelementptr inbounds i32, i32* %P, i32 %A
+  %tmp.6 = icmp eq i32* %tmp.4, %P
+  %tmp.7 = zext i1 %tmp.6 to i32
+  ret i32 %tmp.7
 }
 
 define i32 @test20_as1(i32 addrspace(1)* %P, i32 %A, i32 %B) {
+; CHECK-LABEL: @test20_as1(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[A:%.*]] to i16
+; CHECK-NEXT:    [[TMP_6:%.*]] = icmp eq i16 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP_7:%.*]] = zext i1 [[TMP_6]] to i32
+; CHECK-NEXT:    ret i32 [[TMP_7]]
+;
   %tmp.4 = getelementptr inbounds i32, i32 addrspace(1)* %P, i32 %A
   %tmp.6 = icmp eq i32 addrspace(1)* %tmp.4, %P
   %tmp.7 = zext i1 %tmp.6 to i32
   ret i32 %tmp.7
-; CHECK-LABEL: @test20_as1(
-; CHECK: icmp eq i16 %1, 0
 }
 
 
 define i32 @test21() {
-        %pbob1 = alloca %intstruct
-        %pbob2 = getelementptr %intstruct, %intstruct* %pbob1
-        %pbobel = getelementptr %intstruct, %intstruct* %pbob2, i64 0, i32 0
-        %rval = load i32, i32* %pbobel
-        ret i32 %rval
 ; CHECK-LABEL: @test21(
-; CHECK: getelementptr inbounds %intstruct, %intstruct* %pbob1, i64 0, i32 0
+; CHECK-NEXT:    [[PBOB1:%.*]] = alloca [[INTSTRUCT:%.*]], align 8
+; CHECK-NEXT:    [[PBOBEL:%.*]] = getelementptr inbounds [[INTSTRUCT]], %intstruct* [[PBOB1]], i64 0, i32 0
+; CHECK-NEXT:    [[RVAL:%.*]] = load i32, i32* [[PBOBEL]], align 8
+; CHECK-NEXT:    ret i32 [[RVAL]]
+;
+  %pbob1 = alloca %intstruct
+  %pbob2 = getelementptr %intstruct, %intstruct* %pbob1
+  %pbobel = getelementptr %intstruct, %intstruct* %pbob2, i64 0, i32 0
+  %rval = load i32, i32* %pbobel
+  ret i32 %rval
 }
 
 
@@ -375,36 +472,43 @@ define i32 @test21() {
 @B = global i32 2               ; <i32*> [#uses=1]
 
 define i1 @test22() {
-        %C = icmp ult i32* getelementptr (i32, i32* @A, i64 1),
-                           getelementptr (i32, i32* @B, i64 2)
-        ret i1 %C
 ; CHECK-LABEL: @test22(
-; CHECK: icmp ult (i32* getelementptr inbounds (i32, i32* @A, i64 1), i32* getelementptr (i32, i32* @B, i64 2))
+; CHECK-NEXT:    ret i1 icmp ult (i32* getelementptr inbounds (i32, i32* @A, i64 1), i32* getelementptr (i32, i32* @B, i64 2))
+;
+  %C = icmp ult i32* getelementptr (i32, i32* @A, i64 1),
+  getelementptr (i32, i32* @B, i64 2)
+  ret i1 %C
 }
 
 
 %X = type { [10 x i32], float }
 
 define i1 @test23() {
-        %A = getelementptr %X, %X* null, i64 0, i32 0, i64 0                ; <i32*> [#uses=1]
-        %B = icmp ne i32* %A, null              ; <i1> [#uses=1]
-        ret i1 %B
 ; CHECK-LABEL: @test23(
-; CHECK: ret i1 false
+; CHECK-NEXT:    ret i1 false
+;
+  %A = getelementptr %X, %X* null, i64 0, i32 0, i64 0                ; <i32*> [#uses=1]
+  %B = icmp ne i32* %A, null              ; <i1> [#uses=1]
+  ret i1 %B
 }
 
 define void @test25() {
-entry:
-        %tmp = getelementptr { i64, i64, i64, i64 }, { i64, i64, i64, i64 }* null, i32 0, i32 3         ; <i64*> [#uses=1]
-        %tmp.upgrd.1 = load i64, i64* %tmp           ; <i64> [#uses=1]
-        %tmp8.ui = load i64, i64* null               ; <i64> [#uses=1]
-        %tmp8 = bitcast i64 %tmp8.ui to i64             ; <i64> [#uses=1]
-        %tmp9 = and i64 %tmp8, %tmp.upgrd.1             ; <i64> [#uses=1]
-        %sext = trunc i64 %tmp9 to i32          ; <i32> [#uses=1]
-        %tmp27.i = sext i32 %sext to i64                ; <i64> [#uses=1]
-        tail call void @foo25( i32 0, i64 %tmp27.i )
-        unreachable
 ; CHECK-LABEL: @test25(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    store i64 undef, i64* null, align 536870912
+; CHECK-NEXT:    tail call void @foo25(i32 0, i64 0)
+; CHECK-NEXT:    unreachable
+;
+entry:
+  %tmp = getelementptr { i64, i64, i64, i64 }, { i64, i64, i64, i64 }* null, i32 0, i32 3         ; <i64*> [#uses=1]
+  %tmp.upgrd.1 = load i64, i64* %tmp           ; <i64> [#uses=1]
+  %tmp8.ui = load i64, i64* null               ; <i64> [#uses=1]
+  %tmp8 = bitcast i64 %tmp8.ui to i64             ; <i64> [#uses=1]
+  %tmp9 = and i64 %tmp8, %tmp.upgrd.1             ; <i64> [#uses=1]
+  %sext = trunc i64 %tmp9 to i32          ; <i32> [#uses=1]
+  %tmp27.i = sext i32 %sext to i64                ; <i64> [#uses=1]
+  tail call void @foo25( i32 0, i64 %tmp27.i )
+  unreachable
 }
 
 declare void @foo25(i32, i64)
@@ -412,12 +516,13 @@ declare void @foo25(i32, i64)
 
 ; PR1637
 define i1 @test26(i8* %arr) {
-        %X = getelementptr i8, i8* %arr, i32 1
-        %Y = getelementptr i8, i8* %arr, i32 1
-        %test = icmp uge i8* %X, %Y
-        ret i1 %test
 ; CHECK-LABEL: @test26(
-; CHECK: ret i1 true
+; CHECK-NEXT:    ret i1 true
+;
+  %X = getelementptr i8, i8* %arr, i32 1
+  %Y = getelementptr i8, i8* %arr, i32 1
+  %test = icmp uge i8* %X, %Y
+  ret i1 %test
 }
 
 	%struct.__large_struct = type { [100 x i64] }
@@ -426,21 +531,30 @@ define i1 @test26(i8* %arr) {
 	%struct.sigval_t = type { i8* }
 
 define i32 @test27(%struct.compat_siginfo* %to, %struct.siginfo_t* %from) {
-entry:
-	%from_addr = alloca %struct.siginfo_t*
-	%tmp344 = load %struct.siginfo_t*, %struct.siginfo_t** %from_addr, align 8
-	%tmp345 = getelementptr %struct.siginfo_t, %struct.siginfo_t* %tmp344, i32 0, i32 3
-	%tmp346 = getelementptr { { i32, i32, [0 x i8], %struct.sigval_t, i32 }, [88 x i8] }, { { i32, i32, [0 x i8], %struct.sigval_t, i32 }, [88 x i8] }* %tmp345, i32 0, i32 0
-	%tmp346347 = bitcast { i32, i32, [0 x i8], %struct.sigval_t, i32 }* %tmp346 to { i32, i32, %struct.sigval_t }*
-	%tmp348 = getelementptr { i32, i32, %struct.sigval_t }, { i32, i32, %struct.sigval_t }* %tmp346347, i32 0, i32 2
-	%tmp349 = getelementptr %struct.sigval_t, %struct.sigval_t* %tmp348, i32 0, i32 0
-	%tmp349350 = bitcast i8** %tmp349 to i32*
-	%tmp351 = load i32, i32* %tmp349350, align 8
-	%tmp360 = call i32 asm sideeffect "...",
-        "=r,ir,*m,i,0,~{dirflag},~{fpsr},~{flags}"( i32 %tmp351,
-         %struct.__large_struct* null, i32 -14, i32 0 )
-	unreachable
 ; CHECK-LABEL: @test27(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[FROM_ADDR:%.*]] = alloca %struct.siginfo_t*, align 8
+; CHECK-NEXT:    [[TMP344:%.*]] = load %struct.siginfo_t*, %struct.siginfo_t** [[FROM_ADDR]], align 8
+; CHECK-NEXT:    [[TMP349:%.*]] = getelementptr [[STRUCT_SIGINFO_T:%.*]], %struct.siginfo_t* [[TMP344]], i64 0, i32 3, i32 0, i32 3, i32 0
+; CHECK-NEXT:    [[TMP349350:%.*]] = bitcast i8** [[TMP349]] to i32*
+; CHECK-NEXT:    [[TMP351:%.*]] = load i32, i32* [[TMP349350]], align 8
+; CHECK-NEXT:    [[TMP360:%.*]] = call i32 asm sideeffect "...", "=r,ir,*m,i,0,~{dirflag},~{fpsr},~{flags}"(i32 [[TMP351]], %struct.__large_struct* null, i32 -14, i32 0) #0
+; CHECK-NEXT:    unreachable
+;
+entry:
+  %from_addr = alloca %struct.siginfo_t*
+  %tmp344 = load %struct.siginfo_t*, %struct.siginfo_t** %from_addr, align 8
+  %tmp345 = getelementptr %struct.siginfo_t, %struct.siginfo_t* %tmp344, i32 0, i32 3
+  %tmp346 = getelementptr { { i32, i32, [0 x i8], %struct.sigval_t, i32 }, [88 x i8] }, { { i32, i32, [0 x i8], %struct.sigval_t, i32 }, [88 x i8] }* %tmp345, i32 0, i32 0
+  %tmp346347 = bitcast { i32, i32, [0 x i8], %struct.sigval_t, i32 }* %tmp346 to { i32, i32, %struct.sigval_t }*
+  %tmp348 = getelementptr { i32, i32, %struct.sigval_t }, { i32, i32, %struct.sigval_t }* %tmp346347, i32 0, i32 2
+  %tmp349 = getelementptr %struct.sigval_t, %struct.sigval_t* %tmp348, i32 0, i32 0
+  %tmp349350 = bitcast i8** %tmp349 to i32*
+  %tmp351 = load i32, i32* %tmp349350, align 8
+  %tmp360 = call i32 asm sideeffect "...",
+  "=r,ir,*m,i,0,~{dirflag},~{fpsr},~{flags}"( i32 %tmp351,
+  %struct.__large_struct* null, i32 -14, i32 0 )
+  unreachable
 }
 
 ; PR1978
@@ -449,27 +563,42 @@ entry:
 @.str1 = internal constant [12 x i8] c"destroy %p\0A\00"
 
 define i32 @test28() nounwind  {
+; CHECK-LABEL: @test28(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ORIENTATIONS:%.*]] = alloca [1 x [1 x %struct.x]], align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @puts(i8* nonnull dereferenceable(1) getelementptr inbounds ([6 x i8], [6 x i8]* @.str, i64 0, i64 0)) #0
+; CHECK-NEXT:    br label [[BB10:%.*]]
+; CHECK:       bb10:
+; CHECK-NEXT:    [[INDVAR:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INDVAR_NEXT:%.*]], [[BB10]] ]
+; CHECK-NEXT:    [[TMP12_REC:%.*]] = xor i32 [[INDVAR]], -1
+; CHECK-NEXT:    [[TMP0:%.*]] = sext i32 [[TMP12_REC]] to i64
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds [1 x [1 x %struct.x]], [1 x [1 x %struct.x]]* [[ORIENTATIONS]], i64 1, i64 0, i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP16:%.*]] = call i32 (i8*, ...) @printf(i8* nonnull dereferenceable(1) getelementptr inbounds ([12 x i8], [12 x i8]* @.str1, i64 0, i64 0), %struct.x* nonnull [[TMP12]]) #0
+; CHECK-NEXT:    [[TMP84:%.*]] = icmp eq i32 [[INDVAR]], 0
+; CHECK-NEXT:    [[INDVAR_NEXT]] = add i32 [[INDVAR]], 1
+; CHECK-NEXT:    br i1 [[TMP84]], label [[BB17:%.*]], label [[BB10]]
+; CHECK:       bb17:
+; CHECK-NEXT:    ret i32 0
+;
 entry:
-	%orientations = alloca [1 x [1 x %struct.x]]
-	%tmp3 = call i32 @puts( i8* getelementptr ([6 x i8], [6 x i8]* @.str, i32 0, i32 0) ) nounwind
-	%tmp45 = getelementptr inbounds [1 x [1 x %struct.x]], [1 x [1 x %struct.x]]* %orientations, i32 1, i32 0, i32 0
-	%orientations62 = getelementptr [1 x [1 x %struct.x]], [1 x [1 x %struct.x]]* %orientations, i32 0, i32 0, i32 0
-	br label %bb10
+  %orientations = alloca [1 x [1 x %struct.x]]
+  %tmp3 = call i32 @puts( i8* getelementptr ([6 x i8], [6 x i8]* @.str, i32 0, i32 0) ) nounwind
+  %tmp45 = getelementptr inbounds [1 x [1 x %struct.x]], [1 x [1 x %struct.x]]* %orientations, i32 1, i32 0, i32 0
+  %orientations62 = getelementptr [1 x [1 x %struct.x]], [1 x [1 x %struct.x]]* %orientations, i32 0, i32 0, i32 0
+  br label %bb10
 
 bb10:
-	%indvar = phi i32 [ 0, %entry ], [ %indvar.next, %bb10 ]
-	%tmp.0.reg2mem.0.rec = mul i32 %indvar, -1
-	%tmp12.rec = add i32 %tmp.0.reg2mem.0.rec, -1
-	%tmp12 = getelementptr inbounds %struct.x, %struct.x* %tmp45, i32 %tmp12.rec
-	%tmp16 = call i32 (i8*, ...) @printf( i8* getelementptr ([12 x i8], [12 x i8]* @.str1, i32 0, i32 0), %struct.x* %tmp12 ) nounwind
-	%tmp84 = icmp eq %struct.x* %tmp12, %orientations62
-	%indvar.next = add i32 %indvar, 1
-	br i1 %tmp84, label %bb17, label %bb10
+  %indvar = phi i32 [ 0, %entry ], [ %indvar.next, %bb10 ]
+  %tmp.0.reg2mem.0.rec = mul i32 %indvar, -1
+  %tmp12.rec = add i32 %tmp.0.reg2mem.0.rec, -1
+  %tmp12 = getelementptr inbounds %struct.x, %struct.x* %tmp45, i32 %tmp12.rec
+  %tmp16 = call i32 (i8*, ...) @printf( i8* nonnull dereferenceable(1) getelementptr ([12 x i8], [12 x i8]* @.str1, i32 0, i32 0), %struct.x* %tmp12 ) nounwind
+  %tmp84 = icmp eq %struct.x* %tmp12, %orientations62
+  %indvar.next = add i32 %indvar, 1
+  br i1 %tmp84, label %bb17, label %bb10
 
 bb17:
-	ret i32 0
-; CHECK-LABEL: @test28(
-; CHECK: icmp eq i32 %indvar, 0
+  ret i32 0
 }
 
 declare i32 @puts(i8*)
@@ -482,36 +611,56 @@ declare i32 @printf(i8*, ...)
 ; rdar://6762290
 	%T = type <{ i64, i64, i64 }>
 define i32 @test29(i8* %start, i32 %X) nounwind {
+; CHECK-LABEL: @test29(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    store i64 undef, i64* null, align 536870912
+; CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr i8, i8* [[START:%.*]], i64 undef
+; CHECK-NEXT:    [[TMP0:%.*]] = sext i32 [[X:%.*]] to i64
+; CHECK-NEXT:    [[ADD_PTR212:%.*]] = getelementptr i8, i8* [[START]], i64 [[TMP0]]
+; CHECK-NEXT:    [[CMP214:%.*]] = icmp ugt i8* [[ADD_PTR212]], [[ADD_PTR]]
+; CHECK-NEXT:    br i1 [[CMP214]], label [[IF_THEN216:%.*]], label [[IF_END363:%.*]]
+; CHECK:       if.then216:
+; CHECK-NEXT:    ret i32 1
+; CHECK:       if.end363:
+; CHECK-NEXT:    ret i32 0
+;
 entry:
-	%tmp3 = load i64, i64* null
-	%add.ptr = getelementptr i8, i8* %start, i64 %tmp3
-	%tmp158 = load i32, i32* null
-	%add.ptr159 = getelementptr %T, %T* null, i32 %tmp158
-	%add.ptr209 = getelementptr i8, i8* %start, i64 0
-	%add.ptr212 = getelementptr i8, i8* %add.ptr209, i32 %X
-	%cmp214 = icmp ugt i8* %add.ptr212, %add.ptr
-	br i1 %cmp214, label %if.then216, label %if.end363
+  %tmp3 = load i64, i64* null
+  %add.ptr = getelementptr i8, i8* %start, i64 %tmp3
+  %tmp158 = load i32, i32* null
+  %add.ptr159 = getelementptr %T, %T* null, i32 %tmp158
+  %add.ptr209 = getelementptr i8, i8* %start, i64 0
+  %add.ptr212 = getelementptr i8, i8* %add.ptr209, i32 %X
+  %cmp214 = icmp ugt i8* %add.ptr212, %add.ptr
+  br i1 %cmp214, label %if.then216, label %if.end363
 
 if.then216:
-	ret i32 1
+  ret i32 1
 
 if.end363:
-	ret i32 0
-; CHECK-LABEL: @test29(
+  ret i32 0
 }
 
 
 ; PR3694
 define i32 @test30(i32 %m, i32 %n) nounwind {
-entry:
-	%0 = alloca i32, i32 %n, align 4
-	%1 = bitcast i32* %0 to [0 x i32]*
-	call void @test30f(i32* %0) nounwind
-	%2 = getelementptr [0 x i32], [0 x i32]* %1, i32 0, i32 %m
-	%3 = load i32, i32* %2, align 4
-	ret i32 %3
 ; CHECK-LABEL: @test30(
-; CHECK: getelementptr i32
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = zext i32 [[N:%.*]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca i32, i64 [[TMP0]], align 4
+; CHECK-NEXT:    call void @test30f(i32* nonnull [[TMP1]]) #0
+; CHECK-NEXT:    [[TMP2:%.*]] = sext i32 [[M:%.*]] to i64
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr i32, i32* [[TMP1]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = load i32, i32* [[TMP3]], align 4
+; CHECK-NEXT:    ret i32 [[TMP4]]
+;
+entry:
+  %0 = alloca i32, i32 %n, align 4
+  %1 = bitcast i32* %0 to [0 x i32]*
+  call void @test30f(i32* %0) nounwind
+  %2 = getelementptr [0 x i32], [0 x i32]* %1, i32 0, i32 %m
+  %3 = load i32, i32* %2, align 4
+  ret i32 %3
 }
 
 declare void @test30f(i32*)
@@ -519,30 +668,38 @@ declare void @test30f(i32*)
 
 
 define i1 @test31(i32* %A) {
-        %B = getelementptr i32, i32* %A, i32 1
-        %C = getelementptr i32, i32* %A, i64 1
-        %V = icmp eq i32* %B, %C
-        ret i1 %V
 ; CHECK-LABEL: @test31(
-; CHECK: ret i1 true
+; CHECK-NEXT:    ret i1 true
+;
+  %B = getelementptr i32, i32* %A, i32 1
+  %C = getelementptr i32, i32* %A, i64 1
+  %V = icmp eq i32* %B, %C
+  ret i1 %V
 }
 
 
 ; PR1345
 define i8* @test32(i8* %v) {
-	%A = alloca [4 x i8*], align 16
-	%B = getelementptr [4 x i8*], [4 x i8*]* %A, i32 0, i32 0
-	store i8* null, i8** %B
-	%C = bitcast [4 x i8*]* %A to { [16 x i8] }*
-	%D = getelementptr { [16 x i8] }, { [16 x i8] }* %C, i32 0, i32 0, i32 8
-	%E = bitcast i8* %D to i8**
-	store i8* %v, i8** %E
-	%F = getelementptr [4 x i8*], [4 x i8*]* %A, i32 0, i32 2
-	%G = load i8*, i8** %F
-	ret i8* %G
 ; CHECK-LABEL: @test32(
-; CHECK: %D = getelementptr inbounds [4 x i8*], [4 x i8*]* %A, i64 0, i64 1
-; CHECK: %F = getelementptr inbounds [4 x i8*], [4 x i8*]* %A, i64 0, i64 2
+; CHECK-NEXT:    [[A:%.*]] = alloca [4 x i8*], align 16
+; CHECK-NEXT:    [[B:%.*]] = getelementptr inbounds [4 x i8*], [4 x i8*]* [[A]], i64 0, i64 0
+; CHECK-NEXT:    store i8* null, i8** [[B]], align 16
+; CHECK-NEXT:    [[D:%.*]] = getelementptr inbounds [4 x i8*], [4 x i8*]* [[A]], i64 0, i64 1
+; CHECK-NEXT:    store i8* [[V:%.*]], i8** [[D]], align 8
+; CHECK-NEXT:    [[F:%.*]] = getelementptr inbounds [4 x i8*], [4 x i8*]* [[A]], i64 0, i64 2
+; CHECK-NEXT:    [[G:%.*]] = load i8*, i8** [[F]], align 16
+; CHECK-NEXT:    ret i8* [[G]]
+;
+  %A = alloca [4 x i8*], align 16
+  %B = getelementptr [4 x i8*], [4 x i8*]* %A, i32 0, i32 0
+  store i8* null, i8** %B
+  %C = bitcast [4 x i8*]* %A to { [16 x i8] }*
+  %D = getelementptr { [16 x i8] }, { [16 x i8] }* %C, i32 0, i32 0, i32 8
+  %E = bitcast i8* %D to i8**
+  store i8* %v, i8** %E
+  %F = getelementptr [4 x i8*], [4 x i8*]* %A, i32 0, i32 2
+  %G = load i8*, i8** %F
+  ret i8* %G
 }
 
 ; PR3290
@@ -551,7 +708,9 @@ define i8* @test32(i8* %v) {
 
 define i32* @test33(%struct.Key* %A) {
 ; CHECK-LABEL: @test33(
-; CHECK: getelementptr %struct.Key, %struct.Key* %A, i64 0, i32 0, i32 1
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr [[STRUCT_KEY:%.*]], %struct.Key* [[A:%.*]], i64 0, i32 0, i32 1
+; CHECK-NEXT:    ret i32* [[TMP1]]
+;
   %B = bitcast %struct.Key* %A to %struct.anon*
   %C = getelementptr %struct.anon, %struct.anon* %B, i32 0, i32 2
   ret i32* %C
@@ -559,7 +718,9 @@ define i32* @test33(%struct.Key* %A) {
 
 define i32 addrspace(1)* @test33_as1(%struct.Key addrspace(1)* %A) {
 ; CHECK-LABEL: @test33_as1(
-; CHECK: getelementptr %struct.Key, %struct.Key addrspace(1)* %A, i16 0, i32 0, i32 1
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr [[STRUCT_KEY:%.*]], [[STRUCT_KEY]] addrspace(1)* [[A:%.*]], i16 0, i32 0, i32 1
+; CHECK-NEXT:    ret i32 addrspace(1)* [[TMP1]]
+;
   %B = bitcast %struct.Key addrspace(1)* %A to %struct.anon addrspace(1)*
   %C = getelementptr %struct.anon, %struct.anon addrspace(1)* %B, i32 0, i32 2
   ret i32 addrspace(1)* %C
@@ -567,7 +728,9 @@ define i32 addrspace(1)* @test33_as1(%struct.Key addrspace(1)* %A) {
 
 define i32 addrspace(1)* @test33_array_as1([10 x i32] addrspace(1)* %A) {
 ; CHECK-LABEL: @test33_array_as1(
-; CHECK: getelementptr [10 x i32], [10 x i32] addrspace(1)* %A, i16 0, i16 2
+; CHECK-NEXT:    [[C:%.*]] = getelementptr [10 x i32], [10 x i32] addrspace(1)* [[A:%.*]], i16 0, i16 2
+; CHECK-NEXT:    ret i32 addrspace(1)* [[C]]
+;
   %B = bitcast [10 x i32] addrspace(1)* %A to [5 x i32] addrspace(1)*
   %C = getelementptr [5 x i32], [5 x i32] addrspace(1)* %B, i32 0, i32 2
   ret i32 addrspace(1)* %C
@@ -576,7 +739,9 @@ define i32 addrspace(1)* @test33_array_as1([10 x i32] addrspace(1)* %A) {
 ; Make sure the GEP indices use the right pointer sized integer
 define i32 addrspace(1)* @test33_array_struct_as1([10 x %struct.Key] addrspace(1)* %A) {
 ; CHECK-LABEL: @test33_array_struct_as1(
-; CHECK: getelementptr [10 x %struct.Key], [10 x %struct.Key] addrspace(1)* %A, i16 0, i16 1, i32 0, i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr [10 x %struct.Key], [10 x %struct.Key] addrspace(1)* [[A:%.*]], i16 0, i16 1, i32 0, i32 0
+; CHECK-NEXT:    ret i32 addrspace(1)* [[TMP1]]
+;
   %B = bitcast [10 x %struct.Key] addrspace(1)* %A to [20 x i32] addrspace(1)*
   %C = getelementptr [20 x i32], [20 x i32] addrspace(1)* %B, i32 0, i32 2
   ret i32 addrspace(1)* %C
@@ -584,9 +749,10 @@ define i32 addrspace(1)* @test33_array_struct_as1([10 x %struct.Key] addrspace(1
 
 define i32 addrspace(1)* @test33_addrspacecast(%struct.Key* %A) {
 ; CHECK-LABEL: @test33_addrspacecast(
-; CHECK: %C = getelementptr %struct.Key, %struct.Key* %A, i64 0, i32 0, i32 1
-; CHECK-NEXT: addrspacecast i32* %C to i32 addrspace(1)*
-; CHECK-NEXT: ret
+; CHECK-NEXT:    [[C:%.*]] = getelementptr [[STRUCT_KEY:%.*]], %struct.Key* [[A:%.*]], i64 0, i32 0, i32 1
+; CHECK-NEXT:    [[TMP1:%.*]] = addrspacecast i32* [[C]] to i32 addrspace(1)*
+; CHECK-NEXT:    ret i32 addrspace(1)* [[TMP1]]
+;
   %B = addrspacecast %struct.Key* %A to %struct.anon addrspace(1)*
   %C = getelementptr %struct.anon, %struct.anon addrspace(1)* %B, i32 0, i32 2
   ret i32 addrspace(1)* %C
@@ -594,17 +760,19 @@ define i32 addrspace(1)* @test33_addrspacecast(%struct.Key* %A) {
 
 	%T2 = type { i8*, i8 }
 define i8* @test34(i8* %Val, i64 %V) nounwind {
+; CHECK-LABEL: @test34(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[C_CAST:%.*]] = inttoptr i64 [[V:%.*]] to i8*
+; CHECK-NEXT:    ret i8* [[C_CAST]]
+;
 entry:
-	%A = alloca %T2, align 8
-	%mrv_gep = bitcast %T2* %A to i64*
-	%B = getelementptr %T2, %T2* %A, i64 0, i32 0
+  %A = alloca %T2, align 8
+  %mrv_gep = bitcast %T2* %A to i64*
+  %B = getelementptr %T2, %T2* %A, i64 0, i32 0
 
-      	store i64 %V, i64* %mrv_gep
-	%C = load i8*, i8** %B, align 8
-	ret i8* %C
-; CHECK-LABEL: @test34(
-; CHECK: %[[C:.*]] = inttoptr i64 %V to i8*
-; CHECK: ret i8* %[[C]]
+  store i64 %V, i64* %mrv_gep
+  %C = load i8*, i8** %B, align 8
+  ret i8* %C
 }
 
 %t0 = type { i8*, [19 x i8] }
@@ -618,37 +786,43 @@ entry:
 ; Instcombine should be able to fold this getelementptr.
 
 define i32 @test35() nounwind {
+; CHECK-LABEL: @test35(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 (i8*, ...) @printf(i8* nonnull dereferenceable(1) getelementptr inbounds ([17 x i8], [17 x i8]* @"\01LC8", i64 0, i64 0), i8* getelementptr inbounds (%t0, %t0* @s, i64 0, i32 1, i64 0)) #0
+; CHECK-NEXT:    ret i32 0
+;
   call i32 (i8*, ...) @printf(i8* getelementptr ([17 x i8], [17 x i8]* @"\01LC8", i32 0, i32 0),
-             i8* getelementptr (%t1, %t1* bitcast (%t0* @s to %t1*), i32 0, i32 1, i32 0)) nounwind
+  i8* getelementptr (%t1, %t1* bitcast (%t0* @s to %t1*), i32 0, i32 1, i32 0)) nounwind
   ret i32 0
-; CHECK-LABEL: @test35(
-; CHECK: call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([17 x i8], [17 x i8]* @"\01LC8", i64 0, i64 0), i8* getelementptr inbounds (%t0, %t0* @s, i64 0, i32 1, i64 0)) [[$NUW:#[0-9]+]]
 }
 
 ; Don't treat signed offsets as unsigned.
 define i8* @test36() nounwind {
-  ret i8* getelementptr ([11 x i8], [11 x i8]* @array, i32 0, i64 -1)
 ; CHECK-LABEL: @test36(
-; CHECK: ret i8* getelementptr ([11 x i8], [11 x i8]* @array, i64 0, i64 -1)
+; CHECK-NEXT:    ret i8* getelementptr ([11 x i8], [11 x i8]* @array, i64 0, i64 -1)
+;
+  ret i8* getelementptr ([11 x i8], [11 x i8]* @array, i32 0, i64 -1)
 }
 
 ; Instcombine shouldn't assume that gep(A,0,1) != gep(A,1,0).
 @A37 = external constant [1 x i8]
 define i1 @test37() nounwind {
 ; CHECK-LABEL: @test37(
-; CHECK: ret i1 true
+; CHECK-NEXT:    ret i1 true
+;
   %t = icmp eq i8* getelementptr ([1 x i8], [1 x i8]* @A37, i64 0, i64 1),
-                   getelementptr ([1 x i8], [1 x i8]* @A37, i64 1, i64 0)
+  getelementptr ([1 x i8], [1 x i8]* @A37, i64 1, i64 0)
   ret i1 %t
 }
 
 ; Test index promotion
 define i32* @test38(i32* %I, i32 %n) {
-        %A = getelementptr i32, i32* %I, i32 %n
-        ret i32* %A
 ; CHECK-LABEL: @test38(
-; CHECK: = sext i32 %n to i64
-; CHECK: %A = getelementptr i32, i32* %I, i64 %
+; CHECK-NEXT:    [[TMP1:%.*]] = sext i32 [[N:%.*]] to i64
+; CHECK-NEXT:    [[A:%.*]] = getelementptr i32, i32* [[I:%.*]], i64 [[TMP1]]
+; CHECK-NEXT:    ret i32* [[A]]
+;
+  %A = getelementptr i32, i32* %I, i32 %n
+  ret i32* %A
 }
 
 ; Test that we don't duplicate work when the second gep is a "bitcast".
@@ -656,6 +830,14 @@ define i32* @test38(i32* %I, i32 %n) {
 declare void @pr10322_f2(%pr10322_t*)
 declare void @pr10322_f3(i8**)
 define void @pr10322_f1(%pr10322_t* %foo) {
+; CHECK-LABEL: @pr10322_f1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds [[PR10322_T:%.*]], %pr10322_t* [[FOO:%.*]], i64 2
+; CHECK-NEXT:    call void @pr10322_f2(%pr10322_t* nonnull [[ARRAYIDX8]]) #0
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds [[PR10322_T]], %pr10322_t* [[ARRAYIDX8]], i64 0, i32 0
+; CHECK-NEXT:    call void @pr10322_f3(i8** nonnull [[TMP2]]) #0
+; CHECK-NEXT:    ret void
+;
 entry:
   %arrayidx8 = getelementptr inbounds %pr10322_t, %pr10322_t* %foo, i64 2
   call void @pr10322_f2(%pr10322_t* %arrayidx8) nounwind
@@ -663,8 +845,6 @@ entry:
   call void @pr10322_f3(i8** %tmp2) nounwind
   ret void
 
-; CHECK-LABEL: @pr10322_f1(
-; CHECK: %tmp2 = getelementptr inbounds %pr10322_t, %pr10322_t* %arrayidx8, i64 0, i32 0
 }
 
 ; Test that we combine the last two geps in this sequence, before we
@@ -673,14 +853,19 @@ entry:
 %three_gep_t2 = type {%three_gep_t}
 
 define void @three_gep_f(%three_gep_t2* %x) {
+; CHECK-LABEL: @three_gep_f(
+; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr [[THREE_GEP_T2:%.*]], %three_gep_t2* [[X:%.*]], i64 2
+; CHECK-NEXT:    call void @three_gep_h(%three_gep_t2* [[GEP1]])
+; CHECK-NEXT:    [[GEP3:%.*]] = getelementptr [[THREE_GEP_T2]], %three_gep_t2* [[GEP1]], i64 0, i32 0, i32 0
+; CHECK-NEXT:    call void @three_gep_g(i32* [[GEP3]])
+; CHECK-NEXT:    ret void
+;
   %gep1 = getelementptr %three_gep_t2, %three_gep_t2* %x, i64 2
   call void @three_gep_h(%three_gep_t2* %gep1)
   %gep2 = getelementptr %three_gep_t2, %three_gep_t2* %gep1, i64 0, i32 0
   %gep3 = getelementptr %three_gep_t, %three_gep_t* %gep2, i64 0, i32 0
   call void @three_gep_g(i32* %gep3)
 
-; CHECK-LABEL: @three_gep_f(
-; CHECK: %gep3 = getelementptr %three_gep_t2, %three_gep_t2* %gep1, i64 0, i32 0, i32 0
   ret void
 }
 
@@ -691,6 +876,14 @@ declare void @three_gep_h(%three_gep_t2*)
 %struct.zot = type { i64, i8 }
 
 define void @test39(%struct.ham* %arg, i8 %arg1) nounwind {
+; CHECK-LABEL: @test39(
+; CHECK-NEXT:    [[TMP:%.*]] = getelementptr inbounds [[STRUCT_HAM:%.*]], %struct.ham* [[ARG:%.*]], i64 0, i32 2
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast %struct.zot** [[TMP]] to i8**
+; CHECK-NEXT:    [[TMP21:%.*]] = load i8*, i8** [[TMP1]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, i8* [[TMP21]], i64 -8
+; CHECK-NEXT:    store i8 [[ARG1:%.*]], i8* [[TMP4]], align 8
+; CHECK-NEXT:    ret void
+;
   %tmp = getelementptr inbounds %struct.ham, %struct.ham* %arg, i64 0, i32 2
   %tmp2 = load %struct.zot*, %struct.zot** %tmp, align 8
   %tmp3 = bitcast %struct.zot* %tmp2 to i8*
@@ -698,25 +891,27 @@ define void @test39(%struct.ham* %arg, i8 %arg1) nounwind {
   store i8 %arg1, i8* %tmp4, align 8
   ret void
 
-; CHECK-LABEL: @test39(
-; CHECK: getelementptr inbounds %struct.ham, %struct.ham* %arg, i64 0, i32 2
-; CHECK: getelementptr inbounds i8, i8* %{{.+}}, i64 -8
 }
 
 define i1 @pr16483([1 x i8]* %a, [1 x i8]* %b) {
+; CHECK-LABEL: @pr16483(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult [1 x i8]* [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
   %c = getelementptr [1 x i8], [1 x i8]* %a, i32 0, i32 0
   %d = getelementptr [1 x i8], [1 x i8]* %b, i32 0, i32 0
   %cmp = icmp ult i8* %c, %d
   ret i1 %cmp
 
-; CHECK-LABEL: @pr16483(
-; CHECK-NEXT: icmp ult  [1 x i8]* %a, %b
 }
 
 define i8 @test_gep_bitcast_as1(i32 addrspace(1)* %arr, i16 %N) {
 ; CHECK-LABEL: @test_gep_bitcast_as1(
-; CHECK: getelementptr i32, i32 addrspace(1)* %arr, i16 %N
-; CHECK: bitcast
+; CHECK-NEXT:    [[T1:%.*]] = getelementptr i32, i32 addrspace(1)* [[ARR:%.*]], i16 [[N:%.*]]
+; CHECK-NEXT:    [[T:%.*]] = bitcast i32 addrspace(1)* [[T1]] to i8 addrspace(1)*
+; CHECK-NEXT:    [[X:%.*]] = load i8, i8 addrspace(1)* [[T]], align 1
+; CHECK-NEXT:    ret i8 [[X]]
+;
   %cast = bitcast i32 addrspace(1)* %arr to i8 addrspace(1)*
   %V = mul i16 %N, 4
   %t = getelementptr i8, i8 addrspace(1)* %cast, i16 %V
@@ -727,8 +922,12 @@ define i8 @test_gep_bitcast_as1(i32 addrspace(1)* %arr, i16 %N) {
 ; The element size of the array matches the element size of the pointer
 define i64 @test_gep_bitcast_array_same_size_element([100 x double]* %arr, i64 %N) {
 ; CHECK-LABEL: @test_gep_bitcast_array_same_size_element(
-; CHECK: getelementptr [100 x double], [100 x double]* %arr, i64 0, i64 %V
-; CHECK: bitcast
+; CHECK-NEXT:    [[V:%.*]] = shl i64 [[N:%.*]], 3
+; CHECK-NEXT:    [[T1:%.*]] = getelementptr [100 x double], [100 x double]* [[ARR:%.*]], i64 0, i64 [[V]]
+; CHECK-NEXT:    [[T:%.*]] = bitcast double* [[T1]] to i64*
+; CHECK-NEXT:    [[X:%.*]] = load i64, i64* [[T]], align 4
+; CHECK-NEXT:    ret i64 [[X]]
+;
   %cast = bitcast [100 x double]* %arr to i64*
   %V = mul i64 %N, 8
   %t = getelementptr i64, i64* %cast, i64 %V
@@ -739,10 +938,13 @@ define i64 @test_gep_bitcast_array_same_size_element([100 x double]* %arr, i64 %
 ; gep should be done in the original address space.
 define i64 @test_gep_bitcast_array_same_size_element_addrspacecast([100 x double]* %arr, i64 %N) {
 ; CHECK-LABEL: @test_gep_bitcast_array_same_size_element_addrspacecast(
-; CHECK: getelementptr [100 x double], [100 x double]* %arr, i64 0, i64 %V
-; CHECK-NEXT: bitcast double*
-; CHECK-NEXT: %t = addrspacecast i64*
-; CHECK: load i64, i64 addrspace(3)* %t
+; CHECK-NEXT:    [[V:%.*]] = shl i64 [[N:%.*]], 3
+; CHECK-NEXT:    [[T1:%.*]] = getelementptr [100 x double], [100 x double]* [[ARR:%.*]], i64 0, i64 [[V]]
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast double* [[T1]] to i64*
+; CHECK-NEXT:    [[T:%.*]] = addrspacecast i64* [[TMP1]] to i64 addrspace(3)*
+; CHECK-NEXT:    [[X:%.*]] = load i64, i64 addrspace(3)* [[T]], align 4
+; CHECK-NEXT:    ret i64 [[X]]
+;
   %cast = addrspacecast [100 x double]* %arr to i64 addrspace(3)*
   %V = mul i64 %N, 8
   %t = getelementptr i64, i64 addrspace(3)* %cast, i64 %V
@@ -753,8 +955,11 @@ define i64 @test_gep_bitcast_array_same_size_element_addrspacecast([100 x double
 ; The element size of the array is different the element size of the pointer
 define i8 @test_gep_bitcast_array_different_size_element([100 x double]* %arr, i64 %N) {
 ; CHECK-LABEL: @test_gep_bitcast_array_different_size_element(
-; CHECK: getelementptr [100 x double], [100 x double]* %arr, i64 0, i64 %N
-; CHECK: bitcast
+; CHECK-NEXT:    [[T1:%.*]] = getelementptr [100 x double], [100 x double]* [[ARR:%.*]], i64 0, i64 [[N:%.*]]
+; CHECK-NEXT:    [[T:%.*]] = bitcast double* [[T1]] to i8*
+; CHECK-NEXT:    [[X:%.*]] = load i8, i8* [[T]], align 1
+; CHECK-NEXT:    ret i8 [[X]]
+;
   %cast = bitcast [100 x double]* %arr to i8*
   %V = mul i64 %N, 8
   %t = getelementptr i8, i8* %cast, i64 %V
@@ -764,8 +969,12 @@ define i8 @test_gep_bitcast_array_different_size_element([100 x double]* %arr, i
 
 define i64 @test_gep_bitcast_array_same_size_element_as1([100 x double] addrspace(1)* %arr, i16 %N) {
 ; CHECK-LABEL: @test_gep_bitcast_array_same_size_element_as1(
-; CHECK: getelementptr [100 x double], [100 x double] addrspace(1)* %arr, i16 0, i16 %V
-; CHECK: bitcast
+; CHECK-NEXT:    [[V:%.*]] = shl i16 [[N:%.*]], 3
+; CHECK-NEXT:    [[T1:%.*]] = getelementptr [100 x double], [100 x double] addrspace(1)* [[ARR:%.*]], i16 0, i16 [[V]]
+; CHECK-NEXT:    [[T:%.*]] = bitcast double addrspace(1)* [[T1]] to i64 addrspace(1)*
+; CHECK-NEXT:    [[X:%.*]] = load i64, i64 addrspace(1)* [[T]], align 4
+; CHECK-NEXT:    ret i64 [[X]]
+;
   %cast = bitcast [100 x double] addrspace(1)* %arr to i64 addrspace(1)*
   %V = mul i16 %N, 8
   %t = getelementptr i64, i64 addrspace(1)* %cast, i16 %V
@@ -775,8 +984,11 @@ define i64 @test_gep_bitcast_array_same_size_element_as1([100 x double] addrspac
 
 define i8 @test_gep_bitcast_array_different_size_element_as1([100 x double] addrspace(1)* %arr, i16 %N) {
 ; CHECK-LABEL: @test_gep_bitcast_array_different_size_element_as1(
-; CHECK: getelementptr [100 x double], [100 x double] addrspace(1)* %arr, i16 0, i16 %N
-; CHECK: bitcast
+; CHECK-NEXT:    [[T1:%.*]] = getelementptr [100 x double], [100 x double] addrspace(1)* [[ARR:%.*]], i16 0, i16 [[N:%.*]]
+; CHECK-NEXT:    [[T:%.*]] = bitcast double addrspace(1)* [[T1]] to i8 addrspace(1)*
+; CHECK-NEXT:    [[X:%.*]] = load i8, i8 addrspace(1)* [[T]], align 1
+; CHECK-NEXT:    ret i8 [[X]]
+;
   %cast = bitcast [100 x double] addrspace(1)* %arr to i8 addrspace(1)*
   %V = mul i16 %N, 8
   %t = getelementptr i8, i8 addrspace(1)* %cast, i16 %V
@@ -785,6 +997,9 @@ define i8 @test_gep_bitcast_array_different_size_element_as1([100 x double] addr
 }
 
 define i64 @test40() {
+; CHECK-LABEL: @test40(
+; CHECK-NEXT:    ret i64 8
+;
   %array = alloca [3 x i32], align 4
   %gep = getelementptr inbounds [3 x i32], [3 x i32]* %array, i64 0, i64 2
   %gepi8 = bitcast i32* %gep to i8*
@@ -794,11 +1009,12 @@ define i64 @test40() {
   %ret = ptrtoint i8* %gep2 to i64
   ret i64 %ret
 
-; CHECK-LABEL: @test40
-; CHECK-NEXT: ret i64 8
 }
 
 define i16 @test41([3 x i32] addrspace(1)* %array) {
+; CHECK-LABEL: @test41(
+; CHECK-NEXT:    ret i16 8
+;
   %gep = getelementptr inbounds [3 x i32], [3 x i32] addrspace(1)* %array, i16 0, i16 2
   %gepi8 = bitcast i32 addrspace(1)* %gep to i8 addrspace(1)*
   %p = ptrtoint [3 x i32] addrspace(1)* %array to i16
@@ -807,55 +1023,60 @@ define i16 @test41([3 x i32] addrspace(1)* %array) {
   %ret = ptrtoint i8 addrspace(1)* %gep2 to i16
   ret i16 %ret
 
-; CHECK-LABEL: @test41(
-; CHECK-NEXT: ret i16 8
 }
 
 define i8* @test42(i8* %c1, i8* %c2) {
+; CHECK-LABEL: @test42(
+; CHECK-NEXT:    [[PTRTOINT:%.*]] = ptrtoint i8* [[C1:%.*]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint i8* [[C2:%.*]] to i64
+; CHECK-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], [[PTRTOINT]]
+; CHECK-NEXT:    [[GEP:%.*]] = inttoptr i64 [[TMP2]] to i8*
+; CHECK-NEXT:    ret i8* [[GEP]]
+;
   %ptrtoint = ptrtoint i8* %c1 to i64
   %sub = sub i64 0, %ptrtoint
   %gep = getelementptr inbounds i8, i8* %c2, i64 %sub
   ret i8* %gep
 
-; CHECK-LABEL: @test42(
-; CHECK-NEXT:  [[PTRTOINT1:%.*]] = ptrtoint i8* %c1 to i64
-; CHECK-NEXT:  [[PTRTOINT2:%.*]] = ptrtoint i8* %c2 to i64
-; CHECK-NEXT:  [[SUB:%.*]] = sub i64 [[PTRTOINT2]], [[PTRTOINT1]]
-; CHECK-NEXT:  [[INTTOPTR:%.*]] = inttoptr i64 [[SUB]] to i8*
-; CHECK-NEXT:  ret i8* [[INTTOPTR]]
 }
 
 define i16* @test43(i16* %c1, i16* %c2) {
+; CHECK-LABEL: @test43(
+; CHECK-NEXT:    [[PTRTOINT:%.*]] = ptrtoint i16* [[C1:%.*]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint i16* [[C2:%.*]] to i64
+; CHECK-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], [[PTRTOINT]]
+; CHECK-NEXT:    [[GEP:%.*]] = inttoptr i64 [[TMP2]] to i16*
+; CHECK-NEXT:    ret i16* [[GEP]]
+;
   %ptrtoint = ptrtoint i16* %c1 to i64
   %sub = sub i64 0, %ptrtoint
   %shr = ashr i64 %sub, 1
   %gep = getelementptr inbounds i16, i16* %c2, i64 %shr
   ret i16* %gep
 
-; CHECK-LABEL: @test43(
-; CHECK-NEXT:  [[PTRTOINT1:%.*]] = ptrtoint i16* %c1 to i64
-; CHECK-NEXT:  [[PTRTOINT2:%.*]] = ptrtoint i16* %c2 to i64
-; CHECK-NEXT:  [[SUB:%.*]] = sub i64 [[PTRTOINT2]], [[PTRTOINT1]]
-; CHECK-NEXT:  [[INTTOPTR:%.*]] = inttoptr i64 [[SUB]] to i16*
-; CHECK-NEXT:  ret i16* [[INTTOPTR]]
 }
 
 define %struct.C* @test44(%struct.C* %c1, %struct.C* %c2) {
+; CHECK-LABEL: @test44(
+; CHECK-NEXT:    [[PTRTOINT:%.*]] = ptrtoint %struct.C* [[C1:%.*]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint %struct.C* [[C2:%.*]] to i64
+; CHECK-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], [[PTRTOINT]]
+; CHECK-NEXT:    [[GEP:%.*]] = inttoptr i64 [[TMP2]] to %struct.C*
+; CHECK-NEXT:    ret %struct.C* [[GEP]]
+;
   %ptrtoint = ptrtoint %struct.C* %c1 to i64
   %sub = sub i64 0, %ptrtoint
   %shr = sdiv i64 %sub, 7
   %gep = getelementptr inbounds %struct.C, %struct.C* %c2, i64 %shr
   ret %struct.C* %gep
 
-; CHECK-LABEL: @test44(
-; CHECK-NEXT:  [[PTRTOINT1:%.*]] = ptrtoint %struct.C* %c1 to i64
-; CHECK-NEXT:  [[PTRTOINT2:%.*]] = ptrtoint %struct.C* %c2 to i64
-; CHECK-NEXT:  [[SUB:%.*]] = sub i64 [[PTRTOINT2]], [[PTRTOINT1]]
-; CHECK-NEXT:  [[INTTOPTR:%.*]] = inttoptr i64 [[SUB]] to %struct.C*
-; CHECK-NEXT:  ret %struct.C* [[INTTOPTR]]
 }
 
 define %struct.C* @test45(%struct.C* %c1, %struct.C** %c2) {
+; CHECK-LABEL: @test45(
+; CHECK-NEXT:    [[GEP:%.*]] = bitcast %struct.C** [[C2:%.*]] to %struct.C*
+; CHECK-NEXT:    ret %struct.C* [[GEP]]
+;
   %ptrtoint1 = ptrtoint %struct.C* %c1 to i64
   %ptrtoint2 = ptrtoint %struct.C** %c2 to i64
   %sub = sub i64 %ptrtoint2, %ptrtoint1 ; C2 - C1
@@ -863,57 +1084,62 @@ define %struct.C* @test45(%struct.C* %c1, %struct.C** %c2) {
   %gep = getelementptr inbounds %struct.C, %struct.C* %c1, i64 %shr ; C1 + (C2 - C1)
   ret %struct.C* %gep
 
-; CHECK-LABEL: @test45(
-; CHECK-NEXT:  [[BITCAST:%.*]] = bitcast %struct.C** %c2 to %struct.C*
-; CHECK-NEXT:  ret %struct.C* [[BITCAST]]
 }
 
 define %struct.C* @test46(%struct.C* %c1, %struct.C* %c2, i64 %N) {
+; CHECK-LABEL: @test46(
+; CHECK-NEXT:    [[PTRTOINT:%.*]] = ptrtoint %struct.C* [[C1:%.*]] to i64
+; CHECK-NEXT:    [[SUB:%.*]] = sub i64 0, [[PTRTOINT]]
+; CHECK-NEXT:    [[SDIV:%.*]] = sdiv i64 [[SUB]], [[N:%.*]]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds [[STRUCT_C:%.*]], %struct.C* [[C2:%.*]], i64 [[SDIV]]
+; CHECK-NEXT:    ret %struct.C* [[GEP]]
+;
   %ptrtoint = ptrtoint %struct.C* %c1 to i64
   %sub = sub i64 0, %ptrtoint
   %sdiv = sdiv i64 %sub, %N
   %gep = getelementptr inbounds %struct.C, %struct.C* %c2, i64 %sdiv
   ret %struct.C* %gep
 
-; CHECK-LABEL: @test46(
-; CHECK-NEXT:  [[PTRTOINT:%.*]] = ptrtoint %struct.C* %c1 to i64
-; CHECK-NEXT:  [[SUB:%.*]] = sub i64 0, [[PTRTOINT]]
-; CHECK-NEXT:  [[SDIV:%.*]] = sdiv i64 [[SUB]], %N
-; CHECK-NEXT:  [[GEP:%.*]] = getelementptr inbounds %struct.C, %struct.C* %c2, i64 %sdiv
-; CHECK-NEXT:  ret %struct.C* [[GEP]]
 }
 
 define i32* @test47(i32* %I, i64 %C, i64 %D) {
+; CHECK-LABEL: @test47(
+; CHECK-NEXT:    [[B:%.*]] = getelementptr i32, i32* [[I:%.*]], i64 [[D:%.*]]
+; CHECK-NEXT:    ret i32* [[B]]
+;
   %sub = sub i64 %D, %C
   %A = getelementptr i32, i32* %I, i64 %C
   %B = getelementptr i32, i32* %A, i64 %sub
   ret i32* %B
-; CHECK-LABEL: @test47(
-; CHECK-NEXT: %B = getelementptr i32, i32* %I, i64 %D
 }
 
 define i32* @test48(i32* %I, i64 %C, i64 %D) {
+; CHECK-LABEL: @test48(
+; CHECK-NEXT:    [[B:%.*]] = getelementptr i32, i32* [[I:%.*]], i64 [[D:%.*]]
+; CHECK-NEXT:    ret i32* [[B]]
+;
   %sub = sub i64 %D, %C
   %A = getelementptr i32, i32* %I, i64 %sub
   %B = getelementptr i32, i32* %A, i64 %C
   ret i32* %B
-; CHECK-LABEL: @test48(
-; CHECK-NEXT: %B = getelementptr i32, i32* %I, i64 %D
 }
 
 define i32* @test49(i32* %I, i64 %C) {
+; CHECK-LABEL: @test49(
+; CHECK-NEXT:    [[B:%.*]] = getelementptr i32, i32* [[I:%.*]], i64 -1
+; CHECK-NEXT:    ret i32* [[B]]
+;
   %notC = xor i64 -1, %C
   %A = getelementptr i32, i32* %I, i64 %C
   %B = getelementptr i32, i32* %A, i64 %notC
   ret i32* %B
-; CHECK-LABEL: @test49(
-; CHECK-NEXT: %B = getelementptr i32, i32* %I, i64 -1
 }
 
 define i32 addrspace(1)* @ascast_0_gep(i32* %p) nounwind {
 ; CHECK-LABEL: @ascast_0_gep(
-; CHECK-NOT: getelementptr
-; CHECK: ret
+; CHECK-NEXT:    [[X:%.*]] = addrspacecast i32* [[P:%.*]] to i32 addrspace(1)*
+; CHECK-NEXT:    ret i32 addrspace(1)* [[X]]
+;
   %gep = getelementptr i32, i32* %p, i32 0
   %x = addrspacecast i32* %gep to i32 addrspace(1)*
   ret i32 addrspace(1)* %x
@@ -923,9 +1149,10 @@ define i32 addrspace(1)* @ascast_0_gep(i32* %p) nounwind {
 ; addrspacecast canonicalization.
 define i32 addrspace(1)* @ascast_0_0_gep([128 x i32]* %p) nounwind {
 ; CHECK-LABEL: @ascast_0_0_gep(
-; CHECK-NEXT: getelementptr [128 x i32]
-; CHECK-NEXT: addrspacecast i32*
-; CHECK-NEXT: ret i32 addrspace(1)*
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr [128 x i32], [128 x i32]* [[P:%.*]], i64 0, i64 0
+; CHECK-NEXT:    [[X:%.*]] = addrspacecast i32* [[GEP]] to i32 addrspace(1)*
+; CHECK-NEXT:    ret i32 addrspace(1)* [[X]]
+;
   %gep = getelementptr [128 x i32], [128 x i32]* %p, i32 0, i32 0
   %x = addrspacecast i32* %gep to i32 addrspace(1)*
   ret i32 addrspace(1)* %x
@@ -933,7 +1160,7 @@ define i32 addrspace(1)* @ascast_0_0_gep([128 x i32]* %p) nounwind {
 
 define <2 x i32*> @PR32414(i32** %ptr) {
 ; CHECK-LABEL: @PR32414(
-; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32** %ptr to i32*
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32** [[PTR:%.*]] to i32*
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], <2 x i64> <i64 0, i64 1>
 ; CHECK-NEXT:    ret <2 x i32*> [[TMP1]]
 ;
@@ -942,4 +1169,42 @@ define <2 x i32*> @PR32414(i32** %ptr) {
   ret <2 x i32*> %tmp1
 }
 
-; CHECK: attributes [[$NUW]] = { nounwind }
+define i32* @test_bitcast_nzgep([1 x i32]* %base, i64 %idx) {
+; CHECK-LABEL: @test_bitcast_nzgep(
+; CHECK-NEXT:    [[PTR:%.*]] = getelementptr inbounds [1 x i32], [1 x i32]* [[BASE:%.*]], i64 0, i64 [[IDX:%.*]]
+; CHECK-NEXT:    ret i32* [[PTR]]
+;
+  %base2 = bitcast [1 x i32]* %base to i32*
+  %ptr = getelementptr inbounds i32, i32* %base2, i64 %idx
+  ret i32* %ptr
+}
+
+define i32* @test_zgep_nzgep([1 x i32]* %base, i64 %idx) {
+; CHECK-LABEL: @test_zgep_nzgep(
+; CHECK-NEXT:    [[PTR:%.*]] = getelementptr inbounds [1 x i32], [1 x i32]* [[BASE:%.*]], i64 0, i64 [[IDX:%.*]]
+; CHECK-NEXT:    ret i32* [[PTR]]
+;
+  %base2 = getelementptr [1 x i32], [1 x i32]* %base, i64 0, i64 0
+  %ptr = getelementptr inbounds i32, i32* %base2, i64 %idx
+  ret i32* %ptr
+}
+
+define i32* @test_nzgep_zgep([1 x i32]* %base, i64 %idx) {
+; CHECK-LABEL: @test_nzgep_zgep(
+; CHECK-NEXT:    [[PTR:%.*]] = getelementptr inbounds [1 x i32], [1 x i32]* [[BASE:%.*]], i64 [[IDX:%.*]], i64 0
+; CHECK-NEXT:    ret i32* [[PTR]]
+;
+  %base2 = getelementptr inbounds [1 x i32], [1 x i32]* %base, i64 %idx
+  %ptr = getelementptr [1 x i32], [1 x i32]* %base2, i64 0, i64 0
+  ret i32* %ptr
+}
+
+define i32* @test_gep_inbounds_of_gep(i32* %base) {
+; CHECK-LABEL: @test_gep_inbounds_of_gep(
+; CHECK-NEXT:    [[PTR2:%.*]] = getelementptr i32, i32* [[BASE:%.*]], i64 8
+; CHECK-NEXT:    ret i32* [[PTR2]]
+;
+  %ptr1 = getelementptr i32, i32* %base, i64 4
+  %ptr2 = getelementptr inbounds i32, i32* %ptr1, i64 4
+  ret i32* %ptr2
+}
diff --git a/llvm/test/Transforms/InstCombine/high-bit-signmask-with-trunc.ll b/llvm/test/Transforms/InstCombine/high-bit-signmask-with-trunc.ll
new file mode 100644
index 000000000..034c285dd
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/high-bit-signmask-with-trunc.ll
@@ -0,0 +1,148 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instcombine -S | FileCheck %s
+
+define i32 @t0(i64 %x) {
+; CHECK-LABEL: @t0(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr i64 [[X:%.*]], 63
+; CHECK-NEXT:    [[R:%.*]] = trunc i64 [[TMP1]] to i32
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t0 = lshr i64 %x, 63
+  %t1 = trunc i64 %t0 to i32
+  %r = sub i32 0, %t1
+  ret i32 %r
+}
+define i32 @t1_exact(i64 %x) {
+; CHECK-LABEL: @t1_exact(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr exact i64 [[X:%.*]], 63
+; CHECK-NEXT:    [[R:%.*]] = trunc i64 [[TMP1]] to i32
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t0 = lshr exact i64 %x, 63
+  %t1 = trunc i64 %t0 to i32
+  %r = sub i32 0, %t1
+  ret i32 %r
+}
+define i32 @t2(i64 %x) {
+; CHECK-LABEL: @t2(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i64 [[X:%.*]], 63
+; CHECK-NEXT:    [[R:%.*]] = trunc i64 [[TMP1]] to i32
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t0 = ashr i64 %x, 63
+  %t1 = trunc i64 %t0 to i32
+  %r = sub i32 0, %t1
+  ret i32 %r
+}
+define i32 @t3_exact(i64 %x) {
+; CHECK-LABEL: @t3_exact(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr exact i64 [[X:%.*]], 63
+; CHECK-NEXT:    [[R:%.*]] = trunc i64 [[TMP1]] to i32
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t0 = ashr exact i64 %x, 63
+  %t1 = trunc i64 %t0 to i32
+  %r = sub i32 0, %t1
+  ret i32 %r
+}
+
+define <2 x i32> @t4(<2 x i64> %x) {
+; CHECK-LABEL: @t4(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <2 x i64> [[X:%.*]], <i64 63, i64 63>
+; CHECK-NEXT:    [[R:%.*]] = trunc <2 x i64> [[TMP1]] to <2 x i32>
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %t0 = lshr <2 x i64> %x, <i64 63, i64 63>
+  %t1 = trunc <2 x i64> %t0 to <2 x i32>
+  %r = sub <2 x i32> zeroinitializer, %t1
+  ret <2 x i32> %r
+}
+
+define <2 x i32> @t5(<2 x i64> %x) {
+; CHECK-LABEL: @t5(
+; CHECK-NEXT:    [[T0:%.*]] = lshr <2 x i64> [[X:%.*]], <i64 63, i64 undef>
+; CHECK-NEXT:    [[T1:%.*]] = trunc <2 x i64> [[T0]] to <2 x i32>
+; CHECK-NEXT:    [[R:%.*]] = sub <2 x i32> <i32 0, i32 undef>, [[T1]]
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %t0 = lshr <2 x i64> %x, <i64 63, i64 undef>
+  %t1 = trunc <2 x i64> %t0 to <2 x i32>
+  %r = sub <2 x i32> <i32 0, i32 undef>, %t1
+  ret <2 x i32> %r
+}
+
+declare void @use64(i64)
+declare void @use32(i32)
+
+define i32 @t6(i64 %x) {
+; CHECK-LABEL: @t6(
+; CHECK-NEXT:    [[T0:%.*]] = lshr i64 [[X:%.*]], 63
+; CHECK-NEXT:    call void @use64(i64 [[T0]])
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr i64 [[X]], 63
+; CHECK-NEXT:    [[R:%.*]] = trunc i64 [[TMP1]] to i32
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t0 = lshr i64 %x, 63
+  call void @use64(i64 %t0)
+  %t1 = trunc i64 %t0 to i32
+  %r = sub i32 0, %t1
+  ret i32 %r
+}
+
+define i32 @n7(i64 %x) {
+; CHECK-LABEL: @n7(
+; CHECK-NEXT:    [[T0:%.*]] = lshr i64 [[X:%.*]], 63
+; CHECK-NEXT:    [[T1:%.*]] = trunc i64 [[T0]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[T1]])
+; CHECK-NEXT:    [[R:%.*]] = sub nsw i32 0, [[T1]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t0 = lshr i64 %x, 63
+  %t1 = trunc i64 %t0 to i32
+  call void @use32(i32 %t1)
+  %r = sub i32 0, %t1
+  ret i32 %r
+}
+
+define i32 @n8(i64 %x) {
+; CHECK-LABEL: @n8(
+; CHECK-NEXT:    [[T0:%.*]] = lshr i64 [[X:%.*]], 63
+; CHECK-NEXT:    call void @use64(i64 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = trunc i64 [[T0]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[T1]])
+; CHECK-NEXT:    [[R:%.*]] = sub nsw i32 0, [[T1]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t0 = lshr i64 %x, 63
+  call void @use64(i64 %t0)
+  %t1 = trunc i64 %t0 to i32
+  call void @use32(i32 %t1)
+  %r = sub i32 0, %t1
+  ret i32 %r
+}
+
+define i32 @n9(i64 %x) {
+; CHECK-LABEL: @n9(
+; CHECK-NEXT:    [[T0:%.*]] = lshr i64 [[X:%.*]], 62
+; CHECK-NEXT:    [[T1:%.*]] = trunc i64 [[T0]] to i32
+; CHECK-NEXT:    [[R:%.*]] = sub nsw i32 0, [[T1]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t0 = lshr i64 %x, 62
+  %t1 = trunc i64 %t0 to i32
+  %r = sub i32 0, %t1
+  ret i32 %r
+}
+
+define i32 @n10(i64 %x) {
+; CHECK-LABEL: @n10(
+; CHECK-NEXT:    [[T0:%.*]] = lshr i64 [[X:%.*]], 63
+; CHECK-NEXT:    [[T1:%.*]] = trunc i64 [[T0]] to i32
+; CHECK-NEXT:    [[R:%.*]] = xor i32 [[T1]], 1
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t0 = lshr i64 %x, 63
+  %t1 = trunc i64 %t0 to i32
+  %r = sub i32 1, %t1
+  ret i32 %r
+}
diff --git a/llvm/test/Transforms/InstCombine/high-bit-signmask.ll b/llvm/test/Transforms/InstCombine/high-bit-signmask.ll
new file mode 100644
index 000000000..4a1b395ca
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/high-bit-signmask.ll
@@ -0,0 +1,126 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instcombine -S | FileCheck %s
+
+define i64 @t0(i64 %x) {
+; CHECK-LABEL: @t0(
+; CHECK-NEXT:    [[R:%.*]] = ashr i64 [[X:%.*]], 63
+; CHECK-NEXT:    ret i64 [[R]]
+;
+  %t0 = lshr i64 %x, 63
+  %r = sub i64 0, %t0
+  ret i64 %r
+}
+define i64 @t0_exact(i64 %x) {
+; CHECK-LABEL: @t0_exact(
+; CHECK-NEXT:    [[R:%.*]] = ashr exact i64 [[X:%.*]], 63
+; CHECK-NEXT:    ret i64 [[R]]
+;
+  %t0 = lshr exact i64 %x, 63
+  %r = sub i64 0, %t0
+  ret i64 %r
+}
+define i64 @t2(i64 %x) {
+; CHECK-LABEL: @t2(
+; CHECK-NEXT:    [[R:%.*]] = lshr i64 [[X:%.*]], 63
+; CHECK-NEXT:    ret i64 [[R]]
+;
+  %t0 = ashr i64 %x, 63
+  %r = sub i64 0, %t0
+  ret i64 %r
+}
+define i64 @t3_exact(i64 %x) {
+; CHECK-LABEL: @t3_exact(
+; CHECK-NEXT:    [[R:%.*]] = lshr exact i64 [[X:%.*]], 63
+; CHECK-NEXT:    ret i64 [[R]]
+;
+  %t0 = ashr exact i64 %x, 63
+  %r = sub i64 0, %t0
+  ret i64 %r
+}
+
+define <2 x i64> @t4(<2 x i64> %x) {
+; CHECK-LABEL: @t4(
+; CHECK-NEXT:    [[R:%.*]] = ashr <2 x i64> [[X:%.*]], <i64 63, i64 63>
+; CHECK-NEXT:    ret <2 x i64> [[R]]
+;
+  %t0 = lshr <2 x i64> %x, <i64 63, i64 63>
+  %r = sub <2 x i64> zeroinitializer, %t0
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @t5(<2 x i64> %x) {
+; CHECK-LABEL: @t5(
+; CHECK-NEXT:    [[T0:%.*]] = lshr <2 x i64> [[X:%.*]], <i64 63, i64 undef>
+; CHECK-NEXT:    [[R:%.*]] = sub <2 x i64> <i64 0, i64 undef>, [[T0]]
+; CHECK-NEXT:    ret <2 x i64> [[R]]
+;
+  %t0 = lshr <2 x i64> %x, <i64 63, i64 undef>
+  %r = sub <2 x i64> <i64 0, i64 undef>, %t0
+  ret <2 x i64> %r
+}
+
+declare void @use64(i64)
+declare void @use32(i64)
+
+define i64 @t6(i64 %x) {
+; CHECK-LABEL: @t6(
+; CHECK-NEXT:    [[T0:%.*]] = lshr i64 [[X:%.*]], 63
+; CHECK-NEXT:    call void @use64(i64 [[T0]])
+; CHECK-NEXT:    [[R:%.*]] = ashr i64 [[X]], 63
+; CHECK-NEXT:    ret i64 [[R]]
+;
+  %t0 = lshr i64 %x, 63
+  call void @use64(i64 %t0)
+  %r = sub i64 0, %t0
+  ret i64 %r
+}
+
+define i64 @n7(i64 %x) {
+; CHECK-LABEL: @n7(
+; CHECK-NEXT:    [[T0:%.*]] = lshr i64 [[X:%.*]], 63
+; CHECK-NEXT:    call void @use32(i64 [[T0]])
+; CHECK-NEXT:    [[R:%.*]] = ashr i64 [[X]], 63
+; CHECK-NEXT:    ret i64 [[R]]
+;
+  %t0 = lshr i64 %x, 63
+  call void @use32(i64 %t0)
+  %r = sub i64 0, %t0
+  ret i64 %r
+}
+
+define i64 @n8(i64 %x) {
+; CHECK-LABEL: @n8(
+; CHECK-NEXT:    [[T0:%.*]] = lshr i64 [[X:%.*]], 63
+; CHECK-NEXT:    call void @use64(i64 [[T0]])
+; CHECK-NEXT:    call void @use32(i64 [[T0]])
+; CHECK-NEXT:    [[R:%.*]] = ashr i64 [[X]], 63
+; CHECK-NEXT:    ret i64 [[R]]
+;
+  %t0 = lshr i64 %x, 63
+  call void @use64(i64 %t0)
+  call void @use32(i64 %t0)
+  %r = sub i64 0, %t0
+  ret i64 %r
+}
+
+define i64 @n9(i64 %x) {
+; CHECK-LABEL: @n9(
+; CHECK-NEXT:    [[T0:%.*]] = lshr i64 [[X:%.*]], 62
+; CHECK-NEXT:    [[R:%.*]] = sub nsw i64 0, [[T0]]
+; CHECK-NEXT:    ret i64 [[R]]
+;
+  %t0 = lshr i64 %x, 62
+  %r = sub i64 0, %t0
+  ret i64 %r
+}
+
+define i64 @n10(i64 %x) {
+; CHECK-LABEL: @n10(
+; CHECK-NEXT:    [[T0:%.*]] = lshr i64 [[X:%.*]], 63
+; CHECK-NEXT:    [[R:%.*]] = xor i64 [[T0]], 1
+; CHECK-NEXT:    ret i64 [[R]]
+;
+  %t0 = lshr i64 %x, 63
+  %r = sub i64 1, %t0
+  ret i64 %r
+}
diff --git a/llvm/test/Transforms/InstCombine/hoist-negation-out-of-bias-calculation-with-constant.ll b/llvm/test/Transforms/InstCombine/hoist-negation-out-of-bias-calculation-with-constant.ll
new file mode 100644
index 000000000..d8cabb311
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/hoist-negation-out-of-bias-calculation-with-constant.ll
@@ -0,0 +1,98 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S < %s -instcombine | FileCheck %s
+
+; Fold
+;   (X & C) - X
+; to
+;   - (X & ~C)
+;
+; This allows us to possibly hoist said negation further out,
+; and decreases use count of X.
+
+; https://bugs.llvm.org/show_bug.cgi?id=44427
+
+; Base tests
+
+define i8 @t0(i8 %x) {
+; CHECK-LABEL: @t0(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[X:%.*]], -43
+; CHECK-NEXT:    [[NEGBIAS:%.*]] = sub i8 0, [[TMP1]]
+; CHECK-NEXT:    ret i8 [[NEGBIAS]]
+;
+  %unbiasedx = and i8 %x, 42
+  %negbias = sub i8 %unbiasedx, %x
+  ret i8 %negbias
+}
+
+define <2 x i8> @t1_vec(<2 x i8> %x) {
+; CHECK-LABEL: @t1_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i8> [[X:%.*]], <i8 -43, i8 -43>
+; CHECK-NEXT:    [[NEGBIAS:%.*]] = sub <2 x i8> zeroinitializer, [[TMP1]]
+; CHECK-NEXT:    ret <2 x i8> [[NEGBIAS]]
+;
+  %unbiasedx = and <2 x i8> %x, <i8 42, i8 42>
+  %negbias = sub <2 x i8> %unbiasedx, %x
+  ret <2 x i8> %negbias
+}
+
+define <2 x i8> @t2_vec_undef(<2 x i8> %x) {
+; CHECK-LABEL: @t2_vec_undef(
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i8> [[X:%.*]], <i8 -43, i8 undef>
+; CHECK-NEXT:    [[NEGBIAS:%.*]] = sub <2 x i8> zeroinitializer, [[TMP1]]
+; CHECK-NEXT:    ret <2 x i8> [[NEGBIAS]]
+;
+  %unbiasedx = and <2 x i8> %x, <i8 42, i8 undef>
+  %negbias = sub <2 x i8> %unbiasedx, %x
+  ret <2 x i8> %negbias
+}
+
+define <2 x i8> @t3_vec_nonsplat(<2 x i8> %x) {
+; CHECK-LABEL: @t3_vec_nonsplat(
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i8> [[X:%.*]], <i8 -43, i8 -45>
+; CHECK-NEXT:    [[NEGBIAS:%.*]] = sub <2 x i8> zeroinitializer, [[TMP1]]
+; CHECK-NEXT:    ret <2 x i8> [[NEGBIAS]]
+;
+  %unbiasedx = and <2 x i8> %x, <i8 42, i8 44>
+  %negbias = sub <2 x i8> %unbiasedx, %x
+  ret <2 x i8> %negbias
+}
+
+; Extra uses always prevent fold
+
+declare void @use8(i8)
+
+define i8 @n4_extrause(i8 %x) {
+; CHECK-LABEL: @n4_extrause(
+; CHECK-NEXT:    [[UNBIASEDX:%.*]] = and i8 [[X:%.*]], 42
+; CHECK-NEXT:    call void @use8(i8 [[UNBIASEDX]])
+; CHECK-NEXT:    [[NEGBIAS:%.*]] = sub i8 [[UNBIASEDX]], [[X]]
+; CHECK-NEXT:    ret i8 [[NEGBIAS]]
+;
+  %unbiasedx = and i8 %x, 42
+  call void @use8(i8 %unbiasedx)
+  %negbias = sub i8 %unbiasedx, %x
+  ret i8 %negbias
+}
+
+; Negative tests
+
+define i8 @n5(i8 %x) {
+; CHECK-LABEL: @n5(
+; CHECK-NEXT:    [[NEGBIAS:%.*]] = and i8 [[X:%.*]], -43
+; CHECK-NEXT:    ret i8 [[NEGBIAS]]
+;
+  %unbiasedx = and i8 %x, 42
+  %negbias = sub i8 %x, %unbiasedx ; wrong order
+  ret i8 %negbias
+}
+
+define i8 @n6(i8 %x0, i8 %x1) {
+; CHECK-LABEL: @n6(
+; CHECK-NEXT:    [[UNBIASEDX:%.*]] = and i8 [[X1:%.*]], 42
+; CHECK-NEXT:    [[NEGBIAS:%.*]] = sub i8 [[UNBIASEDX]], [[X0:%.*]]
+; CHECK-NEXT:    ret i8 [[NEGBIAS]]
+;
+  %unbiasedx = and i8 %x1, 42 ; not %x0
+  %negbias = sub i8 %unbiasedx, %x0 ; not %x1
+  ret i8 %negbias
+}
diff --git a/llvm/test/Transforms/InstCombine/hoist-negation-out-of-bias-calculation.ll b/llvm/test/Transforms/InstCombine/hoist-negation-out-of-bias-calculation.ll
new file mode 100644
index 000000000..bcb93c7a5
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/hoist-negation-out-of-bias-calculation.ll
@@ -0,0 +1,158 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S < %s -instcombine | FileCheck %s
+
+; Fold
+;   (X & (- Y)) - X
+; to
+;   - (X & (Y - 1))
+;
+; This allows us to possibly hoist said negation further out.
+
+; https://bugs.llvm.org/show_bug.cgi?id=44448
+
+; Base tests
+
+define i8 @t0(i8 %x, i8 %y) {
+; CHECK-LABEL: @t0(
+; CHECK-NEXT:    [[TMP1:%.*]] = add i8 [[Y:%.*]], -1
+; CHECK-NEXT:    [[TMP2:%.*]] = and i8 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    [[NEGBIAS:%.*]] = sub i8 0, [[TMP2]]
+; CHECK-NEXT:    ret i8 [[NEGBIAS]]
+;
+  %negy = sub i8 0, %y
+  %unbiasedx = and i8 %negy, %x
+  %negbias = sub i8 %unbiasedx, %x
+  ret i8 %negbias
+}
+
+declare i8 @gen8()
+
+define i8 @t1_commutative(i8 %y) {
+; CHECK-LABEL: @t1_commutative(
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[TMP1:%.*]] = add i8 [[Y:%.*]], -1
+; CHECK-NEXT:    [[TMP2:%.*]] = and i8 [[X]], [[TMP1]]
+; CHECK-NEXT:    [[NEGBIAS:%.*]] = sub i8 0, [[TMP2]]
+; CHECK-NEXT:    ret i8 [[NEGBIAS]]
+;
+  %x = call i8 @gen8()
+  %negy = sub i8 0, %y
+  %unbiasedx = and i8 %x, %negy ; commutative, swapped
+  %negbias = sub i8 %unbiasedx, %x
+  ret i8 %negbias
+}
+
+define <2 x i8> @t2_vec(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @t2_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = add <2 x i8> [[Y:%.*]], <i8 -1, i8 -1>
+; CHECK-NEXT:    [[TMP2:%.*]] = and <2 x i8> [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    [[NEGBIAS:%.*]] = sub <2 x i8> zeroinitializer, [[TMP2]]
+; CHECK-NEXT:    ret <2 x i8> [[NEGBIAS]]
+;
+  %negy = sub <2 x i8> <i8 0, i8 0>, %y
+  %unbiasedx = and <2 x i8> %negy, %x
+  %negbias = sub <2 x i8> %unbiasedx, %x
+  ret <2 x i8> %negbias
+}
+
+define <2 x i8> @t3_vec_undef(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @t3_vec_undef(
+; CHECK-NEXT:    [[TMP1:%.*]] = add <2 x i8> [[Y:%.*]], <i8 -1, i8 -1>
+; CHECK-NEXT:    [[TMP2:%.*]] = and <2 x i8> [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    [[NEGBIAS:%.*]] = sub <2 x i8> zeroinitializer, [[TMP2]]
+; CHECK-NEXT:    ret <2 x i8> [[NEGBIAS]]
+;
+  %negy = sub <2 x i8> <i8 0, i8 undef>, %y
+  %unbiasedx = and <2 x i8> %negy, %x
+  %negbias = sub <2 x i8> %unbiasedx, %x
+  ret <2 x i8> %negbias
+}
+
+; Extra uses always prevent fold
+
+declare void @use8(i8)
+
+define i8 @n4_extrause0(i8 %x, i8 %y) {
+; CHECK-LABEL: @n4_extrause0(
+; CHECK-NEXT:    [[NEGY:%.*]] = sub i8 0, [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[NEGY]])
+; CHECK-NEXT:    [[UNBIASEDX:%.*]] = and i8 [[NEGY]], [[X:%.*]]
+; CHECK-NEXT:    [[NEGBIAS:%.*]] = sub i8 [[UNBIASEDX]], [[X]]
+; CHECK-NEXT:    ret i8 [[NEGBIAS]]
+;
+  %negy = sub i8 0, %y
+  call void @use8(i8 %negy)
+  %unbiasedx = and i8 %negy, %x
+  %negbias = sub i8 %unbiasedx, %x
+  ret i8 %negbias
+}
+define i8 @n5_extrause1(i8 %x, i8 %y) {
+; CHECK-LABEL: @n5_extrause1(
+; CHECK-NEXT:    [[NEGY:%.*]] = sub i8 0, [[Y:%.*]]
+; CHECK-NEXT:    [[UNBIASEDX:%.*]] = and i8 [[NEGY]], [[X:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[UNBIASEDX]])
+; CHECK-NEXT:    [[NEGBIAS:%.*]] = sub i8 [[UNBIASEDX]], [[X]]
+; CHECK-NEXT:    ret i8 [[NEGBIAS]]
+;
+  %negy = sub i8 0, %y
+  %unbiasedx = and i8 %negy, %x
+  call void @use8(i8 %unbiasedx)
+  %negbias = sub i8 %unbiasedx, %x
+  ret i8 %negbias
+}
+define i8 @n6_extrause2(i8 %x, i8 %y) {
+; CHECK-LABEL: @n6_extrause2(
+; CHECK-NEXT:    [[NEGY:%.*]] = sub i8 0, [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[NEGY]])
+; CHECK-NEXT:    [[UNBIASEDX:%.*]] = and i8 [[NEGY]], [[X:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[UNBIASEDX]])
+; CHECK-NEXT:    [[NEGBIAS:%.*]] = sub i8 [[UNBIASEDX]], [[X]]
+; CHECK-NEXT:    ret i8 [[NEGBIAS]]
+;
+  %negy = sub i8 0, %y
+  call void @use8(i8 %negy)
+  %unbiasedx = and i8 %negy, %x
+  call void @use8(i8 %unbiasedx)
+  %negbias = sub i8 %unbiasedx, %x
+  ret i8 %negbias
+}
+
+; Negative tests
+
+define i8 @n7(i8 %x, i8 %y) {
+; CHECK-LABEL: @n7(
+; CHECK-NEXT:    [[NEGY_NOT:%.*]] = add i8 [[Y:%.*]], -1
+; CHECK-NEXT:    [[NEGBIAS:%.*]] = and i8 [[NEGY_NOT]], [[X:%.*]]
+; CHECK-NEXT:    ret i8 [[NEGBIAS]]
+;
+  %negy = sub i8 0, %y
+  %unbiasedx = and i8 %negy, %x
+  %negbias = sub i8 %x, %unbiasedx ; wrong order
+  ret i8 %negbias
+}
+
+define i8 @n8(i8 %x, i8 %y) {
+; CHECK-LABEL: @n8(
+; CHECK-NEXT:    [[NEGY:%.*]] = sub i8 1, [[Y:%.*]]
+; CHECK-NEXT:    [[UNBIASEDX:%.*]] = and i8 [[NEGY]], [[X:%.*]]
+; CHECK-NEXT:    [[NEGBIAS:%.*]] = sub i8 [[UNBIASEDX]], [[X]]
+; CHECK-NEXT:    ret i8 [[NEGBIAS]]
+;
+  %negy = sub i8 1, %y ; not negation
+  %unbiasedx = and i8 %negy, %x
+  %negbias = sub i8 %unbiasedx, %x
+  ret i8 %negbias
+}
+
+define i8 @n9(i8 %x0, i8 %x1, i8 %y) {
+; CHECK-LABEL: @n9(
+; CHECK-NEXT:    [[NEGY:%.*]] = sub i8 0, [[Y:%.*]]
+; CHECK-NEXT:    [[UNBIASEDX:%.*]] = and i8 [[NEGY]], [[X1:%.*]]
+; CHECK-NEXT:    [[NEGBIAS:%.*]] = sub i8 [[UNBIASEDX]], [[X0:%.*]]
+; CHECK-NEXT:    ret i8 [[NEGBIAS]]
+;
+  %negy = sub i8 0, %y
+  %unbiasedx = and i8 %x1, %negy ; not %x0
+  %negbias = sub i8 %unbiasedx, %x0 ; not %x1
+  ret i8 %negbias
+}
diff --git a/llvm/test/Transforms/InstCombine/icmp-add.ll b/llvm/test/Transforms/InstCombine/icmp-add.ll
index 86d4d7c57..5cf0cfb8d 100644
--- a/llvm/test/Transforms/InstCombine/icmp-add.ll
+++ b/llvm/test/Transforms/InstCombine/icmp-add.ll
@@ -1,6 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -instcombine -S | FileCheck %s
 
+declare void @use(i32)
+
 ; PR1949
 
 define i1 @test1(i32 %a) {
@@ -75,14 +77,14 @@ define i1 @test4(i32 %a) {
 
 define { i32, i1 } @test4multiuse(i32 %a) {
 ; CHECK-LABEL: @test4multiuse(
-; CHECK-NEXT:    [[B:%.*]] = add i32 [[A:%.*]], -2147483644
-; CHECK-NEXT:    [[C:%.*]] = icmp slt i32 [[B]], -4
+; CHECK-NEXT:    [[B:%.*]] = add nsw i32 [[A:%.*]], -2147483644
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i32 [[A]], 2147483640
 ; CHECK-NEXT:    [[TMP:%.*]] = insertvalue { i32, i1 } undef, i32 [[B]], 0
 ; CHECK-NEXT:    [[RES:%.*]] = insertvalue { i32, i1 } [[TMP]], i1 [[C]], 1
 ; CHECK-NEXT:    ret { i32, i1 } [[RES]]
 ;
 
-  %b = add i32 %a, -2147483644
+  %b = add nsw i32 %a, -2147483644
   %c = icmp slt i32 %b, -4
 
   %tmp = insertvalue { i32, i1 } undef, i32 %b, 0
@@ -463,3 +465,159 @@ define i1 @sum_ult_op_uses(i8 %x, i8 %y, i8* %p) {
   ret i1 %c
 }
 
+; X + Z >s Y + Z -> X > Y if there is no overflow.
+define i1 @common_op_nsw(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @common_op_nsw(
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %lhs = add nsw i32 %x, %z
+  %rhs = add nsw i32 %y, %z
+  %c = icmp sgt i32 %lhs, %rhs
+  ret i1 %c
+}
+
+define i1 @common_op_nsw_extra_uses(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @common_op_nsw_extra_uses(
+; CHECK-NEXT:    [[LHS:%.*]] = add nsw i32 [[X:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    call void @use(i32 [[LHS]])
+; CHECK-NEXT:    [[RHS:%.*]] = add nsw i32 [[Y:%.*]], [[Z]]
+; CHECK-NEXT:    call void @use(i32 [[RHS]])
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i32 [[X]], [[Y]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %lhs = add nsw i32 %x, %z
+  call void @use(i32 %lhs)
+  %rhs = add nsw i32 %y, %z
+  call void @use(i32 %rhs)
+  %c = icmp sgt i32 %lhs, %rhs
+  ret i1 %c
+}
+
+; X + Z >u Z + Y -> X > Y if there is no overflow.
+define i1 @common_op_nuw(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @common_op_nuw(
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %lhs = add nuw i32 %x, %z
+  %rhs = add nuw i32 %z, %y
+  %c = icmp ugt i32 %lhs, %rhs
+  ret i1 %c
+}
+
+define i1 @common_op_nuw_extra_uses(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @common_op_nuw_extra_uses(
+; CHECK-NEXT:    [[LHS:%.*]] = add nuw i32 [[X:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    call void @use(i32 [[LHS]])
+; CHECK-NEXT:    [[RHS:%.*]] = add nuw i32 [[Z]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use(i32 [[RHS]])
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i32 [[X]], [[Y]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %lhs = add nuw i32 %x, %z
+  call void @use(i32 %lhs)
+  %rhs = add nuw i32 %z, %y
+  call void @use(i32 %rhs)
+  %c = icmp ugt i32 %lhs, %rhs
+  ret i1 %c
+}
+
+define i1 @common_op_nsw_commute(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @common_op_nsw_commute(
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %lhs = add nsw i32 %z, %x
+  %rhs = add nsw i32 %y, %z
+  %c = icmp slt i32 %lhs, %rhs
+  ret i1 %c
+}
+
+define i1 @common_op_nuw_commute(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @common_op_nuw_commute(
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %lhs = add nuw i32 %z, %x
+  %rhs = add nuw i32 %z, %y
+  %c = icmp ult i32 %lhs, %rhs
+  ret i1 %c
+}
+
+; X + Y > X -> Y > 0 if there is no overflow.
+define i1 @common_op_test29(i32 %x, i32 %y) {
+; CHECK-LABEL: @common_op_test29(
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i32 [[Y:%.*]], 0
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %lhs = add nsw i32 %x, %y
+  %c = icmp sgt i32 %lhs, %x
+  ret i1 %c
+}
+
+; X + Y > X -> Y > 0 if there is no overflow.
+define i1 @sum_nuw(i32 %x, i32 %y) {
+; CHECK-LABEL: @sum_nuw(
+; CHECK-NEXT:    [[C:%.*]] = icmp ne i32 [[Y:%.*]], 0
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %lhs = add nuw i32 %x, %y
+  %c = icmp ugt i32 %lhs, %x
+  ret i1 %c
+}
+
+; X > X + Y -> 0 > Y if there is no overflow.
+define i1 @sum_nsw_commute(i32 %x, i32 %y) {
+; CHECK-LABEL: @sum_nsw_commute(
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i32 [[Y:%.*]], 0
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %rhs = add nsw i32 %x, %y
+  %c = icmp sgt i32 %x, %rhs
+  ret i1 %c
+}
+
+; X > X + Y -> 0 > Y if there is no overflow.
+define i1 @sum_nuw_commute(i32 %x, i32 %y) {
+; CHECK-LABEL: @sum_nuw_commute(
+; CHECK-NEXT:    ret i1 false
+;
+  %rhs = add nuw i32 %x, %y
+  %c = icmp ugt i32 %x, %rhs
+  ret i1 %c
+}
+
+; PR2698 - https://bugs.llvm.org/show_bug.cgi?id=2698
+
+declare void @use1(i1)
+declare void @use8(i8)
+
+define void @bzip1(i8 %a, i8 %b, i8 %x) {
+; CHECK-LABEL: @bzip1(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    call void @use1(i1 [[CMP]])
+; CHECK-NEXT:    ret void
+;
+  %add1 = add i8 %a, %x
+  %add2 = add i8 %b, %x
+  %cmp = icmp eq i8 %add1, %add2
+  call void @use1(i1 %cmp)
+  ret void
+}
+
+define void @bzip2(i8 %a, i8 %b, i8 %x) {
+; CHECK-LABEL: @bzip2(
+; CHECK-NEXT:    [[ADD1:%.*]] = add i8 [[A:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[A]], [[B:%.*]]
+; CHECK-NEXT:    call void @use1(i1 [[CMP]])
+; CHECK-NEXT:    call void @use8(i8 [[ADD1]])
+; CHECK-NEXT:    ret void
+;
+  %add1 = add i8 %a, %x
+  %add2 = add i8 %b, %x
+  %cmp = icmp eq i8 %add1, %add2
+  call void @use1(i1 %cmp)
+  call void @use8(i8 %add1)
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/icmp-custom-dl.ll b/llvm/test/Transforms/InstCombine/icmp-custom-dl.ll
index ec7828fea..6b7cb1cdd 100644
--- a/llvm/test/Transforms/InstCombine/icmp-custom-dl.ll
+++ b/llvm/test/Transforms/InstCombine/icmp-custom-dl.ll
@@ -8,8 +8,8 @@ declare i32 @test58_d(i64 )
 define i1 @test59(i8* %foo) {
 ; CHECK-LABEL: @test59(
 ; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr inbounds i8, i8* [[FOO:%.*]], i32 8
-; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint i8* [[GEP1]] to i32
-; CHECK-NEXT:    [[USE:%.*]] = zext i32 [[TMP1]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint i8* [[GEP1]] to i40
+; CHECK-NEXT:    [[USE:%.*]] = zext i40 [[TMP1]] to i64
 ; CHECK-NEXT:    [[CALL:%.*]] = call i32 @test58_d(i64 [[USE]])
 ; CHECK-NEXT:    ret i1 true
 ;
@@ -45,7 +45,7 @@ define i1 @test60(i8* %foo, i64 %i, i64 %j) {
 ; CHECK-LABEL: @test60(
 ; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[I:%.*]] to i32
 ; CHECK-NEXT:    [[TMP2:%.*]] = trunc i64 [[J:%.*]] to i32
-; CHECK-NEXT:    [[GEP1_IDX:%.*]] = shl nuw i32 [[TMP1]], 2
+; CHECK-NEXT:    [[GEP1_IDX:%.*]] = shl nsw i32 [[TMP1]], 2
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp slt i32 [[GEP1_IDX]], [[TMP2]]
 ; CHECK-NEXT:    ret i1 [[TMP3]]
 ;
@@ -61,7 +61,7 @@ define i1 @test60_as1(i8 addrspace(1)* %foo, i64 %i, i64 %j) {
 ; CHECK-LABEL: @test60_as1(
 ; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[I:%.*]] to i16
 ; CHECK-NEXT:    [[TMP2:%.*]] = trunc i64 [[J:%.*]] to i16
-; CHECK-NEXT:    [[GEP1_IDX:%.*]] = shl nuw i16 [[TMP1]], 2
+; CHECK-NEXT:    [[GEP1_IDX:%.*]] = shl nsw i16 [[TMP1]], 2
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp slt i16 [[GEP1_IDX]], [[TMP2]]
 ; CHECK-NEXT:    ret i1 [[TMP3]]
 ;
@@ -93,7 +93,7 @@ define i1 @test60_addrspacecast(i8* %foo, i64 %i, i64 %j) {
 
 define i1 @test60_addrspacecast_smaller(i8* %foo, i16 %i, i64 %j) {
 ; CHECK-LABEL: @test60_addrspacecast_smaller(
-; CHECK-NEXT:    [[GEP1_IDX:%.*]] = shl nuw i16 [[I:%.*]], 2
+; CHECK-NEXT:    [[GEP1_IDX:%.*]] = shl nsw i16 [[I:%.*]], 2
 ; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[J:%.*]] to i16
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i16 [[GEP1_IDX]], [[TMP1]]
 ; CHECK-NEXT:    ret i1 [[TMP2]]
diff --git a/llvm/test/Transforms/InstCombine/icmp-div-constant.ll b/llvm/test/Transforms/InstCombine/icmp-div-constant.ll
index 4c0a56825..8028dd636 100644
--- a/llvm/test/Transforms/InstCombine/icmp-div-constant.ll
+++ b/llvm/test/Transforms/InstCombine/icmp-div-constant.ll
@@ -1,6 +1,110 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -instcombine -S | FileCheck %s
 
+define i1 @is_rem2_neg_i8(i8 %x) {
+; CHECK-LABEL: @is_rem2_neg_i8(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[X:%.*]], -127
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i8 [[TMP1]], -127
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %s = srem i8 %x, 2
+  %r = icmp slt i8 %s, 0
+  ret i1 %r
+}
+
+define <2 x i1> @is_rem2_pos_v2i8(<2 x i8> %x) {
+; CHECK-LABEL: @is_rem2_pos_v2i8(
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i8> [[X:%.*]], <i8 -127, i8 -127>
+; CHECK-NEXT:    [[R:%.*]] = icmp eq <2 x i8> [[TMP1]], <i8 1, i8 1>
+; CHECK-NEXT:    ret <2 x i1> [[R]]
+;
+  %s = srem <2 x i8> %x, <i8 2, i8 2>
+  %r = icmp sgt <2 x i8> %s, zeroinitializer
+  ret <2 x i1> %r
+}
+
+; i8 -97 == 159 == 0b10011111
+
+define i1 @is_rem32_pos_i8(i8 %x) {
+; CHECK-LABEL: @is_rem32_pos_i8(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[X:%.*]], -97
+; CHECK-NEXT:    [[R:%.*]] = icmp sgt i8 [[TMP1]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %s = srem i8 %x, 32
+  %r = icmp sgt i8 %s, 0
+  ret i1 %r
+}
+
+; i16 -32765 == 32771 == 0b1000000000000011
+
+define i1 @is_rem4_neg_i16(i16 %x) {
+; CHECK-LABEL: @is_rem4_neg_i16(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i16 [[X:%.*]], -32765
+; CHECK-NEXT:    [[R:%.*]] = icmp ugt i16 [[TMP1]], -32768
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %s = srem i16 %x, 4
+  %r = icmp slt i16 %s, 0
+  ret i1 %r
+}
+
+declare void @use(i32)
+
+; TODO: This is still worth folding because srem is difficult?
+
+define i1 @is_rem32_neg_i32_extra_use(i32 %x) {
+; CHECK-LABEL: @is_rem32_neg_i32_extra_use(
+; CHECK-NEXT:    [[S:%.*]] = srem i32 [[X:%.*]], 32
+; CHECK-NEXT:    call void @use(i32 [[S]])
+; CHECK-NEXT:    [[R:%.*]] = icmp slt i32 [[S]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %s = srem i32 %x, 32
+  call void @use(i32 %s)
+  %r = icmp slt i32 %s, 0
+  ret i1 %r
+}
+
+; Negative test - wrong compare constant
+
+define i1 @is_rem8_nonneg_i16(i16 %x) {
+; CHECK-LABEL: @is_rem8_nonneg_i16(
+; CHECK-NEXT:    [[S:%.*]] = srem i16 [[X:%.*]], 8
+; CHECK-NEXT:    [[R:%.*]] = icmp sgt i16 [[S]], -1
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %s = srem i16 %x, 8
+  %r = icmp sgt i16 %s, -1
+  ret i1 %r
+}
+
+; Negative test - wrong remainder constant
+
+define i1 @is_rem3_neg_i8(i8 %x) {
+; CHECK-LABEL: @is_rem3_neg_i8(
+; CHECK-NEXT:    [[S:%.*]] = srem i8 [[X:%.*]], 3
+; CHECK-NEXT:    [[R:%.*]] = icmp slt i8 [[S]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %s = srem i8 %x, 3
+  %r = icmp slt i8 %s, 0
+  ret i1 %r
+}
+
+; Negative test - wrong compare constant
+
+define i1 @is_rem16_something_i8(i8 %x) {
+; CHECK-LABEL: @is_rem16_something_i8(
+; CHECK-NEXT:    [[S:%.*]] = srem i8 [[X:%.*]], 16
+; CHECK-NEXT:    [[R:%.*]] = icmp slt i8 [[S]], 7
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %s = srem i8 %x, 16
+  %r = icmp slt i8 %s, 7
+  ret i1 %r
+}
+
 ; PR30281 - https://llvm.org/bugs/show_bug.cgi?id=30281
 
 ; All of these tests contain foldable division-by-constant instructions, but we
@@ -9,14 +113,14 @@
 define i32 @icmp_div(i16 %a, i16 %c) {
 ; CHECK-LABEL: @icmp_div(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i16 %a, 0
-; CHECK-NEXT:    br i1 [[TOBOOL]], label %then, label %exit
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i16 [[A:%.*]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL]], label [[THEN:%.*]], label [[EXIT:%.*]]
 ; CHECK:       then:
-; CHECK-NEXT:    [[NOT_CMP:%.*]] = icmp eq i16 %c, 0
-; CHECK-NEXT:    [[PHITMP1:%.*]] = sext i1 [[NOT_CMP]] to i32
-; CHECK-NEXT:    br label %exit
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i16 [[C:%.*]], 0
+; CHECK-NEXT:    [[PHITMP1:%.*]] = sext i1 [[CMP]] to i32
+; CHECK-NEXT:    br label [[EXIT]]
 ; CHECK:       exit:
-; CHECK-NEXT:    [[PHI:%.*]] = phi i32 [ -1, %entry ], [ [[PHITMP1]], %then ]
+; CHECK-NEXT:    [[PHI:%.*]] = phi i32 [ -1, [[ENTRY:%.*]] ], [ [[PHITMP1]], [[THEN]] ]
 ; CHECK-NEXT:    ret i32 [[PHI]]
 ;
 entry:
@@ -38,12 +142,12 @@ exit:
 define i32 @icmp_div2(i16 %a, i16 %c) {
 ; CHECK-LABEL: @icmp_div2(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i16 %a, 0
-; CHECK-NEXT:    br i1 [[TOBOOL]], label %then, label %exit
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i16 [[A:%.*]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL]], label [[THEN:%.*]], label [[EXIT:%.*]]
 ; CHECK:       then:
-; CHECK-NEXT:    br label %exit
+; CHECK-NEXT:    br label [[EXIT]]
 ; CHECK:       exit:
-; CHECK-NEXT:    [[PHI:%.*]] = phi i32 [ -1, %entry ], [ 0, %then ]
+; CHECK-NEXT:    [[PHI:%.*]] = phi i32 [ -1, [[ENTRY:%.*]] ], [ 0, [[THEN]] ]
 ; CHECK-NEXT:    ret i32 [[PHI]]
 ;
 entry:
@@ -65,14 +169,14 @@ exit:
 define i32 @icmp_div3(i16 %a, i16 %c) {
 ; CHECK-LABEL: @icmp_div3(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i16 %a, 0
-; CHECK-NEXT:    br i1 [[TOBOOL]], label %then, label %exit
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i16 [[A:%.*]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL]], label [[THEN:%.*]], label [[EXIT:%.*]]
 ; CHECK:       then:
-; CHECK-NEXT:    [[NOT_CMP:%.*]] = icmp eq i16 %c, 0
-; CHECK-NEXT:    [[PHITMP1:%.*]] = sext i1 [[NOT_CMP]] to i32
-; CHECK-NEXT:    br label %exit
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i16 [[C:%.*]], 0
+; CHECK-NEXT:    [[PHITMP1:%.*]] = sext i1 [[CMP]] to i32
+; CHECK-NEXT:    br label [[EXIT]]
 ; CHECK:       exit:
-; CHECK-NEXT:    [[PHI:%.*]] = phi i32 [ -1, %entry ], [ [[PHITMP1]], %then ]
+; CHECK-NEXT:    [[PHI:%.*]] = phi i32 [ -1, [[ENTRY:%.*]] ], [ [[PHITMP1]], [[THEN]] ]
 ; CHECK-NEXT:    ret i32 [[PHI]]
 ;
 entry:
diff --git a/llvm/test/Transforms/InstCombine/icmp-shr-lt-gt.ll b/llvm/test/Transforms/InstCombine/icmp-shr-lt-gt.ll
index bf1a031a4..c9a803eb8 100644
--- a/llvm/test/Transforms/InstCombine/icmp-shr-lt-gt.ll
+++ b/llvm/test/Transforms/InstCombine/icmp-shr-lt-gt.ll
@@ -1,8 +1,9 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -instcombine -S | FileCheck %s
 
 define i1 @lshrugt_01_00(i4 %x) {
 ; CHECK-LABEL: @lshrugt_01_00(
-; CHECK-NEXT:    [[C:%.*]] = icmp ugt i4 %x, 1
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i4 [[X:%.*]], 1
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = lshr i4 %x, 1
@@ -12,7 +13,7 @@ define i1 @lshrugt_01_00(i4 %x) {
 
 define i1 @lshrugt_01_01(i4 %x) {
 ; CHECK-LABEL: @lshrugt_01_01(
-; CHECK-NEXT:    [[C:%.*]] = icmp ugt i4 %x, 3
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i4 [[X:%.*]], 3
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = lshr i4 %x, 1
@@ -22,7 +23,7 @@ define i1 @lshrugt_01_01(i4 %x) {
 
 define i1 @lshrugt_01_02(i4 %x) {
 ; CHECK-LABEL: @lshrugt_01_02(
-; CHECK-NEXT:    [[C:%.*]] = icmp ugt i4 %x, 5
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i4 [[X:%.*]], 5
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = lshr i4 %x, 1
@@ -32,7 +33,7 @@ define i1 @lshrugt_01_02(i4 %x) {
 
 define i1 @lshrugt_01_03(i4 %x) {
 ; CHECK-LABEL: @lshrugt_01_03(
-; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 %x, 0
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 [[X:%.*]], 0
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = lshr i4 %x, 1
@@ -42,7 +43,7 @@ define i1 @lshrugt_01_03(i4 %x) {
 
 define i1 @lshrugt_01_04(i4 %x) {
 ; CHECK-LABEL: @lshrugt_01_04(
-; CHECK-NEXT:    [[C:%.*]] = icmp ugt i4 %x, -7
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i4 [[X:%.*]], -7
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = lshr i4 %x, 1
@@ -52,7 +53,7 @@ define i1 @lshrugt_01_04(i4 %x) {
 
 define i1 @lshrugt_01_05(i4 %x) {
 ; CHECK-LABEL: @lshrugt_01_05(
-; CHECK-NEXT:    [[C:%.*]] = icmp ugt i4 %x, -5
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i4 [[X:%.*]], -5
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = lshr i4 %x, 1
@@ -62,7 +63,7 @@ define i1 @lshrugt_01_05(i4 %x) {
 
 define i1 @lshrugt_01_06(i4 %x) {
 ; CHECK-LABEL: @lshrugt_01_06(
-; CHECK-NEXT:    [[C:%.*]] = icmp ugt i4 %x, -3
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i4 [[X:%.*]], -3
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = lshr i4 %x, 1
@@ -153,7 +154,7 @@ define i1 @lshrugt_01_15(i4 %x) {
 
 define i1 @lshrugt_02_00(i4 %x) {
 ; CHECK-LABEL: @lshrugt_02_00(
-; CHECK-NEXT:    [[C:%.*]] = icmp ugt i4 %x, 3
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i4 [[X:%.*]], 3
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = lshr i4 %x, 2
@@ -163,7 +164,7 @@ define i1 @lshrugt_02_00(i4 %x) {
 
 define i1 @lshrugt_02_01(i4 %x) {
 ; CHECK-LABEL: @lshrugt_02_01(
-; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 %x, 0
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 [[X:%.*]], 0
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = lshr i4 %x, 2
@@ -173,7 +174,7 @@ define i1 @lshrugt_02_01(i4 %x) {
 
 define i1 @lshrugt_02_02(i4 %x) {
 ; CHECK-LABEL: @lshrugt_02_02(
-; CHECK-NEXT:    [[C:%.*]] = icmp ugt i4 %x, -5
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i4 [[X:%.*]], -5
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = lshr i4 %x, 2
@@ -300,7 +301,7 @@ define i1 @lshrugt_02_15(i4 %x) {
 
 define i1 @lshrugt_03_00(i4 %x) {
 ; CHECK-LABEL: @lshrugt_03_00(
-; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 %x, 0
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 [[X:%.*]], 0
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = lshr i4 %x, 3
@@ -454,7 +455,7 @@ define i1 @lshrult_01_00(i4 %x) {
 
 define i1 @lshrult_01_01(i4 %x) {
 ; CHECK-LABEL: @lshrult_01_01(
-; CHECK-NEXT:    [[C:%.*]] = icmp ult i4 %x, 2
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i4 [[X:%.*]], 2
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = lshr i4 %x, 1
@@ -464,7 +465,7 @@ define i1 @lshrult_01_01(i4 %x) {
 
 define i1 @lshrult_01_02(i4 %x) {
 ; CHECK-LABEL: @lshrult_01_02(
-; CHECK-NEXT:    [[C:%.*]] = icmp ult i4 %x, 4
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i4 [[X:%.*]], 4
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = lshr i4 %x, 1
@@ -474,7 +475,7 @@ define i1 @lshrult_01_02(i4 %x) {
 
 define i1 @lshrult_01_03(i4 %x) {
 ; CHECK-LABEL: @lshrult_01_03(
-; CHECK-NEXT:    [[C:%.*]] = icmp ult i4 %x, 6
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i4 [[X:%.*]], 6
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = lshr i4 %x, 1
@@ -484,7 +485,7 @@ define i1 @lshrult_01_03(i4 %x) {
 
 define i1 @lshrult_01_04(i4 %x) {
 ; CHECK-LABEL: @lshrult_01_04(
-; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 %x, -1
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 [[X:%.*]], -1
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = lshr i4 %x, 1
@@ -494,7 +495,7 @@ define i1 @lshrult_01_04(i4 %x) {
 
 define i1 @lshrult_01_05(i4 %x) {
 ; CHECK-LABEL: @lshrult_01_05(
-; CHECK-NEXT:    [[C:%.*]] = icmp ult i4 %x, -6
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i4 [[X:%.*]], -6
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = lshr i4 %x, 1
@@ -504,7 +505,7 @@ define i1 @lshrult_01_05(i4 %x) {
 
 define i1 @lshrult_01_06(i4 %x) {
 ; CHECK-LABEL: @lshrult_01_06(
-; CHECK-NEXT:    [[C:%.*]] = icmp ult i4 %x, -4
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i4 [[X:%.*]], -4
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = lshr i4 %x, 1
@@ -514,7 +515,7 @@ define i1 @lshrult_01_06(i4 %x) {
 
 define i1 @lshrult_01_07(i4 %x) {
 ; CHECK-LABEL: @lshrult_01_07(
-; CHECK-NEXT:    [[C:%.*]] = icmp ult i4 %x, -2
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i4 [[X:%.*]], -2
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = lshr i4 %x, 1
@@ -605,7 +606,7 @@ define i1 @lshrult_02_00(i4 %x) {
 
 define i1 @lshrult_02_01(i4 %x) {
 ; CHECK-LABEL: @lshrult_02_01(
-; CHECK-NEXT:    [[C:%.*]] = icmp ult i4 %x, 4
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i4 [[X:%.*]], 4
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = lshr i4 %x, 2
@@ -615,7 +616,7 @@ define i1 @lshrult_02_01(i4 %x) {
 
 define i1 @lshrult_02_02(i4 %x) {
 ; CHECK-LABEL: @lshrult_02_02(
-; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 %x, -1
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 [[X:%.*]], -1
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = lshr i4 %x, 2
@@ -625,7 +626,7 @@ define i1 @lshrult_02_02(i4 %x) {
 
 define i1 @lshrult_02_03(i4 %x) {
 ; CHECK-LABEL: @lshrult_02_03(
-; CHECK-NEXT:    [[C:%.*]] = icmp ult i4 %x, -4
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i4 [[X:%.*]], -4
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = lshr i4 %x, 2
@@ -752,7 +753,7 @@ define i1 @lshrult_03_00(i4 %x) {
 
 define i1 @lshrult_03_01(i4 %x) {
 ; CHECK-LABEL: @lshrult_03_01(
-; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 %x, -1
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 [[X:%.*]], -1
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = lshr i4 %x, 3
@@ -888,7 +889,7 @@ define i1 @lshrult_03_15(i4 %x) {
 
 define i1 @ashrsgt_01_00(i4 %x) {
 ; CHECK-LABEL: @ashrsgt_01_00(
-; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 %x, 1
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 [[X:%.*]], 1
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = ashr i4 %x, 1
@@ -898,7 +899,7 @@ define i1 @ashrsgt_01_00(i4 %x) {
 
 define i1 @ashrsgt_01_01(i4 %x) {
 ; CHECK-LABEL: @ashrsgt_01_01(
-; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 %x, 3
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 [[X:%.*]], 3
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = ashr i4 %x, 1
@@ -908,7 +909,7 @@ define i1 @ashrsgt_01_01(i4 %x) {
 
 define i1 @ashrsgt_01_02(i4 %x) {
 ; CHECK-LABEL: @ashrsgt_01_02(
-; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 %x, 5
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 [[X:%.*]], 5
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = ashr i4 %x, 1
@@ -999,7 +1000,7 @@ define i1 @ashrsgt_01_11(i4 %x) {
 
 define i1 @ashrsgt_01_12(i4 %x) {
 ; CHECK-LABEL: @ashrsgt_01_12(
-; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 %x, -7
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 [[X:%.*]], -7
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = ashr i4 %x, 1
@@ -1009,7 +1010,7 @@ define i1 @ashrsgt_01_12(i4 %x) {
 
 define i1 @ashrsgt_01_13(i4 %x) {
 ; CHECK-LABEL: @ashrsgt_01_13(
-; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 %x, -5
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 [[X:%.*]], -5
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = ashr i4 %x, 1
@@ -1019,7 +1020,7 @@ define i1 @ashrsgt_01_13(i4 %x) {
 
 define i1 @ashrsgt_01_14(i4 %x) {
 ; CHECK-LABEL: @ashrsgt_01_14(
-; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 %x, -3
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 [[X:%.*]], -3
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = ashr i4 %x, 1
@@ -1029,7 +1030,7 @@ define i1 @ashrsgt_01_14(i4 %x) {
 
 define i1 @ashrsgt_01_15(i4 %x) {
 ; CHECK-LABEL: @ashrsgt_01_15(
-; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 %x, -1
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 [[X:%.*]], -1
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = ashr i4 %x, 1
@@ -1039,7 +1040,7 @@ define i1 @ashrsgt_01_15(i4 %x) {
 
 define i1 @ashrsgt_02_00(i4 %x) {
 ; CHECK-LABEL: @ashrsgt_02_00(
-; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 %x, 3
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 [[X:%.*]], 3
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = ashr i4 %x, 2
@@ -1166,7 +1167,7 @@ define i1 @ashrsgt_02_13(i4 %x) {
 
 define i1 @ashrsgt_02_14(i4 %x) {
 ; CHECK-LABEL: @ashrsgt_02_14(
-; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 %x, -5
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 [[X:%.*]], -5
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = ashr i4 %x, 2
@@ -1176,7 +1177,7 @@ define i1 @ashrsgt_02_14(i4 %x) {
 
 define i1 @ashrsgt_02_15(i4 %x) {
 ; CHECK-LABEL: @ashrsgt_02_15(
-; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 %x, -1
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 [[X:%.*]], -1
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = ashr i4 %x, 2
@@ -1321,7 +1322,7 @@ define i1 @ashrsgt_03_14(i4 %x) {
 
 define i1 @ashrsgt_03_15(i4 %x) {
 ; CHECK-LABEL: @ashrsgt_03_15(
-; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 %x, -1
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 [[X:%.*]], -1
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = ashr i4 %x, 3
@@ -1331,7 +1332,7 @@ define i1 @ashrsgt_03_15(i4 %x) {
 
 define i1 @ashrslt_01_00(i4 %x) {
 ; CHECK-LABEL: @ashrslt_01_00(
-; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 %x, 0
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 [[X:%.*]], 0
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = ashr i4 %x, 1
@@ -1341,7 +1342,7 @@ define i1 @ashrslt_01_00(i4 %x) {
 
 define i1 @ashrslt_01_01(i4 %x) {
 ; CHECK-LABEL: @ashrslt_01_01(
-; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 %x, 2
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 [[X:%.*]], 2
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = ashr i4 %x, 1
@@ -1351,7 +1352,7 @@ define i1 @ashrslt_01_01(i4 %x) {
 
 define i1 @ashrslt_01_02(i4 %x) {
 ; CHECK-LABEL: @ashrslt_01_02(
-; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 %x, 4
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 [[X:%.*]], 4
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = ashr i4 %x, 1
@@ -1361,7 +1362,7 @@ define i1 @ashrslt_01_02(i4 %x) {
 
 define i1 @ashrslt_01_03(i4 %x) {
 ; CHECK-LABEL: @ashrslt_01_03(
-; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 %x, 6
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 [[X:%.*]], 6
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = ashr i4 %x, 1
@@ -1452,7 +1453,7 @@ define i1 @ashrslt_01_12(i4 %x) {
 
 define i1 @ashrslt_01_13(i4 %x) {
 ; CHECK-LABEL: @ashrslt_01_13(
-; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 %x, -6
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 [[X:%.*]], -6
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = ashr i4 %x, 1
@@ -1462,7 +1463,7 @@ define i1 @ashrslt_01_13(i4 %x) {
 
 define i1 @ashrslt_01_14(i4 %x) {
 ; CHECK-LABEL: @ashrslt_01_14(
-; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 %x, -4
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 [[X:%.*]], -4
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = ashr i4 %x, 1
@@ -1472,7 +1473,7 @@ define i1 @ashrslt_01_14(i4 %x) {
 
 define i1 @ashrslt_01_15(i4 %x) {
 ; CHECK-LABEL: @ashrslt_01_15(
-; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 %x, -2
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 [[X:%.*]], -2
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = ashr i4 %x, 1
@@ -1482,7 +1483,7 @@ define i1 @ashrslt_01_15(i4 %x) {
 
 define i1 @ashrslt_02_00(i4 %x) {
 ; CHECK-LABEL: @ashrslt_02_00(
-; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 %x, 0
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 [[X:%.*]], 0
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = ashr i4 %x, 2
@@ -1492,7 +1493,7 @@ define i1 @ashrslt_02_00(i4 %x) {
 
 define i1 @ashrslt_02_01(i4 %x) {
 ; CHECK-LABEL: @ashrslt_02_01(
-; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 %x, 4
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 [[X:%.*]], 4
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = ashr i4 %x, 2
@@ -1619,7 +1620,7 @@ define i1 @ashrslt_02_14(i4 %x) {
 
 define i1 @ashrslt_02_15(i4 %x) {
 ; CHECK-LABEL: @ashrslt_02_15(
-; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 %x, -4
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 [[X:%.*]], -4
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = ashr i4 %x, 2
@@ -1629,7 +1630,7 @@ define i1 @ashrslt_02_15(i4 %x) {
 
 define i1 @ashrslt_03_00(i4 %x) {
 ; CHECK-LABEL: @ashrslt_03_00(
-; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 %x, 0
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 [[X:%.*]], 0
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = ashr i4 %x, 3
@@ -1774,7 +1775,7 @@ define i1 @ashrslt_03_15(i4 %x) {
 
 define i1 @lshrugt_01_00_exact(i4 %x) {
 ; CHECK-LABEL: @lshrugt_01_00_exact(
-; CHECK-NEXT:    [[C:%.*]] = icmp ne i4 %x, 0
+; CHECK-NEXT:    [[C:%.*]] = icmp ne i4 [[X:%.*]], 0
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = lshr exact i4 %x, 1
@@ -1784,7 +1785,7 @@ define i1 @lshrugt_01_00_exact(i4 %x) {
 
 define i1 @lshrugt_01_01_exact(i4 %x) {
 ; CHECK-LABEL: @lshrugt_01_01_exact(
-; CHECK-NEXT:    [[C:%.*]] = icmp ugt i4 %x, 2
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i4 [[X:%.*]], 2
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = lshr exact i4 %x, 1
@@ -1794,7 +1795,7 @@ define i1 @lshrugt_01_01_exact(i4 %x) {
 
 define i1 @lshrugt_01_02_exact(i4 %x) {
 ; CHECK-LABEL: @lshrugt_01_02_exact(
-; CHECK-NEXT:    [[C:%.*]] = icmp ugt i4 %x, 4
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i4 [[X:%.*]], 4
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = lshr exact i4 %x, 1
@@ -1804,7 +1805,7 @@ define i1 @lshrugt_01_02_exact(i4 %x) {
 
 define i1 @lshrugt_01_03_exact(i4 %x) {
 ; CHECK-LABEL: @lshrugt_01_03_exact(
-; CHECK-NEXT:    [[C:%.*]] = icmp ugt i4 %x, 6
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i4 [[X:%.*]], 6
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = lshr exact i4 %x, 1
@@ -1814,7 +1815,7 @@ define i1 @lshrugt_01_03_exact(i4 %x) {
 
 define i1 @lshrugt_01_04_exact(i4 %x) {
 ; CHECK-LABEL: @lshrugt_01_04_exact(
-; CHECK-NEXT:    [[C:%.*]] = icmp ugt i4 %x, -8
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i4 [[X:%.*]], -8
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = lshr exact i4 %x, 1
@@ -1824,7 +1825,7 @@ define i1 @lshrugt_01_04_exact(i4 %x) {
 
 define i1 @lshrugt_01_05_exact(i4 %x) {
 ; CHECK-LABEL: @lshrugt_01_05_exact(
-; CHECK-NEXT:    [[C:%.*]] = icmp ugt i4 %x, -6
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i4 [[X:%.*]], -6
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = lshr exact i4 %x, 1
@@ -1834,7 +1835,7 @@ define i1 @lshrugt_01_05_exact(i4 %x) {
 
 define i1 @lshrugt_01_06_exact(i4 %x) {
 ; CHECK-LABEL: @lshrugt_01_06_exact(
-; CHECK-NEXT:    [[C:%.*]] = icmp eq i4 %x, -2
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i4 [[X:%.*]], -2
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = lshr exact i4 %x, 1
@@ -1925,7 +1926,7 @@ define i1 @lshrugt_01_15_exact(i4 %x) {
 
 define i1 @lshrugt_02_00_exact(i4 %x) {
 ; CHECK-LABEL: @lshrugt_02_00_exact(
-; CHECK-NEXT:    [[C:%.*]] = icmp ne i4 %x, 0
+; CHECK-NEXT:    [[C:%.*]] = icmp ne i4 [[X:%.*]], 0
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = lshr exact i4 %x, 2
@@ -1935,7 +1936,7 @@ define i1 @lshrugt_02_00_exact(i4 %x) {
 
 define i1 @lshrugt_02_01_exact(i4 %x) {
 ; CHECK-LABEL: @lshrugt_02_01_exact(
-; CHECK-NEXT:    [[C:%.*]] = icmp ugt i4 %x, 4
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i4 [[X:%.*]], 4
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = lshr exact i4 %x, 2
@@ -1945,7 +1946,7 @@ define i1 @lshrugt_02_01_exact(i4 %x) {
 
 define i1 @lshrugt_02_02_exact(i4 %x) {
 ; CHECK-LABEL: @lshrugt_02_02_exact(
-; CHECK-NEXT:    [[C:%.*]] = icmp eq i4 %x, -4
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i4 [[X:%.*]], -4
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = lshr exact i4 %x, 2
@@ -2072,7 +2073,7 @@ define i1 @lshrugt_02_15_exact(i4 %x) {
 
 define i1 @lshrugt_03_00_exact(i4 %x) {
 ; CHECK-LABEL: @lshrugt_03_00_exact(
-; CHECK-NEXT:    [[C:%.*]] = icmp ne i4 %x, 0
+; CHECK-NEXT:    [[C:%.*]] = icmp ne i4 [[X:%.*]], 0
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = lshr exact i4 %x, 3
@@ -2226,7 +2227,7 @@ define i1 @lshrult_01_00_exact(i4 %x) {
 
 define i1 @lshrult_01_01_exact(i4 %x) {
 ; CHECK-LABEL: @lshrult_01_01_exact(
-; CHECK-NEXT:    [[C:%.*]] = icmp eq i4 %x, 0
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i4 [[X:%.*]], 0
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = lshr exact i4 %x, 1
@@ -2236,7 +2237,7 @@ define i1 @lshrult_01_01_exact(i4 %x) {
 
 define i1 @lshrult_01_02_exact(i4 %x) {
 ; CHECK-LABEL: @lshrult_01_02_exact(
-; CHECK-NEXT:    [[C:%.*]] = icmp ult i4 %x, 4
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i4 [[X:%.*]], 4
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = lshr exact i4 %x, 1
@@ -2246,7 +2247,7 @@ define i1 @lshrult_01_02_exact(i4 %x) {
 
 define i1 @lshrult_01_03_exact(i4 %x) {
 ; CHECK-LABEL: @lshrult_01_03_exact(
-; CHECK-NEXT:    [[C:%.*]] = icmp ult i4 %x, 6
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i4 [[X:%.*]], 6
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = lshr exact i4 %x, 1
@@ -2256,7 +2257,7 @@ define i1 @lshrult_01_03_exact(i4 %x) {
 
 define i1 @lshrult_01_04_exact(i4 %x) {
 ; CHECK-LABEL: @lshrult_01_04_exact(
-; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 %x, -1
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 [[X:%.*]], -1
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = lshr exact i4 %x, 1
@@ -2266,7 +2267,7 @@ define i1 @lshrult_01_04_exact(i4 %x) {
 
 define i1 @lshrult_01_05_exact(i4 %x) {
 ; CHECK-LABEL: @lshrult_01_05_exact(
-; CHECK-NEXT:    [[C:%.*]] = icmp ult i4 %x, -6
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i4 [[X:%.*]], -6
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = lshr exact i4 %x, 1
@@ -2276,7 +2277,7 @@ define i1 @lshrult_01_05_exact(i4 %x) {
 
 define i1 @lshrult_01_06_exact(i4 %x) {
 ; CHECK-LABEL: @lshrult_01_06_exact(
-; CHECK-NEXT:    [[C:%.*]] = icmp ult i4 %x, -4
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i4 [[X:%.*]], -4
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = lshr exact i4 %x, 1
@@ -2286,7 +2287,7 @@ define i1 @lshrult_01_06_exact(i4 %x) {
 
 define i1 @lshrult_01_07_exact(i4 %x) {
 ; CHECK-LABEL: @lshrult_01_07_exact(
-; CHECK-NEXT:    [[C:%.*]] = icmp ne i4 %x, -2
+; CHECK-NEXT:    [[C:%.*]] = icmp ne i4 [[X:%.*]], -2
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = lshr exact i4 %x, 1
@@ -2377,7 +2378,7 @@ define i1 @lshrult_02_00_exact(i4 %x) {
 
 define i1 @lshrult_02_01_exact(i4 %x) {
 ; CHECK-LABEL: @lshrult_02_01_exact(
-; CHECK-NEXT:    [[C:%.*]] = icmp eq i4 %x, 0
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i4 [[X:%.*]], 0
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = lshr exact i4 %x, 2
@@ -2387,7 +2388,7 @@ define i1 @lshrult_02_01_exact(i4 %x) {
 
 define i1 @lshrult_02_02_exact(i4 %x) {
 ; CHECK-LABEL: @lshrult_02_02_exact(
-; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 %x, -1
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 [[X:%.*]], -1
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = lshr exact i4 %x, 2
@@ -2397,7 +2398,7 @@ define i1 @lshrult_02_02_exact(i4 %x) {
 
 define i1 @lshrult_02_03_exact(i4 %x) {
 ; CHECK-LABEL: @lshrult_02_03_exact(
-; CHECK-NEXT:    [[C:%.*]] = icmp ne i4 %x, -4
+; CHECK-NEXT:    [[C:%.*]] = icmp ne i4 [[X:%.*]], -4
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = lshr exact i4 %x, 2
@@ -2524,7 +2525,7 @@ define i1 @lshrult_03_00_exact(i4 %x) {
 
 define i1 @lshrult_03_01_exact(i4 %x) {
 ; CHECK-LABEL: @lshrult_03_01_exact(
-; CHECK-NEXT:    [[C:%.*]] = icmp ne i4 %x, -8
+; CHECK-NEXT:    [[C:%.*]] = icmp ne i4 [[X:%.*]], -8
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = lshr exact i4 %x, 3
@@ -2660,7 +2661,7 @@ define i1 @lshrult_03_15_exact(i4 %x) {
 
 define i1 @ashrsgt_01_00_exact(i4 %x) {
 ; CHECK-LABEL: @ashrsgt_01_00_exact(
-; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 %x, 0
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 [[X:%.*]], 0
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = ashr exact i4 %x, 1
@@ -2670,7 +2671,7 @@ define i1 @ashrsgt_01_00_exact(i4 %x) {
 
 define i1 @ashrsgt_01_01_exact(i4 %x) {
 ; CHECK-LABEL: @ashrsgt_01_01_exact(
-; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 %x, 2
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 [[X:%.*]], 2
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = ashr exact i4 %x, 1
@@ -2680,7 +2681,7 @@ define i1 @ashrsgt_01_01_exact(i4 %x) {
 
 define i1 @ashrsgt_01_02_exact(i4 %x) {
 ; CHECK-LABEL: @ashrsgt_01_02_exact(
-; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 %x, 4
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 [[X:%.*]], 4
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = ashr exact i4 %x, 1
@@ -2771,7 +2772,7 @@ define i1 @ashrsgt_01_11_exact(i4 %x) {
 
 define i1 @ashrsgt_01_12_exact(i4 %x) {
 ; CHECK-LABEL: @ashrsgt_01_12_exact(
-; CHECK-NEXT:    [[C:%.*]] = icmp ne i4 %x, -8
+; CHECK-NEXT:    [[C:%.*]] = icmp ne i4 [[X:%.*]], -8
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = ashr exact i4 %x, 1
@@ -2781,7 +2782,7 @@ define i1 @ashrsgt_01_12_exact(i4 %x) {
 
 define i1 @ashrsgt_01_13_exact(i4 %x) {
 ; CHECK-LABEL: @ashrsgt_01_13_exact(
-; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 %x, -6
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 [[X:%.*]], -6
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = ashr exact i4 %x, 1
@@ -2791,7 +2792,7 @@ define i1 @ashrsgt_01_13_exact(i4 %x) {
 
 define i1 @ashrsgt_01_14_exact(i4 %x) {
 ; CHECK-LABEL: @ashrsgt_01_14_exact(
-; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 %x, -4
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 [[X:%.*]], -4
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = ashr exact i4 %x, 1
@@ -2801,7 +2802,7 @@ define i1 @ashrsgt_01_14_exact(i4 %x) {
 
 define i1 @ashrsgt_01_15_exact(i4 %x) {
 ; CHECK-LABEL: @ashrsgt_01_15_exact(
-; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 %x, -1
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 [[X:%.*]], -1
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = ashr exact i4 %x, 1
@@ -2811,7 +2812,7 @@ define i1 @ashrsgt_01_15_exact(i4 %x) {
 
 define i1 @ashrsgt_02_00_exact(i4 %x) {
 ; CHECK-LABEL: @ashrsgt_02_00_exact(
-; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 %x, 0
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 [[X:%.*]], 0
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = ashr exact i4 %x, 2
@@ -2938,7 +2939,7 @@ define i1 @ashrsgt_02_13_exact(i4 %x) {
 
 define i1 @ashrsgt_02_14_exact(i4 %x) {
 ; CHECK-LABEL: @ashrsgt_02_14_exact(
-; CHECK-NEXT:    [[C:%.*]] = icmp ne i4 %x, -8
+; CHECK-NEXT:    [[C:%.*]] = icmp ne i4 [[X:%.*]], -8
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = ashr exact i4 %x, 2
@@ -2948,7 +2949,7 @@ define i1 @ashrsgt_02_14_exact(i4 %x) {
 
 define i1 @ashrsgt_02_15_exact(i4 %x) {
 ; CHECK-LABEL: @ashrsgt_02_15_exact(
-; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 %x, -1
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 [[X:%.*]], -1
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = ashr exact i4 %x, 2
@@ -3093,7 +3094,7 @@ define i1 @ashrsgt_03_14_exact(i4 %x) {
 
 define i1 @ashrsgt_03_15_exact(i4 %x) {
 ; CHECK-LABEL: @ashrsgt_03_15_exact(
-; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 %x, -1
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 [[X:%.*]], -1
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = ashr exact i4 %x, 3
@@ -3103,7 +3104,7 @@ define i1 @ashrsgt_03_15_exact(i4 %x) {
 
 define i1 @ashrslt_01_00_exact(i4 %x) {
 ; CHECK-LABEL: @ashrslt_01_00_exact(
-; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 %x, 0
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 [[X:%.*]], 0
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = ashr exact i4 %x, 1
@@ -3113,7 +3114,7 @@ define i1 @ashrslt_01_00_exact(i4 %x) {
 
 define i1 @ashrslt_01_01_exact(i4 %x) {
 ; CHECK-LABEL: @ashrslt_01_01_exact(
-; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 %x, 2
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 [[X:%.*]], 2
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = ashr exact i4 %x, 1
@@ -3123,7 +3124,7 @@ define i1 @ashrslt_01_01_exact(i4 %x) {
 
 define i1 @ashrslt_01_02_exact(i4 %x) {
 ; CHECK-LABEL: @ashrslt_01_02_exact(
-; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 %x, 4
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 [[X:%.*]], 4
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = ashr exact i4 %x, 1
@@ -3133,7 +3134,7 @@ define i1 @ashrslt_01_02_exact(i4 %x) {
 
 define i1 @ashrslt_01_03_exact(i4 %x) {
 ; CHECK-LABEL: @ashrslt_01_03_exact(
-; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 %x, 6
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 [[X:%.*]], 6
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = ashr exact i4 %x, 1
@@ -3224,7 +3225,7 @@ define i1 @ashrslt_01_12_exact(i4 %x) {
 
 define i1 @ashrslt_01_13_exact(i4 %x) {
 ; CHECK-LABEL: @ashrslt_01_13_exact(
-; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 %x, -6
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 [[X:%.*]], -6
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = ashr exact i4 %x, 1
@@ -3234,7 +3235,7 @@ define i1 @ashrslt_01_13_exact(i4 %x) {
 
 define i1 @ashrslt_01_14_exact(i4 %x) {
 ; CHECK-LABEL: @ashrslt_01_14_exact(
-; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 %x, -4
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 [[X:%.*]], -4
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = ashr exact i4 %x, 1
@@ -3244,7 +3245,7 @@ define i1 @ashrslt_01_14_exact(i4 %x) {
 
 define i1 @ashrslt_01_15_exact(i4 %x) {
 ; CHECK-LABEL: @ashrslt_01_15_exact(
-; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 %x, -2
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 [[X:%.*]], -2
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = ashr exact i4 %x, 1
@@ -3254,7 +3255,7 @@ define i1 @ashrslt_01_15_exact(i4 %x) {
 
 define i1 @ashrslt_02_00_exact(i4 %x) {
 ; CHECK-LABEL: @ashrslt_02_00_exact(
-; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 %x, 0
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 [[X:%.*]], 0
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = ashr exact i4 %x, 2
@@ -3264,7 +3265,7 @@ define i1 @ashrslt_02_00_exact(i4 %x) {
 
 define i1 @ashrslt_02_01_exact(i4 %x) {
 ; CHECK-LABEL: @ashrslt_02_01_exact(
-; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 %x, 4
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 [[X:%.*]], 4
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = ashr exact i4 %x, 2
@@ -3391,7 +3392,7 @@ define i1 @ashrslt_02_14_exact(i4 %x) {
 
 define i1 @ashrslt_02_15_exact(i4 %x) {
 ; CHECK-LABEL: @ashrslt_02_15_exact(
-; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 %x, -4
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 [[X:%.*]], -4
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = ashr exact i4 %x, 2
@@ -3401,7 +3402,7 @@ define i1 @ashrslt_02_15_exact(i4 %x) {
 
 define i1 @ashrslt_03_00_exact(i4 %x) {
 ; CHECK-LABEL: @ashrslt_03_00_exact(
-; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 %x, 0
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 [[X:%.*]], 0
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = ashr exact i4 %x, 3
diff --git a/llvm/test/Transforms/InstCombine/icmp-sub.ll b/llvm/test/Transforms/InstCombine/icmp-sub.ll
index c66581b3b..0486263ff 100644
--- a/llvm/test/Transforms/InstCombine/icmp-sub.ll
+++ b/llvm/test/Transforms/InstCombine/icmp-sub.ll
@@ -84,3 +84,62 @@ define i1 @test_negative_combined_sub_signed_overflow(i8 %x) {
   %z = icmp slt i8 %y, -1
   ret i1 %z
 }
+
+define i1 @test_sub_0_Y_eq_0(i8 %y) {
+; CHECK-LABEL: @test_sub_0_Y_eq_0(
+; CHECK-NEXT:    [[Z:%.*]] = icmp eq i8 [[Y:%.*]], 0
+; CHECK-NEXT:    ret i1 [[Z]]
+;
+  %s = sub i8 0, %y
+  %z = icmp eq i8 %s, 0
+  ret i1 %z
+}
+
+define i1 @test_sub_0_Y_ne_0(i8 %y) {
+; CHECK-LABEL: @test_sub_0_Y_ne_0(
+; CHECK-NEXT:    [[Z:%.*]] = icmp ne i8 [[Y:%.*]], 0
+; CHECK-NEXT:    ret i1 [[Z]]
+;
+  %s = sub i8 0, %y
+  %z = icmp ne i8 %s, 0
+  ret i1 %z
+}
+
+define i1 @test_sub_4_Y_ne_4(i8 %y) {
+; CHECK-LABEL: @test_sub_4_Y_ne_4(
+; CHECK-NEXT:    [[Z:%.*]] = icmp ne i8 [[Y:%.*]], 0
+; CHECK-NEXT:    ret i1 [[Z]]
+;
+  %s = sub i8 4, %y
+  %z = icmp ne i8 %s, 4
+  ret i1 %z
+}
+
+define i1 @test_sub_127_Y_eq_127(i8 %y) {
+; CHECK-LABEL: @test_sub_127_Y_eq_127(
+; CHECK-NEXT:    [[Z:%.*]] = icmp eq i8 [[Y:%.*]], 0
+; CHECK-NEXT:    ret i1 [[Z]]
+;
+  %s = sub i8 127, %y
+  %z = icmp eq i8 %s, 127
+  ret i1 %z
+}
+
+define i1 @test_sub_255_Y_eq_255(i8 %y) {
+; CHECK-LABEL: @test_sub_255_Y_eq_255(
+; CHECK-NEXT:    [[Z:%.*]] = icmp eq i8 [[Y:%.*]], 0
+; CHECK-NEXT:    ret i1 [[Z]]
+;
+  %s = sub i8 255, %y
+  %z = icmp eq i8 %s, 255
+  ret i1 %z
+}
+define <2 x i1> @test_sub_255_Y_eq_255_vec(<2 x i8> %y) {
+; CHECK-LABEL: @test_sub_255_Y_eq_255_vec(
+; CHECK-NEXT:    [[Z:%.*]] = icmp eq <2 x i8> [[Y:%.*]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[Z]]
+;
+  %s = sub <2 x i8> <i8 255, i8 255>, %y
+  %z = icmp eq <2 x i8> %s, <i8 255, i8 255>
+  ret <2 x i1> %z
+}
diff --git a/llvm/test/Transforms/InstCombine/icmp-vec.ll b/llvm/test/Transforms/InstCombine/icmp-vec.ll
index cb83db5f8..8f4cfdbe9 100644
--- a/llvm/test/Transforms/InstCombine/icmp-vec.ll
+++ b/llvm/test/Transforms/InstCombine/icmp-vec.ll
@@ -179,13 +179,22 @@ define <2 x i1> @PR27756_1(<2 x i8> %a) {
 
 ; Undef elements don't prevent the transform of the comparison.
 
-define <2 x i1> @PR27756_2(<2 x i8> %a) {
+define <3 x i1> @PR27756_2(<3 x i8> %a) {
 ; CHECK-LABEL: @PR27756_2(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i8> [[A:%.*]], <i8 undef, i8 1>
-; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <3 x i8> [[A:%.*]], <i8 43, i8 43, i8 1>
+; CHECK-NEXT:    ret <3 x i1> [[CMP]]
 ;
-  %cmp = icmp sle <2 x i8> %a, <i8 undef, i8 0>
-  ret <2 x i1> %cmp
+  %cmp = icmp sle <3 x i8> %a, <i8 42, i8 undef, i8 0>
+  ret <3 x i1> %cmp
+}
+
+define <3 x i1> @PR27756_3(<3 x i8> %a) {
+; CHECK-LABEL: @PR27756_3(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt <3 x i8> [[A:%.*]], <i8 0, i8 0, i8 41>
+; CHECK-NEXT:    ret <3 x i1> [[CMP]]
+;
+  %cmp = icmp sge <3 x i8> %a, <i8 undef, i8 1, i8 42>
+  ret <3 x i1> %cmp
 }
 
 @someglobal = global i32 0
diff --git a/llvm/test/Transforms/InstCombine/icmp.ll b/llvm/test/Transforms/InstCombine/icmp.ll
index 3fecf97e8..4ee2d4ec1 100644
--- a/llvm/test/Transforms/InstCombine/icmp.ll
+++ b/llvm/test/Transforms/InstCombine/icmp.ll
@@ -508,6 +508,21 @@ define i1 @test24(i64 %i) {
   ret i1 %cmp
 }
 
+; Note: offs can be negative, LLVM used to make an incorrect assumption that
+; unsigned overflow does not happen during offset computation
+define i1 @test24_neg_offs(i32* %p, i64 %offs) {
+; CHECK-LABEL: @test24_neg_offs(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i64 [[OFFS:%.*]], -2
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %p1 = getelementptr inbounds i32, i32* %p, i64 %offs
+  %conv1 = ptrtoint i32* %p to i64
+  %conv2 = ptrtoint i32* %p1 to i64
+  %delta = sub i64 %conv1, %conv2
+  %cmp = icmp eq i64 %delta, 8
+  ret i1 %cmp
+}
+
 @X_as1 = addrspace(1) global [1000 x i32] zeroinitializer
 
 define i1 @test24_as1(i64 %i) {
@@ -521,37 +536,31 @@ define i1 @test24_as1(i64 %i) {
   ret i1 %cmp
 }
 
-define i1 @test25(i32 %x, i32 %y, i32 %z) {
-; CHECK-LABEL: @test25(
+; X - Z > Y - Z -> X > Y if there is no overflow.
+define i1 @test27(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @test27(
 ; CHECK-NEXT:    [[C:%.*]] = icmp sgt i32 [[X:%.*]], [[Y:%.*]]
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
-  %lhs = add nsw i32 %x, %z
-  %rhs = add nsw i32 %y, %z
+  %lhs = sub nsw i32 %x, %z
+  %rhs = sub nsw i32 %y, %z
   %c = icmp sgt i32 %lhs, %rhs
   ret i1 %c
 }
 
-; X + Z > Y + Z -> X > Y if there is no overflow.
-define i1 @test26(i32 %x, i32 %y, i32 %z) {
-; CHECK-LABEL: @test26(
-; CHECK-NEXT:    [[C:%.*]] = icmp ugt i32 [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    ret i1 [[C]]
-;
-  %lhs = add nuw i32 %x, %z
-  %rhs = add nuw i32 %y, %z
-  %c = icmp ugt i32 %lhs, %rhs
-  ret i1 %c
-}
-
-; X - Z > Y - Z -> X > Y if there is no overflow.
-define i1 @test27(i32 %x, i32 %y, i32 %z) {
-; CHECK-LABEL: @test27(
-; CHECK-NEXT:    [[C:%.*]] = icmp sgt i32 [[X:%.*]], [[Y:%.*]]
+define i1 @test27_extra_uses(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @test27_extra_uses(
+; CHECK-NEXT:    [[LHS:%.*]] = sub nsw i32 [[X:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    call void @foo(i32 [[LHS]])
+; CHECK-NEXT:    [[RHS:%.*]] = sub nsw i32 [[Y:%.*]], [[Z]]
+; CHECK-NEXT:    call void @foo(i32 [[RHS]])
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i32 [[X]], [[Y]]
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %lhs = sub nsw i32 %x, %z
+  call void @foo(i32 %lhs)
   %rhs = sub nsw i32 %y, %z
+  call void @foo(i32 %rhs)
   %c = icmp sgt i32 %lhs, %rhs
   ret i1 %c
 }
@@ -568,47 +577,49 @@ define i1 @test28(i32 %x, i32 %y, i32 %z) {
   ret i1 %c
 }
 
-; X + Y > X -> Y > 0 if there is no overflow.
-define i1 @test29(i32 %x, i32 %y) {
-; CHECK-LABEL: @test29(
-; CHECK-NEXT:    [[C:%.*]] = icmp sgt i32 [[Y:%.*]], 0
+define i1 @test28_extra_uses(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @test28_extra_uses(
+; CHECK-NEXT:    [[LHS:%.*]] = sub nuw i32 [[X:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    call void @foo(i32 [[LHS]])
+; CHECK-NEXT:    [[RHS:%.*]] = sub nuw i32 [[Y:%.*]], [[Z]]
+; CHECK-NEXT:    call void @foo(i32 [[RHS]])
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i32 [[X]], [[Y]]
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
-  %lhs = add nsw i32 %x, %y
-  %c = icmp sgt i32 %lhs, %x
+  %lhs = sub nuw i32 %x, %z
+  call void @foo(i32 %lhs)
+  %rhs = sub nuw i32 %y, %z
+  call void @foo(i32 %rhs)
+  %c = icmp ugt i32 %lhs, %rhs
   ret i1 %c
 }
 
-; X + Y > X -> Y > 0 if there is no overflow.
-define i1 @test30(i32 %x, i32 %y) {
-; CHECK-LABEL: @test30(
-; CHECK-NEXT:    [[C:%.*]] = icmp ne i32 [[Y:%.*]], 0
-; CHECK-NEXT:    ret i1 [[C]]
-;
-  %lhs = add nuw i32 %x, %y
-  %c = icmp ugt i32 %lhs, %x
-  ret i1 %c
-}
+; PR36969 - https://bugs.llvm.org/show_bug.cgi?id=36969
 
-; X > X + Y -> 0 > Y if there is no overflow.
-define i1 @test31(i32 %x, i32 %y) {
-; CHECK-LABEL: @test31(
-; CHECK-NEXT:    [[C:%.*]] = icmp slt i32 [[Y:%.*]], 0
-; CHECK-NEXT:    ret i1 [[C]]
+define i1 @ugt_sub(i32 %xsrc, i32 %y) {
+; CHECK-LABEL: @ugt_sub(
+; CHECK-NEXT:    [[X:%.*]] = udiv i32 [[XSRC:%.*]], 42
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[X]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
 ;
-  %rhs = add nsw i32 %x, %y
-  %c = icmp sgt i32 %x, %rhs
-  ret i1 %c
+  %x = udiv i32 %xsrc, 42 ; thwart complexity-based canonicalization
+  %sub = sub i32 %x, %y
+  %cmp = icmp ugt i32 %sub, %x
+  ret i1 %cmp
 }
 
-; X > X + Y -> 0 > Y if there is no overflow.
-define i1 @test32(i32 %x, i32 %y) {
-; CHECK-LABEL: @test32(
-; CHECK-NEXT:    ret i1 false
+; Swap operands and predicate. Try a vector type to verify that works too.
+
+define <2 x i1> @ult_sub(<2 x i8> %xsrc, <2 x i8> %y) {
+; CHECK-LABEL: @ult_sub(
+; CHECK-NEXT:    [[X:%.*]] = udiv <2 x i8> [[XSRC:%.*]], <i8 42, i8 -42>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult <2 x i8> [[X]], [[Y:%.*]]
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
-  %rhs = add nuw i32 %x, %y
-  %c = icmp ugt i32 %x, %rhs
-  ret i1 %c
+  %x = udiv <2 x i8> %xsrc, <i8 42, i8 -42> ; thwart complexity-based canonicalization
+  %sub = sub <2 x i8> %x, %y
+  %cmp = icmp ult <2 x i8> %x, %sub
+  ret <2 x i1> %cmp
 }
 
 ; X - Y > X -> 0 > Y if there is no overflow.
@@ -654,34 +665,6 @@ define i1 @test36(i32 %x, i32 %y) {
   ret i1 %c
 }
 
-; PR36969 - https://bugs.llvm.org/show_bug.cgi?id=36969
-
-define i1 @ugt_sub(i32 %xsrc, i32 %y) {
-; CHECK-LABEL: @ugt_sub(
-; CHECK-NEXT:    [[X:%.*]] = udiv i32 [[XSRC:%.*]], 42
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[X]], [[Y:%.*]]
-; CHECK-NEXT:    ret i1 [[CMP]]
-;
-  %x = udiv i32 %xsrc, 42 ; thwart complexity-based canonicalization
-  %sub = sub i32 %x, %y
-  %cmp = icmp ugt i32 %sub, %x
-  ret i1 %cmp
-}
-
-; Swap operands and predicate. Try a vector type to verify that works too.
-
-define <2 x i1> @ult_sub(<2 x i8> %xsrc, <2 x i8> %y) {
-; CHECK-LABEL: @ult_sub(
-; CHECK-NEXT:    [[X:%.*]] = udiv <2 x i8> [[XSRC:%.*]], <i8 42, i8 -42>
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult <2 x i8> [[X]], [[Y:%.*]]
-; CHECK-NEXT:    ret <2 x i1> [[CMP]]
-;
-  %x = udiv <2 x i8> %xsrc, <i8 42, i8 -42> ; thwart complexity-based canonicalization
-  %sub = sub <2 x i8> %x, %y
-  %cmp = icmp ult <2 x i8> %x, %sub
-  ret <2 x i1> %cmp
-}
-
 ; X - Y > X - Z -> Z > Y if there is no overflow.
 define i1 @test37(i32 %x, i32 %y, i32 %z) {
 ; CHECK-LABEL: @test37(
@@ -694,6 +677,53 @@ define i1 @test37(i32 %x, i32 %y, i32 %z) {
   ret i1 %c
 }
 
+define i1 @test37_extra_uses(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @test37_extra_uses(
+; CHECK-NEXT:    [[LHS:%.*]] = sub nsw i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @foo(i32 [[LHS]])
+; CHECK-NEXT:    [[RHS:%.*]] = sub nsw i32 [[X]], [[Z:%.*]]
+; CHECK-NEXT:    call void @foo(i32 [[RHS]])
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i32 [[Z]], [[Y]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %lhs = sub nsw i32 %x, %y
+  call void @foo(i32 %lhs)
+  %rhs = sub nsw i32 %x, %z
+  call void @foo(i32 %rhs)
+  %c = icmp sgt i32 %lhs, %rhs
+  ret i1 %c
+}
+
+; TODO: Min/max pattern should not prevent the fold.
+
+define i32 @neg_max_s32(i32 %x, i32 %y) {
+; CHECK-LABEL: @neg_max_s32(
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i32 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[S_V:%.*]] = select i1 [[C]], i32 [[Y]], i32 [[X]]
+; CHECK-NEXT:    ret i32 [[S_V]]
+;
+  %nx = sub nsw i32 0, %x
+  %ny = sub nsw i32 0, %y
+  %c = icmp slt i32 %nx, %ny
+  %s = select i1 %c, i32 %ny, i32 %nx
+  %r = sub nsw i32 0, %s
+  ret i32 %r
+}
+
+define <4 x i32> @neg_max_v4s32(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: @neg_max_v4s32(
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt <4 x i32> [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[S_V:%.*]] = select <4 x i1> [[C]], <4 x i32> [[X]], <4 x i32> [[Y]]
+; CHECK-NEXT:    ret <4 x i32> [[S_V]]
+;
+  %nx = sub nsw <4 x i32> zeroinitializer, %x
+  %ny = sub nsw <4 x i32> zeroinitializer, %y
+  %c = icmp sgt <4 x i32> %nx, %ny
+  %s = select <4 x i1> %c, <4 x i32> %nx, <4 x i32> %ny
+  %r = sub <4 x i32> zeroinitializer, %s
+  ret <4 x i32> %r
+}
+
 ; X - Y > X - Z -> Z > Y if there is no overflow.
 define i1 @test38(i32 %x, i32 %y, i32 %z) {
 ; CHECK-LABEL: @test38(
@@ -706,6 +736,23 @@ define i1 @test38(i32 %x, i32 %y, i32 %z) {
   ret i1 %c
 }
 
+define i1 @test38_extra_uses(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @test38_extra_uses(
+; CHECK-NEXT:    [[LHS:%.*]] = sub nuw i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @foo(i32 [[LHS]])
+; CHECK-NEXT:    [[RHS:%.*]] = sub nuw i32 [[X]], [[Z:%.*]]
+; CHECK-NEXT:    call void @foo(i32 [[RHS]])
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i32 [[Z]], [[Y]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %lhs = sub nuw i32 %x, %y
+  call void @foo(i32 %lhs)
+  %rhs = sub nuw i32 %x, %z
+  call void @foo(i32 %rhs)
+  %c = icmp ugt i32 %lhs, %rhs
+  ret i1 %c
+}
+
 ; PR9343 #1
 define i1 @test39(i32 %X, i32 %Y) {
 ; CHECK-LABEL: @test39(
@@ -1090,7 +1137,7 @@ define i1 @test59_as1(i8 addrspace(1)* %foo) {
 
 define i1 @test60(i8* %foo, i64 %i, i64 %j) {
 ; CHECK-LABEL: @test60(
-; CHECK-NEXT:    [[GEP1_IDX:%.*]] = shl nuw i64 [[I:%.*]], 2
+; CHECK-NEXT:    [[GEP1_IDX:%.*]] = shl nsw i64 [[I:%.*]], 2
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i64 [[GEP1_IDX]], [[J:%.*]]
 ; CHECK-NEXT:    ret i1 [[TMP1]]
 ;
@@ -1106,7 +1153,7 @@ define i1 @test60_as1(i8 addrspace(1)* %foo, i64 %i, i64 %j) {
 ; CHECK-LABEL: @test60_as1(
 ; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[I:%.*]] to i16
 ; CHECK-NEXT:    [[TMP2:%.*]] = trunc i64 [[J:%.*]] to i16
-; CHECK-NEXT:    [[GEP1_IDX:%.*]] = shl nuw i16 [[TMP1]], 2
+; CHECK-NEXT:    [[GEP1_IDX:%.*]] = shl nsw i16 [[TMP1]], 2
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp slt i16 [[GEP1_IDX]], [[TMP2]]
 ; CHECK-NEXT:    ret i1 [[TMP3]]
 ;
@@ -1122,7 +1169,7 @@ define i1 @test60_as1(i8 addrspace(1)* %foo, i64 %i, i64 %j) {
 ; bitcast. This uses the same sized addrspace.
 define i1 @test60_addrspacecast(i8* %foo, i64 %i, i64 %j) {
 ; CHECK-LABEL: @test60_addrspacecast(
-; CHECK-NEXT:    [[GEP1_IDX:%.*]] = shl nuw i64 [[I:%.*]], 2
+; CHECK-NEXT:    [[GEP1_IDX:%.*]] = shl nsw i64 [[I:%.*]], 2
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i64 [[GEP1_IDX]], [[J:%.*]]
 ; CHECK-NEXT:    ret i1 [[TMP1]]
 ;
@@ -1136,7 +1183,7 @@ define i1 @test60_addrspacecast(i8* %foo, i64 %i, i64 %j) {
 
 define i1 @test60_addrspacecast_smaller(i8* %foo, i16 %i, i64 %j) {
 ; CHECK-LABEL: @test60_addrspacecast_smaller(
-; CHECK-NEXT:    [[GEP1_IDX:%.*]] = shl nuw i16 [[I:%.*]], 2
+; CHECK-NEXT:    [[GEP1_IDX:%.*]] = shl nsw i16 [[I:%.*]], 2
 ; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[J:%.*]] to i16
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i16 [[GEP1_IDX]], [[TMP1]]
 ; CHECK-NEXT:    ret i1 [[TMP2]]
diff --git a/llvm/test/Transforms/InstCombine/insert-extract-shuffle.ll b/llvm/test/Transforms/InstCombine/insert-extract-shuffle.ll
index 3a17acd24..d29bcc7d4 100644
--- a/llvm/test/Transforms/InstCombine/insert-extract-shuffle.ll
+++ b/llvm/test/Transforms/InstCombine/insert-extract-shuffle.ll
@@ -572,3 +572,164 @@ define <4 x float> @insert_in_nonsplat2(float %x, <4 x float> %y) {
   %r = insertelement <4 x float> %splat, float %x, i32 3
   ret <4 x float> %r
 }
+
+define <4 x i8> @shuf_identity_padding(<2 x i8> %x, i8 %y) {
+; CHECK-LABEL: @shuf_identity_padding(
+; CHECK-NEXT:    [[V1:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[V2:%.*]] = insertelement <4 x i8> [[V1]], i8 [[Y:%.*]], i32 2
+; CHECK-NEXT:    ret <4 x i8> [[V2]]
+;
+  %v0 = shufflevector <2 x i8> %x, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+  %x1 = extractelement <2 x i8> %x, i32 1
+  %v1 = insertelement <4 x i8> %v0, i8 %x1, i32 1
+  %v2 = insertelement <4 x i8> %v1, i8 %y, i32 2
+  ret <4 x i8> %v2
+}
+
+define <3 x i8> @shuf_identity_extract(<4 x i8> %x, i8 %y) {
+; CHECK-LABEL: @shuf_identity_extract(
+; CHECK-NEXT:    [[V1:%.*]] = shufflevector <4 x i8> [[X:%.*]], <4 x i8> undef, <3 x i32> <i32 0, i32 1, i32 undef>
+; CHECK-NEXT:    [[V2:%.*]] = insertelement <3 x i8> [[V1]], i8 [[Y:%.*]], i32 2
+; CHECK-NEXT:    ret <3 x i8> [[V2]]
+;
+  %v0 = shufflevector <4 x i8> %x, <4 x i8> undef, <3 x i32> <i32 0, i32 undef, i32 undef>
+  %x1 = extractelement <4 x i8> %x, i32 1
+  %v1 = insertelement <3 x i8> %v0, i8 %x1, i32 1
+  %v2 = insertelement <3 x i8> %v1, i8 %y, i32 2
+  ret <3 x i8> %v2
+}
+
+define <4 x float> @shuf_identity_extract_extra_use(<6 x float> %x, float %y) {
+; CHECK-LABEL: @shuf_identity_extract_extra_use(
+; CHECK-NEXT:    [[V0:%.*]] = shufflevector <6 x float> [[X:%.*]], <6 x float> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 3>
+; CHECK-NEXT:    call void @use(<4 x float> [[V0]])
+; CHECK-NEXT:    [[V1:%.*]] = shufflevector <6 x float> [[X]], <6 x float> undef, <4 x i32> <i32 0, i32 undef, i32 2, i32 3>
+; CHECK-NEXT:    [[V2:%.*]] = insertelement <4 x float> [[V1]], float [[Y:%.*]], i32 1
+; CHECK-NEXT:    ret <4 x float> [[V2]]
+;
+  %v0 = shufflevector <6 x float> %x, <6 x float> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 3>
+  call void @use(<4 x float> %v0)
+  %x1 = extractelement <6 x float> %x, i32 2
+  %v1 = insertelement <4 x float> %v0, float %x1, i32 2
+  %v2 = insertelement <4 x float> %v1, float %y, i32 1
+  ret <4 x float> %v2
+}
+
+; Negative test - can't map variable index to shuffle mask.
+
+define <4 x i8> @shuf_identity_padding_variable_index(<2 x i8> %x, i8 %y, i32 %index) {
+; CHECK-LABEL: @shuf_identity_padding_variable_index(
+; CHECK-NEXT:    [[V0:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[X1:%.*]] = extractelement <2 x i8> [[X]], i32 [[INDEX:%.*]]
+; CHECK-NEXT:    [[V1:%.*]] = insertelement <4 x i8> [[V0]], i8 [[X1]], i32 [[INDEX]]
+; CHECK-NEXT:    [[V2:%.*]] = insertelement <4 x i8> [[V1]], i8 [[Y:%.*]], i32 2
+; CHECK-NEXT:    ret <4 x i8> [[V2]]
+;
+  %v0 = shufflevector <2 x i8> %x, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+  %x1 = extractelement <2 x i8> %x, i32 %index
+  %v1 = insertelement <4 x i8> %v0, i8 %x1, i32 %index
+  %v2 = insertelement <4 x i8> %v1, i8 %y, i32 2
+  ret <4 x i8> %v2
+}
+
+; Negative test - don't create arbitrary shuffle masks.
+
+define <4 x i8> @shuf_identity_padding_wrong_source_vec(<2 x i8> %x, i8 %y, <2 x i8> %other) {
+; CHECK-LABEL: @shuf_identity_padding_wrong_source_vec(
+; CHECK-NEXT:    [[V0:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[X1:%.*]] = extractelement <2 x i8> [[OTHER:%.*]], i32 1
+; CHECK-NEXT:    [[V1:%.*]] = insertelement <4 x i8> [[V0]], i8 [[X1]], i32 1
+; CHECK-NEXT:    [[V2:%.*]] = insertelement <4 x i8> [[V1]], i8 [[Y:%.*]], i32 2
+; CHECK-NEXT:    ret <4 x i8> [[V2]]
+;
+  %v0 = shufflevector <2 x i8> %x, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+  %x1 = extractelement <2 x i8> %other, i32 1
+  %v1 = insertelement <4 x i8> %v0, i8 %x1, i32 1
+  %v2 = insertelement <4 x i8> %v1, i8 %y, i32 2
+  ret <4 x i8> %v2
+}
+
+; Negative test - don't create arbitrary shuffle masks.
+
+define <4 x i8> @shuf_identity_padding_wrong_index(<2 x i8> %x, i8 %y) {
+; CHECK-LABEL: @shuf_identity_padding_wrong_index(
+; CHECK-NEXT:    [[V0:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[X1:%.*]] = extractelement <2 x i8> [[X]], i32 1
+; CHECK-NEXT:    [[V1:%.*]] = insertelement <4 x i8> [[V0]], i8 [[X1]], i32 2
+; CHECK-NEXT:    [[V2:%.*]] = insertelement <4 x i8> [[V1]], i8 [[Y:%.*]], i32 3
+; CHECK-NEXT:    ret <4 x i8> [[V2]]
+;
+  %v0 = shufflevector <2 x i8> %x, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+  %x1 = extractelement <2 x i8> %x, i32 1
+  %v1 = insertelement <4 x i8> %v0, i8 %x1, i32 2
+  %v2 = insertelement <4 x i8> %v1, i8 %y, i32 3
+  ret <4 x i8> %v2
+}
+
+define <4 x float> @insert_undemanded_element_op0(<4 x float> %x, <4 x float> %y) {
+; CHECK-LABEL: @insert_undemanded_element_op0(
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <4 x float> [[X:%.*]], float 4.200000e+01, i32 3
+; CHECK-NEXT:    call void @use(<4 x float> [[INS]])
+; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x float> [[X]], <4 x float> [[Y:%.*]], <4 x i32> <i32 0, i32 7, i32 1, i32 4>
+; CHECK-NEXT:    ret <4 x float> [[S]]
+;
+  %ins = insertelement <4 x float> %x, float 42.0, i32 3
+  call void @use(<4 x float> %ins)
+  %s = shufflevector <4 x float> %ins, <4 x float> %y, <4 x i32> <i32 0, i32 7, i32 1, i32 4>
+  ret <4 x float> %s
+}
+
+define <4 x float> @insert_undemanded_element_op1(<4 x float> %x, <4 x float> %y) {
+; CHECK-LABEL: @insert_undemanded_element_op1(
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <4 x float> [[X:%.*]], float 4.200000e+01, i32 3
+; CHECK-NEXT:    call void @use(<4 x float> [[INS]])
+; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x float> [[Y:%.*]], <4 x float> [[X]], <4 x i32> <i32 3, i32 2, i32 1, i32 4>
+; CHECK-NEXT:    ret <4 x float> [[S]]
+;
+  %ins = insertelement <4 x float> %x, float 42.0, i32 3
+  call void @use(<4 x float> %ins)
+  %s = shufflevector <4 x float> %y, <4 x float> %ins, <4 x i32> <i32 3, i32 2, i32 1, i32 4>
+  ret <4 x float> %s
+}
+
+; Negative test - shuffle chooses the inserted constant.
+
+define <4 x float> @insert_demanded_element_op0(<4 x float> %x, <4 x float> %y) {
+; CHECK-LABEL: @insert_demanded_element_op0(
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <4 x float> [[X:%.*]], float 4.200000e+01, i32 3
+; CHECK-NEXT:    call void @use(<4 x float> [[INS]])
+; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x float> [[INS]], <4 x float> [[Y:%.*]], <4 x i32> <i32 3, i32 2, i32 1, i32 4>
+; CHECK-NEXT:    ret <4 x float> [[S]]
+;
+  %ins = insertelement <4 x float> %x, float 42.0, i32 3
+  call void @use(<4 x float> %ins)
+  %s = shufflevector <4 x float> %ins, <4 x float> %y, <4 x i32> <i32 3, i32 2, i32 1, i32 4>
+  ret <4 x float> %s
+}
+
+; Negative test - shuffle chooses the inserted constant.
+
+define <4 x float> @insert_demanded_element_op1(<4 x float> %x, <4 x float> %y) {
+; CHECK-LABEL: @insert_demanded_element_op1(
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <4 x float> [[X:%.*]], float 4.300000e+01, i32 3
+; CHECK-NEXT:    call void @use(<4 x float> [[INS]])
+; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x float> [[Y:%.*]], <4 x float> [[INS]], <4 x i32> <i32 0, i32 7, i32 1, i32 4>
+; CHECK-NEXT:    ret <4 x float> [[S]]
+;
+  %ins = insertelement <4 x float> %x, float 43.0, i32 3
+  call void @use(<4 x float> %ins)
+  %s = shufflevector <4 x float> %y, <4 x float> %ins, <4 x i32> <i32 0, i32 7, i32 1, i32 4>
+  ret <4 x float> %s
+}
+
+define <4 x float> @splat_constant(<4 x float> %x) {
+; CHECK-LABEL: @splat_constant(
+; CHECK-NEXT:    [[INS3:%.*]] = insertelement <4 x float> [[X:%.*]], float 3.000000e+00, i32 3
+; CHECK-NEXT:    [[R:%.*]] = fadd <4 x float> [[INS3]], <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00>
+; CHECK-NEXT:    ret <4 x float> [[R]]
+;
+  %ins3 = insertelement <4 x float> %x, float 3.0, i32 3
+  %splat3 = shufflevector <4 x float> %ins3, <4 x float> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+  %r = fadd <4 x float> %ins3, %splat3
+  ret <4 x float> %r
+}
diff --git a/llvm/test/Transforms/InstCombine/intptr8.ll b/llvm/test/Transforms/InstCombine/intptr8.ll
new file mode 100644
index 000000000..dd2cc2053
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/intptr8.ll
@@ -0,0 +1,77 @@
+; RUN: opt < %s  -instcombine -S | FileCheck %s
+
+; PR45033: Don't try to insert a cast into a catchswich block.
+
+target datalayout = "e-m:w-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc"
+
+%struct.intrusive_ptr = type { %struct.C* }
+%struct.C = type { %struct.intrusive_ref_counter }
+%struct.intrusive_ref_counter = type { i32 }
+
+declare dso_local %struct.C* @"?mk@@YAPEAUC@@XZ"() #3
+declare dso_local void @"?intrusive_ptr_release@@YAXPEBUintrusive_ref_counter@@@Z"(%struct.intrusive_ref_counter*) #3
+declare dso_local void @"?terminate@@YAXXZ"()
+declare dso_local i32 @__CxxFrameHandler3(...)
+
+define dso_local void @"?crash@@YAXXZ"() local_unnamed_addr #0 personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
+entry:
+  %call1 = invoke %struct.C* @"?mk@@YAPEAUC@@XZ"()
+          to label %invoke.cont2 unwind label %catch.dispatch
+
+invoke.cont2:                                     ; preds = %entry
+  %0 = ptrtoint %struct.C* %call1 to i64
+  %call5 = invoke %struct.C* @"?mk@@YAPEAUC@@XZ"()
+          to label %try.cont unwind label %catch.dispatch
+
+catch.dispatch:                                   ; preds = %invoke.cont2, %entry
+  %a.sroa.0.0 = phi i64 [ %0, %invoke.cont2 ], [ 0, %entry ]
+  %1 = catchswitch within none [label %catch] unwind label %ehcleanup
+
+catch:                                            ; preds = %catch.dispatch
+  %2 = catchpad within %1 [i8* null, i32 64, i8* null]
+  catchret from %2 to label %try.cont
+
+try.cont:                                         ; preds = %invoke.cont2, %catch
+  %a.sroa.0.1 = phi i64 [ %0, %invoke.cont2 ], [ %a.sroa.0.0, %catch ]
+  %3 = inttoptr i64 %a.sroa.0.1 to %struct.C*
+  %tobool.i3 = icmp eq %struct.C* %3, null
+  br i1 %tobool.i3, label %"??1?$intrusive_ptr@UC@@@@QEAA@XZ.exit6", label %if.then.i4
+
+if.then.i4:                                       ; preds = %try.cont
+  %4 = getelementptr %struct.C, %struct.C* %3, i64 0, i32 0
+  invoke void @"?intrusive_ptr_release@@YAXPEBUintrusive_ref_counter@@@Z"(%struct.intrusive_ref_counter* %4)
+          to label %"??1?$intrusive_ptr@UC@@@@QEAA@XZ.exit6" unwind label %terminate.i5
+
+terminate.i5:                                     ; preds = %if.then.i4
+  %5 = cleanuppad within none []
+  call void @"?terminate@@YAXXZ"() #4 [ "funclet"(token %5) ]
+  unreachable
+
+"??1?$intrusive_ptr@UC@@@@QEAA@XZ.exit6":         ; preds = %try.cont, %if.then.i4
+  ret void
+
+ehcleanup:                                        ; preds = %catch.dispatch
+  %6 = cleanuppad within none []
+  %7 = inttoptr i64 %a.sroa.0.0 to %struct.C*
+  %tobool.i = icmp eq %struct.C* %7, null
+  br i1 %tobool.i, label %"??1?$intrusive_ptr@UC@@@@QEAA@XZ.exit", label %if.then.i
+
+if.then.i:                                        ; preds = %ehcleanup
+  %8 = getelementptr %struct.C, %struct.C* %7, i64 0, i32 0
+  invoke void @"?intrusive_ptr_release@@YAXPEBUintrusive_ref_counter@@@Z"(%struct.intrusive_ref_counter* %8) [ "funclet"(token %6) ]
+          to label %"??1?$intrusive_ptr@UC@@@@QEAA@XZ.exit" unwind label %terminate.i
+
+terminate.i:                                      ; preds = %if.then.i
+  %9 = cleanuppad within %6 []
+  call void @"?terminate@@YAXXZ"() #4 [ "funclet"(token %9) ]
+  unreachable
+
+"??1?$intrusive_ptr@UC@@@@QEAA@XZ.exit":          ; preds = %ehcleanup, %if.then.i
+  cleanupret from %6 unwind to caller
+}
+
+; CHECK-LABEL: define dso_local void @"?crash@@YAXXZ"
+; CHECK: catch.dispatch:
+; CHECK-NEXT: %a.sroa.0.0 = phi i64
+; CHECK-NEXT: catchswitch within none [label %catch] unwind label %ehcleanup
diff --git a/llvm/test/Transforms/InstCombine/known-non-zero.ll b/llvm/test/Transforms/InstCombine/known-non-zero.ll
index 0329fbaf9..5467db556 100644
--- a/llvm/test/Transforms/InstCombine/known-non-zero.ll
+++ b/llvm/test/Transforms/InstCombine/known-non-zero.ll
@@ -13,7 +13,7 @@ define i32 @test0(i64 %x) {
 ; CHECK-NEXT:    [[C:%.*]] = icmp eq i64 [[X:%.*]], 0
 ; CHECK-NEXT:    br i1 [[C]], label [[EXIT:%.*]], label [[NON_ZERO:%.*]]
 ; CHECK:       non_zero:
-; CHECK-NEXT:    [[CTZ:%.*]] = call i64 @llvm.cttz.i64(i64 [[X]], i1 false), !range !0
+; CHECK-NEXT:    [[CTZ:%.*]] = call i64 @llvm.cttz.i64(i64 [[X]], i1 true), !range !0
 ; CHECK-NEXT:    [[CTZ32:%.*]] = trunc i64 [[CTZ]] to i32
 ; CHECK-NEXT:    br label [[EXIT]]
 ; CHECK:       exit:
@@ -40,7 +40,7 @@ define i32 @test1(i64 %x) {
 ; CHECK-NEXT:    [[C:%.*]] = icmp eq i64 [[X:%.*]], 0
 ; CHECK-NEXT:    br i1 [[C]], label [[EXIT:%.*]], label [[NON_ZERO:%.*]]
 ; CHECK:       non_zero:
-; CHECK-NEXT:    [[CTZ:%.*]] = call i64 @llvm.ctlz.i64(i64 [[X]], i1 false), !range !0
+; CHECK-NEXT:    [[CTZ:%.*]] = call i64 @llvm.ctlz.i64(i64 [[X]], i1 true), !range !0
 ; CHECK-NEXT:    [[CTZ32:%.*]] = trunc i64 [[CTZ]] to i32
 ; CHECK-NEXT:    br label [[EXIT]]
 ; CHECK:       exit:
@@ -91,3 +91,45 @@ exit:
   %res = phi <8 x i64> [ %ctz, %non_zero ], [ zeroinitializer, %start ]
   ret <8 x i64> %res
 }
+
+; Test that exposed a bug in the PHI handling after D60846. No folding should happen here!
+define void @D60846_misompile(i1* %p) {
+; CHECK-LABEL: @D60846_misompile(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[I:%.*]] = phi i16 [ 0, [[ENTRY:%.*]] ], [ [[I_INC:%.*]], [[COMMON:%.*]] ]
+; CHECK-NEXT:    [[IS_ZERO:%.*]] = icmp eq i16 [[I]], 0
+; CHECK-NEXT:    br i1 [[IS_ZERO]], label [[COMMON]], label [[NON_ZERO:%.*]]
+; CHECK:       non_zero:
+; CHECK-NEXT:    [[IS_ONE:%.*]] = icmp eq i16 [[I]], 1
+; CHECK-NEXT:    store i1 [[IS_ONE]], i1* [[P:%.*]], align 1
+; CHECK-NEXT:    br label [[COMMON]]
+; CHECK:       common:
+; CHECK-NEXT:    [[I_INC]] = add i16 [[I]], 1
+; CHECK-NEXT:    [[LOOP_COND:%.*]] = icmp ult i16 [[I_INC]], 2
+; CHECK-NEXT:    br i1 [[LOOP_COND]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:                                             ; preds = %common, %entry
+  %i = phi i16 [ 0, %entry ], [ %i.inc, %common ]
+  %is_zero = icmp eq i16 %i, 0
+  br i1 %is_zero, label %common, label %non_zero
+
+non_zero:                                         ; preds = %loop
+  %is_one = icmp eq i16 %i, 1
+  store i1 %is_one, i1* %p
+  br label %common
+
+common:                                           ; preds = %non_zero, %loop
+  %i.inc = add i16 %i, 1
+  %loop_cond = icmp ult i16 %i.inc, 2
+  br i1 %loop_cond, label %loop, label %exit
+
+exit:                                             ; preds = %common
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/lifetime-sanitizer.ll b/llvm/test/Transforms/InstCombine/lifetime-sanitizer.ll
new file mode 100644
index 000000000..4b3cad1b9
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/lifetime-sanitizer.ll
@@ -0,0 +1,64 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture)
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture)
+declare void @foo(i8* nocapture)
+
+define void @asan() sanitize_address {
+entry:
+  ; CHECK-LABEL: @asan(
+  %text = alloca i8, align 1
+
+  call void @llvm.lifetime.start.p0i8(i64 1, i8* %text)
+  call void @llvm.lifetime.end.p0i8(i64 1, i8* %text)
+  ; CHECK: call void @llvm.lifetime.start
+  ; CHECK-NEXT: call void @llvm.lifetime.end
+
+  call void @foo(i8* %text) ; Keep alloca alive
+
+  ret void
+}
+
+define void @hwasan() sanitize_hwaddress {
+entry:
+  ; CHECK-LABEL: @hwasan(
+  %text = alloca i8, align 1
+
+  call void @llvm.lifetime.start.p0i8(i64 1, i8* %text)
+  call void @llvm.lifetime.end.p0i8(i64 1, i8* %text)
+  ; CHECK: call void @llvm.lifetime.start
+  ; CHECK-NEXT: call void @llvm.lifetime.end
+
+  call void @foo(i8* %text) ; Keep alloca alive
+
+  ret void
+}
+
+define void @msan() sanitize_memory {
+entry:
+  ; CHECK-LABEL: @msan(
+  %text = alloca i8, align 1
+
+  call void @llvm.lifetime.start.p0i8(i64 1, i8* %text)
+  call void @llvm.lifetime.end.p0i8(i64 1, i8* %text)
+  ; CHECK: call void @llvm.lifetime.start
+  ; CHECK-NEXT: call void @llvm.lifetime.end
+
+  call void @foo(i8* %text) ; Keep alloca alive
+
+  ret void
+}
+
+define void @no_asan() {
+entry:
+  ; CHECK-LABEL: @no_asan(
+  %text = alloca i8, align 1
+
+  call void @llvm.lifetime.start.p0i8(i64 1, i8* %text)
+  call void @llvm.lifetime.end.p0i8(i64 1, i8* %text)
+  ; CHECK-NO: call void @llvm.lifetime
+
+  call void @foo(i8* %text) ; Keep alloca alive
+
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/limit-max-iterations.ll b/llvm/test/Transforms/InstCombine/limit-max-iterations.ll
new file mode 100644
index 000000000..a2ef4ebcb
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/limit-max-iterations.ll
@@ -0,0 +1,44 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine --instcombine-max-iterations=0 -S | FileCheck %s --check-prefix=ZERO
+; RUN: opt < %s -instcombine --instcombine-max-iterations=1 -S | FileCheck %s --check-prefix=ONE
+; RUN: opt < %s -instcombine -S | FileCheck %s --check-prefix=FIXPOINT
+; RUN: not opt < %s -instcombine -S --instcombine-infinite-loop-threshold=3 2>&1 | FileCheck %s --check-prefix=LOOP
+
+; Based on xor-of-icmps-with-extra-uses.ll. This requires multiple iterations of
+; InstCombine to reach a fixpoint.
+
+define i1 @v0_select_of_consts(i32 %X, i32* %selected) {
+; ZERO-LABEL: @v0_select_of_consts(
+; ZERO-NEXT:    [[COND0:%.*]] = icmp sgt i32 [[X:%.*]], 32767
+; ZERO-NEXT:    [[COND1:%.*]] = icmp sgt i32 [[X]], -32768
+; ZERO-NEXT:    [[SELECT:%.*]] = select i1 [[COND0]], i32 32767, i32 -32768
+; ZERO-NEXT:    store i32 [[SELECT]], i32* [[SELECTED:%.*]]
+; ZERO-NEXT:    [[RES:%.*]] = xor i1 [[COND0]], [[COND1]]
+; ZERO-NEXT:    ret i1 [[RES]]
+
+; ONE-LABEL: @v0_select_of_consts(
+; ONE-NEXT:    [[COND0:%.*]] = icmp sle i32 [[X:%.*]], 32767
+; ONE-NEXT:    [[COND0_NOT:%.*]] = xor i1 [[COND0]], true
+; ONE-NEXT:    [[COND1:%.*]] = icmp sgt i32 [[X]], -32768
+; ONE-NEXT:    [[SELECT:%.*]] = select i1 [[COND0_NOT]], i32 32767, i32 -32768
+; ONE-NEXT:    store i32 [[SELECT]], i32* [[SELECTED:%.*]], align 4
+; ONE-NEXT:    [[TMP1:%.*]] = and i1 [[COND0]], [[COND1]]
+; ONE-NEXT:    ret i1 [[TMP1]]
+
+; FIXPOINT-LABEL: @v0_select_of_consts(
+; FIXPOINT-NEXT:    [[COND0_INV:%.*]] = icmp sgt i32 [[X:%.*]], 32767
+; FIXPOINT-NEXT:    [[SELECT:%.*]] = select i1 [[COND0_INV]], i32 32767, i32 -32768
+; FIXPOINT-NEXT:    store i32 [[SELECT]], i32* [[SELECTED:%.*]], align 4
+; FIXPOINT-NEXT:    [[X_OFF:%.*]] = add i32 [[X]], 32767
+; FIXPOINT-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[X_OFF]], 65535
+; FIXPOINT-NEXT:    ret i1 [[TMP1]]
+
+; LOOP: LLVM ERROR: Instruction Combining seems stuck in an infinite loop after 3 iterations.
+
+  %cond0 = icmp sgt i32 %X, 32767
+  %cond1 = icmp sgt i32 %X, -32768
+  %select = select i1 %cond0, i32 32767, i32 -32768
+  store i32 %select, i32* %selected
+  %res = xor i1 %cond0, %cond1
+  ret i1 %res
+}
diff --git a/llvm/test/Transforms/InstCombine/load-bitcast-vec.ll b/llvm/test/Transforms/InstCombine/load-bitcast-vec.ll
index e6540ee70..644f1f5ae 100644
--- a/llvm/test/Transforms/InstCombine/load-bitcast-vec.ll
+++ b/llvm/test/Transforms/InstCombine/load-bitcast-vec.ll
@@ -67,6 +67,52 @@ define float @matching_scalar_small_deref(<4 x float>* dereferenceable(15) %p) {
   ret float %r
 }
 
+define float @matching_scalar_smallest_deref(<4 x float>* dereferenceable(1) %p) {
+; CHECK-LABEL: @matching_scalar_smallest_deref(
+; CHECK-NEXT:    [[BC:%.*]] = getelementptr inbounds <4 x float>, <4 x float>* [[P:%.*]], i64 0, i64 0
+; CHECK-NEXT:    [[R:%.*]] = load float, float* [[BC]], align 16
+; CHECK-NEXT:    ret float [[R]]
+;
+  %bc = bitcast <4 x float>* %p to float*
+  %r = load float, float* %bc, align 16
+  ret float %r
+}
+
+define float @matching_scalar_smallest_deref_or_null(<4 x float>* dereferenceable_or_null(1) %p) {
+; CHECK-LABEL: @matching_scalar_smallest_deref_or_null(
+; CHECK-NEXT:    [[BC:%.*]] = getelementptr inbounds <4 x float>, <4 x float>* [[P:%.*]], i64 0, i64 0
+; CHECK-NEXT:    [[R:%.*]] = load float, float* [[BC]], align 16
+; CHECK-NEXT:    ret float [[R]]
+;
+  %bc = bitcast <4 x float>* %p to float*
+  %r = load float, float* %bc, align 16
+  ret float %r
+}
+
+define float @matching_scalar_smallest_deref_addrspace(<4 x float> addrspace(4)* dereferenceable(1) %p) {
+; CHECK-LABEL: @matching_scalar_smallest_deref_addrspace(
+; CHECK-NEXT:    [[BC:%.*]] = getelementptr inbounds <4 x float>, <4 x float> addrspace(4)* [[P:%.*]], i64 0, i64 0
+; CHECK-NEXT:    [[R:%.*]] = load float, float addrspace(4)* [[BC]], align 16
+; CHECK-NEXT:    ret float [[R]]
+;
+  %bc = bitcast <4 x float> addrspace(4)* %p to float addrspace(4)*
+  %r = load float, float addrspace(4)* %bc, align 16
+  ret float %r
+}
+
+; A null pointer can't be assumed inbounds in a non-default address space.
+
+define float @matching_scalar_smallest_deref_or_null_addrspace(<4 x float> addrspace(4)* dereferenceable_or_null(1) %p) {
+; CHECK-LABEL: @matching_scalar_smallest_deref_or_null_addrspace(
+; CHECK-NEXT:    [[BC:%.*]] = getelementptr <4 x float>, <4 x float> addrspace(4)* [[P:%.*]], i64 0, i64 0
+; CHECK-NEXT:    [[R:%.*]] = load float, float addrspace(4)* [[BC]], align 16
+; CHECK-NEXT:    ret float [[R]]
+;
+  %bc = bitcast <4 x float> addrspace(4)* %p to float addrspace(4)*
+  %r = load float, float addrspace(4)* %bc, align 16
+  ret float %r
+}
+
 define float @matching_scalar_volatile(<4 x float>* dereferenceable(16) %p) {
 ; CHECK-LABEL: @matching_scalar_volatile(
 ; CHECK-NEXT:    [[BC:%.*]] = getelementptr inbounds <4 x float>, <4 x float>* [[P:%.*]], i64 0, i64 0
diff --git a/llvm/test/Transforms/InstCombine/load-bitcast64.ll b/llvm/test/Transforms/InstCombine/load-bitcast64.ll
index d14c686d8..270bd6f5f 100644
--- a/llvm/test/Transforms/InstCombine/load-bitcast64.ll
+++ b/llvm/test/Transforms/InstCombine/load-bitcast64.ll
@@ -1,13 +1,17 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -instcombine -S < %s | FileCheck %s
 
-target datalayout = "p:64:64:64"
+target datalayout = "p:64:64:64-i64:32:32"
 
 
 define i64* @test1(i8* %x) {
-entry:
 ; CHECK-LABEL: @test1(
-; CHECK: load i64*, i64**
-; CHECK: ret
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i8* [[X:%.*]] to i64**
+; CHECK-NEXT:    [[B1:%.*]] = load i64*, i64** [[TMP0]], align 4
+; CHECK-NEXT:    ret i64* [[B1]]
+;
+entry:
   %a = bitcast i8* %x to i64*
   %b = load i64, i64* %a
   %c = inttoptr i64 %b to i64*
@@ -16,10 +20,15 @@ entry:
 }
 
 define i32* @test2(i8* %x) {
-entry:
 ; CHECK-LABEL: @test2(
-; CHECK: load i32, i32*
-; CHECK: ret
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[A:%.*]] = bitcast i8* [[X:%.*]] to i32*
+; CHECK-NEXT:    [[B:%.*]] = load i32, i32* [[A]], align 4
+; CHECK-NEXT:    [[TMP0:%.*]] = zext i32 [[B]] to i64
+; CHECK-NEXT:    [[C:%.*]] = inttoptr i64 [[TMP0]] to i32*
+; CHECK-NEXT:    ret i32* [[C]]
+;
+entry:
   %a = bitcast i8* %x to i32*
   %b = load i32, i32* %a
   %c = inttoptr i32 %b to i32*
@@ -28,10 +37,15 @@ entry:
 }
 
 define i64* @test3(i8* %x) {
-entry:
 ; CHECK-LABEL: @test3(
-; CHECK: load i32, i32*
-; CHECK: ret
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[A:%.*]] = bitcast i8* [[X:%.*]] to i32*
+; CHECK-NEXT:    [[B:%.*]] = load i32, i32* [[A]], align 4
+; CHECK-NEXT:    [[TMP0:%.*]] = zext i32 [[B]] to i64
+; CHECK-NEXT:    [[C:%.*]] = inttoptr i64 [[TMP0]] to i64*
+; CHECK-NEXT:    ret i64* [[C]]
+;
+entry:
   %a = bitcast i8* %x to i32*
   %b = load i32, i32* %a
   %c = inttoptr i32 %b to i64*
@@ -40,10 +54,13 @@ entry:
 }
 
 define i64 @test4(i8* %x) {
-entry:
 ; CHECK-LABEL: @test4(
-; CHECK: load i64, i64*
-; CHECK: ret
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i8* [[X:%.*]] to i64*
+; CHECK-NEXT:    [[B1:%.*]] = load i64, i64* [[TMP0]], align 8
+; CHECK-NEXT:    ret i64 [[B1]]
+;
+entry:
   %a = bitcast i8* %x to i64**
   %b = load i64*, i64** %a
   %c = ptrtoint i64* %b to i64
@@ -52,11 +69,14 @@ entry:
 }
 
 define i32 @test5(i8* %x) {
-entry:
 ; CHECK-LABEL: @test5(
-; CHECK: load i64, i64*
-; CHECK: trunc
-; CHECK: ret
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i8* [[X:%.*]] to i64*
+; CHECK-NEXT:    [[B1:%.*]] = load i64, i64* [[TMP0]], align 8
+; CHECK-NEXT:    [[C:%.*]] = trunc i64 [[B1]] to i32
+; CHECK-NEXT:    ret i32 [[C]]
+;
+entry:
   %a = bitcast i8* %x to i32**
   %b = load i32*, i32** %a
   %c = ptrtoint i32* %b to i32
@@ -65,10 +85,13 @@ entry:
 }
 
 define i64 @test6(i8* %x) {
-entry:
 ; CHECK-LABEL: @test6(
-; CHECK: load i64, i64*
-; CHECK: ret
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i8* [[X:%.*]] to i64*
+; CHECK-NEXT:    [[B1:%.*]] = load i64, i64* [[TMP0]], align 8
+; CHECK-NEXT:    ret i64 [[B1]]
+;
+entry:
   %a = bitcast i8* %x to i32**
   %b = load i32*, i32** %a
   %c = ptrtoint i32* %b to i64
diff --git a/llvm/test/Transforms/InstCombine/load-cmp.ll b/llvm/test/Transforms/InstCombine/load-cmp.ll
index 5746b7aa2..7b24ce309 100644
--- a/llvm/test/Transforms/InstCombine/load-cmp.ll
+++ b/llvm/test/Transforms/InstCombine/load-cmp.ll
@@ -22,7 +22,7 @@
 
 define i1 @test1(i32 %X) {
 ; CHECK-LABEL: @test1(
-; CHECK-NEXT:    [[R:%.*]] = icmp eq i32 %X, 9
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i32 [[X:%.*]], 9
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
   %P = getelementptr inbounds [10 x i16], [10 x i16]* @G16, i32 0, i32 %X
@@ -33,7 +33,7 @@ define i1 @test1(i32 %X) {
 
 define i1 @test1_noinbounds(i32 %X) {
 ; CHECK-LABEL: @test1_noinbounds(
-; CHECK-NEXT:    [[R:%.*]] = icmp eq i32 %X, 9
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i32 [[X:%.*]], 9
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
   %P = getelementptr [10 x i16], [10 x i16]* @G16, i32 0, i32 %X
@@ -44,7 +44,7 @@ define i1 @test1_noinbounds(i32 %X) {
 
 define i1 @test1_noinbounds_i64(i64 %X) {
 ; CHECK-LABEL: @test1_noinbounds_i64(
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 %X to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[X:%.*]] to i32
 ; CHECK-NEXT:    [[R:%.*]] = icmp eq i32 [[TMP1]], 9
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
@@ -56,7 +56,7 @@ define i1 @test1_noinbounds_i64(i64 %X) {
 
 define i1 @test1_noinbounds_as1(i32 %x) {
 ; CHECK-LABEL: @test1_noinbounds_as1(
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 %x to i16
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[X:%.*]] to i16
 ; CHECK-NEXT:    [[R:%.*]] = icmp eq i16 [[TMP1]], 9
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
@@ -69,7 +69,7 @@ define i1 @test1_noinbounds_as1(i32 %x) {
 
 define i1 @test2(i32 %X) {
 ; CHECK-LABEL: @test2(
-; CHECK-NEXT:    [[R:%.*]] = icmp ne i32 %X, 4
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i32 [[X:%.*]], 4
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
   %P = getelementptr inbounds [10 x i16], [10 x i16]* @G16, i32 0, i32 %X
@@ -80,7 +80,7 @@ define i1 @test2(i32 %X) {
 
 define i1 @test3(i32 %X) {
 ; CHECK-LABEL: @test3(
-; CHECK-NEXT:    [[R:%.*]] = icmp eq i32 %X, 1
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i32 [[X:%.*]], 1
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
   %P = getelementptr inbounds [6 x double], [6 x double]* @GD, i32 0, i32 %X
@@ -92,7 +92,7 @@ define i1 @test3(i32 %X) {
 
 define i1 @test4(i32 %X) {
 ; CHECK-LABEL: @test4(
-; CHECK-NEXT:    [[TMP1:%.*]] = lshr i32 933, %X
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i32 933, [[X:%.*]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], 1
 ; CHECK-NEXT:    [[R:%.*]] = icmp ne i32 [[TMP2]], 0
 ; CHECK-NEXT:    ret i1 [[R]]
@@ -105,7 +105,7 @@ define i1 @test4(i32 %X) {
 
 define i1 @test4_i16(i16 %X) {
 ; CHECK-LABEL: @test4_i16(
-; CHECK-NEXT:    [[TMP1:%.*]] = sext i16 %X to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[X:%.*]] to i32
 ; CHECK-NEXT:    [[TMP2:%.*]] = lshr i32 933, [[TMP1]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[TMP2]], 1
 ; CHECK-NEXT:    [[R:%.*]] = icmp ne i32 [[TMP3]], 0
@@ -119,8 +119,8 @@ define i1 @test4_i16(i16 %X) {
 
 define i1 @test5(i32 %X) {
 ; CHECK-LABEL: @test5(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i32 %X, 2
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 %X, 7
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[X:%.*]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[X]], 7
 ; CHECK-NEXT:    [[R:%.*]] = or i1 [[TMP1]], [[TMP2]]
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
@@ -132,7 +132,7 @@ define i1 @test5(i32 %X) {
 
 define i1 @test6(i32 %X) {
 ; CHECK-LABEL: @test6(
-; CHECK-NEXT:    [[TMP1:%.*]] = add i32 %X, -1
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[X:%.*]], -1
 ; CHECK-NEXT:    [[R:%.*]] = icmp ult i32 [[TMP1]], 3
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
@@ -144,7 +144,7 @@ define i1 @test6(i32 %X) {
 
 define i1 @test7(i32 %X) {
 ; CHECK-LABEL: @test7(
-; CHECK-NEXT:    [[TMP1:%.*]] = add i32 %X, -1
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[X:%.*]], -1
 ; CHECK-NEXT:    [[R:%.*]] = icmp ugt i32 [[TMP1]], 2
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
@@ -156,7 +156,7 @@ define i1 @test7(i32 %X) {
 
 define i1 @test8(i32 %X) {
 ; CHECK-LABEL: @test8(
-; CHECK-NEXT:    [[TMP1:%.*]] = or i32 %X, 1
+; CHECK-NEXT:    [[TMP1:%.*]] = or i32 [[X:%.*]], 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], 9
 ; CHECK-NEXT:    ret i1 [[TMP2]]
 ;
@@ -176,9 +176,9 @@ define i1 @test8(i32 %X) {
 
 define i1 @test9(i32 %X) {
 ; CHECK-LABEL: @test9(
-; CHECK-NEXT:    [[X_OFF:%.*]] = add i32 %X, -1
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[X_OFF]], 2
-; CHECK-NEXT:    ret i1 [[TMP1]]
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[X:%.*]], -1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i32 [[TMP1]], 2
+; CHECK-NEXT:    ret i1 [[TMP2]]
 ;
   %P = getelementptr inbounds [4 x { i32, i32 } ], [4 x { i32, i32 } ]* @GA, i32 0, i32 %X, i32 1
   %Q = load i32, i32* %P
@@ -198,7 +198,7 @@ define i1 @test10_struct(i32 %x) {
 
 define i1 @test10_struct_noinbounds(i32 %x) {
 ; CHECK-LABEL: @test10_struct_noinbounds(
-; CHECK-NEXT:    [[P:%.*]] = getelementptr %Foo, %Foo* @GS, i32 %x, i32 0
+; CHECK-NEXT:    [[P:%.*]] = getelementptr [[FOO:%.*]], %Foo* @GS, i32 [[X:%.*]], i32 0
 ; CHECK-NEXT:    [[Q:%.*]] = load i32, i32* [[P]], align 8
 ; CHECK-NEXT:    [[R:%.*]] = icmp eq i32 [[Q]], 9
 ; CHECK-NEXT:    ret i1 [[R]]
@@ -235,8 +235,8 @@ define i1 @test10_struct_i64(i64 %x){
 
 define i1 @test10_struct_noinbounds_i16(i16 %x) {
 ; CHECK-LABEL: @test10_struct_noinbounds_i16(
-; CHECK-NEXT:    [[TMP1:%.*]] = sext i16 %x to i32
-; CHECK-NEXT:    [[P:%.*]] = getelementptr %Foo, %Foo* @GS, i32 [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = sext i16 [[X:%.*]] to i32
+; CHECK-NEXT:    [[P:%.*]] = getelementptr [[FOO:%.*]], %Foo* @GS, i32 [[TMP1]], i32 0
 ; CHECK-NEXT:    [[Q:%.*]] = load i32, i32* [[P]], align 8
 ; CHECK-NEXT:    [[R:%.*]] = icmp eq i32 [[Q]], 0
 ; CHECK-NEXT:    ret i1 [[R]]
@@ -249,7 +249,7 @@ define i1 @test10_struct_noinbounds_i16(i16 %x) {
 
 define i1 @test10_struct_arr(i32 %x) {
 ; CHECK-LABEL: @test10_struct_arr(
-; CHECK-NEXT:    [[R:%.*]] = icmp ne i32 %x, 1
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i32 [[X:%.*]], 1
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
   %p = getelementptr inbounds [4 x %Foo], [4 x %Foo]* @GStructArr, i32 0, i32 %x, i32 2
@@ -260,7 +260,7 @@ define i1 @test10_struct_arr(i32 %x) {
 
 define i1 @test10_struct_arr_noinbounds(i32 %x) {
 ; CHECK-LABEL: @test10_struct_arr_noinbounds(
-; CHECK-NEXT:    [[R:%.*]] = icmp ne i32 %x, 1
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i32 [[X:%.*]], 1
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
   %p = getelementptr [4 x %Foo], [4 x %Foo]* @GStructArr, i32 0, i32 %x, i32 2
@@ -271,7 +271,7 @@ define i1 @test10_struct_arr_noinbounds(i32 %x) {
 
 define i1 @test10_struct_arr_i16(i16 %x) {
 ; CHECK-LABEL: @test10_struct_arr_i16(
-; CHECK-NEXT:    [[R:%.*]] = icmp ne i16 %x, 1
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i16 [[X:%.*]], 1
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
   %p = getelementptr inbounds [4 x %Foo], [4 x %Foo]* @GStructArr, i16 0, i16 %x, i32 2
@@ -282,7 +282,7 @@ define i1 @test10_struct_arr_i16(i16 %x) {
 
 define i1 @test10_struct_arr_i64(i64 %x) {
 ; CHECK-LABEL: @test10_struct_arr_i64(
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 %x to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[X:%.*]] to i32
 ; CHECK-NEXT:    [[R:%.*]] = icmp ne i32 [[TMP1]], 1
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
@@ -294,7 +294,7 @@ define i1 @test10_struct_arr_i64(i64 %x) {
 
 define i1 @test10_struct_arr_noinbounds_i16(i16 %x) {
 ; CHECK-LABEL: @test10_struct_arr_noinbounds_i16(
-; CHECK-NEXT:    [[R:%.*]] = icmp ne i16 %x, 1
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i16 [[X:%.*]], 1
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
   %p = getelementptr [4 x %Foo], [4 x %Foo]* @GStructArr, i32 0, i16 %x, i32 2
@@ -305,7 +305,7 @@ define i1 @test10_struct_arr_noinbounds_i16(i16 %x) {
 
 define i1 @test10_struct_arr_noinbounds_i64(i64 %x) {
 ; CHECK-LABEL: @test10_struct_arr_noinbounds_i64(
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 %x to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[X:%.*]] to i32
 ; CHECK-NEXT:    [[R:%.*]] = icmp ne i32 [[TMP1]], 1
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/load-insert-store.ll b/llvm/test/Transforms/InstCombine/load-insert-store.ll
new file mode 100644
index 000000000..f91b4a97d
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/load-insert-store.ll
@@ -0,0 +1,98 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+define void @insert_store(<16 x i8>* %q, i8 zeroext %s) {
+; CHECK-LABEL: @insert_store(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i8>, <16 x i8>* [[Q:%.*]], align 16
+; CHECK-NEXT:    [[VECINS:%.*]] = insertelement <16 x i8> [[TMP0]], i8 [[S:%.*]], i32 3
+; CHECK-NEXT:    store <16 x i8> [[VECINS]], <16 x i8>* [[Q]], align 16
+; CHECK-NEXT:    ret void
+;
+entry:
+  %0 = load <16 x i8>, <16 x i8>* %q
+  %vecins = insertelement <16 x i8> %0, i8 %s, i32 3
+  store <16 x i8> %vecins, <16 x i8>* %q
+  ret void
+}
+
+define void @single_shuffle_store(<4 x i32>* %a, i32 %b) {
+; CHECK-LABEL: @single_shuffle_store(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i32>, <4 x i32>* [[A:%.*]], align 16
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i32> [[TMP0]], i32 [[B:%.*]], i32 1
+; CHECK-NEXT:    store <4 x i32> [[TMP1]], <4 x i32>* [[A]], align 16, !nontemporal !0
+; CHECK-NEXT:    ret void
+;
+entry:
+  %0 = load <4 x i32>, <4 x i32>* %a
+  %1 = insertelement <4 x i32> %0, i32 %b, i32 1
+  %2 = shufflevector <4 x i32> %0, <4 x i32> %1, <4 x i32> <i32 0, i32 5, i32 2, i32 3>
+  store <4 x i32> %2, <4 x i32>* %a, !nontemporal !0
+  ret void
+}
+
+define void @volatile_update(<16 x i8>* %q, <16 x i8>* %p, i8 zeroext %s) {
+; CHECK-LABEL: @volatile_update(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i8>, <16 x i8>* [[Q:%.*]], align 16
+; CHECK-NEXT:    [[VECINS0:%.*]] = insertelement <16 x i8> [[TMP0]], i8 [[S:%.*]], i32 3
+; CHECK-NEXT:    store volatile <16 x i8> [[VECINS0]], <16 x i8>* [[Q]], align 16
+; CHECK-NEXT:    [[TMP1:%.*]] = load volatile <16 x i8>, <16 x i8>* [[P:%.*]], align 16
+; CHECK-NEXT:    [[VECINS1:%.*]] = insertelement <16 x i8> [[TMP1]], i8 [[S]], i32 1
+; CHECK-NEXT:    store <16 x i8> [[VECINS1]], <16 x i8>* [[P]], align 16
+; CHECK-NEXT:    ret void
+;
+entry:
+  %0 = load <16 x i8>, <16 x i8>* %q
+  %vecins0 = insertelement <16 x i8> %0, i8 %s, i32 3
+  store volatile <16 x i8> %vecins0, <16 x i8>* %q
+
+  %1 = load volatile <16 x i8>, <16 x i8>* %p
+  %vecins1 = insertelement <16 x i8> %1, i8 %s, i32 1
+  store <16 x i8> %vecins1, <16 x i8>* %p
+  ret void
+}
+
+define void @insert_store_addr_differ(<16 x i8>* %p, <16 x i8>* %q, i8 %s) {
+; CHECK-LABEL: @insert_store_addr_differ(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LD:%.*]] = load <16 x i8>, <16 x i8>* [[P:%.*]], align 16
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <16 x i8> [[LD]], i8 [[S:%.*]], i32 3
+; CHECK-NEXT:    store <16 x i8> [[INS]], <16 x i8>* [[Q:%.*]], align 16
+; CHECK-NEXT:    ret void
+;
+entry:
+  %ld = load <16 x i8>, <16 x i8>* %p
+  %ins = insertelement <16 x i8> %ld, i8 %s, i32 3
+  store <16 x i8> %ins, <16 x i8>* %q
+  ret void
+}
+
+define void @insert_store_mem_modify(<16 x i8>* %p, <16 x i8>* %q, <16 x i8>* noalias %r, i8 %s) {
+; CHECK-LABEL: @insert_store_mem_modify(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LD:%.*]] = load <16 x i8>, <16 x i8>* [[P:%.*]], align 16
+; CHECK-NEXT:    store <16 x i8> zeroinitializer, <16 x i8>* [[Q:%.*]], align 16
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <16 x i8> [[LD]], i8 [[S:%.*]], i32 3
+; CHECK-NEXT:    store <16 x i8> [[INS]], <16 x i8>* [[P]], align 16
+; CHECK-NEXT:    [[LD2:%.*]] = load <16 x i8>, <16 x i8>* [[Q]], align 16
+; CHECK-NEXT:    store <16 x i8> zeroinitializer, <16 x i8>* [[R:%.*]], align 16
+; CHECK-NEXT:    [[INS2:%.*]] = insertelement <16 x i8> [[LD2]], i8 [[S]], i32 7
+; CHECK-NEXT:    store <16 x i8> [[INS2]], <16 x i8>* [[Q]], align 16
+; CHECK-NEXT:    ret void
+;
+entry:
+  %ld = load <16 x i8>, <16 x i8>* %p
+  store <16 x i8> zeroinitializer, <16 x i8>* %q
+  %ins = insertelement <16 x i8> %ld, i8 %s, i32 3
+  store <16 x i8> %ins, <16 x i8>* %p
+
+  %ld2 = load <16 x i8>, <16 x i8>* %q
+  store <16 x i8> zeroinitializer, <16 x i8>* %r
+  %ins2 = insertelement <16 x i8> %ld2, i8 %s, i32 7
+  store <16 x i8> %ins2, <16 x i8>* %q
+  ret void
+}
+
+!0 = !{}
diff --git a/llvm/test/Transforms/InstCombine/load.ll b/llvm/test/Transforms/InstCombine/load.ll
index 5129349b3..f2de4ae79 100644
--- a/llvm/test/Transforms/InstCombine/load.ll
+++ b/llvm/test/Transforms/InstCombine/load.ll
@@ -237,6 +237,42 @@ entry:
   ret void
 }
 
+define void @test16-vect(i8* %x, i8* %a, i8* %b, i8* %c) {
+; CHECK-LABEL: @test16-vect(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[C_CAST:%.*]] = bitcast i8* [[C:%.*]] to i32*
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i8* [[X:%.*]] to i32*
+; CHECK-NEXT:    [[X11:%.*]] = load i32, i32* [[TMP0]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8* [[A:%.*]] to i32*
+; CHECK-NEXT:    store i32 [[X11]], i32* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8* [[B:%.*]] to i32*
+; CHECK-NEXT:    store i32 [[X11]], i32* [[TMP2]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8* [[X]] to i32*
+; CHECK-NEXT:    [[X22:%.*]] = load i32, i32* [[TMP3]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8* [[B]] to i32*
+; CHECK-NEXT:    store i32 [[X22]], i32* [[TMP4]], align 4
+; CHECK-NEXT:    store i32 [[X22]], i32* [[C_CAST]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %x.cast = bitcast i8* %x to <4 x i8>*
+  %a.cast = bitcast i8* %a to <4 x i8>*
+  %b.cast = bitcast i8* %b to <4 x i8>*
+  %c.cast = bitcast i8* %c to i32*
+
+  %x1 = load <4 x i8>, <4 x i8>* %x.cast
+  store <4 x i8> %x1, <4 x i8>* %a.cast
+  store <4 x i8> %x1, <4 x i8>* %b.cast
+
+  %x2 = load <4 x i8>, <4 x i8>* %x.cast
+  store <4 x i8> %x2, <4 x i8>* %b.cast
+  %x2.cast = bitcast <4 x i8> %x2 to i32
+  store i32 %x2.cast, i32* %c.cast
+
+  ret void
+}
+
+
 ; Check that in cases similar to @test16 we don't try to rewrite a load when
 ; its only use is a store but it is used as the pointer to that store rather
 ; than the value.
@@ -300,3 +336,15 @@ entry:
   store %swift.error* %err.res, %swift.error** %err, align 8
   ret void
 }
+
+; Make sure we don't canonicalize accesses to scalable vectors.
+define void @test20(<vscale x 4 x i8>* %x, <vscale x 4 x i8>* %y) {
+; CHECK-LABEL: @test20(
+; CHECK-NEXT:    [[X_LOAD:%.*]] = load <vscale x 4 x i8>, <vscale x 4 x i8>* [[X:%.*]], align 1
+; CHECK-NEXT:    store <vscale x 4 x i8> [[X_LOAD]], <vscale x 4 x i8>* [[Y:%.*]], align 1
+; CHECK-NEXT:    ret void
+;
+  %x.load = load <vscale x 4 x i8>, <vscale x 4 x i8>* %x, align 1
+  store <vscale x 4 x i8> %x.load, <vscale x 4 x i8>* %y, align 1
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/log-pow.ll b/llvm/test/Transforms/InstCombine/log-pow.ll
index 4e4a2b261..227fcf0c3 100644
--- a/llvm/test/Transforms/InstCombine/log-pow.ll
+++ b/llvm/test/Transforms/InstCombine/log-pow.ll
@@ -1,62 +1,125 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -instcombine -S | FileCheck %s
 
 define double @log_pow(double %x, double %y) {
-  %pow = call fast double @llvm.pow.f64(double %x, double %y)
-  %call = call fast double @log(double %pow)
-  ret double %call
+; CHECK-LABEL: @log_pow(
+; CHECK-NEXT:    [[LOG1:%.*]] = call fast double @llvm.log.f64(double [[X:%.*]])
+; CHECK-NEXT:    [[MUL:%.*]] = fmul fast double [[LOG1]], [[Y:%.*]]
+; CHECK-NEXT:    ret double [[MUL]]
+;
+  %pow = call fast double @pow(double %x, double %y)
+  %log = call fast double @log(double %pow)
+  ret double %log
 }
 
-; CHECK-LABEL: define double @log_pow(
-; CHECK-NEXT:  %log = call fast double @log(double %x)
-; CHECK-NEXT:  %mul = fmul fast double %log, %y
-; CHECK-NEXT:  ret double %mul
+define float @log10f_powf(float %x, float %y) {
+; CHECK-LABEL: @log10f_powf(
+; CHECK-NEXT:    [[LOG1:%.*]] = call fast float @llvm.log10.f32(float [[X:%.*]])
+; CHECK-NEXT:    [[MUL:%.*]] = fmul fast float [[LOG1]], [[Y:%.*]]
+; CHECK-NEXT:    ret float [[MUL]]
+;
+  %pow = call fast float @powf(float %x, float %y)
+  %log = call fast float @llvm.log10.f32(float %pow)
+  ret float %log
+}
 
-define double @log_pow_not_fast(double %x, double %y) {
-  %pow = call double @llvm.pow.f64(double %x, double %y)
-  %call = call fast double @log(double %pow)
-  ret double %call
+define <2 x double> @log2v_powv(<2 x double> %x, <2 x double> %y) {
+; CHECK-LABEL: @log2v_powv(
+; CHECK-NEXT:    [[LOG1:%.*]] = call fast <2 x double> @llvm.log2.v2f64(<2 x double> [[X:%.*]])
+; CHECK-NEXT:    [[MUL:%.*]] = fmul fast <2 x double> [[LOG1]], [[Y:%.*]]
+; CHECK-NEXT:    ret <2 x double> [[MUL]]
+;
+  %pow = call fast <2 x double> @llvm.pow.v2f64(<2 x double> %x, <2 x double> %y)
+  %log = call fast <2 x double> @llvm.log2.v2f64(<2 x double> %pow)
+  ret <2 x double> %log
 }
 
-; CHECK-LABEL: define double @log_pow_not_fast(
-; CHECK-NEXT:  %pow = call double @llvm.pow.f64(double %x, double %y)
-; CHECK-NEXT:  %call = call fast double @log(double %pow)
-; CHECK-NEXT:  ret double %call
+define double @log_pow_not_fast(double %x, double %y) {
+; CHECK-LABEL: @log_pow_not_fast(
+; CHECK-NEXT:    [[POW:%.*]] = call double @pow(double [[X:%.*]], double [[Y:%.*]])
+; CHECK-NEXT:    [[LOG:%.*]] = call fast double @log(double [[POW]])
+; CHECK-NEXT:    ret double [[LOG]]
+;
+  %pow = call double @pow(double %x, double %y)
+  %log = call fast double @log(double %pow)
+  ret double %log
+}
 
-define double @function_pointer(double ()* %fptr, double %p1) {
-  %call1 = call double %fptr()
-  %pow = call double @log(double %call1)
-  ret double %pow
+define float @function_pointer(float ()* %fptr, float %p1) {
+; CHECK-LABEL: @function_pointer(
+; CHECK-NEXT:    [[PTR:%.*]] = call float [[FPTR:%.*]]()
+; CHECK-NEXT:    [[LOG:%.*]] = call float @logf(float [[PTR]])
+; CHECK-NEXT:    ret float [[LOG]]
+;
+  %ptr = call float %fptr()
+  %log = call float @logf(float %ptr)
+  ret float %log
 }
 
-; CHECK-LABEL: @function_pointer
-; CHECK-NEXT:  %call1 = call double %fptr()
-; CHECK-NEXT:  %pow = call double @log(double %call1)
-; CHECK-NEXT:  ret double %pow
+define double @log10_exp(double %x) {
+; CHECK-LABEL: @log10_exp(
+; CHECK-NEXT:    [[MUL:%.*]] = fmul fast double [[X:%.*]], 0x3FDBCB7B1526E50E
+; CHECK-NEXT:    ret double [[MUL]]
+;
+  %exp = call fast double @exp(double %x)
+  %log = call fast double @log10(double %exp)
+  ret double %log
+}
 
-define double @log_exp2(double %x) {
-  %call2 = call fast double @exp2(double %x)
-  %call3 = call fast double @log(double %call2)
-  ret double %call3
+define <2 x float> @logv_exp2v(<2 x float> %x) {
+; CHECK-LABEL: @logv_exp2v(
+; CHECK-NEXT:    [[MUL:%.*]] = fmul fast <2 x float> [[X:%.*]], <float 0x3FE62E4300000000, float 0x3FE62E4300000000>
+; CHECK-NEXT:    ret <2 x float> [[MUL]]
+;
+  %exp = call fast <2 x float> @llvm.exp2.v2f32(<2 x float> %x)
+  %log = call fast <2 x float> @llvm.log.v2f32(<2 x float> %exp)
+  ret <2 x float> %log
 }
 
-; CHECK-LABEL: @log_exp2
-; CHECK-NEXT:  %call2 = call fast double @exp2(double %x)
-; CHECK-NEXT:  %logmul = fmul fast double %x, 0x3FE62E42FEFA39EF
-; CHECK-NEXT:  ret double %logmul
+define float @log2f_exp10f(float %x) {
+; CHECK-LABEL: @log2f_exp10f(
+; CHECK-NEXT:    [[MUL:%.*]] = fmul fast float [[X:%.*]], 0x400A934F00000000
+; CHECK-NEXT:    ret float [[MUL]]
+;
+  %exp = call fast float @exp10f(float %x)
+  %log = call fast float @log2f(float %exp)
+  ret float %log
+}
 
 define double @log_exp2_not_fast(double %x) {
-  %call2 = call double @exp2(double %x)
-  %call3 = call fast double @log(double %call2)
-  ret double %call3
+; CHECK-LABEL: @log_exp2_not_fast(
+; CHECK-NEXT:    [[EXP:%.*]] = call double @exp2(double [[X:%.*]])
+; CHECK-NEXT:    [[LOG:%.*]] = call fast double @log(double [[EXP]])
+; CHECK-NEXT:    ret double [[LOG]]
+;
+  %exp = call double @exp2(double %x)
+  %log = call fast double @log(double %exp)
+  ret double %log
 }
 
-; CHECK-LABEL: @log_exp2_not_fast
-; CHECK-NEXT:  %call2 = call double @exp2(double %x)
-; CHECK-NEXT:  %call3 = call fast double @log(double %call2)
-; CHECK-NEXT:  ret double %call3
+define double @pr43617(double %d, i32 %i, double (i32)* %f) {
+entry:
+  %sub = fsub double -0.000000e+00, %d
+  %icall = tail call fast double %f(i32 %i)
+  %log = tail call fast double @llvm.log.f64(double %icall)
+  %mul = fmul double %log, %sub
+  ret double %mul
+}
 
 declare double @log(double) #0
-declare double @exp2(double)
-declare double @llvm.pow.f64(double, double)
+declare float @logf(float) #0
+declare double @llvm.log.f64(double) #0
+declare <2 x float> @llvm.log.v2f32(<2 x float>)
+declare float @log2f(float) #0
+declare <2 x double> @llvm.log2.v2f64(<2 x double>)
+declare double @log10(double) #0
+declare float @llvm.log10.f32(float)
+declare double @exp(double %x) #0
+declare double @exp2(double) #0
+declare float @exp10f(float) #0
+declare <2 x float> @llvm.exp2.v2f32(<2 x float>)
+declare double @pow(double, double) #0
+declare float @powf(float, float) #0
+declare <2 x double> @llvm.pow.v2f64(<2 x double>, <2 x double>)
 
 attributes #0 = { nounwind readnone }
diff --git a/llvm/test/Transforms/InstCombine/logical-select.ll b/llvm/test/Transforms/InstCombine/logical-select.ll
index 3f02554e7..82f72a8c6 100644
--- a/llvm/test/Transforms/InstCombine/logical-select.ll
+++ b/llvm/test/Transforms/InstCombine/logical-select.ll
@@ -515,10 +515,10 @@ define <4 x i32> @vec_sel_xor(<4 x i32> %a, <4 x i32> %b, <4 x i1> %c) {
 define <4 x i32> @vec_sel_xor_multi_use(<4 x i32> %a, <4 x i32> %b, <4 x i1> %c) {
 ; CHECK-LABEL: @vec_sel_xor_multi_use(
 ; CHECK-NEXT:    [[TMP1:%.*]] = xor <4 x i1> [[C:%.*]], <i1 true, i1 false, i1 false, i1 false>
+; CHECK-NEXT:    [[MASK_FLIP1:%.*]] = sext <4 x i1> [[TMP1]] to <4 x i32>
 ; CHECK-NEXT:    [[TMP2:%.*]] = xor <4 x i1> [[C]], <i1 false, i1 true, i1 true, i1 true>
 ; CHECK-NEXT:    [[TMP3:%.*]] = select <4 x i1> [[TMP2]], <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]]
-; CHECK-NEXT:    [[TMP4:%.*]] = zext <4 x i1> [[TMP1]] to <4 x i32>
-; CHECK-NEXT:    [[ADD:%.*]] = sub <4 x i32> [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[ADD:%.*]] = add <4 x i32> [[TMP3]], [[MASK_FLIP1]]
 ; CHECK-NEXT:    ret <4 x i32> [[ADD]]
 ;
   %mask = sext <4 x i1> %c to <4 x i32>
diff --git a/llvm/test/Transforms/InstCombine/lower-dbg-declare.ll b/llvm/test/Transforms/InstCombine/lower-dbg-declare.ll
index 44de3fc75..027708665 100644
--- a/llvm/test/Transforms/InstCombine/lower-dbg-declare.ll
+++ b/llvm/test/Transforms/InstCombine/lower-dbg-declare.ll
@@ -3,181 +3,112 @@
 ; This tests dbg.declare lowering for CallInst users of an alloca. The
 ; resulting dbg.value expressions should add a deref to the declare's expression.
 
-; Hand-reduced from this example (-g -Og -fsanitize=address):
-
-;   static volatile int sink;
-;   struct OneElementVector {
-;     int Element;
-;     OneElementVector(int Element) : Element(Element) { sink = Element; }
-;     bool empty() const { return false; }
-;   };
-;   using container = OneElementVector;
-;   static void escape(container &c) { sink = c.Element; }
-;   int main() {
-;     container d1 = {42};
-;     while (!d1.empty())
-;       escape(d1);
-;     return 0;
-;   }
-
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
-target triple = "x86_64-apple-macosx10.13.0"
-
-%struct.OneElementVector = type { i32 }
-
-define i1 @escape(%struct.OneElementVector* %d1) {
-  ret i1 false
-}
+; Hand-reduced from this example (-g -Og -mllvm -disable-llvm-optzns -gno-column-info):
+
+; static volatile int sink;
+; static void escape(int &c) { sink = c; }
+; static bool empty ( int p1 ) { return p1 == 0; }
+; int main() {
+;   int d1 = 42;
+;   while (!empty(d1))
+;     escape(d1);
+;   return 0;
+; }
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@_ZL4sink = internal global i32 0, align 4, !dbg !0
 
 ; CHECK-LABEL: @main
-define i32 @main() !dbg !15 {
+define dso_local i32 @main() !dbg !12 {
 entry:
-  %d1 = alloca %struct.OneElementVector, align 4
-  %0 = bitcast %struct.OneElementVector* %d1 to i8*, !dbg !34
-
-; CHECK: dbg.value(metadata %struct.OneElementVector* [[var:%.*]], metadata !DIExpression(DW_OP_deref))
-; CHECK-NEXT: call i1 @escape
-  call void @llvm.dbg.declare(metadata %struct.OneElementVector* %d1, metadata !19, metadata !DIExpression()), !dbg !35
-  call i1 @escape(%struct.OneElementVector* %d1)
-  br label %while.cond, !dbg !37
+  %retval = alloca i32, align 4
+  %d1 = alloca i32, align 4
+  store i32 0, i32* %retval, align 4
+  %0 = bitcast i32* %d1 to i8*, !dbg !17
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* %0) #4, !dbg !17
+; CHECK: dbg.value(metadata i32 42, metadata [[METADATA_IDX1:![0-9]+]], metadata !DIExpression())
+; CHECK-NEXT: store
+  call void @llvm.dbg.declare(metadata i32* %d1, metadata !16, metadata !DIExpression()), !dbg !17
+  store i32 42, i32* %d1, align 4, !dbg !17
+  br label %while.cond, !dbg !22
 
 while.cond:                                       ; preds = %while.body, %entry
-; CHECK: dbg.value(metadata %struct.OneElementVector* [[var]], metadata !DIExpression(DW_OP_deref))
-; CHECK-NEXT: call i1 @escape
-  %call = call i1 @escape(%struct.OneElementVector* %d1), !dbg !38
-  %lnot = xor i1 %call, true, !dbg !39
-  br i1 %lnot, label %while.body, label %while.end, !dbg !37
+; CHECK: dbg.value(metadata i32 %1, metadata [[METADATA_IDX1]], metadata !DIExpression())
+; CHECK-NEXT: call zeroext i1 @_ZL5emptyi
+  %1 = load i32, i32* %d1, align 4, !dbg !22
+  %call = call zeroext i1 @_ZL5emptyi(i32 %1), !dbg !22
+  %lnot = xor i1 %call, true, !dbg !22
+  br i1 %lnot, label %while.body, label %while.end, !dbg !22
 
 while.body:                                       ; preds = %while.cond
-; CHECK: dbg.value(metadata %struct.OneElementVector* [[var]], metadata !DIExpression(DW_OP_deref))
-; CHECK-NEXT: call i1 @escape
-  call i1 @escape(%struct.OneElementVector* %d1)
-  br label %while.cond, !dbg !37, !llvm.loop !42
+; CHECK: dbg.value(metadata i32* %d1, metadata [[METADATA_IDX1]], metadata !DIExpression(DW_OP_deref))
+; CHECK-NEXT: call void @_ZL6escapeRi
+  call void @_ZL6escapeRi(i32* dereferenceable(4) %d1), !dbg !23
+  br label %while.cond, !dbg !22, !llvm.loop !24
 
 while.end:                                        ; preds = %while.cond
-  ret i32 0, !dbg !45
+  %2 = bitcast i32* %d1 to i8*, !dbg !25
+  call void @llvm.lifetime.end.p0i8(i64 4, i8* %2) #4, !dbg !25
+  ret i32 0, !dbg !26
 }
 
-; CHECK-LABEL: @main2
-define i32 @main2() {
-entry:
-  %d1 = alloca %struct.OneElementVector, align 4
-  %0 = bitcast %struct.OneElementVector* %d1 to i8*, !dbg !34
-
-; CHECK: dbg.value(metadata %struct.OneElementVector* [[var:%.*]], metadata !DIExpression(DW_OP_lit0, DW_OP_mul, DW_OP_deref))
-; CHECK-NEXT: call i1 @escape
-  call void @llvm.dbg.declare(metadata %struct.OneElementVector* %d1, metadata !19, metadata !DIExpression(DW_OP_lit0, DW_OP_mul)), !dbg !35
-  call i1 @escape(%struct.OneElementVector* %d1)
-  br label %while.cond, !dbg !37
+declare void @llvm.lifetime.start.p0i8(i64 immarg, i8* nocapture)
 
-while.cond:                                       ; preds = %while.body, %entry
-; CHECK: dbg.value(metadata %struct.OneElementVector* [[var]], metadata !DIExpression(DW_OP_lit0, DW_OP_mul, DW_OP_deref))
-; CHECK-NEXT: call i1 @escape
-  %call = call i1 @escape(%struct.OneElementVector* %d1), !dbg !38
-  %lnot = xor i1 %call, true, !dbg !39
-  br i1 %lnot, label %while.body, label %while.end, !dbg !37
+declare void @llvm.dbg.declare(metadata, metadata, metadata)
 
-while.body:                                       ; preds = %while.cond
-; CHECK: dbg.value(metadata %struct.OneElementVector* [[var]], metadata !DIExpression(DW_OP_lit0, DW_OP_mul, DW_OP_deref))
-; CHECK-NEXT: call i1 @escape
-  call i1 @escape(%struct.OneElementVector* %d1)
-  br label %while.cond, !dbg !37, !llvm.loop !42
+define internal zeroext i1 @_ZL5emptyi(i32 %p1) !dbg !27 {
+  ret i1 false
+}
 
-while.end:                                        ; preds = %while.cond
-  ret i32 0, !dbg !45
+define internal void @_ZL6escapeRi(i32* dereferenceable(4) %c) #3 !dbg !34 {
+  ret void
 }
 
-declare void @llvm.dbg.declare(metadata, metadata, metadata)
+declare void @llvm.lifetime.end.p0i8(i64 immarg, i8* nocapture) #1
 
 !llvm.dbg.cu = !{!2}
-!llvm.asan.globals = !{!8}
-!llvm.module.flags = !{!10, !11, !12, !13}
-!llvm.ident = !{!14}
+!llvm.module.flags = !{!8, !9, !10}
+!llvm.ident = !{!11}
+
+; CHECK: DIGlobalVariableExpression
+; CHECK: [[METADATA_IDX1]] = !DILocalVariable(name: "d1"
 
 !0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
-!1 = distinct !DIGlobalVariable(name: "sink", linkageName: "_ZL4sink", scope: !2, file: !3, line: 1, type: !6, isLocal: true, isDefinition: true)
-!2 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !3, producer: "clang version 7.0.0 (trunk 337207) (llvm/trunk 337204)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5)
-!3 = !DIFile(filename: "test.cc", directory: "/Users/vsk/src/builds/llvm.org-master-RA")
+!1 = distinct !DIGlobalVariable(name: "sink", linkageName: "_ZL4sink", scope: !2, file: !3, line: 2, type: !6, isLocal: true, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !3, producer: "clang", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5, nameTableKind: None)
+!3 = !DIFile(filename: "lower-dbg-declare.cpp", directory: "")
 !4 = !{}
 !5 = !{!0}
 !6 = !DIDerivedType(tag: DW_TAG_volatile_type, baseType: !7)
 !7 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
-!8 = !{}
-!9 = !{!"test.cc", i32 1, i32 21}
-!10 = !{i32 2, !"Dwarf Version", i32 4}
-!11 = !{i32 2, !"Debug Info Version", i32 3}
-!12 = !{i32 1, !"wchar_size", i32 4}
-!13 = !{i32 7, !"PIC Level", i32 2}
-!14 = !{!"clang version 7.0.0 (trunk 337207) (llvm/trunk 337204)"}
-!15 = distinct !DISubprogram(name: "main", scope: !3, file: !3, line: 18, type: !16, isLocal: false, isDefinition: true, scopeLine: 18, flags: DIFlagPrototyped, isOptimized: true, unit: !2, retainedNodes: !18)
-!16 = !DISubroutineType(types: !17)
-!17 = !{!7}
-!18 = !{!19}
-!19 = !DILocalVariable(name: "d1", scope: !15, file: !3, line: 21, type: !20)
-!20 = !DIDerivedType(tag: DW_TAG_typedef, name: "container", file: !3, line: 12, baseType: !21)
-!21 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "OneElementVector", file: !3, line: 3, size: 32, flags: DIFlagTypePassByValue, elements: !22, identifier: "_ZTS16OneElementVector")
-!22 = !{!23, !24, !28}
-!23 = !DIDerivedType(tag: DW_TAG_member, name: "Element", scope: !21, file: !3, line: 4, baseType: !7, size: 32)
-!24 = !DISubprogram(name: "OneElementVector", scope: !21, file: !3, line: 6, type: !25, isLocal: false, isDefinition: false, scopeLine: 6, flags: DIFlagPrototyped, isOptimized: true)
-!25 = !DISubroutineType(types: !26)
-!26 = !{null, !27, !7}
-!27 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !21, size: 64, flags: DIFlagArtificial | DIFlagObjectPointer)
-!28 = !DISubprogram(name: "empty", linkageName: "_ZNK16OneElementVector5emptyEv", scope: !21, file: !3, line: 8, type: !29, isLocal: false, isDefinition: false, scopeLine: 8, flags: DIFlagPrototyped, isOptimized: true)
-!29 = !DISubroutineType(types: !30)
-!30 = !{!31, !32}
-!31 = !DIBasicType(name: "bool", size: 8, encoding: DW_ATE_boolean)
-!32 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !33, size: 64, flags: DIFlagArtificial | DIFlagObjectPointer)
-!33 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !21)
-!34 = !DILocation(line: 21, column: 3, scope: !15)
-!35 = !DILocation(line: 21, column: 13, scope: !15)
-!36 = !DILocation(line: 21, column: 18, scope: !15)
-!37 = !DILocation(line: 22, column: 3, scope: !15)
-!38 = !DILocation(line: 22, column: 14, scope: !15)
-!39 = !DILocation(line: 22, column: 10, scope: !15)
-!40 = !DILocation(line: 23, column: 5, scope: !41)
-!41 = distinct !DILexicalBlock(scope: !15, file: !3, line: 22, column: 23)
-!42 = distinct !{!42, !37, !43}
-!43 = !DILocation(line: 24, column: 3, scope: !15)
-!44 = !DILocation(line: 26, column: 1, scope: !15)
-!45 = !DILocation(line: 25, column: 3, scope: !15)
-!46 = distinct !DISubprogram(name: "OneElementVector", linkageName: "_ZN16OneElementVectorC1Ei", scope: !21, file: !3, line: 6, type: !25, isLocal: false, isDefinition: true, scopeLine: 6, flags: DIFlagPrototyped, isOptimized: true, unit: !2, declaration: !24, retainedNodes: !47)
-!47 = !{!48, !50}
-!48 = !DILocalVariable(name: "this", arg: 1, scope: !46, type: !49, flags: DIFlagArtificial | DIFlagObjectPointer)
-!49 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !21, size: 64)
-!50 = !DILocalVariable(name: "Element", arg: 2, scope: !46, file: !3, line: 6, type: !7)
-!51 = !DILocation(line: 0, scope: !46)
-!52 = !DILocation(line: 6, column: 24, scope: !46)
-!53 = !DILocation(line: 6, column: 52, scope: !46)
-!54 = !DILocation(line: 6, column: 70, scope: !46)
-!55 = distinct !DISubprogram(name: "empty", linkageName: "_ZNK16OneElementVector5emptyEv", scope: !21, file: !3, line: 8, type: !29, isLocal: false, isDefinition: true, scopeLine: 8, flags: DIFlagPrototyped, isOptimized: true, unit: !2, declaration: !28, retainedNodes: !56)
-!56 = !{!57}
-!57 = !DILocalVariable(name: "this", arg: 1, scope: !55, type: !58, flags: DIFlagArtificial | DIFlagObjectPointer)
-!58 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !33, size: 64)
-!59 = !DILocation(line: 0, scope: !55)
-!60 = !DILocation(line: 8, column: 24, scope: !55)
-!61 = distinct !DISubprogram(name: "escape", linkageName: "_ZL6escapeR16OneElementVector", scope: !3, file: !3, line: 14, type: !62, isLocal: true, isDefinition: true, scopeLine: 14, flags: DIFlagPrototyped, isOptimized: true, unit: !2, retainedNodes: !65)
-!62 = !DISubroutineType(types: !63)
-!63 = !{null, !64}
-!64 = !DIDerivedType(tag: DW_TAG_reference_type, baseType: !20, size: 64)
-!65 = !{!66}
-!66 = !DILocalVariable(name: "c", arg: 1, scope: !61, file: !3, line: 14, type: !64)
-!67 = !DILocation(line: 14, column: 31, scope: !61)
-!68 = !DILocation(line: 15, column: 12, scope: !61)
-!69 = !{!70, !71, i64 0}
-!70 = !{!"_ZTS16OneElementVector", !71, i64 0}
-!71 = !{!"int", !72, i64 0}
-!72 = !{!"omnipotent char", !73, i64 0}
-!73 = !{!"Simple C++ TBAA"}
-!74 = !DILocation(line: 15, column: 8, scope: !61)
-!75 = !{!71, !71, i64 0}
-!76 = !DILocation(line: 16, column: 1, scope: !61)
-!77 = distinct !DISubprogram(name: "OneElementVector", linkageName: "_ZN16OneElementVectorC2Ei", scope: !21, file: !3, line: 6, type: !25, isLocal: false, isDefinition: true, scopeLine: 6, flags: DIFlagPrototyped, isOptimized: true, unit: !2, declaration: !24, retainedNodes: !78)
-!78 = !{!79, !80}
-!79 = !DILocalVariable(name: "this", arg: 1, scope: !77, type: !49, flags: DIFlagArtificial | DIFlagObjectPointer)
-!80 = !DILocalVariable(name: "Element", arg: 2, scope: !77, file: !3, line: 6, type: !7)
-!81 = !DILocation(line: 0, scope: !77)
-!82 = !DILocation(line: 6, column: 24, scope: !77)
-!83 = !DILocation(line: 6, column: 35, scope: !77)
-!84 = !DILocation(line: 6, column: 59, scope: !85)
-!85 = distinct !DILexicalBlock(scope: !77, file: !3, line: 6, column: 52)
-!86 = !DILocation(line: 6, column: 70, scope: !77)
+!8 = !{i32 2, !"Dwarf Version", i32 4}
+!9 = !{i32 2, !"Debug Info Version", i32 3}
+!10 = !{i32 1, !"wchar_size", i32 4}
+!11 = !{!"clang"}
+!12 = distinct !DISubprogram(name: "main", scope: !3, file: !3, line: 5, type: !13, scopeLine: 5, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !2, retainedNodes: !15)
+!13 = !DISubroutineType(types: !14)
+!14 = !{!7}
+!15 = !{!16}
+!16 = !DILocalVariable(name: "d1", scope: !12, file: !3, line: 6, type: !7)
+!17 = !DILocation(line: 6, scope: !12)
+!22 = !DILocation(line: 7, scope: !12)
+!23 = !DILocation(line: 8, scope: !12)
+!24 = distinct !{!24, !22, !23}
+!25 = !DILocation(line: 10, scope: !12)
+!26 = !DILocation(line: 9, scope: !12)
+!27 = distinct !DISubprogram(name: "empty", linkageName: "_ZL5emptyi", scope: !3, file: !3, line: 4, type: !28, scopeLine: 4, flags: DIFlagPrototyped, spFlags: DISPFlagLocalToUnit | DISPFlagDefinition | DISPFlagOptimized, unit: !2, retainedNodes: !31)
+!28 = !DISubroutineType(types: !29)
+!29 = !{!30, !7}
+!30 = !DIBasicType(name: "bool", size: 8, encoding: DW_ATE_boolean)
+!31 = !{!32}
+!32 = !DILocalVariable(name: "p1", arg: 1, scope: !27, file: !3, line: 4, type: !7)
+!33 = !DILocation(line: 4, scope: !27)
+!34 = distinct !DISubprogram(name: "escape", linkageName: "_ZL6escapeRi", scope: !3, file: !3, line: 3, type: !35, scopeLine: 3, flags: DIFlagPrototyped, spFlags: DISPFlagLocalToUnit | DISPFlagDefinition | DISPFlagOptimized, unit: !2, retainedNodes: !38)
+!35 = !DISubroutineType(types: !36)
+!36 = !{null, !37}
+!37 = !DIDerivedType(tag: DW_TAG_reference_type, baseType: !7, size: 64)
+!38 = !{!39}
+!39 = !DILocalVariable(name: "c", arg: 1, scope: !34, file: !3, line: 3, type: !37)
+!42 = !DILocation(line: 3, scope: !34)
diff --git a/llvm/test/Transforms/InstCombine/malloc-free-delete.ll b/llvm/test/Transforms/InstCombine/malloc-free-delete.ll
index 7e7b6d9ae..a0828e1b6 100644
--- a/llvm/test/Transforms/InstCombine/malloc-free-delete.ll
+++ b/llvm/test/Transforms/InstCombine/malloc-free-delete.ll
@@ -1,14 +1,16 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -instcombine -S | FileCheck %s
 ; PR1201
 define i32 @main(i32 %argc, i8** %argv) {
 ; CHECK-LABEL: @main(
-    %c_19 = alloca i8*
-    %malloc_206 = tail call i8* @malloc(i32 mul (i32 ptrtoint (i8* getelementptr (i8, i8* null, i32 1) to i32), i32 10))
-    store i8* %malloc_206, i8** %c_19
-    %tmp_207 = load i8*, i8** %c_19
-    tail call void @free(i8* %tmp_207)
-    ret i32 0
-; CHECK-NEXT: ret i32 0
+; CHECK-NEXT:    ret i32 0
+;
+  %c_19 = alloca i8*
+  %malloc_206 = tail call i8* @malloc(i32 mul (i32 ptrtoint (i8* getelementptr (i8, i8* null, i32 1) to i32), i32 10))
+  store i8* %malloc_206, i8** %c_19
+  %tmp_207 = load i8*, i8** %c_19
+  tail call void @free(i8* %tmp_207)
+  ret i32 0
 }
 
 declare noalias i8* @calloc(i32, i32) nounwind
@@ -17,7 +19,8 @@ declare void @free(i8*)
 
 define i1 @foo() {
 ; CHECK-LABEL: @foo(
-; CHECK-NEXT: ret i1 false
+; CHECK-NEXT:    ret i1 false
+;
   %m = call i8* @malloc(i32 1)
   %z = icmp eq i8* %m, null
   call void @free(i8* %m)
@@ -33,7 +36,8 @@ declare void @llvm.memset.p0i8.i32(i8*, i8, i32, i1) nounwind
 
 define void @test3(i8* %src) {
 ; CHECK-LABEL: @test3(
-; CHECK-NEXT: ret void
+; CHECK-NEXT:    ret void
+;
   %a = call noalias i8* @malloc(i32 10)
   call void @llvm.lifetime.start.p0i8(i64 10, i8* %a)
   call void @llvm.lifetime.end.p0i8(i64 10, i8* %a)
@@ -50,7 +54,8 @@ define void @test3(i8* %src) {
 ;; This used to crash.
 define void @test4() {
 ; CHECK-LABEL: @test4(
-; CHECK-NEXT: ret void
+; CHECK-NEXT:    ret void
+;
   %A = call i8* @malloc(i32 16000)
   %B = bitcast i8* %A to double*
   %C = bitcast double* %B to i8*
@@ -58,23 +63,24 @@ define void @test4() {
   ret void
 }
 
-; CHECK-LABEL: @test5(
 define void @test5(i8* %ptr, i8** %esc) {
-; CHECK-NEXT: call i8* @malloc
-; CHECK-NEXT: call i8* @malloc
-; CHECK-NEXT: call i8* @malloc
-; CHECK-NEXT: call i8* @malloc
-; CHECK-NEXT: call i8* @malloc
-; CHECK-NEXT: call i8* @malloc
-; CHECK-NEXT: call i8* @malloc
-; CHECK-NEXT: call void @llvm.memcpy
-; CHECK-NEXT: call void @llvm.memmove
-; CHECK-NEXT: store
-; CHECK-NEXT: call void @llvm.memcpy
-; CHECK-NEXT: call void @llvm.memmove
-; CHECK-NEXT: call void @llvm.memset
-; CHECK-NEXT: store volatile
-; CHECK-NEXT: ret
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    [[A:%.*]] = call dereferenceable_or_null(700) i8* @malloc(i32 700)
+; CHECK-NEXT:    [[B:%.*]] = call dereferenceable_or_null(700) i8* @malloc(i32 700)
+; CHECK-NEXT:    [[C:%.*]] = call dereferenceable_or_null(700) i8* @malloc(i32 700)
+; CHECK-NEXT:    [[D:%.*]] = call dereferenceable_or_null(700) i8* @malloc(i32 700)
+; CHECK-NEXT:    [[E:%.*]] = call dereferenceable_or_null(700) i8* @malloc(i32 700)
+; CHECK-NEXT:    [[F:%.*]] = call dereferenceable_or_null(700) i8* @malloc(i32 700)
+; CHECK-NEXT:    [[G:%.*]] = call dereferenceable_or_null(700) i8* @malloc(i32 700)
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* nonnull align 1 dereferenceable(32) [[PTR:%.*]], i8* nonnull align 1 dereferenceable(32) [[A]], i32 32, i1 false)
+; CHECK-NEXT:    call void @llvm.memmove.p0i8.p0i8.i32(i8* nonnull align 1 dereferenceable(32) [[PTR]], i8* nonnull align 1 dereferenceable(32) [[B]], i32 32, i1 false)
+; CHECK-NEXT:    store i8* [[C]], i8** [[ESC:%.*]], align 8
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* [[D]], i8* [[PTR]], i32 32, i1 true)
+; CHECK-NEXT:    call void @llvm.memmove.p0i8.p0i8.i32(i8* [[E]], i8* [[PTR]], i32 32, i1 true)
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i32(i8* [[F]], i8 5, i32 32, i1 true)
+; CHECK-NEXT:    store volatile i8 4, i8* [[G]], align 1
+; CHECK-NEXT:    ret void
+;
   %a = call i8* @malloc(i32 700)
   %b = call i8* @malloc(i32 700)
   %c = call i8* @malloc(i32 700)
@@ -98,17 +104,19 @@ define void @test5(i8* %ptr, i8** %esc) {
 ;; Using simplifycfg will remove the empty basic block and the branch operation
 ;; Then, performing a dead elimination will remove the comparison.
 ;; This is what happens with -O1 and upper.
-; CHECK-LABEL: @test6(
 define void @test6(i8* %foo) minsize {
-; CHECK:  %tobool = icmp eq i8* %foo, null
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i8* [[FOO:%.*]], null
+; CHECK-NEXT:    tail call void @free(i8* [[FOO]])
+; CHECK-NEXT:    br i1 [[TOBOOL]], label [[IF_END:%.*]], label [[IF_THEN:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    ret void
+;
 ;; Call to free moved
-; CHECK-NEXT: tail call void @free(i8* %foo)
-; CHECK-NEXT: br i1 %tobool, label %if.end, label %if.then
-; CHECK: if.then:
 ;; Block is now empty and may be simplified by simplifycfg
-; CHECK-NEXT:   br label %if.end
-; CHECK: if.end:
-; CHECK-NEXT:  ret void
 entry:
   %tobool = icmp eq i8* %foo, null
   br i1 %tobool, label %if.end, label %if.then
@@ -126,27 +134,40 @@ declare void @_ZdlPvRKSt9nothrow_t(i8*, i8*) nobuiltin
 declare i32 @__gxx_personality_v0(...)
 declare void @_ZN1AC2Ev(i8* %this)
 
-; CHECK-LABEL: @test7(
 define void @test7() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    invoke void @_ZN1AC2Ev(i8* undef)
+; CHECK-NEXT:    to label [[DOTNOEXC_I:%.*]] unwind label [[LPAD_I:%.*]]
+; CHECK:       .noexc.i:
+; CHECK-NEXT:    unreachable
+; CHECK:       lpad.i:
+; CHECK-NEXT:    [[TMP0:%.*]] = landingpad { i8*, i32 }
+; CHECK-NEXT:    cleanup
+; CHECK-NEXT:    resume { i8*, i32 } [[TMP0]]
+;
 entry:
   %nt = alloca i8
-  ; CHECK-NOT: call {{.*}}@_ZnwmRKSt9nothrow_t(
   %call.i = tail call i8* @_ZnwmRKSt9nothrow_t(i64 1, i8* %nt) builtin nounwind
   invoke void @_ZN1AC2Ev(i8* undef)
-          to label %.noexc.i unwind label %lpad.i
+  to label %.noexc.i unwind label %lpad.i
 
 .noexc.i:                                         ; preds = %entry
   unreachable
 
 lpad.i:                                           ; preds = %entry
   %0 = landingpad { i8*, i32 } cleanup
-  ; CHECK-NOT: call {{.*}}@_ZdlPvRKSt9nothrow_t(
   call void @_ZdlPvRKSt9nothrow_t(i8* %call.i, i8* %nt) builtin nounwind
   resume { i8*, i32 } %0
 }
 
 declare i8* @_Znwm(i64) nobuiltin
 define i8* @_Znwj(i32 %n) nobuiltin {
+; CHECK-LABEL: @_Znwj(
+; CHECK-NEXT:    [[Z:%.*]] = zext i32 [[N:%.*]] to i64
+; CHECK-NEXT:    [[M:%.*]] = call i8* @_Znwm(i64 [[Z]])
+; CHECK-NEXT:    ret i8* [[M]]
+;
   %z = zext i32 %n to i64
   %m = call i8* @_Znwm(i64 %z)
   ret i8* %m
@@ -157,18 +178,34 @@ declare void @_ZdlPv(i8*) nobuiltin
 declare void @_ZdaPv(i8*) nobuiltin
 
 define linkonce void @_ZdlPvm(i8* %p, i64) nobuiltin {
+; CHECK-LABEL: @_ZdlPvm(
+; CHECK-NEXT:    call void @_ZdlPv(i8* [[P:%.*]])
+; CHECK-NEXT:    ret void
+;
   call void @_ZdlPv(i8* %p)
   ret void
 }
 define linkonce void @_ZdlPvj(i8* %p, i32) nobuiltin {
+; CHECK-LABEL: @_ZdlPvj(
+; CHECK-NEXT:    call void @_ZdlPv(i8* [[P:%.*]])
+; CHECK-NEXT:    ret void
+;
   call void @_ZdlPv(i8* %p)
   ret void
 }
 define linkonce void @_ZdaPvm(i8* %p, i64) nobuiltin {
+; CHECK-LABEL: @_ZdaPvm(
+; CHECK-NEXT:    call void @_ZdaPv(i8* [[P:%.*]])
+; CHECK-NEXT:    ret void
+;
   call void @_ZdaPv(i8* %p)
   ret void
 }
 define linkonce void @_ZdaPvj(i8* %p, i32) nobuiltin {
+; CHECK-LABEL: @_ZdaPvj(
+; CHECK-NEXT:    call void @_ZdaPv(i8* [[P:%.*]])
+; CHECK-NEXT:    ret void
+;
   call void @_ZdaPv(i8* %p)
   ret void
 }
@@ -196,9 +233,10 @@ declare void @_ZdlPvSt11align_val_tRKSt9nothrow_t(i8*, i64, i8*) nobuiltin
 declare void @_ZdaPvSt11align_val_tRKSt9nothrow_t(i8*, i64, i8*) nobuiltin
 
 
-; CHECK-LABEL: @test8(
 define void @test8() {
-  ; CHECK-NOT: call
+; CHECK-LABEL: @test8(
+; CHECK-NEXT:    ret void
+;
   %nt = alloca i8
   %nw = call i8* @_Znwm(i64 32) builtin
   call void @_ZdlPv(i8* %nw) builtin
@@ -234,25 +272,30 @@ define void @test8() {
 declare noalias i8* @"\01??2@YAPEAX_K@Z"(i64) nobuiltin
 declare void @"\01??3@YAXPEAX@Z"(i8*) nobuiltin
 
-; CHECK-LABEL: @test9(
 define void @test9() {
-  ; CHECK-NOT: call
+; CHECK-LABEL: @test9(
+; CHECK-NEXT:    ret void
+;
   %new_long_long = call noalias i8* @"\01??2@YAPEAX_K@Z"(i64 32) builtin
   call void @"\01??3@YAXPEAX@Z"(i8* %new_long_long) builtin
   ret void
 }
 
 define void @test10()  {
-; CHECK-LABEL: @test10
-; CHECK: call void @_ZdlPv
+; CHECK-LABEL: @test10(
+; CHECK-NEXT:    call void @_ZdlPv(i8* null)
+; CHECK-NEXT:    ret void
+;
   call void @_ZdlPv(i8* null)
   ret void
 }
 
 define void @test11() {
-; CHECK-LABEL: @test11
-; CHECK: call i8* @_Znwm
-; CHECK: call void @_ZdlPv
+; CHECK-LABEL: @test11(
+; CHECK-NEXT:    [[CALL:%.*]] = call dereferenceable(8) i8* @_Znwm(i64 8) #5
+; CHECK-NEXT:    call void @_ZdlPv(i8* nonnull [[CALL]])
+; CHECK-NEXT:    ret void
+;
   %call = call i8* @_Znwm(i64 8) builtin
   call void @_ZdlPv(i8* %call)
   ret void
@@ -260,19 +303,21 @@ define void @test11() {
 
 ;; Check that the optimization that moves a call to free in its predecessor
 ;; block (see test6) also happens when noop casts are involved.
-; CHECK-LABEL: @test12(
 define void @test12(i32* %foo) minsize {
-; CHECK:  %tobool = icmp eq i32* %foo, null
+; CHECK-LABEL: @test12(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i32* [[FOO:%.*]], null
+; CHECK-NEXT:    [[BITCAST:%.*]] = bitcast i32* [[FOO]] to i8*
+; CHECK-NEXT:    tail call void @free(i8* [[BITCAST]])
+; CHECK-NEXT:    br i1 [[TOBOOL]], label [[IF_END:%.*]], label [[IF_THEN:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    ret void
+;
 ;; Everything before the call to free should have been moved as well.
-; CHECK-NEXT:   %bitcast = bitcast i32* %foo to i8*
 ;; Call to free moved
-; CHECK-NEXT: tail call void @free(i8* %bitcast)
-; CHECK-NEXT: br i1 %tobool, label %if.end, label %if.then
-; CHECK: if.then:
 ;; Block is now empty and may be simplified by simplifycfg
-; CHECK-NEXT:   br label %if.end
-; CHECK: if.end:
-; CHECK-NEXT:  ret void
 entry:
   %tobool = icmp eq i32* %foo, null
   br i1 %tobool, label %if.end, label %if.then
diff --git a/llvm/test/Transforms/InstCombine/mem-deref-bytes-addrspaces.ll b/llvm/test/Transforms/InstCombine/mem-deref-bytes-addrspaces.ll
new file mode 100644
index 000000000..76eeeedfe
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/mem-deref-bytes-addrspaces.ll
@@ -0,0 +1,22 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -instcombine  -S < %s | FileCheck %s
+
+declare i32 @memcmp(i8 addrspace(1)* nocapture, i8* nocapture, i64)
+
+define i32 @memcmp_const_size_update_deref(i8 addrspace(1)* nocapture readonly %d, i8* nocapture readonly %s) {
+; CHECK-LABEL: @memcmp_const_size_update_deref(
+; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8 addrspace(1)* dereferenceable(16) dereferenceable_or_null(40) [[D:%.*]], i8* nonnull dereferenceable(16) [[S:%.*]], i64 16)
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = tail call i32 @memcmp(i8 addrspace(1)* dereferenceable_or_null(40) %d, i8* %s, i64 16)
+  ret i32 %call
+}
+
+define i32 @memcmp_nonconst_size_nonnnull(i8 addrspace(1)* nocapture readonly %d, i8* nocapture readonly %s, i64 %n) {
+; CHECK-LABEL: @memcmp_nonconst_size_nonnnull(
+; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8 addrspace(1)* nonnull dereferenceable_or_null(40) [[D:%.*]], i8* nonnull [[S:%.*]], i64 [[N:%.*]])
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = tail call i32 @memcmp(i8 addrspace(1)* nonnull dereferenceable_or_null(40) %d, i8* nonnull %s, i64 %n)
+  ret i32 %call
+}
diff --git a/llvm/test/Transforms/InstCombine/mem-deref-bytes.ll b/llvm/test/Transforms/InstCombine/mem-deref-bytes.ll
new file mode 100644
index 000000000..c19597a4b
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/mem-deref-bytes.ll
@@ -0,0 +1,163 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -instcombine  -S < %s | FileCheck %s
+
+declare i32 @memcmp(i8* nocapture, i8* nocapture, i64)
+declare i8* @memcpy(i8* nocapture, i8* nocapture, i64)
+declare i8* @memmove(i8* nocapture, i8* nocapture, i64)
+declare i8* @memset(i8* nocapture, i8, i64)
+declare i8* @memchr(i8* nocapture, i32, i64)
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i1)
+declare void @llvm.memmove.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i1)
+declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i1)
+
+define i32 @memcmp_const_size_set_deref(i8* nocapture readonly %d, i8* nocapture readonly %s) {
+; CHECK-LABEL: @memcmp_const_size_set_deref(
+; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* nonnull dereferenceable(16) [[D:%.*]], i8* nonnull dereferenceable(16) [[S:%.*]], i64 16)
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = tail call i32 @memcmp(i8* %d, i8* %s, i64 16)
+  ret i32 %call
+}
+
+define i32 @memcmp_const_size_update_deref(i8* nocapture readonly %d, i8* nocapture readonly %s) {
+; CHECK-LABEL: @memcmp_const_size_update_deref(
+; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* nonnull dereferenceable(16) [[D:%.*]], i8* nonnull dereferenceable(16) [[S:%.*]], i64 16)
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = tail call i32 @memcmp(i8* dereferenceable(4) %d, i8* dereferenceable(8) %s, i64 16)
+  ret i32 %call
+}
+
+define i32 @memcmp_const_size_update_deref2(i8* nocapture readonly %d, i8* nocapture readonly %s) {
+; CHECK-LABEL: @memcmp_const_size_update_deref2(
+; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* nonnull dereferenceable(16) [[D:%.*]], i8* nonnull dereferenceable(16) [[S:%.*]], i64 16)
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = tail call i32 @memcmp(i8* %d, i8* dereferenceable_or_null(8) %s, i64 16)
+  ret i32 %call
+}
+
+define i32 @memcmp_const_size_update_deref3(i8* nocapture readonly %d, i8* nocapture readonly %s) {
+; CHECK-LABEL: @memcmp_const_size_update_deref3(
+; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* nonnull dereferenceable(40) [[D:%.*]], i8* nonnull dereferenceable(16) [[S:%.*]], i64 16)
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = tail call i32 @memcmp(i8* dereferenceable(40) %d, i8* %s, i64 16)
+  ret i32 %call
+}
+
+define i32 @memcmp_const_size_update_deref4(i8* nocapture readonly %d, i8* nocapture readonly %s) {
+; CHECK-LABEL: @memcmp_const_size_update_deref4(
+; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* nonnull dereferenceable(16) [[D:%.*]], i8* nonnull dereferenceable(16) [[S:%.*]], i64 16)
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = tail call i32 @memcmp(i8* dereferenceable_or_null(16) %d, i8* %s, i64 16)
+  ret i32 %call
+}
+
+define i32 @memcmp_const_size_update_deref5(i8* nocapture readonly %d, i8* nocapture readonly %s) "null-pointer-is-valid"="false" {
+; CHECK-LABEL: @memcmp_const_size_update_deref5(
+; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* nonnull dereferenceable(40) [[D:%.*]], i8* nonnull dereferenceable(16) [[S:%.*]], i64 16)
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = tail call i32 @memcmp(i8* dereferenceable_or_null(40) %d, i8* %s, i64 16)
+  ret i32 %call
+}
+
+define i32 @memcmp_const_size_update_deref6(i8* nocapture readonly %d, i8* nocapture readonly %s) "null-pointer-is-valid"="true" {
+; CHECK-LABEL: @memcmp_const_size_update_deref6(
+; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* dereferenceable(16) dereferenceable_or_null(40) [[D:%.*]], i8* dereferenceable(16) [[S:%.*]], i64 16)
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = tail call i32 @memcmp(i8* dereferenceable_or_null(40) %d, i8* %s, i64 16)
+  ret i32 %call
+}
+
+define i32 @memcmp_const_size_update_deref7(i8* nocapture readonly %d, i8* nocapture readonly %s) "null-pointer-is-valid"="true" {
+; CHECK-LABEL: @memcmp_const_size_update_deref7(
+; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* nonnull dereferenceable(40) [[D:%.*]], i8* dereferenceable(16) [[S:%.*]], i64 16)
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = tail call i32 @memcmp(i8* nonnull dereferenceable_or_null(40) %d, i8* %s, i64 16)
+  ret i32 %call
+}
+
+define i32 @memcmp_const_size_no_update_deref(i8* nocapture readonly %d, i8* nocapture readonly %s) {
+; CHECK-LABEL: @memcmp_const_size_no_update_deref(
+; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* nonnull dereferenceable(40) [[D:%.*]], i8* nonnull dereferenceable(20) [[S:%.*]], i64 16)
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = tail call i32 @memcmp(i8* dereferenceable(40) %d, i8* dereferenceable(20) %s, i64 16)
+  ret i32 %call
+}
+
+define i32 @memcmp_nonconst_size(i8* nocapture readonly %d, i8* nocapture readonly %s, i64 %n) {
+; CHECK-LABEL: @memcmp_nonconst_size(
+; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* [[D:%.*]], i8* [[S:%.*]], i64 [[N:%.*]])
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = tail call i32 @memcmp(i8* %d, i8* %s, i64 %n)
+  ret i32 %call
+}
+
+define i8* @memcpy_const_size_set_deref(i8* nocapture readonly %d, i8* nocapture readonly %s) {
+; CHECK-LABEL: @memcpy_const_size_set_deref(
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 1 dereferenceable(64) [[D:%.*]], i8* nonnull align 1 dereferenceable(64) [[S:%.*]], i64 64, i1 false)
+; CHECK-NEXT:    ret i8* [[D]]
+;
+  %call = tail call i8* @memcpy(i8* %d, i8* %s, i64 64)
+  ret i8* %call
+}
+
+define i8* @memmove_const_size_set_deref(i8* nocapture readonly %d, i8* nocapture readonly %s) {
+; CHECK-LABEL: @memmove_const_size_set_deref(
+; CHECK-NEXT:    call void @llvm.memmove.p0i8.p0i8.i64(i8* nonnull align 1 dereferenceable(64) [[D:%.*]], i8* nonnull align 1 dereferenceable(64) [[S:%.*]], i64 64, i1 false)
+; CHECK-NEXT:    ret i8* [[D]]
+;
+  %call = tail call i8* @memmove(i8* %d, i8* %s, i64 64)
+  ret i8* %call
+}
+
+define i8* @memset_const_size_set_deref(i8* nocapture readonly %s, i8 %c) {
+; CHECK-LABEL: @memset_const_size_set_deref(
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* nonnull align 1 dereferenceable(64) [[S:%.*]], i8 [[C:%.*]], i64 64, i1 false)
+; CHECK-NEXT:    ret i8* [[S]]
+;
+  %call = tail call i8* @memset(i8* %s, i8 %c, i64 64)
+  ret i8* %call
+}
+
+define i8* @memchr_const_size_set_deref(i8* nocapture readonly %s, i32 %c) {
+; CHECK-LABEL: @memchr_const_size_set_deref(
+; CHECK-NEXT:    [[CALL:%.*]] = tail call i8* @memchr(i8* nonnull dereferenceable(64) [[S:%.*]], i32 [[C:%.*]], i64 64)
+; CHECK-NEXT:    ret i8* [[CALL]]
+;
+  %call = tail call i8* @memchr(i8* %s, i32 %c, i64 64)
+  ret i8* %call
+}
+
+define i8* @llvm_memcpy_const_size_set_deref(i8* nocapture readonly %d, i8* nocapture readonly %s) {
+; CHECK-LABEL: @llvm_memcpy_const_size_set_deref(
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 1 dereferenceable(16) [[D:%.*]], i8* nonnull align 1 dereferenceable(16) [[S:%.*]], i64 16, i1 false)
+; CHECK-NEXT:    ret i8* [[D]]
+;
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 1 %d, i8* align 1 %s, i64 16, i1 false)
+  ret i8* %d
+}
+
+define i8* @llvm_memmove_const_size_set_deref(i8* nocapture readonly %d, i8* nocapture readonly %s) {
+; CHECK-LABEL: @llvm_memmove_const_size_set_deref(
+; CHECK-NEXT:    call void @llvm.memmove.p0i8.p0i8.i64(i8* nonnull align 1 dereferenceable(16) [[D:%.*]], i8* nonnull align 1 dereferenceable(16) [[S:%.*]], i64 16, i1 false)
+; CHECK-NEXT:    ret i8* [[D]]
+;
+  call void @llvm.memmove.p0i8.p0i8.i64(i8* align 1 %d, i8* align 1 %s, i64 16, i1 false)
+  ret i8* %d
+}
+define i8* @llvm_memset_const_size_set_deref(i8* nocapture readonly %s, i8 %c) {
+; CHECK-LABEL: @llvm_memset_const_size_set_deref(
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* nonnull align 1 dereferenceable(16) [[S:%.*]], i8 [[C:%.*]], i64 16, i1 false)
+; CHECK-NEXT:    ret i8* [[S]]
+;
+  call void @llvm.memset.p0i8.i64(i8* align 1 %s, i8 %c, i64 16, i1 false)
+  ret i8* %s
+}
diff --git a/llvm/test/Transforms/InstCombine/mem-par-metadata-memcpy.ll b/llvm/test/Transforms/InstCombine/mem-par-metadata-memcpy.ll
index 54fe6cb84..4add5a4e4 100644
--- a/llvm/test/Transforms/InstCombine/mem-par-metadata-memcpy.ll
+++ b/llvm/test/Transforms/InstCombine/mem-par-metadata-memcpy.ll
@@ -50,7 +50,7 @@ for.end:                                          ; preds = %for.cond
 ; Function Attrs: argmemonly nounwind
 declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture writeonly, i8* nocapture readonly, i64, i1) #1
 
-attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { argmemonly nounwind }
 
 !llvm.ident = !{!0}
diff --git a/llvm/test/Transforms/InstCombine/memccpy.ll b/llvm/test/Transforms/InstCombine/memccpy.ll
new file mode 100644
index 000000000..cbb6aa38b
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/memccpy.ll
@@ -0,0 +1,214 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+@hello = private constant [11 x i8] c"helloworld\00", align 1
+@NoNulTerminator = private constant [10 x i8] c"helloworld", align 1
+@StopCharAfterNulTerminator = private constant [12 x i8] c"helloworld\00x", align 1
+@StringWithEOF =  constant [14 x i8] c"helloworld\FFab\00", align 1
+
+declare i8* @memccpy(i8*, i8*, i32, i64)
+
+define i8* @memccpy_to_memcpy(i8* %dst) {
+; CHECK-LABEL: @memccpy_to_memcpy(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8* [[DST:%.*]] to i64*
+; CHECK-NEXT:    store i64 8245940763182785896, i64* [[TMP1]], align 1
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, i8* [[DST]], i64 8
+; CHECK-NEXT:    ret i8* [[TMP2]]
+;
+  %call = call i8* @memccpy(i8* %dst, i8* getelementptr inbounds ([11 x i8], [11 x i8]* @hello, i64 0, i64 0), i32 114, i64 12) ; 114 is 'r'
+  ret i8* %call
+}
+
+define i8* @memccpy_to_memcpy2(i8* %dst) {
+; CHECK-LABEL: @memccpy_to_memcpy2(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8* [[DST:%.*]] to i64*
+; CHECK-NEXT:    store i64 8245940763182785896, i64* [[TMP1]], align 1
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, i8* [[DST]], i64 8
+; CHECK-NEXT:    ret i8* [[TMP2]]
+;
+  %call = call i8* @memccpy(i8* %dst, i8* getelementptr inbounds ([11 x i8], [11 x i8]* @hello, i64 0, i64 0), i32 114, i64 8); ; 114 is 'r'
+  ret i8* %call
+}
+
+define void @memccpy_to_memcpy3(i8* %dst) {
+; CHECK-LABEL: @memccpy_to_memcpy3(
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 1 dereferenceable(5) [[DST:%.*]], i8* nonnull align 1 dereferenceable(5) getelementptr inbounds ([11 x i8], [11 x i8]* @hello, i64 0, i64 0), i64 5, i1 false)
+; CHECK-NEXT:    ret void
+;
+  %call = call i8* @memccpy(i8* %dst, i8* getelementptr inbounds ([11 x i8], [11 x i8]* @hello, i64 0, i64 0), i32 111, i64 10) ; 111 is 'o'
+  ret void
+}
+
+define void @memccpy_to_memcpy4(i8* %dst) {
+; CHECK-LABEL: @memccpy_to_memcpy4(
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 1 dereferenceable(11) [[DST:%.*]], i8* nonnull align 1 dereferenceable(11) getelementptr inbounds ([11 x i8], [11 x i8]* @hello, i64 0, i64 0), i64 11, i1 false)
+; CHECK-NEXT:    ret void
+;
+  %call = call i8* @memccpy(i8* %dst, i8* getelementptr inbounds ([11 x i8], [11 x i8]* @hello, i64 0, i64 0), i32 0, i64 12)
+  ret void
+}
+
+define i8* @memccpy_to_memcpy5(i8* %dst) {
+; CHECK-LABEL: @memccpy_to_memcpy5(
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 1 dereferenceable(7) [[DST:%.*]], i8* nonnull align 1 dereferenceable(7) getelementptr inbounds ([11 x i8], [11 x i8]* @hello, i64 0, i64 0), i64 7, i1 false)
+; CHECK-NEXT:    ret i8* null
+;
+  %call = call i8* @memccpy(i8* %dst, i8* getelementptr inbounds ([11 x i8], [11 x i8]* @hello, i64 0, i64 0), i32 114, i64 7)
+  ret i8* %call
+}
+
+define i8* @memccpy_to_memcpy6(i8* %dst) {
+; CHECK-LABEL: @memccpy_to_memcpy6(
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 1 dereferenceable(6) [[DST:%.*]], i8* nonnull align 1 dereferenceable(6) getelementptr inbounds ([11 x i8], [11 x i8]* @hello, i64 0, i64 0), i64 6, i1 false)
+; CHECK-NEXT:    ret i8* null
+;
+  %call = call i8* @memccpy(i8* %dst, i8* getelementptr inbounds ([11 x i8], [11 x i8]* @hello, i64 0, i64 0), i32 114, i64 6);
+  ret i8* %call
+}
+
+define i8* @memccpy_to_memcpy7(i8* %dst) {
+; CHECK-LABEL: @memccpy_to_memcpy7(
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 1 dereferenceable(5) [[DST:%.*]], i8* nonnull align 1 dereferenceable(5) getelementptr inbounds ([11 x i8], [11 x i8]* @hello, i64 0, i64 0), i64 5, i1 false)
+; CHECK-NEXT:    ret i8* null
+;
+  %call = call i8* @memccpy(i8* %dst, i8* getelementptr inbounds ([11 x i8], [11 x i8]* @hello, i64 0, i64 0), i32 115, i64 5) ; 115 is 's'
+  ret i8* %call
+}
+
+define i8* @memccpy_to_memcpy8(i8* %dst) {
+; CHECK-LABEL: @memccpy_to_memcpy8(
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 1 dereferenceable(11) [[DST:%.*]], i8* nonnull align 1 dereferenceable(11) getelementptr inbounds ([11 x i8], [11 x i8]* @hello, i64 0, i64 0), i64 11, i1 false)
+; CHECK-NEXT:    ret i8* null
+;
+  %call = call i8* @memccpy(i8* %dst, i8* getelementptr inbounds ([11 x i8], [11 x i8]* @hello, i64 0, i64 0), i32 115, i64 11) ; 115 is 's'
+  ret i8* %call
+}
+
+define i8* @memccpy_to_memcpy9(i8* %dst, i64 %n) {
+; CHECK-LABEL: @memccpy_to_memcpy9(
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 1 dereferenceable(12) [[DST:%.*]], i8* nonnull align 1 dereferenceable(12) getelementptr inbounds ([12 x i8], [12 x i8]* @StopCharAfterNulTerminator, i64 0, i64 0), i64 12, i1 false)
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, i8* [[DST]], i64 12
+; CHECK-NEXT:    ret i8* [[TMP1]]
+;
+  %call = call i8* @memccpy(i8* %dst, i8* getelementptr inbounds ([12 x i8], [12 x i8]* @StopCharAfterNulTerminator, i64 0, i64 0), i32 120, i64 15) ; 120 is 'x'
+  ret i8* %call
+}
+
+define i8* @memccpy_to_memcpy10(i8* %dst, i64 %n) {
+; CHECK-LABEL: @memccpy_to_memcpy10(
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 1 dereferenceable(11) [[DST:%.*]], i8* nonnull align 1 dereferenceable(11) getelementptr inbounds ([14 x i8], [14 x i8]* @StringWithEOF, i64 0, i64 0), i64 11, i1 false)
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, i8* [[DST]], i64 11
+; CHECK-NEXT:    ret i8* [[TMP1]]
+;
+  %call = call i8* @memccpy(i8* %dst, i8* getelementptr inbounds ([14 x i8], [14 x i8]* @StringWithEOF, i64 0, i64 0), i32 255, i64 15)
+  ret i8* %call
+}
+
+define i8* @memccpy_to_memcpy11(i8* %dst, i64 %n) {
+; CHECK-LABEL: @memccpy_to_memcpy11(
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 1 dereferenceable(11) [[DST:%.*]], i8* nonnull align 1 dereferenceable(11) getelementptr inbounds ([14 x i8], [14 x i8]* @StringWithEOF, i64 0, i64 0), i64 11, i1 false)
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, i8* [[DST]], i64 11
+; CHECK-NEXT:    ret i8* [[TMP1]]
+;
+  %call = call i8* @memccpy(i8* %dst, i8* getelementptr inbounds ([14 x i8], [14 x i8]* @StringWithEOF, i64 0, i64 0), i32 -1, i64 15)
+  ret i8* %call
+}
+
+define i8* @memccpy_to_memcpy12(i8* %dst, i64 %n) {
+; CHECK-LABEL: @memccpy_to_memcpy12(
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 1 dereferenceable(11) [[DST:%.*]], i8* nonnull align 1 dereferenceable(11) getelementptr inbounds ([14 x i8], [14 x i8]* @StringWithEOF, i64 0, i64 0), i64 11, i1 false)
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, i8* [[DST]], i64 11
+; CHECK-NEXT:    ret i8* [[TMP1]]
+;
+  %call = call i8* @memccpy(i8* %dst, i8* getelementptr inbounds ([14 x i8], [14 x i8]* @StringWithEOF, i64 0, i64 0), i32 1023, i64 15)
+  ret i8* %call
+}
+
+define i8* @memccpy_to_null(i8* %dst, i8* %src, i32 %c) {
+; CHECK-LABEL: @memccpy_to_null(
+; CHECK-NEXT:    ret i8* null
+;
+  %call = call i8* @memccpy(i8* %dst, i8* %src, i32 %c, i64 0)
+  ret i8* %call
+}
+
+define void @memccpy_dst_src_same_retval_unused(i8* %dst, i32 %c, i64 %n) {
+; CHECK-LABEL: @memccpy_dst_src_same_retval_unused(
+; CHECK-NEXT:    ret void
+;
+  %call = call i8* @memccpy(i8* %dst, i8* %dst, i32 %c, i64 %n)
+  ret void
+}
+
+; Negative tests
+define i8* @unknown_src(i8* %dst, i8* %src) {
+; CHECK-LABEL: @unknown_src(
+; CHECK-NEXT:    [[CALL:%.*]] = call i8* @memccpy(i8* [[DST:%.*]], i8* [[SRC:%.*]], i32 114, i64 12)
+; CHECK-NEXT:    ret i8* [[CALL]]
+;
+  %call = call i8* @memccpy(i8* %dst, i8* %src, i32 114, i64 12)
+  ret i8* %call
+}
+
+define i8* @unknown_stop_char(i8* %dst, i32 %c) {
+; CHECK-LABEL: @unknown_stop_char(
+; CHECK-NEXT:    [[CALL:%.*]] = call i8* @memccpy(i8* [[DST:%.*]], i8* getelementptr inbounds ([11 x i8], [11 x i8]* @hello, i64 0, i64 0), i32 [[C:%.*]], i64 12)
+; CHECK-NEXT:    ret i8* [[CALL]]
+;
+  %call = call i8* @memccpy(i8* %dst, i8* getelementptr inbounds ([11 x i8], [11 x i8]* @hello, i64 0, i64 0), i32 %c, i64 12)
+  ret i8* %call
+}
+
+define i8* @unknown_size_n(i8* %dst, i64 %n) {
+; CHECK-LABEL: @unknown_size_n(
+; CHECK-NEXT:    [[CALL:%.*]] = call i8* @memccpy(i8* [[DST:%.*]], i8* getelementptr inbounds ([11 x i8], [11 x i8]* @hello, i64 0, i64 0), i32 114, i64 [[N:%.*]])
+; CHECK-NEXT:    ret i8* [[CALL]]
+;
+  %call = call i8* @memccpy(i8* %dst, i8* getelementptr inbounds ([11 x i8], [11 x i8]* @hello, i64 0, i64 0), i32 114, i64 %n)
+  ret i8* %call
+}
+
+define i8* @no_nul_terminator(i8* %dst, i64 %n) {
+; CHECK-LABEL: @no_nul_terminator(
+; CHECK-NEXT:    [[CALL:%.*]] = call i8* @memccpy(i8* [[DST:%.*]], i8* getelementptr inbounds ([12 x i8], [12 x i8]* @StopCharAfterNulTerminator, i64 0, i64 0), i32 120, i64 [[N:%.*]])
+; CHECK-NEXT:    ret i8* [[CALL]]
+;
+  %call = call i8* @memccpy(i8* %dst, i8* getelementptr inbounds ([12 x i8], [12 x i8]* @StopCharAfterNulTerminator, i64 0, i64 0), i32 120, i64 %n) ; 120 is 'x'
+  ret i8* %call
+}
+
+define i8* @possibly_valid_data_after_array(i8* %dst, i64 %n) {
+; CHECK-LABEL: @possibly_valid_data_after_array(
+; CHECK-NEXT:    [[CALL:%.*]] = call i8* @memccpy(i8* [[DST:%.*]], i8* getelementptr inbounds ([10 x i8], [10 x i8]* @NoNulTerminator, i64 0, i64 0), i32 115, i64 [[N:%.*]])
+; CHECK-NEXT:    ret i8* [[CALL]]
+;
+  %call = call i8* @memccpy(i8* %dst, i8* getelementptr inbounds ([10 x i8], [10 x i8]* @NoNulTerminator, i64 0, i64 0), i32 115, i64 %n) ; 115 is 's'
+  ret i8* %call
+}
+
+define i8* @possibly_valid_data_after_array2(i8* %dst, i64 %n) {
+; CHECK-LABEL: @possibly_valid_data_after_array2(
+; CHECK-NEXT:    [[CALL:%.*]] = call i8* @memccpy(i8* [[DST:%.*]], i8* getelementptr inbounds ([11 x i8], [11 x i8]* @hello, i64 0, i64 0), i32 115, i64 [[N:%.*]])
+; CHECK-NEXT:    ret i8* [[CALL]]
+;
+  %call = call i8* @memccpy(i8* %dst, i8* getelementptr inbounds ([11 x i8], [11 x i8]* @hello, i64 0, i64 0), i32 115, i64 %n) ; 115 is 's'
+  ret i8* %call
+}
+
+define i8* @possibly_valid_data_after_array3(i8* %dst) {
+; CHECK-LABEL: @possibly_valid_data_after_array3(
+; CHECK-NEXT:    [[CALL:%.*]] = call i8* @memccpy(i8* [[DST:%.*]], i8* getelementptr inbounds ([11 x i8], [11 x i8]* @hello, i64 0, i64 0), i32 115, i64 12)
+; CHECK-NEXT:    ret i8* [[CALL]]
+;
+  %call = call i8* @memccpy(i8* %dst, i8* getelementptr inbounds ([11 x i8], [11 x i8]* @hello, i64 0, i64 0), i32 115, i64 12) ; 115 is 's'
+  ret i8* %call
+}
+
+define i8* @memccpy_dst_src_same_retval_used(i8* %dst, i32 %c, i64 %n) {
+; CHECK-LABEL: @memccpy_dst_src_same_retval_used(
+; CHECK-NEXT:    [[CALL:%.*]] = call i8* @memccpy(i8* [[DST:%.*]], i8* [[DST]], i32 [[C:%.*]], i64 [[N:%.*]])
+; CHECK-NEXT:    ret i8* [[CALL]]
+;
+  %call = call i8* @memccpy(i8* %dst, i8* %dst, i32 %c, i64 %n)
+  ret i8* %call
+}
diff --git a/llvm/test/Transforms/InstCombine/memchr.ll b/llvm/test/Transforms/InstCombine/memchr.ll
index 83073e206..2e2c29a07 100644
--- a/llvm/test/Transforms/InstCombine/memchr.ll
+++ b/llvm/test/Transforms/InstCombine/memchr.ll
@@ -50,7 +50,7 @@ define void @test3() {
 
 define void @test4(i32 %chr) {
 ; CHECK-LABEL: @test4(
-; CHECK-NEXT:    [[DST:%.*]] = call i8* @memchr(i8* getelementptr inbounds ([14 x i8], [14 x i8]* @hello, i32 0, i32 0), i32 [[CHR:%.*]], i32 14)
+; CHECK-NEXT:    [[DST:%.*]] = call i8* @memchr(i8* nonnull dereferenceable(14) getelementptr inbounds ([14 x i8], [14 x i8]* @hello, i32 0, i32 0), i32 [[CHR:%.*]], i32 14)
 ; CHECK-NEXT:    store i8* [[DST]], i8** @chp, align 4
 ; CHECK-NEXT:    ret void
 ;
@@ -148,7 +148,7 @@ define i1 @test11(i32 %C) {
 ; No 64 bits here
 define i1 @test12(i32 %C) {
 ; CHECK-LABEL: @test12(
-; CHECK-NEXT:    [[DST:%.*]] = call i8* @memchr(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @spaces, i32 0, i32 0), i32 [[C:%.*]], i32 3)
+; CHECK-NEXT:    [[DST:%.*]] = call i8* @memchr(i8* nonnull dereferenceable(3) getelementptr inbounds ([4 x i8], [4 x i8]* @spaces, i32 0, i32 0), i32 [[C:%.*]], i32 3)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i8* [[DST]], null
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -185,7 +185,7 @@ define i1 @test14(i32 %C) {
 
 define i1 @test15(i32 %C) {
 ; CHECK-LABEL: @test15(
-; CHECK-NEXT:    [[DST:%.*]] = call i8* @memchr(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @negative, i32 0, i32 0), i32 [[C:%.*]], i32 3)
+; CHECK-NEXT:    [[DST:%.*]] = call i8* @memchr(i8* nonnull dereferenceable(3) getelementptr inbounds ([3 x i8], [3 x i8]* @negative, i32 0, i32 0), i32 [[C:%.*]], i32 3)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i8* [[DST]], null
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -202,3 +202,43 @@ define i8* @pr32124() {
   %res = tail call i8* @memchr(i8* getelementptr ([1 x i8], [1 x i8]* @s, i64 0, i64 0), i32 0, i32 1)
   ret i8* %res
 }
+
+define i8* @test16(i8* %str, i32 %c, i32 %n) {
+; CHECK-LABEL: @test16(
+; CHECK-NEXT:    [[RET:%.*]] = call i8* @memchr(i8* [[STR:%.*]], i32 [[C:%.*]], i32 [[N:%.*]])
+; CHECK-NEXT:    ret i8* [[RET]]
+;
+
+  %ret = call i8* @memchr(i8* %str, i32 %c, i32 %n)
+  ret i8* %ret
+}
+
+define i8* @test17(i8* %str, i32 %c, i32 %n) {
+; CHECK-LABEL: @test17(
+; CHECK-NEXT:    [[RET:%.*]] = call i8* @memchr(i8* nonnull [[STR:%.*]], i32 [[C:%.*]], i32 [[N:%.*]])
+; CHECK-NEXT:    ret i8* [[RET]]
+;
+
+  %ret = call i8* @memchr(i8* nonnull %str, i32 %c, i32 %n)
+  ret i8* %ret
+}
+
+define i8* @test18(i8* %str, i32 %c) {
+; CHECK-LABEL: @test18(
+; CHECK-NEXT:    [[RET:%.*]] = call i8* @memchr(i8* nonnull dereferenceable(5) [[STR:%.*]], i32 [[C:%.*]], i32 5)
+; CHECK-NEXT:    ret i8* [[RET]]
+;
+
+  %ret = call i8* @memchr(i8* %str, i32 %c, i32 5)
+  ret i8* %ret
+}
+
+define i8* @test19(i8* %str, i32 %c) "null-pointer-is-valid"="true" {
+; CHECK-LABEL: @test19(
+; CHECK-NEXT:    [[RET:%.*]] = call i8* @memchr(i8* dereferenceable(5) [[STR:%.*]], i32 [[C:%.*]], i32 5)
+; CHECK-NEXT:    ret i8* [[RET]]
+;
+
+  %ret = call i8* @memchr(i8* %str, i32 %c, i32 5)
+  ret i8* %ret
+}
diff --git a/llvm/test/Transforms/InstCombine/memcmp-constant-fold.ll b/llvm/test/Transforms/InstCombine/memcmp-constant-fold.ll
index 211b3b5ab..22b946375 100644
--- a/llvm/test/Transforms/InstCombine/memcmp-constant-fold.ll
+++ b/llvm/test/Transforms/InstCombine/memcmp-constant-fold.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -instcombine -S -data-layout=e-n32 | FileCheck %s --check-prefix=ALL --check-prefix=LE
 ; RUN: opt < %s -instcombine -S -data-layout=E-n32 | FileCheck %s --check-prefix=ALL --check-prefix=BE
 
@@ -9,13 +10,13 @@ declare i32 @memcmp(i8*, i8*, i64)
 
 define i1 @memcmp_4bytes_unaligned_constant_i8(i8* align 4 %x) {
 ; LE-LABEL: @memcmp_4bytes_unaligned_constant_i8(
-; LE-NEXT:    [[TMP1:%.*]] = bitcast i8* %x to i32*
+; LE-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i32*
 ; LE-NEXT:    [[LHSV:%.*]] = load i32, i32* [[TMP1]], align 4
 ; LE-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[LHSV]], 16777216
 ; LE-NEXT:    ret i1 [[TMP2]]
 ;
 ; BE-LABEL: @memcmp_4bytes_unaligned_constant_i8(
-; BE-NEXT:    [[TMP1:%.*]] = bitcast i8* %x to i32*
+; BE-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i32*
 ; BE-NEXT:    [[LHSV:%.*]] = load i32, i32* [[TMP1]], align 4
 ; BE-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[LHSV]], 1
 ; BE-NEXT:    ret i1 [[TMP2]]
@@ -32,13 +33,13 @@ define i1 @memcmp_4bytes_unaligned_constant_i8(i8* align 4 %x) {
 
 define i1 @memcmp_4bytes_unaligned_constant_i16(i8* align 4 %x) {
 ; LE-LABEL: @memcmp_4bytes_unaligned_constant_i16(
-; LE-NEXT:    [[TMP1:%.*]] = bitcast i8* %x to i32*
+; LE-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i32*
 ; LE-NEXT:    [[RHSV:%.*]] = load i32, i32* [[TMP1]], align 4
 ; LE-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[RHSV]], 131073
 ; LE-NEXT:    ret i1 [[TMP2]]
 ;
 ; BE-LABEL: @memcmp_4bytes_unaligned_constant_i16(
-; BE-NEXT:    [[TMP1:%.*]] = bitcast i8* %x to i32*
+; BE-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i32*
 ; BE-NEXT:    [[RHSV:%.*]] = load i32, i32* [[TMP1]], align 4
 ; BE-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[RHSV]], 65538
 ; BE-NEXT:    ret i1 [[TMP2]]
@@ -54,7 +55,7 @@ define i1 @memcmp_4bytes_unaligned_constant_i16(i8* align 4 %x) {
 
 define i1 @memcmp_3bytes_aligned_constant_i32(i8* align 4 %x) {
 ; ALL-LABEL: @memcmp_3bytes_aligned_constant_i32(
-; ALL-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* bitcast (i32* getelementptr inbounds ([2 x i32], [2 x i32]* @intbuf, i64 0, i64 1) to i8*), i8* bitcast ([2 x i32]* @intbuf to i8*), i64 3)
+; ALL-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* nonnull dereferenceable(3) bitcast (i32* getelementptr inbounds ([2 x i32], [2 x i32]* @intbuf, i64 0, i64 1) to i8*), i8* nonnull dereferenceable(3) bitcast ([2 x i32]* @intbuf to i8*), i64 3)
 ; ALL-NEXT:    [[CMPEQ0:%.*]] = icmp eq i32 [[CALL]], 0
 ; ALL-NEXT:    ret i1 [[CMPEQ0]]
 ;
@@ -67,7 +68,7 @@ define i1 @memcmp_3bytes_aligned_constant_i32(i8* align 4 %x) {
 
 define i1 @memcmp_4bytes_one_unaligned_i8(i8* align 4 %x, i8* align 1 %y) {
 ; ALL-LABEL: @memcmp_4bytes_one_unaligned_i8(
-; ALL-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* %x, i8* %y, i64 4)
+; ALL-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* nonnull dereferenceable(4) [[X:%.*]], i8* nonnull dereferenceable(4) [[Y:%.*]], i64 4)
 ; ALL-NEXT:    [[CMPEQ0:%.*]] = icmp eq i32 [[CALL]], 0
 ; ALL-NEXT:    ret i1 [[CMPEQ0]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/memcpy-1.ll b/llvm/test/Transforms/InstCombine/memcpy-1.ll
index dceb7c392..789e5ebd7 100644
--- a/llvm/test/Transforms/InstCombine/memcpy-1.ll
+++ b/llvm/test/Transforms/InstCombine/memcpy-1.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; Test that the memcpy library call simplifier works correctly.
 ;
 ; RUN: opt < %s -instcombine -S | FileCheck %s
@@ -10,8 +11,8 @@ declare i8* @memcpy(i8*, i8*, i32)
 
 define i8* @test_simplify1(i8* %mem1, i8* %mem2, i32 %size) {
 ; CHECK-LABEL: @test_simplify1(
-; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 %mem1, i8* align 1 %mem2, i32 %size, i1 false)
-; CHECK-NEXT:    ret i8* %mem1
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 [[MEM1:%.*]], i8* align 1 [[MEM2:%.*]], i32 [[SIZE:%.*]], i1 false)
+; CHECK-NEXT:    ret i8* [[MEM1]]
 ;
   %ret = call i8* @memcpy(i8* %mem1, i8* %mem2, i32 %size)
   ret i8* %ret
@@ -19,10 +20,11 @@ define i8* @test_simplify1(i8* %mem1, i8* %mem2, i32 %size) {
 
 ; Verify that the strictfp attr doesn't block this optimization.
 
-define i8* @test_simplify2(i8* %mem1, i8* %mem2, i32 %size) {
+define i8* @test_simplify2(i8* %mem1, i8* %mem2, i32 %size) strictfp {
 ; CHECK-LABEL: @test_simplify2(
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 [[MEM1:%.*]], i8* align 1 [[MEM2:%.*]], i32 [[SIZE:%.*]], i1 false)
+; CHECK-NEXT:    ret i8* [[MEM1]]
+;
   %ret = call i8* @memcpy(i8* %mem1, i8* %mem2, i32 %size) strictfp
-; CHECK: call void @llvm.memcpy
   ret i8* %ret
-; CHECK: ret i8* %mem1
 }
diff --git a/llvm/test/Transforms/InstCombine/memcpy-from-global.ll b/llvm/test/Transforms/InstCombine/memcpy-from-global.ll
index c14b1a70f..96029fa59 100644
--- a/llvm/test/Transforms/InstCombine/memcpy-from-global.ll
+++ b/llvm/test/Transforms/InstCombine/memcpy-from-global.ll
@@ -1,39 +1,66 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -instcombine -S | FileCheck %s
 target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64"
 @C.0.1248 = internal constant [128 x float] [ float -1.000000e+00, float -1.000000e+00, float -1.000000e+00, float 0.000000e+00, float -1.000000e+00, float -1.000000e+00, float 0.000000e+00, float -1.000000e+00, float -1.000000e+00, float -1.000000e+00, float 0.000000e+00, float 1.000000e+00, float -1.000000e+00, float -1.000000e+00, float 1.000000e+00, float 0.000000e+00, float -1.000000e+00, float 0.000000e+00, float -1.000000e+00, float -1.000000e+00, float -1.000000e+00, float 0.000000e+00, float -1.000000e+00, float 1.000000e+00, float -1.000000e+00, float 0.000000e+00, float 1.000000e+00, float -1.000000e+00, float -1.000000e+00, float 0.000000e+00, float 1.000000e+00, float 1.000000e+00, float -1.000000e+00, float 1.000000e+00, float -1.000000e+00, float 0.000000e+00, float -1.000000e+00, float 1.000000e+00, float 0.000000e+00, float -1.000000e+00, float -1.000000e+00, float 1.000000e+00, float 0.000000e+00, float 1.000000e+00, float -1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 0.000000e+00, float 0.000000e+00, float -1.000000e+00, float -1.000000e+00, float -1.000000e+00, float 0.000000e+00, float -1.000000e+00, float -1.000000e+00, float 1.000000e+00, float 0.000000e+00, float -1.000000e+00, float 1.000000e+00, float -1.000000e+00, float 0.000000e+00, float -1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float -1.000000e+00, float -1.000000e+00, float 0.000000e+00, float 1.000000e+00, float -1.000000e+00, float 0.000000e+00, float -1.000000e+00, float 1.000000e+00, float -1.000000e+00, float 0.000000e+00, float 1.000000e+00, float 1.000000e+00, float -1.000000e+00, float 1.000000e+00, float 0.000000e+00, float 1.000000e+00, float 0.000000e+00, float -1.000000e+00, float -1.000000e+00, float 1.000000e+00, float 0.000000e+00, float -1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 0.000000e+00, float 1.000000e+00, float -1.000000e+00, float 1.000000e+00, float 0.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float -1.000000e+00, float 0.000000e+00, float 1.000000e+00, float 1.000000e+00, float 0.000000e+00, float -1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 0.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 0.000000e+00, float 0.000000e+00, float 1.000000e+00, float -1.000000e+00, float -1.000000e+00, float 0.000000e+00, float 1.000000e+00, float -1.000000e+00, float 1.000000e+00, float 0.000000e+00, float 1.000000e+00, float 1.000000e+00, float -1.000000e+00, float 0.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00 ], align 32		; <[128 x float]*> [#uses=1]
 
 define float @test1(i32 %hash, float %x, float %y, float %z, float %w) {
-entry:
-	%lookupTable = alloca [128 x float], align 16		; <[128 x float]*> [#uses=5]
-	%lookupTable1 = bitcast [128 x float]* %lookupTable to i8*		; <i8*> [#uses=1]
-	call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 16 %lookupTable1, i8* align 16 bitcast ([128 x float]* @C.0.1248 to i8*), i64 512, i1 false)
-
 ; CHECK-LABEL: @test1(
-; CHECK-NOT: alloca
-; CHECK-NOT: call{{.*}}@llvm.memcpy
-
-	%tmp3 = shl i32 %hash, 2		; <i32> [#uses=1]
-	%tmp5 = and i32 %tmp3, 124		; <i32> [#uses=4]
-	%tmp753 = getelementptr [128 x float], [128 x float]* %lookupTable, i32 0, i32 %tmp5		; <float*> [#uses=1]
-	%tmp9 = load float, float* %tmp753		; <float> [#uses=1]
-	%tmp11 = fmul float %tmp9, %x		; <float> [#uses=1]
-	%tmp13 = fadd float %tmp11, 0.000000e+00		; <float> [#uses=1]
-	%tmp17.sum52 = or i32 %tmp5, 1		; <i32> [#uses=1]
-	%tmp1851 = getelementptr [128 x float], [128 x float]* %lookupTable, i32 0, i32 %tmp17.sum52		; <float*> [#uses=1]
-	%tmp19 = load float, float* %tmp1851		; <float> [#uses=1]
-	%tmp21 = fmul float %tmp19, %y		; <float> [#uses=1]
-	%tmp23 = fadd float %tmp21, %tmp13		; <float> [#uses=1]
-	%tmp27.sum50 = or i32 %tmp5, 2		; <i32> [#uses=1]
-	%tmp2849 = getelementptr [128 x float], [128 x float]* %lookupTable, i32 0, i32 %tmp27.sum50		; <float*> [#uses=1]
-	%tmp29 = load float, float* %tmp2849		; <float> [#uses=1]
-	%tmp31 = fmul float %tmp29, %z		; <float> [#uses=1]
-	%tmp33 = fadd float %tmp31, %tmp23		; <float> [#uses=1]
-	%tmp37.sum48 = or i32 %tmp5, 3		; <i32> [#uses=1]
-	%tmp3847 = getelementptr [128 x float], [128 x float]* %lookupTable, i32 0, i32 %tmp37.sum48		; <float*> [#uses=1]
-	%tmp39 = load float, float* %tmp3847		; <float> [#uses=1]
-	%tmp41 = fmul float %tmp39, %w		; <float> [#uses=1]
-	%tmp43 = fadd float %tmp41, %tmp33		; <float> [#uses=1]
-	ret float %tmp43
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP3:%.*]] = shl i32 [[HASH:%.*]], 2
+; CHECK-NEXT:    [[TMP5:%.*]] = and i32 [[TMP3]], 124
+; CHECK-NEXT:    [[TMP0:%.*]] = zext i32 [[TMP5]] to i64
+; CHECK-NEXT:    [[TMP753:%.*]] = getelementptr [128 x float], [128 x float]* @C.0.1248, i64 0, i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP9:%.*]] = load float, float* [[TMP753]], align 16
+; CHECK-NEXT:    [[TMP11:%.*]] = fmul float [[TMP9]], [[X:%.*]]
+; CHECK-NEXT:    [[TMP13:%.*]] = fadd float [[TMP11]], 0.000000e+00
+; CHECK-NEXT:    [[TMP17_SUM52:%.*]] = or i32 [[TMP5]], 1
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[TMP17_SUM52]] to i64
+; CHECK-NEXT:    [[TMP1851:%.*]] = getelementptr [128 x float], [128 x float]* @C.0.1248, i64 0, i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP19:%.*]] = load float, float* [[TMP1851]], align 4
+; CHECK-NEXT:    [[TMP21:%.*]] = fmul float [[TMP19]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP23:%.*]] = fadd float [[TMP21]], [[TMP13]]
+; CHECK-NEXT:    [[TMP27_SUM50:%.*]] = or i32 [[TMP5]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i32 [[TMP27_SUM50]] to i64
+; CHECK-NEXT:    [[TMP2849:%.*]] = getelementptr [128 x float], [128 x float]* @C.0.1248, i64 0, i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP29:%.*]] = load float, float* [[TMP2849]], align 8
+; CHECK-NEXT:    [[TMP31:%.*]] = fmul float [[TMP29]], [[Z:%.*]]
+; CHECK-NEXT:    [[TMP33:%.*]] = fadd float [[TMP31]], [[TMP23]]
+; CHECK-NEXT:    [[TMP37_SUM48:%.*]] = or i32 [[TMP5]], 3
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i32 [[TMP37_SUM48]] to i64
+; CHECK-NEXT:    [[TMP3847:%.*]] = getelementptr [128 x float], [128 x float]* @C.0.1248, i64 0, i64 [[TMP3]]
+; CHECK-NEXT:    [[TMP39:%.*]] = load float, float* [[TMP3847]], align 4
+; CHECK-NEXT:    [[TMP41:%.*]] = fmul float [[TMP39]], [[W:%.*]]
+; CHECK-NEXT:    [[TMP43:%.*]] = fadd float [[TMP41]], [[TMP33]]
+; CHECK-NEXT:    ret float [[TMP43]]
+;
+entry:
+  %lookupTable = alloca [128 x float], align 16		; <[128 x float]*> [#uses=5]
+  %lookupTable1 = bitcast [128 x float]* %lookupTable to i8*		; <i8*> [#uses=1]
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 16 %lookupTable1, i8* align 16 bitcast ([128 x float]* @C.0.1248 to i8*), i64 512, i1 false)
+
+
+  %tmp3 = shl i32 %hash, 2		; <i32> [#uses=1]
+  %tmp5 = and i32 %tmp3, 124		; <i32> [#uses=4]
+  %tmp753 = getelementptr [128 x float], [128 x float]* %lookupTable, i32 0, i32 %tmp5		; <float*> [#uses=1]
+  %tmp9 = load float, float* %tmp753		; <float> [#uses=1]
+  %tmp11 = fmul float %tmp9, %x		; <float> [#uses=1]
+  %tmp13 = fadd float %tmp11, 0.000000e+00		; <float> [#uses=1]
+  %tmp17.sum52 = or i32 %tmp5, 1		; <i32> [#uses=1]
+  %tmp1851 = getelementptr [128 x float], [128 x float]* %lookupTable, i32 0, i32 %tmp17.sum52		; <float*> [#uses=1]
+  %tmp19 = load float, float* %tmp1851		; <float> [#uses=1]
+  %tmp21 = fmul float %tmp19, %y		; <float> [#uses=1]
+  %tmp23 = fadd float %tmp21, %tmp13		; <float> [#uses=1]
+  %tmp27.sum50 = or i32 %tmp5, 2		; <i32> [#uses=1]
+  %tmp2849 = getelementptr [128 x float], [128 x float]* %lookupTable, i32 0, i32 %tmp27.sum50		; <float*> [#uses=1]
+  %tmp29 = load float, float* %tmp2849		; <float> [#uses=1]
+  %tmp31 = fmul float %tmp29, %z		; <float> [#uses=1]
+  %tmp33 = fadd float %tmp31, %tmp23		; <float> [#uses=1]
+  %tmp37.sum48 = or i32 %tmp5, 3		; <i32> [#uses=1]
+  %tmp3847 = getelementptr [128 x float], [128 x float]* %lookupTable, i32 0, i32 %tmp37.sum48		; <float*> [#uses=1]
+  %tmp39 = load float, float* %tmp3847		; <float> [#uses=1]
+  %tmp41 = fmul float %tmp39, %w		; <float> [#uses=1]
+  %tmp43 = fadd float %tmp41, %tmp33		; <float> [#uses=1]
+  ret float %tmp43
 }
 
 declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i1) nounwind
@@ -48,19 +75,22 @@ declare void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)* nocapture, i8 addrspace
 @H = constant [2 x %U] zeroinitializer, align 16
 
 define void @test2() {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[B1:%.*]] = alloca [124 x i8], align 8
+; CHECK-NEXT:    [[B1_SUB:%.*]] = getelementptr inbounds [124 x i8], [124 x i8]* [[B1]], i64 0, i64 0
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 8 dereferenceable(124) [[B1_SUB]], i8* nonnull align 16 dereferenceable(124) getelementptr inbounds (%T, %T* @G, i64 0, i32 0), i64 124, i1 false)
+; CHECK-NEXT:    call void @bar(i8* nonnull [[B1_SUB]])
+; CHECK-NEXT:    ret void
+;
   %A = alloca %T
   %B = alloca %T
   %a = bitcast %T* %A to i8*
   %b = bitcast %T* %B to i8*
 
-; CHECK-LABEL: @test2(
 
 ; %A alloca is deleted
-; CHECK-NEXT: alloca [124 x i8]
-; CHECK-NEXT: getelementptr inbounds [124 x i8], [124 x i8]*
 
 ; use @G instead of %A
-; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 8 %{{.*}}, i8* align 16 getelementptr inbounds (%T, %T* @G, i64 0, i32 0)
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %a, i8* align 4 bitcast (%T* @G to i8*), i64 124, i1 false)
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %b, i8* align 4 %a, i64 124, i1 false)
   call void @bar(i8* %b)
@@ -68,19 +98,22 @@ define void @test2() {
 }
 
 define void @test2_no_null_opt() #0 {
+; CHECK-LABEL: @test2_no_null_opt(
+; CHECK-NEXT:    [[B1:%.*]] = alloca [124 x i8], align 8
+; CHECK-NEXT:    [[B1_SUB:%.*]] = getelementptr inbounds [124 x i8], [124 x i8]* [[B1]], i64 0, i64 0
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 8 dereferenceable(124) [[B1_SUB]], i8* align 16 dereferenceable(124) getelementptr inbounds (%T, %T* @G, i64 0, i32 0), i64 124, i1 false)
+; CHECK-NEXT:    call void @bar(i8* [[B1_SUB]])
+; CHECK-NEXT:    ret void
+;
   %A = alloca %T
   %B = alloca %T
   %a = bitcast %T* %A to i8*
   %b = bitcast %T* %B to i8*
 
-; CHECK-LABEL: @test2_no_null_opt(
 
 ; %A alloca is deleted
-; CHECK-NEXT: alloca [124 x i8]
-; CHECK-NEXT: getelementptr inbounds [124 x i8], [124 x i8]*
 
 ; use @G instead of %A
-; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 8 %{{.*}}, i8* align 16 getelementptr inbounds (%T, %T* @G, i64 0, i32 0)
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %a, i8* align 4 bitcast (%T* @G to i8*), i64 124, i1 false)
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %b, i8* align 4 %a, i64 124, i1 false)
   call void @bar(i8* %b)
@@ -88,22 +121,25 @@ define void @test2_no_null_opt() #0 {
 }
 
 define void @test2_addrspacecast() {
+; CHECK-LABEL: @test2_addrspacecast(
+; CHECK-NEXT:    [[B1:%.*]] = alloca [124 x i8], align 8
+; CHECK-NEXT:    [[B1_SUB:%.*]] = getelementptr inbounds [124 x i8], [124 x i8]* [[B1]], i64 0, i64 0
+; CHECK-NEXT:    [[B:%.*]] = addrspacecast i8* [[B1_SUB]] to i8 addrspace(1)*
+; CHECK-NEXT:    call void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)* align 4 dereferenceable(124) [[B]], i8 addrspace(1)* align 4 dereferenceable(124) addrspacecast (i8* getelementptr inbounds (%T, %T* @G, i64 0, i32 0) to i8 addrspace(1)*), i64 124, i1 false)
+; CHECK-NEXT:    call void @bar_as1(i8 addrspace(1)* [[B]])
+; CHECK-NEXT:    ret void
+;
   %A = alloca %T
   %B = alloca %T
   %a = addrspacecast %T* %A to i8 addrspace(1)*
   %b = addrspacecast %T* %B to i8 addrspace(1)*
 
-; CHECK-LABEL: @test2_addrspacecast(
 
 ; %A alloca is deleted
 ; This doesn't exactly match what test2 does, because folding the type
 ; cast into the alloca doesn't work for the addrspacecast yet.
-; CHECK-NEXT: alloca [124 x i8]
-; CHECK-NEXT: getelementptr
-; CHECK-NEXT: addrspacecast
 
 ; use @G instead of %A
-; CHECK-NEXT: call void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)* align 4 %{{.*}},
   call void @llvm.memcpy.p1i8.p0i8.i64(i8 addrspace(1)* align 4 %a, i8* align 4 bitcast (%T* @G to i8*), i64 124, i1 false)
   call void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)* align 4 %b, i8 addrspace(1)* align 4 %a, i64 124, i1 false)
   call void @bar_as1(i8 addrspace(1)* %b)
@@ -116,45 +152,53 @@ declare void @bar_as1(i8 addrspace(1)*)
 
 ;; Should be able to eliminate the alloca.
 define void @test3() {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    call void @bar(i8* getelementptr inbounds (%T, %T* @G, i64 0, i32 0)) #2
+; CHECK-NEXT:    ret void
+;
   %A = alloca %T
   %a = bitcast %T* %A to i8*
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %a, i8* align 4 bitcast (%T* @G to i8*), i64 124, i1 false)
   call void @bar(i8* %a) readonly
-; CHECK-LABEL: @test3(
-; CHECK-NEXT: call void @bar(i8* getelementptr inbounds (%T, %T* @G, i64 0, i32 0))
   ret void
 }
 
 define void @test3_addrspacecast() {
+; CHECK-LABEL: @test3_addrspacecast(
+; CHECK-NEXT:    call void @bar(i8* getelementptr inbounds (%T, %T* @G, i64 0, i32 0)) #2
+; CHECK-NEXT:    ret void
+;
   %A = alloca %T
   %a = bitcast %T* %A to i8*
   call void @llvm.memcpy.p0i8.p1i8.i64(i8* align 4 %a, i8 addrspace(1)* align 4 addrspacecast (%T* @G to i8 addrspace(1)*), i64 124, i1 false)
   call void @bar(i8* %a) readonly
-; CHECK-LABEL: @test3_addrspacecast(
-; CHECK-NEXT: call void @bar(i8* getelementptr inbounds (%T, %T* @G, i64 0, i32 0))
   ret void
 }
 
 
 define void @test4() {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    call void @baz(i8* byval getelementptr inbounds (%T, %T* @G, i64 0, i32 0))
+; CHECK-NEXT:    ret void
+;
   %A = alloca %T
   %a = bitcast %T* %A to i8*
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %a, i8* align 4 bitcast (%T* @G to i8*), i64 124, i1 false)
   call void @baz(i8* byval %a)
-; CHECK-LABEL: @test4(
-; CHECK-NEXT: call void @baz(i8* byval getelementptr inbounds (%T, %T* @G, i64 0, i32 0))
   ret void
 }
 
 declare void @llvm.lifetime.start.p0i8(i64, i8*)
 define void @test5() {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    call void @baz(i8* byval getelementptr inbounds (%T, %T* @G, i64 0, i32 0))
+; CHECK-NEXT:    ret void
+;
   %A = alloca %T
   %a = bitcast %T* %A to i8*
   call void @llvm.lifetime.start.p0i8(i64 -1, i8* %a)
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %a, i8* align 4 bitcast (%T* @G to i8*), i64 124, i1 false)
   call void @baz(i8* byval %a)
-; CHECK-LABEL: @test5(
-; CHECK-NEXT: call void @baz(i8* byval getelementptr inbounds (%T, %T* @G, i64 0, i32 0))
   ret void
 }
 
@@ -163,65 +207,81 @@ declare void @baz(i8* byval)
 
 
 define void @test6() {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:    call void @bar(i8* bitcast ([2 x %U]* @H to i8*)) #2
+; CHECK-NEXT:    ret void
+;
   %A = alloca %U, align 16
   %a = bitcast %U* %A to i8*
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 16 %a, i8* align 16 bitcast ([2 x %U]* @H to i8*), i64 20, i1 false)
   call void @bar(i8* %a) readonly
-; CHECK-LABEL: @test6(
-; CHECK-NEXT: call void @bar(i8* bitcast ([2 x %U]* @H to i8*))
   ret void
 }
 
 define void @test7() {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:    call void @bar(i8* bitcast ([2 x %U]* @H to i8*)) #2
+; CHECK-NEXT:    ret void
+;
   %A = alloca %U, align 16
   %a = bitcast %U* %A to i8*
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %a, i8* align 4 bitcast (%U* getelementptr ([2 x %U], [2 x %U]* @H, i64 0, i32 0) to i8*), i64 20, i1 false)
   call void @bar(i8* %a) readonly
-; CHECK-LABEL: @test7(
-; CHECK-NEXT: call void @bar(i8* bitcast ([2 x %U]* @H to i8*))
   ret void
 }
 
 define void @test8() {
-  %A = alloca %U, align 16
-  %a = bitcast %U* %A to i8*
+; CHECK-LABEL: @test8(
+; CHECK-NEXT:    [[AL:%.*]] = alloca [[U:%.*]], align 16
+; CHECK-NEXT:    [[A:%.*]] = bitcast %U* [[AL]] to i8*
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 16 dereferenceable(20) [[A]], i8* nonnull align 4 dereferenceable(20) bitcast (%U* getelementptr inbounds ([2 x %U], [2 x %U]* @H, i64 0, i64 1) to i8*), i64 20, i1 false)
+; CHECK-NEXT:    call void @bar(i8* nonnull [[A]]) #2
+; CHECK-NEXT:    ret void
+;
+  %al = alloca %U, align 16
+  %a = bitcast %U* %al to i8*
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %a, i8* align 4 bitcast (%U* getelementptr ([2 x %U], [2 x %U]* @H, i64 0, i32 1) to i8*), i64 20, i1 false)
   call void @bar(i8* %a) readonly
-; CHECK-LABEL: @test8(
-; CHECK: llvm.memcpy
-; CHECK: bar
   ret void
 }
 
 
 define void @test8_addrspacecast() {
-  %A = alloca %U, align 16
-  %a = bitcast %U* %A to i8*
+; CHECK-LABEL: @test8_addrspacecast(
+; CHECK-NEXT:    [[AL:%.*]] = alloca [[U:%.*]], align 16
+; CHECK-NEXT:    [[A:%.*]] = bitcast %U* [[AL]] to i8*
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p1i8.i64(i8* nonnull align 16 dereferenceable(20) [[A]], i8 addrspace(1)* align 4 dereferenceable(20) addrspacecast (i8* bitcast (%U* getelementptr inbounds ([2 x %U], [2 x %U]* @H, i64 0, i64 1) to i8*) to i8 addrspace(1)*), i64 20, i1 false)
+; CHECK-NEXT:    call void @bar(i8* nonnull [[A]]) #2
+; CHECK-NEXT:    ret void
+;
+  %Al = alloca %U, align 16
+  %a = bitcast %U* %Al to i8*
   call void @llvm.memcpy.p0i8.p1i8.i64(i8* align 4 %a, i8 addrspace(1)* align 4 addrspacecast (%U* getelementptr ([2 x %U], [2 x %U]* @H, i64 0, i32 1) to i8 addrspace(1)*), i64 20, i1 false)
   call void @bar(i8* %a) readonly
-; CHECK-LABEL: @test8_addrspacecast(
-; CHECK: llvm.memcpy
-; CHECK: bar
   ret void
 }
 
 define void @test9() {
+; CHECK-LABEL: @test9(
+; CHECK-NEXT:    call void @bar(i8* bitcast (%U* getelementptr inbounds ([2 x %U], [2 x %U]* @H, i64 0, i64 1) to i8*)) #2
+; CHECK-NEXT:    ret void
+;
   %A = alloca %U, align 4
   %a = bitcast %U* %A to i8*
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %a, i8* align 4 bitcast (%U* getelementptr ([2 x %U], [2 x %U]* @H, i64 0, i32 1) to i8*), i64 20, i1 false)
   call void @bar(i8* %a) readonly
-; CHECK-LABEL: @test9(
-; CHECK-NEXT: call void @bar(i8* bitcast (%U* getelementptr inbounds ([2 x %U], [2 x %U]* @H, i64 0, i64 1) to i8*))
   ret void
 }
 
 define void @test9_addrspacecast() {
+; CHECK-LABEL: @test9_addrspacecast(
+; CHECK-NEXT:    call void @bar(i8* bitcast (%U* getelementptr inbounds ([2 x %U], [2 x %U]* @H, i64 0, i64 1) to i8*)) #2
+; CHECK-NEXT:    ret void
+;
   %A = alloca %U, align 4
   %a = bitcast %U* %A to i8*
   call void @llvm.memcpy.p0i8.p1i8.i64(i8* align 4 %a, i8 addrspace(1)* align 4 addrspacecast (%U* getelementptr ([2 x %U], [2 x %U]* @H, i64 0, i32 1) to i8 addrspace(1)*), i64 20, i1 false)
   call void @bar(i8* %a) readonly
-; CHECK-LABEL: @test9_addrspacecast(
-; CHECK-NEXT: call void @bar(i8* bitcast (%U* getelementptr inbounds ([2 x %U], [2 x %U]* @H, i64 0, i64 1) to i8*))
   ret void
 }
 
@@ -231,8 +291,13 @@ define void @test9_addrspacecast() {
 ; Should not replace alloca with global because of size mismatch.
 define void @test9_small_global() {
 ; CHECK-LABEL: @test9_small_global(
-; CHECK-NOT: call void @llvm.memcpy.p0i8.p0i8.i64({{.*}}@bbb,{{.*}}@_ZL3KKK, 
-; CHECK: alloca [1000000 x i8]
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CC:%.*]] = alloca [1000000 x i8], align 16
+; CHECK-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [1000000 x i8], [1000000 x i8]* [[CC]], i64 0, i64 0
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 16 dereferenceable(3) [[ARRAYDECAY]], i8* nonnull align 16 dereferenceable(3) getelementptr inbounds ([3 x i8], [3 x i8]* @_ZL3KKK, i64 0, i64 0), i64 3, i1 false)
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 16 dereferenceable(1000000) getelementptr inbounds ([1000000 x i8], [1000000 x i8]* @bbb, i64 0, i64 0), i8* nonnull align 16 dereferenceable(1000000) [[ARRAYDECAY]], i64 1000000, i1 false)
+; CHECK-NEXT:    ret void
+;
 entry:
   %cc = alloca [1000000 x i8], align 16
   %cc.0..sroa_idx = getelementptr inbounds [1000000 x i8], [1000000 x i8]* %cc, i64 0, i64 0
@@ -245,8 +310,10 @@ entry:
 ; Should replace alloca with global as they have exactly the same size.
 define void @test10_same_global() {
 ; CHECK-LABEL: @test10_same_global(
-; CHECK-NOT: alloca
-; CHECK: call void @llvm.memcpy.p0i8.p0i8.i64({{.*}}@bbb,{{.*}}@_ZL3KKK,{{.*}}, i64 3,
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 16 dereferenceable(3) getelementptr inbounds ([1000000 x i8], [1000000 x i8]* @bbb, i64 0, i64 0), i8* nonnull align 16 dereferenceable(3) getelementptr inbounds ([3 x i8], [3 x i8]* @_ZL3KKK, i64 0, i64 0), i64 3, i1 false)
+; CHECK-NEXT:    ret void
+;
 entry:
   %cc = alloca [3 x i8], align 1
   %cc.0..sroa_idx = getelementptr inbounds [3 x i8], [3 x i8]* %cc, i64 0, i64 0
diff --git a/llvm/test/Transforms/InstCombine/memcpy-to-load.ll b/llvm/test/Transforms/InstCombine/memcpy-to-load.ll
index 614ae1864..2f9d69268 100644
--- a/llvm/test/Transforms/InstCombine/memcpy-to-load.ll
+++ b/llvm/test/Transforms/InstCombine/memcpy-to-load.ll
@@ -34,7 +34,7 @@ define void @copy_2_bytes(i8* %d, i8* %s) {
 
 define void @copy_3_bytes(i8* %d, i8* %s) {
 ; ALL-LABEL: @copy_3_bytes(
-; ALL-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 [[D:%.*]], i8* align 1 [[S:%.*]], i32 3, i1 false)
+; ALL-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* nonnull align 1 dereferenceable(3) [[D:%.*]], i8* nonnull align 1 dereferenceable(3) [[S:%.*]], i32 3, i1 false)
 ; ALL-NEXT:    ret void
 ;
   call void @llvm.memcpy.p0i8.p0i8.i32(i8* %d, i8* %s, i32 3, i1 false)
@@ -57,7 +57,7 @@ define void @copy_4_bytes(i8* %d, i8* %s) {
 
 define void @copy_5_bytes(i8* %d, i8* %s) {
 ; ALL-LABEL: @copy_5_bytes(
-; ALL-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 [[D:%.*]], i8* align 1 [[S:%.*]], i32 5, i1 false)
+; ALL-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* nonnull align 1 dereferenceable(5) [[D:%.*]], i8* nonnull align 1 dereferenceable(5) [[S:%.*]], i32 5, i1 false)
 ; ALL-NEXT:    ret void
 ;
   call void @llvm.memcpy.p0i8.p0i8.i32(i8* %d, i8* %s, i32 5, i1 false)
@@ -78,7 +78,7 @@ define void @copy_8_bytes(i8* %d, i8* %s) {
 
 define void @copy_16_bytes(i8* %d, i8* %s) {
 ; ALL-LABEL: @copy_16_bytes(
-; ALL-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 [[D:%.*]], i8* align 1 [[S:%.*]], i32 16, i1 false)
+; ALL-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* nonnull align 1 dereferenceable(16) [[D:%.*]], i8* nonnull align 1 dereferenceable(16) [[S:%.*]], i32 16, i1 false)
 ; ALL-NEXT:    ret void
 ;
   call void @llvm.memcpy.p0i8.p0i8.i32(i8* %d, i8* %s, i32 16, i1 false)
diff --git a/llvm/test/Transforms/InstCombine/memcpy.ll b/llvm/test/Transforms/InstCombine/memcpy.ll
index ae8f96c90..4f52cb18e 100644
--- a/llvm/test/Transforms/InstCombine/memcpy.ll
+++ b/llvm/test/Transforms/InstCombine/memcpy.ll
@@ -29,7 +29,7 @@ define void @test2(i8* %a) {
 
 define void @test3(i8* %d, i8* %s) {
 ; CHECK-LABEL: @test3(
-; CHECK-NEXT:    tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 [[D:%.*]], i8* align 4 [[S:%.*]], i64 17179869184, i1 false)
+; CHECK-NEXT:    tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 4 dereferenceable(17179869184) [[D:%.*]], i8* nonnull align 4 dereferenceable(17179869184) [[S:%.*]], i64 17179869184, i1 false)
 ; CHECK-NEXT:    ret void
 ;
   tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %d, i8* align 4 %s, i64 17179869184, i1 false)
diff --git a/llvm/test/Transforms/InstCombine/memcpy_chk-1.ll b/llvm/test/Transforms/InstCombine/memcpy_chk-1.ll
index a372ef2fe..c76e4f61f 100644
--- a/llvm/test/Transforms/InstCombine/memcpy_chk-1.ll
+++ b/llvm/test/Transforms/InstCombine/memcpy_chk-1.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; Test lib call simplification of __memcpy_chk calls with various values
 ; for dstlen and len.
 ;
@@ -17,22 +18,22 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3
 
 define i8* @test_simplify1() {
 ; CHECK-LABEL: @test_simplify1(
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 4 dereferenceable(1824) bitcast (%struct.T1* @t1 to i8*), i8* nonnull align 4 dereferenceable(1824) bitcast (%struct.T2* @t2 to i8*), i64 1824, i1 false)
+; CHECK-NEXT:    ret i8* bitcast (%struct.T1* @t1 to i8*)
+;
   %dst = bitcast %struct.T1* @t1 to i8*
   %src = bitcast %struct.T2* @t2 to i8*
-
-; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 bitcast (%struct.T1* @t1 to i8*), i8* align 4 bitcast (%struct.T2* @t2 to i8*), i64 1824, i1 false)
-; CHECK-NEXT: ret i8* bitcast (%struct.T1* @t1 to i8*)
   %ret = call i8* @__memcpy_chk(i8* %dst, i8* %src, i64 1824, i64 1824)
   ret i8* %ret
 }
 
 define i8* @test_simplify2() {
 ; CHECK-LABEL: @test_simplify2(
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 4 dereferenceable(1824) bitcast (%struct.T1* @t1 to i8*), i8* nonnull align 4 dereferenceable(1824) bitcast (%struct.T3* @t3 to i8*), i64 1824, i1 false)
+; CHECK-NEXT:    ret i8* bitcast (%struct.T1* @t1 to i8*)
+;
   %dst = bitcast %struct.T1* @t1 to i8*
   %src = bitcast %struct.T3* @t3 to i8*
-
-; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 bitcast (%struct.T1* @t1 to i8*), i8* align 4 bitcast (%struct.T3* @t3 to i8*), i64 1824, i1 false)
-; CHECK-NEXT: ret i8* bitcast (%struct.T1* @t1 to i8*)
   %ret = call i8* @__memcpy_chk(i8* %dst, i8* %src, i64 1824, i64 2848)
   ret i8* %ret
 }
@@ -41,36 +42,35 @@ define i8* @test_simplify2() {
 
 define i8* @test_no_simplify1() {
 ; CHECK-LABEL: @test_no_simplify1(
+; CHECK-NEXT:    [[RET:%.*]] = call i8* @__memcpy_chk(i8* bitcast (%struct.T3* @t3 to i8*), i8* bitcast (%struct.T1* @t1 to i8*), i64 2848, i64 1824)
+; CHECK-NEXT:    ret i8* [[RET]]
+;
   %dst = bitcast %struct.T3* @t3 to i8*
   %src = bitcast %struct.T1* @t1 to i8*
-
-; CHECK-NEXT: %ret = call i8* @__memcpy_chk(i8* bitcast (%struct.T3* @t3 to i8*), i8* bitcast (%struct.T1* @t1 to i8*), i64 2848, i64 1824)
-; CHECK-NEXT: ret i8* %ret
   %ret = call i8* @__memcpy_chk(i8* %dst, i8* %src, i64 2848, i64 1824)
   ret i8* %ret
 }
 
 define i8* @test_no_simplify2() {
 ; CHECK-LABEL: @test_no_simplify2(
+; CHECK-NEXT:    [[RET:%.*]] = call i8* @__memcpy_chk(i8* bitcast (%struct.T1* @t1 to i8*), i8* bitcast (%struct.T2* @t2 to i8*), i64 1024, i64 0)
+; CHECK-NEXT:    ret i8* [[RET]]
+;
   %dst = bitcast %struct.T1* @t1 to i8*
   %src = bitcast %struct.T2* @t2 to i8*
-
-; CHECK-NEXT: %ret = call i8* @__memcpy_chk(i8* bitcast (%struct.T1* @t1 to i8*), i8* bitcast (%struct.T2* @t2 to i8*), i64 1024, i64 0)
-; CHECK-NEXT: ret i8* %ret
   %ret = call i8* @__memcpy_chk(i8* %dst, i8* %src, i64 1024, i64 0)
   ret i8* %ret
 }
 
 define i8* @test_simplify_return_indcall(i8* ()* %alloc) {
 ; CHECK-LABEL: @test_simplify_return_indcall(
+; CHECK-NEXT:    [[DST:%.*]] = call i8* [[ALLOC:%.*]]()
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 1 dereferenceable(1824) [[DST]], i8* nonnull align 4 dereferenceable(1824) bitcast (%struct.T2* @t2 to i8*), i64 1824, i1 false)
+; CHECK-NEXT:    ret i8* [[DST]]
+;
   %src = bitcast %struct.T2* @t2 to i8*
-
-; CHECK-NEXT: %dst = call i8* %alloc()
   %dst = call i8* %alloc()
-
-; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i64
   %ret = call i8* @__memcpy_chk(i8* %dst, i8* %src, i64 1824, i64 1824)
-; CHECK-NEXT: ret i8* %dst
   ret i8* %ret
 }
 
diff --git a/llvm/test/Transforms/InstCombine/memmove_chk-1.ll b/llvm/test/Transforms/InstCombine/memmove_chk-1.ll
index f00698530..16aa9db81 100644
--- a/llvm/test/Transforms/InstCombine/memmove_chk-1.ll
+++ b/llvm/test/Transforms/InstCombine/memmove_chk-1.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; Test lib call simplification of __memmove_chk calls with various values
 ; for dstlen and len.
 ;
@@ -17,22 +18,24 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3
 
 define i8* @test_simplify1() {
 ; CHECK-LABEL: @test_simplify1(
+; CHECK-NEXT:    call void @llvm.memmove.p0i8.p0i8.i64(i8* nonnull align 4 dereferenceable(1824) bitcast (%struct.T1* @t1 to i8*), i8* nonnull align 4 dereferenceable(1824) bitcast (%struct.T2* @t2 to i8*), i64 1824, i1 false)
+; CHECK-NEXT:    ret i8* bitcast (%struct.T1* @t1 to i8*)
+;
   %dst = bitcast %struct.T1* @t1 to i8*
   %src = bitcast %struct.T2* @t2 to i8*
 
-; CHECK-NEXT: call void @llvm.memmove.p0i8.p0i8.i64(i8* align 4 bitcast (%struct.T1* @t1 to i8*), i8* align 4 bitcast (%struct.T2* @t2 to i8*), i64 1824, i1 false)
-; CHECK-NEXT: ret i8* bitcast (%struct.T1* @t1 to i8*)
   %ret = call i8* @__memmove_chk(i8* %dst, i8* %src, i64 1824, i64 1824)
   ret i8* %ret
 }
 
 define i8* @test_simplify2() {
 ; CHECK-LABEL: @test_simplify2(
+; CHECK-NEXT:    call void @llvm.memmove.p0i8.p0i8.i64(i8* nonnull align 4 dereferenceable(1824) bitcast (%struct.T1* @t1 to i8*), i8* nonnull align 4 dereferenceable(1824) bitcast (%struct.T3* @t3 to i8*), i64 1824, i1 false)
+; CHECK-NEXT:    ret i8* bitcast (%struct.T1* @t1 to i8*)
+;
   %dst = bitcast %struct.T1* @t1 to i8*
   %src = bitcast %struct.T3* @t3 to i8*
 
-; CHECK-NEXT: call void @llvm.memmove.p0i8.p0i8.i64(i8* align 4 bitcast (%struct.T1* @t1 to i8*), i8* align 4 bitcast (%struct.T3* @t3 to i8*), i64 1824, i1 false)
-; CHECK-NEXT: ret i8* bitcast (%struct.T1* @t1 to i8*)
   %ret = call i8* @__memmove_chk(i8* %dst, i8* %src, i64 1824, i64 2848)
   ret i8* %ret
 }
@@ -41,22 +44,24 @@ define i8* @test_simplify2() {
 
 define i8* @test_no_simplify1() {
 ; CHECK-LABEL: @test_no_simplify1(
+; CHECK-NEXT:    [[RET:%.*]] = call i8* @__memmove_chk(i8* bitcast (%struct.T3* @t3 to i8*), i8* bitcast (%struct.T1* @t1 to i8*), i64 2848, i64 1824)
+; CHECK-NEXT:    ret i8* [[RET]]
+;
   %dst = bitcast %struct.T3* @t3 to i8*
   %src = bitcast %struct.T1* @t1 to i8*
 
-; CHECK-NEXT: %ret = call i8* @__memmove_chk(i8* bitcast (%struct.T3* @t3 to i8*), i8* bitcast (%struct.T1* @t1 to i8*), i64 2848, i64 1824)
-; CHECK-NEXT: ret i8* %ret
   %ret = call i8* @__memmove_chk(i8* %dst, i8* %src, i64 2848, i64 1824)
   ret i8* %ret
 }
 
 define i8* @test_no_simplify2() {
 ; CHECK-LABEL: @test_no_simplify2(
+; CHECK-NEXT:    [[RET:%.*]] = call i8* @__memmove_chk(i8* bitcast (%struct.T1* @t1 to i8*), i8* bitcast (%struct.T2* @t2 to i8*), i64 1024, i64 0)
+; CHECK-NEXT:    ret i8* [[RET]]
+;
   %dst = bitcast %struct.T1* @t1 to i8*
   %src = bitcast %struct.T2* @t2 to i8*
 
-; CHECK-NEXT: %ret = call i8* @__memmove_chk(i8* bitcast (%struct.T1* @t1 to i8*), i8* bitcast (%struct.T2* @t2 to i8*), i64 1024, i64 0)
-; CHECK-NEXT: ret i8* %ret
   %ret = call i8* @__memmove_chk(i8* %dst, i8* %src, i64 1024, i64 0)
   ret i8* %ret
 }
diff --git a/llvm/test/Transforms/InstCombine/mempcpy.ll b/llvm/test/Transforms/InstCombine/mempcpy.ll
new file mode 100644
index 000000000..79158a3a0
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/mempcpy.ll
@@ -0,0 +1,56 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+define i8* @memcpy_nonconst_n(i8* %d, i8* nocapture readonly %s, i64 %n) {
+; CHECK-LABEL: @memcpy_nonconst_n(
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 1 [[D:%.*]], i8* align 1 [[S:%.*]], i64 [[N:%.*]], i1 false)
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, i8* [[D]], i64 [[N]]
+; CHECK-NEXT:    ret i8* [[TMP1]]
+;
+  %r = tail call i8* @mempcpy(i8* %d, i8* %s, i64 %n)
+  ret i8* %r
+}
+
+define i8* @memcpy_nonconst_n_copy_attrs(i8* %d, i8* nocapture readonly %s, i64 %n) {
+; CHECK-LABEL: @memcpy_nonconst_n_copy_attrs(
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 1 dereferenceable(16) [[D:%.*]], i8* align 1 [[S:%.*]], i64 [[N:%.*]], i1 false)
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, i8* [[D]], i64 [[N]]
+; CHECK-NEXT:    ret i8* [[TMP1]]
+;
+  %r = tail call i8* @mempcpy(i8* dereferenceable(16) %d, i8* %s, i64 %n)
+  ret i8* %r
+}
+
+define void @memcpy_nonconst_n_unused_retval(i8* %d, i8* nocapture readonly %s, i64 %n) {
+; CHECK-LABEL: @memcpy_nonconst_n_unused_retval(
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 1 [[D:%.*]], i8* align 1 [[S:%.*]], i64 [[N:%.*]], i1 false)
+; CHECK-NEXT:    ret void
+;
+  call i8* @mempcpy(i8* %d, i8* %s, i64 %n)
+  ret void
+}
+
+define i8* @memcpy_small_const_n(i8* %d, i8* nocapture readonly %s) {
+; CHECK-LABEL: @memcpy_small_const_n(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8* [[S:%.*]] to i64*
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8* [[D:%.*]] to i64*
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, i64* [[TMP1]], align 1
+; CHECK-NEXT:    store i64 [[TMP3]], i64* [[TMP2]], align 1
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, i8* [[D]], i64 8
+; CHECK-NEXT:    ret i8* [[TMP4]]
+;
+  %r = tail call i8* @mempcpy(i8* %d, i8* %s, i64 8)
+  ret i8* %r
+}
+
+define i8* @memcpy_big_const_n(i8* %d, i8* nocapture readonly %s) {
+; CHECK-LABEL: @memcpy_big_const_n(
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 1 dereferenceable(1024) [[D:%.*]], i8* nonnull align 1 dereferenceable(1024) [[S:%.*]], i64 1024, i1 false)
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, i8* [[D]], i64 1024
+; CHECK-NEXT:    ret i8* [[TMP1]]
+;
+  %r = tail call i8* @mempcpy(i8* %d, i8* %s, i64 1024)
+  ret i8* %r
+}
+
+declare i8* @mempcpy(i8*, i8* nocapture readonly, i64)
diff --git a/llvm/test/Transforms/InstCombine/memrchr.ll b/llvm/test/Transforms/InstCombine/memrchr.ll
new file mode 100644
index 000000000..feb384702
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/memrchr.ll
@@ -0,0 +1,55 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+
+declare i8* @memrchr(i8*, i32, i32)
+
+define i8* @test1(i8* %str, i32 %c, i32 %n) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[RET:%.*]] = call i8* @memrchr(i8* [[STR:%.*]], i32 [[C:%.*]], i32 [[N:%.*]])
+; CHECK-NEXT:    ret i8* [[RET]]
+;
+
+  %ret = call i8* @memrchr(i8* %str, i32 %c, i32 %n)
+  ret i8* %ret
+}
+
+define i8* @test2(i8* %str, i32 %c, i32 %n) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[RET:%.*]] = call i8* @memrchr(i8* nonnull [[STR:%.*]], i32 [[C:%.*]], i32 [[N:%.*]])
+; CHECK-NEXT:    ret i8* [[RET]]
+;
+
+  %ret = call i8* @memrchr(i8* nonnull %str, i32 %c, i32 %n)
+  ret i8* %ret
+}
+
+define i8* @test3(i8* %str, i32 %c) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    [[RET:%.*]] = call i8* @memrchr(i8* [[STR:%.*]], i32 [[C:%.*]], i32 5)
+; CHECK-NEXT:    ret i8* [[RET]]
+;
+
+  %ret = call i8* @memrchr(i8* %str, i32 %c, i32 5)
+  ret i8* %ret
+}
+
+define i8* @test4(i8* %str, i32 %c) "null-pointer-is-valid"="true" {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    [[RET:%.*]] = call i8* @memrchr(i8* [[STR:%.*]], i32 [[C:%.*]], i32 5)
+; CHECK-NEXT:    ret i8* [[RET]]
+;
+
+  %ret = call i8* @memrchr(i8* %str, i32 %c, i32 5)
+  ret i8* %ret
+}
+
+define i8* @test5(i8* %str, i32 %c) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    [[RET:%.*]] = call i8* @memrchr(i8* [[STR:%.*]], i32 [[C:%.*]], i32 0)
+; CHECK-NEXT:    ret i8* [[RET]]
+;
+
+  %ret = call i8* @memrchr(i8* %str, i32 %c, i32 0)
+  ret i8* %ret
+}
diff --git a/llvm/test/Transforms/InstCombine/memset-1.ll b/llvm/test/Transforms/InstCombine/memset-1.ll
index 7b6341d34..9ce8795c6 100644
--- a/llvm/test/Transforms/InstCombine/memset-1.ll
+++ b/llvm/test/Transforms/InstCombine/memset-1.ll
@@ -49,7 +49,7 @@ define i8* @malloc_and_memset_intrinsic(i32 %n) #0 {
 define i8* @notmalloc_memset(i32 %size, i8*(i32)* %notmalloc) {
 ; CHECK-LABEL: @notmalloc_memset(
 ; CHECK-NEXT:    [[CALL1:%.*]] = call i8* [[NOTMALLOC:%.*]](i32 [[SIZE:%.*]]) #0
-; CHECK-NEXT:    call void @llvm.memset.p0i8.i32(i8* align 1 [[CALL1]], i8 0, i32 [[SIZE]], i1 false)
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i32(i8* align 1 [[CALL1]], i8 0, i32 [[SIZE]], i1 false) #0
 ; CHECK-NEXT:    ret i8* [[CALL1]]
 ;
   %call1 = call i8* %notmalloc(i32 %size) #1
@@ -68,7 +68,7 @@ define float* @pr25892(i32 %size) #0 {
 ; CHECK-NEXT:    br i1 [[CMP]], label [[CLEANUP:%.*]], label [[IF_END:%.*]]
 ; CHECK:       if.end:
 ; CHECK-NEXT:    [[BC:%.*]] = bitcast i8* [[CALL]] to float*
-; CHECK-NEXT:    call void @llvm.memset.p0i8.i32(i8* nonnull align 1 [[CALL]], i8 0, i32 [[SIZE]], i1 false)
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i32(i8* nonnull align 1 [[CALL]], i8 0, i32 [[SIZE]], i1 false) #0
 ; CHECK-NEXT:    br label [[CLEANUP]]
 ; CHECK:       cleanup:
 ; CHECK-NEXT:    [[RETVAL_0:%.*]] = phi float* [ [[BC]], [[IF_END]] ], [ null, [[ENTRY:%.*]] ]
@@ -93,7 +93,7 @@ define i8* @buffer_is_modified_then_memset(i32 %size) {
 ; CHECK-LABEL: @buffer_is_modified_then_memset(
 ; CHECK-NEXT:    [[PTR:%.*]] = tail call i8* @malloc(i32 [[SIZE:%.*]]) #0
 ; CHECK-NEXT:    store i8 1, i8* [[PTR]], align 1
-; CHECK-NEXT:    call void @llvm.memset.p0i8.i32(i8* align 1 [[PTR]], i8 0, i32 [[SIZE]], i1 false)
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i32(i8* nonnull align 1 [[PTR]], i8 0, i32 [[SIZE]], i1 false) #0
 ; CHECK-NEXT:    ret i8* [[PTR]]
 ;
   %ptr = tail call i8* @malloc(i32 %size) #1
@@ -102,6 +102,103 @@ define i8* @buffer_is_modified_then_memset(i32 %size) {
   ret i8* %memset
 }
 
+define i8* @memset_size_select(i1 %b, i8* %ptr) {
+; CHECK-LABEL: @memset_size_select(
+; CHECK-NEXT:    [[SIZE:%.*]] = select i1 [[B:%.*]], i32 10, i32 50
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i32(i8* nonnull align 1 dereferenceable(10) [[PTR:%.*]], i8 0, i32 [[SIZE]], i1 false) #0
+; CHECK-NEXT:    ret i8* [[PTR]]
+;
+  %size = select i1 %b, i32 10, i32 50
+  %memset = tail call i8* @memset(i8* nonnull %ptr, i32 0, i32 %size) #1
+  ret i8* %memset
+}
+
+
+define i8* @memset_size_select2(i1 %b, i8* %ptr) {
+; CHECK-LABEL: @memset_size_select2(
+; CHECK-NEXT:    [[SIZE:%.*]] = select i1 [[B:%.*]], i32 10, i32 50
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i32(i8* nonnull align 1 dereferenceable(80) [[PTR:%.*]], i8 0, i32 [[SIZE]], i1 false) #0
+; CHECK-NEXT:    ret i8* [[PTR]]
+;
+  %size = select i1 %b, i32 10, i32 50
+  %memset = tail call i8* @memset(i8* nonnull dereferenceable(80) %ptr, i32 0, i32 %size) #1
+  ret i8* %memset
+}
+
+define i8* @memset_size_select3(i1 %b, i8* %ptr) {
+; CHECK-LABEL: @memset_size_select3(
+; CHECK-NEXT:    [[SIZE:%.*]] = select i1 [[B:%.*]], i32 10, i32 50
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i32(i8* nonnull align 1 dereferenceable(40) [[PTR:%.*]], i8 0, i32 [[SIZE]], i1 false)
+; CHECK-NEXT:    ret i8* [[PTR]]
+;
+  %size = select i1 %b, i32 10, i32 50
+  %memset = tail call i8* @memset(i8* dereferenceable_or_null(40) %ptr, i32 0, i32 %size)
+  ret i8* %memset
+}
+
+define i8* @memset_size_select4(i1 %b, i8* %ptr) {
+; CHECK-LABEL: @memset_size_select4(
+; CHECK-NEXT:    [[SIZE:%.*]] = select i1 [[B:%.*]], i32 10, i32 50
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i32(i8* nonnull align 1 dereferenceable(40) [[PTR:%.*]], i8 0, i32 [[SIZE]], i1 false) #0
+; CHECK-NEXT:    ret i8* [[PTR]]
+;
+  %size = select i1 %b, i32 10, i32 50
+  %memset = tail call i8* @memset(i8* nonnull dereferenceable_or_null(40) %ptr, i32 0, i32 %size) #1
+  ret i8* %memset
+}
+
+define i8* @memset_size_ashr(i1 %b, i8* %ptr, i32 %v) {
+; CHECK-LABEL: @memset_size_ashr(
+; CHECK-NEXT:    [[SIZE:%.*]] = ashr i32 -2, [[V:%.*]]
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i32(i8* nonnull align 1 [[PTR:%.*]], i8 0, i32 [[SIZE]], i1 false) #0
+; CHECK-NEXT:    ret i8* [[PTR]]
+;
+  %size = ashr i32 -2, %v
+  %memset = tail call i8* @memset(i8* nonnull %ptr, i32 0, i32 %size) #1
+  ret i8* %memset
+}
+
+define i8* @memset_attrs1(i1 %b, i8* %ptr, i32 %size) {
+; CHECK-LABEL: @memset_attrs1(
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i32(i8* align 1 dereferenceable_or_null(40) [[PTR:%.*]], i8 0, i32 [[SIZE:%.*]], i1 false) #0
+; CHECK-NEXT:    ret i8* [[PTR]]
+;
+  %memset = tail call i8* @memset(i8* dereferenceable_or_null(40) %ptr, i32 0, i32 %size) #1
+  ret i8* %memset
+}
+
+; be sure to drop nonnull since size is unknown and can be 0
+; do not change dereferenceable attribute
+define i8* @memset_attrs2(i1 %b, i8* %ptr, i32 %size) {
+; CHECK-LABEL: @memset_attrs2(
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i32(i8* nonnull align 1 dereferenceable(40) [[PTR:%.*]], i8 0, i32 [[SIZE:%.*]], i1 false) #0
+; CHECK-NEXT:    ret i8* [[PTR]]
+;
+  %memset = tail call i8* @memset(i8* nonnull dereferenceable(40) %ptr, i32 0, i32 %size) #1
+  ret i8* %memset
+}
+
+; size is unknown, just copy attrs, no changes in attrs
+define i8* @memset_attrs3(i1 %b, i8* %ptr, i32 %size) {
+; CHECK-LABEL: @memset_attrs3(
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i32(i8* nonnull align 1 dereferenceable_or_null(40) [[PTR:%.*]], i8 0, i32 [[SIZE:%.*]], i1 false) #0
+; CHECK-NEXT:    ret i8* [[PTR]]
+;
+  %memset = tail call i8* @memset(i8* nonnull dereferenceable_or_null(40) %ptr, i32 0, i32 %size) #1
+  ret i8* %memset
+}
+
+; be sure to drop nonnull since size is unknown and can be 0
+define i8* @memset_attrs4(i1 %b, i8* %ptr, i32 %size) {
+; CHECK-LABEL: @memset_attrs4(
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i32(i8* nonnull align 1 [[PTR:%.*]], i8 0, i32 [[SIZE:%.*]], i1 false) #0
+; CHECK-NEXT:    ret i8* [[PTR]]
+;
+  %memset = tail call i8* @memset(i8* nonnull %ptr, i32 0, i32 %size) #1
+  ret i8* %memset
+}
+
+
 attributes #0 = { nounwind ssp uwtable }
 attributes #1 = { nounwind }
 attributes #2 = { nounwind readnone }
diff --git a/llvm/test/Transforms/InstCombine/memset.ll b/llvm/test/Transforms/InstCombine/memset.ll
index 7d531f296..b994d97c7 100644
--- a/llvm/test/Transforms/InstCombine/memset.ll
+++ b/llvm/test/Transforms/InstCombine/memset.ll
@@ -3,7 +3,7 @@
 
 define i32 @test([1024 x i8]* %target) {
 ; CHECK-LABEL: @test(
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds [1024 x i8], [1024 x i8]* [[TARGET:%.*]], i64 0, i64 0
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr [1024 x i8], [1024 x i8]* [[TARGET:%.*]], i64 0, i64 0
 ; CHECK-NEXT:    store i8 1, i8* [[TMP1]], align 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast [1024 x i8]* [[TARGET]] to i16*
 ; CHECK-NEXT:    store i16 257, i16* [[TMP2]], align 2
diff --git a/llvm/test/Transforms/InstCombine/memset_chk-1.ll b/llvm/test/Transforms/InstCombine/memset_chk-1.ll
index 71f95b024..ad45fb862 100644
--- a/llvm/test/Transforms/InstCombine/memset_chk-1.ll
+++ b/llvm/test/Transforms/InstCombine/memset_chk-1.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; Test lib call simplification of __memset_chk calls with various values
 ; for dstlen and len.
 ;
@@ -13,30 +14,33 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3
 
 define i8* @test_simplify1() {
 ; CHECK-LABEL: @test_simplify1(
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* nonnull align 4 dereferenceable(1824) bitcast (%struct.T* @t to i8*), i8 0, i64 1824, i1 false)
+; CHECK-NEXT:    ret i8* bitcast (%struct.T* @t to i8*)
+;
   %dst = bitcast %struct.T* @t to i8*
 
-; CHECK-NEXT: call void @llvm.memset.p0i8.i64(i8* align 4 bitcast (%struct.T* @t to i8*), i8 0, i64 1824, i1 false)
-; CHECK-NEXT: ret i8* bitcast (%struct.T* @t to i8*)
   %ret = call i8* @__memset_chk(i8* %dst, i32 0, i64 1824, i64 1824)
   ret i8* %ret
 }
 
 define i8* @test_simplify2() {
 ; CHECK-LABEL: @test_simplify2(
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* nonnull align 4 dereferenceable(1824) bitcast (%struct.T* @t to i8*), i8 0, i64 1824, i1 false)
+; CHECK-NEXT:    ret i8* bitcast (%struct.T* @t to i8*)
+;
   %dst = bitcast %struct.T* @t to i8*
 
-; CHECK-NEXT: call void @llvm.memset.p0i8.i64(i8* align 4 bitcast (%struct.T* @t to i8*), i8 0, i64 1824, i1 false)
-; CHECK-NEXT: ret i8* bitcast (%struct.T* @t to i8*)
   %ret = call i8* @__memset_chk(i8* %dst, i32 0, i64 1824, i64 3648)
   ret i8* %ret
 }
 
 define i8* @test_simplify3() {
 ; CHECK-LABEL: @test_simplify3(
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* nonnull align 4 dereferenceable(1824) bitcast (%struct.T* @t to i8*), i8 0, i64 1824, i1 false)
+; CHECK-NEXT:    ret i8* bitcast (%struct.T* @t to i8*)
+;
   %dst = bitcast %struct.T* @t to i8*
 
-; CHECK-NEXT: call void @llvm.memset.p0i8.i64(i8* align 4 bitcast (%struct.T* @t to i8*), i8 0, i64 1824, i1 false)
-; CHECK-NEXT: ret i8* bitcast (%struct.T* @t to i8*)
   %ret = call i8* @__memset_chk(i8* %dst, i32 0, i64 1824, i64 -1)
   ret i8* %ret
 }
@@ -45,41 +49,55 @@ define i8* @test_simplify3() {
 
 define i8* @test_no_simplify1() {
 ; CHECK-LABEL: @test_no_simplify1(
+; CHECK-NEXT:    [[RET:%.*]] = call i8* @__memset_chk(i8* bitcast (%struct.T* @t to i8*), i32 0, i64 1824, i64 400)
+; CHECK-NEXT:    ret i8* [[RET]]
+;
   %dst = bitcast %struct.T* @t to i8*
 
-; CHECK-NEXT: %ret = call i8* @__memset_chk(i8* bitcast (%struct.T* @t to i8*), i32 0, i64 1824, i64 400)
-; CHECK-NEXT: ret i8* %ret
   %ret = call i8* @__memset_chk(i8* %dst, i32 0, i64 1824, i64 400)
   ret i8* %ret
 }
 
 define i8* @test_no_simplify2() {
 ; CHECK-LABEL: @test_no_simplify2(
+; CHECK-NEXT:    [[RET:%.*]] = call i8* @__memset_chk(i8* bitcast (%struct.T* @t to i8*), i32 0, i64 1824, i64 0)
+; CHECK-NEXT:    ret i8* [[RET]]
+;
   %dst = bitcast %struct.T* @t to i8*
 
-; CHECK-NEXT: %ret = call i8* @__memset_chk(i8* bitcast (%struct.T* @t to i8*), i32 0, i64 1824, i64 0)
-; CHECK-NEXT: ret i8* %ret
   %ret = call i8* @__memset_chk(i8* %dst, i32 0, i64 1824, i64 0)
   ret i8* %ret
 }
 
 ; Test that RAUW in SimplifyLibCalls for __memset_chk generates valid IR
 define i32 @test_rauw(i8* %a, i8* %b, i8** %c) {
-; CHECK-LABEL: test_rauw
+; CHECK-LABEL: @test_rauw(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CALL49:%.*]] = call i64 @strlen(i8* nonnull dereferenceable(1) [[A:%.*]])
+; CHECK-NEXT:    [[ADD180:%.*]] = add i64 [[CALL49]], 1
+; CHECK-NEXT:    [[YO107:%.*]] = call i64 @llvm.objectsize.i64.p0i8(i8* [[B:%.*]], i1 false, i1 false, i1 false)
+; CHECK-NEXT:    [[CALL50:%.*]] = call i8* @__memmove_chk(i8* [[B]], i8* [[A]], i64 [[ADD180]], i64 [[YO107]])
+; CHECK-NEXT:    [[STRLEN:%.*]] = call i64 @strlen(i8* nonnull dereferenceable(1) [[B]])
+; CHECK-NEXT:    [[STRCHR2:%.*]] = getelementptr i8, i8* [[B]], i64 [[STRLEN]]
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i8** [[C:%.*]] to i64*
+; CHECK-NEXT:    [[D1:%.*]] = load i64, i64* [[TMP0]], align 8
+; CHECK-NEXT:    [[SUB183:%.*]] = ptrtoint i8* [[B]] to i64
+; CHECK-NEXT:    [[SUB184:%.*]] = sub i64 [[D1]], [[SUB183]]
+; CHECK-NEXT:    [[ADD52_I_I:%.*]] = add nsw i64 [[SUB184]], 1
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* align 1 [[STRCHR2]], i8 0, i64 [[ADD52_I_I]], i1 false)
+; CHECK-NEXT:    ret i32 4
+;
 entry:
   %call49 = call i64 @strlen(i8* %a)
   %add180 = add i64 %call49, 1
   %yo107 = call i64 @llvm.objectsize.i64.p0i8(i8* %b, i1 false, i1 false, i1 false)
   %call50 = call i8* @__memmove_chk(i8* %b, i8* %a, i64 %add180, i64 %yo107)
-; CHECK: %strlen = call i64 @strlen(i8* %b)
-; CHECK-NEXT: %strchr2 = getelementptr i8, i8* %b, i64 %strlen
   %call51i = call i8* @strrchr(i8* %b, i32 0)
   %d = load i8*, i8** %c, align 8
   %sub182 = ptrtoint i8* %d to i64
   %sub183 = ptrtoint i8* %b to i64
   %sub184 = sub i64 %sub182, %sub183
   %add52.i.i = add nsw i64 %sub184, 1
-; CHECK: call void @llvm.memset.p0i8.i64(i8* align 1 %strchr2
   %call185 = call i8* @__memset_chk(i8* %call51i, i32 0, i64 %add52.i.i, i64 -1)
   ret i32 4
 }
@@ -94,6 +112,20 @@ declare i8* @__memset_chk(i8*, i32, i64, i64)
 ; FIXME: memset(malloc(x), 0, x) -> calloc(1, x)
 
 define float* @pr25892(i64 %size) #0 {
+; CHECK-LABEL: @pr25892(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CALL:%.*]] = tail call i8* @malloc(i64 [[SIZE:%.*]]) #3
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8* [[CALL]], null
+; CHECK-NEXT:    br i1 [[CMP]], label [[CLEANUP:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.end:
+; CHECK-NEXT:    [[BC:%.*]] = bitcast i8* [[CALL]] to float*
+; CHECK-NEXT:    [[CALL2:%.*]] = tail call i64 @llvm.objectsize.i64.p0i8(i8* nonnull [[CALL]], i1 false, i1 false, i1 false)
+; CHECK-NEXT:    [[CALL3:%.*]] = tail call i8* @__memset_chk(i8* nonnull [[CALL]], i32 0, i64 [[SIZE]], i64 [[CALL2]]) #3
+; CHECK-NEXT:    br label [[CLEANUP]]
+; CHECK:       cleanup:
+; CHECK-NEXT:    [[RETVAL_0:%.*]] = phi float* [ [[BC]], [[IF_END]] ], [ null, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    ret float* [[RETVAL_0]]
+;
 entry:
   %call = tail call i8* @malloc(i64 %size) #1
   %cmp = icmp eq i8* %call, null
@@ -107,19 +139,6 @@ cleanup:
   %retval.0 = phi float* [ %bc, %if.end ], [ null, %entry ]
   ret float* %retval.0
 
-; CHECK-LABEL: @pr25892(
-; CHECK:       entry:
-; CHECK-NEXT:    %call = tail call i8* @malloc(i64 %size)
-; CHECK-NEXT:    %cmp = icmp eq i8* %call, null
-; CHECK-NEXT:    br i1 %cmp, label %cleanup, label %if.end
-; CHECK:       if.end:
-; CHECK-NEXT:    %bc = bitcast i8* %call to float*
-; CHECK-NEXT:    %call2 = tail call i64 @llvm.objectsize.i64.p0i8(i8* nonnull %call, i1 false, i1 false, i1 false)
-; CHECK-NEXT:    %call3 = tail call i8* @__memset_chk(i8* nonnull %call, i32 0, i64 %size, i64 %call2)
-; CHECK-NEXT:    br label %cleanup
-; CHECK:       cleanup:
-; CHECK-NEXT:    %retval.0 = phi float* [ %bc, %if.end ], [ null, %entry ]
-; CHECK-NEXT:    ret float* %retval.0
 }
 
 declare noalias i8* @malloc(i64) #1
diff --git a/llvm/test/Transforms/InstCombine/minmax-fold.ll b/llvm/test/Transforms/InstCombine/minmax-fold.ll
index 264e579db..75d177dbe 100644
--- a/llvm/test/Transforms/InstCombine/minmax-fold.ll
+++ b/llvm/test/Transforms/InstCombine/minmax-fold.ll
@@ -3,7 +3,7 @@
 
 ; This is the canonical form for a type-changing min/max.
 define i64 @t1(i32 %a) {
-; CHECK-LABEL: @t1(
+; CHECK-LABEL: define {{[^@]+}}@t1(
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[A:%.*]], 5
 ; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i32 [[A]], i32 5
 ; CHECK-NEXT:    [[TMP3:%.*]] = sext i32 [[TMP2]] to i64
@@ -17,7 +17,7 @@ define i64 @t1(i32 %a) {
 
 ; Check this is converted into canonical form, as above.
 define i64 @t2(i32 %a) {
-; CHECK-LABEL: @t2(
+; CHECK-LABEL: define {{[^@]+}}@t2(
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[A:%.*]], 5
 ; CHECK-NEXT:    [[NARROW:%.*]] = select i1 [[TMP1]], i32 [[A]], i32 5
 ; CHECK-NEXT:    [[TMP2:%.*]] = sext i32 [[NARROW]] to i64
@@ -31,7 +31,7 @@ define i64 @t2(i32 %a) {
 
 ; Same as @t2, with flipped operands and zext instead of sext.
 define i64 @t3(i32 %a) {
-; CHECK-LABEL: @t3(
+; CHECK-LABEL: define {{[^@]+}}@t3(
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i32 [[A:%.*]], 5
 ; CHECK-NEXT:    [[NARROW:%.*]] = select i1 [[TMP1]], i32 [[A]], i32 5
 ; CHECK-NEXT:    [[TMP2:%.*]] = zext i32 [[NARROW]] to i64
@@ -45,7 +45,7 @@ define i64 @t3(i32 %a) {
 
 ; Same again, with trunc.
 define i32 @t4(i64 %a) {
-; CHECK-LABEL: @t4(
+; CHECK-LABEL: define {{[^@]+}}@t4(
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i64 [[A:%.*]], 5
 ; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i64 [[A]], i64 5
 ; CHECK-NEXT:    [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
@@ -59,7 +59,7 @@ define i32 @t4(i64 %a) {
 
 ; Same as @t3, but with mismatched signedness between icmp and zext.
 define i64 @t5(i32 %a) {
-; CHECK-LABEL: @t5(
+; CHECK-LABEL: define {{[^@]+}}@t5(
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[A:%.*]], 5
 ; CHECK-NEXT:    [[NARROW:%.*]] = select i1 [[TMP1]], i32 [[A]], i32 5
 ; CHECK-NEXT:    [[TMP2:%.*]] = zext i32 [[NARROW]] to i64
@@ -72,7 +72,7 @@ define i64 @t5(i32 %a) {
 }
 
 define float @t6(i32 %a) {
-; CHECK-LABEL: @t6(
+; CHECK-LABEL: define {{[^@]+}}@t6(
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[A:%.*]], 0
 ; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i32 [[A]], i32 0
 ; CHECK-NEXT:    [[TMP3:%.*]] = sitofp i32 [[TMP2]] to float
@@ -85,7 +85,7 @@ define float @t6(i32 %a) {
 }
 
 define i16 @t7(i32 %a) {
-; CHECK-LABEL: @t7(
+; CHECK-LABEL: define {{[^@]+}}@t7(
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[A:%.*]], -32768
 ; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i32 [[A]], i32 -32768
 ; CHECK-NEXT:    [[TMP3:%.*]] = trunc i32 [[TMP2]] to i16
@@ -102,7 +102,7 @@ define i16 @t7(i32 %a) {
 ; which led to a canonicalization fight between different
 ; parts of instcombine.
 define i32 @t8(i64 %a, i32 %b) {
-; CHECK-LABEL: @t8(
+; CHECK-LABEL: define {{[^@]+}}@t8(
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i64 [[A:%.*]], -32767
 ; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i64 [[A]], i64 -32767
 ; CHECK-NEXT:    [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
@@ -124,7 +124,7 @@ define i32 @t8(i64 %a, i32 %b) {
 
 ; Ensure this doesn't get converted to a min/max.
 define i64 @t9(i32 %a) {
-; CHECK-LABEL: @t9(
+; CHECK-LABEL: define {{[^@]+}}@t9(
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[A:%.*]], -1
 ; CHECK-NEXT:    [[TMP2:%.*]] = sext i32 [[A]] to i64
 ; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[TMP1]], i64 [[TMP2]], i64 4294967295
@@ -137,7 +137,7 @@ define i64 @t9(i32 %a) {
 }
 
 define float @t10(i32 %x) {
-; CHECK-LABEL: @t10(
+; CHECK-LABEL: define {{[^@]+}}@t10(
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[X:%.*]], 255
 ; CHECK-NEXT:    [[R1:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 255
 ; CHECK-NEXT:    [[TMP2:%.*]] = sitofp i32 [[R1]] to float
@@ -150,7 +150,7 @@ define float @t10(i32 %x) {
 }
 
 define float @t11(i64 %x) {
-; CHECK-LABEL: @t11(
+; CHECK-LABEL: define {{[^@]+}}@t11(
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i64 [[X:%.*]], 255
 ; CHECK-NEXT:    [[R1:%.*]] = select i1 [[TMP1]], i64 [[X]], i64 255
 ; CHECK-NEXT:    [[TMP2:%.*]] = sitofp i64 [[R1]] to float
@@ -165,7 +165,7 @@ define float @t11(i64 %x) {
 ; Reuse the first 2 bitcasts as the select operands.
 
 define <4 x i32> @bitcasts_fcmp_1(<2 x i64> %a, <2 x i64> %b) {
-; CHECK-LABEL: @bitcasts_fcmp_1(
+; CHECK-LABEL: define {{[^@]+}}@bitcasts_fcmp_1(
 ; CHECK-NEXT:    [[T0:%.*]] = bitcast <2 x i64> [[A:%.*]] to <4 x float>
 ; CHECK-NEXT:    [[T1:%.*]] = bitcast <2 x i64> [[B:%.*]] to <4 x float>
 ; CHECK-NEXT:    [[T2:%.*]] = fcmp olt <4 x float> [[T1]], [[T0]]
@@ -185,7 +185,7 @@ define <4 x i32> @bitcasts_fcmp_1(<2 x i64> %a, <2 x i64> %b) {
 ; Switch cmp operand order.
 
 define <4 x i32> @bitcasts_fcmp_2(<2 x i64> %a, <2 x i64> %b) {
-; CHECK-LABEL: @bitcasts_fcmp_2(
+; CHECK-LABEL: define {{[^@]+}}@bitcasts_fcmp_2(
 ; CHECK-NEXT:    [[T0:%.*]] = bitcast <2 x i64> [[A:%.*]] to <4 x float>
 ; CHECK-NEXT:    [[T1:%.*]] = bitcast <2 x i64> [[B:%.*]] to <4 x float>
 ; CHECK-NEXT:    [[T2:%.*]] = fcmp olt <4 x float> [[T0]], [[T1]]
@@ -205,7 +205,7 @@ define <4 x i32> @bitcasts_fcmp_2(<2 x i64> %a, <2 x i64> %b) {
 ; Integer cmp should have the same transforms.
 
 define <4 x float> @bitcasts_icmp(<2 x i64> %a, <2 x i64> %b) {
-; CHECK-LABEL: @bitcasts_icmp(
+; CHECK-LABEL: define {{[^@]+}}@bitcasts_icmp(
 ; CHECK-NEXT:    [[T0:%.*]] = bitcast <2 x i64> [[A:%.*]] to <4 x i32>
 ; CHECK-NEXT:    [[T1:%.*]] = bitcast <2 x i64> [[B:%.*]] to <4 x i32>
 ; CHECK-NEXT:    [[T2:%.*]] = icmp slt <4 x i32> [[T1]], [[T0]]
@@ -224,7 +224,7 @@ define <4 x float> @bitcasts_icmp(<2 x i64> %a, <2 x i64> %b) {
 
 ; SMIN(SMIN(X, 11), 92) -> SMIN(X, 11)
 define i32 @test68(i32 %x) {
-; CHECK-LABEL: @test68(
+; CHECK-LABEL: define {{[^@]+}}@test68(
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[X:%.*]], 11
 ; CHECK-NEXT:    [[COND:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 11
 ; CHECK-NEXT:    ret i32 [[COND]]
@@ -237,7 +237,7 @@ define i32 @test68(i32 %x) {
 }
 
 define <2 x i32> @test68vec(<2 x i32> %x) {
-; CHECK-LABEL: @test68vec(
+; CHECK-LABEL: define {{[^@]+}}@test68vec(
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt <2 x i32> [[X:%.*]], <i32 11, i32 11>
 ; CHECK-NEXT:    [[COND:%.*]] = select <2 x i1> [[TMP1]], <2 x i32> [[X]], <2 x i32> <i32 11, i32 11>
 ; CHECK-NEXT:    ret <2 x i32> [[COND]]
@@ -251,7 +251,7 @@ define <2 x i32> @test68vec(<2 x i32> %x) {
 
 ; MIN(MIN(X, 24), 83) -> MIN(X, 24)
 define i32 @test69(i32 %x) {
-; CHECK-LABEL: @test69(
+; CHECK-LABEL: define {{[^@]+}}@test69(
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[X:%.*]], 24
 ; CHECK-NEXT:    [[COND:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 24
 ; CHECK-NEXT:    ret i32 [[COND]]
@@ -265,7 +265,7 @@ define i32 @test69(i32 %x) {
 
 ; SMAX(SMAX(X, 75), 36) -> SMAX(X, 75)
 define i32 @test70(i32 %x) {
-; CHECK-LABEL: @test70(
+; CHECK-LABEL: define {{[^@]+}}@test70(
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[X:%.*]], 75
 ; CHECK-NEXT:    [[COND:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 75
 ; CHECK-NEXT:    ret i32 [[COND]]
@@ -279,7 +279,7 @@ define i32 @test70(i32 %x) {
 
 ; MAX(MAX(X, 68), 47) -> MAX(X, 68)
 define i32 @test71(i32 %x) {
-; CHECK-LABEL: @test71(
+; CHECK-LABEL: define {{[^@]+}}@test71(
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i32 [[X:%.*]], 68
 ; CHECK-NEXT:    [[COND:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 68
 ; CHECK-NEXT:    ret i32 [[COND]]
@@ -293,7 +293,7 @@ define i32 @test71(i32 %x) {
 
 ; SMIN(SMIN(X, 92), 11) -> SMIN(X, 11)
 define i32 @test72(i32 %x) {
-; CHECK-LABEL: @test72(
+; CHECK-LABEL: define {{[^@]+}}@test72(
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[X:%.*]], 11
 ; CHECK-NEXT:    [[RETVAL:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 11
 ; CHECK-NEXT:    ret i32 [[RETVAL]]
@@ -306,7 +306,7 @@ define i32 @test72(i32 %x) {
 }
 
 define <2 x i32> @test72vec(<2 x i32> %x) {
-; CHECK-LABEL: @test72vec(
+; CHECK-LABEL: define {{[^@]+}}@test72vec(
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt <2 x i32> [[X:%.*]], <i32 11, i32 11>
 ; CHECK-NEXT:    [[RETVAL:%.*]] = select <2 x i1> [[TMP1]], <2 x i32> [[X]], <2 x i32> <i32 11, i32 11>
 ; CHECK-NEXT:    ret <2 x i32> [[RETVAL]]
@@ -320,7 +320,7 @@ define <2 x i32> @test72vec(<2 x i32> %x) {
 
 ; MIN(MIN(X, 83), 24) -> MIN(X, 24)
 define i32 @test73(i32 %x) {
-; CHECK-LABEL: @test73(
+; CHECK-LABEL: define {{[^@]+}}@test73(
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[X:%.*]], 24
 ; CHECK-NEXT:    [[RETVAL:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 24
 ; CHECK-NEXT:    ret i32 [[RETVAL]]
@@ -334,7 +334,7 @@ define i32 @test73(i32 %x) {
 
 ; SMAX(SMAX(X, 36), 75) -> SMAX(X, 75)
 define i32 @test74(i32 %x) {
-; CHECK-LABEL: @test74(
+; CHECK-LABEL: define {{[^@]+}}@test74(
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[X:%.*]], 75
 ; CHECK-NEXT:    [[RETVAL:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 75
 ; CHECK-NEXT:    ret i32 [[RETVAL]]
@@ -348,7 +348,7 @@ define i32 @test74(i32 %x) {
 
 ; MAX(MAX(X, 47), 68) -> MAX(X, 68)
 define i32 @test75(i32 %x) {
-; CHECK-LABEL: @test75(
+; CHECK-LABEL: define {{[^@]+}}@test75(
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i32 [[X:%.*]], 68
 ; CHECK-NEXT:    [[RETVAL:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 68
 ; CHECK-NEXT:    ret i32 [[RETVAL]]
@@ -366,7 +366,7 @@ define i32 @test75(i32 %x) {
 ; (X <s C1) ? C1 : SMIN(X, C2) ==> SMAX(SMIN(X, C2), C1)
 
 define i32 @clamp_signed1(i32 %x) {
-; CHECK-LABEL: @clamp_signed1(
+; CHECK-LABEL: define {{[^@]+}}@clamp_signed1(
 ; CHECK-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[X:%.*]], 255
 ; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[CMP2]], i32 [[X]], i32 255
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[MIN]], 15
@@ -383,7 +383,7 @@ define i32 @clamp_signed1(i32 %x) {
 ; (X >s C1) ? C1 : SMAX(X, C2) ==> SMIN(SMAX(X, C2), C1)
 
 define i32 @clamp_signed2(i32 %x) {
-; CHECK-LABEL: @clamp_signed2(
+; CHECK-LABEL: define {{[^@]+}}@clamp_signed2(
 ; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i32 [[X:%.*]], 15
 ; CHECK-NEXT:    [[MAX:%.*]] = select i1 [[CMP2]], i32 [[X]], i32 15
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[MAX]], 255
@@ -400,7 +400,7 @@ define i32 @clamp_signed2(i32 %x) {
 ; (X >s C1) ? SMIN(X, C2) : C1 ==> SMAX(SMIN(X, C2), C1)
 
 define i32 @clamp_signed3(i32 %x) {
-; CHECK-LABEL: @clamp_signed3(
+; CHECK-LABEL: define {{[^@]+}}@clamp_signed3(
 ; CHECK-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[X:%.*]], 255
 ; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[CMP2]], i32 [[X]], i32 255
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[MIN]], 15
@@ -417,7 +417,7 @@ define i32 @clamp_signed3(i32 %x) {
 ; (X <s C1) ? SMAX(X, C2) : C1 ==> SMIN(SMAX(X, C1), C2)
 
 define i32 @clamp_signed4(i32 %x) {
-; CHECK-LABEL: @clamp_signed4(
+; CHECK-LABEL: define {{[^@]+}}@clamp_signed4(
 ; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i32 [[X:%.*]], 15
 ; CHECK-NEXT:    [[MAX:%.*]] = select i1 [[CMP2]], i32 [[X]], i32 15
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[MAX]], 255
@@ -434,7 +434,7 @@ define i32 @clamp_signed4(i32 %x) {
 ; (X <u C1) ? C1 : UMIN(X, C2) ==> UMAX(UMIN(X, C2), C1)
 
 define i32 @clamp_unsigned1(i32 %x) {
-; CHECK-LABEL: @clamp_unsigned1(
+; CHECK-LABEL: define {{[^@]+}}@clamp_unsigned1(
 ; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult i32 [[X:%.*]], 255
 ; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[CMP2]], i32 [[X]], i32 255
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i32 [[MIN]], 15
@@ -451,7 +451,7 @@ define i32 @clamp_unsigned1(i32 %x) {
 ; (X >u C1) ? C1 : UMAX(X, C2) ==> UMIN(UMAX(X, C2), C1)
 
 define i32 @clamp_unsigned2(i32 %x) {
-; CHECK-LABEL: @clamp_unsigned2(
+; CHECK-LABEL: define {{[^@]+}}@clamp_unsigned2(
 ; CHECK-NEXT:    [[CMP2:%.*]] = icmp ugt i32 [[X:%.*]], 15
 ; CHECK-NEXT:    [[MAX:%.*]] = select i1 [[CMP2]], i32 [[X]], i32 15
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[MAX]], 255
@@ -468,7 +468,7 @@ define i32 @clamp_unsigned2(i32 %x) {
 ; (X >u C1) ? UMIN(X, C2) : C1 ==> UMAX(UMIN(X, C2), C1)
 
 define i32 @clamp_unsigned3(i32 %x) {
-; CHECK-LABEL: @clamp_unsigned3(
+; CHECK-LABEL: define {{[^@]+}}@clamp_unsigned3(
 ; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult i32 [[X:%.*]], 255
 ; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[CMP2]], i32 [[X]], i32 255
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i32 [[MIN]], 15
@@ -485,7 +485,7 @@ define i32 @clamp_unsigned3(i32 %x) {
 ; (X <u C1) ? UMAX(X, C2) : C1 ==> UMIN(UMAX(X, C2), C1)
 
 define i32 @clamp_unsigned4(i32 %x) {
-; CHECK-LABEL: @clamp_unsigned4(
+; CHECK-LABEL: define {{[^@]+}}@clamp_unsigned4(
 ; CHECK-NEXT:    [[CMP2:%.*]] = icmp ugt i32 [[X:%.*]], 15
 ; CHECK-NEXT:    [[MAX:%.*]] = select i1 [[CMP2]], i32 [[X]], i32 15
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[MAX]], 255
@@ -503,7 +503,7 @@ define i32 @clamp_unsigned4(i32 %x) {
 ; loop because of reverse cmp transformation:
 ; (icmp sgt smin(PositiveA, B) 0) -> (icmp sgt B 0)
 define i32 @clamp_check_for_no_infinite_loop1(i32 %i) {
-; CHECK-LABEL: @clamp_check_for_no_infinite_loop1(
+; CHECK-LABEL: define {{[^@]+}}@clamp_check_for_no_infinite_loop1(
 ; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[I:%.*]], 255
 ; CHECK-NEXT:    [[SEL1:%.*]] = select i1 [[CMP1]], i32 [[I]], i32 255
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[SEL1]], 0
@@ -519,7 +519,7 @@ define i32 @clamp_check_for_no_infinite_loop1(i32 %i) {
 ; Check that there is no infinite loop in case of:
 ; (icmp slt smax(NegativeA, B) 0) -> (icmp slt B 0)
 define i32 @clamp_check_for_no_infinite_loop2(i32 %i) {
-; CHECK-LABEL: @clamp_check_for_no_infinite_loop2(
+; CHECK-LABEL: define {{[^@]+}}@clamp_check_for_no_infinite_loop2(
 ; CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i32 [[I:%.*]], -255
 ; CHECK-NEXT:    [[SEL1:%.*]] = select i1 [[CMP1]], i32 [[I]], i32 -255
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[SEL1]], 0
@@ -536,7 +536,7 @@ define i32 @clamp_check_for_no_infinite_loop2(i32 %i) {
 ; Check that there is no infinite loop because of reverse cmp transformation:
 ; (icmp slt smax(PositiveA, B) 2) -> (icmp eq B 1)
 define i32 @clamp_check_for_no_infinite_loop3(i32 %i) {
-; CHECK-LABEL: @clamp_check_for_no_infinite_loop3(
+; CHECK-LABEL: define {{[^@]+}}@clamp_check_for_no_infinite_loop3(
 ; CHECK-NEXT:    [[I2:%.*]] = icmp sgt i32 [[I:%.*]], 1
 ; CHECK-NEXT:    [[I3:%.*]] = select i1 [[I2]], i32 [[I]], i32 1
 ; CHECK-NEXT:    br i1 true, label [[TRUELABEL:%.*]], label [[FALSELABEL:%.*]]
@@ -567,7 +567,7 @@ falselabel:
 ; The next 3 min tests should canonicalize to the same form...and not infinite loop.
 
 define double @PR31751_umin1(i32 %x) {
-; CHECK-LABEL: @PR31751_umin1(
+; CHECK-LABEL: define {{[^@]+}}@PR31751_umin1(
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[X:%.*]], 2147483647
 ; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 2147483647
 ; CHECK-NEXT:    [[CONV:%.*]] = sitofp i32 [[SEL]] to double
@@ -580,7 +580,7 @@ define double @PR31751_umin1(i32 %x) {
 }
 
 define double @PR31751_umin2(i32 %x) {
-; CHECK-LABEL: @PR31751_umin2(
+; CHECK-LABEL: define {{[^@]+}}@PR31751_umin2(
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[X:%.*]], 2147483647
 ; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 [[X]], i32 2147483647
 ; CHECK-NEXT:    [[CONV:%.*]] = sitofp i32 [[SEL]] to double
@@ -593,7 +593,7 @@ define double @PR31751_umin2(i32 %x) {
 }
 
 define double @PR31751_umin3(i32 %x) {
-; CHECK-LABEL: @PR31751_umin3(
+; CHECK-LABEL: define {{[^@]+}}@PR31751_umin3(
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[X:%.*]], 2147483647
 ; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 2147483647
 ; CHECK-NEXT:    [[CONV:%.*]] = sitofp i32 [[SEL]] to double
@@ -608,7 +608,7 @@ define double @PR31751_umin3(i32 %x) {
 ; The next 3 max tests should canonicalize to the same form...and not infinite loop.
 
 define double @PR31751_umax1(i32 %x) {
-; CHECK-LABEL: @PR31751_umax1(
+; CHECK-LABEL: define {{[^@]+}}@PR31751_umax1(
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i32 [[X:%.*]], -2147483648
 ; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 -2147483648
 ; CHECK-NEXT:    [[CONV:%.*]] = sitofp i32 [[SEL]] to double
@@ -621,7 +621,7 @@ define double @PR31751_umax1(i32 %x) {
 }
 
 define double @PR31751_umax2(i32 %x) {
-; CHECK-LABEL: @PR31751_umax2(
+; CHECK-LABEL: define {{[^@]+}}@PR31751_umax2(
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[X:%.*]], -2147483648
 ; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 [[X]], i32 -2147483648
 ; CHECK-NEXT:    [[CONV:%.*]] = sitofp i32 [[SEL]] to double
@@ -634,7 +634,7 @@ define double @PR31751_umax2(i32 %x) {
 }
 
 define double @PR31751_umax3(i32 %x) {
-; CHECK-LABEL: @PR31751_umax3(
+; CHECK-LABEL: define {{[^@]+}}@PR31751_umax3(
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i32 [[X:%.*]], -2147483648
 ; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 -2147483648
 ; CHECK-NEXT:    [[CONV:%.*]] = sitofp i32 [[SEL]] to double
@@ -649,7 +649,7 @@ define double @PR31751_umax3(i32 %x) {
 ; The icmp/select form a canonical smax, so don't hide that by folding the final bitcast into the select.
 
 define float @bitcast_scalar_smax(float %x, float %y) {
-; CHECK-LABEL: @bitcast_scalar_smax(
+; CHECK-LABEL: define {{[^@]+}}@bitcast_scalar_smax(
 ; CHECK-NEXT:    [[BCX:%.*]] = bitcast float [[X:%.*]] to i32
 ; CHECK-NEXT:    [[BCY:%.*]] = bitcast float [[Y:%.*]] to i32
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[BCX]], [[BCY]]
@@ -668,7 +668,7 @@ define float @bitcast_scalar_smax(float %x, float %y) {
 ; FIXME: Create a canonical umax by bitcasting the select.
 
 define float @bitcast_scalar_umax(float %x, float %y) {
-; CHECK-LABEL: @bitcast_scalar_umax(
+; CHECK-LABEL: define {{[^@]+}}@bitcast_scalar_umax(
 ; CHECK-NEXT:    [[BCX:%.*]] = bitcast float [[X:%.*]] to i32
 ; CHECK-NEXT:    [[BCY:%.*]] = bitcast float [[Y:%.*]] to i32
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[BCX]], [[BCY]]
@@ -686,7 +686,7 @@ define float @bitcast_scalar_umax(float %x, float %y) {
 ; The icmp/select form a canonical smin, so don't hide that by folding the final bitcast into the select.
 
 define <8 x float> @bitcast_vector_smin(<8 x float> %x, <8 x float> %y) {
-; CHECK-LABEL: @bitcast_vector_smin(
+; CHECK-LABEL: define {{[^@]+}}@bitcast_vector_smin(
 ; CHECK-NEXT:    [[BCX:%.*]] = bitcast <8 x float> [[X:%.*]] to <8 x i32>
 ; CHECK-NEXT:    [[BCY:%.*]] = bitcast <8 x float> [[Y:%.*]] to <8 x i32>
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <8 x i32> [[BCX]], [[BCY]]
@@ -705,7 +705,7 @@ define <8 x float> @bitcast_vector_smin(<8 x float> %x, <8 x float> %y) {
 ; FIXME: Create a canonical umin by bitcasting the select.
 
 define <8 x float> @bitcast_vector_umin(<8 x float> %x, <8 x float> %y) {
-; CHECK-LABEL: @bitcast_vector_umin(
+; CHECK-LABEL: define {{[^@]+}}@bitcast_vector_umin(
 ; CHECK-NEXT:    [[BCX:%.*]] = bitcast <8 x float> [[X:%.*]] to <8 x i32>
 ; CHECK-NEXT:    [[BCY:%.*]] = bitcast <8 x float> [[Y:%.*]] to <8 x i32>
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <8 x i32> [[BCX]], [[BCY]]
@@ -722,7 +722,7 @@ define <8 x float> @bitcast_vector_umin(<8 x float> %x, <8 x float> %y) {
 ; Check that we look through cast and recognize min idiom.
 
 define zeroext i8 @look_through_cast1(i32 %x) {
-; CHECK-LABEL: @look_through_cast1(
+; CHECK-LABEL: define {{[^@]+}}@look_through_cast1(
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[X:%.*]], 511
 ; CHECK-NEXT:    [[RES1:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 511
 ; CHECK-NEXT:    [[TMP2:%.*]] = trunc i32 [[RES1]] to i8
@@ -737,7 +737,7 @@ define zeroext i8 @look_through_cast1(i32 %x) {
 ; Check that we look through cast but min is not recognized.
 
 define zeroext i8 @look_through_cast2(i32 %x) {
-; CHECK-LABEL: @look_through_cast2(
+; CHECK-LABEL: define {{[^@]+}}@look_through_cast2(
 ; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[X:%.*]], 510
 ; CHECK-NEXT:    [[X_TRUNC:%.*]] = trunc i32 [[X]] to i8
 ; CHECK-NEXT:    [[RES:%.*]] = select i1 [[CMP1]], i8 [[X_TRUNC]], i8 -1
@@ -750,7 +750,7 @@ define zeroext i8 @look_through_cast2(i32 %x) {
 }
 
 define <2 x i8> @min_through_cast_vec1(<2 x i32> %x) {
-; CHECK-LABEL: @min_through_cast_vec1(
+; CHECK-LABEL: define {{[^@]+}}@min_through_cast_vec1(
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt <2 x i32> [[X:%.*]], <i32 510, i32 511>
 ; CHECK-NEXT:    [[RES1:%.*]] = select <2 x i1> [[TMP1]], <2 x i32> [[X]], <2 x i32> <i32 510, i32 511>
 ; CHECK-NEXT:    [[TMP2:%.*]] = trunc <2 x i32> [[RES1]] to <2 x i8>
@@ -763,7 +763,7 @@ define <2 x i8> @min_through_cast_vec1(<2 x i32> %x) {
 }
 
 define <2 x i8> @min_through_cast_vec2(<2 x i32> %x) {
-; CHECK-LABEL: @min_through_cast_vec2(
+; CHECK-LABEL: define {{[^@]+}}@min_through_cast_vec2(
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt <2 x i32> [[X:%.*]], <i32 511, i32 511>
 ; CHECK-NEXT:    [[RES1:%.*]] = select <2 x i1> [[TMP1]], <2 x i32> [[X]], <2 x i32> <i32 511, i32 511>
 ; CHECK-NEXT:    [[TMP2:%.*]] = trunc <2 x i32> [[RES1]] to <2 x i8>
@@ -781,7 +781,7 @@ define <2 x i8> @min_through_cast_vec2(<2 x i32> %x) {
 ; min(min(a, b), min(b, c)) --> min(min(a, b), c)
 
 define i32 @common_factor_smin(i32 %a, i32 %b, i32 %c) {
-; CHECK-LABEL: @common_factor_smin(
+; CHECK-LABEL: define {{[^@]+}}@common_factor_smin(
 ; CHECK-NEXT:    [[CMP_AB:%.*]] = icmp slt i32 [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    [[MIN_AB:%.*]] = select i1 [[CMP_AB]], i32 [[A]], i32 [[B]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[MIN_AB]], [[C:%.*]]
@@ -800,7 +800,7 @@ define i32 @common_factor_smin(i32 %a, i32 %b, i32 %c) {
 ; max(max(a, b), max(c, b)) --> max(max(a, b), c)
 
 define <2 x i32> @common_factor_smax(<2 x i32> %a, <2 x i32> %b, <2 x i32> %c) {
-; CHECK-LABEL: @common_factor_smax(
+; CHECK-LABEL: define {{[^@]+}}@common_factor_smax(
 ; CHECK-NEXT:    [[CMP_AB:%.*]] = icmp sgt <2 x i32> [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    [[MAX_AB:%.*]] = select <2 x i1> [[CMP_AB]], <2 x i32> [[A]], <2 x i32> [[B]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <2 x i32> [[MAX_AB]], [[C:%.*]]
@@ -819,7 +819,7 @@ define <2 x i32> @common_factor_smax(<2 x i32> %a, <2 x i32> %b, <2 x i32> %c) {
 ; min(min(b, c), min(a, b)) --> min(min(b, c), a)
 
 define <2 x i32> @common_factor_umin(<2 x i32> %a, <2 x i32> %b, <2 x i32> %c) {
-; CHECK-LABEL: @common_factor_umin(
+; CHECK-LABEL: define {{[^@]+}}@common_factor_umin(
 ; CHECK-NEXT:    [[CMP_BC:%.*]] = icmp ult <2 x i32> [[B:%.*]], [[C:%.*]]
 ; CHECK-NEXT:    [[MIN_BC:%.*]] = select <2 x i1> [[CMP_BC]], <2 x i32> [[B]], <2 x i32> [[C]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult <2 x i32> [[MIN_BC]], [[A:%.*]]
@@ -838,7 +838,7 @@ define <2 x i32> @common_factor_umin(<2 x i32> %a, <2 x i32> %b, <2 x i32> %c) {
 ; max(max(b, c), max(b, a)) --> max(max(b, c), a)
 
 define i32 @common_factor_umax(i32 %a, i32 %b, i32 %c) {
-; CHECK-LABEL: @common_factor_umax(
+; CHECK-LABEL: define {{[^@]+}}@common_factor_umax(
 ; CHECK-NEXT:    [[CMP_BC:%.*]] = icmp ugt i32 [[B:%.*]], [[C:%.*]]
 ; CHECK-NEXT:    [[MAX_BC:%.*]] = select i1 [[CMP_BC]], i32 [[B]], i32 [[C]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i32 [[MAX_BC]], [[A:%.*]]
@@ -857,7 +857,7 @@ define i32 @common_factor_umax(i32 %a, i32 %b, i32 %c) {
 declare void @extra_use(i32)
 
 define i32 @common_factor_umax_extra_use_lhs(i32 %a, i32 %b, i32 %c) {
-; CHECK-LABEL: @common_factor_umax_extra_use_lhs(
+; CHECK-LABEL: define {{[^@]+}}@common_factor_umax_extra_use_lhs(
 ; CHECK-NEXT:    [[CMP_BC:%.*]] = icmp ugt i32 [[B:%.*]], [[C:%.*]]
 ; CHECK-NEXT:    [[MAX_BC:%.*]] = select i1 [[CMP_BC]], i32 [[B]], i32 [[C]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i32 [[MAX_BC]], [[A:%.*]]
@@ -876,7 +876,7 @@ define i32 @common_factor_umax_extra_use_lhs(i32 %a, i32 %b, i32 %c) {
 }
 
 define i32 @common_factor_umax_extra_use_rhs(i32 %a, i32 %b, i32 %c) {
-; CHECK-LABEL: @common_factor_umax_extra_use_rhs(
+; CHECK-LABEL: define {{[^@]+}}@common_factor_umax_extra_use_rhs(
 ; CHECK-NEXT:    [[CMP_BA:%.*]] = icmp ugt i32 [[B:%.*]], [[A:%.*]]
 ; CHECK-NEXT:    [[MAX_BA:%.*]] = select i1 [[CMP_BA]], i32 [[B]], i32 [[A]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i32 [[MAX_BA]], [[C:%.*]]
@@ -895,7 +895,7 @@ define i32 @common_factor_umax_extra_use_rhs(i32 %a, i32 %b, i32 %c) {
 }
 
 define i32 @common_factor_umax_extra_use_both(i32 %a, i32 %b, i32 %c) {
-; CHECK-LABEL: @common_factor_umax_extra_use_both(
+; CHECK-LABEL: define {{[^@]+}}@common_factor_umax_extra_use_both(
 ; CHECK-NEXT:    [[CMP_BC:%.*]] = icmp ugt i32 [[B:%.*]], [[C:%.*]]
 ; CHECK-NEXT:    [[MAX_BC:%.*]] = select i1 [[CMP_BC]], i32 [[B]], i32 [[C]]
 ; CHECK-NEXT:    [[CMP_BA:%.*]] = icmp ugt i32 [[B]], [[A:%.*]]
@@ -920,13 +920,13 @@ define i32 @common_factor_umax_extra_use_both(i32 %a, i32 %b, i32 %c) {
 ; This would assert. Don't assume that earlier min/max types match a possible later min/max.
 
 define float @not_min_of_min(i8 %i, float %x) {
-; CHECK-LABEL: @not_min_of_min(
+; CHECK-LABEL: define {{[^@]+}}@not_min_of_min(
 ; CHECK-NEXT:    [[CMP1_INV:%.*]] = fcmp fast oge float [[X:%.*]], 1.000000e+00
-; CHECK-NEXT:    [[MIN1:%.*]] = select i1 [[CMP1_INV]], float 1.000000e+00, float [[X]]
+; CHECK-NEXT:    [[TMP1:%.*]] = select fast i1 [[CMP1_INV]], float 1.000000e+00, float [[X]]
 ; CHECK-NEXT:    [[CMP2_INV:%.*]] = fcmp fast oge float [[X]], 2.000000e+00
-; CHECK-NEXT:    [[MIN2:%.*]] = select i1 [[CMP2_INV]], float 2.000000e+00, float [[X]]
+; CHECK-NEXT:    [[TMP2:%.*]] = select fast i1 [[CMP2_INV]], float 2.000000e+00, float [[X]]
 ; CHECK-NEXT:    [[CMP3:%.*]] = icmp ult i8 [[I:%.*]], 16
-; CHECK-NEXT:    [[R:%.*]] = select i1 [[CMP3]], float [[MIN1]], float [[MIN2]]
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[CMP3]], float [[TMP1]], float [[TMP2]]
 ; CHECK-NEXT:    ret float [[R]]
 ;
   %cmp1 = fcmp fast ult float %x, 1.0
@@ -939,7 +939,7 @@ define float @not_min_of_min(i8 %i, float %x) {
 }
 
 define i32 @add_umin(i32 %x) {
-; CHECK-LABEL: @add_umin(
+; CHECK-LABEL: define {{[^@]+}}@add_umin(
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[X:%.*]], 27
 ; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 27
 ; CHECK-NEXT:    [[R:%.*]] = add nuw nsw i32 [[TMP2]], 15
@@ -952,7 +952,7 @@ define i32 @add_umin(i32 %x) {
 }
 
 define i32 @add_umin_constant_limit(i32 %x) {
-; CHECK-LABEL: @add_umin_constant_limit(
+; CHECK-LABEL: define {{[^@]+}}@add_umin_constant_limit(
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[X:%.*]], 0
 ; CHECK-NEXT:    [[R:%.*]] = select i1 [[TMP1]], i32 41, i32 42
 ; CHECK-NEXT:    ret i32 [[R]]
@@ -967,7 +967,7 @@ define i32 @add_umin_constant_limit(i32 %x) {
 ; TODO: assert that instsimplify always gets this?
 
 define i32 @add_umin_simplify(i32 %x) {
-; CHECK-LABEL: @add_umin_simplify(
+; CHECK-LABEL: define {{[^@]+}}@add_umin_simplify(
 ; CHECK-NEXT:    ret i32 42
 ;
   %a = add nuw i32 %x, 42
@@ -980,7 +980,7 @@ define i32 @add_umin_simplify(i32 %x) {
 ; TODO: assert that instsimplify always gets this?
 
 define i32 @add_umin_simplify2(i32 %x) {
-; CHECK-LABEL: @add_umin_simplify2(
+; CHECK-LABEL: define {{[^@]+}}@add_umin_simplify2(
 ; CHECK-NEXT:    ret i32 42
 ;
   %a = add nuw i32 %x, 43
@@ -992,7 +992,7 @@ define i32 @add_umin_simplify2(i32 %x) {
 ; Negative test
 
 define i32 @add_umin_wrong_pred(i32 %x) {
-; CHECK-LABEL: @add_umin_wrong_pred(
+; CHECK-LABEL: define {{[^@]+}}@add_umin_wrong_pred(
 ; CHECK-NEXT:    [[A:%.*]] = add nuw i32 [[X:%.*]], 15
 ; CHECK-NEXT:    [[C:%.*]] = icmp slt i32 [[A]], 42
 ; CHECK-NEXT:    [[R:%.*]] = select i1 [[C]], i32 [[A]], i32 42
@@ -1007,7 +1007,7 @@ define i32 @add_umin_wrong_pred(i32 %x) {
 ; Negative test
 
 define i32 @add_umin_wrong_wrap(i32 %x) {
-; CHECK-LABEL: @add_umin_wrong_wrap(
+; CHECK-LABEL: define {{[^@]+}}@add_umin_wrong_wrap(
 ; CHECK-NEXT:    [[A:%.*]] = add nsw i32 [[X:%.*]], 15
 ; CHECK-NEXT:    [[C:%.*]] = icmp ult i32 [[A]], 42
 ; CHECK-NEXT:    [[R:%.*]] = select i1 [[C]], i32 [[A]], i32 42
@@ -1022,7 +1022,7 @@ define i32 @add_umin_wrong_wrap(i32 %x) {
 ; Negative test
 
 define i32 @add_umin_extra_use(i32 %x, i32* %p) {
-; CHECK-LABEL: @add_umin_extra_use(
+; CHECK-LABEL: define {{[^@]+}}@add_umin_extra_use(
 ; CHECK-NEXT:    [[A:%.*]] = add nuw i32 [[X:%.*]], 15
 ; CHECK-NEXT:    store i32 [[A]], i32* [[P:%.*]], align 4
 ; CHECK-NEXT:    [[C:%.*]] = icmp ult i32 [[A]], 42
@@ -1037,7 +1037,7 @@ define i32 @add_umin_extra_use(i32 %x, i32* %p) {
 }
 
 define <2 x i16> @add_umin_vec(<2 x i16> %x) {
-; CHECK-LABEL: @add_umin_vec(
+; CHECK-LABEL: define {{[^@]+}}@add_umin_vec(
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult <2 x i16> [[X:%.*]], <i16 225, i16 225>
 ; CHECK-NEXT:    [[TMP2:%.*]] = select <2 x i1> [[TMP1]], <2 x i16> [[X]], <2 x i16> <i16 225, i16 225>
 ; CHECK-NEXT:    [[R:%.*]] = add nuw nsw <2 x i16> [[TMP2]], <i16 15, i16 15>
@@ -1050,7 +1050,7 @@ define <2 x i16> @add_umin_vec(<2 x i16> %x) {
 }
 
 define i37 @add_umax(i37 %x) {
-; CHECK-LABEL: @add_umax(
+; CHECK-LABEL: define {{[^@]+}}@add_umax(
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i37 [[X:%.*]], 37
 ; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i37 [[X]], i37 37
 ; CHECK-NEXT:    [[R:%.*]] = add nuw i37 [[TMP2]], 5
@@ -1063,7 +1063,7 @@ define i37 @add_umax(i37 %x) {
 }
 
 define i37 @add_umax_constant_limit(i37 %x) {
-; CHECK-LABEL: @add_umax_constant_limit(
+; CHECK-LABEL: define {{[^@]+}}@add_umax_constant_limit(
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i37 [[X:%.*]], 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i37 [[X]], i37 1
 ; CHECK-NEXT:    [[R:%.*]] = add nuw i37 [[TMP2]], 81
@@ -1079,9 +1079,9 @@ define i37 @add_umax_constant_limit(i37 %x) {
 ; TODO: assert that instsimplify always gets this?
 
 define i37 @add_umax_simplify(i37 %x) {
-; CHECK-LABEL: @add_umax_simplify(
-; CHECK-NEXT:    [[R:%.*]] = add nuw i37 [[X:%.*]], 42
-; CHECK-NEXT:    ret i37 [[R]]
+; CHECK-LABEL: define {{[^@]+}}@add_umax_simplify(
+; CHECK-NEXT:    [[A:%.*]] = add i37 [[X:%.*]], 42
+; CHECK-NEXT:    ret i37 [[A]]
 ;
   %a = add nuw i37 %x, 42
   %c = icmp ugt i37 %a, 42
@@ -1093,7 +1093,7 @@ define i37 @add_umax_simplify(i37 %x) {
 ; TODO: assert that instsimplify always gets this?
 
 define i32 @add_umax_simplify2(i32 %x) {
-; CHECK-LABEL: @add_umax_simplify2(
+; CHECK-LABEL: define {{[^@]+}}@add_umax_simplify2(
 ; CHECK-NEXT:    [[A:%.*]] = add nuw i32 [[X:%.*]], 57
 ; CHECK-NEXT:    ret i32 [[A]]
 ;
@@ -1106,7 +1106,7 @@ define i32 @add_umax_simplify2(i32 %x) {
 ; Negative test
 
 define i32 @add_umax_wrong_pred(i32 %x) {
-; CHECK-LABEL: @add_umax_wrong_pred(
+; CHECK-LABEL: define {{[^@]+}}@add_umax_wrong_pred(
 ; CHECK-NEXT:    [[A:%.*]] = add nuw i32 [[X:%.*]], 15
 ; CHECK-NEXT:    [[C:%.*]] = icmp sgt i32 [[A]], 42
 ; CHECK-NEXT:    [[R:%.*]] = select i1 [[C]], i32 [[A]], i32 42
@@ -1121,7 +1121,7 @@ define i32 @add_umax_wrong_pred(i32 %x) {
 ; Negative test
 
 define i32 @add_umax_wrong_wrap(i32 %x) {
-; CHECK-LABEL: @add_umax_wrong_wrap(
+; CHECK-LABEL: define {{[^@]+}}@add_umax_wrong_wrap(
 ; CHECK-NEXT:    [[A:%.*]] = add nsw i32 [[X:%.*]], 15
 ; CHECK-NEXT:    [[C:%.*]] = icmp ugt i32 [[A]], 42
 ; CHECK-NEXT:    [[R:%.*]] = select i1 [[C]], i32 [[A]], i32 42
@@ -1136,7 +1136,7 @@ define i32 @add_umax_wrong_wrap(i32 %x) {
 ; Negative test
 
 define i32 @add_umax_extra_use(i32 %x, i32* %p) {
-; CHECK-LABEL: @add_umax_extra_use(
+; CHECK-LABEL: define {{[^@]+}}@add_umax_extra_use(
 ; CHECK-NEXT:    [[A:%.*]] = add nuw i32 [[X:%.*]], 15
 ; CHECK-NEXT:    store i32 [[A]], i32* [[P:%.*]], align 4
 ; CHECK-NEXT:    [[C:%.*]] = icmp ugt i32 [[A]], 42
@@ -1151,7 +1151,7 @@ define i32 @add_umax_extra_use(i32 %x, i32* %p) {
 }
 
 define <2 x i33> @add_umax_vec(<2 x i33> %x) {
-; CHECK-LABEL: @add_umax_vec(
+; CHECK-LABEL: define {{[^@]+}}@add_umax_vec(
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt <2 x i33> [[X:%.*]], <i33 235, i33 235>
 ; CHECK-NEXT:    [[TMP2:%.*]] = select <2 x i1> [[TMP1]], <2 x i33> [[X]], <2 x i33> <i33 235, i33 235>
 ; CHECK-NEXT:    [[R:%.*]] = add nuw <2 x i33> [[TMP2]], <i33 5, i33 5>
@@ -1164,7 +1164,7 @@ define <2 x i33> @add_umax_vec(<2 x i33> %x) {
 }
 
 define i8 @PR14613_umin(i8 %x) {
-; CHECK-LABEL: @PR14613_umin(
+; CHECK-LABEL: define {{[^@]+}}@PR14613_umin(
 ; CHECK-NEXT:    [[U7:%.*]] = call i8 @llvm.uadd.sat.i8(i8 [[X:%.*]], i8 15)
 ; CHECK-NEXT:    ret i8 [[U7]]
 ;
@@ -1177,7 +1177,7 @@ define i8 @PR14613_umin(i8 %x) {
 }
 
 define i8 @PR14613_umax(i8 %x) {
-; CHECK-LABEL: @PR14613_umax(
+; CHECK-LABEL: define {{[^@]+}}@PR14613_umax(
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i8 [[X:%.*]], -16
 ; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i8 [[X]], i8 -16
 ; CHECK-NEXT:    [[U7:%.*]] = add nsw i8 [[TMP2]], 15
@@ -1192,7 +1192,7 @@ define i8 @PR14613_umax(i8 %x) {
 }
 
 define i32 @add_smin(i32 %x) {
-; CHECK-LABEL: @add_smin(
+; CHECK-LABEL: define {{[^@]+}}@add_smin(
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[X:%.*]], 27
 ; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 27
 ; CHECK-NEXT:    [[R:%.*]] = add nsw i32 [[TMP2]], 15
@@ -1205,7 +1205,7 @@ define i32 @add_smin(i32 %x) {
 }
 
 define i32 @add_smin_constant_limit(i32 %x) {
-; CHECK-LABEL: @add_smin_constant_limit(
+; CHECK-LABEL: define {{[^@]+}}@add_smin_constant_limit(
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[X:%.*]], 2147483646
 ; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 2147483646
 ; CHECK-NEXT:    [[R:%.*]] = add nsw i32 [[TMP2]], -3
@@ -1221,7 +1221,7 @@ define i32 @add_smin_constant_limit(i32 %x) {
 ; TODO: assert that instsimplify always gets this?
 
 define i32 @add_smin_simplify(i32 %x) {
-; CHECK-LABEL: @add_smin_simplify(
+; CHECK-LABEL: define {{[^@]+}}@add_smin_simplify(
 ; CHECK-NEXT:    [[R:%.*]] = add nsw i32 [[X:%.*]], -3
 ; CHECK-NEXT:    ret i32 [[R]]
 ;
@@ -1235,7 +1235,7 @@ define i32 @add_smin_simplify(i32 %x) {
 ; TODO: assert that instsimplify always gets this?
 
 define i32 @add_smin_simplify2(i32 %x) {
-; CHECK-LABEL: @add_smin_simplify2(
+; CHECK-LABEL: define {{[^@]+}}@add_smin_simplify2(
 ; CHECK-NEXT:    [[A:%.*]] = add nsw i32 [[X:%.*]], -3
 ; CHECK-NEXT:    ret i32 [[A]]
 ;
@@ -1248,7 +1248,7 @@ define i32 @add_smin_simplify2(i32 %x) {
 ; Negative test
 
 define i32 @add_smin_wrong_pred(i32 %x) {
-; CHECK-LABEL: @add_smin_wrong_pred(
+; CHECK-LABEL: define {{[^@]+}}@add_smin_wrong_pred(
 ; CHECK-NEXT:    [[A:%.*]] = add nsw i32 [[X:%.*]], 15
 ; CHECK-NEXT:    [[C:%.*]] = icmp ult i32 [[A]], 42
 ; CHECK-NEXT:    [[R:%.*]] = select i1 [[C]], i32 [[A]], i32 42
@@ -1263,7 +1263,7 @@ define i32 @add_smin_wrong_pred(i32 %x) {
 ; Negative test
 
 define i32 @add_smin_wrong_wrap(i32 %x) {
-; CHECK-LABEL: @add_smin_wrong_wrap(
+; CHECK-LABEL: define {{[^@]+}}@add_smin_wrong_wrap(
 ; CHECK-NEXT:    [[A:%.*]] = add nuw i32 [[X:%.*]], 15
 ; CHECK-NEXT:    [[C:%.*]] = icmp slt i32 [[A]], 42
 ; CHECK-NEXT:    [[R:%.*]] = select i1 [[C]], i32 [[A]], i32 42
@@ -1278,7 +1278,7 @@ define i32 @add_smin_wrong_wrap(i32 %x) {
 ; Negative test
 
 define i32 @add_smin_extra_use(i32 %x, i32* %p) {
-; CHECK-LABEL: @add_smin_extra_use(
+; CHECK-LABEL: define {{[^@]+}}@add_smin_extra_use(
 ; CHECK-NEXT:    [[A:%.*]] = add nsw i32 [[X:%.*]], 15
 ; CHECK-NEXT:    store i32 [[A]], i32* [[P:%.*]], align 4
 ; CHECK-NEXT:    [[C:%.*]] = icmp slt i32 [[A]], 42
@@ -1293,7 +1293,7 @@ define i32 @add_smin_extra_use(i32 %x, i32* %p) {
 }
 
 define <2 x i16> @add_smin_vec(<2 x i16> %x) {
-; CHECK-LABEL: @add_smin_vec(
+; CHECK-LABEL: define {{[^@]+}}@add_smin_vec(
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt <2 x i16> [[X:%.*]], <i16 225, i16 225>
 ; CHECK-NEXT:    [[TMP2:%.*]] = select <2 x i1> [[TMP1]], <2 x i16> [[X]], <2 x i16> <i16 225, i16 225>
 ; CHECK-NEXT:    [[R:%.*]] = add nsw <2 x i16> [[TMP2]], <i16 15, i16 15>
@@ -1306,7 +1306,7 @@ define <2 x i16> @add_smin_vec(<2 x i16> %x) {
 }
 
 define i37 @add_smax(i37 %x) {
-; CHECK-LABEL: @add_smax(
+; CHECK-LABEL: define {{[^@]+}}@add_smax(
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i37 [[X:%.*]], 37
 ; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i37 [[X]], i37 37
 ; CHECK-NEXT:    [[R:%.*]] = add nuw nsw i37 [[TMP2]], 5
@@ -1319,7 +1319,7 @@ define i37 @add_smax(i37 %x) {
 }
 
 define i8 @add_smax_constant_limit(i8 %x) {
-; CHECK-LABEL: @add_smax_constant_limit(
+; CHECK-LABEL: define {{[^@]+}}@add_smax_constant_limit(
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i8 [[X:%.*]], -127
 ; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i8 [[X]], i8 -127
 ; CHECK-NEXT:    [[R:%.*]] = add nsw i8 [[TMP2]], 125
@@ -1335,7 +1335,7 @@ define i8 @add_smax_constant_limit(i8 %x) {
 ; TODO: assert that instsimplify always gets this?
 
 define i8 @add_smax_simplify(i8 %x) {
-; CHECK-LABEL: @add_smax_simplify(
+; CHECK-LABEL: define {{[^@]+}}@add_smax_simplify(
 ; CHECK-NEXT:    [[R:%.*]] = add nsw i8 [[X:%.*]], 126
 ; CHECK-NEXT:    ret i8 [[R]]
 ;
@@ -1349,7 +1349,7 @@ define i8 @add_smax_simplify(i8 %x) {
 ; TODO: assert that instsimplify always gets this?
 
 define i8 @add_smax_simplify2(i8 %x) {
-; CHECK-LABEL: @add_smax_simplify2(
+; CHECK-LABEL: define {{[^@]+}}@add_smax_simplify2(
 ; CHECK-NEXT:    [[A:%.*]] = add nsw i8 [[X:%.*]], 127
 ; CHECK-NEXT:    ret i8 [[A]]
 ;
@@ -1362,7 +1362,7 @@ define i8 @add_smax_simplify2(i8 %x) {
 ; Negative test
 
 define i32 @add_smax_wrong_pred(i32 %x) {
-; CHECK-LABEL: @add_smax_wrong_pred(
+; CHECK-LABEL: define {{[^@]+}}@add_smax_wrong_pred(
 ; CHECK-NEXT:    [[A:%.*]] = add nsw i32 [[X:%.*]], 15
 ; CHECK-NEXT:    [[C:%.*]] = icmp ugt i32 [[A]], 42
 ; CHECK-NEXT:    [[R:%.*]] = select i1 [[C]], i32 [[A]], i32 42
@@ -1377,7 +1377,7 @@ define i32 @add_smax_wrong_pred(i32 %x) {
 ; Negative test
 
 define i32 @add_smax_wrong_wrap(i32 %x) {
-; CHECK-LABEL: @add_smax_wrong_wrap(
+; CHECK-LABEL: define {{[^@]+}}@add_smax_wrong_wrap(
 ; CHECK-NEXT:    [[A:%.*]] = add nuw i32 [[X:%.*]], 15
 ; CHECK-NEXT:    [[C:%.*]] = icmp sgt i32 [[A]], 42
 ; CHECK-NEXT:    [[R:%.*]] = select i1 [[C]], i32 [[A]], i32 42
@@ -1392,7 +1392,7 @@ define i32 @add_smax_wrong_wrap(i32 %x) {
 ; Negative test
 
 define i32 @add_smax_extra_use(i32 %x, i32* %p) {
-; CHECK-LABEL: @add_smax_extra_use(
+; CHECK-LABEL: define {{[^@]+}}@add_smax_extra_use(
 ; CHECK-NEXT:    [[A:%.*]] = add nsw i32 [[X:%.*]], 15
 ; CHECK-NEXT:    store i32 [[A]], i32* [[P:%.*]], align 4
 ; CHECK-NEXT:    [[C:%.*]] = icmp sgt i32 [[A]], 42
@@ -1407,7 +1407,7 @@ define i32 @add_smax_extra_use(i32 %x, i32* %p) {
 }
 
 define <2 x i33> @add_smax_vec(<2 x i33> %x) {
-; CHECK-LABEL: @add_smax_vec(
+; CHECK-LABEL: define {{[^@]+}}@add_smax_vec(
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <2 x i33> [[X:%.*]], <i33 235, i33 235>
 ; CHECK-NEXT:    [[TMP2:%.*]] = select <2 x i1> [[TMP1]], <2 x i33> [[X]], <2 x i33> <i33 235, i33 235>
 ; CHECK-NEXT:    [[R:%.*]] = add nuw nsw <2 x i33> [[TMP2]], <i33 5, i33 5>
@@ -1420,7 +1420,7 @@ define <2 x i33> @add_smax_vec(<2 x i33> %x) {
 }
 
 define i8 @PR14613_smin(i8 %x) {
-; CHECK-LABEL: @PR14613_smin(
+; CHECK-LABEL: define {{[^@]+}}@PR14613_smin(
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i8 [[X:%.*]], 40
 ; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i8 [[X]], i8 40
 ; CHECK-NEXT:    [[U7:%.*]] = add nsw i8 [[TMP2]], 15
@@ -1435,7 +1435,7 @@ define i8 @PR14613_smin(i8 %x) {
 }
 
 define i8 @PR14613_smax(i8 %x) {
-; CHECK-LABEL: @PR14613_smax(
+; CHECK-LABEL: define {{[^@]+}}@PR14613_smax(
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i8 [[X:%.*]], 40
 ; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i8 [[X]], i8 40
 ; CHECK-NEXT:    [[U7:%.*]] = add nuw i8 [[TMP2]], 15
diff --git a/llvm/test/Transforms/InstCombine/minmax-fp.ll b/llvm/test/Transforms/InstCombine/minmax-fp.ll
index ee1fc7a22..6d4135e17 100644
--- a/llvm/test/Transforms/InstCombine/minmax-fp.ll
+++ b/llvm/test/Transforms/InstCombine/minmax-fp.ll
@@ -3,7 +3,7 @@
 
 ; This is the canonical form for a type-changing min/max.
 define double @t1(float %a) {
-; CHECK-LABEL: @t1(
+; CHECK-LABEL: define {{[^@]+}}@t1(
 ; CHECK-NEXT:    [[DOTINV:%.*]] = fcmp oge float [[A:%.*]], 5.000000e+00
 ; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[DOTINV]], float 5.000000e+00, float [[A]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = fpext float [[TMP1]] to double
@@ -17,7 +17,7 @@ define double @t1(float %a) {
 
 ; Check this is converted into canonical form, as above.
 define double @t2(float %a) {
-; CHECK-LABEL: @t2(
+; CHECK-LABEL: define {{[^@]+}}@t2(
 ; CHECK-NEXT:    [[DOTINV:%.*]] = fcmp oge float [[A:%.*]], 5.000000e+00
 ; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[DOTINV]], float 5.000000e+00, float [[A]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = fpext float [[TMP1]] to double
@@ -31,7 +31,7 @@ define double @t2(float %a) {
 
 ; Same again, with trunc.
 define float @t4(double %a) {
-; CHECK-LABEL: @t4(
+; CHECK-LABEL: define {{[^@]+}}@t4(
 ; CHECK-NEXT:    [[DOTINV:%.*]] = fcmp oge double [[A:%.*]], 5.000000e+00
 ; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[DOTINV]], double 5.000000e+00, double [[A]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = fptrunc double [[TMP1]] to float
@@ -45,7 +45,7 @@ define float @t4(double %a) {
 
 ; different values, should not be converted.
 define double @t5(float %a) {
-; CHECK-LABEL: @t5(
+; CHECK-LABEL: define {{[^@]+}}@t5(
 ; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ult float [[A:%.*]], 5.000000e+00
 ; CHECK-NEXT:    [[TMP2:%.*]] = fpext float [[A]] to double
 ; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[TMP1]], double [[TMP2]], double 5.001000e+00
@@ -61,7 +61,7 @@ define double @t5(float %a) {
 ; So the compare constant may be treated as +0.0, and we sink the fpext.
 
 define double @t6(float %a) {
-; CHECK-LABEL: @t6(
+; CHECK-LABEL: define {{[^@]+}}@t6(
 ; CHECK-NEXT:    [[DOTINV:%.*]] = fcmp oge float [[A:%.*]], 0.000000e+00
 ; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[DOTINV]], float 0.000000e+00, float [[A]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = fpext float [[TMP1]] to double
@@ -77,7 +77,7 @@ define double @t6(float %a) {
 ; So the compare constant may be treated as -0.0, and we sink the fpext.
 
 define double @t7(float %a) {
-; CHECK-LABEL: @t7(
+; CHECK-LABEL: define {{[^@]+}}@t7(
 ; CHECK-NEXT:    [[DOTINV:%.*]] = fcmp oge float [[A:%.*]], 0.000000e+00
 ; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[DOTINV]], float -0.000000e+00, float [[A]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = fpext float [[TMP1]] to double
@@ -92,7 +92,7 @@ define double @t7(float %a) {
 ; min(min(x, 0.0), 0.0) --> min(x, 0.0)
 
 define float @fmin_fmin_zero_mismatch(float %x) {
-; CHECK-LABEL: @fmin_fmin_zero_mismatch(
+; CHECK-LABEL: define {{[^@]+}}@fmin_fmin_zero_mismatch(
 ; CHECK-NEXT:    [[TMP1:%.*]] = fcmp olt float [[X:%.*]], 0.000000e+00
 ; CHECK-NEXT:    [[MIN2:%.*]] = select i1 [[TMP1]], float [[X]], float 0.000000e+00
 ; CHECK-NEXT:    ret float [[MIN2]]
@@ -107,7 +107,7 @@ define float @fmin_fmin_zero_mismatch(float %x) {
 ; max(max(x, -0.0), -0.0) --> max(x, -0.0)
 
 define float @fmax_fmax_zero_mismatch(float %x) {
-; CHECK-LABEL: @fmax_fmax_zero_mismatch(
+; CHECK-LABEL: define {{[^@]+}}@fmax_fmax_zero_mismatch(
 ; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ogt float [[X:%.*]], -0.000000e+00
 ; CHECK-NEXT:    [[MAX11:%.*]] = select i1 [[TMP1]], float [[X]], float -0.000000e+00
 ; CHECK-NEXT:    ret float [[MAX11]]
@@ -120,7 +120,7 @@ define float @fmax_fmax_zero_mismatch(float %x) {
 }
 
 define i64 @t8(float %a) {
-; CHECK-LABEL: @t8(
+; CHECK-LABEL: define {{[^@]+}}@t8(
 ; CHECK-NEXT:    [[DOTINV:%.*]] = fcmp oge float [[A:%.*]], 5.000000e+00
 ; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[DOTINV]], float 5.000000e+00, float [[A]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = fptoui float [[TMP1]] to i64
@@ -133,7 +133,7 @@ define i64 @t8(float %a) {
 }
 
 define i8 @t9(float %a) {
-; CHECK-LABEL: @t9(
+; CHECK-LABEL: define {{[^@]+}}@t9(
 ; CHECK-NEXT:    [[DOTINV:%.*]] = fcmp oge float [[A:%.*]], 0.000000e+00
 ; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[DOTINV]], float 0.000000e+00, float [[A]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = fptosi float [[TMP1]] to i8
@@ -147,11 +147,11 @@ define i8 @t9(float %a) {
 
   ; Either operand could be NaN, but fast modifier applied.
 define i8 @t11(float %a, float %b) {
-; CHECK-LABEL: @t11(
+; CHECK-LABEL: define {{[^@]+}}@t11(
 ; CHECK-NEXT:    [[DOTINV:%.*]] = fcmp fast oge float [[B:%.*]], [[A:%.*]]
-; CHECK-NEXT:    [[DOTV:%.*]] = select i1 [[DOTINV]], float [[A]], float [[B]]
-; CHECK-NEXT:    [[TMP1:%.*]] = fptosi float [[DOTV]] to i8
-; CHECK-NEXT:    ret i8 [[TMP1]]
+; CHECK-NEXT:    [[TMP1:%.*]] = select fast i1 [[DOTINV]], float [[A]], float [[B]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fptosi float [[TMP1]] to i8
+; CHECK-NEXT:    ret i8 [[TMP2]]
 ;
   %1 = fcmp fast ult float %b, %a
   %2 = fptosi float %a to i8
@@ -162,11 +162,11 @@ define i8 @t11(float %a, float %b) {
 
 ; Either operand could be NaN, but nnan modifier applied.
 define i8 @t12(float %a, float %b) {
-; CHECK-LABEL: @t12(
+; CHECK-LABEL: define {{[^@]+}}@t12(
 ; CHECK-NEXT:    [[DOTINV:%.*]] = fcmp nnan oge float [[B:%.*]], [[A:%.*]]
-; CHECK-NEXT:    [[DOTV:%.*]] = select i1 [[DOTINV]], float [[A]], float [[B]]
-; CHECK-NEXT:    [[TMP1:%.*]] = fptosi float [[DOTV]] to i8
-; CHECK-NEXT:    ret i8 [[TMP1]]
+; CHECK-NEXT:    [[TMP1:%.*]] = select nnan i1 [[DOTINV]], float [[A]], float [[B]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fptosi float [[TMP1]] to i8
+; CHECK-NEXT:    ret i8 [[TMP2]]
 ;
   %1 = fcmp nnan ult float %b, %a
   %2 = fptosi float %a to i8
@@ -177,7 +177,7 @@ define i8 @t12(float %a, float %b) {
 
 ; Float and int values do not match.
 define i8 @t13(float %a) {
-; CHECK-LABEL: @t13(
+; CHECK-LABEL: define {{[^@]+}}@t13(
 ; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ult float [[A:%.*]], 1.500000e+00
 ; CHECK-NEXT:    [[TMP2:%.*]] = fptosi float [[A]] to i8
 ; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[TMP1]], i8 [[TMP2]], i8 1
@@ -191,7 +191,7 @@ define i8 @t13(float %a) {
 
 ; %a could be -0.0, but it doesn't matter because the conversion to int is the same for 0.0 or -0.0.
 define i8 @t14(float %a) {
-; CHECK-LABEL: @t14(
+; CHECK-LABEL: define {{[^@]+}}@t14(
 ; CHECK-NEXT:    [[DOTINV:%.*]] = fcmp oge float [[A:%.*]], 0.000000e+00
 ; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[DOTINV]], float 0.000000e+00, float [[A]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = fptosi float [[TMP1]] to i8
@@ -204,7 +204,7 @@ define i8 @t14(float %a) {
 }
 
 define i8 @t14_commute(float %a) {
-; CHECK-LABEL: @t14_commute(
+; CHECK-LABEL: define {{[^@]+}}@t14_commute(
 ; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ogt float [[A:%.*]], 0.000000e+00
 ; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], float [[A]], float 0.000000e+00
 ; CHECK-NEXT:    [[TMP3:%.*]] = fptosi float [[TMP2]] to i8
@@ -217,9 +217,9 @@ define i8 @t14_commute(float %a) {
 }
 
 define i8 @t15(float %a) {
-; CHECK-LABEL: @t15(
+; CHECK-LABEL: define {{[^@]+}}@t15(
 ; CHECK-NEXT:    [[DOTINV:%.*]] = fcmp nsz oge float [[A:%.*]], 0.000000e+00
-; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[DOTINV]], float 0.000000e+00, float [[A]]
+; CHECK-NEXT:    [[TMP1:%.*]] = select nsz i1 [[DOTINV]], float 0.000000e+00, float [[A]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = fptosi float [[TMP1]] to i8
 ; CHECK-NEXT:    ret i8 [[TMP2]]
 ;
@@ -230,7 +230,7 @@ define i8 @t15(float %a) {
 }
 
 define double @t16(i32 %x) {
-; CHECK-LABEL: @t16(
+; CHECK-LABEL: define {{[^@]+}}@t16(
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[X:%.*]], 0
 ; CHECK-NEXT:    [[CST:%.*]] = sitofp i32 [[X]] to double
 ; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], double [[CST]], double 5.000000e-01
@@ -243,7 +243,7 @@ define double @t16(i32 %x) {
 }
 
 define double @t17(i32 %x) {
-; CHECK-LABEL: @t17(
+; CHECK-LABEL: define {{[^@]+}}@t17(
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[X:%.*]], 2
 ; CHECK-NEXT:    [[SEL1:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 2
 ; CHECK-NEXT:    [[TMP2:%.*]] = sitofp i32 [[SEL1]] to double
@@ -256,7 +256,7 @@ define double @t17(i32 %x) {
 }
 
 define float @fneg_fmax(float %x, float %y) {
-; CHECK-LABEL: @fneg_fmax(
+; CHECK-LABEL: define {{[^@]+}}@fneg_fmax(
 ; CHECK-NEXT:    [[COND:%.*]] = fcmp nnan olt float [[X:%.*]], [[Y:%.*]]
 ; CHECK-NEXT:    [[MAX_V:%.*]] = select i1 [[COND]], float [[X]], float [[Y]]
 ; CHECK-NEXT:    [[MAX:%.*]] = fneg float [[MAX_V]]
@@ -270,10 +270,10 @@ define float @fneg_fmax(float %x, float %y) {
 }
 
 define <2 x float> @fsub_fmax(<2 x float> %x, <2 x float> %y) {
-; CHECK-LABEL: @fsub_fmax(
+; CHECK-LABEL: define {{[^@]+}}@fsub_fmax(
 ; CHECK-NEXT:    [[COND_INV:%.*]] = fcmp nnan nsz ogt <2 x float> [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[MAX_V:%.*]] = select <2 x i1> [[COND_INV]], <2 x float> [[Y]], <2 x float> [[X]]
-; CHECK-NEXT:    [[MAX:%.*]] = fsub <2 x float> <float -0.000000e+00, float -0.000000e+00>, [[MAX_V]]
+; CHECK-NEXT:    [[TMP1:%.*]] = select nnan nsz <2 x i1> [[COND_INV]], <2 x float> [[Y]], <2 x float> [[X]]
+; CHECK-NEXT:    [[MAX:%.*]] = fsub <2 x float> <float -0.000000e+00, float -0.000000e+00>, [[TMP1]]
 ; CHECK-NEXT:    ret <2 x float> [[MAX]]
 ;
   %n1 = fsub <2 x float> <float -0.0, float -0.0>, %x
@@ -284,7 +284,7 @@ define <2 x float> @fsub_fmax(<2 x float> %x, <2 x float> %y) {
 }
 
 define <2 x double> @fsub_fmin(<2 x double> %x, <2 x double> %y) {
-; CHECK-LABEL: @fsub_fmin(
+; CHECK-LABEL: define {{[^@]+}}@fsub_fmin(
 ; CHECK-NEXT:    [[COND:%.*]] = fcmp nnan ogt <2 x double> [[X:%.*]], [[Y:%.*]]
 ; CHECK-NEXT:    [[MAX_V:%.*]] = select <2 x i1> [[COND]], <2 x double> [[X]], <2 x double> [[Y]]
 ; CHECK-NEXT:    [[MAX:%.*]] = fsub <2 x double> <double -0.000000e+00, double -0.000000e+00>, [[MAX_V]]
@@ -298,10 +298,10 @@ define <2 x double> @fsub_fmin(<2 x double> %x, <2 x double> %y) {
 }
 
 define double @fneg_fmin(double %x, double %y) {
-; CHECK-LABEL: @fneg_fmin(
+; CHECK-LABEL: define {{[^@]+}}@fneg_fmin(
 ; CHECK-NEXT:    [[COND_INV:%.*]] = fcmp nnan nsz olt double [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[MAX_V:%.*]] = select i1 [[COND_INV]], double [[Y]], double [[X]]
-; CHECK-NEXT:    [[MAX:%.*]] = fneg double [[MAX_V]]
+; CHECK-NEXT:    [[TMP1:%.*]] = select nnan nsz i1 [[COND_INV]], double [[Y]], double [[X]]
+; CHECK-NEXT:    [[MAX:%.*]] = fneg double [[TMP1]]
 ; CHECK-NEXT:    ret double [[MAX]]
 ;
   %n1 = fneg double %x
@@ -312,7 +312,7 @@ define double @fneg_fmin(double %x, double %y) {
 }
 
 define float @maxnum_ogt_fmf_on_select(float %a, float %b) {
-; CHECK-LABEL: @maxnum_ogt_fmf_on_select(
+; CHECK-LABEL: define {{[^@]+}}@maxnum_ogt_fmf_on_select(
 ; CHECK-NEXT:    [[TMP1:%.*]] = call nnan nsz float @llvm.maxnum.f32(float [[A:%.*]], float [[B:%.*]])
 ; CHECK-NEXT:    ret float [[TMP1]]
 ;
@@ -322,7 +322,7 @@ define float @maxnum_ogt_fmf_on_select(float %a, float %b) {
 }
 
 define <2 x float> @maxnum_oge_fmf_on_select(<2 x float> %a, <2 x float> %b) {
-; CHECK-LABEL: @maxnum_oge_fmf_on_select(
+; CHECK-LABEL: define {{[^@]+}}@maxnum_oge_fmf_on_select(
 ; CHECK-NEXT:    [[TMP1:%.*]] = call nnan ninf nsz <2 x float> @llvm.maxnum.v2f32(<2 x float> [[A:%.*]], <2 x float> [[B:%.*]])
 ; CHECK-NEXT:    ret <2 x float> [[TMP1]]
 ;
@@ -332,7 +332,7 @@ define <2 x float> @maxnum_oge_fmf_on_select(<2 x float> %a, <2 x float> %b) {
 }
 
 define float @maxnum_ogt_fmf_on_fcmp(float %a, float %b) {
-; CHECK-LABEL: @maxnum_ogt_fmf_on_fcmp(
+; CHECK-LABEL: define {{[^@]+}}@maxnum_ogt_fmf_on_fcmp(
 ; CHECK-NEXT:    [[COND:%.*]] = fcmp nnan nsz ogt float [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    [[F:%.*]] = select i1 [[COND]], float [[A]], float [[B]]
 ; CHECK-NEXT:    ret float [[F]]
@@ -343,7 +343,7 @@ define float @maxnum_ogt_fmf_on_fcmp(float %a, float %b) {
 }
 
 define <2 x float> @maxnum_oge_fmf_on_fcmp(<2 x float> %a, <2 x float> %b) {
-; CHECK-LABEL: @maxnum_oge_fmf_on_fcmp(
+; CHECK-LABEL: define {{[^@]+}}@maxnum_oge_fmf_on_fcmp(
 ; CHECK-NEXT:    [[COND:%.*]] = fcmp nnan ninf nsz oge <2 x float> [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    [[F:%.*]] = select <2 x i1> [[COND]], <2 x float> [[A]], <2 x float> [[B]]
 ; CHECK-NEXT:    ret <2 x float> [[F]]
@@ -354,7 +354,7 @@ define <2 x float> @maxnum_oge_fmf_on_fcmp(<2 x float> %a, <2 x float> %b) {
 }
 
 define float @maxnum_no_nsz(float %a, float %b) {
-; CHECK-LABEL: @maxnum_no_nsz(
+; CHECK-LABEL: define {{[^@]+}}@maxnum_no_nsz(
 ; CHECK-NEXT:    [[COND:%.*]] = fcmp ogt float [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    [[F:%.*]] = select nnan i1 [[COND]], float [[A]], float [[B]]
 ; CHECK-NEXT:    ret float [[F]]
@@ -365,7 +365,7 @@ define float @maxnum_no_nsz(float %a, float %b) {
 }
 
 define float @maxnum_no_nnan(float %a, float %b) {
-; CHECK-LABEL: @maxnum_no_nnan(
+; CHECK-LABEL: define {{[^@]+}}@maxnum_no_nnan(
 ; CHECK-NEXT:    [[COND:%.*]] = fcmp oge float [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    [[F:%.*]] = select nsz i1 [[COND]], float [[A]], float [[B]]
 ; CHECK-NEXT:    ret float [[F]]
@@ -376,7 +376,7 @@ define float @maxnum_no_nnan(float %a, float %b) {
 }
 
 define float @minnum_olt_fmf_on_select(float %a, float %b) {
-; CHECK-LABEL: @minnum_olt_fmf_on_select(
+; CHECK-LABEL: define {{[^@]+}}@minnum_olt_fmf_on_select(
 ; CHECK-NEXT:    [[TMP1:%.*]] = call nnan nsz float @llvm.minnum.f32(float [[A:%.*]], float [[B:%.*]])
 ; CHECK-NEXT:    ret float [[TMP1]]
 ;
@@ -386,7 +386,7 @@ define float @minnum_olt_fmf_on_select(float %a, float %b) {
 }
 
 define <2 x float> @minnum_ole_fmf_on_select(<2 x float> %a, <2 x float> %b) {
-; CHECK-LABEL: @minnum_ole_fmf_on_select(
+; CHECK-LABEL: define {{[^@]+}}@minnum_ole_fmf_on_select(
 ; CHECK-NEXT:    [[TMP1:%.*]] = call nnan ninf nsz <2 x float> @llvm.minnum.v2f32(<2 x float> [[A:%.*]], <2 x float> [[B:%.*]])
 ; CHECK-NEXT:    ret <2 x float> [[TMP1]]
 ;
@@ -396,7 +396,7 @@ define <2 x float> @minnum_ole_fmf_on_select(<2 x float> %a, <2 x float> %b) {
 }
 
 define float @minnum_olt_fmf_on_fcmp(float %a, float %b) {
-; CHECK-LABEL: @minnum_olt_fmf_on_fcmp(
+; CHECK-LABEL: define {{[^@]+}}@minnum_olt_fmf_on_fcmp(
 ; CHECK-NEXT:    [[COND:%.*]] = fcmp nnan nsz olt float [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    [[F:%.*]] = select i1 [[COND]], float [[A]], float [[B]]
 ; CHECK-NEXT:    ret float [[F]]
@@ -407,7 +407,7 @@ define float @minnum_olt_fmf_on_fcmp(float %a, float %b) {
 }
 
 define <2 x float> @minnum_ole_fmf_on_fcmp(<2 x float> %a, <2 x float> %b) {
-; CHECK-LABEL: @minnum_ole_fmf_on_fcmp(
+; CHECK-LABEL: define {{[^@]+}}@minnum_ole_fmf_on_fcmp(
 ; CHECK-NEXT:    [[COND:%.*]] = fcmp nnan ninf nsz ole <2 x float> [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    [[F:%.*]] = select <2 x i1> [[COND]], <2 x float> [[A]], <2 x float> [[B]]
 ; CHECK-NEXT:    ret <2 x float> [[F]]
@@ -418,7 +418,7 @@ define <2 x float> @minnum_ole_fmf_on_fcmp(<2 x float> %a, <2 x float> %b) {
 }
 
 define float @minnum_no_nsz(float %a, float %b) {
-; CHECK-LABEL: @minnum_no_nsz(
+; CHECK-LABEL: define {{[^@]+}}@minnum_no_nsz(
 ; CHECK-NEXT:    [[COND:%.*]] = fcmp olt float [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    [[F:%.*]] = select nnan i1 [[COND]], float [[A]], float [[B]]
 ; CHECK-NEXT:    ret float [[F]]
@@ -429,7 +429,7 @@ define float @minnum_no_nsz(float %a, float %b) {
 }
 
 define float @minnum_no_nnan(float %a, float %b) {
-; CHECK-LABEL: @minnum_no_nnan(
+; CHECK-LABEL: define {{[^@]+}}@minnum_no_nnan(
 ; CHECK-NEXT:    [[COND:%.*]] = fcmp ole float [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    [[F:%.*]] = select nsz i1 [[COND]], float [[A]], float [[B]]
 ; CHECK-NEXT:    ret float [[F]]
diff --git a/llvm/test/Transforms/InstCombine/minmax-of-minmax.ll b/llvm/test/Transforms/InstCombine/minmax-of-minmax.ll
new file mode 100644
index 000000000..58f2099e4
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/minmax-of-minmax.ll
@@ -0,0 +1,326 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+define i32 @smax_of_smax_smin_commute0(i32 %x, i32 %y) {
+; CHECK-LABEL: @smax_of_smax_smin_commute0(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[MAX:%.*]] = select i1 [[CMP2]], i32 [[X]], i32 [[Y]]
+; CHECK-NEXT:    ret i32 [[MAX]]
+;
+  %cmp1 = icmp slt i32 %x, %y
+  %min = select i1 %cmp1, i32 %x, i32 %y
+  %cmp2 = icmp slt i32 %y, %x
+  %max = select i1 %cmp2, i32 %x, i32 %y
+  %cmp3 = icmp sgt i32 %max, %min
+  %r = select i1 %cmp3, i32 %max, i32 %min
+  ret i32 %r
+}
+
+define i32 @smax_of_smax_smin_commute1(i32 %x, i32 %y) {
+; CHECK-LABEL: @smax_of_smax_smin_commute1(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[MAX:%.*]] = select i1 [[CMP2]], i32 [[X]], i32 [[Y]]
+; CHECK-NEXT:    ret i32 [[MAX]]
+;
+  %cmp1 = icmp sgt i32 %x, %y
+  %min = select i1 %cmp1, i32 %y, i32 %x
+  %cmp2 = icmp sgt i32 %x, %y
+  %max = select i1 %cmp2, i32 %x, i32 %y
+  %cmp3 = icmp sgt i32 %max, %min
+  %r = select i1 %cmp3, i32 %max, i32 %min
+  ret i32 %r
+}
+
+define i32 @smax_of_smax_smin_commute2(i32 %x, i32 %y) {
+; CHECK-LABEL: @smax_of_smax_smin_commute2(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[MAX:%.*]] = select i1 [[CMP2]], i32 [[X]], i32 [[Y]]
+; CHECK-NEXT:    ret i32 [[MAX]]
+;
+  %cmp1 = icmp slt i32 %x, %y
+  %min = select i1 %cmp1, i32 %x, i32 %y
+  %cmp2 = icmp slt i32 %y, %x
+  %max = select i1 %cmp2, i32 %x, i32 %y
+  %cmp3 = icmp slt i32 %min, %max
+  %r = select i1 %cmp3, i32 %max, i32 %min
+  ret i32 %r
+}
+
+define <2 x i32> @smax_of_smax_smin_commute3(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @smax_of_smax_smin_commute3(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt <2 x i32> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[MAX:%.*]] = select <2 x i1> [[CMP2]], <2 x i32> [[X]], <2 x i32> [[Y]]
+; CHECK-NEXT:    ret <2 x i32> [[MAX]]
+;
+  %cmp1 = icmp sgt <2 x i32> %x, %y
+  %min = select <2 x i1> %cmp1, <2 x i32> %y, <2 x i32> %x
+  %cmp2 = icmp sgt <2 x i32> %x, %y
+  %max = select <2 x i1> %cmp2, <2 x i32> %x, <2 x i32> %y
+  %cmp3 = icmp slt <2 x i32> %min, %max
+  %r = select <2 x i1> %cmp3, <2 x i32> %max, <2 x i32> %min
+  ret <2 x i32> %r
+}
+
+define i32 @smin_of_smin_smax_commute0(i32 %x, i32 %y) {
+; CHECK-LABEL: @smin_of_smin_smax_commute0(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i32 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[CMP2]], i32 [[X]], i32 [[Y]]
+; CHECK-NEXT:    ret i32 [[MIN]]
+;
+  %cmp1 = icmp sgt i32 %x, %y
+  %max = select i1 %cmp1, i32 %x, i32 %y
+  %cmp2 = icmp sgt i32 %y, %x
+  %min = select i1 %cmp2, i32 %x, i32 %y
+  %cmp3 = icmp sgt i32 %max, %min
+  %r = select i1 %cmp3, i32 %min, i32 %max
+  ret i32 %r
+}
+
+define i32 @smin_of_smin_smax_commute1(i32 %x, i32 %y) {
+; CHECK-LABEL: @smin_of_smin_smax_commute1(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[CMP2]], i32 [[X]], i32 [[Y]]
+; CHECK-NEXT:    ret i32 [[MIN]]
+;
+  %cmp1 = icmp slt i32 %x, %y
+  %max = select i1 %cmp1, i32 %y, i32 %x
+  %cmp2 = icmp slt i32 %x, %y
+  %min = select i1 %cmp2, i32 %x, i32 %y
+  %cmp3 = icmp sgt i32 %max, %min
+  %r = select i1 %cmp3, i32 %min, i32 %max
+  ret i32 %r
+}
+
+define <2 x i32> @smin_of_smin_smax_commute2(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @smin_of_smin_smax_commute2(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp slt <2 x i32> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[MIN:%.*]] = select <2 x i1> [[CMP2]], <2 x i32> [[X]], <2 x i32> [[Y]]
+; CHECK-NEXT:    ret <2 x i32> [[MIN]]
+;
+  %cmp1 = icmp sgt <2 x i32> %x, %y
+  %max = select <2 x i1> %cmp1, <2 x i32> %x, <2 x i32> %y
+  %cmp2 = icmp slt <2 x i32> %x, %y
+  %min = select <2 x i1> %cmp2, <2 x i32> %x, <2 x i32> %y
+  %cmp3 = icmp slt <2 x i32> %min, %max
+  %r = select <2 x i1> %cmp3, <2 x i32> %min, <2 x i32> %max
+  ret <2 x i32> %r
+}
+
+define i32 @smin_of_smin_smax_commute3(i32 %x, i32 %y) {
+; CHECK-LABEL: @smin_of_smin_smax_commute3(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i32 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[CMP2]], i32 [[X]], i32 [[Y]]
+; CHECK-NEXT:    ret i32 [[MIN]]
+;
+  %cmp1 = icmp slt i32 %x, %y
+  %max = select i1 %cmp1, i32 %y, i32 %x
+  %cmp2 = icmp sgt i32 %y, %x
+  %min = select i1 %cmp2, i32 %x, i32 %y
+  %cmp3 = icmp slt i32 %min, %max
+  %r = select i1 %cmp3, i32 %min, i32 %max
+  ret i32 %r
+}
+
+define i32 @umax_of_umax_umin_commute0(i32 %x, i32 %y) {
+; CHECK-LABEL: @umax_of_umax_umin_commute0(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult i32 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[MAX:%.*]] = select i1 [[CMP2]], i32 [[X]], i32 [[Y]]
+; CHECK-NEXT:    ret i32 [[MAX]]
+;
+  %cmp1 = icmp ult i32 %x, %y
+  %min = select i1 %cmp1, i32 %x, i32 %y
+  %cmp2 = icmp ult i32 %y, %x
+  %max = select i1 %cmp2, i32 %x, i32 %y
+  %cmp3 = icmp ugt i32 %max, %min
+  %r = select i1 %cmp3, i32 %max, i32 %min
+  ret i32 %r
+}
+
+define i32 @umax_of_umax_umin_commute1(i32 %x, i32 %y) {
+; CHECK-LABEL: @umax_of_umax_umin_commute1(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ugt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[MAX:%.*]] = select i1 [[CMP2]], i32 [[X]], i32 [[Y]]
+; CHECK-NEXT:    ret i32 [[MAX]]
+;
+  %cmp1 = icmp ugt i32 %x, %y
+  %min = select i1 %cmp1, i32 %y, i32 %x
+  %cmp2 = icmp ugt i32 %x, %y
+  %max = select i1 %cmp2, i32 %x, i32 %y
+  %cmp3 = icmp ugt i32 %max, %min
+  %r = select i1 %cmp3, i32 %max, i32 %min
+  ret i32 %r
+}
+
+define i32 @umax_of_umax_umin_commute2(i32 %x, i32 %y) {
+; CHECK-LABEL: @umax_of_umax_umin_commute2(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult i32 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[MAX:%.*]] = select i1 [[CMP2]], i32 [[X]], i32 [[Y]]
+; CHECK-NEXT:    ret i32 [[MAX]]
+;
+  %cmp1 = icmp ult i32 %x, %y
+  %min = select i1 %cmp1, i32 %x, i32 %y
+  %cmp2 = icmp ult i32 %y, %x
+  %max = select i1 %cmp2, i32 %x, i32 %y
+  %cmp3 = icmp ult i32 %min, %max
+  %r = select i1 %cmp3, i32 %max, i32 %min
+  ret i32 %r
+}
+
+define <2 x i32> @umax_of_umax_umin_commute3(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @umax_of_umax_umin_commute3(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ugt <2 x i32> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[MAX:%.*]] = select <2 x i1> [[CMP2]], <2 x i32> [[X]], <2 x i32> [[Y]]
+; CHECK-NEXT:    ret <2 x i32> [[MAX]]
+;
+  %cmp1 = icmp ugt <2 x i32> %x, %y
+  %min = select <2 x i1> %cmp1, <2 x i32> %y, <2 x i32> %x
+  %cmp2 = icmp ugt <2 x i32> %x, %y
+  %max = select <2 x i1> %cmp2, <2 x i32> %x, <2 x i32> %y
+  %cmp3 = icmp ult <2 x i32> %min, %max
+  %r = select <2 x i1> %cmp3, <2 x i32> %max, <2 x i32> %min
+  ret <2 x i32> %r
+}
+
+define i32 @umin_of_umin_umax_commute0(i32 %x, i32 %y) {
+; CHECK-LABEL: @umin_of_umin_umax_commute0(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ugt i32 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[CMP2]], i32 [[X]], i32 [[Y]]
+; CHECK-NEXT:    ret i32 [[MIN]]
+;
+  %cmp1 = icmp ugt i32 %x, %y
+  %max = select i1 %cmp1, i32 %x, i32 %y
+  %cmp2 = icmp ugt i32 %y, %x
+  %min = select i1 %cmp2, i32 %x, i32 %y
+  %cmp3 = icmp ugt i32 %max, %min
+  %r = select i1 %cmp3, i32 %min, i32 %max
+  ret i32 %r
+}
+
+define i32 @umin_of_umin_umax_commute1(i32 %x, i32 %y) {
+; CHECK-LABEL: @umin_of_umin_umax_commute1(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[CMP2]], i32 [[X]], i32 [[Y]]
+; CHECK-NEXT:    ret i32 [[MIN]]
+;
+  %cmp1 = icmp ult i32 %x, %y
+  %max = select i1 %cmp1, i32 %y, i32 %x
+  %cmp2 = icmp ult i32 %x, %y
+  %min = select i1 %cmp2, i32 %x, i32 %y
+  %cmp3 = icmp ugt i32 %max, %min
+  %r = select i1 %cmp3, i32 %min, i32 %max
+  ret i32 %r
+}
+
+define <2 x i32> @umin_of_umin_umax_commute2(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @umin_of_umin_umax_commute2(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult <2 x i32> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[MIN:%.*]] = select <2 x i1> [[CMP2]], <2 x i32> [[X]], <2 x i32> [[Y]]
+; CHECK-NEXT:    ret <2 x i32> [[MIN]]
+;
+  %cmp1 = icmp ugt <2 x i32> %x, %y
+  %max = select <2 x i1> %cmp1, <2 x i32> %x, <2 x i32> %y
+  %cmp2 = icmp ult <2 x i32> %x, %y
+  %min = select <2 x i1> %cmp2, <2 x i32> %x, <2 x i32> %y
+  %cmp3 = icmp ult <2 x i32> %min, %max
+  %r = select <2 x i1> %cmp3, <2 x i32> %min, <2 x i32> %max
+  ret <2 x i32> %r
+}
+
+define i32 @umin_of_umin_umax_commute3(i32 %x, i32 %y) {
+; CHECK-LABEL: @umin_of_umin_umax_commute3(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ugt i32 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[CMP2]], i32 [[X]], i32 [[Y]]
+; CHECK-NEXT:    ret i32 [[MIN]]
+;
+  %cmp1 = icmp ult i32 %x, %y
+  %max = select i1 %cmp1, i32 %y, i32 %x
+  %cmp2 = icmp ugt i32 %y, %x
+  %min = select i1 %cmp2, i32 %x, i32 %y
+  %cmp3 = icmp ult i32 %min, %max
+  %r = select i1 %cmp3, i32 %min, i32 %max
+  ret i32 %r
+}
+
+; Negative test - mismatch of min/max flavor
+
+define i32 @umin_of_smin_umax_wrong_pattern(i32 %x, i32 %y) {
+; CHECK-LABEL: @umin_of_smin_umax_wrong_pattern(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ugt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[MAX:%.*]] = select i1 [[CMP1]], i32 [[X]], i32 [[Y]]
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i32 [[Y]], [[X]]
+; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[CMP2]], i32 [[X]], i32 [[Y]]
+; CHECK-NEXT:    [[CMP3:%.*]] = icmp ugt i32 [[MAX]], [[MIN]]
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[CMP3]], i32 [[MIN]], i32 [[MAX]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %cmp1 = icmp ugt i32 %x, %y
+  %max = select i1 %cmp1, i32 %x, i32 %y
+  %cmp2 = icmp sgt i32 %y, %x
+  %min = select i1 %cmp2, i32 %x, i32 %y
+  %cmp3 = icmp ugt i32 %max, %min
+  %r = select i1 %cmp3, i32 %min, i32 %max
+  ret i32 %r
+}
+
+; Negative test - mismatch of min/max flavor
+
+define i32 @smin_of_umin_umax_wrong_pattern2(i32 %x, i32 %y) {
+; CHECK-LABEL: @smin_of_umin_umax_wrong_pattern2(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[MAX:%.*]] = select i1 [[CMP1]], i32 [[Y]], i32 [[X]]
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[CMP2]], i32 [[X]], i32 [[Y]]
+; CHECK-NEXT:    [[CMP3:%.*]] = icmp sgt i32 [[MAX]], [[MIN]]
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[CMP3]], i32 [[MIN]], i32 [[MAX]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %cmp1 = icmp ult i32 %x, %y
+  %max = select i1 %cmp1, i32 %y, i32 %x
+  %cmp2 = icmp ult i32 %x, %y
+  %min = select i1 %cmp2, i32 %x, i32 %y
+  %cmp3 = icmp sgt i32 %max, %min
+  %r = select i1 %cmp3, i32 %min, i32 %max
+  ret i32 %r
+}
+
+; Negative test - operands must match
+
+define <2 x i32> @umin_of_umin_umax_wrong_operand(<2 x i32> %x, <2 x i32> %y, <2 x i32> %z) {
+; CHECK-LABEL: @umin_of_umin_umax_wrong_operand(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ugt <2 x i32> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[MAX:%.*]] = select <2 x i1> [[CMP1]], <2 x i32> [[X]], <2 x i32> [[Y]]
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult <2 x i32> [[X]], [[Z:%.*]]
+; CHECK-NEXT:    [[MIN:%.*]] = select <2 x i1> [[CMP2]], <2 x i32> [[X]], <2 x i32> [[Z]]
+; CHECK-NEXT:    [[CMP3:%.*]] = icmp ult <2 x i32> [[MIN]], [[MAX]]
+; CHECK-NEXT:    [[R:%.*]] = select <2 x i1> [[CMP3]], <2 x i32> [[MIN]], <2 x i32> [[MAX]]
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %cmp1 = icmp ugt <2 x i32> %x, %y
+  %max = select <2 x i1> %cmp1, <2 x i32> %x, <2 x i32> %y
+  %cmp2 = icmp ult <2 x i32> %x, %z
+  %min = select <2 x i1> %cmp2, <2 x i32> %x, <2 x i32> %z
+  %cmp3 = icmp ult <2 x i32> %min, %max
+  %r = select <2 x i1> %cmp3, <2 x i32> %min, <2 x i32> %max
+  ret <2 x i32> %r
+}
+
+; Negative test - operands must match
+
+define i32 @umin_of_umin_umax_wrong_operand2(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @umin_of_umin_umax_wrong_operand2(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult i32 [[X:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    [[MAX:%.*]] = select i1 [[CMP1]], i32 [[Z]], i32 [[X]]
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ugt i32 [[Y:%.*]], [[X]]
+; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[CMP2]], i32 [[X]], i32 [[Y]]
+; CHECK-NEXT:    [[CMP3:%.*]] = icmp ult i32 [[MIN]], [[MAX]]
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[CMP3]], i32 [[MIN]], i32 [[MAX]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %cmp1 = icmp ult i32 %x, %z
+  %max = select i1 %cmp1, i32 %z, i32 %x
+  %cmp2 = icmp ugt i32 %y, %x
+  %min = select i1 %cmp2, i32 %x, i32 %y
+  %cmp3 = icmp ult i32 %min, %max
+  %r = select i1 %cmp3, i32 %min, i32 %max
+  ret i32 %r
+}
diff --git a/llvm/test/Transforms/InstCombine/mul.ll b/llvm/test/Transforms/InstCombine/mul.ll
index b1dc9d513..be8b2459e 100644
--- a/llvm/test/Transforms/InstCombine/mul.ll
+++ b/llvm/test/Transforms/InstCombine/mul.ll
@@ -517,3 +517,93 @@ define i64 @test_mul_canonicalize_neg_is_not_undone(i64 %L1) {
   %B4 = mul i64 %B8, %L1
   ret i64 %B4
 }
+
+define i32 @negate_if_true(i32 %x, i1 %cond) {
+; CHECK-LABEL: @negate_if_true(
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i32 0, [[X:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[COND:%.*]], i32 [[TMP1]], i32 [[X]]
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
+  %sel = select i1 %cond, i32 -1, i32 1
+  %r = mul i32 %sel, %x
+  ret i32 %r
+}
+
+define i32 @negate_if_false(i32 %x, i1 %cond) {
+; CHECK-LABEL: @negate_if_false(
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i32 0, [[X:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[COND:%.*]], i32 [[X]], i32 [[TMP1]]
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
+  %sel = select i1 %cond, i32 1, i32 -1
+  %r = mul i32 %sel, %x
+  ret i32 %r
+}
+
+define <2 x i8> @negate_if_true_commute(<2 x i8> %px, i1 %cond) {
+; CHECK-LABEL: @negate_if_true_commute(
+; CHECK-NEXT:    [[X:%.*]] = sdiv <2 x i8> <i8 42, i8 42>, [[PX:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = sub nsw <2 x i8> zeroinitializer, [[X]]
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[COND:%.*]], <2 x i8> [[TMP1]], <2 x i8> [[X]]
+; CHECK-NEXT:    ret <2 x i8> [[TMP2]]
+;
+  %x = sdiv <2 x i8> <i8 42, i8 42>, %px  ; thwart complexity-based canonicalization
+  %sel = select i1 %cond, <2 x i8> <i8 -1, i8 -1>, <2 x i8> <i8 1, i8 1>
+  %r = mul <2 x i8> %x, %sel
+  ret <2 x i8> %r
+}
+
+define <2 x i8> @negate_if_false_commute(<2 x i8> %px, <2 x i1> %cond) {
+; CHECK-LABEL: @negate_if_false_commute(
+; CHECK-NEXT:    [[X:%.*]] = sdiv <2 x i8> <i8 42, i8 5>, [[PX:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = sub <2 x i8> zeroinitializer, [[X]]
+; CHECK-NEXT:    [[TMP2:%.*]] = select <2 x i1> [[COND:%.*]], <2 x i8> [[X]], <2 x i8> [[TMP1]]
+; CHECK-NEXT:    ret <2 x i8> [[TMP2]]
+;
+  %x = sdiv <2 x i8> <i8 42, i8 5>, %px  ; thwart complexity-based canonicalization
+  %sel = select <2 x i1> %cond, <2 x i8> <i8 1, i8 undef>, <2 x i8> <i8 -1, i8 -1>
+  %r = mul <2 x i8> %x, %sel
+  ret <2 x i8> %r
+}
+
+; Negative test
+
+define i32 @negate_if_true_extra_use(i32 %x, i1 %cond) {
+; CHECK-LABEL: @negate_if_true_extra_use(
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[COND:%.*]], i32 -1, i32 1
+; CHECK-NEXT:    call void @use32(i32 [[SEL]])
+; CHECK-NEXT:    [[R:%.*]] = mul i32 [[SEL]], [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %sel = select i1 %cond, i32 -1, i32 1
+  call void @use32(i32 %sel)
+  %r = mul i32 %sel, %x
+  ret i32 %r
+}
+
+; Negative test
+
+define <2 x i8> @negate_if_true_wrong_constant(<2 x i8> %px, i1 %cond) {
+; CHECK-LABEL: @negate_if_true_wrong_constant(
+; CHECK-NEXT:    [[X:%.*]] = sdiv <2 x i8> <i8 42, i8 42>, [[PX:%.*]]
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[COND:%.*]], <2 x i8> <i8 -1, i8 0>, <2 x i8> <i8 1, i8 1>
+; CHECK-NEXT:    [[R:%.*]] = mul <2 x i8> [[X]], [[SEL]]
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %x = sdiv <2 x i8> <i8 42, i8 42>, %px  ; thwart complexity-based canonicalization
+  %sel = select i1 %cond, <2 x i8> <i8 -1, i8 0>, <2 x i8> <i8 1, i8 1>
+  %r = mul <2 x i8> %x, %sel
+  ret <2 x i8> %r
+}
+
+; (C ? (X /exact Y) : 1) * Y -> C ? X : Y
+define i32 @mul_div_select(i32 %x, i32 %y, i1 %c) {
+; CHECK-LABEL: @mul_div_select(
+; CHECK-NEXT:    [[MUL:%.*]] = select i1 [[C:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[MUL]]
+;
+  %div = udiv exact i32 %x, %y
+  %sel = select i1 %c, i32 %div, i32 1
+  %mul = mul i32 %sel, %y
+  ret i32 %mul
+}
diff --git a/llvm/test/Transforms/InstCombine/not.ll b/llvm/test/Transforms/InstCombine/not.ll
index 42a910aa3..6c7d48630 100644
--- a/llvm/test/Transforms/InstCombine/not.ll
+++ b/llvm/test/Transforms/InstCombine/not.ll
@@ -251,3 +251,156 @@ define <2 x i32> @not_add_vec(<2 x i32> %x) {
   ret <2 x i32> %r
 }
 
+define i1 @not_select_cmp_cmp(i32 %x, i32 %y, float %z, float %w, i1 %cond) {
+; CHECK-LABEL: @not_select_cmp_cmp(
+; CHECK-NEXT:    [[CMPT:%.*]] = icmp sgt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[CMPF:%.*]] = fcmp ole float [[Z:%.*]], [[W:%.*]]
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[COND:%.*]], i1 [[CMPT]], i1 [[CMPF]]
+; CHECK-NEXT:    ret i1 [[SEL]]
+;
+  %cmpt = icmp sle i32 %x, %y
+  %cmpf = fcmp ugt float %z, %w
+  %sel = select i1 %cond, i1 %cmpt, i1 %cmpf
+  %not = xor i1 %sel, true
+  ret i1 %not
+}
+
+declare void @use1(i1)
+
+; TODO: Missed canonicalization - hoist 'not'?
+
+define i1 @not_select_cmp_cmp_extra_use1(i32 %x, i32 %y, float %z, float %w, i1 %cond) {
+; CHECK-LABEL: @not_select_cmp_cmp_extra_use1(
+; CHECK-NEXT:    [[CMPT:%.*]] = icmp sle i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use1(i1 [[CMPT]])
+; CHECK-NEXT:    [[CMPF:%.*]] = fcmp ugt float [[Z:%.*]], [[W:%.*]]
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[COND:%.*]], i1 [[CMPT]], i1 [[CMPF]]
+; CHECK-NEXT:    [[NOT:%.*]] = xor i1 [[SEL]], true
+; CHECK-NEXT:    ret i1 [[NOT]]
+;
+  %cmpt = icmp sle i32 %x, %y
+  call void @use1(i1 %cmpt)
+  %cmpf = fcmp ugt float %z, %w
+  %sel = select i1 %cond, i1 %cmpt, i1 %cmpf
+  %not = xor i1 %sel, true
+  ret i1 %not
+}
+
+; TODO: Missed canonicalization - hoist 'not'?
+
+define i1 @not_select_cmp_cmp_extra_use2(i32 %x, i32 %y, float %z, float %w, i1 %cond) {
+; CHECK-LABEL: @not_select_cmp_cmp_extra_use2(
+; CHECK-NEXT:    [[CMPT:%.*]] = icmp sle i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[CMPF:%.*]] = fcmp ugt float [[Z:%.*]], [[W:%.*]]
+; CHECK-NEXT:    call void @use1(i1 [[CMPF]])
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[COND:%.*]], i1 [[CMPT]], i1 [[CMPF]]
+; CHECK-NEXT:    [[NOT:%.*]] = xor i1 [[SEL]], true
+; CHECK-NEXT:    ret i1 [[NOT]]
+;
+  %cmpt = icmp sle i32 %x, %y
+  %cmpf = fcmp ugt float %z, %w
+  call void @use1(i1 %cmpf)
+  %sel = select i1 %cond, i1 %cmpt, i1 %cmpf
+  %not = xor i1 %sel, true
+  ret i1 %not
+}
+
+; Negative test - extra uses would require more instructions.
+
+define i1 @not_select_cmp_cmp_extra_use3(i32 %x, i32 %y, float %z, float %w, i1 %cond) {
+; CHECK-LABEL: @not_select_cmp_cmp_extra_use3(
+; CHECK-NEXT:    [[CMPT:%.*]] = icmp sle i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use1(i1 [[CMPT]])
+; CHECK-NEXT:    [[CMPF:%.*]] = fcmp ugt float [[Z:%.*]], [[W:%.*]]
+; CHECK-NEXT:    call void @use1(i1 [[CMPF]])
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[COND:%.*]], i1 [[CMPT]], i1 [[CMPF]]
+; CHECK-NEXT:    [[NOT:%.*]] = xor i1 [[SEL]], true
+; CHECK-NEXT:    ret i1 [[NOT]]
+;
+  %cmpt = icmp sle i32 %x, %y
+  call void @use1(i1 %cmpt)
+  %cmpf = fcmp ugt float %z, %w
+  call void @use1(i1 %cmpf)
+  %sel = select i1 %cond, i1 %cmpt, i1 %cmpf
+  %not = xor i1 %sel, true
+  ret i1 %not
+}
+
+; Negative test - extra uses would require more instructions.
+
+define i1 @not_select_cmp_cmp_extra_use4(i32 %x, i32 %y, float %z, float %w, i1 %cond) {
+; CHECK-LABEL: @not_select_cmp_cmp_extra_use4(
+; CHECK-NEXT:    [[CMPT:%.*]] = icmp sle i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[CMPF:%.*]] = fcmp ugt float [[Z:%.*]], [[W:%.*]]
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[COND:%.*]], i1 [[CMPT]], i1 [[CMPF]]
+; CHECK-NEXT:    call void @use1(i1 [[SEL]])
+; CHECK-NEXT:    [[NOT:%.*]] = xor i1 [[SEL]], true
+; CHECK-NEXT:    ret i1 [[NOT]]
+;
+  %cmpt = icmp sle i32 %x, %y
+  %cmpf = fcmp ugt float %z, %w
+  %sel = select i1 %cond, i1 %cmpt, i1 %cmpf
+  call void @use1(i1 %sel)
+  %not = xor i1 %sel, true
+  ret i1 %not
+}
+
+; TODO: Missed canonicalization - hoist 'not'?
+
+define i1 @not_select_cmpt(double %x, double %y, i1 %z, i1 %cond) {
+; CHECK-LABEL: @not_select_cmpt(
+; CHECK-NEXT:    [[CMPT:%.*]] = fcmp oeq double [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[COND:%.*]], i1 [[CMPT]], i1 [[Z:%.*]]
+; CHECK-NEXT:    [[NOT:%.*]] = xor i1 [[SEL]], true
+; CHECK-NEXT:    ret i1 [[NOT]]
+;
+  %cmpt = fcmp oeq double %x, %y
+  %sel = select i1 %cond, i1 %cmpt, i1 %z
+  %not = xor i1 %sel, true
+  ret i1 %not
+}
+
+; TODO: Missed canonicalization - hoist 'not'?
+
+define i1 @not_select_cmpf(i1 %x, i32 %z, i32 %w, i1 %cond) {
+; CHECK-LABEL: @not_select_cmpf(
+; CHECK-NEXT:    [[CMPF:%.*]] = icmp ugt i32 [[Z:%.*]], [[W:%.*]]
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[COND:%.*]], i1 [[X:%.*]], i1 [[CMPF]]
+; CHECK-NEXT:    [[NOT:%.*]] = xor i1 [[SEL]], true
+; CHECK-NEXT:    ret i1 [[NOT]]
+;
+  %cmpf = icmp ugt i32 %z, %w
+  %sel = select i1 %cond, i1 %x, i1 %cmpf
+  %not = xor i1 %sel, true
+  ret i1 %not
+}
+
+define i1 @not_select_cmpt_extra_use(double %x, double %y, i1 %z, i1 %cond) {
+; CHECK-LABEL: @not_select_cmpt_extra_use(
+; CHECK-NEXT:    [[CMPT:%.*]] = fcmp oeq double [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use1(i1 [[CMPT]])
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[COND:%.*]], i1 [[CMPT]], i1 [[Z:%.*]]
+; CHECK-NEXT:    [[NOT:%.*]] = xor i1 [[SEL]], true
+; CHECK-NEXT:    ret i1 [[NOT]]
+;
+  %cmpt = fcmp oeq double %x, %y
+  call void @use1(i1 %cmpt)
+  %sel = select i1 %cond, i1 %cmpt, i1 %z
+  %not = xor i1 %sel, true
+  ret i1 %not
+}
+
+define i1 @not_select_cmpf_extra_use(i1 %x, i32 %z, i32 %w, i1 %cond) {
+; CHECK-LABEL: @not_select_cmpf_extra_use(
+; CHECK-NEXT:    [[CMPF:%.*]] = icmp ugt i32 [[Z:%.*]], [[W:%.*]]
+; CHECK-NEXT:    call void @use1(i1 [[CMPF]])
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[COND:%.*]], i1 [[X:%.*]], i1 [[CMPF]]
+; CHECK-NEXT:    [[NOT:%.*]] = xor i1 [[SEL]], true
+; CHECK-NEXT:    ret i1 [[NOT]]
+;
+  %cmpf = icmp ugt i32 %z, %w
+  call void @use1(i1 %cmpf)
+  %sel = select i1 %cond, i1 %x, i1 %cmpf
+  %not = xor i1 %sel, true
+  ret i1 %not
+}
diff --git a/llvm/test/Transforms/InstCombine/objsize-64.ll b/llvm/test/Transforms/InstCombine/objsize-64.ll
index 866bc4f93..06ad232fc 100644
--- a/llvm/test/Transforms/InstCombine/objsize-64.ll
+++ b/llvm/test/Transforms/InstCombine/objsize-64.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -instcombine -S | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
@@ -7,32 +8,46 @@ declare i32 @__gxx_personality_v0(...)
 declare void @__cxa_call_unexpected(i8*)
 declare i64 @llvm.objectsize.i64(i8*, i1) nounwind readonly
 
-; CHECK-LABEL: @f1(
 define i64 @f1(i8 **%esc) {
+; CHECK-LABEL: @f1(
+; CHECK-NEXT:    [[CALL:%.*]] = call dereferenceable_or_null(4) i8* @malloc(i32 4)
+; CHECK-NEXT:    store i8* [[CALL]], i8** [[ESC:%.*]], align 8
+; CHECK-NEXT:    ret i64 4
+;
   %call = call i8* @malloc(i32 4)
   store i8* %call, i8** %esc
   %size = call i64 @llvm.objectsize.i64(i8* %call, i1 false)
-; CHECK: ret i64 4
   ret i64 %size
 }
 
 
-; CHECK-LABEL: @f2(
 define i64 @f2(i8** %esc) nounwind uwtable ssp personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+; CHECK-LABEL: @f2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CALL:%.*]] = invoke noalias dereferenceable(13) i8* @_Znwm(i64 13)
+; CHECK-NEXT:    to label [[INVOKE_CONT:%.*]] unwind label [[LPAD:%.*]]
+; CHECK:       invoke.cont:
+; CHECK-NEXT:    store i8* [[CALL]], i8** [[ESC:%.*]], align 8
+; CHECK-NEXT:    ret i64 13
+; CHECK:       lpad:
+; CHECK-NEXT:    [[TMP0:%.*]] = landingpad { i8*, i32 }
+; CHECK-NEXT:    filter [0 x i8*] zeroinitializer
+; CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { i8*, i32 } [[TMP0]], 0
+; CHECK-NEXT:    tail call void @__cxa_call_unexpected(i8* [[TMP1]]) #3
+; CHECK-NEXT:    unreachable
+;
 entry:
-; CHECK: invoke noalias i8* @_Znwm(i64 13)
   %call = invoke noalias i8* @_Znwm(i64 13)
-          to label %invoke.cont unwind label %lpad
+  to label %invoke.cont unwind label %lpad
 
 invoke.cont:
-; CHECK: ret i64 13
   store i8* %call, i8** %esc
   %0 = tail call i64 @llvm.objectsize.i64(i8* %call, i1 false)
   ret i64 %0
 
 lpad:
   %1 = landingpad { i8*, i32 }
-          filter [0 x i8*] zeroinitializer
+  filter [0 x i8*] zeroinitializer
   %2 = extractvalue { i8*, i32 } %1, 0
   tail call void @__cxa_call_unexpected(i8* %2) noreturn nounwind
   unreachable
diff --git a/llvm/test/Transforms/InstCombine/objsize.ll b/llvm/test/Transforms/InstCombine/objsize.ll
index 97c708fb6..15f6b44a3 100644
--- a/llvm/test/Transforms/InstCombine/objsize.ll
+++ b/llvm/test/Transforms/InstCombine/objsize.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; Test a pile of objectsize bounds checking.
 ; RUN: opt < %s -instcombine -S | FileCheck %s
 ; We need target data to get the sizes of the arrays and structures.
@@ -7,18 +8,27 @@ target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f3
 @.str = private constant [8 x i8] c"abcdefg\00"   ; <[8 x i8]*>
 define i32 @foo() nounwind {
 ; CHECK-LABEL: @foo(
-; CHECK-NEXT: ret i32 60
+; CHECK-NEXT:    ret i32 60
+;
   %1 = call i32 @llvm.objectsize.i32.p0i8(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i1 false, i1 false, i1 false)
   ret i32 %1
 }
 
 define i8* @bar() nounwind {
 ; CHECK-LABEL: @bar(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[RETVAL:%.*]] = alloca i8*, align 4
+; CHECK-NEXT:    br i1 true, label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+; CHECK:       cond.true:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8*, i8** [[RETVAL]], align 4
+; CHECK-NEXT:    ret i8* [[TMP0]]
+; CHECK:       cond.false:
+; CHECK-NEXT:    ret i8* undef
+;
 entry:
   %retval = alloca i8*
   %0 = call i32 @llvm.objectsize.i32.p0i8(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i1 false, i1 false, i1 false)
   %cmp = icmp ne i32 %0, -1
-; CHECK: br i1 true
   br i1 %cmp, label %cond.true, label %cond.false
 
 cond.true:
@@ -32,7 +42,8 @@ cond.false:
 
 define i32 @f() nounwind {
 ; CHECK-LABEL: @f(
-; CHECK-NEXT: ret i32 0
+; CHECK-NEXT:    ret i32 0
+;
   %1 = call i32 @llvm.objectsize.i32.p0i8(i8* getelementptr ([60 x i8], [60 x i8]* @a, i32 1, i32 0), i1 false, i1 false, i1 false)
   ret i32 %1
 }
@@ -41,7 +52,10 @@ define i32 @f() nounwind {
 
 define i1 @baz() nounwind {
 ; CHECK-LABEL: @baz(
-; CHECK-NEXT: objectsize
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call i32 @llvm.objectsize.i32.p0i8(i8* getelementptr inbounds ([0 x i8], [0 x i8]* @window, i32 0, i32 0), i1 false, i1 false, i1 false)
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], -1
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
   %1 = tail call i32 @llvm.objectsize.i32.p0i8(i8* getelementptr inbounds ([0 x i8], [0 x i8]* @window, i32 0, i32 0), i1 false, i1 false, i1 false)
   %2 = icmp eq i32 %1, -1
   ret i1 %2
@@ -49,7 +63,15 @@ define i1 @baz() nounwind {
 
 define void @test1(i8* %q, i32 %x) nounwind noinline {
 ; CHECK-LABEL: @test1(
-; CHECK: objectsize.i32.p0i8
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.objectsize.i32.p0i8(i8* getelementptr inbounds ([0 x i8], [0 x i8]* @window, i32 0, i32 10), i1 false, i1 false, i1 false)
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[TMP0]], -1
+; CHECK-NEXT:    br i1 [[TMP1]], label %"47", label %"46"
+; CHECK:       "46":
+; CHECK-NEXT:    unreachable
+; CHECK:       "47":
+; CHECK-NEXT:    unreachable
+;
 entry:
   %0 = call i32 @llvm.objectsize.i32.p0i8(i8* getelementptr inbounds ([0 x i8], [0 x i8]* @window, i32 0, i32 10), i1 false, i1 false, i1 false) ; <i64> [#uses=1]
   %1 = icmp eq i32 %0, -1                         ; <i1> [#uses=1]
@@ -66,7 +88,8 @@ entry:
  101, i32 102, i32 103, i32 0], align 4
 define i32 @test2() nounwind {
 ; CHECK-LABEL: @test2(
-; CHECK-NEXT: ret i32 34
+; CHECK-NEXT:    ret i32 34
+;
   %1 = call i32 @llvm.objectsize.i32.p0i8(i8* getelementptr (i8, i8* bitcast ([9 x i32]* @.str5 to i8*), i32 2), i1 false, i1 false, i1 false)
   ret i32 %1
 }
@@ -84,6 +107,14 @@ declare i8* @__inline_memcpy_chk(i8*, i8*, i32) nounwind inlinehint
 
 define void @test3() nounwind {
 ; CHECK-LABEL: @test3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 undef, label [[BB11:%.*]], label [[BB12:%.*]]
+; CHECK:       bb11:
+; CHECK-NEXT:    unreachable
+; CHECK:       bb12:
+; CHECK-NEXT:    [[TMP0:%.*]] = call i8* @__inline_memcpy_chk(i8* bitcast (float* getelementptr inbounds ([480 x float], [480 x float]* @array, i32 0, i32 1) to i8*), i8* undef, i32 512) #3
+; CHECK-NEXT:    unreachable
+;
 entry:
   br i1 undef, label %bb11, label %bb12
 
@@ -92,14 +123,12 @@ bb11:
   %1 = bitcast float* %0 to i8*                   ; <i8*> [#uses=1]
   %2 = call i32 @llvm.objectsize.i32.p0i8(i8* %1, i1 false, i1 false, i1 false) ; <i32> [#uses=1]
   %3 = call i8* @__memcpy_chk(i8* undef, i8* undef, i32 512, i32 %2) nounwind ; <i8*> [#uses=0]
-; CHECK: unreachable
   unreachable
 
 bb12:
   %4 = getelementptr inbounds float, float* getelementptr inbounds ([480 x float], [480 x float]* @array, i32 0, i32 128), i32 -127 ; <float*> [#uses=1]
   %5 = bitcast float* %4 to i8*                   ; <i8*> [#uses=1]
   %6 = call i8* @__inline_memcpy_chk(i8* %5, i8* undef, i32 512) nounwind inlinehint ; <i8*> [#uses=0]
-; CHECK: @__inline_memcpy_chk
   unreachable
 }
 
@@ -109,12 +138,18 @@ bb12:
 
 define i32 @test4(i8** %esc) nounwind ssp {
 ; CHECK-LABEL: @test4(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = alloca [[STRUCT_DATA:%.*]], align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast %struct.data* [[TMP0]] to i8*
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i32(i8* nonnull align 8 dereferenceable(1824) [[TMP1]], i8 0, i32 1824, i1 false) #0
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8** [[ESC:%.*]] to %struct.data**
+; CHECK-NEXT:    store %struct.data* [[TMP0]], %struct.data** [[TMP2]], align 4
+; CHECK-NEXT:    ret i32 0
+;
 entry:
   %0 = alloca %struct.data, align 8
   %1 = bitcast %struct.data* %0 to i8*
   %2 = call i32 @llvm.objectsize.i32.p0i8(i8* %1, i1 false, i1 false, i1 false) nounwind
-; CHECK-NOT: @llvm.objectsize
-; CHECK: @llvm.memset.p0i8.i32(i8* nonnull align 8 %1, i8 0, i32 1824, i1 false)
   %3 = call i8* @__memset_chk(i8* %1, i32 0, i32 1824, i32 %2) nounwind
   store i8* %1, i8** %esc
   ret i32 0
@@ -125,24 +160,32 @@ entry:
 
 define i8* @test5(i32 %n) nounwind ssp {
 ; CHECK-LABEL: @test5(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = tail call noalias dereferenceable_or_null(20) i8* @malloc(i32 20) #0
+; CHECK-NEXT:    [[TMP1:%.*]] = load i8*, i8** @s, align 8
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* nonnull align 1 dereferenceable(10) [[TMP0]], i8* nonnull align 1 dereferenceable(10) [[TMP1]], i32 10, i1 false) #0
+; CHECK-NEXT:    ret i8* [[TMP0]]
+;
 entry:
   %0 = tail call noalias i8* @malloc(i32 20) nounwind
   %1 = tail call i32 @llvm.objectsize.i32.p0i8(i8* %0, i1 false, i1 false, i1 false)
   %2 = load i8*, i8** @s, align 8
-; CHECK-NOT: @llvm.objectsize
-; CHECK: @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 %0, i8* align 1 %1, i32 10, i1 false)
   %3 = tail call i8* @__memcpy_chk(i8* %0, i8* %2, i32 10, i32 %1) nounwind
   ret i8* %0
 }
 
 define void @test6(i32 %n) nounwind ssp {
 ; CHECK-LABEL: @test6(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = tail call noalias dereferenceable_or_null(20) i8* @malloc(i32 20) #0
+; CHECK-NEXT:    [[TMP1:%.*]] = load i8*, i8** @s, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = tail call i8* @__memcpy_chk(i8* [[TMP0]], i8* [[TMP1]], i32 30, i32 20) #0
+; CHECK-NEXT:    ret void
+;
 entry:
   %0 = tail call noalias i8* @malloc(i32 20) nounwind
   %1 = tail call i32 @llvm.objectsize.i32.p0i8(i8* %0, i1 false, i1 false, i1 false)
   %2 = load i8*, i8** @s, align 8
-; CHECK-NOT: @llvm.objectsize
-; CHECK: @__memcpy_chk(i8* %0, i8* %1, i32 30, i32 20)
   %3 = tail call i8* @__memcpy_chk(i8* %0, i8* %2, i32 30, i32 %1) nounwind
   ret void
 }
@@ -153,11 +196,14 @@ declare noalias i8* @malloc(i32) nounwind
 
 define i32 @test7(i8** %esc) {
 ; CHECK-LABEL: @test7(
+; CHECK-NEXT:    [[ALLOC:%.*]] = call noalias dereferenceable_or_null(48) i8* @malloc(i32 48) #0
+; CHECK-NEXT:    store i8* [[ALLOC]], i8** [[ESC:%.*]], align 4
+; CHECK-NEXT:    ret i32 32
+;
   %alloc = call noalias i8* @malloc(i32 48) nounwind
   store i8* %alloc, i8** %esc
   %gep = getelementptr inbounds i8, i8* %alloc, i32 16
   %objsize = call i32 @llvm.objectsize.i32.p0i8(i8* %gep, i1 false, i1 false, i1 false) nounwind readonly
-; CHECK: ret i32 32
   ret i32 %objsize
 }
 
@@ -165,67 +211,86 @@ declare noalias i8* @calloc(i32, i32) nounwind
 
 define i32 @test8(i8** %esc) {
 ; CHECK-LABEL: @test8(
+; CHECK-NEXT:    [[ALLOC:%.*]] = call noalias dereferenceable_or_null(35) i8* @calloc(i32 5, i32 7) #0
+; CHECK-NEXT:    store i8* [[ALLOC]], i8** [[ESC:%.*]], align 4
+; CHECK-NEXT:    ret i32 30
+;
   %alloc = call noalias i8* @calloc(i32 5, i32 7) nounwind
   store i8* %alloc, i8** %esc
   %gep = getelementptr inbounds i8, i8* %alloc, i32 5
   %objsize = call i32 @llvm.objectsize.i32.p0i8(i8* %gep, i1 false, i1 false, i1 false) nounwind readonly
-; CHECK: ret i32 30
   ret i32 %objsize
 }
 
 declare noalias i8* @strdup(i8* nocapture) nounwind
 declare noalias i8* @strndup(i8* nocapture, i32) nounwind
 
-; CHECK-LABEL: @test9(
 define i32 @test9(i8** %esc) {
+; CHECK-LABEL: @test9(
+; CHECK-NEXT:    [[CALL:%.*]] = tail call dereferenceable_or_null(8) i8* @strdup(i8* getelementptr inbounds ([8 x i8], [8 x i8]* @.str, i32 0, i32 0)) #0
+; CHECK-NEXT:    store i8* [[CALL]], i8** [[ESC:%.*]], align 8
+; CHECK-NEXT:    ret i32 8
+;
   %call = tail call i8* @strdup(i8* getelementptr inbounds ([8 x i8], [8 x i8]* @.str, i64 0, i64 0)) nounwind
   store i8* %call, i8** %esc, align 8
   %1 = tail call i32 @llvm.objectsize.i32.p0i8(i8* %call, i1 true, i1 false, i1 false)
-; CHECK: ret i32 8
   ret i32 %1
 }
 
-; CHECK-LABEL: @test10(
 define i32 @test10(i8** %esc) {
+; CHECK-LABEL: @test10(
+; CHECK-NEXT:    [[CALL:%.*]] = tail call dereferenceable_or_null(4) i8* @strndup(i8* dereferenceable(8) getelementptr inbounds ([8 x i8], [8 x i8]* @.str, i32 0, i32 0), i32 3) #0
+; CHECK-NEXT:    store i8* [[CALL]], i8** [[ESC:%.*]], align 8
+; CHECK-NEXT:    ret i32 4
+;
   %call = tail call i8* @strndup(i8* getelementptr inbounds ([8 x i8], [8 x i8]* @.str, i64 0, i64 0), i32 3) nounwind
   store i8* %call, i8** %esc, align 8
   %1 = tail call i32 @llvm.objectsize.i32.p0i8(i8* %call, i1 true, i1 false, i1 false)
-; CHECK: ret i32 4
   ret i32 %1
 }
 
-; CHECK-LABEL: @test11(
 define i32 @test11(i8** %esc) {
+; CHECK-LABEL: @test11(
+; CHECK-NEXT:    [[STRDUP:%.*]] = call dereferenceable_or_null(8) i8* @strdup(i8* getelementptr inbounds ([8 x i8], [8 x i8]* @.str, i32 0, i32 0))
+; CHECK-NEXT:    store i8* [[STRDUP]], i8** [[ESC:%.*]], align 8
+; CHECK-NEXT:    ret i32 8
+;
   %call = tail call i8* @strndup(i8* getelementptr inbounds ([8 x i8], [8 x i8]* @.str, i64 0, i64 0), i32 7) nounwind
   store i8* %call, i8** %esc, align 8
   %1 = tail call i32 @llvm.objectsize.i32.p0i8(i8* %call, i1 true, i1 false, i1 false)
-; CHECK: ret i32 8
   ret i32 %1
 }
 
-; CHECK-LABEL: @test12(
 define i32 @test12(i8** %esc) {
+; CHECK-LABEL: @test12(
+; CHECK-NEXT:    [[STRDUP:%.*]] = call dereferenceable_or_null(8) i8* @strdup(i8* getelementptr inbounds ([8 x i8], [8 x i8]* @.str, i32 0, i32 0))
+; CHECK-NEXT:    store i8* [[STRDUP]], i8** [[ESC:%.*]], align 8
+; CHECK-NEXT:    ret i32 8
+;
   %call = tail call i8* @strndup(i8* getelementptr inbounds ([8 x i8], [8 x i8]* @.str, i64 0, i64 0), i32 8) nounwind
   store i8* %call, i8** %esc, align 8
   %1 = tail call i32 @llvm.objectsize.i32.p0i8(i8* %call, i1 true, i1 false, i1 false)
-; CHECK: ret i32 8
   ret i32 %1
 }
 
-; CHECK-LABEL: @test13(
 define i32 @test13(i8** %esc) {
+; CHECK-LABEL: @test13(
+; CHECK-NEXT:    [[STRDUP:%.*]] = call dereferenceable_or_null(8) i8* @strdup(i8* getelementptr inbounds ([8 x i8], [8 x i8]* @.str, i32 0, i32 0))
+; CHECK-NEXT:    store i8* [[STRDUP]], i8** [[ESC:%.*]], align 8
+; CHECK-NEXT:    ret i32 8
+;
   %call = tail call i8* @strndup(i8* getelementptr inbounds ([8 x i8], [8 x i8]* @.str, i64 0, i64 0), i32 57) nounwind
   store i8* %call, i8** %esc, align 8
   %1 = tail call i32 @llvm.objectsize.i32.p0i8(i8* %call, i1 true, i1 false, i1 false)
-; CHECK: ret i32 8
   ret i32 %1
 }
 
 @globalalias = internal alias [60 x i8], [60 x i8]* @a
 
-; CHECK-LABEL: @test18(
-; CHECK-NEXT: ret i32 60
 define i32 @test18() {
+; CHECK-LABEL: @test18(
+; CHECK-NEXT:    ret i32 60
+;
   %bc = bitcast [60 x i8]* @globalalias to i8*
   %1 = call i32 @llvm.objectsize.i32.p0i8(i8* %bc, i1 false, i1 false, i1 false)
   ret i32 %1
@@ -233,71 +298,87 @@ define i32 @test18() {
 
 @globalalias2 = weak alias [60 x i8], [60 x i8]* @a
 
-; CHECK-LABEL: @test19(
-; CHECK: llvm.objectsize
 define i32 @test19() {
+; CHECK-LABEL: @test19(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.objectsize.i32.p0i8(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @globalalias2, i32 0, i32 0), i1 false, i1 false, i1 false)
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
   %bc = bitcast [60 x i8]* @globalalias2 to i8*
   %1 = call i32 @llvm.objectsize.i32.p0i8(i8* %bc, i1 false, i1 false, i1 false)
   ret i32 %1
 }
 
-; CHECK-LABEL: @test20(
-; CHECK: ret i32 0
 define i32 @test20() {
+; CHECK-LABEL: @test20(
+; CHECK-NEXT:    ret i32 0
+;
   %1 = call i32 @llvm.objectsize.i32.p0i8(i8* null, i1 false, i1 false, i1 false)
   ret i32 %1
 }
 
-; CHECK-LABEL: @test21(
-; CHECK: ret i32 0
 define i32 @test21() {
+; CHECK-LABEL: @test21(
+; CHECK-NEXT:    ret i32 0
+;
   %1 = call i32 @llvm.objectsize.i32.p0i8(i8* null, i1 true, i1 false, i1 false)
   ret i32 %1
 }
 
-; CHECK-LABEL: @test22(
-; CHECK: llvm.objectsize
 define i32 @test22() {
+; CHECK-LABEL: @test22(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.objectsize.i32.p0i8(i8* null, i1 false, i1 true, i1 false)
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
   %1 = call i32 @llvm.objectsize.i32.p0i8(i8* null, i1 false, i1 true, i1 false)
   ret i32 %1
 }
 
-; CHECK-LABEL: @test23(
-; CHECK: llvm.objectsize
 define i32 @test23() {
+; CHECK-LABEL: @test23(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.objectsize.i32.p0i8(i8* null, i1 true, i1 true, i1 false)
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
   %1 = call i32 @llvm.objectsize.i32.p0i8(i8* null, i1 true, i1 true, i1 false)
   ret i32 %1
 }
 
 ; 1 is an arbitrary non-zero address space.
-; CHECK-LABEL: @test24(
-; CHECK: llvm.objectsize
 define i32 @test24() {
+; CHECK-LABEL: @test24(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.objectsize.i32.p1i8(i8 addrspace(1)* null, i1 false, i1 false, i1 false)
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
   %1 = call i32 @llvm.objectsize.i32.p1i8(i8 addrspace(1)* null, i1 false,
-                                          i1 false, i1 false)
+  i1 false, i1 false)
   ret i32 %1
 }
 
-; CHECK-LABEL: @test25(
-; CHECK: llvm.objectsize
 define i32 @test25() {
+; CHECK-LABEL: @test25(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.objectsize.i32.p1i8(i8 addrspace(1)* null, i1 true, i1 false, i1 false)
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
   %1 = call i32 @llvm.objectsize.i32.p1i8(i8 addrspace(1)* null, i1 true,
-                                          i1 false, i1 false)
+  i1 false, i1 false)
   ret i32 %1
 }
 
-; CHECK-LABEL: @test26(
-; CHECK: llvm.objectsize
 define i32 @test26() {
+; CHECK-LABEL: @test26(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.objectsize.i32.p1i8(i8 addrspace(1)* null, i1 false, i1 true, i1 false)
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
   %1 = call i32 @llvm.objectsize.i32.p1i8(i8 addrspace(1)* null, i1 false,
-                                          i1 true, i1 false)
+  i1 true, i1 false)
   ret i32 %1
 }
 
-; CHECK-LABEL: @test27(
-; CHECK: llvm.objectsize
 define i32 @test27() {
+; CHECK-LABEL: @test27(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.objectsize.i32.p1i8(i8 addrspace(1)* null, i1 true, i1 true, i1 false)
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
   %1 = call i32 @llvm.objectsize.i32.p1i8(i8 addrspace(1)* null, i1 true,
-                                          i1 true, i1 false)
+  i1 true, i1 false)
   ret i32 %1
 }
diff --git a/llvm/test/Transforms/InstCombine/overflow_to_sat.ll b/llvm/test/Transforms/InstCombine/overflow_to_sat.ll
new file mode 100644
index 000000000..af56f5703
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/overflow_to_sat.ll
@@ -0,0 +1,710 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+define i32 @uadd(i32 %x, i32 %y) {
+; CHECK-LABEL: @uadd(
+; CHECK-NEXT:    [[S:%.*]] = call i32 @llvm.uadd.sat.i32(i32 [[X:%.*]], i32 [[Y:%.*]])
+; CHECK-NEXT:    ret i32 [[S]]
+;
+  %ao = tail call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 %x, i32 %y)
+  %o = extractvalue { i32, i1 } %ao, 1
+  %a = extractvalue { i32, i1 } %ao, 0
+  %s = select i1 %o, i32 -1, i32 %a
+  ret i32 %s
+}
+
+define i32 @usub(i32 %x, i32 %y) {
+; CHECK-LABEL: @usub(
+; CHECK-NEXT:    [[S:%.*]] = call i32 @llvm.usub.sat.i32(i32 [[X:%.*]], i32 [[Y:%.*]])
+; CHECK-NEXT:    ret i32 [[S]]
+;
+  %ao = tail call { i32, i1 } @llvm.usub.with.overflow.i32(i32 %x, i32 %y)
+  %o = extractvalue { i32, i1 } %ao, 1
+  %a = extractvalue { i32, i1 } %ao, 0
+  %s = select i1 %o, i32 0, i32 %a
+  ret i32 %s
+}
+
+
+define i8 @sadd_x_lt_min(i8 %x, i8 %y) {
+; CHECK-LABEL: @sadd_x_lt_min(
+; CHECK-NEXT:    [[AO:%.*]] = tail call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    [[O:%.*]] = extractvalue { i8, i1 } [[AO]], 1
+; CHECK-NEXT:    [[A:%.*]] = extractvalue { i8, i1 } [[AO]], 0
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i8 [[X]], 0
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[C]], i8 127, i8 -128
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[O]], i8 [[S]], i8 [[A]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ao = tail call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 %x, i8 %y)
+  %o = extractvalue { i8, i1 } %ao, 1
+  %a = extractvalue { i8, i1 } %ao, 0
+  %c = icmp slt i8 %x, 0
+  %s = select i1 %c, i8 127, i8 -128
+  %r = select i1 %o, i8 %s, i8 %a
+  ret i8 %r
+}
+
+define i8 @sadd_x_lt_max(i8 %x, i8 %y) {
+; CHECK-LABEL: @sadd_x_lt_max(
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.sadd.sat.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ao = tail call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 %x, i8 %y)
+  %o = extractvalue { i8, i1 } %ao, 1
+  %a = extractvalue { i8, i1 } %ao, 0
+  %c = icmp slt i8 %x, 0
+  %s = select i1 %c, i8 -128, i8 127
+  %r = select i1 %o, i8 %s, i8 %a
+  ret i8 %r
+}
+
+define i8 @sadd_x_le_min(i8 %x, i8 %y) {
+; CHECK-LABEL: @sadd_x_le_min(
+; CHECK-NEXT:    [[AO:%.*]] = tail call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    [[O:%.*]] = extractvalue { i8, i1 } [[AO]], 1
+; CHECK-NEXT:    [[A:%.*]] = extractvalue { i8, i1 } [[AO]], 0
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i8 [[X]], 1
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[C]], i8 127, i8 -128
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[O]], i8 [[S]], i8 [[A]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ao = tail call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 %x, i8 %y)
+  %o = extractvalue { i8, i1 } %ao, 1
+  %a = extractvalue { i8, i1 } %ao, 0
+  %c = icmp sle i8 %x, 0
+  %s = select i1 %c, i8 127, i8 -128
+  %r = select i1 %o, i8 %s, i8 %a
+  ret i8 %r
+}
+
+define i8 @sadd_x_le_max(i8 %x, i8 %y) {
+; CHECK-LABEL: @sadd_x_le_max(
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.sadd.sat.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ao = tail call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 %x, i8 %y)
+  %o = extractvalue { i8, i1 } %ao, 1
+  %a = extractvalue { i8, i1 } %ao, 0
+  %c = icmp sle i8 %x, 0
+  %s = select i1 %c, i8 -128, i8 127
+  %r = select i1 %o, i8 %s, i8 %a
+  ret i8 %r
+}
+
+define i8 @sadd_x_gt_min(i8 %x, i8 %y) {
+; CHECK-LABEL: @sadd_x_gt_min(
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.sadd.sat.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ao = tail call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 %x, i8 %y)
+  %o = extractvalue { i8, i1 } %ao, 1
+  %a = extractvalue { i8, i1 } %ao, 0
+  %c = icmp sgt i8 %x, 0
+  %s = select i1 %c, i8 127, i8 -128
+  %r = select i1 %o, i8 %s, i8 %a
+  ret i8 %r
+}
+
+define i8 @sadd_x_gt_max(i8 %x, i8 %y) {
+; CHECK-LABEL: @sadd_x_gt_max(
+; CHECK-NEXT:    [[AO:%.*]] = tail call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    [[O:%.*]] = extractvalue { i8, i1 } [[AO]], 1
+; CHECK-NEXT:    [[A:%.*]] = extractvalue { i8, i1 } [[AO]], 0
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i8 [[X]], 0
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[C]], i8 -128, i8 127
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[O]], i8 [[S]], i8 [[A]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ao = tail call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 %x, i8 %y)
+  %o = extractvalue { i8, i1 } %ao, 1
+  %a = extractvalue { i8, i1 } %ao, 0
+  %c = icmp sgt i8 %x, 0
+  %s = select i1 %c, i8 -128, i8 127
+  %r = select i1 %o, i8 %s, i8 %a
+  ret i8 %r
+}
+
+define i8 @sadd_x_ge_min(i8 %x, i8 %y) {
+; CHECK-LABEL: @sadd_x_ge_min(
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.sadd.sat.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ao = tail call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 %x, i8 %y)
+  %o = extractvalue { i8, i1 } %ao, 1
+  %a = extractvalue { i8, i1 } %ao, 0
+  %c = icmp sge i8 %x, 0
+  %s = select i1 %c, i8 127, i8 -128
+  %r = select i1 %o, i8 %s, i8 %a
+  ret i8 %r
+}
+
+define i8 @sadd_x_ge_max(i8 %x, i8 %y) {
+; CHECK-LABEL: @sadd_x_ge_max(
+; CHECK-NEXT:    [[AO:%.*]] = tail call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    [[O:%.*]] = extractvalue { i8, i1 } [[AO]], 1
+; CHECK-NEXT:    [[A:%.*]] = extractvalue { i8, i1 } [[AO]], 0
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i8 [[X]], -1
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[C]], i8 -128, i8 127
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[O]], i8 [[S]], i8 [[A]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ao = tail call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 %x, i8 %y)
+  %o = extractvalue { i8, i1 } %ao, 1
+  %a = extractvalue { i8, i1 } %ao, 0
+  %c = icmp sge i8 %x, 0
+  %s = select i1 %c, i8 -128, i8 127
+  %r = select i1 %o, i8 %s, i8 %a
+  ret i8 %r
+}
+
+
+define i8 @sadd_y_lt_min(i8 %x, i8 %y) {
+; CHECK-LABEL: @sadd_y_lt_min(
+; CHECK-NEXT:    [[AO:%.*]] = tail call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    [[O:%.*]] = extractvalue { i8, i1 } [[AO]], 1
+; CHECK-NEXT:    [[A:%.*]] = extractvalue { i8, i1 } [[AO]], 0
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i8 [[Y]], 0
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[C]], i8 127, i8 -128
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[O]], i8 [[S]], i8 [[A]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ao = tail call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 %x, i8 %y)
+  %o = extractvalue { i8, i1 } %ao, 1
+  %a = extractvalue { i8, i1 } %ao, 0
+  %c = icmp slt i8 %y, 0
+  %s = select i1 %c, i8 127, i8 -128
+  %r = select i1 %o, i8 %s, i8 %a
+  ret i8 %r
+}
+
+define i8 @sadd_y_lt_max(i8 %x, i8 %y) {
+; CHECK-LABEL: @sadd_y_lt_max(
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.sadd.sat.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ao = tail call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 %x, i8 %y)
+  %o = extractvalue { i8, i1 } %ao, 1
+  %a = extractvalue { i8, i1 } %ao, 0
+  %c = icmp slt i8 %y, 0
+  %s = select i1 %c, i8 -128, i8 127
+  %r = select i1 %o, i8 %s, i8 %a
+  ret i8 %r
+}
+
+define i8 @sadd_y_le_min(i8 %x, i8 %y) {
+; CHECK-LABEL: @sadd_y_le_min(
+; CHECK-NEXT:    [[AO:%.*]] = tail call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    [[O:%.*]] = extractvalue { i8, i1 } [[AO]], 1
+; CHECK-NEXT:    [[A:%.*]] = extractvalue { i8, i1 } [[AO]], 0
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i8 [[Y]], 1
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[C]], i8 127, i8 -128
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[O]], i8 [[S]], i8 [[A]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ao = tail call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 %x, i8 %y)
+  %o = extractvalue { i8, i1 } %ao, 1
+  %a = extractvalue { i8, i1 } %ao, 0
+  %c = icmp sle i8 %y, 0
+  %s = select i1 %c, i8 127, i8 -128
+  %r = select i1 %o, i8 %s, i8 %a
+  ret i8 %r
+}
+
+define i8 @sadd_y_le_max(i8 %x, i8 %y) {
+; CHECK-LABEL: @sadd_y_le_max(
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.sadd.sat.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ao = tail call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 %x, i8 %y)
+  %o = extractvalue { i8, i1 } %ao, 1
+  %a = extractvalue { i8, i1 } %ao, 0
+  %c = icmp sle i8 %y, 0
+  %s = select i1 %c, i8 -128, i8 127
+  %r = select i1 %o, i8 %s, i8 %a
+  ret i8 %r
+}
+
+define i8 @sadd_y_gt_min(i8 %x, i8 %y) {
+; CHECK-LABEL: @sadd_y_gt_min(
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.sadd.sat.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ao = tail call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 %x, i8 %y)
+  %o = extractvalue { i8, i1 } %ao, 1
+  %a = extractvalue { i8, i1 } %ao, 0
+  %c = icmp sgt i8 %y, 0
+  %s = select i1 %c, i8 127, i8 -128
+  %r = select i1 %o, i8 %s, i8 %a
+  ret i8 %r
+}
+
+define i8 @sadd_y_gt_max(i8 %x, i8 %y) {
+; CHECK-LABEL: @sadd_y_gt_max(
+; CHECK-NEXT:    [[AO:%.*]] = tail call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    [[O:%.*]] = extractvalue { i8, i1 } [[AO]], 1
+; CHECK-NEXT:    [[A:%.*]] = extractvalue { i8, i1 } [[AO]], 0
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i8 [[Y]], 0
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[C]], i8 -128, i8 127
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[O]], i8 [[S]], i8 [[A]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ao = tail call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 %x, i8 %y)
+  %o = extractvalue { i8, i1 } %ao, 1
+  %a = extractvalue { i8, i1 } %ao, 0
+  %c = icmp sgt i8 %y, 0
+  %s = select i1 %c, i8 -128, i8 127
+  %r = select i1 %o, i8 %s, i8 %a
+  ret i8 %r
+}
+
+define i8 @sadd_y_ge_min(i8 %x, i8 %y) {
+; CHECK-LABEL: @sadd_y_ge_min(
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.sadd.sat.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ao = tail call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 %x, i8 %y)
+  %o = extractvalue { i8, i1 } %ao, 1
+  %a = extractvalue { i8, i1 } %ao, 0
+  %c = icmp sge i8 %y, 0
+  %s = select i1 %c, i8 127, i8 -128
+  %r = select i1 %o, i8 %s, i8 %a
+  ret i8 %r
+}
+
+define i8 @sadd_y_ge_max(i8 %x, i8 %y) {
+; CHECK-LABEL: @sadd_y_ge_max(
+; CHECK-NEXT:    [[AO:%.*]] = tail call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    [[O:%.*]] = extractvalue { i8, i1 } [[AO]], 1
+; CHECK-NEXT:    [[A:%.*]] = extractvalue { i8, i1 } [[AO]], 0
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i8 [[Y]], -1
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[C]], i8 -128, i8 127
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[O]], i8 [[S]], i8 [[A]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ao = tail call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 %x, i8 %y)
+  %o = extractvalue { i8, i1 } %ao, 1
+  %a = extractvalue { i8, i1 } %ao, 0
+  %c = icmp sge i8 %y, 0
+  %s = select i1 %c, i8 -128, i8 127
+  %r = select i1 %o, i8 %s, i8 %a
+  ret i8 %r
+}
+
+
+
+
+define i8 @ssub_x_lt_min(i8 %x, i8 %y) {
+; CHECK-LABEL: @ssub_x_lt_min(
+; CHECK-NEXT:    [[AO:%.*]] = tail call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    [[O:%.*]] = extractvalue { i8, i1 } [[AO]], 1
+; CHECK-NEXT:    [[A:%.*]] = extractvalue { i8, i1 } [[AO]], 0
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i8 [[X]], 0
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[C]], i8 127, i8 -128
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[O]], i8 [[S]], i8 [[A]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ao = tail call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 %x, i8 %y)
+  %o = extractvalue { i8, i1 } %ao, 1
+  %a = extractvalue { i8, i1 } %ao, 0
+  %c = icmp slt i8 %x, 0
+  %s = select i1 %c, i8 127, i8 -128
+  %r = select i1 %o, i8 %s, i8 %a
+  ret i8 %r
+}
+
+define i8 @ssub_x_lt_max(i8 %x, i8 %y) {
+; CHECK-LABEL: @ssub_x_lt_max(
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.ssub.sat.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ao = tail call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 %x, i8 %y)
+  %o = extractvalue { i8, i1 } %ao, 1
+  %a = extractvalue { i8, i1 } %ao, 0
+  %c = icmp slt i8 %x, 0
+  %s = select i1 %c, i8 -128, i8 127
+  %r = select i1 %o, i8 %s, i8 %a
+  ret i8 %r
+}
+
+define i8 @ssub_x_le_min(i8 %x, i8 %y) {
+; CHECK-LABEL: @ssub_x_le_min(
+; CHECK-NEXT:    [[AO:%.*]] = tail call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    [[O:%.*]] = extractvalue { i8, i1 } [[AO]], 1
+; CHECK-NEXT:    [[A:%.*]] = extractvalue { i8, i1 } [[AO]], 0
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i8 [[X]], 1
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[C]], i8 127, i8 -128
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[O]], i8 [[S]], i8 [[A]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ao = tail call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 %x, i8 %y)
+  %o = extractvalue { i8, i1 } %ao, 1
+  %a = extractvalue { i8, i1 } %ao, 0
+  %c = icmp sle i8 %x, 0
+  %s = select i1 %c, i8 127, i8 -128
+  %r = select i1 %o, i8 %s, i8 %a
+  ret i8 %r
+}
+
+define i8 @ssub_x_le_max(i8 %x, i8 %y) {
+; CHECK-LABEL: @ssub_x_le_max(
+; CHECK-NEXT:    [[AO:%.*]] = tail call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    [[O:%.*]] = extractvalue { i8, i1 } [[AO]], 1
+; CHECK-NEXT:    [[A:%.*]] = extractvalue { i8, i1 } [[AO]], 0
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i8 [[X]], 1
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[C]], i8 -128, i8 127
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[O]], i8 [[S]], i8 [[A]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ao = tail call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 %x, i8 %y)
+  %o = extractvalue { i8, i1 } %ao, 1
+  %a = extractvalue { i8, i1 } %ao, 0
+  %c = icmp sle i8 %x, 0
+  %s = select i1 %c, i8 -128, i8 127
+  %r = select i1 %o, i8 %s, i8 %a
+  ret i8 %r
+}
+
+define i8 @ssub_x_lt2_min(i8 %x, i8 %y) {
+; CHECK-LABEL: @ssub_x_lt2_min(
+; CHECK-NEXT:    [[AO:%.*]] = tail call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    [[O:%.*]] = extractvalue { i8, i1 } [[AO]], 1
+; CHECK-NEXT:    [[A:%.*]] = extractvalue { i8, i1 } [[AO]], 0
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i8 [[X]], -1
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[C]], i8 127, i8 -128
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[O]], i8 [[S]], i8 [[A]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ao = tail call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 %x, i8 %y)
+  %o = extractvalue { i8, i1 } %ao, 1
+  %a = extractvalue { i8, i1 } %ao, 0
+  %c = icmp slt i8 %x, -1
+  %s = select i1 %c, i8 127, i8 -128
+  %r = select i1 %o, i8 %s, i8 %a
+  ret i8 %r
+}
+
+define i8 @ssub_x_lt2_max(i8 %x, i8 %y) {
+; CHECK-LABEL: @ssub_x_lt2_max(
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.ssub.sat.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ao = tail call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 %x, i8 %y)
+  %o = extractvalue { i8, i1 } %ao, 1
+  %a = extractvalue { i8, i1 } %ao, 0
+  %c = icmp slt i8 %x, -1
+  %s = select i1 %c, i8 -128, i8 127
+  %r = select i1 %o, i8 %s, i8 %a
+  ret i8 %r
+}
+
+define i8 @ssub_x_gt_min(i8 %x, i8 %y) {
+; CHECK-LABEL: @ssub_x_gt_min(
+; CHECK-NEXT:    [[AO:%.*]] = tail call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    [[O:%.*]] = extractvalue { i8, i1 } [[AO]], 1
+; CHECK-NEXT:    [[A:%.*]] = extractvalue { i8, i1 } [[AO]], 0
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i8 [[X]], 0
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[C]], i8 127, i8 -128
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[O]], i8 [[S]], i8 [[A]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ao = tail call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 %x, i8 %y)
+  %o = extractvalue { i8, i1 } %ao, 1
+  %a = extractvalue { i8, i1 } %ao, 0
+  %c = icmp sgt i8 %x, 0
+  %s = select i1 %c, i8 127, i8 -128
+  %r = select i1 %o, i8 %s, i8 %a
+  ret i8 %r
+}
+
+define i8 @ssub_x_gt_max(i8 %x, i8 %y) {
+; CHECK-LABEL: @ssub_x_gt_max(
+; CHECK-NEXT:    [[AO:%.*]] = tail call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    [[O:%.*]] = extractvalue { i8, i1 } [[AO]], 1
+; CHECK-NEXT:    [[A:%.*]] = extractvalue { i8, i1 } [[AO]], 0
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i8 [[X]], 0
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[C]], i8 -128, i8 127
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[O]], i8 [[S]], i8 [[A]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ao = tail call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 %x, i8 %y)
+  %o = extractvalue { i8, i1 } %ao, 1
+  %a = extractvalue { i8, i1 } %ao, 0
+  %c = icmp sgt i8 %x, 0
+  %s = select i1 %c, i8 -128, i8 127
+  %r = select i1 %o, i8 %s, i8 %a
+  ret i8 %r
+}
+
+define i8 @ssub_x_ge_min(i8 %x, i8 %y) {
+; CHECK-LABEL: @ssub_x_ge_min(
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.ssub.sat.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ao = tail call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 %x, i8 %y)
+  %o = extractvalue { i8, i1 } %ao, 1
+  %a = extractvalue { i8, i1 } %ao, 0
+  %c = icmp sge i8 %x, 0
+  %s = select i1 %c, i8 127, i8 -128
+  %r = select i1 %o, i8 %s, i8 %a
+  ret i8 %r
+}
+
+define i8 @ssub_x_ge_max(i8 %x, i8 %y) {
+; CHECK-LABEL: @ssub_x_ge_max(
+; CHECK-NEXT:    [[AO:%.*]] = tail call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    [[O:%.*]] = extractvalue { i8, i1 } [[AO]], 1
+; CHECK-NEXT:    [[A:%.*]] = extractvalue { i8, i1 } [[AO]], 0
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i8 [[X]], -1
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[C]], i8 -128, i8 127
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[O]], i8 [[S]], i8 [[A]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ao = tail call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 %x, i8 %y)
+  %o = extractvalue { i8, i1 } %ao, 1
+  %a = extractvalue { i8, i1 } %ao, 0
+  %c = icmp sge i8 %x, 0
+  %s = select i1 %c, i8 -128, i8 127
+  %r = select i1 %o, i8 %s, i8 %a
+  ret i8 %r
+}
+
+define i8 @ssub_x_gt2_min(i8 %x, i8 %y) {
+; CHECK-LABEL: @ssub_x_gt2_min(
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.ssub.sat.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ao = tail call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 %x, i8 %y)
+  %o = extractvalue { i8, i1 } %ao, 1
+  %a = extractvalue { i8, i1 } %ao, 0
+  %c = icmp sgt i8 %x, -2
+  %s = select i1 %c, i8 127, i8 -128
+  %r = select i1 %o, i8 %s, i8 %a
+  ret i8 %r
+}
+
+define i8 @ssub_x_gt2_max(i8 %x, i8 %y) {
+; CHECK-LABEL: @ssub_x_gt2_max(
+; CHECK-NEXT:    [[AO:%.*]] = tail call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    [[O:%.*]] = extractvalue { i8, i1 } [[AO]], 1
+; CHECK-NEXT:    [[A:%.*]] = extractvalue { i8, i1 } [[AO]], 0
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i8 [[X]], -2
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[C]], i8 -128, i8 127
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[O]], i8 [[S]], i8 [[A]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ao = tail call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 %x, i8 %y)
+  %o = extractvalue { i8, i1 } %ao, 1
+  %a = extractvalue { i8, i1 } %ao, 0
+  %c = icmp sgt i8 %x, -2
+  %s = select i1 %c, i8 -128, i8 127
+  %r = select i1 %o, i8 %s, i8 %a
+  ret i8 %r
+}
+
+
+define i8 @ssub_y_lt_min(i8 %x, i8 %y) {
+; CHECK-LABEL: @ssub_y_lt_min(
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.ssub.sat.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ao = tail call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 %x, i8 %y)
+  %o = extractvalue { i8, i1 } %ao, 1
+  %a = extractvalue { i8, i1 } %ao, 0
+  %c = icmp slt i8 %y, 0
+  %s = select i1 %c, i8 127, i8 -128
+  %r = select i1 %o, i8 %s, i8 %a
+  ret i8 %r
+}
+
+define i8 @ssub_y_lt_max(i8 %x, i8 %y) {
+; CHECK-LABEL: @ssub_y_lt_max(
+; CHECK-NEXT:    [[AO:%.*]] = tail call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    [[O:%.*]] = extractvalue { i8, i1 } [[AO]], 1
+; CHECK-NEXT:    [[A:%.*]] = extractvalue { i8, i1 } [[AO]], 0
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i8 [[Y]], 0
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[C]], i8 -128, i8 127
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[O]], i8 [[S]], i8 [[A]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ao = tail call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 %x, i8 %y)
+  %o = extractvalue { i8, i1 } %ao, 1
+  %a = extractvalue { i8, i1 } %ao, 0
+  %c = icmp slt i8 %y, 0
+  %s = select i1 %c, i8 -128, i8 127
+  %r = select i1 %o, i8 %s, i8 %a
+  ret i8 %r
+}
+
+define i8 @ssub_y_le_min(i8 %x, i8 %y) {
+; CHECK-LABEL: @ssub_y_le_min(
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.ssub.sat.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ao = tail call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 %x, i8 %y)
+  %o = extractvalue { i8, i1 } %ao, 1
+  %a = extractvalue { i8, i1 } %ao, 0
+  %c = icmp sle i8 %y, 0
+  %s = select i1 %c, i8 127, i8 -128
+  %r = select i1 %o, i8 %s, i8 %a
+  ret i8 %r
+}
+
+define i8 @ssub_y_le_max(i8 %x, i8 %y) {
+; CHECK-LABEL: @ssub_y_le_max(
+; CHECK-NEXT:    [[AO:%.*]] = tail call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    [[O:%.*]] = extractvalue { i8, i1 } [[AO]], 1
+; CHECK-NEXT:    [[A:%.*]] = extractvalue { i8, i1 } [[AO]], 0
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i8 [[Y]], 1
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[C]], i8 -128, i8 127
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[O]], i8 [[S]], i8 [[A]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ao = tail call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 %x, i8 %y)
+  %o = extractvalue { i8, i1 } %ao, 1
+  %a = extractvalue { i8, i1 } %ao, 0
+  %c = icmp sle i8 %y, 0
+  %s = select i1 %c, i8 -128, i8 127
+  %r = select i1 %o, i8 %s, i8 %a
+  ret i8 %r
+}
+
+define i8 @ssub_y_gt_min(i8 %x, i8 %y) {
+; CHECK-LABEL: @ssub_y_gt_min(
+; CHECK-NEXT:    [[AO:%.*]] = tail call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    [[O:%.*]] = extractvalue { i8, i1 } [[AO]], 1
+; CHECK-NEXT:    [[A:%.*]] = extractvalue { i8, i1 } [[AO]], 0
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i8 [[Y]], 0
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[C]], i8 127, i8 -128
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[O]], i8 [[S]], i8 [[A]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ao = tail call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 %x, i8 %y)
+  %o = extractvalue { i8, i1 } %ao, 1
+  %a = extractvalue { i8, i1 } %ao, 0
+  %c = icmp sgt i8 %y, 0
+  %s = select i1 %c, i8 127, i8 -128
+  %r = select i1 %o, i8 %s, i8 %a
+  ret i8 %r
+}
+
+define i8 @ssub_y_gt_max(i8 %x, i8 %y) {
+; CHECK-LABEL: @ssub_y_gt_max(
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.ssub.sat.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ao = tail call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 %x, i8 %y)
+  %o = extractvalue { i8, i1 } %ao, 1
+  %a = extractvalue { i8, i1 } %ao, 0
+  %c = icmp sgt i8 %y, 0
+  %s = select i1 %c, i8 -128, i8 127
+  %r = select i1 %o, i8 %s, i8 %a
+  ret i8 %r
+}
+
+define i8 @ssub_y_ge_min(i8 %x, i8 %y) {
+; CHECK-LABEL: @ssub_y_ge_min(
+; CHECK-NEXT:    [[AO:%.*]] = tail call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    [[O:%.*]] = extractvalue { i8, i1 } [[AO]], 1
+; CHECK-NEXT:    [[A:%.*]] = extractvalue { i8, i1 } [[AO]], 0
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i8 [[Y]], -1
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[C]], i8 127, i8 -128
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[O]], i8 [[S]], i8 [[A]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ao = tail call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 %x, i8 %y)
+  %o = extractvalue { i8, i1 } %ao, 1
+  %a = extractvalue { i8, i1 } %ao, 0
+  %c = icmp sge i8 %y, 0
+  %s = select i1 %c, i8 127, i8 -128
+  %r = select i1 %o, i8 %s, i8 %a
+  ret i8 %r
+}
+
+define i8 @ssub_y_ge_max(i8 %x, i8 %y) {
+; CHECK-LABEL: @ssub_y_ge_max(
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.ssub.sat.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ao = tail call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 %x, i8 %y)
+  %o = extractvalue { i8, i1 } %ao, 1
+  %a = extractvalue { i8, i1 } %ao, 0
+  %c = icmp sge i8 %y, 0
+  %s = select i1 %c, i8 -128, i8 127
+  %r = select i1 %o, i8 %s, i8 %a
+  ret i8 %r
+}
+
+
+define i32 @sadd_i32(i32 %x, i32 %y) {
+; CHECK-LABEL: @sadd_i32(
+; CHECK-NEXT:    [[R:%.*]] = call i32 @llvm.sadd.sat.i32(i32 [[X:%.*]], i32 [[Y:%.*]])
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %ao = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 %x, i32 %y)
+  %o = extractvalue { i32, i1 } %ao, 1
+  %a = extractvalue { i32, i1 } %ao, 0
+  %c = icmp slt i32 %x, 0
+  %s = select i1 %c, i32 -2147483648, i32 2147483647
+  %r = select i1 %o, i32 %s, i32 %a
+  ret i32 %r
+}
+
+define i32 @ssub_i32(i32 %x, i32 %y) {
+; CHECK-LABEL: @ssub_i32(
+; CHECK-NEXT:    [[R:%.*]] = call i32 @llvm.ssub.sat.i32(i32 [[X:%.*]], i32 [[Y:%.*]])
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %ao = tail call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 %x, i32 %y)
+  %o = extractvalue { i32, i1 } %ao, 1
+  %a = extractvalue { i32, i1 } %ao, 0
+  %c = icmp slt i32 %x, 0
+  %s = select i1 %c, i32 -2147483648, i32 2147483647
+  %r = select i1 %o, i32 %s, i32 %a
+  ret i32 %r
+}
+
+define i32 @sadd_bounds(i32 %x, i32 %y) {
+; CHECK-LABEL: @sadd_bounds(
+; CHECK-NEXT:    [[AO:%.*]] = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[X:%.*]], i32 [[Y:%.*]])
+; CHECK-NEXT:    [[O:%.*]] = extractvalue { i32, i1 } [[AO]], 1
+; CHECK-NEXT:    [[A:%.*]] = extractvalue { i32, i1 } [[AO]], 0
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i32 [[X]], 0
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[C]], i32 -128, i32 127
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[O]], i32 [[S]], i32 [[A]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %ao = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 %x, i32 %y)
+  %o = extractvalue { i32, i1 } %ao, 1
+  %a = extractvalue { i32, i1 } %ao, 0
+  %c = icmp slt i32 %x, 0
+  %s = select i1 %c, i32 -128, i32 127
+  %r = select i1 %o, i32 %s, i32 %a
+  ret i32 %r
+}
+
+define i32 @ssub_bounds(i32 %x, i32 %y) {
+; CHECK-LABEL: @ssub_bounds(
+; CHECK-NEXT:    [[AO:%.*]] = tail call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 [[X:%.*]], i32 [[Y:%.*]])
+; CHECK-NEXT:    [[O:%.*]] = extractvalue { i32, i1 } [[AO]], 1
+; CHECK-NEXT:    [[A:%.*]] = extractvalue { i32, i1 } [[AO]], 0
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i32 [[X]], 0
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[C]], i32 -128, i32 127
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[O]], i32 [[S]], i32 [[A]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %ao = tail call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 %x, i32 %y)
+  %o = extractvalue { i32, i1 } %ao, 1
+  %a = extractvalue { i32, i1 } %ao, 0
+  %c = icmp slt i32 %x, 0
+  %s = select i1 %c, i32 -128, i32 127
+  %r = select i1 %o, i32 %s, i32 %a
+  ret i32 %r
+}
+
+declare { i32, i1 } @llvm.uadd.with.overflow.i32(i32 %0, i32 %1)
+declare { i32, i1 } @llvm.usub.with.overflow.i32(i32 %0, i32 %1)
+declare { i8, i1 } @llvm.sadd.with.overflow.i8(i8 %0, i8 %1)
+declare { i8, i1 } @llvm.ssub.with.overflow.i8(i8 %0, i8 %1)
+declare { i32, i1 } @llvm.sadd.with.overflow.i32(i32 %0, i32 %1)
+declare { i32, i1 } @llvm.ssub.with.overflow.i32(i32 %0, i32 %1)
diff --git a/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-after-truncation-variant-a.ll b/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-after-truncation-variant-a.ll
new file mode 100644
index 000000000..bc84e7a61
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-after-truncation-variant-a.ll
@@ -0,0 +1,273 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instcombine -S | FileCheck %s
+
+; If we have some pattern that leaves only some low bits set, and then performs
+; left-shift of those bits, we can combine those two shifts into a shift+mask.
+
+; There are many variants to this pattern:
+;   a)  (trunc (x & ((1 << maskNbits) - 1))) << shiftNbits
+; simplify to:
+;   ((trunc(x)) << shiftNbits) & (~(-1 << (maskNbits+shiftNbits)))
+
+; Simple tests.
+
+declare void @use32(i32)
+declare void @use64(i64)
+
+define i32 @t0_basic(i64 %x, i32 %nbits) {
+; CHECK-LABEL: @t0_basic(
+; CHECK-NEXT:    [[T0:%.*]] = add i32 [[NBITS:%.*]], -1
+; CHECK-NEXT:    [[T1:%.*]] = zext i32 [[T0]] to i64
+; CHECK-NEXT:    [[T2:%.*]] = shl i64 1, [[T1]]
+; CHECK-NEXT:    [[T3:%.*]] = add i64 [[T2]], -1
+; CHECK-NEXT:    [[T4:%.*]] = sub i32 32, [[NBITS]]
+; CHECK-NEXT:    call void @use32(i32 [[T0]])
+; CHECK-NEXT:    call void @use64(i64 [[T1]])
+; CHECK-NEXT:    call void @use64(i64 [[T2]])
+; CHECK-NEXT:    call void @use64(i64 [[T3]])
+; CHECK-NEXT:    call void @use32(i32 [[T4]])
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[X:%.*]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = shl i32 [[TMP1]], [[T4]]
+; CHECK-NEXT:    [[T7:%.*]] = and i32 [[TMP2]], 2147483647
+; CHECK-NEXT:    ret i32 [[T7]]
+;
+  %t0 = add i32 %nbits, -1
+  %t1 = zext i32 %t0 to i64
+  %t2 = shl i64 1, %t1 ; shifting by nbits-1
+  %t3 = add i64 %t2, -1
+  %t4 = sub i32 32, %nbits
+
+  call void @use32(i32 %t0)
+  call void @use64(i64 %t1)
+  call void @use64(i64 %t2)
+  call void @use64(i64 %t3)
+  call void @use32(i32 %t4)
+
+  %t5 = and i64 %t3, %x
+  %t6 = trunc i64 %t5 to i32
+  %t7 = shl i32 %t6, %t4
+  ret i32 %t7
+}
+
+; Vectors
+
+declare void @use8xi32(<8 x i32>)
+declare void @use8xi64(<8 x i64>)
+
+define <8 x i32> @t1_vec_splat(<8 x i64> %x, <8 x i32> %nbits) {
+; CHECK-LABEL: @t1_vec_splat(
+; CHECK-NEXT:    [[T0:%.*]] = add <8 x i32> [[NBITS:%.*]], <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
+; CHECK-NEXT:    [[T1:%.*]] = zext <8 x i32> [[T0]] to <8 x i64>
+; CHECK-NEXT:    [[T2:%.*]] = shl <8 x i64> <i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1>, [[T1]]
+; CHECK-NEXT:    [[T3:%.*]] = add <8 x i64> [[T2]], <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1>
+; CHECK-NEXT:    [[T4:%.*]] = sub <8 x i32> <i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32>, [[NBITS]]
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T0]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T1]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T2]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T3]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T4]])
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <8 x i64> [[X:%.*]] to <8 x i32>
+; CHECK-NEXT:    [[TMP2:%.*]] = shl <8 x i32> [[TMP1]], [[T4]]
+; CHECK-NEXT:    [[T7:%.*]] = and <8 x i32> [[TMP2]], <i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647>
+; CHECK-NEXT:    ret <8 x i32> [[T7]]
+;
+  %t0 = add <8 x i32> %nbits, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
+  %t1 = zext <8 x i32> %t0 to <8 x i64>
+  %t2 = shl <8 x i64> <i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1>, %t1 ; shifting by nbits-1
+  %t3 = add <8 x i64> %t2, <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1>
+  %t4 = sub <8 x i32> <i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32>, %nbits
+
+  call void @use8xi32(<8 x i32> %t0)
+  call void @use8xi64(<8 x i64> %t1)
+  call void @use8xi64(<8 x i64> %t2)
+  call void @use8xi64(<8 x i64> %t3)
+  call void @use8xi32(<8 x i32> %t4)
+
+  %t5 = and <8 x i64> %t3, %x
+  %t6 = trunc <8 x i64> %t5 to <8 x i32>
+  %t7 = shl <8 x i32> %t6, %t4
+  ret <8 x i32> %t7
+}
+
+define <8 x i32> @t2_vec_splat_undef(<8 x i64> %x, <8 x i32> %nbits) {
+; CHECK-LABEL: @t2_vec_splat_undef(
+; CHECK-NEXT:    [[T0:%.*]] = add <8 x i32> [[NBITS:%.*]], <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 undef, i32 -1>
+; CHECK-NEXT:    [[T1:%.*]] = zext <8 x i32> [[T0]] to <8 x i64>
+; CHECK-NEXT:    [[T2:%.*]] = shl <8 x i64> <i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 undef, i64 1>, [[T1]]
+; CHECK-NEXT:    [[T3:%.*]] = add <8 x i64> [[T2]], <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 undef, i64 -1>
+; CHECK-NEXT:    [[T4:%.*]] = sub <8 x i32> <i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 undef, i32 32>, [[NBITS]]
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T0]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T1]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T2]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T3]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T4]])
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <8 x i64> [[X:%.*]] to <8 x i32>
+; CHECK-NEXT:    [[TMP2:%.*]] = shl <8 x i32> [[TMP1]], [[T4]]
+; CHECK-NEXT:    [[T7:%.*]] = and <8 x i32> [[TMP2]], <i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 undef, i32 2147483647>
+; CHECK-NEXT:    ret <8 x i32> [[T7]]
+;
+  %t0 = add <8 x i32> %nbits, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 undef, i32 -1>
+  %t1 = zext <8 x i32> %t0 to <8 x i64>
+  %t2 = shl <8 x i64> <i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 undef, i64 1>, %t1 ; shifting by nbits-1
+  %t3 = add <8 x i64> %t2, <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 undef, i64 -1>
+  %t4 = sub <8 x i32> <i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 undef, i32 32>, %nbits
+
+  call void @use8xi32(<8 x i32> %t0)
+  call void @use8xi64(<8 x i64> %t1)
+  call void @use8xi64(<8 x i64> %t2)
+  call void @use8xi64(<8 x i64> %t3)
+  call void @use8xi32(<8 x i32> %t4)
+
+  %t5 = and <8 x i64> %t3, %x
+  %t6 = trunc <8 x i64> %t5 to <8 x i32>
+  %t7 = shl <8 x i32> %t6, %t4
+  ret <8 x i32> %t7
+}
+
+define <8 x i32> @t3_vec_nonsplat(<8 x i64> %x, <8 x i32> %nbits) {
+; CHECK-LABEL: @t3_vec_nonsplat(
+; CHECK-NEXT:    [[T0:%.*]] = add <8 x i32> [[NBITS:%.*]], <i32 -33, i32 -32, i32 -31, i32 -1, i32 0, i32 1, i32 31, i32 32>
+; CHECK-NEXT:    [[T1:%.*]] = zext <8 x i32> [[T0]] to <8 x i64>
+; CHECK-NEXT:    [[T2:%.*]] = shl <8 x i64> <i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1>, [[T1]]
+; CHECK-NEXT:    [[T3:%.*]] = add <8 x i64> [[T2]], <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1>
+; CHECK-NEXT:    [[T4:%.*]] = sub <8 x i32> <i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32>, [[NBITS]]
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T0]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T1]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T2]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T3]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T4]])
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <8 x i64> [[X:%.*]] to <8 x i32>
+; CHECK-NEXT:    [[TMP2:%.*]] = shl <8 x i32> [[TMP1]], [[T4]]
+; CHECK-NEXT:    [[T7:%.*]] = and <8 x i32> [[TMP2]], <i32 undef, i32 0, i32 1, i32 2147483647, i32 -1, i32 -1, i32 -1, i32 -1>
+; CHECK-NEXT:    ret <8 x i32> [[T7]]
+;
+  %t0 = add <8 x i32> %nbits, <i32 -33, i32 -32, i32 -31, i32 -1, i32 0, i32 1, i32 31, i32 32>
+  %t1 = zext <8 x i32> %t0 to <8 x i64>
+  %t2 = shl <8 x i64> <i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1>, %t1 ; shifting by nbits-1
+  %t3 = add <8 x i64> %t2, <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1>
+  %t4 = sub <8 x i32> <i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32>, %nbits
+
+  call void @use8xi32(<8 x i32> %t0)
+  call void @use8xi64(<8 x i64> %t1)
+  call void @use8xi64(<8 x i64> %t2)
+  call void @use8xi64(<8 x i64> %t3)
+  call void @use8xi32(<8 x i32> %t4)
+
+  %t5 = and <8 x i64> %t3, %x
+  %t6 = trunc <8 x i64> %t5 to <8 x i32>
+  %t7 = shl <8 x i32> %t6, %t4
+  ret <8 x i32> %t7
+}
+
+; Extra uses
+
+define i32 @n4_extrause0(i64 %x, i32 %nbits) {
+; CHECK-LABEL: @n4_extrause0(
+; CHECK-NEXT:    [[T0:%.*]] = add i32 [[NBITS:%.*]], -1
+; CHECK-NEXT:    [[T1:%.*]] = zext i32 [[T0]] to i64
+; CHECK-NEXT:    [[T2:%.*]] = shl i64 1, [[T1]]
+; CHECK-NEXT:    [[T3:%.*]] = add i64 [[T2]], -1
+; CHECK-NEXT:    [[T4:%.*]] = sub i32 32, [[NBITS]]
+; CHECK-NEXT:    call void @use32(i32 [[T0]])
+; CHECK-NEXT:    call void @use64(i64 [[T1]])
+; CHECK-NEXT:    call void @use64(i64 [[T2]])
+; CHECK-NEXT:    call void @use64(i64 [[T3]])
+; CHECK-NEXT:    call void @use32(i32 [[T4]])
+; CHECK-NEXT:    [[T5:%.*]] = and i64 [[T3]], [[X:%.*]]
+; CHECK-NEXT:    call void @use64(i64 [[T5]])
+; CHECK-NEXT:    [[T6:%.*]] = trunc i64 [[T5]] to i32
+; CHECK-NEXT:    [[T7:%.*]] = shl i32 [[T6]], [[T4]]
+; CHECK-NEXT:    ret i32 [[T7]]
+;
+  %t0 = add i32 %nbits, -1
+  %t1 = zext i32 %t0 to i64
+  %t2 = shl i64 1, %t1 ; shifting by nbits-1
+  %t3 = add i64 %t2, -1
+  %t4 = sub i32 32, %nbits
+
+  call void @use32(i32 %t0)
+  call void @use64(i64 %t1)
+  call void @use64(i64 %t2)
+  call void @use64(i64 %t3)
+  call void @use32(i32 %t4)
+
+  %t5 = and i64 %t3, %x
+  call void @use64(i64 %t5)
+  %t6 = trunc i64 %t5 to i32
+  %t7 = shl i32 %t6, %t4
+  ret i32 %t7
+}
+define i32 @n5_extrause1(i64 %x, i32 %nbits) {
+; CHECK-LABEL: @n5_extrause1(
+; CHECK-NEXT:    [[T0:%.*]] = add i32 [[NBITS:%.*]], -1
+; CHECK-NEXT:    [[T1:%.*]] = zext i32 [[T0]] to i64
+; CHECK-NEXT:    [[T2:%.*]] = shl i64 1, [[T1]]
+; CHECK-NEXT:    [[T3:%.*]] = add i64 [[T2]], -1
+; CHECK-NEXT:    [[T4:%.*]] = sub i32 32, [[NBITS]]
+; CHECK-NEXT:    call void @use32(i32 [[T0]])
+; CHECK-NEXT:    call void @use64(i64 [[T1]])
+; CHECK-NEXT:    call void @use64(i64 [[T2]])
+; CHECK-NEXT:    call void @use64(i64 [[T3]])
+; CHECK-NEXT:    call void @use32(i32 [[T4]])
+; CHECK-NEXT:    [[T5:%.*]] = and i64 [[T3]], [[X:%.*]]
+; CHECK-NEXT:    [[T6:%.*]] = trunc i64 [[T5]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[T6]])
+; CHECK-NEXT:    [[T7:%.*]] = shl i32 [[T6]], [[T4]]
+; CHECK-NEXT:    ret i32 [[T7]]
+;
+  %t0 = add i32 %nbits, -1
+  %t1 = zext i32 %t0 to i64
+  %t2 = shl i64 1, %t1 ; shifting by nbits-1
+  %t3 = add i64 %t2, -1
+  %t4 = sub i32 32, %nbits
+
+  call void @use32(i32 %t0)
+  call void @use64(i64 %t1)
+  call void @use64(i64 %t2)
+  call void @use64(i64 %t3)
+  call void @use32(i32 %t4)
+
+  %t5 = and i64 %t3, %x
+  %t6 = trunc i64 %t5 to i32
+  call void @use32(i32 %t6)
+  %t7 = shl i32 %t6, %t4
+  ret i32 %t7
+}
+define i32 @n6_extrause2(i64 %x, i32 %nbits) {
+; CHECK-LABEL: @n6_extrause2(
+; CHECK-NEXT:    [[T0:%.*]] = add i32 [[NBITS:%.*]], -1
+; CHECK-NEXT:    [[T1:%.*]] = zext i32 [[T0]] to i64
+; CHECK-NEXT:    [[T2:%.*]] = shl i64 1, [[T1]]
+; CHECK-NEXT:    [[T3:%.*]] = add i64 [[T2]], -1
+; CHECK-NEXT:    [[T4:%.*]] = sub i32 32, [[NBITS]]
+; CHECK-NEXT:    call void @use32(i32 [[T0]])
+; CHECK-NEXT:    call void @use64(i64 [[T1]])
+; CHECK-NEXT:    call void @use64(i64 [[T2]])
+; CHECK-NEXT:    call void @use64(i64 [[T3]])
+; CHECK-NEXT:    call void @use32(i32 [[T4]])
+; CHECK-NEXT:    [[T5:%.*]] = and i64 [[T3]], [[X:%.*]]
+; CHECK-NEXT:    call void @use64(i64 [[T5]])
+; CHECK-NEXT:    [[T6:%.*]] = trunc i64 [[T5]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[T6]])
+; CHECK-NEXT:    [[T7:%.*]] = shl i32 [[T6]], [[T4]]
+; CHECK-NEXT:    ret i32 [[T7]]
+;
+  %t0 = add i32 %nbits, -1
+  %t1 = zext i32 %t0 to i64
+  %t2 = shl i64 1, %t1 ; shifting by nbits-1
+  %t3 = add i64 %t2, -1
+  %t4 = sub i32 32, %nbits
+
+  call void @use32(i32 %t0)
+  call void @use64(i64 %t1)
+  call void @use64(i64 %t2)
+  call void @use64(i64 %t3)
+  call void @use32(i32 %t4)
+
+  %t5 = and i64 %t3, %x
+  call void @use64(i64 %t5)
+  %t6 = trunc i64 %t5 to i32
+  call void @use32(i32 %t6)
+  %t7 = shl i32 %t6, %t4
+  ret i32 %t7
+}
diff --git a/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-after-truncation-variant-b.ll b/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-after-truncation-variant-b.ll
new file mode 100644
index 000000000..65bc2f244
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-after-truncation-variant-b.ll
@@ -0,0 +1,310 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instcombine -S | FileCheck %s
+
+; If we have some pattern that leaves only some low bits set, and then performs
+; left-shift of those bits, we can combine those two shifts into a shift+mask.
+
+; There are many variants to this pattern:
+;   b)  (trunc ((x & (~(-1 << maskNbits))))) << shiftNbits
+; simplify to:
+;   ((trunc(x)) << shiftNbits) & (~(-1 << (maskNbits+shiftNbits)))
+
+; Simple tests.
+
+declare void @use32(i32)
+declare void @use64(i64)
+
+define i32 @t0_basic(i64 %x, i32 %nbits) {
+; CHECK-LABEL: @t0_basic(
+; CHECK-NEXT:    [[T0:%.*]] = add i32 [[NBITS:%.*]], -1
+; CHECK-NEXT:    [[T1:%.*]] = zext i32 [[T0]] to i64
+; CHECK-NEXT:    [[T2:%.*]] = shl i64 -1, [[T1]]
+; CHECK-NEXT:    [[T3:%.*]] = xor i64 [[T2]], -1
+; CHECK-NEXT:    [[T4:%.*]] = sub i32 32, [[NBITS]]
+; CHECK-NEXT:    call void @use32(i32 [[T0]])
+; CHECK-NEXT:    call void @use64(i64 [[T1]])
+; CHECK-NEXT:    call void @use64(i64 [[T2]])
+; CHECK-NEXT:    call void @use64(i64 [[T3]])
+; CHECK-NEXT:    call void @use32(i32 [[T4]])
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[X:%.*]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = shl i32 [[TMP1]], [[T4]]
+; CHECK-NEXT:    [[T7:%.*]] = and i32 [[TMP2]], 2147483647
+; CHECK-NEXT:    ret i32 [[T7]]
+;
+  %t0 = add i32 %nbits, -1
+  %t1 = zext i32 %t0 to i64
+  %t2 = shl i64 -1, %t1 ; shifting by nbits-1
+  %t3 = xor i64 %t2, -1
+  %t4 = sub i32 32, %nbits
+
+  call void @use32(i32 %t0)
+  call void @use64(i64 %t1)
+  call void @use64(i64 %t2)
+  call void @use64(i64 %t3)
+  call void @use32(i32 %t4)
+
+  %t5 = and i64 %t3, %x
+  %t6 = trunc i64 %t5 to i32
+  %t7 = shl i32 %t6, %t4
+  ret i32 %t7
+}
+
+; Vectors
+
+declare void @use8xi32(<8 x i32>)
+declare void @use8xi64(<8 x i64>)
+
+define <8 x i32> @t1_vec_splat(<8 x i64> %x, <8 x i32> %nbits) {
+; CHECK-LABEL: @t1_vec_splat(
+; CHECK-NEXT:    [[T0:%.*]] = add <8 x i32> [[NBITS:%.*]], <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
+; CHECK-NEXT:    [[T1:%.*]] = zext <8 x i32> [[T0]] to <8 x i64>
+; CHECK-NEXT:    [[T2:%.*]] = shl <8 x i64> <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1>, [[T1]]
+; CHECK-NEXT:    [[T3:%.*]] = xor <8 x i64> [[T2]], <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1>
+; CHECK-NEXT:    [[T4:%.*]] = sub <8 x i32> <i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32>, [[NBITS]]
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T0]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T1]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T2]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T3]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T4]])
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <8 x i64> [[X:%.*]] to <8 x i32>
+; CHECK-NEXT:    [[TMP2:%.*]] = shl <8 x i32> [[TMP1]], [[T4]]
+; CHECK-NEXT:    [[T7:%.*]] = and <8 x i32> [[TMP2]], <i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647>
+; CHECK-NEXT:    ret <8 x i32> [[T7]]
+;
+  %t0 = add <8 x i32> %nbits, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
+  %t1 = zext <8 x i32> %t0 to <8 x i64>
+  %t2 = shl <8 x i64> <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1>, %t1 ; shifting by nbits-1
+  %t3 = xor <8 x i64> %t2, <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1>
+  %t4 = sub <8 x i32> <i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32>, %nbits
+
+  call void @use8xi32(<8 x i32> %t0)
+  call void @use8xi64(<8 x i64> %t1)
+  call void @use8xi64(<8 x i64> %t2)
+  call void @use8xi64(<8 x i64> %t3)
+  call void @use8xi32(<8 x i32> %t4)
+
+  %t5 = and <8 x i64> %t3, %x
+  %t6 = trunc <8 x i64> %t5 to <8 x i32>
+  %t7 = shl <8 x i32> %t6, %t4
+  ret <8 x i32> %t7
+}
+
+define <8 x i32> @t2_vec_splat_undef(<8 x i64> %x, <8 x i32> %nbits) {
+; CHECK-LABEL: @t2_vec_splat_undef(
+; CHECK-NEXT:    [[T0:%.*]] = add <8 x i32> [[NBITS:%.*]], <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 undef, i32 -1>
+; CHECK-NEXT:    [[T1:%.*]] = zext <8 x i32> [[T0]] to <8 x i64>
+; CHECK-NEXT:    [[T2:%.*]] = shl <8 x i64> <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 undef, i64 -1>, [[T1]]
+; CHECK-NEXT:    [[T3:%.*]] = xor <8 x i64> [[T2]], <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 undef, i64 -1>
+; CHECK-NEXT:    [[T4:%.*]] = sub <8 x i32> <i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 undef, i32 32>, [[NBITS]]
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T0]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T1]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T2]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T3]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T4]])
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <8 x i64> [[X:%.*]] to <8 x i32>
+; CHECK-NEXT:    [[TMP2:%.*]] = shl <8 x i32> [[TMP1]], [[T4]]
+; CHECK-NEXT:    [[T7:%.*]] = and <8 x i32> [[TMP2]], <i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 undef, i32 2147483647>
+; CHECK-NEXT:    ret <8 x i32> [[T7]]
+;
+  %t0 = add <8 x i32> %nbits, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 undef, i32 -1>
+  %t1 = zext <8 x i32> %t0 to <8 x i64>
+  %t2 = shl <8 x i64> <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 undef, i64 -1>, %t1 ; shifting by nbits-1
+  %t3 = xor <8 x i64> %t2, <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 undef, i64 -1>
+  %t4 = sub <8 x i32> <i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 undef, i32 32>, %nbits
+
+  call void @use8xi32(<8 x i32> %t0)
+  call void @use8xi64(<8 x i64> %t1)
+  call void @use8xi64(<8 x i64> %t2)
+  call void @use8xi64(<8 x i64> %t3)
+  call void @use8xi32(<8 x i32> %t4)
+
+  %t5 = and <8 x i64> %t3, %x
+  %t6 = trunc <8 x i64> %t5 to <8 x i32>
+  %t7 = shl <8 x i32> %t6, %t4
+  ret <8 x i32> %t7
+}
+
+define <8 x i32> @t3_vec_nonsplat(<8 x i64> %x, <8 x i32> %nbits) {
+; CHECK-LABEL: @t3_vec_nonsplat(
+; CHECK-NEXT:    [[T0:%.*]] = add <8 x i32> [[NBITS:%.*]], <i32 -33, i32 -32, i32 -31, i32 -1, i32 0, i32 1, i32 31, i32 32>
+; CHECK-NEXT:    [[T1:%.*]] = zext <8 x i32> [[T0]] to <8 x i64>
+; CHECK-NEXT:    [[T2:%.*]] = shl <8 x i64> <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1>, [[T1]]
+; CHECK-NEXT:    [[T3:%.*]] = xor <8 x i64> [[T2]], <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1>
+; CHECK-NEXT:    [[T4:%.*]] = sub <8 x i32> <i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32>, [[NBITS]]
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T0]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T1]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T2]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T3]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T4]])
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <8 x i64> [[X:%.*]] to <8 x i32>
+; CHECK-NEXT:    [[TMP2:%.*]] = shl <8 x i32> [[TMP1]], [[T4]]
+; CHECK-NEXT:    [[T7:%.*]] = and <8 x i32> [[TMP2]], <i32 undef, i32 0, i32 1, i32 2147483647, i32 -1, i32 -1, i32 -1, i32 -1>
+; CHECK-NEXT:    ret <8 x i32> [[T7]]
+;
+  %t0 = add <8 x i32> %nbits, <i32 -33, i32 -32, i32 -31, i32 -1, i32 0, i32 1, i32 31, i32 32>
+  %t1 = zext <8 x i32> %t0 to <8 x i64>
+  %t2 = shl <8 x i64> <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1>, %t1 ; shifting by nbits-1
+  %t3 = xor <8 x i64> %t2, <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1>
+  %t4 = sub <8 x i32> <i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32>, %nbits
+
+  call void @use8xi32(<8 x i32> %t0)
+  call void @use8xi64(<8 x i64> %t1)
+  call void @use8xi64(<8 x i64> %t2)
+  call void @use8xi64(<8 x i64> %t3)
+  call void @use8xi32(<8 x i32> %t4)
+
+  %t5 = and <8 x i64> %t3, %x
+  %t6 = trunc <8 x i64> %t5 to <8 x i32>
+  %t7 = shl <8 x i32> %t6, %t4
+  ret <8 x i32> %t7
+}
+
+; -1 can be truncated.
+
+define i32 @t4_allones_trunc(i64 %x, i32 %nbits) {
+; CHECK-LABEL: @t4_allones_trunc(
+; CHECK-NEXT:    [[T0:%.*]] = add i32 [[NBITS:%.*]], -1
+; CHECK-NEXT:    [[T1:%.*]] = zext i32 [[T0]] to i64
+; CHECK-NEXT:    [[T2:%.*]] = shl i64 -1, [[T1]]
+; CHECK-NEXT:    [[T3:%.*]] = xor i64 [[T2]], 4294967295
+; CHECK-NEXT:    [[T4:%.*]] = sub i32 32, [[NBITS]]
+; CHECK-NEXT:    call void @use32(i32 [[T0]])
+; CHECK-NEXT:    call void @use64(i64 [[T1]])
+; CHECK-NEXT:    call void @use64(i64 [[T2]])
+; CHECK-NEXT:    call void @use64(i64 [[T3]])
+; CHECK-NEXT:    call void @use32(i32 [[T4]])
+; CHECK-NEXT:    [[T5:%.*]] = and i64 [[T3]], [[X:%.*]]
+; CHECK-NEXT:    [[T6:%.*]] = trunc i64 [[T5]] to i32
+; CHECK-NEXT:    [[T7:%.*]] = shl i32 [[T6]], [[T4]]
+; CHECK-NEXT:    ret i32 [[T7]]
+;
+  %t0 = add i32 %nbits, -1
+  %t1 = zext i32 %t0 to i64
+  %t2 = shl i64 -1, %t1 ; shifting by nbits-1
+  %t3 = xor i64 %t2, 4294967295 ; we only care about low 32 bits
+  %t4 = sub i32 32, %nbits
+
+  call void @use32(i32 %t0)
+  call void @use64(i64 %t1)
+  call void @use64(i64 %t2)
+  call void @use64(i64 %t3)
+  call void @use32(i32 %t4)
+
+  %t5 = and i64 %t3, %x
+  %t6 = trunc i64 %t5 to i32
+  %t7 = shl i32 %t6, %t4
+  ret i32 %t7
+}
+
+; Extra uses
+
+define i32 @n5_extrause0(i64 %x, i32 %nbits) {
+; CHECK-LABEL: @n5_extrause0(
+; CHECK-NEXT:    [[T0:%.*]] = add i32 [[NBITS:%.*]], -1
+; CHECK-NEXT:    [[T1:%.*]] = zext i32 [[T0]] to i64
+; CHECK-NEXT:    [[T2:%.*]] = shl i64 -1, [[T1]]
+; CHECK-NEXT:    [[T3:%.*]] = xor i64 [[T2]], -1
+; CHECK-NEXT:    [[T4:%.*]] = sub i32 32, [[NBITS]]
+; CHECK-NEXT:    call void @use32(i32 [[T0]])
+; CHECK-NEXT:    call void @use64(i64 [[T1]])
+; CHECK-NEXT:    call void @use64(i64 [[T2]])
+; CHECK-NEXT:    call void @use64(i64 [[T3]])
+; CHECK-NEXT:    call void @use32(i32 [[T4]])
+; CHECK-NEXT:    [[T5:%.*]] = and i64 [[T3]], [[X:%.*]]
+; CHECK-NEXT:    call void @use64(i64 [[T5]])
+; CHECK-NEXT:    [[T6:%.*]] = trunc i64 [[T5]] to i32
+; CHECK-NEXT:    [[T7:%.*]] = shl i32 [[T6]], [[T4]]
+; CHECK-NEXT:    ret i32 [[T7]]
+;
+  %t0 = add i32 %nbits, -1
+  %t1 = zext i32 %t0 to i64
+  %t2 = shl i64 -1, %t1 ; shifting by nbits-1
+  %t3 = xor i64 %t2, -1
+  %t4 = sub i32 32, %nbits
+
+  call void @use32(i32 %t0)
+  call void @use64(i64 %t1)
+  call void @use64(i64 %t2)
+  call void @use64(i64 %t3)
+  call void @use32(i32 %t4)
+
+  %t5 = and i64 %t3, %x
+  call void @use64(i64 %t5)
+  %t6 = trunc i64 %t5 to i32
+  %t7 = shl i32 %t6, %t4
+  ret i32 %t7
+}
+define i32 @n6_extrause1(i64 %x, i32 %nbits) {
+; CHECK-LABEL: @n6_extrause1(
+; CHECK-NEXT:    [[T0:%.*]] = add i32 [[NBITS:%.*]], -1
+; CHECK-NEXT:    [[T1:%.*]] = zext i32 [[T0]] to i64
+; CHECK-NEXT:    [[T2:%.*]] = shl i64 -1, [[T1]]
+; CHECK-NEXT:    [[T3:%.*]] = xor i64 [[T2]], -1
+; CHECK-NEXT:    [[T4:%.*]] = sub i32 32, [[NBITS]]
+; CHECK-NEXT:    call void @use32(i32 [[T0]])
+; CHECK-NEXT:    call void @use64(i64 [[T1]])
+; CHECK-NEXT:    call void @use64(i64 [[T2]])
+; CHECK-NEXT:    call void @use64(i64 [[T3]])
+; CHECK-NEXT:    call void @use32(i32 [[T4]])
+; CHECK-NEXT:    [[T5:%.*]] = and i64 [[T3]], [[X:%.*]]
+; CHECK-NEXT:    [[T6:%.*]] = trunc i64 [[T5]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[T6]])
+; CHECK-NEXT:    [[T7:%.*]] = shl i32 [[T6]], [[T4]]
+; CHECK-NEXT:    ret i32 [[T7]]
+;
+  %t0 = add i32 %nbits, -1
+  %t1 = zext i32 %t0 to i64
+  %t2 = shl i64 -1, %t1 ; shifting by nbits-1
+  %t3 = xor i64 %t2, -1
+  %t4 = sub i32 32, %nbits
+
+  call void @use32(i32 %t0)
+  call void @use64(i64 %t1)
+  call void @use64(i64 %t2)
+  call void @use64(i64 %t3)
+  call void @use32(i32 %t4)
+
+  %t5 = and i64 %t3, %x
+  %t6 = trunc i64 %t5 to i32
+  call void @use32(i32 %t6)
+  %t7 = shl i32 %t6, %t4
+  ret i32 %t7
+}
+define i32 @n7_extrause2(i64 %x, i32 %nbits) {
+; CHECK-LABEL: @n7_extrause2(
+; CHECK-NEXT:    [[T0:%.*]] = add i32 [[NBITS:%.*]], -1
+; CHECK-NEXT:    [[T1:%.*]] = zext i32 [[T0]] to i64
+; CHECK-NEXT:    [[T2:%.*]] = shl i64 -1, [[T1]]
+; CHECK-NEXT:    [[T3:%.*]] = xor i64 [[T2]], -1
+; CHECK-NEXT:    [[T4:%.*]] = sub i32 32, [[NBITS]]
+; CHECK-NEXT:    call void @use32(i32 [[T0]])
+; CHECK-NEXT:    call void @use64(i64 [[T1]])
+; CHECK-NEXT:    call void @use64(i64 [[T2]])
+; CHECK-NEXT:    call void @use64(i64 [[T3]])
+; CHECK-NEXT:    call void @use32(i32 [[T4]])
+; CHECK-NEXT:    [[T5:%.*]] = and i64 [[T3]], [[X:%.*]]
+; CHECK-NEXT:    call void @use64(i64 [[T5]])
+; CHECK-NEXT:    [[T6:%.*]] = trunc i64 [[T5]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[T6]])
+; CHECK-NEXT:    [[T7:%.*]] = shl i32 [[T6]], [[T4]]
+; CHECK-NEXT:    ret i32 [[T7]]
+;
+  %t0 = add i32 %nbits, -1
+  %t1 = zext i32 %t0 to i64
+  %t2 = shl i64 -1, %t1 ; shifting by nbits-1
+  %t3 = xor i64 %t2, -1
+  %t4 = sub i32 32, %nbits
+
+  call void @use32(i32 %t0)
+  call void @use64(i64 %t1)
+  call void @use64(i64 %t2)
+  call void @use64(i64 %t3)
+  call void @use32(i32 %t4)
+
+  %t5 = and i64 %t3, %x
+  call void @use64(i64 %t5)
+  %t6 = trunc i64 %t5 to i32
+  call void @use32(i32 %t6)
+  %t7 = shl i32 %t6, %t4
+  ret i32 %t7
+}
diff --git a/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-after-truncation-variant-c.ll b/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-after-truncation-variant-c.ll
new file mode 100644
index 000000000..9203550b7
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-after-truncation-variant-c.ll
@@ -0,0 +1,219 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instcombine -S | FileCheck %s
+
+; If we have some pattern that leaves only some low bits set, and then performs
+; left-shift of those bits, we can combine those two shifts into a shift+mask.
+
+; There are many variants to this pattern:
+;   c)  (trunc ((x & (-1 >> maskNbits)))) << shiftNbits
+; simplify to:
+;   ((trunc(x)) << shiftNbits) & (-1 >> ((-(maskNbits+shiftNbits))+32))
+
+; Simple tests.
+
+declare void @use32(i32)
+declare void @use64(i64)
+
+define i32 @t0_basic(i64 %x, i32 %nbits) {
+; CHECK-LABEL: @t0_basic(
+; CHECK-NEXT:    [[T0:%.*]] = zext i32 [[NBITS:%.*]] to i64
+; CHECK-NEXT:    [[T1:%.*]] = lshr i64 -1, [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[NBITS]], -33
+; CHECK-NEXT:    call void @use64(i64 [[T0]])
+; CHECK-NEXT:    call void @use64(i64 [[T1]])
+; CHECK-NEXT:    call void @use32(i32 [[T2]])
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[X:%.*]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = shl i32 [[TMP1]], [[T2]]
+; CHECK-NEXT:    [[T5:%.*]] = and i32 [[TMP2]], 2147483647
+; CHECK-NEXT:    ret i32 [[T5]]
+;
+  %t0 = zext i32 %nbits to i64
+  %t1 = lshr i64 -1, %t0
+  %t2 = add i32 %nbits, -33
+
+  call void @use64(i64 %t0)
+  call void @use64(i64 %t1)
+  call void @use32(i32 %t2)
+
+  %t3 = and i64 %t1, %x
+  %t4 = trunc i64 %t3 to i32
+  %t5 = shl i32 %t4, %t2 ; shift is smaller than mask
+  ret i32 %t5
+}
+
+; Vectors
+
+declare void @use8xi32(<8 x i32>)
+declare void @use8xi64(<8 x i64>)
+
+define <8 x i32> @t1_vec_splat(<8 x i64> %x, <8 x i32> %nbits) {
+; CHECK-LABEL: @t1_vec_splat(
+; CHECK-NEXT:    [[T0:%.*]] = zext <8 x i32> [[NBITS:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[T1:%.*]] = lshr <8 x i64> <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1>, [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = add <8 x i32> [[NBITS]], <i32 -33, i32 -33, i32 -33, i32 -33, i32 -33, i32 -33, i32 -33, i32 -33>
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T0]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T1]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T2]])
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <8 x i64> [[X:%.*]] to <8 x i32>
+; CHECK-NEXT:    [[TMP2:%.*]] = shl <8 x i32> [[TMP1]], [[T2]]
+; CHECK-NEXT:    [[T5:%.*]] = and <8 x i32> [[TMP2]], <i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647>
+; CHECK-NEXT:    ret <8 x i32> [[T5]]
+;
+  %t0 = zext <8 x i32> %nbits to <8 x i64>
+  %t1 = lshr <8 x i64> <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1>, %t0
+  %t2 = add <8 x i32> %nbits, <i32 -33, i32 -33, i32 -33, i32 -33, i32 -33, i32 -33, i32 -33, i32 -33>
+
+  call void @use8xi64(<8 x i64> %t0)
+  call void @use8xi64(<8 x i64> %t1)
+  call void @use8xi32(<8 x i32> %t2)
+
+  %t3 = and <8 x i64> %t1, %x
+  %t4 = trunc <8 x i64> %t3 to <8 x i32>
+  %t5 = shl <8 x i32> %t4, %t2 ; shift is smaller than mask
+  ret <8 x i32> %t5
+}
+
+define <8 x i32> @t2_vec_splat_undef(<8 x i64> %x, <8 x i32> %nbits) {
+; CHECK-LABEL: @t2_vec_splat_undef(
+; CHECK-NEXT:    [[T0:%.*]] = zext <8 x i32> [[NBITS:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[T1:%.*]] = lshr <8 x i64> <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 undef, i64 -1>, [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = add <8 x i32> [[NBITS]], <i32 -33, i32 -33, i32 -33, i32 -33, i32 -33, i32 -33, i32 undef, i32 -33>
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T0]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T1]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T2]])
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <8 x i64> [[X:%.*]] to <8 x i32>
+; CHECK-NEXT:    [[TMP2:%.*]] = shl <8 x i32> [[TMP1]], [[T2]]
+; CHECK-NEXT:    [[T5:%.*]] = and <8 x i32> [[TMP2]], <i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 undef, i32 2147483647>
+; CHECK-NEXT:    ret <8 x i32> [[T5]]
+;
+  %t0 = zext <8 x i32> %nbits to <8 x i64>
+  %t1 = lshr <8 x i64> <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 undef, i64 -1>, %t0
+  %t2 = add <8 x i32> %nbits, <i32 -33, i32 -33, i32 -33, i32 -33, i32 -33, i32 -33, i32 undef, i32 -33>
+
+  call void @use8xi64(<8 x i64> %t0)
+  call void @use8xi64(<8 x i64> %t1)
+  call void @use8xi32(<8 x i32> %t2)
+
+  %t3 = and <8 x i64> %t1, %x
+  %t4 = trunc <8 x i64> %t3 to <8 x i32>
+  %t5 = shl <8 x i32> %t4, %t2 ; shift is smaller than mask
+  ret <8 x i32> %t5
+}
+
+define <8 x i32> @t3_vec_nonsplat(<8 x i64> %x, <8 x i32> %nbits) {
+; CHECK-LABEL: @t3_vec_nonsplat(
+; CHECK-NEXT:    [[T0:%.*]] = zext <8 x i32> [[NBITS:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[T1:%.*]] = lshr <8 x i64> <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 undef, i64 -1>, [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = add <8 x i32> [[NBITS]], <i32 -64, i32 -63, i32 -33, i32 -32, i32 63, i32 64, i32 undef, i32 65>
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T0]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T1]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T2]])
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <8 x i64> [[X:%.*]] to <8 x i32>
+; CHECK-NEXT:    [[TMP2:%.*]] = shl <8 x i32> [[TMP1]], [[T2]]
+; CHECK-NEXT:    [[T5:%.*]] = and <8 x i32> [[TMP2]], <i32 undef, i32 1, i32 2147483647, i32 -1, i32 -1, i32 -1, i32 undef, i32 undef>
+; CHECK-NEXT:    ret <8 x i32> [[T5]]
+;
+  %t0 = zext <8 x i32> %nbits to <8 x i64>
+  %t1 = lshr <8 x i64> <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 undef, i64 -1>, %t0
+  %t2 = add <8 x i32> %nbits, <i32 -64, i32 -63, i32 -33, i32 -32, i32 63, i32 64, i32 undef, i32 65>
+
+  call void @use8xi64(<8 x i64> %t0)
+  call void @use8xi64(<8 x i64> %t1)
+  call void @use8xi32(<8 x i32> %t2)
+
+  %t3 = and <8 x i64> %t1, %x
+  %t4 = trunc <8 x i64> %t3 to <8 x i32>
+  %t5 = shl <8 x i32> %t4, %t2 ; shift is smaller than mask
+  ret <8 x i32> %t5
+}
+
+; Extra uses.
+
+define i32 @n4_extrause0(i64 %x, i32 %nbits) {
+; CHECK-LABEL: @n4_extrause0(
+; CHECK-NEXT:    [[T0:%.*]] = zext i32 [[NBITS:%.*]] to i64
+; CHECK-NEXT:    [[T1:%.*]] = lshr i64 -1, [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[NBITS]], -33
+; CHECK-NEXT:    call void @use64(i64 [[T0]])
+; CHECK-NEXT:    call void @use64(i64 [[T1]])
+; CHECK-NEXT:    call void @use32(i32 [[T2]])
+; CHECK-NEXT:    [[T3:%.*]] = and i64 [[T1]], [[X:%.*]]
+; CHECK-NEXT:    call void @use64(i64 [[T3]])
+; CHECK-NEXT:    [[T4:%.*]] = trunc i64 [[T3]] to i32
+; CHECK-NEXT:    [[T5:%.*]] = shl i32 [[T4]], [[T2]]
+; CHECK-NEXT:    ret i32 [[T5]]
+;
+  %t0 = zext i32 %nbits to i64
+  %t1 = lshr i64 -1, %t0
+  %t2 = add i32 %nbits, -33
+
+  call void @use64(i64 %t0)
+  call void @use64(i64 %t1)
+  call void @use32(i32 %t2)
+
+  %t3 = and i64 %t1, %x
+  call void @use64(i64 %t3)
+  %t4 = trunc i64 %t3 to i32
+  %t5 = shl i32 %t4, %t2 ; shift is smaller than mask
+  ret i32 %t5
+}
+
+define i32 @n5_extrause1(i64 %x, i32 %nbits) {
+; CHECK-LABEL: @n5_extrause1(
+; CHECK-NEXT:    [[T0:%.*]] = zext i32 [[NBITS:%.*]] to i64
+; CHECK-NEXT:    [[T1:%.*]] = lshr i64 -1, [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[NBITS]], -33
+; CHECK-NEXT:    call void @use64(i64 [[T0]])
+; CHECK-NEXT:    call void @use64(i64 [[T1]])
+; CHECK-NEXT:    call void @use32(i32 [[T2]])
+; CHECK-NEXT:    [[T3:%.*]] = and i64 [[T1]], [[X:%.*]]
+; CHECK-NEXT:    [[T4:%.*]] = trunc i64 [[T3]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[T4]])
+; CHECK-NEXT:    [[T5:%.*]] = shl i32 [[T4]], [[T2]]
+; CHECK-NEXT:    ret i32 [[T5]]
+;
+  %t0 = zext i32 %nbits to i64
+  %t1 = lshr i64 -1, %t0
+  %t2 = add i32 %nbits, -33
+
+  call void @use64(i64 %t0)
+  call void @use64(i64 %t1)
+  call void @use32(i32 %t2)
+
+  %t3 = and i64 %t1, %x
+  %t4 = trunc i64 %t3 to i32
+  call void @use32(i32 %t4)
+  %t5 = shl i32 %t4, %t2 ; shift is smaller than mask
+  ret i32 %t5
+}
+
+define i32 @n6_extrause2(i64 %x, i32 %nbits) {
+; CHECK-LABEL: @n6_extrause2(
+; CHECK-NEXT:    [[T0:%.*]] = zext i32 [[NBITS:%.*]] to i64
+; CHECK-NEXT:    [[T1:%.*]] = lshr i64 -1, [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[NBITS]], -33
+; CHECK-NEXT:    call void @use64(i64 [[T0]])
+; CHECK-NEXT:    call void @use64(i64 [[T1]])
+; CHECK-NEXT:    call void @use32(i32 [[T2]])
+; CHECK-NEXT:    [[T3:%.*]] = and i64 [[T1]], [[X:%.*]]
+; CHECK-NEXT:    call void @use64(i64 [[T3]])
+; CHECK-NEXT:    [[T4:%.*]] = trunc i64 [[T3]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[T4]])
+; CHECK-NEXT:    [[T5:%.*]] = shl i32 [[T4]], [[T2]]
+; CHECK-NEXT:    ret i32 [[T5]]
+;
+  %t0 = zext i32 %nbits to i64
+  %t1 = lshr i64 -1, %t0
+  %t2 = add i32 %nbits, -33
+
+  call void @use64(i64 %t0)
+  call void @use64(i64 %t1)
+  call void @use32(i32 %t2)
+
+  %t3 = and i64 %t1, %x
+  call void @use64(i64 %t3)
+  %t4 = trunc i64 %t3 to i32
+  call void @use32(i32 %t4)
+  %t5 = shl i32 %t4, %t2 ; shift is smaller than mask
+  ret i32 %t5
+}
diff --git a/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-after-truncation-variant-d.ll b/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-after-truncation-variant-d.ll
new file mode 100644
index 000000000..5c08183b9
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-after-truncation-variant-d.ll
@@ -0,0 +1,247 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instcombine -S | FileCheck %s
+
+; If we have some pattern that leaves only some low bits set, and then performs
+; left-shift of those bits, we can combine those two shifts into a shift+mask.
+
+; There are many variants to this pattern:
+;   d)  (trunc ((x & ((-1 << maskNbits) >> maskNbits)))) << shiftNbits
+; simplify to:
+;   ((trunc(x)) << shiftNbits) & (-1 >> ((-(maskNbits+shiftNbits))+32))
+
+; Simple tests.
+
+declare void @use32(i32)
+declare void @use64(i64)
+
+define i32 @t0_basic(i64 %x, i32 %nbits) {
+; CHECK-LABEL: @t0_basic(
+; CHECK-NEXT:    [[T0:%.*]] = zext i32 [[NBITS:%.*]] to i64
+; CHECK-NEXT:    [[T1:%.*]] = shl i64 -1, [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = lshr i64 [[T1]], [[T0]]
+; CHECK-NEXT:    [[T3:%.*]] = add i32 [[NBITS]], -33
+; CHECK-NEXT:    call void @use64(i64 [[T0]])
+; CHECK-NEXT:    call void @use64(i64 [[T1]])
+; CHECK-NEXT:    call void @use64(i64 [[T2]])
+; CHECK-NEXT:    call void @use32(i32 [[T3]])
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[X:%.*]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = shl i32 [[TMP1]], [[T3]]
+; CHECK-NEXT:    [[T6:%.*]] = and i32 [[TMP2]], 2147483647
+; CHECK-NEXT:    ret i32 [[T6]]
+;
+  %t0 = zext i32 %nbits to i64
+  %t1 = shl i64 -1, %t0
+  %t2 = lshr i64 %t1, %t0
+  %t3 = add i32 %nbits, -33
+
+  call void @use64(i64 %t0)
+  call void @use64(i64 %t1)
+  call void @use64(i64 %t2)
+  call void @use32(i32 %t3)
+
+  %t4 = and i64 %t2, %x
+  %t5 = trunc i64 %t4 to i32
+  %t6 = shl i32 %t5, %t3 ; shift is smaller than mask
+  ret i32 %t6
+}
+
+; Vectors
+
+declare void @use8xi32(<8 x i32>)
+declare void @use8xi64(<8 x i64>)
+
+define <8 x i32> @t1_vec_splat(<8 x i64> %x, <8 x i32> %nbits) {
+; CHECK-LABEL: @t1_vec_splat(
+; CHECK-NEXT:    [[T0:%.*]] = zext <8 x i32> [[NBITS:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[T1:%.*]] = shl <8 x i64> <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1>, [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = lshr <8 x i64> [[T1]], [[T0]]
+; CHECK-NEXT:    [[T3:%.*]] = add <8 x i32> [[NBITS]], <i32 -33, i32 -33, i32 -33, i32 -33, i32 -33, i32 -33, i32 -33, i32 -33>
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T0]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T1]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T2]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T3]])
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <8 x i64> [[X:%.*]] to <8 x i32>
+; CHECK-NEXT:    [[TMP2:%.*]] = shl <8 x i32> [[TMP1]], [[T3]]
+; CHECK-NEXT:    [[T6:%.*]] = and <8 x i32> [[TMP2]], <i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647>
+; CHECK-NEXT:    ret <8 x i32> [[T6]]
+;
+  %t0 = zext <8 x i32> %nbits to <8 x i64>
+  %t1 = shl <8 x i64> <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1>, %t0
+  %t2 = lshr <8 x i64> %t1, %t0
+  %t3 = add <8 x i32> %nbits, <i32 -33, i32 -33, i32 -33, i32 -33, i32 -33, i32 -33, i32 -33, i32 -33>
+
+  call void @use8xi64(<8 x i64> %t0)
+  call void @use8xi64(<8 x i64> %t1)
+  call void @use8xi64(<8 x i64> %t2)
+  call void @use8xi32(<8 x i32> %t3)
+
+  %t4 = and <8 x i64> %t2, %x
+  %t5 = trunc <8 x i64> %t4 to <8 x i32>
+  %t6 = shl <8 x i32> %t5, %t3 ; shift is smaller than mask
+  ret <8 x i32> %t6
+}
+
+define <8 x i32> @t2_vec_splat_undef(<8 x i64> %x, <8 x i32> %nbits) {
+; CHECK-LABEL: @t2_vec_splat_undef(
+; CHECK-NEXT:    [[T0:%.*]] = zext <8 x i32> [[NBITS:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[T1:%.*]] = shl <8 x i64> <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 undef, i64 -1>, [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = lshr <8 x i64> [[T1]], [[T0]]
+; CHECK-NEXT:    [[T3:%.*]] = add <8 x i32> [[NBITS]], <i32 -33, i32 -33, i32 -33, i32 -33, i32 -33, i32 -33, i32 undef, i32 -33>
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T0]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T1]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T2]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T3]])
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <8 x i64> [[X:%.*]] to <8 x i32>
+; CHECK-NEXT:    [[TMP2:%.*]] = shl <8 x i32> [[TMP1]], [[T3]]
+; CHECK-NEXT:    [[T6:%.*]] = and <8 x i32> [[TMP2]], <i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 undef, i32 2147483647>
+; CHECK-NEXT:    ret <8 x i32> [[T6]]
+;
+  %t0 = zext <8 x i32> %nbits to <8 x i64>
+  %t1 = shl <8 x i64> <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 undef, i64 -1>, %t0
+  %t2 = lshr <8 x i64> %t1, %t0
+  %t3 = add <8 x i32> %nbits, <i32 -33, i32 -33, i32 -33, i32 -33, i32 -33, i32 -33, i32 undef, i32 -33>
+
+  call void @use8xi64(<8 x i64> %t0)
+  call void @use8xi64(<8 x i64> %t1)
+  call void @use8xi64(<8 x i64> %t2)
+  call void @use8xi32(<8 x i32> %t3)
+
+  %t4 = and <8 x i64> %t2, %x
+  %t5 = trunc <8 x i64> %t4 to <8 x i32>
+  %t6 = shl <8 x i32> %t5, %t3 ; shift is smaller than mask
+  ret <8 x i32> %t6
+}
+
+define <8 x i32> @t3_vec_nonsplat(<8 x i64> %x, <8 x i32> %nbits) {
+; CHECK-LABEL: @t3_vec_nonsplat(
+; CHECK-NEXT:    [[T0:%.*]] = zext <8 x i32> [[NBITS:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[T1:%.*]] = shl <8 x i64> <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 undef, i64 -1>, [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = lshr <8 x i64> [[T1]], [[T0]]
+; CHECK-NEXT:    [[T3:%.*]] = add <8 x i32> [[NBITS]], <i32 -64, i32 -63, i32 -33, i32 -32, i32 63, i32 64, i32 undef, i32 65>
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T0]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T1]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T2]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T3]])
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <8 x i64> [[X:%.*]] to <8 x i32>
+; CHECK-NEXT:    [[TMP2:%.*]] = shl <8 x i32> [[TMP1]], [[T3]]
+; CHECK-NEXT:    [[T6:%.*]] = and <8 x i32> [[TMP2]], <i32 undef, i32 1, i32 2147483647, i32 -1, i32 -1, i32 -1, i32 undef, i32 undef>
+; CHECK-NEXT:    ret <8 x i32> [[T6]]
+;
+  %t0 = zext <8 x i32> %nbits to <8 x i64>
+  %t1 = shl <8 x i64> <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 undef, i64 -1>, %t0
+  %t2 = lshr <8 x i64> %t1, %t0
+  %t3 = add <8 x i32> %nbits, <i32 -64, i32 -63, i32 -33, i32 -32, i32 63, i32 64, i32 undef, i32 65>
+
+  call void @use8xi64(<8 x i64> %t0)
+  call void @use8xi64(<8 x i64> %t1)
+  call void @use8xi64(<8 x i64> %t2)
+  call void @use8xi32(<8 x i32> %t3)
+
+  %t4 = and <8 x i64> %t2, %x
+  %t5 = trunc <8 x i64> %t4 to <8 x i32>
+  %t6 = shl <8 x i32> %t5, %t3 ; shift is smaller than mask
+  ret <8 x i32> %t6
+}
+
+; Extra uses.
+
+define i32 @n4_extrause0(i64 %x, i32 %nbits) {
+; CHECK-LABEL: @n4_extrause0(
+; CHECK-NEXT:    [[T0:%.*]] = zext i32 [[NBITS:%.*]] to i64
+; CHECK-NEXT:    [[T1:%.*]] = shl i64 -1, [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = lshr i64 [[T1]], [[T0]]
+; CHECK-NEXT:    [[T3:%.*]] = add i32 [[NBITS]], -33
+; CHECK-NEXT:    call void @use64(i64 [[T0]])
+; CHECK-NEXT:    call void @use64(i64 [[T1]])
+; CHECK-NEXT:    call void @use64(i64 [[T2]])
+; CHECK-NEXT:    call void @use32(i32 [[T3]])
+; CHECK-NEXT:    [[T4:%.*]] = and i64 [[T2]], [[X:%.*]]
+; CHECK-NEXT:    call void @use64(i64 [[T4]])
+; CHECK-NEXT:    [[T5:%.*]] = trunc i64 [[T4]] to i32
+; CHECK-NEXT:    [[T6:%.*]] = shl i32 [[T5]], [[T3]]
+; CHECK-NEXT:    ret i32 [[T6]]
+;
+  %t0 = zext i32 %nbits to i64
+  %t1 = shl i64 -1, %t0
+  %t2 = lshr i64 %t1, %t0
+  %t3 = add i32 %nbits, -33
+
+  call void @use64(i64 %t0)
+  call void @use64(i64 %t1)
+  call void @use64(i64 %t2)
+  call void @use32(i32 %t3)
+
+  %t4 = and i64 %t2, %x
+  call void @use64(i64 %t4)
+  %t5 = trunc i64 %t4 to i32
+  %t6 = shl i32 %t5, %t3 ; shift is smaller than mask
+  ret i32 %t6
+}
+
+define i32 @n5_extrause1(i64 %x, i32 %nbits) {
+; CHECK-LABEL: @n5_extrause1(
+; CHECK-NEXT:    [[T0:%.*]] = zext i32 [[NBITS:%.*]] to i64
+; CHECK-NEXT:    [[T1:%.*]] = shl i64 -1, [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = lshr i64 [[T1]], [[T0]]
+; CHECK-NEXT:    [[T3:%.*]] = add i32 [[NBITS]], -33
+; CHECK-NEXT:    call void @use64(i64 [[T0]])
+; CHECK-NEXT:    call void @use64(i64 [[T1]])
+; CHECK-NEXT:    call void @use64(i64 [[T2]])
+; CHECK-NEXT:    call void @use32(i32 [[T3]])
+; CHECK-NEXT:    [[T4:%.*]] = and i64 [[T2]], [[X:%.*]]
+; CHECK-NEXT:    [[T5:%.*]] = trunc i64 [[T4]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[T5]])
+; CHECK-NEXT:    [[T6:%.*]] = shl i32 [[T5]], [[T3]]
+; CHECK-NEXT:    ret i32 [[T6]]
+;
+  %t0 = zext i32 %nbits to i64
+  %t1 = shl i64 -1, %t0
+  %t2 = lshr i64 %t1, %t0
+  %t3 = add i32 %nbits, -33
+
+  call void @use64(i64 %t0)
+  call void @use64(i64 %t1)
+  call void @use64(i64 %t2)
+  call void @use32(i32 %t3)
+
+  %t4 = and i64 %t2, %x
+  %t5 = trunc i64 %t4 to i32
+  call void @use32(i32 %t5)
+  %t6 = shl i32 %t5, %t3 ; shift is smaller than mask
+  ret i32 %t6
+}
+
+define i32 @n6_extrause2(i64 %x, i32 %nbits) {
+; CHECK-LABEL: @n6_extrause2(
+; CHECK-NEXT:    [[T0:%.*]] = zext i32 [[NBITS:%.*]] to i64
+; CHECK-NEXT:    [[T1:%.*]] = shl i64 -1, [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = lshr i64 [[T1]], [[T0]]
+; CHECK-NEXT:    [[T3:%.*]] = add i32 [[NBITS]], -33
+; CHECK-NEXT:    call void @use64(i64 [[T0]])
+; CHECK-NEXT:    call void @use64(i64 [[T1]])
+; CHECK-NEXT:    call void @use64(i64 [[T2]])
+; CHECK-NEXT:    call void @use32(i32 [[T3]])
+; CHECK-NEXT:    [[T4:%.*]] = and i64 [[T2]], [[X:%.*]]
+; CHECK-NEXT:    call void @use64(i64 [[T4]])
+; CHECK-NEXT:    [[T5:%.*]] = trunc i64 [[T4]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[T5]])
+; CHECK-NEXT:    [[T6:%.*]] = shl i32 [[T5]], [[T3]]
+; CHECK-NEXT:    ret i32 [[T6]]
+;
+  %t0 = zext i32 %nbits to i64
+  %t1 = shl i64 -1, %t0
+  %t2 = lshr i64 %t1, %t0
+  %t3 = add i32 %nbits, -33
+
+  call void @use64(i64 %t0)
+  call void @use64(i64 %t1)
+  call void @use64(i64 %t2)
+  call void @use32(i32 %t3)
+
+  %t4 = and i64 %t2, %x
+  call void @use64(i64 %t4)
+  %t5 = trunc i64 %t4 to i32
+  call void @use32(i32 %t5)
+  %t6 = shl i32 %t5, %t3 ; shift is smaller than mask
+  ret i32 %t6
+}
diff --git a/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-after-truncation-variant-e.ll b/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-after-truncation-variant-e.ll
new file mode 100644
index 000000000..1bfd95587
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-after-truncation-variant-e.ll
@@ -0,0 +1,219 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instcombine -S | FileCheck %s
+
+; If we have some pattern that leaves only some low bits set, lshr then performs
+; left-shift of those bits, we can combine those two shifts into a shift+mask.
+
+; There are many variants to this pattern:
+;   e)  (trunc (((x << maskNbits) l>> maskNbits))) << shiftNbits
+; simplify to:
+;   ((trunc(x)) << shiftNbits) & (-1 >> ((-(maskNbits+shiftNbits))+32))
+
+; Simple tests.
+
+declare void @use32(i32)
+declare void @use64(i64)
+
+define i32 @t0_basic(i64 %x, i32 %nbits) {
+; CHECK-LABEL: @t0_basic(
+; CHECK-NEXT:    [[T0:%.*]] = zext i32 [[NBITS:%.*]] to i64
+; CHECK-NEXT:    [[T1:%.*]] = shl i64 [[X:%.*]], [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[NBITS]], -33
+; CHECK-NEXT:    call void @use64(i64 [[T0]])
+; CHECK-NEXT:    call void @use64(i64 [[T1]])
+; CHECK-NEXT:    call void @use32(i32 [[T2]])
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[X]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = shl i32 [[TMP1]], [[T2]]
+; CHECK-NEXT:    [[T5:%.*]] = and i32 [[TMP2]], 2147483647
+; CHECK-NEXT:    ret i32 [[T5]]
+;
+  %t0 = zext i32 %nbits to i64
+  %t1 = shl i64 %x, %t0
+  %t2 = add i32 %nbits, -33
+
+  call void @use64(i64 %t0)
+  call void @use64(i64 %t1)
+  call void @use32(i32 %t2)
+
+  %t3 = lshr i64 %t1, %t0
+  %t4 = trunc i64 %t3 to i32
+  %t5 = shl i32 %t4, %t2 ; shift is smaller than mask
+  ret i32 %t5
+}
+
+; Vectors
+
+declare void @use8xi32(<8 x i32>)
+declare void @use8xi64(<8 x i64>)
+
+define <8 x i32> @t1_vec_splat(<8 x i64> %x, <8 x i32> %nbits) {
+; CHECK-LABEL: @t1_vec_splat(
+; CHECK-NEXT:    [[T0:%.*]] = zext <8 x i32> [[NBITS:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[T1:%.*]] = shl <8 x i64> [[X:%.*]], [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = add <8 x i32> [[NBITS]], <i32 -33, i32 -33, i32 -33, i32 -33, i32 -33, i32 -33, i32 -33, i32 -33>
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T0]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T1]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T2]])
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <8 x i64> [[X]] to <8 x i32>
+; CHECK-NEXT:    [[TMP2:%.*]] = shl <8 x i32> [[TMP1]], [[T2]]
+; CHECK-NEXT:    [[T5:%.*]] = and <8 x i32> [[TMP2]], <i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647>
+; CHECK-NEXT:    ret <8 x i32> [[T5]]
+;
+  %t0 = zext <8 x i32> %nbits to <8 x i64>
+  %t1 = shl <8 x i64> %x, %t0
+  %t2 = add <8 x i32> %nbits, <i32 -33, i32 -33, i32 -33, i32 -33, i32 -33, i32 -33, i32 -33, i32 -33>
+
+  call void @use8xi64(<8 x i64> %t0)
+  call void @use8xi64(<8 x i64> %t1)
+  call void @use8xi32(<8 x i32> %t2)
+
+  %t3 = lshr <8 x i64> %t1, %t0
+  %t4 = trunc <8 x i64> %t3 to <8 x i32>
+  %t5 = shl <8 x i32> %t4, %t2 ; shift is smaller than mask
+  ret <8 x i32> %t5
+}
+
+define <8 x i32> @t2_vec_splat_undef(<8 x i64> %x, <8 x i32> %nbits) {
+; CHECK-LABEL: @t2_vec_splat_undef(
+; CHECK-NEXT:    [[T0:%.*]] = zext <8 x i32> [[NBITS:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[T1:%.*]] = shl <8 x i64> [[X:%.*]], [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = add <8 x i32> [[NBITS]], <i32 -33, i32 -33, i32 -33, i32 -33, i32 -33, i32 -33, i32 undef, i32 -33>
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T0]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T1]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T2]])
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <8 x i64> [[X]] to <8 x i32>
+; CHECK-NEXT:    [[TMP2:%.*]] = shl <8 x i32> [[TMP1]], [[T2]]
+; CHECK-NEXT:    [[T5:%.*]] = and <8 x i32> [[TMP2]], <i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 undef, i32 2147483647>
+; CHECK-NEXT:    ret <8 x i32> [[T5]]
+;
+  %t0 = zext <8 x i32> %nbits to <8 x i64>
+  %t1 = shl <8 x i64> %x, %t0
+  %t2 = add <8 x i32> %nbits, <i32 -33, i32 -33, i32 -33, i32 -33, i32 -33, i32 -33, i32 undef, i32 -33>
+
+  call void @use8xi64(<8 x i64> %t0)
+  call void @use8xi64(<8 x i64> %t1)
+  call void @use8xi32(<8 x i32> %t2)
+
+  %t3 = lshr <8 x i64> %t1, %t0
+  %t4 = trunc <8 x i64> %t3 to <8 x i32>
+  %t5 = shl <8 x i32> %t4, %t2 ; shift is smaller than mask
+  ret <8 x i32> %t5
+}
+
+define <8 x i32> @t3_vec_nonsplat(<8 x i64> %x, <8 x i32> %nbits) {
+; CHECK-LABEL: @t3_vec_nonsplat(
+; CHECK-NEXT:    [[T0:%.*]] = zext <8 x i32> [[NBITS:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[T1:%.*]] = shl <8 x i64> [[X:%.*]], [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = add <8 x i32> [[NBITS]], <i32 -64, i32 -63, i32 -33, i32 -32, i32 63, i32 64, i32 undef, i32 65>
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T0]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T1]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T2]])
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <8 x i64> [[X]] to <8 x i32>
+; CHECK-NEXT:    [[TMP2:%.*]] = shl <8 x i32> [[TMP1]], [[T2]]
+; CHECK-NEXT:    [[T5:%.*]] = and <8 x i32> [[TMP2]], <i32 undef, i32 1, i32 2147483647, i32 -1, i32 -1, i32 -1, i32 undef, i32 undef>
+; CHECK-NEXT:    ret <8 x i32> [[T5]]
+;
+  %t0 = zext <8 x i32> %nbits to <8 x i64>
+  %t1 = shl <8 x i64> %x, %t0
+  %t2 = add <8 x i32> %nbits, <i32 -64, i32 -63, i32 -33, i32 -32, i32 63, i32 64, i32 undef, i32 65>
+
+  call void @use8xi64(<8 x i64> %t0)
+  call void @use8xi64(<8 x i64> %t1)
+  call void @use8xi32(<8 x i32> %t2)
+
+  %t3 = lshr <8 x i64> %t1, %t0
+  %t4 = trunc <8 x i64> %t3 to <8 x i32>
+  %t5 = shl <8 x i32> %t4, %t2 ; shift is smaller than mask
+  ret <8 x i32> %t5
+}
+
+; Extra uses.
+
+define i32 @n4_extrause0(i64 %x, i32 %nbits) {
+; CHECK-LABEL: @n4_extrause0(
+; CHECK-NEXT:    [[T0:%.*]] = zext i32 [[NBITS:%.*]] to i64
+; CHECK-NEXT:    [[T1:%.*]] = shl i64 [[X:%.*]], [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[NBITS]], -33
+; CHECK-NEXT:    call void @use64(i64 [[T0]])
+; CHECK-NEXT:    call void @use64(i64 [[T1]])
+; CHECK-NEXT:    call void @use32(i32 [[T2]])
+; CHECK-NEXT:    [[T3:%.*]] = lshr i64 [[T1]], [[T0]]
+; CHECK-NEXT:    call void @use64(i64 [[T3]])
+; CHECK-NEXT:    [[T4:%.*]] = trunc i64 [[T3]] to i32
+; CHECK-NEXT:    [[T5:%.*]] = shl i32 [[T4]], [[T2]]
+; CHECK-NEXT:    ret i32 [[T5]]
+;
+  %t0 = zext i32 %nbits to i64
+  %t1 = shl i64 %x, %t0
+  %t2 = add i32 %nbits, -33
+
+  call void @use64(i64 %t0)
+  call void @use64(i64 %t1)
+  call void @use32(i32 %t2)
+
+  %t3 = lshr i64 %t1, %t0
+  call void @use64(i64 %t3)
+  %t4 = trunc i64 %t3 to i32
+  %t5 = shl i32 %t4, %t2 ; shift is smaller than mask
+  ret i32 %t5
+}
+
+define i32 @n5_extrause1(i64 %x, i32 %nbits) {
+; CHECK-LABEL: @n5_extrause1(
+; CHECK-NEXT:    [[T0:%.*]] = zext i32 [[NBITS:%.*]] to i64
+; CHECK-NEXT:    [[T1:%.*]] = shl i64 [[X:%.*]], [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[NBITS]], -33
+; CHECK-NEXT:    call void @use64(i64 [[T0]])
+; CHECK-NEXT:    call void @use64(i64 [[T1]])
+; CHECK-NEXT:    call void @use32(i32 [[T2]])
+; CHECK-NEXT:    [[T3:%.*]] = lshr i64 [[T1]], [[T0]]
+; CHECK-NEXT:    [[T4:%.*]] = trunc i64 [[T3]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[T4]])
+; CHECK-NEXT:    [[T5:%.*]] = shl i32 [[T4]], [[T2]]
+; CHECK-NEXT:    ret i32 [[T5]]
+;
+  %t0 = zext i32 %nbits to i64
+  %t1 = shl i64 %x, %t0
+  %t2 = add i32 %nbits, -33
+
+  call void @use64(i64 %t0)
+  call void @use64(i64 %t1)
+  call void @use32(i32 %t2)
+
+  %t3 = lshr i64 %t1, %t0
+  %t4 = trunc i64 %t3 to i32
+  call void @use32(i32 %t4)
+  %t5 = shl i32 %t4, %t2 ; shift is smaller than mask
+  ret i32 %t5
+}
+
+define i32 @n6_extrause2(i64 %x, i32 %nbits) {
+; CHECK-LABEL: @n6_extrause2(
+; CHECK-NEXT:    [[T0:%.*]] = zext i32 [[NBITS:%.*]] to i64
+; CHECK-NEXT:    [[T1:%.*]] = shl i64 [[X:%.*]], [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[NBITS]], -33
+; CHECK-NEXT:    call void @use64(i64 [[T0]])
+; CHECK-NEXT:    call void @use64(i64 [[T1]])
+; CHECK-NEXT:    call void @use32(i32 [[T2]])
+; CHECK-NEXT:    [[T3:%.*]] = lshr i64 [[T1]], [[T0]]
+; CHECK-NEXT:    call void @use64(i64 [[T3]])
+; CHECK-NEXT:    [[T4:%.*]] = trunc i64 [[T3]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[T4]])
+; CHECK-NEXT:    [[T5:%.*]] = shl i32 [[T4]], [[T2]]
+; CHECK-NEXT:    ret i32 [[T5]]
+;
+  %t0 = zext i32 %nbits to i64
+  %t1 = shl i64 %x, %t0
+  %t2 = add i32 %nbits, -33
+
+  call void @use64(i64 %t0)
+  call void @use64(i64 %t1)
+  call void @use32(i32 %t2)
+
+  %t3 = lshr i64 %t1, %t0
+  call void @use64(i64 %t3)
+  %t4 = trunc i64 %t3 to i32
+  call void @use32(i32 %t4)
+  %t5 = shl i32 %t4, %t2 ; shift is smaller than mask
+  ret i32 %t5
+}
diff --git a/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-variant-a.ll b/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-variant-a.ll
new file mode 100644
index 000000000..205dcfbf2
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-variant-a.ll
@@ -0,0 +1,157 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instcombine -S | FileCheck %s
+
+; If we have some pattern that leaves only some low bits set, and then performs
+; left-shift of those bits, we can combine those two shifts into a shift+mask.
+
+; There are many variants to this pattern:
+;   a)  (x & ((1 << maskNbits) - 1)) << shiftNbits
+; simplify to:
+;   (x << shiftNbits) & (~(-1 << (maskNbits+shiftNbits)))
+
+; Simple tests.
+
+declare void @use32(i32)
+
+define i32 @t0_basic(i32 %x, i32 %nbits) {
+; CHECK-LABEL: @t0_basic(
+; CHECK-NEXT:    [[T0:%.*]] = add i32 [[NBITS:%.*]], -1
+; CHECK-NEXT:    [[T1:%.*]] = shl i32 1, [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[T1]], -1
+; CHECK-NEXT:    [[T4:%.*]] = sub i32 32, [[NBITS]]
+; CHECK-NEXT:    call void @use32(i32 [[T0]])
+; CHECK-NEXT:    call void @use32(i32 [[T1]])
+; CHECK-NEXT:    call void @use32(i32 [[T2]])
+; CHECK-NEXT:    call void @use32(i32 [[T4]])
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 [[X:%.*]], [[T4]]
+; CHECK-NEXT:    [[T5:%.*]] = and i32 [[TMP1]], 2147483647
+; CHECK-NEXT:    ret i32 [[T5]]
+;
+  %t0 = add i32 %nbits, -1
+  %t1 = shl i32 1, %t0 ; shifting by nbits-1
+  %t2 = add i32 %t1, -1
+  %t3 = and i32 %t2, %x
+  %t4 = sub i32 32, %nbits
+  call void @use32(i32 %t0)
+  call void @use32(i32 %t1)
+  call void @use32(i32 %t2)
+  call void @use32(i32 %t4)
+  %t5 = shl i32 %t3, %t4
+  ret i32 %t5
+}
+
+; Vectors
+
+declare void @use8xi32(<8 x i32>)
+
+define <8 x i32> @t1_vec_splat(<8 x i32> %x, <8 x i32> %nbits) {
+; CHECK-LABEL: @t1_vec_splat(
+; CHECK-NEXT:    [[T0:%.*]] = add <8 x i32> [[NBITS:%.*]], <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
+; CHECK-NEXT:    [[T1:%.*]] = shl <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>, [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = add <8 x i32> [[T1]], <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
+; CHECK-NEXT:    [[T4:%.*]] = sub <8 x i32> <i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32>, [[NBITS]]
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T0]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T1]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T2]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T4]])
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <8 x i32> [[X:%.*]], [[T4]]
+; CHECK-NEXT:    [[T5:%.*]] = and <8 x i32> [[TMP1]], <i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647>
+; CHECK-NEXT:    ret <8 x i32> [[T5]]
+;
+  %t0 = add <8 x i32> %nbits, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
+  %t1 = shl <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>, %t0
+  %t2 = add <8 x i32> %t1, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
+  %t3 = and <8 x i32> %t2, %x
+  %t4 = sub <8 x i32> <i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32>, %nbits
+  call void @use8xi32(<8 x i32> %t0)
+  call void @use8xi32(<8 x i32> %t1)
+  call void @use8xi32(<8 x i32> %t2)
+  call void @use8xi32(<8 x i32> %t4)
+  %t5 = shl <8 x i32> %t3, %t4
+  ret <8 x i32> %t5
+}
+
+define <8 x i32> @t1_vec_splat_undef(<8 x i32> %x, <8 x i32> %nbits) {
+; CHECK-LABEL: @t1_vec_splat_undef(
+; CHECK-NEXT:    [[T0:%.*]] = add <8 x i32> [[NBITS:%.*]], <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 undef, i32 -1>
+; CHECK-NEXT:    [[T1:%.*]] = shl <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 undef, i32 1>, [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = add <8 x i32> [[T1]], <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 undef, i32 -1>
+; CHECK-NEXT:    [[T4:%.*]] = sub <8 x i32> <i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 undef, i32 32>, [[NBITS]]
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T0]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T1]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T2]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T4]])
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <8 x i32> [[X:%.*]], [[T4]]
+; CHECK-NEXT:    [[T5:%.*]] = and <8 x i32> [[TMP1]], <i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 undef, i32 2147483647>
+; CHECK-NEXT:    ret <8 x i32> [[T5]]
+;
+  %t0 = add <8 x i32> %nbits, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 undef, i32 -1>
+  %t1 = shl <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 undef, i32 1>, %t0
+  %t2 = add <8 x i32> %t1, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 undef, i32 -1>
+  %t3 = and <8 x i32> %t2, %x
+  %t4 = sub <8 x i32> <i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 undef, i32 32>, %nbits
+  call void @use8xi32(<8 x i32> %t0)
+  call void @use8xi32(<8 x i32> %t1)
+  call void @use8xi32(<8 x i32> %t2)
+  call void @use8xi32(<8 x i32> %t4)
+  %t5 = shl <8 x i32> %t3, %t4
+  ret <8 x i32> %t5
+}
+
+define <8 x i32> @t2_vec_nonsplat(<8 x i32> %x, <8 x i32> %nbits) {
+; CHECK-LABEL: @t2_vec_nonsplat(
+; CHECK-NEXT:    [[T0:%.*]] = add <8 x i32> [[NBITS:%.*]], <i32 -33, i32 -32, i32 -31, i32 -1, i32 0, i32 1, i32 31, i32 32>
+; CHECK-NEXT:    [[T1:%.*]] = shl <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>, [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = add <8 x i32> [[T1]], <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
+; CHECK-NEXT:    [[T4:%.*]] = sub <8 x i32> <i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32>, [[NBITS]]
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T0]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T1]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T2]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T4]])
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <8 x i32> [[X:%.*]], [[T4]]
+; CHECK-NEXT:    [[T5:%.*]] = and <8 x i32> [[TMP1]], <i32 undef, i32 0, i32 1, i32 2147483647, i32 -1, i32 -1, i32 -1, i32 undef>
+; CHECK-NEXT:    ret <8 x i32> [[T5]]
+;
+  %t0 = add <8 x i32> %nbits, <i32 -33, i32 -32, i32 -31, i32 -1, i32 0, i32 1, i32 31, i32 32>
+  %t1 = shl <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>, %t0
+  %t2 = add <8 x i32> %t1, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
+  %t3 = and <8 x i32> %t2, %x
+  %t4 = sub <8 x i32> <i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32>, %nbits
+  call void @use8xi32(<8 x i32> %t0)
+  call void @use8xi32(<8 x i32> %t1)
+  call void @use8xi32(<8 x i32> %t2)
+  call void @use8xi32(<8 x i32> %t4)
+  %t5 = shl <8 x i32> %t3, %t4
+  ret <8 x i32> %t5
+}
+
+; Extra uses.
+
+define i32 @n3_extrause(i32 %x, i32 %nbits) {
+; CHECK-LABEL: @n3_extrause(
+; CHECK-NEXT:    [[T0:%.*]] = add i32 [[NBITS:%.*]], -1
+; CHECK-NEXT:    [[T1:%.*]] = shl i32 1, [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[T1]], -1
+; CHECK-NEXT:    [[T3:%.*]] = and i32 [[T2]], [[X:%.*]]
+; CHECK-NEXT:    [[T4:%.*]] = sub i32 32, [[NBITS]]
+; CHECK-NEXT:    call void @use32(i32 [[T0]])
+; CHECK-NEXT:    call void @use32(i32 [[T1]])
+; CHECK-NEXT:    call void @use32(i32 [[T2]])
+; CHECK-NEXT:    call void @use32(i32 [[T3]])
+; CHECK-NEXT:    call void @use32(i32 [[T4]])
+; CHECK-NEXT:    [[T5:%.*]] = shl i32 [[T3]], [[T4]]
+; CHECK-NEXT:    ret i32 [[T5]]
+;
+  %t0 = add i32 %nbits, -1
+  %t1 = shl i32 1, %t0 ; shifting by nbits-1
+  %t2 = add i32 %t1, -1
+  %t3 = and i32 %t2, %x ; this mask must be one-use.
+  %t4 = sub i32 32, %nbits
+  call void @use32(i32 %t0)
+  call void @use32(i32 %t1)
+  call void @use32(i32 %t2)
+  call void @use32(i32 %t3) ; BAD
+  call void @use32(i32 %t4)
+  %t5 = shl i32 %t3, %t4
+  ret i32 %t5
+}
diff --git a/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-variant-b.ll b/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-variant-b.ll
new file mode 100644
index 000000000..4d3d54323
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-variant-b.ll
@@ -0,0 +1,157 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instcombine -S | FileCheck %s
+
+; If we have some pattern that leaves only some low bits set, and then performs
+; left-shift of those bits, we can combine those two shifts into a shift+mask.
+
+; There are many variants to this pattern:
+;   b)  (x & (~(-1 << maskNbits))) << shiftNbits
+; simplify to:
+;   (x << shiftNbits) & (~(-1 << (maskNbits+shiftNbits)))
+
+; Simple tests.
+
+declare void @use32(i32)
+
+define i32 @t0_basic(i32 %x, i32 %nbits) {
+; CHECK-LABEL: @t0_basic(
+; CHECK-NEXT:    [[T0:%.*]] = add i32 [[NBITS:%.*]], -1
+; CHECK-NEXT:    [[T1:%.*]] = shl i32 -1, [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = xor i32 [[T1]], -1
+; CHECK-NEXT:    [[T4:%.*]] = sub i32 32, [[NBITS]]
+; CHECK-NEXT:    call void @use32(i32 [[T0]])
+; CHECK-NEXT:    call void @use32(i32 [[T1]])
+; CHECK-NEXT:    call void @use32(i32 [[T2]])
+; CHECK-NEXT:    call void @use32(i32 [[T4]])
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 [[X:%.*]], [[T4]]
+; CHECK-NEXT:    [[T5:%.*]] = and i32 [[TMP1]], 2147483647
+; CHECK-NEXT:    ret i32 [[T5]]
+;
+  %t0 = add i32 %nbits, -1
+  %t1 = shl i32 -1, %t0 ; shifting by nbits-1
+  %t2 = xor i32 %t1, -1
+  %t3 = and i32 %t2, %x
+  %t4 = sub i32 32, %nbits
+  call void @use32(i32 %t0)
+  call void @use32(i32 %t1)
+  call void @use32(i32 %t2)
+  call void @use32(i32 %t4)
+  %t5 = shl i32 %t3, %t4
+  ret i32 %t5
+}
+
+; Vectors
+
+declare void @use8xi32(<8 x i32>)
+
+define <8 x i32> @t1_vec_splat(<8 x i32> %x, <8 x i32> %nbits) {
+; CHECK-LABEL: @t1_vec_splat(
+; CHECK-NEXT:    [[T0:%.*]] = add <8 x i32> [[NBITS:%.*]], <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
+; CHECK-NEXT:    [[T1:%.*]] = shl <8 x i32> <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>, [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = xor <8 x i32> [[T1]], <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
+; CHECK-NEXT:    [[T4:%.*]] = sub <8 x i32> <i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32>, [[NBITS]]
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T0]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T1]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T2]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T4]])
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <8 x i32> [[X:%.*]], [[T4]]
+; CHECK-NEXT:    [[T5:%.*]] = and <8 x i32> [[TMP1]], <i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647>
+; CHECK-NEXT:    ret <8 x i32> [[T5]]
+;
+  %t0 = add <8 x i32> %nbits, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
+  %t1 = shl <8 x i32> <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>, %t0
+  %t2 = xor <8 x i32> %t1, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
+  %t3 = and <8 x i32> %t2, %x
+  %t4 = sub <8 x i32> <i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32>, %nbits
+  call void @use8xi32(<8 x i32> %t0)
+  call void @use8xi32(<8 x i32> %t1)
+  call void @use8xi32(<8 x i32> %t2)
+  call void @use8xi32(<8 x i32> %t4)
+  %t5 = shl <8 x i32> %t3, %t4
+  ret <8 x i32> %t5
+}
+
+define <8 x i32> @t1_vec_splat_undef(<8 x i32> %x, <8 x i32> %nbits) {
+; CHECK-LABEL: @t1_vec_splat_undef(
+; CHECK-NEXT:    [[T0:%.*]] = add <8 x i32> [[NBITS:%.*]], <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 undef, i32 -1>
+; CHECK-NEXT:    [[T1:%.*]] = shl <8 x i32> <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 undef, i32 -1>, [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = xor <8 x i32> [[T1]], <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 undef, i32 -1>
+; CHECK-NEXT:    [[T4:%.*]] = sub <8 x i32> <i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 undef, i32 32>, [[NBITS]]
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T0]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T1]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T2]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T4]])
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <8 x i32> [[X:%.*]], [[T4]]
+; CHECK-NEXT:    [[T5:%.*]] = and <8 x i32> [[TMP1]], <i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 undef, i32 2147483647>
+; CHECK-NEXT:    ret <8 x i32> [[T5]]
+;
+  %t0 = add <8 x i32> %nbits, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 undef, i32 -1>
+  %t1 = shl <8 x i32> <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 undef, i32 -1>, %t0
+  %t2 = xor <8 x i32> %t1, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 undef, i32 -1>
+  %t3 = and <8 x i32> %t2, %x
+  %t4 = sub <8 x i32> <i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 undef, i32 32>, %nbits
+  call void @use8xi32(<8 x i32> %t0)
+  call void @use8xi32(<8 x i32> %t1)
+  call void @use8xi32(<8 x i32> %t2)
+  call void @use8xi32(<8 x i32> %t4)
+  %t5 = shl <8 x i32> %t3, %t4
+  ret <8 x i32> %t5
+}
+
+define <8 x i32> @t2_vec_nonsplat(<8 x i32> %x, <8 x i32> %nbits) {
+; CHECK-LABEL: @t2_vec_nonsplat(
+; CHECK-NEXT:    [[T0:%.*]] = add <8 x i32> [[NBITS:%.*]], <i32 -33, i32 -32, i32 -31, i32 -1, i32 0, i32 1, i32 31, i32 32>
+; CHECK-NEXT:    [[T1:%.*]] = shl <8 x i32> <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>, [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = xor <8 x i32> [[T1]], <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
+; CHECK-NEXT:    [[T4:%.*]] = sub <8 x i32> <i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32>, [[NBITS]]
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T0]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T1]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T2]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T4]])
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <8 x i32> [[X:%.*]], [[T4]]
+; CHECK-NEXT:    [[T5:%.*]] = and <8 x i32> [[TMP1]], <i32 undef, i32 0, i32 1, i32 2147483647, i32 -1, i32 -1, i32 -1, i32 undef>
+; CHECK-NEXT:    ret <8 x i32> [[T5]]
+;
+  %t0 = add <8 x i32> %nbits, <i32 -33, i32 -32, i32 -31, i32 -1, i32 0, i32 1, i32 31, i32 32>
+  %t1 = shl <8 x i32> <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>, %t0
+  %t2 = xor <8 x i32> %t1, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
+  %t3 = and <8 x i32> %t2, %x
+  %t4 = sub <8 x i32> <i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32>, %nbits
+  call void @use8xi32(<8 x i32> %t0)
+  call void @use8xi32(<8 x i32> %t1)
+  call void @use8xi32(<8 x i32> %t2)
+  call void @use8xi32(<8 x i32> %t4)
+  %t5 = shl <8 x i32> %t3, %t4
+  ret <8 x i32> %t5
+}
+
+; Extra uses.
+
+define i32 @n3_extrause(i32 %x, i32 %nbits) {
+; CHECK-LABEL: @n3_extrause(
+; CHECK-NEXT:    [[T0:%.*]] = add i32 [[NBITS:%.*]], -1
+; CHECK-NEXT:    [[T1:%.*]] = shl i32 -1, [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = xor i32 [[T1]], -1
+; CHECK-NEXT:    [[T3:%.*]] = and i32 [[T2]], [[X:%.*]]
+; CHECK-NEXT:    [[T4:%.*]] = sub i32 32, [[NBITS]]
+; CHECK-NEXT:    call void @use32(i32 [[T0]])
+; CHECK-NEXT:    call void @use32(i32 [[T1]])
+; CHECK-NEXT:    call void @use32(i32 [[T2]])
+; CHECK-NEXT:    call void @use32(i32 [[T3]])
+; CHECK-NEXT:    call void @use32(i32 [[T4]])
+; CHECK-NEXT:    [[T5:%.*]] = shl i32 [[T3]], [[T4]]
+; CHECK-NEXT:    ret i32 [[T5]]
+;
+  %t0 = add i32 %nbits, -1
+  %t1 = shl i32 -1, %t0 ; shifting by nbits-1
+  %t2 = xor i32 %t1, -1
+  %t3 = and i32 %t2, %x ; this mask must be one-use.
+  %t4 = sub i32 32, %nbits
+  call void @use32(i32 %t0)
+  call void @use32(i32 %t1)
+  call void @use32(i32 %t2)
+  call void @use32(i32 %t3) ; BAD
+  call void @use32(i32 %t4)
+  %t5 = shl i32 %t3, %t4
+  ret i32 %t5
+}
diff --git a/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-variant-c.ll b/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-variant-c.ll
new file mode 100644
index 000000000..fc964f872
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-variant-c.ll
@@ -0,0 +1,117 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instcombine -S | FileCheck %s
+
+; If we have some pattern that leaves only some low bits set, and then performs
+; left-shift of those bits, we can combine those two shifts into a shift+mask.
+
+; There are many variants to this pattern:
+;   c)  (x & (-1 >> maskNbits)) << shiftNbits
+; simplify to:
+;   (x << shiftNbits) & (-1 >> ((-(maskNbits+shiftNbits))+32))
+
+; Simple tests.
+
+declare void @use32(i32)
+
+define i32 @t0_basic(i32 %x, i32 %nbits) {
+; CHECK-LABEL: @t0_basic(
+; CHECK-NEXT:    [[T0:%.*]] = lshr i32 -1, [[NBITS:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[NBITS]], -1
+; CHECK-NEXT:    call void @use32(i32 [[T0]])
+; CHECK-NEXT:    call void @use32(i32 [[T2]])
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 [[X:%.*]], [[T2]]
+; CHECK-NEXT:    [[T3:%.*]] = and i32 [[TMP1]], 2147483647
+; CHECK-NEXT:    ret i32 [[T3]]
+;
+  %t0 = lshr i32 -1, %nbits
+  %t1 = and i32 %t0, %x
+  %t2 = add i32 %nbits, -1
+  call void @use32(i32 %t0)
+  call void @use32(i32 %t2)
+  %t3 = shl i32 %t1, %t2 ; shift is smaller than mask
+  ret i32 %t3
+}
+
+; Vectors
+
+declare void @use8xi32(<8 x i32>)
+
+define <8 x i32> @t1_vec_splat(<8 x i32> %x, <8 x i32> %nbits) {
+; CHECK-LABEL: @t1_vec_splat(
+; CHECK-NEXT:    [[T0:%.*]] = lshr <8 x i32> <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>, [[NBITS:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = add <8 x i32> [[NBITS]], <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T0]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T2]])
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <8 x i32> [[X:%.*]], [[T2]]
+; CHECK-NEXT:    [[T3:%.*]] = and <8 x i32> [[TMP1]], <i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647>
+; CHECK-NEXT:    ret <8 x i32> [[T3]]
+;
+  %t0 = lshr <8 x i32> <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>, %nbits
+  %t1 = and <8 x i32> %t0, %x
+  %t2 = add <8 x i32> %nbits, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
+  call void @use8xi32(<8 x i32> %t0)
+  call void @use8xi32(<8 x i32> %t2)
+  %t3 = shl <8 x i32> %t1, %t2 ; shift is smaller than mask
+  ret <8 x i32> %t3
+}
+
+define <8 x i32> @t1_vec_splat_undef(<8 x i32> %x, <8 x i32> %nbits) {
+; CHECK-LABEL: @t1_vec_splat_undef(
+; CHECK-NEXT:    [[T0:%.*]] = lshr <8 x i32> <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 undef, i32 -1>, [[NBITS:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = add <8 x i32> [[NBITS]], <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 undef, i32 -1>
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T0]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T2]])
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <8 x i32> [[X:%.*]], [[T2]]
+; CHECK-NEXT:    [[T3:%.*]] = and <8 x i32> [[TMP1]], <i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 undef, i32 2147483647>
+; CHECK-NEXT:    ret <8 x i32> [[T3]]
+;
+  %t0 = lshr <8 x i32> <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 undef, i32 -1>, %nbits
+  %t1 = and <8 x i32> %t0, %x
+  %t2 = add <8 x i32> %nbits, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 undef, i32 -1>
+  call void @use8xi32(<8 x i32> %t0)
+  call void @use8xi32(<8 x i32> %t2)
+  %t3 = shl <8 x i32> %t1, %t2 ; shift is smaller than mask
+  ret <8 x i32> %t3
+}
+
+define <8 x i32> @t1_vec_nonsplat(<8 x i32> %x, <8 x i32> %nbits) {
+; CHECK-LABEL: @t1_vec_nonsplat(
+; CHECK-NEXT:    [[T0:%.*]] = lshr <8 x i32> <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>, [[NBITS:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = add <8 x i32> [[NBITS]], <i32 -32, i32 -31, i32 -1, i32 0, i32 1, i32 31, i32 32, i32 33>
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T0]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T2]])
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <8 x i32> [[X:%.*]], [[T2]]
+; CHECK-NEXT:    [[T3:%.*]] = and <8 x i32> [[TMP1]], <i32 undef, i32 1, i32 2147483647, i32 -1, i32 -1, i32 -1, i32 -1, i32 undef>
+; CHECK-NEXT:    ret <8 x i32> [[T3]]
+;
+  %t0 = lshr <8 x i32> <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>, %nbits
+  %t1 = and <8 x i32> %t0, %x
+  %t2 = add <8 x i32> %nbits, <i32 -32, i32 -31, i32 -1, i32 0, i32 1, i32 31, i32 32, i32 33>
+  call void @use8xi32(<8 x i32> %t0)
+  call void @use8xi32(<8 x i32> %t2)
+  %t3 = shl <8 x i32> %t1, %t2 ; shift is smaller than mask
+  ret <8 x i32> %t3
+}
+
+; Extra uses.
+
+define i32 @n3_extrause(i32 %x, i32 %nbits) {
+; CHECK-LABEL: @n3_extrause(
+; CHECK-NEXT:    [[T0:%.*]] = lshr i32 -1, [[NBITS:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = and i32 [[T0]], [[X:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[NBITS]], -1
+; CHECK-NEXT:    call void @use32(i32 [[T0]])
+; CHECK-NEXT:    call void @use32(i32 [[T1]])
+; CHECK-NEXT:    call void @use32(i32 [[T2]])
+; CHECK-NEXT:    [[T3:%.*]] = shl i32 [[T1]], [[T2]]
+; CHECK-NEXT:    ret i32 [[T3]]
+;
+  %t0 = lshr i32 -1, %nbits
+  %t1 = and i32 %t0, %x
+  %t2 = add i32 %nbits, -1
+  call void @use32(i32 %t0)
+  call void @use32(i32 %t1) ; BAD
+  call void @use32(i32 %t2)
+  %t3 = shl i32 %t1, %t2 ; shift is smaller than mask
+  ret i32 %t3
+}
diff --git a/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-variant-d.ll b/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-variant-d.ll
new file mode 100644
index 000000000..83c0df1d8
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-variant-d.ll
@@ -0,0 +1,137 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instcombine -S | FileCheck %s
+
+; If we have some pattern that leaves only some low bits set, and then performs
+; left-shift of those bits, we can combine those two shifts into a shift+mask.
+
+; There are many variants to this pattern:
+;   d)  (x & ((-1 << maskNbits) >> maskNbits)) << shiftNbits
+; simplify to:
+;   (x << shiftNbits) & (-1 >> ((-(maskNbits+shiftNbits))+32))
+
+; Simple tests.
+
+declare void @use32(i32)
+
+define i32 @t0_basic(i32 %x, i32 %nbits) {
+; CHECK-LABEL: @t0_basic(
+; CHECK-NEXT:    [[T0:%.*]] = shl i32 -1, [[NBITS:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = lshr i32 [[T0]], [[NBITS]]
+; CHECK-NEXT:    [[T3:%.*]] = add i32 [[NBITS]], -1
+; CHECK-NEXT:    call void @use32(i32 [[T0]])
+; CHECK-NEXT:    call void @use32(i32 [[T1]])
+; CHECK-NEXT:    call void @use32(i32 [[T3]])
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 [[X:%.*]], [[T3]]
+; CHECK-NEXT:    [[T4:%.*]] = and i32 [[TMP1]], 2147483647
+; CHECK-NEXT:    ret i32 [[T4]]
+;
+  %t0 = shl i32 -1, %nbits
+  %t1 = lshr i32 %t0, %nbits
+  %t2 = and i32 %t1, %x
+  %t3 = add i32 %nbits, -1
+  call void @use32(i32 %t0)
+  call void @use32(i32 %t1)
+  call void @use32(i32 %t3)
+  %t4 = shl i32 %t2, %t3 ; shift is smaller than mask
+  ret i32 %t4
+}
+
+; Vectors
+
+declare void @use8xi32(<8 x i32>)
+
+define <8 x i32> @t2_vec_splat(<8 x i32> %x, <8 x i32> %nbits) {
+; CHECK-LABEL: @t2_vec_splat(
+; CHECK-NEXT:    [[T0:%.*]] = shl <8 x i32> <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>, [[NBITS:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = lshr <8 x i32> [[T0]], [[NBITS]]
+; CHECK-NEXT:    [[T3:%.*]] = add <8 x i32> [[NBITS]], <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T0]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T1]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T3]])
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <8 x i32> [[X:%.*]], [[T3]]
+; CHECK-NEXT:    [[T4:%.*]] = and <8 x i32> [[TMP1]], <i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647>
+; CHECK-NEXT:    ret <8 x i32> [[T4]]
+;
+  %t0 = shl <8 x i32> <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>, %nbits
+  %t1 = lshr <8 x i32> %t0, %nbits
+  %t2 = and <8 x i32> %t1, %x
+  %t3 = add <8 x i32> %nbits, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
+  call void @use8xi32(<8 x i32> %t0)
+  call void @use8xi32(<8 x i32> %t1)
+  call void @use8xi32(<8 x i32> %t3)
+  %t4 = shl <8 x i32> %t2, %t3 ; shift is smaller than mask
+  ret <8 x i32> %t4
+}
+
+define <8 x i32> @t2_vec_splat_undef(<8 x i32> %x, <8 x i32> %nbits) {
+; CHECK-LABEL: @t2_vec_splat_undef(
+; CHECK-NEXT:    [[T0:%.*]] = shl <8 x i32> <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 undef, i32 -1>, [[NBITS:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = lshr <8 x i32> [[T0]], [[NBITS]]
+; CHECK-NEXT:    [[T3:%.*]] = add <8 x i32> [[NBITS]], <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 undef, i32 -1>
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T0]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T1]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T3]])
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <8 x i32> [[X:%.*]], [[T3]]
+; CHECK-NEXT:    [[T4:%.*]] = and <8 x i32> [[TMP1]], <i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 undef, i32 2147483647>
+; CHECK-NEXT:    ret <8 x i32> [[T4]]
+;
+  %t0 = shl <8 x i32> <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 undef, i32 -1>, %nbits
+  %t1 = lshr <8 x i32> %t0, %nbits
+  %t2 = and <8 x i32> %t1, %x
+  %t3 = add <8 x i32> %nbits, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 undef, i32 -1>
+  call void @use8xi32(<8 x i32> %t0)
+  call void @use8xi32(<8 x i32> %t1)
+  call void @use8xi32(<8 x i32> %t3)
+  %t4 = shl <8 x i32> %t2, %t3 ; shift is smaller than mask
+  ret <8 x i32> %t4
+}
+
+define <8 x i32> @t2_vec_nonsplat(<8 x i32> %x, <8 x i32> %nbits) {
+; CHECK-LABEL: @t2_vec_nonsplat(
+; CHECK-NEXT:    [[T0:%.*]] = shl <8 x i32> <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>, [[NBITS:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = lshr <8 x i32> [[T0]], [[NBITS]]
+; CHECK-NEXT:    [[T3:%.*]] = add <8 x i32> [[NBITS]], <i32 -32, i32 -31, i32 -1, i32 0, i32 1, i32 31, i32 32, i32 33>
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T0]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T1]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T3]])
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <8 x i32> [[X:%.*]], [[T3]]
+; CHECK-NEXT:    [[T4:%.*]] = and <8 x i32> [[TMP1]], <i32 undef, i32 1, i32 2147483647, i32 -1, i32 -1, i32 -1, i32 -1, i32 undef>
+; CHECK-NEXT:    ret <8 x i32> [[T4]]
+;
+  %t0 = shl <8 x i32> <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>, %nbits
+  %t1 = lshr <8 x i32> %t0, %nbits
+  %t2 = and <8 x i32> %t1, %x
+  %t3 = add <8 x i32> %nbits, <i32 -32, i32 -31, i32 -1, i32 0, i32 1, i32 31, i32 32, i32 33>
+  call void @use8xi32(<8 x i32> %t0)
+  call void @use8xi32(<8 x i32> %t1)
+  call void @use8xi32(<8 x i32> %t3)
+  %t4 = shl <8 x i32> %t2, %t3 ; shift is smaller than mask
+  ret <8 x i32> %t4
+}
+
+; Extra uses.
+
+define i32 @n3_extrause(i32 %x, i32 %nbits) {
+; CHECK-LABEL: @n3_extrause(
+; CHECK-NEXT:    [[T0:%.*]] = shl i32 -1, [[NBITS:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = lshr i32 [[T0]], [[NBITS]]
+; CHECK-NEXT:    [[T2:%.*]] = and i32 [[T1]], [[X:%.*]]
+; CHECK-NEXT:    [[T3:%.*]] = add i32 [[NBITS]], -1
+; CHECK-NEXT:    call void @use32(i32 [[T0]])
+; CHECK-NEXT:    call void @use32(i32 [[T1]])
+; CHECK-NEXT:    call void @use32(i32 [[T2]])
+; CHECK-NEXT:    call void @use32(i32 [[T3]])
+; CHECK-NEXT:    [[T4:%.*]] = shl i32 [[T2]], [[T3]]
+; CHECK-NEXT:    ret i32 [[T4]]
+;
+  %t0 = shl i32 -1, %nbits
+  %t1 = lshr i32 %t0, %nbits
+  %t2 = and i32 %t1, %x
+  %t3 = add i32 %nbits, -1
+  call void @use32(i32 %t0)
+  call void @use32(i32 %t1)
+  call void @use32(i32 %t2) ; BAD
+  call void @use32(i32 %t3)
+  %t4 = shl i32 %t2, %t3 ; shift is smaller than mask
+  ret i32 %t4
+}
diff --git a/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-variant-e.ll b/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-variant-e.ll
new file mode 100644
index 000000000..200d414d8
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-variant-e.ll
@@ -0,0 +1,117 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instcombine -S | FileCheck %s
+
+; If we have some pattern that leaves only some low bits set, and then performs
+; left-shift of those bits, we can combine those two shifts into a shift+mask.
+
+; There are many variants to this pattern:
+;   e)  ((x << maskNbits) l>> maskNbits) << shiftNbits
+; simplify to:
+;   (x << shiftNbits) & (-1 >> ((-(maskNbits+shiftNbits))+32))
+
+; Simple tests.
+
+declare void @use32(i32)
+
+define i32 @t0_basic(i32 %x, i32 %nbits) {
+; CHECK-LABEL: @t0_basic(
+; CHECK-NEXT:    [[T0:%.*]] = shl i32 [[X:%.*]], [[NBITS:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[NBITS]], -1
+; CHECK-NEXT:    call void @use32(i32 [[T0]])
+; CHECK-NEXT:    call void @use32(i32 [[T2]])
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 [[X]], [[T2]]
+; CHECK-NEXT:    [[T3:%.*]] = and i32 [[TMP1]], 2147483647
+; CHECK-NEXT:    ret i32 [[T3]]
+;
+  %t0 = shl i32 %x, %nbits
+  %t1 = lshr i32 %t0, %nbits
+  %t2 = add i32 %nbits, -1
+  call void @use32(i32 %t0)
+  call void @use32(i32 %t2)
+  %t3 = shl i32 %t1, %t2 ; shift is smaller than mask
+  ret i32 %t3
+}
+
+; Vectors
+
+declare void @use8xi32(<8 x i32>)
+
+define <8 x i32> @t1_vec_splat(<8 x i32> %x, <8 x i32> %nbits) {
+; CHECK-LABEL: @t1_vec_splat(
+; CHECK-NEXT:    [[T0:%.*]] = shl <8 x i32> [[X:%.*]], [[NBITS:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = add <8 x i32> [[NBITS]], <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T0]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T2]])
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <8 x i32> [[X]], [[T2]]
+; CHECK-NEXT:    [[T3:%.*]] = and <8 x i32> [[TMP1]], <i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647>
+; CHECK-NEXT:    ret <8 x i32> [[T3]]
+;
+  %t0 = shl <8 x i32> %x, %nbits
+  %t1 = lshr <8 x i32> %t0, %nbits
+  %t2 = add <8 x i32> %nbits, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
+  call void @use8xi32(<8 x i32> %t0)
+  call void @use8xi32(<8 x i32> %t2)
+  %t3 = shl <8 x i32> %t1, %t2 ; shift is smaller than mask
+  ret <8 x i32> %t3
+}
+
+define <8 x i32> @t1_vec_splat_undef(<8 x i32> %x, <8 x i32> %nbits) {
+; CHECK-LABEL: @t1_vec_splat_undef(
+; CHECK-NEXT:    [[T0:%.*]] = shl <8 x i32> [[X:%.*]], [[NBITS:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = add <8 x i32> [[NBITS]], <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 undef, i32 -1>
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T0]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T2]])
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <8 x i32> [[X]], [[T2]]
+; CHECK-NEXT:    [[T3:%.*]] = and <8 x i32> [[TMP1]], <i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 undef, i32 2147483647>
+; CHECK-NEXT:    ret <8 x i32> [[T3]]
+;
+  %t0 = shl <8 x i32> %x, %nbits
+  %t1 = lshr <8 x i32> %t0, %nbits
+  %t2 = add <8 x i32> %nbits, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 undef, i32 -1>
+  call void @use8xi32(<8 x i32> %t0)
+  call void @use8xi32(<8 x i32> %t2)
+  %t3 = shl <8 x i32> %t1, %t2 ; shift is smaller than mask
+  ret <8 x i32> %t3
+}
+
+define <8 x i32> @t1_vec_nonsplat(<8 x i32> %x, <8 x i32> %nbits) {
+; CHECK-LABEL: @t1_vec_nonsplat(
+; CHECK-NEXT:    [[T0:%.*]] = shl <8 x i32> [[X:%.*]], [[NBITS:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = add <8 x i32> [[NBITS]], <i32 -32, i32 -31, i32 -1, i32 0, i32 1, i32 31, i32 32, i32 33>
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T0]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T2]])
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <8 x i32> [[X]], [[T2]]
+; CHECK-NEXT:    [[T3:%.*]] = and <8 x i32> [[TMP1]], <i32 undef, i32 1, i32 2147483647, i32 -1, i32 -1, i32 -1, i32 -1, i32 undef>
+; CHECK-NEXT:    ret <8 x i32> [[T3]]
+;
+  %t0 = shl <8 x i32> %x, %nbits
+  %t1 = lshr <8 x i32> %t0, %nbits
+  %t2 = add <8 x i32> %nbits, <i32 -32, i32 -31, i32 -1, i32 0, i32 1, i32 31, i32 32, i32 33>
+  call void @use8xi32(<8 x i32> %t0)
+  call void @use8xi32(<8 x i32> %t2)
+  %t3 = shl <8 x i32> %t1, %t2 ; shift is smaller than mask
+  ret <8 x i32> %t3
+}
+
+; Extra uses.
+
+define i32 @n3_extrause(i32 %x, i32 %nbits) {
+; CHECK-LABEL: @n3_extrause(
+; CHECK-NEXT:    [[T0:%.*]] = shl i32 [[X:%.*]], [[NBITS:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = lshr i32 [[T0]], [[NBITS]]
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[NBITS]], -1
+; CHECK-NEXT:    call void @use32(i32 [[T0]])
+; CHECK-NEXT:    call void @use32(i32 [[T1]])
+; CHECK-NEXT:    call void @use32(i32 [[T2]])
+; CHECK-NEXT:    [[T3:%.*]] = shl i32 [[T1]], [[T2]]
+; CHECK-NEXT:    ret i32 [[T3]]
+;
+  %t0 = shl i32 %x, %nbits
+  %t1 = lshr i32 %t0, %nbits
+  %t2 = add i32 %nbits, -1
+  call void @use32(i32 %t0)
+  call void @use32(i32 %t1) ; BAD
+  call void @use32(i32 %t2)
+  %t3 = shl i32 %t1, %t2 ; shift is smaller than mask
+  ret i32 %t3
+}
diff --git a/llvm/test/Transforms/InstCombine/phi-equal-incoming-pointers.ll b/llvm/test/Transforms/InstCombine/phi-equal-incoming-pointers.ll
new file mode 100644
index 000000000..db5402bd7
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/phi-equal-incoming-pointers.ll
@@ -0,0 +1,616 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -passes=instcombine,verify -S < %s | FileCheck %s --check-prefixes=ALL,INSTCOMBINE
+
+; Make sure GVN won't undo the transformation:
+; RUN: opt -passes=instcombine,gvn -S < %s | FileCheck %s --check-prefixes=ALL,INSTCOMBINEGVN
+
+declare i8* @get_ptr.i8()
+declare i32* @get_ptr.i32()
+declare void @foo.i8(i8*)
+declare void @foo.i32(i32*)
+
+define i32 @test_gep_and_bitcast(i1 %cond, i1 %cond2) {
+; ALL-LABEL: @test_gep_and_bitcast(
+; ALL-NEXT:  entry:
+; ALL-NEXT:    [[OBJ:%.*]] = call i8* @get_ptr.i8()
+; ALL-NEXT:    br i1 [[COND:%.*]], label [[BB1:%.*]], label [[BB2:%.*]]
+; ALL:       bb1:
+; ALL-NEXT:    [[PTR1:%.*]] = getelementptr inbounds i8, i8* [[OBJ]], i64 16
+; ALL-NEXT:    [[PTR1_TYPED:%.*]] = bitcast i8* [[PTR1]] to i32*
+; ALL-NEXT:    br label [[EXIT:%.*]]
+; ALL:       bb2:
+; ALL-NEXT:    [[PTR2:%.*]] = getelementptr inbounds i8, i8* [[OBJ]], i64 16
+; ALL-NEXT:    [[PTR2_TYPED:%.*]] = bitcast i8* [[PTR2]] to i32*
+; ALL-NEXT:    br label [[EXIT]]
+; ALL:       exit:
+; ALL-NEXT:    [[PTR_TYPED:%.*]] = phi i32* [ [[PTR1_TYPED]], [[BB1]] ], [ [[PTR2_TYPED]], [[BB2]] ]
+; ALL-NEXT:    [[RES_PHI_IN:%.*]] = phi i32* [ [[PTR1_TYPED]], [[BB1]] ], [ [[PTR2_TYPED]], [[BB2]] ]
+; ALL-NEXT:    [[RES_PHI:%.*]] = load i32, i32* [[RES_PHI_IN]], align 4
+; ALL-NEXT:    store i32 1, i32* [[PTR_TYPED]], align 4
+; ALL-NEXT:    [[RES:%.*]] = select i1 [[COND2:%.*]], i32 [[RES_PHI]], i32 1
+; ALL-NEXT:    ret i32 [[RES]]
+;
+entry:
+  %obj = call i8* @get_ptr.i8()
+  br i1 %cond, label %bb1, label %bb2
+
+bb1:
+  %ptr1 = getelementptr inbounds i8, i8* %obj, i64 16
+  %ptr1.typed = bitcast i8* %ptr1 to i32*
+  %res1 = load i32, i32* %ptr1.typed
+  br label %exit
+
+bb2:
+  %ptr2 = getelementptr inbounds i8, i8* %obj, i64 16
+  %ptr2.typed = bitcast i8* %ptr2 to i32*
+  %res2 = load i32, i32* %ptr2.typed
+  br label %exit
+
+exit:
+  %ptr.typed = phi i32* [ %ptr1.typed, %bb1 ], [ %ptr2.typed, %bb2 ]
+  %res.phi = phi i32 [ %res1, %bb1 ], [ %res2, %bb2 ]
+  store i32 1, i32* %ptr.typed
+  %res.load = load i32, i32* %ptr.typed
+  %res = select i1 %cond2, i32 %res.phi, i32 %res.load
+  ret i32 %res
+}
+
+define i32 @test_gep_and_bitcast_arg(i8* %obj, i1 %cond, i1 %cond2) {
+; ALL-LABEL: @test_gep_and_bitcast_arg(
+; ALL-NEXT:  entry:
+; ALL-NEXT:    br i1 [[COND:%.*]], label [[BB1:%.*]], label [[BB2:%.*]]
+; ALL:       bb1:
+; ALL-NEXT:    [[PTR1:%.*]] = getelementptr inbounds i8, i8* [[OBJ:%.*]], i64 16
+; ALL-NEXT:    [[PTR1_TYPED:%.*]] = bitcast i8* [[PTR1]] to i32*
+; ALL-NEXT:    br label [[EXIT:%.*]]
+; ALL:       bb2:
+; ALL-NEXT:    [[PTR2:%.*]] = getelementptr inbounds i8, i8* [[OBJ]], i64 16
+; ALL-NEXT:    [[PTR2_TYPED:%.*]] = bitcast i8* [[PTR2]] to i32*
+; ALL-NEXT:    br label [[EXIT]]
+; ALL:       exit:
+; ALL-NEXT:    [[PTR_TYPED:%.*]] = phi i32* [ [[PTR1_TYPED]], [[BB1]] ], [ [[PTR2_TYPED]], [[BB2]] ]
+; ALL-NEXT:    [[RES_PHI_IN:%.*]] = phi i32* [ [[PTR1_TYPED]], [[BB1]] ], [ [[PTR2_TYPED]], [[BB2]] ]
+; ALL-NEXT:    [[RES_PHI:%.*]] = load i32, i32* [[RES_PHI_IN]], align 4
+; ALL-NEXT:    store i32 1, i32* [[PTR_TYPED]], align 4
+; ALL-NEXT:    [[RES:%.*]] = select i1 [[COND2:%.*]], i32 [[RES_PHI]], i32 1
+; ALL-NEXT:    ret i32 [[RES]]
+;
+entry:
+  br i1 %cond, label %bb1, label %bb2
+
+bb1:
+  %ptr1 = getelementptr inbounds i8, i8* %obj, i64 16
+  %ptr1.typed = bitcast i8* %ptr1 to i32*
+  %res1 = load i32, i32* %ptr1.typed
+  br label %exit
+
+bb2:
+  %ptr2 = getelementptr inbounds i8, i8* %obj, i64 16
+  %ptr2.typed = bitcast i8* %ptr2 to i32*
+  %res2 = load i32, i32* %ptr2.typed
+  br label %exit
+
+exit:
+  %ptr.typed = phi i32* [ %ptr1.typed, %bb1 ], [ %ptr2.typed, %bb2 ]
+  %res.phi = phi i32 [ %res1, %bb1 ], [ %res2, %bb2 ]
+  store i32 1, i32* %ptr.typed
+  %res.load = load i32, i32* %ptr.typed
+  %res = select i1 %cond2, i32 %res.phi, i32 %res.load
+  ret i32 %res
+}
+
+define i32 @test_gep_and_bitcast_phi(i1 %cond, i1 %cond2, i1 %cond3) {
+; ALL-LABEL: @test_gep_and_bitcast_phi(
+; ALL-NEXT:  entry:
+; ALL-NEXT:    br i1 [[COND:%.*]], label [[BB1:%.*]], label [[BB2:%.*]]
+; ALL:       bb1:
+; ALL-NEXT:    [[OBJ1:%.*]] = call i8* @get_ptr.i8()
+; ALL-NEXT:    br label [[MERGE:%.*]]
+; ALL:       bb2:
+; ALL-NEXT:    [[OBJ2_TYPED:%.*]] = call i32* @get_ptr.i32()
+; ALL-NEXT:    [[OBJ2:%.*]] = bitcast i32* [[OBJ2_TYPED]] to i8*
+; ALL-NEXT:    br label [[MERGE]]
+; ALL:       merge:
+; ALL-NEXT:    [[OBJ:%.*]] = phi i8* [ [[OBJ1]], [[BB1]] ], [ [[OBJ2]], [[BB2]] ]
+; ALL-NEXT:    [[ANOTHER_PHI:%.*]] = phi i8* [ [[OBJ1]], [[BB1]] ], [ null, [[BB2]] ]
+; ALL-NEXT:    call void @foo.i8(i8* [[ANOTHER_PHI]])
+; ALL-NEXT:    br i1 [[COND2:%.*]], label [[BB3:%.*]], label [[BB4:%.*]]
+; ALL:       bb3:
+; ALL-NEXT:    [[PTR1:%.*]] = getelementptr inbounds i8, i8* [[OBJ]], i64 16
+; ALL-NEXT:    [[PTR1_TYPED:%.*]] = bitcast i8* [[PTR1]] to i32*
+; ALL-NEXT:    br label [[EXIT:%.*]]
+; ALL:       bb4:
+; ALL-NEXT:    [[PTR2:%.*]] = getelementptr inbounds i8, i8* [[OBJ]], i64 16
+; ALL-NEXT:    [[PTR2_TYPED:%.*]] = bitcast i8* [[PTR2]] to i32*
+; ALL-NEXT:    br label [[EXIT]]
+; ALL:       exit:
+; ALL-NEXT:    [[PTR_TYPED:%.*]] = phi i32* [ [[PTR1_TYPED]], [[BB3]] ], [ [[PTR2_TYPED]], [[BB4]] ]
+; ALL-NEXT:    [[RES_PHI_IN:%.*]] = phi i32* [ [[PTR1_TYPED]], [[BB3]] ], [ [[PTR2_TYPED]], [[BB4]] ]
+; ALL-NEXT:    [[RES_PHI:%.*]] = load i32, i32* [[RES_PHI_IN]], align 4
+; ALL-NEXT:    store i32 1, i32* [[PTR_TYPED]], align 4
+; ALL-NEXT:    [[RES:%.*]] = select i1 [[COND3:%.*]], i32 [[RES_PHI]], i32 1
+; ALL-NEXT:    ret i32 [[RES]]
+;
+entry:
+  br i1 %cond, label %bb1, label %bb2
+
+bb1:
+  %obj1 = call i8* @get_ptr.i8()
+  br label %merge
+
+bb2:
+  %obj2.typed = call i32* @get_ptr.i32()
+  %obj2 = bitcast i32* %obj2.typed to i8*
+  br label %merge
+
+merge:
+  %obj = phi i8* [ %obj1, %bb1 ], [ %obj2, %bb2 ]
+  %another_phi = phi i8* [ %obj1, %bb1 ], [ null, %bb2 ]
+  call void @foo.i8(i8* %another_phi)
+  br i1 %cond2, label %bb3, label %bb4
+
+bb3:
+  %ptr1 = getelementptr inbounds i8, i8* %obj, i64 16
+  %ptr1.typed = bitcast i8* %ptr1 to i32*
+  %res1 = load i32, i32* %ptr1.typed
+  br label %exit
+
+bb4:
+  %ptr2 = getelementptr inbounds i8, i8* %obj, i64 16
+  %ptr2.typed = bitcast i8* %ptr2 to i32*
+  %res2 = load i32, i32* %ptr2.typed
+  br label %exit
+
+exit:
+  %ptr.typed = phi i32* [ %ptr1.typed, %bb3 ], [ %ptr2.typed, %bb4 ]
+  %res.phi = phi i32 [ %res1, %bb3 ], [ %res2, %bb4 ]
+  store i32 1, i32* %ptr.typed
+  %res.load = load i32, i32* %ptr.typed
+  %res = select i1 %cond3, i32 %res.phi, i32 %res.load
+  ret i32 %res
+}
+
+define i32 @test_gep_i32ptr(i1 %cond, i1 %cond2) {
+; ALL-LABEL: @test_gep_i32ptr(
+; ALL-NEXT:  entry:
+; ALL-NEXT:    [[OBJ:%.*]] = call i32* @get_ptr.i32()
+; ALL-NEXT:    br i1 [[COND:%.*]], label [[BB1:%.*]], label [[BB2:%.*]]
+; ALL:       bb1:
+; ALL-NEXT:    [[PTR1_TYPED:%.*]] = getelementptr inbounds i32, i32* [[OBJ]], i64 16
+; ALL-NEXT:    br label [[EXIT:%.*]]
+; ALL:       bb2:
+; ALL-NEXT:    [[PTR2_TYPED:%.*]] = getelementptr inbounds i32, i32* [[OBJ]], i64 16
+; ALL-NEXT:    br label [[EXIT]]
+; ALL:       exit:
+; ALL-NEXT:    [[PTR_TYPED:%.*]] = phi i32* [ [[PTR1_TYPED]], [[BB1]] ], [ [[PTR2_TYPED]], [[BB2]] ]
+; ALL-NEXT:    [[RES_PHI_IN:%.*]] = phi i32* [ [[PTR1_TYPED]], [[BB1]] ], [ [[PTR2_TYPED]], [[BB2]] ]
+; ALL-NEXT:    [[RES_PHI:%.*]] = load i32, i32* [[RES_PHI_IN]], align 4
+; ALL-NEXT:    store i32 1, i32* [[PTR_TYPED]], align 4
+; ALL-NEXT:    [[RES:%.*]] = select i1 [[COND2:%.*]], i32 [[RES_PHI]], i32 1
+; ALL-NEXT:    ret i32 [[RES]]
+;
+entry:
+  %obj = call i32* @get_ptr.i32()
+  br i1 %cond, label %bb1, label %bb2
+
+bb1:
+  %ptr1.typed = getelementptr inbounds i32, i32* %obj, i64 16
+  %res1 = load i32, i32* %ptr1.typed
+  br label %exit
+
+bb2:
+  %ptr2.typed = getelementptr inbounds i32, i32* %obj, i64 16
+  %res2 = load i32, i32* %ptr2.typed
+  br label %exit
+
+exit:
+  %ptr.typed = phi i32* [ %ptr1.typed, %bb1 ], [ %ptr2.typed, %bb2 ]
+  %res.phi = phi i32 [ %res1, %bb1 ], [ %res2, %bb2 ]
+  store i32 1, i32* %ptr.typed
+  %res.load = load i32, i32* %ptr.typed
+  %res = select i1 %cond2, i32 %res.phi, i32 %res.load
+  ret i32 %res
+}
+
+define i32 @test_gep_and_bitcast_gep_base_ptr(i1 %cond, i1 %cond2) {
+; ALL-LABEL: @test_gep_and_bitcast_gep_base_ptr(
+; ALL-NEXT:  entry:
+; ALL-NEXT:    [[OBJ0:%.*]] = call i8* @get_ptr.i8()
+; ALL-NEXT:    br i1 [[COND:%.*]], label [[BB1:%.*]], label [[BB2:%.*]]
+; ALL:       bb1:
+; ALL-NEXT:    [[PTR1:%.*]] = getelementptr inbounds i8, i8* [[OBJ0]], i64 32
+; ALL-NEXT:    [[PTR1_TYPED:%.*]] = bitcast i8* [[PTR1]] to i32*
+; ALL-NEXT:    br label [[EXIT:%.*]]
+; ALL:       bb2:
+; ALL-NEXT:    [[PTR2:%.*]] = getelementptr inbounds i8, i8* [[OBJ0]], i64 32
+; ALL-NEXT:    [[PTR2_TYPED:%.*]] = bitcast i8* [[PTR2]] to i32*
+; ALL-NEXT:    br label [[EXIT]]
+; ALL:       exit:
+; ALL-NEXT:    [[PTR_TYPED:%.*]] = phi i32* [ [[PTR1_TYPED]], [[BB1]] ], [ [[PTR2_TYPED]], [[BB2]] ]
+; ALL-NEXT:    [[RES_PHI_IN:%.*]] = phi i32* [ [[PTR1_TYPED]], [[BB1]] ], [ [[PTR2_TYPED]], [[BB2]] ]
+; ALL-NEXT:    [[RES_PHI:%.*]] = load i32, i32* [[RES_PHI_IN]], align 4
+; ALL-NEXT:    store i32 1, i32* [[PTR_TYPED]], align 4
+; ALL-NEXT:    [[RES:%.*]] = select i1 [[COND2:%.*]], i32 [[RES_PHI]], i32 1
+; ALL-NEXT:    ret i32 [[RES]]
+;
+entry:
+  %obj0 = call i8* @get_ptr.i8()
+  %obj = getelementptr inbounds i8, i8* %obj0, i64 16
+  br i1 %cond, label %bb1, label %bb2
+
+bb1:
+  %ptr1 = getelementptr inbounds i8, i8* %obj, i64 16
+  %ptr1.typed = bitcast i8* %ptr1 to i32*
+  %res1 = load i32, i32* %ptr1.typed
+  br label %exit
+
+bb2:
+  %ptr2 = getelementptr inbounds i8, i8* %obj, i64 16
+  %ptr2.typed = bitcast i8* %ptr2 to i32*
+  %res2 = load i32, i32* %ptr2.typed
+  br label %exit
+
+exit:
+  %ptr.typed = phi i32* [ %ptr1.typed, %bb1 ], [ %ptr2.typed, %bb2 ]
+  %res.phi = phi i32 [ %res1, %bb1 ], [ %res2, %bb2 ]
+  store i32 1, i32* %ptr.typed
+  %res.load = load i32, i32* %ptr.typed
+  %res = select i1 %cond2, i32 %res.phi, i32 %res.load
+  ret i32 %res
+}
+
+define i32 @test_gep_and_bitcast_same_bb(i1 %cond, i1 %cond2) {
+; INSTCOMBINE-LABEL: @test_gep_and_bitcast_same_bb(
+; INSTCOMBINE-NEXT:  entry:
+; INSTCOMBINE-NEXT:    [[OBJ:%.*]] = call i8* @get_ptr.i8()
+; INSTCOMBINE-NEXT:    [[PTR1:%.*]] = getelementptr inbounds i8, i8* [[OBJ]], i64 16
+; INSTCOMBINE-NEXT:    [[PTR1_TYPED:%.*]] = bitcast i8* [[PTR1]] to i32*
+; INSTCOMBINE-NEXT:    br i1 [[COND:%.*]], label [[EXIT:%.*]], label [[BB2:%.*]]
+; INSTCOMBINE:       bb2:
+; INSTCOMBINE-NEXT:    [[PTR2:%.*]] = getelementptr inbounds i8, i8* [[OBJ]], i64 16
+; INSTCOMBINE-NEXT:    [[PTR2_TYPED:%.*]] = bitcast i8* [[PTR2]] to i32*
+; INSTCOMBINE-NEXT:    br label [[EXIT]]
+; INSTCOMBINE:       exit:
+; INSTCOMBINE-NEXT:    [[PTR_TYPED:%.*]] = phi i32* [ [[PTR1_TYPED]], [[ENTRY:%.*]] ], [ [[PTR2_TYPED]], [[BB2]] ]
+; INSTCOMBINE-NEXT:    [[RES_PHI_IN:%.*]] = phi i32* [ [[PTR1_TYPED]], [[ENTRY]] ], [ [[PTR2_TYPED]], [[BB2]] ]
+; INSTCOMBINE-NEXT:    [[RES_PHI:%.*]] = load i32, i32* [[RES_PHI_IN]], align 4
+; INSTCOMBINE-NEXT:    store i32 1, i32* [[PTR_TYPED]], align 4
+; INSTCOMBINE-NEXT:    [[RES:%.*]] = select i1 [[COND2:%.*]], i32 [[RES_PHI]], i32 1
+; INSTCOMBINE-NEXT:    ret i32 [[RES]]
+;
+; INSTCOMBINEGVN-LABEL: @test_gep_and_bitcast_same_bb(
+; INSTCOMBINEGVN-NEXT:  entry:
+; INSTCOMBINEGVN-NEXT:    [[OBJ:%.*]] = call i8* @get_ptr.i8()
+; INSTCOMBINEGVN-NEXT:    [[PTR1:%.*]] = getelementptr inbounds i8, i8* [[OBJ]], i64 16
+; INSTCOMBINEGVN-NEXT:    [[PTR1_TYPED:%.*]] = bitcast i8* [[PTR1]] to i32*
+; INSTCOMBINEGVN-NEXT:    br i1 [[COND:%.*]], label [[EXIT:%.*]], label [[BB2:%.*]]
+; INSTCOMBINEGVN:       bb2:
+; INSTCOMBINEGVN-NEXT:    br label [[EXIT]]
+; INSTCOMBINEGVN:       exit:
+; INSTCOMBINEGVN-NEXT:    [[RES_PHI:%.*]] = load i32, i32* [[PTR1_TYPED]], align 4
+; INSTCOMBINEGVN-NEXT:    store i32 1, i32* [[PTR1_TYPED]], align 4
+; INSTCOMBINEGVN-NEXT:    [[RES:%.*]] = select i1 [[COND2:%.*]], i32 [[RES_PHI]], i32 1
+; INSTCOMBINEGVN-NEXT:    ret i32 [[RES]]
+;
+entry:
+  %obj = call i8* @get_ptr.i8()
+  %ptr1 = getelementptr inbounds i8, i8* %obj, i64 16
+  %ptr1.typed = bitcast i8* %ptr1 to i32*
+  %res1 = load i32, i32* %ptr1.typed
+  br i1 %cond, label %exit, label %bb2
+
+bb2:
+  %ptr2 = getelementptr inbounds i8, i8* %obj, i64 16
+  %ptr2.typed = bitcast i8* %ptr2 to i32*
+  %res2 = load i32, i32* %ptr2.typed
+  br label %exit
+
+exit:
+  %ptr.typed = phi i32* [ %ptr1.typed, %entry ], [ %ptr2.typed, %bb2 ]
+  %res.phi = phi i32 [ %res1, %entry ], [ %res2, %bb2 ]
+  store i32 1, i32* %ptr.typed
+  %res.load = load i32, i32* %ptr.typed
+  %res = select i1 %cond2, i32 %res.phi, i32 %res.load
+  ret i32 %res
+}
+
+define i32 @test_gep_and_bitcast_same_bb_and_extra_use(i1 %cond, i1 %cond2) {
+; INSTCOMBINE-LABEL: @test_gep_and_bitcast_same_bb_and_extra_use(
+; INSTCOMBINE-NEXT:  entry:
+; INSTCOMBINE-NEXT:    [[OBJ:%.*]] = call i8* @get_ptr.i8()
+; INSTCOMBINE-NEXT:    [[PTR1:%.*]] = getelementptr inbounds i8, i8* [[OBJ]], i64 16
+; INSTCOMBINE-NEXT:    [[PTR1_TYPED:%.*]] = bitcast i8* [[PTR1]] to i32*
+; INSTCOMBINE-NEXT:    call void @foo.i32(i32* nonnull [[PTR1_TYPED]])
+; INSTCOMBINE-NEXT:    br i1 [[COND:%.*]], label [[EXIT:%.*]], label [[BB2:%.*]]
+; INSTCOMBINE:       bb2:
+; INSTCOMBINE-NEXT:    [[PTR2:%.*]] = getelementptr inbounds i8, i8* [[OBJ]], i64 16
+; INSTCOMBINE-NEXT:    [[PTR2_TYPED:%.*]] = bitcast i8* [[PTR2]] to i32*
+; INSTCOMBINE-NEXT:    br label [[EXIT]]
+; INSTCOMBINE:       exit:
+; INSTCOMBINE-NEXT:    [[PTR_TYPED:%.*]] = phi i32* [ [[PTR1_TYPED]], [[ENTRY:%.*]] ], [ [[PTR2_TYPED]], [[BB2]] ]
+; INSTCOMBINE-NEXT:    [[RES_PHI_IN:%.*]] = phi i32* [ [[PTR1_TYPED]], [[ENTRY]] ], [ [[PTR2_TYPED]], [[BB2]] ]
+; INSTCOMBINE-NEXT:    [[RES_PHI:%.*]] = load i32, i32* [[RES_PHI_IN]], align 4
+; INSTCOMBINE-NEXT:    store i32 1, i32* [[PTR_TYPED]], align 4
+; INSTCOMBINE-NEXT:    [[RES:%.*]] = select i1 [[COND2:%.*]], i32 [[RES_PHI]], i32 1
+; INSTCOMBINE-NEXT:    ret i32 [[RES]]
+;
+; INSTCOMBINEGVN-LABEL: @test_gep_and_bitcast_same_bb_and_extra_use(
+; INSTCOMBINEGVN-NEXT:  entry:
+; INSTCOMBINEGVN-NEXT:    [[OBJ:%.*]] = call i8* @get_ptr.i8()
+; INSTCOMBINEGVN-NEXT:    [[PTR1:%.*]] = getelementptr inbounds i8, i8* [[OBJ]], i64 16
+; INSTCOMBINEGVN-NEXT:    [[PTR1_TYPED:%.*]] = bitcast i8* [[PTR1]] to i32*
+; INSTCOMBINEGVN-NEXT:    call void @foo.i32(i32* nonnull [[PTR1_TYPED]])
+; INSTCOMBINEGVN-NEXT:    br i1 [[COND:%.*]], label [[EXIT:%.*]], label [[BB2:%.*]]
+; INSTCOMBINEGVN:       bb2:
+; INSTCOMBINEGVN-NEXT:    br label [[EXIT]]
+; INSTCOMBINEGVN:       exit:
+; INSTCOMBINEGVN-NEXT:    [[RES_PHI:%.*]] = load i32, i32* [[PTR1_TYPED]], align 4
+; INSTCOMBINEGVN-NEXT:    store i32 1, i32* [[PTR1_TYPED]], align 4
+; INSTCOMBINEGVN-NEXT:    [[RES:%.*]] = select i1 [[COND2:%.*]], i32 [[RES_PHI]], i32 1
+; INSTCOMBINEGVN-NEXT:    ret i32 [[RES]]
+;
+entry:
+  %obj = call i8* @get_ptr.i8()
+  %ptr1 = getelementptr inbounds i8, i8* %obj, i64 16
+  %ptr1.typed = bitcast i8* %ptr1 to i32*
+  call void @foo.i32(i32* %ptr1.typed)
+  %res1 = load i32, i32* %ptr1.typed
+  br i1 %cond, label %exit, label %bb2
+
+bb2:
+  %ptr2 = getelementptr inbounds i8, i8* %obj, i64 16
+  %ptr2.typed = bitcast i8* %ptr2 to i32*
+  %res2 = load i32, i32* %ptr2.typed
+  br label %exit
+
+exit:
+  %ptr.typed = phi i32* [ %ptr1.typed, %entry ], [ %ptr2.typed, %bb2 ]
+  %res.phi = phi i32 [ %res1, %entry ], [ %res2, %bb2 ]
+  store i32 1, i32* %ptr.typed
+  %res.load = load i32, i32* %ptr.typed
+  %res = select i1 %cond2, i32 %res.phi, i32 %res.load
+  ret i32 %res
+}
+
+define i8 @test_gep(i1 %cond, i1 %cond2) {
+; ALL-LABEL: @test_gep(
+; ALL-NEXT:  entry:
+; ALL-NEXT:    [[OBJ:%.*]] = call i8* @get_ptr.i8()
+; ALL-NEXT:    br i1 [[COND:%.*]], label [[BB1:%.*]], label [[BB2:%.*]]
+; ALL:       bb1:
+; ALL-NEXT:    [[PTR1:%.*]] = getelementptr inbounds i8, i8* [[OBJ]], i64 16
+; ALL-NEXT:    br label [[EXIT:%.*]]
+; ALL:       bb2:
+; ALL-NEXT:    [[PTR2:%.*]] = getelementptr inbounds i8, i8* [[OBJ]], i64 16
+; ALL-NEXT:    br label [[EXIT]]
+; ALL:       exit:
+; ALL-NEXT:    [[PTR_TYPED:%.*]] = phi i8* [ [[PTR1]], [[BB1]] ], [ [[PTR2]], [[BB2]] ]
+; ALL-NEXT:    [[RES_PHI_IN:%.*]] = phi i8* [ [[PTR1]], [[BB1]] ], [ [[PTR2]], [[BB2]] ]
+; ALL-NEXT:    [[RES_PHI:%.*]] = load i8, i8* [[RES_PHI_IN]], align 1
+; ALL-NEXT:    store i8 1, i8* [[PTR_TYPED]], align 1
+; ALL-NEXT:    [[RES:%.*]] = select i1 [[COND2:%.*]], i8 [[RES_PHI]], i8 1
+; ALL-NEXT:    ret i8 [[RES]]
+;
+entry:
+  %obj = call i8* @get_ptr.i8()
+  br i1 %cond, label %bb1, label %bb2
+
+bb1:
+  %ptr1 = getelementptr inbounds i8, i8* %obj, i64 16
+  %res1 = load i8, i8* %ptr1
+  br label %exit
+
+bb2:
+  %ptr2 = getelementptr inbounds i8, i8* %obj, i64 16
+  %res2 = load i8, i8* %ptr2
+  br label %exit
+
+exit:
+  %ptr.typed = phi i8* [ %ptr1, %bb1 ], [ %ptr2, %bb2 ]
+  %res.phi = phi i8 [ %res1, %bb1 ], [ %res2, %bb2 ]
+  store i8 1, i8* %ptr.typed
+  %res.load = load i8, i8* %ptr.typed
+  %res = select i1 %cond2, i8 %res.phi, i8 %res.load
+  ret i8 %res
+}
+
+define i32 @test_extra_uses(i1 %cond, i1 %cond2) {
+; ALL-LABEL: @test_extra_uses(
+; ALL-NEXT:  entry:
+; ALL-NEXT:    [[OBJ:%.*]] = call i8* @get_ptr.i8()
+; ALL-NEXT:    br i1 [[COND:%.*]], label [[BB1:%.*]], label [[BB2:%.*]]
+; ALL:       bb1:
+; ALL-NEXT:    [[PTR1:%.*]] = getelementptr inbounds i8, i8* [[OBJ]], i64 16
+; ALL-NEXT:    [[PTR1_TYPED:%.*]] = bitcast i8* [[PTR1]] to i32*
+; ALL-NEXT:    [[RES1:%.*]] = load i32, i32* [[PTR1_TYPED]], align 4
+; ALL-NEXT:    call void @foo.i32(i32* nonnull [[PTR1_TYPED]])
+; ALL-NEXT:    br label [[EXIT:%.*]]
+; ALL:       bb2:
+; ALL-NEXT:    [[PTR2:%.*]] = getelementptr inbounds i8, i8* [[OBJ]], i64 16
+; ALL-NEXT:    [[PTR2_TYPED:%.*]] = bitcast i8* [[PTR2]] to i32*
+; ALL-NEXT:    [[RES2:%.*]] = load i32, i32* [[PTR2_TYPED]], align 4
+; ALL-NEXT:    call void @foo.i32(i32* nonnull [[PTR2_TYPED]])
+; ALL-NEXT:    br label [[EXIT]]
+; ALL:       exit:
+; ALL-NEXT:    [[PTR_TYPED:%.*]] = phi i32* [ [[PTR1_TYPED]], [[BB1]] ], [ [[PTR2_TYPED]], [[BB2]] ]
+; ALL-NEXT:    [[RES_PHI:%.*]] = phi i32 [ [[RES1]], [[BB1]] ], [ [[RES2]], [[BB2]] ]
+; ALL-NEXT:    store i32 1, i32* [[PTR_TYPED]], align 4
+; ALL-NEXT:    [[RES:%.*]] = select i1 [[COND2:%.*]], i32 [[RES_PHI]], i32 1
+; ALL-NEXT:    ret i32 [[RES]]
+;
+entry:
+  %obj = call i8* @get_ptr.i8()
+  br i1 %cond, label %bb1, label %bb2
+
+bb1:
+  %ptr1 = getelementptr inbounds i8, i8* %obj, i64 16
+  %ptr1.typed = bitcast i8* %ptr1 to i32*
+  %res1 = load i32, i32* %ptr1.typed
+  call void @foo.i32(i32* %ptr1.typed)
+  br label %exit
+
+bb2:
+  %ptr2 = getelementptr inbounds i8, i8* %obj, i64 16
+  %ptr2.typed = bitcast i8* %ptr2 to i32*
+  %res2 = load i32, i32* %ptr2.typed
+  call void @foo.i32(i32* %ptr2.typed)
+  br label %exit
+
+exit:
+  %ptr.typed = phi i32* [ %ptr1.typed, %bb1 ], [ %ptr2.typed, %bb2 ]
+  %res.phi = phi i32 [ %res1, %bb1 ], [ %res2, %bb2 ]
+  store i32 1, i32* %ptr.typed
+  %res.load = load i32, i32* %ptr.typed
+  %res = select i1 %cond2, i32 %res.phi, i32 %res.load
+  ret i32 %res
+}
+
+define i32 @test_extra_uses_non_inbounds(i1 %cond, i1 %cond2) {
+; ALL-LABEL: @test_extra_uses_non_inbounds(
+; ALL-NEXT:  entry:
+; ALL-NEXT:    [[OBJ:%.*]] = call i8* @get_ptr.i8()
+; ALL-NEXT:    br i1 [[COND:%.*]], label [[BB1:%.*]], label [[BB2:%.*]]
+; ALL:       bb1:
+; ALL-NEXT:    [[PTR1:%.*]] = getelementptr i8, i8* [[OBJ]], i64 16
+; ALL-NEXT:    [[PTR1_TYPED:%.*]] = bitcast i8* [[PTR1]] to i32*
+; ALL-NEXT:    [[RES1:%.*]] = load i32, i32* [[PTR1_TYPED]], align 4
+; ALL-NEXT:    call void @foo.i32(i32* nonnull [[PTR1_TYPED]])
+; ALL-NEXT:    br label [[EXIT:%.*]]
+; ALL:       bb2:
+; ALL-NEXT:    [[PTR2:%.*]] = getelementptr i8, i8* [[OBJ]], i64 16
+; ALL-NEXT:    [[PTR2_TYPED:%.*]] = bitcast i8* [[PTR2]] to i32*
+; ALL-NEXT:    [[RES2:%.*]] = load i32, i32* [[PTR2_TYPED]], align 4
+; ALL-NEXT:    call void @foo.i32(i32* nonnull [[PTR2_TYPED]])
+; ALL-NEXT:    br label [[EXIT]]
+; ALL:       exit:
+; ALL-NEXT:    [[PTR_TYPED:%.*]] = phi i32* [ [[PTR1_TYPED]], [[BB1]] ], [ [[PTR2_TYPED]], [[BB2]] ]
+; ALL-NEXT:    [[RES_PHI:%.*]] = phi i32 [ [[RES1]], [[BB1]] ], [ [[RES2]], [[BB2]] ]
+; ALL-NEXT:    store i32 1, i32* [[PTR_TYPED]], align 4
+; ALL-NEXT:    [[RES:%.*]] = select i1 [[COND2:%.*]], i32 [[RES_PHI]], i32 1
+; ALL-NEXT:    ret i32 [[RES]]
+;
+entry:
+  %obj = call i8* @get_ptr.i8()
+  br i1 %cond, label %bb1, label %bb2
+
+bb1:
+  %ptr1 = getelementptr i8, i8* %obj, i64 16
+  %ptr1.typed = bitcast i8* %ptr1 to i32*
+  %res1 = load i32, i32* %ptr1.typed
+  call void @foo.i32(i32* %ptr1.typed)
+  br label %exit
+
+bb2:
+  %ptr2 = getelementptr i8, i8* %obj, i64 16
+  %ptr2.typed = bitcast i8* %ptr2 to i32*
+  %res2 = load i32, i32* %ptr2.typed
+  call void @foo.i32(i32* %ptr2.typed)
+  br label %exit
+
+exit:
+  %ptr.typed = phi i32* [ %ptr1.typed, %bb1 ], [ %ptr2.typed, %bb2 ]
+  %res.phi = phi i32 [ %res1, %bb1 ], [ %res2, %bb2 ]
+  store i32 1, i32* %ptr.typed
+  %res.load = load i32, i32* %ptr.typed
+  %res = select i1 %cond2, i32 %res.phi, i32 %res.load
+  ret i32 %res
+}
+
+define i32 @test_extra_uses_multiple_geps(i1 %cond, i1 %cond2) {
+; ALL-LABEL: @test_extra_uses_multiple_geps(
+; ALL-NEXT:  entry:
+; ALL-NEXT:    [[OBJ:%.*]] = call i8* @get_ptr.i8()
+; ALL-NEXT:    br i1 [[COND:%.*]], label [[BB1:%.*]], label [[BB2:%.*]]
+; ALL:       bb1:
+; ALL-NEXT:    [[PTR1:%.*]] = getelementptr inbounds i8, i8* [[OBJ]], i64 16
+; ALL-NEXT:    [[PTR1_TYPED:%.*]] = bitcast i8* [[PTR1]] to i32*
+; ALL-NEXT:    [[RES1:%.*]] = load i32, i32* [[PTR1_TYPED]], align 4
+; ALL-NEXT:    call void @foo.i32(i32* nonnull [[PTR1_TYPED]])
+; ALL-NEXT:    br label [[EXIT:%.*]]
+; ALL:       bb2:
+; ALL-NEXT:    [[PTR2_1:%.*]] = getelementptr i8, i8* [[OBJ]], i64 16
+; ALL-NEXT:    [[PTR2_TYPED:%.*]] = bitcast i8* [[PTR2_1]] to i32*
+; ALL-NEXT:    [[RES2:%.*]] = load i32, i32* [[PTR2_TYPED]], align 4
+; ALL-NEXT:    call void @foo.i32(i32* nonnull [[PTR2_TYPED]])
+; ALL-NEXT:    br label [[EXIT]]
+; ALL:       exit:
+; ALL-NEXT:    [[PTR_TYPED:%.*]] = phi i32* [ [[PTR1_TYPED]], [[BB1]] ], [ [[PTR2_TYPED]], [[BB2]] ]
+; ALL-NEXT:    [[RES_PHI:%.*]] = phi i32 [ [[RES1]], [[BB1]] ], [ [[RES2]], [[BB2]] ]
+; ALL-NEXT:    store i32 1, i32* [[PTR_TYPED]], align 4
+; ALL-NEXT:    [[RES:%.*]] = select i1 [[COND2:%.*]], i32 [[RES_PHI]], i32 1
+; ALL-NEXT:    ret i32 [[RES]]
+;
+entry:
+  %obj = call i8* @get_ptr.i8()
+  br i1 %cond, label %bb1, label %bb2
+
+bb1:
+  %ptr1 = getelementptr inbounds i8, i8* %obj, i64 16
+  %ptr1.typed = bitcast i8* %ptr1 to i32*
+  %res1 = load i32, i32* %ptr1.typed
+  call void @foo.i32(i32* %ptr1.typed)
+  br label %exit
+
+bb2:
+  %ptr2.0 = getelementptr i8, i8* %obj, i64 8
+  %ptr2.1 = getelementptr inbounds i8, i8* %ptr2.0, i64 8
+  %ptr2.typed = bitcast i8* %ptr2.1 to i32*
+  %res2 = load i32, i32* %ptr2.typed
+  call void @foo.i32(i32* %ptr2.typed)
+  br label %exit
+
+exit:
+  %ptr.typed = phi i32* [ %ptr1.typed, %bb1 ], [ %ptr2.typed, %bb2 ]
+  %res.phi = phi i32 [ %res1, %bb1 ], [ %res2, %bb2 ]
+  store i32 1, i32* %ptr.typed
+  %res.load = load i32, i32* %ptr.typed
+  %res = select i1 %cond2, i32 %res.phi, i32 %res.load
+  ret i32 %res
+}
+
+define i8 @test_gep_extra_uses(i1 %cond, i1 %cond2) {
+; ALL-LABEL: @test_gep_extra_uses(
+; ALL-NEXT:  entry:
+; ALL-NEXT:    [[OBJ:%.*]] = call i8* @get_ptr.i8()
+; ALL-NEXT:    br i1 [[COND:%.*]], label [[BB1:%.*]], label [[BB2:%.*]]
+; ALL:       bb1:
+; ALL-NEXT:    [[PTR1:%.*]] = getelementptr inbounds i8, i8* [[OBJ]], i64 16
+; ALL-NEXT:    [[RES1:%.*]] = load i8, i8* [[PTR1]], align 1
+; ALL-NEXT:    call void @foo.i8(i8* nonnull [[PTR1]])
+; ALL-NEXT:    br label [[EXIT:%.*]]
+; ALL:       bb2:
+; ALL-NEXT:    [[PTR2:%.*]] = getelementptr inbounds i8, i8* [[OBJ]], i64 16
+; ALL-NEXT:    [[RES2:%.*]] = load i8, i8* [[PTR2]], align 1
+; ALL-NEXT:    call void @foo.i8(i8* nonnull [[PTR2]])
+; ALL-NEXT:    br label [[EXIT]]
+; ALL:       exit:
+; ALL-NEXT:    [[PTR_TYPED:%.*]] = phi i8* [ [[PTR1]], [[BB1]] ], [ [[PTR2]], [[BB2]] ]
+; ALL-NEXT:    [[RES_PHI:%.*]] = phi i8 [ [[RES1]], [[BB1]] ], [ [[RES2]], [[BB2]] ]
+; ALL-NEXT:    store i8 1, i8* [[PTR_TYPED]], align 1
+; ALL-NEXT:    [[RES:%.*]] = select i1 [[COND2:%.*]], i8 [[RES_PHI]], i8 1
+; ALL-NEXT:    ret i8 [[RES]]
+;
+entry:
+  %obj = call i8* @get_ptr.i8()
+  br i1 %cond, label %bb1, label %bb2
+
+bb1:
+  %ptr1 = getelementptr inbounds i8, i8* %obj, i64 16
+  %res1 = load i8, i8* %ptr1
+  call void @foo.i8(i8* %ptr1)
+  br label %exit
+
+bb2:
+  %ptr2 = getelementptr inbounds i8, i8* %obj, i64 16
+  %res2 = load i8, i8* %ptr2
+  call void @foo.i8(i8* %ptr2)
+  br label %exit
+
+exit:
+  %ptr.typed = phi i8* [ %ptr1, %bb1 ], [ %ptr2, %bb2 ]
+  %res.phi = phi i8 [ %res1, %bb1 ], [ %res2, %bb2 ]
+  store i8 1, i8* %ptr.typed
+  %res.load = load i8, i8* %ptr.typed
+  %res = select i1 %cond2, i8 %res.phi, i8 %res.load
+  ret i8 %res
+}
diff --git a/llvm/test/Transforms/InstCombine/phi-known-bits-operand-order.ll b/llvm/test/Transforms/InstCombine/phi-known-bits-operand-order.ll
new file mode 100644
index 000000000..e21e388e7
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/phi-known-bits-operand-order.ll
@@ -0,0 +1,99 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+; Check that we can turn the icmp sle into icmp ult, regardless of the
+; order of the incoming values of the PHI node.
+
+declare i1 @cond()
+
+define void @phi_recurrence_start_first() {
+; CHECK-LABEL: @phi_recurrence_start_first(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[WHILE_COND:%.*]]
+; CHECK:       while.cond:
+; CHECK-NEXT:    [[CELL_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[START:%.*]], [[FOR_COND26:%.*]] ]
+; CHECK-NEXT:    [[COND_V:%.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 [[COND_V]], label [[IF_THEN:%.*]], label [[WHILE_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[START]] = add nuw nsw i32 [[CELL_0]], 1
+; CHECK-NEXT:    br i1 [[COND_V]], label [[FOR_COND11:%.*]], label [[FOR_COND26]]
+; CHECK:       for.cond11:
+; CHECK-NEXT:    [[I_1:%.*]] = phi i32 [ [[START]], [[IF_THEN]] ], [ [[STEP:%.*]], [[FOR_COND11]] ]
+; CHECK-NEXT:    [[CMP13:%.*]] = icmp ult i32 [[I_1]], 100
+; CHECK-NEXT:    [[STEP]] = add nuw nsw i32 [[I_1]], 1
+; CHECK-NEXT:    br i1 [[CMP13]], label [[FOR_COND11]], label [[WHILE_END]]
+; CHECK:       for.cond26:
+; CHECK-NEXT:    br label [[WHILE_COND]]
+; CHECK:       while.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %while.cond
+
+while.cond:                                       ; preds = %entry, %for.cond26
+  %cell.0 = phi i32 [ 0, %entry ], [ %start, %for.cond26 ]
+  %cond.v = call i1 @cond()
+  br i1 %cond.v, label %if.then, label %while.end
+
+if.then:                                          ; preds = %while.cond
+  %start = add nsw i32 %cell.0, 1
+  br i1 %cond.v, label %for.cond11, label %for.cond26
+
+for.cond11:                                       ; preds = %for.cond11, %if.then
+  %i.1 = phi i32 [ %start, %if.then ], [ %step, %for.cond11 ]
+  %cmp13 = icmp sle i32 %i.1, 99
+  %step = add nsw i32 %i.1, 1
+  br i1 %cmp13, label %for.cond11, label %while.end
+
+for.cond26:                                       ; preds = %if.then
+  br label %while.cond
+
+while.end:                                        ; preds = %while.cond, %for.cond11
+  ret void
+}
+
+define void @phi_recurrence_step_first() {
+; CHECK-LABEL: @phi_recurrence_step_first(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[WHILE_COND:%.*]]
+; CHECK:       while.cond:
+; CHECK-NEXT:    [[CELL_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[START:%.*]], [[FOR_COND26:%.*]] ]
+; CHECK-NEXT:    [[COND_V:%.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 [[COND_V]], label [[IF_THEN:%.*]], label [[WHILE_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[START]] = add nuw nsw i32 [[CELL_0]], 1
+; CHECK-NEXT:    br i1 [[COND_V]], label [[FOR_COND11:%.*]], label [[FOR_COND26]]
+; CHECK:       for.cond11:
+; CHECK-NEXT:    [[I_1:%.*]] = phi i32 [ [[STEP:%.*]], [[FOR_COND11]] ], [ [[START]], [[IF_THEN]] ]
+; CHECK-NEXT:    [[CMP13:%.*]] = icmp ult i32 [[I_1]], 100
+; CHECK-NEXT:    [[STEP]] = add nuw nsw i32 [[I_1]], 1
+; CHECK-NEXT:    br i1 [[CMP13]], label [[FOR_COND11]], label [[WHILE_END]]
+; CHECK:       for.cond26:
+; CHECK-NEXT:    br label [[WHILE_COND]]
+; CHECK:       while.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %while.cond
+
+while.cond:                                       ; preds = %entry, %for.cond26
+  %cell.0 = phi i32 [ 0, %entry ], [ %start, %for.cond26 ]
+  %cond.v = call i1 @cond()
+  br i1 %cond.v, label %if.then, label %while.end
+
+if.then:                                          ; preds = %while.cond
+  %start = add nsw i32 %cell.0, 1
+  br i1 %cond.v, label %for.cond11, label %for.cond26
+
+for.cond11:                                       ; preds = %for.cond11, %if.then
+  %i.1 = phi i32 [ %step, %for.cond11 ], [ %start, %if.then]
+  %cmp13 = icmp sle i32 %i.1, 99
+  %step = add nsw i32 %i.1, 1
+  br i1 %cmp13, label %for.cond11, label %while.end
+
+for.cond26:                                       ; preds = %if.then
+  br label %while.cond
+
+while.end:                                        ; preds = %while.cond, %for.cond11
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/pow-exp.ll b/llvm/test/Transforms/InstCombine/pow-exp.ll
index f352c3a88..a583c367e 100644
--- a/llvm/test/Transforms/InstCombine/pow-exp.ll
+++ b/llvm/test/Transforms/InstCombine/pow-exp.ll
@@ -212,6 +212,7 @@ declare void @use_f(float)
 
 define double @pow_ok_base(double %e) {
 ; CHECK-LABEL: @pow_ok_base(
+; Do not change 0xBFE0776{{.*}} to the exact constant, see PR42740
 ; CHECK-NEXT:    [[MUL:%.*]] = fmul nnan ninf afn double [[E:%.*]], 0xBFE0776{{.*}}
 ; CHECK-NEXT:    [[EXP2:%.*]] = call nnan ninf afn double @exp2(double [[MUL]])
 ; CHECK-NEXT:    ret double [[EXP2]]
diff --git a/llvm/test/Transforms/InstCombine/pow-sqrt.ll b/llvm/test/Transforms/InstCombine/pow-sqrt.ll
index c87eac705..588d4d129 100644
--- a/llvm/test/Transforms/InstCombine/pow-sqrt.ll
+++ b/llvm/test/Transforms/InstCombine/pow-sqrt.ll
@@ -150,31 +150,94 @@ define double @pow_intrinsic_half_fast(double %x) {
   ret double %pow
 }
 
+; This should not be transformed without some kind of FMF.
 ; -0.5 means take the reciprocal.
 
 define float @pow_libcall_neghalf_no_FMF(float %x) {
 ; CHECK-LABEL: @pow_libcall_neghalf_no_FMF(
-; CHECK-NEXT:    [[SQRTF:%.*]] = call float @sqrtf(float [[X:%.*]])
-; CHECK-NEXT:    [[ABS:%.*]] = call float @llvm.fabs.f32(float [[SQRTF]])
-; CHECK-NEXT:    [[ISINF:%.*]] = fcmp oeq float [[X]], 0xFFF0000000000000
-; CHECK-NEXT:    [[ABS_OP:%.*]] = fdiv float 1.000000e+00, [[ABS]]
+; CHECK-NEXT:    [[POW:%.*]] = call float @powf(float [[X:%.*]], float -5.000000e-01)
+; CHECK-NEXT:    ret float [[POW]]
+;
+  %pow = call float @powf(float %x, float -5.0e-01)
+  ret float %pow
+}
+
+; Transform to sqrt+fdiv because 'reassoc' allows an extra rounding step.
+; Use 'fabs' to handle -0.0 correctly.
+; Use 'select' to handle -INF correctly.
+
+define float @pow_libcall_neghalf_reassoc(float %x) {
+; CHECK-LABEL: @pow_libcall_neghalf_reassoc(
+; CHECK-NEXT:    [[SQRTF:%.*]] = call reassoc float @sqrtf(float [[X:%.*]])
+; CHECK-NEXT:    [[ABS:%.*]] = call reassoc float @llvm.fabs.f32(float [[SQRTF]])
+; CHECK-NEXT:    [[ISINF:%.*]] = fcmp reassoc oeq float [[X]], 0xFFF0000000000000
+; CHECK-NEXT:    [[ABS_OP:%.*]] = fdiv reassoc float 1.000000e+00, [[ABS]]
 ; CHECK-NEXT:    [[RECIPROCAL:%.*]] = select i1 [[ISINF]], float 0.000000e+00, float [[ABS_OP]]
 ; CHECK-NEXT:    ret float [[RECIPROCAL]]
 ;
-  %pow = call float @powf(float %x, float -5.0e-01)
+  %pow = call reassoc float @powf(float %x, float -5.0e-01)
   ret float %pow
 }
 
+; Transform to sqrt+fdiv because 'afn' allows an extra rounding step.
+; Use 'fabs' to handle -0.0 correctly.
+; Use 'select' to handle -INF correctly.
+
+define float @pow_libcall_neghalf_afn(float %x) {
+; CHECK-LABEL: @pow_libcall_neghalf_afn(
+; CHECK-NEXT:    [[SQRTF:%.*]] = call afn float @sqrtf(float [[X:%.*]])
+; CHECK-NEXT:    [[ABS:%.*]] = call afn float @llvm.fabs.f32(float [[SQRTF]])
+; CHECK-NEXT:    [[ISINF:%.*]] = fcmp afn oeq float [[X]], 0xFFF0000000000000
+; CHECK-NEXT:    [[ABS_OP:%.*]] = fdiv afn float 1.000000e+00, [[ABS]]
+; CHECK-NEXT:    [[RECIPROCAL:%.*]] = select i1 [[ISINF]], float 0.000000e+00, float [[ABS_OP]]
+; CHECK-NEXT:    ret float [[RECIPROCAL]]
+;
+  %pow = call afn float @powf(float %x, float -5.0e-01)
+  ret float %pow
+}
+
+; This should not be transformed without some kind of FMF.
+
 define <2 x double> @pow_intrinsic_neghalf_no_FMF(<2 x double> %x) {
 ; CHECK-LABEL: @pow_intrinsic_neghalf_no_FMF(
-; CHECK-NEXT:    [[SQRT:%.*]] = call <2 x double> @llvm.sqrt.v2f64(<2 x double> [[X:%.*]])
-; CHECK-NEXT:    [[ABS:%.*]] = call <2 x double> @llvm.fabs.v2f64(<2 x double> [[SQRT]])
-; CHECK-NEXT:    [[ISINF:%.*]] = fcmp oeq <2 x double> [[X]], <double 0xFFF0000000000000, double 0xFFF0000000000000>
-; CHECK-NEXT:    [[ABS_OP:%.*]] = fdiv <2 x double> <double 1.000000e+00, double 1.000000e+00>, [[ABS]]
+; CHECK-NEXT:    [[POW:%.*]] = call <2 x double> @llvm.pow.v2f64(<2 x double> [[X:%.*]], <2 x double> <double -5.000000e-01, double -5.000000e-01>)
+; CHECK-NEXT:    ret <2 x double> [[POW]]
+;
+  %pow = call <2 x double> @llvm.pow.v2f64(<2 x double> %x, <2 x double> <double -5.0e-01, double -5.0e-01>)
+  ret <2 x double> %pow
+}
+
+; Transform to sqrt+fdiv because 'reassoc' allows an extra rounding step.
+; Use 'fabs' to handle -0.0 correctly.
+; Use 'select' to handle -INF correctly.
+
+define <2 x double> @pow_intrinsic_neghalf_reassoc(<2 x double> %x) {
+; CHECK-LABEL: @pow_intrinsic_neghalf_reassoc(
+; CHECK-NEXT:    [[SQRT:%.*]] = call reassoc <2 x double> @llvm.sqrt.v2f64(<2 x double> [[X:%.*]])
+; CHECK-NEXT:    [[ABS:%.*]] = call reassoc <2 x double> @llvm.fabs.v2f64(<2 x double> [[SQRT]])
+; CHECK-NEXT:    [[ISINF:%.*]] = fcmp reassoc oeq <2 x double> [[X]], <double 0xFFF0000000000000, double 0xFFF0000000000000>
+; CHECK-NEXT:    [[ABS_OP:%.*]] = fdiv reassoc <2 x double> <double 1.000000e+00, double 1.000000e+00>, [[ABS]]
 ; CHECK-NEXT:    [[RECIPROCAL:%.*]] = select <2 x i1> [[ISINF]], <2 x double> zeroinitializer, <2 x double> [[ABS_OP]]
 ; CHECK-NEXT:    ret <2 x double> [[RECIPROCAL]]
 ;
-  %pow = call <2 x double> @llvm.pow.v2f64(<2 x double> %x, <2 x double> <double -5.0e-01, double -5.0e-01>)
+  %pow = call reassoc <2 x double> @llvm.pow.v2f64(<2 x double> %x, <2 x double> <double -5.0e-01, double -5.0e-01>)
+  ret <2 x double> %pow
+}
+
+; Transform to sqrt+fdiv because 'afn' allows an extra rounding step.
+; Use 'fabs' to handle -0.0 correctly.
+; Use 'select' to handle -INF correctly.
+
+define <2 x double> @pow_intrinsic_neghalf_afn(<2 x double> %x) {
+; CHECK-LABEL: @pow_intrinsic_neghalf_afn(
+; CHECK-NEXT:    [[SQRT:%.*]] = call afn <2 x double> @llvm.sqrt.v2f64(<2 x double> [[X:%.*]])
+; CHECK-NEXT:    [[ABS:%.*]] = call afn <2 x double> @llvm.fabs.v2f64(<2 x double> [[SQRT]])
+; CHECK-NEXT:    [[ISINF:%.*]] = fcmp afn oeq <2 x double> [[X]], <double 0xFFF0000000000000, double 0xFFF0000000000000>
+; CHECK-NEXT:    [[ABS_OP:%.*]] = fdiv afn <2 x double> <double 1.000000e+00, double 1.000000e+00>, [[ABS]]
+; CHECK-NEXT:    [[RECIPROCAL:%.*]] = select <2 x i1> [[ISINF]], <2 x double> zeroinitializer, <2 x double> [[ABS_OP]]
+; CHECK-NEXT:    ret <2 x double> [[RECIPROCAL]]
+;
+  %pow = call afn <2 x double> @llvm.pow.v2f64(<2 x double> %x, <2 x double> <double -5.0e-01, double -5.0e-01>)
   ret <2 x double> %pow
 }
 
@@ -182,23 +245,23 @@ define <2 x double> @pow_intrinsic_neghalf_no_FMF(<2 x double> %x) {
 
 define double @pow_libcall_neghalf_ninf(double %x) {
 ; CHECK-LABEL: @pow_libcall_neghalf_ninf(
-; CHECK-NEXT:    [[SQRT:%.*]] = call ninf double @sqrt(double [[X:%.*]])
-; CHECK-NEXT:    [[ABS:%.*]] = call ninf double @llvm.fabs.f64(double [[SQRT]])
-; CHECK-NEXT:    [[RECIPROCAL:%.*]] = fdiv ninf double 1.000000e+00, [[ABS]]
+; CHECK-NEXT:    [[SQRT:%.*]] = call ninf afn double @sqrt(double [[X:%.*]])
+; CHECK-NEXT:    [[ABS:%.*]] = call ninf afn double @llvm.fabs.f64(double [[SQRT]])
+; CHECK-NEXT:    [[RECIPROCAL:%.*]] = fdiv ninf afn double 1.000000e+00, [[ABS]]
 ; CHECK-NEXT:    ret double [[RECIPROCAL]]
 ;
-  %pow = call ninf double @pow(double %x, double -5.0e-01)
+  %pow = call afn ninf double @pow(double %x, double -5.0e-01)
   ret double %pow
 }
 
 define <2 x double> @pow_intrinsic_neghalf_ninf(<2 x double> %x) {
 ; CHECK-LABEL: @pow_intrinsic_neghalf_ninf(
-; CHECK-NEXT:    [[SQRT:%.*]] = call ninf <2 x double> @llvm.sqrt.v2f64(<2 x double> [[X:%.*]])
-; CHECK-NEXT:    [[ABS:%.*]] = call ninf <2 x double> @llvm.fabs.v2f64(<2 x double> [[SQRT]])
-; CHECK-NEXT:    [[RECIPROCAL:%.*]] = fdiv ninf <2 x double> <double 1.000000e+00, double 1.000000e+00>, [[ABS]]
+; CHECK-NEXT:    [[SQRT:%.*]] = call ninf afn <2 x double> @llvm.sqrt.v2f64(<2 x double> [[X:%.*]])
+; CHECK-NEXT:    [[ABS:%.*]] = call ninf afn <2 x double> @llvm.fabs.v2f64(<2 x double> [[SQRT]])
+; CHECK-NEXT:    [[RECIPROCAL:%.*]] = fdiv ninf afn <2 x double> <double 1.000000e+00, double 1.000000e+00>, [[ABS]]
 ; CHECK-NEXT:    ret <2 x double> [[RECIPROCAL]]
 ;
-  %pow = call ninf <2 x double> @llvm.pow.v2f64(<2 x double> %x, <2 x double> <double -5.0e-01, double -5.0e-01>)
+  %pow = call afn ninf <2 x double> @llvm.pow.v2f64(<2 x double> %x, <2 x double> <double -5.0e-01, double -5.0e-01>)
   ret <2 x double> %pow
 }
 
@@ -206,25 +269,25 @@ define <2 x double> @pow_intrinsic_neghalf_ninf(<2 x double> %x) {
 
 define double @pow_libcall_neghalf_nsz(double %x) {
 ; CHECK-LABEL: @pow_libcall_neghalf_nsz(
-; CHECK-NEXT:    [[SQRT:%.*]] = call nsz double @sqrt(double [[X:%.*]])
-; CHECK-NEXT:    [[ISINF:%.*]] = fcmp nsz oeq double [[X]], 0xFFF0000000000000
-; CHECK-NEXT:    [[SQRT_OP:%.*]] = fdiv nsz double 1.000000e+00, [[SQRT]]
+; CHECK-NEXT:    [[SQRT:%.*]] = call nsz afn double @sqrt(double [[X:%.*]])
+; CHECK-NEXT:    [[ISINF:%.*]] = fcmp nsz afn oeq double [[X]], 0xFFF0000000000000
+; CHECK-NEXT:    [[SQRT_OP:%.*]] = fdiv nsz afn double 1.000000e+00, [[SQRT]]
 ; CHECK-NEXT:    [[RECIPROCAL:%.*]] = select i1 [[ISINF]], double 0.000000e+00, double [[SQRT_OP]]
 ; CHECK-NEXT:    ret double [[RECIPROCAL]]
 ;
-  %pow = call nsz double @pow(double %x, double -5.0e-01)
+  %pow = call afn nsz double @pow(double %x, double -5.0e-01)
   ret double %pow
 }
 
 define double @pow_intrinsic_neghalf_nsz(double %x) {
 ; CHECK-LABEL: @pow_intrinsic_neghalf_nsz(
-; CHECK-NEXT:    [[SQRT:%.*]] = call nsz double @llvm.sqrt.f64(double [[X:%.*]])
-; CHECK-NEXT:    [[ISINF:%.*]] = fcmp nsz oeq double [[X]], 0xFFF0000000000000
-; CHECK-NEXT:    [[SQRT_OP:%.*]] = fdiv nsz double 1.000000e+00, [[SQRT]]
+; CHECK-NEXT:    [[SQRT:%.*]] = call nsz afn double @llvm.sqrt.f64(double [[X:%.*]])
+; CHECK-NEXT:    [[ISINF:%.*]] = fcmp nsz afn oeq double [[X]], 0xFFF0000000000000
+; CHECK-NEXT:    [[SQRT_OP:%.*]] = fdiv nsz afn double 1.000000e+00, [[SQRT]]
 ; CHECK-NEXT:    [[RECIPROCAL:%.*]] = select i1 [[ISINF]], double 0.000000e+00, double [[SQRT_OP]]
 ; CHECK-NEXT:    ret double [[RECIPROCAL]]
 ;
-  %pow = call nsz double @llvm.pow.f64(double %x, double -5.0e-01)
+  %pow = call afn nsz double @llvm.pow.f64(double %x, double -5.0e-01)
   ret double %pow
 }
 
@@ -232,21 +295,21 @@ define double @pow_intrinsic_neghalf_nsz(double %x) {
 
 define double @pow_intrinsic_neghalf_ninf_nsz(double %x) {
 ; CHECK-LABEL: @pow_intrinsic_neghalf_ninf_nsz(
-; CHECK-NEXT:    [[SQRT:%.*]] = call ninf nsz double @llvm.sqrt.f64(double [[X:%.*]])
-; CHECK-NEXT:    [[RECIPROCAL:%.*]] = fdiv ninf nsz double 1.000000e+00, [[SQRT]]
+; CHECK-NEXT:    [[SQRT:%.*]] = call ninf nsz afn double @llvm.sqrt.f64(double [[X:%.*]])
+; CHECK-NEXT:    [[RECIPROCAL:%.*]] = fdiv ninf nsz afn double 1.000000e+00, [[SQRT]]
 ; CHECK-NEXT:    ret double [[RECIPROCAL]]
 ;
-  %pow = call ninf nsz double @llvm.pow.f64(double %x, double -5.0e-01)
+  %pow = call afn ninf nsz double @llvm.pow.f64(double %x, double -5.0e-01)
   ret double %pow
 }
 
 define float @pow_libcall_neghalf_ninf_nsz(float %x) {
 ; CHECK-LABEL: @pow_libcall_neghalf_ninf_nsz(
-; CHECK-NEXT:    [[SQRTF:%.*]] = call ninf nsz float @sqrtf(float [[X:%.*]])
-; CHECK-NEXT:    [[RECIPROCAL:%.*]] = fdiv ninf nsz float 1.000000e+00, [[SQRTF]]
+; CHECK-NEXT:    [[SQRTF:%.*]] = call ninf nsz afn float @sqrtf(float [[X:%.*]])
+; CHECK-NEXT:    [[RECIPROCAL:%.*]] = fdiv ninf nsz afn float 1.000000e+00, [[SQRTF]]
 ; CHECK-NEXT:    ret float [[RECIPROCAL]]
 ;
-  %pow = call ninf nsz float @powf(float %x, float -5.0e-01)
+  %pow = call afn ninf nsz float @powf(float %x, float -5.0e-01)
   ret float %pow
 }
 
@@ -262,19 +325,6 @@ define float @pow_libcall_neghalf_fast(float %x) {
   ret float %pow
 }
 
-define float @powf_libcall_neghalf_approx(float %x) {
-; CHECK-LABEL: @powf_libcall_neghalf_approx(
-; CHECK-NEXT:    [[SQRTF:%.*]] = call afn float @sqrtf(float [[X:%.*]])
-; CHECK-NEXT:    [[ABS:%.*]] = call afn float @llvm.fabs.f32(float [[SQRTF]])
-; CHECK-NEXT:    [[ISINF:%.*]] = fcmp afn oeq float [[X]], 0xFFF0000000000000
-; CHECK-NEXT:    [[ABS_OP:%.*]] = fdiv afn float 1.000000e+00, [[ABS]]
-; CHECK-NEXT:    [[RECIPROCAL:%.*]] = select i1 [[ISINF]], float 0.000000e+00, float [[ABS_OP]]
-; CHECK-NEXT:    ret float [[RECIPROCAL]]
-;
-  %pow = call afn float @powf(float %x, float -5.0e-01)
-  ret float %pow
-}
-
 define double @pow_intrinsic_neghalf_fast(double %x) {
 ; CHECK-LABEL: @pow_intrinsic_neghalf_fast(
 ; CHECK-NEXT:    [[SQRT:%.*]] = call fast double @llvm.sqrt.f64(double [[X:%.*]])
diff --git a/llvm/test/Transforms/InstCombine/pow_fp_int.ll b/llvm/test/Transforms/InstCombine/pow_fp_int.ll
index b0cb836da..b4117d4ca 100644
--- a/llvm/test/Transforms/InstCombine/pow_fp_int.ll
+++ b/llvm/test/Transforms/InstCombine/pow_fp_int.ll
@@ -49,6 +49,18 @@ define double @pow_uitofp_double_const_base_fast(i31 %x) {
   ret double %pow
 }
 
+define double @pow_sitofp_double_const_base_2_fast(i32 %x) {
+; CHECK-LABEL: @pow_sitofp_double_const_base_2_fast(
+; CHECK-NEXT:    [[LDEXPF:%.*]] = call afn float @ldexpf(float 1.000000e+00, i32 [[X:%.*]]) #1
+; CHECK-NEXT:    [[RES:%.*]] = fpext float [[LDEXPF]] to double
+; CHECK-NEXT:    ret double [[RES]]
+;
+  %subfp = sitofp i32 %x to float
+  %pow = tail call afn float @llvm.pow.f32(float 2.000000e+00, float %subfp)
+  %res = fpext float %pow to double
+  ret double %res
+}
+
 define double @pow_sitofp_double_const_base_power_of_2_fast(i32 %x) {
 ; CHECK-LABEL: @pow_sitofp_double_const_base_power_of_2_fast(
 ; CHECK-NEXT:    [[SUBFP:%.*]] = sitofp i32 [[X:%.*]] to float
@@ -63,6 +75,19 @@ define double @pow_sitofp_double_const_base_power_of_2_fast(i32 %x) {
   ret double %res
 }
 
+define double @pow_uitofp_const_base_2_fast(i31 %x) {
+; CHECK-LABEL: @pow_uitofp_const_base_2_fast(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i31 [[X:%.*]] to i32
+; CHECK-NEXT:    [[LDEXPF:%.*]] = call afn float @ldexpf(float 1.000000e+00, i32 [[TMP1]]) #1
+; CHECK-NEXT:    [[RES:%.*]] = fpext float [[LDEXPF]] to double
+; CHECK-NEXT:    ret double [[RES]]
+;
+  %subfp = uitofp i31 %x to float
+  %pow = tail call afn float @llvm.pow.f32(float 2.000000e+00, float %subfp)
+  %res = fpext float %pow to double
+  ret double %res
+}
+
 define double @pow_uitofp_const_base_power_of_2_fast(i31 %x) {
 ; CHECK-LABEL: @pow_uitofp_const_base_power_of_2_fast(
 ; CHECK-NEXT:    [[SUBFP:%.*]] = uitofp i31 [[X:%.*]] to float
@@ -210,6 +235,19 @@ define double @pow_uitofp_const_base_fast_i32(i32 %x) {
   ret double %res
 }
 
+define double @pow_uitofp_const_base_2_fast_i32(i32 %x) {
+; CHECK-LABEL: @pow_uitofp_const_base_2_fast_i32(
+; CHECK-NEXT:    [[SUBFP:%.*]] = uitofp i32 [[X:%.*]] to float
+; CHECK-NEXT:    [[EXP2:%.*]] = call fast float @llvm.exp2.f32(float [[SUBFP]])
+; CHECK-NEXT:    [[RES:%.*]] = fpext float [[EXP2]] to double
+; CHECK-NEXT:    ret double [[RES]]
+;
+  %subfp = uitofp i32 %x to float
+  %pow = tail call fast float @llvm.pow.f32(float 2.000000e+00, float %subfp)
+  %res = fpext float %pow to double
+  ret double %res
+}
+
 define double @pow_uitofp_const_base_power_of_2_fast_i32(i32 %x) {
 ; CHECK-LABEL: @pow_uitofp_const_base_power_of_2_fast_i32(
 ; CHECK-NEXT:    [[SUBFP:%.*]] = uitofp i32 [[X:%.*]] to float
@@ -251,6 +289,7 @@ define double @pow_uitofp_double_base_fast_i32(double %base, i32 %x) {
 define double @pow_sitofp_const_base_fast_i64(i64 %x) {
 ; CHECK-LABEL: @pow_sitofp_const_base_fast_i64(
 ; CHECK-NEXT:    [[SUBFP:%.*]] = sitofp i64 [[X:%.*]] to float
+; Do not change 0x400675{{.*}} to the exact constant, see PR42740
 ; CHECK-NEXT:    [[MUL:%.*]] = fmul fast float [[SUBFP]], 0x400675{{.*}}
 ; CHECK-NEXT:    [[EXP2:%.*]] = call fast float @llvm.exp2.f32(float [[MUL]])
 ; CHECK-NEXT:    [[RES:%.*]] = fpext float [[EXP2]] to double
@@ -302,6 +341,18 @@ define double @pow_uitofp_const_base_no_fast(i32 %x) {
   ret double %res
 }
 
+define double @pow_sitofp_const_base_2_no_fast(i32 %x) {
+; CHECK-LABEL: @pow_sitofp_const_base_2_no_fast(
+; CHECK-NEXT:    [[LDEXPF:%.*]] = call float @ldexpf(float 1.000000e+00, i32 [[X:%.*]]) #1
+; CHECK-NEXT:    [[RES:%.*]] = fpext float [[LDEXPF]] to double
+; CHECK-NEXT:    ret double [[RES]]
+;
+  %subfp = sitofp i32 %x to float
+  %pow = tail call float @llvm.pow.f32(float 2.000000e+00, float %subfp)
+  %res = fpext float %pow to double
+  ret double %res
+}
+
 define double @pow_sitofp_const_base_power_of_2_no_fast(i32 %x) {
 ; CHECK-LABEL: @pow_sitofp_const_base_power_of_2_no_fast(
 ; CHECK-NEXT:    [[SUBFP:%.*]] = sitofp i32 [[X:%.*]] to float
@@ -316,6 +367,19 @@ define double @pow_sitofp_const_base_power_of_2_no_fast(i32 %x) {
   ret double %res
 }
 
+define double @pow_uitofp_const_base_2_no_fast(i32 %x) {
+; CHECK-LABEL: @pow_uitofp_const_base_2_no_fast(
+; CHECK-NEXT:    [[SUBFP:%.*]] = uitofp i32 [[X:%.*]] to float
+; CHECK-NEXT:    [[EXP2:%.*]] = call float @llvm.exp2.f32(float [[SUBFP]])
+; CHECK-NEXT:    [[RES:%.*]] = fpext float [[EXP2]] to double
+; CHECK-NEXT:    ret double [[RES]]
+;
+  %subfp = uitofp i32 %x to float
+  %pow = tail call float @llvm.pow.f32(float 2.000000e+00, float %subfp)
+  %res = fpext float %pow to double
+  ret double %res
+}
+
 define double @pow_uitofp_const_base_power_of_2_no_fast(i32 %x) {
 ; CHECK-LABEL: @pow_uitofp_const_base_power_of_2_no_fast(
 ; CHECK-NEXT:    [[SUBFP:%.*]] = uitofp i32 [[X:%.*]] to float
diff --git a/llvm/test/Transforms/InstCombine/pr26992.ll b/llvm/test/Transforms/InstCombine/pr26992.ll
index e5bfb5c0e..da5487780 100644
--- a/llvm/test/Transforms/InstCombine/pr26992.ll
+++ b/llvm/test/Transforms/InstCombine/pr26992.ll
@@ -1,16 +1,37 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -instcombine -S < %s | FileCheck %s
 target triple = "x86_64-pc-windows-msvc"
 
 define i1 @test1(i8* %p) personality i32 (...)* @__CxxFrameHandler3 {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[A:%.*]] = getelementptr i8, i8* [[P:%.*]], i64 1
+; CHECK-NEXT:    invoke void @may_throw()
+; CHECK-NEXT:    to label [[INVOKE_CONT:%.*]] unwind label [[CATCH_DISPATCH:%.*]]
+; CHECK:       invoke.cont:
+; CHECK-NEXT:    [[B:%.*]] = getelementptr i8, i8* [[P]], i64 2
+; CHECK-NEXT:    invoke void @may_throw()
+; CHECK-NEXT:    to label [[EXIT:%.*]] unwind label [[CATCH_DISPATCH]]
+; CHECK:       catch.dispatch:
+; CHECK-NEXT:    [[C:%.*]] = phi i8* [ [[B]], [[INVOKE_CONT]] ], [ [[A]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = catchswitch within none [label %catch] unwind to caller
+; CHECK:       catch:
+; CHECK-NEXT:    [[TMP2:%.*]] = catchpad within [[TMP1]] [i8* null, i32 64, i8* null]
+; CHECK-NEXT:    catchret from [[TMP2]] to label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[D:%.*]] = phi i8* [ [[A]], [[INVOKE_CONT]] ], [ [[C]], [[CATCH:%.*]] ]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8* [[D]], [[A]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
 entry:
   %a = getelementptr i8, i8* %p, i64 1
   invoke void @may_throw()
-          to label %invoke.cont unwind label %catch.dispatch
+  to label %invoke.cont unwind label %catch.dispatch
 
 invoke.cont:
   %b = getelementptr inbounds i8, i8* %a, i64 1
   invoke void @may_throw()
-          to label %exit unwind label %catch.dispatch
+  to label %exit unwind label %catch.dispatch
 
 catch.dispatch:
   %c = phi i8* [ %b, %invoke.cont ], [ %a, %entry ]
@@ -26,12 +47,6 @@ exit:
   ret i1 %cmp
 }
 
-; CHECK-LABEL: define i1 @test1(
-; CHECK:  %[[gep_a:.*]] = getelementptr i8, i8* %p, i64 1
-; CHECK:  %[[gep_b:.*]] = getelementptr inbounds i8, i8* %p, i64 2
-; CHECK:  phi i8* [ %[[gep_b]], {{.*}} ], [ %[[gep_a]], {{.*}} ]
-; CHECK:  %tmp1 = catchswitch within none [label %catch] unwind to caller
-
 declare void @may_throw()
 
 declare i32 @__CxxFrameHandler3(...)
diff --git a/llvm/test/Transforms/InstCombine/pr39177.ll b/llvm/test/Transforms/InstCombine/pr39177.ll
index 35c5ce0d3..c0f0e8604 100644
--- a/llvm/test/Transforms/InstCombine/pr39177.ll
+++ b/llvm/test/Transforms/InstCombine/pr39177.ll
@@ -30,7 +30,7 @@ define void @foo() {
 ; CHECK-LABEL: @foo(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load %struct._IO_FILE*, %struct._IO_FILE** @stderr, align 8
-; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @__fwrite_alias(i8* getelementptr inbounds ([8 x i8], [8 x i8]* @.str, i64 0, i64 0), i64 7, i64 1, %struct._IO_FILE* [[TMP0]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @fwrite(i8* getelementptr inbounds ([8 x i8], [8 x i8]* @.str, i64 0, i64 0), i64 7, i64 1, %struct._IO_FILE* [[TMP0]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
diff --git a/llvm/test/Transforms/InstCombine/pr43081.ll b/llvm/test/Transforms/InstCombine/pr43081.ll
new file mode 100644
index 000000000..42f910043
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/pr43081.ll
@@ -0,0 +1,15 @@
+; RUN: opt < %s -instcombine -disable-builtin strlen -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+declare i8* @strchr(i8*, i32)
+
+define i8* @pr43081(i8* %a) {
+entry:
+  %a.addr = alloca i8*, align 8
+  store i8* %a, i8** %a.addr, align 8
+  %0 = load i8*, i8** %a.addr, align 8
+  %call = call i8* @strchr(i8* %0, i32 0)
+  ret i8* %call
+; CHECK: call i8* @strchr
+}
diff --git a/llvm/test/Transforms/InstCombine/pr43376-getFlippedStrictnessPredicateAndConstant-assert.ll b/llvm/test/Transforms/InstCombine/pr43376-getFlippedStrictnessPredicateAndConstant-assert.ll
new file mode 100644
index 000000000..1e5bf5190
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/pr43376-getFlippedStrictnessPredicateAndConstant-assert.ll
@@ -0,0 +1,36 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; We used to hit an assertion in getFlippedStrictnessPredicateAndConstant due
+; to assuming that edge cases such as %cmp (ult x, 0) already has been
+; simplified. But that depends on the worklist order, so that is not always
+; guaranteed.
+
+define i16 @d(i16* %d.a, i16* %d.b) {
+; CHECK-LABEL: @d(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[T0:%.*]] = load i16, i16* [[D_A:%.*]], align 1
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i16 [[T0]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL]], label [[LAND_END:%.*]], label [[LAND_RHS:%.*]]
+; CHECK:       land.rhs:
+; CHECK-NEXT:    br label [[LAND_END]]
+; CHECK:       land.end:
+; CHECK-NEXT:    ret i16 -1
+;
+entry:
+  %t0 = load i16, i16* %d.a, align 1
+  %tobool = icmp ne i16 %t0, 0
+  br i1 %tobool, label %land.rhs, label %land.end
+
+land.rhs:
+  %t1 = load i16, i16* %d.b, align 1
+  %cmp = icmp ult i16 %t1, 0
+  br label %land.end
+
+land.end:
+  %t2 = phi i1 [ false, %entry ], [ %cmp, %land.rhs ]
+  %land.ext = zext i1 %t2 to i16
+  %mul = mul nsw i16 %land.ext, 3
+  %neg = xor i16 %mul, -1
+  ret i16 %neg
+}
diff --git a/llvm/test/Transforms/InstCombine/pr43893.ll b/llvm/test/Transforms/InstCombine/pr43893.ll
new file mode 100644
index 000000000..5eaae20fe
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/pr43893.ll
@@ -0,0 +1,55 @@
+; Check for setting dbg.value as undef which depends on trivially dead instructions.
+; RUN: opt -instcombine -S -o - %s | FileCheck %s
+
+@a = common dso_local global i8 0, align 1, !dbg !0
+@b = common dso_local global i8 0, align 1, !dbg !6
+
+define dso_local i32 @main() !dbg !13 {
+entry:
+  %0 = load i8, i8* @a, align 1, !dbg !17
+  %dec = add i8 %0, -1, !dbg !17
+  store i8 %dec, i8* @a, align 1, !dbg !17
+;CHECK: call void @llvm.dbg.value(metadata i32 undef
+;CHECK: call void @llvm.dbg.value(metadata i32 -8
+;CHECK: call void @llvm.dbg.value(metadata i32 undef
+  %conv = sext i8 %dec to i32, !dbg !17
+  %udiv = udiv i32 %conv, 4, !dbg !17
+  call void @llvm.dbg.value(metadata i32 %udiv, metadata !18, metadata !DIExpression()), !dbg !19
+  call void @llvm.dbg.value(metadata i32 -8, metadata !20, metadata !DIExpression()), !dbg !19
+  call void @llvm.dbg.value(metadata i32 %udiv, metadata !20, metadata !DIExpression()), !dbg !19
+  store i8 0, i8* @b, align 1, !dbg !21
+  %cmp = icmp sgt i32 %conv, 0, !dbg !22
+  %conv1 = zext i1 %cmp to i32, !dbg !22
+  ret i32 0, !dbg !23
+}
+
+declare void @llvm.dbg.value(metadata, metadata, metadata) 
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!9, !10, !11}
+!llvm.ident = !{!12}
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "a", scope: !2, file: !3, line: 1, type: !8, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 10.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5, nameTableKind: None)
+!3 = !DIFile(filename: "test", directory: "n")
+!4 = !{}
+!5 = !{!0, !6}
+!6 = !DIGlobalVariableExpression(var: !7, expr: !DIExpression())
+!7 = distinct !DIGlobalVariable(name: "b", scope: !2, file: !3, line: 1, type: !8, isLocal: false, isDefinition: true)
+!8 = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_signed_char)
+!9 = !{i32 2, !"Dwarf Version", i32 4}
+!10 = !{i32 2, !"Debug Info Version", i32 3}
+!11 = !{i32 1, !"wchar_size", i32 4}
+!12 = !{!"clang version 10.0.0"}
+!13 = distinct !DISubprogram(name: "main", scope: !3, file: !3, line: 2, type: !14, scopeLine: 2, spFlags: DISPFlagDefinition, unit: !2, retainedNodes: !4)
+!14 = !DISubroutineType(types: !15)
+!15 = !{!16}
+!16 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!17 = !DILocation(line: 4, column: 11, scope: !13)
+!18 = !DILocalVariable(name: "c", scope: !13, file: !3, line: 4, type: !16)
+!19 = !DILocation(line: 0, scope: !13)
+!20 = !DILocalVariable(name: "l_1240", scope: !13, file: !3, line: 6, type: !16)
+!21 = !DILocation(line: 10, column: 10, scope: !13)
+!22 = !DILocation(line: 10, column: 5, scope: !13)
+!23 = !DILocation(line: 12, column: 1, scope: !13)
diff --git a/llvm/test/Transforms/InstCombine/pr44242.ll b/llvm/test/Transforms/InstCombine/pr44242.ll
new file mode 100644
index 000000000..5e783af73
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/pr44242.ll
@@ -0,0 +1,190 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+; Check that we don't create two redundant phi nodes when %val is used in a
+; form where we can't rewrite it in terms of the new phi node.
+
+; Use %val in an instruction type not supported by optimizeBitCastFromPhi.
+define float @sitofp(float %x) {
+; CHECK-LABEL: @sitofp(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP_HEADER:%.*]]
+; CHECK:       loop_header:
+; CHECK-NEXT:    [[VAL:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[VAL_INCR_CASTED:%.*]], [[LOOP:%.*]] ]
+; CHECK-NEXT:    [[VAL_CASTED:%.*]] = bitcast i32 [[VAL]] to float
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp ogt float [[VAL_CASTED]], [[X:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[END:%.*]], label [[LOOP]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[VAL_INCR:%.*]] = fadd float [[VAL_CASTED]], 1.000000e+00
+; CHECK-NEXT:    [[VAL_INCR_CASTED]] = bitcast float [[VAL_INCR]] to i32
+; CHECK-NEXT:    br label [[LOOP_HEADER]]
+; CHECK:       end:
+; CHECK-NEXT:    [[RESULT:%.*]] = sitofp i32 [[VAL]] to float
+; CHECK-NEXT:    ret float [[RESULT]]
+;
+entry:
+  br label %loop_header
+loop_header:
+  %val = phi i32 [ 0, %entry ], [ %val_incr_casted, %loop ]
+  %val_casted = bitcast i32 %val to float
+  %cmp = fcmp ogt float %val_casted, %x
+  br i1 %cmp, label %end, label %loop
+loop:
+  %val_incr = fadd float %val_casted, 1.0
+  %val_incr_casted = bitcast float %val_incr to i32
+  br label %loop_header
+end:
+  %result = sitofp i32 %val to float
+  ret float %result
+}
+
+; Use %val in an incompatible bitcast.
+define <2 x i16> @bitcast(float %x) {
+; CHECK-LABEL: @bitcast(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP_HEADER:%.*]]
+; CHECK:       loop_header:
+; CHECK-NEXT:    [[VAL:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[VAL_INCR_CASTED:%.*]], [[LOOP:%.*]] ]
+; CHECK-NEXT:    [[VAL_CASTED:%.*]] = bitcast i32 [[VAL]] to float
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp ogt float [[VAL_CASTED]], [[X:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[END:%.*]], label [[LOOP]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[VAL_INCR:%.*]] = fadd float [[VAL_CASTED]], 1.000000e+00
+; CHECK-NEXT:    [[VAL_INCR_CASTED]] = bitcast float [[VAL_INCR]] to i32
+; CHECK-NEXT:    br label [[LOOP_HEADER]]
+; CHECK:       end:
+; CHECK-NEXT:    [[RESULT:%.*]] = bitcast i32 [[VAL]] to <2 x i16>
+; CHECK-NEXT:    ret <2 x i16> [[RESULT]]
+;
+entry:
+  br label %loop_header
+loop_header:
+  %val = phi i32 [ 0, %entry ], [ %val_incr_casted, %loop ]
+  %val_casted = bitcast i32 %val to float
+  %cmp = fcmp ogt float %val_casted, %x
+  br i1 %cmp, label %end, label %loop
+loop:
+  %val_incr = fadd float %val_casted, 1.0
+  %val_incr_casted = bitcast float %val_incr to i32
+  br label %loop_header
+end:
+  %result = bitcast i32 %val to <2 x i16>
+  ret <2 x i16> %result
+}
+
+@global = global i32 0
+
+; Use %val with a volatile store.
+define void @store_volatile(float %x) {
+; CHECK-LABEL: @store_volatile(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP_HEADER:%.*]]
+; CHECK:       loop_header:
+; CHECK-NEXT:    [[VAL:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[VAL_INCR_CASTED:%.*]], [[LOOP:%.*]] ]
+; CHECK-NEXT:    [[VAL_CASTED:%.*]] = bitcast i32 [[VAL]] to float
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp ogt float [[VAL_CASTED]], [[X:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[END:%.*]], label [[LOOP]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[VAL_INCR:%.*]] = fadd float [[VAL_CASTED]], 1.000000e+00
+; CHECK-NEXT:    [[VAL_INCR_CASTED]] = bitcast float [[VAL_INCR]] to i32
+; CHECK-NEXT:    br label [[LOOP_HEADER]]
+; CHECK:       end:
+; CHECK-NEXT:    store volatile i32 [[VAL]], i32* @global, align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop_header
+loop_header:
+  %val = phi i32 [ 0, %entry ], [ %val_incr_casted, %loop ]
+  %val_casted = bitcast i32 %val to float
+  %cmp = fcmp ogt float %val_casted, %x
+  br i1 %cmp, label %end, label %loop
+loop:
+  %val_incr = fadd float %val_casted, 1.0
+  %val_incr_casted = bitcast float %val_incr to i32
+  br label %loop_header
+end:
+  store volatile i32 %val, i32* @global
+  ret void
+}
+
+; Use %val with a store where it's actually the address.
+define void @store_address(i32 %x) {
+; CHECK-LABEL: @store_address(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP_HEADER:%.*]]
+; CHECK:       loop_header:
+; CHECK-NEXT:    [[VAL:%.*]] = phi i32* [ @global, [[ENTRY:%.*]] ], [ [[VAL_INCR1:%.*]], [[LOOP:%.*]] ]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[END:%.*]], label [[LOOP]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[VAL_INCR1]] = getelementptr i32, i32* [[VAL]], i64 1
+; CHECK-NEXT:    br label [[LOOP_HEADER]]
+; CHECK:       end:
+; CHECK-NEXT:    store i32 0, i32* [[VAL]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop_header
+loop_header:
+  %val = phi i32* [ @global, %entry ], [ %val_incr_casted, %loop ]
+  %i = phi i32 [ 0, %entry ], [ %i_incr, %loop ]
+  %val_casted = bitcast i32* %val to float*
+  %cmp = icmp sgt i32 %i, %x
+  br i1 %cmp, label %end, label %loop
+loop:
+  %i_incr = add i32 %i, 0
+  %val_incr = getelementptr float, float* %val_casted, i32 1
+  %val_incr_casted = bitcast float* %val_incr to i32*
+  br label %loop_header
+end:
+  store i32 0, i32* %val
+  ret void
+}
+
+; Test where a phi (%val2) other than the original one (%val) has an
+; incompatible use.
+define i32 @multiple_phis(float %x) {
+; CHECK-LABEL: @multiple_phis(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP_HEADER:%.*]]
+; CHECK:       loop_header:
+; CHECK-NEXT:    [[VAL:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[VAL2:%.*]], [[LOOP_END:%.*]] ]
+; CHECK-NEXT:    [[VAL_CASTED:%.*]] = bitcast i32 [[VAL]] to float
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp ogt float [[VAL_CASTED]], [[X:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[END:%.*]], label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[CMP2:%.*]] = fcmp ogt float [[VAL_CASTED]], 2.000000e+00
+; CHECK-NEXT:    br i1 [[CMP2]], label [[IF:%.*]], label [[LOOP_END]]
+; CHECK:       if:
+; CHECK-NEXT:    [[VAL_INCR:%.*]] = fadd float [[VAL_CASTED]], 1.000000e+00
+; CHECK-NEXT:    [[VAL_INCR_CASTED:%.*]] = bitcast float [[VAL_INCR]] to i32
+; CHECK-NEXT:    br label [[LOOP_END]]
+; CHECK:       loop_end:
+; CHECK-NEXT:    [[VAL2]] = phi i32 [ [[VAL]], [[LOOP]] ], [ [[VAL_INCR_CASTED]], [[IF]] ]
+; CHECK-NEXT:    store volatile i32 [[VAL2]], i32* @global, align 4
+; CHECK-NEXT:    br label [[LOOP_HEADER]]
+; CHECK:       end:
+; CHECK-NEXT:    ret i32 [[VAL]]
+;
+entry:
+  br label %loop_header
+loop_header:
+  %val = phi i32 [ 0, %entry ], [ %val2, %loop_end ]
+  %val_casted = bitcast i32 %val to float
+  %cmp = fcmp ogt float %val_casted, %x
+  br i1 %cmp, label %end, label %loop
+loop:
+  %cmp2 = fcmp ogt float %val_casted, 2.0
+  br i1 %cmp2, label %if, label %loop_end
+if:
+  %val_incr = fadd float %val_casted, 1.0
+  %val_incr_casted = bitcast float %val_incr to i32
+  br label %loop_end
+loop_end:
+  %val2 = phi i32 [ %val, %loop ], [ %val_incr_casted, %if ]
+  store volatile i32 %val2, i32* @global ; the incompatible use
+  br label %loop_header
+end:
+  ret i32 %val
+}
diff --git a/llvm/test/Transforms/InstCombine/pr44245.ll b/llvm/test/Transforms/InstCombine/pr44245.ll
new file mode 100644
index 000000000..f75e26e87
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/pr44245.ll
@@ -0,0 +1,192 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -instcombine -instcombine-infinite-loop-threshold=2 < %s | FileCheck %s
+
+; This used to cause on infinite instcombine loop.
+
+define void @test(i1 %c) {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:  bb16:
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[BB17:%.*]], label [[BB24:%.*]]
+; CHECK:       bb17:
+; CHECK-NEXT:    [[TMP0:%.*]] = phi i8* [ [[TMP1:%.*]], [[BB47:%.*]] ], [ undef, [[BB16:%.*]] ]
+; CHECK-NEXT:    store i8* [[TMP0]], i8** undef, align 8
+; CHECK-NEXT:    ret void
+; CHECK:       bb24:
+; CHECK-NEXT:    br i1 [[C]], label [[BB44:%.*]], label [[BB49:%.*]]
+; CHECK:       bb44:
+; CHECK-NEXT:    [[TMP467:%.*]] = load i8*, i8** inttoptr (i64 16 to i8**), align 16
+; CHECK-NEXT:    br label [[BB47]]
+; CHECK:       bb47:
+; CHECK-NEXT:    [[TMP1]] = phi i8* [ [[TMP2:%.*]], [[BB150:%.*]] ], [ [[TMP1221:%.*]], [[BB119:%.*]] ], [ [[TMP1032:%.*]], [[BB101:%.*]] ], [ [[TMP933:%.*]], [[BB91:%.*]] ], [ [[TMP834:%.*]], [[BB81:%.*]] ], [ [[TMP705:%.*]], [[BB67:%.*]] ], [ [[TMP586:%.*]], [[BB56:%.*]] ], [ [[TMP467]], [[BB44]] ]
+; CHECK-NEXT:    br label [[BB17]]
+; CHECK:       bb49:
+; CHECK-NEXT:    br i1 [[C]], label [[BB56]], label [[BB59:%.*]]
+; CHECK:       bb56:
+; CHECK-NEXT:    [[TMP586]] = load i8*, i8** inttoptr (i64 16 to i8**), align 16
+; CHECK-NEXT:    br label [[BB47]]
+; CHECK:       bb59:
+; CHECK-NEXT:    br i1 [[C]], label [[BB67]], label [[BB71:%.*]]
+; CHECK:       bb67:
+; CHECK-NEXT:    [[TMP705]] = load i8*, i8** inttoptr (i64 16 to i8**), align 16
+; CHECK-NEXT:    br label [[BB47]]
+; CHECK:       bb71:
+; CHECK-NEXT:    br i1 [[C]], label [[BB81]], label [[BB84:%.*]]
+; CHECK:       bb81:
+; CHECK-NEXT:    [[TMP834]] = load i8*, i8** inttoptr (i64 16 to i8**), align 16
+; CHECK-NEXT:    br label [[BB47]]
+; CHECK:       bb84:
+; CHECK-NEXT:    br i1 [[C]], label [[BB91]], label [[BB94:%.*]]
+; CHECK:       bb91:
+; CHECK-NEXT:    [[TMP933]] = load i8*, i8** inttoptr (i64 16 to i8**), align 16
+; CHECK-NEXT:    br label [[BB47]]
+; CHECK:       bb94:
+; CHECK-NEXT:    br i1 [[C]], label [[BB101]], label [[BB104:%.*]]
+; CHECK:       bb101:
+; CHECK-NEXT:    [[TMP1032]] = load i8*, i8** inttoptr (i64 16 to i8**), align 16
+; CHECK-NEXT:    br label [[BB47]]
+; CHECK:       bb104:
+; CHECK-NEXT:    br i1 [[C]], label [[BB119]], label [[BB123:%.*]]
+; CHECK:       bb119:
+; CHECK-NEXT:    [[TMP1221]] = load i8*, i8** inttoptr (i64 16 to i8**), align 16
+; CHECK-NEXT:    br label [[BB47]]
+; CHECK:       bb123:
+; CHECK-NEXT:    br i1 [[C]], label [[BB147:%.*]], label [[BB152:%.*]]
+; CHECK:       bb147:
+; CHECK-NEXT:    [[TMP1499:%.*]] = load i8*, i8** inttoptr (i64 16 to i8**), align 16
+; CHECK-NEXT:    br label [[BB150]]
+; CHECK:       bb150:
+; CHECK-NEXT:    [[TMP2]] = phi i8* [ [[TMP1848:%.*]], [[BB152]] ], [ [[TMP1499]], [[BB147]] ]
+; CHECK-NEXT:    br label [[BB47]]
+; CHECK:       bb152:
+; CHECK-NEXT:    [[TMP1848]] = load i8*, i8** inttoptr (i64 16 to i8**), align 16
+; CHECK-NEXT:    call void undef()
+; CHECK-NEXT:    br label [[BB150]]
+;
+bb16:                                             ; preds = %bb
+  br i1 %c, label %bb17, label %bb24
+
+bb17:                                             ; preds = %bb47, %bb17
+  %0 = phi i8* [ %1, %bb47 ], [ undef, %bb16 ]
+  store i8* %0, i8** undef, align 8
+  ret void
+
+bb24:                                             ; preds = %bb24
+  br i1 %c, label %bb44, label %bb49
+
+bb44:                                             ; preds = %bb43
+  %tmp46 = load i64*, i64** inttoptr (i64 16 to i64**), align 16
+  br label %bb47
+
+bb47:                                             ; preds = %bb150, %bb119, %bb101, %bb91, %bb81, %bb67, %bb56, %bb44
+  %.in1 = phi i64* [ %.in, %bb150 ], [ %tmp122, %bb119 ], [ %tmp103, %bb101 ], [ %tmp93, %bb91 ], [ %tmp83, %bb81 ], [ %tmp70, %bb67 ], [ %tmp58, %bb56 ], [ %tmp46, %bb44 ]
+  %1 = bitcast i64* %.in1 to i8*
+  br label %bb17
+
+bb49:                                             ; preds = %bb49
+  br i1 %c, label %bb56, label %bb59
+
+bb56:                                             ; preds = %bb55
+  %tmp58 = load i64*, i64** inttoptr (i64 16 to i64**), align 16
+  br label %bb47
+
+bb59:                                             ; preds = %bb59
+  br i1 %c, label %bb67, label %bb71
+
+bb67:                                             ; preds = %bb66
+  %tmp70 = load i64*, i64** inttoptr (i64 16 to i64**), align 16
+  br label %bb47
+
+bb71:                                             ; preds = %bb71
+  br i1 %c, label %bb81, label %bb84
+
+bb81:                                             ; preds = %bb80
+  %tmp83 = load i64*, i64** inttoptr (i64 16 to i64**), align 16
+  br label %bb47
+
+bb84:                                             ; preds = %bb84
+  br i1 %c, label %bb91, label %bb94
+
+bb91:                                             ; preds = %bb90
+  %tmp93 = load i64*, i64** inttoptr (i64 16 to i64**), align 16
+  br label %bb47
+
+bb94:                                            ; preds = %bb94
+  br i1 %c, label %bb101, label %bb104
+
+bb101:                                            ; preds = %bb100
+  %tmp103 = load i64*, i64** inttoptr (i64 16 to i64**), align 16
+  br label %bb47
+
+bb104:                                            ; preds = %bb104
+  br i1 %c, label %bb119, label %bb123
+
+bb119:                                            ; preds = %bb118
+  %tmp122 = load i64*, i64** inttoptr (i64 16 to i64**), align 16
+  br label %bb47
+
+bb123:                                            ; preds = %bb123
+  br i1 %c, label %bb147, label %bb152
+
+bb147:                                            ; preds = %bb146
+  %tmp149 = load i64*, i64** inttoptr (i64 16 to i64**), align 16
+  br label %bb150
+
+bb150:                                            ; preds = %bb152, %bb147
+  %.in = phi i64* [ %tmp184, %bb152 ], [ %tmp149, %bb147 ]
+  br label %bb47
+
+bb152:                                            ; preds = %bb146
+  %tmp184 = load i64*, i64** inttoptr (i64 16 to i64**), align 16
+  call void undef()
+  br label %bb150
+}
+
+; This used to cause an instcombine loop when the problem above was
+; addressed in a non-robust fashion.
+
+%type_1 = type {}
+%type_2 = type {}
+%type_3 = type {}
+
+define void @test_2(i1 %c) local_unnamed_addr {
+; CHECK-LABEL: @test_2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[WHILE_COND:%.*]]
+; CHECK:       while.cond:
+; CHECK-NEXT:    br label [[FOR_COND:%.*]]
+; CHECK:       for.cond:
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[COND_TRUE133:%.*]], label [[COND_FALSE138:%.*]]
+; CHECK:       cond.true133:
+; CHECK-NEXT:    store %type_3* undef, %type_3** null, align 536870912
+; CHECK-NEXT:    br label [[COND_END144:%.*]]
+; CHECK:       cond.false138:
+; CHECK-NEXT:    store %type_3* undef, %type_3** null, align 536870912
+; CHECK-NEXT:    br label [[COND_END144]]
+; CHECK:       cond.end144:
+; CHECK-NEXT:    br label [[WHILE_COND]]
+;
+entry:
+  br label %while.cond
+
+while.cond:                                       ; preds = %cond.end144, %entry
+  %link.0 = phi %type_2* [ undef, %entry ], [ %cond145, %cond.end144 ]
+  %os115 = bitcast %type_2* %link.0 to %type_3*
+  %ou116 = getelementptr inbounds %type_3, %type_3* %os115, i32 0
+  %os1117 = bitcast %type_3* %ou116 to %type_1*
+  br label %for.cond
+
+for.cond:                                         ; preds = %while.cond
+  br i1 %c, label %cond.true133, label %cond.false138
+
+cond.true133:                                     ; preds = %sw.epilog
+  %0 = load %type_2*, %type_2** undef, align 8
+  br label %cond.end144
+
+cond.false138:                                    ; preds = %sw.epilog
+  %1 = load %type_2*, %type_2** undef, align 8
+  br label %cond.end144
+
+cond.end144:                                      ; preds = %cond.false138, %cond.true133
+  %cond145 = phi %type_2* [ %0, %cond.true133 ], [ %1, %cond.false138 ]
+  br label %while.cond
+}
diff --git a/llvm/test/Transforms/InstCombine/pr44541.ll b/llvm/test/Transforms/InstCombine/pr44541.ll
new file mode 100644
index 000000000..782526a10
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/pr44541.ll
@@ -0,0 +1,25 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -instcombine -expensive-combines=0 -instcombine-infinite-loop-threshold=3 < %s | FileCheck %s
+
+; This test used to cause an infinite combine loop.
+
+define i16 @passthru(i16 returned %x) {
+; CHECK-LABEL: @passthru(
+; CHECK-NEXT:    ret i16 [[X:%.*]]
+;
+  ret i16 %x
+}
+
+define i16 @test(i16 %arg) {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:    [[ZERO:%.*]] = call i16 @passthru(i16 0)
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i16 [[ARG:%.*]], 0
+; CHECK-NEXT:    [[RET:%.*]] = select i1 [[TMP1]], i16 [[ARG]], i16 0
+; CHECK-NEXT:    ret i16 [[RET]]
+;
+  %zero = call i16 @passthru(i16 0)
+  %sub = sub nuw nsw i16 %arg, %zero
+  %cmp = icmp slt i16 %sub, 0
+  %ret = select i1 %cmp, i16 0, i16 %sub
+  ret i16 %ret
+}
diff --git a/llvm/test/Transforms/InstCombine/pr44552.ll b/llvm/test/Transforms/InstCombine/pr44552.ll
new file mode 100644
index 000000000..adefe829d
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/pr44552.ll
@@ -0,0 +1,59 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -instcombine -instcombine-infinite-loop-threshold=2 < %s | FileCheck %s
+
+; This used to require 10 instcombine iterations to fully optimize.
+; The number of iterations grew linearly with the number of DSEd stores,
+; resulting in overall quadratic runtime.
+
+%struct.S3 = type { i64 }
+
+@csmith_sink_ = dso_local global i64 0, align 1
+@g_302_7 = internal constant i32 0, align 1
+@g_313_0 = internal global i16 0, align 1
+@g_313_1 = internal global i32 0, align 1
+@g_313_2 = internal global i32 0, align 1
+@g_313_3 = internal global i32 0, align 1
+@g_313_4 = internal global i16 0, align 1
+@g_313_5 = internal global i16 0, align 1
+@g_313_6 = internal global i16 0, align 1
+@g_316 = internal global %struct.S3 zeroinitializer, align 1
+@g_316_1_0 = internal global i16 0, align 1
+
+define i16 @main() {
+; CHECK-LABEL: @main(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    store i64 0, i64* @csmith_sink_, align 8
+; CHECK-NEXT:    ret i16 0
+;
+entry:
+  store i64 0, i64* @csmith_sink_, align 1
+  %0 = load i16, i16* @g_313_0, align 1
+  %conv2 = sext i16 %0 to i64
+  store i64 %conv2, i64* @csmith_sink_, align 1
+  %1 = load i32, i32* @g_313_1, align 1
+  %conv3 = zext i32 %1 to i64
+  store i64 %conv3, i64* @csmith_sink_, align 1
+  %2 = load i32, i32* @g_313_2, align 1
+  %conv4 = sext i32 %2 to i64
+  store i64 %conv4, i64* @csmith_sink_, align 1
+  %3 = load i32, i32* @g_313_3, align 1
+  %conv5 = zext i32 %3 to i64
+  store i64 %conv5, i64* @csmith_sink_, align 1
+  %4 = load i16, i16* @g_313_4, align 1
+  %conv6 = sext i16 %4 to i64
+  store i64 %conv6, i64* @csmith_sink_, align 1
+  %5 = load i16, i16* @g_313_5, align 1
+  %conv7 = sext i16 %5 to i64
+  store i64 %conv7, i64* @csmith_sink_, align 1
+  %6 = load i16, i16* @g_313_6, align 1
+  %conv8 = sext i16 %6 to i64
+  store i64 %conv8, i64* @csmith_sink_, align 1
+  %7 = load i64, i64* getelementptr inbounds (%struct.S3, %struct.S3* @g_316, i32 0, i32 0), align 1
+  store i64 %7, i64* @csmith_sink_, align 1
+  %8 = load i16, i16* @g_316_1_0, align 1
+  %conv9 = sext i16 %8 to i64
+  store i64 %conv9, i64* @csmith_sink_, align 1
+  store i64 0, i64* @csmith_sink_, align 1
+  ret i16 0
+}
+
diff --git a/llvm/test/Transforms/InstCombine/pr44835.ll b/llvm/test/Transforms/InstCombine/pr44835.ll
new file mode 100644
index 000000000..46c9fca6b
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/pr44835.ll
@@ -0,0 +1,29 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+; This test used to cause an infinite loop in the load/store min/max bitcast
+; transform.
+
+define void @test(i32* %p, i32* %p2) {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:    [[V:%.*]] = load i32, i32* [[P:%.*]], align 4
+; CHECK-NEXT:    [[V2:%.*]] = load i32, i32* [[P2:%.*]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[V2]], [[V]]
+; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[CMP]], i32 [[V2]], i32 [[V]]
+; CHECK-NEXT:    store i32 [[TMP1]], i32* [[P]], align 4
+; CHECK-NEXT:    ret void
+;
+  %v = load i32, i32* %p, align 4
+  %v2 = load i32, i32* %p2, align 4
+  %cmp = icmp ult i32 %v2, %v
+  %sel = select i1 %cmp, i32* %p2, i32* %p
+  %p8 = bitcast i32* %p to i8*
+  %sel8 = bitcast i32* %sel to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %p8, i8* align 4 %sel8, i64 4, i1 false)
+  ret void
+}
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* noalias nocapture writeonly, i8* noalias nocapture readonly, i64, i1 immarg) #0
+
+attributes #0 = { argmemonly nounwind willreturn }
diff --git a/llvm/test/Transforms/InstCombine/printf-1.ll b/llvm/test/Transforms/InstCombine/printf-1.ll
index 9d13b360f..6249ca5b1 100644
--- a/llvm/test/Transforms/InstCombine/printf-1.ll
+++ b/llvm/test/Transforms/InstCombine/printf-1.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; Test that the printf library call simplifier works correctly.
 ;
 ; RUN: opt < %s -instcombine -S | FileCheck %s
@@ -22,110 +23,164 @@ declare i32 @printf(i8*, ...)
 
 define void @test_simplify1() {
 ; CHECK-LABEL: @test_simplify1(
+; CHECK-NEXT:    ret void
+;
+; CHECK-IPRINTF-LABEL: @test_simplify1(
+; CHECK-IPRINTF-NEXT:    ret void
+;
   %fmt = getelementptr [1 x i8], [1 x i8]* @empty, i32 0, i32 0
   call i32 (i8*, ...) @printf(i8* %fmt)
   ret void
-; CHECK-NEXT: ret void
 }
 
 ; Check printf("x") -> putchar('x'), even for '%'.
 
 define void @test_simplify2() {
 ; CHECK-LABEL: @test_simplify2(
+; CHECK-NEXT:    [[PUTCHAR:%.*]] = call i32 @putchar(i32 104)
+; CHECK-NEXT:    ret void
+;
+; CHECK-IPRINTF-LABEL: @test_simplify2(
+; CHECK-IPRINTF-NEXT:    [[PUTCHAR:%.*]] = call i32 @putchar(i32 104)
+; CHECK-IPRINTF-NEXT:    ret void
+;
   %fmt = getelementptr [2 x i8], [2 x i8]* @h, i32 0, i32 0
   call i32 (i8*, ...) @printf(i8* %fmt)
-; CHECK-NEXT: call i32 @putchar(i32 104)
   ret void
-; CHECK-NEXT: ret void
 }
 
 ; Special case: printf("%%") -> putchar('%').
 
 define void @test_simplify2b() {
 ; CHECK-LABEL: @test_simplify2b(
+; CHECK-NEXT:    [[PUTCHAR:%.*]] = call i32 @putchar(i32 37)
+; CHECK-NEXT:    ret void
+;
+; CHECK-IPRINTF-LABEL: @test_simplify2b(
+; CHECK-IPRINTF-NEXT:    [[PUTCHAR:%.*]] = call i32 @putchar(i32 37)
+; CHECK-IPRINTF-NEXT:    ret void
+;
   %fmt = getelementptr [3 x i8], [3 x i8]* @h2, i32 0, i32 0
   call i32 (i8*, ...) @printf(i8* %fmt)
-; CHECK-NEXT: call i32 @putchar(i32 37)
   ret void
-; CHECK-NEXT: ret void
 }
 
 define void @test_simplify3() {
 ; CHECK-LABEL: @test_simplify3(
+; CHECK-NEXT:    [[PUTCHAR:%.*]] = call i32 @putchar(i32 37)
+; CHECK-NEXT:    ret void
+;
+; CHECK-IPRINTF-LABEL: @test_simplify3(
+; CHECK-IPRINTF-NEXT:    [[PUTCHAR:%.*]] = call i32 @putchar(i32 37)
+; CHECK-IPRINTF-NEXT:    ret void
+;
   %fmt = getelementptr [2 x i8], [2 x i8]* @percent, i32 0, i32 0
   call i32 (i8*, ...) @printf(i8* %fmt)
-; CHECK-NEXT: call i32 @putchar(i32 37)
   ret void
-; CHECK-NEXT: ret void
 }
 
 ; Check printf("foo\n") -> puts("foo").
 
 define void @test_simplify4() {
 ; CHECK-LABEL: @test_simplify4(
+; CHECK-NEXT:    [[PUTS:%.*]] = call i32 @puts(i8* nonnull dereferenceable(1) getelementptr inbounds ([12 x i8], [12 x i8]* @str, i32 0, i32 0))
+; CHECK-NEXT:    ret void
+;
+; CHECK-IPRINTF-LABEL: @test_simplify4(
+; CHECK-IPRINTF-NEXT:    [[PUTS:%.*]] = call i32 @puts(i8* nonnull dereferenceable(1) getelementptr inbounds ([12 x i8], [12 x i8]* @str, i32 0, i32 0))
+; CHECK-IPRINTF-NEXT:    ret void
+;
   %fmt = getelementptr [13 x i8], [13 x i8]* @hello_world, i32 0, i32 0
   call i32 (i8*, ...) @printf(i8* %fmt)
-; CHECK-NEXT: call i32 @puts(i8* getelementptr inbounds ([12 x i8], [12 x i8]* [[$STR]], i32 0, i32 0))
   ret void
-; CHECK-NEXT: ret void
 }
 
 ; Check printf("%c", chr) -> putchar(chr).
 
 define void @test_simplify5() {
 ; CHECK-LABEL: @test_simplify5(
+; CHECK-NEXT:    [[PUTCHAR:%.*]] = call i32 @putchar(i32 104)
+; CHECK-NEXT:    ret void
+;
+; CHECK-IPRINTF-LABEL: @test_simplify5(
+; CHECK-IPRINTF-NEXT:    [[PUTCHAR:%.*]] = call i32 @putchar(i32 104)
+; CHECK-IPRINTF-NEXT:    ret void
+;
   %fmt = getelementptr [3 x i8], [3 x i8]* @percent_c, i32 0, i32 0
   call i32 (i8*, ...) @printf(i8* %fmt, i8 104)
-; CHECK-NEXT: call i32 @putchar(i32 104)
   ret void
-; CHECK-NEXT: ret void
 }
 
 ; Check printf("%s\n", str) -> puts(str).
 
 define void @test_simplify6() {
 ; CHECK-LABEL: @test_simplify6(
+; CHECK-NEXT:    [[PUTS:%.*]] = call i32 @puts(i8* nonnull dereferenceable(1) getelementptr inbounds ([13 x i8], [13 x i8]* @hello_world, i32 0, i32 0))
+; CHECK-NEXT:    ret void
+;
+; CHECK-IPRINTF-LABEL: @test_simplify6(
+; CHECK-IPRINTF-NEXT:    [[PUTS:%.*]] = call i32 @puts(i8* nonnull dereferenceable(1) getelementptr inbounds ([13 x i8], [13 x i8]* @hello_world, i32 0, i32 0))
+; CHECK-IPRINTF-NEXT:    ret void
+;
   %fmt = getelementptr [4 x i8], [4 x i8]* @percent_s, i32 0, i32 0
   %str = getelementptr [13 x i8], [13 x i8]* @hello_world, i32 0, i32 0
   call i32 (i8*, ...) @printf(i8* %fmt, i8* %str)
-; CHECK-NEXT: call i32 @puts(i8* getelementptr inbounds ([13 x i8], [13 x i8]* @hello_world, i32 0, i32 0))
   ret void
-; CHECK-NEXT: ret void
 }
 
 ; Check printf(format, ...) -> iprintf(format, ...) if no floating point.
 
 define void @test_simplify7() {
+; CHECK-LABEL: @test_simplify7(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 (i8*, ...) @printf(i8* nonnull dereferenceable(1) getelementptr inbounds ([3 x i8], [3 x i8]* @percent_d, i32 0, i32 0), i32 187)
+; CHECK-NEXT:    ret void
+;
 ; CHECK-IPRINTF-LABEL: @test_simplify7(
+; CHECK-IPRINTF-NEXT:    [[TMP1:%.*]] = call i32 (i8*, ...) @iprintf(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @percent_d, i32 0, i32 0), i32 187)
+; CHECK-IPRINTF-NEXT:    ret void
+;
   %fmt = getelementptr [3 x i8], [3 x i8]* @percent_d, i32 0, i32 0
   call i32 (i8*, ...) @printf(i8* %fmt, i32 187)
-; CHECK-IPRINTF-NEXT: call i32 (i8*, ...) @iprintf(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @percent_d, i32 0, i32 0), i32 187)
   ret void
-; CHECK-IPRINTF-NEXT: ret void
 }
 
 define void @test_no_simplify1() {
+; CHECK-LABEL: @test_no_simplify1(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 (i8*, ...) @printf(i8* nonnull dereferenceable(1) getelementptr inbounds ([3 x i8], [3 x i8]* @percent_f, i32 0, i32 0), double 1.870000e+00)
+; CHECK-NEXT:    ret void
+;
 ; CHECK-IPRINTF-LABEL: @test_no_simplify1(
+; CHECK-IPRINTF-NEXT:    [[TMP1:%.*]] = call i32 (i8*, ...) @printf(i8* nonnull dereferenceable(1) getelementptr inbounds ([3 x i8], [3 x i8]* @percent_f, i32 0, i32 0), double 1.870000e+00)
+; CHECK-IPRINTF-NEXT:    ret void
+;
   %fmt = getelementptr [3 x i8], [3 x i8]* @percent_f, i32 0, i32 0
   call i32 (i8*, ...) @printf(i8* %fmt, double 1.87)
-; CHECK-IPRINTF-NEXT: call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @percent_f, i32 0, i32 0), double 1.870000e+00)
   ret void
-; CHECK-IPRINTF-NEXT: ret void
 }
 
 define void @test_no_simplify2(i8* %fmt, double %d) {
 ; CHECK-LABEL: @test_no_simplify2(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 (i8*, ...) @printf(i8* nonnull dereferenceable(1) [[FMT:%.*]], double [[D:%.*]])
+; CHECK-NEXT:    ret void
+;
+; CHECK-IPRINTF-LABEL: @test_no_simplify2(
+; CHECK-IPRINTF-NEXT:    [[TMP1:%.*]] = call i32 (i8*, ...) @printf(i8* nonnull dereferenceable(1) [[FMT:%.*]], double [[D:%.*]])
+; CHECK-IPRINTF-NEXT:    ret void
+;
   call i32 (i8*, ...) @printf(i8* %fmt, double %d)
-; CHECK-NEXT: call i32 (i8*, ...) @printf(i8* %fmt, double %d)
   ret void
-; CHECK-NEXT: ret void
 }
 
 define i32 @test_no_simplify3() {
 ; CHECK-LABEL: @test_no_simplify3(
+; CHECK-NEXT:    [[RET:%.*]] = call i32 (i8*, ...) @printf(i8* nonnull dereferenceable(1) getelementptr inbounds ([2 x i8], [2 x i8]* @h, i32 0, i32 0))
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+; CHECK-IPRINTF-LABEL: @test_no_simplify3(
+; CHECK-IPRINTF-NEXT:    [[TMP1:%.*]] = call i32 (i8*, ...) @iprintf(i8* getelementptr inbounds ([2 x i8], [2 x i8]* @h, i32 0, i32 0))
+; CHECK-IPRINTF-NEXT:    ret i32 [[TMP1]]
+;
   %fmt = getelementptr [2 x i8], [2 x i8]* @h, i32 0, i32 0
   %ret = call i32 (i8*, ...) @printf(i8* %fmt)
-; CHECK-NEXT: call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([2 x i8], [2 x i8]* @h, i32 0, i32 0))
   ret i32 %ret
-; CHECK-NEXT: ret i32 %ret
 }
diff --git a/llvm/test/Transforms/InstCombine/printf-2.ll b/llvm/test/Transforms/InstCombine/printf-2.ll
index fbd5b1bb9..66678bc55 100644
--- a/llvm/test/Transforms/InstCombine/printf-2.ll
+++ b/llvm/test/Transforms/InstCombine/printf-2.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; Test that the printf library call simplifier works correctly.
 ;
 ; RUN: opt < %s -instcombine -S | FileCheck %s
@@ -16,38 +17,42 @@ declare void @printf(i8*, ...)
 
 define void @test_simplify1() {
 ; CHECK-LABEL: @test_simplify1(
+; CHECK-NEXT:    [[PUTCHAR:%.*]] = call i32 @putchar(i32 104)
+; CHECK-NEXT:    ret void
+;
   %fmt = getelementptr [2 x i8], [2 x i8]* @h, i32 0, i32 0
   call void (i8*, ...) @printf(i8* %fmt)
-; CHECK-NEXT: call i32 @putchar(i32 104)
   ret void
-; CHECK-NEXT: ret void
 }
 
 define void @test_simplify2() {
 ; CHECK-LABEL: @test_simplify2(
+; CHECK-NEXT:    [[PUTS:%.*]] = call i32 @puts(i8* nonnull dereferenceable(1) getelementptr inbounds ([12 x i8], [12 x i8]* @str, i32 0, i32 0))
+; CHECK-NEXT:    ret void
+;
   %fmt = getelementptr [13 x i8], [13 x i8]* @hello_world, i32 0, i32 0
   call void (i8*, ...) @printf(i8* %fmt)
-; CHECK-NEXT: call i32 @puts(i8* getelementptr inbounds ([12 x i8], [12 x i8]* @str, i32 0, i32 0))
   ret void
-; CHECK-NEXT: ret void
 }
 
 define void @test_simplify6() {
 ; CHECK-LABEL: @test_simplify6(
+; CHECK-NEXT:    [[PUTS:%.*]] = call i32 @puts(i8* nonnull dereferenceable(1) getelementptr inbounds ([13 x i8], [13 x i8]* @hello_world, i32 0, i32 0))
+; CHECK-NEXT:    ret void
+;
   %fmt = getelementptr [4 x i8], [4 x i8]* @percent_s, i32 0, i32 0
   %str = getelementptr [13 x i8], [13 x i8]* @hello_world, i32 0, i32 0
   call void (i8*, ...) @printf(i8* %fmt, i8* %str)
-; CHECK-NEXT: call i32 @puts(i8* getelementptr inbounds ([13 x i8], [13 x i8]* @hello_world, i32 0, i32 0))
   ret void
-; CHECK-NEXT: ret void
 }
 
 define void @test_simplify7() {
 ; CHECK-LABEL: @test_simplify7(
+; CHECK-NEXT:    [[PUTCHAR:%.*]] = call i32 @putchar(i32 97)
+; CHECK-NEXT:    ret void
+;
   %fmt = getelementptr [3 x i8], [3 x i8]* @format_str, i32 0, i32 0
   %str = getelementptr [2 x i8], [2 x i8]* @charstr, i32 0, i32 0
   call void (i8*, ...) @printf(i8* %fmt, i8* %str)
-; CHECK-NEXT: call i32 @putchar(i32 97)
   ret void
-; CHECK-NEXT: ret void
 }
diff --git a/llvm/test/Transforms/InstCombine/puts-1.ll b/llvm/test/Transforms/InstCombine/puts-1.ll
index 210286848..aa18a30f4 100644
--- a/llvm/test/Transforms/InstCombine/puts-1.ll
+++ b/llvm/test/Transforms/InstCombine/puts-1.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; Test that the puts library call simplifier works correctly.
 ;
 ; RUN: opt < %s -instcombine -S | FileCheck %s
@@ -12,20 +13,22 @@ declare i32 @puts(i8*)
 
 define void @test_simplify1() {
 ; CHECK-LABEL: @test_simplify1(
+; CHECK-NEXT:    [[PUTCHAR:%.*]] = call i32 @putchar(i32 10)
+; CHECK-NEXT:    ret void
+;
   %str = getelementptr [1 x i8], [1 x i8]* @empty, i32 0, i32 0
   call i32 @puts(i8* %str)
-; CHECK-NEXT: call i32 @putchar(i32 10)
   ret void
-; CHECK-NEXT: ret void
 }
 
 ; Don't simplify if the return value is used.
 
 define i32 @test_no_simplify1() {
 ; CHECK-LABEL: @test_no_simplify1(
+; CHECK-NEXT:    [[RET:%.*]] = call i32 @puts(i8* nonnull dereferenceable(1) getelementptr inbounds ([1 x i8], [1 x i8]* @empty, i32 0, i32 0))
+; CHECK-NEXT:    ret i32 [[RET]]
+;
   %str = getelementptr [1 x i8], [1 x i8]* @empty, i32 0, i32 0
   %ret = call i32 @puts(i8* %str)
-; CHECK-NEXT: call i32 @puts(i8* getelementptr inbounds ([1 x i8], [1 x i8]* @empty, i32 0, i32 0))
   ret i32 %ret
-; CHECK-NEXT: ret i32 %ret
 }
diff --git a/llvm/test/Transforms/InstCombine/realloc.ll b/llvm/test/Transforms/InstCombine/realloc.ll
index 22f37f1d2..7304b2403 100644
--- a/llvm/test/Transforms/InstCombine/realloc.ll
+++ b/llvm/test/Transforms/InstCombine/realloc.ll
@@ -7,7 +7,7 @@ declare noalias i8* @malloc(i64) #1
 
 define i8* @realloc_null_ptr() #0 {
 ; CHECK-LABEL: @realloc_null_ptr(
-; CHECK-NEXT:    [[MALLOC:%.*]] = call i8* @malloc(i64 100)
+; CHECK-NEXT:    [[MALLOC:%.*]] = call dereferenceable_or_null(100) i8* @malloc(i64 100)
 ; CHECK-NEXT:    ret i8* [[MALLOC]]
 ;
   %call = call i8* @realloc(i8* null, i64 100) #2
@@ -16,7 +16,7 @@ define i8* @realloc_null_ptr() #0 {
 
 define i8* @realloc_unknown_ptr(i8* %ptr) #0 {
 ; CHECK-LABEL: @realloc_unknown_ptr(
-; CHECK-NEXT:    [[CALL:%.*]] = call i8* @realloc(i8* [[PTR:%.*]], i64 100)
+; CHECK-NEXT:    [[CALL:%.*]] = call dereferenceable_or_null(100) i8* @realloc(i8* [[PTR:%.*]], i64 100)
 ; CHECK-NEXT:    ret i8* [[CALL]]
 ;
   %call = call i8* @realloc(i8* %ptr, i64 100) #2
diff --git a/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-after-truncation-variant-a.ll b/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-after-truncation-variant-a.ll
new file mode 100644
index 000000000..d11a5c322
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-after-truncation-variant-a.ll
@@ -0,0 +1,199 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instcombine -S | FileCheck %s
+
+; If we have some pattern that leaves only some low bits set, and then performs
+; left-shift of those bits, we can combine those two shifts into a shift+mask.
+
+; There are many variants to this pattern:
+;   a)  (trunc (x & ((1 << maskNbits) - 1))) << shiftNbits
+; simplify to:
+;   (trunc(x)) << shiftNbits
+
+; Simple tests.
+
+declare void @use32(i32)
+declare void @use64(i64)
+
+define i32 @t0_basic(i64 %x, i32 %nbits) {
+; CHECK-LABEL: @t0_basic(
+; CHECK-NEXT:    [[T0:%.*]] = zext i32 [[NBITS:%.*]] to i64
+; CHECK-NEXT:    [[T1:%.*]] = shl i64 1, [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = add i64 [[T1]], -1
+; CHECK-NEXT:    [[T3:%.*]] = sub i32 32, [[NBITS]]
+; CHECK-NEXT:    [[T4:%.*]] = and i64 [[T2]], [[X:%.*]]
+; CHECK-NEXT:    call void @use32(i32 [[NBITS]])
+; CHECK-NEXT:    call void @use64(i64 [[T0]])
+; CHECK-NEXT:    call void @use64(i64 [[T1]])
+; CHECK-NEXT:    call void @use64(i64 [[T2]])
+; CHECK-NEXT:    call void @use32(i32 [[T3]])
+; CHECK-NEXT:    call void @use64(i64 [[T4]])
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[X]] to i32
+; CHECK-NEXT:    [[T6:%.*]] = shl i32 [[TMP1]], [[T3]]
+; CHECK-NEXT:    ret i32 [[T6]]
+;
+  %t0 = zext i32 %nbits to i64
+  %t1 = shl i64 1, %t0
+  %t2 = add i64 %t1, -1
+  %t3 = sub i32 32, %nbits
+  %t4 = and i64 %t2, %x
+
+  call void @use32(i32 %nbits)
+  call void @use64(i64 %t0)
+  call void @use64(i64 %t1)
+  call void @use64(i64 %t2)
+  call void @use32(i32 %t3)
+  call void @use64(i64 %t4)
+
+  %t5 = trunc i64 %t4 to i32
+  %t6 = shl i32 %t5, %t3
+  ret i32 %t6
+}
+
+; Vectors
+
+declare void @use8xi32(<8 x i32>)
+declare void @use8xi64(<8 x i64>)
+
+define <8 x i32> @t1_vec_splat(<8 x i64> %x, <8 x i32> %nbits) {
+; CHECK-LABEL: @t1_vec_splat(
+; CHECK-NEXT:    [[T0:%.*]] = zext <8 x i32> [[NBITS:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[T1:%.*]] = shl <8 x i64> <i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1>, [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = add <8 x i64> [[T1]], <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1>
+; CHECK-NEXT:    [[T3:%.*]] = sub <8 x i32> <i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32>, [[NBITS]]
+; CHECK-NEXT:    [[T4:%.*]] = and <8 x i64> [[T2]], [[X:%.*]]
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[NBITS]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T0]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T1]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T2]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T3]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T4]])
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <8 x i64> [[X]] to <8 x i32>
+; CHECK-NEXT:    [[T6:%.*]] = shl <8 x i32> [[TMP1]], [[T3]]
+; CHECK-NEXT:    ret <8 x i32> [[T6]]
+;
+  %t0 = zext <8 x i32> %nbits to <8 x i64>
+  %t1 = shl <8 x i64> <i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1>, %t0
+  %t2 = add <8 x i64> %t1, <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1>
+  %t3 = sub <8 x i32> <i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32>, %nbits
+  %t4 = and <8 x i64> %t2, %x
+
+  call void @use8xi32(<8 x i32> %nbits)
+  call void @use8xi64(<8 x i64> %t0)
+  call void @use8xi64(<8 x i64> %t1)
+  call void @use8xi64(<8 x i64> %t2)
+  call void @use8xi32(<8 x i32> %t3)
+  call void @use8xi64(<8 x i64> %t4)
+
+  %t5 = trunc <8 x i64> %t4 to <8 x i32>
+  %t6 = shl <8 x i32> %t5, %t3
+  ret <8 x i32> %t6
+}
+
+define <8 x i32> @t2_vec_splat_undef(<8 x i64> %x, <8 x i32> %nbits) {
+; CHECK-LABEL: @t2_vec_splat_undef(
+; CHECK-NEXT:    [[T0:%.*]] = zext <8 x i32> [[NBITS:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[T1:%.*]] = shl <8 x i64> <i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 undef, i64 1>, [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = add <8 x i64> [[T1]], <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 undef, i64 -1>
+; CHECK-NEXT:    [[T3:%.*]] = sub <8 x i32> <i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 undef, i32 32>, [[NBITS]]
+; CHECK-NEXT:    [[T4:%.*]] = and <8 x i64> [[T2]], [[X:%.*]]
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[NBITS]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T0]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T1]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T2]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T3]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T4]])
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <8 x i64> [[X]] to <8 x i32>
+; CHECK-NEXT:    [[T6:%.*]] = shl <8 x i32> [[TMP1]], [[T3]]
+; CHECK-NEXT:    ret <8 x i32> [[T6]]
+;
+  %t0 = zext <8 x i32> %nbits to <8 x i64>
+  %t1 = shl <8 x i64> <i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 undef, i64 1>, %t0
+  %t2 = add <8 x i64> %t1, <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 undef, i64 -1>
+  %t3 = sub <8 x i32> <i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 undef, i32 32>, %nbits
+  %t4 = and <8 x i64> %t2, %x
+
+  call void @use8xi32(<8 x i32> %nbits)
+  call void @use8xi64(<8 x i64> %t0)
+  call void @use8xi64(<8 x i64> %t1)
+  call void @use8xi64(<8 x i64> %t2)
+  call void @use8xi32(<8 x i32> %t3)
+  call void @use8xi64(<8 x i64> %t4)
+
+  %t5 = trunc <8 x i64> %t4 to <8 x i32>
+  %t6 = shl <8 x i32> %t5, %t3
+  ret <8 x i32> %t6
+}
+
+define <8 x i32> @t3_vec_nonsplat(<8 x i64> %x, <8 x i32> %nbits) {
+; CHECK-LABEL: @t3_vec_nonsplat(
+; CHECK-NEXT:    [[T0:%.*]] = zext <8 x i32> [[NBITS:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[T1:%.*]] = shl <8 x i64> <i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1>, [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = add <8 x i64> [[T1]], <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1>
+; CHECK-NEXT:    [[T3:%.*]] = sub <8 x i32> <i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32>, [[NBITS]]
+; CHECK-NEXT:    [[T4:%.*]] = and <8 x i64> [[T2]], [[X:%.*]]
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[NBITS]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T0]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T1]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T2]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T3]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T4]])
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <8 x i64> [[X]] to <8 x i32>
+; CHECK-NEXT:    [[T6:%.*]] = shl <8 x i32> [[TMP1]], [[T3]]
+; CHECK-NEXT:    ret <8 x i32> [[T6]]
+;
+  %t0 = zext <8 x i32> %nbits to <8 x i64>
+  %t1 = shl <8 x i64> <i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1>, %t0
+  %t2 = add <8 x i64> %t1, <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1>
+  %t3 = sub <8 x i32> <i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32>, %nbits
+  %t4 = and <8 x i64> %t2, %x
+
+  call void @use8xi32(<8 x i32> %nbits)
+  call void @use8xi64(<8 x i64> %t0)
+  call void @use8xi64(<8 x i64> %t1)
+  call void @use8xi64(<8 x i64> %t2)
+  call void @use8xi32(<8 x i32> %t3)
+  call void @use8xi64(<8 x i64> %t4)
+
+  %t5 = trunc <8 x i64> %t4 to <8 x i32>
+  %t6 = shl <8 x i32> %t5, %t3
+  ret <8 x i32> %t6
+}
+
+; Extra uses
+
+define i32 @n4_extrause(i64 %x, i32 %nbits) {
+; CHECK-LABEL: @n4_extrause(
+; CHECK-NEXT:    [[T0:%.*]] = zext i32 [[NBITS:%.*]] to i64
+; CHECK-NEXT:    [[T1:%.*]] = shl i64 1, [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = add i64 [[T1]], -1
+; CHECK-NEXT:    [[T3:%.*]] = sub i32 32, [[NBITS]]
+; CHECK-NEXT:    [[T4:%.*]] = and i64 [[T2]], [[X:%.*]]
+; CHECK-NEXT:    call void @use32(i32 [[NBITS]])
+; CHECK-NEXT:    call void @use64(i64 [[T0]])
+; CHECK-NEXT:    call void @use64(i64 [[T1]])
+; CHECK-NEXT:    call void @use64(i64 [[T2]])
+; CHECK-NEXT:    call void @use32(i32 [[T3]])
+; CHECK-NEXT:    call void @use64(i64 [[T4]])
+; CHECK-NEXT:    [[T5:%.*]] = trunc i64 [[T4]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[T5]])
+; CHECK-NEXT:    [[T6:%.*]] = shl i32 [[T5]], [[T3]]
+; CHECK-NEXT:    ret i32 [[T6]]
+;
+  %t0 = zext i32 %nbits to i64
+  %t1 = shl i64 1, %t0
+  %t2 = add i64 %t1, -1
+  %t3 = sub i32 32, %nbits
+  %t4 = and i64 %t2, %x
+
+  call void @use32(i32 %nbits)
+  call void @use64(i64 %t0)
+  call void @use64(i64 %t1)
+  call void @use64(i64 %t2)
+  call void @use32(i32 %t3)
+  call void @use64(i64 %t4)
+
+  %t5 = trunc i64 %t4 to i32
+  call void @use32(i32 %t5)
+  %t6 = shl i32 %t5, %t3
+  ret i32 %t6
+}
diff --git a/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-after-truncation-variant-b.ll b/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-after-truncation-variant-b.ll
new file mode 100644
index 000000000..960f2c9e8
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-after-truncation-variant-b.ll
@@ -0,0 +1,238 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instcombine -S | FileCheck %s
+
+; If we have some pattern that leaves only some low bits set, and then performs
+; left-shift of those bits, we can combine those two shifts into a shift+mask.
+
+; There are many variants to this pattern:
+;   b)  (trunc ((x & (~(-1 << maskNbits))))) << shiftNbits
+; simplify to:
+;   ((trunc(x)) << shiftNbits) & (~(-1 << (maskNbits+shiftNbits)))
+
+; Simple tests.
+
+declare void @use32(i32)
+declare void @use64(i64)
+
+define i32 @t0_basic(i64 %x, i32 %nbits) {
+; CHECK-LABEL: @t0_basic(
+; CHECK-NEXT:    [[T0:%.*]] = zext i32 [[NBITS:%.*]] to i64
+; CHECK-NEXT:    [[T1:%.*]] = shl i64 -1, [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = xor i64 [[T1]], -1
+; CHECK-NEXT:    [[T3:%.*]] = sub i32 32, [[NBITS]]
+; CHECK-NEXT:    [[T4:%.*]] = and i64 [[T2]], [[X:%.*]]
+; CHECK-NEXT:    call void @use32(i32 [[NBITS]])
+; CHECK-NEXT:    call void @use64(i64 [[T0]])
+; CHECK-NEXT:    call void @use64(i64 [[T1]])
+; CHECK-NEXT:    call void @use64(i64 [[T2]])
+; CHECK-NEXT:    call void @use32(i32 [[T3]])
+; CHECK-NEXT:    call void @use64(i64 [[T4]])
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[X]] to i32
+; CHECK-NEXT:    [[T6:%.*]] = shl i32 [[TMP1]], [[T3]]
+; CHECK-NEXT:    ret i32 [[T6]]
+;
+  %t0 = zext i32 %nbits to i64
+  %t1 = shl i64 -1, %t0
+  %t2 = xor i64 %t1, -1
+  %t3 = sub i32 32, %nbits
+  %t4 = and i64 %t2, %x
+
+  call void @use32(i32 %nbits)
+  call void @use64(i64 %t0)
+  call void @use64(i64 %t1)
+  call void @use64(i64 %t2)
+  call void @use32(i32 %t3)
+  call void @use64(i64 %t4)
+
+  %t5 = trunc i64 %t4 to i32
+  %t6 = shl i32 %t5, %t3
+  ret i32 %t6
+}
+
+; Vectors
+
+declare void @use8xi32(<8 x i32>)
+declare void @use8xi64(<8 x i64>)
+
+define <8 x i32> @t1_vec_splat(<8 x i64> %x, <8 x i32> %nbits) {
+; CHECK-LABEL: @t1_vec_splat(
+; CHECK-NEXT:    [[T0:%.*]] = zext <8 x i32> [[NBITS:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[T1:%.*]] = shl <8 x i64> <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1>, [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = xor <8 x i64> [[T1]], <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1>
+; CHECK-NEXT:    [[T3:%.*]] = sub <8 x i32> <i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32>, [[NBITS]]
+; CHECK-NEXT:    [[T4:%.*]] = and <8 x i64> [[T2]], [[X:%.*]]
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[NBITS]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T0]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T1]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T2]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T3]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T4]])
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <8 x i64> [[X]] to <8 x i32>
+; CHECK-NEXT:    [[T6:%.*]] = shl <8 x i32> [[TMP1]], [[T3]]
+; CHECK-NEXT:    ret <8 x i32> [[T6]]
+;
+  %t0 = zext <8 x i32> %nbits to <8 x i64>
+  %t1 = shl <8 x i64> <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1>, %t0
+  %t2 = xor <8 x i64> %t1, <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1>
+  %t3 = sub <8 x i32> <i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32>, %nbits
+  %t4 = and <8 x i64> %t2, %x
+
+  call void @use8xi32(<8 x i32> %nbits)
+  call void @use8xi64(<8 x i64> %t0)
+  call void @use8xi64(<8 x i64> %t1)
+  call void @use8xi64(<8 x i64> %t2)
+  call void @use8xi32(<8 x i32> %t3)
+  call void @use8xi64(<8 x i64> %t4)
+
+  %t5 = trunc <8 x i64> %t4 to <8 x i32>
+  %t6 = shl <8 x i32> %t5, %t3
+  ret <8 x i32> %t6
+}
+
+define <8 x i32> @t2_vec_splat_undef(<8 x i64> %x, <8 x i32> %nbits) {
+; CHECK-LABEL: @t2_vec_splat_undef(
+; CHECK-NEXT:    [[T0:%.*]] = zext <8 x i32> [[NBITS:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[T1:%.*]] = shl <8 x i64> <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 undef, i64 -1>, [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = xor <8 x i64> [[T1]], <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 undef, i64 -1>
+; CHECK-NEXT:    [[T3:%.*]] = sub <8 x i32> <i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 undef, i32 32>, [[NBITS]]
+; CHECK-NEXT:    [[T4:%.*]] = and <8 x i64> [[T2]], [[X:%.*]]
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[NBITS]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T0]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T1]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T2]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T3]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T4]])
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <8 x i64> [[X]] to <8 x i32>
+; CHECK-NEXT:    [[T6:%.*]] = shl <8 x i32> [[TMP1]], [[T3]]
+; CHECK-NEXT:    ret <8 x i32> [[T6]]
+;
+  %t0 = zext <8 x i32> %nbits to <8 x i64>
+  %t1 = shl <8 x i64> <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 undef, i64 -1>, %t0
+  %t2 = xor <8 x i64> %t1, <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 undef, i64 -1>
+  %t3 = sub <8 x i32> <i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 undef, i32 32>, %nbits
+  %t4 = and <8 x i64> %t2, %x
+
+  call void @use8xi32(<8 x i32> %nbits)
+  call void @use8xi64(<8 x i64> %t0)
+  call void @use8xi64(<8 x i64> %t1)
+  call void @use8xi64(<8 x i64> %t2)
+  call void @use8xi32(<8 x i32> %t3)
+  call void @use8xi64(<8 x i64> %t4)
+
+  %t5 = trunc <8 x i64> %t4 to <8 x i32>
+  %t6 = shl <8 x i32> %t5, %t3
+  ret <8 x i32> %t6
+}
+
+define <8 x i32> @t3_vec_nonsplat(<8 x i64> %x, <8 x i32> %nbits) {
+; CHECK-LABEL: @t3_vec_nonsplat(
+; CHECK-NEXT:    [[T0:%.*]] = add <8 x i32> [[NBITS:%.*]], <i32 -1, i32 0, i32 0, i32 1, i32 0, i32 0, i32 0, i32 0>
+; CHECK-NEXT:    [[T1:%.*]] = zext <8 x i32> [[T0]] to <8 x i64>
+; CHECK-NEXT:    [[T2:%.*]] = shl <8 x i64> <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1>, [[T1]]
+; CHECK-NEXT:    [[T3:%.*]] = xor <8 x i64> [[T2]], <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1>
+; CHECK-NEXT:    [[T4:%.*]] = sub <8 x i32> <i32 33, i32 32, i32 33, i32 32, i32 32, i32 32, i32 32, i32 32>, [[NBITS]]
+; CHECK-NEXT:    [[T5:%.*]] = and <8 x i64> [[T3]], [[X:%.*]]
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T0]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T1]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T2]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T3]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T4]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T5]])
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <8 x i64> [[X]] to <8 x i32>
+; CHECK-NEXT:    [[T7:%.*]] = shl <8 x i32> [[TMP1]], [[T4]]
+; CHECK-NEXT:    ret <8 x i32> [[T7]]
+;
+  %t0 = add <8 x i32> %nbits, <i32 -1, i32 0, i32 0, i32 1, i32 0, i32 0, i32 0, i32 0>
+  %t1 = zext <8 x i32> %t0 to <8 x i64>
+  %t2 = shl <8 x i64> <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1>, %t1 ; shifting by nbits-1
+  %t3 = xor <8 x i64> %t2, <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1>
+  %t4 = sub <8 x i32> <i32 33, i32 32, i32 33, i32 32, i32 32, i32 32, i32 32, i32 32>, %nbits
+  %t5 = and <8 x i64> %t3, %x
+
+  call void @use8xi32(<8 x i32> %t0)
+  call void @use8xi64(<8 x i64> %t1)
+  call void @use8xi64(<8 x i64> %t2)
+  call void @use8xi64(<8 x i64> %t3)
+  call void @use8xi32(<8 x i32> %t4)
+  call void @use8xi64(<8 x i64> %t5)
+
+  %t6 = trunc <8 x i64> %t5 to <8 x i32>
+  %t7 = shl <8 x i32> %t6, %t4
+  ret <8 x i32> %t7
+}
+
+; -1 can be truncated.
+
+define i32 @t4_allones_trunc(i64 %x, i32 %nbits) {
+; CHECK-LABEL: @t4_allones_trunc(
+; CHECK-NEXT:    [[T0:%.*]] = zext i32 [[NBITS:%.*]] to i64
+; CHECK-NEXT:    [[T1:%.*]] = shl i64 -1, [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = xor i64 [[T1]], 4294967295
+; CHECK-NEXT:    [[T3:%.*]] = sub i32 32, [[NBITS]]
+; CHECK-NEXT:    [[T4:%.*]] = and i64 [[T2]], [[X:%.*]]
+; CHECK-NEXT:    call void @use32(i32 [[NBITS]])
+; CHECK-NEXT:    call void @use64(i64 [[T0]])
+; CHECK-NEXT:    call void @use64(i64 [[T1]])
+; CHECK-NEXT:    call void @use64(i64 [[T2]])
+; CHECK-NEXT:    call void @use32(i32 [[T3]])
+; CHECK-NEXT:    call void @use64(i64 [[T4]])
+; CHECK-NEXT:    [[T5:%.*]] = trunc i64 [[T4]] to i32
+; CHECK-NEXT:    [[T6:%.*]] = shl i32 [[T5]], [[T3]]
+; CHECK-NEXT:    ret i32 [[T6]]
+;
+  %t0 = zext i32 %nbits to i64
+  %t1 = shl i64 -1, %t0
+  %t2 = xor i64 %t1, 4294967295
+  %t3 = sub i32 32, %nbits
+  %t4 = and i64 %t2, %x
+
+  call void @use32(i32 %nbits)
+  call void @use64(i64 %t0)
+  call void @use64(i64 %t1)
+  call void @use64(i64 %t2)
+  call void @use32(i32 %t3)
+  call void @use64(i64 %t4)
+
+  %t5 = trunc i64 %t4 to i32
+  %t6 = shl i32 %t5, %t3
+  ret i32 %t6
+}
+
+; Extra uses
+
+define i32 @n5_extrause(i64 %x, i32 %nbits) {
+; CHECK-LABEL: @n5_extrause(
+; CHECK-NEXT:    [[T0:%.*]] = zext i32 [[NBITS:%.*]] to i64
+; CHECK-NEXT:    [[T1:%.*]] = shl i64 -1, [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = xor i64 [[T1]], -1
+; CHECK-NEXT:    [[T3:%.*]] = sub i32 32, [[NBITS]]
+; CHECK-NEXT:    [[T4:%.*]] = and i64 [[T2]], [[X:%.*]]
+; CHECK-NEXT:    call void @use32(i32 [[NBITS]])
+; CHECK-NEXT:    call void @use64(i64 [[T0]])
+; CHECK-NEXT:    call void @use64(i64 [[T1]])
+; CHECK-NEXT:    call void @use64(i64 [[T2]])
+; CHECK-NEXT:    call void @use32(i32 [[T3]])
+; CHECK-NEXT:    call void @use64(i64 [[T4]])
+; CHECK-NEXT:    [[T5:%.*]] = trunc i64 [[T4]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[T5]])
+; CHECK-NEXT:    [[T6:%.*]] = shl i32 [[T5]], [[T3]]
+; CHECK-NEXT:    ret i32 [[T6]]
+;
+  %t0 = zext i32 %nbits to i64
+  %t1 = shl i64 -1, %t0
+  %t2 = xor i64 %t1, -1
+  %t3 = sub i32 32, %nbits
+  %t4 = and i64 %t2, %x
+
+  call void @use32(i32 %nbits)
+  call void @use64(i64 %t0)
+  call void @use64(i64 %t1)
+  call void @use64(i64 %t2)
+  call void @use32(i32 %t3)
+  call void @use64(i64 %t4)
+
+  %t5 = trunc i64 %t4 to i32
+  call void @use32(i32 %t5)
+  %t6 = shl i32 %t5, %t3
+  ret i32 %t6
+}
diff --git a/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-after-truncation-variant-c.ll b/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-after-truncation-variant-c.ll
new file mode 100644
index 000000000..3cc3e1028
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-after-truncation-variant-c.ll
@@ -0,0 +1,169 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instcombine -S | FileCheck %s
+
+; If we have some pattern that leaves only some low bits set, and then performs
+; left-shift of those bits, we can combine those two shifts into a shift+mask.
+
+; There are many variants to this pattern:
+;   c)  (trunc ((x & (-1 >> maskNbits)))) << shiftNbits
+; simplify to:
+;   (trunc(x)) << shiftNbits
+
+; Simple tests.
+
+declare void @use32(i32)
+declare void @use64(i64)
+
+define i32 @t0_basic(i64 %x, i32 %nbits) {
+; CHECK-LABEL: @t0_basic(
+; CHECK-NEXT:    [[T0:%.*]] = zext i32 [[NBITS:%.*]] to i64
+; CHECK-NEXT:    [[T1:%.*]] = lshr i64 -1, [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[NBITS]], -32
+; CHECK-NEXT:    [[T3:%.*]] = and i64 [[T1]], [[X:%.*]]
+; CHECK-NEXT:    call void @use64(i64 [[T0]])
+; CHECK-NEXT:    call void @use64(i64 [[T1]])
+; CHECK-NEXT:    call void @use32(i32 [[T2]])
+; CHECK-NEXT:    call void @use64(i64 [[T3]])
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[X]] to i32
+; CHECK-NEXT:    [[T5:%.*]] = shl i32 [[TMP1]], [[T2]]
+; CHECK-NEXT:    ret i32 [[T5]]
+;
+  %t0 = zext i32 %nbits to i64
+  %t1 = lshr i64 -1, %t0
+  %t2 = add i32 %nbits, -32
+  %t3 = and i64 %t1, %x
+
+  call void @use64(i64 %t0)
+  call void @use64(i64 %t1)
+  call void @use32(i32 %t2)
+  call void @use64(i64 %t3)
+
+  %t4 = trunc i64 %t3 to i32
+  %t5 = shl i32 %t4, %t2
+  ret i32 %t5
+}
+
+; Vectors
+
+declare void @use8xi32(<8 x i32>)
+declare void @use8xi64(<8 x i64>)
+
+define <8 x i32> @t1_vec_splat(<8 x i64> %x, <8 x i32> %nbits) {
+; CHECK-LABEL: @t1_vec_splat(
+; CHECK-NEXT:    [[T0:%.*]] = zext <8 x i32> [[NBITS:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[T1:%.*]] = lshr <8 x i64> <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1>, [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = add <8 x i32> [[NBITS]], <i32 -32, i32 -32, i32 -32, i32 -32, i32 -32, i32 -32, i32 -32, i32 -32>
+; CHECK-NEXT:    [[T3:%.*]] = and <8 x i64> [[T1]], [[X:%.*]]
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T0]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T1]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T2]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T3]])
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <8 x i64> [[X]] to <8 x i32>
+; CHECK-NEXT:    [[T5:%.*]] = shl <8 x i32> [[TMP1]], [[T2]]
+; CHECK-NEXT:    ret <8 x i32> [[T5]]
+;
+  %t0 = zext <8 x i32> %nbits to <8 x i64>
+  %t1 = lshr <8 x i64> <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1>, %t0
+  %t2 = add <8 x i32> %nbits, <i32 -32, i32 -32, i32 -32, i32 -32, i32 -32, i32 -32, i32 -32, i32 -32>
+  %t3 = and <8 x i64> %t1, %x
+
+  call void @use8xi64(<8 x i64> %t0)
+  call void @use8xi64(<8 x i64> %t1)
+  call void @use8xi32(<8 x i32> %t2)
+  call void @use8xi64(<8 x i64> %t3)
+
+  %t4 = trunc <8 x i64> %t3 to <8 x i32>
+  %t5 = shl <8 x i32> %t4, %t2
+  ret <8 x i32> %t5
+}
+
+define <8 x i32> @t2_vec_splat_undef(<8 x i64> %x, <8 x i32> %nbits) {
+; CHECK-LABEL: @t2_vec_splat_undef(
+; CHECK-NEXT:    [[T0:%.*]] = zext <8 x i32> [[NBITS:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[T1:%.*]] = lshr <8 x i64> <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 undef, i64 -1>, [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = add <8 x i32> [[NBITS]], <i32 -32, i32 -32, i32 -32, i32 -32, i32 -32, i32 -32, i32 undef, i32 -32>
+; CHECK-NEXT:    [[T3:%.*]] = and <8 x i64> [[T1]], [[X:%.*]]
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T0]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T1]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T2]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T3]])
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <8 x i64> [[X]] to <8 x i32>
+; CHECK-NEXT:    [[T5:%.*]] = shl <8 x i32> [[TMP1]], [[T2]]
+; CHECK-NEXT:    ret <8 x i32> [[T5]]
+;
+  %t0 = zext <8 x i32> %nbits to <8 x i64>
+  %t1 = lshr <8 x i64> <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 undef, i64 -1>, %t0
+  %t2 = add <8 x i32> %nbits, <i32 -32, i32 -32, i32 -32, i32 -32, i32 -32, i32 -32, i32 undef, i32 -32>
+  %t3 = and <8 x i64> %t1, %x
+
+  call void @use8xi64(<8 x i64> %t0)
+  call void @use8xi64(<8 x i64> %t1)
+  call void @use8xi32(<8 x i32> %t2)
+  call void @use8xi64(<8 x i64> %t3)
+
+  %t4 = trunc <8 x i64> %t3 to <8 x i32>
+  %t5 = shl <8 x i32> %t4, %t2
+  ret <8 x i32> %t5
+}
+
+define <8 x i32> @t3_vec_nonsplat(<8 x i64> %x, <8 x i32> %nbits) {
+; CHECK-LABEL: @t3_vec_nonsplat(
+; CHECK-NEXT:    [[T0:%.*]] = zext <8 x i32> [[NBITS:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[T1:%.*]] = lshr <8 x i64> <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 undef, i64 -1>, [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = add <8 x i32> [[NBITS]], <i32 -32, i32 -1, i32 0, i32 1, i32 31, i32 32, i32 undef, i32 64>
+; CHECK-NEXT:    [[T3:%.*]] = and <8 x i64> [[T1]], [[X:%.*]]
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T0]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T1]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T2]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T3]])
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <8 x i64> [[X]] to <8 x i32>
+; CHECK-NEXT:    [[T5:%.*]] = shl <8 x i32> [[TMP1]], [[T2]]
+; CHECK-NEXT:    ret <8 x i32> [[T5]]
+;
+  %t0 = zext <8 x i32> %nbits to <8 x i64>
+  %t1 = lshr <8 x i64> <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 undef, i64 -1>, %t0
+  %t2 = add <8 x i32> %nbits, <i32 -32, i32 -1, i32 0, i32 1, i32 31, i32 32, i32 undef, i32 64>
+  %t3 = and <8 x i64> %t1, %x
+
+  call void @use8xi64(<8 x i64> %t0)
+  call void @use8xi64(<8 x i64> %t1)
+  call void @use8xi32(<8 x i32> %t2)
+  call void @use8xi64(<8 x i64> %t3)
+
+  %t4 = trunc <8 x i64> %t3 to <8 x i32>
+  %t5 = shl <8 x i32> %t4, %t2
+  ret <8 x i32> %t5
+}
+
+; Extra uses.
+
+define i32 @n4_extrause(i64 %x, i32 %nbits) {
+; CHECK-LABEL: @n4_extrause(
+; CHECK-NEXT:    [[T0:%.*]] = zext i32 [[NBITS:%.*]] to i64
+; CHECK-NEXT:    [[T1:%.*]] = lshr i64 -1, [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[NBITS]], -32
+; CHECK-NEXT:    [[T3:%.*]] = and i64 [[T1]], [[X:%.*]]
+; CHECK-NEXT:    call void @use64(i64 [[T0]])
+; CHECK-NEXT:    call void @use64(i64 [[T1]])
+; CHECK-NEXT:    call void @use32(i32 [[T2]])
+; CHECK-NEXT:    call void @use64(i64 [[T3]])
+; CHECK-NEXT:    [[T4:%.*]] = trunc i64 [[T3]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[T4]])
+; CHECK-NEXT:    [[T5:%.*]] = shl i32 [[T4]], [[T2]]
+; CHECK-NEXT:    ret i32 [[T5]]
+;
+  %t0 = zext i32 %nbits to i64
+  %t1 = lshr i64 -1, %t0
+  %t2 = add i32 %nbits, -32
+  %t3 = and i64 %t1, %x
+
+  call void @use64(i64 %t0)
+  call void @use64(i64 %t1)
+  call void @use32(i32 %t2)
+  call void @use64(i64 %t3)
+
+  %t4 = trunc i64 %t3 to i32
+  call void @use32(i32 %t4)
+  %t5 = shl i32 %t4, %t2
+  ret i32 %t5
+}
diff --git a/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-after-truncation-variant-d.ll b/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-after-truncation-variant-d.ll
new file mode 100644
index 000000000..04a0966f1
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-after-truncation-variant-d.ll
@@ -0,0 +1,189 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instcombine -S | FileCheck %s
+
+; If we have some pattern that leaves only some low bits set, and then performs
+; left-shift of those bits, we can combine those two shifts into a shift+mask.
+
+; There are many variants to this pattern:
+;   d)  (trunc ((x & ((-1 << maskNbits) >> maskNbits)))) << shiftNbits
+; simplify to:
+;   (trunc(x)) << shiftNbits
+
+; Simple tests.
+
+declare void @use32(i32)
+declare void @use64(i64)
+
+define i32 @t0_basic(i64 %x, i32 %nbits) {
+; CHECK-LABEL: @t0_basic(
+; CHECK-NEXT:    [[T0:%.*]] = zext i32 [[NBITS:%.*]] to i64
+; CHECK-NEXT:    [[T1:%.*]] = shl i64 -1, [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = lshr i64 [[T1]], [[T0]]
+; CHECK-NEXT:    [[T3:%.*]] = add i32 [[NBITS]], -32
+; CHECK-NEXT:    [[T4:%.*]] = and i64 [[T2]], [[X:%.*]]
+; CHECK-NEXT:    call void @use64(i64 [[T0]])
+; CHECK-NEXT:    call void @use64(i64 [[T1]])
+; CHECK-NEXT:    call void @use64(i64 [[T2]])
+; CHECK-NEXT:    call void @use32(i32 [[T3]])
+; CHECK-NEXT:    call void @use64(i64 [[T4]])
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[X]] to i32
+; CHECK-NEXT:    [[T6:%.*]] = shl i32 [[TMP1]], [[T3]]
+; CHECK-NEXT:    ret i32 [[T6]]
+;
+  %t0 = zext i32 %nbits to i64
+  %t1 = shl i64 -1, %t0
+  %t2 = lshr i64 %t1, %t0
+  %t3 = add i32 %nbits, -32
+  %t4 = and i64 %t2, %x
+
+  call void @use64(i64 %t0)
+  call void @use64(i64 %t1)
+  call void @use64(i64 %t2)
+  call void @use32(i32 %t3)
+  call void @use64(i64 %t4)
+
+  %t5 = trunc i64 %t4 to i32
+  %t6 = shl i32 %t5, %t3
+  ret i32 %t6
+}
+
+; Vectors
+
+declare void @use8xi32(<8 x i32>)
+declare void @use8xi64(<8 x i64>)
+
+define <8 x i32> @t1_vec_splat(<8 x i64> %x, <8 x i32> %nbits) {
+; CHECK-LABEL: @t1_vec_splat(
+; CHECK-NEXT:    [[T0:%.*]] = zext <8 x i32> [[NBITS:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[T1:%.*]] = shl <8 x i64> <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1>, [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = lshr <8 x i64> [[T1]], [[T0]]
+; CHECK-NEXT:    [[T3:%.*]] = add <8 x i32> [[NBITS]], <i32 -32, i32 -32, i32 -32, i32 -32, i32 -32, i32 -32, i32 -32, i32 -32>
+; CHECK-NEXT:    [[T4:%.*]] = and <8 x i64> [[T2]], [[X:%.*]]
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T0]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T1]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T2]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T3]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T4]])
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <8 x i64> [[X]] to <8 x i32>
+; CHECK-NEXT:    [[T6:%.*]] = shl <8 x i32> [[TMP1]], [[T3]]
+; CHECK-NEXT:    ret <8 x i32> [[T6]]
+;
+  %t0 = zext <8 x i32> %nbits to <8 x i64>
+  %t1 = shl <8 x i64> <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1>, %t0
+  %t2 = lshr <8 x i64> %t1, %t0
+  %t3 = add <8 x i32> %nbits, <i32 -32, i32 -32, i32 -32, i32 -32, i32 -32, i32 -32, i32 -32, i32 -32>
+  %t4 = and <8 x i64> %t2, %x
+
+  call void @use8xi64(<8 x i64> %t0)
+  call void @use8xi64(<8 x i64> %t1)
+  call void @use8xi64(<8 x i64> %t2)
+  call void @use8xi32(<8 x i32> %t3)
+  call void @use8xi64(<8 x i64> %t4)
+
+  %t5 = trunc <8 x i64> %t4 to <8 x i32>
+  %t6 = shl <8 x i32> %t5, %t3
+  ret <8 x i32> %t6
+}
+
+define <8 x i32> @t2_vec_splat_undef(<8 x i64> %x, <8 x i32> %nbits) {
+; CHECK-LABEL: @t2_vec_splat_undef(
+; CHECK-NEXT:    [[T0:%.*]] = zext <8 x i32> [[NBITS:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[T1:%.*]] = shl <8 x i64> <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 undef, i64 -1>, [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = lshr <8 x i64> [[T1]], [[T0]]
+; CHECK-NEXT:    [[T3:%.*]] = add <8 x i32> [[NBITS]], <i32 -32, i32 -32, i32 -32, i32 -32, i32 -32, i32 -32, i32 undef, i32 -32>
+; CHECK-NEXT:    [[T4:%.*]] = and <8 x i64> [[T2]], [[X:%.*]]
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T0]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T1]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T2]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T3]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T4]])
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <8 x i64> [[X]] to <8 x i32>
+; CHECK-NEXT:    [[T6:%.*]] = shl <8 x i32> [[TMP1]], [[T3]]
+; CHECK-NEXT:    ret <8 x i32> [[T6]]
+;
+  %t0 = zext <8 x i32> %nbits to <8 x i64>
+  %t1 = shl <8 x i64> <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 undef, i64 -1>, %t0
+  %t2 = lshr <8 x i64> %t1, %t0
+  %t3 = add <8 x i32> %nbits, <i32 -32, i32 -32, i32 -32, i32 -32, i32 -32, i32 -32, i32 undef, i32 -32>
+  %t4 = and <8 x i64> %t2, %x
+
+  call void @use8xi64(<8 x i64> %t0)
+  call void @use8xi64(<8 x i64> %t1)
+  call void @use8xi64(<8 x i64> %t2)
+  call void @use8xi32(<8 x i32> %t3)
+  call void @use8xi64(<8 x i64> %t4)
+
+  %t5 = trunc <8 x i64> %t4 to <8 x i32>
+  %t6 = shl <8 x i32> %t5, %t3
+  ret <8 x i32> %t6
+}
+
+define <8 x i32> @t3_vec_nonsplat(<8 x i64> %x, <8 x i32> %nbits) {
+; CHECK-LABEL: @t3_vec_nonsplat(
+; CHECK-NEXT:    [[T0:%.*]] = zext <8 x i32> [[NBITS:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[T1:%.*]] = shl <8 x i64> <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 undef, i64 -1>, [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = lshr <8 x i64> [[T1]], [[T0]]
+; CHECK-NEXT:    [[T3:%.*]] = add <8 x i32> [[NBITS]], <i32 -32, i32 -1, i32 0, i32 1, i32 31, i32 32, i32 undef, i32 64>
+; CHECK-NEXT:    [[T4:%.*]] = and <8 x i64> [[T2]], [[X:%.*]]
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T0]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T1]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T2]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T3]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T4]])
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <8 x i64> [[X]] to <8 x i32>
+; CHECK-NEXT:    [[T6:%.*]] = shl <8 x i32> [[TMP1]], [[T3]]
+; CHECK-NEXT:    ret <8 x i32> [[T6]]
+;
+  %t0 = zext <8 x i32> %nbits to <8 x i64>
+  %t1 = shl <8 x i64> <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 undef, i64 -1>, %t0
+  %t2 = lshr <8 x i64> %t1, %t0
+  %t3 = add <8 x i32> %nbits, <i32 -32, i32 -1, i32 0, i32 1, i32 31, i32 32, i32 undef, i32 64>
+  %t4 = and <8 x i64> %t2, %x
+
+  call void @use8xi64(<8 x i64> %t0)
+  call void @use8xi64(<8 x i64> %t1)
+  call void @use8xi64(<8 x i64> %t2)
+  call void @use8xi32(<8 x i32> %t3)
+  call void @use8xi64(<8 x i64> %t4)
+
+  %t5 = trunc <8 x i64> %t4 to <8 x i32>
+  %t6 = shl <8 x i32> %t5, %t3
+  ret <8 x i32> %t6
+}
+
+; Extra uses.
+
+define i32 @n4_extrause(i64 %x, i32 %nbits) {
+; CHECK-LABEL: @n4_extrause(
+; CHECK-NEXT:    [[T0:%.*]] = zext i32 [[NBITS:%.*]] to i64
+; CHECK-NEXT:    [[T1:%.*]] = shl i64 -1, [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = lshr i64 [[T1]], [[T0]]
+; CHECK-NEXT:    [[T3:%.*]] = add i32 [[NBITS]], -32
+; CHECK-NEXT:    [[T4:%.*]] = and i64 [[T2]], [[X:%.*]]
+; CHECK-NEXT:    call void @use64(i64 [[T0]])
+; CHECK-NEXT:    call void @use64(i64 [[T1]])
+; CHECK-NEXT:    call void @use64(i64 [[T2]])
+; CHECK-NEXT:    call void @use32(i32 [[T3]])
+; CHECK-NEXT:    call void @use64(i64 [[T4]])
+; CHECK-NEXT:    [[T5:%.*]] = trunc i64 [[T4]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[T5]])
+; CHECK-NEXT:    [[T6:%.*]] = shl i32 [[T5]], [[T3]]
+; CHECK-NEXT:    ret i32 [[T6]]
+;
+  %t0 = zext i32 %nbits to i64
+  %t1 = shl i64 -1, %t0
+  %t2 = lshr i64 %t1, %t0
+  %t3 = add i32 %nbits, -32
+  %t4 = and i64 %t2, %x
+
+  call void @use64(i64 %t0)
+  call void @use64(i64 %t1)
+  call void @use64(i64 %t2)
+  call void @use32(i32 %t3)
+  call void @use64(i64 %t4)
+
+  %t5 = trunc i64 %t4 to i32
+  call void @use32(i32 %t5)
+  %t6 = shl i32 %t5, %t3
+  ret i32 %t6
+}
diff --git a/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-after-truncation-variant-e.ll b/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-after-truncation-variant-e.ll
new file mode 100644
index 000000000..64e7c6bb6
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-after-truncation-variant-e.ll
@@ -0,0 +1,169 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instcombine -S | FileCheck %s
+
+; If we have some pattern that leaves only some low bits set, lshr then performs
+; left-shift of those bits, we can combine those two shifts into a shift+mask.
+
+; There are many variants to this pattern:
+;   e)  (trunc (((x << maskNbits) l>> maskNbits))) << shiftNbits
+; simplify to:
+;   ((trunc(x)) << shiftNbits) & (-1 >> ((-(maskNbits+shiftNbits))+32))
+
+; Simple tests.
+
+declare void @use32(i32)
+declare void @use64(i64)
+
+define i32 @t0_basic(i64 %x, i32 %nbits) {
+; CHECK-LABEL: @t0_basic(
+; CHECK-NEXT:    [[T0:%.*]] = zext i32 [[NBITS:%.*]] to i64
+; CHECK-NEXT:    [[T1:%.*]] = shl i64 [[X:%.*]], [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[NBITS]], -32
+; CHECK-NEXT:    [[T3:%.*]] = lshr i64 [[T1]], [[T0]]
+; CHECK-NEXT:    call void @use64(i64 [[T0]])
+; CHECK-NEXT:    call void @use64(i64 [[T1]])
+; CHECK-NEXT:    call void @use32(i32 [[T2]])
+; CHECK-NEXT:    call void @use64(i64 [[T3]])
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[X]] to i32
+; CHECK-NEXT:    [[T5:%.*]] = shl i32 [[TMP1]], [[T2]]
+; CHECK-NEXT:    ret i32 [[T5]]
+;
+  %t0 = zext i32 %nbits to i64
+  %t1 = shl i64 %x, %t0
+  %t2 = add i32 %nbits, -32
+  %t3 = lshr i64 %t1, %t0
+
+  call void @use64(i64 %t0)
+  call void @use64(i64 %t1)
+  call void @use32(i32 %t2)
+  call void @use64(i64 %t3)
+
+  %t4 = trunc i64 %t3 to i32
+  %t5 = shl i32 %t4, %t2
+  ret i32 %t5
+}
+
+; Vectors
+
+declare void @use8xi32(<8 x i32>)
+declare void @use8xi64(<8 x i64>)
+
+define <8 x i32> @t1_vec_splat(<8 x i64> %x, <8 x i32> %nbits) {
+; CHECK-LABEL: @t1_vec_splat(
+; CHECK-NEXT:    [[T0:%.*]] = zext <8 x i32> [[NBITS:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[T1:%.*]] = shl <8 x i64> [[X:%.*]], [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = add <8 x i32> [[NBITS]], <i32 -32, i32 -32, i32 -32, i32 -32, i32 -32, i32 -32, i32 -32, i32 -32>
+; CHECK-NEXT:    [[T3:%.*]] = lshr <8 x i64> [[T1]], [[T0]]
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T0]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T1]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T2]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T3]])
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <8 x i64> [[X]] to <8 x i32>
+; CHECK-NEXT:    [[T5:%.*]] = shl <8 x i32> [[TMP1]], [[T2]]
+; CHECK-NEXT:    ret <8 x i32> [[T5]]
+;
+  %t0 = zext <8 x i32> %nbits to <8 x i64>
+  %t1 = shl <8 x i64> %x, %t0
+  %t2 = add <8 x i32> %nbits, <i32 -32, i32 -32, i32 -32, i32 -32, i32 -32, i32 -32, i32 -32, i32 -32>
+  %t3 = lshr <8 x i64> %t1, %t0
+
+  call void @use8xi64(<8 x i64> %t0)
+  call void @use8xi64(<8 x i64> %t1)
+  call void @use8xi32(<8 x i32> %t2)
+  call void @use8xi64(<8 x i64> %t3)
+
+  %t4 = trunc <8 x i64> %t3 to <8 x i32>
+  %t5 = shl <8 x i32> %t4, %t2
+  ret <8 x i32> %t5
+}
+
+define <8 x i32> @t2_vec_splat_undef(<8 x i64> %x, <8 x i32> %nbits) {
+; CHECK-LABEL: @t2_vec_splat_undef(
+; CHECK-NEXT:    [[T0:%.*]] = zext <8 x i32> [[NBITS:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[T1:%.*]] = shl <8 x i64> [[X:%.*]], [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = add <8 x i32> [[NBITS]], <i32 -32, i32 -32, i32 -32, i32 -32, i32 -32, i32 -32, i32 undef, i32 -32>
+; CHECK-NEXT:    [[T3:%.*]] = lshr <8 x i64> [[T1]], [[T0]]
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T0]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T1]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T2]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T3]])
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <8 x i64> [[X]] to <8 x i32>
+; CHECK-NEXT:    [[T5:%.*]] = shl <8 x i32> [[TMP1]], [[T2]]
+; CHECK-NEXT:    ret <8 x i32> [[T5]]
+;
+  %t0 = zext <8 x i32> %nbits to <8 x i64>
+  %t1 = shl <8 x i64> %x, %t0
+  %t2 = add <8 x i32> %nbits, <i32 -32, i32 -32, i32 -32, i32 -32, i32 -32, i32 -32, i32 undef, i32 -32>
+  %t3 = lshr <8 x i64> %t1, %t0
+
+  call void @use8xi64(<8 x i64> %t0)
+  call void @use8xi64(<8 x i64> %t1)
+  call void @use8xi32(<8 x i32> %t2)
+  call void @use8xi64(<8 x i64> %t3)
+
+  %t4 = trunc <8 x i64> %t3 to <8 x i32>
+  %t5 = shl <8 x i32> %t4, %t2
+  ret <8 x i32> %t5
+}
+
+define <8 x i32> @t3_vec_nonsplat(<8 x i64> %x, <8 x i32> %nbits) {
+; CHECK-LABEL: @t3_vec_nonsplat(
+; CHECK-NEXT:    [[T0:%.*]] = zext <8 x i32> [[NBITS:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[T1:%.*]] = shl <8 x i64> [[X:%.*]], [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = add <8 x i32> [[NBITS]], <i32 -32, i32 -1, i32 0, i32 1, i32 31, i32 32, i32 undef, i32 64>
+; CHECK-NEXT:    [[T3:%.*]] = lshr <8 x i64> [[T1]], [[T0]]
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T0]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T1]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T2]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T3]])
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <8 x i64> [[X]] to <8 x i32>
+; CHECK-NEXT:    [[T5:%.*]] = shl <8 x i32> [[TMP1]], [[T2]]
+; CHECK-NEXT:    ret <8 x i32> [[T5]]
+;
+  %t0 = zext <8 x i32> %nbits to <8 x i64>
+  %t1 = shl <8 x i64> %x, %t0
+  %t2 = add <8 x i32> %nbits, <i32 -32, i32 -1, i32 0, i32 1, i32 31, i32 32, i32 undef, i32 64>
+  %t3 = lshr <8 x i64> %t1, %t0
+
+  call void @use8xi64(<8 x i64> %t0)
+  call void @use8xi64(<8 x i64> %t1)
+  call void @use8xi32(<8 x i32> %t2)
+  call void @use8xi64(<8 x i64> %t3)
+
+  %t4 = trunc <8 x i64> %t3 to <8 x i32>
+  %t5 = shl <8 x i32> %t4, %t2
+  ret <8 x i32> %t5
+}
+
+; Extra uses.
+
+define i32 @n4_extrause(i64 %x, i32 %nbits) {
+; CHECK-LABEL: @n4_extrause(
+; CHECK-NEXT:    [[T0:%.*]] = zext i32 [[NBITS:%.*]] to i64
+; CHECK-NEXT:    [[T1:%.*]] = shl i64 [[X:%.*]], [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[NBITS]], -32
+; CHECK-NEXT:    [[T3:%.*]] = lshr i64 [[T1]], [[T0]]
+; CHECK-NEXT:    call void @use64(i64 [[T0]])
+; CHECK-NEXT:    call void @use64(i64 [[T1]])
+; CHECK-NEXT:    call void @use32(i32 [[T2]])
+; CHECK-NEXT:    call void @use64(i64 [[T3]])
+; CHECK-NEXT:    [[T4:%.*]] = trunc i64 [[T3]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[T4]])
+; CHECK-NEXT:    [[T5:%.*]] = shl i32 [[T4]], [[T2]]
+; CHECK-NEXT:    ret i32 [[T5]]
+;
+  %t0 = zext i32 %nbits to i64
+  %t1 = shl i64 %x, %t0
+  %t2 = add i32 %nbits, -32
+  %t3 = lshr i64 %t1, %t0
+
+  call void @use64(i64 %t0)
+  call void @use64(i64 %t1)
+  call void @use32(i32 %t2)
+  call void @use64(i64 %t3)
+
+  %t4 = trunc i64 %t3 to i32
+  call void @use32(i32 %t4)
+  %t5 = shl i32 %t4, %t2
+  ret i32 %t5
+}
diff --git a/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-after-truncation-variant-f.ll b/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-after-truncation-variant-f.ll
new file mode 100644
index 000000000..aaaeb43fd
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-after-truncation-variant-f.ll
@@ -0,0 +1,198 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instcombine -S | FileCheck %s
+
+; If we have some pattern that leaves only some low bits set, ashr then performs
+; left-shift of those bits, we can combine those two shifts into a shift+mask.
+
+; There are many variants to this pattern:
+;   e)  (trunc (((x << maskNbits) a>> maskNbits))) << shiftNbits
+; simplify to:
+;   (trunc(x)) << shiftNbits
+
+; Simple tests.
+
+declare void @use32(i32)
+declare void @use64(i64)
+
+define i32 @t0_basic(i64 %x, i32 %nbits) {
+; CHECK-LABEL: @t0_basic(
+; CHECK-NEXT:    [[T0:%.*]] = zext i32 [[NBITS:%.*]] to i64
+; CHECK-NEXT:    [[T1:%.*]] = shl i64 [[X:%.*]], [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[NBITS]], -32
+; CHECK-NEXT:    [[T3:%.*]] = ashr i64 [[T1]], [[T0]]
+; CHECK-NEXT:    call void @use64(i64 [[T0]])
+; CHECK-NEXT:    call void @use64(i64 [[T1]])
+; CHECK-NEXT:    call void @use32(i32 [[T2]])
+; CHECK-NEXT:    call void @use64(i64 [[T3]])
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[X]] to i32
+; CHECK-NEXT:    [[T5:%.*]] = shl i32 [[TMP1]], [[T2]]
+; CHECK-NEXT:    ret i32 [[T5]]
+;
+  %t0 = zext i32 %nbits to i64
+  %t1 = shl i64 %x, %t0
+  %t2 = add i32 %nbits, -32
+  %t3 = ashr i64 %t1, %t0
+
+  call void @use64(i64 %t0)
+  call void @use64(i64 %t1)
+  call void @use32(i32 %t2)
+  call void @use64(i64 %t3)
+
+  %t4 = trunc i64 %t3 to i32
+  %t5 = shl i32 %t4, %t2
+  ret i32 %t5
+}
+
+; Vectors
+
+declare void @use8xi32(<8 x i32>)
+declare void @use8xi64(<8 x i64>)
+
+define <8 x i32> @t1_vec_splat(<8 x i64> %x, <8 x i32> %nbits) {
+; CHECK-LABEL: @t1_vec_splat(
+; CHECK-NEXT:    [[T0:%.*]] = zext <8 x i32> [[NBITS:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[T1:%.*]] = shl <8 x i64> [[X:%.*]], [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = add <8 x i32> [[NBITS]], <i32 -32, i32 -32, i32 -32, i32 -32, i32 -32, i32 -32, i32 -32, i32 -32>
+; CHECK-NEXT:    [[T3:%.*]] = ashr <8 x i64> [[T1]], [[T0]]
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T0]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T1]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T2]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T3]])
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <8 x i64> [[X]] to <8 x i32>
+; CHECK-NEXT:    [[T5:%.*]] = shl <8 x i32> [[TMP1]], [[T2]]
+; CHECK-NEXT:    ret <8 x i32> [[T5]]
+;
+  %t0 = zext <8 x i32> %nbits to <8 x i64>
+  %t1 = shl <8 x i64> %x, %t0
+  %t2 = add <8 x i32> %nbits, <i32 -32, i32 -32, i32 -32, i32 -32, i32 -32, i32 -32, i32 -32, i32 -32>
+  %t3 = ashr <8 x i64> %t1, %t0
+
+  call void @use8xi64(<8 x i64> %t0)
+  call void @use8xi64(<8 x i64> %t1)
+  call void @use8xi32(<8 x i32> %t2)
+  call void @use8xi64(<8 x i64> %t3)
+
+  %t4 = trunc <8 x i64> %t3 to <8 x i32>
+  %t5 = shl <8 x i32> %t4, %t2
+  ret <8 x i32> %t5
+}
+
+define <8 x i32> @t2_vec_splat_undef(<8 x i64> %x, <8 x i32> %nbits) {
+; CHECK-LABEL: @t2_vec_splat_undef(
+; CHECK-NEXT:    [[T0:%.*]] = zext <8 x i32> [[NBITS:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[T1:%.*]] = shl <8 x i64> [[X:%.*]], [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = add <8 x i32> [[NBITS]], <i32 -32, i32 -32, i32 -32, i32 -32, i32 -32, i32 -32, i32 undef, i32 -32>
+; CHECK-NEXT:    [[T3:%.*]] = ashr <8 x i64> [[T1]], [[T0]]
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T0]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T1]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T2]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T3]])
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <8 x i64> [[X]] to <8 x i32>
+; CHECK-NEXT:    [[T5:%.*]] = shl <8 x i32> [[TMP1]], [[T2]]
+; CHECK-NEXT:    ret <8 x i32> [[T5]]
+;
+  %t0 = zext <8 x i32> %nbits to <8 x i64>
+  %t1 = shl <8 x i64> %x, %t0
+  %t2 = add <8 x i32> %nbits, <i32 -32, i32 -32, i32 -32, i32 -32, i32 -32, i32 -32, i32 undef, i32 -32>
+  %t3 = ashr <8 x i64> %t1, %t0
+
+  call void @use8xi64(<8 x i64> %t0)
+  call void @use8xi64(<8 x i64> %t1)
+  call void @use8xi32(<8 x i32> %t2)
+  call void @use8xi64(<8 x i64> %t3)
+
+  %t4 = trunc <8 x i64> %t3 to <8 x i32>
+  %t5 = shl <8 x i32> %t4, %t2
+  ret <8 x i32> %t5
+}
+
+define <8 x i32> @t3_vec_nonsplat(<8 x i64> %x, <8 x i32> %nbits) {
+; CHECK-LABEL: @t3_vec_nonsplat(
+; CHECK-NEXT:    [[T0:%.*]] = zext <8 x i32> [[NBITS:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[T1:%.*]] = shl <8 x i64> [[X:%.*]], [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = add <8 x i32> [[NBITS]], <i32 -32, i32 -1, i32 0, i32 1, i32 31, i32 32, i32 undef, i32 64>
+; CHECK-NEXT:    [[T3:%.*]] = ashr <8 x i64> [[T1]], [[T0]]
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T0]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T1]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T2]])
+; CHECK-NEXT:    call void @use8xi64(<8 x i64> [[T3]])
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <8 x i64> [[X]] to <8 x i32>
+; CHECK-NEXT:    [[T5:%.*]] = shl <8 x i32> [[TMP1]], [[T2]]
+; CHECK-NEXT:    ret <8 x i32> [[T5]]
+;
+  %t0 = zext <8 x i32> %nbits to <8 x i64>
+  %t1 = shl <8 x i64> %x, %t0
+  %t2 = add <8 x i32> %nbits, <i32 -32, i32 -1, i32 0, i32 1, i32 31, i32 32, i32 undef, i32 64>
+  %t3 = ashr <8 x i64> %t1, %t0
+
+  call void @use8xi64(<8 x i64> %t0)
+  call void @use8xi64(<8 x i64> %t1)
+  call void @use8xi32(<8 x i32> %t2)
+  call void @use8xi64(<8 x i64> %t3)
+
+  %t4 = trunc <8 x i64> %t3 to <8 x i32>
+  %t5 = shl <8 x i32> %t4, %t2
+  ret <8 x i32> %t5
+}
+
+; Extra uses.
+
+define i32 @n4_extrause(i64 %x, i32 %nbits) {
+; CHECK-LABEL: @n4_extrause(
+; CHECK-NEXT:    [[T0:%.*]] = zext i32 [[NBITS:%.*]] to i64
+; CHECK-NEXT:    [[T1:%.*]] = shl i64 [[X:%.*]], [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[NBITS]], -32
+; CHECK-NEXT:    [[T3:%.*]] = ashr i64 [[T1]], [[T0]]
+; CHECK-NEXT:    call void @use64(i64 [[T0]])
+; CHECK-NEXT:    call void @use64(i64 [[T1]])
+; CHECK-NEXT:    call void @use32(i32 [[T2]])
+; CHECK-NEXT:    call void @use64(i64 [[T3]])
+; CHECK-NEXT:    [[T4:%.*]] = trunc i64 [[T3]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[T4]])
+; CHECK-NEXT:    [[T5:%.*]] = shl i32 [[T4]], [[T2]]
+; CHECK-NEXT:    ret i32 [[T5]]
+;
+  %t0 = zext i32 %nbits to i64
+  %t1 = shl i64 %x, %t0
+  %t2 = add i32 %nbits, -32
+  %t3 = ashr i64 %t1, %t0
+
+  call void @use64(i64 %t0)
+  call void @use64(i64 %t1)
+  call void @use32(i32 %t2)
+  call void @use64(i64 %t3)
+
+  %t4 = trunc i64 %t3 to i32
+  call void @use32(i32 %t4)
+  %t5 = shl i32 %t4, %t2
+  ret i32 %t5
+}
+
+; If mask is needed - we can't fold.
+
+define i32 @n5_mask(i64 %x, i32 %nbits) {
+; CHECK-LABEL: @n5_mask(
+; CHECK-NEXT:    [[T0:%.*]] = zext i32 [[NBITS:%.*]] to i64
+; CHECK-NEXT:    [[T1:%.*]] = shl i64 [[X:%.*]], [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[NBITS]], -33
+; CHECK-NEXT:    call void @use64(i64 [[T0]])
+; CHECK-NEXT:    call void @use64(i64 [[T1]])
+; CHECK-NEXT:    call void @use32(i32 [[T2]])
+; CHECK-NEXT:    [[T3:%.*]] = ashr i64 [[T1]], [[T0]]
+; CHECK-NEXT:    [[T4:%.*]] = trunc i64 [[T3]] to i32
+; CHECK-NEXT:    [[T5:%.*]] = shl i32 [[T4]], [[T2]]
+; CHECK-NEXT:    ret i32 [[T5]]
+;
+  %t0 = zext i32 %nbits to i64
+  %t1 = shl i64 %x, %t0
+  %t2 = add i32 %nbits, -33
+
+  call void @use64(i64 %t0)
+  call void @use64(i64 %t1)
+  call void @use32(i32 %t2)
+
+  %t3 = ashr i64 %t1, %t0
+  %t4 = trunc i64 %t3 to i32
+  %t5 = shl i32 %t4, %t2
+  ret i32 %t5
+}
diff --git a/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-variant-a.ll b/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-variant-a.ll
index 381b28511..e7145fccc 100644
--- a/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-variant-a.ll
+++ b/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-variant-a.ll
@@ -25,7 +25,7 @@ define i32 @t0_basic(i32 %x, i32 %nbits) {
 ; CHECK-NEXT:    call void @use32(i32 [[T1]])
 ; CHECK-NEXT:    call void @use32(i32 [[T2]])
 ; CHECK-NEXT:    call void @use32(i32 [[T3]])
-; CHECK-NEXT:    [[T4:%.*]] = shl i32 [[T2]], [[T3]]
+; CHECK-NEXT:    [[T4:%.*]] = shl i32 [[X]], [[T3]]
 ; CHECK-NEXT:    ret i32 [[T4]]
 ;
   %t0 = shl i32 1, %nbits
@@ -50,7 +50,7 @@ define i32 @t1_bigger_shift(i32 %x, i32 %nbits) {
 ; CHECK-NEXT:    call void @use32(i32 [[T1]])
 ; CHECK-NEXT:    call void @use32(i32 [[T2]])
 ; CHECK-NEXT:    call void @use32(i32 [[T3]])
-; CHECK-NEXT:    [[T4:%.*]] = shl i32 [[T2]], [[T3]]
+; CHECK-NEXT:    [[T4:%.*]] = shl i32 [[X]], [[T3]]
 ; CHECK-NEXT:    ret i32 [[T4]]
 ;
   %t0 = shl i32 1, %nbits
@@ -77,7 +77,7 @@ define i32 @t2_bigger_mask(i32 %x, i32 %nbits) {
 ; CHECK-NEXT:    call void @use32(i32 [[T2]])
 ; CHECK-NEXT:    call void @use32(i32 [[T3]])
 ; CHECK-NEXT:    call void @use32(i32 [[T4]])
-; CHECK-NEXT:    [[T5:%.*]] = shl i32 [[T3]], [[T4]]
+; CHECK-NEXT:    [[T5:%.*]] = shl i32 [[X]], [[T4]]
 ; CHECK-NEXT:    ret i32 [[T5]]
 ;
   %t0 = add i32 %nbits, 1
@@ -109,7 +109,7 @@ define <3 x i32> @t3_vec_splat(<3 x i32> %x, <3 x i32> %nbits) {
 ; CHECK-NEXT:    call void @use3xi32(<3 x i32> [[T2]])
 ; CHECK-NEXT:    call void @use3xi32(<3 x i32> [[T3]])
 ; CHECK-NEXT:    call void @use3xi32(<3 x i32> [[T4]])
-; CHECK-NEXT:    [[T5:%.*]] = shl <3 x i32> [[T3]], [[T4]]
+; CHECK-NEXT:    [[T5:%.*]] = shl <3 x i32> [[X]], [[T4]]
 ; CHECK-NEXT:    ret <3 x i32> [[T5]]
 ;
   %t0 = add <3 x i32> %nbits, <i32 0, i32 0, i32 0>
@@ -138,7 +138,7 @@ define <3 x i32> @t4_vec_nonsplat(<3 x i32> %x, <3 x i32> %nbits) {
 ; CHECK-NEXT:    call void @use3xi32(<3 x i32> [[T2]])
 ; CHECK-NEXT:    call void @use3xi32(<3 x i32> [[T3]])
 ; CHECK-NEXT:    call void @use3xi32(<3 x i32> [[T4]])
-; CHECK-NEXT:    [[T5:%.*]] = shl <3 x i32> [[T3]], [[T4]]
+; CHECK-NEXT:    [[T5:%.*]] = shl <3 x i32> [[X]], [[T4]]
 ; CHECK-NEXT:    ret <3 x i32> [[T5]]
 ;
   %t0 = add <3 x i32> %nbits, <i32 -1, i32 0, i32 1>
@@ -166,7 +166,7 @@ define <3 x i32> @t5_vec_undef(<3 x i32> %x, <3 x i32> %nbits) {
 ; CHECK-NEXT:    call void @use3xi32(<3 x i32> [[T2]])
 ; CHECK-NEXT:    call void @use3xi32(<3 x i32> [[T3]])
 ; CHECK-NEXT:    call void @use3xi32(<3 x i32> [[T4]])
-; CHECK-NEXT:    [[T5:%.*]] = shl <3 x i32> [[T3]], [[T4]]
+; CHECK-NEXT:    [[T5:%.*]] = shl <3 x i32> [[X]], [[T4]]
 ; CHECK-NEXT:    ret <3 x i32> [[T5]]
 ;
   %t0 = add <3 x i32> %nbits, <i32 0, i32 undef, i32 0>
@@ -198,7 +198,7 @@ define i32 @t6_commutativity0(i32 %nbits) {
 ; CHECK-NEXT:    call void @use32(i32 [[T1]])
 ; CHECK-NEXT:    call void @use32(i32 [[T2]])
 ; CHECK-NEXT:    call void @use32(i32 [[T3]])
-; CHECK-NEXT:    [[T4:%.*]] = shl i32 [[T2]], [[T3]]
+; CHECK-NEXT:    [[T4:%.*]] = shl i32 [[X]], [[T3]]
 ; CHECK-NEXT:    ret i32 [[T4]]
 ;
   %x = call i32 @gen32()
@@ -260,7 +260,7 @@ define i32 @t8_commutativity2(i32 %nbits0, i32 %nbits1) {
 ; CHECK-NEXT:    call void @use32(i32 [[T3]])
 ; CHECK-NEXT:    call void @use32(i32 [[T4]])
 ; CHECK-NEXT:    call void @use32(i32 [[T5]])
-; CHECK-NEXT:    [[T6:%.*]] = shl i32 [[T4]], [[T5]]
+; CHECK-NEXT:    [[T6:%.*]] = shl i32 [[T1]], [[T5]]
 ; CHECK-NEXT:    ret i32 [[T6]]
 ;
   %t0 = shl i32 1, %nbits0
@@ -291,7 +291,7 @@ define i32 @t9_nuw(i32 %x, i32 %nbits) {
 ; CHECK-NEXT:    call void @use32(i32 [[T1]])
 ; CHECK-NEXT:    call void @use32(i32 [[T2]])
 ; CHECK-NEXT:    call void @use32(i32 [[T3]])
-; CHECK-NEXT:    [[T4:%.*]] = shl nuw i32 [[T2]], [[T3]]
+; CHECK-NEXT:    [[T4:%.*]] = shl i32 [[X]], [[T3]]
 ; CHECK-NEXT:    ret i32 [[T4]]
 ;
   %t0 = shl i32 1, %nbits
@@ -316,7 +316,7 @@ define i32 @t10_nsw(i32 %x, i32 %nbits) {
 ; CHECK-NEXT:    call void @use32(i32 [[T1]])
 ; CHECK-NEXT:    call void @use32(i32 [[T2]])
 ; CHECK-NEXT:    call void @use32(i32 [[T3]])
-; CHECK-NEXT:    [[T4:%.*]] = shl nsw i32 [[T2]], [[T3]]
+; CHECK-NEXT:    [[T4:%.*]] = shl i32 [[X]], [[T3]]
 ; CHECK-NEXT:    ret i32 [[T4]]
 ;
   %t0 = shl i32 1, %nbits
@@ -341,7 +341,7 @@ define i32 @t11_nuw_nsw(i32 %x, i32 %nbits) {
 ; CHECK-NEXT:    call void @use32(i32 [[T1]])
 ; CHECK-NEXT:    call void @use32(i32 [[T2]])
 ; CHECK-NEXT:    call void @use32(i32 [[T3]])
-; CHECK-NEXT:    [[T4:%.*]] = shl nuw nsw i32 [[T2]], [[T3]]
+; CHECK-NEXT:    [[T4:%.*]] = shl i32 [[X]], [[T3]]
 ; CHECK-NEXT:    ret i32 [[T4]]
 ;
   %t0 = shl i32 1, %nbits
@@ -407,28 +407,3 @@ define i32 @n13_not_minus_one(i32 %x, i32 %nbits) {
   %t4 = shl i32 %t2, %t3
   ret i32 %t4
 }
-
-define i32 @n14_insifficient_sum(i32 %x, i32 %nbits) {
-; CHECK-LABEL: @n14_insifficient_sum(
-; CHECK-NEXT:    [[T0:%.*]] = shl i32 1, [[NBITS:%.*]]
-; CHECK-NEXT:    [[T1:%.*]] = add nsw i32 [[T0]], -1
-; CHECK-NEXT:    [[T2:%.*]] = and i32 [[T1]], [[X:%.*]]
-; CHECK-NEXT:    [[T3:%.*]] = sub i32 31, [[NBITS]]
-; CHECK-NEXT:    call void @use32(i32 [[T0]])
-; CHECK-NEXT:    call void @use32(i32 [[T1]])
-; CHECK-NEXT:    call void @use32(i32 [[T2]])
-; CHECK-NEXT:    call void @use32(i32 [[T3]])
-; CHECK-NEXT:    [[T4:%.*]] = shl i32 [[T2]], [[T3]]
-; CHECK-NEXT:    ret i32 [[T4]]
-;
-  %t0 = shl i32 1, %nbits
-  %t1 = add nsw i32 %t0, -1
-  %t2 = and i32 %t1, %x
-  %t3 = sub i32 31, %nbits ; summary shift amount is less than 32
-  call void @use32(i32 %t0)
-  call void @use32(i32 %t1)
-  call void @use32(i32 %t2)
-  call void @use32(i32 %t3)
-  %t4 = shl i32 %t2, %t3
-  ret i32 %t4
-}
diff --git a/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-variant-b.ll b/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-variant-b.ll
index ac415438d..94e786d38 100644
--- a/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-variant-b.ll
+++ b/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-variant-b.ll
@@ -25,7 +25,7 @@ define i32 @t0_basic(i32 %x, i32 %nbits) {
 ; CHECK-NEXT:    call void @use32(i32 [[T1]])
 ; CHECK-NEXT:    call void @use32(i32 [[T2]])
 ; CHECK-NEXT:    call void @use32(i32 [[T3]])
-; CHECK-NEXT:    [[T4:%.*]] = shl i32 [[T2]], [[T3]]
+; CHECK-NEXT:    [[T4:%.*]] = shl i32 [[X]], [[T3]]
 ; CHECK-NEXT:    ret i32 [[T4]]
 ;
   %t0 = shl i32 -1, %nbits
@@ -50,7 +50,7 @@ define i32 @t1_bigger_shift(i32 %x, i32 %nbits) {
 ; CHECK-NEXT:    call void @use32(i32 [[T1]])
 ; CHECK-NEXT:    call void @use32(i32 [[T2]])
 ; CHECK-NEXT:    call void @use32(i32 [[T3]])
-; CHECK-NEXT:    [[T4:%.*]] = shl i32 [[T2]], [[T3]]
+; CHECK-NEXT:    [[T4:%.*]] = shl i32 [[X]], [[T3]]
 ; CHECK-NEXT:    ret i32 [[T4]]
 ;
   %t0 = shl i32 -1, %nbits
@@ -77,7 +77,7 @@ define i32 @t2_bigger_mask(i32 %x, i32 %nbits) {
 ; CHECK-NEXT:    call void @use32(i32 [[T2]])
 ; CHECK-NEXT:    call void @use32(i32 [[T3]])
 ; CHECK-NEXT:    call void @use32(i32 [[T4]])
-; CHECK-NEXT:    [[T5:%.*]] = shl i32 [[T3]], [[T4]]
+; CHECK-NEXT:    [[T5:%.*]] = shl i32 [[X]], [[T4]]
 ; CHECK-NEXT:    ret i32 [[T5]]
 ;
   %t0 = add i32 %nbits, 1
@@ -109,7 +109,7 @@ define <3 x i32> @t3_vec_splat(<3 x i32> %x, <3 x i32> %nbits) {
 ; CHECK-NEXT:    call void @use3xi32(<3 x i32> [[T2]])
 ; CHECK-NEXT:    call void @use3xi32(<3 x i32> [[T3]])
 ; CHECK-NEXT:    call void @use3xi32(<3 x i32> [[T4]])
-; CHECK-NEXT:    [[T5:%.*]] = shl <3 x i32> [[T3]], [[T4]]
+; CHECK-NEXT:    [[T5:%.*]] = shl <3 x i32> [[X]], [[T4]]
 ; CHECK-NEXT:    ret <3 x i32> [[T5]]
 ;
   %t0 = add <3 x i32> %nbits, <i32 0, i32 0, i32 0>
@@ -138,7 +138,7 @@ define <3 x i32> @t4_vec_nonsplat(<3 x i32> %x, <3 x i32> %nbits) {
 ; CHECK-NEXT:    call void @use3xi32(<3 x i32> [[T2]])
 ; CHECK-NEXT:    call void @use3xi32(<3 x i32> [[T3]])
 ; CHECK-NEXT:    call void @use3xi32(<3 x i32> [[T4]])
-; CHECK-NEXT:    [[T5:%.*]] = shl <3 x i32> [[T3]], [[T4]]
+; CHECK-NEXT:    [[T5:%.*]] = shl <3 x i32> [[X]], [[T4]]
 ; CHECK-NEXT:    ret <3 x i32> [[T5]]
 ;
   %t0 = add <3 x i32> %nbits, <i32 -1, i32 0, i32 1>
@@ -166,7 +166,7 @@ define <3 x i32> @t5_vec_undef(<3 x i32> %x, <3 x i32> %nbits) {
 ; CHECK-NEXT:    call void @use3xi32(<3 x i32> [[T2]])
 ; CHECK-NEXT:    call void @use3xi32(<3 x i32> [[T3]])
 ; CHECK-NEXT:    call void @use3xi32(<3 x i32> [[T4]])
-; CHECK-NEXT:    [[T5:%.*]] = shl <3 x i32> [[T3]], [[T4]]
+; CHECK-NEXT:    [[T5:%.*]] = shl <3 x i32> [[X]], [[T4]]
 ; CHECK-NEXT:    ret <3 x i32> [[T5]]
 ;
   %t0 = add <3 x i32> %nbits, <i32 0, i32 undef, i32 0>
@@ -198,7 +198,7 @@ define i32 @t6_commutativity0(i32 %nbits) {
 ; CHECK-NEXT:    call void @use32(i32 [[T1]])
 ; CHECK-NEXT:    call void @use32(i32 [[T2]])
 ; CHECK-NEXT:    call void @use32(i32 [[T3]])
-; CHECK-NEXT:    [[T4:%.*]] = shl i32 [[T2]], [[T3]]
+; CHECK-NEXT:    [[T4:%.*]] = shl i32 [[X]], [[T3]]
 ; CHECK-NEXT:    ret i32 [[T4]]
 ;
   %x = call i32 @gen32()
@@ -260,7 +260,7 @@ define i32 @t8_commutativity2(i32 %nbits0, i32 %nbits1) {
 ; CHECK-NEXT:    call void @use32(i32 [[T3]])
 ; CHECK-NEXT:    call void @use32(i32 [[T4]])
 ; CHECK-NEXT:    call void @use32(i32 [[T5]])
-; CHECK-NEXT:    [[T6:%.*]] = shl i32 [[T4]], [[T5]]
+; CHECK-NEXT:    [[T6:%.*]] = shl i32 [[T1]], [[T5]]
 ; CHECK-NEXT:    ret i32 [[T6]]
 ;
   %t0 = shl i32 -1, %nbits0
@@ -291,7 +291,7 @@ define i32 @t9_nuw(i32 %x, i32 %nbits) {
 ; CHECK-NEXT:    call void @use32(i32 [[T1]])
 ; CHECK-NEXT:    call void @use32(i32 [[T2]])
 ; CHECK-NEXT:    call void @use32(i32 [[T3]])
-; CHECK-NEXT:    [[T4:%.*]] = shl nuw i32 [[T2]], [[T3]]
+; CHECK-NEXT:    [[T4:%.*]] = shl i32 [[X]], [[T3]]
 ; CHECK-NEXT:    ret i32 [[T4]]
 ;
   %t0 = shl i32 -1, %nbits
@@ -316,7 +316,7 @@ define i32 @t10_nsw(i32 %x, i32 %nbits) {
 ; CHECK-NEXT:    call void @use32(i32 [[T1]])
 ; CHECK-NEXT:    call void @use32(i32 [[T2]])
 ; CHECK-NEXT:    call void @use32(i32 [[T3]])
-; CHECK-NEXT:    [[T4:%.*]] = shl nsw i32 [[T2]], [[T3]]
+; CHECK-NEXT:    [[T4:%.*]] = shl i32 [[X]], [[T3]]
 ; CHECK-NEXT:    ret i32 [[T4]]
 ;
   %t0 = shl i32 -1, %nbits
@@ -341,7 +341,7 @@ define i32 @t11_nuw_nsw(i32 %x, i32 %nbits) {
 ; CHECK-NEXT:    call void @use32(i32 [[T1]])
 ; CHECK-NEXT:    call void @use32(i32 [[T2]])
 ; CHECK-NEXT:    call void @use32(i32 [[T3]])
-; CHECK-NEXT:    [[T4:%.*]] = shl nuw nsw i32 [[T2]], [[T3]]
+; CHECK-NEXT:    [[T4:%.*]] = shl i32 [[X]], [[T3]]
 ; CHECK-NEXT:    ret i32 [[T4]]
 ;
   %t0 = shl i32 -1, %nbits
@@ -382,28 +382,3 @@ define i32 @n12_not_minus_one(i32 %x, i32 %nbits) {
   %t4 = shl i32 %t2, %t3
   ret i32 %t4
 }
-
-define i32 @n13_insufficient_sum(i32 %x, i32 %nbits) {
-; CHECK-LABEL: @n13_insufficient_sum(
-; CHECK-NEXT:    [[T0:%.*]] = shl i32 -1, [[NBITS:%.*]]
-; CHECK-NEXT:    [[T1:%.*]] = xor i32 [[T0]], -1
-; CHECK-NEXT:    [[T2:%.*]] = and i32 [[T1]], [[X:%.*]]
-; CHECK-NEXT:    [[T3:%.*]] = sub i32 31, [[NBITS]]
-; CHECK-NEXT:    call void @use32(i32 [[T0]])
-; CHECK-NEXT:    call void @use32(i32 [[T1]])
-; CHECK-NEXT:    call void @use32(i32 [[T2]])
-; CHECK-NEXT:    call void @use32(i32 [[T3]])
-; CHECK-NEXT:    [[T4:%.*]] = shl i32 [[T2]], [[T3]]
-; CHECK-NEXT:    ret i32 [[T4]]
-;
-  %t0 = shl i32 -1, %nbits
-  %t1 = xor i32 %t0, -1
-  %t2 = and i32 %t1, %x
-  %t3 = sub i32 31, %nbits ; summary shift amount is less than 32
-  call void @use32(i32 %t0)
-  call void @use32(i32 %t1)
-  call void @use32(i32 %t2)
-  call void @use32(i32 %t3)
-  %t4 = shl i32 %t2, %t3
-  ret i32 %t4
-}
diff --git a/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-variant-c.ll b/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-variant-c.ll
index 453458197..fb480e3f3 100644
--- a/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-variant-c.ll
+++ b/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-variant-c.ll
@@ -21,7 +21,7 @@ define i32 @t0_basic(i32 %x, i32 %nbits) {
 ; CHECK-NEXT:    [[T1:%.*]] = and i32 [[T0]], [[X:%.*]]
 ; CHECK-NEXT:    call void @use32(i32 [[T0]])
 ; CHECK-NEXT:    call void @use32(i32 [[T1]])
-; CHECK-NEXT:    [[T2:%.*]] = shl i32 [[T1]], [[NBITS]]
+; CHECK-NEXT:    [[T2:%.*]] = shl i32 [[X]], [[NBITS]]
 ; CHECK-NEXT:    ret i32 [[T2]]
 ;
   %t0 = lshr i32 -1, %nbits
@@ -40,7 +40,7 @@ define i32 @t1_bigger_shift(i32 %x, i32 %nbits) {
 ; CHECK-NEXT:    call void @use32(i32 [[T0]])
 ; CHECK-NEXT:    call void @use32(i32 [[T1]])
 ; CHECK-NEXT:    call void @use32(i32 [[T2]])
-; CHECK-NEXT:    [[T3:%.*]] = shl i32 [[T1]], [[T2]]
+; CHECK-NEXT:    [[T3:%.*]] = shl i32 [[X]], [[T2]]
 ; CHECK-NEXT:    ret i32 [[T3]]
 ;
   %t0 = lshr i32 -1, %nbits
@@ -65,7 +65,7 @@ define <3 x i32> @t2_vec_splat(<3 x i32> %x, <3 x i32> %nbits) {
 ; CHECK-NEXT:    call void @use3xi32(<3 x i32> [[T0]])
 ; CHECK-NEXT:    call void @use3xi32(<3 x i32> [[T1]])
 ; CHECK-NEXT:    call void @use3xi32(<3 x i32> [[T2]])
-; CHECK-NEXT:    [[T3:%.*]] = shl <3 x i32> [[T1]], [[T2]]
+; CHECK-NEXT:    [[T3:%.*]] = shl <3 x i32> [[X]], [[T2]]
 ; CHECK-NEXT:    ret <3 x i32> [[T3]]
 ;
   %t0 = lshr <3 x i32> <i32 -1, i32 -1, i32 -1>, %nbits
@@ -86,7 +86,7 @@ define <3 x i32> @t3_vec_nonsplat(<3 x i32> %x, <3 x i32> %nbits) {
 ; CHECK-NEXT:    call void @use3xi32(<3 x i32> [[T0]])
 ; CHECK-NEXT:    call void @use3xi32(<3 x i32> [[T1]])
 ; CHECK-NEXT:    call void @use3xi32(<3 x i32> [[T2]])
-; CHECK-NEXT:    [[T3:%.*]] = shl <3 x i32> [[T1]], [[T2]]
+; CHECK-NEXT:    [[T3:%.*]] = shl <3 x i32> [[X]], [[T2]]
 ; CHECK-NEXT:    ret <3 x i32> [[T3]]
 ;
   %t0 = lshr <3 x i32> <i32 -1, i32 -1, i32 -1>, %nbits
@@ -107,7 +107,7 @@ define <3 x i32> @t4_vec_undef(<3 x i32> %x, <3 x i32> %nbits) {
 ; CHECK-NEXT:    call void @use3xi32(<3 x i32> [[T0]])
 ; CHECK-NEXT:    call void @use3xi32(<3 x i32> [[T1]])
 ; CHECK-NEXT:    call void @use3xi32(<3 x i32> [[T2]])
-; CHECK-NEXT:    [[T3:%.*]] = shl <3 x i32> [[T1]], [[T2]]
+; CHECK-NEXT:    [[T3:%.*]] = shl <3 x i32> [[X]], [[T2]]
 ; CHECK-NEXT:    ret <3 x i32> [[T3]]
 ;
   %t0 = lshr <3 x i32> <i32 -1, i32 undef, i32 -1>, %nbits
@@ -131,7 +131,7 @@ define i32 @t5_commutativity0(i32 %nbits) {
 ; CHECK-NEXT:    [[T1:%.*]] = and i32 [[X]], [[T0]]
 ; CHECK-NEXT:    call void @use32(i32 [[T0]])
 ; CHECK-NEXT:    call void @use32(i32 [[T1]])
-; CHECK-NEXT:    [[T2:%.*]] = shl i32 [[T1]], [[NBITS]]
+; CHECK-NEXT:    [[T2:%.*]] = shl i32 [[X]], [[NBITS]]
 ; CHECK-NEXT:    ret i32 [[T2]]
 ;
   %x = call i32 @gen32()
@@ -151,7 +151,7 @@ define i32 @t6_commutativity1(i32 %nbits0, i32 %nbits1) {
 ; CHECK-NEXT:    call void @use32(i32 [[T0]])
 ; CHECK-NEXT:    call void @use32(i32 [[T1]])
 ; CHECK-NEXT:    call void @use32(i32 [[T2]])
-; CHECK-NEXT:    [[T3:%.*]] = shl i32 [[T2]], [[NBITS0]]
+; CHECK-NEXT:    [[T3:%.*]] = shl i32 [[T1]], [[NBITS0]]
 ; CHECK-NEXT:    ret i32 [[T3]]
 ;
   %t0 = lshr i32 -1, %nbits0
@@ -192,7 +192,7 @@ define i32 @t8_nuw(i32 %x, i32 %nbits) {
 ; CHECK-NEXT:    [[T1:%.*]] = and i32 [[T0]], [[X:%.*]]
 ; CHECK-NEXT:    call void @use32(i32 [[T0]])
 ; CHECK-NEXT:    call void @use32(i32 [[T1]])
-; CHECK-NEXT:    [[T2:%.*]] = shl nuw i32 [[T1]], [[NBITS]]
+; CHECK-NEXT:    [[T2:%.*]] = shl i32 [[X]], [[NBITS]]
 ; CHECK-NEXT:    ret i32 [[T2]]
 ;
   %t0 = lshr i32 -1, %nbits
@@ -209,7 +209,7 @@ define i32 @t9_nsw(i32 %x, i32 %nbits) {
 ; CHECK-NEXT:    [[T1:%.*]] = and i32 [[T0]], [[X:%.*]]
 ; CHECK-NEXT:    call void @use32(i32 [[T0]])
 ; CHECK-NEXT:    call void @use32(i32 [[T1]])
-; CHECK-NEXT:    [[T2:%.*]] = shl nsw i32 [[T1]], [[NBITS]]
+; CHECK-NEXT:    [[T2:%.*]] = shl i32 [[X]], [[NBITS]]
 ; CHECK-NEXT:    ret i32 [[T2]]
 ;
   %t0 = lshr i32 -1, %nbits
@@ -226,7 +226,7 @@ define i32 @t10_nuw_nsw(i32 %x, i32 %nbits) {
 ; CHECK-NEXT:    [[T1:%.*]] = and i32 [[T0]], [[X:%.*]]
 ; CHECK-NEXT:    call void @use32(i32 [[T0]])
 ; CHECK-NEXT:    call void @use32(i32 [[T1]])
-; CHECK-NEXT:    [[T2:%.*]] = shl nuw nsw i32 [[T1]], [[NBITS]]
+; CHECK-NEXT:    [[T2:%.*]] = shl i32 [[X]], [[NBITS]]
 ; CHECK-NEXT:    ret i32 [[T2]]
 ;
   %t0 = lshr i32 -1, %nbits
@@ -237,41 +237,47 @@ define i32 @t10_nuw_nsw(i32 %x, i32 %nbits) {
   ret i32 %t2
 }
 
-; Negative tests
+; Special test
 
-define i32 @n11_not_minus_one(i32 %x, i32 %nbits) {
-; CHECK-LABEL: @n11_not_minus_one(
-; CHECK-NEXT:    [[T0:%.*]] = lshr i32 -2, [[NBITS:%.*]]
+declare void @llvm.assume(i1 %cond)
+
+; We can't simplify (%shiftnbits-%masknbits) but we have an assumption.
+define i32 @t11_assume_uge(i32 %x, i32 %masknbits, i32 %shiftnbits) {
+; CHECK-LABEL: @t11_assume_uge(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp uge i32 [[SHIFTNBITS:%.*]], [[MASKNBITS:%.*]]
+; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    [[T0:%.*]] = lshr i32 -1, [[MASKNBITS]]
 ; CHECK-NEXT:    [[T1:%.*]] = and i32 [[T0]], [[X:%.*]]
 ; CHECK-NEXT:    call void @use32(i32 [[T0]])
 ; CHECK-NEXT:    call void @use32(i32 [[T1]])
-; CHECK-NEXT:    [[T2:%.*]] = shl i32 [[T1]], [[NBITS]]
+; CHECK-NEXT:    [[T2:%.*]] = shl i32 [[T1]], [[SHIFTNBITS]]
 ; CHECK-NEXT:    ret i32 [[T2]]
 ;
-  %t0 = lshr i32 -2, %nbits ; shifting not '-1'
+  %cmp = icmp uge i32 %shiftnbits, %masknbits
+  call void @llvm.assume(i1 %cmp)
+  %t0 = lshr i32 -1, %masknbits
   %t1 = and i32 %t0, %x
   call void @use32(i32 %t0)
   call void @use32(i32 %t1)
-  %t2 = shl i32 %t1, %nbits
+  %t2 = shl i32 %t1, %shiftnbits
   ret i32 %t2
 }
 
-define i32 @n12_shamt_is_smaller(i32 %x, i32 %nbits) {
-; CHECK-LABEL: @n12_shamt_is_smaller(
-; CHECK-NEXT:    [[T0:%.*]] = lshr i32 -1, [[NBITS:%.*]]
+; Negative tests
+
+define i32 @n12_not_minus_one(i32 %x, i32 %nbits) {
+; CHECK-LABEL: @n12_not_minus_one(
+; CHECK-NEXT:    [[T0:%.*]] = lshr i32 -2, [[NBITS:%.*]]
 ; CHECK-NEXT:    [[T1:%.*]] = and i32 [[T0]], [[X:%.*]]
-; CHECK-NEXT:    [[T2:%.*]] = add i32 [[NBITS]], -1
 ; CHECK-NEXT:    call void @use32(i32 [[T0]])
 ; CHECK-NEXT:    call void @use32(i32 [[T1]])
-; CHECK-NEXT:    call void @use32(i32 [[T2]])
+; CHECK-NEXT:    [[T2:%.*]] = shl i32 [[T1]], [[NBITS]]
 ; CHECK-NEXT:    ret i32 [[T2]]
 ;
-  %t0 = lshr i32 -1, %nbits
+  %t0 = lshr i32 -2, %nbits ; shifting not '-1'
   %t1 = and i32 %t0, %x
-  %t2 = add i32 %nbits, -1
   call void @use32(i32 %t0)
   call void @use32(i32 %t1)
-  call void @use32(i32 %t2)
-  %t3 = shl i32 %t1, %t2 ; shift is smaller than mask
+  %t2 = shl i32 %t1, %nbits
   ret i32 %t2
 }
diff --git a/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-variant-d.ll b/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-variant-d.ll
index 601e14927..0871e82dd 100644
--- a/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-variant-d.ll
+++ b/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-variant-d.ll
@@ -23,7 +23,7 @@ define i32 @t0_basic(i32 %x, i32 %nbits) {
 ; CHECK-NEXT:    call void @use32(i32 [[T0]])
 ; CHECK-NEXT:    call void @use32(i32 [[T1]])
 ; CHECK-NEXT:    call void @use32(i32 [[T2]])
-; CHECK-NEXT:    [[T4:%.*]] = shl i32 [[T2]], [[NBITS]]
+; CHECK-NEXT:    [[T4:%.*]] = shl i32 [[X]], [[NBITS]]
 ; CHECK-NEXT:    ret i32 [[T4]]
 ;
   %t0 = shl i32 -1, %nbits
@@ -46,7 +46,7 @@ define i32 @t1_bigger_shift(i32 %x, i32 %nbits) {
 ; CHECK-NEXT:    call void @use32(i32 [[T1]])
 ; CHECK-NEXT:    call void @use32(i32 [[T2]])
 ; CHECK-NEXT:    call void @use32(i32 [[T3]])
-; CHECK-NEXT:    [[T4:%.*]] = shl i32 [[T2]], [[T3]]
+; CHECK-NEXT:    [[T4:%.*]] = shl i32 [[X]], [[T3]]
 ; CHECK-NEXT:    ret i32 [[T4]]
 ;
   %t0 = shl i32 -1, %nbits
@@ -75,7 +75,7 @@ define <3 x i32> @t2_vec_splat(<3 x i32> %x, <3 x i32> %nbits) {
 ; CHECK-NEXT:    call void @use3xi32(<3 x i32> [[T1]])
 ; CHECK-NEXT:    call void @use3xi32(<3 x i32> [[T2]])
 ; CHECK-NEXT:    call void @use3xi32(<3 x i32> [[T3]])
-; CHECK-NEXT:    [[T4:%.*]] = shl <3 x i32> [[T2]], [[T3]]
+; CHECK-NEXT:    [[T4:%.*]] = shl <3 x i32> [[X]], [[T3]]
 ; CHECK-NEXT:    ret <3 x i32> [[T4]]
 ;
   %t0 = shl <3 x i32> <i32 -1, i32 -1, i32 -1>, %nbits
@@ -100,7 +100,7 @@ define <3 x i32> @t3_vec_nonsplat(<3 x i32> %x, <3 x i32> %nbits) {
 ; CHECK-NEXT:    call void @use3xi32(<3 x i32> [[T1]])
 ; CHECK-NEXT:    call void @use3xi32(<3 x i32> [[T2]])
 ; CHECK-NEXT:    call void @use3xi32(<3 x i32> [[T3]])
-; CHECK-NEXT:    [[T4:%.*]] = shl <3 x i32> [[T2]], [[T3]]
+; CHECK-NEXT:    [[T4:%.*]] = shl <3 x i32> [[X]], [[T3]]
 ; CHECK-NEXT:    ret <3 x i32> [[T4]]
 ;
   %t0 = shl <3 x i32> <i32 -1, i32 -1, i32 -1>, %nbits
@@ -124,7 +124,7 @@ define <3 x i32> @t4_vec_undef(<3 x i32> %x, <3 x i32> %nbits) {
 ; CHECK-NEXT:    call void @use3xi32(<3 x i32> [[T1]])
 ; CHECK-NEXT:    call void @use3xi32(<3 x i32> [[T2]])
 ; CHECK-NEXT:    call void @use3xi32(<3 x i32> [[NBITS]])
-; CHECK-NEXT:    [[T4:%.*]] = shl <3 x i32> [[T2]], [[NBITS]]
+; CHECK-NEXT:    [[T4:%.*]] = shl <3 x i32> [[X]], [[NBITS]]
 ; CHECK-NEXT:    ret <3 x i32> [[T4]]
 ;
   %t0 = shl <3 x i32> <i32 -1, i32 undef, i32 -1>, %nbits
@@ -152,7 +152,7 @@ define i32 @t5_commutativity0(i32 %nbits) {
 ; CHECK-NEXT:    call void @use32(i32 [[T0]])
 ; CHECK-NEXT:    call void @use32(i32 [[T1]])
 ; CHECK-NEXT:    call void @use32(i32 [[T2]])
-; CHECK-NEXT:    [[T3:%.*]] = shl i32 [[T2]], [[NBITS]]
+; CHECK-NEXT:    [[T3:%.*]] = shl i32 [[X]], [[NBITS]]
 ; CHECK-NEXT:    ret i32 [[T3]]
 ;
   %x = call i32 @gen32()
@@ -178,7 +178,7 @@ define i32 @t6_commutativity1(i32 %nbits0, i32 %nbits1) {
 ; CHECK-NEXT:    call void @use32(i32 [[T2]])
 ; CHECK-NEXT:    call void @use32(i32 [[T3]])
 ; CHECK-NEXT:    call void @use32(i32 [[T4]])
-; CHECK-NEXT:    [[T5:%.*]] = shl i32 [[T4]], [[NBITS0]]
+; CHECK-NEXT:    [[T5:%.*]] = shl i32 [[T3]], [[NBITS0]]
 ; CHECK-NEXT:    ret i32 [[T5]]
 ;
   %t0 = shl i32 -1, %nbits0
@@ -233,7 +233,7 @@ define i32 @t8_nuw(i32 %x, i32 %nbits) {
 ; CHECK-NEXT:    call void @use32(i32 [[T0]])
 ; CHECK-NEXT:    call void @use32(i32 [[T1]])
 ; CHECK-NEXT:    call void @use32(i32 [[T2]])
-; CHECK-NEXT:    [[T3:%.*]] = shl nuw i32 [[T2]], [[NBITS]]
+; CHECK-NEXT:    [[T3:%.*]] = shl i32 [[X]], [[NBITS]]
 ; CHECK-NEXT:    ret i32 [[T3]]
 ;
   %t0 = shl i32 -1, %nbits
@@ -254,7 +254,7 @@ define i32 @t9_nsw(i32 %x, i32 %nbits) {
 ; CHECK-NEXT:    call void @use32(i32 [[T0]])
 ; CHECK-NEXT:    call void @use32(i32 [[T1]])
 ; CHECK-NEXT:    call void @use32(i32 [[T2]])
-; CHECK-NEXT:    [[T3:%.*]] = shl nsw i32 [[T2]], [[NBITS]]
+; CHECK-NEXT:    [[T3:%.*]] = shl i32 [[X]], [[NBITS]]
 ; CHECK-NEXT:    ret i32 [[T3]]
 ;
   %t0 = shl i32 -1, %nbits
@@ -275,7 +275,7 @@ define i32 @t10_nuw_nsw(i32 %x, i32 %nbits) {
 ; CHECK-NEXT:    call void @use32(i32 [[T0]])
 ; CHECK-NEXT:    call void @use32(i32 [[T1]])
 ; CHECK-NEXT:    call void @use32(i32 [[T2]])
-; CHECK-NEXT:    [[T3:%.*]] = shl nuw nsw i32 [[T2]], [[NBITS]]
+; CHECK-NEXT:    [[T3:%.*]] = shl i32 [[X]], [[NBITS]]
 ; CHECK-NEXT:    ret i32 [[T3]]
 ;
   %t0 = shl i32 -1, %nbits
@@ -288,10 +288,40 @@ define i32 @t10_nuw_nsw(i32 %x, i32 %nbits) {
   ret i32 %t3
 }
 
+; Special test
+
+declare void @llvm.assume(i1 %cond)
+
+; We can't simplify (%shiftnbits-%masknbits) but we have an assumption.
+define i32 @t11_assume_uge(i32 %x, i32 %masknbits, i32 %shiftnbits) {
+; CHECK-LABEL: @t11_assume_uge(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp uge i32 [[SHIFTNBITS:%.*]], [[MASKNBITS:%.*]]
+; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    [[T0:%.*]] = shl i32 -1, [[MASKNBITS]]
+; CHECK-NEXT:    [[T1:%.*]] = lshr i32 [[T0]], [[MASKNBITS]]
+; CHECK-NEXT:    [[T2:%.*]] = and i32 [[T1]], [[X:%.*]]
+; CHECK-NEXT:    call void @use32(i32 [[T0]])
+; CHECK-NEXT:    call void @use32(i32 [[T1]])
+; CHECK-NEXT:    call void @use32(i32 [[T2]])
+; CHECK-NEXT:    [[T4:%.*]] = shl i32 [[T2]], [[SHIFTNBITS]]
+; CHECK-NEXT:    ret i32 [[T4]]
+;
+  %cmp = icmp uge i32 %shiftnbits, %masknbits
+  call void @llvm.assume(i1 %cmp)
+  %t0 = shl i32 -1, %masknbits
+  %t1 = lshr i32 %t0, %masknbits
+  %t2 = and i32 %t1, %x
+  call void @use32(i32 %t0)
+  call void @use32(i32 %t1)
+  call void @use32(i32 %t2)
+  %t4 = shl i32 %t2, %shiftnbits
+  ret i32 %t4
+}
+
 ; Negative tests
 
-define i32 @n11_different_shamts0(i32 %x, i32 %nbits0, i32 %nbits1) {
-; CHECK-LABEL: @n11_different_shamts0(
+define i32 @n12_different_shamts0(i32 %x, i32 %nbits0, i32 %nbits1) {
+; CHECK-LABEL: @n12_different_shamts0(
 ; CHECK-NEXT:    [[T0:%.*]] = shl i32 [[X:%.*]], [[NBITS0:%.*]]
 ; CHECK-NEXT:    [[T1:%.*]] = lshr i32 [[T0]], [[NBITS1:%.*]]
 ; CHECK-NEXT:    [[T2:%.*]] = and i32 [[T1]], [[X]]
@@ -311,8 +341,8 @@ define i32 @n11_different_shamts0(i32 %x, i32 %nbits0, i32 %nbits1) {
   ret i32 %t3
 }
 
-define i32 @n12_different_shamts1(i32 %x, i32 %nbits0, i32 %nbits1) {
-; CHECK-LABEL: @n12_different_shamts1(
+define i32 @n13_different_shamts1(i32 %x, i32 %nbits0, i32 %nbits1) {
+; CHECK-LABEL: @n13_different_shamts1(
 ; CHECK-NEXT:    [[T0:%.*]] = shl i32 [[X:%.*]], [[NBITS0:%.*]]
 ; CHECK-NEXT:    [[T1:%.*]] = lshr i32 [[T0]], [[NBITS1:%.*]]
 ; CHECK-NEXT:    [[T2:%.*]] = and i32 [[T1]], [[X]]
@@ -331,26 +361,3 @@ define i32 @n12_different_shamts1(i32 %x, i32 %nbits0, i32 %nbits1) {
   %t3 = shl i32 %t2, %nbits1
   ret i32 %t3
 }
-
-define i32 @n13_shamt_is_smaller(i32 %x, i32 %nbits) {
-; CHECK-LABEL: @n13_shamt_is_smaller(
-; CHECK-NEXT:    [[T0:%.*]] = shl i32 -1, [[NBITS:%.*]]
-; CHECK-NEXT:    [[T1:%.*]] = lshr i32 [[T0]], [[NBITS]]
-; CHECK-NEXT:    [[T2:%.*]] = and i32 [[T1]], [[X:%.*]]
-; CHECK-NEXT:    [[T3:%.*]] = add i32 [[NBITS]], -1
-; CHECK-NEXT:    call void @use32(i32 [[T0]])
-; CHECK-NEXT:    call void @use32(i32 [[T1]])
-; CHECK-NEXT:    call void @use32(i32 [[T2]])
-; CHECK-NEXT:    [[T4:%.*]] = shl i32 [[T2]], [[T3]]
-; CHECK-NEXT:    ret i32 [[T4]]
-;
-  %t0 = shl i32 -1, %nbits
-  %t1 = lshr i32 %t0, %nbits
-  %t2 = and i32 %t1, %x
-  %t3 = add i32 %nbits, -1 ; shift is smaller than mask
-  call void @use32(i32 %t0)
-  call void @use32(i32 %t1)
-  call void @use32(i32 %t2)
-  %t4 = shl i32 %t2, %t3
-  ret i32 %t4
-}
diff --git a/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-variant-e.ll b/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-variant-e.ll
index 691db006a..f3682ef8c 100644
--- a/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-variant-e.ll
+++ b/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-variant-e.ll
@@ -21,7 +21,7 @@ define i32 @t0_basic(i32 %x, i32 %nbits) {
 ; CHECK-NEXT:    [[T1:%.*]] = lshr i32 [[T0]], [[NBITS]]
 ; CHECK-NEXT:    call void @use32(i32 [[T0]])
 ; CHECK-NEXT:    call void @use32(i32 [[T1]])
-; CHECK-NEXT:    [[T2:%.*]] = shl i32 [[T1]], [[NBITS]]
+; CHECK-NEXT:    [[T2:%.*]] = shl i32 [[X]], [[NBITS]]
 ; CHECK-NEXT:    ret i32 [[T2]]
 ;
   %t0 = shl i32 %x, %nbits
@@ -40,7 +40,7 @@ define i32 @t1_bigger_shift(i32 %x, i32 %nbits) {
 ; CHECK-NEXT:    call void @use32(i32 [[T0]])
 ; CHECK-NEXT:    call void @use32(i32 [[T1]])
 ; CHECK-NEXT:    call void @use32(i32 [[T2]])
-; CHECK-NEXT:    [[T3:%.*]] = shl i32 [[T1]], [[T2]]
+; CHECK-NEXT:    [[T3:%.*]] = shl i32 [[X]], [[T2]]
 ; CHECK-NEXT:    ret i32 [[T3]]
 ;
   %t0 = shl i32 %x, %nbits
@@ -65,7 +65,7 @@ define <3 x i32> @t2_vec_splat(<3 x i32> %x, <3 x i32> %nbits) {
 ; CHECK-NEXT:    call void @use3xi32(<3 x i32> [[T0]])
 ; CHECK-NEXT:    call void @use3xi32(<3 x i32> [[T1]])
 ; CHECK-NEXT:    call void @use3xi32(<3 x i32> [[T2]])
-; CHECK-NEXT:    [[T3:%.*]] = shl <3 x i32> [[T1]], [[T2]]
+; CHECK-NEXT:    [[T3:%.*]] = shl <3 x i32> [[X]], [[T2]]
 ; CHECK-NEXT:    ret <3 x i32> [[T3]]
 ;
   %t0 = shl <3 x i32> %x, %nbits
@@ -86,7 +86,7 @@ define <3 x i32> @t3_vec_nonsplat(<3 x i32> %x, <3 x i32> %nbits) {
 ; CHECK-NEXT:    call void @use3xi32(<3 x i32> [[T0]])
 ; CHECK-NEXT:    call void @use3xi32(<3 x i32> [[T1]])
 ; CHECK-NEXT:    call void @use3xi32(<3 x i32> [[T2]])
-; CHECK-NEXT:    [[T3:%.*]] = shl <3 x i32> [[T1]], [[T2]]
+; CHECK-NEXT:    [[T3:%.*]] = shl <3 x i32> [[X]], [[T2]]
 ; CHECK-NEXT:    ret <3 x i32> [[T3]]
 ;
   %t0 = shl <3 x i32> %x, %nbits
@@ -107,7 +107,7 @@ define <3 x i32> @t4_vec_undef(<3 x i32> %x, <3 x i32> %nbits) {
 ; CHECK-NEXT:    call void @use3xi32(<3 x i32> [[T0]])
 ; CHECK-NEXT:    call void @use3xi32(<3 x i32> [[T1]])
 ; CHECK-NEXT:    call void @use3xi32(<3 x i32> [[T2]])
-; CHECK-NEXT:    [[T3:%.*]] = shl <3 x i32> [[T1]], [[T2]]
+; CHECK-NEXT:    [[T3:%.*]] = shl <3 x i32> [[X]], [[T2]]
 ; CHECK-NEXT:    ret <3 x i32> [[T3]]
 ;
   %t0 = shl <3 x i32> %x, %nbits
@@ -128,7 +128,7 @@ define i32 @t5_nuw(i32 %x, i32 %nbits) {
 ; CHECK-NEXT:    [[T1:%.*]] = lshr i32 [[T0]], [[NBITS]]
 ; CHECK-NEXT:    call void @use32(i32 [[T0]])
 ; CHECK-NEXT:    call void @use32(i32 [[T1]])
-; CHECK-NEXT:    [[T2:%.*]] = shl nuw i32 [[T1]], [[NBITS]]
+; CHECK-NEXT:    [[T2:%.*]] = shl i32 [[X]], [[NBITS]]
 ; CHECK-NEXT:    ret i32 [[T2]]
 ;
   %t0 = shl i32 %x, %nbits
@@ -145,7 +145,7 @@ define i32 @t6_nsw(i32 %x, i32 %nbits) {
 ; CHECK-NEXT:    [[T1:%.*]] = lshr i32 [[T0]], [[NBITS]]
 ; CHECK-NEXT:    call void @use32(i32 [[T0]])
 ; CHECK-NEXT:    call void @use32(i32 [[T1]])
-; CHECK-NEXT:    [[T2:%.*]] = shl nsw i32 [[T1]], [[NBITS]]
+; CHECK-NEXT:    [[T2:%.*]] = shl i32 [[X]], [[NBITS]]
 ; CHECK-NEXT:    ret i32 [[T2]]
 ;
   %t0 = shl i32 %x, %nbits
@@ -162,7 +162,7 @@ define i32 @t7_nuw_nsw(i32 %x, i32 %nbits) {
 ; CHECK-NEXT:    [[T1:%.*]] = lshr i32 [[T0]], [[NBITS]]
 ; CHECK-NEXT:    call void @use32(i32 [[T0]])
 ; CHECK-NEXT:    call void @use32(i32 [[T1]])
-; CHECK-NEXT:    [[T2:%.*]] = shl nuw nsw i32 [[T1]], [[NBITS]]
+; CHECK-NEXT:    [[T2:%.*]] = shl i32 [[X]], [[NBITS]]
 ; CHECK-NEXT:    ret i32 [[T2]]
 ;
   %t0 = shl i32 %x, %nbits
@@ -173,58 +173,64 @@ define i32 @t7_nuw_nsw(i32 %x, i32 %nbits) {
   ret i32 %t2
 }
 
-; Negative tests
+; Special test
 
-define i32 @n8_different_shamts0(i32 %x, i32 %nbits0, i32 %nbits1) {
-; CHECK-LABEL: @n8_different_shamts0(
-; CHECK-NEXT:    [[T0:%.*]] = shl i32 [[X:%.*]], [[NBITS0:%.*]]
-; CHECK-NEXT:    [[T1:%.*]] = lshr i32 [[T0]], [[NBITS1:%.*]]
+declare void @llvm.assume(i1 %cond)
+
+; We can't simplify (%shiftnbits-%masknbits) but we have an assumption.
+define i32 @t8_assume_uge(i32 %x, i32 %masknbits, i32 %shiftnbits) {
+; CHECK-LABEL: @t8_assume_uge(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp uge i32 [[SHIFTNBITS:%.*]], [[MASKNBITS:%.*]]
+; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    [[T0:%.*]] = shl i32 [[X:%.*]], [[MASKNBITS]]
+; CHECK-NEXT:    [[T1:%.*]] = lshr i32 [[T0]], [[MASKNBITS]]
 ; CHECK-NEXT:    call void @use32(i32 [[T0]])
 ; CHECK-NEXT:    call void @use32(i32 [[T1]])
-; CHECK-NEXT:    [[T2:%.*]] = shl i32 [[T1]], [[NBITS0]]
+; CHECK-NEXT:    [[T2:%.*]] = shl i32 [[T1]], [[SHIFTNBITS]]
 ; CHECK-NEXT:    ret i32 [[T2]]
 ;
-  %t0 = shl i32 %x, %nbits0 ; different shift amts
-  %t1 = lshr i32 %t0, %nbits1 ; different shift amts
+  %cmp = icmp uge i32 %shiftnbits, %masknbits
+  call void @llvm.assume(i1 %cmp)
+  %t0 = shl i32 %x, %masknbits
+  %t1 = lshr i32 %t0, %masknbits
   call void @use32(i32 %t0)
   call void @use32(i32 %t1)
-  %t2 = shl i32 %t1, %nbits0
+  %t2 = shl i32 %t1, %shiftnbits
   ret i32 %t2
 }
 
-define i32 @n9_different_shamts1(i32 %x, i32 %nbits0, i32 %nbits1) {
-; CHECK-LABEL: @n9_different_shamts1(
+; Negative tests
+
+define i32 @n9_different_shamts0(i32 %x, i32 %nbits0, i32 %nbits1) {
+; CHECK-LABEL: @n9_different_shamts0(
 ; CHECK-NEXT:    [[T0:%.*]] = shl i32 [[X:%.*]], [[NBITS0:%.*]]
 ; CHECK-NEXT:    [[T1:%.*]] = lshr i32 [[T0]], [[NBITS1:%.*]]
 ; CHECK-NEXT:    call void @use32(i32 [[T0]])
 ; CHECK-NEXT:    call void @use32(i32 [[T1]])
-; CHECK-NEXT:    [[T2:%.*]] = shl i32 [[T1]], [[NBITS1]]
+; CHECK-NEXT:    [[T2:%.*]] = shl i32 [[T1]], [[NBITS0]]
 ; CHECK-NEXT:    ret i32 [[T2]]
 ;
   %t0 = shl i32 %x, %nbits0 ; different shift amts
   %t1 = lshr i32 %t0, %nbits1 ; different shift amts
   call void @use32(i32 %t0)
   call void @use32(i32 %t1)
-  %t2 = shl i32 %t1, %nbits1
+  %t2 = shl i32 %t1, %nbits0
   ret i32 %t2
 }
 
-define i32 @n10_shamt_is_smaller(i32 %x, i32 %nbits) {
-; CHECK-LABEL: @n10_shamt_is_smaller(
-; CHECK-NEXT:    [[T0:%.*]] = shl i32 [[X:%.*]], [[NBITS:%.*]]
-; CHECK-NEXT:    [[T1:%.*]] = lshr i32 [[T0]], [[NBITS]]
-; CHECK-NEXT:    [[T2:%.*]] = add i32 [[NBITS]], -1
+define i32 @n10_different_shamts1(i32 %x, i32 %nbits0, i32 %nbits1) {
+; CHECK-LABEL: @n10_different_shamts1(
+; CHECK-NEXT:    [[T0:%.*]] = shl i32 [[X:%.*]], [[NBITS0:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = lshr i32 [[T0]], [[NBITS1:%.*]]
 ; CHECK-NEXT:    call void @use32(i32 [[T0]])
 ; CHECK-NEXT:    call void @use32(i32 [[T1]])
-; CHECK-NEXT:    call void @use32(i32 [[T2]])
+; CHECK-NEXT:    [[T2:%.*]] = shl i32 [[T1]], [[NBITS1]]
 ; CHECK-NEXT:    ret i32 [[T2]]
 ;
-  %t0 = shl i32 %x, %nbits
-  %t1 = lshr i32 %t0, %nbits
-  %t2 = add i32 %nbits, -1
+  %t0 = shl i32 %x, %nbits0 ; different shift amts
+  %t1 = lshr i32 %t0, %nbits1 ; different shift amts
   call void @use32(i32 %t0)
   call void @use32(i32 %t1)
-  call void @use32(i32 %t2)
-  %t3 = shl i32 %t1, %t2 ; shift is smaller than mask
+  %t2 = shl i32 %t1, %nbits1
   ret i32 %t2
 }
diff --git a/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-variant-f.ll b/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-variant-f.ll
index cfdbe39c5..3e2bb330a 100644
--- a/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-variant-f.ll
+++ b/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-variant-f.ll
@@ -21,7 +21,7 @@ define i32 @t0_basic(i32 %x, i32 %nbits) {
 ; CHECK-NEXT:    [[T1:%.*]] = ashr i32 [[T0]], [[NBITS]]
 ; CHECK-NEXT:    call void @use32(i32 [[T0]])
 ; CHECK-NEXT:    call void @use32(i32 [[T1]])
-; CHECK-NEXT:    [[T2:%.*]] = shl i32 [[T1]], [[NBITS]]
+; CHECK-NEXT:    [[T2:%.*]] = shl i32 [[X]], [[NBITS]]
 ; CHECK-NEXT:    ret i32 [[T2]]
 ;
   %t0 = shl i32 %x, %nbits
@@ -40,7 +40,7 @@ define i32 @t1_bigger_shift(i32 %x, i32 %nbits) {
 ; CHECK-NEXT:    call void @use32(i32 [[T0]])
 ; CHECK-NEXT:    call void @use32(i32 [[T1]])
 ; CHECK-NEXT:    call void @use32(i32 [[T2]])
-; CHECK-NEXT:    [[T3:%.*]] = shl i32 [[T1]], [[T2]]
+; CHECK-NEXT:    [[T3:%.*]] = shl i32 [[X]], [[T2]]
 ; CHECK-NEXT:    ret i32 [[T3]]
 ;
   %t0 = shl i32 %x, %nbits
@@ -65,7 +65,7 @@ define <3 x i32> @t2_vec_splat(<3 x i32> %x, <3 x i32> %nbits) {
 ; CHECK-NEXT:    call void @use3xi32(<3 x i32> [[T0]])
 ; CHECK-NEXT:    call void @use3xi32(<3 x i32> [[T1]])
 ; CHECK-NEXT:    call void @use3xi32(<3 x i32> [[T2]])
-; CHECK-NEXT:    [[T3:%.*]] = shl <3 x i32> [[T1]], [[T2]]
+; CHECK-NEXT:    [[T3:%.*]] = shl <3 x i32> [[X]], [[T2]]
 ; CHECK-NEXT:    ret <3 x i32> [[T3]]
 ;
   %t0 = shl <3 x i32> %x, %nbits
@@ -86,7 +86,7 @@ define <3 x i32> @t3_vec_nonsplat(<3 x i32> %x, <3 x i32> %nbits) {
 ; CHECK-NEXT:    call void @use3xi32(<3 x i32> [[T0]])
 ; CHECK-NEXT:    call void @use3xi32(<3 x i32> [[T1]])
 ; CHECK-NEXT:    call void @use3xi32(<3 x i32> [[T2]])
-; CHECK-NEXT:    [[T3:%.*]] = shl <3 x i32> [[T1]], [[T2]]
+; CHECK-NEXT:    [[T3:%.*]] = shl <3 x i32> [[X]], [[T2]]
 ; CHECK-NEXT:    ret <3 x i32> [[T3]]
 ;
   %t0 = shl <3 x i32> %x, %nbits
@@ -107,7 +107,7 @@ define <3 x i32> @t4_vec_undef(<3 x i32> %x, <3 x i32> %nbits) {
 ; CHECK-NEXT:    call void @use3xi32(<3 x i32> [[T0]])
 ; CHECK-NEXT:    call void @use3xi32(<3 x i32> [[T1]])
 ; CHECK-NEXT:    call void @use3xi32(<3 x i32> [[T2]])
-; CHECK-NEXT:    [[T3:%.*]] = shl <3 x i32> [[T1]], [[T2]]
+; CHECK-NEXT:    [[T3:%.*]] = shl <3 x i32> [[X]], [[T2]]
 ; CHECK-NEXT:    ret <3 x i32> [[T3]]
 ;
   %t0 = shl <3 x i32> %x, %nbits
@@ -128,7 +128,7 @@ define i32 @t5_nuw(i32 %x, i32 %nbits) {
 ; CHECK-NEXT:    [[T1:%.*]] = ashr i32 [[T0]], [[NBITS]]
 ; CHECK-NEXT:    call void @use32(i32 [[T0]])
 ; CHECK-NEXT:    call void @use32(i32 [[T1]])
-; CHECK-NEXT:    [[T2:%.*]] = shl nuw i32 [[T1]], [[NBITS]]
+; CHECK-NEXT:    [[T2:%.*]] = shl i32 [[X]], [[NBITS]]
 ; CHECK-NEXT:    ret i32 [[T2]]
 ;
   %t0 = shl i32 %x, %nbits
@@ -145,7 +145,7 @@ define i32 @t6_nsw(i32 %x, i32 %nbits) {
 ; CHECK-NEXT:    [[T1:%.*]] = ashr i32 [[T0]], [[NBITS]]
 ; CHECK-NEXT:    call void @use32(i32 [[T0]])
 ; CHECK-NEXT:    call void @use32(i32 [[T1]])
-; CHECK-NEXT:    [[T2:%.*]] = shl nsw i32 [[T1]], [[NBITS]]
+; CHECK-NEXT:    [[T2:%.*]] = shl i32 [[X]], [[NBITS]]
 ; CHECK-NEXT:    ret i32 [[T2]]
 ;
   %t0 = shl i32 %x, %nbits
@@ -162,7 +162,7 @@ define i32 @t7_nuw_nsw(i32 %x, i32 %nbits) {
 ; CHECK-NEXT:    [[T1:%.*]] = ashr i32 [[T0]], [[NBITS]]
 ; CHECK-NEXT:    call void @use32(i32 [[T0]])
 ; CHECK-NEXT:    call void @use32(i32 [[T1]])
-; CHECK-NEXT:    [[T2:%.*]] = shl nuw nsw i32 [[T1]], [[NBITS]]
+; CHECK-NEXT:    [[T2:%.*]] = shl i32 [[X]], [[NBITS]]
 ; CHECK-NEXT:    ret i32 [[T2]]
 ;
   %t0 = shl i32 %x, %nbits
@@ -173,10 +173,36 @@ define i32 @t7_nuw_nsw(i32 %x, i32 %nbits) {
   ret i32 %t2
 }
 
+; Special test
+
+declare void @llvm.assume(i1 %cond)
+
+; We can't simplify (%shiftnbits-%masknbits) but we have an assumption.
+define i32 @t8_assume_uge(i32 %x, i32 %masknbits, i32 %shiftnbits) {
+; CHECK-LABEL: @t8_assume_uge(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp uge i32 [[SHIFTNBITS:%.*]], [[MASKNBITS:%.*]]
+; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    [[T0:%.*]] = shl i32 [[X:%.*]], [[MASKNBITS]]
+; CHECK-NEXT:    [[T1:%.*]] = ashr i32 [[T0]], [[MASKNBITS]]
+; CHECK-NEXT:    call void @use32(i32 [[T0]])
+; CHECK-NEXT:    call void @use32(i32 [[T1]])
+; CHECK-NEXT:    [[T2:%.*]] = shl i32 [[T1]], [[SHIFTNBITS]]
+; CHECK-NEXT:    ret i32 [[T2]]
+;
+  %cmp = icmp uge i32 %shiftnbits, %masknbits
+  call void @llvm.assume(i1 %cmp)
+  %t0 = shl i32 %x, %masknbits
+  %t1 = ashr i32 %t0, %masknbits
+  call void @use32(i32 %t0)
+  call void @use32(i32 %t1)
+  %t2 = shl i32 %t1, %shiftnbits
+  ret i32 %t2
+}
+
 ; Negative tests
 
-define i32 @n8_different_shamts0(i32 %x, i32 %nbits0, i32 %nbits1) {
-; CHECK-LABEL: @n8_different_shamts0(
+define i32 @n9_different_shamts0(i32 %x, i32 %nbits0, i32 %nbits1) {
+; CHECK-LABEL: @n9_different_shamts0(
 ; CHECK-NEXT:    [[T0:%.*]] = shl i32 [[X:%.*]], [[NBITS0:%.*]]
 ; CHECK-NEXT:    [[T1:%.*]] = ashr i32 [[T0]], [[NBITS1:%.*]]
 ; CHECK-NEXT:    call void @use32(i32 [[T0]])
@@ -192,8 +218,8 @@ define i32 @n8_different_shamts0(i32 %x, i32 %nbits0, i32 %nbits1) {
   ret i32 %t2
 }
 
-define i32 @n9_different_shamts1(i32 %x, i32 %nbits0, i32 %nbits1) {
-; CHECK-LABEL: @n9_different_shamts1(
+define i32 @n10_different_shamts1(i32 %x, i32 %nbits0, i32 %nbits1) {
+; CHECK-LABEL: @n10_different_shamts1(
 ; CHECK-NEXT:    [[T0:%.*]] = shl i32 [[X:%.*]], [[NBITS0:%.*]]
 ; CHECK-NEXT:    [[T1:%.*]] = ashr i32 [[T0]], [[NBITS1:%.*]]
 ; CHECK-NEXT:    call void @use32(i32 [[T0]])
@@ -209,22 +235,21 @@ define i32 @n9_different_shamts1(i32 %x, i32 %nbits0, i32 %nbits1) {
   ret i32 %t2
 }
 
-define i32 @n10_shamt_is_smaller(i32 %x, i32 %nbits) {
-; CHECK-LABEL: @n10_shamt_is_smaller(
+define i32 @n11_shamt_is_smaller(i32 %x, i32 %nbits) {
+; CHECK-LABEL: @n11_shamt_is_smaller(
 ; CHECK-NEXT:    [[T0:%.*]] = shl i32 [[X:%.*]], [[NBITS:%.*]]
 ; CHECK-NEXT:    [[T1:%.*]] = ashr i32 [[T0]], [[NBITS]]
 ; CHECK-NEXT:    [[T2:%.*]] = add i32 [[NBITS]], -1
 ; CHECK-NEXT:    call void @use32(i32 [[T0]])
-; CHECK-NEXT:    call void @use32(i32 [[T1]])
 ; CHECK-NEXT:    call void @use32(i32 [[T2]])
-; CHECK-NEXT:    ret i32 [[T2]]
+; CHECK-NEXT:    [[T3:%.*]] = shl i32 [[T1]], [[T2]]
+; CHECK-NEXT:    ret i32 [[T3]]
 ;
   %t0 = shl i32 %x, %nbits
   %t1 = ashr i32 %t0, %nbits
   %t2 = add i32 %nbits, -1
   call void @use32(i32 %t0)
-  call void @use32(i32 %t1)
   call void @use32(i32 %t2)
   %t3 = shl i32 %t1, %t2 ; shift is smaller than mask
-  ret i32 %t2
+  ret i32 %t3
 }
diff --git a/llvm/test/Transforms/InstCombine/rem.ll b/llvm/test/Transforms/InstCombine/rem.ll
index 200e0389f..c833acc16 100644
--- a/llvm/test/Transforms/InstCombine/rem.ll
+++ b/llvm/test/Transforms/InstCombine/rem.ll
@@ -201,8 +201,8 @@ define <2 x i19> @weird_vec_power_of_2_constant_splat_divisor(<2 x i19> %A) {
 
 define i1 @test3a(i32 %A) {
 ; CHECK-LABEL: @test3a(
-; CHECK-NEXT:    [[B1:%.*]] = and i32 [[A:%.*]], 7
-; CHECK-NEXT:    [[C:%.*]] = icmp ne i32 [[B1]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[A:%.*]], 7
+; CHECK-NEXT:    [[C:%.*]] = icmp ne i32 [[TMP1]], 0
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %B = srem i32 %A, -8
@@ -212,8 +212,8 @@ define i1 @test3a(i32 %A) {
 
 define <2 x i1> @test3a_vec(<2 x i32> %A) {
 ; CHECK-LABEL: @test3a_vec(
-; CHECK-NEXT:    [[B1:%.*]] = and <2 x i32> [[A:%.*]], <i32 7, i32 7>
-; CHECK-NEXT:    [[C:%.*]] = icmp ne <2 x i32> [[B1]], zeroinitializer
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[A:%.*]], <i32 7, i32 7>
+; CHECK-NEXT:    [[C:%.*]] = icmp ne <2 x i32> [[TMP1]], zeroinitializer
 ; CHECK-NEXT:    ret <2 x i1> [[C]]
 ;
   %B = srem <2 x i32> %A, <i32 -8, i32 -8>
@@ -367,8 +367,8 @@ define i32 @test16(i32 %x, i32 %y) {
 define i32 @test17(i32 %X) {
 ; CHECK-LABEL: @test17(
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne i32 [[X:%.*]], 1
-; CHECK-NEXT:    [[TMP2:%.*]] = zext i1 [[TMP1]] to i32
-; CHECK-NEXT:    ret i32 [[TMP2]]
+; CHECK-NEXT:    [[A:%.*]] = zext i1 [[TMP1]] to i32
+; CHECK-NEXT:    ret i32 [[A]]
 ;
   %A = urem i32 1, %X
   ret i32 %A
@@ -657,6 +657,88 @@ define <2 x i32> @test23(<2 x i32> %A) {
   ret <2 x i32> %mul
 }
 
+define i1 @test24(i32 %A) {
+; CHECK-LABEL: @test24(
+; CHECK-NEXT:    [[B:%.*]] = and i32 [[A:%.*]], 2147483647
+; CHECK-NEXT:    [[C:%.*]] = icmp ne i32 [[B]], 0
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %B = urem i32 %A, 2147483648 ; signbit
+  %C = icmp ne i32 %B, 0
+  ret i1 %C
+}
+
+define <2 x i1> @test24_vec(<2 x i32> %A) {
+; CHECK-LABEL: @test24_vec(
+; CHECK-NEXT:    [[B:%.*]] = and <2 x i32> [[A:%.*]], <i32 2147483647, i32 2147483647>
+; CHECK-NEXT:    [[C:%.*]] = icmp ne <2 x i32> [[B]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[C]]
+;
+  %B = urem <2 x i32> %A, <i32 2147483648, i32 2147483648>
+  %C = icmp ne <2 x i32> %B, zeroinitializer
+  ret <2 x i1> %C
+}
+
+define i1 @test25(i32 %A) {
+; CHECK-LABEL: @test25(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[A:%.*]], 2147483647
+; CHECK-NEXT:    [[C:%.*]] = icmp ne i32 [[TMP1]], 0
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %B = srem i32 %A, 2147483648 ; signbit
+  %C = icmp ne i32 %B, 0
+  ret i1 %C
+}
+
+define <2 x i1> @test25_vec(<2 x i32> %A) {
+; CHECK-LABEL: @test25_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[A:%.*]], <i32 2147483647, i32 2147483647>
+; CHECK-NEXT:    [[C:%.*]] = icmp ne <2 x i32> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[C]]
+;
+  %B = srem <2 x i32> %A, <i32 2147483648, i32 2147483648>
+  %C = icmp ne <2 x i32> %B, zeroinitializer
+  ret <2 x i1> %C
+}
+
+define i1 @test26(i32 %A, i32 %B) {
+; CHECK-LABEL: @test26(
+; CHECK-NEXT:    [[NOTMASK:%.*]] = shl nsw i32 -1, [[B:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[NOTMASK]], -1
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], [[A:%.*]]
+; CHECK-NEXT:    [[E:%.*]] = icmp ne i32 [[TMP2]], 0
+; CHECK-NEXT:    ret i1 [[E]]
+;
+  %C = shl i32 1, %B ; not a constant
+  %D = srem i32 %A, %C
+  %E = icmp ne i32 %D, 0
+  ret i1 %E
+}
+
+define i1 @test27(i32 %A, i32* %remdst) {
+; CHECK-LABEL: @test27(
+; CHECK-NEXT:    [[B:%.*]] = srem i32 [[A:%.*]], -2147483648
+; CHECK-NEXT:    store i32 [[B]], i32* [[REMDST:%.*]], align 1
+; CHECK-NEXT:    [[C:%.*]] = icmp ne i32 [[B]], 0
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %B = srem i32 %A, 2147483648 ; signbit
+  store i32 %B, i32* %remdst, align 1 ; extra use of rem
+  %C = icmp ne i32 %B, 0
+  ret i1 %C
+}
+
+define i1 @test28(i32 %A) {
+; CHECK-LABEL: @test28(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[A:%.*]], 2147483647
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %B = srem i32 %A, 2147483648 ; signbit
+  %C = icmp eq i32 %B, 0 ; another equality predicate
+  ret i1 %C
+}
+
 ; FP division-by-zero is not UB.
 
 define double @PR34870(i1 %cond, double %x, double %y) {
diff --git a/llvm/test/Transforms/InstCombine/result-of-add-of-negative-is-non-zero-and-no-underflow.ll b/llvm/test/Transforms/InstCombine/result-of-add-of-negative-is-non-zero-and-no-underflow.ll
new file mode 100644
index 000000000..5b48b5e30
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/result-of-add-of-negative-is-non-zero-and-no-underflow.ll
@@ -0,0 +1,247 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instcombine -S | FileCheck %s
+
+declare void @use8(i8)
+
+declare void @use1(i1)
+declare void @llvm.assume(i1)
+
+; Here we don't know that at least one of the values being added is non-zero
+define i1 @t0_bad(i8 %base, i8 %offset) {
+; CHECK-LABEL: @t0_bad(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = add i8 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp ne i8 [[ADJUSTED]], 0
+; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp ult i8 [[ADJUSTED]], [[BASE]]
+; CHECK-NEXT:    [[R:%.*]] = and i1 [[NOT_NULL]], [[NO_UNDERFLOW]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %adjusted = add i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp ne i8 %adjusted, 0
+  %no_underflow = icmp ult i8 %adjusted, %base
+  %r = and i1 %not_null, %no_underflow
+  ret i1 %r
+}
+
+; Ok, base is non-zero.
+define i1 @t1(i8 %base, i8 %offset) {
+; CHECK-LABEL: @t1(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[BASE:%.*]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = add i8 [[BASE]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i8 0, [[BASE]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i8 [[TMP1]], [[OFFSET]]
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %cmp = icmp slt i8 %base, 0
+  call void @llvm.assume(i1 %cmp)
+
+  %adjusted = add i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp ne i8 %adjusted, 0
+  %no_underflow = icmp ult i8 %adjusted, %base
+  %r = and i1 %not_null, %no_underflow
+  ret i1 %r
+}
+
+; Ok, offset is non-zero.
+define i1 @t2(i8 %base, i8 %offset) {
+; CHECK-LABEL: @t2(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[OFFSET:%.*]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = add i8 [[BASE:%.*]], [[OFFSET]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i8 0, [[OFFSET]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i8 [[TMP1]], [[BASE]]
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %cmp = icmp slt i8 %offset, 0
+  call void @llvm.assume(i1 %cmp)
+
+  %adjusted = add i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp ne i8 %adjusted, 0
+  %no_underflow = icmp ult i8 %adjusted, %base
+  %r = and i1 %not_null, %no_underflow
+  ret i1 %r
+}
+
+; We need to produce extra instruction, so one of icmp's must go away.
+define i1 @t3_oneuse0(i8 %base, i8 %offset) {
+; CHECK-LABEL: @t3_oneuse0(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[BASE:%.*]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = add i8 [[BASE]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp ne i8 [[ADJUSTED]], 0
+; CHECK-NEXT:    call void @use1(i1 [[NOT_NULL]])
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i8 0, [[BASE]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i8 [[TMP1]], [[OFFSET]]
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %cmp = icmp slt i8 %base, 0
+  call void @llvm.assume(i1 %cmp)
+
+  %adjusted = add i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp ne i8 %adjusted, 0
+  call void @use1(i1 %not_null)
+  %no_underflow = icmp ult i8 %adjusted, %base
+  %r = and i1 %not_null, %no_underflow
+  ret i1 %r
+}
+define i1 @t4_oneuse1(i8 %base, i8 %offset) {
+; CHECK-LABEL: @t4_oneuse1(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[BASE:%.*]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = add i8 [[BASE]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp ult i8 [[ADJUSTED]], [[BASE]]
+; CHECK-NEXT:    call void @use1(i1 [[NO_UNDERFLOW]])
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i8 0, [[BASE]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i8 [[TMP1]], [[OFFSET]]
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %cmp = icmp slt i8 %base, 0
+  call void @llvm.assume(i1 %cmp)
+
+  %adjusted = add i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp ne i8 %adjusted, 0
+  %no_underflow = icmp ult i8 %adjusted, %base
+  call void @use1(i1 %no_underflow)
+  %r = and i1 %not_null, %no_underflow
+  ret i1 %r
+}
+define i1 @t5_oneuse2_bad(i8 %base, i8 %offset) {
+; CHECK-LABEL: @t5_oneuse2_bad(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[BASE:%.*]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = add i8 [[BASE]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp ne i8 [[ADJUSTED]], 0
+; CHECK-NEXT:    call void @use1(i1 [[NOT_NULL]])
+; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp ult i8 [[ADJUSTED]], [[BASE]]
+; CHECK-NEXT:    call void @use1(i1 [[NO_UNDERFLOW]])
+; CHECK-NEXT:    [[R:%.*]] = and i1 [[NOT_NULL]], [[NO_UNDERFLOW]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %cmp = icmp slt i8 %base, 0
+  call void @llvm.assume(i1 %cmp)
+
+  %adjusted = add i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp ne i8 %adjusted, 0
+  call void @use1(i1 %not_null)
+  %no_underflow = icmp ult i8 %adjusted, %base
+  call void @use1(i1 %no_underflow)
+  %r = and i1 %not_null, %no_underflow
+  ret i1 %r
+}
+
+define i1 @t6_commutativity0(i8 %base, i8 %offset) {
+; CHECK-LABEL: @t6_commutativity0(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[BASE:%.*]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = add i8 [[BASE]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i8 0, [[BASE]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i8 [[TMP1]], [[OFFSET]]
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %cmp = icmp slt i8 %base, 0
+  call void @llvm.assume(i1 %cmp)
+
+  %adjusted = add i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp ne i8 %adjusted, 0
+  %no_underflow = icmp ult i8 %adjusted, %base
+  %r = and i1 %no_underflow, %not_null ; swapped
+  ret i1 %r
+}
+define i1 @t7_commutativity1(i8 %base, i8 %offset) {
+; CHECK-LABEL: @t7_commutativity1(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[BASE:%.*]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = add i8 [[BASE]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i8 0, [[BASE]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i8 [[TMP1]], [[OFFSET]]
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %cmp = icmp slt i8 %base, 0
+  call void @llvm.assume(i1 %cmp)
+
+  %adjusted = add i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp ne i8 %adjusted, 0
+  %no_underflow = icmp ugt i8 %base, %adjusted ; swapped
+  %r = and i1 %not_null, %no_underflow
+  ret i1 %r
+}
+define i1 @t7_commutativity3(i8 %base, i8 %offset) {
+; CHECK-LABEL: @t7_commutativity3(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[BASE:%.*]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = add i8 [[BASE]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i8 0, [[BASE]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i8 [[TMP1]], [[OFFSET]]
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %cmp = icmp slt i8 %base, 0
+  call void @llvm.assume(i1 %cmp)
+
+  %adjusted = add i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp ne i8 %adjusted, 0
+  %no_underflow = icmp ugt i8 %base, %adjusted ; swapped
+  %r = and i1 %no_underflow, %not_null ; swapped
+  ret i1 %r
+}
+
+; We could have the opposite question, did we get null or overflow happened?
+define i1 @t8(i8 %base, i8 %offset) {
+; CHECK-LABEL: @t8(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[BASE:%.*]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = add i8 [[BASE]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i8 0, [[BASE]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp uge i8 [[TMP1]], [[OFFSET]]
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %cmp = icmp slt i8 %base, 0
+  call void @llvm.assume(i1 %cmp)
+
+  %adjusted = add i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp eq i8 %adjusted, 0
+  %no_underflow = icmp uge i8 %adjusted, %base
+  %r = or i1 %not_null, %no_underflow
+  ret i1 %r
+}
+
+; The comparison can be with any of the values being added.
+define i1 @t9(i8 %base, i8 %offset) {
+; CHECK-LABEL: @t9(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[BASE:%.*]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = add i8 [[BASE]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i8 0, [[BASE]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i8 [[TMP1]], [[OFFSET]]
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %cmp = icmp slt i8 %base, 0
+  call void @llvm.assume(i1 %cmp)
+
+  %adjusted = add i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp ne i8 %adjusted, 0
+  %no_underflow = icmp ult i8 %adjusted, %offset
+  %r = and i1 %not_null, %no_underflow
+  ret i1 %r
+}
diff --git a/llvm/test/Transforms/InstCombine/result-of-add-of-negative-or-zero-is-non-zero-and-no-underflow.ll b/llvm/test/Transforms/InstCombine/result-of-add-of-negative-or-zero-is-non-zero-and-no-underflow.ll
new file mode 100644
index 000000000..b8a29e4fa
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/result-of-add-of-negative-or-zero-is-non-zero-and-no-underflow.ll
@@ -0,0 +1,161 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instcombine -S | FileCheck %s
+
+declare void @use8(i8)
+
+declare void @use1(i1)
+declare void @llvm.assume(i1)
+
+define i1 @t0(i8 %base, i8 %offset) {
+; CHECK-LABEL: @t0(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = add i8 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i8 0, [[OFFSET]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i8 [[TMP1]], [[BASE]]
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %adjusted = add i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp ne i8 %adjusted, 0
+  %no_underflow = icmp ule i8 %adjusted, %base
+  %r = and i1 %not_null, %no_underflow
+  ret i1 %r
+}
+
+; We need to produce extra instruction, so one of icmp's must go away.
+define i1 @t1_oneuse0(i8 %base, i8 %offset) {
+; CHECK-LABEL: @t1_oneuse0(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = add i8 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp ne i8 [[ADJUSTED]], 0
+; CHECK-NEXT:    call void @use1(i1 [[NOT_NULL]])
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i8 0, [[OFFSET]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i8 [[TMP1]], [[BASE]]
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %adjusted = add i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp ne i8 %adjusted, 0
+  call void @use1(i1 %not_null)
+  %no_underflow = icmp ule i8 %adjusted, %base
+  %r = and i1 %not_null, %no_underflow
+  ret i1 %r
+}
+define i1 @t2_oneuse1(i8 %base, i8 %offset) {
+; CHECK-LABEL: @t2_oneuse1(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = add i8 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp ule i8 [[ADJUSTED]], [[BASE]]
+; CHECK-NEXT:    call void @use1(i1 [[NO_UNDERFLOW]])
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i8 0, [[OFFSET]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i8 [[TMP1]], [[BASE]]
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %adjusted = add i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp ne i8 %adjusted, 0
+  %no_underflow = icmp ule i8 %adjusted, %base
+  call void @use1(i1 %no_underflow)
+  %r = and i1 %not_null, %no_underflow
+  ret i1 %r
+}
+define i1 @n3_oneuse2_bad(i8 %base, i8 %offset) {
+; CHECK-LABEL: @n3_oneuse2_bad(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = add i8 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp ne i8 [[ADJUSTED]], 0
+; CHECK-NEXT:    call void @use1(i1 [[NOT_NULL]])
+; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp ule i8 [[ADJUSTED]], [[BASE]]
+; CHECK-NEXT:    call void @use1(i1 [[NO_UNDERFLOW]])
+; CHECK-NEXT:    [[R:%.*]] = and i1 [[NOT_NULL]], [[NO_UNDERFLOW]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %adjusted = add i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp ne i8 %adjusted, 0
+  call void @use1(i1 %not_null)
+  %no_underflow = icmp ule i8 %adjusted, %base
+  call void @use1(i1 %no_underflow)
+  %r = and i1 %not_null, %no_underflow
+  ret i1 %r
+}
+
+define i1 @t4_commutativity0(i8 %base, i8 %offset) {
+; CHECK-LABEL: @t4_commutativity0(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = add i8 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i8 0, [[OFFSET]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i8 [[TMP1]], [[BASE]]
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %adjusted = add i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp ne i8 %adjusted, 0
+  %no_underflow = icmp ule i8 %adjusted, %base
+  %r = and i1 %no_underflow, %not_null ; swapped
+  ret i1 %r
+}
+define i1 @t5_commutativity1(i8 %base, i8 %offset) {
+; CHECK-LABEL: @t5_commutativity1(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = add i8 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i8 0, [[OFFSET]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i8 [[TMP1]], [[BASE]]
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %adjusted = add i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp ne i8 %adjusted, 0
+  %no_underflow = icmp uge i8 %base, %adjusted ; swapped
+  %r = and i1 %not_null, %no_underflow
+  ret i1 %r
+}
+define i1 @t6_commutativity3(i8 %base, i8 %offset) {
+; CHECK-LABEL: @t6_commutativity3(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = add i8 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i8 0, [[OFFSET]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i8 [[TMP1]], [[BASE]]
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %adjusted = add i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp ne i8 %adjusted, 0
+  %no_underflow = icmp uge i8 %base, %adjusted ; swapped
+  %r = and i1 %no_underflow, %not_null ; swapped
+  ret i1 %r
+}
+
+; We could have the opposite question, did we get null or overflow happened?
+define i1 @t7(i8 %base, i8 %offset) {
+; CHECK-LABEL: @t7(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = add i8 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[TMP1:%.*]] = add i8 [[ADJUSTED]], -1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp uge i8 [[TMP1]], [[BASE]]
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %adjusted = add i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp eq i8 %adjusted, 0
+  %no_underflow = icmp ugt i8 %adjusted, %base
+  %r = or i1 %not_null, %no_underflow
+  ret i1 %r
+}
+
+; The comparison can be with any of the values being added.
+define i1 @t8(i8 %base, i8 %offset) {
+; CHECK-LABEL: @t8(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = add i8 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i8 0, [[BASE]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i8 [[TMP1]], [[OFFSET]]
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %adjusted = add i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp ne i8 %adjusted, 0
+  %no_underflow = icmp ule i8 %adjusted, %offset
+  %r = and i1 %not_null, %no_underflow
+  ret i1 %r
+}
diff --git a/llvm/test/Transforms/InstCombine/result-of-usub-is-non-zero-and-no-overflow.ll b/llvm/test/Transforms/InstCombine/result-of-usub-is-non-zero-and-no-overflow.ll
new file mode 100644
index 000000000..d46da5ec8
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/result-of-usub-is-non-zero-and-no-overflow.ll
@@ -0,0 +1,430 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instcombine -S | FileCheck %s
+
+; Here we subtract two values, check that subtraction did not overflow AND
+; that the result is non-zero. This can be simplified just to a comparison
+; between the base and offset.
+
+declare void @use8(i8)
+declare void @use64(i64)
+declare void @use1(i1)
+
+declare {i8, i1} @llvm.usub.with.overflow(i8, i8)
+declare void @useagg({i8, i1})
+
+declare void @llvm.assume(i1)
+
+; There is a number of base patterns..
+
+define i1 @t0_noncanonical_ignoreme(i8 %base, i8 %offset) {
+; CHECK-LABEL: @t0_noncanonical_ignoreme(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i8 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp uge i8 [[BASE]], [[OFFSET]]
+; CHECK-NEXT:    call void @use1(i1 [[NO_UNDERFLOW]])
+; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp ne i8 [[ADJUSTED]], 0
+; CHECK-NEXT:    call void @use1(i1 [[NOT_NULL]])
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i8 [[BASE]], [[OFFSET]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %adjusted = sub i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %no_underflow = icmp ule i8 %adjusted, %base
+  call void @use1(i1 %no_underflow)
+  %not_null = icmp ne i8 %adjusted, 0
+  call void @use1(i1 %not_null)
+  %r = and i1 %not_null, %no_underflow
+  ret i1 %r
+}
+
+define i1 @t1(i8 %base, i8 %offset) {
+; CHECK-LABEL: @t1(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i8 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp uge i8 [[BASE]], [[OFFSET]]
+; CHECK-NEXT:    call void @use1(i1 [[NO_UNDERFLOW]])
+; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp ne i8 [[ADJUSTED]], 0
+; CHECK-NEXT:    call void @use1(i1 [[NOT_NULL]])
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i8 [[BASE]], [[OFFSET]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %adjusted = sub i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %no_underflow = icmp uge i8 %base, %offset
+  call void @use1(i1 %no_underflow)
+  %not_null = icmp ne i8 %adjusted, 0
+  call void @use1(i1 %not_null)
+  %r = and i1 %not_null, %no_underflow
+  ret i1 %r
+}
+define i1 @t1_strict(i8 %base, i8 %offset) {
+; CHECK-LABEL: @t1_strict(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i8 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp ugt i8 [[BASE]], [[OFFSET]]
+; CHECK-NEXT:    call void @use1(i1 [[NO_UNDERFLOW]])
+; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp ne i8 [[ADJUSTED]], 0
+; CHECK-NEXT:    call void @use1(i1 [[NOT_NULL]])
+; CHECK-NEXT:    ret i1 [[NO_UNDERFLOW]]
+;
+  %adjusted = sub i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %no_underflow = icmp ugt i8 %base, %offset ; same is valid for strict predicate
+  call void @use1(i1 %no_underflow)
+  %not_null = icmp ne i8 %adjusted, 0
+  call void @use1(i1 %not_null)
+  %r = and i1 %not_null, %no_underflow
+  ret i1 %r
+}
+
+define i1 @t2(i8 %base, i8 %offset) {
+; CHECK-LABEL: @t2(
+; CHECK-NEXT:    [[AGG:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[BASE:%.*]], i8 [[OFFSET:%.*]])
+; CHECK-NEXT:    call void @useagg({ i8, i1 } [[AGG]])
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = extractvalue { i8, i1 } [[AGG]], 0
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[UNDERFLOW:%.*]] = extractvalue { i8, i1 } [[AGG]], 1
+; CHECK-NEXT:    call void @use1(i1 [[UNDERFLOW]])
+; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = xor i1 [[UNDERFLOW]], true
+; CHECK-NEXT:    call void @use1(i1 [[NO_UNDERFLOW]])
+; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp ne i8 [[ADJUSTED]], 0
+; CHECK-NEXT:    [[R:%.*]] = and i1 [[NOT_NULL]], [[NO_UNDERFLOW]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %agg = call {i8, i1} @llvm.usub.with.overflow(i8 %base, i8 %offset)
+  call void @useagg({i8, i1} %agg)
+  %adjusted = extractvalue {i8, i1} %agg, 0
+  call void @use8(i8 %adjusted)
+  %underflow = extractvalue {i8, i1} %agg, 1
+  call void @use1(i1 %underflow)
+  %no_underflow = xor i1 %underflow, -1
+  call void @use1(i1 %no_underflow)
+  %not_null = icmp ne i8 %adjusted, 0
+  %r = and i1 %not_null, %no_underflow
+  ret i1 %r
+}
+
+; Commutativity
+
+define i1 @t3_commutability0(i8 %base, i8 %offset) {
+; CHECK-LABEL: @t3_commutability0(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i8 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp uge i8 [[BASE]], [[OFFSET]]
+; CHECK-NEXT:    call void @use1(i1 [[NO_UNDERFLOW]])
+; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp ne i8 [[ADJUSTED]], 0
+; CHECK-NEXT:    call void @use1(i1 [[NOT_NULL]])
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i8 [[BASE]], [[OFFSET]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %adjusted = sub i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %no_underflow = icmp ule i8 %offset, %base ; swapped
+  call void @use1(i1 %no_underflow)
+  %not_null = icmp ne i8 %adjusted, 0
+  call void @use1(i1 %not_null)
+  %r = and i1 %not_null, %no_underflow
+  ret i1 %r
+}
+define i1 @t4_commutability1(i8 %base, i8 %offset) {
+; CHECK-LABEL: @t4_commutability1(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i8 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp uge i8 [[BASE]], [[OFFSET]]
+; CHECK-NEXT:    call void @use1(i1 [[NO_UNDERFLOW]])
+; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp ne i8 [[ADJUSTED]], 0
+; CHECK-NEXT:    call void @use1(i1 [[NOT_NULL]])
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i8 [[BASE]], [[OFFSET]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %adjusted = sub i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %no_underflow = icmp uge i8 %base, %offset
+  call void @use1(i1 %no_underflow)
+  %not_null = icmp ne i8 %adjusted, 0
+  call void @use1(i1 %not_null)
+  %r = and i1 %no_underflow, %not_null ; swapped
+  ret i1 %r
+}
+define i1 @t5_commutability2(i8 %base, i8 %offset) {
+; CHECK-LABEL: @t5_commutability2(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i8 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp uge i8 [[BASE]], [[OFFSET]]
+; CHECK-NEXT:    call void @use1(i1 [[NO_UNDERFLOW]])
+; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp ne i8 [[ADJUSTED]], 0
+; CHECK-NEXT:    call void @use1(i1 [[NOT_NULL]])
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i8 [[BASE]], [[OFFSET]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %adjusted = sub i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %no_underflow = icmp ule i8 %offset, %base ; swapped
+  call void @use1(i1 %no_underflow)
+  %not_null = icmp ne i8 %adjusted, 0
+  call void @use1(i1 %not_null)
+  %r = and i1 %no_underflow, %not_null ; swapped
+  ret i1 %r
+}
+
+define i1 @t6_commutability(i8 %base, i8 %offset) {
+; CHECK-LABEL: @t6_commutability(
+; CHECK-NEXT:    [[AGG:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[BASE:%.*]], i8 [[OFFSET:%.*]])
+; CHECK-NEXT:    call void @useagg({ i8, i1 } [[AGG]])
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = extractvalue { i8, i1 } [[AGG]], 0
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[UNDERFLOW:%.*]] = extractvalue { i8, i1 } [[AGG]], 1
+; CHECK-NEXT:    call void @use1(i1 [[UNDERFLOW]])
+; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = xor i1 [[UNDERFLOW]], true
+; CHECK-NEXT:    call void @use1(i1 [[NO_UNDERFLOW]])
+; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp ne i8 [[ADJUSTED]], 0
+; CHECK-NEXT:    [[R:%.*]] = and i1 [[NOT_NULL]], [[NO_UNDERFLOW]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %agg = call {i8, i1} @llvm.usub.with.overflow(i8 %base, i8 %offset)
+  call void @useagg({i8, i1} %agg)
+  %adjusted = extractvalue {i8, i1} %agg, 0
+  call void @use8(i8 %adjusted)
+  %underflow = extractvalue {i8, i1} %agg, 1
+  call void @use1(i1 %underflow)
+  %no_underflow = xor i1 %underflow, -1
+  call void @use1(i1 %no_underflow)
+  %not_null = icmp ne i8 %adjusted, 0
+  %r = and i1 %no_underflow, %not_null ; swapped
+  ret i1 %r
+}
+
+; What if we were checking the opposite question, that we either got null,
+; or overflow happened?
+
+define i1 @t7(i8 %base, i8 %offset) {
+; CHECK-LABEL: @t7(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i8 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[UNDERFLOW:%.*]] = icmp ult i8 [[BASE]], [[OFFSET]]
+; CHECK-NEXT:    call void @use1(i1 [[UNDERFLOW]])
+; CHECK-NEXT:    [[NULL:%.*]] = icmp eq i8 [[ADJUSTED]], 0
+; CHECK-NEXT:    call void @use1(i1 [[NULL]])
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ule i8 [[BASE]], [[OFFSET]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %adjusted = sub i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %underflow = icmp ult i8 %base, %offset
+  call void @use1(i1 %underflow)
+  %null = icmp eq i8 %adjusted, 0
+  call void @use1(i1 %null)
+  %r = or i1 %null, %underflow
+  ret i1 %r
+}
+define i1 @t7_nonstrict(i8 %base, i8 %offset) {
+; CHECK-LABEL: @t7_nonstrict(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i8 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[UNDERFLOW:%.*]] = icmp ule i8 [[BASE]], [[OFFSET]]
+; CHECK-NEXT:    call void @use1(i1 [[UNDERFLOW]])
+; CHECK-NEXT:    [[NULL:%.*]] = icmp eq i8 [[ADJUSTED]], 0
+; CHECK-NEXT:    call void @use1(i1 [[NULL]])
+; CHECK-NEXT:    ret i1 [[UNDERFLOW]]
+;
+  %adjusted = sub i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %underflow = icmp ule i8 %base, %offset ; same is valid for non-strict predicate
+  call void @use1(i1 %underflow)
+  %null = icmp eq i8 %adjusted, 0
+  call void @use1(i1 %null)
+  %r = or i1 %null, %underflow
+  ret i1 %r
+}
+
+define i1 @t8(i8 %base, i8 %offset) {
+; CHECK-LABEL: @t8(
+; CHECK-NEXT:    [[AGG:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[BASE:%.*]], i8 [[OFFSET:%.*]])
+; CHECK-NEXT:    call void @useagg({ i8, i1 } [[AGG]])
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = extractvalue { i8, i1 } [[AGG]], 0
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[UNDERFLOW:%.*]] = extractvalue { i8, i1 } [[AGG]], 1
+; CHECK-NEXT:    call void @use1(i1 [[UNDERFLOW]])
+; CHECK-NEXT:    [[NULL:%.*]] = icmp eq i8 [[ADJUSTED]], 0
+; CHECK-NEXT:    [[R:%.*]] = or i1 [[NULL]], [[UNDERFLOW]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %agg = call {i8, i1} @llvm.usub.with.overflow(i8 %base, i8 %offset)
+  call void @useagg({i8, i1} %agg)
+  %adjusted = extractvalue {i8, i1} %agg, 0
+  call void @use8(i8 %adjusted)
+  %underflow = extractvalue {i8, i1} %agg, 1
+  call void @use1(i1 %underflow)
+  %null = icmp eq i8 %adjusted, 0
+  %r = or i1 %null, %underflow
+  ret i1 %r
+}
+
+; And these patterns also have commutative variants
+
+define i1 @t9_commutative(i8 %base, i8 %offset) {
+; CHECK-LABEL: @t9_commutative(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i8 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[UNDERFLOW:%.*]] = icmp ult i8 [[BASE]], [[OFFSET]]
+; CHECK-NEXT:    call void @use1(i1 [[UNDERFLOW]])
+; CHECK-NEXT:    [[NULL:%.*]] = icmp eq i8 [[ADJUSTED]], 0
+; CHECK-NEXT:    call void @use1(i1 [[NULL]])
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ule i8 [[BASE]], [[OFFSET]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %adjusted = sub i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %underflow = icmp ult i8 %base, %adjusted ; swapped
+  call void @use1(i1 %underflow)
+  %null = icmp eq i8 %adjusted, 0
+  call void @use1(i1 %null)
+  %r = or i1 %null, %underflow
+  ret i1 %r
+}
+
+;-------------------------------------------------------------------------------
+
+define i1 @t10(i64 %base, i64* nonnull %offsetptr) {
+; CHECK-LABEL: @t10(
+; CHECK-NEXT:    [[OFFSET:%.*]] = ptrtoint i64* [[OFFSETPTR:%.*]] to i64
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i64 [[BASE:%.*]], [[OFFSET]]
+; CHECK-NEXT:    call void @use64(i64 [[ADJUSTED]])
+; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp ule i64 [[OFFSET]], [[BASE]]
+; CHECK-NEXT:    call void @use1(i1 [[NO_UNDERFLOW]])
+; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp ne i64 [[ADJUSTED]], 0
+; CHECK-NEXT:    call void @use1(i1 [[NOT_NULL]])
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i64 [[OFFSET]], [[BASE]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %offset = ptrtoint i64* %offsetptr to i64
+
+  %adjusted = sub i64 %base, %offset
+  call void @use64(i64 %adjusted)
+  %no_underflow = icmp ult i64 %adjusted, %base
+  call void @use1(i1 %no_underflow)
+  %not_null = icmp ne i64 %adjusted, 0
+  call void @use1(i1 %not_null)
+  %r = and i1 %not_null, %no_underflow
+  ret i1 %r
+}
+define i1 @t11_commutative(i64 %base, i64* nonnull %offsetptr) {
+; CHECK-LABEL: @t11_commutative(
+; CHECK-NEXT:    [[OFFSET:%.*]] = ptrtoint i64* [[OFFSETPTR:%.*]] to i64
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i64 [[BASE:%.*]], [[OFFSET]]
+; CHECK-NEXT:    call void @use64(i64 [[ADJUSTED]])
+; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp ule i64 [[OFFSET]], [[BASE]]
+; CHECK-NEXT:    call void @use1(i1 [[NO_UNDERFLOW]])
+; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp ne i64 [[ADJUSTED]], 0
+; CHECK-NEXT:    call void @use1(i1 [[NOT_NULL]])
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i64 [[OFFSET]], [[BASE]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %offset = ptrtoint i64* %offsetptr to i64
+
+  %adjusted = sub i64 %base, %offset
+  call void @use64(i64 %adjusted)
+  %no_underflow = icmp ugt i64 %base, %adjusted ; swapped
+  call void @use1(i1 %no_underflow)
+  %not_null = icmp ne i64 %adjusted, 0
+  call void @use1(i1 %not_null)
+  %r = and i1 %not_null, %no_underflow
+  ret i1 %r
+}
+
+define i1 @t12(i64 %base, i64* nonnull %offsetptr) {
+; CHECK-LABEL: @t12(
+; CHECK-NEXT:    [[OFFSET:%.*]] = ptrtoint i64* [[OFFSETPTR:%.*]] to i64
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i64 [[BASE:%.*]], [[OFFSET]]
+; CHECK-NEXT:    call void @use64(i64 [[ADJUSTED]])
+; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp ugt i64 [[OFFSET]], [[BASE]]
+; CHECK-NEXT:    call void @use1(i1 [[NO_UNDERFLOW]])
+; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp eq i64 [[ADJUSTED]], 0
+; CHECK-NEXT:    call void @use1(i1 [[NOT_NULL]])
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp uge i64 [[OFFSET]], [[BASE]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %offset = ptrtoint i64* %offsetptr to i64
+
+  %adjusted = sub i64 %base, %offset
+  call void @use64(i64 %adjusted)
+  %no_underflow = icmp uge i64 %adjusted, %base
+  call void @use1(i1 %no_underflow)
+  %not_null = icmp eq i64 %adjusted, 0
+  call void @use1(i1 %not_null)
+  %r = or i1 %not_null, %no_underflow
+  ret i1 %r
+}
+define i1 @t13(i64 %base, i64* nonnull %offsetptr) {
+; CHECK-LABEL: @t13(
+; CHECK-NEXT:    [[OFFSET:%.*]] = ptrtoint i64* [[OFFSETPTR:%.*]] to i64
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i64 [[BASE:%.*]], [[OFFSET]]
+; CHECK-NEXT:    call void @use64(i64 [[ADJUSTED]])
+; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp ugt i64 [[OFFSET]], [[BASE]]
+; CHECK-NEXT:    call void @use1(i1 [[NO_UNDERFLOW]])
+; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp eq i64 [[ADJUSTED]], 0
+; CHECK-NEXT:    call void @use1(i1 [[NOT_NULL]])
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp uge i64 [[OFFSET]], [[BASE]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %offset = ptrtoint i64* %offsetptr to i64
+
+  %adjusted = sub i64 %base, %offset
+  call void @use64(i64 %adjusted)
+  %no_underflow = icmp ule i64 %base, %adjusted ; swapped
+  call void @use1(i1 %no_underflow)
+  %not_null = icmp eq i64 %adjusted, 0
+  call void @use1(i1 %not_null)
+  %r = or i1 %not_null, %no_underflow
+  ret i1 %r
+}
+
+define i1 @t14_bad(i64 %base, i64 %offset) {
+; CHECK-LABEL: @t14_bad(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i64 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use64(i64 [[ADJUSTED]])
+; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp ult i64 [[ADJUSTED]], [[BASE]]
+; CHECK-NEXT:    call void @use1(i1 [[NO_UNDERFLOW]])
+; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp ne i64 [[ADJUSTED]], 0
+; CHECK-NEXT:    call void @use1(i1 [[NOT_NULL]])
+; CHECK-NEXT:    [[R:%.*]] = and i1 [[NOT_NULL]], [[NO_UNDERFLOW]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %adjusted = sub i64 %base, %offset
+  call void @use64(i64 %adjusted)
+  %no_underflow = icmp ult i64 %adjusted, %base
+  call void @use1(i1 %no_underflow)
+  %not_null = icmp ne i64 %adjusted, 0
+  call void @use1(i1 %not_null)
+  %r = and i1 %not_null, %no_underflow
+  ret i1 %r
+}
+
+define i1 @base_ult_offset(i8 %base, i8 %offset) {
+; CHECK-LABEL: @base_ult_offset(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i8 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i8 [[BASE]], [[OFFSET]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %adjusted = sub i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp ne i8 %adjusted, 0
+  %no_underflow = icmp ule i8 %base, %offset
+  %r = and i1 %no_underflow, %not_null
+  ret i1 %r
+}
+define i1 @base_uge_offset(i8 %base, i8 %offset) {
+; CHECK-LABEL: @base_uge_offset(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i8 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp uge i8 [[BASE]], [[OFFSET]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %adjusted = sub i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp eq i8 %adjusted, 0
+  %no_underflow = icmp ugt i8 %base, %offset
+  %r = or i1 %no_underflow, %not_null
+  ret i1 %r
+}
diff --git a/llvm/test/Transforms/InstCombine/reuse-constant-from-select-in-icmp.ll b/llvm/test/Transforms/InstCombine/reuse-constant-from-select-in-icmp.ll
new file mode 100644
index 000000000..6eb21466e
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/reuse-constant-from-select-in-icmp.ll
@@ -0,0 +1,336 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; If we have a relational comparison with a constant, and said comparison is
+; used in a select, and there is a constant in select, see if we can make
+; those constants match.
+
+; We can't ever get non-canonical scalar predicates.
+
+; Likewise, while we can get non-canonical vector predicates, there must be an
+; extra use on that `icmp`, which precludes the fold from happening.
+
+;------------------------------------------------------------------------------;
+; Canonical scalar predicates
+;------------------------------------------------------------------------------;
+
+!0 = !{!"branch_weights", i32 2000, i32 1}
+
+define i32 @p0_ult_65536(i32 %x, i32 %y) {
+; CHECK-LABEL: @p0_ult_65536(
+; CHECK-NEXT:    [[T_INV:%.*]] = icmp ugt i32 [[X:%.*]], 65535
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[T_INV]], i32 65535, i32 [[Y:%.*]], !prof !0
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t = icmp ult i32 %x, 65536
+  %r = select i1 %t, i32 %y, i32 65535, !prof !0
+  ret i32 %r
+}
+define i32 @p1_ugt(i32 %x, i32 %y) {
+; CHECK-LABEL: @p1_ugt(
+; CHECK-NEXT:    [[T_INV:%.*]] = icmp ult i32 [[X:%.*]], 65535
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[T_INV]], i32 65535, i32 [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t = icmp ugt i32 %x, 65534
+  %r = select i1 %t, i32 %y, i32 65535
+  ret i32 %r
+}
+define i32 @p2_slt_65536(i32 %x, i32 %y) {
+; CHECK-LABEL: @p2_slt_65536(
+; CHECK-NEXT:    [[T_INV:%.*]] = icmp sgt i32 [[X:%.*]], 65535
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[T_INV]], i32 65535, i32 [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t = icmp slt i32 %x, 65536
+  %r = select i1 %t, i32 %y, i32 65535
+  ret i32 %r
+}
+define i32 @p3_sgt(i32 %x, i32 %y) {
+; CHECK-LABEL: @p3_sgt(
+; CHECK-NEXT:    [[T_INV:%.*]] = icmp slt i32 [[X:%.*]], 65535
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[T_INV]], i32 65535, i32 [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t = icmp sgt i32 %x, 65534
+  %r = select i1 %t, i32 %y, i32 65535
+  ret i32 %r
+}
+
+;------------------------------------------------------------------------------;
+; Vectors
+;------------------------------------------------------------------------------;
+
+define <2 x i32> @p4_vec_splat_ult_65536(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @p4_vec_splat_ult_65536(
+; CHECK-NEXT:    [[T_INV:%.*]] = icmp ugt <2 x i32> [[X:%.*]], <i32 65535, i32 65535>
+; CHECK-NEXT:    [[R:%.*]] = select <2 x i1> [[T_INV]], <2 x i32> <i32 65535, i32 65535>, <2 x i32> [[Y:%.*]]
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %t = icmp ult <2 x i32> %x, <i32 65536, i32 65536>
+  %r = select <2 x i1> %t, <2 x i32> %y, <2 x i32> <i32 65535, i32 65535>
+  ret <2 x i32> %r
+}
+define <2 x i32> @p5_vec_splat_ugt(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @p5_vec_splat_ugt(
+; CHECK-NEXT:    [[T_INV:%.*]] = icmp ult <2 x i32> [[X:%.*]], <i32 65535, i32 65535>
+; CHECK-NEXT:    [[R:%.*]] = select <2 x i1> [[T_INV]], <2 x i32> <i32 65535, i32 65535>, <2 x i32> [[Y:%.*]]
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %t = icmp ugt <2 x i32> %x, <i32 65534, i32 65534>
+  %r = select <2 x i1> %t, <2 x i32> %y, <2 x i32> <i32 65535, i32 65535>
+  ret <2 x i32> %r
+}
+define <2 x i32> @p6_vec_splat_slt_65536(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @p6_vec_splat_slt_65536(
+; CHECK-NEXT:    [[T_INV:%.*]] = icmp sgt <2 x i32> [[X:%.*]], <i32 65535, i32 65535>
+; CHECK-NEXT:    [[R:%.*]] = select <2 x i1> [[T_INV]], <2 x i32> <i32 65535, i32 65535>, <2 x i32> [[Y:%.*]]
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %t = icmp slt <2 x i32> %x, <i32 65536, i32 65536>
+  %r = select <2 x i1> %t, <2 x i32> %y, <2 x i32> <i32 65535, i32 65535>
+  ret <2 x i32> %r
+}
+define <2 x i32> @p7_vec_splat_sgt(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @p7_vec_splat_sgt(
+; CHECK-NEXT:    [[T_INV:%.*]] = icmp slt <2 x i32> [[X:%.*]], <i32 65535, i32 65535>
+; CHECK-NEXT:    [[R:%.*]] = select <2 x i1> [[T_INV]], <2 x i32> <i32 65535, i32 65535>, <2 x i32> [[Y:%.*]]
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %t = icmp sgt <2 x i32> %x, <i32 65534, i32 65534>
+  %r = select <2 x i1> %t, <2 x i32> %y, <2 x i32> <i32 65535, i32 65535>
+  ret <2 x i32> %r
+}
+
+; Vectors with undef
+
+define <2 x i32> @p8_vec_nonsplat_undef0(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @p8_vec_nonsplat_undef0(
+; CHECK-NEXT:    [[T_INV:%.*]] = icmp ugt <2 x i32> [[X:%.*]], <i32 65535, i32 65535>
+; CHECK-NEXT:    [[R:%.*]] = select <2 x i1> [[T_INV]], <2 x i32> <i32 65535, i32 65535>, <2 x i32> [[Y:%.*]]
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %t = icmp ult <2 x i32> %x, <i32 65536, i32 undef>
+  %r = select <2 x i1> %t, <2 x i32> %y, <2 x i32> <i32 65535, i32 65535>
+  ret <2 x i32> %r
+}
+define <2 x i32> @p9_vec_nonsplat_undef1(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @p9_vec_nonsplat_undef1(
+; CHECK-NEXT:    [[T_INV:%.*]] = icmp ugt <2 x i32> [[X:%.*]], <i32 65535, i32 65535>
+; CHECK-NEXT:    [[R:%.*]] = select <2 x i1> [[T_INV]], <2 x i32> <i32 65535, i32 undef>, <2 x i32> [[Y:%.*]]
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %t = icmp ult <2 x i32> %x, <i32 65536, i32 65536>
+  %r = select <2 x i1> %t, <2 x i32> %y, <2 x i32> <i32 65535, i32 undef>
+  ret <2 x i32> %r
+}
+define <2 x i32> @p10_vec_nonsplat_undef2(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @p10_vec_nonsplat_undef2(
+; CHECK-NEXT:    [[T_INV:%.*]] = icmp ugt <2 x i32> [[X:%.*]], <i32 65535, i32 65535>
+; CHECK-NEXT:    [[R:%.*]] = select <2 x i1> [[T_INV]], <2 x i32> <i32 65535, i32 undef>, <2 x i32> [[Y:%.*]]
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %t = icmp ult <2 x i32> %x, <i32 65536, i32 undef>
+  %r = select <2 x i1> %t, <2 x i32> %y, <2 x i32> <i32 65535, i32 undef>
+  ret <2 x i32> %r
+}
+
+; Non-splat vectors
+
+define <2 x i32> @p11_vec_nonsplat(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @p11_vec_nonsplat(
+; CHECK-NEXT:    [[T_INV:%.*]] = icmp ugt <2 x i32> [[X:%.*]], <i32 65535, i32 32767>
+; CHECK-NEXT:    [[R:%.*]] = select <2 x i1> [[T_INV]], <2 x i32> <i32 65535, i32 32767>, <2 x i32> [[Y:%.*]]
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %t = icmp ult <2 x i32> %x, <i32 65536, i32 32768>
+  %r = select <2 x i1> %t, <2 x i32> %y, <2 x i32> <i32 65535, i32 32767>
+  ret <2 x i32> %r
+}
+
+;------------------------------------------------------------------------------;
+; Extra uses prevent the fold.
+;------------------------------------------------------------------------------;
+
+declare void @use1(i1)
+
+define i32 @n12_extrause(i32 %x, i32 %y) {
+; CHECK-LABEL: @n12_extrause(
+; CHECK-NEXT:    [[T:%.*]] = icmp ult i32 [[X:%.*]], 65536
+; CHECK-NEXT:    call void @use1(i1 [[T]])
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[T]], i32 [[Y:%.*]], i32 65535
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t = icmp ult i32 %x, 65536
+  call void @use1(i1 %t)
+  %r = select i1 %t, i32 %y, i32 65535
+  ret i32 %r
+}
+
+;------------------------------------------------------------------------------;
+; Commutativity
+;------------------------------------------------------------------------------;
+
+; We don't care if the constant in select is true value or false value
+define i32 @p13_commutativity0(i32 %x, i32 %y) {
+; CHECK-LABEL: @p13_commutativity0(
+; CHECK-NEXT:    [[T_INV:%.*]] = icmp ugt i32 [[X:%.*]], 65535
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[T_INV]], i32 [[Y:%.*]], i32 65535
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t = icmp ult i32 %x, 65536
+  %r = select i1 %t, i32 65535, i32 %y
+  ret i32 %r
+}
+
+; Which means, if both possibilities are constants, we must check both of them.
+define i32 @p14_commutativity1(i32 %x, i32 %y) {
+; CHECK-LABEL: @p14_commutativity1(
+; CHECK-NEXT:    [[T_INV:%.*]] = icmp ugt i32 [[X:%.*]], 65535
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[T_INV]], i32 42, i32 65535
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t = icmp ult i32 %x, 65536
+  %r = select i1 %t, i32 65535, i32 42
+  ret i32 %r
+}
+define i32 @p15_commutativity2(i32 %x, i32 %y) {
+; CHECK-LABEL: @p15_commutativity2(
+; CHECK-NEXT:    [[T_INV:%.*]] = icmp ugt i32 [[X:%.*]], 65535
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[T_INV]], i32 65535, i32 42
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t = icmp ult i32 %x, 65536
+  %r = select i1 %t, i32 42, i32 65535
+  ret i32 %r
+}
+
+;------------------------------------------------------------------------------;
+; Negative tests
+;------------------------------------------------------------------------------;
+
+; For vectors, make sure we handle edge cases correctly
+define <2 x i32> @n17_ult_zero(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @n17_ult_zero(
+; CHECK-NEXT:    [[T:%.*]] = icmp ult <2 x i32> [[X:%.*]], <i32 65536, i32 0>
+; CHECK-NEXT:    [[R:%.*]] = select <2 x i1> [[T]], <2 x i32> [[Y:%.*]], <2 x i32> <i32 65535, i32 -1>
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %t = icmp ult <2 x i32> %x, <i32 65536, i32 0>
+  %r = select <2 x i1> %t, <2 x i32> %y, <2 x i32> <i32 65535, i32 -1>
+  ret <2 x i32> %r
+}
+define <2 x i32> @n18_ugt_allones(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @n18_ugt_allones(
+; CHECK-NEXT:    [[T:%.*]] = icmp ugt <2 x i32> [[X:%.*]], <i32 65534, i32 -1>
+; CHECK-NEXT:    [[R:%.*]] = select <2 x i1> [[T]], <2 x i32> [[Y:%.*]], <2 x i32> <i32 65535, i32 0>
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %t = icmp ugt <2 x i32> %x, <i32 65534, i32 -1>
+  %r = select <2 x i1> %t, <2 x i32> %y, <2 x i32> <i32 65535, i32 0>
+  ret <2 x i32> %r
+}
+define <2 x i32> @n19_slt_int_min(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @n19_slt_int_min(
+; CHECK-NEXT:    [[T:%.*]] = icmp slt <2 x i32> [[X:%.*]], <i32 65536, i32 -2147483648>
+; CHECK-NEXT:    [[R:%.*]] = select <2 x i1> [[T]], <2 x i32> [[Y:%.*]], <2 x i32> <i32 65535, i32 2147483647>
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %t = icmp slt <2 x i32> %x, <i32 65536, i32 -2147483648>
+  %r = select <2 x i1> %t, <2 x i32> %y, <2 x i32> <i32 65535, i32 2147483647>
+  ret <2 x i32> %r
+}
+define <2 x i32> @n20_sgt_int_max(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @n20_sgt_int_max(
+; CHECK-NEXT:    [[T:%.*]] = icmp sgt <2 x i32> [[X:%.*]], <i32 65534, i32 2147483647>
+; CHECK-NEXT:    [[R:%.*]] = select <2 x i1> [[T]], <2 x i32> [[Y:%.*]], <2 x i32> <i32 65535, i32 -2147483648>
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %t = icmp sgt <2 x i32> %x, <i32 65534, i32 2147483647>
+  %r = select <2 x i1> %t, <2 x i32> %y, <2 x i32> <i32 65535, i32 -2147483648>
+  ret <2 x i32> %r
+}
+
+; We don't do anything for non-relational comparisons.
+define i32 @n21_equality(i32 %x, i32 %y) {
+; CHECK-LABEL: @n21_equality(
+; CHECK-NEXT:    [[T:%.*]] = icmp eq i32 [[X:%.*]], -2147483648
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[T]], i32 2147483647, i32 [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t = icmp eq i32 %x, -2147483648
+  %r = select i1 %t, i32 2147483647, i32 %y
+  ret i32 %r
+}
+
+; There is nothing special about sign-bit-tests, we can fold them.
+define i32 @t22_sign_check(i32 %x, i32 %y) {
+; CHECK-LABEL: @t22_sign_check(
+; CHECK-NEXT:    [[T_INV:%.*]] = icmp sgt i32 [[X:%.*]], -1
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[T_INV]], i32 [[Y:%.*]], i32 -1
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t = icmp slt i32 %x, 0
+  %r = select i1 %t, i32 -1, i32 %y
+  ret i32 %r
+}
+define i32 @t22_sign_check2(i32 %x, i32 %y) {
+; CHECK-LABEL: @t22_sign_check2(
+; CHECK-NEXT:    [[T_INV:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[T_INV]], i32 [[Y:%.*]], i32 0
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t = icmp sgt i32 %x, -1
+  %r = select i1 %t, i32 0, i32 %y
+  ret i32 %r
+}
+
+; If the types don't match we currently don't do anything.
+define i32 @n23_type_mismatch(i64 %x, i32 %y) {
+; CHECK-LABEL: @n23_type_mismatch(
+; CHECK-NEXT:    [[T:%.*]] = icmp ult i64 [[X:%.*]], 65536
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[T]], i32 [[Y:%.*]], i32 65535
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t = icmp ult i64 %x, 65536
+  %r = select i1 %t, i32 %y, i32 65535
+  ret i32 %r
+}
+
+; Don't do wrong tranform
+define i32 @n24_ult_65534(i32 %x, i32 %y) {
+; CHECK-LABEL: @n24_ult_65534(
+; CHECK-NEXT:    [[T:%.*]] = icmp ult i32 [[X:%.*]], 65534
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[T]], i32 [[Y:%.*]], i32 65535
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t = icmp ult i32 %x, 65534
+  %r = select i1 %t, i32 %y, i32 65535
+  ret i32 %r
+}
+
+; If we already have a match, it's good enough.
+define i32 @n25_all_good0(i32 %x, i32 %y) {
+; CHECK-LABEL: @n25_all_good0(
+; CHECK-NEXT:    [[T:%.*]] = icmp ult i32 [[X:%.*]], 65536
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[T]], i32 65535, i32 65536
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t = icmp ult i32 %x, 65536
+  %r = select i1 %t, i32 65535, i32 65536
+  ret i32 %r
+}
+define i32 @n26_all_good1(i32 %x, i32 %y) {
+; CHECK-LABEL: @n26_all_good1(
+; CHECK-NEXT:    [[T:%.*]] = icmp ult i32 [[X:%.*]], 65536
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[T]], i32 65536, i32 65535
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t = icmp ult i32 %x, 65536
+  %r = select i1 %t, i32 65536, i32 65535
+  ret i32 %r
+}
+
+
+
+; CHECK: !0 = !{!"branch_weights", i32 1, i32 2000}
diff --git a/llvm/test/Transforms/InstCombine/sadd_sat.ll b/llvm/test/Transforms/InstCombine/sadd_sat.ll
new file mode 100644
index 000000000..04dd4f5c0
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/sadd_sat.ll
@@ -0,0 +1,503 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
+
+define i32 @sadd_sat32(i32 %a, i32 %b) {
+; CHECK-LABEL: @sadd_sat32(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.sadd.sat.i32(i32 [[B:%.*]], i32 [[A:%.*]])
+; CHECK-NEXT:    ret i32 [[TMP0]]
+;
+entry:
+  %conv = sext i32 %a to i64
+  %conv1 = sext i32 %b to i64
+  %add = add i64 %conv1, %conv
+  %0 = icmp slt i64 %add, 2147483647
+  %spec.store.select = select i1 %0, i64 %add, i64 2147483647
+  %1 = icmp sgt i64 %spec.store.select, -2147483648
+  %spec.store.select8 = select i1 %1, i64 %spec.store.select, i64 -2147483648
+  %conv7 = trunc i64 %spec.store.select8 to i32
+  ret i32 %conv7
+}
+
+define i32 @ssub_sat32(i32 %a, i32 %b) {
+; CHECK-LABEL: @ssub_sat32(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.ssub.sat.i32(i32 [[A:%.*]], i32 [[B:%.*]])
+; CHECK-NEXT:    ret i32 [[TMP0]]
+;
+entry:
+  %conv = sext i32 %a to i64
+  %conv1 = sext i32 %b to i64
+  %sub = sub i64 %conv, %conv1
+  %0 = icmp slt i64 %sub, 2147483647
+  %spec.store.select = select i1 %0, i64 %sub, i64 2147483647
+  %1 = icmp sgt i64 %spec.store.select, -2147483648
+  %spec.store.select8 = select i1 %1, i64 %spec.store.select, i64 -2147483648
+  %conv7 = trunc i64 %spec.store.select8 to i32
+  ret i32 %conv7
+}
+
+define i32 @smul_sat32(i32 %a, i32 %b) {
+; CHECK-LABEL: @smul_sat32(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CONV:%.*]] = sext i32 [[A:%.*]] to i64
+; CHECK-NEXT:    [[CONV1:%.*]] = sext i32 [[B:%.*]] to i64
+; CHECK-NEXT:    [[ADD:%.*]] = mul nsw i64 [[CONV1]], [[CONV]]
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp slt i64 [[ADD]], 2147483647
+; CHECK-NEXT:    [[SPEC_STORE_SELECT:%.*]] = select i1 [[TMP0]], i64 [[ADD]], i64 2147483647
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i64 [[SPEC_STORE_SELECT]], -2147483648
+; CHECK-NEXT:    [[SPEC_STORE_SELECT8:%.*]] = select i1 [[TMP1]], i64 [[SPEC_STORE_SELECT]], i64 -2147483648
+; CHECK-NEXT:    [[CONV7:%.*]] = trunc i64 [[SPEC_STORE_SELECT8]] to i32
+; CHECK-NEXT:    ret i32 [[CONV7]]
+;
+entry:
+  %conv = sext i32 %a to i64
+  %conv1 = sext i32 %b to i64
+  %add = mul i64 %conv1, %conv
+  %0 = icmp slt i64 %add, 2147483647
+  %spec.store.select = select i1 %0, i64 %add, i64 2147483647
+  %1 = icmp sgt i64 %spec.store.select, -2147483648
+  %spec.store.select8 = select i1 %1, i64 %spec.store.select, i64 -2147483648
+  %conv7 = trunc i64 %spec.store.select8 to i32
+  ret i32 %conv7
+}
+
+define signext i16 @sadd_sat16(i16 signext %a, i16 signext %b) {
+; CHECK-LABEL: @sadd_sat16(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = call i16 @llvm.sadd.sat.i16(i16 [[B:%.*]], i16 [[A:%.*]])
+; CHECK-NEXT:    ret i16 [[TMP0]]
+;
+entry:
+  %conv = sext i16 %a to i32
+  %conv1 = sext i16 %b to i32
+  %add = add i32 %conv1, %conv
+  %0 = icmp slt i32 %add, 32767
+  %spec.store.select = select i1 %0, i32 %add, i32 32767
+  %1 = icmp sgt i32 %spec.store.select, -32768
+  %spec.store.select10 = select i1 %1, i32 %spec.store.select, i32 -32768
+  %conv9 = trunc i32 %spec.store.select10 to i16
+  ret i16 %conv9
+}
+
+define signext i16 @ssub_sat16(i16 signext %a, i16 signext %b) {
+; CHECK-LABEL: @ssub_sat16(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = call i16 @llvm.ssub.sat.i16(i16 [[A:%.*]], i16 [[B:%.*]])
+; CHECK-NEXT:    ret i16 [[TMP0]]
+;
+entry:
+  %conv = sext i16 %a to i32
+  %conv1 = sext i16 %b to i32
+  %sub = sub i32 %conv, %conv1
+  %0 = icmp slt i32 %sub, 32767
+  %spec.store.select = select i1 %0, i32 %sub, i32 32767
+  %1 = icmp sgt i32 %spec.store.select, -32768
+  %spec.store.select10 = select i1 %1, i32 %spec.store.select, i32 -32768
+  %conv9 = trunc i32 %spec.store.select10 to i16
+  ret i16 %conv9
+}
+
+define signext i8 @sadd_sat8(i8 signext %a, i8 signext %b) {
+; CHECK-LABEL: @sadd_sat8(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = call i8 @llvm.sadd.sat.i8(i8 [[B:%.*]], i8 [[A:%.*]])
+; CHECK-NEXT:    ret i8 [[TMP0]]
+;
+entry:
+  %conv = sext i8 %a to i32
+  %conv1 = sext i8 %b to i32
+  %add = add i32 %conv1, %conv
+  %0 = icmp slt i32 %add, 127
+  %spec.store.select = select i1 %0, i32 %add, i32 127
+  %1 = icmp sgt i32 %spec.store.select, -128
+  %spec.store.select10 = select i1 %1, i32 %spec.store.select, i32 -128
+  %conv9 = trunc i32 %spec.store.select10 to i8
+  ret i8 %conv9
+}
+
+define signext i8 @ssub_sat8(i8 signext %a, i8 signext %b) {
+; CHECK-LABEL: @ssub_sat8(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = call i8 @llvm.ssub.sat.i8(i8 [[A:%.*]], i8 [[B:%.*]])
+; CHECK-NEXT:    ret i8 [[TMP0]]
+;
+entry:
+  %conv = sext i8 %a to i32
+  %conv1 = sext i8 %b to i32
+  %sub = sub i32 %conv, %conv1
+  %0 = icmp slt i32 %sub, 127
+  %spec.store.select = select i1 %0, i32 %sub, i32 127
+  %1 = icmp sgt i32 %spec.store.select, -128
+  %spec.store.select10 = select i1 %1, i32 %spec.store.select, i32 -128
+  %conv9 = trunc i32 %spec.store.select10 to i8
+  ret i8 %conv9
+}
+
+define signext i64 @sadd_sat64(i64 signext %a, i64 signext %b) {
+; CHECK-LABEL: @sadd_sat64(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.sadd.sat.i64(i64 [[B:%.*]], i64 [[A:%.*]])
+; CHECK-NEXT:    ret i64 [[TMP0]]
+;
+entry:
+  %conv = sext i64 %a to i65
+  %conv1 = sext i64 %b to i65
+  %add = add i65 %conv1, %conv
+  %0 = icmp slt i65 %add, 9223372036854775807
+  %spec.store.select = select i1 %0, i65 %add, i65 9223372036854775807
+  %1 = icmp sgt i65 %spec.store.select, -9223372036854775808
+  %spec.store.select10 = select i1 %1, i65 %spec.store.select, i65 -9223372036854775808
+  %conv9 = trunc i65 %spec.store.select10 to i64
+  ret i64 %conv9
+}
+
+define signext i64 @ssub_sat64(i64 signext %a, i64 signext %b) {
+; CHECK-LABEL: @ssub_sat64(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.ssub.sat.i64(i64 [[A:%.*]], i64 [[B:%.*]])
+; CHECK-NEXT:    ret i64 [[TMP0]]
+;
+entry:
+  %conv = sext i64 %a to i65
+  %conv1 = sext i64 %b to i65
+  %sub = sub i65 %conv, %conv1
+  %0 = icmp slt i65 %sub, 9223372036854775807
+  %spec.store.select = select i1 %0, i65 %sub, i65 9223372036854775807
+  %1 = icmp sgt i65 %spec.store.select, -9223372036854775808
+  %spec.store.select10 = select i1 %1, i65 %spec.store.select, i65 -9223372036854775808
+  %conv9 = trunc i65 %spec.store.select10 to i64
+  ret i64 %conv9
+}
+
+define signext i4 @sadd_sat4(i4 signext %a, i4 signext %b) {
+; CHECK-LABEL: @sadd_sat4(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CONV:%.*]] = sext i4 [[A:%.*]] to i32
+; CHECK-NEXT:    [[CONV1:%.*]] = sext i4 [[B:%.*]] to i32
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[CONV1]], [[CONV]]
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp slt i32 [[ADD]], 7
+; CHECK-NEXT:    [[SPEC_STORE_SELECT:%.*]] = select i1 [[TMP0]], i32 [[ADD]], i32 7
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[SPEC_STORE_SELECT]], -8
+; CHECK-NEXT:    [[SPEC_STORE_SELECT10:%.*]] = select i1 [[TMP1]], i32 [[SPEC_STORE_SELECT]], i32 -8
+; CHECK-NEXT:    [[CONV9:%.*]] = trunc i32 [[SPEC_STORE_SELECT10]] to i4
+; CHECK-NEXT:    ret i4 [[CONV9]]
+;
+entry:
+  %conv = sext i4 %a to i32
+  %conv1 = sext i4 %b to i32
+  %add = add i32 %conv1, %conv
+  %0 = icmp slt i32 %add, 7
+  %spec.store.select = select i1 %0, i32 %add, i32 7
+  %1 = icmp sgt i32 %spec.store.select, -8
+  %spec.store.select10 = select i1 %1, i32 %spec.store.select, i32 -8
+  %conv9 = trunc i32 %spec.store.select10 to i4
+  ret i4 %conv9
+}
+
+define signext i4 @ssub_sat4(i4 signext %a, i4 signext %b) {
+; CHECK-LABEL: @ssub_sat4(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CONV:%.*]] = sext i4 [[A:%.*]] to i32
+; CHECK-NEXT:    [[CONV1:%.*]] = sext i4 [[B:%.*]] to i32
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 [[CONV]], [[CONV1]]
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp slt i32 [[SUB]], 7
+; CHECK-NEXT:    [[SPEC_STORE_SELECT:%.*]] = select i1 [[TMP0]], i32 [[SUB]], i32 7
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[SPEC_STORE_SELECT]], -8
+; CHECK-NEXT:    [[SPEC_STORE_SELECT10:%.*]] = select i1 [[TMP1]], i32 [[SPEC_STORE_SELECT]], i32 -8
+; CHECK-NEXT:    [[CONV9:%.*]] = trunc i32 [[SPEC_STORE_SELECT10]] to i4
+; CHECK-NEXT:    ret i4 [[CONV9]]
+;
+entry:
+  %conv = sext i4 %a to i32
+  %conv1 = sext i4 %b to i32
+  %sub = sub i32 %conv, %conv1
+  %0 = icmp slt i32 %sub, 7
+  %spec.store.select = select i1 %0, i32 %sub, i32 7
+  %1 = icmp sgt i32 %spec.store.select, -8
+  %spec.store.select10 = select i1 %1, i32 %spec.store.select, i32 -8
+  %conv9 = trunc i32 %spec.store.select10 to i4
+  ret i4 %conv9
+}
+
+define <4 x i32> @sadd_satv4i32(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: @sadd_satv4i32(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> [[B:%.*]], <4 x i32> [[A:%.*]])
+; CHECK-NEXT:    ret <4 x i32> [[TMP0]]
+;
+entry:
+  %conv = sext <4 x i32> %a to <4 x i64>
+  %conv1 = sext <4 x i32> %b to <4 x i64>
+  %add = add <4 x i64> %conv1, %conv
+  %0 = icmp slt <4 x i64> %add, <i64 2147483647, i64 2147483647, i64 2147483647, i64 2147483647>
+  %spec.store.select = select <4 x i1> %0, <4 x i64> %add, <4 x i64> <i64 2147483647, i64 2147483647, i64 2147483647, i64 2147483647>
+  %1 = icmp sgt <4 x i64> %spec.store.select, <i64 -2147483648, i64 -2147483648, i64 -2147483648, i64 -2147483648>
+  %spec.store.select8 = select <4 x i1> %1, <4 x i64> %spec.store.select, <4 x i64> <i64 -2147483648, i64 -2147483648, i64 -2147483648, i64 -2147483648>
+  %conv7 = trunc <4 x i64> %spec.store.select8 to <4 x i32>
+  ret <4 x i32> %conv7
+}
+
+define <4 x i32> @ssub_satv4i32(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: @ssub_satv4i32(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> [[B:%.*]], <4 x i32> [[A:%.*]])
+; CHECK-NEXT:    ret <4 x i32> [[TMP0]]
+;
+entry:
+  %conv = sext <4 x i32> %a to <4 x i64>
+  %conv1 = sext <4 x i32> %b to <4 x i64>
+  %add = sub <4 x i64> %conv1, %conv
+  %0 = icmp slt <4 x i64> %add, <i64 2147483647, i64 2147483647, i64 2147483647, i64 2147483647>
+  %spec.store.select = select <4 x i1> %0, <4 x i64> %add, <4 x i64> <i64 2147483647, i64 2147483647, i64 2147483647, i64 2147483647>
+  %1 = icmp sgt <4 x i64> %spec.store.select, <i64 -2147483648, i64 -2147483648, i64 -2147483648, i64 -2147483648>
+  %spec.store.select8 = select <4 x i1> %1, <4 x i64> %spec.store.select, <4 x i64> <i64 -2147483648, i64 -2147483648, i64 -2147483648, i64 -2147483648>
+  %conv7 = trunc <4 x i64> %spec.store.select8 to <4 x i32>
+  ret <4 x i32> %conv7
+}
+
+define <4 x i32> @sadd_satv4i4(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: @sadd_satv4i4(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ADD:%.*]] = add <4 x i32> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp slt <4 x i32> [[ADD]], <i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT:    [[SPEC_STORE_SELECT:%.*]] = select <4 x i1> [[TMP0]], <4 x i32> [[ADD]], <4 x i32> <i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <4 x i32> [[SPEC_STORE_SELECT]], <i32 -16, i32 -16, i32 -16, i32 -16>
+; CHECK-NEXT:    [[SPEC_STORE_SELECT8:%.*]] = select <4 x i1> [[TMP1]], <4 x i32> [[SPEC_STORE_SELECT]], <4 x i32> <i32 -16, i32 -16, i32 -16, i32 -16>
+; CHECK-NEXT:    ret <4 x i32> [[SPEC_STORE_SELECT8]]
+;
+entry:
+  %add = add <4 x i32> %a, %b
+  %0 = icmp slt <4 x i32> %add, <i32 15, i32 15, i32 15, i32 15>
+  %spec.store.select = select <4 x i1> %0, <4 x i32> %add, <4 x i32> <i32 15, i32 15, i32 15, i32 15>
+  %1 = icmp sgt <4 x i32> %spec.store.select, <i32 -16, i32 -16, i32 -16, i32 -16>
+  %spec.store.select8 = select <4 x i1> %1, <4 x i32> %spec.store.select, <4 x i32> <i32 -16, i32 -16, i32 -16, i32 -16>
+  ret <4 x i32> %spec.store.select8
+}
+
+define <4 x i32> @ssub_satv4i4(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: @ssub_satv4i4(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ADD:%.*]] = sub <4 x i32> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp slt <4 x i32> [[ADD]], <i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT:    [[SPEC_STORE_SELECT:%.*]] = select <4 x i1> [[TMP0]], <4 x i32> [[ADD]], <4 x i32> <i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <4 x i32> [[SPEC_STORE_SELECT]], <i32 -16, i32 -16, i32 -16, i32 -16>
+; CHECK-NEXT:    [[SPEC_STORE_SELECT8:%.*]] = select <4 x i1> [[TMP1]], <4 x i32> [[SPEC_STORE_SELECT]], <4 x i32> <i32 -16, i32 -16, i32 -16, i32 -16>
+; CHECK-NEXT:    ret <4 x i32> [[SPEC_STORE_SELECT8]]
+;
+entry:
+  %add = sub <4 x i32> %a, %b
+  %0 = icmp slt <4 x i32> %add, <i32 15, i32 15, i32 15, i32 15>
+  %spec.store.select = select <4 x i1> %0, <4 x i32> %add, <4 x i32> <i32 15, i32 15, i32 15, i32 15>
+  %1 = icmp sgt <4 x i32> %spec.store.select, <i32 -16, i32 -16, i32 -16, i32 -16>
+  %spec.store.select8 = select <4 x i1> %1, <4 x i32> %spec.store.select, <4 x i32> <i32 -16, i32 -16, i32 -16, i32 -16>
+  ret <4 x i32> %spec.store.select8
+}
+
+
+define i32 @sadd_sat32_extrause_1(i32 %a, i32 %b) {
+; CHECK-LABEL: @sadd_sat32_extrause_1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.sadd.sat.i32(i32 [[B:%.*]], i32 [[A:%.*]])
+; CHECK-NEXT:    [[SPEC_STORE_SELECT8:%.*]] = sext i32 [[TMP0]] to i64
+; CHECK-NEXT:    call void @use64(i64 [[SPEC_STORE_SELECT8]])
+; CHECK-NEXT:    ret i32 [[TMP0]]
+;
+entry:
+  %conv = sext i32 %a to i64
+  %conv1 = sext i32 %b to i64
+  %add = add i64 %conv1, %conv
+  %0 = icmp slt i64 %add, 2147483647
+  %spec.store.select = select i1 %0, i64 %add, i64 2147483647
+  %1 = icmp sgt i64 %spec.store.select, -2147483648
+  %spec.store.select8 = select i1 %1, i64 %spec.store.select, i64 -2147483648
+  %conv7 = trunc i64 %spec.store.select8 to i32
+  call void @use64(i64 %spec.store.select8)
+  ret i32 %conv7
+}
+
+define i32 @sadd_sat32_extrause_2(i32 %a, i32 %b) {
+; CHECK-LABEL: @sadd_sat32_extrause_2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CONV:%.*]] = sext i32 [[A:%.*]] to i64
+; CHECK-NEXT:    [[CONV1:%.*]] = sext i32 [[B:%.*]] to i64
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i64 [[CONV1]], [[CONV]]
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp slt i64 [[ADD]], 2147483647
+; CHECK-NEXT:    [[SPEC_STORE_SELECT:%.*]] = select i1 [[TMP0]], i64 [[ADD]], i64 2147483647
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i64 [[SPEC_STORE_SELECT]], -2147483648
+; CHECK-NEXT:    [[SPEC_STORE_SELECT8:%.*]] = select i1 [[TMP1]], i64 [[SPEC_STORE_SELECT]], i64 -2147483648
+; CHECK-NEXT:    [[CONV7:%.*]] = trunc i64 [[SPEC_STORE_SELECT8]] to i32
+; CHECK-NEXT:    call void @use64(i64 [[SPEC_STORE_SELECT]])
+; CHECK-NEXT:    ret i32 [[CONV7]]
+;
+entry:
+  %conv = sext i32 %a to i64
+  %conv1 = sext i32 %b to i64
+  %add = add i64 %conv1, %conv
+  %0 = icmp slt i64 %add, 2147483647
+  %spec.store.select = select i1 %0, i64 %add, i64 2147483647
+  %1 = icmp sgt i64 %spec.store.select, -2147483648
+  %spec.store.select8 = select i1 %1, i64 %spec.store.select, i64 -2147483648
+  %conv7 = trunc i64 %spec.store.select8 to i32
+  call void @use64(i64 %spec.store.select)
+  ret i32 %conv7
+}
+
+define i32 @sadd_sat32_extrause_3(i32 %a, i32 %b) {
+; CHECK-LABEL: @sadd_sat32_extrause_3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CONV:%.*]] = sext i32 [[A:%.*]] to i64
+; CHECK-NEXT:    [[CONV1:%.*]] = sext i32 [[B:%.*]] to i64
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i64 [[CONV1]], [[CONV]]
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp slt i64 [[ADD]], 2147483647
+; CHECK-NEXT:    [[SPEC_STORE_SELECT:%.*]] = select i1 [[TMP0]], i64 [[ADD]], i64 2147483647
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i64 [[SPEC_STORE_SELECT]], -2147483648
+; CHECK-NEXT:    [[SPEC_STORE_SELECT8:%.*]] = select i1 [[TMP1]], i64 [[SPEC_STORE_SELECT]], i64 -2147483648
+; CHECK-NEXT:    [[CONV7:%.*]] = trunc i64 [[SPEC_STORE_SELECT8]] to i32
+; CHECK-NEXT:    call void @use64(i64 [[ADD]])
+; CHECK-NEXT:    ret i32 [[CONV7]]
+;
+entry:
+  %conv = sext i32 %a to i64
+  %conv1 = sext i32 %b to i64
+  %add = add i64 %conv1, %conv
+  %0 = icmp slt i64 %add, 2147483647
+  %spec.store.select = select i1 %0, i64 %add, i64 2147483647
+  %1 = icmp sgt i64 %spec.store.select, -2147483648
+  %spec.store.select8 = select i1 %1, i64 %spec.store.select, i64 -2147483648
+  %conv7 = trunc i64 %spec.store.select8 to i32
+  call void @use64(i64 %add)
+  ret i32 %conv7
+}
+
+define i32 @sadd_sat32_trunc(i32 %a, i32 %b) {
+; CHECK-LABEL: @sadd_sat32_trunc(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CONV:%.*]] = sext i32 [[A:%.*]] to i64
+; CHECK-NEXT:    [[CONV1:%.*]] = sext i32 [[B:%.*]] to i64
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i64 [[CONV1]], [[CONV]]
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp slt i64 [[ADD]], 32767
+; CHECK-NEXT:    [[SPEC_STORE_SELECT:%.*]] = select i1 [[TMP0]], i64 [[ADD]], i64 32767
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i64 [[SPEC_STORE_SELECT]], -32768
+; CHECK-NEXT:    [[SPEC_STORE_SELECT8:%.*]] = select i1 [[TMP1]], i64 [[SPEC_STORE_SELECT]], i64 -32768
+; CHECK-NEXT:    [[CONV7:%.*]] = trunc i64 [[SPEC_STORE_SELECT8]] to i32
+; CHECK-NEXT:    ret i32 [[CONV7]]
+;
+entry:
+  %conv = sext i32 %a to i64
+  %conv1 = sext i32 %b to i64
+  %add = add i64 %conv1, %conv
+  %0 = icmp slt i64 %add, 32767
+  %spec.store.select = select i1 %0, i64 %add, i64 32767
+  %1 = icmp sgt i64 %spec.store.select, -32768
+  %spec.store.select8 = select i1 %1, i64 %spec.store.select, i64 -32768
+  %conv7 = trunc i64 %spec.store.select8 to i32
+  ret i32 %conv7
+}
+
+define i32 @sadd_sat32_ext16(i32 %a, i16 %b) {
+; CHECK-LABEL: @sadd_sat32_ext16(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = sext i16 [[B:%.*]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.sadd.sat.i32(i32 [[TMP0]], i32 [[A:%.*]])
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+entry:
+  %conv = sext i32 %a to i64
+  %conv1 = sext i16 %b to i64
+  %add = add i64 %conv1, %conv
+  %0 = icmp slt i64 %add, 2147483647
+  %spec.store.select = select i1 %0, i64 %add, i64 2147483647
+  %1 = icmp sgt i64 %spec.store.select, -2147483648
+  %spec.store.select8 = select i1 %1, i64 %spec.store.select, i64 -2147483648
+  %conv7 = trunc i64 %spec.store.select8 to i32
+  ret i32 %conv7
+}
+
+define i8 @sadd_sat8_ext8(i8 %a, i16 %b) {
+; CHECK-LABEL: @sadd_sat8_ext8(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CONV:%.*]] = sext i8 [[A:%.*]] to i32
+; CHECK-NEXT:    [[CONV1:%.*]] = sext i16 [[B:%.*]] to i32
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[CONV1]], [[CONV]]
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp slt i32 [[ADD]], 127
+; CHECK-NEXT:    [[SPEC_STORE_SELECT:%.*]] = select i1 [[TMP0]], i32 [[ADD]], i32 127
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[SPEC_STORE_SELECT]], -128
+; CHECK-NEXT:    [[SPEC_STORE_SELECT8:%.*]] = select i1 [[TMP1]], i32 [[SPEC_STORE_SELECT]], i32 -128
+; CHECK-NEXT:    [[CONV7:%.*]] = trunc i32 [[SPEC_STORE_SELECT8]] to i8
+; CHECK-NEXT:    ret i8 [[CONV7]]
+;
+entry:
+  %conv = sext i8 %a to i32
+  %conv1 = sext i16 %b to i32
+  %add = add i32 %conv1, %conv
+  %0 = icmp slt i32 %add, 127
+  %spec.store.select = select i1 %0, i32 %add, i32 127
+  %1 = icmp sgt i32 %spec.store.select, -128
+  %spec.store.select8 = select i1 %1, i32 %spec.store.select, i32 -128
+  %conv7 = trunc i32 %spec.store.select8 to i8
+  ret i8 %conv7
+}
+
+define i32 @sadd_sat32_zext(i32 %a, i32 %b) {
+; CHECK-LABEL: @sadd_sat32_zext(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CONV:%.*]] = zext i32 [[A:%.*]] to i64
+; CHECK-NEXT:    [[CONV1:%.*]] = zext i32 [[B:%.*]] to i64
+; CHECK-NEXT:    [[ADD:%.*]] = add nuw nsw i64 [[CONV1]], [[CONV]]
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ult i64 [[ADD]], 2147483647
+; CHECK-NEXT:    [[SPEC_STORE_SELECT:%.*]] = select i1 [[TMP0]], i64 [[ADD]], i64 2147483647
+; CHECK-NEXT:    [[CONV7:%.*]] = trunc i64 [[SPEC_STORE_SELECT]] to i32
+; CHECK-NEXT:    ret i32 [[CONV7]]
+;
+entry:
+  %conv = zext i32 %a to i64
+  %conv1 = zext i32 %b to i64
+  %add = add i64 %conv1, %conv
+  %0 = icmp slt i64 %add, 2147483647
+  %spec.store.select = select i1 %0, i64 %add, i64 2147483647
+  %1 = icmp sgt i64 %spec.store.select, -2147483648
+  %spec.store.select8 = select i1 %1, i64 %spec.store.select, i64 -2147483648
+  %conv7 = trunc i64 %spec.store.select8 to i32
+  ret i32 %conv7
+}
+
+define i32 @sadd_sat32_maxmin(i32 %a, i32 %b) {
+; CHECK-LABEL: @sadd_sat32_maxmin(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.sadd.sat.i32(i32 [[B:%.*]], i32 [[A:%.*]])
+; CHECK-NEXT:    ret i32 [[TMP0]]
+;
+entry:
+  %conv = sext i32 %a to i64
+  %conv1 = sext i32 %b to i64
+  %add = add i64 %conv1, %conv
+  %0 = icmp sgt i64 %add, -2147483648
+  %spec.store.select = select i1 %0, i64 %add, i64 -2147483648
+  %1 = icmp slt i64 %spec.store.select, 2147483647
+  %spec.store.select8 = select i1 %1, i64 %spec.store.select, i64 2147483647
+  %conv7 = trunc i64 %spec.store.select8 to i32
+  ret i32 %conv7
+}
+
+define i64 @sadd_sat32_notrunc(i32 %a, i32 %b) {
+; CHECK-LABEL: @sadd_sat32_notrunc(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.sadd.sat.i32(i32 [[B:%.*]], i32 [[A:%.*]])
+; CHECK-NEXT:    [[SPEC_STORE_SELECT8:%.*]] = sext i32 [[TMP0]] to i64
+; CHECK-NEXT:    ret i64 [[SPEC_STORE_SELECT8]]
+;
+entry:
+  %conv = sext i32 %a to i64
+  %conv1 = sext i32 %b to i64
+  %add = add i64 %conv1, %conv
+  %0 = icmp sgt i64 %add, -2147483648
+  %spec.store.select = select i1 %0, i64 %add, i64 -2147483648
+  %1 = icmp slt i64 %spec.store.select, 2147483647
+  %spec.store.select8 = select i1 %1, i64 %spec.store.select, i64 2147483647
+  ret i64 %spec.store.select8
+}
+
+declare void @use64(i64)
diff --git a/llvm/test/Transforms/InstCombine/saturating-add-sub.ll b/llvm/test/Transforms/InstCombine/saturating-add-sub.ll
index 364c80d20..57ef7515e 100644
--- a/llvm/test/Transforms/InstCombine/saturating-add-sub.ll
+++ b/llvm/test/Transforms/InstCombine/saturating-add-sub.ll
@@ -1156,6 +1156,48 @@ define i8 @test_scalar_uadd_sub_const(i8 %a) {
   ret i8 %res
 }
 
+define i1 @scalar_uadd_eq_zero(i8 %a, i8 %b) {
+; CHECK-LABEL: @scalar_uadd_eq_zero(
+; CHECK-NEXT:    [[TMP1:%.*]] = or i8 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i8 [[TMP1]], 0
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %sat = call i8 @llvm.uadd.sat.i8(i8 %a, i8 %b)
+  %cmp = icmp eq i8 %sat, 0
+  ret i1 %cmp
+}
+
+define i1 @scalar_uadd_ne_zero(i8 %a, i8 %b) {
+; CHECK-LABEL: @scalar_uadd_ne_zero(
+; CHECK-NEXT:    [[TMP1:%.*]] = or i8 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i8 [[TMP1]], 0
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %sat = call i8 @llvm.uadd.sat.i8(i8 %a, i8 %b)
+  %cmp = icmp ne i8 %sat, 0
+  ret i1 %cmp
+}
+
+define i1 @scalar_usub_eq_zero(i8 %a, i8 %b) {
+; CHECK-LABEL: @scalar_usub_eq_zero(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ule i8 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %sat = call i8 @llvm.usub.sat.i8(i8 %a, i8 %b)
+  %cmp = icmp eq i8 %sat, 0
+  ret i1 %cmp
+}
+
+define i1 @scalar_usub_ne_zero(i8 %a, i8 %b) {
+; CHECK-LABEL: @scalar_usub_ne_zero(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i8 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %sat = call i8 @llvm.usub.sat.i8(i8 %a, i8 %b)
+  %cmp = icmp ne i8 %sat, 0
+  ret i1 %cmp
+}
+
 ; Raw IR tests
 
 define i32 @uadd_sat(i32 %x, i32 %y) {
@@ -1442,6 +1484,50 @@ define i32 @uadd_sat_constant_commute(i32 %x) {
   ret i32 %r
 }
 
+define i32 @uadd_sat_canon(i32 %x, i32 %y) {
+; CHECK-LABEL: @uadd_sat_canon(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.uadd.sat.i32(i32 [[X:%.*]], i32 [[Y:%.*]])
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %a = add i32 %x, %y
+  %c = icmp ult i32 %a, %x
+  %r = select i1 %c, i32 -1, i32 %a
+  ret i32 %r
+}
+
+define i32 @uadd_sat_canon_y(i32 %x, i32 %y) {
+; CHECK-LABEL: @uadd_sat_canon_y(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.uadd.sat.i32(i32 [[Y:%.*]], i32 [[X:%.*]])
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %a = add i32 %x, %y
+  %c = icmp ult i32 %a, %y
+  %r = select i1 %c, i32 -1, i32 %a
+  ret i32 %r
+}
+
+define i32 @uadd_sat_canon_nuw(i32 %x, i32 %y) {
+; CHECK-LABEL: @uadd_sat_canon_nuw(
+; CHECK-NEXT:    [[A:%.*]] = add nuw i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[A]]
+;
+  %a = add nuw i32 %x, %y
+  %c = icmp ult i32 %a, %x
+  %r = select i1 %c, i32 -1, i32 %a
+  ret i32 %r
+}
+
+define i32 @uadd_sat_canon_y_nuw(i32 %x, i32 %y) {
+; CHECK-LABEL: @uadd_sat_canon_y_nuw(
+; CHECK-NEXT:    [[A:%.*]] = add nuw i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[A]]
+;
+  %a = add nuw i32 %x, %y
+  %c = icmp ult i32 %a, %y
+  %r = select i1 %c, i32 -1, i32 %a
+  ret i32 %r
+}
+
 define <4 x i32> @uadd_sat_constant_vec(<4 x i32> %x) {
 ; CHECK-LABEL: @uadd_sat_constant_vec(
 ; CHECK-NEXT:    [[A:%.*]] = add <4 x i32> [[X:%.*]], <i32 42, i32 42, i32 42, i32 42>
diff --git a/llvm/test/Transforms/InstCombine/select-bitext-bitwise-ops.ll b/llvm/test/Transforms/InstCombine/select-bitext-bitwise-ops.ll
index 25aadd651..d05f7245e 100644
--- a/llvm/test/Transforms/InstCombine/select-bitext-bitwise-ops.ll
+++ b/llvm/test/Transforms/InstCombine/select-bitext-bitwise-ops.ll
@@ -3,13 +3,11 @@
 
 define i64 @sel_false_val_is_a_masked_shl_of_true_val1(i32 %x, i64 %y) {
 ; CHECK-LABEL: @sel_false_val_is_a_masked_shl_of_true_val1(
-; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 15
-; CHECK-NEXT:    [[TMP2:%.*]] = shl nuw nsw i32 [[TMP1]], 2
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[TMP1]], 0
-; CHECK-NEXT:    [[NARROW:%.*]] = select i1 [[TMP3]], i32 0, i32 [[TMP2]]
-; CHECK-NEXT:    [[TMP4:%.*]] = zext i32 [[NARROW]] to i64
-; CHECK-NEXT:    [[TMP5:%.*]] = ashr i64 [[Y:%.*]], [[TMP4]]
-; CHECK-NEXT:    ret i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 [[X:%.*]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], 60
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i32 [[TMP2]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = ashr i64 [[Y:%.*]], [[TMP3]]
+; CHECK-NEXT:    ret i64 [[TMP4]]
 ;
   %1 = and i32 %x, 15
   %2 = shl nuw nsw i32 %1, 2
@@ -39,13 +37,11 @@ define i64 @sel_false_val_is_a_masked_shl_of_true_val2(i32 %x, i64 %y) {
 
 define i64 @sel_false_val_is_a_masked_lshr_of_true_val1(i32 %x, i64 %y) {
 ; CHECK-LABEL: @sel_false_val_is_a_masked_lshr_of_true_val1(
-; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 60
-; CHECK-NEXT:    [[TMP2:%.*]] = lshr exact i32 [[TMP1]], 2
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[TMP1]], 0
-; CHECK-NEXT:    [[NARROW:%.*]] = select i1 [[TMP3]], i32 0, i32 [[TMP2]]
-; CHECK-NEXT:    [[TMP4:%.*]] = zext i32 [[NARROW]] to i64
-; CHECK-NEXT:    [[TMP5:%.*]] = ashr i64 [[Y:%.*]], [[TMP4]]
-; CHECK-NEXT:    ret i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i32 [[X:%.*]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], 15
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i32 [[TMP2]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = ashr i64 [[Y:%.*]], [[TMP3]]
+; CHECK-NEXT:    ret i64 [[TMP4]]
 ;
   %1 = and i32 %x, 60
   %2 = lshr i32 %1, 2
@@ -75,13 +71,11 @@ define i64 @sel_false_val_is_a_masked_lshr_of_true_val2(i32 %x, i64 %y) {
 
 define i64 @sel_false_val_is_a_masked_ashr_of_true_val1(i32 %x, i64 %y) {
 ; CHECK-LABEL: @sel_false_val_is_a_masked_ashr_of_true_val1(
-; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], -2147483588
-; CHECK-NEXT:    [[TMP2:%.*]] = ashr exact i32 [[TMP1]], 2
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[TMP1]], 0
-; CHECK-NEXT:    [[NARROW:%.*]] = select i1 [[TMP3]], i32 0, i32 [[TMP2]]
-; CHECK-NEXT:    [[TMP4:%.*]] = zext i32 [[NARROW]] to i64
-; CHECK-NEXT:    [[TMP5:%.*]] = ashr i64 [[Y:%.*]], [[TMP4]]
-; CHECK-NEXT:    ret i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr i32 [[X:%.*]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], -536870897
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i32 [[TMP2]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = ashr i64 [[Y:%.*]], [[TMP3]]
+; CHECK-NEXT:    ret i64 [[TMP4]]
 ;
   %1 = and i32 %x, -2147483588
   %2 = ashr i32 %1, 2
diff --git a/llvm/test/Transforms/InstCombine/select-crash.ll b/llvm/test/Transforms/InstCombine/select-crash.ll
index 41b69d251..9de338d95 100644
--- a/llvm/test/Transforms/InstCombine/select-crash.ll
+++ b/llvm/test/Transforms/InstCombine/select-crash.ll
@@ -4,7 +4,7 @@
 define fastcc double @gimp_operation_color_balance_map(float %value, double %highlights) nounwind readnone inlinehint {
 entry:
 ; CHECK: gimp_operation_color_balance_map
-; CHECK: fsub double -0.000000
+; CHECK: fneg double
   %conv = fpext float %value to double
   %div = fdiv double %conv, 1.600000e+01
   %add = fadd double %div, 1.000000e+00
@@ -22,7 +22,7 @@ entry:
 ; PR10180: same crash, but with vectors
 define <4 x float> @foo(i1 %b, <4 x float> %x, <4 x float> %y, <4 x float> %z) {
 ; CHECK-LABEL: @foo(
-; CHECK: fsub <4 x float>
+; CHECK: fneg <4 x float>
 ; CHECK: select
 ; CHECK: fadd <4 x float>
   %a = fadd <4 x float> %x, %y
diff --git a/llvm/test/Transforms/InstCombine/select-ctlz-to-cttz.ll b/llvm/test/Transforms/InstCombine/select-ctlz-to-cttz.ll
new file mode 100644
index 000000000..e785ad35e
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/select-ctlz-to-cttz.ll
@@ -0,0 +1,219 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+
+declare i32 @llvm.cttz.i32(i32, i1 immarg)
+declare i32 @llvm.ctlz.i32(i32, i1 immarg)
+declare i64 @llvm.cttz.i64(i64, i1 immarg)
+declare i64 @llvm.ctlz.i64(i64, i1 immarg)
+
+declare <2 x i32> @llvm.ctlz.v2i32(<2 x i32>, i1)
+declare <2 x i32> @llvm.cttz.v2i32(<2 x i32>, i1)
+
+
+declare void @use(i32)
+declare void @use2(i1)
+
+define i32 @select_clz_to_ctz(i32 %a) {
+; CHECK-LABEL: @select_clz_to_ctz(
+; CHECK-NEXT:    [[COND:%.*]] = call i32 @llvm.cttz.i32(i32 [[A:%.*]], i1 true), !range !0
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+  %sub = sub i32 0, %a
+  %and = and i32 %sub, %a
+  %lz = tail call i32 @llvm.ctlz.i32(i32 %and, i1 true)
+  %tobool = icmp eq i32 %a, 0
+  %sub1 = xor i32 %lz, 31
+  %cond = select i1 %tobool, i32 %lz, i32 %sub1
+  ret i32 %cond
+}
+
+define i32 @select_clz_to_ctz_preserve_flag(i32 %a) {
+; CHECK-LABEL: @select_clz_to_ctz_preserve_flag(
+; CHECK-NEXT:    [[COND:%.*]] = call i32 @llvm.cttz.i32(i32 [[A:%.*]], i1 false), !range !0
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+  %sub = sub i32 0, %a
+  %and = and i32 %sub, %a
+  %lz = tail call i32 @llvm.ctlz.i32(i32 %and, i1 false)
+  %tobool = icmp eq i32 %a, 0
+  %sub1 = xor i32 %lz, 31
+  %cond = select i1 %tobool, i32 %lz, i32 %sub1
+  ret i32 %cond
+}
+
+define <2 x i32> @select_clz_to_ctz_vec(<2 x i32> %a) {
+; CHECK-LABEL: @select_clz_to_ctz_vec(
+; CHECK-NEXT:    [[COND:%.*]] = call <2 x i32> @llvm.cttz.v2i32(<2 x i32> [[A:%.*]], i1 true)
+; CHECK-NEXT:    ret <2 x i32> [[COND]]
+;
+  %sub = sub <2 x i32> zeroinitializer, %a
+  %and = and <2 x i32> %sub, %a
+  %lz = tail call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> %and, i1 true)
+  %tobool = icmp eq <2 x i32> %a, zeroinitializer
+  %sub1 = xor <2 x i32> %lz, <i32 31, i32 31>
+  %cond = select <2 x i1> %tobool, <2 x i32> %lz, <2 x i32> %sub1
+  ret <2 x i32> %cond
+}
+
+define i32 @select_clz_to_ctz_extra_use(i32 %a) {
+; CHECK-LABEL: @select_clz_to_ctz_extra_use(
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 0, [[A:%.*]]
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[SUB]], [[A]]
+; CHECK-NEXT:    [[LZ:%.*]] = tail call i32 @llvm.ctlz.i32(i32 [[AND]], i1 true), !range !0
+; CHECK-NEXT:    [[SUB1:%.*]] = xor i32 [[LZ]], 31
+; CHECK-NEXT:    call void @use(i32 [[SUB1]])
+; CHECK-NEXT:    [[COND:%.*]] = call i32 @llvm.cttz.i32(i32 [[A]], i1 true), !range !0
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+  %sub = sub i32 0, %a
+  %and = and i32 %sub, %a
+  %lz = tail call i32 @llvm.ctlz.i32(i32 %and, i1 true)
+  %tobool = icmp eq i32 %a, 0
+  %sub1 = xor i32 %lz, 31
+  call void @use(i32 %sub1)
+  %cond = select i1 %tobool, i32 %lz, i32 %sub1
+  ret i32 %cond
+}
+
+define i32 @select_clz_to_ctz_and_commuted(i32 %a) {
+; CHECK-LABEL: @select_clz_to_ctz_and_commuted(
+; CHECK-NEXT:    [[COND:%.*]] = call i32 @llvm.cttz.i32(i32 [[A:%.*]], i1 true), !range !0
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+  %sub = sub i32 0, %a
+  %and = and i32 %a, %sub
+  %lz = tail call i32 @llvm.ctlz.i32(i32 %and, i1 true)
+  %tobool = icmp eq i32 %a, 0
+  %sub1 = xor i32 %lz, 31
+  %cond = select i1 %tobool, i32 %lz, i32 %sub1
+  ret i32 %cond
+}
+
+define i32 @select_clz_to_ctz_icmp_ne(i32 %a) {
+; CHECK-LABEL: @select_clz_to_ctz_icmp_ne(
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne i32 [[A:%.*]], 0
+; CHECK-NEXT:    call void @use2(i1 [[TOBOOL]])
+; CHECK-NEXT:    [[COND:%.*]] = call i32 @llvm.cttz.i32(i32 [[A]], i1 true), !range !0
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+  %sub = sub i32 0, %a
+  %and = and i32 %sub, %a
+  %lz = tail call i32 @llvm.ctlz.i32(i32 %and, i1 true)
+  %tobool = icmp ne i32 %a, 0
+  call void @use2(i1 %tobool)
+  %sub1 = xor i32 %lz, 31
+  %cond = select i1 %tobool, i32 %sub1, i32 %lz
+  ret i32 %cond
+}
+
+define i64 @select_clz_to_ctz_i64(i64 %a) {
+; CHECK-LABEL: @select_clz_to_ctz_i64(
+; CHECK-NEXT:    [[COND:%.*]] = call i64 @llvm.cttz.i64(i64 [[A:%.*]], i1 true), !range !1
+; CHECK-NEXT:    ret i64 [[COND]]
+;
+  %sub = sub i64 0, %a
+  %and = and i64 %sub, %a
+  %lz = tail call i64 @llvm.ctlz.i64(i64 %and, i1 true)
+  %tobool = icmp eq i64 %a, 0
+  %sub1 = xor i64 %lz, 63
+  %cond = select i1 %tobool, i64 %lz, i64 %sub1
+  ret i64 %cond
+}
+
+; Negative tests
+
+define i32 @select_clz_to_ctz_wrong_sub(i32 %a) {
+; CHECK-LABEL: @select_clz_to_ctz_wrong_sub(
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 1, [[A:%.*]]
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[SUB]], [[A]]
+; CHECK-NEXT:    [[LZ:%.*]] = tail call i32 @llvm.ctlz.i32(i32 [[AND]], i1 true), !range !0
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[A]], 0
+; CHECK-NEXT:    [[SUB1:%.*]] = xor i32 [[LZ]], 31
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[TOBOOL]], i32 [[LZ]], i32 [[SUB1]]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+  %sub = sub i32 1, %a
+  %and = and i32 %sub, %a
+  %lz = tail call i32 @llvm.ctlz.i32(i32 %and, i1 true)
+  %tobool = icmp eq i32 %a, 0
+  %sub1 = xor i32 %lz, 31
+  %cond = select i1 %tobool, i32 %lz, i32 %sub1
+  ret i32 %cond
+}
+
+define i64 @select_clz_to_ctz_i64_wrong_xor(i64 %a) {
+; CHECK-LABEL: @select_clz_to_ctz_i64_wrong_xor(
+; CHECK-NEXT:    [[SUB:%.*]] = sub i64 0, [[A:%.*]]
+; CHECK-NEXT:    [[AND:%.*]] = and i64 [[SUB]], [[A]]
+; CHECK-NEXT:    [[LZ:%.*]] = tail call i64 @llvm.ctlz.i64(i64 [[AND]], i1 true), !range !1
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i64 [[A]], 0
+; CHECK-NEXT:    [[SUB11:%.*]] = or i64 [[LZ]], 64
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[TOBOOL]], i64 [[LZ]], i64 [[SUB11]]
+; CHECK-NEXT:    ret i64 [[COND]]
+;
+  %sub = sub i64 0, %a
+  %and = and i64 %sub, %a
+  %lz = tail call i64 @llvm.ctlz.i64(i64 %and, i1 true)
+  %tobool = icmp eq i64 %a, 0
+  %sub1 = xor i64 %lz, 64
+  %cond = select i1 %tobool, i64 %lz, i64 %sub1
+  ret i64 %cond
+}
+
+define i64 @select_clz_to_ctz_i64_wrong_icmp_cst(i64 %a) {
+; CHECK-LABEL: @select_clz_to_ctz_i64_wrong_icmp_cst(
+; CHECK-NEXT:    [[SUB:%.*]] = sub i64 0, [[A:%.*]]
+; CHECK-NEXT:    [[AND:%.*]] = and i64 [[SUB]], [[A]]
+; CHECK-NEXT:    [[LZ:%.*]] = tail call i64 @llvm.ctlz.i64(i64 [[AND]], i1 true), !range !1
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i64 [[A]], 1
+; CHECK-NEXT:    [[SUB1:%.*]] = xor i64 [[LZ]], 63
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[TOBOOL]], i64 [[LZ]], i64 [[SUB1]]
+; CHECK-NEXT:    ret i64 [[COND]]
+;
+  %sub = sub i64 0, %a
+  %and = and i64 %sub, %a
+  %lz = tail call i64 @llvm.ctlz.i64(i64 %and, i1 true)
+  %tobool = icmp eq i64 %a, 1
+  %sub1 = xor i64 %lz, 63
+  %cond = select i1 %tobool, i64 %lz, i64 %sub1
+  ret i64 %cond
+}
+
+define i64 @select_clz_to_ctz_i64_wrong_icmp_pred(i64 %a) {
+; CHECK-LABEL: @select_clz_to_ctz_i64_wrong_icmp_pred(
+; CHECK-NEXT:    [[SUB:%.*]] = sub i64 0, [[A:%.*]]
+; CHECK-NEXT:    [[AND:%.*]] = and i64 [[SUB]], [[A]]
+; CHECK-NEXT:    [[LZ:%.*]] = tail call i64 @llvm.ctlz.i64(i64 [[AND]], i1 true), !range !1
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp slt i64 [[A]], 0
+; CHECK-NEXT:    [[SUB1:%.*]] = xor i64 [[LZ]], 63
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[TOBOOL]], i64 [[LZ]], i64 [[SUB1]]
+; CHECK-NEXT:    ret i64 [[COND]]
+;
+  %sub = sub i64 0, %a
+  %and = and i64 %sub, %a
+  %lz = tail call i64 @llvm.ctlz.i64(i64 %and, i1 true)
+  %tobool = icmp slt i64 %a, 0
+  %sub1 = xor i64 %lz, 63
+  %cond = select i1 %tobool, i64 %lz, i64 %sub1
+  ret i64 %cond
+}
+
+define <2 x i32> @select_clz_to_ctz_vec_with_undef(<2 x i32> %a) {
+; CHECK-LABEL: @select_clz_to_ctz_vec_with_undef(
+; CHECK-NEXT:    [[SUB:%.*]] = sub <2 x i32> zeroinitializer, [[A:%.*]]
+; CHECK-NEXT:    [[AND:%.*]] = and <2 x i32> [[SUB]], [[A]]
+; CHECK-NEXT:    [[LZ:%.*]] = tail call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> [[AND]], i1 true)
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq <2 x i32> [[A]], zeroinitializer
+; CHECK-NEXT:    [[SUB1:%.*]] = xor <2 x i32> [[LZ]], <i32 31, i32 undef>
+; CHECK-NEXT:    [[COND:%.*]] = select <2 x i1> [[TOBOOL]], <2 x i32> [[LZ]], <2 x i32> [[SUB1]]
+; CHECK-NEXT:    ret <2 x i32> [[COND]]
+;
+  %sub = sub <2 x i32> zeroinitializer, %a
+  %and = and <2 x i32> %sub, %a
+  %lz = tail call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> %and, i1 true)
+  %tobool = icmp eq <2 x i32> %a, zeroinitializer
+  %sub1 = xor <2 x i32> %lz, <i32 31, i32 undef>
+  %cond = select <2 x i1> %tobool, <2 x i32> %lz, <2 x i32> %sub1
+  ret <2 x i32> %cond
+}
diff --git a/llvm/test/Transforms/InstCombine/select-extractelement.ll b/llvm/test/Transforms/InstCombine/select-extractelement.ll
index 79d0b47f9..fb3f4fa83 100644
--- a/llvm/test/Transforms/InstCombine/select-extractelement.ll
+++ b/llvm/test/Transforms/InstCombine/select-extractelement.ll
@@ -1,11 +1,12 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -S -instcombine < %s | FileCheck %s
 
 declare void @v4float_user(<4 x float>) #0
 
 define float @extract_one_select(<4 x float> %a, <4 x float> %b, i32 %c) #0 {
 ; CHECK-LABEL: @extract_one_select(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 %c, 0
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], <4 x float> %b, <4 x float> %a
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[C:%.*]], 0
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], <4 x float> [[B:%.*]], <4 x float> [[A:%.*]]
 ; CHECK-NEXT:    [[EXTRACT:%.*]] = extractelement <4 x float> [[SEL]], i32 2
 ; CHECK-NEXT:    ret float [[EXTRACT]]
 ;
@@ -18,8 +19,8 @@ define float @extract_one_select(<4 x float> %a, <4 x float> %b, i32 %c) #0 {
 ; Multiple extractelements
 define <2 x float> @extract_two_select(<4 x float> %a, <4 x float> %b, i32 %c) #0 {
 ; CHECK-LABEL: @extract_two_select(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 %c, 0
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], <4 x float> %b, <4 x float> %a
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[C:%.*]], 0
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], <4 x float> [[B:%.*]], <4 x float> [[A:%.*]]
 ; CHECK-NEXT:    [[BUILD2:%.*]] = shufflevector <4 x float> [[SEL]], <4 x float> undef, <2 x i32> <i32 1, i32 2>
 ; CHECK-NEXT:    ret <2 x float> [[BUILD2]]
 ;
@@ -35,8 +36,8 @@ define <2 x float> @extract_two_select(<4 x float> %a, <4 x float> %b, i32 %c) #
 ; Select has an extra non-extractelement user, don't change it
 define float @extract_one_select_user(<4 x float> %a, <4 x float> %b, i32 %c) #0 {
 ; CHECK-LABEL: @extract_one_select_user(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 %c, 0
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], <4 x float> %b, <4 x float> %a
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[C:%.*]], 0
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], <4 x float> [[B:%.*]], <4 x float> [[A:%.*]]
 ; CHECK-NEXT:    [[EXTRACT:%.*]] = extractelement <4 x float> [[SEL]], i32 2
 ; CHECK-NEXT:    call void @v4float_user(<4 x float> [[SEL]])
 ; CHECK-NEXT:    ret float [[EXTRACT]]
@@ -50,8 +51,8 @@ define float @extract_one_select_user(<4 x float> %a, <4 x float> %b, i32 %c) #0
 
 define float @extract_one_vselect_user(<4 x float> %a, <4 x float> %b, <4 x i32> %c) #0 {
 ; CHECK-LABEL: @extract_one_vselect_user(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <4 x i32> %c, zeroinitializer
-; CHECK-NEXT:    [[SEL:%.*]] = select <4 x i1> [[CMP]], <4 x float> %b, <4 x float> %a
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <4 x i32> [[C:%.*]], zeroinitializer
+; CHECK-NEXT:    [[SEL:%.*]] = select <4 x i1> [[CMP]], <4 x float> [[B:%.*]], <4 x float> [[A:%.*]]
 ; CHECK-NEXT:    [[EXTRACT:%.*]] = extractelement <4 x float> [[SEL]], i32 2
 ; CHECK-NEXT:    call void @v4float_user(<4 x float> [[SEL]])
 ; CHECK-NEXT:    ret float [[EXTRACT]]
@@ -63,13 +64,13 @@ define float @extract_one_vselect_user(<4 x float> %a, <4 x float> %b, <4 x i32>
   ret float %extract
 }
 
-; Do not convert the vector select into a scalar select. That would increase 
+; Do not convert the vector select into a scalar select. That would increase
 ; the instruction count and potentially obfuscate a vector min/max idiom.
 
 define float @extract_one_vselect(<4 x float> %a, <4 x float> %b, <4 x i32> %c) #0 {
 ; CHECK-LABEL: @extract_one_vselect(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <4 x i32> %c, zeroinitializer
-; CHECK-NEXT:    [[SELECT:%.*]] = select <4 x i1> [[CMP]], <4 x float> %b, <4 x float> %a
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <4 x i32> [[C:%.*]], zeroinitializer
+; CHECK-NEXT:    [[SELECT:%.*]] = select <4 x i1> [[CMP]], <4 x float> [[B:%.*]], <4 x float> [[A:%.*]]
 ; CHECK-NEXT:    [[EXTRACT:%.*]] = extractelement <4 x float> [[SELECT]], i32 0
 ; CHECK-NEXT:    ret float [[EXTRACT]]
 ;
@@ -82,8 +83,8 @@ define float @extract_one_vselect(<4 x float> %a, <4 x float> %b, <4 x i32> %c)
 ; Multiple extractelements from a vector select
 define <2 x float> @extract_two_vselect(<4 x float> %a, <4 x float> %b, <4 x i32> %c) #0 {
 ; CHECK-LABEL: @extract_two_vselect(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <4 x i32> %c, zeroinitializer
-; CHECK-NEXT:    [[SEL:%.*]] = select <4 x i1> [[CMP]], <4 x float> %b, <4 x float> %a
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <4 x i32> [[C:%.*]], zeroinitializer
+; CHECK-NEXT:    [[SEL:%.*]] = select <4 x i1> [[CMP]], <4 x float> [[B:%.*]], <4 x float> [[A:%.*]]
 ; CHECK-NEXT:    [[BUILD2:%.*]] = shufflevector <4 x float> [[SEL]], <4 x float> undef, <2 x i32> <i32 1, i32 2>
 ; CHECK-NEXT:    ret <2 x float> [[BUILD2]]
 ;
@@ -102,20 +103,20 @@ define <2 x float> @extract_two_vselect(<4 x float> %a, <4 x float> %b, <4 x i32
 define <4 x float> @simple_vector_select(<4 x float> %a, <4 x float> %b, <4 x i32> %c) #0 {
 ; CHECK-LABEL: @simple_vector_select(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <4 x i32> %c, i32 0
+; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <4 x i32> [[C:%.*]], i32 0
 ; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[TMP0]], 0
-; CHECK-NEXT:    [[A_SINK:%.*]] = select i1 [[TOBOOL]], <4 x float> %b, <4 x float> %a
-; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x i32> %c, i32 1
+; CHECK-NEXT:    [[A_SINK:%.*]] = select i1 [[TOBOOL]], <4 x float> [[B:%.*]], <4 x float> [[A:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x i32> [[C]], i32 1
 ; CHECK-NEXT:    [[TOBOOL1:%.*]] = icmp eq i32 [[TMP1]], 0
-; CHECK-NEXT:    [[A_SINK1:%.*]] = select i1 [[TOBOOL1]], <4 x float> %b, <4 x float> %a
+; CHECK-NEXT:    [[A_SINK1:%.*]] = select i1 [[TOBOOL1]], <4 x float> [[B]], <4 x float> [[A]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x float> [[A_SINK]], <4 x float> [[A_SINK1]], <4 x i32> <i32 0, i32 5, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x i32> %c, i32 2
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x i32> [[C]], i32 2
 ; CHECK-NEXT:    [[TOBOOL6:%.*]] = icmp eq i32 [[TMP3]], 0
-; CHECK-NEXT:    [[A_SINK2:%.*]] = select i1 [[TOBOOL6]], <4 x float> %b, <4 x float> %a
+; CHECK-NEXT:    [[A_SINK2:%.*]] = select i1 [[TOBOOL6]], <4 x float> [[B]], <4 x float> [[A]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> [[A_SINK2]], <4 x i32> <i32 0, i32 1, i32 6, i32 undef>
-; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x i32> %c, i32 3
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x i32> [[C]], i32 3
 ; CHECK-NEXT:    [[TOBOOL11:%.*]] = icmp eq i32 [[TMP5]], 0
-; CHECK-NEXT:    [[A_SINK3:%.*]] = select i1 [[TOBOOL11]], <4 x float> %b, <4 x float> %a
+; CHECK-NEXT:    [[A_SINK3:%.*]] = select i1 [[TOBOOL11]], <4 x float> [[B]], <4 x float> [[A]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <4 x float> [[TMP4]], <4 x float> [[A_SINK3]], <4 x i32> <i32 0, i32 1, i32 2, i32 7>
 ; CHECK-NEXT:    ret <4 x float> [[TMP6]]
 ;
@@ -143,4 +144,70 @@ entry:
   ret <4 x float> %11
 }
 
-attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf"="true" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+define <4 x i32> @extract_cond(<4 x i32> %x, <4 x i32> %y, <4 x i1> %condv) {
+; CHECK-LABEL: @extract_cond(
+; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <4 x i1> [[CONDV:%.*]], <4 x i1> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; CHECK-NEXT:    [[R:%.*]] = select <4 x i1> [[DOTSPLAT]], <4 x i32> [[X:%.*]], <4 x i32> [[Y:%.*]]
+; CHECK-NEXT:    ret <4 x i32> [[R]]
+;
+  %cond = extractelement <4 x i1> %condv, i32 3
+  %r = select i1 %cond, <4 x i32> %x, <4 x i32> %y
+  ret <4 x i32> %r
+}
+
+define <4 x i32> @splat_cond(<4 x i32> %x, <4 x i32> %y, <4 x i1> %condv) {
+; CHECK-LABEL: @splat_cond(
+; CHECK-NEXT:    [[SPLATCOND:%.*]] = shufflevector <4 x i1> [[CONDV:%.*]], <4 x i1> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; CHECK-NEXT:    [[R:%.*]] = select <4 x i1> [[SPLATCOND]], <4 x i32> [[X:%.*]], <4 x i32> [[Y:%.*]]
+; CHECK-NEXT:    ret <4 x i32> [[R]]
+;
+  %splatcond = shufflevector <4 x i1> %condv, <4 x i1> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+  %r = select <4 x i1> %splatcond, <4 x i32> %x, <4 x i32> %y
+  ret <4 x i32> %r
+}
+
+declare void @extra_use(i1)
+
+; Negative test
+
+define <4 x i32> @extract_cond_extra_use(<4 x i32> %x, <4 x i32> %y, <4 x i1> %condv) {
+; CHECK-LABEL: @extract_cond_extra_use(
+; CHECK-NEXT:    [[COND:%.*]] = extractelement <4 x i1> [[CONDV:%.*]], i32 3
+; CHECK-NEXT:    call void @extra_use(i1 [[COND]])
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[COND]], <4 x i32> [[X:%.*]], <4 x i32> [[Y:%.*]]
+; CHECK-NEXT:    ret <4 x i32> [[R]]
+;
+  %cond = extractelement <4 x i1> %condv, i32 3
+  call void @extra_use(i1 %cond)
+  %r = select i1 %cond, <4 x i32> %x, <4 x i32> %y
+  ret <4 x i32> %r
+}
+
+; Negative test
+
+define <4 x i32> @extract_cond_variable_index(<4 x i32> %x, <4 x i32> %y, <4 x i1> %condv, i32 %index) {
+; CHECK-LABEL: @extract_cond_variable_index(
+; CHECK-NEXT:    [[COND:%.*]] = extractelement <4 x i1> [[CONDV:%.*]], i32 [[INDEX:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[COND]], <4 x i32> [[X:%.*]], <4 x i32> [[Y:%.*]]
+; CHECK-NEXT:    ret <4 x i32> [[R]]
+;
+  %cond = extractelement <4 x i1> %condv, i32 %index
+  %r = select i1 %cond, <4 x i32> %x, <4 x i32> %y
+  ret <4 x i32> %r
+}
+
+; IR shuffle can alter the number of elements in the vector, so this is ok.
+
+define <4 x i32> @extract_cond_type_mismatch(<4 x i32> %x, <4 x i32> %y, <5 x i1> %condv) {
+; CHECK-LABEL: @extract_cond_type_mismatch(
+; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <5 x i1> [[CONDV:%.*]], <5 x i1> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[R:%.*]] = select <4 x i1> [[DOTSPLAT]], <4 x i32> [[X:%.*]], <4 x i32> [[Y:%.*]]
+; CHECK-NEXT:    ret <4 x i32> [[R]]
+;
+  %cond = extractelement <5 x i1> %condv, i32 1
+  %r = select i1 %cond, <4 x i32> %x, <4 x i32> %y
+  ret <4 x i32> %r
+}
+
+
+attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/Transforms/InstCombine/select-imm-canon.ll b/llvm/test/Transforms/InstCombine/select-imm-canon.ll
new file mode 100644
index 000000000..b00d9e3f2
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/select-imm-canon.ll
@@ -0,0 +1,70 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define i8 @single(i32 %A) {
+; CHECK-LABEL: @single(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp sgt i32 [[A:%.*]], -128
+; CHECK-NEXT:    [[L2:%.*]] = select i1 [[TMP0]], i32 [[A]], i32 -128
+; CHECK-NEXT:    [[CONV7:%.*]] = trunc i32 [[L2]] to i8
+; CHECK-NEXT:    ret i8 [[CONV7]]
+;
+entry:
+  %l1 = icmp slt i32 %A, -128
+  %l2 = select i1 %l1, i32 128, i32 %A
+  %conv7 = trunc i32 %l2 to i8
+  ret i8 %conv7
+}
+
+define i8 @double(i32 %A) {
+; CHECK-LABEL: @double(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp sgt i32 [[A:%.*]], -128
+; CHECK-NEXT:    [[L2:%.*]] = select i1 [[TMP0]], i32 [[A]], i32 -128
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[L2]], 127
+; CHECK-NEXT:    [[SPEC_SELECT_I:%.*]] = select i1 [[TMP1]], i32 [[L2]], i32 127
+; CHECK-NEXT:    [[CONV7:%.*]] = trunc i32 [[SPEC_SELECT_I]] to i8
+; CHECK-NEXT:    ret i8 [[CONV7]]
+;
+entry:
+  %l1 = icmp slt i32 %A, -128
+  %l2 = select i1 %l1, i32 128, i32 %A
+  %.inv = icmp sgt i32 %A, 127
+  %spec.select.i = select i1 %.inv, i32 127, i32 %l2
+  %conv7 = trunc i32 %spec.select.i to i8
+  ret i8 %conv7
+}
+
+define i8 @thisdoesnotloop(i32 %A, i32 %B) {
+; CHECK-LABEL: @thisdoesnotloop(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[L1:%.*]] = icmp slt i32 [[A:%.*]], -128
+; CHECK-NEXT:    [[L2:%.*]] = select i1 [[L1]], i32 -128, i32 [[B:%.*]]
+; CHECK-NEXT:    [[CONV7:%.*]] = trunc i32 [[L2]] to i8
+; CHECK-NEXT:    ret i8 [[CONV7]]
+;
+entry:
+  %l1 = icmp slt i32 %A, -128
+  %l2 = select i1 %l1, i32 128, i32 %B
+  %conv7 = trunc i32 %l2 to i8
+  ret i8 %conv7
+}
+
+define i8 @original(i32 %A, i32 %B) {
+; CHECK-LABEL: @original(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[A:%.*]], -128
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i32 [[A]], i32 -128
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp slt i32 [[TMP2]], 127
+; CHECK-NEXT:    [[SPEC_SELECT_I:%.*]] = select i1 [[TMP3]], i32 [[TMP2]], i32 127
+; CHECK-NEXT:    [[CONV7:%.*]] = trunc i32 [[SPEC_SELECT_I]] to i8
+; CHECK-NEXT:    ret i8 [[CONV7]]
+;
+  %cmp4.i = icmp slt i32 127, %A
+  %cmp6.i = icmp sle i32 -128, %A
+  %retval.0.i = select i1 %cmp4.i, i32 127, i32 -128
+  %not.cmp4.i = xor i1 %cmp4.i, true
+  %cleanup.dest.slot.0.i = and i1 %cmp6.i, %not.cmp4.i
+  %spec.select.i = select i1 %cleanup.dest.slot.0.i, i32 %A, i32 %retval.0.i
+  %conv7 = trunc i32 %spec.select.i to i8
+  ret i8 %conv7
+}
diff --git a/llvm/test/Transforms/InstCombine/select-obo-peo-ops.ll b/llvm/test/Transforms/InstCombine/select-obo-peo-ops.ll
index c57904ab9..54c1d0f54 100644
--- a/llvm/test/Transforms/InstCombine/select-obo-peo-ops.ll
+++ b/llvm/test/Transforms/InstCombine/select-obo-peo-ops.ll
@@ -3,13 +3,11 @@
 
 define i64 @test_shl_nuw_nsw__all_are_safe(i32 %x, i64 %y) {
 ; CHECK-LABEL: @test_shl_nuw_nsw__all_are_safe(
-; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 15
-; CHECK-NEXT:    [[TMP2:%.*]] = shl nuw nsw i32 [[TMP1]], 2
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[TMP1]], 0
-; CHECK-NEXT:    [[NARROW:%.*]] = select i1 [[TMP3]], i32 0, i32 [[TMP2]]
-; CHECK-NEXT:    [[TMP4:%.*]] = zext i32 [[NARROW]] to i64
-; CHECK-NEXT:    [[TMP5:%.*]] = ashr i64 [[Y:%.*]], [[TMP4]]
-; CHECK-NEXT:    ret i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 [[X:%.*]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], 60
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i32 [[TMP2]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = ashr i64 [[Y:%.*]], [[TMP3]]
+; CHECK-NEXT:    ret i64 [[TMP4]]
 ;
   %1 = and i32 %x, 15
   %2 = shl nuw nsw i32 %1, 2
@@ -22,13 +20,11 @@ define i64 @test_shl_nuw_nsw__all_are_safe(i32 %x, i64 %y) {
 
 define i64 @test_shl_nuw__all_are_safe(i32 %x, i64 %y) {
 ; CHECK-LABEL: @test_shl_nuw__all_are_safe(
-; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 15
-; CHECK-NEXT:    [[TMP2:%.*]] = shl nuw nsw i32 [[TMP1]], 2
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[TMP1]], 0
-; CHECK-NEXT:    [[NARROW:%.*]] = select i1 [[TMP3]], i32 0, i32 [[TMP2]]
-; CHECK-NEXT:    [[TMP4:%.*]] = zext i32 [[NARROW]] to i64
-; CHECK-NEXT:    [[TMP5:%.*]] = ashr i64 [[Y:%.*]], [[TMP4]]
-; CHECK-NEXT:    ret i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 [[X:%.*]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], 60
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i32 [[TMP2]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = ashr i64 [[Y:%.*]], [[TMP3]]
+; CHECK-NEXT:    ret i64 [[TMP4]]
 ;
   %1 = and i32 %x, 15
   %2 = shl nuw i32 %1, 2
@@ -41,13 +37,11 @@ define i64 @test_shl_nuw__all_are_safe(i32 %x, i64 %y) {
 
 define i64 @test_shl_nsw__all_are_safe(i32 %x, i64 %y) {
 ; CHECK-LABEL: @test_shl_nsw__all_are_safe(
-; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 15
-; CHECK-NEXT:    [[TMP2:%.*]] = shl nuw nsw i32 [[TMP1]], 2
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[TMP1]], 0
-; CHECK-NEXT:    [[NARROW:%.*]] = select i1 [[TMP3]], i32 0, i32 [[TMP2]]
-; CHECK-NEXT:    [[TMP4:%.*]] = zext i32 [[NARROW]] to i64
-; CHECK-NEXT:    [[TMP5:%.*]] = ashr i64 [[Y:%.*]], [[TMP4]]
-; CHECK-NEXT:    ret i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 [[X:%.*]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], 60
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i32 [[TMP2]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = ashr i64 [[Y:%.*]], [[TMP3]]
+; CHECK-NEXT:    ret i64 [[TMP4]]
 ;
   %1 = and i32 %x, 15
   %2 = shl nsw i32 %1, 2
@@ -60,13 +54,11 @@ define i64 @test_shl_nsw__all_are_safe(i32 %x, i64 %y) {
 
 define i64 @test_shl__all_are_safe(i32 %x, i64 %y) {
 ; CHECK-LABEL: @test_shl__all_are_safe(
-; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 15
-; CHECK-NEXT:    [[TMP2:%.*]] = shl nuw nsw i32 [[TMP1]], 2
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[TMP1]], 0
-; CHECK-NEXT:    [[NARROW:%.*]] = select i1 [[TMP3]], i32 0, i32 [[TMP2]]
-; CHECK-NEXT:    [[TMP4:%.*]] = zext i32 [[NARROW]] to i64
-; CHECK-NEXT:    [[TMP5:%.*]] = ashr i64 [[Y:%.*]], [[TMP4]]
-; CHECK-NEXT:    ret i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 [[X:%.*]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], 60
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i32 [[TMP2]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = ashr i64 [[Y:%.*]], [[TMP3]]
+; CHECK-NEXT:    ret i64 [[TMP4]]
 ;
   %1 = and i32 %x, 15
   %2 = shl i32 %1, 2
@@ -79,13 +71,11 @@ define i64 @test_shl__all_are_safe(i32 %x, i64 %y) {
 
 define i64 @test_shl_nuw_nsw__nuw_is_safe(i32 %x, i64 %y) {
 ; CHECK-LABEL: @test_shl_nuw_nsw__nuw_is_safe(
-; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 1073741822
-; CHECK-NEXT:    [[TMP2:%.*]] = shl nuw nsw i32 [[TMP1]], 2
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[TMP1]], 0
-; CHECK-NEXT:    [[NARROW:%.*]] = select i1 [[TMP3]], i32 0, i32 [[TMP2]]
-; CHECK-NEXT:    [[TMP4:%.*]] = zext i32 [[NARROW]] to i64
-; CHECK-NEXT:    [[TMP5:%.*]] = ashr i64 [[Y:%.*]], [[TMP4]]
-; CHECK-NEXT:    ret i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 [[X:%.*]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], -8
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i32 [[TMP2]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = ashr i64 [[Y:%.*]], [[TMP3]]
+; CHECK-NEXT:    ret i64 [[TMP4]]
 ;
   %1 = and i32 %x, 1073741822
   %2 = shl nuw nsw i32 %1, 2
@@ -98,13 +88,11 @@ define i64 @test_shl_nuw_nsw__nuw_is_safe(i32 %x, i64 %y) {
 
 define i64 @test_shl_nuw__nuw_is_safe(i32 %x, i64 %y) {
 ; CHECK-LABEL: @test_shl_nuw__nuw_is_safe(
-; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 1073741822
-; CHECK-NEXT:    [[TMP2:%.*]] = shl nuw i32 [[TMP1]], 2
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[TMP1]], 0
-; CHECK-NEXT:    [[NARROW:%.*]] = select i1 [[TMP3]], i32 0, i32 [[TMP2]]
-; CHECK-NEXT:    [[TMP4:%.*]] = zext i32 [[NARROW]] to i64
-; CHECK-NEXT:    [[TMP5:%.*]] = ashr i64 [[Y:%.*]], [[TMP4]]
-; CHECK-NEXT:    ret i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 [[X:%.*]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], -8
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i32 [[TMP2]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = ashr i64 [[Y:%.*]], [[TMP3]]
+; CHECK-NEXT:    ret i64 [[TMP4]]
 ;
   %1 = and i32 %x, 1073741822
   %2 = shl nuw i32 %1, 2
@@ -117,13 +105,11 @@ define i64 @test_shl_nuw__nuw_is_safe(i32 %x, i64 %y) {
 
 define i64 @test_shl_nsw__nuw_is_safe(i32 %x, i64 %y) {
 ; CHECK-LABEL: @test_shl_nsw__nuw_is_safe(
-; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 1073741822
-; CHECK-NEXT:    [[TMP2:%.*]] = shl nuw nsw i32 [[TMP1]], 2
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[TMP1]], 0
-; CHECK-NEXT:    [[NARROW:%.*]] = select i1 [[TMP3]], i32 0, i32 [[TMP2]]
-; CHECK-NEXT:    [[TMP4:%.*]] = zext i32 [[NARROW]] to i64
-; CHECK-NEXT:    [[TMP5:%.*]] = ashr i64 [[Y:%.*]], [[TMP4]]
-; CHECK-NEXT:    ret i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 [[X:%.*]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], -8
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i32 [[TMP2]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = ashr i64 [[Y:%.*]], [[TMP3]]
+; CHECK-NEXT:    ret i64 [[TMP4]]
 ;
   %1 = and i32 %x, 1073741822
   %2 = shl nsw i32 %1, 2
@@ -136,13 +122,11 @@ define i64 @test_shl_nsw__nuw_is_safe(i32 %x, i64 %y) {
 
 define i64 @test_shl__nuw_is_safe(i32 %x, i64 %y) {
 ; CHECK-LABEL: @test_shl__nuw_is_safe(
-; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 1073741822
-; CHECK-NEXT:    [[TMP2:%.*]] = shl nuw i32 [[TMP1]], 2
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[TMP1]], 0
-; CHECK-NEXT:    [[NARROW:%.*]] = select i1 [[TMP3]], i32 0, i32 [[TMP2]]
-; CHECK-NEXT:    [[TMP4:%.*]] = zext i32 [[NARROW]] to i64
-; CHECK-NEXT:    [[TMP5:%.*]] = ashr i64 [[Y:%.*]], [[TMP4]]
-; CHECK-NEXT:    ret i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 [[X:%.*]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], -8
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i32 [[TMP2]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = ashr i64 [[Y:%.*]], [[TMP3]]
+; CHECK-NEXT:    ret i64 [[TMP4]]
 ;
   %1 = and i32 %x, 1073741822
   %2 = shl i32 %1, 2
@@ -156,12 +140,10 @@ define i64 @test_shl__nuw_is_safe(i32 %x, i64 %y) {
 define i32 @test_shl_nuw_nsw__nsw_is_safe(i32 %x) {
 ; CHECK-LABEL: @test_shl_nuw_nsw__nsw_is_safe(
 ; CHECK-NEXT:    [[TMP1:%.*]] = or i32 [[X:%.*]], -83886080
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], -83886079
-; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw nsw i32 [[TMP1]], 2
-; CHECK-NEXT:    [[TMP4:%.*]] = select i1 [[TMP2]], i32 -335544316, i32 [[TMP3]]
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i32 [[TMP4]], [[TMP1]]
-; CHECK-NEXT:    [[TMP6:%.*]] = mul i32 [[TMP5]], [[TMP3]]
-; CHECK-NEXT:    ret i32 [[TMP6]]
+; CHECK-NEXT:    [[TMP2:%.*]] = shl nsw i32 [[TMP1]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = mul i32 [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    [[TMP4:%.*]] = mul i32 [[TMP3]], [[TMP2]]
+; CHECK-NEXT:    ret i32 [[TMP4]]
 ;
   %1 = or i32 %x, -83886080
   %2 = icmp eq i32 %1, -83886079
@@ -175,12 +157,10 @@ define i32 @test_shl_nuw_nsw__nsw_is_safe(i32 %x) {
 define i32 @test_shl_nuw__nsw_is_safe(i32 %x) {
 ; CHECK-LABEL: @test_shl_nuw__nsw_is_safe(
 ; CHECK-NEXT:    [[TMP1:%.*]] = or i32 [[X:%.*]], -83886080
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], -83886079
-; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw nsw i32 [[TMP1]], 2
-; CHECK-NEXT:    [[TMP4:%.*]] = select i1 [[TMP2]], i32 -335544316, i32 [[TMP3]]
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i32 [[TMP4]], [[TMP1]]
-; CHECK-NEXT:    [[TMP6:%.*]] = mul i32 [[TMP5]], [[TMP3]]
-; CHECK-NEXT:    ret i32 [[TMP6]]
+; CHECK-NEXT:    [[TMP2:%.*]] = shl nsw i32 [[TMP1]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = mul i32 [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    [[TMP4:%.*]] = mul i32 [[TMP3]], [[TMP2]]
+; CHECK-NEXT:    ret i32 [[TMP4]]
 ;
   %1 = or i32 %x, -83886080
   %2 = icmp eq i32 %1, -83886079
@@ -194,12 +174,10 @@ define i32 @test_shl_nuw__nsw_is_safe(i32 %x) {
 define i32 @test_shl_nsw__nsw_is_safe(i32 %x) {
 ; CHECK-LABEL: @test_shl_nsw__nsw_is_safe(
 ; CHECK-NEXT:    [[TMP1:%.*]] = or i32 [[X:%.*]], -83886080
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], -83886079
-; CHECK-NEXT:    [[TMP3:%.*]] = shl nsw i32 [[TMP1]], 2
-; CHECK-NEXT:    [[TMP4:%.*]] = select i1 [[TMP2]], i32 -335544316, i32 [[TMP3]]
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i32 [[TMP4]], [[TMP1]]
-; CHECK-NEXT:    [[TMP6:%.*]] = mul i32 [[TMP5]], [[TMP3]]
-; CHECK-NEXT:    ret i32 [[TMP6]]
+; CHECK-NEXT:    [[TMP2:%.*]] = shl nsw i32 [[TMP1]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = mul i32 [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    [[TMP4:%.*]] = mul i32 [[TMP3]], [[TMP2]]
+; CHECK-NEXT:    ret i32 [[TMP4]]
 ;
   %1 = or i32 %x, -83886080
   %2 = icmp eq i32 %1, -83886079
@@ -213,12 +191,10 @@ define i32 @test_shl_nsw__nsw_is_safe(i32 %x) {
 define i32 @test_shl__nsw_is_safe(i32 %x) {
 ; CHECK-LABEL: @test_shl__nsw_is_safe(
 ; CHECK-NEXT:    [[TMP1:%.*]] = or i32 [[X:%.*]], -83886080
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], -83886079
-; CHECK-NEXT:    [[TMP3:%.*]] = shl nsw i32 [[TMP1]], 2
-; CHECK-NEXT:    [[TMP4:%.*]] = select i1 [[TMP2]], i32 -335544316, i32 [[TMP3]]
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i32 [[TMP4]], [[TMP1]]
-; CHECK-NEXT:    [[TMP6:%.*]] = mul i32 [[TMP5]], [[TMP3]]
-; CHECK-NEXT:    ret i32 [[TMP6]]
+; CHECK-NEXT:    [[TMP2:%.*]] = shl nsw i32 [[TMP1]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = mul i32 [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    [[TMP4:%.*]] = mul i32 [[TMP3]], [[TMP2]]
+; CHECK-NEXT:    ret i32 [[TMP4]]
 ;
   %1 = or i32 %x, -83886080
   %2 = icmp eq i32 %1, -83886079
@@ -232,13 +208,11 @@ define i32 @test_shl__nsw_is_safe(i32 %x) {
 
 define i64 @test_shl_nuw_nsw__none_are_safe(i32 %x, i64 %y) {
 ; CHECK-LABEL: @test_shl_nuw_nsw__none_are_safe(
-; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], -2
-; CHECK-NEXT:    [[TMP2:%.*]] = shl nuw nsw i32 [[TMP1]], 2
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[TMP1]], 0
-; CHECK-NEXT:    [[NARROW:%.*]] = select i1 [[TMP3]], i32 0, i32 [[TMP2]]
-; CHECK-NEXT:    [[TMP4:%.*]] = zext i32 [[NARROW]] to i64
-; CHECK-NEXT:    [[TMP5:%.*]] = ashr i64 [[Y:%.*]], [[TMP4]]
-; CHECK-NEXT:    ret i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 [[X:%.*]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], -8
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i32 [[TMP2]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = ashr i64 [[Y:%.*]], [[TMP3]]
+; CHECK-NEXT:    ret i64 [[TMP4]]
 ;
   %1 = and i32 %x, 4294967294
   %2 = shl nuw nsw i32 %1, 2
@@ -251,13 +225,11 @@ define i64 @test_shl_nuw_nsw__none_are_safe(i32 %x, i64 %y) {
 
 define i64 @test_shl_nuw__none_are_safe(i32 %x, i64 %y) {
 ; CHECK-LABEL: @test_shl_nuw__none_are_safe(
-; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], -2
-; CHECK-NEXT:    [[TMP2:%.*]] = shl nuw i32 [[TMP1]], 2
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[TMP1]], 0
-; CHECK-NEXT:    [[NARROW:%.*]] = select i1 [[TMP3]], i32 0, i32 [[TMP2]]
-; CHECK-NEXT:    [[TMP4:%.*]] = zext i32 [[NARROW]] to i64
-; CHECK-NEXT:    [[TMP5:%.*]] = ashr i64 [[Y:%.*]], [[TMP4]]
-; CHECK-NEXT:    ret i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 [[X:%.*]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], -8
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i32 [[TMP2]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = ashr i64 [[Y:%.*]], [[TMP3]]
+; CHECK-NEXT:    ret i64 [[TMP4]]
 ;
   %1 = and i32 %x, 4294967294
   %2 = shl nuw i32 %1, 2
@@ -270,13 +242,11 @@ define i64 @test_shl_nuw__none_are_safe(i32 %x, i64 %y) {
 
 define i64 @test_shl_nsw__none_are_safe(i32 %x, i64 %y) {
 ; CHECK-LABEL: @test_shl_nsw__none_are_safe(
-; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], -2
-; CHECK-NEXT:    [[TMP2:%.*]] = shl nsw i32 [[TMP1]], 2
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[TMP1]], 0
-; CHECK-NEXT:    [[NARROW:%.*]] = select i1 [[TMP3]], i32 0, i32 [[TMP2]]
-; CHECK-NEXT:    [[TMP4:%.*]] = zext i32 [[NARROW]] to i64
-; CHECK-NEXT:    [[TMP5:%.*]] = ashr i64 [[Y:%.*]], [[TMP4]]
-; CHECK-NEXT:    ret i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 [[X:%.*]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], -8
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i32 [[TMP2]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = ashr i64 [[Y:%.*]], [[TMP3]]
+; CHECK-NEXT:    ret i64 [[TMP4]]
 ;
   %1 = and i32 %x, 4294967294
   %2 = shl nsw i32 %1, 2
@@ -306,13 +276,11 @@ define i64 @test_shl__none_are_safe(i32 %x, i64 %y) {
 
 define i64 @test_lshr_exact__exact_is_safe(i32 %x, i64 %y) {
 ; CHECK-LABEL: @test_lshr_exact__exact_is_safe(
-; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 60
-; CHECK-NEXT:    [[TMP2:%.*]] = lshr exact i32 [[TMP1]], 2
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[TMP1]], 0
-; CHECK-NEXT:    [[NARROW:%.*]] = select i1 [[TMP3]], i32 0, i32 [[TMP2]]
-; CHECK-NEXT:    [[TMP4:%.*]] = zext i32 [[NARROW]] to i64
-; CHECK-NEXT:    [[TMP5:%.*]] = ashr i64 [[Y:%.*]], [[TMP4]]
-; CHECK-NEXT:    ret i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i32 [[X:%.*]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], 15
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i32 [[TMP2]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = ashr i64 [[Y:%.*]], [[TMP3]]
+; CHECK-NEXT:    ret i64 [[TMP4]]
 ;
   %1 = and i32 %x, 60
   %2 = lshr exact i32 %1, 2
@@ -325,13 +293,11 @@ define i64 @test_lshr_exact__exact_is_safe(i32 %x, i64 %y) {
 
 define i64 @test_lshr__exact_is_safe(i32 %x, i64 %y) {
 ; CHECK-LABEL: @test_lshr__exact_is_safe(
-; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 60
-; CHECK-NEXT:    [[TMP2:%.*]] = lshr exact i32 [[TMP1]], 2
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[TMP1]], 0
-; CHECK-NEXT:    [[NARROW:%.*]] = select i1 [[TMP3]], i32 0, i32 [[TMP2]]
-; CHECK-NEXT:    [[TMP4:%.*]] = zext i32 [[NARROW]] to i64
-; CHECK-NEXT:    [[TMP5:%.*]] = ashr i64 [[Y:%.*]], [[TMP4]]
-; CHECK-NEXT:    ret i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i32 [[X:%.*]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], 15
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i32 [[TMP2]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = ashr i64 [[Y:%.*]], [[TMP3]]
+; CHECK-NEXT:    ret i64 [[TMP4]]
 ;
   %1 = and i32 %x, 60
   %2 = lshr i32 %1, 2
@@ -344,13 +310,11 @@ define i64 @test_lshr__exact_is_safe(i32 %x, i64 %y) {
 
 define i64 @test_lshr_exact__exact_is_unsafe(i32 %x, i64 %y) {
 ; CHECK-LABEL: @test_lshr_exact__exact_is_unsafe(
-; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 63
-; CHECK-NEXT:    [[TMP2:%.*]] = lshr exact i32 [[TMP1]], 2
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[TMP1]], 0
-; CHECK-NEXT:    [[NARROW:%.*]] = select i1 [[TMP3]], i32 0, i32 [[TMP2]]
-; CHECK-NEXT:    [[TMP4:%.*]] = zext i32 [[NARROW]] to i64
-; CHECK-NEXT:    [[TMP5:%.*]] = ashr i64 [[Y:%.*]], [[TMP4]]
-; CHECK-NEXT:    ret i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i32 [[X:%.*]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], 15
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i32 [[TMP2]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = ashr i64 [[Y:%.*]], [[TMP3]]
+; CHECK-NEXT:    ret i64 [[TMP4]]
 ;
   %1 = and i32 %x, 63
   %2 = lshr exact i32 %1, 2
@@ -380,13 +344,11 @@ define i64 @test_lshr__exact_is_unsafe(i32 %x, i64 %y) {
 
 define i64 @test_ashr_exact__exact_is_safe(i32 %x, i64 %y) {
 ; CHECK-LABEL: @test_ashr_exact__exact_is_safe(
-; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], -2147483588
-; CHECK-NEXT:    [[TMP2:%.*]] = ashr exact i32 [[TMP1]], 2
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[TMP1]], 0
-; CHECK-NEXT:    [[NARROW:%.*]] = select i1 [[TMP3]], i32 0, i32 [[TMP2]]
-; CHECK-NEXT:    [[TMP4:%.*]] = zext i32 [[NARROW]] to i64
-; CHECK-NEXT:    [[TMP5:%.*]] = ashr i64 [[Y:%.*]], [[TMP4]]
-; CHECK-NEXT:    ret i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr i32 [[X:%.*]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], -536870897
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i32 [[TMP2]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = ashr i64 [[Y:%.*]], [[TMP3]]
+; CHECK-NEXT:    ret i64 [[TMP4]]
 ;
   %1 = and i32 %x, -2147483588
   %2 = ashr exact i32 %1, 2
@@ -399,13 +361,11 @@ define i64 @test_ashr_exact__exact_is_safe(i32 %x, i64 %y) {
 
 define i64 @test_ashr__exact_is_safe(i32 %x, i64 %y) {
 ; CHECK-LABEL: @test_ashr__exact_is_safe(
-; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], -2147483588
-; CHECK-NEXT:    [[TMP2:%.*]] = ashr exact i32 [[TMP1]], 2
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[TMP1]], 0
-; CHECK-NEXT:    [[NARROW:%.*]] = select i1 [[TMP3]], i32 0, i32 [[TMP2]]
-; CHECK-NEXT:    [[TMP4:%.*]] = zext i32 [[NARROW]] to i64
-; CHECK-NEXT:    [[TMP5:%.*]] = ashr i64 [[Y:%.*]], [[TMP4]]
-; CHECK-NEXT:    ret i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr i32 [[X:%.*]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], -536870897
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i32 [[TMP2]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = ashr i64 [[Y:%.*]], [[TMP3]]
+; CHECK-NEXT:    ret i64 [[TMP4]]
 ;
   %1 = and i32 %x, -2147483588
   %2 = ashr i32 %1, 2
@@ -418,13 +378,11 @@ define i64 @test_ashr__exact_is_safe(i32 %x, i64 %y) {
 
 define i64 @test_ashr_exact__exact_is_unsafe(i32 %x, i64 %y) {
 ; CHECK-LABEL: @test_ashr_exact__exact_is_unsafe(
-; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], -2147483585
-; CHECK-NEXT:    [[TMP2:%.*]] = ashr exact i32 [[TMP1]], 2
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[TMP1]], 0
-; CHECK-NEXT:    [[NARROW:%.*]] = select i1 [[TMP3]], i32 0, i32 [[TMP2]]
-; CHECK-NEXT:    [[TMP4:%.*]] = zext i32 [[NARROW]] to i64
-; CHECK-NEXT:    [[TMP5:%.*]] = ashr i64 [[Y:%.*]], [[TMP4]]
-; CHECK-NEXT:    ret i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr i32 [[X:%.*]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], -536870897
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i32 [[TMP2]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = ashr i64 [[Y:%.*]], [[TMP3]]
+; CHECK-NEXT:    ret i64 [[TMP4]]
 ;
   %1 = and i32 %x, -2147483585
   %2 = ashr exact i32 %1, 2
@@ -455,10 +413,8 @@ define i64 @test_ashr__exact_is_unsafe(i32 %x, i64 %y) {
 define i32 @test_add_nuw_nsw__all_are_safe(i32 %x) {
 ; CHECK-LABEL: @test_add_nuw_nsw__all_are_safe(
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 1073741823
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 3
 ; CHECK-NEXT:    [[ADD:%.*]] = add nuw nsw i32 [[AND]], 1
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 4, i32 [[ADD]]
-; CHECK-NEXT:    ret i32 [[SEL]]
+; CHECK-NEXT:    ret i32 [[ADD]]
 ;
   %and = and i32 %x, 1073741823
   %cmp = icmp eq i32 %and, 3
@@ -470,10 +426,8 @@ define i32 @test_add_nuw_nsw__all_are_safe(i32 %x) {
 define i32 @test_add_nuw__all_are_safe(i32 %x) {
 ; CHECK-LABEL: @test_add_nuw__all_are_safe(
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 1073741823
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 3
 ; CHECK-NEXT:    [[ADD:%.*]] = add nuw nsw i32 [[AND]], 1
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 4, i32 [[ADD]]
-; CHECK-NEXT:    ret i32 [[SEL]]
+; CHECK-NEXT:    ret i32 [[ADD]]
 ;
   %and = and i32 %x, 1073741823
   %cmp = icmp eq i32 %and, 3
@@ -485,10 +439,8 @@ define i32 @test_add_nuw__all_are_safe(i32 %x) {
 define i32 @test_add_nsw__all_are_safe(i32 %x) {
 ; CHECK-LABEL: @test_add_nsw__all_are_safe(
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 1073741823
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 3
 ; CHECK-NEXT:    [[ADD:%.*]] = add nuw nsw i32 [[AND]], 1
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 4, i32 [[ADD]]
-; CHECK-NEXT:    ret i32 [[SEL]]
+; CHECK-NEXT:    ret i32 [[ADD]]
 ;
   %and = and i32 %x, 1073741823
   %cmp = icmp eq i32 %and, 3
@@ -500,10 +452,8 @@ define i32 @test_add_nsw__all_are_safe(i32 %x) {
 define i32 @test_add__all_are_safe(i32 %x) {
 ; CHECK-LABEL: @test_add__all_are_safe(
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 1073741823
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 3
 ; CHECK-NEXT:    [[ADD:%.*]] = add nuw nsw i32 [[AND]], 1
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 4, i32 [[ADD]]
-; CHECK-NEXT:    ret i32 [[SEL]]
+; CHECK-NEXT:    ret i32 [[ADD]]
 ;
   %and = and i32 %x, 1073741823
   %cmp = icmp eq i32 %and, 3
@@ -515,10 +465,8 @@ define i32 @test_add__all_are_safe(i32 %x) {
 define i32 @test_add_nuw_nsw__nuw_is_safe(i32 %x) {
 ; CHECK-LABEL: @test_add_nuw_nsw__nuw_is_safe(
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 2147483647
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 2147483647
-; CHECK-NEXT:    [[ADD:%.*]] = add nuw nsw i32 [[AND]], 1
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 -2147483648, i32 [[ADD]]
-; CHECK-NEXT:    ret i32 [[SEL]]
+; CHECK-NEXT:    [[ADD:%.*]] = add nuw i32 [[AND]], 1
+; CHECK-NEXT:    ret i32 [[ADD]]
 ;
   %and = and i32 %x, 2147483647
   %cmp = icmp eq i32 %and, 2147483647
@@ -530,10 +478,8 @@ define i32 @test_add_nuw_nsw__nuw_is_safe(i32 %x) {
 define i32 @test_add_nuw__nuw_is_safe(i32 %x) {
 ; CHECK-LABEL: @test_add_nuw__nuw_is_safe(
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 2147483647
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 2147483647
 ; CHECK-NEXT:    [[ADD:%.*]] = add nuw i32 [[AND]], 1
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 -2147483648, i32 [[ADD]]
-; CHECK-NEXT:    ret i32 [[SEL]]
+; CHECK-NEXT:    ret i32 [[ADD]]
 ;
   %and = and i32 %x, 2147483647
   %cmp = icmp eq i32 %and, 2147483647
@@ -545,10 +491,8 @@ define i32 @test_add_nuw__nuw_is_safe(i32 %x) {
 define i32 @test_add_nsw__nuw_is_safe(i32 %x) {
 ; CHECK-LABEL: @test_add_nsw__nuw_is_safe(
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 2147483647
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 2147483647
-; CHECK-NEXT:    [[ADD:%.*]] = add nuw nsw i32 [[AND]], 1
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 -2147483648, i32 [[ADD]]
-; CHECK-NEXT:    ret i32 [[SEL]]
+; CHECK-NEXT:    [[ADD:%.*]] = add nuw i32 [[AND]], 1
+; CHECK-NEXT:    ret i32 [[ADD]]
 ;
   %and = and i32 %x, 2147483647
   %cmp = icmp eq i32 %and, 2147483647
@@ -560,10 +504,8 @@ define i32 @test_add_nsw__nuw_is_safe(i32 %x) {
 define i32 @test_add__nuw_is_safe(i32 %x) {
 ; CHECK-LABEL: @test_add__nuw_is_safe(
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 2147483647
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 2147483647
 ; CHECK-NEXT:    [[ADD:%.*]] = add nuw i32 [[AND]], 1
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 -2147483648, i32 [[ADD]]
-; CHECK-NEXT:    ret i32 [[SEL]]
+; CHECK-NEXT:    ret i32 [[ADD]]
 ;
   %and = and i32 %x, 2147483647
   %cmp = icmp eq i32 %and, 2147483647
@@ -575,10 +517,8 @@ define i32 @test_add__nuw_is_safe(i32 %x) {
 define i32 @test_add_nuw_nsw__nsw_is_safe(i32 %x) {
 ; CHECK-LABEL: @test_add_nuw_nsw__nsw_is_safe(
 ; CHECK-NEXT:    [[OR:%.*]] = or i32 [[X:%.*]], -2147483648
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[OR]], -1
-; CHECK-NEXT:    [[ADD:%.*]] = add nuw nsw i32 [[OR]], 1
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 0, i32 [[ADD]]
-; CHECK-NEXT:    ret i32 [[SEL]]
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[OR]], 1
+; CHECK-NEXT:    ret i32 [[ADD]]
 ;
   %or = or i32 %x, -2147483648
   %cmp = icmp eq i32 %or, -1
@@ -590,10 +530,8 @@ define i32 @test_add_nuw_nsw__nsw_is_safe(i32 %x) {
 define i32 @test_add_nuw__nsw_is_safe(i32 %x) {
 ; CHECK-LABEL: @test_add_nuw__nsw_is_safe(
 ; CHECK-NEXT:    [[OR:%.*]] = or i32 [[X:%.*]], -2147483648
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[OR]], -1
-; CHECK-NEXT:    [[ADD:%.*]] = add nuw nsw i32 [[OR]], 1
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 0, i32 [[ADD]]
-; CHECK-NEXT:    ret i32 [[SEL]]
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[OR]], 1
+; CHECK-NEXT:    ret i32 [[ADD]]
 ;
   %or = or i32 %x, -2147483648
   %cmp = icmp eq i32 %or, -1
@@ -605,10 +543,8 @@ define i32 @test_add_nuw__nsw_is_safe(i32 %x) {
 define i32 @test_add_nsw__nsw_is_safe(i32 %x) {
 ; CHECK-LABEL: @test_add_nsw__nsw_is_safe(
 ; CHECK-NEXT:    [[OR:%.*]] = or i32 [[X:%.*]], -2147483648
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[OR]], -1
 ; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[OR]], 1
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 0, i32 [[ADD]]
-; CHECK-NEXT:    ret i32 [[SEL]]
+; CHECK-NEXT:    ret i32 [[ADD]]
 ;
   %or = or i32 %x, -2147483648
   %cmp = icmp eq i32 %or, -1
@@ -620,10 +556,8 @@ define i32 @test_add_nsw__nsw_is_safe(i32 %x) {
 define i32 @test_add__nsw_is_safe(i32 %x) {
 ; CHECK-LABEL: @test_add__nsw_is_safe(
 ; CHECK-NEXT:    [[OR:%.*]] = or i32 [[X:%.*]], -2147483648
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[OR]], -1
 ; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[OR]], 1
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 0, i32 [[ADD]]
-; CHECK-NEXT:    ret i32 [[SEL]]
+; CHECK-NEXT:    ret i32 [[ADD]]
 ;
   %or = or i32 %x, -2147483648
   %cmp = icmp eq i32 %or, -1
@@ -634,10 +568,8 @@ define i32 @test_add__nsw_is_safe(i32 %x) {
 
 define i32 @test_add_nuw_nsw__none_are_safe(i32 %x) {
 ; CHECK-LABEL: @test_add_nuw_nsw__none_are_safe(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X:%.*]], 3
-; CHECK-NEXT:    [[ADD:%.*]] = add nuw nsw i32 [[X]], 1
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 4, i32 [[ADD]]
-; CHECK-NEXT:    ret i32 [[SEL]]
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[X:%.*]], 1
+; CHECK-NEXT:    ret i32 [[ADD]]
 ;
   %cmp = icmp eq i32 %x, 3
   %add = add nuw nsw i32 %x, 1
@@ -647,10 +579,8 @@ define i32 @test_add_nuw_nsw__none_are_safe(i32 %x) {
 
 define i32 @test_add_nuw__none_are_safe(i32 %x) {
 ; CHECK-LABEL: @test_add_nuw__none_are_safe(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X:%.*]], 3
-; CHECK-NEXT:    [[ADD:%.*]] = add nuw i32 [[X]], 1
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 4, i32 [[ADD]]
-; CHECK-NEXT:    ret i32 [[SEL]]
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[X:%.*]], 1
+; CHECK-NEXT:    ret i32 [[ADD]]
 ;
   %cmp = icmp eq i32 %x, 3
   %add = add nuw i32 %x, 1
@@ -660,10 +590,8 @@ define i32 @test_add_nuw__none_are_safe(i32 %x) {
 
 define i32 @test_add_nsw__none_are_safe(i32 %x) {
 ; CHECK-LABEL: @test_add_nsw__none_are_safe(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X:%.*]], 3
-; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[X]], 1
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 4, i32 [[ADD]]
-; CHECK-NEXT:    ret i32 [[SEL]]
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[X:%.*]], 1
+; CHECK-NEXT:    ret i32 [[ADD]]
 ;
   %cmp = icmp eq i32 %x, 3
   %add = add nsw i32 %x, 1
@@ -685,10 +613,8 @@ define i32 @test_add__none_are_safe(i32 %x) {
 define i32 @test_sub_nuw_nsw__all_are_safe(i32 %x) {
 ; CHECK-LABEL: @test_sub_nuw_nsw__all_are_safe(
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 255
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 6
 ; CHECK-NEXT:    [[SUB:%.*]] = sub nuw nsw i32 -254, [[AND]]
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 -260, i32 [[SUB]]
-; CHECK-NEXT:    ret i32 [[SEL]]
+; CHECK-NEXT:    ret i32 [[SUB]]
 ;
   %and = and i32 %x, 255
   %cmp = icmp eq i32 %and, 6
@@ -700,10 +626,8 @@ define i32 @test_sub_nuw_nsw__all_are_safe(i32 %x) {
 define i32 @test_sub_nuw__all_are_safe(i32 %x) {
 ; CHECK-LABEL: @test_sub_nuw__all_are_safe(
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 255
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 6
 ; CHECK-NEXT:    [[SUB:%.*]] = sub nuw nsw i32 -254, [[AND]]
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 -260, i32 [[SUB]]
-; CHECK-NEXT:    ret i32 [[SEL]]
+; CHECK-NEXT:    ret i32 [[SUB]]
 ;
   %and = and i32 %x, 255
   %cmp = icmp eq i32 %and, 6
@@ -715,10 +639,8 @@ define i32 @test_sub_nuw__all_are_safe(i32 %x) {
 define i32 @test_sub_nsw__all_are_safe(i32 %x) {
 ; CHECK-LABEL: @test_sub_nsw__all_are_safe(
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 255
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 6
 ; CHECK-NEXT:    [[SUB:%.*]] = sub nuw nsw i32 -254, [[AND]]
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 -260, i32 [[SUB]]
-; CHECK-NEXT:    ret i32 [[SEL]]
+; CHECK-NEXT:    ret i32 [[SUB]]
 ;
   %and = and i32 %x, 255
   %cmp = icmp eq i32 %and, 6
@@ -730,10 +652,8 @@ define i32 @test_sub_nsw__all_are_safe(i32 %x) {
 define i32 @test_sub__all_are_safe(i32 %x) {
 ; CHECK-LABEL: @test_sub__all_are_safe(
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 255
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 6
 ; CHECK-NEXT:    [[SUB:%.*]] = sub nuw nsw i32 -254, [[AND]]
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 -260, i32 [[SUB]]
-; CHECK-NEXT:    ret i32 [[SEL]]
+; CHECK-NEXT:    ret i32 [[SUB]]
 ;
   %and = and i32 %x, 255
   %cmp = icmp eq i32 %and, 6
@@ -745,10 +665,8 @@ define i32 @test_sub__all_are_safe(i32 %x) {
 define i32 @test_sub_nuw_nsw__nuw_is_safe(i32 %x) {
 ; CHECK-LABEL: @test_sub_nuw_nsw__nuw_is_safe(
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 2147483647
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 1073741824
-; CHECK-NEXT:    [[SUB:%.*]] = sub nuw nsw i32 -2147483648, [[AND]]
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 1073741824, i32 [[SUB]]
-; CHECK-NEXT:    ret i32 [[SEL]]
+; CHECK-NEXT:    [[SUB:%.*]] = sub nuw i32 -2147483648, [[AND]]
+; CHECK-NEXT:    ret i32 [[SUB]]
 ;
   %and = and i32 %x, 2147483647
   %cmp = icmp eq i32 %and, 1073741824
@@ -760,10 +678,8 @@ define i32 @test_sub_nuw_nsw__nuw_is_safe(i32 %x) {
 define i32 @test_sub_nuw__nuw_is_safe(i32 %x) {
 ; CHECK-LABEL: @test_sub_nuw__nuw_is_safe(
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 2147483647
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 1073741824
 ; CHECK-NEXT:    [[SUB:%.*]] = sub nuw i32 -2147483648, [[AND]]
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 1073741824, i32 [[SUB]]
-; CHECK-NEXT:    ret i32 [[SEL]]
+; CHECK-NEXT:    ret i32 [[SUB]]
 ;
   %and = and i32 %x, 2147483647
   %cmp = icmp eq i32 %and, 1073741824
@@ -775,10 +691,8 @@ define i32 @test_sub_nuw__nuw_is_safe(i32 %x) {
 define i32 @test_sub_nsw__nuw_is_safe(i32 %x) {
 ; CHECK-LABEL: @test_sub_nsw__nuw_is_safe(
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 2147483647
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 1073741824
-; CHECK-NEXT:    [[SUB:%.*]] = sub nuw nsw i32 -2147483648, [[AND]]
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 1073741824, i32 [[SUB]]
-; CHECK-NEXT:    ret i32 [[SEL]]
+; CHECK-NEXT:    [[SUB:%.*]] = sub nuw i32 -2147483648, [[AND]]
+; CHECK-NEXT:    ret i32 [[SUB]]
 ;
   %and = and i32 %x, 2147483647
   %cmp = icmp eq i32 %and, 1073741824
@@ -790,10 +704,8 @@ define i32 @test_sub_nsw__nuw_is_safe(i32 %x) {
 define i32 @test_sub__nuw_is_safe(i32 %x) {
 ; CHECK-LABEL: @test_sub__nuw_is_safe(
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 2147483647
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 1073741824
 ; CHECK-NEXT:    [[SUB:%.*]] = sub nuw i32 -2147483648, [[AND]]
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 1073741824, i32 [[SUB]]
-; CHECK-NEXT:    ret i32 [[SEL]]
+; CHECK-NEXT:    ret i32 [[SUB]]
 ;
   %and = and i32 %x, 2147483647
   %cmp = icmp eq i32 %and, 1073741824
@@ -805,10 +717,8 @@ define i32 @test_sub__nuw_is_safe(i32 %x) {
 define i32 @test_sub_nuw_nsw__nsw_is_safe(i32 %x) {
 ; CHECK-LABEL: @test_sub_nuw_nsw__nsw_is_safe(
 ; CHECK-NEXT:    [[OR:%.*]] = or i32 [[X:%.*]], -2147483648
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[OR]], -2147483647
-; CHECK-NEXT:    [[SUB:%.*]] = sub nuw nsw i32 -2147483648, [[OR]]
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 -1, i32 [[SUB]]
-; CHECK-NEXT:    ret i32 [[SEL]]
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 -2147483648, [[OR]]
+; CHECK-NEXT:    ret i32 [[SUB]]
 ;
   %or = or i32 %x, -2147483648
   %cmp = icmp eq i32 %or, -2147483647
@@ -820,10 +730,8 @@ define i32 @test_sub_nuw_nsw__nsw_is_safe(i32 %x) {
 define i32 @test_sub_nuw__nsw_is_safe(i32 %x) {
 ; CHECK-LABEL: @test_sub_nuw__nsw_is_safe(
 ; CHECK-NEXT:    [[OR:%.*]] = or i32 [[X:%.*]], -2147483648
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[OR]], -2147483647
-; CHECK-NEXT:    [[SUB:%.*]] = sub nuw nsw i32 -2147483648, [[OR]]
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 -1, i32 [[SUB]]
-; CHECK-NEXT:    ret i32 [[SEL]]
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 -2147483648, [[OR]]
+; CHECK-NEXT:    ret i32 [[SUB]]
 ;
   %or = or i32 %x, -2147483648
   %cmp = icmp eq i32 %or, -2147483647
@@ -835,10 +743,8 @@ define i32 @test_sub_nuw__nsw_is_safe(i32 %x) {
 define i32 @test_sub_nsw__nsw_is_safe(i32 %x) {
 ; CHECK-LABEL: @test_sub_nsw__nsw_is_safe(
 ; CHECK-NEXT:    [[OR:%.*]] = or i32 [[X:%.*]], -2147483648
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[OR]], -2147483647
 ; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 -2147483648, [[OR]]
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 -1, i32 [[SUB]]
-; CHECK-NEXT:    ret i32 [[SEL]]
+; CHECK-NEXT:    ret i32 [[SUB]]
 ;
   %or = or i32 %x, -2147483648
   %cmp = icmp eq i32 %or, -2147483647
@@ -850,10 +756,8 @@ define i32 @test_sub_nsw__nsw_is_safe(i32 %x) {
 define i32 @test_sub__nsw_is_safe(i32 %x) {
 ; CHECK-LABEL: @test_sub__nsw_is_safe(
 ; CHECK-NEXT:    [[OR:%.*]] = or i32 [[X:%.*]], -2147483648
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[OR]], -2147483647
 ; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 -2147483648, [[OR]]
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 -1, i32 [[SUB]]
-; CHECK-NEXT:    ret i32 [[SEL]]
+; CHECK-NEXT:    ret i32 [[SUB]]
 ;
   %or = or i32 %x, -2147483648
   %cmp = icmp eq i32 %or, -2147483647
@@ -864,10 +768,8 @@ define i32 @test_sub__nsw_is_safe(i32 %x) {
 
 define i32 @test_sub_nuw_nsw__none_are_safe(i32 %x) {
 ; CHECK-LABEL: @test_sub_nuw_nsw__none_are_safe(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X:%.*]], 1
-; CHECK-NEXT:    [[SUB:%.*]] = sub nuw nsw i32 -2147483648, [[X]]
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 2147483647, i32 [[SUB]]
-; CHECK-NEXT:    ret i32 [[SEL]]
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 -2147483648, [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[SUB]]
 ;
   %cmp = icmp eq i32 %x, 1
   %sub = sub nuw nsw i32 -2147483648, %x
@@ -877,10 +779,8 @@ define i32 @test_sub_nuw_nsw__none_are_safe(i32 %x) {
 
 define i32 @test_sub_nuw__none_are_safe(i32 %x) {
 ; CHECK-LABEL: @test_sub_nuw__none_are_safe(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X:%.*]], 1
-; CHECK-NEXT:    [[SUB:%.*]] = sub nuw i32 -2147483648, [[X]]
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 2147483647, i32 [[SUB]]
-; CHECK-NEXT:    ret i32 [[SEL]]
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 -2147483648, [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[SUB]]
 ;
   %cmp = icmp eq i32 %x, 1
   %sub = sub nuw i32 -2147483648, %x
@@ -890,10 +790,8 @@ define i32 @test_sub_nuw__none_are_safe(i32 %x) {
 
 define i32 @test_sub_nsw__none_are_safe(i32 %x) {
 ; CHECK-LABEL: @test_sub_nsw__none_are_safe(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X:%.*]], 1
-; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 -2147483648, [[X]]
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 2147483647, i32 [[SUB]]
-; CHECK-NEXT:    ret i32 [[SEL]]
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 -2147483648, [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[SUB]]
 ;
   %cmp = icmp eq i32 %x, 1
   %sub = sub nsw i32 -2147483648, %x
@@ -915,10 +813,8 @@ define i32 @test_sub__none_are_safe(i32 %x) {
 define i32 @test_mul_nuw_nsw__all_are_safe(i32 %x) {
 ; CHECK-LABEL: @test_mul_nuw_nsw__all_are_safe(
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 255
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 17
 ; CHECK-NEXT:    [[MUL:%.*]] = mul nuw nsw i32 [[AND]], 9
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 153, i32 [[MUL]]
-; CHECK-NEXT:    ret i32 [[SEL]]
+; CHECK-NEXT:    ret i32 [[MUL]]
 ;
   %and = and i32 %x, 255
   %cmp = icmp eq i32 %and, 17
@@ -930,10 +826,8 @@ define i32 @test_mul_nuw_nsw__all_are_safe(i32 %x) {
 define i32 @test_mul_nuw__all_are_safe(i32 %x) {
 ; CHECK-LABEL: @test_mul_nuw__all_are_safe(
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 255
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 17
 ; CHECK-NEXT:    [[MUL:%.*]] = mul nuw nsw i32 [[AND]], 9
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 153, i32 [[MUL]]
-; CHECK-NEXT:    ret i32 [[SEL]]
+; CHECK-NEXT:    ret i32 [[MUL]]
 ;
   %and = and i32 %x, 255
   %cmp = icmp eq i32 %and, 17
@@ -945,10 +839,8 @@ define i32 @test_mul_nuw__all_are_safe(i32 %x) {
 define i32 @test_mul_nsw__all_are_safe(i32 %x) {
 ; CHECK-LABEL: @test_mul_nsw__all_are_safe(
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 255
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 17
 ; CHECK-NEXT:    [[MUL:%.*]] = mul nuw nsw i32 [[AND]], 9
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 153, i32 [[MUL]]
-; CHECK-NEXT:    ret i32 [[SEL]]
+; CHECK-NEXT:    ret i32 [[MUL]]
 ;
   %and = and i32 %x, 255
   %cmp = icmp eq i32 %and, 17
@@ -960,10 +852,8 @@ define i32 @test_mul_nsw__all_are_safe(i32 %x) {
 define i32 @test_mul__all_are_safe(i32 %x) {
 ; CHECK-LABEL: @test_mul__all_are_safe(
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 255
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 17
 ; CHECK-NEXT:    [[MUL:%.*]] = mul nuw nsw i32 [[AND]], 9
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 153, i32 [[MUL]]
-; CHECK-NEXT:    ret i32 [[SEL]]
+; CHECK-NEXT:    ret i32 [[MUL]]
 ;
   %and = and i32 %x, 255
   %cmp = icmp eq i32 %and, 17
@@ -975,10 +865,8 @@ define i32 @test_mul__all_are_safe(i32 %x) {
 define i32 @test_mul_nuw_nsw__nuw_is_safe(i32 %x) {
 ; CHECK-LABEL: @test_mul_nuw_nsw__nuw_is_safe(
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 268435457
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 268435456
-; CHECK-NEXT:    [[MUL:%.*]] = mul nuw nsw i32 [[AND]], 9
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 -1879048192, i32 [[MUL]]
-; CHECK-NEXT:    ret i32 [[SEL]]
+; CHECK-NEXT:    [[MUL:%.*]] = mul nuw i32 [[AND]], 9
+; CHECK-NEXT:    ret i32 [[MUL]]
 ;
   %and = and i32 %x, 268435457
   %cmp = icmp eq i32 %and, 268435456
@@ -990,10 +878,8 @@ define i32 @test_mul_nuw_nsw__nuw_is_safe(i32 %x) {
 define i32 @test_mul_nuw__nuw_is_safe(i32 %x) {
 ; CHECK-LABEL: @test_mul_nuw__nuw_is_safe(
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 268435457
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 268435456
 ; CHECK-NEXT:    [[MUL:%.*]] = mul nuw i32 [[AND]], 9
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 -1879048192, i32 [[MUL]]
-; CHECK-NEXT:    ret i32 [[SEL]]
+; CHECK-NEXT:    ret i32 [[MUL]]
 ;
   %and = and i32 %x, 268435457
   %cmp = icmp eq i32 %and, 268435456
@@ -1005,10 +891,8 @@ define i32 @test_mul_nuw__nuw_is_safe(i32 %x) {
 define i32 @test_mul_nsw__nuw_is_safe(i32 %x) {
 ; CHECK-LABEL: @test_mul_nsw__nuw_is_safe(
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 268435457
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 268435456
-; CHECK-NEXT:    [[MUL:%.*]] = mul nuw nsw i32 [[AND]], 9
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 -1879048192, i32 [[MUL]]
-; CHECK-NEXT:    ret i32 [[SEL]]
+; CHECK-NEXT:    [[MUL:%.*]] = mul nuw i32 [[AND]], 9
+; CHECK-NEXT:    ret i32 [[MUL]]
 ;
   %and = and i32 %x, 268435457
   %cmp = icmp eq i32 %and, 268435456
@@ -1020,10 +904,8 @@ define i32 @test_mul_nsw__nuw_is_safe(i32 %x) {
 define i32 @test_mul__nuw_is_safe(i32 %x) {
 ; CHECK-LABEL: @test_mul__nuw_is_safe(
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 268435457
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 268435456
 ; CHECK-NEXT:    [[MUL:%.*]] = mul nuw i32 [[AND]], 9
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 -1879048192, i32 [[MUL]]
-; CHECK-NEXT:    ret i32 [[SEL]]
+; CHECK-NEXT:    ret i32 [[MUL]]
 ;
   %and = and i32 %x, 268435457
   %cmp = icmp eq i32 %and, 268435456
@@ -1035,10 +917,8 @@ define i32 @test_mul__nuw_is_safe(i32 %x) {
 define i32 @test_mul_nuw_nsw__nsw_is_safe(i32 %x) {
 ; CHECK-LABEL: @test_mul_nuw_nsw__nsw_is_safe(
 ; CHECK-NEXT:    [[AND:%.*]] = or i32 [[X:%.*]], -83886080
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], -83886079
-; CHECK-NEXT:    [[MUL:%.*]] = mul nuw nsw i32 [[AND]], 9
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 -754974711, i32 [[MUL]]
-; CHECK-NEXT:    ret i32 [[SEL]]
+; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[AND]], 9
+; CHECK-NEXT:    ret i32 [[MUL]]
 ;
   %and = or i32 %x, -83886080
   %cmp = icmp eq i32 %and, -83886079
@@ -1050,10 +930,8 @@ define i32 @test_mul_nuw_nsw__nsw_is_safe(i32 %x) {
 define i32 @test_mul_nuw__nsw_is_safe(i32 %x) {
 ; CHECK-LABEL: @test_mul_nuw__nsw_is_safe(
 ; CHECK-NEXT:    [[AND:%.*]] = or i32 [[X:%.*]], -83886080
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], -83886079
-; CHECK-NEXT:    [[MUL:%.*]] = mul nuw nsw i32 [[AND]], 9
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 -754974711, i32 [[MUL]]
-; CHECK-NEXT:    ret i32 [[SEL]]
+; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[AND]], 9
+; CHECK-NEXT:    ret i32 [[MUL]]
 ;
   %and = or i32 %x, -83886080
   %cmp = icmp eq i32 %and, -83886079
@@ -1065,10 +943,8 @@ define i32 @test_mul_nuw__nsw_is_safe(i32 %x) {
 define i32 @test_mul_nsw__nsw_is_safe(i32 %x) {
 ; CHECK-LABEL: @test_mul_nsw__nsw_is_safe(
 ; CHECK-NEXT:    [[AND:%.*]] = or i32 [[X:%.*]], -83886080
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], -83886079
 ; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[AND]], 9
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 -754974711, i32 [[MUL]]
-; CHECK-NEXT:    ret i32 [[SEL]]
+; CHECK-NEXT:    ret i32 [[MUL]]
 ;
   %and = or i32 %x, -83886080
   %cmp = icmp eq i32 %and, -83886079
@@ -1080,10 +956,8 @@ define i32 @test_mul_nsw__nsw_is_safe(i32 %x) {
 define i32 @test_mul__nsw_is_safe(i32 %x) {
 ; CHECK-LABEL: @test_mul__nsw_is_safe(
 ; CHECK-NEXT:    [[AND:%.*]] = or i32 [[X:%.*]], -83886080
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], -83886079
 ; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[AND]], 9
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 -754974711, i32 [[MUL]]
-; CHECK-NEXT:    ret i32 [[SEL]]
+; CHECK-NEXT:    ret i32 [[MUL]]
 ;
   %and = or i32 %x, -83886080
   %cmp = icmp eq i32 %and, -83886079
@@ -1094,10 +968,8 @@ define i32 @test_mul__nsw_is_safe(i32 %x) {
 
 define i32 @test_mul_nuw_nsw__none_are_safe(i32 %x) {
 ; CHECK-LABEL: @test_mul_nuw_nsw__none_are_safe(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X:%.*]], 805306368
-; CHECK-NEXT:    [[MUL:%.*]] = mul nuw nsw i32 [[X]], 9
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 -1342177280, i32 [[MUL]]
-; CHECK-NEXT:    ret i32 [[SEL]]
+; CHECK-NEXT:    [[MUL:%.*]] = mul i32 [[X:%.*]], 9
+; CHECK-NEXT:    ret i32 [[MUL]]
 ;
   %cmp = icmp eq i32 %x, 805306368
   %mul = mul nuw nsw i32 %x, 9
@@ -1107,10 +979,8 @@ define i32 @test_mul_nuw_nsw__none_are_safe(i32 %x) {
 
 define i32 @test_mul_nuw__none_are_safe(i32 %x) {
 ; CHECK-LABEL: @test_mul_nuw__none_are_safe(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X:%.*]], 805306368
-; CHECK-NEXT:    [[MUL:%.*]] = mul nuw i32 [[X]], 9
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 -1342177280, i32 [[MUL]]
-; CHECK-NEXT:    ret i32 [[SEL]]
+; CHECK-NEXT:    [[MUL:%.*]] = mul i32 [[X:%.*]], 9
+; CHECK-NEXT:    ret i32 [[MUL]]
 ;
   %cmp = icmp eq i32 %x, 805306368
   %mul = mul nuw i32 %x, 9
@@ -1120,10 +990,8 @@ define i32 @test_mul_nuw__none_are_safe(i32 %x) {
 
 define i32 @test_mul_nsw__none_are_safe(i32 %x) {
 ; CHECK-LABEL: @test_mul_nsw__none_are_safe(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X:%.*]], 805306368
-; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[X]], 9
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 -1342177280, i32 [[MUL]]
-; CHECK-NEXT:    ret i32 [[SEL]]
+; CHECK-NEXT:    [[MUL:%.*]] = mul i32 [[X:%.*]], 9
+; CHECK-NEXT:    ret i32 [[MUL]]
 ;
   %cmp = icmp eq i32 %x, 805306368
   %mul = mul nsw i32 %x, 9
diff --git a/llvm/test/Transforms/InstCombine/select.ll b/llvm/test/Transforms/InstCombine/select.ll
index 972037682..2f04efe44 100644
--- a/llvm/test/Transforms/InstCombine/select.ll
+++ b/llvm/test/Transforms/InstCombine/select.ll
@@ -5,39 +5,6 @@
 
 target datalayout = "e-p:64:64-p1:16:16-p2:32:32:32-p3:64:64:64"
 
-define i32 @test1(i32 %A, i32 %B) {
-; CHECK-LABEL: @test1(
-; CHECK-NEXT:    ret i32 [[B:%.*]]
-;
-  %C = select i1 false, i32 %A, i32 %B
-  ret i32 %C
-}
-
-define i32 @test2(i32 %A, i32 %B) {
-; CHECK-LABEL: @test2(
-; CHECK-NEXT:    ret i32 [[A:%.*]]
-;
-  %C = select i1 true, i32 %A, i32 %B
-  ret i32 %C
-}
-
-
-define i32 @test3(i1 %C, i32 %I) {
-; CHECK-LABEL: @test3(
-; CHECK-NEXT:    ret i32 [[I:%.*]]
-;
-  %V = select i1 %C, i32 %I, i32 %I
-  ret i32 %V
-}
-
-define i1 @test4(i1 %C) {
-; CHECK-LABEL: @test4(
-; CHECK-NEXT:    ret i1 [[C:%.*]]
-;
-  %V = select i1 %C, i1 true, i1 false
-  ret i1 %V
-}
-
 define i1 @test5(i1 %C) {
 ; CHECK-LABEL: @test5(
 ; CHECK-NEXT:    [[NOT_C:%.*]] = xor i1 [[C:%.*]], true
@@ -694,8 +661,8 @@ define i32 @test41(i1 %cond, i32 %x, i32 %y) {
 define i32 @test42(i32 %x, i32 %y) {
 ; CHECK-LABEL: @test42(
 ; CHECK-NEXT:    [[COND:%.*]] = icmp eq i32 [[X:%.*]], 0
-; CHECK-NEXT:    [[TMP1:%.*]] = zext i1 [[COND]] to i32
-; CHECK-NEXT:    [[C:%.*]] = sub i32 [[Y:%.*]], [[TMP1]]
+; CHECK-NEXT:    [[B:%.*]] = sext i1 [[COND]] to i32
+; CHECK-NEXT:    [[C:%.*]] = add i32 [[B]], [[Y:%.*]]
 ; CHECK-NEXT:    ret i32 [[C]]
 ;
   %b = add i32 %y, -1
@@ -707,8 +674,8 @@ define i32 @test42(i32 %x, i32 %y) {
 define <2 x i32> @test42vec(<2 x i32> %x, <2 x i32> %y) {
 ; CHECK-LABEL: @test42vec(
 ; CHECK-NEXT:    [[COND:%.*]] = icmp eq <2 x i32> [[X:%.*]], zeroinitializer
-; CHECK-NEXT:    [[TMP1:%.*]] = zext <2 x i1> [[COND]] to <2 x i32>
-; CHECK-NEXT:    [[C:%.*]] = sub <2 x i32> [[Y:%.*]], [[TMP1]]
+; CHECK-NEXT:    [[B:%.*]] = sext <2 x i1> [[COND]] to <2 x i32>
+; CHECK-NEXT:    [[C:%.*]] = add <2 x i32> [[B]], [[Y:%.*]]
 ; CHECK-NEXT:    ret <2 x i32> [[C]]
 ;
   %b = add <2 x i32> %y, <i32 -1, i32 -1>
@@ -1318,10 +1285,8 @@ define i32 @test_select_select1(i32 %a, i32 %r0, i32 %r1, i32 %v1, i32 %v2) {
 
 define i32 @PR23757(i32 %x) {
 ; CHECK-LABEL: @PR23757(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X:%.*]], 2147483647
-; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[X]], 1
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 -2147483648, i32 [[ADD]]
-; CHECK-NEXT:    ret i32 [[SEL]]
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[X:%.*]], 1
+; CHECK-NEXT:    ret i32 [[ADD]]
 ;
   %cmp = icmp eq i32 %x, 2147483647
   %add = add nsw i32 %x, 1
@@ -1329,6 +1294,43 @@ define i32 @PR23757(i32 %x) {
   ret i32 %sel
 }
 
+define i32 @PR23757_swapped(i32 %x) {
+; CHECK-LABEL: @PR23757_swapped(
+; CHECK-NEXT:    ret i32 -2147483648
+;
+  %cmp = icmp eq i32 %x, 2147483647
+  %add = add nsw i32 %x, 1
+  %sel = select i1 %cmp, i32 %add, i32 -2147483648
+  ret i32 %sel
+}
+
+define i32 @PR23757_ne(i32 %x, i1* %p) {
+; CHECK-LABEL: @PR23757_ne(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[X:%.*]], 2147483647
+; CHECK-NEXT:    store i1 [[CMP]], i1* [[P:%.*]], align 1
+; CHECK-NEXT:    ret i32 -2147483648
+;
+  %cmp = icmp ne i32 %x, 2147483647
+  store i1 %cmp, i1* %p ; thwart predicate canonicalization
+  %add = add nsw i32 %x, 1
+  %sel = select i1 %cmp, i32 -2147483648, i32 %add
+  ret i32 %sel
+}
+
+define i32 @PR23757_ne_swapped(i32 %x, i1* %p) {
+; CHECK-LABEL: @PR23757_ne_swapped(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[X:%.*]], 2147483647
+; CHECK-NEXT:    store i1 [[CMP]], i1* [[P:%.*]], align 1
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[X]], 1
+; CHECK-NEXT:    ret i32 [[ADD]]
+;
+  %cmp = icmp ne i32 %x, 2147483647
+  store i1 %cmp, i1* %p ; thwart predicate canonicalization
+  %add = add nsw i32 %x, 1
+  %sel = select i1 %cmp, i32 %add, i32 -2147483648
+  ret i32 %sel
+}
+
 ; max(max(~a, -1), -1) --> ~min(a, 0)
 
 define i32 @PR27137(i32 %a) {
@@ -1504,3 +1506,137 @@ define i8 @test90(i1 %cond, i8 %w, i8 %x, i8 %y, i8 %z) {
   ret i8 %c
 }
 
+define i32 @test_shl_zext_bool(i1 %t) {
+; CHECK-LABEL: @test_shl_zext_bool(
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[T:%.*]], i32 4, i32 0
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %r = select i1 %t, i32 4, i32 0
+  ret i32 %r
+}
+
+define <2 x i32> @test_shl_zext_bool_splat(<2 x i1> %t) {
+; CHECK-LABEL: @test_shl_zext_bool_splat(
+; CHECK-NEXT:    [[R:%.*]] = select <2 x i1> [[T:%.*]], <2 x i32> <i32 8, i32 8>, <2 x i32> zeroinitializer
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %r = select <2 x i1> %t, <2 x i32> <i32 8, i32 8>, <2 x i32> zeroinitializer
+  ret <2 x i32> %r
+}
+
+define <2 x i32> @test_shl_zext_bool_vec(<2 x i1> %t) {
+; CHECK-LABEL: @test_shl_zext_bool_vec(
+; CHECK-NEXT:    [[R:%.*]] = select <2 x i1> [[T:%.*]], <2 x i32> <i32 4, i32 8>, <2 x i32> zeroinitializer
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %r = select <2 x i1> %t, <2 x i32> <i32 4, i32 8>, <2 x i32> zeroinitializer
+  ret <2 x i32> %r
+}
+
+define float @copysign1(float %x) {
+; CHECK-LABEL: @copysign1(
+; CHECK-NEXT:    [[I:%.*]] = bitcast float [[X:%.*]] to i32
+; CHECK-NEXT:    [[ISPOS:%.*]] = icmp sgt i32 [[I]], -1
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[ISPOS]], float 1.000000e+00, float -1.000000e+00
+; CHECK-NEXT:    ret float [[R]]
+;
+  %i = bitcast float %x to i32
+  %ispos = icmp sgt i32 %i, -1
+  %r = select i1 %ispos, float 1.0, float -1.0
+  ret float %r
+}
+
+define <2 x float> @copysign2(<2 x float> %x) {
+; CHECK-LABEL: @copysign2(
+; CHECK-NEXT:    [[I:%.*]] = bitcast <2 x float> [[X:%.*]] to <2 x i32>
+; CHECK-NEXT:    [[ISNEG:%.*]] = icmp slt <2 x i32> [[I]], zeroinitializer
+; CHECK-NEXT:    [[R:%.*]] = select nsz <2 x i1> [[ISNEG]], <2 x float> <float 4.200000e+01, float 4.200000e+01>, <2 x float> <float -4.200000e+01, float -4.200000e+01>
+; CHECK-NEXT:    ret <2 x float> [[R]]
+;
+  %i = bitcast <2 x float> %x to <2 x i32>
+  %isneg = icmp slt <2 x i32> %i, zeroinitializer
+  %r = select nsz <2 x i1> %isneg, <2 x float> <float 42.0, float 42.0>, <2 x float> <float -42.0, float -42.0>
+  ret <2 x float> %r
+}
+
+define float @copysign3(float %x) {
+; CHECK-LABEL: @copysign3(
+; CHECK-NEXT:    [[I:%.*]] = bitcast float [[X:%.*]] to i32
+; CHECK-NEXT:    [[ISPOS:%.*]] = icmp sgt i32 [[I]], -1
+; CHECK-NEXT:    [[R:%.*]] = select fast i1 [[ISPOS]], float -4.300000e+01, float 4.300000e+01
+; CHECK-NEXT:    ret float [[R]]
+;
+  %i = bitcast float %x to i32
+  %ispos = icmp ult i32 %i, 2147483648
+  %r = select fast i1 %ispos, float -43.0, float 43.0
+  ret float %r
+}
+
+define <2 x float> @copysign4(<2 x float> %x) {
+; CHECK-LABEL: @copysign4(
+; CHECK-NEXT:    [[I:%.*]] = bitcast <2 x float> [[X:%.*]] to <2 x i32>
+; CHECK-NEXT:    [[ISNEG:%.*]] = icmp slt <2 x i32> [[I]], zeroinitializer
+; CHECK-NEXT:    [[R:%.*]] = select nnan arcp <2 x i1> [[ISNEG]], <2 x float> <float 4.200000e+01, float undef>, <2 x float> <float -4.200000e+01, float -4.200000e+01>
+; CHECK-NEXT:    ret <2 x float> [[R]]
+;
+  %i = bitcast <2 x float> %x to <2 x i32>
+  %isneg = icmp ugt <2 x i32> %i, <i32 2147483647, i32 2147483647>
+  %r = select arcp nnan <2 x i1> %isneg, <2 x float> <float 42.0, float undef>, <2 x float> <float -42.0, float -42.0>
+  ret <2 x float> %r
+}
+
+declare void @use1(i1)
+
+define float @copysign_extra_use(float %x) {
+; CHECK-LABEL: @copysign_extra_use(
+; CHECK-NEXT:    [[I:%.*]] = bitcast float [[X:%.*]] to i32
+; CHECK-NEXT:    [[ISNEG:%.*]] = icmp slt i32 [[I]], 0
+; CHECK-NEXT:    call void @use1(i1 [[ISNEG]])
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[ISNEG]], float -4.400000e+01, float 4.400000e+01
+; CHECK-NEXT:    ret float [[R]]
+;
+  %i = bitcast float %x to i32
+  %isneg = icmp ugt i32 %i, 2147483647
+  call void @use1(i1 %isneg)
+  %r = select i1 %isneg, float -44.0, float 44.0
+  ret float %r
+}
+
+define float @copysign_type_mismatch(double %x) {
+; CHECK-LABEL: @copysign_type_mismatch(
+; CHECK-NEXT:    [[I:%.*]] = bitcast double [[X:%.*]] to i64
+; CHECK-NEXT:    [[ISPOS:%.*]] = icmp sgt i64 [[I]], -1
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[ISPOS]], float 1.000000e+00, float -1.000000e+00
+; CHECK-NEXT:    ret float [[R]]
+;
+  %i = bitcast double %x to i64
+  %ispos = icmp sgt i64 %i, -1
+  %r = select i1 %ispos, float 1.0, float -1.0
+  ret float %r
+}
+
+define float @copysign_wrong_cmp(float %x) {
+; CHECK-LABEL: @copysign_wrong_cmp(
+; CHECK-NEXT:    [[I:%.*]] = bitcast float [[X:%.*]] to i32
+; CHECK-NEXT:    [[ISPOS:%.*]] = icmp sgt i32 [[I]], 0
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[ISPOS]], float 1.000000e+00, float -1.000000e+00
+; CHECK-NEXT:    ret float [[R]]
+;
+  %i = bitcast float %x to i32
+  %ispos = icmp sgt i32 %i, 0
+  %r = select i1 %ispos, float 1.0, float -1.0
+  ret float %r
+}
+
+define float @copysign_wrong_const(float %x) {
+; CHECK-LABEL: @copysign_wrong_const(
+; CHECK-NEXT:    [[I:%.*]] = bitcast float [[X:%.*]] to i32
+; CHECK-NEXT:    [[ISPOS:%.*]] = icmp sgt i32 [[I]], -1
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[ISPOS]], float 2.000000e+00, float -1.000000e+00
+; CHECK-NEXT:    ret float [[R]]
+;
+  %i = bitcast float %x to i32
+  %ispos = icmp sgt i32 %i, -1
+  %r = select i1 %ispos, float 2.0, float -1.0
+  ret float %r
+}
diff --git a/llvm/test/Transforms/InstCombine/select_arithmetic.ll b/llvm/test/Transforms/InstCombine/select_arithmetic.ll
index 642fa6ccc..535e62a3c 100644
--- a/llvm/test/Transforms/InstCombine/select_arithmetic.ll
+++ b/llvm/test/Transforms/InstCombine/select_arithmetic.ll
@@ -92,3 +92,48 @@ define float @test5(i1 zeroext %arg, float %div) {
   ret float %mul
 }
 
+define float @fmul_nnan_nsz(i1 %cond, float %val) {
+; CHECK-LABEL: @fmul_nnan_nsz(
+; CHECK-NEXT:    ret float 0.000000e+00
+;
+  %lhs = select i1 %cond, float %val, float +0.0
+  %rhs = select i1 %cond, float -0.0, float %val
+  %mul = fmul nnan nsz float %lhs, %rhs
+  ret float %mul
+}
+
+define <2 x float> @fadd_nsz(<2 x i1> %cond, <2 x float> %val) {
+; CHECK-LABEL: @fadd_nsz(
+; CHECK-NEXT:    ret <2 x float> [[VAL:%.*]]
+;
+  %lhs = select <2 x i1> %cond, <2 x float> %val, <2 x float> <float +0.0, float +0.0>
+  %rhs = select <2 x i1> %cond, <2 x float> <float +0.0, float +0.0>, <2 x float> %val
+  %add = fadd nsz <2 x float> %lhs, %rhs
+  ret <2 x float> %add
+}
+
+define double @fsub_nnan(i1 %cond, double %val, double %val2) {
+; CHECK-LABEL: @fsub_nnan(
+; CHECK-NEXT:    [[TMP1:%.*]] = fadd nnan double [[VAL2:%.*]], -7.000000e+00
+; CHECK-NEXT:    [[ADD:%.*]] = select nnan i1 [[COND:%.*]], double 0.000000e+00, double [[TMP1]]
+; CHECK-NEXT:    ret double [[ADD]]
+;
+  %lhs = select i1 %cond, double %val, double %val2
+  %rhs = select i1 %cond, double %val, double 7.0
+  %add = fsub nnan double %lhs, %rhs
+  ret double %add
+}
+
+; TODO combine selects feeding fdiv like we do for fmul, fadd and fsub
+define double @fdiv_nnan_nsz(i1 %cond, double %val, double %val2) {
+; CHECK-LABEL: @fdiv_nnan_nsz(
+; CHECK-NEXT:    [[LHS:%.*]] = select i1 [[COND:%.*]], double [[VAL2:%.*]], double 0.000000e+00
+; CHECK-NEXT:    [[RHS:%.*]] = select i1 [[COND]], double 4.200000e+01, double [[VAL:%.*]]
+; CHECK-NEXT:    [[ADD:%.*]] = fdiv nnan nsz double [[LHS]], [[RHS]]
+; CHECK-NEXT:    ret double [[ADD]]
+;
+  %lhs = select i1 %cond, double %val2, double 0.0
+  %rhs = select i1 %cond, double 42.0, double %val
+  %add = fdiv nnan nsz double %lhs, %rhs
+  ret double %add
+}
diff --git a/llvm/test/Transforms/InstCombine/shift-amount-reassociation-in-bittest-with-truncation-lshr.ll b/llvm/test/Transforms/InstCombine/shift-amount-reassociation-in-bittest-with-truncation-lshr.ll
new file mode 100644
index 000000000..118186f98
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/shift-amount-reassociation-in-bittest-with-truncation-lshr.ll
@@ -0,0 +1,468 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instcombine -S | FileCheck %s
+
+; Given pattern:
+;   icmp eq/ne (and ((x shift Q), (y oppositeshift K))), 0
+; we should move shifts to the same hand of 'and', i.e. e.g. rewrite as
+;   icmp eq/ne (and (((x shift Q) shift K), y)), 0
+; We are only interested in opposite logical shifts here.
+; We still can handle the case where there is a truncation between a shift
+; and an 'and', thought the legality check isn't obvious.
+
+;-------------------------------------------------------------------------------
+; Basic scalar tests
+;-------------------------------------------------------------------------------
+
+; This fold can't be performed for fully variable %x and %y
+define i1 @n0(i32 %x, i64 %y, i32 %len) {
+; CHECK-LABEL: @n0(
+; CHECK-NEXT:    [[T0:%.*]] = sub i32 32, [[LEN:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = shl i32 [[X:%.*]], [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[LEN]], -16
+; CHECK-NEXT:    [[T2_WIDE:%.*]] = zext i32 [[T2]] to i64
+; CHECK-NEXT:    [[T3:%.*]] = lshr i64 [[Y:%.*]], [[T2_WIDE]]
+; CHECK-NEXT:    [[T3_TRUNC:%.*]] = trunc i64 [[T3]] to i32
+; CHECK-NEXT:    [[T4:%.*]] = and i32 [[T1]], [[T3_TRUNC]]
+; CHECK-NEXT:    [[T5:%.*]] = icmp ne i32 [[T4]], 0
+; CHECK-NEXT:    ret i1 [[T5]]
+;
+  %t0 = sub i32 32, %len
+  %t1 = shl i32 %x, %t0
+  %t2 = add i32 %len, -16
+  %t2_wide = zext i32 %t2 to i64
+  %t3 = lshr i64 %y, %t2_wide
+  %t3_trunc = trunc i64 %t3 to i32
+  %t4 = and i32 %t1, %t3_trunc
+  %t5 = icmp ne i32 %t4, 0
+  ret i1 %t5
+}
+
+; However we can fold if %x/%y are constants that pass extra legality check.
+
+; New shift amount would be 16, %x has 16 leading zeros - can fold.
+define i1 @t1(i64 %y, i32 %len) {
+; CHECK-LABEL: @t1(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[Y:%.*]], 4294901760
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %t0 = sub i32 32, %len
+  %t1 = shl i32 65535, %t0
+  %t2 = add i32 %len, -16
+  %t2_wide = zext i32 %t2 to i64
+  %t3 = lshr i64 %y, %t2_wide
+  %t3_trunc = trunc i64 %t3 to i32
+  %t4 = and i32 %t1, %t3_trunc
+  %t5 = icmp ne i32 %t4, 0
+  ret i1 %t5
+}
+; Note that we indeed look at leading zeros!
+define i1 @t1_single_bit(i64 %y, i32 %len) {
+; CHECK-LABEL: @t1_single_bit(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[Y:%.*]], 2147483648
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %t0 = sub i32 32, %len
+  %t1 = shl i32 32768, %t0
+  %t2 = add i32 %len, -16
+  %t2_wide = zext i32 %t2 to i64
+  %t3 = lshr i64 %y, %t2_wide
+  %t3_trunc = trunc i64 %t3 to i32
+  %t4 = and i32 %t1, %t3_trunc
+  %t5 = icmp ne i32 %t4, 0
+  ret i1 %t5
+}
+; New shift amount would be 16, %x has 15 leading zeros - can not fold.
+define i1 @n2(i64 %y, i32 %len) {
+; CHECK-LABEL: @n2(
+; CHECK-NEXT:    [[T0:%.*]] = sub i32 32, [[LEN:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = shl i32 131071, [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[LEN]], -16
+; CHECK-NEXT:    [[T2_WIDE:%.*]] = zext i32 [[T2]] to i64
+; CHECK-NEXT:    [[T3:%.*]] = lshr i64 [[Y:%.*]], [[T2_WIDE]]
+; CHECK-NEXT:    [[T3_TRUNC:%.*]] = trunc i64 [[T3]] to i32
+; CHECK-NEXT:    [[T4:%.*]] = and i32 [[T1]], [[T3_TRUNC]]
+; CHECK-NEXT:    [[T5:%.*]] = icmp ne i32 [[T4]], 0
+; CHECK-NEXT:    ret i1 [[T5]]
+;
+  %t0 = sub i32 32, %len
+  %t1 = shl i32 131071, %t0
+  %t2 = add i32 %len, -16
+  %t2_wide = zext i32 %t2 to i64
+  %t3 = lshr i64 %y, %t2_wide
+  %t3_trunc = trunc i64 %t3 to i32
+  %t4 = and i32 %t1, %t3_trunc
+  %t5 = icmp ne i32 %t4, 0
+  ret i1 %t5
+}
+
+; New shift amount would be 16, %y has 47 leading zeros - can fold.
+define i1 @t3(i32 %x, i32 %len) {
+; CHECK-LABEL: @t3(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i32 [[TMP1]], 0
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %t0 = sub i32 32, %len
+  %t1 = shl i32 %x, %t0
+  %t2 = add i32 %len, -16
+  %t2_wide = zext i32 %t2 to i64
+  %t3 = lshr i64 131071, %t2_wide
+  %t3_trunc = trunc i64 %t3 to i32
+  %t4 = and i32 %t1, %t3_trunc
+  %t5 = icmp ne i32 %t4, 0
+  ret i1 %t5
+}
+; Note that we indeed look at leading zeros!
+define i1 @t3_singlebit(i32 %x, i32 %len) {
+; CHECK-LABEL: @t3_singlebit(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i32 [[TMP1]], 0
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %t0 = sub i32 32, %len
+  %t1 = shl i32 %x, %t0
+  %t2 = add i32 %len, -16
+  %t2_wide = zext i32 %t2 to i64
+  %t3 = lshr i64 65536, %t2_wide
+  %t3_trunc = trunc i64 %t3 to i32
+  %t4 = and i32 %t1, %t3_trunc
+  %t5 = icmp ne i32 %t4, 0
+  ret i1 %t5
+}
+; New shift amount would be 16, %y has 48 leading zeros - can not fold.
+define i1 @n4(i32 %x, i32 %len) {
+; CHECK-LABEL: @n4(
+; CHECK-NEXT:    [[T0:%.*]] = sub i32 32, [[LEN:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = shl i32 [[X:%.*]], [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[LEN]], -16
+; CHECK-NEXT:    [[T2_WIDE:%.*]] = zext i32 [[T2]] to i64
+; CHECK-NEXT:    [[T3:%.*]] = lshr i64 262143, [[T2_WIDE]]
+; CHECK-NEXT:    [[T3_TRUNC:%.*]] = trunc i64 [[T3]] to i32
+; CHECK-NEXT:    [[T4:%.*]] = and i32 [[T1]], [[T3_TRUNC]]
+; CHECK-NEXT:    [[T5:%.*]] = icmp ne i32 [[T4]], 0
+; CHECK-NEXT:    ret i1 [[T5]]
+;
+  %t0 = sub i32 32, %len
+  %t1 = shl i32 %x, %t0
+  %t2 = add i32 %len, -16
+  %t2_wide = zext i32 %t2 to i64
+  %t3 = lshr i64 262143, %t2_wide
+  %t3_trunc = trunc i64 %t3 to i32
+  %t4 = and i32 %t1, %t3_trunc
+  %t5 = icmp ne i32 %t4, 0
+  ret i1 %t5
+}
+
+; While we could still deal with arbitrary values if KnownBits can answer
+; the question, it isn't obvious it's worth it, so let's not for now.
+
+;-------------------------------------------------------------------------------
+; Vector tests
+;-------------------------------------------------------------------------------
+
+; New shift amount would be 16, minimal count of leading zeros in %x is 16. Ok.
+define <2 x i1> @t5_vec(<2 x i64> %y, <2 x i32> %len) {
+; CHECK-LABEL: @t5_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <2 x i64> [[Y:%.*]], <i64 16, i64 16>
+; CHECK-NEXT:    [[TMP2:%.*]] = and <2 x i64> [[TMP1]], <i64 65535, i64 32767>
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <2 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[TMP3]]
+;
+  %t0 = sub <2 x i32> <i32 32, i32 32>, %len
+  %t1 = shl <2 x i32> <i32 65535, i32 32767>, %t0
+  %t2 = add <2 x i32> %len, <i32 -16, i32 -16>
+  %t2_wide = zext <2 x i32> %t2 to <2 x i64>
+  %t3 = lshr <2 x i64> %y, %t2_wide
+  %t3_trunc = trunc <2 x i64> %t3 to <2 x i32>
+  %t4 = and <2 x i32> %t1, %t3_trunc
+  %t5 = icmp ne <2 x i32> %t4, <i32 0, i32 0>
+  ret <2 x i1> %t5
+}
+; New shift amount would be 16, minimal count of leading zeros in %x is 15, not ok to fold.
+define <2 x i1> @n6_vec(<2 x i64> %y, <2 x i32> %len) {
+; CHECK-LABEL: @n6_vec(
+; CHECK-NEXT:    [[T0:%.*]] = sub <2 x i32> <i32 32, i32 32>, [[LEN:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = shl <2 x i32> <i32 65535, i32 131071>, [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = add <2 x i32> [[LEN]], <i32 -16, i32 -16>
+; CHECK-NEXT:    [[T2_WIDE:%.*]] = zext <2 x i32> [[T2]] to <2 x i64>
+; CHECK-NEXT:    [[T3:%.*]] = lshr <2 x i64> [[Y:%.*]], [[T2_WIDE]]
+; CHECK-NEXT:    [[T3_TRUNC:%.*]] = trunc <2 x i64> [[T3]] to <2 x i32>
+; CHECK-NEXT:    [[T4:%.*]] = and <2 x i32> [[T1]], [[T3_TRUNC]]
+; CHECK-NEXT:    [[T5:%.*]] = icmp ne <2 x i32> [[T4]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[T5]]
+;
+  %t0 = sub <2 x i32> <i32 32, i32 32>, %len
+  %t1 = shl <2 x i32> <i32 65535, i32 131071>, %t0
+  %t2 = add <2 x i32> %len, <i32 -16, i32 -16>
+  %t2_wide = zext <2 x i32> %t2 to <2 x i64>
+  %t3 = lshr <2 x i64> %y, %t2_wide
+  %t3_trunc = trunc <2 x i64> %t3 to <2 x i32>
+  %t4 = and <2 x i32> %t1, %t3_trunc
+  %t5 = icmp ne <2 x i32> %t4, <i32 0, i32 0>
+  ret <2 x i1> %t5
+}
+
+; New shift amount would be 16, minimal count of leading zeros in %x is 47. Ok.
+define <2 x i1> @t7_vec(<2 x i32> %x, <2 x i32> %len) {
+; CHECK-LABEL: @t7_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[X:%.*]], <i32 1, i32 0>
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne <2 x i32> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[TMP2]]
+;
+  %t0 = sub <2 x i32> <i32 32, i32 32>, %len
+  %t1 = shl <2 x i32> %x, %t0
+  %t2 = add <2 x i32> %len, <i32 -16, i32 -16>
+  %t2_wide = zext <2 x i32> %t2 to <2 x i64>
+  %t3 = lshr <2 x i64> <i64 131071, i64 65535>, %t2_wide
+  %t3_trunc = trunc <2 x i64> %t3 to <2 x i32>
+  %t4 = and <2 x i32> %t1, %t3_trunc
+  %t5 = icmp ne <2 x i32> %t4, <i32 0, i32 0>
+  ret <2 x i1> %t5
+}
+; New shift amount would be 16, minimal count of leading zeros in %x is 48, not ok to fold.
+define <2 x i1> @n8_vec(<2 x i32> %x, <2 x i32> %len) {
+; CHECK-LABEL: @n8_vec(
+; CHECK-NEXT:    [[T0:%.*]] = sub <2 x i32> <i32 32, i32 32>, [[LEN:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = shl <2 x i32> [[X:%.*]], [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = add <2 x i32> [[LEN]], <i32 -16, i32 -16>
+; CHECK-NEXT:    [[T2_WIDE:%.*]] = zext <2 x i32> [[T2]] to <2 x i64>
+; CHECK-NEXT:    [[T3:%.*]] = lshr <2 x i64> <i64 131071, i64 262143>, [[T2_WIDE]]
+; CHECK-NEXT:    [[T3_TRUNC:%.*]] = trunc <2 x i64> [[T3]] to <2 x i32>
+; CHECK-NEXT:    [[T4:%.*]] = and <2 x i32> [[T1]], [[T3_TRUNC]]
+; CHECK-NEXT:    [[T5:%.*]] = icmp ne <2 x i32> [[T4]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[T5]]
+;
+  %t0 = sub <2 x i32> <i32 32, i32 32>, %len
+  %t1 = shl <2 x i32> %x, %t0
+  %t2 = add <2 x i32> %len, <i32 -16, i32 -16>
+  %t2_wide = zext <2 x i32> %t2 to <2 x i64>
+  %t3 = lshr <2 x i64> <i64 131071, i64 262143>, %t2_wide
+  %t3_trunc = trunc <2 x i64> %t3 to <2 x i32>
+  %t4 = and <2 x i32> %t1, %t3_trunc
+  %t5 = icmp ne <2 x i32> %t4, <i32 0, i32 0>
+  ret <2 x i1> %t5
+}
+
+;-------------------------------------------------------------------------------
+
+; Ok if the final shift amount is exactly one less than widest bit width.
+define i1 @t9_highest_bit(i32 %x, i64 %y, i32 %len) {
+; CHECK-LABEL: @t9_highest_bit(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[X:%.*]] to i64
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr i64 [[Y:%.*]], 63
+; CHECK-NEXT:    [[TMP3:%.*]] = and i64 [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne i64 [[TMP3]], 0
+; CHECK-NEXT:    ret i1 [[TMP4]]
+;
+  %t0 = sub i32 64, %len
+  %t1 = shl i32 %x, %t0
+  %t2 = add i32 %len, -1
+  %t2_wide = zext i32 %t2 to i64
+  %t3 = lshr i64 %y, %t2_wide
+  %t3_trunc = trunc i64 %t3 to i32
+  %t4 = and i32 %t1, %t3_trunc
+  %t5 = icmp ne i32 %t4, 0
+  ret i1 %t5
+}
+; Not highest bit.
+define i1 @t10_almost_highest_bit(i32 %x, i64 %y, i32 %len) {
+; CHECK-LABEL: @t10_almost_highest_bit(
+; CHECK-NEXT:    [[T0:%.*]] = sub i32 64, [[LEN:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = shl i32 [[X:%.*]], [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[LEN]], -2
+; CHECK-NEXT:    [[T2_WIDE:%.*]] = zext i32 [[T2]] to i64
+; CHECK-NEXT:    [[T3:%.*]] = lshr i64 [[Y:%.*]], [[T2_WIDE]]
+; CHECK-NEXT:    [[T3_TRUNC:%.*]] = trunc i64 [[T3]] to i32
+; CHECK-NEXT:    [[T4:%.*]] = and i32 [[T1]], [[T3_TRUNC]]
+; CHECK-NEXT:    [[T5:%.*]] = icmp ne i32 [[T4]], 0
+; CHECK-NEXT:    ret i1 [[T5]]
+;
+  %t0 = sub i32 64, %len
+  %t1 = shl i32 %x, %t0
+  %t2 = add i32 %len, -2
+  %t2_wide = zext i32 %t2 to i64
+  %t3 = lshr i64 %y, %t2_wide
+  %t3_trunc = trunc i64 %t3 to i32
+  %t4 = and i32 %t1, %t3_trunc
+  %t5 = icmp ne i32 %t4, 0
+  ret i1 %t5
+}
+
+; Ok if the final shift amount is zero.
+define i1 @t11_no_shift(i32 %x, i64 %y, i32 %len) {
+; CHECK-LABEL: @t11_no_shift(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[X:%.*]] to i64
+; CHECK-NEXT:    [[TMP2:%.*]] = and i64 [[TMP1]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    ret i1 [[TMP3]]
+;
+  %t0 = sub i32 64, %len
+  %t1 = shl i32 %x, %t0
+  %t2 = add i32 %len, -64
+  %t2_wide = zext i32 %t2 to i64
+  %t3 = lshr i64 %y, %t2_wide
+  %t3_trunc = trunc i64 %t3 to i32
+  %t4 = and i32 %t1, %t3_trunc
+  %t5 = icmp ne i32 %t4, 0
+  ret i1 %t5
+}
+; Not zero-shift.
+define i1 @t10_shift_by_one(i32 %x, i64 %y, i32 %len) {
+; CHECK-LABEL: @t10_shift_by_one(
+; CHECK-NEXT:    [[T0:%.*]] = sub i32 64, [[LEN:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = shl i32 [[X:%.*]], [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[LEN]], -63
+; CHECK-NEXT:    [[T2_WIDE:%.*]] = zext i32 [[T2]] to i64
+; CHECK-NEXT:    [[T3:%.*]] = lshr i64 [[Y:%.*]], [[T2_WIDE]]
+; CHECK-NEXT:    [[T3_TRUNC:%.*]] = trunc i64 [[T3]] to i32
+; CHECK-NEXT:    [[T4:%.*]] = and i32 [[T1]], [[T3_TRUNC]]
+; CHECK-NEXT:    [[T5:%.*]] = icmp ne i32 [[T4]], 0
+; CHECK-NEXT:    ret i1 [[T5]]
+;
+  %t0 = sub i32 64, %len
+  %t1 = shl i32 %x, %t0
+  %t2 = add i32 %len, -63
+  %t2_wide = zext i32 %t2 to i64
+  %t3 = lshr i64 %y, %t2_wide
+  %t3_trunc = trunc i64 %t3 to i32
+  %t4 = and i32 %t1, %t3_trunc
+  %t5 = icmp ne i32 %t4, 0
+  ret i1 %t5
+}
+
+; A mix of those conditions is ok.
+define <2 x i1> @t11_zero_and_almost_bitwidth(<2 x i32> %x, <2 x i64> %y, <2 x i32> %len) {
+; CHECK-LABEL: @t11_zero_and_almost_bitwidth(
+; CHECK-NEXT:    [[T0:%.*]] = sub <2 x i32> <i32 64, i32 64>, [[LEN:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = shl <2 x i32> [[X:%.*]], [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = add <2 x i32> [[LEN]], <i32 -1, i32 -64>
+; CHECK-NEXT:    [[T2_WIDE:%.*]] = zext <2 x i32> [[T2]] to <2 x i64>
+; CHECK-NEXT:    [[T3:%.*]] = lshr <2 x i64> [[Y:%.*]], [[T2_WIDE]]
+; CHECK-NEXT:    [[T3_TRUNC:%.*]] = trunc <2 x i64> [[T3]] to <2 x i32>
+; CHECK-NEXT:    [[T4:%.*]] = and <2 x i32> [[T1]], [[T3_TRUNC]]
+; CHECK-NEXT:    [[T5:%.*]] = icmp ne <2 x i32> [[T4]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[T5]]
+;
+  %t0 = sub <2 x i32> <i32 64, i32 64>, %len
+  %t1 = shl <2 x i32> %x, %t0
+  %t2 = add <2 x i32> %len, <i32 -1, i32 -64>
+  %t2_wide = zext <2 x i32> %t2 to <2 x i64>
+  %t3 = lshr <2 x i64> %y, %t2_wide
+  %t3_trunc = trunc <2 x i64> %t3 to <2 x i32>
+  %t4 = and <2 x i32> %t1, %t3_trunc
+  %t5 = icmp ne <2 x i32> %t4, <i32 0, i32 0>
+  ret <2 x i1> %t5
+}
+define <2 x i1> @n12_bad(<2 x i32> %x, <2 x i64> %y, <2 x i32> %len) {
+; CHECK-LABEL: @n12_bad(
+; CHECK-NEXT:    [[T0:%.*]] = sub <2 x i32> <i32 64, i32 64>, [[LEN:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = shl <2 x i32> [[X:%.*]], [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = add <2 x i32> [[LEN]], <i32 -2, i32 -64>
+; CHECK-NEXT:    [[T2_WIDE:%.*]] = zext <2 x i32> [[T2]] to <2 x i64>
+; CHECK-NEXT:    [[T3:%.*]] = lshr <2 x i64> [[Y:%.*]], [[T2_WIDE]]
+; CHECK-NEXT:    [[T3_TRUNC:%.*]] = trunc <2 x i64> [[T3]] to <2 x i32>
+; CHECK-NEXT:    [[T4:%.*]] = and <2 x i32> [[T1]], [[T3_TRUNC]]
+; CHECK-NEXT:    [[T5:%.*]] = icmp ne <2 x i32> [[T4]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[T5]]
+;
+  %t0 = sub <2 x i32> <i32 64, i32 64>, %len
+  %t1 = shl <2 x i32> %x, %t0
+  %t2 = add <2 x i32> %len, <i32 -2, i32 -64>
+  %t2_wide = zext <2 x i32> %t2 to <2 x i64>
+  %t3 = lshr <2 x i64> %y, %t2_wide
+  %t3_trunc = trunc <2 x i64> %t3 to <2 x i32>
+  %t4 = and <2 x i32> %t1, %t3_trunc
+  %t5 = icmp ne <2 x i32> %t4, <i32 0, i32 0>
+  ret <2 x i1> %t5
+}
+
+;------------------------------------------------------------------------------;
+
+; Ok if one of the values being shifted is 1
+define i1 @t13_x_is_one(i64 %y, i32 %len) {
+; CHECK-LABEL: @t13_x_is_one(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[Y:%.*]], 65536
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %t0 = sub i32 32, %len
+  %t1 = shl i32 1, %t0
+  %t2 = add i32 %len, -16
+  %t2_wide = zext i32 %t2 to i64
+  %t3 = lshr i64 %y, %t2_wide
+  %t3_trunc = trunc i64 %t3 to i32
+  %t4 = and i32 %t1, %t3_trunc
+  %t5 = icmp ne i32 %t4, 0
+  ret i1 %t5
+}
+define i1 @t14_x_is_one(i32 %x, i32 %len) {
+; CHECK-LABEL: @t14_x_is_one(
+; CHECK-NEXT:    ret i1 false
+;
+  %t0 = sub i32 32, %len
+  %t1 = shl i32 %x, %t0
+  %t2 = add i32 %len, -16
+  %t2_wide = zext i32 %t2 to i64
+  %t3 = lshr i64 1, %t2_wide
+  %t3_trunc = trunc i64 %t3 to i32
+  %t4 = and i32 %t1, %t3_trunc
+  %t5 = icmp ne i32 %t4, 0
+  ret i1 %t5
+}
+
+define <2 x i1> @t15_vec_x_is_one_or_zero(<2 x i64> %y, <2 x i32> %len) {
+; CHECK-LABEL: @t15_vec_x_is_one_or_zero(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <2 x i64> [[Y:%.*]], <i64 48, i64 48>
+; CHECK-NEXT:    [[TMP2:%.*]] = and <2 x i64> [[TMP1]], <i64 1, i64 0>
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <2 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[TMP3]]
+;
+  %t0 = sub <2 x i32> <i32 64, i32 64>, %len
+  %t1 = shl <2 x i32> <i32 1, i32 0>, %t0
+  %t2 = add <2 x i32> %len, <i32 -16, i32 -16>
+  %t2_wide = zext <2 x i32> %t2 to <2 x i64>
+  %t3 = lshr <2 x i64> %y, %t2_wide
+  %t3_trunc = trunc <2 x i64> %t3 to <2 x i32>
+  %t4 = and <2 x i32> %t1, %t3_trunc
+  %t5 = icmp ne <2 x i32> %t4, <i32 0, i32 0>
+  ret <2 x i1> %t5
+}
+define <2 x i1> @t16_vec_y_is_one_or_zero(<2 x i32> %x, <2 x i32> %len) {
+; CHECK-LABEL: @t16_vec_y_is_one_or_zero(
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+  %t0 = sub <2 x i32> <i32 64, i32 64>, %len
+  %t1 = shl <2 x i32> %x, %t0
+  %t2 = add <2 x i32> %len, <i32 -16, i32 -16>
+  %t2_wide = zext <2 x i32> %t2 to <2 x i64>
+  %t3 = lshr <2 x i64> <i64 1, i64 0>, %t2_wide
+  %t3_trunc = trunc <2 x i64> %t3 to <2 x i32>
+  %t4 = and <2 x i32> %t1, %t3_trunc
+  %t5 = icmp ne <2 x i32> %t4, <i32 0, i32 0>
+  ret <2 x i1> %t5
+}
+
+;------------------------------------------------------------------------------;
+
+; All other tests - extra uses, etc are already covered in
+; shift-amount-reassociation-in-bittest-with-truncation-shl.ll and
+; shift-amount-reassociation-in-bittest.ll
+
+; And that's the main motivational pattern:
+define i1 @rawspeed_signbit(i64 %storage, i32 %nbits) {
+; CHECK-LABEL: @rawspeed_signbit(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i64 [[STORAGE:%.*]], -1
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %skipnbits = sub nsw i32 64, %nbits
+  %skipnbitswide = zext i32 %skipnbits to i64
+  %datawide = lshr i64 %storage, %skipnbitswide
+  %data = trunc i64 %datawide to i32
+  %nbitsminusone = add nsw i32 %nbits, -1
+  %bitmask = shl i32 1, %nbitsminusone
+  %bitmasked = and i32 %bitmask, %data
+  %isbitunset = icmp eq i32 %bitmasked, 0
+  ret i1 %isbitunset
+}
diff --git a/llvm/test/Transforms/InstCombine/shift-amount-reassociation-in-bittest-with-truncation-shl.ll b/llvm/test/Transforms/InstCombine/shift-amount-reassociation-in-bittest-with-truncation-shl.ll
new file mode 100644
index 000000000..0c3ee4609
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/shift-amount-reassociation-in-bittest-with-truncation-shl.ll
@@ -0,0 +1,463 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instcombine -S | FileCheck %s
+
+; Given pattern:
+;   icmp eq/ne (and ((x shift Q), (y oppositeshift K))), 0
+; we should move shifts to the same hand of 'and', i.e. e.g. rewrite as
+;   icmp eq/ne (and (((x shift Q) shift K), y)), 0
+; We are only interested in opposite logical shifts here.
+; We still can handle the case where there is a truncation between a shift and
+; an 'and'. If it's trunc-of-shl - no extra legality check is needed.
+
+;-------------------------------------------------------------------------------
+; Basic scalar tests
+;-------------------------------------------------------------------------------
+
+define i1 @t0_const_after_fold_lshr_shl_ne(i32 %x, i64 %y, i32 %len) {
+; CHECK-LABEL: @t0_const_after_fold_lshr_shl_ne(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i32 [[X:%.*]], 31
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i32 [[TMP1]] to i64
+; CHECK-NEXT:    [[TMP3:%.*]] = and i64 [[TMP2]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne i64 [[TMP3]], 0
+; CHECK-NEXT:    ret i1 [[TMP4]]
+;
+  %t0 = sub i32 32, %len
+  %t1 = lshr i32 %x, %t0
+  %t2 = add i32 %len, -1
+  %t2_wide = zext i32 %t2 to i64
+  %t3 = shl i64 %y, %t2_wide
+  %t3_trunc = trunc i64 %t3 to i32
+  %t4 = and i32 %t1, %t3_trunc
+  %t5 = icmp ne i32 %t4, 0
+  ret i1 %t5
+}
+
+;-------------------------------------------------------------------------------
+; Very basic vector tests
+;-------------------------------------------------------------------------------
+
+define <2 x i1> @t1_vec_splat(<2 x i32> %x, <2 x i64> %y, <2 x i32> %len) {
+; CHECK-LABEL: @t1_vec_splat(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <2 x i32> [[X:%.*]], <i32 31, i32 31>
+; CHECK-NEXT:    [[TMP2:%.*]] = zext <2 x i32> [[TMP1]] to <2 x i64>
+; CHECK-NEXT:    [[TMP3:%.*]] = and <2 x i64> [[TMP2]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <2 x i64> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[TMP4]]
+;
+  %t0 = sub <2 x i32> <i32 32, i32 32>, %len
+  %t1 = lshr <2 x i32> %x, %t0
+  %t2 = add <2 x i32> %len, <i32 -1, i32 -1>
+  %t2_wide = zext <2 x i32> %t2 to <2 x i64>
+  %t3 = shl <2 x i64> %y, %t2_wide
+  %t3_trunc = trunc <2 x i64> %t3 to <2 x i32>
+  %t4 = and <2 x i32> %t1, %t3_trunc
+  %t5 = icmp ne <2 x i32> %t4, <i32 0, i32 0>
+  ret <2 x i1> %t5
+}
+
+define <2 x i1> @t2_vec_nonsplat(<2 x i32> %x, <2 x i64> %y, <2 x i32> %len) {
+; CHECK-LABEL: @t2_vec_nonsplat(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext <2 x i32> [[X:%.*]] to <2 x i64>
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr <2 x i64> [[TMP1]], <i64 31, i64 30>
+; CHECK-NEXT:    [[TMP3:%.*]] = and <2 x i64> [[TMP2]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <2 x i64> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[TMP4]]
+;
+  %t0 = sub <2 x i32> <i32 30, i32 32>, %len
+  %t1 = lshr <2 x i32> %x, %t0
+  %t2 = add <2 x i32> %len, <i32 1, i32 -2>
+  %t2_wide = zext <2 x i32> %t2 to <2 x i64>
+  %t3 = shl <2 x i64> %y, %t2_wide
+  %t3_trunc = trunc <2 x i64> %t3 to <2 x i32>
+  %t4 = and <2 x i32> %t1, %t3_trunc
+  %t5 = icmp ne <2 x i32> %t4, <i32 0, i32 0>
+  ret <2 x i1> %t5
+}
+
+;-------------------------------------------------------------------------------
+; Commutativity tests
+;-------------------------------------------------------------------------------
+
+declare i32 @gen32()
+declare i64 @gen64()
+
+; While 'and' is commutative, the 'trunc' *always* seems to be getting
+; canonicalized to the RHS, it does not seem possible to prevent that.
+
+;-------------------------------------------------------------------------------
+; One-use tests
+;-------------------------------------------------------------------------------
+
+declare void @use32(i32)
+declare void @use64(i64)
+
+; Nope, everything has extra uses.
+define i1 @t3_oneuse0(i32 %x, i64 %y, i32 %len) {
+; CHECK-LABEL: @t3_oneuse0(
+; CHECK-NEXT:    [[T0:%.*]] = sub i32 32, [[LEN:%.*]]
+; CHECK-NEXT:    call void @use32(i32 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = lshr i32 [[X:%.*]], [[T0]]
+; CHECK-NEXT:    call void @use32(i32 [[T1]])
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[LEN]], -1
+; CHECK-NEXT:    call void @use32(i32 [[T2]])
+; CHECK-NEXT:    [[T2_WIDE:%.*]] = zext i32 [[T2]] to i64
+; CHECK-NEXT:    call void @use64(i64 [[T2_WIDE]])
+; CHECK-NEXT:    [[T3:%.*]] = shl i64 [[Y:%.*]], [[T2_WIDE]]
+; CHECK-NEXT:    call void @use64(i64 [[T3]])
+; CHECK-NEXT:    [[T3_TRUNC:%.*]] = trunc i64 [[T3]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[T3_TRUNC]])
+; CHECK-NEXT:    [[T4:%.*]] = and i32 [[T1]], [[T3_TRUNC]]
+; CHECK-NEXT:    call void @use32(i32 [[T4]])
+; CHECK-NEXT:    [[T5:%.*]] = icmp ne i32 [[T4]], 0
+; CHECK-NEXT:    ret i1 [[T5]]
+;
+  %t0 = sub i32 32, %len
+  call void @use32(i32 %t0)
+  %t1 = lshr i32 %x, %t0
+  call void @use32(i32 %t1)
+  %t2 = add i32 %len, -1
+  call void @use32(i32 %t2)
+  %t2_wide = zext i32 %t2 to i64
+  call void @use64(i64 %t2_wide)
+  %t3 = shl i64 %y, %t2_wide
+  call void @use64(i64 %t3)
+  %t3_trunc = trunc i64 %t3 to i32
+  call void @use32(i32 %t3_trunc)
+  %t4 = and i32 %t1, %t3_trunc
+  call void @use32(i32 %t4)
+  %t5 = icmp ne i32 %t4, 0
+  ret i1 %t5
+}
+
+; Nope, still too much extra uses.
+define i1 @t4_oneuse1(i32 %x, i64 %y, i32 %len) {
+; CHECK-LABEL: @t4_oneuse1(
+; CHECK-NEXT:    [[T0:%.*]] = sub i32 32, [[LEN:%.*]]
+; CHECK-NEXT:    call void @use32(i32 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = lshr i32 [[X:%.*]], [[T0]]
+; CHECK-NEXT:    call void @use32(i32 [[T1]])
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[LEN]], -1
+; CHECK-NEXT:    call void @use32(i32 [[T2]])
+; CHECK-NEXT:    [[T2_WIDE:%.*]] = zext i32 [[T2]] to i64
+; CHECK-NEXT:    call void @use64(i64 [[T2_WIDE]])
+; CHECK-NEXT:    [[T3:%.*]] = shl i64 [[Y:%.*]], [[T2_WIDE]]
+; CHECK-NEXT:    call void @use64(i64 [[T3]])
+; CHECK-NEXT:    [[T3_TRUNC:%.*]] = trunc i64 [[T3]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[T3_TRUNC]])
+; CHECK-NEXT:    [[T4:%.*]] = and i32 [[T1]], [[T3_TRUNC]]
+; CHECK-NEXT:    [[T5:%.*]] = icmp ne i32 [[T4]], 0
+; CHECK-NEXT:    ret i1 [[T5]]
+;
+  %t0 = sub i32 32, %len
+  call void @use32(i32 %t0)
+  %t1 = lshr i32 %x, %t0
+  call void @use32(i32 %t1)
+  %t2 = add i32 %len, -1
+  call void @use32(i32 %t2)
+  %t2_wide = zext i32 %t2 to i64
+  call void @use64(i64 %t2_wide)
+  %t3 = shl i64 %y, %t2_wide
+  call void @use64(i64 %t3)
+  %t3_trunc = trunc i64 %t3 to i32
+  call void @use32(i32 %t3_trunc)
+  %t4 = and i32 %t1, %t3_trunc ; no extra uses
+  %t5 = icmp ne i32 %t4, 0
+  ret i1 %t5
+}
+
+; Still too much extra uses.
+define i1 @t5_oneuse2(i32 %x, i64 %y, i32 %len) {
+; CHECK-LABEL: @t5_oneuse2(
+; CHECK-NEXT:    [[T0:%.*]] = sub i32 32, [[LEN:%.*]]
+; CHECK-NEXT:    call void @use32(i32 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = lshr i32 [[X:%.*]], [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[LEN]], -1
+; CHECK-NEXT:    call void @use32(i32 [[T2]])
+; CHECK-NEXT:    [[T2_WIDE:%.*]] = zext i32 [[T2]] to i64
+; CHECK-NEXT:    call void @use64(i64 [[T2_WIDE]])
+; CHECK-NEXT:    [[T3:%.*]] = shl i64 [[Y:%.*]], [[T2_WIDE]]
+; CHECK-NEXT:    call void @use64(i64 [[T3]])
+; CHECK-NEXT:    [[T3_TRUNC:%.*]] = trunc i64 [[T3]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[T3_TRUNC]])
+; CHECK-NEXT:    [[T4:%.*]] = and i32 [[T1]], [[T3_TRUNC]]
+; CHECK-NEXT:    [[T5:%.*]] = icmp ne i32 [[T4]], 0
+; CHECK-NEXT:    ret i1 [[T5]]
+;
+  %t0 = sub i32 32, %len
+  call void @use32(i32 %t0)
+  %t1 = lshr i32 %x, %t0 ; no extra uses
+  %t2 = add i32 %len, -1
+  call void @use32(i32 %t2)
+  %t2_wide = zext i32 %t2 to i64
+  call void @use64(i64 %t2_wide)
+  %t3 = shl i64 %y, %t2_wide
+  call void @use64(i64 %t3)
+  %t3_trunc = trunc i64 %t3 to i32
+  call void @use32(i32 %t3_trunc)
+  %t4 = and i32 %t1, %t3_trunc ; no extra uses
+  %t5 = icmp ne i32 %t4, 0
+  ret i1 %t5
+}
+
+; Ok, trunc has no extra uses.
+define i1 @t6_oneuse3(i32 %x, i64 %y, i32 %len) {
+; CHECK-LABEL: @t6_oneuse3(
+; CHECK-NEXT:    [[T0:%.*]] = sub i32 32, [[LEN:%.*]]
+; CHECK-NEXT:    call void @use32(i32 [[T0]])
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[LEN]], -1
+; CHECK-NEXT:    call void @use32(i32 [[T2]])
+; CHECK-NEXT:    [[T2_WIDE:%.*]] = zext i32 [[T2]] to i64
+; CHECK-NEXT:    call void @use64(i64 [[T2_WIDE]])
+; CHECK-NEXT:    [[T3:%.*]] = shl i64 [[Y:%.*]], [[T2_WIDE]]
+; CHECK-NEXT:    call void @use64(i64 [[T3]])
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i32 [[X:%.*]], 31
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i32 [[TMP1]] to i64
+; CHECK-NEXT:    [[TMP3:%.*]] = and i64 [[TMP2]], [[Y]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne i64 [[TMP3]], 0
+; CHECK-NEXT:    ret i1 [[TMP4]]
+;
+  %t0 = sub i32 32, %len
+  call void @use32(i32 %t0)
+  %t1 = lshr i32 %x, %t0 ; no extra uses
+  %t2 = add i32 %len, -1
+  call void @use32(i32 %t2)
+  %t2_wide = zext i32 %t2 to i64
+  call void @use64(i64 %t2_wide)
+  %t3 = shl i64 %y, %t2_wide
+  call void @use64(i64 %t3)
+  %t3_trunc = trunc i64 %t3 to i32 ; no extra uses
+  %t4 = and i32 %t1, %t3_trunc ; no extra uses
+  %t5 = icmp ne i32 %t4, 0
+  ret i1 %t5
+}
+
+; Ok, shift amount of non-truncated shift has no extra uses;
+define i1 @t7_oneuse4(i32 %x, i64 %y, i32 %len) {
+; CHECK-LABEL: @t7_oneuse4(
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[LEN:%.*]], -1
+; CHECK-NEXT:    call void @use32(i32 [[T2]])
+; CHECK-NEXT:    [[T2_WIDE:%.*]] = zext i32 [[T2]] to i64
+; CHECK-NEXT:    call void @use64(i64 [[T2_WIDE]])
+; CHECK-NEXT:    [[T3:%.*]] = shl i64 [[Y:%.*]], [[T2_WIDE]]
+; CHECK-NEXT:    call void @use64(i64 [[T3]])
+; CHECK-NEXT:    [[T3_TRUNC:%.*]] = trunc i64 [[T3]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[T3_TRUNC]])
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i32 [[X:%.*]], 31
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i32 [[TMP1]] to i64
+; CHECK-NEXT:    [[TMP3:%.*]] = and i64 [[TMP2]], [[Y]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne i64 [[TMP3]], 0
+; CHECK-NEXT:    ret i1 [[TMP4]]
+;
+  %t0 = sub i32 32, %len ; no extra uses
+  %t1 = lshr i32 %x, %t0 ; no extra uses
+  %t2 = add i32 %len, -1
+  call void @use32(i32 %t2)
+  %t2_wide = zext i32 %t2 to i64
+  call void @use64(i64 %t2_wide)
+  %t3 = shl i64 %y, %t2_wide
+  call void @use64(i64 %t3)
+  %t3_trunc = trunc i64 %t3 to i32
+  call void @use32(i32 %t3_trunc)
+  %t4 = and i32 %t1, %t3_trunc ; no extra uses
+  %t5 = icmp ne i32 %t4, 0
+  ret i1 %t5
+}
+
+; Ok, non-truncated shift is of constant;
+define i1 @t8_oneuse5(i32 %x, i64 %y, i32 %len) {
+; CHECK-LABEL: @t8_oneuse5(
+; CHECK-NEXT:    [[T0:%.*]] = sub i32 32, [[LEN:%.*]]
+; CHECK-NEXT:    call void @use32(i32 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = lshr i32 -52543054, [[T0]]
+; CHECK-NEXT:    call void @use32(i32 [[T1]])
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[LEN]], -1
+; CHECK-NEXT:    call void @use32(i32 [[T2]])
+; CHECK-NEXT:    [[T2_WIDE:%.*]] = zext i32 [[T2]] to i64
+; CHECK-NEXT:    call void @use64(i64 [[T2_WIDE]])
+; CHECK-NEXT:    [[T3:%.*]] = shl i64 [[Y:%.*]], [[T2_WIDE]]
+; CHECK-NEXT:    call void @use64(i64 [[T3]])
+; CHECK-NEXT:    [[T3_TRUNC:%.*]] = trunc i64 [[T3]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[T3_TRUNC]])
+; CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[Y]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %t0 = sub i32 32, %len
+  call void @use32(i32 %t0)
+  %t1 = lshr i32 4242424242, %t0 ; shift-of-constant
+  call void @use32(i32 %t1)
+  %t2 = add i32 %len, -1
+  call void @use32(i32 %t2)
+  %t2_wide = zext i32 %t2 to i64
+  call void @use64(i64 %t2_wide)
+  %t3 = shl i64 %y, %t2_wide
+  call void @use64(i64 %t3)
+  %t3_trunc = trunc i64 %t3 to i32
+  call void @use32(i32 %t3_trunc)
+  %t4 = and i32 %t1, %t3_trunc ; no extra uses
+  %t5 = icmp ne i32 %t4, 0
+  ret i1 %t5
+}
+
+; Ok, truncated shift is of constant;
+define i1 @t9_oneuse5(i32 %x, i64 %y, i32 %len) {
+; CHECK-LABEL: @t9_oneuse5(
+; CHECK-NEXT:    [[T0:%.*]] = sub i32 32, [[LEN:%.*]]
+; CHECK-NEXT:    call void @use32(i32 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = lshr i32 [[X:%.*]], [[T0]]
+; CHECK-NEXT:    call void @use32(i32 [[T1]])
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[LEN]], -1
+; CHECK-NEXT:    call void @use32(i32 [[T2]])
+; CHECK-NEXT:    [[T2_WIDE:%.*]] = zext i32 [[T2]] to i64
+; CHECK-NEXT:    call void @use64(i64 [[T2_WIDE]])
+; CHECK-NEXT:    [[T3:%.*]] = shl i64 4242424242, [[T2_WIDE]]
+; CHECK-NEXT:    call void @use64(i64 [[T3]])
+; CHECK-NEXT:    [[T3_TRUNC:%.*]] = trunc i64 [[T3]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[T3_TRUNC]])
+; CHECK-NEXT:    ret i1 false
+;
+  %t0 = sub i32 32, %len
+  call void @use32(i32 %t0)
+  %t1 = lshr i32 %x, %t0 ; shift-of-constant
+  call void @use32(i32 %t1)
+  %t2 = add i32 %len, -1
+  call void @use32(i32 %t2)
+  %t2_wide = zext i32 %t2 to i64
+  call void @use64(i64 %t2_wide)
+  %t3 = shl i64 4242424242, %t2_wide
+  call void @use64(i64 %t3)
+  %t3_trunc = trunc i64 %t3 to i32
+  call void @use32(i32 %t3_trunc)
+  %t4 = and i32 %t1, %t3_trunc ; no extra uses
+  %t5 = icmp ne i32 %t4, 0
+  ret i1 %t5
+}
+
+;-------------------------------------------------------------------------------
+; Commutativity with extra uses
+;-------------------------------------------------------------------------------
+
+; While 'and' is commutative, the 'trunc' *always* seems to be getting
+; canonicalized to the RHS, it does not seem possible to prevent that.
+
+;
+
+; Constant shift amounts
+
+define i1 @t10_constants(i32 %x, i64 %y) {
+; CHECK-LABEL: @t10_constants(
+; CHECK-NEXT:    [[Y_TR:%.*]] = trunc i64 [[Y:%.*]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i32 [[X:%.*]], 26
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], [[Y_TR]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne i32 [[TMP2]], 0
+; CHECK-NEXT:    ret i1 [[TMP3]]
+;
+  %t0 = lshr i32 %x, 12
+  %t1 = shl i64 %y, 14
+  %t1_trunc = trunc i64 %t1 to i32
+  %t2 = and i32 %t0, %t1_trunc
+  %t3 = icmp ne i32 %t2, 0
+  ret i1 %t3
+}
+
+define <2 x i1> @t11_constants_vec_splat(<2 x i32> %x, <2 x i64> %y) {
+; CHECK-LABEL: @t11_constants_vec_splat(
+; CHECK-NEXT:    [[T0:%.*]] = lshr <2 x i32> [[X:%.*]], <i32 12, i32 12>
+; CHECK-NEXT:    [[T1:%.*]] = shl <2 x i64> [[Y:%.*]], <i64 14, i64 14>
+; CHECK-NEXT:    [[T1_TRUNC:%.*]] = trunc <2 x i64> [[T1]] to <2 x i32>
+; CHECK-NEXT:    [[T2:%.*]] = and <2 x i32> [[T0]], [[T1_TRUNC]]
+; CHECK-NEXT:    [[T3:%.*]] = icmp ne <2 x i32> [[T2]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[T3]]
+;
+  %t0 = lshr <2 x i32> %x, <i32 12, i32 12>
+  %t1 = shl <2 x i64> %y, <i64 14, i64 14>
+  %t1_trunc = trunc <2 x i64> %t1 to <2 x i32>
+  %t2 = and <2 x i32> %t0, %t1_trunc
+  %t3 = icmp ne <2 x i32> %t2, <i32 0, i32 0>
+  ret <2 x i1> %t3
+}
+define <2 x i1> @t12_constants_vec_nonsplat(<2 x i32> %x, <2 x i64> %y) {
+; CHECK-LABEL: @t12_constants_vec_nonsplat(
+; CHECK-NEXT:    [[T0:%.*]] = lshr <2 x i32> [[X:%.*]], <i32 12, i32 14>
+; CHECK-NEXT:    [[T1:%.*]] = shl <2 x i64> [[Y:%.*]], <i64 16, i64 14>
+; CHECK-NEXT:    [[T1_TRUNC:%.*]] = trunc <2 x i64> [[T1]] to <2 x i32>
+; CHECK-NEXT:    [[T2:%.*]] = and <2 x i32> [[T0]], [[T1_TRUNC]]
+; CHECK-NEXT:    [[T3:%.*]] = icmp ne <2 x i32> [[T2]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[T3]]
+;
+  %t0 = lshr <2 x i32> %x, <i32 12, i32 14>
+  %t1 = shl <2 x i64> %y, <i64 16, i64 14>
+  %t1_trunc = trunc <2 x i64> %t1 to <2 x i32>
+  %t2 = and <2 x i32> %t0, %t1_trunc
+  %t3 = icmp ne <2 x i32> %t2, <i32 0, i32 0>
+  ret <2 x i1> %t3
+}
+
+;-------------------------------------------------------------------------------
+; Negative tests
+;-------------------------------------------------------------------------------
+
+define i1 @n13_overshift(i32 %x, i64 %y, i32 %len) {
+; CHECK-LABEL: @n13_overshift(
+; CHECK-NEXT:    [[T0:%.*]] = sub i32 32, [[LEN:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = lshr i32 [[X:%.*]], [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[LEN]], 32
+; CHECK-NEXT:    [[T2_WIDE:%.*]] = zext i32 [[T2]] to i64
+; CHECK-NEXT:    [[T3:%.*]] = shl i64 [[Y:%.*]], [[T2_WIDE]]
+; CHECK-NEXT:    [[T3_TRUNC:%.*]] = trunc i64 [[T3]] to i32
+; CHECK-NEXT:    [[T4:%.*]] = and i32 [[T1]], [[T3_TRUNC]]
+; CHECK-NEXT:    [[T5:%.*]] = icmp ne i32 [[T4]], 0
+; CHECK-NEXT:    ret i1 [[T5]]
+;
+  %t0 = sub i32 32, %len
+  %t1 = lshr i32 %x, %t0
+  %t2 = add i32 %len, 32 ; too much
+  %t2_wide = zext i32 %t2 to i64
+  %t3 = shl i64 %y, %t2_wide
+  %t3_trunc = trunc i64 %t3 to i32
+  %t4 = and i32 %t1, %t3_trunc
+  %t5 = icmp ne i32 %t4, 0
+  ret i1 %t5
+}
+
+define i1 @n14_trunc_of_lshr(i64 %x, i32 %y, i32 %len) {
+; CHECK-LABEL: @n14_trunc_of_lshr(
+; CHECK-NEXT:    [[T0:%.*]] = sub i32 32, [[LEN:%.*]]
+; CHECK-NEXT:    [[T0_WIDE:%.*]] = zext i32 [[T0]] to i64
+; CHECK-NEXT:    [[T1:%.*]] = lshr i64 [[X:%.*]], [[T0_WIDE]]
+; CHECK-NEXT:    [[T1_TRUNC:%.*]] = trunc i64 [[T1]] to i32
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[LEN]], -1
+; CHECK-NEXT:    [[T3:%.*]] = shl i32 [[Y:%.*]], [[T2]]
+; CHECK-NEXT:    [[T4:%.*]] = and i32 [[T3]], [[T1_TRUNC]]
+; CHECK-NEXT:    [[T5:%.*]] = icmp ne i32 [[T4]], 0
+; CHECK-NEXT:    ret i1 [[T5]]
+;
+  %t0 = sub i32 32, %len
+  %t0_wide = zext i32 %t0 to i64
+  %t1 = lshr i64 %x, %t0_wide
+  %t1_trunc = trunc i64 %t1 to i32
+  %t2 = add i32 %len, -1
+  %t3 = shl i32 %y, %t2
+  %t4 = and i32 %t1_trunc, %t3
+  %t5 = icmp ne i32 %t4, 0
+  ret i1 %t5
+}
+
+; Completely variable shift amounts
+
+define i1 @n15_variable_shamts(i32 %x, i64 %y, i32 %shamt0, i64 %shamt1) {
+; CHECK-LABEL: @n15_variable_shamts(
+; CHECK-NEXT:    [[T0:%.*]] = lshr i32 [[X:%.*]], [[SHAMT0:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = shl i64 [[Y:%.*]], [[SHAMT1:%.*]]
+; CHECK-NEXT:    [[T1_TRUNC:%.*]] = trunc i64 [[T1]] to i32
+; CHECK-NEXT:    [[T2:%.*]] = and i32 [[T0]], [[T1_TRUNC]]
+; CHECK-NEXT:    [[T3:%.*]] = icmp ne i32 [[T2]], 0
+; CHECK-NEXT:    ret i1 [[T3]]
+;
+  %t0 = lshr i32 %x, %shamt0
+  %t1 = shl i64 %y, %shamt1
+  %t1_trunc = trunc i64 %t1 to i32
+  %t2 = and i32 %t1_trunc, %t0
+  %t3 = icmp ne i32 %t2, 0
+  ret i1 %t3
+}
diff --git a/llvm/test/Transforms/InstCombine/shift-amount-reassociation-in-bittest.ll b/llvm/test/Transforms/InstCombine/shift-amount-reassociation-in-bittest.ll
index 9a2cfa597..97506e193 100644
--- a/llvm/test/Transforms/InstCombine/shift-amount-reassociation-in-bittest.ll
+++ b/llvm/test/Transforms/InstCombine/shift-amount-reassociation-in-bittest.ll
@@ -671,6 +671,14 @@ define <2 x i1> @n38_overshift(<2 x i32> %x, <2 x i32> %y) {
 ; As usual, don't crash given constantexpr's :/
 @f.a = internal global i16 0
 define i1 @constantexpr() {
+; CHECK-LABEL: @constantexpr(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i16, i16* @f.a, align 2
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i16 [[TMP0]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = and i16 [[TMP1]], shl (i16 1, i16 zext (i1 icmp ne (i16 ptrtoint (i16* @f.a to i16), i16 1) to i16))
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne i16 [[TMP2]], 0
+; CHECK-NEXT:    ret i1 [[TOBOOL]]
+;
 entry:
   %0 = load i16, i16* @f.a
   %shr = ashr i16 %0, 1
@@ -679,3 +687,23 @@ entry:
   %tobool = icmp ne i16 %and, 0
   ret i1 %tobool
 }
+
+; See https://bugs.llvm.org/show_bug.cgi?id=44802
+define i1 @pr44802(i3 %a, i3 %x, i3 %y) {
+; CHECK-LABEL: @pr44802(
+; CHECK-NEXT:    [[T0:%.*]] = icmp ne i3 [[A:%.*]], 0
+; CHECK-NEXT:    [[T1:%.*]] = zext i1 [[T0]] to i3
+; CHECK-NEXT:    [[T2:%.*]] = lshr i3 [[X:%.*]], [[T1]]
+; CHECK-NEXT:    [[T3:%.*]] = shl i3 [[Y:%.*]], [[T1]]
+; CHECK-NEXT:    [[T4:%.*]] = and i3 [[T2]], [[T3]]
+; CHECK-NEXT:    [[T5:%.*]] = icmp ne i3 [[T4]], 0
+; CHECK-NEXT:    ret i1 [[T5]]
+;
+  %t0 = icmp ne i3 %a, 0
+  %t1 = zext i1 %t0 to i3
+  %t2 = lshr i3 %x, %t1
+  %t3 = shl i3 %y, %t1
+  %t4 = and i3 %t2, %t3
+  %t5 = icmp ne i3 %t4, 0
+  ret i1 %t5
+}
diff --git a/llvm/test/Transforms/InstCombine/shift-amount-reassociation-with-truncation-ashr.ll b/llvm/test/Transforms/InstCombine/shift-amount-reassociation-with-truncation-ashr.ll
new file mode 100644
index 000000000..c4e337548
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/shift-amount-reassociation-with-truncation-ashr.ll
@@ -0,0 +1,200 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instcombine -S | FileCheck %s
+
+; Given pattern:
+;   (trunc (iSrc x a>> Q) to iDst) a>> K
+; we should rewrite it as
+;   (trunc (iSrc x a>> (Q+K)) to iDst)
+; iff (Q+K) is bitwidth(iSrc)-1
+; THIS FOLD DOES *NOT* REQUIRE ANY 'nuw'/`nsw` FLAGS!
+
+; Basic scalar test
+
+define i16 @t0(i32 %x, i16 %y) {
+; CHECK-LABEL: @t0(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr i32 [[X:%.*]], 31
+; CHECK-NEXT:    [[T5:%.*]] = trunc i32 [[TMP1]] to i16
+; CHECK-NEXT:    ret i16 [[T5]]
+;
+  %t0 = sub i16 32, %y
+  %t1 = zext i16 %t0 to i32
+  %t2 = ashr i32 %x, %t1
+  %t3 = trunc i32 %t2 to i16
+  %t4 = add i16 %y, -1
+  %t5 = ashr i16 %t3, %t4
+  ret i16 %t5
+}
+
+; Basic vector tests
+
+define <2 x i16> @t1_vec_splat(<2 x i32> %x, <2 x i16> %y) {
+; CHECK-LABEL: @t1_vec_splat(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <2 x i32> [[X:%.*]], <i32 31, i32 31>
+; CHECK-NEXT:    [[T5:%.*]] = trunc <2 x i32> [[TMP1]] to <2 x i16>
+; CHECK-NEXT:    ret <2 x i16> [[T5]]
+;
+  %t0 = sub <2 x i16> <i16 32, i16 32>, %y
+  %t1 = zext <2 x i16> %t0 to <2 x i32>
+  %t2 = ashr <2 x i32> %x, %t1
+  %t3 = trunc <2 x i32> %t2 to <2 x i16>
+  %t4 = add <2 x i16> %y, <i16 -1, i16 -1>
+  %t5 = ashr <2 x i16> %t3, %t4
+  ret <2 x i16> %t5
+}
+
+define <3 x i16> @t3_vec_nonsplat_undef0(<3 x i32> %x, <3 x i16> %y) {
+; CHECK-LABEL: @t3_vec_nonsplat_undef0(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <3 x i32> [[X:%.*]], <i32 31, i32 0, i32 31>
+; CHECK-NEXT:    [[T5:%.*]] = trunc <3 x i32> [[TMP1]] to <3 x i16>
+; CHECK-NEXT:    ret <3 x i16> [[T5]]
+;
+  %t0 = sub <3 x i16> <i16 32, i16 undef, i16 32>, %y
+  %t1 = zext <3 x i16> %t0 to <3 x i32>
+  %t2 = ashr <3 x i32> %x, %t1
+  %t3 = trunc <3 x i32> %t2 to <3 x i16>
+  %t4 = add <3 x i16> %y, <i16 -1, i16 -1, i16 -1>
+  %t5 = ashr <3 x i16> %t3, %t4
+  ret <3 x i16> %t5
+}
+
+define <3 x i16> @t4_vec_nonsplat_undef1(<3 x i32> %x, <3 x i16> %y) {
+; CHECK-LABEL: @t4_vec_nonsplat_undef1(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <3 x i32> [[X:%.*]], <i32 31, i32 0, i32 31>
+; CHECK-NEXT:    [[T5:%.*]] = trunc <3 x i32> [[TMP1]] to <3 x i16>
+; CHECK-NEXT:    ret <3 x i16> [[T5]]
+;
+  %t0 = sub <3 x i16> <i16 32, i16 32, i16 32>, %y
+  %t1 = zext <3 x i16> %t0 to <3 x i32>
+  %t2 = ashr <3 x i32> %x, %t1
+  %t3 = trunc <3 x i32> %t2 to <3 x i16>
+  %t4 = add <3 x i16> %y, <i16 -1, i16 undef, i16 -1>
+  %t5 = ashr <3 x i16> %t3, %t4
+  ret <3 x i16> %t5
+}
+
+define <3 x i16> @t5_vec_nonsplat_undef1(<3 x i32> %x, <3 x i16> %y) {
+; CHECK-LABEL: @t5_vec_nonsplat_undef1(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <3 x i32> [[X:%.*]], <i32 31, i32 0, i32 31>
+; CHECK-NEXT:    [[T5:%.*]] = trunc <3 x i32> [[TMP1]] to <3 x i16>
+; CHECK-NEXT:    ret <3 x i16> [[T5]]
+;
+  %t0 = sub <3 x i16> <i16 32, i16 undef, i16 32>, %y
+  %t1 = zext <3 x i16> %t0 to <3 x i32>
+  %t2 = ashr <3 x i32> %x, %t1
+  %t3 = trunc <3 x i32> %t2 to <3 x i16>
+  %t4 = add <3 x i16> %y, <i16 -1, i16 undef, i16 -1>
+  %t5 = ashr <3 x i16> %t3, %t4
+  ret <3 x i16> %t5
+}
+
+; One-use tests
+
+declare void @use16(i16)
+declare void @use32(i32)
+
+define i16 @t6_extrause0(i32 %x, i16 %y) {
+; CHECK-LABEL: @t6_extrause0(
+; CHECK-NEXT:    [[T0:%.*]] = sub i16 32, [[Y:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = zext i16 [[T0]] to i32
+; CHECK-NEXT:    [[T2:%.*]] = ashr i32 [[X:%.*]], [[T1]]
+; CHECK-NEXT:    [[T3:%.*]] = trunc i32 [[T2]] to i16
+; CHECK-NEXT:    call void @use16(i16 [[T3]])
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr i32 [[X]], 31
+; CHECK-NEXT:    [[T5:%.*]] = trunc i32 [[TMP1]] to i16
+; CHECK-NEXT:    ret i16 [[T5]]
+;
+  %t0 = sub i16 32, %y
+  %t1 = zext i16 %t0 to i32
+  %t2 = ashr i32 %x, %t1
+  %t3 = trunc i32 %t2 to i16
+  %t4 = add i16 %y, -1
+  call void @use16(i16 %t3)
+  %t5 = ashr i16 %t3, %t4
+  ret i16 %t5
+}
+
+define i16 @t7_extrause1(i32 %x, i16 %y) {
+; CHECK-LABEL: @t7_extrause1(
+; CHECK-NEXT:    [[T4:%.*]] = add i16 [[Y:%.*]], -1
+; CHECK-NEXT:    call void @use16(i16 [[T4]])
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr i32 [[X:%.*]], 31
+; CHECK-NEXT:    [[T5:%.*]] = trunc i32 [[TMP1]] to i16
+; CHECK-NEXT:    ret i16 [[T5]]
+;
+  %t0 = sub i16 32, %y
+  %t1 = zext i16 %t0 to i32
+  %t2 = ashr i32 %x, %t1
+  %t3 = trunc i32 %t2 to i16
+  %t4 = add i16 %y, -1
+  call void @use16(i16 %t4)
+  %t5 = ashr i16 %t3, %t4
+  ret i16 %t5
+}
+
+define i16 @t8_extrause2(i32 %x, i16 %y) {
+; CHECK-LABEL: @t8_extrause2(
+; CHECK-NEXT:    [[T0:%.*]] = sub i16 32, [[Y:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = zext i16 [[T0]] to i32
+; CHECK-NEXT:    [[T2:%.*]] = ashr i32 [[X:%.*]], [[T1]]
+; CHECK-NEXT:    [[T3:%.*]] = trunc i32 [[T2]] to i16
+; CHECK-NEXT:    [[T4:%.*]] = add i16 [[Y]], -1
+; CHECK-NEXT:    call void @use16(i16 [[T3]])
+; CHECK-NEXT:    call void @use16(i16 [[T4]])
+; CHECK-NEXT:    [[T5:%.*]] = ashr i16 [[T3]], [[T4]]
+; CHECK-NEXT:    ret i16 [[T5]]
+;
+  %t0 = sub i16 32, %y
+  %t1 = zext i16 %t0 to i32
+  %t2 = ashr i32 %x, %t1
+  %t3 = trunc i32 %t2 to i16
+  %t4 = add i16 %y, -1
+  call void @use16(i16 %t3)
+  call void @use16(i16 %t4)
+  %t5 = ashr i16 %t3, %t4
+  ret i16 %t5
+}
+
+; No 'nuw'/'nsw' flags are to be propagated!
+; But we can't test that, such IR does not reach that code.
+
+; Negative tests
+
+; Can only fold if we are extracting the sign bit.
+define i16 @t9_ashr(i32 %x, i16 %y) {
+; CHECK-LABEL: @t9_ashr(
+; CHECK-NEXT:    [[T0:%.*]] = sub i16 32, [[Y:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = zext i16 [[T0]] to i32
+; CHECK-NEXT:    [[T2:%.*]] = ashr i32 [[X:%.*]], [[T1]]
+; CHECK-NEXT:    [[T3:%.*]] = trunc i32 [[T2]] to i16
+; CHECK-NEXT:    [[T4:%.*]] = add i16 [[Y]], -2
+; CHECK-NEXT:    [[T5:%.*]] = ashr i16 [[T3]], [[T4]]
+; CHECK-NEXT:    ret i16 [[T5]]
+;
+  %t0 = sub i16 32, %y
+  %t1 = zext i16 %t0 to i32
+  %t2 = ashr i32 %x, %t1
+  %t3 = trunc i32 %t2 to i16
+  %t4 = add i16 %y, -2
+  %t5 = ashr i16 %t3, %t4
+  ret i16 %t5
+}
+
+; If we have different right-shifts, in general, we can't do anything with it.
+define i16 @n10_lshr_ashr(i32 %x, i16 %y) {
+; CHECK-LABEL: @n10_lshr_ashr(
+; CHECK-NEXT:    [[T0:%.*]] = sub i16 32, [[Y:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = zext i16 [[T0]] to i32
+; CHECK-NEXT:    [[T2:%.*]] = lshr i32 [[X:%.*]], [[T1]]
+; CHECK-NEXT:    [[T3:%.*]] = trunc i32 [[T2]] to i16
+; CHECK-NEXT:    [[T4:%.*]] = add i16 [[Y]], -1
+; CHECK-NEXT:    [[T5:%.*]] = ashr i16 [[T3]], [[T4]]
+; CHECK-NEXT:    ret i16 [[T5]]
+;
+  %t0 = sub i16 32, %y
+  %t1 = zext i16 %t0 to i32
+  %t2 = lshr i32 %x, %t1
+  %t3 = trunc i32 %t2 to i16
+  %t4 = add i16 %y, -1
+  %t5 = ashr i16 %t3, %t4
+  ret i16 %t5
+}
diff --git a/llvm/test/Transforms/InstCombine/shift-amount-reassociation-with-truncation-lshr.ll b/llvm/test/Transforms/InstCombine/shift-amount-reassociation-with-truncation-lshr.ll
new file mode 100644
index 000000000..edb34fbb9
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/shift-amount-reassociation-with-truncation-lshr.ll
@@ -0,0 +1,200 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instcombine -S | FileCheck %s
+
+; Given pattern:
+;   (trunc (iSrc x l>> Q) to iDst) l>> K
+; we should rewrite it as
+;   (trunc (iSrc x l>> (Q+K)) to iDst)
+; iff (Q+K) is bitwidth(iSrc)-1
+; THIS FOLD DOES *NOT* REQUIRE ANY 'nuw'/`nsw` FLAGS!
+
+; Basic scalar test
+
+define i16 @t0(i32 %x, i16 %y) {
+; CHECK-LABEL: @t0(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i32 [[X:%.*]], 31
+; CHECK-NEXT:    [[T5:%.*]] = trunc i32 [[TMP1]] to i16
+; CHECK-NEXT:    ret i16 [[T5]]
+;
+  %t0 = sub i16 32, %y
+  %t1 = zext i16 %t0 to i32
+  %t2 = lshr i32 %x, %t1
+  %t3 = trunc i32 %t2 to i16
+  %t4 = add i16 %y, -1
+  %t5 = lshr i16 %t3, %t4
+  ret i16 %t5
+}
+
+; Basic vector tests
+
+define <2 x i16> @t1_vec_splat(<2 x i32> %x, <2 x i16> %y) {
+; CHECK-LABEL: @t1_vec_splat(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <2 x i32> [[X:%.*]], <i32 31, i32 31>
+; CHECK-NEXT:    [[T5:%.*]] = trunc <2 x i32> [[TMP1]] to <2 x i16>
+; CHECK-NEXT:    ret <2 x i16> [[T5]]
+;
+  %t0 = sub <2 x i16> <i16 32, i16 32>, %y
+  %t1 = zext <2 x i16> %t0 to <2 x i32>
+  %t2 = lshr <2 x i32> %x, %t1
+  %t3 = trunc <2 x i32> %t2 to <2 x i16>
+  %t4 = add <2 x i16> %y, <i16 -1, i16 -1>
+  %t5 = lshr <2 x i16> %t3, %t4
+  ret <2 x i16> %t5
+}
+
+define <3 x i16> @t3_vec_nonsplat_undef0(<3 x i32> %x, <3 x i16> %y) {
+; CHECK-LABEL: @t3_vec_nonsplat_undef0(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <3 x i32> [[X:%.*]], <i32 31, i32 0, i32 31>
+; CHECK-NEXT:    [[T5:%.*]] = trunc <3 x i32> [[TMP1]] to <3 x i16>
+; CHECK-NEXT:    ret <3 x i16> [[T5]]
+;
+  %t0 = sub <3 x i16> <i16 32, i16 undef, i16 32>, %y
+  %t1 = zext <3 x i16> %t0 to <3 x i32>
+  %t2 = lshr <3 x i32> %x, %t1
+  %t3 = trunc <3 x i32> %t2 to <3 x i16>
+  %t4 = add <3 x i16> %y, <i16 -1, i16 -1, i16 -1>
+  %t5 = lshr <3 x i16> %t3, %t4
+  ret <3 x i16> %t5
+}
+
+define <3 x i16> @t4_vec_nonsplat_undef1(<3 x i32> %x, <3 x i16> %y) {
+; CHECK-LABEL: @t4_vec_nonsplat_undef1(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <3 x i32> [[X:%.*]], <i32 31, i32 0, i32 31>
+; CHECK-NEXT:    [[T5:%.*]] = trunc <3 x i32> [[TMP1]] to <3 x i16>
+; CHECK-NEXT:    ret <3 x i16> [[T5]]
+;
+  %t0 = sub <3 x i16> <i16 32, i16 32, i16 32>, %y
+  %t1 = zext <3 x i16> %t0 to <3 x i32>
+  %t2 = lshr <3 x i32> %x, %t1
+  %t3 = trunc <3 x i32> %t2 to <3 x i16>
+  %t4 = add <3 x i16> %y, <i16 -1, i16 undef, i16 -1>
+  %t5 = lshr <3 x i16> %t3, %t4
+  ret <3 x i16> %t5
+}
+
+define <3 x i16> @t5_vec_nonsplat_undef1(<3 x i32> %x, <3 x i16> %y) {
+; CHECK-LABEL: @t5_vec_nonsplat_undef1(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <3 x i32> [[X:%.*]], <i32 31, i32 0, i32 31>
+; CHECK-NEXT:    [[T5:%.*]] = trunc <3 x i32> [[TMP1]] to <3 x i16>
+; CHECK-NEXT:    ret <3 x i16> [[T5]]
+;
+  %t0 = sub <3 x i16> <i16 32, i16 undef, i16 32>, %y
+  %t1 = zext <3 x i16> %t0 to <3 x i32>
+  %t2 = lshr <3 x i32> %x, %t1
+  %t3 = trunc <3 x i32> %t2 to <3 x i16>
+  %t4 = add <3 x i16> %y, <i16 -1, i16 undef, i16 -1>
+  %t5 = lshr <3 x i16> %t3, %t4
+  ret <3 x i16> %t5
+}
+
+; One-use tests
+
+declare void @use16(i16)
+declare void @use32(i32)
+
+define i16 @t6_extrause0(i32 %x, i16 %y) {
+; CHECK-LABEL: @t6_extrause0(
+; CHECK-NEXT:    [[T0:%.*]] = sub i16 32, [[Y:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = zext i16 [[T0]] to i32
+; CHECK-NEXT:    [[T2:%.*]] = lshr i32 [[X:%.*]], [[T1]]
+; CHECK-NEXT:    [[T3:%.*]] = trunc i32 [[T2]] to i16
+; CHECK-NEXT:    call void @use16(i16 [[T3]])
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i32 [[X]], 31
+; CHECK-NEXT:    [[T5:%.*]] = trunc i32 [[TMP1]] to i16
+; CHECK-NEXT:    ret i16 [[T5]]
+;
+  %t0 = sub i16 32, %y
+  %t1 = zext i16 %t0 to i32
+  %t2 = lshr i32 %x, %t1
+  %t3 = trunc i32 %t2 to i16
+  %t4 = add i16 %y, -1
+  call void @use16(i16 %t3)
+  %t5 = lshr i16 %t3, %t4
+  ret i16 %t5
+}
+
+define i16 @t7_extrause1(i32 %x, i16 %y) {
+; CHECK-LABEL: @t7_extrause1(
+; CHECK-NEXT:    [[T4:%.*]] = add i16 [[Y:%.*]], -1
+; CHECK-NEXT:    call void @use16(i16 [[T4]])
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i32 [[X:%.*]], 31
+; CHECK-NEXT:    [[T5:%.*]] = trunc i32 [[TMP1]] to i16
+; CHECK-NEXT:    ret i16 [[T5]]
+;
+  %t0 = sub i16 32, %y
+  %t1 = zext i16 %t0 to i32
+  %t2 = lshr i32 %x, %t1
+  %t3 = trunc i32 %t2 to i16
+  %t4 = add i16 %y, -1
+  call void @use16(i16 %t4)
+  %t5 = lshr i16 %t3, %t4
+  ret i16 %t5
+}
+
+define i16 @t8_extrause2(i32 %x, i16 %y) {
+; CHECK-LABEL: @t8_extrause2(
+; CHECK-NEXT:    [[T0:%.*]] = sub i16 32, [[Y:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = zext i16 [[T0]] to i32
+; CHECK-NEXT:    [[T2:%.*]] = lshr i32 [[X:%.*]], [[T1]]
+; CHECK-NEXT:    [[T3:%.*]] = trunc i32 [[T2]] to i16
+; CHECK-NEXT:    [[T4:%.*]] = add i16 [[Y]], -1
+; CHECK-NEXT:    call void @use16(i16 [[T3]])
+; CHECK-NEXT:    call void @use16(i16 [[T4]])
+; CHECK-NEXT:    [[T5:%.*]] = lshr i16 [[T3]], [[T4]]
+; CHECK-NEXT:    ret i16 [[T5]]
+;
+  %t0 = sub i16 32, %y
+  %t1 = zext i16 %t0 to i32
+  %t2 = lshr i32 %x, %t1
+  %t3 = trunc i32 %t2 to i16
+  %t4 = add i16 %y, -1
+  call void @use16(i16 %t3)
+  call void @use16(i16 %t4)
+  %t5 = lshr i16 %t3, %t4
+  ret i16 %t5
+}
+
+; No 'nuw'/'nsw' flags are to be propagated!
+; But we can't test that, such IR does not reach that code.
+
+; Negative tests
+
+; Can only fold if we are extracting the sign bit.
+define i16 @t9_lshr(i32 %x, i16 %y) {
+; CHECK-LABEL: @t9_lshr(
+; CHECK-NEXT:    [[T0:%.*]] = sub i16 32, [[Y:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = zext i16 [[T0]] to i32
+; CHECK-NEXT:    [[T2:%.*]] = lshr i32 [[X:%.*]], [[T1]]
+; CHECK-NEXT:    [[T3:%.*]] = trunc i32 [[T2]] to i16
+; CHECK-NEXT:    [[T4:%.*]] = add i16 [[Y]], -2
+; CHECK-NEXT:    [[T5:%.*]] = lshr i16 [[T3]], [[T4]]
+; CHECK-NEXT:    ret i16 [[T5]]
+;
+  %t0 = sub i16 32, %y
+  %t1 = zext i16 %t0 to i32
+  %t2 = lshr i32 %x, %t1
+  %t3 = trunc i32 %t2 to i16
+  %t4 = add i16 %y, -2
+  %t5 = lshr i16 %t3, %t4
+  ret i16 %t5
+}
+
+; If we have different right-shifts, in general, we can't do anything with it.
+define i16 @n10_ashr_lshr(i32 %x, i16 %y) {
+; CHECK-LABEL: @n10_ashr_lshr(
+; CHECK-NEXT:    [[T0:%.*]] = sub i16 32, [[Y:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = zext i16 [[T0]] to i32
+; CHECK-NEXT:    [[T2:%.*]] = ashr i32 [[X:%.*]], [[T1]]
+; CHECK-NEXT:    [[T3:%.*]] = trunc i32 [[T2]] to i16
+; CHECK-NEXT:    [[T4:%.*]] = add i16 [[Y]], -1
+; CHECK-NEXT:    [[T5:%.*]] = lshr i16 [[T3]], [[T4]]
+; CHECK-NEXT:    ret i16 [[T5]]
+;
+  %t0 = sub i16 32, %y
+  %t1 = zext i16 %t0 to i32
+  %t2 = ashr i32 %x, %t1
+  %t3 = trunc i32 %t2 to i16
+  %t4 = add i16 %y, -1
+  %t5 = lshr i16 %t3, %t4
+  ret i16 %t5
+}
diff --git a/llvm/test/Transforms/InstCombine/shift-amount-reassociation-with-truncation-shl.ll b/llvm/test/Transforms/InstCombine/shift-amount-reassociation-with-truncation-shl.ll
new file mode 100644
index 000000000..7ae9b76fa
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/shift-amount-reassociation-with-truncation-shl.ll
@@ -0,0 +1,212 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instcombine -S | FileCheck %s
+
+; Given pattern:
+;   (trunc (x << Q) to iDst) << K
+; we should rewrite it as
+;   (trunc (x << (Q+K)) to iDst)  iff (Q+K) u< iDst
+; This is only valid for shl.
+; THIS FOLD DOES *NOT* REQUIRE ANY 'nuw'/`nsw` FLAGS!
+
+; Basic scalar test
+
+define i16 @t0(i32 %x, i16 %y) {
+; CHECK-LABEL: @t0(
+; CHECK-NEXT:    [[X_TR:%.*]] = trunc i32 [[X:%.*]] to i16
+; CHECK-NEXT:    [[T5:%.*]] = shl i16 [[X_TR]], 8
+; CHECK-NEXT:    ret i16 [[T5]]
+;
+  %t0 = sub i16 32, %y
+  %t1 = zext i16 %t0 to i32
+  %t2 = shl i32 %x, %t1
+  %t3 = trunc i32 %t2 to i16
+  %t4 = add i16 %y, -24
+  %t5 = shl i16 %t3, %t4
+  ret i16 %t5
+}
+
+define <2 x i16> @t1_vec_splat(<2 x i32> %x, <2 x i16> %y) {
+; CHECK-LABEL: @t1_vec_splat(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <2 x i32> [[X:%.*]], <i32 8, i32 8>
+; CHECK-NEXT:    [[T5:%.*]] = trunc <2 x i32> [[TMP1]] to <2 x i16>
+; CHECK-NEXT:    ret <2 x i16> [[T5]]
+;
+  %t0 = sub <2 x i16> <i16 32, i16 32>, %y
+  %t1 = zext <2 x i16> %t0 to <2 x i32>
+  %t2 = shl <2 x i32> %x, %t1
+  %t3 = trunc <2 x i32> %t2 to <2 x i16>
+  %t4 = add <2 x i16> %y, <i16 -24, i16 -24>
+  %t5 = shl <2 x i16> %t3, %t4
+  ret <2 x i16> %t5
+}
+
+define <2 x i16> @t2_vec_nonsplat(<2 x i32> %x, <2 x i16> %y) {
+; CHECK-LABEL: @t2_vec_nonsplat(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <2 x i32> [[X:%.*]], <i32 8, i32 30>
+; CHECK-NEXT:    [[T5:%.*]] = trunc <2 x i32> [[TMP1]] to <2 x i16>
+; CHECK-NEXT:    ret <2 x i16> [[T5]]
+;
+  %t0 = sub <2 x i16> <i16 32, i16 30>, %y
+  %t1 = zext <2 x i16> %t0 to <2 x i32>
+  %t2 = shl <2 x i32> %x, %t1
+  %t3 = trunc <2 x i32> %t2 to <2 x i16>
+  %t4 = add <2 x i16> %y, <i16 -24, i16 0>
+  %t5 = shl <2 x i16> %t3, %t4
+  ret <2 x i16> %t5
+}
+
+; Basic vector tests
+
+define <3 x i16> @t3_vec_nonsplat_undef0(<3 x i32> %x, <3 x i16> %y) {
+; CHECK-LABEL: @t3_vec_nonsplat_undef0(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <3 x i32> [[X:%.*]], <i32 8, i32 0, i32 8>
+; CHECK-NEXT:    [[T5:%.*]] = trunc <3 x i32> [[TMP1]] to <3 x i16>
+; CHECK-NEXT:    ret <3 x i16> [[T5]]
+;
+  %t0 = sub <3 x i16> <i16 32, i16 undef, i16 32>, %y
+  %t1 = zext <3 x i16> %t0 to <3 x i32>
+  %t2 = shl <3 x i32> %x, %t1
+  %t3 = trunc <3 x i32> %t2 to <3 x i16>
+  %t4 = add <3 x i16> %y, <i16 -24, i16 -24, i16 -24>
+  %t5 = shl <3 x i16> %t3, %t4
+  ret <3 x i16> %t5
+}
+
+define <3 x i16> @t4_vec_nonsplat_undef1(<3 x i32> %x, <3 x i16> %y) {
+; CHECK-LABEL: @t4_vec_nonsplat_undef1(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <3 x i32> [[X:%.*]], <i32 8, i32 0, i32 8>
+; CHECK-NEXT:    [[T5:%.*]] = trunc <3 x i32> [[TMP1]] to <3 x i16>
+; CHECK-NEXT:    ret <3 x i16> [[T5]]
+;
+  %t0 = sub <3 x i16> <i16 32, i16 32, i16 32>, %y
+  %t1 = zext <3 x i16> %t0 to <3 x i32>
+  %t2 = shl <3 x i32> %x, %t1
+  %t3 = trunc <3 x i32> %t2 to <3 x i16>
+  %t4 = add <3 x i16> %y, <i16 -24, i16 undef, i16 -24>
+  %t5 = shl <3 x i16> %t3, %t4
+  ret <3 x i16> %t5
+}
+
+define <3 x i16> @t5_vec_nonsplat_undef1(<3 x i32> %x, <3 x i16> %y) {
+; CHECK-LABEL: @t5_vec_nonsplat_undef1(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <3 x i32> [[X:%.*]], <i32 8, i32 0, i32 8>
+; CHECK-NEXT:    [[T5:%.*]] = trunc <3 x i32> [[TMP1]] to <3 x i16>
+; CHECK-NEXT:    ret <3 x i16> [[T5]]
+;
+  %t0 = sub <3 x i16> <i16 32, i16 undef, i16 32>, %y
+  %t1 = zext <3 x i16> %t0 to <3 x i32>
+  %t2 = shl <3 x i32> %x, %t1
+  %t3 = trunc <3 x i32> %t2 to <3 x i16>
+  %t4 = add <3 x i16> %y, <i16 -24, i16 undef, i16 -24>
+  %t5 = shl <3 x i16> %t3, %t4
+  ret <3 x i16> %t5
+}
+
+; One-use tests
+
+declare void @use16(i16)
+declare void @use32(i32)
+
+define i16 @t6_extrause0(i32 %x, i16 %y) {
+; CHECK-LABEL: @t6_extrause0(
+; CHECK-NEXT:    [[T0:%.*]] = sub i16 32, [[Y:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = zext i16 [[T0]] to i32
+; CHECK-NEXT:    [[T2:%.*]] = shl i32 [[X:%.*]], [[T1]]
+; CHECK-NEXT:    [[T3:%.*]] = trunc i32 [[T2]] to i16
+; CHECK-NEXT:    call void @use16(i16 [[T3]])
+; CHECK-NEXT:    [[X_TR:%.*]] = trunc i32 [[X]] to i16
+; CHECK-NEXT:    [[T5:%.*]] = shl i16 [[X_TR]], 8
+; CHECK-NEXT:    ret i16 [[T5]]
+;
+  %t0 = sub i16 32, %y
+  %t1 = zext i16 %t0 to i32
+  %t2 = shl i32 %x, %t1
+  %t3 = trunc i32 %t2 to i16
+  %t4 = add i16 %y, -24
+  call void @use16(i16 %t3)
+  %t5 = shl i16 %t3, %t4
+  ret i16 %t5
+}
+
+define i16 @t7_extrause1(i32 %x, i16 %y) {
+; CHECK-LABEL: @t7_extrause1(
+; CHECK-NEXT:    [[T4:%.*]] = add i16 [[Y:%.*]], -24
+; CHECK-NEXT:    call void @use16(i16 [[T4]])
+; CHECK-NEXT:    [[X_TR:%.*]] = trunc i32 [[X:%.*]] to i16
+; CHECK-NEXT:    [[T5:%.*]] = shl i16 [[X_TR]], 8
+; CHECK-NEXT:    ret i16 [[T5]]
+;
+  %t0 = sub i16 32, %y
+  %t1 = zext i16 %t0 to i32
+  %t2 = shl i32 %x, %t1
+  %t3 = trunc i32 %t2 to i16
+  %t4 = add i16 %y, -24
+  call void @use16(i16 %t4)
+  %t5 = shl i16 %t3, %t4
+  ret i16 %t5
+}
+
+define i16 @t8_extrause2(i32 %x, i16 %y) {
+; CHECK-LABEL: @t8_extrause2(
+; CHECK-NEXT:    [[T0:%.*]] = sub i16 32, [[Y:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = zext i16 [[T0]] to i32
+; CHECK-NEXT:    [[T2:%.*]] = shl i32 [[X:%.*]], [[T1]]
+; CHECK-NEXT:    [[T3:%.*]] = trunc i32 [[T2]] to i16
+; CHECK-NEXT:    [[T4:%.*]] = add i16 [[Y]], -24
+; CHECK-NEXT:    call void @use16(i16 [[T3]])
+; CHECK-NEXT:    call void @use16(i16 [[T4]])
+; CHECK-NEXT:    [[T5:%.*]] = shl i16 [[T3]], [[T4]]
+; CHECK-NEXT:    ret i16 [[T5]]
+;
+  %t0 = sub i16 32, %y
+  %t1 = zext i16 %t0 to i32
+  %t2 = shl i32 %x, %t1
+  %t3 = trunc i32 %t2 to i16
+  %t4 = add i16 %y, -24
+  call void @use16(i16 %t3)
+  call void @use16(i16 %t4)
+  %t5 = shl i16 %t3, %t4
+  ret i16 %t5
+}
+
+; No 'nuw'/'nsw' flags are to be propagated!
+; But we can't test that, such IR does not reach that code.
+
+; Negative tests
+
+; Can't fold, total shift would be 32
+define i16 @n11(i32 %x, i16 %y) {
+; CHECK-LABEL: @n11(
+; CHECK-NEXT:    [[T0:%.*]] = sub i16 30, [[Y:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = zext i16 [[T0]] to i32
+; CHECK-NEXT:    [[T2:%.*]] = shl i32 [[X:%.*]], [[T1]]
+; CHECK-NEXT:    [[T3:%.*]] = trunc i32 [[T2]] to i16
+; CHECK-NEXT:    [[T4:%.*]] = add i16 [[Y]], -31
+; CHECK-NEXT:    [[T5:%.*]] = shl i16 [[T3]], [[T4]]
+; CHECK-NEXT:    ret i16 [[T5]]
+;
+  %t0 = sub i16 30, %y
+  %t1 = zext i16 %t0 to i32
+  %t2 = shl i32 %x, %t1
+  %t3 = trunc i32 %t2 to i16
+  %t4 = add i16 %y, -31
+  %t5 = shl i16 %t3, %t4
+  ret i16 %t5
+}
+
+; Bit width mismatch of shit amount
+
+@Y32 = global i32 42
+@Y16 = global i16 42
+define i16 @t01(i32 %x) {
+; CHECK-LABEL: @t01(
+; CHECK-NEXT:    [[T0:%.*]] = shl i32 [[X:%.*]], ptrtoint (i32* @Y32 to i32)
+; CHECK-NEXT:    [[T1:%.*]] = trunc i32 [[T0]] to i16
+; CHECK-NEXT:    [[T2:%.*]] = shl i16 [[T1]], ptrtoint (i16* @Y16 to i16)
+; CHECK-NEXT:    ret i16 [[T2]]
+;
+  %t0 = shl i32 %x, ptrtoint (i32* @Y32 to i32)
+  %t1 = trunc i32 %t0 to i16
+  %t2 = shl i16 %t1, ptrtoint (i16* @Y16 to i16)
+  ret i16 %t2
+}
diff --git a/llvm/test/Transforms/InstCombine/shift-amount-reassociation.ll b/llvm/test/Transforms/InstCombine/shift-amount-reassociation.ll
index e124a0358..96461691e 100644
--- a/llvm/test/Transforms/InstCombine/shift-amount-reassociation.ll
+++ b/llvm/test/Transforms/InstCombine/shift-amount-reassociation.ll
@@ -203,3 +203,137 @@ define <2 x i32> @t13_vec(<2 x i32> %x, <2 x i32> %y) {
   %t3 = lshr <2 x i32> %t1, %t2
   ret <2 x i32> %t3
 }
+
+; If we have different right-shifts, in general, we can't do anything with it.
+define i32 @n13(i32 %x, i32 %y) {
+; CHECK-LABEL: @n13(
+; CHECK-NEXT:    [[T0:%.*]] = sub i32 32, [[Y:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = lshr i32 [[X:%.*]], [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[Y]], -2
+; CHECK-NEXT:    [[T3:%.*]] = ashr i32 [[T1]], [[T2]]
+; CHECK-NEXT:    ret i32 [[T3]]
+;
+  %t0 = sub i32 32, %y
+  %t1 = lshr i32 %x, %t0
+  %t2 = add i32 %y, -2
+  %t3 = ashr i32 %t1, %t2
+  ret i32 %t3
+}
+define i32 @n14(i32 %x, i32 %y) {
+; CHECK-LABEL: @n14(
+; CHECK-NEXT:    [[T0:%.*]] = sub i32 32, [[Y:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = lshr i32 [[X:%.*]], [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[Y]], -1
+; CHECK-NEXT:    [[T3:%.*]] = ashr i32 [[T1]], [[T2]]
+; CHECK-NEXT:    ret i32 [[T3]]
+;
+  %t0 = sub i32 32, %y
+  %t1 = lshr i32 %x, %t0
+  %t2 = add i32 %y, -1
+  %t3 = ashr i32 %t1, %t2
+  ret i32 %t3
+}
+define i32 @n15(i32 %x, i32 %y) {
+; CHECK-LABEL: @n15(
+; CHECK-NEXT:    [[T0:%.*]] = sub i32 32, [[Y:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = ashr i32 [[X:%.*]], [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[Y]], -2
+; CHECK-NEXT:    [[T3:%.*]] = lshr i32 [[T1]], [[T2]]
+; CHECK-NEXT:    ret i32 [[T3]]
+;
+  %t0 = sub i32 32, %y
+  %t1 = ashr i32 %x, %t0
+  %t2 = add i32 %y, -2
+  %t3 = lshr i32 %t1, %t2
+  ret i32 %t3
+}
+define i32 @n16(i32 %x, i32 %y) {
+; CHECK-LABEL: @n16(
+; CHECK-NEXT:    [[T0:%.*]] = sub i32 32, [[Y:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = ashr i32 [[X:%.*]], [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[Y]], -1
+; CHECK-NEXT:    [[T3:%.*]] = lshr i32 [[T1]], [[T2]]
+; CHECK-NEXT:    ret i32 [[T3]]
+;
+  %t0 = sub i32 32, %y
+  %t1 = ashr i32 %x, %t0
+  %t2 = add i32 %y, -1
+  %t3 = lshr i32 %t1, %t2
+  ret i32 %t3
+}
+
+; If the shift direction is different, then this should be handled elsewhere.
+define i32 @n17(i32 %x, i32 %y) {
+; CHECK-LABEL: @n17(
+; CHECK-NEXT:    [[T0:%.*]] = sub i32 32, [[Y:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = shl i32 [[X:%.*]], [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[Y]], -1
+; CHECK-NEXT:    [[T3:%.*]] = lshr i32 [[T1]], [[T2]]
+; CHECK-NEXT:    ret i32 [[T3]]
+;
+  %t0 = sub i32 32, %y
+  %t1 = shl i32 %x, %t0
+  %t2 = add i32 %y, -1
+  %t3 = lshr i32 %t1, %t2
+  ret i32 %t3
+}
+define i32 @n18(i32 %x, i32 %y) {
+; CHECK-LABEL: @n18(
+; CHECK-NEXT:    [[T0:%.*]] = sub i32 32, [[Y:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = shl i32 [[X:%.*]], [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[Y]], -1
+; CHECK-NEXT:    [[T3:%.*]] = ashr i32 [[T1]], [[T2]]
+; CHECK-NEXT:    ret i32 [[T3]]
+;
+  %t0 = sub i32 32, %y
+  %t1 = shl i32 %x, %t0
+  %t2 = add i32 %y, -1
+  %t3 = ashr i32 %t1, %t2
+  ret i32 %t3
+}
+define i32 @n19(i32 %x, i32 %y) {
+; CHECK-LABEL: @n19(
+; CHECK-NEXT:    [[T0:%.*]] = sub i32 32, [[Y:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = lshr i32 [[X:%.*]], [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[Y]], -1
+; CHECK-NEXT:    [[T3:%.*]] = shl i32 [[T1]], [[T2]]
+; CHECK-NEXT:    ret i32 [[T3]]
+;
+  %t0 = sub i32 32, %y
+  %t1 = lshr i32 %x, %t0
+  %t2 = add i32 %y, -1
+  %t3 = shl i32 %t1, %t2
+  ret i32 %t3
+}
+define i32 @n20(i32 %x, i32 %y) {
+; CHECK-LABEL: @n20(
+; CHECK-NEXT:    [[T0:%.*]] = sub i32 32, [[Y:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = ashr i32 [[X:%.*]], [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[Y]], -1
+; CHECK-NEXT:    [[T3:%.*]] = shl i32 [[T1]], [[T2]]
+; CHECK-NEXT:    ret i32 [[T3]]
+;
+  %t0 = sub i32 32, %y
+  %t1 = ashr i32 %x, %t0
+  %t2 = add i32 %y, -1
+  %t3 = shl i32 %t1, %t2
+  ret i32 %t3
+}
+
+; See https://bugs.llvm.org/show_bug.cgi?id=44802
+define i3 @pr44802(i3 %t0) {
+; CHECK-LABEL: @pr44802(
+; CHECK-NEXT:    [[T1:%.*]] = sub i3 0, [[T0:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = icmp ne i3 [[T0]], 0
+; CHECK-NEXT:    [[T3:%.*]] = zext i1 [[T2]] to i3
+; CHECK-NEXT:    [[T4:%.*]] = lshr i3 [[T1]], [[T3]]
+; CHECK-NEXT:    [[T5:%.*]] = lshr i3 [[T4]], [[T3]]
+; CHECK-NEXT:    ret i3 [[T5]]
+;
+  %t1 = sub i3 0, %t0
+  %t2 = icmp ne i3 %t0, 0
+  %t3 = zext i1 %t2 to i3
+  %t4 = lshr i3 %t1, %t3
+  %t5 = lshr i3 %t4, %t3
+  ret i3 %t5
+}
diff --git a/llvm/test/Transforms/InstCombine/shift-by-signext.ll b/llvm/test/Transforms/InstCombine/shift-by-signext.ll
new file mode 100644
index 000000000..b049dfac6
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/shift-by-signext.ll
@@ -0,0 +1,173 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instcombine -S | FileCheck %s
+
+; If we have a shift by sign-extended value, we can replace sign-extension
+; with zero-extension.
+
+define i32 @t0_shl(i32 %x, i8 %shamt) {
+; CHECK-LABEL: @t0_shl(
+; CHECK-NEXT:    [[SHAMT_WIDE1:%.*]] = zext i8 [[SHAMT:%.*]] to i32
+; CHECK-NEXT:    [[R:%.*]] = shl i32 [[X:%.*]], [[SHAMT_WIDE1]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %shamt_wide = sext i8 %shamt to i32
+  %r = shl i32 %x, %shamt_wide
+  ret i32 %r
+}
+define i32 @t1_lshr(i32 %x, i8 %shamt) {
+; CHECK-LABEL: @t1_lshr(
+; CHECK-NEXT:    [[SHAMT_WIDE1:%.*]] = zext i8 [[SHAMT:%.*]] to i32
+; CHECK-NEXT:    [[R:%.*]] = lshr i32 [[X:%.*]], [[SHAMT_WIDE1]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %shamt_wide = sext i8 %shamt to i32
+  %r = lshr i32 %x, %shamt_wide
+  ret i32 %r
+}
+define i32 @t2_ashr(i32 %x, i8 %shamt) {
+; CHECK-LABEL: @t2_ashr(
+; CHECK-NEXT:    [[SHAMT_WIDE1:%.*]] = zext i8 [[SHAMT:%.*]] to i32
+; CHECK-NEXT:    [[R:%.*]] = ashr i32 [[X:%.*]], [[SHAMT_WIDE1]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %shamt_wide = sext i8 %shamt to i32
+  %r = ashr i32 %x, %shamt_wide
+  ret i32 %r
+}
+
+define <2 x i32> @t3_vec_shl(<2 x i32> %x, <2 x i8> %shamt) {
+; CHECK-LABEL: @t3_vec_shl(
+; CHECK-NEXT:    [[SHAMT_WIDE1:%.*]] = zext <2 x i8> [[SHAMT:%.*]] to <2 x i32>
+; CHECK-NEXT:    [[R:%.*]] = shl <2 x i32> [[X:%.*]], [[SHAMT_WIDE1]]
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %shamt_wide = sext <2 x i8> %shamt to <2 x i32>
+  %r = shl <2 x i32> %x, %shamt_wide
+  ret <2 x i32> %r
+}
+define <2 x i32> @t4_vec_lshr(<2 x i32> %x, <2 x i8> %shamt) {
+; CHECK-LABEL: @t4_vec_lshr(
+; CHECK-NEXT:    [[SHAMT_WIDE1:%.*]] = zext <2 x i8> [[SHAMT:%.*]] to <2 x i32>
+; CHECK-NEXT:    [[R:%.*]] = lshr <2 x i32> [[X:%.*]], [[SHAMT_WIDE1]]
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %shamt_wide = sext <2 x i8> %shamt to <2 x i32>
+  %r = lshr <2 x i32> %x, %shamt_wide
+  ret <2 x i32> %r
+}
+define <2 x i32> @t5_vec_ashr(<2 x i32> %x, <2 x i8> %shamt) {
+; CHECK-LABEL: @t5_vec_ashr(
+; CHECK-NEXT:    [[SHAMT_WIDE1:%.*]] = zext <2 x i8> [[SHAMT:%.*]] to <2 x i32>
+; CHECK-NEXT:    [[R:%.*]] = ashr <2 x i32> [[X:%.*]], [[SHAMT_WIDE1]]
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %shamt_wide = sext <2 x i8> %shamt to <2 x i32>
+  %r = ashr <2 x i32> %x, %shamt_wide
+  ret <2 x i32> %r
+}
+
+define i32 @t6_twoshifts(i32 %x, i8 %shamt) {
+; CHECK-LABEL: @t6_twoshifts(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[SHAMT_WIDE:%.*]] = sext i8 [[SHAMT:%.*]] to i32
+; CHECK-NEXT:    br label [[WORK:%.*]]
+; CHECK:       work:
+; CHECK-NEXT:    br label [[END:%.*]]
+; CHECK:       end:
+; CHECK-NEXT:    [[N0:%.*]] = shl i32 [[X:%.*]], [[SHAMT_WIDE]]
+; CHECK-NEXT:    [[R:%.*]] = ashr i32 [[N0]], [[SHAMT_WIDE]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+bb:
+  %shamt_wide = sext i8 %shamt to i32
+  br label %work
+work:
+  %n0 = shl i32 %x, %shamt_wide
+  %r = ashr i32 %n0, %shamt_wide
+  br label %end
+end:
+  ret i32 %r
+}
+
+; This is not valid for funnel shifts in general
+declare i7 @llvm.fshl.i7(i7 %a, i7 %b, i7 %c)
+declare i7 @llvm.fshr.i7(i7 %a, i7 %b, i7 %c)
+define i7 @n7_fshl(i7 %x, i7 %y, i6 %shamt) {
+; CHECK-LABEL: @n7_fshl(
+; CHECK-NEXT:    [[SHAMT_WIDE:%.*]] = sext i6 [[SHAMT:%.*]] to i7
+; CHECK-NEXT:    [[R:%.*]] = call i7 @llvm.fshl.i7(i7 [[X:%.*]], i7 [[Y:%.*]], i7 [[SHAMT_WIDE]])
+; CHECK-NEXT:    ret i7 [[R]]
+;
+  %shamt_wide = sext i6 %shamt to i7
+  %r = call i7 @llvm.fshl.i7(i7 %x, i7 %y, i7 %shamt_wide)
+  ret i7 %r
+}
+define i7 @n8_fshr(i7 %x, i7 %y, i6 %shamt) {
+; CHECK-LABEL: @n8_fshr(
+; CHECK-NEXT:    [[SHAMT_WIDE:%.*]] = sext i6 [[SHAMT:%.*]] to i7
+; CHECK-NEXT:    [[R:%.*]] = call i7 @llvm.fshr.i7(i7 [[X:%.*]], i7 [[Y:%.*]], i7 [[SHAMT_WIDE]])
+; CHECK-NEXT:    ret i7 [[R]]
+;
+  %shamt_wide = sext i6 %shamt to i7
+  %r = call i7 @llvm.fshr.i7(i7 %x, i7 %y, i7 %shamt_wide)
+  ret i7 %r
+}
+; And the cases that are safe are handled by SimplifyDemandedBits().
+declare i8 @llvm.fshl.i8(i8 %a, i8 %b, i8 %c)
+declare i8 @llvm.fshr.i8(i8 %a, i8 %b, i8 %c)
+define i8 @t9_fshl(i8 %x, i8 %y, i6 %shamt) {
+; CHECK-LABEL: @t9_fshl(
+; CHECK-NEXT:    [[SHAMT_WIDE1:%.*]] = zext i6 [[SHAMT:%.*]] to i8
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.fshl.i8(i8 [[X:%.*]], i8 [[Y:%.*]], i8 [[SHAMT_WIDE1]])
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %shamt_wide = sext i6 %shamt to i8
+  %r = call i8 @llvm.fshl.i8(i8 %x, i8 %y, i8 %shamt_wide)
+  ret i8 %r
+}
+define i8 @t10_fshr(i8 %x, i8 %y, i6 %shamt) {
+; CHECK-LABEL: @t10_fshr(
+; CHECK-NEXT:    [[SHAMT_WIDE1:%.*]] = zext i6 [[SHAMT:%.*]] to i8
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.fshr.i8(i8 [[X:%.*]], i8 [[Y:%.*]], i8 [[SHAMT_WIDE1]])
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %shamt_wide = sext i6 %shamt to i8
+  %r = call i8 @llvm.fshr.i8(i8 %x, i8 %y, i8 %shamt_wide)
+  ret i8 %r
+}
+
+declare void @use32(i32)
+define i32 @n11_extrause(i32 %x, i8 %shamt) {
+; CHECK-LABEL: @n11_extrause(
+; CHECK-NEXT:    [[SHAMT_WIDE:%.*]] = sext i8 [[SHAMT:%.*]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[SHAMT_WIDE]])
+; CHECK-NEXT:    [[R:%.*]] = shl i32 [[X:%.*]], [[SHAMT_WIDE]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %shamt_wide = sext i8 %shamt to i32
+  call void @use32(i32 %shamt_wide)
+  %r = shl i32 %x, %shamt_wide
+  ret i32 %r
+}
+define i32 @n12_twoshifts_and_extrause(i32 %x, i8 %shamt) {
+; CHECK-LABEL: @n12_twoshifts_and_extrause(
+; CHECK-NEXT:    [[SHAMT_WIDE:%.*]] = sext i8 [[SHAMT:%.*]] to i32
+; CHECK-NEXT:    br label [[WORK:%.*]]
+; CHECK:       work:
+; CHECK-NEXT:    br label [[END:%.*]]
+; CHECK:       end:
+; CHECK-NEXT:    [[N0:%.*]] = shl i32 [[X:%.*]], [[SHAMT_WIDE]]
+; CHECK-NEXT:    [[R:%.*]] = ashr i32 [[N0]], [[SHAMT_WIDE]]
+; CHECK-NEXT:    call void @use32(i32 [[SHAMT_WIDE]])
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %shamt_wide = sext i8 %shamt to i32
+  br label %work
+work:
+  %n0 = shl i32 %x, %shamt_wide
+  %r = ashr i32 %n0, %shamt_wide
+  br label %end
+end:
+  call void @use32(i32 %shamt_wide)
+  ret i32 %r
+}
diff --git a/llvm/test/Transforms/InstCombine/shift-logic.ll b/llvm/test/Transforms/InstCombine/shift-logic.ll
new file mode 100644
index 000000000..d0d06a387
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/shift-logic.ll
@@ -0,0 +1,188 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define i8 @shl_and(i8 %x, i8 %y) {
+; CHECK-LABEL: @shl_and(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i8 [[X:%.*]], 5
+; CHECK-NEXT:    [[TMP2:%.*]] = shl i8 [[Y:%.*]], 2
+; CHECK-NEXT:    [[SH1:%.*]] = and i8 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret i8 [[SH1]]
+;
+  %sh0 = shl i8 %x, 3
+  %r = and i8 %sh0, %y
+  %sh1 = shl i8 %r, 2
+  ret i8 %sh1
+}
+
+define i16 @shl_or(i16 %x, i16 %py) {
+; CHECK-LABEL: @shl_or(
+; CHECK-NEXT:    [[Y:%.*]] = srem i16 [[PY:%.*]], 42
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i16 [[X:%.*]], 12
+; CHECK-NEXT:    [[TMP2:%.*]] = shl nsw i16 [[Y]], 7
+; CHECK-NEXT:    [[SH1:%.*]] = or i16 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret i16 [[SH1]]
+;
+  %y = srem i16 %py, 42 ; thwart complexity-based canonicalization
+  %sh0 = shl i16 %x, 5
+  %r = or i16 %y, %sh0
+  %sh1 = shl i16 %r, 7
+  ret i16 %sh1
+}
+
+define i32 @shl_xor(i32 %x, i32 %y) {
+; CHECK-LABEL: @shl_xor(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 [[X:%.*]], 12
+; CHECK-NEXT:    [[TMP2:%.*]] = shl i32 [[Y:%.*]], 7
+; CHECK-NEXT:    [[SH1:%.*]] = xor i32 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret i32 [[SH1]]
+;
+  %sh0 = shl i32 %x, 5
+  %r = xor i32 %sh0, %y
+  %sh1 = shl i32 %r, 7
+  ret i32 %sh1
+}
+
+define i64 @lshr_and(i64 %x, i64 %py) {
+; CHECK-LABEL: @lshr_and(
+; CHECK-NEXT:    [[Y:%.*]] = srem i64 [[PY:%.*]], 42
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i64 [[X:%.*]], 12
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr i64 [[Y]], 7
+; CHECK-NEXT:    [[SH1:%.*]] = and i64 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret i64 [[SH1]]
+;
+  %y = srem i64 %py, 42 ; thwart complexity-based canonicalization
+  %sh0 = lshr i64 %x, 5
+  %r = and i64 %y, %sh0
+  %sh1 = lshr i64 %r, 7
+  ret i64 %sh1
+}
+
+define <4 x i32> @lshr_or(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: @lshr_or(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <4 x i32> [[X:%.*]], <i32 12, i32 12, i32 12, i32 12>
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr <4 x i32> [[Y:%.*]], <i32 7, i32 7, i32 7, i32 7>
+; CHECK-NEXT:    [[SH1:%.*]] = or <4 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret <4 x i32> [[SH1]]
+;
+  %sh0 = lshr <4 x i32> %x, <i32 5, i32 5, i32 5, i32 5>
+  %r = or <4 x i32> %sh0, %y
+  %sh1 = lshr <4 x i32> %r, <i32 7, i32 7, i32 7, i32 7>
+  ret <4 x i32> %sh1
+}
+
+define <8 x i16> @lshr_xor(<8 x i16> %x, <8 x i16> %py) {
+; CHECK-LABEL: @lshr_xor(
+; CHECK-NEXT:    [[Y:%.*]] = srem <8 x i16> [[PY:%.*]], <i16 42, i16 42, i16 42, i16 42, i16 42, i16 42, i16 42, i16 42>
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <8 x i16> [[X:%.*]], <i16 12, i16 12, i16 12, i16 12, i16 12, i16 12, i16 12, i16 12>
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr <8 x i16> [[Y]], <i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7>
+; CHECK-NEXT:    [[SH1:%.*]] = xor <8 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret <8 x i16> [[SH1]]
+;
+  %y = srem <8 x i16> %py, <i16 42, i16 42, i16 42, i16 42, i16 42, i16 42, i16 42, i16 -42> ; thwart complexity-based canonicalization
+  %sh0 = lshr <8 x i16> %x, <i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5>
+  %r = xor <8 x i16> %y, %sh0
+  %sh1 = lshr <8 x i16> %r, <i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7>
+  ret <8 x i16> %sh1
+}
+
+
+define <16 x i8> @ashr_and(<16 x i8> %x, <16 x i8> %py, <16 x i8> %pz) {
+; CHECK-LABEL: @ashr_and(
+; CHECK-NEXT:    [[Y:%.*]] = srem <16 x i8> [[PY:%.*]], [[PZ:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <16 x i8> [[X:%.*]], <i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5>
+; CHECK-NEXT:    [[TMP2:%.*]] = ashr <16 x i8> [[Y]], <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>
+; CHECK-NEXT:    [[SH1:%.*]] = and <16 x i8> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret <16 x i8> [[SH1]]
+;
+  %y = srem <16 x i8> %py, %pz ; thwart complexity-based canonicalization
+  %sh0 = ashr <16 x i8> %x, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
+  %r = and <16 x i8> %y, %sh0
+  %sh1 = ashr <16 x i8> %r, <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>
+  ret <16 x i8> %sh1
+}
+
+define <2 x i64> @ashr_or(<2 x i64> %x, <2 x i64> %y) {
+; CHECK-LABEL: @ashr_or(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <2 x i64> [[X:%.*]], <i64 12, i64 12>
+; CHECK-NEXT:    [[TMP2:%.*]] = ashr <2 x i64> [[Y:%.*]], <i64 7, i64 7>
+; CHECK-NEXT:    [[SH1:%.*]] = or <2 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret <2 x i64> [[SH1]]
+;
+  %sh0 = ashr <2 x i64> %x, <i64 5, i64 5>
+  %r = or <2 x i64> %sh0, %y
+  %sh1 = ashr <2 x i64> %r, <i64 7, i64 7>
+  ret <2 x i64> %sh1
+}
+
+define i32 @ashr_xor(i32 %x, i32 %py) {
+; CHECK-LABEL: @ashr_xor(
+; CHECK-NEXT:    [[Y:%.*]] = srem i32 [[PY:%.*]], 42
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr i32 [[X:%.*]], 12
+; CHECK-NEXT:    [[TMP2:%.*]] = ashr i32 [[Y]], 7
+; CHECK-NEXT:    [[SH1:%.*]] = xor i32 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret i32 [[SH1]]
+;
+  %y = srem i32 %py, 42 ; thwart complexity-based canonicalization
+  %sh0 = ashr i32 %x, 5
+  %r = xor i32 %y, %sh0
+  %sh1 = ashr i32 %r, 7
+  ret i32 %sh1
+}
+
+define i32 @shr_mismatch_xor(i32 %x, i32 %y) {
+; CHECK-LABEL: @shr_mismatch_xor(
+; CHECK-NEXT:    [[SH0:%.*]] = ashr i32 [[X:%.*]], 5
+; CHECK-NEXT:    [[R:%.*]] = xor i32 [[SH0]], [[Y:%.*]]
+; CHECK-NEXT:    [[SH1:%.*]] = lshr i32 [[R]], 7
+; CHECK-NEXT:    ret i32 [[SH1]]
+;
+  %sh0 = ashr i32 %x, 5
+  %r = xor i32 %y, %sh0
+  %sh1 = lshr i32 %r, 7
+  ret i32 %sh1
+}
+
+define i32 @ashr_overshift_xor(i32 %x, i32 %y) {
+; CHECK-LABEL: @ashr_overshift_xor(
+; CHECK-NEXT:    [[SH0:%.*]] = ashr i32 [[X:%.*]], 15
+; CHECK-NEXT:    [[R:%.*]] = xor i32 [[SH0]], [[Y:%.*]]
+; CHECK-NEXT:    [[SH1:%.*]] = ashr i32 [[R]], 17
+; CHECK-NEXT:    ret i32 [[SH1]]
+;
+  %sh0 = ashr i32 %x, 15
+  %r = xor i32 %y, %sh0
+  %sh1 = ashr i32 %r, 17
+  ret i32 %sh1
+}
+
+define i32 @lshr_or_extra_use(i32 %x, i32 %y, i32* %p) {
+; CHECK-LABEL: @lshr_or_extra_use(
+; CHECK-NEXT:    [[SH0:%.*]] = lshr i32 [[X:%.*]], 5
+; CHECK-NEXT:    [[R:%.*]] = or i32 [[SH0]], [[Y:%.*]]
+; CHECK-NEXT:    store i32 [[R]], i32* [[P:%.*]], align 4
+; CHECK-NEXT:    [[SH1:%.*]] = lshr i32 [[R]], 7
+; CHECK-NEXT:    ret i32 [[SH1]]
+;
+  %sh0 = lshr i32 %x, 5
+  %r = or i32 %sh0, %y
+  store i32 %r, i32* %p
+  %sh1 = lshr i32 %r, 7
+  ret i32 %sh1
+}
+
+; Avoid crashing on constant expressions.
+
+@g = external global i32
+
+define i32 @PR44028(i32 %x) {
+; CHECK-LABEL: @PR44028(
+; CHECK-NEXT:    [[SH1:%.*]] = ashr exact i32 [[X:%.*]], 16
+; CHECK-NEXT:    [[T0:%.*]] = xor i32 [[SH1]], shl (i32 ptrtoint (i32* @g to i32), i32 16)
+; CHECK-NEXT:    [[T27:%.*]] = ashr exact i32 [[T0]], 16
+; CHECK-NEXT:    ret i32 [[T27]]
+;
+  %sh1 = ashr exact i32 %x, 16
+  %t0 = xor i32 %sh1, shl (i32 ptrtoint (i32* @g to i32), i32 16)
+  %t27 = ashr exact i32 %t0, 16
+  ret i32 %t27
+}
diff --git a/llvm/test/Transforms/InstCombine/shift.ll b/llvm/test/Transforms/InstCombine/shift.ll
index 9ac32439b..822f915d5 100644
--- a/llvm/test/Transforms/InstCombine/shift.ll
+++ b/llvm/test/Transforms/InstCombine/shift.ll
@@ -21,7 +21,7 @@ define <4 x i32> @shl_non_splat_vector(<4 x i32> %A) {
 
 define i32 @test6(i32 %A) {
 ; CHECK-LABEL: @test6(
-; CHECK-NEXT:    [[C:%.*]] = mul i32 %A, 6
+; CHECK-NEXT:    [[C:%.*]] = mul i32 [[A:%.*]], 6
 ; CHECK-NEXT:    ret i32 [[C]]
 ;
   %B = shl i32 %A, 1      ;; convert to an mul instruction
@@ -31,7 +31,7 @@ define i32 @test6(i32 %A) {
 
 define i32 @test6a(i32 %A) {
 ; CHECK-LABEL: @test6a(
-; CHECK-NEXT:    [[C:%.*]] = mul i32 %A, 6
+; CHECK-NEXT:    [[C:%.*]] = mul i32 [[A:%.*]], 6
 ; CHECK-NEXT:    ret i32 [[C]]
 ;
   %B = mul i32 %A, 3
@@ -52,7 +52,7 @@ define i8 @test8(i8 %A) {
 ;; (A << 7) >> 7 === A & 1
 define i8 @test9(i8 %A) {
 ; CHECK-LABEL: @test9(
-; CHECK-NEXT:    [[B:%.*]] = and i8 %A, 1
+; CHECK-NEXT:    [[B:%.*]] = and i8 [[A:%.*]], 1
 ; CHECK-NEXT:    ret i8 [[B]]
 ;
   %B = shl i8 %A, 7
@@ -64,7 +64,7 @@ define i8 @test9(i8 %A) {
 
 define i8 @test10(i8 %A) {
 ; CHECK-LABEL: @test10(
-; CHECK-NEXT:    [[B:%.*]] = and i8 %A, -128
+; CHECK-NEXT:    [[B:%.*]] = and i8 [[A:%.*]], -128
 ; CHECK-NEXT:    ret i8 [[B]]
 ;
   %B = lshr i8 %A, 7
@@ -75,7 +75,7 @@ define i8 @test10(i8 %A) {
 ;; Allow the simplification when the lshr shift is exact.
 define i8 @test10a(i8 %A) {
 ; CHECK-LABEL: @test10a(
-; CHECK-NEXT:    ret i8 %A
+; CHECK-NEXT:    ret i8 [[A:%.*]]
 ;
   %B = lshr exact i8 %A, 7
   %C = shl i8 %B, 7
@@ -85,14 +85,14 @@ define i8 @test10a(i8 %A) {
 ;; This transformation is deferred to DAGCombine:
 ;; (A >> 3) << 4 === (A & 0x1F) << 1
 ;; The shl may be valuable to scalar evolution.
-define i8 @test11(i8 %A) {
+define i8 @test11(i8 %x) {
 ; CHECK-LABEL: @test11(
-; CHECK-NEXT:    [[A:%.*]] = mul i8 %A, 3
+; CHECK-NEXT:    [[A:%.*]] = mul i8 [[X:%.*]], 3
 ; CHECK-NEXT:    [[B:%.*]] = lshr i8 [[A]], 3
 ; CHECK-NEXT:    [[C:%.*]] = shl i8 [[B]], 4
 ; CHECK-NEXT:    ret i8 [[C]]
 ;
-  %a = mul i8 %A, 3
+  %a = mul i8 %x, 3
   %B = lshr i8 %a, 3
   %C = shl i8 %B, 4
   ret i8 %C
@@ -101,7 +101,7 @@ define i8 @test11(i8 %A) {
 ;; Allow the simplification in InstCombine when the lshr shift is exact.
 define i8 @test11a(i8 %A) {
 ; CHECK-LABEL: @test11a(
-; CHECK-NEXT:    [[C:%.*]] = mul i8 %A, 6
+; CHECK-NEXT:    [[C:%.*]] = mul i8 [[A:%.*]], 6
 ; CHECK-NEXT:    ret i8 [[C]]
 ;
   %a = mul i8 %A, 3
@@ -114,8 +114,8 @@ define i8 @test11a(i8 %A) {
 ;; (A >> 8) << 8 === A & -256
 define i32 @test12(i32 %A) {
 ; CHECK-LABEL: @test12(
-; CHECK-NEXT:    [[B1:%.*]] = and i32 %A, -256
-; CHECK-NEXT:    ret i32 [[B1]]
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[A:%.*]], -256
+; CHECK-NEXT:    ret i32 [[TMP1]]
 ;
   %B = ashr i32 %A, 8
   %C = shl i32 %B, 8
@@ -141,14 +141,14 @@ define i8 @shishi(i8 %x) {
 ;; This transformation is deferred to DAGCombine:
 ;; (A >> 3) << 4 === (A & -8) * 2
 ;; The shl may be valuable to scalar evolution.
-define i8 @test13(i8 %A) {
+define i8 @test13(i8 %x) {
 ; CHECK-LABEL: @test13(
-; CHECK-NEXT:    [[A:%.*]] = mul i8 %A, 3
-; CHECK-NEXT:    [[B1:%.*]] = lshr i8 [[A]], 3
-; CHECK-NEXT:    [[C:%.*]] = shl i8 [[B1]], 4
+; CHECK-NEXT:    [[A:%.*]] = mul i8 [[X:%.*]], 3
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i8 [[A]], 3
+; CHECK-NEXT:    [[C:%.*]] = shl i8 [[TMP1]], 4
 ; CHECK-NEXT:    ret i8 [[C]]
 ;
-  %a = mul i8 %A, 3
+  %a = mul i8 %x, 3
   %B = ashr i8 %a, 3
   %C = shl i8 %B, 4
   ret i8 %C
@@ -156,7 +156,7 @@ define i8 @test13(i8 %A) {
 
 define i8 @test13a(i8 %A) {
 ; CHECK-LABEL: @test13a(
-; CHECK-NEXT:    [[C:%.*]] = mul i8 %A, 6
+; CHECK-NEXT:    [[C:%.*]] = mul i8 [[A:%.*]], 6
 ; CHECK-NEXT:    ret i8 [[C]]
 ;
   %a = mul i8 %A, 3
@@ -168,7 +168,7 @@ define i8 @test13a(i8 %A) {
 ;; D = ((B | 1234) << 4) === ((B << 4)|(1234 << 4)
 define i32 @test14(i32 %A) {
 ; CHECK-LABEL: @test14(
-; CHECK-NEXT:    [[B:%.*]] = and i32 %A, -19760
+; CHECK-NEXT:    [[B:%.*]] = and i32 [[A:%.*]], -19760
 ; CHECK-NEXT:    [[C:%.*]] = or i32 [[B]], 19744
 ; CHECK-NEXT:    ret i32 [[C]]
 ;
@@ -181,7 +181,7 @@ define i32 @test14(i32 %A) {
 ;; D = ((B | 1234) << 4) === ((B << 4)|(1234 << 4)
 define i32 @test14a(i32 %A) {
 ; CHECK-LABEL: @test14a(
-; CHECK-NEXT:    [[C:%.*]] = and i32 %A, 77
+; CHECK-NEXT:    [[C:%.*]] = and i32 [[A:%.*]], 77
 ; CHECK-NEXT:    ret i32 [[C]]
 ;
   %B = shl i32 %A, 4
@@ -192,7 +192,7 @@ define i32 @test14a(i32 %A) {
 
 define i32 @test15(i1 %C) {
 ; CHECK-LABEL: @test15(
-; CHECK-NEXT:    [[A:%.*]] = select i1 %C, i32 12, i32 4
+; CHECK-NEXT:    [[A:%.*]] = select i1 [[C:%.*]], i32 12, i32 4
 ; CHECK-NEXT:    ret i32 [[A]]
 ;
   %A = select i1 %C, i32 3, i32 1
@@ -202,7 +202,7 @@ define i32 @test15(i1 %C) {
 
 define i32 @test15a(i1 %C) {
 ; CHECK-LABEL: @test15a(
-; CHECK-NEXT:    [[V:%.*]] = select i1 %C, i32 512, i32 128
+; CHECK-NEXT:    [[V:%.*]] = select i1 [[C:%.*]], i32 512, i32 128
 ; CHECK-NEXT:    ret i32 [[V]]
 ;
   %A = select i1 %C, i8 3, i8 1
@@ -213,7 +213,7 @@ define i32 @test15a(i1 %C) {
 
 define i1 @test16(i32 %X) {
 ; CHECK-LABEL: @test16(
-; CHECK-NEXT:    [[TMP_6:%.*]] = and i32 %X, 16
+; CHECK-NEXT:    [[TMP_6:%.*]] = and i32 [[X:%.*]], 16
 ; CHECK-NEXT:    [[TMP_7:%.*]] = icmp ne i32 [[TMP_6]], 0
 ; CHECK-NEXT:    ret i1 [[TMP_7]]
 ;
@@ -225,7 +225,7 @@ define i1 @test16(i32 %X) {
 
 define i1 @test17(i32 %A) {
 ; CHECK-LABEL: @test17(
-; CHECK-NEXT:    [[B_MASK:%.*]] = and i32 %A, -8
+; CHECK-NEXT:    [[B_MASK:%.*]] = and i32 [[A:%.*]], -8
 ; CHECK-NEXT:    [[C:%.*]] = icmp eq i32 [[B_MASK]], 9872
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
@@ -236,7 +236,7 @@ define i1 @test17(i32 %A) {
 
 define <2 x i1> @test17vec(<2 x i32> %A) {
 ; CHECK-LABEL: @test17vec(
-; CHECK-NEXT:    [[B_MASK:%.*]] = and <2 x i32> %A, <i32 -8, i32 -8>
+; CHECK-NEXT:    [[B_MASK:%.*]] = and <2 x i32> [[A:%.*]], <i32 -8, i32 -8>
 ; CHECK-NEXT:    [[C:%.*]] = icmp eq <2 x i32> [[B_MASK]], <i32 9872, i32 9872>
 ; CHECK-NEXT:    ret <2 x i1> [[C]]
 ;
@@ -257,7 +257,7 @@ define i1 @test18(i8 %A) {
 
 define i1 @test19(i32 %A) {
 ; CHECK-LABEL: @test19(
-; CHECK-NEXT:    [[C:%.*]] = icmp ult i32 %A, 4
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i32 [[A:%.*]], 4
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %B = ashr i32 %A, 2
@@ -268,7 +268,7 @@ define i1 @test19(i32 %A) {
 
 define <2 x i1> @test19vec(<2 x i32> %A) {
 ; CHECK-LABEL: @test19vec(
-; CHECK-NEXT:    [[C:%.*]] = icmp ult <2 x i32> %A, <i32 4, i32 4>
+; CHECK-NEXT:    [[C:%.*]] = icmp ult <2 x i32> [[A:%.*]], <i32 4, i32 4>
 ; CHECK-NEXT:    ret <2 x i1> [[C]]
 ;
   %B = ashr <2 x i32> %A, <i32 2, i32 2>
@@ -279,7 +279,7 @@ define <2 x i1> @test19vec(<2 x i32> %A) {
 ;; X >u ~4
 define i1 @test19a(i32 %A) {
 ; CHECK-LABEL: @test19a(
-; CHECK-NEXT:    [[C:%.*]] = icmp ugt i32 %A, -5
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i32 [[A:%.*]], -5
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %B = ashr i32 %A, 2
@@ -289,7 +289,7 @@ define i1 @test19a(i32 %A) {
 
 define <2 x i1> @test19a_vec(<2 x i32> %A) {
 ; CHECK-LABEL: @test19a_vec(
-; CHECK-NEXT:    [[C:%.*]] = icmp ugt <2 x i32> %A, <i32 -5, i32 -5>
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt <2 x i32> [[A:%.*]], <i32 -5, i32 -5>
 ; CHECK-NEXT:    ret <2 x i1> [[C]]
 ;
   %B = ashr <2 x i32> %A, <i32 2, i32 2>
@@ -309,7 +309,7 @@ define i1 @test20(i8 %A) {
 
 define i1 @test21(i8 %A) {
 ; CHECK-LABEL: @test21(
-; CHECK-NEXT:    [[B_MASK:%.*]] = and i8 %A, 15
+; CHECK-NEXT:    [[B_MASK:%.*]] = and i8 [[A:%.*]], 15
 ; CHECK-NEXT:    [[C:%.*]] = icmp eq i8 [[B_MASK]], 8
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
@@ -320,7 +320,7 @@ define i1 @test21(i8 %A) {
 
 define i1 @test22(i8 %A) {
 ; CHECK-LABEL: @test22(
-; CHECK-NEXT:    [[B_MASK:%.*]] = and i8 %A, 15
+; CHECK-NEXT:    [[B_MASK:%.*]] = and i8 [[A:%.*]], 15
 ; CHECK-NEXT:    [[C:%.*]] = icmp eq i8 [[B_MASK]], 0
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
@@ -331,7 +331,7 @@ define i1 @test22(i8 %A) {
 
 define i8 @test23(i32 %A) {
 ; CHECK-LABEL: @test23(
-; CHECK-NEXT:    [[D:%.*]] = trunc i32 %A to i8
+; CHECK-NEXT:    [[D:%.*]] = trunc i32 [[A:%.*]] to i8
 ; CHECK-NEXT:    ret i8 [[D]]
 ;
   ;; casts not needed
@@ -343,7 +343,7 @@ define i8 @test23(i32 %A) {
 
 define i8 @test24(i8 %X) {
 ; CHECK-LABEL: @test24(
-; CHECK-NEXT:    [[Z:%.*]] = and i8 %X, 3
+; CHECK-NEXT:    [[Z:%.*]] = and i8 [[X:%.*]], 3
 ; CHECK-NEXT:    ret i8 [[Z]]
 ;
   %Y = and i8 %X, -5
@@ -354,8 +354,8 @@ define i8 @test24(i8 %X) {
 
 define i32 @test25(i32 %tmp.2, i32 %AA) {
 ; CHECK-LABEL: @test25(
-; CHECK-NEXT:    [[TMP_3:%.*]] = and i32 %tmp.2, -131072
-; CHECK-NEXT:    [[X2:%.*]] = add i32 [[TMP_3]], %AA
+; CHECK-NEXT:    [[TMP_3:%.*]] = and i32 [[TMP_2:%.*]], -131072
+; CHECK-NEXT:    [[X2:%.*]] = add i32 [[TMP_3]], [[AA:%.*]]
 ; CHECK-NEXT:    [[TMP_6:%.*]] = and i32 [[X2]], -131072
 ; CHECK-NEXT:    ret i32 [[TMP_6]]
 ;
@@ -368,8 +368,8 @@ define i32 @test25(i32 %tmp.2, i32 %AA) {
 
 define <2 x i32> @test25_vector(<2 x i32> %tmp.2, <2 x i32> %AA) {
 ; CHECK-LABEL: @test25_vector(
-; CHECK-NEXT:    [[TMP_3:%.*]] = and <2 x i32> %tmp.2, <i32 -131072, i32 -131072>
-; CHECK-NEXT:    [[X2:%.*]] = add <2 x i32> [[TMP_3]], %AA
+; CHECK-NEXT:    [[TMP_3:%.*]] = and <2 x i32> [[TMP_2:%.*]], <i32 -131072, i32 -131072>
+; CHECK-NEXT:    [[X2:%.*]] = add <2 x i32> [[TMP_3]], [[AA:%.*]]
 ; CHECK-NEXT:    [[TMP_6:%.*]] = and <2 x i32> [[X2]], <i32 -131072, i32 -131072>
 ; CHECK-NEXT:    ret <2 x i32> [[TMP_6]]
 ;
@@ -383,7 +383,7 @@ define <2 x i32> @test25_vector(<2 x i32> %tmp.2, <2 x i32> %AA) {
 ;; handle casts between shifts.
 define i32 @test26(i32 %A) {
 ; CHECK-LABEL: @test26(
-; CHECK-NEXT:    [[B:%.*]] = and i32 %A, -2
+; CHECK-NEXT:    [[B:%.*]] = and i32 [[A:%.*]], -2
 ; CHECK-NEXT:    ret i32 [[B]]
 ;
   %B = lshr i32 %A, 1
@@ -395,7 +395,7 @@ define i32 @test26(i32 %A) {
 
 define i1 @test27(i32 %x) nounwind {
 ; CHECK-LABEL: @test27(
-; CHECK-NEXT:    [[TMP1:%.*]] = and i32 %x, 8
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 8
 ; CHECK-NEXT:    [[Z:%.*]] = icmp ne i32 [[TMP1]], 0
 ; CHECK-NEXT:    ret i1 [[Z]]
 ;
@@ -406,7 +406,7 @@ define i1 @test27(i32 %x) nounwind {
 
 define i1 @test28(i8 %x) {
 ; CHECK-LABEL: @test28(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 %x, 0
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[X:%.*]], 0
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %shr = lshr i8 %x, 7
@@ -416,7 +416,7 @@ define i1 @test28(i8 %x) {
 
 define <2 x i1> @test28vec(<2 x i8> %x) {
 ; CHECK-LABEL: @test28vec(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i8> %x, zeroinitializer
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i8> [[X:%.*]], zeroinitializer
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %shr = lshr <2 x i8> %x, <i8 7, i8 7>
@@ -427,13 +427,13 @@ define <2 x i1> @test28vec(<2 x i8> %x) {
 define i8 @test28a(i8 %x, i8 %y) {
 ; CHECK-LABEL: @test28a(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP1:%.*]] = lshr i8 %x, 7
-; CHECK-NEXT:    [[COND1:%.*]] = icmp eq i8 [[TMP1]], 0
-; CHECK-NEXT:    br i1 [[COND1]], label %bb2, label %bb1
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i8 [[X:%.*]], 7
+; CHECK-NEXT:    [[COND1:%.*]] = icmp slt i8 [[X]], 0
+; CHECK-NEXT:    br i1 [[COND1]], label [[BB1:%.*]], label [[BB2:%.*]]
 ; CHECK:       bb1:
 ; CHECK-NEXT:    ret i8 [[TMP1]]
 ; CHECK:       bb2:
-; CHECK-NEXT:    [[TMP2:%.*]] = add i8 [[TMP1]], %y
+; CHECK-NEXT:    [[TMP2:%.*]] = add i8 [[TMP1]], [[Y:%.*]]
 ; CHECK-NEXT:    ret i8 [[TMP2]]
 ;
 entry:
@@ -452,7 +452,7 @@ bb2:
 define i32 @test29(i64 %d18) {
 ; CHECK-LABEL: @test29(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP916:%.*]] = lshr i64 %d18, 63
+; CHECK-NEXT:    [[TMP916:%.*]] = lshr i64 [[D18:%.*]], 63
 ; CHECK-NEXT:    [[TMP10:%.*]] = trunc i64 [[TMP916]] to i32
 ; CHECK-NEXT:    ret i32 [[TMP10]]
 ;
@@ -466,8 +466,8 @@ entry:
 
 define i32 @test30(i32 %A, i32 %B, i32 %C) {
 ; CHECK-LABEL: @test30(
-; CHECK-NEXT:    [[X1:%.*]] = and i32 %A, %B
-; CHECK-NEXT:    [[Z:%.*]] = shl i32 [[X1]], %C
+; CHECK-NEXT:    [[X1:%.*]] = and i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[Z:%.*]] = shl i32 [[X1]], [[C:%.*]]
 ; CHECK-NEXT:    ret i32 [[Z]]
 ;
   %X = shl i32 %A, %C
@@ -478,8 +478,8 @@ define i32 @test30(i32 %A, i32 %B, i32 %C) {
 
 define i32 @test31(i32 %A, i32 %B, i32 %C) {
 ; CHECK-LABEL: @test31(
-; CHECK-NEXT:    [[X1:%.*]] = or i32 %A, %B
-; CHECK-NEXT:    [[Z:%.*]] = lshr i32 [[X1]], %C
+; CHECK-NEXT:    [[X1:%.*]] = or i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[Z:%.*]] = lshr i32 [[X1]], [[C:%.*]]
 ; CHECK-NEXT:    ret i32 [[Z]]
 ;
   %X = lshr i32 %A, %C
@@ -490,8 +490,8 @@ define i32 @test31(i32 %A, i32 %B, i32 %C) {
 
 define i32 @test32(i32 %A, i32 %B, i32 %C) {
 ; CHECK-LABEL: @test32(
-; CHECK-NEXT:    [[X1:%.*]] = xor i32 %A, %B
-; CHECK-NEXT:    [[Z:%.*]] = ashr i32 [[X1]], %C
+; CHECK-NEXT:    [[X1:%.*]] = xor i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[Z:%.*]] = ashr i32 [[X1]], [[C:%.*]]
 ; CHECK-NEXT:    ret i32 [[Z]]
 ;
   %X = ashr i32 %A, %C
@@ -502,7 +502,7 @@ define i32 @test32(i32 %A, i32 %B, i32 %C) {
 
 define i1 @test33(i32 %X) {
 ; CHECK-LABEL: @test33(
-; CHECK-NEXT:    [[TMP1_MASK:%.*]] = and i32 %X, 16777216
+; CHECK-NEXT:    [[TMP1_MASK:%.*]] = and i32 [[X:%.*]], 16777216
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i32 [[TMP1_MASK]], 0
 ; CHECK-NEXT:    ret i1 [[TMP2]]
 ;
@@ -513,7 +513,7 @@ define i1 @test33(i32 %X) {
 
 define <2 x i1> @test33vec(<2 x i32> %X) {
 ; CHECK-LABEL: @test33vec(
-; CHECK-NEXT:    [[TMP1_MASK:%.*]] = and <2 x i32> %X, <i32 16777216, i32 16777216>
+; CHECK-NEXT:    [[TMP1_MASK:%.*]] = and <2 x i32> [[X:%.*]], <i32 16777216, i32 16777216>
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne <2 x i32> [[TMP1_MASK]], zeroinitializer
 ; CHECK-NEXT:    ret <2 x i1> [[TMP2]]
 ;
@@ -533,7 +533,7 @@ define i1 @test34(i32 %X) {
 
 define i1 @test35(i32 %X) {
 ; CHECK-LABEL: @test35(
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i32 %X, 0
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i32 [[X:%.*]], 0
 ; CHECK-NEXT:    ret i1 [[TMP2]]
 ;
   %tmp1 = ashr i32 %X, 7
@@ -543,7 +543,7 @@ define i1 @test35(i32 %X) {
 
 define <2 x i1> @test35vec(<2 x i32> %X) {
 ; CHECK-LABEL: @test35vec(
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt <2 x i32> %X, zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt <2 x i32> [[X:%.*]], zeroinitializer
 ; CHECK-NEXT:    ret <2 x i1> [[TMP2]]
 ;
   %tmp1 = ashr <2 x i32> %X, <i32 7, i32 7>
@@ -553,7 +553,7 @@ define <2 x i1> @test35vec(<2 x i32> %X) {
 
 define i128 @test36(i128 %A, i128 %B) {
 ; CHECK-LABEL: @test36(
-; CHECK-NEXT:    [[TMP231:%.*]] = or i128 %B, %A
+; CHECK-NEXT:    [[TMP231:%.*]] = or i128 [[B:%.*]], [[A:%.*]]
 ; CHECK-NEXT:    [[INS:%.*]] = and i128 [[TMP231]], 18446744073709551615
 ; CHECK-NEXT:    ret i128 [[INS]]
 ;
@@ -566,9 +566,9 @@ define i128 @test36(i128 %A, i128 %B) {
 
 define i64 @test37(i128 %A, i32 %B) {
 ; CHECK-LABEL: @test37(
-; CHECK-NEXT:    [[TMP22:%.*]] = zext i32 %B to i128
+; CHECK-NEXT:    [[TMP22:%.*]] = zext i32 [[B:%.*]] to i128
 ; CHECK-NEXT:    [[TMP23:%.*]] = shl nuw nsw i128 [[TMP22]], 32
-; CHECK-NEXT:    [[INS:%.*]] = or i128 [[TMP23]], %A
+; CHECK-NEXT:    [[INS:%.*]] = or i128 [[TMP23]], [[A:%.*]]
 ; CHECK-NEXT:    [[TMP46:%.*]] = trunc i128 [[INS]] to i64
 ; CHECK-NEXT:    ret i64 [[TMP46]]
 ;
@@ -583,7 +583,7 @@ define i64 @test37(i128 %A, i32 %B) {
 
 define <2 x i32> @shl_nuw_nsw_splat_vec(<2 x i8> %x) {
 ; CHECK-LABEL: @shl_nuw_nsw_splat_vec(
-; CHECK-NEXT:    [[T2:%.*]] = zext <2 x i8> %x to <2 x i32>
+; CHECK-NEXT:    [[T2:%.*]] = zext <2 x i8> [[X:%.*]] to <2 x i32>
 ; CHECK-NEXT:    [[T3:%.*]] = shl nuw nsw <2 x i32> [[T2]], <i32 17, i32 17>
 ; CHECK-NEXT:    ret <2 x i32> [[T3]]
 ;
@@ -594,7 +594,7 @@ define <2 x i32> @shl_nuw_nsw_splat_vec(<2 x i8> %x) {
 
 define i32 @test38(i32 %x) nounwind readnone {
 ; CHECK-LABEL: @test38(
-; CHECK-NEXT:    [[REM1:%.*]] = and i32 %x, 31
+; CHECK-NEXT:    [[REM1:%.*]] = and i32 [[X:%.*]], 31
 ; CHECK-NEXT:    [[SHL:%.*]] = shl i32 1, [[REM1]]
 ; CHECK-NEXT:    ret i32 [[SHL]]
 ;
@@ -607,7 +607,7 @@ define i32 @test38(i32 %x) nounwind readnone {
 define i8 @test39(i32 %a0) {
 ; CHECK-LABEL: @test39(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP4:%.*]] = trunc i32 %a0 to i8
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc i32 [[A0:%.*]] to i8
 ; CHECK-NEXT:    [[TMP5:%.*]] = shl i8 [[TMP4]], 5
 ; CHECK-NEXT:    [[TMP49:%.*]] = shl i8 [[TMP4]], 6
 ; CHECK-NEXT:    [[TMP50:%.*]] = and i8 [[TMP49]], 64
@@ -634,8 +634,8 @@ entry:
 ; PR9809
 define i32 @test40(i32 %a, i32 %b) nounwind {
 ; CHECK-LABEL: @test40(
-; CHECK-NEXT:    [[TMP1:%.*]] = add i32 %b, 2
-; CHECK-NEXT:    [[DIV:%.*]] = lshr i32 %a, [[TMP1]]
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[B:%.*]], 2
+; CHECK-NEXT:    [[DIV:%.*]] = lshr i32 [[A:%.*]], [[TMP1]]
 ; CHECK-NEXT:    ret i32 [[DIV]]
 ;
   %shl1 = shl i32 1, %b
@@ -646,7 +646,7 @@ define i32 @test40(i32 %a, i32 %b) nounwind {
 
 define i32 @test41(i32 %a, i32 %b) nounwind {
 ; CHECK-LABEL: @test41(
-; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 8, %b
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 8, [[B:%.*]]
 ; CHECK-NEXT:    ret i32 [[TMP1]]
 ;
   %1 = shl i32 1, %b
@@ -656,8 +656,8 @@ define i32 @test41(i32 %a, i32 %b) nounwind {
 
 define i32 @test42(i32 %a, i32 %b) nounwind {
 ; CHECK-LABEL: @test42(
-; CHECK-NEXT:    [[DIV:%.*]] = lshr exact i32 4096, %b
-; CHECK-NEXT:    [[DIV2:%.*]] = udiv i32 %a, [[DIV]]
+; CHECK-NEXT:    [[DIV:%.*]] = lshr exact i32 4096, [[B:%.*]]
+; CHECK-NEXT:    [[DIV2:%.*]] = udiv i32 [[A:%.*]], [[DIV]]
 ; CHECK-NEXT:    ret i32 [[DIV2]]
 ;
   %div = lshr i32 4096, %b    ; must be exact otherwise we'd divide by zero
@@ -667,8 +667,8 @@ define i32 @test42(i32 %a, i32 %b) nounwind {
 
 define <2 x i32> @test42vec(<2 x i32> %a, <2 x i32> %b) {
 ; CHECK-LABEL: @test42vec(
-; CHECK-NEXT:    [[DIV:%.*]] = lshr exact <2 x i32> <i32 4096, i32 4096>, %b
-; CHECK-NEXT:    [[DIV2:%.*]] = udiv <2 x i32> %a, [[DIV]]
+; CHECK-NEXT:    [[DIV:%.*]] = lshr exact <2 x i32> <i32 4096, i32 4096>, [[B:%.*]]
+; CHECK-NEXT:    [[DIV2:%.*]] = udiv <2 x i32> [[A:%.*]], [[DIV]]
 ; CHECK-NEXT:    ret <2 x i32> [[DIV2]]
 ;
   %div = lshr <2 x i32> <i32 4096, i32 4096>, %b    ; must be exact otherwise we'd divide by zero
@@ -678,8 +678,8 @@ define <2 x i32> @test42vec(<2 x i32> %a, <2 x i32> %b) {
 
 define i32 @test43(i32 %a, i32 %b) nounwind {
 ; CHECK-LABEL: @test43(
-; CHECK-NEXT:    [[TMP1:%.*]] = add i32 %b, 12
-; CHECK-NEXT:    [[DIV2:%.*]] = lshr i32 %a, [[TMP1]]
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[B:%.*]], 12
+; CHECK-NEXT:    [[DIV2:%.*]] = lshr i32 [[A:%.*]], [[TMP1]]
 ; CHECK-NEXT:    ret i32 [[DIV2]]
 ;
   %div = shl i32 4096, %b    ; must be exact otherwise we'd divide by zero
@@ -689,7 +689,7 @@ define i32 @test43(i32 %a, i32 %b) nounwind {
 
 define i32 @test44(i32 %a) nounwind {
 ; CHECK-LABEL: @test44(
-; CHECK-NEXT:    [[Y:%.*]] = shl i32 %a, 5
+; CHECK-NEXT:    [[Y:%.*]] = shl i32 [[A:%.*]], 5
 ; CHECK-NEXT:    ret i32 [[Y]]
 ;
   %y = shl nuw i32 %a, 1
@@ -699,7 +699,7 @@ define i32 @test44(i32 %a) nounwind {
 
 define i32 @test45(i32 %a) nounwind {
 ; CHECK-LABEL: @test45(
-; CHECK-NEXT:    [[Y:%.*]] = lshr i32 %a, 5
+; CHECK-NEXT:    [[Y:%.*]] = lshr i32 [[A:%.*]], 5
 ; CHECK-NEXT:    ret i32 [[Y]]
 ;
   %y = lshr exact i32 %a, 1
@@ -711,7 +711,7 @@ define i32 @test45(i32 %a) nounwind {
 
 define i32 @test46(i32 %a) {
 ; CHECK-LABEL: @test46(
-; CHECK-NEXT:    [[Z:%.*]] = ashr exact i32 %a, 2
+; CHECK-NEXT:    [[Z:%.*]] = ashr exact i32 [[A:%.*]], 2
 ; CHECK-NEXT:    ret i32 [[Z]]
 ;
   %y = ashr exact i32 %a, 3
@@ -723,7 +723,7 @@ define i32 @test46(i32 %a) {
 
 define <2 x i32> @test46_splat_vec(<2 x i32> %a) {
 ; CHECK-LABEL: @test46_splat_vec(
-; CHECK-NEXT:    [[Z:%.*]] = ashr exact <2 x i32> %a, <i32 2, i32 2>
+; CHECK-NEXT:    [[Z:%.*]] = ashr exact <2 x i32> [[A:%.*]], <i32 2, i32 2>
 ; CHECK-NEXT:    ret <2 x i32> [[Z]]
 ;
   %y = ashr exact <2 x i32> %a, <i32 3, i32 3>
@@ -735,7 +735,7 @@ define <2 x i32> @test46_splat_vec(<2 x i32> %a) {
 
 define i8 @test47(i8 %a) {
 ; CHECK-LABEL: @test47(
-; CHECK-NEXT:    [[Z:%.*]] = lshr exact i8 %a, 2
+; CHECK-NEXT:    [[Z:%.*]] = lshr exact i8 [[A:%.*]], 2
 ; CHECK-NEXT:    ret i8 [[Z]]
 ;
   %y = lshr exact i8 %a, 3
@@ -747,7 +747,7 @@ define i8 @test47(i8 %a) {
 
 define <2 x i8> @test47_splat_vec(<2 x i8> %a) {
 ; CHECK-LABEL: @test47_splat_vec(
-; CHECK-NEXT:    [[Z:%.*]] = lshr exact <2 x i8> %a, <i8 2, i8 2>
+; CHECK-NEXT:    [[Z:%.*]] = lshr exact <2 x i8> [[A:%.*]], <i8 2, i8 2>
 ; CHECK-NEXT:    ret <2 x i8> [[Z]]
 ;
   %y = lshr exact <2 x i8> %a, <i8 3, i8 3>
@@ -759,7 +759,7 @@ define <2 x i8> @test47_splat_vec(<2 x i8> %a) {
 
 define i32 @test48(i32 %x) {
 ; CHECK-LABEL: @test48(
-; CHECK-NEXT:    [[B:%.*]] = shl i32 %x, 2
+; CHECK-NEXT:    [[B:%.*]] = shl i32 [[X:%.*]], 2
 ; CHECK-NEXT:    ret i32 [[B]]
 ;
   %A = lshr exact i32 %x, 1
@@ -771,7 +771,7 @@ define i32 @test48(i32 %x) {
 
 define i32 @test48_nuw_nsw(i32 %x) {
 ; CHECK-LABEL: @test48_nuw_nsw(
-; CHECK-NEXT:    [[B:%.*]] = shl nuw nsw i32 %x, 2
+; CHECK-NEXT:    [[B:%.*]] = shl nuw nsw i32 [[X:%.*]], 2
 ; CHECK-NEXT:    ret i32 [[B]]
 ;
   %A = lshr exact i32 %x, 1
@@ -783,7 +783,7 @@ define i32 @test48_nuw_nsw(i32 %x) {
 
 define <2 x i32> @test48_splat_vec(<2 x i32> %x) {
 ; CHECK-LABEL: @test48_splat_vec(
-; CHECK-NEXT:    [[B:%.*]] = shl nuw nsw <2 x i32> %x, <i32 2, i32 2>
+; CHECK-NEXT:    [[B:%.*]] = shl nuw nsw <2 x i32> [[X:%.*]], <i32 2, i32 2>
 ; CHECK-NEXT:    ret <2 x i32> [[B]]
 ;
   %A = lshr exact <2 x i32> %x, <i32 1, i32 1>
@@ -795,7 +795,7 @@ define <2 x i32> @test48_splat_vec(<2 x i32> %x) {
 
 define i32 @test49(i32 %x) {
 ; CHECK-LABEL: @test49(
-; CHECK-NEXT:    [[B:%.*]] = shl i32 %x, 2
+; CHECK-NEXT:    [[B:%.*]] = shl i32 [[X:%.*]], 2
 ; CHECK-NEXT:    ret i32 [[B]]
 ;
   %A = ashr exact i32 %x, 1
@@ -807,7 +807,7 @@ define i32 @test49(i32 %x) {
 
 define i32 @test49_nuw_nsw(i32 %x) {
 ; CHECK-LABEL: @test49_nuw_nsw(
-; CHECK-NEXT:    [[B:%.*]] = shl nuw nsw i32 %x, 2
+; CHECK-NEXT:    [[B:%.*]] = shl nuw nsw i32 [[X:%.*]], 2
 ; CHECK-NEXT:    ret i32 [[B]]
 ;
   %A = ashr exact i32 %x, 1
@@ -819,7 +819,7 @@ define i32 @test49_nuw_nsw(i32 %x) {
 
 define <2 x i32> @test49_splat_vec(<2 x i32> %x) {
 ; CHECK-LABEL: @test49_splat_vec(
-; CHECK-NEXT:    [[B:%.*]] = shl nuw nsw <2 x i32> %x, <i32 2, i32 2>
+; CHECK-NEXT:    [[B:%.*]] = shl nuw nsw <2 x i32> [[X:%.*]], <i32 2, i32 2>
 ; CHECK-NEXT:    ret <2 x i32> [[B]]
 ;
   %A = ashr exact <2 x i32> %x, <i32 1, i32 1>
@@ -831,7 +831,7 @@ define <2 x i32> @test49_splat_vec(<2 x i32> %x) {
 
 define i32 @test50(i32 %x) {
 ; CHECK-LABEL: @test50(
-; CHECK-NEXT:    [[B:%.*]] = ashr i32 %x, 2
+; CHECK-NEXT:    [[B:%.*]] = ashr i32 [[X:%.*]], 2
 ; CHECK-NEXT:    ret i32 [[B]]
 ;
   %A = shl nsw i32 %x, 1
@@ -844,7 +844,7 @@ define i32 @test50(i32 %x) {
 
 define <2 x i32> @test50_splat_vec(<2 x i32> %x) {
 ; CHECK-LABEL: @test50_splat_vec(
-; CHECK-NEXT:    [[B:%.*]] = ashr exact <2 x i32> %x, <i32 2, i32 2>
+; CHECK-NEXT:    [[B:%.*]] = ashr exact <2 x i32> [[X:%.*]], <i32 2, i32 2>
 ; CHECK-NEXT:    ret <2 x i32> [[B]]
 ;
   %A = shl nsw <2 x i32> %x, <i32 1, i32 1>
@@ -856,7 +856,7 @@ define <2 x i32> @test50_splat_vec(<2 x i32> %x) {
 
 define i32 @test51(i32 %x) {
 ; CHECK-LABEL: @test51(
-; CHECK-NEXT:    [[B:%.*]] = lshr i32 %x, 2
+; CHECK-NEXT:    [[B:%.*]] = lshr i32 [[X:%.*]], 2
 ; CHECK-NEXT:    ret i32 [[B]]
 ;
   %A = shl nuw i32 %x, 1
@@ -869,7 +869,7 @@ define i32 @test51(i32 %x) {
 
 define <2 x i32> @test51_splat_vec(<2 x i32> %x) {
 ; CHECK-LABEL: @test51_splat_vec(
-; CHECK-NEXT:    [[B:%.*]] = lshr exact <2 x i32> %x, <i32 2, i32 2>
+; CHECK-NEXT:    [[B:%.*]] = lshr exact <2 x i32> [[X:%.*]], <i32 2, i32 2>
 ; CHECK-NEXT:    ret <2 x i32> [[B]]
 ;
   %A = shl nuw <2 x i32> %x, <i32 1, i32 1>
@@ -882,7 +882,7 @@ define <2 x i32> @test51_splat_vec(<2 x i32> %x) {
 
 define i32 @test51_no_nuw(i32 %x) {
 ; CHECK-LABEL: @test51_no_nuw(
-; CHECK-NEXT:    [[TMP1:%.*]] = lshr exact i32 %x, 2
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr exact i32 [[X:%.*]], 2
 ; CHECK-NEXT:    [[B:%.*]] = and i32 [[TMP1]], 536870911
 ; CHECK-NEXT:    ret i32 [[B]]
 ;
@@ -895,7 +895,7 @@ define i32 @test51_no_nuw(i32 %x) {
 
 define <2 x i32> @test51_no_nuw_splat_vec(<2 x i32> %x) {
 ; CHECK-LABEL: @test51_no_nuw_splat_vec(
-; CHECK-NEXT:    [[TMP1:%.*]] = lshr <2 x i32> %x, <i32 2, i32 2>
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <2 x i32> [[X:%.*]], <i32 2, i32 2>
 ; CHECK-NEXT:    [[B:%.*]] = and <2 x i32> [[TMP1]], <i32 536870911, i32 536870911>
 ; CHECK-NEXT:    ret <2 x i32> [[B]]
 ;
@@ -908,7 +908,7 @@ define <2 x i32> @test51_no_nuw_splat_vec(<2 x i32> %x) {
 
 define i32 @test52(i32 %x) {
 ; CHECK-LABEL: @test52(
-; CHECK-NEXT:    [[B:%.*]] = shl nsw i32 %x, 2
+; CHECK-NEXT:    [[B:%.*]] = shl nsw i32 [[X:%.*]], 2
 ; CHECK-NEXT:    ret i32 [[B]]
 ;
   %A = shl nsw i32 %x, 3
@@ -920,7 +920,7 @@ define i32 @test52(i32 %x) {
 
 define <2 x i32> @test52_splat_vec(<2 x i32> %x) {
 ; CHECK-LABEL: @test52_splat_vec(
-; CHECK-NEXT:    [[B:%.*]] = shl nsw <2 x i32> %x, <i32 2, i32 2>
+; CHECK-NEXT:    [[B:%.*]] = shl nsw <2 x i32> [[X:%.*]], <i32 2, i32 2>
 ; CHECK-NEXT:    ret <2 x i32> [[B]]
 ;
   %A = shl nsw <2 x i32> %x, <i32 3, i32 3>
@@ -932,7 +932,7 @@ define <2 x i32> @test52_splat_vec(<2 x i32> %x) {
 
 define i32 @test53(i32 %x) {
 ; CHECK-LABEL: @test53(
-; CHECK-NEXT:    [[B:%.*]] = shl nuw i32 %x, 2
+; CHECK-NEXT:    [[B:%.*]] = shl nuw i32 [[X:%.*]], 2
 ; CHECK-NEXT:    ret i32 [[B]]
 ;
   %A = shl nuw i32 %x, 3
@@ -944,7 +944,7 @@ define i32 @test53(i32 %x) {
 
 define <2 x i32> @test53_splat_vec(<2 x i32> %x) {
 ; CHECK-LABEL: @test53_splat_vec(
-; CHECK-NEXT:    [[B:%.*]] = shl nuw <2 x i32> %x, <i32 2, i32 2>
+; CHECK-NEXT:    [[B:%.*]] = shl nuw <2 x i32> [[X:%.*]], <i32 2, i32 2>
 ; CHECK-NEXT:    ret <2 x i32> [[B]]
 ;
   %A = shl nuw <2 x i32> %x, <i32 3, i32 3>
@@ -956,7 +956,7 @@ define <2 x i32> @test53_splat_vec(<2 x i32> %x) {
 
 define i8 @test53_no_nuw(i8 %x) {
 ; CHECK-LABEL: @test53_no_nuw(
-; CHECK-NEXT:    [[TMP1:%.*]] = shl i8 %x, 2
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i8 [[X:%.*]], 2
 ; CHECK-NEXT:    [[B:%.*]] = and i8 [[TMP1]], 124
 ; CHECK-NEXT:    ret i8 [[B]]
 ;
@@ -969,7 +969,7 @@ define i8 @test53_no_nuw(i8 %x) {
 
 define <2 x i8> @test53_no_nuw_splat_vec(<2 x i8> %x) {
 ; CHECK-LABEL: @test53_no_nuw_splat_vec(
-; CHECK-NEXT:    [[TMP1:%.*]] = shl <2 x i8> %x, <i8 2, i8 2>
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <2 x i8> [[X:%.*]], <i8 2, i8 2>
 ; CHECK-NEXT:    [[B:%.*]] = and <2 x i8> [[TMP1]], <i8 124, i8 124>
 ; CHECK-NEXT:    ret <2 x i8> [[B]]
 ;
@@ -980,7 +980,7 @@ define <2 x i8> @test53_no_nuw_splat_vec(<2 x i8> %x) {
 
 define i32 @test54(i32 %x) {
 ; CHECK-LABEL: @test54(
-; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 %x, 3
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 [[X:%.*]], 3
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[TMP1]], 16
 ; CHECK-NEXT:    ret i32 [[AND]]
 ;
@@ -992,7 +992,7 @@ define i32 @test54(i32 %x) {
 
 define <2 x i32> @test54_splat_vec(<2 x i32> %x) {
 ; CHECK-LABEL: @test54_splat_vec(
-; CHECK-NEXT:    [[TMP1:%.*]] = shl <2 x i32> %x, <i32 3, i32 3>
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <2 x i32> [[X:%.*]], <i32 3, i32 3>
 ; CHECK-NEXT:    [[AND:%.*]] = and <2 x i32> [[TMP1]], <i32 16, i32 16>
 ; CHECK-NEXT:    ret <2 x i32> [[AND]]
 ;
@@ -1004,7 +1004,7 @@ define <2 x i32> @test54_splat_vec(<2 x i32> %x) {
 
 define i32 @test55(i32 %x) {
 ; CHECK-LABEL: @test55(
-; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 %x, 3
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 [[X:%.*]], 3
 ; CHECK-NEXT:    [[OR:%.*]] = or i32 [[TMP1]], 8
 ; CHECK-NEXT:    ret i32 [[OR]]
 ;
@@ -1016,7 +1016,7 @@ define i32 @test55(i32 %x) {
 
 define i32 @test56(i32 %x) {
 ; CHECK-LABEL: @test56(
-; CHECK-NEXT:    [[SHR2:%.*]] = lshr i32 %x, 1
+; CHECK-NEXT:    [[SHR2:%.*]] = lshr i32 [[X:%.*]], 1
 ; CHECK-NEXT:    [[SHL:%.*]] = shl i32 [[SHR2]], 4
 ; CHECK-NEXT:    [[OR:%.*]] = or i32 [[SHL]], 7
 ; CHECK-NEXT:    ret i32 [[OR]]
@@ -1029,8 +1029,8 @@ define i32 @test56(i32 %x) {
 
 define i32 @test57(i32 %x) {
 ; CHECK-LABEL: @test57(
-; CHECK-NEXT:    [[SHR1:%.*]] = lshr i32 %x, 1
-; CHECK-NEXT:    [[SHL:%.*]] = shl i32 [[SHR1]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i32 [[X:%.*]], 1
+; CHECK-NEXT:    [[SHL:%.*]] = shl i32 [[TMP1]], 4
 ; CHECK-NEXT:    [[OR:%.*]] = or i32 [[SHL]], 7
 ; CHECK-NEXT:    ret i32 [[OR]]
 ;
@@ -1042,7 +1042,7 @@ define i32 @test57(i32 %x) {
 
 define i32 @test58(i32 %x) {
 ; CHECK-LABEL: @test58(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr i32 %x, 3
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr i32 [[X:%.*]], 3
 ; CHECK-NEXT:    [[OR:%.*]] = or i32 [[TMP1]], 1
 ; CHECK-NEXT:    ret i32 [[OR]]
 ;
@@ -1054,7 +1054,7 @@ define i32 @test58(i32 %x) {
 
 define <2 x i32> @test58_splat_vec(<2 x i32> %x) {
 ; CHECK-LABEL: @test58_splat_vec(
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr <2 x i32> %x, <i32 3, i32 3>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <2 x i32> [[X:%.*]], <i32 3, i32 3>
 ; CHECK-NEXT:    [[OR:%.*]] = or <2 x i32> [[TMP1]], <i32 1, i32 1>
 ; CHECK-NEXT:    ret <2 x i32> [[OR]]
 ;
@@ -1066,7 +1066,7 @@ define <2 x i32> @test58_splat_vec(<2 x i32> %x) {
 
 define i32 @test59(i32 %x) {
 ; CHECK-LABEL: @test59(
-; CHECK-NEXT:    [[SHR:%.*]] = ashr i32 %x, 4
+; CHECK-NEXT:    [[SHR:%.*]] = ashr i32 [[X:%.*]], 4
 ; CHECK-NEXT:    [[SHL:%.*]] = shl nsw i32 [[SHR]], 1
 ; CHECK-NEXT:    [[OR:%.*]] = or i32 [[SHL]], 2
 ; CHECK-NEXT:    ret i32 [[OR]]
@@ -1080,7 +1080,7 @@ define i32 @test59(i32 %x) {
 ; propagate "exact" trait
 define i32 @test60(i32 %x) {
 ; CHECK-LABEL: @test60(
-; CHECK-NEXT:    [[SHL:%.*]] = ashr exact i32 %x, 3
+; CHECK-NEXT:    [[SHL:%.*]] = ashr exact i32 [[X:%.*]], 3
 ; CHECK-NEXT:    [[OR:%.*]] = or i32 [[SHL]], 1
 ; CHECK-NEXT:    ret i32 [[OR]]
 ;
@@ -1094,19 +1094,19 @@ define i32 @test60(i32 %x) {
 define void @test61(i128 %arg) {
 ; CHECK-LABEL: @test61(
 ; CHECK-NEXT:  bb:
-; CHECK-NEXT:    br i1 undef, label %bb1, label %bb12
+; CHECK-NEXT:    br i1 undef, label [[BB1:%.*]], label [[BB12:%.*]]
 ; CHECK:       bb1:
-; CHECK-NEXT:    br label %bb2
+; CHECK-NEXT:    br label [[BB2:%.*]]
 ; CHECK:       bb2:
-; CHECK-NEXT:    br i1 undef, label %bb3, label %bb7
+; CHECK-NEXT:    br i1 undef, label [[BB3:%.*]], label [[BB7:%.*]]
 ; CHECK:       bb3:
-; CHECK-NEXT:    br label %bb8
+; CHECK-NEXT:    br label [[BB8:%.*]]
 ; CHECK:       bb7:
-; CHECK-NEXT:    br i1 undef, label %bb8, label %bb2
+; CHECK-NEXT:    br i1 undef, label [[BB8]], label [[BB2]]
 ; CHECK:       bb8:
-; CHECK-NEXT:    br i1 undef, label %bb11, label %bb12
+; CHECK-NEXT:    br i1 undef, label [[BB11:%.*]], label [[BB12]]
 ; CHECK:       bb11:
-; CHECK-NEXT:    br i1 undef, label %bb1, label %bb12
+; CHECK-NEXT:    br i1 undef, label [[BB1]], label [[BB12]]
 ; CHECK:       bb12:
 ; CHECK-NEXT:    ret void
 ;
@@ -1159,7 +1159,7 @@ define <4 x i32> @test62_splat_vector(<4 x i32> %a) {
 
 define <4 x i32> @test62_non_splat_vector(<4 x i32> %a) {
 ; CHECK-LABEL: @test62_non_splat_vector(
-; CHECK-NEXT:    [[B:%.*]] = ashr <4 x i32> %a, <i32 32, i32 0, i32 1, i32 2>
+; CHECK-NEXT:    [[B:%.*]] = ashr <4 x i32> [[A:%.*]], <i32 32, i32 0, i32 1, i32 2>
 ; CHECK-NEXT:    ret <4 x i32> [[B]]
 ;
   %b = ashr <4 x i32> %a, <i32 32, i32 0, i32 1, i32 2>  ; shift all bits out
@@ -1168,7 +1168,7 @@ define <4 x i32> @test62_non_splat_vector(<4 x i32> %a) {
 
 define <2 x i65> @test_63(<2 x i64> %t) {
 ; CHECK-LABEL: @test_63(
-; CHECK-NEXT:    [[A:%.*]] = zext <2 x i64> %t to <2 x i65>
+; CHECK-NEXT:    [[A:%.*]] = zext <2 x i64> [[T:%.*]] to <2 x i65>
 ; CHECK-NEXT:    [[SEXT:%.*]] = shl <2 x i65> [[A]], <i65 33, i65 33>
 ; CHECK-NEXT:    [[B:%.*]] = ashr exact <2 x i65> [[SEXT]], <i65 33, i65 33>
 ; CHECK-NEXT:    ret <2 x i65> [[B]]
@@ -1179,9 +1179,50 @@ define <2 x i65> @test_63(<2 x i64> %t) {
   ret <2 x i65> %b
 }
 
-define i64 @test_64(i32 %t) {
-; CHECK-LABEL: @test_64(
-; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 %t, 8
+define i32 @test_shl_zext_bool(i1 %t) {
+; CHECK-LABEL: @test_shl_zext_bool(
+; CHECK-NEXT:    [[SHL:%.*]] = select i1 [[T:%.*]], i32 4, i32 0
+; CHECK-NEXT:    ret i32 [[SHL]]
+;
+  %ext = zext i1 %t to i32
+  %shl = shl i32 %ext, 2
+  ret i32 %shl
+}
+
+define <2 x i32> @test_shl_zext_bool_splat(<2 x i1> %t) {
+; CHECK-LABEL: @test_shl_zext_bool_splat(
+; CHECK-NEXT:    [[SHL:%.*]] = select <2 x i1> [[T:%.*]], <2 x i32> <i32 8, i32 8>, <2 x i32> zeroinitializer
+; CHECK-NEXT:    ret <2 x i32> [[SHL]]
+;
+  %ext = zext <2 x i1> %t to <2 x i32>
+  %shl = shl <2 x i32> %ext, <i32 3, i32 3>
+  ret <2 x i32> %shl
+}
+
+define <2 x i32> @test_shl_zext_bool_vec(<2 x i1> %t) {
+; CHECK-LABEL: @test_shl_zext_bool_vec(
+; CHECK-NEXT:    [[SHL:%.*]] = select <2 x i1> [[T:%.*]], <2 x i32> <i32 4, i32 8>, <2 x i32> zeroinitializer
+; CHECK-NEXT:    ret <2 x i32> [[SHL]]
+;
+  %ext = zext <2 x i1> %t to <2 x i32>
+  %shl = shl <2 x i32> %ext, <i32 2, i32 3>
+  ret <2 x i32> %shl
+}
+
+define i32 @test_shl_zext_bool_not_constant(i1 %cmp, i32 %shamt) {
+; CHECK-LABEL: @test_shl_zext_bool_not_constant(
+; CHECK-NEXT:    [[CONV3:%.*]] = zext i1 [[CMP:%.*]] to i32
+; CHECK-NEXT:    [[SHL:%.*]] = shl i32 [[CONV3]], [[SHAMT:%.*]]
+; CHECK-NEXT:    ret i32 [[SHL]]
+;
+  %conv3 = zext i1 %cmp to i32
+  %shl = shl i32 %conv3, %shamt
+  ret i32 %shl
+}
+
+define i64 @shl_zext(i32 %t) {
+; CHECK-LABEL: @shl_zext(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 [[T:%.*]], 8
 ; CHECK-NEXT:    [[SHL:%.*]] = zext i32 [[TMP1]] to i64
 ; CHECK-NEXT:    ret i64 [[SHL]]
 ;
@@ -1191,9 +1232,27 @@ define i64 @test_64(i32 %t) {
   ret i64 %shl
 }
 
-define <2 x i64> @test_64_splat_vec(<2 x i32> %t) {
-; CHECK-LABEL: @test_64_splat_vec(
-; CHECK-NEXT:    [[TMP1:%.*]] = shl <2 x i32> %t, <i32 8, i32 8>
+declare void @use(i64)
+
+define i64 @shl_zext_extra_use(i32 %t) {
+; CHECK-LABEL: @shl_zext_extra_use(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[T:%.*]], 16777215
+; CHECK-NEXT:    [[EXT:%.*]] = zext i32 [[AND]] to i64
+; CHECK-NEXT:    call void @use(i64 [[EXT]])
+; CHECK-NEXT:    [[SHL:%.*]] = shl nuw nsw i64 [[EXT]], 8
+; CHECK-NEXT:    ret i64 [[SHL]]
+;
+  %and = and i32 %t, 16777215
+  %ext = zext i32 %and to i64
+  call void @use(i64 %ext)
+  %shl = shl i64 %ext, 8
+  ret i64 %shl
+}
+
+
+define <2 x i64> @shl_zext_splat_vec(<2 x i32> %t) {
+; CHECK-LABEL: @shl_zext_splat_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <2 x i32> [[T:%.*]], <i32 8, i32 8>
 ; CHECK-NEXT:    [[SHL:%.*]] = zext <2 x i32> [[TMP1]] to <2 x i64>
 ; CHECK-NEXT:    ret <2 x i64> [[SHL]]
 ;
@@ -1203,9 +1262,80 @@ define <2 x i64> @test_64_splat_vec(<2 x i32> %t) {
   ret <2 x i64> %shl
 }
 
+define i64 @shl_zext_mul(i32 %t) {
+; CHECK-LABEL: @shl_zext_mul(
+; CHECK-NEXT:    [[MUL:%.*]] = mul i32 [[T:%.*]], 16777215
+; CHECK-NEXT:    [[EXT:%.*]] = zext i32 [[MUL]] to i64
+; CHECK-NEXT:    [[SHL:%.*]] = shl nuw i64 [[EXT]], 32
+; CHECK-NEXT:    ret i64 [[SHL]]
+;
+  %mul = mul i32 %t, 16777215
+  %ext = zext i32 %mul to i64
+  %shl = shl i64 %ext, 32
+  ret i64 %shl
+}
+
+define <3 x i17> @shl_zext_mul_splat(<3 x i5> %t) {
+; CHECK-LABEL: @shl_zext_mul_splat(
+; CHECK-NEXT:    [[MUL:%.*]] = mul <3 x i5> [[T:%.*]], <i5 13, i5 13, i5 13>
+; CHECK-NEXT:    [[EXT:%.*]] = zext <3 x i5> [[MUL]] to <3 x i17>
+; CHECK-NEXT:    [[SHL:%.*]] = shl nuw <3 x i17> [[EXT]], <i17 12, i17 12, i17 12>
+; CHECK-NEXT:    ret <3 x i17> [[SHL]]
+;
+  %mul = mul <3 x i5> %t, <i5 13, i5 13, i5 13>
+  %ext = zext <3 x i5> %mul to <3 x i17>
+  %shl = shl <3 x i17> %ext, <i17 12, i17 12, i17 12>
+  ret <3 x i17> %shl
+}
+
+define i64 @shl_zext_mul_low_shift_amount(i32 %t) {
+; CHECK-LABEL: @shl_zext_mul_low_shift_amount(
+; CHECK-NEXT:    [[MUL:%.*]] = mul i32 [[T:%.*]], 16777215
+; CHECK-NEXT:    [[EXT:%.*]] = zext i32 [[MUL]] to i64
+; CHECK-NEXT:    [[SHL:%.*]] = shl nuw nsw i64 [[EXT]], 31
+; CHECK-NEXT:    ret i64 [[SHL]]
+;
+  %mul = mul i32 %t, 16777215
+  %ext = zext i32 %mul to i64
+  %shl = shl i64 %ext, 31
+  ret i64 %shl
+}
+
+define i64 @shl_zext_mul_extra_use1(i32 %t) {
+; CHECK-LABEL: @shl_zext_mul_extra_use1(
+; CHECK-NEXT:    [[MUL:%.*]] = mul i32 [[T:%.*]], 16777215
+; CHECK-NEXT:    [[EXT:%.*]] = zext i32 [[MUL]] to i64
+; CHECK-NEXT:    call void @use(i64 [[EXT]])
+; CHECK-NEXT:    [[SHL:%.*]] = shl nuw i64 [[EXT]], 32
+; CHECK-NEXT:    ret i64 [[SHL]]
+;
+  %mul = mul i32 %t, 16777215
+  %ext = zext i32 %mul to i64
+  call void @use(i64 %ext)
+  %shl = shl i64 %ext, 32
+  ret i64 %shl
+}
+
+declare void @use_i32(i32)
+
+define i64 @shl_zext_mul_extra_use2(i32 %t) {
+; CHECK-LABEL: @shl_zext_mul_extra_use2(
+; CHECK-NEXT:    [[MUL:%.*]] = mul i32 [[T:%.*]], 16777215
+; CHECK-NEXT:    call void @use_i32(i32 [[MUL]])
+; CHECK-NEXT:    [[EXT:%.*]] = zext i32 [[MUL]] to i64
+; CHECK-NEXT:    [[SHL:%.*]] = shl nuw i64 [[EXT]], 32
+; CHECK-NEXT:    ret i64 [[SHL]]
+;
+  %mul = mul i32 %t, 16777215
+  call void @use_i32(i32 %mul)
+  %ext = zext i32 %mul to i64
+  %shl = shl i64 %ext, 32
+  ret i64 %shl
+}
+
 define <2 x i8> @ashr_demanded_bits_splat(<2 x i8> %x) {
 ; CHECK-LABEL: @ashr_demanded_bits_splat(
-; CHECK-NEXT:    [[SHR:%.*]] = ashr <2 x i8> %x, <i8 7, i8 7>
+; CHECK-NEXT:    [[SHR:%.*]] = ashr <2 x i8> [[X:%.*]], <i8 7, i8 7>
 ; CHECK-NEXT:    ret <2 x i8> [[SHR]]
 ;
   %and = and <2 x i8> %x, <i8 128, i8 128>
@@ -1215,7 +1345,7 @@ define <2 x i8> @ashr_demanded_bits_splat(<2 x i8> %x) {
 
 define <2 x i8> @lshr_demanded_bits_splat(<2 x i8> %x) {
 ; CHECK-LABEL: @lshr_demanded_bits_splat(
-; CHECK-NEXT:    [[SHR:%.*]] = lshr <2 x i8> %x, <i8 7, i8 7>
+; CHECK-NEXT:    [[SHR:%.*]] = lshr <2 x i8> [[X:%.*]], <i8 7, i8 7>
 ; CHECK-NEXT:    ret <2 x i8> [[SHR]]
 ;
   %and = and <2 x i8> %x, <i8 128, i8 128>
@@ -1498,7 +1628,12 @@ define i32 @ashr_select_xor_false(i32 %x, i1 %cond) {
 ; https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=4871
 define i177 @lshr_out_of_range(i177 %Y, i177** %A2) {
 ; CHECK-LABEL: @lshr_out_of_range(
-; CHECK-NEXT:    store i177** [[A2:%.*]], i177*** undef, align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne i177 [[Y:%.*]], -1
+; CHECK-NEXT:    [[B4:%.*]] = sext i1 [[TMP1]] to i177
+; CHECK-NEXT:    [[C8:%.*]] = icmp ult i177 [[B4]], [[Y]]
+; CHECK-NEXT:    [[TMP2:%.*]] = sext i1 [[C8]] to i64
+; CHECK-NEXT:    [[G18:%.*]] = getelementptr i177*, i177** [[A2:%.*]], i64 [[TMP2]]
+; CHECK-NEXT:    store i177** [[G18]], i177*** undef, align 8
 ; CHECK-NEXT:    ret i177 0
 ;
   %B5 = udiv i177 %Y, -1
@@ -1519,6 +1654,18 @@ define i177 @lshr_out_of_range(i177 %Y, i177** %A2) {
 ; https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=5032
 define void @ashr_out_of_range(i177* %A) {
 ; CHECK-LABEL: @ashr_out_of_range(
+; CHECK-NEXT:    [[L:%.*]] = load i177, i177* [[A:%.*]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i177 [[L]], -1
+; CHECK-NEXT:    [[B2:%.*]] = select i1 [[TMP1]], i64 -1, i64 -2
+; CHECK-NEXT:    [[G11:%.*]] = getelementptr i177, i177* [[A]], i64 [[B2]]
+; CHECK-NEXT:    [[L7:%.*]] = load i177, i177* [[G11]], align 4
+; CHECK-NEXT:    [[B36:%.*]] = select i1 [[TMP1]], i177 0, i177 [[L7]]
+; CHECK-NEXT:    [[C17:%.*]] = icmp sgt i177 [[B36]], [[L7]]
+; CHECK-NEXT:    [[TMP2:%.*]] = sext i1 [[C17]] to i64
+; CHECK-NEXT:    [[G62:%.*]] = getelementptr i177, i177* [[G11]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i177 [[L7]], -1
+; CHECK-NEXT:    [[B28:%.*]] = select i1 [[TMP3]], i177 0, i177 [[L7]]
+; CHECK-NEXT:    store i177 [[B28]], i177* [[G62]], align 4
 ; CHECK-NEXT:    ret void
 ;
   %L = load i177, i177* %A
diff --git a/llvm/test/Transforms/InstCombine/shuffle_select.ll b/llvm/test/Transforms/InstCombine/shuffle_select.ll
index 6cda586f9..d3de3e853 100644
--- a/llvm/test/Transforms/InstCombine/shuffle_select.ll
+++ b/llvm/test/Transforms/InstCombine/shuffle_select.ll
@@ -1458,7 +1458,8 @@ define <4 x i8> @or_add_2_vars(<4 x i8> %v, <4 x i8> %v1) {
 
 define <4 x i32> @PR41419(<4 x i32> %v) {
 ; CHECK-LABEL: @PR41419(
-; CHECK-NEXT:    ret <4 x i32> [[V:%.*]]
+; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x i32> [[V:%.*]], <4 x i32> undef, <4 x i32> <i32 undef, i32 undef, i32 2, i32 undef>
+; CHECK-NEXT:    ret <4 x i32> [[S]]
 ;
   %s = shufflevector <4 x i32> %v, <4 x i32> undef, <4 x i32> <i32 4, i32 5, i32 2, i32 7>
   ret <4 x i32> %s
diff --git a/llvm/test/Transforms/InstCombine/shufflevector-div-rem.ll b/llvm/test/Transforms/InstCombine/shufflevector-div-rem.ll
new file mode 100644
index 000000000..8364aab0a
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/shufflevector-div-rem.ll
@@ -0,0 +1,122 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S -o - | FileCheck %s
+
+; This test case was added as a reproducer for a miscompile, where instcombine
+; introduced an
+;   srem <2 x i16> %1, <i16 undef, i16 2>
+; instruction, which makes the whole srem undefined (even if we only end up
+; extracting the second element in the vector).
+define i16 @test_srem_orig(i16 %a, i1 %cmp) {
+; CHECK-LABEL: @test_srem_orig(
+; CHECK-NEXT:    [[SPLATINSERT:%.*]] = insertelement <2 x i16> undef, i16 [[A:%.*]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = srem <2 x i16> [[SPLATINSERT]], <i16 2, i16 1>
+; CHECK-NEXT:    [[SPLAT_OP:%.*]] = shufflevector <2 x i16> [[TMP1]], <2 x i16> undef, <2 x i32> <i32 undef, i32 0>
+; CHECK-NEXT:    [[T2:%.*]] = select i1 [[CMP:%.*]], <2 x i16> <i16 undef, i16 1>, <2 x i16> [[SPLAT_OP]]
+; CHECK-NEXT:    [[T3:%.*]] = extractelement <2 x i16> [[T2]], i32 1
+; CHECK-NEXT:    ret i16 [[T3]]
+;
+  %splatinsert = insertelement <2 x i16> undef, i16 %a, i32 0
+  %splat = shufflevector <2 x i16> %splatinsert, <2 x i16> undef, <2 x i32> zeroinitializer
+  %t1 = select i1 %cmp, <2 x i16> <i16 1, i16 1>, <2 x i16> %splat
+  %t2 = srem <2 x i16> %t1, <i16 2, i16 2>
+  %t3 = extractelement <2 x i16> %t2, i32 1
+  ret i16 %t3
+}
+
+; This is basically a reduced version of test_srem_orig (based on what the
+; code would look like after a few iterations of instcombine, just before we
+; try to transform the shufflevector by doing
+; "evaluateInDifferentElementOrder".
+define <2 x i16> @test_srem(i16 %a, i1 %cmp) {
+; CHECK-LABEL: @test_srem(
+; CHECK-NEXT:    [[SPLATINSERT:%.*]] = insertelement <2 x i16> undef, i16 [[A:%.*]], i32 0
+; CHECK-NEXT:    [[T1:%.*]] = srem <2 x i16> [[SPLATINSERT]], <i16 2, i16 1>
+; CHECK-NEXT:    [[SPLAT_OP:%.*]] = shufflevector <2 x i16> [[T1]], <2 x i16> undef, <2 x i32> <i32 undef, i32 0>
+; CHECK-NEXT:    [[T2:%.*]] = select i1 [[CMP:%.*]], <2 x i16> <i16 77, i16 99>, <2 x i16> [[SPLAT_OP]]
+; CHECK-NEXT:    ret <2 x i16> [[T2]]
+;
+  %splatinsert = insertelement <2 x i16> undef, i16 %a, i32 0
+  %t1 = srem <2 x i16> %splatinsert, <i16 2, i16 1>
+  %splat.op = shufflevector <2 x i16> %t1, <2 x i16> undef, <2 x i32> <i32 undef, i32 0>
+  %t2 = select i1 %cmp, <2 x i16> <i16 77, i16 99>, <2 x i16> %splat.op
+  ret <2 x i16> %t2
+}
+
+define <2 x i16> @test_urem(i16 %a, i1 %cmp) {
+; CHECK-LABEL: @test_urem(
+; CHECK-NEXT:    [[SPLATINSERT:%.*]] = insertelement <2 x i16> undef, i16 [[A:%.*]], i32 0
+; CHECK-NEXT:    [[T1:%.*]] = urem <2 x i16> [[SPLATINSERT]], <i16 3, i16 1>
+; CHECK-NEXT:    [[SPLAT_OP:%.*]] = shufflevector <2 x i16> [[T1]], <2 x i16> undef, <2 x i32> <i32 undef, i32 0>
+; CHECK-NEXT:    [[T2:%.*]] = select i1 [[CMP:%.*]], <2 x i16> <i16 77, i16 99>, <2 x i16> [[SPLAT_OP]]
+; CHECK-NEXT:    ret <2 x i16> [[T2]]
+;
+  %splatinsert = insertelement <2 x i16> undef, i16 %a, i32 0
+  %t1 = urem <2 x i16> %splatinsert, <i16 3, i16 1>
+  %splat.op = shufflevector <2 x i16> %t1, <2 x i16> undef, <2 x i32> <i32 undef, i32 0>
+  %t2 = select i1 %cmp, <2 x i16> <i16 77, i16 99>, <2 x i16> %splat.op
+  ret <2 x i16> %t2
+}
+
+define <2 x i16> @test_sdiv(i16 %a, i1 %cmp) {
+; CHECK-LABEL: @test_sdiv(
+; CHECK-NEXT:    [[SPLATINSERT:%.*]] = insertelement <2 x i16> undef, i16 [[A:%.*]], i32 0
+; CHECK-NEXT:    [[T1:%.*]] = sdiv <2 x i16> [[SPLATINSERT]], <i16 2, i16 1>
+; CHECK-NEXT:    [[SPLAT_OP:%.*]] = shufflevector <2 x i16> [[T1]], <2 x i16> undef, <2 x i32> <i32 undef, i32 0>
+; CHECK-NEXT:    [[T2:%.*]] = select i1 [[CMP:%.*]], <2 x i16> <i16 77, i16 99>, <2 x i16> [[SPLAT_OP]]
+; CHECK-NEXT:    ret <2 x i16> [[T2]]
+;
+  %splatinsert = insertelement <2 x i16> undef, i16 %a, i32 0
+  %t1 = sdiv <2 x i16> %splatinsert, <i16 2, i16 1>
+  %splat.op = shufflevector <2 x i16> %t1, <2 x i16> undef, <2 x i32> <i32 undef, i32 0>
+  %t2 = select i1 %cmp, <2 x i16> <i16 77, i16 99>, <2 x i16> %splat.op
+  ret <2 x i16> %t2
+}
+
+define <2 x i16> @test_udiv(i16 %a, i1 %cmp) {
+; CHECK-LABEL: @test_udiv(
+; CHECK-NEXT:    [[SPLATINSERT:%.*]] = insertelement <2 x i16> undef, i16 [[A:%.*]], i32 0
+; CHECK-NEXT:    [[T1:%.*]] = udiv <2 x i16> [[SPLATINSERT]], <i16 3, i16 1>
+; CHECK-NEXT:    [[SPLAT_OP:%.*]] = shufflevector <2 x i16> [[T1]], <2 x i16> undef, <2 x i32> <i32 undef, i32 0>
+; CHECK-NEXT:    [[T2:%.*]] = select i1 [[CMP:%.*]], <2 x i16> <i16 77, i16 99>, <2 x i16> [[SPLAT_OP]]
+; CHECK-NEXT:    ret <2 x i16> [[T2]]
+;
+  %splatinsert = insertelement <2 x i16> undef, i16 %a, i32 0
+  %t1 = udiv <2 x i16> %splatinsert, <i16 3, i16 1>
+  %splat.op = shufflevector <2 x i16> %t1, <2 x i16> undef, <2 x i32> <i32 undef, i32 0>
+  %t2 = select i1 %cmp, <2 x i16> <i16 77, i16 99>, <2 x i16> %splat.op
+  ret <2 x i16> %t2
+}
+
+; For fdiv we do not need to worry about div by undef. Verify that the
+; shufflevector is eliminated here.
+define <2 x float> @test_fdiv(float %a, float %b, i1 %cmp) {
+; CHECK-LABEL: @test_fdiv(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x float> undef, float [[A:%.*]], i32 1
+; CHECK-NEXT:    [[TMP2:%.*]] = fdiv <2 x float> [[TMP1]], <float undef, float 3.000000e+00>
+; CHECK-NEXT:    [[T2:%.*]] = select i1 [[CMP:%.*]], <2 x float> <float 7.700000e+01, float 9.900000e+01>, <2 x float> [[TMP2]]
+; CHECK-NEXT:    ret <2 x float> [[T2]]
+;
+  %splatinsert = insertelement <2 x float> undef, float %a, i32 0
+  %denom = insertelement <2 x float> <float 3.0, float undef>, float 1.0, i32 1
+  %t1 = fdiv <2 x float> %splatinsert, %denom
+  %splat.op = shufflevector <2 x float> %t1, <2 x float> undef, <2 x i32> <i32 undef, i32 0>
+  %t2 = select i1 %cmp, <2 x float> <float 77.0, float 99.0>, <2 x float> %splat.op
+  ret <2 x float> %t2
+}
+
+; For frem we do not need to worry about div by undef. Verify that the
+; shufflevector is eliminated here.
+define <2 x float> @test_frem(float %a, float %b, i1 %cmp) {
+; CHECK-LABEL: @test_frem(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x float> undef, float [[A:%.*]], i32 1
+; CHECK-NEXT:    [[TMP2:%.*]] = frem <2 x float> [[TMP1]], <float undef, float 3.000000e+00>
+; CHECK-NEXT:    [[T2:%.*]] = select i1 [[CMP:%.*]], <2 x float> <float 7.700000e+01, float 9.900000e+01>, <2 x float> [[TMP2]]
+; CHECK-NEXT:    ret <2 x float> [[T2]]
+;
+  %splatinsert = insertelement <2 x float> undef, float %a, i32 0
+  %denom = insertelement <2 x float> <float 3.0, float undef>, float 1.0, i32 1
+  %t1 = frem <2 x float> %splatinsert, %denom
+  %splat.op = shufflevector <2 x float> %t1, <2 x float> undef, <2 x i32> <i32 undef, i32 0>
+  %t2 = select i1 %cmp, <2 x float> <float 77.0, float 99.0>, <2 x float> %splat.op
+  ret <2 x float> %t2
+}
diff --git a/llvm/test/Transforms/InstCombine/sign-bit-test-via-right-shifting-all-other-bits.ll b/llvm/test/Transforms/InstCombine/sign-bit-test-via-right-shifting-all-other-bits.ll
new file mode 100644
index 000000000..8e89a0649
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/sign-bit-test-via-right-shifting-all-other-bits.ll
@@ -0,0 +1,362 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+declare void @use32(i32)
+declare void @use64(i64)
+
+define i1 @highest_bit_test_via_lshr(i32 %data, i32 %nbits) {
+; CHECK-LABEL: @highest_bit_test_via_lshr(
+; CHECK-NEXT:    [[NUM_LOW_BITS_TO_SKIP:%.*]] = sub i32 32, [[NBITS:%.*]]
+; CHECK-NEXT:    [[HIGH_BITS_EXTRACTED:%.*]] = lshr i32 [[DATA:%.*]], [[NUM_LOW_BITS_TO_SKIP]]
+; CHECK-NEXT:    [[SKIP_ALL_BITS_TILL_SIGNBIT:%.*]] = add i32 [[NBITS]], -1
+; CHECK-NEXT:    [[SIGNBIT:%.*]] = lshr i32 [[DATA]], 31
+; CHECK-NEXT:    call void @use32(i32 [[NUM_LOW_BITS_TO_SKIP]])
+; CHECK-NEXT:    call void @use32(i32 [[HIGH_BITS_EXTRACTED]])
+; CHECK-NEXT:    call void @use32(i32 [[SKIP_ALL_BITS_TILL_SIGNBIT]])
+; CHECK-NEXT:    call void @use32(i32 [[SIGNBIT]])
+; CHECK-NEXT:    [[ISNEG:%.*]] = icmp slt i32 [[DATA]], 0
+; CHECK-NEXT:    ret i1 [[ISNEG]]
+;
+  %num_low_bits_to_skip = sub i32 32, %nbits
+  %high_bits_extracted = lshr i32 %data, %num_low_bits_to_skip
+  %skip_all_bits_till_signbit = sub i32 %nbits, 1
+  %signbit = lshr i32 %high_bits_extracted, %skip_all_bits_till_signbit
+
+  call void @use32(i32 %num_low_bits_to_skip)
+  call void @use32(i32 %high_bits_extracted)
+  call void @use32(i32 %skip_all_bits_till_signbit)
+  call void @use32(i32 %signbit)
+
+  %isneg = icmp ne i32 %signbit, 0
+  ret i1 %isneg
+}
+
+define i1 @highest_bit_test_via_lshr_with_truncation(i64 %data, i32 %nbits) {
+; CHECK-LABEL: @highest_bit_test_via_lshr_with_truncation(
+; CHECK-NEXT:    [[NUM_LOW_BITS_TO_SKIP:%.*]] = sub i32 64, [[NBITS:%.*]]
+; CHECK-NEXT:    [[NUM_LOW_BITS_TO_SKIP_WIDE:%.*]] = zext i32 [[NUM_LOW_BITS_TO_SKIP]] to i64
+; CHECK-NEXT:    [[HIGH_BITS_EXTRACTED:%.*]] = lshr i64 [[DATA:%.*]], [[NUM_LOW_BITS_TO_SKIP_WIDE]]
+; CHECK-NEXT:    [[HIGH_BITS_EXTRACTED_NARROW:%.*]] = trunc i64 [[HIGH_BITS_EXTRACTED]] to i32
+; CHECK-NEXT:    [[SKIP_ALL_BITS_TILL_SIGNBIT:%.*]] = add i32 [[NBITS]], -1
+; CHECK-NEXT:    [[SIGNBIT:%.*]] = lshr i32 [[HIGH_BITS_EXTRACTED_NARROW]], [[SKIP_ALL_BITS_TILL_SIGNBIT]]
+; CHECK-NEXT:    call void @use32(i32 [[NUM_LOW_BITS_TO_SKIP]])
+; CHECK-NEXT:    call void @use64(i64 [[NUM_LOW_BITS_TO_SKIP_WIDE]])
+; CHECK-NEXT:    call void @use64(i64 [[HIGH_BITS_EXTRACTED]])
+; CHECK-NEXT:    call void @use32(i32 [[HIGH_BITS_EXTRACTED_NARROW]])
+; CHECK-NEXT:    call void @use32(i32 [[SKIP_ALL_BITS_TILL_SIGNBIT]])
+; CHECK-NEXT:    call void @use32(i32 [[SIGNBIT]])
+; CHECK-NEXT:    [[ISNEG:%.*]] = icmp slt i64 [[DATA]], 0
+; CHECK-NEXT:    ret i1 [[ISNEG]]
+;
+  %num_low_bits_to_skip = sub i32 64, %nbits
+  %num_low_bits_to_skip_wide = zext i32 %num_low_bits_to_skip to i64
+  %high_bits_extracted = lshr i64 %data, %num_low_bits_to_skip_wide
+  %high_bits_extracted_narrow = trunc i64 %high_bits_extracted to i32
+  %skip_all_bits_till_signbit = sub i32 %nbits, 1
+  %signbit = lshr i32 %high_bits_extracted_narrow, %skip_all_bits_till_signbit
+
+  call void @use32(i32 %num_low_bits_to_skip)
+  call void @use64(i64 %num_low_bits_to_skip_wide)
+  call void @use64(i64 %high_bits_extracted)
+  call void @use32(i32 %high_bits_extracted_narrow)
+  call void @use32(i32 %skip_all_bits_till_signbit)
+  call void @use32(i32 %signbit)
+
+  %isneg = icmp ne i32 %signbit, 0
+  ret i1 %isneg
+}
+
+define i1 @highest_bit_test_via_ashr(i32 %data, i32 %nbits) {
+; CHECK-LABEL: @highest_bit_test_via_ashr(
+; CHECK-NEXT:    [[NUM_LOW_BITS_TO_SKIP:%.*]] = sub i32 32, [[NBITS:%.*]]
+; CHECK-NEXT:    [[HIGH_BITS_EXTRACTED:%.*]] = ashr i32 [[DATA:%.*]], [[NUM_LOW_BITS_TO_SKIP]]
+; CHECK-NEXT:    [[SKIP_ALL_BITS_TILL_SIGNBIT:%.*]] = add i32 [[NBITS]], -1
+; CHECK-NEXT:    [[SIGNBIT:%.*]] = ashr i32 [[DATA]], 31
+; CHECK-NEXT:    call void @use32(i32 [[NUM_LOW_BITS_TO_SKIP]])
+; CHECK-NEXT:    call void @use32(i32 [[HIGH_BITS_EXTRACTED]])
+; CHECK-NEXT:    call void @use32(i32 [[SKIP_ALL_BITS_TILL_SIGNBIT]])
+; CHECK-NEXT:    call void @use32(i32 [[SIGNBIT]])
+; CHECK-NEXT:    [[ISNEG:%.*]] = icmp slt i32 [[DATA]], 0
+; CHECK-NEXT:    ret i1 [[ISNEG]]
+;
+  %num_low_bits_to_skip = sub i32 32, %nbits
+  %high_bits_extracted = ashr i32 %data, %num_low_bits_to_skip
+  %skip_all_bits_till_signbit = sub i32 %nbits, 1
+  %signbit = ashr i32 %high_bits_extracted, %skip_all_bits_till_signbit
+
+  call void @use32(i32 %num_low_bits_to_skip)
+  call void @use32(i32 %high_bits_extracted)
+  call void @use32(i32 %skip_all_bits_till_signbit)
+  call void @use32(i32 %signbit)
+
+  %isneg = icmp ne i32 %signbit, 0
+  ret i1 %isneg
+}
+
+define i1 @highest_bit_test_via_ashr_with_truncation(i64 %data, i32 %nbits) {
+; CHECK-LABEL: @highest_bit_test_via_ashr_with_truncation(
+; CHECK-NEXT:    [[NUM_LOW_BITS_TO_SKIP:%.*]] = sub i32 64, [[NBITS:%.*]]
+; CHECK-NEXT:    [[NUM_LOW_BITS_TO_SKIP_WIDE:%.*]] = zext i32 [[NUM_LOW_BITS_TO_SKIP]] to i64
+; CHECK-NEXT:    [[HIGH_BITS_EXTRACTED:%.*]] = ashr i64 [[DATA:%.*]], [[NUM_LOW_BITS_TO_SKIP_WIDE]]
+; CHECK-NEXT:    [[HIGH_BITS_EXTRACTED_NARROW:%.*]] = trunc i64 [[HIGH_BITS_EXTRACTED]] to i32
+; CHECK-NEXT:    [[SKIP_ALL_BITS_TILL_SIGNBIT:%.*]] = add i32 [[NBITS]], -1
+; CHECK-NEXT:    [[SIGNBIT:%.*]] = ashr i32 [[HIGH_BITS_EXTRACTED_NARROW]], [[SKIP_ALL_BITS_TILL_SIGNBIT]]
+; CHECK-NEXT:    call void @use32(i32 [[NUM_LOW_BITS_TO_SKIP]])
+; CHECK-NEXT:    call void @use64(i64 [[NUM_LOW_BITS_TO_SKIP_WIDE]])
+; CHECK-NEXT:    call void @use64(i64 [[HIGH_BITS_EXTRACTED]])
+; CHECK-NEXT:    call void @use32(i32 [[HIGH_BITS_EXTRACTED_NARROW]])
+; CHECK-NEXT:    call void @use32(i32 [[SKIP_ALL_BITS_TILL_SIGNBIT]])
+; CHECK-NEXT:    call void @use32(i32 [[SIGNBIT]])
+; CHECK-NEXT:    [[ISNEG:%.*]] = icmp slt i64 [[DATA]], 0
+; CHECK-NEXT:    ret i1 [[ISNEG]]
+;
+  %num_low_bits_to_skip = sub i32 64, %nbits
+  %num_low_bits_to_skip_wide = zext i32 %num_low_bits_to_skip to i64
+  %high_bits_extracted = ashr i64 %data, %num_low_bits_to_skip_wide
+  %high_bits_extracted_narrow = trunc i64 %high_bits_extracted to i32
+  %skip_all_bits_till_signbit = sub i32 %nbits, 1
+  %signbit = ashr i32 %high_bits_extracted_narrow, %skip_all_bits_till_signbit
+
+  call void @use32(i32 %num_low_bits_to_skip)
+  call void @use64(i64 %num_low_bits_to_skip_wide)
+  call void @use64(i64 %high_bits_extracted)
+  call void @use32(i32 %high_bits_extracted_narrow)
+  call void @use32(i32 %skip_all_bits_till_signbit)
+  call void @use32(i32 %signbit)
+
+  %isneg = icmp ne i32 %signbit, 0
+  ret i1 %isneg
+}
+
+define i1 @highest_bit_test_via_lshr_ashr(i32 %data, i32 %nbits) {
+; CHECK-LABEL: @highest_bit_test_via_lshr_ashr(
+; CHECK-NEXT:    [[NUM_LOW_BITS_TO_SKIP:%.*]] = sub i32 32, [[NBITS:%.*]]
+; CHECK-NEXT:    [[HIGH_BITS_EXTRACTED:%.*]] = lshr i32 [[DATA:%.*]], [[NUM_LOW_BITS_TO_SKIP]]
+; CHECK-NEXT:    [[SKIP_ALL_BITS_TILL_SIGNBIT:%.*]] = add i32 [[NBITS]], -1
+; CHECK-NEXT:    [[SIGNBIT:%.*]] = ashr i32 [[HIGH_BITS_EXTRACTED]], [[SKIP_ALL_BITS_TILL_SIGNBIT]]
+; CHECK-NEXT:    call void @use32(i32 [[NUM_LOW_BITS_TO_SKIP]])
+; CHECK-NEXT:    call void @use32(i32 [[HIGH_BITS_EXTRACTED]])
+; CHECK-NEXT:    call void @use32(i32 [[SKIP_ALL_BITS_TILL_SIGNBIT]])
+; CHECK-NEXT:    call void @use32(i32 [[SIGNBIT]])
+; CHECK-NEXT:    [[ISNEG:%.*]] = icmp slt i32 [[DATA]], 0
+; CHECK-NEXT:    ret i1 [[ISNEG]]
+;
+  %num_low_bits_to_skip = sub i32 32, %nbits
+  %high_bits_extracted = lshr i32 %data, %num_low_bits_to_skip
+  %skip_all_bits_till_signbit = sub i32 %nbits, 1
+  %signbit = ashr i32 %high_bits_extracted, %skip_all_bits_till_signbit
+
+  call void @use32(i32 %num_low_bits_to_skip)
+  call void @use32(i32 %high_bits_extracted)
+  call void @use32(i32 %skip_all_bits_till_signbit)
+  call void @use32(i32 %signbit)
+
+  %isneg = icmp ne i32 %signbit, 0
+  ret i1 %isneg
+}
+
+define i1 @highest_bit_test_via_lshr_ashe_with_truncation(i64 %data, i32 %nbits) {
+; CHECK-LABEL: @highest_bit_test_via_lshr_ashe_with_truncation(
+; CHECK-NEXT:    [[NUM_LOW_BITS_TO_SKIP:%.*]] = sub i32 64, [[NBITS:%.*]]
+; CHECK-NEXT:    [[NUM_LOW_BITS_TO_SKIP_WIDE:%.*]] = zext i32 [[NUM_LOW_BITS_TO_SKIP]] to i64
+; CHECK-NEXT:    [[HIGH_BITS_EXTRACTED:%.*]] = lshr i64 [[DATA:%.*]], [[NUM_LOW_BITS_TO_SKIP_WIDE]]
+; CHECK-NEXT:    [[HIGH_BITS_EXTRACTED_NARROW:%.*]] = trunc i64 [[HIGH_BITS_EXTRACTED]] to i32
+; CHECK-NEXT:    [[SKIP_ALL_BITS_TILL_SIGNBIT:%.*]] = add i32 [[NBITS]], -1
+; CHECK-NEXT:    [[SIGNBIT:%.*]] = ashr i32 [[HIGH_BITS_EXTRACTED_NARROW]], [[SKIP_ALL_BITS_TILL_SIGNBIT]]
+; CHECK-NEXT:    call void @use32(i32 [[NUM_LOW_BITS_TO_SKIP]])
+; CHECK-NEXT:    call void @use64(i64 [[NUM_LOW_BITS_TO_SKIP_WIDE]])
+; CHECK-NEXT:    call void @use64(i64 [[HIGH_BITS_EXTRACTED]])
+; CHECK-NEXT:    call void @use32(i32 [[HIGH_BITS_EXTRACTED_NARROW]])
+; CHECK-NEXT:    call void @use32(i32 [[SKIP_ALL_BITS_TILL_SIGNBIT]])
+; CHECK-NEXT:    call void @use32(i32 [[SIGNBIT]])
+; CHECK-NEXT:    [[ISNEG:%.*]] = icmp slt i64 [[DATA]], 0
+; CHECK-NEXT:    ret i1 [[ISNEG]]
+;
+  %num_low_bits_to_skip = sub i32 64, %nbits
+  %num_low_bits_to_skip_wide = zext i32 %num_low_bits_to_skip to i64
+  %high_bits_extracted = lshr i64 %data, %num_low_bits_to_skip_wide
+  %high_bits_extracted_narrow = trunc i64 %high_bits_extracted to i32
+  %skip_all_bits_till_signbit = sub i32 %nbits, 1
+  %signbit = ashr i32 %high_bits_extracted_narrow, %skip_all_bits_till_signbit
+
+  call void @use32(i32 %num_low_bits_to_skip)
+  call void @use64(i64 %num_low_bits_to_skip_wide)
+  call void @use64(i64 %high_bits_extracted)
+  call void @use32(i32 %high_bits_extracted_narrow)
+  call void @use32(i32 %skip_all_bits_till_signbit)
+  call void @use32(i32 %signbit)
+
+  %isneg = icmp ne i32 %signbit, 0
+  ret i1 %isneg
+}
+
+define i1 @highest_bit_test_via_ashr_lshr(i32 %data, i32 %nbits) {
+; CHECK-LABEL: @highest_bit_test_via_ashr_lshr(
+; CHECK-NEXT:    [[NUM_LOW_BITS_TO_SKIP:%.*]] = sub i32 32, [[NBITS:%.*]]
+; CHECK-NEXT:    [[HIGH_BITS_EXTRACTED:%.*]] = ashr i32 [[DATA:%.*]], [[NUM_LOW_BITS_TO_SKIP]]
+; CHECK-NEXT:    [[SKIP_ALL_BITS_TILL_SIGNBIT:%.*]] = add i32 [[NBITS]], -1
+; CHECK-NEXT:    [[SIGNBIT:%.*]] = lshr i32 [[HIGH_BITS_EXTRACTED]], [[SKIP_ALL_BITS_TILL_SIGNBIT]]
+; CHECK-NEXT:    call void @use32(i32 [[NUM_LOW_BITS_TO_SKIP]])
+; CHECK-NEXT:    call void @use32(i32 [[HIGH_BITS_EXTRACTED]])
+; CHECK-NEXT:    call void @use32(i32 [[SKIP_ALL_BITS_TILL_SIGNBIT]])
+; CHECK-NEXT:    call void @use32(i32 [[SIGNBIT]])
+; CHECK-NEXT:    [[ISNEG:%.*]] = icmp slt i32 [[DATA]], 0
+; CHECK-NEXT:    ret i1 [[ISNEG]]
+;
+  %num_low_bits_to_skip = sub i32 32, %nbits
+  %high_bits_extracted = ashr i32 %data, %num_low_bits_to_skip
+  %skip_all_bits_till_signbit = sub i32 %nbits, 1
+  %signbit = lshr i32 %high_bits_extracted, %skip_all_bits_till_signbit
+
+  call void @use32(i32 %num_low_bits_to_skip)
+  call void @use32(i32 %high_bits_extracted)
+  call void @use32(i32 %skip_all_bits_till_signbit)
+  call void @use32(i32 %signbit)
+
+  %isneg = icmp ne i32 %signbit, 0
+  ret i1 %isneg
+}
+
+define i1 @highest_bit_test_via_ashr_lshr_with_truncation(i64 %data, i32 %nbits) {
+; CHECK-LABEL: @highest_bit_test_via_ashr_lshr_with_truncation(
+; CHECK-NEXT:    [[NUM_LOW_BITS_TO_SKIP:%.*]] = sub i32 64, [[NBITS:%.*]]
+; CHECK-NEXT:    [[NUM_LOW_BITS_TO_SKIP_WIDE:%.*]] = zext i32 [[NUM_LOW_BITS_TO_SKIP]] to i64
+; CHECK-NEXT:    [[HIGH_BITS_EXTRACTED:%.*]] = ashr i64 [[DATA:%.*]], [[NUM_LOW_BITS_TO_SKIP_WIDE]]
+; CHECK-NEXT:    [[HIGH_BITS_EXTRACTED_NARROW:%.*]] = trunc i64 [[HIGH_BITS_EXTRACTED]] to i32
+; CHECK-NEXT:    [[SKIP_ALL_BITS_TILL_SIGNBIT:%.*]] = add i32 [[NBITS]], -1
+; CHECK-NEXT:    [[SIGNBIT:%.*]] = lshr i32 [[HIGH_BITS_EXTRACTED_NARROW]], [[SKIP_ALL_BITS_TILL_SIGNBIT]]
+; CHECK-NEXT:    call void @use32(i32 [[NUM_LOW_BITS_TO_SKIP]])
+; CHECK-NEXT:    call void @use64(i64 [[NUM_LOW_BITS_TO_SKIP_WIDE]])
+; CHECK-NEXT:    call void @use64(i64 [[HIGH_BITS_EXTRACTED]])
+; CHECK-NEXT:    call void @use32(i32 [[HIGH_BITS_EXTRACTED_NARROW]])
+; CHECK-NEXT:    call void @use32(i32 [[SKIP_ALL_BITS_TILL_SIGNBIT]])
+; CHECK-NEXT:    call void @use32(i32 [[SIGNBIT]])
+; CHECK-NEXT:    [[ISNEG:%.*]] = icmp slt i64 [[DATA]], 0
+; CHECK-NEXT:    ret i1 [[ISNEG]]
+;
+  %num_low_bits_to_skip = sub i32 64, %nbits
+  %num_low_bits_to_skip_wide = zext i32 %num_low_bits_to_skip to i64
+  %high_bits_extracted = ashr i64 %data, %num_low_bits_to_skip_wide
+  %high_bits_extracted_narrow = trunc i64 %high_bits_extracted to i32
+  %skip_all_bits_till_signbit = sub i32 %nbits, 1
+  %signbit = lshr i32 %high_bits_extracted_narrow, %skip_all_bits_till_signbit
+
+  call void @use32(i32 %num_low_bits_to_skip)
+  call void @use64(i64 %num_low_bits_to_skip_wide)
+  call void @use64(i64 %high_bits_extracted)
+  call void @use32(i32 %high_bits_extracted_narrow)
+  call void @use32(i32 %skip_all_bits_till_signbit)
+  call void @use32(i32 %signbit)
+
+  %isneg = icmp ne i32 %signbit, 0
+  ret i1 %isneg
+}
+
+;------------------------------------------------------------------------------;
+
+define i1 @unsigned_sign_bit_extract(i32 %x) {
+; CHECK-LABEL: @unsigned_sign_bit_extract(
+; CHECK-NEXT:    [[ISNEG:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    ret i1 [[ISNEG]]
+;
+  %signbit = lshr i32 %x, 31
+  %isneg = icmp ne i32 %signbit, 0
+  ret i1 %isneg
+}
+define i1 @unsigned_sign_bit_extract_extrause(i32 %x) {
+; CHECK-LABEL: @unsigned_sign_bit_extract_extrause(
+; CHECK-NEXT:    [[SIGNBIT:%.*]] = lshr i32 [[X:%.*]], 31
+; CHECK-NEXT:    call void @use32(i32 [[SIGNBIT]])
+; CHECK-NEXT:    [[ISNEG:%.*]] = icmp slt i32 [[X]], 0
+; CHECK-NEXT:    ret i1 [[ISNEG]]
+;
+  %signbit = lshr i32 %x, 31
+  call void @use32(i32 %signbit)
+  %isneg = icmp ne i32 %signbit, 0
+  ret i1 %isneg
+}
+define i1 @unsigned_sign_bit_extract_extrause__ispositive(i32 %x) {
+; CHECK-LABEL: @unsigned_sign_bit_extract_extrause__ispositive(
+; CHECK-NEXT:    [[SIGNBIT:%.*]] = lshr i32 [[X:%.*]], 31
+; CHECK-NEXT:    call void @use32(i32 [[SIGNBIT]])
+; CHECK-NEXT:    [[ISNEG:%.*]] = icmp sgt i32 [[X]], -1
+; CHECK-NEXT:    ret i1 [[ISNEG]]
+;
+  %signbit = lshr i32 %x, 31
+  call void @use32(i32 %signbit)
+  %isneg = icmp eq i32 %signbit, 0
+  ret i1 %isneg
+}
+define i1 @signed_sign_bit_extract(i32 %x) {
+; CHECK-LABEL: @signed_sign_bit_extract(
+; CHECK-NEXT:    [[ISNEG:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    ret i1 [[ISNEG]]
+;
+  %signsmear = ashr i32 %x, 31
+  %isneg = icmp ne i32 %signsmear, 0
+  ret i1 %isneg
+}
+define i1 @signed_sign_bit_extract_extrause(i32 %x) {
+; CHECK-LABEL: @signed_sign_bit_extract_extrause(
+; CHECK-NEXT:    [[SIGNSMEAR:%.*]] = ashr i32 [[X:%.*]], 31
+; CHECK-NEXT:    call void @use32(i32 [[SIGNSMEAR]])
+; CHECK-NEXT:    [[ISNEG:%.*]] = icmp slt i32 [[X]], 0
+; CHECK-NEXT:    ret i1 [[ISNEG]]
+;
+  %signsmear = ashr i32 %x, 31
+  call void @use32(i32 %signsmear)
+  %isneg = icmp ne i32 %signsmear, 0
+  ret i1 %isneg
+}
+define i1 @unsigned_sign_bit_extract_with_trunc(i64 %x) {
+; CHECK-LABEL: @unsigned_sign_bit_extract_with_trunc(
+; CHECK-NEXT:    [[ISNEG:%.*]] = icmp slt i64 [[X:%.*]], 0
+; CHECK-NEXT:    ret i1 [[ISNEG]]
+;
+  %signbit = lshr i64 %x, 63
+  %signbit_narrow = trunc i64 %signbit to i32
+  %isneg = icmp ne i32 %signbit_narrow, 0
+  ret i1 %isneg
+}
+define i1 @unsigned_sign_bit_extract_with_trunc_extrause(i64 %x) {
+; CHECK-LABEL: @unsigned_sign_bit_extract_with_trunc_extrause(
+; CHECK-NEXT:    [[SIGNBIT:%.*]] = lshr i64 [[X:%.*]], 63
+; CHECK-NEXT:    call void @use64(i64 [[SIGNBIT]])
+; CHECK-NEXT:    [[SIGNBIT_NARROW:%.*]] = trunc i64 [[SIGNBIT]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[SIGNBIT_NARROW]])
+; CHECK-NEXT:    [[ISNEG:%.*]] = icmp slt i64 [[X]], 0
+; CHECK-NEXT:    ret i1 [[ISNEG]]
+;
+  %signbit = lshr i64 %x, 63
+  call void @use64(i64 %signbit)
+  %signbit_narrow = trunc i64 %signbit to i32
+  call void @use32(i32 %signbit_narrow)
+  %isneg = icmp ne i32 %signbit_narrow, 0
+  ret i1 %isneg
+}
+define i1 @signed_sign_bit_extract_trunc(i64 %x) {
+; CHECK-LABEL: @signed_sign_bit_extract_trunc(
+; CHECK-NEXT:    [[ISNEG:%.*]] = icmp slt i64 [[X:%.*]], 0
+; CHECK-NEXT:    ret i1 [[ISNEG]]
+;
+  %signsmear = ashr i64 %x, 63
+  %signsmear_narrow = trunc i64 %signsmear to i32
+  %isneg = icmp ne i32 %signsmear_narrow, 0
+  ret i1 %isneg
+}
+define i1 @signed_sign_bit_extract_trunc_extrause(i64 %x) {
+; CHECK-LABEL: @signed_sign_bit_extract_trunc_extrause(
+; CHECK-NEXT:    [[SIGNSMEAR:%.*]] = ashr i64 [[X:%.*]], 63
+; CHECK-NEXT:    call void @use64(i64 [[SIGNSMEAR]])
+; CHECK-NEXT:    [[SIGNSMEAR_NARROW:%.*]] = trunc i64 [[SIGNSMEAR]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[SIGNSMEAR_NARROW]])
+; CHECK-NEXT:    [[ISNEG:%.*]] = icmp slt i64 [[X]], 0
+; CHECK-NEXT:    ret i1 [[ISNEG]]
+;
+  %signsmear = ashr i64 %x, 63
+  call void @use64(i64 %signsmear)
+  %signsmear_narrow = trunc i64 %signsmear to i32
+  call void @use32(i32 %signsmear_narrow)
+  %isneg = icmp ne i32 %signsmear_narrow, 0
+  ret i1 %isneg
+}
diff --git a/llvm/test/Transforms/InstCombine/sink-alloca.ll b/llvm/test/Transforms/InstCombine/sink-alloca.ll
index f2de74ff5..dc5012680 100644
--- a/llvm/test/Transforms/InstCombine/sink-alloca.ll
+++ b/llvm/test/Transforms/InstCombine/sink-alloca.ll
@@ -47,6 +47,6 @@ ret:                                              ; preds = %sinktarget, %nonent
 ; CHECK:   %p = call i32* @use_and_return(i32* nonnull %argmem)
 ; CHECK:   store i32 13, i32* %p
 ; CHECK:   call void @llvm.stackrestore(i8* %sp)
-; CHECK:   %0 = call i32* @use_and_return(i32* %p)
+; CHECK:   %0 = call i32* @use_and_return(i32* nonnull %p)
 
 attributes #0 = { nounwind }
diff --git a/llvm/test/Transforms/InstCombine/snprintf.ll b/llvm/test/Transforms/InstCombine/snprintf.ll
index f323bf9bf..0b9d88048 100644
--- a/llvm/test/Transforms/InstCombine/snprintf.ll
+++ b/llvm/test/Transforms/InstCombine/snprintf.ll
@@ -10,13 +10,14 @@ declare i32 @snprintf(i8*, i64, i8*, ...) #1
 
 define void @test_not_const_fmt(i8* %buf, i8* %fmt) #0 {
 ; CHECK-LABEL: @test_not_const_fmt(
-; CHECK-NEXT:    [[CALL:%.*]] = call i32 (i8*, i64, i8*, ...) @snprintf(i8* [[BUF:%.*]], i64 32, i8* [[FMT:%.*]])
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 (i8*, i64, i8*, ...) @snprintf(i8* nonnull dereferenceable(1) [[BUF:%.*]], i64 32, i8* [[FMT:%.*]])
 ; CHECK-NEXT:    ret void
 ;
   %call = call i32 (i8*, i64, i8*, ...) @snprintf(i8* %buf, i64 32, i8* %fmt) #2
   ret void
 }
 
+; size is '0', do not add nonnull attribute
 define void @test_not_const_fmt_zero_size_return_value(i8* %buf, i8* %fmt) #0 {
 ; CHECK-LABEL: @test_not_const_fmt_zero_size_return_value(
 ; CHECK-NEXT:    [[CALL:%.*]] = call i32 (i8*, i64, i8*, ...) @snprintf(i8* [[BUF:%.*]], i64 0, i8* [[FMT:%.*]])
@@ -26,7 +27,6 @@ define void @test_not_const_fmt_zero_size_return_value(i8* %buf, i8* %fmt) #0 {
   ret void
 }
 
-
 define void @test_not_const_size(i8* %buf, i64 %size) #0 {
 ; CHECK-LABEL: @test_not_const_size(
 ; CHECK-NEXT:    [[CALL:%.*]] = call i32 (i8*, i64, i8*, ...) @snprintf(i8* [[BUF:%.*]], i64 [[SIZE:%.*]], i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str, i64 0, i64 0))
@@ -47,7 +47,7 @@ define i32 @test_return_value(i8* %buf) #0 {
 
 define void @test_percentage(i8* %buf) #0 {
 ; CHECK-LABEL: @test_percentage(
-; CHECK-NEXT:    [[CALL:%.*]] = call i32 (i8*, i64, i8*, ...) @snprintf(i8* [[BUF:%.*]], i64 32, i8* getelementptr inbounds ([3 x i8], [3 x i8]* @.str.1, i64 0, i64 0))
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 (i8*, i64, i8*, ...) @snprintf(i8* nonnull dereferenceable(1) [[BUF:%.*]], i64 32, i8* getelementptr inbounds ([3 x i8], [3 x i8]* @.str.1, i64 0, i64 0))
 ; CHECK-NEXT:    ret void
 ;
   %call = call i32 (i8*, i64, i8*, ...) @snprintf(i8* %buf, i64 32, i8* getelementptr inbounds ([3 x i8], [3 x i8]* @.str.1, i64 0, i64 0)) #2
@@ -92,7 +92,7 @@ define i32 @test_char_zero_size(i8* %buf) #0 {
 
 define i32 @test_char_wrong_size(i8* %buf) #0 {
 ; CHECK-LABEL: @test_char_wrong_size(
-; CHECK-NEXT:    [[CALL:%.*]] = call i32 (i8*, i64, i8*, ...) @snprintf(i8* [[BUF:%.*]], i64 1, i8* getelementptr inbounds ([3 x i8], [3 x i8]* @.str.2, i64 0, i64 0), i32 65)
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 (i8*, i64, i8*, ...) @snprintf(i8* nonnull dereferenceable(1) [[BUF:%.*]], i64 1, i8* getelementptr inbounds ([3 x i8], [3 x i8]* @.str.2, i64 0, i64 0), i32 65)
 ; CHECK-NEXT:    ret i32 [[CALL]]
 ;
   %call = call i32 (i8*, i64, i8*, ...) @snprintf(i8* %buf, i64 1, i8* getelementptr inbounds ([3 x i8], [3 x i8]* @.str.2, i64 0, i64 0), i32 65) #2
@@ -120,7 +120,7 @@ define i32 @test_str_zero_size(i8* %buf) #0 {
 
 define i32 @test_str_wrong_size(i8* %buf) #0 {
 ; CHECK-LABEL: @test_str_wrong_size(
-; CHECK-NEXT:    [[CALL:%.*]] = call i32 (i8*, i64, i8*, ...) @snprintf(i8* [[BUF:%.*]], i64 1, i8* getelementptr inbounds ([3 x i8], [3 x i8]* @.str.3, i64 0, i64 0), i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str, i64 0, i64 0))
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 (i8*, i64, i8*, ...) @snprintf(i8* nonnull dereferenceable(1) [[BUF:%.*]], i64 1, i8* getelementptr inbounds ([3 x i8], [3 x i8]* @.str.3, i64 0, i64 0), i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str, i64 0, i64 0))
 ; CHECK-NEXT:    ret i32 [[CALL]]
 ;
   %call = call i32 (i8*, i64, i8*, ...) @snprintf(i8* %buf, i64 1, i8* getelementptr inbounds ([3 x i8], [3 x i8]* @.str.3, i64 0, i64 0), i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str, i64 0, i64 0)) #2
diff --git a/llvm/test/Transforms/InstCombine/sprintf-1.ll b/llvm/test/Transforms/InstCombine/sprintf-1.ll
index 1fbdc43bd..9dbfecaec 100644
--- a/llvm/test/Transforms/InstCombine/sprintf-1.ll
+++ b/llvm/test/Transforms/InstCombine/sprintf-1.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; Test that the sprintf library call simplifier works correctly.
 ;
 ; RUN: opt < %s -instcombine -S | FileCheck %s
@@ -20,81 +21,125 @@ declare i32 @sprintf(i8*, i8*, ...)
 
 define void @test_simplify1(i8* %dst) {
 ; CHECK-LABEL: @test_simplify1(
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* nonnull align 1 dereferenceable(13) [[DST:%.*]], i8* nonnull align 1 dereferenceable(13) getelementptr inbounds ([13 x i8], [13 x i8]* @hello_world, i32 0, i32 0), i32 13, i1 false)
+; CHECK-NEXT:    ret void
+;
+; CHECK-IPRINTF-LABEL: @test_simplify1(
+; CHECK-IPRINTF-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* nonnull align 1 dereferenceable(13) [[DST:%.*]], i8* nonnull align 1 dereferenceable(13) getelementptr inbounds ([13 x i8], [13 x i8]* @hello_world, i32 0, i32 0), i32 13, i1 false)
+; CHECK-IPRINTF-NEXT:    ret void
+;
   %fmt = getelementptr [13 x i8], [13 x i8]* @hello_world, i32 0, i32 0
   call i32 (i8*, i8*, ...) @sprintf(i8* %dst, i8* %fmt)
-; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 %dst, i8* align 1 getelementptr inbounds ([13 x i8], [13 x i8]* @hello_world, i32 0, i32 0), i32 13, i1 false)
   ret void
-; CHECK-NEXT: ret void
 }
 
 define void @test_simplify2(i8* %dst) {
 ; CHECK-LABEL: @test_simplify2(
+; CHECK-NEXT:    store i8 0, i8* [[DST:%.*]], align 1
+; CHECK-NEXT:    ret void
+;
+; CHECK-IPRINTF-LABEL: @test_simplify2(
+; CHECK-IPRINTF-NEXT:    store i8 0, i8* [[DST:%.*]], align 1
+; CHECK-IPRINTF-NEXT:    ret void
+;
   %fmt = getelementptr [1 x i8], [1 x i8]* @null, i32 0, i32 0
   call i32 (i8*, i8*, ...) @sprintf(i8* %dst, i8* %fmt)
-; CHECK-NEXT: store i8 0, i8* %dst, align 1
   ret void
-; CHECK-NEXT: ret void
 }
 
 define void @test_simplify3(i8* %dst) {
 ; CHECK-LABEL: @test_simplify3(
+; CHECK-NEXT:    store i8 0, i8* [[DST:%.*]], align 1
+; CHECK-NEXT:    ret void
+;
+; CHECK-IPRINTF-LABEL: @test_simplify3(
+; CHECK-IPRINTF-NEXT:    store i8 0, i8* [[DST:%.*]], align 1
+; CHECK-IPRINTF-NEXT:    ret void
+;
   %fmt = getelementptr [7 x i8], [7 x i8]* @null_hello, i32 0, i32 0
   call i32 (i8*, i8*, ...) @sprintf(i8* %dst, i8* %fmt)
-; CHECK-NEXT: store i8 0, i8* %dst, align 1
   ret void
-; CHECK-NEXT: ret void
 }
 
 ; Check sprintf(dst, "%c", chr) -> *(i8*)dst = chr; *((i8*)dst + 1) = 0.
 
 define void @test_simplify4(i8* %dst) {
 ; CHECK-LABEL: @test_simplify4(
+; CHECK-NEXT:    store i8 104, i8* [[DST:%.*]], align 1
+; CHECK-NEXT:    [[NUL:%.*]] = getelementptr i8, i8* [[DST]], i32 1
+; CHECK-NEXT:    store i8 0, i8* [[NUL]], align 1
+; CHECK-NEXT:    ret void
+;
+; CHECK-IPRINTF-LABEL: @test_simplify4(
+; CHECK-IPRINTF-NEXT:    store i8 104, i8* [[DST:%.*]], align 1
+; CHECK-IPRINTF-NEXT:    [[NUL:%.*]] = getelementptr i8, i8* [[DST]], i32 1
+; CHECK-IPRINTF-NEXT:    store i8 0, i8* [[NUL]], align 1
+; CHECK-IPRINTF-NEXT:    ret void
+;
   %fmt = getelementptr [3 x i8], [3 x i8]* @percent_c, i32 0, i32 0
   call i32 (i8*, i8*, ...) @sprintf(i8* %dst, i8* %fmt, i8 104)
-; CHECK-NEXT: store i8 104, i8* %dst, align 1
-; CHECK-NEXT: [[NUL:%[a-z0-9]+]] = getelementptr i8, i8* %dst, i32 1
-; CHECK-NEXT: store i8 0, i8* [[NUL]], align 1
   ret void
-; CHECK-NEXT: ret void
 }
 
 ; Check sprintf(dst, "%s", str) -> llvm.memcpy(dest, str, strlen(str) + 1, 1).
 
 define void @test_simplify5(i8* %dst, i8* %str) {
 ; CHECK-LABEL: @test_simplify5(
+; CHECK-NEXT:    [[STRLEN:%.*]] = call i32 @strlen(i8* nonnull dereferenceable(1) [[STR:%.*]])
+; CHECK-NEXT:    [[LENINC:%.*]] = add i32 [[STRLEN]], 1
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 [[DST:%.*]], i8* align 1 [[STR]], i32 [[LENINC]], i1 false)
+; CHECK-NEXT:    ret void
+;
+; CHECK-IPRINTF-LABEL: @test_simplify5(
+; CHECK-IPRINTF-NEXT:    [[STRLEN:%.*]] = call i32 @strlen(i8* nonnull dereferenceable(1) [[STR:%.*]])
+; CHECK-IPRINTF-NEXT:    [[LENINC:%.*]] = add i32 [[STRLEN]], 1
+; CHECK-IPRINTF-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 [[DST:%.*]], i8* align 1 [[STR]], i32 [[LENINC]], i1 false)
+; CHECK-IPRINTF-NEXT:    ret void
+;
   %fmt = getelementptr [3 x i8], [3 x i8]* @percent_s, i32 0, i32 0
   call i32 (i8*, i8*, ...) @sprintf(i8* %dst, i8* %fmt, i8* %str)
-; CHECK-NEXT: [[STRLEN:%[a-z0-9]+]] = call i32 @strlen(i8* %str)
-; CHECK-NEXT: [[LENINC:%[a-z0-9]+]] = add i32 [[STRLEN]], 1
-; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 %dst, i8* align 1 %str, i32 [[LENINC]], i1 false)
   ret void
-; CHECK-NEXT: ret void
 }
 
 ; Check sprintf(dst, format, ...) -> siprintf(str, format, ...) if no floating.
 
 define void @test_simplify6(i8* %dst) {
+; CHECK-LABEL: @test_simplify6(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 (i8*, i8*, ...) @sprintf(i8* nonnull dereferenceable(1) [[DST:%.*]], i8* nonnull dereferenceable(1) getelementptr inbounds ([3 x i8], [3 x i8]* @percent_d, i32 0, i32 0), i32 187)
+; CHECK-NEXT:    ret void
+;
 ; CHECK-IPRINTF-LABEL: @test_simplify6(
+; CHECK-IPRINTF-NEXT:    [[TMP1:%.*]] = call i32 (i8*, i8*, ...) @siprintf(i8* [[DST:%.*]], i8* getelementptr inbounds ([3 x i8], [3 x i8]* @percent_d, i32 0, i32 0), i32 187)
+; CHECK-IPRINTF-NEXT:    ret void
+;
   %fmt = getelementptr [3 x i8], [3 x i8]* @percent_d, i32 0, i32 0
   call i32 (i8*, i8*, ...) @sprintf(i8* %dst, i8* %fmt, i32 187)
-; CHECK-IPRINTF-NEXT: call i32 (i8*, i8*, ...) @siprintf(i8* %dst, i8* getelementptr inbounds ([3 x i8], [3 x i8]* @percent_d, i32 0, i32 0), i32 187)
   ret void
-; CHECK-IPRINTF-NEXT: ret void
 }
 
 define void @test_no_simplify1(i8* %dst) {
+; CHECK-LABEL: @test_no_simplify1(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 (i8*, i8*, ...) @sprintf(i8* nonnull dereferenceable(1) [[DST:%.*]], i8* nonnull dereferenceable(1) getelementptr inbounds ([3 x i8], [3 x i8]* @percent_f, i32 0, i32 0), double 1.870000e+00)
+; CHECK-NEXT:    ret void
+;
 ; CHECK-IPRINTF-LABEL: @test_no_simplify1(
+; CHECK-IPRINTF-NEXT:    [[TMP1:%.*]] = call i32 (i8*, i8*, ...) @sprintf(i8* nonnull dereferenceable(1) [[DST:%.*]], i8* nonnull dereferenceable(1) getelementptr inbounds ([3 x i8], [3 x i8]* @percent_f, i32 0, i32 0), double 1.870000e+00)
+; CHECK-IPRINTF-NEXT:    ret void
+;
   %fmt = getelementptr [3 x i8], [3 x i8]* @percent_f, i32 0, i32 0
   call i32 (i8*, i8*, ...) @sprintf(i8* %dst, i8* %fmt, double 1.87)
-; CHECK-IPRINTF-NEXT: call i32 (i8*, i8*, ...) @sprintf(i8* %dst, i8* getelementptr inbounds ([3 x i8], [3 x i8]* @percent_f, i32 0, i32 0), double 1.870000e+00)
   ret void
-; CHECK-IPRINTF-NEXT: ret void
 }
 
 define void @test_no_simplify2(i8* %dst, i8* %fmt, double %d) {
 ; CHECK-LABEL: @test_no_simplify2(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 (i8*, i8*, ...) @sprintf(i8* nonnull dereferenceable(1) [[DST:%.*]], i8* nonnull dereferenceable(1) [[FMT:%.*]], double [[D:%.*]])
+; CHECK-NEXT:    ret void
+;
+; CHECK-IPRINTF-LABEL: @test_no_simplify2(
+; CHECK-IPRINTF-NEXT:    [[TMP1:%.*]] = call i32 (i8*, i8*, ...) @sprintf(i8* nonnull dereferenceable(1) [[DST:%.*]], i8* nonnull dereferenceable(1) [[FMT:%.*]], double [[D:%.*]])
+; CHECK-IPRINTF-NEXT:    ret void
+;
   call i32 (i8*, i8*, ...) @sprintf(i8* %dst, i8* %fmt, double %d)
-; CHECK-NEXT: call i32 (i8*, i8*, ...) @sprintf(i8* %dst, i8* %fmt, double %d)
   ret void
-; CHECK-NEXT: ret void
 }
diff --git a/llvm/test/Transforms/InstCombine/sqrt.ll b/llvm/test/Transforms/InstCombine/sqrt.ll
index 57719eb8a..55f952fe5 100644
--- a/llvm/test/Transforms/InstCombine/sqrt.ll
+++ b/llvm/test/Transforms/InstCombine/sqrt.ll
@@ -47,5 +47,17 @@ define float @test3(float* %v) nounwind uwtable ssp {
   ret float %conv38
 }
 
+; PR43347 - https://bugs.llvm.org/show_bug.cgi?id=43347
+
+define void @0(float %f) {
+; CHECK-LABEL: @0(
+; CHECK-NEXT:    [[SQRTF:%.*]] = call float @sqrtf(float [[F:%.*]]) #2
+; CHECK-NEXT:    ret void
+;
+  %d = fpext float %f to double
+  %r = call double @sqrt(double %d)
+  ret void
+}
+
 declare i32 @foo(double)
 declare double @sqrt(double) readnone
diff --git a/llvm/test/Transforms/InstCombine/srem-via-sdiv-mul-sub.ll b/llvm/test/Transforms/InstCombine/srem-via-sdiv-mul-sub.ll
new file mode 100644
index 000000000..8690faa7a
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/srem-via-sdiv-mul-sub.ll
@@ -0,0 +1,117 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instcombine -S | FileCheck %s
+
+; Fold
+;   x - ((x / y) * y)
+; to
+;   x % y
+
+; Also,
+;   ((x / y) * y)
+; can then be simplified to
+;   x - (x % y)
+
+declare void @use8(i8)
+declare void @use2xi8(<2 x i8>)
+
+define i8 @t0_basic(i8 %x, i8 %y) {
+; CHECK-LABEL: @t0_basic(
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[DIV]])
+; CHECK-NEXT:    [[ROUNDXDOWNTOMULTIPLEOFY:%.*]] = mul i8 [[DIV]], [[Y]]
+; CHECK-NEXT:    [[REM:%.*]] = sub i8 [[X]], [[ROUNDXDOWNTOMULTIPLEOFY]]
+; CHECK-NEXT:    ret i8 [[REM]]
+;
+  %div = sdiv i8 %x, %y
+  call void @use8(i8 %div)
+  %roundXdownToMultipleOfY = mul i8 %div, %y
+  %rem = sub i8 %x, %roundXdownToMultipleOfY
+  ret i8 %rem
+}
+
+define <2 x i8> @t1_vector(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @t1_vector(
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv <2 x i8> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use2xi8(<2 x i8> [[DIV]])
+; CHECK-NEXT:    [[ROUNDXDOWNTOMULTIPLEOFY:%.*]] = mul <2 x i8> [[DIV]], [[Y]]
+; CHECK-NEXT:    [[REM:%.*]] = sub <2 x i8> [[X]], [[ROUNDXDOWNTOMULTIPLEOFY]]
+; CHECK-NEXT:    ret <2 x i8> [[REM]]
+;
+  %div = sdiv <2 x i8> %x, %y
+  call void @use2xi8(<2 x i8> %div)
+  %roundXdownToMultipleOfY = mul <2 x i8> %div, %y
+  %rem = sub <2 x i8> %x, %roundXdownToMultipleOfY
+  ret <2 x i8> %rem
+}
+
+; Extra use
+
+define i8 @t4_extrause(i8 %x, i8 %y) {
+; CHECK-LABEL: @t4_extrause(
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[DIV]])
+; CHECK-NEXT:    [[ROUNDXDOWNTOMULTIPLEOFY:%.*]] = mul i8 [[DIV]], [[Y]]
+; CHECK-NEXT:    call void @use8(i8 [[ROUNDXDOWNTOMULTIPLEOFY]])
+; CHECK-NEXT:    [[REM:%.*]] = sub i8 [[X]], [[ROUNDXDOWNTOMULTIPLEOFY]]
+; CHECK-NEXT:    ret i8 [[REM]]
+;
+  %div = sdiv i8 %x, %y
+  call void @use8(i8 %div)
+  %roundXdownToMultipleOfY = mul i8 %div, %y
+  call void @use8(i8 %roundXdownToMultipleOfY)
+  %rem = sub i8 %x, %roundXdownToMultipleOfY
+  ret i8 %rem
+}
+
+; Commutativity
+
+declare i8 @gen8()
+
+define i8 @t5_commutative(i8 %x) {
+; CHECK-LABEL: @t5_commutative(
+; CHECK-NEXT:    [[Y:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv i8 [[X:%.*]], [[Y]]
+; CHECK-NEXT:    call void @use8(i8 [[DIV]])
+; CHECK-NEXT:    [[ROUNDXDOWNTOMULTIPLEOFY:%.*]] = mul i8 [[Y]], [[DIV]]
+; CHECK-NEXT:    [[REM:%.*]] = sub i8 [[X]], [[ROUNDXDOWNTOMULTIPLEOFY]]
+; CHECK-NEXT:    ret i8 [[REM]]
+;
+  %y = call i8 @gen8()
+  %div = sdiv i8 %x, %y
+  call void @use8(i8 %div)
+  %roundXdownToMultipleOfY = mul i8 %y, %div ; swapped
+  %rem = sub i8 %x, %roundXdownToMultipleOfY
+  ret i8 %rem
+}
+
+; Negative tests
+
+define i8 @n6_different_x(i8 %x0, i8 %x1, i8 %y) {
+; CHECK-LABEL: @n6_different_x(
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv i8 [[X0:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[DIV]])
+; CHECK-NEXT:    [[ROUNDXDOWNTOMULTIPLEOFY:%.*]] = mul i8 [[DIV]], [[Y]]
+; CHECK-NEXT:    [[REM:%.*]] = sub i8 [[X1:%.*]], [[ROUNDXDOWNTOMULTIPLEOFY]]
+; CHECK-NEXT:    ret i8 [[REM]]
+;
+  %div = sdiv i8 %x0, %y
+  call void @use8(i8 %div)
+  %roundXdownToMultipleOfY = mul i8 %div, %y
+  %rem = sub i8 %x1, %roundXdownToMultipleOfY
+  ret i8 %rem
+}
+
+define i8 @n6_different_y(i8 %x, i8 %y0, i8 %y1) {
+; CHECK-LABEL: @n6_different_y(
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv i8 [[X:%.*]], [[Y0:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[DIV]])
+; CHECK-NEXT:    [[ROUNDXDOWNTOMULTIPLEOFY:%.*]] = mul i8 [[DIV]], [[Y1:%.*]]
+; CHECK-NEXT:    [[REM:%.*]] = sub i8 [[X]], [[ROUNDXDOWNTOMULTIPLEOFY]]
+; CHECK-NEXT:    ret i8 [[REM]]
+;
+  %div = sdiv i8 %x, %y0
+  call void @use8(i8 %div)
+  %roundXdownToMultipleOfY = mul i8 %div, %y1
+  %rem = sub i8 %x, %roundXdownToMultipleOfY
+  ret i8 %rem
+}
diff --git a/llvm/test/Transforms/InstCombine/stacksave-debuginfo.ll b/llvm/test/Transforms/InstCombine/stacksave-debuginfo.ll
index 3c31c4c78..693ed870a 100644
--- a/llvm/test/Transforms/InstCombine/stacksave-debuginfo.ll
+++ b/llvm/test/Transforms/InstCombine/stacksave-debuginfo.ll
@@ -8,10 +8,11 @@ declare void @llvm.stackrestore(i8*) #0
 
 define i32* @test1(i32 %P) !dbg !6 {
 ; CHECK-LABEL: @test1(
-; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[P:%.*]] to i64, !dbg !12
-; CHECK-NEXT:    [[A:%.*]] = alloca i32, i64 [[TMP1]], align 4, !dbg !12
-; CHECK-NEXT:    call void @llvm.dbg.value(metadata i32* [[A]], metadata !11, metadata !DIExpression()), !dbg !12
-; CHECK-NEXT:    ret i32* [[A]], !dbg !13
+; CHECK-NEXT:    call void @llvm.dbg.value(metadata i8* undef
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[P:%.*]] to i64, !dbg !13
+; CHECK-NEXT:    [[A:%.*]] = alloca i32, i64 [[TMP1]], align 4, !dbg !13
+; CHECK-NEXT:    call void @llvm.dbg.value(metadata i32* [[A]]
+; CHECK-NEXT:    ret i32* [[A]], !dbg !14
 ;
   %tmp = call i8* @llvm.stacksave(), !dbg !12
   call void @llvm.dbg.value(metadata i8* %tmp, metadata !9, metadata !DIExpression()), !dbg !12
diff --git a/llvm/test/Transforms/InstCombine/stdio-custom-dl.ll b/llvm/test/Transforms/InstCombine/stdio-custom-dl.ll
new file mode 100644
index 000000000..c4a666db2
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/stdio-custom-dl.ll
@@ -0,0 +1,22 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S -mtriple=x86_64-unknown-linux-gnu | FileCheck %s
+
+target datalayout = "e-m:o-p:40:64:64:32-i64:64-f80:128-n8:16:32:64-S128"
+%struct._IO_FILE = type { i32, i8*, i8*, i8*, i8*, i8*, i8*, i8*, i8*, i8*, i8*, i8*, %struct._IO_marker*, %struct._IO_FILE*, i32, i32, i64, i16, i8, [1 x i8], i8*, i64, i8*, i8*, i8*, i8*, i64, i32, [20 x i8] }
+%struct._IO_marker = type { %struct._IO_marker*, %struct._IO_FILE*, i32 }
+@.str = private unnamed_addr constant [5 x i8] c"file\00", align 1
+@.str.1 = private unnamed_addr constant [2 x i8] c"w\00", align 1
+@.str.2 = private unnamed_addr constant [4 x i8] c"str\00", align 1
+
+; Check fwrite is generated with arguments of ptr size, not index size
+define internal void @fputs_test_custom_dl() {
+; CHECK-LABEL: @fputs_test_custom_dl(
+; CHECK-NEXT:    [[TMP1:%.*]] = call %struct._IO_FILE* @fopen(i8* getelementptr inbounds ([5 x i8], [5 x i8]* @.str, i32 0, i32 0), i8* getelementptr inbounds ([2 x i8], [2 x i8]* @.str.1, i32 0, i32 0))
+;
+  %call = call %struct._IO_FILE* @fopen(i8* getelementptr inbounds ([5 x i8], [5 x i8]* @.str, i64 0, i64 0), i8* getelementptr inbounds ([2 x i8], [2 x i8]* @.str.1, i64 0, i64 0))
+  %call1 = call i32 @fputs(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str.2, i64 0, i64 0), %struct._IO_FILE* %call)
+  ret void
+}
+
+declare %struct._IO_FILE* @fopen(i8*, i8*)
+declare i32 @fputs(i8* nocapture readonly, %struct._IO_FILE* nocapture)
diff --git a/llvm/test/Transforms/InstCombine/stpcpy-1.ll b/llvm/test/Transforms/InstCombine/stpcpy-1.ll
index cc8289937..260ccf6c0 100644
--- a/llvm/test/Transforms/InstCombine/stpcpy-1.ll
+++ b/llvm/test/Transforms/InstCombine/stpcpy-1.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; Test that the stpcpy library call simplifier works correctly.
 ; RUN: opt < %s -instcombine -S | FileCheck %s
 ;
@@ -13,34 +14,51 @@ declare i8* @stpcpy(i8*, i8*)
 
 define i8* @test_simplify1() {
 ; CHECK-LABEL: @test_simplify1(
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* nonnull align 1 dereferenceable(6) getelementptr inbounds ([32 x i8], [32 x i8]* @a, i32 0, i32 0), i8* nonnull align 1 dereferenceable(6) getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i32 0, i32 0), i32 6, i1 false)
+; CHECK-NEXT:    ret i8* getelementptr inbounds ([32 x i8], [32 x i8]* @a, i32 0, i32 5)
+;
 
   %dst = getelementptr [32 x i8], [32 x i8]* @a, i32 0, i32 0
   %src = getelementptr [6 x i8], [6 x i8]* @hello, i32 0, i32 0
 
   %ret = call i8* @stpcpy(i8* %dst, i8* %src)
-; CHECK: @llvm.memcpy.p0i8.p0i8.i32
-; CHECK-NEXT: getelementptr inbounds ([32 x i8], [32 x i8]* @a, i32 0, i32 5)
   ret i8* %ret
 }
 
 define i8* @test_simplify2() {
 ; CHECK-LABEL: @test_simplify2(
+; CHECK-NEXT:    [[STRLEN:%.*]] = call i32 @strlen(i8* nonnull dereferenceable(1) getelementptr inbounds ([32 x i8], [32 x i8]* @a, i32 0, i32 0))
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds [32 x i8], [32 x i8]* @a, i32 0, i32 [[STRLEN]]
+; CHECK-NEXT:    ret i8* [[TMP1]]
+;
 
   %dst = getelementptr [32 x i8], [32 x i8]* @a, i32 0, i32 0
 
   %ret = call i8* @stpcpy(i8* %dst, i8* %dst)
-; CHECK: [[LEN:%[a-z]+]] = call i32 @strlen
-; CHECK-NEXT: getelementptr inbounds [32 x i8], [32 x i8]* @a, i32 0, i32 [[LEN]]
   ret i8* %ret
 }
 
+define void @test_simplify3(i8* %dst) {
+; CHECK-LABEL: @test_simplify3(
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* nonnull align 1 dereferenceable(80) [[DST:%.*]], i8* nonnull align 1 dereferenceable(6) getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i32 0, i32 0), i32 6, i1 false)
+; CHECK-NEXT:    ret void
+;
+
+  %src = getelementptr [6 x i8], [6 x i8]* @hello, i32 0, i32 0
+
+  call i8* @stpcpy(i8* dereferenceable(80) %dst, i8* %src)
+  ret void
+}
+
 define i8* @test_no_simplify1() {
 ; CHECK-LABEL: @test_no_simplify1(
+; CHECK-NEXT:    [[RET:%.*]] = call i8* @stpcpy(i8* getelementptr inbounds ([32 x i8], [32 x i8]* @a, i32 0, i32 0), i8* getelementptr inbounds ([32 x i8], [32 x i8]* @b, i32 0, i32 0))
+; CHECK-NEXT:    ret i8* [[RET]]
+;
 
   %dst = getelementptr [32 x i8], [32 x i8]* @a, i32 0, i32 0
   %src = getelementptr [32 x i8], [32 x i8]* @b, i32 0, i32 0
 
   %ret = call i8* @stpcpy(i8* %dst, i8* %src)
-; CHECK: call i8* @stpcpy
   ret i8* %ret
 }
diff --git a/llvm/test/Transforms/InstCombine/stpcpy_chk-1.ll b/llvm/test/Transforms/InstCombine/stpcpy_chk-1.ll
index b2e0416ae..27244245a 100644
--- a/llvm/test/Transforms/InstCombine/stpcpy_chk-1.ll
+++ b/llvm/test/Transforms/InstCombine/stpcpy_chk-1.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; Test lib call simplification of __stpcpy_chk calls with various values
 ; for src, dst, and slen.
 ;
@@ -13,33 +14,36 @@ target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f3
 
 define i8* @test_simplify1() {
 ; CHECK-LABEL: @test_simplify1(
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* nonnull align 1 dereferenceable(12) getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* nonnull align 1 dereferenceable(12) getelementptr inbounds ([12 x i8], [12 x i8]* @.str, i32 0, i32 0), i32 12, i1 false)
+; CHECK-NEXT:    ret i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 11)
+;
   %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
   %src = getelementptr inbounds [12 x i8], [12 x i8]* @.str, i32 0, i32 0
 
-; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* align 1 getelementptr inbounds ([12 x i8], [12 x i8]* @.str, i32 0, i32 0), i32 12, i1 false)
-; CHECK-NEXT: ret i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 11)
   %ret = call i8* @__stpcpy_chk(i8* %dst, i8* %src, i32 60)
   ret i8* %ret
 }
 
 define i8* @test_simplify2() {
 ; CHECK-LABEL: @test_simplify2(
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* nonnull align 1 dereferenceable(12) getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* nonnull align 1 dereferenceable(12) getelementptr inbounds ([12 x i8], [12 x i8]* @.str, i32 0, i32 0), i32 12, i1 false)
+; CHECK-NEXT:    ret i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 11)
+;
   %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
   %src = getelementptr inbounds [12 x i8], [12 x i8]* @.str, i32 0, i32 0
 
-; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* align 1 getelementptr inbounds ([12 x i8], [12 x i8]* @.str, i32 0, i32 0), i32 12, i1 false)
-; CHECK-NEXT: ret i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 11)
   %ret = call i8* @__stpcpy_chk(i8* %dst, i8* %src, i32 12)
   ret i8* %ret
 }
 
 define i8* @test_simplify3() {
 ; CHECK-LABEL: @test_simplify3(
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* nonnull align 1 dereferenceable(12) getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* nonnull align 1 dereferenceable(12) getelementptr inbounds ([12 x i8], [12 x i8]* @.str, i32 0, i32 0), i32 12, i1 false)
+; CHECK-NEXT:    ret i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 11)
+;
   %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
   %src = getelementptr inbounds [12 x i8], [12 x i8]* @.str, i32 0, i32 0
 
-; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* align 1 getelementptr inbounds ([12 x i8], [12 x i8]* @.str, i32 0, i32 0), i32 12, i1 false)
-; CHECK-NEXT: ret i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 11)
   %ret = call i8* @__stpcpy_chk(i8* %dst, i8* %src, i32 -1)
   ret i8* %ret
 }
@@ -48,11 +52,12 @@ define i8* @test_simplify3() {
 
 define i8* @test_simplify4() {
 ; CHECK-LABEL: @test_simplify4(
+; CHECK-NEXT:    [[STPCPY:%.*]] = call i8* @stpcpy(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i32 0, i32 0))
+; CHECK-NEXT:    ret i8* [[STPCPY]]
+;
   %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
   %src = getelementptr inbounds [60 x i8], [60 x i8]* @b, i32 0, i32 0
 
-; CHECK-NEXT: %stpcpy = call i8* @stpcpy(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i32 0, i32 0))
-; CHECK-NEXT: ret i8* %stpcpy
   %ret = call i8* @__stpcpy_chk(i8* %dst, i8* %src, i32 -1)
   ret i8* %ret
 }
@@ -61,12 +66,13 @@ define i8* @test_simplify4() {
 
 define i8* @test_simplify5() {
 ; CHECK-LABEL: @test_simplify5(
+; CHECK-NEXT:    [[LEN:%.*]] = call i32 @llvm.objectsize.i32.p0i8(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i1 false, i1 false, i1 false)
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8* @__memcpy_chk(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* getelementptr inbounds ([12 x i8], [12 x i8]* @.str, i32 0, i32 0), i32 12, i32 [[LEN]])
+; CHECK-NEXT:    ret i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 11)
+;
   %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
   %src = getelementptr inbounds [12 x i8], [12 x i8]* @.str, i32 0, i32 0
 
-; CHECK-NEXT: %len = call i32 @llvm.objectsize.i32.p0i8(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i1 false, i1 false, i1 false)
-; CHECK-NEXT: %1 = call i8* @__memcpy_chk(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* getelementptr inbounds ([12 x i8], [12 x i8]* @.str, i32 0, i32 0), i32 12, i32 %len)
-; CHECK-NEXT: ret i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 11)
   %len = call i32 @llvm.objectsize.i32.p0i8(i8* %dst, i1 false, i1 false, i1 false)
   %ret = call i8* @__stpcpy_chk(i8* %dst, i8* %src, i32 %len)
   ret i8* %ret
@@ -76,11 +82,12 @@ define i8* @test_simplify5() {
 
 define i8* @test_simplify6() {
 ; CHECK-LABEL: @test_simplify6(
+; CHECK-NEXT:    [[STRLEN:%.*]] = call i32 @strlen(i8* nonnull dereferenceable(1) getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0))
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 [[STRLEN]]
+; CHECK-NEXT:    ret i8* [[TMP1]]
+;
   %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
 
-; CHECK-NEXT: %strlen = call i32 @strlen(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0))
-; CHECK-NEXT: %1 = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 %strlen
-; CHECK-NEXT: ret i8* %1
   %len = call i32 @llvm.objectsize.i32.p0i8(i8* %dst, i1 false, i1 false, i1 false)
   %ret = call i8* @__stpcpy_chk(i8* %dst, i8* %dst, i32 %len)
   ret i8* %ret
@@ -90,11 +97,12 @@ define i8* @test_simplify6() {
 
 define i8* @test_no_simplify1() {
 ; CHECK-LABEL: @test_no_simplify1(
+; CHECK-NEXT:    [[RET:%.*]] = call i8* @__stpcpy_chk(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i32 0, i32 0), i32 8)
+; CHECK-NEXT:    ret i8* [[RET]]
+;
   %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
   %src = getelementptr inbounds [60 x i8], [60 x i8]* @b, i32 0, i32 0
 
-; CHECK-NEXT: %ret = call i8* @__stpcpy_chk(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i32 0, i32 0), i32 8)
-; CHECK-NEXT: ret i8* %ret
   %ret = call i8* @__stpcpy_chk(i8* %dst, i8* %src, i32 8)
   ret i8* %ret
 }
diff --git a/llvm/test/Transforms/InstCombine/strchr-1.ll b/llvm/test/Transforms/InstCombine/strchr-1.ll
index 4fce378f5..081c3bf7a 100644
--- a/llvm/test/Transforms/InstCombine/strchr-1.ll
+++ b/llvm/test/Transforms/InstCombine/strchr-1.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; Test that the strchr library call simplifier works correctly.
 ; RUN: opt < %s -instcombine -S | FileCheck %s
 
@@ -11,9 +12,10 @@ target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f3
 declare i8* @strchr(i8*, i32)
 
 define void @test_simplify1() {
-; CHECK: store i8* getelementptr inbounds ([14 x i8], [14 x i8]* @hello, i32 0, i32 6)
-; CHECK-NOT: call i8* @strchr
-; CHECK: ret void
+; CHECK-LABEL: @test_simplify1(
+; CHECK-NEXT:    store i8* getelementptr inbounds ([14 x i8], [14 x i8]* @hello, i32 0, i32 6), i8** @chp, align 4
+; CHECK-NEXT:    ret void
+;
 
   %str = getelementptr [14 x i8], [14 x i8]* @hello, i32 0, i32 0
   %dst = call i8* @strchr(i8* %str, i32 119)
@@ -22,9 +24,10 @@ define void @test_simplify1() {
 }
 
 define void @test_simplify2() {
-; CHECK: store i8* null, i8** @chp, align 4
-; CHECK-NOT: call i8* @strchr
-; CHECK: ret void
+; CHECK-LABEL: @test_simplify2(
+; CHECK-NEXT:    store i8* null, i8** @chp, align 4
+; CHECK-NEXT:    ret void
+;
 
   %str = getelementptr [1 x i8], [1 x i8]* @null, i32 0, i32 0
   %dst = call i8* @strchr(i8* %str, i32 119)
@@ -33,9 +36,10 @@ define void @test_simplify2() {
 }
 
 define void @test_simplify3() {
-; CHECK: store i8* getelementptr inbounds ([14 x i8], [14 x i8]* @hello, i32 0, i32 13)
-; CHECK-NOT: call i8* @strchr
-; CHECK: ret void
+; CHECK-LABEL: @test_simplify3(
+; CHECK-NEXT:    store i8* getelementptr inbounds ([14 x i8], [14 x i8]* @hello, i32 0, i32 13), i8** @chp, align 4
+; CHECK-NEXT:    ret void
+;
 
   %src = getelementptr [14 x i8], [14 x i8]* @hello, i32 0, i32 0
   %dst = call i8* @strchr(i8* %src, i32 0)
@@ -44,9 +48,11 @@ define void @test_simplify3() {
 }
 
 define void @test_simplify4(i32 %chr) {
-; CHECK: call i8* @memchr
-; CHECK-NOT: call i8* @strchr
-; CHECK: ret void
+; CHECK-LABEL: @test_simplify4(
+; CHECK-NEXT:    [[MEMCHR:%.*]] = call i8* @memchr(i8* nonnull dereferenceable(14) getelementptr inbounds ([14 x i8], [14 x i8]* @hello, i32 0, i32 0), i32 [[CHR:%.*]], i32 14)
+; CHECK-NEXT:    store i8* [[MEMCHR]], i8** @chp, align 4
+; CHECK-NEXT:    ret void
+;
 
   %src = getelementptr [14 x i8], [14 x i8]* @hello, i32 0, i32 0
   %dst = call i8* @strchr(i8* %src, i32 %chr)
@@ -55,9 +61,10 @@ define void @test_simplify4(i32 %chr) {
 }
 
 define void @test_simplify5() {
-; CHECK: store i8* getelementptr inbounds ([14 x i8], [14 x i8]* @hello, i32 0, i32 13)
-; CHECK-NOT: call i8* @strchr
-; CHECK: ret void
+; CHECK-LABEL: @test_simplify5(
+; CHECK-NEXT:    store i8* getelementptr inbounds ([14 x i8], [14 x i8]* @hello, i32 0, i32 13), i8** @chp, align 4
+; CHECK-NEXT:    ret void
+;
 
   %src = getelementptr [14 x i8], [14 x i8]* @hello, i32 0, i32 0
   %dst = call i8* @strchr(i8* %src, i32 65280)
@@ -67,11 +74,12 @@ define void @test_simplify5() {
 
 ; Check transformation strchr(p, 0) -> p + strlen(p)
 define void @test_simplify6(i8* %str) {
-; CHECK: %strlen = call i32 @strlen(i8* %str)
-; CHECK-NOT: call i8* @strchr
-; CHECK: %strchr = getelementptr i8, i8* %str, i32 %strlen
-; CHECK: store i8* %strchr, i8** @chp, align 4
-; CHECK: ret void
+; CHECK-LABEL: @test_simplify6(
+; CHECK-NEXT:    [[STRLEN:%.*]] = call i32 @strlen(i8* nonnull dereferenceable(1) [[STR:%.*]])
+; CHECK-NEXT:    [[STRCHR:%.*]] = getelementptr i8, i8* [[STR]], i32 [[STRLEN]]
+; CHECK-NEXT:    store i8* [[STRCHR]], i8** @chp, align 4
+; CHECK-NEXT:    ret void
+;
 
   %dst = call i8* @strchr(i8* %str, i32 0)
   store i8* %dst, i8** @chp
@@ -80,17 +88,38 @@ define void @test_simplify6(i8* %str) {
 
 ; Check transformation strchr("\r\n", C) != nullptr -> (C & 9217) != 0
 define i1 @test_simplify7(i32 %C) {
-; CHECK-LABEL: @test_simplify7
-; CHECK-NEXT: [[TRUNC:%.*]] = trunc i32 %C to i16
-; CHECK-NEXT: [[TRUNC_AND:%.*]] = and i16 [[TRUNC]], 255
-; CHECK-NEXT: %memchr.bounds = icmp ult i16 [[TRUNC_AND]], 16
-; CHECK-NEXT: [[SHL:%.*]] = shl i16 1, [[TRUNC_AND]]
-; CHECK-NEXT: [[AND:%.*]] = and i16 [[SHL]], 9217
-; CHECK-NEXT: %memchr.bits = icmp ne i16 [[AND]], 0
-; CHECK-NEXT: %memchr1 = and i1 %memchr.bounds, %memchr.bits
-; CHECK-NEXT: ret i1 %memchr1
+; CHECK-LABEL: @test_simplify7(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[C:%.*]] to i16
+; CHECK-NEXT:    [[TMP2:%.*]] = and i16 [[TMP1]], 255
+; CHECK-NEXT:    [[MEMCHR_BOUNDS:%.*]] = icmp ult i16 [[TMP2]], 16
+; CHECK-NEXT:    [[TMP3:%.*]] = shl i16 1, [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = and i16 [[TMP3]], 9217
+; CHECK-NEXT:    [[MEMCHR_BITS:%.*]] = icmp ne i16 [[TMP4]], 0
+; CHECK-NEXT:    [[MEMCHR1:%.*]] = and i1 [[MEMCHR_BOUNDS]], [[MEMCHR_BITS]]
+; CHECK-NEXT:    ret i1 [[MEMCHR1]]
+;
 
   %dst = call i8* @strchr(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @newlines, i64 0, i64 0), i32 %C)
   %cmp = icmp ne i8* %dst, null
   ret i1 %cmp
 }
+
+define i8* @test1(i8* %str, i32 %c) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[RET:%.*]] = call i8* @strchr(i8* nonnull dereferenceable(1) [[STR:%.*]], i32 [[C:%.*]])
+; CHECK-NEXT:    ret i8* [[RET]]
+;
+
+  %ret = call i8* @strchr(i8* %str, i32 %c)
+  ret i8* %ret
+}
+
+define i8* @test2(i8* %str, i32 %c) "null-pointer-is-valid"="true" {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[RET:%.*]] = call i8* @strchr(i8* [[STR:%.*]], i32 [[C:%.*]])
+; CHECK-NEXT:    ret i8* [[RET]]
+;
+
+  %ret = call i8* @strchr(i8* %str, i32 %c)
+  ret i8* %ret
+}
diff --git a/llvm/test/Transforms/InstCombine/strcmp-1.ll b/llvm/test/Transforms/InstCombine/strcmp-1.ll
index 4dfda0472..9634999a3 100644
--- a/llvm/test/Transforms/InstCombine/strcmp-1.ll
+++ b/llvm/test/Transforms/InstCombine/strcmp-1.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; Test that the strcmp library call simplifier works correctly.
 ; RUN: opt < %s -instcombine -S | FileCheck %s --check-prefix=NOBCMP
 ; RUN: opt < %s -instcombine -mtriple=unknown-unknown-linux-gnu -S | FileCheck %s --check-prefix=BCMP
@@ -18,7 +19,18 @@ define i32 @test1(i8* %str2) {
 ; CHECK: %1 = zext i8 %strcmpload to i32
 ; CHECK: %2 = sub nsw i32 0, %1
 ; CHECK: ret i32 %2
-
+; NOBCMP-LABEL: @test1(
+; NOBCMP-NEXT:    [[STRCMPLOAD:%.*]] = load i8, i8* [[STR2:%.*]], align 1
+; NOBCMP-NEXT:    [[TMP1:%.*]] = zext i8 [[STRCMPLOAD]] to i32
+; NOBCMP-NEXT:    [[TMP2:%.*]] = sub nsw i32 0, [[TMP1]]
+; NOBCMP-NEXT:    ret i32 [[TMP2]]
+;
+; BCMP-LABEL: @test1(
+; BCMP-NEXT:    [[STRCMPLOAD:%.*]] = load i8, i8* [[STR2:%.*]], align 1
+; BCMP-NEXT:    [[TMP1:%.*]] = zext i8 [[STRCMPLOAD]] to i32
+; BCMP-NEXT:    [[TMP2:%.*]] = sub nsw i32 0, [[TMP1]]
+; BCMP-NEXT:    ret i32 [[TMP2]]
+;
   %str1 = getelementptr inbounds [1 x i8], [1 x i8]* @null, i32 0, i32 0
   %temp1 = call i32 @strcmp(i8* %str1, i8* %str2)
   ret i32 %temp1
@@ -31,7 +43,16 @@ define i32 @test2(i8* %str1) {
 ; CHECK: %strcmpload = load i8, i8* %str
 ; CHECK: %1 = zext i8 %strcmpload to i32
 ; CHECK: ret i32 %1
-
+; NOBCMP-LABEL: @test2(
+; NOBCMP-NEXT:    [[STRCMPLOAD:%.*]] = load i8, i8* [[STR1:%.*]], align 1
+; NOBCMP-NEXT:    [[TMP1:%.*]] = zext i8 [[STRCMPLOAD]] to i32
+; NOBCMP-NEXT:    ret i32 [[TMP1]]
+;
+; BCMP-LABEL: @test2(
+; BCMP-NEXT:    [[STRCMPLOAD:%.*]] = load i8, i8* [[STR1:%.*]], align 1
+; BCMP-NEXT:    [[TMP1:%.*]] = zext i8 [[STRCMPLOAD]] to i32
+; BCMP-NEXT:    ret i32 [[TMP1]]
+;
   %str2 = getelementptr inbounds [1 x i8], [1 x i8]* @null, i32 0, i32 0
   %temp1 = call i32 @strcmp(i8* %str1, i8* %str2)
   ret i32 %temp1
@@ -41,7 +62,12 @@ define i32 @test2(i8* %str1) {
 define i32 @test3() {
 ; CHECK-LABEL: @test3(
 ; CHECK: ret i32 -1
-
+; NOBCMP-LABEL: @test3(
+; NOBCMP-NEXT:    ret i32 -1
+;
+; BCMP-LABEL: @test3(
+; BCMP-NEXT:    ret i32 -1
+;
   %str1 = getelementptr inbounds [5 x i8], [5 x i8]* @hell, i32 0, i32 0
   %str2 = getelementptr inbounds [6 x i8], [6 x i8]* @hello, i32 0, i32 0
   %temp1 = call i32 @strcmp(i8* %str1, i8* %str2)
@@ -51,7 +77,12 @@ define i32 @test3() {
 define i32 @test4() {
 ; CHECK-LABEL: @test4(
 ; CHECK: ret i32 1
-
+; NOBCMP-LABEL: @test4(
+; NOBCMP-NEXT:    ret i32 1
+;
+; BCMP-LABEL: @test4(
+; BCMP-NEXT:    ret i32 1
+;
   %str1 = getelementptr inbounds [5 x i8], [5 x i8]* @hell, i32 0, i32 0
   %str2 = getelementptr inbounds [1 x i8], [1 x i8]* @null, i32 0, i32 0
   %temp1 = call i32 @strcmp(i8* %str1, i8* %str2)
@@ -64,7 +95,16 @@ define i32 @test5(i1 %b) {
 ; CHECK-LABEL: @test5(
 ; CHECK: %memcmp = call i32 @memcmp(i8* getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i32 0, i32 0), i8* %str2, i32 5)
 ; CHECK: ret i32 %memcmp
-
+; NOBCMP-LABEL: @test5(
+; NOBCMP-NEXT:    [[STR2:%.*]] = select i1 [[B:%.*]], i8* getelementptr inbounds ([5 x i8], [5 x i8]* @hell, i32 0, i32 0), i8* getelementptr inbounds ([5 x i8], [5 x i8]* @bell, i32 0, i32 0)
+; NOBCMP-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(5) getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i32 0, i32 0), i8* nonnull dereferenceable(5) [[STR2]], i32 5)
+; NOBCMP-NEXT:    ret i32 [[MEMCMP]]
+;
+; BCMP-LABEL: @test5(
+; BCMP-NEXT:    [[STR2:%.*]] = select i1 [[B:%.*]], i8* getelementptr inbounds ([5 x i8], [5 x i8]* @hell, i32 0, i32 0), i8* getelementptr inbounds ([5 x i8], [5 x i8]* @bell, i32 0, i32 0)
+; BCMP-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(5) getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i32 0, i32 0), i8* nonnull dereferenceable(5) [[STR2]], i32 5)
+; BCMP-NEXT:    ret i32 [[MEMCMP]]
+;
   %str1 = getelementptr inbounds [6 x i8], [6 x i8]* @hello, i32 0, i32 0
   %temp1 = getelementptr inbounds [5 x i8], [5 x i8]* @hell, i32 0, i32 0
   %temp2 = getelementptr inbounds [5 x i8], [5 x i8]* @bell, i32 0, i32 0
@@ -77,22 +117,31 @@ define i32 @test5(i1 %b) {
 define i32 @test6(i8* %str) {
 ; CHECK-LABEL: @test6(
 ; CHECK: ret i32 0
-
+; NOBCMP-LABEL: @test6(
+; NOBCMP-NEXT:    ret i32 0
+;
+; BCMP-LABEL: @test6(
+; BCMP-NEXT:    ret i32 0
+;
   %temp1 = call i32 @strcmp(i8* %str, i8* %str)
   ret i32 %temp1
 }
 
 ; strcmp(x, y) == 0  -> bcmp(x, y, <known length>)
 define i1 @test7(i1 %b) {
+; NOBCMP-LABEL: @test7(
+; NOBCMP-NEXT:    [[STR2:%.*]] = select i1 [[B:%.*]], i8* getelementptr inbounds ([5 x i8], [5 x i8]* @hell, i32 0, i32 0), i8* getelementptr inbounds ([5 x i8], [5 x i8]* @bell, i32 0, i32 0)
+; NOBCMP-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(5) getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i32 0, i32 0), i8* nonnull dereferenceable(5) [[STR2]], i32 5)
+; NOBCMP-NEXT:    [[RES:%.*]] = icmp eq i32 [[MEMCMP]], 0
+; NOBCMP-NEXT:    ret i1 [[RES]]
+;
 ; BCMP-LABEL: @test7(
-; BCMP: %bcmp = call i32 @bcmp(i8* getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i32 0, i32 0), i8* %str2, i32 5)
-; BCMP: %res = icmp eq i32 %bcmp, 0
-; BCMP: ret i1 %res
+; BCMP-NEXT:    [[STR2:%.*]] = select i1 [[B:%.*]], i8* getelementptr inbounds ([5 x i8], [5 x i8]* @hell, i32 0, i32 0), i8* getelementptr inbounds ([5 x i8], [5 x i8]* @bell, i32 0, i32 0)
+; BCMP-NEXT:    [[BCMP:%.*]] = call i32 @bcmp(i8* nonnull dereferenceable(5) getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i32 0, i32 0), i8* nonnull dereferenceable(5) [[STR2]], i32 5)
+; BCMP-NEXT:    [[RES:%.*]] = icmp eq i32 [[BCMP]], 0
+; BCMP-NEXT:    ret i1 [[RES]]
+;
 
-; NOBCMP-LABEL: @test7(
-; NOBCMP: %memcmp = call i32 @memcmp(i8* getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i32 0, i32 0), i8* %str2, i32 5)
-; NOBCMP: %res = icmp eq i32 %memcmp, 0
-; NOBCMP: ret i1 %res
 
   %str1 = getelementptr inbounds [6 x i8], [6 x i8]* @hello, i32 0, i32 0
   %temp1 = getelementptr inbounds [5 x i8], [5 x i8]* @hell, i32 0, i32 0
diff --git a/llvm/test/Transforms/InstCombine/strcmp-memcmp.ll b/llvm/test/Transforms/InstCombine/strcmp-memcmp.ll
index 092a47fe1..4c89b81b8 100644
--- a/llvm/test/Transforms/InstCombine/strcmp-memcmp.ll
+++ b/llvm/test/Transforms/InstCombine/strcmp-memcmp.ll
@@ -11,7 +11,7 @@ declare void @use(i32)
 define i32 @strcmp_memcmp([12 x i8]* dereferenceable (12) %buf) {
 ; CHECK-LABEL: @strcmp_memcmp(
 ; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
-; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull [[STRING]], i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i64 4)
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(4) [[STRING]], i8* nonnull dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i64 4)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[MEMCMP]], 0
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; CHECK-NEXT:    ret i32 [[CONV]]
@@ -28,7 +28,7 @@ declare i32 @strcmp(i8* nocapture, i8* nocapture)
 define i32 @strcmp_memcmp2([12 x i8]* dereferenceable (12) %buf) {
 ; CHECK-LABEL: @strcmp_memcmp2(
 ; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
-; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull [[STRING]], i64 4)
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull dereferenceable(4) [[STRING]], i64 4)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[MEMCMP]], 0
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; CHECK-NEXT:    ret i32 [[CONV]]
@@ -43,7 +43,7 @@ define i32 @strcmp_memcmp2([12 x i8]* dereferenceable (12) %buf) {
 define i32 @strcmp_memcmp3([12 x i8]* dereferenceable (12) %buf) {
 ; CHECK-LABEL: @strcmp_memcmp3(
 ; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
-; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull [[STRING]], i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i64 4)
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(4) [[STRING]], i8* nonnull dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i64 4)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[MEMCMP]], 0
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; CHECK-NEXT:    ret i32 [[CONV]]
@@ -58,7 +58,7 @@ define i32 @strcmp_memcmp3([12 x i8]* dereferenceable (12) %buf) {
 define i32 @strcmp_memcmp4([12 x i8]* dereferenceable (12) %buf) {
 ; CHECK-LABEL: @strcmp_memcmp4(
 ; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
-; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull [[STRING]], i64 4)
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull dereferenceable(4) [[STRING]], i64 4)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[MEMCMP]], 0
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; CHECK-NEXT:    ret i32 [[CONV]]
@@ -73,7 +73,7 @@ define i32 @strcmp_memcmp4([12 x i8]* dereferenceable (12) %buf) {
 define i32 @strcmp_memcmp5([5 x i8]* dereferenceable (5) %buf) {
 ; CHECK-LABEL: @strcmp_memcmp5(
 ; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [5 x i8], [5 x i8]* [[BUF:%.*]], i64 0, i64 0
-; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull [[STRING]], i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i64 4)
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(4) [[STRING]], i8* nonnull dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i64 4)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[MEMCMP]], 0
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; CHECK-NEXT:    ret i32 [[CONV]]
@@ -88,7 +88,7 @@ define i32 @strcmp_memcmp5([5 x i8]* dereferenceable (5) %buf) {
 define i32 @strcmp_memcmp6([12 x i8]* dereferenceable (12) %buf) {
 ; CHECK-LABEL: @strcmp_memcmp6(
 ; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
-; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull [[STRING]], i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i64 4)
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(4) [[STRING]], i8* nonnull dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i64 4)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[MEMCMP]], 0
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; CHECK-NEXT:    ret i32 [[CONV]]
@@ -103,7 +103,7 @@ define i32 @strcmp_memcmp6([12 x i8]* dereferenceable (12) %buf) {
 define i32 @strcmp_memcmp7([12 x i8]* dereferenceable (12) %buf) {
 ; CHECK-LABEL: @strcmp_memcmp7(
 ; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
-; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull [[STRING]], i64 4)
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull dereferenceable(4) [[STRING]], i64 4)
 ; CHECK-NEXT:    [[MEMCMP_LOBIT:%.*]] = lshr i32 [[MEMCMP]], 31
 ; CHECK-NEXT:    ret i32 [[MEMCMP_LOBIT]]
 ;
@@ -117,7 +117,7 @@ define i32 @strcmp_memcmp7([12 x i8]* dereferenceable (12) %buf) {
 define i32 @strcmp_memcmp8([4 x i8]* dereferenceable (4) %buf) {
 ; CHECK-LABEL: @strcmp_memcmp8(
 ; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [4 x i8], [4 x i8]* [[BUF:%.*]], i64 0, i64 0
-; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull [[STRING]], i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i64 4)
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(4) [[STRING]], i8* nonnull dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i64 4)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[MEMCMP]], 0
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; CHECK-NEXT:    ret i32 [[CONV]]
@@ -132,7 +132,7 @@ define i32 @strcmp_memcmp8([4 x i8]* dereferenceable (4) %buf) {
 define i32 @strcmp_memcmp9([12 x i8]* dereferenceable (12) %buf) {
 ; CHECK-LABEL: @strcmp_memcmp9(
 ; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
-; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull [[STRING]], i8* getelementptr inbounds ([8 x i8], [8 x i8]* @abc, i64 0, i64 0), i64 4)
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(4) [[STRING]], i8* nonnull dereferenceable(4) getelementptr inbounds ([8 x i8], [8 x i8]* @abc, i64 0, i64 0), i64 4)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[MEMCMP]], 0
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; CHECK-NEXT:    ret i32 [[CONV]]
@@ -148,7 +148,7 @@ define i32 @strcmp_memcmp9([12 x i8]* dereferenceable (12) %buf) {
 define i32 @strncmp_memcmp([12 x i8]* dereferenceable (12) %buf) {
 ; CHECK-LABEL: @strncmp_memcmp(
 ; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
-; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull [[STRING]], i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i64 2)
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(2) [[STRING]], i8* nonnull dereferenceable(2) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i64 2)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[MEMCMP]], 0
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; CHECK-NEXT:    ret i32 [[CONV]]
@@ -165,7 +165,7 @@ declare i32 @strncmp(i8* nocapture, i8* nocapture, i64)
 define i32 @strncmp_memcmp2([12 x i8]* dereferenceable (12) %buf) {
 ; CHECK-LABEL: @strncmp_memcmp2(
 ; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
-; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull [[STRING]], i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i64 4)
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(4) [[STRING]], i8* nonnull dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i64 4)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[MEMCMP]], 0
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; CHECK-NEXT:    ret i32 [[CONV]]
@@ -180,7 +180,7 @@ define i32 @strncmp_memcmp2([12 x i8]* dereferenceable (12) %buf) {
 define i32 @strncmp_memcmp3([12 x i8]* dereferenceable (12) %buf) {
 ; CHECK-LABEL: @strncmp_memcmp3(
 ; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
-; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull [[STRING]], i64 4)
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull dereferenceable(4) [[STRING]], i64 4)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[MEMCMP]], 0
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; CHECK-NEXT:    ret i32 [[CONV]]
@@ -195,7 +195,7 @@ define i32 @strncmp_memcmp3([12 x i8]* dereferenceable (12) %buf) {
 define i32 @strncmp_memcmp4([12 x i8]* dereferenceable (12) %buf) {
 ; CHECK-LABEL: @strncmp_memcmp4(
 ; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
-; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull [[STRING]], i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i64 4)
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(4) [[STRING]], i8* nonnull dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i64 4)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[MEMCMP]], 0
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; CHECK-NEXT:    ret i32 [[CONV]]
@@ -210,7 +210,7 @@ define i32 @strncmp_memcmp4([12 x i8]* dereferenceable (12) %buf) {
 define i32 @strncmp_memcmp5([12 x i8]* dereferenceable (12) %buf) {
 ; CHECK-LABEL: @strncmp_memcmp5(
 ; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
-; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull [[STRING]], i64 4)
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull dereferenceable(4) [[STRING]], i64 4)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[MEMCMP]], 0
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; CHECK-NEXT:    ret i32 [[CONV]]
@@ -226,7 +226,7 @@ define i32 @strncmp_memcmp5([12 x i8]* dereferenceable (12) %buf) {
 define i32 @strncmp_memcmp6([12 x i8]* dereferenceable (12) %buf) {
 ; CHECK-LABEL: @strncmp_memcmp6(
 ; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
-; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull [[STRING]], i64 4)
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull dereferenceable(4) [[STRING]], i64 4)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[MEMCMP]], 0
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; CHECK-NEXT:    ret i32 [[CONV]]
@@ -241,7 +241,7 @@ define i32 @strncmp_memcmp6([12 x i8]* dereferenceable (12) %buf) {
 define i32 @strncmp_memcmp7([12 x i8]* dereferenceable (12) %buf) {
 ; CHECK-LABEL: @strncmp_memcmp7(
 ; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
-; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull [[STRING]], i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i64 4)
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(4) [[STRING]], i8* nonnull dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i64 4)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[MEMCMP]], 0
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; CHECK-NEXT:    ret i32 [[CONV]]
@@ -256,7 +256,7 @@ define i32 @strncmp_memcmp7([12 x i8]* dereferenceable (12) %buf) {
 define i32 @strncmp_memcmp8([12 x i8]* dereferenceable (12) %buf) {
 ; CHECK-LABEL: @strncmp_memcmp8(
 ; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
-; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull [[STRING]], i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i64 3)
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(3) [[STRING]], i8* nonnull dereferenceable(3) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i64 3)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[MEMCMP]], 0
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; CHECK-NEXT:    ret i32 [[CONV]]
@@ -271,7 +271,7 @@ define i32 @strncmp_memcmp8([12 x i8]* dereferenceable (12) %buf) {
 define i32 @strncmp_memcmp9([12 x i8]* dereferenceable (12) %buf) {
 ; CHECK-LABEL: @strncmp_memcmp9(
 ; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
-; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull [[STRING]], i64 4)
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull dereferenceable(4) [[STRING]], i64 4)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[MEMCMP]], 0
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; CHECK-NEXT:    ret i32 [[CONV]]
@@ -286,7 +286,7 @@ define i32 @strncmp_memcmp9([12 x i8]* dereferenceable (12) %buf) {
 define i32 @strncmp_memcmp10([12 x i8]* dereferenceable (12) %buf) {
 ; CHECK-LABEL: @strncmp_memcmp10(
 ; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
-; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull [[STRING]], i64 4)
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull dereferenceable(4) [[STRING]], i64 4)
 ; CHECK-NEXT:    [[MEMCMP_LOBIT:%.*]] = lshr i32 [[MEMCMP]], 31
 ; CHECK-NEXT:    ret i32 [[MEMCMP_LOBIT]]
 ;
@@ -300,7 +300,7 @@ define i32 @strncmp_memcmp10([12 x i8]* dereferenceable (12) %buf) {
 define i32 @strncmp_memcmp11([12 x i8]* dereferenceable (12) %buf) {
 ; CHECK-LABEL: @strncmp_memcmp11(
 ; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
-; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull [[STRING]], i64 4)
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull dereferenceable(4) [[STRING]], i64 4)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[MEMCMP]], 0
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; CHECK-NEXT:    ret i32 [[CONV]]
@@ -315,7 +315,7 @@ define i32 @strncmp_memcmp11([12 x i8]* dereferenceable (12) %buf) {
 define i32 @strncmp_memcmp12([12 x i8]* dereferenceable (12) %buf) {
 ; CHECK-LABEL: @strncmp_memcmp12(
 ; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
-; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull [[STRING]], i64 4)
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull dereferenceable(4) [[STRING]], i64 4)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[MEMCMP]], 0
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; CHECK-NEXT:    ret i32 [[CONV]]
@@ -330,7 +330,7 @@ define i32 @strncmp_memcmp12([12 x i8]* dereferenceable (12) %buf) {
 define i32 @strncmp_memcmp13([12 x i8]* dereferenceable (12) %buf) {
 ; CHECK-LABEL: @strncmp_memcmp13(
 ; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
-; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull [[STRING]], i8* getelementptr inbounds ([8 x i8], [8 x i8]* @abc, i64 0, i64 0), i64 2)
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(2) [[STRING]], i8* nonnull dereferenceable(2) getelementptr inbounds ([8 x i8], [8 x i8]* @abc, i64 0, i64 0), i64 2)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[MEMCMP]], 0
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; CHECK-NEXT:    ret i32 [[CONV]]
@@ -345,7 +345,7 @@ define i32 @strncmp_memcmp13([12 x i8]* dereferenceable (12) %buf) {
 define i32 @strncmp_memcmp14([12 x i8]* dereferenceable (12) %buf) {
 ; CHECK-LABEL: @strncmp_memcmp14(
 ; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
-; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull [[STRING]], i8* getelementptr inbounds ([8 x i8], [8 x i8]* @abc, i64 0, i64 0), i64 4)
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(4) [[STRING]], i8* nonnull dereferenceable(4) getelementptr inbounds ([8 x i8], [8 x i8]* @abc, i64 0, i64 0), i64 4)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[MEMCMP]], 0
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; CHECK-NEXT:    ret i32 [[CONV]]
@@ -361,7 +361,7 @@ define i32 @strncmp_memcmp14([12 x i8]* dereferenceable (12) %buf) {
 define i32 @strcmp_memcmp_bad([12 x i8]* dereferenceable (12) %buf) {
 ; CHECK-LABEL: @strcmp_memcmp_bad(
 ; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
-; CHECK-NEXT:    [[CALL:%.*]] = call i32 @strcmp(i8* nonnull [[STRING]], i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0))
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @strcmp(i8* nonnull [[STRING]], i8* nonnull dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0))
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[CALL]], 3
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; CHECK-NEXT:    ret i32 [[CONV]]
@@ -376,7 +376,7 @@ define i32 @strcmp_memcmp_bad([12 x i8]* dereferenceable (12) %buf) {
 define i32 @strcmp_memcmp_bad2([12 x i8]* dereferenceable (12) %buf) {
 ; CHECK-LABEL: @strcmp_memcmp_bad2(
 ; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
-; CHECK-NEXT:    [[CALL:%.*]] = call i32 @strcmp(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull [[STRING]])
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @strcmp(i8* nonnull dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull [[STRING]])
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[CALL]], 3
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; CHECK-NEXT:    ret i32 [[CONV]]
@@ -391,7 +391,7 @@ define i32 @strcmp_memcmp_bad2([12 x i8]* dereferenceable (12) %buf) {
 define i32 @strcmp_memcmp_bad3([12 x i8]* dereferenceable (12) %buf) {
 ; CHECK-LABEL: @strcmp_memcmp_bad3(
 ; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
-; CHECK-NEXT:    [[CALL:%.*]] = call i32 @strcmp(i8* nonnull [[STRING]], i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0))
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @strcmp(i8* nonnull [[STRING]], i8* nonnull dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0))
 ; CHECK-NEXT:    ret i32 [[CALL]]
 ;
   %string = getelementptr inbounds [12 x i8], [12 x i8]* %buf, i64 0, i64 0
@@ -402,7 +402,7 @@ define i32 @strcmp_memcmp_bad3([12 x i8]* dereferenceable (12) %buf) {
 
 define i32 @strcmp_memcmp_bad4(i8* nocapture readonly %buf) {
 ; CHECK-LABEL: @strcmp_memcmp_bad4(
-; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 @strcmp(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* [[BUF:%.*]])
+; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 @strcmp(i8* nonnull dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull dereferenceable(1) [[BUF:%.*]])
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[CALL]], 0
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; CHECK-NEXT:    ret i32 [[CONV]]
@@ -417,7 +417,7 @@ define i32 @strcmp_memcmp_bad4(i8* nocapture readonly %buf) {
 define i32 @strcmp_memcmp_bad5([3 x i8]* dereferenceable (3) %buf) {
 ; CHECK-LABEL: @strcmp_memcmp_bad5(
 ; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [3 x i8], [3 x i8]* [[BUF:%.*]], i64 0, i64 0
-; CHECK-NEXT:    [[CALL:%.*]] = call i32 @strcmp(i8* nonnull [[STRING]], i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0))
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @strcmp(i8* nonnull [[STRING]], i8* nonnull dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0))
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[CALL]], 0
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; CHECK-NEXT:    ret i32 [[CONV]]
@@ -432,7 +432,7 @@ define i32 @strcmp_memcmp_bad5([3 x i8]* dereferenceable (3) %buf) {
 define i32 @strcmp_memcmp_bad6([4 x i8]* dereferenceable (4) %buf, i8* nocapture readonly %k) {
 ; CHECK-LABEL: @strcmp_memcmp_bad6(
 ; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [4 x i8], [4 x i8]* [[BUF:%.*]], i64 0, i64 0
-; CHECK-NEXT:    [[CALL:%.*]] = call i32 @strcmp(i8* nonnull [[STRING]], i8* [[K:%.*]])
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @strcmp(i8* nonnull [[STRING]], i8* nonnull dereferenceable(1) [[K:%.*]])
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[CALL]], 0
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; CHECK-NEXT:    ret i32 [[CONV]]
@@ -446,7 +446,7 @@ define i32 @strcmp_memcmp_bad6([4 x i8]* dereferenceable (4) %buf, i8* nocapture
 
 define i32 @strcmp_memcmp_bad7(i8* nocapture readonly %k) {
 ; CHECK-LABEL: @strcmp_memcmp_bad7(
-; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 @strcmp(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* [[K:%.*]])
+; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 @strcmp(i8* nonnull dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull dereferenceable(1) [[K:%.*]])
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[CALL]], 0
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; CHECK-NEXT:    ret i32 [[CONV]]
@@ -460,7 +460,7 @@ define i32 @strcmp_memcmp_bad7(i8* nocapture readonly %k) {
 define i32 @strcmp_memcmp_bad8([4 x i8]* dereferenceable (4) %buf) {
 ; CHECK-LABEL: @strcmp_memcmp_bad8(
 ; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [4 x i8], [4 x i8]* [[BUF:%.*]], i64 0, i64 0
-; CHECK-NEXT:    [[CALL:%.*]] = call i32 @strcmp(i8* nonnull [[STRING]], i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0))
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @strcmp(i8* nonnull [[STRING]], i8* nonnull dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0))
 ; CHECK-NEXT:    tail call void @use(i32 [[CALL]])
 ; CHECK-NEXT:    ret i32 0
 ;
@@ -473,7 +473,7 @@ define i32 @strcmp_memcmp_bad8([4 x i8]* dereferenceable (4) %buf) {
 define i32 @strncmp_memcmp_bad([12 x i8]* dereferenceable (12) %buf) {
 ; CHECK-LABEL: @strncmp_memcmp_bad(
 ; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
-; CHECK-NEXT:    [[CALL:%.*]] = call i32 @strncmp(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull [[STRING]], i64 5)
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @strncmp(i8* nonnull dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull [[STRING]], i64 5)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[CALL]], 3
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; CHECK-NEXT:    ret i32 [[CONV]]
@@ -489,7 +489,7 @@ define i32 @strncmp_memcmp_bad([12 x i8]* dereferenceable (12) %buf) {
 define i32 @strncmp_memcmp_bad1([12 x i8]* dereferenceable (12) %buf) {
 ; CHECK-LABEL: @strncmp_memcmp_bad1(
 ; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
-; CHECK-NEXT:    [[CALL:%.*]] = call i32 @strncmp(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull [[STRING]], i64 5)
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @strncmp(i8* nonnull dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull [[STRING]], i64 5)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[CALL]], 3
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; CHECK-NEXT:    ret i32 [[CONV]]
@@ -518,7 +518,7 @@ define i32 @strncmp_memcmp_bad2([12 x i8]* dereferenceable (12) %buf, i64 %n) {
 
 define i32 @strncmp_memcmp_bad3(i8* nocapture readonly %k) {
 ; CHECK-LABEL: @strncmp_memcmp_bad3(
-; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 @strncmp(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* [[K:%.*]], i64 2)
+; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 @strncmp(i8* nonnull dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull dereferenceable(1) [[K:%.*]], i64 2)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[CALL]], 0
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; CHECK-NEXT:    ret i32 [[CONV]]
@@ -532,7 +532,7 @@ define i32 @strncmp_memcmp_bad3(i8* nocapture readonly %k) {
 define i32 @strncmp_memcmp_bad4([4 x i8]* dereferenceable (4) %buf) {
 ; CHECK-LABEL: @strncmp_memcmp_bad4(
 ; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [4 x i8], [4 x i8]* [[BUF:%.*]], i64 0, i64 0
-; CHECK-NEXT:    [[CALL:%.*]] = call i32 @strncmp(i8* nonnull [[STRING]], i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i64 2)
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @strncmp(i8* nonnull [[STRING]], i8* nonnull dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i64 2)
 ; CHECK-NEXT:    tail call void @use(i32 [[CALL]])
 ; CHECK-NEXT:    ret i32 0
 ;
@@ -545,7 +545,7 @@ define i32 @strncmp_memcmp_bad4([4 x i8]* dereferenceable (4) %buf) {
 define i32 @strcmp_memcmp_msan([12 x i8]* dereferenceable (12) %buf) sanitize_memory {
 ; CHECK-LABEL: @strcmp_memcmp_msan(
 ; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
-; CHECK-NEXT:    [[CALL:%.*]] = call i32 @strcmp(i8* nonnull [[STRING]], i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0))
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @strcmp(i8* nonnull [[STRING]], i8* nonnull dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0))
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[CALL]], 0
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; CHECK-NEXT:    ret i32 [[CONV]]
diff --git a/llvm/test/Transforms/InstCombine/strcpy-1.ll b/llvm/test/Transforms/InstCombine/strcpy-1.ll
index 24c70c18f..0ccf983a7 100644
--- a/llvm/test/Transforms/InstCombine/strcpy-1.ll
+++ b/llvm/test/Transforms/InstCombine/strcpy-1.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; Test that the strcpy library call simplifier works correctly.
 ; rdar://6839935
 ; RUN: opt < %s -instcombine -S | FileCheck %s
@@ -14,32 +15,50 @@ declare i8* @strcpy(i8*, i8*)
 
 define void @test_simplify1() {
 ; CHECK-LABEL: @test_simplify1(
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* nonnull align 1 dereferenceable(6) getelementptr inbounds ([32 x i8], [32 x i8]* @a, i32 0, i32 0), i8* nonnull align 1 dereferenceable(6) getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i32 0, i32 0), i32 6, i1 false)
+; CHECK-NEXT:    ret void
+;
 
   %dst = getelementptr [32 x i8], [32 x i8]* @a, i32 0, i32 0
   %src = getelementptr [6 x i8], [6 x i8]* @hello, i32 0, i32 0
 
   call i8* @strcpy(i8* %dst, i8* %src)
-; CHECK: @llvm.memcpy.p0i8.p0i8.i32
   ret void
 }
 
 define i8* @test_simplify2() {
 ; CHECK-LABEL: @test_simplify2(
+; CHECK-NEXT:    ret i8* getelementptr inbounds ([32 x i8], [32 x i8]* @a, i32 0, i32 0)
+;
 
   %dst = getelementptr [32 x i8], [32 x i8]* @a, i32 0, i32 0
 
   %ret = call i8* @strcpy(i8* %dst, i8* %dst)
-; CHECK: ret i8* getelementptr inbounds ([32 x i8], [32 x i8]* @a, i32 0, i32 0)
   ret i8* %ret
 }
 
+
+define void @test_simplify3(i8* %dst) {
+; CHECK-LABEL: @test_simplify3(
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* nonnull align 1 dereferenceable(80) [[DST:%.*]], i8* nonnull align 1 dereferenceable(6) getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i32 0, i32 0), i32 6, i1 false)
+; CHECK-NEXT:    ret void
+;
+
+  %src = getelementptr [6 x i8], [6 x i8]* @hello, i32 0, i32 0
+
+  call i8* @strcpy(i8* dereferenceable(80) %dst, i8* %src)
+  ret void
+}
+
 define i8* @test_no_simplify1() {
 ; CHECK-LABEL: @test_no_simplify1(
+; CHECK-NEXT:    [[RET:%.*]] = call i8* @strcpy(i8* nonnull dereferenceable(1) getelementptr inbounds ([32 x i8], [32 x i8]* @a, i32 0, i32 0), i8* nonnull dereferenceable(1) getelementptr inbounds ([32 x i8], [32 x i8]* @b, i32 0, i32 0))
+; CHECK-NEXT:    ret i8* [[RET]]
+;
 
   %dst = getelementptr [32 x i8], [32 x i8]* @a, i32 0, i32 0
   %src = getelementptr [32 x i8], [32 x i8]* @b, i32 0, i32 0
 
   %ret = call i8* @strcpy(i8* %dst, i8* %src)
-; CHECK: call i8* @strcpy
   ret i8* %ret
 }
diff --git a/llvm/test/Transforms/InstCombine/strcpy_chk-1.ll b/llvm/test/Transforms/InstCombine/strcpy_chk-1.ll
index 859d810d3..02a4b5cbd 100644
--- a/llvm/test/Transforms/InstCombine/strcpy_chk-1.ll
+++ b/llvm/test/Transforms/InstCombine/strcpy_chk-1.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; Test lib call simplification of __strcpy_chk calls with various values
 ; for src, dst, and slen.
 ;
@@ -13,33 +14,36 @@ target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f3
 
 define i8* @test_simplify1() {
 ; CHECK-LABEL: @test_simplify1(
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* nonnull align 1 dereferenceable(12) getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* nonnull align 1 dereferenceable(12) getelementptr inbounds ([12 x i8], [12 x i8]* @.str, i32 0, i32 0), i32 12, i1 false)
+; CHECK-NEXT:    ret i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0)
+;
   %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
   %src = getelementptr inbounds [12 x i8], [12 x i8]* @.str, i32 0, i32 0
 
-; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* align 1 getelementptr inbounds ([12 x i8], [12 x i8]* @.str, i32 0, i32 0), i32 12, i1 false)
-; CHECK-NEXT: ret i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0)
   %ret = call i8* @__strcpy_chk(i8* %dst, i8* %src, i32 60)
   ret i8* %ret
 }
 
 define i8* @test_simplify2() {
 ; CHECK-LABEL: @test_simplify2(
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* nonnull align 1 dereferenceable(12) getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* nonnull align 1 dereferenceable(12) getelementptr inbounds ([12 x i8], [12 x i8]* @.str, i32 0, i32 0), i32 12, i1 false)
+; CHECK-NEXT:    ret i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0)
+;
   %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
   %src = getelementptr inbounds [12 x i8], [12 x i8]* @.str, i32 0, i32 0
 
-; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* align 1 getelementptr inbounds ([12 x i8], [12 x i8]* @.str, i32 0, i32 0), i32 12, i1 false)
-; CHECK-NEXT: ret i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0)
   %ret = call i8* @__strcpy_chk(i8* %dst, i8* %src, i32 12)
   ret i8* %ret
 }
 
 define i8* @test_simplify3() {
 ; CHECK-LABEL: @test_simplify3(
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* nonnull align 1 dereferenceable(12) getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* nonnull align 1 dereferenceable(12) getelementptr inbounds ([12 x i8], [12 x i8]* @.str, i32 0, i32 0), i32 12, i1 false)
+; CHECK-NEXT:    ret i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0)
+;
   %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
   %src = getelementptr inbounds [12 x i8], [12 x i8]* @.str, i32 0, i32 0
 
-; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* align 1 getelementptr inbounds ([12 x i8], [12 x i8]* @.str, i32 0, i32 0), i32 12, i1 false)
-; CHECK-NEXT: ret i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0)
   %ret = call i8* @__strcpy_chk(i8* %dst, i8* %src, i32 -1)
   ret i8* %ret
 }
@@ -48,11 +52,12 @@ define i8* @test_simplify3() {
 
 define i8* @test_simplify4() {
 ; CHECK-LABEL: @test_simplify4(
+; CHECK-NEXT:    [[STRCPY:%.*]] = call i8* @strcpy(i8* nonnull dereferenceable(1) getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* nonnull dereferenceable(1) getelementptr inbounds ([60 x i8], [60 x i8]* @b, i32 0, i32 0))
+; CHECK-NEXT:    ret i8* [[STRCPY]]
+;
   %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
   %src = getelementptr inbounds [60 x i8], [60 x i8]* @b, i32 0, i32 0
 
-; CHECK-NEXT: %strcpy = call i8* @strcpy(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i32 0, i32 0))
-; CHECK-NEXT: ret i8* %strcpy
   %ret = call i8* @__strcpy_chk(i8* %dst, i8* %src, i32 -1)
   ret i8* %ret
 }
@@ -61,12 +66,13 @@ define i8* @test_simplify4() {
 
 define i8* @test_simplify5() {
 ; CHECK-LABEL: @test_simplify5(
+; CHECK-NEXT:    [[LEN:%.*]] = call i32 @llvm.objectsize.i32.p0i8(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i1 false, i1 false, i1 false)
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8* @__memcpy_chk(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* getelementptr inbounds ([12 x i8], [12 x i8]* @.str, i32 0, i32 0), i32 12, i32 [[LEN]])
+; CHECK-NEXT:    ret i8* [[TMP1]]
+;
   %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
   %src = getelementptr inbounds [12 x i8], [12 x i8]* @.str, i32 0, i32 0
 
-; CHECK-NEXT: %len = call i32 @llvm.objectsize.i32.p0i8(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i1 false, i1 false, i1 false)
-; CHECK-NEXT: %1 = call i8* @__memcpy_chk(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* getelementptr inbounds ([12 x i8], [12 x i8]* @.str, i32 0, i32 0), i32 12, i32 %len)
-; CHECK-NEXT: ret i8* %1
   %len = call i32 @llvm.objectsize.i32.p0i8(i8* %dst, i1 false, i1 false, i1 false)
   %ret = call i8* @__strcpy_chk(i8* %dst, i8* %src, i32 %len)
   ret i8* %ret
@@ -76,11 +82,12 @@ define i8* @test_simplify5() {
 
 define i8* @test_simplify6() {
 ; CHECK-LABEL: @test_simplify6(
+; CHECK-NEXT:    [[LEN:%.*]] = call i32 @llvm.objectsize.i32.p0i8(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i1 false, i1 false, i1 false)
+; CHECK-NEXT:    [[RET:%.*]] = call i8* @__strcpy_chk(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i32 [[LEN]])
+; CHECK-NEXT:    ret i8* [[RET]]
+;
   %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
 
-; CHECK-NEXT: %len = call i32 @llvm.objectsize.i32.p0i8(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i1 false, i1 false, i1 false)
-; CHECK-NEXT: %ret = call i8* @__strcpy_chk(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i32 %len)
-; CHECK-NEXT: ret i8* %ret
   %len = call i32 @llvm.objectsize.i32.p0i8(i8* %dst, i1 false, i1 false, i1 false)
   %ret = call i8* @__strcpy_chk(i8* %dst, i8* %dst, i32 %len)
   ret i8* %ret
@@ -90,11 +97,12 @@ define i8* @test_simplify6() {
 
 define i8* @test_no_simplify1() {
 ; CHECK-LABEL: @test_no_simplify1(
+; CHECK-NEXT:    [[RET:%.*]] = call i8* @__strcpy_chk(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i32 0, i32 0), i32 8)
+; CHECK-NEXT:    ret i8* [[RET]]
+;
   %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
   %src = getelementptr inbounds [60 x i8], [60 x i8]* @b, i32 0, i32 0
 
-; CHECK-NEXT: %ret = call i8* @__strcpy_chk(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i32 0, i32 0), i32 8)
-; CHECK-NEXT: ret i8* %ret
   %ret = call i8* @__strcpy_chk(i8* %dst, i8* %src, i32 8)
   ret i8* %ret
 }
diff --git a/llvm/test/Transforms/InstCombine/strcspn-1.ll b/llvm/test/Transforms/InstCombine/strcspn-1.ll
index 8d441a9ee..c1f728584 100644
--- a/llvm/test/Transforms/InstCombine/strcspn-1.ll
+++ b/llvm/test/Transforms/InstCombine/strcspn-1.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; Test that the strcspn library call simplifier works correctly.
 ;
 ; RUN: opt < %s -instcombine -S | FileCheck %s
@@ -14,44 +15,48 @@ declare i64 @strcspn(i8*, i8*)
 
 define i64 @test_simplify1(i8* %str) {
 ; CHECK-LABEL: @test_simplify1(
+; CHECK-NEXT:    [[STRLEN:%.*]] = call i64 @strlen(i8* nonnull dereferenceable(1) [[STR:%.*]])
+; CHECK-NEXT:    ret i64 [[STRLEN]]
+;
   %pat = getelementptr [1 x i8], [1 x i8]* @null, i32 0, i32 0
 
   %ret = call i64 @strcspn(i8* %str, i8* %pat)
-; CHECK-NEXT: [[VAR:%[a-z]+]] = call i64 @strlen(i8* %str)
   ret i64 %ret
-; CHECK-NEXT: ret i64 [[VAR]]
 }
 
 ; Check strcspn("", s) -> 0.
 
 define i64 @test_simplify2(i8* %pat) {
 ; CHECK-LABEL: @test_simplify2(
+; CHECK-NEXT:    ret i64 0
+;
   %str = getelementptr [1 x i8], [1 x i8]* @null, i32 0, i32 0
 
   %ret = call i64 @strcspn(i8* %str, i8* %pat)
   ret i64 %ret
-; CHECK-NEXT: ret i64 0
 }
 
 ; Check strcspn(s1, s2), where s1 and s2 are constants.
 
 define i64 @test_simplify3() {
 ; CHECK-LABEL: @test_simplify3(
+; CHECK-NEXT:    ret i64 0
+;
   %str = getelementptr [6 x i8], [6 x i8]* @abcba, i32 0, i32 0
   %pat = getelementptr [4 x i8], [4 x i8]* @abc, i32 0, i32 0
 
   %ret = call i64 @strcspn(i8* %str, i8* %pat)
   ret i64 %ret
-; CHECK-NEXT: ret i64 0
 }
 
 ; Check cases that shouldn't be simplified.
 
 define i64 @test_no_simplify1(i8* %str, i8* %pat) {
 ; CHECK-LABEL: @test_no_simplify1(
+; CHECK-NEXT:    [[RET:%.*]] = call i64 @strcspn(i8* [[STR:%.*]], i8* [[PAT:%.*]])
+; CHECK-NEXT:    ret i64 [[RET]]
+;
 
   %ret = call i64 @strcspn(i8* %str, i8* %pat)
-; CHECK-NEXT: %ret = call i64 @strcspn(i8* %str, i8* %pat)
   ret i64 %ret
-; CHECK-NEXT: ret i64 %ret
 }
diff --git a/llvm/test/Transforms/InstCombine/strict-sub-underflow-check-to-comparison-of-sub-operands.ll b/llvm/test/Transforms/InstCombine/strict-sub-underflow-check-to-comparison-of-sub-operands.ll
new file mode 100644
index 000000000..9abb588a4
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/strict-sub-underflow-check-to-comparison-of-sub-operands.ll
@@ -0,0 +1,111 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instcombine -S | FileCheck %s
+
+declare void @llvm.assume(i1)
+declare i8 @gen8()
+declare void @use8(i8)
+
+define i1 @t0(i8 %base, i8 %offset) {
+; CHECK-LABEL: @t0(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[OFFSET:%.*]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i8 [[BASE:%.*]], [[OFFSET]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[RES:%.*]] = icmp uge i8 [[BASE]], [[OFFSET]]
+; CHECK-NEXT:    ret i1 [[RES]]
+;
+  %cmp = icmp slt i8 %offset, 0
+  call void @llvm.assume(i1 %cmp)
+
+  %adjusted = sub i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %res = icmp ult i8 %adjusted, %base
+  ret i1 %res
+}
+define i1 @t1(i8 %base, i8 %offset) {
+; CHECK-LABEL: @t1(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[OFFSET:%.*]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i8 [[BASE:%.*]], [[OFFSET]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[RES:%.*]] = icmp ult i8 [[BASE]], [[OFFSET]]
+; CHECK-NEXT:    ret i1 [[RES]]
+;
+  %cmp = icmp slt i8 %offset, 0
+  call void @llvm.assume(i1 %cmp)
+
+  %adjusted = sub i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %res = icmp uge i8 %adjusted, %base
+  ret i1 %res
+}
+define i1 @t2(i8 %offset) {
+; CHECK-LABEL: @t2(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[OFFSET:%.*]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    [[BASE:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i8 [[BASE]], [[OFFSET]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[RES:%.*]] = icmp uge i8 [[BASE]], [[OFFSET]]
+; CHECK-NEXT:    ret i1 [[RES]]
+;
+  %cmp = icmp slt i8 %offset, 0
+  call void @llvm.assume(i1 %cmp)
+
+  %base = call i8 @gen8()
+  %adjusted = sub i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %res = icmp ugt i8 %base, %adjusted
+  ret i1 %res
+}
+define i1 @t3(i8 %offset) {
+; CHECK-LABEL: @t3(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[OFFSET:%.*]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    [[BASE:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i8 [[BASE]], [[OFFSET]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[RES:%.*]] = icmp ult i8 [[BASE]], [[OFFSET]]
+; CHECK-NEXT:    ret i1 [[RES]]
+;
+  %cmp = icmp slt i8 %offset, 0
+  call void @llvm.assume(i1 %cmp)
+
+  %base = call i8 @gen8()
+  %adjusted = sub i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %res = icmp ule i8 %base, %adjusted
+  ret i1 %res
+}
+
+; Here we don't know that offset is non-zero. Can't fold.
+define i1 @n4_maybezero(i8 %base, i8 %offset) {
+; CHECK-LABEL: @n4_maybezero(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i8 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[RES:%.*]] = icmp ult i8 [[ADJUSTED]], [[BASE]]
+; CHECK-NEXT:    ret i1 [[RES]]
+;
+  %adjusted = sub i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %res = icmp ult i8 %adjusted, %base
+  ret i1 %res
+}
+; We need to know that about %offset, %base won't do. Can't fold.
+define i1 @n5_wrongnonzero(i8 %base, i8 %offset) {
+; CHECK-LABEL: @n5_wrongnonzero(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i8 [[BASE:%.*]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i8 [[BASE]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[RES:%.*]] = icmp ult i8 [[ADJUSTED]], [[BASE]]
+; CHECK-NEXT:    ret i1 [[RES]]
+;
+  %cmp = icmp sgt i8 %base, 0
+  call void @llvm.assume(i1 %cmp)
+
+  %adjusted = sub i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %res = icmp ult i8 %adjusted, %base
+  ret i1 %res
+}
diff --git a/llvm/test/Transforms/InstCombine/strlen-1.ll b/llvm/test/Transforms/InstCombine/strlen-1.ll
index aaf1b8994..0ecf233a2 100644
--- a/llvm/test/Transforms/InstCombine/strlen-1.ll
+++ b/llvm/test/Transforms/InstCombine/strlen-1.ll
@@ -100,7 +100,7 @@ define i1 @test_simplify8(i8* %str_p) {
 
 define i32 @test_simplify9(i1 %x) {
 ; CHECK-LABEL: @test_simplify9(
-; CHECK-NEXT:    [[TMP1:%.*]] = select i1 %x, i32 5, i32 6
+; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[X:%.*]], i32 5, i32 6
 ; CHECK-NEXT:    ret i32 [[TMP1]]
 ;
   %hello = getelementptr [6 x i8], [6 x i8]* @hello, i32 0, i32 0
@@ -115,7 +115,7 @@ define i32 @test_simplify9(i1 %x) {
 
 define i32 @test_simplify10(i32 %x) {
 ; CHECK-LABEL: @test_simplify10(
-; CHECK-NEXT:    [[TMP1:%.*]] = sub i32 5, %x
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i32 5, [[X:%.*]]
 ; CHECK-NEXT:    ret i32 [[TMP1]]
 ;
   %hello_p = getelementptr inbounds [6 x i8], [6 x i8]* @hello, i32 0, i32 %x
@@ -127,7 +127,7 @@ define i32 @test_simplify10(i32 %x) {
 
 define i32 @test_simplify11(i32 %x) {
 ; CHECK-LABEL: @test_simplify11(
-; CHECK-NEXT:    [[AND:%.*]] = and i32 %x, 7
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 7
 ; CHECK-NEXT:    [[TMP1:%.*]] = sub nuw nsw i32 9, [[AND]]
 ; CHECK-NEXT:    ret i32 [[TMP1]]
 ;
@@ -141,7 +141,7 @@ define i32 @test_simplify11(i32 %x) {
 
 define i32 @test_no_simplify1() {
 ; CHECK-LABEL: @test_no_simplify1(
-; CHECK-NEXT:    [[A_L:%.*]] = call i32 @strlen(i8* getelementptr inbounds ([32 x i8], [32 x i8]* @a, i32 0, i32 0))
+; CHECK-NEXT:    [[A_L:%.*]] = call i32 @strlen(i8* nonnull dereferenceable(1) getelementptr inbounds ([32 x i8], [32 x i8]* @a, i32 0, i32 0))
 ; CHECK-NEXT:    ret i32 [[A_L]]
 ;
   %a_p = getelementptr [32 x i8], [32 x i8]* @a, i32 0, i32 0
@@ -153,7 +153,7 @@ define i32 @test_no_simplify1() {
 
 define i32 @test_no_simplify2(i32 %x) {
 ; CHECK-LABEL: @test_no_simplify2(
-; CHECK-NEXT:    [[HELLO_P:%.*]] = getelementptr inbounds [7 x i8], [7 x i8]* @null_hello, i32 0, i32 %x
+; CHECK-NEXT:    [[HELLO_P:%.*]] = getelementptr inbounds [7 x i8], [7 x i8]* @null_hello, i32 0, i32 [[X:%.*]]
 ; CHECK-NEXT:    [[HELLO_L:%.*]] = call i32 @strlen(i8* nonnull [[HELLO_P]])
 ; CHECK-NEXT:    ret i32 [[HELLO_L]]
 ;
@@ -164,7 +164,7 @@ define i32 @test_no_simplify2(i32 %x) {
 
 define i32 @test_no_simplify2_no_null_opt(i32 %x) #0 {
 ; CHECK-LABEL: @test_no_simplify2_no_null_opt(
-; CHECK-NEXT:    [[HELLO_P:%.*]] = getelementptr inbounds [7 x i8], [7 x i8]* @null_hello, i32 0, i32 %x
+; CHECK-NEXT:    [[HELLO_P:%.*]] = getelementptr inbounds [7 x i8], [7 x i8]* @null_hello, i32 0, i32 [[X:%.*]]
 ; CHECK-NEXT:    [[HELLO_L:%.*]] = call i32 @strlen(i8* [[HELLO_P]])
 ; CHECK-NEXT:    ret i32 [[HELLO_L]]
 ;
@@ -177,7 +177,7 @@ define i32 @test_no_simplify2_no_null_opt(i32 %x) #0 {
 
 define i32 @test_no_simplify3(i32 %x) {
 ; CHECK-LABEL: @test_no_simplify3(
-; CHECK-NEXT:    [[AND:%.*]] = and i32 %x, 15
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 15
 ; CHECK-NEXT:    [[HELLO_P:%.*]] = getelementptr inbounds [13 x i8], [13 x i8]* @null_hello_mid, i32 0, i32 [[AND]]
 ; CHECK-NEXT:    [[HELLO_L:%.*]] = call i32 @strlen(i8* nonnull [[HELLO_P]])
 ; CHECK-NEXT:    ret i32 [[HELLO_L]]
@@ -190,7 +190,7 @@ define i32 @test_no_simplify3(i32 %x) {
 
 define i32 @test_no_simplify3_on_null_opt(i32 %x) #0 {
 ; CHECK-LABEL: @test_no_simplify3_on_null_opt(
-; CHECK-NEXT:    [[AND:%.*]] = and i32 %x, 15
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 15
 ; CHECK-NEXT:    [[HELLO_P:%.*]] = getelementptr inbounds [13 x i8], [13 x i8]* @null_hello_mid, i32 0, i32 [[AND]]
 ; CHECK-NEXT:    [[HELLO_L:%.*]] = call i32 @strlen(i8* [[HELLO_P]])
 ; CHECK-NEXT:    ret i32 [[HELLO_L]]
@@ -201,4 +201,22 @@ define i32 @test_no_simplify3_on_null_opt(i32 %x) #0 {
   ret i32 %hello_l
 }
 
+define i32 @test1(i8* %str) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[LEN:%.*]] = tail call i32 @strlen(i8* nonnull dereferenceable(1) [[STR:%.*]]) #1
+; CHECK-NEXT:    ret i32 [[LEN]]
+;
+  %len = tail call i32 @strlen(i8* %str) nounwind
+  ret i32 %len
+}
+
+define i32 @test2(i8* %str) #0 {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[LEN:%.*]] = tail call i32 @strlen(i8* [[STR:%.*]]) #1
+; CHECK-NEXT:    ret i32 [[LEN]]
+;
+  %len = tail call i32 @strlen(i8* %str) nounwind
+  ret i32 %len
+}
+
 attributes #0 = { "null-pointer-is-valid"="true" }
diff --git a/llvm/test/Transforms/InstCombine/strncat-2.ll b/llvm/test/Transforms/InstCombine/strncat-2.ll
index 3533e1eac..4aed71829 100644
--- a/llvm/test/Transforms/InstCombine/strncat-2.ll
+++ b/llvm/test/Transforms/InstCombine/strncat-2.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; Test that the strncat libcall simplifier works correctly.
 ;
 ; RUN: opt < %s -instcombine -S | FileCheck %s
@@ -7,13 +8,15 @@ target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f3
 @hello = constant [6 x i8] c"hello\00"
 @empty = constant [1 x i8] c"\00"
 @a = common global [32 x i8] zeroinitializer, align 1
-
 declare i8* @strncat(i8*, i8*, i32)
 
 define void @test_simplify1() {
 ; CHECK-LABEL: @test_simplify1(
-; CHECK-NOT: call i8* @strncat
-; CHECK: ret void
+; CHECK-NEXT:    [[STRLEN:%.*]] = call i32 @strlen(i8* nonnull dereferenceable(1) getelementptr inbounds ([32 x i8], [32 x i8]* @a, i32 0, i32 0))
+; CHECK-NEXT:    [[ENDPTR:%.*]] = getelementptr [32 x i8], [32 x i8]* @a, i32 0, i32 [[STRLEN]]
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* nonnull align 1 dereferenceable(6) [[ENDPTR]], i8* nonnull align 1 dereferenceable(6) getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i32 0, i32 0), i32 6, i1 false)
+; CHECK-NEXT:    ret void
+;
 
   %dst = getelementptr [32 x i8], [32 x i8]* @a, i32 0, i32 0
   %src = getelementptr [6 x i8], [6 x i8]* @hello, i32 0, i32 0
@@ -23,7 +26,8 @@ define void @test_simplify1() {
 
 define void @test_simplify2() {
 ; CHECK-LABEL: @test_simplify2(
-; CHECK-NEXT: ret void
+; CHECK-NEXT:    ret void
+;
 
   %dst = getelementptr [32 x i8], [32 x i8]* @a, i32 0, i32 0
   %src = getelementptr [1 x i8], [1 x i8]* @empty, i32 0, i32 0
@@ -33,7 +37,8 @@ define void @test_simplify2() {
 
 define void @test_simplify3() {
 ; CHECK-LABEL: @test_simplify3(
-; CHECK-NEXT: ret void
+; CHECK-NEXT:    ret void
+;
 
   %dst = getelementptr [32 x i8], [32 x i8]* @a, i32 0, i32 0
   %src = getelementptr [6 x i8], [6 x i8]* @hello, i32 0, i32 0
@@ -43,11 +48,66 @@ define void @test_simplify3() {
 
 define void @test_nosimplify1() {
 ; CHECK-LABEL: @test_nosimplify1(
-; CHECK: call i8* @strncat
-; CHECK: ret void
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8* @strncat(i8* nonnull dereferenceable(1) getelementptr inbounds ([32 x i8], [32 x i8]* @a, i32 0, i32 0), i8* nonnull dereferenceable(6) getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i32 0, i32 0), i32 1)
+; CHECK-NEXT:    ret void
+;
 
   %dst = getelementptr [32 x i8], [32 x i8]* @a, i32 0, i32 0
   %src = getelementptr [6 x i8], [6 x i8]* @hello, i32 0, i32 0
   call i8* @strncat(i8* %dst, i8* %src, i32 1)
   ret void
 }
+
+; strncat(nonnull x, nonnull y, n)  -> strncat(nonnull x, y, n)
+define i8* @test1(i8* %str1, i8* %str2, i32 %n) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[TEMP1:%.*]] = call i8* @strncat(i8* nonnull [[STR1:%.*]], i8* nonnull [[STR2:%.*]], i32 [[N:%.*]])
+; CHECK-NEXT:    ret i8* [[TEMP1]]
+;
+
+  %temp1 = call i8* @strncat(i8* nonnull %str1, i8* nonnull %str2, i32 %n)
+  ret i8* %temp1
+}
+
+; strncat(x, y, 0)  -> x
+define i8* @test2(i8* %str1, i8* %str2, i32 %n) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    ret i8* [[STR1:%.*]]
+;
+
+  %temp1 = call i8* @strncat(i8* %str1, i8* %str2, i32 0)
+  ret i8* %temp1
+}
+
+; strncat(x, y, 5)  -> strncat(nonnull x, nonnull y, 5)
+define i8* @test3(i8* %str1, i8* %str2, i32 %n) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    [[TEMP1:%.*]] = call i8* @strncat(i8* nonnull dereferenceable(1) [[STR1:%.*]], i8* nonnull dereferenceable(1) [[STR2:%.*]], i32 5)
+; CHECK-NEXT:    ret i8* [[TEMP1]]
+;
+
+  %temp1 = call i8* @strncat(i8* %str1, i8* %str2, i32 5)
+  ret i8* %temp1
+}
+
+define i8* @test4(i8* %str1, i8* %str2, i32 %n) "null-pointer-is-valid"="true" {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    [[TEMP1:%.*]] = call i8* @strncat(i8* [[STR1:%.*]], i8* [[STR2:%.*]], i32 [[N:%.*]])
+; CHECK-NEXT:    ret i8* [[TEMP1]]
+;
+
+  %temp1 = call i8* @strncat(i8* %str1, i8* %str2, i32 %n)
+  ret i8* %temp1
+}
+
+define i8* @test5(i8* %str, i32 %n) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    [[STRLEN:%.*]] = call i32 @strlen(i8* nonnull dereferenceable(1) [[STR:%.*]])
+; CHECK-NEXT:    [[ENDPTR:%.*]] = getelementptr i8, i8* [[STR]], i32 [[STRLEN]]
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* nonnull align 1 dereferenceable(6) [[ENDPTR]], i8* nonnull align 1 dereferenceable(6) getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i32 0, i32 0), i32 6, i1 false)
+; CHECK-NEXT:    ret i8* [[STR]]
+;
+  %src = getelementptr [6 x i8], [6 x i8]* @hello, i32 0, i32 0
+  %temp1 = call i8* @strncat(i8* %str, i8* %src, i32 10)
+  ret i8* %temp1
+}
diff --git a/llvm/test/Transforms/InstCombine/strncat-3.ll b/llvm/test/Transforms/InstCombine/strncat-3.ll
index 0f6964df9..d87688378 100644
--- a/llvm/test/Transforms/InstCombine/strncat-3.ll
+++ b/llvm/test/Transforms/InstCombine/strncat-3.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; Test that the strncat libcall simplifier works correctly.
 ;
 ; RUN: opt < %s -instcombine -S | FileCheck %s
@@ -12,8 +13,9 @@ declare i16* @strncat(i8*, i8*, i32)
 
 define void @test_nosimplify1() {
 ; CHECK-LABEL: @test_nosimplify1(
-; CHECK: call i16* @strncat
-; CHECK: ret void
+; CHECK-NEXT:    [[TMP1:%.*]] = call i16* @strncat(i8* getelementptr inbounds ([32 x i8], [32 x i8]* @a, i32 0, i32 0), i8* getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i32 0, i32 0), i32 13)
+; CHECK-NEXT:    ret void
+;
 
   %dst = getelementptr [32 x i8], [32 x i8]* @a, i32 0, i32 0
   %src = getelementptr [6 x i8], [6 x i8]* @hello, i32 0, i32 0
diff --git a/llvm/test/Transforms/InstCombine/strncmp-1.ll b/llvm/test/Transforms/InstCombine/strncmp-1.ll
index a1121821f..4bb03cc7a 100644
--- a/llvm/test/Transforms/InstCombine/strncmp-1.ll
+++ b/llvm/test/Transforms/InstCombine/strncmp-1.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; Test that the strncmp library call simplifier works correctly.
 ; RUN: opt < %s -instcombine -S | FileCheck %s
 
@@ -13,10 +14,11 @@ declare i32 @strncmp(i8*, i8*, i32)
 ; strncmp("", x, n) -> -*x
 define i32 @test1(i8* %str2) {
 ; CHECK-LABEL: @test1(
-; CHECK: %strcmpload = load i8, i8* %str
-; CHECK: %1 = zext i8 %strcmpload to i32
-; CHECK: %2 = sub nsw i32 0, %1
-; CHECK: ret i32 %2
+; CHECK-NEXT:    [[STRCMPLOAD:%.*]] = load i8, i8* [[STR2:%.*]], align 1
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[STRCMPLOAD]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = sub nsw i32 0, [[TMP1]]
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
 
   %str1 = getelementptr inbounds [1 x i8], [1 x i8]* @null, i32 0, i32 0
   %temp1 = call i32 @strncmp(i8* %str1, i8* %str2, i32 10)
@@ -26,9 +28,10 @@ define i32 @test1(i8* %str2) {
 ; strncmp(x, "", n) -> *x
 define i32 @test2(i8* %str1) {
 ; CHECK-LABEL: @test2(
-; CHECK: %strcmpload = load i8, i8* %str1
-; CHECK: %1 = zext i8 %strcmpload to i32
-; CHECK: ret i32 %1
+; CHECK-NEXT:    [[STRCMPLOAD:%.*]] = load i8, i8* [[STR1:%.*]], align 1
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[STRCMPLOAD]] to i32
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
 
   %str2 = getelementptr inbounds [1 x i8], [1 x i8]* @null, i32 0, i32 0
   %temp1 = call i32 @strncmp(i8* %str1, i8* %str2, i32 10)
@@ -38,7 +41,8 @@ define i32 @test2(i8* %str1) {
 ; strncmp(x, y, n)  -> cnst
 define i32 @test3() {
 ; CHECK-LABEL: @test3(
-; CHECK: ret i32 -1
+; CHECK-NEXT:    ret i32 -1
+;
 
   %str1 = getelementptr inbounds [5 x i8], [5 x i8]* @hell, i32 0, i32 0
   %str2 = getelementptr inbounds [6 x i8], [6 x i8]* @hello, i32 0, i32 0
@@ -48,7 +52,8 @@ define i32 @test3() {
 
 define i32 @test4() {
 ; CHECK-LABEL: @test4(
-; CHECK: ret i32 1
+; CHECK-NEXT:    ret i32 1
+;
 
   %str1 = getelementptr inbounds [5 x i8], [5 x i8]* @hell, i32 0, i32 0
   %str2 = getelementptr inbounds [1 x i8], [1 x i8]* @null, i32 0, i32 0
@@ -58,7 +63,8 @@ define i32 @test4() {
 
 define i32 @test5() {
 ; CHECK-LABEL: @test5(
-; CHECK: ret i32 0
+; CHECK-NEXT:    ret i32 0
+;
 
   %str1 = getelementptr inbounds [5 x i8], [5 x i8]* @hell, i32 0, i32 0
   %str2 = getelementptr inbounds [6 x i8], [6 x i8]* @hello, i32 0, i32 0
@@ -69,12 +75,13 @@ define i32 @test5() {
 ; strncmp(x,y,1) -> memcmp(x,y,1)
 define i32 @test6(i8* %str1, i8* %str2) {
 ; CHECK-LABEL: @test6(
-; CHECK: [[LOAD1:%[a-z]+]] = load i8, i8* %str1, align 1
-; CHECK: [[ZEXT1:%[a-z]+]] = zext i8 [[LOAD1]] to i32
-; CHECK: [[LOAD2:%[a-z]+]] = load i8, i8* %str2, align 1
-; CHECK: [[ZEXT2:%[a-z]+]] = zext i8 [[LOAD2]] to i32
-; CHECK: [[RET:%[a-z]+]] = sub nsw i32 [[ZEXT1]], [[ZEXT2]]
-; CHECK: ret i32 [[RET]]
+; CHECK-NEXT:    [[LHSC:%.*]] = load i8, i8* [[STR1:%.*]], align 1
+; CHECK-NEXT:    [[LHSV:%.*]] = zext i8 [[LHSC]] to i32
+; CHECK-NEXT:    [[RHSC:%.*]] = load i8, i8* [[STR2:%.*]], align 1
+; CHECK-NEXT:    [[RHSV:%.*]] = zext i8 [[RHSC]] to i32
+; CHECK-NEXT:    [[CHARDIFF:%.*]] = sub nsw i32 [[LHSV]], [[RHSV]]
+; CHECK-NEXT:    ret i32 [[CHARDIFF]]
+;
 
   %temp1 = call i32 @strncmp(i8* %str1, i8* %str2, i32 1)
   ret i32 %temp1
@@ -83,7 +90,8 @@ define i32 @test6(i8* %str1, i8* %str2) {
 ; strncmp(x,y,0)   -> 0
 define i32 @test7(i8* %str1, i8* %str2) {
 ; CHECK-LABEL: @test7(
-; CHECK: ret i32 0
+; CHECK-NEXT:    ret i32 0
+;
 
   %temp1 = call i32 @strncmp(i8* %str1, i8* %str2, i32 0)
   ret i32 %temp1
@@ -92,8 +100,51 @@ define i32 @test7(i8* %str1, i8* %str2) {
 ; strncmp(x,x,n)  -> 0
 define i32 @test8(i8* %str, i32 %n) {
 ; CHECK-LABEL: @test8(
-; CHECK: ret i32 0
+; CHECK-NEXT:    ret i32 0
+;
 
   %temp1 = call i32 @strncmp(i8* %str, i8* %str, i32 %n)
   ret i32 %temp1
 }
+
+; strncmp(nonnull x, nonnull y, n)  -> strncmp(x, y, n)
+define i32 @test9(i8* %str1, i8* %str2, i32 %n) {
+; CHECK-LABEL: @test9(
+; CHECK-NEXT:    [[TEMP1:%.*]] = call i32 @strncmp(i8* nonnull [[STR1:%.*]], i8* nonnull [[STR2:%.*]], i32 [[N:%.*]])
+; CHECK-NEXT:    ret i32 [[TEMP1]]
+;
+
+  %temp1 = call i32 @strncmp(i8* nonnull %str1, i8* nonnull %str2, i32 %n)
+  ret i32 %temp1
+}
+
+; strncmp(nonnull x, nonnull y, 0)  -> 0
+define i32 @test10(i8* %str1, i8* %str2, i32 %n) {
+; CHECK-LABEL: @test10(
+; CHECK-NEXT:    ret i32 0
+;
+
+  %temp1 = call i32 @strncmp(i8* nonnull %str1, i8* nonnull %str2, i32 0)
+  ret i32 %temp1
+}
+
+; strncmp(x, y, 5)  -> strncmp(nonnull x, nonnull y, 5)
+define i32 @test11(i8* %str1, i8* %str2, i32 %n) {
+; CHECK-LABEL: @test11(
+; CHECK-NEXT:    [[TEMP1:%.*]] = call i32 @strncmp(i8* nonnull dereferenceable(1) [[STR1:%.*]], i8* nonnull dereferenceable(1) [[STR2:%.*]], i32 5)
+; CHECK-NEXT:    ret i32 [[TEMP1]]
+;
+
+  %temp1 = call i32 @strncmp(i8* %str1, i8* %str2, i32 5)
+  ret i32 %temp1
+}
+
+define i32 @test12(i8* %str1, i8* %str2, i32 %n) "null-pointer-is-valid"="true" {
+; CHECK-LABEL: @test12(
+; CHECK-NEXT:    [[TEMP1:%.*]] = call i32 @strncmp(i8* [[STR1:%.*]], i8* [[STR2:%.*]], i32 [[N:%.*]])
+; CHECK-NEXT:    ret i32 [[TEMP1]]
+;
+
+  %temp1 = call i32 @strncmp(i8* %str1, i8* %str2, i32 %n)
+  ret i32 %temp1
+}
diff --git a/llvm/test/Transforms/InstCombine/strncmp-2.ll b/llvm/test/Transforms/InstCombine/strncmp-2.ll
index 9e19781e0..5afad9e19 100644
--- a/llvm/test/Transforms/InstCombine/strncmp-2.ll
+++ b/llvm/test/Transforms/InstCombine/strncmp-2.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; Test that the strncmp library call simplifier works correctly.
 ; RUN: opt < %s -instcombine -S | FileCheck %s
 
@@ -10,8 +11,9 @@ declare i16 @strncmp(i8*, i8*, i32)
 
 define i16 @test_nosimplify() {
 ; CHECK-LABEL: @test_nosimplify(
-; CHECK: call i16 @strncmp
-; CHECK: ret i16 %temp1
+; CHECK-NEXT:    [[TEMP1:%.*]] = call i16 @strncmp(i8* getelementptr inbounds ([5 x i8], [5 x i8]* @hell, i32 0, i32 0), i8* getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i32 0, i32 0), i32 10)
+; CHECK-NEXT:    ret i16 [[TEMP1]]
+;
 
   %str1 = getelementptr inbounds [5 x i8], [5 x i8]* @hell, i32 0, i32 0
   %str2 = getelementptr inbounds [6 x i8], [6 x i8]* @hello, i32 0, i32 0
diff --git a/llvm/test/Transforms/InstCombine/strncpy-1.ll b/llvm/test/Transforms/InstCombine/strncpy-1.ll
index 28cb26b17..969358eb4 100644
--- a/llvm/test/Transforms/InstCombine/strncpy-1.ll
+++ b/llvm/test/Transforms/InstCombine/strncpy-1.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; Test that the strncpy library call simplifier works correctly.
 ;
 ; RUN: opt < %s -instcombine -S | FileCheck %s
@@ -17,8 +18,15 @@ declare i32 @puts(i8*)
 
 define i32 @test_simplify1() {
 ; CHECK-LABEL: @test_simplify1(
-; CHECK-NOT: call i8* @strncpy
-; CHECK: call i32 @puts
+; CHECK-NEXT:    [[TARGET:%.*]] = alloca [1024 x i8], align 1
+; CHECK-NEXT:    [[ARG1:%.*]] = getelementptr inbounds [1024 x i8], [1024 x i8]* [[TARGET]], i32 0, i32 0
+; CHECK-NEXT:    store i8 0, i8* [[ARG1]], align 1
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* nonnull align 1 dereferenceable(6) [[ARG1]], i8* nonnull align 1 dereferenceable(6) getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i32 0, i32 0), i32 6, i1 false)
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i32(i8* nonnull align 1 dereferenceable(42) [[ARG1]], i8 0, i32 42, i1 false)
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i32(i8* nonnull align 1 dereferenceable(42) [[ARG1]], i8 0, i32 42, i1 false)
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @puts(i8* nonnull [[ARG1]])
+; CHECK-NEXT:    ret i32 0
+;
   %target = alloca [1024 x i8]
   %arg1 = getelementptr [1024 x i8], [1024 x i8]* %target, i32 0, i32 0
   store i8 0, i8* %arg1
@@ -40,11 +48,13 @@ define i32 @test_simplify1() {
 
 define void @test_simplify2() {
 ; CHECK-LABEL: @test_simplify2(
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i32(i8* nonnull align 1 dereferenceable(32) getelementptr inbounds ([32 x i8], [32 x i8]* @a, i32 0, i32 0), i8 0, i32 32, i1 false)
+; CHECK-NEXT:    ret void
+;
   %dst = getelementptr [32 x i8], [32 x i8]* @a, i32 0, i32 0
   %src = getelementptr [1 x i8], [1 x i8]* @null, i32 0, i32 0
 
   call i8* @strncpy(i8* %dst, i8* %src, i32 32)
-; CHECK: call void @llvm.memset.p0i8.i32
   ret void
 }
 
@@ -52,44 +62,110 @@ define void @test_simplify2() {
 
 define i8* @test_simplify3() {
 ; CHECK-LABEL: @test_simplify3(
+; CHECK-NEXT:    ret i8* getelementptr inbounds ([32 x i8], [32 x i8]* @a, i32 0, i32 0)
+;
   %dst = getelementptr [32 x i8], [32 x i8]* @a, i32 0, i32 0
   %src = getelementptr [6 x i8], [6 x i8]* @hello, i32 0, i32 0
 
   %ret = call i8* @strncpy(i8* %dst, i8* %src, i32 0)
   ret i8* %ret
-; CHECK: ret i8* getelementptr inbounds ([32 x i8], [32 x i8]* @a, i32 0, i32 0)
 }
 
 ; Check  strncpy(x, s, c) -> memcpy(x, s, c, 1) [s and c are constant].
 
 define void @test_simplify4() {
 ; CHECK-LABEL: @test_simplify4(
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* nonnull align 1 dereferenceable(6) getelementptr inbounds ([32 x i8], [32 x i8]* @a, i32 0, i32 0), i8* nonnull align 1 dereferenceable(6) getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i32 0, i32 0), i32 6, i1 false)
+; CHECK-NEXT:    ret void
+;
   %dst = getelementptr [32 x i8], [32 x i8]* @a, i32 0, i32 0
   %src = getelementptr [6 x i8], [6 x i8]* @hello, i32 0, i32 0
 
   call i8* @strncpy(i8* %dst, i8* %src, i32 6)
-; CHECK: call void @llvm.memcpy.p0i8.p0i8.i32
   ret void
 }
 
+define void @test_simplify5(i8* %dst) {
+; CHECK-LABEL: @test_simplify5(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8* @strncpy(i8* nonnull dereferenceable(8) [[DST:%.*]], i8* nonnull dereferenceable(6) getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i32 0, i32 0), i32 32)
+; CHECK-NEXT:    ret void
+;
+  %src = getelementptr [6 x i8], [6 x i8]* @hello, i32 0, i32 0
+  call i8* @strncpy(i8* dereferenceable(8) %dst, i8* %src, i32 32)
+  ret void
+}
+
+define void @test_simplify6(i8* %dst) {
+; CHECK-LABEL: @test_simplify6(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8* @strncpy(i8* nonnull dereferenceable(80) [[DST:%.*]], i8* nonnull dereferenceable(6) getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i32 0, i32 0), i32 32)
+; CHECK-NEXT:    ret void
+;
+  %src = getelementptr [6 x i8], [6 x i8]* @hello, i32 0, i32 0
+  call i8* @strncpy(i8* dereferenceable(80) %dst, i8* %src, i32 32)
+  ret void
+}
+
+define void @test_simplify7(i8* %dst, i32 %n) {
+; CHECK-LABEL: @test_simplify7(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8* @strncpy(i8* nonnull dereferenceable(80) [[DST:%.*]], i8* getelementptr inbounds ([1 x i8], [1 x i8]* @null, i32 0, i32 0), i32 [[N:%.*]])
+; CHECK-NEXT:    ret void
+;
+  %src = getelementptr [1 x i8], [1 x i8]* @null, i32 0, i32 0
+  call i8* @strncpy(i8* dereferenceable(80) %dst, i8* %src, i32 %n)
+  ret void
+}
+
+define i8* @test1(i8* %dst, i8* %src, i32 %n) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[RET:%.*]] = call i8* @strncpy(i8* nonnull [[DST:%.*]], i8* nonnull [[SRC:%.*]], i32 [[N:%.*]])
+; CHECK-NEXT:    ret i8* [[RET]]
+;
+  %ret = call i8* @strncpy(i8* nonnull %dst, i8* nonnull %src, i32 %n)
+  ret i8* %ret
+}
+
+define i8* @test2(i8* %dst) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* nonnull align 1 dereferenceable(5) [[DST:%.*]], i8* nonnull align 1 dereferenceable(6) getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i32 0, i32 0), i32 5, i1 false)
+; CHECK-NEXT:    ret i8* [[DST]]
+;
+  %src = getelementptr [6 x i8], [6 x i8]* @hello, i32 0, i32 0
+  %ret = call i8* @strncpy(i8* nonnull %dst, i8* nonnull %src, i32 5)
+  ret i8* %ret
+}
+
+define i8* @test3(i8* %dst, i32 %n) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i32(i8* noalias nonnull align 1 dereferenceable(5) [[DST:%.*]], i8 0, i32 5, i1 false)
+; CHECK-NEXT:    ret i8* [[DST]]
+;
+  %src = getelementptr [1 x i8], [1 x i8]* @null, i32 0, i32 0
+  %ret = call i8* @strncpy(i8* noalias nonnull %dst, i8* nonnull %src, i32 5);
+  ret i8* %ret
+}
+
 ; Check cases that shouldn't be simplified.
 
 define void @test_no_simplify1() {
 ; CHECK-LABEL: @test_no_simplify1(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8* @strncpy(i8* nonnull dereferenceable(1) getelementptr inbounds ([32 x i8], [32 x i8]* @a, i32 0, i32 0), i8* nonnull dereferenceable(1) getelementptr inbounds ([32 x i8], [32 x i8]* @b, i32 0, i32 0), i32 32)
+; CHECK-NEXT:    ret void
+;
   %dst = getelementptr [32 x i8], [32 x i8]* @a, i32 0, i32 0
   %src = getelementptr [32 x i8], [32 x i8]* @b, i32 0, i32 0
 
   call i8* @strncpy(i8* %dst, i8* %src, i32 32)
-; CHECK: call i8* @strncpy
   ret void
 }
 
 define void @test_no_simplify2() {
 ; CHECK-LABEL: @test_no_simplify2(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8* @strncpy(i8* nonnull dereferenceable(1) getelementptr inbounds ([32 x i8], [32 x i8]* @a, i32 0, i32 0), i8* nonnull dereferenceable(6) getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i32 0, i32 0), i32 8)
+; CHECK-NEXT:    ret void
+;
   %dst = getelementptr [32 x i8], [32 x i8]* @a, i32 0, i32 0
   %src = getelementptr [6 x i8], [6 x i8]* @hello, i32 0, i32 0
 
   call i8* @strncpy(i8* %dst, i8* %src, i32 8)
-; CHECK: call i8* @strncpy
   ret void
 }
diff --git a/llvm/test/Transforms/InstCombine/strncpy-2.ll b/llvm/test/Transforms/InstCombine/strncpy-2.ll
index 5c45f9f24..ed4de382c 100644
--- a/llvm/test/Transforms/InstCombine/strncpy-2.ll
+++ b/llvm/test/Transforms/InstCombine/strncpy-2.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; Test that the strncpy library call simplifier works correctly.
 ;
 ; RUN: opt < %s -instcombine -S | FileCheck %s
@@ -13,10 +14,12 @@ declare i16* @strncpy(i8*, i8*, i32)
 
 define void @test_no_simplify1() {
 ; CHECK-LABEL: @test_no_simplify1(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i16* @strncpy(i8* getelementptr inbounds ([32 x i8], [32 x i8]* @a, i32 0, i32 0), i8* getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i32 0, i32 0), i32 6)
+; CHECK-NEXT:    ret void
+;
   %dst = getelementptr [32 x i8], [32 x i8]* @a, i32 0, i32 0
   %src = getelementptr [6 x i8], [6 x i8]* @hello, i32 0, i32 0
 
   call i16* @strncpy(i8* %dst, i8* %src, i32 6)
-; CHECK: call i16* @strncpy
   ret void
 }
diff --git a/llvm/test/Transforms/InstCombine/strncpy_chk-1.ll b/llvm/test/Transforms/InstCombine/strncpy_chk-1.ll
index bc3ff35ef..ed90303b2 100644
--- a/llvm/test/Transforms/InstCombine/strncpy_chk-1.ll
+++ b/llvm/test/Transforms/InstCombine/strncpy_chk-1.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; Test lib call simplification of __strncpy_chk calls with various values
 ; for len and dstlen.
 ;
@@ -13,33 +14,36 @@ target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f3
 
 define i8* @test_simplify1() {
 ; CHECK-LABEL: @test_simplify1(
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* nonnull align 1 dereferenceable(12) getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* nonnull align 1 dereferenceable(12) getelementptr inbounds ([12 x i8], [12 x i8]* @.str, i32 0, i32 0), i32 12, i1 false)
+; CHECK-NEXT:    ret i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0)
+;
   %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
   %src = getelementptr inbounds [12 x i8], [12 x i8]* @.str, i32 0, i32 0
 
-; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* align 1 getelementptr inbounds ([12 x i8], [12 x i8]* @.str, i32 0, i32 0), i32 12, i1 false)
-; CHECK-NEXT: ret i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0)
   %ret = call i8* @__strncpy_chk(i8* %dst, i8* %src, i32 12, i32 60)
   ret i8* %ret
 }
 
 define i8* @test_simplify2() {
 ; CHECK-LABEL: @test_simplify2(
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* nonnull align 1 dereferenceable(12) getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* nonnull align 1 dereferenceable(12) getelementptr inbounds ([12 x i8], [12 x i8]* @.str, i32 0, i32 0), i32 12, i1 false)
+; CHECK-NEXT:    ret i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0)
+;
   %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
   %src = getelementptr inbounds [12 x i8], [12 x i8]* @.str, i32 0, i32 0
 
-; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* align 1 getelementptr inbounds ([12 x i8], [12 x i8]* @.str, i32 0, i32 0), i32 12, i1 false)
-; CHECK-NEXT: ret i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0)
   %ret = call i8* @__strncpy_chk(i8* %dst, i8* %src, i32 12, i32 12)
   ret i8* %ret
 }
 
 define i8* @test_simplify3() {
 ; CHECK-LABEL: @test_simplify3(
+; CHECK-NEXT:    [[STRNCPY:%.*]] = call i8* @strncpy(i8* nonnull dereferenceable(1) getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* nonnull dereferenceable(1) getelementptr inbounds ([60 x i8], [60 x i8]* @b, i32 0, i32 0), i32 12)
+; CHECK-NEXT:    ret i8* [[STRNCPY]]
+;
   %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
   %src = getelementptr inbounds [60 x i8], [60 x i8]* @b, i32 0, i32 0
 
-; CHECK-NEXT: %strncpy = call i8* @strncpy(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i32 0, i32 0), i32 12)
-; CHECK-NEXT: ret i8* %strncpy
   %ret = call i8* @__strncpy_chk(i8* %dst, i8* %src, i32 12, i32 60)
   ret i8* %ret
 }
@@ -48,22 +52,24 @@ define i8* @test_simplify3() {
 
 define i8* @test_no_simplify1() {
 ; CHECK-LABEL: @test_no_simplify1(
+; CHECK-NEXT:    [[RET:%.*]] = call i8* @__strncpy_chk(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* getelementptr inbounds ([12 x i8], [12 x i8]* @.str, i32 0, i32 0), i32 8, i32 4)
+; CHECK-NEXT:    ret i8* [[RET]]
+;
   %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
   %src = getelementptr inbounds [12 x i8], [12 x i8]* @.str, i32 0, i32 0
 
-; CHECK-NEXT: %ret = call i8* @__strncpy_chk(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* getelementptr inbounds ([12 x i8], [12 x i8]* @.str, i32 0, i32 0), i32 8, i32 4)
-; CHECK-NEXT: ret i8* %ret
   %ret = call i8* @__strncpy_chk(i8* %dst, i8* %src, i32 8, i32 4)
   ret i8* %ret
 }
 
 define i8* @test_no_simplify2() {
 ; CHECK-LABEL: @test_no_simplify2(
+; CHECK-NEXT:    [[RET:%.*]] = call i8* @__strncpy_chk(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i32 0, i32 0), i32 8, i32 0)
+; CHECK-NEXT:    ret i8* [[RET]]
+;
   %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
   %src = getelementptr inbounds [60 x i8], [60 x i8]* @b, i32 0, i32 0
 
-; CHECK-NEXT: %ret = call i8* @__strncpy_chk(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i32 0, i32 0), i32 8, i32 0)
-; CHECK-NEXT: ret i8* %ret
   %ret = call i8* @__strncpy_chk(i8* %dst, i8* %src, i32 8, i32 0)
   ret i8* %ret
 }
diff --git a/llvm/test/Transforms/InstCombine/strndup.ll b/llvm/test/Transforms/InstCombine/strndup.ll
new file mode 100644
index 000000000..b3f927934
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/strndup.ll
@@ -0,0 +1,67 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+@hello = constant [6 x i8] c"hello\00"
+@null = constant [1 x i8] zeroinitializer
+
+declare i8* @strndup(i8*, i32)
+
+define i8* @test1() {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[STRDUP:%.*]] = call dereferenceable_or_null(1) i8* @strdup(i8* getelementptr inbounds ([1 x i8], [1 x i8]* @null, i64 0, i64 0))
+; CHECK-NEXT:    ret i8* [[STRDUP]]
+;
+  %src = getelementptr [1 x i8], [1 x i8]* @null, i32 0, i32 0
+  %ret = call i8* @strndup(i8* %src, i32 0)
+  ret i8* %ret
+}
+
+define i8* @test2() {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[RET:%.*]] = call dereferenceable_or_null(5) i8* @strndup(i8* dereferenceable(6) getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i64 0, i64 0), i32 4)
+; CHECK-NEXT:    ret i8* [[RET]]
+;
+  %src = getelementptr [6 x i8], [6 x i8]* @hello, i32 0, i32 0
+  %ret = call i8* @strndup(i8* %src, i32 4)
+  ret i8* %ret
+}
+
+define i8* @test3() {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    [[STRDUP:%.*]] = call dereferenceable_or_null(6) i8* @strdup(i8* getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i64 0, i64 0))
+; CHECK-NEXT:    ret i8* [[STRDUP]]
+;
+  %src = getelementptr [6 x i8], [6 x i8]* @hello, i32 0, i32 0
+  %ret = call i8* @strndup(i8* %src, i32 5)
+  ret i8* %ret
+}
+
+define i8* @test4() {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    [[STRDUP:%.*]] = call dereferenceable_or_null(6) i8* @strdup(i8* getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i64 0, i64 0))
+; CHECK-NEXT:    ret i8* [[STRDUP]]
+;
+  %src = getelementptr [6 x i8], [6 x i8]* @hello, i32 0, i32 0
+  %ret = call i8* @strndup(i8* %src, i32 6)
+  ret i8* %ret
+}
+
+define i8* @test5() {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    [[STRDUP:%.*]] = call dereferenceable_or_null(6) i8* @strdup(i8* getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i64 0, i64 0))
+; CHECK-NEXT:    ret i8* [[STRDUP]]
+;
+  %src = getelementptr [6 x i8], [6 x i8]* @hello, i32 0, i32 0
+  %ret = call i8* @strndup(i8* %src, i32 7)
+  ret i8* %ret
+}
+
+define i8* @test6(i32 %n) {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:    [[RET:%.*]] = call i8* @strndup(i8* getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i64 0, i64 0), i32 [[N:%.*]])
+; CHECK-NEXT:    ret i8* [[RET]]
+;
+  %src = getelementptr [6 x i8], [6 x i8]* @hello, i32 0, i32 0
+  %ret = call i8* @strndup(i8* %src, i32 %n)
+  ret i8* %ret
+}
diff --git a/llvm/test/Transforms/InstCombine/strpbrk-1.ll b/llvm/test/Transforms/InstCombine/strpbrk-1.ll
index a61100deb..c58140980 100644
--- a/llvm/test/Transforms/InstCombine/strpbrk-1.ll
+++ b/llvm/test/Transforms/InstCombine/strpbrk-1.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; Test that the strpbrk library call simplifier works correctly.
 ;
 ; RUN: opt < %s -instcombine -S | FileCheck %s
@@ -14,55 +15,60 @@ declare i8* @strpbrk(i8*, i8*)
 
 define i8* @test_simplify1(i8* %str) {
 ; CHECK-LABEL: @test_simplify1(
+; CHECK-NEXT:    ret i8* null
+;
   %pat = getelementptr [1 x i8], [1 x i8]* @null, i32 0, i32 0
 
   %ret = call i8* @strpbrk(i8* %str, i8* %pat)
   ret i8* %ret
-; CHECK-NEXT: ret i8* null
 }
 
 ; Check strpbrk("", s) -> NULL.
 
 define i8* @test_simplify2(i8* %pat) {
 ; CHECK-LABEL: @test_simplify2(
+; CHECK-NEXT:    ret i8* null
+;
   %str = getelementptr [1 x i8], [1 x i8]* @null, i32 0, i32 0
 
   %ret = call i8* @strpbrk(i8* %str, i8* %pat)
   ret i8* %ret
-; CHECK-NEXT: ret i8* null
 }
 
 ; Check strpbrk(s1, s2), where s1 and s2 are constants.
 
 define i8* @test_simplify3() {
 ; CHECK-LABEL: @test_simplify3(
+; CHECK-NEXT:    ret i8* getelementptr inbounds ([12 x i8], [12 x i8]* @hello, i32 0, i32 6)
+;
   %str = getelementptr [12 x i8], [12 x i8]* @hello, i32 0, i32 0
   %pat = getelementptr [2 x i8], [2 x i8]* @w, i32 0, i32 0
 
   %ret = call i8* @strpbrk(i8* %str, i8* %pat)
   ret i8* %ret
-; CHECK-NEXT: ret i8* getelementptr inbounds ([12 x i8], [12 x i8]* @hello, i32 0, i32 6)
 }
 
 ; Check strpbrk(s, "a") -> strchr(s, 'a').
 
 define i8* @test_simplify4(i8* %str) {
 ; CHECK-LABEL: @test_simplify4(
+; CHECK-NEXT:    [[STRCHR:%.*]] = call i8* @strchr(i8* nonnull dereferenceable(1) [[STR:%.*]], i32 119)
+; CHECK-NEXT:    ret i8* [[STRCHR]]
+;
   %pat = getelementptr [2 x i8], [2 x i8]* @w, i32 0, i32 0
 
   %ret = call i8* @strpbrk(i8* %str, i8* %pat)
-; CHECK-NEXT: [[VAR:%[a-z]+]] = call i8* @strchr(i8* %str, i32 119)
   ret i8* %ret
-; CHECK-NEXT: ret i8* [[VAR]]
 }
 
 ; Check cases that shouldn't be simplified.
 
 define i8* @test_no_simplify1(i8* %str, i8* %pat) {
 ; CHECK-LABEL: @test_no_simplify1(
+; CHECK-NEXT:    [[RET:%.*]] = call i8* @strpbrk(i8* [[STR:%.*]], i8* [[PAT:%.*]])
+; CHECK-NEXT:    ret i8* [[RET]]
+;
 
   %ret = call i8* @strpbrk(i8* %str, i8* %pat)
-; CHECK-NEXT: %ret = call i8* @strpbrk(i8* %str, i8* %pat)
   ret i8* %ret
-; CHECK-NEXT: ret i8* %ret
 }
diff --git a/llvm/test/Transforms/InstCombine/strrchr-1.ll b/llvm/test/Transforms/InstCombine/strrchr-1.ll
index 3ae68fba9..e60e9bcd3 100644
--- a/llvm/test/Transforms/InstCombine/strrchr-1.ll
+++ b/llvm/test/Transforms/InstCombine/strrchr-1.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; Test that the strrchr library call simplifier works correctly.
 ; RUN: opt < %s -instcombine -S | FileCheck %s
 
@@ -10,9 +11,10 @@ target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f3
 declare i8* @strrchr(i8*, i32)
 
 define void @test_simplify1() {
-; CHECK: store i8* getelementptr inbounds ([14 x i8], [14 x i8]* @hello, i32 0, i32 6)
-; CHECK-NOT: call i8* @strrchr
-; CHECK: ret void
+; CHECK-LABEL: @test_simplify1(
+; CHECK-NEXT:    store i8* getelementptr inbounds ([14 x i8], [14 x i8]* @hello, i32 0, i32 6), i8** @chp, align 4
+; CHECK-NEXT:    ret void
+;
 
   %str = getelementptr [14 x i8], [14 x i8]* @hello, i32 0, i32 0
   %dst = call i8* @strrchr(i8* %str, i32 119)
@@ -21,9 +23,10 @@ define void @test_simplify1() {
 }
 
 define void @test_simplify2() {
-; CHECK: store i8* null, i8** @chp, align 4
-; CHECK-NOT: call i8* @strrchr
-; CHECK: ret void
+; CHECK-LABEL: @test_simplify2(
+; CHECK-NEXT:    store i8* null, i8** @chp, align 4
+; CHECK-NEXT:    ret void
+;
 
   %str = getelementptr [1 x i8], [1 x i8]* @null, i32 0, i32 0
   %dst = call i8* @strrchr(i8* %str, i32 119)
@@ -32,9 +35,10 @@ define void @test_simplify2() {
 }
 
 define void @test_simplify3() {
-; CHECK: store i8* getelementptr inbounds ([14 x i8], [14 x i8]* @hello, i32 0, i32 13)
-; CHECK-NOT: call i8* @strrchr
-; CHECK: ret void
+; CHECK-LABEL: @test_simplify3(
+; CHECK-NEXT:    store i8* getelementptr inbounds ([14 x i8], [14 x i8]* @hello, i32 0, i32 13), i8** @chp, align 4
+; CHECK-NEXT:    ret void
+;
 
   %src = getelementptr [14 x i8], [14 x i8]* @hello, i32 0, i32 0
   %dst = call i8* @strrchr(i8* %src, i32 0)
@@ -43,9 +47,10 @@ define void @test_simplify3() {
 }
 
 define void @test_simplify4() {
-; CHECK: store i8* getelementptr inbounds ([14 x i8], [14 x i8]* @hello, i32 0, i32 13)
-; CHECK-NOT: call i8* @strrchr
-; CHECK: ret void
+; CHECK-LABEL: @test_simplify4(
+; CHECK-NEXT:    store i8* getelementptr inbounds ([14 x i8], [14 x i8]* @hello, i32 0, i32 13), i8** @chp, align 4
+; CHECK-NEXT:    ret void
+;
 
   %src = getelementptr [14 x i8], [14 x i8]* @hello, i32 0, i32 0
   %dst = call i8* @strrchr(i8* %src, i32 65280)
@@ -55,11 +60,33 @@ define void @test_simplify4() {
 
 define void @test_nosimplify1(i32 %chr) {
 ; CHECK-LABEL: @test_nosimplify1(
-; CHECK: call i8* @strrchr
-; CHECK: ret void
+; CHECK-NEXT:    [[DST:%.*]] = call i8* @strrchr(i8* nonnull dereferenceable(1) getelementptr inbounds ([14 x i8], [14 x i8]* @hello, i32 0, i32 0), i32 [[CHR:%.*]])
+; CHECK-NEXT:    store i8* [[DST]], i8** @chp, align 4
+; CHECK-NEXT:    ret void
+;
 
   %src = getelementptr [14 x i8], [14 x i8]* @hello, i32 0, i32 0
   %dst = call i8* @strrchr(i8* %src, i32 %chr)
   store i8* %dst, i8** @chp
   ret void
 }
+
+define i8* @test1(i8* %str, i32 %c) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[RET:%.*]] = call i8* @strrchr(i8* nonnull dereferenceable(1) [[STR:%.*]], i32 [[C:%.*]])
+; CHECK-NEXT:    ret i8* [[RET]]
+;
+
+  %ret = call i8* @strrchr(i8* %str, i32 %c)
+  ret i8* %ret
+}
+
+define i8* @test2(i8* %str, i32 %c) "null-pointer-is-valid"="true" {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[RET:%.*]] = call i8* @strrchr(i8* [[STR:%.*]], i32 [[C:%.*]])
+; CHECK-NEXT:    ret i8* [[RET]]
+;
+
+  %ret = call i8* @strrchr(i8* %str, i32 %c)
+  ret i8* %ret
+}
diff --git a/llvm/test/Transforms/InstCombine/strstr-1.ll b/llvm/test/Transforms/InstCombine/strstr-1.ll
index d57e56ccd..4a9e62045 100644
--- a/llvm/test/Transforms/InstCombine/strstr-1.ll
+++ b/llvm/test/Transforms/InstCombine/strstr-1.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; Test that the strstr library call simplifier works correctly.
 ;
 ; RUN: opt < %s -instcombine -S | FileCheck %s
@@ -15,51 +16,75 @@ declare i8* @strstr(i8*, i8*)
 
 define i8* @test_simplify1(i8* %str) {
 ; CHECK-LABEL: @test_simplify1(
+; CHECK-NEXT:    ret i8* [[STR:%.*]]
+;
   %pat = getelementptr inbounds [1 x i8], [1 x i8]* @.str, i32 0, i32 0
   %ret = call i8* @strstr(i8* %str, i8* %pat)
   ret i8* %ret
-; CHECK-NEXT: ret i8* %str
 }
 
 ; Check strstr(str, "a") -> strchr(str, 'a').
 
 define i8* @test_simplify2(i8* %str) {
 ; CHECK-LABEL: @test_simplify2(
+; CHECK-NEXT:    [[STRCHR:%.*]] = call i8* @strchr(i8* nonnull dereferenceable(1) [[STR:%.*]], i32 97)
+; CHECK-NEXT:    ret i8* [[STRCHR]]
+;
   %pat = getelementptr inbounds [2 x i8], [2 x i8]* @.str1, i32 0, i32 0
   %ret = call i8* @strstr(i8* %str, i8* %pat)
   ret i8* %ret
-; CHECK-NEXT: @strchr(i8* %str, i32 97)
 }
 
 ; Check strstr("abcde", "bcd") -> "abcde" + 1.
 
 define i8* @test_simplify3() {
 ; CHECK-LABEL: @test_simplify3(
+; CHECK-NEXT:    ret i8* getelementptr inbounds ([6 x i8], [6 x i8]* @.str2, i64 0, i64 1)
+;
   %str = getelementptr inbounds [6 x i8], [6 x i8]* @.str2, i32 0, i32 0
   %pat = getelementptr inbounds [4 x i8], [4 x i8]* @.str3, i32 0, i32 0
   %ret = call i8* @strstr(i8* %str, i8* %pat)
   ret i8* %ret
-; CHECK-NEXT: getelementptr inbounds ([6 x i8], [6 x i8]* @.str2, i64 0, i64 1)
 }
 
 ; Check strstr(str, str) -> str.
 
 define i8* @test_simplify4(i8* %str) {
 ; CHECK-LABEL: @test_simplify4(
+; CHECK-NEXT:    ret i8* [[STR:%.*]]
+;
   %ret = call i8* @strstr(i8* %str, i8* %str)
   ret i8* %ret
-; CHECK-NEXT: ret i8* %str
 }
 
 ; Check strstr(str, pat) == str -> strncmp(str, pat, strlen(str)) == 0.
 
 define i1 @test_simplify5(i8* %str, i8* %pat) {
 ; CHECK-LABEL: @test_simplify5(
+; CHECK-NEXT:    [[STRLEN:%.*]] = call i64 @strlen(i8* nonnull dereferenceable(1) [[PAT:%.*]])
+; CHECK-NEXT:    [[STRNCMP:%.*]] = call i32 @strncmp(i8* [[STR:%.*]], i8* [[PAT]], i64 [[STRLEN]])
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i32 [[STRNCMP]], 0
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
   %ret = call i8* @strstr(i8* %str, i8* %pat)
   %cmp = icmp eq i8* %ret, %str
   ret i1 %cmp
-; CHECK: [[LEN:%[a-z]+]] = call {{i[0-9]+}} @strlen(i8* %pat)
-; CHECK: [[NCMP:%[a-z]+]] = call {{i[0-9]+}} @strncmp(i8* %str, i8* %pat, {{i[0-9]+}} [[LEN]])
-; CHECK: icmp eq {{i[0-9]+}} [[NCMP]], 0
-; CHECK: ret i1
+}
+
+define i8* @test1(i8* %str1, i8* %str2) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[RET:%.*]] = call i8* @strstr(i8* nonnull dereferenceable(1) [[STR1:%.*]], i8* nonnull dereferenceable(1) [[STR2:%.*]])
+; CHECK-NEXT:    ret i8* [[RET]]
+;
+  %ret = call i8* @strstr(i8* %str1, i8* %str2)
+  ret i8* %ret
+}
+
+define i8* @test2(i8* %str1, i8* %str2) "null-pointer-is-valid"="true" {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[RET:%.*]] = call i8* @strstr(i8* [[STR1:%.*]], i8* [[STR2:%.*]])
+; CHECK-NEXT:    ret i8* [[RET]]
+;
+  %ret = call i8* @strstr(i8* %str1, i8* %str2)
+  ret i8* %ret
 }
diff --git a/llvm/test/Transforms/InstCombine/sub-and-or-neg-xor.ll b/llvm/test/Transforms/InstCombine/sub-and-or-neg-xor.ll
new file mode 100644
index 000000000..980433e2d
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/sub-and-or-neg-xor.ll
@@ -0,0 +1,115 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+declare void @use(i32)
+
+define i32 @sub_to_xor(i32 %x, i32 %y) {
+; CHECK-LABEL: @sub_to_xor(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 0, [[TMP1]]
+; CHECK-NEXT:    ret i32 [[SUB]]
+;
+  %or = or i32 %x, %y
+  %and = and i32 %x, %y
+  %sub = sub i32 %and, %or
+  ret i32 %sub
+}
+
+define i32 @sub_to_xor_extra_use_sub(i32 %x, i32 %y) {
+; CHECK-LABEL: @sub_to_xor_extra_use_sub(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 0, [[TMP1]]
+; CHECK-NEXT:    call void @use(i32 [[SUB]])
+; CHECK-NEXT:    ret i32 [[SUB]]
+;
+  %or = or i32 %x, %y
+  %and = and i32 %x, %y
+  %sub = sub i32 %and, %or
+  call void @use(i32 %sub)
+  ret i32 %sub
+}
+
+define i32 @sub_to_xor_extra_use_and(i32 %x, i32 %y) {
+; CHECK-LABEL: @sub_to_xor_extra_use_and(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use(i32 [[AND]])
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 0, [[TMP1]]
+; CHECK-NEXT:    ret i32 [[SUB]]
+;
+  %or = or i32 %x, %y
+  %and = and i32 %x, %y
+  call void @use(i32 %and)
+  %sub = sub i32 %and, %or
+  ret i32 %sub
+}
+
+define i32 @sub_to_xor_extra_use_or(i32 %x, i32 %y) {
+; CHECK-LABEL: @sub_to_xor_extra_use_or(
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use(i32 [[OR]])
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 0, [[TMP1]]
+; CHECK-NEXT:    ret i32 [[SUB]]
+;
+  %or = or i32 %x, %y
+  call void @use(i32 %or)
+  %and = and i32 %x, %y
+  %sub = sub i32 %and, %or
+  ret i32 %sub
+}
+
+define i32 @sub_to_xor_or_commuted(i32 %x, i32 %y) {
+; CHECK-LABEL: @sub_to_xor_or_commuted(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 0, [[TMP1]]
+; CHECK-NEXT:    ret i32 [[SUB]]
+;
+  %or = or i32 %y, %x
+  %and = and i32 %x, %y
+  %sub = sub i32 %and, %or
+  ret i32 %sub
+}
+
+define i32 @sub_to_xor_and_commuted(i32 %x, i32 %y) {
+; CHECK-LABEL: @sub_to_xor_and_commuted(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 0, [[TMP1]]
+; CHECK-NEXT:    ret i32 [[SUB]]
+;
+  %or = or i32 %x, %y
+  %and = and i32 %y, %x
+  %sub = sub i32 %and, %or
+  ret i32 %sub
+}
+
+define <2 x i32> @sub_to_xor_vec(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @sub_to_xor_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor <2 x i32> [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[SUB:%.*]] = sub <2 x i32> zeroinitializer, [[TMP1]]
+; CHECK-NEXT:    ret <2 x i32> [[SUB]]
+;
+  %or = or <2 x i32> %x, %y
+  %and = and <2 x i32> %y, %x
+  %sub = sub <2 x i32> %and, %or
+  ret <2 x i32> %sub
+}
+
+; Negative tests
+
+define i32 @sub_to_xor_extra_use_and_or(i32 %x, i32 %y) {
+; CHECK-LABEL: @sub_to_xor_extra_use_and_or(
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use(i32 [[OR]])
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X]], [[Y]]
+; CHECK-NEXT:    call void @use(i32 [[AND]])
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 [[AND]], [[OR]]
+; CHECK-NEXT:    ret i32 [[SUB]]
+;
+  %or = or i32 %x, %y
+  call void @use(i32 %or)
+  %and = and i32 %x, %y
+  call void @use(i32 %and)
+  %sub = sub i32 %and, %or
+  ret i32 %sub
+}
diff --git a/llvm/test/Transforms/InstCombine/sub-ashr-and-to-icmp-select.ll b/llvm/test/Transforms/InstCombine/sub-ashr-and-to-icmp-select.ll
new file mode 100644
index 000000000..66609f141
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/sub-ashr-and-to-icmp-select.ll
@@ -0,0 +1,213 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -instcombine %s -S -o - | FileCheck %s
+
+; Clamp negative to zero:
+; E.g., clamp0 implemented in a shifty way, could be optimized as v > 0 ? v : 0, where sub hasNoSignedWrap.
+; int32 clamp0(int32 v) {
+;   return ((-(v) >> 31) & (v));
+; }
+;
+
+; Scalar Types
+
+define i8 @sub_ashr_and_i8(i8 %x, i8 %y) {
+; CHECK-LABEL: @sub_ashr_and_i8(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[AND:%.*]] = select i1 [[TMP1]], i8 [[X]], i8 0
+; CHECK-NEXT:    ret i8 [[AND]]
+;
+  %sub = sub nsw i8 %y, %x
+  %shr = ashr i8 %sub, 7
+  %and = and i8 %shr, %x
+  ret i8 %and
+}
+
+define i16 @sub_ashr_and_i16(i16 %x, i16 %y) {
+; CHECK-LABEL: @sub_ashr_and_i16(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i16 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[AND:%.*]] = select i1 [[TMP1]], i16 [[X]], i16 0
+; CHECK-NEXT:    ret i16 [[AND]]
+;
+
+  %sub = sub nsw i16 %y, %x
+  %shr = ashr i16 %sub, 15
+  %and = and i16 %shr, %x
+  ret i16 %and
+}
+
+define i32 @sub_ashr_and_i32(i32 %x, i32 %y) {
+; CHECK-LABEL: @sub_ashr_and_i32(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[AND:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 0
+; CHECK-NEXT:    ret i32 [[AND]]
+;
+  %sub = sub nsw i32 %y, %x
+  %shr = ashr i32 %sub, 31
+  %and = and i32 %shr, %x
+  ret i32 %and
+}
+
+define i64 @sub_ashr_and_i64(i64 %x, i64 %y) {
+; CHECK-LABEL: @sub_ashr_and_i64(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i64 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[AND:%.*]] = select i1 [[TMP1]], i64 [[X]], i64 0
+; CHECK-NEXT:    ret i64 [[AND]]
+;
+  %sub = sub nsw i64 %y, %x
+  %shr = ashr i64 %sub, 63
+  %and = and i64 %shr, %x
+  ret i64 %and
+}
+
+; nuw nsw
+
+define i32 @sub_ashr_and_i32_nuw_nsw(i32 %x, i32 %y) {
+; CHECK-LABEL: @sub_ashr_and_i32_nuw_nsw(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[AND:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 0
+; CHECK-NEXT:    ret i32 [[AND]]
+;
+  %sub = sub nuw nsw i32 %y, %x
+  %shr = ashr i32 %sub, 31
+  %and = and i32 %shr, %x
+  ret i32 %and
+}
+
+; Commute
+
+define i32 @sub_ashr_and_i32_commute(i32 %x, i32 %y) {
+; CHECK-LABEL: @sub_ashr_and_i32_commute(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[AND:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 0
+; CHECK-NEXT:    ret i32 [[AND]]
+;
+  %sub = sub nsw i32 %y, %x
+  %shr = ashr i32 %sub, 31
+  %and = and i32 %x, %shr  ; commute %x and %shr
+  ret i32 %and
+}
+
+; Vector Types
+
+define <4 x i32> @sub_ashr_and_i32_vec(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: @sub_ashr_and_i32_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <4 x i32> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[AND:%.*]] = select <4 x i1> [[TMP1]], <4 x i32> [[X]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    ret <4 x i32> [[AND]]
+;
+  %sub = sub nsw <4 x i32> %y, %x
+  %shr = ashr <4 x i32> %sub, <i32 31, i32 31, i32 31, i32 31>
+  %and = and <4 x i32> %shr, %x
+  ret <4 x i32> %and
+}
+
+define <4 x i32> @sub_ashr_and_i32_vec_nuw_nsw(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: @sub_ashr_and_i32_vec_nuw_nsw(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <4 x i32> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[AND:%.*]] = select <4 x i1> [[TMP1]], <4 x i32> [[X]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    ret <4 x i32> [[AND]]
+;
+  %sub = sub nuw nsw <4 x i32> %y, %x
+  %shr = ashr <4 x i32> %sub, <i32 31, i32 31, i32 31, i32 31>
+  %and = and <4 x i32> %shr, %x
+  ret <4 x i32> %and
+}
+
+define <4 x i32> @sub_ashr_and_i32_vec_commute(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: @sub_ashr_and_i32_vec_commute(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <4 x i32> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[AND:%.*]] = select <4 x i1> [[TMP1]], <4 x i32> [[X]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    ret <4 x i32> [[AND]]
+;
+  %sub = sub nsw <4 x i32> %y, %x
+  %shr = ashr <4 x i32> %sub, <i32 31, i32 31, i32 31, i32 31>
+  %and = and <4 x i32> %x, %shr  ; commute %x and %shr
+  ret <4 x i32> %and
+}
+
+; Extra uses
+
+define i32 @sub_ashr_and_i32_extra_use_sub(i32 %x, i32 %y, i32* %p) {
+; CHECK-LABEL: @sub_ashr_and_i32_extra_use_sub(
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    store i32 [[SUB]], i32* [[P:%.*]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[Y]], [[X]]
+; CHECK-NEXT:    [[AND:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 0
+; CHECK-NEXT:    ret i32 [[AND]]
+;
+  %sub = sub nsw i32 %y, %x
+  store i32 %sub, i32* %p
+  %shr = ashr i32 %sub, 31
+  %and = and i32 %shr, %x
+  ret i32 %and
+}
+
+define i32 @sub_ashr_and_i32_extra_use_and(i32 %x, i32 %y, i32* %p) {
+; CHECK-LABEL: @sub_ashr_and_i32_extra_use_and(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[AND:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 0
+; CHECK-NEXT:    store i32 [[AND]], i32* [[P:%.*]], align 4
+; CHECK-NEXT:    ret i32 [[AND]]
+;
+  %sub = sub nsw i32 %y, %x
+  %shr = ashr i32 %sub, 31
+  %and = and i32 %shr, %x
+  store i32 %and, i32* %p
+  ret i32 %and
+}
+
+; Negative Tests
+
+define i32 @sub_ashr_and_i32_extra_use_ashr(i32 %x, i32 %y, i32* %p) {
+; CHECK-LABEL: @sub_ashr_and_i32_extra_use_ashr(
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[SHR:%.*]] = ashr i32 [[SUB]], 31
+; CHECK-NEXT:    store i32 [[SHR]], i32* [[P:%.*]], align 4
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[SHR]], [[X]]
+; CHECK-NEXT:    ret i32 [[AND]]
+;
+  %sub = sub nsw i32 %y, %x
+  %shr = ashr i32 %sub, 31
+  store i32 %shr, i32* %p
+  %and = and i32 %shr, %x
+  ret i32 %and
+}
+
+define i32 @sub_ashr_and_i32_no_nuw_nsw(i32 %x, i32 %y) {
+; CHECK-LABEL: @sub_ashr_and_i32_no_nuw_nsw(
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[SHR:%.*]] = ashr i32 [[SUB]], 7
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[SHR]], [[X]]
+; CHECK-NEXT:    ret i32 [[AND]]
+;
+  %sub = sub i32 %y, %x
+  %shr = ashr i32 %sub, 7
+  %and = and i32 %shr, %x
+  ret i32 %and
+}
+
+define <4 x i32> @sub_ashr_and_i32_vec_undef(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: @sub_ashr_and_i32_vec_undef(
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw <4 x i32> [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[SHR:%.*]] = ashr <4 x i32> [[SUB]], <i32 31, i32 31, i32 31, i32 undef>
+; CHECK-NEXT:    [[AND:%.*]] = and <4 x i32> [[SHR]], [[X]]
+; CHECK-NEXT:    ret <4 x i32> [[AND]]
+;
+  %sub = sub nsw <4 x i32> %y, %x
+  %shr = ashr <4 x i32> %sub, <i32 31, i32 31, i32 31, i32 undef>
+  %and = and <4 x i32> %shr, %x
+  ret <4 x i32> %and
+}
+
+define i32 @sub_ashr_and_i32_shift_wrong_bit(i32 %x, i32 %y) {
+; CHECK-LABEL: @sub_ashr_and_i32_shift_wrong_bit(
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[SHR:%.*]] = ashr i32 [[SUB]], 15
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[SHR]], [[X]]
+; CHECK-NEXT:    ret i32 [[AND]]
+;
+  %sub = sub nsw i32 %y, %x
+  %shr = ashr i32 %sub, 15
+  %and = and i32 %shr, %x
+  ret i32 %and
+}
diff --git a/llvm/test/Transforms/InstCombine/sub-ashr-or-to-icmp-select.ll b/llvm/test/Transforms/InstCombine/sub-ashr-or-to-icmp-select.ll
new file mode 100644
index 000000000..937bff62d
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/sub-ashr-or-to-icmp-select.ll
@@ -0,0 +1,239 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -instcombine %s -S -o - | FileCheck %s
+
+; Clamp positive to allOnes:
+; E.g., clamp255 implemented in a shifty way, could be optimized as v > 255 ? 255 : v, where sub hasNoSignedWrap.
+; int32 clamp255(int32 v) {
+;   return (((255 - (v)) >> 31) | (v)) & 255;
+; }
+;
+
+; Scalar Types
+
+define i32 @clamp255_i32(i32 %x) {
+; CHECK-LABEL: @clamp255_i32(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[X:%.*]], 255
+; CHECK-NEXT:    [[OR:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 255
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[OR]], 255
+; CHECK-NEXT:    ret i32 [[AND]]
+;
+  %sub = sub nsw i32 255, %x
+  %shr = ashr i32 %sub, 31
+  %or = or i32 %shr, %x
+  %and = and i32 %or, 255
+  ret i32 %and
+}
+
+define i8 @sub_ashr_or_i8(i8 %x, i8 %y) {
+; CHECK-LABEL: @sub_ashr_or_i8(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[OR:%.*]] = select i1 [[TMP1]], i8 -1, i8 [[X]]
+; CHECK-NEXT:    ret i8 [[OR]]
+;
+  %sub = sub nsw i8 %y, %x
+  %shr = ashr i8 %sub, 7
+  %or = or i8 %shr, %x
+  ret i8 %or
+}
+
+define i16 @sub_ashr_or_i16(i16 %x, i16 %y) {
+; CHECK-LABEL: @sub_ashr_or_i16(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i16 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[OR:%.*]] = select i1 [[TMP1]], i16 -1, i16 [[X]]
+; CHECK-NEXT:    ret i16 [[OR]]
+;
+  %sub = sub nsw i16 %y, %x
+  %shr = ashr i16 %sub, 15
+  %or = or i16 %shr, %x
+  ret i16 %or
+}
+
+define i32 @sub_ashr_or_i32(i32 %x, i32 %y) {
+; CHECK-LABEL: @sub_ashr_or_i32(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[OR:%.*]] = select i1 [[TMP1]], i32 -1, i32 [[X]]
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %sub = sub nsw i32 %y, %x
+  %shr = ashr i32 %sub, 31
+  %or = or i32 %shr, %x
+  ret i32 %or
+}
+
+define i64 @sub_ashr_or_i64(i64 %x, i64 %y) {
+; CHECK-LABEL: @sub_ashr_or_i64(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i64 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[OR:%.*]] = select i1 [[TMP1]], i64 -1, i64 [[X]]
+; CHECK-NEXT:    ret i64 [[OR]]
+;
+  %sub = sub nsw i64 %y, %x
+  %shr = ashr i64 %sub, 63
+  %or = or i64 %shr, %x
+  ret i64 %or
+}
+
+; nuw nsw
+
+define i32 @sub_ashr_or_i32_nuw_nsw(i32 %x, i32 %y) {
+; CHECK-LABEL: @sub_ashr_or_i32_nuw_nsw(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[OR:%.*]] = select i1 [[TMP1]], i32 -1, i32 [[X]]
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %sub = sub nuw nsw i32 %y, %x
+  %shr = ashr i32 %sub, 31
+  %or = or i32 %shr, %x
+  ret i32 %or
+}
+
+; Commute
+
+define i32 @sub_ashr_or_i32_commute(i32 %x, i32 %y) {
+; CHECK-LABEL: @sub_ashr_or_i32_commute(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[OR:%.*]] = select i1 [[TMP1]], i32 -1, i32 [[X]]
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %sub = sub nsw i32 %y, %x
+  %shr = ashr i32 %sub, 31
+  %or = or i32 %x, %shr  ; commute %shr and %x
+  ret i32 %or
+}
+
+; Vector Types
+
+define <4 x i32> @sub_ashr_or_i32_vec(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: @sub_ashr_or_i32_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <4 x i32> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[OR:%.*]] = select <4 x i1> [[TMP1]], <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>, <4 x i32> [[X]]
+; CHECK-NEXT:    ret <4 x i32> [[OR]]
+;
+  %sub = sub nsw <4 x i32> %y, %x
+  %shr = ashr <4 x i32> %sub, <i32 31, i32 31, i32 31, i32 31>
+  %or = or <4 x i32> %shr, %x
+  ret <4 x i32> %or
+}
+
+define <4 x i32> @sub_ashr_or_i32_vec_nuw_nsw(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: @sub_ashr_or_i32_vec_nuw_nsw(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <4 x i32> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[OR:%.*]] = select <4 x i1> [[TMP1]], <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>, <4 x i32> [[X]]
+; CHECK-NEXT:    ret <4 x i32> [[OR]]
+;
+  %sub = sub nuw nsw <4 x i32> %y, %x
+  %shr = ashr <4 x i32> %sub, <i32 31, i32 31, i32 31, i32 31>
+  %or = or <4 x i32> %shr, %x
+  ret <4 x i32> %or
+}
+
+define <4 x i32> @sub_ashr_or_i32_vec_commute(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: @sub_ashr_or_i32_vec_commute(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <4 x i32> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[OR:%.*]] = select <4 x i1> [[TMP1]], <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>, <4 x i32> [[X]]
+; CHECK-NEXT:    ret <4 x i32> [[OR]]
+;
+  %sub = sub nsw <4 x i32> %y, %x
+  %shr = ashr <4 x i32> %sub, <i32 31, i32 31, i32 31, i32 31>
+  %or = or <4 x i32> %x, %shr
+  ret <4 x i32> %or
+}
+
+; Extra uses
+
+define i32 @sub_ashr_or_i32_extra_use_sub(i32 %x, i32 %y, i32* %p) {
+; CHECK-LABEL: @sub_ashr_or_i32_extra_use_sub(
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    store i32 [[SUB]], i32* [[P:%.*]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[Y]], [[X]]
+; CHECK-NEXT:    [[OR:%.*]] = select i1 [[TMP1]], i32 -1, i32 [[X]]
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %sub = sub nsw i32 %y, %x
+  store i32 %sub, i32* %p
+  %shr = ashr i32 %sub, 31
+  %or = or i32 %shr, %x
+  ret i32 %or
+}
+
+define i32 @sub_ashr_or_i32_extra_use_or(i32 %x, i32 %y, i32* %p) {
+; CHECK-LABEL: @sub_ashr_or_i32_extra_use_or(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[OR:%.*]] = select i1 [[TMP1]], i32 -1, i32 [[X]]
+; CHECK-NEXT:    store i32 [[OR]], i32* [[P:%.*]], align 4
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %sub = sub nsw i32 %y, %x
+  %shr = ashr i32 %sub, 31
+  %or = or i32 %shr, %x
+  store i32 %or, i32* %p
+  ret i32 %or
+}
+
+; Negative Tests
+
+define i32 @sub_ashr_or_i32_extra_use_ashr(i32 %x, i32 %y, i32* %p) {
+; CHECK-LABEL: @sub_ashr_or_i32_extra_use_ashr(
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[SHR:%.*]] = ashr i32 [[SUB]], 31
+; CHECK-NEXT:    store i32 [[SHR]], i32* [[P:%.*]], align 4
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[SHR]], [[X]]
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %sub = sub nsw i32 %y, %x
+  %shr = ashr i32 %sub, 31
+  store i32 %shr, i32* %p
+  %or = or i32 %shr, %x
+  ret i32 %or
+}
+
+define i32 @sub_ashr_or_i32_no_nsw_nuw(i32 %x, i32 %y) {
+; CHECK-LABEL: @sub_ashr_or_i32_no_nsw_nuw(
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[SHR:%.*]] = ashr i32 [[SUB]], 31
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[SHR]], [[X]]
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %sub = sub i32 %y, %x
+  %shr = ashr i32 %sub, 31
+  %or = or i32 %shr, %x
+  ret i32 %or
+}
+
+define <4 x i32> @sub_ashr_or_i32_vec_undef1(<4 x i32> %x) {
+; CHECK-LABEL: @sub_ashr_or_i32_vec_undef1(
+; CHECK-NEXT:    [[SUB:%.*]] = sub <4 x i32> <i32 255, i32 255, i32 undef, i32 255>, [[X:%.*]]
+; CHECK-NEXT:    [[SHR:%.*]] = ashr <4 x i32> [[SUB]], <i32 31, i32 31, i32 31, i32 31>
+; CHECK-NEXT:    [[OR:%.*]] = or <4 x i32> [[SHR]], [[X]]
+; CHECK-NEXT:    ret <4 x i32> [[OR]]
+;
+  %sub = sub <4 x i32> <i32 255, i32 255, i32 undef, i32 255>, %x
+  %shr = ashr <4 x i32> %sub, <i32 31, i32 31, i32 31, i32 31>
+  %or = or <4 x i32> %shr, %x
+  ret <4 x i32> %or
+}
+
+define <4 x i32> @sub_ashr_or_i32_vec_undef2(<4 x i32> %x) {
+; CHECK-LABEL: @sub_ashr_or_i32_vec_undef2(
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw <4 x i32> <i32 255, i32 255, i32 255, i32 255>, [[X:%.*]]
+; CHECK-NEXT:    [[SHR:%.*]] = ashr <4 x i32> [[SUB]], <i32 undef, i32 31, i32 31, i32 31>
+; CHECK-NEXT:    [[OR:%.*]] = or <4 x i32> [[SHR]], [[X]]
+; CHECK-NEXT:    ret <4 x i32> [[OR]]
+;
+  %sub = sub nsw <4 x i32> <i32 255, i32 255, i32 255, i32 255>, %x
+  %shr = ashr <4 x i32> %sub, <i32 undef, i32 31, i32 31, i32 31>
+  %or = or <4 x i32> %shr, %x
+  ret <4 x i32> %or
+}
+
+define i32 @sub_ashr_or_i32_shift_wrong_bit(i32 %x, i32 %y) {
+; CHECK-LABEL: @sub_ashr_or_i32_shift_wrong_bit(
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[SHR:%.*]] = ashr i32 [[SUB]], 11
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[SHR]], [[X]]
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %sub = sub nsw i32 %y, %x
+  %shr = ashr i32 %sub, 11
+  %or = or i32 %shr, %x
+  ret i32 %or
+}
diff --git a/llvm/test/Transforms/InstCombine/sub-gep.ll b/llvm/test/Transforms/InstCombine/sub-gep.ll
new file mode 100644
index 000000000..5798614c7
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/sub-gep.ll
@@ -0,0 +1,101 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+define i64 @test_inbounds([0 x i32]* %base, i64 %idx) {
+; CHECK-LABEL: @test_inbounds(
+; CHECK-NEXT:    [[P2_IDX:%.*]] = shl nsw i64 [[IDX:%.*]], 2
+; CHECK-NEXT:    ret i64 [[P2_IDX]]
+;
+  %p1 = getelementptr inbounds [0 x i32], [0 x i32]* %base, i64 0, i64 0
+  %p2 = getelementptr inbounds [0 x i32], [0 x i32]* %base, i64 0, i64 %idx
+  %i1 = ptrtoint i32* %p1 to i64
+  %i2 = ptrtoint i32* %p2 to i64
+  %d = sub i64 %i2, %i1
+  ret i64 %d
+}
+
+define i64 @test_inbounds_nuw([0 x i32]* %base, i64 %idx) {
+; CHECK-LABEL: @test_inbounds_nuw(
+; CHECK-NEXT:    [[P2_IDX:%.*]] = shl nuw nsw i64 [[IDX:%.*]], 2
+; CHECK-NEXT:    ret i64 [[P2_IDX]]
+;
+  %p1 = getelementptr inbounds [0 x i32], [0 x i32]* %base, i64 0, i64 0
+  %p2 = getelementptr inbounds [0 x i32], [0 x i32]* %base, i64 0, i64 %idx
+  %i1 = ptrtoint i32* %p1 to i64
+  %i2 = ptrtoint i32* %p2 to i64
+  %d = sub nuw i64 %i2, %i1
+  ret i64 %d
+}
+
+define i64 @test_nuw([0 x i32]* %base, i64 %idx) {
+; CHECK-LABEL: @test_nuw(
+; CHECK-NEXT:    [[P2_IDX:%.*]] = shl i64 [[IDX:%.*]], 2
+; CHECK-NEXT:    ret i64 [[P2_IDX]]
+;
+  %p1 = getelementptr [0 x i32], [0 x i32]* %base, i64 0, i64 0
+  %p2 = getelementptr [0 x i32], [0 x i32]* %base, i64 0, i64 %idx
+  %i1 = ptrtoint i32* %p1 to i64
+  %i2 = ptrtoint i32* %p2 to i64
+  %d = sub nuw i64 %i2, %i1
+  ret i64 %d
+}
+
+define i32 @test_inbounds_nuw_trunc([0 x i32]* %base, i64 %idx) {
+; CHECK-LABEL: @test_inbounds_nuw_trunc(
+; CHECK-NEXT:    [[IDX_TR:%.*]] = trunc i64 [[IDX:%.*]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 [[IDX_TR]], 2
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %p1 = getelementptr inbounds [0 x i32], [0 x i32]* %base, i64 0, i64 0
+  %p2 = getelementptr inbounds [0 x i32], [0 x i32]* %base, i64 0, i64 %idx
+  %i1 = ptrtoint i32* %p1 to i64
+  %i2 = ptrtoint i32* %p2 to i64
+  %t1 = trunc i64 %i1 to i32
+  %t2 = trunc i64 %i2 to i32
+  %d = sub nuw i32 %t2, %t1
+  ret i32 %d
+}
+
+define i64 @test_inbounds_nuw_swapped([0 x i32]* %base, i64 %idx) {
+; CHECK-LABEL: @test_inbounds_nuw_swapped(
+; CHECK-NEXT:    [[P2_IDX:%.*]] = shl nsw i64 [[IDX:%.*]], 2
+; CHECK-NEXT:    [[DIFF_NEG:%.*]] = sub i64 0, [[P2_IDX]]
+; CHECK-NEXT:    ret i64 [[DIFF_NEG]]
+;
+  %p1 = getelementptr inbounds [0 x i32], [0 x i32]* %base, i64 0, i64 0
+  %p2 = getelementptr inbounds [0 x i32], [0 x i32]* %base, i64 0, i64 %idx
+  %i1 = ptrtoint i32* %p2 to i64
+  %i2 = ptrtoint i32* %p1 to i64
+  %d = sub nuw i64 %i2, %i1
+  ret i64 %d
+}
+
+; The sub and shl here could be nuw, but this is harder to handle.
+define i64 @test_inbounds_nuw_two_gep([0 x i32]* %base, i64 %idx, i64 %idx2) {
+; CHECK-LABEL: @test_inbounds_nuw_two_gep(
+; CHECK-NEXT:    [[P2_IDX1:%.*]] = sub i64 [[IDX2:%.*]], [[IDX:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[P2_IDX1]], 2
+; CHECK-NEXT:    ret i64 [[TMP1]]
+;
+  %p1 = getelementptr inbounds [0 x i32], [0 x i32]* %base, i64 0, i64 %idx
+  %p2 = getelementptr inbounds [0 x i32], [0 x i32]* %base, i64 0, i64 %idx2
+  %i1 = ptrtoint i32* %p1 to i64
+  %i2 = ptrtoint i32* %p2 to i64
+  %d = sub nuw i64 %i2, %i1
+  ret i64 %d
+}
+
+define i64 @test_inbounds_nuw_multi_index([0 x [2 x i32]]* %base, i64 %idx, i64 %idx2) {
+; CHECK-LABEL: @test_inbounds_nuw_multi_index(
+; CHECK-NEXT:    [[P2_IDX:%.*]] = shl nsw i64 [[IDX:%.*]], 3
+; CHECK-NEXT:    [[P2_IDX1:%.*]] = shl nsw i64 [[IDX2:%.*]], 2
+; CHECK-NEXT:    [[P2_OFFS2:%.*]] = add i64 [[P2_IDX]], [[P2_IDX1]]
+; CHECK-NEXT:    ret i64 [[P2_OFFS2]]
+;
+  %p1 = getelementptr inbounds [0 x [2 x i32]], [0 x [2 x i32]]* %base, i64 0, i64 0, i64 0
+  %p2 = getelementptr inbounds [0 x [2 x i32]], [0 x [2 x i32]]* %base, i64 0, i64 %idx, i64 %idx2
+  %i1 = ptrtoint i32* %p1 to i64
+  %i2 = ptrtoint i32* %p2 to i64
+  %d = sub nuw i64 %i2, %i1
+  ret i64 %d
+}
diff --git a/llvm/test/Transforms/InstCombine/sub-minmax.ll b/llvm/test/Transforms/InstCombine/sub-minmax.ll
index ccc3483ce..58edf3719 100644
--- a/llvm/test/Transforms/InstCombine/sub-minmax.ll
+++ b/llvm/test/Transforms/InstCombine/sub-minmax.ll
@@ -4,10 +4,9 @@
 define i32 @max_na_b_minux_na(i32 %A, i32 %B) {
 ; CHECK-LABEL: @max_na_b_minux_na(
 ; CHECK-NEXT:    [[NOT:%.*]] = xor i32 [[A:%.*]], -1
-; CHECK-NEXT:    [[L0:%.*]] = icmp ult i32 [[NOT]], [[B:%.*]]
-; CHECK-NEXT:    [[L1:%.*]] = select i1 [[L0]], i32 [[NOT]], i32 [[B]]
-; CHECK-NEXT:    [[X:%.*]] = sub i32 [[L1]], [[NOT]]
-; CHECK-NEXT:    ret i32 [[X]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.usub.sat.i32(i32 [[NOT]], i32 [[B:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = sub i32 0, [[TMP1]]
+; CHECK-NEXT:    ret i32 [[TMP2]]
 ;
   %not = xor i32 %A, -1
   %l0 = icmp ult i32 %not, %B
@@ -19,10 +18,8 @@ define i32 @max_na_b_minux_na(i32 %A, i32 %B) {
 define i32 @na_minus_max_na_b(i32 %A, i32 %B) {
 ; CHECK-LABEL: @na_minus_max_na_b(
 ; CHECK-NEXT:    [[NOT:%.*]] = xor i32 [[A:%.*]], -1
-; CHECK-NEXT:    [[L0:%.*]] = icmp ult i32 [[NOT]], [[B:%.*]]
-; CHECK-NEXT:    [[L1:%.*]] = select i1 [[L0]], i32 [[NOT]], i32 [[B]]
-; CHECK-NEXT:    [[X:%.*]] = sub i32 [[NOT]], [[L1]]
-; CHECK-NEXT:    ret i32 [[X]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.usub.sat.i32(i32 [[NOT]], i32 [[B:%.*]])
+; CHECK-NEXT:    ret i32 [[TMP1]]
 ;
   %not = xor i32 %A, -1
   %l0 = icmp ult i32 %not, %B
@@ -34,10 +31,9 @@ define i32 @na_minus_max_na_b(i32 %A, i32 %B) {
 define i32 @max_b_na_minus_na(i32 %A, i32 %B) {
 ; CHECK-LABEL: @max_b_na_minus_na(
 ; CHECK-NEXT:    [[NOT:%.*]] = xor i32 [[A:%.*]], -1
-; CHECK-NEXT:    [[L0:%.*]] = icmp ugt i32 [[NOT]], [[B:%.*]]
-; CHECK-NEXT:    [[L1:%.*]] = select i1 [[L0]], i32 [[B]], i32 [[NOT]]
-; CHECK-NEXT:    [[X:%.*]] = sub i32 [[L1]], [[NOT]]
-; CHECK-NEXT:    ret i32 [[X]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.usub.sat.i32(i32 [[NOT]], i32 [[B:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = sub i32 0, [[TMP1]]
+; CHECK-NEXT:    ret i32 [[TMP2]]
 ;
   %not = xor i32 %A, -1
   %l0 = icmp ugt i32 %not, %B
@@ -49,10 +45,8 @@ define i32 @max_b_na_minus_na(i32 %A, i32 %B) {
 define i32 @na_minus_max_b_na(i32 %A, i32 %B) {
 ; CHECK-LABEL: @na_minus_max_b_na(
 ; CHECK-NEXT:    [[NOT:%.*]] = xor i32 [[A:%.*]], -1
-; CHECK-NEXT:    [[L0:%.*]] = icmp ugt i32 [[NOT]], [[B:%.*]]
-; CHECK-NEXT:    [[L1:%.*]] = select i1 [[L0]], i32 [[B]], i32 [[NOT]]
-; CHECK-NEXT:    [[X:%.*]] = sub i32 [[NOT]], [[L1]]
-; CHECK-NEXT:    ret i32 [[X]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.usub.sat.i32(i32 [[NOT]], i32 [[B:%.*]])
+; CHECK-NEXT:    ret i32 [[TMP1]]
 ;
   %not = xor i32 %A, -1
   %l0 = icmp ugt i32 %not, %B
@@ -64,10 +58,9 @@ define i32 @na_minus_max_b_na(i32 %A, i32 %B) {
 
 define i32 @max_na_bi_minux_na(i32 %A, i32 %Bi) {
 ; CHECK-LABEL: @max_na_bi_minux_na(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[A:%.*]], [[BI:%.*]]
-; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i32 [[BI]], i32 [[A]]
-; CHECK-NEXT:    [[X:%.*]] = sub i32 [[A]], [[TMP2]]
-; CHECK-NEXT:    ret i32 [[X]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.usub.sat.i32(i32 [[BI:%.*]], i32 [[A:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = sub i32 0, [[TMP1]]
+; CHECK-NEXT:    ret i32 [[TMP2]]
 ;
   %B =  xor i32 %Bi, -1
   %not = xor i32 %A, -1
@@ -79,10 +72,8 @@ define i32 @max_na_bi_minux_na(i32 %A, i32 %Bi) {
 
 define i32 @na_minus_max_na_bi(i32 %A, i32 %Bi) {
 ; CHECK-LABEL: @na_minus_max_na_bi(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[A:%.*]], [[BI:%.*]]
-; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i32 [[BI]], i32 [[A]]
-; CHECK-NEXT:    [[X:%.*]] = sub i32 [[TMP2]], [[A]]
-; CHECK-NEXT:    ret i32 [[X]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.usub.sat.i32(i32 [[BI:%.*]], i32 [[A:%.*]])
+; CHECK-NEXT:    ret i32 [[TMP1]]
 ;
   %B =  xor i32 %Bi, -1
   %not = xor i32 %A, -1
@@ -94,10 +85,9 @@ define i32 @na_minus_max_na_bi(i32 %A, i32 %Bi) {
 
 define i32 @max_bi_na_minus_na(i32 %A, i32 %Bi) {
 ; CHECK-LABEL: @max_bi_na_minus_na(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[A:%.*]], [[BI:%.*]]
-; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i32 [[BI]], i32 [[A]]
-; CHECK-NEXT:    [[X:%.*]] = sub i32 [[A]], [[TMP2]]
-; CHECK-NEXT:    ret i32 [[X]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.usub.sat.i32(i32 [[BI:%.*]], i32 [[A:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = sub i32 0, [[TMP1]]
+; CHECK-NEXT:    ret i32 [[TMP2]]
 ;
   %B =  xor i32 %Bi, -1
   %not = xor i32 %A, -1
@@ -109,10 +99,8 @@ define i32 @max_bi_na_minus_na(i32 %A, i32 %Bi) {
 
 define i32 @na_minus_max_bi_na(i32 %A, i32 %Bi) {
 ; CHECK-LABEL: @na_minus_max_bi_na(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[A:%.*]], [[BI:%.*]]
-; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i32 [[BI]], i32 [[A]]
-; CHECK-NEXT:    [[X:%.*]] = sub i32 [[TMP2]], [[A]]
-; CHECK-NEXT:    ret i32 [[X]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.usub.sat.i32(i32 [[BI:%.*]], i32 [[A:%.*]])
+; CHECK-NEXT:    ret i32 [[TMP1]]
 ;
   %B =  xor i32 %Bi, -1
   %not = xor i32 %A, -1
diff --git a/llvm/test/Transforms/InstCombine/sub-of-negatible.ll b/llvm/test/Transforms/InstCombine/sub-of-negatible.ll
new file mode 100644
index 000000000..01084312c
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/sub-of-negatible.ll
@@ -0,0 +1,377 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instcombine -S | FileCheck %s
+
+declare void @use8(i8)
+
+; Constant can be freely negated.
+define i8 @t0(i8 %x) {
+; CHECK-LABEL: @t0(
+; CHECK-NEXT:    [[T0:%.*]] = add i8 [[X:%.*]], 42
+; CHECK-NEXT:    ret i8 [[T0]]
+;
+  %t0 = sub i8 %x, -42
+  ret i8 %t0
+}
+
+; Negation can be negated for free
+define i8 @t1(i8 %x, i8 %y) {
+; CHECK-LABEL: @t1(
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = add i8 [[X:%.*]], [[Y]]
+; CHECK-NEXT:    ret i8 [[T1]]
+;
+  %t0 = sub i8 0, %y
+  call void @use8(i8 %t0)
+  %t1 = sub i8 %x, %t0
+  ret i8 %t1
+}
+
+; Shift-left can be negated if all uses can be updated
+define i8 @t2(i8 %x, i8 %y) {
+; CHECK-LABEL: @t2(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 -42, [[Y:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = sub i8 [[X:%.*]], [[T0]]
+; CHECK-NEXT:    ret i8 [[T1]]
+;
+  %t0 = shl i8 -42, %y
+  %t1 = sub i8 %x, %t0
+  ret i8 %t1
+}
+define i8 @n2(i8 %x, i8 %y) {
+; CHECK-LABEL: @n2(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 -42, [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = sub i8 [[X:%.*]], [[T0]]
+; CHECK-NEXT:    ret i8 [[T1]]
+;
+  %t0 = shl i8 -42, %y
+  call void @use8(i8 %t0)
+  %t1 = sub i8 %x, %t0
+  ret i8 %t1
+}
+define i8 @t3(i8 %x, i8 %y, i8 %z) {
+; CHECK-LABEL: @t3(
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[Z:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = shl i8 [[T0]], [[Y:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = sub i8 [[X:%.*]], [[T1]]
+; CHECK-NEXT:    ret i8 [[T2]]
+;
+  %t0 = sub i8 0, %z
+  call void @use8(i8 %t0)
+  %t1 = shl i8 %t0, %y
+  %t2 = sub i8 %x, %t1
+  ret i8 %t2
+}
+define i8 @n3(i8 %x, i8 %y, i8 %z) {
+; CHECK-LABEL: @n3(
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[Z:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = shl i8 [[T0]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T1]])
+; CHECK-NEXT:    [[T2:%.*]] = sub i8 [[X:%.*]], [[T1]]
+; CHECK-NEXT:    ret i8 [[T2]]
+;
+  %t0 = sub i8 0, %z
+  call void @use8(i8 %t0)
+  %t1 = shl i8 %t0, %y
+  call void @use8(i8 %t1)
+  %t2 = sub i8 %x, %t1
+  ret i8 %t2
+}
+
+; Select can be negated if all it's operands can be negated and all the users of select can be updated
+define i8 @t4(i8 %x, i1 %y) {
+; CHECK-LABEL: @t4(
+; CHECK-NEXT:    [[T0:%.*]] = select i1 [[Y:%.*]], i8 -42, i8 44
+; CHECK-NEXT:    [[T1:%.*]] = sub i8 [[X:%.*]], [[T0]]
+; CHECK-NEXT:    ret i8 [[T1]]
+;
+  %t0 = select i1 %y, i8 -42, i8 44
+  %t1 = sub i8 %x, %t0
+  ret i8 %t1
+}
+define i8 @n4(i8 %x, i1 %y) {
+; CHECK-LABEL: @n4(
+; CHECK-NEXT:    [[T0:%.*]] = select i1 [[Y:%.*]], i8 -42, i8 44
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = sub i8 [[X:%.*]], [[T0]]
+; CHECK-NEXT:    ret i8 [[T1]]
+;
+  %t0 = select i1 %y, i8 -42, i8 44
+  call void @use8(i8 %t0)
+  %t1 = sub i8 %x, %t0
+  ret i8 %t1
+}
+define i8 @n5(i8 %x, i1 %y, i8 %z) {
+; CHECK-LABEL: @n5(
+; CHECK-NEXT:    [[T0:%.*]] = select i1 [[Y:%.*]], i8 -42, i8 [[Z:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = sub i8 [[X:%.*]], [[T0]]
+; CHECK-NEXT:    ret i8 [[T1]]
+;
+  %t0 = select i1 %y, i8 -42, i8 %z
+  %t1 = sub i8 %x, %t0
+  ret i8 %t1
+}
+define i8 @t6(i8 %x, i1 %y, i8 %z) {
+; CHECK-LABEL: @t6(
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[Z:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = select i1 [[Y:%.*]], i8 -42, i8 [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = sub i8 [[X:%.*]], [[T1]]
+; CHECK-NEXT:    ret i8 [[T2]]
+;
+  %t0 = sub i8 0, %z
+  call void @use8(i8 %t0)
+  %t1 = select i1 %y, i8 -42, i8 %t0
+  %t2 = sub i8 %x, %t1
+  ret i8 %t2
+}
+define i8 @t7(i8 %x, i1 %y, i8 %z) {
+; CHECK-LABEL: @t7(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 1, [[Z:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = select i1 [[Y:%.*]], i8 0, i8 [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = sub i8 [[X:%.*]], [[T1]]
+; CHECK-NEXT:    ret i8 [[T2]]
+;
+  %t0 = shl i8 1, %z
+  %t1 = select i1 %y, i8 0, i8 %t0
+  %t2 = sub i8 %x, %t1
+  ret i8 %t2
+}
+define i8 @n8(i8 %x, i1 %y, i8 %z) {
+; CHECK-LABEL: @n8(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 1, [[Z:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = select i1 [[Y:%.*]], i8 0, i8 [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = sub i8 [[X:%.*]], [[T1]]
+; CHECK-NEXT:    ret i8 [[T2]]
+;
+  %t0 = shl i8 1, %z
+  call void @use8(i8 %t0)
+  %t1 = select i1 %y, i8 0, i8 %t0
+  %t2 = sub i8 %x, %t1
+  ret i8 %t2
+}
+
+; Subtraction can be negated by swapping its operands.
+; x - (y - z) -> x - y + z -> x + (z - y)
+define i8 @t9(i8 %x, i8 %y) {
+; CHECK-LABEL: @t9(
+; CHECK-NEXT:    [[T01:%.*]] = sub i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i8 [[T01]]
+;
+  %t0 = sub i8 %y, %x
+  %t1 = sub i8 0, %t0
+  ret i8 %t1
+}
+define i8 @n10(i8 %x, i8 %y, i8 %z) {
+; CHECK-LABEL: @n10(
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = sub i8 0, [[T0]]
+; CHECK-NEXT:    ret i8 [[T1]]
+;
+  %t0 = sub i8 %y, %x
+  call void @use8(i8 %t0)
+  %t1 = sub i8 0, %t0
+  ret i8 %t1
+}
+
+; Addition can be negated if both operands can be negated
+; x - (y + z) -> x - y - z -> x + ((-y) + (-z)))
+define i8 @t12(i8 %x, i8 %y, i8 %z) {
+; CHECK-LABEL: @t12(
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = sub i8 0, [[Z:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T1]])
+; CHECK-NEXT:    [[TMP1:%.*]] = add i8 [[Y]], [[Z]]
+; CHECK-NEXT:    [[T3:%.*]] = add i8 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    ret i8 [[T3]]
+;
+  %t0 = sub i8 0, %y
+  call void @use8(i8 %t0)
+  %t1 = sub i8 0, %z
+  call void @use8(i8 %t1)
+  %t2 = add i8 %t0, %t1
+  %t3 = sub i8 %x, %t2
+  ret i8 %t3
+}
+define i8 @n13(i8 %x, i8 %y, i8 %z) {
+; CHECK-LABEL: @n13(
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[T11:%.*]] = sub i8 [[Y]], [[Z:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = add i8 [[T11]], [[X:%.*]]
+; CHECK-NEXT:    ret i8 [[T2]]
+;
+  %t0 = sub i8 0, %y
+  call void @use8(i8 %t0)
+  %t1 = add i8 %t0, %z
+  %t2 = sub i8 %x, %t1
+  ret i8 %t2
+}
+define i8 @n14(i8 %x, i8 %y, i8 %z) {
+; CHECK-LABEL: @n14(
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = sub i8 0, [[Z:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T1]])
+; CHECK-NEXT:    [[TMP1:%.*]] = add i8 [[Y]], [[Z]]
+; CHECK-NEXT:    [[T2:%.*]] = sub i8 0, [[TMP1]]
+; CHECK-NEXT:    call void @use8(i8 [[T2]])
+; CHECK-NEXT:    [[T3:%.*]] = add i8 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    ret i8 [[T3]]
+;
+  %t0 = sub i8 0, %y
+  call void @use8(i8 %t0)
+  %t1 = sub i8 0, %z
+  call void @use8(i8 %t1)
+  %t2 = add i8 %t0, %t1
+  call void @use8(i8 %t2)
+  %t3 = sub i8 %x, %t2
+  ret i8 %t3
+}
+
+; Multiplication can be negated if either one of operands can be negated
+; x - (y * z) -> x + ((-y) * z) or  x + ((-z) * y)
+define i8 @t15(i8 %x, i8 %y, i8 %z) {
+; CHECK-LABEL: @t15(
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i8 [[Z:%.*]], [[Y]]
+; CHECK-NEXT:    [[T2:%.*]] = add i8 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    ret i8 [[T2]]
+;
+  %t0 = sub i8 0, %y
+  call void @use8(i8 %t0)
+  %t1 = mul i8 %t0, %z
+  %t2 = sub i8 %x, %t1
+  ret i8 %t2
+}
+define i8 @n16(i8 %x, i8 %y, i8 %z) {
+; CHECK-LABEL: @n16(
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = mul i8 [[T0]], [[Z:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T1]])
+; CHECK-NEXT:    [[T2:%.*]] = sub i8 [[X:%.*]], [[T1]]
+; CHECK-NEXT:    ret i8 [[T2]]
+;
+  %t0 = sub i8 0, %y
+  call void @use8(i8 %t0)
+  %t1 = mul i8 %t0, %z
+  call void @use8(i8 %t1)
+  %t2 = sub i8 %x, %t1
+  ret i8 %t2
+}
+
+; Phi can be negated if all incoming values can be negated
+define i8 @t16(i1 %c, i8 %x) {
+; CHECK-LABEL: @t16(
+; CHECK-NEXT:  begin:
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    br label [[END:%.*]]
+; CHECK:       else:
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    [[Z:%.*]] = phi i8 [ [[X:%.*]], [[THEN]] ], [ 42, [[ELSE]] ]
+; CHECK-NEXT:    ret i8 [[Z]]
+;
+begin:
+  br i1 %c, label %then, label %else
+then:
+  %y = sub i8 0, %x
+  br label %end
+else:
+  br label %end
+end:
+  %z = phi i8 [ %y, %then], [ -42, %else ]
+  %n = sub i8 0, %z
+  ret i8 %n
+}
+define i8 @n17(i1 %c, i8 %x) {
+; CHECK-LABEL: @n17(
+; CHECK-NEXT:  begin:
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    [[Y:%.*]] = sub i8 0, [[X:%.*]]
+; CHECK-NEXT:    br label [[END:%.*]]
+; CHECK:       else:
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    [[Z:%.*]] = phi i8 [ [[Y]], [[THEN]] ], [ -42, [[ELSE]] ]
+; CHECK-NEXT:    call void @use8(i8 [[Z]])
+; CHECK-NEXT:    [[N:%.*]] = sub i8 0, [[Z]]
+; CHECK-NEXT:    ret i8 [[N]]
+;
+begin:
+  br i1 %c, label %then, label %else
+then:
+  %y = sub i8 0, %x
+  br label %end
+else:
+  br label %end
+end:
+  %z = phi i8 [ %y, %then], [ -42, %else ]
+  call void @use8(i8 %z)
+  %n = sub i8 0, %z
+  ret i8 %n
+}
+define i8 @n19(i1 %c, i8 %x, i8 %y) {
+; CHECK-LABEL: @n19(
+; CHECK-NEXT:  begin:
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    [[Z:%.*]] = sub i8 0, [[X:%.*]]
+; CHECK-NEXT:    br label [[END:%.*]]
+; CHECK:       else:
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    [[R:%.*]] = phi i8 [ [[Z]], [[THEN]] ], [ [[Y:%.*]], [[ELSE]] ]
+; CHECK-NEXT:    [[N:%.*]] = sub i8 0, [[R]]
+; CHECK-NEXT:    ret i8 [[N]]
+;
+begin:
+  br i1 %c, label %then, label %else
+then:
+  %z = sub i8 0, %x
+  br label %end
+else:
+  br label %end
+end:
+  %r = phi i8 [ %z, %then], [ %y, %else ]
+  %n = sub i8 0, %r
+  ret i8 %n
+}
+
+; truncation can be negated if it's operand can be negated
+define i8 @t20(i8 %x, i16 %y) {
+; CHECK-LABEL: @t20(
+; CHECK-NEXT:    [[T0:%.*]] = shl i16 -42, [[Y:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = trunc i16 [[T0]] to i8
+; CHECK-NEXT:    [[T2:%.*]] = sub i8 [[X:%.*]], [[T1]]
+; CHECK-NEXT:    ret i8 [[T2]]
+;
+  %t0 = shl i16 -42, %y
+  %t1 = trunc i16 %t0 to i8
+  %t2 = sub i8 %x, %t1
+  ret i8 %t2
+}
+define i8 @n21(i8 %x, i16 %y) {
+; CHECK-LABEL: @n21(
+; CHECK-NEXT:    [[T0:%.*]] = shl i16 -42, [[Y:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = trunc i16 [[T0]] to i8
+; CHECK-NEXT:    call void @use8(i8 [[T1]])
+; CHECK-NEXT:    [[T2:%.*]] = sub i8 [[X:%.*]], [[T1]]
+; CHECK-NEXT:    ret i8 [[T2]]
+;
+  %t0 = shl i16 -42, %y
+  %t1 = trunc i16 %t0 to i8
+  call void @use8(i8 %t1)
+  %t2 = sub i8 %x, %t1
+  ret i8 %t2
+}
diff --git a/llvm/test/Transforms/InstCombine/sub-or-and-xor.ll b/llvm/test/Transforms/InstCombine/sub-or-and-xor.ll
new file mode 100644
index 000000000..59d3c6f6b
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/sub-or-and-xor.ll
@@ -0,0 +1,104 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+declare void @use(i32)
+
+define i32 @sub_to_xor(i32 %x, i32 %y) {
+; CHECK-LABEL: @sub_to_xor(
+; CHECK-NEXT:    [[SUB:%.*]] = xor i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[SUB]]
+;
+  %or = or i32 %x, %y
+  %and = and i32 %x, %y
+  %sub = sub i32 %or, %and
+  ret i32 %sub
+}
+
+define i32 @sub_to_xor_extra_use_sub(i32 %x, i32 %y) {
+; CHECK-LABEL: @sub_to_xor_extra_use_sub(
+; CHECK-NEXT:    [[SUB:%.*]] = xor i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use(i32 [[SUB]])
+; CHECK-NEXT:    ret i32 [[SUB]]
+;
+  %or = or i32 %x, %y
+  %and = and i32 %x, %y
+  %sub = sub i32 %or, %and
+  call void @use(i32 %sub)
+  ret i32 %sub
+}
+
+define i32 @sub_to_xor_extra_use_and(i32 %x, i32 %y) {
+; CHECK-LABEL: @sub_to_xor_extra_use_and(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use(i32 [[AND]])
+; CHECK-NEXT:    [[SUB:%.*]] = xor i32 [[X]], [[Y]]
+; CHECK-NEXT:    ret i32 [[SUB]]
+;
+  %or = or i32 %x, %y
+  %and = and i32 %x, %y
+  call void @use(i32 %and)
+  %sub = sub i32 %or, %and
+  ret i32 %sub
+}
+
+define i32 @sub_to_xor_extra_use_or(i32 %x, i32 %y) {
+; CHECK-LABEL: @sub_to_xor_extra_use_or(
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use(i32 [[OR]])
+; CHECK-NEXT:    [[SUB:%.*]] = xor i32 [[X]], [[Y]]
+; CHECK-NEXT:    ret i32 [[SUB]]
+;
+  %or = or i32 %x, %y
+  call void @use(i32 %or)
+  %and = and i32 %x, %y
+  %sub = sub i32 %or, %and
+  ret i32 %sub
+}
+
+define i32 @sub_to_xor_or_commuted(i32 %x, i32 %y) {
+; CHECK-LABEL: @sub_to_xor_or_commuted(
+; CHECK-NEXT:    [[SUB:%.*]] = xor i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[SUB]]
+;
+  %or = or i32 %y, %x
+  %and = and i32 %x, %y
+  %sub = sub i32 %or, %and
+  ret i32 %sub
+}
+
+define i32 @sub_to_xor_and_commuted(i32 %x, i32 %y) {
+; CHECK-LABEL: @sub_to_xor_and_commuted(
+; CHECK-NEXT:    [[SUB:%.*]] = xor i32 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[SUB]]
+;
+  %or = or i32 %x, %y
+  %and = and i32 %y, %x
+  %sub = sub i32 %or, %and
+  ret i32 %sub
+}
+
+define <2 x i32> @sub_to_xor_vec(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @sub_to_xor_vec(
+; CHECK-NEXT:    [[SUB:%.*]] = xor <2 x i32> [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    ret <2 x i32> [[SUB]]
+;
+  %or = or <2 x i32> %x, %y
+  %and = and <2 x i32> %y, %x
+  %sub = sub <2 x i32> %or, %and
+  ret <2 x i32> %sub
+}
+
+; Negative tests
+
+define i32 @sub_to_xor_wrong_arg(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @sub_to_xor_wrong_arg(
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X]], [[Z:%.*]]
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 [[AND]], [[OR]]
+; CHECK-NEXT:    ret i32 [[SUB]]
+;
+  %or = or i32 %x, %y
+  %and = and i32 %x, %z
+  %sub = sub i32 %and, %or
+  ret i32 %sub
+}
diff --git a/llvm/test/Transforms/InstCombine/sub-xor-or-neg-and.ll b/llvm/test/Transforms/InstCombine/sub-xor-or-neg-and.ll
new file mode 100644
index 000000000..ab32fe528
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/sub-xor-or-neg-and.ll
@@ -0,0 +1,115 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+declare void @use(i32)
+
+define i32 @sub_to_and(i32 %x, i32 %y) {
+; CHECK-LABEL: @sub_to_and(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 0, [[TMP1]]
+; CHECK-NEXT:    ret i32 [[SUB]]
+;
+  %or = or i32 %x, %y
+  %xor = xor i32 %x, %y
+  %sub = sub i32 %xor, %or
+  ret i32 %sub
+}
+
+define i32 @sub_to_and_extra_use_sub(i32 %x, i32 %y) {
+; CHECK-LABEL: @sub_to_and_extra_use_sub(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 0, [[TMP1]]
+; CHECK-NEXT:    call void @use(i32 [[SUB]])
+; CHECK-NEXT:    ret i32 [[SUB]]
+;
+  %or = or i32 %x, %y
+  %xor = xor i32 %x, %y
+  %sub = sub i32 %xor, %or
+  call void @use(i32 %sub)
+  ret i32 %sub
+}
+
+define i32 @sub_to_and_extra_use_and(i32 %x, i32 %y) {
+; CHECK-LABEL: @sub_to_and_extra_use_and(
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use(i32 [[XOR]])
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 0, [[TMP1]]
+; CHECK-NEXT:    ret i32 [[SUB]]
+;
+  %or = or i32 %x, %y
+  %xor = xor i32 %x, %y
+  call void @use(i32 %xor)
+  %sub = sub i32 %xor, %or
+  ret i32 %sub
+}
+
+define i32 @sub_to_and_extra_use_or(i32 %x, i32 %y) {
+; CHECK-LABEL: @sub_to_and_extra_use_or(
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use(i32 [[OR]])
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 0, [[TMP1]]
+; CHECK-NEXT:    ret i32 [[SUB]]
+;
+  %or = or i32 %x, %y
+  call void @use(i32 %or)
+  %xor = xor i32 %x, %y
+  %sub = sub i32 %xor, %or
+  ret i32 %sub
+}
+
+define i32 @sub_to_and_or_commuted(i32 %x, i32 %y) {
+; CHECK-LABEL: @sub_to_and_or_commuted(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 0, [[TMP1]]
+; CHECK-NEXT:    ret i32 [[SUB]]
+;
+  %or = or i32 %y, %x
+  %xor = xor i32 %x, %y
+  %sub = sub i32 %xor, %or
+  ret i32 %sub
+}
+
+define i32 @sub_to_and_and_commuted(i32 %x, i32 %y) {
+; CHECK-LABEL: @sub_to_and_and_commuted(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 0, [[TMP1]]
+; CHECK-NEXT:    ret i32 [[SUB]]
+;
+  %or = or i32 %x, %y
+  %xor = xor i32 %y, %x
+  %sub = sub i32 %xor, %or
+  ret i32 %sub
+}
+
+define <2 x i32> @sub_to_and_vec(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @sub_to_and_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[SUB:%.*]] = sub <2 x i32> zeroinitializer, [[TMP1]]
+; CHECK-NEXT:    ret <2 x i32> [[SUB]]
+;
+  %or = or <2 x i32> %x, %y
+  %xor = xor <2 x i32> %y, %x
+  %sub = sub <2 x i32> %xor, %or
+  ret <2 x i32> %sub
+}
+
+; Negative tests
+
+define i32 @sub_to_and_extra_use_and_or(i32 %x, i32 %y) {
+; CHECK-LABEL: @sub_to_and_extra_use_and_or(
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use(i32 [[OR]])
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[X]], [[Y]]
+; CHECK-NEXT:    call void @use(i32 [[XOR]])
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 [[XOR]], [[OR]]
+; CHECK-NEXT:    ret i32 [[SUB]]
+;
+  %or = or i32 %x, %y
+  call void @use(i32 %or)
+  %xor = xor i32 %x, %y
+  call void @use(i32 %xor)
+  %sub = sub i32 %xor, %or
+  ret i32 %sub
+}
diff --git a/llvm/test/Transforms/InstCombine/sub.ll b/llvm/test/Transforms/InstCombine/sub.ll
index 88c681c8c..1ab24c6d6 100644
--- a/llvm/test/Transforms/InstCombine/sub.ll
+++ b/llvm/test/Transforms/InstCombine/sub.ll
@@ -494,7 +494,7 @@ define i16 @test24a_as1(i8 addrspace(1)* %P, i16 %A) {
 
 define i64 @test24b(i8* %P, i64 %A){
 ; CHECK-LABEL: @test24b(
-; CHECK-NEXT:    [[B_IDX:%.*]] = shl nuw i64 [[A:%.*]], 1
+; CHECK-NEXT:    [[B_IDX:%.*]] = shl nsw i64 [[A:%.*]], 1
 ; CHECK-NEXT:    ret i64 [[B_IDX]]
 ;
   %B = getelementptr inbounds [42 x i16], [42 x i16]* @Arr, i64 0, i64 %A
@@ -506,9 +506,9 @@ define i64 @test24b(i8* %P, i64 %A){
 
 define i64 @test25(i8* %P, i64 %A){
 ; CHECK-LABEL: @test25(
-; CHECK-NEXT:    [[B_IDX:%.*]] = shl nuw i64 [[A:%.*]], 1
-; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[B_IDX]], -84
-; CHECK-NEXT:    ret i64 [[TMP1]]
+; CHECK-NEXT:    [[B_IDX:%.*]] = shl nsw i64 [[A:%.*]], 1
+; CHECK-NEXT:    [[DIFF_NEG:%.*]] = add i64 [[B_IDX]], -84
+; CHECK-NEXT:    ret i64 [[DIFF_NEG]]
 ;
   %B = getelementptr inbounds [42 x i16], [42 x i16]* @Arr, i64 0, i64 %A
   %C = ptrtoint i16* %B to i64
@@ -521,9 +521,9 @@ define i64 @test25(i8* %P, i64 %A){
 define i16 @test25_as1(i8 addrspace(1)* %P, i64 %A) {
 ; CHECK-LABEL: @test25_as1(
 ; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[A:%.*]] to i16
-; CHECK-NEXT:    [[B_IDX:%.*]] = shl nuw i16 [[TMP1]], 1
-; CHECK-NEXT:    [[TMP2:%.*]] = add i16 [[B_IDX]], -84
-; CHECK-NEXT:    ret i16 [[TMP2]]
+; CHECK-NEXT:    [[B_IDX:%.*]] = shl nsw i16 [[TMP1]], 1
+; CHECK-NEXT:    [[DIFF_NEG:%.*]] = add i16 [[B_IDX]], -84
+; CHECK-NEXT:    ret i16 [[DIFF_NEG]]
 ;
   %B = getelementptr inbounds [42 x i16], [42 x i16] addrspace(1)* @Arr_as1, i64 0, i64 %A
   %C = ptrtoint i16 addrspace(1)* %B to i16
@@ -668,7 +668,7 @@ define i64 @test29(i8* %foo, i64 %i, i64 %j) {
 
 define i64 @test30(i8* %foo, i64 %i, i64 %j) {
 ; CHECK-LABEL: @test30(
-; CHECK-NEXT:    [[GEP1_IDX:%.*]] = shl nuw i64 [[I:%.*]], 2
+; CHECK-NEXT:    [[GEP1_IDX:%.*]] = shl nsw i64 [[I:%.*]], 2
 ; CHECK-NEXT:    [[TMP1:%.*]] = sub i64 [[GEP1_IDX]], [[J:%.*]]
 ; CHECK-NEXT:    ret i64 [[TMP1]]
 ;
@@ -683,7 +683,7 @@ define i64 @test30(i8* %foo, i64 %i, i64 %j) {
 
 define i16 @test30_as1(i8 addrspace(1)* %foo, i16 %i, i16 %j) {
 ; CHECK-LABEL: @test30_as1(
-; CHECK-NEXT:    [[GEP1_IDX:%.*]] = shl nuw i16 [[I:%.*]], 2
+; CHECK-NEXT:    [[GEP1_IDX:%.*]] = shl nsw i16 [[I:%.*]], 2
 ; CHECK-NEXT:    [[TMP1:%.*]] = sub i16 [[GEP1_IDX]], [[J:%.*]]
 ; CHECK-NEXT:    ret i16 [[TMP1]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/subtract-from-one-hand-of-select.ll b/llvm/test/Transforms/InstCombine/subtract-from-one-hand-of-select.ll
new file mode 100644
index 000000000..8ff2e0ee2
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/subtract-from-one-hand-of-select.ll
@@ -0,0 +1,78 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S < %s -instcombine | FileCheck %s
+
+; Fold
+;   sub %Op0, (select %Cond, %TrueVal, %FalseVal)
+; to
+;   select %Cond, (sub %Op0, %TrueVal), (sub %Op0, %FalseVal)
+
+; https://bugs.llvm.org/show_bug.cgi?id=44426
+
+; Base tests
+
+define i8 @t0_sub_from_trueval(i1 %c, i8 %Op0, i8 %FalseVal) {
+; CHECK-LABEL: @t0_sub_from_trueval(
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i8 [[OP0:%.*]], [[FALSEVAL:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[C:%.*]], i8 0, i8 [[TMP1]], !prof !0
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %o = select i1 %c, i8 %Op0, i8 %FalseVal, !prof !0 ; while there, ensure preservation of prof md
+  %r = sub i8 %Op0, %o
+  ret i8 %r
+}
+define i8 @t1_sub_from_falseval(i1 %c, i8 %TrueVal, i8 %Op0) {
+; CHECK-LABEL: @t1_sub_from_falseval(
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i8 [[OP0:%.*]], [[TRUEVAL:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[C:%.*]], i8 [[TMP1]], i8 0, !prof !0
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %o = select i1 %c, i8 %TrueVal, i8 %Op0, !prof !0 ; while there, ensure preservation of prof md
+  %r = sub i8 %Op0, %o
+  ret i8 %r
+}
+
+; Vectors
+
+define <2 x i8> @t2_vec(i1 %c, <2 x i8> %Op0, <2 x i8> %FalseVal) {
+; CHECK-LABEL: @t2_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = sub <2 x i8> [[OP0:%.*]], [[FALSEVAL:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[C:%.*]], <2 x i8> zeroinitializer, <2 x i8> [[TMP1]]
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %o = select i1 %c, <2 x i8> %Op0, <2 x i8> %FalseVal
+  %r = sub <2 x i8> %Op0, %o
+  ret <2 x i8> %r
+}
+
+; Extra use
+
+declare void @use8(i8)
+
+define i8 @n3_extrause(i1 %c, i8 %Op0, i8 %FalseVal) {
+; CHECK-LABEL: @n3_extrause(
+; CHECK-NEXT:    [[O:%.*]] = select i1 [[C:%.*]], i8 [[OP0:%.*]], i8 [[FALSEVAL:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[O]])
+; CHECK-NEXT:    [[R:%.*]] = sub i8 [[OP0]], [[O]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %o = select i1 %c, i8 %Op0, i8 %FalseVal
+  call void @use8(i8 %o)
+  %r = sub i8 %Op0, %o
+  ret i8 %r
+}
+
+; Negative tests
+
+define i8 @n4_wrong_hands(i1 %c, i8 %TrueVal, i8 %FalseVal, i8 %Op0) {
+; CHECK-LABEL: @n4_wrong_hands(
+; CHECK-NEXT:    [[O:%.*]] = select i1 [[C:%.*]], i8 [[TRUEVAL:%.*]], i8 [[FALSEVAL:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = sub i8 [[OP0:%.*]], [[O]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %o = select i1 %c, i8 %TrueVal, i8 %FalseVal ; none of the hands is %Op0
+  %r = sub i8 %Op0, %o
+  ret i8 %r
+}
+
+; CHECK: !0 = !{!"branch_weights", i32 0, i32 100}
+!0  = !{!"branch_weights", i32 0, i32 100}
diff --git a/llvm/test/Transforms/InstCombine/subtract-of-one-hand-of-select.ll b/llvm/test/Transforms/InstCombine/subtract-of-one-hand-of-select.ll
new file mode 100644
index 000000000..cfe35f06b
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/subtract-of-one-hand-of-select.ll
@@ -0,0 +1,78 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S < %s -instcombine | FileCheck %s
+
+; Fold
+;   sub (select %Cond, %TrueVal, %FalseVal), %Op1
+; to
+;   select %Cond, (sub %TrueVal, %Op1), (sub %FalseVal, %Op1)
+
+; https://bugs.llvm.org/show_bug.cgi?id=44426
+
+; Base tests
+
+define i8 @t0_sub_of_trueval(i1 %c, i8 %Op1, i8 %FalseVal) {
+; CHECK-LABEL: @t0_sub_of_trueval(
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i8 [[FALSEVAL:%.*]], [[OP1:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[C:%.*]], i8 0, i8 [[TMP1]], !prof !0
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %o = select i1 %c, i8 %Op1, i8 %FalseVal, !prof !0 ; while there, ensure preservation of prof md
+  %r = sub i8 %o, %Op1
+  ret i8 %r
+}
+define i8 @t1_sub_of_falseval(i1 %c, i8 %TrueVal, i8 %Op1) {
+; CHECK-LABEL: @t1_sub_of_falseval(
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i8 [[TRUEVAL:%.*]], [[OP1:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[C:%.*]], i8 [[TMP1]], i8 0, !prof !0
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %o = select i1 %c, i8 %TrueVal, i8 %Op1, !prof !0 ; while there, ensure preservation of prof md
+  %r = sub i8 %o, %Op1
+  ret i8 %r
+}
+
+; Vectors
+
+define <2 x i8> @t2_vec(i1 %c, <2 x i8> %Op1, <2 x i8> %FalseVal) {
+; CHECK-LABEL: @t2_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = sub <2 x i8> [[FALSEVAL:%.*]], [[OP1:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[C:%.*]], <2 x i8> zeroinitializer, <2 x i8> [[TMP1]]
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %o = select i1 %c, <2 x i8> %Op1, <2 x i8> %FalseVal
+  %r = sub <2 x i8> %o, %Op1
+  ret <2 x i8> %r
+}
+
+; Extra use
+
+declare void @use8(i8)
+
+define i8 @n3_extrause(i1 %c, i8 %Op1, i8 %FalseVal) {
+; CHECK-LABEL: @n3_extrause(
+; CHECK-NEXT:    [[O:%.*]] = select i1 [[C:%.*]], i8 [[OP1:%.*]], i8 [[FALSEVAL:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[O]])
+; CHECK-NEXT:    [[R:%.*]] = sub i8 [[O]], [[OP1]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %o = select i1 %c, i8 %Op1, i8 %FalseVal
+  call void @use8(i8 %o)
+  %r = sub i8 %o, %Op1
+  ret i8 %r
+}
+
+; Negative tests
+
+define i8 @n4_wrong_hands(i1 %c, i8 %TrueVal, i8 %FalseVal, i8 %Op1) {
+; CHECK-LABEL: @n4_wrong_hands(
+; CHECK-NEXT:    [[O:%.*]] = select i1 [[C:%.*]], i8 [[TRUEVAL:%.*]], i8 [[FALSEVAL:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = sub i8 [[O]], [[OP1:%.*]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %o = select i1 %c, i8 %TrueVal, i8 %FalseVal ; none of the hands is %Op1
+  %r = sub i8 %o, %Op1
+  ret i8 %r
+}
+
+; CHECK: !0 = !{!"branch_weights", i32 0, i32 100}
+!0  = !{!"branch_weights", i32 0, i32 100}
diff --git a/llvm/test/Transforms/InstCombine/trunc-load.ll b/llvm/test/Transforms/InstCombine/trunc-load.ll
index 8088ce1b7..e17e99b13 100644
--- a/llvm/test/Transforms/InstCombine/trunc-load.ll
+++ b/llvm/test/Transforms/InstCombine/trunc-load.ll
@@ -2,8 +2,10 @@
 ; RUN: opt < %s -instcombine -S -data-layout="e-n16:32:64" | FileCheck %s --check-prefixes=CHECK,LE
 ; RUN: opt < %s -instcombine -S -data-layout="E-n16:32:64" | FileCheck %s --check-prefixes=CHECK,BE
 
-define i32 @truncload(i64* %ptr) {
-; CHECK-LABEL: @truncload(
+; Don't narrow if it would lose information about the dereferenceable range of the pointer.
+
+define i32 @truncload_no_deref(i64* %ptr) {
+; CHECK-LABEL: @truncload_no_deref(
 ; CHECK-NEXT:    [[X:%.*]] = load i64, i64* [[PTR:%.*]], align 4
 ; CHECK-NEXT:    [[R:%.*]] = trunc i64 [[X]] to i32
 ; CHECK-NEXT:    ret i32 [[R]]
@@ -13,7 +15,33 @@ define i32 @truncload(i64* %ptr) {
   ret i32 %r
 }
 
-define i16 @truncload_align(i32* %ptr) {
+define i32 @truncload_small_deref(i64* dereferenceable(7) %ptr) {
+; CHECK-LABEL: @truncload_small_deref(
+; CHECK-NEXT:    [[X:%.*]] = load i64, i64* [[PTR:%.*]], align 4
+; CHECK-NEXT:    [[R:%.*]] = trunc i64 [[X]] to i32
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %x = load i64, i64* %ptr
+  %r = trunc i64 %x to i32
+  ret i32 %r
+}
+
+; On little-endian, we can narrow the load without an offset.
+
+define i32 @truncload_deref(i64* dereferenceable(8) %ptr) {
+; CHECK-LABEL: @truncload_deref(
+; CHECK-NEXT:    [[X:%.*]] = load i64, i64* [[PTR:%.*]], align 4
+; CHECK-NEXT:    [[R:%.*]] = trunc i64 [[X]] to i32
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %x = load i64, i64* %ptr
+  %r = trunc i64 %x to i32
+  ret i32 %r
+}
+
+; Preserve alignment.
+
+define i16 @truncload_align(i32* dereferenceable(14) %ptr) {
 ; CHECK-LABEL: @truncload_align(
 ; CHECK-NEXT:    [[X:%.*]] = load i32, i32* [[PTR:%.*]], align 16
 ; CHECK-NEXT:    [[R:%.*]] = trunc i32 [[X]] to i16
@@ -24,9 +52,11 @@ define i16 @truncload_align(i32* %ptr) {
   ret i16 %r
 }
 
+; Negative test - extra use means we would not eliminate the original load.
+
 declare void @use(i64)
 
-define i32 @truncload_extra_use(i64* %ptr) {
+define i32 @truncload_extra_use(i64* dereferenceable(100) %ptr) {
 ; CHECK-LABEL: @truncload_extra_use(
 ; CHECK-NEXT:    [[X:%.*]] = load i64, i64* [[PTR:%.*]], align 2
 ; CHECK-NEXT:    call void @use(i64 [[X]])
@@ -39,7 +69,9 @@ define i32 @truncload_extra_use(i64* %ptr) {
   ret i32 %r
 }
 
-define i8 @truncload_type(i64* %ptr) {
+; Negative test - don't create a load if the type is not allowed by the data-layout.
+
+define i8 @truncload_type(i64* dereferenceable(9) %ptr) {
 ; CHECK-LABEL: @truncload_type(
 ; CHECK-NEXT:    [[X:%.*]] = load i64, i64* [[PTR:%.*]], align 2
 ; CHECK-NEXT:    [[R:%.*]] = trunc i64 [[X]] to i8
@@ -50,7 +82,9 @@ define i8 @truncload_type(i64* %ptr) {
   ret i8 %r
 }
 
-define i32 @truncload_volatile(i64* %ptr) {
+; Negative test - don't transform volatiles.
+
+define i32 @truncload_volatile(i64* dereferenceable(8) %ptr) {
 ; CHECK-LABEL: @truncload_volatile(
 ; CHECK-NEXT:    [[X:%.*]] = load volatile i64, i64* [[PTR:%.*]], align 8
 ; CHECK-NEXT:    [[R:%.*]] = trunc i64 [[X]] to i32
@@ -61,7 +95,9 @@ define i32 @truncload_volatile(i64* %ptr) {
   ret i32 %r
 }
 
-define i32 @truncload_address_space(i64 addrspace(1)* %ptr) {
+; Preserve address space.
+
+define i32 @truncload_address_space(i64 addrspace(1)* dereferenceable(8) %ptr) {
 ; CHECK-LABEL: @truncload_address_space(
 ; CHECK-NEXT:    [[X:%.*]] = load i64, i64 addrspace(1)* [[PTR:%.*]], align 4
 ; CHECK-NEXT:    [[R:%.*]] = trunc i64 [[X]] to i32
diff --git a/llvm/test/Transforms/InstCombine/trunc.ll b/llvm/test/Transforms/InstCombine/trunc.ll
index 750f62433..146d0fffa 100644
--- a/llvm/test/Transforms/InstCombine/trunc.ll
+++ b/llvm/test/Transforms/InstCombine/trunc.ll
@@ -626,3 +626,21 @@ define <2 x i8> @narrow_sub_vec_constant(<2 x i32> %x) {
   ret <2 x i8> %tr
 }
 
+; FIXME: If the select is narrowed based on the target's datalayout, we allow more optimizations.
+
+define i16 @PR44545(i32 %t0, i32 %data) {
+; CHECK-LABEL: @PR44545(
+; CHECK-NEXT:    [[T1:%.*]] = add nuw nsw i32 [[T0:%.*]], 1
+; CHECK-NEXT:    [[ISZERO:%.*]] = icmp eq i32 [[DATA:%.*]], 0
+; CHECK-NEXT:    [[FFS:%.*]] = select i1 [[ISZERO]], i32 0, i32 [[T1]]
+; CHECK-NEXT:    [[CAST:%.*]] = trunc i32 [[FFS]] to i16
+; CHECK-NEXT:    [[SUB:%.*]] = add nsw i16 [[CAST]], -1
+; CHECK-NEXT:    ret i16 [[SUB]]
+;
+  %t1 = add nuw nsw i32 %t0, 1
+  %iszero = icmp eq i32 %data, 0
+  %ffs = select i1 %iszero, i32 0, i32 %t1
+  %cast = trunc i32 %ffs to i16
+  %sub = add nsw i16 %cast, -1
+  ret i16 %sub
+}
diff --git a/llvm/test/Transforms/InstCombine/unavailable-debug.ll b/llvm/test/Transforms/InstCombine/unavailable-debug.ll
index 703c1c289..e3f0d7cf2 100644
--- a/llvm/test/Transforms/InstCombine/unavailable-debug.ll
+++ b/llvm/test/Transforms/InstCombine/unavailable-debug.ll
@@ -2,7 +2,7 @@
 
 ; Make sure to update the debug value after dead code elimination.
 ; CHECK: %call = call signext i8 @b(i32 6), !dbg !39
-; CHECK-NEXT: call void @llvm.dbg.value(metadata i32 undef, metadata !30, metadata !DIExpression()), !dbg !38
+; CHECK-NEXT: call void @llvm.dbg.value(metadata i8 %call, metadata !30, metadata !DIExpression(DW_OP_LLVM_convert, 8, DW_ATE_signed, DW_OP_LLVM_convert, 32, DW_ATE_signed, DW_OP_stack_value)), !dbg !38
 
 @e = common local_unnamed_addr global i8 0, align 1, !dbg !0
 @c = common local_unnamed_addr global i32 0, align 4, !dbg !6
diff --git a/llvm/test/Transforms/InstCombine/unordered-fcmp-select.ll b/llvm/test/Transforms/InstCombine/unordered-fcmp-select.ll
index 0eb729047..d2e1518cd 100644
--- a/llvm/test/Transforms/InstCombine/unordered-fcmp-select.ll
+++ b/llvm/test/Transforms/InstCombine/unordered-fcmp-select.ll
@@ -1,125 +1,138 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -S -instcombine < %s | FileCheck %s
 
-; CHECK-LABEL: @select_max_ugt(
-; CHECK: %cmp.inv = fcmp ole float %a, %b
-; CHECK-NEXT: %sel = select i1 %cmp.inv, float %b, float %a
-; CHECK-NEXT: ret float %sel
 define float @select_max_ugt(float %a, float %b) {
-  %cmp = fcmp ugt float %a, %b
-  %sel = select i1 %cmp, float %a, float %b
+; CHECK-LABEL: define {{[^@]+}}@select_max_ugt(
+; CHECK-NEXT:    [[CMP_INV:%.*]] = fcmp arcp ole float [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = select arcp i1 [[CMP_INV]], float [[B]], float [[A]]
+; CHECK-NEXT:    ret float [[TMP1]]
+;
+  %cmp = fcmp arcp ugt float %a, %b
+  %sel = select arcp i1 %cmp, float %a, float %b
   ret float %sel
 }
 
-; CHECK-LABEL: @select_max_uge(
-; CHECK: %cmp.inv = fcmp olt float %a, %b
-; CHECK-NEXT: %sel = select i1 %cmp.inv, float %b, float %a
-; CHECK-NEXT: ret float %sel
 define float @select_max_uge(float %a, float %b) {
-  %cmp = fcmp uge float %a, %b
-  %sel = select i1 %cmp, float %a, float %b
+; CHECK-LABEL: define {{[^@]+}}@select_max_uge(
+; CHECK-NEXT:    [[CMP_INV:%.*]] = fcmp nnan olt float [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = select nnan i1 [[CMP_INV]], float [[B]], float [[A]]
+; CHECK-NEXT:    ret float [[TMP1]]
+;
+  %cmp = fcmp nnan uge float %a, %b
+  %sel = select ninf i1 %cmp, float %a, float %b
   ret float %sel
 }
 
-; CHECK-LABEL: @select_min_ugt(
-; CHECK: %cmp.inv = fcmp ole float %a, %b
-; CHECK-NEXT: %sel = select i1 %cmp.inv, float %a, float %b
-; CHECK-NEXT: ret float %sel
 define float @select_min_ugt(float %a, float %b) {
-  %cmp = fcmp ugt float %a, %b
-  %sel = select i1 %cmp, float %b, float %a
+; CHECK-LABEL: define {{[^@]+}}@select_min_ugt(
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast float @llvm.minnum.f32(float [[A:%.*]], float [[B:%.*]])
+; CHECK-NEXT:    ret float [[TMP1]]
+;
+  %cmp = fcmp fast ugt float %a, %b
+  %sel = select reassoc i1 %cmp, float %b, float %a
   ret float %sel
 }
 
-; CHECK-LABEL: @select_min_uge(
-; CHECK: %cmp.inv = fcmp olt float %a, %b
-; CHECK-NEXT: %sel = select i1 %cmp.inv, float %a, float %b
-; CHECK-NEXT: ret float %sel
 define float @select_min_uge(float %a, float %b) {
-  %cmp = fcmp uge float %a, %b
-  %sel = select i1 %cmp, float %b, float %a
+; CHECK-LABEL: define {{[^@]+}}@select_min_uge(
+; CHECK-NEXT:    [[CMP_INV:%.*]] = fcmp nsz olt float [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = select nsz i1 [[CMP_INV]], float [[A]], float [[B]]
+; CHECK-NEXT:    ret float [[TMP1]]
+;
+  %cmp = fcmp nsz uge float %a, %b
+  %sel = select fast i1 %cmp, float %b, float %a
   ret float %sel
 }
 
-; CHECK-LABEL: @select_max_ult(
-; CHECK: %cmp.inv = fcmp oge float %a, %b
-; CHECK-NEXT: %sel = select i1 %cmp.inv, float %a, float %b
-; CHECK-NEXT: ret float %sel
 define float @select_max_ult(float %a, float %b) {
-  %cmp = fcmp ult float %a, %b
-  %sel = select i1 %cmp, float %b, float %a
+; CHECK-LABEL: define {{[^@]+}}@select_max_ult(
+; CHECK-NEXT:    [[CMP_INV:%.*]] = fcmp arcp oge float [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = select arcp i1 [[CMP_INV]], float [[A]], float [[B]]
+; CHECK-NEXT:    ret float [[TMP1]]
+;
+  %cmp = fcmp arcp ult float %a, %b
+  %sel = select ninf nnan i1 %cmp, float %b, float %a
   ret float %sel
 }
 
-; CHECK-LABEL: @select_max_ule(
-; CHECK: %cmp.inv = fcmp ogt float %a, %b
-; CHECK-NEXT: %sel = select i1 %cmp.inv, float %a, float %b
-; CHECK: ret float %sel
 define float @select_max_ule(float %a, float %b) {
-  %cmp = fcmp ule float %a, %b
-  %sel = select i1 %cmp, float %b, float %a
+; CHECK-LABEL: define {{[^@]+}}@select_max_ule(
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast float @llvm.maxnum.f32(float [[A:%.*]], float [[B:%.*]])
+; CHECK-NEXT:    ret float [[TMP1]]
+;
+  %cmp = fcmp fast ule float %a, %b
+  %sel = select nsz i1 %cmp, float %b, float %a
   ret float %sel
 }
 
-; CHECK-LABEL: @select_min_ult(
-; CHECK: %cmp.inv = fcmp oge float %a, %b
-; CHECK-NEXT: %sel = select i1 %cmp.inv, float %b, float %a
-; CHECK-NEXT: ret float %sel
 define float @select_min_ult(float %a, float %b) {
-  %cmp = fcmp ult float %a, %b
-  %sel = select i1 %cmp, float %a, float %b
+; CHECK-LABEL: define {{[^@]+}}@select_min_ult(
+; CHECK-NEXT:    [[CMP_INV:%.*]] = fcmp nsz oge float [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = select nsz i1 [[CMP_INV]], float [[B]], float [[A]]
+; CHECK-NEXT:    ret float [[TMP1]]
+;
+  %cmp = fcmp nsz ult float %a, %b
+  %sel = select fast i1 %cmp, float %a, float %b
   ret float %sel
 }
 
-; CHECK-LABEL: @select_min_ule(
-; CHECK: %cmp.inv = fcmp ogt float %a, %b
-; CHECK-NEXT: %sel = select i1 %cmp.inv, float %b, float %a
-; CHECK-NEXT: ret float %sel
 define float @select_min_ule(float %a, float %b) {
-  %cmp = fcmp ule float %a, %b
-  %sel = select i1 %cmp, float %a, float %b
+; CHECK-LABEL: define {{[^@]+}}@select_min_ule(
+; CHECK-NEXT:    [[CMP_INV:%.*]] = fcmp arcp ogt float [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = select arcp i1 [[CMP_INV]], float [[B]], float [[A]]
+; CHECK-NEXT:    ret float [[TMP1]]
+;
+  %cmp = fcmp arcp ule float %a, %b
+  %sel = select ninf i1 %cmp, float %a, float %b
   ret float %sel
 }
 
-; CHECK-LABEL: @select_fcmp_une(
-; CHECK:  %cmp.inv = fcmp oeq float %a, %b
-; CHECK-NEXT:  %sel = select i1 %cmp.inv, float %b, float %a
-; CHECK-NEXT: ret float %sel
 define float @select_fcmp_une(float %a, float %b) {
-  %cmp = fcmp une float %a, %b
-  %sel = select i1 %cmp, float %a, float %b
+; CHECK-LABEL: define {{[^@]+}}@select_fcmp_une(
+; CHECK-NEXT:    [[CMP_INV:%.*]] = fcmp reassoc oeq float [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = select reassoc i1 [[CMP_INV]], float [[B]], float [[A]]
+; CHECK-NEXT:    ret float [[TMP1]]
+;
+  %cmp = fcmp reassoc une float %a, %b
+  %sel = select nnan i1 %cmp, float %a, float %b
   ret float %sel
 }
 
-; CHECK-LABEL: @select_fcmp_ueq
-; CHECK:  %cmp.inv = fcmp one float %a, %b
-; CHECK-NEXT:  %sel = select i1 %cmp.inv, float %b, float %a
-; CHECK-NEXT: ret float %sel
 define float @select_fcmp_ueq(float %a, float %b) {
-  %cmp = fcmp ueq float %a, %b
-  %sel = select i1 %cmp, float %a, float %b
+; CHECK-LABEL: define {{[^@]+}}@select_fcmp_ueq(
+; CHECK-NEXT:    [[CMP_INV:%.*]] = fcmp reassoc one float [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = select reassoc i1 [[CMP_INV]], float [[B]], float [[A]]
+; CHECK-NEXT:    ret float [[TMP1]]
+;
+  %cmp = fcmp reassoc ueq float %a, %b
+  %sel = select arcp nnan i1 %cmp, float %a, float %b
   ret float %sel
 }
 
 declare void @foo(i1)
 
-; CHECK-LABEL: @select_max_ugt_2_use_cmp(
-; CHECK: fcmp ugt
-; CHECK-NOT: fcmp
-; CHECK: ret
 define float @select_max_ugt_2_use_cmp(float %a, float %b) {
-  %cmp = fcmp ugt float %a, %b
+; CHECK-LABEL: define {{[^@]+}}@select_max_ugt_2_use_cmp(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp reassoc ugt float [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    call void @foo(i1 [[CMP]])
+; CHECK-NEXT:    [[SEL:%.*]] = select fast i1 [[CMP]], float [[A]], float [[B]]
+; CHECK-NEXT:    ret float [[SEL]]
+;
+  %cmp = fcmp reassoc ugt float %a, %b
   call void @foo(i1 %cmp)
-  %sel = select i1 %cmp, float %a, float %b
+  %sel = select fast i1 %cmp, float %a, float %b
   ret float %sel
 }
 
-; CHECK-LABEL: @select_min_uge_2_use_cmp(
-; CHECK: fcmp uge
-; CHECK-NOT: fcmp
-; CHECK: ret
 define float @select_min_uge_2_use_cmp(float %a, float %b) {
-  %cmp = fcmp uge float %a, %b
+; CHECK-LABEL: define {{[^@]+}}@select_min_uge_2_use_cmp(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp ninf uge float [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    call void @foo(i1 [[CMP]])
+; CHECK-NEXT:    [[SEL:%.*]] = select nsz i1 [[CMP]], float [[B]], float [[A]]
+; CHECK-NEXT:    ret float [[SEL]]
+;
+  %cmp = fcmp ninf uge float %a, %b
   call void @foo(i1 %cmp)
-  %sel = select i1 %cmp, float %b, float %a
+  %sel = select nsz i1 %cmp, float %b, float %a
   ret float %sel
 }
diff --git a/llvm/test/Transforms/InstCombine/unpack-fca.ll b/llvm/test/Transforms/InstCombine/unpack-fca.ll
index 3c5e4177d..1bfd53f40 100644
--- a/llvm/test/Transforms/InstCombine/unpack-fca.ll
+++ b/llvm/test/Transforms/InstCombine/unpack-fca.ll
@@ -13,7 +13,7 @@ declare i32 @A.foo(%A* nocapture %this)
 
 define void @storeA(%A* %a.ptr) {
 ; CHECK-LABEL: storeA
-; CHECK-NEXT: [[GEP:%[a-z0-9\.]+]] = getelementptr inbounds %A, %A* %a.ptr, i64 0, i32 0
+; CHECK-NEXT: [[GEP:%[a-z0-9\.]+]] = getelementptr %A, %A* %a.ptr, i64 0, i32 0
 ; CHECK-NEXT: store %A__vtbl* @A__vtblZ, %A__vtbl** [[GEP]], align 8
 ; CHECK-NEXT: ret void
   store %A { %A__vtbl* @A__vtblZ }, %A* %a.ptr, align 8
@@ -33,7 +33,7 @@ define void @storeB(%B* %b.ptr) {
 
 define void @storeStructOfA({ %A }* %sa.ptr) {
 ; CHECK-LABEL: storeStructOfA
-; CHECK-NEXT: [[GEP:%[a-z0-9\.]+]] = getelementptr inbounds { %A }, { %A }* %sa.ptr, i64 0, i32 0, i32 0
+; CHECK-NEXT: [[GEP:%[a-z0-9\.]+]] = getelementptr { %A }, { %A }* %sa.ptr, i64 0, i32 0, i32 0
 ; CHECK-NEXT: store %A__vtbl* @A__vtblZ, %A__vtbl** [[GEP]], align 8
 ; CHECK-NEXT: ret void
   store { %A } { %A { %A__vtbl* @A__vtblZ } }, { %A }* %sa.ptr, align 8
@@ -42,7 +42,7 @@ define void @storeStructOfA({ %A }* %sa.ptr) {
 
 define void @storeArrayOfA([1 x %A]* %aa.ptr) {
 ; CHECK-LABEL: storeArrayOfA
-; CHECK-NEXT: [[GEP:%[a-z0-9\.]+]] = getelementptr inbounds [1 x %A], [1 x %A]* %aa.ptr, i64 0, i64 0, i32 0
+; CHECK-NEXT: [[GEP:%[a-z0-9\.]+]] = getelementptr [1 x %A], [1 x %A]* %aa.ptr, i64 0, i64 0, i32 0
 ; CHECK-NEXT: store %A__vtbl* @A__vtblZ, %A__vtbl** [[GEP]], align 8
 ; CHECK-NEXT: ret void
   store [1 x %A] [%A { %A__vtbl* @A__vtblZ }], [1 x %A]* %aa.ptr, align 8
@@ -60,7 +60,7 @@ define void @storeLargeArrayOfA([2000 x %A]* %aa.ptr) {
 
 define void @storeStructOfArrayOfA({ [1 x %A] }* %saa.ptr) {
 ; CHECK-LABEL: storeStructOfArrayOfA
-; CHECK-NEXT: [[GEP:%[a-z0-9\.]+]] = getelementptr inbounds { [1 x %A] }, { [1 x %A] }* %saa.ptr, i64 0, i32 0, i64 0, i32 0
+; CHECK-NEXT: [[GEP:%[a-z0-9\.]+]] = getelementptr { [1 x %A] }, { [1 x %A] }* %saa.ptr, i64 0, i32 0, i64 0, i32 0
 ; CHECK-NEXT: store %A__vtbl* @A__vtblZ, %A__vtbl** [[GEP]], align 8
 ; CHECK-NEXT: ret void
   store { [1 x %A] } { [1 x %A] [%A { %A__vtbl* @A__vtblZ }] }, { [1 x %A] }* %saa.ptr, align 8
@@ -90,7 +90,7 @@ define void @storeArrayOfB([2 x %B]* %ab.ptr, [2 x %B] %ab) {
 
 define %A @loadA(%A* %a.ptr) {
 ; CHECK-LABEL: loadA
-; CHECK-NEXT: [[GEP:%[a-z0-9\.]+]] = getelementptr inbounds %A, %A* %a.ptr, i64 0, i32 0
+; CHECK-NEXT: [[GEP:%[a-z0-9\.]+]] = getelementptr %A, %A* %a.ptr, i64 0, i32 0
 ; CHECK-NEXT: [[LOAD:%[a-z0-9\.]+]] = load %A__vtbl*, %A__vtbl** [[GEP]], align 8
 ; CHECK-NEXT: [[IV:%[a-z0-9\.]+]] = insertvalue %A undef, %A__vtbl* [[LOAD]], 0
 ; CHECK-NEXT: ret %A [[IV]]
@@ -113,7 +113,7 @@ define %B @loadB(%B* %b.ptr) {
 
 define { %A } @loadStructOfA({ %A }* %sa.ptr) {
 ; CHECK-LABEL: loadStructOfA
-; CHECK-NEXT: [[GEP:%[a-z0-9\.]+]] = getelementptr inbounds { %A }, { %A }* %sa.ptr, i64 0, i32 0, i32 0
+; CHECK-NEXT: [[GEP:%[a-z0-9\.]+]] = getelementptr { %A }, { %A }* %sa.ptr, i64 0, i32 0, i32 0
 ; CHECK-NEXT: [[LOAD:%[a-z0-9\.]+]] = load %A__vtbl*, %A__vtbl** [[GEP]], align 8
 ; CHECK-NEXT: [[IV1:%[a-z0-9\.]+]] = insertvalue %A undef, %A__vtbl* [[LOAD]], 0
 ; CHECK-NEXT: [[IV2:%[a-z0-9\.]+]] = insertvalue { %A } undef, %A [[IV1]], 0
@@ -124,7 +124,7 @@ define { %A } @loadStructOfA({ %A }* %sa.ptr) {
 
 define [1 x %A] @loadArrayOfA([1 x %A]* %aa.ptr) {
 ; CHECK-LABEL: loadArrayOfA
-; CHECK-NEXT: [[GEP:%[a-z0-9\.]+]] = getelementptr inbounds [1 x %A], [1 x %A]* %aa.ptr, i64 0, i64 0, i32 0
+; CHECK-NEXT: [[GEP:%[a-z0-9\.]+]] = getelementptr [1 x %A], [1 x %A]* %aa.ptr, i64 0, i64 0, i32 0
 ; CHECK-NEXT: [[LOAD:%[a-z0-9\.]+]] = load %A__vtbl*, %A__vtbl** [[GEP]], align 8
 ; CHECK-NEXT: [[IV1:%[a-z0-9\.]+]] = insertvalue %A undef, %A__vtbl* [[LOAD]], 0
 ; CHECK-NEXT: [[IV2:%[a-z0-9\.]+]] = insertvalue [1 x %A] undef, %A [[IV1]], 0
@@ -135,7 +135,7 @@ define [1 x %A] @loadArrayOfA([1 x %A]* %aa.ptr) {
 
 define { [1 x %A] } @loadStructOfArrayOfA({ [1 x %A] }* %saa.ptr) {
 ; CHECK-LABEL: loadStructOfArrayOfA
-; CHECK-NEXT: [[GEP:%[a-z0-9\.]+]] = getelementptr inbounds { [1 x %A] }, { [1 x %A] }* %saa.ptr, i64 0, i32 0, i64 0, i32 0
+; CHECK-NEXT: [[GEP:%[a-z0-9\.]+]] = getelementptr { [1 x %A] }, { [1 x %A] }* %saa.ptr, i64 0, i32 0, i64 0, i32 0
 ; CHECK-NEXT: [[LOAD:%[a-z0-9\.]+]] = load %A__vtbl*, %A__vtbl** [[GEP]], align 8
 ; CHECK-NEXT: [[IV1:%[a-z0-9\.]+]] = insertvalue %A undef, %A__vtbl* [[LOAD]], 0
 ; CHECK-NEXT: [[IV2:%[a-z0-9\.]+]] = insertvalue [1 x %A] undef, %A [[IV1]], 0
@@ -147,7 +147,7 @@ define { [1 x %A] } @loadStructOfArrayOfA({ [1 x %A] }* %saa.ptr) {
 
 define { %A } @structOfA({ %A }* %sa.ptr) {
 ; CHECK-LABEL: structOfA
-; CHECK-NEXT: [[GEP:%[a-z0-9\.]+]] = getelementptr inbounds { %A }, { %A }* %sa.ptr, i64 0, i32 0, i32 0
+; CHECK-NEXT: [[GEP:%[a-z0-9\.]+]] = getelementptr { %A }, { %A }* %sa.ptr, i64 0, i32 0, i32 0
 ; CHECK-NEXT: store %A__vtbl* @A__vtblZ, %A__vtbl** [[GEP]], align 8
 ; CHECK-NEXT: ret { %A } { %A { %A__vtbl* @A__vtblZ } }
   store { %A } { %A { %A__vtbl* @A__vtblZ } }, { %A }* %sa.ptr, align 8
diff --git a/llvm/test/Transforms/InstCombine/unrecognized_three-way-comparison.ll b/llvm/test/Transforms/InstCombine/unrecognized_three-way-comparison.ll
index dcd046e67..14e0b5573 100644
--- a/llvm/test/Transforms/InstCombine/unrecognized_three-way-comparison.ll
+++ b/llvm/test/Transforms/InstCombine/unrecognized_three-way-comparison.ll
@@ -457,3 +457,225 @@ exit:
   ret i32 42
 }
 
+declare void @use1(i1)
+define i32 @compare_against_fortytwo_commutatibility_0(i32 %x) {
+; CHECK-LABEL: @compare_against_fortytwo_commutatibility_0(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp sgt i32 [[X:%.*]], 42
+; CHECK-NEXT:    br i1 [[TMP0]], label [[CALLFOO:%.*]], label [[EXIT:%.*]]
+; CHECK:       callfoo:
+; CHECK-NEXT:    call void @foo(i32 1)
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i32 84
+;
+entry:
+  %cmp1 = icmp eq i32 %x, 42
+  %cmp2 = icmp slt i32 %x, 42
+  %select1 = select i1 %cmp2, i32 -1, i32 1
+  %select2 = select i1 %cmp1, i32 0, i32 %select1
+  %cond = icmp sgt i32 %select2, 0
+  br i1 %cond, label %callfoo, label %exit
+
+callfoo:
+  call void @foo(i32 %select2)
+  br label %exit
+
+exit:
+  ret i32 84
+}
+define i32 @compare_against_fortytwo_commutatibility_1(i32 %x) {
+; CHECK-LABEL: @compare_against_fortytwo_commutatibility_1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ne i32 [[X:%.*]], 42
+; CHECK-NEXT:    call void @use1(i1 [[CMP1]])
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp sgt i32 [[X]], 42
+; CHECK-NEXT:    br i1 [[TMP0]], label [[CALLFOO:%.*]], label [[EXIT:%.*]]
+; CHECK:       callfoo:
+; CHECK-NEXT:    call void @foo(i32 1)
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i32 84
+;
+entry:
+  %cmp1 = icmp ne i32 %x, 42 ; inverted
+  call void @use1(i1 %cmp1)
+  %cmp2 = icmp slt i32 %x, 42
+  %select1 = select i1 %cmp2, i32 -1, i32 1
+  %select2 = select i1 %cmp1, i32 %select1, i32 0 ; swapped
+  %cond = icmp sgt i32 %select2, 0
+  br i1 %cond, label %callfoo, label %exit
+
+callfoo:
+  call void @foo(i32 %select2)
+  br label %exit
+
+exit:
+  ret i32 84
+}
+define i32 @compare_against_fortytwo_commutatibility_2(i32 %x) {
+; CHECK-LABEL: @compare_against_fortytwo_commutatibility_2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp sgt i32 [[X:%.*]], 42
+; CHECK-NEXT:    br i1 [[TMP0]], label [[CALLFOO:%.*]], label [[EXIT:%.*]]
+; CHECK:       callfoo:
+; CHECK-NEXT:    call void @foo(i32 1)
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i32 84
+;
+entry:
+  %cmp1 = icmp eq i32 %x, 42
+  %cmp2 = icmp sgt i32 %x, 41 ; inverted
+  %select1 = select i1 %cmp2, i32 1, i32 -1 ; swapped
+  %select2 = select i1 %cmp1, i32 0, i32 %select1
+  %cond = icmp sgt i32 %select2, 0
+  br i1 %cond, label %callfoo, label %exit
+
+callfoo:
+  call void @foo(i32 %select2)
+  br label %exit
+
+exit:
+  ret i32 84
+}
+define i32 @compare_against_fortytwo_commutatibility_3(i32 %x) {
+; CHECK-LABEL: @compare_against_fortytwo_commutatibility_3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ne i32 [[X:%.*]], 42
+; CHECK-NEXT:    call void @use1(i1 [[CMP1]])
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp sgt i32 [[X]], 42
+; CHECK-NEXT:    br i1 [[TMP0]], label [[CALLFOO:%.*]], label [[EXIT:%.*]]
+; CHECK:       callfoo:
+; CHECK-NEXT:    call void @foo(i32 1)
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i32 84
+;
+entry:
+  %cmp1 = icmp ne i32 %x, 42 ; inverted
+  call void @use1(i1 %cmp1)
+  %cmp2 = icmp sgt i32 %x, 41 ; inverted
+  %select1 = select i1 %cmp2, i32 1, i32 -1 ; swapped
+  %select2 = select i1 %cmp1, i32 %select1, i32 0 ; swapped
+  %cond = icmp sgt i32 %select2, 0
+  br i1 %cond, label %callfoo, label %exit
+
+callfoo:
+  call void @foo(i32 %select2)
+  br label %exit
+
+exit:
+  ret i32 84
+}
+
+define i32 @compare_against_arbitrary_value_commutativity0(i32 %x, i32 %c) {
+; CHECK-LABEL: @compare_against_arbitrary_value_commutativity0(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp sgt i32 [[X:%.*]], [[C:%.*]]
+; CHECK-NEXT:    br i1 [[TMP0]], label [[CALLFOO:%.*]], label [[EXIT:%.*]]
+; CHECK:       callfoo:
+; CHECK-NEXT:    call void @foo(i32 1)
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i32 42
+;
+entry:
+  %cmp1 = icmp eq i32 %x, %c
+  %cmp2 = icmp slt i32 %x, %c
+  %select1 = select i1 %cmp2, i32 -1, i32 1
+  %select2 = select i1 %cmp1, i32 0, i32 %select1
+  %cond = icmp sgt i32 %select2, 0
+  br i1 %cond, label %callfoo, label %exit
+
+callfoo:
+  call void @foo(i32 %select2)
+  br label %exit
+
+exit:
+  ret i32 42
+}
+define i32 @compare_against_arbitrary_value_commutativity1(i32 %x, i32 %c) {
+; CHECK-LABEL: @compare_against_arbitrary_value_commutativity1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp sgt i32 [[X:%.*]], [[C:%.*]]
+; CHECK-NEXT:    br i1 [[TMP0]], label [[CALLFOO:%.*]], label [[EXIT:%.*]]
+; CHECK:       callfoo:
+; CHECK-NEXT:    call void @foo(i32 1)
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i32 42
+;
+entry:
+  %cmp1 = icmp eq i32 %x, %c
+  %cmp2 = icmp sgt i32 %c, %x ; swapped
+  %select1 = select i1 %cmp2, i32 -1, i32 1
+  %select2 = select i1 %cmp1, i32 0, i32 %select1
+  %cond = icmp sgt i32 %select2, 0
+  br i1 %cond, label %callfoo, label %exit
+
+callfoo:
+  call void @foo(i32 %select2)
+  br label %exit
+
+exit:
+  ret i32 42
+}
+define i32 @compare_against_arbitrary_value_commutativity2(i32 %x, i32 %c) {
+; CHECK-LABEL: @compare_against_arbitrary_value_commutativity2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ne i32 [[X:%.*]], [[C:%.*]]
+; CHECK-NEXT:    call void @use1(i1 [[CMP1]])
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp sgt i32 [[X]], [[C]]
+; CHECK-NEXT:    br i1 [[TMP0]], label [[CALLFOO:%.*]], label [[EXIT:%.*]]
+; CHECK:       callfoo:
+; CHECK-NEXT:    call void @foo(i32 1)
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i32 42
+;
+entry:
+  %cmp1 = icmp ne i32 %x, %c ; inverted
+  call void @use1(i1 %cmp1)
+  %cmp2 = icmp slt i32 %x, %c
+  %select1 = select i1 %cmp2, i32 -1, i32 1
+  %select2 = select i1 %cmp1, i32 %select1, i32 0 ; swapped
+  %cond = icmp sgt i32 %select2, 0
+  br i1 %cond, label %callfoo, label %exit
+
+callfoo:
+  call void @foo(i32 %select2)
+  br label %exit
+
+exit:
+  ret i32 42
+}
+define i32 @compare_against_arbitrary_value_commutativity3(i32 %x, i32 %c) {
+; CHECK-LABEL: @compare_against_arbitrary_value_commutativity3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ne i32 [[X:%.*]], [[C:%.*]]
+; CHECK-NEXT:    call void @use1(i1 [[CMP1]])
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp sgt i32 [[X]], [[C]]
+; CHECK-NEXT:    br i1 [[TMP0]], label [[CALLFOO:%.*]], label [[EXIT:%.*]]
+; CHECK:       callfoo:
+; CHECK-NEXT:    call void @foo(i32 1)
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i32 42
+;
+entry:
+  %cmp1 = icmp ne i32 %x, %c ; inverted
+  call void @use1(i1 %cmp1)
+  %cmp2 = icmp sgt i32 %c, %x ; swapped
+  %select1 = select i1 %cmp2, i32 -1, i32 1
+  %select2 = select i1 %cmp1, i32 %select1, i32 0 ; swapped
+  %cond = icmp sgt i32 %select2, 0
+  br i1 %cond, label %callfoo, label %exit
+
+callfoo:
+  call void @foo(i32 %select2)
+  br label %exit
+
+exit:
+  ret i32 42
+}
diff --git a/llvm/test/Transforms/InstCombine/unsigned-add-lack-of-overflow-check-via-add.ll b/llvm/test/Transforms/InstCombine/unsigned-add-lack-of-overflow-check-via-add.ll
new file mode 100644
index 000000000..b39f2bb8b
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/unsigned-add-lack-of-overflow-check-via-add.ll
@@ -0,0 +1,236 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instcombine -S | FileCheck %s
+
+; Should fold
+;   (%x + %y) u>= %x
+; or
+;   (%x + %y) u>= %y
+; to
+;   @llvm.uadd.with.overflow(%x, %y) + extractvalue + not
+;
+; All tests here have extra uses, to ensure that the pattern isn't perturbed.
+
+declare void @use8(i8)
+declare void @use2x8(<2 x i8>)
+
+define i1 @t0_basic(i8 %x, i8 %y) {
+; CHECK-LABEL: @t0_basic(
+; CHECK-NEXT:    [[T0:%.*]] = add i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[R:%.*]] = icmp uge i8 [[T0]], [[Y]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = add i8 %x, %y
+  call void @use8(i8 %t0)
+  %r = icmp uge i8 %t0, %y
+  ret i1 %r
+}
+
+define <2 x i1> @t1_vec(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @t1_vec(
+; CHECK-NEXT:    [[T0:%.*]] = add <2 x i8> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use2x8(<2 x i8> [[T0]])
+; CHECK-NEXT:    [[R:%.*]] = icmp uge <2 x i8> [[T0]], [[Y]]
+; CHECK-NEXT:    ret <2 x i1> [[R]]
+;
+  %t0 = add <2 x i8> %x, %y
+  call void @use2x8(<2 x i8> %t0)
+  %r = icmp uge <2 x i8> %t0, %y
+  ret <2 x i1> %r
+}
+
+; Commutativity
+
+define i1 @t2_symmetry(i8 %x, i8 %y) {
+; CHECK-LABEL: @t2_symmetry(
+; CHECK-NEXT:    [[T0:%.*]] = add i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[R:%.*]] = icmp uge i8 [[T0]], [[X]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = add i8 %x, %y
+  call void @use8(i8 %t0)
+  %r = icmp uge i8 %t0, %x ; can check against either of `add` arguments
+  ret i1 %r
+}
+
+declare i8 @gen8()
+
+define i1 @t3_commutative(i8 %x) {
+; CHECK-LABEL: @t3_commutative(
+; CHECK-NEXT:    [[Y:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[T0:%.*]] = add i8 [[Y]], [[X:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[R:%.*]] = icmp uge i8 [[T0]], [[Y]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %y = call i8 @gen8()
+  %t0 = add i8 %y, %x ; swapped
+  call void @use8(i8 %t0)
+  %r = icmp uge i8 %t0, %y
+  ret i1 %r
+}
+
+define i1 @t4_commutative() {
+; CHECK-LABEL: @t4_commutative(
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[Y:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[T0:%.*]] = add i8 [[X]], [[Y]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[R:%.*]] = icmp ule i8 [[Y]], [[T0]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %x = call i8 @gen8()
+  %y = call i8 @gen8()
+  %t0 = add i8 %x, %y
+  call void @use8(i8 %t0)
+  %r = icmp ule i8 %y, %t0 ; swapped
+  ret i1 %r
+}
+
+define i1 @t5_commutative(i8 %x) {
+; CHECK-LABEL: @t5_commutative(
+; CHECK-NEXT:    [[Y:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[T0:%.*]] = add i8 [[Y]], [[X:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[R:%.*]] = icmp ule i8 [[Y]], [[T0]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %y = call i8 @gen8()
+  %t0 = add i8 %y, %x ; swapped
+  call void @use8(i8 %t0)
+  %r = icmp ule i8 %y, %t0 ; swapped
+  ret i1 %r
+}
+
+; Extra-use tests
+
+define i1 @t6_no_extrause(i8 %x, i8 %y) {
+; CHECK-LABEL: @t6_no_extrause(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i8 [[Y:%.*]], -1
+; CHECK-NEXT:    [[R:%.*]] = icmp uge i8 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = add i8 %x, %y
+  %r = icmp uge i8 %t0, %y
+  ret i1 %r
+}
+
+; Negative tests
+
+define i1 @n7_different_y(i8 %x, i8 %y0, i8 %y1) {
+; CHECK-LABEL: @n7_different_y(
+; CHECK-NEXT:    [[T0:%.*]] = add i8 [[X:%.*]], [[Y0:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[R:%.*]] = icmp uge i8 [[T0]], [[Y1:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = add i8 %x, %y0
+  call void @use8(i8 %t0)
+  %r = icmp uge i8 %t0, %y1
+  ret i1 %r
+}
+
+define i1 @n8_wrong_pred0(i8 %x, i8 %y) {
+; CHECK-LABEL: @n8_wrong_pred0(
+; CHECK-NEXT:    [[T0:%.*]] = add i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[R:%.*]] = icmp ule i8 [[T0]], [[Y]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = add i8 %x, %y
+  call void @use8(i8 %t0)
+  %r = icmp ule i8 %t0, %y
+  ret i1 %r
+}
+
+define i1 @n9_wrong_pred1(i8 %x, i8 %y) {
+; CHECK-LABEL: @n9_wrong_pred1(
+; CHECK-NEXT:    [[T0:%.*]] = add i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[R:%.*]] = icmp ugt i8 [[T0]], [[Y]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = add i8 %x, %y
+  call void @use8(i8 %t0)
+  %r = icmp ugt i8 %t0, %y
+  ret i1 %r
+}
+
+define i1 @n10_wrong_pred2(i8 %x, i8 %y) {
+; CHECK-LABEL: @n10_wrong_pred2(
+; CHECK-NEXT:    [[T0:%.*]] = add i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i8 [[X]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = add i8 %x, %y
+  call void @use8(i8 %t0)
+  %r = icmp eq i8 %t0, %y
+  ret i1 %r
+}
+
+define i1 @n11_wrong_pred3(i8 %x, i8 %y) {
+; CHECK-LABEL: @n11_wrong_pred3(
+; CHECK-NEXT:    [[T0:%.*]] = add i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i8 [[X]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = add i8 %x, %y
+  call void @use8(i8 %t0)
+  %r = icmp ne i8 %t0, %y
+  ret i1 %r
+}
+
+define i1 @n12_wrong_pred4(i8 %x, i8 %y) {
+; CHECK-LABEL: @n12_wrong_pred4(
+; CHECK-NEXT:    [[T0:%.*]] = add i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[R:%.*]] = icmp slt i8 [[T0]], [[Y]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = add i8 %x, %y
+  call void @use8(i8 %t0)
+  %r = icmp slt i8 %t0, %y
+  ret i1 %r
+}
+
+define i1 @n13_wrong_pred5(i8 %x, i8 %y) {
+; CHECK-LABEL: @n13_wrong_pred5(
+; CHECK-NEXT:    [[T0:%.*]] = add i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[R:%.*]] = icmp sle i8 [[T0]], [[Y]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = add i8 %x, %y
+  call void @use8(i8 %t0)
+  %r = icmp sle i8 %t0, %y
+  ret i1 %r
+}
+
+define i1 @n14_wrong_pred6(i8 %x, i8 %y) {
+; CHECK-LABEL: @n14_wrong_pred6(
+; CHECK-NEXT:    [[T0:%.*]] = add i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[R:%.*]] = icmp sgt i8 [[T0]], [[Y]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = add i8 %x, %y
+  call void @use8(i8 %t0)
+  %r = icmp sgt i8 %t0, %y
+  ret i1 %r
+}
+
+define i1 @n15_wrong_pred7(i8 %x, i8 %y) {
+; CHECK-LABEL: @n15_wrong_pred7(
+; CHECK-NEXT:    [[T0:%.*]] = add i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[R:%.*]] = icmp sge i8 [[T0]], [[Y]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = add i8 %x, %y
+  call void @use8(i8 %t0)
+  %r = icmp sge i8 %t0, %y
+  ret i1 %r
+}
diff --git a/llvm/test/Transforms/InstCombine/unsigned-add-lack-of-overflow-check-via-xor.ll b/llvm/test/Transforms/InstCombine/unsigned-add-lack-of-overflow-check-via-xor.ll
new file mode 100644
index 000000000..f48bd6c9a
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/unsigned-add-lack-of-overflow-check-via-xor.ll
@@ -0,0 +1,189 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instcombine -S | FileCheck %s
+
+; Should fold
+;   (%y ^ -1) u>= %x
+; to
+;   @llvm.uadd.with.overflow(%x, %y) + extractvalue + not
+;
+; All tests here have extra uses, to ensure that the pattern isn't perturbed.
+
+declare void @use8(i8)
+declare void @use2x8(<2 x i8>)
+
+define i1 @t0_basic(i8 %x, i8 %y) {
+; CHECK-LABEL: @t0_basic(
+; CHECK-NEXT:    [[T0:%.*]] = xor i8 [[Y:%.*]], -1
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[R:%.*]] = icmp uge i8 [[T0]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = xor i8 %y, -1
+  call void @use8(i8 %t0)
+  %r = icmp uge i8 %t0, %x
+  ret i1 %r
+}
+
+define <2 x i1> @t1_vec(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @t1_vec(
+; CHECK-NEXT:    [[T0:%.*]] = xor <2 x i8> [[Y:%.*]], <i8 -1, i8 -1>
+; CHECK-NEXT:    call void @use2x8(<2 x i8> [[T0]])
+; CHECK-NEXT:    [[R:%.*]] = icmp uge <2 x i8> [[T0]], [[X:%.*]]
+; CHECK-NEXT:    ret <2 x i1> [[R]]
+;
+  %t0 = xor <2 x i8> %y, <i8 -1, i8 -1>
+  call void @use2x8(<2 x i8> %t0)
+  %r = icmp uge <2 x i8> %t0, %x
+  ret <2 x i1> %r
+}
+
+; Commutativity
+
+declare i8 @gen8()
+
+define i1 @t2_commutative(i8 %y) {
+; CHECK-LABEL: @t2_commutative(
+; CHECK-NEXT:    [[T0:%.*]] = xor i8 [[Y:%.*]], -1
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[R:%.*]] = icmp ule i8 [[X]], [[T0]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = xor i8 %y, -1
+  call void @use8(i8 %t0)
+  %x = call i8 @gen8()
+  %r = icmp ule i8 %x, %t0 ; swapped
+  ret i1 %r
+}
+
+; Extra-use tests
+
+define i1 @t3_no_extrause(i8 %x, i8 %y) {
+; CHECK-LABEL: @t3_no_extrause(
+; CHECK-NEXT:    [[T0:%.*]] = xor i8 [[Y:%.*]], -1
+; CHECK-NEXT:    [[R:%.*]] = icmp uge i8 [[T0]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = xor i8 %y, -1
+  %r = icmp uge i8 %t0, %x
+  ret i1 %r
+}
+
+; Negative tests
+
+define i1 @n4_wrong_pred0(i8 %x, i8 %y) {
+; CHECK-LABEL: @n4_wrong_pred0(
+; CHECK-NEXT:    [[T0:%.*]] = xor i8 [[Y:%.*]], -1
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[R:%.*]] = icmp ule i8 [[T0]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = xor i8 %y, -1
+  call void @use8(i8 %t0)
+  %r = icmp ule i8 %t0, %x
+  ret i1 %r
+}
+
+define i1 @n5_wrong_pred1(i8 %x, i8 %y) {
+; CHECK-LABEL: @n5_wrong_pred1(
+; CHECK-NEXT:    [[T0:%.*]] = xor i8 [[Y:%.*]], -1
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[R:%.*]] = icmp ugt i8 [[T0]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = xor i8 %y, -1
+  call void @use8(i8 %t0)
+  %r = icmp ugt i8 %t0, %x
+  ret i1 %r
+}
+
+define i1 @n6_wrong_pred2(i8 %x, i8 %y) {
+; CHECK-LABEL: @n6_wrong_pred2(
+; CHECK-NEXT:    [[T0:%.*]] = xor i8 [[Y:%.*]], -1
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i8 [[T0]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = xor i8 %y, -1
+  call void @use8(i8 %t0)
+  %r = icmp eq i8 %t0, %x
+  ret i1 %r
+}
+
+define i1 @n7_wrong_pred3(i8 %x, i8 %y) {
+; CHECK-LABEL: @n7_wrong_pred3(
+; CHECK-NEXT:    [[T0:%.*]] = xor i8 [[Y:%.*]], -1
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i8 [[T0]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = xor i8 %y, -1
+  call void @use8(i8 %t0)
+  %r = icmp ne i8 %t0, %x
+  ret i1 %r
+}
+
+define i1 @n8_wrong_pred4(i8 %x, i8 %y) {
+; CHECK-LABEL: @n8_wrong_pred4(
+; CHECK-NEXT:    [[T0:%.*]] = xor i8 [[Y:%.*]], -1
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[R:%.*]] = icmp slt i8 [[T0]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = xor i8 %y, -1
+  call void @use8(i8 %t0)
+  %r = icmp slt i8 %t0, %x
+  ret i1 %r
+}
+
+define i1 @n9_wrong_pred5(i8 %x, i8 %y) {
+; CHECK-LABEL: @n9_wrong_pred5(
+; CHECK-NEXT:    [[T0:%.*]] = xor i8 [[Y:%.*]], -1
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[R:%.*]] = icmp sle i8 [[T0]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = xor i8 %y, -1
+  call void @use8(i8 %t0)
+  %r = icmp sle i8 %t0, %x
+  ret i1 %r
+}
+
+define i1 @n10_wrong_pred6(i8 %x, i8 %y) {
+; CHECK-LABEL: @n10_wrong_pred6(
+; CHECK-NEXT:    [[T0:%.*]] = xor i8 [[Y:%.*]], -1
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[R:%.*]] = icmp sgt i8 [[T0]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = xor i8 %y, -1
+  call void @use8(i8 %t0)
+  %r = icmp sgt i8 %t0, %x
+  ret i1 %r
+}
+
+define i1 @n11_wrong_pred7(i8 %x, i8 %y) {
+; CHECK-LABEL: @n11_wrong_pred7(
+; CHECK-NEXT:    [[T0:%.*]] = xor i8 [[Y:%.*]], -1
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[R:%.*]] = icmp sge i8 [[T0]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = xor i8 %y, -1
+  call void @use8(i8 %t0)
+  %r = icmp sge i8 %t0, %x
+  ret i1 %r
+}
+
+define <2 x i1> @n12_vec_nonsplat(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @n12_vec_nonsplat(
+; CHECK-NEXT:    [[T0:%.*]] = xor <2 x i8> [[Y:%.*]], <i8 -1, i8 -2>
+; CHECK-NEXT:    call void @use2x8(<2 x i8> [[T0]])
+; CHECK-NEXT:    [[R:%.*]] = icmp uge <2 x i8> [[T0]], [[X:%.*]]
+; CHECK-NEXT:    ret <2 x i1> [[R]]
+;
+  %t0 = xor <2 x i8> %y, <i8 -1, i8 -2> ; must be -1.
+  call void @use2x8(<2 x i8> %t0)
+  %r = icmp uge <2 x i8> %t0, %x
+  ret <2 x i1> %r
+}
diff --git a/llvm/test/Transforms/InstCombine/unsigned-add-lack-of-overflow-check.ll b/llvm/test/Transforms/InstCombine/unsigned-add-lack-of-overflow-check.ll
new file mode 100644
index 000000000..29c195f3a
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/unsigned-add-lack-of-overflow-check.ll
@@ -0,0 +1,199 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instcombine -S | FileCheck %s
+
+; Should fold
+;   (%x + %y) u>= %x
+; or
+;   (%x + %y) u>= %y
+; to
+;   @llvm.uadd.with.overflow(%x, %y) + extractvalue + not
+
+define i1 @t0_basic(i8 %x, i8 %y) {
+; CHECK-LABEL: @t0_basic(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i8 [[Y:%.*]], -1
+; CHECK-NEXT:    [[R:%.*]] = icmp uge i8 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = add i8 %x, %y
+  %r = icmp uge i8 %t0, %y
+  ret i1 %r
+}
+
+define <2 x i1> @t1_vec(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @t1_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor <2 x i8> [[Y:%.*]], <i8 -1, i8 -1>
+; CHECK-NEXT:    [[R:%.*]] = icmp uge <2 x i8> [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    ret <2 x i1> [[R]]
+;
+  %t0 = add <2 x i8> %x, %y
+  %r = icmp uge <2 x i8> %t0, %y
+  ret <2 x i1> %r
+}
+
+; Commutativity
+
+define i1 @t2_symmetry(i8 %x, i8 %y) {
+; CHECK-LABEL: @t2_symmetry(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i8 [[X:%.*]], -1
+; CHECK-NEXT:    [[R:%.*]] = icmp uge i8 [[TMP1]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = add i8 %x, %y
+  %r = icmp uge i8 %t0, %x ; can check against either of `add` arguments
+  ret i1 %r
+}
+
+declare i8 @gen8()
+
+define i1 @t3_commutative(i8 %x) {
+; CHECK-LABEL: @t3_commutative(
+; CHECK-NEXT:    [[Y:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i8 [[Y]], -1
+; CHECK-NEXT:    [[R:%.*]] = icmp uge i8 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %y = call i8 @gen8()
+  %t0 = add i8 %y, %x ; swapped
+  %r = icmp uge i8 %t0, %y
+  ret i1 %r
+}
+
+define i1 @t4_commutative(i8 %x, i8 %y) {
+; CHECK-LABEL: @t4_commutative(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i8 [[Y:%.*]], -1
+; CHECK-NEXT:    [[R:%.*]] = icmp uge i8 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = add i8 %x, %y
+  %r = icmp ule i8 %y, %t0 ; swapped
+  ret i1 %r
+}
+
+define i1 @t5_commutative(i8 %x) {
+; CHECK-LABEL: @t5_commutative(
+; CHECK-NEXT:    [[Y:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i8 [[Y]], -1
+; CHECK-NEXT:    [[R:%.*]] = icmp uge i8 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %y = call i8 @gen8()
+  %t0 = add i8 %y, %x ; swapped
+  %r = icmp ule i8 %y, %t0 ; swapped
+  ret i1 %r
+}
+
+; Extra-use tests
+
+declare void @use8(i8)
+
+define i1 @t6_extrause(i8 %x, i8 %y) {
+; CHECK-LABEL: @t6_extrause(
+; CHECK-NEXT:    [[T0:%.*]] = add i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[R:%.*]] = icmp uge i8 [[T0]], [[Y]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = add i8 %x, %y
+  call void @use8(i8 %t0)
+  %r = icmp uge i8 %t0, %y
+  ret i1 %r
+}
+
+; Negative tests
+
+define i1 @n7_different_y(i8 %x, i8 %y0, i8 %y1) {
+; CHECK-LABEL: @n7_different_y(
+; CHECK-NEXT:    [[T0:%.*]] = add i8 [[X:%.*]], [[Y0:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp uge i8 [[T0]], [[Y1:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = add i8 %x, %y0
+  %r = icmp uge i8 %t0, %y1
+  ret i1 %r
+}
+
+define i1 @n8_wrong_pred0(i8 %x, i8 %y) {
+; CHECK-LABEL: @n8_wrong_pred0(
+; CHECK-NEXT:    [[T0:%.*]] = add i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp ule i8 [[T0]], [[Y]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = add i8 %x, %y
+  %r = icmp ule i8 %t0, %y
+  ret i1 %r
+}
+
+define i1 @n9_wrong_pred1(i8 %x, i8 %y) {
+; CHECK-LABEL: @n9_wrong_pred1(
+; CHECK-NEXT:    [[T0:%.*]] = add i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp ugt i8 [[T0]], [[Y]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = add i8 %x, %y
+  %r = icmp ugt i8 %t0, %y
+  ret i1 %r
+}
+
+define i1 @n10_wrong_pred2(i8 %x, i8 %y) {
+; CHECK-LABEL: @n10_wrong_pred2(
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i8 [[X:%.*]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = add i8 %x, %y
+  %r = icmp eq i8 %t0, %y
+  ret i1 %r
+}
+
+define i1 @n11_wrong_pred3(i8 %x, i8 %y) {
+; CHECK-LABEL: @n11_wrong_pred3(
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i8 [[X:%.*]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = add i8 %x, %y
+  %r = icmp ne i8 %t0, %y
+  ret i1 %r
+}
+
+define i1 @n12_wrong_pred4(i8 %x, i8 %y) {
+; CHECK-LABEL: @n12_wrong_pred4(
+; CHECK-NEXT:    [[T0:%.*]] = add i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp slt i8 [[T0]], [[Y]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = add i8 %x, %y
+  %r = icmp slt i8 %t0, %y
+  ret i1 %r
+}
+
+define i1 @n13_wrong_pred5(i8 %x, i8 %y) {
+; CHECK-LABEL: @n13_wrong_pred5(
+; CHECK-NEXT:    [[T0:%.*]] = add i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp sle i8 [[T0]], [[Y]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = add i8 %x, %y
+  %r = icmp sle i8 %t0, %y
+  ret i1 %r
+}
+
+define i1 @n14_wrong_pred6(i8 %x, i8 %y) {
+; CHECK-LABEL: @n14_wrong_pred6(
+; CHECK-NEXT:    [[T0:%.*]] = add i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp sgt i8 [[T0]], [[Y]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = add i8 %x, %y
+  %r = icmp sgt i8 %t0, %y
+  ret i1 %r
+}
+
+define i1 @n15_wrong_pred7(i8 %x, i8 %y) {
+; CHECK-LABEL: @n15_wrong_pred7(
+; CHECK-NEXT:    [[T0:%.*]] = add i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp sge i8 [[T0]], [[Y]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = add i8 %x, %y
+  %r = icmp sge i8 %t0, %y
+  ret i1 %r
+}
diff --git a/llvm/test/Transforms/InstCombine/unsigned-add-overflow-check-via-add.ll b/llvm/test/Transforms/InstCombine/unsigned-add-overflow-check-via-add.ll
new file mode 100644
index 000000000..a614142ef
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/unsigned-add-overflow-check-via-add.ll
@@ -0,0 +1,232 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instcombine -S | FileCheck %s
+
+; Should fold
+;   (%x + %y) u< %x
+; or
+;   (%x + %y) u< %y
+; to
+;   @llvm.uadd.with.overflow(%x, %y) + extractvalue
+;
+; All tests here have extra uses, to ensure that the pattern isn't perturbed.
+
+declare void @use8(i8)
+declare void @use2x8(<2 x i8>)
+
+define i1 @t0_basic(i8 %x, i8 %y) {
+; CHECK-LABEL: @t0_basic(
+; CHECK-NEXT:    [[T0:%.*]] = add i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[R:%.*]] = icmp ult i8 [[T0]], [[Y]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = add i8 %x, %y
+  call void @use8(i8 %t0)
+  %r = icmp ult i8 %t0, %y
+  ret i1 %r
+}
+
+define <2 x i1> @t1_vec(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @t1_vec(
+; CHECK-NEXT:    [[T0:%.*]] = add <2 x i8> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use2x8(<2 x i8> [[T0]])
+; CHECK-NEXT:    [[R:%.*]] = icmp ult <2 x i8> [[T0]], [[Y]]
+; CHECK-NEXT:    ret <2 x i1> [[R]]
+;
+  %t0 = add <2 x i8> %x, %y
+  call void @use2x8(<2 x i8> %t0)
+  %r = icmp ult <2 x i8> %t0, %y
+  ret <2 x i1> %r
+}
+
+; Commutativity
+
+define i1 @t2_symmetry(i8 %x, i8 %y) {
+; CHECK-LABEL: @t2_symmetry(
+; CHECK-NEXT:    [[T0:%.*]] = add i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[R:%.*]] = icmp ult i8 [[T0]], [[X]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = add i8 %x, %y
+  call void @use8(i8 %t0)
+  %r = icmp ult i8 %t0, %x ; can check against either of `add` arguments
+  ret i1 %r
+}
+
+declare i8 @gen8()
+
+define i1 @t3_commutative(i8 %x) {
+; CHECK-LABEL: @t3_commutative(
+; CHECK-NEXT:    [[Y:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[T0:%.*]] = add i8 [[Y]], [[X:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[R:%.*]] = icmp ult i8 [[T0]], [[Y]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %y = call i8 @gen8()
+  %t0 = add i8 %y, %x ; swapped
+  call void @use8(i8 %t0)
+  %r = icmp ult i8 %t0, %y
+  ret i1 %r
+}
+
+define i1 @t4_commutative(i8 %x, i8 %y) {
+; CHECK-LABEL: @t4_commutative(
+; CHECK-NEXT:    [[T0:%.*]] = add i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[R:%.*]] = icmp ult i8 [[T0]], [[Y]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = add i8 %x, %y
+  call void @use8(i8 %t0)
+  %r = icmp ugt i8 %y, %t0 ; swapped
+  ret i1 %r
+}
+
+define i1 @t5_commutative(i8 %x) {
+; CHECK-LABEL: @t5_commutative(
+; CHECK-NEXT:    [[Y:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[T0:%.*]] = add i8 [[Y]], [[X:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[R:%.*]] = icmp ugt i8 [[Y]], [[T0]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %y = call i8 @gen8()
+  %t0 = add i8 %y, %x ; swapped
+  call void @use8(i8 %t0)
+  %r = icmp ugt i8 %y, %t0 ; swapped
+  ret i1 %r
+}
+
+; Extra-use tests
+
+define i1 @t6_no_extrause(i8 %x, i8 %y) {
+; CHECK-LABEL: @t6_no_extrause(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i8 [[Y:%.*]], -1
+; CHECK-NEXT:    [[R:%.*]] = icmp ult i8 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = add i8 %x, %y
+  %r = icmp ult i8 %t0, %y
+  ret i1 %r
+}
+
+; Negative tests
+
+define i1 @n7_different_y(i8 %x, i8 %y0, i8 %y1) {
+; CHECK-LABEL: @n7_different_y(
+; CHECK-NEXT:    [[T0:%.*]] = add i8 [[X:%.*]], [[Y0:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[R:%.*]] = icmp ult i8 [[T0]], [[Y1:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = add i8 %x, %y0
+  call void @use8(i8 %t0)
+  %r = icmp ult i8 %t0, %y1
+  ret i1 %r
+}
+
+define i1 @n8_wrong_pred0(i8 %x, i8 %y) {
+; CHECK-LABEL: @n8_wrong_pred0(
+; CHECK-NEXT:    [[T0:%.*]] = add i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[R:%.*]] = icmp ule i8 [[T0]], [[Y]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = add i8 %x, %y
+  call void @use8(i8 %t0)
+  %r = icmp ule i8 %t0, %y
+  ret i1 %r
+}
+
+define i1 @n9_wrong_pred1(i8 %x, i8 %y) {
+; CHECK-LABEL: @n9_wrong_pred1(
+; CHECK-NEXT:    [[T0:%.*]] = add i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[R:%.*]] = icmp ugt i8 [[T0]], [[Y]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = add i8 %x, %y
+  call void @use8(i8 %t0)
+  %r = icmp ugt i8 %t0, %y
+  ret i1 %r
+}
+
+define i1 @n10_wrong_pred2(i8 %x, i8 %y) {
+; CHECK-LABEL: @n10_wrong_pred2(
+; CHECK-NEXT:    [[T0:%.*]] = add i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i8 [[X]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = add i8 %x, %y
+  call void @use8(i8 %t0)
+  %r = icmp eq i8 %t0, %y
+  ret i1 %r
+}
+
+define i1 @n11_wrong_pred3(i8 %x, i8 %y) {
+; CHECK-LABEL: @n11_wrong_pred3(
+; CHECK-NEXT:    [[T0:%.*]] = add i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i8 [[X]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = add i8 %x, %y
+  call void @use8(i8 %t0)
+  %r = icmp ne i8 %t0, %y
+  ret i1 %r
+}
+
+define i1 @n12_wrong_pred4(i8 %x, i8 %y) {
+; CHECK-LABEL: @n12_wrong_pred4(
+; CHECK-NEXT:    [[T0:%.*]] = add i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[R:%.*]] = icmp slt i8 [[T0]], [[Y]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = add i8 %x, %y
+  call void @use8(i8 %t0)
+  %r = icmp slt i8 %t0, %y
+  ret i1 %r
+}
+
+define i1 @n13_wrong_pred5(i8 %x, i8 %y) {
+; CHECK-LABEL: @n13_wrong_pred5(
+; CHECK-NEXT:    [[T0:%.*]] = add i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[R:%.*]] = icmp sle i8 [[T0]], [[Y]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = add i8 %x, %y
+  call void @use8(i8 %t0)
+  %r = icmp sle i8 %t0, %y
+  ret i1 %r
+}
+
+define i1 @n14_wrong_pred6(i8 %x, i8 %y) {
+; CHECK-LABEL: @n14_wrong_pred6(
+; CHECK-NEXT:    [[T0:%.*]] = add i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[R:%.*]] = icmp sgt i8 [[T0]], [[Y]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = add i8 %x, %y
+  call void @use8(i8 %t0)
+  %r = icmp sgt i8 %t0, %y
+  ret i1 %r
+}
+
+define i1 @n15_wrong_pred7(i8 %x, i8 %y) {
+; CHECK-LABEL: @n15_wrong_pred7(
+; CHECK-NEXT:    [[T0:%.*]] = add i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[R:%.*]] = icmp sge i8 [[T0]], [[Y]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = add i8 %x, %y
+  call void @use8(i8 %t0)
+  %r = icmp sge i8 %t0, %y
+  ret i1 %r
+}
diff --git a/llvm/test/Transforms/InstCombine/unsigned-add-overflow-check-via-xor.ll b/llvm/test/Transforms/InstCombine/unsigned-add-overflow-check-via-xor.ll
new file mode 100644
index 000000000..e8fa5c84d
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/unsigned-add-overflow-check-via-xor.ll
@@ -0,0 +1,189 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instcombine -S | FileCheck %s
+
+; Should fold
+;   (%y ^ -1) u< %x
+; to
+;   @llvm.uadd.with.overflow(%x, %y) + extractvalue + not
+;
+; All tests here have extra uses, to ensure that the pattern isn't perturbed.
+
+declare void @use8(i8)
+declare void @use2x8(<2 x i8>)
+
+define i1 @t0_basic(i8 %x, i8 %y) {
+; CHECK-LABEL: @t0_basic(
+; CHECK-NEXT:    [[T0:%.*]] = xor i8 [[Y:%.*]], -1
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[R:%.*]] = icmp ult i8 [[T0]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = xor i8 %y, -1
+  call void @use8(i8 %t0)
+  %r = icmp ult i8 %t0, %x
+  ret i1 %r
+}
+
+define <2 x i1> @t1_vec(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @t1_vec(
+; CHECK-NEXT:    [[T0:%.*]] = xor <2 x i8> [[Y:%.*]], <i8 -1, i8 -1>
+; CHECK-NEXT:    call void @use2x8(<2 x i8> [[T0]])
+; CHECK-NEXT:    [[R:%.*]] = icmp ult <2 x i8> [[T0]], [[X:%.*]]
+; CHECK-NEXT:    ret <2 x i1> [[R]]
+;
+  %t0 = xor <2 x i8> %y, <i8 -1, i8 -1>
+  call void @use2x8(<2 x i8> %t0)
+  %r = icmp ult <2 x i8> %t0, %x
+  ret <2 x i1> %r
+}
+
+; Commutativity
+
+declare i8 @gen8()
+
+define i1 @t2_commutative(i8 %y) {
+; CHECK-LABEL: @t2_commutative(
+; CHECK-NEXT:    [[T0:%.*]] = xor i8 [[Y:%.*]], -1
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[R:%.*]] = icmp ule i8 [[X]], [[T0]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = xor i8 %y, -1
+  call void @use8(i8 %t0)
+  %x = call i8 @gen8()
+  %r = icmp ule i8 %x, %t0 ; swapped
+  ret i1 %r
+}
+
+; Extra-use tests
+
+define i1 @t3_no_extrause(i8 %x, i8 %y) {
+; CHECK-LABEL: @t3_no_extrause(
+; CHECK-NEXT:    [[T0:%.*]] = xor i8 [[Y:%.*]], -1
+; CHECK-NEXT:    [[R:%.*]] = icmp ult i8 [[T0]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = xor i8 %y, -1
+  %r = icmp ult i8 %t0, %x
+  ret i1 %r
+}
+
+; Negative tests
+
+define i1 @n4_wrong_pred0(i8 %x, i8 %y) {
+; CHECK-LABEL: @n4_wrong_pred0(
+; CHECK-NEXT:    [[T0:%.*]] = xor i8 [[Y:%.*]], -1
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[R:%.*]] = icmp ule i8 [[T0]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = xor i8 %y, -1
+  call void @use8(i8 %t0)
+  %r = icmp ule i8 %t0, %x
+  ret i1 %r
+}
+
+define i1 @n5_wrong_pred1(i8 %x, i8 %y) {
+; CHECK-LABEL: @n5_wrong_pred1(
+; CHECK-NEXT:    [[T0:%.*]] = xor i8 [[Y:%.*]], -1
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[R:%.*]] = icmp ugt i8 [[T0]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = xor i8 %y, -1
+  call void @use8(i8 %t0)
+  %r = icmp ugt i8 %t0, %x
+  ret i1 %r
+}
+
+define i1 @n6_wrong_pred2(i8 %x, i8 %y) {
+; CHECK-LABEL: @n6_wrong_pred2(
+; CHECK-NEXT:    [[T0:%.*]] = xor i8 [[Y:%.*]], -1
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i8 [[T0]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = xor i8 %y, -1
+  call void @use8(i8 %t0)
+  %r = icmp eq i8 %t0, %x
+  ret i1 %r
+}
+
+define i1 @n7_wrong_pred3(i8 %x, i8 %y) {
+; CHECK-LABEL: @n7_wrong_pred3(
+; CHECK-NEXT:    [[T0:%.*]] = xor i8 [[Y:%.*]], -1
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i8 [[T0]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = xor i8 %y, -1
+  call void @use8(i8 %t0)
+  %r = icmp ne i8 %t0, %x
+  ret i1 %r
+}
+
+define i1 @n8_wrong_pred4(i8 %x, i8 %y) {
+; CHECK-LABEL: @n8_wrong_pred4(
+; CHECK-NEXT:    [[T0:%.*]] = xor i8 [[Y:%.*]], -1
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[R:%.*]] = icmp slt i8 [[T0]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = xor i8 %y, -1
+  call void @use8(i8 %t0)
+  %r = icmp slt i8 %t0, %x
+  ret i1 %r
+}
+
+define i1 @n9_wrong_pred5(i8 %x, i8 %y) {
+; CHECK-LABEL: @n9_wrong_pred5(
+; CHECK-NEXT:    [[T0:%.*]] = xor i8 [[Y:%.*]], -1
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[R:%.*]] = icmp sle i8 [[T0]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = xor i8 %y, -1
+  call void @use8(i8 %t0)
+  %r = icmp sle i8 %t0, %x
+  ret i1 %r
+}
+
+define i1 @n10_wrong_pred6(i8 %x, i8 %y) {
+; CHECK-LABEL: @n10_wrong_pred6(
+; CHECK-NEXT:    [[T0:%.*]] = xor i8 [[Y:%.*]], -1
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[R:%.*]] = icmp sgt i8 [[T0]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = xor i8 %y, -1
+  call void @use8(i8 %t0)
+  %r = icmp sgt i8 %t0, %x
+  ret i1 %r
+}
+
+define i1 @n11_wrong_pred7(i8 %x, i8 %y) {
+; CHECK-LABEL: @n11_wrong_pred7(
+; CHECK-NEXT:    [[T0:%.*]] = xor i8 [[Y:%.*]], -1
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[R:%.*]] = icmp sge i8 [[T0]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = xor i8 %y, -1
+  call void @use8(i8 %t0)
+  %r = icmp sge i8 %t0, %x
+  ret i1 %r
+}
+
+define <2 x i1> @n12_vec_nonsplat(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @n12_vec_nonsplat(
+; CHECK-NEXT:    [[T0:%.*]] = xor <2 x i8> [[Y:%.*]], <i8 -1, i8 -2>
+; CHECK-NEXT:    call void @use2x8(<2 x i8> [[T0]])
+; CHECK-NEXT:    [[R:%.*]] = icmp ult <2 x i8> [[T0]], [[X:%.*]]
+; CHECK-NEXT:    ret <2 x i1> [[R]]
+;
+  %t0 = xor <2 x i8> %y, <i8 -1, i8 -2> ; must be -1.
+  call void @use2x8(<2 x i8> %t0)
+  %r = icmp ult <2 x i8> %t0, %x
+  ret <2 x i1> %r
+}
diff --git a/llvm/test/Transforms/InstCombine/unsigned-add-overflow-check.ll b/llvm/test/Transforms/InstCombine/unsigned-add-overflow-check.ll
new file mode 100644
index 000000000..d1ab9ae41
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/unsigned-add-overflow-check.ll
@@ -0,0 +1,199 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instcombine -S | FileCheck %s
+
+; Should fold
+;   (%x + %y) u< %x
+; or
+;   (%x + %y) u< %y
+; to
+;   @llvm.uadd.with.overflow(%x, %y) + extractvalue
+
+define i1 @t0_basic(i8 %x, i8 %y) {
+; CHECK-LABEL: @t0_basic(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i8 [[Y:%.*]], -1
+; CHECK-NEXT:    [[R:%.*]] = icmp ult i8 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = add i8 %x, %y
+  %r = icmp ult i8 %t0, %y
+  ret i1 %r
+}
+
+define <2 x i1> @t1_vec(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @t1_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor <2 x i8> [[Y:%.*]], <i8 -1, i8 -1>
+; CHECK-NEXT:    [[R:%.*]] = icmp ult <2 x i8> [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    ret <2 x i1> [[R]]
+;
+  %t0 = add <2 x i8> %x, %y
+  %r = icmp ult <2 x i8> %t0, %y
+  ret <2 x i1> %r
+}
+
+; Commutativity
+
+define i1 @t2_symmetry(i8 %x, i8 %y) {
+; CHECK-LABEL: @t2_symmetry(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i8 [[X:%.*]], -1
+; CHECK-NEXT:    [[R:%.*]] = icmp ult i8 [[TMP1]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = add i8 %x, %y
+  %r = icmp ult i8 %t0, %x ; can check against either of `add` arguments
+  ret i1 %r
+}
+
+declare i8 @gen8()
+
+define i1 @t3_commutative(i8 %x) {
+; CHECK-LABEL: @t3_commutative(
+; CHECK-NEXT:    [[Y:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i8 [[Y]], -1
+; CHECK-NEXT:    [[R:%.*]] = icmp ult i8 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %y = call i8 @gen8()
+  %t0 = add i8 %y, %x ; swapped
+  %r = icmp ult i8 %t0, %y
+  ret i1 %r
+}
+
+define i1 @t4_commutative(i8 %x, i8 %y) {
+; CHECK-LABEL: @t4_commutative(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i8 [[Y:%.*]], -1
+; CHECK-NEXT:    [[R:%.*]] = icmp ult i8 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = add i8 %x, %y
+  %r = icmp ugt i8 %y, %t0 ; swapped
+  ret i1 %r
+}
+
+define i1 @t5_commutative(i8 %x) {
+; CHECK-LABEL: @t5_commutative(
+; CHECK-NEXT:    [[Y:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i8 [[Y]], -1
+; CHECK-NEXT:    [[R:%.*]] = icmp ult i8 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %y = call i8 @gen8()
+  %t0 = add i8 %y, %x ; swapped
+  %r = icmp ugt i8 %y, %t0 ; swapped
+  ret i1 %r
+}
+
+; Extra-use tests
+
+declare void @use8(i8)
+
+define i1 @t6_extrause(i8 %x, i8 %y) {
+; CHECK-LABEL: @t6_extrause(
+; CHECK-NEXT:    [[T0:%.*]] = add i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[R:%.*]] = icmp ult i8 [[T0]], [[Y]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = add i8 %x, %y
+  call void @use8(i8 %t0)
+  %r = icmp ult i8 %t0, %y
+  ret i1 %r
+}
+
+; Negative tests
+
+define i1 @n7_different_y(i8 %x, i8 %y0, i8 %y1) {
+; CHECK-LABEL: @n7_different_y(
+; CHECK-NEXT:    [[T0:%.*]] = add i8 [[X:%.*]], [[Y0:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp ult i8 [[T0]], [[Y1:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = add i8 %x, %y0
+  %r = icmp ult i8 %t0, %y1
+  ret i1 %r
+}
+
+define i1 @n8_wrong_pred0(i8 %x, i8 %y) {
+; CHECK-LABEL: @n8_wrong_pred0(
+; CHECK-NEXT:    [[T0:%.*]] = add i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp ule i8 [[T0]], [[Y]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = add i8 %x, %y
+  %r = icmp ule i8 %t0, %y
+  ret i1 %r
+}
+
+define i1 @n9_wrong_pred1(i8 %x, i8 %y) {
+; CHECK-LABEL: @n9_wrong_pred1(
+; CHECK-NEXT:    [[T0:%.*]] = add i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp ugt i8 [[T0]], [[Y]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = add i8 %x, %y
+  %r = icmp ugt i8 %t0, %y
+  ret i1 %r
+}
+
+define i1 @n10_wrong_pred2(i8 %x, i8 %y) {
+; CHECK-LABEL: @n10_wrong_pred2(
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i8 [[X:%.*]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = add i8 %x, %y
+  %r = icmp eq i8 %t0, %y
+  ret i1 %r
+}
+
+define i1 @n11_wrong_pred3(i8 %x, i8 %y) {
+; CHECK-LABEL: @n11_wrong_pred3(
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i8 [[X:%.*]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = add i8 %x, %y
+  %r = icmp ne i8 %t0, %y
+  ret i1 %r
+}
+
+define i1 @n12_wrong_pred4(i8 %x, i8 %y) {
+; CHECK-LABEL: @n12_wrong_pred4(
+; CHECK-NEXT:    [[T0:%.*]] = add i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp slt i8 [[T0]], [[Y]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = add i8 %x, %y
+  %r = icmp slt i8 %t0, %y
+  ret i1 %r
+}
+
+define i1 @n13_wrong_pred5(i8 %x, i8 %y) {
+; CHECK-LABEL: @n13_wrong_pred5(
+; CHECK-NEXT:    [[T0:%.*]] = add i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp sle i8 [[T0]], [[Y]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = add i8 %x, %y
+  %r = icmp sle i8 %t0, %y
+  ret i1 %r
+}
+
+define i1 @n14_wrong_pred6(i8 %x, i8 %y) {
+; CHECK-LABEL: @n14_wrong_pred6(
+; CHECK-NEXT:    [[T0:%.*]] = add i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp sgt i8 [[T0]], [[Y]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = add i8 %x, %y
+  %r = icmp sgt i8 %t0, %y
+  ret i1 %r
+}
+
+define i1 @n15_wrong_pred7(i8 %x, i8 %y) {
+; CHECK-LABEL: @n15_wrong_pred7(
+; CHECK-NEXT:    [[T0:%.*]] = add i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp sge i8 [[T0]], [[Y]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = add i8 %x, %y
+  %r = icmp sge i8 %t0, %y
+  ret i1 %r
+}
diff --git a/llvm/test/Transforms/InstCombine/unsigned-mul-lack-of-overflow-check-via-mul-udiv.ll b/llvm/test/Transforms/InstCombine/unsigned-mul-lack-of-overflow-check-via-mul-udiv.ll
new file mode 100644
index 000000000..faa4bedb2
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/unsigned-mul-lack-of-overflow-check-via-mul-udiv.ll
@@ -0,0 +1,173 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instcombine -S | FileCheck %s
+
+; Fold
+;   ((%x * %y) u/ %x) == %y
+; to
+;   @llvm.umul.with.overflow(%x, %y) + extractvalue + not
+
+define i1 @t0_basic(i8 %x, i8 %y) {
+; CHECK-LABEL: @t0_basic(
+; CHECK-NEXT:    [[UMUL:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    [[UMUL_OV:%.*]] = extractvalue { i8, i1 } [[UMUL]], 1
+; CHECK-NEXT:    [[UMUL_NOT_OV:%.*]] = xor i1 [[UMUL_OV]], true
+; CHECK-NEXT:    ret i1 [[UMUL_NOT_OV]]
+;
+  %t0 = mul i8 %x, %y
+  %t1 = udiv i8 %t0, %x
+  %r = icmp eq i8 %t1, %y
+  ret i1 %r
+}
+
+define <2 x i1> @t1_vec(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @t1_vec(
+; CHECK-NEXT:    [[UMUL:%.*]] = call { <2 x i8>, <2 x i1> } @llvm.umul.with.overflow.v2i8(<2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]])
+; CHECK-NEXT:    [[UMUL_OV:%.*]] = extractvalue { <2 x i8>, <2 x i1> } [[UMUL]], 1
+; CHECK-NEXT:    [[UMUL_NOT_OV:%.*]] = xor <2 x i1> [[UMUL_OV]], <i1 true, i1 true>
+; CHECK-NEXT:    ret <2 x i1> [[UMUL_NOT_OV]]
+;
+  %t0 = mul <2 x i8> %x, %y
+  %t1 = udiv <2 x i8> %t0, %x
+  %r = icmp eq <2 x i8> %t1, %y
+  ret <2 x i1> %r
+}
+
+declare i8 @gen8()
+
+define i1 @t2_commutative(i8 %x) {
+; CHECK-LABEL: @t2_commutative(
+; CHECK-NEXT:    [[Y:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[UMUL:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[X:%.*]], i8 [[Y]])
+; CHECK-NEXT:    [[UMUL_OV:%.*]] = extractvalue { i8, i1 } [[UMUL]], 1
+; CHECK-NEXT:    [[UMUL_NOT_OV:%.*]] = xor i1 [[UMUL_OV]], true
+; CHECK-NEXT:    ret i1 [[UMUL_NOT_OV]]
+;
+  %y = call i8 @gen8()
+  %t0 = mul i8 %y, %x ; swapped
+  %t1 = udiv i8 %t0, %x
+  %r = icmp eq i8 %t1, %y
+  ret i1 %r
+}
+
+define i1 @t3_commutative(i8 %x) {
+; CHECK-LABEL: @t3_commutative(
+; CHECK-NEXT:    [[Y:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[UMUL:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[X:%.*]], i8 [[Y]])
+; CHECK-NEXT:    [[UMUL_OV:%.*]] = extractvalue { i8, i1 } [[UMUL]], 1
+; CHECK-NEXT:    [[UMUL_NOT_OV:%.*]] = xor i1 [[UMUL_OV]], true
+; CHECK-NEXT:    ret i1 [[UMUL_NOT_OV]]
+;
+  %y = call i8 @gen8()
+  %t0 = mul i8 %y, %x ; swapped
+  %t1 = udiv i8 %t0, %x
+  %r = icmp eq i8 %t1, %y
+  ret i1 %r
+}
+
+define i1 @t4_commutative(i8 %x) {
+; CHECK-LABEL: @t4_commutative(
+; CHECK-NEXT:    [[Y:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[UMUL:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[X:%.*]], i8 [[Y]])
+; CHECK-NEXT:    [[UMUL_OV:%.*]] = extractvalue { i8, i1 } [[UMUL]], 1
+; CHECK-NEXT:    [[UMUL_NOT_OV:%.*]] = xor i1 [[UMUL_OV]], true
+; CHECK-NEXT:    ret i1 [[UMUL_NOT_OV]]
+;
+  %y = call i8 @gen8()
+  %t0 = mul i8 %y, %x ; swapped
+  %t1 = udiv i8 %t0, %x
+  %r = icmp eq i8 %y, %t1 ; swapped
+  ret i1 %r
+}
+
+; Extra-use tests
+
+declare void @use8(i8)
+
+define i1 @t5_extrause0(i8 %x, i8 %y) {
+; CHECK-LABEL: @t5_extrause0(
+; CHECK-NEXT:    [[UMUL:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    [[UMUL_VAL:%.*]] = extractvalue { i8, i1 } [[UMUL]], 0
+; CHECK-NEXT:    [[UMUL_OV:%.*]] = extractvalue { i8, i1 } [[UMUL]], 1
+; CHECK-NEXT:    [[UMUL_NOT_OV:%.*]] = xor i1 [[UMUL_OV]], true
+; CHECK-NEXT:    call void @use8(i8 [[UMUL_VAL]])
+; CHECK-NEXT:    ret i1 [[UMUL_NOT_OV]]
+;
+  %t0 = mul i8 %x, %y
+  call void @use8(i8 %t0)
+  %t1 = udiv i8 %t0, %x
+  %r = icmp eq i8 %t1, %y
+  ret i1 %r
+}
+
+define i1 @t6_extrause1(i8 %x, i8 %y) {
+; CHECK-LABEL: @t6_extrause1(
+; CHECK-NEXT:    [[T0:%.*]] = mul i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = udiv i8 [[T0]], [[X]]
+; CHECK-NEXT:    call void @use8(i8 [[T1]])
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i8 [[T1]], [[Y]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = mul i8 %x, %y
+  %t1 = udiv i8 %t0, %x
+  call void @use8(i8 %t1)
+  %r = icmp eq i8 %t1, %y
+  ret i1 %r
+}
+
+define i1 @t7_extrause2(i8 %x, i8 %y) {
+; CHECK-LABEL: @t7_extrause2(
+; CHECK-NEXT:    [[T0:%.*]] = mul i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = udiv i8 [[T0]], [[X]]
+; CHECK-NEXT:    call void @use8(i8 [[T1]])
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i8 [[T1]], [[Y]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = mul i8 %x, %y
+  call void @use8(i8 %t0)
+  %t1 = udiv i8 %t0, %x
+  call void @use8(i8 %t1)
+  %r = icmp eq i8 %t1, %y
+  ret i1 %r
+}
+
+; Negative tests
+
+define i1 @n8_different_x(i8 %x0, i8 %x1, i8 %y) {
+; CHECK-LABEL: @n8_different_x(
+; CHECK-NEXT:    [[T0:%.*]] = mul i8 [[X0:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = udiv i8 [[T0]], [[X1:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i8 [[T1]], [[Y]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = mul i8 %x0, %y
+  %t1 = udiv i8 %t0, %x1
+  %r = icmp eq i8 %t1, %y
+  ret i1 %r
+}
+
+define i1 @n9_different_y(i8 %x, i8 %y0, i8 %y1) {
+; CHECK-LABEL: @n9_different_y(
+; CHECK-NEXT:    [[T0:%.*]] = mul i8 [[X:%.*]], [[Y0:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = udiv i8 [[T0]], [[X]]
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i8 [[T1]], [[Y1:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = mul i8 %x, %y0
+  %t1 = udiv i8 %t0, %x
+  %r = icmp eq i8 %t1, %y1
+  ret i1 %r
+}
+
+define i1 @n10_wrong_pred(i8 %x, i8 %y) {
+; CHECK-LABEL: @n10_wrong_pred(
+; CHECK-NEXT:    [[T0:%.*]] = mul i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = udiv i8 [[T0]], [[X]]
+; CHECK-NEXT:    [[R:%.*]] = icmp ult i8 [[T1]], [[Y]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = mul i8 %x, %y
+  %t1 = udiv i8 %t0, %x
+  %r = icmp ult i8 %t1, %y
+  ret i1 %r
+}
diff --git a/llvm/test/Transforms/InstCombine/unsigned-mul-lack-of-overflow-check-via-udiv-of-allones.ll b/llvm/test/Transforms/InstCombine/unsigned-mul-lack-of-overflow-check-via-udiv-of-allones.ll
new file mode 100644
index 000000000..cfc7a01d7
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/unsigned-mul-lack-of-overflow-check-via-udiv-of-allones.ll
@@ -0,0 +1,109 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instcombine -S | FileCheck %s
+
+; Fold
+;   (-1 u/ %x) u>= %y
+; to
+;   @llvm.umul.with.overflow(%x, %y) + extractvalue + not
+
+define i1 @t0_basic(i8 %x, i8 %y) {
+; CHECK-LABEL: @t0_basic(
+; CHECK-NEXT:    [[UMUL:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    [[UMUL_OV:%.*]] = extractvalue { i8, i1 } [[UMUL]], 1
+; CHECK-NEXT:    [[UMUL_NOT_OV:%.*]] = xor i1 [[UMUL_OV]], true
+; CHECK-NEXT:    ret i1 [[UMUL_NOT_OV]]
+;
+  %t0 = udiv i8 -1, %x
+  %r = icmp uge i8 %t0, %y
+  ret i1 %r
+}
+
+define <2 x i1> @t1_vec(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @t1_vec(
+; CHECK-NEXT:    [[UMUL:%.*]] = call { <2 x i8>, <2 x i1> } @llvm.umul.with.overflow.v2i8(<2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]])
+; CHECK-NEXT:    [[UMUL_OV:%.*]] = extractvalue { <2 x i8>, <2 x i1> } [[UMUL]], 1
+; CHECK-NEXT:    [[UMUL_NOT_OV:%.*]] = xor <2 x i1> [[UMUL_OV]], <i1 true, i1 true>
+; CHECK-NEXT:    ret <2 x i1> [[UMUL_NOT_OV]]
+;
+  %t0 = udiv <2 x i8> <i8 -1, i8 -1>, %x
+  %r = icmp uge <2 x i8> %t0, %y
+  ret <2 x i1> %r
+}
+
+define <3 x i1> @t2_vec_undef(<3 x i8> %x, <3 x i8> %y) {
+; CHECK-LABEL: @t2_vec_undef(
+; CHECK-NEXT:    [[UMUL:%.*]] = call { <3 x i8>, <3 x i1> } @llvm.umul.with.overflow.v3i8(<3 x i8> [[X:%.*]], <3 x i8> [[Y:%.*]])
+; CHECK-NEXT:    [[UMUL_OV:%.*]] = extractvalue { <3 x i8>, <3 x i1> } [[UMUL]], 1
+; CHECK-NEXT:    [[UMUL_NOT_OV:%.*]] = xor <3 x i1> [[UMUL_OV]], <i1 true, i1 true, i1 true>
+; CHECK-NEXT:    ret <3 x i1> [[UMUL_NOT_OV]]
+;
+  %t0 = udiv <3 x i8> <i8 -1, i8 undef, i8 -1>, %x
+  %r = icmp uge <3 x i8> %t0, %y
+  ret <3 x i1> %r
+}
+
+declare i8 @gen8()
+
+define i1 @t3_commutative(i8 %x) {
+; CHECK-LABEL: @t3_commutative(
+; CHECK-NEXT:    [[Y:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[UMUL:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[X:%.*]], i8 [[Y]])
+; CHECK-NEXT:    [[UMUL_OV:%.*]] = extractvalue { i8, i1 } [[UMUL]], 1
+; CHECK-NEXT:    [[UMUL_NOT_OV:%.*]] = xor i1 [[UMUL_OV]], true
+; CHECK-NEXT:    ret i1 [[UMUL_NOT_OV]]
+;
+  %t0 = udiv i8 -1, %x
+  %y = call i8 @gen8()
+  %r = icmp ule i8 %y, %t0 ; swapped
+  ret i1 %r
+}
+
+; Negative tests
+
+declare void @use8(i8)
+
+define i1 @n4_extrause(i8 %x, i8 %y) {
+; CHECK-LABEL: @n4_extrause(
+; CHECK-NEXT:    [[T0:%.*]] = udiv i8 -1, [[X:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[R:%.*]] = icmp uge i8 [[T0]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = udiv i8 -1, %x
+  call void @use8(i8 %t0)
+  %r = icmp uge i8 %t0, %y
+  ret i1 %r
+}
+
+define i1 @n5_not_negone(i8 %x, i8 %y) {
+; CHECK-LABEL: @n5_not_negone(
+; CHECK-NEXT:    [[T0:%.*]] = udiv i8 -2, [[X:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp uge i8 [[T0]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = udiv i8 -2, %x ; not -1
+  %r = icmp uge i8 %t0, %y
+  ret i1 %r
+}
+
+define i1 @n6_wrong_pred0(i8 %x, i8 %y) {
+; CHECK-LABEL: @n6_wrong_pred0(
+; CHECK-NEXT:    [[T0:%.*]] = udiv i8 -1, [[X:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp ule i8 [[T0]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = udiv i8 -1, %x
+  %r = icmp ule i8 %t0, %y ; not uge
+  ret i1 %r
+}
+
+define i1 @n6_wrong_pred1(i8 %x, i8 %y) {
+; CHECK-LABEL: @n6_wrong_pred1(
+; CHECK-NEXT:    [[T0:%.*]] = udiv i8 -1, [[X:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp ugt i8 [[T0]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = udiv i8 -1, %x
+  %r = icmp ugt i8 %t0, %y ; not uge
+  ret i1 %r
+}
diff --git a/llvm/test/Transforms/InstCombine/unsigned-mul-overflow-check-via-mul-udiv.ll b/llvm/test/Transforms/InstCombine/unsigned-mul-overflow-check-via-mul-udiv.ll
new file mode 100644
index 000000000..a1c09f567
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/unsigned-mul-overflow-check-via-mul-udiv.ll
@@ -0,0 +1,167 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instcombine -S | FileCheck %s
+
+; Fold
+;   ((%x * %y) u/ %x) != %y
+; to
+;   @llvm.umul.with.overflow(%x, %y) + extractvalue
+
+define i1 @t0_basic(i8 %x, i8 %y) {
+; CHECK-LABEL: @t0_basic(
+; CHECK-NEXT:    [[UMUL:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    [[UMUL_OV:%.*]] = extractvalue { i8, i1 } [[UMUL]], 1
+; CHECK-NEXT:    ret i1 [[UMUL_OV]]
+;
+  %t0 = mul i8 %x, %y
+  %t1 = udiv i8 %t0, %x
+  %r = icmp ne i8 %t1, %y
+  ret i1 %r
+}
+
+define <2 x i1> @t1_vec(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @t1_vec(
+; CHECK-NEXT:    [[UMUL:%.*]] = call { <2 x i8>, <2 x i1> } @llvm.umul.with.overflow.v2i8(<2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]])
+; CHECK-NEXT:    [[UMUL_OV:%.*]] = extractvalue { <2 x i8>, <2 x i1> } [[UMUL]], 1
+; CHECK-NEXT:    ret <2 x i1> [[UMUL_OV]]
+;
+  %t0 = mul <2 x i8> %x, %y
+  %t1 = udiv <2 x i8> %t0, %x
+  %r = icmp ne <2 x i8> %t1, %y
+  ret <2 x i1> %r
+}
+
+declare i8 @gen8()
+
+define i1 @t2_commutative(i8 %x) {
+; CHECK-LABEL: @t2_commutative(
+; CHECK-NEXT:    [[Y:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[UMUL:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[X:%.*]], i8 [[Y]])
+; CHECK-NEXT:    [[UMUL_OV:%.*]] = extractvalue { i8, i1 } [[UMUL]], 1
+; CHECK-NEXT:    ret i1 [[UMUL_OV]]
+;
+  %y = call i8 @gen8()
+  %t0 = mul i8 %y, %x ; swapped
+  %t1 = udiv i8 %t0, %x
+  %r = icmp ne i8 %t1, %y
+  ret i1 %r
+}
+
+define i1 @t3_commutative(i8 %x) {
+; CHECK-LABEL: @t3_commutative(
+; CHECK-NEXT:    [[Y:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[UMUL:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[X:%.*]], i8 [[Y]])
+; CHECK-NEXT:    [[UMUL_OV:%.*]] = extractvalue { i8, i1 } [[UMUL]], 1
+; CHECK-NEXT:    ret i1 [[UMUL_OV]]
+;
+  %y = call i8 @gen8()
+  %t0 = mul i8 %y, %x ; swapped
+  %t1 = udiv i8 %t0, %x
+  %r = icmp ne i8 %t1, %y
+  ret i1 %r
+}
+
+define i1 @t4_commutative(i8 %x) {
+; CHECK-LABEL: @t4_commutative(
+; CHECK-NEXT:    [[Y:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[UMUL:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[X:%.*]], i8 [[Y]])
+; CHECK-NEXT:    [[UMUL_OV:%.*]] = extractvalue { i8, i1 } [[UMUL]], 1
+; CHECK-NEXT:    ret i1 [[UMUL_OV]]
+;
+  %y = call i8 @gen8()
+  %t0 = mul i8 %y, %x ; swapped
+  %t1 = udiv i8 %t0, %x
+  %r = icmp ne i8 %y, %t1 ; swapped
+  ret i1 %r
+}
+
+; Extra-use tests
+
+declare void @use8(i8)
+
+define i1 @t5_extrause0(i8 %x, i8 %y) {
+; CHECK-LABEL: @t5_extrause0(
+; CHECK-NEXT:    [[UMUL:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    [[UMUL_VAL:%.*]] = extractvalue { i8, i1 } [[UMUL]], 0
+; CHECK-NEXT:    [[UMUL_OV:%.*]] = extractvalue { i8, i1 } [[UMUL]], 1
+; CHECK-NEXT:    call void @use8(i8 [[UMUL_VAL]])
+; CHECK-NEXT:    ret i1 [[UMUL_OV]]
+;
+  %t0 = mul i8 %x, %y
+  call void @use8(i8 %t0)
+  %t1 = udiv i8 %t0, %x
+  %r = icmp ne i8 %t1, %y
+  ret i1 %r
+}
+
+define i1 @t6_extrause1(i8 %x, i8 %y) {
+; CHECK-LABEL: @t6_extrause1(
+; CHECK-NEXT:    [[T0:%.*]] = mul i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = udiv i8 [[T0]], [[X]]
+; CHECK-NEXT:    call void @use8(i8 [[T1]])
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i8 [[T1]], [[Y]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = mul i8 %x, %y
+  %t1 = udiv i8 %t0, %x
+  call void @use8(i8 %t1)
+  %r = icmp ne i8 %t1, %y
+  ret i1 %r
+}
+
+define i1 @t7_extrause2(i8 %x, i8 %y) {
+; CHECK-LABEL: @t7_extrause2(
+; CHECK-NEXT:    [[T0:%.*]] = mul i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = udiv i8 [[T0]], [[X]]
+; CHECK-NEXT:    call void @use8(i8 [[T1]])
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i8 [[T1]], [[Y]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = mul i8 %x, %y
+  call void @use8(i8 %t0)
+  %t1 = udiv i8 %t0, %x
+  call void @use8(i8 %t1)
+  %r = icmp ne i8 %t1, %y
+  ret i1 %r
+}
+
+; Negative tests
+
+define i1 @n8_different_x(i8 %x0, i8 %x1, i8 %y) {
+; CHECK-LABEL: @n8_different_x(
+; CHECK-NEXT:    [[T0:%.*]] = mul i8 [[X0:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = udiv i8 [[T0]], [[X1:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i8 [[T1]], [[Y]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = mul i8 %x0, %y
+  %t1 = udiv i8 %t0, %x1
+  %r = icmp ne i8 %t1, %y
+  ret i1 %r
+}
+
+define i1 @n9_different_y(i8 %x, i8 %y0, i8 %y1) {
+; CHECK-LABEL: @n9_different_y(
+; CHECK-NEXT:    [[T0:%.*]] = mul i8 [[X:%.*]], [[Y0:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = udiv i8 [[T0]], [[X]]
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i8 [[T1]], [[Y1:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = mul i8 %x, %y0
+  %t1 = udiv i8 %t0, %x
+  %r = icmp ne i8 %t1, %y1
+  ret i1 %r
+}
+
+define i1 @n10_wrong_pred(i8 %x, i8 %y) {
+; CHECK-LABEL: @n10_wrong_pred(
+; CHECK-NEXT:    [[T0:%.*]] = mul i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = udiv i8 [[T0]], [[X]]
+; CHECK-NEXT:    [[R:%.*]] = icmp ugt i8 [[T1]], [[Y]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = mul i8 %x, %y
+  %t1 = udiv i8 %t0, %x
+  %r = icmp ugt i8 %t1, %y
+  ret i1 %r
+}
diff --git a/llvm/test/Transforms/InstCombine/unsigned-mul-overflow-check-via-udiv-of-allones.ll b/llvm/test/Transforms/InstCombine/unsigned-mul-overflow-check-via-udiv-of-allones.ll
new file mode 100644
index 000000000..f4ab3c23b
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/unsigned-mul-overflow-check-via-udiv-of-allones.ll
@@ -0,0 +1,105 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instcombine -S | FileCheck %s
+
+; Fold
+;   (-1 u/ %x) u< %y
+; to
+;   @llvm.umul.with.overflow(%x, %y) + extractvalue
+
+define i1 @t0_basic(i8 %x, i8 %y) {
+; CHECK-LABEL: @t0_basic(
+; CHECK-NEXT:    [[UMUL:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    [[UMUL_OV:%.*]] = extractvalue { i8, i1 } [[UMUL]], 1
+; CHECK-NEXT:    ret i1 [[UMUL_OV]]
+;
+  %t0 = udiv i8 -1, %x
+  %r = icmp ult i8 %t0, %y
+  ret i1 %r
+}
+
+define <2 x i1> @t1_vec(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @t1_vec(
+; CHECK-NEXT:    [[UMUL:%.*]] = call { <2 x i8>, <2 x i1> } @llvm.umul.with.overflow.v2i8(<2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]])
+; CHECK-NEXT:    [[UMUL_OV:%.*]] = extractvalue { <2 x i8>, <2 x i1> } [[UMUL]], 1
+; CHECK-NEXT:    ret <2 x i1> [[UMUL_OV]]
+;
+  %t0 = udiv <2 x i8> <i8 -1, i8 -1>, %x
+  %r = icmp ult <2 x i8> %t0, %y
+  ret <2 x i1> %r
+}
+
+define <3 x i1> @t2_vec_undef(<3 x i8> %x, <3 x i8> %y) {
+; CHECK-LABEL: @t2_vec_undef(
+; CHECK-NEXT:    [[UMUL:%.*]] = call { <3 x i8>, <3 x i1> } @llvm.umul.with.overflow.v3i8(<3 x i8> [[X:%.*]], <3 x i8> [[Y:%.*]])
+; CHECK-NEXT:    [[UMUL_OV:%.*]] = extractvalue { <3 x i8>, <3 x i1> } [[UMUL]], 1
+; CHECK-NEXT:    ret <3 x i1> [[UMUL_OV]]
+;
+  %t0 = udiv <3 x i8> <i8 -1, i8 undef, i8 -1>, %x
+  %r = icmp ult <3 x i8> %t0, %y
+  ret <3 x i1> %r
+}
+
+declare i8 @gen8()
+
+define i1 @t3_commutative(i8 %x) {
+; CHECK-LABEL: @t3_commutative(
+; CHECK-NEXT:    [[Y:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[UMUL:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[X:%.*]], i8 [[Y]])
+; CHECK-NEXT:    [[UMUL_OV:%.*]] = extractvalue { i8, i1 } [[UMUL]], 1
+; CHECK-NEXT:    ret i1 [[UMUL_OV]]
+;
+  %t0 = udiv i8 -1, %x
+  %y = call i8 @gen8()
+  %r = icmp ugt i8 %y, %t0 ; swapped
+  ret i1 %r
+}
+
+; Negative tests
+
+declare void @use8(i8)
+
+define i1 @n4_extrause(i8 %x, i8 %y) {
+; CHECK-LABEL: @n4_extrause(
+; CHECK-NEXT:    [[T0:%.*]] = udiv i8 -1, [[X:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[R:%.*]] = icmp ult i8 [[T0]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = udiv i8 -1, %x
+  call void @use8(i8 %t0)
+  %r = icmp ult i8 %t0, %y
+  ret i1 %r
+}
+
+define i1 @n5_not_negone(i8 %x, i8 %y) {
+; CHECK-LABEL: @n5_not_negone(
+; CHECK-NEXT:    [[T0:%.*]] = udiv i8 -2, [[X:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp ult i8 [[T0]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = udiv i8 -2, %x ; not -1
+  %r = icmp ult i8 %t0, %y
+  ret i1 %r
+}
+
+define i1 @n6_wrong_pred0(i8 %x, i8 %y) {
+; CHECK-LABEL: @n6_wrong_pred0(
+; CHECK-NEXT:    [[T0:%.*]] = udiv i8 -1, [[X:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp ule i8 [[T0]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = udiv i8 -1, %x
+  %r = icmp ule i8 %t0, %y ; not ult
+  ret i1 %r
+}
+
+define i1 @n6_wrong_pred1(i8 %x, i8 %y) {
+; CHECK-LABEL: @n6_wrong_pred1(
+; CHECK-NEXT:    [[T0:%.*]] = udiv i8 -1, [[X:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp ugt i8 [[T0]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = udiv i8 -1, %x
+  %r = icmp ugt i8 %t0, %y ; not ult
+  ret i1 %r
+}
diff --git a/llvm/test/Transforms/InstCombine/unsigned-sub-lack-of-overflow-check.ll b/llvm/test/Transforms/InstCombine/unsigned-sub-lack-of-overflow-check.ll
new file mode 100644
index 000000000..d048841c9
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/unsigned-sub-lack-of-overflow-check.ll
@@ -0,0 +1,155 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instcombine -S | FileCheck %s
+
+; Fold
+;   (%x - %y) u<= %x
+; to
+;   @llvm.sub.with.overflow(%x, %y) + extractvalue + not
+
+define i1 @t0_basic(i8 %x, i8 %y) {
+; CHECK-LABEL: @t0_basic(
+; CHECK-NEXT:    [[R:%.*]] = icmp ule i8 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = sub i8 %x, %y
+  %r = icmp ule i8 %t0, %x
+  ret i1 %r
+}
+
+define <2 x i1> @t1_vec(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @t1_vec(
+; CHECK-NEXT:    [[R:%.*]] = icmp ule <2 x i8> [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    ret <2 x i1> [[R]]
+;
+  %t0 = sub <2 x i8> %x, %y
+  %r = icmp ule <2 x i8> %t0, %x
+  ret <2 x i1> %r
+}
+
+; Commutativity
+
+define i1 @t2_commutative(i8 %x, i8 %y) {
+; CHECK-LABEL: @t2_commutative(
+; CHECK-NEXT:    [[R:%.*]] = icmp ule i8 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = sub i8 %x, %y
+  %r = icmp uge i8 %x, %t0 ; swapped
+  ret i1 %r
+}
+
+; Extra-use tests
+
+declare void @use8(i8)
+
+define i1 @t3_extrause0(i8 %x, i8 %y) {
+; CHECK-LABEL: @t3_extrause0(
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[R:%.*]] = icmp uge i8 [[X]], [[Y]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = sub i8 %x, %y
+  call void @use8(i8 %t0)
+  %r = icmp ule i8 %t0, %x
+  ret i1 %r
+}
+
+; Negative tests
+
+define i1 @n4_not_commutative(i8 %x, i8 %y) {
+; CHECK-LABEL: @n4_not_commutative(
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp ule i8 [[T0]], [[Y]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = sub i8 %x, %y
+  %r = icmp ule i8 %t0, %y ; can't check against %y.
+  ret i1 %r
+}
+
+define i1 @n5_wrong_pred0(i8 %x, i8 %y) {
+; CHECK-LABEL: @n5_wrong_pred0(
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp ult i8 [[T0]], [[X]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = sub i8 %x, %y
+  %r = icmp ult i8 %t0, %x
+  ret i1 %r
+}
+
+define i1 @n6_wrong_pred1(i8 %x, i8 %y) {
+; CHECK-LABEL: @n6_wrong_pred1(
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp uge i8 [[T0]], [[X]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = sub i8 %x, %y
+  %r = icmp uge i8 %t0, %x
+  ret i1 %r
+}
+
+define i1 @n7_wrong_pred2(i8 %x, i8 %y) {
+; CHECK-LABEL: @n7_wrong_pred2(
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i8 [[Y:%.*]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = sub i8 %x, %y
+  %r = icmp eq i8 %t0, %x
+  ret i1 %r
+}
+
+define i1 @n8_wrong_pred3(i8 %x, i8 %y) {
+; CHECK-LABEL: @n8_wrong_pred3(
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i8 [[Y:%.*]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = sub i8 %x, %y
+  %r = icmp ne i8 %t0, %x
+  ret i1 %r
+}
+
+define i1 @n9_wrong_pred4(i8 %x, i8 %y) {
+; CHECK-LABEL: @n9_wrong_pred4(
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp slt i8 [[T0]], [[X]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = sub i8 %x, %y
+  %r = icmp slt i8 %t0, %x
+  ret i1 %r
+}
+
+define i1 @n10_wrong_pred5(i8 %x, i8 %y) {
+; CHECK-LABEL: @n10_wrong_pred5(
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp sle i8 [[T0]], [[X]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = sub i8 %x, %y
+  %r = icmp sle i8 %t0, %x
+  ret i1 %r
+}
+
+define i1 @n11_wrong_pred6(i8 %x, i8 %y) {
+; CHECK-LABEL: @n11_wrong_pred6(
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp sgt i8 [[T0]], [[X]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = sub i8 %x, %y
+  %r = icmp sgt i8 %t0, %x
+  ret i1 %r
+}
+
+define i1 @n12_wrong_pred7(i8 %x, i8 %y) {
+; CHECK-LABEL: @n12_wrong_pred7(
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp sge i8 [[T0]], [[X]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = sub i8 %x, %y
+  %r = icmp sge i8 %t0, %x
+  ret i1 %r
+}
diff --git a/llvm/test/Transforms/InstCombine/unsigned-sub-overflow-check.ll b/llvm/test/Transforms/InstCombine/unsigned-sub-overflow-check.ll
new file mode 100644
index 000000000..124150cac
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/unsigned-sub-overflow-check.ll
@@ -0,0 +1,155 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instcombine -S | FileCheck %s
+
+; Fold
+;   (%x - %y) u> %x
+; to
+;   @llvm.sub.with.overflow(%x, %y) + extractvalue
+
+define i1 @t0_basic(i8 %x, i8 %y) {
+; CHECK-LABEL: @t0_basic(
+; CHECK-NEXT:    [[R:%.*]] = icmp ugt i8 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = sub i8 %x, %y
+  %r = icmp ugt i8 %t0, %x
+  ret i1 %r
+}
+
+define <2 x i1> @t1_vec(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @t1_vec(
+; CHECK-NEXT:    [[R:%.*]] = icmp ugt <2 x i8> [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    ret <2 x i1> [[R]]
+;
+  %t0 = sub <2 x i8> %x, %y
+  %r = icmp ugt <2 x i8> %t0, %x
+  ret <2 x i1> %r
+}
+
+; Commutativity
+
+define i1 @t2_commutative(i8 %x, i8 %y) {
+; CHECK-LABEL: @t2_commutative(
+; CHECK-NEXT:    [[R:%.*]] = icmp ugt i8 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = sub i8 %x, %y
+  %r = icmp ult i8 %x, %t0 ; swapped
+  ret i1 %r
+}
+
+; Extra-use tests
+
+declare void @use8(i8)
+
+define i1 @t3_extrause0(i8 %x, i8 %y) {
+; CHECK-LABEL: @t3_extrause0(
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[R:%.*]] = icmp ult i8 [[X]], [[Y]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = sub i8 %x, %y
+  call void @use8(i8 %t0)
+  %r = icmp ugt i8 %t0, %x
+  ret i1 %r
+}
+
+; Negative tests
+
+define i1 @n4_not_commutative(i8 %x, i8 %y) {
+; CHECK-LABEL: @n4_not_commutative(
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp ugt i8 [[T0]], [[Y]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = sub i8 %x, %y
+  %r = icmp ugt i8 %t0, %y ; can't check against %y.
+  ret i1 %r
+}
+
+define i1 @n5_wrong_pred0(i8 %x, i8 %y) {
+; CHECK-LABEL: @n5_wrong_pred0(
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp uge i8 [[T0]], [[X]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = sub i8 %x, %y
+  %r = icmp uge i8 %t0, %x
+  ret i1 %r
+}
+
+define i1 @n6_wrong_pred1(i8 %x, i8 %y) {
+; CHECK-LABEL: @n6_wrong_pred1(
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp ult i8 [[T0]], [[X]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = sub i8 %x, %y
+  %r = icmp ult i8 %t0, %x
+  ret i1 %r
+}
+
+define i1 @n7_wrong_pred2(i8 %x, i8 %y) {
+; CHECK-LABEL: @n7_wrong_pred2(
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i8 [[Y:%.*]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = sub i8 %x, %y
+  %r = icmp eq i8 %t0, %x
+  ret i1 %r
+}
+
+define i1 @n8_wrong_pred3(i8 %x, i8 %y) {
+; CHECK-LABEL: @n8_wrong_pred3(
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i8 [[Y:%.*]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = sub i8 %x, %y
+  %r = icmp ne i8 %t0, %x
+  ret i1 %r
+}
+
+define i1 @n9_wrong_pred4(i8 %x, i8 %y) {
+; CHECK-LABEL: @n9_wrong_pred4(
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp slt i8 [[T0]], [[X]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = sub i8 %x, %y
+  %r = icmp slt i8 %t0, %x
+  ret i1 %r
+}
+
+define i1 @n10_wrong_pred5(i8 %x, i8 %y) {
+; CHECK-LABEL: @n10_wrong_pred5(
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp sle i8 [[T0]], [[X]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = sub i8 %x, %y
+  %r = icmp sle i8 %t0, %x
+  ret i1 %r
+}
+
+define i1 @n11_wrong_pred6(i8 %x, i8 %y) {
+; CHECK-LABEL: @n11_wrong_pred6(
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp sgt i8 [[T0]], [[X]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = sub i8 %x, %y
+  %r = icmp sgt i8 %t0, %x
+  ret i1 %r
+}
+
+define i1 @n12_wrong_pred7(i8 %x, i8 %y) {
+; CHECK-LABEL: @n12_wrong_pred7(
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp sge i8 [[T0]], [[X]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = sub i8 %x, %y
+  %r = icmp sge i8 %t0, %x
+  ret i1 %r
+}
diff --git a/llvm/test/Transforms/InstCombine/unsigned_saturated_sub.ll b/llvm/test/Transforms/InstCombine/unsigned_saturated_sub.ll
index 44aa7deb4..60da66b7a 100644
--- a/llvm/test/Transforms/InstCombine/unsigned_saturated_sub.ll
+++ b/llvm/test/Transforms/InstCombine/unsigned_saturated_sub.ll
@@ -5,6 +5,8 @@
 ; usub.sat() intrinsics is tested here.
 
 declare void @use(i64)
+declare void @usei32(i32)
+declare void @usei1(i1)
 
 ; (a > b) ? a - b : 0 -> usub.sat(a, b)
 
@@ -32,6 +34,51 @@ define i64 @max_sub_uge(i64 %a, i64 %b) {
   ret i64 %sel
 }
 
+define i64 @max_sub_uge_extrause1(i64 %a, i64 %b) {
+; CHECK-LABEL: @max_sub_uge_extrause1(
+; CHECK-NEXT:    [[SUB:%.*]] = sub i64 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[A]], i64 [[B]])
+; CHECK-NEXT:    call void @use(i64 [[SUB]])
+; CHECK-NEXT:    ret i64 [[TMP1]]
+;
+  %cmp = icmp uge i64 %a, %b
+  %sub = sub i64 %a, %b
+  %sel = select i1 %cmp, i64 %sub ,i64 0
+  call void @use(i64 %sub)
+  ret i64 %sel
+}
+
+define i64 @max_sub_uge_extrause2(i64 %a, i64 %b) {
+; CHECK-LABEL: @max_sub_uge_extrause2(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp uge i64 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[A]], i64 [[B]])
+; CHECK-NEXT:    call void @usei1(i1 [[CMP]])
+; CHECK-NEXT:    ret i64 [[TMP1]]
+;
+  %cmp = icmp uge i64 %a, %b
+  %sub = sub i64 %a, %b
+  %sel = select i1 %cmp, i64 %sub ,i64 0
+  call void @usei1(i1 %cmp)
+  ret i64 %sel
+}
+
+define i64 @max_sub_uge_extrause3(i64 %a, i64 %b) {
+; CHECK-LABEL: @max_sub_uge_extrause3(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp uge i64 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[SUB:%.*]] = sub i64 [[A]], [[B]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[A]], i64 [[B]])
+; CHECK-NEXT:    call void @use(i64 [[SUB]])
+; CHECK-NEXT:    call void @usei1(i1 [[CMP]])
+; CHECK-NEXT:    ret i64 [[TMP1]]
+;
+  %cmp = icmp uge i64 %a, %b
+  %sub = sub i64 %a, %b
+  %sel = select i1 %cmp, i64 %sub ,i64 0
+  call void @use(i64 %sub)
+  call void @usei1(i1 %cmp)
+  ret i64 %sel
+}
+
 ; Again, with vectors:
 ; (a > b) ? a - b : 0 -> usub.sat(a, b)
 
@@ -140,6 +187,53 @@ define i64 @neg_max_sub_ugt_sel_swapped(i64 %a, i64 %b) {
   ret i64 %sel
 }
 
+define i64 @neg_max_sub_ugt_sel_swapped_extrause1(i64 %a, i64 %b) {
+; CHECK-LABEL: @neg_max_sub_ugt_sel_swapped_extrause1(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[B:%.*]], [[A:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[A]], i64 [[B]])
+; CHECK-NEXT:    [[TMP2:%.*]] = sub i64 0, [[TMP1]]
+; CHECK-NEXT:    call void @usei1(i1 [[CMP]])
+; CHECK-NEXT:    ret i64 [[TMP2]]
+;
+  %cmp = icmp ugt i64 %b, %a
+  %sub = sub i64 %b, %a
+  %sel = select i1 %cmp, i64 0 ,i64 %sub
+  call void @usei1(i1 %cmp)
+  ret i64 %sel
+}
+
+define i64 @neg_max_sub_ugt_sel_swapped_extrause2(i64 %a, i64 %b) {
+; CHECK-LABEL: @neg_max_sub_ugt_sel_swapped_extrause2(
+; CHECK-NEXT:    [[SUB:%.*]] = sub i64 [[B:%.*]], [[A:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[A]], i64 [[B]])
+; CHECK-NEXT:    [[TMP2:%.*]] = sub i64 0, [[TMP1]]
+; CHECK-NEXT:    call void @use(i64 [[SUB]])
+; CHECK-NEXT:    ret i64 [[TMP2]]
+;
+  %cmp = icmp ugt i64 %b, %a
+  %sub = sub i64 %b, %a
+  %sel = select i1 %cmp, i64 0 ,i64 %sub
+  call void @use(i64 %sub)
+  ret i64 %sel
+}
+
+define i64 @neg_max_sub_ugt_sel_swapped_extrause3(i64 %a, i64 %b) {
+; CHECK-LABEL: @neg_max_sub_ugt_sel_swapped_extrause3(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[B:%.*]], [[A:%.*]]
+; CHECK-NEXT:    [[SUB:%.*]] = sub i64 [[B]], [[A]]
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i64 0, i64 [[SUB]]
+; CHECK-NEXT:    call void @use(i64 [[SUB]])
+; CHECK-NEXT:    call void @usei1(i1 [[CMP]])
+; CHECK-NEXT:    ret i64 [[SEL]]
+;
+  %cmp = icmp ugt i64 %b, %a
+  %sub = sub i64 %b, %a
+  %sel = select i1 %cmp, i64 0 ,i64 %sub
+  call void @use(i64 %sub)
+  call void @usei1(i1 %cmp)
+  ret i64 %sel
+}
+
 ; ((a < b) ? 0 : b - a) -> -usub.sat(a, b)
 
 define i64 @neg_max_sub_ult_sel_swapped(i64 %a, i64 %b) {
@@ -158,3 +252,202 @@ define i64 @neg_max_sub_ult_sel_swapped(i64 %a, i64 %b) {
   ret i64 %sel
 }
 
+define i32 @max_sub_ugt_c1(i32 %a) {
+; CHECK-LABEL: @max_sub_ugt_c1(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.usub.sat.i32(i32 [[A:%.*]], i32 1)
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %cmp = icmp ugt i32 %a, 1
+  %sub = add i32 %a, -1
+  %sel = select i1 %cmp, i32 %sub ,i32 0
+  ret i32 %sel
+}
+
+define i32 @max_sub_ugt_c01(i32 %a) {
+; CHECK-LABEL: @max_sub_ugt_c01(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[A:%.*]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = add i32 [[A]], -1
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 0, i32 [[SUB]]
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %cmp = icmp ugt i32 %a, 0
+  %sub = add i32 %a, -1
+  %sel = select i1 %cmp, i32 %sub ,i32 0
+  ret i32 %sel
+}
+
+define i32 @max_sub_ugt_c10(i32 %a) {
+; CHECK-LABEL: @max_sub_ugt_c10(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.usub.sat.i32(i32 [[A:%.*]], i32 10)
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %cmp = icmp ugt i32 %a, 10
+  %sub = add i32 %a, -10
+  %sel = select i1 %cmp, i32 %sub, i32 0
+  ret i32 %sel
+}
+
+define i32 @max_sub_ugt_c910(i32 %a) {
+; CHECK-LABEL: @max_sub_ugt_c910(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[A:%.*]], 9
+; CHECK-NEXT:    [[SUB:%.*]] = add i32 [[A]], -10
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 [[SUB]], i32 0
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %cmp = icmp ugt i32 %a, 9
+  %sub = add i32 %a, -10
+  %sel = select i1 %cmp, i32 %sub, i32 0
+  ret i32 %sel
+}
+
+define i32 @max_sub_ugt_c1110(i32 %a) {
+; CHECK-LABEL: @max_sub_ugt_c1110(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[A:%.*]], 11
+; CHECK-NEXT:    [[SUB:%.*]] = add i32 [[A]], -10
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 [[SUB]], i32 0
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %cmp = icmp ugt i32 %a, 11
+  %sub = add i32 %a, -10
+  %sel = select i1 %cmp, i32 %sub, i32 0
+  ret i32 %sel
+}
+
+define i32 @max_sub_ugt_c0(i32 %a) {
+; CHECK-LABEL: @max_sub_ugt_c0(
+; CHECK-NEXT:    ret i32 0
+;
+  %cmp = icmp ugt i32 %a, -1
+  %sub = add i32 %a, 0
+  %sel = select i1 %cmp, i32 %sub, i32 0
+  ret i32 %sel
+}
+
+define i32 @max_sub_ugt_cmiss(i32 %a) {
+; CHECK-LABEL: @max_sub_ugt_cmiss(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[A:%.*]], 1
+; CHECK-NEXT:    [[SUB:%.*]] = add i32 [[A]], -2
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 [[SUB]], i32 0
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %cmp = icmp ugt i32 %a, 1
+  %sub = add i32 %a, -2
+  %sel = select i1 %cmp, i32 %sub, i32 0
+  ret i32 %sel
+}
+
+define i32 @max_sub_ult_c1(i32 %a) {
+; CHECK-LABEL: @max_sub_ult_c1(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[A:%.*]], 0
+; CHECK-NEXT:    [[SEL:%.*]] = sext i1 [[CMP]] to i32
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %cmp = icmp ult i32 %a, 1
+  %sub = add i32 %a, -1
+  %sel = select i1 %cmp, i32 %sub, i32 0
+  ret i32 %sel
+}
+
+define i32 @max_sub_ult_c2(i32 %a) {
+; CHECK-LABEL: @max_sub_ult_c2(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.usub.sat.i32(i32 2, i32 [[A:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = sub nsw i32 0, [[TMP1]]
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
+  %cmp = icmp ult i32 %a, 2
+  %sub = add i32 %a, -2
+  %sel = select i1 %cmp, i32 %sub, i32 0
+  ret i32 %sel
+}
+
+define i32 @max_sub_ult_c2_oneuseicmp(i32 %a) {
+; CHECK-LABEL: @max_sub_ult_c2_oneuseicmp(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[A:%.*]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.usub.sat.i32(i32 2, i32 [[A]])
+; CHECK-NEXT:    [[TMP2:%.*]] = sub nsw i32 0, [[TMP1]]
+; CHECK-NEXT:    call void @usei1(i1 [[CMP]])
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
+  %cmp = icmp ult i32 %a, 2
+  %sub = add i32 %a, -2
+  %sel = select i1 %cmp, i32 %sub, i32 0
+  call void @usei1(i1 %cmp)
+  ret i32 %sel
+}
+
+define i32 @max_sub_ult_c2_oneusesub(i32 %a) {
+; CHECK-LABEL: @max_sub_ult_c2_oneusesub(
+; CHECK-NEXT:    [[SUB:%.*]] = add i32 [[A:%.*]], -2
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.usub.sat.i32(i32 2, i32 [[A]])
+; CHECK-NEXT:    [[TMP2:%.*]] = sub nsw i32 0, [[TMP1]]
+; CHECK-NEXT:    call void @usei32(i32 [[SUB]])
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
+  %cmp = icmp ult i32 %a, 2
+  %sub = add i32 %a, -2
+  %sel = select i1 %cmp, i32 %sub, i32 0
+  call void @usei32(i32 %sub)
+  ret i32 %sel
+}
+
+define i32 @max_sub_ult_c32(i32 %a) {
+; CHECK-LABEL: @max_sub_ult_c32(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[A:%.*]], 3
+; CHECK-NEXT:    [[SUB:%.*]] = add i32 [[A]], -2
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 [[SUB]], i32 0
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %cmp = icmp ult i32 %a, 3
+  %sub = add i32 %a, -2
+  %sel = select i1 %cmp, i32 %sub, i32 0
+  ret i32 %sel
+}
+
+define i32 @max_sub_ugt_c32(i32 %a) {
+; CHECK-LABEL: @max_sub_ugt_c32(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[A:%.*]], 3
+; CHECK-NEXT:    [[SUB:%.*]] = add i32 [[A]], -2
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 [[SUB]], i32 0
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %cmp = icmp ugt i32 3, %a
+  %sub = add i32 %a, -2
+  %sel = select i1 %cmp, i32 %sub, i32 0
+  ret i32 %sel
+}
+
+define i32 @max_sub_uge_c32(i32 %a) {
+; CHECK-LABEL: @max_sub_uge_c32(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[A:%.*]], 3
+; CHECK-NEXT:    [[SUB:%.*]] = add i32 [[A]], -2
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 [[SUB]], i32 0
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %cmp = icmp uge i32 2, %a
+  %sub = add i32 %a, -2
+  %sel = select i1 %cmp, i32 %sub, i32 0
+  ret i32 %sel
+}
+
+define i32 @max_sub_ult_c12(i32 %a) {
+; CHECK-LABEL: @max_sub_ult_c12(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[A:%.*]], 0
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 -2, i32 0
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %cmp = icmp ult i32 %a, 1
+  %sub = add i32 %a, -2
+  %sel = select i1 %cmp, i32 %sub, i32 0
+  ret i32 %sel
+}
+
+define i32 @max_sub_ult_c0(i32 %a) {
+; CHECK-LABEL: @max_sub_ult_c0(
+; CHECK-NEXT:    ret i32 0
+;
+  %cmp = icmp ult i32 %a, 0
+  %sub = add i32 %a, -1
+  %sel = select i1 %cmp, i32 %sub, i32 0
+  ret i32 %sel
+}
+
diff --git a/llvm/test/Transforms/InstCombine/unused-nonnull.ll b/llvm/test/Transforms/InstCombine/unused-nonnull.ll
new file mode 100644
index 000000000..518cd268e
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/unused-nonnull.ll
@@ -0,0 +1,45 @@
+; PR44154: LLVM c3b06d0c393e caused the body of @main to be replaced with
+; unreachable. Check that we perform the expected calls and optimizations.
+;
+; RUN: opt -S -O3 -o - %s | FileCheck %s
+; CHECK-LABEL: @main
+; CHECK:       %0 = icmp slt i32 %argc, 2
+; CHECK-NEXT:  br i1 %0, label %done, label %do_work
+; CHECK-LABEL: do_work:
+; CHECK:       %1 = tail call i32 @compute(
+; CHECK-LABEL: done:
+; CHECK:       %retval = phi i32 [ 0, %entry ], [ %1, %do_work ]
+; CHECK-NEXT:  ret i32 %retval
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define i32 @main(i32 %argc, i8** %argv) #0 {
+entry:
+  %0 = getelementptr inbounds i8*, i8** %argv, i32 0
+  %ptr = load i8*, i8** %0
+  %1 = call i32 @compute(i8* %ptr, i32 %argc)
+  %2 = icmp slt i32 %argc, 2
+  br i1 %2, label %done, label %do_work
+
+do_work:
+  %3 = icmp eq i8* %ptr, null
+  br i1 %3, label %null, label %done
+
+null:
+  call void @call_if_null(i8* %ptr)
+  br label %done
+
+done:
+  %retval = phi i32 [0, %entry], [%1, %do_work], [%1, %null]
+  ret i32 %retval
+}
+
+define i32 @compute(i8* nonnull %ptr, i32 %x) #1 {
+  ret i32 %x
+}
+
+declare void @call_if_null(i8* %ptr) #0
+
+attributes #0 = { nounwind }
+attributes #1 = { noinline nounwind readonly }
diff --git a/llvm/test/Transforms/InstCombine/urem-via-udiv-mul-sub.ll b/llvm/test/Transforms/InstCombine/urem-via-udiv-mul-sub.ll
new file mode 100644
index 000000000..0e28a0d72
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/urem-via-udiv-mul-sub.ll
@@ -0,0 +1,117 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instcombine -S | FileCheck %s
+
+; Fold
+;   x - ((x / y) * y)
+; to
+;   x % y
+
+; Also,
+;   ((x / y) * y)
+; can then be simplified to
+;   x - (x % y)
+
+declare void @use8(i8)
+declare void @use2xi8(<2 x i8>)
+
+define i8 @t0_basic(i8 %x, i8 %y) {
+; CHECK-LABEL: @t0_basic(
+; CHECK-NEXT:    [[DIV:%.*]] = udiv i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[DIV]])
+; CHECK-NEXT:    [[ROUNDXDOWNTOMULTIPLEOFY:%.*]] = mul i8 [[DIV]], [[Y]]
+; CHECK-NEXT:    [[REM:%.*]] = sub i8 [[X]], [[ROUNDXDOWNTOMULTIPLEOFY]]
+; CHECK-NEXT:    ret i8 [[REM]]
+;
+  %div = udiv i8 %x, %y
+  call void @use8(i8 %div)
+  %roundXdownToMultipleOfY = mul i8 %div, %y
+  %rem = sub i8 %x, %roundXdownToMultipleOfY
+  ret i8 %rem
+}
+
+define <2 x i8> @t1_vector(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @t1_vector(
+; CHECK-NEXT:    [[DIV:%.*]] = udiv <2 x i8> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use2xi8(<2 x i8> [[DIV]])
+; CHECK-NEXT:    [[ROUNDXDOWNTOMULTIPLEOFY:%.*]] = mul <2 x i8> [[DIV]], [[Y]]
+; CHECK-NEXT:    [[REM:%.*]] = sub <2 x i8> [[X]], [[ROUNDXDOWNTOMULTIPLEOFY]]
+; CHECK-NEXT:    ret <2 x i8> [[REM]]
+;
+  %div = udiv <2 x i8> %x, %y
+  call void @use2xi8(<2 x i8> %div)
+  %roundXdownToMultipleOfY = mul <2 x i8> %div, %y
+  %rem = sub <2 x i8> %x, %roundXdownToMultipleOfY
+  ret <2 x i8> %rem
+}
+
+; Extra use
+
+define i8 @t4_extrause(i8 %x, i8 %y) {
+; CHECK-LABEL: @t4_extrause(
+; CHECK-NEXT:    [[DIV:%.*]] = udiv i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[DIV]])
+; CHECK-NEXT:    [[ROUNDXDOWNTOMULTIPLEOFY:%.*]] = mul i8 [[DIV]], [[Y]]
+; CHECK-NEXT:    call void @use8(i8 [[ROUNDXDOWNTOMULTIPLEOFY]])
+; CHECK-NEXT:    [[REM:%.*]] = sub i8 [[X]], [[ROUNDXDOWNTOMULTIPLEOFY]]
+; CHECK-NEXT:    ret i8 [[REM]]
+;
+  %div = udiv i8 %x, %y
+  call void @use8(i8 %div)
+  %roundXdownToMultipleOfY = mul i8 %div, %y
+  call void @use8(i8 %roundXdownToMultipleOfY)
+  %rem = sub i8 %x, %roundXdownToMultipleOfY
+  ret i8 %rem
+}
+
+; Commutativity
+
+declare i8 @gen8()
+
+define i8 @t5_commutative(i8 %x) {
+; CHECK-LABEL: @t5_commutative(
+; CHECK-NEXT:    [[Y:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[DIV:%.*]] = udiv i8 [[X:%.*]], [[Y]]
+; CHECK-NEXT:    call void @use8(i8 [[DIV]])
+; CHECK-NEXT:    [[ROUNDXDOWNTOMULTIPLEOFY:%.*]] = mul i8 [[Y]], [[DIV]]
+; CHECK-NEXT:    [[REM:%.*]] = sub i8 [[X]], [[ROUNDXDOWNTOMULTIPLEOFY]]
+; CHECK-NEXT:    ret i8 [[REM]]
+;
+  %y = call i8 @gen8()
+  %div = udiv i8 %x, %y
+  call void @use8(i8 %div)
+  %roundXdownToMultipleOfY = mul i8 %y, %div ; swapped
+  %rem = sub i8 %x, %roundXdownToMultipleOfY
+  ret i8 %rem
+}
+
+; Negative tests
+
+define i8 @n6_different_x(i8 %x0, i8 %x1, i8 %y) {
+; CHECK-LABEL: @n6_different_x(
+; CHECK-NEXT:    [[DIV:%.*]] = udiv i8 [[X0:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[DIV]])
+; CHECK-NEXT:    [[ROUNDXDOWNTOMULTIPLEOFY:%.*]] = mul i8 [[DIV]], [[Y]]
+; CHECK-NEXT:    [[REM:%.*]] = sub i8 [[X1:%.*]], [[ROUNDXDOWNTOMULTIPLEOFY]]
+; CHECK-NEXT:    ret i8 [[REM]]
+;
+  %div = udiv i8 %x0, %y
+  call void @use8(i8 %div)
+  %roundXdownToMultipleOfY = mul i8 %div, %y
+  %rem = sub i8 %x1, %roundXdownToMultipleOfY
+  ret i8 %rem
+}
+
+define i8 @n6_different_y(i8 %x, i8 %y0, i8 %y1) {
+; CHECK-LABEL: @n6_different_y(
+; CHECK-NEXT:    [[DIV:%.*]] = udiv i8 [[X:%.*]], [[Y0:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[DIV]])
+; CHECK-NEXT:    [[ROUNDXDOWNTOMULTIPLEOFY:%.*]] = mul i8 [[DIV]], [[Y1:%.*]]
+; CHECK-NEXT:    [[REM:%.*]] = sub i8 [[X]], [[ROUNDXDOWNTOMULTIPLEOFY]]
+; CHECK-NEXT:    ret i8 [[REM]]
+;
+  %div = udiv i8 %x, %y0
+  call void @use8(i8 %div)
+  %roundXdownToMultipleOfY = mul i8 %div, %y1
+  %rem = sub i8 %x, %roundXdownToMultipleOfY
+  ret i8 %rem
+}
diff --git a/llvm/test/Transforms/InstCombine/variable-signext-of-variable-high-bit-extraction.ll b/llvm/test/Transforms/InstCombine/variable-signext-of-variable-high-bit-extraction.ll
new file mode 100644
index 000000000..a5f38735a
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/variable-signext-of-variable-high-bit-extraction.ll
@@ -0,0 +1,575 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instcombine -S | FileCheck %s
+
+declare void @use16(i16)
+declare void @use32(i32)
+declare void @use64(i64)
+
+define i32 @t0(i64 %data, i32 %nbits) {
+; CHECK-LABEL: @t0(
+; CHECK-NEXT:    [[SKIP_HIGH:%.*]] = sub i32 64, [[NBITS:%.*]]
+; CHECK-NEXT:    call void @use32(i32 [[SKIP_HIGH]])
+; CHECK-NEXT:    [[SKIP_HIGH_WIDE:%.*]] = zext i32 [[SKIP_HIGH]] to i64
+; CHECK-NEXT:    call void @use64(i64 [[SKIP_HIGH_WIDE]])
+; CHECK-NEXT:    [[EXTRACTED:%.*]] = lshr i64 [[DATA:%.*]], [[SKIP_HIGH_WIDE]]
+; CHECK-NEXT:    call void @use64(i64 [[EXTRACTED]])
+; CHECK-NEXT:    [[EXTRACTED_NARROW:%.*]] = trunc i64 [[EXTRACTED]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[EXTRACTED_NARROW]])
+; CHECK-NEXT:    [[NUM_HIGH_BITS_TO_SMEAR_NARROW:%.*]] = sub i32 32, [[NBITS]]
+; CHECK-NEXT:    call void @use32(i32 [[NUM_HIGH_BITS_TO_SMEAR_NARROW]])
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr i64 [[DATA]], [[SKIP_HIGH_WIDE]]
+; CHECK-NEXT:    [[SIGNEXTENDED:%.*]] = trunc i64 [[TMP1]] to i32
+; CHECK-NEXT:    ret i32 [[SIGNEXTENDED]]
+;
+  %skip_high = sub i32 64, %nbits
+  call void @use32(i32 %skip_high)
+  %skip_high_wide = zext i32 %skip_high to i64
+  call void @use64(i64 %skip_high_wide)
+  %extracted = lshr i64 %data, %skip_high_wide
+  call void @use64(i64 %extracted)
+  %extracted_narrow = trunc i64 %extracted to i32
+  call void @use32(i32 %extracted_narrow)
+  %num_high_bits_to_smear_narrow = sub i32 32, %nbits
+  call void @use32(i32 %num_high_bits_to_smear_narrow)
+  %signbit_positioned = shl i32 %extracted_narrow, %num_high_bits_to_smear_narrow
+  %signextended = ashr i32 %signbit_positioned, %num_high_bits_to_smear_narrow
+  ret i32 %signextended
+}
+define i32 @t0_zext_of_nbits(i64 %data, i8 %nbits_narrow) {
+; CHECK-LABEL: @t0_zext_of_nbits(
+; CHECK-NEXT:    [[NBITS:%.*]] = zext i8 [[NBITS_NARROW:%.*]] to i16
+; CHECK-NEXT:    call void @use16(i16 [[NBITS]])
+; CHECK-NEXT:    [[SKIP_HIGH:%.*]] = sub nsw i16 64, [[NBITS]]
+; CHECK-NEXT:    call void @use16(i16 [[SKIP_HIGH]])
+; CHECK-NEXT:    [[SKIP_HIGH_WIDE:%.*]] = zext i16 [[SKIP_HIGH]] to i64
+; CHECK-NEXT:    call void @use64(i64 [[SKIP_HIGH_WIDE]])
+; CHECK-NEXT:    [[EXTRACTED:%.*]] = lshr i64 [[DATA:%.*]], [[SKIP_HIGH_WIDE]]
+; CHECK-NEXT:    call void @use64(i64 [[EXTRACTED]])
+; CHECK-NEXT:    [[EXTRACTED_NARROW:%.*]] = trunc i64 [[EXTRACTED]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[EXTRACTED_NARROW]])
+; CHECK-NEXT:    [[NUM_HIGH_BITS_TO_SMEAR_NARROW_NARROW:%.*]] = sub nsw i16 32, [[NBITS]]
+; CHECK-NEXT:    call void @use16(i16 [[NUM_HIGH_BITS_TO_SMEAR_NARROW_NARROW]])
+; CHECK-NEXT:    [[NUM_HIGH_BITS_TO_SMEAR_NARROW:%.*]] = zext i16 [[NUM_HIGH_BITS_TO_SMEAR_NARROW_NARROW]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[NUM_HIGH_BITS_TO_SMEAR_NARROW]])
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr i64 [[DATA]], [[SKIP_HIGH_WIDE]]
+; CHECK-NEXT:    [[SIGNEXTENDED:%.*]] = trunc i64 [[TMP1]] to i32
+; CHECK-NEXT:    ret i32 [[SIGNEXTENDED]]
+;
+  %nbits = zext i8 %nbits_narrow to i16
+  call void @use16(i16 %nbits)
+  %skip_high = sub i16 64, %nbits
+  call void @use16(i16 %skip_high)
+  %skip_high_wide = zext i16 %skip_high to i64
+  call void @use64(i64 %skip_high_wide)
+  %extracted = lshr i64 %data, %skip_high_wide
+  call void @use64(i64 %extracted)
+  %extracted_narrow = trunc i64 %extracted to i32
+  call void @use32(i32 %extracted_narrow)
+  %num_high_bits_to_smear_narrow_narrow = sub i16 32, %nbits
+  call void @use16(i16 %num_high_bits_to_smear_narrow_narrow)
+  %num_high_bits_to_smear_narrow = zext i16 %num_high_bits_to_smear_narrow_narrow to i32
+  call void @use32(i32 %num_high_bits_to_smear_narrow)
+  %signbit_positioned = shl i32 %extracted_narrow, %num_high_bits_to_smear_narrow
+  %signextended = ashr i32 %signbit_positioned, %num_high_bits_to_smear_narrow
+  ret i32 %signextended
+}
+define i32 @t0_exact(i64 %data, i32 %nbits) {
+; CHECK-LABEL: @t0_exact(
+; CHECK-NEXT:    [[SKIP_HIGH:%.*]] = sub i32 64, [[NBITS:%.*]]
+; CHECK-NEXT:    call void @use32(i32 [[SKIP_HIGH]])
+; CHECK-NEXT:    [[SKIP_HIGH_WIDE:%.*]] = zext i32 [[SKIP_HIGH]] to i64
+; CHECK-NEXT:    call void @use64(i64 [[SKIP_HIGH_WIDE]])
+; CHECK-NEXT:    [[EXTRACTED:%.*]] = lshr exact i64 [[DATA:%.*]], [[SKIP_HIGH_WIDE]]
+; CHECK-NEXT:    call void @use64(i64 [[EXTRACTED]])
+; CHECK-NEXT:    [[EXTRACTED_NARROW:%.*]] = trunc i64 [[EXTRACTED]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[EXTRACTED_NARROW]])
+; CHECK-NEXT:    [[NUM_HIGH_BITS_TO_SMEAR_NARROW:%.*]] = sub i32 32, [[NBITS]]
+; CHECK-NEXT:    call void @use32(i32 [[NUM_HIGH_BITS_TO_SMEAR_NARROW]])
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr exact i64 [[DATA]], [[SKIP_HIGH_WIDE]]
+; CHECK-NEXT:    [[SIGNEXTENDED:%.*]] = trunc i64 [[TMP1]] to i32
+; CHECK-NEXT:    ret i32 [[SIGNEXTENDED]]
+;
+  %skip_high = sub i32 64, %nbits
+  call void @use32(i32 %skip_high)
+  %skip_high_wide = zext i32 %skip_high to i64
+  call void @use64(i64 %skip_high_wide)
+  %extracted = lshr exact i64 %data, %skip_high_wide ; We can preserve `exact`-ness of the original shift.
+  call void @use64(i64 %extracted)
+  %extracted_narrow = trunc i64 %extracted to i32
+  call void @use32(i32 %extracted_narrow)
+  %num_high_bits_to_smear_narrow = sub i32 32, %nbits
+  call void @use32(i32 %num_high_bits_to_smear_narrow)
+  %signbit_positioned = shl i32 %extracted_narrow, %num_high_bits_to_smear_narrow
+  %signextended = ashr i32 %signbit_positioned, %num_high_bits_to_smear_narrow
+  ret i32 %signextended
+}
+
+define i32 @t1_redundant_sext(i64 %data, i32 %nbits) {
+; CHECK-LABEL: @t1_redundant_sext(
+; CHECK-NEXT:    [[SKIP_HIGH:%.*]] = sub i32 64, [[NBITS:%.*]]
+; CHECK-NEXT:    call void @use32(i32 [[SKIP_HIGH]])
+; CHECK-NEXT:    [[SKIP_HIGH_WIDE:%.*]] = zext i32 [[SKIP_HIGH]] to i64
+; CHECK-NEXT:    call void @use64(i64 [[SKIP_HIGH_WIDE]])
+; CHECK-NEXT:    [[EXTRACTED_WITH_SIGNEXTENSION:%.*]] = ashr i64 [[DATA:%.*]], [[SKIP_HIGH_WIDE]]
+; CHECK-NEXT:    call void @use64(i64 [[EXTRACTED_WITH_SIGNEXTENSION]])
+; CHECK-NEXT:    [[EXTRACTED_WITH_SIGNEXTENSION_NARROW:%.*]] = trunc i64 [[EXTRACTED_WITH_SIGNEXTENSION]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[EXTRACTED_WITH_SIGNEXTENSION_NARROW]])
+; CHECK-NEXT:    [[NUM_HIGH_BITS_TO_SMEAR_NARROW:%.*]] = sub i32 32, [[NBITS]]
+; CHECK-NEXT:    call void @use32(i32 [[NUM_HIGH_BITS_TO_SMEAR_NARROW]])
+; CHECK-NEXT:    [[SIGNBIT_POSITIONED:%.*]] = shl i32 [[EXTRACTED_WITH_SIGNEXTENSION_NARROW]], [[NUM_HIGH_BITS_TO_SMEAR_NARROW]]
+; CHECK-NEXT:    call void @use32(i32 [[SIGNBIT_POSITIONED]])
+; CHECK-NEXT:    ret i32 [[EXTRACTED_WITH_SIGNEXTENSION_NARROW]]
+;
+  %skip_high = sub i32 64, %nbits
+  call void @use32(i32 %skip_high)
+  %skip_high_wide = zext i32 %skip_high to i64
+  call void @use64(i64 %skip_high_wide)
+  %extracted_with_signextension = ashr i64 %data, %skip_high_wide
+  call void @use64(i64 %extracted_with_signextension)
+  %extracted_with_signextension_narrow = trunc i64 %extracted_with_signextension to i32 ; this is already the answer.
+  call void @use32(i32 %extracted_with_signextension_narrow)
+  %num_high_bits_to_smear_narrow = sub i32 32, %nbits
+  call void @use32(i32 %num_high_bits_to_smear_narrow)
+  %signbit_positioned = shl i32 %extracted_with_signextension_narrow, %num_high_bits_to_smear_narrow
+  call void @use32(i32 %signbit_positioned)
+  %signextended = ashr i32 %signbit_positioned, %num_high_bits_to_smear_narrow
+  ret i32 %signextended
+}
+
+define i64 @t2_notrunc(i64 %data, i64 %nbits) {
+; CHECK-LABEL: @t2_notrunc(
+; CHECK-NEXT:    [[SKIP_HIGH:%.*]] = sub i64 64, [[NBITS:%.*]]
+; CHECK-NEXT:    call void @use64(i64 [[SKIP_HIGH]])
+; CHECK-NEXT:    [[EXTRACTED:%.*]] = lshr i64 [[DATA:%.*]], [[SKIP_HIGH]]
+; CHECK-NEXT:    call void @use64(i64 [[EXTRACTED]])
+; CHECK-NEXT:    [[NUM_HIGH_BITS_TO_SMEAR:%.*]] = sub i64 64, [[NBITS]]
+; CHECK-NEXT:    call void @use64(i64 [[NUM_HIGH_BITS_TO_SMEAR]])
+; CHECK-NEXT:    [[SIGNBIT_POSITIONED:%.*]] = shl i64 [[EXTRACTED]], [[NUM_HIGH_BITS_TO_SMEAR]]
+; CHECK-NEXT:    call void @use64(i64 [[SIGNBIT_POSITIONED]])
+; CHECK-NEXT:    [[SIGNEXTENDED:%.*]] = ashr i64 [[DATA]], [[SKIP_HIGH]]
+; CHECK-NEXT:    ret i64 [[SIGNEXTENDED]]
+;
+  %skip_high = sub i64 64, %nbits
+  call void @use64(i64 %skip_high)
+  %extracted = lshr i64 %data, %skip_high
+  call void @use64(i64 %extracted)
+  %num_high_bits_to_smear = sub i64 64, %nbits
+  call void @use64(i64 %num_high_bits_to_smear)
+  %signbit_positioned = shl i64 %extracted, %num_high_bits_to_smear ;
+  call void @use64(i64 %signbit_positioned)
+  %signextended = ashr i64 %signbit_positioned, %num_high_bits_to_smear ; can just shift %data itself.
+  ret i64 %signextended
+}
+
+define i64 @t3_notrunc_redundant_sext(i64 %data, i64 %nbits) {
+; CHECK-LABEL: @t3_notrunc_redundant_sext(
+; CHECK-NEXT:    [[SKIP_HIGH:%.*]] = sub i64 64, [[NBITS:%.*]]
+; CHECK-NEXT:    call void @use64(i64 [[SKIP_HIGH]])
+; CHECK-NEXT:    [[EXTRACTED:%.*]] = ashr i64 [[DATA:%.*]], [[SKIP_HIGH]]
+; CHECK-NEXT:    call void @use64(i64 [[EXTRACTED]])
+; CHECK-NEXT:    [[NUM_HIGH_BITS_TO_SMEAR:%.*]] = sub i64 64, [[NBITS]]
+; CHECK-NEXT:    call void @use64(i64 [[NUM_HIGH_BITS_TO_SMEAR]])
+; CHECK-NEXT:    [[SIGNBIT_POSITIONED:%.*]] = shl i64 [[EXTRACTED]], [[NUM_HIGH_BITS_TO_SMEAR]]
+; CHECK-NEXT:    call void @use64(i64 [[SIGNBIT_POSITIONED]])
+; CHECK-NEXT:    ret i64 [[EXTRACTED]]
+;
+  %skip_high = sub i64 64, %nbits
+  call void @use64(i64 %skip_high)
+  %extracted = ashr i64 %data, %skip_high ; this is already the answer.
+  call void @use64(i64 %extracted)
+  %num_high_bits_to_smear = sub i64 64, %nbits
+  call void @use64(i64 %num_high_bits_to_smear)
+  %signbit_positioned = shl i64 %extracted, %num_high_bits_to_smear
+  call void @use64(i64 %signbit_positioned)
+  %signextended = ashr i64 %signbit_positioned, %num_high_bits_to_smear
+  ret i64 %signextended
+}
+
+define <2 x i32> @t4_vec(<2 x i64> %data, <2 x i32> %nbits) {
+; CHECK-LABEL: @t4_vec(
+; CHECK-NEXT:    [[SKIP_HIGH:%.*]] = sub <2 x i32> <i32 64, i32 64>, [[NBITS:%.*]]
+; CHECK-NEXT:    [[SKIP_HIGH_WIDE:%.*]] = zext <2 x i32> [[SKIP_HIGH]] to <2 x i64>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <2 x i64> [[DATA:%.*]], [[SKIP_HIGH_WIDE]]
+; CHECK-NEXT:    [[SIGNEXTENDED:%.*]] = trunc <2 x i64> [[TMP1]] to <2 x i32>
+; CHECK-NEXT:    ret <2 x i32> [[SIGNEXTENDED]]
+;
+  %skip_high = sub <2 x i32> <i32 64, i32 64>, %nbits
+  %skip_high_wide = zext <2 x i32> %skip_high to <2 x i64>
+  %extracted = lshr <2 x i64> %data, %skip_high_wide
+  %extracted_narrow = trunc <2 x i64> %extracted to <2 x i32>
+  %num_high_bits_to_smear_narrow = sub <2 x i32> <i32 32, i32 32>, %nbits
+  %signbit_positioned = shl <2 x i32> %extracted_narrow, %num_high_bits_to_smear_narrow
+  %signextended = ashr <2 x i32> %signbit_positioned, %num_high_bits_to_smear_narrow
+  ret <2 x i32> %signextended
+}
+
+define <3 x i32> @t5_vec_undef(<3 x i64> %data, <3 x i32> %nbits) {
+; CHECK-LABEL: @t5_vec_undef(
+; CHECK-NEXT:    [[SKIP_HIGH:%.*]] = sub <3 x i32> <i32 64, i32 64, i32 undef>, [[NBITS:%.*]]
+; CHECK-NEXT:    [[SKIP_HIGH_WIDE:%.*]] = zext <3 x i32> [[SKIP_HIGH]] to <3 x i64>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <3 x i64> [[DATA:%.*]], [[SKIP_HIGH_WIDE]]
+; CHECK-NEXT:    [[SIGNEXTENDED:%.*]] = trunc <3 x i64> [[TMP1]] to <3 x i32>
+; CHECK-NEXT:    ret <3 x i32> [[SIGNEXTENDED]]
+;
+  %skip_high = sub <3 x i32> <i32 64, i32 64, i32 undef>, %nbits
+  %skip_high_wide = zext <3 x i32> %skip_high to <3 x i64>
+  %extracted = lshr <3 x i64> %data, %skip_high_wide
+  %extracted_narrow = trunc <3 x i64> %extracted to <3 x i32>
+  %num_high_bits_to_smear_narrow0 = sub <3 x i32> <i32 32, i32 32, i32 undef>, %nbits
+  %num_high_bits_to_smear_narrow1 = sub <3 x i32> <i32 undef, i32 32, i32 32>, %nbits
+  %signbit_positioned = shl <3 x i32> %extracted_narrow, %num_high_bits_to_smear_narrow0
+  %signextended = ashr <3 x i32> %signbit_positioned, %num_high_bits_to_smear_narrow1
+  ret <3 x i32> %signextended
+}
+
+; Extra-uses
+define i32 @t6_extrause_good0(i64 %data, i32 %nbits) {
+; CHECK-LABEL: @t6_extrause_good0(
+; CHECK-NEXT:    [[SKIP_HIGH:%.*]] = sub i32 64, [[NBITS:%.*]]
+; CHECK-NEXT:    call void @use32(i32 [[SKIP_HIGH]])
+; CHECK-NEXT:    [[SKIP_HIGH_WIDE:%.*]] = zext i32 [[SKIP_HIGH]] to i64
+; CHECK-NEXT:    call void @use64(i64 [[SKIP_HIGH_WIDE]])
+; CHECK-NEXT:    [[EXTRACTED:%.*]] = lshr i64 [[DATA:%.*]], [[SKIP_HIGH_WIDE]]
+; CHECK-NEXT:    call void @use64(i64 [[EXTRACTED]])
+; CHECK-NEXT:    [[EXTRACTED_NARROW:%.*]] = trunc i64 [[EXTRACTED]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[EXTRACTED_NARROW]])
+; CHECK-NEXT:    [[NUM_HIGH_BITS_TO_SMEAR_NARROW:%.*]] = sub i32 32, [[NBITS]]
+; CHECK-NEXT:    call void @use32(i32 [[NUM_HIGH_BITS_TO_SMEAR_NARROW]])
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr i64 [[DATA]], [[SKIP_HIGH_WIDE]]
+; CHECK-NEXT:    [[SIGNEXTENDED:%.*]] = trunc i64 [[TMP1]] to i32
+; CHECK-NEXT:    ret i32 [[SIGNEXTENDED]]
+;
+  %skip_high = sub i32 64, %nbits
+  call void @use32(i32 %skip_high)
+  %skip_high_wide = zext i32 %skip_high to i64
+  call void @use64(i64 %skip_high_wide)
+  %extracted = lshr i64 %data, %skip_high_wide
+  call void @use64(i64 %extracted)
+  %extracted_narrow = trunc i64 %extracted to i32
+  call void @use32(i32 %extracted_narrow)
+  %num_high_bits_to_smear_narrow = sub i32 32, %nbits
+  call void @use32(i32 %num_high_bits_to_smear_narrow)
+  %signbit_positioned = shl i32 %extracted_narrow, %num_high_bits_to_smear_narrow ; will go away
+  %signextended = ashr i32 %signbit_positioned, %num_high_bits_to_smear_narrow
+  ret i32 %signextended
+}
+define i32 @t7_extrause_good1(i64 %data, i32 %nbits) {
+; CHECK-LABEL: @t7_extrause_good1(
+; CHECK-NEXT:    [[SKIP_HIGH:%.*]] = sub i32 64, [[NBITS:%.*]]
+; CHECK-NEXT:    call void @use32(i32 [[SKIP_HIGH]])
+; CHECK-NEXT:    [[SKIP_HIGH_WIDE:%.*]] = zext i32 [[SKIP_HIGH]] to i64
+; CHECK-NEXT:    call void @use64(i64 [[SKIP_HIGH_WIDE]])
+; CHECK-NEXT:    [[EXTRACTED:%.*]] = lshr i64 [[DATA:%.*]], [[SKIP_HIGH_WIDE]]
+; CHECK-NEXT:    call void @use64(i64 [[EXTRACTED]])
+; CHECK-NEXT:    [[EXTRACTED_NARROW:%.*]] = trunc i64 [[EXTRACTED]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[EXTRACTED_NARROW]])
+; CHECK-NEXT:    [[NUM_HIGH_BITS_TO_SMEAR_NARROW0:%.*]] = sub i32 32, [[NBITS]]
+; CHECK-NEXT:    call void @use32(i32 [[NUM_HIGH_BITS_TO_SMEAR_NARROW0]])
+; CHECK-NEXT:    [[SIGNBIT_POSITIONED:%.*]] = shl i32 [[EXTRACTED_NARROW]], [[NUM_HIGH_BITS_TO_SMEAR_NARROW0]]
+; CHECK-NEXT:    call void @use32(i32 [[SIGNBIT_POSITIONED]])
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr i64 [[DATA]], [[SKIP_HIGH_WIDE]]
+; CHECK-NEXT:    [[SIGNEXTENDED:%.*]] = trunc i64 [[TMP1]] to i32
+; CHECK-NEXT:    ret i32 [[SIGNEXTENDED]]
+;
+  %skip_high = sub i32 64, %nbits
+  call void @use32(i32 %skip_high)
+  %skip_high_wide = zext i32 %skip_high to i64
+  call void @use64(i64 %skip_high_wide)
+  %extracted = lshr i64 %data, %skip_high_wide
+  call void @use64(i64 %extracted)
+  %extracted_narrow = trunc i64 %extracted to i32
+  call void @use32(i32 %extracted_narrow)
+  %num_high_bits_to_smear_narrow0 = sub i32 32, %nbits
+  call void @use32(i32 %num_high_bits_to_smear_narrow0)
+  %num_high_bits_to_smear_narrow1 = sub i32 32, %nbits ; will go away.
+  %signbit_positioned = shl i32 %extracted_narrow, %num_high_bits_to_smear_narrow0
+  call void @use32(i32 %signbit_positioned)
+  %signextended = ashr i32 %signbit_positioned, %num_high_bits_to_smear_narrow1
+  ret i32 %signextended
+}
+define i32 @n8_extrause_bad(i64 %data, i32 %nbits) {
+; CHECK-LABEL: @n8_extrause_bad(
+; CHECK-NEXT:    [[SKIP_HIGH:%.*]] = sub i32 64, [[NBITS:%.*]]
+; CHECK-NEXT:    call void @use32(i32 [[SKIP_HIGH]])
+; CHECK-NEXT:    [[SKIP_HIGH_WIDE:%.*]] = zext i32 [[SKIP_HIGH]] to i64
+; CHECK-NEXT:    call void @use64(i64 [[SKIP_HIGH_WIDE]])
+; CHECK-NEXT:    [[EXTRACTED:%.*]] = lshr i64 [[DATA:%.*]], [[SKIP_HIGH_WIDE]]
+; CHECK-NEXT:    call void @use64(i64 [[EXTRACTED]])
+; CHECK-NEXT:    [[EXTRACTED_NARROW:%.*]] = trunc i64 [[EXTRACTED]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[EXTRACTED_NARROW]])
+; CHECK-NEXT:    [[NUM_HIGH_BITS_TO_SMEAR_NARROW:%.*]] = sub i32 32, [[NBITS]]
+; CHECK-NEXT:    call void @use32(i32 [[NUM_HIGH_BITS_TO_SMEAR_NARROW]])
+; CHECK-NEXT:    [[SIGNBIT_POSITIONED:%.*]] = shl i32 [[EXTRACTED_NARROW]], [[NUM_HIGH_BITS_TO_SMEAR_NARROW]]
+; CHECK-NEXT:    call void @use32(i32 [[SIGNBIT_POSITIONED]])
+; CHECK-NEXT:    [[SIGNEXTENDED:%.*]] = ashr i32 [[SIGNBIT_POSITIONED]], [[NUM_HIGH_BITS_TO_SMEAR_NARROW]]
+; CHECK-NEXT:    ret i32 [[SIGNEXTENDED]]
+;
+  %skip_high = sub i32 64, %nbits
+  call void @use32(i32 %skip_high)
+  %skip_high_wide = zext i32 %skip_high to i64
+  call void @use64(i64 %skip_high_wide)
+  %extracted = lshr i64 %data, %skip_high_wide
+  call void @use64(i64 %extracted)
+  %extracted_narrow = trunc i64 %extracted to i32
+  call void @use32(i32 %extracted_narrow)
+  %num_high_bits_to_smear_narrow = sub i32 32, %nbits
+  call void @use32(i32 %num_high_bits_to_smear_narrow)
+  %signbit_positioned = shl i32 %extracted_narrow, %num_high_bits_to_smear_narrow
+  call void @use32(i32 %signbit_positioned)
+  %signextended = ashr i32 %signbit_positioned, %num_high_bits_to_smear_narrow ; neither of operands will go away.
+  ret i32 %signextended
+}
+
+; Negative tests
+define i32 @n9(i64 %data, i32 %nbits) {
+; CHECK-LABEL: @n9(
+; CHECK-NEXT:    [[SKIP_HIGH:%.*]] = sub i32 63, [[NBITS:%.*]]
+; CHECK-NEXT:    call void @use32(i32 [[SKIP_HIGH]])
+; CHECK-NEXT:    [[SKIP_HIGH_WIDE:%.*]] = zext i32 [[SKIP_HIGH]] to i64
+; CHECK-NEXT:    call void @use64(i64 [[SKIP_HIGH_WIDE]])
+; CHECK-NEXT:    [[EXTRACTED:%.*]] = lshr i64 [[DATA:%.*]], [[SKIP_HIGH_WIDE]]
+; CHECK-NEXT:    call void @use64(i64 [[EXTRACTED]])
+; CHECK-NEXT:    [[EXTRACTED_NARROW:%.*]] = trunc i64 [[EXTRACTED]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[EXTRACTED_NARROW]])
+; CHECK-NEXT:    [[NUM_HIGH_BITS_TO_SMEAR_NARROW:%.*]] = sub i32 32, [[NBITS]]
+; CHECK-NEXT:    call void @use32(i32 [[NUM_HIGH_BITS_TO_SMEAR_NARROW]])
+; CHECK-NEXT:    [[SIGNBIT_POSITIONED:%.*]] = shl i32 [[EXTRACTED_NARROW]], [[NUM_HIGH_BITS_TO_SMEAR_NARROW]]
+; CHECK-NEXT:    [[SIGNEXTENDED:%.*]] = ashr i32 [[SIGNBIT_POSITIONED]], [[NUM_HIGH_BITS_TO_SMEAR_NARROW]]
+; CHECK-NEXT:    ret i32 [[SIGNEXTENDED]]
+;
+  %skip_high = sub i32 63, %nbits ; not 64
+  call void @use32(i32 %skip_high)
+  %skip_high_wide = zext i32 %skip_high to i64
+  call void @use64(i64 %skip_high_wide)
+  %extracted = lshr i64 %data, %skip_high_wide
+  call void @use64(i64 %extracted)
+  %extracted_narrow = trunc i64 %extracted to i32
+  call void @use32(i32 %extracted_narrow)
+  %num_high_bits_to_smear_narrow = sub i32 32, %nbits
+  call void @use32(i32 %num_high_bits_to_smear_narrow)
+  %signbit_positioned = shl i32 %extracted_narrow, %num_high_bits_to_smear_narrow
+  %signextended = ashr i32 %signbit_positioned, %num_high_bits_to_smear_narrow
+  ret i32 %signextended
+}
+
+define i32 @n10(i64 %data, i32 %nbits) {
+; CHECK-LABEL: @n10(
+; CHECK-NEXT:    [[SKIP_HIGH:%.*]] = sub i32 64, [[NBITS:%.*]]
+; CHECK-NEXT:    call void @use32(i32 [[SKIP_HIGH]])
+; CHECK-NEXT:    [[SKIP_HIGH_WIDE:%.*]] = zext i32 [[SKIP_HIGH]] to i64
+; CHECK-NEXT:    call void @use64(i64 [[SKIP_HIGH_WIDE]])
+; CHECK-NEXT:    [[EXTRACTED:%.*]] = lshr i64 [[DATA:%.*]], [[SKIP_HIGH_WIDE]]
+; CHECK-NEXT:    call void @use64(i64 [[EXTRACTED]])
+; CHECK-NEXT:    [[EXTRACTED_NARROW:%.*]] = trunc i64 [[EXTRACTED]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[EXTRACTED_NARROW]])
+; CHECK-NEXT:    [[NUM_HIGH_BITS_TO_SMEAR_NARROW:%.*]] = sub i32 31, [[NBITS]]
+; CHECK-NEXT:    call void @use32(i32 [[NUM_HIGH_BITS_TO_SMEAR_NARROW]])
+; CHECK-NEXT:    [[SIGNBIT_POSITIONED:%.*]] = shl i32 [[EXTRACTED_NARROW]], [[NUM_HIGH_BITS_TO_SMEAR_NARROW]]
+; CHECK-NEXT:    [[SIGNEXTENDED:%.*]] = ashr i32 [[SIGNBIT_POSITIONED]], [[NUM_HIGH_BITS_TO_SMEAR_NARROW]]
+; CHECK-NEXT:    ret i32 [[SIGNEXTENDED]]
+;
+  %skip_high = sub i32 64, %nbits
+  call void @use32(i32 %skip_high)
+  %skip_high_wide = zext i32 %skip_high to i64
+  call void @use64(i64 %skip_high_wide)
+  %extracted = lshr i64 %data, %skip_high_wide
+  call void @use64(i64 %extracted)
+  %extracted_narrow = trunc i64 %extracted to i32
+  call void @use32(i32 %extracted_narrow)
+  %num_high_bits_to_smear_narrow = sub i32 31, %nbits ; not 32
+  call void @use32(i32 %num_high_bits_to_smear_narrow)
+  %signbit_positioned = shl i32 %extracted_narrow, %num_high_bits_to_smear_narrow
+  %signextended = ashr i32 %signbit_positioned, %num_high_bits_to_smear_narrow
+  ret i32 %signextended
+}
+
+define i32 @n11(i64 %data, i32 %nbits1, i32 %nbits2) {
+; CHECK-LABEL: @n11(
+; CHECK-NEXT:    [[SKIP_HIGH:%.*]] = sub i32 64, [[NBITS1:%.*]]
+; CHECK-NEXT:    call void @use32(i32 [[SKIP_HIGH]])
+; CHECK-NEXT:    [[SKIP_HIGH_WIDE:%.*]] = zext i32 [[SKIP_HIGH]] to i64
+; CHECK-NEXT:    call void @use64(i64 [[SKIP_HIGH_WIDE]])
+; CHECK-NEXT:    [[EXTRACTED:%.*]] = lshr i64 [[DATA:%.*]], [[SKIP_HIGH_WIDE]]
+; CHECK-NEXT:    call void @use64(i64 [[EXTRACTED]])
+; CHECK-NEXT:    [[EXTRACTED_NARROW:%.*]] = trunc i64 [[EXTRACTED]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[EXTRACTED_NARROW]])
+; CHECK-NEXT:    [[NUM_HIGH_BITS_TO_SMEAR_NARROW:%.*]] = sub i32 32, [[NBITS2:%.*]]
+; CHECK-NEXT:    call void @use32(i32 [[NUM_HIGH_BITS_TO_SMEAR_NARROW]])
+; CHECK-NEXT:    [[SIGNBIT_POSITIONED:%.*]] = shl i32 [[EXTRACTED_NARROW]], [[NUM_HIGH_BITS_TO_SMEAR_NARROW]]
+; CHECK-NEXT:    [[SIGNEXTENDED:%.*]] = ashr i32 [[SIGNBIT_POSITIONED]], [[NUM_HIGH_BITS_TO_SMEAR_NARROW]]
+; CHECK-NEXT:    ret i32 [[SIGNEXTENDED]]
+;
+  %skip_high = sub i32 64, %nbits1 ; not %nbits2
+  call void @use32(i32 %skip_high)
+  %skip_high_wide = zext i32 %skip_high to i64
+  call void @use64(i64 %skip_high_wide)
+  %extracted = lshr i64 %data, %skip_high_wide
+  call void @use64(i64 %extracted)
+  %extracted_narrow = trunc i64 %extracted to i32
+  call void @use32(i32 %extracted_narrow)
+  %num_high_bits_to_smear_narrow = sub i32 32, %nbits2 ; not %nbits1
+  call void @use32(i32 %num_high_bits_to_smear_narrow)
+  %signbit_positioned = shl i32 %extracted_narrow, %num_high_bits_to_smear_narrow
+  %signextended = ashr i32 %signbit_positioned, %num_high_bits_to_smear_narrow
+  ret i32 %signextended
+}
+
+define i32 @n12(i64 %data, i32 %nbits1, i32 %nbits2) {
+; CHECK-LABEL: @n12(
+; CHECK-NEXT:    [[SKIP_HIGH:%.*]] = sub i32 64, [[NBITS1:%.*]]
+; CHECK-NEXT:    call void @use32(i32 [[SKIP_HIGH]])
+; CHECK-NEXT:    [[SKIP_HIGH_WIDE:%.*]] = zext i32 [[SKIP_HIGH]] to i64
+; CHECK-NEXT:    call void @use64(i64 [[SKIP_HIGH_WIDE]])
+; CHECK-NEXT:    [[EXTRACTED:%.*]] = lshr i64 [[DATA:%.*]], [[SKIP_HIGH_WIDE]]
+; CHECK-NEXT:    call void @use64(i64 [[EXTRACTED]])
+; CHECK-NEXT:    [[EXTRACTED_NARROW:%.*]] = trunc i64 [[EXTRACTED]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[EXTRACTED_NARROW]])
+; CHECK-NEXT:    [[NUM_HIGH_BITS_TO_SMEAR_NARROW1:%.*]] = sub i32 32, [[NBITS1]]
+; CHECK-NEXT:    [[NUM_HIGH_BITS_TO_SMEAR_NARROW2:%.*]] = sub i32 32, [[NBITS2:%.*]]
+; CHECK-NEXT:    call void @use32(i32 [[NUM_HIGH_BITS_TO_SMEAR_NARROW1]])
+; CHECK-NEXT:    call void @use32(i32 [[NUM_HIGH_BITS_TO_SMEAR_NARROW2]])
+; CHECK-NEXT:    [[SIGNBIT_POSITIONED:%.*]] = shl i32 [[EXTRACTED_NARROW]], [[NUM_HIGH_BITS_TO_SMEAR_NARROW1]]
+; CHECK-NEXT:    [[SIGNEXTENDED:%.*]] = ashr i32 [[SIGNBIT_POSITIONED]], [[NUM_HIGH_BITS_TO_SMEAR_NARROW2]]
+; CHECK-NEXT:    ret i32 [[SIGNEXTENDED]]
+;
+  %skip_high = sub i32 64, %nbits1
+  call void @use32(i32 %skip_high)
+  %skip_high_wide = zext i32 %skip_high to i64
+  call void @use64(i64 %skip_high_wide)
+  %extracted = lshr i64 %data, %skip_high_wide
+  call void @use64(i64 %extracted)
+  %extracted_narrow = trunc i64 %extracted to i32
+  call void @use32(i32 %extracted_narrow)
+  %num_high_bits_to_smear_narrow1 = sub i32 32, %nbits1 ; not %nbits2
+  %num_high_bits_to_smear_narrow2 = sub i32 32, %nbits2 ; not %nbits1
+  call void @use32(i32 %num_high_bits_to_smear_narrow1)
+  call void @use32(i32 %num_high_bits_to_smear_narrow2)
+  %signbit_positioned = shl i32 %extracted_narrow, %num_high_bits_to_smear_narrow1
+  %signextended = ashr i32 %signbit_positioned, %num_high_bits_to_smear_narrow2
+  ret i32 %signextended
+}
+
+define i32 @n13(i64 %data, i32 %nbits) {
+; CHECK-LABEL: @n13(
+; CHECK-NEXT:    [[SKIP_HIGH:%.*]] = sub i32 64, [[NBITS:%.*]]
+; CHECK-NEXT:    call void @use32(i32 [[SKIP_HIGH]])
+; CHECK-NEXT:    [[SKIP_HIGH_WIDE:%.*]] = zext i32 [[SKIP_HIGH]] to i64
+; CHECK-NEXT:    call void @use64(i64 [[SKIP_HIGH_WIDE]])
+; CHECK-NEXT:    [[EXTRACTED:%.*]] = lshr i64 [[DATA:%.*]], [[SKIP_HIGH_WIDE]]
+; CHECK-NEXT:    call void @use64(i64 [[EXTRACTED]])
+; CHECK-NEXT:    [[EXTRACTED_NARROW:%.*]] = trunc i64 [[EXTRACTED]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[EXTRACTED_NARROW]])
+; CHECK-NEXT:    [[NUM_HIGH_BITS_TO_SMEAR_NARROW:%.*]] = sub i32 32, [[NBITS]]
+; CHECK-NEXT:    call void @use32(i32 [[NUM_HIGH_BITS_TO_SMEAR_NARROW]])
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i32 -1, [[NUM_HIGH_BITS_TO_SMEAR_NARROW]]
+; CHECK-NEXT:    [[RES:%.*]] = and i32 [[TMP1]], [[EXTRACTED_NARROW]]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %skip_high = sub i32 64, %nbits
+  call void @use32(i32 %skip_high)
+  %skip_high_wide = zext i32 %skip_high to i64
+  call void @use64(i64 %skip_high_wide)
+  %extracted = lshr i64 %data, %skip_high_wide
+  call void @use64(i64 %extracted)
+  %extracted_narrow = trunc i64 %extracted to i32
+  call void @use32(i32 %extracted_narrow)
+  %num_high_bits_to_smear_narrow = sub i32 32, %nbits
+  call void @use32(i32 %num_high_bits_to_smear_narrow)
+  %highbits_cleaned = shl i32 %extracted_narrow, %num_high_bits_to_smear_narrow
+  %res = lshr i32 %highbits_cleaned, %num_high_bits_to_smear_narrow ; not ashr
+  ret i32 %res
+}
+define i32 @n13_extrause(i64 %data, i32 %nbits) {
+; CHECK-LABEL: @n13_extrause(
+; CHECK-NEXT:    [[SKIP_HIGH:%.*]] = sub i32 64, [[NBITS:%.*]]
+; CHECK-NEXT:    call void @use32(i32 [[SKIP_HIGH]])
+; CHECK-NEXT:    [[SKIP_HIGH_WIDE:%.*]] = zext i32 [[SKIP_HIGH]] to i64
+; CHECK-NEXT:    call void @use64(i64 [[SKIP_HIGH_WIDE]])
+; CHECK-NEXT:    [[EXTRACTED:%.*]] = lshr i64 [[DATA:%.*]], [[SKIP_HIGH_WIDE]]
+; CHECK-NEXT:    call void @use64(i64 [[EXTRACTED]])
+; CHECK-NEXT:    [[EXTRACTED_NARROW:%.*]] = trunc i64 [[EXTRACTED]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[EXTRACTED_NARROW]])
+; CHECK-NEXT:    [[NUM_HIGH_BITS_TO_SMEAR_NARROW:%.*]] = sub i32 32, [[NBITS]]
+; CHECK-NEXT:    call void @use32(i32 [[NUM_HIGH_BITS_TO_SMEAR_NARROW]])
+; CHECK-NEXT:    [[HIGHBITS_CLEANED:%.*]] = shl i32 [[EXTRACTED_NARROW]], [[NUM_HIGH_BITS_TO_SMEAR_NARROW]]
+; CHECK-NEXT:    call void @use32(i32 [[HIGHBITS_CLEANED]])
+; CHECK-NEXT:    [[RES:%.*]] = lshr i32 [[HIGHBITS_CLEANED]], [[NUM_HIGH_BITS_TO_SMEAR_NARROW]]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %skip_high = sub i32 64, %nbits
+  call void @use32(i32 %skip_high)
+  %skip_high_wide = zext i32 %skip_high to i64
+  call void @use64(i64 %skip_high_wide)
+  %extracted = lshr i64 %data, %skip_high_wide
+  call void @use64(i64 %extracted)
+  %extracted_narrow = trunc i64 %extracted to i32
+  call void @use32(i32 %extracted_narrow)
+  %num_high_bits_to_smear_narrow = sub i32 32, %nbits
+  call void @use32(i32 %num_high_bits_to_smear_narrow)
+  %highbits_cleaned = shl i32 %extracted_narrow, %num_high_bits_to_smear_narrow
+  call void @use32(i32 %highbits_cleaned)
+  %res = lshr i32 %highbits_cleaned, %num_high_bits_to_smear_narrow ; not ashr
+  ret i32 %res
+}
+define i32 @n14(i64 %data, i32 %nbits) {
+; CHECK-LABEL: @n14(
+; CHECK-NEXT:    [[SKIP_HIGH:%.*]] = sub i32 64, [[NBITS:%.*]]
+; CHECK-NEXT:    call void @use32(i32 [[SKIP_HIGH]])
+; CHECK-NEXT:    [[SKIP_HIGH_WIDE:%.*]] = zext i32 [[SKIP_HIGH]] to i64
+; CHECK-NEXT:    call void @use64(i64 [[SKIP_HIGH_WIDE]])
+; CHECK-NEXT:    [[EXTRACTED:%.*]] = ashr i64 [[DATA:%.*]], [[SKIP_HIGH_WIDE]]
+; CHECK-NEXT:    call void @use64(i64 [[EXTRACTED]])
+; CHECK-NEXT:    [[EXTRACTED_NARROW:%.*]] = trunc i64 [[EXTRACTED]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[EXTRACTED_NARROW]])
+; CHECK-NEXT:    [[NUM_HIGH_BITS_TO_SMEAR_NARROW:%.*]] = sub i32 32, [[NBITS]]
+; CHECK-NEXT:    call void @use32(i32 [[NUM_HIGH_BITS_TO_SMEAR_NARROW]])
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i32 -1, [[NUM_HIGH_BITS_TO_SMEAR_NARROW]]
+; CHECK-NEXT:    [[RES:%.*]] = and i32 [[TMP1]], [[EXTRACTED_NARROW]]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %skip_high = sub i32 64, %nbits
+  call void @use32(i32 %skip_high)
+  %skip_high_wide = zext i32 %skip_high to i64
+  call void @use64(i64 %skip_high_wide)
+  %extracted = ashr i64 %data, %skip_high_wide
+  call void @use64(i64 %extracted)
+  %extracted_narrow = trunc i64 %extracted to i32
+  call void @use32(i32 %extracted_narrow)
+  %num_high_bits_to_smear_narrow = sub i32 32, %nbits
+  call void @use32(i32 %num_high_bits_to_smear_narrow)
+  %highbits_cleaned = shl i32 %extracted_narrow, %num_high_bits_to_smear_narrow
+  %res = lshr i32 %highbits_cleaned, %num_high_bits_to_smear_narrow ; not ashr
+  ret i32 %res
+}
+define i32 @n14_extrause(i64 %data, i32 %nbits) {
+; CHECK-LABEL: @n14_extrause(
+; CHECK-NEXT:    [[SKIP_HIGH:%.*]] = sub i32 64, [[NBITS:%.*]]
+; CHECK-NEXT:    call void @use32(i32 [[SKIP_HIGH]])
+; CHECK-NEXT:    [[SKIP_HIGH_WIDE:%.*]] = zext i32 [[SKIP_HIGH]] to i64
+; CHECK-NEXT:    call void @use64(i64 [[SKIP_HIGH_WIDE]])
+; CHECK-NEXT:    [[EXTRACTED:%.*]] = ashr i64 [[DATA:%.*]], [[SKIP_HIGH_WIDE]]
+; CHECK-NEXT:    call void @use64(i64 [[EXTRACTED]])
+; CHECK-NEXT:    [[EXTRACTED_NARROW:%.*]] = trunc i64 [[EXTRACTED]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[EXTRACTED_NARROW]])
+; CHECK-NEXT:    [[NUM_HIGH_BITS_TO_SMEAR_NARROW:%.*]] = sub i32 32, [[NBITS]]
+; CHECK-NEXT:    call void @use32(i32 [[NUM_HIGH_BITS_TO_SMEAR_NARROW]])
+; CHECK-NEXT:    [[HIGHBITS_CLEANED:%.*]] = shl i32 [[EXTRACTED_NARROW]], [[NUM_HIGH_BITS_TO_SMEAR_NARROW]]
+; CHECK-NEXT:    call void @use32(i32 [[HIGHBITS_CLEANED]])
+; CHECK-NEXT:    [[RES:%.*]] = lshr i32 [[HIGHBITS_CLEANED]], [[NUM_HIGH_BITS_TO_SMEAR_NARROW]]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %skip_high = sub i32 64, %nbits
+  call void @use32(i32 %skip_high)
+  %skip_high_wide = zext i32 %skip_high to i64
+  call void @use64(i64 %skip_high_wide)
+  %extracted = ashr i64 %data, %skip_high_wide
+  call void @use64(i64 %extracted)
+  %extracted_narrow = trunc i64 %extracted to i32
+  call void @use32(i32 %extracted_narrow)
+  %num_high_bits_to_smear_narrow = sub i32 32, %nbits
+  call void @use32(i32 %num_high_bits_to_smear_narrow)
+  %highbits_cleaned = shl i32 %extracted_narrow, %num_high_bits_to_smear_narrow
+  call void @use32(i32 %highbits_cleaned)
+  %res = lshr i32 %highbits_cleaned, %num_high_bits_to_smear_narrow ; not ashr
+  ret i32 %res
+}
diff --git a/llvm/test/Transforms/InstCombine/vec_demanded_elts.ll b/llvm/test/Transforms/InstCombine/vec_demanded_elts.ll
index 3f172be3d..932dcf8e5 100644
--- a/llvm/test/Transforms/InstCombine/vec_demanded_elts.ll
+++ b/llvm/test/Transforms/InstCombine/vec_demanded_elts.ll
@@ -175,7 +175,8 @@ define <4 x i32> @inselt_shuf_no_demand_multiuse(i32 %a0, i32 %a1, <4 x i32> %b)
 ; CHECK-NEXT:    [[OUT0:%.*]] = insertelement <4 x i32> undef, i32 [[A0:%.*]], i32 0
 ; CHECK-NEXT:    [[OUT01:%.*]] = insertelement <4 x i32> [[OUT0]], i32 [[A1:%.*]], i32 1
 ; CHECK-NEXT:    [[FOO:%.*]] = add <4 x i32> [[OUT01]], [[B:%.*]]
-; CHECK-NEXT:    ret <4 x i32> [[FOO]]
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x i32> [[FOO]], <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    ret <4 x i32> [[SHUFFLE]]
 ;
   %out0 = insertelement <4 x i32> undef, i32 %a0, i32 0
   %out01 = insertelement <4 x i32> %out0, i32 %a1, i32 1
@@ -189,7 +190,8 @@ define <4 x i32> @inselt_shuf_no_demand_multiuse(i32 %a0, i32 %a1, <4 x i32> %b)
 define <4 x float> @inselt_shuf_no_demand_bogus_insert_index_in_chain(float %a1, float %a2, float %a3, i32 %variable_index) {
 ; CHECK-LABEL: @inselt_shuf_no_demand_bogus_insert_index_in_chain(
 ; CHECK-NEXT:    [[OUT12:%.*]] = insertelement <4 x float> undef, float [[A2:%.*]], i32 [[VARIABLE_INDEX:%.*]]
-; CHECK-NEXT:    ret <4 x float> [[OUT12]]
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x float> [[OUT12]], <4 x float> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    ret <4 x float> [[SHUFFLE]]
 ;
   %out1 = insertelement <4 x float> undef, float %a1, i32 1
   %out12 = insertelement <4 x float> %out1, float %a2, i32 %variable_index ; something unexpected
@@ -214,7 +216,8 @@ define <3 x i8> @shuf_add(<3 x i8> %x) {
 define <3 x i8> @shuf_sub(<3 x i8> %x) {
 ; CHECK-LABEL: @shuf_sub(
 ; CHECK-NEXT:    [[BO:%.*]] = sub <3 x i8> <i8 1, i8 undef, i8 3>, [[X:%.*]]
-; CHECK-NEXT:    ret <3 x i8> [[BO]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 0, i32 undef, i32 2>
+; CHECK-NEXT:    ret <3 x i8> [[R]]
 ;
   %bo = sub nuw <3 x i8> <i8 1, i8 2, i8 3>, %x
   %r = shufflevector <3 x i8> %bo, <3 x i8> undef, <3 x i32> <i32 0, i32 undef, i32 2>
@@ -569,7 +572,7 @@ define i32* @gep_cvbase_w_s_idx(<2 x i32*> %base, i64 %raw_addr) {
 
 define i32* @gep_cvbase_w_cv_idx(<2 x i32*> %base, i64 %raw_addr) {
 ; CHECK-LABEL: @gep_cvbase_w_cv_idx(
-; CHECK-NEXT:    ret i32* extractelement (<2 x i32*> getelementptr (i32, <2 x i32*> <i32* @GLOBAL, i32* @GLOBAL>, <2 x i64> <i64 0, i64 1>), i32 1)
+; CHECK-NEXT:    ret i32* getelementptr inbounds (i32, i32* @GLOBAL, i64 1)
 ;
   %gep = getelementptr i32, <2 x i32*> <i32* @GLOBAL, i32* @GLOBAL>, <2 x i64> <i64 0, i64 1>
   %ee = extractelement <2 x i32*> %gep, i32 1
@@ -644,10 +647,28 @@ define i32* @gep_demanded_lane_undef(i32* %base, i64 %idx) {
 ;; indices.
 define i32* @PR41624(<2 x { i32, i32 }*> %a) {
 ; CHECK-LABEL: @PR41624(
-; CHECK-NEXT:   %w = getelementptr { i32, i32 }, <2 x { i32, i32 }*> %a, <2 x i64> <i64 5, i64 5>, <2 x i32> zeroinitializer
-; CHECK-NEXT:   %r = extractelement <2 x i32*> %w, i32 0
-; CHECK-NEXT:   ret i32* %r
+; CHECK-NEXT:    [[W:%.*]] = getelementptr { i32, i32 }, <2 x { i32, i32 }*> [[A:%.*]], <2 x i64> <i64 5, i64 5>, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[R:%.*]] = extractelement <2 x i32*> [[W]], i32 0
+; CHECK-NEXT:    ret i32* [[R]]
+;
   %w = getelementptr { i32, i32 }, <2 x { i32, i32 }*> %a, <2 x i64> <i64 5, i64 5>, <2 x i32> zeroinitializer
   %r = extractelement <2 x i32*> %w, i32 0
   ret i32* %r
 }
+
+@global = external global [0 x i32], align 4
+
+; Make sure we don't get stuck in a loop turning the zeroinitializer into
+; <0, undef, undef, undef> and then changing it back.
+define i32* @zero_sized_type_extract(<4 x i64> %arg, i64 %arg1) {
+; CHECK-LABEL: @zero_sized_type_extract(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP:%.*]] = getelementptr inbounds [0 x i32], <4 x [0 x i32]*> <[0 x i32]* @global, [0 x i32]* undef, [0 x i32]* undef, [0 x i32]* undef>, <4 x i64> <i64 0, i64 undef, i64 undef, i64 undef>, <4 x i64> [[ARG:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x i32*> [[TMP]], i64 0
+; CHECK-NEXT:    ret i32* [[TMP2]]
+;
+bb:
+  %tmp = getelementptr inbounds [0 x i32], <4 x [0 x i32]*> <[0 x i32]* @global, [0 x i32]* @global, [0 x i32]* @global, [0 x i32]* @global>, <4 x i64> zeroinitializer, <4 x i64> %arg
+  %tmp2 = extractelement <4 x i32*> %tmp, i64 0
+  ret i32* %tmp2
+}
diff --git a/llvm/test/Transforms/InstCombine/vec_shuffle.ll b/llvm/test/Transforms/InstCombine/vec_shuffle.ll
index 65af43ea2..a6257a249 100644
--- a/llvm/test/Transforms/InstCombine/vec_shuffle.ll
+++ b/llvm/test/Transforms/InstCombine/vec_shuffle.ll
@@ -61,7 +61,8 @@ define float @test6(<4 x float> %X) {
 
 define <4 x float> @test7(<4 x float> %x) {
 ; CHECK-LABEL: @test7(
-; CHECK-NEXT:    ret <4 x float> [[X:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[X:%.*]], <4 x float> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    ret <4 x float> [[R]]
 ;
   %r = shufflevector <4 x float> %x, <4 x float> undef, <4 x i32> < i32 0, i32 1, i32 6, i32 7 >
   ret <4 x float> %r
@@ -1004,6 +1005,207 @@ define <2 x i32> @and_splat_constant(<2 x i32> %x) {
   ret <2 x i32> %r
 }
 
+; AND does not fold to undef for undef operands, we cannot move it
+; across a shuffle with undef masks.
+define <4 x i16> @and_constant_mask_undef(<4 x i16> %add) {
+; CHECK-LABEL: @and_constant_mask_undef(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x i16> [[ADD:%.*]], <4 x i16> undef, <4 x i32> <i32 undef, i32 undef, i32 1, i32 1>
+; CHECK-NEXT:    [[AND:%.*]] = and <4 x i16> [[SHUFFLE]], <i16 0, i16 0, i16 -1, i16 -1>
+; CHECK-NEXT:    ret <4 x i16> [[AND]]
+;
+entry:
+  %shuffle = shufflevector <4 x i16> %add, <4 x i16> undef, <4 x i32> <i32 undef, i32 undef, i32 1, i32 1>
+  %and = and <4 x i16> %shuffle, <i16 0, i16 0, i16 -1, i16 -1>
+  ret <4 x i16> %and
+}
+
+; AND does not fold to undef for undef operands, we cannot move it
+; across a shuffle with undef masks.
+define <4 x i16> @and_constant_mask_undef_2(<4 x i16> %add) {
+; CHECK-LABEL: @and_constant_mask_undef_2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x i16> [[ADD:%.*]], <4 x i16> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 undef>
+; CHECK-NEXT:    [[AND:%.*]] = and <4 x i16> [[SHUFFLE]], <i16 -1, i16 -1, i16 -1, i16 0>
+; CHECK-NEXT:    ret <4 x i16> [[AND]]
+;
+entry:
+  %shuffle = shufflevector <4 x i16> %add, <4 x i16> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 undef>
+  %and = and <4 x i16> %shuffle, <i16 -1, i16 -1, i16 -1, i16 -0>
+  ret <4 x i16> %and
+}
+
+; We can move the AND across the shuffle, as -1 (AND identity value) is used for undef lanes.
+define <4 x i16> @and_constant_mask_undef_3(<4 x i16> %add) {
+; CHECK-LABEL: @and_constant_mask_undef_3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <4 x i16> <i16 0, i16 0, i16 0, i16 undef>
+;
+entry:
+  %shuffle = shufflevector <4 x i16> %add, <4 x i16> undef, <4 x i32> <i32 0, i32 1, i32 1, i32 undef>
+  %and = and <4 x i16> %shuffle, <i16 0, i16 0, i16 0, i16 -1>
+  ret <4 x i16> %and
+}
+
+; We can move the AND across the shuffle, as -1 (AND identity value) is used for undef lanes.
+define <4 x i16> @and_constant_mask_undef_4(<4 x i16> %add) {
+; CHECK-LABEL: @and_constant_mask_undef_4(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = and <4 x i16> [[ADD:%.*]], <i16 9, i16 20, i16 undef, i16 undef>
+; CHECK-NEXT:    [[AND:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> undef, <4 x i32> <i32 0, i32 1, i32 1, i32 undef>
+; CHECK-NEXT:    ret <4 x i16> [[AND]]
+;
+entry:
+  %shuffle = shufflevector <4 x i16> %add, <4 x i16> undef, <4 x i32> <i32 0, i32 1, i32 1, i32 undef>
+  %and = and <4 x i16> %shuffle, <i16 9, i16 20, i16 20, i16 -1>
+  ret <4 x i16> %and
+}
+
+define <4 x i16> @and_constant_mask_not_undef(<4 x i16> %add) {
+; CHECK-LABEL: @and_constant_mask_not_undef(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = and <4 x i16> [[ADD:%.*]], <i16 undef, i16 -1, i16 0, i16 0>
+; CHECK-NEXT:    [[AND:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> undef, <4 x i32> <i32 2, i32 3, i32 1, i32 1>
+; CHECK-NEXT:    ret <4 x i16> [[AND]]
+;
+entry:
+  %shuffle = shufflevector <4 x i16> %add, <4 x i16> undef, <4 x i32> <i32 2, i32 3, i32 1, i32 1>
+  %and = and <4 x i16> %shuffle, <i16 0, i16 0, i16 -1, i16 -1>
+  ret <4 x i16> %and
+}
+
+; OR does not fold to undef for undef operands, we cannot move it
+; across a shuffle with undef masks.
+define <4 x i16> @or_constant_mask_undef(<4 x i16> %in) {
+; CHECK-LABEL: @or_constant_mask_undef(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x i16> [[IN:%.*]], <4 x i16> undef, <4 x i32> <i32 undef, i32 undef, i32 1, i32 1>
+; CHECK-NEXT:    [[OR:%.*]] = or <4 x i16> [[SHUFFLE]], <i16 -1, i16 -1, i16 0, i16 0>
+; CHECK-NEXT:    ret <4 x i16> [[OR]]
+;
+entry:
+  %shuffle = shufflevector <4 x i16> %in, <4 x i16> undef, <4 x i32> <i32 undef, i32 undef, i32 1, i32 1>
+  %or = or <4 x i16> %shuffle, <i16 -1, i16 -1, i16 0, i16 0>
+  ret <4 x i16> %or
+}
+
+; OR does not fold to undef for undef operands, we cannot move it
+; across a shuffle with undef masks.
+define <4 x i16> @or_constant_mask_undef_2(<4 x i16> %in) {
+; CHECK-LABEL: @or_constant_mask_undef_2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x i16> [[IN:%.*]], <4 x i16> undef, <4 x i32> <i32 undef, i32 1, i32 1, i32 undef>
+; CHECK-NEXT:    [[OR:%.*]] = or <4 x i16> [[SHUFFLE]], <i16 -1, i16 0, i16 0, i16 -1>
+; CHECK-NEXT:    ret <4 x i16> [[OR]]
+;
+entry:
+  %shuffle = shufflevector <4 x i16> %in, <4 x i16> undef, <4 x i32> <i32 undef, i32 1, i32 1, i32 undef>
+  %or = or <4 x i16> %shuffle, <i16 -1, i16 0, i16 0, i16 -1>
+  ret <4 x i16> %or
+}
+
+; We can move the OR across the shuffle, as 0 (OR identity value) is used for undef lanes.
+define <4 x i16> @or_constant_mask_undef_3(<4 x i16> %in) {
+; CHECK-LABEL: @or_constant_mask_undef_3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <4 x i16> <i16 undef, i16 -1, i16 -1, i16 undef>
+;
+entry:
+  %shuffle = shufflevector <4 x i16> %in, <4 x i16> undef, <4 x i32> <i32 undef, i32 1, i32 1, i32 undef>
+  %or = or <4 x i16> %shuffle, <i16 0, i16 -1, i16 -1, i16 0>
+  ret <4 x i16> %or
+}
+
+; We can move the OR across the shuffle, as 0 (OR identity value) is used for undef lanes.
+define <4 x i16> @or_constant_mask_undef_4(<4 x i16> %in) {
+; CHECK-LABEL: @or_constant_mask_undef_4(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = or <4 x i16> [[IN:%.*]], <i16 undef, i16 99, i16 undef, i16 undef>
+; CHECK-NEXT:    [[OR:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> undef, <4 x i32> <i32 undef, i32 1, i32 1, i32 undef>
+; CHECK-NEXT:    ret <4 x i16> [[OR]]
+;
+entry:
+  %shuffle = shufflevector <4 x i16> %in, <4 x i16> undef, <4 x i32> <i32 undef, i32 1, i32 1, i32 undef>
+  %or = or <4 x i16> %shuffle, <i16 0, i16 99, i16 99, i16 0>
+  ret <4 x i16> %or
+}
+
+define <4 x i16> @or_constant_mask_not_undef(<4 x i16> %in) {
+; CHECK-LABEL: @or_constant_mask_not_undef(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = or <4 x i16> [[IN:%.*]], <i16 undef, i16 -1, i16 0, i16 0>
+; CHECK-NEXT:    [[AND:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> undef, <4 x i32> <i32 2, i32 3, i32 1, i32 1>
+; CHECK-NEXT:    ret <4 x i16> [[AND]]
+;
+entry:
+  %shuffle = shufflevector <4 x i16> %in, <4 x i16> undef, <4 x i32> <i32 2, i32 3, i32 1, i32 1>
+  %and = or <4 x i16> %shuffle, <i16 0, i16 0, i16 -1, i16 -1>
+  ret <4 x i16> %and
+}
+
+define <4 x i16> @shl_constant_mask_undef(<4 x i16> %in) {
+; CHECK-LABEL: @shl_constant_mask_undef(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x i16> [[IN:%.*]], <4 x i16> undef, <4 x i32> <i32 0, i32 undef, i32 1, i32 1>
+; CHECK-NEXT:    [[SHL:%.*]] = shl <4 x i16> [[SHUFFLE]], <i16 10, i16 3, i16 0, i16 0>
+; CHECK-NEXT:    ret <4 x i16> [[SHL]]
+;
+entry:
+  %shuffle = shufflevector <4 x i16> %in, <4 x i16> undef, <4 x i32> <i32 0, i32 undef, i32 1, i32 1>
+  %shl = shl <4 x i16> %shuffle, <i16 10, i16 3, i16 0, i16 0>
+  ret <4 x i16> %shl
+}
+
+define <4 x i16> @add_constant_mask_undef(<4 x i16> %in) {
+; CHECK-LABEL: @add_constant_mask_undef(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ADD:%.*]] = shufflevector <4 x i16> [[IN:%.*]], <4 x i16> undef, <4 x i32> <i32 undef, i32 undef, i32 1, i32 1>
+; CHECK-NEXT:    ret <4 x i16> [[ADD]]
+;
+entry:
+  %shuffle = shufflevector <4 x i16> %in, <4 x i16> undef, <4 x i32> <i32 undef, i32 undef, i32 1, i32 1>
+  %add = add <4 x i16> %shuffle, <i16 10, i16 3, i16 0, i16 0>
+  ret <4 x i16> %add
+}
+
+define <4 x i16> @add_constant_mask_undef_2(<4 x i16> %in) {
+; CHECK-LABEL: @add_constant_mask_undef_2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = add <4 x i16> [[IN:%.*]], <i16 undef, i16 0, i16 3, i16 undef>
+; CHECK-NEXT:    [[ADD:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> undef, <4 x i32> <i32 undef, i32 2, i32 1, i32 1>
+; CHECK-NEXT:    ret <4 x i16> [[ADD]]
+;
+entry:
+  %shuffle = shufflevector <4 x i16> %in, <4 x i16> undef, <4 x i32> <i32 undef, i32 2, i32 1, i32 1>
+  %add = add <4 x i16> %shuffle, <i16 10, i16 3, i16 0, i16 0>
+  ret <4 x i16> %add
+}
+
+define <4 x i16> @sub_constant_mask_undef(<4 x i16> %in) {
+; CHECK-LABEL: @sub_constant_mask_undef(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SUB:%.*]] = shufflevector <4 x i16> [[IN:%.*]], <4 x i16> undef, <4 x i32> <i32 undef, i32 undef, i32 1, i32 1>
+; CHECK-NEXT:    ret <4 x i16> [[SUB]]
+;
+entry:
+  %shuffle = shufflevector <4 x i16> %in, <4 x i16> undef, <4 x i32> <i32 undef, i32 undef, i32 1, i32 1>
+  %sub = sub <4 x i16> %shuffle, <i16 10, i16 3, i16 0, i16 0>
+  ret <4 x i16> %sub
+}
+
+define <4 x i16> @sub_constant_mask_undef_2(<4 x i16> %in) {
+; CHECK-LABEL: @sub_constant_mask_undef_2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = add <4 x i16> [[IN:%.*]], <i16 undef, i16 0, i16 -10, i16 undef>
+; CHECK-NEXT:    [[SUB:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> undef, <4 x i32> <i32 1, i32 1, i32 2, i32 undef>
+; CHECK-NEXT:    ret <4 x i16> [[SUB]]
+;
+entry:
+  %shuffle = shufflevector <4 x i16> %in, <4 x i16> undef, <4 x i32> <i32 1, i32 1, i32 2, i32 undef>
+  %sub = sub <4 x i16> %shuffle, <i16 0, i16 0, i16 10, i16 99>
+  ret <4 x i16> %sub
+}
+
 define <2 x i32> @or_splat_constant(<2 x i32> %x) {
 ; CHECK-LABEL: @or_splat_constant(
 ; CHECK-NEXT:    [[TMP1:%.*]] = or <2 x i32> [[X:%.*]], <i32 42, i32 undef>
diff --git a/llvm/test/Transforms/InstCombine/vec_udiv_to_shift.ll b/llvm/test/Transforms/InstCombine/vec_udiv_to_shift.ll
new file mode 100644
index 000000000..2f9a0ef3e
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/vec_udiv_to_shift.ll
@@ -0,0 +1,16 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define <8 x i16> @udiv_vec8x16(<8 x i16> %var) {
+entry:
+; CHECK: lshr <8 x i16> %var, <i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5>
+%0 = udiv <8 x i16> %var, <i16 32, i16 32, i16 32, i16 32, i16 32, i16 32, i16 32, i16 32>
+ret <8 x i16> %0
+}
+
+define <4 x i32> @udiv_vec4x32(<4 x i32> %var) {
+entry:
+; CHECK: lshr <4 x i32> %var, <i32 4, i32 4, i32 4, i32 4>
+%0 = udiv <4 x i32> %var, <i32 16, i32 16, i32 16, i32 16>
+ret <4 x i32> %0
+}
diff --git a/llvm/test/Transforms/InstCombine/vector-urem.ll b/llvm/test/Transforms/InstCombine/vector-urem.ll
index 113451f84..c9b79b219 100644
--- a/llvm/test/Transforms/InstCombine/vector-urem.ll
+++ b/llvm/test/Transforms/InstCombine/vector-urem.ll
@@ -39,7 +39,7 @@ define <4 x i32> @test_v4i32_one(<4 x i32> %a0) {
 
 define <4 x i32> @test_v4i32_one_undef(<4 x i32> %a0) {
 ; CHECK-LABEL: @test_v4i32_one_undef(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne <4 x i32> [[A0:%.*]], <i32 1, i32 1, i32 1, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne <4 x i32> [[A0:%.*]], <i32 1, i32 1, i32 1, i32 1>
 ; CHECK-NEXT:    [[TMP2:%.*]] = zext <4 x i1> [[TMP1]] to <4 x i32>
 ; CHECK-NEXT:    ret <4 x i32> [[TMP2]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/widenable-conditions.ll b/llvm/test/Transforms/InstCombine/widenable-conditions.ll
new file mode 100644
index 000000000..4f3664724
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/widenable-conditions.ll
@@ -0,0 +1,156 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define i1 @test1(i1 %a, i1 %b) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[WC:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[LHS:%.*]] = and i1 [[WC]], [[B:%.*]]
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[LHS]], [[A:%.*]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  %wc = call i1 @llvm.experimental.widenable.condition()
+  %lhs = and i1 %b, %wc
+  %and = and i1 %lhs, %a
+  ret i1 %and
+}
+
+; Negative test - profitability of dropping WC from first use unclear
+define i1 @test1b(i1 %a, i1 %b) {
+; CHECK-LABEL: @test1b(
+; CHECK-NEXT:    [[WC:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[LHS:%.*]] = and i1 [[WC]], [[B:%.*]]
+; CHECK-NEXT:    call void @use(i1 [[LHS]])
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[LHS]], [[A:%.*]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  %wc = call i1 @llvm.experimental.widenable.condition()
+  %lhs = and i1 %b, %wc
+  call void @use(i1 %lhs)
+  %and = and i1 %lhs, %a
+  ret i1 %and
+}
+
+; multiple uses of A, B, WC doesn't change result
+define i1 @test1c(i1 %a, i1 %b) {
+; CHECK-LABEL: @test1c(
+; CHECK-NEXT:    call void @use(i1 [[A:%.*]])
+; CHECK-NEXT:    call void @use(i1 [[B:%.*]])
+; CHECK-NEXT:    [[WC:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    call void @use(i1 [[WC]])
+; CHECK-NEXT:    [[LHS:%.*]] = and i1 [[WC]], [[B]]
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[LHS]], [[A]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  call void @use(i1 %a)
+  call void @use(i1 %b)
+  %wc = call i1 @llvm.experimental.widenable.condition()
+  call void @use(i1 %wc)
+  %lhs = and i1 %b, %wc
+  %and = and i1 %lhs, %a
+  ret i1 %and
+}
+
+define i1 @test2(i1 %a, i1 %b) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[WC:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[LHS:%.*]] = and i1 [[WC]], [[B:%.*]]
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[LHS]], [[A:%.*]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  %wc = call i1 @llvm.experimental.widenable.condition()
+  %lhs = and i1 %wc, %b
+  %and = and i1 %lhs, %a
+  ret i1 %and
+}
+
+; To test the rhs side, an instruction on lhs to prevent complexity
+; canonicalization reducing to above.
+define i1 @test3(i1 %a, i1 %b, i1 %c) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    [[WC:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[LHS:%.*]] = and i1 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[RHS:%.*]] = and i1 [[WC]], [[C:%.*]]
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[LHS]], [[RHS]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  %wc = call i1 @llvm.experimental.widenable.condition()
+  %lhs = and i1 %a, %b
+  %rhs = and i1 %c, %wc
+  %and = and i1 %lhs, %rhs
+  ret i1 %and
+}
+
+define i1 @test4(i1 %a, i1 %b, i1 %c) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    [[WC:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[LHS:%.*]] = and i1 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[RHS:%.*]] = and i1 [[WC]], [[C:%.*]]
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[LHS]], [[RHS]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  %wc = call i1 @llvm.experimental.widenable.condition()
+  %lhs = and i1 %a, %b
+  %rhs = and i1 %wc, %c
+  %and = and i1 %lhs, %rhs
+  ret i1 %and
+}
+
+define i1 @test5(i1 %a, i1 %b) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    [[WC:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    ret i1 [[WC]]
+;
+  %wc = call i1 @llvm.experimental.widenable.condition()
+  %and = and i1 %wc, %wc
+  ret i1 %and
+}
+
+define i1 @test6(i1 %a, i1 %b) {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:    [[WC:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[WC2:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[WC]], [[WC2]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  %wc = call i1 @llvm.experimental.widenable.condition()
+  %wc2 = call i1 @llvm.experimental.widenable.condition()
+  %and = and i1 %wc, %wc2
+  ret i1 %and
+}
+
+define i1 @test7(i1 %a, i1 %b) {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:    [[WC:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    call void @use(i1 [[WC]])
+; CHECK-NEXT:    [[WC2:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[WC]], [[WC2]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  %wc = call i1 @llvm.experimental.widenable.condition()
+  call void @use(i1 %wc)
+  %wc2 = call i1 @llvm.experimental.widenable.condition()
+  %and = and i1 %wc, %wc2
+  ret i1 %and
+}
+
+define i1 @test8(i1 %a, i1 %b) {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT:    [[WC:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[WC2:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    call void @use(i1 [[WC2]])
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[WC]], [[WC2]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  %wc = call i1 @llvm.experimental.widenable.condition()
+  %wc2 = call i1 @llvm.experimental.widenable.condition()
+  call void @use(i1 %wc2)
+  %and = and i1 %wc, %wc2
+  ret i1 %and
+}
+
+
+declare void @use(i1)
+declare i1 @llvm.experimental.widenable.condition()
diff --git a/llvm/test/Transforms/InstCombine/with_overflow.ll b/llvm/test/Transforms/InstCombine/with_overflow.ll
index 04e8097ca..fc6102bc7 100644
--- a/llvm/test/Transforms/InstCombine/with_overflow.ll
+++ b/llvm/test/Transforms/InstCombine/with_overflow.ll
@@ -356,8 +356,7 @@ define i1 @uadd_res_ult_x(i32 %x, i32 %y, i1* %p) nounwind {
 ; CHECK-NEXT:    [[A:%.*]] = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 [[X:%.*]], i32 [[Y:%.*]])
 ; CHECK-NEXT:    [[B:%.*]] = extractvalue { i32, i1 } [[A]], 1
 ; CHECK-NEXT:    store i1 [[B]], i1* [[P:%.*]], align 1
-; CHECK-NEXT:    [[C:%.*]] = extractvalue { i32, i1 } [[A]], 0
-; CHECK-NEXT:    [[D:%.*]] = icmp ult i32 [[C]], [[X]]
+; CHECK-NEXT:    [[D:%.*]] = extractvalue { i32, i1 } [[A]], 1
 ; CHECK-NEXT:    ret i1 [[D]]
 ;
   %a = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 %x, i32 %y)
@@ -373,8 +372,7 @@ define i1 @uadd_res_ult_y(i32 %x, i32 %y, i1* %p) nounwind {
 ; CHECK-NEXT:    [[A:%.*]] = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 [[X:%.*]], i32 [[Y:%.*]])
 ; CHECK-NEXT:    [[B:%.*]] = extractvalue { i32, i1 } [[A]], 1
 ; CHECK-NEXT:    store i1 [[B]], i1* [[P:%.*]], align 1
-; CHECK-NEXT:    [[C:%.*]] = extractvalue { i32, i1 } [[A]], 0
-; CHECK-NEXT:    [[D:%.*]] = icmp ult i32 [[C]], [[Y]]
+; CHECK-NEXT:    [[D:%.*]] = extractvalue { i32, i1 } [[A]], 1
 ; CHECK-NEXT:    ret i1 [[D]]
 ;
   %a = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 %x, i32 %y)
@@ -391,8 +389,7 @@ define i1 @uadd_res_ugt_x(i32 %xx, i32 %y, i1* %p) nounwind {
 ; CHECK-NEXT:    [[A:%.*]] = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 [[X]], i32 [[Y:%.*]])
 ; CHECK-NEXT:    [[B:%.*]] = extractvalue { i32, i1 } [[A]], 1
 ; CHECK-NEXT:    store i1 [[B]], i1* [[P:%.*]], align 1
-; CHECK-NEXT:    [[C:%.*]] = extractvalue { i32, i1 } [[A]], 0
-; CHECK-NEXT:    [[D:%.*]] = icmp ugt i32 [[X]], [[C]]
+; CHECK-NEXT:    [[D:%.*]] = extractvalue { i32, i1 } [[A]], 1
 ; CHECK-NEXT:    ret i1 [[D]]
 ;
   %x = urem i32 42, %xx ; Thwart complexity-based canonicalization
@@ -410,8 +407,7 @@ define i1 @uadd_res_ugt_y(i32 %x, i32 %yy, i1* %p) nounwind {
 ; CHECK-NEXT:    [[A:%.*]] = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 [[X:%.*]], i32 [[Y]])
 ; CHECK-NEXT:    [[B:%.*]] = extractvalue { i32, i1 } [[A]], 1
 ; CHECK-NEXT:    store i1 [[B]], i1* [[P:%.*]], align 1
-; CHECK-NEXT:    [[C:%.*]] = extractvalue { i32, i1 } [[A]], 0
-; CHECK-NEXT:    [[D:%.*]] = icmp ugt i32 [[Y]], [[C]]
+; CHECK-NEXT:    [[D:%.*]] = extractvalue { i32, i1 } [[A]], 1
 ; CHECK-NEXT:    ret i1 [[D]]
 ;
   %y = urem i32 42, %yy ; Thwart complexity-based canonicalization
@@ -428,8 +424,7 @@ define i1 @uadd_res_ult_const(i32 %x, i1* %p) nounwind {
 ; CHECK-NEXT:    [[A:%.*]] = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 [[X:%.*]], i32 42)
 ; CHECK-NEXT:    [[B:%.*]] = extractvalue { i32, i1 } [[A]], 1
 ; CHECK-NEXT:    store i1 [[B]], i1* [[P:%.*]], align 1
-; CHECK-NEXT:    [[C:%.*]] = extractvalue { i32, i1 } [[A]], 0
-; CHECK-NEXT:    [[D:%.*]] = icmp ult i32 [[C]], 42
+; CHECK-NEXT:    [[D:%.*]] = extractvalue { i32, i1 } [[A]], 1
 ; CHECK-NEXT:    ret i1 [[D]]
 ;
   %a = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 %x, i32 42)
@@ -445,8 +440,7 @@ define i1 @uadd_res_ult_const_one(i32 %x, i1* %p) nounwind {
 ; CHECK-NEXT:    [[A:%.*]] = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 [[X:%.*]], i32 1)
 ; CHECK-NEXT:    [[B:%.*]] = extractvalue { i32, i1 } [[A]], 1
 ; CHECK-NEXT:    store i1 [[B]], i1* [[P:%.*]], align 1
-; CHECK-NEXT:    [[C:%.*]] = extractvalue { i32, i1 } [[A]], 0
-; CHECK-NEXT:    [[D:%.*]] = icmp eq i32 [[C]], 0
+; CHECK-NEXT:    [[D:%.*]] = extractvalue { i32, i1 } [[A]], 1
 ; CHECK-NEXT:    ret i1 [[D]]
 ;
   %a = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 %x, i32 1)
@@ -462,8 +456,7 @@ define i1 @uadd_res_ult_const_minus_one(i32 %x, i1* %p) nounwind {
 ; CHECK-NEXT:    [[A:%.*]] = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 [[X:%.*]], i32 -1)
 ; CHECK-NEXT:    [[B:%.*]] = extractvalue { i32, i1 } [[A]], 1
 ; CHECK-NEXT:    store i1 [[B]], i1* [[P:%.*]], align 1
-; CHECK-NEXT:    [[C:%.*]] = extractvalue { i32, i1 } [[A]], 0
-; CHECK-NEXT:    [[D:%.*]] = icmp ne i32 [[C]], -1
+; CHECK-NEXT:    [[D:%.*]] = extractvalue { i32, i1 } [[A]], 1
 ; CHECK-NEXT:    ret i1 [[D]]
 ;
   %a = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 %x, i32 -1)
diff --git a/llvm/test/Transforms/InstCombine/xor-of-icmps-with-extra-uses.ll b/llvm/test/Transforms/InstCombine/xor-of-icmps-with-extra-uses.ll
new file mode 100644
index 000000000..a795ead93
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/xor-of-icmps-with-extra-uses.ll
@@ -0,0 +1,163 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; These xor-of-icmps could be replaced with and-of-icmps, but %cond0 has extra
+; uses, so we don't consider it, even though some cases are freely invertible.
+
+; %cond0 is extra-used in select, which is freely invertible.
+define i1 @v0_select_of_consts(i32 %X, i32* %selected) {
+; CHECK-LABEL: @v0_select_of_consts(
+; CHECK-NEXT:    [[COND0_INV:%.*]] = icmp sgt i32 [[X:%.*]], 32767
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[COND0_INV]], i32 32767, i32 -32768
+; CHECK-NEXT:    store i32 [[SELECT]], i32* [[SELECTED:%.*]], align 4
+; CHECK-NEXT:    [[X_OFF:%.*]] = add i32 [[X]], 32767
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[X_OFF]], 65535
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cond0 = icmp sgt i32 %X, 32767
+  %cond1 = icmp sgt i32 %X, -32768
+  %select = select i1 %cond0, i32 32767, i32 -32768
+  store i32 %select, i32* %selected
+  %res = xor i1 %cond0, %cond1
+  ret i1 %res
+}
+define i1 @v1_select_of_var_and_const(i32 %X, i32 %Y, i32* %selected) {
+; CHECK-LABEL: @v1_select_of_var_and_const(
+; CHECK-NEXT:    [[COND0:%.*]] = icmp slt i32 [[X:%.*]], 32768
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[COND0]], i32 -32768, i32 [[Y:%.*]]
+; CHECK-NEXT:    store i32 [[SELECT]], i32* [[SELECTED:%.*]], align 4
+; CHECK-NEXT:    [[X_OFF:%.*]] = add i32 [[X]], 32767
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[X_OFF]], 65535
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cond0 = icmp sgt i32 %X, 32767
+  %cond1 = icmp sgt i32 %X, -32768
+  %select = select i1 %cond0, i32 %Y, i32 -32768
+  store i32 %select, i32* %selected
+  %res = xor i1 %cond0, %cond1
+  ret i1 %res
+}
+define i1 @v2_select_of_const_and_var(i32 %X, i32 %Y, i32* %selected) {
+; CHECK-LABEL: @v2_select_of_const_and_var(
+; CHECK-NEXT:    [[COND0_INV:%.*]] = icmp sgt i32 [[X:%.*]], 32767
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[COND0_INV]], i32 32767, i32 [[Y:%.*]]
+; CHECK-NEXT:    store i32 [[SELECT]], i32* [[SELECTED:%.*]], align 4
+; CHECK-NEXT:    [[X_OFF:%.*]] = add i32 [[X]], 32767
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[X_OFF]], 65535
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cond0 = icmp sgt i32 %X, 32767
+  %cond1 = icmp sgt i32 %X, -32768
+  %select = select i1 %cond0, i32 32767, i32 %Y
+  store i32 %select, i32* %selected
+  %res = xor i1 %cond0, %cond1
+  ret i1 %res
+}
+
+; Branch is also freely invertible
+define i1 @v3_branch(i32 %X, i32* %dst0, i32* %dst1) {
+; CHECK-LABEL: @v3_branch(
+; CHECK-NEXT:  begin:
+; CHECK-NEXT:    [[COND0:%.*]] = icmp slt i32 [[X:%.*]], 32768
+; CHECK-NEXT:    br i1 [[COND0]], label [[BB1:%.*]], label [[BB0:%.*]]
+; CHECK:       bb0:
+; CHECK-NEXT:    store i32 0, i32* [[DST0:%.*]], align 4
+; CHECK-NEXT:    br label [[END:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    store i32 0, i32* [[DST1:%.*]], align 4
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    [[X_OFF:%.*]] = add i32 [[X]], 32767
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[X_OFF]], 65535
+; CHECK-NEXT:    ret i1 [[TMP0]]
+;
+begin:
+  %cond0 = icmp sgt i32 %X, 32767
+  %cond1 = icmp sgt i32 %X, -32768
+  br i1 %cond0, label %bb0, label %bb1
+bb0:
+  store i32 0, i32* %dst0
+  br label %end
+bb1:
+  store i32 0, i32* %dst1
+  br label %end
+end:
+  %res = xor i1 %cond0, %cond1
+  ret i1 %res
+}
+
+; Can invert 'not'.
+define i1 @v4_not_store(i32 %X, i1* %not_cond) {
+; CHECK-LABEL: @v4_not_store(
+; CHECK-NEXT:    [[COND0:%.*]] = icmp slt i32 [[X:%.*]], 32768
+; CHECK-NEXT:    store i1 [[COND0]], i1* [[NOT_COND:%.*]], align 1
+; CHECK-NEXT:    [[X_OFF:%.*]] = add i32 [[X]], 32767
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[X_OFF]], 65535
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cond0 = icmp sgt i32 %X, 32767
+  %not_cond0 = xor i1 %cond0, -1
+  store i1 %not_cond0, i1* %not_cond
+  %cond1 = icmp sgt i32 %X, -32768
+  %select = select i1 %cond0, i32 32767, i32 -32768
+  %res = xor i1 %cond0, %cond1
+  ret i1 %res
+}
+
+; All extra uses are invertible.
+define i1 @v5_select_and_not(i32 %X, i32 %Y, i32* %selected, i1* %not_cond) {
+; CHECK-LABEL: @v5_select_and_not(
+; CHECK-NEXT:    [[COND0:%.*]] = icmp slt i32 [[X:%.*]], 32768
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[COND0]], i32 [[Y:%.*]], i32 32767
+; CHECK-NEXT:    store i1 [[COND0]], i1* [[NOT_COND:%.*]], align 1
+; CHECK-NEXT:    store i32 [[SELECT]], i32* [[SELECTED:%.*]], align 4
+; CHECK-NEXT:    [[X_OFF:%.*]] = add i32 [[X]], 32767
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[X_OFF]], 65535
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cond0 = icmp sgt i32 %X, 32767
+  %cond1 = icmp sgt i32 %X, -32768
+  %select = select i1 %cond0, i32 32767, i32 %Y
+  %not_cond0 = xor i1 %cond0, -1
+  store i1 %not_cond0, i1* %not_cond
+  store i32 %select, i32* %selected
+  %res = xor i1 %cond0, %cond1
+  ret i1 %res
+}
+
+; Not all extra uses are invertible.
+define i1 @n6_select_and_not(i32 %X, i32 %Y, i32* %selected, i1* %not_cond) {
+; CHECK-LABEL: @n6_select_and_not(
+; CHECK-NEXT:    [[COND0:%.*]] = icmp sgt i32 [[X:%.*]], 32767
+; CHECK-NEXT:    [[COND1:%.*]] = icmp sgt i32 [[X]], -32768
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[COND0]], i32 32767, i32 [[Y:%.*]]
+; CHECK-NEXT:    store i1 [[COND0]], i1* [[NOT_COND:%.*]], align 1
+; CHECK-NEXT:    store i32 [[SELECT]], i32* [[SELECTED:%.*]], align 4
+; CHECK-NEXT:    [[RES:%.*]] = xor i1 [[COND0]], [[COND1]]
+; CHECK-NEXT:    ret i1 [[RES]]
+;
+  %cond0 = icmp sgt i32 %X, 32767
+  %cond1 = icmp sgt i32 %X, -32768
+  %select = select i1 %cond0, i32 32767, i32 %Y
+  store i1 %cond0, i1* %not_cond
+  store i32 %select, i32* %selected
+  %res = xor i1 %cond0, %cond1
+  ret i1 %res
+}
+
+; Not freely invertible, would require extra 'not' instruction.
+define i1 @n7_store(i32 %X, i1* %cond) {
+; CHECK-LABEL: @n7_store(
+; CHECK-NEXT:    [[COND0:%.*]] = icmp sgt i32 [[X:%.*]], 32767
+; CHECK-NEXT:    store i1 [[COND0]], i1* [[COND:%.*]], align 1
+; CHECK-NEXT:    [[COND1:%.*]] = icmp sgt i32 [[X]], -32768
+; CHECK-NEXT:    [[RES:%.*]] = xor i1 [[COND0]], [[COND1]]
+; CHECK-NEXT:    ret i1 [[RES]]
+;
+  %cond0 = icmp sgt i32 %X, 32767
+  store i1 %cond0, i1* %cond
+  %cond1 = icmp sgt i32 %X, -32768
+  %select = select i1 %cond0, i32 32767, i32 -32768
+  %res = xor i1 %cond0, %cond1
+  ret i1 %res
+}
diff --git a/llvm/test/Transforms/InstCombine/zext-bool-add-sub.ll b/llvm/test/Transforms/InstCombine/zext-bool-add-sub.ll
index 86c206972..71fa9a795 100644
--- a/llvm/test/Transforms/InstCombine/zext-bool-add-sub.ll
+++ b/llvm/test/Transforms/InstCombine/zext-bool-add-sub.ll
@@ -5,9 +5,9 @@
 
 define i32 @a(i1 zeroext %x, i1 zeroext %y) {
 ; CHECK-LABEL: @a(
+; CHECK-NEXT:    [[CONV3_NEG:%.*]] = sext i1 [[Y:%.*]] to i32
 ; CHECK-NEXT:    [[SUB:%.*]] = select i1 [[X:%.*]], i32 2, i32 1
-; CHECK-NEXT:    [[TMP1:%.*]] = zext i1 [[Y:%.*]] to i32
-; CHECK-NEXT:    [[ADD:%.*]] = sub nsw i32 [[SUB]], [[TMP1]]
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[SUB]], [[CONV3_NEG]]
 ; CHECK-NEXT:    ret i32 [[ADD]]
 ;
   %conv = zext i1 %x to i32
@@ -317,8 +317,8 @@ define i8 @sext_sub_nuw(i8 %x, i1 %y) {
 
 define i32 @sextbool_add(i1 %c, i32 %x) {
 ; CHECK-LABEL: @sextbool_add(
-; CHECK-NEXT:    [[TMP1:%.*]] = zext i1 [[C:%.*]] to i32
-; CHECK-NEXT:    [[S:%.*]] = sub i32 [[X:%.*]], [[TMP1]]
+; CHECK-NEXT:    [[B:%.*]] = sext i1 [[C:%.*]] to i32
+; CHECK-NEXT:    [[S:%.*]] = add i32 [[B]], [[X:%.*]]
 ; CHECK-NEXT:    ret i32 [[S]]
 ;
   %b = sext i1 %c to i32
@@ -329,8 +329,8 @@ define i32 @sextbool_add(i1 %c, i32 %x) {
 define i32 @sextbool_add_commute(i1 %c, i32 %px) {
 ; CHECK-LABEL: @sextbool_add_commute(
 ; CHECK-NEXT:    [[X:%.*]] = urem i32 [[PX:%.*]], 42
-; CHECK-NEXT:    [[TMP1:%.*]] = zext i1 [[C:%.*]] to i32
-; CHECK-NEXT:    [[S:%.*]] = sub nsw i32 [[X]], [[TMP1]]
+; CHECK-NEXT:    [[B:%.*]] = sext i1 [[C:%.*]] to i32
+; CHECK-NEXT:    [[S:%.*]] = add nsw i32 [[X]], [[B]]
 ; CHECK-NEXT:    ret i32 [[S]]
 ;
   %x = urem i32 %px, 42 ; thwart complexity-based canonicalization
@@ -358,8 +358,8 @@ define i32 @sextbool_add_uses(i1 %c, i32 %x) {
 
 define <4 x i32> @sextbool_add_vector(<4 x i1> %c, <4 x i32> %x) {
 ; CHECK-LABEL: @sextbool_add_vector(
-; CHECK-NEXT:    [[TMP1:%.*]] = zext <4 x i1> [[C:%.*]] to <4 x i32>
-; CHECK-NEXT:    [[S:%.*]] = sub <4 x i32> [[X:%.*]], [[TMP1]]
+; CHECK-NEXT:    [[B:%.*]] = sext <4 x i1> [[C:%.*]] to <4 x i32>
+; CHECK-NEXT:    [[S:%.*]] = add <4 x i32> [[B]], [[X:%.*]]
 ; CHECK-NEXT:    ret <4 x i32> [[S]]
 ;
   %b = sext <4 x i1> %c to <4 x i32>
@@ -393,8 +393,8 @@ define i32 @zextbool_sub_uses(i1 %c, i32 %x) {
 
 define <4 x i32> @zextbool_sub_vector(<4 x i1> %c, <4 x i32> %x) {
 ; CHECK-LABEL: @zextbool_sub_vector(
-; CHECK-NEXT:    [[B:%.*]] = zext <4 x i1> [[C:%.*]] to <4 x i32>
-; CHECK-NEXT:    [[S:%.*]] = sub <4 x i32> [[X:%.*]], [[B]]
+; CHECK-NEXT:    [[TMP1:%.*]] = sext <4 x i1> [[C:%.*]] to <4 x i32>
+; CHECK-NEXT:    [[S:%.*]] = add <4 x i32> [[TMP1]], [[X:%.*]]
 ; CHECK-NEXT:    ret <4 x i32> [[S]]
 ;
   %b = zext <4 x i1> %c to <4 x i32>
diff --git a/llvm/test/Transforms/InstCombine/zext-or-icmp.ll b/llvm/test/Transforms/InstCombine/zext-or-icmp.ll
index afbe36da3..6c65032ce 100644
--- a/llvm/test/Transforms/InstCombine/zext-or-icmp.ll
+++ b/llvm/test/Transforms/InstCombine/zext-or-icmp.ll
@@ -15,7 +15,7 @@ define i8 @zext_or_icmp_icmp(i8 %a, i8 %b) {
 ; CHECK-NEXT:    %toBool2 = icmp eq i8 %b, 0
 ; CHECK-NEXT:    %toBool22 = zext i1 %toBool2 to i8
 ; CHECK-NEXT:    %1 = xor i8 %mask, 1
-; CHECK-NEXT:    %zext = or i8 %1, %toBool22
+; CHECK-NEXT:    %zext3 = or i8 %1, %toBool22
 ; CHECK-NEXT:    ret i8 %zext
 }
 
diff --git a/llvm/test/Transforms/InstCombine/zext.ll b/llvm/test/Transforms/InstCombine/zext.ll
index a53bf6cc0..1dbb9ffd7 100644
--- a/llvm/test/Transforms/InstCombine/zext.ll
+++ b/llvm/test/Transforms/InstCombine/zext.ll
@@ -3,7 +3,7 @@
 
 define i64 @test_sext_zext(i16 %A) {
 ; CHECK-LABEL: @test_sext_zext(
-; CHECK-NEXT:    [[C2:%.*]] = zext i16 %A to i64
+; CHECK-NEXT:    [[C2:%.*]] = zext i16 [[A:%.*]] to i64
 ; CHECK-NEXT:    ret i64 [[C2]]
 ;
   %c1 = zext i16 %A to i32
@@ -13,7 +13,7 @@ define i64 @test_sext_zext(i16 %A) {
 
 define <2 x i64> @test2(<2 x i1> %A) {
 ; CHECK-LABEL: @test2(
-; CHECK-NEXT:    [[XOR:%.*]] = xor <2 x i1> %A, <i1 true, i1 true>
+; CHECK-NEXT:    [[XOR:%.*]] = xor <2 x i1> [[A:%.*]], <i1 true, i1 true>
 ; CHECK-NEXT:    [[ZEXT:%.*]] = zext <2 x i1> [[XOR]] to <2 x i64>
 ; CHECK-NEXT:    ret <2 x i64> [[ZEXT]]
 ;
@@ -24,8 +24,8 @@ define <2 x i64> @test2(<2 x i1> %A) {
 
 define <2 x i64> @test3(<2 x i64> %A) {
 ; CHECK-LABEL: @test3(
-; CHECK-NEXT:    [[AND:%.*]] = and <2 x i64> %A, <i64 23, i64 42>
-; CHECK-NEXT:    ret <2 x i64> [[AND]]
+; CHECK-NEXT:    [[ZEXT:%.*]] = and <2 x i64> [[A:%.*]], <i64 23, i64 42>
+; CHECK-NEXT:    ret <2 x i64> [[ZEXT]]
 ;
   %trunc = trunc <2 x i64> %A to <2 x i32>
   %and = and <2 x i32> %trunc, <i32 23, i32 42>
@@ -35,9 +35,9 @@ define <2 x i64> @test3(<2 x i64> %A) {
 
 define <2 x i64> @test4(<2 x i64> %A) {
 ; CHECK-LABEL: @test4(
-; CHECK-NEXT:    [[AND:%.*]] = and <2 x i64> [[A:%.*]], <i64 23, i64 42>
-; CHECK-NEXT:    [[XOR:%.*]] = xor <2 x i64> [[AND]], <i64 23, i64 42>
-; CHECK-NEXT:    ret <2 x i64> [[XOR]]
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i64> [[A:%.*]], <i64 23, i64 42>
+; CHECK-NEXT:    [[ZEXT:%.*]] = xor <2 x i64> [[TMP1]], <i64 23, i64 42>
+; CHECK-NEXT:    ret <2 x i64> [[ZEXT]]
 ;
   %trunc = trunc <2 x i64> %A to <2 x i32>
   %and = and <2 x i32> %trunc, <i32 23, i32 42>
@@ -48,7 +48,7 @@ define <2 x i64> @test4(<2 x i64> %A) {
 
 define i64 @fold_xor_zext_sandwich(i1 %a) {
 ; CHECK-LABEL: @fold_xor_zext_sandwich(
-; CHECK-NEXT:    [[TMP1:%.*]] = xor i1 %a, true
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i1 [[A:%.*]], true
 ; CHECK-NEXT:    [[ZEXT2:%.*]] = zext i1 [[TMP1]] to i64
 ; CHECK-NEXT:    ret i64 [[ZEXT2]]
 ;
@@ -60,7 +60,7 @@ define i64 @fold_xor_zext_sandwich(i1 %a) {
 
 define <2 x i64> @fold_xor_zext_sandwich_vec(<2 x i1> %a) {
 ; CHECK-LABEL: @fold_xor_zext_sandwich_vec(
-; CHECK-NEXT:    [[TMP1:%.*]] = xor <2 x i1> %a, <i1 true, i1 true>
+; CHECK-NEXT:    [[TMP1:%.*]] = xor <2 x i1> [[A:%.*]], <i1 true, i1 true>
 ; CHECK-NEXT:    [[ZEXT2:%.*]] = zext <2 x i1> [[TMP1]] to <2 x i64>
 ; CHECK-NEXT:    ret <2 x i64> [[ZEXT2]]
 ;
@@ -74,8 +74,8 @@ define <2 x i64> @fold_xor_zext_sandwich_vec(<2 x i1> %a) {
 
 define i8 @fold_and_zext_icmp(i64 %a, i64 %b, i64 %c) {
 ; CHECK-LABEL: @fold_and_zext_icmp(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i64 %a, %b
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i64 %a, %c
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i64 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i64 [[A]], [[C:%.*]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = and i1 [[TMP1]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = zext i1 [[TMP3]] to i8
 ; CHECK-NEXT:    ret i8 [[TMP4]]
@@ -92,8 +92,8 @@ define i8 @fold_and_zext_icmp(i64 %a, i64 %b, i64 %c) {
 
 define i8 @fold_or_zext_icmp(i64 %a, i64 %b, i64 %c) {
 ; CHECK-LABEL: @fold_or_zext_icmp(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i64 %a, %b
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i64 %a, %c
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i64 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i64 [[A]], [[C:%.*]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = or i1 [[TMP1]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = zext i1 [[TMP3]] to i8
 ; CHECK-NEXT:    ret i8 [[TMP4]]
@@ -110,8 +110,8 @@ define i8 @fold_or_zext_icmp(i64 %a, i64 %b, i64 %c) {
 
 define i8 @fold_xor_zext_icmp(i64 %a, i64 %b, i64 %c) {
 ; CHECK-LABEL: @fold_xor_zext_icmp(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i64 %a, %b
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i64 %a, %c
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i64 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i64 [[A]], [[C:%.*]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = xor i1 [[TMP1]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = zext i1 [[TMP3]] to i8
 ; CHECK-NEXT:    ret i8 [[TMP4]]
@@ -129,10 +129,10 @@ define i8 @fold_xor_zext_icmp(i64 %a, i64 %b, i64 %c) {
 
 define i8 @fold_nested_logic_zext_icmp(i64 %a, i64 %b, i64 %c, i64 %d) {
 ; CHECK-LABEL: @fold_nested_logic_zext_icmp(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i64 %a, %b
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i64 %a, %c
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i64 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i64 [[A]], [[C:%.*]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = and i1 [[TMP1]], [[TMP2]]
-; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i64 %a, %d
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[A]], [[D:%.*]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = zext i1 [[TMP5]] to i8
 ; CHECK-NEXT:    ret i8 [[TMP6]]
@@ -152,7 +152,7 @@ define i8 @fold_nested_logic_zext_icmp(i64 %a, i64 %b, i64 %c, i64 %d) {
 
 define i1024 @sext_zext_apint1(i77 %A) {
 ; CHECK-LABEL: @sext_zext_apint1(
-; CHECK-NEXT:    [[C2:%.*]] = zext i77 %A to i1024
+; CHECK-NEXT:    [[C2:%.*]] = zext i77 [[A:%.*]] to i1024
 ; CHECK-NEXT:    ret i1024 [[C2]]
 ;
   %c1 = zext i77 %A to i533
@@ -164,7 +164,7 @@ define i1024 @sext_zext_apint1(i77 %A) {
 
 define i47 @sext_zext_apint2(i11 %A) {
 ; CHECK-LABEL: @sext_zext_apint2(
-; CHECK-NEXT:    [[C2:%.*]] = zext i11 %A to i47
+; CHECK-NEXT:    [[C2:%.*]] = zext i11 [[A:%.*]] to i47
 ; CHECK-NEXT:    ret i47 [[C2]]
 ;
   %c1 = zext i11 %A to i39
@@ -172,3 +172,231 @@ define i47 @sext_zext_apint2(i11 %A) {
   ret i47 %c2
 }
 
+declare void @use1(i1)
+declare void @use32(i32)
+
+define i32 @masked_bit_set(i32 %x, i32 %y) {
+; CHECK-LABEL: @masked_bit_set(
+; CHECK-NEXT:    [[SH1:%.*]] = shl i32 1, [[Y:%.*]]
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[SH1]], [[X:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[AND]], 0
+; CHECK-NEXT:    [[R:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %sh1 = shl i32 1, %y
+  %and = and i32 %sh1, %x
+  %cmp = icmp ne i32 %and, 0
+  %r = zext i1 %cmp to i32
+  ret i32 %r
+}
+
+define <2 x i32> @masked_bit_clear(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @masked_bit_clear(
+; CHECK-NEXT:    [[SH1:%.*]] = shl <2 x i32> <i32 1, i32 1>, [[Y:%.*]]
+; CHECK-NEXT:    [[AND:%.*]] = and <2 x i32> [[SH1]], [[X:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> [[AND]], zeroinitializer
+; CHECK-NEXT:    [[R:%.*]] = zext <2 x i1> [[CMP]] to <2 x i32>
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %sh1 = shl <2 x i32> <i32 1, i32 1>, %y
+  %and = and <2 x i32> %sh1, %x
+  %cmp = icmp eq <2 x i32> %and, zeroinitializer
+  %r = zext <2 x i1> %cmp to <2 x i32>
+  ret <2 x i32> %r
+}
+
+define <2 x i32> @masked_bit_set_commute(<2 x i32> %px, <2 x i32> %y) {
+; CHECK-LABEL: @masked_bit_set_commute(
+; CHECK-NEXT:    [[X:%.*]] = srem <2 x i32> <i32 42, i32 3>, [[PX:%.*]]
+; CHECK-NEXT:    [[SH1:%.*]] = shl <2 x i32> <i32 1, i32 1>, [[Y:%.*]]
+; CHECK-NEXT:    [[AND:%.*]] = and <2 x i32> [[X]], [[SH1]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> [[AND]], zeroinitializer
+; CHECK-NEXT:    [[R:%.*]] = zext <2 x i1> [[CMP]] to <2 x i32>
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %x = srem <2 x i32> <i32 42, i32 3>, %px ; thwart complexity-based canonicalization
+  %sh1 = shl <2 x i32> <i32 1, i32 1>, %y
+  %and = and <2 x i32> %x, %sh1
+  %cmp = icmp ne <2 x i32> %and, zeroinitializer
+  %r = zext <2 x i1> %cmp to <2 x i32>
+  ret <2 x i32> %r
+}
+
+define i32 @masked_bit_clear_commute(i32 %px, i32 %y) {
+; CHECK-LABEL: @masked_bit_clear_commute(
+; CHECK-NEXT:    [[X:%.*]] = srem i32 42, [[PX:%.*]]
+; CHECK-NEXT:    [[SH1:%.*]] = shl i32 1, [[Y:%.*]]
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X]], [[SH1]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    [[R:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %x = srem i32 42, %px ; thwart complexity-based canonicalization
+  %sh1 = shl i32 1, %y
+  %and = and i32 %x, %sh1
+  %cmp = icmp eq i32 %and, 0
+  %r = zext i1 %cmp to i32
+  ret i32 %r
+}
+
+define i32 @masked_bit_set_use1(i32 %x, i32 %y) {
+; CHECK-LABEL: @masked_bit_set_use1(
+; CHECK-NEXT:    [[SH1:%.*]] = shl i32 1, [[Y:%.*]]
+; CHECK-NEXT:    call void @use32(i32 [[SH1]])
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[SH1]], [[X:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[AND]], 0
+; CHECK-NEXT:    [[R:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %sh1 = shl i32 1, %y
+  call void @use32(i32 %sh1)
+  %and = and i32 %sh1, %x
+  %cmp = icmp ne i32 %and, 0
+  %r = zext i1 %cmp to i32
+  ret i32 %r
+}
+
+define i32 @masked_bit_set_use2(i32 %x, i32 %y) {
+; CHECK-LABEL: @masked_bit_set_use2(
+; CHECK-NEXT:    [[SH1:%.*]] = shl i32 1, [[Y:%.*]]
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[SH1]], [[X:%.*]]
+; CHECK-NEXT:    call void @use32(i32 [[AND]])
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[AND]], 0
+; CHECK-NEXT:    [[R:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %sh1 = shl i32 1, %y
+  %and = and i32 %sh1, %x
+  call void @use32(i32 %and)
+  %cmp = icmp ne i32 %and, 0
+  %r = zext i1 %cmp to i32
+  ret i32 %r
+}
+
+define i32 @masked_bit_set_use3(i32 %x, i32 %y) {
+; CHECK-LABEL: @masked_bit_set_use3(
+; CHECK-NEXT:    [[SH1:%.*]] = shl i32 1, [[Y:%.*]]
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[SH1]], [[X:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[AND]], 0
+; CHECK-NEXT:    call void @use1(i1 [[CMP]])
+; CHECK-NEXT:    [[R:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %sh1 = shl i32 1, %y
+  %and = and i32 %sh1, %x
+  %cmp = icmp ne i32 %and, 0
+  call void @use1(i1 %cmp)
+  %r = zext i1 %cmp to i32
+  ret i32 %r
+}
+
+define i32 @masked_bit_clear_use1(i32 %x, i32 %y) {
+; CHECK-LABEL: @masked_bit_clear_use1(
+; CHECK-NEXT:    [[SH1:%.*]] = shl i32 1, [[Y:%.*]]
+; CHECK-NEXT:    call void @use32(i32 [[SH1]])
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[SH1]], [[X:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    [[R:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %sh1 = shl i32 1, %y
+  call void @use32(i32 %sh1)
+  %and = and i32 %sh1, %x
+  %cmp = icmp eq i32 %and, 0
+  %r = zext i1 %cmp to i32
+  ret i32 %r
+}
+
+define i32 @masked_bit_clear_use2(i32 %x, i32 %y) {
+; CHECK-LABEL: @masked_bit_clear_use2(
+; CHECK-NEXT:    [[SH1:%.*]] = shl i32 1, [[Y:%.*]]
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[SH1]], [[X:%.*]]
+; CHECK-NEXT:    call void @use32(i32 [[AND]])
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    [[R:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %sh1 = shl i32 1, %y
+  %and = and i32 %sh1, %x
+  call void @use32(i32 %and)
+  %cmp = icmp eq i32 %and, 0
+  %r = zext i1 %cmp to i32
+  ret i32 %r
+}
+
+define i32 @masked_bit_clear_use3(i32 %x, i32 %y) {
+; CHECK-LABEL: @masked_bit_clear_use3(
+; CHECK-NEXT:    [[SH1:%.*]] = shl i32 1, [[Y:%.*]]
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[SH1]], [[X:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    call void @use1(i1 [[CMP]])
+; CHECK-NEXT:    [[R:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %sh1 = shl i32 1, %y
+  %and = and i32 %sh1, %x
+  %cmp = icmp eq i32 %and, 0
+  call void @use1(i1 %cmp)
+  %r = zext i1 %cmp to i32
+  ret i32 %r
+}
+
+define i32 @masked_bits_set(i32 %x, i32 %y) {
+; CHECK-LABEL: @masked_bits_set(
+; CHECK-NEXT:    [[SH1:%.*]] = shl i32 3, [[Y:%.*]]
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[SH1]], [[X:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[AND]], 0
+; CHECK-NEXT:    [[R:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %sh1 = shl i32 3, %y
+  %and = and i32 %sh1, %x
+  %cmp = icmp ne i32 %and, 0
+  %r = zext i1 %cmp to i32
+  ret i32 %r
+}
+
+define i32 @div_bit_set(i32 %x, i32 %y) {
+; CHECK-LABEL: @div_bit_set(
+; CHECK-NEXT:    [[SH1:%.*]] = shl i32 1, [[Y:%.*]]
+; CHECK-NEXT:    [[AND:%.*]] = sdiv i32 [[SH1]], [[X:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[AND]], 0
+; CHECK-NEXT:    [[R:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %sh1 = shl i32 1, %y
+  %and = sdiv i32 %sh1, %x
+  %cmp = icmp ne i32 %and, 0
+  %r = zext i1 %cmp to i32
+  ret i32 %r
+}
+
+define i32 @masked_bit_set_nonzero_cmp(i32 %x, i32 %y) {
+; CHECK-LABEL: @masked_bit_set_nonzero_cmp(
+; CHECK-NEXT:    [[SH1:%.*]] = shl i32 1, [[Y:%.*]]
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[SH1]], [[X:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[AND]], 1
+; CHECK-NEXT:    [[R:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %sh1 = shl i32 1, %y
+  %and = and i32 %sh1, %x
+  %cmp = icmp ne i32 %and, 1
+  %r = zext i1 %cmp to i32
+  ret i32 %r
+}
+
+define i32 @masked_bit_wrong_pred(i32 %x, i32 %y) {
+; CHECK-LABEL: @masked_bit_wrong_pred(
+; CHECK-NEXT:    [[SH1:%.*]] = shl i32 1, [[Y:%.*]]
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[SH1]], [[X:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[AND]], 0
+; CHECK-NEXT:    [[R:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %sh1 = shl i32 1, %y
+  %and = and i32 %sh1, %x
+  %cmp = icmp sgt i32 %and, 0
+  %r = zext i1 %cmp to i32
+  ret i32 %r
+}
diff --git a/llvm/test/Transforms/InstMerge/st_sink_split_bb.ll b/llvm/test/Transforms/InstMerge/st_sink_split_bb.ll
new file mode 100644
index 000000000..ee4d5647d
--- /dev/null
+++ b/llvm/test/Transforms/InstMerge/st_sink_split_bb.ll
@@ -0,0 +1,94 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; Test to make sure that a new block is inserted if we
+; have more than 2 predecessors for the block we're going to sink to.
+; RUN: opt -basicaa -memdep -mldst-motion -S < %s | FileCheck %s --check-prefix=CHECK-NO
+; RUN: opt -debug-pass-manager -aa-pipeline=basic-aa -passes='require<memdep>,mldst-motion' -S < %s 2>&1 | FileCheck %s --check-prefixes=CHECK-NO,CHECK-INV-NO
+; RUN: opt -debug-pass-manager -aa-pipeline=basic-aa -passes='require<memdep>,mldst-motion<split-footer-bb>' -S < %s 2>&1 | FileCheck %s --check-prefixes=CHECK-YES,CHECK-INV-YES
+target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+
+; When passing split-footer-bb to MLSM, it invalidates CFG analyses
+; CHECK-INV-NO: Running pass: MergedLoadStoreMotionPass
+; CHECK-INV-NO-NOT: Invalidating analysis: DominatorTreeAnalysis
+; CHECK-INV-YES: Running pass: MergedLoadStoreMotionPass
+; CHECK-INV-YES: Invalidating analysis: DominatorTreeAnalysis
+
+; Function Attrs: nounwind uwtable
+define dso_local void @st_sink_split_bb(i32* nocapture %arg, i32* nocapture %arg1, i1 zeroext %arg2, i1 zeroext %arg3) local_unnamed_addr {
+; CHECK-NO-LABEL: @st_sink_split_bb(
+; CHECK-NO-NEXT:  bb:
+; CHECK-NO-NEXT:    br i1 [[ARG2:%.*]], label [[BB4:%.*]], label [[BB5:%.*]]
+; CHECK-NO:       bb4:
+; CHECK-NO-NEXT:    store i32 1, i32* [[ARG:%.*]], align 4
+; CHECK-NO-NEXT:    br label [[BB9:%.*]]
+; CHECK-NO:       bb5:
+; CHECK-NO-NEXT:    br i1 [[ARG3:%.*]], label [[BB6:%.*]], label [[BB7:%.*]]
+; CHECK-NO:       bb6:
+; CHECK-NO-NEXT:    store i32 2, i32* [[ARG]], align 4
+; CHECK-NO-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, i32* [[ARG1:%.*]], i64 1
+; CHECK-NO-NEXT:    store i32 3, i32* [[TMP1]], align 4
+; CHECK-NO-NEXT:    [[TMP:%.*]] = getelementptr inbounds i32, i32* [[ARG1]], i64 2
+; CHECK-NO-NEXT:    store i32 3, i32* [[TMP]], align 4
+; CHECK-NO-NEXT:    br label [[BB9]]
+; CHECK-NO:       bb7:
+; CHECK-NO-NEXT:    store i32 3, i32* [[ARG]], align 4
+; CHECK-NO-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, i32* [[ARG1]], i64 1
+; CHECK-NO-NEXT:    store i32 3, i32* [[TMP2]], align 4
+; CHECK-NO-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, i32* [[ARG1]], i64 2
+; CHECK-NO-NEXT:    store i32 3, i32* [[TMP8]], align 4
+; CHECK-NO-NEXT:    br label [[BB9]]
+; CHECK-NO:       bb9:
+; CHECK-NO-NEXT:    ret void
+;
+; CHECK-YES-LABEL: @st_sink_split_bb(
+; CHECK-YES-NEXT:  bb:
+; CHECK-YES-NEXT:    br i1 [[ARG2:%.*]], label [[BB4:%.*]], label [[BB5:%.*]]
+; CHECK-YES:       bb4:
+; CHECK-YES-NEXT:    store i32 1, i32* [[ARG:%.*]], align 4
+; CHECK-YES-NEXT:    br label [[BB9:%.*]]
+; CHECK-YES:       bb5:
+; CHECK-YES-NEXT:    br i1 [[ARG3:%.*]], label [[BB6:%.*]], label [[BB7:%.*]]
+; CHECK-YES:       bb6:
+; CHECK-YES-NEXT:    store i32 2, i32* [[ARG]], align 4
+; CHECK-YES-NEXT:    br label [[BB9_SINK_SPLIT:%.*]]
+; CHECK-YES:       bb7:
+; CHECK-YES-NEXT:    store i32 3, i32* [[ARG]], align 4
+; CHECK-YES-NEXT:    br label [[BB9_SINK_SPLIT]]
+; CHECK-YES:       bb9.sink.split:
+; CHECK-YES-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, i32* [[ARG1:%.*]], i64 1
+; CHECK-YES-NEXT:    store i32 3, i32* [[TMP0]], align 4
+; CHECK-YES-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, i32* [[ARG1]], i64 2
+; CHECK-YES-NEXT:    store i32 3, i32* [[TMP1]], align 4
+; CHECK-YES-NEXT:    br label [[BB9]]
+; CHECK-YES:       bb9:
+; CHECK-YES-NEXT:    ret void
+;
+bb:
+  br i1 %arg2, label %bb4, label %bb5
+
+bb4:                                              ; preds = %bb
+  store i32 1, i32* %arg, align 4
+  br label %bb9
+
+bb5:                                              ; preds = %bb
+  br i1 %arg3, label %bb6, label %bb7
+
+bb6:                                              ; preds = %bb5
+  store i32 2, i32* %arg, align 4
+  %tmp1 = getelementptr inbounds i32, i32* %arg1, i64 1
+  store i32 3, i32* %tmp1, align 4
+  %tmp = getelementptr inbounds i32, i32* %arg1, i64 2
+  store i32 3, i32* %tmp, align 4
+  br label %bb9
+
+bb7:                                              ; preds = %bb5
+  store i32 3, i32* %arg, align 4
+  %tmp2 = getelementptr inbounds i32, i32* %arg1, i64 1
+  store i32 3, i32* %tmp2, align 4
+  %tmp8 = getelementptr inbounds i32, i32* %arg1, i64 2
+  store i32 3, i32* %tmp8, align 4
+  br label %bb9
+
+
+bb9:                                              ; preds = %bb7, %bb6, %bb4
+  ret void
+}
diff --git a/llvm/test/Transforms/InstSimplify/assume-non-zero.ll b/llvm/test/Transforms/InstSimplify/assume-non-zero.ll
new file mode 100644
index 000000000..a506b2559
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/assume-non-zero.ll
@@ -0,0 +1,233 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare void @llvm.assume(i1) #1
+
+define i1 @nonnull0_true(i8* %x) {
+; CHECK-LABEL: @nonnull0_true(
+; CHECK-NEXT:    [[A:%.*]] = icmp ne i8* [[X:%.*]], null
+; CHECK-NEXT:    call void @llvm.assume(i1 [[A]])
+; CHECK-NEXT:    ret i1 true
+;
+  %a = icmp ne i8* %x, null
+  call void @llvm.assume(i1 %a)
+  %q = icmp ne i8* %x, null
+  ret i1 %q
+}
+
+define i1 @nonnull1_true(i8* %x) {
+; CHECK-LABEL: @nonnull1_true(
+; CHECK-NEXT:    [[INTPTR:%.*]] = ptrtoint i8* [[X:%.*]] to i64
+; CHECK-NEXT:    [[A:%.*]] = icmp ne i64 [[INTPTR]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[A]])
+; CHECK-NEXT:    ret i1 true
+;
+  %intptr = ptrtoint i8* %x to i64
+  %a = icmp ne i64 %intptr, 0
+  call void @llvm.assume(i1 %a)
+  %q = icmp ne i8* %x, null
+  ret i1 %q
+}
+
+define i1 @nonnull2_true(i8 %x, i8 %y) {
+; CHECK-LABEL: @nonnull2_true(
+; CHECK-NEXT:    [[A:%.*]] = icmp ugt i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @llvm.assume(i1 [[A]])
+; CHECK-NEXT:    ret i1 true
+;
+  %a = icmp ugt i8 %x, %y
+  call void @llvm.assume(i1 %a)
+  %q = icmp ne i8 %x, 0
+  ret i1 %q
+}
+define i1 @nonnull2_true_swapped(i8 %x, i8 %y) {
+; CHECK-LABEL: @nonnull2_true_swapped(
+; CHECK-NEXT:    [[A:%.*]] = icmp ult i8 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    call void @llvm.assume(i1 [[A]])
+; CHECK-NEXT:    ret i1 true
+;
+  %a = icmp ult i8 %y, %x
+  call void @llvm.assume(i1 %a)
+  %q = icmp ne i8 %x, 0
+  ret i1 %q
+}
+
+define i1 @nonnull3_unknown(i8 %x) {
+; CHECK-LABEL: @nonnull3_unknown(
+; CHECK-NEXT:    [[Q:%.*]] = icmp ne i8 [[X:%.*]], 0
+; CHECK-NEXT:    ret i1 [[Q]]
+;
+  %a = icmp uge i8 %x, 0
+  call void @llvm.assume(i1 %a)
+  %q = icmp ne i8 %x, 0
+  ret i1 %q
+}
+define i1 @nonnull4_true(i8 %x) {
+; CHECK-LABEL: @nonnull4_true(
+; CHECK-NEXT:    [[A:%.*]] = icmp uge i8 [[X:%.*]], 1
+; CHECK-NEXT:    call void @llvm.assume(i1 [[A]])
+; CHECK-NEXT:    ret i1 true
+;
+  %a = icmp uge i8 %x, 1
+  call void @llvm.assume(i1 %a)
+  %q = icmp ne i8 %x, 0
+  ret i1 %q
+}
+
+define i1 @nonnull5_unknown(i8 %x) {
+; CHECK-LABEL: @nonnull5_unknown(
+; CHECK-NEXT:    [[A:%.*]] = icmp sgt i8 [[X:%.*]], -1
+; CHECK-NEXT:    call void @llvm.assume(i1 [[A]])
+; CHECK-NEXT:    [[Q:%.*]] = icmp ne i8 [[X]], 0
+; CHECK-NEXT:    ret i1 [[Q]]
+;
+  %a = icmp sgt i8 %x, -1
+  call void @llvm.assume(i1 %a)
+  %q = icmp ne i8 %x, 0
+  ret i1 %q
+}
+define i1 @nonnull6_true(i8 %x) {
+; CHECK-LABEL: @nonnull6_true(
+; CHECK-NEXT:    [[A:%.*]] = icmp sgt i8 [[X:%.*]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[A]])
+; CHECK-NEXT:    ret i1 true
+;
+  %a = icmp sgt i8 %x, 0
+  call void @llvm.assume(i1 %a)
+  %q = icmp ne i8 %x, 0
+  ret i1 %q
+}
+define i1 @nonnull7_true(i8 %x) {
+; CHECK-LABEL: @nonnull7_true(
+; CHECK-NEXT:    [[A:%.*]] = icmp sgt i8 [[X:%.*]], 1
+; CHECK-NEXT:    call void @llvm.assume(i1 [[A]])
+; CHECK-NEXT:    ret i1 true
+;
+  %a = icmp sgt i8 %x, 1
+  call void @llvm.assume(i1 %a)
+  %q = icmp ne i8 %x, 0
+  ret i1 %q
+}
+
+define i1 @nonnull8_unknown(i8 %x) {
+; CHECK-LABEL: @nonnull8_unknown(
+; CHECK-NEXT:    [[A:%.*]] = icmp sge i8 [[X:%.*]], -1
+; CHECK-NEXT:    call void @llvm.assume(i1 [[A]])
+; CHECK-NEXT:    [[Q:%.*]] = icmp ne i8 [[X]], 0
+; CHECK-NEXT:    ret i1 [[Q]]
+;
+  %a = icmp sge i8 %x, -1
+  call void @llvm.assume(i1 %a)
+  %q = icmp ne i8 %x, 0
+  ret i1 %q
+}
+define i1 @nonnull9_unknown(i8 %x) {
+; CHECK-LABEL: @nonnull9_unknown(
+; CHECK-NEXT:    [[A:%.*]] = icmp sge i8 [[X:%.*]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[A]])
+; CHECK-NEXT:    [[Q:%.*]] = icmp ne i8 [[X]], 0
+; CHECK-NEXT:    ret i1 [[Q]]
+;
+  %a = icmp sge i8 %x, 0
+  call void @llvm.assume(i1 %a)
+  %q = icmp ne i8 %x, 0
+  ret i1 %q
+}
+define i1 @nonnull10_true(i8 %x) {
+; CHECK-LABEL: @nonnull10_true(
+; CHECK-NEXT:    [[A:%.*]] = icmp sge i8 [[X:%.*]], 1
+; CHECK-NEXT:    call void @llvm.assume(i1 [[A]])
+; CHECK-NEXT:    ret i1 true
+;
+  %a = icmp sge i8 %x, 1
+  call void @llvm.assume(i1 %a)
+  %q = icmp ne i8 %x, 0
+  ret i1 %q
+}
+
+define i1 @nonnull11_true(i8 %x) {
+; CHECK-LABEL: @nonnull11_true(
+; CHECK-NEXT:    [[A:%.*]] = icmp slt i8 [[X:%.*]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[A]])
+; CHECK-NEXT:    ret i1 true
+;
+  %a = icmp slt i8 %x, 0
+  call void @llvm.assume(i1 %a)
+  %q = icmp ne i8 %x, 0
+  ret i1 %q
+}
+define i1 @nonnull12_unknown(i8 %x) {
+; CHECK-LABEL: @nonnull12_unknown(
+; CHECK-NEXT:    [[A:%.*]] = icmp slt i8 [[X:%.*]], 1
+; CHECK-NEXT:    call void @llvm.assume(i1 [[A]])
+; CHECK-NEXT:    [[Q:%.*]] = icmp ne i8 [[X]], 0
+; CHECK-NEXT:    ret i1 [[Q]]
+;
+  %a = icmp slt i8 %x, 1
+  call void @llvm.assume(i1 %a)
+  %q = icmp ne i8 %x, 0
+  ret i1 %q
+}
+define i1 @nonnull13_unknown(i8 %x) {
+; CHECK-LABEL: @nonnull13_unknown(
+; CHECK-NEXT:    [[A:%.*]] = icmp slt i8 [[X:%.*]], 2
+; CHECK-NEXT:    call void @llvm.assume(i1 [[A]])
+; CHECK-NEXT:    [[Q:%.*]] = icmp ne i8 [[X]], 0
+; CHECK-NEXT:    ret i1 [[Q]]
+;
+  %a = icmp slt i8 %x, 2
+  call void @llvm.assume(i1 %a)
+  %q = icmp ne i8 %x, 0
+  ret i1 %q
+}
+
+define i1 @nonnull14_true(i8 %x) {
+; CHECK-LABEL: @nonnull14_true(
+; CHECK-NEXT:    [[A:%.*]] = icmp sle i8 [[X:%.*]], -1
+; CHECK-NEXT:    call void @llvm.assume(i1 [[A]])
+; CHECK-NEXT:    ret i1 true
+;
+  %a = icmp sle i8 %x, -1
+  call void @llvm.assume(i1 %a)
+  %q = icmp ne i8 %x, 0
+  ret i1 %q
+}
+define i1 @nonnull15_unknown(i8 %x) {
+; CHECK-LABEL: @nonnull15_unknown(
+; CHECK-NEXT:    [[A:%.*]] = icmp sle i8 [[X:%.*]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[A]])
+; CHECK-NEXT:    [[Q:%.*]] = icmp ne i8 [[X]], 0
+; CHECK-NEXT:    ret i1 [[Q]]
+;
+  %a = icmp sle i8 %x, 0
+  call void @llvm.assume(i1 %a)
+  %q = icmp ne i8 %x, 0
+  ret i1 %q
+}
+define i1 @nonnull16_unknown(i8 %x) {
+; CHECK-LABEL: @nonnull16_unknown(
+; CHECK-NEXT:    [[A:%.*]] = icmp sle i8 [[X:%.*]], 1
+; CHECK-NEXT:    call void @llvm.assume(i1 [[A]])
+; CHECK-NEXT:    [[Q:%.*]] = icmp ne i8 [[X]], 0
+; CHECK-NEXT:    ret i1 [[Q]]
+;
+  %a = icmp sle i8 %x, 1
+  call void @llvm.assume(i1 %a)
+  %q = icmp ne i8 %x, 0
+  ret i1 %q
+}
+define i1 @nonnull17_unknown(i8 %x) {
+; CHECK-LABEL: @nonnull17_unknown(
+; CHECK-NEXT:    [[A:%.*]] = icmp sle i8 [[X:%.*]], 2
+; CHECK-NEXT:    call void @llvm.assume(i1 [[A]])
+; CHECK-NEXT:    [[Q:%.*]] = icmp ne i8 [[X]], 0
+; CHECK-NEXT:    ret i1 [[Q]]
+;
+  %a = icmp sle i8 %x, 2
+  call void @llvm.assume(i1 %a)
+  %q = icmp ne i8 %x, 0
+  ret i1 %q
+}
diff --git a/llvm/test/Transforms/InstSimplify/assume.ll b/llvm/test/Transforms/InstSimplify/assume.ll
index 99b5759e9..157d8de8f 100644
--- a/llvm/test/Transforms/InstSimplify/assume.ll
+++ b/llvm/test/Transforms/InstSimplify/assume.ll
@@ -33,15 +33,15 @@ define i64 @PR31809() !dbg !7 {
 }
 
 ; Similar to above: there's no way to know which assumption is truthful,
-; so just don't crash. The second icmp+assume gets processed later, so that
-; determines the return value.
+; so just don't crash.
 
 define i8 @conflicting_assumptions(i8 %x) !dbg !10 {
 ; CHECK-LABEL: @conflicting_assumptions(
+; CHECK-NEXT:    [[ADD:%.*]] = add i8 [[X:%.*]], 1, !dbg !10
 ; CHECK-NEXT:    call void @llvm.assume(i1 false)
-; CHECK-NEXT:    [[COND2:%.*]] = icmp eq i8 %x, 4
+; CHECK-NEXT:    [[COND2:%.*]] = icmp eq i8 [[X]], 4
 ; CHECK-NEXT:    call void @llvm.assume(i1 [[COND2]])
-; CHECK-NEXT:    ret i8 5
+; CHECK-NEXT:    ret i8 [[ADD]]
 ;
   %add = add i8 %x, 1, !dbg !11
   %cond1 = icmp eq i8 %x, 3
diff --git a/llvm/test/Transforms/InstSimplify/call.ll b/llvm/test/Transforms/InstSimplify/call.ll
index 27497e5e8..15c56d233 100644
--- a/llvm/test/Transforms/InstSimplify/call.ll
+++ b/llvm/test/Transforms/InstSimplify/call.ll
@@ -742,3 +742,238 @@ define i9 @fshr_shift_undef(i9 %x, i9 %y) {
   ret i9 %r
 }
 
+declare double @llvm.fma.f64(double,double,double)
+declare double @llvm.fmuladd.f64(double,double,double)
+
+define double @fma_undef_op0(double %x, double %y) {
+; CHECK-LABEL: @fma_undef_op0(
+; CHECK-NEXT:    ret double 0x7FF8000000000000
+;
+  %r = call double @llvm.fma.f64(double undef, double %x, double %y)
+  ret double %r
+}
+
+define double @fma_undef_op1(double %x, double %y) {
+; CHECK-LABEL: @fma_undef_op1(
+; CHECK-NEXT:    ret double 0x7FF8000000000000
+;
+  %r = call double @llvm.fma.f64(double %x, double undef, double %y)
+  ret double %r
+}
+
+define double @fma_undef_op2(double %x, double %y) {
+; CHECK-LABEL: @fma_undef_op2(
+; CHECK-NEXT:    ret double 0x7FF8000000000000
+;
+  %r = call double @llvm.fma.f64(double %x, double %y, double undef)
+  ret double %r
+}
+
+define double @fmuladd_undef_op0(double %x, double %y) {
+; CHECK-LABEL: @fmuladd_undef_op0(
+; CHECK-NEXT:    ret double 0x7FF8000000000000
+;
+  %r = call double @llvm.fmuladd.f64(double undef, double %x, double %y)
+  ret double %r
+}
+
+define double @fmuladd_undef_op1(double %x, double %y) {
+; CHECK-LABEL: @fmuladd_undef_op1(
+; CHECK-NEXT:    ret double 0x7FF8000000000000
+;
+  %r = call double @llvm.fmuladd.f64(double %x, double undef, double %y)
+  ret double %r
+}
+
+define double @fmuladd_undef_op2(double %x, double %y) {
+; CHECK-LABEL: @fmuladd_undef_op2(
+; CHECK-NEXT:    ret double 0x7FF8000000000000
+;
+  %r = call double @llvm.fmuladd.f64(double %x, double %y, double undef)
+  ret double %r
+}
+
+define double @fma_nan_op0(double %x, double %y) {
+; CHECK-LABEL: @fma_nan_op0(
+; CHECK-NEXT:    ret double 0x7FF8000000000000
+;
+  %r = call double @llvm.fma.f64(double 0x7ff8000000000000, double %x, double %y)
+  ret double %r
+}
+
+define double @fma_nan_op1(double %x, double %y) {
+; CHECK-LABEL: @fma_nan_op1(
+; CHECK-NEXT:    ret double 0x7FF8000000000001
+;
+  %r = call double @llvm.fma.f64(double %x, double 0x7ff8000000000001, double %y)
+  ret double %r
+}
+
+define double @fma_nan_op2(double %x, double %y) {
+; CHECK-LABEL: @fma_nan_op2(
+; CHECK-NEXT:    ret double 0x7FF8000000000002
+;
+  %r = call double @llvm.fma.f64(double %x, double %y, double 0x7ff8000000000002)
+  ret double %r
+}
+
+define double @fmuladd_nan_op0_op1(double %x) {
+; CHECK-LABEL: @fmuladd_nan_op0_op1(
+; CHECK-NEXT:    ret double 0x7FF8000000001234
+;
+  %r = call double @llvm.fmuladd.f64(double 0x7ff8000000001234, double 0x7ff800000000dead, double %x)
+  ret double %r
+}
+
+define double @fmuladd_nan_op0_op2(double %x) {
+; CHECK-LABEL: @fmuladd_nan_op0_op2(
+; CHECK-NEXT:    ret double 0x7FF8000000005678
+;
+  %r = call double @llvm.fmuladd.f64(double 0x7ff8000000005678, double %x, double 0x7ff800000000dead)
+  ret double %r
+}
+
+define double @fmuladd_nan_op1_op2(double %x) {
+; CHECK-LABEL: @fmuladd_nan_op1_op2(
+; CHECK-NEXT:    ret double 0x7FF80000AAAAAAAA
+;
+  %r = call double @llvm.fmuladd.f64(double %x, double 0x7ff80000aaaaaaaa, double 0x7ff800000000dead)
+  ret double %r
+}
+
+define double @fma_nan_multiplicand_inf_zero(double %x) {
+; CHECK-LABEL: @fma_nan_multiplicand_inf_zero(
+; CHECK-NEXT:    [[R:%.*]] = call double @llvm.fma.f64(double 0x7FF0000000000000, double 0.000000e+00, double [[X:%.*]])
+; CHECK-NEXT:    ret double [[R]]
+;
+  %r = call double @llvm.fma.f64(double 0x7ff0000000000000, double 0.0, double %x)
+  ret double %r
+}
+
+define double @fma_nan_multiplicand_zero_inf(double %x) {
+; CHECK-LABEL: @fma_nan_multiplicand_zero_inf(
+; CHECK-NEXT:    [[R:%.*]] = call double @llvm.fma.f64(double 0.000000e+00, double 0x7FF0000000000000, double [[X:%.*]])
+; CHECK-NEXT:    ret double [[R]]
+;
+  %r = call double @llvm.fma.f64(double 0.0, double 0x7ff0000000000000, double %x)
+  ret double %r
+}
+
+define double @fma_nan_addend_inf_neginf(double %x, i32 %y) {
+; CHECK-LABEL: @fma_nan_addend_inf_neginf(
+; CHECK-NEXT:    [[NOTNAN:%.*]] = uitofp i32 [[Y:%.*]] to double
+; CHECK-NEXT:    [[R:%.*]] = call double @llvm.fma.f64(double 0x7FF0000000000000, double [[NOTNAN]], double 0xFFF0000000000000)
+; CHECK-NEXT:    ret double [[R]]
+;
+  %notnan = uitofp i32 %y to double
+  %r = call double @llvm.fma.f64(double 0x7ff0000000000000, double %notnan, double 0xfff0000000000000)
+  ret double %r
+}
+
+define double @fma_nan_addend_neginf_inf(double %x, i1 %y) {
+; CHECK-LABEL: @fma_nan_addend_neginf_inf(
+; CHECK-NEXT:    [[NOTNAN:%.*]] = select i1 [[Y:%.*]], double 4.200000e+01, double -1.000000e-01
+; CHECK-NEXT:    [[R:%.*]] = call double @llvm.fma.f64(double [[NOTNAN]], double 0xFFF0000000000000, double 0x7FF0000000000000)
+; CHECK-NEXT:    ret double [[R]]
+;
+  %notnan = select i1 %y, double 42.0, double -0.1
+  %r = call double @llvm.fma.f64(double %notnan, double 0xfff0000000000000, double 0x7ff0000000000000)
+  ret double %r
+}
+
+define double @fmuladd_nan_multiplicand_neginf_zero(double %x) {
+; CHECK-LABEL: @fmuladd_nan_multiplicand_neginf_zero(
+; CHECK-NEXT:    [[R:%.*]] = call double @llvm.fmuladd.f64(double 0xFFF0000000000000, double 0.000000e+00, double [[X:%.*]])
+; CHECK-NEXT:    ret double [[R]]
+;
+  %r = call double @llvm.fmuladd.f64(double 0xfff0000000000000, double 0.0, double %x)
+  ret double %r
+}
+
+define double @fmuladd_nan_multiplicand_negzero_inf(double %x) {
+; CHECK-LABEL: @fmuladd_nan_multiplicand_negzero_inf(
+; CHECK-NEXT:    [[R:%.*]] = call double @llvm.fmuladd.f64(double -0.000000e+00, double 0x7FF0000000000000, double [[X:%.*]])
+; CHECK-NEXT:    ret double [[R]]
+;
+  %r = call double @llvm.fmuladd.f64(double -0.0, double 0x7ff0000000000000, double %x)
+  ret double %r
+}
+
+define double @fmuladd_nan_addend_inf_neginf(double %x, i32 %y) {
+; CHECK-LABEL: @fmuladd_nan_addend_inf_neginf(
+; CHECK-NEXT:    [[NOTNAN:%.*]] = sitofp i32 [[Y:%.*]] to double
+; CHECK-NEXT:    [[R:%.*]] = call double @llvm.fmuladd.f64(double 0x7FF0000000000000, double [[NOTNAN]], double 0xFFF0000000000000)
+; CHECK-NEXT:    ret double [[R]]
+;
+  %notnan = sitofp i32 %y to double
+  %r = call double @llvm.fmuladd.f64(double 0x7ff0000000000000, double %notnan, double 0xfff0000000000000)
+  ret double %r
+}
+
+define double @fmuladd_nan_addend_neginf_inf(double %x, i1 %y) {
+; CHECK-LABEL: @fmuladd_nan_addend_neginf_inf(
+; CHECK-NEXT:    [[NOTNAN:%.*]] = select i1 [[Y:%.*]], double 4.200000e+01, double -1.000000e-01
+; CHECK-NEXT:    [[R:%.*]] = call double @llvm.fmuladd.f64(double [[NOTNAN]], double 0xFFF0000000000000, double 0x7FF0000000000000)
+; CHECK-NEXT:    ret double [[R]]
+;
+  %notnan = select i1 %y, double 42.0, double -0.1
+  %r = call double @llvm.fmuladd.f64(double %notnan, double 0xfff0000000000000, double 0x7ff0000000000000)
+  ret double %r
+}
+
+declare float @llvm.copysign.f32(float, float)
+declare <2 x double> @llvm.copysign.v2f64(<2 x double>, <2 x double>)
+
+define float @copysign_same_operand(float %x) {
+; CHECK-LABEL: @copysign_same_operand(
+; CHECK-NEXT:    ret float [[X:%.*]]
+;
+  %r = call float @llvm.copysign.f32(float %x, float %x)
+  ret float %r
+}
+
+define <2 x double> @copysign_same_operand_vec(<2 x double> %x) {
+; CHECK-LABEL: @copysign_same_operand_vec(
+; CHECK-NEXT:    ret <2 x double> [[X:%.*]]
+;
+  %r = call <2 x double> @llvm.copysign.v2f64(<2 x double> %x, <2 x double> %x)
+  ret <2 x double> %r
+}
+
+define float @negated_sign_arg(float %x) {
+; CHECK-LABEL: @negated_sign_arg(
+; CHECK-NEXT:    [[NEGX:%.*]] = fsub ninf float -0.000000e+00, [[X:%.*]]
+; CHECK-NEXT:    ret float [[NEGX]]
+;
+  %negx = fsub ninf float -0.0, %x
+  %r = call arcp float @llvm.copysign.f32(float %x, float %negx)
+  ret float %r
+}
+
+define <2 x double> @negated_sign_arg_vec(<2 x double> %x) {
+; CHECK-LABEL: @negated_sign_arg_vec(
+; CHECK-NEXT:    [[NEGX:%.*]] = fneg afn <2 x double> [[X:%.*]]
+; CHECK-NEXT:    ret <2 x double> [[NEGX]]
+;
+  %negx = fneg afn <2 x double> %x
+  %r = call arcp <2 x double> @llvm.copysign.v2f64(<2 x double> %x, <2 x double> %negx)
+  ret <2 x double> %r
+}
+
+define float @negated_mag_arg(float %x) {
+; CHECK-LABEL: @negated_mag_arg(
+; CHECK-NEXT:    ret float [[X:%.*]]
+;
+  %negx = fneg nnan float %x
+  %r = call ninf float @llvm.copysign.f32(float %negx, float %x)
+  ret float %r
+}
+
+define <2 x double> @negated_mag_arg_vec(<2 x double> %x) {
+; CHECK-LABEL: @negated_mag_arg_vec(
+; CHECK-NEXT:    ret <2 x double> [[X:%.*]]
+;
+  %negx = fneg afn <2 x double> %x
+  %r = call arcp <2 x double> @llvm.copysign.v2f64(<2 x double> %negx, <2 x double> %x)
+  ret <2 x double> %r
+}
diff --git a/llvm/test/Transforms/InstSimplify/div-by-0-guard-before-smul_ov-not.ll b/llvm/test/Transforms/InstSimplify/div-by-0-guard-before-smul_ov-not.ll
new file mode 100644
index 000000000..6ed3b1e57
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/div-by-0-guard-before-smul_ov-not.ll
@@ -0,0 +1,102 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instsimplify -S | FileCheck %s
+
+declare { i4, i1 } @llvm.smul.with.overflow.i4(i4, i4) #1
+
+define i1 @t0_umul(i4 %size, i4 %nmemb) {
+; CHECK-LABEL: @t0_umul(
+; CHECK-NEXT:    [[SMUL:%.*]] = tail call { i4, i1 } @llvm.smul.with.overflow.i4(i4 [[SIZE:%.*]], i4 [[NMEMB:%.*]])
+; CHECK-NEXT:    [[SMUL_OV:%.*]] = extractvalue { i4, i1 } [[SMUL]], 1
+; CHECK-NEXT:    [[PHITMP:%.*]] = xor i1 [[SMUL_OV]], true
+; CHECK-NEXT:    ret i1 [[PHITMP]]
+;
+  %cmp = icmp eq i4 %size, 0
+  %smul = tail call { i4, i1 } @llvm.smul.with.overflow.i4(i4 %size, i4 %nmemb)
+  %smul.ov = extractvalue { i4, i1 } %smul, 1
+  %phitmp = xor i1 %smul.ov, true
+  %or = or i1 %cmp, %phitmp
+  ret i1 %or
+}
+
+define i1 @t1_commutative(i4 %size, i4 %nmemb) {
+; CHECK-LABEL: @t1_commutative(
+; CHECK-NEXT:    [[SMUL:%.*]] = tail call { i4, i1 } @llvm.smul.with.overflow.i4(i4 [[SIZE:%.*]], i4 [[NMEMB:%.*]])
+; CHECK-NEXT:    [[SMUL_OV:%.*]] = extractvalue { i4, i1 } [[SMUL]], 1
+; CHECK-NEXT:    [[PHITMP:%.*]] = xor i1 [[SMUL_OV]], true
+; CHECK-NEXT:    ret i1 [[PHITMP]]
+;
+  %cmp = icmp eq i4 %size, 0
+  %smul = tail call { i4, i1 } @llvm.smul.with.overflow.i4(i4 %size, i4 %nmemb)
+  %smul.ov = extractvalue { i4, i1 } %smul, 1
+  %phitmp = xor i1 %smul.ov, true
+  %or = or i1 %phitmp, %cmp ; swapped
+  ret i1 %or
+}
+
+define i1 @n2_wrong_size(i4 %size0, i4 %size1, i4 %nmemb) {
+; CHECK-LABEL: @n2_wrong_size(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i4 [[SIZE1:%.*]], 0
+; CHECK-NEXT:    [[SMUL:%.*]] = tail call { i4, i1 } @llvm.smul.with.overflow.i4(i4 [[SIZE0:%.*]], i4 [[NMEMB:%.*]])
+; CHECK-NEXT:    [[SMUL_OV:%.*]] = extractvalue { i4, i1 } [[SMUL]], 1
+; CHECK-NEXT:    [[PHITMP:%.*]] = xor i1 [[SMUL_OV]], true
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[CMP]], [[PHITMP]]
+; CHECK-NEXT:    ret i1 [[OR]]
+;
+  %cmp = icmp eq i4 %size1, 0 ; not %size0
+  %smul = tail call { i4, i1 } @llvm.smul.with.overflow.i4(i4 %size0, i4 %nmemb)
+  %smul.ov = extractvalue { i4, i1 } %smul, 1
+  %phitmp = xor i1 %smul.ov, true
+  %or = or i1 %cmp, %phitmp
+  ret i1 %or
+}
+
+define i1 @n3_wrong_pred(i4 %size, i4 %nmemb) {
+; CHECK-LABEL: @n3_wrong_pred(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i4 [[SIZE:%.*]], 0
+; CHECK-NEXT:    [[SMUL:%.*]] = tail call { i4, i1 } @llvm.smul.with.overflow.i4(i4 [[SIZE]], i4 [[NMEMB:%.*]])
+; CHECK-NEXT:    [[SMUL_OV:%.*]] = extractvalue { i4, i1 } [[SMUL]], 1
+; CHECK-NEXT:    [[PHITMP:%.*]] = xor i1 [[SMUL_OV]], true
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[CMP]], [[PHITMP]]
+; CHECK-NEXT:    ret i1 [[OR]]
+;
+  %cmp = icmp ne i4 %size, 0 ; not 'eq'
+  %smul = tail call { i4, i1 } @llvm.smul.with.overflow.i4(i4 %size, i4 %nmemb)
+  %smul.ov = extractvalue { i4, i1 } %smul, 1
+  %phitmp = xor i1 %smul.ov, true
+  %or = or i1 %cmp, %phitmp
+  ret i1 %or
+}
+
+define i1 @n4_not_and(i4 %size, i4 %nmemb) {
+; CHECK-LABEL: @n4_not_and(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i4 [[SIZE:%.*]], 0
+; CHECK-NEXT:    [[SMUL:%.*]] = tail call { i4, i1 } @llvm.smul.with.overflow.i4(i4 [[SIZE]], i4 [[NMEMB:%.*]])
+; CHECK-NEXT:    [[SMUL_OV:%.*]] = extractvalue { i4, i1 } [[SMUL]], 1
+; CHECK-NEXT:    [[PHITMP:%.*]] = xor i1 [[SMUL_OV]], true
+; CHECK-NEXT:    [[OR:%.*]] = and i1 [[CMP]], [[PHITMP]]
+; CHECK-NEXT:    ret i1 [[OR]]
+;
+  %cmp = icmp eq i4 %size, 0
+  %smul = tail call { i4, i1 } @llvm.smul.with.overflow.i4(i4 %size, i4 %nmemb)
+  %smul.ov = extractvalue { i4, i1 } %smul, 1
+  %phitmp = xor i1 %smul.ov, true
+  %or = and i1 %cmp, %phitmp ; not 'or'
+  ret i1 %or
+}
+
+define i1 @n5_not_zero(i4 %size, i4 %nmemb) {
+; CHECK-LABEL: @n5_not_zero(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i4 [[SIZE:%.*]], 1
+; CHECK-NEXT:    [[SMUL:%.*]] = tail call { i4, i1 } @llvm.smul.with.overflow.i4(i4 [[SIZE]], i4 [[NMEMB:%.*]])
+; CHECK-NEXT:    [[SMUL_OV:%.*]] = extractvalue { i4, i1 } [[SMUL]], 1
+; CHECK-NEXT:    [[PHITMP:%.*]] = xor i1 [[SMUL_OV]], true
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[CMP]], [[PHITMP]]
+; CHECK-NEXT:    ret i1 [[OR]]
+;
+  %cmp = icmp eq i4 %size, 1 ; should be '0'
+  %smul = tail call { i4, i1 } @llvm.smul.with.overflow.i4(i4 %size, i4 %nmemb)
+  %smul.ov = extractvalue { i4, i1 } %smul, 1
+  %phitmp = xor i1 %smul.ov, true
+  %or = or i1 %cmp, %phitmp
+  ret i1 %or
+}
diff --git a/llvm/test/Transforms/InstSimplify/div-by-0-guard-before-smul_ov.ll b/llvm/test/Transforms/InstSimplify/div-by-0-guard-before-smul_ov.ll
new file mode 100644
index 000000000..004c36560
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/div-by-0-guard-before-smul_ov.ll
@@ -0,0 +1,90 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instsimplify -S | FileCheck %s
+
+declare { i4, i1 } @llvm.smul.with.overflow.i4(i4, i4) #1
+
+define i1 @t0_smul(i4 %size, i4 %nmemb) {
+; CHECK-LABEL: @t0_smul(
+; CHECK-NEXT:    [[SMUL:%.*]] = tail call { i4, i1 } @llvm.smul.with.overflow.i4(i4 [[SIZE:%.*]], i4 [[NMEMB:%.*]])
+; CHECK-NEXT:    [[SMUL_OV:%.*]] = extractvalue { i4, i1 } [[SMUL]], 1
+; CHECK-NEXT:    ret i1 [[SMUL_OV]]
+;
+  %cmp = icmp ne i4 %size, 0
+  %smul = tail call { i4, i1 } @llvm.smul.with.overflow.i4(i4 %size, i4 %nmemb)
+  %smul.ov = extractvalue { i4, i1 } %smul, 1
+  %and = and i1 %smul.ov, %cmp
+  ret i1 %and
+}
+
+define i1 @t1_commutative(i4 %size, i4 %nmemb) {
+; CHECK-LABEL: @t1_commutative(
+; CHECK-NEXT:    [[SMUL:%.*]] = tail call { i4, i1 } @llvm.smul.with.overflow.i4(i4 [[SIZE:%.*]], i4 [[NMEMB:%.*]])
+; CHECK-NEXT:    [[SMUL_OV:%.*]] = extractvalue { i4, i1 } [[SMUL]], 1
+; CHECK-NEXT:    ret i1 [[SMUL_OV]]
+;
+  %cmp = icmp ne i4 %size, 0
+  %smul = tail call { i4, i1 } @llvm.smul.with.overflow.i4(i4 %size, i4 %nmemb)
+  %smul.ov = extractvalue { i4, i1 } %smul, 1
+  %and = and i1 %cmp, %smul.ov ; swapped
+  ret i1 %and
+}
+
+define i1 @n2_wrong_size(i4 %size0, i4 %size1, i4 %nmemb) {
+; CHECK-LABEL: @n2_wrong_size(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i4 [[SIZE1:%.*]], 0
+; CHECK-NEXT:    [[SMUL:%.*]] = tail call { i4, i1 } @llvm.smul.with.overflow.i4(i4 [[SIZE0:%.*]], i4 [[NMEMB:%.*]])
+; CHECK-NEXT:    [[SMUL_OV:%.*]] = extractvalue { i4, i1 } [[SMUL]], 1
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[SMUL_OV]], [[CMP]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  %cmp = icmp ne i4 %size1, 0 ; not %size0
+  %smul = tail call { i4, i1 } @llvm.smul.with.overflow.i4(i4 %size0, i4 %nmemb)
+  %smul.ov = extractvalue { i4, i1 } %smul, 1
+  %and = and i1 %smul.ov, %cmp
+  ret i1 %and
+}
+
+define i1 @n3_wrong_pred(i4 %size, i4 %nmemb) {
+; CHECK-LABEL: @n3_wrong_pred(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i4 [[SIZE:%.*]], 0
+; CHECK-NEXT:    [[SMUL:%.*]] = tail call { i4, i1 } @llvm.smul.with.overflow.i4(i4 [[SIZE]], i4 [[NMEMB:%.*]])
+; CHECK-NEXT:    [[SMUL_OV:%.*]] = extractvalue { i4, i1 } [[SMUL]], 1
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[SMUL_OV]], [[CMP]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  %cmp = icmp eq i4 %size, 0 ; not 'ne'
+  %smul = tail call { i4, i1 } @llvm.smul.with.overflow.i4(i4 %size, i4 %nmemb)
+  %smul.ov = extractvalue { i4, i1 } %smul, 1
+  %and = and i1 %smul.ov, %cmp
+  ret i1 %and
+}
+
+define i1 @n4_not_and(i4 %size, i4 %nmemb) {
+; CHECK-LABEL: @n4_not_and(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i4 [[SIZE:%.*]], 0
+; CHECK-NEXT:    [[SMUL:%.*]] = tail call { i4, i1 } @llvm.smul.with.overflow.i4(i4 [[SIZE]], i4 [[NMEMB:%.*]])
+; CHECK-NEXT:    [[SMUL_OV:%.*]] = extractvalue { i4, i1 } [[SMUL]], 1
+; CHECK-NEXT:    [[AND:%.*]] = or i1 [[SMUL_OV]], [[CMP]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  %cmp = icmp ne i4 %size, 0
+  %smul = tail call { i4, i1 } @llvm.smul.with.overflow.i4(i4 %size, i4 %nmemb)
+  %smul.ov = extractvalue { i4, i1 } %smul, 1
+  %and = or i1 %smul.ov, %cmp ; not 'and'
+  ret i1 %and
+}
+
+define i1 @n5_not_zero(i4 %size, i4 %nmemb) {
+; CHECK-LABEL: @n5_not_zero(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i4 [[SIZE:%.*]], 1
+; CHECK-NEXT:    [[SMUL:%.*]] = tail call { i4, i1 } @llvm.smul.with.overflow.i4(i4 [[SIZE]], i4 [[NMEMB:%.*]])
+; CHECK-NEXT:    [[SMUL_OV:%.*]] = extractvalue { i4, i1 } [[SMUL]], 1
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[SMUL_OV]], [[CMP]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  %cmp = icmp ne i4 %size, 1 ; should be '0'
+  %smul = tail call { i4, i1 } @llvm.smul.with.overflow.i4(i4 %size, i4 %nmemb)
+  %smul.ov = extractvalue { i4, i1 } %smul, 1
+  %and = and i1 %smul.ov, %cmp
+  ret i1 %and
+}
diff --git a/llvm/test/Transforms/InstSimplify/div-by-0-guard-before-umul_ov-not.ll b/llvm/test/Transforms/InstSimplify/div-by-0-guard-before-umul_ov-not.ll
new file mode 100644
index 000000000..c3c28a726
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/div-by-0-guard-before-umul_ov-not.ll
@@ -0,0 +1,102 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instsimplify -S | FileCheck %s
+
+declare { i4, i1 } @llvm.umul.with.overflow.i4(i4, i4) #1
+
+define i1 @t0_umul(i4 %size, i4 %nmemb) {
+; CHECK-LABEL: @t0_umul(
+; CHECK-NEXT:    [[UMUL:%.*]] = tail call { i4, i1 } @llvm.umul.with.overflow.i4(i4 [[SIZE:%.*]], i4 [[NMEMB:%.*]])
+; CHECK-NEXT:    [[UMUL_OV:%.*]] = extractvalue { i4, i1 } [[UMUL]], 1
+; CHECK-NEXT:    [[PHITMP:%.*]] = xor i1 [[UMUL_OV]], true
+; CHECK-NEXT:    ret i1 [[PHITMP]]
+;
+  %cmp = icmp eq i4 %size, 0
+  %umul = tail call { i4, i1 } @llvm.umul.with.overflow.i4(i4 %size, i4 %nmemb)
+  %umul.ov = extractvalue { i4, i1 } %umul, 1
+  %phitmp = xor i1 %umul.ov, true
+  %or = or i1 %cmp, %phitmp
+  ret i1 %or
+}
+
+define i1 @t1_commutative(i4 %size, i4 %nmemb) {
+; CHECK-LABEL: @t1_commutative(
+; CHECK-NEXT:    [[UMUL:%.*]] = tail call { i4, i1 } @llvm.umul.with.overflow.i4(i4 [[SIZE:%.*]], i4 [[NMEMB:%.*]])
+; CHECK-NEXT:    [[UMUL_OV:%.*]] = extractvalue { i4, i1 } [[UMUL]], 1
+; CHECK-NEXT:    [[PHITMP:%.*]] = xor i1 [[UMUL_OV]], true
+; CHECK-NEXT:    ret i1 [[PHITMP]]
+;
+  %cmp = icmp eq i4 %size, 0
+  %umul = tail call { i4, i1 } @llvm.umul.with.overflow.i4(i4 %size, i4 %nmemb)
+  %umul.ov = extractvalue { i4, i1 } %umul, 1
+  %phitmp = xor i1 %umul.ov, true
+  %or = or i1 %phitmp, %cmp ; swapped
+  ret i1 %or
+}
+
+define i1 @n2_wrong_size(i4 %size0, i4 %size1, i4 %nmemb) {
+; CHECK-LABEL: @n2_wrong_size(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i4 [[SIZE1:%.*]], 0
+; CHECK-NEXT:    [[UMUL:%.*]] = tail call { i4, i1 } @llvm.umul.with.overflow.i4(i4 [[SIZE0:%.*]], i4 [[NMEMB:%.*]])
+; CHECK-NEXT:    [[UMUL_OV:%.*]] = extractvalue { i4, i1 } [[UMUL]], 1
+; CHECK-NEXT:    [[PHITMP:%.*]] = xor i1 [[UMUL_OV]], true
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[CMP]], [[PHITMP]]
+; CHECK-NEXT:    ret i1 [[OR]]
+;
+  %cmp = icmp eq i4 %size1, 0 ; not %size0
+  %umul = tail call { i4, i1 } @llvm.umul.with.overflow.i4(i4 %size0, i4 %nmemb)
+  %umul.ov = extractvalue { i4, i1 } %umul, 1
+  %phitmp = xor i1 %umul.ov, true
+  %or = or i1 %cmp, %phitmp
+  ret i1 %or
+}
+
+define i1 @n3_wrong_pred(i4 %size, i4 %nmemb) {
+; CHECK-LABEL: @n3_wrong_pred(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i4 [[SIZE:%.*]], 0
+; CHECK-NEXT:    [[UMUL:%.*]] = tail call { i4, i1 } @llvm.umul.with.overflow.i4(i4 [[SIZE]], i4 [[NMEMB:%.*]])
+; CHECK-NEXT:    [[UMUL_OV:%.*]] = extractvalue { i4, i1 } [[UMUL]], 1
+; CHECK-NEXT:    [[PHITMP:%.*]] = xor i1 [[UMUL_OV]], true
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[CMP]], [[PHITMP]]
+; CHECK-NEXT:    ret i1 [[OR]]
+;
+  %cmp = icmp ne i4 %size, 0 ; not 'eq'
+  %umul = tail call { i4, i1 } @llvm.umul.with.overflow.i4(i4 %size, i4 %nmemb)
+  %umul.ov = extractvalue { i4, i1 } %umul, 1
+  %phitmp = xor i1 %umul.ov, true
+  %or = or i1 %cmp, %phitmp
+  ret i1 %or
+}
+
+define i1 @n4_not_and(i4 %size, i4 %nmemb) {
+; CHECK-LABEL: @n4_not_and(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i4 [[SIZE:%.*]], 0
+; CHECK-NEXT:    [[UMUL:%.*]] = tail call { i4, i1 } @llvm.umul.with.overflow.i4(i4 [[SIZE]], i4 [[NMEMB:%.*]])
+; CHECK-NEXT:    [[UMUL_OV:%.*]] = extractvalue { i4, i1 } [[UMUL]], 1
+; CHECK-NEXT:    [[PHITMP:%.*]] = xor i1 [[UMUL_OV]], true
+; CHECK-NEXT:    [[OR:%.*]] = and i1 [[CMP]], [[PHITMP]]
+; CHECK-NEXT:    ret i1 [[OR]]
+;
+  %cmp = icmp eq i4 %size, 0
+  %umul = tail call { i4, i1 } @llvm.umul.with.overflow.i4(i4 %size, i4 %nmemb)
+  %umul.ov = extractvalue { i4, i1 } %umul, 1
+  %phitmp = xor i1 %umul.ov, true
+  %or = and i1 %cmp, %phitmp ; not 'or'
+  ret i1 %or
+}
+
+define i1 @n5_not_zero(i4 %size, i4 %nmemb) {
+; CHECK-LABEL: @n5_not_zero(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i4 [[SIZE:%.*]], 1
+; CHECK-NEXT:    [[UMUL:%.*]] = tail call { i4, i1 } @llvm.umul.with.overflow.i4(i4 [[SIZE]], i4 [[NMEMB:%.*]])
+; CHECK-NEXT:    [[UMUL_OV:%.*]] = extractvalue { i4, i1 } [[UMUL]], 1
+; CHECK-NEXT:    [[PHITMP:%.*]] = xor i1 [[UMUL_OV]], true
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[CMP]], [[PHITMP]]
+; CHECK-NEXT:    ret i1 [[OR]]
+;
+  %cmp = icmp eq i4 %size, 1 ; should be '0'
+  %umul = tail call { i4, i1 } @llvm.umul.with.overflow.i4(i4 %size, i4 %nmemb)
+  %umul.ov = extractvalue { i4, i1 } %umul, 1
+  %phitmp = xor i1 %umul.ov, true
+  %or = or i1 %cmp, %phitmp
+  ret i1 %or
+}
diff --git a/llvm/test/Transforms/InstSimplify/div-by-0-guard-before-umul_ov.ll b/llvm/test/Transforms/InstSimplify/div-by-0-guard-before-umul_ov.ll
new file mode 100644
index 000000000..337de40f5
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/div-by-0-guard-before-umul_ov.ll
@@ -0,0 +1,90 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instsimplify -S | FileCheck %s
+
+declare { i4, i1 } @llvm.umul.with.overflow.i4(i4, i4) #1
+
+define i1 @t0_umul(i4 %size, i4 %nmemb) {
+; CHECK-LABEL: @t0_umul(
+; CHECK-NEXT:    [[UMUL:%.*]] = tail call { i4, i1 } @llvm.umul.with.overflow.i4(i4 [[SIZE:%.*]], i4 [[NMEMB:%.*]])
+; CHECK-NEXT:    [[UMUL_OV:%.*]] = extractvalue { i4, i1 } [[UMUL]], 1
+; CHECK-NEXT:    ret i1 [[UMUL_OV]]
+;
+  %cmp = icmp ne i4 %size, 0
+  %umul = tail call { i4, i1 } @llvm.umul.with.overflow.i4(i4 %size, i4 %nmemb)
+  %umul.ov = extractvalue { i4, i1 } %umul, 1
+  %and = and i1 %umul.ov, %cmp
+  ret i1 %and
+}
+
+define i1 @t1_commutative(i4 %size, i4 %nmemb) {
+; CHECK-LABEL: @t1_commutative(
+; CHECK-NEXT:    [[UMUL:%.*]] = tail call { i4, i1 } @llvm.umul.with.overflow.i4(i4 [[SIZE:%.*]], i4 [[NMEMB:%.*]])
+; CHECK-NEXT:    [[UMUL_OV:%.*]] = extractvalue { i4, i1 } [[UMUL]], 1
+; CHECK-NEXT:    ret i1 [[UMUL_OV]]
+;
+  %cmp = icmp ne i4 %size, 0
+  %umul = tail call { i4, i1 } @llvm.umul.with.overflow.i4(i4 %size, i4 %nmemb)
+  %umul.ov = extractvalue { i4, i1 } %umul, 1
+  %and = and i1 %cmp, %umul.ov ; swapped
+  ret i1 %and
+}
+
+define i1 @n2_wrong_size(i4 %size0, i4 %size1, i4 %nmemb) {
+; CHECK-LABEL: @n2_wrong_size(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i4 [[SIZE1:%.*]], 0
+; CHECK-NEXT:    [[UMUL:%.*]] = tail call { i4, i1 } @llvm.umul.with.overflow.i4(i4 [[SIZE0:%.*]], i4 [[NMEMB:%.*]])
+; CHECK-NEXT:    [[UMUL_OV:%.*]] = extractvalue { i4, i1 } [[UMUL]], 1
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[UMUL_OV]], [[CMP]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  %cmp = icmp ne i4 %size1, 0 ; not %size0
+  %umul = tail call { i4, i1 } @llvm.umul.with.overflow.i4(i4 %size0, i4 %nmemb)
+  %umul.ov = extractvalue { i4, i1 } %umul, 1
+  %and = and i1 %umul.ov, %cmp
+  ret i1 %and
+}
+
+define i1 @n3_wrong_pred(i4 %size, i4 %nmemb) {
+; CHECK-LABEL: @n3_wrong_pred(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i4 [[SIZE:%.*]], 0
+; CHECK-NEXT:    [[UMUL:%.*]] = tail call { i4, i1 } @llvm.umul.with.overflow.i4(i4 [[SIZE]], i4 [[NMEMB:%.*]])
+; CHECK-NEXT:    [[UMUL_OV:%.*]] = extractvalue { i4, i1 } [[UMUL]], 1
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[UMUL_OV]], [[CMP]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  %cmp = icmp eq i4 %size, 0 ; not 'ne'
+  %umul = tail call { i4, i1 } @llvm.umul.with.overflow.i4(i4 %size, i4 %nmemb)
+  %umul.ov = extractvalue { i4, i1 } %umul, 1
+  %and = and i1 %umul.ov, %cmp
+  ret i1 %and
+}
+
+define i1 @n4_not_and(i4 %size, i4 %nmemb) {
+; CHECK-LABEL: @n4_not_and(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i4 [[SIZE:%.*]], 0
+; CHECK-NEXT:    [[UMUL:%.*]] = tail call { i4, i1 } @llvm.umul.with.overflow.i4(i4 [[SIZE]], i4 [[NMEMB:%.*]])
+; CHECK-NEXT:    [[UMUL_OV:%.*]] = extractvalue { i4, i1 } [[UMUL]], 1
+; CHECK-NEXT:    [[AND:%.*]] = or i1 [[UMUL_OV]], [[CMP]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  %cmp = icmp ne i4 %size, 0
+  %umul = tail call { i4, i1 } @llvm.umul.with.overflow.i4(i4 %size, i4 %nmemb)
+  %umul.ov = extractvalue { i4, i1 } %umul, 1
+  %and = or i1 %umul.ov, %cmp ; not 'and'
+  ret i1 %and
+}
+
+define i1 @n5_not_zero(i4 %size, i4 %nmemb) {
+; CHECK-LABEL: @n5_not_zero(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i4 [[SIZE:%.*]], 1
+; CHECK-NEXT:    [[UMUL:%.*]] = tail call { i4, i1 } @llvm.umul.with.overflow.i4(i4 [[SIZE]], i4 [[NMEMB:%.*]])
+; CHECK-NEXT:    [[UMUL_OV:%.*]] = extractvalue { i4, i1 } [[UMUL]], 1
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[UMUL_OV]], [[CMP]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  %cmp = icmp ne i4 %size, 1 ; should be '0'
+  %umul = tail call { i4, i1 } @llvm.umul.with.overflow.i4(i4 %size, i4 %nmemb)
+  %umul.ov = extractvalue { i4, i1 } %umul, 1
+  %and = and i1 %umul.ov, %cmp
+  ret i1 %and
+}
diff --git a/llvm/test/Transforms/InstSimplify/fcmp-select.ll b/llvm/test/Transforms/InstSimplify/fcmp-select.ll
index eae885c84..e10069126 100644
--- a/llvm/test/Transforms/InstSimplify/fcmp-select.ll
+++ b/llvm/test/Transforms/InstSimplify/fcmp-select.ll
@@ -48,6 +48,174 @@ define double @une_swapped(double %x) {
   ret double %cond
 }
 
+; X == 0.0 ? X : 0.0 --> ? (X could be -0.0)
+
+define double @oeq_zero(double %x) {
+; CHECK-LABEL: @oeq_zero(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp oeq double [[X:%.*]], 0.000000e+00
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], double [[X]], double 0.000000e+00
+; CHECK-NEXT:    ret double [[COND]]
+;
+  %cmp = fcmp oeq double %x, 0.0
+  %cond = select i1 %cmp, double %x, double 0.0
+  ret double %cond
+}
+
+; X == 0.0 ? 0.0 : X --> ? (change sign if X is -0.0)
+
+define float @oeq_zero_swapped(float %x) {
+; CHECK-LABEL: @oeq_zero_swapped(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp oeq float [[X:%.*]], 0.000000e+00
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], float 0.000000e+00, float [[X]]
+; CHECK-NEXT:    ret float [[COND]]
+;
+  %cmp = fcmp oeq float %x, 0.0
+  %cond = select i1 %cmp, float 0.0, float %x
+  ret float %cond
+}
+
+; X != 0.0 ? X : -0.0 --> ? (change sign if X is 0.0)
+
+define double @une_zero(double %x) {
+; CHECK-LABEL: @une_zero(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp une double [[X:%.*]], 0.000000e+00
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], double [[X]], double -0.000000e+00
+; CHECK-NEXT:    ret double [[COND]]
+;
+  %cmp = fcmp une double %x, 0.0
+  %cond = select i1 %cmp, double %x, double -0.0
+  ret double %cond
+}
+
+; X != 0.0 ? -0.0 : X --> ? (X could be 0.0)
+
+define double @une_zero_swapped(double %x) {
+; CHECK-LABEL: @une_zero_swapped(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp une double [[X:%.*]], 0.000000e+00
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], double -0.000000e+00, double [[X]]
+; CHECK-NEXT:    ret double [[COND]]
+;
+  %cmp = fcmp une double %x, 0.0
+  %cond = select i1 %cmp, double -0.0, double %x
+  ret double %cond
+}
+
+; X == 0.0 ? X : 0.0 --> 0.0
+
+define double @oeq_zero_nsz(double %x) {
+; CHECK-LABEL: @oeq_zero_nsz(
+; CHECK-NEXT:    ret double 0.000000e+00
+;
+  %cmp = fcmp oeq double %x, 0.0
+  %cond = select nsz i1 %cmp, double %x, double 0.0
+  ret double %cond
+}
+
+; X == 0.0 ? 0.0 : X --> X
+
+define float @oeq_zero_swapped_nsz(float %x) {
+; CHECK-LABEL: @oeq_zero_swapped_nsz(
+; CHECK-NEXT:    ret float [[X:%.*]]
+;
+  %cmp = fcmp oeq float %x, 0.0
+  %cond = select fast i1 %cmp, float 0.0, float %x
+  ret float %cond
+}
+
+; X != 0.0 ? X : 0.0 --> X
+
+define double @une_zero_nsz(double %x) {
+; CHECK-LABEL: @une_zero_nsz(
+; CHECK-NEXT:    ret double [[X:%.*]]
+;
+  %cmp = fcmp une double %x, 0.0
+  %cond = select nsz ninf i1 %cmp, double %x, double 0.0
+  ret double %cond
+}
+
+; X != 0.0 ? 0.0 : X --> 0.0
+
+define <2 x double> @une_zero_swapped_nsz(<2 x double> %x) {
+; CHECK-LABEL: @une_zero_swapped_nsz(
+; CHECK-NEXT:    ret <2 x double> zeroinitializer
+;
+  %cmp = fcmp une <2 x double> %x, <double 0.0, double 0.0>
+  %cond = select nsz <2 x i1> %cmp, <2 x double> <double 0.0, double 0.0>, <2 x double> %x
+  ret <2 x double> %cond
+}
+
+; X == Y ? X : Y --> Y
+
+define double @oeq_nsz(double %x, double %y) {
+; CHECK-LABEL: @oeq_nsz(
+; CHECK-NEXT:    ret double [[Y:%.*]]
+;
+  %cmp = fcmp oeq double %x, %y
+  %cond = select fast i1 %cmp, double %x, double %y
+  ret double %cond
+}
+
+; X == Y ? Y : X --> X
+
+define <2 x float> @oeq_swapped_nsz(<2 x float> %x, <2 x float> %y) {
+; CHECK-LABEL: @oeq_swapped_nsz(
+; CHECK-NEXT:    ret <2 x float> [[X:%.*]]
+;
+  %cmp = fcmp oeq <2 x float> %x, %y
+  %cond = select nsz nnan <2 x i1> %cmp, <2 x float> %y, <2 x float> %x
+  ret <2 x float> %cond
+}
+
+; X != Y ? X : Y --> X
+
+define double @une_nsz(double %x, double %y) {
+; CHECK-LABEL: @une_nsz(
+; CHECK-NEXT:    ret double [[X:%.*]]
+;
+  %cmp = fcmp une double %x, %y
+  %cond = select nsz i1 %cmp, double %x, double %y
+  ret double %cond
+}
+
+; X != Y ? Y : X --> Y
+
+define <2 x double> @une_swapped_nsz(<2 x double> %x, <2 x double> %y) {
+; CHECK-LABEL: @une_swapped_nsz(
+; CHECK-NEXT:    ret <2 x double> [[Y:%.*]]
+;
+  %cmp = fcmp une <2 x double> %x, %y
+  %cond = select fast <2 x i1> %cmp, <2 x double> %y, <2 x double> %x
+  ret <2 x double> %cond
+}
+
+; Harder - mismatched zero constants (not typical due to canonicalization):
+; X != 0.0 ? X : -0.0 --> X
+
+define double @une_zero_mismatch_nsz(double %x) {
+; CHECK-LABEL: @une_zero_mismatch_nsz(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp une double [[X:%.*]], 0.000000e+00
+; CHECK-NEXT:    [[COND:%.*]] = select ninf nsz i1 [[CMP]], double [[X]], double -0.000000e+00
+; CHECK-NEXT:    ret double [[COND]]
+;
+  %cmp = fcmp une double %x, 0.0
+  %cond = select nsz ninf i1 %cmp, double %x, double -0.0
+  ret double %cond
+}
+
+; Even harder - mismatched vector zero constants (not typical due to canonicalization):
+; X != 0.0 ? -0.0 : X --> 0.0
+
+define <2 x double> @une_zero_mismatch_swapped_nsz(<2 x double> %x) {
+; CHECK-LABEL: @une_zero_mismatch_swapped_nsz(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp une <2 x double> [[X:%.*]], <double -0.000000e+00, double 0.000000e+00>
+; CHECK-NEXT:    [[COND:%.*]] = select nsz <2 x i1> [[CMP]], <2 x double> <double 0.000000e+00, double -0.000000e+00>, <2 x double> [[X]]
+; CHECK-NEXT:    ret <2 x double> [[COND]]
+;
+  %cmp = fcmp une <2 x double> %x, <double -0.0, double 0.0>
+  %cond = select nsz <2 x i1> %cmp, <2 x double> <double 0.0, double -0.0>, <2 x double> %x
+  ret <2 x double> %cond
+}
+
 define double @une_could_be_negzero(double %x, double %y) {
 ; CHECK-LABEL: @une_could_be_negzero(
 ; CHECK-NEXT:    [[CMP:%.*]] = fcmp une double [[X:%.*]], [[Y:%.*]]
diff --git a/llvm/test/Transforms/InstSimplify/freeze.ll b/llvm/test/Transforms/InstSimplify/freeze.ll
new file mode 100644
index 000000000..ac287b949
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/freeze.ll
@@ -0,0 +1,20 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+define i32 @fold(i32 %x) {
+; CHECK-LABEL: @fold(
+; CHECK-NEXT:    [[Y:%.*]] = freeze i32 [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[Y]]
+;
+  %y = freeze i32 %x
+  %z = freeze i32 %y
+  ret i32 %z
+}
+
+define i32 @make_const() {
+; CHECK-LABEL: @make_const(
+; CHECK-NEXT:    ret i32 10
+;
+  %x = freeze i32 10
+  ret i32 %x
+}
diff --git a/llvm/test/Transforms/InstSimplify/icmp-abs-nabs.ll b/llvm/test/Transforms/InstSimplify/icmp-abs-nabs.ll
index be2e7b4d4..31f71cba8 100644
--- a/llvm/test/Transforms/InstSimplify/icmp-abs-nabs.ll
+++ b/llvm/test/Transforms/InstSimplify/icmp-abs-nabs.ll
@@ -401,3 +401,20 @@ define i1 @nabs_no_intersection(i32 %a) {
   ret i1 %r
 }
 
+; We can't fold this to false unless both subs have nsw.
+define i1 @abs_sub_sub_missing_nsw(i32 %x, i32 %y) {
+; CHECK-LABEL: @abs_sub_sub_missing_nsw(
+; CHECK-NEXT:    [[A:%.*]] = sub i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[B:%.*]] = sub nsw i32 [[Y]], [[X]]
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i32 [[A]], -1
+; CHECK-NEXT:    [[D:%.*]] = select i1 [[C]], i32 [[A]], i32 [[B]]
+; CHECK-NEXT:    [[E:%.*]] = icmp slt i32 [[D]], 0
+; CHECK-NEXT:    ret i1 [[E]]
+;
+  %a = sub i32 %x, %y
+  %b = sub nsw i32 %y, %x
+  %c = icmp sgt i32 %a, -1
+  %d = select i1 %c, i32 %a, i32 %b
+  %e = icmp slt i32 %d, 0
+  ret i1 %e
+}
diff --git a/llvm/test/Transforms/InstSimplify/insertelement.ll b/llvm/test/Transforms/InstSimplify/insertelement.ll
index e487eeb96..97f656a7c 100644
--- a/llvm/test/Transforms/InstSimplify/insertelement.ll
+++ b/llvm/test/Transforms/InstSimplify/insertelement.ll
@@ -67,3 +67,26 @@ define <8 x i8> @extract_insert_same_vec_and_index2(<8 x i8> %in, i32 %index) {
   %vec = insertelement <8 x i8> %in, i8 %val, i32 %index
   ret <8 x i8> %vec
 }
+
+; The insert is in an unreachable block, so it is allowed to point to itself.
+; This would crash via stack overflow.
+
+define void @PR43218() {
+; CHECK-LABEL: @PR43218(
+; CHECK-NEXT:  end:
+; CHECK-NEXT:    ret void
+; CHECK:       unreachable_infloop:
+; CHECK-NEXT:    [[EXTRACT:%.*]] = extractelement <2 x i64> [[BOGUS:%.*]], i32 0
+; CHECK-NEXT:    [[T0:%.*]] = inttoptr i64 [[EXTRACT]] to i16****
+; CHECK-NEXT:    [[BOGUS]] = insertelement <2 x i64> [[BOGUS]], i64 undef, i32 1
+; CHECK-NEXT:    br label [[UNREACHABLE_INFLOOP:%.*]]
+;
+end:
+  ret void
+
+unreachable_infloop:
+  %extract = extractelement <2 x i64> %bogus, i32 0
+  %t0 = inttoptr i64 %extract to i16****
+  %bogus = insertelement <2 x i64> %bogus, i64 undef, i32 1
+  br label %unreachable_infloop
+}
diff --git a/llvm/test/Transforms/InstSimplify/known-never-nan.ll b/llvm/test/Transforms/InstSimplify/known-never-nan.ll
index 37cfc932a..109775607 100644
--- a/llvm/test/Transforms/InstSimplify/known-never-nan.ll
+++ b/llvm/test/Transforms/InstSimplify/known-never-nan.ll
@@ -158,6 +158,18 @@ define i1 @known_nan_select(i1 %cond, double %arg0, double %arg1) {
   ret i1 %tmp
 }
 
+define i1 @nnan_ninf_known_nan_select(i1 %cond, double %arg0, double %arg1) {
+; CHECK-LABEL: @nnan_ninf_known_nan_select(
+; CHECK-NEXT:    ret i1 true
+;
+  %lhs = fadd nnan ninf double %arg0, 1.0
+  %rhs = fadd nnan ninf double %arg1, 2.0
+  %op = select i1 %cond, double %lhs, double %rhs
+  %mul = fmul double %op, 2.0
+  %tmp = fcmp ord double %mul, %mul
+  ret i1 %tmp
+}
+
 define i1 @select_maybe_nan_lhs(i1 %cond, double %lhs, double %arg1) {
 ; CHECK-LABEL: @select_maybe_nan_lhs(
 ; CHECK-NEXT:    [[RHS:%.*]] = fadd nnan double [[ARG1:%.*]], 1.000000e+00
@@ -258,9 +270,7 @@ define i1 @nnan_fsub(double %arg0, double %arg1) {
 define i1 @nnan_binary_fneg() {
 ; CHECK-LABEL: @nnan_binary_fneg(
 ; CHECK-NEXT:    [[NNAN:%.*]] = call nnan double @func()
-; CHECK-NEXT:    [[OP:%.*]] = fsub double -0.000000e+00, [[NNAN]]
-; CHECK-NEXT:    [[TMP:%.*]] = fcmp ord double [[OP]], [[OP]]
-; CHECK-NEXT:    ret i1 [[TMP]]
+; CHECK-NEXT:    ret i1 true
 ;
   %nnan = call nnan double @func()
   %op = fsub double -0.0, %nnan
@@ -299,6 +309,29 @@ define i1 @uitofp(i32 %arg0) {
   ret i1 %tmp
 }
 
+define i1 @uitofp_add(i32 %arg0) {
+; CHECK-LABEL: @uitofp_add(
+; CHECK-NEXT:    ret i1 true
+;
+  %op = uitofp i32 %arg0 to double
+  %add = fadd double %op, %op
+  %tmp = fcmp ord double %add, %add
+  ret i1 %tmp
+}
+
+define i1 @uitofp_add_big(i1024 %arg0) {
+; CHECK-LABEL: @uitofp_add_big(
+; CHECK-NEXT:    [[OP:%.*]] = uitofp i1024 [[ARG0:%.*]] to double
+; CHECK-NEXT:    [[ADD:%.*]] = fadd double [[OP]], [[OP]]
+; CHECK-NEXT:    [[TMP:%.*]] = fcmp ord double [[ADD]], [[ADD]]
+; CHECK-NEXT:    ret i1 [[TMP]]
+;
+  %op = uitofp i1024 %arg0 to double
+  %add = fadd double %op, %op
+  %tmp = fcmp ord double %add, %add
+  ret i1 %tmp
+}
+
 define i1 @fpext(float %arg0) {
 ; CHECK-LABEL: @fpext(
 ; CHECK-NEXT:    ret i1 false
diff --git a/llvm/test/Transforms/InstSimplify/known-non-zero.ll b/llvm/test/Transforms/InstSimplify/known-non-zero.ll
index 7e819e82b..524e51be7 100644
--- a/llvm/test/Transforms/InstSimplify/known-non-zero.ll
+++ b/llvm/test/Transforms/InstSimplify/known-non-zero.ll
@@ -7,8 +7,7 @@ define i64 @test0(i64 %x) {
 ; CHECK-NEXT:    [[A:%.*]] = icmp eq i64 [[X:%.*]], 0
 ; CHECK-NEXT:    br i1 [[A]], label [[EXIT:%.*]], label [[NON_ZERO:%.*]]
 ; CHECK:       non_zero:
-; CHECK-NEXT:    [[B:%.*]] = icmp eq i64 [[X]], 0
-; CHECK-NEXT:    br i1 [[B]], label [[UNREACHABLE:%.*]], label [[EXIT]]
+; CHECK-NEXT:    br i1 false, label [[UNREACHABLE:%.*]], label [[EXIT]]
 ; CHECK:       unreachable:
 ; CHECK-NEXT:    br label [[EXIT]]
 ; CHECK:       exit:
@@ -37,8 +36,7 @@ define i64 @test1(i64 %x) {
 ; CHECK-NEXT:    [[A:%.*]] = icmp eq i64 [[X:%.*]], 0
 ; CHECK-NEXT:    br i1 [[A]], label [[EXIT:%.*]], label [[NON_ZERO:%.*]]
 ; CHECK:       non_zero:
-; CHECK-NEXT:    [[B:%.*]] = icmp ugt i64 [[X]], 0
-; CHECK-NEXT:    br i1 [[B]], label [[EXIT]], label [[UNREACHABLE:%.*]]
+; CHECK-NEXT:    br i1 true, label [[EXIT]], label [[UNREACHABLE:%.*]]
 ; CHECK:       unreachable:
 ; CHECK-NEXT:    br label [[EXIT]]
 ; CHECK:       exit:
@@ -73,11 +71,9 @@ define i1 @test2(i64 %x, i1 %y) {
 ; CHECK:       two:
 ; CHECK-NEXT:    br label [[MAINBLOCK]]
 ; CHECK:       mainblock:
-; CHECK-NEXT:    [[P:%.*]] = phi i64 [ [[X]], [[ONE]] ], [ 42, [[TWO]] ]
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i64 [[P]], 0
 ; CHECK-NEXT:    br label [[EXIT]]
 ; CHECK:       exit:
-; CHECK-NEXT:    [[RES:%.*]] = phi i1 [ [[CMP]], [[MAINBLOCK]] ], [ true, [[START:%.*]] ]
+; CHECK-NEXT:    [[RES:%.*]] = phi i1 [ false, [[MAINBLOCK]] ], [ true, [[START:%.*]] ]
 ; CHECK-NEXT:    ret i1 [[RES]]
 ;
 start:
@@ -102,3 +98,50 @@ exit:
   %res = phi i1 [ %cmp, %mainblock ], [ 1, %start ]
   ret i1 %res
 }
+
+
+; The code below exposed a bug similar to the one exposed by D60846, see the commit 6ea477590085.
+; In a nutshell, we should not replace %result.0 with 0 here.
+
+define zeroext i8 @update_phi_query_loc_in_recursive_call(i8* nocapture readonly %p){
+; CHECK-LABEL: @update_phi_query_loc_in_recursive_call(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_COND:%.*]]
+; CHECK:       for.cond:
+; CHECK-NEXT:    [[RESULT_0:%.*]] = phi i8 [ 0, [[ENTRY:%.*]] ], [ [[CONV2:%.*]], [[FOR_BODY:%.*]] ]
+; CHECK-NEXT:    [[SHIFT_0:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ 1, [[FOR_BODY]] ]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[SHIFT_0]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_COND_CLEANUP:%.*]]
+; CHECK:       for.cond.cleanup:
+; CHECK-NEXT:    ret i8 [[RESULT_0]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, i8* [[P:%.*]], align 1
+; CHECK-NEXT:    [[CONV:%.*]] = zext i8 [[TMP0]] to i32
+; CHECK-NEXT:    [[MUL:%.*]] = shl nuw nsw i32 [[SHIFT_0]], 3
+; CHECK-NEXT:    [[SHL:%.*]] = shl nuw nsw i32 [[CONV]], [[MUL]]
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[SHL]] to i8
+; CHECK-NEXT:    [[CONV2]] = or i8 [[RESULT_0]], [[TMP1]]
+; CHECK-NEXT:    br label [[FOR_COND]]
+;
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.body, %entry
+  %result.0 = phi i8 [ 0, %entry ], [ %conv2, %for.body ]
+  %shift.0 = phi i32 [ 0, %entry ], [ 1, %for.body ]
+  %cmp = icmp eq i32 %shift.0, 0
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.cond
+  ret i8 %result.0
+
+for.body:                                         ; preds = %for.cond
+  %0 = load i8, i8* %p, align 1
+  %conv = zext i8 %0 to i32
+  %mul = shl nuw nsw i32 %shift.0, 3
+  %shl = shl nuw nsw i32 %conv, %mul
+  %1 = trunc i32 %shl to i8
+  %conv2 = or i8 %result.0, %1
+  %inc = add nuw nsw i32 %shift.0, 1
+  br label %for.cond
+}
diff --git a/llvm/test/Transforms/InstSimplify/redundant-null-check-in-uadd_with_overflow-of-nonnull-ptr.ll b/llvm/test/Transforms/InstSimplify/redundant-null-check-in-uadd_with_overflow-of-nonnull-ptr.ll
new file mode 100644
index 000000000..d409aecf5
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/redundant-null-check-in-uadd_with_overflow-of-nonnull-ptr.ll
@@ -0,0 +1,265 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instsimplify -S | FileCheck %s
+
+; Here we have add some offset to a non-null pointer,
+; and check that the result does not overflow and is not a null pointer.
+; But since the base pointer is already non-null, and we check for overflow,
+; that will already catch the get null pointer,
+; so the separate null check is redundant and can be dropped.
+
+define i1 @t0(i8* nonnull %base, i64 %offset) {
+; CHECK-LABEL: @t0(
+; CHECK-NEXT:    [[BASE_INT:%.*]] = ptrtoint i8* [[BASE:%.*]] to i64
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = add i64 [[BASE_INT]], [[OFFSET:%.*]]
+; CHECK-NEXT:    [[NO_OVERFLOW_DURING_ADJUSTMENT:%.*]] = icmp uge i64 [[ADJUSTED]], [[BASE_INT]]
+; CHECK-NEXT:    ret i1 [[NO_OVERFLOW_DURING_ADJUSTMENT]]
+;
+  %base_int = ptrtoint i8* %base to i64
+  %adjusted = add i64 %base_int, %offset
+  %non_null_after_adjustment = icmp ne i64 %adjusted, 0
+  %no_overflow_during_adjustment = icmp uge i64 %adjusted, %base_int
+  %res = and i1 %non_null_after_adjustment, %no_overflow_during_adjustment
+  ret i1 %res
+}
+define i1 @t1(i8* nonnull %base, i64 %offset) {
+; CHECK-LABEL: @t1(
+; CHECK-NEXT:    [[BASE_INT:%.*]] = ptrtoint i8* [[BASE:%.*]] to i64
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = add i64 [[BASE_INT]], [[OFFSET:%.*]]
+; CHECK-NEXT:    [[NO_OVERFLOW_DURING_ADJUSTMENT:%.*]] = icmp ule i64 [[BASE_INT]], [[ADJUSTED]]
+; CHECK-NEXT:    ret i1 [[NO_OVERFLOW_DURING_ADJUSTMENT]]
+;
+  %base_int = ptrtoint i8* %base to i64
+  %adjusted = add i64 %base_int, %offset
+  %non_null_after_adjustment = icmp ne i64 %adjusted, 0
+  %no_overflow_during_adjustment = icmp ule i64 %base_int, %adjusted ; swapped
+  %res = and i1 %non_null_after_adjustment, %no_overflow_during_adjustment
+  ret i1 %res
+}
+define i1 @t2(i8* nonnull %base, i64 %offset) {
+; CHECK-LABEL: @t2(
+; CHECK-NEXT:    [[BASE_INT:%.*]] = ptrtoint i8* [[BASE:%.*]] to i64
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = add i64 [[BASE_INT]], [[OFFSET:%.*]]
+; CHECK-NEXT:    [[NO_OVERFLOW_DURING_ADJUSTMENT:%.*]] = icmp uge i64 [[ADJUSTED]], [[BASE_INT]]
+; CHECK-NEXT:    ret i1 [[NO_OVERFLOW_DURING_ADJUSTMENT]]
+;
+  %base_int = ptrtoint i8* %base to i64
+  %adjusted = add i64 %base_int, %offset
+  %non_null_after_adjustment = icmp ne i64 %adjusted, 0
+  %no_overflow_during_adjustment = icmp uge i64 %adjusted, %base_int
+  %res = and i1 %no_overflow_during_adjustment, %non_null_after_adjustment ; swapped
+  ret i1 %res
+}
+define i1 @t3(i8* nonnull %base, i64 %offset) {
+; CHECK-LABEL: @t3(
+; CHECK-NEXT:    [[BASE_INT:%.*]] = ptrtoint i8* [[BASE:%.*]] to i64
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = add i64 [[BASE_INT]], [[OFFSET:%.*]]
+; CHECK-NEXT:    [[NO_OVERFLOW_DURING_ADJUSTMENT:%.*]] = icmp ule i64 [[BASE_INT]], [[ADJUSTED]]
+; CHECK-NEXT:    ret i1 [[NO_OVERFLOW_DURING_ADJUSTMENT]]
+;
+  %base_int = ptrtoint i8* %base to i64
+  %adjusted = add i64 %base_int, %offset
+  %non_null_after_adjustment = icmp ne i64 %adjusted, 0
+  %no_overflow_during_adjustment = icmp ule i64 %base_int, %adjusted ; swapped
+  %res = and i1 %no_overflow_during_adjustment, %non_null_after_adjustment ; swapped
+  ret i1 %res
+}
+
+; If the joining operator was 'or', i.e. we check that either we produced non-null
+; pointer, or no overflow happened, then the overflow check itself is redundant.
+
+define i1 @t4(i8* nonnull %base, i64 %offset) {
+; CHECK-LABEL: @t4(
+; CHECK-NEXT:    [[BASE_INT:%.*]] = ptrtoint i8* [[BASE:%.*]] to i64
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = add i64 [[BASE_INT]], [[OFFSET:%.*]]
+; CHECK-NEXT:    [[NON_NULL_AFTER_ADJUSTMENT:%.*]] = icmp ne i64 [[ADJUSTED]], 0
+; CHECK-NEXT:    ret i1 [[NON_NULL_AFTER_ADJUSTMENT]]
+;
+  %base_int = ptrtoint i8* %base to i64
+  %adjusted = add i64 %base_int, %offset
+  %non_null_after_adjustment = icmp ne i64 %adjusted, 0
+  %no_overflow_during_adjustment = icmp uge i64 %adjusted, %base_int
+  %res = or i1 %non_null_after_adjustment, %no_overflow_during_adjustment
+  ret i1 %res
+}
+define i1 @t5(i8* nonnull %base, i64 %offset) {
+; CHECK-LABEL: @t5(
+; CHECK-NEXT:    [[BASE_INT:%.*]] = ptrtoint i8* [[BASE:%.*]] to i64
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = add i64 [[BASE_INT]], [[OFFSET:%.*]]
+; CHECK-NEXT:    [[NON_NULL_AFTER_ADJUSTMENT:%.*]] = icmp ne i64 [[ADJUSTED]], 0
+; CHECK-NEXT:    ret i1 [[NON_NULL_AFTER_ADJUSTMENT]]
+;
+  %base_int = ptrtoint i8* %base to i64
+  %adjusted = add i64 %base_int, %offset
+  %non_null_after_adjustment = icmp ne i64 %adjusted, 0
+  %no_overflow_during_adjustment = icmp ule i64 %base_int, %adjusted ; swapped
+  %res = or i1 %non_null_after_adjustment, %no_overflow_during_adjustment
+  ret i1 %res
+}
+define i1 @t6(i8* nonnull %base, i64 %offset) {
+; CHECK-LABEL: @t6(
+; CHECK-NEXT:    [[BASE_INT:%.*]] = ptrtoint i8* [[BASE:%.*]] to i64
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = add i64 [[BASE_INT]], [[OFFSET:%.*]]
+; CHECK-NEXT:    [[NON_NULL_AFTER_ADJUSTMENT:%.*]] = icmp ne i64 [[ADJUSTED]], 0
+; CHECK-NEXT:    ret i1 [[NON_NULL_AFTER_ADJUSTMENT]]
+;
+  %base_int = ptrtoint i8* %base to i64
+  %adjusted = add i64 %base_int, %offset
+  %non_null_after_adjustment = icmp ne i64 %adjusted, 0
+  %no_overflow_during_adjustment = icmp uge i64 %adjusted, %base_int
+  %res = or i1 %no_overflow_during_adjustment, %non_null_after_adjustment ; swapped
+  ret i1 %res
+}
+define i1 @t7(i8* nonnull %base, i64 %offset) {
+; CHECK-LABEL: @t7(
+; CHECK-NEXT:    [[BASE_INT:%.*]] = ptrtoint i8* [[BASE:%.*]] to i64
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = add i64 [[BASE_INT]], [[OFFSET:%.*]]
+; CHECK-NEXT:    [[NON_NULL_AFTER_ADJUSTMENT:%.*]] = icmp ne i64 [[ADJUSTED]], 0
+; CHECK-NEXT:    ret i1 [[NON_NULL_AFTER_ADJUSTMENT]]
+;
+  %base_int = ptrtoint i8* %base to i64
+  %adjusted = add i64 %base_int, %offset
+  %non_null_after_adjustment = icmp ne i64 %adjusted, 0
+  %no_overflow_during_adjustment = icmp ule i64 %base_int, %adjusted ; swapped
+  %res = or i1 %no_overflow_during_adjustment, %non_null_after_adjustment ; swapped
+  ret i1 %res
+}
+
+; Or, we could be checking the reverse condition, that we either get null pointer,
+; or overflow happens, then again, the standalone null check is redundant and
+; can be dropped.
+
+define i1 @t8(i8* nonnull %base, i64 %offset) {
+; CHECK-LABEL: @t8(
+; CHECK-NEXT:    [[BASE_INT:%.*]] = ptrtoint i8* [[BASE:%.*]] to i64
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = add i64 [[BASE_INT]], [[OFFSET:%.*]]
+; CHECK-NEXT:    [[NO_OVERFLOW_DURING_ADJUSTMENT:%.*]] = icmp ult i64 [[ADJUSTED]], [[BASE_INT]]
+; CHECK-NEXT:    ret i1 [[NO_OVERFLOW_DURING_ADJUSTMENT]]
+;
+  %base_int = ptrtoint i8* %base to i64
+  %adjusted = add i64 %base_int, %offset
+  %non_null_after_adjustment = icmp eq i64 %adjusted, 0
+  %no_overflow_during_adjustment = icmp ult i64 %adjusted, %base_int
+  %res = or i1 %non_null_after_adjustment, %no_overflow_during_adjustment
+  ret i1 %res
+}
+define i1 @t9(i8* nonnull %base, i64 %offset) {
+; CHECK-LABEL: @t9(
+; CHECK-NEXT:    [[BASE_INT:%.*]] = ptrtoint i8* [[BASE:%.*]] to i64
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = add i64 [[BASE_INT]], [[OFFSET:%.*]]
+; CHECK-NEXT:    [[NO_OVERFLOW_DURING_ADJUSTMENT:%.*]] = icmp ugt i64 [[BASE_INT]], [[ADJUSTED]]
+; CHECK-NEXT:    ret i1 [[NO_OVERFLOW_DURING_ADJUSTMENT]]
+;
+  %base_int = ptrtoint i8* %base to i64
+  %adjusted = add i64 %base_int, %offset
+  %non_null_after_adjustment = icmp eq i64 %adjusted, 0
+  %no_overflow_during_adjustment = icmp ugt i64 %base_int, %adjusted ; swapped
+  %res = or i1 %non_null_after_adjustment, %no_overflow_during_adjustment
+  ret i1 %res
+}
+define i1 @t10(i8* nonnull %base, i64 %offset) {
+; CHECK-LABEL: @t10(
+; CHECK-NEXT:    [[BASE_INT:%.*]] = ptrtoint i8* [[BASE:%.*]] to i64
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = add i64 [[BASE_INT]], [[OFFSET:%.*]]
+; CHECK-NEXT:    [[NO_OVERFLOW_DURING_ADJUSTMENT:%.*]] = icmp ult i64 [[ADJUSTED]], [[BASE_INT]]
+; CHECK-NEXT:    ret i1 [[NO_OVERFLOW_DURING_ADJUSTMENT]]
+;
+  %base_int = ptrtoint i8* %base to i64
+  %adjusted = add i64 %base_int, %offset
+  %non_null_after_adjustment = icmp eq i64 %adjusted, 0
+  %no_overflow_during_adjustment = icmp ult i64 %adjusted, %base_int
+  %res = or i1 %no_overflow_during_adjustment, %non_null_after_adjustment ; swapped
+  ret i1 %res
+}
+define i1 @t11(i8* nonnull %base, i64 %offset) {
+; CHECK-LABEL: @t11(
+; CHECK-NEXT:    [[BASE_INT:%.*]] = ptrtoint i8* [[BASE:%.*]] to i64
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = add i64 [[BASE_INT]], [[OFFSET:%.*]]
+; CHECK-NEXT:    [[NO_OVERFLOW_DURING_ADJUSTMENT:%.*]] = icmp ugt i64 [[BASE_INT]], [[ADJUSTED]]
+; CHECK-NEXT:    ret i1 [[NO_OVERFLOW_DURING_ADJUSTMENT]]
+;
+  %base_int = ptrtoint i8* %base to i64
+  %adjusted = add i64 %base_int, %offset
+  %non_null_after_adjustment = icmp eq i64 %adjusted, 0
+  %no_overflow_during_adjustment = icmp ugt i64 %base_int, %adjusted ; swapped
+  %res = or i1 %no_overflow_during_adjustment, %non_null_after_adjustment ; swapped
+  ret i1 %res
+}
+
+; If the joining operator was 'and', i.e. we check that we both get null pointer
+; AND overflow happens, then the overflow check is redundant.
+
+define i1 @t12(i8* nonnull %base, i64 %offset) {
+; CHECK-LABEL: @t12(
+; CHECK-NEXT:    [[BASE_INT:%.*]] = ptrtoint i8* [[BASE:%.*]] to i64
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = add i64 [[BASE_INT]], [[OFFSET:%.*]]
+; CHECK-NEXT:    [[NON_NULL_AFTER_ADJUSTMENT:%.*]] = icmp eq i64 [[ADJUSTED]], 0
+; CHECK-NEXT:    ret i1 [[NON_NULL_AFTER_ADJUSTMENT]]
+;
+  %base_int = ptrtoint i8* %base to i64
+  %adjusted = add i64 %base_int, %offset
+  %non_null_after_adjustment = icmp eq i64 %adjusted, 0
+  %no_overflow_during_adjustment = icmp ult i64 %adjusted, %base_int
+  %res = and i1 %non_null_after_adjustment, %no_overflow_during_adjustment
+  ret i1 %res
+}
+define i1 @t13(i8* nonnull %base, i64 %offset) {
+; CHECK-LABEL: @t13(
+; CHECK-NEXT:    [[BASE_INT:%.*]] = ptrtoint i8* [[BASE:%.*]] to i64
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = add i64 [[BASE_INT]], [[OFFSET:%.*]]
+; CHECK-NEXT:    [[NON_NULL_AFTER_ADJUSTMENT:%.*]] = icmp eq i64 [[ADJUSTED]], 0
+; CHECK-NEXT:    ret i1 [[NON_NULL_AFTER_ADJUSTMENT]]
+;
+  %base_int = ptrtoint i8* %base to i64
+  %adjusted = add i64 %base_int, %offset
+  %non_null_after_adjustment = icmp eq i64 %adjusted, 0
+  %no_overflow_during_adjustment = icmp ugt i64 %base_int, %adjusted ; swapped
+  %res = and i1 %non_null_after_adjustment, %no_overflow_during_adjustment
+  ret i1 %res
+}
+define i1 @t14(i8* nonnull %base, i64 %offset) {
+; CHECK-LABEL: @t14(
+; CHECK-NEXT:    [[BASE_INT:%.*]] = ptrtoint i8* [[BASE:%.*]] to i64
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = add i64 [[BASE_INT]], [[OFFSET:%.*]]
+; CHECK-NEXT:    [[NON_NULL_AFTER_ADJUSTMENT:%.*]] = icmp eq i64 [[ADJUSTED]], 0
+; CHECK-NEXT:    ret i1 [[NON_NULL_AFTER_ADJUSTMENT]]
+;
+  %base_int = ptrtoint i8* %base to i64
+  %adjusted = add i64 %base_int, %offset
+  %non_null_after_adjustment = icmp eq i64 %adjusted, 0
+  %no_overflow_during_adjustment = icmp ult i64 %adjusted, %base_int
+  %res = and i1 %no_overflow_during_adjustment, %non_null_after_adjustment ; swapped
+  ret i1 %res
+}
+define i1 @t15(i8* nonnull %base, i64 %offset) {
+; CHECK-LABEL: @t15(
+; CHECK-NEXT:    [[BASE_INT:%.*]] = ptrtoint i8* [[BASE:%.*]] to i64
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = add i64 [[BASE_INT]], [[OFFSET:%.*]]
+; CHECK-NEXT:    [[NON_NULL_AFTER_ADJUSTMENT:%.*]] = icmp eq i64 [[ADJUSTED]], 0
+; CHECK-NEXT:    ret i1 [[NON_NULL_AFTER_ADJUSTMENT]]
+;
+  %base_int = ptrtoint i8* %base to i64
+  %adjusted = add i64 %base_int, %offset
+  %non_null_after_adjustment = icmp eq i64 %adjusted, 0
+  %no_overflow_during_adjustment = icmp ugt i64 %base_int, %adjusted ; swapped
+  %res = and i1 %no_overflow_during_adjustment, %non_null_after_adjustment ; swapped
+  ret i1 %res
+}
+
+declare void @llvm.assume(i1)
+define i1 @t16(i64 %base, i64 %offset) {
+; CHECK-LABEL: @t16(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i64 [[BASE:%.*]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = add i64 [[BASE]], [[OFFSET:%.*]]
+; CHECK-NEXT:    [[NO_OVERFLOW_DURING_ADJUSTMENT:%.*]] = icmp uge i64 [[ADJUSTED]], [[BASE]]
+; CHECK-NEXT:    ret i1 [[NO_OVERFLOW_DURING_ADJUSTMENT]]
+;
+  %cmp = icmp slt i64 %base, 0
+  call void @llvm.assume(i1 %cmp)
+
+  %adjusted = add i64 %base, %offset
+  %non_null_after_adjustment = icmp ne i64 %adjusted, 0
+  %no_overflow_during_adjustment = icmp uge i64 %adjusted, %base
+  %res = and i1 %non_null_after_adjustment, %no_overflow_during_adjustment
+  ret i1 %res
+}
diff --git a/llvm/test/Transforms/InstSimplify/remove-dead-call.ll b/llvm/test/Transforms/InstSimplify/remove-dead-call.ll
new file mode 100755
index 000000000..172a99131
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/remove-dead-call.ll
@@ -0,0 +1,28 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instsimplify -S -o /dev/null -debug-pass=Details 2>&1 | FileCheck --check-prefix DETAILS %s
+; RUN: opt < %s -instsimplify -S -o - | FileCheck %s
+
+; Verify that InstSimplifyLegacyPass notifies the pass manager about changes
+; being made (when a call is removed CGSCC must be updated).
+;
+; DETAILS: Made Modification 'Remove redundant instructions' on Function 'main'
+
+define internal void @func_1(i64* nocapture readnone %0) #0 {
+; CHECK-LABEL: @func_1(
+; CHECK-NEXT:    unreachable
+;
+  unreachable
+}
+
+define i16 @main(i16 %0, i16** nocapture readnone %1) #1 {
+; CHECK-LABEL: @main(
+; CHECK-NEXT:  bb1:
+; CHECK-NEXT:    unreachable
+;
+bb1:
+  call void @func_1(i64* undef)
+  unreachable
+}
+
+attributes #0 = { noinline norecurse nounwind readnone }
+attributes #1 = { norecurse nounwind readnone }
diff --git a/llvm/test/Transforms/InstSimplify/result-of-add-of-negative-is-non-zero-and-no-underflow.ll b/llvm/test/Transforms/InstSimplify/result-of-add-of-negative-is-non-zero-and-no-underflow.ll
new file mode 100644
index 000000000..9521a71bd
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/result-of-add-of-negative-is-non-zero-and-no-underflow.ll
@@ -0,0 +1,125 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instsimplify -S | FileCheck %s
+
+; Here we add unsigned two values, check that addition did not underflow AND
+; that the result is non-zero. This can be simplified just to a comparison
+; between the base and negated offset.
+
+declare void @use8(i8)
+
+declare void @use1(i1)
+declare void @llvm.assume(i1)
+
+; If we are checking that the result is not null or no underflow happened,
+; it is tautological (always-true).
+define i1 @t1(i8 %base, i8 %offset) {
+; CHECK-LABEL: @t1(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[BASE:%.*]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = add i8 [[BASE]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp ne i8 [[ADJUSTED]], 0
+; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp ult i8 [[ADJUSTED]], [[BASE]]
+; CHECK-NEXT:    [[R:%.*]] = or i1 [[NOT_NULL]], [[NO_UNDERFLOW]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %cmp = icmp slt i8 %base, 0
+  call void @llvm.assume(i1 %cmp)
+
+  %adjusted = add i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp ne i8 %adjusted, 0
+  %no_underflow = icmp ult i8 %adjusted, %base
+  %r = or i1 %not_null, %no_underflow
+  ret i1 %r
+}
+define i1 @t2_commutative(i8 %base, i8 %offset) {
+; CHECK-LABEL: @t2_commutative(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[BASE:%.*]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = add i8 [[BASE]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp ne i8 [[ADJUSTED]], 0
+; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp ugt i8 [[BASE]], [[ADJUSTED]]
+; CHECK-NEXT:    [[R:%.*]] = or i1 [[NOT_NULL]], [[NO_UNDERFLOW]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %cmp = icmp slt i8 %base, 0
+  call void @llvm.assume(i1 %cmp)
+
+  %adjusted = add i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp ne i8 %adjusted, 0
+  %no_underflow = icmp ugt i8 %base, %adjusted
+  %r = or i1 %not_null, %no_underflow
+  ret i1 %r
+}
+
+; If we are checking that the result is null and underflow happened,
+; it is tautological (always-false).
+define i1 @t3(i8 %base, i8 %offset) {
+; CHECK-LABEL: @t3(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[BASE:%.*]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = add i8 [[BASE]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp eq i8 [[ADJUSTED]], 0
+; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp uge i8 [[ADJUSTED]], [[BASE]]
+; CHECK-NEXT:    [[R:%.*]] = and i1 [[NOT_NULL]], [[NO_UNDERFLOW]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %cmp = icmp slt i8 %base, 0
+  call void @llvm.assume(i1 %cmp)
+
+  %adjusted = add i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp eq i8 %adjusted, 0
+  %no_underflow = icmp uge i8 %adjusted, %base
+  %r = and i1 %not_null, %no_underflow
+  ret i1 %r
+}
+define i1 @t4_commutative(i8 %base, i8 %offset) {
+; CHECK-LABEL: @t4_commutative(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[BASE:%.*]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = add i8 [[BASE]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp eq i8 [[ADJUSTED]], 0
+; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp ule i8 [[BASE]], [[ADJUSTED]]
+; CHECK-NEXT:    [[R:%.*]] = and i1 [[NOT_NULL]], [[NO_UNDERFLOW]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %cmp = icmp slt i8 %base, 0
+  call void @llvm.assume(i1 %cmp)
+
+  %adjusted = add i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp eq i8 %adjusted, 0
+  %no_underflow = icmp ule i8 %base, %adjusted
+  %r = and i1 %not_null, %no_underflow
+  ret i1 %r
+}
+
+; We only need to know that any of the 'add' operands is non-zero,
+; not necessarily the one used in the comparison.
+define i1 @t5(i8 %base, i8 %offset) {
+; CHECK-LABEL: @t5(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[OFFSET:%.*]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = add i8 [[BASE:%.*]], [[OFFSET]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp ne i8 [[ADJUSTED]], 0
+; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp ult i8 [[ADJUSTED]], [[BASE]]
+; CHECK-NEXT:    [[R:%.*]] = or i1 [[NOT_NULL]], [[NO_UNDERFLOW]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %cmp = icmp slt i8 %offset, 0
+  call void @llvm.assume(i1 %cmp)
+
+  %adjusted = add i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp ne i8 %adjusted, 0
+  %no_underflow = icmp ult i8 %adjusted, %base
+  %r = or i1 %not_null, %no_underflow
+  ret i1 %r
+}
diff --git a/llvm/test/Transforms/InstSimplify/result-of-usub-by-nonzero-is-non-zero-and-no-overflow.ll b/llvm/test/Transforms/InstSimplify/result-of-usub-by-nonzero-is-non-zero-and-no-overflow.ll
new file mode 100644
index 000000000..f52f5e55d
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/result-of-usub-by-nonzero-is-non-zero-and-no-overflow.ll
@@ -0,0 +1,86 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instsimplify -S | FileCheck %s
+
+; Here we subtract two values, check that subtraction did not overflow AND
+; that the result is non-zero. This can be simplified just to a comparison
+; between the base and offset.
+
+define i1 @t0(i64 %base, i64* nonnull %offsetptr) {
+; CHECK-LABEL: @t0(
+; CHECK-NEXT:    [[OFFSET:%.*]] = ptrtoint i64* [[OFFSETPTR:%.*]] to i64
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i64 [[BASE:%.*]], [[OFFSET]]
+; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp uge i64 [[ADJUSTED]], [[BASE]]
+; CHECK-NEXT:    ret i1 [[NO_UNDERFLOW]]
+;
+  %offset = ptrtoint i64* %offsetptr to i64
+
+  %adjusted = sub i64 %base, %offset
+  %no_underflow = icmp uge i64 %adjusted, %base
+  %not_null = icmp ne i64 %adjusted, 0
+  %r = and i1 %not_null, %no_underflow
+  ret i1 %r
+}
+
+define i1 @t1(i64 %base, i64* nonnull %offsetptr) {
+; CHECK-LABEL: @t1(
+; CHECK-NEXT:    [[OFFSET:%.*]] = ptrtoint i64* [[OFFSETPTR:%.*]] to i64
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i64 [[BASE:%.*]], [[OFFSET]]
+; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp ult i64 [[ADJUSTED]], [[BASE]]
+; CHECK-NEXT:    ret i1 [[NO_UNDERFLOW]]
+;
+  %offset = ptrtoint i64* %offsetptr to i64
+
+  %adjusted = sub i64 %base, %offset
+  %no_underflow = icmp ult i64 %adjusted, %base
+  %not_null = icmp eq i64 %adjusted, 0
+  %r = or i1 %not_null, %no_underflow
+  ret i1 %r
+}
+
+define i1 @t2_commutative(i64 %base, i64* nonnull %offsetptr) {
+; CHECK-LABEL: @t2_commutative(
+; CHECK-NEXT:    [[OFFSET:%.*]] = ptrtoint i64* [[OFFSETPTR:%.*]] to i64
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i64 [[BASE:%.*]], [[OFFSET]]
+; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp ule i64 [[BASE]], [[ADJUSTED]]
+; CHECK-NEXT:    ret i1 [[NO_UNDERFLOW]]
+;
+  %offset = ptrtoint i64* %offsetptr to i64
+
+  %adjusted = sub i64 %base, %offset
+  %no_underflow = icmp ule i64 %base, %adjusted
+  %not_null = icmp ne i64 %adjusted, 0
+  %r = and i1 %not_null, %no_underflow
+  ret i1 %r
+}
+
+define i1 @t3_commutative(i64 %base, i64* nonnull %offsetptr) {
+; CHECK-LABEL: @t3_commutative(
+; CHECK-NEXT:    [[OFFSET:%.*]] = ptrtoint i64* [[OFFSETPTR:%.*]] to i64
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i64 [[BASE:%.*]], [[OFFSET]]
+; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp ugt i64 [[BASE]], [[ADJUSTED]]
+; CHECK-NEXT:    ret i1 [[NO_UNDERFLOW]]
+;
+  %offset = ptrtoint i64* %offsetptr to i64
+
+  %adjusted = sub i64 %base, %offset
+  %no_underflow = icmp ugt i64 %base, %adjusted
+  %not_null = icmp eq i64 %adjusted, 0
+  %r = or i1 %not_null, %no_underflow
+  ret i1 %r
+}
+
+; We don't know that offset is non-zero, so we can't fold.
+define i1 @t4_bad(i64 %base, i64 %offset) {
+; CHECK-LABEL: @t4_bad(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i64 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp uge i64 [[ADJUSTED]], [[BASE]]
+; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp ne i64 [[ADJUSTED]], 0
+; CHECK-NEXT:    [[R:%.*]] = and i1 [[NOT_NULL]], [[NO_UNDERFLOW]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %adjusted = sub i64 %base, %offset
+  %no_underflow = icmp uge i64 %adjusted, %base
+  %not_null = icmp ne i64 %adjusted, 0
+  %r = and i1 %not_null, %no_underflow
+  ret i1 %r
+}
diff --git a/llvm/test/Transforms/InstSimplify/result-of-usub-is-non-zero-and-no-overflow.ll b/llvm/test/Transforms/InstSimplify/result-of-usub-is-non-zero-and-no-overflow.ll
new file mode 100644
index 000000000..fd473ac9d
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/result-of-usub-is-non-zero-and-no-overflow.ll
@@ -0,0 +1,339 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instsimplify -S | FileCheck %s
+
+; Here we subtract two values, check that subtraction did not overflow AND
+; that the result is non-zero. This can be simplified just to a comparison
+; between the base and offset.
+
+declare void @use8(i8)
+declare void @use64(i64)
+declare void @use1(i1)
+
+declare void @llvm.assume(i1)
+
+; If we are checking that we either did not get null or got no overflow,
+; this is tautological and is always true.
+
+define i1 @commutativity0(i8 %base, i8 %offset) {
+; CHECK-LABEL: @commutativity0(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i8 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp uge i8 [[BASE]], [[OFFSET]]
+; CHECK-NEXT:    call void @use1(i1 [[NO_UNDERFLOW]])
+; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp ne i8 [[ADJUSTED]], 0
+; CHECK-NEXT:    call void @use1(i1 [[NOT_NULL]])
+; CHECK-NEXT:    ret i1 true
+;
+  %adjusted = sub i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %no_underflow = icmp uge i8 %base, %offset
+  call void @use1(i1 %no_underflow)
+  %not_null = icmp ne i8 %adjusted, 0
+  call void @use1(i1 %not_null)
+  %r = or i1 %not_null, %no_underflow
+  ret i1 %r
+}
+define i1 @commutativity1(i8 %base, i8 %offset) {
+; CHECK-LABEL: @commutativity1(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i8 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp ule i8 [[OFFSET]], [[BASE]]
+; CHECK-NEXT:    call void @use1(i1 [[NO_UNDERFLOW]])
+; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp ne i8 [[ADJUSTED]], 0
+; CHECK-NEXT:    call void @use1(i1 [[NOT_NULL]])
+; CHECK-NEXT:    ret i1 true
+;
+  %adjusted = sub i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %no_underflow = icmp ule i8 %offset, %base ; swapped
+  call void @use1(i1 %no_underflow)
+  %not_null = icmp ne i8 %adjusted, 0
+  call void @use1(i1 %not_null)
+  %r = or i1 %not_null, %no_underflow
+  ret i1 %r
+}
+define i1 @commutativity2(i8 %base, i8 %offset) {
+; CHECK-LABEL: @commutativity2(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i8 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp uge i8 [[BASE]], [[OFFSET]]
+; CHECK-NEXT:    call void @use1(i1 [[NO_UNDERFLOW]])
+; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp ne i8 [[ADJUSTED]], 0
+; CHECK-NEXT:    call void @use1(i1 [[NOT_NULL]])
+; CHECK-NEXT:    ret i1 true
+;
+  %adjusted = sub i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %no_underflow = icmp uge i8 %base, %offset
+  call void @use1(i1 %no_underflow)
+  %not_null = icmp ne i8 %adjusted, 0
+  call void @use1(i1 %not_null)
+  %r = or i1 %no_underflow, %not_null ; swapped
+  ret i1 %r
+}
+
+define i1 @commutativity3(i8 %base, i8 %offset) {
+; CHECK-LABEL: @commutativity3(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i8 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp ule i8 [[OFFSET]], [[BASE]]
+; CHECK-NEXT:    call void @use1(i1 [[NO_UNDERFLOW]])
+; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp ne i8 [[ADJUSTED]], 0
+; CHECK-NEXT:    call void @use1(i1 [[NOT_NULL]])
+; CHECK-NEXT:    ret i1 true
+;
+  %adjusted = sub i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %no_underflow = icmp ule i8 %offset, %base ; swapped
+  call void @use1(i1 %no_underflow)
+  %not_null = icmp ne i8 %adjusted, 0
+  call void @use1(i1 %not_null)
+  %r = or i1 %no_underflow, %not_null ; swapped
+  ret i1 %r
+}
+
+;-------------------------------------------------------------------------------
+
+define i1 @exaustive_t0_no_underflow(i8 %base, i8 %offset) {
+; CHECK-LABEL: @exaustive_t0_no_underflow(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i8 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp ult i8 [[BASE]], [[OFFSET]]
+; CHECK-NEXT:    ret i1 [[NO_UNDERFLOW]]
+;
+  %adjusted = sub i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp ne i8 %adjusted, 0
+  %no_underflow = icmp ult i8 %base, %offset
+  %r = and i1 %no_underflow, %not_null
+  ret i1 %r
+}
+
+define i1 @exaustive_t1_not_null(i8 %base, i8 %offset) {
+; CHECK-LABEL: @exaustive_t1_not_null(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i8 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp ne i8 [[ADJUSTED]], 0
+; CHECK-NEXT:    ret i1 [[NOT_NULL]]
+;
+  %adjusted = sub i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp ne i8 %adjusted, 0
+  %no_underflow = icmp ult i8 %base, %offset
+  %r = or i1 %no_underflow, %not_null
+  ret i1 %r
+}
+
+define i1 @exaustive_t2_false(i8 %base, i8 %offset) {
+; CHECK-LABEL: @exaustive_t2_false(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i8 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    ret i1 false
+;
+  %adjusted = sub i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp eq i8 %adjusted, 0
+  %no_underflow = icmp ult i8 %base, %offset
+  %r = and i1 %no_underflow, %not_null
+  ret i1 %r
+}
+
+define i1 @exaustive_t3_bad(i8 %base, i8 %offset) {
+; CHECK-LABEL: @exaustive_t3_bad(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i8 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp eq i8 [[ADJUSTED]], 0
+; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp ult i8 [[BASE]], [[OFFSET]]
+; CHECK-NEXT:    [[R:%.*]] = or i1 [[NO_UNDERFLOW]], [[NOT_NULL]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %adjusted = sub i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp eq i8 %adjusted, 0
+  %no_underflow = icmp ult i8 %base, %offset
+  %r = or i1 %no_underflow, %not_null
+  ret i1 %r
+}
+
+define i1 @exaustive_t4_no_underflow(i8 %base, i8 %offset) {
+; CHECK-LABEL: @exaustive_t4_no_underflow(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i8 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp ugt i8 [[BASE]], [[OFFSET]]
+; CHECK-NEXT:    ret i1 [[NO_UNDERFLOW]]
+;
+  %adjusted = sub i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp ne i8 %adjusted, 0
+  %no_underflow = icmp ugt i8 %base, %offset
+  %r = and i1 %no_underflow, %not_null
+  ret i1 %r
+}
+
+define i1 @exaustive_t5_not_null(i8 %base, i8 %offset) {
+; CHECK-LABEL: @exaustive_t5_not_null(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i8 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp ne i8 [[ADJUSTED]], 0
+; CHECK-NEXT:    ret i1 [[NOT_NULL]]
+;
+  %adjusted = sub i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp ne i8 %adjusted, 0
+  %no_underflow = icmp ugt i8 %base, %offset
+  %r = or i1 %no_underflow, %not_null
+  ret i1 %r
+}
+
+define i1 @exaustive_t6_false(i8 %base, i8 %offset) {
+; CHECK-LABEL: @exaustive_t6_false(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i8 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    ret i1 false
+;
+  %adjusted = sub i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp eq i8 %adjusted, 0
+  %no_underflow = icmp ugt i8 %base, %offset
+  %r = and i1 %no_underflow, %not_null
+  ret i1 %r
+}
+
+define i1 @exaustive_t7_bad(i8 %base, i8 %offset) {
+; CHECK-LABEL: @exaustive_t7_bad(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i8 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp eq i8 [[ADJUSTED]], 0
+; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp ugt i8 [[BASE]], [[OFFSET]]
+; CHECK-NEXT:    [[R:%.*]] = or i1 [[NO_UNDERFLOW]], [[NOT_NULL]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %adjusted = sub i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp eq i8 %adjusted, 0
+  %no_underflow = icmp ugt i8 %base, %offset
+  %r = or i1 %no_underflow, %not_null
+  ret i1 %r
+}
+
+define i1 @exaustive_t8_bad(i8 %base, i8 %offset) {
+; CHECK-LABEL: @exaustive_t8_bad(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i8 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp ne i8 [[ADJUSTED]], 0
+; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp ule i8 [[BASE]], [[OFFSET]]
+; CHECK-NEXT:    [[R:%.*]] = and i1 [[NO_UNDERFLOW]], [[NOT_NULL]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %adjusted = sub i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp ne i8 %adjusted, 0
+  %no_underflow = icmp ule i8 %base, %offset
+  %r = and i1 %no_underflow, %not_null
+  ret i1 %r
+}
+
+define i1 @exaustive_t9_true(i8 %base, i8 %offset) {
+; CHECK-LABEL: @exaustive_t9_true(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i8 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    ret i1 true
+;
+  %adjusted = sub i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp ne i8 %adjusted, 0
+  %no_underflow = icmp ule i8 %base, %offset
+  %r = or i1 %no_underflow, %not_null
+  ret i1 %r
+}
+
+define i1 @exaustive_t10_not_null(i8 %base, i8 %offset) {
+; CHECK-LABEL: @exaustive_t10_not_null(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i8 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp eq i8 [[ADJUSTED]], 0
+; CHECK-NEXT:    ret i1 [[NOT_NULL]]
+;
+  %adjusted = sub i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp eq i8 %adjusted, 0
+  %no_underflow = icmp ule i8 %base, %offset
+  %r = and i1 %no_underflow, %not_null
+  ret i1 %r
+}
+
+define i1 @exaustive_t11_no_underflow(i8 %base, i8 %offset) {
+; CHECK-LABEL: @exaustive_t11_no_underflow(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i8 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp ule i8 [[BASE]], [[OFFSET]]
+; CHECK-NEXT:    ret i1 [[NO_UNDERFLOW]]
+;
+  %adjusted = sub i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp eq i8 %adjusted, 0
+  %no_underflow = icmp ule i8 %base, %offset
+  %r = or i1 %no_underflow, %not_null
+  ret i1 %r
+}
+
+define i1 @exaustive_t12_bad(i8 %base, i8 %offset) {
+; CHECK-LABEL: @exaustive_t12_bad(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i8 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp ne i8 [[ADJUSTED]], 0
+; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp uge i8 [[BASE]], [[OFFSET]]
+; CHECK-NEXT:    [[R:%.*]] = and i1 [[NO_UNDERFLOW]], [[NOT_NULL]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %adjusted = sub i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp ne i8 %adjusted, 0
+  %no_underflow = icmp uge i8 %base, %offset
+  %r = and i1 %no_underflow, %not_null
+  ret i1 %r
+}
+
+define i1 @exaustive_t13_true(i8 %base, i8 %offset) {
+; CHECK-LABEL: @exaustive_t13_true(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i8 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    ret i1 true
+;
+  %adjusted = sub i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp ne i8 %adjusted, 0
+  %no_underflow = icmp uge i8 %base, %offset
+  %r = or i1 %no_underflow, %not_null
+  ret i1 %r
+}
+
+define i1 @exaustive_t14_not_null(i8 %base, i8 %offset) {
+; CHECK-LABEL: @exaustive_t14_not_null(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i8 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp eq i8 [[ADJUSTED]], 0
+; CHECK-NEXT:    ret i1 [[NOT_NULL]]
+;
+  %adjusted = sub i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp eq i8 %adjusted, 0
+  %no_underflow = icmp uge i8 %base, %offset
+  %r = and i1 %no_underflow, %not_null
+  ret i1 %r
+}
+
+define i1 @exaustive_t15_no_underflow(i8 %base, i8 %offset) {
+; CHECK-LABEL: @exaustive_t15_no_underflow(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i8 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp uge i8 [[BASE]], [[OFFSET]]
+; CHECK-NEXT:    ret i1 [[NO_UNDERFLOW]]
+;
+  %adjusted = sub i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp eq i8 %adjusted, 0
+  %no_underflow = icmp uge i8 %base, %offset
+  %r = or i1 %no_underflow, %not_null
+  ret i1 %r
+}
diff --git a/llvm/test/Transforms/InstSimplify/select.ll b/llvm/test/Transforms/InstSimplify/select.ll
index d640805f8..433b4e495 100644
--- a/llvm/test/Transforms/InstSimplify/select.ll
+++ b/llvm/test/Transforms/InstSimplify/select.ll
@@ -1,9 +1,74 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -instsimplify -S | FileCheck %s
 
+define i1 @bool_true_or_false(i1 %cond) {
+; CHECK-LABEL: @bool_true_or_false(
+; CHECK-NEXT:    ret i1 [[COND:%.*]]
+;
+  %s = select i1 %cond, i1 true, i1 false
+  ret i1 %s
+}
+
+define <2 x i1> @bool_true_or_false_vec(<2 x i1> %cond) {
+; CHECK-LABEL: @bool_true_or_false_vec(
+; CHECK-NEXT:    ret <2 x i1> [[COND:%.*]]
+;
+  %s = select <2 x i1> %cond, <2 x i1> <i1 true, i1 true>, <2 x i1> zeroinitializer
+  ret <2 x i1> %s
+}
+
+define <2 x i1> @bool_true_or_false_vec_undef(<2 x i1> %cond) {
+; CHECK-LABEL: @bool_true_or_false_vec_undef(
+; CHECK-NEXT:    ret <2 x i1> [[COND:%.*]]
+;
+  %s = select <2 x i1> %cond, <2 x i1> <i1 undef, i1 true>, <2 x i1> <i1 false, i1 undef>
+  ret <2 x i1> %s
+}
+
+define i32 @cond_is_false(i32 %A, i32 %B) {
+; CHECK-LABEL: @cond_is_false(
+; CHECK-NEXT:    ret i32 [[B:%.*]]
+;
+  %C = select i1 false, i32 %A, i32 %B
+  ret i32 %C
+}
+
+define i32 @cond_is_true(i32 %A, i32 %B) {
+; CHECK-LABEL: @cond_is_true(
+; CHECK-NEXT:    ret i32 [[A:%.*]]
+;
+  %C = select i1 true, i32 %A, i32 %B
+  ret i32 %C
+}
+
+define i32 @equal_arms(i1 %cond, i32 %x) {
+; CHECK-LABEL: @equal_arms(
+; CHECK-NEXT:    ret i32 [[X:%.*]]
+;
+  %V = select i1 %cond, i32 %x, i32 %x
+  ret i32 %V
+}
+
+define <2 x i32> @equal_arms_vec(<2 x i1> %cond, <2 x i32> %x) {
+; CHECK-LABEL: @equal_arms_vec(
+; CHECK-NEXT:    ret <2 x i32> [[X:%.*]]
+;
+  %V = select <2 x i1> %cond, <2 x i32> %x, <2 x i32> %x
+  ret <2 x i32> %V
+}
+
+define <2 x i32> @equal_arms_vec_undef(<2 x i1> %cond) {
+; CHECK-LABEL: @equal_arms_vec_undef(
+; CHECK-NEXT:    [[V:%.*]] = select <2 x i1> [[COND:%.*]], <2 x i32> <i32 42, i32 undef>, <2 x i32> <i32 undef, i32 42>
+; CHECK-NEXT:    ret <2 x i32> [[V]]
+;
+  %V = select <2 x i1> %cond, <2 x i32> <i32 42, i32 undef>, <2 x i32> <i32 undef, i32 42>
+  ret <2 x i32> %V
+}
+
 define <2 x i8> @vsel_tvec(<2 x i8> %x, <2 x i8> %y) {
 ; CHECK-LABEL: @vsel_tvec(
-; CHECK-NEXT:    ret <2 x i8> %x
+; CHECK-NEXT:    ret <2 x i8> [[X:%.*]]
 ;
   %s = select <2 x i1><i1 true, i1 true>, <2 x i8> %x, <2 x i8> %y
   ret <2 x i8> %s
@@ -11,7 +76,7 @@ define <2 x i8> @vsel_tvec(<2 x i8> %x, <2 x i8> %y) {
 
 define <2 x i8> @vsel_fvec(<2 x i8> %x, <2 x i8> %y) {
 ; CHECK-LABEL: @vsel_fvec(
-; CHECK-NEXT:    ret <2 x i8> %y
+; CHECK-NEXT:    ret <2 x i8> [[Y:%.*]]
 ;
   %s = select <2 x i1><i1 false, i1 false>, <2 x i8> %x, <2 x i8> %y
   ret <2 x i8> %s
@@ -47,7 +112,7 @@ define <3 x i4> @vsel_undef_false_op(<3 x i4> %x, <3 x i4> %y) {
 
 define i32 @test1(i32 %x) {
 ; CHECK-LABEL: @test1(
-; CHECK-NEXT:    ret i32 %x
+; CHECK-NEXT:    ret i32 [[X:%.*]]
 ;
   %and = and i32 %x, 1
   %cmp = icmp eq i32 %and, 0
@@ -58,7 +123,7 @@ define i32 @test1(i32 %x) {
 
 define i32 @test2(i32 %x) {
 ; CHECK-LABEL: @test2(
-; CHECK-NEXT:    ret i32 %x
+; CHECK-NEXT:    ret i32 [[X:%.*]]
 ;
   %and = and i32 %x, 1
   %cmp = icmp ne i32 %and, 0
@@ -69,7 +134,7 @@ define i32 @test2(i32 %x) {
 
 define i32 @test3(i32 %x) {
 ; CHECK-LABEL: @test3(
-; CHECK-NEXT:    [[AND1:%.*]] = and i32 %x, -2
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[X:%.*]], -2
 ; CHECK-NEXT:    ret i32 [[AND1]]
 ;
   %and = and i32 %x, 1
@@ -81,7 +146,7 @@ define i32 @test3(i32 %x) {
 
 define i32 @test4(i32 %X) {
 ; CHECK-LABEL: @test4(
-; CHECK-NEXT:    [[OR:%.*]] = or i32 %X, -2147483648
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[X:%.*]], -2147483648
 ; CHECK-NEXT:    ret i32 [[OR]]
 ;
   %cmp = icmp slt i32 %X, 0
@@ -104,7 +169,7 @@ define i32 @test4noncanon(i32 %X) {
 
 define i32 @test5(i32 %X) {
 ; CHECK-LABEL: @test5(
-; CHECK-NEXT:    ret i32 %X
+; CHECK-NEXT:    ret i32 [[X:%.*]]
 ;
   %cmp = icmp slt i32 %X, 0
   %or = or i32 %X, -2147483648
@@ -114,7 +179,7 @@ define i32 @test5(i32 %X) {
 
 define i32 @test6(i32 %X) {
 ; CHECK-LABEL: @test6(
-; CHECK-NEXT:    [[AND:%.*]] = and i32 %X, 2147483647
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 2147483647
 ; CHECK-NEXT:    ret i32 [[AND]]
 ;
   %cmp = icmp slt i32 %X, 0
@@ -125,7 +190,7 @@ define i32 @test6(i32 %X) {
 
 define i32 @test7(i32 %X) {
 ; CHECK-LABEL: @test7(
-; CHECK-NEXT:    ret i32 %X
+; CHECK-NEXT:    ret i32 [[X:%.*]]
 ;
   %cmp = icmp slt i32 %X, 0
   %and = and i32 %X, 2147483647
@@ -135,7 +200,7 @@ define i32 @test7(i32 %X) {
 
 define i32 @test8(i32 %X) {
 ; CHECK-LABEL: @test8(
-; CHECK-NEXT:    ret i32 %X
+; CHECK-NEXT:    ret i32 [[X:%.*]]
 ;
   %cmp = icmp sgt i32 %X, -1
   %or = or i32 %X, -2147483648
@@ -145,7 +210,7 @@ define i32 @test8(i32 %X) {
 
 define i32 @test9(i32 %X) {
 ; CHECK-LABEL: @test9(
-; CHECK-NEXT:    [[OR:%.*]] = or i32 %X, -2147483648
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[X:%.*]], -2147483648
 ; CHECK-NEXT:    ret i32 [[OR]]
 ;
   %cmp = icmp sgt i32 %X, -1
@@ -168,7 +233,7 @@ define i32 @test9noncanon(i32 %X) {
 
 define i32 @test10(i32 %X) {
 ; CHECK-LABEL: @test10(
-; CHECK-NEXT:    ret i32 %X
+; CHECK-NEXT:    ret i32 [[X:%.*]]
 ;
   %cmp = icmp sgt i32 %X, -1
   %and = and i32 %X, 2147483647
@@ -178,7 +243,7 @@ define i32 @test10(i32 %X) {
 
 define i32 @test11(i32 %X) {
 ; CHECK-LABEL: @test11(
-; CHECK-NEXT:    [[AND:%.*]] = and i32 %X, 2147483647
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 2147483647
 ; CHECK-NEXT:    ret i32 [[AND]]
 ;
   %cmp = icmp sgt i32 %X, -1
@@ -189,7 +254,7 @@ define i32 @test11(i32 %X) {
 
 define <2 x i8> @test11vec(<2 x i8> %X) {
 ; CHECK-LABEL: @test11vec(
-; CHECK-NEXT:    [[AND:%.*]] = and <2 x i8> %X, <i8 127, i8 127>
+; CHECK-NEXT:    [[AND:%.*]] = and <2 x i8> [[X:%.*]], <i8 127, i8 127>
 ; CHECK-NEXT:    ret <2 x i8> [[AND]]
 ;
   %cmp = icmp sgt <2 x i8> %X, <i8 -1, i8 -1>
@@ -246,7 +311,7 @@ define i32 @test13noncanon(i32 %X) {
 
 define i32 @select_icmp_and_8_eq_0_or_8(i32 %x) {
 ; CHECK-LABEL: @select_icmp_and_8_eq_0_or_8(
-; CHECK-NEXT:    [[OR:%.*]] = or i32 %x, 8
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[X:%.*]], 8
 ; CHECK-NEXT:    ret i32 [[OR]]
 ;
   %and = and i32 %x, 8
@@ -258,7 +323,7 @@ define i32 @select_icmp_and_8_eq_0_or_8(i32 %x) {
 
 define i32 @select_icmp_and_8_eq_0_or_8_alt(i32 %x) {
 ; CHECK-LABEL: @select_icmp_and_8_eq_0_or_8_alt(
-; CHECK-NEXT:    [[OR:%.*]] = or i32 %x, 8
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[X:%.*]], 8
 ; CHECK-NEXT:    ret i32 [[OR]]
 ;
   %and = and i32 %x, 8
@@ -270,7 +335,7 @@ define i32 @select_icmp_and_8_eq_0_or_8_alt(i32 %x) {
 
 define i32 @select_icmp_and_8_ne_0_or_8(i32 %x) {
 ; CHECK-LABEL: @select_icmp_and_8_ne_0_or_8(
-; CHECK-NEXT:    ret i32 %x
+; CHECK-NEXT:    ret i32 [[X:%.*]]
 ;
   %and = and i32 %x, 8
   %cmp = icmp ne i32 %and, 0
@@ -281,7 +346,7 @@ define i32 @select_icmp_and_8_ne_0_or_8(i32 %x) {
 
 define i32 @select_icmp_and_8_ne_0_or_8_alt(i32 %x) {
 ; CHECK-LABEL: @select_icmp_and_8_ne_0_or_8_alt(
-; CHECK-NEXT:    ret i32 %x
+; CHECK-NEXT:    ret i32 [[X:%.*]]
 ;
   %and = and i32 %x, 8
   %cmp = icmp eq i32 %and, 0
@@ -292,7 +357,7 @@ define i32 @select_icmp_and_8_ne_0_or_8_alt(i32 %x) {
 
 define i32 @select_icmp_and_8_eq_0_and_not_8(i32 %x) {
 ; CHECK-LABEL: @select_icmp_and_8_eq_0_and_not_8(
-; CHECK-NEXT:    [[AND1:%.*]] = and i32 %x, -9
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[X:%.*]], -9
 ; CHECK-NEXT:    ret i32 [[AND1]]
 ;
   %and = and i32 %x, 8
@@ -304,7 +369,7 @@ define i32 @select_icmp_and_8_eq_0_and_not_8(i32 %x) {
 
 define i32 @select_icmp_and_8_eq_0_and_not_8_alt(i32 %x) {
 ; CHECK-LABEL: @select_icmp_and_8_eq_0_and_not_8_alt(
-; CHECK-NEXT:    [[AND1:%.*]] = and i32 %x, -9
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[X:%.*]], -9
 ; CHECK-NEXT:    ret i32 [[AND1]]
 ;
   %and = and i32 %x, 8
@@ -316,7 +381,7 @@ define i32 @select_icmp_and_8_eq_0_and_not_8_alt(i32 %x) {
 
 define i32 @select_icmp_and_8_ne_0_and_not_8(i32 %x) {
 ; CHECK-LABEL: @select_icmp_and_8_ne_0_and_not_8(
-; CHECK-NEXT:    ret i32 %x
+; CHECK-NEXT:    ret i32 [[X:%.*]]
 ;
   %and = and i32 %x, 8
   %cmp = icmp ne i32 %and, 0
@@ -327,7 +392,7 @@ define i32 @select_icmp_and_8_ne_0_and_not_8(i32 %x) {
 
 define i32 @select_icmp_and_8_ne_0_and_not_8_alt(i32 %x) {
 ; CHECK-LABEL: @select_icmp_and_8_ne_0_and_not_8_alt(
-; CHECK-NEXT:    ret i32 %x
+; CHECK-NEXT:    ret i32 [[X:%.*]]
 ;
   %and = and i32 %x, 8
   %cmp = icmp eq i32 %and, 0
@@ -342,7 +407,7 @@ define i32 @select_icmp_and_8_ne_0_and_not_8_alt(i32 %x) {
 
 define i32 @select_icmp_trunc_8_ne_0_or_128(i32 %x) {
 ; CHECK-LABEL: @select_icmp_trunc_8_ne_0_or_128(
-; CHECK-NEXT:    [[OR:%.*]] = or i32 %x, 128
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[X:%.*]], 128
 ; CHECK-NEXT:    ret i32 [[OR]]
 ;
   %trunc = trunc i32 %x to i8
@@ -354,7 +419,7 @@ define i32 @select_icmp_trunc_8_ne_0_or_128(i32 %x) {
 
 define i32 @select_icmp_trunc_8_ne_0_or_128_alt(i32 %x) {
 ; CHECK-LABEL: @select_icmp_trunc_8_ne_0_or_128_alt(
-; CHECK-NEXT:    [[OR:%.*]] = or i32 %x, 128
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[X:%.*]], 128
 ; CHECK-NEXT:    ret i32 [[OR]]
 ;
   %trunc = trunc i32 %x to i8
@@ -366,7 +431,7 @@ define i32 @select_icmp_trunc_8_ne_0_or_128_alt(i32 %x) {
 
 define i32 @select_icmp_trunc_8_eq_0_or_128(i32 %x) {
 ; CHECK-LABEL: @select_icmp_trunc_8_eq_0_or_128(
-; CHECK-NEXT:    ret i32 %x
+; CHECK-NEXT:    ret i32 [[X:%.*]]
 ;
   %trunc = trunc i32 %x to i8
   %cmp = icmp slt i8 %trunc, 0
@@ -377,7 +442,7 @@ define i32 @select_icmp_trunc_8_eq_0_or_128(i32 %x) {
 
 define i32 @select_icmp_trunc_8_eq_0_or_128_alt(i32 %x) {
 ; CHECK-LABEL: @select_icmp_trunc_8_eq_0_or_128_alt(
-; CHECK-NEXT:    ret i32 %x
+; CHECK-NEXT:    ret i32 [[X:%.*]]
 ;
   %trunc = trunc i32 %x to i8
   %cmp = icmp sgt i8 %trunc, -1
@@ -388,7 +453,7 @@ define i32 @select_icmp_trunc_8_eq_0_or_128_alt(i32 %x) {
 
 define i32 @select_icmp_trunc_8_eq_0_and_not_8(i32 %x) {
 ; CHECK-LABEL: @select_icmp_trunc_8_eq_0_and_not_8(
-; CHECK-NEXT:    [[AND:%.*]] = and i32 %x, -9
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], -9
 ; CHECK-NEXT:    ret i32 [[AND]]
 ;
   %trunc = trunc i32 %x to i4
@@ -400,7 +465,7 @@ define i32 @select_icmp_trunc_8_eq_0_and_not_8(i32 %x) {
 
 define i32 @select_icmp_trunc_8_eq_0_and_not_8_alt(i32 %x) {
 ; CHECK-LABEL: @select_icmp_trunc_8_eq_0_and_not_8_alt(
-; CHECK-NEXT:    [[AND:%.*]] = and i32 %x, -9
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], -9
 ; CHECK-NEXT:    ret i32 [[AND]]
 ;
   %trunc = trunc i32 %x to i4
@@ -412,7 +477,7 @@ define i32 @select_icmp_trunc_8_eq_0_and_not_8_alt(i32 %x) {
 
 define i32 @select_icmp_trunc_8_ne_0_and_not_8(i32 %x) {
 ; CHECK-LABEL: @select_icmp_trunc_8_ne_0_and_not_8(
-; CHECK-NEXT:    ret i32 %x
+; CHECK-NEXT:    ret i32 [[X:%.*]]
 ;
   %trunc = trunc i32 %x to i4
   %cmp = icmp slt i4 %trunc, 0
@@ -423,7 +488,7 @@ define i32 @select_icmp_trunc_8_ne_0_and_not_8(i32 %x) {
 
 define i32 @select_icmp_trunc_8_ne_0_and_not_8_alt(i32 %x) {
 ; CHECK-LABEL: @select_icmp_trunc_8_ne_0_and_not_8_alt(
-; CHECK-NEXT:    ret i32 %x
+; CHECK-NEXT:    ret i32 [[X:%.*]]
 ;
   %trunc = trunc i32 %x to i4
   %cmp = icmp sgt i4 %trunc, -1
@@ -436,7 +501,7 @@ define i32 @select_icmp_trunc_8_ne_0_and_not_8_alt(i32 %x) {
 
 define <2 x i32> @select_icmp_and_8_ne_0_and_not_8_vec(<2 x i32> %x) {
 ; CHECK-LABEL: @select_icmp_and_8_ne_0_and_not_8_vec(
-; CHECK-NEXT:    ret <2 x i32> %x
+; CHECK-NEXT:    ret <2 x i32> [[X:%.*]]
 ;
   %and = and <2 x i32> %x, <i32 8, i32 8>
   %cmp = icmp ne <2 x i32> %and, zeroinitializer
@@ -447,7 +512,7 @@ define <2 x i32> @select_icmp_and_8_ne_0_and_not_8_vec(<2 x i32> %x) {
 
 define <2 x i32> @select_icmp_trunc_8_ne_0_and_not_8_alt_vec(<2 x i32> %x) {
 ; CHECK-LABEL: @select_icmp_trunc_8_ne_0_and_not_8_alt_vec(
-; CHECK-NEXT:    ret <2 x i32> %x
+; CHECK-NEXT:    ret <2 x i32> [[X:%.*]]
 ;
   %trunc = trunc <2 x i32> %x to <2 x i4>
   %cmp = icmp sgt <2 x i4> %trunc, <i4 -1, i4 -1>
@@ -460,10 +525,10 @@ define <2 x i32> @select_icmp_trunc_8_ne_0_and_not_8_alt_vec(<2 x i32> %x) {
 
 define i32 @select_icmp_x_and_8_eq_0_y_and_not_8(i32 %x, i32 %y) {
 ; CHECK-LABEL: @select_icmp_x_and_8_eq_0_y_and_not_8(
-; CHECK-NEXT:    [[AND:%.*]] = and i32 %x, 8
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 8
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
-; CHECK-NEXT:    [[AND1:%.*]] = and i32 %y, -9
-; CHECK-NEXT:    [[Y_AND1:%.*]] = select i1 [[CMP]], i32 %y, i32 [[AND1]]
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[Y:%.*]], -9
+; CHECK-NEXT:    [[Y_AND1:%.*]] = select i1 [[CMP]], i32 [[Y]], i32 [[AND1]]
 ; CHECK-NEXT:    ret i32 [[Y_AND1]]
 ;
   %and = and i32 %x, 8
@@ -475,10 +540,10 @@ define i32 @select_icmp_x_and_8_eq_0_y_and_not_8(i32 %x, i32 %y) {
 
 define i64 @select_icmp_x_and_8_eq_0_y64_and_not_8(i32 %x, i64 %y) {
 ; CHECK-LABEL: @select_icmp_x_and_8_eq_0_y64_and_not_8(
-; CHECK-NEXT:    [[AND:%.*]] = and i32 %x, 8
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 8
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
-; CHECK-NEXT:    [[AND1:%.*]] = and i64 %y, -9
-; CHECK-NEXT:    [[Y_AND1:%.*]] = select i1 [[CMP]], i64 %y, i64 [[AND1]]
+; CHECK-NEXT:    [[AND1:%.*]] = and i64 [[Y:%.*]], -9
+; CHECK-NEXT:    [[Y_AND1:%.*]] = select i1 [[CMP]], i64 [[Y]], i64 [[AND1]]
 ; CHECK-NEXT:    ret i64 [[Y_AND1]]
 ;
   %and = and i32 %x, 8
@@ -490,10 +555,10 @@ define i64 @select_icmp_x_and_8_eq_0_y64_and_not_8(i32 %x, i64 %y) {
 
 define i64 @select_icmp_x_and_8_ne_0_y64_and_not_8(i32 %x, i64 %y) {
 ; CHECK-LABEL: @select_icmp_x_and_8_ne_0_y64_and_not_8(
-; CHECK-NEXT:    [[AND:%.*]] = and i32 %x, 8
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 8
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
-; CHECK-NEXT:    [[AND1:%.*]] = and i64 %y, -9
-; CHECK-NEXT:    [[AND1_Y:%.*]] = select i1 [[CMP]], i64 [[AND1]], i64 %y
+; CHECK-NEXT:    [[AND1:%.*]] = and i64 [[Y:%.*]], -9
+; CHECK-NEXT:    [[AND1_Y:%.*]] = select i1 [[CMP]], i64 [[AND1]], i64 [[Y]]
 ; CHECK-NEXT:    ret i64 [[AND1_Y]]
 ;
   %and = and i32 %x, 8
@@ -507,8 +572,8 @@ define i64 @select_icmp_x_and_8_ne_0_y64_and_not_8(i32 %x, i64 %y) {
 
 define i32* @select_icmp_pointers(i32* %x, i32* %y) {
 ; CHECK-LABEL: @select_icmp_pointers(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32* %x, null
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32* %x, i32* %y
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32* [[X:%.*]], null
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32* [[X]], i32* [[Y:%.*]]
 ; CHECK-NEXT:    ret i32* [[SEL]]
 ;
   %cmp = icmp slt i32* %x, null
@@ -523,8 +588,8 @@ declare void @llvm.assume(i1)
 
 define i8 @assume_sel_cond(i1 %cond, i8 %x, i8 %y) {
 ; CHECK-LABEL: @assume_sel_cond(
-; CHECK-NEXT:    call void @llvm.assume(i1 %cond)
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 %cond, i8 %x, i8 %y
+; CHECK-NEXT:    call void @llvm.assume(i1 [[COND:%.*]])
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[COND]], i8 [[X:%.*]], i8 [[Y:%.*]]
 ; CHECK-NEXT:    ret i8 [[SEL]]
 ;
   call void @llvm.assume(i1 %cond)
@@ -534,9 +599,9 @@ define i8 @assume_sel_cond(i1 %cond, i8 %x, i8 %y) {
 
 define i8 @do_not_assume_sel_cond(i1 %cond, i8 %x, i8 %y) {
 ; CHECK-LABEL: @do_not_assume_sel_cond(
-; CHECK-NEXT:    [[NOTCOND:%.*]] = icmp eq i1 %cond, false
+; CHECK-NEXT:    [[NOTCOND:%.*]] = icmp eq i1 [[COND:%.*]], false
 ; CHECK-NEXT:    call void @llvm.assume(i1 [[NOTCOND]])
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 %cond, i8 %x, i8 %y
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[COND]], i8 [[X:%.*]], i8 [[Y:%.*]]
 ; CHECK-NEXT:    ret i8 [[SEL]]
 ;
   %notcond = icmp eq i1 %cond, false
@@ -547,8 +612,9 @@ define i8 @do_not_assume_sel_cond(i1 %cond, i8 %x, i8 %y) {
 
 define i32* @select_icmp_eq_0_gep_operand(i32* %base, i64 %n) {
 ; CHECK-LABEL: @select_icmp_eq_0_gep_operand(
-; CHECK-NEXT: [[GEP:%.*]] = getelementptr
-; CHECK-NEXT: ret i32* [[GEP]]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i32, i32* [[BASE:%.*]], i64 [[N:%.*]]
+; CHECK-NEXT:    ret i32* [[GEP]]
+;
   %cond = icmp eq i64 %n, 0
   %gep = getelementptr i32, i32* %base, i64 %n
   %r = select i1 %cond, i32* %base, i32* %gep
@@ -557,8 +623,9 @@ define i32* @select_icmp_eq_0_gep_operand(i32* %base, i64 %n) {
 
 define i32* @select_icmp_ne_0_gep_operand(i32* %base, i64 %n) {
 ; CHECK-LABEL: @select_icmp_ne_0_gep_operand(
-; CHECK-NEXT: [[GEP:%.*]] = getelementptr
-; CHECK-NEXT: ret i32* [[GEP]]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i32, i32* [[BASE:%.*]], i64 [[N:%.*]]
+; CHECK-NEXT:    ret i32* [[GEP]]
+;
   %cond = icmp ne i64 %n, 0
   %gep = getelementptr i32, i32* %base, i64 %n
   %r = select i1 %cond, i32* %gep, i32* %base
diff --git a/llvm/test/Transforms/InstSimplify/shufflevector.ll b/llvm/test/Transforms/InstSimplify/shufflevector.ll
index cc49ae355..14c4205f4 100644
--- a/llvm/test/Transforms/InstSimplify/shufflevector.ll
+++ b/llvm/test/Transforms/InstSimplify/shufflevector.ll
@@ -247,3 +247,30 @@ define <2 x float> @PR32872(<2 x float> %x) {
   %tmp4 = shufflevector <4 x float> zeroinitializer, <4 x float> %tmp1, <2 x i32> <i32 4, i32 5>
   ret <2 x float> %tmp4
 }
+
+define <5 x i8> @splat_inserted_constant(<4 x i8> %x) {
+; CHECK-LABEL: @splat_inserted_constant(
+; CHECK-NEXT:    ret <5 x i8> <i8 42, i8 42, i8 42, i8 42, i8 42>
+;
+  %ins3 = insertelement <4 x i8> %x, i8 42, i64 3
+  %splat5 = shufflevector <4 x i8> %ins3, <4 x i8> undef, <5 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3>
+  ret <5 x i8> %splat5
+}
+
+define <4 x float> @splat_inserted_constant_undef_elt(<4 x float> %x) {
+; CHECK-LABEL: @splat_inserted_constant_undef_elt(
+; CHECK-NEXT:    ret <4 x float> <float 1.200000e+01, float 1.200000e+01, float undef, float 1.200000e+01>
+;
+  %ins1 = insertelement <4 x float> %x, float 12.0, i32 1
+  %splat1 = shufflevector <4 x float> %ins1, <4 x float> undef, <4 x i32> <i32 1, i32 1, i32 undef, i32 1>
+  ret <4 x float> %splat1
+}
+
+define <2 x i8> @splat_inserted_constant_not_canonical(<3 x i8> %x, <3 x i8> %y) {
+; CHECK-LABEL: @splat_inserted_constant_not_canonical(
+; CHECK-NEXT:    ret <2 x i8> <i8 undef, i8 23>
+;
+  %ins2 = insertelement <3 x i8> %x, i8 23, i7 2
+  %splat2 = shufflevector <3 x i8> %y, <3 x i8> %ins2, <2 x i32> <i32 undef, i32 5>
+  ret <2 x i8> %splat2
+}
diff --git a/llvm/test/Transforms/InstSimplify/unsigned-range-checks.ll b/llvm/test/Transforms/InstSimplify/unsigned-range-checks.ll
new file mode 100644
index 000000000..e0c3a7c8d
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/unsigned-range-checks.ll
@@ -0,0 +1,130 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instsimplify -S | FileCheck %s
+
+; Here we add unsigned two values, check that addition did not underflow AND
+; that the result is non-zero. This can be simplified just to a comparison
+; between the base and negated offset.
+
+; If we are checking that the result is not null or no underflow happened,
+; it is tautological (always-true).
+define i1 @t1(i8 %x, i8 %y) {
+; CHECK-LABEL: @t1(
+; CHECK-NEXT:    ret i1 true
+;
+  %not_null = icmp ne i8 %y, 0
+  %no_underflow = icmp ule i8 %y, %x
+  %r = or i1 %not_null, %no_underflow
+  ret i1 %r
+}
+define i1 @t2_commutative(i8 %x, i8 %y) {
+; CHECK-LABEL: @t2_commutative(
+; CHECK-NEXT:    ret i1 true
+;
+  %not_null = icmp ne i8 %y, 0
+  %no_underflow = icmp uge i8 %x, %y ; swapped
+  %r = or i1 %not_null, %no_underflow
+  ret i1 %r
+}
+
+define i1 @t3_commutative(i8 %x, i8 %y) {
+; CHECK-LABEL: @t3_commutative(
+; CHECK-NEXT:    ret i1 true
+;
+  %not_null = icmp ne i8 %y, 0
+  %no_underflow = icmp ule i8 %y, %x
+  %r = or i1 %no_underflow, %not_null ; swapped
+  ret i1 %r
+}
+define i1 @t4_commutative(i8 %x, i8 %y) {
+; CHECK-LABEL: @t4_commutative(
+; CHECK-NEXT:    ret i1 true
+;
+  %not_null = icmp ne i8 %y, 0
+  %no_underflow = icmp uge i8 %x, %y ; swapped
+  %r = or i1 %no_underflow, %not_null ; swapped
+  ret i1 %r
+}
+
+; If we are checking that the result is null and underflow happened,
+; it is tautological (always-false).
+define i1 @t5(i8 %x, i8 %y) {
+; CHECK-LABEL: @t5(
+; CHECK-NEXT:    ret i1 false
+;
+  %not_null = icmp eq i8 %y, 0
+  %no_underflow = icmp ugt i8 %y, %x
+  %r = and i1 %not_null, %no_underflow
+  ret i1 %r
+}
+define i1 @t6_commutative(i8 %x, i8 %y) {
+; CHECK-LABEL: @t6_commutative(
+; CHECK-NEXT:    ret i1 false
+;
+  %not_null = icmp eq i8 %y, 0
+  %no_underflow = icmp ult i8 %x, %y ; swapped
+  %r = and i1 %not_null, %no_underflow
+  ret i1 %r
+}
+
+; We only need to know that any of the 'add' operands is non-zero,
+; not necessarily the one used in the comparison.
+define i1 @t7(i8 %x, i8 %y) {
+; CHECK-LABEL: @t7(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp = icmp slt i8 %y, 0
+  %not_null = icmp ne i8 %y, 0
+  %no_underflow = icmp ule i8 %y, %x
+  %r = or i1 %not_null, %no_underflow
+  ret i1 %r
+}
+
+; If we check that no underflow happened and that the result is null,
+; we can just check for null.
+define i1 @t8(i8 %x, i8 %y) {
+; CHECK-LABEL: @t8(
+; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp eq i8 [[Y:%.*]], 0
+; CHECK-NEXT:    ret i1 [[NOT_NULL]]
+;
+  %not_null = icmp eq i8 %y, 0
+  %no_underflow = icmp ule i8 %y, %x
+  %r = and i1 %not_null, %no_underflow
+  ret i1 %r
+}
+; Likewise, if we check that result is non-null or underflow happened,
+; we can just check for null.
+define i1 @t9(i8 %x, i8 %y) {
+; CHECK-LABEL: @t9(
+; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp ne i8 [[Y:%.*]], 0
+; CHECK-NEXT:    ret i1 [[NOT_NULL]]
+;
+  %not_null = icmp ne i8 %y, 0
+  %no_underflow = icmp ugt i8 %y, %x
+  %r = or i1 %not_null, %no_underflow
+  ret i1 %r
+}
+
+; If we check that no underflow happened or that the result is not null,
+; we can just check for lack of underflow.
+define i1 @t10(i8 %x, i8 %y) {
+; CHECK-LABEL: @t10(
+; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp ule i8 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[NO_UNDERFLOW]]
+;
+  %not_null = icmp eq i8 %y, 0
+  %no_underflow = icmp ule i8 %y, %x
+  %r = or i1 %not_null, %no_underflow
+  ret i1 %r
+}
+; Likewise, if we check that underflow happened and that the result is not null,
+; we can just check for lack of underflow.
+define i1 @t11(i8 %x, i8 %y) {
+; CHECK-LABEL: @t11(
+; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp ugt i8 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[NO_UNDERFLOW]]
+;
+  %not_null = icmp ne i8 %y, 0
+  %no_underflow = icmp ugt i8 %y, %x
+  %r = and i1 %not_null, %no_underflow
+  ret i1 %r
+}
diff --git a/llvm/test/Transforms/InterleavedAccess/ARM/interleaved-accesses.ll b/llvm/test/Transforms/InterleavedAccess/ARM/interleaved-accesses.ll
index fdf6e1fed..2491b72e7 100644
--- a/llvm/test/Transforms/InterleavedAccess/ARM/interleaved-accesses.ll
+++ b/llvm/test/Transforms/InterleavedAccess/ARM/interleaved-accesses.ll
@@ -1,19 +1,30 @@
-; RUN: opt < %s -mattr=+neon -interleaved-access -S | FileCheck %s -check-prefixes=NEON,ALL
-; RUN: opt < %s -interleaved-access -S | FileCheck %s -check-prefixes=NO_NEON,ALL
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mattr=+neon -interleaved-access -S | FileCheck %s --check-prefix=CHECK-NEON
+; RUN: opt < %s -mattr=+mve.fp -interleaved-access -S | FileCheck %s --check-prefix=CHECK-MVE
+; RUN: opt < %s -interleaved-access -S | FileCheck %s --check-prefix=CHECK-NONE
 
 target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-n32-S64"
 target triple = "arm---eabi"
 
 define void @load_factor2(<16 x i8>* %ptr) {
-; NEON-LABEL:    @load_factor2(
-; NEON-NEXT:       [[TMP1:%.*]] = bitcast <16 x i8>* %ptr to i8*
-; NEON-NEXT:       [[VLDN:%.*]] = call { <8 x i8>, <8 x i8> } @llvm.arm.neon.vld2.v8i8.p0i8(i8* [[TMP1]], i32 4)
-; NEON-NEXT:       [[TMP2:%.*]] = extractvalue { <8 x i8>, <8 x i8> } [[VLDN]], 1
-; NEON-NEXT:       [[TMP3:%.*]] = extractvalue { <8 x i8>, <8 x i8> } [[VLDN]], 0
-; NEON-NEXT:       ret void
-; NO_NEON-LABEL: @load_factor2(
-; NO_NEON-NOT:     @llvm.arm.neon
-; NO_NEON:         ret void
+; CHECK-NEON-LABEL: @load_factor2(
+; CHECK-NEON-NEXT:    [[TMP1:%.*]] = bitcast <16 x i8>* [[PTR:%.*]] to i8*
+; CHECK-NEON-NEXT:    [[VLDN:%.*]] = call { <8 x i8>, <8 x i8> } @llvm.arm.neon.vld2.v8i8.p0i8(i8* [[TMP1]], i32 4)
+; CHECK-NEON-NEXT:    [[TMP2:%.*]] = extractvalue { <8 x i8>, <8 x i8> } [[VLDN]], 1
+; CHECK-NEON-NEXT:    [[TMP3:%.*]] = extractvalue { <8 x i8>, <8 x i8> } [[VLDN]], 0
+; CHECK-NEON-NEXT:    ret void
+;
+; CHECK-MVE-LABEL: @load_factor2(
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = load <16 x i8>, <16 x i8>* [[PTR:%.*]], align 4
+; CHECK-MVE-NEXT:    [[V0:%.*]] = shufflevector <16 x i8> [[INTERLEAVED_VEC]], <16 x i8> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+; CHECK-MVE-NEXT:    [[V1:%.*]] = shufflevector <16 x i8> [[INTERLEAVED_VEC]], <16 x i8> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+; CHECK-MVE-NEXT:    ret void
+;
+; CHECK-NONE-LABEL: @load_factor2(
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = load <16 x i8>, <16 x i8>* [[PTR:%.*]], align 4
+; CHECK-NONE-NEXT:    [[V0:%.*]] = shufflevector <16 x i8> [[INTERLEAVED_VEC]], <16 x i8> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+; CHECK-NONE-NEXT:    [[V1:%.*]] = shufflevector <16 x i8> [[INTERLEAVED_VEC]], <16 x i8> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+; CHECK-NONE-NEXT:    ret void
 ;
   %interleaved.vec = load <16 x i8>, <16 x i8>* %ptr, align 4
   %v0 = shufflevector <16 x i8> %interleaved.vec, <16 x i8> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
@@ -22,16 +33,27 @@ define void @load_factor2(<16 x i8>* %ptr) {
 }
 
 define void @load_factor3(<6 x i32>* %ptr) {
-; NEON-LABEL:    @load_factor3(
-; NEON-NEXT:       [[TMP1:%.*]] = bitcast <6 x i32>* %ptr to i8*
-; NEON-NEXT:       [[VLDN:%.*]] = call { <2 x i32>, <2 x i32>, <2 x i32> } @llvm.arm.neon.vld3.v2i32.p0i8(i8* [[TMP1]], i32 4)
-; NEON-NEXT:       [[TMP2:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[VLDN]], 2
-; NEON-NEXT:       [[TMP3:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[VLDN]], 1
-; NEON-NEXT:       [[TMP4:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[VLDN]], 0
-; NEON-NEXT:       ret void
-; NO_NEON-LABEL: @load_factor3(
-; NO_NEON-NOT:     @llvm.arm.neon
-; NO_NEON:         ret void
+; CHECK-NEON-LABEL: @load_factor3(
+; CHECK-NEON-NEXT:    [[TMP1:%.*]] = bitcast <6 x i32>* [[PTR:%.*]] to i8*
+; CHECK-NEON-NEXT:    [[VLDN:%.*]] = call { <2 x i32>, <2 x i32>, <2 x i32> } @llvm.arm.neon.vld3.v2i32.p0i8(i8* [[TMP1]], i32 4)
+; CHECK-NEON-NEXT:    [[TMP2:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[VLDN]], 2
+; CHECK-NEON-NEXT:    [[TMP3:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[VLDN]], 1
+; CHECK-NEON-NEXT:    [[TMP4:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[VLDN]], 0
+; CHECK-NEON-NEXT:    ret void
+;
+; CHECK-MVE-LABEL: @load_factor3(
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = load <6 x i32>, <6 x i32>* [[PTR:%.*]], align 4
+; CHECK-MVE-NEXT:    [[V0:%.*]] = shufflevector <6 x i32> [[INTERLEAVED_VEC]], <6 x i32> undef, <2 x i32> <i32 0, i32 3>
+; CHECK-MVE-NEXT:    [[V1:%.*]] = shufflevector <6 x i32> [[INTERLEAVED_VEC]], <6 x i32> undef, <2 x i32> <i32 1, i32 4>
+; CHECK-MVE-NEXT:    [[V2:%.*]] = shufflevector <6 x i32> [[INTERLEAVED_VEC]], <6 x i32> undef, <2 x i32> <i32 2, i32 5>
+; CHECK-MVE-NEXT:    ret void
+;
+; CHECK-NONE-LABEL: @load_factor3(
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = load <6 x i32>, <6 x i32>* [[PTR:%.*]], align 4
+; CHECK-NONE-NEXT:    [[V0:%.*]] = shufflevector <6 x i32> [[INTERLEAVED_VEC]], <6 x i32> undef, <2 x i32> <i32 0, i32 3>
+; CHECK-NONE-NEXT:    [[V1:%.*]] = shufflevector <6 x i32> [[INTERLEAVED_VEC]], <6 x i32> undef, <2 x i32> <i32 1, i32 4>
+; CHECK-NONE-NEXT:    [[V2:%.*]] = shufflevector <6 x i32> [[INTERLEAVED_VEC]], <6 x i32> undef, <2 x i32> <i32 2, i32 5>
+; CHECK-NONE-NEXT:    ret void
 ;
   %interleaved.vec = load <6 x i32>, <6 x i32>* %ptr, align 4
   %v0 = shufflevector <6 x i32> %interleaved.vec, <6 x i32> undef, <2 x i32> <i32 0, i32 3>
@@ -41,17 +63,30 @@ define void @load_factor3(<6 x i32>* %ptr) {
 }
 
 define void @load_factor4(<16 x i32>* %ptr) {
-; NEON-LABEL:    @load_factor4(
-; NEON-NEXT:       [[TMP1:%.*]] = bitcast <16 x i32>* %ptr to i8*
-; NEON-NEXT:       [[VLDN:%.*]] = call { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } @llvm.arm.neon.vld4.v4i32.p0i8(i8* [[TMP1]], i32 4)
-; NEON-NEXT:       [[TMP2:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN]], 3
-; NEON-NEXT:       [[TMP3:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN]], 2
-; NEON-NEXT:       [[TMP4:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN]], 1
-; NEON-NEXT:       [[TMP5:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN]], 0
-; NEON-NEXT:       ret void
-; NO_NEON-LABEL: @load_factor4(
-; NO_NEON-NOT:     @llvm.arm.neon
-; NO_NEON:         ret void
+; CHECK-NEON-LABEL: @load_factor4(
+; CHECK-NEON-NEXT:    [[TMP1:%.*]] = bitcast <16 x i32>* [[PTR:%.*]] to i8*
+; CHECK-NEON-NEXT:    [[VLDN:%.*]] = call { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } @llvm.arm.neon.vld4.v4i32.p0i8(i8* [[TMP1]], i32 4)
+; CHECK-NEON-NEXT:    [[TMP2:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN]], 3
+; CHECK-NEON-NEXT:    [[TMP3:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN]], 2
+; CHECK-NEON-NEXT:    [[TMP4:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN]], 1
+; CHECK-NEON-NEXT:    [[TMP5:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN]], 0
+; CHECK-NEON-NEXT:    ret void
+;
+; CHECK-MVE-LABEL: @load_factor4(
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = load <16 x i32>, <16 x i32>* [[PTR:%.*]], align 4
+; CHECK-MVE-NEXT:    [[V0:%.*]] = shufflevector <16 x i32> [[INTERLEAVED_VEC]], <16 x i32> undef, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+; CHECK-MVE-NEXT:    [[V1:%.*]] = shufflevector <16 x i32> [[INTERLEAVED_VEC]], <16 x i32> undef, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
+; CHECK-MVE-NEXT:    [[V2:%.*]] = shufflevector <16 x i32> [[INTERLEAVED_VEC]], <16 x i32> undef, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
+; CHECK-MVE-NEXT:    [[V3:%.*]] = shufflevector <16 x i32> [[INTERLEAVED_VEC]], <16 x i32> undef, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
+; CHECK-MVE-NEXT:    ret void
+;
+; CHECK-NONE-LABEL: @load_factor4(
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = load <16 x i32>, <16 x i32>* [[PTR:%.*]], align 4
+; CHECK-NONE-NEXT:    [[V0:%.*]] = shufflevector <16 x i32> [[INTERLEAVED_VEC]], <16 x i32> undef, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+; CHECK-NONE-NEXT:    [[V1:%.*]] = shufflevector <16 x i32> [[INTERLEAVED_VEC]], <16 x i32> undef, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
+; CHECK-NONE-NEXT:    [[V2:%.*]] = shufflevector <16 x i32> [[INTERLEAVED_VEC]], <16 x i32> undef, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
+; CHECK-NONE-NEXT:    [[V3:%.*]] = shufflevector <16 x i32> [[INTERLEAVED_VEC]], <16 x i32> undef, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
+; CHECK-NONE-NEXT:    ret void
 ;
   %interleaved.vec = load <16 x i32>, <16 x i32>* %ptr, align 4
   %v0 = shufflevector <16 x i32> %interleaved.vec, <16 x i32> undef, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
@@ -62,15 +97,22 @@ define void @load_factor4(<16 x i32>* %ptr) {
 }
 
 define void @store_factor2(<16 x i8>* %ptr, <8 x i8> %v0, <8 x i8> %v1) {
-; NEON-LABEL:    @store_factor2(
-; NEON-NEXT:       [[TMP1:%.*]] = bitcast <16 x i8>* %ptr to i8*
-; NEON-NEXT:       [[TMP2:%.*]] = shufflevector <8 x i8> %v0, <8 x i8> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; NEON-NEXT:       [[TMP3:%.*]] = shufflevector <8 x i8> %v0, <8 x i8> %v1, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; NEON-NEXT:       call void @llvm.arm.neon.vst2.p0i8.v8i8(i8* [[TMP1]], <8 x i8> [[TMP2]], <8 x i8> [[TMP3]], i32 4)
-; NEON-NEXT:       ret void
-; NO_NEON-LABEL: @store_factor2(
-; NO_NEON-NOT:     @llvm.arm.neon
-; NO_NEON:         ret void
+; CHECK-NEON-LABEL: @store_factor2(
+; CHECK-NEON-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i8> [[V0:%.*]], <8 x i8> [[V1:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEON-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i8> [[V0]], <8 x i8> [[V1]], <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEON-NEXT:    [[TMP3:%.*]] = bitcast <16 x i8>* [[PTR:%.*]] to i8*
+; CHECK-NEON-NEXT:    call void @llvm.arm.neon.vst2.p0i8.v8i8(i8* [[TMP3]], <8 x i8> [[TMP1]], <8 x i8> [[TMP2]], i32 4)
+; CHECK-NEON-NEXT:    ret void
+;
+; CHECK-MVE-LABEL: @store_factor2(
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i8> [[V0:%.*]], <8 x i8> [[V1:%.*]], <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
+; CHECK-MVE-NEXT:    store <16 x i8> [[INTERLEAVED_VEC]], <16 x i8>* [[PTR:%.*]], align 4
+; CHECK-MVE-NEXT:    ret void
+;
+; CHECK-NONE-LABEL: @store_factor2(
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i8> [[V0:%.*]], <8 x i8> [[V1:%.*]], <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
+; CHECK-NONE-NEXT:    store <16 x i8> [[INTERLEAVED_VEC]], <16 x i8>* [[PTR:%.*]], align 4
+; CHECK-NONE-NEXT:    ret void
 ;
   %interleaved.vec = shufflevector <8 x i8> %v0, <8 x i8> %v1, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
   store <16 x i8> %interleaved.vec, <16 x i8>* %ptr, align 4
@@ -78,16 +120,29 @@ define void @store_factor2(<16 x i8>* %ptr, <8 x i8> %v0, <8 x i8> %v1) {
 }
 
 define void @store_factor3(<12 x i32>* %ptr, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2) {
-; NEON-LABEL:    @store_factor3(
-; NEON:            [[TMP1:%.*]] = bitcast <12 x i32>* %ptr to i8*
-; NEON-NEXT:       [[TMP2:%.*]] = shufflevector <8 x i32> %s0, <8 x i32> %s1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; NEON-NEXT:       [[TMP3:%.*]] = shufflevector <8 x i32> %s0, <8 x i32> %s1, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; NEON-NEXT:       [[TMP4:%.*]] = shufflevector <8 x i32> %s0, <8 x i32> %s1, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
-; NEON-NEXT:       call void @llvm.arm.neon.vst3.p0i8.v4i32(i8* [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], i32 4)
-; NEON-NEXT:       ret void
-; NO_NEON-LABEL: @store_factor3(
-; NO_NEON-NOT:     @llvm.arm.neon
-; NO_NEON:         ret void
+; CHECK-NEON-LABEL: @store_factor3(
+; CHECK-NEON-NEXT:    [[S0:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEON-NEXT:    [[S1:%.*]] = shufflevector <4 x i32> [[V2:%.*]], <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEON-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[S0]], <8 x i32> [[S1]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEON-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[S0]], <8 x i32> [[S1]], <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEON-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[S0]], <8 x i32> [[S1]], <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEON-NEXT:    [[TMP4:%.*]] = bitcast <12 x i32>* [[PTR:%.*]] to i8*
+; CHECK-NEON-NEXT:    call void @llvm.arm.neon.vst3.p0i8.v4i32(i8* [[TMP4]], <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> [[TMP3]], i32 4)
+; CHECK-NEON-NEXT:    ret void
+;
+; CHECK-MVE-LABEL: @store_factor3(
+; CHECK-MVE-NEXT:    [[S0:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-MVE-NEXT:    [[S1:%.*]] = shufflevector <4 x i32> [[V2:%.*]], <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i32> [[S0]], <8 x i32> [[S1]], <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+; CHECK-MVE-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], <12 x i32>* [[PTR:%.*]], align 4
+; CHECK-MVE-NEXT:    ret void
+;
+; CHECK-NONE-LABEL: @store_factor3(
+; CHECK-NONE-NEXT:    [[S0:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NONE-NEXT:    [[S1:%.*]] = shufflevector <4 x i32> [[V2:%.*]], <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i32> [[S0]], <8 x i32> [[S1]], <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+; CHECK-NONE-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], <12 x i32>* [[PTR:%.*]], align 4
+; CHECK-NONE-NEXT:    ret void
 ;
   %s0 = shufflevector <4 x i32> %v0, <4 x i32> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   %s1 = shufflevector <4 x i32> %v2, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -97,17 +152,30 @@ define void @store_factor3(<12 x i32>* %ptr, <4 x i32> %v0, <4 x i32> %v1, <4 x
 }
 
 define void @store_factor4(<16 x i32>* %ptr, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2, <4 x i32> %v3) {
-; NEON-LABEL:    @store_factor4(
-; NEON:            [[TMP1:%.*]] = bitcast <16 x i32>* %ptr to i8*
-; NEON-NEXT:       [[TMP2:%.*]] = shufflevector <8 x i32> %s0, <8 x i32> %s1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; NEON-NEXT:       [[TMP3:%.*]] = shufflevector <8 x i32> %s0, <8 x i32> %s1, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; NEON-NEXT:       [[TMP4:%.*]] = shufflevector <8 x i32> %s0, <8 x i32> %s1, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
-; NEON-NEXT:       [[TMP5:%.*]] = shufflevector <8 x i32> %s0, <8 x i32> %s1, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
-; NEON-NEXT:       call void @llvm.arm.neon.vst4.p0i8.v4i32(i8* [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <4 x i32> [[TMP5]], i32 4)
-; NEON-NEXT:       ret void
-; NO_NEON-LABEL: @store_factor4(
-; NO_NEON-NOT:     @llvm.arm.neon
-; NO_NEON:         ret void
+; CHECK-NEON-LABEL: @store_factor4(
+; CHECK-NEON-NEXT:    [[S0:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEON-NEXT:    [[S1:%.*]] = shufflevector <4 x i32> [[V2:%.*]], <4 x i32> [[V3:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEON-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[S0]], <8 x i32> [[S1]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEON-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[S0]], <8 x i32> [[S1]], <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEON-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[S0]], <8 x i32> [[S1]], <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEON-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i32> [[S0]], <8 x i32> [[S1]], <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEON-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32>* [[PTR:%.*]] to i8*
+; CHECK-NEON-NEXT:    call void @llvm.arm.neon.vst4.p0i8.v4i32(i8* [[TMP5]], <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], i32 4)
+; CHECK-NEON-NEXT:    ret void
+;
+; CHECK-MVE-LABEL: @store_factor4(
+; CHECK-MVE-NEXT:    [[S0:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-MVE-NEXT:    [[S1:%.*]] = shufflevector <4 x i32> [[V2:%.*]], <4 x i32> [[V3:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i32> [[S0]], <8 x i32> [[S1]], <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 1, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
+; CHECK-MVE-NEXT:    store <16 x i32> [[INTERLEAVED_VEC]], <16 x i32>* [[PTR:%.*]], align 4
+; CHECK-MVE-NEXT:    ret void
+;
+; CHECK-NONE-LABEL: @store_factor4(
+; CHECK-NONE-NEXT:    [[S0:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NONE-NEXT:    [[S1:%.*]] = shufflevector <4 x i32> [[V2:%.*]], <4 x i32> [[V3:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i32> [[S0]], <8 x i32> [[S1]], <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 1, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
+; CHECK-NONE-NEXT:    store <16 x i32> [[INTERLEAVED_VEC]], <16 x i32>* [[PTR:%.*]], align 4
+; CHECK-NONE-NEXT:    ret void
 ;
   %s0 = shufflevector <4 x i32> %v0, <4 x i32> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   %s1 = shufflevector <4 x i32> %v2, <4 x i32> %v3, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -117,15 +185,22 @@ define void @store_factor4(<16 x i32>* %ptr, <4 x i32> %v0, <4 x i32> %v1, <4 x
 }
 
 define void @load_ptrvec_factor2(<4 x i32*>* %ptr) {
-; NEON-LABEL:    @load_ptrvec_factor2(
-; NEON-NEXT:       [[TMP1:%.*]] = bitcast <4 x i32*>* %ptr to i8*
-; NEON-NEXT:       [[VLDN:%.*]] = call { <2 x i32>, <2 x i32> } @llvm.arm.neon.vld2.v2i32.p0i8(i8* [[TMP1]], i32 4)
-; NEON-NEXT:       [[TMP2:%.*]] = extractvalue { <2 x i32>, <2 x i32> } [[VLDN]], 0
-; NEON-NEXT:       [[TMP3:%.*]] = inttoptr <2 x i32> [[TMP2]] to <2 x i32*>
-; NEON-NEXT:       ret void
-; NO_NEON-LABEL: @load_ptrvec_factor2(
-; NO_NEON-NOT:     @llvm.arm.neon
-; NO_NEON:         ret void
+; CHECK-NEON-LABEL: @load_ptrvec_factor2(
+; CHECK-NEON-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32*>* [[PTR:%.*]] to i8*
+; CHECK-NEON-NEXT:    [[VLDN:%.*]] = call { <2 x i32>, <2 x i32> } @llvm.arm.neon.vld2.v2i32.p0i8(i8* [[TMP1]], i32 4)
+; CHECK-NEON-NEXT:    [[TMP2:%.*]] = extractvalue { <2 x i32>, <2 x i32> } [[VLDN]], 0
+; CHECK-NEON-NEXT:    [[TMP3:%.*]] = inttoptr <2 x i32> [[TMP2]] to <2 x i32*>
+; CHECK-NEON-NEXT:    ret void
+;
+; CHECK-MVE-LABEL: @load_ptrvec_factor2(
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = load <4 x i32*>, <4 x i32*>* [[PTR:%.*]], align 4
+; CHECK-MVE-NEXT:    [[V0:%.*]] = shufflevector <4 x i32*> [[INTERLEAVED_VEC]], <4 x i32*> undef, <2 x i32> <i32 0, i32 2>
+; CHECK-MVE-NEXT:    ret void
+;
+; CHECK-NONE-LABEL: @load_ptrvec_factor2(
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = load <4 x i32*>, <4 x i32*>* [[PTR:%.*]], align 4
+; CHECK-NONE-NEXT:    [[V0:%.*]] = shufflevector <4 x i32*> [[INTERLEAVED_VEC]], <4 x i32*> undef, <2 x i32> <i32 0, i32 2>
+; CHECK-NONE-NEXT:    ret void
 ;
   %interleaved.vec = load <4 x i32*>, <4 x i32*>* %ptr, align 4
   %v0 = shufflevector <4 x i32*> %interleaved.vec, <4 x i32*> undef, <2 x i32> <i32 0, i32 2>
@@ -133,19 +208,30 @@ define void @load_ptrvec_factor2(<4 x i32*>* %ptr) {
 }
 
 define void @load_ptrvec_factor3(<6 x i32*>* %ptr) {
-; NEON-LABEL:    @load_ptrvec_factor3(
-; NEON-NEXT:       [[TMP1:%.*]] = bitcast <6 x i32*>* %ptr to i8*
-; NEON-NEXT:       [[VLDN:%.*]] = call { <2 x i32>, <2 x i32>, <2 x i32> } @llvm.arm.neon.vld3.v2i32.p0i8(i8* [[TMP1]], i32 4)
-; NEON-NEXT:       [[TMP2:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[VLDN]], 2
-; NEON-NEXT:       [[TMP3:%.*]] = inttoptr <2 x i32> [[TMP2]] to <2 x i32*>
-; NEON-NEXT:       [[TMP4:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[VLDN]], 1
-; NEON-NEXT:       [[TMP5:%.*]] = inttoptr <2 x i32> [[TMP4]] to <2 x i32*>
-; NEON-NEXT:       [[TMP6:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[VLDN]], 0
-; NEON-NEXT:       [[TMP7:%.*]] = inttoptr <2 x i32> [[TMP6]] to <2 x i32*>
-; NEON-NEXT:       ret void
-; NO_NEON-LABEL: @load_ptrvec_factor3(
-; NO_NEON-NOT:     @llvm.arm.neon
-; NO_NEON:         ret void
+; CHECK-NEON-LABEL: @load_ptrvec_factor3(
+; CHECK-NEON-NEXT:    [[TMP1:%.*]] = bitcast <6 x i32*>* [[PTR:%.*]] to i8*
+; CHECK-NEON-NEXT:    [[VLDN:%.*]] = call { <2 x i32>, <2 x i32>, <2 x i32> } @llvm.arm.neon.vld3.v2i32.p0i8(i8* [[TMP1]], i32 4)
+; CHECK-NEON-NEXT:    [[TMP2:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[VLDN]], 2
+; CHECK-NEON-NEXT:    [[TMP3:%.*]] = inttoptr <2 x i32> [[TMP2]] to <2 x i32*>
+; CHECK-NEON-NEXT:    [[TMP4:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[VLDN]], 1
+; CHECK-NEON-NEXT:    [[TMP5:%.*]] = inttoptr <2 x i32> [[TMP4]] to <2 x i32*>
+; CHECK-NEON-NEXT:    [[TMP6:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[VLDN]], 0
+; CHECK-NEON-NEXT:    [[TMP7:%.*]] = inttoptr <2 x i32> [[TMP6]] to <2 x i32*>
+; CHECK-NEON-NEXT:    ret void
+;
+; CHECK-MVE-LABEL: @load_ptrvec_factor3(
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = load <6 x i32*>, <6 x i32*>* [[PTR:%.*]], align 4
+; CHECK-MVE-NEXT:    [[V0:%.*]] = shufflevector <6 x i32*> [[INTERLEAVED_VEC]], <6 x i32*> undef, <2 x i32> <i32 0, i32 3>
+; CHECK-MVE-NEXT:    [[V1:%.*]] = shufflevector <6 x i32*> [[INTERLEAVED_VEC]], <6 x i32*> undef, <2 x i32> <i32 1, i32 4>
+; CHECK-MVE-NEXT:    [[V2:%.*]] = shufflevector <6 x i32*> [[INTERLEAVED_VEC]], <6 x i32*> undef, <2 x i32> <i32 2, i32 5>
+; CHECK-MVE-NEXT:    ret void
+;
+; CHECK-NONE-LABEL: @load_ptrvec_factor3(
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = load <6 x i32*>, <6 x i32*>* [[PTR:%.*]], align 4
+; CHECK-NONE-NEXT:    [[V0:%.*]] = shufflevector <6 x i32*> [[INTERLEAVED_VEC]], <6 x i32*> undef, <2 x i32> <i32 0, i32 3>
+; CHECK-NONE-NEXT:    [[V1:%.*]] = shufflevector <6 x i32*> [[INTERLEAVED_VEC]], <6 x i32*> undef, <2 x i32> <i32 1, i32 4>
+; CHECK-NONE-NEXT:    [[V2:%.*]] = shufflevector <6 x i32*> [[INTERLEAVED_VEC]], <6 x i32*> undef, <2 x i32> <i32 2, i32 5>
+; CHECK-NONE-NEXT:    ret void
 ;
   %interleaved.vec = load <6 x i32*>, <6 x i32*>* %ptr, align 4
   %v0 = shufflevector <6 x i32*> %interleaved.vec, <6 x i32*> undef, <2 x i32> <i32 0, i32 3>
@@ -155,21 +241,34 @@ define void @load_ptrvec_factor3(<6 x i32*>* %ptr) {
 }
 
 define void @load_ptrvec_factor4(<8 x i32*>* %ptr) {
-; NEON-LABEL:    @load_ptrvec_factor4(
-; NEON-NEXT:       [[TMP1:%.*]] = bitcast <8 x i32*>* %ptr to i8*
-; NEON-NEXT:       [[VLDN:%.*]] = call { <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32> } @llvm.arm.neon.vld4.v2i32.p0i8(i8* [[TMP1]], i32 4)
-; NEON-NEXT:       [[TMP2:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32> } [[VLDN]], 3
-; NEON-NEXT:       [[TMP3:%.*]] = inttoptr <2 x i32> [[TMP2]] to <2 x i32*>
-; NEON-NEXT:       [[TMP4:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32> } [[VLDN]], 2
-; NEON-NEXT:       [[TMP5:%.*]] = inttoptr <2 x i32> [[TMP4]] to <2 x i32*>
-; NEON-NEXT:       [[TMP6:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32> } [[VLDN]], 1
-; NEON-NEXT:       [[TMP7:%.*]] = inttoptr <2 x i32> [[TMP6]] to <2 x i32*>
-; NEON-NEXT:       [[TMP8:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32> } [[VLDN]], 0
-; NEON-NEXT:       [[TMP9:%.*]] = inttoptr <2 x i32> [[TMP8]] to <2 x i32*>
-; NEON-NEXT:       ret void
-; NO_NEON-LABEL: @load_ptrvec_factor4(
-; NO_NEON-NOT:     @llvm.arm.neon
-; NO_NEON:         ret void
+; CHECK-NEON-LABEL: @load_ptrvec_factor4(
+; CHECK-NEON-NEXT:    [[TMP1:%.*]] = bitcast <8 x i32*>* [[PTR:%.*]] to i8*
+; CHECK-NEON-NEXT:    [[VLDN:%.*]] = call { <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32> } @llvm.arm.neon.vld4.v2i32.p0i8(i8* [[TMP1]], i32 4)
+; CHECK-NEON-NEXT:    [[TMP2:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32> } [[VLDN]], 3
+; CHECK-NEON-NEXT:    [[TMP3:%.*]] = inttoptr <2 x i32> [[TMP2]] to <2 x i32*>
+; CHECK-NEON-NEXT:    [[TMP4:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32> } [[VLDN]], 2
+; CHECK-NEON-NEXT:    [[TMP5:%.*]] = inttoptr <2 x i32> [[TMP4]] to <2 x i32*>
+; CHECK-NEON-NEXT:    [[TMP6:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32> } [[VLDN]], 1
+; CHECK-NEON-NEXT:    [[TMP7:%.*]] = inttoptr <2 x i32> [[TMP6]] to <2 x i32*>
+; CHECK-NEON-NEXT:    [[TMP8:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32> } [[VLDN]], 0
+; CHECK-NEON-NEXT:    [[TMP9:%.*]] = inttoptr <2 x i32> [[TMP8]] to <2 x i32*>
+; CHECK-NEON-NEXT:    ret void
+;
+; CHECK-MVE-LABEL: @load_ptrvec_factor4(
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = load <8 x i32*>, <8 x i32*>* [[PTR:%.*]], align 4
+; CHECK-MVE-NEXT:    [[V0:%.*]] = shufflevector <8 x i32*> [[INTERLEAVED_VEC]], <8 x i32*> undef, <2 x i32> <i32 0, i32 4>
+; CHECK-MVE-NEXT:    [[V1:%.*]] = shufflevector <8 x i32*> [[INTERLEAVED_VEC]], <8 x i32*> undef, <2 x i32> <i32 1, i32 5>
+; CHECK-MVE-NEXT:    [[V2:%.*]] = shufflevector <8 x i32*> [[INTERLEAVED_VEC]], <8 x i32*> undef, <2 x i32> <i32 2, i32 6>
+; CHECK-MVE-NEXT:    [[V3:%.*]] = shufflevector <8 x i32*> [[INTERLEAVED_VEC]], <8 x i32*> undef, <2 x i32> <i32 3, i32 7>
+; CHECK-MVE-NEXT:    ret void
+;
+; CHECK-NONE-LABEL: @load_ptrvec_factor4(
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = load <8 x i32*>, <8 x i32*>* [[PTR:%.*]], align 4
+; CHECK-NONE-NEXT:    [[V0:%.*]] = shufflevector <8 x i32*> [[INTERLEAVED_VEC]], <8 x i32*> undef, <2 x i32> <i32 0, i32 4>
+; CHECK-NONE-NEXT:    [[V1:%.*]] = shufflevector <8 x i32*> [[INTERLEAVED_VEC]], <8 x i32*> undef, <2 x i32> <i32 1, i32 5>
+; CHECK-NONE-NEXT:    [[V2:%.*]] = shufflevector <8 x i32*> [[INTERLEAVED_VEC]], <8 x i32*> undef, <2 x i32> <i32 2, i32 6>
+; CHECK-NONE-NEXT:    [[V3:%.*]] = shufflevector <8 x i32*> [[INTERLEAVED_VEC]], <8 x i32*> undef, <2 x i32> <i32 3, i32 7>
+; CHECK-NONE-NEXT:    ret void
 ;
   %interleaved.vec = load <8 x i32*>, <8 x i32*>* %ptr, align 4
   %v0 = shufflevector <8 x i32*> %interleaved.vec, <8 x i32*> undef, <2 x i32> <i32 0, i32 4>
@@ -180,17 +279,24 @@ define void @load_ptrvec_factor4(<8 x i32*>* %ptr) {
 }
 
 define void @store_ptrvec_factor2(<4 x i32*>* %ptr, <2 x i32*> %v0, <2 x i32*> %v1) {
-; NEON-LABEL:    @store_ptrvec_factor2(
-; NEON-NEXT:       [[TMP1:%.*]] = ptrtoint <2 x i32*> %v0 to <2 x i32>
-; NEON-NEXT:       [[TMP2:%.*]] = ptrtoint <2 x i32*> %v1 to <2 x i32>
-; NEON-NEXT:       [[TMP3:%.*]] = bitcast <4 x i32*>* %ptr to i8*
-; NEON-NEXT:       [[TMP4:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> [[TMP2]], <2 x i32> <i32 0, i32 1>
-; NEON-NEXT:       [[TMP5:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> [[TMP2]], <2 x i32> <i32 2, i32 3>
-; NEON-NEXT:       call void @llvm.arm.neon.vst2.p0i8.v2i32(i8* [[TMP3]], <2 x i32> [[TMP4]], <2 x i32> [[TMP5]], i32 4)
-; NEON-NEXT:       ret void
-; NO_NEON-LABEL: @store_ptrvec_factor2(
-; NO_NEON-NOT:     @llvm.arm.neon
-; NO_NEON:         ret void
+; CHECK-NEON-LABEL: @store_ptrvec_factor2(
+; CHECK-NEON-NEXT:    [[TMP1:%.*]] = ptrtoint <2 x i32*> [[V0:%.*]] to <2 x i32>
+; CHECK-NEON-NEXT:    [[TMP2:%.*]] = ptrtoint <2 x i32*> [[V1:%.*]] to <2 x i32>
+; CHECK-NEON-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> [[TMP2]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEON-NEXT:    [[TMP4:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> [[TMP2]], <2 x i32> <i32 2, i32 3>
+; CHECK-NEON-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32*>* [[PTR:%.*]] to i8*
+; CHECK-NEON-NEXT:    call void @llvm.arm.neon.vst2.p0i8.v2i32(i8* [[TMP5]], <2 x i32> [[TMP3]], <2 x i32> [[TMP4]], i32 4)
+; CHECK-NEON-NEXT:    ret void
+;
+; CHECK-MVE-LABEL: @store_ptrvec_factor2(
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <2 x i32*> [[V0:%.*]], <2 x i32*> [[V1:%.*]], <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; CHECK-MVE-NEXT:    store <4 x i32*> [[INTERLEAVED_VEC]], <4 x i32*>* [[PTR:%.*]], align 4
+; CHECK-MVE-NEXT:    ret void
+;
+; CHECK-NONE-LABEL: @store_ptrvec_factor2(
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <2 x i32*> [[V0:%.*]], <2 x i32*> [[V1:%.*]], <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; CHECK-NONE-NEXT:    store <4 x i32*> [[INTERLEAVED_VEC]], <4 x i32*>* [[PTR:%.*]], align 4
+; CHECK-NONE-NEXT:    ret void
 ;
   %interleaved.vec = shufflevector <2 x i32*> %v0, <2 x i32*> %v1, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
   store <4 x i32*> %interleaved.vec, <4 x i32*>* %ptr, align 4
@@ -198,18 +304,31 @@ define void @store_ptrvec_factor2(<4 x i32*>* %ptr, <2 x i32*> %v0, <2 x i32*> %
 }
 
 define void @store_ptrvec_factor3(<6 x i32*>* %ptr, <2 x i32*> %v0, <2 x i32*> %v1, <2 x i32*> %v2) {
-; NEON-LABEL:    @store_ptrvec_factor3(
-; NEON:            [[TMP1:%.*]] = ptrtoint <4 x i32*> %s0 to <4 x i32>
-; NEON-NEXT:       [[TMP2:%.*]] = ptrtoint <4 x i32*> %s1 to <4 x i32>
-; NEON-NEXT:       [[TMP3:%.*]] = bitcast <6 x i32*>* %ptr to i8*
-; NEON-NEXT:       [[TMP4:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <2 x i32> <i32 0, i32 1>
-; NEON-NEXT:       [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <2 x i32> <i32 2, i32 3>
-; NEON-NEXT:       [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <2 x i32> <i32 4, i32 5>
-; NEON-NEXT:       call void @llvm.arm.neon.vst3.p0i8.v2i32(i8* [[TMP3]], <2 x i32> [[TMP4]], <2 x i32> [[TMP5]], <2 x i32> [[TMP6]], i32 4)
-; NEON-NEXT:       ret void
-; NO_NEON-LABEL: @store_ptrvec_factor3(
-; NO_NEON-NOT:     @llvm.arm.neon
-; NO_NEON:         ret void
+; CHECK-NEON-LABEL: @store_ptrvec_factor3(
+; CHECK-NEON-NEXT:    [[S0:%.*]] = shufflevector <2 x i32*> [[V0:%.*]], <2 x i32*> [[V1:%.*]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEON-NEXT:    [[S1:%.*]] = shufflevector <2 x i32*> [[V2:%.*]], <2 x i32*> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEON-NEXT:    [[TMP1:%.*]] = ptrtoint <4 x i32*> [[S0]] to <4 x i32>
+; CHECK-NEON-NEXT:    [[TMP2:%.*]] = ptrtoint <4 x i32*> [[S1]] to <4 x i32>
+; CHECK-NEON-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEON-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <2 x i32> <i32 2, i32 3>
+; CHECK-NEON-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <2 x i32> <i32 4, i32 5>
+; CHECK-NEON-NEXT:    [[TMP6:%.*]] = bitcast <6 x i32*>* [[PTR:%.*]] to i8*
+; CHECK-NEON-NEXT:    call void @llvm.arm.neon.vst3.p0i8.v2i32(i8* [[TMP6]], <2 x i32> [[TMP3]], <2 x i32> [[TMP4]], <2 x i32> [[TMP5]], i32 4)
+; CHECK-NEON-NEXT:    ret void
+;
+; CHECK-MVE-LABEL: @store_ptrvec_factor3(
+; CHECK-MVE-NEXT:    [[S0:%.*]] = shufflevector <2 x i32*> [[V0:%.*]], <2 x i32*> [[V1:%.*]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-MVE-NEXT:    [[S1:%.*]] = shufflevector <2 x i32*> [[V2:%.*]], <2 x i32*> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i32*> [[S0]], <4 x i32*> [[S1]], <6 x i32> <i32 0, i32 2, i32 4, i32 1, i32 3, i32 5>
+; CHECK-MVE-NEXT:    store <6 x i32*> [[INTERLEAVED_VEC]], <6 x i32*>* [[PTR:%.*]], align 4
+; CHECK-MVE-NEXT:    ret void
+;
+; CHECK-NONE-LABEL: @store_ptrvec_factor3(
+; CHECK-NONE-NEXT:    [[S0:%.*]] = shufflevector <2 x i32*> [[V0:%.*]], <2 x i32*> [[V1:%.*]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NONE-NEXT:    [[S1:%.*]] = shufflevector <2 x i32*> [[V2:%.*]], <2 x i32*> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i32*> [[S0]], <4 x i32*> [[S1]], <6 x i32> <i32 0, i32 2, i32 4, i32 1, i32 3, i32 5>
+; CHECK-NONE-NEXT:    store <6 x i32*> [[INTERLEAVED_VEC]], <6 x i32*>* [[PTR:%.*]], align 4
+; CHECK-NONE-NEXT:    ret void
 ;
   %s0 = shufflevector <2 x i32*> %v0, <2 x i32*> %v1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %s1 = shufflevector <2 x i32*> %v2, <2 x i32*> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
@@ -219,19 +338,32 @@ define void @store_ptrvec_factor3(<6 x i32*>* %ptr, <2 x i32*> %v0, <2 x i32*> %
 }
 
 define void @store_ptrvec_factor4(<8 x i32*>* %ptr, <2 x i32*> %v0, <2 x i32*> %v1, <2 x i32*> %v2, <2 x i32*> %v3) {
-; NEON-LABEL:    @store_ptrvec_factor4(
-; NEON:            [[TMP1:%.*]] = ptrtoint <4 x i32*> %s0 to <4 x i32>
-; NEON-NEXT:       [[TMP2:%.*]] = ptrtoint <4 x i32*> %s1 to <4 x i32>
-; NEON-NEXT:       [[TMP3:%.*]] = bitcast <8 x i32*>* %ptr to i8*
-; NEON-NEXT:       [[TMP4:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <2 x i32> <i32 0, i32 1>
-; NEON-NEXT:       [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <2 x i32> <i32 2, i32 3>
-; NEON-NEXT:       [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <2 x i32> <i32 4, i32 5>
-; NEON-NEXT:       [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <2 x i32> <i32 6, i32 7>
-; NEON-NEXT:       call void @llvm.arm.neon.vst4.p0i8.v2i32(i8* [[TMP3]], <2 x i32> [[TMP4]], <2 x i32> [[TMP5]], <2 x i32> [[TMP6]], <2 x i32> [[TMP7]], i32 4)
-; NEON-NEXT:       ret void
-; NO_NEON-LABEL: @store_ptrvec_factor4(
-; NO_NEON-NOT:     @llvm.arm.neon
-; NO_NEON:         ret void
+; CHECK-NEON-LABEL: @store_ptrvec_factor4(
+; CHECK-NEON-NEXT:    [[S0:%.*]] = shufflevector <2 x i32*> [[V0:%.*]], <2 x i32*> [[V1:%.*]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEON-NEXT:    [[S1:%.*]] = shufflevector <2 x i32*> [[V2:%.*]], <2 x i32*> [[V3:%.*]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEON-NEXT:    [[TMP1:%.*]] = ptrtoint <4 x i32*> [[S0]] to <4 x i32>
+; CHECK-NEON-NEXT:    [[TMP2:%.*]] = ptrtoint <4 x i32*> [[S1]] to <4 x i32>
+; CHECK-NEON-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEON-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <2 x i32> <i32 2, i32 3>
+; CHECK-NEON-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <2 x i32> <i32 4, i32 5>
+; CHECK-NEON-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <2 x i32> <i32 6, i32 7>
+; CHECK-NEON-NEXT:    [[TMP7:%.*]] = bitcast <8 x i32*>* [[PTR:%.*]] to i8*
+; CHECK-NEON-NEXT:    call void @llvm.arm.neon.vst4.p0i8.v2i32(i8* [[TMP7]], <2 x i32> [[TMP3]], <2 x i32> [[TMP4]], <2 x i32> [[TMP5]], <2 x i32> [[TMP6]], i32 4)
+; CHECK-NEON-NEXT:    ret void
+;
+; CHECK-MVE-LABEL: @store_ptrvec_factor4(
+; CHECK-MVE-NEXT:    [[S0:%.*]] = shufflevector <2 x i32*> [[V0:%.*]], <2 x i32*> [[V1:%.*]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-MVE-NEXT:    [[S1:%.*]] = shufflevector <2 x i32*> [[V2:%.*]], <2 x i32*> [[V3:%.*]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i32*> [[S0]], <4 x i32*> [[S1]], <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 1, i32 3, i32 5, i32 7>
+; CHECK-MVE-NEXT:    store <8 x i32*> [[INTERLEAVED_VEC]], <8 x i32*>* [[PTR:%.*]], align 4
+; CHECK-MVE-NEXT:    ret void
+;
+; CHECK-NONE-LABEL: @store_ptrvec_factor4(
+; CHECK-NONE-NEXT:    [[S0:%.*]] = shufflevector <2 x i32*> [[V0:%.*]], <2 x i32*> [[V1:%.*]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NONE-NEXT:    [[S1:%.*]] = shufflevector <2 x i32*> [[V2:%.*]], <2 x i32*> [[V3:%.*]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i32*> [[S0]], <4 x i32*> [[S1]], <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 1, i32 3, i32 5, i32 7>
+; CHECK-NONE-NEXT:    store <8 x i32*> [[INTERLEAVED_VEC]], <8 x i32*>* [[PTR:%.*]], align 4
+; CHECK-NONE-NEXT:    ret void
 ;
   %s0 = shufflevector <2 x i32*> %v0, <2 x i32*> %v1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %s1 = shufflevector <2 x i32*> %v2, <2 x i32*> %v3, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -241,15 +373,25 @@ define void @store_ptrvec_factor4(<8 x i32*>* %ptr, <2 x i32*> %v0, <2 x i32*> %
 }
 
 define void @load_undef_mask_factor2(<8 x i32>* %ptr) {
-; NEON-LABEL:    @load_undef_mask_factor2(
-; NEON-NEXT:       [[TMP1:%.*]] = bitcast <8 x i32>* %ptr to i8*
-; NEON-NEXT:       [[VLDN:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.arm.neon.vld2.v4i32.p0i8(i8* [[TMP1]], i32 4)
-; NEON-NEXT:       [[TMP2:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN]], 1
-; NEON-NEXT:       [[TMP3:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN]], 0
-; NEON-NEXT:       ret void
-; NO_NEON-LABEL: @load_undef_mask_factor2(
-; NO_NEON-NOT:     @llvm.arm.neon
-; NO_NEON:         ret void
+; CHECK-NEON-LABEL: @load_undef_mask_factor2(
+; CHECK-NEON-NEXT:    [[TMP1:%.*]] = bitcast <8 x i32>* [[PTR:%.*]] to i8*
+; CHECK-NEON-NEXT:    [[VLDN:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.arm.neon.vld2.v4i32.p0i8(i8* [[TMP1]], i32 4)
+; CHECK-NEON-NEXT:    [[TMP2:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN]], 1
+; CHECK-NEON-NEXT:    [[TMP3:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN]], 0
+; CHECK-NEON-NEXT:    ret void
+;
+; CHECK-MVE-LABEL: @load_undef_mask_factor2(
+; CHECK-MVE-NEXT:    [[TMP1:%.*]] = bitcast <8 x i32>* [[PTR:%.*]] to i32*
+; CHECK-MVE-NEXT:    [[VLDN:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.arm.mve.vld2q.v4i32.p0i32(i32* [[TMP1]])
+; CHECK-MVE-NEXT:    [[TMP2:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN]], 1
+; CHECK-MVE-NEXT:    [[TMP3:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN]], 0
+; CHECK-MVE-NEXT:    ret void
+;
+; CHECK-NONE-LABEL: @load_undef_mask_factor2(
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = load <8 x i32>, <8 x i32>* [[PTR:%.*]], align 4
+; CHECK-NONE-NEXT:    [[V0:%.*]] = shufflevector <8 x i32> [[INTERLEAVED_VEC]], <8 x i32> undef, <4 x i32> <i32 undef, i32 2, i32 undef, i32 6>
+; CHECK-NONE-NEXT:    [[V1:%.*]] = shufflevector <8 x i32> [[INTERLEAVED_VEC]], <8 x i32> undef, <4 x i32> <i32 undef, i32 3, i32 undef, i32 7>
+; CHECK-NONE-NEXT:    ret void
 ;
   %interleaved.vec = load <8 x i32>, <8 x i32>* %ptr, align 4
   %v0 = shufflevector <8 x i32> %interleaved.vec, <8 x i32> undef, <4 x i32> <i32 undef, i32 2, i32 undef, i32 6>
@@ -258,16 +400,27 @@ define void @load_undef_mask_factor2(<8 x i32>* %ptr) {
 }
 
 define void @load_undef_mask_factor3(<12 x i32>* %ptr) {
-; NEON-LABEL:    @load_undef_mask_factor3(
-; NEON-NEXT:       [[TMP1:%.*]] = bitcast <12 x i32>* %ptr to i8*
-; NEON-NEXT:       [[VLDN:%.*]] = call { <4 x i32>, <4 x i32>, <4 x i32> } @llvm.arm.neon.vld3.v4i32.p0i8(i8* [[TMP1]], i32 4)
-; NEON-NEXT:       [[TMP2:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN]], 2
-; NEON-NEXT:       [[TMP3:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN]], 1
-; NEON-NEXT:       [[TMP4:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN]], 0
-; NEON-NEXT:       ret void
-; NO_NEON-LABEL: @load_undef_mask_factor3(
-; NO_NEON-NOT:     @llvm.arm.neon
-; NO_NEON:         ret void
+; CHECK-NEON-LABEL: @load_undef_mask_factor3(
+; CHECK-NEON-NEXT:    [[TMP1:%.*]] = bitcast <12 x i32>* [[PTR:%.*]] to i8*
+; CHECK-NEON-NEXT:    [[VLDN:%.*]] = call { <4 x i32>, <4 x i32>, <4 x i32> } @llvm.arm.neon.vld3.v4i32.p0i8(i8* [[TMP1]], i32 4)
+; CHECK-NEON-NEXT:    [[TMP2:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN]], 2
+; CHECK-NEON-NEXT:    [[TMP3:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN]], 1
+; CHECK-NEON-NEXT:    [[TMP4:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN]], 0
+; CHECK-NEON-NEXT:    ret void
+;
+; CHECK-MVE-LABEL: @load_undef_mask_factor3(
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = load <12 x i32>, <12 x i32>* [[PTR:%.*]], align 4
+; CHECK-MVE-NEXT:    [[V0:%.*]] = shufflevector <12 x i32> [[INTERLEAVED_VEC]], <12 x i32> undef, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
+; CHECK-MVE-NEXT:    [[V1:%.*]] = shufflevector <12 x i32> [[INTERLEAVED_VEC]], <12 x i32> undef, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
+; CHECK-MVE-NEXT:    [[V2:%.*]] = shufflevector <12 x i32> [[INTERLEAVED_VEC]], <12 x i32> undef, <4 x i32> <i32 2, i32 undef, i32 undef, i32 undef>
+; CHECK-MVE-NEXT:    ret void
+;
+; CHECK-NONE-LABEL: @load_undef_mask_factor3(
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = load <12 x i32>, <12 x i32>* [[PTR:%.*]], align 4
+; CHECK-NONE-NEXT:    [[V0:%.*]] = shufflevector <12 x i32> [[INTERLEAVED_VEC]], <12 x i32> undef, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
+; CHECK-NONE-NEXT:    [[V1:%.*]] = shufflevector <12 x i32> [[INTERLEAVED_VEC]], <12 x i32> undef, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
+; CHECK-NONE-NEXT:    [[V2:%.*]] = shufflevector <12 x i32> [[INTERLEAVED_VEC]], <12 x i32> undef, <4 x i32> <i32 2, i32 undef, i32 undef, i32 undef>
+; CHECK-NONE-NEXT:    ret void
 ;
   %interleaved.vec = load <12 x i32>, <12 x i32>* %ptr, align 4
   %v0 = shufflevector <12 x i32> %interleaved.vec, <12 x i32> undef, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
@@ -277,17 +430,30 @@ define void @load_undef_mask_factor3(<12 x i32>* %ptr) {
 }
 
 define void @load_undef_mask_factor4(<16 x i32>* %ptr) {
-; NEON-LABEL:    @load_undef_mask_factor4(
-; NEON-NEXT:       [[TMP1:%.*]] = bitcast <16 x i32>* %ptr to i8*
-; NEON-NEXT:       [[VLDN:%.*]] = call { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } @llvm.arm.neon.vld4.v4i32.p0i8(i8* [[TMP1]], i32 4)
-; NEON-NEXT:       [[TMP2:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN]], 3
-; NEON-NEXT:       [[TMP3:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN]], 2
-; NEON-NEXT:       [[TMP4:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN]], 1
-; NEON-NEXT:       [[TMP5:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN]], 0
-; NEON-NEXT:       ret void
-; NO_NEON-LABEL: @load_undef_mask_factor4(
-; NO_NEON-NOT:     @llvm.arm.neon
-; NO_NEON:         ret void
+; CHECK-NEON-LABEL: @load_undef_mask_factor4(
+; CHECK-NEON-NEXT:    [[TMP1:%.*]] = bitcast <16 x i32>* [[PTR:%.*]] to i8*
+; CHECK-NEON-NEXT:    [[VLDN:%.*]] = call { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } @llvm.arm.neon.vld4.v4i32.p0i8(i8* [[TMP1]], i32 4)
+; CHECK-NEON-NEXT:    [[TMP2:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN]], 3
+; CHECK-NEON-NEXT:    [[TMP3:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN]], 2
+; CHECK-NEON-NEXT:    [[TMP4:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN]], 1
+; CHECK-NEON-NEXT:    [[TMP5:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN]], 0
+; CHECK-NEON-NEXT:    ret void
+;
+; CHECK-MVE-LABEL: @load_undef_mask_factor4(
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = load <16 x i32>, <16 x i32>* [[PTR:%.*]], align 4
+; CHECK-MVE-NEXT:    [[V0:%.*]] = shufflevector <16 x i32> [[INTERLEAVED_VEC]], <16 x i32> undef, <4 x i32> <i32 0, i32 4, i32 undef, i32 undef>
+; CHECK-MVE-NEXT:    [[V1:%.*]] = shufflevector <16 x i32> [[INTERLEAVED_VEC]], <16 x i32> undef, <4 x i32> <i32 1, i32 5, i32 undef, i32 undef>
+; CHECK-MVE-NEXT:    [[V2:%.*]] = shufflevector <16 x i32> [[INTERLEAVED_VEC]], <16 x i32> undef, <4 x i32> <i32 2, i32 6, i32 undef, i32 undef>
+; CHECK-MVE-NEXT:    [[V3:%.*]] = shufflevector <16 x i32> [[INTERLEAVED_VEC]], <16 x i32> undef, <4 x i32> <i32 3, i32 7, i32 undef, i32 undef>
+; CHECK-MVE-NEXT:    ret void
+;
+; CHECK-NONE-LABEL: @load_undef_mask_factor4(
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = load <16 x i32>, <16 x i32>* [[PTR:%.*]], align 4
+; CHECK-NONE-NEXT:    [[V0:%.*]] = shufflevector <16 x i32> [[INTERLEAVED_VEC]], <16 x i32> undef, <4 x i32> <i32 0, i32 4, i32 undef, i32 undef>
+; CHECK-NONE-NEXT:    [[V1:%.*]] = shufflevector <16 x i32> [[INTERLEAVED_VEC]], <16 x i32> undef, <4 x i32> <i32 1, i32 5, i32 undef, i32 undef>
+; CHECK-NONE-NEXT:    [[V2:%.*]] = shufflevector <16 x i32> [[INTERLEAVED_VEC]], <16 x i32> undef, <4 x i32> <i32 2, i32 6, i32 undef, i32 undef>
+; CHECK-NONE-NEXT:    [[V3:%.*]] = shufflevector <16 x i32> [[INTERLEAVED_VEC]], <16 x i32> undef, <4 x i32> <i32 3, i32 7, i32 undef, i32 undef>
+; CHECK-NONE-NEXT:    ret void
 ;
   %interleaved.vec = load <16 x i32>, <16 x i32>* %ptr, align 4
   %v0 = shufflevector <16 x i32> %interleaved.vec, <16 x i32> undef, <4 x i32> <i32 0, i32 4, i32 undef, i32 undef>
@@ -298,15 +464,25 @@ define void @load_undef_mask_factor4(<16 x i32>* %ptr) {
 }
 
 define void @store_undef_mask_factor2(<8 x i32>* %ptr, <4 x i32> %v0, <4 x i32> %v1) {
-; NEON-LABEL:    @store_undef_mask_factor2(
-; NEON-NEXT:       [[TMP1:%.*]] = bitcast <8 x i32>* %ptr to i8*
-; NEON-NEXT:       [[TMP2:%.*]] = shufflevector <4 x i32> %v0, <4 x i32> %v1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; NEON-NEXT:       [[TMP3:%.*]] = shufflevector <4 x i32> %v0, <4 x i32> %v1, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; NEON-NEXT:       call void @llvm.arm.neon.vst2.p0i8.v4i32(i8* [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> [[TMP3]], i32 4)
-; NEON-NEXT:       ret void
-; NO_NEON-LABEL: @store_undef_mask_factor2(
-; NO_NEON-NOT:     @llvm.arm.neon
-; NO_NEON:         ret void
+; CHECK-NEON-LABEL: @store_undef_mask_factor2(
+; CHECK-NEON-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEON-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i32> [[V0]], <4 x i32> [[V1]], <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEON-NEXT:    [[TMP3:%.*]] = bitcast <8 x i32>* [[PTR:%.*]] to i8*
+; CHECK-NEON-NEXT:    call void @llvm.arm.neon.vst2.p0i8.v4i32(i8* [[TMP3]], <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], i32 4)
+; CHECK-NEON-NEXT:    ret void
+;
+; CHECK-MVE-LABEL: @store_undef_mask_factor2(
+; CHECK-MVE-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-MVE-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i32> [[V0]], <4 x i32> [[V1]], <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-MVE-NEXT:    [[TMP3:%.*]] = bitcast <8 x i32>* [[PTR:%.*]] to i32*
+; CHECK-MVE-NEXT:    call void @llvm.arm.mve.vst2q.p0i32.v4i32(i32* [[TMP3]], <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], i32 0)
+; CHECK-MVE-NEXT:    call void @llvm.arm.mve.vst2q.p0i32.v4i32(i32* [[TMP3]], <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], i32 1)
+; CHECK-MVE-NEXT:    ret void
+;
+; CHECK-NONE-LABEL: @store_undef_mask_factor2(
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <8 x i32> <i32 undef, i32 undef, i32 undef, i32 undef, i32 2, i32 6, i32 3, i32 7>
+; CHECK-NONE-NEXT:    store <8 x i32> [[INTERLEAVED_VEC]], <8 x i32>* [[PTR:%.*]], align 4
+; CHECK-NONE-NEXT:    ret void
 ;
   %interleaved.vec = shufflevector <4 x i32> %v0, <4 x i32> %v1, <8 x i32> <i32 undef, i32 undef, i32 undef, i32 undef, i32 2, i32 6, i32 3, i32 7>
   store <8 x i32> %interleaved.vec, <8 x i32>* %ptr, align 4
@@ -314,16 +490,29 @@ define void @store_undef_mask_factor2(<8 x i32>* %ptr, <4 x i32> %v0, <4 x i32>
 }
 
 define void @store_undef_mask_factor3(<12 x i32>* %ptr, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2) {
-; NEON-LABEL:    @store_undef_mask_factor3(
-; NEON:            [[TMP1:%.*]] = bitcast <12 x i32>* %ptr to i8*
-; NEON-NEXT:       [[TMP2:%.*]] = shufflevector <8 x i32> %s0, <8 x i32> %s1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; NEON-NEXT:       [[TMP3:%.*]] = shufflevector <8 x i32> %s0, <8 x i32> %s1, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; NEON-NEXT:       [[TMP4:%.*]] = shufflevector <8 x i32> %s0, <8 x i32> %s1, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
-; NEON-NEXT:       call void @llvm.arm.neon.vst3.p0i8.v4i32(i8* [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], i32 4)
-; NEON-NEXT:       ret void
-; NO_NEON-LABEL: @store_undef_mask_factor3(
-; NO_NEON-NOT:     @llvm.arm.neon
-; NO_NEON:         ret void
+; CHECK-NEON-LABEL: @store_undef_mask_factor3(
+; CHECK-NEON-NEXT:    [[S0:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEON-NEXT:    [[S1:%.*]] = shufflevector <4 x i32> [[V2:%.*]], <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEON-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[S0]], <8 x i32> [[S1]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEON-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[S0]], <8 x i32> [[S1]], <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEON-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[S0]], <8 x i32> [[S1]], <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEON-NEXT:    [[TMP4:%.*]] = bitcast <12 x i32>* [[PTR:%.*]] to i8*
+; CHECK-NEON-NEXT:    call void @llvm.arm.neon.vst3.p0i8.v4i32(i8* [[TMP4]], <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> [[TMP3]], i32 4)
+; CHECK-NEON-NEXT:    ret void
+;
+; CHECK-MVE-LABEL: @store_undef_mask_factor3(
+; CHECK-MVE-NEXT:    [[S0:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-MVE-NEXT:    [[S1:%.*]] = shufflevector <4 x i32> [[V2:%.*]], <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i32> [[S0]], <8 x i32> [[S1]], <12 x i32> <i32 0, i32 4, i32 undef, i32 1, i32 undef, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+; CHECK-MVE-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], <12 x i32>* [[PTR:%.*]], align 4
+; CHECK-MVE-NEXT:    ret void
+;
+; CHECK-NONE-LABEL: @store_undef_mask_factor3(
+; CHECK-NONE-NEXT:    [[S0:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NONE-NEXT:    [[S1:%.*]] = shufflevector <4 x i32> [[V2:%.*]], <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i32> [[S0]], <8 x i32> [[S1]], <12 x i32> <i32 0, i32 4, i32 undef, i32 1, i32 undef, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+; CHECK-NONE-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], <12 x i32>* [[PTR:%.*]], align 4
+; CHECK-NONE-NEXT:    ret void
 ;
   %s0 = shufflevector <4 x i32> %v0, <4 x i32> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   %s1 = shufflevector <4 x i32> %v2, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -333,17 +522,30 @@ define void @store_undef_mask_factor3(<12 x i32>* %ptr, <4 x i32> %v0, <4 x i32>
 }
 
 define void @store_undef_mask_factor4(<16 x i32>* %ptr, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2, <4 x i32> %v3) {
-; NEON-LABEL:    @store_undef_mask_factor4(
-; NEON:            [[TMP1:%.*]] = bitcast <16 x i32>* %ptr to i8*
-; NEON-NEXT:       [[TMP2:%.*]] = shufflevector <8 x i32> %s0, <8 x i32> %s1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; NEON-NEXT:       [[TMP3:%.*]] = shufflevector <8 x i32> %s0, <8 x i32> %s1, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; NEON-NEXT:       [[TMP4:%.*]] = shufflevector <8 x i32> %s0, <8 x i32> %s1, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
-; NEON-NEXT:       [[TMP5:%.*]] = shufflevector <8 x i32> %s0, <8 x i32> %s1, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
-; NEON-NEXT:       call void @llvm.arm.neon.vst4.p0i8.v4i32(i8* [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <4 x i32> [[TMP5]], i32 4)
-; NEON-NEXT:       ret void
-; NO_NEON-LABEL: @store_undef_mask_factor4(
-; NO_NEON-NOT:     @llvm.arm.neon
-; NO_NEON:         ret void
+; CHECK-NEON-LABEL: @store_undef_mask_factor4(
+; CHECK-NEON-NEXT:    [[S0:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEON-NEXT:    [[S1:%.*]] = shufflevector <4 x i32> [[V2:%.*]], <4 x i32> [[V3:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEON-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[S0]], <8 x i32> [[S1]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEON-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[S0]], <8 x i32> [[S1]], <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEON-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[S0]], <8 x i32> [[S1]], <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEON-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i32> [[S0]], <8 x i32> [[S1]], <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEON-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32>* [[PTR:%.*]] to i8*
+; CHECK-NEON-NEXT:    call void @llvm.arm.neon.vst4.p0i8.v4i32(i8* [[TMP5]], <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], i32 4)
+; CHECK-NEON-NEXT:    ret void
+;
+; CHECK-MVE-LABEL: @store_undef_mask_factor4(
+; CHECK-MVE-NEXT:    [[S0:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-MVE-NEXT:    [[S1:%.*]] = shufflevector <4 x i32> [[V2:%.*]], <4 x i32> [[V3:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i32> [[S0]], <8 x i32> [[S1]], <16 x i32> <i32 0, i32 4, i32 8, i32 undef, i32 undef, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
+; CHECK-MVE-NEXT:    store <16 x i32> [[INTERLEAVED_VEC]], <16 x i32>* [[PTR:%.*]], align 4
+; CHECK-MVE-NEXT:    ret void
+;
+; CHECK-NONE-LABEL: @store_undef_mask_factor4(
+; CHECK-NONE-NEXT:    [[S0:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NONE-NEXT:    [[S1:%.*]] = shufflevector <4 x i32> [[V2:%.*]], <4 x i32> [[V3:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i32> [[S0]], <8 x i32> [[S1]], <16 x i32> <i32 0, i32 4, i32 8, i32 undef, i32 undef, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
+; CHECK-NONE-NEXT:    store <16 x i32> [[INTERLEAVED_VEC]], <16 x i32>* [[PTR:%.*]], align 4
+; CHECK-NONE-NEXT:    ret void
 ;
   %s0 = shufflevector <4 x i32> %v0, <4 x i32> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   %s1 = shufflevector <4 x i32> %v2, <4 x i32> %v3, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -353,16 +555,27 @@ define void @store_undef_mask_factor4(<16 x i32>* %ptr, <4 x i32> %v0, <4 x i32>
 }
 
 define void @load_address_space(<8 x i32> addrspace(1)* %ptr) {
-; NEON-LABEL:    @load_address_space(
-; NEON-NEXT:       [[TMP1:%.*]] = bitcast <8 x i32> addrspace(1)* %ptr to i8 addrspace(1)*
-; NEON-NEXT:       [[VLDN:%.*]] = call { <2 x i32>, <2 x i32>, <2 x i32> } @llvm.arm.neon.vld3.v2i32.p1i8(i8 addrspace(1)* [[TMP1]], i32 0)
-; NEON-NEXT:       [[TMP2:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[VLDN]], 2
-; NEON-NEXT:       [[TMP3:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[VLDN]], 1
-; NEON-NEXT:       [[TMP4:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[VLDN]], 0
-; NEON-NEXT:       ret void
-; NO_NEON-LABEL: @load_address_space(
-; NO_NEON-NOT:     @llvm.arm.neon
-; NO_NEON:         ret void
+; CHECK-NEON-LABEL: @load_address_space(
+; CHECK-NEON-NEXT:    [[TMP1:%.*]] = bitcast <8 x i32> addrspace(1)* [[PTR:%.*]] to i8 addrspace(1)*
+; CHECK-NEON-NEXT:    [[VLDN:%.*]] = call { <2 x i32>, <2 x i32>, <2 x i32> } @llvm.arm.neon.vld3.v2i32.p1i8(i8 addrspace(1)* [[TMP1]], i32 0)
+; CHECK-NEON-NEXT:    [[TMP2:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[VLDN]], 2
+; CHECK-NEON-NEXT:    [[TMP3:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[VLDN]], 1
+; CHECK-NEON-NEXT:    [[TMP4:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[VLDN]], 0
+; CHECK-NEON-NEXT:    ret void
+;
+; CHECK-MVE-LABEL: @load_address_space(
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = load <8 x i32>, <8 x i32> addrspace(1)* [[PTR:%.*]]
+; CHECK-MVE-NEXT:    [[V0:%.*]] = shufflevector <8 x i32> [[INTERLEAVED_VEC]], <8 x i32> undef, <2 x i32> <i32 0, i32 3>
+; CHECK-MVE-NEXT:    [[V1:%.*]] = shufflevector <8 x i32> [[INTERLEAVED_VEC]], <8 x i32> undef, <2 x i32> <i32 1, i32 4>
+; CHECK-MVE-NEXT:    [[V2:%.*]] = shufflevector <8 x i32> [[INTERLEAVED_VEC]], <8 x i32> undef, <2 x i32> <i32 2, i32 5>
+; CHECK-MVE-NEXT:    ret void
+;
+; CHECK-NONE-LABEL: @load_address_space(
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = load <8 x i32>, <8 x i32> addrspace(1)* [[PTR:%.*]]
+; CHECK-NONE-NEXT:    [[V0:%.*]] = shufflevector <8 x i32> [[INTERLEAVED_VEC]], <8 x i32> undef, <2 x i32> <i32 0, i32 3>
+; CHECK-NONE-NEXT:    [[V1:%.*]] = shufflevector <8 x i32> [[INTERLEAVED_VEC]], <8 x i32> undef, <2 x i32> <i32 1, i32 4>
+; CHECK-NONE-NEXT:    [[V2:%.*]] = shufflevector <8 x i32> [[INTERLEAVED_VEC]], <8 x i32> undef, <2 x i32> <i32 2, i32 5>
+; CHECK-NONE-NEXT:    ret void
 ;
   %interleaved.vec = load <8 x i32>, <8 x i32> addrspace(1)* %ptr
   %v0 = shufflevector <8 x i32> %interleaved.vec, <8 x i32> undef, <2 x i32> <i32 0, i32 3>
@@ -372,15 +585,22 @@ define void @load_address_space(<8 x i32> addrspace(1)* %ptr) {
 }
 
 define void @store_address_space(<4 x i32> addrspace(1)* %ptr, <2 x i32> %v0, <2 x i32> %v1) {
-; NEON-LABEL:    @store_address_space(
-; NEON-NEXT:       [[TMP1:%.*]] = bitcast <4 x i32> addrspace(1)* %ptr to i8 addrspace(1)*
-; NEON-NEXT:       [[TMP2:%.*]] = shufflevector <2 x i32> %v0, <2 x i32> %v1, <2 x i32> <i32 0, i32 1>
-; NEON-NEXT:       [[TMP3:%.*]] = shufflevector <2 x i32> %v0, <2 x i32> %v1, <2 x i32> <i32 2, i32 3>
-; NEON-NEXT:       call void @llvm.arm.neon.vst2.p1i8.v2i32(i8 addrspace(1)* [[TMP1]], <2 x i32> [[TMP2]], <2 x i32> [[TMP3]], i32 0)
-; NEON-NEXT:       ret void
-; NO_NEON-LABEL: @store_address_space(
-; NO_NEON-NOT:     @llvm.arm.neon
-; NO_NEON:         ret void
+; CHECK-NEON-LABEL: @store_address_space(
+; CHECK-NEON-NEXT:    [[TMP1:%.*]] = shufflevector <2 x i32> [[V0:%.*]], <2 x i32> [[V1:%.*]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEON-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i32> [[V0]], <2 x i32> [[V1]], <2 x i32> <i32 2, i32 3>
+; CHECK-NEON-NEXT:    [[TMP3:%.*]] = bitcast <4 x i32> addrspace(1)* [[PTR:%.*]] to i8 addrspace(1)*
+; CHECK-NEON-NEXT:    call void @llvm.arm.neon.vst2.p1i8.v2i32(i8 addrspace(1)* [[TMP3]], <2 x i32> [[TMP1]], <2 x i32> [[TMP2]], i32 0)
+; CHECK-NEON-NEXT:    ret void
+;
+; CHECK-MVE-LABEL: @store_address_space(
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <2 x i32> [[V0:%.*]], <2 x i32> [[V1:%.*]], <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; CHECK-MVE-NEXT:    store <4 x i32> [[INTERLEAVED_VEC]], <4 x i32> addrspace(1)* [[PTR:%.*]]
+; CHECK-MVE-NEXT:    ret void
+;
+; CHECK-NONE-LABEL: @store_address_space(
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <2 x i32> [[V0:%.*]], <2 x i32> [[V1:%.*]], <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; CHECK-NONE-NEXT:    store <4 x i32> [[INTERLEAVED_VEC]], <4 x i32> addrspace(1)* [[PTR:%.*]]
+; CHECK-NONE-NEXT:    ret void
 ;
   %interleaved.vec = shufflevector <2 x i32> %v0, <2 x i32> %v1, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
   store <4 x i32> %interleaved.vec, <4 x i32> addrspace(1)* %ptr
@@ -388,9 +608,23 @@ define void @store_address_space(<4 x i32> addrspace(1)* %ptr, <2 x i32> %v0, <2
 }
 
 define void @load_f16_factor2(<8 x half>* %ptr) {
-; ALL-LABEL: @load_f16_factor2(
-; ALL-NOT:     @llvm.arm.neon
-; ALL:         ret void
+; CHECK-NEON-LABEL: @load_f16_factor2(
+; CHECK-NEON-NEXT:    [[INTERLEAVED_VEC:%.*]] = load <8 x half>, <8 x half>* [[PTR:%.*]], align 4
+; CHECK-NEON-NEXT:    [[V0:%.*]] = shufflevector <8 x half> [[INTERLEAVED_VEC]], <8 x half> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEON-NEXT:    [[V1:%.*]] = shufflevector <8 x half> [[INTERLEAVED_VEC]], <8 x half> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEON-NEXT:    ret void
+;
+; CHECK-MVE-LABEL: @load_f16_factor2(
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = load <8 x half>, <8 x half>* [[PTR:%.*]], align 4
+; CHECK-MVE-NEXT:    [[V0:%.*]] = shufflevector <8 x half> [[INTERLEAVED_VEC]], <8 x half> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-MVE-NEXT:    [[V1:%.*]] = shufflevector <8 x half> [[INTERLEAVED_VEC]], <8 x half> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-MVE-NEXT:    ret void
+;
+; CHECK-NONE-LABEL: @load_f16_factor2(
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = load <8 x half>, <8 x half>* [[PTR:%.*]], align 4
+; CHECK-NONE-NEXT:    [[V0:%.*]] = shufflevector <8 x half> [[INTERLEAVED_VEC]], <8 x half> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NONE-NEXT:    [[V1:%.*]] = shufflevector <8 x half> [[INTERLEAVED_VEC]], <8 x half> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NONE-NEXT:    ret void
 ;
   %interleaved.vec = load <8 x half>, <8 x half>* %ptr, align 4
   %v0 = shufflevector <8 x half> %interleaved.vec, <8 x half> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
@@ -399,9 +633,20 @@ define void @load_f16_factor2(<8 x half>* %ptr) {
 }
 
 define void @store_f16_factor2(<8 x half>* %ptr, <4 x half> %v0, <4 x half> %v1) {
-; ALL-LABEL: @store_f16_factor2(
-; ALL-NOT:     @llvm.arm.neon
-; ALL:         ret void
+; CHECK-NEON-LABEL: @store_f16_factor2(
+; CHECK-NEON-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x half> [[V0:%.*]], <4 x half> [[V1:%.*]], <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; CHECK-NEON-NEXT:    store <8 x half> [[INTERLEAVED_VEC]], <8 x half>* [[PTR:%.*]], align 4
+; CHECK-NEON-NEXT:    ret void
+;
+; CHECK-MVE-LABEL: @store_f16_factor2(
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x half> [[V0:%.*]], <4 x half> [[V1:%.*]], <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; CHECK-MVE-NEXT:    store <8 x half> [[INTERLEAVED_VEC]], <8 x half>* [[PTR:%.*]], align 4
+; CHECK-MVE-NEXT:    ret void
+;
+; CHECK-NONE-LABEL: @store_f16_factor2(
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x half> [[V0:%.*]], <4 x half> [[V1:%.*]], <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; CHECK-NONE-NEXT:    store <8 x half> [[INTERLEAVED_VEC]], <8 x half>* [[PTR:%.*]], align 4
+; CHECK-NONE-NEXT:    ret void
 ;
   %interleaved.vec = shufflevector <4 x half> %v0, <4 x half> %v1, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
   store <8 x half> %interleaved.vec, <8 x half>* %ptr, align 4
@@ -409,9 +654,20 @@ define void @store_f16_factor2(<8 x half>* %ptr, <4 x half> %v0, <4 x half> %v1)
 }
 
 define void @load_illegal_factor2(<3 x float>* %ptr) nounwind {
-; ALL-LABEL:    @load_illegal_factor2(
-; ALL-NOT:        @llvm.arm.neon
-; ALL:            ret void
+; CHECK-NEON-LABEL: @load_illegal_factor2(
+; CHECK-NEON-NEXT:    [[INTERLEAVED_VEC:%.*]] = load <3 x float>, <3 x float>* [[PTR:%.*]], align 16
+; CHECK-NEON-NEXT:    [[V0:%.*]] = shufflevector <3 x float> [[INTERLEAVED_VEC]], <3 x float> undef, <3 x i32> <i32 0, i32 2, i32 undef>
+; CHECK-NEON-NEXT:    ret void
+;
+; CHECK-MVE-LABEL: @load_illegal_factor2(
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = load <3 x float>, <3 x float>* [[PTR:%.*]], align 16
+; CHECK-MVE-NEXT:    [[V0:%.*]] = shufflevector <3 x float> [[INTERLEAVED_VEC]], <3 x float> undef, <3 x i32> <i32 0, i32 2, i32 undef>
+; CHECK-MVE-NEXT:    ret void
+;
+; CHECK-NONE-LABEL: @load_illegal_factor2(
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = load <3 x float>, <3 x float>* [[PTR:%.*]], align 16
+; CHECK-NONE-NEXT:    [[V0:%.*]] = shufflevector <3 x float> [[INTERLEAVED_VEC]], <3 x float> undef, <3 x i32> <i32 0, i32 2, i32 undef>
+; CHECK-NONE-NEXT:    ret void
 ;
   %interleaved.vec = load <3 x float>, <3 x float>* %ptr, align 16
   %v0 = shufflevector <3 x float> %interleaved.vec, <3 x float> undef, <3 x i32> <i32 0, i32 2, i32 undef>
@@ -419,9 +675,20 @@ define void @load_illegal_factor2(<3 x float>* %ptr) nounwind {
 }
 
 define void @store_illegal_factor2(<3 x float>* %ptr, <3 x float> %v0) nounwind {
-; ALL-LABEL: @store_illegal_factor2(
-; ALL-NOT:     @llvm.arm.neon
-; ALL:         ret void
+; CHECK-NEON-LABEL: @store_illegal_factor2(
+; CHECK-NEON-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <3 x float> [[V0:%.*]], <3 x float> undef, <3 x i32> <i32 0, i32 2, i32 undef>
+; CHECK-NEON-NEXT:    store <3 x float> [[INTERLEAVED_VEC]], <3 x float>* [[PTR:%.*]], align 16
+; CHECK-NEON-NEXT:    ret void
+;
+; CHECK-MVE-LABEL: @store_illegal_factor2(
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <3 x float> [[V0:%.*]], <3 x float> undef, <3 x i32> <i32 0, i32 2, i32 undef>
+; CHECK-MVE-NEXT:    store <3 x float> [[INTERLEAVED_VEC]], <3 x float>* [[PTR:%.*]], align 16
+; CHECK-MVE-NEXT:    ret void
+;
+; CHECK-NONE-LABEL: @store_illegal_factor2(
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <3 x float> [[V0:%.*]], <3 x float> undef, <3 x i32> <i32 0, i32 2, i32 undef>
+; CHECK-NONE-NEXT:    store <3 x float> [[INTERLEAVED_VEC]], <3 x float>* [[PTR:%.*]], align 16
+; CHECK-NONE-NEXT:    ret void
 ;
   %interleaved.vec = shufflevector <3 x float> %v0, <3 x float> undef, <3 x i32> <i32 0, i32 2, i32 undef>
   store <3 x float> %interleaved.vec, <3 x float>* %ptr, align 16
@@ -429,17 +696,24 @@ define void @store_illegal_factor2(<3 x float>* %ptr, <3 x float> %v0) nounwind
 }
 
 define void @store_general_mask_factor4(<8 x i32>* %ptr, <32 x i32> %v0, <32 x i32> %v1) {
-; NEON-LABEL:    @store_general_mask_factor4(
-; NEON-NEXT:       [[TMP1:%.*]] = bitcast <8 x i32>* %ptr to i8*
-; NEON-NEXT:       [[TMP2:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 4, i32 5>
-; NEON-NEXT:       [[TMP3:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 16, i32 17>
-; NEON-NEXT:       [[TMP4:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 32, i32 33>
-; NEON-NEXT:       [[TMP5:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 8, i32 9>
-; NEON-NEXT:       call void @llvm.arm.neon.vst4.p0i8.v2i32(i8* [[TMP1]], <2 x i32> [[TMP2]], <2 x i32> [[TMP3]], <2 x i32> [[TMP4]], <2 x i32> [[TMP5]], i32 4)
-; NEON-NEXT:       ret void
-; NO_NEON-LABEL: @store_general_mask_factor4(
-; NO_NEON-NOT:     @llvm.arm.neon
-; NO_NEON:         ret void
+; CHECK-NEON-LABEL: @store_general_mask_factor4(
+; CHECK-NEON-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <2 x i32> <i32 4, i32 5>
+; CHECK-NEON-NEXT:    [[TMP2:%.*]] = shufflevector <32 x i32> [[V0]], <32 x i32> [[V1]], <2 x i32> <i32 16, i32 17>
+; CHECK-NEON-NEXT:    [[TMP3:%.*]] = shufflevector <32 x i32> [[V0]], <32 x i32> [[V1]], <2 x i32> <i32 32, i32 33>
+; CHECK-NEON-NEXT:    [[TMP4:%.*]] = shufflevector <32 x i32> [[V0]], <32 x i32> [[V1]], <2 x i32> <i32 8, i32 9>
+; CHECK-NEON-NEXT:    [[TMP5:%.*]] = bitcast <8 x i32>* [[PTR:%.*]] to i8*
+; CHECK-NEON-NEXT:    call void @llvm.arm.neon.vst4.p0i8.v2i32(i8* [[TMP5]], <2 x i32> [[TMP1]], <2 x i32> [[TMP2]], <2 x i32> [[TMP3]], <2 x i32> [[TMP4]], i32 4)
+; CHECK-NEON-NEXT:    ret void
+;
+; CHECK-MVE-LABEL: @store_general_mask_factor4(
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <8 x i32> <i32 4, i32 16, i32 32, i32 8, i32 5, i32 17, i32 33, i32 9>
+; CHECK-MVE-NEXT:    store <8 x i32> [[INTERLEAVED_VEC]], <8 x i32>* [[PTR:%.*]], align 4
+; CHECK-MVE-NEXT:    ret void
+;
+; CHECK-NONE-LABEL: @store_general_mask_factor4(
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <8 x i32> <i32 4, i32 16, i32 32, i32 8, i32 5, i32 17, i32 33, i32 9>
+; CHECK-NONE-NEXT:    store <8 x i32> [[INTERLEAVED_VEC]], <8 x i32>* [[PTR:%.*]], align 4
+; CHECK-NONE-NEXT:    ret void
 ;
   %interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <8 x i32> <i32 4, i32 16, i32 32, i32 8, i32 5, i32 17, i32 33, i32 9>
   store <8 x i32> %interleaved.vec, <8 x i32>* %ptr, align 4
@@ -447,17 +721,24 @@ define void @store_general_mask_factor4(<8 x i32>* %ptr, <32 x i32> %v0, <32 x i
 }
 
 define void @store_general_mask_factor4_undefbeg(<8 x i32>* %ptr, <32 x i32> %v0, <32 x i32> %v1) {
-; NEON-LABEL:    @store_general_mask_factor4_undefbeg(
-; NEON-NEXT:       [[TMP1:%.*]] = bitcast <8 x i32>* %ptr to i8*
-; NEON-NEXT:       [[TMP2:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 4, i32 5>
-; NEON-NEXT:       [[TMP3:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 16, i32 17>
-; NEON-NEXT:       [[TMP4:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 32, i32 33>
-; NEON-NEXT:       [[TMP5:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 8, i32 9>
-; NEON-NEXT:       call void @llvm.arm.neon.vst4.p0i8.v2i32(i8* [[TMP1]], <2 x i32> [[TMP2]], <2 x i32> [[TMP3]], <2 x i32> [[TMP4]], <2 x i32> [[TMP5]], i32 4)
-; NEON-NEXT:       ret void
-; NO_NEON-LABEL: @store_general_mask_factor4_undefbeg(
-; NO_NEON-NOT:     @llvm.arm.neon
-; NO_NEON:         ret void
+; CHECK-NEON-LABEL: @store_general_mask_factor4_undefbeg(
+; CHECK-NEON-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <2 x i32> <i32 4, i32 5>
+; CHECK-NEON-NEXT:    [[TMP2:%.*]] = shufflevector <32 x i32> [[V0]], <32 x i32> [[V1]], <2 x i32> <i32 16, i32 17>
+; CHECK-NEON-NEXT:    [[TMP3:%.*]] = shufflevector <32 x i32> [[V0]], <32 x i32> [[V1]], <2 x i32> <i32 32, i32 33>
+; CHECK-NEON-NEXT:    [[TMP4:%.*]] = shufflevector <32 x i32> [[V0]], <32 x i32> [[V1]], <2 x i32> <i32 8, i32 9>
+; CHECK-NEON-NEXT:    [[TMP5:%.*]] = bitcast <8 x i32>* [[PTR:%.*]] to i8*
+; CHECK-NEON-NEXT:    call void @llvm.arm.neon.vst4.p0i8.v2i32(i8* [[TMP5]], <2 x i32> [[TMP1]], <2 x i32> [[TMP2]], <2 x i32> [[TMP3]], <2 x i32> [[TMP4]], i32 4)
+; CHECK-NEON-NEXT:    ret void
+;
+; CHECK-MVE-LABEL: @store_general_mask_factor4_undefbeg(
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <8 x i32> <i32 undef, i32 16, i32 32, i32 8, i32 5, i32 17, i32 33, i32 9>
+; CHECK-MVE-NEXT:    store <8 x i32> [[INTERLEAVED_VEC]], <8 x i32>* [[PTR:%.*]], align 4
+; CHECK-MVE-NEXT:    ret void
+;
+; CHECK-NONE-LABEL: @store_general_mask_factor4_undefbeg(
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <8 x i32> <i32 undef, i32 16, i32 32, i32 8, i32 5, i32 17, i32 33, i32 9>
+; CHECK-NONE-NEXT:    store <8 x i32> [[INTERLEAVED_VEC]], <8 x i32>* [[PTR:%.*]], align 4
+; CHECK-NONE-NEXT:    ret void
 ;
   %interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <8 x i32> <i32 undef, i32 16, i32 32, i32 8, i32 5, i32 17, i32 33, i32 9>
   store <8 x i32> %interleaved.vec, <8 x i32>* %ptr, align 4
@@ -465,17 +746,24 @@ define void @store_general_mask_factor4_undefbeg(<8 x i32>* %ptr, <32 x i32> %v0
 }
 
 define void @store_general_mask_factor4_undefend(<8 x i32>* %ptr, <32 x i32> %v0, <32 x i32> %v1) {
-; NEON-LABEL:    @store_general_mask_factor4_undefend(
-; NEON-NEXT:       [[TMP1:%.*]] = bitcast <8 x i32>* %ptr to i8*
-; NEON-NEXT:       [[TMP2:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 4, i32 5>
-; NEON-NEXT:       [[TMP3:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 16, i32 17>
-; NEON-NEXT:       [[TMP4:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 32, i32 33>
-; NEON-NEXT:       [[TMP5:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 8, i32 9>
-; NEON-NEXT:       call void @llvm.arm.neon.vst4.p0i8.v2i32(i8* [[TMP1]], <2 x i32> [[TMP2]], <2 x i32> [[TMP3]], <2 x i32> [[TMP4]], <2 x i32> [[TMP5]], i32 4)
-; NEON-NEXT:       ret void
-; NO_NEON-LABEL: @store_general_mask_factor4_undefend(
-; NO_NEON-NOT:     @llvm.arm.neon
-; NO_NEON:         ret void
+; CHECK-NEON-LABEL: @store_general_mask_factor4_undefend(
+; CHECK-NEON-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <2 x i32> <i32 4, i32 5>
+; CHECK-NEON-NEXT:    [[TMP2:%.*]] = shufflevector <32 x i32> [[V0]], <32 x i32> [[V1]], <2 x i32> <i32 16, i32 17>
+; CHECK-NEON-NEXT:    [[TMP3:%.*]] = shufflevector <32 x i32> [[V0]], <32 x i32> [[V1]], <2 x i32> <i32 32, i32 33>
+; CHECK-NEON-NEXT:    [[TMP4:%.*]] = shufflevector <32 x i32> [[V0]], <32 x i32> [[V1]], <2 x i32> <i32 8, i32 9>
+; CHECK-NEON-NEXT:    [[TMP5:%.*]] = bitcast <8 x i32>* [[PTR:%.*]] to i8*
+; CHECK-NEON-NEXT:    call void @llvm.arm.neon.vst4.p0i8.v2i32(i8* [[TMP5]], <2 x i32> [[TMP1]], <2 x i32> [[TMP2]], <2 x i32> [[TMP3]], <2 x i32> [[TMP4]], i32 4)
+; CHECK-NEON-NEXT:    ret void
+;
+; CHECK-MVE-LABEL: @store_general_mask_factor4_undefend(
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <8 x i32> <i32 4, i32 16, i32 32, i32 8, i32 5, i32 17, i32 33, i32 undef>
+; CHECK-MVE-NEXT:    store <8 x i32> [[INTERLEAVED_VEC]], <8 x i32>* [[PTR:%.*]], align 4
+; CHECK-MVE-NEXT:    ret void
+;
+; CHECK-NONE-LABEL: @store_general_mask_factor4_undefend(
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <8 x i32> <i32 4, i32 16, i32 32, i32 8, i32 5, i32 17, i32 33, i32 undef>
+; CHECK-NONE-NEXT:    store <8 x i32> [[INTERLEAVED_VEC]], <8 x i32>* [[PTR:%.*]], align 4
+; CHECK-NONE-NEXT:    ret void
 ;
   %interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <8 x i32> <i32 4, i32 16, i32 32, i32 8, i32 5, i32 17, i32 33, i32 undef>
   store <8 x i32> %interleaved.vec, <8 x i32>* %ptr, align 4
@@ -483,17 +771,24 @@ define void @store_general_mask_factor4_undefend(<8 x i32>* %ptr, <32 x i32> %v0
 }
 
 define void @store_general_mask_factor4_undefmid(<8 x i32>* %ptr, <32 x i32> %v0, <32 x i32> %v1) {
-; NEON-LABEL:    @store_general_mask_factor4_undefmid(
-; NEON-NEXT:       [[TMP1:%.*]] = bitcast <8 x i32>* %ptr to i8*
-; NEON-NEXT:       [[TMP2:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 4, i32 5>
-; NEON-NEXT:       [[TMP3:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 16, i32 17>
-; NEON-NEXT:       [[TMP4:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 32, i32 33>
-; NEON-NEXT:       [[TMP5:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 8, i32 9>
-; NEON-NEXT:       call void @llvm.arm.neon.vst4.p0i8.v2i32(i8* [[TMP1]], <2 x i32> [[TMP2]], <2 x i32> [[TMP3]], <2 x i32> [[TMP4]], <2 x i32> [[TMP5]], i32 4)
-; NEON-NEXT:       ret void
-; NO_NEON-LABEL: @store_general_mask_factor4_undefmid(
-; NO_NEON-NOT:     @llvm.arm.neon
-; NO_NEON:         ret void
+; CHECK-NEON-LABEL: @store_general_mask_factor4_undefmid(
+; CHECK-NEON-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <2 x i32> <i32 4, i32 5>
+; CHECK-NEON-NEXT:    [[TMP2:%.*]] = shufflevector <32 x i32> [[V0]], <32 x i32> [[V1]], <2 x i32> <i32 16, i32 17>
+; CHECK-NEON-NEXT:    [[TMP3:%.*]] = shufflevector <32 x i32> [[V0]], <32 x i32> [[V1]], <2 x i32> <i32 32, i32 33>
+; CHECK-NEON-NEXT:    [[TMP4:%.*]] = shufflevector <32 x i32> [[V0]], <32 x i32> [[V1]], <2 x i32> <i32 8, i32 9>
+; CHECK-NEON-NEXT:    [[TMP5:%.*]] = bitcast <8 x i32>* [[PTR:%.*]] to i8*
+; CHECK-NEON-NEXT:    call void @llvm.arm.neon.vst4.p0i8.v2i32(i8* [[TMP5]], <2 x i32> [[TMP1]], <2 x i32> [[TMP2]], <2 x i32> [[TMP3]], <2 x i32> [[TMP4]], i32 4)
+; CHECK-NEON-NEXT:    ret void
+;
+; CHECK-MVE-LABEL: @store_general_mask_factor4_undefmid(
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <8 x i32> <i32 4, i32 undef, i32 32, i32 8, i32 5, i32 17, i32 undef, i32 9>
+; CHECK-MVE-NEXT:    store <8 x i32> [[INTERLEAVED_VEC]], <8 x i32>* [[PTR:%.*]], align 4
+; CHECK-MVE-NEXT:    ret void
+;
+; CHECK-NONE-LABEL: @store_general_mask_factor4_undefmid(
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <8 x i32> <i32 4, i32 undef, i32 32, i32 8, i32 5, i32 17, i32 undef, i32 9>
+; CHECK-NONE-NEXT:    store <8 x i32> [[INTERLEAVED_VEC]], <8 x i32>* [[PTR:%.*]], align 4
+; CHECK-NONE-NEXT:    ret void
 ;
   %interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <8 x i32> <i32 4, i32 undef, i32 32, i32 8, i32 5, i32 17, i32 undef, i32 9>
   store <8 x i32> %interleaved.vec, <8 x i32>* %ptr, align 4
@@ -501,17 +796,24 @@ define void @store_general_mask_factor4_undefmid(<8 x i32>* %ptr, <32 x i32> %v0
 }
 
 define void @store_general_mask_factor4_undefmulti(<8 x i32>* %ptr, <32 x i32> %v0, <32 x i32> %v1) {
-; NEON-LABEL:    @store_general_mask_factor4_undefmulti(
-; NEON-NEXT:       [[TMP1:%.*]] = bitcast <8 x i32>* %ptr to i8*
-; NEON-NEXT:       [[TMP2:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 4, i32 5>
-; NEON-NEXT:       [[TMP3:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 0, i32 1>
-; NEON-NEXT:       [[TMP4:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 0, i32 1>
-; NEON-NEXT:       [[TMP5:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 8, i32 9>
-; NEON-NEXT:       call void @llvm.arm.neon.vst4.p0i8.v2i32(i8* [[TMP1]], <2 x i32> [[TMP2]], <2 x i32> [[TMP3]], <2 x i32> [[TMP4]], <2 x i32> [[TMP5]], i32 4)
-; NEON-NEXT:       ret void
-; NO_NEON-LABEL: @store_general_mask_factor4_undefmulti(
-; NO_NEON-NOT:     @llvm.arm.neon
-; NO_NEON:         ret void
+; CHECK-NEON-LABEL: @store_general_mask_factor4_undefmulti(
+; CHECK-NEON-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <2 x i32> <i32 4, i32 5>
+; CHECK-NEON-NEXT:    [[TMP2:%.*]] = shufflevector <32 x i32> [[V0]], <32 x i32> [[V1]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEON-NEXT:    [[TMP3:%.*]] = shufflevector <32 x i32> [[V0]], <32 x i32> [[V1]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEON-NEXT:    [[TMP4:%.*]] = shufflevector <32 x i32> [[V0]], <32 x i32> [[V1]], <2 x i32> <i32 8, i32 9>
+; CHECK-NEON-NEXT:    [[TMP5:%.*]] = bitcast <8 x i32>* [[PTR:%.*]] to i8*
+; CHECK-NEON-NEXT:    call void @llvm.arm.neon.vst4.p0i8.v2i32(i8* [[TMP5]], <2 x i32> [[TMP1]], <2 x i32> [[TMP2]], <2 x i32> [[TMP3]], <2 x i32> [[TMP4]], i32 4)
+; CHECK-NEON-NEXT:    ret void
+;
+; CHECK-MVE-LABEL: @store_general_mask_factor4_undefmulti(
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <8 x i32> <i32 4, i32 undef, i32 undef, i32 8, i32 undef, i32 undef, i32 undef, i32 9>
+; CHECK-MVE-NEXT:    store <8 x i32> [[INTERLEAVED_VEC]], <8 x i32>* [[PTR:%.*]], align 4
+; CHECK-MVE-NEXT:    ret void
+;
+; CHECK-NONE-LABEL: @store_general_mask_factor4_undefmulti(
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <8 x i32> <i32 4, i32 undef, i32 undef, i32 8, i32 undef, i32 undef, i32 undef, i32 9>
+; CHECK-NONE-NEXT:    store <8 x i32> [[INTERLEAVED_VEC]], <8 x i32>* [[PTR:%.*]], align 4
+; CHECK-NONE-NEXT:    ret void
 ;
   %interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <8 x i32> <i32 4, i32 undef, i32 undef, i32 8, i32 undef, i32 undef, i32 undef, i32 9>
   store <8 x i32> %interleaved.vec, <8 x i32>* %ptr, align 4
@@ -519,16 +821,23 @@ define void @store_general_mask_factor4_undefmulti(<8 x i32>* %ptr, <32 x i32> %
 }
 
 define void @store_general_mask_factor3(<12 x i32>* %ptr, <32 x i32> %v0, <32 x i32> %v1) {
-; NEON-LABEL:    @store_general_mask_factor3(
-; NEON-NEXT:       [[TMP1:%.*]] = bitcast <12 x i32>* %ptr to i8*
-; NEON-NEXT:       [[TMP2:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; NEON-NEXT:       [[TMP3:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <4 x i32> <i32 32, i32 33, i32 34, i32 35>
-; NEON-NEXT:       [[TMP4:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
-; NEON-NEXT:       call void @llvm.arm.neon.vst3.p0i8.v4i32(i8* [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], i32 4)
-; NEON-NEXT:       ret void
-; NO_NEON-LABEL: @store_general_mask_factor3(
-; NO_NEON-NOT:     @llvm.arm.neon
-; NO_NEON:         ret void
+; CHECK-NEON-LABEL: @store_general_mask_factor3(
+; CHECK-NEON-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEON-NEXT:    [[TMP2:%.*]] = shufflevector <32 x i32> [[V0]], <32 x i32> [[V1]], <4 x i32> <i32 32, i32 33, i32 34, i32 35>
+; CHECK-NEON-NEXT:    [[TMP3:%.*]] = shufflevector <32 x i32> [[V0]], <32 x i32> [[V1]], <4 x i32> <i32 16, i32 17, i32 18, i32 19>
+; CHECK-NEON-NEXT:    [[TMP4:%.*]] = bitcast <12 x i32>* [[PTR:%.*]] to i8*
+; CHECK-NEON-NEXT:    call void @llvm.arm.neon.vst3.p0i8.v4i32(i8* [[TMP4]], <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> [[TMP3]], i32 4)
+; CHECK-NEON-NEXT:    ret void
+;
+; CHECK-MVE-LABEL: @store_general_mask_factor3(
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 4, i32 32, i32 16, i32 5, i32 33, i32 17, i32 6, i32 34, i32 18, i32 7, i32 35, i32 19>
+; CHECK-MVE-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], <12 x i32>* [[PTR:%.*]], align 4
+; CHECK-MVE-NEXT:    ret void
+;
+; CHECK-NONE-LABEL: @store_general_mask_factor3(
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 4, i32 32, i32 16, i32 5, i32 33, i32 17, i32 6, i32 34, i32 18, i32 7, i32 35, i32 19>
+; CHECK-NONE-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], <12 x i32>* [[PTR:%.*]], align 4
+; CHECK-NONE-NEXT:    ret void
 ;
   %interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <12 x i32> <i32 4, i32 32, i32 16, i32 5, i32 33, i32 17, i32 6, i32 34, i32 18, i32 7, i32 35, i32 19>
   store <12 x i32> %interleaved.vec, <12 x i32>* %ptr, align 4
@@ -536,16 +845,23 @@ define void @store_general_mask_factor3(<12 x i32>* %ptr, <32 x i32> %v0, <32 x
 }
 
 define void @store_general_mask_factor3_undefmultimid(<12 x i32>* %ptr, <32 x i32> %v0, <32 x i32> %v1) {
-; NEON-LABEL:    @store_general_mask_factor3_undefmultimid(
-; NEON-NEXT:       [[TMP1:%.*]] = bitcast <12 x i32>* %ptr to i8*
-; NEON-NEXT:       [[TMP2:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; NEON-NEXT:       [[TMP3:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <4 x i32> <i32 32, i32 33, i32 34, i32 35>
-; NEON-NEXT:       [[TMP4:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
-; NEON-NEXT:       call void @llvm.arm.neon.vst3.p0i8.v4i32(i8* [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], i32 4)
-; NEON-NEXT:       ret void
-; NO_NEON-LABEL: @store_general_mask_factor3_undefmultimid(
-; NO_NEON-NOT:     @llvm.arm.neon
-; NO_NEON:         ret void
+; CHECK-NEON-LABEL: @store_general_mask_factor3_undefmultimid(
+; CHECK-NEON-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEON-NEXT:    [[TMP2:%.*]] = shufflevector <32 x i32> [[V0]], <32 x i32> [[V1]], <4 x i32> <i32 32, i32 33, i32 34, i32 35>
+; CHECK-NEON-NEXT:    [[TMP3:%.*]] = shufflevector <32 x i32> [[V0]], <32 x i32> [[V1]], <4 x i32> <i32 16, i32 17, i32 18, i32 19>
+; CHECK-NEON-NEXT:    [[TMP4:%.*]] = bitcast <12 x i32>* [[PTR:%.*]] to i8*
+; CHECK-NEON-NEXT:    call void @llvm.arm.neon.vst3.p0i8.v4i32(i8* [[TMP4]], <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> [[TMP3]], i32 4)
+; CHECK-NEON-NEXT:    ret void
+;
+; CHECK-MVE-LABEL: @store_general_mask_factor3_undefmultimid(
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 4, i32 32, i32 16, i32 undef, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 7, i32 35, i32 19>
+; CHECK-MVE-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], <12 x i32>* [[PTR:%.*]], align 4
+; CHECK-MVE-NEXT:    ret void
+;
+; CHECK-NONE-LABEL: @store_general_mask_factor3_undefmultimid(
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 4, i32 32, i32 16, i32 undef, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 7, i32 35, i32 19>
+; CHECK-NONE-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], <12 x i32>* [[PTR:%.*]], align 4
+; CHECK-NONE-NEXT:    ret void
 ;
   %interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <12 x i32> <i32 4, i32 32, i32 16, i32 undef, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 7, i32 35, i32 19>
   store <12 x i32> %interleaved.vec, <12 x i32>* %ptr, align 4
@@ -553,9 +869,20 @@ define void @store_general_mask_factor3_undefmultimid(<12 x i32>* %ptr, <32 x i3
 }
 
 define void @store_general_mask_factor3_undef_fail(<12 x i32>* %ptr, <32 x i32> %v0, <32 x i32> %v1) {
-; ALL-LABEL: @store_general_mask_factor3_undef_fail(
-; ALL-NOT:     @llvm.arm.neon
-; ALL:         ret void
+; CHECK-NEON-LABEL: @store_general_mask_factor3_undef_fail(
+; CHECK-NEON-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 4, i32 32, i32 16, i32 undef, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 8, i32 35, i32 19>
+; CHECK-NEON-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], <12 x i32>* [[PTR:%.*]], align 4
+; CHECK-NEON-NEXT:    ret void
+;
+; CHECK-MVE-LABEL: @store_general_mask_factor3_undef_fail(
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 4, i32 32, i32 16, i32 undef, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 8, i32 35, i32 19>
+; CHECK-MVE-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], <12 x i32>* [[PTR:%.*]], align 4
+; CHECK-MVE-NEXT:    ret void
+;
+; CHECK-NONE-LABEL: @store_general_mask_factor3_undef_fail(
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 4, i32 32, i32 16, i32 undef, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 8, i32 35, i32 19>
+; CHECK-NONE-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], <12 x i32>* [[PTR:%.*]], align 4
+; CHECK-NONE-NEXT:    ret void
 ;
   %interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <12 x i32> <i32 4, i32 32, i32 16, i32 undef, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 8, i32 35, i32 19>
   store <12 x i32> %interleaved.vec, <12 x i32>* %ptr, align 4
@@ -563,16 +890,23 @@ define void @store_general_mask_factor3_undef_fail(<12 x i32>* %ptr, <32 x i32>
 }
 
 define void @store_general_mask_factor3_undeflane(<12 x i32>* %ptr, <32 x i32> %v0, <32 x i32> %v1) {
-; NEON-LABEL:    @store_general_mask_factor3_undeflane(
-; NEON-NEXT:       [[TMP1:%.*]] = bitcast <12 x i32>* %ptr to i8*
-; NEON-NEXT:       [[TMP2:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; NEON-NEXT:       [[TMP3:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <4 x i32> <i32 32, i32 33, i32 34, i32 35>
-; NEON-NEXT:       [[TMP4:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
-; NEON-NEXT:       call void @llvm.arm.neon.vst3.p0i8.v4i32(i8* [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], i32 4)
-; NEON-NEXT:       ret void
-; NO_NEON-LABEL: @store_general_mask_factor3_undeflane(
-; NO_NEON-NOT:     @llvm.arm.neon
-; NO_NEON:         ret void
+; CHECK-NEON-LABEL: @store_general_mask_factor3_undeflane(
+; CHECK-NEON-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEON-NEXT:    [[TMP2:%.*]] = shufflevector <32 x i32> [[V0]], <32 x i32> [[V1]], <4 x i32> <i32 32, i32 33, i32 34, i32 35>
+; CHECK-NEON-NEXT:    [[TMP3:%.*]] = shufflevector <32 x i32> [[V0]], <32 x i32> [[V1]], <4 x i32> <i32 16, i32 17, i32 18, i32 19>
+; CHECK-NEON-NEXT:    [[TMP4:%.*]] = bitcast <12 x i32>* [[PTR:%.*]] to i8*
+; CHECK-NEON-NEXT:    call void @llvm.arm.neon.vst3.p0i8.v4i32(i8* [[TMP4]], <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> [[TMP3]], i32 4)
+; CHECK-NEON-NEXT:    ret void
+;
+; CHECK-MVE-LABEL: @store_general_mask_factor3_undeflane(
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 undef, i32 32, i32 16, i32 undef, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 undef, i32 35, i32 19>
+; CHECK-MVE-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], <12 x i32>* [[PTR:%.*]], align 4
+; CHECK-MVE-NEXT:    ret void
+;
+; CHECK-NONE-LABEL: @store_general_mask_factor3_undeflane(
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 undef, i32 32, i32 16, i32 undef, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 undef, i32 35, i32 19>
+; CHECK-NONE-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], <12 x i32>* [[PTR:%.*]], align 4
+; CHECK-NONE-NEXT:    ret void
 ;
   %interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <12 x i32> <i32 undef, i32 32, i32 16, i32 undef, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 undef, i32 35, i32 19>
   store <12 x i32> %interleaved.vec, <12 x i32>* %ptr, align 4
@@ -580,9 +914,20 @@ define void @store_general_mask_factor3_undeflane(<12 x i32>* %ptr, <32 x i32> %
 }
 
 define void @store_general_mask_factor3_endstart_fail(<12 x i32>* %ptr, <32 x i32> %v0, <32 x i32> %v1) {
-; ALL-LABEL:    @store_general_mask_factor3_endstart_fail(
-; ALL-NOT:        @llvm.arm.neon
-; ALL:            ret void
+; CHECK-NEON-LABEL: @store_general_mask_factor3_endstart_fail(
+; CHECK-NEON-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 undef, i32 32, i32 16, i32 undef, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 2, i32 35, i32 19>
+; CHECK-NEON-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], <12 x i32>* [[PTR:%.*]], align 4
+; CHECK-NEON-NEXT:    ret void
+;
+; CHECK-MVE-LABEL: @store_general_mask_factor3_endstart_fail(
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 undef, i32 32, i32 16, i32 undef, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 2, i32 35, i32 19>
+; CHECK-MVE-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], <12 x i32>* [[PTR:%.*]], align 4
+; CHECK-MVE-NEXT:    ret void
+;
+; CHECK-NONE-LABEL: @store_general_mask_factor3_endstart_fail(
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 undef, i32 32, i32 16, i32 undef, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 2, i32 35, i32 19>
+; CHECK-NONE-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], <12 x i32>* [[PTR:%.*]], align 4
+; CHECK-NONE-NEXT:    ret void
 ;
   %interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <12 x i32> <i32 undef, i32 32, i32 16, i32 undef, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 2, i32 35, i32 19>
   store <12 x i32> %interleaved.vec, <12 x i32>* %ptr, align 4
@@ -590,16 +935,23 @@ define void @store_general_mask_factor3_endstart_fail(<12 x i32>* %ptr, <32 x i3
 }
 
 define void @store_general_mask_factor3_endstart_pass(<12 x i32>* %ptr, <32 x i32> %v0, <32 x i32> %v1) {
-; NEON-LABEL:    @store_general_mask_factor3_endstart_pass(
-; NEON-NEXT:       [[TMP1:%.*]] = bitcast <12 x i32>* %ptr to i8*
-; NEON-NEXT:       [[TMP2:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; NEON-NEXT:       [[TMP3:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <4 x i32> <i32 32, i32 33, i32 34, i32 35>
-; NEON-NEXT:       [[TMP4:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
-; NEON-NEXT:       call void @llvm.arm.neon.vst3.p0i8.v4i32(i8* [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], i32 4)
-; NEON-NEXT:       ret void
-; NO_NEON-LABEL: @store_general_mask_factor3_endstart_pass(
-; NO_NEON-NOT:     @llvm.arm.neon
-; NO_NEON:         ret void
+; CHECK-NEON-LABEL: @store_general_mask_factor3_endstart_pass(
+; CHECK-NEON-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEON-NEXT:    [[TMP2:%.*]] = shufflevector <32 x i32> [[V0]], <32 x i32> [[V1]], <4 x i32> <i32 32, i32 33, i32 34, i32 35>
+; CHECK-NEON-NEXT:    [[TMP3:%.*]] = shufflevector <32 x i32> [[V0]], <32 x i32> [[V1]], <4 x i32> <i32 16, i32 17, i32 18, i32 19>
+; CHECK-NEON-NEXT:    [[TMP4:%.*]] = bitcast <12 x i32>* [[PTR:%.*]] to i8*
+; CHECK-NEON-NEXT:    call void @llvm.arm.neon.vst3.p0i8.v4i32(i8* [[TMP4]], <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> [[TMP3]], i32 4)
+; CHECK-NEON-NEXT:    ret void
+;
+; CHECK-MVE-LABEL: @store_general_mask_factor3_endstart_pass(
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 undef, i32 32, i32 16, i32 undef, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 7, i32 35, i32 19>
+; CHECK-MVE-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], <12 x i32>* [[PTR:%.*]], align 4
+; CHECK-MVE-NEXT:    ret void
+;
+; CHECK-NONE-LABEL: @store_general_mask_factor3_endstart_pass(
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 undef, i32 32, i32 16, i32 undef, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 7, i32 35, i32 19>
+; CHECK-NONE-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], <12 x i32>* [[PTR:%.*]], align 4
+; CHECK-NONE-NEXT:    ret void
 ;
   %interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <12 x i32> <i32 undef, i32 32, i32 16, i32 undef, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 7, i32 35, i32 19>
   store <12 x i32> %interleaved.vec, <12 x i32>* %ptr, align 4
@@ -607,9 +959,20 @@ define void @store_general_mask_factor3_endstart_pass(<12 x i32>* %ptr, <32 x i3
 }
 
 define void @store_general_mask_factor3_midstart_fail(<12 x i32>* %ptr, <32 x i32> %v0, <32 x i32> %v1) {
-; ALL-LABEL:    @store_general_mask_factor3_midstart_fail(
-; ALL-NOT:        @llvm.arm.neon
-; ALL:            ret void
+; CHECK-NEON-LABEL: @store_general_mask_factor3_midstart_fail(
+; CHECK-NEON-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 undef, i32 32, i32 16, i32 0, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 undef, i32 35, i32 19>
+; CHECK-NEON-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], <12 x i32>* [[PTR:%.*]], align 4
+; CHECK-NEON-NEXT:    ret void
+;
+; CHECK-MVE-LABEL: @store_general_mask_factor3_midstart_fail(
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 undef, i32 32, i32 16, i32 0, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 undef, i32 35, i32 19>
+; CHECK-MVE-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], <12 x i32>* [[PTR:%.*]], align 4
+; CHECK-MVE-NEXT:    ret void
+;
+; CHECK-NONE-LABEL: @store_general_mask_factor3_midstart_fail(
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 undef, i32 32, i32 16, i32 0, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 undef, i32 35, i32 19>
+; CHECK-NONE-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], <12 x i32>* [[PTR:%.*]], align 4
+; CHECK-NONE-NEXT:    ret void
 ;
   %interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <12 x i32> <i32 undef, i32 32, i32 16, i32 0, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 undef, i32 35, i32 19>
   store <12 x i32> %interleaved.vec, <12 x i32>* %ptr, align 4
@@ -617,16 +980,23 @@ define void @store_general_mask_factor3_midstart_fail(<12 x i32>* %ptr, <32 x i3
 }
 
 define void @store_general_mask_factor3_midstart_pass(<12 x i32>* %ptr, <32 x i32> %v0, <32 x i32> %v1) {
-; NEON-LABEL:    @store_general_mask_factor3_midstart_pass(
-; NEON-NEXT:       [[TMP1:%.*]] = bitcast <12 x i32>* %ptr to i8*
-; NEON-NEXT:       [[TMP2:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; NEON-NEXT:       [[TMP3:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <4 x i32> <i32 32, i32 33, i32 34, i32 35>
-; NEON-NEXT:       [[TMP4:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
-; NEON-NEXT:       call void @llvm.arm.neon.vst3.p0i8.v4i32(i8* [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], i32 4)
-; NEON-NEXT:       ret void
-; NO_NEON-LABEL: @store_general_mask_factor3_midstart_pass(
-; NO_NEON-NOT:     @llvm.arm.neon
-; NO_NEON:         ret void
+; CHECK-NEON-LABEL: @store_general_mask_factor3_midstart_pass(
+; CHECK-NEON-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEON-NEXT:    [[TMP2:%.*]] = shufflevector <32 x i32> [[V0]], <32 x i32> [[V1]], <4 x i32> <i32 32, i32 33, i32 34, i32 35>
+; CHECK-NEON-NEXT:    [[TMP3:%.*]] = shufflevector <32 x i32> [[V0]], <32 x i32> [[V1]], <4 x i32> <i32 16, i32 17, i32 18, i32 19>
+; CHECK-NEON-NEXT:    [[TMP4:%.*]] = bitcast <12 x i32>* [[PTR:%.*]] to i8*
+; CHECK-NEON-NEXT:    call void @llvm.arm.neon.vst3.p0i8.v4i32(i8* [[TMP4]], <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> [[TMP3]], i32 4)
+; CHECK-NEON-NEXT:    ret void
+;
+; CHECK-MVE-LABEL: @store_general_mask_factor3_midstart_pass(
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 undef, i32 32, i32 16, i32 1, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 undef, i32 35, i32 19>
+; CHECK-MVE-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], <12 x i32>* [[PTR:%.*]], align 4
+; CHECK-MVE-NEXT:    ret void
+;
+; CHECK-NONE-LABEL: @store_general_mask_factor3_midstart_pass(
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 undef, i32 32, i32 16, i32 1, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 undef, i32 35, i32 19>
+; CHECK-NONE-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], <12 x i32>* [[PTR:%.*]], align 4
+; CHECK-NONE-NEXT:    ret void
 ;
   %interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <12 x i32> <i32 undef, i32 32, i32 16, i32 1, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 undef, i32 35, i32 19>
   store <12 x i32> %interleaved.vec, <12 x i32>* %ptr, align 4
@@ -636,35 +1006,61 @@ define void @store_general_mask_factor3_midstart_pass(<12 x i32>* %ptr, <32 x i3
 @g = external global <4 x float>
 
 ; The following does not give a valid interleaved store
-; ALL-LABEL: define void @no_interleave
-; ALL-NOT: call void @llvm.arm.neon.vst2
-; ALL: shufflevector
-; ALL: store
-; ALL: ret void
 define void @no_interleave(<4 x float> %a0) {
+; CHECK-NEON-LABEL: @no_interleave(
+; CHECK-NEON-NEXT:    [[V0:%.*]] = shufflevector <4 x float> [[A0:%.*]], <4 x float> [[A0]], <4 x i32> <i32 0, i32 7, i32 1, i32 undef>
+; CHECK-NEON-NEXT:    store <4 x float> [[V0]], <4 x float>* @g, align 16
+; CHECK-NEON-NEXT:    ret void
+;
+; CHECK-MVE-LABEL: @no_interleave(
+; CHECK-MVE-NEXT:    [[V0:%.*]] = shufflevector <4 x float> [[A0:%.*]], <4 x float> [[A0]], <4 x i32> <i32 0, i32 7, i32 1, i32 undef>
+; CHECK-MVE-NEXT:    store <4 x float> [[V0]], <4 x float>* @g, align 16
+; CHECK-MVE-NEXT:    ret void
+;
+; CHECK-NONE-LABEL: @no_interleave(
+; CHECK-NONE-NEXT:    [[V0:%.*]] = shufflevector <4 x float> [[A0:%.*]], <4 x float> [[A0]], <4 x i32> <i32 0, i32 7, i32 1, i32 undef>
+; CHECK-NONE-NEXT:    store <4 x float> [[V0]], <4 x float>* @g, align 16
+; CHECK-NONE-NEXT:    ret void
+;
   %v0 = shufflevector <4 x float> %a0, <4 x float> %a0, <4 x i32> <i32 0, i32 7, i32 1, i32 undef>
   store <4 x float> %v0, <4 x float>* @g, align 16
   ret void
 }
 
 define void @load_factor2_wide2(<16 x i32>* %ptr) {
-; NEON-LABEL:    @load_factor2_wide2(
-; NEON-NEXT:       [[TMP1:%.*]] = bitcast <16 x i32>* %ptr to i32*
-; NEON-NEXT:       [[TMP2:%.*]] = bitcast i32* [[TMP1]] to i8*
-; NEON-NEXT:       [[VLDN:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.arm.neon.vld2.v4i32.p0i8(i8* [[TMP2]], i32 4)
-; NEON-NEXT:       [[TMP3:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN]], 1
-; NEON-NEXT:       [[TMP4:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN]], 0
-; NEON-NEXT:       [[TMP5:%.*]] = getelementptr i32, i32* [[TMP1]], i32 8
-; NEON-NEXT:       [[TMP6:%.*]] = bitcast i32* [[TMP5]] to i8*
-; NEON-NEXT:       [[VLDN1:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.arm.neon.vld2.v4i32.p0i8(i8* [[TMP6]], i32 4)
-; NEON-NEXT:       [[TMP7:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN1]], 1
-; NEON-NEXT:       [[TMP8:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN1]], 0
-; NEON-NEXT:       [[TMP9:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP7]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; NEON-NEXT:       [[TMP10:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; NEON-NEXT:       ret void
-; NO_NEON-LABEL: @load_factor2_wide2(
-; NO_NEON-NOT:     @llvm.arm.neon
-; NO_NEON:         ret void
+; CHECK-NEON-LABEL: @load_factor2_wide2(
+; CHECK-NEON-NEXT:    [[TMP1:%.*]] = bitcast <16 x i32>* [[PTR:%.*]] to i32*
+; CHECK-NEON-NEXT:    [[TMP2:%.*]] = bitcast i32* [[TMP1]] to i8*
+; CHECK-NEON-NEXT:    [[VLDN:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.arm.neon.vld2.v4i32.p0i8(i8* [[TMP2]], i32 4)
+; CHECK-NEON-NEXT:    [[TMP3:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN]], 1
+; CHECK-NEON-NEXT:    [[TMP4:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN]], 0
+; CHECK-NEON-NEXT:    [[TMP5:%.*]] = getelementptr i32, i32* [[TMP1]], i32 8
+; CHECK-NEON-NEXT:    [[TMP6:%.*]] = bitcast i32* [[TMP5]] to i8*
+; CHECK-NEON-NEXT:    [[VLDN1:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.arm.neon.vld2.v4i32.p0i8(i8* [[TMP6]], i32 4)
+; CHECK-NEON-NEXT:    [[TMP7:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN1]], 1
+; CHECK-NEON-NEXT:    [[TMP8:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN1]], 0
+; CHECK-NEON-NEXT:    [[TMP9:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP7]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEON-NEXT:    [[TMP10:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEON-NEXT:    ret void
+;
+; CHECK-MVE-LABEL: @load_factor2_wide2(
+; CHECK-MVE-NEXT:    [[TMP1:%.*]] = bitcast <16 x i32>* [[PTR:%.*]] to i32*
+; CHECK-MVE-NEXT:    [[VLDN:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.arm.mve.vld2q.v4i32.p0i32(i32* [[TMP1]])
+; CHECK-MVE-NEXT:    [[TMP2:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN]], 1
+; CHECK-MVE-NEXT:    [[TMP3:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN]], 0
+; CHECK-MVE-NEXT:    [[TMP4:%.*]] = getelementptr i32, i32* [[TMP1]], i32 8
+; CHECK-MVE-NEXT:    [[VLDN1:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.arm.mve.vld2q.v4i32.p0i32(i32* [[TMP4]])
+; CHECK-MVE-NEXT:    [[TMP5:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN1]], 1
+; CHECK-MVE-NEXT:    [[TMP6:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN1]], 0
+; CHECK-MVE-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> [[TMP5]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-MVE-NEXT:    [[TMP8:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-MVE-NEXT:    ret void
+;
+; CHECK-NONE-LABEL: @load_factor2_wide2(
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = load <16 x i32>, <16 x i32>* [[PTR:%.*]], align 4
+; CHECK-NONE-NEXT:    [[V0:%.*]] = shufflevector <16 x i32> [[INTERLEAVED_VEC]], <16 x i32> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+; CHECK-NONE-NEXT:    [[V1:%.*]] = shufflevector <16 x i32> [[INTERLEAVED_VEC]], <16 x i32> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+; CHECK-NONE-NEXT:    ret void
 ;
   %interleaved.vec = load <16 x i32>, <16 x i32>* %ptr, align 4
   %v0 = shufflevector <16 x i32> %interleaved.vec, <16 x i32> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
@@ -673,32 +1069,56 @@ define void @load_factor2_wide2(<16 x i32>* %ptr) {
 }
 
 define void @load_factor2_wide3(<24 x i32>* %ptr) {
-; NEON-LABEL:    @load_factor2_wide3(
-; NEON-NEXT:       [[TMP1:%.*]] = bitcast <24 x i32>* [[PTR:%.*]] to i32*
-; NEON-NEXT:       [[TMP2:%.*]] = bitcast i32* [[TMP1]] to i8*
-; NEON-NEXT:       [[VLDN:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.arm.neon.vld2.v4i32.p0i8(i8* [[TMP2]], i32 4)
-; NEON-NEXT:       [[TMP3:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN]], 1
-; NEON-NEXT:       [[TMP4:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN]], 0
-; NEON-NEXT:       [[TMP5:%.*]] = getelementptr i32, i32* [[TMP1]], i32 8
-; NEON-NEXT:       [[TMP6:%.*]] = bitcast i32* [[TMP5]] to i8*
-; NEON-NEXT:       [[VLDN1:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.arm.neon.vld2.v4i32.p0i8(i8* [[TMP6]], i32 4)
-; NEON-NEXT:       [[TMP7:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN1]], 1
-; NEON-NEXT:       [[TMP8:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN1]], 0
-; NEON-NEXT:       [[TMP9:%.*]] = getelementptr i32, i32* [[TMP5]], i32 8
-; NEON-NEXT:       [[TMP10:%.*]] = bitcast i32* [[TMP9]] to i8*
-; NEON-NEXT:       [[VLDN2:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.arm.neon.vld2.v4i32.p0i8(i8* [[TMP10]], i32 4)
-; NEON-NEXT:       [[TMP11:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN2]], 1
-; NEON-NEXT:       [[TMP12:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN2]], 0
-; NEON-NEXT:       [[TMP13:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP7]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; NEON-NEXT:       [[TMP14:%.*]] = shufflevector <4 x i32> [[TMP11]], <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; NEON-NEXT:       [[TMP15:%.*]] = shufflevector <8 x i32> [[TMP13]], <8 x i32> [[TMP14]], <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-; NEON-NEXT:       [[TMP16:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; NEON-NEXT:       [[TMP17:%.*]] = shufflevector <4 x i32> [[TMP12]], <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; NEON-NEXT:       [[TMP18:%.*]] = shufflevector <8 x i32> [[TMP16]], <8 x i32> [[TMP17]], <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-; NEON-NEXT:       ret void
-; NO_NEON-LABEL: @load_factor2_wide3(
-; NO_NEON-NOT:     @llvm.arm.neon
-; NO_NEON:         ret void
+; CHECK-NEON-LABEL: @load_factor2_wide3(
+; CHECK-NEON-NEXT:    [[TMP1:%.*]] = bitcast <24 x i32>* [[PTR:%.*]] to i32*
+; CHECK-NEON-NEXT:    [[TMP2:%.*]] = bitcast i32* [[TMP1]] to i8*
+; CHECK-NEON-NEXT:    [[VLDN:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.arm.neon.vld2.v4i32.p0i8(i8* [[TMP2]], i32 4)
+; CHECK-NEON-NEXT:    [[TMP3:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN]], 1
+; CHECK-NEON-NEXT:    [[TMP4:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN]], 0
+; CHECK-NEON-NEXT:    [[TMP5:%.*]] = getelementptr i32, i32* [[TMP1]], i32 8
+; CHECK-NEON-NEXT:    [[TMP6:%.*]] = bitcast i32* [[TMP5]] to i8*
+; CHECK-NEON-NEXT:    [[VLDN1:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.arm.neon.vld2.v4i32.p0i8(i8* [[TMP6]], i32 4)
+; CHECK-NEON-NEXT:    [[TMP7:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN1]], 1
+; CHECK-NEON-NEXT:    [[TMP8:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN1]], 0
+; CHECK-NEON-NEXT:    [[TMP9:%.*]] = getelementptr i32, i32* [[TMP5]], i32 8
+; CHECK-NEON-NEXT:    [[TMP10:%.*]] = bitcast i32* [[TMP9]] to i8*
+; CHECK-NEON-NEXT:    [[VLDN2:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.arm.neon.vld2.v4i32.p0i8(i8* [[TMP10]], i32 4)
+; CHECK-NEON-NEXT:    [[TMP11:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN2]], 1
+; CHECK-NEON-NEXT:    [[TMP12:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN2]], 0
+; CHECK-NEON-NEXT:    [[TMP13:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP7]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEON-NEXT:    [[TMP14:%.*]] = shufflevector <4 x i32> [[TMP11]], <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEON-NEXT:    [[TMP15:%.*]] = shufflevector <8 x i32> [[TMP13]], <8 x i32> [[TMP14]], <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEON-NEXT:    [[TMP16:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEON-NEXT:    [[TMP17:%.*]] = shufflevector <4 x i32> [[TMP12]], <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEON-NEXT:    [[TMP18:%.*]] = shufflevector <8 x i32> [[TMP16]], <8 x i32> [[TMP17]], <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEON-NEXT:    ret void
+;
+; CHECK-MVE-LABEL: @load_factor2_wide3(
+; CHECK-MVE-NEXT:    [[TMP1:%.*]] = bitcast <24 x i32>* [[PTR:%.*]] to i32*
+; CHECK-MVE-NEXT:    [[VLDN:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.arm.mve.vld2q.v4i32.p0i32(i32* [[TMP1]])
+; CHECK-MVE-NEXT:    [[TMP2:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN]], 1
+; CHECK-MVE-NEXT:    [[TMP3:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN]], 0
+; CHECK-MVE-NEXT:    [[TMP4:%.*]] = getelementptr i32, i32* [[TMP1]], i32 8
+; CHECK-MVE-NEXT:    [[VLDN1:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.arm.mve.vld2q.v4i32.p0i32(i32* [[TMP4]])
+; CHECK-MVE-NEXT:    [[TMP5:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN1]], 1
+; CHECK-MVE-NEXT:    [[TMP6:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN1]], 0
+; CHECK-MVE-NEXT:    [[TMP7:%.*]] = getelementptr i32, i32* [[TMP4]], i32 8
+; CHECK-MVE-NEXT:    [[VLDN2:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.arm.mve.vld2q.v4i32.p0i32(i32* [[TMP7]])
+; CHECK-MVE-NEXT:    [[TMP8:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN2]], 1
+; CHECK-MVE-NEXT:    [[TMP9:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN2]], 0
+; CHECK-MVE-NEXT:    [[TMP10:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> [[TMP5]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-MVE-NEXT:    [[TMP11:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-MVE-NEXT:    [[TMP12:%.*]] = shufflevector <8 x i32> [[TMP10]], <8 x i32> [[TMP11]], <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
+; CHECK-MVE-NEXT:    [[TMP13:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-MVE-NEXT:    [[TMP14:%.*]] = shufflevector <4 x i32> [[TMP9]], <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-MVE-NEXT:    [[TMP15:%.*]] = shufflevector <8 x i32> [[TMP13]], <8 x i32> [[TMP14]], <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
+; CHECK-MVE-NEXT:    ret void
+;
+; CHECK-NONE-LABEL: @load_factor2_wide3(
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = load <24 x i32>, <24 x i32>* [[PTR:%.*]], align 4
+; CHECK-NONE-NEXT:    [[V0:%.*]] = shufflevector <24 x i32> [[INTERLEAVED_VEC]], <24 x i32> undef, <12 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22>
+; CHECK-NONE-NEXT:    [[V1:%.*]] = shufflevector <24 x i32> [[INTERLEAVED_VEC]], <24 x i32> undef, <12 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23>
+; CHECK-NONE-NEXT:    ret void
 ;
   %interleaved.vec = load <24 x i32>, <24 x i32>* %ptr, align 4
   %v0 = shufflevector <24 x i32> %interleaved.vec, <24 x i32> undef, <12 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22>
@@ -707,26 +1127,37 @@ define void @load_factor2_wide3(<24 x i32>* %ptr) {
 }
 
 define void @load_factor3_wide(<24 x i32>* %ptr) {
-; NEON-LABEL:    @load_factor3_wide(
-; NEON-NEXT:       [[TMP1:%.*]] = bitcast <24 x i32>* %ptr to i32*
-; NEON-NEXT:       [[TMP2:%.*]] = bitcast i32* [[TMP1]] to i8*
-; NEON-NEXT:       [[VLDN:%.*]] = call { <4 x i32>, <4 x i32>, <4 x i32> } @llvm.arm.neon.vld3.v4i32.p0i8(i8* [[TMP2]], i32 4)
-; NEON-NEXT:       [[TMP3:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN]], 2
-; NEON-NEXT:       [[TMP4:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN]], 1
-; NEON-NEXT:       [[TMP5:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN]], 0
-; NEON-NEXT:       [[TMP6:%.*]] = getelementptr i32, i32* [[TMP1]], i32 12
-; NEON-NEXT:       [[TMP7:%.*]] = bitcast i32* [[TMP6]] to i8*
-; NEON-NEXT:       [[VLDN1:%.*]] = call { <4 x i32>, <4 x i32>, <4 x i32> } @llvm.arm.neon.vld3.v4i32.p0i8(i8* [[TMP7]], i32 4)
-; NEON-NEXT:       [[TMP8:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN1]], 2
-; NEON-NEXT:       [[TMP9:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN1]], 1
-; NEON-NEXT:       [[TMP10:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN1]], 0
-; NEON-NEXT:       [[TMP11:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; NEON-NEXT:       [[TMP12:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> [[TMP9]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; NEON-NEXT:       [[TMP13:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> [[TMP10]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; NEON-NEXT:       ret void
-; NO_NEON-LABEL: @load_factor3_wide(
-; NO_NEON-NOT:     @llvm.arm.neon
-; NO_NEON:         ret void
+; CHECK-NEON-LABEL: @load_factor3_wide(
+; CHECK-NEON-NEXT:    [[TMP1:%.*]] = bitcast <24 x i32>* [[PTR:%.*]] to i32*
+; CHECK-NEON-NEXT:    [[TMP2:%.*]] = bitcast i32* [[TMP1]] to i8*
+; CHECK-NEON-NEXT:    [[VLDN:%.*]] = call { <4 x i32>, <4 x i32>, <4 x i32> } @llvm.arm.neon.vld3.v4i32.p0i8(i8* [[TMP2]], i32 4)
+; CHECK-NEON-NEXT:    [[TMP3:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN]], 2
+; CHECK-NEON-NEXT:    [[TMP4:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN]], 1
+; CHECK-NEON-NEXT:    [[TMP5:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN]], 0
+; CHECK-NEON-NEXT:    [[TMP6:%.*]] = getelementptr i32, i32* [[TMP1]], i32 12
+; CHECK-NEON-NEXT:    [[TMP7:%.*]] = bitcast i32* [[TMP6]] to i8*
+; CHECK-NEON-NEXT:    [[VLDN1:%.*]] = call { <4 x i32>, <4 x i32>, <4 x i32> } @llvm.arm.neon.vld3.v4i32.p0i8(i8* [[TMP7]], i32 4)
+; CHECK-NEON-NEXT:    [[TMP8:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN1]], 2
+; CHECK-NEON-NEXT:    [[TMP9:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN1]], 1
+; CHECK-NEON-NEXT:    [[TMP10:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN1]], 0
+; CHECK-NEON-NEXT:    [[TMP11:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEON-NEXT:    [[TMP12:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> [[TMP9]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEON-NEXT:    [[TMP13:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> [[TMP10]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEON-NEXT:    ret void
+;
+; CHECK-MVE-LABEL: @load_factor3_wide(
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = load <24 x i32>, <24 x i32>* [[PTR:%.*]], align 4
+; CHECK-MVE-NEXT:    [[V0:%.*]] = shufflevector <24 x i32> [[INTERLEAVED_VEC]], <24 x i32> undef, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
+; CHECK-MVE-NEXT:    [[V1:%.*]] = shufflevector <24 x i32> [[INTERLEAVED_VEC]], <24 x i32> undef, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22>
+; CHECK-MVE-NEXT:    [[V2:%.*]] = shufflevector <24 x i32> [[INTERLEAVED_VEC]], <24 x i32> undef, <8 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23>
+; CHECK-MVE-NEXT:    ret void
+;
+; CHECK-NONE-LABEL: @load_factor3_wide(
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = load <24 x i32>, <24 x i32>* [[PTR:%.*]], align 4
+; CHECK-NONE-NEXT:    [[V0:%.*]] = shufflevector <24 x i32> [[INTERLEAVED_VEC]], <24 x i32> undef, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
+; CHECK-NONE-NEXT:    [[V1:%.*]] = shufflevector <24 x i32> [[INTERLEAVED_VEC]], <24 x i32> undef, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22>
+; CHECK-NONE-NEXT:    [[V2:%.*]] = shufflevector <24 x i32> [[INTERLEAVED_VEC]], <24 x i32> undef, <8 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23>
+; CHECK-NONE-NEXT:    ret void
 ;
   %interleaved.vec = load <24 x i32>, <24 x i32>* %ptr, align 4
   %v0 = shufflevector <24 x i32> %interleaved.vec, <24 x i32> undef, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
@@ -736,29 +1167,42 @@ define void @load_factor3_wide(<24 x i32>* %ptr) {
 }
 
 define void @load_factor4_wide(<32 x i32>* %ptr) {
-; NEON-LABEL:    @load_factor4_wide(
-; NEON-NEXT:       [[TMP1:%.*]] = bitcast <32 x i32>* %ptr to i32*
-; NEON-NEXT:       [[TMP2:%.*]] = bitcast i32* [[TMP1]] to i8*
-; NEON-NEXT:       [[VLDN:%.*]] = call { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } @llvm.arm.neon.vld4.v4i32.p0i8(i8* [[TMP2]], i32 4)
-; NEON-NEXT:       [[TMP3:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN]], 3
-; NEON-NEXT:       [[TMP4:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN]], 2
-; NEON-NEXT:       [[TMP5:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN]], 1
-; NEON-NEXT:       [[TMP6:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN]], 0
-; NEON-NEXT:       [[TMP7:%.*]] = getelementptr i32, i32* [[TMP1]], i32 16
-; NEON-NEXT:       [[TMP8:%.*]] = bitcast i32* [[TMP7]] to i8*
-; NEON-NEXT:       [[VLDN1:%.*]] = call { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } @llvm.arm.neon.vld4.v4i32.p0i8(i8* [[TMP8]], i32 4)
-; NEON-NEXT:       [[TMP9:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN1]], 3
-; NEON-NEXT:       [[TMP10:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN1]], 2
-; NEON-NEXT:       [[TMP11:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN1]], 1
-; NEON-NEXT:       [[TMP12:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN1]], 0
-; NEON-NEXT:       [[TMP13:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP9]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; NEON-NEXT:       [[TMP14:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> [[TMP10]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; NEON-NEXT:       [[TMP15:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> [[TMP11]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; NEON-NEXT:       [[TMP16:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> [[TMP12]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; NEON-NEXT:       ret void
-; NO_NEON-LABEL: @load_factor4_wide(
-; NO_NEON-NOT:     @llvm.arm.neon
-; NO_NEON:         ret void
+; CHECK-NEON-LABEL: @load_factor4_wide(
+; CHECK-NEON-NEXT:    [[TMP1:%.*]] = bitcast <32 x i32>* [[PTR:%.*]] to i32*
+; CHECK-NEON-NEXT:    [[TMP2:%.*]] = bitcast i32* [[TMP1]] to i8*
+; CHECK-NEON-NEXT:    [[VLDN:%.*]] = call { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } @llvm.arm.neon.vld4.v4i32.p0i8(i8* [[TMP2]], i32 4)
+; CHECK-NEON-NEXT:    [[TMP3:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN]], 3
+; CHECK-NEON-NEXT:    [[TMP4:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN]], 2
+; CHECK-NEON-NEXT:    [[TMP5:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN]], 1
+; CHECK-NEON-NEXT:    [[TMP6:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN]], 0
+; CHECK-NEON-NEXT:    [[TMP7:%.*]] = getelementptr i32, i32* [[TMP1]], i32 16
+; CHECK-NEON-NEXT:    [[TMP8:%.*]] = bitcast i32* [[TMP7]] to i8*
+; CHECK-NEON-NEXT:    [[VLDN1:%.*]] = call { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } @llvm.arm.neon.vld4.v4i32.p0i8(i8* [[TMP8]], i32 4)
+; CHECK-NEON-NEXT:    [[TMP9:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN1]], 3
+; CHECK-NEON-NEXT:    [[TMP10:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN1]], 2
+; CHECK-NEON-NEXT:    [[TMP11:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN1]], 1
+; CHECK-NEON-NEXT:    [[TMP12:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN1]], 0
+; CHECK-NEON-NEXT:    [[TMP13:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP9]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEON-NEXT:    [[TMP14:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> [[TMP10]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEON-NEXT:    [[TMP15:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> [[TMP11]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEON-NEXT:    [[TMP16:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> [[TMP12]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEON-NEXT:    ret void
+;
+; CHECK-MVE-LABEL: @load_factor4_wide(
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = load <32 x i32>, <32 x i32>* [[PTR:%.*]], align 4
+; CHECK-MVE-NEXT:    [[V0:%.*]] = shufflevector <32 x i32> [[INTERLEAVED_VEC]], <32 x i32> undef, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
+; CHECK-MVE-NEXT:    [[V1:%.*]] = shufflevector <32 x i32> [[INTERLEAVED_VEC]], <32 x i32> undef, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>
+; CHECK-MVE-NEXT:    [[V2:%.*]] = shufflevector <32 x i32> [[INTERLEAVED_VEC]], <32 x i32> undef, <8 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30>
+; CHECK-MVE-NEXT:    [[V3:%.*]] = shufflevector <32 x i32> [[INTERLEAVED_VEC]], <32 x i32> undef, <8 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31>
+; CHECK-MVE-NEXT:    ret void
+;
+; CHECK-NONE-LABEL: @load_factor4_wide(
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = load <32 x i32>, <32 x i32>* [[PTR:%.*]], align 4
+; CHECK-NONE-NEXT:    [[V0:%.*]] = shufflevector <32 x i32> [[INTERLEAVED_VEC]], <32 x i32> undef, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
+; CHECK-NONE-NEXT:    [[V1:%.*]] = shufflevector <32 x i32> [[INTERLEAVED_VEC]], <32 x i32> undef, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>
+; CHECK-NONE-NEXT:    [[V2:%.*]] = shufflevector <32 x i32> [[INTERLEAVED_VEC]], <32 x i32> undef, <8 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30>
+; CHECK-NONE-NEXT:    [[V3:%.*]] = shufflevector <32 x i32> [[INTERLEAVED_VEC]], <32 x i32> undef, <8 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31>
+; CHECK-NONE-NEXT:    ret void
 ;
   %interleaved.vec = load <32 x i32>, <32 x i32>* %ptr, align 4
   %v0 = shufflevector <32 x i32> %interleaved.vec, <32 x i32> undef, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
@@ -769,21 +1213,36 @@ define void @load_factor4_wide(<32 x i32>* %ptr) {
 }
 
 define void @store_factor2_wide(<16 x i32>* %ptr, <8 x i32> %v0, <8 x i32> %v1) {
-; NEON-LABEL:    @store_factor2_wide(
-; NEON-NEXT:       [[TMP1:%.*]] = bitcast <16 x i32>* %ptr to i32*
-; NEON-NEXT:       [[TMP2:%.*]] = bitcast i32* [[TMP1]] to i8*
-; NEON-NEXT:       [[TMP3:%.*]] = shufflevector <8 x i32> %v0, <8 x i32> %v1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; NEON-NEXT:       [[TMP4:%.*]] = shufflevector <8 x i32> %v0, <8 x i32> %v1, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
-; NEON-NEXT:       call void @llvm.arm.neon.vst2.p0i8.v4i32(i8* [[TMP2]], <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], i32 4)
-; NEON-NEXT:       [[TMP5:%.*]] = getelementptr i32, i32* [[TMP1]], i32 8
-; NEON-NEXT:       [[TMP6:%.*]] = bitcast i32* [[TMP5]] to i8*
-; NEON-NEXT:       [[TMP7:%.*]] = shufflevector <8 x i32> %v0, <8 x i32> %v1, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; NEON-NEXT:       [[TMP8:%.*]] = shufflevector <8 x i32> %v0, <8 x i32> %v1, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
-; NEON-NEXT:       call void @llvm.arm.neon.vst2.p0i8.v4i32(i8* [[TMP6]], <4 x i32> [[TMP7]], <4 x i32> [[TMP8]], i32 4)
-; NEON-NEXT:       ret void
-; NO_NEON-LABEL: @store_factor2_wide(
-; NO_NEON-NOT:     @llvm.arm.neon
-; NO_NEON:         ret void
+; CHECK-NEON-LABEL: @store_factor2_wide(
+; CHECK-NEON-NEXT:    [[TMP1:%.*]] = bitcast <16 x i32>* [[PTR:%.*]] to i32*
+; CHECK-NEON-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[V0:%.*]], <8 x i32> [[V1:%.*]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEON-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[V0]], <8 x i32> [[V1]], <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEON-NEXT:    [[TMP4:%.*]] = bitcast i32* [[TMP1]] to i8*
+; CHECK-NEON-NEXT:    call void @llvm.arm.neon.vst2.p0i8.v4i32(i8* [[TMP4]], <4 x i32> [[TMP2]], <4 x i32> [[TMP3]], i32 4)
+; CHECK-NEON-NEXT:    [[TMP5:%.*]] = getelementptr i32, i32* [[TMP1]], i32 8
+; CHECK-NEON-NEXT:    [[TMP6:%.*]] = shufflevector <8 x i32> [[V0]], <8 x i32> [[V1]], <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEON-NEXT:    [[TMP7:%.*]] = shufflevector <8 x i32> [[V0]], <8 x i32> [[V1]], <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEON-NEXT:    [[TMP8:%.*]] = bitcast i32* [[TMP5]] to i8*
+; CHECK-NEON-NEXT:    call void @llvm.arm.neon.vst2.p0i8.v4i32(i8* [[TMP8]], <4 x i32> [[TMP6]], <4 x i32> [[TMP7]], i32 4)
+; CHECK-NEON-NEXT:    ret void
+;
+; CHECK-MVE-LABEL: @store_factor2_wide(
+; CHECK-MVE-NEXT:    [[TMP1:%.*]] = bitcast <16 x i32>* [[PTR:%.*]] to i32*
+; CHECK-MVE-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[V0:%.*]], <8 x i32> [[V1:%.*]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-MVE-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[V0]], <8 x i32> [[V1]], <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; CHECK-MVE-NEXT:    call void @llvm.arm.mve.vst2q.p0i32.v4i32(i32* [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> [[TMP3]], i32 0)
+; CHECK-MVE-NEXT:    call void @llvm.arm.mve.vst2q.p0i32.v4i32(i32* [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> [[TMP3]], i32 1)
+; CHECK-MVE-NEXT:    [[TMP4:%.*]] = getelementptr i32, i32* [[TMP1]], i32 8
+; CHECK-MVE-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i32> [[V0]], <8 x i32> [[V1]], <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-MVE-NEXT:    [[TMP6:%.*]] = shufflevector <8 x i32> [[V0]], <8 x i32> [[V1]], <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; CHECK-MVE-NEXT:    call void @llvm.arm.mve.vst2q.p0i32.v4i32(i32* [[TMP4]], <4 x i32> [[TMP5]], <4 x i32> [[TMP6]], i32 0)
+; CHECK-MVE-NEXT:    call void @llvm.arm.mve.vst2q.p0i32.v4i32(i32* [[TMP4]], <4 x i32> [[TMP5]], <4 x i32> [[TMP6]], i32 1)
+; CHECK-MVE-NEXT:    ret void
+;
+; CHECK-NONE-LABEL: @store_factor2_wide(
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i32> [[V0:%.*]], <8 x i32> [[V1:%.*]], <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
+; CHECK-NONE-NEXT:    store <16 x i32> [[INTERLEAVED_VEC]], <16 x i32>* [[PTR:%.*]], align 4
+; CHECK-NONE-NEXT:    ret void
 ;
   %interleaved.vec = shufflevector <8 x i32> %v0, <8 x i32> %v1, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
   store <16 x i32> %interleaved.vec, <16 x i32>* %ptr, align 4
@@ -791,23 +1250,36 @@ define void @store_factor2_wide(<16 x i32>* %ptr, <8 x i32> %v0, <8 x i32> %v1)
 }
 
 define void @store_factor3_wide(<24 x i32>* %ptr, <8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2) {
-; NEON-LABEL:    @store_factor3_wide(
-; NEON:            [[TMP1:%.*]] = bitcast <24 x i32>* %ptr to i32*
-; NEON-NEXT:       [[TMP2:%.*]] = bitcast i32* [[TMP1]] to i8*
-; NEON-NEXT:       [[TMP3:%.*]] = shufflevector <16 x i32> %s0, <16 x i32> %s1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; NEON-NEXT:       [[TMP4:%.*]] = shufflevector <16 x i32> %s0, <16 x i32> %s1, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
-; NEON-NEXT:       [[TMP5:%.*]] = shufflevector <16 x i32> %s0, <16 x i32> %s1, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
-; NEON-NEXT:       call void @llvm.arm.neon.vst3.p0i8.v4i32(i8* [[TMP2]], <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <4 x i32> [[TMP5]], i32 4)
-; NEON-NEXT:       [[TMP6:%.*]] = getelementptr i32, i32* [[TMP1]], i32 12
-; NEON-NEXT:       [[TMP7:%.*]] = bitcast i32* [[TMP6]] to i8*
-; NEON-NEXT:       [[TMP8:%.*]] = shufflevector <16 x i32> %s0, <16 x i32> %s1, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; NEON-NEXT:       [[TMP9:%.*]] = shufflevector <16 x i32> %s0, <16 x i32> %s1, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
-; NEON-NEXT:       [[TMP10:%.*]] = shufflevector <16 x i32> %s0, <16 x i32> %s1, <4 x i32> <i32 20, i32 21, i32 22, i32 23>
-; NEON-NEXT:       call void @llvm.arm.neon.vst3.p0i8.v4i32(i8* [[TMP7]], <4 x i32> [[TMP8]], <4 x i32> [[TMP9]], <4 x i32> [[TMP10]], i32 4)
-; NEON-NEXT:       ret void
-; NO_NEON-LABEL: @store_factor3_wide(
-; NO_NEON-NOT:     @llvm.arm.neon
-; NO_NEON:         ret void
+; CHECK-NEON-LABEL: @store_factor3_wide(
+; CHECK-NEON-NEXT:    [[S0:%.*]] = shufflevector <8 x i32> [[V0:%.*]], <8 x i32> [[V1:%.*]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEON-NEXT:    [[S1:%.*]] = shufflevector <8 x i32> [[V2:%.*]], <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEON-NEXT:    [[TMP1:%.*]] = bitcast <24 x i32>* [[PTR:%.*]] to i32*
+; CHECK-NEON-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i32> [[S0]], <16 x i32> [[S1]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEON-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i32> [[S0]], <16 x i32> [[S1]], <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEON-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i32> [[S0]], <16 x i32> [[S1]], <4 x i32> <i32 16, i32 17, i32 18, i32 19>
+; CHECK-NEON-NEXT:    [[TMP5:%.*]] = bitcast i32* [[TMP1]] to i8*
+; CHECK-NEON-NEXT:    call void @llvm.arm.neon.vst3.p0i8.v4i32(i8* [[TMP5]], <4 x i32> [[TMP2]], <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], i32 4)
+; CHECK-NEON-NEXT:    [[TMP6:%.*]] = getelementptr i32, i32* [[TMP1]], i32 12
+; CHECK-NEON-NEXT:    [[TMP7:%.*]] = shufflevector <16 x i32> [[S0]], <16 x i32> [[S1]], <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEON-NEXT:    [[TMP8:%.*]] = shufflevector <16 x i32> [[S0]], <16 x i32> [[S1]], <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEON-NEXT:    [[TMP9:%.*]] = shufflevector <16 x i32> [[S0]], <16 x i32> [[S1]], <4 x i32> <i32 20, i32 21, i32 22, i32 23>
+; CHECK-NEON-NEXT:    [[TMP10:%.*]] = bitcast i32* [[TMP6]] to i8*
+; CHECK-NEON-NEXT:    call void @llvm.arm.neon.vst3.p0i8.v4i32(i8* [[TMP10]], <4 x i32> [[TMP7]], <4 x i32> [[TMP8]], <4 x i32> [[TMP9]], i32 4)
+; CHECK-NEON-NEXT:    ret void
+;
+; CHECK-MVE-LABEL: @store_factor3_wide(
+; CHECK-MVE-NEXT:    [[S0:%.*]] = shufflevector <8 x i32> [[V0:%.*]], <8 x i32> [[V1:%.*]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-MVE-NEXT:    [[S1:%.*]] = shufflevector <8 x i32> [[V2:%.*]], <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <16 x i32> [[S0]], <16 x i32> [[S1]], <24 x i32> <i32 0, i32 8, i32 16, i32 1, i32 9, i32 17, i32 2, i32 10, i32 18, i32 3, i32 11, i32 19, i32 4, i32 12, i32 20, i32 5, i32 13, i32 21, i32 6, i32 14, i32 22, i32 7, i32 15, i32 23>
+; CHECK-MVE-NEXT:    store <24 x i32> [[INTERLEAVED_VEC]], <24 x i32>* [[PTR:%.*]], align 4
+; CHECK-MVE-NEXT:    ret void
+;
+; CHECK-NONE-LABEL: @store_factor3_wide(
+; CHECK-NONE-NEXT:    [[S0:%.*]] = shufflevector <8 x i32> [[V0:%.*]], <8 x i32> [[V1:%.*]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NONE-NEXT:    [[S1:%.*]] = shufflevector <8 x i32> [[V2:%.*]], <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <16 x i32> [[S0]], <16 x i32> [[S1]], <24 x i32> <i32 0, i32 8, i32 16, i32 1, i32 9, i32 17, i32 2, i32 10, i32 18, i32 3, i32 11, i32 19, i32 4, i32 12, i32 20, i32 5, i32 13, i32 21, i32 6, i32 14, i32 22, i32 7, i32 15, i32 23>
+; CHECK-NONE-NEXT:    store <24 x i32> [[INTERLEAVED_VEC]], <24 x i32>* [[PTR:%.*]], align 4
+; CHECK-NONE-NEXT:    ret void
 ;
   %s0 = shufflevector <8 x i32> %v0, <8 x i32> %v1, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
   %s1 = shufflevector <8 x i32> %v2, <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -817,25 +1289,38 @@ define void @store_factor3_wide(<24 x i32>* %ptr, <8 x i32> %v0, <8 x i32> %v1,
 }
 
 define void @store_factor4_wide(<32 x i32>* %ptr, <8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2, <8 x i32> %v3) {
-; NEON-LABEL:    @store_factor4_wide(
-; NEON:            [[TMP1:%.*]] = bitcast <32 x i32>* %ptr to i32*
-; NEON-NEXT:       [[TMP2:%.*]] = bitcast i32* [[TMP1]] to i8*
-; NEON-NEXT:       [[TMP3:%.*]] = shufflevector <16 x i32> %s0, <16 x i32> %s1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; NEON-NEXT:       [[TMP4:%.*]] = shufflevector <16 x i32> %s0, <16 x i32> %s1, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
-; NEON-NEXT:       [[TMP5:%.*]] = shufflevector <16 x i32> %s0, <16 x i32> %s1, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
-; NEON-NEXT:       [[TMP6:%.*]] = shufflevector <16 x i32> %s0, <16 x i32> %s1, <4 x i32> <i32 24, i32 25, i32 26, i32 27>
-; NEON-NEXT:       call void @llvm.arm.neon.vst4.p0i8.v4i32(i8* [[TMP2]], <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <4 x i32> [[TMP5]], <4 x i32> [[TMP6]], i32 4)
-; NEON-NEXT:       [[TMP7:%.*]] = getelementptr i32, i32* [[TMP1]], i32 16
-; NEON-NEXT:       [[TMP8:%.*]] = bitcast i32* [[TMP7]] to i8*
-; NEON-NEXT:       [[TMP9:%.*]] = shufflevector <16 x i32> %s0, <16 x i32> %s1, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; NEON-NEXT:       [[TMP10:%.*]] = shufflevector <16 x i32> %s0, <16 x i32> %s1, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
-; NEON-NEXT:       [[TMP11:%.*]] = shufflevector <16 x i32> %s0, <16 x i32> %s1, <4 x i32> <i32 20, i32 21, i32 22, i32 23>
-; NEON-NEXT:       [[TMP12:%.*]] = shufflevector <16 x i32> %s0, <16 x i32> %s1, <4 x i32> <i32 28, i32 29, i32 30, i32 31>
-; NEON-NEXT:       call void @llvm.arm.neon.vst4.p0i8.v4i32(i8* [[TMP8]], <4 x i32> [[TMP9]], <4 x i32> [[TMP10]], <4 x i32> [[TMP11]], <4 x i32> [[TMP12]], i32 4)
-; NEON-NEXT:       ret void
-; NO_NEON-LABEL: @store_factor4_wide(
-; NO_NEON-NOT:     @llvm.arm.neon
-; NO_NEON:         ret void
+; CHECK-NEON-LABEL: @store_factor4_wide(
+; CHECK-NEON-NEXT:    [[S0:%.*]] = shufflevector <8 x i32> [[V0:%.*]], <8 x i32> [[V1:%.*]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEON-NEXT:    [[S1:%.*]] = shufflevector <8 x i32> [[V2:%.*]], <8 x i32> [[V3:%.*]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEON-NEXT:    [[TMP1:%.*]] = bitcast <32 x i32>* [[PTR:%.*]] to i32*
+; CHECK-NEON-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i32> [[S0]], <16 x i32> [[S1]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEON-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i32> [[S0]], <16 x i32> [[S1]], <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEON-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i32> [[S0]], <16 x i32> [[S1]], <4 x i32> <i32 16, i32 17, i32 18, i32 19>
+; CHECK-NEON-NEXT:    [[TMP5:%.*]] = shufflevector <16 x i32> [[S0]], <16 x i32> [[S1]], <4 x i32> <i32 24, i32 25, i32 26, i32 27>
+; CHECK-NEON-NEXT:    [[TMP6:%.*]] = bitcast i32* [[TMP1]] to i8*
+; CHECK-NEON-NEXT:    call void @llvm.arm.neon.vst4.p0i8.v4i32(i8* [[TMP6]], <4 x i32> [[TMP2]], <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <4 x i32> [[TMP5]], i32 4)
+; CHECK-NEON-NEXT:    [[TMP7:%.*]] = getelementptr i32, i32* [[TMP1]], i32 16
+; CHECK-NEON-NEXT:    [[TMP8:%.*]] = shufflevector <16 x i32> [[S0]], <16 x i32> [[S1]], <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEON-NEXT:    [[TMP9:%.*]] = shufflevector <16 x i32> [[S0]], <16 x i32> [[S1]], <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEON-NEXT:    [[TMP10:%.*]] = shufflevector <16 x i32> [[S0]], <16 x i32> [[S1]], <4 x i32> <i32 20, i32 21, i32 22, i32 23>
+; CHECK-NEON-NEXT:    [[TMP11:%.*]] = shufflevector <16 x i32> [[S0]], <16 x i32> [[S1]], <4 x i32> <i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEON-NEXT:    [[TMP12:%.*]] = bitcast i32* [[TMP7]] to i8*
+; CHECK-NEON-NEXT:    call void @llvm.arm.neon.vst4.p0i8.v4i32(i8* [[TMP12]], <4 x i32> [[TMP8]], <4 x i32> [[TMP9]], <4 x i32> [[TMP10]], <4 x i32> [[TMP11]], i32 4)
+; CHECK-NEON-NEXT:    ret void
+;
+; CHECK-MVE-LABEL: @store_factor4_wide(
+; CHECK-MVE-NEXT:    [[S0:%.*]] = shufflevector <8 x i32> [[V0:%.*]], <8 x i32> [[V1:%.*]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-MVE-NEXT:    [[S1:%.*]] = shufflevector <8 x i32> [[V2:%.*]], <8 x i32> [[V3:%.*]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <16 x i32> [[S0]], <16 x i32> [[S1]], <32 x i32> <i32 0, i32 8, i32 16, i32 24, i32 1, i32 9, i32 17, i32 25, i32 2, i32 10, i32 18, i32 26, i32 3, i32 11, i32 19, i32 27, i32 4, i32 12, i32 20, i32 28, i32 5, i32 13, i32 21, i32 29, i32 6, i32 14, i32 22, i32 30, i32 7, i32 15, i32 23, i32 31>
+; CHECK-MVE-NEXT:    store <32 x i32> [[INTERLEAVED_VEC]], <32 x i32>* [[PTR:%.*]], align 4
+; CHECK-MVE-NEXT:    ret void
+;
+; CHECK-NONE-LABEL: @store_factor4_wide(
+; CHECK-NONE-NEXT:    [[S0:%.*]] = shufflevector <8 x i32> [[V0:%.*]], <8 x i32> [[V1:%.*]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NONE-NEXT:    [[S1:%.*]] = shufflevector <8 x i32> [[V2:%.*]], <8 x i32> [[V3:%.*]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <16 x i32> [[S0]], <16 x i32> [[S1]], <32 x i32> <i32 0, i32 8, i32 16, i32 24, i32 1, i32 9, i32 17, i32 25, i32 2, i32 10, i32 18, i32 26, i32 3, i32 11, i32 19, i32 27, i32 4, i32 12, i32 20, i32 28, i32 5, i32 13, i32 21, i32 29, i32 6, i32 14, i32 22, i32 30, i32 7, i32 15, i32 23, i32 31>
+; CHECK-NONE-NEXT:    store <32 x i32> [[INTERLEAVED_VEC]], <32 x i32>* [[PTR:%.*]], align 4
+; CHECK-NONE-NEXT:    ret void
 ;
   %s0 = shufflevector <8 x i32> %v0, <8 x i32> %v1, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
   %s1 = shufflevector <8 x i32> %v2, <8 x i32> %v3, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -845,9 +1330,23 @@ define void @store_factor4_wide(<32 x i32>* %ptr, <8 x i32> %v0, <8 x i32> %v1,
 }
 
 define void @load_factor2_fp128(<4 x fp128>* %ptr) {
-; ALL-LABEL: @load_factor2_fp128(
-; ALL-NOT:     @llvm.arm.neon
-; ALL:         ret void
+; CHECK-NEON-LABEL: @load_factor2_fp128(
+; CHECK-NEON-NEXT:    [[INTERLEAVED_VEC:%.*]] = load <4 x fp128>, <4 x fp128>* [[PTR:%.*]], align 16
+; CHECK-NEON-NEXT:    [[V0:%.*]] = shufflevector <4 x fp128> [[INTERLEAVED_VEC]], <4 x fp128> undef, <2 x i32> <i32 0, i32 2>
+; CHECK-NEON-NEXT:    [[V1:%.*]] = shufflevector <4 x fp128> [[INTERLEAVED_VEC]], <4 x fp128> undef, <2 x i32> <i32 1, i32 3>
+; CHECK-NEON-NEXT:    ret void
+;
+; CHECK-MVE-LABEL: @load_factor2_fp128(
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = load <4 x fp128>, <4 x fp128>* [[PTR:%.*]], align 16
+; CHECK-MVE-NEXT:    [[V0:%.*]] = shufflevector <4 x fp128> [[INTERLEAVED_VEC]], <4 x fp128> undef, <2 x i32> <i32 0, i32 2>
+; CHECK-MVE-NEXT:    [[V1:%.*]] = shufflevector <4 x fp128> [[INTERLEAVED_VEC]], <4 x fp128> undef, <2 x i32> <i32 1, i32 3>
+; CHECK-MVE-NEXT:    ret void
+;
+; CHECK-NONE-LABEL: @load_factor2_fp128(
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = load <4 x fp128>, <4 x fp128>* [[PTR:%.*]], align 16
+; CHECK-NONE-NEXT:    [[V0:%.*]] = shufflevector <4 x fp128> [[INTERLEAVED_VEC]], <4 x fp128> undef, <2 x i32> <i32 0, i32 2>
+; CHECK-NONE-NEXT:    [[V1:%.*]] = shufflevector <4 x fp128> [[INTERLEAVED_VEC]], <4 x fp128> undef, <2 x i32> <i32 1, i32 3>
+; CHECK-NONE-NEXT:    ret void
 ;
   %interleaved.vec = load <4 x fp128>, <4 x fp128>* %ptr, align 16
   %v0 = shufflevector <4 x fp128> %interleaved.vec, <4 x fp128> undef, <2 x i32> <i32 0, i32 2>
@@ -856,27 +1355,47 @@ define void @load_factor2_fp128(<4 x fp128>* %ptr) {
 }
 
 define void @load_factor2_wide_pointer(<16 x i32*>* %ptr) {
-; NEON-LABEL:    @load_factor2_wide_pointer(
-; NEON-NEXT:       [[TMP1:%.*]] = bitcast <16 x i32*>* %ptr to i32*
-; NEON-NEXT:       [[TMP2:%.*]] = bitcast i32* [[TMP1]] to i8*
-; NEON-NEXT:       [[VLDN:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.arm.neon.vld2.v4i32.p0i8(i8* [[TMP2]], i32 4)
-; NEON-NEXT:       [[TMP3:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN]], 1
-; NEON-NEXT:       [[TMP4:%.*]] = inttoptr <4 x i32> [[TMP3]] to <4 x i32*>
-; NEON-NEXT:       [[TMP5:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN]], 0
-; NEON-NEXT:       [[TMP6:%.*]] = inttoptr <4 x i32> [[TMP5]] to <4 x i32*>
-; NEON-NEXT:       [[TMP7:%.*]] = getelementptr i32, i32* [[TMP1]], i32 8
-; NEON-NEXT:       [[TMP8:%.*]] = bitcast i32* [[TMP7]] to i8*
-; NEON-NEXT:       [[VLDN1:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.arm.neon.vld2.v4i32.p0i8(i8* [[TMP8]], i32 4)
-; NEON-NEXT:       [[TMP9:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN1]], 1
-; NEON-NEXT:       [[TMP10:%.*]] = inttoptr <4 x i32> [[TMP9]] to <4 x i32*>
-; NEON-NEXT:       [[TMP11:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN1]], 0
-; NEON-NEXT:       [[TMP12:%.*]] = inttoptr <4 x i32> [[TMP11]] to <4 x i32*>
-; NEON-NEXT:       [[TMP13:%.*]] = shufflevector <4 x i32*> [[TMP4]], <4 x i32*> [[TMP10]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; NEON-NEXT:       [[TMP14:%.*]] = shufflevector <4 x i32*> [[TMP6]], <4 x i32*> [[TMP12]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; NEON-NEXT:       ret void
-; NO_NEON-LABEL: @load_factor2_wide_pointer(
-; NO_NEON-NOT:     @llvm.arm.neon
-; NO_NEON:         ret void
+; CHECK-NEON-LABEL: @load_factor2_wide_pointer(
+; CHECK-NEON-NEXT:    [[TMP1:%.*]] = bitcast <16 x i32*>* [[PTR:%.*]] to i32*
+; CHECK-NEON-NEXT:    [[TMP2:%.*]] = bitcast i32* [[TMP1]] to i8*
+; CHECK-NEON-NEXT:    [[VLDN:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.arm.neon.vld2.v4i32.p0i8(i8* [[TMP2]], i32 4)
+; CHECK-NEON-NEXT:    [[TMP3:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN]], 1
+; CHECK-NEON-NEXT:    [[TMP4:%.*]] = inttoptr <4 x i32> [[TMP3]] to <4 x i32*>
+; CHECK-NEON-NEXT:    [[TMP5:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN]], 0
+; CHECK-NEON-NEXT:    [[TMP6:%.*]] = inttoptr <4 x i32> [[TMP5]] to <4 x i32*>
+; CHECK-NEON-NEXT:    [[TMP7:%.*]] = getelementptr i32, i32* [[TMP1]], i32 8
+; CHECK-NEON-NEXT:    [[TMP8:%.*]] = bitcast i32* [[TMP7]] to i8*
+; CHECK-NEON-NEXT:    [[VLDN1:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.arm.neon.vld2.v4i32.p0i8(i8* [[TMP8]], i32 4)
+; CHECK-NEON-NEXT:    [[TMP9:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN1]], 1
+; CHECK-NEON-NEXT:    [[TMP10:%.*]] = inttoptr <4 x i32> [[TMP9]] to <4 x i32*>
+; CHECK-NEON-NEXT:    [[TMP11:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN1]], 0
+; CHECK-NEON-NEXT:    [[TMP12:%.*]] = inttoptr <4 x i32> [[TMP11]] to <4 x i32*>
+; CHECK-NEON-NEXT:    [[TMP13:%.*]] = shufflevector <4 x i32*> [[TMP4]], <4 x i32*> [[TMP10]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEON-NEXT:    [[TMP14:%.*]] = shufflevector <4 x i32*> [[TMP6]], <4 x i32*> [[TMP12]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEON-NEXT:    ret void
+;
+; CHECK-MVE-LABEL: @load_factor2_wide_pointer(
+; CHECK-MVE-NEXT:    [[TMP1:%.*]] = bitcast <16 x i32*>* [[PTR:%.*]] to i32*
+; CHECK-MVE-NEXT:    [[VLDN:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.arm.mve.vld2q.v4i32.p0i32(i32* [[TMP1]])
+; CHECK-MVE-NEXT:    [[TMP2:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN]], 1
+; CHECK-MVE-NEXT:    [[TMP3:%.*]] = inttoptr <4 x i32> [[TMP2]] to <4 x i32*>
+; CHECK-MVE-NEXT:    [[TMP4:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN]], 0
+; CHECK-MVE-NEXT:    [[TMP5:%.*]] = inttoptr <4 x i32> [[TMP4]] to <4 x i32*>
+; CHECK-MVE-NEXT:    [[TMP6:%.*]] = getelementptr i32, i32* [[TMP1]], i32 8
+; CHECK-MVE-NEXT:    [[VLDN1:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.arm.mve.vld2q.v4i32.p0i32(i32* [[TMP6]])
+; CHECK-MVE-NEXT:    [[TMP7:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN1]], 1
+; CHECK-MVE-NEXT:    [[TMP8:%.*]] = inttoptr <4 x i32> [[TMP7]] to <4 x i32*>
+; CHECK-MVE-NEXT:    [[TMP9:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN1]], 0
+; CHECK-MVE-NEXT:    [[TMP10:%.*]] = inttoptr <4 x i32> [[TMP9]] to <4 x i32*>
+; CHECK-MVE-NEXT:    [[TMP11:%.*]] = shufflevector <4 x i32*> [[TMP3]], <4 x i32*> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-MVE-NEXT:    [[TMP12:%.*]] = shufflevector <4 x i32*> [[TMP5]], <4 x i32*> [[TMP10]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-MVE-NEXT:    ret void
+;
+; CHECK-NONE-LABEL: @load_factor2_wide_pointer(
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = load <16 x i32*>, <16 x i32*>* [[PTR:%.*]], align 4
+; CHECK-NONE-NEXT:    [[V0:%.*]] = shufflevector <16 x i32*> [[INTERLEAVED_VEC]], <16 x i32*> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+; CHECK-NONE-NEXT:    [[V1:%.*]] = shufflevector <16 x i32*> [[INTERLEAVED_VEC]], <16 x i32*> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+; CHECK-NONE-NEXT:    ret void
 ;
   %interleaved.vec = load <16 x i32*>, <16 x i32*>* %ptr, align 4
   %v0 = shufflevector <16 x i32*> %interleaved.vec, <16 x i32*> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
@@ -888,9 +1407,24 @@ define void @load_factor2_wide_pointer(<16 x i32*>* %ptr) {
 ; actually load enough elements to satisfy the shuffle masks. (It would be
 ; possible to produce a vld2.v2i32, but that currently isn't implemented.)
 define void @load_out_of_range(<4 x i32>* %ptr) {
-; ALL-LABEL: @load_out_of_range(
-; ALL-NOT:     @llvm.arm.neon
-; ALL:         ret void
+; CHECK-NEON-LABEL: @load_out_of_range(
+; CHECK-NEON-NEXT:    [[INTERLEAVED_VEC:%.*]] = load <4 x i32>, <4 x i32>* [[PTR:%.*]], align 4
+; CHECK-NEON-NEXT:    [[V0:%.*]] = shufflevector <4 x i32> [[INTERLEAVED_VEC]], <4 x i32> undef, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
+; CHECK-NEON-NEXT:    [[V1:%.*]] = shufflevector <4 x i32> [[INTERLEAVED_VEC]], <4 x i32> undef, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
+; CHECK-NEON-NEXT:    ret void
+;
+; CHECK-MVE-LABEL: @load_out_of_range(
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = load <4 x i32>, <4 x i32>* [[PTR:%.*]], align 4
+; CHECK-MVE-NEXT:    [[V0:%.*]] = shufflevector <4 x i32> [[INTERLEAVED_VEC]], <4 x i32> undef, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
+; CHECK-MVE-NEXT:    [[V1:%.*]] = shufflevector <4 x i32> [[INTERLEAVED_VEC]], <4 x i32> undef, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
+; CHECK-MVE-NEXT:    ret void
+;
+; CHECK-NONE-LABEL: @load_out_of_range(
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = load <4 x i32>, <4 x i32>* [[PTR:%.*]], align 4
+; CHECK-NONE-NEXT:    [[V0:%.*]] = shufflevector <4 x i32> [[INTERLEAVED_VEC]], <4 x i32> undef, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
+; CHECK-NONE-NEXT:    [[V1:%.*]] = shufflevector <4 x i32> [[INTERLEAVED_VEC]], <4 x i32> undef, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
+; CHECK-NONE-NEXT:    ret void
+;
   %interleaved.vec = load <4 x i32>, <4 x i32>* %ptr, align 4
   %v0 = shufflevector <4 x i32> %interleaved.vec, <4 x i32> undef, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
   %v1 = shufflevector <4 x i32> %interleaved.vec, <4 x i32> undef, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
diff --git a/llvm/test/Transforms/Internalize/vcall-visibility.ll b/llvm/test/Transforms/Internalize/vcall-visibility.ll
new file mode 100644
index 000000000..dd4419502
--- /dev/null
+++ b/llvm/test/Transforms/Internalize/vcall-visibility.ll
@@ -0,0 +1,64 @@
+; RUN: opt < %s -internalize -S | FileCheck %s
+
+%struct.A = type { i32 (...)** }
+%struct.B = type { i32 (...)** }
+%struct.C = type { i32 (...)** }
+
+; Class A has default visibility, so has no !vcall_visibility metadata before
+; or after LTO.
+; CHECK-NOT: @_ZTV1A = {{.*}}!vcall_visibility
+@_ZTV1A = dso_local unnamed_addr constant { [3 x i8*] } { [3 x i8*] [i8* null, i8* null, i8* bitcast (void (%struct.A*)* @_ZN1A3fooEv to i8*)] }, align 8, !type !0, !type !1
+
+; Class B has hidden visibility but public LTO visibility, so has no
+; !vcall_visibility metadata before or after LTO.
+; CHECK-NOT: @_ZTV1B = {{.*}}!vcall_visibility
+@_ZTV1B = hidden unnamed_addr constant { [3 x i8*] } { [3 x i8*] [i8* null, i8* null, i8* bitcast (void (%struct.B*)* @_ZN1B3fooEv to i8*)] }, align 8, !type !2, !type !3
+
+; Class C has hidden visibility, so the !vcall_visibility metadata is set to 1
+; (linkage unit) before LTO, and 2 (translation unit) after LTO.
+; CHECK: @_ZTV1C ={{.*}}!vcall_visibility [[MD_TU_VIS:![0-9]+]]
+@_ZTV1C = hidden unnamed_addr constant { [3 x i8*] } { [3 x i8*] [i8* null, i8* null, i8* bitcast (void (%struct.C*)* @_ZN1C3fooEv to i8*)] }, align 8, !type !4, !type !5, !vcall_visibility !6
+
+; Class D has translation unit visibility before LTO, and this is not changed
+; by LTO.
+; CHECK: @_ZTVN12_GLOBAL__N_11DE = {{.*}}!vcall_visibility [[MD_TU_VIS:![0-9]+]]
+@_ZTVN12_GLOBAL__N_11DE = internal unnamed_addr constant { [3 x i8*] } zeroinitializer, align 8, !type !7, !type !9, !vcall_visibility !11
+
+define dso_local void @_ZN1A3fooEv(%struct.A* nocapture %this) {
+entry:
+  ret void
+}
+
+define hidden void @_ZN1B3fooEv(%struct.B* nocapture %this) {
+entry:
+  ret void
+}
+
+define hidden void @_ZN1C3fooEv(%struct.C* nocapture %this) {
+entry:
+  ret void
+}
+
+define hidden noalias nonnull i8* @_Z6make_dv() {
+entry:
+  %call = tail call i8* @_Znwm(i64 8) #3
+  %0 = bitcast i8* %call to i32 (...)***
+  store i32 (...)** bitcast (i8** getelementptr inbounds ({ [3 x i8*] }, { [3 x i8*] }* @_ZTVN12_GLOBAL__N_11DE, i64 0, inrange i32 0, i64 2) to i32 (...)**), i32 (...)*** %0, align 8
+  ret i8* %call
+}
+
+declare dso_local noalias nonnull i8* @_Znwm(i64)
+
+; CHECK: [[MD_TU_VIS]] = !{i64 2}
+!0 = !{i64 16, !"_ZTS1A"}
+!1 = !{i64 16, !"_ZTSM1AFvvE.virtual"}
+!2 = !{i64 16, !"_ZTS1B"}
+!3 = !{i64 16, !"_ZTSM1BFvvE.virtual"}
+!4 = !{i64 16, !"_ZTS1C"}
+!5 = !{i64 16, !"_ZTSM1CFvvE.virtual"}
+!6 = !{i64 1}
+!7 = !{i64 16, !8}
+!8 = distinct !{}
+!9 = !{i64 16, !10}
+!10 = distinct !{}
+!11 = !{i64 2}
diff --git a/llvm/test/Transforms/JumpThreading/bb-unreachable-from-entry.ll b/llvm/test/Transforms/JumpThreading/bb-unreachable-from-entry.ll
index 5900350f5..958811764 100644
--- a/llvm/test/Transforms/JumpThreading/bb-unreachable-from-entry.ll
+++ b/llvm/test/Transforms/JumpThreading/bb-unreachable-from-entry.ll
@@ -1,8 +1,11 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -S < %s -jump-threading | FileCheck %s
-; CHECK-LABEL: @foo
-; CHECK-NEXT: exit2:
-; CHECK-NEXT: ret void
+
 define void @foo() {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  exit2:
+; CHECK-NEXT:    ret void
+;
 entry:
   br label %bb1
 
diff --git a/llvm/test/Transforms/JumpThreading/ddt-crash3.ll b/llvm/test/Transforms/JumpThreading/ddt-crash3.ll
index 50ac86a3f..f36f0410f 100644
--- a/llvm/test/Transforms/JumpThreading/ddt-crash3.ll
+++ b/llvm/test/Transforms/JumpThreading/ddt-crash3.ll
@@ -32,7 +32,7 @@ bb27:                                             ; preds = %bb1
   br label %bb26
 }
 
-attributes #0 = { norecurse noreturn nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { norecurse noreturn nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.ident = !{!0}
 
diff --git a/llvm/test/Transforms/JumpThreading/header-succ.ll b/llvm/test/Transforms/JumpThreading/header-succ.ll
index 859d44cff..d4da77c5d 100644
--- a/llvm/test/Transforms/JumpThreading/header-succ.ll
+++ b/llvm/test/Transforms/JumpThreading/header-succ.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -S -jump-threading < %s | FileCheck %s
 
 ; Check that the heuristic for avoiding accidental introduction of irreducible
@@ -7,93 +8,126 @@
 declare void @opaque_body()
 
 define void @jump_threading_loopheader() {
-; CHECK-LABEL: @jump_threading_loopheader
+; CHECK-LABEL: @jump_threading_loopheader(
+; CHECK-NEXT:  top:
+; CHECK-NEXT:    br label [[ENTRY:%.*]]
+; CHECK:       entry:
+; CHECK-NEXT:    [[IND:%.*]] = phi i32 [ 0, [[TOP:%.*]] ], [ [[NEXTIND:%.*]], [[LATCH:%.*]] ]
+; CHECK-NEXT:    [[NEXTIND]] = add i32 [[IND]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ule i32 [[IND]], 10
+; CHECK-NEXT:    br i1 [[CMP]], label [[LATCH]], label [[EXIT:%.*]]
+; CHECK:       latch:
+; CHECK-NEXT:    call void @opaque_body()
+; CHECK-NEXT:    br label [[ENTRY]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
 top:
-    br label %entry
+  br label %entry
 
 entry:
-    %ind = phi i32 [0, %top], [%nextind, %latch]
-    %nextind = add i32 %ind, 1
-    %cmp = icmp ule i32 %ind, 10
-; CHECK: br i1 %cmp, label %latch, label %exit
-    br i1 %cmp, label %body, label %latch
+  %ind = phi i32 [0, %top], [%nextind, %latch]
+  %nextind = add i32 %ind, 1
+  %cmp = icmp ule i32 %ind, 10
+  br i1 %cmp, label %body, label %latch
 
 body:
-    call void @opaque_body()
-; CHECK: br label %entry
-    br label %latch
+  call void @opaque_body()
+  br label %latch
 
 latch:
-    %cond = phi i2 [1, %entry], [2, %body]
-    switch i2 %cond, label %unreach [
-        i2 2, label %entry
-        i2 1, label %exit
-    ]
+  %cond = phi i2 [1, %entry], [2, %body]
+  switch i2 %cond, label %unreach [
+  i2 2, label %entry
+  i2 1, label %exit
+  ]
 
 unreach:
-    unreachable
+  unreachable
 
 exit:
-    ret void
+  ret void
 }
 
 ; We also need to check the opposite order of the branches, in the switch
 ; instruction because jump-threading relies on that to decide which edge to
 ; try to thread first.
 define void @jump_threading_loopheader2() {
-; CHECK-LABEL: @jump_threading_loopheader2
+; CHECK-LABEL: @jump_threading_loopheader2(
+; CHECK-NEXT:  top:
+; CHECK-NEXT:    br label [[ENTRY:%.*]]
+; CHECK:       entry:
+; CHECK-NEXT:    [[IND:%.*]] = phi i32 [ 0, [[TOP:%.*]] ], [ [[NEXTIND:%.*]], [[LATCH:%.*]] ]
+; CHECK-NEXT:    [[NEXTIND]] = add i32 [[IND]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ule i32 [[IND]], 10
+; CHECK-NEXT:    br i1 [[CMP]], label [[EXIT:%.*]], label [[LATCH]]
+; CHECK:       latch:
+; CHECK-NEXT:    br label [[ENTRY]]
+; CHECK:       exit:
+; CHECK-NEXT:    call void @opaque_body()
+; CHECK-NEXT:    ret void
+;
 top:
-    br label %entry
+  br label %entry
 
 entry:
-    %ind = phi i32 [0, %top], [%nextind, %latch]
-    %nextind = add i32 %ind, 1
-    %cmp = icmp ule i32 %ind, 10
-; CHECK: br i1 %cmp, label %exit, label %latch
-    br i1 %cmp, label %body, label %latch
+  %ind = phi i32 [0, %top], [%nextind, %latch]
+  %nextind = add i32 %ind, 1
+  %cmp = icmp ule i32 %ind, 10
+  br i1 %cmp, label %body, label %latch
 
 body:
-    call void @opaque_body()
-; CHECK: br label %entry
-    br label %latch
+  call void @opaque_body()
+  br label %latch
 
 latch:
-    %cond = phi i2 [1, %entry], [2, %body]
-    switch i2 %cond, label %unreach [
-        i2 1, label %entry
-        i2 2, label %exit
-    ]
+  %cond = phi i2 [1, %entry], [2, %body]
+  switch i2 %cond, label %unreach [
+  i2 1, label %entry
+  i2 2, label %exit
+  ]
 
 unreach:
-    unreachable
+  unreachable
 
 exit:
-    ret void
+  ret void
 }
 
 ; Check if we can handle undef branch condition.
 define void @jump_threading_loopheader3() {
-; CHECK-LABEL: @jump_threading_loopheader3
+; CHECK-LABEL: @jump_threading_loopheader3(
+; CHECK-NEXT:  top:
+; CHECK-NEXT:    br label [[ENTRY:%.*]]
+; CHECK:       entry:
+; CHECK-NEXT:    [[IND:%.*]] = phi i32 [ 0, [[TOP:%.*]] ], [ [[NEXTIND:%.*]], [[LATCH:%.*]] ]
+; CHECK-NEXT:    [[NEXTIND]] = add i32 [[IND]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ule i32 [[IND]], 10
+; CHECK-NEXT:    br i1 [[CMP]], label [[LATCH]], label [[EXIT:%.*]]
+; CHECK:       latch:
+; CHECK-NEXT:    call void @opaque_body()
+; CHECK-NEXT:    br label [[ENTRY]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
 top:
-    br label %entry
+  br label %entry
 
 entry:
-    %ind = phi i32 [0, %top], [%nextind, %latch]
-    %nextind = add i32 %ind, 1
-    %cmp = icmp ule i32 %ind, 10
-; CHECK: br i1 %cmp, label %latch, label %exit
-    br i1 %cmp, label %body, label %latch
+  %ind = phi i32 [0, %top], [%nextind, %latch]
+  %nextind = add i32 %ind, 1
+  %cmp = icmp ule i32 %ind, 10
+  br i1 %cmp, label %body, label %latch
 
 body:
-    call void @opaque_body()
-; CHECK: br label %entry
-    br label %latch
+  call void @opaque_body()
+  br label %latch
 
 latch:
-   %phi = phi i32 [undef, %entry], [0, %body]
-   %cmp1 = icmp eq i32 %phi, 0
-   br i1 %cmp1, label %entry, label %exit
+  %phi = phi i32 [undef, %entry], [0, %body]
+  %cmp1 = icmp eq i32 %phi, 0
+  br i1 %cmp1, label %entry, label %exit
 
 exit:
-    ret void
+  ret void
 }
diff --git a/llvm/test/Transforms/JumpThreading/induction.ll b/llvm/test/Transforms/JumpThreading/induction.ll
index 714c28d14..6124aab33 100644
--- a/llvm/test/Transforms/JumpThreading/induction.ll
+++ b/llvm/test/Transforms/JumpThreading/induction.ll
@@ -1,15 +1,22 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -S -jump-threading < %s | FileCheck %s
 
 define i8 @test(i32 %a, i32 %length) {
-; CHECK-LABEL: @test
+; CHECK-LABEL: @test(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[BACKEDGE:%.*]]
+; CHECK:       backedge:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[BACKEDGE]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i32 [[IV]], 1
+; CHECK-NEXT:    [[CONT:%.*]] = icmp slt i32 [[IV_NEXT]], 400
+; CHECK-NEXT:    br i1 [[CONT]], label [[BACKEDGE]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i8 0
+;
 entry:
-; CHECK: br label %backedge
   br label %loop
 
 loop:
-; CHECK-LABEL: backedge:
-; CHECK: phi i32
-; CHECK: br i1 %cont, label %backedge, label %exit
   %iv = phi i32 [0, %entry], [%iv.next, %backedge]
   ;; We can use an inductive argument to prove %iv is always positive
   %cnd = icmp sge i32 %iv, 0
diff --git a/llvm/test/Transforms/JumpThreading/thread-loads.ll b/llvm/test/Transforms/JumpThreading/thread-loads.ll
index 8e2edbddc..47ef01ebe 100644
--- a/llvm/test/Transforms/JumpThreading/thread-loads.ll
+++ b/llvm/test/Transforms/JumpThreading/thread-loads.ll
@@ -496,7 +496,7 @@ declare void @fn3(i64)
 ; merge fully redudant and then we can jump-thread the block with the
 ; store.
 ;
-; CHECK-LABEL: define i32 @phi_translate_partial_redundant_loads(i32, i32*, i32*
+; CHECK-LABEL: define i32 @phi_translate_partial_redundant_loads(i32 %0, i32* %1, i32* %2
 ; CHECK: merge.thread:
 ; CHECK: store
 ; CHECK: br label %left_x
diff --git a/llvm/test/Transforms/JumpThreading/unreachable-loops.ll b/llvm/test/Transforms/JumpThreading/unreachable-loops.ll
new file mode 100644
index 000000000..3f75aeae9
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/unreachable-loops.ll
@@ -0,0 +1,63 @@
+; RUN: opt -jump-threading -S < %s | FileCheck %s
+; RUN: opt -passes=jump-threading -S < %s | FileCheck %s
+; Check the unreachable loop won't cause infinite loop
+; in jump-threading when it tries to update the predecessors'
+; profile metadata from a phi node.
+
+define void @unreachable_single_bb_loop() {
+; CHECK-LABEL: @unreachable_single_bb_loop()
+bb:
+  %tmp = call i32 @a()
+  %tmp1 = icmp eq i32 %tmp, 1
+  br i1 %tmp1, label %bb5, label %bb8
+
+; unreachable single bb loop.
+bb2:                                              ; preds = %bb2
+  %tmp4 = icmp ne i32 %tmp, 1
+  switch i1 %tmp4, label %bb2 [
+    i1 0, label %bb5
+    i1 1, label %bb8
+  ]
+
+bb5:                                              ; preds = %bb2, %bb
+  %tmp6 = phi i1 [ %tmp1, %bb ], [ false, %bb2 ]
+  br i1 %tmp6, label %bb8, label %bb7, !prof !0
+
+bb7:                                              ; preds = %bb5
+  br label %bb8
+
+bb8:                                              ; preds = %bb8, %bb7, %bb5, %bb2
+  ret void
+}
+
+define void @unreachable_multi_bbs_loop() {
+; CHECK-LABEL: @unreachable_multi_bbs_loop()
+bb:
+  %tmp = call i32 @a()
+  %tmp1 = icmp eq i32 %tmp, 1
+  br i1 %tmp1, label %bb5, label %bb8
+
+; unreachable two bbs loop.
+bb3:                                              ; preds = %bb2
+  br label %bb2
+
+bb2:                                              ; preds = %bb3
+  %tmp4 = icmp ne i32 %tmp, 1
+  switch i1 %tmp4, label %bb3 [
+    i1 0, label %bb5
+    i1 1, label %bb8
+  ]
+
+bb5:                                              ; preds = %bb2, %bb
+  %tmp6 = phi i1 [ %tmp1, %bb ], [ false, %bb2 ]
+  br i1 %tmp6, label %bb8, label %bb7, !prof !0
+
+bb7:                                              ; preds = %bb5
+  br label %bb8
+
+bb8:                                              ; preds = %bb8, %bb7, %bb5, %bb2
+  ret void
+}
+declare i32 @a()
+
+!0 = !{!"branch_weights", i32 2146410443, i32 1073205}
diff --git a/llvm/test/Transforms/JumpThreading/update-edge-weight.ll b/llvm/test/Transforms/JumpThreading/update-edge-weight.ll
index 58cd71861..e6a609327 100644
--- a/llvm/test/Transforms/JumpThreading/update-edge-weight.ll
+++ b/llvm/test/Transforms/JumpThreading/update-edge-weight.ll
@@ -1,4 +1,5 @@
 ; RUN: opt -S -jump-threading %s | FileCheck %s
+; RUN: opt -S -passes=jump-threading %s | FileCheck %s
 
 ; Test if edge weights are properly updated after jump threading.
 
diff --git a/llvm/test/Transforms/LCSSA/pr44058.ll b/llvm/test/Transforms/LCSSA/pr44058.ll
new file mode 100644
index 000000000..b6fa92ab7
--- /dev/null
+++ b/llvm/test/Transforms/LCSSA/pr44058.ll
@@ -0,0 +1,37 @@
+; RUN: opt -passes="verify<scalar-evolution>,lcssa,verify<scalar-evolution>" -verify-scev-strict -S %s
+
+; The first SCEV verification is required because it queries SCEV and populates
+; SCEV caches. Second SCEV verification checks if the caches are in valid state.
+
+; Check that the second SCEV verification doesn't fail.
+define void @foo(i32* %arg, i32* %arg1, i1 %arg2) {
+bb:
+  br label %bb3
+
+bb3:                                              ; preds = %bb13, %bb
+  %tmp = load i32, i32* %arg
+  %tmp4 = load i32, i32* %arg1
+  %tmp5 = add i32 %tmp4, %tmp
+  %tmp6 = icmp sgt i32 %tmp5, %tmp
+  br i1 %tmp6, label %bb7, label %bb11
+
+bb7:                                              ; preds = %bb3
+  br i1 %arg2, label %bb10, label %bb8
+
+bb8:                                              ; preds = %bb7
+  %tmp9 = add nsw i32 %tmp, 1
+  ret void
+
+bb10:                                             ; preds = %bb7
+  br label %bb11
+
+bb11:                                             ; preds = %bb10, %bb3
+  %tmp12 = phi i32 [ 0, %bb3 ], [ %tmp4, %bb10 ]
+  br label %bb13
+
+bb13:                                             ; preds = %bb13, %bb11
+  %tmp14 = phi i32 [ %tmp15, %bb13 ], [ 0, %bb11 ]
+  %tmp15 = add nuw nsw i32 %tmp14, 1
+  %tmp16 = icmp slt i32 %tmp15, %tmp12
+  br i1 %tmp16, label %bb13, label %bb3
+}
diff --git a/llvm/test/Transforms/LICM/AMDGPU/bitcast.ll b/llvm/test/Transforms/LICM/AMDGPU/bitcast.ll
new file mode 100644
index 000000000..5f4546551
--- /dev/null
+++ b/llvm/test/Transforms/LICM/AMDGPU/bitcast.ll
@@ -0,0 +1,21 @@
+; RUN: opt -licm -mtriple=amdgcn -S -o - %s | FileCheck %s
+
+; CHECK-LABEL: foo
+; CHECK: ret
+define void @foo(i8* %d, <1 x i32>* %s, i32 %idx) {
+entry:
+  br label %for.body
+
+for.body:
+  %v0 = load <1 x i32>, <1 x i32>* %s
+  %v1 = bitcast <1 x i32> %v0 to <4 x i8>
+  br label %for.cond
+
+for.cond:
+  %e0 = extractelement <4 x i8> %v1, i32 %idx
+  store i8 %e0, i8* %d
+  br i1 false, label %for.exit, label %for.body
+
+for.exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LICM/argmemonly-call.ll b/llvm/test/Transforms/LICM/argmemonly-call.ll
index 86c92a558..505025385 100644
--- a/llvm/test/Transforms/LICM/argmemonly-call.ll
+++ b/llvm/test/Transforms/LICM/argmemonly-call.ll
@@ -1,8 +1,10 @@
-; RUN: opt -S -basicaa -licm -licm-n2-threshold=0 %s | FileCheck %s
+; RUN: opt -S -basicaa -licm -licm-n2-threshold=0 -enable-mssa-loop-dependency=false %s | FileCheck %s
+; RUN: opt -S -basicaa -licm -licm-n2-threshold=0 -enable-mssa-loop-dependency=true -verify-memoryssa %s | FileCheck %s --check-prefix=ALIAS-N2
 ; RUN: opt -licm -basicaa -licm-n2-threshold=200 < %s -S | FileCheck %s --check-prefix=ALIAS-N2
+
 ; RUN: opt -aa-pipeline=basic-aa -licm-n2-threshold=0 -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' < %s -S | FileCheck %s
+; RUN: opt -aa-pipeline=basic-aa -licm-n2-threshold=0 -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop-mssa(licm)' < %s -S | FileCheck %s --check-prefix=ALIAS-N2
 ; RUN: opt -aa-pipeline=basic-aa -licm-n2-threshold=200 -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' < %s -S | FileCheck %s --check-prefix=ALIAS-N2
-; RUN: opt -S -basicaa -licm -licm-n2-threshold=0 -enable-mssa-loop-dependency=true -verify-memoryssa %s | FileCheck %s --check-prefix=ALIAS-N2
 
 declare i32 @foo() readonly argmemonly nounwind
 declare i32 @foo2() readonly nounwind
diff --git a/llvm/test/Transforms/LICM/atomics.ll b/llvm/test/Transforms/LICM/atomics.ll
index 23dc60c59..214078e6f 100644
--- a/llvm/test/Transforms/LICM/atomics.ll
+++ b/llvm/test/Transforms/LICM/atomics.ll
@@ -1,5 +1,7 @@
-; RUN: opt < %s -S -basicaa -licm | FileCheck %s
-; RUN: opt -aa-pipeline=basic-aa -passes='require<opt-remark-emit>,loop(licm)' < %s -S | FileCheck %s
+; RUN: opt < %s -S -basicaa -licm -enable-mssa-loop-dependency=false | FileCheck -check-prefixes=CHECK,AST %s
+; RUN: opt < %s -S -basicaa -licm -enable-mssa-loop-dependency=true  | FileCheck -check-prefixes=CHECK,MSSA %s
+; RUN: opt -aa-pipeline=basic-aa -passes='require<opt-remark-emit>,loop(licm)' < %s -S | FileCheck -check-prefixes=CHECK,AST %s
+; RUN: opt -aa-pipeline=basic-aa -passes='require<opt-remark-emit>,loop-mssa(licm)' < %s -S | FileCheck -check-prefixes=CHECK,MSSA %s
 
 ; Check that we can hoist unordered loads
 define i32 @test1(i32* nocapture %y) nounwind uwtable ssp {
@@ -173,11 +175,12 @@ loop:
 end:
   ret i32 %vala
 ; CHECK-LABEL: define i32 @test7b(
-; CHECK-LABEL: entry:
-; CHECK: store i32 5, i32* %x
+; AST-LABEL: entry:
+; AST: store i32 5, i32* %x
 ; CHECK-LABEL: loop:
 ; CHECK: load atomic i32, i32* %y monotonic
 ; CHECK-LABEL: end:
+; MSSA: store i32 5, i32* %x
 ; CHECK: store atomic i32 %{{.+}}, i32* %z unordered, align 4
 }
 
diff --git a/llvm/test/Transforms/LICM/callbr-crash.ll b/llvm/test/Transforms/LICM/callbr-crash.ll
new file mode 100644
index 000000000..e5e7215d5
--- /dev/null
+++ b/llvm/test/Transforms/LICM/callbr-crash.ll
@@ -0,0 +1,18 @@
+; RUN: opt -licm -disable-output < %s
+
+define i32 @j() {
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %cond.true.i, %entry
+  callbr void asm sideeffect "", "X,~{dirflag},~{fpsr},~{flags}"(i8* blockaddress(@j, %for.end))
+          to label %cond.true.i [label %for.end]
+
+cond.true.i:                                      ; preds = %for.cond
+  %asmresult1.i.i = extractvalue { i8, i32 } zeroinitializer, 1
+  br i1 undef, label %for.end, label %for.cond
+
+for.end:                                          ; preds = %cond.true.i, %for.cond
+  %asmresult1.i.i2 = phi i32 [ %asmresult1.i.i, %cond.true.i ], [ undef, %for.cond ]
+  ret i32 undef
+}
diff --git a/llvm/test/Transforms/LICM/explicit_guards.ll b/llvm/test/Transforms/LICM/explicit_guards.ll
index c9e091359..a684ebe9d 100644
--- a/llvm/test/Transforms/LICM/explicit_guards.ll
+++ b/llvm/test/Transforms/LICM/explicit_guards.ll
@@ -2,21 +2,54 @@
 ; RUN: opt -S -make-guards-explicit -basicaa -licm < %s        | FileCheck %s
 ; RUN: opt -S -aa-pipeline=basic-aa -passes='require<opt-remark-emit>,make-guards-explicit,loop(licm)' < %s | FileCheck %s
 
-; Test interaction between explicit guards and LICM: make sure that we do not
-; hoist explicit conditions while we can hoist invariant loads in presence of
-; explicit guards.
-
 declare void @llvm.experimental.guard(i1,...)
+declare void @maythrow()
 
 ; Make sure that we do not hoist widenable_cond out of loop.
-define void @do_not_hoist_widenable_cond(i1 %cond, i32 %N, i32 %M) {
-; CHECK-LABEL: @do_not_hoist_widenable_cond(
+define void @hoist_widenable_cond(i1 %cond, i32 %N, i32 %M) {
+; CHECK-LABEL: @hoist_widenable_cond(
 ; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[GUARDED:%.*]] ]
 ; CHECK-NEXT:    [[GUARD_COND:%.*]] = icmp slt i32 [[IV]], [[N:%.*]]
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[GUARD_COND]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded:
+; CHECK-NEXT:    [[LOOP_COND:%.*]] = icmp slt i32 [[IV]], [[M:%.*]]
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    br i1 [[LOOP_COND]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+  %guard_cond = icmp slt i32 %iv, %N
+  call void(i1, ...) @llvm.experimental.guard(i1 %guard_cond) [ "deopt"() ]
+  %loop_cond = icmp slt i32 %iv, %M
+  %iv.next = add i32 %iv, 1
+  br i1 %loop_cond, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+define void @hoist_widenable_cond_speculate(i1 %cond, i32 %N, i32 %M) {
+; CHECK-LABEL: @hoist_widenable_cond_speculate(
+; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[GUARDED:%.*]] ]
+; CHECK-NEXT:    [[GUARD_COND:%.*]] = icmp slt i32 [[IV]], [[N:%.*]]
+; CHECK-NEXT:    call void @maythrow()
 ; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[GUARD_COND]], [[WIDENABLE_COND]]
 ; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
 ; CHECK:       deopt:
@@ -35,6 +68,7 @@ entry:
 loop:
   %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
   %guard_cond = icmp slt i32 %iv, %N
+  call void @maythrow()
   call void(i1, ...) @llvm.experimental.guard(i1 %guard_cond) [ "deopt"() ]
   %loop_cond = icmp slt i32 %iv, %M
   %iv.next = add i32 %iv, 1
@@ -44,15 +78,16 @@ exit:
   ret void
 }
 
+
 define void @hoist_invariant_load(i1 %cond, i32* %np, i32 %M) {
 ; CHECK-LABEL: @hoist_invariant_load(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[N:%.*]] = load i32, i32* [[NP:%.*]]
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[GUARDED:%.*]] ]
 ; CHECK-NEXT:    [[GUARD_COND:%.*]] = icmp slt i32 [[IV]], [[N]]
-; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
 ; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[GUARD_COND]], [[WIDENABLE_COND]]
 ; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
 ; CHECK:       deopt:
diff --git a/llvm/test/Transforms/LICM/guards.ll b/llvm/test/Transforms/LICM/guards.ll
index 6c25eb167..2343e0917 100644
--- a/llvm/test/Transforms/LICM/guards.ll
+++ b/llvm/test/Transforms/LICM/guards.ll
@@ -1,7 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; REQUIRES: asserts
-; RUN: opt -licm -basicaa -ipt-expensive-asserts=true < %s -S | FileCheck %s
+; RUN: opt -licm -basicaa -enable-mssa-loop-dependency=false -ipt-expensive-asserts=true < %s -S | FileCheck %s
+; RUN: opt -licm -basicaa -enable-mssa-loop-dependency=true -ipt-expensive-asserts=true < %s -S | FileCheck %s --check-prefixes=CHECK,MSSA
 ; RUN: opt -aa-pipeline=basic-aa -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' -ipt-expensive-asserts=true < %s -S | FileCheck %s
+; RUN: opt -aa-pipeline=basic-aa -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop-mssa(licm)' -ipt-expensive-asserts=true < %s -S | FileCheck %s --check-prefixes=CHECK,MSSA
 
 ; Hoist guard and load.
 define void @test1(i1 %cond, i32* %ptr) {
@@ -61,13 +63,40 @@ define void @test2b(i1 %cond, i32* %ptr) {
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[X:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[X_INC:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    store i32 0, i32* [[P2]]
+; CHECK-NEXT:    store i32 [[X]], i32* [[P2]]
 ; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[COND:%.*]]) [ "deopt"(i32 0) ]
 ; CHECK-NEXT:    [[VAL:%.*]] = load i32, i32* [[PTR]]
 ; CHECK-NEXT:    [[X_INC]] = add i32 [[X]], [[VAL]]
 ; CHECK-NEXT:    br label [[LOOP]]
 ;
 
+entry:
+  br label %loop
+
+loop:
+  %x = phi i32 [ 0, %entry ], [ %x.inc, %loop ]
+  %p2 = getelementptr i32, i32* %ptr, i32 1
+  store i32 %x, i32* %p2
+  call void (i1, ...) @llvm.experimental.guard(i1 %cond) ["deopt" (i32 0)]
+  %val = load i32, i32* %ptr
+  %x.inc = add i32 %x, %val
+  br label %loop
+}
+
+; But can hoist if the side effect is hoisted with MSSA
+define void @test2b_prime(i1 %cond, i32* noalias %ptr) {
+; MSSA-LABEL: @test2b_prime(
+; MSSA-NEXT:  entry:
+; MSSA-NEXT:    [[P2:%.*]] = getelementptr i32, i32* [[PTR:%.*]], i32 1
+; MSSA-NEXT:    store i32 0, i32* [[P2]]
+; MSSA-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[COND:%.*]]) [ "deopt"(i32 0) ]
+; MSSA-NEXT:    [[VAL:%.*]] = load i32, i32* [[PTR]]
+; MSSA-NEXT:    br label [[LOOP:%.*]]
+; MSSA:       loop:
+; MSSA-NEXT:    [[X:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[X_INC:%.*]], [[LOOP]] ]
+; MSSA-NEXT:    [[X_INC]] = add i32 [[X]], [[VAL]]
+; MSSA-NEXT:    br label [[LOOP]]
+
 entry:
   br label %loop
 
diff --git a/llvm/test/Transforms/LICM/hoist-deref-load.ll b/llvm/test/Transforms/LICM/hoist-deref-load.ll
index aacff88ac..9a486fd5b 100644
--- a/llvm/test/Transforms/LICM/hoist-deref-load.ll
+++ b/llvm/test/Transforms/LICM/hoist-deref-load.ll
@@ -1,7 +1,7 @@
 ; RUN: opt -S -basicaa -licm < %s | FileCheck %s
 ; RUN: opt -aa-pipeline=basic-aa -passes='require<opt-remark-emit>,loop(simplify-cfg,licm)' -S < %s | FileCheck %s
 ; RUN: opt -S -basicaa -licm -enable-mssa-loop-dependency=true -verify-memoryssa < %s | FileCheck %s
-; RUN: opt -aa-pipeline=basic-aa -passes='require<opt-remark-emit>,loop(simplify-cfg,licm)' -enable-mssa-loop-dependency=true -verify-memoryssa -S < %s | FileCheck %s
+; RUN: opt -aa-pipeline=basic-aa -passes='require<opt-remark-emit>,loop-mssa(simplify-cfg,licm)' -verify-memoryssa -S < %s | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/Transforms/LICM/hoist-mustexec.ll b/llvm/test/Transforms/LICM/hoist-mustexec.ll
index 521d35296..59184eb54 100644
--- a/llvm/test/Transforms/LICM/hoist-mustexec.ll
+++ b/llvm/test/Transforms/LICM/hoist-mustexec.ll
@@ -129,8 +129,6 @@ fail:
 }
 
 ; requires fact length is non-zero
-; TODO: IsKnownNonNullFromDominatingConditions is currently only be done for
-; pointers; should handle integers too
 define i32 @test4(i32* noalias nocapture readonly %a) nounwind uwtable {
 ; CHECK-LABEL: @test4(
 ; CHECK-NEXT:  entry:
@@ -138,6 +136,7 @@ define i32 @test4(i32* noalias nocapture readonly %a) nounwind uwtable {
 ; CHECK-NEXT:    [[IS_ZERO:%.*]] = icmp eq i32 [[LEN]], 0
 ; CHECK-NEXT:    br i1 [[IS_ZERO]], label [[FAIL:%.*]], label [[PREHEADER:%.*]]
 ; CHECK:       preheader:
+; CHECK-NEXT:    [[I1:%.*]] = load i32, i32* [[A]], align 4
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[PREHEADER]] ], [ [[INC:%.*]], [[CONTINUE:%.*]] ]
@@ -145,7 +144,6 @@ define i32 @test4(i32* noalias nocapture readonly %a) nounwind uwtable {
 ; CHECK-NEXT:    [[R_CHK:%.*]] = icmp ult i32 [[IV]], [[LEN]]
 ; CHECK-NEXT:    br i1 [[R_CHK]], label [[CONTINUE]], label [[FAIL_LOOPEXIT:%.*]]
 ; CHECK:       continue:
-; CHECK-NEXT:    [[I1:%.*]] = load i32, i32* [[A]], align 4
 ; CHECK-NEXT:    [[ADD]] = add nsw i32 [[I1]], [[ACC]]
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[IV]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 1000
diff --git a/llvm/test/Transforms/LICM/hoist-phi.ll b/llvm/test/Transforms/LICM/hoist-phi.ll
index f3a137db5..26970eff2 100644
--- a/llvm/test/Transforms/LICM/hoist-phi.ll
+++ b/llvm/test/Transforms/LICM/hoist-phi.ll
@@ -5,9 +5,9 @@
 ; RUN: opt -passes='require<opt-remark-emit>,loop(licm)' -licm-control-flow-hoisting=1 -S < %s | FileCheck %s -check-prefixes=CHECK,CHECK-ENABLED
 ; RUN: opt -passes='require<opt-remark-emit>,loop(licm)' -licm-control-flow-hoisting=0 -S < %s | FileCheck %s -check-prefixes=CHECK,CHECK-DISABLED
 
-; RUN: opt -passes='require<opt-remark-emit>,loop(licm)' -licm-control-flow-hoisting=1 -enable-mssa-loop-dependency=true -verify-memoryssa -S < %s | FileCheck %s -check-prefixes=CHECK,CHECK-ENABLED
+; RUN: opt -passes='require<opt-remark-emit>,loop-mssa(licm)' -licm-control-flow-hoisting=1 -verify-memoryssa -S < %s | FileCheck %s -check-prefixes=CHECK,CHECK-ENABLED
 ; Enable run below when adding promotion. e.g. "store i32 %phi, i32* %p" is promoted to phi.lcssa.
-; opt -passes='require<opt-remark-emit>,loop(licm)' -licm-control-flow-hoisting=0 -enable-mssa-loop-dependency=true -verify-memoryssa -S < %s | FileCheck %s -check-prefixes=CHECK,CHECK-DISABLED
+; opt -passes='require<opt-remark-emit>,loop-mssa(licm)' -licm-control-flow-hoisting=0 -verify-memoryssa -S < %s | FileCheck %s -check-prefixes=CHECK,CHECK-DISABLED
 
 
 ; CHECK-LABEL: @triangle_phi
diff --git a/llvm/test/Transforms/LICM/hoisting.ll b/llvm/test/Transforms/LICM/hoisting.ll
index f65b965d5..97609fa39 100644
--- a/llvm/test/Transforms/LICM/hoisting.ll
+++ b/llvm/test/Transforms/LICM/hoisting.ll
@@ -126,10 +126,32 @@ ifend:                                            ; preds = %tailrecurse
   ret { i32*, i32 } %d
 }
 
-; CHECK: define i32 @hoist_bitreverse(i32)
+; CHECK: define void @test6(float %f)
+; CHECK: fneg
+; CHECK: br label %for.body
+define void @test6(float %f) #2 {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %i = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  call void @foo_may_call_exit(i32 0)
+  %neg = fneg float %f
+  call void @use(float %neg)
+  %inc = add nsw i32 %i, 1
+  %cmp = icmp slt i32 %inc, 10000
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+declare void @use(float)
+
+; CHECK: define i32 @hoist_bitreverse(i32 %0)
 ; CHECK: bitreverse
 ; CHECK: br label %header
-define i32 @hoist_bitreverse(i32)  {
+define i32 @hoist_bitreverse(i32 %0)  {
   br label %header
 
 header:
diff --git a/llvm/test/Transforms/LICM/pr38513.ll b/llvm/test/Transforms/LICM/pr38513.ll
new file mode 100644
index 000000000..ebc275106
--- /dev/null
+++ b/llvm/test/Transforms/LICM/pr38513.ll
@@ -0,0 +1,27 @@
+; RUN: opt -enable-mssa-loop-dependency=false -disable-basicaa -alias-set-saturation-threshold=2 -licm -S < %s | FileCheck %s
+; REQUIRES: asserts
+
+; CHECK-LABEL: @f1()
+define void @f1() {
+  %lc1.10 = alloca [3 x i16]
+  br label %bb1
+
+bb1:                                              ; preds = %bb6, %0
+  store i16 undef, i16* undef
+  br label %bb2
+
+bb2:                                              ; preds = %bb8, %bb1
+  %_tmp18.fca.0.gep = getelementptr inbounds [3 x i16], [3 x i16]* %lc1.10, i32 0, i32 0
+  %_tmp18.fca.0.load = load i16, i16* %_tmp18.fca.0.gep
+  %_tmp18.fca.1.gep = getelementptr inbounds [3 x i16], [3 x i16]* %lc1.10, i32 0, i32 1
+  %_tmp18.fca.1.load = load i16, i16* %_tmp18.fca.1.gep
+  %_tmp18.fca.2.gep = getelementptr inbounds [3 x i16], [3 x i16]* %lc1.10, i32 0, i32 2
+  %_tmp18.fca.2.load = load i16, i16* %_tmp18.fca.2.gep
+  br label %bb8
+
+bb8:                                              ; preds = %bb2
+  br i1 undef, label %bb2, label %bb6
+
+bb6:                                              ; preds = %bb8
+  br label %bb1
+}
diff --git a/llvm/test/Transforms/LICM/pr42969.ll b/llvm/test/Transforms/LICM/pr42969.ll
new file mode 100644
index 000000000..7fa36f0cf
--- /dev/null
+++ b/llvm/test/Transforms/LICM/pr42969.ll
@@ -0,0 +1,70 @@
+; RUN: opt %s -S -scoped-noalias -enable-mssa-loop-dependency=true -licm | FileCheck %s
+
+define i16 @main(i1 %a_b_mayalias, i16* %a, i16* %b) {
+; CHECK:       scalar.body:
+; CHECK-NEXT:    [[J:%.*]] = phi i64
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16
+; CHECK-NEXT:    [[RESULT:%.*]] = add i16 [[TMP3]], 1
+; CHECK-NEXT:    store i16 [[RESULT]]
+
+entry:
+  br label %outer
+
+outer:                                            ; preds = %scalar.cleanup, %entry
+; 4 = MemoryPhi({entry,liveOnEntry},{scalar.cleanup,2})
+  %i = phi i16 [ 0, %entry ], [ %i.next, %scalar.cleanup ]
+  br i1 %a_b_mayalias, label %scalar.ph, label %vector.ph
+
+vector.ph:                                        ; preds = %outer
+; MemoryUse(4) MayAlias
+  %tmp1 = load i16, i16* %a, align 1, !alias.scope !0, !tbaa !7
+  %tmp2 = add i16 %tmp1, 1
+; 1 = MemoryDef(4)
+  store i16 %tmp2, i16* %b, align 1, !alias.scope !3, !noalias !0, !tbaa !7
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i64 [ %index.next, %vector.body ], [ 0, %vector.ph ]
+  %index.next = add i64 %index, 1
+  %cmp1 = icmp eq i64 %index.next, 16
+  br i1 %cmp1, label %middle.block, label %vector.body
+
+middle.block:                                     ; preds = %vector.body
+  br label %scalar.ph
+
+scalar.ph:                                        ; preds = %middle.block, %outer
+; 5 = MemoryPhi({outer,4},{middle.block,1})
+  %j.start = phi i64 [ 0, %outer ], [ 16, %middle.block ]
+  br label %scalar.body
+
+scalar.body:                                      ; preds = %scalar.body, %scalar.ph
+; 3 = MemoryPhi({scalar.ph,5},{scalar.body,2})
+  %j = phi i64 [ %j.next, %scalar.body ], [ %j.start, %scalar.ph ]
+; MemoryUse(3) MayAlias
+  %tmp3 = load i16, i16* %a, align 1, !tbaa !7
+  %result = add i16 %tmp3, 1
+; 2 = MemoryDef(3)
+  store i16 %result, i16* %b, align 1, !tbaa !7
+  %j.next = add nuw nsw i64 %j, 1
+  %cmp2 = icmp ult i64 %j.next, 20
+  br i1 %cmp2, label %scalar.body, label %scalar.cleanup
+
+scalar.cleanup:                                   ; preds = %scalar.body
+  %result.lcssa = phi i16 [ %result, %scalar.body ]
+  %i.next = add nuw nsw i16 %i, 1
+  %exitcond = icmp eq i16 %i.next, 10
+  br i1 %exitcond, label %exit.block, label %outer
+
+exit.block:                                       ; preds = %scalar.cleanup
+  %result.lcssa.lcssa = phi i16 [ %result.lcssa, %scalar.cleanup ]
+  ret i16 %result.lcssa.lcssa
+}
+
+!0 = !{!1}
+!1 = distinct !{!1, !2}
+!2 = distinct !{!2, !"LVerDomain"}
+!3 = !{!4}
+!4 = distinct !{!4, !2}
+!5 = !{!"Simple C++ TBAA"}
+!6 = !{!"omnipotent char", !5, i64 0}
+!7 = !{!6, !6, i64 0}
diff --git a/llvm/test/Transforms/LICM/pragma-licm-disable.ll b/llvm/test/Transforms/LICM/pragma-licm-disable.ll
new file mode 100644
index 000000000..cba109943
--- /dev/null
+++ b/llvm/test/Transforms/LICM/pragma-licm-disable.ll
@@ -0,0 +1,33 @@
+; RUN: opt < %s -S -basicaa -licm | FileCheck %s
+
+; Check that the LICM pass does not operate on a loop which has the
+; llvm.licm.disable metadata.
+; CHECK-LABEL: @licm_disable
+; CHECK: entry:
+; CHECK-NOT: load
+; CHECK: do.body:
+; CHECK: load i64, i64* bitcast (i32** @in to i64*)
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc"
+
+@in = internal unnamed_addr global i32* null, align 8
+@out = internal unnamed_addr global i32* null, align 8
+
+define void @licm_disable(i32 %N) {
+entry:
+  br label %do.body
+
+do.body:                                          ; preds = %entry
+  %i.0 = phi i32 [ 0, %entry ], [ %inc, %do.body ]
+  %v1 = load i64, i64* bitcast (i32** @in to i64*), align 8
+  store i64 %v1, i64* bitcast (i32** @out to i64*), align 8
+  %inc = add nsw i32 %i.0, 1
+  %cmp = icmp slt i32 %inc, %N
+  br i1 %cmp, label %do.body, label %do.end, !llvm.loop !1
+
+do.end:                                           ; preds = %do.body
+  ret void
+}
+!1 = !{!1, !2}
+!2 = !{!"llvm.licm.disable"}
diff --git a/llvm/test/Transforms/LICM/promote-order.ll b/llvm/test/Transforms/LICM/promote-order.ll
index 79aa7899b..e187404f3 100644
--- a/llvm/test/Transforms/LICM/promote-order.ll
+++ b/llvm/test/Transforms/LICM/promote-order.ll
@@ -1,5 +1,7 @@
-; RUN: opt -tbaa -basicaa -licm -S < %s | FileCheck %s
-; RUN: opt -aa-pipeline=type-based-aa,basic-aa -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' -S %s | FileCheck %s
+; RUN: opt -tbaa -basicaa -licm -enable-mssa-loop-dependency=false -S < %s | FileCheck %s --check-prefixes=CHECK,AST
+; RUN: opt -tbaa -basicaa -licm -enable-mssa-loop-dependency=true -S < %s | FileCheck %s --check-prefixes=CHECK,MSSA
+; RUN: opt -aa-pipeline=type-based-aa,basic-aa -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' -S %s | FileCheck %s  --check-prefixes=CHECK,AST
+; RUN: opt -aa-pipeline=type-based-aa,basic-aa -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop-mssa(licm)' -S %s | FileCheck %s --check-prefixes=CHECK,MSSA
 
 ; LICM should keep the stores in their original order when it sinks/promotes them.
 ; rdar://12045203
@@ -12,8 +14,9 @@ target triple = "x86_64-apple-macosx10.8.0"
 define i32* @_Z4doiti(i32 %n, float* %tmp1, i32* %tmp3) nounwind {
 ; CHECK-LABEL: for.body.lr.ph:
 ; CHECK: store float 1.000000e+00, float* %tmp1
-; CHECK-LABEL: for.cond.for.end_crit_edge:
+; AST-LABEL: for.cond.for.end_crit_edge:
 ; CHECK: store i32 1, i32* %tmp3
+; MSSA-LABEL: for.cond.for.end_crit_edge:
 
 entry:
   %cmp1 = icmp slt i32 0, %n
diff --git a/llvm/test/Transforms/LICM/read-only-calls.ll b/llvm/test/Transforms/LICM/read-only-calls.ll
index 0a378144f..19e9a118c 100644
--- a/llvm/test/Transforms/LICM/read-only-calls.ll
+++ b/llvm/test/Transforms/LICM/read-only-calls.ll
@@ -1,6 +1,9 @@
-; RUN: opt -S -basicaa -licm -licm-n2-threshold=0 %s | FileCheck %s
+; RUN: opt -S -basicaa -licm -licm-n2-threshold=0 -enable-mssa-loop-dependency=false %s | FileCheck %s
+; RUN: opt -S -basicaa -licm -licm-n2-threshold=0 -enable-mssa-loop-dependency=true %s | FileCheck %s --check-prefix=ALIAS-N2
 ; RUN: opt -licm -basicaa -licm-n2-threshold=200 < %s -S | FileCheck %s --check-prefix=ALIAS-N2
+
 ; RUN: opt -aa-pipeline=basic-aa -licm-n2-threshold=0 -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' < %s -S | FileCheck %s
+; RUN: opt -aa-pipeline=basic-aa -licm-n2-threshold=0 -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop-mssa(licm)' < %s -S | FileCheck %s --check-prefix=ALIAS-N2
 ; RUN: opt -aa-pipeline=basic-aa -licm-n2-threshold=200 -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' < %s -S | FileCheck %s --check-prefix=ALIAS-N2
 
 ; We should be able to hoist loads in presence of read only calls and stores
diff --git a/llvm/test/Transforms/LICM/store-hoisting.ll b/llvm/test/Transforms/LICM/store-hoisting.ll
index 95994eccc..c1df6efdc 100644
--- a/llvm/test/Transforms/LICM/store-hoisting.ll
+++ b/llvm/test/Transforms/LICM/store-hoisting.ll
@@ -1,6 +1,7 @@
-; RUN: opt -S -basicaa -licm %s | FileCheck -check-prefixes=CHECK,AST %s
+; RUN: opt -S -basicaa -licm -enable-mssa-loop-dependency=false %s | FileCheck -check-prefixes=CHECK,AST %s
 ; RUN: opt -S -basicaa -licm -enable-mssa-loop-dependency=true %s | FileCheck  -check-prefixes=CHECK,MSSA %s
 ; RUN: opt -aa-pipeline=basic-aa -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' < %s -S | FileCheck -check-prefixes=CHECK,AST %s
+; RUN: opt -aa-pipeline=basic-aa -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop-mssa(licm)' < %s -S | FileCheck -check-prefixes=CHECK,MSSA %s
 
 define void @test(i32* %loc) {
 ; CHECK-LABEL: @test
diff --git a/llvm/test/Transforms/LICM/volatile-alias.ll b/llvm/test/Transforms/LICM/volatile-alias.ll
index c1f727c4e..605720a3b 100644
--- a/llvm/test/Transforms/LICM/volatile-alias.ll
+++ b/llvm/test/Transforms/LICM/volatile-alias.ll
@@ -53,4 +53,4 @@ for.end:                                          ; preds = %for.cond
   ret i32 %8
 }
 
-attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/vect-ptr-ptr-size-mismatch.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/vect-ptr-ptr-size-mismatch.ll
new file mode 100644
index 000000000..1cb8d14f1
--- /dev/null
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/vect-ptr-ptr-size-mismatch.ll
@@ -0,0 +1,96 @@
+; RUN: opt -load-store-vectorizer -S < %s | FileCheck %s
+
+target datalayout = "e-p:64:64-p1:64:64-p5:32:32"
+
+; Size mismatch between the 32 bit pointer in address space 5 and 64 bit
+; pointer in address space 0 it was cast to caused below test to crash.
+; The p5:32:32 portion of the data layout is critical for the test.
+
+; CHECK-LABEL: @cast_to_ptr
+; CHECK: store i32* undef, i32** %tmp9, align 8
+; CHECK: store i32* undef, i32** %tmp7, align 8
+define void @cast_to_ptr() {
+entry:
+  %ascast = addrspacecast i32* addrspace(5)* null to i32**
+  %tmp4 = icmp eq i32 undef, 0
+  %tmp6 = select i1 false, i32** undef, i32** undef
+  %tmp7 = select i1 %tmp4, i32** null, i32** %tmp6
+  %tmp9 = select i1 %tmp4, i32** %ascast, i32** null
+  store i32* undef, i32** %tmp9, align 8
+  store i32* undef, i32** %tmp7, align 8
+  unreachable
+}
+
+; CHECK-LABEL: @cast_to_cast
+; CHECK: %tmp4 = load i32*, i32** %tmp1, align 8
+; CHECK: %tmp5 = load i32*, i32** %tmp3, align 8
+define void @cast_to_cast() {
+entry:
+  %a.ascast = addrspacecast i32* addrspace(5)* undef to i32**
+  %b.ascast = addrspacecast i32* addrspace(5)* null to i32**
+  %tmp1 = select i1 false, i32** %a.ascast, i32** undef
+  %tmp3 = select i1 false, i32** %b.ascast, i32** undef
+  %tmp4 = load i32*, i32** %tmp1, align 8
+  %tmp5 = load i32*, i32** %tmp3, align 8
+  unreachable
+}
+
+; CHECK-LABEL: @all_to_cast
+; CHECK: load <4 x float>
+define void @all_to_cast(i8* nocapture readonly align 16 dereferenceable(16) %alloc1) {
+entry:
+  %alloc16 = addrspacecast i8* %alloc1 to i8 addrspace(1)*
+  %tmp = bitcast i8 addrspace(1)* %alloc16 to float addrspace(1)*
+  %tmp1 = load float, float addrspace(1)* %tmp, align 16, !invariant.load !0
+  %tmp6 = getelementptr inbounds i8, i8 addrspace(1)* %alloc16, i64 4
+  %tmp7 = bitcast i8 addrspace(1)* %tmp6 to float addrspace(1)*
+  %tmp8 = load float, float addrspace(1)* %tmp7, align 4, !invariant.load !0
+  %tmp15 = getelementptr inbounds i8, i8 addrspace(1)* %alloc16, i64 8
+  %tmp16 = bitcast i8 addrspace(1)* %tmp15 to float addrspace(1)*
+  %tmp17 = load float, float addrspace(1)* %tmp16, align 8, !invariant.load !0
+  %tmp24 = getelementptr inbounds i8, i8 addrspace(1)* %alloc16, i64 12
+  %tmp25 = bitcast i8 addrspace(1)* %tmp24 to float addrspace(1)*
+  %tmp26 = load float, float addrspace(1)* %tmp25, align 4, !invariant.load !0
+  ret void
+}
+
+; CHECK-LABEL: @ext_ptr
+; CHECK  load <2 x i32>
+define void @ext_ptr(i32 addrspace(5)* %p) {
+entry:
+  %gep1 = getelementptr inbounds i32, i32 addrspace(5)* %p, i64 0
+  %gep2 = getelementptr inbounds i32, i32 addrspace(5)* %p, i64 1
+  %a.ascast = addrspacecast i32 addrspace(5)* %gep1 to i32*
+  %b.ascast = addrspacecast i32 addrspace(5)* %gep2 to i32*
+  %tmp1 = load i32, i32* %a.ascast, align 8
+  %tmp2 = load i32, i32* %b.ascast, align 8
+  unreachable
+}
+
+; CHECK-LABEL: @shrink_ptr
+; CHECK  load <2 x i32>
+define void @shrink_ptr(i32* %p) {
+entry:
+  %gep1 = getelementptr inbounds i32, i32* %p, i64 0
+  %gep2 = getelementptr inbounds i32, i32* %p, i64 1
+  %a.ascast = addrspacecast i32* %gep1 to i32 addrspace(5)*
+  %b.ascast = addrspacecast i32* %gep2 to i32 addrspace(5)*
+  %tmp1 = load i32, i32 addrspace(5)* %a.ascast, align 8
+  %tmp2 = load i32, i32 addrspace(5)* %b.ascast, align 8
+  unreachable
+}
+
+; CHECK-LABEL: @ext_ptr_wrap
+; CHECK: load <2 x i8>
+define void @ext_ptr_wrap(i8 addrspace(5)* %p) {
+entry:
+  %gep1 = getelementptr inbounds i8, i8 addrspace(5)* %p, i64 0
+  %gep2 = getelementptr inbounds i8, i8 addrspace(5)* %p, i64 4294967295
+  %a.ascast = addrspacecast i8 addrspace(5)* %gep1 to i8*
+  %b.ascast = addrspacecast i8 addrspace(5)* %gep2 to i8*
+  %tmp1 = load i8, i8* %a.ascast, align 1
+  %tmp2 = load i8, i8* %b.ascast, align 1
+  unreachable
+}
+
+!0 = !{}
diff --git a/llvm/test/Transforms/LoopFusion/cannot_fuse.ll b/llvm/test/Transforms/LoopFusion/cannot_fuse.ll
index d742a5b03..db099c990 100644
--- a/llvm/test/Transforms/LoopFusion/cannot_fuse.ll
+++ b/llvm/test/Transforms/LoopFusion/cannot_fuse.ll
@@ -15,60 +15,64 @@
 ; CHECK: bb20.preheader
 ; CHECK: ****************************
 ; CHECK: Loop Fusion complete
-define void @non_cfe(i32* noalias %arg) {
+define void @non_cfe(i32* noalias %arg, i32 %N) {
 bb:
-  br label %bb5
-
-bb5:                                              ; preds = %bb14, %bb
-  %indvars.iv2 = phi i64 [ %indvars.iv.next3, %bb14 ], [ 0, %bb ]
-  %.01 = phi i32 [ 0, %bb ], [ %tmp15, %bb14 ]
-  %exitcond4 = icmp ne i64 %indvars.iv2, 100
-  br i1 %exitcond4, label %bb7, label %bb16
+  br label %bb7
 
-bb7:                                              ; preds = %bb5
-  %tmp = add nsw i32 %.01, -3
-  %tmp8 = add nuw nsw i64 %indvars.iv2, 3
+bb7:                                              ; preds = %bb, %bb14
+  %.014 = phi i32 [ 0, %bb ], [ %tmp15, %bb14 ]
+  %indvars.iv23 = phi i64 [ 0, %bb ], [ %indvars.iv.next3, %bb14 ]
+  %tmp = add nsw i32 %.014, -3
+  %tmp8 = add nuw nsw i64 %indvars.iv23, 3
   %tmp9 = trunc i64 %tmp8 to i32
   %tmp10 = mul nsw i32 %tmp, %tmp9
-  %tmp11 = trunc i64 %indvars.iv2 to i32
+  %tmp11 = trunc i64 %indvars.iv23 to i32
   %tmp12 = srem i32 %tmp10, %tmp11
-  %tmp13 = getelementptr inbounds i32, i32* %arg, i64 %indvars.iv2
+  %tmp13 = getelementptr inbounds i32, i32* %arg, i64 %indvars.iv23
   store i32 %tmp12, i32* %tmp13, align 4
   br label %bb14
 
 bb14:                                             ; preds = %bb7
-  %indvars.iv.next3 = add nuw nsw i64 %indvars.iv2, 1
-  %tmp15 = add nuw nsw i32 %.01, 1
-  br label %bb5
+  %indvars.iv.next3 = add nuw nsw i64 %indvars.iv23, 1
+  %tmp15 = add nuw nsw i32 %.014, 1
+  %exitcond4 = icmp ne i64 %indvars.iv.next3, 100
+  br i1 %exitcond4, label %bb7, label %bb34
 
-bb16:                                             ; preds = %bb5
+bb34:
+  %cmp = icmp slt i32 %N, 50
+  br i1 %cmp, label %bb16, label %bb33
+
+bb16:                                             ; preds = %bb34
   %tmp17 = load i32, i32* %arg, align 4
   %tmp18 = icmp slt i32 %tmp17, 0
-  br i1 %tmp18, label %bb20, label %bb33
+  br i1 %tmp18, label %bb20.preheader, label %bb33
 
-bb20:                                             ; preds = %bb30, %bb16
-  %indvars.iv = phi i64 [ %indvars.iv.next, %bb30 ], [ 0, %bb16 ]
-  %.0 = phi i32 [ 0, %bb16 ], [ %tmp31, %bb30 ]
-  %exitcond = icmp ne i64 %indvars.iv, 100
-  br i1 %exitcond, label %bb22, label %bb33
+bb20.preheader:                                   ; preds = %bb16
+  br label %bb22
 
-bb22:                                             ; preds = %bb20
-  %tmp23 = add nsw i32 %.0, -3
-  %tmp24 = add nuw nsw i64 %indvars.iv, 3
+bb22:                                             ; preds = %bb20.preheader, %bb30
+  %.02 = phi i32 [ 0, %bb20.preheader ], [ %tmp31, %bb30 ]
+  %indvars.iv1 = phi i64 [ 0, %bb20.preheader ], [ %indvars.iv.next, %bb30 ]
+  %tmp23 = add nsw i32 %.02, -3
+  %tmp24 = add nuw nsw i64 %indvars.iv1, 3
   %tmp25 = trunc i64 %tmp24 to i32
   %tmp26 = mul nsw i32 %tmp23, %tmp25
-  %tmp27 = trunc i64 %indvars.iv to i32
+  %tmp27 = trunc i64 %indvars.iv1 to i32
   %tmp28 = srem i32 %tmp26, %tmp27
-  %tmp29 = getelementptr inbounds [1024 x i32], [1024 x i32]* @B, i64 0, i64 %indvars.iv
+  %tmp29 = getelementptr inbounds [1024 x i32], [1024 x i32]* @B, i64 0, i64 %indvars.iv1
   store i32 %tmp28, i32* %tmp29, align 4
   br label %bb30
 
 bb30:                                             ; preds = %bb22
-  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
-  %tmp31 = add nuw nsw i32 %.0, 1
-  br label %bb20
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv1, 1
+  %tmp31 = add nuw nsw i32 %.02, 1
+  %exitcond = icmp ne i64 %indvars.iv.next, 100
+  br i1 %exitcond, label %bb22, label %bb33.loopexit
+
+bb33.loopexit:                                    ; preds = %bb30
+  br label %bb33
 
-bb33:                                             ; preds = %bb20, %bb16
+bb33:                                             ; preds = %bb33.loopexit, %bb16, %bb34
   ret void
 }
 
@@ -88,54 +92,48 @@ bb33:                                             ; preds = %bb20, %bb16
 ; CHECK: Loop Fusion complete
 define void @non_adjacent(i32* noalias %arg) {
 bb:
-  br label %bb3
-
-bb3:                                              ; preds = %bb11, %bb
-  %.01 = phi i64 [ 0, %bb ], [ %tmp12, %bb11 ]
-  %exitcond2 = icmp ne i64 %.01, 100
-  br i1 %exitcond2, label %bb5, label %bb4
+  br label %bb5
 
-bb4:                                              ; preds = %bb3
+bb4:                                              ; preds = %bb11
   br label %bb13
 
-bb5:                                              ; preds = %bb3
-  %tmp = add nsw i64 %.01, -3
-  %tmp6 = add nuw nsw i64 %.01, 3
+bb5:                                              ; preds = %bb, %bb11
+  %.013 = phi i64 [ 0, %bb ], [ %tmp12, %bb11 ]
+  %tmp = add nsw i64 %.013, -3
+  %tmp6 = add nuw nsw i64 %.013, 3
   %tmp7 = mul nsw i64 %tmp, %tmp6
-  %tmp8 = srem i64 %tmp7, %.01
+  %tmp8 = srem i64 %tmp7, %.013
   %tmp9 = trunc i64 %tmp8 to i32
-  %tmp10 = getelementptr inbounds i32, i32* %arg, i64 %.01
+  %tmp10 = getelementptr inbounds i32, i32* %arg, i64 %.013
   store i32 %tmp9, i32* %tmp10, align 4
   br label %bb11
 
 bb11:                                             ; preds = %bb5
-  %tmp12 = add nuw nsw i64 %.01, 1
-  br label %bb3
+  %tmp12 = add nuw nsw i64 %.013, 1
+  %exitcond2 = icmp ne i64 %tmp12, 100
+  br i1 %exitcond2, label %bb5, label %bb4
 
 bb13:                                             ; preds = %bb4
-  br label %bb14
-
-bb14:                                             ; preds = %bb23, %bb13
-  %.0 = phi i64 [ 0, %bb13 ], [ %tmp24, %bb23 ]
-  %exitcond = icmp ne i64 %.0, 100
-  br i1 %exitcond, label %bb16, label %bb15
+  br label %bb16
 
-bb15:                                             ; preds = %bb14
+bb15:                                             ; preds = %bb23
   br label %bb25
 
-bb16:                                             ; preds = %bb14
-  %tmp17 = add nsw i64 %.0, -3
-  %tmp18 = add nuw nsw i64 %.0, 3
+bb16:                                             ; preds = %bb13, %bb23
+  %.02 = phi i64 [ 0, %bb13 ], [ %tmp24, %bb23 ]
+  %tmp17 = add nsw i64 %.02, -3
+  %tmp18 = add nuw nsw i64 %.02, 3
   %tmp19 = mul nsw i64 %tmp17, %tmp18
-  %tmp20 = srem i64 %tmp19, %.0
+  %tmp20 = srem i64 %tmp19, %.02
   %tmp21 = trunc i64 %tmp20 to i32
-  %tmp22 = getelementptr inbounds [1024 x i32], [1024 x i32]* @B, i64 0, i64 %.0
+  %tmp22 = getelementptr inbounds [1024 x i32], [1024 x i32]* @B, i64 0, i64 %.02
   store i32 %tmp21, i32* %tmp22, align 4
   br label %bb23
 
 bb23:                                             ; preds = %bb16
-  %tmp24 = add nuw nsw i64 %.0, 1
-  br label %bb14
+  %tmp24 = add nuw nsw i64 %.02, 1
+  %exitcond = icmp ne i64 %tmp24, 100
+  br i1 %exitcond, label %bb16, label %bb15
 
 bb25:                                             ; preds = %bb15
   ret void
@@ -156,54 +154,48 @@ bb25:                                             ; preds = %bb15
 ; CHECK: Loop Fusion complete
 define void @different_bounds(i32* noalias %arg) {
 bb:
-  br label %bb3
-
-bb3:                                              ; preds = %bb11, %bb
-  %.01 = phi i64 [ 0, %bb ], [ %tmp12, %bb11 ]
-  %exitcond2 = icmp ne i64 %.01, 100
-  br i1 %exitcond2, label %bb5, label %bb4
+  br label %bb5
 
-bb4:                                              ; preds = %bb3
+bb4:                                              ; preds = %bb11
   br label %bb13
 
-bb5:                                              ; preds = %bb3
-  %tmp = add nsw i64 %.01, -3
-  %tmp6 = add nuw nsw i64 %.01, 3
+bb5:                                              ; preds = %bb, %bb11
+  %.013 = phi i64 [ 0, %bb ], [ %tmp12, %bb11 ]
+  %tmp = add nsw i64 %.013, -3
+  %tmp6 = add nuw nsw i64 %.013, 3
   %tmp7 = mul nsw i64 %tmp, %tmp6
-  %tmp8 = srem i64 %tmp7, %.01
+  %tmp8 = srem i64 %tmp7, %.013
   %tmp9 = trunc i64 %tmp8 to i32
-  %tmp10 = getelementptr inbounds i32, i32* %arg, i64 %.01
+  %tmp10 = getelementptr inbounds i32, i32* %arg, i64 %.013
   store i32 %tmp9, i32* %tmp10, align 4
   br label %bb11
 
 bb11:                                             ; preds = %bb5
-  %tmp12 = add nuw nsw i64 %.01, 1
-  br label %bb3
+  %tmp12 = add nuw nsw i64 %.013, 1
+  %exitcond2 = icmp ne i64 %tmp12, 100
+  br i1 %exitcond2, label %bb5, label %bb4
 
 bb13:                                             ; preds = %bb4
-  br label %bb14
+  br label %bb16
 
-bb14:                                             ; preds = %bb23, %bb13
-  %.0 = phi i64 [ 0, %bb13 ], [ %tmp24, %bb23 ]
-  %exitcond = icmp ne i64 %.0, 200
-  br i1 %exitcond, label %bb16, label %bb15
-
-bb15:                                             ; preds = %bb14
+bb15:                                             ; preds = %bb23
   br label %bb25
 
-bb16:                                             ; preds = %bb14
-  %tmp17 = add nsw i64 %.0, -3
-  %tmp18 = add nuw nsw i64 %.0, 3
+bb16:                                             ; preds = %bb13, %bb23
+  %.02 = phi i64 [ 0, %bb13 ], [ %tmp24, %bb23 ]
+  %tmp17 = add nsw i64 %.02, -3
+  %tmp18 = add nuw nsw i64 %.02, 3
   %tmp19 = mul nsw i64 %tmp17, %tmp18
-  %tmp20 = srem i64 %tmp19, %.0
+  %tmp20 = srem i64 %tmp19, %.02
   %tmp21 = trunc i64 %tmp20 to i32
-  %tmp22 = getelementptr inbounds [1024 x i32], [1024 x i32]* @B, i64 0, i64 %.0
+  %tmp22 = getelementptr inbounds [1024 x i32], [1024 x i32]* @B, i64 0, i64 %.02
   store i32 %tmp21, i32* %tmp22, align 4
   br label %bb23
 
 bb23:                                             ; preds = %bb16
-  %tmp24 = add nuw nsw i64 %.0, 1
-  br label %bb14
+  %tmp24 = add nuw nsw i64 %.02, 1
+  %exitcond = icmp ne i64 %tmp24, 200
+  br i1 %exitcond, label %bb16, label %bb15
 
 bb25:                                             ; preds = %bb15
   ret void
@@ -225,41 +217,38 @@ bb25:                                             ; preds = %bb15
 ; CHECK: Loop Fusion complete
 define void @negative_dependence(i32* noalias %arg) {
 bb:
-  br label %bb5
+  br label %bb7
 
-bb5:                                              ; preds = %bb9, %bb
-  %indvars.iv2 = phi i64 [ %indvars.iv.next3, %bb9 ], [ 0, %bb ]
-  %exitcond4 = icmp ne i64 %indvars.iv2, 100
-  br i1 %exitcond4, label %bb7, label %bb11
+bb11.preheader:                                   ; preds = %bb9
+  br label %bb13
 
-bb7:                                              ; preds = %bb5
-  %tmp = getelementptr inbounds i32, i32* %arg, i64 %indvars.iv2
-  %tmp8 = trunc i64 %indvars.iv2 to i32
+bb7:                                              ; preds = %bb, %bb9
+  %indvars.iv22 = phi i64 [ 0, %bb ], [ %indvars.iv.next3, %bb9 ]
+  %tmp = getelementptr inbounds i32, i32* %arg, i64 %indvars.iv22
+  %tmp8 = trunc i64 %indvars.iv22 to i32
   store i32 %tmp8, i32* %tmp, align 4
   br label %bb9
 
 bb9:                                              ; preds = %bb7
-  %indvars.iv.next3 = add nuw nsw i64 %indvars.iv2, 1
-  br label %bb5
+  %indvars.iv.next3 = add nuw nsw i64 %indvars.iv22, 1
+  %exitcond4 = icmp ne i64 %indvars.iv.next3, 100
+  br i1 %exitcond4, label %bb7, label %bb11.preheader
 
-bb11:                                             ; preds = %bb18, %bb5
-  %indvars.iv = phi i64 [ %indvars.iv.next, %bb18 ], [ 0, %bb5 ]
-  %exitcond = icmp ne i64 %indvars.iv, 100
-  br i1 %exitcond, label %bb13, label %bb19
-
-bb13:                                             ; preds = %bb11
-  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+bb13:                                             ; preds = %bb11.preheader, %bb18
+  %indvars.iv1 = phi i64 [ 0, %bb11.preheader ], [ %indvars.iv.next, %bb18 ]
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv1, 1
   %tmp14 = getelementptr inbounds i32, i32* %arg, i64 %indvars.iv.next
   %tmp15 = load i32, i32* %tmp14, align 4
   %tmp16 = shl nsw i32 %tmp15, 1
-  %tmp17 = getelementptr inbounds [1024 x i32], [1024 x i32]* @B, i64 0, i64 %indvars.iv
+  %tmp17 = getelementptr inbounds [1024 x i32], [1024 x i32]* @B, i64 0, i64 %indvars.iv1
   store i32 %tmp16, i32* %tmp17, align 4
   br label %bb18
 
 bb18:                                             ; preds = %bb13
-  br label %bb11
+  %exitcond = icmp ne i64 %indvars.iv.next, 100
+  br i1 %exitcond, label %bb13, label %bb19
 
-bb19:                                             ; preds = %bb11
+bb19:                                             ; preds = %bb18
   ret void
 }
 
@@ -282,41 +271,38 @@ bb19:                                             ; preds = %bb11
 ; CHECK: Loop Fusion complete
 define i32 @sumTest(i32* noalias %arg) {
 bb:
-  br label %bb6
+  br label %bb9
 
-bb6:                                              ; preds = %bb9, %bb
-  %indvars.iv3 = phi i64 [ %indvars.iv.next4, %bb9 ], [ 0, %bb ]
-  %.01 = phi i32 [ 0, %bb ], [ %tmp11, %bb9 ]
-  %exitcond5 = icmp ne i64 %indvars.iv3, 100
-  br i1 %exitcond5, label %bb9, label %bb13
+bb13.preheader:                                   ; preds = %bb9
+  br label %bb15
 
-bb9:                                              ; preds = %bb6
-  %tmp = getelementptr inbounds i32, i32* %arg, i64 %indvars.iv3
+bb9:                                              ; preds = %bb, %bb9
+  %.01.lcssa = phi i32 [ 0, %bb ], [ %tmp11, %bb9 ]
+  %.013 = phi i32 [ 0, %bb ], [ %tmp11, %bb9 ]
+  %indvars.iv32 = phi i64 [ 0, %bb ], [ %indvars.iv.next4, %bb9 ]
+  %tmp = getelementptr inbounds i32, i32* %arg, i64 %indvars.iv32
   %tmp10 = load i32, i32* %tmp, align 4
-  %tmp11 = add nsw i32 %.01, %tmp10
-  %indvars.iv.next4 = add nuw nsw i64 %indvars.iv3, 1
-  br label %bb6
+  %tmp11 = add nsw i32 %.013, %tmp10
+  %indvars.iv.next4 = add nuw nsw i64 %indvars.iv32, 1
+  %exitcond5 = icmp ne i64 %indvars.iv.next4, 100
+  br i1 %exitcond5, label %bb9, label %bb13.preheader
 
-bb13:                                             ; preds = %bb20, %bb6
-  %.01.lcssa = phi i32 [ %.01, %bb6 ], [ %.01.lcssa, %bb20 ]
-  %indvars.iv = phi i64 [ %indvars.iv.next, %bb20 ], [ 0, %bb6 ]
-  %exitcond = icmp ne i64 %indvars.iv, 100
-  br i1 %exitcond, label %bb15, label %bb14
-
-bb14:                                             ; preds = %bb13
+bb14:                                             ; preds = %bb20
   br label %bb21
 
-bb15:                                             ; preds = %bb13
-  %tmp16 = getelementptr inbounds i32, i32* %arg, i64 %indvars.iv
+bb15:                                             ; preds = %bb13.preheader, %bb20
+  %indvars.iv1 = phi i64 [ 0, %bb13.preheader ], [ %indvars.iv.next, %bb20 ]
+  %tmp16 = getelementptr inbounds i32, i32* %arg, i64 %indvars.iv1
   %tmp17 = load i32, i32* %tmp16, align 4
   %tmp18 = sdiv i32 %tmp17, %.01.lcssa
-  %tmp19 = getelementptr inbounds [1024 x i32], [1024 x i32]* @B, i64 0, i64 %indvars.iv
+  %tmp19 = getelementptr inbounds [1024 x i32], [1024 x i32]* @B, i64 0, i64 %indvars.iv1
   store i32 %tmp18, i32* %tmp19, align 4
   br label %bb20
 
 bb20:                                             ; preds = %bb15
-  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
-  br label %bb13
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv1, 1
+  %exitcond = icmp ne i64 %indvars.iv.next, 100
+  br i1 %exitcond, label %bb15, label %bb14
 
 bb21:                                             ; preds = %bb14
   ret i32 %.01.lcssa
@@ -369,3 +355,62 @@ for.cond.cleanup7:                                ; preds = %for.body8, %entry
   %sum1.0.lcssa36 = phi float [ 0.000000e+00, %entry ], [ %add, %for.body8 ]
   ret float %sum1.0.lcssa36
 }
+
+; Check that non-rotated loops are not considered for fusion.
+; CHECK: Performing Loop Fusion on function notRotated
+; CHECK: Loop bb{{.*}} is not rotated!
+; CHECK: Loop bb{{.*}} is not rotated!
+define void @notRotated(i32* noalias %arg) {
+bb:
+  br label %bb5
+
+bb5:                                              ; preds = %bb14, %bb
+  %indvars.iv2 = phi i64 [ %indvars.iv.next3, %bb14 ], [ 0, %bb ]
+  %.01 = phi i32 [ 0, %bb ], [ %tmp15, %bb14 ]
+  %exitcond4 = icmp ne i64 %indvars.iv2, 100
+  br i1 %exitcond4, label %bb7, label %bb17
+
+bb7:                                              ; preds = %bb5
+  %tmp = add nsw i32 %.01, -3
+  %tmp8 = add nuw nsw i64 %indvars.iv2, 3
+  %tmp9 = trunc i64 %tmp8 to i32
+  %tmp10 = mul nsw i32 %tmp, %tmp9
+  %tmp11 = trunc i64 %indvars.iv2 to i32
+  %tmp12 = srem i32 %tmp10, %tmp11
+  %tmp13 = getelementptr inbounds i32, i32* %arg, i64 %indvars.iv2
+  store i32 %tmp12, i32* %tmp13, align 4
+  br label %bb14
+
+bb14:                                             ; preds = %bb7
+  %indvars.iv.next3 = add nuw nsw i64 %indvars.iv2, 1
+  %tmp15 = add nuw nsw i32 %.01, 1
+  br label %bb5
+
+bb17:                                             ; preds = %bb27, %bb5
+  %indvars.iv = phi i64 [ %indvars.iv.next, %bb27 ], [ 0, %bb5 ]
+  %.0 = phi i32 [ 0, %bb5 ], [ %tmp28, %bb27 ]
+  %exitcond = icmp ne i64 %indvars.iv, 100
+  br i1 %exitcond, label %bb19, label %bb18
+
+bb18:                                             ; preds = %bb17
+  br label %bb29
+
+bb19:                                             ; preds = %bb17
+  %tmp20 = add nsw i32 %.0, -3
+  %tmp21 = add nuw nsw i64 %indvars.iv, 3
+  %tmp22 = trunc i64 %tmp21 to i32
+  %tmp23 = mul nsw i32 %tmp20, %tmp22
+  %tmp24 = trunc i64 %indvars.iv to i32
+  %tmp25 = srem i32 %tmp23, %tmp24
+  %tmp26 = getelementptr inbounds [1024 x i32], [1024 x i32]* @B, i64 0, i64 %indvars.iv
+  store i32 %tmp25, i32* %tmp26, align 4
+  br label %bb27
+
+bb27:                                             ; preds = %bb19
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %tmp28 = add nuw nsw i32 %.0, 1
+  br label %bb17
+
+bb29:                                             ; preds = %bb18
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopFusion/diagnostics_analysis.ll b/llvm/test/Transforms/LoopFusion/diagnostics_analysis.ll
new file mode 100644
index 000000000..901fa0d8e
--- /dev/null
+++ b/llvm/test/Transforms/LoopFusion/diagnostics_analysis.ll
@@ -0,0 +1,130 @@
+; RUN: opt -S -loop-fusion -pass-remarks-analysis=loop-fusion -disable-output < %s 2>&1 | FileCheck %s
+
+@B = common global [1024 x i32] zeroinitializer, align 16
+
+; CHECK: remark: diagnostics_analysis.c:6:3: [test]: Loop is not a candidate for fusion: Loop contains a volatile access
+; CHECK: remark: diagnostics_analysis.c:10:3: [test]: Loop is not a candidate for fusion: Loop has unknown trip count
+define void @test(i32* %A, i32 %n) !dbg !15 {
+entry:
+  %A.addr = alloca i32*, align 8
+  %n.addr = alloca i32, align 4
+  %i = alloca i32, align 4
+  %i1 = alloca i32, align 4
+  store i32* %A, i32** %A.addr, align 8
+  store i32 %n, i32* %n.addr, align 4
+  %0 = bitcast i32* %i to i8*
+  store i32 0, i32* %i, align 4
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %1 = load i32, i32* %i, align 4
+  %2 = load i32, i32* %n.addr, align 4
+  %cmp = icmp slt i32 %1, %2
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.cond
+  %3 = bitcast i32* %i to i8*, !dbg !42
+  br label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %4 = load i32, i32* %i, align 4
+  %sub = sub nsw i32 %4, 3
+  %5 = load i32, i32* %i, align 4
+  %add = add nsw i32 %5, 3
+  %mul = mul nsw i32 %sub, %add
+  %6 = load i32, i32* %i, align 4
+  %rem = srem i32 %mul, %6
+  %7 = load i32*, i32** %A.addr, align 8
+  %8 = load i32, i32* %i, align 4
+  %idxprom = sext i32 %8 to i64
+  %arrayidx = getelementptr inbounds i32, i32* %7, i64 %idxprom
+  store volatile i32 %rem, i32* %arrayidx, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body
+  %9 = load i32, i32* %i, align 4, !dbg !49
+  %inc = add nsw i32 %9, 1, !dbg !49
+  store i32 %inc, i32* %i, align 4, !dbg !49
+  br label %for.cond, !dbg !42, !llvm.loop !50
+
+for.end:                                          ; preds = %for.cond.cleanup
+  %10 = bitcast i32* %i1 to i8*
+  store i32 0, i32* %i1, align 4
+  br label %for.cond2
+
+for.cond2:                                        ; preds = %for.inc12, %for.end
+  %11 = load i32, i32* %i1, align 4
+  %12 = load i32, i32* %n.addr, align 4
+  %cmp3 = icmp slt i32 %11, %12
+  br i1 %cmp3, label %for.body5, label %for.cond.cleanup4
+
+for.cond.cleanup4:                                ; preds = %for.cond2
+  %13 = bitcast i32* %i1 to i8*
+  br label %for.end14
+
+for.body5:                                        ; preds = %for.cond2
+  %14 = load i32, i32* %i1, align 4
+  %sub6 = sub nsw i32 %14, 3
+  %15 = load i32, i32* %i1, align 4
+  %add7 = add nsw i32 %15, 3
+  %mul8 = mul nsw i32 %sub6, %add7
+  %16 = load i32, i32* %i1, align 4
+  %rem9 = srem i32 %mul8, %16
+  %17 = load i32, i32* %i1, align 4
+  %idxprom10 = sext i32 %17 to i64
+  %arrayidx11 = getelementptr inbounds [1024 x i32], [1024 x i32]* @B, i64 0, i64 %idxprom10
+  store i32 %rem9, i32* %arrayidx11, align 4
+  br label %for.inc12
+
+for.inc12:                                        ; preds = %for.body5
+  %18 = load i32, i32* %i1, align 4
+  %inc13 = add nsw i32 %18, 1
+  store i32 %inc13, i32* %i1, align 4
+  br label %for.cond2, !dbg !59, !llvm.loop !67
+
+for.end14:                                        ; preds = %for.cond.cleanup4
+  ret void
+}
+
+!llvm.module.flags = !{!10, !11, !13}
+!llvm.ident = !{!14}
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "B", scope: !2, file: !3, line: 1, type: !6, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 9.0.0 (git@github.ibm.com:compiler/llvm-project.git c019c32c5a2b0ed4487a738337d35fd3f630ac0a)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5, nameTableKind: GNU)
+!3 = !DIFile(filename: "diagnostics_analysis.c", directory: "/tmp")
+!4 = !{}
+!5 = !{!0}
+!6 = !DICompositeType(tag: DW_TAG_array_type, baseType: !7, size: 32768, elements: !8)
+!7 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!8 = !{!9}
+!9 = !DISubrange(count: 1024)
+!10 = !{i32 2, !"Dwarf Version", i32 4}
+!11 = !{i32 2, !"Debug Info Version", i32 3}
+!13 = !{i32 7, !"PIC Level", i32 2}
+!14 = !{!"clang version 9.0.0 (git@github.ibm.com:compiler/llvm-project.git c019c32c5a2b0ed4487a738337d35fd3f630ac0a)"}
+!15 = distinct !DISubprogram(name: "test", scope: !3, file: !3, line: 5, type: !16, scopeLine: 5, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !2, retainedNodes: !20)
+!16 = !DISubroutineType(types: !17)
+!17 = !{null, !18, !7}
+!18 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !19, size: 64)
+!19 = !DIDerivedType(tag: DW_TAG_volatile_type, baseType: !7)
+!20 = !{!21, !22, !23, !25}
+!21 = !DILocalVariable(name: "A", arg: 1, scope: !15, file: !3, line: 5, type: !18)
+!22 = !DILocalVariable(name: "n", arg: 2, scope: !15, file: !3, line: 5, type: !7)
+!23 = !DILocalVariable(name: "i", scope: !24, file: !3, line: 6, type: !7)
+!24 = distinct !DILexicalBlock(scope: !15, file: !3, line: 6, column: 3)
+!25 = !DILocalVariable(name: "i", scope: !26, file: !3, line: 10, type: !7)
+!26 = distinct !DILexicalBlock(scope: !15, file: !3, line: 10, column: 3)
+!38 = distinct !DILexicalBlock(scope: !24, file: !3, line: 6, column: 3)
+!41 = !DILocation(line: 6, column: 3, scope: !24)
+!42 = !DILocation(line: 6, column: 3, scope: !38)
+!44 = distinct !DILexicalBlock(scope: !38, file: !3, line: 6, column: 31)
+!49 = !DILocation(line: 6, column: 27, scope: !38)
+!50 = distinct !{!50, !41, !51}
+!51 = !DILocation(line: 8, column: 3, scope: !24)
+!55 = distinct !DILexicalBlock(scope: !26, file: !3, line: 10, column: 3)
+!58 = !DILocation(line: 10, column: 3, scope: !26)
+!59 = !DILocation(line: 10, column: 3, scope: !55)
+!67 = distinct !{!67, !58, !68}
+!68 = !DILocation(line: 12, column: 3, scope: !26)
+!69 = !DILocation(line: 13, column: 1, scope: !15)
diff --git a/llvm/test/Transforms/LoopFusion/diagnostics_missed.ll b/llvm/test/Transforms/LoopFusion/diagnostics_missed.ll
new file mode 100644
index 000000000..0926e0353
--- /dev/null
+++ b/llvm/test/Transforms/LoopFusion/diagnostics_missed.ll
@@ -0,0 +1,269 @@
+; RUN: opt -S -loop-fusion -pass-remarks-missed=loop-fusion -disable-output < %s 2>&1 | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+@B = common global [1024 x i32] zeroinitializer, align 16, !dbg !0
+
+; CHECK: remark: diagnostics_missed.c:18:3: [non_adjacent]: entry and for.end: Loops are not adjacent
+define void @non_adjacent(i32* noalias %A) !dbg !14 {
+entry:
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.inc
+  br label %for.end
+
+for.body:                                         ; preds = %entry, %for.inc
+  %i.02 = phi i64 [ 0, %entry ], [ %inc, %for.inc ]
+  %sub = add nsw i64 %i.02, -3
+  %add = add nuw nsw i64 %i.02, 3
+  %mul = mul nsw i64 %sub, %add
+  %rem = srem i64 %mul, %i.02
+  %conv = trunc i64 %rem to i32
+  %arrayidx = getelementptr inbounds i32, i32* %A, i64 %i.02
+  store i32 %conv, i32* %arrayidx, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body
+  %inc = add nuw nsw i64 %i.02, 1, !dbg !26
+  %exitcond1 = icmp ne i64 %inc, 100
+  br i1 %exitcond1, label %for.body, label %for.cond.cleanup, !llvm.loop !28
+
+for.end:                                          ; preds = %for.cond.cleanup
+  br label %for.body6
+
+for.cond.cleanup5:                                ; preds = %for.inc13
+  br label %for.end15
+
+for.body6:                                        ; preds = %for.end, %for.inc13
+  %i1.01 = phi i64 [ 0, %for.end ], [ %inc14, %for.inc13 ]
+  %sub7 = add nsw i64 %i1.01, -3
+  %add8 = add nuw nsw i64 %i1.01, 3
+  %mul9 = mul nsw i64 %sub7, %add8
+  %rem10 = srem i64 %mul9, %i1.01
+  %conv11 = trunc i64 %rem10 to i32
+  %arrayidx12 = getelementptr inbounds [1024 x i32], [1024 x i32]* @B, i64 0, i64 %i1.01
+  store i32 %conv11, i32* %arrayidx12, align 4
+  br label %for.inc13
+
+for.inc13:                                        ; preds = %for.body6
+  %inc14 = add nuw nsw i64 %i1.01, 1, !dbg !31
+  %exitcond = icmp ne i64 %inc14, 100
+  br i1 %exitcond, label %for.body6, label %for.cond.cleanup5, !llvm.loop !33
+
+for.end15:                                        ; preds = %for.cond.cleanup5
+  ret void
+}
+
+; CHECK: remark: diagnostics_missed.c:28:3: [different_bounds]: entry and for.end: Loop trip counts are not the same
+define void @different_bounds(i32* noalias %A) !dbg !36 {
+entry:
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.inc
+  br label %for.end
+
+for.body:                                         ; preds = %entry, %for.inc
+  %i.02 = phi i64 [ 0, %entry ], [ %inc, %for.inc ]
+  %sub = add nsw i64 %i.02, -3
+  %add = add nuw nsw i64 %i.02, 3
+  %mul = mul nsw i64 %sub, %add
+  %rem = srem i64 %mul, %i.02
+  %conv = trunc i64 %rem to i32
+  %arrayidx = getelementptr inbounds i32, i32* %A, i64 %i.02
+  store i32 %conv, i32* %arrayidx, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body
+  %inc = add nuw nsw i64 %i.02, 1, !dbg !43
+  %exitcond1 = icmp ne i64 %inc, 100
+  br i1 %exitcond1, label %for.body, label %for.cond.cleanup, !llvm.loop !45
+
+for.end:                                          ; preds = %for.cond.cleanup
+  br label %for.body6
+
+for.cond.cleanup5:                                ; preds = %for.inc13
+  br label %for.end15
+
+for.body6:                                        ; preds = %for.end, %for.inc13
+  %i1.01 = phi i64 [ 0, %for.end ], [ %inc14, %for.inc13 ]
+  %sub7 = add nsw i64 %i1.01, -3
+  %add8 = add nuw nsw i64 %i1.01, 3
+  %mul9 = mul nsw i64 %sub7, %add8
+  %rem10 = srem i64 %mul9, %i1.01
+  %conv11 = trunc i64 %rem10 to i32
+  %arrayidx12 = getelementptr inbounds [1024 x i32], [1024 x i32]* @B, i64 0, i64 %i1.01
+  store i32 %conv11, i32* %arrayidx12, align 4
+  br label %for.inc13
+
+for.inc13:                                        ; preds = %for.body6
+  %inc14 = add nuw nsw i64 %i1.01, 1
+  %exitcond = icmp ne i64 %inc14, 200
+  br i1 %exitcond, label %for.body6, label %for.cond.cleanup5, !llvm.loop !48
+
+for.end15:                                        ; preds = %for.cond.cleanup5
+  ret void
+}
+
+; CHECK: remark: diagnostics_missed.c:38:3: [negative_dependence]: entry and for.end: Loop has a non-empty preheader
+define void @negative_dependence(i32* noalias %A) !dbg !51 {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.inc
+  %indvars.iv13 = phi i64 [ 0, %entry ], [ %indvars.iv.next2, %for.inc ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i64 %indvars.iv13
+  %tmp = trunc i64 %indvars.iv13 to i32
+  store i32 %tmp, i32* %arrayidx, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body
+  %indvars.iv.next2 = add nuw nsw i64 %indvars.iv13, 1
+  %exitcond3 = icmp ne i64 %indvars.iv.next2, 100
+  br i1 %exitcond3, label %for.body, label %for.end, !llvm.loop !58
+
+for.end:                                          ; preds = %for.inc
+  call void @llvm.dbg.value(metadata i32 0, metadata !56, metadata !DIExpression()), !dbg !61
+  br label %for.body5
+
+for.body5:                                        ; preds = %for.end, %for.inc10
+  %indvars.iv2 = phi i64 [ 0, %for.end ], [ %indvars.iv.next, %for.inc10 ]
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv2, 1
+  %arrayidx7 = getelementptr inbounds i32, i32* %A, i64 %indvars.iv.next
+  %tmp4 = load i32, i32* %arrayidx7, align 4
+  %mul = shl nsw i32 %tmp4, 1
+  %arrayidx9 = getelementptr inbounds [1024 x i32], [1024 x i32]* @B, i64 0, i64 %indvars.iv2
+  store i32 %mul, i32* %arrayidx9, align 4
+  br label %for.inc10
+
+for.inc10:                                        ; preds = %for.body5
+  %exitcond = icmp ne i64 %indvars.iv.next, 100
+  br i1 %exitcond, label %for.body5, label %for.end12
+
+for.end12:                                        ; preds = %for.inc10
+  ret void, !dbg !62
+}
+
+; CHECK: remark: diagnostics_missed.c:51:3: [sumTest]: entry and for.cond2.preheader: Dependencies prevent fusion
+define i32 @sumTest(i32* noalias %A) !dbg !63 {
+entry:
+  br label %for.body
+
+for.cond2.preheader:                              ; preds = %for.inc
+  br label %for.body5
+
+for.body:                                         ; preds = %entry, %for.inc
+  %sum.04 = phi i32 [ 0, %entry ], [ %add, %for.inc ]
+  %indvars.iv13 = phi i64 [ 0, %entry ], [ %indvars.iv.next2, %for.inc ]
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body
+  %arrayidx = getelementptr inbounds i32, i32* %A, i64 %indvars.iv13
+  %tmp = load i32, i32* %arrayidx, align 4
+  %add = add nsw i32 %sum.04, %tmp
+  %indvars.iv.next2 = add nuw nsw i64 %indvars.iv13, 1
+  %exitcond3 = icmp ne i64 %indvars.iv.next2, 100
+  br i1 %exitcond3, label %for.body, label %for.cond2.preheader, !llvm.loop !73
+
+for.body5:                                        ; preds = %for.cond2.preheader, %for.inc10
+  %indvars.iv2 = phi i64 [ 0, %for.cond2.preheader ], [ %indvars.iv.next, %for.inc10 ]
+  %arrayidx7 = getelementptr inbounds i32, i32* %A, i64 %indvars.iv2
+  %tmp4 = load i32, i32* %arrayidx7, align 4
+  %div = sdiv i32 %tmp4, %add
+  %arrayidx9 = getelementptr inbounds [1024 x i32], [1024 x i32]* @B, i64 0, i64 %indvars.iv2
+  store i32 %div, i32* %arrayidx9, align 4
+  br label %for.inc10
+
+for.inc10:                                        ; preds = %for.body5
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv2, 1
+  %exitcond = icmp ne i64 %indvars.iv.next, 100
+  br i1 %exitcond, label %for.body5, label %for.end12
+
+for.end12:                                        ; preds = %for.inc10
+  ret i32 %add, !dbg !76
+}
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.value(metadata, metadata, metadata) #0
+
+attributes #0 = { nounwind readnone speculatable willreturn }
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!10, !11, !12, !13}
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "B", scope: !2, file: !3, line: 46, type: !6, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 9.0.0 (git@github.ibm.com:compiler/llvm-project.git 23c4baaa9f5b33d2d52eda981d376c6b0a7a3180)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5, nameTableKind: GNU)
+!3 = !DIFile(filename: "diagnostics_missed.c", directory: "/tmp")
+!4 = !{}
+!5 = !{!0}
+!6 = !DICompositeType(tag: DW_TAG_array_type, baseType: !7, size: 32768, elements: !8)
+!7 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!8 = !{!9}
+!9 = !DISubrange(count: 1024)
+!10 = !{i32 2, !"Dwarf Version", i32 4}
+!11 = !{i32 2, !"Debug Info Version", i32 3}
+!12 = !{i32 1, !"wchar_size", i32 4}
+!13 = !{i32 7, !"PIC Level", i32 2}
+!14 = distinct !DISubprogram(name: "non_adjacent", scope: !3, file: !3, line: 17, type: !15, scopeLine: 17, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !2, retainedNodes: !19)
+!15 = !DISubroutineType(types: !16)
+!16 = !{null, !17}
+!17 = !DIDerivedType(tag: DW_TAG_restrict_type, baseType: !18)
+!18 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !7, size: 64)
+!19 = !{!20, !21, !24}
+!20 = !DILocalVariable(name: "A", arg: 1, scope: !14, file: !3, line: 17, type: !17)
+!21 = !DILocalVariable(name: "i", scope: !22, file: !3, line: 18, type: !23)
+!22 = distinct !DILexicalBlock(scope: !14, file: !3, line: 18, column: 3)
+!23 = !DIBasicType(name: "long int", size: 64, encoding: DW_ATE_signed)
+!24 = !DILocalVariable(name: "i", scope: !25, file: !3, line: 22, type: !23)
+!25 = distinct !DILexicalBlock(scope: !14, file: !3, line: 22, column: 3)
+!26 = !DILocation(line: 18, column: 30, scope: !27)
+!27 = distinct !DILexicalBlock(scope: !22, file: !3, line: 18, column: 3)
+!28 = distinct !{!28, !29, !30}
+!29 = !DILocation(line: 18, column: 3, scope: !22)
+!30 = !DILocation(line: 20, column: 3, scope: !22)
+!31 = !DILocation(line: 22, column: 30, scope: !32)
+!32 = distinct !DILexicalBlock(scope: !25, file: !3, line: 22, column: 3)
+!33 = distinct !{!33, !34, !35}
+!34 = !DILocation(line: 22, column: 3, scope: !25)
+!35 = !DILocation(line: 24, column: 3, scope: !25)
+!36 = distinct !DISubprogram(name: "different_bounds", scope: !3, file: !3, line: 27, type: !15, scopeLine: 27, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !2, retainedNodes: !37)
+!37 = !{!38, !39, !41}
+!38 = !DILocalVariable(name: "A", arg: 1, scope: !36, file: !3, line: 27, type: !17)
+!39 = !DILocalVariable(name: "i", scope: !40, file: !3, line: 28, type: !23)
+!40 = distinct !DILexicalBlock(scope: !36, file: !3, line: 28, column: 3)
+!41 = !DILocalVariable(name: "i", scope: !42, file: !3, line: 32, type: !23)
+!42 = distinct !DILexicalBlock(scope: !36, file: !3, line: 32, column: 3)
+!43 = !DILocation(line: 28, column: 30, scope: !44)
+!44 = distinct !DILexicalBlock(scope: !40, file: !3, line: 28, column: 3)
+!45 = distinct !{!45, !46, !47}
+!46 = !DILocation(line: 28, column: 3, scope: !40)
+!47 = !DILocation(line: 30, column: 3, scope: !40)
+!48 = distinct !{!48, !49, !50}
+!49 = !DILocation(line: 32, column: 3, scope: !42)
+!50 = !DILocation(line: 34, column: 3, scope: !42)
+!51 = distinct !DISubprogram(name: "negative_dependence", scope: !3, file: !3, line: 37, type: !15, scopeLine: 37, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !2, retainedNodes: !52)
+!52 = !{!53, !54, !56}
+!53 = !DILocalVariable(name: "A", arg: 1, scope: !51, file: !3, line: 37, type: !17)
+!54 = !DILocalVariable(name: "i", scope: !55, file: !3, line: 38, type: !7)
+!55 = distinct !DILexicalBlock(scope: !51, file: !3, line: 38, column: 3)
+!56 = !DILocalVariable(name: "i", scope: !57, file: !3, line: 42, type: !7)
+!57 = distinct !DILexicalBlock(scope: !51, file: !3, line: 42, column: 3)
+!58 = distinct !{!58, !59, !60}
+!59 = !DILocation(line: 38, column: 3, scope: !55)
+!60 = !DILocation(line: 40, column: 3, scope: !55)
+!61 = !DILocation(line: 0, scope: !57)
+!62 = !DILocation(line: 45, column: 1, scope: !51)
+!63 = distinct !DISubprogram(name: "sumTest", scope: !3, file: !3, line: 48, type: !64, scopeLine: 48, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !2, retainedNodes: !66)
+!64 = !DISubroutineType(types: !65)
+!65 = !{!7, !17}
+!66 = !{!67, !68, !69, !71}
+!67 = !DILocalVariable(name: "A", arg: 1, scope: !63, file: !3, line: 48, type: !17)
+!68 = !DILocalVariable(name: "sum", scope: !63, file: !3, line: 49, type: !7)
+!69 = !DILocalVariable(name: "i", scope: !70, file: !3, line: 51, type: !7)
+!70 = distinct !DILexicalBlock(scope: !63, file: !3, line: 51, column: 3)
+!71 = !DILocalVariable(name: "i", scope: !72, file: !3, line: 54, type: !7)
+!72 = distinct !DILexicalBlock(scope: !63, file: !3, line: 54, column: 3)
+!73 = distinct !{!73, !74, !75}
+!74 = !DILocation(line: 51, column: 3, scope: !70)
+!75 = !DILocation(line: 52, column: 15, scope: !70)
+!76 = !DILocation(line: 57, column: 3, scope: !63)
diff --git a/llvm/test/Transforms/LoopFusion/four_loops.ll b/llvm/test/Transforms/LoopFusion/four_loops.ll
index e03de6584..8f3822b8a 100644
--- a/llvm/test/Transforms/LoopFusion/four_loops.ll
+++ b/llvm/test/Transforms/LoopFusion/four_loops.ll
@@ -9,127 +9,107 @@
 ; CHECK-NEXT: bb:
 ; CHECK-NEXT: br label %[[LOOP1HEADER:bb[0-9]+]]
 ; CHECK: [[LOOP1HEADER]]
-; CHECK: br i1 %exitcond12, label %[[LOOP1BODY:bb[0-9]+]], label %[[LOOP2PREHEADER:bb[0-9]+]]
-; CHECK: [[LOOP1BODY]]
+; CHECK: br label %[[LOOP2BODY:bb[0-9]+]]
+; CHECK: [[LOOP2BODY]]
+; CHECK: br label %[[LOOP3BODY:bb[0-9]+]]
+; CHECK: [[LOOP3BODY]]
+; CHECK: br label %[[LOOP4BODY:bb[0-9]+]]
+; CHECK: [[LOOP4BODY]]
 ; CHECK: br label %[[LOOP1LATCH:bb[0-9]+]]
 ; CHECK: [[LOOP1LATCH]]
-; CHECK: br label %[[LOOP2PREHEADER]]
-; CHECK: [[LOOP2PREHEADER]]
-; CHECK: br i1 %exitcond9, label %[[LOOP2HEADER:bb[0-9]+]], label %[[LOOP3PREHEADER:bb[0-9]+]]
-; CHECK: [[LOOP2HEADER]]
-; CHECK: br label %[[LOOP2LATCH:bb[0-9]+]]
-; CHECK: [[LOOP2LATCH]]
-; CHECK: br label %[[LOOP3PREHEADER]]
-; CHECK: [[LOOP3PREHEADER]]
-; CHECK: br i1 %exitcond6, label %[[LOOP3HEADER:bb[0-9]+]], label %[[LOOP4PREHEADER:bb[0-9]+]]
-; CHECK: [[LOOP3HEADER]]
-; CHECK: br label %[[LOOP3LATCH:bb[0-9]+]]
-; CHECK: [[LOOP3LATCH]]
-; CHECK: br label %[[LOOP4PREHEADER]]
-; CHECK: [[LOOP4PREHEADER]]
-; CHECK: br i1 %exitcond, label %[[LOOP4HEADER:bb[0-9]+]], label %[[LOOP4EXIT:bb[0-9]+]]
-; CHECK: [[LOOP4EXIT]]
-; CHECK: br label %[[FUNCEXIT:bb[0-9]+]]
-; CHECK: [[LOOP4HEADER]]
-; CHECK: br label %[[LOOP4LATCH:bb[0-9]+]]
-; CHECK: [[LOOP4LATCH]]
-; CHECK: br label %[[LOOP1HEADER]]
-; CHECK: [[FUNCEXIT]]
+; CHECK: br i1 %{{.*}}, label %[[LOOP1HEADER]], label %[[LOOPEXIT:bb[0-9]+]]
 ; CHECK: ret void
 define void @dep_free() {
 bb:
-  br label %bb13
+  br label %bb15
 
-bb13:                                             ; preds = %bb22, %bb
-  %indvars.iv10 = phi i64 [ %indvars.iv.next11, %bb22 ], [ 0, %bb ]
-  %.0 = phi i32 [ 0, %bb ], [ %tmp23, %bb22 ]
-  %exitcond12 = icmp ne i64 %indvars.iv10, 100
-  br i1 %exitcond12, label %bb15, label %bb25
+bb25.preheader:                                   ; preds = %bb22
+  br label %bb27
 
-bb15:                                             ; preds = %bb13
-  %tmp = add nsw i32 %.0, -3
-  %tmp16 = add nuw nsw i64 %indvars.iv10, 3
+bb15:                                             ; preds = %bb, %bb22
+  %.08 = phi i32 [ 0, %bb ], [ %tmp23, %bb22 ]
+  %indvars.iv107 = phi i64 [ 0, %bb ], [ %indvars.iv.next11, %bb22 ]
+  %tmp = add nsw i32 %.08, -3
+  %tmp16 = add nuw nsw i64 %indvars.iv107, 3
   %tmp17 = trunc i64 %tmp16 to i32
   %tmp18 = mul nsw i32 %tmp, %tmp17
-  %tmp19 = trunc i64 %indvars.iv10 to i32
+  %tmp19 = trunc i64 %indvars.iv107 to i32
   %tmp20 = srem i32 %tmp18, %tmp19
-  %tmp21 = getelementptr inbounds [1024 x i32], [1024 x i32]* @A, i64 0, i64 %indvars.iv10
+  %tmp21 = getelementptr inbounds [1024 x i32], [1024 x i32]* @A, i64 0, i64 %indvars.iv107
   store i32 %tmp20, i32* %tmp21, align 4
   br label %bb22
 
 bb22:                                             ; preds = %bb15
-  %indvars.iv.next11 = add nuw nsw i64 %indvars.iv10, 1
-  %tmp23 = add nuw nsw i32 %.0, 1
-  br label %bb13
-
-bb25:                                             ; preds = %bb35, %bb13
-  %indvars.iv7 = phi i64 [ %indvars.iv.next8, %bb35 ], [ 0, %bb13 ]
-  %.01 = phi i32 [ 0, %bb13 ], [ %tmp36, %bb35 ]
-  %exitcond9 = icmp ne i64 %indvars.iv7, 100
-  br i1 %exitcond9, label %bb27, label %bb38
-
-bb27:                                             ; preds = %bb25
-  %tmp28 = add nsw i32 %.01, -3
-  %tmp29 = add nuw nsw i64 %indvars.iv7, 3
+  %indvars.iv.next11 = add nuw nsw i64 %indvars.iv107, 1
+  %tmp23 = add nuw nsw i32 %.08, 1
+  %exitcond12 = icmp ne i64 %indvars.iv.next11, 100
+  br i1 %exitcond12, label %bb15, label %bb25.preheader
+
+bb38.preheader:                                   ; preds = %bb35
+  br label %bb40
+
+bb27:                                             ; preds = %bb25.preheader, %bb35
+  %.016 = phi i32 [ 0, %bb25.preheader ], [ %tmp36, %bb35 ]
+  %indvars.iv75 = phi i64 [ 0, %bb25.preheader ], [ %indvars.iv.next8, %bb35 ]
+  %tmp28 = add nsw i32 %.016, -3
+  %tmp29 = add nuw nsw i64 %indvars.iv75, 3
   %tmp30 = trunc i64 %tmp29 to i32
   %tmp31 = mul nsw i32 %tmp28, %tmp30
-  %tmp32 = trunc i64 %indvars.iv7 to i32
+  %tmp32 = trunc i64 %indvars.iv75 to i32
   %tmp33 = srem i32 %tmp31, %tmp32
-  %tmp34 = getelementptr inbounds [1024 x i32], [1024 x i32]* @B, i64 0, i64 %indvars.iv7
+  %tmp34 = getelementptr inbounds [1024 x i32], [1024 x i32]* @B, i64 0, i64 %indvars.iv75
   store i32 %tmp33, i32* %tmp34, align 4
   br label %bb35
 
 bb35:                                             ; preds = %bb27
-  %indvars.iv.next8 = add nuw nsw i64 %indvars.iv7, 1
-  %tmp36 = add nuw nsw i32 %.01, 1
-  br label %bb25
-
-bb38:                                             ; preds = %bb48, %bb25
-  %indvars.iv4 = phi i64 [ %indvars.iv.next5, %bb48 ], [ 0, %bb25 ]
-  %.02 = phi i32 [ 0, %bb25 ], [ %tmp49, %bb48 ]
-  %exitcond6 = icmp ne i64 %indvars.iv4, 100
-  br i1 %exitcond6, label %bb40, label %bb51
-
-bb40:                                             ; preds = %bb38
-  %tmp41 = add nsw i32 %.02, -3
-  %tmp42 = add nuw nsw i64 %indvars.iv4, 3
+  %indvars.iv.next8 = add nuw nsw i64 %indvars.iv75, 1
+  %tmp36 = add nuw nsw i32 %.016, 1
+  %exitcond9 = icmp ne i64 %indvars.iv.next8, 100
+  br i1 %exitcond9, label %bb27, label %bb38.preheader
+
+bb51.preheader:                                   ; preds = %bb48
+  br label %bb53
+
+bb40:                                             ; preds = %bb38.preheader, %bb48
+  %.024 = phi i32 [ 0, %bb38.preheader ], [ %tmp49, %bb48 ]
+  %indvars.iv43 = phi i64 [ 0, %bb38.preheader ], [ %indvars.iv.next5, %bb48 ]
+  %tmp41 = add nsw i32 %.024, -3
+  %tmp42 = add nuw nsw i64 %indvars.iv43, 3
   %tmp43 = trunc i64 %tmp42 to i32
   %tmp44 = mul nsw i32 %tmp41, %tmp43
-  %tmp45 = trunc i64 %indvars.iv4 to i32
+  %tmp45 = trunc i64 %indvars.iv43 to i32
   %tmp46 = srem i32 %tmp44, %tmp45
-  %tmp47 = getelementptr inbounds [1024 x i32], [1024 x i32]* @C, i64 0, i64 %indvars.iv4
+  %tmp47 = getelementptr inbounds [1024 x i32], [1024 x i32]* @C, i64 0, i64 %indvars.iv43
   store i32 %tmp46, i32* %tmp47, align 4
   br label %bb48
 
 bb48:                                             ; preds = %bb40
-  %indvars.iv.next5 = add nuw nsw i64 %indvars.iv4, 1
-  %tmp49 = add nuw nsw i32 %.02, 1
-  br label %bb38
+  %indvars.iv.next5 = add nuw nsw i64 %indvars.iv43, 1
+  %tmp49 = add nuw nsw i32 %.024, 1
+  %exitcond6 = icmp ne i64 %indvars.iv.next5, 100
+  br i1 %exitcond6, label %bb40, label %bb51.preheader
 
-bb51:                                             ; preds = %bb61, %bb38
-  %indvars.iv = phi i64 [ %indvars.iv.next, %bb61 ], [ 0, %bb38 ]
-  %.03 = phi i32 [ 0, %bb38 ], [ %tmp62, %bb61 ]
-  %exitcond = icmp ne i64 %indvars.iv, 100
-  br i1 %exitcond, label %bb53, label %bb52
-
-bb52:                                             ; preds = %bb51
+bb52:                                             ; preds = %bb61
   br label %bb63
 
-bb53:                                             ; preds = %bb51
-  %tmp54 = add nsw i32 %.03, -3
-  %tmp55 = add nuw nsw i64 %indvars.iv, 3
+bb53:                                             ; preds = %bb51.preheader, %bb61
+  %.032 = phi i32 [ 0, %bb51.preheader ], [ %tmp62, %bb61 ]
+  %indvars.iv1 = phi i64 [ 0, %bb51.preheader ], [ %indvars.iv.next, %bb61 ]
+  %tmp54 = add nsw i32 %.032, -3
+  %tmp55 = add nuw nsw i64 %indvars.iv1, 3
   %tmp56 = trunc i64 %tmp55 to i32
   %tmp57 = mul nsw i32 %tmp54, %tmp56
-  %tmp58 = trunc i64 %indvars.iv to i32
+  %tmp58 = trunc i64 %indvars.iv1 to i32
   %tmp59 = srem i32 %tmp57, %tmp58
-  %tmp60 = getelementptr inbounds [1024 x i32], [1024 x i32]* @D, i64 0, i64 %indvars.iv
+  %tmp60 = getelementptr inbounds [1024 x i32], [1024 x i32]* @D, i64 0, i64 %indvars.iv1
   store i32 %tmp59, i32* %tmp60, align 4
   br label %bb61
 
 bb61:                                             ; preds = %bb53
-  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
-  %tmp62 = add nuw nsw i32 %.03, 1
-  br label %bb51
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv1, 1
+  %tmp62 = add nuw nsw i32 %.032, 1
+  %exitcond = icmp ne i64 %indvars.iv.next, 100
+  br i1 %exitcond, label %bb53, label %bb52
 
 bb63:                                             ; preds = %bb52
   ret void
diff --git a/llvm/test/Transforms/LoopFusion/guarded.ll b/llvm/test/Transforms/LoopFusion/guarded.ll
new file mode 100644
index 000000000..9242e0a51
--- /dev/null
+++ b/llvm/test/Transforms/LoopFusion/guarded.ll
@@ -0,0 +1,65 @@
+; RUN: opt -S -loop-fusion < %s | FileCheck %s
+
+@B = common global [1024 x i32] zeroinitializer, align 16
+
+; CHECK: void @dep_free_parametric
+; CHECK-next: entry:
+; CHECK: br i1 %{{.*}}, label %[[LOOP1PREHEADER:bb[0-9]*]], label %[[LOOP1SUCC:bb[0-9]+]]
+; CHECK: [[LOOP1PREHEADER]]
+; CHECK-NEXT: br label %[[LOOP1BODY:bb[0-9]*]]
+; CHECK: [[LOOP1BODY]]
+; CHECK: br i1 %{{.*}}, label %[[LOOP1BODY]], label %[[LOOP2EXIT:bb[0-9]+]]
+; CHECK: [[LOOP2EXIT]]
+; CHECK: br label %[[LOOP1SUCC]]
+; CHECK: [[LOOP1SUCC]]
+; CHECK: ret void
+define void @dep_free_parametric(i32* noalias %A, i64 %N) {
+entry:
+  %cmp4 = icmp slt i64 0, %N
+  br i1 %cmp4, label %bb3, label %bb14
+
+bb3:                               ; preds = %entry
+  br label %bb5
+
+bb5:                                         ; preds = %bb3, %bb5
+  %i.05 = phi i64 [ %inc, %bb5 ], [ 0, %bb3 ]
+  %sub = sub nsw i64 %i.05, 3
+  %add = add nsw i64 %i.05, 3
+  %mul = mul nsw i64 %sub, %add
+  %rem = srem i64 %mul, %i.05
+  %conv = trunc i64 %rem to i32
+  %arrayidx = getelementptr inbounds i32, i32* %A, i64 %i.05
+  store i32 %conv, i32* %arrayidx, align 4
+  %inc = add nsw i64 %i.05, 1
+  %cmp = icmp slt i64 %inc, %N
+  br i1 %cmp, label %bb5, label %bb10
+
+bb10:                                 ; preds = %bb5
+  br label %bb14
+
+bb14:                                          ; preds = %bb10, %entry
+  %cmp31 = icmp slt i64 0, %N
+  br i1 %cmp31, label %bb8, label %bb12
+
+bb8:                              ; preds = %bb14
+  br label %bb9
+
+bb9:                                        ; preds = %bb8, %bb9
+  %i1.02 = phi i64 [ %inc14, %bb9 ], [ 0, %bb8 ]
+  %sub7 = sub nsw i64 %i1.02, 3
+  %add8 = add nsw i64 %i1.02, 3
+  %mul9 = mul nsw i64 %sub7, %add8
+  %rem10 = srem i64 %mul9, %i1.02
+  %conv11 = trunc i64 %rem10 to i32
+  %arrayidx12 = getelementptr inbounds [1024 x i32], [1024 x i32]* @B, i64 0, i64 %i1.02
+  store i32 %conv11, i32* %arrayidx12, align 4
+  %inc14 = add nsw i64 %i1.02, 1
+  %cmp3 = icmp slt i64 %inc14, %N
+  br i1 %cmp3, label %bb9, label %bb15
+
+bb15:                               ; preds = %bb9
+  br label %bb12
+
+bb12:                                        ; preds = %bb15, %bb14
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopFusion/loop_nest.ll b/llvm/test/Transforms/LoopFusion/loop_nest.ll
index d6cf21450..44a0ac809 100644
--- a/llvm/test/Transforms/LoopFusion/loop_nest.ll
+++ b/llvm/test/Transforms/LoopFusion/loop_nest.ll
@@ -23,24 +23,18 @@
 ; CHECK-NEXT: bb:
 ; CHECK-NEXT: br label %[[LOOP1HEADER:bb[0-9]+]]
 ; CHECK: [[LOOP1HEADER]]
-; CHECK: br i1 %exitcond12, label %[[LOOP3PREHEADER:bb[0-9]+.preheader]], label %[[LOOP2HEADER:bb[0-9]+]]
-; CHECK: [[LOOP3PREHEADER]]
 ; CHECK: br label %[[LOOP3HEADER:bb[0-9]+]]
 ; CHECK: [[LOOP3HEADER]]
-; CHECK: br i1 %exitcond9, label %[[LOOP3BODY:bb[0-9]+]], label %[[LOOP1LATCH:bb[0-9]+]]
-; CHECK: [[LOOP1LATCH]]
 ; CHECK: br label %[[LOOP2HEADER:bb[0-9]+]]
 ; CHECK: [[LOOP2HEADER]]
-; CHECK: br i1 %exitcond6, label %[[LOOP4PREHEADER:bb[0-9]+.preheader]], label %[[LOOP2EXITBLOCK:bb[0-9]+]]
-; CHECK: [[LOOP4PREHEADER]]
 ; CHECK: br label %[[LOOP4HEADER:bb[0-9]+]]
-; CHECK: [[LOOP2EXITBLOCK]]
-; CHECK-NEXT: br label %[[FUNCEXIT:bb[0-9]+]]
 ; CHECK: [[LOOP4HEADER]]
-; CHECK: br i1 %exitcond, label %[[LOOP4BODY:bb[0-9]+]], label %[[LOOP2LATCH:bb[0-9]+]]
-; CHECK: [[LOOP2LATCH]]
-; CHECK: br label %[[LOOP1HEADER:bb[0-9]+]]
-; CHECK: [[FUNCEXIT]]
+; CHECK: br i1 %{{.*}}, label %[[LOOP3HEADER]], label %[[LOOP1LATCH:bb[0-9]+]]
+; CHECK: [[LOOP1LATCH]]
+; CHECK-NEXT: %inc.outer.fc0 = add nuw nsw i64 %indvars.iv105, 1
+; CHECK-NEXT: %add.outer.fc0 = add nuw nsw i32 %.06, 1
+; CHECK-NEXT: %cmp.outer.fc0 = icmp ne i64 %inc.outer.fc0, 100
+; CHECK: br i1 %{{.*}}, label %[[LOOP1HEADER]], label %[[LOOP1EXIT:bb[0-9]*]]
 ; CHECK: ret void
 
 ; TODO: The current version of loop fusion does not allow the inner loops to be
@@ -48,72 +42,69 @@
 ; limitations that can be addressed in future improvements to fusion.
 define void @dep_free() {
 bb:
-  br label %bb13
+  br label %bb16
 
-bb13:                                             ; preds = %bb27, %bb
-  %indvars.iv10 = phi i64 [ %indvars.iv.next11, %bb27 ], [ 0, %bb ]
-  %.0 = phi i32 [ 0, %bb ], [ %tmp28, %bb27 ]
-  %exitcond12 = icmp ne i64 %indvars.iv10, 100
-  br i1 %exitcond12, label %bb16, label %bb30
+bb16:                                   ; preds = %bb, %bb27
+  %.06 = phi i32 [ 0, %bb ], [ %add.outer.fc0, %bb27 ]
+  %indvars.iv105 = phi i64 [ 0, %bb ], [ %inc.outer.fc0, %bb27 ]
+  br label %bb18
 
-bb16:                                             ; preds = %bb25, %bb13
-  %indvars.iv7 = phi i64 [ %indvars.iv.next8, %bb25 ], [ 0, %bb13 ]
-  %exitcond9 = icmp ne i64 %indvars.iv7, 100
-  br i1 %exitcond9, label %bb18, label %bb27
+bb30:                                   ; preds = %bb27
+  br label %bb33
 
-bb18:                                             ; preds = %bb16
-  %tmp = add nsw i32 %.0, -3
-  %tmp19 = add nuw nsw i64 %indvars.iv10, 3
+bb18:                                             ; preds = %bb16, %bb25
+  %indvars.iv74 = phi i64 [ 0, %bb16 ], [ %indvars.iv.next8, %bb25 ]
+  %tmp = add nsw i32 %.06, -3
+  %tmp19 = add nuw nsw i64 %indvars.iv105, 3
   %tmp20 = trunc i64 %tmp19 to i32
   %tmp21 = mul nsw i32 %tmp, %tmp20
-  %tmp22 = trunc i64 %indvars.iv10 to i32
+  %tmp22 = trunc i64 %indvars.iv105 to i32
   %tmp23 = srem i32 %tmp21, %tmp22
-  %tmp24 = getelementptr inbounds [1024 x [1024 x i32]], [1024 x [1024 x i32]]* @A, i64 0, i64 %indvars.iv10, i64 %indvars.iv7
+  %tmp24 = getelementptr inbounds [1024 x [1024 x i32]], [1024 x [1024 x i32]]* @A, i64 0, i64 %indvars.iv105, i64 %indvars.iv74
   store i32 %tmp23, i32* %tmp24, align 4
   br label %bb25
 
 bb25:                                             ; preds = %bb18
-  %indvars.iv.next8 = add nuw nsw i64 %indvars.iv7, 1
-  br label %bb16
+  %indvars.iv.next8 = add nuw nsw i64 %indvars.iv74, 1
+  %exitcond9 = icmp ne i64 %indvars.iv.next8, 100
+  br i1 %exitcond9, label %bb18, label %bb27
 
-bb27:                                             ; preds = %bb16
-  %indvars.iv.next11 = add nuw nsw i64 %indvars.iv10, 1
-  %tmp28 = add nuw nsw i32 %.0, 1
-  br label %bb13
+bb27:                                             ; preds = %bb25
+  %inc.outer.fc0 = add nuw nsw i64 %indvars.iv105, 1
+  %add.outer.fc0 = add nuw nsw i32 %.06, 1
+  %cmp.outer.fc0 = icmp ne i64 %inc.outer.fc0, 100
+  br i1 %cmp.outer.fc0, label %bb16, label %bb30
 
-bb30:                                             ; preds = %bb45, %bb13
-  %indvars.iv4 = phi i64 [ %indvars.iv.next5, %bb45 ], [ 0, %bb13 ]
-  %.02 = phi i32 [ 0, %bb13 ], [ %tmp46, %bb45 ]
-  %exitcond6 = icmp ne i64 %indvars.iv4, 100
-  br i1 %exitcond6, label %bb33, label %bb31
+bb33:                                   ; preds = %bb30, %bb45
+  %.023 = phi i32 [ 0, %bb30 ], [ %tmp46, %bb45 ]
+  %indvars.iv42 = phi i64 [ 0, %bb30 ], [ %indvars.iv.next5, %bb45 ]
+  br label %bb35
 
-bb31:                                             ; preds = %bb30
+bb31:                                             ; preds = %bb45
   br label %bb47
 
-bb33:                                             ; preds = %bb43, %bb30
-  %indvars.iv = phi i64 [ %indvars.iv.next, %bb43 ], [ 0, %bb30 ]
-  %exitcond = icmp ne i64 %indvars.iv, 100
-  br i1 %exitcond, label %bb35, label %bb45
-
-bb35:                                             ; preds = %bb33
-  %tmp36 = add nsw i32 %.02, -3
-  %tmp37 = add nuw nsw i64 %indvars.iv4, 3
+bb35:                                             ; preds = %bb33, %bb43
+  %indvars.iv1 = phi i64 [ 0, %bb33 ], [ %indvars.iv.next, %bb43 ]
+  %tmp36 = add nsw i32 %.023, -3
+  %tmp37 = add nuw nsw i64 %indvars.iv42, 3
   %tmp38 = trunc i64 %tmp37 to i32
   %tmp39 = mul nsw i32 %tmp36, %tmp38
-  %tmp40 = trunc i64 %indvars.iv4 to i32
+  %tmp40 = trunc i64 %indvars.iv42 to i32
   %tmp41 = srem i32 %tmp39, %tmp40
-  %tmp42 = getelementptr inbounds [1024 x [1024 x i32]], [1024 x [1024 x i32]]* @B, i64 0, i64 %indvars.iv4, i64 %indvars.iv
+  %tmp42 = getelementptr inbounds [1024 x [1024 x i32]], [1024 x [1024 x i32]]* @B, i64 0, i64 %indvars.iv42, i64 %indvars.iv1
   store i32 %tmp41, i32* %tmp42, align 4
   br label %bb43
 
 bb43:                                             ; preds = %bb35
-  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
-  br label %bb33
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv1, 1
+  %exitcond = icmp ne i64 %indvars.iv.next, 100
+  br i1 %exitcond, label %bb35, label %bb45
 
-bb45:                                             ; preds = %bb33
-  %indvars.iv.next5 = add nuw nsw i64 %indvars.iv4, 1
-  %tmp46 = add nuw nsw i32 %.02, 1
-  br label %bb30
+bb45:                                             ; preds = %bb43
+  %indvars.iv.next5 = add nuw nsw i64 %indvars.iv42, 1
+  %tmp46 = add nuw nsw i32 %.023, 1
+  %exitcond6 = icmp ne i64 %indvars.iv.next5, 100
+  br i1 %exitcond6, label %bb33, label %bb31
 
 bb47:                                             ; preds = %bb31
   ret void
diff --git a/llvm/test/Transforms/LoopFusion/simple.ll b/llvm/test/Transforms/LoopFusion/simple.ll
index 9d2db2b91..aeb626126 100644
--- a/llvm/test/Transforms/LoopFusion/simple.ll
+++ b/llvm/test/Transforms/LoopFusion/simple.ll
@@ -6,68 +6,59 @@
 ; CHECK-NEXT: bb:
 ; CHECK-NEXT: br label %[[LOOP1HEADER:bb[0-9]*]]
 ; CHECK: [[LOOP1HEADER]]
-; CHECK: br i1 %{{.*}}, label %[[LOOP1BODY:bb[0-9]*]], label %[[LOOP2PREHEADER:bb[0-9]+]]
-; CHECK: [[LOOP1BODY]]
-; CHECK: br label %[[LOOP1LATCH:bb[0-9]*]]
-; CHECK: [[LOOP1LATCH]]
-; CHECK: br label %[[LOOP2PREHEADER:bb[0-9]+]]
-; CHECK: [[LOOP2PREHEADER]]
-; CHECK: br i1 %{{.*}}, label %[[LOOP2BODY:bb[0-9]*]], label %[[LOOP2EXIT:bb[0-9]*]]
-; CHECK: [[LOOP2BODY]]
+; CHECK: br label %[[LOOP2HEADER:bb[0-9]*]]
+; CHECK: [[LOOP2HEADER]]
 ; CHECK: br label %[[LOOP2LATCH:bb[0-9]+]]
 ; CHECK: [[LOOP2LATCH]]
-; CHECK: br label %[[LOOP1HEADER]]
+; CHECK: br i1 %{{.*}}, label %[[LOOP1HEADER]], label %{{.*}}
 ; CHECK: ret void
 define void @dep_free(i32* noalias %arg) {
 bb:
-  br label %bb5
+  br label %bb7
 
-bb5:                                              ; preds = %bb14, %bb
-  %indvars.iv2 = phi i64 [ %indvars.iv.next3, %bb14 ], [ 0, %bb ]
-  %.01 = phi i32 [ 0, %bb ], [ %tmp15, %bb14 ]
-  %exitcond4 = icmp ne i64 %indvars.iv2, 100
-  br i1 %exitcond4, label %bb7, label %bb17
-
-bb7:                                              ; preds = %bb5
-  %tmp = add nsw i32 %.01, -3
-  %tmp8 = add nuw nsw i64 %indvars.iv2, 3
+bb7:                                              ; preds = %bb, %bb14
+  %.014 = phi i32 [ 0, %bb ], [ %tmp15, %bb14 ]
+  %indvars.iv23 = phi i64 [ 0, %bb ], [ %indvars.iv.next3, %bb14 ]
+  %tmp = add nsw i32 %.014, -3
+  %tmp8 = add nuw nsw i64 %indvars.iv23, 3
   %tmp9 = trunc i64 %tmp8 to i32
   %tmp10 = mul nsw i32 %tmp, %tmp9
-  %tmp11 = trunc i64 %indvars.iv2 to i32
+  %tmp11 = trunc i64 %indvars.iv23 to i32
   %tmp12 = srem i32 %tmp10, %tmp11
-  %tmp13 = getelementptr inbounds i32, i32* %arg, i64 %indvars.iv2
+  %tmp13 = getelementptr inbounds i32, i32* %arg, i64 %indvars.iv23
   store i32 %tmp12, i32* %tmp13, align 4
   br label %bb14
 
 bb14:                                             ; preds = %bb7
-  %indvars.iv.next3 = add nuw nsw i64 %indvars.iv2, 1
-  %tmp15 = add nuw nsw i32 %.01, 1
-  br label %bb5
-
-bb17:                                             ; preds = %bb27, %bb5
-  %indvars.iv = phi i64 [ %indvars.iv.next, %bb27 ], [ 0, %bb5 ]
-  %.0 = phi i32 [ 0, %bb5 ], [ %tmp28, %bb27 ]
-  %exitcond = icmp ne i64 %indvars.iv, 100
-  br i1 %exitcond, label %bb19, label %bb18
-
-bb18:                                             ; preds = %bb17
-  br label %bb29
-
-bb19:                                             ; preds = %bb17
-  %tmp20 = add nsw i32 %.0, -3
-  %tmp21 = add nuw nsw i64 %indvars.iv, 3
+  %indvars.iv.next3 = add nuw nsw i64 %indvars.iv23, 1
+  %tmp15 = add nuw nsw i32 %.014, 1
+  %exitcond4 = icmp ne i64 %indvars.iv.next3, 100
+  br i1 %exitcond4, label %bb7, label %bb17.preheader
+
+bb17.preheader:                                   ; preds = %bb14
+  br label %bb19
+
+bb19:                                             ; preds = %bb17.preheader, %bb27
+  %.02 = phi i32 [ 0, %bb17.preheader ], [ %tmp28, %bb27 ]
+  %indvars.iv1 = phi i64 [ 0, %bb17.preheader ], [ %indvars.iv.next, %bb27 ]
+  %tmp20 = add nsw i32 %.02, -3
+  %tmp21 = add nuw nsw i64 %indvars.iv1, 3
   %tmp22 = trunc i64 %tmp21 to i32
   %tmp23 = mul nsw i32 %tmp20, %tmp22
-  %tmp24 = trunc i64 %indvars.iv to i32
+  %tmp24 = trunc i64 %indvars.iv1 to i32
   %tmp25 = srem i32 %tmp23, %tmp24
-  %tmp26 = getelementptr inbounds [1024 x i32], [1024 x i32]* @B, i64 0, i64 %indvars.iv
+  %tmp26 = getelementptr inbounds [1024 x i32], [1024 x i32]* @B, i64 0, i64 %indvars.iv1
   store i32 %tmp25, i32* %tmp26, align 4
   br label %bb27
 
 bb27:                                             ; preds = %bb19
-  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
-  %tmp28 = add nuw nsw i32 %.0, 1
-  br label %bb17
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv1, 1
+  %tmp28 = add nuw nsw i32 %.02, 1
+  %exitcond = icmp ne i64 %indvars.iv.next, 100
+  br i1 %exitcond, label %bb19, label %bb18
+
+bb18:                                             ; preds = %bb27
+  br label %bb29
 
 bb29:                                             ; preds = %bb18
   ret void
@@ -75,64 +66,56 @@ bb29:                                             ; preds = %bb18
 
 ; CHECK: void @dep_free_parametric
 ; CHECK-NEXT: bb:
-; CHECK-NEXT: br label %[[LOOP1HEADER:bb[0-9]*]]
+; CHECK: br i1 %{{.*}}, label %[[LOOP1PREHEADER:bb[0-9.a-z]*]], label %[[EXITBLOCK:bb[0-9]*]]
+; CHECK: [[LOOP1PREHEADER]]
+; CHECK: br label %[[LOOP1HEADER:bb[0-9]*]]
 ; CHECK: [[LOOP1HEADER]]
-; CHECK: br i1 %{{.*}}, label %[[LOOP1BODY:bb[0-9]*]], label %[[LOOP2PREHEADER:bb[0-9]+]]
-; CHECK: [[LOOP1BODY]]
-; CHECK: br label %[[LOOP1LATCH:bb[0-9]*]]
-; CHECK: [[LOOP1LATCH]]
-; CHECK: br label %[[LOOP2PREHEADER:bb[0-9]+]]
-; CHECK: [[LOOP2PREHEADER]]
-; CHECK: br i1 %{{.*}}, label %[[LOOP2BODY:bb[0-9]*]], label %[[LOOP2EXIT:bb[0-9]*]]
-; CHECK: [[LOOP2BODY]]
+; CHECK: br label %[[LOOP2HEADER:bb[0-9]*]]
+; CHECK: [[LOOP2HEADER]]
 ; CHECK: br label %[[LOOP2LATCH:bb[0-9]+]]
 ; CHECK: [[LOOP2LATCH]]
-; CHECK: br label %[[LOOP1HEADER]]
+; CHECK: br i1 %{{.*}}, label %[[LOOP1HEADER]], label %[[EXITBLOCK]]
 ; CHECK: ret void
 define void @dep_free_parametric(i32* noalias %arg, i64 %arg2) {
 bb:
-  br label %bb3
-
-bb3:                                              ; preds = %bb12, %bb
-  %.01 = phi i64 [ 0, %bb ], [ %tmp13, %bb12 ]
-  %tmp = icmp slt i64 %.01, %arg2
-  br i1 %tmp, label %bb5, label %bb15
+  %tmp3 = icmp slt i64 0, %arg2
+  br i1 %tmp3, label %bb5, label %bb15.preheader
 
-bb5:                                              ; preds = %bb3
-  %tmp6 = add nsw i64 %.01, -3
-  %tmp7 = add nuw nsw i64 %.01, 3
+bb5:                                              ; preds = %bb5, %bb12
+  %.014 = phi i64 [ 0, %bb ], [ %tmp13, %bb12 ]
+  %tmp6 = add nsw i64 %.014, -3
+  %tmp7 = add nuw nsw i64 %.014, 3
   %tmp8 = mul nsw i64 %tmp6, %tmp7
-  %tmp9 = srem i64 %tmp8, %.01
+  %tmp9 = srem i64 %tmp8, %.014
   %tmp10 = trunc i64 %tmp9 to i32
-  %tmp11 = getelementptr inbounds i32, i32* %arg, i64 %.01
+  %tmp11 = getelementptr inbounds i32, i32* %arg, i64 %.014
   store i32 %tmp10, i32* %tmp11, align 4
   br label %bb12
 
 bb12:                                             ; preds = %bb5
-  %tmp13 = add nuw nsw i64 %.01, 1
-  br label %bb3
-
-bb15:                                             ; preds = %bb25, %bb3
-  %.0 = phi i64 [ 0, %bb3 ], [ %tmp26, %bb25 ]
-  %tmp16 = icmp slt i64 %.0, %arg2
-  br i1 %tmp16, label %bb18, label %bb17
-
-bb17:                                             ; preds = %bb15
-  br label %bb27
-
-bb18:                                             ; preds = %bb15
-  %tmp19 = add nsw i64 %.0, -3
-  %tmp20 = add nuw nsw i64 %.0, 3
+  %tmp13 = add nuw nsw i64 %.014, 1
+  %tmp = icmp slt i64 %tmp13, %arg2
+  br i1 %tmp, label %bb5, label %bb15.preheader
+
+bb15.preheader:                                   ; preds = %bb12, %bb
+  %tmp161 = icmp slt i64 0, %arg2
+  br i1 %tmp161, label %bb18, label %bb27
+
+bb18:                                             ; preds = %bb15.preheader, %bb25
+  %.02 = phi i64 [ 0, %bb15.preheader ], [ %tmp26, %bb25 ]
+  %tmp19 = add nsw i64 %.02, -3
+  %tmp20 = add nuw nsw i64 %.02, 3
   %tmp21 = mul nsw i64 %tmp19, %tmp20
-  %tmp22 = srem i64 %tmp21, %.0
+  %tmp22 = srem i64 %tmp21, %.02
   %tmp23 = trunc i64 %tmp22 to i32
-  %tmp24 = getelementptr inbounds [1024 x i32], [1024 x i32]* @B, i64 0, i64 %.0
+  %tmp24 = getelementptr inbounds [1024 x i32], [1024 x i32]* @B, i64 0, i64 %.02
   store i32 %tmp23, i32* %tmp24, align 4
   br label %bb25
 
 bb25:                                             ; preds = %bb18
-  %tmp26 = add nuw nsw i64 %.0, 1
-  br label %bb15
+  %tmp26 = add nuw nsw i64 %.02, 1
+  %tmp16 = icmp slt i64 %tmp26, %arg2
+  br i1 %tmp16, label %bb18, label %bb27
 
 bb27:                                             ; preds = %bb17
   ret void
@@ -142,113 +125,83 @@ bb27:                                             ; preds = %bb17
 ; CHECK-NEXT: bb:
 ; CHECK-NEXT: br label %[[LOOP1HEADER:bb[0-9]*]]
 ; CHECK: [[LOOP1HEADER]]
-; CHECK: br i1 %{{.*}}, label %[[LOOP1BODY:bb[0-9]*]], label %[[LOOP2PREHEADER:bb[0-9]+]]
-; CHECK: [[LOOP1BODY]]
-; CHECK: br label %[[LOOP1LATCH:bb[0-9]*]]
-; CHECK: [[LOOP1LATCH]]
-; CHECK: br label %[[LOOP2PREHEADER:bb[0-9]+]]
-; CHECK: [[LOOP2PREHEADER]]
-; CHECK: br i1 %{{.*}}, label %[[LOOP2BODY:bb[0-9]*]], label %[[LOOP2EXIT:bb[0-9]*]]
-; CHECK: [[LOOP2BODY]]
+; CHECK: br label %[[LOOP2HEADER:bb[0-9]*]]
+; CHECK: [[LOOP2HEADER]]
 ; CHECK: br label %[[LOOP2LATCH:bb[0-9]+]]
 ; CHECK: [[LOOP2LATCH]]
-; CHECK: br label %[[LOOP1HEADER]]
+; CHECK: br i1 %{{.*}}, label %[[LOOP1HEADER]], label %{{.*}}
 ; CHECK: ret void
 define void @raw_only(i32* noalias %arg) {
 bb:
-  br label %bb5
+  br label %bb7
 
-bb5:                                              ; preds = %bb9, %bb
-  %indvars.iv2 = phi i64 [ %indvars.iv.next3, %bb9 ], [ 0, %bb ]
-  %exitcond4 = icmp ne i64 %indvars.iv2, 100
-  br i1 %exitcond4, label %bb7, label %bb11
+bb11.preheader:                                   ; preds = %bb9
+  br label %bb13
 
-bb7:                                              ; preds = %bb5
-  %tmp = getelementptr inbounds i32, i32* %arg, i64 %indvars.iv2
-  %tmp8 = trunc i64 %indvars.iv2 to i32
+bb7:                                              ; preds = %bb, %bb9
+  %indvars.iv22 = phi i64 [ 0, %bb ], [ %indvars.iv.next3, %bb9 ]
+  %tmp = getelementptr inbounds i32, i32* %arg, i64 %indvars.iv22
+  %tmp8 = trunc i64 %indvars.iv22 to i32
   store i32 %tmp8, i32* %tmp, align 4
   br label %bb9
 
 bb9:                                              ; preds = %bb7
-  %indvars.iv.next3 = add nuw nsw i64 %indvars.iv2, 1
-  br label %bb5
+  %indvars.iv.next3 = add nuw nsw i64 %indvars.iv22, 1
+  %exitcond4 = icmp ne i64 %indvars.iv.next3, 100
+  br i1 %exitcond4, label %bb7, label %bb11.preheader
 
-bb11:                                             ; preds = %bb18, %bb5
-  %indvars.iv = phi i64 [ %indvars.iv.next, %bb18 ], [ 0, %bb5 ]
-  %exitcond = icmp ne i64 %indvars.iv, 100
-  br i1 %exitcond, label %bb13, label %bb19
-
-bb13:                                             ; preds = %bb11
-  %tmp14 = getelementptr inbounds i32, i32* %arg, i64 %indvars.iv
+bb13:                                             ; preds = %bb11.preheader, %bb18
+  %indvars.iv1 = phi i64 [ 0, %bb11.preheader ], [ %indvars.iv.next, %bb18 ]
+  %tmp14 = getelementptr inbounds i32, i32* %arg, i64 %indvars.iv1
   %tmp15 = load i32, i32* %tmp14, align 4
   %tmp16 = shl nsw i32 %tmp15, 1
-  %tmp17 = getelementptr inbounds [1024 x i32], [1024 x i32]* @B, i64 0, i64 %indvars.iv
+  %tmp17 = getelementptr inbounds [1024 x i32], [1024 x i32]* @B, i64 0, i64 %indvars.iv1
   store i32 %tmp16, i32* %tmp17, align 4
   br label %bb18
 
 bb18:                                             ; preds = %bb13
-  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
-  br label %bb11
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv1, 1
+  %exitcond = icmp ne i64 %indvars.iv.next, 100 br i1 %exitcond, label %bb13, label %bb19
 
-bb19:                                             ; preds = %bb11
+bb19:                                             ; preds = %bb18
   ret void
 }
 
 ; CHECK: void @raw_only_parametric
 ; CHECK-NEXT: bb:
+; CHECK: br i1 %{{.*}}, label %[[LOOP1PREHEADER:bb[0-9.a-z]*]], label %[[EXITBLOCK:bb[0-9]*]]
+; CHECK: [[LOOP1PREHEADER]]
 ; CHECK: br label %[[LOOP1HEADER:bb[0-9]*]]
 ; CHECK: [[LOOP1HEADER]]
-; CHECK: br i1 %{{.*}}, label %[[LOOP1BODY:bb[0-9]*]], label %[[LOOP2PREHEADER:bb[0-9]+]]
-; CHECK: [[LOOP1BODY]]
-; CHECK: br label %[[LOOP1LATCH:bb[0-9]*]]
-; CHECK: [[LOOP1LATCH]]
-; CHECK: br label %[[LOOP2PREHEADER:bb[0-9]+]]
-; CHECK: [[LOOP2PREHEADER]]
-; CHECK: br i1 %{{.*}}, label %[[LOOP2BODY:bb[0-9]*]], label %[[LOOP2EXIT:bb[0-9]*]]
-; CHECK: [[LOOP2BODY]]
-; CHECK: br label %[[LOOP2LATCH:bb[0-9]+]]
-; CHECK: [[LOOP2LATCH]]
-; CHECK: br label %[[LOOP1HEADER]]
+; CHECK: br i1 %{{.*}}, label %[[LOOP1HEADER]], label %[[EXITBLOCK]]
 ; CHECK: ret void
 define void @raw_only_parametric(i32* noalias %arg, i32 %arg4) {
 bb:
-  br label %bb5
-
-bb5:                                              ; preds = %bb11, %bb
-  %indvars.iv2 = phi i64 [ %indvars.iv.next3, %bb11 ], [ 0, %bb ]
   %tmp = sext i32 %arg4 to i64
-  %tmp6 = icmp slt i64 %indvars.iv2, %tmp
-  br i1 %tmp6, label %bb8, label %bb14
+  %tmp64 = icmp sgt i32 %arg4, 0
+  br i1 %tmp64, label %bb8, label %bb23
 
-bb8:                                              ; preds = %bb5
-  %tmp9 = getelementptr inbounds i32, i32* %arg, i64 %indvars.iv2
-  %tmp10 = trunc i64 %indvars.iv2 to i32
+bb8:                                              ; preds = %bb, %bb8
+  %indvars.iv25 = phi i64 [ %indvars.iv.next3, %bb8 ], [ 0, %bb ]
+  %tmp9 = getelementptr inbounds i32, i32* %arg, i64 %indvars.iv25
+  %tmp10 = trunc i64 %indvars.iv25 to i32
   store i32 %tmp10, i32* %tmp9, align 4
-  br label %bb11
-
-bb11:                                             ; preds = %bb8
-  %indvars.iv.next3 = add nuw nsw i64 %indvars.iv2, 1
-  br label %bb5
-
-bb14:                                             ; preds = %bb22, %bb5
-  %indvars.iv = phi i64 [ %indvars.iv.next, %bb22 ], [ 0, %bb5 ]
-  %tmp13 = sext i32 %arg4 to i64
-  %tmp15 = icmp slt i64 %indvars.iv, %tmp13
-  br i1 %tmp15, label %bb17, label %bb23
+  %indvars.iv.next3 = add nuw nsw i64 %indvars.iv25, 1
+  %tmp6 = icmp slt i64 %indvars.iv.next3, %tmp
+  br i1 %tmp6, label %bb8, label %bb17
 
-bb17:                                             ; preds = %bb14
-  %tmp18 = getelementptr inbounds i32, i32* %arg, i64 %indvars.iv
+bb17:                                             ; preds = %bb8, %bb17
+  %indvars.iv3 = phi i64 [ %indvars.iv.next, %bb17 ], [ 0, %bb8 ]
+  %tmp18 = getelementptr inbounds i32, i32* %arg, i64 %indvars.iv3
   %tmp19 = load i32, i32* %tmp18, align 4
   %tmp20 = shl nsw i32 %tmp19, 1
-  %tmp21 = getelementptr inbounds [1024 x i32], [1024 x i32]* @B, i64 0, i64 %indvars.iv
+  %tmp21 = getelementptr inbounds [1024 x i32], [1024 x i32]* @B, i64 0, i64 %indvars.iv3
   store i32 %tmp20, i32* %tmp21, align 4
-  br label %bb22
-
-bb22:                                             ; preds = %bb17
-  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
-  br label %bb14
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv3, 1
+  %tmp15 = icmp slt i64 %indvars.iv.next, %tmp
+  br i1 %tmp15, label %bb17, label %bb23
 
-bb23:                                             ; preds = %bb14
+bb23:                                             ; preds = %bb17, %bb
   ret void
 }
 
@@ -256,62 +209,100 @@ bb23:                                             ; preds = %bb14
 ; CHECK-NEXT: bb:
 ; CHECK: br label %[[LOOP1HEADER:bb[0-9]*]]
 ; CHECK: [[LOOP1HEADER]]
-; CHECK: br i1 %{{.*}}, label %[[LOOP1BODY:bb[0-9]*]], label %[[LOOP2PREHEADER:bb[0-9]+]]
-; CHECK: [[LOOP1BODY]]
-; CHECK: br label %[[LOOP1LATCH:bb[0-9]*]]
-; CHECK: [[LOOP1LATCH]]
-; CHECK: br label %[[LOOP2PREHEADER:bb[0-9]+]]
-; CHECK: [[LOOP2PREHEADER]]
-; CHECK: br i1 %{{.*}}, label %[[LOOP2BODY:bb[0-9]*]], label %[[LOOP2EXIT:bb[0-9]*]]
-; CHECK: [[LOOP2BODY]]
+; CHECK: br label %[[LOOP2HEADER:bb[0-9]*]]
+; CHECK: [[LOOP2HEADER]]
 ; CHECK: br label %[[LOOP2LATCH:bb[0-9]+]]
 ; CHECK: [[LOOP2LATCH]]
-; CHECK: br label %[[LOOP1HEADER]]
+; CHECK: br i1 %{{.*}}, label %[[LOOP1HEADER]], label %{{.*}}
 ; CHECK: ret void
 define void @forward_dep(i32* noalias %arg) {
 bb:
-  br label %bb5
+  br label %bb7
 
-bb5:                                              ; preds = %bb14, %bb
-  %indvars.iv2 = phi i64 [ %indvars.iv.next3, %bb14 ], [ 0, %bb ]
-  %.01 = phi i32 [ 0, %bb ], [ %tmp15, %bb14 ]
-  %exitcond4 = icmp ne i64 %indvars.iv2, 100
-  br i1 %exitcond4, label %bb7, label %bb17
-
-bb7:                                              ; preds = %bb5
-  %tmp = add nsw i32 %.01, -3
-  %tmp8 = add nuw nsw i64 %indvars.iv2, 3
+bb7:                                              ; preds = %bb, %bb14
+  %.013 = phi i32 [ 0, %bb ], [ %tmp15, %bb14 ]
+  %indvars.iv22 = phi i64 [ 0, %bb ], [ %indvars.iv.next3, %bb14 ]
+  %tmp = add nsw i32 %.013, -3
+  %tmp8 = add nuw nsw i64 %indvars.iv22, 3
   %tmp9 = trunc i64 %tmp8 to i32
   %tmp10 = mul nsw i32 %tmp, %tmp9
-  %tmp11 = trunc i64 %indvars.iv2 to i32
+  %tmp11 = trunc i64 %indvars.iv22 to i32
   %tmp12 = srem i32 %tmp10, %tmp11
-  %tmp13 = getelementptr inbounds i32, i32* %arg, i64 %indvars.iv2
+  %tmp13 = getelementptr inbounds i32, i32* %arg, i64 %indvars.iv22
   store i32 %tmp12, i32* %tmp13, align 4
   br label %bb14
 
 bb14:                                             ; preds = %bb7
-  %indvars.iv.next3 = add nuw nsw i64 %indvars.iv2, 1
-  %tmp15 = add nuw nsw i32 %.01, 1
-  br label %bb5
-
-bb17:                                             ; preds = %bb25, %bb5
-  %indvars.iv = phi i64 [ %indvars.iv.next, %bb25 ], [ 0, %bb5 ]
-  %exitcond = icmp ne i64 %indvars.iv, 100
-  br i1 %exitcond, label %bb19, label %bb26
-
-bb19:                                             ; preds = %bb17
-  %tmp20 = add nsw i64 %indvars.iv, -3
+  %indvars.iv.next3 = add nuw nsw i64 %indvars.iv22, 1
+  %tmp15 = add nuw nsw i32 %.013, 1
+  %exitcond4 = icmp ne i64 %indvars.iv.next3, 100
+  br i1 %exitcond4, label %bb7, label %bb19
+
+bb19:                                             ; preds = %bb14, %bb25
+  %indvars.iv1 = phi i64 [ 0, %bb14 ], [ %indvars.iv.next, %bb25 ]
+  %tmp20 = add nsw i64 %indvars.iv1, -3
   %tmp21 = getelementptr inbounds i32, i32* %arg, i64 %tmp20
   %tmp22 = load i32, i32* %tmp21, align 4
   %tmp23 = mul nsw i32 %tmp22, 3
-  %tmp24 = getelementptr inbounds i32, i32* %arg, i64 %indvars.iv
+  %tmp24 = getelementptr inbounds i32, i32* %arg, i64 %indvars.iv1
   store i32 %tmp23, i32* %tmp24, align 4
   br label %bb25
 
 bb25:                                             ; preds = %bb19
-  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
-  br label %bb17
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv1, 1
+  %exitcond = icmp ne i64 %indvars.iv.next, 100
+  br i1 %exitcond, label %bb19, label %bb26
+
+bb26:                                             ; preds = %bb25
+  ret void
+}
 
-bb26:                                             ; preds = %bb17
+; Test that instructions in loop 1 latch are moved to the beginning of loop 2
+; latch iff it is proven safe. %inc.first and %cmp.first are moved, but
+; `store i32 0, i32* %Ai.first` is not.
+
+; CHECK: void @flow_dep
+; CHECK-LABEL: entry:
+; CHECK-NEXT: br label %for.first
+; CHECK-LABEL: for.first:
+; CHECK: store i32 0, i32* %Ai.first
+; CHECK: %Ai.second =
+; CHECK: br label %for.second.latch
+; CHECK-LABEL: for.second.latch:
+; CHECK-NEXT: %inc.first = add nsw i64 %i.first, 1
+; CHECK-NEXT: %cmp.first = icmp slt i64 %inc.first, 100
+; CHECK: br i1 %cmp.second, label %for.first, label %for.end
+; CHECK-LABEL: for.end:
+; CHECK-NEXT: ret void
+
+define void @flow_dep(i32* noalias %A, i32* noalias %B) {
+entry:
+  br label %for.first
+
+for.first:
+  %i.first = phi i64 [ 0, %entry ], [ %inc.first, %for.first ]
+  %Ai.first = getelementptr inbounds i32, i32* %A, i64 %i.first
+  store i32 0, i32* %Ai.first, align 4
+  %inc.first = add nsw i64 %i.first, 1
+  %cmp.first = icmp slt i64 %inc.first, 100
+  br i1 %cmp.first, label %for.first, label %for.second.preheader
+
+for.second.preheader:
+  br label %for.second
+
+for.second:
+  %i.second = phi i64 [ %inc.second, %for.second.latch ], [ 0, %for.second.preheader ]
+  %Ai.second = getelementptr inbounds i32, i32* %A, i64 %i.second
+  %0 = load i32, i32* %Ai.second, align 4
+  %Bi = getelementptr inbounds i32, i32* %B, i64 %i.second
+  store i32 %0, i32* %Bi, align 4
+  br label %for.second.latch
+
+for.second.latch:
+  %inc.second = add nsw i64 %i.second, 1
+  %cmp.second = icmp slt i64 %inc.second, 100
+  br i1 %cmp.second, label %for.second, label %for.end
+
+for.end:
   ret void
 }
diff --git a/llvm/test/Transforms/LoopInstSimplify/basic.ll b/llvm/test/Transforms/LoopInstSimplify/basic.ll
index d6f177f07..a5f7222fa 100644
--- a/llvm/test/Transforms/LoopInstSimplify/basic.ll
+++ b/llvm/test/Transforms/LoopInstSimplify/basic.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -S %s -passes=loop-instsimplify | FileCheck %s
-; RUN: opt -S %s -passes=loop-instsimplify -enable-mssa-loop-dependency=true -verify-memoryssa | FileCheck %s
+; RUN: opt -S %s -passes='loop-mssa(loop-instsimplify)' -verify-memoryssa | FileCheck %s
 
 ; Test very basic folding and propagation occurs within a loop body. This should
 ; collapse to the loop iteration structure and the LCSSA PHI node.
diff --git a/llvm/test/Transforms/LoopInterchange/interchangeable.ll b/llvm/test/Transforms/LoopInterchange/interchangeable.ll
index a97981c51..3dc6bdc43 100644
--- a/llvm/test/Transforms/LoopInterchange/interchangeable.ll
+++ b/llvm/test/Transforms/LoopInterchange/interchangeable.ll
@@ -18,26 +18,28 @@ define void @interchange_01(i64 %k, i64 %N) {
 ; CHECK:       for1.header.preheader:
 ; CHECK-NEXT:    br label [[FOR1_HEADER:%.*]]
 ; CHECK:       for1.header:
-; CHECK-NEXT:    [[INDVARS_IV23:%.*]] = phi i64 [ [[INDVARS_IV_NEXT24:%.*]], [[FOR1_INC10:%.*]] ], [ 0, [[FOR1_HEADER_PREHEADER:%.*]] ]
+; CHECK-NEXT:    [[J23:%.*]] = phi i64 [ [[J_NEXT24:%.*]], [[FOR1_INC10:%.*]] ], [ 0, [[FOR1_HEADER_PREHEADER:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR2_SPLIT1:%.*]]
 ; CHECK:       for2.preheader:
 ; CHECK-NEXT:    br label [[FOR2:%.*]]
 ; CHECK:       for2:
-; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR2_SPLIT:%.*]] ], [ 0, [[FOR2_PREHEADER]] ]
+; CHECK-NEXT:    [[J:%.*]] = phi i64 [ [[TMP0:%.*]], [[FOR2_SPLIT:%.*]] ], [ 0, [[FOR2_PREHEADER]] ]
 ; CHECK-NEXT:    br label [[FOR1_HEADER_PREHEADER]]
 ; CHECK:       for2.split1:
-; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds [100 x [100 x i64]], [100 x [100 x i64]]* @A, i64 0, i64 [[INDVARS_IV]], i64 [[INDVARS_IV23]]
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds [100 x [100 x i64]], [100 x [100 x i64]]* @A, i64 0, i64 [[J]], i64 [[J23]]
 ; CHECK-NEXT:    [[LV:%.*]] = load i64, i64* [[ARRAYIDX5]]
 ; CHECK-NEXT:    [[ADD:%.*]] = add nsw i64 [[LV]], [[K:%.*]]
 ; CHECK-NEXT:    store i64 [[ADD]], i64* [[ARRAYIDX5]]
+; CHECK-NEXT:    [[J_NEXT:%.*]] = add nuw nsw i64 [[J]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[J]], 99
 ; CHECK-NEXT:    br label [[FOR1_INC10]]
 ; CHECK:       for2.split:
-; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
-; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV]], 99
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END12:%.*]], label [[FOR2]]
+; CHECK-NEXT:    [[TMP0]] = add nuw nsw i64 [[J]], 1
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i64 [[J]], 99
+; CHECK-NEXT:    br i1 [[TMP1]], label [[FOR_END12:%.*]], label [[FOR2]]
 ; CHECK:       for1.inc10:
-; CHECK-NEXT:    [[INDVARS_IV_NEXT24]] = add nuw nsw i64 [[INDVARS_IV23]], 1
-; CHECK-NEXT:    [[EXITCOND26:%.*]] = icmp eq i64 [[INDVARS_IV23]], 99
+; CHECK-NEXT:    [[J_NEXT24]] = add nuw nsw i64 [[J23]], 1
+; CHECK-NEXT:    [[EXITCOND26:%.*]] = icmp eq i64 [[J23]], 99
 ; CHECK-NEXT:    br i1 [[EXITCOND26]], label [[FOR2_SPLIT]], label [[FOR1_HEADER]]
 ; CHECK:       for.end12:
 ; CHECK-NEXT:    ret void
@@ -79,26 +81,28 @@ define void @interchange_02(i64 %k) {
 ; CHECK:       for1.header.preheader:
 ; CHECK-NEXT:    br label [[FOR1_HEADER:%.*]]
 ; CHECK:       for1.header:
-; CHECK-NEXT:    [[INDVARS_IV19:%.*]] = phi i64 [ [[INDVARS_IV_NEXT20:%.*]], [[FOR1_INC10:%.*]] ], [ 0, [[FOR1_HEADER_PREHEADER:%.*]] ]
+; CHECK-NEXT:    [[J19:%.*]] = phi i64 [ [[J_NEXT20:%.*]], [[FOR1_INC10:%.*]] ], [ 0, [[FOR1_HEADER_PREHEADER:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR3_SPLIT1:%.*]]
 ; CHECK:       for3.preheader:
 ; CHECK-NEXT:    br label [[FOR3:%.*]]
 ; CHECK:       for3:
-; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR3_SPLIT:%.*]] ], [ 100, [[FOR3_PREHEADER]] ]
+; CHECK-NEXT:    [[J:%.*]] = phi i64 [ [[TMP1:%.*]], [[FOR3_SPLIT:%.*]] ], [ 100, [[FOR3_PREHEADER]] ]
 ; CHECK-NEXT:    br label [[FOR1_HEADER_PREHEADER]]
 ; CHECK:       for3.split1:
-; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds [100 x [100 x i64]], [100 x [100 x i64]]* @A, i64 0, i64 [[INDVARS_IV]], i64 [[INDVARS_IV19]]
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds [100 x [100 x i64]], [100 x [100 x i64]]* @A, i64 0, i64 [[J]], i64 [[J19]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i64, i64* [[ARRAYIDX5]]
 ; CHECK-NEXT:    [[ADD:%.*]] = add nsw i64 [[TMP0]], [[K:%.*]]
 ; CHECK-NEXT:    store i64 [[ADD]], i64* [[ARRAYIDX5]]
+; CHECK-NEXT:    [[J_NEXT:%.*]] = add nsw i64 [[J]], -1
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i64 [[J]], 0
 ; CHECK-NEXT:    br label [[FOR1_INC10]]
 ; CHECK:       for3.split:
-; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nsw i64 [[INDVARS_IV]], -1
-; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i64 [[INDVARS_IV]], 0
-; CHECK-NEXT:    br i1 [[CMP2]], label [[FOR3]], label [[FOR_END11:%.*]]
+; CHECK-NEXT:    [[TMP1]] = add nsw i64 [[J]], -1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp sgt i64 [[J]], 0
+; CHECK-NEXT:    br i1 [[TMP2]], label [[FOR3]], label [[FOR_END11:%.*]]
 ; CHECK:       for1.inc10:
-; CHECK-NEXT:    [[INDVARS_IV_NEXT20]] = add nuw nsw i64 [[INDVARS_IV19]], 1
-; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT20]], 100
+; CHECK-NEXT:    [[J_NEXT20]] = add nuw nsw i64 [[J19]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[J_NEXT20]], 100
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR3_SPLIT]], label [[FOR1_HEADER]]
 ; CHECK:       for.end11:
 ; CHECK-NEXT:    ret void
@@ -139,6 +143,28 @@ for.end11:
 ;; FIXME: DA misses this case after D35430
 
 define void @interchange_10() {
+; CHECK-LABEL: @interchange_10(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR1_HEADER:%.*]]
+; CHECK:       for1.header:
+; CHECK-NEXT:    [[J23:%.*]] = phi i64 [ 1, [[ENTRY:%.*]] ], [ [[J_NEXT24:%.*]], [[FOR1_INC10:%.*]] ]
+; CHECK-NEXT:    [[J_NEXT24]] = add nuw nsw i64 [[J23]], 1
+; CHECK-NEXT:    br label [[FOR2:%.*]]
+; CHECK:       for2:
+; CHECK-NEXT:    [[J:%.*]] = phi i64 [ [[J_NEXT:%.*]], [[FOR2]] ], [ 1, [[FOR1_HEADER]] ]
+; CHECK-NEXT:    [[J_NEXT]] = add nuw nsw i64 [[J]], 1
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds [100 x [100 x i64]], [100 x [100 x i64]]* @A, i64 0, i64 [[J]], i64 [[J23]]
+; CHECK-NEXT:    store i64 [[J]], i64* [[ARRAYIDX5]]
+; CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds [100 x [100 x i64]], [100 x [100 x i64]]* @A, i64 0, i64 [[J]], i64 [[J_NEXT24]]
+; CHECK-NEXT:    store i64 [[J23]], i64* [[ARRAYIDX10]]
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[J]], 99
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR1_INC10]], label [[FOR2]]
+; CHECK:       for1.inc10:
+; CHECK-NEXT:    [[EXITCOND26:%.*]] = icmp eq i64 [[J23]], 98
+; CHECK-NEXT:    br i1 [[EXITCOND26]], label [[FOR_END12:%.*]], label [[FOR1_HEADER]]
+; CHECK:       for.end12:
+; CHECK-NEXT:    ret void
+;
 entry:
   br label %for1.header
 
diff --git a/llvm/test/Transforms/LoopInterchange/perserve-lcssa.ll b/llvm/test/Transforms/LoopInterchange/perserve-lcssa.ll
index af6170987..a1a4d6736 100644
--- a/llvm/test/Transforms/LoopInterchange/perserve-lcssa.ll
+++ b/llvm/test/Transforms/LoopInterchange/perserve-lcssa.ll
@@ -10,13 +10,18 @@
 
 define void @test_lcssa_indvars1()  {
 ; CHECK-LABEL: @test_lcssa_indvars1()
+; CHECK-LABEL: inner.body:
+; CHECK-NEXT:    %iv.inner = phi i64 [ %[[IVNEXT:[0-9]+]], %inner.body.split ], [ 5, %inner.body.preheader ]
+
 ; CHECK-LABEL: inner.body.split:
 ; CHECK-NEXT:    %0 = phi i64 [ %iv.outer.next, %outer.latch ]
-; CHECK-NEXT:    %iv.inner.next = add nsw i64 %iv.inner, -1
+; CHECK-NEXT:    %[[IVNEXT]] = add nsw i64 %iv.inner, -1
+; CHECK-NEXT:    %[[COND:[0-9]+]] = icmp eq i64 %iv.inner, 0
+; CHECK-NEXT:    br i1 %[[COND]], label %exit, label %inner.body
 
 ; CHECK-LABEL: exit:
 ; CHECK-NEXT:    %v4.lcssa = phi i64 [ %0, %inner.body.split ]
-; CHECK-NEXT:    %v8.lcssa.lcssa = phi i64 [ %iv.inner.next, %inner.body.split ]
+; CHECK-NEXT:    %v8.lcssa.lcssa = phi i64 [ %[[IVNEXT]], %inner.body.split ]
 ; CHECK-NEXT:    store i64 %v8.lcssa.lcssa, i64* @b, align 4
 ; CHECK-NEXT:    store i64 %v4.lcssa, i64* @a, align 4
 
@@ -52,9 +57,14 @@ exit:                                             ; preds = %outer.latch
 
 define void @test_lcssa_indvars2()  {
 ; CHECK-LABEL: @test_lcssa_indvars2()
+; CHECK-LABEL: inner.body:
+; CHECK-NEXT:    %iv.inner = phi i64 [ %[[IVNEXT:[0-9]+]], %inner.body.split ], [ 5, %inner.body.preheader ]
+
 ; CHECK-LABEL: inner.body.split:
 ; CHECK-NEXT:    %0 = phi i64 [ %iv.outer, %outer.latch ]
-; CHECK-NEXT:    %iv.inner.next = add nsw i64 %iv.inner, -1
+; CHECK-NEXT:    %[[IVNEXT]] = add nsw i64 %iv.inner, -1
+; CHECK-NEXT:    %[[COND:[0-9]+]] = icmp eq i64 %[[IVNEXT]], 0
+; CHECK-NEXT:    br i1 %[[COND]], label %exit, label %inner.body
 
 ; CHECK-LABEL: exit:
 ; CHECK-NEXT:    %v4.lcssa = phi i64 [ %0, %inner.body.split ]
@@ -93,14 +103,19 @@ exit:                                             ; preds = %outer.latch
 
 define void @test_lcssa_indvars3()  {
 ; CHECK-LABEL: @test_lcssa_indvars3()
+; CHECK-LABEL: inner.body:
+; CHECK-NEXT:    %iv.inner = phi i64 [ %[[IVNEXT:[0-9]+]], %inner.body.split ], [ 5, %inner.body.preheader ]
+
 ; CHECK-LABEL: inner.body.split:
 ; CHECK-NEXT:    %0 = phi i64 [ %iv.outer.next, %outer.latch ]
-; CHECK-NEXT:    %iv.inner.next = add nsw i64 %iv.inner, -1
+; CHECK-NEXT:    %[[IVNEXT]] = add nsw i64 %iv.inner, -1
+; CHECK-NEXT:    %[[COND:[0-9]+]] = icmp eq i64 %iv.inner, 0
+; CHECK-NEXT:    br i1 %[[COND]], label %exit, label %inner.body
 
 ; CHECK-LABEL: exit:
 ; CHECK-NEXT:    %v4.lcssa = phi i64 [ %0, %inner.body.split ]
-; CHECK-NEXT:    %v8.lcssa.lcssa = phi i64 [ %iv.inner.next, %inner.body.split ]
-; CHECK-NEXT:    %v8.lcssa.lcssa.2 = phi i64 [ %iv.inner.next, %inner.body.split ]
+; CHECK-NEXT:    %v8.lcssa.lcssa = phi i64 [ %[[IVNEXT]], %inner.body.split ]
+; CHECK-NEXT:    %v8.lcssa.lcssa.2 = phi i64 [ %[[IVNEXT]], %inner.body.split ]
 ; CHECK-NEXT:    %r1 = add i64 %v8.lcssa.lcssa, %v8.lcssa.lcssa.2
 ; CHECK-NEXT:    store i64 %r1, i64* @b, align 4
 ; CHECK-NEXT:    store i64 %v4.lcssa, i64* @a, align 4
@@ -150,8 +165,12 @@ define void @no_reachable_exits() {
 ; CHECK-LABEL: inner.ph:
 ; CHECK-NEXT:    br label %inner.body
 ; CHECK-LABEL: inner.body:
-; CHECK-NEXT:    %tmp31 = phi i32 [ 0, %inner.ph ], [ %tmp6, %inner.body.split ]
+; CHECK-NEXT:    %tmp31 = phi i32 [ 0, %inner.ph ], [ %[[IVNEXT:[0-9]]], %inner.body.split ]
 ; CHECK-NEXT:    br label %outer.ph
+; CHECK-LABEL: inner.body.split:
+; CHECK-NEXT:    %[[IVNEXT]] = add nsw i32 %tmp31, 1
+; CHECK-NEXT:    br i1 false, label %inner.body, label %exit
+
 
 bb:
   br label %outer.ph
diff --git a/llvm/test/Transforms/LoopInterchange/phi-ordering.ll b/llvm/test/Transforms/LoopInterchange/phi-ordering.ll
index 2854fe19f..00dc5c3d7 100644
--- a/llvm/test/Transforms/LoopInterchange/phi-ordering.ll
+++ b/llvm/test/Transforms/LoopInterchange/phi-ordering.ll
@@ -27,7 +27,7 @@ define void @test(i32 %T, [90 x i32]* noalias nocapture %C, i16* noalias nocaptu
 ; CHECK:       for3.preheader:
 ; CHECK-NEXT:    br label [[FOR3:%.*]]
 ; CHECK:       for3:
-; CHECK-NEXT:    [[K:%.*]] = phi i32 [ [[INC:%.*]], [[FOR3_SPLIT:%.*]] ], [ 1, [[FOR3_PREHEADER]] ]
+; CHECK-NEXT:    [[K:%.*]] = phi i32 [ [[TMP1:%.*]], [[FOR3_SPLIT:%.*]] ], [ 1, [[FOR3_PREHEADER]] ]
 ; CHECK-NEXT:    br label [[FOR1_HEADER_PREHEADER]]
 ; CHECK:       for3.split1:
 ; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[K]], [[MUL]]
@@ -35,11 +35,13 @@ define void @test(i32 %T, [90 x i32]* noalias nocapture %C, i16* noalias nocaptu
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i16, i16* [[ARRAYIDX]], align 2
 ; CHECK-NEXT:    [[ADD15:%.*]] = add nsw i16 [[TMP0]], 1
 ; CHECK-NEXT:    store i16 [[ADD15]], i16* [[ARRAYIDX]]
+; CHECK-NEXT:    [[INC:%.*]] = add nuw nsw i32 [[K]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 90
 ; CHECK-NEXT:    br label [[FOR2_INC16]]
 ; CHECK:       for3.split:
-; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[K]], 1
-; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 90
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR1_LOOPEXIT:%.*]], label [[FOR3]]
+; CHECK-NEXT:    [[TMP1]] = add nuw nsw i32 [[K]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], 90
+; CHECK-NEXT:    br i1 [[TMP2]], label [[FOR1_LOOPEXIT:%.*]], label [[FOR3]]
 ; CHECK:       for2.inc16:
 ; CHECK-NEXT:    [[INC17]] = add nuw nsw i32 [[J]], 1
 ; CHECK-NEXT:    [[EXITCOND47:%.*]] = icmp eq i32 [[INC17]], 90
diff --git a/llvm/test/Transforms/LoopInterchange/pr43176-move-to-new-latch.ll b/llvm/test/Transforms/LoopInterchange/pr43176-move-to-new-latch.ll
new file mode 100644
index 000000000..ef0c5ad6c
--- /dev/null
+++ b/llvm/test/Transforms/LoopInterchange/pr43176-move-to-new-latch.ll
@@ -0,0 +1,140 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -loop-interchange -verify-loop-lcssa -verify-dom-info -S %s | FileCheck %s
+
+@b = external dso_local global [5 x i32], align 16
+
+define void @test1() {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_BODY2_PREHEADER:%.*]]
+; CHECK:       for.body.preheader:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INC41:%.*]] = phi i32 [ [[INC4:%.*]], [[FOR_INC3:%.*]] ], [ undef, [[FOR_BODY_PREHEADER:%.*]] ]
+; CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[INC41]] to i64
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [5 x i32], [5 x i32]* @b, i64 0, i64 [[IDXPROM]]
+; CHECK-NEXT:    br label [[FOR_BODY2_SPLIT:%.*]]
+; CHECK:       for.body2.preheader:
+; CHECK-NEXT:    br label [[FOR_BODY2:%.*]]
+; CHECK:       for.body2:
+; CHECK-NEXT:    [[LSR_IV:%.*]] = phi i32 [ [[TMP1:%.*]], [[FOR_INC_SPLIT:%.*]] ], [ 1, [[FOR_BODY2_PREHEADER]] ]
+; CHECK-NEXT:    br label [[FOR_BODY_PREHEADER]]
+; CHECK:       for.body2.split:
+; CHECK-NEXT:    br label [[FOR_INC:%.*]]
+; CHECK:       for.inc:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    store i32 undef, i32* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[LSR_IV]], 4
+; CHECK-NEXT:    [[LSR_IV_NEXT:%.*]] = add nuw nsw i32 [[LSR_IV]], 1
+; CHECK-NEXT:    br label [[FOR_COND1_FOR_END_CRIT_EDGE:%.*]]
+; CHECK:       for.inc.split:
+; CHECK-NEXT:    [[TMP1]] = add nuw nsw i32 [[LSR_IV]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i32 [[LSR_IV]], 4
+; CHECK-NEXT:    br i1 [[TMP2]], label [[FOR_BODY2]], label [[FOR_COND_FOR_END5_CRIT_EDGE:%.*]]
+; CHECK:       for.cond1.for.end_crit_edge:
+; CHECK-NEXT:    br label [[FOR_INC3]]
+; CHECK:       for.inc3:
+; CHECK-NEXT:    [[INC4]] = add nsw i32 [[INC41]], 1
+; CHECK-NEXT:    br i1 false, label [[FOR_BODY]], label [[FOR_INC_SPLIT]]
+; CHECK:       for.cond.for.end5_crit_edge:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.inc3, %entry
+  %inc41 = phi i32 [ %inc4, %for.inc3 ], [ undef, %entry ]
+  br label %for.body2
+
+for.body2:                                        ; preds = %for.inc, %for.body
+  %lsr.iv = phi i32 [ %lsr.iv.next, %for.inc ], [ 1, %for.body ]
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body2
+  %idxprom = sext i32 %inc41 to i64
+  %arrayidx = getelementptr inbounds [5 x i32], [5 x i32]* @b, i64 0, i64 %idxprom
+  %0 = load i32, i32* %arrayidx, align 4
+  store i32 undef, i32* %arrayidx, align 4
+  %cmp = icmp slt i32 %lsr.iv, 4
+  %lsr.iv.next = add nuw nsw i32 %lsr.iv, 1
+  br i1 %cmp, label %for.body2, label %for.cond1.for.end_crit_edge
+
+for.cond1.for.end_crit_edge:                      ; preds = %for.inc
+  br label %for.inc3
+
+for.inc3:                                         ; preds = %for.cond1.for.end_crit_edge
+  %inc4 = add nsw i32 %inc41, 1
+  br i1 undef, label %for.body, label %for.cond.for.end5_crit_edge
+
+for.cond.for.end5_crit_edge:                      ; preds = %for.inc3
+  ret void
+}
+
+define void @test2() {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_BODY2_PREHEADER:%.*]]
+; CHECK:       for.body.preheader:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INC41:%.*]] = phi i32 [ [[INC4:%.*]], [[FOR_INC3:%.*]] ], [ undef, [[FOR_BODY_PREHEADER:%.*]] ]
+; CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[INC41]] to i64
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [5 x i32], [5 x i32]* @b, i64 0, i64 [[IDXPROM]]
+; CHECK-NEXT:    br label [[FOR_BODY2_SPLIT:%.*]]
+; CHECK:       for.body2.preheader:
+; CHECK-NEXT:    br label [[FOR_BODY2:%.*]]
+; CHECK:       for.body2:
+; CHECK-NEXT:    [[LSR_IV:%.*]] = phi i32 [ [[TMP1:%.*]], [[FOR_INC_SPLIT:%.*]] ], [ 1, [[FOR_BODY2_PREHEADER]] ]
+; CHECK-NEXT:    br label [[FOR_BODY_PREHEADER]]
+; CHECK:       for.body2.split:
+; CHECK-NEXT:    br label [[FOR_INC:%.*]]
+; CHECK:       for.inc:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[LSR_IV]], 4
+; CHECK-NEXT:    [[CMP_ZEXT:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    store i32 [[CMP_ZEXT]], i32* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[LSR_IV_NEXT:%.*]] = add nuw nsw i32 [[LSR_IV]], 1
+; CHECK-NEXT:    br label [[FOR_COND1_FOR_END_CRIT_EDGE:%.*]]
+; CHECK:       for.inc.split:
+; CHECK-NEXT:    [[TMP1]] = add nuw nsw i32 [[LSR_IV]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i32 [[LSR_IV]], 4
+; CHECK-NEXT:    br i1 [[TMP2]], label [[FOR_BODY2]], label [[FOR_COND_FOR_END5_CRIT_EDGE:%.*]]
+; CHECK:       for.cond1.for.end_crit_edge:
+; CHECK-NEXT:    br label [[FOR_INC3]]
+; CHECK:       for.inc3:
+; CHECK-NEXT:    [[INC4]] = add nsw i32 [[INC41]], 1
+; CHECK-NEXT:    br i1 false, label [[FOR_BODY]], label [[FOR_INC_SPLIT]]
+; CHECK:       for.cond.for.end5_crit_edge:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.inc3, %entry
+  %inc41 = phi i32 [ %inc4, %for.inc3 ], [ undef, %entry ]
+  br label %for.body2
+
+for.body2:                                        ; preds = %for.inc, %for.body
+  %lsr.iv = phi i32 [ %lsr.iv.next, %for.inc ], [ 1, %for.body ]
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body2
+  %idxprom = sext i32 %inc41 to i64
+  %arrayidx = getelementptr inbounds [5 x i32], [5 x i32]* @b, i64 0, i64 %idxprom
+  %0 = load i32, i32* %arrayidx, align 4
+  %cmp = icmp slt i32 %lsr.iv, 4
+  %cmp.zext = zext i1 %cmp to i32
+  store i32 %cmp.zext, i32* %arrayidx, align 4
+  %lsr.iv.next = add nuw nsw i32 %lsr.iv, 1
+  br i1 %cmp, label %for.body2, label %for.cond1.for.end_crit_edge
+
+for.cond1.for.end_crit_edge:                      ; preds = %for.inc
+  br label %for.inc3
+
+for.inc3:                                         ; preds = %for.cond1.for.end_crit_edge
+  %inc4 = add nsw i32 %inc41, 1
+  br i1 undef, label %for.body, label %for.cond.for.end5_crit_edge
+
+for.cond.for.end5_crit_edge:                      ; preds = %for.inc3
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopInterchange/pr43473-invalid-lcssa-phis-in-inner-exit.ll b/llvm/test/Transforms/LoopInterchange/pr43473-invalid-lcssa-phis-in-inner-exit.ll
new file mode 100644
index 000000000..9ae5f42e1
--- /dev/null
+++ b/llvm/test/Transforms/LoopInterchange/pr43473-invalid-lcssa-phis-in-inner-exit.ll
@@ -0,0 +1,108 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -loop-interchange -S < %s | FileCheck %s
+
+; Test cases for PR43473.
+
+; In the 2 test cases below, we have a LCSSA PHI in the inner loop exit, which
+; is used in the outer loop latch. This is not supported.
+
+define void @test1() {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[OUTER_HEADER:%.*]]
+; CHECK:       outer.header:
+; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ undef, [[ENTRY:%.*]] ], [ [[OUTER_IV_NEXT:%.*]], [[OUTER_LATCH:%.*]] ]
+; CHECK-NEXT:    [[IDX:%.*]] = getelementptr inbounds double, double* undef, i64 [[OUTER_IV]]
+; CHECK-NEXT:    br label [[INNER:%.*]]
+; CHECK:       inner:
+; CHECK-NEXT:    [[INNER_IV:%.*]] = phi i64 [ 0, [[OUTER_HEADER]] ], [ [[INNER_IV_NEXT:%.*]], [[INNER]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = load double, double* [[IDX]], align 8
+; CHECK-NEXT:    store double undef, double* [[IDX]], align 8
+; CHECK-NEXT:    [[INNER_IV_NEXT]] = add nuw nsw i64 [[INNER_IV]], 1
+; CHECK-NEXT:    br i1 false, label [[INNER]], label [[OUTER_LATCH]]
+; CHECK:       outer.latch:
+; CHECK-NEXT:    [[INC43_LCSSA_WIDE_US:%.*]] = phi i64 [ [[INNER_IV_NEXT]], [[INNER]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[INC43_LCSSA_WIDE_US]] to i32
+; CHECK-NEXT:    [[OUTER_IV_NEXT]] = add nsw i64 [[OUTER_IV]], 1
+; CHECK-NEXT:    br i1 false, label [[OUTER_HEADER]], label [[OUTER_EXIT:%.*]]
+; CHECK:       outer.exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %outer.header
+
+outer.header:                                    ; preds = %for.cond26.for.end44_crit_edge.us, %entry
+  %outer.iv = phi i64 [ undef, %entry ], [ %outer.iv.next, %outer.latch ]
+  %idx = getelementptr inbounds double, double* undef, i64 %outer.iv
+  br label %inner
+
+inner:                                    ; preds = %for.body28.us, %for.body25.us
+  %inner.iv = phi i64 [ 0, %outer.header ], [ %inner.iv.next, %inner ]
+  %0 = load double, double* %idx, align 8
+  store double undef, double* %idx, align 8
+  %inner.iv.next = add nuw nsw i64 %inner.iv, 1
+  br i1 undef, label %inner, label %outer.latch
+
+outer.latch:                ; preds = %inner
+  %inc43.lcssa.wide.us = phi i64 [ %inner.iv.next, %inner ]
+  %1 = trunc i64 %inc43.lcssa.wide.us to i32
+  %outer.iv.next = add nsw i64 %outer.iv, 1
+  br i1 undef, label %outer.header, label %outer.exit
+
+outer.exit:       ; preds = %for.cond26.for.end44_crit_edge.us
+  ret void
+}
+
+; Same as @test1, but with a dedicated inner loop exit block.
+define void @test2() {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[OUTER_HEADER:%.*]]
+; CHECK:       outer.header:
+; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ undef, [[ENTRY:%.*]] ], [ [[OUTER_IV_NEXT:%.*]], [[OUTER_LATCH:%.*]] ]
+; CHECK-NEXT:    [[IDX:%.*]] = getelementptr inbounds double, double* undef, i64 [[OUTER_IV]]
+; CHECK-NEXT:    br label [[INNER:%.*]]
+; CHECK:       inner:
+; CHECK-NEXT:    [[INNER_IV:%.*]] = phi i64 [ 0, [[OUTER_HEADER]] ], [ [[INNER_IV_NEXT:%.*]], [[INNER]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = load double, double* [[IDX]], align 8
+; CHECK-NEXT:    store double undef, double* [[IDX]], align 8
+; CHECK-NEXT:    [[INNER_IV_NEXT]] = add nuw nsw i64 [[INNER_IV]], 1
+; CHECK-NEXT:    br i1 false, label [[INNER]], label [[INNER_EXIT:%.*]]
+; CHECK:       inner.exit:
+; CHECK-NEXT:    [[INC43_LCSSA_WIDE_US:%.*]] = phi i64 [ [[INNER_IV_NEXT]], [[INNER]] ]
+; CHECK-NEXT:    br label [[OUTER_LATCH]]
+; CHECK:       outer.latch:
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[INC43_LCSSA_WIDE_US]] to i32
+; CHECK-NEXT:    [[OUTER_IV_NEXT]] = add nsw i64 [[OUTER_IV]], 1
+; CHECK-NEXT:    br i1 false, label [[OUTER_HEADER]], label [[OUTER_EXIT:%.*]]
+; CHECK:       outer.exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %outer.header
+
+outer.header:                                    ; preds = %for.cond26.for.end44_crit_edge.us, %entry
+  %outer.iv = phi i64 [ undef, %entry ], [ %outer.iv.next, %outer.latch ]
+  %idx = getelementptr inbounds double, double* undef, i64 %outer.iv
+  br label %inner
+
+inner:                                    ; preds = %for.body28.us, %for.body25.us
+  %inner.iv = phi i64 [ 0, %outer.header ], [ %inner.iv.next, %inner ]
+  %0 = load double, double* %idx, align 8
+  store double undef, double* %idx, align 8
+  %inner.iv.next = add nuw nsw i64 %inner.iv, 1
+  br i1 undef, label %inner, label %inner.exit
+
+inner.exit:
+  %inc43.lcssa.wide.us = phi i64 [ %inner.iv.next, %inner ]
+  br label %outer.latch
+
+outer.latch:                ; preds = %inner
+  %1 = trunc i64 %inc43.lcssa.wide.us to i32
+  %outer.iv.next = add nsw i64 %outer.iv, 1
+  br i1 undef, label %outer.header, label %outer.exit
+
+outer.exit:       ; preds = %for.cond26.for.end44_crit_edge.us
+  ret void
+}
+
diff --git a/llvm/test/Transforms/LoopInterchange/pr43797-lcssa-for-multiple-outer-loop-blocks.ll b/llvm/test/Transforms/LoopInterchange/pr43797-lcssa-for-multiple-outer-loop-blocks.ll
new file mode 100644
index 000000000..445fffecc
--- /dev/null
+++ b/llvm/test/Transforms/LoopInterchange/pr43797-lcssa-for-multiple-outer-loop-blocks.ll
@@ -0,0 +1,150 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -loop-interchange -verify-loop-lcssa -S %s | FileCheck %s
+
+; Tests for PR43797.
+
+@wdtdr = external dso_local global [5 x [5 x double]], align 16
+
+define void @test1() {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[INNER_HEADER_PREHEADER:%.*]]
+; CHECK:       outer.header.preheader:
+; CHECK-NEXT:    br label [[OUTER_HEADER:%.*]]
+; CHECK:       outer.header:
+; CHECK-NEXT:    [[OUTER_IDX:%.*]] = phi i64 [ [[OUTER_IDX_INC:%.*]], [[OUTER_LATCH:%.*]] ], [ 0, [[OUTER_HEADER_PREHEADER:%.*]] ]
+; CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds [5 x [5 x double]], [5 x [5 x double]]* @wdtdr, i64 0, i64 0, i64 [[OUTER_IDX]]
+; CHECK-NEXT:    br label [[INNER_HEADER_SPLIT:%.*]]
+; CHECK:       inner.header.preheader:
+; CHECK-NEXT:    br label [[INNER_HEADER:%.*]]
+; CHECK:       inner.header:
+; CHECK-NEXT:    [[INNER_IDX:%.*]] = phi i64 [ [[TMP3:%.*]], [[INNER_LATCH_SPLIT:%.*]] ], [ 0, [[INNER_HEADER_PREHEADER]] ]
+; CHECK-NEXT:    br label [[OUTER_HEADER_PREHEADER]]
+; CHECK:       inner.header.split:
+; CHECK-NEXT:    [[TMP0:%.*]] = load double, double* [[ARRAYIDX8]], align 8
+; CHECK-NEXT:    store double undef, double* [[ARRAYIDX8]], align 8
+; CHECK-NEXT:    br label [[INNER_LATCH:%.*]]
+; CHECK:       inner.latch:
+; CHECK-NEXT:    [[INNER_IDX_INC:%.*]] = add nsw i64 [[INNER_IDX]], 1
+; CHECK-NEXT:    br label [[INNER_EXIT:%.*]]
+; CHECK:       inner.latch.split:
+; CHECK-NEXT:    [[TMP1:%.*]] = phi i64 [ [[OUTER_V:%.*]], [[OUTER_LATCH]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = phi i64 [ [[OUTER_IDX_INC]], [[OUTER_LATCH]] ]
+; CHECK-NEXT:    [[TMP3]] = add nsw i64 [[INNER_IDX]], 1
+; CHECK-NEXT:    br i1 false, label [[INNER_HEADER]], label [[OUTER_EXIT:%.*]]
+; CHECK:       inner.exit:
+; CHECK-NEXT:    [[OUTER_V]] = add nsw i64 [[OUTER_IDX]], 1
+; CHECK-NEXT:    br label [[OUTER_LATCH]]
+; CHECK:       outer.latch:
+; CHECK-NEXT:    [[OUTER_IDX_INC]] = add nsw i64 [[OUTER_IDX]], 1
+; CHECK-NEXT:    br i1 false, label [[OUTER_HEADER]], label [[INNER_LATCH_SPLIT]]
+; CHECK:       outer.exit:
+; CHECK-NEXT:    [[EXIT1_LCSSA:%.*]] = phi i64 [ [[TMP1]], [[INNER_LATCH_SPLIT]] ]
+; CHECK-NEXT:    [[EXIT2_LCSSA:%.*]] = phi i64 [ [[TMP2]], [[INNER_LATCH_SPLIT]] ]
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %outer.header
+
+outer.header:                                         ; preds = %for.inc27, %entry
+  %outer.idx = phi i64 [ 0, %entry ], [ %outer.idx.inc, %outer.latch ]
+  %arrayidx8 = getelementptr inbounds [5 x [5 x double]], [5 x [5 x double]]* @wdtdr, i64 0, i64 0, i64 %outer.idx
+  br label %inner.header
+
+inner.header:                                        ; preds = %for.inc, %for.body
+  %inner.idx = phi i64 [ 0, %outer.header ], [ %inner.idx.inc, %inner.latch]
+  %0 = load double, double* %arrayidx8, align 8
+  store double undef, double* %arrayidx8, align 8
+  br label %inner.latch
+
+inner.latch:                                          ; preds = %for.body6
+  %inner.idx.inc = add nsw i64 %inner.idx, 1
+  br i1 undef, label %inner.header, label %inner.exit
+
+inner.exit:                                          ; preds = %for.inc
+  %outer.v = add nsw i64 %outer.idx, 1
+  br label %outer.latch
+
+outer.latch:                                        ; preds = %for.end
+  %outer.idx.inc = add nsw i64 %outer.idx, 1
+  br i1 undef, label %outer.header, label %outer.exit
+
+outer.exit:                                        ; preds = %for.inc27
+  %exit1.lcssa = phi i64 [ %outer.v, %outer.latch ]
+  %exit2.lcssa = phi i64 [ %outer.idx.inc, %outer.latch ]
+  ret void
+}
+
+define void @test2(i1 %cond) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[INNER_HEADER_PREHEADER:%.*]], label [[OUTER_EXIT:%.*]]
+; CHECK:       outer.header.preheader:
+; CHECK-NEXT:    br label [[OUTER_HEADER:%.*]]
+; CHECK:       outer.header:
+; CHECK-NEXT:    [[OUTER_IDX:%.*]] = phi i64 [ [[OUTER_IDX_INC:%.*]], [[OUTER_LATCH:%.*]] ], [ 0, [[OUTER_HEADER_PREHEADER:%.*]] ]
+; CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds [5 x [5 x double]], [5 x [5 x double]]* @wdtdr, i64 0, i64 0, i64 [[OUTER_IDX]]
+; CHECK-NEXT:    br label [[INNER_HEADER_SPLIT:%.*]]
+; CHECK:       inner.header.preheader:
+; CHECK-NEXT:    br label [[INNER_HEADER:%.*]]
+; CHECK:       inner.header:
+; CHECK-NEXT:    [[INNER_IDX:%.*]] = phi i64 [ [[TMP3:%.*]], [[INNER_LATCH_SPLIT:%.*]] ], [ 0, [[INNER_HEADER_PREHEADER]] ]
+; CHECK-NEXT:    br label [[OUTER_HEADER_PREHEADER]]
+; CHECK:       inner.header.split:
+; CHECK-NEXT:    [[TMP0:%.*]] = load double, double* [[ARRAYIDX8]], align 8
+; CHECK-NEXT:    store double undef, double* [[ARRAYIDX8]], align 8
+; CHECK-NEXT:    br label [[INNER_LATCH:%.*]]
+; CHECK:       inner.latch:
+; CHECK-NEXT:    [[INNER_IDX_INC:%.*]] = add nsw i64 [[INNER_IDX]], 1
+; CHECK-NEXT:    br label [[INNER_EXIT:%.*]]
+; CHECK:       inner.latch.split:
+; CHECK-NEXT:    [[TMP1:%.*]] = phi i64 [ [[OUTER_IDX_INC]], [[OUTER_LATCH]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = phi i64 [ [[OUTER_V:%.*]], [[OUTER_LATCH]] ]
+; CHECK-NEXT:    [[TMP3]] = add nsw i64 [[INNER_IDX]], 1
+; CHECK-NEXT:    br i1 false, label [[INNER_HEADER]], label [[OUTER_EXIT_LOOPEXIT:%.*]]
+; CHECK:       inner.exit:
+; CHECK-NEXT:    [[OUTER_V]] = add nsw i64 [[OUTER_IDX]], 1
+; CHECK-NEXT:    br label [[OUTER_LATCH]]
+; CHECK:       outer.latch:
+; CHECK-NEXT:    [[OUTER_IDX_INC]] = add nsw i64 [[OUTER_IDX]], 1
+; CHECK-NEXT:    br i1 false, label [[OUTER_HEADER]], label [[INNER_LATCH_SPLIT]]
+; CHECK:       outer.exit.loopexit:
+; CHECK-NEXT:    [[OUTER_IDX_INC_LCSSA:%.*]] = phi i64 [ [[TMP1]], [[INNER_LATCH_SPLIT]] ]
+; CHECK-NEXT:    [[OUTER_V_LCSSA:%.*]] = phi i64 [ [[TMP2]], [[INNER_LATCH_SPLIT]] ]
+; CHECK-NEXT:    br label [[OUTER_EXIT]]
+; CHECK:       outer.exit:
+; CHECK-NEXT:    [[EXIT1_LCSSA:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[OUTER_V_LCSSA]], [[OUTER_EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    [[EXIT2_LCSSA:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[OUTER_IDX_INC_LCSSA]], [[OUTER_EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret void
+;
+entry:
+  br i1 %cond, label %outer.header, label %outer.exit
+
+outer.header:                                         ; preds = %for.inc27, %entry
+  %outer.idx = phi i64 [ 0, %entry ], [ %outer.idx.inc, %outer.latch ]
+  %arrayidx8 = getelementptr inbounds [5 x [5 x double]], [5 x [5 x double]]* @wdtdr, i64 0, i64 0, i64 %outer.idx
+  br label %inner.header
+
+inner.header:                                        ; preds = %for.inc, %for.body
+  %inner.idx = phi i64 [ 0, %outer.header ], [ %inner.idx.inc, %inner.latch]
+  %0 = load double, double* %arrayidx8, align 8
+  store double undef, double* %arrayidx8, align 8
+  br label %inner.latch
+
+inner.latch:                                          ; preds = %for.body6
+  %inner.idx.inc = add nsw i64 %inner.idx , 1
+  br i1 undef, label %inner.header, label %inner.exit
+
+inner.exit:                                          ; preds = %for.inc
+  %outer.v = add nsw i64 %outer.idx, 1
+  br label %outer.latch
+
+outer.latch:                                        ; preds = %for.end
+  %outer.idx.inc = add nsw i64 %outer.idx, 1
+  br i1 undef, label %outer.header, label %outer.exit
+
+outer.exit:                                        ; preds = %for.inc27
+  %exit1.lcssa = phi i64 [ 0, %entry ], [ %outer.v, %outer.latch ]
+  %exit2.lcssa = phi i64 [ 0, %entry ], [ %outer.idx.inc, %outer.latch ]
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopInterchange/reductions-across-inner-and-outer-loop.ll b/llvm/test/Transforms/LoopInterchange/reductions-across-inner-and-outer-loop.ll
index 7ac96f148..deff8ccc4 100644
--- a/llvm/test/Transforms/LoopInterchange/reductions-across-inner-and-outer-loop.ll
+++ b/llvm/test/Transforms/LoopInterchange/reductions-across-inner-and-outer-loop.ll
@@ -31,6 +31,8 @@ define i64 @test1([100 x [100 x i64]]* %Arr) {
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [100 x [100 x i64]], [100 x [100 x i64]]* [[ARR:%.*]], i64 0, i64 [[INDVARS_IV]], i64 [[INDVARS_IV23]]
 ; CHECK-NEXT:    [[LV:%.*]] = load i64, i64* [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[SUM_INC]] = add i64 [[SUM_INNER]], [[LV]]
+; CHECK-NEXT:    [[IV_ORIGINAL:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[EXIT1_ORIGINAL:%.*]] = icmp eq i64 [[IV_ORIGINAL]], 100
 ; CHECK-NEXT:    br label [[FOR1_INC]]
 ; CHECK:       for2.split:
 ; CHECK-NEXT:    [[SUM_INC_LCSSA]] = phi i64 [ [[SUM_INC]], %for1.inc ]
diff --git a/llvm/test/Transforms/LoopInterchange/update-condbranch-duplicate-successors.ll b/llvm/test/Transforms/LoopInterchange/update-condbranch-duplicate-successors.ll
new file mode 100644
index 000000000..3f178443e
--- /dev/null
+++ b/llvm/test/Transforms/LoopInterchange/update-condbranch-duplicate-successors.ll
@@ -0,0 +1,145 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -loop-interchange -S %s | FileCheck %s
+
+
+@global = external dso_local global [1000 x [1000 x i32]], align 16
+
+; Test that we support updating conditional branches where both targets are the same
+; in the predecessor of the outer loop header.
+
+define void @foo(i1 %cmp) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[CMP:%.*]], label [[INNER_HEADER_PREHEADER:%.*]], label [[INNER_HEADER_PREHEADER]]
+; CHECK:       bb1:
+; CHECK-NEXT:    br label [[OUTER_HEADER:%.*]]
+; CHECK:       outer.header:
+; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ 0, [[BB1:%.*]] ], [ [[OUTER_IV_NEXT:%.*]], [[OUTER_LATCH:%.*]] ]
+; CHECK-NEXT:    br label [[INNER_HEADER_SPLIT1:%.*]]
+; CHECK:       inner.header.preheader:
+; CHECK-NEXT:    br label [[INNER_HEADER:%.*]]
+; CHECK:       inner.header:
+; CHECK-NEXT:    [[INNER_IV:%.*]] = phi i64 [ [[TMP0:%.*]], [[INNER_HEADER_SPLIT:%.*]] ], [ 5, [[INNER_HEADER_PREHEADER]] ]
+; CHECK-NEXT:    br label [[BB1]]
+; CHECK:       inner.header.split1:
+; CHECK-NEXT:    [[PTR:%.*]] = getelementptr inbounds [1000 x [1000 x i32]], [1000 x [1000 x i32]]* @global, i64 0, i64 [[INNER_IV]], i64 [[OUTER_IV]]
+; CHECK-NEXT:    [[LV:%.*]] = load i32, i32* [[PTR]]
+; CHECK-NEXT:    [[V:%.*]] = mul i32 [[LV]], 100
+; CHECK-NEXT:    store i32 [[V]], i32* [[PTR]]
+; CHECK-NEXT:    [[INNER_IV_NEXT:%.*]] = add nsw i64 [[INNER_IV]], 1
+; CHECK-NEXT:    [[COND1:%.*]] = icmp eq i64 [[INNER_IV_NEXT]], 1000
+; CHECK-NEXT:    br label [[OUTER_LATCH]]
+; CHECK:       inner.header.split:
+; CHECK-NEXT:    [[TMP0]] = add nsw i64 [[INNER_IV]], 1
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i64 [[TMP0]], 1000
+; CHECK-NEXT:    br i1 [[TMP1]], label [[BB9:%.*]], label [[INNER_HEADER]]
+; CHECK:       outer.latch:
+; CHECK-NEXT:    [[OUTER_IV_NEXT]] = add nuw nsw i64 [[OUTER_IV]], 1
+; CHECK-NEXT:    [[COND2:%.*]] = icmp eq i64 [[OUTER_IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[COND2]], label [[INNER_HEADER_SPLIT]], label [[OUTER_HEADER]]
+; CHECK:       bb9:
+; CHECK-NEXT:    br label [[BB10:%.*]]
+; CHECK:       bb10:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br i1 %cmp, label %bb1, label %bb1
+
+bb1:                                              ; preds = %entry, %entry
+  br label %outer.header
+
+outer.header:                                              ; preds = %outer.latch, %bb1
+  %outer.iv = phi i64 [ 0, %bb1], [ %outer.iv.next, %outer.latch ]
+  br label %inner.header
+
+inner.header:                                              ; preds = %inner.header, %outer.header
+  %inner.iv = phi i64 [ %inner.iv.next, %inner.header ], [ 5, %outer.header ]
+  %ptr = getelementptr inbounds [1000 x [1000 x i32]], [1000 x [1000 x i32]]* @global, i64 0, i64 %inner.iv, i64 %outer.iv
+  %lv = load i32, i32* %ptr
+  %v = mul i32 %lv, 100
+  store i32 %v, i32* %ptr
+  %inner.iv.next = add nsw i64 %inner.iv, 1
+  %cond1 = icmp eq i64 %inner.iv.next , 1000
+  br i1 %cond1, label %outer.latch, label %inner.header
+
+outer.latch:                                              ; preds = %inner.header
+  %outer.iv.next = add nuw nsw i64 %outer.iv, 1
+  %cond2 = icmp eq i64 %outer.iv.next, 1000
+  br i1 %cond2, label %bb9, label %outer.header
+
+bb9:                                              ; preds = %outer.latch
+  br label %bb10
+
+bb10:                                             ; preds = %bb9
+  ret void
+}
+
+
+define void @foo1(i1 %cmp) {
+; CHECK-LABEL: @foo1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[CMP:%.*]], label [[BB1:%.*]], label [[BB1]]
+; CHECK:       bb1:
+; CHECK-NEXT:    br i1 [[CMP]], label [[INNER_HEADER_PREHEADER:%.*]], label [[INNER_HEADER_PREHEADER]]
+; CHECK:       outer.header.preheader:
+; CHECK-NEXT:    br label [[OUTER_HEADER:%.*]]
+; CHECK:       outer.header:
+; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ [[OUTER_IV_NEXT:%.*]], [[OUTER_LATCH:%.*]] ], [ 0, [[OUTER_HEADER_PREHEADER:%.*]] ]
+; CHECK-NEXT:    br i1 [[CMP]], label [[INNER_HEADER_SPLIT1:%.*]], label [[INNER_HEADER_SPLIT1]]
+; CHECK:       inner.header.preheader:
+; CHECK-NEXT:    br label [[INNER_HEADER:%.*]]
+; CHECK:       inner.header:
+; CHECK-NEXT:    [[INNER_IV:%.*]] = phi i64 [ [[TMP0:%.*]], [[INNER_HEADER_SPLIT:%.*]] ], [ 5, [[INNER_HEADER_PREHEADER]] ]
+; CHECK-NEXT:    br label [[OUTER_HEADER_PREHEADER]]
+; CHECK:       inner.header.split1:
+; CHECK-NEXT:    [[PTR:%.*]] = getelementptr inbounds [1000 x [1000 x i32]], [1000 x [1000 x i32]]* @global, i64 0, i64 [[INNER_IV]], i64 [[OUTER_IV]]
+; CHECK-NEXT:    [[LV:%.*]] = load i32, i32* [[PTR]]
+; CHECK-NEXT:    [[V:%.*]] = mul i32 [[LV]], 100
+; CHECK-NEXT:    store i32 [[V]], i32* [[PTR]]
+; CHECK-NEXT:    [[INNER_IV_NEXT:%.*]] = add nsw i64 [[INNER_IV]], 1
+; CHECK-NEXT:    [[COND1:%.*]] = icmp eq i64 [[INNER_IV_NEXT]], 1000
+; CHECK-NEXT:    br label [[OUTER_LATCH]]
+; CHECK:       inner.header.split:
+; CHECK-NEXT:    [[TMP0]] = add nsw i64 [[INNER_IV]], 1
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i64 [[TMP0]], 1000
+; CHECK-NEXT:    br i1 [[TMP1]], label [[BB9:%.*]], label [[INNER_HEADER]]
+; CHECK:       outer.latch:
+; CHECK-NEXT:    [[OUTER_IV_NEXT]] = add nuw nsw i64 [[OUTER_IV]], 1
+; CHECK-NEXT:    [[COND2:%.*]] = icmp eq i64 [[OUTER_IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[COND2]], label [[INNER_HEADER_SPLIT]], label [[OUTER_HEADER]]
+; CHECK:       bb9:
+; CHECK-NEXT:    br label [[BB10:%.*]]
+; CHECK:       bb10:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br i1 %cmp, label %bb1, label %bb1
+
+bb1:                                              ; preds = %entry, %entry
+  br i1 %cmp, label %outer.header, label %outer.header
+
+outer.header:                                              ; preds = %outer.latch, %bb1
+  %outer.iv = phi i64 [ 0, %bb1 ], [ 0, %bb1 ], [ %outer.iv.next, %outer.latch ]
+  br i1 %cmp, label %inner.header, label %inner.header
+
+inner.header:                                              ; preds = %inner.header, %outer.header
+  %inner.iv = phi i64 [ %inner.iv.next, %inner.header ], [ 5, %outer.header ], [ 5, %outer.header ]
+  %ptr = getelementptr inbounds [1000 x [1000 x i32]], [1000 x [1000 x i32]]* @global, i64 0, i64 %inner.iv, i64 %outer.iv
+  %lv = load i32, i32* %ptr
+  %v = mul i32 %lv, 100
+  store i32 %v, i32* %ptr
+  %inner.iv.next = add nsw i64 %inner.iv, 1
+  %cond1 = icmp eq i64 %inner.iv.next , 1000
+  br i1 %cond1, label %outer.latch, label %inner.header
+
+outer.latch:                                              ; preds = %inner.header
+  %outer.iv.next = add nuw nsw i64 %outer.iv, 1
+  %cond2 = icmp eq i64 %outer.iv.next, 1000
+  br i1 %cond2, label %bb9, label %outer.header
+
+bb9:                                              ; preds = %outer.latch
+  br label %bb10
+
+bb10:                                             ; preds = %bb9
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopPredication/basic_widenable_branch_guards.ll b/llvm/test/Transforms/LoopPredication/basic_widenable_branch_guards.ll
index 85d4be0d7..70cf1e9dd 100644
--- a/llvm/test/Transforms/LoopPredication/basic_widenable_branch_guards.ll
+++ b/llvm/test/Transforms/LoopPredication/basic_widenable_branch_guards.ll
@@ -1924,6 +1924,72 @@ exit:                                             ; preds = %guarded, %entry
   ret i32 0
 }
 
+define i32 @swapped_wb(i32* %array, i32 %length, i32 %n) {
+; CHECK-LABEL: @swapped_wb(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ule i32 [[N]], [[LENGTH:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 0, [[LENGTH]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i1 [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    [[TMP3:%.*]] = and i1 [[TMP2]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    br i1 [[TMP3]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    [[DEOPTCALL:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTCALL]]
+; CHECK:       guarded:
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ult i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]], !prof !1
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %tmp5 = icmp eq i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:                                   ; preds = %entry
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  %within.bounds = icmp ult i32 %i, %length
+  %exiplicit_guard_cond = and i1 %widenable_cond, %within.bounds
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  %deoptcall = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+  ret i32 %deoptcall
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp ult i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit, !prof !2
+
+exit:                                             ; preds = %guarded, %entry
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
 
 declare i32 @llvm.experimental.deoptimize.i32(...)
 
diff --git a/llvm/test/Transforms/LoopPredication/predicate-exits.ll b/llvm/test/Transforms/LoopPredication/predicate-exits.ll
new file mode 100644
index 000000000..83d85eb74
--- /dev/null
+++ b/llvm/test/Transforms/LoopPredication/predicate-exits.ll
@@ -0,0 +1,1110 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -loop-predication -S | FileCheck %s
+
+declare void @prevent_merging()
+
+; Base case - with side effects in loop
+define i32 @test1(i32* %array, i32 %length, i32 %n, i1 %cond_0) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ugt i32 [[N:%.*]], 1
+; CHECK-NEXT:    [[UMAX:%.*]] = select i1 [[TMP0]], i32 [[N]], i32 1
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[UMAX]], -1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i32 [[LENGTH:%.*]], [[TMP1]]
+; CHECK-NEXT:    [[UMIN:%.*]] = select i1 [[TMP2]], i32 [[LENGTH]], i32 [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ugt i32 [[LENGTH]], [[UMIN]]
+; CHECK-NEXT:    [[TMP4:%.*]] = freeze i1 [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = and i1 [[TMP4]], [[COND_0:%.*]]
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[TMP5]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[LOOP_PREHEADER:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    [[DEOPTRET:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTRET]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[I]], [[LENGTH]]
+; CHECK-NEXT:    br i1 true, label [[GUARDED]], label [[DEOPT2:%.*]], !prof !0
+; CHECK:       deopt2:
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    [[DEOPTRET2:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTRET2]]
+; CHECK:       guarded:
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    store i32 0, i32* [[ARRAY_I_PTR]]
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ult i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %cond_0, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %loop.preheader, label %deopt, !prof !0
+
+deopt:
+  %deoptret = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+  ret i32 %deoptret
+
+loop.preheader:
+  br label %loop
+
+loop:
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  call void @unknown()
+  %within.bounds = icmp ult i32 %i, %length
+  br i1 %within.bounds, label %guarded, label %deopt2, !prof !0
+
+deopt2:
+  call void @unknown()
+  %deoptret2 = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+  ret i32 %deoptret2
+
+guarded:
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  store i32 0, i32* %array.i.ptr
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp ult i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:
+  %result = phi i32 [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+
+
+define i32 @test_non_canonical(i32* %array, i32 %length, i1 %cond_0) {
+; CHECK-LABEL: @test_non_canonical(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ugt i32 [[LENGTH:%.*]], 1
+; CHECK-NEXT:    [[UMAX:%.*]] = select i1 [[TMP0]], i32 [[LENGTH]], i32 1
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[UMAX]], -1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i32 [[LENGTH]], [[TMP1]]
+; CHECK-NEXT:    [[UMIN:%.*]] = select i1 [[TMP2]], i32 [[LENGTH]], i32 [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ugt i32 [[LENGTH]], [[UMIN]]
+; CHECK-NEXT:    [[TMP4:%.*]] = freeze i1 [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = and i1 [[TMP4]], [[COND_0:%.*]]
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[TMP5]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[LOOP_PREHEADER:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    [[DEOPTRET:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTRET]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[I]], [[LENGTH]]
+; CHECK-NEXT:    br i1 true, label [[GUARDED]], label [[DEOPT2:%.*]], !prof !0
+; CHECK:       deopt2:
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    [[DEOPTRET2:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTRET2]]
+; CHECK:       guarded:
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    store i32 0, i32* [[ARRAY_I_PTR]]
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ult i32 [[I_NEXT]], [[LENGTH]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %cond_0, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %loop.preheader, label %deopt, !prof !0
+
+deopt:
+  %deoptret = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+  ret i32 %deoptret
+
+loop.preheader:
+  br label %loop
+
+loop:
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  call void @unknown()
+  %within.bounds = icmp ult i32 %i, %length
+  br i1 %within.bounds, label %guarded, label %deopt2, !prof !0
+
+deopt2:
+  call void @unknown()
+  %deoptret2 = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+  ret i32 %deoptret2
+
+guarded:
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  store i32 0, i32* %array.i.ptr
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp ult i32 %i.next, %length
+  br i1 %continue, label %loop, label %exit
+
+exit:
+  %result = phi i32 [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+
+define i32 @test_two_range_checks(i32* %array, i32 %length.1, i32 %length.2, i32 %n, i1 %cond_0) {
+; CHECK-LABEL: @test_two_range_checks(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[LENGTH_2:%.*]], [[LENGTH_1:%.*]]
+; CHECK-NEXT:    [[UMIN:%.*]] = select i1 [[TMP0]], i32 [[LENGTH_2]], i32 [[LENGTH_1]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i32 [[N:%.*]], 1
+; CHECK-NEXT:    [[UMAX:%.*]] = select i1 [[TMP1]], i32 [[N]], i32 1
+; CHECK-NEXT:    [[TMP2:%.*]] = add i32 [[UMAX]], -1
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ult i32 [[UMIN]], [[TMP2]]
+; CHECK-NEXT:    [[UMIN1:%.*]] = select i1 [[TMP3]], i32 [[UMIN]], i32 [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ugt i32 [[LENGTH_1]], [[UMIN1]]
+; CHECK-NEXT:    [[TMP5:%.*]] = freeze i1 [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = and i1 [[TMP5]], [[COND_0:%.*]]
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp ugt i32 [[LENGTH_2]], [[UMIN1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = freeze i1 [[TMP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = and i1 [[TMP8]], [[TMP6]]
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[TMP9]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[LOOP_PREHEADER:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    [[DEOPTRET:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTRET]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED2:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED2]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[I]], [[LENGTH_1]]
+; CHECK-NEXT:    br i1 true, label [[GUARDED:%.*]], label [[DEOPT2:%.*]], !prof !0
+; CHECK:       deopt2:
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    [[DEOPTRET2:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTRET2]]
+; CHECK:       guarded:
+; CHECK-NEXT:    [[WITHIN_BOUNDS2:%.*]] = icmp ult i32 [[I]], [[LENGTH_2]]
+; CHECK-NEXT:    br i1 true, label [[GUARDED2]], label [[DEOPT3:%.*]], !prof !0
+; CHECK:       deopt3:
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    [[DEOPTRET3:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTRET3]]
+; CHECK:       guarded2:
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    store i32 0, i32* [[ARRAY_I_PTR]]
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ult i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED2]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %cond_0, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %loop.preheader, label %deopt, !prof !0
+
+deopt:
+  %deoptret = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+  ret i32 %deoptret
+
+loop.preheader:
+  br label %loop
+
+loop:
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded2 ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded2 ], [ 0, %loop.preheader ]
+  call void @unknown()
+  %within.bounds = icmp ult i32 %i, %length.1
+  br i1 %within.bounds, label %guarded, label %deopt2, !prof !0
+
+deopt2:
+  call void @unknown()
+  %deoptret2 = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+  ret i32 %deoptret2
+
+guarded:
+  %within.bounds2 = icmp ult i32 %i, %length.2
+  br i1 %within.bounds2, label %guarded2, label %deopt3, !prof !0
+
+deopt3:
+  call void @unknown()
+  %deoptret3 = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+  ret i32 %deoptret3
+
+guarded2:
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  store i32 0, i32* %array.i.ptr
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp ult i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:
+  %result = phi i32 [ %loop.acc.next, %guarded2 ]
+  ret i32 %result
+}
+
+@G = external global i32
+
+define i32 @test_unanalyzeable_exit(i32* %array, i32 %length, i32 %n, i1 %cond_0) {
+; CHECK-LABEL: @test_unanalyzeable_exit(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_0:%.*]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[LOOP_PREHEADER:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    [[DEOPTRET:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTRET]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED2:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED2]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    [[VOL:%.*]] = load volatile i32, i32* @G
+; CHECK-NEXT:    [[UNKNOWN:%.*]] = icmp eq i32 [[VOL]], 0
+; CHECK-NEXT:    br i1 [[UNKNOWN]], label [[GUARDED2]], label [[DEOPT3:%.*]], !prof !0
+; CHECK:       deopt3:
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    [[DEOPTRET3:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTRET3]]
+; CHECK:       guarded2:
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    store i32 0, i32* [[ARRAY_I_PTR]]
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ult i32 [[I_NEXT]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED2]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %cond_0, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %loop.preheader, label %deopt, !prof !0
+
+deopt:
+  %deoptret = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+  ret i32 %deoptret
+
+loop.preheader:
+  br label %loop
+
+loop:
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded2 ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded2 ], [ 0, %loop.preheader ]
+  call void @unknown()
+  %vol = load volatile i32, i32* @G
+  %unknown = icmp eq i32 %vol, 0
+  br i1 %unknown, label %guarded2, label %deopt3, !prof !0
+
+deopt3:
+  call void @unknown()
+  %deoptret3 = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+  ret i32 %deoptret3
+
+guarded2:
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  store i32 0, i32* %array.i.ptr
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp ult i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:
+  %result = phi i32 [ %loop.acc.next, %guarded2 ]
+  ret i32 %result
+}
+
+define i32 @test_unanalyzeable_exit2(i32* %array, i32 %length, i32 %n, i1 %cond_0) {
+; CHECK-LABEL: @test_unanalyzeable_exit2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ugt i32 [[N:%.*]], 1
+; CHECK-NEXT:    [[UMAX:%.*]] = select i1 [[TMP0]], i32 [[N]], i32 1
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[UMAX]], -1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i32 [[LENGTH:%.*]], [[TMP1]]
+; CHECK-NEXT:    [[UMIN:%.*]] = select i1 [[TMP2]], i32 [[LENGTH]], i32 [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ugt i32 [[LENGTH]], [[UMIN]]
+; CHECK-NEXT:    [[TMP4:%.*]] = freeze i1 [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = and i1 [[TMP4]], [[COND_0:%.*]]
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[TMP5]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[LOOP_PREHEADER:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    [[DEOPTRET:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTRET]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED2:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED2]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[I]], [[LENGTH]]
+; CHECK-NEXT:    br i1 true, label [[GUARDED:%.*]], label [[DEOPT2:%.*]], !prof !0
+; CHECK:       deopt2:
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    [[DEOPTRET2:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTRET2]]
+; CHECK:       guarded:
+; CHECK-NEXT:    [[VOL:%.*]] = load volatile i32, i32* @G
+; CHECK-NEXT:    [[UNKNOWN:%.*]] = icmp eq i32 [[VOL]], 0
+; CHECK-NEXT:    br i1 [[UNKNOWN]], label [[GUARDED2]], label [[DEOPT3:%.*]], !prof !0
+; CHECK:       deopt3:
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    [[DEOPTRET3:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTRET3]]
+; CHECK:       guarded2:
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    store i32 0, i32* [[ARRAY_I_PTR]]
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ult i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED2]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %cond_0, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %loop.preheader, label %deopt, !prof !0
+
+deopt:
+  %deoptret = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+  ret i32 %deoptret
+
+loop.preheader:
+  br label %loop
+
+loop:
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded2 ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded2 ], [ 0, %loop.preheader ]
+  call void @unknown()
+  %within.bounds = icmp ult i32 %i, %length
+  br i1 %within.bounds, label %guarded, label %deopt2, !prof !0
+
+deopt2:
+  call void @unknown()
+  %deoptret2 = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+  ret i32 %deoptret2
+
+guarded:
+  %vol = load volatile i32, i32* @G
+  %unknown = icmp eq i32 %vol, 0
+  br i1 %unknown, label %guarded2, label %deopt3, !prof !0
+
+deopt3:
+  call void @unknown()
+  %deoptret3 = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+  ret i32 %deoptret3
+
+guarded2:
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  store i32 0, i32* %array.i.ptr
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp ult i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:
+  %result = phi i32 [ %loop.acc.next, %guarded2 ]
+  ret i32 %result
+}
+
+
+define i32 @test_unanalyzeable_latch(i32* %array, i32 %length, i32 %n, i1 %cond_0) {
+; CHECK-LABEL: @test_unanalyzeable_latch(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_0:%.*]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[LOOP_PREHEADER:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    [[DEOPTRET:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTRET]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[I]], [[LENGTH:%.*]]
+; CHECK-NEXT:    br i1 [[WITHIN_BOUNDS]], label [[GUARDED]], label [[DEOPT2:%.*]], !prof !0
+; CHECK:       deopt2:
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    [[DEOPTRET2:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTRET2]]
+; CHECK:       guarded:
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    store i32 0, i32* [[ARRAY_I_PTR]]
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[VOL:%.*]] = load volatile i32, i32* @G
+; CHECK-NEXT:    [[UNKNOWN:%.*]] = icmp eq i32 [[VOL]], 0
+; CHECK-NEXT:    br i1 [[UNKNOWN]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %cond_0, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %loop.preheader, label %deopt, !prof !0
+
+deopt:
+  %deoptret = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+  ret i32 %deoptret
+
+loop.preheader:
+  br label %loop
+
+loop:
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  call void @unknown()
+  %within.bounds = icmp ult i32 %i, %length
+  br i1 %within.bounds, label %guarded, label %deopt2, !prof !0
+
+deopt2:
+  call void @unknown()
+  %deoptret2 = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+  ret i32 %deoptret2
+
+guarded:
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  store i32 0, i32* %array.i.ptr
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %i.next = add nuw i32 %i, 1
+  %vol = load volatile i32, i32* @G
+  %unknown = icmp eq i32 %vol, 0
+  br i1 %unknown, label %loop, label %exit
+
+exit:
+  %result = phi i32 [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+
+define i32 @provably_taken(i32* %array, i1 %cond_0) {
+; CHECK-LABEL: @provably_taken(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[TMP0:%.*]] = freeze i1 false
+; CHECK-NEXT:    [[TMP1:%.*]] = and i1 [[TMP0]], [[COND_0:%.*]]
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[TMP1]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[LOOP_PREHEADER:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    [[DEOPTRET:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTRET]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[I]], 198
+; CHECK-NEXT:    br i1 true, label [[GUARDED]], label [[DEOPT2:%.*]], !prof !0
+; CHECK:       deopt2:
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    [[DEOPTRET2:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTRET2]]
+; CHECK:       guarded:
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    store i32 0, i32* [[ARRAY_I_PTR]]
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ult i32 [[I_NEXT]], 200
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %cond_0, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %loop.preheader, label %deopt, !prof !0
+
+deopt:
+  %deoptret = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+  ret i32 %deoptret
+
+loop.preheader:
+  br label %loop
+
+loop:
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  call void @unknown()
+  %within.bounds = icmp ult i32 %i, 198
+  br i1 %within.bounds, label %guarded, label %deopt2, !prof !0
+
+deopt2:
+  call void @unknown()
+  %deoptret2 = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+  ret i32 %deoptret2
+
+guarded:
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  store i32 0, i32* %array.i.ptr
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp ult i32 %i.next, 200
+  br i1 %continue, label %loop, label %exit
+
+exit:
+  %result = phi i32 [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+define i32 @provably_not_taken(i32* %array, i1 %cond_0) {
+; CHECK-LABEL: @provably_not_taken(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[TMP0:%.*]] = freeze i1 true
+; CHECK-NEXT:    [[TMP1:%.*]] = and i1 [[TMP0]], [[COND_0:%.*]]
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[TMP1]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[LOOP_PREHEADER:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    [[DEOPTRET:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTRET]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[I]], 205
+; CHECK-NEXT:    br i1 true, label [[GUARDED]], label [[DEOPT2:%.*]], !prof !0
+; CHECK:       deopt2:
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    [[DEOPTRET2:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTRET2]]
+; CHECK:       guarded:
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    store i32 0, i32* [[ARRAY_I_PTR]]
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ult i32 [[I_NEXT]], 200
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %cond_0, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %loop.preheader, label %deopt, !prof !0
+
+deopt:
+  %deoptret = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+  ret i32 %deoptret
+
+loop.preheader:
+  br label %loop
+
+loop:
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  call void @unknown()
+  %within.bounds = icmp ult i32 %i, 205
+  br i1 %within.bounds, label %guarded, label %deopt2, !prof !0
+
+deopt2:
+  call void @unknown()
+  %deoptret2 = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+  ret i32 %deoptret2
+
+guarded:
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  store i32 0, i32* %array.i.ptr
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp ult i32 %i.next, 200
+  br i1 %continue, label %loop, label %exit
+
+exit:
+  %result = phi i32 [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+
+;; Unswitch likes to produce some ugly exit blocks without simplifications
+;; being applied.  Make sure we can handle that form.
+define i32 @unswitch_exit_form(i32* %array, i32 %length, i32 %n, i1 %cond_0) {
+; CHECK-LABEL: @unswitch_exit_form(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ugt i32 [[N:%.*]], 1
+; CHECK-NEXT:    [[UMAX:%.*]] = select i1 [[TMP0]], i32 [[N]], i32 1
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[UMAX]], -1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i32 [[LENGTH:%.*]], [[TMP1]]
+; CHECK-NEXT:    [[UMIN:%.*]] = select i1 [[TMP2]], i32 [[LENGTH]], i32 [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ugt i32 [[LENGTH]], [[UMIN]]
+; CHECK-NEXT:    [[TMP4:%.*]] = freeze i1 [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = and i1 [[TMP4]], [[COND_0:%.*]]
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[TMP5]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[LOOP_PREHEADER:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt.loopexit:
+; CHECK-NEXT:    br label [[DEOPT]]
+; CHECK:       deopt:
+; CHECK-NEXT:    [[PHI:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ 1, [[DEOPT_LOOPEXIT:%.*]] ]
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    br label [[ACTUAL_DEOPT:%.*]]
+; CHECK:       actual_deopt:
+; CHECK-NEXT:    [[DEOPTRET:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"(i32 [[PHI]]) ]
+; CHECK-NEXT:    ret i32 [[DEOPTRET]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[I]], [[LENGTH]]
+; CHECK-NEXT:    br i1 true, label [[GUARDED]], label [[DEOPT_LOOPEXIT]], !prof !0
+; CHECK:       guarded:
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    store i32 0, i32* [[ARRAY_I_PTR]]
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ult i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %cond_0, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %loop.preheader, label %deopt, !prof !0
+
+deopt:
+  ;; This is written to look like an unsimplified loop exit after unswitch
+  ;; (i.e. phis, merge, and branch to actual block)
+  %phi = phi i32 [0, %entry], [1, %loop]
+  call void @unknown() ;; it's okay to skip possible throws
+  br label %actual_deopt
+
+actual_deopt:
+  %deoptret = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"(i32 %phi) ]
+  ret i32 %deoptret
+
+loop.preheader:
+  br label %loop
+
+loop:
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  call void @unknown()
+  %within.bounds = icmp ult i32 %i, %length
+  br i1 %within.bounds, label %guarded, label %deopt, !prof !0
+
+guarded:
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  store i32 0, i32* %array.i.ptr
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp ult i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:
+  %result = phi i32 [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+define i32 @swapped_wb(i32* %array, i32 %length, i32 %n, i1 %cond_0) {
+; CHECK-LABEL: @swapped_wb(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ugt i32 [[N:%.*]], 1
+; CHECK-NEXT:    [[UMAX:%.*]] = select i1 [[TMP0]], i32 [[N]], i32 1
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[UMAX]], -1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i32 [[LENGTH:%.*]], [[TMP1]]
+; CHECK-NEXT:    [[UMIN:%.*]] = select i1 [[TMP2]], i32 [[LENGTH]], i32 [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ugt i32 [[LENGTH]], [[UMIN]]
+; CHECK-NEXT:    [[TMP4:%.*]] = freeze i1 [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = and i1 [[TMP4]], [[COND_0:%.*]]
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[WIDENABLE_COND]], [[TMP5]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[LOOP_PREHEADER:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    [[DEOPTRET:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTRET]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[I]], [[LENGTH]]
+; CHECK-NEXT:    br i1 true, label [[GUARDED]], label [[DEOPT2:%.*]], !prof !0
+; CHECK:       deopt2:
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    [[DEOPTRET2:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTRET2]]
+; CHECK:       guarded:
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    store i32 0, i32* [[ARRAY_I_PTR]]
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ult i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %widenable_cond, %cond_0
+  br i1 %exiplicit_guard_cond, label %loop.preheader, label %deopt, !prof !0
+
+deopt:
+  %deoptret = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+  ret i32 %deoptret
+
+loop.preheader:
+  br label %loop
+
+loop:
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  call void @unknown()
+  %within.bounds = icmp ult i32 %i, %length
+  br i1 %within.bounds, label %guarded, label %deopt2, !prof !0
+
+deopt2:
+  call void @unknown()
+  %deoptret2 = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+  ret i32 %deoptret2
+
+guarded:
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  store i32 0, i32* %array.i.ptr
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp ult i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:
+  %result = phi i32 [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+define i32 @trivial_wb(i32* %array, i32 %length, i32 %n) {
+; CHECK-LABEL: @trivial_wb(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ugt i32 [[N:%.*]], 1
+; CHECK-NEXT:    [[UMAX:%.*]] = select i1 [[TMP0]], i32 [[N]], i32 1
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[UMAX]], -1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i32 [[LENGTH:%.*]], [[TMP1]]
+; CHECK-NEXT:    [[UMIN:%.*]] = select i1 [[TMP2]], i32 [[LENGTH]], i32 [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ugt i32 [[LENGTH]], [[UMIN]]
+; CHECK-NEXT:    [[TMP4:%.*]] = freeze i1 [[TMP3]]
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[TMP5:%.*]] = and i1 [[TMP4]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[TMP5]], label [[LOOP_PREHEADER:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    [[DEOPTRET:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTRET]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[I]], [[LENGTH]]
+; CHECK-NEXT:    br i1 true, label [[GUARDED]], label [[DEOPT2:%.*]], !prof !0
+; CHECK:       deopt2:
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    [[DEOPTRET2:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTRET2]]
+; CHECK:       guarded:
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    store i32 0, i32* [[ARRAY_I_PTR]]
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ult i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  br i1 %widenable_cond, label %loop.preheader, label %deopt, !prof !0
+
+deopt:
+  %deoptret = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+  ret i32 %deoptret
+
+loop.preheader:
+  br label %loop
+
+loop:
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  call void @unknown()
+  %within.bounds = icmp ult i32 %i, %length
+  br i1 %within.bounds, label %guarded, label %deopt2, !prof !0
+
+deopt2:
+  call void @unknown()
+  %deoptret2 = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+  ret i32 %deoptret2
+
+guarded:
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  store i32 0, i32* %array.i.ptr
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp ult i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:
+  %result = phi i32 [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+; TODO: Non-latch exits can still be predicated
+; This is currently prevented by an overly restrictive profitability check.
+define i32 @todo_unconditional_latch(i32* %array, i32 %length, i1 %cond_0) {
+; CHECK-LABEL: @todo_unconditional_latch(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_0:%.*]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[LOOP_PREHEADER:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    [[DEOPTRET:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTRET]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[I]], [[LENGTH:%.*]]
+; CHECK-NEXT:    br i1 [[WITHIN_BOUNDS]], label [[GUARDED]], label [[DEOPT2:%.*]], !prof !0
+; CHECK:       deopt2:
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    [[DEOPTRET2:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTRET2]]
+; CHECK:       guarded:
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    store i32 0, i32* [[ARRAY_I_PTR]]
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    br label [[LOOP]]
+;
+entry:
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %cond_0, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %loop.preheader, label %deopt, !prof !0
+
+deopt:
+  %deoptret = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+  ret i32 %deoptret
+
+loop.preheader:
+  br label %loop
+
+loop:
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  call void @unknown()
+  %within.bounds = icmp ult i32 %i, %length
+  br i1 %within.bounds, label %guarded, label %deopt2, !prof !0
+
+deopt2:
+  call void @unknown()
+  %deoptret2 = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+  ret i32 %deoptret2
+
+guarded:
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  store i32 0, i32* %array.i.ptr
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %i.next = add nuw i32 %i, 1
+  br label %loop
+}
+
+
+; If we have a stray widenable branch in the loop, we should still be able to
+; run.  This can happen when unswitching's cost model avoids unswitching some
+; branches.
+define i32 @wb_in_loop(i32* %array, i32 %length, i32 %n, i1 %cond_0) {
+; CHECK-LABEL: @wb_in_loop(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[WC2:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ugt i32 [[N:%.*]], 1
+; CHECK-NEXT:    [[UMAX:%.*]] = select i1 [[TMP0]], i32 [[N]], i32 1
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[UMAX]], -1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i32 [[LENGTH:%.*]], [[TMP1]]
+; CHECK-NEXT:    [[UMIN:%.*]] = select i1 [[TMP2]], i32 [[LENGTH]], i32 [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ugt i32 [[LENGTH]], [[UMIN]]
+; CHECK-NEXT:    [[TMP4:%.*]] = freeze i1 [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = and i1 [[TMP4]], [[COND_0:%.*]]
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ugt i32 [[LENGTH]], [[UMIN]]
+; CHECK-NEXT:    [[TMP7:%.*]] = freeze i1 [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = and i1 [[TMP7]], [[TMP5]]
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[TMP8]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[LOOP_PREHEADER:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    [[DEOPTRET:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTRET]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED2:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED2]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[I]], [[LENGTH]]
+; CHECK-NEXT:    br i1 true, label [[GUARDED:%.*]], label [[DEOPT2:%.*]], !prof !0
+; CHECK:       deopt2:
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    [[DEOPTRET2:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTRET2]]
+; CHECK:       guarded:
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    [[WITHIN_BOUNDS2:%.*]] = icmp ult i32 [[I]], [[LENGTH]]
+; CHECK-NEXT:    [[WB_COND:%.*]] = and i1 [[WITHIN_BOUNDS2]], true
+; CHECK-NEXT:    br i1 true, label [[GUARDED2]], label [[DEOPT3:%.*]], !prof !0
+; CHECK:       deopt3:
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    [[DEOPTRET3:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTRET3]]
+; CHECK:       guarded2:
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    store i32 0, i32* [[ARRAY_I_PTR]]
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ult i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED2]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %wc2 = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %cond_0, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %loop.preheader, label %deopt, !prof !0
+
+deopt:
+  %deoptret = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+  ret i32 %deoptret
+
+loop.preheader:
+  br label %loop
+
+loop:
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded2 ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded2 ], [ 0, %loop.preheader ]
+  call void @unknown()
+  %within.bounds = icmp ult i32 %i, %length
+  br i1 %within.bounds, label %guarded, label %deopt2, !prof !0
+
+deopt2:
+  call void @unknown()
+  %deoptret2 = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+  ret i32 %deoptret2
+
+guarded:
+  call void @unknown()
+  %within.bounds2 = icmp ult i32 %i, %length
+  %wb_cond = and i1 %within.bounds2, %wc2
+  br i1 %wb_cond, label %guarded2, label %deopt3, !prof !0
+
+deopt3:
+  call void @unknown()
+  %deoptret3 = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+  ret i32 %deoptret3
+
+guarded2:
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  store i32 0, i32* %array.i.ptr
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp ult i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:
+  %result = phi i32 [ %loop.acc.next, %guarded2 ]
+  ret i32 %result
+}
+
+
+
+declare void @unknown()
+
+declare i1 @llvm.experimental.widenable.condition()
+declare i32 @llvm.experimental.deoptimize.i32(...)
+
+!0 = !{!"branch_weights", i32 1048576, i32 1}
+!1 = !{i32 1, i32 -2147483648}
+!2 = !{i32 0, i32 50}
diff --git a/llvm/test/Transforms/LoopReroll/reroll_with_dbg.ll b/llvm/test/Transforms/LoopReroll/reroll_with_dbg.ll
index 3a752b23b..a57ccd42a 100644
--- a/llvm/test/Transforms/LoopReroll/reroll_with_dbg.ll
+++ b/llvm/test/Transforms/LoopReroll/reroll_with_dbg.ll
@@ -78,7 +78,7 @@ for.body:                                         ; preds = %for.body.preheader,
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.value(metadata, metadata, metadata) #1
 
-attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="arm7tdmi" "target-features"="+strict-align" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="arm7tdmi" "target-features"="+strict-align" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/Transforms/LoopRotate/basic.ll b/llvm/test/Transforms/LoopRotate/basic.ll
index d01d19f7f..dc13ea91c 100644
--- a/llvm/test/Transforms/LoopRotate/basic.ll
+++ b/llvm/test/Transforms/LoopRotate/basic.ll
@@ -1,7 +1,7 @@
 ; RUN: opt -S -loop-rotate < %s | FileCheck %s
 ; RUN: opt -S -loop-rotate -enable-mssa-loop-dependency=true -verify-memoryssa < %s | FileCheck %s
 ; RUN: opt -S -passes='require<targetir>,require<assumptions>,loop(rotate)' < %s | FileCheck %s
-; RUN: opt -S -passes='require<targetir>,require<assumptions>,loop(rotate)' -enable-mssa-loop-dependency=true -verify-memoryssa  < %s | FileCheck %s
+; RUN: opt -S -passes='require<targetir>,require<assumptions>,loop-mssa(rotate)' -verify-memoryssa  < %s | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 target triple = "x86_64-apple-darwin10.0.0"
diff --git a/llvm/test/Transforms/LoopRotate/dbg-value-duplicates-2.ll b/llvm/test/Transforms/LoopRotate/dbg-value-duplicates-2.ll
new file mode 100644
index 000000000..318d3e6e0
--- /dev/null
+++ b/llvm/test/Transforms/LoopRotate/dbg-value-duplicates-2.ll
@@ -0,0 +1,77 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -loop-rotate -S | FileCheck %s
+
+define dso_local i16 @main() local_unnamed_addr #0 !dbg !7 {
+; CHECK-LABEL: @main(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    call void @llvm.dbg.value(metadata i16 9, metadata !12, metadata !DIExpression()), !dbg !13
+; CHECK-NEXT:    br label [[BB2:%.*]], !dbg !14
+; CHECK:       bb2:
+; CHECK-NEXT:    [[TMP1:%.*]] = phi i16 [ 9, [[ENTRY:%.*]] ], [ [[TMP5:%.*]], [[BB2]] ]
+; CHECK-NEXT:    call void @llvm.dbg.value(metadata i16 [[TMP1]], metadata !12, metadata !DIExpression()), !dbg !13
+; CHECK-NEXT:    [[TMP4:%.*]] = call i16 @wibble(i16 [[TMP1]]), !dbg !14
+; CHECK-NEXT:    [[TMP5]] = add nsw i16 [[TMP4]], [[TMP1]], !dbg !14
+; CHECK-NEXT:    call void @llvm.dbg.value(metadata i16 [[TMP5]], metadata !12, metadata !DIExpression()), !dbg !13
+; CHECK-NEXT:    [[TMP6:%.*]] = call i16 @wibble(i16 [[TMP4]]), !dbg !14
+; CHECK-NEXT:    [[TMP7:%.*]] = mul nsw i16 [[TMP6]], 3, !dbg !14
+; CHECK-NEXT:    [[TMP8:%.*]] = call i16 @wibble(i16 [[TMP7]]), !dbg !14
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i16 [[TMP5]], 17, !dbg !14
+; CHECK-NEXT:    br i1 [[TMP2]], label [[BB2]], label [[BB3:%.*]], !dbg !14
+; CHECK:       bb3:
+; CHECK-NEXT:    [[TMP10:%.*]] = call i16 @wibble(i16 8), !dbg !14
+; CHECK-NEXT:    ret i16 [[TMP10]], !dbg !14
+;
+entry:
+  call void @llvm.dbg.value(metadata i16 9, metadata !12, metadata !DIExpression()), !dbg !13
+  br label %bb1, !dbg !14
+
+bb1:
+  %tmp = phi i16 [ 9, %entry ], [ %tmp5, %bb2 ], !dbg !13
+  call void @llvm.dbg.value(metadata i16 %tmp, metadata !12, metadata !DIExpression()), !dbg !13
+  %tmp2 = icmp slt i16 %tmp, 17, !dbg !16
+  br i1 %tmp2, label %bb2, label %bb3, !dbg !16
+
+bb2:
+  %tmp4 = call i16 @wibble(i16 %tmp), !dbg !16
+  %tmp5 = add nsw i16 %tmp4, %tmp, !dbg !16
+  call void @llvm.dbg.value(metadata i16 %tmp5, metadata !12, metadata !DIExpression()), !dbg !13
+  %tmp6 = call i16 @wibble(i16 %tmp4), !dbg !16
+  %tmp7 = mul nsw i16 %tmp6, 3, !dbg !16
+  %tmp8 = call i16 @wibble(i16 %tmp7), !dbg !16
+  br label %bb1, !dbg !16
+
+bb3:
+  %tmp10 = call i16 @wibble(i16 8), !dbg !16
+  ret i16 %tmp10, !dbg !16
+}
+
+declare void @llvm.dbg.value(metadata, metadata, metadata) #1
+
+declare i16 @wibble(i16) #2
+
+attributes #0 = { noinline nounwind }
+attributes #1 = { nounwind readnone speculatable willreturn }
+attributes #2 = { noinline nounwind readnone }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 10.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, globals: !2, nameTableKind: None)
+!1 = !DIFile(filename: "foo.c", directory: "")
+!2 = !{}
+!3 = !{i32 7, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 1}
+!6 = !{!"clang version 10.0.0"}
+!7 = distinct !DISubprogram(name: "main", scope: !1, file: !1, line: 8, type: !8, scopeLine: 8, flags: DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !11)
+!8 = !DISubroutineType(types: !9)
+!9 = !{!10}
+!10 = !DIBasicType(name: "int", size: 16, encoding: DW_ATE_signed)
+!11 = !{!12}
+!12 = !DILocalVariable(name: "x", scope: !7, file: !1, line: 9, type: !10)
+!13 = !DILocation(line: 0, scope: !7)
+!14 = !DILocation(line: 11, column: 8, scope: !15)
+!15 = distinct !DILexicalBlock(scope: !7, file: !1, line: 11, column: 3)
+!16 = !DILocation(line: 11, column: 21, scope: !17)
+!17 = distinct !DILexicalBlock(scope: !15, file: !1, line: 11, column: 3)
diff --git a/llvm/test/Transforms/LoopRotate/pr35210.ll b/llvm/test/Transforms/LoopRotate/pr35210.ll
index a705642c4..982a4c0c8 100644
--- a/llvm/test/Transforms/LoopRotate/pr35210.ll
+++ b/llvm/test/Transforms/LoopRotate/pr35210.ll
@@ -1,5 +1,5 @@
 ;RUN: opt %s -passes='adce,loop(rotate),adce' -S -debug-pass-manager -debug-only=loop-rotate 2>&1 | FileCheck %s
-;RUN: opt %s -passes='adce,loop(rotate),adce' -S -debug-pass-manager -debug-only=loop-rotate -enable-mssa-loop-dependency=true -verify-memoryssa 2>&1 | FileCheck %s --check-prefix=MSSA
+;RUN: opt %s -passes='adce,loop-mssa(rotate),adce' -S -debug-pass-manager -debug-only=loop-rotate -verify-memoryssa 2>&1 | FileCheck %s --check-prefix=MSSA
 ;REQUIRES: asserts
 
 ; This test is to make sure we invalidate the post dominator pass after loop rotate simplifies the loop latch.
diff --git a/llvm/test/Transforms/LoopRotate/switch.ll b/llvm/test/Transforms/LoopRotate/switch.ll
new file mode 100644
index 000000000..08d64eace
--- /dev/null
+++ b/llvm/test/Transforms/LoopRotate/switch.ll
@@ -0,0 +1,166 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -loop-rotate < %s | FileCheck %s
+
+define i64 @switch_multi_entry_known_entry() {
+; CHECK-LABEL: @switch_multi_entry_known_entry(
+; CHECK-NEXT:  start:
+; CHECK-NEXT:    br label [[HEADER:%.*]]
+; CHECK:       header:
+; CHECK-NEXT:    [[STATE:%.*]] = phi i8 [ 2, [[START:%.*]] ], [ [[NEXT_STATE:%.*]], [[LATCH:%.*]] ]
+; CHECK-NEXT:    [[COUNT:%.*]] = phi i64 [ 0, [[START]] ], [ [[INC:%.*]], [[LATCH]] ]
+; CHECK-NEXT:    switch i8 [[STATE]], label [[EXIT:%.*]] [
+; CHECK-NEXT:    i8 0, label [[LATCH]]
+; CHECK-NEXT:    i8 2, label [[LATCH]]
+; CHECK-NEXT:    ]
+; CHECK:       latch:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i64 [[COUNT]], 999
+; CHECK-NEXT:    [[NEXT_STATE]] = zext i1 [[CMP]] to i8
+; CHECK-NEXT:    [[INC]] = add i64 [[COUNT]], 1
+; CHECK-NEXT:    br label [[HEADER]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[COUNT_LCSSA:%.*]] = phi i64 [ [[COUNT]], [[HEADER]] ]
+; CHECK-NEXT:    ret i64 [[COUNT_LCSSA]]
+;
+start:
+  br label %header
+
+header:                                           ; preds = %latch, %start
+  %state = phi i8 [ 2, %start ], [ %next_state, %latch ]
+  %count = phi i64 [ 0, %start ], [ %inc, %latch ]
+  switch i8 %state, label %exit [
+  i8 0, label %latch
+  i8 2, label %latch
+  ]
+
+latch:                                            ; preds = %header, %header
+  %cmp = icmp eq i64 %count, 999
+  %next_state = zext i1 %cmp to i8
+  %inc = add i64 %count, 1
+  br label %header
+
+exit:                                             ; preds = %header
+  ret i64 %count
+}
+
+define i64 @switch_multi_entry_unknown_entry(i8 %start_state) {
+; CHECK-LABEL: @switch_multi_entry_unknown_entry(
+; CHECK-NEXT:  start:
+; CHECK-NEXT:    br label [[HEADER:%.*]]
+; CHECK:       header:
+; CHECK-NEXT:    [[STATE:%.*]] = phi i8 [ [[START_STATE:%.*]], [[START:%.*]] ], [ [[NEXT_STATE:%.*]], [[LATCH:%.*]] ]
+; CHECK-NEXT:    [[COUNT:%.*]] = phi i64 [ 0, [[START]] ], [ [[INC:%.*]], [[LATCH]] ]
+; CHECK-NEXT:    switch i8 [[STATE]], label [[EXIT:%.*]] [
+; CHECK-NEXT:    i8 0, label [[LATCH]]
+; CHECK-NEXT:    i8 2, label [[LATCH]]
+; CHECK-NEXT:    ]
+; CHECK:       latch:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i64 [[COUNT]], 999
+; CHECK-NEXT:    [[NEXT_STATE]] = zext i1 [[CMP]] to i8
+; CHECK-NEXT:    [[INC]] = add i64 [[COUNT]], 1
+; CHECK-NEXT:    br label [[HEADER]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[COUNT_LCSSA:%.*]] = phi i64 [ [[COUNT]], [[HEADER]] ]
+; CHECK-NEXT:    ret i64 [[COUNT_LCSSA]]
+;
+start:
+  br label %header
+
+header:                                           ; preds = %latch, %start
+  %state = phi i8 [ %start_state, %start ], [ %next_state, %latch ]
+  %count = phi i64 [ 0, %start ], [ %inc, %latch ]
+  switch i8 %state, label %exit [
+  i8 0, label %latch
+  i8 2, label %latch
+  ]
+
+latch:                                            ; preds = %header, %header
+  %cmp = icmp eq i64 %count, 999
+  %next_state = zext i1 %cmp to i8
+  %inc = add i64 %count, 1
+  br label %header
+
+exit:                                             ; preds = %header
+  ret i64 %count
+}
+
+define i64 @switch_multi_exit_known_entry() {
+; CHECK-LABEL: @switch_multi_exit_known_entry(
+; CHECK-NEXT:  start:
+; CHECK-NEXT:    br label [[HEADER:%.*]]
+; CHECK:       header:
+; CHECK-NEXT:    [[STATE:%.*]] = phi i8 [ 0, [[START:%.*]] ], [ [[NEXT_STATE:%.*]], [[LATCH:%.*]] ]
+; CHECK-NEXT:    [[COUNT:%.*]] = phi i64 [ 0, [[START]] ], [ [[INC:%.*]], [[LATCH]] ]
+; CHECK-NEXT:    switch i8 [[STATE]], label [[LATCH]] [
+; CHECK-NEXT:    i8 1, label [[EXIT:%.*]]
+; CHECK-NEXT:    i8 2, label [[EXIT]]
+; CHECK-NEXT:    ]
+; CHECK:       latch:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i64 [[COUNT]], 999
+; CHECK-NEXT:    [[NEXT_STATE]] = zext i1 [[CMP]] to i8
+; CHECK-NEXT:    [[INC]] = add i64 [[COUNT]], 1
+; CHECK-NEXT:    br label [[HEADER]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[COUNT_LCSSA:%.*]] = phi i64 [ [[COUNT]], [[HEADER]] ], [ [[COUNT]], [[HEADER]] ]
+; CHECK-NEXT:    ret i64 [[COUNT_LCSSA]]
+;
+start:
+  br label %header
+
+header:                                           ; preds = %latch, %start
+  %state = phi i8 [ 0, %start ], [ %next_state, %latch ]
+  %count = phi i64 [ 0, %start ], [ %inc, %latch ]
+  switch i8 %state, label %latch [
+  i8 1, label %exit
+  i8 2, label %exit
+  ]
+
+latch:                                            ; preds = %header, %header
+  %cmp = icmp eq i64 %count, 999
+  %next_state = zext i1 %cmp to i8
+  %inc = add i64 %count, 1
+  br label %header
+
+exit:                                             ; preds = %header
+  ret i64 %count
+}
+
+define i64 @switch_multi_exit_unknown_entry(i8 %start_state) {
+; CHECK-LABEL: @switch_multi_exit_unknown_entry(
+; CHECK-NEXT:  start:
+; CHECK-NEXT:    br label [[HEADER:%.*]]
+; CHECK:       header:
+; CHECK-NEXT:    [[STATE:%.*]] = phi i8 [ [[START_STATE:%.*]], [[START:%.*]] ], [ [[NEXT_STATE:%.*]], [[LATCH:%.*]] ]
+; CHECK-NEXT:    [[COUNT:%.*]] = phi i64 [ 0, [[START]] ], [ [[INC:%.*]], [[LATCH]] ]
+; CHECK-NEXT:    switch i8 [[STATE]], label [[LATCH]] [
+; CHECK-NEXT:    i8 1, label [[EXIT:%.*]]
+; CHECK-NEXT:    i8 2, label [[EXIT]]
+; CHECK-NEXT:    ]
+; CHECK:       latch:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i64 [[COUNT]], 999
+; CHECK-NEXT:    [[NEXT_STATE]] = zext i1 [[CMP]] to i8
+; CHECK-NEXT:    [[INC]] = add i64 [[COUNT]], 1
+; CHECK-NEXT:    br label [[HEADER]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[COUNT_LCSSA:%.*]] = phi i64 [ [[COUNT]], [[HEADER]] ], [ [[COUNT]], [[HEADER]] ]
+; CHECK-NEXT:    ret i64 [[COUNT_LCSSA]]
+;
+start:
+  br label %header
+
+header:                                           ; preds = %latch, %start
+  %state = phi i8 [ %start_state, %start ], [ %next_state, %latch ]
+  %count = phi i64 [ 0, %start ], [ %inc, %latch ]
+  switch i8 %state, label %latch [
+  i8 1, label %exit
+  i8 2, label %exit
+  ]
+
+latch:                                            ; preds = %header, %header
+  %cmp = icmp eq i64 %count, 999
+  %next_state = zext i1 %cmp to i8
+  %inc = add i64 %count, 1
+  br label %header
+
+exit:                                             ; preds = %header
+  ret i64 %count
+}
diff --git a/llvm/test/Transforms/LoopSimplify/basictest.ll b/llvm/test/Transforms/LoopSimplify/basictest.ll
index e5fb7b990..fc3864bdf 100644
--- a/llvm/test/Transforms/LoopSimplify/basictest.ll
+++ b/llvm/test/Transforms/LoopSimplify/basictest.ll
@@ -1,236 +1,228 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -S -loop-simplify | FileCheck %s
 ; RUN: opt < %s -S -passes=loop-simplify | FileCheck %s
 
 ; This function should get a preheader inserted before bb3, that is jumped
 ; to by bb1 & bb2
 define void @test() {
-; CHECK-LABEL: define void @test(
+; CHECK-LABEL: @test(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 true, label [[BB1:%.*]], label [[BB2:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    br label [[BB3_PREHEADER:%.*]]
+; CHECK:       bb2:
+; CHECK-NEXT:    br label [[BB3_PREHEADER]]
+; CHECK:       bb3.preheader:
+; CHECK-NEXT:    br label [[BB3:%.*]]
+; CHECK:       bb3:
+; CHECK-NEXT:    br label [[BB3]]
+;
 entry:
   br i1 true, label %bb1, label %bb2
 
 bb1:
   br label %bb3
-; CHECK:      bb1:
-; CHECK-NEXT:   br label %[[PH:.*]]
 
 bb2:
   br label %bb3
-; CHECK:      bb2:
-; CHECK-NEXT:   br label %[[PH]]
 
 bb3:
   br label %bb3
-; CHECK:      [[PH]]:
-; CHECK-NEXT:   br label %bb3
-;
-; CHECK:      bb3:
-; CHECK-NEXT:   br label %bb3
 }
 
 ; Test a case where we have multiple exit blocks as successors of a single loop
 ; block that need to be made dedicated exit blocks. We also have multiple
 ; exiting edges to one of the exit blocks that all should be rewritten.
 define void @test_multiple_exits_from_single_block(i8 %a, i8* %b.ptr) {
-; CHECK-LABEL: define void @test_multiple_exits_from_single_block(
+; CHECK-LABEL: @test_multiple_exits_from_single_block(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    switch i8 [[A:%.*]], label [[LOOP_PREHEADER:%.*]] [
+; CHECK-NEXT:    i8 0, label [[EXIT_A:%.*]]
+; CHECK-NEXT:    i8 1, label [[EXIT_B:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[B:%.*]] = load volatile i8, i8* [[B_PTR:%.*]]
+; CHECK-NEXT:    switch i8 [[B]], label [[LOOP_BACKEDGE:%.*]] [
+; CHECK-NEXT:    i8 0, label [[EXIT_A_LOOPEXIT:%.*]]
+; CHECK-NEXT:    i8 1, label [[EXIT_B_LOOPEXIT:%.*]]
+; CHECK-NEXT:    i8 2, label [[LOOP_BACKEDGE]]
+; CHECK-NEXT:    i8 3, label [[EXIT_A_LOOPEXIT]]
+; CHECK-NEXT:    i8 4, label [[LOOP_BACKEDGE]]
+; CHECK-NEXT:    i8 5, label [[EXIT_A_LOOPEXIT]]
+; CHECK-NEXT:    i8 6, label [[LOOP_BACKEDGE]]
+; CHECK-NEXT:    ]
+; CHECK:       loop.backedge:
+; CHECK-NEXT:    br label [[LOOP]]
+; CHECK:       exit.a.loopexit:
+; CHECK-NEXT:    br label [[EXIT_A]]
+; CHECK:       exit.a:
+; CHECK-NEXT:    ret void
+; CHECK:       exit.b.loopexit:
+; CHECK-NEXT:    br label [[EXIT_B]]
+; CHECK:       exit.b:
+; CHECK-NEXT:    ret void
+;
 entry:
   switch i8 %a, label %loop [
-    i8 0, label %exit.a
-    i8 1, label %exit.b
+  i8 0, label %exit.a
+  i8 1, label %exit.b
   ]
-; CHECK:      entry:
-; CHECK-NEXT:   switch i8 %a, label %[[PH:.*]] [
-; CHECK-NEXT:     i8 0, label %exit.a
-; CHECK-NEXT:     i8 1, label %exit.b
-; CHECK-NEXT:   ]
 
 loop:
   %b = load volatile i8, i8* %b.ptr
   switch i8 %b, label %loop [
-    i8 0, label %exit.a
-    i8 1, label %exit.b
-    i8 2, label %loop
-    i8 3, label %exit.a
-    i8 4, label %loop
-    i8 5, label %exit.a
-    i8 6, label %loop
+  i8 0, label %exit.a
+  i8 1, label %exit.b
+  i8 2, label %loop
+  i8 3, label %exit.a
+  i8 4, label %loop
+  i8 5, label %exit.a
+  i8 6, label %loop
   ]
-; CHECK:      [[PH]]:
-; CHECK-NEXT:   br label %loop
-;
-; CHECK:      loop:
-; CHECK-NEXT:   %[[B:.*]] = load volatile i8, i8* %b.ptr
-; CHECK-NEXT:   switch i8 %[[B]], label %[[BACKEDGE:.*]] [
-; CHECK-NEXT:     i8 0, label %[[LOOPEXIT_A:.*]]
-; CHECK-NEXT:     i8 1, label %[[LOOPEXIT_B:.*]]
-; CHECK-NEXT:     i8 2, label %[[BACKEDGE]]
-; CHECK-NEXT:     i8 3, label %[[LOOPEXIT_A]]
-; CHECK-NEXT:     i8 4, label %[[BACKEDGE]]
-; CHECK-NEXT:     i8 5, label %[[LOOPEXIT_A]]
-; CHECK-NEXT:     i8 6, label %[[BACKEDGE]]
-; CHECK-NEXT:   ]
-;
-; CHECK:      [[BACKEDGE]]:
-; CHECK-NEXT:   br label %loop
 
 exit.a:
   ret void
-; CHECK:      [[LOOPEXIT_A]]:
-; CHECK-NEXT:   br label %exit.a
-;
-; CHECK:      exit.a:
-; CHECK-NEXT:   ret void
 
 exit.b:
   ret void
-; CHECK:      [[LOOPEXIT_B]]:
-; CHECK-NEXT:   br label %exit.b
-;
-; CHECK:      exit.b:
-; CHECK-NEXT:   ret void
 }
 
 ; Check that we leave already dedicated exits alone when forming dedicated exit
 ; blocks.
 define void @test_pre_existing_dedicated_exits(i1 %a, i1* %ptr) {
-; CHECK-LABEL: define void @test_pre_existing_dedicated_exits(
+; CHECK-LABEL: @test_pre_existing_dedicated_exits(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[A:%.*]], label [[LOOP_PH:%.*]], label [[NON_DEDICATED_EXIT:%.*]]
+; CHECK:       loop.ph:
+; CHECK-NEXT:    br label [[LOOP_HEADER:%.*]]
+; CHECK:       loop.header:
+; CHECK-NEXT:    [[C1:%.*]] = load volatile i1, i1* [[PTR:%.*]]
+; CHECK-NEXT:    br i1 [[C1]], label [[LOOP_BODY1:%.*]], label [[DEDICATED_EXIT1:%.*]]
+; CHECK:       loop.body1:
+; CHECK-NEXT:    [[C2:%.*]] = load volatile i1, i1* [[PTR]]
+; CHECK-NEXT:    br i1 [[C2]], label [[LOOP_BODY2:%.*]], label [[NON_DEDICATED_EXIT_LOOPEXIT:%.*]]
+; CHECK:       loop.body2:
+; CHECK-NEXT:    [[C3:%.*]] = load volatile i1, i1* [[PTR]]
+; CHECK-NEXT:    br i1 [[C3]], label [[LOOP_BACKEDGE:%.*]], label [[DEDICATED_EXIT2:%.*]]
+; CHECK:       loop.backedge:
+; CHECK-NEXT:    br label [[LOOP_HEADER]]
+; CHECK:       dedicated_exit1:
+; CHECK-NEXT:    ret void
+; CHECK:       dedicated_exit2:
+; CHECK-NEXT:    ret void
+; CHECK:       non_dedicated_exit.loopexit:
+; CHECK-NEXT:    br label [[NON_DEDICATED_EXIT]]
+; CHECK:       non_dedicated_exit:
+; CHECK-NEXT:    ret void
+;
 entry:
   br i1 %a, label %loop.ph, label %non_dedicated_exit
-; CHECK:      entry:
-; CHECK-NEXT:   br i1 %a, label %loop.ph, label %non_dedicated_exit
 
 loop.ph:
   br label %loop.header
-; CHECK:      loop.ph:
-; CHECK-NEXT:   br label %loop.header
 
 loop.header:
   %c1 = load volatile i1, i1* %ptr
   br i1 %c1, label %loop.body1, label %dedicated_exit1
-; CHECK:      loop.header:
-; CHECK-NEXT:   %[[C1:.*]] = load volatile i1, i1* %ptr
-; CHECK-NEXT:   br i1 %[[C1]], label %loop.body1, label %dedicated_exit1
 
 loop.body1:
   %c2 = load volatile i1, i1* %ptr
   br i1 %c2, label %loop.body2, label %non_dedicated_exit
-; CHECK:      loop.body1:
-; CHECK-NEXT:   %[[C2:.*]] = load volatile i1, i1* %ptr
-; CHECK-NEXT:   br i1 %[[C2]], label %loop.body2, label %[[LOOPEXIT:.*]]
 
 loop.body2:
   %c3 = load volatile i1, i1* %ptr
   br i1 %c3, label %loop.backedge, label %dedicated_exit2
-; CHECK:      loop.body2:
-; CHECK-NEXT:   %[[C3:.*]] = load volatile i1, i1* %ptr
-; CHECK-NEXT:   br i1 %[[C3]], label %loop.backedge, label %dedicated_exit2
 
 loop.backedge:
   br label %loop.header
-; CHECK:      loop.backedge:
-; CHECK-NEXT:   br label %loop.header
 
 dedicated_exit1:
   ret void
 ; Check that there isn't a split loop exit.
-; CHECK-NOT:    br label %dedicated_exit1
-;
-; CHECK:      dedicated_exit1:
-; CHECK-NEXT:   ret void
 
 dedicated_exit2:
   ret void
 ; Check that there isn't a split loop exit.
-; CHECK-NOT:    br label %dedicated_exit2
-;
-; CHECK:      dedicated_exit2:
-; CHECK-NEXT:   ret void
 
 non_dedicated_exit:
   ret void
-; CHECK:      [[LOOPEXIT]]:
-; CHECK-NEXT:   br label %non_dedicated_exit
-;
-; CHECK:      non_dedicated_exit:
-; CHECK-NEXT:   ret void
 }
 
 ; Check that we form what dedicated exits we can even when some exits are
 ; reached via indirectbr which precludes forming dedicated exits.
 define void @test_form_some_dedicated_exits_despite_indirectbr(i8 %a, i8* %ptr, i8** %addr.ptr) {
-; CHECK-LABEL: define void @test_form_some_dedicated_exits_despite_indirectbr(
+; CHECK-LABEL: @test_form_some_dedicated_exits_despite_indirectbr(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    switch i8 [[A:%.*]], label [[LOOP_PH:%.*]] [
+; CHECK-NEXT:    i8 0, label [[EXIT_A:%.*]]
+; CHECK-NEXT:    i8 1, label [[EXIT_B:%.*]]
+; CHECK-NEXT:    i8 2, label [[EXIT_C:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       loop.ph:
+; CHECK-NEXT:    br label [[LOOP_HEADER:%.*]]
+; CHECK:       loop.header:
+; CHECK-NEXT:    [[ADDR1:%.*]] = load volatile i8*, i8** [[ADDR_PTR:%.*]]
+; CHECK-NEXT:    indirectbr i8* [[ADDR1]], [label [[LOOP_BODY1:%.*]], label %exit.a]
+; CHECK:       loop.body1:
+; CHECK-NEXT:    [[B:%.*]] = load volatile i8, i8* [[PTR:%.*]]
+; CHECK-NEXT:    switch i8 [[B]], label [[LOOP_BODY2:%.*]] [
+; CHECK-NEXT:    i8 0, label [[EXIT_A]]
+; CHECK-NEXT:    i8 1, label [[EXIT_B_LOOPEXIT:%.*]]
+; CHECK-NEXT:    i8 2, label [[EXIT_C]]
+; CHECK-NEXT:    ]
+; CHECK:       loop.body2:
+; CHECK-NEXT:    [[ADDR2:%.*]] = load volatile i8*, i8** [[ADDR_PTR]]
+; CHECK-NEXT:    indirectbr i8* [[ADDR2]], [label [[LOOP_BACKEDGE:%.*]], label %exit.c]
+; CHECK:       loop.backedge:
+; CHECK-NEXT:    br label [[LOOP_HEADER]]
+; CHECK:       exit.a:
+; CHECK-NEXT:    ret void
+; CHECK:       exit.b.loopexit:
+; CHECK-NEXT:    br label [[EXIT_B]]
+; CHECK:       exit.b:
+; CHECK-NEXT:    ret void
+; CHECK:       exit.c:
+; CHECK-NEXT:    ret void
+;
 entry:
   switch i8 %a, label %loop.ph [
-    i8 0, label %exit.a
-    i8 1, label %exit.b
-    i8 2, label %exit.c
+  i8 0, label %exit.a
+  i8 1, label %exit.b
+  i8 2, label %exit.c
   ]
-; CHECK:      entry:
-; CHECK-NEXT:   switch i8 %a, label %loop.ph [
-; CHECK-NEXT:     i8 0, label %exit.a
-; CHECK-NEXT:     i8 1, label %exit.b
-; CHECK-NEXT:     i8 2, label %exit.c
-; CHECK-NEXT:   ]
 
 loop.ph:
   br label %loop.header
-; CHECK:      loop.ph:
-; CHECK-NEXT:   br label %loop.header
 
 loop.header:
   %addr1 = load volatile i8*, i8** %addr.ptr
   indirectbr i8* %addr1, [label %loop.body1, label %exit.a]
-; CHECK:      loop.header:
-; CHECK-NEXT:   %[[ADDR1:.*]] = load volatile i8*, i8** %addr.ptr
-; CHECK-NEXT:   indirectbr i8* %[[ADDR1]], [label %loop.body1, label %exit.a]
 
 loop.body1:
   %b = load volatile i8, i8* %ptr
   switch i8 %b, label %loop.body2 [
-    i8 0, label %exit.a
-    i8 1, label %exit.b
-    i8 2, label %exit.c
+  i8 0, label %exit.a
+  i8 1, label %exit.b
+  i8 2, label %exit.c
   ]
-; CHECK:      loop.body1:
-; CHECK-NEXT:   %[[B:.*]] = load volatile i8, i8* %ptr
-; CHECK-NEXT:   switch i8 %[[B]], label %loop.body2 [
-; CHECK-NEXT:     i8 0, label %exit.a
-; CHECK-NEXT:     i8 1, label %[[LOOPEXIT:.*]]
-; CHECK-NEXT:     i8 2, label %exit.c
-; CHECK-NEXT:   ]
 
 loop.body2:
   %addr2 = load volatile i8*, i8** %addr.ptr
   indirectbr i8* %addr2, [label %loop.backedge, label %exit.c]
-; CHECK:      loop.body2:
-; CHECK-NEXT:   %[[ADDR2:.*]] = load volatile i8*, i8** %addr.ptr
-; CHECK-NEXT:   indirectbr i8* %[[ADDR2]], [label %loop.backedge, label %exit.c]
 
 loop.backedge:
   br label %loop.header
-; CHECK:      loop.backedge:
-; CHECK-NEXT:   br label %loop.header
 
 exit.a:
   ret void
 ; Check that there isn't a split loop exit.
-; CHECK-NOT:    br label %exit.a
-;
-; CHECK:      exit.a:
-; CHECK-NEXT:   ret void
 
 exit.b:
   ret void
-; CHECK:      [[LOOPEXIT]]:
-; CHECK-NEXT:   br label %exit.b
-;
-; CHECK:      exit.b:
-; CHECK-NEXT:   ret void
 
 exit.c:
   ret void
 ; Check that there isn't a split loop exit.
-; CHECK-NOT:    br label %exit.c
-;
-; CHECK:      exit.c:
-; CHECK-NEXT:   ret void
 }
diff --git a/llvm/test/Transforms/LoopStrengthReduce/AMDGPU/different-addrspace-addressing-mode-loops.ll b/llvm/test/Transforms/LoopStrengthReduce/AMDGPU/different-addrspace-addressing-mode-loops.ll
index 568809b09..79ec1dba5 100644
--- a/llvm/test/Transforms/LoopStrengthReduce/AMDGPU/different-addrspace-addressing-mode-loops.ll
+++ b/llvm/test/Transforms/LoopStrengthReduce/AMDGPU/different-addrspace-addressing-mode-loops.ll
@@ -153,4 +153,4 @@ bb:
   br i1 %exitcond, label %._crit_edge.loopexit, label %.lr.ph
 }
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="hawaii" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="hawaii" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/Transforms/LoopStrengthReduce/X86/pr17473.ll b/llvm/test/Transforms/LoopStrengthReduce/X86/pr17473.ll
index 5b7bb8846..609a1c202 100644
--- a/llvm/test/Transforms/LoopStrengthReduce/X86/pr17473.ll
+++ b/llvm/test/Transforms/LoopStrengthReduce/X86/pr17473.ll
@@ -62,6 +62,6 @@ for.end:                                          ; preds = %fn3.exit
 ; Function Attrs: nounwind optsize
 declare i32 @printf(i8* nocapture readonly, ...) #1
 
-attributes #0 = { nounwind optsize ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { nounwind optsize "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind optsize ssp uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind optsize "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind optsize }
diff --git a/llvm/test/Transforms/LoopStrengthReduce/pr18165.ll b/llvm/test/Transforms/LoopStrengthReduce/pr18165.ll
index 11c9c4ec4..02268beca 100644
--- a/llvm/test/Transforms/LoopStrengthReduce/pr18165.ll
+++ b/llvm/test/Transforms/LoopStrengthReduce/pr18165.ll
@@ -71,8 +71,8 @@ fn1.exit:                                         ; preds = %lor.end.i
 ; Function Attrs: nounwind optsize
 declare i32 @printf(i8* nocapture readonly, ...) #1
 
-attributes #0 = { nounwind optsize ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { nounwind optsize "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind optsize ssp uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind optsize "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind optsize }
 
 !llvm.ident = !{!0}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/two-combinations-bug.ll b/llvm/test/Transforms/LoopStrengthReduce/two-combinations-bug.ll
index f0b872a30..25767e901 100644
--- a/llvm/test/Transforms/LoopStrengthReduce/two-combinations-bug.ll
+++ b/llvm/test/Transforms/LoopStrengthReduce/two-combinations-bug.ll
@@ -46,7 +46,7 @@ for.body3:                                        ; preds = %for.body3, %for.bod
   br i1 %exitcond, label %for.cond.loopexit, label %for.body3
 }
 
-attributes #0 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !0 = !{!1, !2, i64 16}
 !1 = !{!"planet", !2, i64 0, !2, i64 8, !2, i64 16, !2, i64 24, !2, i64 32, !2, i64 40, !2, i64 48}
diff --git a/llvm/test/Transforms/LoopUnroll/AArch64/unroll-optsize.ll b/llvm/test/Transforms/LoopUnroll/AArch64/unroll-optsize.ll
index c0bf00b93..b0b3a5578 100644
--- a/llvm/test/Transforms/LoopUnroll/AArch64/unroll-optsize.ll
+++ b/llvm/test/Transforms/LoopUnroll/AArch64/unroll-optsize.ll
@@ -136,7 +136,7 @@ define void @fully_unrolled_bigger() #0 {
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [4 x i32], [4 x i32]* [[ARR]], i64 0, i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    store i32 [[OR]], i32* [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
-; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV]], 6
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV]], 7
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY]]
 ; CHECK:       for.cond.cleanup:
 ; CHECK-NEXT:    [[PTR:%.*]] = bitcast [4 x i32]* [[ARR]] to i32*
@@ -156,7 +156,7 @@ for.body:                                         ; preds = %for.body, %entry
   %arrayidx = getelementptr inbounds [4 x i32], [4 x i32]* %arr, i64 0, i64 %indvars.iv
   store i32 %or, i32* %arrayidx, align 4
   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
-  %exitcond = icmp eq i64 %indvars.iv, 6
+  %exitcond = icmp eq i64 %indvars.iv, 7
   br i1 %exitcond, label %for.cond.cleanup, label %for.body
 
 for.cond.cleanup:                                 ; preds = %for.cond
diff --git a/llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-for-private.ll b/llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-for-private.ll
index e2606bb69..d1fbc8760 100644
--- a/llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-for-private.ll
+++ b/llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-for-private.ll
@@ -1,4 +1,4 @@
-; RUN: opt -data-layout=A5 -mtriple=amdgcn-unknown-amdhsa -loop-unroll -S -amdgpu-unroll-threshold-private=20000 %s | FileCheck %s
+; RUN: opt -data-layout=A5 -mtriple=amdgcn-unknown-amdhsa -loop-unroll -S %s | FileCheck %s
 
 ; Check that we full unroll loop to be able to eliminate alloca
 ; CHECK-LABEL: @non_invariant_ind
diff --git a/llvm/test/Transforms/LoopUnroll/ARM/dont-unroll-loopdec.ll b/llvm/test/Transforms/LoopUnroll/ARM/dont-unroll-loopdec.ll
new file mode 100644
index 000000000..32aceb4a5
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/ARM/dont-unroll-loopdec.ll
@@ -0,0 +1,40 @@
+; RUN: opt -mtriple=thumbv8.1m.main -mattr=+mve.fp -loop-unroll -S < %s | FileCheck %s
+
+; CHECK-LABEL:  foo
+; CHECK:        5:
+; CHECK:        6:                 ; preds = %6, %5
+; CHECK:        15:                ; preds = %6
+; CHECK:          br label %16
+; CHECK:        16:                ; preds = %15, %3
+; CHECK:          ret void
+; CHECK:        }
+
+define void @foo(i8* nocapture, i8* nocapture readonly, i32) {
+  %4 = icmp sgt i32 %2, 0
+  br i1 %4, label %5, label %16
+
+; <label>:5:
+  br label %6
+
+; <label>:6:
+  %7 = phi i32 [ %13, %6 ], [ %2, %5 ]
+  %8 = phi i8* [ %10, %6 ], [ %1, %5 ]
+  %9 = phi i8* [ %12, %6 ], [ %0, %5 ]
+  %10 = getelementptr inbounds i8, i8* %8, i32 1
+  %11 = load i8, i8* %8, align 1
+  %12 = getelementptr inbounds i8, i8* %9, i32 1
+  store i8 %11, i8* %9, align 1
+
+  %13 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %7, i32 1)
+
+  %14 = icmp sgt i32 %7, 1
+  br i1 %14, label %6, label %15
+
+; <label>:15:
+  br label %16
+
+; <label>:16:
+  ret void
+}
+
+declare i32 @llvm.loop.decrement.reg.i32.i32.i32(i32, i32)
diff --git a/llvm/test/Transforms/LoopUnroll/ARM/mve-nounroll.ll b/llvm/test/Transforms/LoopUnroll/ARM/mve-nounroll.ll
new file mode 100644
index 000000000..15016c1f7
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/ARM/mve-nounroll.ll
@@ -0,0 +1,176 @@
+; RUN: opt -mtriple=thumbv8.1m.main -mattr=+mve.fp -loop-unroll -S < %s -o - | FileCheck %s
+
+; CHECK-LABEL: @loopfn
+; CHECK: vector.body:
+; CHECK:   br i1 %7, label %middle.block, label %vector.body, !llvm.loop !0
+; CHECK: middle.block:
+; CHECK:   br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader13
+; CHECK: for.body:
+; CHECK:   br i1 %exitcond, label %for.cond.cleanup.loopexit, label %for.body, !llvm.loop !2
+
+define void @loopfn(float* %s1, float* %s2, float* %d, i32 %n) {
+entry:
+  %cmp10 = icmp sgt i32 %n, 0
+  br i1 %cmp10, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:                               ; preds = %entry
+  %min.iters.check = icmp ult i32 %n, 4
+  br i1 %min.iters.check, label %for.body.preheader13, label %vector.ph
+
+for.body.preheader13:                             ; preds = %middle.block, %for.body.preheader
+  %i.011.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
+  br label %for.body
+
+vector.ph:                                        ; preds = %for.body.preheader
+  %n.vec = and i32 %n, -4
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %0 = getelementptr inbounds float, float* %s1, i32 %index
+  %1 = bitcast float* %0 to <4 x float>*
+  %wide.load = load <4 x float>, <4 x float>* %1, align 4
+  %2 = getelementptr inbounds float, float* %s2, i32 %index
+  %3 = bitcast float* %2 to <4 x float>*
+  %wide.load12 = load <4 x float>, <4 x float>* %3, align 4
+  %4 = fadd fast <4 x float> %wide.load12, %wide.load
+  %5 = getelementptr inbounds float, float* %d, i32 %index
+  %6 = bitcast float* %5 to <4 x float>*
+  store <4 x float> %4, <4 x float>* %6, align 4
+  %index.next = add i32 %index, 4
+  %7 = icmp eq i32 %index.next, %n.vec
+  br i1 %7, label %middle.block, label %vector.body, !llvm.loop !0
+
+middle.block:                                     ; preds = %vector.body
+  %cmp.n = icmp eq i32 %n.vec, %n
+  br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader13
+
+for.cond.cleanup.loopexit:                        ; preds = %for.body
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.cond.cleanup.loopexit, %middle.block, %entry
+  ret void
+
+for.body:                                         ; preds = %for.body.preheader13, %for.body
+  %i.011 = phi i32 [ %add3, %for.body ], [ %i.011.ph, %for.body.preheader13 ]
+  %arrayidx = getelementptr inbounds float, float* %s1, i32 %i.011
+  %8 = load float, float* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds float, float* %s2, i32 %i.011
+  %9 = load float, float* %arrayidx1, align 4
+  %add = fadd fast float %9, %8
+  %arrayidx2 = getelementptr inbounds float, float* %d, i32 %i.011
+  store float %add, float* %arrayidx2, align 4
+  %add3 = add nuw nsw i32 %i.011, 1
+  %exitcond = icmp eq i32 %add3, %n
+  br i1 %exitcond, label %for.cond.cleanup.loopexit, label %for.body, !llvm.loop !2
+}
+
+
+
+; CHECK-LABEL: @nested
+; CHECK: for.outer:
+; CHECK:   br label %vector.body
+; CHECK: vector.body:
+; CHECK:   br i1 %8, label %for.latch, label %vector.body, !llvm.loop !0
+; CHECK: for.latch:
+; CHECK:   br i1 %exitcond34, label %for.cond.cleanup.loopexit, label %for.outer
+
+define void @nested(float* %s1, float* %s2, float* %d, i32 %n) {
+entry:
+  %cmp31 = icmp eq i32 %n, 0
+  br i1 %cmp31, label %for.cond.cleanup, label %for.outer.preheader
+
+for.outer.preheader:                 ; preds = %entry
+  %min.iters.check = icmp ult i32 %n, 4
+  %n.vec = and i32 %n, -4
+  %cmp.n = icmp eq i32 %n.vec, %n
+  br label %for.outer
+
+for.outer:                           ; preds = %for.outer.preheader, %for.cond1.for.cond.cleanup3_crit_edge.us
+  %j.032.us = phi i32 [ %inc.us, %for.latch ], [ 0, %for.outer.preheader ]
+  %mul.us = mul i32 %j.032.us, %n
+  br label %vector.body
+
+vector.body:                                      ; preds = %for.outer, %vector.body
+  %index = phi i32 [ %index.next, %vector.body ], [ 0, %for.outer ]
+  %0 = add i32 %index, %mul.us
+  %1 = getelementptr inbounds float, float* %s1, i32 %0
+  %2 = bitcast float* %1 to <4 x float>*
+  %wide.load = load <4 x float>, <4 x float>* %2, align 4
+  %3 = getelementptr inbounds float, float* %s2, i32 %0
+  %4 = bitcast float* %3 to <4 x float>*
+  %wide.load35 = load <4 x float>, <4 x float>* %4, align 4
+  %5 = fadd fast <4 x float> %wide.load35, %wide.load
+  %6 = getelementptr inbounds float, float* %d, i32 %0
+  %7 = bitcast float* %6 to <4 x float>*
+  store <4 x float> %5, <4 x float>* %7, align 4
+  %index.next = add i32 %index, 4
+  %8 = icmp eq i32 %index.next, %n.vec
+  br i1 %8, label %for.latch, label %vector.body, !llvm.loop !0
+
+for.latch:                           ; preds = %vector.body, %for.outer
+  %i.030.us.ph = phi i32 [ %n.vec, %vector.body ]
+  %inc.us = add nuw i32 %j.032.us, 1
+  %exitcond34 = icmp eq i32 %inc.us, %n
+  br i1 %exitcond34, label %for.cond.cleanup.loopexit, label %for.outer
+
+for.cond.cleanup.loopexit:
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.cond1.for.cond.cleanup3_crit_edge.us, %entry
+  ret void
+}
+
+; Test that we don't unroll loops that only contain vector intrinsics.
+; CHECK-LABEL: test_intrinsics
+; CHECK: call <16 x i8> @llvm.arm.mve.sub
+; CHECK-NOT: call <16 x i8> @llvm.arm.mve.sub
+define dso_local arm_aapcs_vfpcc void @test_intrinsics(i8* noalias nocapture readonly %a, i8* noalias nocapture readonly %b, i8* noalias nocapture %c, i32 %N) {
+entry:
+  %cmp8 = icmp eq i32 %N, 0
+  %tmp8 = add i32 %N, 15
+  %tmp9 = lshr i32 %tmp8, 4
+  %tmp10 = shl nuw i32 %tmp9, 4
+  %tmp11 = add i32 %tmp10, -16
+  %tmp12 = lshr i32 %tmp11, 4
+  %tmp13 = add nuw nsw i32 %tmp12, 1
+  br i1 %cmp8, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:
+  br label %vector.body
+
+vector.body:
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %tmp14 = phi i32 [ %tmp13, %vector.ph ], [ %tmp15, %vector.body ]
+  %0 = phi i32 [ %N, %vector.ph ], [ %2, %vector.body ]
+  %tmp = getelementptr inbounds i8, i8* %a, i32 %index
+  %1 = call <16 x i1> @llvm.arm.mve.vctp8(i32 %0)
+  %2 = sub i32 %0, 16
+  %tmp2 = bitcast i8* %tmp to <16 x i8>*
+  %wide.masked.load = tail call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %tmp2, i32 4, <16 x i1> %1, <16 x i8> undef)
+  %tmp3 = getelementptr inbounds i8, i8* %b, i32 %index
+  %tmp4 = bitcast i8* %tmp3 to <16 x i8>*
+  %wide.masked.load2 = tail call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %tmp4, i32 4, <16 x i1> %1, <16 x i8> undef)
+  %sub = call <16 x i8> @llvm.arm.mve.sub.predicated.v16i8.v16i1(<16 x i8> %wide.masked.load2, <16 x i8> %wide.masked.load, <16 x i1> %1, <16 x i8> undef)
+  %tmp6 = getelementptr inbounds i8, i8* %c, i32 %index
+  %tmp7 = bitcast i8* %tmp6 to <16 x i8>*
+  tail call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %sub, <16 x i8>* %tmp7, i32 4, <16 x i1> %1)
+  %index.next = add i32 %index, 16
+  %tmp15 = sub i32 %tmp14, 1
+  %tmp16 = icmp ne i32 %tmp15, 0
+  br i1 %tmp16, label %vector.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %vector.body, %entry
+  ret void
+}
+
+declare <16 x i1> @llvm.arm.mve.vctp8(i32)
+declare <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>*, i32, <16 x i1>, <16 x i8>)
+declare <16 x i8> @llvm.arm.mve.sub.predicated.v16i8.v16i1(<16 x i8>, <16 x i8>, <16 x i1>, <16 x i8>)
+declare void @llvm.masked.store.v16i8.p0v16i8(<16 x i8>, <16 x i8>*, i32, <16 x i1>)
+
+
+!0 = distinct !{!0, !1}
+!1 = !{!"llvm.loop.isvectorized", i32 1}
+!2 = distinct !{!2, !3, !1}
+!3 = !{!"llvm.loop.unroll.runtime.disable"}
diff --git a/llvm/test/Transforms/LoopUnroll/disable-full-unroll-by-opt.ll b/llvm/test/Transforms/LoopUnroll/disable-full-unroll-by-opt.ll
new file mode 100644
index 000000000..c89948ad5
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/disable-full-unroll-by-opt.ll
@@ -0,0 +1,35 @@
+; Default behavior
+; RUN: opt < %s -passes='unroll' -S | FileCheck %s -check-prefixes=ENABLE,COMMON
+
+; Pass option
+; RUN: opt < %s -passes='unroll<full-unroll-max=0>'  -S | FileCheck %s -check-prefixes=DISABLE,COMMON
+; RUN: opt < %s -passes='unroll<full-unroll-max=30>' -S | FileCheck %s -check-prefixes=DISABLE,COMMON
+; RUN: opt < %s -passes='unroll<full-unroll-max=36>' -S | FileCheck %s -check-prefixes=ENABLE,COMMON
+
+; cl::opt option
+; RUN: opt < %s -passes='unroll' -unroll-full-max-count=0 -S | FileCheck %s -check-prefixes=DISABLE,COMMON
+; RUN: opt < %s -passes='unroll' -unroll-full-max-count=30 -S | FileCheck %s -check-prefixes=DISABLE,COMMON
+; RUN: opt < %s -passes='unroll' -unroll-full-max-count=36 -S | FileCheck %s -check-prefixes=ENABLE,COMMON
+
+; Pass option has a priority over cl::opt
+; RUN: opt < %s -passes='unroll<full-unroll-max=30>' -unroll-full-max-count=36 -S | FileCheck %s -check-prefixes=DISABLE,COMMON
+; RUN: opt < %s -passes='unroll<full-unroll-max=36>' -unroll-full-max-count=30 -S | FileCheck %s -check-prefixes=ENABLE,COMMON
+
+define void @test() {
+; COMMON-LABEL: @test(
+ entry:
+  br label %loop
+
+ loop:
+  %idx = phi i32 [ 0, %entry ], [ %idx.inc, %loop ]
+  %idx.inc = add i32 %idx, 1
+  %be = icmp slt i32 %idx, 32
+  br i1 %be, label %loop, label %exit
+
+; COMMON: loop:
+; DISABLE:  %be = icmp slt i32 %idx, 32
+; ENABLE-NOT:  %be = icmp slt i32 %idx, 32
+
+ exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopUnroll/optsize-loop-size.ll b/llvm/test/Transforms/LoopUnroll/optsize-loop-size.ll
new file mode 100644
index 000000000..4ca397a1c
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/optsize-loop-size.ll
@@ -0,0 +1,58 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -loop-unroll -S < %s | FileCheck %s
+
+define i32 @test(i32 %a, i32 %b, i32 %c) optsize {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[REF_TMP:%.*]] = alloca [3 x i32], align 4
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast [3 x i32]* [[REF_TMP]] to i8*
+; CHECK-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [3 x i32], [3 x i32]* [[REF_TMP]], i64 0, i64 0
+; CHECK-NEXT:    store i32 [[A:%.*]], i32* [[ARRAYINIT_BEGIN]], align 4
+; CHECK-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [3 x i32], [3 x i32]* [[REF_TMP]], i64 0, i64 1
+; CHECK-NEXT:    store i32 [[B:%.*]], i32* [[ARRAYINIT_ELEMENT]], align 4
+; CHECK-NEXT:    [[ARRAYINIT_ELEMENT1:%.*]] = getelementptr inbounds [3 x i32], [3 x i32]* [[REF_TMP]], i64 0, i64 2
+; CHECK-NEXT:    store i32 [[C:%.*]], i32* [[ARRAYINIT_ELEMENT1]], align 4
+; CHECK-NEXT:    [[ADD_PTR_I_I:%.*]] = getelementptr inbounds [3 x i32], [3 x i32]* [[REF_TMP]], i64 0, i64 3
+; CHECK-NEXT:    [[CMP_I_I_I3:%.*]] = icmp slt i32 [[A]], [[B]]
+; CHECK-NEXT:    [[SPEC_SELECT_I_I4:%.*]] = select i1 [[CMP_I_I_I3]], i32* [[ARRAYINIT_ELEMENT]], i32* [[ARRAYINIT_BEGIN]]
+; CHECK-NEXT:    [[INCDEC_PTR_I_I5:%.*]] = getelementptr inbounds [3 x i32], [3 x i32]* [[REF_TMP]], i64 0, i64 2
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[DOTPRE:%.*]] = load i32, i32* [[SPEC_SELECT_I_I4]], align 4
+; CHECK-NEXT:    [[DOTPRE2:%.*]] = load i32, i32* [[INCDEC_PTR_I_I5]], align 4
+; CHECK-NEXT:    [[CMP_I_I_I:%.*]] = icmp slt i32 [[DOTPRE]], [[DOTPRE2]]
+; CHECK-NEXT:    [[SPEC_SELECT_I_I:%.*]] = select i1 [[CMP_I_I_I]], i32* [[INCDEC_PTR_I_I5]], i32* [[SPEC_SELECT_I_I4]]
+; CHECK-NEXT:    [[INCDEC_PTR_I_I:%.*]] = getelementptr inbounds i32, i32* [[INCDEC_PTR_I_I5]], i64 1
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[SPEC_SELECT_I_I]], align 4
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+entry:
+  %ref.tmp = alloca [3 x i32], align 4
+  %0 = bitcast [3 x i32]* %ref.tmp to i8*
+  %arrayinit.begin = getelementptr inbounds [3 x i32], [3 x i32]* %ref.tmp, i64 0, i64 0
+  store i32 %a, i32* %arrayinit.begin, align 4
+  %arrayinit.element = getelementptr inbounds [3 x i32], [3 x i32]* %ref.tmp, i64 0, i64 1
+  store i32 %b, i32* %arrayinit.element, align 4
+  %arrayinit.element1 = getelementptr inbounds [3 x i32], [3 x i32]* %ref.tmp, i64 0, i64 2
+  store i32 %c, i32* %arrayinit.element1, align 4
+  %add.ptr.i.i = getelementptr inbounds [3 x i32], [3 x i32]* %ref.tmp, i64 0, i64 3
+  %cmp.i.i.i3 = icmp slt i32 %a, %b
+  %spec.select.i.i4 = select i1 %cmp.i.i.i3, i32* %arrayinit.element, i32* %arrayinit.begin
+  %incdec.ptr.i.i5 = getelementptr inbounds [3 x i32], [3 x i32]* %ref.tmp, i64 0, i64 2
+  br label %loop
+
+loop:          ; preds = %entry, %loop
+  %incdec.ptr.i.i7 = phi i32* [ %incdec.ptr.i.i5, %entry ], [ %incdec.ptr.i.i, %loop ]
+  %spec.select.i.i6 = phi i32* [ %spec.select.i.i4, %entry ], [ %spec.select.i.i, %loop ]
+  %.pre = load i32, i32* %spec.select.i.i6, align 4
+  %.pre2 = load i32, i32* %incdec.ptr.i.i7, align 4
+  %cmp.i.i.i = icmp slt i32 %.pre, %.pre2
+  %spec.select.i.i = select i1 %cmp.i.i.i, i32* %incdec.ptr.i.i7, i32* %spec.select.i.i6
+  %incdec.ptr.i.i = getelementptr inbounds i32, i32* %incdec.ptr.i.i7, i64 1
+  %cmp1.i.i = icmp eq i32* %incdec.ptr.i.i, %add.ptr.i.i
+  br i1 %cmp1.i.i, label %exit, label %loop
+
+exit:           ; preds = %loop
+  %1 = load i32, i32* %spec.select.i.i, align 4
+  ret i32 %1
+}
diff --git a/llvm/test/Transforms/LoopUnroll/peel-loop-conditions-pgo-1.ll b/llvm/test/Transforms/LoopUnroll/peel-loop-conditions-pgo-1.ll
new file mode 100644
index 000000000..0c56229d0
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/peel-loop-conditions-pgo-1.ll
@@ -0,0 +1,43 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -S -loop-unroll -loop-unroll -verify-dom-info -debug-only=loop-unroll -unroll-peel-max-count=7 2>&1 | FileCheck %s
+; REQUIRES: asserts
+
+declare void @f1()
+declare void @f2()
+
+; Check that we can peel off iterations that make conditions true.
+; The second invocation of loop-unroll will do profile based peeling of
+; remained iterations.
+define void @test1(i32 %k) !prof !4 {
+; CHECK: Loop Unroll: F[test1] Loop %for.body
+; CHECK: PEELING loop %for.body with iteration count 2!
+; CHECK: PEELING loop %for.body with iteration count 4!
+; CHECK: llvm.loop.unroll.disable
+for.body.lr.ph:
+  br label %for.body
+
+for.body:
+  %i.05 = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %for.inc ]
+  %cmp1 = icmp ult i32 %i.05, 2
+  br i1 %cmp1, label %if.then, label %if.else
+
+if.then:
+  call void @f1()
+  br label %for.inc
+
+if.else:
+  call void @f2()
+  br label %for.inc
+
+for.inc:
+  %inc = add nsw i32 %i.05, 1
+  %cmp = icmp slt i32 %inc, %k
+  br i1 %cmp, label %for.body, label %for.end, !llvm.loop !1, !prof !2
+
+for.end:
+  ret void
+}
+
+!1 = distinct !{!1}
+!2 = !{!"branch_weights", i32 6, i32 1}
+!4 = !{!"function_entry_count", i64 1}
diff --git a/llvm/test/Transforms/LoopUnroll/peel-loop-conditions-pgo-2.ll b/llvm/test/Transforms/LoopUnroll/peel-loop-conditions-pgo-2.ll
new file mode 100644
index 000000000..6462283df
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/peel-loop-conditions-pgo-2.ll
@@ -0,0 +1,43 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -S -loop-unroll -loop-unroll -verify-dom-info -debug-only=loop-unroll -unroll-peel-max-count=7 2>&1 | FileCheck %s
+; REQUIRES: asserts
+
+declare void @f1()
+declare void @f2()
+
+; Check that we can peel off iterations that make conditions true.
+; The second invocation of loop-unroll will NOT do profile based peeling of
+; remained iterations because the total number of peeled iterations exceeds
+; threashold specified with -unroll-peel-max-count=7.
+define void @test2(i32 %k) !prof !4 {
+; CHECK: Loop Unroll: F[test2] Loop %for.body
+; CHECK: PEELING loop %for.body with iteration count 2!
+; CHECK-NOT: llvm.loop.unroll.disable
+for.body.lr.ph:
+  br label %for.body
+
+for.body:
+  %i.05 = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %for.inc ]
+  %cmp1 = icmp ult i32 %i.05, 2
+  br i1 %cmp1, label %if.then, label %if.else
+
+if.then:
+  call void @f1()
+  br label %for.inc
+
+if.else:
+  call void @f2()
+  br label %for.inc
+
+for.inc:
+  %inc = add nsw i32 %i.05, 1
+  %cmp = icmp slt i32 %inc, %k
+  br i1 %cmp, label %for.body, label %for.end, !llvm.loop !1, !prof !3
+
+for.end:
+  ret void
+}
+
+!1 = distinct !{!1}
+!3 = !{!"branch_weights", i32 8, i32 1}
+!4 = !{!"function_entry_count", i64 1}
diff --git a/llvm/test/Transforms/LoopUnroll/peel-loop-conditions.ll b/llvm/test/Transforms/LoopUnroll/peel-loop-conditions.ll
index a9937d9db..5c84884c6 100644
--- a/llvm/test/Transforms/LoopUnroll/peel-loop-conditions.ll
+++ b/llvm/test/Transforms/LoopUnroll/peel-loop-conditions.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -S -loop-unroll -verify-dom-info | FileCheck %s
+; RUN: opt < %s -S -loop-unroll -unroll-peel-max-count=4 -verify-dom-info | FileCheck %s
 
 declare void @f1()
 declare void @f2()
@@ -522,22 +522,75 @@ for.end:
 }
 
 define void @test7(i32 %k) {
-; FIXME: Could simplify loop body by peeling one additional iteration after
-;        i != 3 becomes false
 ; CHECK-LABEL: @test7(
 ; CHECK-NEXT:  for.body.lr.ph:
+; CHECK-NEXT:    br label [[FOR_BODY_PEEL_BEGIN:%.*]]
+; CHECK:       for.body.peel.begin:
+; CHECK-NEXT:    br label [[FOR_BODY_PEEL:%.*]]
+; CHECK:       for.body.peel:
+; CHECK-NEXT:    [[CMP1_PEEL:%.*]] = icmp ne i32 0, 3
+; CHECK-NEXT:    br i1 [[CMP1_PEEL]], label [[IF_THEN_PEEL:%.*]], label [[FOR_INC_PEEL:%.*]]
+; CHECK:       if.then.peel:
+; CHECK-NEXT:    call void @f1()
+; CHECK-NEXT:    br label [[FOR_INC_PEEL]]
+; CHECK:       for.inc.peel:
+; CHECK-NEXT:    [[INC_PEEL:%.*]] = add nsw i32 0, 1
+; CHECK-NEXT:    [[CMP_PEEL:%.*]] = icmp slt i32 [[INC_PEEL]], [[K:%.*]]
+; CHECK-NEXT:    br i1 [[CMP_PEEL]], label [[FOR_BODY_PEEL_NEXT:%.*]], label [[FOR_END:%.*]]
+; CHECK:       for.body.peel.next:
+; CHECK-NEXT:    br label [[FOR_BODY_PEEL2:%.*]]
+; CHECK:       for.body.peel2:
+; CHECK-NEXT:    [[CMP1_PEEL3:%.*]] = icmp ne i32 [[INC_PEEL]], 3
+; CHECK-NEXT:    br i1 [[CMP1_PEEL3]], label [[IF_THEN_PEEL4:%.*]], label [[FOR_INC_PEEL5:%.*]]
+; CHECK:       if.then.peel4:
+; CHECK-NEXT:    call void @f1()
+; CHECK-NEXT:    br label [[FOR_INC_PEEL5]]
+; CHECK:       for.inc.peel5:
+; CHECK-NEXT:    [[INC_PEEL6:%.*]] = add nsw i32 [[INC_PEEL]], 1
+; CHECK-NEXT:    [[CMP_PEEL7:%.*]] = icmp slt i32 [[INC_PEEL6]], [[K]]
+; CHECK-NEXT:    br i1 [[CMP_PEEL7]], label [[FOR_BODY_PEEL_NEXT1:%.*]], label [[FOR_END]]
+; CHECK:       for.body.peel.next1:
+; CHECK-NEXT:    br label [[FOR_BODY_PEEL9:%.*]]
+; CHECK:       for.body.peel9:
+; CHECK-NEXT:    [[CMP1_PEEL10:%.*]] = icmp ne i32 [[INC_PEEL6]], 3
+; CHECK-NEXT:    br i1 [[CMP1_PEEL10]], label [[IF_THEN_PEEL11:%.*]], label [[FOR_INC_PEEL12:%.*]]
+; CHECK:       if.then.peel11:
+; CHECK-NEXT:    call void @f1()
+; CHECK-NEXT:    br label [[FOR_INC_PEEL12]]
+; CHECK:       for.inc.peel12:
+; CHECK-NEXT:    [[INC_PEEL13:%.*]] = add nsw i32 [[INC_PEEL6]], 1
+; CHECK-NEXT:    [[CMP_PEEL14:%.*]] = icmp slt i32 [[INC_PEEL13]], [[K]]
+; CHECK-NEXT:    br i1 [[CMP_PEEL14]], label [[FOR_BODY_PEEL_NEXT8:%.*]], label [[FOR_END]]
+; CHECK:       for.body.peel.next8:
+; CHECK-NEXT:    br label [[FOR_BODY_PEEL16:%.*]]
+; CHECK:       for.body.peel16:
+; CHECK-NEXT:    [[CMP1_PEEL17:%.*]] = icmp ne i32 [[INC_PEEL13]], 3
+; CHECK-NEXT:    br i1 [[CMP1_PEEL17]], label [[IF_THEN_PEEL18:%.*]], label [[FOR_INC_PEEL19:%.*]]
+; CHECK:       if.then.peel18:
+; CHECK-NEXT:    call void @f1()
+; CHECK-NEXT:    br label [[FOR_INC_PEEL19]]
+; CHECK:       for.inc.peel19:
+; CHECK-NEXT:    [[INC_PEEL20:%.*]] = add nsw i32 [[INC_PEEL13]], 1
+; CHECK-NEXT:    [[CMP_PEEL21:%.*]] = icmp slt i32 [[INC_PEEL20]], [[K]]
+; CHECK-NEXT:    br i1 [[CMP_PEEL21]], label [[FOR_BODY_PEEL_NEXT15:%.*]], label [[FOR_END]]
+; CHECK:       for.body.peel.next15:
+; CHECK-NEXT:    br label [[FOR_BODY_PEEL_NEXT22:%.*]]
+; CHECK:       for.body.peel.next22:
+; CHECK-NEXT:    br label [[FOR_BODY_LR_PH_PEEL_NEWPH:%.*]]
+; CHECK:       for.body.lr.ph.peel.newph:
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[I_05:%.*]] = phi i32 [ 0, [[FOR_BODY_LR_PH:%.*]] ], [ [[INC:%.*]], [[FOR_INC:%.*]] ]
-; CHECK-NEXT:    [[CMP1:%.*]] = icmp ne i32 [[I_05]], 3
-; CHECK-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[FOR_INC]]
+; CHECK-NEXT:    [[I_05:%.*]] = phi i32 [ [[INC_PEEL20]], [[FOR_BODY_LR_PH_PEEL_NEWPH]] ], [ [[INC:%.*]], [[FOR_INC:%.*]] ]
+; CHECK-NEXT:    br i1 true, label [[IF_THEN:%.*]], label [[FOR_INC]]
 ; CHECK:       if.then:
 ; CHECK-NEXT:    call void @f1()
 ; CHECK-NEXT:    br label [[FOR_INC]]
 ; CHECK:       for.inc:
-; CHECK-NEXT:    [[INC]] = add nsw i32 [[I_05]], 1
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[INC]], [[K:%.*]]
-; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END:%.*]]
+; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_05]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[INC]], [[K]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END_LOOPEXIT:%.*]], !llvm.loop !6
+; CHECK:       for.end.loopexit:
+; CHECK-NEXT:    br label [[FOR_END]]
 ; CHECK:       for.end:
 ; CHECK-NEXT:    ret void
 ;
@@ -563,22 +616,75 @@ for.end:
 }
 
 define void @test8(i32 %k) {
-; FIXME: Could simplify loop body by peeling one additional iteration after
-;        i == 3 becomes true.
 ; CHECK-LABEL: @test8(
 ; CHECK-NEXT:  for.body.lr.ph:
+; CHECK-NEXT:    br label [[FOR_BODY_PEEL_BEGIN:%.*]]
+; CHECK:       for.body.peel.begin:
+; CHECK-NEXT:    br label [[FOR_BODY_PEEL:%.*]]
+; CHECK:       for.body.peel:
+; CHECK-NEXT:    [[CMP1_PEEL:%.*]] = icmp eq i32 0, 3
+; CHECK-NEXT:    br i1 [[CMP1_PEEL]], label [[IF_THEN_PEEL:%.*]], label [[FOR_INC_PEEL:%.*]]
+; CHECK:       if.then.peel:
+; CHECK-NEXT:    call void @f1()
+; CHECK-NEXT:    br label [[FOR_INC_PEEL]]
+; CHECK:       for.inc.peel:
+; CHECK-NEXT:    [[INC_PEEL:%.*]] = add nsw i32 0, 1
+; CHECK-NEXT:    [[CMP_PEEL:%.*]] = icmp slt i32 [[INC_PEEL]], [[K:%.*]]
+; CHECK-NEXT:    br i1 [[CMP_PEEL]], label [[FOR_BODY_PEEL_NEXT:%.*]], label [[FOR_END:%.*]]
+; CHECK:       for.body.peel.next:
+; CHECK-NEXT:    br label [[FOR_BODY_PEEL2:%.*]]
+; CHECK:       for.body.peel2:
+; CHECK-NEXT:    [[CMP1_PEEL3:%.*]] = icmp eq i32 [[INC_PEEL]], 3
+; CHECK-NEXT:    br i1 [[CMP1_PEEL3]], label [[IF_THEN_PEEL4:%.*]], label [[FOR_INC_PEEL5:%.*]]
+; CHECK:       if.then.peel4:
+; CHECK-NEXT:    call void @f1()
+; CHECK-NEXT:    br label [[FOR_INC_PEEL5]]
+; CHECK:       for.inc.peel5:
+; CHECK-NEXT:    [[INC_PEEL6:%.*]] = add nsw i32 [[INC_PEEL]], 1
+; CHECK-NEXT:    [[CMP_PEEL7:%.*]] = icmp slt i32 [[INC_PEEL6]], [[K]]
+; CHECK-NEXT:    br i1 [[CMP_PEEL7]], label [[FOR_BODY_PEEL_NEXT1:%.*]], label [[FOR_END]]
+; CHECK:       for.body.peel.next1:
+; CHECK-NEXT:    br label [[FOR_BODY_PEEL9:%.*]]
+; CHECK:       for.body.peel9:
+; CHECK-NEXT:    [[CMP1_PEEL10:%.*]] = icmp eq i32 [[INC_PEEL6]], 3
+; CHECK-NEXT:    br i1 [[CMP1_PEEL10]], label [[IF_THEN_PEEL11:%.*]], label [[FOR_INC_PEEL12:%.*]]
+; CHECK:       if.then.peel11:
+; CHECK-NEXT:    call void @f1()
+; CHECK-NEXT:    br label [[FOR_INC_PEEL12]]
+; CHECK:       for.inc.peel12:
+; CHECK-NEXT:    [[INC_PEEL13:%.*]] = add nsw i32 [[INC_PEEL6]], 1
+; CHECK-NEXT:    [[CMP_PEEL14:%.*]] = icmp slt i32 [[INC_PEEL13]], [[K]]
+; CHECK-NEXT:    br i1 [[CMP_PEEL14]], label [[FOR_BODY_PEEL_NEXT8:%.*]], label [[FOR_END]]
+; CHECK:       for.body.peel.next8:
+; CHECK-NEXT:    br label [[FOR_BODY_PEEL16:%.*]]
+; CHECK:       for.body.peel16:
+; CHECK-NEXT:    [[CMP1_PEEL17:%.*]] = icmp eq i32 [[INC_PEEL13]], 3
+; CHECK-NEXT:    br i1 [[CMP1_PEEL17]], label [[IF_THEN_PEEL18:%.*]], label [[FOR_INC_PEEL19:%.*]]
+; CHECK:       if.then.peel18:
+; CHECK-NEXT:    call void @f1()
+; CHECK-NEXT:    br label [[FOR_INC_PEEL19]]
+; CHECK:       for.inc.peel19:
+; CHECK-NEXT:    [[INC_PEEL20:%.*]] = add nsw i32 [[INC_PEEL13]], 1
+; CHECK-NEXT:    [[CMP_PEEL21:%.*]] = icmp slt i32 [[INC_PEEL20]], [[K]]
+; CHECK-NEXT:    br i1 [[CMP_PEEL21]], label [[FOR_BODY_PEEL_NEXT15:%.*]], label [[FOR_END]]
+; CHECK:       for.body.peel.next15:
+; CHECK-NEXT:    br label [[FOR_BODY_PEEL_NEXT22:%.*]]
+; CHECK:       for.body.peel.next22:
+; CHECK-NEXT:    br label [[FOR_BODY_LR_PH_PEEL_NEWPH:%.*]]
+; CHECK:       for.body.lr.ph.peel.newph:
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[I_05:%.*]] = phi i32 [ 0, [[FOR_BODY_LR_PH:%.*]] ], [ [[INC:%.*]], [[FOR_INC:%.*]] ]
-; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i32 [[I_05]], 3
-; CHECK-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[FOR_INC]]
+; CHECK-NEXT:    [[I_05:%.*]] = phi i32 [ [[INC_PEEL20]], [[FOR_BODY_LR_PH_PEEL_NEWPH]] ], [ [[INC:%.*]], [[FOR_INC:%.*]] ]
+; CHECK-NEXT:    br i1 false, label [[IF_THEN:%.*]], label [[FOR_INC]]
 ; CHECK:       if.then:
 ; CHECK-NEXT:    call void @f1()
 ; CHECK-NEXT:    br label [[FOR_INC]]
 ; CHECK:       for.inc:
-; CHECK-NEXT:    [[INC]] = add nsw i32 [[I_05]], 1
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[INC]], [[K:%.*]]
-; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END:%.*]]
+; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_05]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[INC]], [[K]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END_LOOPEXIT:%.*]], !llvm.loop !7
+; CHECK:       for.end.loopexit:
+; CHECK-NEXT:    br label [[FOR_END]]
 ; CHECK:       for.end:
 ; CHECK-NEXT:    ret void
 ;
@@ -643,3 +749,461 @@ for.inc:
 for.end:
   ret void
 }
+; CHECK-NOT: llvm.loop.unroll.disable
+
+define void @test_10__peel_first_iter_via_slt_pred(i32 %len) {
+; CHECK-LABEL: @test_10__peel_first_iter_via_slt_pred(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP5:%.*]] = icmp sgt i32 [[LEN:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP5]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
+; CHECK:       for.body.preheader:
+; CHECK-NEXT:    br label [[FOR_BODY_PEEL_BEGIN:%.*]]
+; CHECK:       for.body.peel.begin:
+; CHECK-NEXT:    br label [[FOR_BODY_PEEL:%.*]]
+; CHECK:       for.body.peel:
+; CHECK-NEXT:    [[CMP1_PEEL:%.*]] = icmp slt i32 0, 1
+; CHECK-NEXT:    br i1 [[CMP1_PEEL]], label [[IF_THEN_PEEL:%.*]], label [[IF_END_PEEL:%.*]]
+; CHECK:       if.then.peel:
+; CHECK-NEXT:    call void @init()
+; CHECK-NEXT:    br label [[IF_END_PEEL]]
+; CHECK:       if.end.peel:
+; CHECK-NEXT:    call void @sink()
+; CHECK-NEXT:    [[INC_PEEL:%.*]] = add nuw nsw i32 0, 1
+; CHECK-NEXT:    [[EXITCOND_PEEL:%.*]] = icmp eq i32 [[INC_PEEL]], [[LEN]]
+; CHECK-NEXT:    br i1 [[EXITCOND_PEEL]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[FOR_BODY_PEEL_NEXT:%.*]]
+; CHECK:       for.body.peel.next:
+; CHECK-NEXT:    br label [[FOR_BODY_PEEL_NEXT1:%.*]]
+; CHECK:       for.body.peel.next1:
+; CHECK-NEXT:    br label [[FOR_BODY_PREHEADER_PEEL_NEWPH:%.*]]
+; CHECK:       for.body.preheader.peel.newph:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.cond.cleanup.loopexit.loopexit:
+; CHECK-NEXT:    br label [[FOR_COND_CLEANUP_LOOPEXIT]]
+; CHECK:       for.cond.cleanup.loopexit:
+; CHECK-NEXT:    br label [[FOR_COND_CLEANUP]]
+; CHECK:       for.cond.cleanup:
+; CHECK-NEXT:    ret void
+; CHECK:       for.body:
+; CHECK-NEXT:    [[I_06:%.*]] = phi i32 [ [[INC:%.*]], [[IF_END:%.*]] ], [ [[INC_PEEL]], [[FOR_BODY_PREHEADER_PEEL_NEWPH]] ]
+; CHECK-NEXT:    br i1 false, label [[IF_THEN:%.*]], label [[IF_END]]
+; CHECK:       if.then:
+; CHECK-NEXT:    call void @init()
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    call void @sink()
+; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_06]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[LEN]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT:%.*]], label [[FOR_BODY]], !llvm.loop !8
+;
+entry:
+  %cmp5 = icmp sgt i32 %len, 0
+  br i1 %cmp5, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %if.end, %entry
+  ret void
+
+for.body:                                         ; preds = %entry, %if.end
+  %i.06 = phi i32 [ %inc, %if.end ], [ 0, %entry ]
+  %cmp1 = icmp slt i32 %i.06, 1
+  br i1 %cmp1, label %if.then, label %if.end
+
+if.then:                                          ; preds = %for.body
+  call void @init()
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %for.body
+  call void @sink()
+  %inc = add nuw nsw i32 %i.06, 1
+  %exitcond = icmp eq i32 %inc, %len
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+define void @test_11__peel_first_iter_via_sgt_pred(i32 %len) {
+; CHECK-LABEL: @test_11__peel_first_iter_via_sgt_pred(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP5:%.*]] = icmp sgt i32 [[LEN:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP5]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
+; CHECK:       for.body.preheader:
+; CHECK-NEXT:    br label [[FOR_BODY_PEEL_BEGIN:%.*]]
+; CHECK:       for.body.peel.begin:
+; CHECK-NEXT:    br label [[FOR_BODY_PEEL:%.*]]
+; CHECK:       for.body.peel:
+; CHECK-NEXT:    [[CMP1_PEEL:%.*]] = icmp sgt i32 0, 0
+; CHECK-NEXT:    br i1 [[CMP1_PEEL]], label [[IF_END_PEEL:%.*]], label [[IF_THEN_PEEL:%.*]]
+; CHECK:       if.then.peel:
+; CHECK-NEXT:    call void @init()
+; CHECK-NEXT:    br label [[IF_END_PEEL]]
+; CHECK:       if.end.peel:
+; CHECK-NEXT:    call void @sink()
+; CHECK-NEXT:    [[INC_PEEL:%.*]] = add nuw nsw i32 0, 1
+; CHECK-NEXT:    [[EXITCOND_PEEL:%.*]] = icmp eq i32 [[INC_PEEL]], [[LEN]]
+; CHECK-NEXT:    br i1 [[EXITCOND_PEEL]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[FOR_BODY_PEEL_NEXT:%.*]]
+; CHECK:       for.body.peel.next:
+; CHECK-NEXT:    br label [[FOR_BODY_PEEL_NEXT1:%.*]]
+; CHECK:       for.body.peel.next1:
+; CHECK-NEXT:    br label [[FOR_BODY_PREHEADER_PEEL_NEWPH:%.*]]
+; CHECK:       for.body.preheader.peel.newph:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.cond.cleanup.loopexit.loopexit:
+; CHECK-NEXT:    br label [[FOR_COND_CLEANUP_LOOPEXIT]]
+; CHECK:       for.cond.cleanup.loopexit:
+; CHECK-NEXT:    br label [[FOR_COND_CLEANUP]]
+; CHECK:       for.cond.cleanup:
+; CHECK-NEXT:    ret void
+; CHECK:       for.body:
+; CHECK-NEXT:    [[I_06:%.*]] = phi i32 [ [[INC:%.*]], [[IF_END:%.*]] ], [ [[INC_PEEL]], [[FOR_BODY_PREHEADER_PEEL_NEWPH]] ]
+; CHECK-NEXT:    br i1 true, label [[IF_END]], label [[IF_THEN:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    call void @init()
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    call void @sink()
+; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_06]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[LEN]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT:%.*]], label [[FOR_BODY]], !llvm.loop !10
+;
+entry:
+  %cmp5 = icmp sgt i32 %len, 0
+  br i1 %cmp5, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %if.end, %entry
+  ret void
+
+for.body:                                         ; preds = %entry, %if.end
+  %i.06 = phi i32 [ %inc, %if.end ], [ 0, %entry ]
+  %cmp1 = icmp sgt i32 %i.06, 0
+  br i1 %cmp1, label %if.end, label %if.then
+
+if.then:                                          ; preds = %for.body
+  call void @init()
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %for.body
+  call void @sink()
+  %inc = add nuw nsw i32 %i.06, 1
+  %exitcond = icmp eq i32 %inc, %len
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+; NOTE: here we should only peel the first iteration,
+;       i.e. all calls to sink() must stay in loop.
+define void @test12__peel_first_iter_via_eq_pred(i32 %len) {
+; CHECK-LABEL: @test12__peel_first_iter_via_eq_pred(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP5:%.*]] = icmp sgt i32 [[LEN:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP5]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
+; CHECK:       for.body.preheader:
+; CHECK-NEXT:    br label [[FOR_BODY_PEEL_BEGIN:%.*]]
+; CHECK:       for.body.peel.begin:
+; CHECK-NEXT:    br label [[FOR_BODY_PEEL:%.*]]
+; CHECK:       for.body.peel:
+; CHECK-NEXT:    [[CMP1_PEEL:%.*]] = icmp eq i32 0, 0
+; CHECK-NEXT:    br i1 [[CMP1_PEEL]], label [[IF_THEN_PEEL:%.*]], label [[IF_END_PEEL:%.*]]
+; CHECK:       if.then.peel:
+; CHECK-NEXT:    call void @init()
+; CHECK-NEXT:    br label [[IF_END_PEEL]]
+; CHECK:       if.end.peel:
+; CHECK-NEXT:    call void @sink()
+; CHECK-NEXT:    [[INC_PEEL:%.*]] = add nuw nsw i32 0, 1
+; CHECK-NEXT:    [[EXITCOND_PEEL:%.*]] = icmp eq i32 [[INC_PEEL]], [[LEN]]
+; CHECK-NEXT:    br i1 [[EXITCOND_PEEL]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[FOR_BODY_PEEL_NEXT:%.*]]
+; CHECK:       for.body.peel.next:
+; CHECK-NEXT:    br label [[FOR_BODY_PEEL_NEXT1:%.*]]
+; CHECK:       for.body.peel.next1:
+; CHECK-NEXT:    br label [[FOR_BODY_PREHEADER_PEEL_NEWPH:%.*]]
+; CHECK:       for.body.preheader.peel.newph:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.cond.cleanup.loopexit.loopexit:
+; CHECK-NEXT:    br label [[FOR_COND_CLEANUP_LOOPEXIT]]
+; CHECK:       for.cond.cleanup.loopexit:
+; CHECK-NEXT:    br label [[FOR_COND_CLEANUP]]
+; CHECK:       for.cond.cleanup:
+; CHECK-NEXT:    ret void
+; CHECK:       for.body:
+; CHECK-NEXT:    [[I_06:%.*]] = phi i32 [ [[INC:%.*]], [[IF_END:%.*]] ], [ [[INC_PEEL]], [[FOR_BODY_PREHEADER_PEEL_NEWPH]] ]
+; CHECK-NEXT:    br i1 false, label [[IF_THEN:%.*]], label [[IF_END]]
+; CHECK:       if.then:
+; CHECK-NEXT:    call void @init()
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    call void @sink()
+; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_06]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[LEN]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT:%.*]], label [[FOR_BODY]], !llvm.loop !11
+;
+entry:
+  %cmp5 = icmp sgt i32 %len, 0
+  br i1 %cmp5, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %if.end, %entry
+  ret void
+
+for.body:                                         ; preds = %entry, %if.end
+  %i.06 = phi i32 [ %inc, %if.end ], [ 0, %entry ]
+  %cmp1 = icmp eq i32 %i.06, 0
+  br i1 %cmp1, label %if.then, label %if.end
+
+if.then:                                          ; preds = %for.body
+  call void @init()
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %for.body
+  call void @sink()
+  %inc = add nuw nsw i32 %i.06, 1
+  %exitcond = icmp eq i32 %inc, %len
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+; NOTE: here we should only peel the first iteration,
+;       i.e. all calls to sink() must stay in loop.
+define void @test13__peel_first_iter_via_ne_pred(i32 %len) {
+; CHECK-LABEL: @test13__peel_first_iter_via_ne_pred(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP5:%.*]] = icmp sgt i32 [[LEN:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP5]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
+; CHECK:       for.body.preheader:
+; CHECK-NEXT:    br label [[FOR_BODY_PEEL_BEGIN:%.*]]
+; CHECK:       for.body.peel.begin:
+; CHECK-NEXT:    br label [[FOR_BODY_PEEL:%.*]]
+; CHECK:       for.body.peel:
+; CHECK-NEXT:    [[CMP1_PEEL:%.*]] = icmp ne i32 0, 0
+; CHECK-NEXT:    br i1 [[CMP1_PEEL]], label [[IF_END_PEEL:%.*]], label [[IF_THEN_PEEL:%.*]]
+; CHECK:       if.then.peel:
+; CHECK-NEXT:    call void @init()
+; CHECK-NEXT:    br label [[IF_END_PEEL]]
+; CHECK:       if.end.peel:
+; CHECK-NEXT:    call void @sink()
+; CHECK-NEXT:    [[INC_PEEL:%.*]] = add nuw nsw i32 0, 1
+; CHECK-NEXT:    [[EXITCOND_PEEL:%.*]] = icmp eq i32 [[INC_PEEL]], [[LEN]]
+; CHECK-NEXT:    br i1 [[EXITCOND_PEEL]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[FOR_BODY_PEEL_NEXT:%.*]]
+; CHECK:       for.body.peel.next:
+; CHECK-NEXT:    br label [[FOR_BODY_PEEL_NEXT1:%.*]]
+; CHECK:       for.body.peel.next1:
+; CHECK-NEXT:    br label [[FOR_BODY_PREHEADER_PEEL_NEWPH:%.*]]
+; CHECK:       for.body.preheader.peel.newph:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.cond.cleanup.loopexit.loopexit:
+; CHECK-NEXT:    br label [[FOR_COND_CLEANUP_LOOPEXIT]]
+; CHECK:       for.cond.cleanup.loopexit:
+; CHECK-NEXT:    br label [[FOR_COND_CLEANUP]]
+; CHECK:       for.cond.cleanup:
+; CHECK-NEXT:    ret void
+; CHECK:       for.body:
+; CHECK-NEXT:    [[I_06:%.*]] = phi i32 [ [[INC:%.*]], [[IF_END:%.*]] ], [ [[INC_PEEL]], [[FOR_BODY_PREHEADER_PEEL_NEWPH]] ]
+; CHECK-NEXT:    br i1 true, label [[IF_END]], label [[IF_THEN:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    call void @init()
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    call void @sink()
+; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_06]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[LEN]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT:%.*]], label [[FOR_BODY]], !llvm.loop !12
+;
+entry:
+  %cmp5 = icmp sgt i32 %len, 0
+  br i1 %cmp5, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %if.end, %entry
+  ret void
+
+for.body:                                         ; preds = %entry, %if.end
+  %i.06 = phi i32 [ %inc, %if.end ], [ 0, %entry ]
+  %cmp1 = icmp ne i32 %i.06, 0
+  br i1 %cmp1, label %if.end, label %if.then
+
+if.then:                                          ; preds = %for.body
+  call void @init()
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %for.body
+  call void @sink()
+  %inc = add nuw nsw i32 %i.06, 1
+  %exitcond = icmp eq i32 %inc, %len
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+; No peeling is profitable here.
+define void @test14__ivar_mod2_is_1(i32 %len) {
+; CHECK-LABEL: @test14__ivar_mod2_is_1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP5:%.*]] = icmp sgt i32 [[LEN:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP5]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
+; CHECK:       for.body.preheader:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.cond.cleanup.loopexit:
+; CHECK-NEXT:    br label [[FOR_COND_CLEANUP]]
+; CHECK:       for.cond.cleanup:
+; CHECK-NEXT:    ret void
+; CHECK:       for.body:
+; CHECK-NEXT:    [[I_06:%.*]] = phi i32 [ [[INC:%.*]], [[IF_END:%.*]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[I_06]], 1
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i32 [[AND]], 1
+; CHECK-NEXT:    br i1 [[CMP1]], label [[IF_END]], label [[IF_THEN:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    call void @init()
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    call void @sink()
+; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_06]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[LEN]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[FOR_BODY]]
+;
+entry:
+  %cmp5 = icmp sgt i32 %len, 0
+  br i1 %cmp5, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %if.end, %entry
+  ret void
+
+for.body:                                         ; preds = %entry, %if.end
+  %i.06 = phi i32 [ %inc, %if.end ], [ 0, %entry ]
+  %and = and i32 %i.06, 1
+  %cmp1 = icmp eq i32 %and, 1
+  br i1 %cmp1, label %if.end, label %if.then
+
+if.then:                                          ; preds = %for.body
+  call void @init()
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %for.body
+  call void @sink()
+  %inc = add nuw nsw i32 %i.06, 1
+  %exitcond = icmp eq i32 %inc, %len
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+; No peeling is profitable here.
+define void @test15__ivar_mod2_is_0(i32 %len) {
+; CHECK-LABEL: @test15__ivar_mod2_is_0(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP5:%.*]] = icmp sgt i32 [[LEN:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP5]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
+; CHECK:       for.body.preheader:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.cond.cleanup.loopexit:
+; CHECK-NEXT:    br label [[FOR_COND_CLEANUP]]
+; CHECK:       for.cond.cleanup:
+; CHECK-NEXT:    ret void
+; CHECK:       for.body:
+; CHECK-NEXT:    [[I_06:%.*]] = phi i32 [ [[INC:%.*]], [[IF_END:%.*]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[I_06]], 1
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    br i1 [[CMP1]], label [[IF_END]], label [[IF_THEN:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    call void @init()
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    call void @sink()
+; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_06]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[LEN]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[FOR_BODY]]
+;
+entry:
+  %cmp5 = icmp sgt i32 %len, 0
+  br i1 %cmp5, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %if.end, %entry
+  ret void
+
+for.body:                                         ; preds = %entry, %if.end
+  %i.06 = phi i32 [ %inc, %if.end ], [ 0, %entry ]
+  %and = and i32 %i.06, 1
+  %cmp1 = icmp eq i32 %and, 0
+  br i1 %cmp1, label %if.end, label %if.then
+
+if.then:                                          ; preds = %for.body
+  call void @init()
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %for.body
+  call void @sink()
+  %inc = add nuw nsw i32 %i.06, 1
+  %exitcond = icmp eq i32 %inc, %len
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+; Similar to @test7, we need to peel one extra iteration, and we can't do that
+; as per the -unroll-peel-max-count=4, so this shouldn't be peeled at all.
+define void @test16(i32 %k) {
+; CHECK-LABEL: @test16(
+; CHECK-NEXT:  for.body.lr.ph:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[I_05:%.*]] = phi i32 [ 0, [[FOR_BODY_LR_PH:%.*]] ], [ [[INC:%.*]], [[FOR_INC:%.*]] ]
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ne i32 [[I_05]], 4
+; CHECK-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[FOR_INC]]
+; CHECK:       if.then:
+; CHECK-NEXT:    call void @f1()
+; CHECK-NEXT:    br label [[FOR_INC]]
+; CHECK:       for.inc:
+; CHECK-NEXT:    [[INC]] = add nsw i32 [[I_05]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[INC]], [[K:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END:%.*]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
+for.body.lr.ph:
+  br label %for.body
+
+for.body:
+  %i.05 = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %for.inc ]
+  %cmp1 = icmp ne i32 %i.05, 4
+  br i1 %cmp1, label %if.then, label %for.inc
+
+if.then:
+  call void @f1()
+  br label %for.inc
+
+for.inc:
+  %inc = add nsw i32 %i.05, 1
+  %cmp = icmp slt i32 %inc, %k
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+; Similar to @test8, we need to peel one extra iteration, and we can't do that
+; as per the -unroll-peel-max-count=4, so this shouldn't be peeled at all.
+define void @test17(i32 %k) {
+; CHECK-LABEL: @test17(
+; CHECK-NEXT:  for.body.lr.ph:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[I_05:%.*]] = phi i32 [ 0, [[FOR_BODY_LR_PH:%.*]] ], [ [[INC:%.*]], [[FOR_INC:%.*]] ]
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i32 [[I_05]], 4
+; CHECK-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[FOR_INC]]
+; CHECK:       if.then:
+; CHECK-NEXT:    call void @f1()
+; CHECK-NEXT:    br label [[FOR_INC]]
+; CHECK:       for.inc:
+; CHECK-NEXT:    [[INC]] = add nsw i32 [[I_05]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[INC]], [[K:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END:%.*]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
+for.body.lr.ph:
+  br label %for.body
+
+for.body:
+  %i.05 = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %for.inc ]
+  %cmp1 = icmp eq i32 %i.05, 4
+  br i1 %cmp1, label %if.then, label %for.inc
+
+if.then:
+  call void @f1()
+  br label %for.inc
+
+for.inc:
+  %inc = add nsw i32 %i.05, 1
+  %cmp = icmp slt i32 %inc, %k
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+declare void @init()
+declare void @sink()
diff --git a/llvm/test/Transforms/LoopUnroll/peel-loop-inner.ll b/llvm/test/Transforms/LoopUnroll/peel-loop-inner.ll
new file mode 100644
index 000000000..a83f3bfb9
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/peel-loop-inner.ll
@@ -0,0 +1,50 @@
+; RUN: opt < %s -S -passes='require<opt-remark-emit>,unroll<peeling;no-runtime>,simplify-cfg,instcombine' -unroll-force-peel-count=3 -verify-dom-info | FileCheck %s
+
+define void @basic(i32 %K, i32 %N) {
+; CHECK-LABEL: @basic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[OUTER:%.*]]
+; CHECK:       outer:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[I_INC:%.*]], [[OUTER_BACKEDGE:%.*]] ]
+; CHECK-NEXT:    [[CMP_INNER_PEEL:%.*]] = icmp sgt i32 [[K:%.*]], 1
+; CHECK-NEXT:    br i1 [[CMP_INNER_PEEL]], label [[INNER_PEEL2:%.*]], label [[OUTER_BACKEDGE]]
+; CHECK:       inner.peel2:
+; CHECK-NEXT:    [[CMP_INNER_PEEL4:%.*]] = icmp eq i32 [[K]], 2
+; CHECK-NEXT:    br i1 [[CMP_INNER_PEEL4]], label [[OUTER_BACKEDGE]], label [[INNER_PEEL6:%.*]]
+; CHECK:       inner.peel6:
+; CHECK-NEXT:    [[CMP_INNER_PEEL8:%.*]] = icmp sgt i32 [[K]], 3
+; CHECK-NEXT:    br i1 [[CMP_INNER_PEEL8]], label [[INNER:%.*]], label [[OUTER_BACKEDGE]]
+; CHECK:       inner:
+; CHECK-NEXT:    [[J:%.*]] = phi i32 [ [[J_INC:%.*]], [[INNER]] ], [ 3, [[INNER_PEEL6]] ]
+; CHECK-NEXT:    [[J_INC]] = add nuw nsw i32 [[J]], 1
+; CHECK-NEXT:    [[CMP_INNER:%.*]] = icmp slt i32 [[J_INC]], [[K]]
+; CHECK-NEXT:    br i1 [[CMP_INNER]], label [[INNER]], label [[OUTER_BACKEDGE]], !llvm.loop !0
+; CHECK:       outer.backedge:
+; CHECK-NEXT:    [[I_INC]] = add i32 [[I]], 1
+; CHECK-NEXT:    [[CMP_OUTER:%.*]] = icmp slt i32 [[I_INC]], [[N:%.*]]
+; CHECK-NOT:    !llvm.loop
+; CHECK:       end:
+;
+entry:
+  br label %outer
+
+outer:
+  %i = phi i32 [ 0, %entry ], [ %i.inc, %outer.backedge ]
+  br label %inner
+
+inner:
+  %j = phi i32 [ 0, %outer ], [ %j.inc, %inner ]
+  %j.inc = add i32 %j, 1
+  %cmp.inner = icmp slt i32 %j.inc, %K
+  br i1 %cmp.inner, label %inner, label %outer.backedge, !llvm.loop !1
+
+outer.backedge:
+  %i.inc = add i32 %i, 1
+  %cmp.outer = icmp slt i32 %i.inc, %N
+  br i1 %cmp.outer, label %outer, label %end
+
+end:
+  ret void
+}
+
+!1 = distinct !{!1}
diff --git a/llvm/test/Transforms/LoopUnroll/peel-loop-pgo-deopt-idom-2.ll b/llvm/test/Transforms/LoopUnroll/peel-loop-pgo-deopt-idom-2.ll
new file mode 100644
index 000000000..17fd5a741
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/peel-loop-pgo-deopt-idom-2.ll
@@ -0,0 +1,46 @@
+; REQUIRES: asserts
+; RUN: opt < %s -S -debug-only=loop-unroll -loop-unroll -unroll-runtime -unroll-peel-multi-deopt-exit 2>&1 | FileCheck %s
+; RUN: opt < %s -S -debug-only=loop-unroll -unroll-peel-multi-deopt-exit -passes='require<profile-summary>,function(require<opt-remark-emit>,unroll)' 2>&1 | FileCheck %s
+
+; Regression test for setting the correct idom for exit blocks.
+
+; CHECK: Loop Unroll: F[basic]
+; CHECK: PEELING loop %for.body with iteration count 1!
+
+define i32 @basic(i32* %p, i32 %k, i1 %c1, i1 %c2) #0 !prof !3 {
+entry:
+  br label %for.body
+
+for.body:
+  %i.05 = phi i32 [ 0, %entry ], [ %inc, %latch ]
+  %p.addr.04 = phi i32* [ %p, %entry ], [ %incdec.ptr, %latch ]
+  %incdec.ptr = getelementptr inbounds i32, i32* %p.addr.04, i32 1
+  store i32 %i.05, i32* %p.addr.04, align 4
+  %inc = add nsw i32 %i.05, 1
+  %cmp = icmp slt i32 %inc, %k
+  br i1 %c1, label %left, label %right
+
+left:
+  br label %latch
+
+right:
+  br i1 %c1, label %latch, label %side_exit, !prof !2
+
+latch:
+  br i1 %cmp, label %for.body, label %for.end, !prof !1
+
+for.end:
+  ret i32 %inc
+
+side_exit:
+  %rval = call i32(...) @llvm.experimental.deoptimize.i32() [ "deopt"(i32 %inc) ]
+  ret i32 %rval
+}
+
+declare i32 @llvm.experimental.deoptimize.i32(...)
+
+attributes #0 = { nounwind }
+
+!1 = !{!"branch_weights", i32 1, i32 1}
+!2 = !{!"branch_weights", i32 1, i32 0}
+!3 = !{!"function_entry_count", i64 1}
diff --git a/llvm/test/Transforms/LoopUnroll/peel-loop-pgo-deopt.ll b/llvm/test/Transforms/LoopUnroll/peel-loop-pgo-deopt.ll
index dc023f36d..abad07948 100644
--- a/llvm/test/Transforms/LoopUnroll/peel-loop-pgo-deopt.ll
+++ b/llvm/test/Transforms/LoopUnroll/peel-loop-pgo-deopt.ll
@@ -1,20 +1,26 @@
 ; REQUIRES: asserts
 ; RUN: opt < %s -S -debug-only=loop-unroll -loop-unroll -unroll-runtime -unroll-peel-multi-deopt-exit 2>&1 | FileCheck %s
 ; RUN: opt < %s -S -debug-only=loop-unroll -unroll-peel-multi-deopt-exit -passes='require<profile-summary>,function(require<opt-remark-emit>,unroll)' 2>&1 | FileCheck %s
+; RUN: opt < %s -S -debug-only=loop-unroll -unroll-peel-multi-deopt-exit -passes='require<profile-summary>,function(require<opt-remark-emit>,unroll<no-profile-peeling>)' 2>&1 | FileCheck %s --check-prefixes=CHECK-NO-PEEL
 
 ; Make sure we use the profile information correctly to peel-off 3 iterations
 ; from the loop, and update the branch weights for the peeled loop properly.
+; All side exits to deopt does not change weigths.
 
 ; CHECK: Loop Unroll: F[basic]
 ; CHECK: PEELING loop %for.body with iteration count 3!
-
+; CHECK-NO-PEEL-NOT: PEELING loop %for.body
 ; CHECK-LABEL: @basic
+; CHECK: br i1 %c, label %{{.*}}, label %side_exit, !prof !15
 ; CHECK: br i1 %{{.*}}, label %[[NEXT0:.*]], label %for.cond.for.end_crit_edge, !prof !16
 ; CHECK: [[NEXT0]]:
+; CHECK: br i1 %c, label %{{.*}}, label %side_exit, !prof !15
 ; CHECK: br i1 %{{.*}}, label %[[NEXT1:.*]], label %for.cond.for.end_crit_edge, !prof !17
 ; CHECK: [[NEXT1]]:
+; CHECK: br i1 %c, label %{{.*}}, label %side_exit, !prof !15
 ; CHECK: br i1 %{{.*}}, label %[[NEXT2:.*]], label %for.cond.for.end_crit_edge, !prof !18
 ; CHECK: [[NEXT2]]:
+; CHECK: br i1 %c, label %{{.*}}, label %side_exit.loopexit, !prof !15
 ; CHECK: br i1 %{{.*}}, label %for.body, label %{{.*}}, !prof !19
 
 define i32 @basic(i32* %p, i32 %k, i1 %c) #0 !prof !15 {
@@ -74,8 +80,11 @@ attributes #1 = { nounwind optsize }
 !16 = !{!"branch_weights", i32 3001, i32 1001}
 !17 = !{!"branch_weights", i32 1, i32 0}
 
-;CHECK: !16 = !{!"branch_weights", i32 900, i32 101}
-;CHECK: !17 = !{!"branch_weights", i32 540, i32 360}
-;CHECK: !18 = !{!"branch_weights", i32 162, i32 378}
-;CHECK: !19 = !{!"branch_weights", i32 1399, i32 162}
+; This is a weights of deopt side-exit.
+;CHECK: !15 = !{!"branch_weights", i32 1, i32 0}
+; This is a weights of latch and its copies.
+;CHECK: !16 = !{!"branch_weights", i32 3001, i32 1001}
+;CHECK: !17 = !{!"branch_weights", i32 2000, i32 1001}
+;CHECK: !18 = !{!"branch_weights", i32 999, i32 1001}
+;CHECK: !19 = !{!"branch_weights", i32 1, i32 1001}
 
diff --git a/llvm/test/Transforms/LoopUnroll/peel-loop-pgo.ll b/llvm/test/Transforms/LoopUnroll/peel-loop-pgo.ll
index 361a2ca71..f34d3d46c 100644
--- a/llvm/test/Transforms/LoopUnroll/peel-loop-pgo.ll
+++ b/llvm/test/Transforms/LoopUnroll/peel-loop-pgo.ll
@@ -103,8 +103,8 @@ attributes #1 = { nounwind optsize }
 !15 = !{!"function_entry_count", i64 1}
 !16 = !{!"branch_weights", i32 3001, i32 1001}
 
-;CHECK: !15 = !{!"branch_weights", i32 900, i32 101}
-;CHECK: !16 = !{!"branch_weights", i32 540, i32 360}
-;CHECK: !17 = !{!"branch_weights", i32 162, i32 378}
-;CHECK: !18 = !{!"branch_weights", i32 1399, i32 162}
+;CHECK: !15 = !{!"branch_weights", i32 3001, i32 1001}
+;CHECK: !16 = !{!"branch_weights", i32 2000, i32 1001}
+;CHECK: !17 = !{!"branch_weights", i32 999, i32 1001}
+;CHECK: !18 = !{!"branch_weights", i32 1, i32 1001}
 
diff --git a/llvm/test/Transforms/LoopUnroll/runtime-epilog-debuginfo.ll b/llvm/test/Transforms/LoopUnroll/runtime-epilog-debuginfo.ll
index 98077ea6c..3a0049032 100644
--- a/llvm/test/Transforms/LoopUnroll/runtime-epilog-debuginfo.ll
+++ b/llvm/test/Transforms/LoopUnroll/runtime-epilog-debuginfo.ll
@@ -66,7 +66,7 @@ lee1.exit:                                        ; preds = %lee1.exit.loopexit,
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.value(metadata, i64, metadata, metadata) #1
 
-attributes #0 = { nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="arm7tdmi" "target-features"="+neon,+strict-align,+vfp3,-crypto,-fp-armv8,-fp16,-vfp4" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="arm7tdmi" "target-features"="+neon,+strict-align,+vfp3,-crypto,-fp-armv8,-fp16,-vfp4" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/Transforms/LoopUnroll/runtime-loop-multiple-exits.ll b/llvm/test/Transforms/LoopUnroll/runtime-loop-multiple-exits.ll
index 9f9735866..e91bd2eff 100644
--- a/llvm/test/Transforms/LoopUnroll/runtime-loop-multiple-exits.ll
+++ b/llvm/test/Transforms/LoopUnroll/runtime-loop-multiple-exits.ll
@@ -578,10 +578,10 @@ define void @test8() {
 ; PROLOG:      %lcmp.mod = icmp eq i64
 ; PROLOG-NEXT: br i1 %lcmp.mod, label %innerH.prol.loopexit, label %innerH.prol.preheader
 ; PROLOG: latch.6:
-; PROLOG-NEXT: %tmp4.7 = add nuw nsw i64 %tmp3, 8
 ; PROLOG-NEXT: br i1 false, label %outerloop.loopexit.loopexit, label %latch.7
-; PROLOG: latch.7
-; PROLOG-NEXT: %tmp6.7 = icmp ult i64 %tmp4.7, 100
+; PROLOG: latch.7:
+; PROLOG-NEXT: %tmp4.7 = add nuw nsw i64 %tmp3, 8
+; PROLOG-NEXT: %tmp6.7 = icmp ult i64 %tmp3, 92
 ; PROLOG-NEXT: br i1 %tmp6.7, label %innerH, label %exit.unr-lcssa
 bb:
   br label %outerloop
diff --git a/llvm/test/Transforms/LoopUnroll/runtime-small-upperbound.ll b/llvm/test/Transforms/LoopUnroll/runtime-small-upperbound.ll
new file mode 100644
index 000000000..95632a5a3
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/runtime-small-upperbound.ll
@@ -0,0 +1,70 @@
+; RUN: opt -S -loop-unroll -unroll-runtime %s -o - | FileCheck %s
+; RUN: opt -S -loop-unroll -unroll-runtime -unroll-max-upperbound=6 %s -o - | FileCheck %s --check-prefix=UPPER
+
+target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
+
+@global = dso_local local_unnamed_addr global i32 0, align 4
+@global.1 = dso_local local_unnamed_addr global i8* null, align 4
+
+; Check that loop in hoge_3, with a runtime upperbound of 3, is not unrolled.
+; CHECK-LABEL: hoge_3
+; CHECK: loop:
+; CHECK: store
+; CHECK-NOT: store
+; CHECK: br i1 %{{.*}}, label %loop
+; UPPER-LABEL: hoge_3
+; UPPER: loop:
+; UPPER: store
+; UPPER-NOT: store
+; UPPER: br i1 %{{.*}}, label %loop
+define dso_local void @hoge_3(i8 %arg) {
+entry:
+  %x = load i32, i32* @global, align 4
+  %y = load i8*, i8** @global.1, align 4
+  %0 = icmp ult i32 %x, 17
+  br i1 %0, label %loop, label %exit
+
+loop:
+  %iv = phi i32 [ %x, %entry ], [ %iv.next, %loop ]
+  %ptr = phi i8* [ %y, %entry ], [ %ptr.next, %loop ]
+  %iv.next = add nuw i32 %iv, 8
+  %ptr.next = getelementptr inbounds i8, i8* %ptr, i32 1
+  store i8 %arg, i8* %ptr.next, align 1
+  %1 = icmp ult i32 %iv.next, 17
+  br i1 %1, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+; Check that loop in hoge_5, with a runtime upperbound of 5, is unrolled when -unroll-max-upperbound=4
+; CHECK-LABEL: hoge_5
+; CHECK: loop:
+; CHECK: store
+; CHECK-NOT: store
+; CHECK: br i1 %{{.*}}, label %loop
+; UPPER-LABEL: hoge_5
+; UPPER: loop:
+; UPPER: store
+; UPPER: store
+; UPPER: store
+; UPPER: br i1 %{{.*}}, label %loop
+define dso_local void @hoge_5(i8 %arg) {
+entry:
+  %x = load i32, i32* @global, align 4
+  %y = load i8*, i8** @global.1, align 4
+  %0 = icmp ult i32 %x, 17
+  br i1 %0, label %loop, label %exit
+
+loop:
+  %iv = phi i32 [ %x, %entry ], [ %iv.next, %loop ]
+  %ptr = phi i8* [ %y, %entry ], [ %ptr.next, %loop ]
+  %iv.next = add nuw i32 %iv, 4
+  %ptr.next = getelementptr inbounds i8, i8* %ptr, i32 1
+  store i8 %arg, i8* %ptr.next, align 1
+  %1 = icmp ult i32 %iv.next, 17
+  br i1 %1, label %loop, label %exit
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopUnroll/scevunroll.ll b/llvm/test/Transforms/LoopUnroll/scevunroll.ll
index afee41e11..ea473e1cc 100644
--- a/llvm/test/Transforms/LoopUnroll/scevunroll.ll
+++ b/llvm/test/Transforms/LoopUnroll/scevunroll.ll
@@ -67,12 +67,10 @@ exit2:
 ; SCEV properly unrolls multi-exit loops.
 ;
 ; CHECK-LABEL: @multiExit(
-; CHECK: getelementptr i32, i32* %base, i32 10
+; CHECK: getelementptr i32, i32* %base, i32 %iv
 ; CHECK-NEXT: load i32, i32*
-; CHECK: br i1 false, label %l2.10, label %exit1
-; CHECK: l2.10:
-; CHECK-NOT: br
-; CHECK: ret i32
+; CHECK: br i1 false, label %l2, label %exit1
+; CHECK: br i1 true, label %l1, label %exit2
 define i32 @multiExit(i32* %base) nounwind {
 entry:
   br label %l1
@@ -184,7 +182,7 @@ for.body87:
 ; CHECK: for.body:
 ; CHECK: %b.03 = phi i32 [ 0, %entry ], [ %add, %for.cond ]
 ; CHECK: return:
-; CHECK: %b.03.lcssa = phi i32 [ %b.03, %for.body ], [ 0, %for.cond ]
+; CHECK: %b.03.lcssa = phi i32 [ 8, %for.body ], [ 0, %for.cond ]
 define void @nsw_latch(i32* %a) nounwind {
 entry:
   br label %for.body
diff --git a/llvm/test/Transforms/LoopUnroll/unroll-preserve-scev-lcssa.ll b/llvm/test/Transforms/LoopUnroll/unroll-preserve-scev-lcssa.ll
new file mode 100644
index 000000000..205a51ae3
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/unroll-preserve-scev-lcssa.ll
@@ -0,0 +1,71 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -loop-reduce -loop-unroll -unroll-runtime -verify-scev -verify-loop-lcssa < %s -S | FileCheck %s
+
+; Test case for PR43458. We check that we do properly invalidate LCSSA phi
+; users in SCEV. Run -loop-reduce first, so SCEV is already populated.
+
+define void @spam() {
+; CHECK-LABEL: @spam(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    br label [[BB3:%.*]]
+; CHECK:       bb3:
+; CHECK-NEXT:    br i1 false, label [[BB9:%.*]], label [[BB5_PREHEADER:%.*]]
+; CHECK:       bb5.preheader:
+; CHECK-NEXT:    br label [[BB5:%.*]]
+; CHECK:       bb5:
+; CHECK-NEXT:    br label [[BB9]]
+; CHECK:       bb9:
+; CHECK-NEXT:    [[TMP10:%.*]] = phi i64 [ 0, [[BB3]] ], [ 5, [[BB5]] ]
+; CHECK-NEXT:    [[TMP11:%.*]] = trunc i64 [[TMP10]] to i32
+; CHECK-NEXT:    [[TMP21:%.*]] = sext i32 [[TMP11]] to i64
+; CHECK-NEXT:    [[TMP22:%.*]] = icmp slt i64 0, [[TMP21]]
+; CHECK-NEXT:    br i1 [[TMP22]], label [[BB24_PREHEADER:%.*]], label [[BB29:%.*]]
+; CHECK:       bb24.preheader:
+; CHECK-NEXT:    br label [[BB24:%.*]]
+; CHECK:       bb24:
+; CHECK-NEXT:    [[TMP25:%.*]] = phi i64 [ [[TMP26:%.*]], [[BB24]] ], [ 0, [[BB24_PREHEADER]] ]
+; CHECK-NEXT:    [[TMP26]] = add nuw nsw i64 [[TMP25]], 1
+; CHECK-NEXT:    [[TMP27:%.*]] = icmp slt i64 [[TMP26]], [[TMP21]]
+; CHECK-NEXT:    br i1 [[TMP27]], label [[BB24]], label [[BB29_LOOPEXIT:%.*]]
+; CHECK:       bb29.loopexit:
+; CHECK-NEXT:    br label [[BB29]]
+; CHECK:       bb29:
+; CHECK-NEXT:    ret void
+;
+bb:
+  br label %bb3
+
+bb3:                                              ; preds = %bb9, %bb
+  %pv1 = phi i64 [ 0, %bb ], [ %iv1, %bb9]
+  %c1 = icmp eq i64 %pv1, 5
+  br i1 %c1, label %bb9, label %bb5
+
+bb5:                                              ; preds = %bb5, %bb3
+  %pv = phi i64 [ 0, %bb3], [ %tmp6, %bb5]
+  %tmp6 = add nsw i64 %pv, 1
+  %cond = icmp eq i64 %tmp6, 5
+  br i1 %cond, label %bb8, label %bb5
+
+bb8:                                              ; preds = %bb5
+  br label %bb9
+
+bb9:                                              ; preds = %bb8, %bb3
+  %tmp10 = phi i64 [ 0, %bb3 ], [ %tmp6, %bb8 ]
+  %tmp11 = trunc i64 %tmp10 to i32
+  %iv1 = add nsw i64 %pv1, 1
+  br i1 false, label %bb3, label %bb20
+
+bb20:                                             ; preds = %bb9
+  %tmp21 = sext i32 %tmp11 to i64
+  %tmp22 = icmp slt i64 0, %tmp21
+  br i1 %tmp22, label %bb24, label %bb29
+
+bb24:                                             ; preds = %bb24, %bb20
+  %tmp25 = phi i64 [ %tmp26, %bb24 ], [ 0, %bb20 ]
+  %tmp26 = add nuw nsw i64 %tmp25, 1
+  %tmp27 = icmp slt i64 %tmp26, %tmp21
+  br i1 %tmp27, label %bb24, label %bb29
+
+bb29:                                             ; preds = %bb24, %bb20
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopUnrollAndJam/dependencies.ll b/llvm/test/Transforms/LoopUnrollAndJam/dependencies.ll
index 890683043..d58423874 100644
--- a/llvm/test/Transforms/LoopUnrollAndJam/dependencies.ll
+++ b/llvm/test/Transforms/LoopUnrollAndJam/dependencies.ll
@@ -1,4 +1,5 @@
 ; RUN: opt -basicaa -loop-unroll-and-jam -allow-unroll-and-jam -unroll-and-jam-count=4 < %s -S | FileCheck %s
+; RUN: opt -aa-pipeline=basic-aa -passes='unroll-and-jam' -allow-unroll-and-jam -unroll-and-jam-count=4 < %s -S | FileCheck %s
 
 target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
 
diff --git a/llvm/test/Transforms/LoopUnrollAndJam/disable.ll b/llvm/test/Transforms/LoopUnrollAndJam/disable.ll
index 4a00937b9..6e879896c 100644
--- a/llvm/test/Transforms/LoopUnrollAndJam/disable.ll
+++ b/llvm/test/Transforms/LoopUnrollAndJam/disable.ll
@@ -1,4 +1,5 @@
 ; RUN: opt -loop-unroll-and-jam -allow-unroll-and-jam -unroll-and-jam-count=4 -pass-remarks=loop-unroll-and-jam < %s -S 2>&1 | FileCheck %s
+; RUN: opt -passes='unroll-and-jam' -allow-unroll-and-jam -unroll-and-jam-count=4 -pass-remarks=loop-unroll-and-jam < %s -S 2>&1 | FileCheck %s
 
 target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
 
diff --git a/llvm/test/Transforms/LoopUnrollAndJam/pragma-explicit.ll b/llvm/test/Transforms/LoopUnrollAndJam/pragma-explicit.ll
index 5254c779d..f2c12702f 100644
--- a/llvm/test/Transforms/LoopUnrollAndJam/pragma-explicit.ll
+++ b/llvm/test/Transforms/LoopUnrollAndJam/pragma-explicit.ll
@@ -1,4 +1,5 @@
 ; RUN: opt -loop-unroll-and-jam -allow-unroll-and-jam -unroll-runtime -unroll-partial-threshold=60 < %s -S | FileCheck %s
+; RUN: opt -aa-pipeline=type-based-aa,basic-aa -passes='unroll-and-jam' -allow-unroll-and-jam -unroll-runtime -unroll-partial-threshold=60 < %s -S | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/Transforms/LoopUnrollAndJam/unroll-and-jam.ll b/llvm/test/Transforms/LoopUnrollAndJam/unroll-and-jam.ll
index bdb47c27f..7580b50f8 100644
--- a/llvm/test/Transforms/LoopUnrollAndJam/unroll-and-jam.ll
+++ b/llvm/test/Transforms/LoopUnrollAndJam/unroll-and-jam.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -basicaa -tbaa -loop-unroll-and-jam -allow-unroll-and-jam -unroll-and-jam-count=4 -unroll-remainder < %s -S | FileCheck %s
+; RUN: opt -aa-pipeline=type-based-aa,basic-aa -passes='unroll-and-jam' -allow-unroll-and-jam -unroll-and-jam-count=4 -unroll-remainder < %s -S | FileCheck %s
 
 target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
 
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/arbitrary-induction-step.ll b/llvm/test/Transforms/LoopVectorize/AArch64/arbitrary-induction-step.ll
index 81ecd57b2..8aa196b43 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/arbitrary-induction-step.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/arbitrary-induction-step.ll
@@ -15,15 +15,15 @@ target triple = "aarch64--linux-gnueabi"
 ; CHECK: load <4 x i32>, <4 x i32>*
 ; CHECK: mul nsw <4 x i32>
 ; CHECK: mul nsw <4 x i32>
-; CHECK: add nsw <4 x i32>
-; CHECK: add nsw <4 x i32>
+; CHECK: add <4 x i32>
+; CHECK: add <4 x i32>
 ; CHECK: %index.next = add i64 %index, 8
 ; CHECK: icmp eq i64 %index.next, 512
 
 ; FORCE-VEC-LABEL: @ind_plus2(
 ; FORCE-VEC: %wide.load = load <2 x i32>, <2 x i32>*
 ; FORCE-VEC: mul nsw <2 x i32>
-; FORCE-VEC: add nsw <2 x i32>
+; FORCE-VEC: add <2 x i32>
 ; FORCE-VEC: %index.next = add i64 %index, 2
 ; FORCE-VEC: icmp eq i64 %index.next, 512
 define i32 @ind_plus2(i32* %A) {
@@ -59,15 +59,15 @@ for.end:                                          ; preds = %for.body
 ; CHECK: load <4 x i32>, <4 x i32>*
 ; CHECK: mul nsw <4 x i32>
 ; CHECK: mul nsw <4 x i32>
-; CHECK: add nsw <4 x i32>
-; CHECK: add nsw <4 x i32>
+; CHECK: add <4 x i32>
+; CHECK: add <4 x i32>
 ; CHECK: %index.next = add i64 %index, 8
 ; CHECK: icmp eq i64 %index.next, 512
 
 ; FORCE-VEC-LABEL: @ind_minus2(
 ; FORCE-VEC: %wide.load = load <2 x i32>, <2 x i32>*
 ; FORCE-VEC: mul nsw <2 x i32>
-; FORCE-VEC: add nsw <2 x i32>
+; FORCE-VEC: add <2 x i32>
 ; FORCE-VEC: %index.next = add i64 %index, 2
 ; FORCE-VEC: icmp eq i64 %index.next, 512
 define i32 @ind_minus2(i32* %A) {
@@ -110,8 +110,8 @@ for.end:                                          ; preds = %for.body
 ; CHECK: shufflevector <8 x i32> %[[V1]], <8 x i32> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
 ; CHECK: mul nsw <4 x i32>
 ; CHECK: mul nsw <4 x i32>
-; CHECK: add nsw <4 x i32>
-; CHECK: add nsw <4 x i32>
+; CHECK: add <4 x i32>
+; CHECK: add <4 x i32>
 ; CHECK: %index.next = add i64 %index, 8
 ; CHECK: icmp eq i64 %index.next, 1024
 
@@ -120,7 +120,7 @@ for.end:                                          ; preds = %for.body
 ; FORCE-VEC: shufflevector <4 x i32> %[[V]], <4 x i32> undef, <2 x i32> <i32 0, i32 2>
 ; FORCE-VEC: shufflevector <4 x i32> %[[V]], <4 x i32> undef, <2 x i32> <i32 1, i32 3>
 ; FORCE-VEC: mul nsw <2 x i32>
-; FORCE-VEC: add nsw <2 x i32>
+; FORCE-VEC: add <2 x i32>
 ; FORCE-VEC: %index.next = add i64 %index, 2
 ; FORCE-VEC: icmp eq i64 %index.next, 1024
 define i32 @ptr_ind_plus2(i32* %A) {
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/pr33053.ll b/llvm/test/Transforms/LoopVectorize/AArch64/pr33053.ll
index 6763940bf..eaed8276a 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/pr33053.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/pr33053.ll
@@ -42,7 +42,7 @@ for.end:                                          ; preds = %for.body, %entry
   ret i32 %conv27
 }
 
-attributes #0 = { norecurse nounwind readonly "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="generic" "target-features"="+neon" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { norecurse nounwind readonly "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="generic" "target-features"="+neon" "unsafe-fp-math"="false" "use-soft-float"="false" }
 !llvm.ident = !{!0}
 
 !0 = !{!"clang"}
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/arm-ieee-vectorize.ll b/llvm/test/Transforms/LoopVectorize/ARM/arm-ieee-vectorize.ll
index 369568f6d..8a8be7646 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/arm-ieee-vectorize.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/arm-ieee-vectorize.ll
@@ -1,5 +1,6 @@
 ; RUN: opt -mtriple armv7-linux-gnueabihf -loop-vectorize -S %s -debug-only=loop-vectorize -o /dev/null 2>&1 | FileCheck %s --check-prefix=CHECK --check-prefix=LINUX
 ; RUN: opt -mtriple armv8-linux-gnu -loop-vectorize -S %s -debug-only=loop-vectorize -o /dev/null 2>&1 | FileCheck %s --check-prefix=CHECK --check-prefix=LINUX
+; RUN: opt -mtriple armv8.1.m-none-eabi -mattr=+mve.fp -loop-vectorize -S %s -debug-only=loop-vectorize -o /dev/null 2>&1 | FileCheck %s --check-prefix=CHECK --check-prefix=MVE
 ; RUN: opt -mtriple armv7-unknwon-darwin -loop-vectorize -S %s -debug-only=loop-vectorize -o /dev/null 2>&1 | FileCheck %s --check-prefix=CHECK --check-prefix=DARWIN
 ; REQUIRES: asserts
 
@@ -44,6 +45,8 @@ for.end:                                          ; preds = %for.end.loopexit, %
 ; Floating-point loops need fast-math to be vectorizeable
 ; LINUX: Checking a loop in "sumf"
 ; LINUX: Potentially unsafe FP op prevents vectorization
+; MVE: Checking a loop in "sumf"
+; MVE: We can vectorize this loop!
 ; DARWIN: Checking a loop in "sumf"
 ; DARWIN: We can vectorize this loop!
 define void @sumf(float* noalias nocapture readonly %A, float* noalias nocapture readonly %B, float* noalias nocapture %C, i32 %N) {
@@ -110,6 +113,8 @@ for.end:                                          ; preds = %for.end.loopexit, %
 ; Floating-point loops need fast-math to be vectorizeable
 ; LINUX: Checking a loop in "redf"
 ; LINUX: Potentially unsafe FP op prevents vectorization
+; MVE: Checking a loop in "redf"
+; MVE: We can vectorize this loop!
 ; DARWIN: Checking a loop in "redf"
 ; DARWIN: We can vectorize this loop!
 define float @redf(float* noalias nocapture readonly %a, float* noalias nocapture readonly %b, i32 %N) {
@@ -326,5 +331,5 @@ for.end:                                          ; preds = %for.body, %entry
 
 declare float @fabsf(float)
 
-attributes #1 = { nounwind readnone "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="cortex-a8" "target-features"="+dsp,+neon,+vfp3" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #2 = { nounwind readnone "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="true" "no-nans-fp-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="cortex-a8" "target-features"="+dsp,+neon,+vfp3" "unsafe-fp-math"="true" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="cortex-a8" "target-features"="+dsp,+neon,+vfp3" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { nounwind readnone "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="true" "no-nans-fp-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="cortex-a8" "target-features"="+dsp,+neon,+vfp3" "unsafe-fp-math"="true" "use-soft-float"="false" }
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/mve-interleaved-cost.ll b/llvm/test/Transforms/LoopVectorize/ARM/mve-interleaved-cost.ll
new file mode 100644
index 000000000..4cd63b422
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/ARM/mve-interleaved-cost.ll
@@ -0,0 +1,1111 @@
+; RUN: opt -loop-vectorize -force-vector-width=2 -debug-only=loop-vectorize -disable-output < %s 2>&1 | FileCheck %s --check-prefix=VF_2
+; RUN: opt -loop-vectorize -force-vector-width=4 -debug-only=loop-vectorize -disable-output < %s 2>&1 | FileCheck %s --check-prefix=VF_4
+; RUN: opt -loop-vectorize -force-vector-width=8 -debug-only=loop-vectorize -disable-output < %s 2>&1 | FileCheck %s --check-prefix=VF_8
+; RUN: opt -loop-vectorize -force-vector-width=16 -debug-only=loop-vectorize -disable-output < %s 2>&1 | FileCheck %s --check-prefix=VF_16
+; REQUIRES: asserts
+
+target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
+target triple = "thumbv8.1-m.main-none-eabi"
+
+; Factor 2
+
+%i8.2 = type {i8, i8}
+define void @i8_factor_2(%i8.2* %data, i64 %n) #0 {
+entry:
+  br label %for.body
+
+; VF_2-LABEL:  Checking a loop in "i8_factor_2"
+; VF_2:          Found an estimated cost of 20 for VF 2 For instruction: %tmp2 = load i8, i8* %tmp0, align 1
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp3 = load i8, i8* %tmp1, align 1
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i8 0, i8* %tmp0, align 1
+; VF_2-NEXT:     Found an estimated cost of 12 for VF 2 For instruction: store i8 0, i8* %tmp1, align 1
+; VF_4-LABEL:  Checking a loop in "i8_factor_2"
+; VF_4:          Found an estimated cost of 4 for VF 4 For instruction: %tmp2 = load i8, i8* %tmp0, align 1
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp3 = load i8, i8* %tmp1, align 1
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i8 0, i8* %tmp0, align 1
+; VF_4-NEXT:     Found an estimated cost of 4 for VF 4 For instruction: store i8 0, i8* %tmp1, align 1
+; VF_8-LABEL:  Checking a loop in "i8_factor_2"
+; VF_8:          Found an estimated cost of 4 for VF 8 For instruction: %tmp2 = load i8, i8* %tmp0, align 1
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp3 = load i8, i8* %tmp1, align 1
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i8 0, i8* %tmp0, align 1
+; VF_8-NEXT:     Found an estimated cost of 4 for VF 8 For instruction: store i8 0, i8* %tmp1, align 1
+; VF_16-LABEL: Checking a loop in "i8_factor_2"
+; VF_16:         Found an estimated cost of 4 for VF 16 For instruction: %tmp2 = load i8, i8* %tmp0, align 1
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp3 = load i8, i8* %tmp1, align 1
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i8 0, i8* %tmp0, align 1
+; VF_16-NEXT:    Found an estimated cost of 4 for VF 16 For instruction: store i8 0, i8* %tmp1, align 1
+for.body:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
+  %tmp0 = getelementptr inbounds %i8.2, %i8.2* %data, i64 %i, i32 0
+  %tmp1 = getelementptr inbounds %i8.2, %i8.2* %data, i64 %i, i32 1
+  %tmp2 = load i8, i8* %tmp0, align 1
+  %tmp3 = load i8, i8* %tmp1, align 1
+  store i8 0, i8* %tmp0, align 1
+  store i8 0, i8* %tmp1, align 1
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+%i16.2 = type {i16, i16}
+define void @i16_factor_2(%i16.2* %data, i64 %n) #0 {
+entry:
+  br label %for.body
+
+; VF_2-LABEL:  Checking a loop in "i16_factor_2"
+; VF_2:          Found an estimated cost of 20 for VF 2 For instruction: %tmp2 = load i16, i16* %tmp0, align 2
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp3 = load i16, i16* %tmp1, align 2
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i16 0, i16* %tmp0, align 2
+; VF_2-NEXT:     Found an estimated cost of 12 for VF 2 For instruction: store i16 0, i16* %tmp1, align 2
+; VF_4-LABEL:  Checking a loop in "i16_factor_2"
+; VF_4:          Found an estimated cost of 4 for VF 4 For instruction: %tmp2 = load i16, i16* %tmp0, align 2
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp3 = load i16, i16* %tmp1, align 2
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i16 0, i16* %tmp0, align 2
+; VF_4-NEXT:     Found an estimated cost of 4 for VF 4 For instruction: store i16 0, i16* %tmp1, align 2
+; VF_8-LABEL:  Checking a loop in "i16_factor_2"
+; VF_8:          Found an estimated cost of 4 for VF 8 For instruction: %tmp2 = load i16, i16* %tmp0, align 2
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp3 = load i16, i16* %tmp1, align 2
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i16 0, i16* %tmp0, align 2
+; VF_8-NEXT:     Found an estimated cost of 4 for VF 8 For instruction: store i16 0, i16* %tmp1, align 2
+; VF_16-LABEL: Checking a loop in "i16_factor_2"
+; VF_16:         Found an estimated cost of 8 for VF 16 For instruction: %tmp2 = load i16, i16* %tmp0, align 2
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp3 = load i16, i16* %tmp1, align 2
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i16 0, i16* %tmp0, align 2
+; VF_16-NEXT:    Found an estimated cost of 8 for VF 16 For instruction: store i16 0, i16* %tmp1, align 2
+for.body:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
+  %tmp0 = getelementptr inbounds %i16.2, %i16.2* %data, i64 %i, i32 0
+  %tmp1 = getelementptr inbounds %i16.2, %i16.2* %data, i64 %i, i32 1
+  %tmp2 = load i16, i16* %tmp0, align 2
+  %tmp3 = load i16, i16* %tmp1, align 2
+  store i16 0, i16* %tmp0, align 2
+  store i16 0, i16* %tmp1, align 2
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+%i32.2 = type {i32, i32}
+define void @i32_factor_2(%i32.2* %data, i64 %n) #0 {
+entry:
+  br label %for.body
+
+; VF_2-LABEL:  Checking a loop in "i32_factor_2"
+; VF_2:          Found an estimated cost of 20 for VF 2 For instruction: %tmp2 = load i32, i32* %tmp0, align 4
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp3 = load i32, i32* %tmp1, align 4
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i32 0, i32* %tmp0, align 4
+; VF_2-NEXT:     Found an estimated cost of 12 for VF 2 For instruction: store i32 0, i32* %tmp1, align 4
+; VF_4-LABEL:  Checking a loop in "i32_factor_2"
+; VF_4:          Found an estimated cost of 4 for VF 4 For instruction: %tmp2 = load i32, i32* %tmp0, align 4
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp3 = load i32, i32* %tmp1, align 4
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i32 0, i32* %tmp0, align 4
+; VF_4-NEXT:     Found an estimated cost of 4 for VF 4 For instruction: store i32 0, i32* %tmp1, align 4
+; VF_8-LABEL:  Checking a loop in "i32_factor_2"
+; VF_8:          Found an estimated cost of 8 for VF 8 For instruction: %tmp2 = load i32, i32* %tmp0, align 4
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp3 = load i32, i32* %tmp1, align 4
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i32 0, i32* %tmp0, align 4
+; VF_8-NEXT:     Found an estimated cost of 8 for VF 8 For instruction: store i32 0, i32* %tmp1, align 4
+; VF_16-LABEL: Checking a loop in "i32_factor_2"
+; VF_16:         Found an estimated cost of 16 for VF 16 For instruction: %tmp2 = load i32, i32* %tmp0, align 4
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp3 = load i32, i32* %tmp1, align 4
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i32 0, i32* %tmp0, align 4
+; VF_16-NEXT:    Found an estimated cost of 16 for VF 16 For instruction: store i32 0, i32* %tmp1, align 4
+for.body:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
+  %tmp0 = getelementptr inbounds %i32.2, %i32.2* %data, i64 %i, i32 0
+  %tmp1 = getelementptr inbounds %i32.2, %i32.2* %data, i64 %i, i32 1
+  %tmp2 = load i32, i32* %tmp0, align 4
+  %tmp3 = load i32, i32* %tmp1, align 4
+  store i32 0, i32* %tmp0, align 4
+  store i32 0, i32* %tmp1, align 4
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+%i64.2 = type {i64, i64}
+define void @i64_factor_2(%i64.2* %data, i64 %n) #0 {
+entry:
+  br label %for.body
+
+; VF_2-LABEL:  Checking a loop in "i64_factor_2"
+; VF_2:          Found an estimated cost of 24 for VF 2 For instruction: %tmp2 = load i64, i64* %tmp0, align 8
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp3 = load i64, i64* %tmp1, align 8
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i64 0, i64* %tmp0, align 8
+; VF_2-NEXT:     Found an estimated cost of 16 for VF 2 For instruction: store i64 0, i64* %tmp1, align 8
+; VF_4-LABEL:  Checking a loop in "i64_factor_2"
+; VF_4:          Found an estimated cost of 80 for VF 4 For instruction: %tmp2 = load i64, i64* %tmp0, align 8
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp3 = load i64, i64* %tmp1, align 8
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i64 0, i64* %tmp0, align 8
+; VF_4-NEXT:     Found an estimated cost of 48 for VF 4 For instruction: store i64 0, i64* %tmp1, align 8
+; VF_8-LABEL:  Checking a loop in "i64_factor_2"
+; VF_8:          Found an estimated cost of 288 for VF 8 For instruction: %tmp2 = load i64, i64* %tmp0, align 8
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp3 = load i64, i64* %tmp1, align 8
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i64 0, i64* %tmp0, align 8
+; VF_8-NEXT:     Found an estimated cost of 160 for VF 8 For instruction: store i64 0, i64* %tmp1, align 8
+; VF_16-LABEL: Checking a loop in "i64_factor_2"
+; VF_16:         Found an estimated cost of 1088 for VF 16 For instruction: %tmp2 = load i64, i64* %tmp0, align 8
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp3 = load i64, i64* %tmp1, align 8
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i64 0, i64* %tmp0, align 8
+; VF_16-NEXT:    Found an estimated cost of 576 for VF 16 For instruction: store i64 0, i64* %tmp1, align 8
+for.body:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
+  %tmp0 = getelementptr inbounds %i64.2, %i64.2* %data, i64 %i, i32 0
+  %tmp1 = getelementptr inbounds %i64.2, %i64.2* %data, i64 %i, i32 1
+  %tmp2 = load i64, i64* %tmp0, align 8
+  %tmp3 = load i64, i64* %tmp1, align 8
+  store i64 0, i64* %tmp0, align 8
+  store i64 0, i64* %tmp1, align 8
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+%f16.2 = type {half, half}
+define void @f16_factor_2(%f16.2* %data, i64 %n) #0 {
+entry:
+  br label %for.body
+
+; VF_2-LABEL:  Checking a loop in "f16_factor_2"
+; VF_2:          Found an estimated cost of 20 for VF 2 For instruction: %tmp2 = load half, half* %tmp0, align 2
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp3 = load half, half* %tmp1, align 2
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store half 0xH0000, half* %tmp0, align 2
+; VF_2-NEXT:     Found an estimated cost of 12 for VF 2 For instruction: store half 0xH0000, half* %tmp1, align 2
+; VF_4-LABEL:  Checking a loop in "f16_factor_2"
+; VF_4:          Found an estimated cost of 72 for VF 4 For instruction: %tmp2 = load half, half* %tmp0, align 2
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp3 = load half, half* %tmp1, align 2
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store half 0xH0000, half* %tmp0, align 2
+; VF_4-NEXT:     Found an estimated cost of 40 for VF 4 For instruction: store half 0xH0000, half* %tmp1, align 2
+; VF_8-LABEL:  Checking a loop in "f16_factor_2"
+; VF_8:          Found an estimated cost of 4 for VF 8 For instruction: %tmp2 = load half, half* %tmp0, align 2
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp3 = load half, half* %tmp1, align 2
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store half 0xH0000, half* %tmp0, align 2
+; VF_8-NEXT:     Found an estimated cost of 4 for VF 8 For instruction: store half 0xH0000, half* %tmp1, align 2
+; VF_16-LABEL: Checking a loop in "f16_factor_2"
+; VF_16:         Found an estimated cost of 8 for VF 16 For instruction: %tmp2 = load half, half* %tmp0, align 2
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp3 = load half, half* %tmp1, align 2
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store half 0xH0000, half* %tmp0, align 2
+; VF_16-NEXT:    Found an estimated cost of 8 for VF 16 For instruction: store half 0xH0000, half* %tmp1, align 2
+for.body:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
+  %tmp0 = getelementptr inbounds %f16.2, %f16.2* %data, i64 %i, i32 0
+  %tmp1 = getelementptr inbounds %f16.2, %f16.2* %data, i64 %i, i32 1
+  %tmp2 = load half, half* %tmp0, align 2
+  %tmp3 = load half, half* %tmp1, align 2
+  store half 0.0, half* %tmp0, align 2
+  store half 0.0, half* %tmp1, align 2
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+%f32.2 = type {float, float}
+define void @f32_factor_2(%f32.2* %data, i64 %n) #0 {
+entry:
+  br label %for.body
+
+; VF_2-LABEL:  Checking a loop in "f32_factor_2"
+; VF_2:          Found an estimated cost of 20 for VF 2 For instruction: %tmp2 = load float, float* %tmp0, align 4
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp3 = load float, float* %tmp1, align 4
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store float 0.000000e+00, float* %tmp0, align 4
+; VF_2-NEXT:     Found an estimated cost of 12 for VF 2 For instruction: store float 0.000000e+00, float* %tmp1, align 4
+; VF_4-LABEL:  Checking a loop in "f32_factor_2"
+; VF_4:          Found an estimated cost of 4 for VF 4 For instruction: %tmp2 = load float, float* %tmp0, align 4
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp3 = load float, float* %tmp1, align 4
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store float 0.000000e+00, float* %tmp0, align 4
+; VF_4-NEXT:     Found an estimated cost of 4 for VF 4 For instruction: store float 0.000000e+00, float* %tmp1, align 4
+; VF_8-LABEL:  Checking a loop in "f32_factor_2"
+; VF_8:          Found an estimated cost of 8 for VF 8 For instruction: %tmp2 = load float, float* %tmp0, align 4
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp3 = load float, float* %tmp1, align 4
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store float 0.000000e+00, float* %tmp0, align 4
+; VF_8-NEXT:     Found an estimated cost of 8 for VF 8 For instruction: store float 0.000000e+00, float* %tmp1, align 4
+; VF_16-LABEL: Checking a loop in "f32_factor_2"
+; VF_16:         Found an estimated cost of 16 for VF 16 For instruction: %tmp2 = load float, float* %tmp0, align 4
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp3 = load float, float* %tmp1, align 4
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store float 0.000000e+00, float* %tmp0, align 4
+; VF_16-NEXT:    Found an estimated cost of 16 for VF 16 For instruction: store float 0.000000e+00, float* %tmp1, align 4
+for.body:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
+  %tmp0 = getelementptr inbounds %f32.2, %f32.2* %data, i64 %i, i32 0
+  %tmp1 = getelementptr inbounds %f32.2, %f32.2* %data, i64 %i, i32 1
+  %tmp2 = load float, float* %tmp0, align 4
+  %tmp3 = load float, float* %tmp1, align 4
+  store float 0.0, float* %tmp0, align 4
+  store float 0.0, float* %tmp1, align 4
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+%f64.2 = type {double, double}
+define void @f64_factor_2(%f64.2* %data, i64 %n) #0 {
+entry:
+  br label %for.body
+
+; VF_2-LABEL:  Checking a loop in "f64_factor_2"
+; VF_2:          Found an estimated cost of 20 for VF 2 For instruction: %tmp2 = load double, double* %tmp0, align 8
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp3 = load double, double* %tmp1, align 8
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store double 0.000000e+00, double* %tmp0, align 8
+; VF_2-NEXT:     Found an estimated cost of 12 for VF 2 For instruction: store double 0.000000e+00, double* %tmp1, align 8
+; VF_4-LABEL:  Checking a loop in "f64_factor_2"
+; VF_4:          Found an estimated cost of 72 for VF 4 For instruction: %tmp2 = load double, double* %tmp0, align 8
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp3 = load double, double* %tmp1, align 8
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store double 0.000000e+00, double* %tmp0, align 8
+; VF_4-NEXT:     Found an estimated cost of 40 for VF 4 For instruction: store double 0.000000e+00, double* %tmp1, align 8
+; VF_8-LABEL:  Checking a loop in "f64_factor_2"
+; VF_8:          Found an estimated cost of 272 for VF 8 For instruction: %tmp2 = load double, double* %tmp0, align 8
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp3 = load double, double* %tmp1, align 8
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store double 0.000000e+00, double* %tmp0, align 8
+; VF_8-NEXT:     Found an estimated cost of 144 for VF 8 For instruction: store double 0.000000e+00, double* %tmp1, align 8
+; VF_16-LABEL: Checking a loop in "f64_factor_2"
+; VF_16:         Found an estimated cost of 1056 for VF 16 For instruction: %tmp2 = load double, double* %tmp0, align 8
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp3 = load double, double* %tmp1, align 8
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store double 0.000000e+00, double* %tmp0, align 8
+; VF_16-NEXT:    Found an estimated cost of 544 for VF 16 For instruction: store double 0.000000e+00, double* %tmp1, align 8
+for.body:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
+  %tmp0 = getelementptr inbounds %f64.2, %f64.2* %data, i64 %i, i32 0
+  %tmp1 = getelementptr inbounds %f64.2, %f64.2* %data, i64 %i, i32 1
+  %tmp2 = load double, double* %tmp0, align 8
+  %tmp3 = load double, double* %tmp1, align 8
+  store double 0.0, double* %tmp0, align 8
+  store double 0.0, double* %tmp1, align 8
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+
+
+; Factor 3
+
+%i8.3 = type {i8, i8, i8}
+define void @i8_factor_3(%i8.3* %data, i64 %n) #0 {
+entry:
+  br label %for.body
+
+; VF_2-LABEL:  Checking a loop in "i8_factor_3"
+; VF_2:          Found an estimated cost of 30 for VF 2 For instruction: %tmp3 = load i8, i8* %tmp0, align 1
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp4 = load i8, i8* %tmp1, align 1
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp5 = load i8, i8* %tmp2, align 1
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i8 0, i8* %tmp0, align 1
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i8 0, i8* %tmp1, align 1
+; VF_2-NEXT:     Found an estimated cost of 18 for VF 2 For instruction: store i8 0, i8* %tmp2, align 1
+; VF_4-LABEL:  Checking a loop in "i8_factor_3"
+; VF_4:          Found an estimated cost of 108 for VF 4 For instruction: %tmp3 = load i8, i8* %tmp0, align 1
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp4 = load i8, i8* %tmp1, align 1
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp5 = load i8, i8* %tmp2, align 1
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i8 0, i8* %tmp0, align 1
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i8 0, i8* %tmp1, align 1
+; VF_4-NEXT:     Found an estimated cost of 60 for VF 4 For instruction: store i8 0, i8* %tmp2, align 1
+; VF_8-LABEL:  Checking a loop in "i8_factor_3"
+; VF_8:          Found an estimated cost of 408 for VF 8 For instruction: %tmp3 = load i8, i8* %tmp0, align 1
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp4 = load i8, i8* %tmp1, align 1
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp5 = load i8, i8* %tmp2, align 1
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i8 0, i8* %tmp0, align 1
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i8 0, i8* %tmp1, align 1
+; VF_8-NEXT:     Found an estimated cost of 216 for VF 8 For instruction: store i8 0, i8* %tmp2, align 1
+; VF_16-LABEL: Checking a loop in "i8_factor_3"
+; VF_16:         Found an estimated cost of 1584 for VF 16 For instruction: %tmp3 = load i8, i8* %tmp0, align 1
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp4 = load i8, i8* %tmp1, align 1
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp5 = load i8, i8* %tmp2, align 1
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i8 0, i8* %tmp0, align 1
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i8 0, i8* %tmp1, align 1
+; VF_16-NEXT:    Found an estimated cost of 816 for VF 16 For instruction: store i8 0, i8* %tmp2, align 1
+for.body:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
+  %tmp0 = getelementptr inbounds %i8.3, %i8.3* %data, i64 %i, i32 0
+  %tmp1 = getelementptr inbounds %i8.3, %i8.3* %data, i64 %i, i32 1
+  %tmp2 = getelementptr inbounds %i8.3, %i8.3* %data, i64 %i, i32 2
+  %tmp3 = load i8, i8* %tmp0, align 1
+  %tmp4 = load i8, i8* %tmp1, align 1
+  %tmp5 = load i8, i8* %tmp2, align 1
+  store i8 0, i8* %tmp0, align 1
+  store i8 0, i8* %tmp1, align 1
+  store i8 0, i8* %tmp2, align 1
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+%i16.3 = type {i16, i16, i16}
+define void @i16_factor_3(%i16.3* %data, i64 %n) #0 {
+entry:
+  br label %for.body
+
+; VF_2-LABEL:  Checking a loop in "i16_factor_3"
+; VF_2:          Found an estimated cost of 30 for VF 2 For instruction: %tmp3 = load i16, i16* %tmp0, align 2
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp4 = load i16, i16* %tmp1, align 2
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp5 = load i16, i16* %tmp2, align 2
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i16 0, i16* %tmp0, align 2
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i16 0, i16* %tmp1, align 2
+; VF_2-NEXT:     Found an estimated cost of 18 for VF 2 For instruction: store i16 0, i16* %tmp2, align 2
+; VF_4-LABEL:  Checking a loop in "i16_factor_3"
+; VF_4:          Found an estimated cost of 108 for VF 4 For instruction: %tmp3 = load i16, i16* %tmp0, align 2
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp4 = load i16, i16* %tmp1, align 2
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp5 = load i16, i16* %tmp2, align 2
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i16 0, i16* %tmp0, align 2
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i16 0, i16* %tmp1, align 2
+; VF_4-NEXT:     Found an estimated cost of 60 for VF 4 For instruction: store i16 0, i16* %tmp2, align 2
+; VF_8-LABEL:  Checking a loop in "i16_factor_3"
+; VF_8:          Found an estimated cost of 408 for VF 8 For instruction: %tmp3 = load i16, i16* %tmp0, align 2
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp4 = load i16, i16* %tmp1, align 2
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp5 = load i16, i16* %tmp2, align 2
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i16 0, i16* %tmp0, align 2
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i16 0, i16* %tmp1, align 2
+; VF_8-NEXT:     Found an estimated cost of 216 for VF 8 For instruction: store i16 0, i16* %tmp2, align 2
+; VF_16-LABEL: Checking a loop in "i16_factor_3"
+; VF_16:         Found an estimated cost of 1584 for VF 16 For instruction: %tmp3 = load i16, i16* %tmp0, align 2
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp4 = load i16, i16* %tmp1, align 2
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp5 = load i16, i16* %tmp2, align 2
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i16 0, i16* %tmp0, align 2
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i16 0, i16* %tmp1, align 2
+; VF_16-NEXT:    Found an estimated cost of 816 for VF 16 For instruction: store i16 0, i16* %tmp2, align 2
+for.body:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
+  %tmp0 = getelementptr inbounds %i16.3, %i16.3* %data, i64 %i, i32 0
+  %tmp1 = getelementptr inbounds %i16.3, %i16.3* %data, i64 %i, i32 1
+  %tmp2 = getelementptr inbounds %i16.3, %i16.3* %data, i64 %i, i32 2
+  %tmp3 = load i16, i16* %tmp0, align 2
+  %tmp4 = load i16, i16* %tmp1, align 2
+  %tmp5 = load i16, i16* %tmp2, align 2
+  store i16 0, i16* %tmp0, align 2
+  store i16 0, i16* %tmp1, align 2
+  store i16 0, i16* %tmp2, align 2
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+%i32.3 = type {i32, i32, i32}
+define void @i32_factor_3(%i32.3* %data, i64 %n) #0 {
+entry:
+  br label %for.body
+
+; VF_2-LABEL:  Checking a loop in "i32_factor_3"
+; VF_2:          Found an estimated cost of 30 for VF 2 For instruction: %tmp3 = load i32, i32* %tmp0, align 4
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp4 = load i32, i32* %tmp1, align 4
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp5 = load i32, i32* %tmp2, align 4
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i32 0, i32* %tmp0, align 4
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i32 0, i32* %tmp1, align 4
+; VF_2-NEXT:     Found an estimated cost of 18 for VF 2 For instruction: store i32 0, i32* %tmp2, align 4
+; VF_4-LABEL:  Checking a loop in "i32_factor_3"
+; VF_4:          Found an estimated cost of 108 for VF 4 For instruction: %tmp3 = load i32, i32* %tmp0, align 4
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp4 = load i32, i32* %tmp1, align 4
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp5 = load i32, i32* %tmp2, align 4
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i32 0, i32* %tmp0, align 4
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i32 0, i32* %tmp1, align 4
+; VF_4-NEXT:     Found an estimated cost of 60 for VF 4 For instruction: store i32 0, i32* %tmp2, align 4
+; VF_8-LABEL:  Checking a loop in "i32_factor_3"
+; VF_8:          Found an estimated cost of 408 for VF 8 For instruction: %tmp3 = load i32, i32* %tmp0, align 4
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp4 = load i32, i32* %tmp1, align 4
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp5 = load i32, i32* %tmp2, align 4
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i32 0, i32* %tmp0, align 4
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i32 0, i32* %tmp1, align 4
+; VF_8-NEXT:     Found an estimated cost of 216 for VF 8 For instruction: store i32 0, i32* %tmp2, align 4
+; VF_16-LABEL: Checking a loop in "i32_factor_3"
+; VF_16:         Found an estimated cost of 1584 for VF 16 For instruction: %tmp3 = load i32, i32* %tmp0, align 4
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp4 = load i32, i32* %tmp1, align 4
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp5 = load i32, i32* %tmp2, align 4
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i32 0, i32* %tmp0, align 4
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i32 0, i32* %tmp1, align 4
+; VF_16-NEXT:    Found an estimated cost of 816 for VF 16 For instruction: store i32 0, i32* %tmp2, align 4
+for.body:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
+  %tmp0 = getelementptr inbounds %i32.3, %i32.3* %data, i64 %i, i32 0
+  %tmp1 = getelementptr inbounds %i32.3, %i32.3* %data, i64 %i, i32 1
+  %tmp2 = getelementptr inbounds %i32.3, %i32.3* %data, i64 %i, i32 2
+  %tmp3 = load i32, i32* %tmp0, align 4
+  %tmp4 = load i32, i32* %tmp1, align 4
+  %tmp5 = load i32, i32* %tmp2, align 4
+  store i32 0, i32* %tmp0, align 4
+  store i32 0, i32* %tmp1, align 4
+  store i32 0, i32* %tmp2, align 4
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+%i64.3 = type {i64, i64, i64}
+define void @i64_factor_3(%i64.3* %data, i64 %n) #0 {
+entry:
+  br label %for.body
+
+; VF_2-LABEL:  Checking a loop in "i64_factor_3"
+; VF_2:          Found an estimated cost of 36 for VF 2 For instruction: %tmp3 = load i64, i64* %tmp0, align 8
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp4 = load i64, i64* %tmp1, align 8
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp5 = load i64, i64* %tmp2, align 8
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i64 0, i64* %tmp0, align 8
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i64 0, i64* %tmp1, align 8
+; VF_2-NEXT:     Found an estimated cost of 24 for VF 2 For instruction: store i64 0, i64* %tmp2, align 8
+; VF_4-LABEL:  Checking a loop in "i64_factor_3"
+; VF_4:          Found an estimated cost of 120 for VF 4 For instruction: %tmp3 = load i64, i64* %tmp0, align 8
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp4 = load i64, i64* %tmp1, align 8
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp5 = load i64, i64* %tmp2, align 8
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i64 0, i64* %tmp0, align 8
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i64 0, i64* %tmp1, align 8
+; VF_4-NEXT:     Found an estimated cost of 72 for VF 4 For instruction: store i64 0, i64* %tmp2, align 8
+; VF_8-LABEL:  Checking a loop in "i64_factor_3"
+; VF_8:          Found an estimated cost of 432 for VF 8 For instruction: %tmp3 = load i64, i64* %tmp0, align 8
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp4 = load i64, i64* %tmp1, align 8
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp5 = load i64, i64* %tmp2, align 8
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i64 0, i64* %tmp0, align 8
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i64 0, i64* %tmp1, align 8
+; VF_8-NEXT:     Found an estimated cost of 240 for VF 8 For instruction: store i64 0, i64* %tmp2, align 8
+; VF_16-LABEL: Checking a loop in "i64_factor_3"
+; VF_16:         Found an estimated cost of 1632 for VF 16 For instruction: %tmp3 = load i64, i64* %tmp0, align 8
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp4 = load i64, i64* %tmp1, align 8
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp5 = load i64, i64* %tmp2, align 8
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i64 0, i64* %tmp0, align 8
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i64 0, i64* %tmp1, align 8
+; VF_16-NEXT:    Found an estimated cost of 864 for VF 16 For instruction: store i64 0, i64* %tmp2, align 8
+for.body:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
+  %tmp0 = getelementptr inbounds %i64.3, %i64.3* %data, i64 %i, i32 0
+  %tmp1 = getelementptr inbounds %i64.3, %i64.3* %data, i64 %i, i32 1
+  %tmp2 = getelementptr inbounds %i64.3, %i64.3* %data, i64 %i, i32 2
+  %tmp3 = load i64, i64* %tmp0, align 8
+  %tmp4 = load i64, i64* %tmp1, align 8
+  %tmp5 = load i64, i64* %tmp2, align 8
+  store i64 0, i64* %tmp0, align 8
+  store i64 0, i64* %tmp1, align 8
+  store i64 0, i64* %tmp2, align 8
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+%f16.3 = type {half, half, half}
+define void @f16_factor_3(%f16.3* %data, i64 %n) #0 {
+entry:
+  br label %for.body
+
+; VF_2-LABEL:  Checking a loop in "f16_factor_3"
+; VF_2:          Found an estimated cost of 30 for VF 2 For instruction: %tmp3 = load half, half* %tmp0, align 2
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp4 = load half, half* %tmp1, align 2
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp5 = load half, half* %tmp2, align 2
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store half 0xH0000, half* %tmp0, align 2
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store half 0xH0000, half* %tmp1, align 2
+; VF_2-NEXT:     Found an estimated cost of 18 for VF 2 For instruction: store half 0xH0000, half* %tmp2, align 2
+; VF_4-LABEL:  Checking a loop in "f16_factor_3"
+; VF_4:          Found an estimated cost of 108 for VF 4 For instruction: %tmp3 = load half, half* %tmp0, align 2
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp4 = load half, half* %tmp1, align 2
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp5 = load half, half* %tmp2, align 2
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store half 0xH0000, half* %tmp0, align 2
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store half 0xH0000, half* %tmp1, align 2
+; VF_4-NEXT:     Found an estimated cost of 60 for VF 4 For instruction: store half 0xH0000, half* %tmp2, align 2
+; VF_8-LABEL:  Checking a loop in "f16_factor_3"
+; VF_8:          Found an estimated cost of 408 for VF 8 For instruction: %tmp3 = load half, half* %tmp0, align 2
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp4 = load half, half* %tmp1, align 2
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp5 = load half, half* %tmp2, align 2
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store half 0xH0000, half* %tmp0, align 2
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store half 0xH0000, half* %tmp1, align 2
+; VF_8-NEXT:     Found an estimated cost of 216 for VF 8 For instruction: store half 0xH0000, half* %tmp2, align 2
+; VF_16-LABEL: Checking a loop in "f16_factor_3"
+; VF_16:         Found an estimated cost of 1584 for VF 16 For instruction: %tmp3 = load half, half* %tmp0, align 2
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp4 = load half, half* %tmp1, align 2
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp5 = load half, half* %tmp2, align 2
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store half 0xH0000, half* %tmp0, align 2
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store half 0xH0000, half* %tmp1, align 2
+; VF_16-NEXT:    Found an estimated cost of 816 for VF 16 For instruction: store half 0xH0000, half* %tmp2, align 2
+for.body:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
+  %tmp0 = getelementptr inbounds %f16.3, %f16.3* %data, i64 %i, i32 0
+  %tmp1 = getelementptr inbounds %f16.3, %f16.3* %data, i64 %i, i32 1
+  %tmp2 = getelementptr inbounds %f16.3, %f16.3* %data, i64 %i, i32 2
+  %tmp3 = load half, half* %tmp0, align 2
+  %tmp4 = load half, half* %tmp1, align 2
+  %tmp5 = load half, half* %tmp2, align 2
+  store half 0.0, half* %tmp0, align 2
+  store half 0.0, half* %tmp1, align 2
+  store half 0.0, half* %tmp2, align 2
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+%f32.3 = type {float, float, float}
+define void @f32_factor_3(%f32.3* %data, i64 %n) #0 {
+entry:
+  br label %for.body
+
+; VF_2-LABEL:  Checking a loop in "f32_factor_3"
+; VF_2:          Found an estimated cost of 30 for VF 2 For instruction: %tmp3 = load float, float* %tmp0, align 4
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp4 = load float, float* %tmp1, align 4
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp5 = load float, float* %tmp2, align 4
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store float 0.000000e+00, float* %tmp0, align 4
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store float 0.000000e+00, float* %tmp1, align 4
+; VF_2-NEXT:     Found an estimated cost of 18 for VF 2 For instruction: store float 0.000000e+00, float* %tmp2, align 4
+; VF_4-LABEL:  Checking a loop in "f32_factor_3"
+; VF_4:          Found an estimated cost of 108 for VF 4 For instruction: %tmp3 = load float, float* %tmp0, align 4
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp4 = load float, float* %tmp1, align 4
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp5 = load float, float* %tmp2, align 4
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store float 0.000000e+00, float* %tmp0, align 4
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store float 0.000000e+00, float* %tmp1, align 4
+; VF_4-NEXT:     Found an estimated cost of 60 for VF 4 For instruction: store float 0.000000e+00, float* %tmp2, align 4
+; VF_8-LABEL:  Checking a loop in "f32_factor_3"
+; VF_8:          Found an estimated cost of 408 for VF 8 For instruction: %tmp3 = load float, float* %tmp0, align 4
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp4 = load float, float* %tmp1, align 4
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp5 = load float, float* %tmp2, align 4
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store float 0.000000e+00, float* %tmp0, align 4
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store float 0.000000e+00, float* %tmp1, align 4
+; VF_8-NEXT:     Found an estimated cost of 216 for VF 8 For instruction: store float 0.000000e+00, float* %tmp2, align 4
+; VF_16-LABEL: Checking a loop in "f32_factor_3"
+; VF_16:         Found an estimated cost of 1584 for VF 16 For instruction: %tmp3 = load float, float* %tmp0, align 4
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp4 = load float, float* %tmp1, align 4
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp5 = load float, float* %tmp2, align 4
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store float 0.000000e+00, float* %tmp0, align 4
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store float 0.000000e+00, float* %tmp1, align 4
+; VF_16-NEXT:    Found an estimated cost of 816 for VF 16 For instruction: store float 0.000000e+00, float* %tmp2, align 4
+for.body:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
+  %tmp0 = getelementptr inbounds %f32.3, %f32.3* %data, i64 %i, i32 0
+  %tmp1 = getelementptr inbounds %f32.3, %f32.3* %data, i64 %i, i32 1
+  %tmp2 = getelementptr inbounds %f32.3, %f32.3* %data, i64 %i, i32 2
+  %tmp3 = load float, float* %tmp0, align 4
+  %tmp4 = load float, float* %tmp1, align 4
+  %tmp5 = load float, float* %tmp2, align 4
+  store float 0.0, float* %tmp0, align 4
+  store float 0.0, float* %tmp1, align 4
+  store float 0.0, float* %tmp2, align 4
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+%f64.3 = type {double, double, double}
+define void @f64_factor_3(%f64.3* %data, i64 %n) #0 {
+entry:
+  br label %for.body
+
+; VF_2-LABEL:  Checking a loop in "f64_factor_3"
+; VF_2:          Found an estimated cost of 30 for VF 2 For instruction: %tmp3 = load double, double* %tmp0, align 8
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp4 = load double, double* %tmp1, align 8
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp5 = load double, double* %tmp2, align 8
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store double 0.000000e+00, double* %tmp0, align 8
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store double 0.000000e+00, double* %tmp1, align 8
+; VF_2-NEXT:     Found an estimated cost of 18 for VF 2 For instruction: store double 0.000000e+00, double* %tmp2, align 8
+; VF_4-LABEL:  Checking a loop in "f64_factor_3"
+; VF_4:          Found an estimated cost of 108 for VF 4 For instruction: %tmp3 = load double, double* %tmp0, align 8
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp4 = load double, double* %tmp1, align 8
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp5 = load double, double* %tmp2, align 8
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store double 0.000000e+00, double* %tmp0, align 8
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store double 0.000000e+00, double* %tmp1, align 8
+; VF_4-NEXT:     Found an estimated cost of 60 for VF 4 For instruction: store double 0.000000e+00, double* %tmp2, align 8
+; VF_8-LABEL:  Checking a loop in "f64_factor_3"
+; VF_8:          Found an estimated cost of 408 for VF 8 For instruction: %tmp3 = load double, double* %tmp0, align 8
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp4 = load double, double* %tmp1, align 8
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp5 = load double, double* %tmp2, align 8
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store double 0.000000e+00, double* %tmp0, align 8
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store double 0.000000e+00, double* %tmp1, align 8
+; VF_8-NEXT:     Found an estimated cost of 216 for VF 8 For instruction: store double 0.000000e+00, double* %tmp2, align 8
+; VF_16-LABEL: Checking a loop in "f64_factor_3"
+; VF_16:         Found an estimated cost of 1584 for VF 16 For instruction: %tmp3 = load double, double* %tmp0, align 8
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp4 = load double, double* %tmp1, align 8
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp5 = load double, double* %tmp2, align 8
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store double 0.000000e+00, double* %tmp0, align 8
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store double 0.000000e+00, double* %tmp1, align 8
+; VF_16-NEXT:    Found an estimated cost of 816 for VF 16 For instruction: store double 0.000000e+00, double* %tmp2, align 8
+for.body:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
+  %tmp0 = getelementptr inbounds %f64.3, %f64.3* %data, i64 %i, i32 0
+  %tmp1 = getelementptr inbounds %f64.3, %f64.3* %data, i64 %i, i32 1
+  %tmp2 = getelementptr inbounds %f64.3, %f64.3* %data, i64 %i, i32 2
+  %tmp3 = load double, double* %tmp0, align 8
+  %tmp4 = load double, double* %tmp1, align 8
+  %tmp5 = load double, double* %tmp2, align 8
+  store double 0.0, double* %tmp0, align 8
+  store double 0.0, double* %tmp1, align 8
+  store double 0.0, double* %tmp2, align 8
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+
+; Factor 4
+
+%i8.4 = type {i8, i8, i8, i8}
+define void @i8_factor_4(%i8.4* %data, i64 %n) #0 {
+entry:
+  br label %for.body
+
+; VF_2-LABEL:  Checking a loop in "i8_factor_4"
+; VF_2:          Found an estimated cost of 40 for VF 2 For instruction: %tmp4 = load i8, i8* %tmp0, align 1
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp5 = load i8, i8* %tmp1, align 1
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp6 = load i8, i8* %tmp2, align 1
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp7 = load i8, i8* %tmp3, align 1
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i8 0, i8* %tmp0, align 1
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i8 0, i8* %tmp1, align 1
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i8 0, i8* %tmp2, align 1
+; VF_2-NEXT:     Found an estimated cost of 24 for VF 2 For instruction: store i8 0, i8* %tmp3, align 1
+; VF_4-LABEL: Checking a loop in "i8_factor_4"
+; VF_4:         Found an estimated cost of 144 for VF 4 For instruction: %tmp4 = load i8, i8* %tmp0, align 1
+; VF_4-NEXT:    Found an estimated cost of 0 for VF 4 For instruction: %tmp5 = load i8, i8* %tmp1, align 1
+; VF_4-NEXT:    Found an estimated cost of 0 for VF 4 For instruction: %tmp6 = load i8, i8* %tmp2, align 1
+; VF_4-NEXT:    Found an estimated cost of 0 for VF 4 For instruction: %tmp7 = load i8, i8* %tmp3, align 1
+; VF_4-NEXT:    Found an estimated cost of 0 for VF 4 For instruction: store i8 0, i8* %tmp0, align 1
+; VF_4-NEXT:    Found an estimated cost of 0 for VF 4 For instruction: store i8 0, i8* %tmp1, align 1
+; VF_4-NEXT:    Found an estimated cost of 0 for VF 4 For instruction: store i8 0, i8* %tmp2, align 1
+; VF_4-NEXT:    Found an estimated cost of 80 for VF 4 For instruction: store i8 0, i8* %tmp3, align 1
+; VF_8-LABEL:  Checking a loop in "i8_factor_4"
+; VF_8:          Found an estimated cost of 544 for VF 8 For instruction: %tmp4 = load i8, i8* %tmp0, align 1
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp5 = load i8, i8* %tmp1, align 1
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp6 = load i8, i8* %tmp2, align 1
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp7 = load i8, i8* %tmp3, align 1
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i8 0, i8* %tmp0, align 1
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i8 0, i8* %tmp1, align 1
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i8 0, i8* %tmp2, align 1
+; VF_8-NEXT:     Found an estimated cost of 288 for VF 8 For instruction: store i8 0, i8* %tmp3, align 1
+; VF_16-LABEL: Checking a loop in "i8_factor_4"
+; VF_16:         Found an estimated cost of 2112 for VF 16 For instruction: %tmp4 = load i8, i8* %tmp0, align 1
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp5 = load i8, i8* %tmp1, align 1
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp6 = load i8, i8* %tmp2, align 1
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp7 = load i8, i8* %tmp3, align 1
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i8 0, i8* %tmp0, align 1
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i8 0, i8* %tmp1, align 1
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i8 0, i8* %tmp2, align 1
+; VF_16-NEXT:    Found an estimated cost of 1088 for VF 16 For instruction: store i8 0, i8* %tmp3, align 1
+for.body:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
+  %tmp0 = getelementptr inbounds %i8.4, %i8.4* %data, i64 %i, i32 0
+  %tmp1 = getelementptr inbounds %i8.4, %i8.4* %data, i64 %i, i32 1
+  %tmp2 = getelementptr inbounds %i8.4, %i8.4* %data, i64 %i, i32 2
+  %tmp3 = getelementptr inbounds %i8.4, %i8.4* %data, i64 %i, i32 3
+  %tmp4 = load i8, i8* %tmp0, align 1
+  %tmp5 = load i8, i8* %tmp1, align 1
+  %tmp6 = load i8, i8* %tmp2, align 1
+  %tmp7 = load i8, i8* %tmp3, align 1
+  store i8 0, i8* %tmp0, align 1
+  store i8 0, i8* %tmp1, align 1
+  store i8 0, i8* %tmp2, align 1
+  store i8 0, i8* %tmp3, align 1
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+%i16.4 = type {i16, i16, i16, i16}
+define void @i16_factor_4(%i16.4* %data, i64 %n) #0 {
+entry:
+  br label %for.body
+
+; VF_2-LABEL:  Checking a loop in "i16_factor_4"
+; VF_2:          Found an estimated cost of 40 for VF 2 For instruction: %tmp4 = load i16, i16* %tmp0, align 2
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp5 = load i16, i16* %tmp1, align 2
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp6 = load i16, i16* %tmp2, align 2
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp7 = load i16, i16* %tmp3, align 2
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i16 0, i16* %tmp0, align 2
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i16 0, i16* %tmp1, align 2
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i16 0, i16* %tmp2, align 2
+; VF_2-NEXT:     Found an estimated cost of 24 for VF 2 For instruction: store i16 0, i16* %tmp3, align 2
+; VF_4-LABEL:  Checking a loop in "i16_factor_4"
+; VF_4:          Found an estimated cost of 144 for VF 4 For instruction: %tmp4 = load i16, i16* %tmp0, align 2
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp5 = load i16, i16* %tmp1, align 2
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp6 = load i16, i16* %tmp2, align 2
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp7 = load i16, i16* %tmp3, align 2
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i16 0, i16* %tmp0, align 2
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i16 0, i16* %tmp1, align 2
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i16 0, i16* %tmp2, align 2
+; VF_4-NEXT:     Found an estimated cost of 80 for VF 4 For instruction: store i16 0, i16* %tmp3, align 2
+; VF_8-LABEL:  Checking a loop in "i16_factor_4"
+; VF_8:          Found an estimated cost of 544 for VF 8 For instruction: %tmp4 = load i16, i16* %tmp0, align 2
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp5 = load i16, i16* %tmp1, align 2
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp6 = load i16, i16* %tmp2, align 2
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp7 = load i16, i16* %tmp3, align 2
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i16 0, i16* %tmp0, align 2
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i16 0, i16* %tmp1, align 2
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i16 0, i16* %tmp2, align 2
+; VF_8-NEXT:     Found an estimated cost of 288 for VF 8 For instruction: store i16 0, i16* %tmp3, align 2
+; VF_16-LABEL: Checking a loop in "i16_factor_4"
+; VF_16:         Found an estimated cost of 2112 for VF 16 For instruction: %tmp4 = load i16, i16* %tmp0, align 2
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp5 = load i16, i16* %tmp1, align 2
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp6 = load i16, i16* %tmp2, align 2
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp7 = load i16, i16* %tmp3, align 2
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i16 0, i16* %tmp0, align 2
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i16 0, i16* %tmp1, align 2
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i16 0, i16* %tmp2, align 2
+; VF_16-NEXT:    Found an estimated cost of 1088 for VF 16 For instruction: store i16 0, i16* %tmp3, align 2
+for.body:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
+  %tmp0 = getelementptr inbounds %i16.4, %i16.4* %data, i64 %i, i32 0
+  %tmp1 = getelementptr inbounds %i16.4, %i16.4* %data, i64 %i, i32 1
+  %tmp2 = getelementptr inbounds %i16.4, %i16.4* %data, i64 %i, i32 2
+  %tmp3 = getelementptr inbounds %i16.4, %i16.4* %data, i64 %i, i32 3
+  %tmp4 = load i16, i16* %tmp0, align 2
+  %tmp5 = load i16, i16* %tmp1, align 2
+  %tmp6 = load i16, i16* %tmp2, align 2
+  %tmp7 = load i16, i16* %tmp3, align 2
+  store i16 0, i16* %tmp0, align 2
+  store i16 0, i16* %tmp1, align 2
+  store i16 0, i16* %tmp2, align 2
+  store i16 0, i16* %tmp3, align 2
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+%i32.4 = type {i32, i32, i32, i32}
+define void @i32_factor_4(%i32.4* %data, i64 %n) #0 {
+entry:
+  br label %for.body
+
+; VF_2-LABEL:  Checking a loop in "i32_factor_4"
+; VF_2:          Found an estimated cost of 40 for VF 2 For instruction: %tmp4 = load i32, i32* %tmp0, align 4
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp5 = load i32, i32* %tmp1, align 4
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp6 = load i32, i32* %tmp2, align 4
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp7 = load i32, i32* %tmp3, align 4
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i32 0, i32* %tmp0, align 4
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i32 0, i32* %tmp1, align 4
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i32 0, i32* %tmp2, align 4
+; VF_2-NEXT:     Found an estimated cost of 24 for VF 2 For instruction: store i32 0, i32* %tmp3, align 4
+; VF_4-LABEL:  Checking a loop in "i32_factor_4"
+; VF_4:          Found an estimated cost of 144 for VF 4 For instruction: %tmp4 = load i32, i32* %tmp0, align 4
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp5 = load i32, i32* %tmp1, align 4
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp6 = load i32, i32* %tmp2, align 4
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp7 = load i32, i32* %tmp3, align 4
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i32 0, i32* %tmp0, align 4
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i32 0, i32* %tmp1, align 4
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i32 0, i32* %tmp2, align 4
+; VF_4-NEXT:     Found an estimated cost of 80 for VF 4 For instruction: store i32 0, i32* %tmp3, align 4
+; VF_8-LABEL:  Checking a loop in "i32_factor_4"
+; VF_8:          Found an estimated cost of 544 for VF 8 For instruction: %tmp4 = load i32, i32* %tmp0, align 4
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp5 = load i32, i32* %tmp1, align 4
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp6 = load i32, i32* %tmp2, align 4
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp7 = load i32, i32* %tmp3, align 4
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i32 0, i32* %tmp0, align 4
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i32 0, i32* %tmp1, align 4
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i32 0, i32* %tmp2, align 4
+; VF_8-NEXT:     Found an estimated cost of 288 for VF 8 For instruction: store i32 0, i32* %tmp3, align 4
+; VF_16-LABEL: Checking a loop in "i32_factor_4"
+; VF_16:         Found an estimated cost of 2112 for VF 16 For instruction: %tmp4 = load i32, i32* %tmp0, align 4
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp5 = load i32, i32* %tmp1, align 4
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp6 = load i32, i32* %tmp2, align 4
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp7 = load i32, i32* %tmp3, align 4
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i32 0, i32* %tmp0, align 4
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i32 0, i32* %tmp1, align 4
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i32 0, i32* %tmp2, align 4
+; VF_16-NEXT:    Found an estimated cost of 1088 for VF 16 For instruction: store i32 0, i32* %tmp3, align 4
+for.body:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
+  %tmp0 = getelementptr inbounds %i32.4, %i32.4* %data, i64 %i, i32 0
+  %tmp1 = getelementptr inbounds %i32.4, %i32.4* %data, i64 %i, i32 1
+  %tmp2 = getelementptr inbounds %i32.4, %i32.4* %data, i64 %i, i32 2
+  %tmp3 = getelementptr inbounds %i32.4, %i32.4* %data, i64 %i, i32 3
+  %tmp4 = load i32, i32* %tmp0, align 4
+  %tmp5 = load i32, i32* %tmp1, align 4
+  %tmp6 = load i32, i32* %tmp2, align 4
+  %tmp7 = load i32, i32* %tmp3, align 4
+  store i32 0, i32* %tmp0, align 4
+  store i32 0, i32* %tmp1, align 4
+  store i32 0, i32* %tmp2, align 4
+  store i32 0, i32* %tmp3, align 4
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+%i64.4 = type {i64, i64, i64, i64}
+define void @i64_factor_4(%i64.4* %data, i64 %n) #0 {
+entry:
+  br label %for.body
+
+; VF_2-LABEL:  Checking a loop in "i64_factor_4"
+; VF_2:          Found an estimated cost of 48 for VF 2 For instruction: %tmp4 = load i64, i64* %tmp0, align 8
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp5 = load i64, i64* %tmp1, align 8
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp6 = load i64, i64* %tmp2, align 8
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp7 = load i64, i64* %tmp3, align 8
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i64 0, i64* %tmp0, align 8
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i64 0, i64* %tmp1, align 8
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i64 0, i64* %tmp2, align 8
+; VF_2-NEXT:     Found an estimated cost of 32 for VF 2 For instruction: store i64 0, i64* %tmp3, align 8
+; VF_4-LABEL:  Checking a loop in "i64_factor_4"
+; VF_4:          Found an estimated cost of 160 for VF 4 For instruction: %tmp4 = load i64, i64* %tmp0, align 8
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp5 = load i64, i64* %tmp1, align 8
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp6 = load i64, i64* %tmp2, align 8
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp7 = load i64, i64* %tmp3, align 8
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i64 0, i64* %tmp0, align 8
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i64 0, i64* %tmp1, align 8
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i64 0, i64* %tmp2, align 8
+; VF_4-NEXT:     Found an estimated cost of 96 for VF 4 For instruction: store i64 0, i64* %tmp3, align 8
+; VF_8-LABEL:  Checking a loop in "i64_factor_4"
+; VF_8:          Found an estimated cost of 576 for VF 8 For instruction: %tmp4 = load i64, i64* %tmp0, align 8
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp5 = load i64, i64* %tmp1, align 8
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp6 = load i64, i64* %tmp2, align 8
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp7 = load i64, i64* %tmp3, align 8
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i64 0, i64* %tmp0, align 8
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i64 0, i64* %tmp1, align 8
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i64 0, i64* %tmp2, align 8
+; VF_8-NEXT:     Found an estimated cost of 320 for VF 8 For instruction: store i64 0, i64* %tmp3, align 8
+; VF_16-LABEL: Checking a loop in "i64_factor_4"
+; VF_16:         Found an estimated cost of 2176 for VF 16 For instruction: %tmp4 = load i64, i64* %tmp0, align 8
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp5 = load i64, i64* %tmp1, align 8
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp6 = load i64, i64* %tmp2, align 8
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp7 = load i64, i64* %tmp3, align 8
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i64 0, i64* %tmp0, align 8
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i64 0, i64* %tmp1, align 8
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i64 0, i64* %tmp2, align 8
+; VF_16-NEXT:    Found an estimated cost of 1152 for VF 16 For instruction: store i64 0, i64* %tmp3, align 8
+for.body:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
+  %tmp0 = getelementptr inbounds %i64.4, %i64.4* %data, i64 %i, i32 0
+  %tmp1 = getelementptr inbounds %i64.4, %i64.4* %data, i64 %i, i32 1
+  %tmp2 = getelementptr inbounds %i64.4, %i64.4* %data, i64 %i, i32 2
+  %tmp3 = getelementptr inbounds %i64.4, %i64.4* %data, i64 %i, i32 3
+  %tmp4 = load i64, i64* %tmp0, align 8
+  %tmp5 = load i64, i64* %tmp1, align 8
+  %tmp6 = load i64, i64* %tmp2, align 8
+  %tmp7 = load i64, i64* %tmp3, align 8
+  store i64 0, i64* %tmp0, align 8
+  store i64 0, i64* %tmp1, align 8
+  store i64 0, i64* %tmp2, align 8
+  store i64 0, i64* %tmp3, align 8
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+%f16.4 = type {half, half, half, half}
+define void @f16_factor_4(%f16.4* %data, i64 %n) #0 {
+entry:
+  br label %for.body
+
+; VF_2-LABEL:  Checking a loop in "f16_factor_4"
+; VF_2:          Found an estimated cost of 40 for VF 2 For instruction: %tmp4 = load half, half* %tmp0, align 2
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp5 = load half, half* %tmp1, align 2
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp6 = load half, half* %tmp2, align 2
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp7 = load half, half* %tmp3, align 2
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store half 0xH0000, half* %tmp0, align 2
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store half 0xH0000, half* %tmp1, align 2
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store half 0xH0000, half* %tmp2, align 2
+; VF_2-NEXT:     Found an estimated cost of 24 for VF 2 For instruction: store half 0xH0000, half* %tmp3, align 2
+; VF_4-LABEL:  Checking a loop in "f16_factor_4"
+; VF_4:          Found an estimated cost of 144 for VF 4 For instruction: %tmp4 = load half, half* %tmp0, align 2
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp5 = load half, half* %tmp1, align 2
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp6 = load half, half* %tmp2, align 2
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp7 = load half, half* %tmp3, align 2
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store half 0xH0000, half* %tmp0, align 2
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store half 0xH0000, half* %tmp1, align 2
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store half 0xH0000, half* %tmp2, align 2
+; VF_4-NEXT:     Found an estimated cost of 80 for VF 4 For instruction: store half 0xH0000, half* %tmp3, align 2
+; VF_8-LABEL:  Checking a loop in "f16_factor_4"
+; VF_8:          Found an estimated cost of 544 for VF 8 For instruction: %tmp4 = load half, half* %tmp0, align 2
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp5 = load half, half* %tmp1, align 2
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp6 = load half, half* %tmp2, align 2
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp7 = load half, half* %tmp3, align 2
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store half 0xH0000, half* %tmp0, align 2
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store half 0xH0000, half* %tmp1, align 2
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store half 0xH0000, half* %tmp2, align 2
+; VF_8-NEXT:     Found an estimated cost of 288 for VF 8 For instruction: store half 0xH0000, half* %tmp3, align 2
+; VF_16-LABEL: Checking a loop in "f16_factor_4"
+; VF_16:         Found an estimated cost of 2112 for VF 16 For instruction: %tmp4 = load half, half* %tmp0, align 2
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp5 = load half, half* %tmp1, align 2
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp6 = load half, half* %tmp2, align 2
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp7 = load half, half* %tmp3, align 2
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store half 0xH0000, half* %tmp0, align 2
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store half 0xH0000, half* %tmp1, align 2
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store half 0xH0000, half* %tmp2, align 2
+; VF_16-NEXT:    Found an estimated cost of 1088 for VF 16 For instruction: store half 0xH0000, half* %tmp3, align 2
+for.body:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
+  %tmp0 = getelementptr inbounds %f16.4, %f16.4* %data, i64 %i, i32 0
+  %tmp1 = getelementptr inbounds %f16.4, %f16.4* %data, i64 %i, i32 1
+  %tmp2 = getelementptr inbounds %f16.4, %f16.4* %data, i64 %i, i32 2
+  %tmp3 = getelementptr inbounds %f16.4, %f16.4* %data, i64 %i, i32 3
+  %tmp4 = load half, half* %tmp0, align 2
+  %tmp5 = load half, half* %tmp1, align 2
+  %tmp6 = load half, half* %tmp2, align 2
+  %tmp7 = load half, half* %tmp3, align 2
+  store half 0.0, half* %tmp0, align 2
+  store half 0.0, half* %tmp1, align 2
+  store half 0.0, half* %tmp2, align 2
+  store half 0.0, half* %tmp3, align 2
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+%f32.4 = type {float, float, float, float}
+define void @f32_factor_4(%f32.4* %data, i64 %n) #0 {
+entry:
+  br label %for.body
+
+; VF_2-LABEL:  Checking a loop in "f32_factor_4"
+; VF_2:          Found an estimated cost of 40 for VF 2 For instruction: %tmp4 = load float, float* %tmp0, align 4
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp5 = load float, float* %tmp1, align 4
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp6 = load float, float* %tmp2, align 4
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp7 = load float, float* %tmp3, align 4
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store float 0.000000e+00, float* %tmp0, align 4
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store float 0.000000e+00, float* %tmp1, align 4
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store float 0.000000e+00, float* %tmp2, align 4
+; VF_2-NEXT:     Found an estimated cost of 24 for VF 2 For instruction: store float 0.000000e+00, float* %tmp3, align 4
+; VF_4-LABEL:  Checking a loop in "f32_factor_4"
+; VF_4:          Found an estimated cost of 144 for VF 4 For instruction: %tmp4 = load float, float* %tmp0, align 4
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp5 = load float, float* %tmp1, align 4
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp6 = load float, float* %tmp2, align 4
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp7 = load float, float* %tmp3, align 4
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store float 0.000000e+00, float* %tmp0, align 4
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store float 0.000000e+00, float* %tmp1, align 4
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store float 0.000000e+00, float* %tmp2, align 4
+; VF_4-NEXT:     Found an estimated cost of 80 for VF 4 For instruction: store float 0.000000e+00, float* %tmp3, align 4
+; VF_8-LABEL:  Checking a loop in "f32_factor_4"
+; VF_8:          Found an estimated cost of 544 for VF 8 For instruction: %tmp4 = load float, float* %tmp0, align 4
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp5 = load float, float* %tmp1, align 4
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp6 = load float, float* %tmp2, align 4
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp7 = load float, float* %tmp3, align 4
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store float 0.000000e+00, float* %tmp0, align 4
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store float 0.000000e+00, float* %tmp1, align 4
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store float 0.000000e+00, float* %tmp2, align 4
+; VF_8-NEXT:     Found an estimated cost of 288 for VF 8 For instruction: store float 0.000000e+00, float* %tmp3, align 4
+; VF_16-LABEL: Checking a loop in "f32_factor_4"
+; VF_16:         Found an estimated cost of 2112 for VF 16 For instruction: %tmp4 = load float, float* %tmp0, align 4
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp5 = load float, float* %tmp1, align 4
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp6 = load float, float* %tmp2, align 4
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp7 = load float, float* %tmp3, align 4
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store float 0.000000e+00, float* %tmp0, align 4
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store float 0.000000e+00, float* %tmp1, align 4
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store float 0.000000e+00, float* %tmp2, align 4
+; VF_16-NEXT:    Found an estimated cost of 1088 for VF 16 For instruction: store float 0.000000e+00, float* %tmp3, align 4
+for.body:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
+  %tmp0 = getelementptr inbounds %f32.4, %f32.4* %data, i64 %i, i32 0
+  %tmp1 = getelementptr inbounds %f32.4, %f32.4* %data, i64 %i, i32 1
+  %tmp2 = getelementptr inbounds %f32.4, %f32.4* %data, i64 %i, i32 2
+  %tmp3 = getelementptr inbounds %f32.4, %f32.4* %data, i64 %i, i32 3
+  %tmp4 = load float, float* %tmp0, align 4
+  %tmp5 = load float, float* %tmp1, align 4
+  %tmp6 = load float, float* %tmp2, align 4
+  %tmp7 = load float, float* %tmp3, align 4
+  store float 0.0, float* %tmp0, align 4
+  store float 0.0, float* %tmp1, align 4
+  store float 0.0, float* %tmp2, align 4
+  store float 0.0, float* %tmp3, align 4
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+%f64.4 = type {double, double, double, double}
+define void @f64_factor_4(%f64.4* %data, i64 %n) #0 {
+entry:
+  br label %for.body
+
+; VF_2-LABEL:  Checking a loop in "f64_factor_4"
+; VF_2:          Found an estimated cost of 40 for VF 2 For instruction: %tmp4 = load double, double* %tmp0, align 8
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp5 = load double, double* %tmp1, align 8
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp6 = load double, double* %tmp2, align 8
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp7 = load double, double* %tmp3, align 8
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store double 0.000000e+00, double* %tmp0, align 8
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store double 0.000000e+00, double* %tmp1, align 8
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store double 0.000000e+00, double* %tmp2, align 8
+; VF_2-NEXT:     Found an estimated cost of 24 for VF 2 For instruction: store double 0.000000e+00, double* %tmp3, align 8
+; VF_4-LABEL:  Checking a loop in "f64_factor_4"
+; VF_4:          Found an estimated cost of 144 for VF 4 For instruction: %tmp4 = load double, double* %tmp0, align 8
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp5 = load double, double* %tmp1, align 8
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp6 = load double, double* %tmp2, align 8
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp7 = load double, double* %tmp3, align 8
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store double 0.000000e+00, double* %tmp0, align 8
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store double 0.000000e+00, double* %tmp1, align 8
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store double 0.000000e+00, double* %tmp2, align 8
+; VF_4-NEXT:     Found an estimated cost of 80 for VF 4 For instruction: store double 0.000000e+00, double* %tmp3, align 8
+; VF_8-LABEL:  Checking a loop in "f64_factor_4"
+; VF_8:          Found an estimated cost of 544 for VF 8 For instruction: %tmp4 = load double, double* %tmp0, align 8
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp5 = load double, double* %tmp1, align 8
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp6 = load double, double* %tmp2, align 8
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp7 = load double, double* %tmp3, align 8
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store double 0.000000e+00, double* %tmp0, align 8
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store double 0.000000e+00, double* %tmp1, align 8
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store double 0.000000e+00, double* %tmp2, align 8
+; VF_8-NEXT:     Found an estimated cost of 288 for VF 8 For instruction: store double 0.000000e+00, double* %tmp3, align 8
+; VF_16-LABEL: Checking a loop in "f64_factor_4"
+; VF_16:         Found an estimated cost of 2112 for VF 16 For instruction: %tmp4 = load double, double* %tmp0, align 8
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp5 = load double, double* %tmp1, align 8
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp6 = load double, double* %tmp2, align 8
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp7 = load double, double* %tmp3, align 8
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store double 0.000000e+00, double* %tmp0, align 8
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store double 0.000000e+00, double* %tmp1, align 8
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store double 0.000000e+00, double* %tmp2, align 8
+; VF_16-NEXT:    Found an estimated cost of 1088 for VF 16 For instruction: store double 0.000000e+00, double* %tmp3, align 8
+for.body:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
+  %tmp0 = getelementptr inbounds %f64.4, %f64.4* %data, i64 %i, i32 0
+  %tmp1 = getelementptr inbounds %f64.4, %f64.4* %data, i64 %i, i32 1
+  %tmp2 = getelementptr inbounds %f64.4, %f64.4* %data, i64 %i, i32 2
+  %tmp3 = getelementptr inbounds %f64.4, %f64.4* %data, i64 %i, i32 3
+  %tmp4 = load double, double* %tmp0, align 8
+  %tmp5 = load double, double* %tmp1, align 8
+  %tmp6 = load double, double* %tmp2, align 8
+  %tmp7 = load double, double* %tmp3, align 8
+  store double 0.0, double* %tmp0, align 8
+  store double 0.0, double* %tmp1, align 8
+  store double 0.0, double* %tmp2, align 8
+  store double 0.0, double* %tmp3, align 8
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+attributes #0 = { "target-features"="+mve.fp" }
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/mve-maskedldst.ll b/llvm/test/Transforms/LoopVectorize/ARM/mve-maskedldst.ll
new file mode 100644
index 000000000..6b036c3b1
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/ARM/mve-maskedldst.ll
@@ -0,0 +1,176 @@
+; RUN: opt -loop-vectorize < %s -S -o - | FileCheck %s
+
+target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
+target triple = "thumbv8.1-m.main-none-eabi"
+
+; CHECK-LABEL: test_i32_align4
+; CHECK: call void @llvm.masked.store.v4i32.p0v4i32
+define void @test_i32_align4(i32* nocapture %A, i32 %n) #0 {
+entry:
+  %cmp12 = icmp sgt i32 %n, 0
+  br i1 %cmp12, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.inc
+  %i.013 = phi i32 [ %inc, %for.inc ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i32 %i.013
+  %0 = load i32, i32* %arrayidx, align 4
+  %.off = add i32 %0, 9
+  %1 = icmp ult i32 %.off, 19
+  br i1 %1, label %if.then, label %for.inc
+
+if.then:                                          ; preds = %for.body
+  store i32 0, i32* %arrayidx, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body, %if.then
+  %inc = add nuw nsw i32 %i.013, 1
+  %exitcond = icmp eq i32 %inc, %n
+  br i1 %exitcond, label %for.cond.cleanup.loopexit, label %for.body
+
+for.cond.cleanup.loopexit:                        ; preds = %for.inc
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.cond.cleanup.loopexit, %entry
+  ret void
+}
+
+; CHECK-LABEL: test_i32_align2
+; CHECK-NOT: call void @llvm.masked.store
+define void @test_i32_align2(i32* nocapture %A, i32 %n) #0 {
+entry:
+  %cmp12 = icmp sgt i32 %n, 0
+  br i1 %cmp12, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.inc
+  %i.013 = phi i32 [ %inc, %for.inc ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i32 %i.013
+  %0 = load i32, i32* %arrayidx, align 2
+  %.off = add i32 %0, 9
+  %1 = icmp ult i32 %.off, 19
+  br i1 %1, label %if.then, label %for.inc
+
+if.then:                                          ; preds = %for.body
+  store i32 0, i32* %arrayidx, align 2
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body, %if.then
+  %inc = add nuw nsw i32 %i.013, 1
+  %exitcond = icmp eq i32 %inc, %n
+  br i1 %exitcond, label %for.cond.cleanup.loopexit, label %for.body
+
+for.cond.cleanup.loopexit:                        ; preds = %for.inc
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.cond.cleanup.loopexit, %entry
+  ret void
+}
+
+; CHECK-LABEL: test_i32_noalign
+; CHECK: call void @llvm.masked.store.v4i32.p0v4i32
+define void @test_i32_noalign(i32* nocapture %A, i32 %n) #0 {
+entry:
+  %cmp12 = icmp sgt i32 %n, 0
+  br i1 %cmp12, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.inc
+  %i.013 = phi i32 [ %inc, %for.inc ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i32 %i.013
+  %0 = load i32, i32* %arrayidx
+  %.off = add i32 %0, 9
+  %1 = icmp ult i32 %.off, 19
+  br i1 %1, label %if.then, label %for.inc
+
+if.then:                                          ; preds = %for.body
+  store i32 0, i32* %arrayidx
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body, %if.then
+  %inc = add nuw nsw i32 %i.013, 1
+  %exitcond = icmp eq i32 %inc, %n
+  br i1 %exitcond, label %for.cond.cleanup.loopexit, label %for.body
+
+for.cond.cleanup.loopexit:                        ; preds = %for.inc
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.cond.cleanup.loopexit, %entry
+  ret void
+}
+
+; CHECK-LABEL: test_i16_align2
+; CHECK: call void @llvm.masked.store.v8i16.p0v8i16
+define void @test_i16_align2(i16* nocapture %A, i32 %n) #0 {
+entry:
+  %cmp12 = icmp sgt i32 %n, 0
+  br i1 %cmp12, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.inc
+  %i.013 = phi i32 [ %inc, %for.inc ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i16, i16* %A, i32 %i.013
+  %0 = load i16, i16* %arrayidx, align 2
+  %.off = add i16 %0, 9
+  %1 = icmp ult i16 %.off, 19
+  br i1 %1, label %if.then, label %for.inc
+
+if.then:                                          ; preds = %for.body
+  store i16 0, i16* %arrayidx, align 2
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body, %if.then
+  %inc = add nuw nsw i32 %i.013, 1
+  %exitcond = icmp eq i32 %inc, %n
+  br i1 %exitcond, label %for.cond.cleanup.loopexit, label %for.body
+
+for.cond.cleanup.loopexit:                        ; preds = %for.inc
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.cond.cleanup.loopexit, %entry
+  ret void
+}
+
+; CHECK-LABEL: test_i16_align1
+; CHECK-NOT: call void @llvm.masked.store
+define void @test_i16_align1(i16* nocapture %A, i32 %n) #0 {
+entry:
+  %cmp12 = icmp sgt i32 %n, 0
+  br i1 %cmp12, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.inc
+  %i.013 = phi i32 [ %inc, %for.inc ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i16, i16* %A, i32 %i.013
+  %0 = load i16, i16* %arrayidx, align 1
+  %.off = add i16 %0, 9
+  %1 = icmp ult i16 %.off, 19
+  br i1 %1, label %if.then, label %for.inc
+
+if.then:                                          ; preds = %for.body
+  store i16 0, i16* %arrayidx, align 1
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body, %if.then
+  %inc = add nuw nsw i32 %i.013, 1
+  %exitcond = icmp eq i32 %inc, %n
+  br i1 %exitcond, label %for.cond.cleanup.loopexit, label %for.body
+
+for.cond.cleanup.loopexit:                        ; preds = %for.inc
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.cond.cleanup.loopexit, %entry
+  ret void
+}
+
+attributes #0 = { "target-features"="+mve" }
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/mve-shiftcost.ll b/llvm/test/Transforms/LoopVectorize/ARM/mve-shiftcost.ll
new file mode 100644
index 000000000..7195e61c2
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/ARM/mve-shiftcost.ll
@@ -0,0 +1,87 @@
+; RUN: opt -loop-vectorize < %s -S -o - | FileCheck %s --check-prefix=CHECK
+; RUN: opt -loop-vectorize -debug-only=loop-vectorize -disable-output < %s 2>&1 | FileCheck %s --check-prefix=CHECK-COST
+; REQUIRES: asserts
+
+target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
+target triple = "thumbv8.1m.main-arm-none-eabi"
+
+; CHECK-LABEL: test
+; CHECK-COST: LV: Found an estimated cost of 0 for VF 1 For instruction:   %and515 = shl i32 %l41, 3
+; CHECK-COST: LV: Found an estimated cost of 1 for VF 1 For instruction:   %l45 = and i32 %and515, 131072
+; CHECK-COST: LV: Found an estimated cost of 2 for VF 4 For instruction:   %and515 = shl i32 %l41, 3
+; CHECK-COST: LV: Found an estimated cost of 2 for VF 4 For instruction:   %l45 = and i32 %and515, 131072
+; CHECK-NOT: vector.body
+
+define void @test([101 x i32] *%src, i32 %N) #0 {
+entry:
+  br label %for.body386
+  
+for.body386:                                      ; preds = %entry, %l77
+  %add387 = phi i32 [ %inc532, %l77 ], [ 0, %entry ]
+  %arrayidx388 = getelementptr inbounds [101 x i32], [101 x i32]* %src, i32 0, i32 %add387
+  %l41 = load i32, i32* %arrayidx388, align 4
+  %l42 = and i32 %l41, 65535
+  %l43 = icmp eq i32 %l42, 0
+  br i1 %l43, label %l77, label %l44
+
+l44:                                               ; preds = %for.body386
+  %and515 = shl i32 %l41, 3
+  %l45 = and i32 %and515, 131072
+  %and506 = shl i32 %l41, 5
+  %l46 = and i32 %and506, 262144
+  %and497 = shl i32 %l41, 7
+  %l47 = and i32 %and497, 524288
+  %and488 = shl i32 %l41, 9
+  %l48 = and i32 %and488, 1048576
+  %and479 = shl i32 %l41, 11
+  %l49 = and i32 %and479, 2097152
+  %and470 = shl i32 %l41, 13
+  %l50 = and i32 %and470, 4194304
+  %and461 = shl i32 %l41, 15
+  %l51 = and i32 %and461, 8388608
+  %and452 = shl i32 %l41, 17
+  %l52 = and i32 %and452, 16777216
+  %and443 = shl i32 %l41, 19
+  %l53 = and i32 %and443, 33554432
+  %and434 = shl i32 %l41, 21
+  %l54 = and i32 %and434, 67108864
+  %and425 = shl i32 %l41, 23
+  %l55 = and i32 %and425, 134217728
+  %and416 = shl i32 %l41, 25
+  %l56 = and i32 %and416, 268435456
+  %and407 = shl i32 %l41, 27
+  %l57 = and i32 %and407, 536870912
+  %and398 = shl i32 %l41, 29
+  %l58 = and i32 %and398, 1073741824
+  %l59 = shl i32 %l41, 31
+  %l60 = or i32 %l59, %l41
+  %l61 = or i32 %l58, %l60
+  %l62 = or i32 %l57, %l61
+  %l63 = or i32 %l56, %l62
+  %l64 = or i32 %l55, %l63
+  %l65 = or i32 %l54, %l64
+  %l66 = or i32 %l53, %l65
+  %l67 = or i32 %l52, %l66
+  %l68 = or i32 %l51, %l67
+  %l69 = or i32 %l50, %l68
+  %l70 = or i32 %l49, %l69
+  %l71 = or i32 %l48, %l70
+  %l72 = or i32 %l47, %l71
+  %l73 = or i32 %l46, %l72
+  %l74 = or i32 %l45, %l73
+  %and524 = shl i32 %l41, 1
+  %l75 = and i32 %and524, 65536
+  %l76 = or i32 %l75, %l74
+  store i32 %l76, i32* %arrayidx388, align 4
+  br label %l77
+
+l77:                                               ; preds = %for.body386, %l44
+  %inc532 = add nuw nsw i32 %add387, 1
+  %exitcond649 = icmp eq i32 %inc532, %N
+  br i1 %exitcond649, label %exit, label %for.body386
+
+exit:
+  ret void
+}
+
+attributes #0 = { nounwind "min-legal-vector-width"="0" "target-cpu"="generic" "target-features"="+armv8.1-m.main,+fp-armv8d16sp,+fp16,+fullfp16,+hwdiv,+lob,+mve.fp,+ras,+strict-align,+thumb-mode,+vfp2sp,+vfp3d16sp,+vfp4d16sp" "use-soft-float"="false" }
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/mve-vldn.ll b/llvm/test/Transforms/LoopVectorize/ARM/mve-vldn.ll
new file mode 100644
index 000000000..353e72558
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/ARM/mve-vldn.ll
@@ -0,0 +1,87 @@
+; RUN: opt -loop-vectorize < %s -S -o - | FileCheck %s --check-prefixes=CHECK,CHECK-2,CHECK-NO4
+; RUN: opt -loop-vectorize -mve-max-interleave-factor=1 < %s -S -o - | FileCheck %s --check-prefixes=CHECK,CHECK-NO2,CHECK-NO4
+; RUN: opt -loop-vectorize -mve-max-interleave-factor=2 < %s -S -o - | FileCheck %s --check-prefixes=CHECK,CHECK-2,CHECK-NO4
+; RUN: opt -loop-vectorize -mve-max-interleave-factor=4 < %s -S -o - | FileCheck %s --check-prefixes=CHECK,CHECK-2,CHECK-4
+
+target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
+target triple = "thumbv8.1m.main-arm-none-eabi"
+
+; CHECK-LABEL: vld2
+; CHECK-2: vector.body
+; CHECK-NO2-NOT: vector.body
+define void @vld2(half* nocapture readonly %pIn, half* nocapture %pOut, i32 %numRows, i32 %numCols, i32 %scale.coerce) #0 {
+entry:
+  %tmp.0.extract.trunc = trunc i32 %scale.coerce to i16
+  %0 = bitcast i16 %tmp.0.extract.trunc to half
+  %mul = mul i32 %numCols, %numRows
+  %shr = lshr i32 %mul, 2
+  %cmp26 = icmp eq i32 %shr, 0
+  br i1 %cmp26, label %while.end, label %while.body
+
+while.body:                                       ; preds = %entry, %while.body
+  %pIn.addr.029 = phi half* [ %add.ptr, %while.body ], [ %pIn, %entry ]
+  %pOut.addr.028 = phi half* [ %add.ptr7, %while.body ], [ %pOut, %entry ]
+  %blkCnt.027 = phi i32 [ %dec, %while.body ], [ %shr, %entry ]
+  %1 = load half, half* %pIn.addr.029, align 2
+  %arrayidx2 = getelementptr inbounds half, half* %pIn.addr.029, i32 1
+  %2 = load half, half* %arrayidx2, align 2
+  %mul3 = fmul half %1, %0
+  %mul4 = fmul half %2, %0
+  store half %mul3, half* %pOut.addr.028, align 2
+  %arrayidx6 = getelementptr inbounds half, half* %pOut.addr.028, i32 1
+  store half %mul4, half* %arrayidx6, align 2
+  %add.ptr = getelementptr inbounds half, half* %pIn.addr.029, i32 2
+  %add.ptr7 = getelementptr inbounds half, half* %pOut.addr.028, i32 2
+  %dec = add nsw i32 %blkCnt.027, -1
+  %cmp = icmp eq i32 %dec, 0
+  br i1 %cmp, label %while.end, label %while.body
+
+while.end:                                        ; preds = %while.body, %entry
+  ret void
+}
+
+; CHECK-LABEL: vld4
+; CHECK-4: vector.body
+; CHECK-NO4-NOT: vector.body
+define void @vld4(half* nocapture readonly %pIn, half* nocapture %pOut, i32 %numRows, i32 %numCols, i32 %scale.coerce) #0 {
+entry:
+  %tmp.0.extract.trunc = trunc i32 %scale.coerce to i16
+  %0 = bitcast i16 %tmp.0.extract.trunc to half
+  %mul = mul i32 %numCols, %numRows
+  %shr = lshr i32 %mul, 2
+  %cmp38 = icmp eq i32 %shr, 0
+  br i1 %cmp38, label %while.end, label %while.body
+
+while.body:                                       ; preds = %entry, %while.body
+  %pIn.addr.041 = phi half* [ %add.ptr, %while.body ], [ %pIn, %entry ]
+  %pOut.addr.040 = phi half* [ %add.ptr13, %while.body ], [ %pOut, %entry ]
+  %blkCnt.039 = phi i32 [ %dec, %while.body ], [ %shr, %entry ]
+  %1 = load half, half* %pIn.addr.041, align 2
+  %arrayidx2 = getelementptr inbounds half, half* %pIn.addr.041, i32 1
+  %2 = load half, half* %arrayidx2, align 2
+  %arrayidx3 = getelementptr inbounds half, half* %pIn.addr.041, i32 2
+  %3 = load half, half* %arrayidx3, align 2
+  %arrayidx4 = getelementptr inbounds half, half* %pIn.addr.041, i32 3
+  %4 = load half, half* %arrayidx4, align 2
+  %mul5 = fmul half %1, %0
+  %mul6 = fmul half %2, %0
+  %mul7 = fmul half %3, %0
+  %mul8 = fmul half %4, %0
+  store half %mul5, half* %pOut.addr.040, align 2
+  %arrayidx10 = getelementptr inbounds half, half* %pOut.addr.040, i32 1
+  store half %mul6, half* %arrayidx10, align 2
+  %arrayidx11 = getelementptr inbounds half, half* %pOut.addr.040, i32 2
+  store half %mul7, half* %arrayidx11, align 2
+  %arrayidx12 = getelementptr inbounds half, half* %pOut.addr.040, i32 3
+  store half %mul8, half* %arrayidx12, align 2
+  %add.ptr = getelementptr inbounds half, half* %pIn.addr.041, i32 4
+  %add.ptr13 = getelementptr inbounds half, half* %pOut.addr.040, i32 4
+  %dec = add nsw i32 %blkCnt.039, -1
+  %cmp = icmp eq i32 %dec, 0
+  br i1 %cmp, label %while.end, label %while.body
+
+while.end:                                        ; preds = %while.body, %entry
+  ret void
+}
+
+attributes #0 = { "target-features"="+armv8.1-m.main,+fp-armv8d16,+fp-armv8d16sp,+fp16,+fp64,+fullfp16,+hwdiv,+lob,+mve.fp,+ras,+strict-align,+thumb-mode,+vfp2,+vfp2sp,+vfp3d16,+vfp3d16sp,+vfp4d16,+vfp4d16sp,-crypto,-d32,-fp-armv8,-fp-armv8sp,-neon,-vfp3,-vfp3sp,-vfp4,-vfp4sp" }
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/prefer-tail-loop-folding.ll b/llvm/test/Transforms/LoopVectorize/ARM/prefer-tail-loop-folding.ll
new file mode 100644
index 000000000..32c08efd0
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/ARM/prefer-tail-loop-folding.ll
@@ -0,0 +1,655 @@
+; RUN: opt -mtriple=thumbv8.1m.main-arm-eabihf \
+; RUN:   -disable-mve-tail-predication=false -loop-vectorize -S < %s | \
+; RUN:   FileCheck %s -check-prefixes=CHECK,PREFER-FOLDING
+
+; RUN: opt -mtriple=thumbv8.1m.main-arm-eabihf -mattr=-mve \
+; RUN:   -disable-mve-tail-predication=false -loop-vectorize \
+; RUN:   -enable-arm-maskedldst=true -S < %s | \
+; RUN:   FileCheck %s -check-prefixes=CHECK,NO-FOLDING
+
+; RUN: opt -mtriple=thumbv8.1m.main-arm-eabihf -mattr=+mve \
+; RUN:   -disable-mve-tail-predication=false -loop-vectorize \
+; RUN:   -enable-arm-maskedldst=false -S < %s | \
+; RUN:   FileCheck %s -check-prefixes=CHECK,NO-FOLDING
+
+; RUN: opt -mtriple=thumbv8.1m.main-arm-eabihf -mattr=+mve \
+; RUN:   -disable-mve-tail-predication=true -loop-vectorize \
+; RUN:   -enable-arm-maskedldst=true -S < %s | \
+; RUN:   FileCheck %s -check-prefixes=CHECK,NO-FOLDING
+
+; Disabling the low-overhead branch extension will make
+; 'isHardwareLoopProfitable' return false, so that we test avoiding folding for
+; these cases.
+; RUN: opt -mtriple=thumbv8.1m.main-arm-eabihf -mattr=+mve,-lob \
+; RUN:   -disable-mve-tail-predication=false -loop-vectorize \
+; RUN:   -enable-arm-maskedldst=true -S < %s | \
+; RUN:   FileCheck %s -check-prefixes=CHECK,NO-FOLDING
+
+; RUN: opt -mtriple=thumbv8.1m.main-arm-eabihf -mattr=+mve.fp \
+; RUN:   -disable-mve-tail-predication=false -loop-vectorize \
+; RUN:   -enable-arm-maskedldst=true -S < %s | \
+; RUN:   FileCheck %s -check-prefixes=CHECK,PREFER-FOLDING
+
+; RUN: opt -mtriple=thumbv8.1m.main-arm-eabihf -mattr=+mve.fp \
+; RUN:   -prefer-predicate-over-epilog=false \
+; RUN:   -disable-mve-tail-predication=false -loop-vectorize \
+; RUN:   -enable-arm-maskedldst=true -S < %s | \
+; RUN:   FileCheck %s -check-prefixes=CHECK,NO-FOLDING
+
+; RUN: opt -mtriple=thumbv8.1m.main-arm-eabihf -mattr=+mve.fp \
+; RUN:   -prefer-predicate-over-epilog=true \
+; RUN:   -disable-mve-tail-predication=false -loop-vectorize \
+; RUN:   -enable-arm-maskedldst=true -S < %s | \
+; RUN:   FileCheck %s -check-prefixes=CHECK,FOLDING-OPT
+
+define void @prefer_folding(i32* noalias nocapture %A, i32* noalias nocapture readonly %B, i32* noalias nocapture readonly %C) #0 {
+; CHECK-LABEL:    prefer_folding(
+; PREFER-FOLDING: vector.body:
+; PREFER-FOLDING: call <4 x i32> @llvm.masked.load.v4i32.p0v4i32
+; PREFER-FOLDING: call <4 x i32> @llvm.masked.load.v4i32.p0v4i32
+; PREFER-FOLDING: call void @llvm.masked.store.v4i32.p0v4i32
+; PREFER-FOLDING: br i1 %{{.*}}, label %{{.*}}, label %vector.body
+;
+; NO-FOLDING-NOT: call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(
+; NO-FOLDING-NOT: call void @llvm.masked.store.v4i32.p0v4i32(
+; NO-FOLDING:     br i1 %{{.*}}, label %{{.*}}, label %for.body
+entry:
+  br label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.09 = phi i32 [ 0, %entry ], [ %add3, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %B, i32 %i.09
+  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds i32, i32* %C, i32 %i.09
+  %1 = load i32, i32* %arrayidx1, align 4
+  %add = add nsw i32 %1, %0
+  %arrayidx2 = getelementptr inbounds i32, i32* %A, i32 %i.09
+  store i32 %add, i32* %arrayidx2, align 4
+  %add3 = add nuw nsw i32 %i.09, 1
+  %exitcond = icmp eq i32 %add3, 431
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+define void @mixed_types(i16* noalias nocapture %A, i16* noalias nocapture readonly %B, i16* noalias nocapture readonly %C, i32* noalias nocapture %D, i32* noalias nocapture readonly %E, i32* noalias nocapture readonly %F) #0 {
+; CHECK-LABEL:        mixed_types(
+; PREFER-FOLDING:     vector.body:
+; PREFER-FOLDING:     call <4 x i16> @llvm.masked.load.v4i16.p0v4i16
+; PREFER-FOLDING:     call <4 x i16> @llvm.masked.load.v4i16.p0v4i16
+; PREFER-FOLDING:     call void @llvm.masked.store.v4i16.p0v4i16
+; PREFER-FOLDING:     call <4 x i32> @llvm.masked.load.v4i32.p0v4i32
+; PREFER-FOLDING:     call <4 x i32> @llvm.masked.load.v4i32.p0v4i32
+; PREFER-FOLDING:     call void @llvm.masked.store.v4i32.p0v4i32
+; PREFER-FOLDING:     br i1 %{{.*}}, label %{{.*}}, label %vector.body
+entry:
+  br label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.018 = phi i32 [ 0, %entry ], [ %add9, %for.body ]
+  %arrayidx = getelementptr inbounds i16, i16* %B, i32 %i.018
+  %0 = load i16, i16* %arrayidx, align 2
+  %arrayidx1 = getelementptr inbounds i16, i16* %C, i32 %i.018
+  %1 = load i16, i16* %arrayidx1, align 2
+  %add = add i16 %1, %0
+  %arrayidx4 = getelementptr inbounds i16, i16* %A, i32 %i.018
+  store i16 %add, i16* %arrayidx4, align 2
+  %arrayidx5 = getelementptr inbounds i32, i32* %E, i32 %i.018
+  %2 = load i32, i32* %arrayidx5, align 4
+  %arrayidx6 = getelementptr inbounds i32, i32* %F, i32 %i.018
+  %3 = load i32, i32* %arrayidx6, align 4
+  %add7 = add nsw i32 %3, %2
+  %arrayidx8 = getelementptr inbounds i32, i32* %D, i32 %i.018
+  store i32 %add7, i32* %arrayidx8, align 4
+  %add9 = add nuw nsw i32 %i.018, 1
+  %exitcond = icmp eq i32 %add9, 431
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+define void @unsupported_i64_type(i64* noalias nocapture %A, i64* noalias nocapture readonly %B, i64* noalias nocapture readonly %C) #0 {
+; CHECK-LABEL:        unsupported_i64_type(
+; PREFER-FOLDING-NOT: vector.body:
+; PREFER-FOLDING-NOT: llvm.masked.load
+; PREFER-FOLDING-NOT: llvm.masked.store
+; PREFER-FOLDING:     for.body:
+entry:
+  br label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.09 = phi i32 [ 0, %entry ], [ %add3, %for.body ]
+  %arrayidx = getelementptr inbounds i64, i64* %B, i32 %i.09
+  %0 = load i64, i64* %arrayidx, align 8
+  %arrayidx1 = getelementptr inbounds i64, i64* %C, i32 %i.09
+  %1 = load i64, i64* %arrayidx1, align 8
+  %add = add nsw i64 %1, %0
+  %arrayidx2 = getelementptr inbounds i64, i64* %A, i32 %i.09
+  store i64 %add, i64* %arrayidx2, align 8
+  %add3 = add nuw nsw i32 %i.09, 1
+  %exitcond = icmp eq i32 %add3, 431
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+define void @zero_extending_load_allowed(i32* noalias nocapture %A, i8* noalias nocapture readonly %B, i32* noalias nocapture readonly %C) #0 {
+; CHECK-LABEL:    zero_extending_load_allowed(
+; PREFER-FOLDING: vector.body:
+; PREFER-FOLDING: call <4 x i8> @llvm.masked.load.v4i8.p0v4i8
+; PREFER-FOLDING: call <4 x i32> @llvm.masked.load.v4i32.p0v4i32
+; PREFER-FOLDING: call void @llvm.masked.store.v4i32.p0v4i32
+; PREFER-FOLDING: br i1 %{{.*}}, label %{{.*}}, label %vector.body
+entry:
+  br label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.09 = phi i32 [ 0, %entry ], [ %add3, %for.body ]
+  %arrayidx = getelementptr inbounds i8, i8* %B, i32 %i.09
+  %0 = load i8, i8* %arrayidx, align 1
+  %conv = zext i8 %0 to i32
+  %arrayidx1 = getelementptr inbounds i32, i32* %C, i32 %i.09
+  %1 = load i32, i32* %arrayidx1, align 4
+  %add = add nsw i32 %1, %conv
+  %arrayidx2 = getelementptr inbounds i32, i32* %A, i32 %i.09
+  store i32 %add, i32* %arrayidx2, align 4
+  %add3 = add nuw nsw i32 %i.09, 1
+  %exitcond = icmp eq i32 %add3, 431
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+define void @sign_extending_load_allowed(i32* noalias nocapture %A, i8* noalias nocapture readonly %B, i32* noalias nocapture readonly %C) #0 {
+; CHECK-LABEL:    sign_extending_load_allowed(
+; PREFER-FOLDING: vector.body:
+; PREFER-FOLDING: call <4 x i8> @llvm.masked.load.v4i8.p0v4i8
+; PREFER-FOLDING: call <4 x i32> @llvm.masked.load.v4i32.p0v4i32
+; PREFER-FOLDING: call void @llvm.masked.store.v4i32.p0v4i32
+; PREFER-FOLDING: br i1 %{{.*}}, label %{{.*}}, label %vector.body
+entry:
+  br label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.09 = phi i32 [ 0, %entry ], [ %add3, %for.body ]
+  %arrayidx = getelementptr inbounds i8, i8* %B, i32 %i.09
+  %0 = load i8, i8* %arrayidx, align 1
+  %conv = sext i8 %0 to i32
+  %arrayidx1 = getelementptr inbounds i32, i32* %C, i32 %i.09
+  %1 = load i32, i32* %arrayidx1, align 4
+  %add = add nsw i32 %1, %conv
+  %arrayidx2 = getelementptr inbounds i32, i32* %A, i32 %i.09
+  store i32 %add, i32* %arrayidx2, align 4
+  %add3 = add nuw nsw i32 %i.09, 1
+  %exitcond = icmp eq i32 %add3, 431
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+define void @narrowing_load_not_allowed(i8* noalias nocapture %A, i8* noalias nocapture readonly %B, i16* noalias nocapture readonly %C) #0 {
+; CHECK-LABEL:        narrowing_load_not_allowed(
+; PREFER-FOLDING:     vector.body:
+; PREFER-FOLDING-NOT: llvm.masked.load
+; PREFER-FOLDING-NOT: llvm.masked.store
+; PREFER-FOLDING:     br i1 %{{.*}}, label %{{.*}}, label %vector.body
+
+; FOLDING-OPT:        vector.body:
+; FOLDING-OPT         call <8 x i16> @llvm.masked.load.v8i16.p0v8i16
+; FOLDING-OPT         call <8 x i8> @llvm.masked.load.v8i8.p0v8i8
+; FOLDING-OPT         call void @llvm.masked.store.v8i8.p0v8i8
+; FOLDING-OPT:        br i1 %{{.*}}, label %{{.*}}, label %vector.body
+entry:
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body
+  ret void
+
+for.body:                                         ; preds = %for.body, %entry
+  %i.012 = phi i32 [ 0, %entry ], [ %add6, %for.body ]
+  %arrayidx = getelementptr inbounds i16, i16* %C, i32 %i.012
+  %0 = load i16, i16* %arrayidx, align 2
+  %arrayidx1 = getelementptr inbounds i8, i8* %B, i32 %i.012
+  %1 = load i8, i8* %arrayidx1, align 1
+  %conv3 = trunc i16 %0 to i8
+  %add = add i8 %1, %conv3
+  %arrayidx5 = getelementptr inbounds i8, i8* %A, i32 %i.012
+  store i8 %add, i8* %arrayidx5, align 1
+  %add6 = add nuw nsw i32 %i.012, 1
+  %exitcond = icmp eq i32 %add6, 431
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+define void @narrowing_store_allowed(i8* noalias nocapture %A, i32* noalias nocapture readonly %B, i32* noalias nocapture readonly %C) #0 {
+; CHECK-LABEL:    narrowing_store_allowed(
+; PREFER-FOLDING: call void @llvm.masked.store.v4i8.p0v4i8
+; PREFER-FOLDING: br i1 %{{.*}}, label %{{.*}}, label %vector.body
+entry:
+  br label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.09 = phi i32 [ 0, %entry ], [ %add3, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %B, i32 %i.09
+  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds i32, i32* %C, i32 %i.09
+  %1 = load i32, i32* %arrayidx1, align 4
+  %add = add nsw i32 %1, %0
+  %conv = trunc i32 %add to i8
+  %arrayidx2 = getelementptr inbounds i8, i8* %A, i32 %i.09
+  store i8 %conv, i8* %arrayidx2, align 1
+  %add3 = add nuw nsw i32 %i.09, 1
+  %exitcond = icmp eq i32 %add3, 431
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+; This is a trunc not connected to a store, so we don't allow this.
+; TODO: this is conservative, because the trunc is only used in the
+; loop control statements, and thus not affecting element sizes, so
+; we could allow this case.
+define void @trunc_not_allowed(i32* noalias nocapture %A, i32* noalias nocapture readonly %B, i32* noalias nocapture readonly %C) #0 {
+; CHECK-LABEL:        trunc_not_allowed(
+; PREFER-FOLDING:     vector.body:
+; PREFER-FOLDING-NOT: llvm.masked.load
+; PREFER-FOLDING-NOT: llvm.masked.store
+; PREFER-FOLDING:     br i1 %{{.*}}, label %{{.*}}, label %vector.body
+entry:
+  br label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.09 = phi i32 [ 0, %entry ], [ %add3, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %B, i32 %i.09
+  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds i32, i32* %C, i32 %i.09
+  %1 = load i32, i32* %arrayidx1, align 4
+  %add = add nsw i32 %1, %0
+  %arrayidx2 = getelementptr inbounds i32, i32* %A, i32 %i.09
+  store i32 %add, i32* %arrayidx2, align 4
+  %add3 = add nuw nsw i32 %i.09, 1
+
+  %add.iv = trunc i32 %add3 to i16
+
+  %exitcond = icmp eq i16 %add.iv, 431
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+define void @trunc_not_allowed_different_vec_elemns(i32* noalias nocapture %A, i32* noalias nocapture readonly %B, i32* noalias nocapture readonly %C, i16* noalias nocapture %D) #0 {
+; CHECK-LABEL:        trunc_not_allowed_different_vec_elemns(
+; PREFER-FOLDING:     vector.body:
+; PREFER-FOLDING-NOT: llvm.masked.load
+; PREFER-FOLDING-NOT: llvm.masked.store
+; PREFER-FOLDING:     br i1 %{{.*}}, label %{{.*}}, label %vector.body
+entry:
+  br label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.021 = phi i32 [ 0, %entry ], [ %add9, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %B, i32 %i.021
+  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds i32, i32* %C, i32 %i.021
+  %1 = load i32, i32* %arrayidx1, align 4
+  %add = add nsw i32 %1, %0
+  %arrayidx2 = getelementptr inbounds i32, i32* %A, i32 %i.021
+  store i32 %add, i32* %arrayidx2, align 4
+  %add.tr = trunc i32 %add to i16
+  %conv7 = shl i16 %add.tr, 1
+  %arrayidx8 = getelementptr inbounds i16, i16* %D, i32 %i.021
+  store i16 %conv7, i16* %arrayidx8, align 2
+  %add9 = add nuw nsw i32 %i.021, 1
+  %exitcond = icmp eq i32 %add9, 431
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+
+@tab = common global [32 x i8] zeroinitializer, align 1
+
+define i32 @icmp_not_allowed() #0 {
+; CHECK-LABEL:        icmp_not_allowed(
+; PREFER-FOLDING:     vector.body:
+; PREFER-FOLDING-NOT: llvm.masked.load
+; PREFER-FOLDING-NOT: llvm.masked.store
+; PREFER-FOLDING:     br i1 %{{.*}}, label %{{.*}}, label %vector.body
+entry:
+  br label %for.body
+
+for.body:
+  %i.08 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %arrayidx = getelementptr inbounds [32 x i8], [32 x i8]* @tab, i32 0, i32 %i.08
+  %0 = load i8, i8* %arrayidx, align 1
+  %cmp1 = icmp eq i8 %0, 0
+  %. = select i1 %cmp1, i8 2, i8 1
+  store i8 %., i8* %arrayidx, align 1
+  %inc = add nsw i32 %i.08, 1
+  %exitcond = icmp slt i32 %inc, 1000
+  br i1 %exitcond, label %for.body, label %for.end
+
+for.end:
+  ret i32 0
+}
+
+@ftab = common global [32 x float] zeroinitializer, align 1
+
+define float @fcmp_not_allowed() #0 {
+; CHECK-LABEL:        fcmp_not_allowed(
+; PREFER-FOLDING:     vector.body:
+; PREFER-FOLDING-NOT: llvm.masked.load
+; PREFER-FOLDING-NOT: llvm.masked.store
+; PREFER-FOLDING:     br i1 %{{.*}}, label %{{.*}}, label %vector.body
+entry:
+  br label %for.body
+
+for.body:
+  %i.08 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %arrayidx = getelementptr inbounds [32 x float], [32 x float]* @ftab, i32 0, i32 %i.08
+  %0 = load float, float* %arrayidx, align 4
+  %cmp1 = fcmp oeq float %0, 0.000000e+00
+  %. = select i1 %cmp1, float 2.000000e+00, float 1.000000e+00
+  store float %., float* %arrayidx, align 4
+  %inc = add nsw i32 %i.08, 1
+  %exitcond = icmp slt i32 %inc, 999
+  br i1 %exitcond, label %for.body, label %for.end
+
+for.end:
+  ret float 0.000000e+00
+}
+
+define void @pragma_vect_predicate_disable(i32* noalias nocapture %A, i32* noalias nocapture readonly %B, i32* noalias nocapture readonly %C) #0 {
+; CHECK-LABEL:        pragma_vect_predicate_disable(
+; PREFER-FOLDING:     vector.body:
+; PREFER-FOLDING-NOT: call <4 x i32> @llvm.masked.load.v4i32.p0v4i32
+; PREFER-FOLDING-NOT: call <4 x i32> @llvm.masked.load.v4i32.p0v4i32
+; PREFER-FOLDING-NOT: call void @llvm.masked.store.v4i32.p0v4i32
+; PREFER-FOLDING:     br i1 %{{.*}}, label %{{.*}}, label %vector.body
+entry:
+  br label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.09 = phi i32 [ 0, %entry ], [ %add3, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %B, i32 %i.09
+  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds i32, i32* %C, i32 %i.09
+  %1 = load i32, i32* %arrayidx1, align 4
+  %add = add nsw i32 %1, %0
+  %arrayidx2 = getelementptr inbounds i32, i32* %A, i32 %i.09
+  store i32 %add, i32* %arrayidx2, align 4
+  %add3 = add nuw nsw i32 %i.09, 1
+  %exitcond = icmp eq i32 %add3, 431
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body, !llvm.loop !7
+}
+
+; Test directions for array indices i and N-1. I.e. check strides 1 and -1, and
+; force vectorisation with a loop hint.
+define void @strides_different_direction(i32* noalias nocapture %A, i32* noalias nocapture readonly %B, i32* noalias nocapture readonly %C, i32 %N) #0 {
+; CHECK-LABEL: strides_different_direction(
+; PREFER-FOLDING:     vector.body:
+; PREFER-FOLDING-NOT: llvm.masked.load
+; PREFER-FOLDING-NOT: llvm.masked.store
+; PREFER-FOLDING:     br i1 %{{.*}}, label %{{.*}}, label %vector.body
+entry:
+  br label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.09 = phi i32 [ 0, %entry ], [ %add3, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %B, i32 %i.09
+  %0 = load i32, i32* %arrayidx, align 4
+  %sub = sub nsw i32 %N, %i.09
+  %arrayidx1 = getelementptr inbounds i32, i32* %C, i32 %sub
+  %1 = load i32, i32* %arrayidx1, align 4
+  %add = add nsw i32 %1, %0
+  %arrayidx2 = getelementptr inbounds i32, i32* %A, i32 %i.09
+  store i32 %add, i32* %arrayidx2, align 4
+  %add3 = add nuw nsw i32 %i.09, 1
+  %exitcond = icmp eq i32 %add3, 431
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body, !llvm.loop !10
+}
+
+define void @stride_4(i32* noalias nocapture %A, i32* noalias nocapture readonly %B, i32* noalias nocapture readonly %C) #0 {
+; CHECK-LABEL:        stride_4(
+; PREFER-FOLDING:     vector.body:
+; PREFER-FOLDING-NOT: llvm.masked.load
+; PREFER-FOLDING-NOT: llvm.masked.store
+; PREFER-FOLDING:     br i1 %{{.*}}, label %{{.*}}, label %vector.body
+entry:
+  br label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.09 = phi i32 [ 0, %entry ], [ %add3, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %B, i32 %i.09
+  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds i32, i32* %C, i32 %i.09
+  %1 = load i32, i32* %arrayidx1, align 4
+  %add = add nsw i32 %1, %0
+  %arrayidx2 = getelementptr inbounds i32, i32* %A, i32 %i.09
+  store i32 %add, i32* %arrayidx2, align 4
+  %add3 = add nuw nsw i32 %i.09, 4
+  %cmp = icmp ult i32 %add3, 731
+  br i1 %cmp, label %for.body, label %for.cond.cleanup, !llvm.loop !5
+}
+
+define void @too_many_loop_blocks(i32* noalias nocapture %A, i32* noalias nocapture readonly %B, i32* noalias nocapture readonly %C) #0 {
+; CHECK-LABEL:        too_many_loop_blocks(
+; PREFER-FOLDING:     vector.body:
+; PREFER-FOLDING-NOT: llvm.masked.load
+; PREFER-FOLDING-NOT: llvm.masked.store
+; PREFER-FOLDING:     br i1 %{{.*}}, label %{{.*}}, label %vector.body
+entry:
+  br label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.09 = phi i32 [ 0, %entry ], [ %add3, %loopincr ]
+  %arrayidx = getelementptr inbounds i32, i32* %B, i32 %i.09
+  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds i32, i32* %C, i32 %i.09
+  %1 = load i32, i32* %arrayidx1, align 4
+  %add = add nsw i32 %1, %0
+  %arrayidx2 = getelementptr inbounds i32, i32* %A, i32 %i.09
+  store i32 %add, i32* %arrayidx2, align 4
+  br label %loopincr
+
+loopincr:
+  %add3 = add nuw nsw i32 %i.09, 1
+  %exitcond = icmp eq i32 %add3, 431
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+define dso_local void @half(half* noalias nocapture %A, half* noalias nocapture readonly %B, half* noalias nocapture readonly %C) #0 {
+; CHECK-LABEL:    half(
+; PREFER-FOLDING: vector.body:
+; PREFER-FOLDING: call <8 x half> @llvm.masked.load.v8f16.p0v8f16
+; PREFER-FOLDING: call <8 x half> @llvm.masked.load.v8f16.p0v8f16
+; PREFER-FOLDING: call void @llvm.masked.store.v8f16.p0v8f16
+; PREFER-FOLDING: br i1 %{{.*}}, label %{{.*}}, label %vector.body
+entry:
+  br label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.09 = phi i32 [ 0, %entry ], [ %add3, %for.body ]
+  %arrayidx = getelementptr inbounds half, half* %B, i32 %i.09
+  %0 = load half, half* %arrayidx, align 2
+  %arrayidx1 = getelementptr inbounds half, half* %C, i32 %i.09
+  %1 = load half, half* %arrayidx1, align 2
+  %add = fadd fast half %1, %0
+  %arrayidx2 = getelementptr inbounds half, half* %A, i32 %i.09
+  store half %add, half* %arrayidx2, align 2
+  %add3 = add nuw nsw i32 %i.09, 1
+  %exitcond = icmp eq i32 %add3, 431
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+define void @float(float* noalias nocapture %A, float* noalias nocapture readonly %B, float* noalias nocapture readonly %C) #0 {
+; CHECK-LABEL:    float(
+; PREFER-FOLDING: vector.body:
+; PREFER-FOLDING: call <4 x float> @llvm.masked.load.v4f32.p0v4f32
+; PREFER-FOLDING: call <4 x float> @llvm.masked.load.v4f32.p0v4f32
+; PREFER-FOLDING: call void @llvm.masked.store.v4f32.p0v4f32
+; PREFER-FOLDING: br i1 %{{.*}}, label %{{.*}}, label %vector.body
+entry:
+  br label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.09 = phi i32 [ 0, %entry ], [ %add3, %for.body ]
+  %arrayidx = getelementptr inbounds float, float* %B, i32 %i.09
+  %0 = load float, float* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds float, float* %C, i32 %i.09
+  %1 = load float, float* %arrayidx1, align 4
+  %add = fadd fast float %1, %0
+  %arrayidx2 = getelementptr inbounds float, float* %A, i32 %i.09
+  store float %add, float* %arrayidx2, align 4
+  %add3 = add nuw nsw i32 %i.09, 1
+  %exitcond = icmp eq i32 %add3, 431
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body, !llvm.loop !10
+}
+
+define void @double(double* noalias nocapture %A, double* noalias nocapture readonly %B, double* noalias nocapture readonly %C) #0 {
+; CHECK-LABEL:        double(
+; PREFER-FOLDING:     for.body:
+; PREFER-FOLDING-NOT: vector.body:
+entry:
+  br label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.09 = phi i32 [ 0, %entry ], [ %add3, %for.body ]
+  %arrayidx = getelementptr inbounds double, double* %B, i32 %i.09
+  %0 = load double, double* %arrayidx, align 8
+  %arrayidx1 = getelementptr inbounds double, double* %C, i32 %i.09
+  %1 = load double, double* %arrayidx1, align 8
+  %add = fadd fast double %1, %0
+  %arrayidx2 = getelementptr inbounds double, double* %A, i32 %i.09
+  store double %add, double* %arrayidx2, align 8
+  %add3 = add nuw nsw i32 %i.09, 1
+  %exitcond = icmp eq i32 %add3, 431
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+; TODO: this fpext could be allowed, but we don't lower it very efficiently yet,
+; so reject this for now.
+define void @fpext_allowed(float* noalias nocapture %A, half* noalias nocapture readonly %B, float* noalias nocapture readonly %C) #0 {
+; CHECK-LABEL:        fpext_allowed(
+; PREFER-FOLDING:     vector.body:
+; PREFER-FOLDING-NOT: llvm.masked.load
+; PREFER-FOLDING-NOT: llvm.masked.store
+; PREFER-FOLDING:     br i1 %{{.*}}, label %{{.*}}, label %vector.body
+entry:
+  br label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.09 = phi i32 [ 0, %entry ], [ %add3, %for.body ]
+  %arrayidx = getelementptr inbounds half, half* %B, i32 %i.09
+  %0 = load half, half* %arrayidx, align 2
+  %conv = fpext half %0 to float
+  %arrayidx1 = getelementptr inbounds float, float* %C, i32 %i.09
+  %1 = load float, float* %arrayidx1, align 4
+  %add = fadd fast float %1, %conv
+  %arrayidx2 = getelementptr inbounds float, float* %A, i32 %i.09
+  store float %add, float* %arrayidx2, align 4
+  %add3 = add nuw nsw i32 %i.09, 1
+  %exitcond = icmp eq i32 %add3, 431
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+; TODO: this fptrunc could be allowed, but we don't lower it very efficiently yet,
+; so reject this for now.
+define void @fptrunc_allowed(half* noalias nocapture %A, float* noalias nocapture readonly %B, float* noalias nocapture readonly %C) #0 {
+; CHECK-LABEL:        fptrunc_allowed(
+; PREFER-FOLDING:     vector.body:
+; PREFER-FOLDING-NOT: llvm.masked.load
+; PREFER-FOLDING-NOT: llvm.masked.store
+; PREFER-FOLDING:     br i1 %{{.*}}, label %{{.*}}, label %vector.body
+entry:
+  br label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.09 = phi i32 [ 0, %entry ], [ %add3, %for.body ]
+  %arrayidx = getelementptr inbounds float, float* %B, i32 %i.09
+  %0 = load float, float* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds float, float* %C, i32 %i.09
+  %1 = load float, float* %arrayidx1, align 4
+  %add = fadd fast float %1, %0
+  %conv = fptrunc float %add to half
+  %arrayidx2 = getelementptr inbounds half, half* %A, i32 %i.09
+  store half %conv, half* %arrayidx2, align 2
+  %add3 = add nuw nsw i32 %i.09, 1
+  %exitcond = icmp eq i32 %add3, 431
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+define void @fptrunc_not_allowed(float* noalias nocapture %A, float* noalias nocapture readonly %B, float* noalias nocapture readonly %C, half* noalias nocapture %D) #0 {
+; CHECK-LABEL:        fptrunc_not_allowed(
+; PREFER-FOLDING:     vector.body:
+; PREFER-FOLDING-NOT: llvm.masked.load
+; PREFER-FOLDING-NOT: llvm.masked.store
+; PREFER-FOLDING:     br i1 %{{.*}}, label %{{.*}}, label %vector.body
+entry:
+  br label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.017 = phi i32 [ 0, %entry ], [ %add6, %for.body ]
+  %arrayidx = getelementptr inbounds float, float* %B, i32 %i.017
+  %0 = load float, float* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds float, float* %C, i32 %i.017
+  %1 = load float, float* %arrayidx1, align 4
+  %add = fadd fast float %1, %0
+  %arrayidx2 = getelementptr inbounds float, float* %A, i32 %i.017
+  store float %add, float* %arrayidx2, align 4
+  %conv = fptrunc float %add to half
+  %factor = fmul fast half %conv, 0xH4000
+  %arrayidx5 = getelementptr inbounds half, half* %D, i32 %i.017
+  store half %factor, half* %arrayidx5, align 2
+  %add6 = add nuw nsw i32 %i.017, 1
+  %exitcond = icmp eq i32 %add6, 431
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+attributes #0 = { nofree norecurse nounwind "target-features"="+armv8.1-m.main,+mve.fp" }
+
+!5 = distinct !{!5, !6}
+!6 = !{!"llvm.loop.vectorize.enable", i1 true}
+
+!7 = distinct !{!7, !8}
+!8 = !{!"llvm.loop.vectorize.predicate.enable", i1 false}
+
+!10 = distinct !{!10, !11}
+!11 = !{!"llvm.loop.vectorize.width", i32 4}
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/tail-folding-counting-down.ll b/llvm/test/Transforms/LoopVectorize/ARM/tail-folding-counting-down.ll
new file mode 100644
index 000000000..5a3438230
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/ARM/tail-folding-counting-down.ll
@@ -0,0 +1,47 @@
+; RUN: opt < %s -loop-vectorize -S | FileCheck %s
+; RUN: opt < %s -loop-vectorize -prefer-predicate-over-epilog -S | FileCheck %s
+; RUN: opt < %s -loop-vectorize -disable-mve-tail-predication=false -S | FileCheck %s
+
+; Check that when we can't predicate this loop that it is still vectorised (with
+; an epilogue).
+; TODO: the reason this can't be predicated is because a primary induction
+; variable can't be found (not yet) for this counting down loop. But with that
+; fixed, this should be able to be predicated.
+
+; CHECK-LABEL: vector.body:
+
+target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
+target triple = "thumbv8.1m.main-arm-unknown-eabihf"
+
+define dso_local void @foo(i8* noalias nocapture readonly %A, i8* noalias nocapture readonly %B, i8* noalias nocapture %C, i32 %N) #0 {
+entry:
+  %cmp6 = icmp eq i32 %N, 0
+  br i1 %cmp6, label %while.end, label %while.body.preheader
+
+while.body.preheader:
+  br label %while.body
+
+while.body:
+  %N.addr.010 = phi i32 [ %dec, %while.body ], [ %N, %while.body.preheader ]
+  %C.addr.09 = phi i8* [ %incdec.ptr4, %while.body ], [ %C, %while.body.preheader ]
+  %B.addr.08 = phi i8* [ %incdec.ptr1, %while.body ], [ %B, %while.body.preheader ]
+  %A.addr.07 = phi i8* [ %incdec.ptr, %while.body ], [ %A, %while.body.preheader ]
+  %incdec.ptr = getelementptr inbounds i8, i8* %A.addr.07, i32 1
+  %0 = load i8, i8* %A.addr.07, align 1
+  %incdec.ptr1 = getelementptr inbounds i8, i8* %B.addr.08, i32 1
+  %1 = load i8, i8* %B.addr.08, align 1
+  %add = add i8 %1, %0
+  %incdec.ptr4 = getelementptr inbounds i8, i8* %C.addr.09, i32 1
+  store i8 %add, i8* %C.addr.09, align 1
+  %dec = add i32 %N.addr.010, -1
+  %cmp = icmp eq i32 %dec, 0
+  br i1 %cmp, label %while.end.loopexit, label %while.body
+
+while.end.loopexit:
+  br label %while.end
+
+while.end:
+  ret void
+}
+
+attributes #0 = { nofree norecurse nounwind "target-features"="+armv8.1-m.main,+mve.fp" }
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/tail-loop-folding.ll b/llvm/test/Transforms/LoopVectorize/ARM/tail-loop-folding.ll
new file mode 100644
index 000000000..72321f040
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/ARM/tail-loop-folding.ll
@@ -0,0 +1,120 @@
+; RUN: opt < %s -loop-vectorize -S | \
+; RUN:  FileCheck %s -check-prefixes=COMMON,CHECK
+
+; RUN: opt < %s -loop-vectorize -prefer-predicate-over-epilog -S | \
+; RUN:   FileCheck -check-prefixes=COMMON,PREDFLAG %s
+
+target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
+target triple = "thumbv8.1m.main-arm-unknown-eabihf"
+
+define dso_local void @tail_folding(i32* noalias nocapture %A, i32* noalias nocapture readonly %B, i32* noalias nocapture readonly %C) #0 {
+; CHECK-LABEL: tail_folding(
+; CHECK: vector.body:
+;
+; This needs implementation of TTI::preferPredicateOverEpilogue,
+; then this will be tail-folded too:
+;
+; CHECK-NOT:  call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(
+; CHECK-NOT:  call void @llvm.masked.store.v4i32.p0v4i32(
+; CHECK:      br i1 %{{.*}}, label %{{.*}}, label %vector.body
+entry:
+  br label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %B, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds i32, i32* %C, i64 %indvars.iv
+  %1 = load i32, i32* %arrayidx2, align 4
+  %add = add nsw i32 %1, %0
+  %arrayidx4 = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  store i32 %add, i32* %arrayidx4, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 430
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+
+define dso_local void @tail_folding_enabled(i32* noalias nocapture %A, i32* noalias nocapture readonly %B, i32* noalias nocapture readonly %C) local_unnamed_addr #0 {
+; COMMON-LABEL: tail_folding_enabled(
+; COMMON: vector.body:
+; COMMON:   %[[WML1:.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(
+; COMMON:   %[[WML2:.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(
+; COMMON:   %[[ADD:.*]] = add nsw <4 x i32> %[[WML2]], %[[WML1]]
+; COMMON:   call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %[[ADD]]
+; COMMON:   br i1 %12, label %{{.*}}, label %vector.body
+entry:
+  br label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %B, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds i32, i32* %C, i64 %indvars.iv
+  %1 = load i32, i32* %arrayidx2, align 4
+  %add = add nsw i32 %1, %0
+  %arrayidx4 = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  store i32 %add, i32* %arrayidx4, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 430
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body, !llvm.loop !6
+}
+
+define dso_local void @tail_folding_disabled(i32* noalias nocapture %A, i32* noalias nocapture readonly %B, i32* noalias nocapture readonly %C) local_unnamed_addr #0 {
+; CHECK-LABEL: tail_folding_disabled(
+; CHECK:      vector.body:
+; CHECK-NOT:  @llvm.masked.load.v8i32.p0v8i32(
+; CHECK-NOT:  @llvm.masked.store.v8i32.p0v8i32(
+; CHECK:      br i1 %{{.*}}, label {{.*}}, label %vector.body
+
+; PREDFLAG-LABEL: tail_folding_disabled(
+; PREDFLAG:  vector.body:
+; PREDFLAG:  %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(
+; PREDFLAG:  %wide.masked.load1 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(
+; PREDFLAG:  %{{.*}} = add nsw <4 x i32> %wide.masked.load1, %wide.masked.load
+; PREDFLAG:  call void @llvm.masked.store.v4i32.p0v4i32(
+; PREDFLAG:  %index.next = add i64 %index, 4
+; PREDFLAG:  %12 = icmp eq i64 %index.next, 432
+; PREDFLAG:  br i1 %{{.*}}, label %middle.block, label %vector.body, !llvm.loop !6
+entry:
+  br label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %B, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds i32, i32* %C, i64 %indvars.iv
+  %1 = load i32, i32* %arrayidx2, align 4
+  %add = add nsw i32 %1, %0
+  %arrayidx4 = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  store i32 %add, i32* %arrayidx4, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 430
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body, !llvm.loop !10
+}
+
+; CHECK:      !0 = distinct !{!0, !1}
+; CHECK-NEXT: !1 = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK-NEXT: !2 = distinct !{!2, !3, !1}
+; CHECK-NEXT: !3 = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK-NEXT: !4 = distinct !{!4, !1}
+; CHECK-NEXT: !5 = distinct !{!5, !3, !1}
+; CHECK-NEXT: !6 = distinct !{!6, !1}
+attributes #0 = { nofree norecurse nounwind "target-features"="+armv8.1-m.main,+mve.fp" }
+
+!6 = distinct !{!6, !7, !8}
+!7 = !{!"llvm.loop.vectorize.predicate.enable", i1 true}
+!8 = !{!"llvm.loop.vectorize.enable", i1 true}
+
+!10 = distinct !{!10, !11, !12}
+!11 = !{!"llvm.loop.vectorize.predicate.enable", i1 false}
+!12 = !{!"llvm.loop.vectorize.enable", i1 true}
diff --git a/llvm/test/Transforms/LoopVectorize/PowerPC/reg-usage.ll b/llvm/test/Transforms/LoopVectorize/PowerPC/reg-usage.ll
new file mode 100644
index 000000000..13c03f01d
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/PowerPC/reg-usage.ll
@@ -0,0 +1,281 @@
+; RUN: opt < %s -debug-only=loop-vectorize -loop-vectorize -vectorizer-maximize-bandwidth -O2 -mtriple=powerpc64-unknown-linux -S -mcpu=pwr8 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-PWR8
+; RUN: opt < %s -debug-only=loop-vectorize -loop-vectorize -vectorizer-maximize-bandwidth -O2 -mtriple=powerpc64le-unknown-linux -S -mcpu=pwr9 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-PWR9
+; REQUIRES: asserts
+
+@a = global [1024 x i8] zeroinitializer, align 16
+@b = global [1024 x i8] zeroinitializer, align 16
+
+define i32 @foo() {
+; CHECK-LABEL: foo
+
+; CHECK-PWR8: Setting best plan to VF=16, UF=4
+
+; CHECK-PWR9: Setting best plan to VF=8, UF=8
+
+
+entry:
+  br label %for.body
+
+for.cond.cleanup:
+  %add.lcssa = phi i32 [ %add, %for.body ]
+  ret i32 %add.lcssa
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %s.015 = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %arrayidx = getelementptr inbounds [1024 x i8], [1024 x i8]* @a, i64 0, i64 %indvars.iv
+  %0 = load i8, i8* %arrayidx, align 1
+  %conv = zext i8 %0 to i32
+  %arrayidx2 = getelementptr inbounds [1024 x i8], [1024 x i8]* @b, i64 0, i64 %indvars.iv
+  %1 = load i8, i8* %arrayidx2, align 1
+  %conv3 = zext i8 %1 to i32
+  %sub = sub nsw i32 %conv, %conv3
+  %ispos = icmp sgt i32 %sub, -1
+  %neg = sub nsw i32 0, %sub
+  %2 = select i1 %ispos, i32 %sub, i32 %neg
+  %add = add nsw i32 %2, %s.015
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 1024
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+define i32 @goo() {
+; For indvars.iv used in a computating chain only feeding into getelementptr or cmp,
+; it will not have vector version and the vector register usage will not exceed the
+; available vector register number.
+
+; CHECK-LABEL: goo
+
+; CHECK: Setting best plan to VF=16, UF=4
+
+entry:
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body
+  %add.lcssa = phi i32 [ %add, %for.body ]
+  ret i32 %add.lcssa
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %s.015 = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %tmp1 = add nsw i64 %indvars.iv, 3
+  %arrayidx = getelementptr inbounds [1024 x i8], [1024 x i8]* @a, i64 0, i64 %tmp1
+  %tmp = load i8, i8* %arrayidx, align 1
+  %conv = zext i8 %tmp to i32
+  %tmp2 = add nsw i64 %indvars.iv, 2
+  %arrayidx2 = getelementptr inbounds [1024 x i8], [1024 x i8]* @b, i64 0, i64 %tmp2
+  %tmp3 = load i8, i8* %arrayidx2, align 1
+  %conv3 = zext i8 %tmp3 to i32
+  %sub = sub nsw i32 %conv, %conv3
+  %ispos = icmp sgt i32 %sub, -1
+  %neg = sub nsw i32 0, %sub
+  %tmp4 = select i1 %ispos, i32 %sub, i32 %neg
+  %add = add nsw i32 %tmp4, %s.015
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 1024
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+define i64 @bar(i64* nocapture %a) {
+; CHECK-LABEL: bar
+
+; CHECK: Setting best plan to VF=2, UF=12
+
+entry:
+  br label %for.body
+
+for.cond.cleanup:
+  %add2.lcssa = phi i64 [ %add2, %for.body ]
+  ret i64 %add2.lcssa
+
+for.body:
+  %i.012 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
+  %s.011 = phi i64 [ 0, %entry ], [ %add2, %for.body ]
+  %arrayidx = getelementptr inbounds i64, i64* %a, i64 %i.012
+  %0 = load i64, i64* %arrayidx, align 8
+  %add = add nsw i64 %0, %i.012
+  store i64 %add, i64* %arrayidx, align 8
+  %add2 = add nsw i64 %add, %s.011
+  %inc = add nuw nsw i64 %i.012, 1
+  %exitcond = icmp eq i64 %inc, 1024
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+@d = external global [0 x i64], align 8
+@e = external global [0 x i32], align 4
+@c = external global [0 x i32], align 4
+
+define void @hoo(i32 %n) {
+; CHECK-LABEL: hoo
+; CHECK: Setting best plan to VF=1, UF=12
+
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds [0 x i64], [0 x i64]* @d, i64 0, i64 %indvars.iv
+  %tmp = load i64, i64* %arrayidx, align 8
+  %arrayidx1 = getelementptr inbounds [0 x i32], [0 x i32]* @e, i64 0, i64 %tmp
+  %tmp1 = load i32, i32* %arrayidx1, align 4
+  %arrayidx3 = getelementptr inbounds [0 x i32], [0 x i32]* @c, i64 0, i64 %indvars.iv
+  store i32 %tmp1, i32* %arrayidx3, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 10000
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+define float @float_(float* nocapture readonly %a, float* nocapture readonly %b, i32 %n) {
+;CHECK-LABEL: float_
+;CHECK: LV(REG): VF = 1
+;CHECK: LV(REG): Found max usage: 2 item
+;CHECK-NEXT: LV(REG): RegisterClass: PPC::GPRRC, 2 registers
+;CHECK-NEXT: LV(REG): RegisterClass: PPC::VSXRC, 3 registers
+;CHECK: LV(REG): Found invariant usage: 1 item
+;CHECK-NEXT: LV(REG): RegisterClass: PPC::GPRRC, 1 registers
+
+entry:
+  %cmp = icmp sgt i32 %n, 0
+  br i1 %cmp, label %preheader, label %for.end
+
+preheader:
+  %t0 = sext i32 %n to i64
+  br label %for
+
+for:
+  %indvars.iv = phi i64 [ 0, %preheader ], [ %indvars.iv.next, %for ]
+  %s.02 = phi float [ 0.0, %preheader ], [ %add4, %for ]
+  %arrayidx = getelementptr inbounds float, float* %a, i64 %indvars.iv
+  %t1 = load float, float* %arrayidx, align 4
+  %arrayidx3 = getelementptr inbounds float, float* %b, i64 %indvars.iv
+  %t2 = load float, float* %arrayidx3, align 4
+  %add = fadd fast float %t1, %s.02
+  %add4 = fadd fast float %add, %t2
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 32
+  %cmp1 = icmp slt i64 %indvars.iv.next, %t0
+  br i1 %cmp1, label %for, label %loopexit
+
+loopexit:
+  %add4.lcssa = phi float [ %add4, %for ]
+  br label %for.end
+
+for.end:
+  %s.0.lcssa = phi float [ 0.0, %entry ], [ %add4.lcssa, %loopexit ]
+  ret float %s.0.lcssa
+}
+
+
+define void @double_(double* nocapture %A, i32 %n) nounwind uwtable ssp {
+;CHECK-LABEL: double_
+;CHECK-PWR8: LV(REG): VF = 2
+;CHECK-PWR8: LV(REG): Found max usage: 2 item
+;CHECK-PWR8-NEXT: LV(REG): RegisterClass: PPC::GPRRC, 2 registers
+;CHECK-PWR8-NEXT: LV(REG): RegisterClass: PPC::VSXRC, 5 registers
+;CHECK-PWR8: LV(REG): Found invariant usage: 1 item
+;CHECK-PWR8-NEXT: LV(REG): RegisterClass: PPC::VSXRC, 1 registers
+
+;CHECK-PWR9: LV(REG): VF = 1
+;CHECK-PWR9: LV(REG): Found max usage: 2 item
+;CHECK-PWR9-NEXT: LV(REG): RegisterClass: PPC::GPRRC, 2 registers
+;CHECK-PWR9-NEXT: LV(REG): RegisterClass: PPC::VSXRC, 5 registers
+;CHECK-PWR9: LV(REG): Found invariant usage: 1 item
+;CHECK-PWR9-NEXT: LV(REG): RegisterClass: PPC::GPRRC, 1 registers
+
+  %1 = sext i32 %n to i64
+  br label %2
+
+; <label>:2                                       ; preds = %2, %0
+  %indvars.iv = phi i64 [ %indvars.iv.next, %2 ], [ %1, %0 ]
+  %3 = getelementptr inbounds double, double* %A, i64 %indvars.iv
+  %4 = load double, double* %3, align 8
+  %5 = fadd double %4, 3.000000e+00
+  %6 = fmul double %4, 2.000000e+00
+  %7 = fadd double %5, %6
+  %8 = fadd double %7, 2.000000e+00
+  %9 = fmul double %8, 5.000000e-01
+  %10 = fadd double %6, %9
+  %11 = fsub double %10, %5
+  %12 = fadd double %4, %11
+  %13 = fdiv double %8, %12
+  %14 = fmul double %13, %8
+  %15 = fmul double %6, %14
+  %16 = fmul double %5, %15
+  %17 = fadd double %16, -3.000000e+00
+  %18 = fsub double %4, %5
+  %19 = fadd double %6, %18
+  %20 = fadd double %13, %19
+  %21 = fadd double %20, %17
+  %22 = fadd double %21, 3.000000e+00
+  %23 = fmul double %4, %22
+  store double %23, double* %3, align 8
+  %indvars.iv.next = add i64 %indvars.iv, -1
+  %24 = trunc i64 %indvars.iv to i32
+  %25 = icmp eq i32 %24, 0
+  br i1 %25, label %26, label %2
+
+; <label>:26                                      ; preds = %2
+  ret void
+}
+
+define ppc_fp128 @fp128_(ppc_fp128* nocapture %n, ppc_fp128 %d) nounwind readonly {
+;CHECK-LABEL: fp128_
+;CHECK: LV(REG): VF = 1
+;CHECK: LV(REG): Found max usage: 2 item
+;CHECK: LV(REG): RegisterClass: PPC::GPRRC, 2 registers
+;CHECK: LV(REG): RegisterClass: PPC::VRRC, 2 registers
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %i.06 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %x.05 = phi ppc_fp128 [ %d, %entry ], [ %sub, %for.body ]
+  %arrayidx = getelementptr inbounds ppc_fp128, ppc_fp128* %n, i32 %i.06
+  %0 = load ppc_fp128, ppc_fp128* %arrayidx, align 8
+  %sub = fsub ppc_fp128 %x.05, %0
+  %inc = add nsw i32 %i.06, 1
+  %exitcond = icmp eq i32 %inc, 2048
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret ppc_fp128 %sub
+}
+
+
+define void @fp16_(half* nocapture readonly %pIn, half* nocapture %pOut, i32 %numRows, i32 %numCols, i32 %scale.coerce) #0 {
+;CHECK-LABEL: fp16_
+;CHECK: LV(REG): VF = 1
+;CHECK: LV(REG): Found max usage: 2 item
+;CHECK: LV(REG): RegisterClass: PPC::GPRRC, 4 registers
+;CHECK: LV(REG): RegisterClass: PPC::VSXRC, 2 registers
+entry:
+  %tmp.0.extract.trunc = trunc i32 %scale.coerce to i16
+  %0 = bitcast i16 %tmp.0.extract.trunc to half
+  %mul = mul i32 %numCols, %numRows
+  %shr = lshr i32 %mul, 2
+  %cmp26 = icmp eq i32 %shr, 0
+  br i1 %cmp26, label %while.end, label %while.body
+
+while.body:                                       ; preds = %entry, %while.body
+  %pIn.addr.029 = phi half* [ %add.ptr, %while.body ], [ %pIn, %entry ]
+  %pOut.addr.028 = phi half* [ %add.ptr7, %while.body ], [ %pOut, %entry ]
+  %blkCnt.027 = phi i32 [ %dec, %while.body ], [ %shr, %entry ]
+  %1 = load half, half* %pIn.addr.029, align 2
+  %arrayidx2 = getelementptr inbounds half, half* %pIn.addr.029, i32 1
+  %2 = load half, half* %arrayidx2, align 2
+  %mul3 = fmul half %1, %0
+  %mul4 = fmul half %2, %0
+  store half %mul3, half* %pOut.addr.028, align 2
+  %arrayidx6 = getelementptr inbounds half, half* %pOut.addr.028, i32 1
+  store half %mul4, half* %arrayidx6, align 2
+  %add.ptr = getelementptr inbounds half, half* %pIn.addr.029, i32 2
+  %add.ptr7 = getelementptr inbounds half, half* %pOut.addr.028, i32 2
+  %dec = add nsw i32 %blkCnt.027, -1
+  %cmp = icmp eq i32 %dec, 0
+  br i1 %cmp, label %while.end, label %while.body
+
+while.end:                                        ; preds = %while.body, %entry
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/PowerPC/vectorize-bswap.ll b/llvm/test/Transforms/LoopVectorize/PowerPC/vectorize-bswap.ll
new file mode 100644
index 000000000..d9e517805
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/PowerPC/vectorize-bswap.ll
@@ -0,0 +1,97 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -loop-vectorize -mtriple=powerpc64le-unknown-unknown \
+; RUN:   -force-target-max-vector-interleave=1 -mcpu=pwr9 < %s | FileCheck %s
+define dso_local void @test(i32* %Arr, i32 signext %Len) {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 0, [[LEN:%.*]]
+; CHECK-NEXT:    br i1 [[CMP1]], label [[FOR_BODY_LR_PH:%.*]], label [[FOR_COND_CLEANUP:%.*]]
+; CHECK:       for.body.lr.ph:
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[LEN]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[LEN]], 4
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i32 [[LEN]], [[N_MOD_VF]]
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> undef, i32 [[INDEX]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <4 x i32> [[BROADCAST_SPLAT]], <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = sext i32 [[TMP0]] to i64
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, i32* [[ARR:%.*]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i32* [[TMP3]] to <4 x i32>*
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, <4 x i32>* [[TMP4]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = call <4 x i32> @llvm.bswap.v4i32(<4 x i32> [[WIDE_LOAD]])
+; CHECK-NEXT:    [[TMP6:%.*]] = sext i32 [[TMP0]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, i32* [[ARR]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, i32* [[TMP7]], i32 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i32* [[TMP8]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP5]], <4 x i32>* [[TMP9]], align 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !0
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[LEN]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_FOR_COND_CLEANUP_CRIT_EDGE:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_LR_PH]] ]
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.cond.for.cond.cleanup_crit_edge:
+; CHECK-NEXT:    br label [[FOR_COND_CLEANUP]]
+; CHECK:       for.cond.cleanup:
+; CHECK-NEXT:    br label [[FOR_END:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[I_02:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_INC:%.*]] ]
+; CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[I_02]] to i64
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[ARR]], i64 [[IDXPROM]]
+; CHECK-NEXT:    [[TMP11:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[TMP12:%.*]] = call i32 @llvm.bswap.i32(i32 [[TMP11]])
+; CHECK-NEXT:    [[IDXPROM1:%.*]] = sext i32 [[I_02]] to i64
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, i32* [[ARR]], i64 [[IDXPROM1]]
+; CHECK-NEXT:    store i32 [[TMP12]], i32* [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    br label [[FOR_INC]]
+; CHECK:       for.inc:
+; CHECK-NEXT:    [[INC]] = add nsw i32 [[I_02]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[INC]], [[LEN]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_COND_FOR_COND_CLEANUP_CRIT_EDGE]], !llvm.loop !2
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cmp1 = icmp slt i32 0, %Len
+  br i1 %cmp1, label %for.body.lr.ph, label %for.cond.cleanup
+
+for.body.lr.ph:                                   ; preds = %entry
+  br label %for.body
+
+for.cond.for.cond.cleanup_crit_edge:              ; preds = %for.inc
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.cond.for.cond.cleanup_crit_edge, %entry
+  br label %for.end
+
+for.body:                                         ; preds = %for.body.lr.ph, %for.inc
+  %i.02 = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %for.inc ]
+  %idxprom = sext i32 %i.02 to i64
+  %arrayidx = getelementptr inbounds i32, i32* %Arr, i64 %idxprom
+  %0 = load i32, i32* %arrayidx, align 4
+  %1 = call i32 @llvm.bswap.i32(i32 %0)
+  %idxprom1 = sext i32 %i.02 to i64
+  %arrayidx2 = getelementptr inbounds i32, i32* %Arr, i64 %idxprom1
+  store i32 %1, i32* %arrayidx2, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body
+  %inc = add nsw i32 %i.02, 1
+  %cmp = icmp slt i32 %inc, %Len
+  br i1 %cmp, label %for.body, label %for.cond.for.cond.cleanup_crit_edge
+
+for.end:                                          ; preds = %for.cond.cleanup
+  ret void
+}
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare i32 @llvm.bswap.i32(i32)
diff --git a/llvm/test/Transforms/LoopVectorize/SystemZ/predicated-first-order-recurrence.ll b/llvm/test/Transforms/LoopVectorize/SystemZ/predicated-first-order-recurrence.ll
new file mode 100644
index 000000000..7c5b5c53e
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/SystemZ/predicated-first-order-recurrence.ll
@@ -0,0 +1,103 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -loop-vectorize -mtriple=s390x-ibm-linux -mcpu=z13 -force-vector-width=2 -S %s | FileCheck %s
+
+; Test case from PR44020.
+
+; In func_21, %rec forms a first order recurrence and we predicate to avoid
+; scalar iteration overhead for a low trip count loop. Make sure we pick
+; the correct insertion point when fixing first order recurrences.
+
+@A = external dso_local global [5 x i32], align 4
+@B = external dso_local global [5 x i32], align 4
+
+define void @func_21() {
+; CHECK-LABEL: @func_21(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE4:%.*]] ]
+; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <2 x i32> [ <i32 undef, i32 0>, [[VECTOR_PH]] ], [ [[TMP12:%.*]], [[PRED_STORE_CONTINUE4]] ]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> undef, i64 [[INDEX]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <2 x i64> [[BROADCAST_SPLAT]], <i64 0, i64 1>
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ule <2 x i64> [[INDUCTION]], <i64 4, i64 4>
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x i1> [[TMP2]], i32 0
+; CHECK-NEXT:    br i1 [[TMP3]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
+; CHECK:       pred.load.if:
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds [5 x i32], [5 x i32]* @A, i64 0, i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP5:%.*]] = load i32, i32* [[TMP4]], align 4
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i32> undef, i32 [[TMP5]], i32 0
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE]]
+; CHECK:       pred.load.continue:
+; CHECK-NEXT:    [[TMP7:%.*]] = phi <2 x i32> [ undef, [[VECTOR_BODY]] ], [ [[TMP6]], [[PRED_LOAD_IF]] ]
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x i1> [[TMP2]], i32 1
+; CHECK-NEXT:    br i1 [[TMP8]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2:%.*]]
+; CHECK:       pred.load.if1:
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds [5 x i32], [5 x i32]* @A, i64 0, i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP10:%.*]] = load i32, i32* [[TMP9]], align 4
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <2 x i32> [[TMP7]], i32 [[TMP10]], i32 1
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE2]]
+; CHECK:       pred.load.continue2:
+; CHECK-NEXT:    [[TMP12]] = phi <2 x i32> [ [[TMP7]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP11]], [[PRED_LOAD_IF1]] ]
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <2 x i32> [[VECTOR_RECUR]], <2 x i32> [[TMP12]], <2 x i32> <i32 1, i32 2>
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x i1> [[TMP2]], i32 0
+; CHECK-NEXT:    br i1 [[TMP14]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
+; CHECK:       pred.store.if:
+; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds [5 x i32], [5 x i32]* @B, i64 0, i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <2 x i32> [[TMP13]], i32 0
+; CHECK-NEXT:    store i32 [[TMP16]], i32* [[TMP15]], align 4
+; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE]]
+; CHECK:       pred.store.continue:
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <2 x i1> [[TMP2]], i32 1
+; CHECK-NEXT:    br i1 [[TMP17]], label [[PRED_STORE_IF3:%.*]], label [[PRED_STORE_CONTINUE4]]
+; CHECK:       pred.store.if3:
+; CHECK-NEXT:    [[TMP18:%.*]] = getelementptr inbounds [5 x i32], [5 x i32]* @B, i64 0, i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP19:%.*]] = extractelement <2 x i32> [[TMP13]], i32 1
+; CHECK-NEXT:    store i32 [[TMP19]], i32* [[TMP18]], align 4
+; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE4]]
+; CHECK:       pred.store.continue4:
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], 6
+; CHECK-NEXT:    br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !0
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <2 x i32> [[TMP12]], i32 1
+; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <2 x i32> [[TMP12]], i32 0
+; CHECK-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 6, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[SCALAR_RECUR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[LV:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[A_PTR:%.*]] = getelementptr inbounds [5 x i32], [5 x i32]* @A, i64 0, i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[LV]] = load i32, i32* [[A_PTR]], align 4
+; CHECK-NEXT:    [[B_PTR:%.*]] = getelementptr inbounds [5 x i32], [5 x i32]* @B, i64 0, i64 [[INDVARS_IV]]
+; CHECK-NEXT:    store i32 [[SCALAR_RECUR]], i32* [[B_PTR]], align 4
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 5
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[EXIT]], label [[LOOP]], !llvm.loop !2
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:                                    ; preds = %loop, %entry
+  %rec = phi i32 [ 0, %entry], [ %lv, %loop ]
+  %indvars.iv = phi i64 [ 0, %entry], [ %indvars.iv.next, %loop ]
+  %A.ptr= getelementptr inbounds [5 x i32], [5 x i32]* @A, i64 0, i64 %indvars.iv
+  %lv = load i32, i32* %A.ptr, align 4
+  %B.ptr = getelementptr inbounds [5 x i32], [5 x i32]* @B, i64 0, i64 %indvars.iv
+  store i32 %rec, i32* %B.ptr, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 5
+  br i1 %exitcond, label %exit, label %loop
+
+exit:                                             ; preds = %loop
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/consecutive-ptr-uniforms.ll b/llvm/test/Transforms/LoopVectorize/X86/consecutive-ptr-uniforms.ll
index e18159f24..93285accb 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/consecutive-ptr-uniforms.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/consecutive-ptr-uniforms.ll
@@ -1,5 +1,6 @@
 ; REQUIRES: asserts
 ; RUN: opt < %s -loop-vectorize -instcombine -S -debug-only=loop-vectorize -disable-output -print-after=instcombine 2>&1 | FileCheck %s
+; RUN: opt < %s -loop-vectorize -force-vector-width=2 -S | FileCheck %s -check-prefix=FORCE
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
@@ -65,3 +66,85 @@ for.end:
 }
 
 attributes #0 = { "target-cpu"="knl" }
+
+; CHECK-LABEL: PR40816
+;
+; Check that scalar with predication instructions are not considered uniform
+; after vectorization, because that results in replicating a region instead of
+; having a single instance (out of VF). The predication stems from a tiny count
+; of 3 leading to folding the tail by masking using icmp ule <i, i+1> <= <2, 2>.
+;
+; CHECK:     LV: Found trip count: 3
+; CHECK:     LV: Found uniform instruction:   {{%.*}} = icmp eq i32 {{%.*}}, 0
+; CHECK-NOT: LV: Found uniform instruction:   {{%.*}} = load i32, i32* {{%.*}}, align 1
+; CHECK:     LV: Found not uniform being ScalarWithPredication:  {{%.*}} = load i32, i32* {{%.*}}, align 1
+; CHECK:     LV: Found scalar instruction:   {{%.*}} = getelementptr inbounds [3 x i32], [3 x i32]* @a, i32 0, i32 {{%.*}}
+;
+; FORCE-LABEL: @PR40816(
+; FORCE-NEXT:  entry:
+; FORCE-NEXT:    br i1 false, label {{%.*}}, label [[VECTOR_PH:%.*]]
+; FORCE:       vector.ph:
+; FORCE-NEXT:    br label [[VECTOR_BODY:%.*]]
+; FORCE:       vector.body:
+; FORCE-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE4:%.*]] ]
+; FORCE-NEXT:    [[VEC_IND:%.*]] = phi <2 x i32> [ <i32 0, i32 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_LOAD_CONTINUE4]] ]
+; FORCE-NEXT:    [[TMP0:%.*]] = add i32 [[INDEX]], 0
+; FORCE-NEXT:    [[TMP1:%.*]] = add i32 [[INDEX]], 1
+; FORCE-NEXT:    [[TMP2:%.*]] = icmp ule <2 x i32> [[VEC_IND]], <i32 2, i32 2>
+; FORCE-NEXT:    [[TMP3:%.*]] = extractelement <2 x i1> [[TMP2]], i32 0
+; FORCE-NEXT:    br i1 [[TMP3]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
+; FORCE:       pred.store.if:
+; FORCE-NEXT:    store i32 [[TMP0]], i32* @b, align 1
+; FORCE-NEXT:    br label [[PRED_STORE_CONTINUE]]
+; FORCE:       pred.store.continue:
+; FORCE-NEXT:    [[TMP4:%.*]] = extractelement <2 x i1> [[TMP2]], i32 1
+; FORCE-NEXT:    br i1 [[TMP4]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE2:%.*]]
+; FORCE:       pred.store.if1:
+; FORCE-NEXT:    store i32 [[TMP1]], i32* @b, align 1
+; FORCE-NEXT:    br label [[PRED_STORE_CONTINUE2]]
+; FORCE:       pred.store.continue2:
+; FORCE-NEXT:    [[TMP5:%.*]] = extractelement <2 x i1> [[TMP2]], i32 0
+; FORCE-NEXT:    br i1 [[TMP5]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
+; FORCE:       pred.load.if:
+; FORCE-NEXT:    [[TMP6:%.*]] = getelementptr inbounds [3 x i32], [3 x i32]* @a, i32 0, i32 [[TMP0]]
+; FORCE-NEXT:    [[TMP7:%.*]] = load i32, i32* [[TMP6]], align 1
+; FORCE-NEXT:    [[TMP8:%.*]] = insertelement <2 x i32> undef, i32 [[TMP7]], i32 0
+; FORCE-NEXT:    br label [[PRED_LOAD_CONTINUE]]
+; FORCE:       pred.load.continue:
+; FORCE-NEXT:    [[TMP9:%.*]] = phi <2 x i32> [ undef, [[PRED_STORE_CONTINUE2]] ], [ [[TMP8]], [[PRED_LOAD_IF]] ]
+; FORCE-NEXT:    [[TMP10:%.*]] = extractelement <2 x i1> [[TMP2]], i32 1
+; FORCE-NEXT:    br i1 [[TMP10]], label [[PRED_LOAD_IF3:%.*]], label [[PRED_LOAD_CONTINUE4]]
+; FORCE:       pred.load.if3:
+; FORCE-NEXT:    [[TMP11:%.*]] = getelementptr inbounds [3 x i32], [3 x i32]* @a, i32 0, i32 [[TMP1]]
+; FORCE-NEXT:    [[TMP12:%.*]] = load i32, i32* [[TMP11]], align 1
+; FORCE-NEXT:    [[TMP13:%.*]] = insertelement <2 x i32> [[TMP9]], i32 [[TMP12]], i32 1
+; FORCE-NEXT:    br label [[PRED_LOAD_CONTINUE4]]
+; FORCE:       pred.load.continue4:
+; FORCE-NEXT:    [[TMP14:%.*]] = phi <2 x i32> [ [[TMP9]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP13]], [[PRED_LOAD_IF3]] ]
+; FORCE-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 2
+; FORCE-NEXT:    [[VEC_IND_NEXT]] = add <2 x i32> [[VEC_IND]], <i32 2, i32 2>
+; FORCE-NEXT:    [[TMP15:%.*]] = icmp eq i32 [[INDEX_NEXT]], 4
+; FORCE-NEXT:    br i1 [[TMP15]], label {{%.*}}, label [[VECTOR_BODY]]
+;
+@a = internal constant [3 x i32] [i32 7, i32 7, i32 0], align 1
+@b = external global i32, align 1
+
+define void @PR40816() #1 {
+
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %0 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  store i32 %0, i32* @b, align 1
+  %arrayidx1 = getelementptr inbounds [3 x i32], [3 x i32]* @a, i32 0, i32 %0
+  %1 = load i32, i32* %arrayidx1, align 1
+  %cmp2 = icmp eq i32 %1, 0
+  %inc = add nuw nsw i32 %0, 1
+  br i1 %cmp2, label %return, label %for.body
+
+return:                                           ; preds = %for.body
+  ret void
+}
+
+attributes #1 = { "target-cpu"="core2" }
diff --git a/llvm/test/Transforms/LoopVectorize/X86/constant-fold.ll b/llvm/test/Transforms/LoopVectorize/X86/constant-fold.ll
index a73d91ba3..360a52c61 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/constant-fold.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/constant-fold.ll
@@ -27,7 +27,7 @@ define void @f1() {
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr [2 x i16*], [2 x i16*]* @b, i16 0, i64 [[TMP1]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr i16*, i16** [[TMP2]], i32 0
 ; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i16** [[TMP3]] to <2 x i16*>*
-; CHECK-NEXT:    store <2 x i16*> <i16* getelementptr inbounds (%rec8, %rec8* extractelement (<2 x %rec8*> getelementptr ([1 x %rec8], [1 x %rec8]* @a, <2 x i16> zeroinitializer, <2 x i64> zeroinitializer), i32 0), i32 0, i32 0), i16* getelementptr inbounds (%rec8, %rec8* extractelement (<2 x %rec8*> getelementptr ([1 x %rec8], [1 x %rec8]* @a, <2 x i16> zeroinitializer, <2 x i64> zeroinitializer), i32 1), i32 0, i32 0)>, <2 x i16*>* [[TMP4]], align 8
+; CHECK-NEXT:    store <2 x i16*> <i16* getelementptr inbounds ([1 x %rec8], [1 x %rec8]* @a, i32 0, i32 0, i32 0), i16* getelementptr inbounds ([1 x %rec8], [1 x %rec8]* @a, i32 0, i32 0, i32 0)>, <2 x i16*>* [[TMP4]], align 8
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 2
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], 2
 ; CHECK-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !0
diff --git a/llvm/test/Transforms/LoopVectorize/X86/cost-model-assert.ll b/llvm/test/Transforms/LoopVectorize/X86/cost-model-assert.ll
new file mode 100644
index 000000000..90f0ae4cd
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/cost-model-assert.ll
@@ -0,0 +1,127 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s  -loop-vectorize -S | FileCheck %s
+
+; This is a bugpoint reduction of a test from PR43582:
+; https://bugs.llvm.org/show_bug.cgi?id=43582
+
+; ...but it's over-simplifying the underlying question:
+; TODO: Should this be vectorized rather than allowing the backend to load combine?
+;       The original code is a bswap pattern.
+
+target datalayout = "e-m:w-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-w64-windows-gnu"
+
+define void @cff_index_load_offsets(i1 %cond, i8 %x, i8* %p) #0 {
+; CHECK-LABEL: @cff_index_load_offsets(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[IF_THEN:%.*]], label [[EXIT:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    br i1 true, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i8> undef, i8 [[X:%.*]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i8> [[BROADCAST_SPLATINSERT]], <4 x i8> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, i8* null, i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP2:%.*]] = zext <4 x i8> [[BROADCAST_SPLAT]] to <4 x i32>
+; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw <4 x i32> [[TMP2]], <i32 24, i32 24, i32 24, i32 24>
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, i8* [[P:%.*]], align 1, !tbaa !1
+; CHECK-NEXT:    [[TMP5:%.*]] = load i8, i8* [[P]], align 1, !tbaa !1
+; CHECK-NEXT:    [[TMP6:%.*]] = load i8, i8* [[P]], align 1, !tbaa !1
+; CHECK-NEXT:    [[TMP7:%.*]] = load i8, i8* [[P]], align 1, !tbaa !1
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <4 x i8> undef, i8 [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <4 x i8> [[TMP8]], i8 [[TMP5]], i32 1
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <4 x i8> [[TMP9]], i8 [[TMP6]], i32 2
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <4 x i8> [[TMP10]], i8 [[TMP7]], i32 3
+; CHECK-NEXT:    [[TMP12:%.*]] = zext <4 x i8> [[TMP11]] to <4 x i32>
+; CHECK-NEXT:    [[TMP13:%.*]] = shl nuw nsw <4 x i32> [[TMP12]], <i32 16, i32 16, i32 16, i32 16>
+; CHECK-NEXT:    [[TMP14:%.*]] = or <4 x i32> [[TMP13]], [[TMP3]]
+; CHECK-NEXT:    [[TMP15:%.*]] = load i8, i8* undef, align 1, !tbaa !1
+; CHECK-NEXT:    [[TMP16:%.*]] = load i8, i8* undef, align 1, !tbaa !1
+; CHECK-NEXT:    [[TMP17:%.*]] = load i8, i8* undef, align 1, !tbaa !1
+; CHECK-NEXT:    [[TMP18:%.*]] = load i8, i8* undef, align 1, !tbaa !1
+; CHECK-NEXT:    [[TMP19:%.*]] = or <4 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = or <4 x i32> [[TMP19]], zeroinitializer
+; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <4 x i32> [[TMP20]], i32 0
+; CHECK-NEXT:    store i32 [[TMP21]], i32* undef, align 4, !tbaa !4
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <4 x i32> [[TMP20]], i32 1
+; CHECK-NEXT:    store i32 [[TMP22]], i32* undef, align 4, !tbaa !4
+; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <4 x i32> [[TMP20]], i32 2
+; CHECK-NEXT:    store i32 [[TMP23]], i32* undef, align 4, !tbaa !4
+; CHECK-NEXT:    [[TMP24:%.*]] = extractelement <4 x i32> [[TMP20]], i32 3
+; CHECK-NEXT:    store i32 [[TMP24]], i32* undef, align 4, !tbaa !4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP25:%.*]] = icmp eq i64 [[INDEX_NEXT]], 0
+; CHECK-NEXT:    br i1 [[TMP25]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !6
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1, 0
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[SW_EPILOG:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i8* [ null, [[MIDDLE_BLOCK]] ], [ null, [[IF_THEN]] ]
+; CHECK-NEXT:    br label [[FOR_BODY68:%.*]]
+; CHECK:       for.body68:
+; CHECK-NEXT:    [[P_359:%.*]] = phi i8* [ [[ADD_PTR86:%.*]], [[FOR_BODY68]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[CONV70:%.*]] = zext i8 [[X]] to i32
+; CHECK-NEXT:    [[SHL71:%.*]] = shl nuw i32 [[CONV70]], 24
+; CHECK-NEXT:    [[TMP26:%.*]] = load i8, i8* [[P]], align 1, !tbaa !1
+; CHECK-NEXT:    [[CONV73:%.*]] = zext i8 [[TMP26]] to i32
+; CHECK-NEXT:    [[SHL74:%.*]] = shl nuw nsw i32 [[CONV73]], 16
+; CHECK-NEXT:    [[OR75:%.*]] = or i32 [[SHL74]], [[SHL71]]
+; CHECK-NEXT:    [[TMP27:%.*]] = load i8, i8* undef, align 1, !tbaa !1
+; CHECK-NEXT:    [[SHL78:%.*]] = shl nuw nsw i32 undef, 8
+; CHECK-NEXT:    [[OR79:%.*]] = or i32 [[OR75]], [[SHL78]]
+; CHECK-NEXT:    [[CONV81:%.*]] = zext i8 undef to i32
+; CHECK-NEXT:    [[OR83:%.*]] = or i32 [[OR79]], [[CONV81]]
+; CHECK-NEXT:    store i32 [[OR83]], i32* undef, align 4, !tbaa !4
+; CHECK-NEXT:    [[ADD_PTR86]] = getelementptr inbounds i8, i8* [[P_359]], i64 4
+; CHECK-NEXT:    [[CMP66:%.*]] = icmp ult i8* [[ADD_PTR86]], undef
+; CHECK-NEXT:    br i1 [[CMP66]], label [[FOR_BODY68]], label [[SW_EPILOG]], !llvm.loop !8
+; CHECK:       sw.epilog:
+; CHECK-NEXT:    unreachable
+; CHECK:       Exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br i1 %cond, label %if.then, label %Exit
+
+if.then:                                          ; preds = %entry
+  br label %for.body68
+
+for.body68:                                       ; preds = %for.body68, %if.then
+  %p.359 = phi i8* [ %add.ptr86, %for.body68 ], [ null, %if.then ]
+  %conv70 = zext i8 %x to i32
+  %shl71 = shl nuw i32 %conv70, 24
+  %0 = load i8, i8* %p, align 1, !tbaa !1
+  %conv73 = zext i8 %0 to i32
+  %shl74 = shl nuw nsw i32 %conv73, 16
+  %or75 = or i32 %shl74, %shl71
+  %1 = load i8, i8* undef, align 1, !tbaa !1
+  %shl78 = shl nuw nsw i32 undef, 8
+  %or79 = or i32 %or75, %shl78
+  %conv81 = zext i8 undef to i32
+  %or83 = or i32 %or79, %conv81
+  store i32 %or83, i32* undef, align 4, !tbaa !4
+  %add.ptr86 = getelementptr inbounds i8, i8* %p.359, i64 4
+  %cmp66 = icmp ult i8* %add.ptr86, undef
+  br i1 %cmp66, label %for.body68, label %sw.epilog
+
+sw.epilog:                                        ; preds = %for.body68
+  unreachable
+
+Exit:                                             ; preds = %entry
+  ret void
+}
+
+attributes #0 = { "use-soft-float"="false" }
+
+!llvm.ident = !{!0}
+
+!0 = !{!"clang version 10.0.0 (https://github.com/llvm/llvm-project.git 0fedc26a0dc0066f3968b9fea6a4e1f746c8d5a4)"}
+!1 = !{!2, !2, i64 0}
+!2 = !{!"omnipotent char", !3, i64 0}
+!3 = !{!"Simple C/C++ TBAA"}
+!4 = !{!5, !5, i64 0}
+!5 = !{!"long", !2, i64 0}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/illegal-parallel-loop-uniform-write.ll b/llvm/test/Transforms/LoopVectorize/X86/illegal-parallel-loop-uniform-write.ll
index 43c834ed8..3bb73e97a 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/illegal-parallel-loop-uniform-write.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/illegal-parallel-loop-uniform-write.ll
@@ -232,7 +232,7 @@ for.end15:                                        ; preds = %for.end.us, %entry
   ret void
 }
 
-attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !3 = !{!4, !5}
 !4 = !{!4}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/int128_no_gather.ll b/llvm/test/Transforms/LoopVectorize/X86/int128_no_gather.ll
index 4d7c0b6f6..d438fa050 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/int128_no_gather.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/int128_no_gather.ll
@@ -71,6 +71,6 @@ declare i32 @printf(i8*, ...) #1
 ; Function Attrs: nounwind
 declare i32 @puts(i8* nocapture readonly) #2
 
-attributes #0 = { noinline nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="skylake-avx512" "target-features"="+adx,+aes,+avx,+avx2,+avx512bw,+avx512cd,+avx512dq,+avx512f,+avx512vl,+bmi,+bmi2,+clflushopt,+clwb,+cx16,+f16c,+fma,+fsgsbase,+fxsr,+lzcnt,+mmx,+movbe,+mpx,+pclmul,+pku,+popcnt,+rdrnd,+rdseed,+rtm,+sgx,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87,+xsave,+xsavec,+xsaveopt,+xsaves" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="skylake-avx512" "target-features"="+adx,+aes,+avx,+avx2,+avx512bw,+avx512cd,+avx512dq,+avx512f,+avx512vl,+bmi,+bmi2,+clflushopt,+clwb,+cx16,+f16c,+fma,+fsgsbase,+fxsr,+lzcnt,+mmx,+movbe,+mpx,+pclmul,+pku,+popcnt,+rdrnd,+rdseed,+rtm,+sgx,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87,+xsave,+xsavec,+xsaveopt,+xsaves" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noinline nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="skylake-avx512" "target-features"="+adx,+aes,+avx,+avx2,+avx512bw,+avx512cd,+avx512dq,+avx512f,+avx512vl,+bmi,+bmi2,+clflushopt,+clwb,+cx16,+f16c,+fma,+fsgsbase,+fxsr,+lzcnt,+mmx,+movbe,+mpx,+pclmul,+pku,+popcnt,+rdrnd,+rdseed,+rtm,+sgx,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87,+xsave,+xsavec,+xsaveopt,+xsaves" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="skylake-avx512" "target-features"="+adx,+aes,+avx,+avx2,+avx512bw,+avx512cd,+avx512dq,+avx512f,+avx512vl,+bmi,+bmi2,+clflushopt,+clwb,+cx16,+f16c,+fma,+fsgsbase,+fxsr,+lzcnt,+mmx,+movbe,+mpx,+pclmul,+pku,+popcnt,+rdrnd,+rdseed,+rtm,+sgx,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87,+xsave,+xsavec,+xsaveopt,+xsaves" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind }
diff --git a/llvm/test/Transforms/LoopVectorize/X86/interleave_short_tc.ll b/llvm/test/Transforms/LoopVectorize/X86/interleave_short_tc.ll
new file mode 100644
index 000000000..78e69cb2d
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/interleave_short_tc.ll
@@ -0,0 +1,59 @@
+; Check that we won't interleave by more than "best known" estimated trip count.
+
+; The loop is expected to be vectorized by 4 and interleaving suppresed due to
+; short trip count which is controled by "tiny-trip-count-interleave-threshold".
+; RUN: opt  -passes=loop-vectorize -force-vector-width=4 -vectorizer-min-trip-count=4 -S < %s |  FileCheck %s
+; 
+; The loop is expected to be vectorized by 4 and computed interleaving factor is 1.
+; Thus the resulting step is 4.
+; RUN: opt  -passes=loop-vectorize -force-vector-width=4 -vectorizer-min-trip-count=4 -tiny-trip-count-interleave-threshold=4 -S < %s |  FileCheck %s
+
+; The loop is expected to be vectorized by 2 and computed interleaving factor is 2.
+; Thus the resulting step is 4.
+; RUN: opt  -passes=loop-vectorize -force-vector-width=2 -vectorizer-min-trip-count=4 -tiny-trip-count-interleave-threshold=4 -S < %s |  FileCheck %s
+
+; Check that we won't interleave by more than "best known" estimated trip count.
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@a = dso_local global [5 x i32] zeroinitializer, align 16
+@b = dso_local global [5 x i32] zeroinitializer, align 16
+
+; Function Attrs: nofree norecurse nounwind uwtable
+define dso_local void @_Z3fooi(i32 %M) local_unnamed_addr {
+; CHECK-LABEL: @_Z3fooi(
+; CHECK:       [[VECTOR_BODY:vector\.body]]:
+; CHECK:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH:%.*]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+;
+entry:
+  %cmp8 = icmp sgt i32 %M, 0
+  br i1 %cmp8, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:                               ; preds = %entry
+  %wide.trip.count = zext i32 %M to i64
+  br label %for.body
+
+for.cond.cleanup.loopexit:                        ; preds = %for.body
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.cond.cleanup.loopexit, %entry
+  ret void
+
+for.body:                                         ; preds = %for.body, %for.body.preheader
+  %indvars.iv = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds [5 x i32], [5 x i32]* @b, i64 0, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %1 = trunc i64 %indvars.iv to i32
+  %mul = mul nsw i32 %0, %1
+  %arrayidx2 = getelementptr inbounds [5 x i32], [5 x i32]* @a, i64 0, i64 %indvars.iv
+  %2 = load i32, i32* %arrayidx2, align 4
+  %add = add nsw i32 %2, %mul
+  store i32 %add, i32* %arrayidx2, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond, label %for.cond.cleanup.loopexit, label %for.body, !prof !1
+}
+
+!1 = !{!"branch_weights", i32 1, i32 5}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/interleaved-accesses-waw-dependency.ll b/llvm/test/Transforms/LoopVectorize/X86/interleaved-accesses-waw-dependency.ll
new file mode 100644
index 000000000..fa5ef4fa3
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/interleaved-accesses-waw-dependency.ll
@@ -0,0 +1,110 @@
+; RUN: opt < %s -loop-vectorize -force-vector-width=4 -force-vector-interleave=2 -debug-only=vectorutils -disable-output -enable-interleaved-mem-accesses=true 2>&1 | FileCheck %s
+; REQUIRES: asserts
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; PR40291
+; The loop does the following operation 3 times:
+; 1. Load x from memory;
+; 2. Store (x + 1) to this memory;
+; 3. if (x < 1), store 0 to this memory.
+
+; When scalar version stores 0 in all locations, the vector version should do
+; the same thing. However, with interleaving it does not honour the WAW dependency between
+; store 0 and store (x + 1) to the same memory.
+; For now, we identify such unsafe dependency and disable adding the
+; store into the interleaved group.
+; In this test case, because we disable adding store into i32* %storeaddr12 and
+; storeaddr22, we create interleaved groups with gaps and
+; disable that interleaved group. So, we are only left with valid interleaved
+; groups.
+
+
+
+
+; CHECK:      LV: Analyzing interleaved accesses...
+; CHECK:      LV: Creating an interleave group with:  store i32 %tmp34, i32* %storeaddr32, align 4
+; CHECK-NEXT: LV: Inserted:  store i32 %tmp24, i32* %storeaddr22, align 4
+; CHECK-NEXT:     into the interleave group with  store i32 %tmp34, i32* %storeaddr32, align 4
+; CHECK-NEXT: LV: Inserted:  store i32 %tmp14, i32* %storeaddr12, align 4
+; CHECK-NEXT:     into the interleave group with  store i32 %tmp34, i32* %storeaddr32, align 4
+; CHECK:      LV: Invalidated store group due to dependence between   store i32 %tmp24, i32* %storeaddr22, align 4 and   store i32 0, i32* %storeaddr22, align 4
+; CHECK-NEXT: LV: Creating an interleave group with:  store i32 %tmp24, i32* %storeaddr22, align 4
+; CHECK-NEXT: LV: Inserted:  store i32 %tmp14, i32* %storeaddr12, align 4
+; CHECK-NEXT:     into the interleave group with  store i32 %tmp24, i32* %storeaddr22, align 4
+; CHECK-NEXT: LV: Invalidated store group due to dependence between   store i32 %tmp14, i32* %storeaddr12, align 4 and   store i32 0, i32* %storeaddr12, align 4
+
+
+define void @test(i8* nonnull align 8 dereferenceable_or_null(24) %arg) {
+bb:
+  %tmp = getelementptr inbounds i8, i8* %arg, i64 16
+  %tmp1 = bitcast i8* %tmp to i8**
+  %tmp2 = load i8*, i8** %tmp1, align 8
+  %tmp3 = getelementptr inbounds i8, i8* %arg, i64 8
+  %tmp4 = bitcast i8* %tmp3 to i8**
+  %tmp5 = load i8*, i8** %tmp4, align 8
+  %tmp6 = getelementptr inbounds i8, i8* %tmp5, i64 12
+  %tmp7 = bitcast i8* %tmp6 to i32*
+  %tmp8 = getelementptr inbounds i8, i8* %tmp2, i64 12
+  br label %header
+
+header:                                              ; preds = %latch, %bb
+  %tmp10 = phi i64 [ %tmp41, %latch ], [ 3, %bb ]
+  %tmp11 = add nsw i64 %tmp10, -1
+  %storeaddr12 = getelementptr inbounds i32, i32* %tmp7, i64 %tmp11
+  %tmp13 = load i32, i32* %storeaddr12, align 4
+  %tmp14 = add i32 %tmp13, 1
+  store i32 %tmp14, i32* %storeaddr12, align 4
+  %tmp15 = icmp slt i32 %tmp13, 1
+  %tmp16 = xor i1 %tmp15, true
+  %tmp17 = zext i1 %tmp16 to i8
+  %tmp18 = getelementptr inbounds i8, i8* %tmp8, i64 %tmp10
+  store i8 %tmp17, i8* %tmp18, align 1
+  br i1 %tmp15, label %bb19, label %bb20
+
+bb19:                                             ; preds = %header
+  store i32 0, i32* %storeaddr12, align 4
+  br label %bb20
+
+bb20:                                             ; preds = %bb19, %header
+  %tmp21 = add nuw nsw i64 %tmp10, 1
+  %storeaddr22 = getelementptr inbounds i32, i32* %tmp7, i64 %tmp10
+  %tmp23 = load i32, i32* %storeaddr22, align 4
+  %tmp24 = add i32 %tmp23, 1
+  store i32 %tmp24, i32* %storeaddr22, align 4
+  %tmp25 = icmp slt i32 %tmp23, 1
+  %tmp26 = xor i1 %tmp25, true
+  %tmp27 = zext i1 %tmp26 to i8
+  %tmp28 = getelementptr inbounds i8, i8* %tmp8, i64 %tmp21
+  store i8 %tmp27, i8* %tmp28, align 1
+  br i1 %tmp25, label %bb29, label %bb30
+
+bb29:                                             ; preds = %bb20
+  store i32 0, i32* %storeaddr22, align 4
+  br label %bb30
+
+bb30:                                             ; preds = %bb29, %bb20
+  %tmp31 = add nuw nsw i64 %tmp10, 2
+  %storeaddr32 = getelementptr inbounds i32, i32* %tmp7, i64 %tmp21
+  %tmp33 = load i32, i32* %storeaddr32, align 4
+  %tmp34 = add i32 %tmp33, 1
+  store i32 %tmp34, i32* %storeaddr32, align 4
+  %tmp35 = icmp slt i32 %tmp33, 1
+  %tmp36 = xor i1 %tmp35, true
+  %tmp37 = zext i1 %tmp36 to i8
+  %tmp38 = getelementptr inbounds i8, i8* %tmp8, i64 %tmp31
+  store i8 %tmp37, i8* %tmp38, align 1
+  br i1 %tmp35, label %bb39, label %latch
+
+bb39:                                             ; preds = %bb30
+  store i32 0, i32* %storeaddr32, align 4
+  br label %latch
+
+latch:                                             ; preds = %bb39, %bb30
+  %tmp41 = add nuw nsw i64 %tmp10, 3
+  %tmp42 = icmp ugt i64 %tmp31, 67
+  br i1 %tmp42, label %exit, label %header
+
+exit:                                             ; preds = %latch
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/interleaving.ll b/llvm/test/Transforms/LoopVectorize/X86/interleaving.ll
index 9294c92b5..f12f35702 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/interleaving.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/interleaving.ll
@@ -1,6 +1,6 @@
 ; RUN: opt -S -mtriple=x86_64-pc_linux -loop-vectorize -instcombine < %s | FileCheck %s --check-prefix=NORMAL
-; RUN: opt -S -mtriple=x86_64-pc_linux -loop-vectorize -instcombine -mcpu=slm < %s | FileCheck %s --check-prefix=NORMAL
-; RUN: opt -S -mtriple=x86_64-pc_linux -loop-vectorize -instcombine -mcpu=atom < %s | FileCheck %s --check-prefix=ATOM
+; RUN: opt -S -mtriple=x86_64-pc_linux -loop-vectorize -instcombine -mcpu=slm < %s | FileCheck %s --check-prefix=SLOW
+; RUN: opt -S -mtriple=x86_64-pc_linux -loop-vectorize -instcombine -mcpu=atom < %s | FileCheck %s --check-prefix=SLOW
 
 ; NORMAL-LABEL: foo
 ; NORMAL: %[[WIDE:.*]] = load <8 x i32>, <8 x i32>* %{{.*}}, align 4
@@ -8,10 +8,10 @@
 ; NORMAL: %[[STRIDED2:.*]] = shufflevector <8 x i32> %wide.vec, <8 x i32> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
 ; NORMAL: add nsw <4 x i32> %[[STRIDED2]], %[[STRIDED1]]
 
-; ATOM-LABEL: foo
-; ATOM: load i32
-; ATOM: load i32
-; ATOM: store i32
+; SLOW-LABEL: foo
+; SLOW: load i32
+; SLOW: load i32
+; SLOW: store i32
 define void @foo(i32* noalias nocapture %a, i32* noalias nocapture readonly %b) {
 entry:
   br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/X86/load-deref-pred.ll b/llvm/test/Transforms/LoopVectorize/X86/load-deref-pred.ll
new file mode 100644
index 000000000..3accbcdd1
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/load-deref-pred.ll
@@ -0,0 +1,2128 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -force-vector-width=4 -loop-vectorize -mcpu=haswell < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128-ni:1"
+target triple = "x86_64-unknown-linux-gnu"
+
+;; This file includes tests for avoiding the need for a masked.load
+;; We don't need a masked.load for this due to deref facts, and can instead
+;; use a plain vector load.
+
+declare void @init(i32*)
+
+;; For ease of explanation, this one demonstrates
+;; with a range check, but there are better lowering options specifically for
+;; this test (i.e. reducing the iteration space of the vector copy), so
+;; following tests are written more generically.
+define i32 @test_explicit_pred(i64 %len) {
+; CHECK-LABEL: @test_explicit_pred(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ALLOCA:%.*]] = alloca [4096 x i32]
+; CHECK-NEXT:    [[BASE:%.*]] = bitcast [4096 x i32]* [[ALLOCA]] to i32*
+; CHECK-NEXT:    call void @init(i32* [[BASE]])
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> undef, i64 [[LEN:%.*]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT7:%.*]] = insertelement <4 x i64> undef, i64 [[LEN]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT8:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT7]], <4 x i64> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT9:%.*]] = insertelement <4 x i64> undef, i64 [[LEN]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT10:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT9]], <4 x i64> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT11:%.*]] = insertelement <4 x i64> undef, i64 [[LEN]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT12:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT11]], <4 x i64> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP36:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI4:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP37:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI5:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP38:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI6:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP39:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[STEP_ADD:%.*]] = add <4 x i64> [[VEC_IND]], <i64 4, i64 4, i64 4, i64 4>
+; CHECK-NEXT:    [[STEP_ADD1:%.*]] = add <4 x i64> [[STEP_ADD]], <i64 4, i64 4, i64 4, i64 4>
+; CHECK-NEXT:    [[STEP_ADD2:%.*]] = add <4 x i64> [[STEP_ADD1]], <i64 4, i64 4, i64 4, i64 4>
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[INDEX]], 3
+; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = add i64 [[INDEX]], 5
+; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[INDEX]], 6
+; CHECK-NEXT:    [[TMP7:%.*]] = add i64 [[INDEX]], 7
+; CHECK-NEXT:    [[TMP8:%.*]] = add i64 [[INDEX]], 8
+; CHECK-NEXT:    [[TMP9:%.*]] = add i64 [[INDEX]], 9
+; CHECK-NEXT:    [[TMP10:%.*]] = add i64 [[INDEX]], 10
+; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[INDEX]], 11
+; CHECK-NEXT:    [[TMP12:%.*]] = add i64 [[INDEX]], 12
+; CHECK-NEXT:    [[TMP13:%.*]] = add i64 [[INDEX]], 13
+; CHECK-NEXT:    [[TMP14:%.*]] = add i64 [[INDEX]], 14
+; CHECK-NEXT:    [[TMP15:%.*]] = add i64 [[INDEX]], 15
+; CHECK-NEXT:    [[TMP16:%.*]] = icmp slt <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[TMP17:%.*]] = icmp slt <4 x i64> [[STEP_ADD]], [[BROADCAST_SPLAT8]]
+; CHECK-NEXT:    [[TMP18:%.*]] = icmp slt <4 x i64> [[STEP_ADD1]], [[BROADCAST_SPLAT10]]
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp slt <4 x i64> [[STEP_ADD2]], [[BROADCAST_SPLAT12]]
+; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[TMP12]]
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i32, i32* [[TMP20]], i32 0
+; CHECK-NEXT:    [[TMP25:%.*]] = bitcast i32* [[TMP24]] to <4 x i32>*
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, <4 x i32>* [[TMP25]], align 4
+; CHECK-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i32, i32* [[TMP20]], i32 4
+; CHECK-NEXT:    [[TMP27:%.*]] = bitcast i32* [[TMP26]] to <4 x i32>*
+; CHECK-NEXT:    [[WIDE_LOAD13:%.*]] = load <4 x i32>, <4 x i32>* [[TMP27]], align 4
+; CHECK-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i32, i32* [[TMP20]], i32 8
+; CHECK-NEXT:    [[TMP29:%.*]] = bitcast i32* [[TMP28]] to <4 x i32>*
+; CHECK-NEXT:    [[WIDE_LOAD14:%.*]] = load <4 x i32>, <4 x i32>* [[TMP29]], align 4
+; CHECK-NEXT:    [[TMP30:%.*]] = getelementptr inbounds i32, i32* [[TMP20]], i32 12
+; CHECK-NEXT:    [[TMP31:%.*]] = bitcast i32* [[TMP30]] to <4 x i32>*
+; CHECK-NEXT:    [[WIDE_LOAD15:%.*]] = load <4 x i32>, <4 x i32>* [[TMP31]], align 4
+; CHECK-NEXT:    [[TMP32:%.*]] = xor <4 x i1> [[TMP16]], <i1 true, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    [[TMP33:%.*]] = xor <4 x i1> [[TMP17]], <i1 true, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    [[TMP34:%.*]] = xor <4 x i1> [[TMP18]], <i1 true, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    [[TMP35:%.*]] = xor <4 x i1> [[TMP19]], <i1 true, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP16]], <4 x i32> [[WIDE_LOAD]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[PREDPHI16:%.*]] = select <4 x i1> [[TMP17]], <4 x i32> [[WIDE_LOAD13]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[PREDPHI17:%.*]] = select <4 x i1> [[TMP18]], <4 x i32> [[WIDE_LOAD14]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[PREDPHI18:%.*]] = select <4 x i1> [[TMP19]], <4 x i32> [[WIDE_LOAD15]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP36]] = add <4 x i32> [[VEC_PHI]], [[PREDPHI]]
+; CHECK-NEXT:    [[TMP37]] = add <4 x i32> [[VEC_PHI4]], [[PREDPHI16]]
+; CHECK-NEXT:    [[TMP38]] = add <4 x i32> [[VEC_PHI5]], [[PREDPHI17]]
+; CHECK-NEXT:    [[TMP39]] = add <4 x i32> [[VEC_PHI6]], [[PREDPHI18]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 16
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[STEP_ADD2]], <i64 4, i64 4, i64 4, i64 4>
+; CHECK-NEXT:    [[TMP40:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4096
+; CHECK-NEXT:    br i1 [[TMP40]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !0
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <4 x i32> [[TMP37]], [[TMP36]]
+; CHECK-NEXT:    [[BIN_RDX19:%.*]] = add <4 x i32> [[TMP38]], [[BIN_RDX]]
+; CHECK-NEXT:    [[BIN_RDX20:%.*]] = add <4 x i32> [[TMP39]], [[BIN_RDX19]]
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[BIN_RDX20]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX21:%.*]] = add <4 x i32> [[BIN_RDX20]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF22:%.*]] = shufflevector <4 x i32> [[BIN_RDX21]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX23:%.*]] = add <4 x i32> [[BIN_RDX21]], [[RDX_SHUF22]]
+; CHECK-NEXT:    [[TMP41:%.*]] = extractelement <4 x i32> [[BIN_RDX23]], i32 0
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 4096, 4096
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[LOOP_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 4096, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[TMP41]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
+; CHECK-NEXT:    [[ACCUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ACCUM_NEXT:%.*]], [[LATCH]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT:    [[EARLYCND:%.*]] = icmp slt i64 [[IV]], [[LEN]]
+; CHECK-NEXT:    br i1 [[EARLYCND]], label [[PRED:%.*]], label [[LATCH]]
+; CHECK:       pred:
+; CHECK-NEXT:    [[ADDR:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[IV]]
+; CHECK-NEXT:    [[VAL:%.*]] = load i32, i32* [[ADDR]]
+; CHECK-NEXT:    br label [[LATCH]]
+; CHECK:       latch:
+; CHECK-NEXT:    [[VAL_PHI:%.*]] = phi i32 [ 0, [[LOOP]] ], [ [[VAL]], [[PRED]] ]
+; CHECK-NEXT:    [[ACCUM_NEXT]] = add i32 [[ACCUM]], [[VAL_PHI]]
+; CHECK-NEXT:    [[EXIT:%.*]] = icmp ugt i64 [[IV]], 4094
+; CHECK-NEXT:    br i1 [[EXIT]], label [[LOOP_EXIT]], label [[LOOP]], !llvm.loop !2
+; CHECK:       loop_exit:
+; CHECK-NEXT:    [[ACCUM_NEXT_LCSSA:%.*]] = phi i32 [ [[ACCUM_NEXT]], [[LATCH]] ], [ [[TMP41]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    ret i32 [[ACCUM_NEXT_LCSSA]]
+;
+entry:
+  %alloca = alloca [4096 x i32]
+  %base = bitcast [4096 x i32]* %alloca to i32*
+  call void @init(i32* %base)
+  br label %loop
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
+  %accum = phi i32 [ 0, %entry ], [ %accum.next, %latch ]
+  %iv.next = add i64 %iv, 1
+  %earlycnd = icmp slt i64 %iv, %len
+  br i1 %earlycnd, label %pred, label %latch
+pred:
+  %addr = getelementptr inbounds i32, i32* %base, i64 %iv
+  %val = load i32, i32* %addr
+  br label %latch
+latch:
+  %val.phi = phi i32 [0, %loop], [%val, %pred]
+  %accum.next = add i32 %accum, %val.phi
+  %exit = icmp ugt i64 %iv, 4094
+  br i1 %exit, label %loop_exit, label %loop
+
+loop_exit:
+  ret i32 %accum.next
+}
+
+;; Similiar to the above, but without an analyzeable condition.
+define i32 @test_explicit_pred_generic(i64 %len, i1* %test_base) {
+; CHECK-LABEL: @test_explicit_pred_generic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ALLOCA:%.*]] = alloca [4096 x i32]
+; CHECK-NEXT:    [[BASE:%.*]] = bitcast [4096 x i32]* [[ALLOCA]] to i32*
+; CHECK-NEXT:    call void @init(i32* [[BASE]])
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP80:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI4:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP81:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI5:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP82:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI6:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP83:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> undef, i64 [[INDEX]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], <i64 0, i64 1, i64 2, i64 3>
+; CHECK-NEXT:    [[INDUCTION1:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], <i64 4, i64 5, i64 6, i64 7>
+; CHECK-NEXT:    [[INDUCTION2:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], <i64 8, i64 9, i64 10, i64 11>
+; CHECK-NEXT:    [[INDUCTION3:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], <i64 12, i64 13, i64 14, i64 15>
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[INDEX]], 3
+; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = add i64 [[INDEX]], 5
+; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[INDEX]], 6
+; CHECK-NEXT:    [[TMP7:%.*]] = add i64 [[INDEX]], 7
+; CHECK-NEXT:    [[TMP8:%.*]] = add i64 [[INDEX]], 8
+; CHECK-NEXT:    [[TMP9:%.*]] = add i64 [[INDEX]], 9
+; CHECK-NEXT:    [[TMP10:%.*]] = add i64 [[INDEX]], 10
+; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[INDEX]], 11
+; CHECK-NEXT:    [[TMP12:%.*]] = add i64 [[INDEX]], 12
+; CHECK-NEXT:    [[TMP13:%.*]] = add i64 [[INDEX]], 13
+; CHECK-NEXT:    [[TMP14:%.*]] = add i64 [[INDEX]], 14
+; CHECK-NEXT:    [[TMP15:%.*]] = add i64 [[INDEX]], 15
+; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE:%.*]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP3]]
+; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP25:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP9]]
+; CHECK-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP10]]
+; CHECK-NEXT:    [[TMP27:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP11]]
+; CHECK-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP12]]
+; CHECK-NEXT:    [[TMP29:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP13]]
+; CHECK-NEXT:    [[TMP30:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP14]]
+; CHECK-NEXT:    [[TMP31:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP15]]
+; CHECK-NEXT:    [[TMP32:%.*]] = load i1, i1* [[TMP16]]
+; CHECK-NEXT:    [[TMP33:%.*]] = load i1, i1* [[TMP17]]
+; CHECK-NEXT:    [[TMP34:%.*]] = load i1, i1* [[TMP18]]
+; CHECK-NEXT:    [[TMP35:%.*]] = load i1, i1* [[TMP19]]
+; CHECK-NEXT:    [[TMP36:%.*]] = insertelement <4 x i1> undef, i1 [[TMP32]], i32 0
+; CHECK-NEXT:    [[TMP37:%.*]] = insertelement <4 x i1> [[TMP36]], i1 [[TMP33]], i32 1
+; CHECK-NEXT:    [[TMP38:%.*]] = insertelement <4 x i1> [[TMP37]], i1 [[TMP34]], i32 2
+; CHECK-NEXT:    [[TMP39:%.*]] = insertelement <4 x i1> [[TMP38]], i1 [[TMP35]], i32 3
+; CHECK-NEXT:    [[TMP40:%.*]] = load i1, i1* [[TMP20]]
+; CHECK-NEXT:    [[TMP41:%.*]] = load i1, i1* [[TMP21]]
+; CHECK-NEXT:    [[TMP42:%.*]] = load i1, i1* [[TMP22]]
+; CHECK-NEXT:    [[TMP43:%.*]] = load i1, i1* [[TMP23]]
+; CHECK-NEXT:    [[TMP44:%.*]] = insertelement <4 x i1> undef, i1 [[TMP40]], i32 0
+; CHECK-NEXT:    [[TMP45:%.*]] = insertelement <4 x i1> [[TMP44]], i1 [[TMP41]], i32 1
+; CHECK-NEXT:    [[TMP46:%.*]] = insertelement <4 x i1> [[TMP45]], i1 [[TMP42]], i32 2
+; CHECK-NEXT:    [[TMP47:%.*]] = insertelement <4 x i1> [[TMP46]], i1 [[TMP43]], i32 3
+; CHECK-NEXT:    [[TMP48:%.*]] = load i1, i1* [[TMP24]]
+; CHECK-NEXT:    [[TMP49:%.*]] = load i1, i1* [[TMP25]]
+; CHECK-NEXT:    [[TMP50:%.*]] = load i1, i1* [[TMP26]]
+; CHECK-NEXT:    [[TMP51:%.*]] = load i1, i1* [[TMP27]]
+; CHECK-NEXT:    [[TMP52:%.*]] = insertelement <4 x i1> undef, i1 [[TMP48]], i32 0
+; CHECK-NEXT:    [[TMP53:%.*]] = insertelement <4 x i1> [[TMP52]], i1 [[TMP49]], i32 1
+; CHECK-NEXT:    [[TMP54:%.*]] = insertelement <4 x i1> [[TMP53]], i1 [[TMP50]], i32 2
+; CHECK-NEXT:    [[TMP55:%.*]] = insertelement <4 x i1> [[TMP54]], i1 [[TMP51]], i32 3
+; CHECK-NEXT:    [[TMP56:%.*]] = load i1, i1* [[TMP28]]
+; CHECK-NEXT:    [[TMP57:%.*]] = load i1, i1* [[TMP29]]
+; CHECK-NEXT:    [[TMP58:%.*]] = load i1, i1* [[TMP30]]
+; CHECK-NEXT:    [[TMP59:%.*]] = load i1, i1* [[TMP31]]
+; CHECK-NEXT:    [[TMP60:%.*]] = insertelement <4 x i1> undef, i1 [[TMP56]], i32 0
+; CHECK-NEXT:    [[TMP61:%.*]] = insertelement <4 x i1> [[TMP60]], i1 [[TMP57]], i32 1
+; CHECK-NEXT:    [[TMP62:%.*]] = insertelement <4 x i1> [[TMP61]], i1 [[TMP58]], i32 2
+; CHECK-NEXT:    [[TMP63:%.*]] = insertelement <4 x i1> [[TMP62]], i1 [[TMP59]], i32 3
+; CHECK-NEXT:    [[TMP64:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP65:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP66:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP67:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[TMP12]]
+; CHECK-NEXT:    [[TMP68:%.*]] = getelementptr inbounds i32, i32* [[TMP64]], i32 0
+; CHECK-NEXT:    [[TMP69:%.*]] = bitcast i32* [[TMP68]] to <4 x i32>*
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, <4 x i32>* [[TMP69]], align 4
+; CHECK-NEXT:    [[TMP70:%.*]] = getelementptr inbounds i32, i32* [[TMP64]], i32 4
+; CHECK-NEXT:    [[TMP71:%.*]] = bitcast i32* [[TMP70]] to <4 x i32>*
+; CHECK-NEXT:    [[WIDE_LOAD7:%.*]] = load <4 x i32>, <4 x i32>* [[TMP71]], align 4
+; CHECK-NEXT:    [[TMP72:%.*]] = getelementptr inbounds i32, i32* [[TMP64]], i32 8
+; CHECK-NEXT:    [[TMP73:%.*]] = bitcast i32* [[TMP72]] to <4 x i32>*
+; CHECK-NEXT:    [[WIDE_LOAD8:%.*]] = load <4 x i32>, <4 x i32>* [[TMP73]], align 4
+; CHECK-NEXT:    [[TMP74:%.*]] = getelementptr inbounds i32, i32* [[TMP64]], i32 12
+; CHECK-NEXT:    [[TMP75:%.*]] = bitcast i32* [[TMP74]] to <4 x i32>*
+; CHECK-NEXT:    [[WIDE_LOAD9:%.*]] = load <4 x i32>, <4 x i32>* [[TMP75]], align 4
+; CHECK-NEXT:    [[TMP76:%.*]] = xor <4 x i1> [[TMP39]], <i1 true, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    [[TMP77:%.*]] = xor <4 x i1> [[TMP47]], <i1 true, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    [[TMP78:%.*]] = xor <4 x i1> [[TMP55]], <i1 true, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    [[TMP79:%.*]] = xor <4 x i1> [[TMP63]], <i1 true, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP39]], <4 x i32> [[WIDE_LOAD]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[PREDPHI10:%.*]] = select <4 x i1> [[TMP47]], <4 x i32> [[WIDE_LOAD7]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[PREDPHI11:%.*]] = select <4 x i1> [[TMP55]], <4 x i32> [[WIDE_LOAD8]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[PREDPHI12:%.*]] = select <4 x i1> [[TMP63]], <4 x i32> [[WIDE_LOAD9]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP80]] = add <4 x i32> [[VEC_PHI]], [[PREDPHI]]
+; CHECK-NEXT:    [[TMP81]] = add <4 x i32> [[VEC_PHI4]], [[PREDPHI10]]
+; CHECK-NEXT:    [[TMP82]] = add <4 x i32> [[VEC_PHI5]], [[PREDPHI11]]
+; CHECK-NEXT:    [[TMP83]] = add <4 x i32> [[VEC_PHI6]], [[PREDPHI12]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 16
+; CHECK-NEXT:    [[TMP84:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4096
+; CHECK-NEXT:    br i1 [[TMP84]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !4
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <4 x i32> [[TMP81]], [[TMP80]]
+; CHECK-NEXT:    [[BIN_RDX13:%.*]] = add <4 x i32> [[TMP82]], [[BIN_RDX]]
+; CHECK-NEXT:    [[BIN_RDX14:%.*]] = add <4 x i32> [[TMP83]], [[BIN_RDX13]]
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[BIN_RDX14]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX15:%.*]] = add <4 x i32> [[BIN_RDX14]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF16:%.*]] = shufflevector <4 x i32> [[BIN_RDX15]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX17:%.*]] = add <4 x i32> [[BIN_RDX15]], [[RDX_SHUF16]]
+; CHECK-NEXT:    [[TMP85:%.*]] = extractelement <4 x i32> [[BIN_RDX17]], i32 0
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 4096, 4096
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[LOOP_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 4096, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[TMP85]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
+; CHECK-NEXT:    [[ACCUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ACCUM_NEXT:%.*]], [[LATCH]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT:    [[TEST_ADDR:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[IV]]
+; CHECK-NEXT:    [[EARLYCND:%.*]] = load i1, i1* [[TEST_ADDR]]
+; CHECK-NEXT:    br i1 [[EARLYCND]], label [[PRED:%.*]], label [[LATCH]]
+; CHECK:       pred:
+; CHECK-NEXT:    [[ADDR:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[IV]]
+; CHECK-NEXT:    [[VAL:%.*]] = load i32, i32* [[ADDR]]
+; CHECK-NEXT:    br label [[LATCH]]
+; CHECK:       latch:
+; CHECK-NEXT:    [[VAL_PHI:%.*]] = phi i32 [ 0, [[LOOP]] ], [ [[VAL]], [[PRED]] ]
+; CHECK-NEXT:    [[ACCUM_NEXT]] = add i32 [[ACCUM]], [[VAL_PHI]]
+; CHECK-NEXT:    [[EXIT:%.*]] = icmp ugt i64 [[IV]], 4094
+; CHECK-NEXT:    br i1 [[EXIT]], label [[LOOP_EXIT]], label [[LOOP]], !llvm.loop !5
+; CHECK:       loop_exit:
+; CHECK-NEXT:    [[ACCUM_NEXT_LCSSA:%.*]] = phi i32 [ [[ACCUM_NEXT]], [[LATCH]] ], [ [[TMP85]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    ret i32 [[ACCUM_NEXT_LCSSA]]
+;
+entry:
+  %alloca = alloca [4096 x i32]
+  %base = bitcast [4096 x i32]* %alloca to i32*
+  call void @init(i32* %base)
+  br label %loop
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
+  %accum = phi i32 [ 0, %entry ], [ %accum.next, %latch ]
+  %iv.next = add i64 %iv, 1
+  %test_addr = getelementptr inbounds i1, i1* %test_base, i64 %iv
+  %earlycnd = load i1, i1* %test_addr
+  br i1 %earlycnd, label %pred, label %latch
+pred:
+  %addr = getelementptr inbounds i32, i32* %base, i64 %iv
+  %val = load i32, i32* %addr
+  br label %latch
+latch:
+  %val.phi = phi i32 [0, %loop], [%val, %pred]
+  %accum.next = add i32 %accum, %val.phi
+  %exit = icmp ugt i64 %iv, 4094
+  br i1 %exit, label %loop_exit, label %loop
+
+loop_exit:
+  ret i32 %accum.next
+}
+
+; Trivial case where the address loaded from it loop invariant (and yes,
+; there are better lowerings, this is a test of robustness of vectorization,
+; nothing more.)
+; TODO: currently shows predication which can be removed
+define i32 @test_invariant_address(i64 %len, i1* %test_base) {
+; CHECK-LABEL: @test_invariant_address(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ALLOCA:%.*]] = alloca [4096 x i32]
+; CHECK-NEXT:    [[BASE:%.*]] = bitcast [4096 x i32]* [[ALLOCA]] to i32*
+; CHECK-NEXT:    call void @init(i32* [[BASE]])
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP100:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI4:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP101:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI5:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP102:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI6:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP103:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> undef, i64 [[INDEX]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], <i64 0, i64 1, i64 2, i64 3>
+; CHECK-NEXT:    [[INDUCTION1:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], <i64 4, i64 5, i64 6, i64 7>
+; CHECK-NEXT:    [[INDUCTION2:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], <i64 8, i64 9, i64 10, i64 11>
+; CHECK-NEXT:    [[INDUCTION3:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], <i64 12, i64 13, i64 14, i64 15>
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[INDEX]], 3
+; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = add i64 [[INDEX]], 5
+; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[INDEX]], 6
+; CHECK-NEXT:    [[TMP7:%.*]] = add i64 [[INDEX]], 7
+; CHECK-NEXT:    [[TMP8:%.*]] = add i64 [[INDEX]], 8
+; CHECK-NEXT:    [[TMP9:%.*]] = add i64 [[INDEX]], 9
+; CHECK-NEXT:    [[TMP10:%.*]] = add i64 [[INDEX]], 10
+; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[INDEX]], 11
+; CHECK-NEXT:    [[TMP12:%.*]] = add i64 [[INDEX]], 12
+; CHECK-NEXT:    [[TMP13:%.*]] = add i64 [[INDEX]], 13
+; CHECK-NEXT:    [[TMP14:%.*]] = add i64 [[INDEX]], 14
+; CHECK-NEXT:    [[TMP15:%.*]] = add i64 [[INDEX]], 15
+; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE:%.*]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP3]]
+; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP25:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP9]]
+; CHECK-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP10]]
+; CHECK-NEXT:    [[TMP27:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP11]]
+; CHECK-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP12]]
+; CHECK-NEXT:    [[TMP29:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP13]]
+; CHECK-NEXT:    [[TMP30:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP14]]
+; CHECK-NEXT:    [[TMP31:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP15]]
+; CHECK-NEXT:    [[TMP32:%.*]] = load i1, i1* [[TMP16]]
+; CHECK-NEXT:    [[TMP33:%.*]] = load i1, i1* [[TMP17]]
+; CHECK-NEXT:    [[TMP34:%.*]] = load i1, i1* [[TMP18]]
+; CHECK-NEXT:    [[TMP35:%.*]] = load i1, i1* [[TMP19]]
+; CHECK-NEXT:    [[TMP36:%.*]] = insertelement <4 x i1> undef, i1 [[TMP32]], i32 0
+; CHECK-NEXT:    [[TMP37:%.*]] = insertelement <4 x i1> [[TMP36]], i1 [[TMP33]], i32 1
+; CHECK-NEXT:    [[TMP38:%.*]] = insertelement <4 x i1> [[TMP37]], i1 [[TMP34]], i32 2
+; CHECK-NEXT:    [[TMP39:%.*]] = insertelement <4 x i1> [[TMP38]], i1 [[TMP35]], i32 3
+; CHECK-NEXT:    [[TMP40:%.*]] = load i1, i1* [[TMP20]]
+; CHECK-NEXT:    [[TMP41:%.*]] = load i1, i1* [[TMP21]]
+; CHECK-NEXT:    [[TMP42:%.*]] = load i1, i1* [[TMP22]]
+; CHECK-NEXT:    [[TMP43:%.*]] = load i1, i1* [[TMP23]]
+; CHECK-NEXT:    [[TMP44:%.*]] = insertelement <4 x i1> undef, i1 [[TMP40]], i32 0
+; CHECK-NEXT:    [[TMP45:%.*]] = insertelement <4 x i1> [[TMP44]], i1 [[TMP41]], i32 1
+; CHECK-NEXT:    [[TMP46:%.*]] = insertelement <4 x i1> [[TMP45]], i1 [[TMP42]], i32 2
+; CHECK-NEXT:    [[TMP47:%.*]] = insertelement <4 x i1> [[TMP46]], i1 [[TMP43]], i32 3
+; CHECK-NEXT:    [[TMP48:%.*]] = load i1, i1* [[TMP24]]
+; CHECK-NEXT:    [[TMP49:%.*]] = load i1, i1* [[TMP25]]
+; CHECK-NEXT:    [[TMP50:%.*]] = load i1, i1* [[TMP26]]
+; CHECK-NEXT:    [[TMP51:%.*]] = load i1, i1* [[TMP27]]
+; CHECK-NEXT:    [[TMP52:%.*]] = insertelement <4 x i1> undef, i1 [[TMP48]], i32 0
+; CHECK-NEXT:    [[TMP53:%.*]] = insertelement <4 x i1> [[TMP52]], i1 [[TMP49]], i32 1
+; CHECK-NEXT:    [[TMP54:%.*]] = insertelement <4 x i1> [[TMP53]], i1 [[TMP50]], i32 2
+; CHECK-NEXT:    [[TMP55:%.*]] = insertelement <4 x i1> [[TMP54]], i1 [[TMP51]], i32 3
+; CHECK-NEXT:    [[TMP56:%.*]] = load i1, i1* [[TMP28]]
+; CHECK-NEXT:    [[TMP57:%.*]] = load i1, i1* [[TMP29]]
+; CHECK-NEXT:    [[TMP58:%.*]] = load i1, i1* [[TMP30]]
+; CHECK-NEXT:    [[TMP59:%.*]] = load i1, i1* [[TMP31]]
+; CHECK-NEXT:    [[TMP60:%.*]] = insertelement <4 x i1> undef, i1 [[TMP56]], i32 0
+; CHECK-NEXT:    [[TMP61:%.*]] = insertelement <4 x i1> [[TMP60]], i1 [[TMP57]], i32 1
+; CHECK-NEXT:    [[TMP62:%.*]] = insertelement <4 x i1> [[TMP61]], i1 [[TMP58]], i32 2
+; CHECK-NEXT:    [[TMP63:%.*]] = insertelement <4 x i1> [[TMP62]], i1 [[TMP59]], i32 3
+; CHECK-NEXT:    [[TMP64:%.*]] = load i32, i32* [[BASE]]
+; CHECK-NEXT:    [[TMP65:%.*]] = load i32, i32* [[BASE]]
+; CHECK-NEXT:    [[TMP66:%.*]] = load i32, i32* [[BASE]]
+; CHECK-NEXT:    [[TMP67:%.*]] = load i32, i32* [[BASE]]
+; CHECK-NEXT:    [[TMP68:%.*]] = insertelement <4 x i32> undef, i32 [[TMP64]], i32 0
+; CHECK-NEXT:    [[TMP69:%.*]] = insertelement <4 x i32> [[TMP68]], i32 [[TMP65]], i32 1
+; CHECK-NEXT:    [[TMP70:%.*]] = insertelement <4 x i32> [[TMP69]], i32 [[TMP66]], i32 2
+; CHECK-NEXT:    [[TMP71:%.*]] = insertelement <4 x i32> [[TMP70]], i32 [[TMP67]], i32 3
+; CHECK-NEXT:    [[TMP72:%.*]] = load i32, i32* [[BASE]]
+; CHECK-NEXT:    [[TMP73:%.*]] = load i32, i32* [[BASE]]
+; CHECK-NEXT:    [[TMP74:%.*]] = load i32, i32* [[BASE]]
+; CHECK-NEXT:    [[TMP75:%.*]] = load i32, i32* [[BASE]]
+; CHECK-NEXT:    [[TMP76:%.*]] = insertelement <4 x i32> undef, i32 [[TMP72]], i32 0
+; CHECK-NEXT:    [[TMP77:%.*]] = insertelement <4 x i32> [[TMP76]], i32 [[TMP73]], i32 1
+; CHECK-NEXT:    [[TMP78:%.*]] = insertelement <4 x i32> [[TMP77]], i32 [[TMP74]], i32 2
+; CHECK-NEXT:    [[TMP79:%.*]] = insertelement <4 x i32> [[TMP78]], i32 [[TMP75]], i32 3
+; CHECK-NEXT:    [[TMP80:%.*]] = load i32, i32* [[BASE]]
+; CHECK-NEXT:    [[TMP81:%.*]] = load i32, i32* [[BASE]]
+; CHECK-NEXT:    [[TMP82:%.*]] = load i32, i32* [[BASE]]
+; CHECK-NEXT:    [[TMP83:%.*]] = load i32, i32* [[BASE]]
+; CHECK-NEXT:    [[TMP84:%.*]] = insertelement <4 x i32> undef, i32 [[TMP80]], i32 0
+; CHECK-NEXT:    [[TMP85:%.*]] = insertelement <4 x i32> [[TMP84]], i32 [[TMP81]], i32 1
+; CHECK-NEXT:    [[TMP86:%.*]] = insertelement <4 x i32> [[TMP85]], i32 [[TMP82]], i32 2
+; CHECK-NEXT:    [[TMP87:%.*]] = insertelement <4 x i32> [[TMP86]], i32 [[TMP83]], i32 3
+; CHECK-NEXT:    [[TMP88:%.*]] = load i32, i32* [[BASE]]
+; CHECK-NEXT:    [[TMP89:%.*]] = load i32, i32* [[BASE]]
+; CHECK-NEXT:    [[TMP90:%.*]] = load i32, i32* [[BASE]]
+; CHECK-NEXT:    [[TMP91:%.*]] = load i32, i32* [[BASE]]
+; CHECK-NEXT:    [[TMP92:%.*]] = insertelement <4 x i32> undef, i32 [[TMP88]], i32 0
+; CHECK-NEXT:    [[TMP93:%.*]] = insertelement <4 x i32> [[TMP92]], i32 [[TMP89]], i32 1
+; CHECK-NEXT:    [[TMP94:%.*]] = insertelement <4 x i32> [[TMP93]], i32 [[TMP90]], i32 2
+; CHECK-NEXT:    [[TMP95:%.*]] = insertelement <4 x i32> [[TMP94]], i32 [[TMP91]], i32 3
+; CHECK-NEXT:    [[TMP96:%.*]] = xor <4 x i1> [[TMP39]], <i1 true, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    [[TMP97:%.*]] = xor <4 x i1> [[TMP47]], <i1 true, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    [[TMP98:%.*]] = xor <4 x i1> [[TMP55]], <i1 true, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    [[TMP99:%.*]] = xor <4 x i1> [[TMP63]], <i1 true, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP39]], <4 x i32> [[TMP71]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[PREDPHI7:%.*]] = select <4 x i1> [[TMP47]], <4 x i32> [[TMP79]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[PREDPHI8:%.*]] = select <4 x i1> [[TMP55]], <4 x i32> [[TMP87]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[PREDPHI9:%.*]] = select <4 x i1> [[TMP63]], <4 x i32> [[TMP95]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP100]] = add <4 x i32> [[VEC_PHI]], [[PREDPHI]]
+; CHECK-NEXT:    [[TMP101]] = add <4 x i32> [[VEC_PHI4]], [[PREDPHI7]]
+; CHECK-NEXT:    [[TMP102]] = add <4 x i32> [[VEC_PHI5]], [[PREDPHI8]]
+; CHECK-NEXT:    [[TMP103]] = add <4 x i32> [[VEC_PHI6]], [[PREDPHI9]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 16
+; CHECK-NEXT:    [[TMP104:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4096
+; CHECK-NEXT:    br i1 [[TMP104]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !6
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <4 x i32> [[TMP101]], [[TMP100]]
+; CHECK-NEXT:    [[BIN_RDX10:%.*]] = add <4 x i32> [[TMP102]], [[BIN_RDX]]
+; CHECK-NEXT:    [[BIN_RDX11:%.*]] = add <4 x i32> [[TMP103]], [[BIN_RDX10]]
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[BIN_RDX11]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX12:%.*]] = add <4 x i32> [[BIN_RDX11]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF13:%.*]] = shufflevector <4 x i32> [[BIN_RDX12]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX14:%.*]] = add <4 x i32> [[BIN_RDX12]], [[RDX_SHUF13]]
+; CHECK-NEXT:    [[TMP105:%.*]] = extractelement <4 x i32> [[BIN_RDX14]], i32 0
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 4096, 4096
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[LOOP_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 4096, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[TMP105]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
+; CHECK-NEXT:    [[ACCUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ACCUM_NEXT:%.*]], [[LATCH]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT:    [[TEST_ADDR:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[IV]]
+; CHECK-NEXT:    [[EARLYCND:%.*]] = load i1, i1* [[TEST_ADDR]]
+; CHECK-NEXT:    br i1 [[EARLYCND]], label [[PRED:%.*]], label [[LATCH]]
+; CHECK:       pred:
+; CHECK-NEXT:    [[VAL:%.*]] = load i32, i32* [[BASE]]
+; CHECK-NEXT:    br label [[LATCH]]
+; CHECK:       latch:
+; CHECK-NEXT:    [[VAL_PHI:%.*]] = phi i32 [ 0, [[LOOP]] ], [ [[VAL]], [[PRED]] ]
+; CHECK-NEXT:    [[ACCUM_NEXT]] = add i32 [[ACCUM]], [[VAL_PHI]]
+; CHECK-NEXT:    [[EXIT:%.*]] = icmp ugt i64 [[IV]], 4094
+; CHECK-NEXT:    br i1 [[EXIT]], label [[LOOP_EXIT]], label [[LOOP]], !llvm.loop !7
+; CHECK:       loop_exit:
+; CHECK-NEXT:    [[ACCUM_NEXT_LCSSA:%.*]] = phi i32 [ [[ACCUM_NEXT]], [[LATCH]] ], [ [[TMP105]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    ret i32 [[ACCUM_NEXT_LCSSA]]
+;
+entry:
+  %alloca = alloca [4096 x i32]
+  %base = bitcast [4096 x i32]* %alloca to i32*
+  call void @init(i32* %base)
+  br label %loop
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
+  %accum = phi i32 [ 0, %entry ], [ %accum.next, %latch ]
+  %iv.next = add i64 %iv, 1
+  %test_addr = getelementptr inbounds i1, i1* %test_base, i64 %iv
+  %earlycnd = load i1, i1* %test_addr
+  br i1 %earlycnd, label %pred, label %latch
+pred:
+  %val = load i32, i32* %base
+  br label %latch
+latch:
+  %val.phi = phi i32 [0, %loop], [%val, %pred]
+  %accum.next = add i32 %accum, %val.phi
+  %exit = icmp ugt i64 %iv, 4094
+  br i1 %exit, label %loop_exit, label %loop
+
+loop_exit:
+  ret i32 %accum.next
+}
+
+; Overlapping loads - Fails alignment checking, not dereferenceability
+define i32 @test_step_narrower_than_access(i64 %len, i1* %test_base) {
+; CHECK-LABEL: @test_step_narrower_than_access(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ALLOCA:%.*]] = alloca [4096 x i32]
+; CHECK-NEXT:    [[BASE:%.*]] = bitcast [4096 x i32]* [[ALLOCA]] to i32*
+; CHECK-NEXT:    call void @init(i32* [[BASE]])
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE36:%.*]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP180:%.*]], [[PRED_LOAD_CONTINUE36]] ]
+; CHECK-NEXT:    [[VEC_PHI4:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP181:%.*]], [[PRED_LOAD_CONTINUE36]] ]
+; CHECK-NEXT:    [[VEC_PHI5:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP182:%.*]], [[PRED_LOAD_CONTINUE36]] ]
+; CHECK-NEXT:    [[VEC_PHI6:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP183:%.*]], [[PRED_LOAD_CONTINUE36]] ]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> undef, i64 [[INDEX]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], <i64 0, i64 1, i64 2, i64 3>
+; CHECK-NEXT:    [[INDUCTION1:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], <i64 4, i64 5, i64 6, i64 7>
+; CHECK-NEXT:    [[INDUCTION2:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], <i64 8, i64 9, i64 10, i64 11>
+; CHECK-NEXT:    [[INDUCTION3:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], <i64 12, i64 13, i64 14, i64 15>
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[INDEX]], 3
+; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = add i64 [[INDEX]], 5
+; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[INDEX]], 6
+; CHECK-NEXT:    [[TMP7:%.*]] = add i64 [[INDEX]], 7
+; CHECK-NEXT:    [[TMP8:%.*]] = add i64 [[INDEX]], 8
+; CHECK-NEXT:    [[TMP9:%.*]] = add i64 [[INDEX]], 9
+; CHECK-NEXT:    [[TMP10:%.*]] = add i64 [[INDEX]], 10
+; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[INDEX]], 11
+; CHECK-NEXT:    [[TMP12:%.*]] = add i64 [[INDEX]], 12
+; CHECK-NEXT:    [[TMP13:%.*]] = add i64 [[INDEX]], 13
+; CHECK-NEXT:    [[TMP14:%.*]] = add i64 [[INDEX]], 14
+; CHECK-NEXT:    [[TMP15:%.*]] = add i64 [[INDEX]], 15
+; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE:%.*]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP3]]
+; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP25:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP9]]
+; CHECK-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP10]]
+; CHECK-NEXT:    [[TMP27:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP11]]
+; CHECK-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP12]]
+; CHECK-NEXT:    [[TMP29:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP13]]
+; CHECK-NEXT:    [[TMP30:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP14]]
+; CHECK-NEXT:    [[TMP31:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP15]]
+; CHECK-NEXT:    [[TMP32:%.*]] = load i1, i1* [[TMP16]]
+; CHECK-NEXT:    [[TMP33:%.*]] = load i1, i1* [[TMP17]]
+; CHECK-NEXT:    [[TMP34:%.*]] = load i1, i1* [[TMP18]]
+; CHECK-NEXT:    [[TMP35:%.*]] = load i1, i1* [[TMP19]]
+; CHECK-NEXT:    [[TMP36:%.*]] = insertelement <4 x i1> undef, i1 [[TMP32]], i32 0
+; CHECK-NEXT:    [[TMP37:%.*]] = insertelement <4 x i1> [[TMP36]], i1 [[TMP33]], i32 1
+; CHECK-NEXT:    [[TMP38:%.*]] = insertelement <4 x i1> [[TMP37]], i1 [[TMP34]], i32 2
+; CHECK-NEXT:    [[TMP39:%.*]] = insertelement <4 x i1> [[TMP38]], i1 [[TMP35]], i32 3
+; CHECK-NEXT:    [[TMP40:%.*]] = load i1, i1* [[TMP20]]
+; CHECK-NEXT:    [[TMP41:%.*]] = load i1, i1* [[TMP21]]
+; CHECK-NEXT:    [[TMP42:%.*]] = load i1, i1* [[TMP22]]
+; CHECK-NEXT:    [[TMP43:%.*]] = load i1, i1* [[TMP23]]
+; CHECK-NEXT:    [[TMP44:%.*]] = insertelement <4 x i1> undef, i1 [[TMP40]], i32 0
+; CHECK-NEXT:    [[TMP45:%.*]] = insertelement <4 x i1> [[TMP44]], i1 [[TMP41]], i32 1
+; CHECK-NEXT:    [[TMP46:%.*]] = insertelement <4 x i1> [[TMP45]], i1 [[TMP42]], i32 2
+; CHECK-NEXT:    [[TMP47:%.*]] = insertelement <4 x i1> [[TMP46]], i1 [[TMP43]], i32 3
+; CHECK-NEXT:    [[TMP48:%.*]] = load i1, i1* [[TMP24]]
+; CHECK-NEXT:    [[TMP49:%.*]] = load i1, i1* [[TMP25]]
+; CHECK-NEXT:    [[TMP50:%.*]] = load i1, i1* [[TMP26]]
+; CHECK-NEXT:    [[TMP51:%.*]] = load i1, i1* [[TMP27]]
+; CHECK-NEXT:    [[TMP52:%.*]] = insertelement <4 x i1> undef, i1 [[TMP48]], i32 0
+; CHECK-NEXT:    [[TMP53:%.*]] = insertelement <4 x i1> [[TMP52]], i1 [[TMP49]], i32 1
+; CHECK-NEXT:    [[TMP54:%.*]] = insertelement <4 x i1> [[TMP53]], i1 [[TMP50]], i32 2
+; CHECK-NEXT:    [[TMP55:%.*]] = insertelement <4 x i1> [[TMP54]], i1 [[TMP51]], i32 3
+; CHECK-NEXT:    [[TMP56:%.*]] = load i1, i1* [[TMP28]]
+; CHECK-NEXT:    [[TMP57:%.*]] = load i1, i1* [[TMP29]]
+; CHECK-NEXT:    [[TMP58:%.*]] = load i1, i1* [[TMP30]]
+; CHECK-NEXT:    [[TMP59:%.*]] = load i1, i1* [[TMP31]]
+; CHECK-NEXT:    [[TMP60:%.*]] = insertelement <4 x i1> undef, i1 [[TMP56]], i32 0
+; CHECK-NEXT:    [[TMP61:%.*]] = insertelement <4 x i1> [[TMP60]], i1 [[TMP57]], i32 1
+; CHECK-NEXT:    [[TMP62:%.*]] = insertelement <4 x i1> [[TMP61]], i1 [[TMP58]], i32 2
+; CHECK-NEXT:    [[TMP63:%.*]] = insertelement <4 x i1> [[TMP62]], i1 [[TMP59]], i32 3
+; CHECK-NEXT:    [[TMP64:%.*]] = extractelement <4 x i1> [[TMP39]], i32 0
+; CHECK-NEXT:    br i1 [[TMP64]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
+; CHECK:       pred.load.if:
+; CHECK-NEXT:    [[TMP65:%.*]] = bitcast i32* [[BASE]] to i16*
+; CHECK-NEXT:    [[TMP66:%.*]] = getelementptr inbounds i16, i16* [[TMP65]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP67:%.*]] = bitcast i16* [[TMP66]] to i32*
+; CHECK-NEXT:    [[TMP68:%.*]] = load i32, i32* [[TMP67]]
+; CHECK-NEXT:    [[TMP69:%.*]] = insertelement <4 x i32> undef, i32 [[TMP68]], i32 0
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE]]
+; CHECK:       pred.load.continue:
+; CHECK-NEXT:    [[TMP70:%.*]] = phi <4 x i32> [ undef, [[VECTOR_BODY]] ], [ [[TMP69]], [[PRED_LOAD_IF]] ]
+; CHECK-NEXT:    [[TMP71:%.*]] = extractelement <4 x i1> [[TMP39]], i32 1
+; CHECK-NEXT:    br i1 [[TMP71]], label [[PRED_LOAD_IF7:%.*]], label [[PRED_LOAD_CONTINUE8:%.*]]
+; CHECK:       pred.load.if7:
+; CHECK-NEXT:    [[TMP72:%.*]] = bitcast i32* [[BASE]] to i16*
+; CHECK-NEXT:    [[TMP73:%.*]] = getelementptr inbounds i16, i16* [[TMP72]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP74:%.*]] = bitcast i16* [[TMP73]] to i32*
+; CHECK-NEXT:    [[TMP75:%.*]] = load i32, i32* [[TMP74]]
+; CHECK-NEXT:    [[TMP76:%.*]] = insertelement <4 x i32> [[TMP70]], i32 [[TMP75]], i32 1
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE8]]
+; CHECK:       pred.load.continue8:
+; CHECK-NEXT:    [[TMP77:%.*]] = phi <4 x i32> [ [[TMP70]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP76]], [[PRED_LOAD_IF7]] ]
+; CHECK-NEXT:    [[TMP78:%.*]] = extractelement <4 x i1> [[TMP39]], i32 2
+; CHECK-NEXT:    br i1 [[TMP78]], label [[PRED_LOAD_IF9:%.*]], label [[PRED_LOAD_CONTINUE10:%.*]]
+; CHECK:       pred.load.if9:
+; CHECK-NEXT:    [[TMP79:%.*]] = bitcast i32* [[BASE]] to i16*
+; CHECK-NEXT:    [[TMP80:%.*]] = getelementptr inbounds i16, i16* [[TMP79]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP81:%.*]] = bitcast i16* [[TMP80]] to i32*
+; CHECK-NEXT:    [[TMP82:%.*]] = load i32, i32* [[TMP81]]
+; CHECK-NEXT:    [[TMP83:%.*]] = insertelement <4 x i32> [[TMP77]], i32 [[TMP82]], i32 2
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE10]]
+; CHECK:       pred.load.continue10:
+; CHECK-NEXT:    [[TMP84:%.*]] = phi <4 x i32> [ [[TMP77]], [[PRED_LOAD_CONTINUE8]] ], [ [[TMP83]], [[PRED_LOAD_IF9]] ]
+; CHECK-NEXT:    [[TMP85:%.*]] = extractelement <4 x i1> [[TMP39]], i32 3
+; CHECK-NEXT:    br i1 [[TMP85]], label [[PRED_LOAD_IF11:%.*]], label [[PRED_LOAD_CONTINUE12:%.*]]
+; CHECK:       pred.load.if11:
+; CHECK-NEXT:    [[TMP86:%.*]] = bitcast i32* [[BASE]] to i16*
+; CHECK-NEXT:    [[TMP87:%.*]] = getelementptr inbounds i16, i16* [[TMP86]], i64 [[TMP3]]
+; CHECK-NEXT:    [[TMP88:%.*]] = bitcast i16* [[TMP87]] to i32*
+; CHECK-NEXT:    [[TMP89:%.*]] = load i32, i32* [[TMP88]]
+; CHECK-NEXT:    [[TMP90:%.*]] = insertelement <4 x i32> [[TMP84]], i32 [[TMP89]], i32 3
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE12]]
+; CHECK:       pred.load.continue12:
+; CHECK-NEXT:    [[TMP91:%.*]] = phi <4 x i32> [ [[TMP84]], [[PRED_LOAD_CONTINUE10]] ], [ [[TMP90]], [[PRED_LOAD_IF11]] ]
+; CHECK-NEXT:    [[TMP92:%.*]] = extractelement <4 x i1> [[TMP47]], i32 0
+; CHECK-NEXT:    br i1 [[TMP92]], label [[PRED_LOAD_IF13:%.*]], label [[PRED_LOAD_CONTINUE14:%.*]]
+; CHECK:       pred.load.if13:
+; CHECK-NEXT:    [[TMP93:%.*]] = bitcast i32* [[BASE]] to i16*
+; CHECK-NEXT:    [[TMP94:%.*]] = getelementptr inbounds i16, i16* [[TMP93]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP95:%.*]] = bitcast i16* [[TMP94]] to i32*
+; CHECK-NEXT:    [[TMP96:%.*]] = load i32, i32* [[TMP95]]
+; CHECK-NEXT:    [[TMP97:%.*]] = insertelement <4 x i32> undef, i32 [[TMP96]], i32 0
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE14]]
+; CHECK:       pred.load.continue14:
+; CHECK-NEXT:    [[TMP98:%.*]] = phi <4 x i32> [ undef, [[PRED_LOAD_CONTINUE12]] ], [ [[TMP97]], [[PRED_LOAD_IF13]] ]
+; CHECK-NEXT:    [[TMP99:%.*]] = extractelement <4 x i1> [[TMP47]], i32 1
+; CHECK-NEXT:    br i1 [[TMP99]], label [[PRED_LOAD_IF15:%.*]], label [[PRED_LOAD_CONTINUE16:%.*]]
+; CHECK:       pred.load.if15:
+; CHECK-NEXT:    [[TMP100:%.*]] = bitcast i32* [[BASE]] to i16*
+; CHECK-NEXT:    [[TMP101:%.*]] = getelementptr inbounds i16, i16* [[TMP100]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP102:%.*]] = bitcast i16* [[TMP101]] to i32*
+; CHECK-NEXT:    [[TMP103:%.*]] = load i32, i32* [[TMP102]]
+; CHECK-NEXT:    [[TMP104:%.*]] = insertelement <4 x i32> [[TMP98]], i32 [[TMP103]], i32 1
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE16]]
+; CHECK:       pred.load.continue16:
+; CHECK-NEXT:    [[TMP105:%.*]] = phi <4 x i32> [ [[TMP98]], [[PRED_LOAD_CONTINUE14]] ], [ [[TMP104]], [[PRED_LOAD_IF15]] ]
+; CHECK-NEXT:    [[TMP106:%.*]] = extractelement <4 x i1> [[TMP47]], i32 2
+; CHECK-NEXT:    br i1 [[TMP106]], label [[PRED_LOAD_IF17:%.*]], label [[PRED_LOAD_CONTINUE18:%.*]]
+; CHECK:       pred.load.if17:
+; CHECK-NEXT:    [[TMP107:%.*]] = bitcast i32* [[BASE]] to i16*
+; CHECK-NEXT:    [[TMP108:%.*]] = getelementptr inbounds i16, i16* [[TMP107]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP109:%.*]] = bitcast i16* [[TMP108]] to i32*
+; CHECK-NEXT:    [[TMP110:%.*]] = load i32, i32* [[TMP109]]
+; CHECK-NEXT:    [[TMP111:%.*]] = insertelement <4 x i32> [[TMP105]], i32 [[TMP110]], i32 2
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE18]]
+; CHECK:       pred.load.continue18:
+; CHECK-NEXT:    [[TMP112:%.*]] = phi <4 x i32> [ [[TMP105]], [[PRED_LOAD_CONTINUE16]] ], [ [[TMP111]], [[PRED_LOAD_IF17]] ]
+; CHECK-NEXT:    [[TMP113:%.*]] = extractelement <4 x i1> [[TMP47]], i32 3
+; CHECK-NEXT:    br i1 [[TMP113]], label [[PRED_LOAD_IF19:%.*]], label [[PRED_LOAD_CONTINUE20:%.*]]
+; CHECK:       pred.load.if19:
+; CHECK-NEXT:    [[TMP114:%.*]] = bitcast i32* [[BASE]] to i16*
+; CHECK-NEXT:    [[TMP115:%.*]] = getelementptr inbounds i16, i16* [[TMP114]], i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP116:%.*]] = bitcast i16* [[TMP115]] to i32*
+; CHECK-NEXT:    [[TMP117:%.*]] = load i32, i32* [[TMP116]]
+; CHECK-NEXT:    [[TMP118:%.*]] = insertelement <4 x i32> [[TMP112]], i32 [[TMP117]], i32 3
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE20]]
+; CHECK:       pred.load.continue20:
+; CHECK-NEXT:    [[TMP119:%.*]] = phi <4 x i32> [ [[TMP112]], [[PRED_LOAD_CONTINUE18]] ], [ [[TMP118]], [[PRED_LOAD_IF19]] ]
+; CHECK-NEXT:    [[TMP120:%.*]] = extractelement <4 x i1> [[TMP55]], i32 0
+; CHECK-NEXT:    br i1 [[TMP120]], label [[PRED_LOAD_IF21:%.*]], label [[PRED_LOAD_CONTINUE22:%.*]]
+; CHECK:       pred.load.if21:
+; CHECK-NEXT:    [[TMP121:%.*]] = bitcast i32* [[BASE]] to i16*
+; CHECK-NEXT:    [[TMP122:%.*]] = getelementptr inbounds i16, i16* [[TMP121]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP123:%.*]] = bitcast i16* [[TMP122]] to i32*
+; CHECK-NEXT:    [[TMP124:%.*]] = load i32, i32* [[TMP123]]
+; CHECK-NEXT:    [[TMP125:%.*]] = insertelement <4 x i32> undef, i32 [[TMP124]], i32 0
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE22]]
+; CHECK:       pred.load.continue22:
+; CHECK-NEXT:    [[TMP126:%.*]] = phi <4 x i32> [ undef, [[PRED_LOAD_CONTINUE20]] ], [ [[TMP125]], [[PRED_LOAD_IF21]] ]
+; CHECK-NEXT:    [[TMP127:%.*]] = extractelement <4 x i1> [[TMP55]], i32 1
+; CHECK-NEXT:    br i1 [[TMP127]], label [[PRED_LOAD_IF23:%.*]], label [[PRED_LOAD_CONTINUE24:%.*]]
+; CHECK:       pred.load.if23:
+; CHECK-NEXT:    [[TMP128:%.*]] = bitcast i32* [[BASE]] to i16*
+; CHECK-NEXT:    [[TMP129:%.*]] = getelementptr inbounds i16, i16* [[TMP128]], i64 [[TMP9]]
+; CHECK-NEXT:    [[TMP130:%.*]] = bitcast i16* [[TMP129]] to i32*
+; CHECK-NEXT:    [[TMP131:%.*]] = load i32, i32* [[TMP130]]
+; CHECK-NEXT:    [[TMP132:%.*]] = insertelement <4 x i32> [[TMP126]], i32 [[TMP131]], i32 1
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE24]]
+; CHECK:       pred.load.continue24:
+; CHECK-NEXT:    [[TMP133:%.*]] = phi <4 x i32> [ [[TMP126]], [[PRED_LOAD_CONTINUE22]] ], [ [[TMP132]], [[PRED_LOAD_IF23]] ]
+; CHECK-NEXT:    [[TMP134:%.*]] = extractelement <4 x i1> [[TMP55]], i32 2
+; CHECK-NEXT:    br i1 [[TMP134]], label [[PRED_LOAD_IF25:%.*]], label [[PRED_LOAD_CONTINUE26:%.*]]
+; CHECK:       pred.load.if25:
+; CHECK-NEXT:    [[TMP135:%.*]] = bitcast i32* [[BASE]] to i16*
+; CHECK-NEXT:    [[TMP136:%.*]] = getelementptr inbounds i16, i16* [[TMP135]], i64 [[TMP10]]
+; CHECK-NEXT:    [[TMP137:%.*]] = bitcast i16* [[TMP136]] to i32*
+; CHECK-NEXT:    [[TMP138:%.*]] = load i32, i32* [[TMP137]]
+; CHECK-NEXT:    [[TMP139:%.*]] = insertelement <4 x i32> [[TMP133]], i32 [[TMP138]], i32 2
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE26]]
+; CHECK:       pred.load.continue26:
+; CHECK-NEXT:    [[TMP140:%.*]] = phi <4 x i32> [ [[TMP133]], [[PRED_LOAD_CONTINUE24]] ], [ [[TMP139]], [[PRED_LOAD_IF25]] ]
+; CHECK-NEXT:    [[TMP141:%.*]] = extractelement <4 x i1> [[TMP55]], i32 3
+; CHECK-NEXT:    br i1 [[TMP141]], label [[PRED_LOAD_IF27:%.*]], label [[PRED_LOAD_CONTINUE28:%.*]]
+; CHECK:       pred.load.if27:
+; CHECK-NEXT:    [[TMP142:%.*]] = bitcast i32* [[BASE]] to i16*
+; CHECK-NEXT:    [[TMP143:%.*]] = getelementptr inbounds i16, i16* [[TMP142]], i64 [[TMP11]]
+; CHECK-NEXT:    [[TMP144:%.*]] = bitcast i16* [[TMP143]] to i32*
+; CHECK-NEXT:    [[TMP145:%.*]] = load i32, i32* [[TMP144]]
+; CHECK-NEXT:    [[TMP146:%.*]] = insertelement <4 x i32> [[TMP140]], i32 [[TMP145]], i32 3
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE28]]
+; CHECK:       pred.load.continue28:
+; CHECK-NEXT:    [[TMP147:%.*]] = phi <4 x i32> [ [[TMP140]], [[PRED_LOAD_CONTINUE26]] ], [ [[TMP146]], [[PRED_LOAD_IF27]] ]
+; CHECK-NEXT:    [[TMP148:%.*]] = extractelement <4 x i1> [[TMP63]], i32 0
+; CHECK-NEXT:    br i1 [[TMP148]], label [[PRED_LOAD_IF29:%.*]], label [[PRED_LOAD_CONTINUE30:%.*]]
+; CHECK:       pred.load.if29:
+; CHECK-NEXT:    [[TMP149:%.*]] = bitcast i32* [[BASE]] to i16*
+; CHECK-NEXT:    [[TMP150:%.*]] = getelementptr inbounds i16, i16* [[TMP149]], i64 [[TMP12]]
+; CHECK-NEXT:    [[TMP151:%.*]] = bitcast i16* [[TMP150]] to i32*
+; CHECK-NEXT:    [[TMP152:%.*]] = load i32, i32* [[TMP151]]
+; CHECK-NEXT:    [[TMP153:%.*]] = insertelement <4 x i32> undef, i32 [[TMP152]], i32 0
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE30]]
+; CHECK:       pred.load.continue30:
+; CHECK-NEXT:    [[TMP154:%.*]] = phi <4 x i32> [ undef, [[PRED_LOAD_CONTINUE28]] ], [ [[TMP153]], [[PRED_LOAD_IF29]] ]
+; CHECK-NEXT:    [[TMP155:%.*]] = extractelement <4 x i1> [[TMP63]], i32 1
+; CHECK-NEXT:    br i1 [[TMP155]], label [[PRED_LOAD_IF31:%.*]], label [[PRED_LOAD_CONTINUE32:%.*]]
+; CHECK:       pred.load.if31:
+; CHECK-NEXT:    [[TMP156:%.*]] = bitcast i32* [[BASE]] to i16*
+; CHECK-NEXT:    [[TMP157:%.*]] = getelementptr inbounds i16, i16* [[TMP156]], i64 [[TMP13]]
+; CHECK-NEXT:    [[TMP158:%.*]] = bitcast i16* [[TMP157]] to i32*
+; CHECK-NEXT:    [[TMP159:%.*]] = load i32, i32* [[TMP158]]
+; CHECK-NEXT:    [[TMP160:%.*]] = insertelement <4 x i32> [[TMP154]], i32 [[TMP159]], i32 1
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE32]]
+; CHECK:       pred.load.continue32:
+; CHECK-NEXT:    [[TMP161:%.*]] = phi <4 x i32> [ [[TMP154]], [[PRED_LOAD_CONTINUE30]] ], [ [[TMP160]], [[PRED_LOAD_IF31]] ]
+; CHECK-NEXT:    [[TMP162:%.*]] = extractelement <4 x i1> [[TMP63]], i32 2
+; CHECK-NEXT:    br i1 [[TMP162]], label [[PRED_LOAD_IF33:%.*]], label [[PRED_LOAD_CONTINUE34:%.*]]
+; CHECK:       pred.load.if33:
+; CHECK-NEXT:    [[TMP163:%.*]] = bitcast i32* [[BASE]] to i16*
+; CHECK-NEXT:    [[TMP164:%.*]] = getelementptr inbounds i16, i16* [[TMP163]], i64 [[TMP14]]
+; CHECK-NEXT:    [[TMP165:%.*]] = bitcast i16* [[TMP164]] to i32*
+; CHECK-NEXT:    [[TMP166:%.*]] = load i32, i32* [[TMP165]]
+; CHECK-NEXT:    [[TMP167:%.*]] = insertelement <4 x i32> [[TMP161]], i32 [[TMP166]], i32 2
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE34]]
+; CHECK:       pred.load.continue34:
+; CHECK-NEXT:    [[TMP168:%.*]] = phi <4 x i32> [ [[TMP161]], [[PRED_LOAD_CONTINUE32]] ], [ [[TMP167]], [[PRED_LOAD_IF33]] ]
+; CHECK-NEXT:    [[TMP169:%.*]] = extractelement <4 x i1> [[TMP63]], i32 3
+; CHECK-NEXT:    br i1 [[TMP169]], label [[PRED_LOAD_IF35:%.*]], label [[PRED_LOAD_CONTINUE36]]
+; CHECK:       pred.load.if35:
+; CHECK-NEXT:    [[TMP170:%.*]] = bitcast i32* [[BASE]] to i16*
+; CHECK-NEXT:    [[TMP171:%.*]] = getelementptr inbounds i16, i16* [[TMP170]], i64 [[TMP15]]
+; CHECK-NEXT:    [[TMP172:%.*]] = bitcast i16* [[TMP171]] to i32*
+; CHECK-NEXT:    [[TMP173:%.*]] = load i32, i32* [[TMP172]]
+; CHECK-NEXT:    [[TMP174:%.*]] = insertelement <4 x i32> [[TMP168]], i32 [[TMP173]], i32 3
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE36]]
+; CHECK:       pred.load.continue36:
+; CHECK-NEXT:    [[TMP175:%.*]] = phi <4 x i32> [ [[TMP168]], [[PRED_LOAD_CONTINUE34]] ], [ [[TMP174]], [[PRED_LOAD_IF35]] ]
+; CHECK-NEXT:    [[TMP176:%.*]] = xor <4 x i1> [[TMP39]], <i1 true, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    [[TMP177:%.*]] = xor <4 x i1> [[TMP47]], <i1 true, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    [[TMP178:%.*]] = xor <4 x i1> [[TMP55]], <i1 true, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    [[TMP179:%.*]] = xor <4 x i1> [[TMP63]], <i1 true, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP39]], <4 x i32> [[TMP91]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[PREDPHI37:%.*]] = select <4 x i1> [[TMP47]], <4 x i32> [[TMP119]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[PREDPHI38:%.*]] = select <4 x i1> [[TMP55]], <4 x i32> [[TMP147]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[PREDPHI39:%.*]] = select <4 x i1> [[TMP63]], <4 x i32> [[TMP175]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP180]] = add <4 x i32> [[VEC_PHI]], [[PREDPHI]]
+; CHECK-NEXT:    [[TMP181]] = add <4 x i32> [[VEC_PHI4]], [[PREDPHI37]]
+; CHECK-NEXT:    [[TMP182]] = add <4 x i32> [[VEC_PHI5]], [[PREDPHI38]]
+; CHECK-NEXT:    [[TMP183]] = add <4 x i32> [[VEC_PHI6]], [[PREDPHI39]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 16
+; CHECK-NEXT:    [[TMP184:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4096
+; CHECK-NEXT:    br i1 [[TMP184]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !8
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <4 x i32> [[TMP181]], [[TMP180]]
+; CHECK-NEXT:    [[BIN_RDX40:%.*]] = add <4 x i32> [[TMP182]], [[BIN_RDX]]
+; CHECK-NEXT:    [[BIN_RDX41:%.*]] = add <4 x i32> [[TMP183]], [[BIN_RDX40]]
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[BIN_RDX41]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX42:%.*]] = add <4 x i32> [[BIN_RDX41]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF43:%.*]] = shufflevector <4 x i32> [[BIN_RDX42]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX44:%.*]] = add <4 x i32> [[BIN_RDX42]], [[RDX_SHUF43]]
+; CHECK-NEXT:    [[TMP185:%.*]] = extractelement <4 x i32> [[BIN_RDX44]], i32 0
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 4096, 4096
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[LOOP_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 4096, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[TMP185]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
+; CHECK-NEXT:    [[ACCUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ACCUM_NEXT:%.*]], [[LATCH]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT:    [[TEST_ADDR:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[IV]]
+; CHECK-NEXT:    [[EARLYCND:%.*]] = load i1, i1* [[TEST_ADDR]]
+; CHECK-NEXT:    br i1 [[EARLYCND]], label [[PRED:%.*]], label [[LATCH]]
+; CHECK:       pred:
+; CHECK-NEXT:    [[BASE_I16P:%.*]] = bitcast i32* [[BASE]] to i16*
+; CHECK-NEXT:    [[ADDR_I16P:%.*]] = getelementptr inbounds i16, i16* [[BASE_I16P]], i64 [[IV]]
+; CHECK-NEXT:    [[ADDR:%.*]] = bitcast i16* [[ADDR_I16P]] to i32*
+; CHECK-NEXT:    [[VAL:%.*]] = load i32, i32* [[ADDR]]
+; CHECK-NEXT:    br label [[LATCH]]
+; CHECK:       latch:
+; CHECK-NEXT:    [[VAL_PHI:%.*]] = phi i32 [ 0, [[LOOP]] ], [ [[VAL]], [[PRED]] ]
+; CHECK-NEXT:    [[ACCUM_NEXT]] = add i32 [[ACCUM]], [[VAL_PHI]]
+; CHECK-NEXT:    [[EXIT:%.*]] = icmp ugt i64 [[IV]], 4094
+; CHECK-NEXT:    br i1 [[EXIT]], label [[LOOP_EXIT]], label [[LOOP]], !llvm.loop !9
+; CHECK:       loop_exit:
+; CHECK-NEXT:    [[ACCUM_NEXT_LCSSA:%.*]] = phi i32 [ [[ACCUM_NEXT]], [[LATCH]] ], [ [[TMP185]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    ret i32 [[ACCUM_NEXT_LCSSA]]
+;
+entry:
+  %alloca = alloca [4096 x i32]
+  %base = bitcast [4096 x i32]* %alloca to i32*
+  call void @init(i32* %base)
+  br label %loop
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
+  %accum = phi i32 [ 0, %entry ], [ %accum.next, %latch ]
+  %iv.next = add i64 %iv, 1
+  %test_addr = getelementptr inbounds i1, i1* %test_base, i64 %iv
+  %earlycnd = load i1, i1* %test_addr
+  br i1 %earlycnd, label %pred, label %latch
+pred:
+  %base.i16p = bitcast i32* %base to i16*
+  %addr.i16p = getelementptr inbounds i16, i16* %base.i16p, i64 %iv
+  %addr = bitcast i16* %addr.i16p to i32*
+  %val = load i32, i32* %addr
+  br label %latch
+latch:
+  %val.phi = phi i32 [0, %loop], [%val, %pred]
+  %accum.next = add i32 %accum, %val.phi
+  %exit = icmp ugt i64 %iv, 4094
+  br i1 %exit, label %loop_exit, label %loop
+
+loop_exit:
+  ret i32 %accum.next
+}
+
+define i32 @test_max_trip_count(i64 %len, i1* %test_base, i64 %n) {
+; CHECK-LABEL: @test_max_trip_count(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ALLOCA:%.*]] = alloca [4096 x i32]
+; CHECK-NEXT:    [[BASE:%.*]] = bitcast [4096 x i32]* [[ALLOCA]] to i32*
+; CHECK-NEXT:    call void @init(i32* [[BASE]])
+; CHECK-NEXT:    [[MIN_CMP:%.*]] = icmp ult i64 4096, [[N:%.*]]
+; CHECK-NEXT:    [[MIN_N:%.*]] = select i1 [[MIN_CMP]], i64 4096, i64 [[N]]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[MIN_N]], 2
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 16
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 16
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP81:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI4:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP82:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI5:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP83:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI6:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP84:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> undef, i64 [[INDEX]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], <i64 0, i64 1, i64 2, i64 3>
+; CHECK-NEXT:    [[INDUCTION1:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], <i64 4, i64 5, i64 6, i64 7>
+; CHECK-NEXT:    [[INDUCTION2:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], <i64 8, i64 9, i64 10, i64 11>
+; CHECK-NEXT:    [[INDUCTION3:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], <i64 12, i64 13, i64 14, i64 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[INDEX]], 3
+; CHECK-NEXT:    [[TMP5:%.*]] = add i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[INDEX]], 5
+; CHECK-NEXT:    [[TMP7:%.*]] = add i64 [[INDEX]], 6
+; CHECK-NEXT:    [[TMP8:%.*]] = add i64 [[INDEX]], 7
+; CHECK-NEXT:    [[TMP9:%.*]] = add i64 [[INDEX]], 8
+; CHECK-NEXT:    [[TMP10:%.*]] = add i64 [[INDEX]], 9
+; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[INDEX]], 10
+; CHECK-NEXT:    [[TMP12:%.*]] = add i64 [[INDEX]], 11
+; CHECK-NEXT:    [[TMP13:%.*]] = add i64 [[INDEX]], 12
+; CHECK-NEXT:    [[TMP14:%.*]] = add i64 [[INDEX]], 13
+; CHECK-NEXT:    [[TMP15:%.*]] = add i64 [[INDEX]], 14
+; CHECK-NEXT:    [[TMP16:%.*]] = add i64 [[INDEX]], 15
+; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE:%.*]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP3]]
+; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP25:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP9]]
+; CHECK-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP10]]
+; CHECK-NEXT:    [[TMP27:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP11]]
+; CHECK-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP12]]
+; CHECK-NEXT:    [[TMP29:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP13]]
+; CHECK-NEXT:    [[TMP30:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP14]]
+; CHECK-NEXT:    [[TMP31:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP15]]
+; CHECK-NEXT:    [[TMP32:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP16]]
+; CHECK-NEXT:    [[TMP33:%.*]] = load i1, i1* [[TMP17]]
+; CHECK-NEXT:    [[TMP34:%.*]] = load i1, i1* [[TMP18]]
+; CHECK-NEXT:    [[TMP35:%.*]] = load i1, i1* [[TMP19]]
+; CHECK-NEXT:    [[TMP36:%.*]] = load i1, i1* [[TMP20]]
+; CHECK-NEXT:    [[TMP37:%.*]] = insertelement <4 x i1> undef, i1 [[TMP33]], i32 0
+; CHECK-NEXT:    [[TMP38:%.*]] = insertelement <4 x i1> [[TMP37]], i1 [[TMP34]], i32 1
+; CHECK-NEXT:    [[TMP39:%.*]] = insertelement <4 x i1> [[TMP38]], i1 [[TMP35]], i32 2
+; CHECK-NEXT:    [[TMP40:%.*]] = insertelement <4 x i1> [[TMP39]], i1 [[TMP36]], i32 3
+; CHECK-NEXT:    [[TMP41:%.*]] = load i1, i1* [[TMP21]]
+; CHECK-NEXT:    [[TMP42:%.*]] = load i1, i1* [[TMP22]]
+; CHECK-NEXT:    [[TMP43:%.*]] = load i1, i1* [[TMP23]]
+; CHECK-NEXT:    [[TMP44:%.*]] = load i1, i1* [[TMP24]]
+; CHECK-NEXT:    [[TMP45:%.*]] = insertelement <4 x i1> undef, i1 [[TMP41]], i32 0
+; CHECK-NEXT:    [[TMP46:%.*]] = insertelement <4 x i1> [[TMP45]], i1 [[TMP42]], i32 1
+; CHECK-NEXT:    [[TMP47:%.*]] = insertelement <4 x i1> [[TMP46]], i1 [[TMP43]], i32 2
+; CHECK-NEXT:    [[TMP48:%.*]] = insertelement <4 x i1> [[TMP47]], i1 [[TMP44]], i32 3
+; CHECK-NEXT:    [[TMP49:%.*]] = load i1, i1* [[TMP25]]
+; CHECK-NEXT:    [[TMP50:%.*]] = load i1, i1* [[TMP26]]
+; CHECK-NEXT:    [[TMP51:%.*]] = load i1, i1* [[TMP27]]
+; CHECK-NEXT:    [[TMP52:%.*]] = load i1, i1* [[TMP28]]
+; CHECK-NEXT:    [[TMP53:%.*]] = insertelement <4 x i1> undef, i1 [[TMP49]], i32 0
+; CHECK-NEXT:    [[TMP54:%.*]] = insertelement <4 x i1> [[TMP53]], i1 [[TMP50]], i32 1
+; CHECK-NEXT:    [[TMP55:%.*]] = insertelement <4 x i1> [[TMP54]], i1 [[TMP51]], i32 2
+; CHECK-NEXT:    [[TMP56:%.*]] = insertelement <4 x i1> [[TMP55]], i1 [[TMP52]], i32 3
+; CHECK-NEXT:    [[TMP57:%.*]] = load i1, i1* [[TMP29]]
+; CHECK-NEXT:    [[TMP58:%.*]] = load i1, i1* [[TMP30]]
+; CHECK-NEXT:    [[TMP59:%.*]] = load i1, i1* [[TMP31]]
+; CHECK-NEXT:    [[TMP60:%.*]] = load i1, i1* [[TMP32]]
+; CHECK-NEXT:    [[TMP61:%.*]] = insertelement <4 x i1> undef, i1 [[TMP57]], i32 0
+; CHECK-NEXT:    [[TMP62:%.*]] = insertelement <4 x i1> [[TMP61]], i1 [[TMP58]], i32 1
+; CHECK-NEXT:    [[TMP63:%.*]] = insertelement <4 x i1> [[TMP62]], i1 [[TMP59]], i32 2
+; CHECK-NEXT:    [[TMP64:%.*]] = insertelement <4 x i1> [[TMP63]], i1 [[TMP60]], i32 3
+; CHECK-NEXT:    [[TMP65:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP66:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP67:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[TMP9]]
+; CHECK-NEXT:    [[TMP68:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[TMP13]]
+; CHECK-NEXT:    [[TMP69:%.*]] = getelementptr inbounds i32, i32* [[TMP65]], i32 0
+; CHECK-NEXT:    [[TMP70:%.*]] = bitcast i32* [[TMP69]] to <4 x i32>*
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* [[TMP70]], i32 4, <4 x i1> [[TMP40]], <4 x i32> undef)
+; CHECK-NEXT:    [[TMP71:%.*]] = getelementptr inbounds i32, i32* [[TMP65]], i32 4
+; CHECK-NEXT:    [[TMP72:%.*]] = bitcast i32* [[TMP71]] to <4 x i32>*
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD7:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* [[TMP72]], i32 4, <4 x i1> [[TMP48]], <4 x i32> undef)
+; CHECK-NEXT:    [[TMP73:%.*]] = getelementptr inbounds i32, i32* [[TMP65]], i32 8
+; CHECK-NEXT:    [[TMP74:%.*]] = bitcast i32* [[TMP73]] to <4 x i32>*
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD8:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* [[TMP74]], i32 4, <4 x i1> [[TMP56]], <4 x i32> undef)
+; CHECK-NEXT:    [[TMP75:%.*]] = getelementptr inbounds i32, i32* [[TMP65]], i32 12
+; CHECK-NEXT:    [[TMP76:%.*]] = bitcast i32* [[TMP75]] to <4 x i32>*
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD9:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* [[TMP76]], i32 4, <4 x i1> [[TMP64]], <4 x i32> undef)
+; CHECK-NEXT:    [[TMP77:%.*]] = xor <4 x i1> [[TMP40]], <i1 true, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    [[TMP78:%.*]] = xor <4 x i1> [[TMP48]], <i1 true, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    [[TMP79:%.*]] = xor <4 x i1> [[TMP56]], <i1 true, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    [[TMP80:%.*]] = xor <4 x i1> [[TMP64]], <i1 true, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP40]], <4 x i32> [[WIDE_MASKED_LOAD]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[PREDPHI10:%.*]] = select <4 x i1> [[TMP48]], <4 x i32> [[WIDE_MASKED_LOAD7]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[PREDPHI11:%.*]] = select <4 x i1> [[TMP56]], <4 x i32> [[WIDE_MASKED_LOAD8]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[PREDPHI12:%.*]] = select <4 x i1> [[TMP64]], <4 x i32> [[WIDE_MASKED_LOAD9]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP81]] = add <4 x i32> [[VEC_PHI]], [[PREDPHI]]
+; CHECK-NEXT:    [[TMP82]] = add <4 x i32> [[VEC_PHI4]], [[PREDPHI10]]
+; CHECK-NEXT:    [[TMP83]] = add <4 x i32> [[VEC_PHI5]], [[PREDPHI11]]
+; CHECK-NEXT:    [[TMP84]] = add <4 x i32> [[VEC_PHI6]], [[PREDPHI12]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 16
+; CHECK-NEXT:    [[TMP85:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP85]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !10
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <4 x i32> [[TMP82]], [[TMP81]]
+; CHECK-NEXT:    [[BIN_RDX13:%.*]] = add <4 x i32> [[TMP83]], [[BIN_RDX]]
+; CHECK-NEXT:    [[BIN_RDX14:%.*]] = add <4 x i32> [[TMP84]], [[BIN_RDX13]]
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[BIN_RDX14]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX15:%.*]] = add <4 x i32> [[BIN_RDX14]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF16:%.*]] = shufflevector <4 x i32> [[BIN_RDX15]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX17:%.*]] = add <4 x i32> [[BIN_RDX15]], [[RDX_SHUF16]]
+; CHECK-NEXT:    [[TMP86:%.*]] = extractelement <4 x i32> [[BIN_RDX17]], i32 0
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[LOOP_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[TMP86]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
+; CHECK-NEXT:    [[ACCUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ACCUM_NEXT:%.*]], [[LATCH]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT:    [[TEST_ADDR:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[IV]]
+; CHECK-NEXT:    [[EARLYCND:%.*]] = load i1, i1* [[TEST_ADDR]]
+; CHECK-NEXT:    br i1 [[EARLYCND]], label [[PRED:%.*]], label [[LATCH]]
+; CHECK:       pred:
+; CHECK-NEXT:    [[ADDR:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[IV]]
+; CHECK-NEXT:    [[VAL:%.*]] = load i32, i32* [[ADDR]]
+; CHECK-NEXT:    br label [[LATCH]]
+; CHECK:       latch:
+; CHECK-NEXT:    [[VAL_PHI:%.*]] = phi i32 [ 0, [[LOOP]] ], [ [[VAL]], [[PRED]] ]
+; CHECK-NEXT:    [[ACCUM_NEXT]] = add i32 [[ACCUM]], [[VAL_PHI]]
+; CHECK-NEXT:    [[EXIT:%.*]] = icmp ugt i64 [[IV]], [[MIN_N]]
+; CHECK-NEXT:    br i1 [[EXIT]], label [[LOOP_EXIT]], label [[LOOP]], !llvm.loop !11
+; CHECK:       loop_exit:
+; CHECK-NEXT:    [[ACCUM_NEXT_LCSSA:%.*]] = phi i32 [ [[ACCUM_NEXT]], [[LATCH]] ], [ [[TMP86]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    ret i32 [[ACCUM_NEXT_LCSSA]]
+;
+entry:
+  %alloca = alloca [4096 x i32]
+  %base = bitcast [4096 x i32]* %alloca to i32*
+  call void @init(i32* %base)
+  %min.cmp = icmp ult i64 4096, %n
+  %min.n = select i1 %min.cmp, i64 4096, i64 %n
+  br label %loop
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
+  %accum = phi i32 [ 0, %entry ], [ %accum.next, %latch ]
+  %iv.next = add i64 %iv, 1
+  %test_addr = getelementptr inbounds i1, i1* %test_base, i64 %iv
+  %earlycnd = load i1, i1* %test_addr
+  br i1 %earlycnd, label %pred, label %latch
+pred:
+  %addr = getelementptr inbounds i32, i32* %base, i64 %iv
+  %val = load i32, i32* %addr
+  br label %latch
+latch:
+  %val.phi = phi i32 [0, %loop], [%val, %pred]
+  %accum.next = add i32 %accum, %val.phi
+  %exit = icmp ugt i64 %iv, %min.n
+  br i1 %exit, label %loop_exit, label %loop
+
+loop_exit:
+  ret i32 %accum.next
+}
+
+
+
+
+define i32 @test_non_zero_start(i64 %len, i1* %test_base) {
+; CHECK-LABEL: @test_non_zero_start(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ALLOCA:%.*]] = alloca [4096 x i32]
+; CHECK-NEXT:    [[BASE:%.*]] = bitcast [4096 x i32]* [[ALLOCA]] to i32*
+; CHECK-NEXT:    call void @init(i32* [[BASE]])
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP80:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI4:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP81:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI5:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP82:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI6:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP83:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 1024, [[INDEX]]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> undef, i64 [[OFFSET_IDX]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], <i64 0, i64 1, i64 2, i64 3>
+; CHECK-NEXT:    [[INDUCTION1:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], <i64 4, i64 5, i64 6, i64 7>
+; CHECK-NEXT:    [[INDUCTION2:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], <i64 8, i64 9, i64 10, i64 11>
+; CHECK-NEXT:    [[INDUCTION3:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], <i64 12, i64 13, i64 14, i64 15>
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[OFFSET_IDX]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[OFFSET_IDX]], 3
+; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[OFFSET_IDX]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = add i64 [[OFFSET_IDX]], 5
+; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[OFFSET_IDX]], 6
+; CHECK-NEXT:    [[TMP7:%.*]] = add i64 [[OFFSET_IDX]], 7
+; CHECK-NEXT:    [[TMP8:%.*]] = add i64 [[OFFSET_IDX]], 8
+; CHECK-NEXT:    [[TMP9:%.*]] = add i64 [[OFFSET_IDX]], 9
+; CHECK-NEXT:    [[TMP10:%.*]] = add i64 [[OFFSET_IDX]], 10
+; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[OFFSET_IDX]], 11
+; CHECK-NEXT:    [[TMP12:%.*]] = add i64 [[OFFSET_IDX]], 12
+; CHECK-NEXT:    [[TMP13:%.*]] = add i64 [[OFFSET_IDX]], 13
+; CHECK-NEXT:    [[TMP14:%.*]] = add i64 [[OFFSET_IDX]], 14
+; CHECK-NEXT:    [[TMP15:%.*]] = add i64 [[OFFSET_IDX]], 15
+; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE:%.*]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP3]]
+; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP25:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP9]]
+; CHECK-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP10]]
+; CHECK-NEXT:    [[TMP27:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP11]]
+; CHECK-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP12]]
+; CHECK-NEXT:    [[TMP29:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP13]]
+; CHECK-NEXT:    [[TMP30:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP14]]
+; CHECK-NEXT:    [[TMP31:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP15]]
+; CHECK-NEXT:    [[TMP32:%.*]] = load i1, i1* [[TMP16]]
+; CHECK-NEXT:    [[TMP33:%.*]] = load i1, i1* [[TMP17]]
+; CHECK-NEXT:    [[TMP34:%.*]] = load i1, i1* [[TMP18]]
+; CHECK-NEXT:    [[TMP35:%.*]] = load i1, i1* [[TMP19]]
+; CHECK-NEXT:    [[TMP36:%.*]] = insertelement <4 x i1> undef, i1 [[TMP32]], i32 0
+; CHECK-NEXT:    [[TMP37:%.*]] = insertelement <4 x i1> [[TMP36]], i1 [[TMP33]], i32 1
+; CHECK-NEXT:    [[TMP38:%.*]] = insertelement <4 x i1> [[TMP37]], i1 [[TMP34]], i32 2
+; CHECK-NEXT:    [[TMP39:%.*]] = insertelement <4 x i1> [[TMP38]], i1 [[TMP35]], i32 3
+; CHECK-NEXT:    [[TMP40:%.*]] = load i1, i1* [[TMP20]]
+; CHECK-NEXT:    [[TMP41:%.*]] = load i1, i1* [[TMP21]]
+; CHECK-NEXT:    [[TMP42:%.*]] = load i1, i1* [[TMP22]]
+; CHECK-NEXT:    [[TMP43:%.*]] = load i1, i1* [[TMP23]]
+; CHECK-NEXT:    [[TMP44:%.*]] = insertelement <4 x i1> undef, i1 [[TMP40]], i32 0
+; CHECK-NEXT:    [[TMP45:%.*]] = insertelement <4 x i1> [[TMP44]], i1 [[TMP41]], i32 1
+; CHECK-NEXT:    [[TMP46:%.*]] = insertelement <4 x i1> [[TMP45]], i1 [[TMP42]], i32 2
+; CHECK-NEXT:    [[TMP47:%.*]] = insertelement <4 x i1> [[TMP46]], i1 [[TMP43]], i32 3
+; CHECK-NEXT:    [[TMP48:%.*]] = load i1, i1* [[TMP24]]
+; CHECK-NEXT:    [[TMP49:%.*]] = load i1, i1* [[TMP25]]
+; CHECK-NEXT:    [[TMP50:%.*]] = load i1, i1* [[TMP26]]
+; CHECK-NEXT:    [[TMP51:%.*]] = load i1, i1* [[TMP27]]
+; CHECK-NEXT:    [[TMP52:%.*]] = insertelement <4 x i1> undef, i1 [[TMP48]], i32 0
+; CHECK-NEXT:    [[TMP53:%.*]] = insertelement <4 x i1> [[TMP52]], i1 [[TMP49]], i32 1
+; CHECK-NEXT:    [[TMP54:%.*]] = insertelement <4 x i1> [[TMP53]], i1 [[TMP50]], i32 2
+; CHECK-NEXT:    [[TMP55:%.*]] = insertelement <4 x i1> [[TMP54]], i1 [[TMP51]], i32 3
+; CHECK-NEXT:    [[TMP56:%.*]] = load i1, i1* [[TMP28]]
+; CHECK-NEXT:    [[TMP57:%.*]] = load i1, i1* [[TMP29]]
+; CHECK-NEXT:    [[TMP58:%.*]] = load i1, i1* [[TMP30]]
+; CHECK-NEXT:    [[TMP59:%.*]] = load i1, i1* [[TMP31]]
+; CHECK-NEXT:    [[TMP60:%.*]] = insertelement <4 x i1> undef, i1 [[TMP56]], i32 0
+; CHECK-NEXT:    [[TMP61:%.*]] = insertelement <4 x i1> [[TMP60]], i1 [[TMP57]], i32 1
+; CHECK-NEXT:    [[TMP62:%.*]] = insertelement <4 x i1> [[TMP61]], i1 [[TMP58]], i32 2
+; CHECK-NEXT:    [[TMP63:%.*]] = insertelement <4 x i1> [[TMP62]], i1 [[TMP59]], i32 3
+; CHECK-NEXT:    [[TMP64:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP65:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP66:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP67:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[TMP12]]
+; CHECK-NEXT:    [[TMP68:%.*]] = getelementptr inbounds i32, i32* [[TMP64]], i32 0
+; CHECK-NEXT:    [[TMP69:%.*]] = bitcast i32* [[TMP68]] to <4 x i32>*
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* [[TMP69]], i32 4, <4 x i1> [[TMP39]], <4 x i32> undef)
+; CHECK-NEXT:    [[TMP70:%.*]] = getelementptr inbounds i32, i32* [[TMP64]], i32 4
+; CHECK-NEXT:    [[TMP71:%.*]] = bitcast i32* [[TMP70]] to <4 x i32>*
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD7:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* [[TMP71]], i32 4, <4 x i1> [[TMP47]], <4 x i32> undef)
+; CHECK-NEXT:    [[TMP72:%.*]] = getelementptr inbounds i32, i32* [[TMP64]], i32 8
+; CHECK-NEXT:    [[TMP73:%.*]] = bitcast i32* [[TMP72]] to <4 x i32>*
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD8:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* [[TMP73]], i32 4, <4 x i1> [[TMP55]], <4 x i32> undef)
+; CHECK-NEXT:    [[TMP74:%.*]] = getelementptr inbounds i32, i32* [[TMP64]], i32 12
+; CHECK-NEXT:    [[TMP75:%.*]] = bitcast i32* [[TMP74]] to <4 x i32>*
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD9:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* [[TMP75]], i32 4, <4 x i1> [[TMP63]], <4 x i32> undef)
+; CHECK-NEXT:    [[TMP76:%.*]] = xor <4 x i1> [[TMP39]], <i1 true, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    [[TMP77:%.*]] = xor <4 x i1> [[TMP47]], <i1 true, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    [[TMP78:%.*]] = xor <4 x i1> [[TMP55]], <i1 true, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    [[TMP79:%.*]] = xor <4 x i1> [[TMP63]], <i1 true, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP39]], <4 x i32> [[WIDE_MASKED_LOAD]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[PREDPHI10:%.*]] = select <4 x i1> [[TMP47]], <4 x i32> [[WIDE_MASKED_LOAD7]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[PREDPHI11:%.*]] = select <4 x i1> [[TMP55]], <4 x i32> [[WIDE_MASKED_LOAD8]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[PREDPHI12:%.*]] = select <4 x i1> [[TMP63]], <4 x i32> [[WIDE_MASKED_LOAD9]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP80]] = add <4 x i32> [[VEC_PHI]], [[PREDPHI]]
+; CHECK-NEXT:    [[TMP81]] = add <4 x i32> [[VEC_PHI4]], [[PREDPHI10]]
+; CHECK-NEXT:    [[TMP82]] = add <4 x i32> [[VEC_PHI5]], [[PREDPHI11]]
+; CHECK-NEXT:    [[TMP83]] = add <4 x i32> [[VEC_PHI6]], [[PREDPHI12]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 16
+; CHECK-NEXT:    [[TMP84:%.*]] = icmp eq i64 [[INDEX_NEXT]], 3072
+; CHECK-NEXT:    br i1 [[TMP84]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !12
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <4 x i32> [[TMP81]], [[TMP80]]
+; CHECK-NEXT:    [[BIN_RDX13:%.*]] = add <4 x i32> [[TMP82]], [[BIN_RDX]]
+; CHECK-NEXT:    [[BIN_RDX14:%.*]] = add <4 x i32> [[TMP83]], [[BIN_RDX13]]
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[BIN_RDX14]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX15:%.*]] = add <4 x i32> [[BIN_RDX14]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF16:%.*]] = shufflevector <4 x i32> [[BIN_RDX15]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX17:%.*]] = add <4 x i32> [[BIN_RDX15]], [[RDX_SHUF16]]
+; CHECK-NEXT:    [[TMP85:%.*]] = extractelement <4 x i32> [[BIN_RDX17]], i32 0
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 3072, 3072
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[LOOP_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 4096, [[MIDDLE_BLOCK]] ], [ 1024, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[TMP85]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
+; CHECK-NEXT:    [[ACCUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ACCUM_NEXT:%.*]], [[LATCH]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT:    [[TEST_ADDR:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[IV]]
+; CHECK-NEXT:    [[EARLYCND:%.*]] = load i1, i1* [[TEST_ADDR]]
+; CHECK-NEXT:    br i1 [[EARLYCND]], label [[PRED:%.*]], label [[LATCH]]
+; CHECK:       pred:
+; CHECK-NEXT:    [[ADDR:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[IV]]
+; CHECK-NEXT:    [[VAL:%.*]] = load i32, i32* [[ADDR]]
+; CHECK-NEXT:    br label [[LATCH]]
+; CHECK:       latch:
+; CHECK-NEXT:    [[VAL_PHI:%.*]] = phi i32 [ 0, [[LOOP]] ], [ [[VAL]], [[PRED]] ]
+; CHECK-NEXT:    [[ACCUM_NEXT]] = add i32 [[ACCUM]], [[VAL_PHI]]
+; CHECK-NEXT:    [[EXIT:%.*]] = icmp ugt i64 [[IV]], 4094
+; CHECK-NEXT:    br i1 [[EXIT]], label [[LOOP_EXIT]], label [[LOOP]], !llvm.loop !13
+; CHECK:       loop_exit:
+; CHECK-NEXT:    [[ACCUM_NEXT_LCSSA:%.*]] = phi i32 [ [[ACCUM_NEXT]], [[LATCH]] ], [ [[TMP85]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    ret i32 [[ACCUM_NEXT_LCSSA]]
+;
+entry:
+  %alloca = alloca [4096 x i32]
+  %base = bitcast [4096 x i32]* %alloca to i32*
+  call void @init(i32* %base)
+  br label %loop
+loop:
+  %iv = phi i64 [ 1024, %entry ], [ %iv.next, %latch ]
+  %accum = phi i32 [ 0, %entry ], [ %accum.next, %latch ]
+  %iv.next = add i64 %iv, 1
+  %test_addr = getelementptr inbounds i1, i1* %test_base, i64 %iv
+  %earlycnd = load i1, i1* %test_addr
+  br i1 %earlycnd, label %pred, label %latch
+pred:
+  %addr = getelementptr inbounds i32, i32* %base, i64 %iv
+  %val = load i32, i32* %addr
+  br label %latch
+latch:
+  %val.phi = phi i32 [0, %loop], [%val, %pred]
+  %accum.next = add i32 %accum, %val.phi
+  %exit = icmp ugt i64 %iv, 4094
+  br i1 %exit, label %loop_exit, label %loop
+
+loop_exit:
+  ret i32 %accum.next
+}
+
+define i32 @neg_out_of_bounds_start(i64 %len, i1* %test_base) {
+; CHECK-LABEL: @neg_out_of_bounds_start(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ALLOCA:%.*]] = alloca [4096 x i32]
+; CHECK-NEXT:    [[BASE:%.*]] = bitcast [4096 x i32]* [[ALLOCA]] to i32*
+; CHECK-NEXT:    call void @init(i32* [[BASE]])
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ -10, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
+; CHECK-NEXT:    [[ACCUM:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[ACCUM_NEXT:%.*]], [[LATCH]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT:    [[TEST_ADDR:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE:%.*]], i64 [[IV]]
+; CHECK-NEXT:    [[EARLYCND:%.*]] = load i1, i1* [[TEST_ADDR]]
+; CHECK-NEXT:    br i1 [[EARLYCND]], label [[PRED:%.*]], label [[LATCH]]
+; CHECK:       pred:
+; CHECK-NEXT:    [[ADDR:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[IV]]
+; CHECK-NEXT:    [[VAL:%.*]] = load i32, i32* [[ADDR]]
+; CHECK-NEXT:    br label [[LATCH]]
+; CHECK:       latch:
+; CHECK-NEXT:    [[VAL_PHI:%.*]] = phi i32 [ 0, [[LOOP]] ], [ [[VAL]], [[PRED]] ]
+; CHECK-NEXT:    [[ACCUM_NEXT]] = add i32 [[ACCUM]], [[VAL_PHI]]
+; CHECK-NEXT:    [[EXIT:%.*]] = icmp ugt i64 [[IV]], 4094
+; CHECK-NEXT:    br i1 [[EXIT]], label [[LOOP_EXIT:%.*]], label [[LOOP]]
+; CHECK:       loop_exit:
+; CHECK-NEXT:    [[ACCUM_NEXT_LCSSA:%.*]] = phi i32 [ [[ACCUM_NEXT]], [[LATCH]] ]
+; CHECK-NEXT:    ret i32 [[ACCUM_NEXT_LCSSA]]
+;
+entry:
+  %alloca = alloca [4096 x i32]
+  %base = bitcast [4096 x i32]* %alloca to i32*
+  call void @init(i32* %base)
+  br label %loop
+loop:
+  %iv = phi i64 [ -10, %entry ], [ %iv.next, %latch ]
+  %accum = phi i32 [ 0, %entry ], [ %accum.next, %latch ]
+  %iv.next = add i64 %iv, 1
+  %test_addr = getelementptr inbounds i1, i1* %test_base, i64 %iv
+  %earlycnd = load i1, i1* %test_addr
+  br i1 %earlycnd, label %pred, label %latch
+pred:
+  %addr = getelementptr inbounds i32, i32* %base, i64 %iv
+  %val = load i32, i32* %addr
+  br label %latch
+latch:
+  %val.phi = phi i32 [0, %loop], [%val, %pred]
+  %accum.next = add i32 %accum, %val.phi
+  %exit = icmp ugt i64 %iv, 4094
+  br i1 %exit, label %loop_exit, label %loop
+
+loop_exit:
+  ret i32 %accum.next
+}
+
+
+;; TODO: handle non-unit strides
+define i32 @test_non_unit_stride(i64 %len, i1* %test_base) {
+; CHECK-LABEL: @test_non_unit_stride(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ALLOCA:%.*]] = alloca [4096 x i32]
+; CHECK-NEXT:    [[BASE:%.*]] = bitcast [4096 x i32]* [[ALLOCA]] to i32*
+; CHECK-NEXT:    call void @init(i32* [[BASE]])
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE36:%.*]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP148:%.*]], [[PRED_LOAD_CONTINUE36]] ]
+; CHECK-NEXT:    [[VEC_PHI4:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP149:%.*]], [[PRED_LOAD_CONTINUE36]] ]
+; CHECK-NEXT:    [[VEC_PHI5:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP150:%.*]], [[PRED_LOAD_CONTINUE36]] ]
+; CHECK-NEXT:    [[VEC_PHI6:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP151:%.*]], [[PRED_LOAD_CONTINUE36]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 2
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> undef, i64 [[OFFSET_IDX]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], <i64 0, i64 2, i64 4, i64 6>
+; CHECK-NEXT:    [[INDUCTION1:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], <i64 8, i64 10, i64 12, i64 14>
+; CHECK-NEXT:    [[INDUCTION2:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], <i64 16, i64 18, i64 20, i64 22>
+; CHECK-NEXT:    [[INDUCTION3:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], <i64 24, i64 26, i64 28, i64 30>
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[OFFSET_IDX]], 4
+; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[OFFSET_IDX]], 6
+; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[OFFSET_IDX]], 8
+; CHECK-NEXT:    [[TMP5:%.*]] = add i64 [[OFFSET_IDX]], 10
+; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[OFFSET_IDX]], 12
+; CHECK-NEXT:    [[TMP7:%.*]] = add i64 [[OFFSET_IDX]], 14
+; CHECK-NEXT:    [[TMP8:%.*]] = add i64 [[OFFSET_IDX]], 16
+; CHECK-NEXT:    [[TMP9:%.*]] = add i64 [[OFFSET_IDX]], 18
+; CHECK-NEXT:    [[TMP10:%.*]] = add i64 [[OFFSET_IDX]], 20
+; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[OFFSET_IDX]], 22
+; CHECK-NEXT:    [[TMP12:%.*]] = add i64 [[OFFSET_IDX]], 24
+; CHECK-NEXT:    [[TMP13:%.*]] = add i64 [[OFFSET_IDX]], 26
+; CHECK-NEXT:    [[TMP14:%.*]] = add i64 [[OFFSET_IDX]], 28
+; CHECK-NEXT:    [[TMP15:%.*]] = add i64 [[OFFSET_IDX]], 30
+; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE:%.*]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP3]]
+; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP25:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP9]]
+; CHECK-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP10]]
+; CHECK-NEXT:    [[TMP27:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP11]]
+; CHECK-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP12]]
+; CHECK-NEXT:    [[TMP29:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP13]]
+; CHECK-NEXT:    [[TMP30:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP14]]
+; CHECK-NEXT:    [[TMP31:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP15]]
+; CHECK-NEXT:    [[TMP32:%.*]] = load i1, i1* [[TMP16]]
+; CHECK-NEXT:    [[TMP33:%.*]] = load i1, i1* [[TMP17]]
+; CHECK-NEXT:    [[TMP34:%.*]] = load i1, i1* [[TMP18]]
+; CHECK-NEXT:    [[TMP35:%.*]] = load i1, i1* [[TMP19]]
+; CHECK-NEXT:    [[TMP36:%.*]] = insertelement <4 x i1> undef, i1 [[TMP32]], i32 0
+; CHECK-NEXT:    [[TMP37:%.*]] = insertelement <4 x i1> [[TMP36]], i1 [[TMP33]], i32 1
+; CHECK-NEXT:    [[TMP38:%.*]] = insertelement <4 x i1> [[TMP37]], i1 [[TMP34]], i32 2
+; CHECK-NEXT:    [[TMP39:%.*]] = insertelement <4 x i1> [[TMP38]], i1 [[TMP35]], i32 3
+; CHECK-NEXT:    [[TMP40:%.*]] = load i1, i1* [[TMP20]]
+; CHECK-NEXT:    [[TMP41:%.*]] = load i1, i1* [[TMP21]]
+; CHECK-NEXT:    [[TMP42:%.*]] = load i1, i1* [[TMP22]]
+; CHECK-NEXT:    [[TMP43:%.*]] = load i1, i1* [[TMP23]]
+; CHECK-NEXT:    [[TMP44:%.*]] = insertelement <4 x i1> undef, i1 [[TMP40]], i32 0
+; CHECK-NEXT:    [[TMP45:%.*]] = insertelement <4 x i1> [[TMP44]], i1 [[TMP41]], i32 1
+; CHECK-NEXT:    [[TMP46:%.*]] = insertelement <4 x i1> [[TMP45]], i1 [[TMP42]], i32 2
+; CHECK-NEXT:    [[TMP47:%.*]] = insertelement <4 x i1> [[TMP46]], i1 [[TMP43]], i32 3
+; CHECK-NEXT:    [[TMP48:%.*]] = load i1, i1* [[TMP24]]
+; CHECK-NEXT:    [[TMP49:%.*]] = load i1, i1* [[TMP25]]
+; CHECK-NEXT:    [[TMP50:%.*]] = load i1, i1* [[TMP26]]
+; CHECK-NEXT:    [[TMP51:%.*]] = load i1, i1* [[TMP27]]
+; CHECK-NEXT:    [[TMP52:%.*]] = insertelement <4 x i1> undef, i1 [[TMP48]], i32 0
+; CHECK-NEXT:    [[TMP53:%.*]] = insertelement <4 x i1> [[TMP52]], i1 [[TMP49]], i32 1
+; CHECK-NEXT:    [[TMP54:%.*]] = insertelement <4 x i1> [[TMP53]], i1 [[TMP50]], i32 2
+; CHECK-NEXT:    [[TMP55:%.*]] = insertelement <4 x i1> [[TMP54]], i1 [[TMP51]], i32 3
+; CHECK-NEXT:    [[TMP56:%.*]] = load i1, i1* [[TMP28]]
+; CHECK-NEXT:    [[TMP57:%.*]] = load i1, i1* [[TMP29]]
+; CHECK-NEXT:    [[TMP58:%.*]] = load i1, i1* [[TMP30]]
+; CHECK-NEXT:    [[TMP59:%.*]] = load i1, i1* [[TMP31]]
+; CHECK-NEXT:    [[TMP60:%.*]] = insertelement <4 x i1> undef, i1 [[TMP56]], i32 0
+; CHECK-NEXT:    [[TMP61:%.*]] = insertelement <4 x i1> [[TMP60]], i1 [[TMP57]], i32 1
+; CHECK-NEXT:    [[TMP62:%.*]] = insertelement <4 x i1> [[TMP61]], i1 [[TMP58]], i32 2
+; CHECK-NEXT:    [[TMP63:%.*]] = insertelement <4 x i1> [[TMP62]], i1 [[TMP59]], i32 3
+; CHECK-NEXT:    [[TMP64:%.*]] = extractelement <4 x i1> [[TMP39]], i32 0
+; CHECK-NEXT:    br i1 [[TMP64]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
+; CHECK:       pred.load.if:
+; CHECK-NEXT:    [[TMP65:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP66:%.*]] = load i32, i32* [[TMP65]]
+; CHECK-NEXT:    [[TMP67:%.*]] = insertelement <4 x i32> undef, i32 [[TMP66]], i32 0
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE]]
+; CHECK:       pred.load.continue:
+; CHECK-NEXT:    [[TMP68:%.*]] = phi <4 x i32> [ undef, [[VECTOR_BODY]] ], [ [[TMP67]], [[PRED_LOAD_IF]] ]
+; CHECK-NEXT:    [[TMP69:%.*]] = extractelement <4 x i1> [[TMP39]], i32 1
+; CHECK-NEXT:    br i1 [[TMP69]], label [[PRED_LOAD_IF7:%.*]], label [[PRED_LOAD_CONTINUE8:%.*]]
+; CHECK:       pred.load.if7:
+; CHECK-NEXT:    [[TMP70:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP71:%.*]] = load i32, i32* [[TMP70]]
+; CHECK-NEXT:    [[TMP72:%.*]] = insertelement <4 x i32> [[TMP68]], i32 [[TMP71]], i32 1
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE8]]
+; CHECK:       pred.load.continue8:
+; CHECK-NEXT:    [[TMP73:%.*]] = phi <4 x i32> [ [[TMP68]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP72]], [[PRED_LOAD_IF7]] ]
+; CHECK-NEXT:    [[TMP74:%.*]] = extractelement <4 x i1> [[TMP39]], i32 2
+; CHECK-NEXT:    br i1 [[TMP74]], label [[PRED_LOAD_IF9:%.*]], label [[PRED_LOAD_CONTINUE10:%.*]]
+; CHECK:       pred.load.if9:
+; CHECK-NEXT:    [[TMP75:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP76:%.*]] = load i32, i32* [[TMP75]]
+; CHECK-NEXT:    [[TMP77:%.*]] = insertelement <4 x i32> [[TMP73]], i32 [[TMP76]], i32 2
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE10]]
+; CHECK:       pred.load.continue10:
+; CHECK-NEXT:    [[TMP78:%.*]] = phi <4 x i32> [ [[TMP73]], [[PRED_LOAD_CONTINUE8]] ], [ [[TMP77]], [[PRED_LOAD_IF9]] ]
+; CHECK-NEXT:    [[TMP79:%.*]] = extractelement <4 x i1> [[TMP39]], i32 3
+; CHECK-NEXT:    br i1 [[TMP79]], label [[PRED_LOAD_IF11:%.*]], label [[PRED_LOAD_CONTINUE12:%.*]]
+; CHECK:       pred.load.if11:
+; CHECK-NEXT:    [[TMP80:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[TMP3]]
+; CHECK-NEXT:    [[TMP81:%.*]] = load i32, i32* [[TMP80]]
+; CHECK-NEXT:    [[TMP82:%.*]] = insertelement <4 x i32> [[TMP78]], i32 [[TMP81]], i32 3
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE12]]
+; CHECK:       pred.load.continue12:
+; CHECK-NEXT:    [[TMP83:%.*]] = phi <4 x i32> [ [[TMP78]], [[PRED_LOAD_CONTINUE10]] ], [ [[TMP82]], [[PRED_LOAD_IF11]] ]
+; CHECK-NEXT:    [[TMP84:%.*]] = extractelement <4 x i1> [[TMP47]], i32 0
+; CHECK-NEXT:    br i1 [[TMP84]], label [[PRED_LOAD_IF13:%.*]], label [[PRED_LOAD_CONTINUE14:%.*]]
+; CHECK:       pred.load.if13:
+; CHECK-NEXT:    [[TMP85:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP86:%.*]] = load i32, i32* [[TMP85]]
+; CHECK-NEXT:    [[TMP87:%.*]] = insertelement <4 x i32> undef, i32 [[TMP86]], i32 0
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE14]]
+; CHECK:       pred.load.continue14:
+; CHECK-NEXT:    [[TMP88:%.*]] = phi <4 x i32> [ undef, [[PRED_LOAD_CONTINUE12]] ], [ [[TMP87]], [[PRED_LOAD_IF13]] ]
+; CHECK-NEXT:    [[TMP89:%.*]] = extractelement <4 x i1> [[TMP47]], i32 1
+; CHECK-NEXT:    br i1 [[TMP89]], label [[PRED_LOAD_IF15:%.*]], label [[PRED_LOAD_CONTINUE16:%.*]]
+; CHECK:       pred.load.if15:
+; CHECK-NEXT:    [[TMP90:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP91:%.*]] = load i32, i32* [[TMP90]]
+; CHECK-NEXT:    [[TMP92:%.*]] = insertelement <4 x i32> [[TMP88]], i32 [[TMP91]], i32 1
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE16]]
+; CHECK:       pred.load.continue16:
+; CHECK-NEXT:    [[TMP93:%.*]] = phi <4 x i32> [ [[TMP88]], [[PRED_LOAD_CONTINUE14]] ], [ [[TMP92]], [[PRED_LOAD_IF15]] ]
+; CHECK-NEXT:    [[TMP94:%.*]] = extractelement <4 x i1> [[TMP47]], i32 2
+; CHECK-NEXT:    br i1 [[TMP94]], label [[PRED_LOAD_IF17:%.*]], label [[PRED_LOAD_CONTINUE18:%.*]]
+; CHECK:       pred.load.if17:
+; CHECK-NEXT:    [[TMP95:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP96:%.*]] = load i32, i32* [[TMP95]]
+; CHECK-NEXT:    [[TMP97:%.*]] = insertelement <4 x i32> [[TMP93]], i32 [[TMP96]], i32 2
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE18]]
+; CHECK:       pred.load.continue18:
+; CHECK-NEXT:    [[TMP98:%.*]] = phi <4 x i32> [ [[TMP93]], [[PRED_LOAD_CONTINUE16]] ], [ [[TMP97]], [[PRED_LOAD_IF17]] ]
+; CHECK-NEXT:    [[TMP99:%.*]] = extractelement <4 x i1> [[TMP47]], i32 3
+; CHECK-NEXT:    br i1 [[TMP99]], label [[PRED_LOAD_IF19:%.*]], label [[PRED_LOAD_CONTINUE20:%.*]]
+; CHECK:       pred.load.if19:
+; CHECK-NEXT:    [[TMP100:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP101:%.*]] = load i32, i32* [[TMP100]]
+; CHECK-NEXT:    [[TMP102:%.*]] = insertelement <4 x i32> [[TMP98]], i32 [[TMP101]], i32 3
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE20]]
+; CHECK:       pred.load.continue20:
+; CHECK-NEXT:    [[TMP103:%.*]] = phi <4 x i32> [ [[TMP98]], [[PRED_LOAD_CONTINUE18]] ], [ [[TMP102]], [[PRED_LOAD_IF19]] ]
+; CHECK-NEXT:    [[TMP104:%.*]] = extractelement <4 x i1> [[TMP55]], i32 0
+; CHECK-NEXT:    br i1 [[TMP104]], label [[PRED_LOAD_IF21:%.*]], label [[PRED_LOAD_CONTINUE22:%.*]]
+; CHECK:       pred.load.if21:
+; CHECK-NEXT:    [[TMP105:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP106:%.*]] = load i32, i32* [[TMP105]]
+; CHECK-NEXT:    [[TMP107:%.*]] = insertelement <4 x i32> undef, i32 [[TMP106]], i32 0
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE22]]
+; CHECK:       pred.load.continue22:
+; CHECK-NEXT:    [[TMP108:%.*]] = phi <4 x i32> [ undef, [[PRED_LOAD_CONTINUE20]] ], [ [[TMP107]], [[PRED_LOAD_IF21]] ]
+; CHECK-NEXT:    [[TMP109:%.*]] = extractelement <4 x i1> [[TMP55]], i32 1
+; CHECK-NEXT:    br i1 [[TMP109]], label [[PRED_LOAD_IF23:%.*]], label [[PRED_LOAD_CONTINUE24:%.*]]
+; CHECK:       pred.load.if23:
+; CHECK-NEXT:    [[TMP110:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[TMP9]]
+; CHECK-NEXT:    [[TMP111:%.*]] = load i32, i32* [[TMP110]]
+; CHECK-NEXT:    [[TMP112:%.*]] = insertelement <4 x i32> [[TMP108]], i32 [[TMP111]], i32 1
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE24]]
+; CHECK:       pred.load.continue24:
+; CHECK-NEXT:    [[TMP113:%.*]] = phi <4 x i32> [ [[TMP108]], [[PRED_LOAD_CONTINUE22]] ], [ [[TMP112]], [[PRED_LOAD_IF23]] ]
+; CHECK-NEXT:    [[TMP114:%.*]] = extractelement <4 x i1> [[TMP55]], i32 2
+; CHECK-NEXT:    br i1 [[TMP114]], label [[PRED_LOAD_IF25:%.*]], label [[PRED_LOAD_CONTINUE26:%.*]]
+; CHECK:       pred.load.if25:
+; CHECK-NEXT:    [[TMP115:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[TMP10]]
+; CHECK-NEXT:    [[TMP116:%.*]] = load i32, i32* [[TMP115]]
+; CHECK-NEXT:    [[TMP117:%.*]] = insertelement <4 x i32> [[TMP113]], i32 [[TMP116]], i32 2
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE26]]
+; CHECK:       pred.load.continue26:
+; CHECK-NEXT:    [[TMP118:%.*]] = phi <4 x i32> [ [[TMP113]], [[PRED_LOAD_CONTINUE24]] ], [ [[TMP117]], [[PRED_LOAD_IF25]] ]
+; CHECK-NEXT:    [[TMP119:%.*]] = extractelement <4 x i1> [[TMP55]], i32 3
+; CHECK-NEXT:    br i1 [[TMP119]], label [[PRED_LOAD_IF27:%.*]], label [[PRED_LOAD_CONTINUE28:%.*]]
+; CHECK:       pred.load.if27:
+; CHECK-NEXT:    [[TMP120:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[TMP11]]
+; CHECK-NEXT:    [[TMP121:%.*]] = load i32, i32* [[TMP120]]
+; CHECK-NEXT:    [[TMP122:%.*]] = insertelement <4 x i32> [[TMP118]], i32 [[TMP121]], i32 3
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE28]]
+; CHECK:       pred.load.continue28:
+; CHECK-NEXT:    [[TMP123:%.*]] = phi <4 x i32> [ [[TMP118]], [[PRED_LOAD_CONTINUE26]] ], [ [[TMP122]], [[PRED_LOAD_IF27]] ]
+; CHECK-NEXT:    [[TMP124:%.*]] = extractelement <4 x i1> [[TMP63]], i32 0
+; CHECK-NEXT:    br i1 [[TMP124]], label [[PRED_LOAD_IF29:%.*]], label [[PRED_LOAD_CONTINUE30:%.*]]
+; CHECK:       pred.load.if29:
+; CHECK-NEXT:    [[TMP125:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[TMP12]]
+; CHECK-NEXT:    [[TMP126:%.*]] = load i32, i32* [[TMP125]]
+; CHECK-NEXT:    [[TMP127:%.*]] = insertelement <4 x i32> undef, i32 [[TMP126]], i32 0
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE30]]
+; CHECK:       pred.load.continue30:
+; CHECK-NEXT:    [[TMP128:%.*]] = phi <4 x i32> [ undef, [[PRED_LOAD_CONTINUE28]] ], [ [[TMP127]], [[PRED_LOAD_IF29]] ]
+; CHECK-NEXT:    [[TMP129:%.*]] = extractelement <4 x i1> [[TMP63]], i32 1
+; CHECK-NEXT:    br i1 [[TMP129]], label [[PRED_LOAD_IF31:%.*]], label [[PRED_LOAD_CONTINUE32:%.*]]
+; CHECK:       pred.load.if31:
+; CHECK-NEXT:    [[TMP130:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[TMP13]]
+; CHECK-NEXT:    [[TMP131:%.*]] = load i32, i32* [[TMP130]]
+; CHECK-NEXT:    [[TMP132:%.*]] = insertelement <4 x i32> [[TMP128]], i32 [[TMP131]], i32 1
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE32]]
+; CHECK:       pred.load.continue32:
+; CHECK-NEXT:    [[TMP133:%.*]] = phi <4 x i32> [ [[TMP128]], [[PRED_LOAD_CONTINUE30]] ], [ [[TMP132]], [[PRED_LOAD_IF31]] ]
+; CHECK-NEXT:    [[TMP134:%.*]] = extractelement <4 x i1> [[TMP63]], i32 2
+; CHECK-NEXT:    br i1 [[TMP134]], label [[PRED_LOAD_IF33:%.*]], label [[PRED_LOAD_CONTINUE34:%.*]]
+; CHECK:       pred.load.if33:
+; CHECK-NEXT:    [[TMP135:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[TMP14]]
+; CHECK-NEXT:    [[TMP136:%.*]] = load i32, i32* [[TMP135]]
+; CHECK-NEXT:    [[TMP137:%.*]] = insertelement <4 x i32> [[TMP133]], i32 [[TMP136]], i32 2
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE34]]
+; CHECK:       pred.load.continue34:
+; CHECK-NEXT:    [[TMP138:%.*]] = phi <4 x i32> [ [[TMP133]], [[PRED_LOAD_CONTINUE32]] ], [ [[TMP137]], [[PRED_LOAD_IF33]] ]
+; CHECK-NEXT:    [[TMP139:%.*]] = extractelement <4 x i1> [[TMP63]], i32 3
+; CHECK-NEXT:    br i1 [[TMP139]], label [[PRED_LOAD_IF35:%.*]], label [[PRED_LOAD_CONTINUE36]]
+; CHECK:       pred.load.if35:
+; CHECK-NEXT:    [[TMP140:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[TMP15]]
+; CHECK-NEXT:    [[TMP141:%.*]] = load i32, i32* [[TMP140]]
+; CHECK-NEXT:    [[TMP142:%.*]] = insertelement <4 x i32> [[TMP138]], i32 [[TMP141]], i32 3
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE36]]
+; CHECK:       pred.load.continue36:
+; CHECK-NEXT:    [[TMP143:%.*]] = phi <4 x i32> [ [[TMP138]], [[PRED_LOAD_CONTINUE34]] ], [ [[TMP142]], [[PRED_LOAD_IF35]] ]
+; CHECK-NEXT:    [[TMP144:%.*]] = xor <4 x i1> [[TMP39]], <i1 true, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    [[TMP145:%.*]] = xor <4 x i1> [[TMP47]], <i1 true, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    [[TMP146:%.*]] = xor <4 x i1> [[TMP55]], <i1 true, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    [[TMP147:%.*]] = xor <4 x i1> [[TMP63]], <i1 true, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP39]], <4 x i32> [[TMP83]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[PREDPHI37:%.*]] = select <4 x i1> [[TMP47]], <4 x i32> [[TMP103]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[PREDPHI38:%.*]] = select <4 x i1> [[TMP55]], <4 x i32> [[TMP123]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[PREDPHI39:%.*]] = select <4 x i1> [[TMP63]], <4 x i32> [[TMP143]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP148]] = add <4 x i32> [[VEC_PHI]], [[PREDPHI]]
+; CHECK-NEXT:    [[TMP149]] = add <4 x i32> [[VEC_PHI4]], [[PREDPHI37]]
+; CHECK-NEXT:    [[TMP150]] = add <4 x i32> [[VEC_PHI5]], [[PREDPHI38]]
+; CHECK-NEXT:    [[TMP151]] = add <4 x i32> [[VEC_PHI6]], [[PREDPHI39]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 16
+; CHECK-NEXT:    [[TMP152:%.*]] = icmp eq i64 [[INDEX_NEXT]], 2048
+; CHECK-NEXT:    br i1 [[TMP152]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !14
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <4 x i32> [[TMP149]], [[TMP148]]
+; CHECK-NEXT:    [[BIN_RDX40:%.*]] = add <4 x i32> [[TMP150]], [[BIN_RDX]]
+; CHECK-NEXT:    [[BIN_RDX41:%.*]] = add <4 x i32> [[TMP151]], [[BIN_RDX40]]
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[BIN_RDX41]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX42:%.*]] = add <4 x i32> [[BIN_RDX41]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF43:%.*]] = shufflevector <4 x i32> [[BIN_RDX42]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX44:%.*]] = add <4 x i32> [[BIN_RDX42]], [[RDX_SHUF43]]
+; CHECK-NEXT:    [[TMP153:%.*]] = extractelement <4 x i32> [[BIN_RDX44]], i32 0
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 2048, 2048
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[LOOP_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 4096, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[TMP153]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
+; CHECK-NEXT:    [[ACCUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ACCUM_NEXT:%.*]], [[LATCH]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 2
+; CHECK-NEXT:    [[TEST_ADDR:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[IV]]
+; CHECK-NEXT:    [[EARLYCND:%.*]] = load i1, i1* [[TEST_ADDR]]
+; CHECK-NEXT:    br i1 [[EARLYCND]], label [[PRED:%.*]], label [[LATCH]]
+; CHECK:       pred:
+; CHECK-NEXT:    [[ADDR:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[IV]]
+; CHECK-NEXT:    [[VAL:%.*]] = load i32, i32* [[ADDR]]
+; CHECK-NEXT:    br label [[LATCH]]
+; CHECK:       latch:
+; CHECK-NEXT:    [[VAL_PHI:%.*]] = phi i32 [ 0, [[LOOP]] ], [ [[VAL]], [[PRED]] ]
+; CHECK-NEXT:    [[ACCUM_NEXT]] = add i32 [[ACCUM]], [[VAL_PHI]]
+; CHECK-NEXT:    [[EXIT:%.*]] = icmp ugt i64 [[IV]], 4093
+; CHECK-NEXT:    br i1 [[EXIT]], label [[LOOP_EXIT]], label [[LOOP]], !llvm.loop !15
+; CHECK:       loop_exit:
+; CHECK-NEXT:    [[ACCUM_NEXT_LCSSA:%.*]] = phi i32 [ [[ACCUM_NEXT]], [[LATCH]] ], [ [[TMP153]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    ret i32 [[ACCUM_NEXT_LCSSA]]
+;
+entry:
+  %alloca = alloca [4096 x i32]
+  %base = bitcast [4096 x i32]* %alloca to i32*
+  call void @init(i32* %base)
+  br label %loop
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
+  %accum = phi i32 [ 0, %entry ], [ %accum.next, %latch ]
+  %iv.next = add i64 %iv, 2
+  %test_addr = getelementptr inbounds i1, i1* %test_base, i64 %iv
+  %earlycnd = load i1, i1* %test_addr
+  br i1 %earlycnd, label %pred, label %latch
+pred:
+  %addr = getelementptr inbounds i32, i32* %base, i64 %iv
+  %val = load i32, i32* %addr
+  br label %latch
+latch:
+  %val.phi = phi i32 [0, %loop], [%val, %pred]
+  %accum.next = add i32 %accum, %val.phi
+  %exit = icmp ugt i64 %iv, 4093
+  br i1 %exit, label %loop_exit, label %loop
+
+loop_exit:
+  ret i32 %accum.next
+}
+
+define i32 @neg_off_by_many(i64 %len, i1* %test_base) {
+; CHECK-LABEL: @neg_off_by_many(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ALLOCA:%.*]] = alloca [1024 x i32]
+; CHECK-NEXT:    [[BASE:%.*]] = bitcast [1024 x i32]* [[ALLOCA]] to i32*
+; CHECK-NEXT:    call void @init(i32* [[BASE]])
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP80:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI4:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP81:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI5:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP82:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI6:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP83:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> undef, i64 [[INDEX]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], <i64 0, i64 1, i64 2, i64 3>
+; CHECK-NEXT:    [[INDUCTION1:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], <i64 4, i64 5, i64 6, i64 7>
+; CHECK-NEXT:    [[INDUCTION2:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], <i64 8, i64 9, i64 10, i64 11>
+; CHECK-NEXT:    [[INDUCTION3:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], <i64 12, i64 13, i64 14, i64 15>
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[INDEX]], 3
+; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = add i64 [[INDEX]], 5
+; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[INDEX]], 6
+; CHECK-NEXT:    [[TMP7:%.*]] = add i64 [[INDEX]], 7
+; CHECK-NEXT:    [[TMP8:%.*]] = add i64 [[INDEX]], 8
+; CHECK-NEXT:    [[TMP9:%.*]] = add i64 [[INDEX]], 9
+; CHECK-NEXT:    [[TMP10:%.*]] = add i64 [[INDEX]], 10
+; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[INDEX]], 11
+; CHECK-NEXT:    [[TMP12:%.*]] = add i64 [[INDEX]], 12
+; CHECK-NEXT:    [[TMP13:%.*]] = add i64 [[INDEX]], 13
+; CHECK-NEXT:    [[TMP14:%.*]] = add i64 [[INDEX]], 14
+; CHECK-NEXT:    [[TMP15:%.*]] = add i64 [[INDEX]], 15
+; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE:%.*]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP3]]
+; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP25:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP9]]
+; CHECK-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP10]]
+; CHECK-NEXT:    [[TMP27:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP11]]
+; CHECK-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP12]]
+; CHECK-NEXT:    [[TMP29:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP13]]
+; CHECK-NEXT:    [[TMP30:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP14]]
+; CHECK-NEXT:    [[TMP31:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP15]]
+; CHECK-NEXT:    [[TMP32:%.*]] = load i1, i1* [[TMP16]]
+; CHECK-NEXT:    [[TMP33:%.*]] = load i1, i1* [[TMP17]]
+; CHECK-NEXT:    [[TMP34:%.*]] = load i1, i1* [[TMP18]]
+; CHECK-NEXT:    [[TMP35:%.*]] = load i1, i1* [[TMP19]]
+; CHECK-NEXT:    [[TMP36:%.*]] = insertelement <4 x i1> undef, i1 [[TMP32]], i32 0
+; CHECK-NEXT:    [[TMP37:%.*]] = insertelement <4 x i1> [[TMP36]], i1 [[TMP33]], i32 1
+; CHECK-NEXT:    [[TMP38:%.*]] = insertelement <4 x i1> [[TMP37]], i1 [[TMP34]], i32 2
+; CHECK-NEXT:    [[TMP39:%.*]] = insertelement <4 x i1> [[TMP38]], i1 [[TMP35]], i32 3
+; CHECK-NEXT:    [[TMP40:%.*]] = load i1, i1* [[TMP20]]
+; CHECK-NEXT:    [[TMP41:%.*]] = load i1, i1* [[TMP21]]
+; CHECK-NEXT:    [[TMP42:%.*]] = load i1, i1* [[TMP22]]
+; CHECK-NEXT:    [[TMP43:%.*]] = load i1, i1* [[TMP23]]
+; CHECK-NEXT:    [[TMP44:%.*]] = insertelement <4 x i1> undef, i1 [[TMP40]], i32 0
+; CHECK-NEXT:    [[TMP45:%.*]] = insertelement <4 x i1> [[TMP44]], i1 [[TMP41]], i32 1
+; CHECK-NEXT:    [[TMP46:%.*]] = insertelement <4 x i1> [[TMP45]], i1 [[TMP42]], i32 2
+; CHECK-NEXT:    [[TMP47:%.*]] = insertelement <4 x i1> [[TMP46]], i1 [[TMP43]], i32 3
+; CHECK-NEXT:    [[TMP48:%.*]] = load i1, i1* [[TMP24]]
+; CHECK-NEXT:    [[TMP49:%.*]] = load i1, i1* [[TMP25]]
+; CHECK-NEXT:    [[TMP50:%.*]] = load i1, i1* [[TMP26]]
+; CHECK-NEXT:    [[TMP51:%.*]] = load i1, i1* [[TMP27]]
+; CHECK-NEXT:    [[TMP52:%.*]] = insertelement <4 x i1> undef, i1 [[TMP48]], i32 0
+; CHECK-NEXT:    [[TMP53:%.*]] = insertelement <4 x i1> [[TMP52]], i1 [[TMP49]], i32 1
+; CHECK-NEXT:    [[TMP54:%.*]] = insertelement <4 x i1> [[TMP53]], i1 [[TMP50]], i32 2
+; CHECK-NEXT:    [[TMP55:%.*]] = insertelement <4 x i1> [[TMP54]], i1 [[TMP51]], i32 3
+; CHECK-NEXT:    [[TMP56:%.*]] = load i1, i1* [[TMP28]]
+; CHECK-NEXT:    [[TMP57:%.*]] = load i1, i1* [[TMP29]]
+; CHECK-NEXT:    [[TMP58:%.*]] = load i1, i1* [[TMP30]]
+; CHECK-NEXT:    [[TMP59:%.*]] = load i1, i1* [[TMP31]]
+; CHECK-NEXT:    [[TMP60:%.*]] = insertelement <4 x i1> undef, i1 [[TMP56]], i32 0
+; CHECK-NEXT:    [[TMP61:%.*]] = insertelement <4 x i1> [[TMP60]], i1 [[TMP57]], i32 1
+; CHECK-NEXT:    [[TMP62:%.*]] = insertelement <4 x i1> [[TMP61]], i1 [[TMP58]], i32 2
+; CHECK-NEXT:    [[TMP63:%.*]] = insertelement <4 x i1> [[TMP62]], i1 [[TMP59]], i32 3
+; CHECK-NEXT:    [[TMP64:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP65:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP66:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP67:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[TMP12]]
+; CHECK-NEXT:    [[TMP68:%.*]] = getelementptr inbounds i32, i32* [[TMP64]], i32 0
+; CHECK-NEXT:    [[TMP69:%.*]] = bitcast i32* [[TMP68]] to <4 x i32>*
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* [[TMP69]], i32 4, <4 x i1> [[TMP39]], <4 x i32> undef)
+; CHECK-NEXT:    [[TMP70:%.*]] = getelementptr inbounds i32, i32* [[TMP64]], i32 4
+; CHECK-NEXT:    [[TMP71:%.*]] = bitcast i32* [[TMP70]] to <4 x i32>*
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD7:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* [[TMP71]], i32 4, <4 x i1> [[TMP47]], <4 x i32> undef)
+; CHECK-NEXT:    [[TMP72:%.*]] = getelementptr inbounds i32, i32* [[TMP64]], i32 8
+; CHECK-NEXT:    [[TMP73:%.*]] = bitcast i32* [[TMP72]] to <4 x i32>*
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD8:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* [[TMP73]], i32 4, <4 x i1> [[TMP55]], <4 x i32> undef)
+; CHECK-NEXT:    [[TMP74:%.*]] = getelementptr inbounds i32, i32* [[TMP64]], i32 12
+; CHECK-NEXT:    [[TMP75:%.*]] = bitcast i32* [[TMP74]] to <4 x i32>*
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD9:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* [[TMP75]], i32 4, <4 x i1> [[TMP63]], <4 x i32> undef)
+; CHECK-NEXT:    [[TMP76:%.*]] = xor <4 x i1> [[TMP39]], <i1 true, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    [[TMP77:%.*]] = xor <4 x i1> [[TMP47]], <i1 true, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    [[TMP78:%.*]] = xor <4 x i1> [[TMP55]], <i1 true, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    [[TMP79:%.*]] = xor <4 x i1> [[TMP63]], <i1 true, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP39]], <4 x i32> [[WIDE_MASKED_LOAD]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[PREDPHI10:%.*]] = select <4 x i1> [[TMP47]], <4 x i32> [[WIDE_MASKED_LOAD7]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[PREDPHI11:%.*]] = select <4 x i1> [[TMP55]], <4 x i32> [[WIDE_MASKED_LOAD8]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[PREDPHI12:%.*]] = select <4 x i1> [[TMP63]], <4 x i32> [[WIDE_MASKED_LOAD9]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP80]] = add <4 x i32> [[VEC_PHI]], [[PREDPHI]]
+; CHECK-NEXT:    [[TMP81]] = add <4 x i32> [[VEC_PHI4]], [[PREDPHI10]]
+; CHECK-NEXT:    [[TMP82]] = add <4 x i32> [[VEC_PHI5]], [[PREDPHI11]]
+; CHECK-NEXT:    [[TMP83]] = add <4 x i32> [[VEC_PHI6]], [[PREDPHI12]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 16
+; CHECK-NEXT:    [[TMP84:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4096
+; CHECK-NEXT:    br i1 [[TMP84]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !16
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <4 x i32> [[TMP81]], [[TMP80]]
+; CHECK-NEXT:    [[BIN_RDX13:%.*]] = add <4 x i32> [[TMP82]], [[BIN_RDX]]
+; CHECK-NEXT:    [[BIN_RDX14:%.*]] = add <4 x i32> [[TMP83]], [[BIN_RDX13]]
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[BIN_RDX14]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX15:%.*]] = add <4 x i32> [[BIN_RDX14]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF16:%.*]] = shufflevector <4 x i32> [[BIN_RDX15]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX17:%.*]] = add <4 x i32> [[BIN_RDX15]], [[RDX_SHUF16]]
+; CHECK-NEXT:    [[TMP85:%.*]] = extractelement <4 x i32> [[BIN_RDX17]], i32 0
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 4096, 4096
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[LOOP_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 4096, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[TMP85]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
+; CHECK-NEXT:    [[ACCUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ACCUM_NEXT:%.*]], [[LATCH]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT:    [[TEST_ADDR:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[IV]]
+; CHECK-NEXT:    [[EARLYCND:%.*]] = load i1, i1* [[TEST_ADDR]]
+; CHECK-NEXT:    br i1 [[EARLYCND]], label [[PRED:%.*]], label [[LATCH]]
+; CHECK:       pred:
+; CHECK-NEXT:    [[ADDR:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[IV]]
+; CHECK-NEXT:    [[VAL:%.*]] = load i32, i32* [[ADDR]]
+; CHECK-NEXT:    br label [[LATCH]]
+; CHECK:       latch:
+; CHECK-NEXT:    [[VAL_PHI:%.*]] = phi i32 [ 0, [[LOOP]] ], [ [[VAL]], [[PRED]] ]
+; CHECK-NEXT:    [[ACCUM_NEXT]] = add i32 [[ACCUM]], [[VAL_PHI]]
+; CHECK-NEXT:    [[EXIT:%.*]] = icmp ugt i64 [[IV]], 4094
+; CHECK-NEXT:    br i1 [[EXIT]], label [[LOOP_EXIT]], label [[LOOP]], !llvm.loop !17
+; CHECK:       loop_exit:
+; CHECK-NEXT:    [[ACCUM_NEXT_LCSSA:%.*]] = phi i32 [ [[ACCUM_NEXT]], [[LATCH]] ], [ [[TMP85]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    ret i32 [[ACCUM_NEXT_LCSSA]]
+;
+entry:
+  %alloca = alloca [1024 x i32]
+  %base = bitcast [1024 x i32]* %alloca to i32*
+  call void @init(i32* %base)
+  br label %loop
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
+  %accum = phi i32 [ 0, %entry ], [ %accum.next, %latch ]
+  %iv.next = add i64 %iv, 1
+  %test_addr = getelementptr inbounds i1, i1* %test_base, i64 %iv
+  %earlycnd = load i1, i1* %test_addr
+  br i1 %earlycnd, label %pred, label %latch
+pred:
+  %addr = getelementptr inbounds i32, i32* %base, i64 %iv
+  %val = load i32, i32* %addr
+  br label %latch
+latch:
+  %val.phi = phi i32 [0, %loop], [%val, %pred]
+  %accum.next = add i32 %accum, %val.phi
+  %exit = icmp ugt i64 %iv, 4094
+  br i1 %exit, label %loop_exit, label %loop
+
+loop_exit:
+  ret i32 %accum.next
+}
+
+define i32 @neg_off_by_one_iteration(i64 %len, i1* %test_base) {
+; CHECK-LABEL: @neg_off_by_one_iteration(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ALLOCA:%.*]] = alloca [4095 x i32]
+; CHECK-NEXT:    [[BASE:%.*]] = bitcast [4095 x i32]* [[ALLOCA]] to i32*
+; CHECK-NEXT:    call void @init(i32* [[BASE]])
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP80:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI4:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP81:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI5:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP82:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI6:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP83:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> undef, i64 [[INDEX]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], <i64 0, i64 1, i64 2, i64 3>
+; CHECK-NEXT:    [[INDUCTION1:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], <i64 4, i64 5, i64 6, i64 7>
+; CHECK-NEXT:    [[INDUCTION2:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], <i64 8, i64 9, i64 10, i64 11>
+; CHECK-NEXT:    [[INDUCTION3:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], <i64 12, i64 13, i64 14, i64 15>
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[INDEX]], 3
+; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = add i64 [[INDEX]], 5
+; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[INDEX]], 6
+; CHECK-NEXT:    [[TMP7:%.*]] = add i64 [[INDEX]], 7
+; CHECK-NEXT:    [[TMP8:%.*]] = add i64 [[INDEX]], 8
+; CHECK-NEXT:    [[TMP9:%.*]] = add i64 [[INDEX]], 9
+; CHECK-NEXT:    [[TMP10:%.*]] = add i64 [[INDEX]], 10
+; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[INDEX]], 11
+; CHECK-NEXT:    [[TMP12:%.*]] = add i64 [[INDEX]], 12
+; CHECK-NEXT:    [[TMP13:%.*]] = add i64 [[INDEX]], 13
+; CHECK-NEXT:    [[TMP14:%.*]] = add i64 [[INDEX]], 14
+; CHECK-NEXT:    [[TMP15:%.*]] = add i64 [[INDEX]], 15
+; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE:%.*]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP3]]
+; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP25:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP9]]
+; CHECK-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP10]]
+; CHECK-NEXT:    [[TMP27:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP11]]
+; CHECK-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP12]]
+; CHECK-NEXT:    [[TMP29:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP13]]
+; CHECK-NEXT:    [[TMP30:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP14]]
+; CHECK-NEXT:    [[TMP31:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP15]]
+; CHECK-NEXT:    [[TMP32:%.*]] = load i1, i1* [[TMP16]]
+; CHECK-NEXT:    [[TMP33:%.*]] = load i1, i1* [[TMP17]]
+; CHECK-NEXT:    [[TMP34:%.*]] = load i1, i1* [[TMP18]]
+; CHECK-NEXT:    [[TMP35:%.*]] = load i1, i1* [[TMP19]]
+; CHECK-NEXT:    [[TMP36:%.*]] = insertelement <4 x i1> undef, i1 [[TMP32]], i32 0
+; CHECK-NEXT:    [[TMP37:%.*]] = insertelement <4 x i1> [[TMP36]], i1 [[TMP33]], i32 1
+; CHECK-NEXT:    [[TMP38:%.*]] = insertelement <4 x i1> [[TMP37]], i1 [[TMP34]], i32 2
+; CHECK-NEXT:    [[TMP39:%.*]] = insertelement <4 x i1> [[TMP38]], i1 [[TMP35]], i32 3
+; CHECK-NEXT:    [[TMP40:%.*]] = load i1, i1* [[TMP20]]
+; CHECK-NEXT:    [[TMP41:%.*]] = load i1, i1* [[TMP21]]
+; CHECK-NEXT:    [[TMP42:%.*]] = load i1, i1* [[TMP22]]
+; CHECK-NEXT:    [[TMP43:%.*]] = load i1, i1* [[TMP23]]
+; CHECK-NEXT:    [[TMP44:%.*]] = insertelement <4 x i1> undef, i1 [[TMP40]], i32 0
+; CHECK-NEXT:    [[TMP45:%.*]] = insertelement <4 x i1> [[TMP44]], i1 [[TMP41]], i32 1
+; CHECK-NEXT:    [[TMP46:%.*]] = insertelement <4 x i1> [[TMP45]], i1 [[TMP42]], i32 2
+; CHECK-NEXT:    [[TMP47:%.*]] = insertelement <4 x i1> [[TMP46]], i1 [[TMP43]], i32 3
+; CHECK-NEXT:    [[TMP48:%.*]] = load i1, i1* [[TMP24]]
+; CHECK-NEXT:    [[TMP49:%.*]] = load i1, i1* [[TMP25]]
+; CHECK-NEXT:    [[TMP50:%.*]] = load i1, i1* [[TMP26]]
+; CHECK-NEXT:    [[TMP51:%.*]] = load i1, i1* [[TMP27]]
+; CHECK-NEXT:    [[TMP52:%.*]] = insertelement <4 x i1> undef, i1 [[TMP48]], i32 0
+; CHECK-NEXT:    [[TMP53:%.*]] = insertelement <4 x i1> [[TMP52]], i1 [[TMP49]], i32 1
+; CHECK-NEXT:    [[TMP54:%.*]] = insertelement <4 x i1> [[TMP53]], i1 [[TMP50]], i32 2
+; CHECK-NEXT:    [[TMP55:%.*]] = insertelement <4 x i1> [[TMP54]], i1 [[TMP51]], i32 3
+; CHECK-NEXT:    [[TMP56:%.*]] = load i1, i1* [[TMP28]]
+; CHECK-NEXT:    [[TMP57:%.*]] = load i1, i1* [[TMP29]]
+; CHECK-NEXT:    [[TMP58:%.*]] = load i1, i1* [[TMP30]]
+; CHECK-NEXT:    [[TMP59:%.*]] = load i1, i1* [[TMP31]]
+; CHECK-NEXT:    [[TMP60:%.*]] = insertelement <4 x i1> undef, i1 [[TMP56]], i32 0
+; CHECK-NEXT:    [[TMP61:%.*]] = insertelement <4 x i1> [[TMP60]], i1 [[TMP57]], i32 1
+; CHECK-NEXT:    [[TMP62:%.*]] = insertelement <4 x i1> [[TMP61]], i1 [[TMP58]], i32 2
+; CHECK-NEXT:    [[TMP63:%.*]] = insertelement <4 x i1> [[TMP62]], i1 [[TMP59]], i32 3
+; CHECK-NEXT:    [[TMP64:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP65:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP66:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP67:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[TMP12]]
+; CHECK-NEXT:    [[TMP68:%.*]] = getelementptr inbounds i32, i32* [[TMP64]], i32 0
+; CHECK-NEXT:    [[TMP69:%.*]] = bitcast i32* [[TMP68]] to <4 x i32>*
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* [[TMP69]], i32 4, <4 x i1> [[TMP39]], <4 x i32> undef)
+; CHECK-NEXT:    [[TMP70:%.*]] = getelementptr inbounds i32, i32* [[TMP64]], i32 4
+; CHECK-NEXT:    [[TMP71:%.*]] = bitcast i32* [[TMP70]] to <4 x i32>*
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD7:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* [[TMP71]], i32 4, <4 x i1> [[TMP47]], <4 x i32> undef)
+; CHECK-NEXT:    [[TMP72:%.*]] = getelementptr inbounds i32, i32* [[TMP64]], i32 8
+; CHECK-NEXT:    [[TMP73:%.*]] = bitcast i32* [[TMP72]] to <4 x i32>*
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD8:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* [[TMP73]], i32 4, <4 x i1> [[TMP55]], <4 x i32> undef)
+; CHECK-NEXT:    [[TMP74:%.*]] = getelementptr inbounds i32, i32* [[TMP64]], i32 12
+; CHECK-NEXT:    [[TMP75:%.*]] = bitcast i32* [[TMP74]] to <4 x i32>*
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD9:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* [[TMP75]], i32 4, <4 x i1> [[TMP63]], <4 x i32> undef)
+; CHECK-NEXT:    [[TMP76:%.*]] = xor <4 x i1> [[TMP39]], <i1 true, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    [[TMP77:%.*]] = xor <4 x i1> [[TMP47]], <i1 true, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    [[TMP78:%.*]] = xor <4 x i1> [[TMP55]], <i1 true, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    [[TMP79:%.*]] = xor <4 x i1> [[TMP63]], <i1 true, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP39]], <4 x i32> [[WIDE_MASKED_LOAD]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[PREDPHI10:%.*]] = select <4 x i1> [[TMP47]], <4 x i32> [[WIDE_MASKED_LOAD7]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[PREDPHI11:%.*]] = select <4 x i1> [[TMP55]], <4 x i32> [[WIDE_MASKED_LOAD8]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[PREDPHI12:%.*]] = select <4 x i1> [[TMP63]], <4 x i32> [[WIDE_MASKED_LOAD9]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP80]] = add <4 x i32> [[VEC_PHI]], [[PREDPHI]]
+; CHECK-NEXT:    [[TMP81]] = add <4 x i32> [[VEC_PHI4]], [[PREDPHI10]]
+; CHECK-NEXT:    [[TMP82]] = add <4 x i32> [[VEC_PHI5]], [[PREDPHI11]]
+; CHECK-NEXT:    [[TMP83]] = add <4 x i32> [[VEC_PHI6]], [[PREDPHI12]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 16
+; CHECK-NEXT:    [[TMP84:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4096
+; CHECK-NEXT:    br i1 [[TMP84]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !18
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <4 x i32> [[TMP81]], [[TMP80]]
+; CHECK-NEXT:    [[BIN_RDX13:%.*]] = add <4 x i32> [[TMP82]], [[BIN_RDX]]
+; CHECK-NEXT:    [[BIN_RDX14:%.*]] = add <4 x i32> [[TMP83]], [[BIN_RDX13]]
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[BIN_RDX14]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX15:%.*]] = add <4 x i32> [[BIN_RDX14]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF16:%.*]] = shufflevector <4 x i32> [[BIN_RDX15]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX17:%.*]] = add <4 x i32> [[BIN_RDX15]], [[RDX_SHUF16]]
+; CHECK-NEXT:    [[TMP85:%.*]] = extractelement <4 x i32> [[BIN_RDX17]], i32 0
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 4096, 4096
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[LOOP_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 4096, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[TMP85]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
+; CHECK-NEXT:    [[ACCUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ACCUM_NEXT:%.*]], [[LATCH]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT:    [[TEST_ADDR:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[IV]]
+; CHECK-NEXT:    [[EARLYCND:%.*]] = load i1, i1* [[TEST_ADDR]]
+; CHECK-NEXT:    br i1 [[EARLYCND]], label [[PRED:%.*]], label [[LATCH]]
+; CHECK:       pred:
+; CHECK-NEXT:    [[ADDR:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[IV]]
+; CHECK-NEXT:    [[VAL:%.*]] = load i32, i32* [[ADDR]]
+; CHECK-NEXT:    br label [[LATCH]]
+; CHECK:       latch:
+; CHECK-NEXT:    [[VAL_PHI:%.*]] = phi i32 [ 0, [[LOOP]] ], [ [[VAL]], [[PRED]] ]
+; CHECK-NEXT:    [[ACCUM_NEXT]] = add i32 [[ACCUM]], [[VAL_PHI]]
+; CHECK-NEXT:    [[EXIT:%.*]] = icmp ugt i64 [[IV]], 4094
+; CHECK-NEXT:    br i1 [[EXIT]], label [[LOOP_EXIT]], label [[LOOP]], !llvm.loop !19
+; CHECK:       loop_exit:
+; CHECK-NEXT:    [[ACCUM_NEXT_LCSSA:%.*]] = phi i32 [ [[ACCUM_NEXT]], [[LATCH]] ], [ [[TMP85]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    ret i32 [[ACCUM_NEXT_LCSSA]]
+;
+entry:
+  %alloca = alloca [4095 x i32]
+  %base = bitcast [4095 x i32]* %alloca to i32*
+  call void @init(i32* %base)
+  br label %loop
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
+  %accum = phi i32 [ 0, %entry ], [ %accum.next, %latch ]
+  %iv.next = add i64 %iv, 1
+  %test_addr = getelementptr inbounds i1, i1* %test_base, i64 %iv
+  %earlycnd = load i1, i1* %test_addr
+  br i1 %earlycnd, label %pred, label %latch
+pred:
+  %addr = getelementptr inbounds i32, i32* %base, i64 %iv
+  %val = load i32, i32* %addr
+  br label %latch
+latch:
+  %val.phi = phi i32 [0, %loop], [%val, %pred]
+  %accum.next = add i32 %accum, %val.phi
+  %exit = icmp ugt i64 %iv, 4094
+  br i1 %exit, label %loop_exit, label %loop
+
+loop_exit:
+  ret i32 %accum.next
+}
+
+define i32 @neg_off_by_one_byte(i64 %len, i1* %test_base) {
+; CHECK-LABEL: @neg_off_by_one_byte(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ALLOCA:%.*]] = alloca [16383 x i8]
+; CHECK-NEXT:    [[BASE:%.*]] = bitcast [16383 x i8]* [[ALLOCA]] to i32*
+; CHECK-NEXT:    call void @init(i32* [[BASE]])
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP80:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI4:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP81:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI5:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP82:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI6:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP83:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> undef, i64 [[INDEX]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], <i64 0, i64 1, i64 2, i64 3>
+; CHECK-NEXT:    [[INDUCTION1:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], <i64 4, i64 5, i64 6, i64 7>
+; CHECK-NEXT:    [[INDUCTION2:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], <i64 8, i64 9, i64 10, i64 11>
+; CHECK-NEXT:    [[INDUCTION3:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], <i64 12, i64 13, i64 14, i64 15>
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[INDEX]], 3
+; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = add i64 [[INDEX]], 5
+; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[INDEX]], 6
+; CHECK-NEXT:    [[TMP7:%.*]] = add i64 [[INDEX]], 7
+; CHECK-NEXT:    [[TMP8:%.*]] = add i64 [[INDEX]], 8
+; CHECK-NEXT:    [[TMP9:%.*]] = add i64 [[INDEX]], 9
+; CHECK-NEXT:    [[TMP10:%.*]] = add i64 [[INDEX]], 10
+; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[INDEX]], 11
+; CHECK-NEXT:    [[TMP12:%.*]] = add i64 [[INDEX]], 12
+; CHECK-NEXT:    [[TMP13:%.*]] = add i64 [[INDEX]], 13
+; CHECK-NEXT:    [[TMP14:%.*]] = add i64 [[INDEX]], 14
+; CHECK-NEXT:    [[TMP15:%.*]] = add i64 [[INDEX]], 15
+; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE:%.*]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP3]]
+; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP25:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP9]]
+; CHECK-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP10]]
+; CHECK-NEXT:    [[TMP27:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP11]]
+; CHECK-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP12]]
+; CHECK-NEXT:    [[TMP29:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP13]]
+; CHECK-NEXT:    [[TMP30:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP14]]
+; CHECK-NEXT:    [[TMP31:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP15]]
+; CHECK-NEXT:    [[TMP32:%.*]] = load i1, i1* [[TMP16]]
+; CHECK-NEXT:    [[TMP33:%.*]] = load i1, i1* [[TMP17]]
+; CHECK-NEXT:    [[TMP34:%.*]] = load i1, i1* [[TMP18]]
+; CHECK-NEXT:    [[TMP35:%.*]] = load i1, i1* [[TMP19]]
+; CHECK-NEXT:    [[TMP36:%.*]] = insertelement <4 x i1> undef, i1 [[TMP32]], i32 0
+; CHECK-NEXT:    [[TMP37:%.*]] = insertelement <4 x i1> [[TMP36]], i1 [[TMP33]], i32 1
+; CHECK-NEXT:    [[TMP38:%.*]] = insertelement <4 x i1> [[TMP37]], i1 [[TMP34]], i32 2
+; CHECK-NEXT:    [[TMP39:%.*]] = insertelement <4 x i1> [[TMP38]], i1 [[TMP35]], i32 3
+; CHECK-NEXT:    [[TMP40:%.*]] = load i1, i1* [[TMP20]]
+; CHECK-NEXT:    [[TMP41:%.*]] = load i1, i1* [[TMP21]]
+; CHECK-NEXT:    [[TMP42:%.*]] = load i1, i1* [[TMP22]]
+; CHECK-NEXT:    [[TMP43:%.*]] = load i1, i1* [[TMP23]]
+; CHECK-NEXT:    [[TMP44:%.*]] = insertelement <4 x i1> undef, i1 [[TMP40]], i32 0
+; CHECK-NEXT:    [[TMP45:%.*]] = insertelement <4 x i1> [[TMP44]], i1 [[TMP41]], i32 1
+; CHECK-NEXT:    [[TMP46:%.*]] = insertelement <4 x i1> [[TMP45]], i1 [[TMP42]], i32 2
+; CHECK-NEXT:    [[TMP47:%.*]] = insertelement <4 x i1> [[TMP46]], i1 [[TMP43]], i32 3
+; CHECK-NEXT:    [[TMP48:%.*]] = load i1, i1* [[TMP24]]
+; CHECK-NEXT:    [[TMP49:%.*]] = load i1, i1* [[TMP25]]
+; CHECK-NEXT:    [[TMP50:%.*]] = load i1, i1* [[TMP26]]
+; CHECK-NEXT:    [[TMP51:%.*]] = load i1, i1* [[TMP27]]
+; CHECK-NEXT:    [[TMP52:%.*]] = insertelement <4 x i1> undef, i1 [[TMP48]], i32 0
+; CHECK-NEXT:    [[TMP53:%.*]] = insertelement <4 x i1> [[TMP52]], i1 [[TMP49]], i32 1
+; CHECK-NEXT:    [[TMP54:%.*]] = insertelement <4 x i1> [[TMP53]], i1 [[TMP50]], i32 2
+; CHECK-NEXT:    [[TMP55:%.*]] = insertelement <4 x i1> [[TMP54]], i1 [[TMP51]], i32 3
+; CHECK-NEXT:    [[TMP56:%.*]] = load i1, i1* [[TMP28]]
+; CHECK-NEXT:    [[TMP57:%.*]] = load i1, i1* [[TMP29]]
+; CHECK-NEXT:    [[TMP58:%.*]] = load i1, i1* [[TMP30]]
+; CHECK-NEXT:    [[TMP59:%.*]] = load i1, i1* [[TMP31]]
+; CHECK-NEXT:    [[TMP60:%.*]] = insertelement <4 x i1> undef, i1 [[TMP56]], i32 0
+; CHECK-NEXT:    [[TMP61:%.*]] = insertelement <4 x i1> [[TMP60]], i1 [[TMP57]], i32 1
+; CHECK-NEXT:    [[TMP62:%.*]] = insertelement <4 x i1> [[TMP61]], i1 [[TMP58]], i32 2
+; CHECK-NEXT:    [[TMP63:%.*]] = insertelement <4 x i1> [[TMP62]], i1 [[TMP59]], i32 3
+; CHECK-NEXT:    [[TMP64:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP65:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP66:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP67:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[TMP12]]
+; CHECK-NEXT:    [[TMP68:%.*]] = getelementptr inbounds i32, i32* [[TMP64]], i32 0
+; CHECK-NEXT:    [[TMP69:%.*]] = bitcast i32* [[TMP68]] to <4 x i32>*
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* [[TMP69]], i32 4, <4 x i1> [[TMP39]], <4 x i32> undef)
+; CHECK-NEXT:    [[TMP70:%.*]] = getelementptr inbounds i32, i32* [[TMP64]], i32 4
+; CHECK-NEXT:    [[TMP71:%.*]] = bitcast i32* [[TMP70]] to <4 x i32>*
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD7:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* [[TMP71]], i32 4, <4 x i1> [[TMP47]], <4 x i32> undef)
+; CHECK-NEXT:    [[TMP72:%.*]] = getelementptr inbounds i32, i32* [[TMP64]], i32 8
+; CHECK-NEXT:    [[TMP73:%.*]] = bitcast i32* [[TMP72]] to <4 x i32>*
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD8:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* [[TMP73]], i32 4, <4 x i1> [[TMP55]], <4 x i32> undef)
+; CHECK-NEXT:    [[TMP74:%.*]] = getelementptr inbounds i32, i32* [[TMP64]], i32 12
+; CHECK-NEXT:    [[TMP75:%.*]] = bitcast i32* [[TMP74]] to <4 x i32>*
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD9:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* [[TMP75]], i32 4, <4 x i1> [[TMP63]], <4 x i32> undef)
+; CHECK-NEXT:    [[TMP76:%.*]] = xor <4 x i1> [[TMP39]], <i1 true, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    [[TMP77:%.*]] = xor <4 x i1> [[TMP47]], <i1 true, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    [[TMP78:%.*]] = xor <4 x i1> [[TMP55]], <i1 true, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    [[TMP79:%.*]] = xor <4 x i1> [[TMP63]], <i1 true, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP39]], <4 x i32> [[WIDE_MASKED_LOAD]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[PREDPHI10:%.*]] = select <4 x i1> [[TMP47]], <4 x i32> [[WIDE_MASKED_LOAD7]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[PREDPHI11:%.*]] = select <4 x i1> [[TMP55]], <4 x i32> [[WIDE_MASKED_LOAD8]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[PREDPHI12:%.*]] = select <4 x i1> [[TMP63]], <4 x i32> [[WIDE_MASKED_LOAD9]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP80]] = add <4 x i32> [[VEC_PHI]], [[PREDPHI]]
+; CHECK-NEXT:    [[TMP81]] = add <4 x i32> [[VEC_PHI4]], [[PREDPHI10]]
+; CHECK-NEXT:    [[TMP82]] = add <4 x i32> [[VEC_PHI5]], [[PREDPHI11]]
+; CHECK-NEXT:    [[TMP83]] = add <4 x i32> [[VEC_PHI6]], [[PREDPHI12]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 16
+; CHECK-NEXT:    [[TMP84:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4096
+; CHECK-NEXT:    br i1 [[TMP84]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !20
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <4 x i32> [[TMP81]], [[TMP80]]
+; CHECK-NEXT:    [[BIN_RDX13:%.*]] = add <4 x i32> [[TMP82]], [[BIN_RDX]]
+; CHECK-NEXT:    [[BIN_RDX14:%.*]] = add <4 x i32> [[TMP83]], [[BIN_RDX13]]
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[BIN_RDX14]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX15:%.*]] = add <4 x i32> [[BIN_RDX14]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF16:%.*]] = shufflevector <4 x i32> [[BIN_RDX15]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX17:%.*]] = add <4 x i32> [[BIN_RDX15]], [[RDX_SHUF16]]
+; CHECK-NEXT:    [[TMP85:%.*]] = extractelement <4 x i32> [[BIN_RDX17]], i32 0
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 4096, 4096
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[LOOP_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 4096, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[TMP85]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
+; CHECK-NEXT:    [[ACCUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ACCUM_NEXT:%.*]], [[LATCH]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT:    [[TEST_ADDR:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[IV]]
+; CHECK-NEXT:    [[EARLYCND:%.*]] = load i1, i1* [[TEST_ADDR]]
+; CHECK-NEXT:    br i1 [[EARLYCND]], label [[PRED:%.*]], label [[LATCH]]
+; CHECK:       pred:
+; CHECK-NEXT:    [[ADDR:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[IV]]
+; CHECK-NEXT:    [[VAL:%.*]] = load i32, i32* [[ADDR]]
+; CHECK-NEXT:    br label [[LATCH]]
+; CHECK:       latch:
+; CHECK-NEXT:    [[VAL_PHI:%.*]] = phi i32 [ 0, [[LOOP]] ], [ [[VAL]], [[PRED]] ]
+; CHECK-NEXT:    [[ACCUM_NEXT]] = add i32 [[ACCUM]], [[VAL_PHI]]
+; CHECK-NEXT:    [[EXIT:%.*]] = icmp ugt i64 [[IV]], 4094
+; CHECK-NEXT:    br i1 [[EXIT]], label [[LOOP_EXIT]], label [[LOOP]], !llvm.loop !21
+; CHECK:       loop_exit:
+; CHECK-NEXT:    [[ACCUM_NEXT_LCSSA:%.*]] = phi i32 [ [[ACCUM_NEXT]], [[LATCH]] ], [ [[TMP85]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    ret i32 [[ACCUM_NEXT_LCSSA]]
+;
+entry:
+  %alloca = alloca [16383 x i8]
+  %base = bitcast [16383 x i8]* %alloca to i32*
+  call void @init(i32* %base)
+  br label %loop
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
+  %accum = phi i32 [ 0, %entry ], [ %accum.next, %latch ]
+  %iv.next = add i64 %iv, 1
+  %test_addr = getelementptr inbounds i1, i1* %test_base, i64 %iv
+  %earlycnd = load i1, i1* %test_addr
+  br i1 %earlycnd, label %pred, label %latch
+pred:
+  %addr = getelementptr inbounds i32, i32* %base, i64 %iv
+  %val = load i32, i32* %addr
+  br label %latch
+latch:
+  %val.phi = phi i32 [0, %loop], [%val, %pred]
+  %accum.next = add i32 %accum, %val.phi
+  %exit = icmp ugt i64 %iv, 4094
+  br i1 %exit, label %loop_exit, label %loop
+
+loop_exit:
+  ret i32 %accum.next
+}
+
+
diff --git a/llvm/test/Transforms/LoopVectorize/X86/no_fpmath_with_hotness.ll b/llvm/test/Transforms/LoopVectorize/X86/no_fpmath_with_hotness.ll
index fbc4ab2aa..f593b9423 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/no_fpmath_with_hotness.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/no_fpmath_with_hotness.ll
@@ -3,7 +3,7 @@
 
 ; CHECK: remark: no_fpmath.c:6:11: loop not vectorized: cannot prove it is safe to reorder floating-point operations (hotness: 300)
 ; CHECK: remark: no_fpmath.c:6:14: loop not vectorized
-; CHECK: remark: no_fpmath.c:17:14: vectorized loop (vectorization width: 2, interleaved count: 2) (hotness: 300)
+; CHECK: remark: no_fpmath.c:17:14: vectorized loop (vectorization width: 2, interleaved count: 1) (hotness: 300)
 
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.10.0"
diff --git a/llvm/test/Transforms/LoopVectorize/X86/pr42674.ll b/llvm/test/Transforms/LoopVectorize/X86/pr42674.ll
new file mode 100644
index 000000000..adaae12f4
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/pr42674.ll
@@ -0,0 +1,61 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -loop-vectorize -instcombine -simplifycfg -mtriple=x86_64-unknown-linux-gnu -mattr=avx512vl,avx512dq,avx512bw -S | FileCheck %s
+
+@bytes = global [128 x i8] zeroinitializer, align 16
+
+; Make sure we end up with vector code for this loop. We used to try to create
+; a VF=64,UF=4 loop, but the scalar trip count is only 128 so
+; the vector loop was dead code leaving only a scalar remainder.
+define zeroext i8 @sum() {
+; CHECK-LABEL: @sum(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <64 x i8> [ zeroinitializer, [[ENTRY]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI2:%.*]] = phi <64 x i8> [ zeroinitializer, [[ENTRY]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds [128 x i8], [128 x i8]* @bytes, i64 0, i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8* [[TMP0]] to <64 x i8>*
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <64 x i8>, <64 x i8>* [[TMP1]], align 16
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, i8* [[TMP0]], i64 64
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8* [[TMP2]] to <64 x i8>*
+; CHECK-NEXT:    [[WIDE_LOAD3:%.*]] = load <64 x i8>, <64 x i8>* [[TMP3]], align 16
+; CHECK-NEXT:    [[TMP4]] = add <64 x i8> [[WIDE_LOAD]], [[VEC_PHI]]
+; CHECK-NEXT:    [[TMP5]] = add <64 x i8> [[WIDE_LOAD3]], [[VEC_PHI2]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 128
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX]], 0
+; CHECK-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !0
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <64 x i8> [[TMP5]], [[TMP4]]
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <64 x i8> [[BIN_RDX]], <64 x i8> undef, <64 x i32> <i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX4:%.*]] = add <64 x i8> [[BIN_RDX]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF5:%.*]] = shufflevector <64 x i8> [[BIN_RDX4]], <64 x i8> undef, <64 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX6:%.*]] = add <64 x i8> [[BIN_RDX4]], [[RDX_SHUF5]]
+; CHECK-NEXT:    [[RDX_SHUF7:%.*]] = shufflevector <64 x i8> [[BIN_RDX6]], <64 x i8> undef, <64 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX8:%.*]] = add <64 x i8> [[BIN_RDX6]], [[RDX_SHUF7]]
+; CHECK-NEXT:    [[RDX_SHUF9:%.*]] = shufflevector <64 x i8> [[BIN_RDX8]], <64 x i8> undef, <64 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX10:%.*]] = add <64 x i8> [[BIN_RDX8]], [[RDX_SHUF9]]
+; CHECK-NEXT:    [[RDX_SHUF11:%.*]] = shufflevector <64 x i8> [[BIN_RDX10]], <64 x i8> undef, <64 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX12:%.*]] = add <64 x i8> [[BIN_RDX10]], [[RDX_SHUF11]]
+; CHECK-NEXT:    [[RDX_SHUF13:%.*]] = shufflevector <64 x i8> [[BIN_RDX12]], <64 x i8> undef, <64 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX14:%.*]] = add <64 x i8> [[BIN_RDX12]], [[RDX_SHUF13]]
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <64 x i8> [[BIN_RDX14]], i32 0
+; CHECK-NEXT:    ret i8 [[TMP7]]
+;
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %r.010 = phi i8 [ 0, %entry ], [ %add, %for.body ]
+  %arrayidx = getelementptr inbounds [128 x i8], [128 x i8]* @bytes, i64 0, i64 %indvars.iv
+  %0 = load i8, i8* %arrayidx, align 1
+  %add = add i8 %0, %r.010
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 128
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  %add.lcssa = phi i8 [ %add, %for.body ]
+  ret i8 %add.lcssa
+}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/reg-usage-debug.ll b/llvm/test/Transforms/LoopVectorize/X86/reg-usage-debug.ll
index 8205092de..b6254a4f8 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/reg-usage-debug.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/reg-usage-debug.ll
@@ -22,7 +22,11 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 ; CHECK: LV: Checking a loop in "test_g"
-; CHECK: LV(REG): Found max usage: 2
+; CHECK: LV(REG): Found max usage: 2 item
+; CHECK-NEXT: LV(REG): RegisterClass: Generic::ScalarRC, 2 registers
+; CHECK-NEXT: LV(REG): RegisterClass: Generic::VectorRC, 2 registers
+; CHECK-NEXT: LV(REG): Found invariant usage: 1 item
+; CHECK-NEXT: LV(REG): RegisterClass: Generic::VectorRC, 2 registers
 
 define i32 @test_g(i32* nocapture readonly %a, i32 %n) local_unnamed_addr !dbg !6 {
 entry:
@@ -60,7 +64,11 @@ for.end:                                          ; preds = %for.end.loopexit, %
 }
 
 ; CHECK: LV: Checking a loop in "test"
-; CHECK: LV(REG): Found max usage: 2
+; CHECK: LV(REG): Found max usage: 2 item
+; CHECK-NEXT: LV(REG): RegisterClass: Generic::ScalarRC, 2 registers
+; CHECK-NEXT: LV(REG): RegisterClass: Generic::VectorRC, 2 registers
+; CHECK-NEXT: LV(REG): Found invariant usage: 1 item
+; CHECK-NEXT: LV(REG): RegisterClass: Generic::VectorRC, 2 registers
 
 define i32 @test(i32* nocapture readonly %a, i32 %n) local_unnamed_addr {
 entry:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/reg-usage.ll b/llvm/test/Transforms/LoopVectorize/X86/reg-usage.ll
index 9b276aa2b..cae9360e0 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/reg-usage.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/reg-usage.ll
@@ -11,9 +11,15 @@ define i32 @foo() {
 ;
 ; CHECK-LABEL: foo
 ; CHECK:      LV(REG): VF = 8
-; CHECK-NEXT: LV(REG): Found max usage: 7
+; CHECK-NEXT: LV(REG): Found max usage: 2 item
+; CHECK-NEXT: LV(REG): RegisterClass: Generic::ScalarRC, 2 registers
+; CHECK-NEXT: LV(REG): RegisterClass: Generic::VectorRC, 7 registers
+; CHECK-NEXT: LV(REG): Found invariant usage: 0 item
 ; CHECK:      LV(REG): VF = 16
-; CHECK-NEXT: LV(REG): Found max usage: 13
+; CHECK-NEXT: LV(REG): Found max usage: 2 item
+; CHECK-NEXT: LV(REG): RegisterClass: Generic::ScalarRC, 2 registers
+; CHECK-NEXT: LV(REG): RegisterClass: Generic::VectorRC, 13 registers
+; CHECK-NEXT: LV(REG): Found invariant usage: 0 item
 
 entry:
   br label %for.body
@@ -47,9 +53,15 @@ define i32 @goo() {
 ; available vector register number.
 ; CHECK-LABEL: goo
 ; CHECK:      LV(REG): VF = 8
-; CHECK-NEXT: LV(REG): Found max usage: 7
+; CHECK-NEXT: LV(REG): Found max usage: 2 item
+; CHECK-NEXT: LV(REG): RegisterClass: Generic::ScalarRC, 2 registers
+; CHECK-NEXT: LV(REG): RegisterClass: Generic::VectorRC, 7 registers
+; CHECK-NEXT: LV(REG): Found invariant usage: 0 item
 ; CHECK:      LV(REG): VF = 16
-; CHECK-NEXT: LV(REG): Found max usage: 13
+; CHECK-NEXT: LV(REG): Found max usage: 2 item
+; CHECK-NEXT: LV(REG): RegisterClass: Generic::ScalarRC, 2 registers
+; CHECK-NEXT: LV(REG): RegisterClass: Generic::VectorRC, 13 registers
+; CHECK-NEXT: LV(REG): Found invariant usage: 0 item
 entry:
   br label %for.body
 
@@ -81,8 +93,11 @@ for.body:                                         ; preds = %for.body, %entry
 define i64 @bar(i64* nocapture %a) {
 ; CHECK-LABEL: bar
 ; CHECK:       LV(REG): VF = 2
-; CHECK:       LV(REG): Found max usage: 3
-;
+; CHECK-NEXT: LV(REG): Found max usage: 2 item
+; CHECK-NEXT: LV(REG): RegisterClass: Generic::VectorRC, 3 registers
+; CHECK-NEXT: LV(REG): RegisterClass: Generic::ScalarRC, 1 registers
+; CHECK-NEXT: LV(REG): Found invariant usage: 0 item
+
 entry:
   br label %for.body
 
@@ -113,8 +128,11 @@ define void @hoo(i32 %n) {
 ; so the max usage of AVX512 vector register will be 2.
 ; AVX512F-LABEL: bar
 ; AVX512F:       LV(REG): VF = 16
-; AVX512F:       LV(REG): Found max usage: 2
-;
+; AVX512F-CHECK: LV(REG): Found max usage: 2 item
+; AVX512F-CHECK: LV(REG): RegisterClass: Generic::ScalarRC, 2 registers
+; AVX512F-CHECK: LV(REG): RegisterClass: Generic::VectorRC, 2 registers
+; AVX512F-CHECK: LV(REG): Found invariant usage: 0 item
+
 entry:
   br label %for.body
 
diff --git a/llvm/test/Transforms/LoopVectorize/X86/scatter_crash.ll b/llvm/test/Transforms/LoopVectorize/X86/scatter_crash.ll
index aff372b56..42828ce47 100755
--- a/llvm/test/Transforms/LoopVectorize/X86/scatter_crash.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/scatter_crash.ll
@@ -111,4 +111,4 @@ for.body:                                         ; preds = %for.body.preheader,
   br i1 %cmp, label %for.body, label %for.cond.cleanup.loopexit99
 }
 
-attributes #0 = { norecurse nounwind ssp uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="knl" "target-features"="+adx,+aes,+avx,+avx2,+avx512cd,+avx512er,+avx512f,+avx512pf,+bmi,+bmi2,+cx16,+f16c,+fma,+fsgsbase,+fxsr,+lzcnt,+mmx,+movbe,+pclmul,+popcnt,+prefetchwt1,+rdrnd,+rdseed,+rtm,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87,+xsave,+xsaveopt" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { norecurse nounwind ssp uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="knl" "target-features"="+adx,+aes,+avx,+avx2,+avx512cd,+avx512er,+avx512f,+avx512pf,+bmi,+bmi2,+cx16,+f16c,+fma,+fsgsbase,+fxsr,+lzcnt,+mmx,+movbe,+pclmul,+popcnt,+prefetchwt1,+rdrnd,+rdseed,+rtm,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87,+xsave,+xsaveopt" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/Transforms/LoopVectorize/X86/tail_folding_and_assume_safety.ll b/llvm/test/Transforms/LoopVectorize/X86/tail_folding_and_assume_safety.ll
new file mode 100644
index 000000000..10477ddce
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/tail_folding_and_assume_safety.ll
@@ -0,0 +1,166 @@
+; RUN: opt -mcpu=skx -S -loop-vectorize -instcombine -force-vector-width=8 -force-vector-interleave=1 < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-linux-gnu"
+
+; Case1: With pragma predicate to force tail-folding.
+; All memory opertions are masked.
+;void fold_tail(int * restrict p, int * restrict q1, int * restrict q2, int guard) {
+;   #pragma clang loop vectorize_predicate(enable)
+;   for(int ix=0; ix < 1021; ++ix) {
+;     if (ix > guard) {
+;       p[ix] = q1[ix] + q2[ix];
+;     }
+;   }
+;}
+
+;CHECK-LABEL: @fold_tail
+;CHECK: vector.body:
+;CHECK: call <8 x i32> @llvm.masked.load
+;CHECK: call <8 x i32> @llvm.masked.load
+;CHECK: call void @llvm.masked.store
+
+; Function Attrs: nofree norecurse nounwind uwtable
+define dso_local void @fold_tail(i32* noalias nocapture %p, i32* noalias nocapture readonly %q1, i32* noalias nocapture readonly %q2,
+i32 %guard) local_unnamed_addr #0 {
+entry:
+  %0 = sext i32 %guard to i64
+  br label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.inc ]
+  %cmp1 = icmp sgt i64 %indvars.iv, %0
+  br i1 %cmp1, label %if.then, label %for.inc
+
+if.then:
+  %arrayidx = getelementptr inbounds i32, i32* %q1, i64 %indvars.iv
+  %1 = load i32, i32* %arrayidx, align 4, !tbaa !2
+  %arrayidx3 = getelementptr inbounds i32, i32* %q2, i64 %indvars.iv
+  %2 = load i32, i32* %arrayidx3, align 4, !tbaa !2
+  %add = add nsw i32 %2, %1
+  %arrayidx5 = getelementptr inbounds i32, i32* %p, i64 %indvars.iv
+  store i32 %add, i32* %arrayidx5, align 4, !tbaa !2
+  br label %for.inc
+
+for.inc:
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 1021
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body, !llvm.loop !8
+}
+
+; Case2: With pragma assume_safety only the store is masked.
+; void assume_safety(int * p, int * q1, int * q2, int guard) {
+;   #pragma clang loop vectorize(assume_safety)
+;   for(int ix=0; ix < 1021; ++ix) {
+;     if (ix > guard) {
+;       p[ix] = q1[ix] + q2[ix];
+;     }
+;   }
+;}
+
+;CHECK-LABEL: @assume_safety
+;CHECK: vector.body:
+;CHECK-NOT: @llvm.masked.load
+;CHECK:  call void @llvm.masked.store
+
+; Function Attrs: norecurse nounwind uwtable
+define void @assume_safety(i32* nocapture, i32* nocapture readonly, i32* nocapture readonly, i32) local_unnamed_addr #0 {
+  %5 = sext i32 %3 to i64
+  br label %7
+
+; <label>:6:
+  ret void
+
+; <label>:7:
+  %8 = phi i64 [ 0, %4 ], [ %18, %17 ]
+  %9 = icmp sgt i64 %8, %5
+  br i1 %9, label %10, label %17
+
+; <label>:10:
+  %11 = getelementptr inbounds i32, i32* %1, i64 %8
+  %12 = load i32, i32* %11, align 4, !tbaa !2, !llvm.mem.parallel_loop_access !6
+  %13 = getelementptr inbounds i32, i32* %2, i64 %8
+  %14 = load i32, i32* %13, align 4, !tbaa !2, !llvm.mem.parallel_loop_access !6
+  %15 = add nsw i32 %14, %12
+  %16 = getelementptr inbounds i32, i32* %0, i64 %8
+  store i32 %15, i32* %16, align 4, !tbaa !2, !llvm.mem.parallel_loop_access !6
+  br label %17
+
+; <label>:17:
+  %18 = add nuw nsw i64 %8, 1
+  %19 = icmp eq i64 %18, 1021
+  br i1 %19, label %6, label %7, !llvm.loop !6
+}
+
+; Case3: With pragma assume_safety and pragma predicate both the store and the
+; load are masked.
+; void fold_tail_and_assume_safety(int * p, int * q1, int * q2, int guard) {
+;   #pragma clang loop vectorize(assume_safety) vectorize_predicate(enable)
+;   for(int ix=0; ix < 1021; ++ix) {
+;     if (ix > guard) {
+;       p[ix] = q1[ix] + q2[ix];
+;     }
+;   }
+;}
+
+;CHECK-LABEL: @fold_tail_and_assume_safety
+;CHECK: vector.body:
+;CHECK: call <8 x i32> @llvm.masked.load
+;CHECK: call <8 x i32> @llvm.masked.load
+;CHECK: call void @llvm.masked.store
+
+; Function Attrs: nofree norecurse nounwind uwtable
+define dso_local void @fold_tail_and_assume_safety(i32* noalias nocapture %p, i32* noalias nocapture readonly %q1, i32* noalias nocapture readonly %q2,
+i32 %guard) local_unnamed_addr #0 {
+entry:
+  %0 = sext i32 %guard to i64
+  br label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.inc ]
+  %cmp1 = icmp sgt i64 %indvars.iv, %0
+  br i1 %cmp1, label %if.then, label %for.inc
+
+if.then:
+  %arrayidx = getelementptr inbounds i32, i32* %q1, i64 %indvars.iv
+  %1 = load i32, i32* %arrayidx, align 4, !tbaa !2, !llvm.access.group !10
+  %arrayidx3 = getelementptr inbounds i32, i32* %q2, i64 %indvars.iv
+  %2 = load i32, i32* %arrayidx3, align 4, !tbaa !2, !llvm.access.group !10
+  %add = add nsw i32 %2, %1
+  %arrayidx5 = getelementptr inbounds i32, i32* %p, i64 %indvars.iv
+  store i32 %add, i32* %arrayidx5, align 4, !tbaa !2, !llvm.access.group !10
+  br label %for.inc
+
+for.inc:
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 1021
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body, !llvm.loop !11
+}
+
+attributes #0 = { norecurse nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.module.flags = !{!0}
+!llvm.ident = !{!1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{!"clang version 6.0.0-1ubuntu2 (tags/RELEASE_600/final)"}
+!2 = !{!3, !3, i64 0}
+!3 = !{!"int", !4, i64 0}
+!4 = !{!"omnipotent char", !5, i64 0}
+!5 = !{!"Simple C/C++ TBAA"}
+!6 = distinct !{!6, !7}
+!7 = !{!"llvm.loop.vectorize.enable", i1 true}
+
+!8 = distinct !{!8, !9}
+!9 = !{!"llvm.loop.vectorize.predicate.enable", i1 true}
+
+!10 = distinct !{}
+!11 = distinct !{!11, !12, !13}
+!12 = !{!"llvm.loop.parallel_accesses", !10}
+!13 = !{!"llvm.loop.vectorize.predicate.enable", i1 true}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/tail_loop_folding.ll b/llvm/test/Transforms/LoopVectorize/X86/tail_loop_folding.ll
new file mode 100644
index 000000000..7fc8d518e
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/tail_loop_folding.ll
@@ -0,0 +1,152 @@
+; RUN: opt < %s -loop-vectorize -S | FileCheck %s
+; RUN: opt < %s -loop-vectorize -prefer-predicate-over-epilog -S | FileCheck -check-prefix=PREDFLAG %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define dso_local void @tail_folding_enabled(i32* noalias nocapture %A, i32* noalias nocapture readonly %B, i32* noalias nocapture readonly %C) local_unnamed_addr #0 {
+; CHECK-LABEL: tail_folding_enabled(
+; CHECK:  vector.body:
+; CHECK:  %wide.masked.load = call <8 x i32> @llvm.masked.load.v8i32.p0v8i32(
+; CHECK:  %wide.masked.load1 = call <8 x i32> @llvm.masked.load.v8i32.p0v8i32(
+; CHECK:  %8 = add nsw <8 x i32> %wide.masked.load1, %wide.masked.load
+; CHECK:  call void @llvm.masked.store.v8i32.p0v8i32(
+; CHECK:  %index.next = add i64 %index, 8
+; CHECK:  %12 = icmp eq i64 %index.next, 432
+; CHECK:  br i1 %12, label %middle.block, label %vector.body, !llvm.loop !0
+; PREDFLAG-LABEL: tail_folding_enabled(
+; PREDFLAG:  vector.body:
+; PREDFLAG:  %wide.masked.load = call <8 x i32> @llvm.masked.load.v8i32.p0v8i32(
+; PREDFLAG:  %wide.masked.load1 = call <8 x i32> @llvm.masked.load.v8i32.p0v8i32(
+; PREDFLAG:  %8 = add nsw <8 x i32> %wide.masked.load1, %wide.masked.load
+; PREDFLAG:  call void @llvm.masked.store.v8i32.p0v8i32(
+; PREDFLAG:  %index.next = add i64 %index, 8
+; PREDFLAG:  %12 = icmp eq i64 %index.next, 432
+; PREDFLAG:  br i1 %12, label %middle.block, label %vector.body, !llvm.loop !0
+entry:
+  br label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %B, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds i32, i32* %C, i64 %indvars.iv
+  %1 = load i32, i32* %arrayidx2, align 4
+  %add = add nsw i32 %1, %0
+  %arrayidx4 = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  store i32 %add, i32* %arrayidx4, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 430
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body, !llvm.loop !6
+}
+
+define dso_local void @tail_folding_disabled(i32* noalias nocapture %A, i32* noalias nocapture readonly %B, i32* noalias nocapture readonly %C) local_unnamed_addr #0 {
+; CHECK-LABEL: tail_folding_disabled(
+; CHECK:      vector.body:
+; CHECK-NOT:  @llvm.masked.load.v8i32.p0v8i32(
+; CHECK-NOT:  @llvm.masked.store.v8i32.p0v8i32(
+; CHECK:      br i1 %44, label {{.*}}, label %vector.body
+; PREDFLAG-LABEL: tail_folding_disabled(
+; PREDFLAG:  vector.body:
+; PREDFLAG:  %wide.masked.load = call <8 x i32> @llvm.masked.load.v8i32.p0v8i32(
+; PREDFLAG:  %wide.masked.load1 = call <8 x i32> @llvm.masked.load.v8i32.p0v8i32(
+; PREDFLAG:  %8 = add nsw <8 x i32> %wide.masked.load1, %wide.masked.load
+; PREDFLAG:  call void @llvm.masked.store.v8i32.p0v8i32(
+; PREDFLAG:  %index.next = add i64 %index, 8
+; PREDFLAG:  %12 = icmp eq i64 %index.next, 432
+; PREDFLAG:  br i1 %12, label %middle.block, label %vector.body, !llvm.loop !4
+entry:
+  br label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %B, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds i32, i32* %C, i64 %indvars.iv
+  %1 = load i32, i32* %arrayidx2, align 4
+  %add = add nsw i32 %1, %0
+  %arrayidx4 = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  store i32 %add, i32* %arrayidx4, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 430
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body, !llvm.loop !10
+}
+
+; Check that fold tail under optsize passes the reduction live-out value
+; through a select.
+; int reduction_i32(int *A, int *B, int N) {
+;   int sum = 0;
+;   for (int i = 0; i < N; ++i)
+;     sum += (A[i] + B[i]);
+;   return sum;
+; }
+;
+define i32 @reduction_i32(i32* nocapture readonly %A, i32* nocapture readonly %B, i32 %N) #0 {
+; CHECK-LABEL: @reduction_i32(
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %vector.body ]
+; CHECK-NEXT:    [[ACCUM_PHI:%.*]] = phi <8 x i32> [ zeroinitializer, %vector.ph ], [ [[ACCUM:%.*]], %vector.body ]
+; CHECK:         [[ICMPULE:%.*]] = icmp ule <8 x i64>
+; CHECK:         [[LOAD1:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0v8i32(<8 x i32>* {{.*}}, i32 4, <8 x i1> [[ICMPULE]], <8 x i32> undef)
+; CHECK:         [[LOAD2:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0v8i32(<8 x i32>* {{.*}}, i32 4, <8 x i1> [[ICMPULE]], <8 x i32> undef)
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw <8 x i32> [[LOAD2]], [[LOAD1]]
+; CHECK-NEXT:    [[ACCUM]] = add <8 x i32> [[ADD]], [[ACCUM_PHI]]
+; CHECK:         [[LIVEOUT:%.*]] = select <8 x i1> [[ICMPULE]], <8 x i32> [[ACCUM]], <8 x i32> [[ACCUM_PHI]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 8
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x i32> [[LIVEOUT]], <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <8 x i32> [[LIVEOUT]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF4:%.*]] = shufflevector <8 x i32> [[BIN_RDX]], <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX5:%.*]] = add <8 x i32> [[BIN_RDX]], [[RDX_SHUF4]]
+; CHECK-NEXT:    [[RDX_SHUF6:%.*]] = shufflevector <8 x i32> [[BIN_RDX5]], <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX7:%.*]] = add <8 x i32> [[BIN_RDX5]], [[RDX_SHUF6]]
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <8 x i32> [[BIN_RDX7]], i32 0
+; CHECK-NEXT:    br i1 true, label %for.cond.cleanup, label %scalar.ph
+; CHECK:       scalar.ph:
+; CHECK:       for.cond.cleanup:
+; CHECK-NEXT:    [[SUM_1_LCSSA:%.*]] = phi i32 [ {{.*}}, %for.body ], [ [[TMP17]], %middle.block ]
+; CHECK-NEXT:    ret i32 [[SUM_1_LCSSA]]
+;
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %sum.0 = phi i32 [ %sum.1, %for.body ], [ 0, %entry ]
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %arrayidxA = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidxA, align 4
+  %arrayidxB = getelementptr inbounds i32, i32* %B, i64 %indvars.iv
+  %1 = load i32, i32* %arrayidxB, align 4
+  %add = add nsw i32 %1, %0
+  %sum.1 = add nuw nsw i32 %add, %sum.0
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %N
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret i32 %sum.1
+}
+
+; CHECK:      !0 = distinct !{!0, !1}
+; CHECK-NEXT: !1 = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK-NEXT: !2 = distinct !{!2, !3, !1}
+; CHECK-NEXT: !3 = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK-NEXT: !4 = distinct !{!4, !1}
+; CHECK-NEXT: !5 = distinct !{!5, !3, !1}
+
+attributes #0 = { nounwind optsize uwtable "target-cpu"="core-avx2" "target-features"="+avx,+avx2" }
+
+!6 = distinct !{!6, !7, !8}
+!7 = !{!"llvm.loop.vectorize.predicate.enable", i1 true}
+!8 = !{!"llvm.loop.vectorize.enable", i1 true}
+
+!10 = distinct !{!10, !11, !12}
+!11 = !{!"llvm.loop.vectorize.predicate.enable", i1 false}
+!12 = !{!"llvm.loop.vectorize.enable", i1 true}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/vect.omp.force.small-tc.ll b/llvm/test/Transforms/LoopVectorize/X86/vect.omp.force.small-tc.ll
index 7c249a1b4..1e8f1409d 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/vect.omp.force.small-tc.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/vect.omp.force.small-tc.ll
@@ -102,17 +102,17 @@ define void @vectorized1(float* noalias nocapture %A, float* noalias nocapture r
 ; CHECK-NEXT:    [[INDUCTION:%.*]] = add <8 x i64> [[BROADCAST_SPLAT]], <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>
 ; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds float, float* [[B:%.*]], i64 [[TMP0]]
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds float, float* [[TMP1]], i32 0
-; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float* [[TMP2]] to <8 x float>*
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x float>, <8 x float>* [[TMP3]], align 4
-; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, float* [[A:%.*]], i64 [[TMP0]]
-; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds float, float* [[TMP4]], i32 0
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast float* [[TMP5]] to <8 x float>*
-; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <8 x float>, <8 x float>* [[TMP6]], align 4
-; CHECK-NEXT:    [[TMP7:%.*]] = fadd fast <8 x float> [[WIDE_LOAD]], [[WIDE_LOAD1]]
-; CHECK-NEXT:    [[TMP8:%.*]] = icmp ule <8 x i64> [[INDUCTION]], <i64 19, i64 19, i64 19, i64 19, i64 19, i64 19, i64 19, i64 19>
-; CHECK-NEXT:    [[TMP9:%.*]] = bitcast float* [[TMP5]] to <8 x float>*
-; CHECK-NEXT:    call void @llvm.masked.store.v8f32.p0v8f32(<8 x float> [[TMP7]], <8 x float>* [[TMP9]], i32 4, <8 x i1> [[TMP8]])
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ule <8 x i64> [[INDUCTION]], <i64 19, i64 19, i64 19, i64 19, i64 19, i64 19, i64 19, i64 19>
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds float, float* [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast float* [[TMP3]] to <8 x float>*
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* [[TMP4]], i32 4, <8 x i1> [[TMP2]], <8 x float> undef), !llvm.access.group !6
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds float, float* [[A:%.*]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds float, float* [[TMP5]], i32 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast float* [[TMP6]] to <8 x float>*
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD1:%.*]] = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* [[TMP7]], i32 4, <8 x i1> [[TMP2]], <8 x float> undef), !llvm.access.group !6
+; CHECK-NEXT:    [[TMP8:%.*]] = fadd fast <8 x float> [[WIDE_MASKED_LOAD]], [[WIDE_MASKED_LOAD1]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast float* [[TMP6]] to <8 x float>*
+; CHECK-NEXT:    call void @llvm.masked.store.v8f32.p0v8f32(<8 x float> [[TMP8]], <8 x float>* [[TMP9]], i32 4, <8 x i1> [[TMP2]])
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 8
 ; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 24
 ; CHECK-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !7
diff --git a/llvm/test/Transforms/LoopVectorize/X86/vectorization-remarks-profitable.ll b/llvm/test/Transforms/LoopVectorize/X86/vectorization-remarks-profitable.ll
index 228515efc..e16f8cf4e 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/vectorization-remarks-profitable.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/vectorization-remarks-profitable.ll
@@ -76,7 +76,7 @@ for.end:                                          ; preds = %for.body, %entry
   ret void, !dbg !27
 }
 
-attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!7, !8}
diff --git a/llvm/test/Transforms/LoopVectorize/dbg.value.ll b/llvm/test/Transforms/LoopVectorize/dbg.value.ll
index cc8df3019..f2c628983 100644
--- a/llvm/test/Transforms/LoopVectorize/dbg.value.ll
+++ b/llvm/test/Transforms/LoopVectorize/dbg.value.ll
@@ -42,7 +42,7 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.value(metadata, metadata, metadata) #1
 
-attributes #0 = { nounwind ssp uwtable "fp-contract-model"="standard" "no-frame-pointer-elim" "no-frame-pointer-elim-non-leaf" "relocation-model"="pic" "ssp-buffers-size"="8" }
+attributes #0 = { nounwind ssp uwtable "fp-contract-model"="standard" "frame-pointer"="non-leaf" "relocation-model"="pic" "ssp-buffers-size"="8" }
 attributes #1 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!11}
diff --git a/llvm/test/Transforms/LoopVectorize/debugloc.ll b/llvm/test/Transforms/LoopVectorize/debugloc.ll
index 358f49b99..5cf59191f 100644
--- a/llvm/test/Transforms/LoopVectorize/debugloc.ll
+++ b/llvm/test/Transforms/LoopVectorize/debugloc.ll
@@ -60,7 +60,7 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.value(metadata, metadata, metadata) #1
 
-attributes #0 = { nounwind readonly ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf"="true" "no-infs-fp-math"="true" "no-nans-fp-math"="true" "unsafe-fp-math"="true" "use-soft-float"="false" }
+attributes #0 = { nounwind readonly ssp uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="true" "no-nans-fp-math"="true" "unsafe-fp-math"="true" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/Transforms/LoopVectorize/diag-missing-instr-debug-loc.ll b/llvm/test/Transforms/LoopVectorize/diag-missing-instr-debug-loc.ll
index bf4eb0876..94ff8e525 100644
--- a/llvm/test/Transforms/LoopVectorize/diag-missing-instr-debug-loc.ll
+++ b/llvm/test/Transforms/LoopVectorize/diag-missing-instr-debug-loc.ll
@@ -50,7 +50,7 @@ for.body:                                         ; preds = %for.body.preheader,
   br i1 %exitcond, label %for.cond.cleanup, label %for.body, !dbg !9, !llvm.loop !18
 }
 
-attributes #0 = { norecurse nounwind ssp uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="core2" "target-features"="+cx16,+fxsr,+mmx,+sse,+sse2,+sse3,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { norecurse nounwind ssp uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="core2" "target-features"="+cx16,+fxsr,+mmx,+sse,+sse2,+sse3,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!3, !4}
diff --git a/llvm/test/Transforms/LoopVectorize/diag-with-hotness-info-2.ll b/llvm/test/Transforms/LoopVectorize/diag-with-hotness-info-2.ll
index 34641849d..834cefa70 100644
--- a/llvm/test/Transforms/LoopVectorize/diag-with-hotness-info-2.ll
+++ b/llvm/test/Transforms/LoopVectorize/diag-with-hotness-info-2.ll
@@ -132,7 +132,7 @@ for.cond.cleanup:
   ret void, !dbg !44
 }
 
-attributes #0 = { norecurse nounwind ssp uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="core2" "target-features"="+cx16,+fxsr,+mmx,+sse,+sse2,+sse3,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { norecurse nounwind ssp uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="core2" "target-features"="+cx16,+fxsr,+mmx,+sse,+sse2,+sse3,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!3, !4, !5}
diff --git a/llvm/test/Transforms/LoopVectorize/diag-with-hotness-info.ll b/llvm/test/Transforms/LoopVectorize/diag-with-hotness-info.ll
index 3076d3194..9e3712148 100644
--- a/llvm/test/Transforms/LoopVectorize/diag-with-hotness-info.ll
+++ b/llvm/test/Transforms/LoopVectorize/diag-with-hotness-info.ll
@@ -145,7 +145,7 @@ for.body:                                         ; preds = %entry, %for.body
   br i1 %exitcond, label %for.cond.cleanup, label %for.body, !dbg !43, !llvm.loop !55
 }
 
-attributes #0 = { norecurse nounwind ssp uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="core2" "target-features"="+cx16,+fxsr,+mmx,+sse,+sse2,+sse3,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { norecurse nounwind ssp uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="core2" "target-features"="+cx16,+fxsr,+mmx,+sse,+sse2,+sse3,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!3, !4, !5}
diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-complex.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-complex.ll
new file mode 100644
index 000000000..aa913172f
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-complex.ll
@@ -0,0 +1,275 @@
+; RUN: opt -loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S %s | FileCheck %s
+
+
+@p = external local_unnamed_addr global [257 x i32], align 16
+@q = external local_unnamed_addr global [257 x i32], align 16
+
+; Test case for PR43398.
+
+define void @can_sink_after_store(i32 %x, i32* %ptr, i64 %tc) local_unnamed_addr #0 {
+; CHECK-LABEL: vector.ph:
+; CHECK:        %broadcast.splatinsert1 = insertelement <4 x i32> undef, i32 %x, i32 0
+; CHECK-NEXT:   %broadcast.splat2 = shufflevector <4 x i32> %broadcast.splatinsert1, <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:   %vector.recur.init = insertelement <4 x i32> undef, i32 %.pre, i32 3
+; CHECK-NEXT:    br label %vector.body
+
+; CHECK-LABEL: vector.body:
+; CHECK-NEXT:   %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+; CHECK-NEXT:   %vector.recur = phi <4 x i32> [ %vector.recur.init, %vector.ph ], [ %wide.load, %vector.body ]
+; CHECK-NEXT:   %offset.idx = add i64 1, %index
+; CHECK-NEXT:   %broadcast.splatinsert = insertelement <4 x i64> undef, i64 %offset.idx, i32 0
+; CHECK-NEXT:   %broadcast.splat = shufflevector <4 x i64> %broadcast.splatinsert, <4 x i64> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:   %induction = add <4 x i64> %broadcast.splat, <i64 0, i64 1, i64 2, i64 3>
+; CHECK-NEXT:   %0 = add i64 %offset.idx, 0
+; CHECK-NEXT:   %1 = getelementptr inbounds [257 x i32], [257 x i32]* @p, i64 0, i64 %0
+; CHECK-NEXT:   %2 = getelementptr inbounds i32, i32* %1, i32 0
+; CHECK-NEXT:   %3 = bitcast i32* %2 to <4 x i32>*
+; CHECK-NEXT:   %wide.load = load <4 x i32>, <4 x i32>* %3, align 4
+; CHECK-NEXT:   %4 = shufflevector <4 x i32> %vector.recur, <4 x i32> %wide.load, <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; CHECK-NEXT:   %5 = add <4 x i32> %4, %broadcast.splat2
+; CHECK-NEXT:   %6 = add <4 x i32> %5, %wide.load
+; CHECK-NEXT:   %7 = getelementptr inbounds [257 x i32], [257 x i32]* @q, i64 0, i64 %0
+; CHECK-NEXT:   %8 = getelementptr inbounds i32, i32* %7, i32 0
+; CHECK-NEXT:   %9 = bitcast i32* %8 to <4 x i32>*
+; CHECK-NEXT:   store <4 x i32> %6, <4 x i32>* %9, align 4
+; CHECK-NEXT:   %index.next = add i64 %index, 4
+; CHECK-NEXT:   %10 = icmp eq i64 %index.next, 1996
+; CHECK-NEXT:   br i1 %10, label %middle.block, label %vector.body
+;
+entry:
+  br label %preheader
+
+preheader:
+  %idx.phi.trans = getelementptr inbounds [257 x i32], [257 x i32]* @p, i64 0, i64 1
+  %.pre = load i32, i32* %idx.phi.trans, align 4
+  br label %for
+
+for:
+  %pre.phi = phi i32 [ %.pre, %preheader ], [ %pre.next, %for ]
+  %iv = phi i64 [ 1, %preheader ], [ %iv.next, %for ]
+  %add.1 = add i32 %pre.phi, %x
+  %idx.1 = getelementptr inbounds [257 x i32], [257 x i32]* @p, i64 0, i64 %iv
+  %pre.next = load i32, i32* %idx.1, align 4
+  %add.2 = add i32 %add.1, %pre.next
+  %idx.2 = getelementptr inbounds [257 x i32], [257 x i32]* @q, i64 0, i64 %iv
+  store i32 %add.2, i32* %idx.2, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 2000
+  br i1 %exitcond, label %exit, label %for
+
+exit:
+  ret void
+}
+
+; We can sink potential trapping instructions, as this will only delay the trap
+; and not introduce traps on additional paths.
+define void @sink_sdiv(i32 %x, i32* %ptr, i64 %tc) local_unnamed_addr #0 {
+; CHECK-LABEL: vector.ph:
+; CHECK:        %broadcast.splatinsert1 = insertelement <4 x i32> undef, i32 %x, i32 0
+; CHECK-NEXT:   %broadcast.splat2 = shufflevector <4 x i32> %broadcast.splatinsert1, <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:   %vector.recur.init = insertelement <4 x i32> undef, i32 %.pre, i32 3
+; CHECK-NEXT:    br label %vector.body
+
+; CHECK-LABEL: vector.body:
+; CHECK-NEXT:   %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+; CHECK-NEXT:   %vector.recur = phi <4 x i32> [ %vector.recur.init, %vector.ph ], [ %wide.load, %vector.body ]
+; CHECK-NEXT:   %offset.idx = add i64 1, %index
+; CHECK-NEXT:   %broadcast.splatinsert = insertelement <4 x i64> undef, i64 %offset.idx, i32 0
+; CHECK-NEXT:   %broadcast.splat = shufflevector <4 x i64> %broadcast.splatinsert, <4 x i64> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:   %induction = add <4 x i64> %broadcast.splat, <i64 0, i64 1, i64 2, i64 3>
+; CHECK-NEXT:   %0 = add i64 %offset.idx, 0
+; CHECK-NEXT:   %1 = getelementptr inbounds [257 x i32], [257 x i32]* @p, i64 0, i64 %0
+; CHECK-NEXT:   %2 = getelementptr inbounds i32, i32* %1, i32 0
+; CHECK-NEXT:   %3 = bitcast i32* %2 to <4 x i32>*
+; CHECK-NEXT:   %wide.load = load <4 x i32>, <4 x i32>* %3, align 4
+; CHECK-NEXT:   %4 = shufflevector <4 x i32> %vector.recur, <4 x i32> %wide.load, <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; CHECK-NEXT:   %5 = sdiv <4 x i32> %4, %broadcast.splat2
+; CHECK-NEXT:   %6 = add <4 x i32> %5, %wide.load
+; CHECK-NEXT:   %7 = getelementptr inbounds [257 x i32], [257 x i32]* @q, i64 0, i64 %0
+; CHECK-NEXT:   %8 = getelementptr inbounds i32, i32* %7, i32 0
+; CHECK-NEXT:   %9 = bitcast i32* %8 to <4 x i32>*
+; CHECK-NEXT:   store <4 x i32> %6, <4 x i32>* %9, align 4
+; CHECK-NEXT:   %index.next = add i64 %index, 4
+; CHECK-NEXT:   %10 = icmp eq i64 %index.next, 1996
+; CHECK-NEXT:   br i1 %10, label %middle.block, label %vector.body
+;
+entry:
+  br label %preheader
+
+preheader:
+  %idx.phi.trans = getelementptr inbounds [257 x i32], [257 x i32]* @p, i64 0, i64 1
+  %.pre = load i32, i32* %idx.phi.trans, align 4
+  br label %for
+
+for:
+  %pre.phi = phi i32 [ %.pre, %preheader ], [ %pre.next, %for ]
+  %iv = phi i64 [ 1, %preheader ], [ %iv.next, %for ]
+  %div.1 = sdiv i32 %pre.phi, %x
+  %idx.1 = getelementptr inbounds [257 x i32], [257 x i32]* @p, i64 0, i64 %iv
+  %pre.next = load i32, i32* %idx.1, align 4
+  %add.2 = add i32 %div.1, %pre.next
+  %idx.2 = getelementptr inbounds [257 x i32], [257 x i32]* @q, i64 0, i64 %iv
+  store i32 %add.2, i32* %idx.2, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 2000
+  br i1 %exitcond, label %exit, label %for
+
+exit:
+  ret void
+}
+
+; FIXME: Currently we can only sink a single instruction. For the example below,
+;        we also have to sink users.
+define void @cannot_sink_with_additional_user(i32 %x, i32* %ptr, i64 %tc) {
+; CHECK-LABEL: define void @cannot_sink_with_additional_user(
+; CHECK-NEXT: entry:
+; CHECK-NEXT:   br label %preheader
+
+; CHECK-LABEL: preheader:                                        ; preds = %entry
+; CHECK:  br label %for
+
+; CHECK-LABEL: for:                                              ; preds = %for, %preheader
+; CHECK  br i1 %exitcond, label %exit, label %for
+
+; CHECK-LABEL: exit:
+; CHECK-NEXT:    ret void
+
+entry:
+  br label %preheader
+
+preheader:
+  %idx.phi.trans = getelementptr inbounds [257 x i32], [257 x i32]* @p, i64 0, i64 1
+  %.pre = load i32, i32* %idx.phi.trans, align 4
+  br label %for
+
+for:
+  %pre.phi = phi i32 [ %.pre, %preheader ], [ %pre.next, %for ]
+  %iv = phi i64 [ 1, %preheader ], [ %iv.next, %for ]
+  %add.1 = add i32 %pre.phi, %x
+  %add.2 = add i32 %add.1, %x
+  %idx.1 = getelementptr inbounds [257 x i32], [257 x i32]* @p, i64 0, i64 %iv
+  %pre.next = load i32, i32* %idx.1, align 4
+  %add.3 = add i32 %add.1, %pre.next
+  %add.4 = add i32 %add.2, %add.3
+  %idx.2 = getelementptr inbounds [257 x i32], [257 x i32]* @q, i64 0, i64 %iv
+  store i32 %add.4, i32* %idx.2, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 2000
+  br i1 %exitcond, label %exit, label %for
+
+exit:
+  ret void
+}
+
+; FIXME: We can sink a store, if we can guarantee that it does not alias any
+;        loads/stores in between.
+define void @cannot_sink_store(i32 %x, i32* %ptr, i64 %tc) {
+; CHECK-LABEL: define void @cannot_sink_store(
+; CHECK-NEXT: entry:
+; CHECK-NEXT:   br label %preheader
+
+; CHECK-LABEL: preheader:                                        ; preds = %entry
+; CHECK:  br label %for
+
+; CHECK-LABEL: for:                                              ; preds = %for, %preheader
+; CHECK  br i1 %exitcond, label %exit, label %for
+
+; CHECK-LABEL: exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %preheader
+
+preheader:
+  %idx.phi.trans = getelementptr inbounds [257 x i32], [257 x i32]* @p, i64 0, i64 1
+  %.pre = load i32, i32* %idx.phi.trans, align 4
+  br label %for
+
+for:
+  %pre.phi = phi i32 [ %.pre, %preheader ], [ %pre.next, %for ]
+  %iv = phi i64 [ 1, %preheader ], [ %iv.next, %for ]
+  %add.1 = add i32 %pre.phi, %x
+  store i32 %add.1, i32* %ptr
+  %idx.1 = getelementptr inbounds [257 x i32], [257 x i32]* @p, i64 0, i64 %iv
+  %pre.next = load i32, i32* %idx.1, align 4
+  %add.2 = add i32 %add.1, %pre.next
+  %idx.2 = getelementptr inbounds [257 x i32], [257 x i32]* @q, i64 0, i64 %iv
+  store i32 %add.2, i32* %idx.2, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 2000
+  br i1 %exitcond, label %exit, label %for
+
+exit:
+  ret void
+}
+
+; Some kinds of reductions are not detected by IVDescriptors. If we have a
+; cycle, we cannot sink it.
+define void @cannot_sink_reduction(i32 %x, i32* %ptr, i64 %tc) {
+; CHECK-LABEL: define void @cannot_sink_reduction(
+; CHECK-NEXT: entry:
+; CHECK-NEXT:   br label %preheader
+
+; CHECK-LABEL: preheader:                                        ; preds = %entry
+; CHECK:  br label %for
+
+; CHECK-LABEL: for:                                              ; preds = %for, %preheader
+; CHECK  br i1 %exitcond, label %exit, label %for
+
+; CHECK-LABEL: exit:                                    ; preds = %for
+; CHECK-NET:     ret void
+;
+entry:
+  br label %preheader
+
+preheader:
+  %idx.phi.trans = getelementptr inbounds [257 x i32], [257 x i32]* @p, i64 0, i64 1
+  %.pre = load i32, i32* %idx.phi.trans, align 4
+  br label %for
+
+for:
+  %pre.phi = phi i32 [ %.pre, %preheader ], [ %d, %for ]
+  %iv = phi i64 [ 1, %preheader ], [ %iv.next, %for ]
+  %d = sdiv i32 %pre.phi, %x
+  %idx.1 = getelementptr inbounds [257 x i32], [257 x i32]* @p, i64 0, i64 %iv
+  %pre.next = load i32, i32* %idx.1, align 4
+  %add.2 = add i32 %x, %pre.next
+  %idx.2 = getelementptr inbounds [257 x i32], [257 x i32]* @q, i64 0, i64 %iv
+  store i32 %add.2, i32* %idx.2, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 2000
+  br i1 %exitcond, label %exit, label %for
+
+exit:
+  ret void
+}
+
+; TODO: We should be able to sink %tmp38 after %tmp60.
+define void @instruction_with_2_FOR_operands() {
+; CHECK-LABEL: define void @instruction_with_2_FOR_operands(
+; CHECK-NEXT: bb:
+; CHECK-NEXT:   br label %bb13
+
+; CHECK-LABEL: bb13:
+; CHECK:         br i1 %tmp12, label %bb13, label %bb74
+
+; CHECK-LABEL: bb74:
+; CHECK-NEXT:    ret void
+;
+bb:
+  br label %bb13
+
+bb13:                                             ; preds = %bb13, %bb
+  %tmp37 = phi float [ %tmp60, %bb13 ], [ undef, %bb ]
+  %tmp27 = phi float [ %tmp49, %bb13 ], [ undef, %bb ]
+  %indvars.iv = phi i64 [ %indvars.iv.next, %bb13 ], [ 0, %bb ]
+  %tmp38 = fmul fast float %tmp37, %tmp27
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %tmp49 = load float, float* undef, align 4
+  %tmp60 = load float, float* undef, align 4
+  %tmp12 = icmp slt i64 %indvars.iv, undef
+  br i1 %tmp12, label %bb13, label %bb74
+
+bb74:                                             ; preds = %bb13
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-multiply-recurrences.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-multiply-recurrences.ll
new file mode 100644
index 000000000..5027362c7
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-multiply-recurrences.ll
@@ -0,0 +1,45 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN:opt -loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S %s | FileCheck %s
+
+; For %for.1, we are fine initially, because the previous value %for.1.next dominates the
+; user of %for.1. But for %for.2, we have to sink the user (%for.1.next) past the previous
+; value %for.2.next. This however breaks the condition we have for %for.1. We cannot fix
+; both first order recurrences and cannot vectorize the loop.
+define i32 @c(i32 %N) {
+; CHECK-LABEL: @c(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 10, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[FOR_1:%.*]] = phi i32 [ [[FOR_1_NEXT:%.*]], [[FOR_BODY]] ], [ 20, [[ENTRY]] ]
+; CHECK-NEXT:    [[FOR_2:%.*]] = phi i32 [ [[FOR_2_NEXT:%.*]], [[FOR_BODY]] ], [ 11, [[ENTRY]] ]
+; CHECK-NEXT:    [[FOR_1_NEXT]] = add nsw i32 [[FOR_2]], 1
+; CHECK-NEXT:    [[FOR_2_NEXT]] = shl i32 [[FOR_1]], 24
+; CHECK-NEXT:    [[INC]] = add nsw i32 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND1_FOR_END_CRIT_EDGE:%.*]], label [[FOR_BODY]]
+; CHECK:       for.cond1.for.end_crit_edge:
+; CHECK-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[FOR_1_NEXT]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[SEXT_LCSSA:%.*]] = phi i32 [ [[FOR_2_NEXT]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[RES:%.*]] = add i32 [[ADD_LCSSA]], [[SEXT_LCSSA]]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %iv  = phi i32 [ %inc, %for.body ], [ 10, %entry ]
+  %for.1 = phi i32 [ %for.1.next, %for.body ], [ 20, %entry ]
+  %for.2 = phi i32 [ %for.2.next, %for.body ], [ 11, %entry ]
+  %for.1.next = add nsw i32 %for.2, 1
+  %for.2.next = shl i32 %for.1, 24
+  %inc = add nsw i32 %iv, 1
+  %exitcond = icmp eq i32 %inc, %N
+  br i1 %exitcond, label %for.cond1.for.end_crit_edge, label %for.body
+
+for.cond1.for.end_crit_edge:                      ; preds = %for.body
+  %add.lcssa = phi i32 [ %for.1.next, %for.body ]
+  %sext.lcssa = phi i32 [ %for.2.next, %for.body ]
+  %res = add i32 %add.lcssa, %sext.lcssa
+  ret i32 %res
+}
diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll
index 998f41267..5a250984b 100644
--- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll
+++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll
@@ -572,3 +572,76 @@ for.body:
 for.end:
   ret void
 }
+
+; Do not sink branches: While branches are if-converted and do not require
+; sinking, instructions with side effects (e.g. loads) conditioned by those
+; branches will become users of the condition bit after vectorization and would
+; need to be sunk if the loop is vectorized.
+define void @do_not_sink_branch(i32 %x, i32* %in, i32* %out, i32 %tc) local_unnamed_addr #0 {
+; NO-SINK-AFTER-LABEL: do_not_sink_branch
+; NO-SINK-AFTER-NOT:   vector.ph:
+; NO-SINK-AFTER:       }
+entry:
+  %cmp530 = icmp slt i32 0, %tc
+  br label %for.body4
+
+for.body4:                                        ; preds = %cond.end, %entry
+  %indvars.iv = phi i32 [ 0, %entry ], [ %indvars.iv.next, %cond.end ]
+  %cmp534 = phi i1 [ %cmp530, %entry ], [ %cmp5, %cond.end ]
+  br i1 %cmp534, label %cond.true, label %cond.end
+
+cond.true:                                        ; preds = %for.body4
+  %arrayidx7 = getelementptr inbounds i32, i32* %in, i32 %indvars.iv
+  %in.val = load i32, i32* %arrayidx7, align 4
+  br label %cond.end
+
+cond.end:                                         ; preds = %for.body4, %cond.true
+  %cond = phi i32 [ %in.val, %cond.true ], [ 0, %for.body4 ]
+  %arrayidx8 = getelementptr inbounds i32, i32* %out, i32 %indvars.iv
+  store i32 %cond, i32* %arrayidx8, align 4
+  %indvars.iv.next = add nuw nsw i32 %indvars.iv, 1
+  %cmp5 = icmp slt i32 %indvars.iv.next, %tc
+  %exitcond = icmp eq i32 %indvars.iv.next, %x
+  br i1 %exitcond, label %for.end12.loopexit, label %for.body4
+
+for.end12.loopexit:                               ; preds = %cond.end
+  ret void
+}
+
+; Dead instructions, like the exit condition are not part of the actual VPlan
+; and do not need to be sunk. PR44634.
+define void @sink_dead_inst() {
+; SINK-AFTER-LABEL: define void @sink_dead_inst(
+; SINK-AFTER-LABEL: vector.body:                                      ; preds = %vector.body, %vector.ph
+; SINK-AFTER-NEXT:    %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+; SINK-AFTER-NEXT:    %vec.ind = phi <4 x i16> [ <i16 -27, i16 -26, i16 -25, i16 -24>, %vector.ph ], [ %vec.ind.next, %vector.body ]
+; SINK-AFTER-NEXT:    %vector.recur = phi <4 x i16> [ <i16 undef, i16 undef, i16 undef, i16 0>, %vector.ph ], [ %3, %vector.body ]
+; SINK-AFTER-NEXT:    %vector.recur2 = phi <4 x i32> [ <i32 undef, i32 undef, i32 undef, i32 -27>, %vector.ph ], [ %1, %vector.body ]
+; SINK-AFTER-NEXT:    %0 = add <4 x i16> %vec.ind, <i16 1, i16 1, i16 1, i16 1>
+; SINK-AFTER-NEXT:    %1 = zext <4 x i16> %0 to <4 x i32>
+; SINK-AFTER-NEXT:    %2 = shufflevector <4 x i32> %vector.recur2, <4 x i32> %1, <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; SINK-AFTER-NEXT:    %3 = add <4 x i16> %0, <i16 5, i16 5, i16 5, i16 5>
+; SINK-AFTER-NEXT:    %4 = shufflevector <4 x i16> %vector.recur, <4 x i16> %3, <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; SINK-AFTER-NEXT:    %5 = sub <4 x i16> %4, <i16 10, i16 10, i16 10, i16 10>
+; SINK-AFTER-NEXT:    %index.next = add i32 %index, 4
+; SINK-AFTER-NEXT:    %vec.ind.next = add <4 x i16> %vec.ind, <i16 4, i16 4, i16 4, i16 4>
+; SINK-AFTER-NEXT:    %6 = icmp eq i32 %index.next, 40
+; SINK-AFTER-NEXT:    br i1 %6, label %middle.block, label %vector.body, !llvm.loop !43
+;
+entry:
+  br label %for.cond
+
+for.cond:
+  %iv = phi i16 [ -27, %entry ], [ %iv.next, %for.cond ]
+  %rec.1 = phi i16 [ 0, %entry ], [ %rec.1.prev, %for.cond ]
+  %rec.2 = phi i32 [ -27, %entry ], [ %rec.2.prev, %for.cond ]
+  %use.rec.1 = sub i16 %rec.1, 10
+  %cmp = icmp eq i32 %rec.2, 15
+  %iv.next = add i16 %iv, 1
+  %rec.2.prev = zext i16 %iv.next to i32
+  %rec.1.prev = add i16 %iv.next, 5
+  br i1 %cmp, label %for.end, label %for.cond
+
+for.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/hoist-loads.ll b/llvm/test/Transforms/LoopVectorize/hoist-loads.ll
index db4774d8b..7e110d8de 100644
--- a/llvm/test/Transforms/LoopVectorize/hoist-loads.ll
+++ b/llvm/test/Transforms/LoopVectorize/hoist-loads.ll
@@ -42,12 +42,12 @@ for.end:
 ; CHECK: load <2 x float>
 ; CHECK-NOT: load <2 x float>
 
-define void @dont_hoist_cond_load() {
+define void @dont_hoist_cond_load([1024 x float]* %a) {
 entry:
   br label %for.body
 for.body:
   %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %if.end9 ]
-  %arrayidx = getelementptr inbounds [1024 x float], [1024 x float]* @A, i64 0, i64 %indvars.iv
+  %arrayidx = getelementptr inbounds [1024 x float], [1024 x float]* %a, i64 0, i64 %indvars.iv
   %arrayidx2 = getelementptr inbounds [1024 x float], [1024 x float]* @B, i64 0, i64 %indvars.iv
   %0 = load float, float* %arrayidx2, align 4
   %cmp3 = fcmp oeq float %0, 0.000000e+00
diff --git a/llvm/test/Transforms/LoopVectorize/if-conversion-nest.ll b/llvm/test/Transforms/LoopVectorize/if-conversion-nest.ll
index f254bc81a..c1b29f80c 100644
--- a/llvm/test/Transforms/LoopVectorize/if-conversion-nest.ll
+++ b/llvm/test/Transforms/LoopVectorize/if-conversion-nest.ll
@@ -12,7 +12,7 @@ define i32 @foo(i32* nocapture %A, i32* nocapture %B, i32 %n) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[N]], -1
 ; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[TMP0]] to i64
 ; CHECK-NEXT:    [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 4
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP0]], 3
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
 ; CHECK:       vector.memcheck:
 ; CHECK-NEXT:    [[TMP3:%.*]] = add i32 [[N]], -1
@@ -22,33 +22,33 @@ define i32 @foo(i32* nocapture %A, i32* nocapture %B, i32 %n) {
 ; CHECK-NEXT:    [[SCEVGEP4:%.*]] = getelementptr i32, i32* [[B:%.*]], i64 [[TMP5]]
 ; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ugt i32* [[SCEVGEP4]], [[A]]
 ; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ugt i32* [[SCEVGEP]], [[B]]
-; CHECK-NEXT:    [[MEMCHECK_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
-; CHECK-NEXT:    br i1 [[MEMCHECK_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
+; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT:    br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[TMP2]], 8589934588
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i32* [[TMP7]] to <4 x i32>*
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, <4 x i32>* [[TMP8]], align 4, !alias.scope !0, !noalias !3
-; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i32* [[TMP9]] to <4 x i32>*
-; CHECK-NEXT:    [[WIDE_LOAD6:%.*]] = load <4 x i32>, <4 x i32>* [[TMP10]], align 4, !alias.scope !3
-; CHECK-NEXT:    [[TMP11:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD6]]
-; CHECK-NEXT:    [[TMP12:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD]], <i32 19, i32 19, i32 19, i32 19>
-; CHECK-NEXT:    [[TMP13:%.*]] = icmp slt <4 x i32> [[WIDE_LOAD6]], <i32 4, i32 4, i32 4, i32 4>
-; CHECK-NEXT:    [[TMP14:%.*]] = select <4 x i1> [[TMP13]], <4 x i32> <i32 4, i32 4, i32 4, i32 4>, <4 x i32> <i32 5, i32 5, i32 5, i32 5>
-; CHECK-NEXT:    [[TMP15:%.*]] = and <4 x i1> [[TMP12]], [[TMP11]]
-; CHECK-NEXT:    [[TMP16:%.*]] = xor <4 x i1> [[TMP12]], <i1 true, i1 true, i1 true, i1 true>
-; CHECK-NEXT:    [[TMP17:%.*]] = and <4 x i1> [[TMP11]], [[TMP16]]
-; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP15]], <4 x i32> <i32 3, i32 3, i32 3, i32 3>, <4 x i32> <i32 9, i32 9, i32 9, i32 9>
-; CHECK-NEXT:    [[PREDPHI7:%.*]] = select <4 x i1> [[TMP17]], <4 x i32> [[TMP14]], <4 x i32> [[PREDPHI]]
-; CHECK-NEXT:    [[TMP18:%.*]] = bitcast i32* [[TMP7]] to <4 x i32>*
-; CHECK-NEXT:    store <4 x i32> [[PREDPHI7]], <4 x i32>* [[TMP18]], align 4, !alias.scope !0, !noalias !3
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i32* [[TMP6]] to <4 x i32>*
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, <4 x i32>* [[TMP7]], align 4, !alias.scope !0, !noalias !3
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i32* [[TMP8]] to <4 x i32>*
+; CHECK-NEXT:    [[WIDE_LOAD6:%.*]] = load <4 x i32>, <4 x i32>* [[TMP9]], align 4, !alias.scope !3
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD6]]
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD]], <i32 19, i32 19, i32 19, i32 19>
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp slt <4 x i32> [[WIDE_LOAD6]], <i32 4, i32 4, i32 4, i32 4>
+; CHECK-NEXT:    [[TMP13:%.*]] = select <4 x i1> [[TMP12]], <4 x i32> <i32 4, i32 4, i32 4, i32 4>, <4 x i32> <i32 5, i32 5, i32 5, i32 5>
+; CHECK-NEXT:    [[TMP14:%.*]] = and <4 x i1> [[TMP11]], [[TMP10]]
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <4 x i1> [[TMP11]], <i1 true, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    [[TMP16:%.*]] = and <4 x i1> [[TMP10]], [[TMP15]]
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP14]], <4 x i32> <i32 3, i32 3, i32 3, i32 3>, <4 x i32> <i32 9, i32 9, i32 9, i32 9>
+; CHECK-NEXT:    [[PREDPHI7:%.*]] = select <4 x i1> [[TMP16]], <4 x i32> [[TMP13]], <4 x i32> [[PREDPHI]]
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast i32* [[TMP6]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[PREDPHI7]], <4 x i32>* [[TMP17]], align 4, !alias.scope !0, !noalias !3
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
-; CHECK-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !5
+; CHECK-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !5
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]
@@ -58,16 +58,16 @@ define i32 @foo(i32* nocapture %A, i32* nocapture %B, i32 %n) {
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[IF_END14:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV]]
-; CHECK-NEXT:    [[TMP20:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[TMP19:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV]]
-; CHECK-NEXT:    [[TMP21:%.*]] = load i32, i32* [[ARRAYIDX2]], align 4
-; CHECK-NEXT:    [[CMP3:%.*]] = icmp sgt i32 [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[TMP20:%.*]] = load i32, i32* [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[CMP3:%.*]] = icmp sgt i32 [[TMP19]], [[TMP20]]
 ; CHECK-NEXT:    br i1 [[CMP3]], label [[IF_THEN:%.*]], label [[IF_END14]]
 ; CHECK:       if.then:
-; CHECK-NEXT:    [[CMP6:%.*]] = icmp sgt i32 [[TMP20]], 19
+; CHECK-NEXT:    [[CMP6:%.*]] = icmp sgt i32 [[TMP19]], 19
 ; CHECK-NEXT:    br i1 [[CMP6]], label [[IF_END14]], label [[IF_ELSE:%.*]]
 ; CHECK:       if.else:
-; CHECK-NEXT:    [[CMP10:%.*]] = icmp slt i32 [[TMP21]], 4
+; CHECK-NEXT:    [[CMP10:%.*]] = icmp slt i32 [[TMP20]], 4
 ; CHECK-NEXT:    [[DOT:%.*]] = select i1 [[CMP10]], i32 4, i32 5
 ; CHECK-NEXT:    br label [[IF_END14]]
 ; CHECK:       if.end14:
diff --git a/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll b/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll
index 353087f66..7cdd0b984 100644
--- a/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll
@@ -248,8 +248,8 @@ define void @bug18724(i1 %cond) {
 ; UNROLL-NOSIMPLIFY-NEXT:    store i32 2, i32* [[TMP1]], align 4
 ; UNROLL-NOSIMPLIFY-NEXT:    br label [[PRED_STORE_CONTINUE4]]
 ; UNROLL-NOSIMPLIFY:       pred.store.continue4:
-; UNROLL-NOSIMPLIFY-NEXT:    [[TMP4:%.*]] = add nsw i32 [[VEC_PHI]], 1
-; UNROLL-NOSIMPLIFY-NEXT:    [[TMP5:%.*]] = add nsw i32 [[VEC_PHI2]], 1
+; UNROLL-NOSIMPLIFY-NEXT:    [[TMP4:%.*]] = add i32 [[VEC_PHI]], 1
+; UNROLL-NOSIMPLIFY-NEXT:    [[TMP5:%.*]] = add i32 [[VEC_PHI2]], 1
 ; UNROLL-NOSIMPLIFY-NEXT:    [[PREDPHI]] = select i1 undef, i32 [[VEC_PHI]], i32 [[TMP4]]
 ; UNROLL-NOSIMPLIFY-NEXT:    [[PREDPHI5]] = select i1 undef, i32 [[VEC_PHI2]], i32 [[TMP5]]
 ; UNROLL-NOSIMPLIFY-NEXT:    [[OFFSET_IDX6:%.*]] = add i64 undef, [[INDEX]]
diff --git a/llvm/test/Transforms/LoopVectorize/incorrect-dom-info.ll b/llvm/test/Transforms/LoopVectorize/incorrect-dom-info.ll
index 798793a64..5417dfd44 100644
--- a/llvm/test/Transforms/LoopVectorize/incorrect-dom-info.ll
+++ b/llvm/test/Transforms/LoopVectorize/incorrect-dom-info.ll
@@ -135,7 +135,7 @@ thread-pre-split5:                                ; preds = %.lr.ph
   ret void
 }
 
-attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.ident = !{!0}
 
diff --git a/llvm/test/Transforms/LoopVectorize/interleaved-accesses-uniform-load.ll b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-uniform-load.ll
new file mode 100644
index 000000000..b56470cec
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-uniform-load.ll
@@ -0,0 +1,49 @@
+; RUN: opt -S -loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -enable-interleaved-mem-accesses=true < %s
+
+; Make sure the vectorizer can handle this loop: The strided load is only used
+; by the loop's exit condition, which is not vectorized, and is therefore
+; considered uniform while also forming an interleave group.
+  
+%0 = type { i32 ()*, i32 }
+
+@0 = internal unnamed_addr constant [59 x %0] [%0 zeroinitializer,
+%0 zeroinitializer, %0 zeroinitializer, %0 zeroinitializer,
+%0 zeroinitializer, %0 zeroinitializer, %0 zeroinitializer,
+%0 zeroinitializer, %0 zeroinitializer, %0 zeroinitializer,
+%0 zeroinitializer, %0 zeroinitializer, %0 zeroinitializer,
+%0 zeroinitializer, %0 zeroinitializer, %0 zeroinitializer,
+%0 zeroinitializer, %0 zeroinitializer, %0 zeroinitializer,
+%0 zeroinitializer, %0 zeroinitializer, %0 zeroinitializer,
+%0 zeroinitializer, %0 zeroinitializer, %0 zeroinitializer,
+%0 zeroinitializer, %0 zeroinitializer, %0 zeroinitializer,
+%0 zeroinitializer, %0 zeroinitializer, %0 zeroinitializer,
+%0 zeroinitializer, %0 zeroinitializer, %0 zeroinitializer,
+%0 zeroinitializer, %0 zeroinitializer, %0 zeroinitializer,
+%0 zeroinitializer, %0 zeroinitializer, %0 zeroinitializer,
+%0 zeroinitializer, %0 zeroinitializer, %0 zeroinitializer,
+%0 zeroinitializer, %0 zeroinitializer, %0 zeroinitializer,
+%0 zeroinitializer, %0 zeroinitializer, %0 zeroinitializer,
+%0 zeroinitializer, %0 zeroinitializer, %0 zeroinitializer,
+%0 {i32 ()* null, i32 258}, %0 zeroinitializer, %0 zeroinitializer,
+%0 zeroinitializer, %0 zeroinitializer, %0 zeroinitializer,
+%0 zeroinitializer], align 8
+
+define dso_local void @test_dead_load(i32 %arg) {
+; CHECK-LABEL: @test_dead_load(
+; CHECK: vector.body:
+; CHECK: %wide.vec = load <16 x i32>, <16 x i32>* %3, align 8
+; CHECK: %strided.vec = shufflevector <16 x i32> %wide.vec, <16 x i32> undef, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+bb1:
+  br label %bb2
+
+bb2:
+  %tmp = phi %0* [ %tmp6, %bb2 ], [ getelementptr inbounds ([59 x %0], [59 x %0]* @0, i64 0, i64 0), %bb1 ]
+  %tmp3 = getelementptr inbounds %0, %0* %tmp, i64 0, i32 1
+  %tmp4 = load i32, i32* %tmp3, align 8
+  %tmp5 = icmp eq i32 %tmp4, 258
+  %tmp6 = getelementptr inbounds %0, %0* %tmp, i64 1
+  br i1 %tmp5, label %bb65, label %bb2
+
+bb65:
+  unreachable
+}
diff --git a/llvm/test/Transforms/LoopVectorize/interleaved-accesses.ll b/llvm/test/Transforms/LoopVectorize/interleaved-accesses.ll
index 2d8e9afbb..5e5dc5e74 100644
--- a/llvm/test/Transforms/LoopVectorize/interleaved-accesses.ll
+++ b/llvm/test/Transforms/LoopVectorize/interleaved-accesses.ll
@@ -139,9 +139,9 @@ for.end:                                          ; preds = %for.body
 ; CHECK: shufflevector <16 x i32> %wide.vec, <16 x i32> undef, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
 ; CHECK: shufflevector <16 x i32> %wide.vec, <16 x i32> undef, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
 ; CHECK: shufflevector <16 x i32> %wide.vec, <16 x i32> undef, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
-; CHECK: add nsw <4 x i32>
+; CHECK: add <4 x i32>
 ; CHECK: sub <4 x i32>
-; CHECK: add nsw <4 x i32>
+; CHECK: add <4 x i32>
 ; CHECK: sub <4 x i32>
 
 %struct.ST4 = type { i32, i32, i32, i32 }
@@ -529,7 +529,7 @@ for.body:                                         ; preds = %for.body, %entry
 ; CHECK: %[[V0:.*]] = shufflevector <8 x i32> %wide.vec, <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
 ; CHECK: %[[V1:.*]] = shufflevector <8 x i32> %wide.vec, <8 x i32> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
 ; CHECK: bitcast <4 x i32> %[[V1]] to <4 x float>
-; CHECK: add nsw <4 x i32>
+; CHECK: add <4 x i32>
 ; CHECK: fadd fast <4 x float>
 
 %struct.IntFloat = type { i32, float }
@@ -645,7 +645,7 @@ for.end:
 ; CHECK:   store i32 %[[X4:.+]], {{.*}}
 ; CHECK:   %[[L2:.+]] = load <8 x i32>, <8 x i32>* {{.*}}
 ; CHECK:   %[[S1:.+]] = shufflevector <8 x i32> %[[L2]], <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK:   add nsw <4 x i32> %[[S1]], %[[Phi]]
+; CHECK:   add <4 x i32> %[[S1]], %[[Phi]]
 
 define i32 @PR27626_1(%pair.i32 *%p, i64 %n) {
 entry:
@@ -746,7 +746,7 @@ for.end:
 ; CHECK:   store i32 %[[X4:.+]], {{.*}}
 ; CHECK:   %[[L2:.+]] = load <8 x i32>, <8 x i32>* {{.*}}
 ; CHECK:   %[[S1:.+]] = shufflevector <8 x i32> %[[L2]], <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK:   add nsw <4 x i32> %[[S1]], %[[Phi]]
+; CHECK:   add <4 x i32> %[[S1]], %[[Phi]]
 
 define i32 @PR27626_3(%pair.i32 *%p, i64 %n, i32 %z) {
 entry:
diff --git a/llvm/test/Transforms/LoopVectorize/metadata-width.ll b/llvm/test/Transforms/LoopVectorize/metadata-width.ll
index 455a4e195..7c7973247 100644
--- a/llvm/test/Transforms/LoopVectorize/metadata-width.ll
+++ b/llvm/test/Transforms/LoopVectorize/metadata-width.ll
@@ -24,7 +24,7 @@ for.end:                                          ; preds = %for.body, %entry
   ret void
 }
 
-attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !0 = !{!0, !1}
 !1 = !{!"llvm.loop.vectorize.width", i32 8}
diff --git a/llvm/test/Transforms/LoopVectorize/multiple-address-spaces.ll b/llvm/test/Transforms/LoopVectorize/multiple-address-spaces.ll
index cb6478f10..84e533ee7 100644
--- a/llvm/test/Transforms/LoopVectorize/multiple-address-spaces.ll
+++ b/llvm/test/Transforms/LoopVectorize/multiple-address-spaces.ll
@@ -40,4 +40,4 @@ for.end:                                          ; preds = %for.body
   ret i32 0
 }
 
-attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/Transforms/LoopVectorize/no_int_induction.ll b/llvm/test/Transforms/LoopVectorize/no_int_induction.ll
index 7e6b26cd9..cac4076da 100644
--- a/llvm/test/Transforms/LoopVectorize/no_int_induction.ll
+++ b/llvm/test/Transforms/LoopVectorize/no_int_induction.ll
@@ -10,7 +10,7 @@ target datalayout = "e-p:64:64:64-p1:16:16:16-i1:8:8-i8:8:8-i16:16:16-i32:32:32-
 ;CHECK: phi i64
 ;CHECK: phi <4 x i32>
 ;CHECK: load <4 x i32>
-;CHECK: add nsw <4 x i32>
+;CHECK: add <4 x i32>
 ;CHECK: ret i32
 define i32 @sum_array(i32* %A, i32 %n) nounwind uwtable readonly noinline ssp {
   %1 = sext i32 %n to i64
@@ -37,7 +37,7 @@ _ZSt10accumulateIPiiET0_T_S2_S1_.exit:            ; preds = %.lr.ph.i, %0
 ;CHECK: phi i16
 ;CHECK: phi <4 x i32>
 ;CHECK: load <4 x i32>
-;CHECK: add nsw <4 x i32>
+;CHECK: add <4 x i32>
 ;CHECK: ret i32
 define i32 @sum_array_as1(i32 addrspace(1)* %A, i32 %n) nounwind uwtable readonly noinline ssp {
   %1 = sext i32 %n to i64
diff --git a/llvm/test/Transforms/LoopVectorize/nofloat-report.ll b/llvm/test/Transforms/LoopVectorize/nofloat-report.ll
new file mode 100644
index 000000000..39ba1e9e4
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/nofloat-report.ll
@@ -0,0 +1,27 @@
+; REQUIRES: asserts
+; RUN: opt -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -debug-only=loop-vectorize\
+; RUN:     -disable-output -S %s 2>&1 | FileCheck %s
+
+; Make sure that we report about not vectorizing functions with 'noimplicitfloat' attributes
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+@a = common global [2048 x i32] zeroinitializer, align 16
+
+; CHECK: LV: Not vectorizing: Can't vectorize when the NoImplicitFloat attribute is used
+define void @example_nofloat() noimplicitfloat { ;           <--------- "noimplicitfloat" attribute here!
+  br label %1
+
+; <label>:1                                       ; preds = %1, %0
+  %indvars.iv = phi i64 [ 0, %0 ], [ %indvars.iv.next, %1 ]
+  %2 = getelementptr inbounds [2048 x i32], [2048 x i32]* @a, i64 0, i64 %indvars.iv
+  %3 = trunc i64 %indvars.iv to i32
+  store i32 %3, i32* %2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 1024
+  br i1 %exitcond, label %4, label %1
+
+; <label>:4                                       ; preds = %1
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/nuw.ll b/llvm/test/Transforms/LoopVectorize/nuw.ll
new file mode 100644
index 000000000..cf1565637
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/nuw.ll
@@ -0,0 +1,58 @@
+; RUN: opt %s -loop-vectorize -force-vector-interleave=2 -force-vector-width=4 -S | FileCheck %s
+
+; Fixes PR43828
+
+define void @test(i32* %B) {
+; CHECK-LABEL: @test(
+; CHECK:       vector.body:
+; CHECK-COUNT-2: sub <4 x i32>
+entry:
+  br label %outer_loop
+
+outer_loop:
+  %local_4 = phi i32 [ 2, %entry ], [ %4, %outer_tail]
+  br label %inner_loop
+
+inner_loop:
+  %local_2 = phi i32 [ 0, %outer_loop ], [ %1, %inner_loop ]
+  %local_3 = phi i32 [ -104, %outer_loop ], [ %0, %inner_loop ]
+  %0 = sub nuw nsw i32 %local_3, %local_4
+  %1 = add nuw nsw i32 %local_2, 1
+  %2 = icmp ugt i32 %local_2, 126
+  br i1 %2, label %outer_tail, label %inner_loop
+
+outer_tail:
+  %3 = phi i32 [ %0, %inner_loop ]
+  store atomic i32 %3, i32 * %B unordered, align 8
+  %4 = add i32 %local_4, 1
+  %5 = icmp slt i32 %4, 6
+  br i1 %5, label %outer_loop, label %exit
+
+exit:
+  ret void
+}
+
+define i32 @multi-instr(i32* noalias nocapture %A, i32* noalias nocapture %B, i32 %inc) {
+; CHECK-LABEL: @multi-instr(
+; CHECK:       vector.body:
+; CHECK-COUNT-4: add <4 x i32>
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [0, %entry], [%iv_inc, %loop]
+  %redu = phi i32 [0, %entry], [%3, %loop]
+  %gepa = getelementptr inbounds i32, i32* %A, i32 %iv
+  %gepb = getelementptr inbounds i32, i32* %B, i32 %iv
+  %0 = load i32, i32* %gepa
+  %1 = load i32, i32* %gepb
+  %2 = add nuw nsw i32 %redu, %0
+  %3 = add nuw nsw i32 %2, %1
+  %iv_inc = add nuw nsw i32 %iv, 1
+  %4 = icmp ult i32 %iv_inc, 128
+  br i1 %4, label %loop, label %exit
+
+exit:
+  %lcssa = phi i32 [%3, %loop]
+  ret i32 %lcssa
+}
diff --git a/llvm/test/Transforms/LoopVectorize/optsize.ll b/llvm/test/Transforms/LoopVectorize/optsize.ll
index 403c006ee..199190649 100644
--- a/llvm/test/Transforms/LoopVectorize/optsize.ll
+++ b/llvm/test/Transforms/LoopVectorize/optsize.ll
@@ -84,6 +84,43 @@ for.end:                                          ; preds = %for.body
   ret i32 0
 }
 
+; PR43371: don't run into an assert due to emitting SCEV runtime checks
+; with OptForSize.
+;
+@cm_array = external global [2592 x i16], align 1
+
+define void @pr43371() optsize {
+;
+; CHECK-LABEL: @pr43371
+; CHECK-NOT:   vector.scevcheck
+;
+; We do not want to generate SCEV predicates when optimising for size, because
+; that will lead to extra code generation such as the SCEV overflow runtime
+; checks. Not generating SCEV predicates can still result in vectorisation as
+; the non-consecutive loads/stores can be scalarized:
+;
+; CHECK: vector.body:
+; CHECK: store i16 0, i16* %{{.*}}, align 1
+; CHECK: store i16 0, i16* %{{.*}}, align 1
+; CHECK: br i1 {{.*}}, label %vector.body
+;
+entry:
+  br label %for.body29
+
+for.cond.cleanup28:
+  unreachable
+
+for.body29:
+  %i24.0170 = phi i16 [ 0, %entry], [ %inc37, %for.body29]
+  %add33 = add i16 undef, %i24.0170
+  %idxprom34 = zext i16 %add33 to i32
+  %arrayidx35 = getelementptr [2592 x i16], [2592 x i16] * @cm_array, i32 0, i32 %idxprom34
+  store i16 0, i16 * %arrayidx35, align 1
+  %inc37 = add i16 %i24.0170, 1
+  %cmp26 = icmp ult i16 %inc37, 756
+  br i1 %cmp26, label %for.body29, label %for.cond.cleanup28
+}
+
 !llvm.module.flags = !{!0}
 !0 = !{i32 1, !"ProfileSummary", !1}
 !1 = !{!2, !3, !4, !5, !6, !7, !8, !9}
diff --git a/llvm/test/Transforms/LoopVectorize/pr44488-predication.ll b/llvm/test/Transforms/LoopVectorize/pr44488-predication.ll
new file mode 100644
index 000000000..f049eef03
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/pr44488-predication.ll
@@ -0,0 +1,79 @@
+; RUN: opt < %s -force-vector-width=2 -force-vector-interleave=1 -loop-vectorize -S | FileCheck %s
+
+; Test case for PR44488. Checks that the correct predicates are created for
+; branches where true and false successors are equal. See the checks involving
+; CMP1 and CMP2.
+
+@v_38 = global i16 12061, align 1
+@v_39 = global i16 11333, align 1
+
+define i16 @test_true_and_false_branch_equal() {
+; CHECK-LABEL: @test_true_and_false_branch_equal(
+; CHECK-LABEL: vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], [[PRED_SREM_CONTINUE2:%.*]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = trunc i32 [[INDEX]] to i16
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i16 99, [[TMP0]]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i16> undef, i16 [[OFFSET_IDX]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i16> [[BROADCAST_SPLATINSERT]], <2 x i16> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <2 x i16> [[BROADCAST_SPLAT]], <i16 0, i16 1>
+; CHECK-NEXT:    [[TMP1:%.*]] = add i16 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, i16* @v_38, align 1
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, i16* @v_38, align 1
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <2 x i16> undef, i16 [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <2 x i16> [[TMP4]], i16 [[TMP3]], i32 1
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq <2 x i16> [[TMP5]], <i16 32767, i16 32767>
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp eq <2 x i16> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[NOT_CMP2:%.*]] = xor <2 x i1> [[CMP2]], <i1 true, i1 true>
+; CHECK-NEXT:    [[PRED1:%.*]] = extractelement <2 x i1> [[NOT_CMP2]], i32 0
+; CHECK-NEXT:    br i1 [[PRED1]], label [[PRED_SREM_IF:%.*]], label [[PRED_SREM_CONTINUE:%.*]]
+; CHECK:       pred.srem.if:
+; CHECK-NEXT:    [[TMP14:%.*]] = srem i16 5786, [[TMP2]]
+; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <2 x i16> undef, i16 [[TMP14]], i32 0
+; CHECK-NEXT:    br label [[PRED_SREM_CONTINUE]]
+; CHECK:       pred.srem.continue:
+; CHECK-NEXT:    [[TMP16:%.*]] = phi <2 x i16> [ undef, %vector.body ], [ [[TMP15]], [[PRED_SREM_IF]] ]
+; CHECK-NEXT:    [[PRED2:%.*]] = extractelement <2 x i1> [[NOT_CMP2]], i32 1
+; CHECK-NEXT:    br i1 [[PRED2]], label [[PRED_SREM_IF1:%.*]], label [[PRED_SREM_CONTINUE2]]
+; CHECK:       pred.srem.if1:
+; CHECK-NEXT:    [[TMP18:%.*]] = srem i16 5786, [[TMP3]]
+; CHECK-NEXT:    [[TMP19:%.*]] = insertelement <2 x i16> [[TMP16]], i16 [[TMP18]], i32 1
+; CHECK-NEXT:    br label [[PRED_SREM_CONTINUE2]]
+; CHECK:       pred.srem.continue2:
+; CHECK-NEXT:    [[TMP20:%.*]] = phi <2 x i16> [ [[TMP16]], [[PRED_SREM_CONTINUE]] ], [ [[TMP19]], [[PRED_SREM_IF1]] ]
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[CMP2]], <2 x i16> <i16 5786, i16 5786>, <2 x i16> [[TMP20]]
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <2 x i16> [[PREDPHI]], i32 0
+; CHECK-NEXT:    store i16 [[TMP22]], i16* @v_39, align 1
+; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <2 x i16> [[PREDPHI]], i32 1
+; CHECK-NEXT:    store i16 [[TMP23]], i16* @v_39, align 1
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP24:%.*]] = icmp eq i32 [[INDEX_NEXT]], 12
+; CHECK-NEXT:    br i1 [[TMP24]], label %middle.block, label %vector.body, !llvm.loop !0
+;
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.latch
+  %i.07 = phi i16 [ 99, %entry ], [ %inc7, %for.latch ]
+  %lv = load i16, i16* @v_38, align 1
+  %cmp1 = icmp eq i16 %lv, 32767
+  br i1 %cmp1, label %cond.end, label %cond.end
+
+cond.end:                                         ; preds = %for.body, %for.body
+  %cmp2 = icmp eq i16 %lv, 0
+  br i1 %cmp2, label %for.latch, label %cond.false4
+
+cond.false4:                                      ; preds = %cond.end
+  %rem = srem i16 5786, %lv
+  br label %for.latch
+
+for.latch:                                        ; preds = %cond.end, %cond.false4
+  %cond6 = phi i16 [ %rem, %cond.false4 ], [ 5786, %cond.end ]
+  store i16 %cond6, i16* @v_39, align 1
+  %inc7 = add nsw i16 %i.07, 1
+  %cmp = icmp slt i16 %inc7, 111
+  br i1 %cmp, label %for.body, label %exit
+
+exit:                                 ; preds = %for.latch
+  %rv = load i16, i16* @v_39, align 1
+  ret i16 %rv
+}
diff --git a/llvm/test/Transforms/LoopVectorize/reduction-small-size.ll b/llvm/test/Transforms/LoopVectorize/reduction-small-size.ll
index 879f1c3c5..c68f73331 100644
--- a/llvm/test/Transforms/LoopVectorize/reduction-small-size.ll
+++ b/llvm/test/Transforms/LoopVectorize/reduction-small-size.ll
@@ -8,7 +8,7 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 ; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ [[TMP17:%.*]], %[[LATCH]] ]
 ; CHECK:       [[LATCH]]:
 ; CHECK:         [[TMP13:%.*]] = and <4 x i32> [[VEC_PHI]], <i32 255, i32 255, i32 255, i32 255>
-; CHECK-NEXT:    [[TMP14:%.*]] = add nuw nsw <4 x i32> [[TMP13]], {{.*}}
+; CHECK-NEXT:    [[TMP14:%.*]] = add <4 x i32> [[TMP13]], {{.*}}
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
 ; CHECK:         [[TMP16:%.*]] = trunc <4 x i32> [[TMP14]] to <4 x i8>
 ; CHECK-NEXT:    [[TMP17]] = zext <4 x i8> [[TMP16]] to <4 x i32>
diff --git a/llvm/test/Transforms/LoopVectorize/reduction.ll b/llvm/test/Transforms/LoopVectorize/reduction.ll
index 1246cd5ca..4599899f7 100644
--- a/llvm/test/Transforms/LoopVectorize/reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/reduction.ll
@@ -300,7 +300,7 @@ for.end:                                          ; preds = %for.body, %entry
 ; In this test the reduction variable is on the LHS and we can vectorize it.
 ;CHECK-LABEL: @reduction_sub_lhs(
 ;CHECK: phi <4 x i32>
-;CHECK: sub nsw <4 x i32>
+;CHECK: sub <4 x i32>
 ;CHECK: ret i32
 define i32 @reduction_sub_lhs(i32 %n, i32* noalias nocapture %A) nounwind uwtable readonly {
 entry:
diff --git a/llvm/test/Transforms/LoopVectorize/runtime-check-address-space.ll b/llvm/test/Transforms/LoopVectorize/runtime-check-address-space.ll
index 8e7ac1f86..b97cd43e3 100644
--- a/llvm/test/Transforms/LoopVectorize/runtime-check-address-space.ll
+++ b/llvm/test/Transforms/LoopVectorize/runtime-check-address-space.ll
@@ -218,4 +218,4 @@ for.end:                                          ; preds = %for.body, %entry
   ret void
 }
 
-attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/Transforms/LoopVectorize/runtime-check-readonly-address-space.ll b/llvm/test/Transforms/LoopVectorize/runtime-check-readonly-address-space.ll
index 6ee983d22..c02a49157 100644
--- a/llvm/test/Transforms/LoopVectorize/runtime-check-readonly-address-space.ll
+++ b/llvm/test/Transforms/LoopVectorize/runtime-check-readonly-address-space.ll
@@ -129,4 +129,4 @@ for.end:                                          ; preds = %for.body
   ret void
 }
 
-attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/Transforms/LoopVectorize/runtime-check.ll b/llvm/test/Transforms/LoopVectorize/runtime-check.ll
index 2a665e56a..ecabef22f 100644
--- a/llvm/test/Transforms/LoopVectorize/runtime-check.ll
+++ b/llvm/test/Transforms/LoopVectorize/runtime-check.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s  -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -dce -instcombine -S | FileCheck %s
+; RUN: opt < %s -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -dce -instcombine -S | FileCheck %s
+; RUN: opt < %s -loop-vectorize -disable-basicaa -S -pass-remarks-analysis='loop-vectorize' 2>&1 | FileCheck %s -check-prefix=FORCED_OPTSIZE
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
@@ -18,7 +19,7 @@ define i32 @foo(float* nocapture %a, float* nocapture %b, i32 %n) nounwind uwtab
 ; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[N]], -1, !dbg !9
 ; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[TMP0]] to i64, !dbg !9
 ; CHECK-NEXT:    [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1, !dbg !9
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 4, !dbg !9
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP0]], 3, !dbg !9
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]], !dbg !9
 ; CHECK:       vector.memcheck:
 ; CHECK-NEXT:    [[TMP3:%.*]] = add i32 [[N]], -1, !dbg !9
@@ -28,34 +29,34 @@ define i32 @foo(float* nocapture %a, float* nocapture %b, i32 %n) nounwind uwtab
 ; CHECK-NEXT:    [[SCEVGEP4:%.*]] = getelementptr float, float* [[B:%.*]], i64 [[TMP5]], !dbg !9
 ; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ugt float* [[SCEVGEP4]], [[A]], !dbg !9
 ; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ugt float* [[SCEVGEP]], [[B]], !dbg !9
-; CHECK-NEXT:    [[MEMCHECK_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]], !dbg !9
-; CHECK-NEXT:    br i1 [[MEMCHECK_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]], !dbg !9
+; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]], !dbg !9
+; CHECK-NEXT:    br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]], !dbg !9
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[TMP2]], 8589934588, !dbg !9
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]], !dbg !9
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ], !dbg !9
-; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, float* [[B]], i64 [[INDEX]], !dbg !9
-; CHECK-NEXT:    [[TMP8:%.*]] = bitcast float* [[TMP7]] to <4 x float>*, !dbg !9
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, <4 x float>* [[TMP8]], align 4, !dbg !9, !alias.scope !10
-; CHECK-NEXT:    [[TMP9:%.*]] = fmul <4 x float> [[WIDE_LOAD]], <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00>, !dbg !9
-; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[INDEX]], !dbg !9
-; CHECK-NEXT:    [[TMP11:%.*]] = bitcast float* [[TMP10]] to <4 x float>*, !dbg !9
-; CHECK-NEXT:    store <4 x float> [[TMP9]], <4 x float>* [[TMP11]], align 4, !dbg !9, !alias.scope !13, !noalias !10
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds float, float* [[B]], i64 [[INDEX]], !dbg !9
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast float* [[TMP6]] to <4 x float>*, !dbg !9
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, <4 x float>* [[TMP7]], align 4, !dbg !9, !alias.scope !10
+; CHECK-NEXT:    [[TMP8:%.*]] = fmul <4 x float> [[WIDE_LOAD]], <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00>, !dbg !9
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[INDEX]], !dbg !9
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast float* [[TMP9]] to <4 x float>*, !dbg !9
+; CHECK-NEXT:    store <4 x float> [[TMP8]], <4 x float>* [[TMP10]], align 4, !dbg !9, !alias.scope !13, !noalias !10
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4, !dbg !9
-; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]], !dbg !9
-; CHECK-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !dbg !9, !llvm.loop !15
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]], !dbg !9
+; CHECK-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !dbg !9, !llvm.loop !15
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]], !dbg !9
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]], !dbg !9
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ], [ 0, [[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ], [ 0, [[VECTOR_MEMCHECK]] ], !dbg !9
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]], !dbg !9
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], !dbg !9
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, float* [[B]], i64 [[INDVARS_IV]], !dbg !9
-; CHECK-NEXT:    [[TMP13:%.*]] = load float, float* [[ARRAYIDX]], align 4, !dbg !9
-; CHECK-NEXT:    [[MUL:%.*]] = fmul float [[TMP13]], 3.000000e+00, !dbg !9
+; CHECK-NEXT:    [[TMP12:%.*]] = load float, float* [[ARRAYIDX]], align 4, !dbg !9
+; CHECK-NEXT:    [[MUL:%.*]] = fmul float [[TMP12]], 3.000000e+00, !dbg !9
 ; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[INDVARS_IV]], !dbg !9
 ; CHECK-NEXT:    store float [[MUL]], float* [[ARRAYIDX2]], align 4, !dbg !9
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 1, !dbg !9
@@ -159,6 +160,32 @@ loopexit:
 
 ; CHECK: !9 = !DILocation(line: 101, column: 1, scope: !{{.*}})
 
+define dso_local void @forced_optsize(i64* noalias nocapture readonly %x_p, i64* noalias nocapture readonly %y_p, i64* noalias nocapture %z_p) minsize optsize {
+;
+; FORCED_OPTSIZE: remark: <unknown>:0:0: Code-size may be reduced by not forcing vectorization, or by source-code modifications eliminating the need for runtime checks (e.g., adding 'restrict').
+; FORCED_OPTSIZE-LABEL: @forced_optsize(
+; FORCED_OPTSIZE:       vector.body:
+;
+entry:
+  br label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i64, i64* %x_p, i64 %indvars.iv
+  %0 = load i64, i64* %arrayidx, align 8
+  %arrayidx2 = getelementptr inbounds i64, i64* %y_p, i64 %indvars.iv
+  %1 = load i64, i64* %arrayidx2, align 8
+  %add = add nsw i64 %1, %0
+  %arrayidx4 = getelementptr inbounds i64, i64* %z_p, i64 %indvars.iv
+  store i64 %add, i64* %arrayidx4, align 8
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 128
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body, !llvm.loop !12
+}
+
 !llvm.module.flags = !{!0, !1}
 !llvm.dbg.cu = !{!9}
 !0 = !{i32 2, !"Dwarf Version", i32 4}
@@ -177,3 +204,6 @@ loopexit:
                              splitDebugFilename: "abc.debug", emissionKind: 2)
 !10 = !DIFile(filename: "path/to/file", directory: "/path/to/dir")
 !11 = !{i32 2, !"Debug Info Version", i32 3}
+!12 = distinct !{!12, !13, !14}
+!13 = !{!"llvm.loop.vectorize.width", i32 2}
+!14 = !{!"llvm.loop.vectorize.enable", i1 true}
diff --git a/llvm/test/Transforms/LoopVectorize/scev-exitlim-crash.ll b/llvm/test/Transforms/LoopVectorize/scev-exitlim-crash.ll
index 833db9310..23b980620 100644
--- a/llvm/test/Transforms/LoopVectorize/scev-exitlim-crash.ll
+++ b/llvm/test/Transforms/LoopVectorize/scev-exitlim-crash.ll
@@ -102,8 +102,8 @@ for.end26:                                        ; preds = %for.cond4.for.end26
 }
 declare i32 @fn2(double) #1
 
-attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf"="true" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf"="true" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !0 = !{!"int", !1}
 !1 = !{!"omnipotent char", !2}
diff --git a/llvm/test/Transforms/LoopVectorize/tail-folding-counting-down.ll b/llvm/test/Transforms/LoopVectorize/tail-folding-counting-down.ll
new file mode 100644
index 000000000..2667bfe68
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/tail-folding-counting-down.ll
@@ -0,0 +1,42 @@
+; RUN: opt < %s -loop-vectorize -prefer-predicate-over-epilog -S | FileCheck %s
+
+; Check that when we can't predicate this loop that it is still vectorised (with
+; an epilogue).
+; TODO: the reason this can't be predicated is because a primary induction
+; variable can't be found (not yet) for this counting down loop. But with that
+; fixed, this should be able to be predicated.
+
+; CHECK-LABEL: vector.body:
+
+target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
+
+define dso_local void @foo(i8* noalias nocapture readonly %A, i8* noalias nocapture readonly %B, i8* noalias nocapture %C, i32 %N) {
+entry:
+  %cmp6 = icmp eq i32 %N, 0
+  br i1 %cmp6, label %while.end, label %while.body.preheader
+
+while.body.preheader:
+  br label %while.body
+
+while.body:
+  %N.addr.010 = phi i32 [ %dec, %while.body ], [ %N, %while.body.preheader ]
+  %C.addr.09 = phi i8* [ %incdec.ptr4, %while.body ], [ %C, %while.body.preheader ]
+  %B.addr.08 = phi i8* [ %incdec.ptr1, %while.body ], [ %B, %while.body.preheader ]
+  %A.addr.07 = phi i8* [ %incdec.ptr, %while.body ], [ %A, %while.body.preheader ]
+  %incdec.ptr = getelementptr inbounds i8, i8* %A.addr.07, i32 1
+  %0 = load i8, i8* %A.addr.07, align 1
+  %incdec.ptr1 = getelementptr inbounds i8, i8* %B.addr.08, i32 1
+  %1 = load i8, i8* %B.addr.08, align 1
+  %add = add i8 %1, %0
+  %incdec.ptr4 = getelementptr inbounds i8, i8* %C.addr.09, i32 1
+  store i8 %add, i8* %C.addr.09, align 1
+  %dec = add i32 %N.addr.010, -1
+  %cmp = icmp eq i32 %dec, 0
+  br i1 %cmp, label %while.end.loopexit, label %while.body
+
+while.end.loopexit:
+  br label %while.end
+
+while.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/vectorize-once.ll b/llvm/test/Transforms/LoopVectorize/vectorize-once.ll
index d407ac171..04c246366 100644
--- a/llvm/test/Transforms/LoopVectorize/vectorize-once.ll
+++ b/llvm/test/Transforms/LoopVectorize/vectorize-once.ll
@@ -65,7 +65,7 @@ _ZSt10accumulateIPiiET0_T_S2_S1_.exit:            ; preds = %for.body.i, %entry
   ret i32 %__init.addr.0.lcssa.i
 }
 
-attributes #0 = { nounwind readonly ssp uwtable "fp-contract-model"="standard" "no-frame-pointer-elim" "no-frame-pointer-elim-non-leaf" "realign-stack" "relocation-model"="pic" "ssp-buffers-size"="8" }
+attributes #0 = { nounwind readonly ssp uwtable "fp-contract-model"="standard" "frame-pointer"="non-leaf" "realign-stack" "relocation-model"="pic" "ssp-buffers-size"="8" }
 
 ; CHECK: !0 = distinct !{!0, !1}
 ; CHECK: !1 = !{!"llvm.loop.isvectorized", i32 1}
diff --git a/llvm/test/Transforms/LowerConstantIntrinsics/constant-intrinsics.ll b/llvm/test/Transforms/LowerConstantIntrinsics/constant-intrinsics.ll
new file mode 100644
index 000000000..b2c98d204
--- /dev/null
+++ b/llvm/test/Transforms/LowerConstantIntrinsics/constant-intrinsics.ll
@@ -0,0 +1,114 @@
+; RUN: opt -lower-constant-intrinsics -S < %s | FileCheck %s
+
+;; Ensure that an unfoldable is.constant gets lowered reasonably in
+;; optimized codegen, in particular, that the "true" branch is
+;; eliminated.
+
+;; Also ensure that any unfoldable objectsize is resolved in order.
+
+;; CHECK-NOT: tail call i32 @subfun_1()
+;; CHECK:     tail call i32 @subfun_2()
+;; CHECK-NOT: tail call i32 @subfun_1()
+
+declare i1 @llvm.is.constant.i32(i32 %a) nounwind readnone
+declare i1 @llvm.is.constant.i64(i64 %a) nounwind readnone
+declare i1 @llvm.is.constant.i256(i256 %a) nounwind readnone
+declare i1 @llvm.is.constant.v2i64(<2 x i64> %a) nounwind readnone
+declare i1 @llvm.is.constant.f32(float %a) nounwind readnone
+declare i1 @llvm.is.constant.sl_i32i32s({i32, i32} %a) nounwind readnone
+declare i1 @llvm.is.constant.a2i64([2 x i64] %a) nounwind readnone
+declare i1 @llvm.is.constant.p0i64(i64* %a) nounwind readnone
+
+declare i64 @llvm.objectsize.i64.p0i8(i8*, i1, i1, i1) nounwind readnone
+
+declare i32 @subfun_1()
+declare i32 @subfun_2()
+
+define i32 @test_branch(i32 %in) nounwind {
+  %v = call i1 @llvm.is.constant.i32(i32 %in)
+  br i1 %v, label %True, label %False
+
+True:
+  %call1 = tail call i32 @subfun_1()
+  ret i32 %call1
+
+False:
+  %call2 = tail call i32 @subfun_2()
+  ret i32 %call2
+}
+
+;; llvm.objectsize is another tricky case which gets folded to -1 very
+;; late in the game. We'd like to ensure that llvm.is.constant of
+;; llvm.objectsize is true.
+define i1 @test_objectsize(i8* %obj) nounwind {
+;; CHECK-LABEL:    test_objectsize
+;; CHECK-NOT:      llvm.objectsize
+;; CHECK-NOT:      llvm.is.constant
+;; CHECK:          ret i1 true
+  %os = call i64 @llvm.objectsize.i64.p0i8(i8* %obj, i1 false, i1 false, i1 false)
+  %os1 = add i64 %os, 1
+  %v = call i1 @llvm.is.constant.i64(i64 %os1)
+  ret i1 %v
+}
+
+@test_phi_a = dso_local global i32 0, align 4
+declare dso_local i32 @test_phi_b(...)
+
+; Function Attrs: nounwind uwtable
+define dso_local i32 @test_phi() {
+entry:
+  %0 = load i32, i32* @test_phi_a, align 4
+  %1 = tail call i1 @llvm.is.constant.i32(i32 %0)
+  br i1 %1, label %cond.end, label %cond.false
+
+cond.false:                                       ; preds = %entry
+  %call = tail call i32 bitcast (i32 (...)* @test_phi_b to i32 ()*)() #3
+  %.pre = load i32, i32* @test_phi_a, align 4
+  br label %cond.end
+
+cond.end:                                         ; preds = %entry, %cond.false
+  %2 = phi i32 [ %.pre, %cond.false ], [ %0, %entry ]
+  %cond = phi i32 [ %call, %cond.false ], [ 1, %entry ]
+  %cmp = icmp eq i32 %cond, %2
+  br i1 %cmp, label %cond.true1, label %cond.end4
+
+cond.true1:                                       ; preds = %cond.end
+  %call2 = tail call i32 bitcast (i32 (...)* @test_phi_b to i32 ()*)() #3
+  br label %cond.end4
+
+cond.end4:                                        ; preds = %cond.end, %cond.true1
+  ret i32 undef
+}
+
+define i1 @test_various_types(i256 %int, float %float, <2 x i64> %vec, {i32, i32} %struct, [2 x i64] %arr, i64* %ptr) #0 {
+; CHECK-LABEL: @test_various_types(
+; CHECK-NOT: llvm.is.constant
+  %v1 = call i1 @llvm.is.constant.i256(i256 %int)
+  %v2 = call i1 @llvm.is.constant.f32(float %float)
+  %v3 = call i1 @llvm.is.constant.v2i64(<2 x i64> %vec)
+  %v4 = call i1 @llvm.is.constant.sl_i32i32s({i32, i32} %struct)
+  %v5 = call i1 @llvm.is.constant.a2i64([2 x i64] %arr)
+  %v6 = call i1 @llvm.is.constant.p0i64(i64* %ptr)
+
+  %c1 = call i1 @llvm.is.constant.i256(i256 -1)
+  %c2 = call i1 @llvm.is.constant.f32(float 17.0)
+  %c3 = call i1 @llvm.is.constant.v2i64(<2 x i64> <i64 -1, i64 44>)
+  %c4 = call i1 @llvm.is.constant.sl_i32i32s({i32, i32} {i32 -1, i32 32})
+  %c5 = call i1 @llvm.is.constant.a2i64([2 x i64] [i64 -1, i64 32])
+  %c6 = call i1 @llvm.is.constant.p0i64(i64* inttoptr (i32 42 to i64*))
+
+  %x1 = add i1 %v1, %c1
+  %x2 = add i1 %v2, %c2
+  %x3 = add i1 %v3, %c3
+  %x4 = add i1 %v4, %c4
+  %x5 = add i1 %v5, %c5
+  %x6 = add i1 %v6, %c6
+
+  %res2 = add i1 %x1, %x2
+  %res3 = add i1 %res2, %x3
+  %res4 = add i1 %res3, %x4
+  %res5 = add i1 %res4, %x5
+  %res6 = add i1 %res5, %x6
+
+  ret i1 %res6
+}
diff --git a/llvm/test/Transforms/LowerConstantIntrinsics/crash-on-large-allocas.ll b/llvm/test/Transforms/LowerConstantIntrinsics/crash-on-large-allocas.ll
new file mode 100644
index 000000000..e4e981161
--- /dev/null
+++ b/llvm/test/Transforms/LowerConstantIntrinsics/crash-on-large-allocas.ll
@@ -0,0 +1,16 @@
+; RUN: opt -S -lower-constant-intrinsics %s -o - | FileCheck %s
+;
+; Ensure that we don't {crash,return a bad value} when given an alloca larger
+; than what a pointer can represent.
+
+target datalayout = "p:16:16"
+
+; CHECK-LABEL: @alloca_overflow_is_unknown(
+define i16 @alloca_overflow_is_unknown() {
+  %i = alloca i8, i32 65537
+  %j = call i16 @llvm.objectsize.i16.p0i8(i8* %i, i1 false, i1 false, i1 false)
+  ; CHECK: ret i16 -1
+  ret i16 %j
+}
+
+declare i16 @llvm.objectsize.i16.p0i8(i8*, i1, i1, i1)
diff --git a/llvm/test/Transforms/LowerConstantIntrinsics/objectsize_basic.ll b/llvm/test/Transforms/LowerConstantIntrinsics/objectsize_basic.ll
new file mode 100644
index 000000000..94aba44fc
--- /dev/null
+++ b/llvm/test/Transforms/LowerConstantIntrinsics/objectsize_basic.ll
@@ -0,0 +1,83 @@
+; RUN: opt -lower-constant-intrinsics -S < %s | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-apple-darwin10.0.0"
+
+declare i64 @llvm.objectsize.i64(i8*, i1, i1, i1) nounwind readonly
+declare i64 @llvm.objectsize.i64.p1i8(i8 addrspace(1)*, i1, i1, i1) nounwind readonly
+declare void @llvm.trap() nounwind
+
+; CHECK-LABEL: @test1(
+; objectsize should fold to a constant, which causes the branch to fold to an
+; uncond branch.
+define i32 @test1(i8* %ptr) nounwind ssp noredzone align 2 {
+entry:
+  %0 = tail call i64 @llvm.objectsize.i64(i8* %ptr, i1 false, i1 false, i1 false)
+  %1 = icmp ugt i64 %0, 3
+  br i1 %1, label %T, label %trap
+
+; CHECK: entry:
+; CHECK-NOT: label %trap
+
+trap:                                             ; preds = %0, %entry
+  tail call void @llvm.trap() noreturn nounwind
+  unreachable
+
+T:
+; CHECK: ret i32 4
+  ret i32 4
+}
+
+; CHECK-LABEL: @test_objectsize_null_flag(
+define i64 @test_objectsize_null_flag(i8* %ptr) {
+entry:
+  ; CHECK: ret i64 -1
+  %0 = tail call i64 @llvm.objectsize.i64(i8* null, i1 false, i1 true, i1 false)
+  ret i64 %0
+}
+
+; CHECK-LABEL: @test_objectsize_null_flag_min(
+define i64 @test_objectsize_null_flag_min(i8* %ptr) {
+entry:
+  ; CHECK: ret i64 0
+  %0 = tail call i64 @llvm.objectsize.i64(i8* null, i1 true, i1 true, i1 false)
+  ret i64 %0
+}
+
+; Test foldable null pointers because we evaluate them with non-exact modes in
+; CodeGenPrepare.
+; CHECK-LABEL: @test_objectsize_null_flag_noas0(
+define i64 @test_objectsize_null_flag_noas0() {
+entry:
+  ; CHECK: ret i64 -1
+  %0 = tail call i64 @llvm.objectsize.i64.p1i8(i8 addrspace(1)* null, i1 false,
+                                               i1 true, i1 false)
+  ret i64 %0
+}
+
+; CHECK-LABEL: @test_objectsize_null_flag_min_noas0(
+define i64 @test_objectsize_null_flag_min_noas0() {
+entry:
+  ; CHECK: ret i64 0
+  %0 = tail call i64 @llvm.objectsize.i64.p1i8(i8 addrspace(1)* null, i1 true,
+                                               i1 true, i1 false)
+  ret i64 %0
+}
+
+; CHECK-LABEL: @test_objectsize_null_known_flag_noas0
+define i64 @test_objectsize_null_known_flag_noas0() {
+entry:
+  ; CHECK: ret i64 -1
+  %0 = tail call i64 @llvm.objectsize.i64.p1i8(i8 addrspace(1)* null, i1 false,
+                                               i1 false, i1 false)
+  ret i64 %0
+}
+
+; CHECK-LABEL: @test_objectsize_null_known_flag_min_noas0
+define i64 @test_objectsize_null_known_flag_min_noas0() {
+entry:
+  ; CHECK: ret i64 0
+  %0 = tail call i64 @llvm.objectsize.i64.p1i8(i8 addrspace(1)* null, i1 true,
+                                               i1 false, i1 false)
+  ret i64 %0
+}
diff --git a/llvm/test/Transforms/LowerExpectIntrinsic/basic.ll b/llvm/test/Transforms/LowerExpectIntrinsic/basic.ll
index d1335e834..4324b7bd3 100644
--- a/llvm/test/Transforms/LowerExpectIntrinsic/basic.ll
+++ b/llvm/test/Transforms/LowerExpectIntrinsic/basic.ll
@@ -13,7 +13,7 @@ entry:
   %conv1 = sext i32 %conv to i64
   %expval = call i64 @llvm.expect.i64(i64 %conv1, i64 1)
   %tobool = icmp ne i64 %expval, 0
-; CHECK: !prof !0
+; CHECK: !prof !0, !misexpect !1
 ; CHECK-NOT: @llvm.expect
   br i1 %tobool, label %if.then, label %if.end
 
@@ -45,7 +45,7 @@ entry:
   %conv = sext i32 %tmp to i64
   %expval = call i64 @llvm.expect.i64(i64 %conv, i64 1)
   %tobool = icmp ne i64 %expval, 0
-; CHECK: !prof !0
+; CHECK: !prof !0, !misexpect !1
 ; CHECK-NOT: @llvm.expect
   br i1 %tobool, label %if.then, label %if.end
 
@@ -76,7 +76,7 @@ entry:
   %conv = sext i32 %lnot.ext to i64
   %expval = call i64 @llvm.expect.i64(i64 %conv, i64 1)
   %tobool1 = icmp ne i64 %expval, 0
-; CHECK: !prof !0
+; CHECK: !prof !0, !misexpect !1
 ; CHECK-NOT: @llvm.expect
   br i1 %tobool1, label %if.then, label %if.end
 
@@ -108,7 +108,7 @@ entry:
   %conv = sext i32 %lnot.ext to i64
   %expval = call i64 @llvm.expect.i64(i64 %conv, i64 1)
   %tobool2 = icmp ne i64 %expval, 0
-; CHECK: !prof !0
+; CHECK: !prof !0, !misexpect !1
 ; CHECK-NOT: @llvm.expect
   br i1 %tobool2, label %if.then, label %if.end
 
@@ -138,7 +138,7 @@ entry:
   %conv1 = sext i32 %conv to i64
   %expval = call i64 @llvm.expect.i64(i64 %conv1, i64 0)
   %tobool = icmp ne i64 %expval, 0
-; CHECK: !prof !1
+; CHECK: !prof !2, !misexpect !3
 ; CHECK-NOT: @llvm.expect
   br i1 %tobool, label %if.then, label %if.end
 
@@ -164,8 +164,8 @@ entry:
   store i32 %x, i32* %x.addr, align 4
   %tmp = load i32, i32* %x.addr, align 4
   %conv = sext i32 %tmp to i64
-  %expval = call i64 @llvm.expect.i64(i64 %conv, i64 1)
-; CHECK: !prof !2
+  %expval = call i64 @llvm.expect.i64(i64 %conv, i64 2)
+; CHECK: !prof !4, !misexpect !5
 ; CHECK-NOT: @llvm.expect
   switch i64 %expval, label %sw.epilog [
     i64 1, label %sw.bb
@@ -194,7 +194,7 @@ entry:
   %tmp = load i32, i32* %x.addr, align 4
   %conv = sext i32 %tmp to i64
   %expval = call i64 @llvm.expect.i64(i64 %conv, i64 1)
-; CHECK: !prof !3
+; CHECK: !prof !6, !misexpect !1
 ; CHECK-NOT: @llvm.expect
   switch i64 %expval, label %sw.epilog [
     i64 2, label %sw.bb
@@ -226,7 +226,7 @@ entry:
   %conv = zext i1 %cmp to i32
   %expval = call i32 @llvm.expect.i32(i32 %conv, i32 1)
   %tobool = icmp ne i32 %expval, 0
-; CHECK: !prof !0
+; CHECK: !prof !0, !misexpect !1
 ; CHECK-NOT: @llvm.expect
   br i1 %tobool, label %if.then, label %if.end
 
@@ -255,7 +255,7 @@ entry:
   %tmp = load i32, i32* %x.addr, align 4
   %cmp = icmp sgt i32 %tmp, 1
   %expval = call i1 @llvm.expect.i1(i1 %cmp, i1 1)
-; CHECK: !prof !0
+; CHECK: !prof !0, !misexpect !1
 ; CHECK-NOT: @llvm.expect
   br i1 %expval, label %if.then, label %if.end
 
@@ -278,7 +278,7 @@ define i32 @test10(i64 %t6) {
   %t7 = call i64 @llvm.expect.i64(i64 %t6, i64 0)
   %t8 = icmp ne i64 %t7, 0
   %t9 = select i1 %t8, i32 1, i32 2
-; CHECK: select{{.*}}, !prof !1
+; CHECK: select{{.*}}, !prof !2, !misexpect !3
   ret i32 %t9
 }
 
@@ -286,6 +286,9 @@ define i32 @test10(i64 %t6) {
 declare i1 @llvm.expect.i1(i1, i1) nounwind readnone
 
 ; CHECK: !0 = !{!"branch_weights", i32 2000, i32 1}
-; CHECK: !1 = !{!"branch_weights", i32 1, i32 2000}
-; CHECK: !2 = !{!"branch_weights", i32 1, i32 2000, i32 1}
-; CHECK: !3 = !{!"branch_weights", i32 2000, i32 1, i32 1}
+; CHECK: !1 = !{!"misexpect", i64 0, i64 2000, i64 1}
+; CHECK: !2 = !{!"branch_weights", i32 1, i32 2000}
+; CHECK: !3 = !{!"misexpect", i64 1, i64 2000, i64 1}
+; CHECK: !4 = !{!"branch_weights", i32 1, i32 1, i32 2000}
+; CHECK: !5 = !{!"misexpect", i64 2, i64 2000, i64 1}
+; CHECK: !6 = !{!"branch_weights", i32 2000, i32 1, i32 1}
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/bigger-expressions-double.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/bigger-expressions-double.ll
new file mode 100644
index 000000000..b72ecf5c0
--- /dev/null
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/bigger-expressions-double.ll
@@ -0,0 +1,513 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -lower-matrix-intrinsics -S < %s | FileCheck  %s
+; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s
+
+define void @transpose_multiply(<9 x double>* %A.Ptr, <9 x double>* %B.Ptr, <9 x double>* %C.Ptr) {
+; CHECK-LABEL: @transpose_multiply(
+; CHECK-NEXT:  entry:
+
+; Load columns of input matrixes %A and %B.
+
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <9 x double>* [[A_PTR:%.*]] to double*
+; CHECK-NEXT:    [[COL_CAST:%.*]] = bitcast double* [[TMP0]] to <3 x double>*
+; CHECK-NEXT:    [[COL_LOAD:%.*]] = load <3 x double>, <3 x double>* [[COL_CAST]], align 8
+; CHECK-NEXT:    [[COL_GEP:%.*]] = getelementptr double, double* [[TMP0]], i32 3
+; CHECK-NEXT:    [[COL_CAST1:%.*]] = bitcast double* [[COL_GEP]] to <3 x double>*
+; CHECK-NEXT:    [[COL_LOAD2:%.*]] = load <3 x double>, <3 x double>* [[COL_CAST1]], align 8
+; CHECK-NEXT:    [[COL_GEP3:%.*]] = getelementptr double, double* [[TMP0]], i32 6
+; CHECK-NEXT:    [[COL_CAST4:%.*]] = bitcast double* [[COL_GEP3]] to <3 x double>*
+; CHECK-NEXT:    [[COL_LOAD5:%.*]] = load <3 x double>, <3 x double>* [[COL_CAST4]], align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <9 x double>* [[B_PTR:%.*]] to double*
+; CHECK-NEXT:    [[COL_CAST6:%.*]] = bitcast double* [[TMP1]] to <3 x double>*
+; CHECK-NEXT:    [[COL_LOAD7:%.*]] = load <3 x double>, <3 x double>* [[COL_CAST6]], align 8
+; CHECK-NEXT:    [[COL_GEP8:%.*]] = getelementptr double, double* [[TMP1]], i32 3
+; CHECK-NEXT:    [[COL_CAST9:%.*]] = bitcast double* [[COL_GEP8]] to <3 x double>*
+; CHECK-NEXT:    [[COL_LOAD10:%.*]] = load <3 x double>, <3 x double>* [[COL_CAST9]], align 8
+; CHECK-NEXT:    [[COL_GEP11:%.*]] = getelementptr double, double* [[TMP1]], i32 6
+; CHECK-NEXT:    [[COL_CAST12:%.*]] = bitcast double* [[COL_GEP11]] to <3 x double>*
+; CHECK-NEXT:    [[COL_LOAD13:%.*]] = load <3 x double>, <3 x double>* [[COL_CAST12]], align 8
+
+; Transpose %A.
+
+; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <3 x double> [[COL_LOAD]], i64 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <3 x double> undef, double [[TMP0]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <3 x double> [[COL_LOAD2]], i64 0
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <3 x double> [[TMP1]], double [[TMP2]], i64 1
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <3 x double> [[COL_LOAD5]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <3 x double> [[TMP3]], double [[TMP4]], i64 2
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <3 x double> [[COL_LOAD]], i64 1
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <3 x double> undef, double [[TMP6]], i64 0
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <3 x double> [[COL_LOAD2]], i64 1
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <3 x double> [[TMP7]], double [[TMP8]], i64 1
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <3 x double> [[COL_LOAD5]], i64 1
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <3 x double> [[TMP9]], double [[TMP10]], i64 2
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <3 x double> [[COL_LOAD]], i64 2
+; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <3 x double> undef, double [[TMP12]], i64 0
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <3 x double> [[COL_LOAD2]], i64 2
+; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <3 x double> [[TMP13]], double [[TMP14]], i64 1
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <3 x double> [[COL_LOAD5]], i64 2
+; CHECK-NEXT:    [[TMP17:%.*]] = insertelement <3 x double> [[TMP15]], double [[TMP16]], i64 2
+
+; Lower multiply(transpose(%A), %B)
+
+; CHECK-NEXT:    [[BLOCK:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <3 x double> [[COL_LOAD7]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <1 x double> undef, double [[TMP18]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP19:%.*]] = fmul <1 x double> [[BLOCK]], [[SPLAT_SPLAT]]
+; CHECK-NEXT:    [[BLOCK6:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <3 x double> [[COL_LOAD7]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT7:%.*]] = insertelement <1 x double> undef, double [[TMP20]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT8:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT7]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP21:%.*]] = fmul <1 x double> [[BLOCK6]], [[SPLAT_SPLAT8]]
+; CHECK-NEXT:    [[TMP22:%.*]] = fadd <1 x double> [[TMP19]], [[TMP21]]
+; CHECK-NEXT:    [[BLOCK9:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <3 x double> [[COL_LOAD7]], i64 2
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT10:%.*]] = insertelement <1 x double> undef, double [[TMP23]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT11:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT10]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = fmul <1 x double> [[BLOCK9]], [[SPLAT_SPLAT11]]
+; CHECK-NEXT:    [[TMP25:%.*]] = fadd <1 x double> [[TMP22]], [[TMP24]]
+; CHECK-NEXT:    [[TMP26:%.*]] = shufflevector <1 x double> [[TMP25]], <1 x double> undef, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP27:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP26]], <3 x i32> <i32 3, i32 1, i32 2>
+; CHECK-NEXT:    [[BLOCK12:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP28:%.*]] = extractelement <3 x double> [[COL_LOAD7]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT13:%.*]] = insertelement <1 x double> undef, double [[TMP28]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT14:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT13]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP29:%.*]] = fmul <1 x double> [[BLOCK12]], [[SPLAT_SPLAT14]]
+; CHECK-NEXT:    [[BLOCK15:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP30:%.*]] = extractelement <3 x double> [[COL_LOAD7]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT16:%.*]] = insertelement <1 x double> undef, double [[TMP30]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT17:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT16]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP31:%.*]] = fmul <1 x double> [[BLOCK15]], [[SPLAT_SPLAT17]]
+; CHECK-NEXT:    [[TMP32:%.*]] = fadd <1 x double> [[TMP29]], [[TMP31]]
+; CHECK-NEXT:    [[BLOCK18:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP33:%.*]] = extractelement <3 x double> [[COL_LOAD7]], i64 2
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT19:%.*]] = insertelement <1 x double> undef, double [[TMP33]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT20:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT19]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP34:%.*]] = fmul <1 x double> [[BLOCK18]], [[SPLAT_SPLAT20]]
+; CHECK-NEXT:    [[TMP35:%.*]] = fadd <1 x double> [[TMP32]], [[TMP34]]
+; CHECK-NEXT:    [[TMP36:%.*]] = shufflevector <1 x double> [[TMP35]], <1 x double> undef, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP37:%.*]] = shufflevector <3 x double> [[TMP27]], <3 x double> [[TMP36]], <3 x i32> <i32 0, i32 3, i32 2>
+; CHECK-NEXT:    [[BLOCK21:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> undef, <1 x i32> <i32 2>
+; CHECK-NEXT:    [[TMP38:%.*]] = extractelement <3 x double> [[COL_LOAD7]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT22:%.*]] = insertelement <1 x double> undef, double [[TMP38]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT23:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT22]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP39:%.*]] = fmul <1 x double> [[BLOCK21]], [[SPLAT_SPLAT23]]
+; CHECK-NEXT:    [[BLOCK24:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> undef, <1 x i32> <i32 2>
+; CHECK-NEXT:    [[TMP40:%.*]] = extractelement <3 x double> [[COL_LOAD7]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT25:%.*]] = insertelement <1 x double> undef, double [[TMP40]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT26:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT25]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP41:%.*]] = fmul <1 x double> [[BLOCK24]], [[SPLAT_SPLAT26]]
+; CHECK-NEXT:    [[TMP42:%.*]] = fadd <1 x double> [[TMP39]], [[TMP41]]
+; CHECK-NEXT:    [[BLOCK27:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> undef, <1 x i32> <i32 2>
+; CHECK-NEXT:    [[TMP43:%.*]] = extractelement <3 x double> [[COL_LOAD7]], i64 2
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT28:%.*]] = insertelement <1 x double> undef, double [[TMP43]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT29:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT28]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP44:%.*]] = fmul <1 x double> [[BLOCK27]], [[SPLAT_SPLAT29]]
+; CHECK-NEXT:    [[TMP45:%.*]] = fadd <1 x double> [[TMP42]], [[TMP44]]
+; CHECK-NEXT:    [[TMP46:%.*]] = shufflevector <1 x double> [[TMP45]], <1 x double> undef, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP47:%.*]] = shufflevector <3 x double> [[TMP37]], <3 x double> [[TMP46]], <3 x i32> <i32 0, i32 1, i32 3>
+; CHECK-NEXT:    [[BLOCK30:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP48:%.*]] = extractelement <3 x double> [[COL_LOAD10]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT31:%.*]] = insertelement <1 x double> undef, double [[TMP48]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT32:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT31]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP49:%.*]] = fmul <1 x double> [[BLOCK30]], [[SPLAT_SPLAT32]]
+; CHECK-NEXT:    [[BLOCK33:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP50:%.*]] = extractelement <3 x double> [[COL_LOAD10]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT34:%.*]] = insertelement <1 x double> undef, double [[TMP50]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT35:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT34]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP51:%.*]] = fmul <1 x double> [[BLOCK33]], [[SPLAT_SPLAT35]]
+; CHECK-NEXT:    [[TMP52:%.*]] = fadd <1 x double> [[TMP49]], [[TMP51]]
+; CHECK-NEXT:    [[BLOCK36:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP53:%.*]] = extractelement <3 x double> [[COL_LOAD10]], i64 2
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT37:%.*]] = insertelement <1 x double> undef, double [[TMP53]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT38:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT37]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP54:%.*]] = fmul <1 x double> [[BLOCK36]], [[SPLAT_SPLAT38]]
+; CHECK-NEXT:    [[TMP55:%.*]] = fadd <1 x double> [[TMP52]], [[TMP54]]
+; CHECK-NEXT:    [[TMP56:%.*]] = shufflevector <1 x double> [[TMP55]], <1 x double> undef, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP57:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP56]], <3 x i32> <i32 3, i32 1, i32 2>
+; CHECK-NEXT:    [[BLOCK39:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP58:%.*]] = extractelement <3 x double> [[COL_LOAD10]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT40:%.*]] = insertelement <1 x double> undef, double [[TMP58]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT41:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT40]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP59:%.*]] = fmul <1 x double> [[BLOCK39]], [[SPLAT_SPLAT41]]
+; CHECK-NEXT:    [[BLOCK42:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP60:%.*]] = extractelement <3 x double> [[COL_LOAD10]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT43:%.*]] = insertelement <1 x double> undef, double [[TMP60]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT44:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT43]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP61:%.*]] = fmul <1 x double> [[BLOCK42]], [[SPLAT_SPLAT44]]
+; CHECK-NEXT:    [[TMP62:%.*]] = fadd <1 x double> [[TMP59]], [[TMP61]]
+; CHECK-NEXT:    [[BLOCK45:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP63:%.*]] = extractelement <3 x double> [[COL_LOAD10]], i64 2
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT46:%.*]] = insertelement <1 x double> undef, double [[TMP63]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT47:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT46]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP64:%.*]] = fmul <1 x double> [[BLOCK45]], [[SPLAT_SPLAT47]]
+; CHECK-NEXT:    [[TMP65:%.*]] = fadd <1 x double> [[TMP62]], [[TMP64]]
+; CHECK-NEXT:    [[TMP66:%.*]] = shufflevector <1 x double> [[TMP65]], <1 x double> undef, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP67:%.*]] = shufflevector <3 x double> [[TMP57]], <3 x double> [[TMP66]], <3 x i32> <i32 0, i32 3, i32 2>
+; CHECK-NEXT:    [[BLOCK48:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> undef, <1 x i32> <i32 2>
+; CHECK-NEXT:    [[TMP68:%.*]] = extractelement <3 x double> [[COL_LOAD10]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT49:%.*]] = insertelement <1 x double> undef, double [[TMP68]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT50:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT49]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP69:%.*]] = fmul <1 x double> [[BLOCK48]], [[SPLAT_SPLAT50]]
+; CHECK-NEXT:    [[BLOCK51:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> undef, <1 x i32> <i32 2>
+; CHECK-NEXT:    [[TMP70:%.*]] = extractelement <3 x double> [[COL_LOAD10]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT52:%.*]] = insertelement <1 x double> undef, double [[TMP70]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT53:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT52]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP71:%.*]] = fmul <1 x double> [[BLOCK51]], [[SPLAT_SPLAT53]]
+; CHECK-NEXT:    [[TMP72:%.*]] = fadd <1 x double> [[TMP69]], [[TMP71]]
+; CHECK-NEXT:    [[BLOCK54:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> undef, <1 x i32> <i32 2>
+; CHECK-NEXT:    [[TMP73:%.*]] = extractelement <3 x double> [[COL_LOAD10]], i64 2
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT55:%.*]] = insertelement <1 x double> undef, double [[TMP73]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT56:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT55]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP74:%.*]] = fmul <1 x double> [[BLOCK54]], [[SPLAT_SPLAT56]]
+; CHECK-NEXT:    [[TMP75:%.*]] = fadd <1 x double> [[TMP72]], [[TMP74]]
+; CHECK-NEXT:    [[TMP76:%.*]] = shufflevector <1 x double> [[TMP75]], <1 x double> undef, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP77:%.*]] = shufflevector <3 x double> [[TMP67]], <3 x double> [[TMP76]], <3 x i32> <i32 0, i32 1, i32 3>
+; CHECK-NEXT:    [[BLOCK57:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP78:%.*]] = extractelement <3 x double> [[COL_LOAD13]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT58:%.*]] = insertelement <1 x double> undef, double [[TMP78]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT59:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT58]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP79:%.*]] = fmul <1 x double> [[BLOCK57]], [[SPLAT_SPLAT59]]
+; CHECK-NEXT:    [[BLOCK60:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP80:%.*]] = extractelement <3 x double> [[COL_LOAD13]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT61:%.*]] = insertelement <1 x double> undef, double [[TMP80]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT62:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT61]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP81:%.*]] = fmul <1 x double> [[BLOCK60]], [[SPLAT_SPLAT62]]
+; CHECK-NEXT:    [[TMP82:%.*]] = fadd <1 x double> [[TMP79]], [[TMP81]]
+; CHECK-NEXT:    [[BLOCK63:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP83:%.*]] = extractelement <3 x double> [[COL_LOAD13]], i64 2
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT64:%.*]] = insertelement <1 x double> undef, double [[TMP83]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT65:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT64]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP84:%.*]] = fmul <1 x double> [[BLOCK63]], [[SPLAT_SPLAT65]]
+; CHECK-NEXT:    [[TMP85:%.*]] = fadd <1 x double> [[TMP82]], [[TMP84]]
+; CHECK-NEXT:    [[TMP86:%.*]] = shufflevector <1 x double> [[TMP85]], <1 x double> undef, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP87:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP86]], <3 x i32> <i32 3, i32 1, i32 2>
+; CHECK-NEXT:    [[BLOCK66:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP88:%.*]] = extractelement <3 x double> [[COL_LOAD13]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT67:%.*]] = insertelement <1 x double> undef, double [[TMP88]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT68:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT67]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP89:%.*]] = fmul <1 x double> [[BLOCK66]], [[SPLAT_SPLAT68]]
+; CHECK-NEXT:    [[BLOCK69:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP90:%.*]] = extractelement <3 x double> [[COL_LOAD13]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT70:%.*]] = insertelement <1 x double> undef, double [[TMP90]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT71:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT70]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP91:%.*]] = fmul <1 x double> [[BLOCK69]], [[SPLAT_SPLAT71]]
+; CHECK-NEXT:    [[TMP92:%.*]] = fadd <1 x double> [[TMP89]], [[TMP91]]
+; CHECK-NEXT:    [[BLOCK72:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP93:%.*]] = extractelement <3 x double> [[COL_LOAD13]], i64 2
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT73:%.*]] = insertelement <1 x double> undef, double [[TMP93]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT74:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT73]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP94:%.*]] = fmul <1 x double> [[BLOCK72]], [[SPLAT_SPLAT74]]
+; CHECK-NEXT:    [[TMP95:%.*]] = fadd <1 x double> [[TMP92]], [[TMP94]]
+; CHECK-NEXT:    [[TMP96:%.*]] = shufflevector <1 x double> [[TMP95]], <1 x double> undef, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP97:%.*]] = shufflevector <3 x double> [[TMP87]], <3 x double> [[TMP96]], <3 x i32> <i32 0, i32 3, i32 2>
+; CHECK-NEXT:    [[BLOCK75:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> undef, <1 x i32> <i32 2>
+; CHECK-NEXT:    [[TMP98:%.*]] = extractelement <3 x double> [[COL_LOAD13]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT76:%.*]] = insertelement <1 x double> undef, double [[TMP98]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT77:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT76]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP99:%.*]] = fmul <1 x double> [[BLOCK75]], [[SPLAT_SPLAT77]]
+; CHECK-NEXT:    [[BLOCK78:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> undef, <1 x i32> <i32 2>
+; CHECK-NEXT:    [[TMP100:%.*]] = extractelement <3 x double> [[COL_LOAD13]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT79:%.*]] = insertelement <1 x double> undef, double [[TMP100]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT80:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT79]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP101:%.*]] = fmul <1 x double> [[BLOCK78]], [[SPLAT_SPLAT80]]
+; CHECK-NEXT:    [[TMP102:%.*]] = fadd <1 x double> [[TMP99]], [[TMP101]]
+; CHECK-NEXT:    [[BLOCK81:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> undef, <1 x i32> <i32 2>
+; CHECK-NEXT:    [[TMP103:%.*]] = extractelement <3 x double> [[COL_LOAD13]], i64 2
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT82:%.*]] = insertelement <1 x double> undef, double [[TMP103]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT83:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT82]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP104:%.*]] = fmul <1 x double> [[BLOCK81]], [[SPLAT_SPLAT83]]
+; CHECK-NEXT:    [[TMP105:%.*]] = fadd <1 x double> [[TMP102]], [[TMP104]]
+; CHECK-NEXT:    [[TMP106:%.*]] = shufflevector <1 x double> [[TMP105]], <1 x double> undef, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP107:%.*]] = shufflevector <3 x double> [[TMP97]], <3 x double> [[TMP106]], <3 x i32> <i32 0, i32 1, i32 3>
+
+; Store result columns.
+
+; CHECK-NEXT:    [[TMP108:%.*]] = bitcast <9 x double>* [[C_PTR:%.*]] to double*
+; CHECK-NEXT:    [[TMP109:%.*]] = bitcast double* [[TMP108]] to <3 x double>*
+; CHECK-NEXT:    store <3 x double> [[TMP47]], <3 x double>* [[TMP109]], align 8
+; CHECK-NEXT:    [[TMP110:%.*]] = getelementptr double, double* [[TMP108]], i32 3
+; CHECK-NEXT:    [[TMP111:%.*]] = bitcast double* [[TMP110]] to <3 x double>*
+; CHECK-NEXT:    store <3 x double> [[TMP77]], <3 x double>* [[TMP111]], align 8
+; CHECK-NEXT:    [[TMP112:%.*]] = getelementptr double, double* [[TMP108]], i32 6
+; CHECK-NEXT:    [[TMP113:%.*]] = bitcast double* [[TMP112]] to <3 x double>*
+; CHECK-NEXT:    store <3 x double> [[TMP107]], <3 x double>* [[TMP113]], align 8
+; CHECK-NEXT:    ret void
+;
+
+entry:
+  %a = load <9 x double>, <9 x double>* %A.Ptr
+  %b = load <9 x double>, <9 x double>* %B.Ptr
+  %a.trans  = call <9 x double> @llvm.matrix.transpose(<9 x double> %a, i32 3, i32 3)
+  %c = call <9 x double> @llvm.matrix.multiply.v9f64.v9f64.v9f64(<9 x double> %a.trans, <9 x double> %b, i32 3, i32 3, i32 3)
+  store <9 x double> %c, <9 x double>* %C.Ptr
+  ret void
+}
+
+declare <9 x double> @llvm.matrix.transpose(<9 x double>, i32, i32)
+declare <9 x double> @llvm.matrix.multiply.v9f64.v9f64.v9f64(<9 x double>, <9 x double>, i32, i32, i32)
+
+define void @transpose_multiply_add(<9 x double>* %A.Ptr, <9 x double>* %B.Ptr, <9 x double>* %C.Ptr) {
+; CHECK-LABEL: @transpose_multiply_add(
+; CHECK-NEXT:  entry:
+
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <9 x double>* [[A_PTR:%.*]] to double*
+; CHECK-NEXT:    [[COL_CAST:%.*]] = bitcast double* [[TMP0]] to <3 x double>*
+; CHECK-NEXT:    [[COL_LOAD:%.*]] = load <3 x double>, <3 x double>* [[COL_CAST]], align 8
+; CHECK-NEXT:    [[COL_GEP:%.*]] = getelementptr double, double* [[TMP0]], i32 3
+; CHECK-NEXT:    [[COL_CAST1:%.*]] = bitcast double* [[COL_GEP]] to <3 x double>*
+; CHECK-NEXT:    [[COL_LOAD2:%.*]] = load <3 x double>, <3 x double>* [[COL_CAST1]], align 8
+; CHECK-NEXT:    [[COL_GEP3:%.*]] = getelementptr double, double* [[TMP0]], i32 6
+; CHECK-NEXT:    [[COL_CAST4:%.*]] = bitcast double* [[COL_GEP3]] to <3 x double>*
+; CHECK-NEXT:    [[COL_LOAD5:%.*]] = load <3 x double>, <3 x double>* [[COL_CAST4]], align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <9 x double>* [[B_PTR:%.*]] to double*
+; CHECK-NEXT:    [[COL_CAST6:%.*]] = bitcast double* [[TMP1]] to <3 x double>*
+; CHECK-NEXT:    [[COL_LOAD7:%.*]] = load <3 x double>, <3 x double>* [[COL_CAST6]], align 8
+; CHECK-NEXT:    [[COL_GEP8:%.*]] = getelementptr double, double* [[TMP1]], i32 3
+; CHECK-NEXT:    [[COL_CAST9:%.*]] = bitcast double* [[COL_GEP8]] to <3 x double>*
+; CHECK-NEXT:    [[COL_LOAD10:%.*]] = load <3 x double>, <3 x double>* [[COL_CAST9]], align 8
+; CHECK-NEXT:    [[COL_GEP11:%.*]] = getelementptr double, double* [[TMP1]], i32 6
+; CHECK-NEXT:    [[COL_CAST12:%.*]] = bitcast double* [[COL_GEP11]] to <3 x double>*
+; CHECK-NEXT:    [[COL_LOAD13:%.*]] = load <3 x double>, <3 x double>* [[COL_CAST12]], align 8
+
+; Transpose %A.
+
+; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <3 x double> [[COL_LOAD]], i64 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <3 x double> undef, double [[TMP0]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <3 x double> [[COL_LOAD2]], i64 0
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <3 x double> [[TMP1]], double [[TMP2]], i64 1
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <3 x double> [[COL_LOAD5]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <3 x double> [[TMP3]], double [[TMP4]], i64 2
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <3 x double> [[COL_LOAD]], i64 1
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <3 x double> undef, double [[TMP6]], i64 0
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <3 x double> [[COL_LOAD2]], i64 1
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <3 x double> [[TMP7]], double [[TMP8]], i64 1
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <3 x double> [[COL_LOAD5]], i64 1
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <3 x double> [[TMP9]], double [[TMP10]], i64 2
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <3 x double> [[COL_LOAD]], i64 2
+; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <3 x double> undef, double [[TMP12]], i64 0
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <3 x double> [[COL_LOAD2]], i64 2
+; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <3 x double> [[TMP13]], double [[TMP14]], i64 1
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <3 x double> [[COL_LOAD5]], i64 2
+; CHECK-NEXT:    [[TMP17:%.*]] = insertelement <3 x double> [[TMP15]], double [[TMP16]], i64 2
+
+; Lower multiply(transpose(%A), %B)
+
+; CHECK-NEXT:    [[BLOCK:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <3 x double> [[COL_LOAD7]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <1 x double> undef, double [[TMP18]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP19:%.*]] = fmul <1 x double> [[BLOCK]], [[SPLAT_SPLAT]]
+; CHECK-NEXT:    [[BLOCK6:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <3 x double> [[COL_LOAD7]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT7:%.*]] = insertelement <1 x double> undef, double [[TMP20]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT8:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT7]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP21:%.*]] = fmul <1 x double> [[BLOCK6]], [[SPLAT_SPLAT8]]
+; CHECK-NEXT:    [[TMP22:%.*]] = fadd <1 x double> [[TMP19]], [[TMP21]]
+; CHECK-NEXT:    [[BLOCK9:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <3 x double> [[COL_LOAD7]], i64 2
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT10:%.*]] = insertelement <1 x double> undef, double [[TMP23]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT11:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT10]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = fmul <1 x double> [[BLOCK9]], [[SPLAT_SPLAT11]]
+; CHECK-NEXT:    [[TMP25:%.*]] = fadd <1 x double> [[TMP22]], [[TMP24]]
+; CHECK-NEXT:    [[TMP26:%.*]] = shufflevector <1 x double> [[TMP25]], <1 x double> undef, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP27:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP26]], <3 x i32> <i32 3, i32 1, i32 2>
+; CHECK-NEXT:    [[BLOCK12:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP28:%.*]] = extractelement <3 x double> [[COL_LOAD7]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT13:%.*]] = insertelement <1 x double> undef, double [[TMP28]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT14:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT13]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP29:%.*]] = fmul <1 x double> [[BLOCK12]], [[SPLAT_SPLAT14]]
+; CHECK-NEXT:    [[BLOCK15:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP30:%.*]] = extractelement <3 x double> [[COL_LOAD7]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT16:%.*]] = insertelement <1 x double> undef, double [[TMP30]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT17:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT16]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP31:%.*]] = fmul <1 x double> [[BLOCK15]], [[SPLAT_SPLAT17]]
+; CHECK-NEXT:    [[TMP32:%.*]] = fadd <1 x double> [[TMP29]], [[TMP31]]
+; CHECK-NEXT:    [[BLOCK18:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP33:%.*]] = extractelement <3 x double> [[COL_LOAD7]], i64 2
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT19:%.*]] = insertelement <1 x double> undef, double [[TMP33]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT20:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT19]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP34:%.*]] = fmul <1 x double> [[BLOCK18]], [[SPLAT_SPLAT20]]
+; CHECK-NEXT:    [[TMP35:%.*]] = fadd <1 x double> [[TMP32]], [[TMP34]]
+; CHECK-NEXT:    [[TMP36:%.*]] = shufflevector <1 x double> [[TMP35]], <1 x double> undef, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP37:%.*]] = shufflevector <3 x double> [[TMP27]], <3 x double> [[TMP36]], <3 x i32> <i32 0, i32 3, i32 2>
+; CHECK-NEXT:    [[BLOCK21:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> undef, <1 x i32> <i32 2>
+; CHECK-NEXT:    [[TMP38:%.*]] = extractelement <3 x double> [[COL_LOAD7]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT22:%.*]] = insertelement <1 x double> undef, double [[TMP38]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT23:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT22]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP39:%.*]] = fmul <1 x double> [[BLOCK21]], [[SPLAT_SPLAT23]]
+; CHECK-NEXT:    [[BLOCK24:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> undef, <1 x i32> <i32 2>
+; CHECK-NEXT:    [[TMP40:%.*]] = extractelement <3 x double> [[COL_LOAD7]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT25:%.*]] = insertelement <1 x double> undef, double [[TMP40]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT26:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT25]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP41:%.*]] = fmul <1 x double> [[BLOCK24]], [[SPLAT_SPLAT26]]
+; CHECK-NEXT:    [[TMP42:%.*]] = fadd <1 x double> [[TMP39]], [[TMP41]]
+; CHECK-NEXT:    [[BLOCK27:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> undef, <1 x i32> <i32 2>
+; CHECK-NEXT:    [[TMP43:%.*]] = extractelement <3 x double> [[COL_LOAD7]], i64 2
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT28:%.*]] = insertelement <1 x double> undef, double [[TMP43]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT29:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT28]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP44:%.*]] = fmul <1 x double> [[BLOCK27]], [[SPLAT_SPLAT29]]
+; CHECK-NEXT:    [[TMP45:%.*]] = fadd <1 x double> [[TMP42]], [[TMP44]]
+; CHECK-NEXT:    [[TMP46:%.*]] = shufflevector <1 x double> [[TMP45]], <1 x double> undef, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP47:%.*]] = shufflevector <3 x double> [[TMP37]], <3 x double> [[TMP46]], <3 x i32> <i32 0, i32 1, i32 3>
+; CHECK-NEXT:    [[BLOCK30:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP48:%.*]] = extractelement <3 x double> [[COL_LOAD10]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT31:%.*]] = insertelement <1 x double> undef, double [[TMP48]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT32:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT31]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP49:%.*]] = fmul <1 x double> [[BLOCK30]], [[SPLAT_SPLAT32]]
+; CHECK-NEXT:    [[BLOCK33:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP50:%.*]] = extractelement <3 x double> [[COL_LOAD10]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT34:%.*]] = insertelement <1 x double> undef, double [[TMP50]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT35:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT34]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP51:%.*]] = fmul <1 x double> [[BLOCK33]], [[SPLAT_SPLAT35]]
+; CHECK-NEXT:    [[TMP52:%.*]] = fadd <1 x double> [[TMP49]], [[TMP51]]
+; CHECK-NEXT:    [[BLOCK36:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP53:%.*]] = extractelement <3 x double> [[COL_LOAD10]], i64 2
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT37:%.*]] = insertelement <1 x double> undef, double [[TMP53]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT38:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT37]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP54:%.*]] = fmul <1 x double> [[BLOCK36]], [[SPLAT_SPLAT38]]
+; CHECK-NEXT:    [[TMP55:%.*]] = fadd <1 x double> [[TMP52]], [[TMP54]]
+; CHECK-NEXT:    [[TMP56:%.*]] = shufflevector <1 x double> [[TMP55]], <1 x double> undef, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP57:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP56]], <3 x i32> <i32 3, i32 1, i32 2>
+; CHECK-NEXT:    [[BLOCK39:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP58:%.*]] = extractelement <3 x double> [[COL_LOAD10]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT40:%.*]] = insertelement <1 x double> undef, double [[TMP58]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT41:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT40]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP59:%.*]] = fmul <1 x double> [[BLOCK39]], [[SPLAT_SPLAT41]]
+; CHECK-NEXT:    [[BLOCK42:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP60:%.*]] = extractelement <3 x double> [[COL_LOAD10]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT43:%.*]] = insertelement <1 x double> undef, double [[TMP60]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT44:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT43]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP61:%.*]] = fmul <1 x double> [[BLOCK42]], [[SPLAT_SPLAT44]]
+; CHECK-NEXT:    [[TMP62:%.*]] = fadd <1 x double> [[TMP59]], [[TMP61]]
+; CHECK-NEXT:    [[BLOCK45:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP63:%.*]] = extractelement <3 x double> [[COL_LOAD10]], i64 2
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT46:%.*]] = insertelement <1 x double> undef, double [[TMP63]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT47:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT46]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP64:%.*]] = fmul <1 x double> [[BLOCK45]], [[SPLAT_SPLAT47]]
+; CHECK-NEXT:    [[TMP65:%.*]] = fadd <1 x double> [[TMP62]], [[TMP64]]
+; CHECK-NEXT:    [[TMP66:%.*]] = shufflevector <1 x double> [[TMP65]], <1 x double> undef, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP67:%.*]] = shufflevector <3 x double> [[TMP57]], <3 x double> [[TMP66]], <3 x i32> <i32 0, i32 3, i32 2>
+; CHECK-NEXT:    [[BLOCK48:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> undef, <1 x i32> <i32 2>
+; CHECK-NEXT:    [[TMP68:%.*]] = extractelement <3 x double> [[COL_LOAD10]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT49:%.*]] = insertelement <1 x double> undef, double [[TMP68]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT50:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT49]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP69:%.*]] = fmul <1 x double> [[BLOCK48]], [[SPLAT_SPLAT50]]
+; CHECK-NEXT:    [[BLOCK51:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> undef, <1 x i32> <i32 2>
+; CHECK-NEXT:    [[TMP70:%.*]] = extractelement <3 x double> [[COL_LOAD10]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT52:%.*]] = insertelement <1 x double> undef, double [[TMP70]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT53:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT52]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP71:%.*]] = fmul <1 x double> [[BLOCK51]], [[SPLAT_SPLAT53]]
+; CHECK-NEXT:    [[TMP72:%.*]] = fadd <1 x double> [[TMP69]], [[TMP71]]
+; CHECK-NEXT:    [[BLOCK54:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> undef, <1 x i32> <i32 2>
+; CHECK-NEXT:    [[TMP73:%.*]] = extractelement <3 x double> [[COL_LOAD10]], i64 2
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT55:%.*]] = insertelement <1 x double> undef, double [[TMP73]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT56:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT55]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP74:%.*]] = fmul <1 x double> [[BLOCK54]], [[SPLAT_SPLAT56]]
+; CHECK-NEXT:    [[TMP75:%.*]] = fadd <1 x double> [[TMP72]], [[TMP74]]
+; CHECK-NEXT:    [[TMP76:%.*]] = shufflevector <1 x double> [[TMP75]], <1 x double> undef, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP77:%.*]] = shufflevector <3 x double> [[TMP67]], <3 x double> [[TMP76]], <3 x i32> <i32 0, i32 1, i32 3>
+; CHECK-NEXT:    [[BLOCK57:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP78:%.*]] = extractelement <3 x double> [[COL_LOAD13]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT58:%.*]] = insertelement <1 x double> undef, double [[TMP78]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT59:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT58]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP79:%.*]] = fmul <1 x double> [[BLOCK57]], [[SPLAT_SPLAT59]]
+; CHECK-NEXT:    [[BLOCK60:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP80:%.*]] = extractelement <3 x double> [[COL_LOAD13]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT61:%.*]] = insertelement <1 x double> undef, double [[TMP80]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT62:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT61]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP81:%.*]] = fmul <1 x double> [[BLOCK60]], [[SPLAT_SPLAT62]]
+; CHECK-NEXT:    [[TMP82:%.*]] = fadd <1 x double> [[TMP79]], [[TMP81]]
+; CHECK-NEXT:    [[BLOCK63:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP83:%.*]] = extractelement <3 x double> [[COL_LOAD13]], i64 2
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT64:%.*]] = insertelement <1 x double> undef, double [[TMP83]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT65:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT64]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP84:%.*]] = fmul <1 x double> [[BLOCK63]], [[SPLAT_SPLAT65]]
+; CHECK-NEXT:    [[TMP85:%.*]] = fadd <1 x double> [[TMP82]], [[TMP84]]
+; CHECK-NEXT:    [[TMP86:%.*]] = shufflevector <1 x double> [[TMP85]], <1 x double> undef, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP87:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP86]], <3 x i32> <i32 3, i32 1, i32 2>
+; CHECK-NEXT:    [[BLOCK66:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP88:%.*]] = extractelement <3 x double> [[COL_LOAD13]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT67:%.*]] = insertelement <1 x double> undef, double [[TMP88]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT68:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT67]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP89:%.*]] = fmul <1 x double> [[BLOCK66]], [[SPLAT_SPLAT68]]
+; CHECK-NEXT:    [[BLOCK69:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP90:%.*]] = extractelement <3 x double> [[COL_LOAD13]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT70:%.*]] = insertelement <1 x double> undef, double [[TMP90]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT71:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT70]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP91:%.*]] = fmul <1 x double> [[BLOCK69]], [[SPLAT_SPLAT71]]
+; CHECK-NEXT:    [[TMP92:%.*]] = fadd <1 x double> [[TMP89]], [[TMP91]]
+; CHECK-NEXT:    [[BLOCK72:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP93:%.*]] = extractelement <3 x double> [[COL_LOAD13]], i64 2
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT73:%.*]] = insertelement <1 x double> undef, double [[TMP93]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT74:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT73]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP94:%.*]] = fmul <1 x double> [[BLOCK72]], [[SPLAT_SPLAT74]]
+; CHECK-NEXT:    [[TMP95:%.*]] = fadd <1 x double> [[TMP92]], [[TMP94]]
+; CHECK-NEXT:    [[TMP96:%.*]] = shufflevector <1 x double> [[TMP95]], <1 x double> undef, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP97:%.*]] = shufflevector <3 x double> [[TMP87]], <3 x double> [[TMP96]], <3 x i32> <i32 0, i32 3, i32 2>
+; CHECK-NEXT:    [[BLOCK75:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> undef, <1 x i32> <i32 2>
+; CHECK-NEXT:    [[TMP98:%.*]] = extractelement <3 x double> [[COL_LOAD13]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT76:%.*]] = insertelement <1 x double> undef, double [[TMP98]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT77:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT76]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP99:%.*]] = fmul <1 x double> [[BLOCK75]], [[SPLAT_SPLAT77]]
+; CHECK-NEXT:    [[BLOCK78:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> undef, <1 x i32> <i32 2>
+; CHECK-NEXT:    [[TMP100:%.*]] = extractelement <3 x double> [[COL_LOAD13]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT79:%.*]] = insertelement <1 x double> undef, double [[TMP100]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT80:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT79]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP101:%.*]] = fmul <1 x double> [[BLOCK78]], [[SPLAT_SPLAT80]]
+; CHECK-NEXT:    [[TMP102:%.*]] = fadd <1 x double> [[TMP99]], [[TMP101]]
+; CHECK-NEXT:    [[BLOCK81:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> undef, <1 x i32> <i32 2>
+; CHECK-NEXT:    [[TMP103:%.*]] = extractelement <3 x double> [[COL_LOAD13]], i64 2
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT82:%.*]] = insertelement <1 x double> undef, double [[TMP103]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT83:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT82]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP104:%.*]] = fmul <1 x double> [[BLOCK81]], [[SPLAT_SPLAT83]]
+; CHECK-NEXT:    [[TMP105:%.*]] = fadd <1 x double> [[TMP102]], [[TMP104]]
+
+;  Embed result of multiply into flat vector.
+
+; CHECK-NEXT:    [[TMP106:%.*]] = shufflevector <1 x double> [[TMP105]], <1 x double> undef, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP107:%.*]] = shufflevector <3 x double> [[TMP97]], <3 x double> [[TMP106]], <3 x i32> <i32 0, i32 1, i32 3>
+
+; Load %C.
+
+; CHECK-NEXT:    [[TMP110:%.*]] = bitcast <9 x double>* [[C_PTR:%.*]] to double*
+; CHECK-NEXT:    [[COL_CAST92:%.*]] = bitcast double* [[TMP110]] to <3 x double>*
+; CHECK-NEXT:    [[COL_LOAD93:%.*]] = load <3 x double>, <3 x double>* [[COL_CAST92]], align 8
+; CHECK-NEXT:    [[COL_GEP94:%.*]] = getelementptr double, double* [[TMP110]], i32 3
+; CHECK-NEXT:    [[COL_CAST95:%.*]] = bitcast double* [[COL_GEP94]] to <3 x double>*
+; CHECK-NEXT:    [[COL_LOAD96:%.*]] = load <3 x double>, <3 x double>* [[COL_CAST95]], align 8
+; CHECK-NEXT:    [[COL_GEP97:%.*]] = getelementptr double, double* [[TMP110]], i32 6
+; CHECK-NEXT:    [[COL_CAST98:%.*]] = bitcast double* [[COL_GEP97]] to <3 x double>*
+; CHECK-NEXT:    [[COL_LOAD99:%.*]] = load <3 x double>, <3 x double>* [[COL_CAST98]], align 8
+
+; Add column vectors.
+
+; CHECK-NEXT:    [[TMP108:%.*]] = fadd <3 x double> [[COL_LOAD93]], [[TMP47]]
+; CHECK-NEXT:    [[TMP109:%.*]] = fadd <3 x double> [[COL_LOAD96]], [[TMP77]]
+; CHECK-NEXT:    [[TMP110:%.*]] = fadd <3 x double> [[COL_LOAD99]], [[TMP107]]
+
+; Store result columns.
+
+; CHECK-NEXT:    [[TMP111:%.*]] = bitcast <9 x double>* [[C_PTR]] to double*
+; CHECK-NEXT:    [[TMP112:%.*]] = bitcast double* [[TMP111]] to <3 x double>*
+; CHECK-NEXT:    store <3 x double> [[TMP108]], <3 x double>* [[TMP112]], align 8
+; CHECK-NEXT:    [[TMP113:%.*]] = getelementptr double, double* [[TMP111]], i32 3
+; CHECK-NEXT:    [[TMP114:%.*]] = bitcast double* [[TMP113]] to <3 x double>*
+; CHECK-NEXT:    store <3 x double> [[TMP109]], <3 x double>* [[TMP114]], align 8
+; CHECK-NEXT:    [[TMP115:%.*]] = getelementptr double, double* [[TMP111]], i32 6
+; CHECK-NEXT:    [[TMP116:%.*]] = bitcast double* [[TMP115]] to <3 x double>*
+; CHECK-NEXT:    store <3 x double> [[TMP110]], <3 x double>* [[TMP116]], align 8
+; CHECK-NEXT:    ret void
+;
+entry:
+  %a = load <9 x double>, <9 x double>* %A.Ptr
+  %b = load <9 x double>, <9 x double>* %B.Ptr
+  %a.trans  = call <9 x double> @llvm.matrix.transpose(<9 x double> %a, i32 3, i32 3)
+  %mult = call <9 x double> @llvm.matrix.multiply.v9f64.v9f64.v9f64(<9 x double> %a.trans, <9 x double> %b, i32 3, i32 3, i32 3)
+  %c = load <9 x double>, <9 x double>* %C.Ptr
+  %res = fadd <9 x double> %c, %mult
+
+  store <9 x double> %res, <9 x double>* %C.Ptr
+  ret void
+}
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-double-contraction-fmf.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-double-contraction-fmf.ll
new file mode 100644
index 000000000..1b1e8dccc
--- /dev/null
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-double-contraction-fmf.ll
@@ -0,0 +1,69 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -lower-matrix-intrinsics -S < %s | FileCheck %s
+; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s
+
+
+define <4 x double> @multiply_2x2(<4 x double> %a, <4 x double> %b) {
+; CHECK-LABEL: @multiply_2x2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <4 x double> [[A]], <4 x double> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[SPLIT2:%.*]] = shufflevector <4 x double> [[B:%.*]], <4 x double> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[SPLIT3:%.*]] = shufflevector <4 x double> [[B]], <4 x double> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[BLOCK:%.*]] = shufflevector <2 x double> [[SPLIT]], <2 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <2 x double> [[SPLIT2]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <1 x double> undef, double [[TMP0]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul <1 x double> [[BLOCK]], [[SPLAT_SPLAT]]
+; CHECK-NEXT:    [[BLOCK4:%.*]] = shufflevector <2 x double> [[SPLIT1]], <2 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x double> [[SPLIT2]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT5:%.*]] = insertelement <1 x double> undef, double [[TMP2]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT6:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT5]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = call <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[BLOCK4]], <1 x double> [[SPLAT_SPLAT6]], <1 x double> [[TMP1]])
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <1 x double> [[TMP3]], <1 x double> undef, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <2 x double> undef, <2 x double> [[TMP4]], <2 x i32> <i32 2, i32 1>
+; CHECK-NEXT:    [[BLOCK7:%.*]] = shufflevector <2 x double> [[SPLIT]], <2 x double> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x double> [[SPLIT2]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT8:%.*]] = insertelement <1 x double> undef, double [[TMP6]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT9:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT8]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = fmul <1 x double> [[BLOCK7]], [[SPLAT_SPLAT9]]
+; CHECK-NEXT:    [[BLOCK10:%.*]] = shufflevector <2 x double> [[SPLIT1]], <2 x double> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x double> [[SPLIT2]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT11:%.*]] = insertelement <1 x double> undef, double [[TMP8]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT12:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT11]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = call <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[BLOCK10]], <1 x double> [[SPLAT_SPLAT12]], <1 x double> [[TMP7]])
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <1 x double> [[TMP9]], <1 x double> undef, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <2 x double> [[TMP5]], <2 x double> [[TMP10]], <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:    [[BLOCK13:%.*]] = shufflevector <2 x double> [[SPLIT]], <2 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <2 x double> [[SPLIT3]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT14:%.*]] = insertelement <1 x double> undef, double [[TMP12]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT15:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT14]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = fmul <1 x double> [[BLOCK13]], [[SPLAT_SPLAT15]]
+; CHECK-NEXT:    [[BLOCK16:%.*]] = shufflevector <2 x double> [[SPLIT1]], <2 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x double> [[SPLIT3]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT17:%.*]] = insertelement <1 x double> undef, double [[TMP14]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT18:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT17]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = call <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[BLOCK16]], <1 x double> [[SPLAT_SPLAT18]], <1 x double> [[TMP13]])
+; CHECK-NEXT:    [[TMP16:%.*]] = shufflevector <1 x double> [[TMP15]], <1 x double> undef, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP17:%.*]] = shufflevector <2 x double> undef, <2 x double> [[TMP16]], <2 x i32> <i32 2, i32 1>
+; CHECK-NEXT:    [[BLOCK19:%.*]] = shufflevector <2 x double> [[SPLIT]], <2 x double> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <2 x double> [[SPLIT3]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT20:%.*]] = insertelement <1 x double> undef, double [[TMP18]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT21:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT20]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP19:%.*]] = fmul <1 x double> [[BLOCK19]], [[SPLAT_SPLAT21]]
+; CHECK-NEXT:    [[BLOCK22:%.*]] = shufflevector <2 x double> [[SPLIT1]], <2 x double> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <2 x double> [[SPLIT3]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT23:%.*]] = insertelement <1 x double> undef, double [[TMP20]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT24:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT23]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP21:%.*]] = call <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[BLOCK22]], <1 x double> [[SPLAT_SPLAT24]], <1 x double> [[TMP19]])
+; CHECK-NEXT:    [[TMP22:%.*]] = shufflevector <1 x double> [[TMP21]], <1 x double> undef, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP23:%.*]] = shufflevector <2 x double> [[TMP17]], <2 x double> [[TMP22]], <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:    [[TMP24:%.*]] = shufflevector <2 x double> [[TMP11]], <2 x double> [[TMP23]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    ret <4 x double> [[TMP24]]
+;
+entry:
+  %c = call contract <4 x double> @llvm.matrix.multiply.v4f64.v4f64.v4f64(<4 x double> %a, <4 x double> %b, i32 2, i32 2, i32 2)
+  ret <4 x double> %c
+}
+
+declare <4 x double> @llvm.matrix.multiply.v4f64.v4f64.v4f64(<4 x double>, <4 x double>, i32, i32, i32)
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-double-contraction.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-double-contraction.ll
new file mode 100644
index 000000000..624599682
--- /dev/null
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-double-contraction.ll
@@ -0,0 +1,69 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -lower-matrix-intrinsics -matrix-allow-contract -S < %s | FileCheck %s
+; RUN: opt -passes='lower-matrix-intrinsics' -matrix-allow-contract -S < %s | FileCheck %s
+
+
+define <4 x double> @multiply_2x2(<4 x double> %a, <4 x double> %b) {
+; CHECK-LABEL: @multiply_2x2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <4 x double> [[A]], <4 x double> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[SPLIT2:%.*]] = shufflevector <4 x double> [[B:%.*]], <4 x double> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[SPLIT3:%.*]] = shufflevector <4 x double> [[B]], <4 x double> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[BLOCK:%.*]] = shufflevector <2 x double> [[SPLIT]], <2 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <2 x double> [[SPLIT2]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <1 x double> undef, double [[TMP0]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul <1 x double> [[BLOCK]], [[SPLAT_SPLAT]]
+; CHECK-NEXT:    [[BLOCK4:%.*]] = shufflevector <2 x double> [[SPLIT1]], <2 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x double> [[SPLIT2]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT5:%.*]] = insertelement <1 x double> undef, double [[TMP2]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT6:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT5]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = call <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[BLOCK4]], <1 x double> [[SPLAT_SPLAT6]], <1 x double> [[TMP1]])
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <1 x double> [[TMP3]], <1 x double> undef, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <2 x double> undef, <2 x double> [[TMP4]], <2 x i32> <i32 2, i32 1>
+; CHECK-NEXT:    [[BLOCK7:%.*]] = shufflevector <2 x double> [[SPLIT]], <2 x double> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x double> [[SPLIT2]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT8:%.*]] = insertelement <1 x double> undef, double [[TMP6]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT9:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT8]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = fmul <1 x double> [[BLOCK7]], [[SPLAT_SPLAT9]]
+; CHECK-NEXT:    [[BLOCK10:%.*]] = shufflevector <2 x double> [[SPLIT1]], <2 x double> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x double> [[SPLIT2]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT11:%.*]] = insertelement <1 x double> undef, double [[TMP8]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT12:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT11]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = call <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[BLOCK10]], <1 x double> [[SPLAT_SPLAT12]], <1 x double> [[TMP7]])
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <1 x double> [[TMP9]], <1 x double> undef, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <2 x double> [[TMP5]], <2 x double> [[TMP10]], <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:    [[BLOCK13:%.*]] = shufflevector <2 x double> [[SPLIT]], <2 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <2 x double> [[SPLIT3]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT14:%.*]] = insertelement <1 x double> undef, double [[TMP12]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT15:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT14]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = fmul <1 x double> [[BLOCK13]], [[SPLAT_SPLAT15]]
+; CHECK-NEXT:    [[BLOCK16:%.*]] = shufflevector <2 x double> [[SPLIT1]], <2 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x double> [[SPLIT3]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT17:%.*]] = insertelement <1 x double> undef, double [[TMP14]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT18:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT17]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = call <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[BLOCK16]], <1 x double> [[SPLAT_SPLAT18]], <1 x double> [[TMP13]])
+; CHECK-NEXT:    [[TMP16:%.*]] = shufflevector <1 x double> [[TMP15]], <1 x double> undef, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP17:%.*]] = shufflevector <2 x double> undef, <2 x double> [[TMP16]], <2 x i32> <i32 2, i32 1>
+; CHECK-NEXT:    [[BLOCK19:%.*]] = shufflevector <2 x double> [[SPLIT]], <2 x double> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <2 x double> [[SPLIT3]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT20:%.*]] = insertelement <1 x double> undef, double [[TMP18]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT21:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT20]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP19:%.*]] = fmul <1 x double> [[BLOCK19]], [[SPLAT_SPLAT21]]
+; CHECK-NEXT:    [[BLOCK22:%.*]] = shufflevector <2 x double> [[SPLIT1]], <2 x double> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <2 x double> [[SPLIT3]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT23:%.*]] = insertelement <1 x double> undef, double [[TMP20]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT24:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT23]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP21:%.*]] = call <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[BLOCK22]], <1 x double> [[SPLAT_SPLAT24]], <1 x double> [[TMP19]])
+; CHECK-NEXT:    [[TMP22:%.*]] = shufflevector <1 x double> [[TMP21]], <1 x double> undef, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP23:%.*]] = shufflevector <2 x double> [[TMP17]], <2 x double> [[TMP22]], <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:    [[TMP24:%.*]] = shufflevector <2 x double> [[TMP11]], <2 x double> [[TMP23]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    ret <4 x double> [[TMP24]]
+;
+entry:
+  %c = call <4 x double> @llvm.matrix.multiply.v4f64.v4f64.v4f64(<4 x double> %a, <4 x double> %b, i32 2, i32 2, i32 2)
+  ret <4 x double> %c
+}
+
+declare <4 x double> @llvm.matrix.multiply.v4f64.v4f64.v4f64(<4 x double>, <4 x double>, i32, i32, i32)
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-double.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-double.ll
new file mode 100644
index 000000000..4e2e17e77
--- /dev/null
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-double.ll
@@ -0,0 +1,254 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -lower-matrix-intrinsics -S < %s | FileCheck %s
+; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s
+
+
+define <4 x double> @multiply_2x2(<4 x double> %a, <4 x double> %b) {
+; CHECK-LABEL: @multiply_2x2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <4 x double> [[A]], <4 x double> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[SPLIT2:%.*]] = shufflevector <4 x double> [[B:%.*]], <4 x double> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[SPLIT3:%.*]] = shufflevector <4 x double> [[B]], <4 x double> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[BLOCK:%.*]] = shufflevector <2 x double> [[SPLIT]], <2 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <2 x double> [[SPLIT2]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <1 x double> undef, double [[TMP0]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul <1 x double> [[BLOCK]], [[SPLAT_SPLAT]]
+; CHECK-NEXT:    [[BLOCK4:%.*]] = shufflevector <2 x double> [[SPLIT1]], <2 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x double> [[SPLIT2]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT5:%.*]] = insertelement <1 x double> undef, double [[TMP2]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT6:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT5]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = fmul <1 x double> [[BLOCK4]], [[SPLAT_SPLAT6]]
+; CHECK-NEXT:    [[TMP4:%.*]] = fadd <1 x double> [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <1 x double> [[TMP4]], <1 x double> undef, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <2 x double> undef, <2 x double> [[TMP5]], <2 x i32> <i32 2, i32 1>
+; CHECK-NEXT:    [[BLOCK7:%.*]] = shufflevector <2 x double> [[SPLIT]], <2 x double> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x double> [[SPLIT2]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT8:%.*]] = insertelement <1 x double> undef, double [[TMP7]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT9:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT8]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = fmul <1 x double> [[BLOCK7]], [[SPLAT_SPLAT9]]
+; CHECK-NEXT:    [[BLOCK10:%.*]] = shufflevector <2 x double> [[SPLIT1]], <2 x double> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <2 x double> [[SPLIT2]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT11:%.*]] = insertelement <1 x double> undef, double [[TMP9]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT12:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT11]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = fmul <1 x double> [[BLOCK10]], [[SPLAT_SPLAT12]]
+; CHECK-NEXT:    [[TMP11:%.*]] = fadd <1 x double> [[TMP8]], [[TMP10]]
+; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <1 x double> [[TMP11]], <1 x double> undef, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <2 x double> [[TMP6]], <2 x double> [[TMP12]], <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:    [[BLOCK13:%.*]] = shufflevector <2 x double> [[SPLIT]], <2 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x double> [[SPLIT3]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT14:%.*]] = insertelement <1 x double> undef, double [[TMP14]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT15:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT14]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = fmul <1 x double> [[BLOCK13]], [[SPLAT_SPLAT15]]
+; CHECK-NEXT:    [[BLOCK16:%.*]] = shufflevector <2 x double> [[SPLIT1]], <2 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <2 x double> [[SPLIT3]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT17:%.*]] = insertelement <1 x double> undef, double [[TMP16]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT18:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT17]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = fmul <1 x double> [[BLOCK16]], [[SPLAT_SPLAT18]]
+; CHECK-NEXT:    [[TMP18:%.*]] = fadd <1 x double> [[TMP15]], [[TMP17]]
+; CHECK-NEXT:    [[TMP19:%.*]] = shufflevector <1 x double> [[TMP18]], <1 x double> undef, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP20:%.*]] = shufflevector <2 x double> undef, <2 x double> [[TMP19]], <2 x i32> <i32 2, i32 1>
+; CHECK-NEXT:    [[BLOCK19:%.*]] = shufflevector <2 x double> [[SPLIT]], <2 x double> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <2 x double> [[SPLIT3]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT20:%.*]] = insertelement <1 x double> undef, double [[TMP21]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT21:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT20]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP22:%.*]] = fmul <1 x double> [[BLOCK19]], [[SPLAT_SPLAT21]]
+; CHECK-NEXT:    [[BLOCK22:%.*]] = shufflevector <2 x double> [[SPLIT1]], <2 x double> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <2 x double> [[SPLIT3]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT23:%.*]] = insertelement <1 x double> undef, double [[TMP23]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT24:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT23]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = fmul <1 x double> [[BLOCK22]], [[SPLAT_SPLAT24]]
+; CHECK-NEXT:    [[TMP25:%.*]] = fadd <1 x double> [[TMP22]], [[TMP24]]
+; CHECK-NEXT:    [[TMP26:%.*]] = shufflevector <1 x double> [[TMP25]], <1 x double> undef, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP27:%.*]] = shufflevector <2 x double> [[TMP20]], <2 x double> [[TMP26]], <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:    [[TMP28:%.*]] = shufflevector <2 x double> [[TMP13]], <2 x double> [[TMP27]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    ret <4 x double> [[TMP28]]
+;
+entry:
+  %c = call <4 x double> @llvm.matrix.multiply.v4f64.v4f64.v4f64(<4 x double> %a, <4 x double> %b, i32 2, i32 2, i32 2)
+  ret <4 x double> %c
+}
+
+declare <4 x double> @llvm.matrix.multiply.v4f64.v4f64.v4f64(<4 x double>, <4 x double>, i32, i32, i32)
+
+define <4 x double> @multiply_1x2(<2 x double> %a, <2 x double> %b) {
+; CHECK-LABEL: @multiply_1x2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <2 x double> [[A:%.*]], <2 x double> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <2 x double> [[B:%.*]], <2 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[SPLIT2:%.*]] = shufflevector <2 x double> [[B]], <2 x double> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[BLOCK:%.*]] = shufflevector <2 x double> [[SPLIT]], <2 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <1 x double> [[SPLIT1]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <1 x double> undef, double [[TMP0]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul <1 x double> [[BLOCK]], [[SPLAT_SPLAT]]
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <1 x double> [[TMP1]], <1 x double> undef, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <2 x double> undef, <2 x double> [[TMP2]], <2 x i32> <i32 2, i32 1>
+; CHECK-NEXT:    [[BLOCK3:%.*]] = shufflevector <2 x double> [[SPLIT]], <2 x double> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <1 x double> [[SPLIT1]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT4:%.*]] = insertelement <1 x double> undef, double [[TMP4]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT5:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT4]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = fmul <1 x double> [[BLOCK3]], [[SPLAT_SPLAT5]]
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <1 x double> [[TMP5]], <1 x double> undef, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> [[TMP6]], <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:    [[BLOCK6:%.*]] = shufflevector <2 x double> [[SPLIT]], <2 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <1 x double> [[SPLIT2]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT7:%.*]] = insertelement <1 x double> undef, double [[TMP8]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT8:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT7]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = fmul <1 x double> [[BLOCK6]], [[SPLAT_SPLAT8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <1 x double> [[TMP9]], <1 x double> undef, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <2 x double> undef, <2 x double> [[TMP10]], <2 x i32> <i32 2, i32 1>
+; CHECK-NEXT:    [[BLOCK9:%.*]] = shufflevector <2 x double> [[SPLIT]], <2 x double> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <1 x double> [[SPLIT2]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT10:%.*]] = insertelement <1 x double> undef, double [[TMP12]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT11:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT10]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = fmul <1 x double> [[BLOCK9]], [[SPLAT_SPLAT11]]
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <1 x double> [[TMP13]], <1 x double> undef, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP15:%.*]] = shufflevector <2 x double> [[TMP11]], <2 x double> [[TMP14]], <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:    [[TMP16:%.*]] = shufflevector <2 x double> [[TMP7]], <2 x double> [[TMP15]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    ret <4 x double> [[TMP16]]
+;
+entry:
+  %c = call <4 x double> @llvm.matrix.multiply.v4f64.v2f64.v2f64(<2 x double> %a, <2 x double> %b, i32 2, i32 1, i32 2)
+  ret <4 x double> %c
+}
+
+declare <4 x double> @llvm.matrix.multiply.v4f64.v2f64.v2f64(<2 x double>, <2 x double>, i32, i32, i32)
+
+define <9 x double> @multiply_2x3(<6 x double> %a, <6 x double> %b) {
+; CHECK-LABEL: @multiply_2x3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <6 x double> [[A:%.*]], <6 x double> undef, <3 x i32> <i32 0, i32 1, i32 2>
+; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <6 x double> [[A]], <6 x double> undef, <3 x i32> <i32 3, i32 4, i32 5>
+; CHECK-NEXT:    [[SPLIT2:%.*]] = shufflevector <6 x double> [[B:%.*]], <6 x double> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[SPLIT3:%.*]] = shufflevector <6 x double> [[B]], <6 x double> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[SPLIT4:%.*]] = shufflevector <6 x double> [[B]], <6 x double> undef, <2 x i32> <i32 4, i32 5>
+; CHECK-NEXT:    [[BLOCK:%.*]] = shufflevector <3 x double> [[SPLIT]], <3 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <2 x double> [[SPLIT2]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <1 x double> undef, double [[TMP0]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul <1 x double> [[BLOCK]], [[SPLAT_SPLAT]]
+; CHECK-NEXT:    [[BLOCK5:%.*]] = shufflevector <3 x double> [[SPLIT1]], <3 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x double> [[SPLIT2]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT6:%.*]] = insertelement <1 x double> undef, double [[TMP2]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT7:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT6]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = fmul <1 x double> [[BLOCK5]], [[SPLAT_SPLAT7]]
+; CHECK-NEXT:    [[TMP4:%.*]] = fadd <1 x double> [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <1 x double> [[TMP4]], <1 x double> undef, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP5]], <3 x i32> <i32 3, i32 1, i32 2>
+; CHECK-NEXT:    [[BLOCK8:%.*]] = shufflevector <3 x double> [[SPLIT]], <3 x double> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x double> [[SPLIT2]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT9:%.*]] = insertelement <1 x double> undef, double [[TMP7]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT10:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT9]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = fmul <1 x double> [[BLOCK8]], [[SPLAT_SPLAT10]]
+; CHECK-NEXT:    [[BLOCK11:%.*]] = shufflevector <3 x double> [[SPLIT1]], <3 x double> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <2 x double> [[SPLIT2]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT12:%.*]] = insertelement <1 x double> undef, double [[TMP9]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT13:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT12]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = fmul <1 x double> [[BLOCK11]], [[SPLAT_SPLAT13]]
+; CHECK-NEXT:    [[TMP11:%.*]] = fadd <1 x double> [[TMP8]], [[TMP10]]
+; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <1 x double> [[TMP11]], <1 x double> undef, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <3 x double> [[TMP6]], <3 x double> [[TMP12]], <3 x i32> <i32 0, i32 3, i32 2>
+; CHECK-NEXT:    [[BLOCK14:%.*]] = shufflevector <3 x double> [[SPLIT]], <3 x double> undef, <1 x i32> <i32 2>
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x double> [[SPLIT2]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT15:%.*]] = insertelement <1 x double> undef, double [[TMP14]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT16:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT15]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = fmul <1 x double> [[BLOCK14]], [[SPLAT_SPLAT16]]
+; CHECK-NEXT:    [[BLOCK17:%.*]] = shufflevector <3 x double> [[SPLIT1]], <3 x double> undef, <1 x i32> <i32 2>
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <2 x double> [[SPLIT2]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT18:%.*]] = insertelement <1 x double> undef, double [[TMP16]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT19:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT18]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = fmul <1 x double> [[BLOCK17]], [[SPLAT_SPLAT19]]
+; CHECK-NEXT:    [[TMP18:%.*]] = fadd <1 x double> [[TMP15]], [[TMP17]]
+; CHECK-NEXT:    [[TMP19:%.*]] = shufflevector <1 x double> [[TMP18]], <1 x double> undef, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP20:%.*]] = shufflevector <3 x double> [[TMP13]], <3 x double> [[TMP19]], <3 x i32> <i32 0, i32 1, i32 3>
+; CHECK-NEXT:    [[BLOCK20:%.*]] = shufflevector <3 x double> [[SPLIT]], <3 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <2 x double> [[SPLIT3]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT21:%.*]] = insertelement <1 x double> undef, double [[TMP21]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT22:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT21]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP22:%.*]] = fmul <1 x double> [[BLOCK20]], [[SPLAT_SPLAT22]]
+; CHECK-NEXT:    [[BLOCK23:%.*]] = shufflevector <3 x double> [[SPLIT1]], <3 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <2 x double> [[SPLIT3]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT24:%.*]] = insertelement <1 x double> undef, double [[TMP23]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT25:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT24]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = fmul <1 x double> [[BLOCK23]], [[SPLAT_SPLAT25]]
+; CHECK-NEXT:    [[TMP25:%.*]] = fadd <1 x double> [[TMP22]], [[TMP24]]
+; CHECK-NEXT:    [[TMP26:%.*]] = shufflevector <1 x double> [[TMP25]], <1 x double> undef, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP27:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP26]], <3 x i32> <i32 3, i32 1, i32 2>
+; CHECK-NEXT:    [[BLOCK26:%.*]] = shufflevector <3 x double> [[SPLIT]], <3 x double> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP28:%.*]] = extractelement <2 x double> [[SPLIT3]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT27:%.*]] = insertelement <1 x double> undef, double [[TMP28]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT28:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT27]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP29:%.*]] = fmul <1 x double> [[BLOCK26]], [[SPLAT_SPLAT28]]
+; CHECK-NEXT:    [[BLOCK29:%.*]] = shufflevector <3 x double> [[SPLIT1]], <3 x double> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP30:%.*]] = extractelement <2 x double> [[SPLIT3]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT30:%.*]] = insertelement <1 x double> undef, double [[TMP30]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT31:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT30]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP31:%.*]] = fmul <1 x double> [[BLOCK29]], [[SPLAT_SPLAT31]]
+; CHECK-NEXT:    [[TMP32:%.*]] = fadd <1 x double> [[TMP29]], [[TMP31]]
+; CHECK-NEXT:    [[TMP33:%.*]] = shufflevector <1 x double> [[TMP32]], <1 x double> undef, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP34:%.*]] = shufflevector <3 x double> [[TMP27]], <3 x double> [[TMP33]], <3 x i32> <i32 0, i32 3, i32 2>
+; CHECK-NEXT:    [[BLOCK32:%.*]] = shufflevector <3 x double> [[SPLIT]], <3 x double> undef, <1 x i32> <i32 2>
+; CHECK-NEXT:    [[TMP35:%.*]] = extractelement <2 x double> [[SPLIT3]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT33:%.*]] = insertelement <1 x double> undef, double [[TMP35]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT34:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT33]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP36:%.*]] = fmul <1 x double> [[BLOCK32]], [[SPLAT_SPLAT34]]
+; CHECK-NEXT:    [[BLOCK35:%.*]] = shufflevector <3 x double> [[SPLIT1]], <3 x double> undef, <1 x i32> <i32 2>
+; CHECK-NEXT:    [[TMP37:%.*]] = extractelement <2 x double> [[SPLIT3]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT36:%.*]] = insertelement <1 x double> undef, double [[TMP37]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT37:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT36]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP38:%.*]] = fmul <1 x double> [[BLOCK35]], [[SPLAT_SPLAT37]]
+; CHECK-NEXT:    [[TMP39:%.*]] = fadd <1 x double> [[TMP36]], [[TMP38]]
+; CHECK-NEXT:    [[TMP40:%.*]] = shufflevector <1 x double> [[TMP39]], <1 x double> undef, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP41:%.*]] = shufflevector <3 x double> [[TMP34]], <3 x double> [[TMP40]], <3 x i32> <i32 0, i32 1, i32 3>
+; CHECK-NEXT:    [[BLOCK38:%.*]] = shufflevector <3 x double> [[SPLIT]], <3 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP42:%.*]] = extractelement <2 x double> [[SPLIT4]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT39:%.*]] = insertelement <1 x double> undef, double [[TMP42]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT40:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT39]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP43:%.*]] = fmul <1 x double> [[BLOCK38]], [[SPLAT_SPLAT40]]
+; CHECK-NEXT:    [[BLOCK41:%.*]] = shufflevector <3 x double> [[SPLIT1]], <3 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP44:%.*]] = extractelement <2 x double> [[SPLIT4]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT42:%.*]] = insertelement <1 x double> undef, double [[TMP44]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT43:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT42]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP45:%.*]] = fmul <1 x double> [[BLOCK41]], [[SPLAT_SPLAT43]]
+; CHECK-NEXT:    [[TMP46:%.*]] = fadd <1 x double> [[TMP43]], [[TMP45]]
+; CHECK-NEXT:    [[TMP47:%.*]] = shufflevector <1 x double> [[TMP46]], <1 x double> undef, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP48:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP47]], <3 x i32> <i32 3, i32 1, i32 2>
+; CHECK-NEXT:    [[BLOCK44:%.*]] = shufflevector <3 x double> [[SPLIT]], <3 x double> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP49:%.*]] = extractelement <2 x double> [[SPLIT4]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT45:%.*]] = insertelement <1 x double> undef, double [[TMP49]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT46:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT45]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP50:%.*]] = fmul <1 x double> [[BLOCK44]], [[SPLAT_SPLAT46]]
+; CHECK-NEXT:    [[BLOCK47:%.*]] = shufflevector <3 x double> [[SPLIT1]], <3 x double> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP51:%.*]] = extractelement <2 x double> [[SPLIT4]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT48:%.*]] = insertelement <1 x double> undef, double [[TMP51]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT49:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT48]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP52:%.*]] = fmul <1 x double> [[BLOCK47]], [[SPLAT_SPLAT49]]
+; CHECK-NEXT:    [[TMP53:%.*]] = fadd <1 x double> [[TMP50]], [[TMP52]]
+; CHECK-NEXT:    [[TMP54:%.*]] = shufflevector <1 x double> [[TMP53]], <1 x double> undef, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP55:%.*]] = shufflevector <3 x double> [[TMP48]], <3 x double> [[TMP54]], <3 x i32> <i32 0, i32 3, i32 2>
+; CHECK-NEXT:    [[BLOCK50:%.*]] = shufflevector <3 x double> [[SPLIT]], <3 x double> undef, <1 x i32> <i32 2>
+; CHECK-NEXT:    [[TMP56:%.*]] = extractelement <2 x double> [[SPLIT4]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT51:%.*]] = insertelement <1 x double> undef, double [[TMP56]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT52:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT51]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP57:%.*]] = fmul <1 x double> [[BLOCK50]], [[SPLAT_SPLAT52]]
+; CHECK-NEXT:    [[BLOCK53:%.*]] = shufflevector <3 x double> [[SPLIT1]], <3 x double> undef, <1 x i32> <i32 2>
+; CHECK-NEXT:    [[TMP58:%.*]] = extractelement <2 x double> [[SPLIT4]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT54:%.*]] = insertelement <1 x double> undef, double [[TMP58]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT55:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT54]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP59:%.*]] = fmul <1 x double> [[BLOCK53]], [[SPLAT_SPLAT55]]
+; CHECK-NEXT:    [[TMP60:%.*]] = fadd <1 x double> [[TMP57]], [[TMP59]]
+; CHECK-NEXT:    [[TMP61:%.*]] = shufflevector <1 x double> [[TMP60]], <1 x double> undef, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP62:%.*]] = shufflevector <3 x double> [[TMP55]], <3 x double> [[TMP61]], <3 x i32> <i32 0, i32 1, i32 3>
+; CHECK-NEXT:    [[TMP63:%.*]] = shufflevector <3 x double> [[TMP20]], <3 x double> [[TMP41]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
+; CHECK-NEXT:    [[TMP64:%.*]] = shufflevector <3 x double> [[TMP62]], <3 x double> undef, <6 x i32> <i32 0, i32 1, i32 2, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP65:%.*]] = shufflevector <6 x double> [[TMP63]], <6 x double> [[TMP64]], <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
+; CHECK-NEXT:    ret <9 x double> [[TMP65]]
+;
+entry:
+  %c = call <9 x double> @llvm.matrix.multiply.v6f64.v6f64.v6f64(<6 x double> %a, <6 x double> %b, i32 3, i32 2, i32 3)
+  ret <9 x double> %c
+}
+
+declare <9 x double> @llvm.matrix.multiply.v6f64.v6f64.v6f64(<6 x double>, <6 x double>, i32, i32, i32)
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-float-contraction-fmf.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-float-contraction-fmf.ll
new file mode 100644
index 000000000..f9ac72937
--- /dev/null
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-float-contraction-fmf.ll
@@ -0,0 +1,69 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -lower-matrix-intrinsics -S < %s | FileCheck %s
+; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s
+
+
+define <4 x float> @multiply_2x2(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: @multiply_2x2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <4 x float> [[A:%.*]], <4 x float> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <4 x float> [[A]], <4 x float> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[SPLIT2:%.*]] = shufflevector <4 x float> [[B:%.*]], <4 x float> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[SPLIT3:%.*]] = shufflevector <4 x float> [[B]], <4 x float> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[BLOCK:%.*]] = shufflevector <2 x float> [[SPLIT]], <2 x float> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <2 x float> [[SPLIT2]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <1 x float> undef, float [[TMP0]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT]], <1 x float> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul <1 x float> [[BLOCK]], [[SPLAT_SPLAT]]
+; CHECK-NEXT:    [[BLOCK4:%.*]] = shufflevector <2 x float> [[SPLIT1]], <2 x float> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x float> [[SPLIT2]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT5:%.*]] = insertelement <1 x float> undef, float [[TMP2]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT6:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT5]], <1 x float> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = call <1 x float> @llvm.fmuladd.v1f32(<1 x float> [[BLOCK4]], <1 x float> [[SPLAT_SPLAT6]], <1 x float> [[TMP1]])
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <1 x float> [[TMP3]], <1 x float> undef, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> undef, <2 x float> [[TMP4]], <2 x i32> <i32 2, i32 1>
+; CHECK-NEXT:    [[BLOCK7:%.*]] = shufflevector <2 x float> [[SPLIT]], <2 x float> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x float> [[SPLIT2]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT8:%.*]] = insertelement <1 x float> undef, float [[TMP6]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT9:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT8]], <1 x float> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = fmul <1 x float> [[BLOCK7]], [[SPLAT_SPLAT9]]
+; CHECK-NEXT:    [[BLOCK10:%.*]] = shufflevector <2 x float> [[SPLIT1]], <2 x float> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x float> [[SPLIT2]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT11:%.*]] = insertelement <1 x float> undef, float [[TMP8]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT12:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT11]], <1 x float> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = call <1 x float> @llvm.fmuladd.v1f32(<1 x float> [[BLOCK10]], <1 x float> [[SPLAT_SPLAT12]], <1 x float> [[TMP7]])
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <1 x float> [[TMP9]], <1 x float> undef, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <2 x float> [[TMP5]], <2 x float> [[TMP10]], <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:    [[BLOCK13:%.*]] = shufflevector <2 x float> [[SPLIT]], <2 x float> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <2 x float> [[SPLIT3]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT14:%.*]] = insertelement <1 x float> undef, float [[TMP12]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT15:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT14]], <1 x float> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = fmul <1 x float> [[BLOCK13]], [[SPLAT_SPLAT15]]
+; CHECK-NEXT:    [[BLOCK16:%.*]] = shufflevector <2 x float> [[SPLIT1]], <2 x float> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x float> [[SPLIT3]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT17:%.*]] = insertelement <1 x float> undef, float [[TMP14]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT18:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT17]], <1 x float> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = call <1 x float> @llvm.fmuladd.v1f32(<1 x float> [[BLOCK16]], <1 x float> [[SPLAT_SPLAT18]], <1 x float> [[TMP13]])
+; CHECK-NEXT:    [[TMP16:%.*]] = shufflevector <1 x float> [[TMP15]], <1 x float> undef, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP17:%.*]] = shufflevector <2 x float> undef, <2 x float> [[TMP16]], <2 x i32> <i32 2, i32 1>
+; CHECK-NEXT:    [[BLOCK19:%.*]] = shufflevector <2 x float> [[SPLIT]], <2 x float> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <2 x float> [[SPLIT3]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT20:%.*]] = insertelement <1 x float> undef, float [[TMP18]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT21:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT20]], <1 x float> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP19:%.*]] = fmul <1 x float> [[BLOCK19]], [[SPLAT_SPLAT21]]
+; CHECK-NEXT:    [[BLOCK22:%.*]] = shufflevector <2 x float> [[SPLIT1]], <2 x float> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <2 x float> [[SPLIT3]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT23:%.*]] = insertelement <1 x float> undef, float [[TMP20]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT24:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT23]], <1 x float> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP21:%.*]] = call <1 x float> @llvm.fmuladd.v1f32(<1 x float> [[BLOCK22]], <1 x float> [[SPLAT_SPLAT24]], <1 x float> [[TMP19]])
+; CHECK-NEXT:    [[TMP22:%.*]] = shufflevector <1 x float> [[TMP21]], <1 x float> undef, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP23:%.*]] = shufflevector <2 x float> [[TMP17]], <2 x float> [[TMP22]], <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:    [[TMP24:%.*]] = shufflevector <2 x float> [[TMP11]], <2 x float> [[TMP23]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    ret <4 x float> [[TMP24]]
+;
+entry:
+  %c = call contract <4 x float> @llvm.matrix.multiply.v4f32.v4f32.v4f32(<4 x float> %a, <4 x float> %b, i32 2, i32 2, i32 2)
+  ret <4 x float> %c
+}
+
+declare <4 x float> @llvm.matrix.multiply.v4f32.v4f32.v4f32(<4 x float>, <4 x float>, i32, i32, i32)
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-float-contraction.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-float-contraction.ll
new file mode 100644
index 000000000..2e2cecfdd
--- /dev/null
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-float-contraction.ll
@@ -0,0 +1,69 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -lower-matrix-intrinsics -matrix-allow-contract -S < %s | FileCheck %s
+; RUN: opt -passes='lower-matrix-intrinsics' -matrix-allow-contract -S < %s | FileCheck %s
+
+
+define <4 x float> @multiply_2x2(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: @multiply_2x2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <4 x float> [[A:%.*]], <4 x float> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <4 x float> [[A]], <4 x float> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[SPLIT2:%.*]] = shufflevector <4 x float> [[B:%.*]], <4 x float> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[SPLIT3:%.*]] = shufflevector <4 x float> [[B]], <4 x float> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[BLOCK:%.*]] = shufflevector <2 x float> [[SPLIT]], <2 x float> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <2 x float> [[SPLIT2]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <1 x float> undef, float [[TMP0]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT]], <1 x float> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul <1 x float> [[BLOCK]], [[SPLAT_SPLAT]]
+; CHECK-NEXT:    [[BLOCK4:%.*]] = shufflevector <2 x float> [[SPLIT1]], <2 x float> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x float> [[SPLIT2]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT5:%.*]] = insertelement <1 x float> undef, float [[TMP2]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT6:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT5]], <1 x float> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = call <1 x float> @llvm.fmuladd.v1f32(<1 x float> [[BLOCK4]], <1 x float> [[SPLAT_SPLAT6]], <1 x float> [[TMP1]])
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <1 x float> [[TMP3]], <1 x float> undef, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> undef, <2 x float> [[TMP4]], <2 x i32> <i32 2, i32 1>
+; CHECK-NEXT:    [[BLOCK7:%.*]] = shufflevector <2 x float> [[SPLIT]], <2 x float> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x float> [[SPLIT2]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT8:%.*]] = insertelement <1 x float> undef, float [[TMP6]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT9:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT8]], <1 x float> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = fmul <1 x float> [[BLOCK7]], [[SPLAT_SPLAT9]]
+; CHECK-NEXT:    [[BLOCK10:%.*]] = shufflevector <2 x float> [[SPLIT1]], <2 x float> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x float> [[SPLIT2]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT11:%.*]] = insertelement <1 x float> undef, float [[TMP8]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT12:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT11]], <1 x float> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = call <1 x float> @llvm.fmuladd.v1f32(<1 x float> [[BLOCK10]], <1 x float> [[SPLAT_SPLAT12]], <1 x float> [[TMP7]])
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <1 x float> [[TMP9]], <1 x float> undef, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <2 x float> [[TMP5]], <2 x float> [[TMP10]], <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:    [[BLOCK13:%.*]] = shufflevector <2 x float> [[SPLIT]], <2 x float> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <2 x float> [[SPLIT3]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT14:%.*]] = insertelement <1 x float> undef, float [[TMP12]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT15:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT14]], <1 x float> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = fmul <1 x float> [[BLOCK13]], [[SPLAT_SPLAT15]]
+; CHECK-NEXT:    [[BLOCK16:%.*]] = shufflevector <2 x float> [[SPLIT1]], <2 x float> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x float> [[SPLIT3]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT17:%.*]] = insertelement <1 x float> undef, float [[TMP14]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT18:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT17]], <1 x float> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = call <1 x float> @llvm.fmuladd.v1f32(<1 x float> [[BLOCK16]], <1 x float> [[SPLAT_SPLAT18]], <1 x float> [[TMP13]])
+; CHECK-NEXT:    [[TMP16:%.*]] = shufflevector <1 x float> [[TMP15]], <1 x float> undef, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP17:%.*]] = shufflevector <2 x float> undef, <2 x float> [[TMP16]], <2 x i32> <i32 2, i32 1>
+; CHECK-NEXT:    [[BLOCK19:%.*]] = shufflevector <2 x float> [[SPLIT]], <2 x float> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <2 x float> [[SPLIT3]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT20:%.*]] = insertelement <1 x float> undef, float [[TMP18]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT21:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT20]], <1 x float> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP19:%.*]] = fmul <1 x float> [[BLOCK19]], [[SPLAT_SPLAT21]]
+; CHECK-NEXT:    [[BLOCK22:%.*]] = shufflevector <2 x float> [[SPLIT1]], <2 x float> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <2 x float> [[SPLIT3]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT23:%.*]] = insertelement <1 x float> undef, float [[TMP20]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT24:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT23]], <1 x float> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP21:%.*]] = call <1 x float> @llvm.fmuladd.v1f32(<1 x float> [[BLOCK22]], <1 x float> [[SPLAT_SPLAT24]], <1 x float> [[TMP19]])
+; CHECK-NEXT:    [[TMP22:%.*]] = shufflevector <1 x float> [[TMP21]], <1 x float> undef, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP23:%.*]] = shufflevector <2 x float> [[TMP17]], <2 x float> [[TMP22]], <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:    [[TMP24:%.*]] = shufflevector <2 x float> [[TMP11]], <2 x float> [[TMP23]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    ret <4 x float> [[TMP24]]
+;
+entry:
+  %c = call contract <4 x float> @llvm.matrix.multiply.v4f32.v4f32.v4f32(<4 x float> %a, <4 x float> %b, i32 2, i32 2, i32 2)
+  ret <4 x float> %c
+}
+
+declare <4 x float> @llvm.matrix.multiply.v4f32.v4f32.v4f32(<4 x float>, <4 x float>, i32, i32, i32)
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-float.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-float.ll
new file mode 100644
index 000000000..427e42fbf
--- /dev/null
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-float.ll
@@ -0,0 +1,254 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -lower-matrix-intrinsics -S < %s | FileCheck %s
+; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s
+
+
+define <4 x float> @multiply_2x2(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: @multiply_2x2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <4 x float> [[A:%.*]], <4 x float> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <4 x float> [[A]], <4 x float> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[SPLIT2:%.*]] = shufflevector <4 x float> [[B:%.*]], <4 x float> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[SPLIT3:%.*]] = shufflevector <4 x float> [[B]], <4 x float> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[BLOCK:%.*]] = shufflevector <2 x float> [[SPLIT]], <2 x float> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <2 x float> [[SPLIT2]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <1 x float> undef, float [[TMP0]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT]], <1 x float> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul <1 x float> [[BLOCK]], [[SPLAT_SPLAT]]
+; CHECK-NEXT:    [[BLOCK4:%.*]] = shufflevector <2 x float> [[SPLIT1]], <2 x float> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x float> [[SPLIT2]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT5:%.*]] = insertelement <1 x float> undef, float [[TMP2]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT6:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT5]], <1 x float> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = fmul <1 x float> [[BLOCK4]], [[SPLAT_SPLAT6]]
+; CHECK-NEXT:    [[TMP4:%.*]] = fadd <1 x float> [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <1 x float> [[TMP4]], <1 x float> undef, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <2 x float> undef, <2 x float> [[TMP5]], <2 x i32> <i32 2, i32 1>
+; CHECK-NEXT:    [[BLOCK7:%.*]] = shufflevector <2 x float> [[SPLIT]], <2 x float> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x float> [[SPLIT2]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT8:%.*]] = insertelement <1 x float> undef, float [[TMP7]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT9:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT8]], <1 x float> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = fmul <1 x float> [[BLOCK7]], [[SPLAT_SPLAT9]]
+; CHECK-NEXT:    [[BLOCK10:%.*]] = shufflevector <2 x float> [[SPLIT1]], <2 x float> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <2 x float> [[SPLIT2]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT11:%.*]] = insertelement <1 x float> undef, float [[TMP9]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT12:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT11]], <1 x float> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = fmul <1 x float> [[BLOCK10]], [[SPLAT_SPLAT12]]
+; CHECK-NEXT:    [[TMP11:%.*]] = fadd <1 x float> [[TMP8]], [[TMP10]]
+; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <1 x float> [[TMP11]], <1 x float> undef, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <2 x float> [[TMP6]], <2 x float> [[TMP12]], <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:    [[BLOCK13:%.*]] = shufflevector <2 x float> [[SPLIT]], <2 x float> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x float> [[SPLIT3]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT14:%.*]] = insertelement <1 x float> undef, float [[TMP14]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT15:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT14]], <1 x float> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = fmul <1 x float> [[BLOCK13]], [[SPLAT_SPLAT15]]
+; CHECK-NEXT:    [[BLOCK16:%.*]] = shufflevector <2 x float> [[SPLIT1]], <2 x float> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <2 x float> [[SPLIT3]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT17:%.*]] = insertelement <1 x float> undef, float [[TMP16]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT18:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT17]], <1 x float> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = fmul <1 x float> [[BLOCK16]], [[SPLAT_SPLAT18]]
+; CHECK-NEXT:    [[TMP18:%.*]] = fadd <1 x float> [[TMP15]], [[TMP17]]
+; CHECK-NEXT:    [[TMP19:%.*]] = shufflevector <1 x float> [[TMP18]], <1 x float> undef, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP20:%.*]] = shufflevector <2 x float> undef, <2 x float> [[TMP19]], <2 x i32> <i32 2, i32 1>
+; CHECK-NEXT:    [[BLOCK19:%.*]] = shufflevector <2 x float> [[SPLIT]], <2 x float> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <2 x float> [[SPLIT3]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT20:%.*]] = insertelement <1 x float> undef, float [[TMP21]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT21:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT20]], <1 x float> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP22:%.*]] = fmul <1 x float> [[BLOCK19]], [[SPLAT_SPLAT21]]
+; CHECK-NEXT:    [[BLOCK22:%.*]] = shufflevector <2 x float> [[SPLIT1]], <2 x float> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <2 x float> [[SPLIT3]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT23:%.*]] = insertelement <1 x float> undef, float [[TMP23]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT24:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT23]], <1 x float> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = fmul <1 x float> [[BLOCK22]], [[SPLAT_SPLAT24]]
+; CHECK-NEXT:    [[TMP25:%.*]] = fadd <1 x float> [[TMP22]], [[TMP24]]
+; CHECK-NEXT:    [[TMP26:%.*]] = shufflevector <1 x float> [[TMP25]], <1 x float> undef, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP27:%.*]] = shufflevector <2 x float> [[TMP20]], <2 x float> [[TMP26]], <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:    [[TMP28:%.*]] = shufflevector <2 x float> [[TMP13]], <2 x float> [[TMP27]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    ret <4 x float> [[TMP28]]
+;
+entry:
+  %c = call <4 x float> @llvm.matrix.multiply.v4f32.v4f32.v4f32(<4 x float> %a, <4 x float> %b, i32 2, i32 2, i32 2)
+  ret <4 x float> %c
+}
+
+declare <4 x float> @llvm.matrix.multiply.v4f32.v4f32.v4f32(<4 x float>, <4 x float>, i32, i32, i32)
+
+define <4 x float> @multiply_1x2(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: @multiply_1x2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <2 x float> [[A:%.*]], <2 x float> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <2 x float> [[B:%.*]], <2 x float> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[SPLIT2:%.*]] = shufflevector <2 x float> [[B]], <2 x float> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[BLOCK:%.*]] = shufflevector <2 x float> [[SPLIT]], <2 x float> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <1 x float> [[SPLIT1]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <1 x float> undef, float [[TMP0]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT]], <1 x float> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul <1 x float> [[BLOCK]], [[SPLAT_SPLAT]]
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <1 x float> [[TMP1]], <1 x float> undef, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <2 x float> undef, <2 x float> [[TMP2]], <2 x i32> <i32 2, i32 1>
+; CHECK-NEXT:    [[BLOCK3:%.*]] = shufflevector <2 x float> [[SPLIT]], <2 x float> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <1 x float> [[SPLIT1]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT4:%.*]] = insertelement <1 x float> undef, float [[TMP4]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT5:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT4]], <1 x float> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = fmul <1 x float> [[BLOCK3]], [[SPLAT_SPLAT5]]
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <1 x float> [[TMP5]], <1 x float> undef, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <2 x float> [[TMP3]], <2 x float> [[TMP6]], <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:    [[BLOCK6:%.*]] = shufflevector <2 x float> [[SPLIT]], <2 x float> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <1 x float> [[SPLIT2]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT7:%.*]] = insertelement <1 x float> undef, float [[TMP8]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT8:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT7]], <1 x float> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = fmul <1 x float> [[BLOCK6]], [[SPLAT_SPLAT8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <1 x float> [[TMP9]], <1 x float> undef, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <2 x float> undef, <2 x float> [[TMP10]], <2 x i32> <i32 2, i32 1>
+; CHECK-NEXT:    [[BLOCK9:%.*]] = shufflevector <2 x float> [[SPLIT]], <2 x float> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <1 x float> [[SPLIT2]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT10:%.*]] = insertelement <1 x float> undef, float [[TMP12]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT11:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT10]], <1 x float> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = fmul <1 x float> [[BLOCK9]], [[SPLAT_SPLAT11]]
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <1 x float> [[TMP13]], <1 x float> undef, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP15:%.*]] = shufflevector <2 x float> [[TMP11]], <2 x float> [[TMP14]], <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:    [[TMP16:%.*]] = shufflevector <2 x float> [[TMP7]], <2 x float> [[TMP15]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    ret <4 x float> [[TMP16]]
+;
+entry:
+  %c = call <4 x float> @llvm.matrix.multiply.v4f32.v2f32.v2f32(<2 x float> %a, <2 x float> %b, i32 2, i32 1, i32 2)
+  ret <4 x float> %c
+}
+
+declare <4 x float> @llvm.matrix.multiply.v4f32.v2f32.v2f32(<2 x float>, <2 x float>, i32, i32, i32)
+
+define <9 x float> @multiply_2x3(<6 x float> %a, <6 x float> %b) {
+; CHECK-LABEL: @multiply_2x3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <6 x float> [[A:%.*]], <6 x float> undef, <3 x i32> <i32 0, i32 1, i32 2>
+; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <6 x float> [[A]], <6 x float> undef, <3 x i32> <i32 3, i32 4, i32 5>
+; CHECK-NEXT:    [[SPLIT2:%.*]] = shufflevector <6 x float> [[B:%.*]], <6 x float> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[SPLIT3:%.*]] = shufflevector <6 x float> [[B]], <6 x float> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[SPLIT4:%.*]] = shufflevector <6 x float> [[B]], <6 x float> undef, <2 x i32> <i32 4, i32 5>
+; CHECK-NEXT:    [[BLOCK:%.*]] = shufflevector <3 x float> [[SPLIT]], <3 x float> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <2 x float> [[SPLIT2]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <1 x float> undef, float [[TMP0]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT]], <1 x float> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul <1 x float> [[BLOCK]], [[SPLAT_SPLAT]]
+; CHECK-NEXT:    [[BLOCK5:%.*]] = shufflevector <3 x float> [[SPLIT1]], <3 x float> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x float> [[SPLIT2]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT6:%.*]] = insertelement <1 x float> undef, float [[TMP2]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT7:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT6]], <1 x float> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = fmul <1 x float> [[BLOCK5]], [[SPLAT_SPLAT7]]
+; CHECK-NEXT:    [[TMP4:%.*]] = fadd <1 x float> [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <1 x float> [[TMP4]], <1 x float> undef, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <3 x float> undef, <3 x float> [[TMP5]], <3 x i32> <i32 3, i32 1, i32 2>
+; CHECK-NEXT:    [[BLOCK8:%.*]] = shufflevector <3 x float> [[SPLIT]], <3 x float> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x float> [[SPLIT2]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT9:%.*]] = insertelement <1 x float> undef, float [[TMP7]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT10:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT9]], <1 x float> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = fmul <1 x float> [[BLOCK8]], [[SPLAT_SPLAT10]]
+; CHECK-NEXT:    [[BLOCK11:%.*]] = shufflevector <3 x float> [[SPLIT1]], <3 x float> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <2 x float> [[SPLIT2]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT12:%.*]] = insertelement <1 x float> undef, float [[TMP9]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT13:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT12]], <1 x float> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = fmul <1 x float> [[BLOCK11]], [[SPLAT_SPLAT13]]
+; CHECK-NEXT:    [[TMP11:%.*]] = fadd <1 x float> [[TMP8]], [[TMP10]]
+; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <1 x float> [[TMP11]], <1 x float> undef, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <3 x float> [[TMP6]], <3 x float> [[TMP12]], <3 x i32> <i32 0, i32 3, i32 2>
+; CHECK-NEXT:    [[BLOCK14:%.*]] = shufflevector <3 x float> [[SPLIT]], <3 x float> undef, <1 x i32> <i32 2>
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x float> [[SPLIT2]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT15:%.*]] = insertelement <1 x float> undef, float [[TMP14]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT16:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT15]], <1 x float> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = fmul <1 x float> [[BLOCK14]], [[SPLAT_SPLAT16]]
+; CHECK-NEXT:    [[BLOCK17:%.*]] = shufflevector <3 x float> [[SPLIT1]], <3 x float> undef, <1 x i32> <i32 2>
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <2 x float> [[SPLIT2]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT18:%.*]] = insertelement <1 x float> undef, float [[TMP16]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT19:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT18]], <1 x float> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = fmul <1 x float> [[BLOCK17]], [[SPLAT_SPLAT19]]
+; CHECK-NEXT:    [[TMP18:%.*]] = fadd <1 x float> [[TMP15]], [[TMP17]]
+; CHECK-NEXT:    [[TMP19:%.*]] = shufflevector <1 x float> [[TMP18]], <1 x float> undef, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP20:%.*]] = shufflevector <3 x float> [[TMP13]], <3 x float> [[TMP19]], <3 x i32> <i32 0, i32 1, i32 3>
+; CHECK-NEXT:    [[BLOCK20:%.*]] = shufflevector <3 x float> [[SPLIT]], <3 x float> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <2 x float> [[SPLIT3]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT21:%.*]] = insertelement <1 x float> undef, float [[TMP21]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT22:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT21]], <1 x float> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP22:%.*]] = fmul <1 x float> [[BLOCK20]], [[SPLAT_SPLAT22]]
+; CHECK-NEXT:    [[BLOCK23:%.*]] = shufflevector <3 x float> [[SPLIT1]], <3 x float> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <2 x float> [[SPLIT3]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT24:%.*]] = insertelement <1 x float> undef, float [[TMP23]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT25:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT24]], <1 x float> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = fmul <1 x float> [[BLOCK23]], [[SPLAT_SPLAT25]]
+; CHECK-NEXT:    [[TMP25:%.*]] = fadd <1 x float> [[TMP22]], [[TMP24]]
+; CHECK-NEXT:    [[TMP26:%.*]] = shufflevector <1 x float> [[TMP25]], <1 x float> undef, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP27:%.*]] = shufflevector <3 x float> undef, <3 x float> [[TMP26]], <3 x i32> <i32 3, i32 1, i32 2>
+; CHECK-NEXT:    [[BLOCK26:%.*]] = shufflevector <3 x float> [[SPLIT]], <3 x float> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP28:%.*]] = extractelement <2 x float> [[SPLIT3]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT27:%.*]] = insertelement <1 x float> undef, float [[TMP28]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT28:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT27]], <1 x float> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP29:%.*]] = fmul <1 x float> [[BLOCK26]], [[SPLAT_SPLAT28]]
+; CHECK-NEXT:    [[BLOCK29:%.*]] = shufflevector <3 x float> [[SPLIT1]], <3 x float> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP30:%.*]] = extractelement <2 x float> [[SPLIT3]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT30:%.*]] = insertelement <1 x float> undef, float [[TMP30]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT31:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT30]], <1 x float> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP31:%.*]] = fmul <1 x float> [[BLOCK29]], [[SPLAT_SPLAT31]]
+; CHECK-NEXT:    [[TMP32:%.*]] = fadd <1 x float> [[TMP29]], [[TMP31]]
+; CHECK-NEXT:    [[TMP33:%.*]] = shufflevector <1 x float> [[TMP32]], <1 x float> undef, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP34:%.*]] = shufflevector <3 x float> [[TMP27]], <3 x float> [[TMP33]], <3 x i32> <i32 0, i32 3, i32 2>
+; CHECK-NEXT:    [[BLOCK32:%.*]] = shufflevector <3 x float> [[SPLIT]], <3 x float> undef, <1 x i32> <i32 2>
+; CHECK-NEXT:    [[TMP35:%.*]] = extractelement <2 x float> [[SPLIT3]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT33:%.*]] = insertelement <1 x float> undef, float [[TMP35]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT34:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT33]], <1 x float> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP36:%.*]] = fmul <1 x float> [[BLOCK32]], [[SPLAT_SPLAT34]]
+; CHECK-NEXT:    [[BLOCK35:%.*]] = shufflevector <3 x float> [[SPLIT1]], <3 x float> undef, <1 x i32> <i32 2>
+; CHECK-NEXT:    [[TMP37:%.*]] = extractelement <2 x float> [[SPLIT3]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT36:%.*]] = insertelement <1 x float> undef, float [[TMP37]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT37:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT36]], <1 x float> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP38:%.*]] = fmul <1 x float> [[BLOCK35]], [[SPLAT_SPLAT37]]
+; CHECK-NEXT:    [[TMP39:%.*]] = fadd <1 x float> [[TMP36]], [[TMP38]]
+; CHECK-NEXT:    [[TMP40:%.*]] = shufflevector <1 x float> [[TMP39]], <1 x float> undef, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP41:%.*]] = shufflevector <3 x float> [[TMP34]], <3 x float> [[TMP40]], <3 x i32> <i32 0, i32 1, i32 3>
+; CHECK-NEXT:    [[BLOCK38:%.*]] = shufflevector <3 x float> [[SPLIT]], <3 x float> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP42:%.*]] = extractelement <2 x float> [[SPLIT4]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT39:%.*]] = insertelement <1 x float> undef, float [[TMP42]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT40:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT39]], <1 x float> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP43:%.*]] = fmul <1 x float> [[BLOCK38]], [[SPLAT_SPLAT40]]
+; CHECK-NEXT:    [[BLOCK41:%.*]] = shufflevector <3 x float> [[SPLIT1]], <3 x float> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP44:%.*]] = extractelement <2 x float> [[SPLIT4]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT42:%.*]] = insertelement <1 x float> undef, float [[TMP44]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT43:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT42]], <1 x float> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP45:%.*]] = fmul <1 x float> [[BLOCK41]], [[SPLAT_SPLAT43]]
+; CHECK-NEXT:    [[TMP46:%.*]] = fadd <1 x float> [[TMP43]], [[TMP45]]
+; CHECK-NEXT:    [[TMP47:%.*]] = shufflevector <1 x float> [[TMP46]], <1 x float> undef, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP48:%.*]] = shufflevector <3 x float> undef, <3 x float> [[TMP47]], <3 x i32> <i32 3, i32 1, i32 2>
+; CHECK-NEXT:    [[BLOCK44:%.*]] = shufflevector <3 x float> [[SPLIT]], <3 x float> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP49:%.*]] = extractelement <2 x float> [[SPLIT4]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT45:%.*]] = insertelement <1 x float> undef, float [[TMP49]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT46:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT45]], <1 x float> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP50:%.*]] = fmul <1 x float> [[BLOCK44]], [[SPLAT_SPLAT46]]
+; CHECK-NEXT:    [[BLOCK47:%.*]] = shufflevector <3 x float> [[SPLIT1]], <3 x float> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP51:%.*]] = extractelement <2 x float> [[SPLIT4]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT48:%.*]] = insertelement <1 x float> undef, float [[TMP51]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT49:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT48]], <1 x float> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP52:%.*]] = fmul <1 x float> [[BLOCK47]], [[SPLAT_SPLAT49]]
+; CHECK-NEXT:    [[TMP53:%.*]] = fadd <1 x float> [[TMP50]], [[TMP52]]
+; CHECK-NEXT:    [[TMP54:%.*]] = shufflevector <1 x float> [[TMP53]], <1 x float> undef, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP55:%.*]] = shufflevector <3 x float> [[TMP48]], <3 x float> [[TMP54]], <3 x i32> <i32 0, i32 3, i32 2>
+; CHECK-NEXT:    [[BLOCK50:%.*]] = shufflevector <3 x float> [[SPLIT]], <3 x float> undef, <1 x i32> <i32 2>
+; CHECK-NEXT:    [[TMP56:%.*]] = extractelement <2 x float> [[SPLIT4]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT51:%.*]] = insertelement <1 x float> undef, float [[TMP56]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT52:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT51]], <1 x float> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP57:%.*]] = fmul <1 x float> [[BLOCK50]], [[SPLAT_SPLAT52]]
+; CHECK-NEXT:    [[BLOCK53:%.*]] = shufflevector <3 x float> [[SPLIT1]], <3 x float> undef, <1 x i32> <i32 2>
+; CHECK-NEXT:    [[TMP58:%.*]] = extractelement <2 x float> [[SPLIT4]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT54:%.*]] = insertelement <1 x float> undef, float [[TMP58]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT55:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT54]], <1 x float> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP59:%.*]] = fmul <1 x float> [[BLOCK53]], [[SPLAT_SPLAT55]]
+; CHECK-NEXT:    [[TMP60:%.*]] = fadd <1 x float> [[TMP57]], [[TMP59]]
+; CHECK-NEXT:    [[TMP61:%.*]] = shufflevector <1 x float> [[TMP60]], <1 x float> undef, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP62:%.*]] = shufflevector <3 x float> [[TMP55]], <3 x float> [[TMP61]], <3 x i32> <i32 0, i32 1, i32 3>
+; CHECK-NEXT:    [[TMP63:%.*]] = shufflevector <3 x float> [[TMP20]], <3 x float> [[TMP41]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
+; CHECK-NEXT:    [[TMP64:%.*]] = shufflevector <3 x float> [[TMP62]], <3 x float> undef, <6 x i32> <i32 0, i32 1, i32 2, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP65:%.*]] = shufflevector <6 x float> [[TMP63]], <6 x float> [[TMP64]], <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
+; CHECK-NEXT:    ret <9 x float> [[TMP65]]
+;
+entry:
+  %c = call <9 x float> @llvm.matrix.multiply.v6f32.v6f32.v6f32(<6 x float> %a, <6 x float> %b, i32 3, i32 2, i32 3)
+  ret <9 x float> %c
+}
+
+declare <9 x float> @llvm.matrix.multiply.v6f32.v6f32.v6f32(<6 x float>, <6 x float>, i32, i32, i32)
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-i32.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-i32.ll
new file mode 100644
index 000000000..4da36f9ff
--- /dev/null
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-i32.ll
@@ -0,0 +1,254 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -lower-matrix-intrinsics -S < %s | FileCheck %s
+; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s
+
+
+define <4 x i32> @multiply_2x2(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: @multiply_2x2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <4 x i32> [[A:%.*]], <4 x i32> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <4 x i32> [[A]], <4 x i32> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[SPLIT2:%.*]] = shufflevector <4 x i32> [[B:%.*]], <4 x i32> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[SPLIT3:%.*]] = shufflevector <4 x i32> [[B]], <4 x i32> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[BLOCK:%.*]] = shufflevector <2 x i32> [[SPLIT]], <2 x i32> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <2 x i32> [[SPLIT2]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <1 x i32> undef, i32 [[TMP0]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <1 x i32> [[SPLAT_SPLATINSERT]], <1 x i32> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP1:%.*]] = mul <1 x i32> [[BLOCK]], [[SPLAT_SPLAT]]
+; CHECK-NEXT:    [[BLOCK4:%.*]] = shufflevector <2 x i32> [[SPLIT1]], <2 x i32> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x i32> [[SPLIT2]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT5:%.*]] = insertelement <1 x i32> undef, i32 [[TMP2]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT6:%.*]] = shufflevector <1 x i32> [[SPLAT_SPLATINSERT5]], <1 x i32> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = mul <1 x i32> [[BLOCK4]], [[SPLAT_SPLAT6]]
+; CHECK-NEXT:    [[TMP4:%.*]] = add <1 x i32> [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <1 x i32> [[TMP4]], <1 x i32> undef, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <2 x i32> undef, <2 x i32> [[TMP5]], <2 x i32> <i32 2, i32 1>
+; CHECK-NEXT:    [[BLOCK7:%.*]] = shufflevector <2 x i32> [[SPLIT]], <2 x i32> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x i32> [[SPLIT2]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT8:%.*]] = insertelement <1 x i32> undef, i32 [[TMP7]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT9:%.*]] = shufflevector <1 x i32> [[SPLAT_SPLATINSERT8]], <1 x i32> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = mul <1 x i32> [[BLOCK7]], [[SPLAT_SPLAT9]]
+; CHECK-NEXT:    [[BLOCK10:%.*]] = shufflevector <2 x i32> [[SPLIT1]], <2 x i32> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <2 x i32> [[SPLIT2]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT11:%.*]] = insertelement <1 x i32> undef, i32 [[TMP9]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT12:%.*]] = shufflevector <1 x i32> [[SPLAT_SPLATINSERT11]], <1 x i32> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = mul <1 x i32> [[BLOCK10]], [[SPLAT_SPLAT12]]
+; CHECK-NEXT:    [[TMP11:%.*]] = add <1 x i32> [[TMP8]], [[TMP10]]
+; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <1 x i32> [[TMP11]], <1 x i32> undef, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <2 x i32> [[TMP6]], <2 x i32> [[TMP12]], <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:    [[BLOCK13:%.*]] = shufflevector <2 x i32> [[SPLIT]], <2 x i32> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x i32> [[SPLIT3]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT14:%.*]] = insertelement <1 x i32> undef, i32 [[TMP14]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT15:%.*]] = shufflevector <1 x i32> [[SPLAT_SPLATINSERT14]], <1 x i32> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = mul <1 x i32> [[BLOCK13]], [[SPLAT_SPLAT15]]
+; CHECK-NEXT:    [[BLOCK16:%.*]] = shufflevector <2 x i32> [[SPLIT1]], <2 x i32> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <2 x i32> [[SPLIT3]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT17:%.*]] = insertelement <1 x i32> undef, i32 [[TMP16]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT18:%.*]] = shufflevector <1 x i32> [[SPLAT_SPLATINSERT17]], <1 x i32> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = mul <1 x i32> [[BLOCK16]], [[SPLAT_SPLAT18]]
+; CHECK-NEXT:    [[TMP18:%.*]] = add <1 x i32> [[TMP15]], [[TMP17]]
+; CHECK-NEXT:    [[TMP19:%.*]] = shufflevector <1 x i32> [[TMP18]], <1 x i32> undef, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP20:%.*]] = shufflevector <2 x i32> undef, <2 x i32> [[TMP19]], <2 x i32> <i32 2, i32 1>
+; CHECK-NEXT:    [[BLOCK19:%.*]] = shufflevector <2 x i32> [[SPLIT]], <2 x i32> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <2 x i32> [[SPLIT3]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT20:%.*]] = insertelement <1 x i32> undef, i32 [[TMP21]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT21:%.*]] = shufflevector <1 x i32> [[SPLAT_SPLATINSERT20]], <1 x i32> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP22:%.*]] = mul <1 x i32> [[BLOCK19]], [[SPLAT_SPLAT21]]
+; CHECK-NEXT:    [[BLOCK22:%.*]] = shufflevector <2 x i32> [[SPLIT1]], <2 x i32> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <2 x i32> [[SPLIT3]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT23:%.*]] = insertelement <1 x i32> undef, i32 [[TMP23]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT24:%.*]] = shufflevector <1 x i32> [[SPLAT_SPLATINSERT23]], <1 x i32> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = mul <1 x i32> [[BLOCK22]], [[SPLAT_SPLAT24]]
+; CHECK-NEXT:    [[TMP25:%.*]] = add <1 x i32> [[TMP22]], [[TMP24]]
+; CHECK-NEXT:    [[TMP26:%.*]] = shufflevector <1 x i32> [[TMP25]], <1 x i32> undef, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP27:%.*]] = shufflevector <2 x i32> [[TMP20]], <2 x i32> [[TMP26]], <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:    [[TMP28:%.*]] = shufflevector <2 x i32> [[TMP13]], <2 x i32> [[TMP27]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    ret <4 x i32> [[TMP28]]
+;
+entry:
+  %c = call <4 x i32> @llvm.matrix.multiply.v4i32.v4i32.v4i32(<4 x i32> %a, <4 x i32> %b, i32 2, i32 2, i32 2)
+  ret <4 x i32> %c
+}
+
+declare <4 x i32> @llvm.matrix.multiply.v4i32.v4i32.v4i32(<4 x i32>, <4 x i32>, i32, i32, i32)
+
+define <4 x i32> @multiply_1x2(<2 x i32> %a, <2 x i32> %b) {
+; CHECK-LABEL: @multiply_1x2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <2 x i32> [[A:%.*]], <2 x i32> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <2 x i32> [[B:%.*]], <2 x i32> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[SPLIT2:%.*]] = shufflevector <2 x i32> [[B]], <2 x i32> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[BLOCK:%.*]] = shufflevector <2 x i32> [[SPLIT]], <2 x i32> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <1 x i32> [[SPLIT1]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <1 x i32> undef, i32 [[TMP0]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <1 x i32> [[SPLAT_SPLATINSERT]], <1 x i32> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP1:%.*]] = mul <1 x i32> [[BLOCK]], [[SPLAT_SPLAT]]
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <1 x i32> [[TMP1]], <1 x i32> undef, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i32> undef, <2 x i32> [[TMP2]], <2 x i32> <i32 2, i32 1>
+; CHECK-NEXT:    [[BLOCK3:%.*]] = shufflevector <2 x i32> [[SPLIT]], <2 x i32> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <1 x i32> [[SPLIT1]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT4:%.*]] = insertelement <1 x i32> undef, i32 [[TMP4]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT5:%.*]] = shufflevector <1 x i32> [[SPLAT_SPLATINSERT4]], <1 x i32> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = mul <1 x i32> [[BLOCK3]], [[SPLAT_SPLAT5]]
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <1 x i32> [[TMP5]], <1 x i32> undef, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <2 x i32> [[TMP3]], <2 x i32> [[TMP6]], <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:    [[BLOCK6:%.*]] = shufflevector <2 x i32> [[SPLIT]], <2 x i32> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <1 x i32> [[SPLIT2]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT7:%.*]] = insertelement <1 x i32> undef, i32 [[TMP8]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT8:%.*]] = shufflevector <1 x i32> [[SPLAT_SPLATINSERT7]], <1 x i32> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = mul <1 x i32> [[BLOCK6]], [[SPLAT_SPLAT8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <1 x i32> [[TMP9]], <1 x i32> undef, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <2 x i32> undef, <2 x i32> [[TMP10]], <2 x i32> <i32 2, i32 1>
+; CHECK-NEXT:    [[BLOCK9:%.*]] = shufflevector <2 x i32> [[SPLIT]], <2 x i32> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <1 x i32> [[SPLIT2]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT10:%.*]] = insertelement <1 x i32> undef, i32 [[TMP12]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT11:%.*]] = shufflevector <1 x i32> [[SPLAT_SPLATINSERT10]], <1 x i32> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = mul <1 x i32> [[BLOCK9]], [[SPLAT_SPLAT11]]
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <1 x i32> [[TMP13]], <1 x i32> undef, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP15:%.*]] = shufflevector <2 x i32> [[TMP11]], <2 x i32> [[TMP14]], <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:    [[TMP16:%.*]] = shufflevector <2 x i32> [[TMP7]], <2 x i32> [[TMP15]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    ret <4 x i32> [[TMP16]]
+;
+entry:
+  %c = call <4 x i32> @llvm.matrix.multiply.v4i32.v2i32.v2i32(<2 x i32> %a, <2 x i32> %b, i32 2, i32 1, i32 2)
+  ret <4 x i32> %c
+}
+
+declare <4 x i32> @llvm.matrix.multiply.v4i32.v2i32.v2i32(<2 x i32>, <2 x i32>, i32, i32, i32)
+
+define <9 x i32> @multiply_2x3(<6 x i32> %a, <6 x i32> %b) {
+; CHECK-LABEL: @multiply_2x3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <6 x i32> [[A:%.*]], <6 x i32> undef, <3 x i32> <i32 0, i32 1, i32 2>
+; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <6 x i32> [[A]], <6 x i32> undef, <3 x i32> <i32 3, i32 4, i32 5>
+; CHECK-NEXT:    [[SPLIT2:%.*]] = shufflevector <6 x i32> [[B:%.*]], <6 x i32> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[SPLIT3:%.*]] = shufflevector <6 x i32> [[B]], <6 x i32> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[SPLIT4:%.*]] = shufflevector <6 x i32> [[B]], <6 x i32> undef, <2 x i32> <i32 4, i32 5>
+; CHECK-NEXT:    [[BLOCK:%.*]] = shufflevector <3 x i32> [[SPLIT]], <3 x i32> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <2 x i32> [[SPLIT2]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <1 x i32> undef, i32 [[TMP0]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <1 x i32> [[SPLAT_SPLATINSERT]], <1 x i32> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP1:%.*]] = mul <1 x i32> [[BLOCK]], [[SPLAT_SPLAT]]
+; CHECK-NEXT:    [[BLOCK5:%.*]] = shufflevector <3 x i32> [[SPLIT1]], <3 x i32> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x i32> [[SPLIT2]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT6:%.*]] = insertelement <1 x i32> undef, i32 [[TMP2]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT7:%.*]] = shufflevector <1 x i32> [[SPLAT_SPLATINSERT6]], <1 x i32> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = mul <1 x i32> [[BLOCK5]], [[SPLAT_SPLAT7]]
+; CHECK-NEXT:    [[TMP4:%.*]] = add <1 x i32> [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <1 x i32> [[TMP4]], <1 x i32> undef, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <3 x i32> undef, <3 x i32> [[TMP5]], <3 x i32> <i32 3, i32 1, i32 2>
+; CHECK-NEXT:    [[BLOCK8:%.*]] = shufflevector <3 x i32> [[SPLIT]], <3 x i32> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x i32> [[SPLIT2]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT9:%.*]] = insertelement <1 x i32> undef, i32 [[TMP7]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT10:%.*]] = shufflevector <1 x i32> [[SPLAT_SPLATINSERT9]], <1 x i32> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = mul <1 x i32> [[BLOCK8]], [[SPLAT_SPLAT10]]
+; CHECK-NEXT:    [[BLOCK11:%.*]] = shufflevector <3 x i32> [[SPLIT1]], <3 x i32> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <2 x i32> [[SPLIT2]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT12:%.*]] = insertelement <1 x i32> undef, i32 [[TMP9]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT13:%.*]] = shufflevector <1 x i32> [[SPLAT_SPLATINSERT12]], <1 x i32> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = mul <1 x i32> [[BLOCK11]], [[SPLAT_SPLAT13]]
+; CHECK-NEXT:    [[TMP11:%.*]] = add <1 x i32> [[TMP8]], [[TMP10]]
+; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <1 x i32> [[TMP11]], <1 x i32> undef, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <3 x i32> [[TMP6]], <3 x i32> [[TMP12]], <3 x i32> <i32 0, i32 3, i32 2>
+; CHECK-NEXT:    [[BLOCK14:%.*]] = shufflevector <3 x i32> [[SPLIT]], <3 x i32> undef, <1 x i32> <i32 2>
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x i32> [[SPLIT2]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT15:%.*]] = insertelement <1 x i32> undef, i32 [[TMP14]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT16:%.*]] = shufflevector <1 x i32> [[SPLAT_SPLATINSERT15]], <1 x i32> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = mul <1 x i32> [[BLOCK14]], [[SPLAT_SPLAT16]]
+; CHECK-NEXT:    [[BLOCK17:%.*]] = shufflevector <3 x i32> [[SPLIT1]], <3 x i32> undef, <1 x i32> <i32 2>
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <2 x i32> [[SPLIT2]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT18:%.*]] = insertelement <1 x i32> undef, i32 [[TMP16]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT19:%.*]] = shufflevector <1 x i32> [[SPLAT_SPLATINSERT18]], <1 x i32> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = mul <1 x i32> [[BLOCK17]], [[SPLAT_SPLAT19]]
+; CHECK-NEXT:    [[TMP18:%.*]] = add <1 x i32> [[TMP15]], [[TMP17]]
+; CHECK-NEXT:    [[TMP19:%.*]] = shufflevector <1 x i32> [[TMP18]], <1 x i32> undef, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP20:%.*]] = shufflevector <3 x i32> [[TMP13]], <3 x i32> [[TMP19]], <3 x i32> <i32 0, i32 1, i32 3>
+; CHECK-NEXT:    [[BLOCK20:%.*]] = shufflevector <3 x i32> [[SPLIT]], <3 x i32> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <2 x i32> [[SPLIT3]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT21:%.*]] = insertelement <1 x i32> undef, i32 [[TMP21]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT22:%.*]] = shufflevector <1 x i32> [[SPLAT_SPLATINSERT21]], <1 x i32> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP22:%.*]] = mul <1 x i32> [[BLOCK20]], [[SPLAT_SPLAT22]]
+; CHECK-NEXT:    [[BLOCK23:%.*]] = shufflevector <3 x i32> [[SPLIT1]], <3 x i32> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <2 x i32> [[SPLIT3]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT24:%.*]] = insertelement <1 x i32> undef, i32 [[TMP23]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT25:%.*]] = shufflevector <1 x i32> [[SPLAT_SPLATINSERT24]], <1 x i32> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = mul <1 x i32> [[BLOCK23]], [[SPLAT_SPLAT25]]
+; CHECK-NEXT:    [[TMP25:%.*]] = add <1 x i32> [[TMP22]], [[TMP24]]
+; CHECK-NEXT:    [[TMP26:%.*]] = shufflevector <1 x i32> [[TMP25]], <1 x i32> undef, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP27:%.*]] = shufflevector <3 x i32> undef, <3 x i32> [[TMP26]], <3 x i32> <i32 3, i32 1, i32 2>
+; CHECK-NEXT:    [[BLOCK26:%.*]] = shufflevector <3 x i32> [[SPLIT]], <3 x i32> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP28:%.*]] = extractelement <2 x i32> [[SPLIT3]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT27:%.*]] = insertelement <1 x i32> undef, i32 [[TMP28]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT28:%.*]] = shufflevector <1 x i32> [[SPLAT_SPLATINSERT27]], <1 x i32> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP29:%.*]] = mul <1 x i32> [[BLOCK26]], [[SPLAT_SPLAT28]]
+; CHECK-NEXT:    [[BLOCK29:%.*]] = shufflevector <3 x i32> [[SPLIT1]], <3 x i32> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP30:%.*]] = extractelement <2 x i32> [[SPLIT3]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT30:%.*]] = insertelement <1 x i32> undef, i32 [[TMP30]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT31:%.*]] = shufflevector <1 x i32> [[SPLAT_SPLATINSERT30]], <1 x i32> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP31:%.*]] = mul <1 x i32> [[BLOCK29]], [[SPLAT_SPLAT31]]
+; CHECK-NEXT:    [[TMP32:%.*]] = add <1 x i32> [[TMP29]], [[TMP31]]
+; CHECK-NEXT:    [[TMP33:%.*]] = shufflevector <1 x i32> [[TMP32]], <1 x i32> undef, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP34:%.*]] = shufflevector <3 x i32> [[TMP27]], <3 x i32> [[TMP33]], <3 x i32> <i32 0, i32 3, i32 2>
+; CHECK-NEXT:    [[BLOCK32:%.*]] = shufflevector <3 x i32> [[SPLIT]], <3 x i32> undef, <1 x i32> <i32 2>
+; CHECK-NEXT:    [[TMP35:%.*]] = extractelement <2 x i32> [[SPLIT3]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT33:%.*]] = insertelement <1 x i32> undef, i32 [[TMP35]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT34:%.*]] = shufflevector <1 x i32> [[SPLAT_SPLATINSERT33]], <1 x i32> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP36:%.*]] = mul <1 x i32> [[BLOCK32]], [[SPLAT_SPLAT34]]
+; CHECK-NEXT:    [[BLOCK35:%.*]] = shufflevector <3 x i32> [[SPLIT1]], <3 x i32> undef, <1 x i32> <i32 2>
+; CHECK-NEXT:    [[TMP37:%.*]] = extractelement <2 x i32> [[SPLIT3]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT36:%.*]] = insertelement <1 x i32> undef, i32 [[TMP37]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT37:%.*]] = shufflevector <1 x i32> [[SPLAT_SPLATINSERT36]], <1 x i32> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP38:%.*]] = mul <1 x i32> [[BLOCK35]], [[SPLAT_SPLAT37]]
+; CHECK-NEXT:    [[TMP39:%.*]] = add <1 x i32> [[TMP36]], [[TMP38]]
+; CHECK-NEXT:    [[TMP40:%.*]] = shufflevector <1 x i32> [[TMP39]], <1 x i32> undef, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP41:%.*]] = shufflevector <3 x i32> [[TMP34]], <3 x i32> [[TMP40]], <3 x i32> <i32 0, i32 1, i32 3>
+; CHECK-NEXT:    [[BLOCK38:%.*]] = shufflevector <3 x i32> [[SPLIT]], <3 x i32> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP42:%.*]] = extractelement <2 x i32> [[SPLIT4]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT39:%.*]] = insertelement <1 x i32> undef, i32 [[TMP42]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT40:%.*]] = shufflevector <1 x i32> [[SPLAT_SPLATINSERT39]], <1 x i32> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP43:%.*]] = mul <1 x i32> [[BLOCK38]], [[SPLAT_SPLAT40]]
+; CHECK-NEXT:    [[BLOCK41:%.*]] = shufflevector <3 x i32> [[SPLIT1]], <3 x i32> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP44:%.*]] = extractelement <2 x i32> [[SPLIT4]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT42:%.*]] = insertelement <1 x i32> undef, i32 [[TMP44]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT43:%.*]] = shufflevector <1 x i32> [[SPLAT_SPLATINSERT42]], <1 x i32> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP45:%.*]] = mul <1 x i32> [[BLOCK41]], [[SPLAT_SPLAT43]]
+; CHECK-NEXT:    [[TMP46:%.*]] = add <1 x i32> [[TMP43]], [[TMP45]]
+; CHECK-NEXT:    [[TMP47:%.*]] = shufflevector <1 x i32> [[TMP46]], <1 x i32> undef, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP48:%.*]] = shufflevector <3 x i32> undef, <3 x i32> [[TMP47]], <3 x i32> <i32 3, i32 1, i32 2>
+; CHECK-NEXT:    [[BLOCK44:%.*]] = shufflevector <3 x i32> [[SPLIT]], <3 x i32> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP49:%.*]] = extractelement <2 x i32> [[SPLIT4]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT45:%.*]] = insertelement <1 x i32> undef, i32 [[TMP49]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT46:%.*]] = shufflevector <1 x i32> [[SPLAT_SPLATINSERT45]], <1 x i32> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP50:%.*]] = mul <1 x i32> [[BLOCK44]], [[SPLAT_SPLAT46]]
+; CHECK-NEXT:    [[BLOCK47:%.*]] = shufflevector <3 x i32> [[SPLIT1]], <3 x i32> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP51:%.*]] = extractelement <2 x i32> [[SPLIT4]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT48:%.*]] = insertelement <1 x i32> undef, i32 [[TMP51]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT49:%.*]] = shufflevector <1 x i32> [[SPLAT_SPLATINSERT48]], <1 x i32> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP52:%.*]] = mul <1 x i32> [[BLOCK47]], [[SPLAT_SPLAT49]]
+; CHECK-NEXT:    [[TMP53:%.*]] = add <1 x i32> [[TMP50]], [[TMP52]]
+; CHECK-NEXT:    [[TMP54:%.*]] = shufflevector <1 x i32> [[TMP53]], <1 x i32> undef, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP55:%.*]] = shufflevector <3 x i32> [[TMP48]], <3 x i32> [[TMP54]], <3 x i32> <i32 0, i32 3, i32 2>
+; CHECK-NEXT:    [[BLOCK50:%.*]] = shufflevector <3 x i32> [[SPLIT]], <3 x i32> undef, <1 x i32> <i32 2>
+; CHECK-NEXT:    [[TMP56:%.*]] = extractelement <2 x i32> [[SPLIT4]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT51:%.*]] = insertelement <1 x i32> undef, i32 [[TMP56]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT52:%.*]] = shufflevector <1 x i32> [[SPLAT_SPLATINSERT51]], <1 x i32> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP57:%.*]] = mul <1 x i32> [[BLOCK50]], [[SPLAT_SPLAT52]]
+; CHECK-NEXT:    [[BLOCK53:%.*]] = shufflevector <3 x i32> [[SPLIT1]], <3 x i32> undef, <1 x i32> <i32 2>
+; CHECK-NEXT:    [[TMP58:%.*]] = extractelement <2 x i32> [[SPLIT4]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT54:%.*]] = insertelement <1 x i32> undef, i32 [[TMP58]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT55:%.*]] = shufflevector <1 x i32> [[SPLAT_SPLATINSERT54]], <1 x i32> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP59:%.*]] = mul <1 x i32> [[BLOCK53]], [[SPLAT_SPLAT55]]
+; CHECK-NEXT:    [[TMP60:%.*]] = add <1 x i32> [[TMP57]], [[TMP59]]
+; CHECK-NEXT:    [[TMP61:%.*]] = shufflevector <1 x i32> [[TMP60]], <1 x i32> undef, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP62:%.*]] = shufflevector <3 x i32> [[TMP55]], <3 x i32> [[TMP61]], <3 x i32> <i32 0, i32 1, i32 3>
+; CHECK-NEXT:    [[TMP63:%.*]] = shufflevector <3 x i32> [[TMP20]], <3 x i32> [[TMP41]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
+; CHECK-NEXT:    [[TMP64:%.*]] = shufflevector <3 x i32> [[TMP62]], <3 x i32> undef, <6 x i32> <i32 0, i32 1, i32 2, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP65:%.*]] = shufflevector <6 x i32> [[TMP63]], <6 x i32> [[TMP64]], <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
+; CHECK-NEXT:    ret <9 x i32> [[TMP65]]
+;
+entry:
+  %c = call <9 x i32> @llvm.matrix.multiply.v6i32.v6i32.v6i32(<6 x i32> %a, <6 x i32> %b, i32 3, i32 2, i32 3)
+  ret <9 x i32> %c
+}
+
+declare <9 x i32> @llvm.matrix.multiply.v6i32.v6i32.v6i32(<6 x i32>, <6 x i32>, i32, i32, i32)
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/propagate-backward.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/propagate-backward.ll
new file mode 100644
index 000000000..89ca79649
--- /dev/null
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/propagate-backward.ll
@@ -0,0 +1,96 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -lower-matrix-intrinsics -S < %s | FileCheck  %s
+; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s
+
+define <8 x double> @fadd_transpose(<8 x double> %a, <8 x double> %b) {
+; CHECK-LABEL: @fadd_transpose(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <8 x double> [[A:%.*]], <8 x double> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <8 x double> [[A]], <8 x double> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[SPLIT2:%.*]] = shufflevector <8 x double> [[A]], <8 x double> undef, <2 x i32> <i32 4, i32 5>
+; CHECK-NEXT:    [[SPLIT3:%.*]] = shufflevector <8 x double> [[A]], <8 x double> undef, <2 x i32> <i32 6, i32 7>
+; CHECK-NEXT:    [[SPLIT4:%.*]] = shufflevector <8 x double> [[B:%.*]], <8 x double> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[SPLIT5:%.*]] = shufflevector <8 x double> [[B]], <8 x double> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[SPLIT6:%.*]] = shufflevector <8 x double> [[B]], <8 x double> undef, <2 x i32> <i32 4, i32 5>
+; CHECK-NEXT:    [[SPLIT7:%.*]] = shufflevector <8 x double> [[B]], <8 x double> undef, <2 x i32> <i32 6, i32 7>
+; CHECK-NEXT:    [[TMP0:%.*]] = fadd <2 x double> [[SPLIT]], [[SPLIT4]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fadd <2 x double> [[SPLIT1]], [[SPLIT5]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fadd <2 x double> [[SPLIT2]], [[SPLIT6]]
+; CHECK-NEXT:    [[TMP3:%.*]] = fadd <2 x double> [[SPLIT3]], [[SPLIT7]]
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x double> [[TMP0]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <4 x double> undef, double [[TMP4]], i64 0
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x double> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <4 x double> [[TMP5]], double [[TMP6]], i64 1
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x double> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <4 x double> [[TMP7]], double [[TMP8]], i64 2
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <2 x double> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <4 x double> [[TMP9]], double [[TMP10]], i64 3
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <2 x double> [[TMP0]], i64 1
+; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <4 x double> undef, double [[TMP12]], i64 0
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x double> [[TMP1]], i64 1
+; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <4 x double> [[TMP13]], double [[TMP14]], i64 1
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <2 x double> [[TMP2]], i64 1
+; CHECK-NEXT:    [[TMP17:%.*]] = insertelement <4 x double> [[TMP15]], double [[TMP16]], i64 2
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <2 x double> [[TMP3]], i64 1
+; CHECK-NEXT:    [[TMP19:%.*]] = insertelement <4 x double> [[TMP17]], double [[TMP18]], i64 3
+; CHECK-NEXT:    [[TMP20:%.*]] = shufflevector <4 x double> [[TMP11]], <4 x double> [[TMP19]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    ret <8 x double> [[TMP20]]
+;
+entry:
+  %add = fadd <8 x double> %a, %b
+  %c  = call <8 x double> @llvm.matrix.transpose(<8 x double> %add, i32 2, i32 4)
+  ret <8 x double> %c
+}
+
+define <8 x double> @load_fadd_transpose(<8 x double>* %A.Ptr, <8 x double> %b) {
+; CHECK-LABEL: @load_fadd_transpose(
+; CHECK-NEXT:  entry:
+
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x double>* [[A_PTR:%.*]] to double*
+; CHECK-NEXT:    [[COL_CAST:%.*]] = bitcast double* [[TMP0]] to <2 x double>*
+; CHECK-NEXT:    [[COL_LOAD:%.*]] = load <2 x double>, <2 x double>* [[COL_CAST]], align 8
+; CHECK-NEXT:    [[COL_GEP:%.*]] = getelementptr double, double* [[TMP0]], i32 2
+; CHECK-NEXT:    [[COL_CAST1:%.*]] = bitcast double* [[COL_GEP]] to <2 x double>*
+; CHECK-NEXT:    [[COL_LOAD2:%.*]] = load <2 x double>, <2 x double>* [[COL_CAST1]], align 8
+; CHECK-NEXT:    [[COL_GEP3:%.*]] = getelementptr double, double* [[TMP0]], i32 4
+; CHECK-NEXT:    [[COL_CAST4:%.*]] = bitcast double* [[COL_GEP3]] to <2 x double>*
+; CHECK-NEXT:    [[COL_LOAD5:%.*]] = load <2 x double>, <2 x double>* [[COL_CAST4]], align 8
+; CHECK-NEXT:    [[COL_GEP6:%.*]] = getelementptr double, double* [[TMP0]], i32 6
+; CHECK-NEXT:    [[COL_CAST7:%.*]] = bitcast double* [[COL_GEP6]] to <2 x double>*
+; CHECK-NEXT:    [[COL_LOAD8:%.*]] = load <2 x double>, <2 x double>* [[COL_CAST7]], align 8
+; CHECK-NEXT:    [[SPLIT4:%.*]] = shufflevector <8 x double> [[B:%.*]], <8 x double> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[SPLIT5:%.*]] = shufflevector <8 x double> [[B]], <8 x double> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[SPLIT6:%.*]] = shufflevector <8 x double> [[B]], <8 x double> undef, <2 x i32> <i32 4, i32 5>
+; CHECK-NEXT:    [[SPLIT7:%.*]] = shufflevector <8 x double> [[B]], <8 x double> undef, <2 x i32> <i32 6, i32 7>
+
+; CHECK-NEXT:    [[TMP0:%.*]] = fadd <2 x double> [[COL_LOAD]], [[SPLIT4]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fadd <2 x double> [[COL_LOAD2]], [[SPLIT5]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fadd <2 x double> [[COL_LOAD5]], [[SPLIT6]]
+; CHECK-NEXT:    [[TMP3:%.*]] = fadd <2 x double> [[COL_LOAD8]], [[SPLIT7]]
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x double> [[TMP0]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <4 x double> undef, double [[TMP4]], i64 0
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x double> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <4 x double> [[TMP5]], double [[TMP6]], i64 1
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x double> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <4 x double> [[TMP7]], double [[TMP8]], i64 2
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <2 x double> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <4 x double> [[TMP9]], double [[TMP10]], i64 3
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <2 x double> [[TMP0]], i64 1
+; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <4 x double> undef, double [[TMP12]], i64 0
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x double> [[TMP1]], i64 1
+; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <4 x double> [[TMP13]], double [[TMP14]], i64 1
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <2 x double> [[TMP2]], i64 1
+; CHECK-NEXT:    [[TMP17:%.*]] = insertelement <4 x double> [[TMP15]], double [[TMP16]], i64 2
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <2 x double> [[TMP3]], i64 1
+; CHECK-NEXT:    [[TMP19:%.*]] = insertelement <4 x double> [[TMP17]], double [[TMP18]], i64 3
+; CHECK-NEXT:    [[TMP20:%.*]] = shufflevector <4 x double> [[TMP11]], <4 x double> [[TMP19]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    ret <8 x double> [[TMP20]]
+;
+entry:
+  %a = load <8 x double>, <8 x double>* %A.Ptr
+  %add = fadd <8 x double> %a, %b
+  %c  = call <8 x double> @llvm.matrix.transpose(<8 x double> %add, i32 2, i32 4)
+  ret <8 x double> %c
+}
+
+declare <8 x double> @llvm.matrix.transpose(<8 x double>, i32, i32)
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/propagate-backwards-unsupported.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/propagate-backwards-unsupported.ll
new file mode 100644
index 000000000..591cddd26
--- /dev/null
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/propagate-backwards-unsupported.ll
@@ -0,0 +1,135 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -lower-matrix-intrinsics -S < %s | FileCheck %s
+; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s
+
+; Check that we we use flattened vectors for PHI operands and extract the columns afterwards.
+define <9 x double> @unsupported_phi(i1 %cond, <9 x double> %A, <9 x double> %B, <9 x double> %C) {
+; CHECK-LABEL: @unsupported_phi(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+
+; CHECK-LABEL: if.then:
+; CHECK-NEXT:    [[SPLIT3:%.*]] = shufflevector <9 x double> [[A:%.*]], <9 x double> undef, <3 x i32> <i32 0, i32 1, i32 2>
+; CHECK-NEXT:    [[SPLIT4:%.*]] = shufflevector <9 x double> [[A]], <9 x double> undef, <3 x i32> <i32 3, i32 4, i32 5>
+; CHECK-NEXT:    [[SPLIT5:%.*]] = shufflevector <9 x double> [[A]], <9 x double> undef, <3 x i32> <i32 6, i32 7, i32 8>
+; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <3 x double> [[SPLIT3]], i64 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <3 x double> undef, double [[TMP0]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <3 x double> [[SPLIT4]], i64 0
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <3 x double> [[TMP1]], double [[TMP2]], i64 1
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <3 x double> [[SPLIT5]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <3 x double> [[TMP3]], double [[TMP4]], i64 2
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <3 x double> [[SPLIT3]], i64 1
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <3 x double> undef, double [[TMP6]], i64 0
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <3 x double> [[SPLIT4]], i64 1
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <3 x double> [[TMP7]], double [[TMP8]], i64 1
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <3 x double> [[SPLIT5]], i64 1
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <3 x double> [[TMP9]], double [[TMP10]], i64 2
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <3 x double> [[SPLIT3]], i64 2
+; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <3 x double> undef, double [[TMP12]], i64 0
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <3 x double> [[SPLIT4]], i64 2
+; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <3 x double> [[TMP13]], double [[TMP14]], i64 1
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <3 x double> [[SPLIT5]], i64 2
+; CHECK-NEXT:    [[TMP17:%.*]] = insertelement <3 x double> [[TMP15]], double [[TMP16]], i64 2
+; CHECK-NEXT:    [[TMP18:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> [[TMP11]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
+; CHECK-NEXT:    [[TMP19:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> undef, <6 x i32> <i32 0, i32 1, i32 2, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP20:%.*]] = shufflevector <6 x double> [[TMP18]], <6 x double> [[TMP19]], <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
+; CHECK-NEXT:    br label [[IF_END:%.*]]
+
+; CHECK-LABEL: if.else:
+; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <9 x double> [[B:%.*]], <9 x double> undef, <3 x i32> <i32 0, i32 1, i32 2>
+; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <9 x double> [[B]], <9 x double> undef, <3 x i32> <i32 3, i32 4, i32 5>
+; CHECK-NEXT:    [[SPLIT2:%.*]] = shufflevector <9 x double> [[B]], <9 x double> undef, <3 x i32> <i32 6, i32 7, i32 8>
+; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <3 x double> [[SPLIT]], i64 0
+; CHECK-NEXT:    [[TMP22:%.*]] = insertelement <3 x double> undef, double [[TMP21]], i64 0
+; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <3 x double> [[SPLIT1]], i64 0
+; CHECK-NEXT:    [[TMP24:%.*]] = insertelement <3 x double> [[TMP22]], double [[TMP23]], i64 1
+; CHECK-NEXT:    [[TMP25:%.*]] = extractelement <3 x double> [[SPLIT2]], i64 0
+; CHECK-NEXT:    [[TMP26:%.*]] = insertelement <3 x double> [[TMP24]], double [[TMP25]], i64 2
+; CHECK-NEXT:    [[TMP27:%.*]] = extractelement <3 x double> [[SPLIT]], i64 1
+; CHECK-NEXT:    [[TMP28:%.*]] = insertelement <3 x double> undef, double [[TMP27]], i64 0
+; CHECK-NEXT:    [[TMP29:%.*]] = extractelement <3 x double> [[SPLIT1]], i64 1
+; CHECK-NEXT:    [[TMP30:%.*]] = insertelement <3 x double> [[TMP28]], double [[TMP29]], i64 1
+; CHECK-NEXT:    [[TMP31:%.*]] = extractelement <3 x double> [[SPLIT2]], i64 1
+; CHECK-NEXT:    [[TMP32:%.*]] = insertelement <3 x double> [[TMP30]], double [[TMP31]], i64 2
+; CHECK-NEXT:    [[TMP33:%.*]] = extractelement <3 x double> [[SPLIT]], i64 2
+; CHECK-NEXT:    [[TMP34:%.*]] = insertelement <3 x double> undef, double [[TMP33]], i64 0
+; CHECK-NEXT:    [[TMP35:%.*]] = extractelement <3 x double> [[SPLIT1]], i64 2
+; CHECK-NEXT:    [[TMP36:%.*]] = insertelement <3 x double> [[TMP34]], double [[TMP35]], i64 1
+; CHECK-NEXT:    [[TMP37:%.*]] = extractelement <3 x double> [[SPLIT2]], i64 2
+; CHECK-NEXT:    [[TMP38:%.*]] = insertelement <3 x double> [[TMP36]], double [[TMP37]], i64 2
+; CHECK-NEXT:    [[TMP39:%.*]] = shufflevector <3 x double> [[TMP26]], <3 x double> [[TMP32]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
+; CHECK-NEXT:    [[TMP40:%.*]] = shufflevector <3 x double> [[TMP38]], <3 x double> undef, <6 x i32> <i32 0, i32 1, i32 2, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP41:%.*]] = shufflevector <6 x double> [[TMP39]], <6 x double> [[TMP40]], <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
+; CHECK-NEXT:    br label [[IF_END]]
+
+; CHECK-LABEL: if.end:
+; CHECK-NEXT:    [[MERGE:%.*]] = phi <9 x double> [ [[TMP20]], [[IF_THEN]] ], [ [[TMP41]], [[IF_ELSE]] ]
+; CHECK-NEXT:    [[SPLIT6:%.*]] = shufflevector <9 x double> [[C:%.*]], <9 x double> undef, <3 x i32> <i32 0, i32 1, i32 2>
+; CHECK-NEXT:    [[SPLIT7:%.*]] = shufflevector <9 x double> [[C]], <9 x double> undef, <3 x i32> <i32 3, i32 4, i32 5>
+; CHECK-NEXT:    [[SPLIT8:%.*]] = shufflevector <9 x double> [[C]], <9 x double> undef, <3 x i32> <i32 6, i32 7, i32 8>
+; CHECK-NEXT:    [[SPLIT9:%.*]] = shufflevector <9 x double> [[MERGE]], <9 x double> undef, <3 x i32> <i32 0, i32 1, i32 2>
+; CHECK-NEXT:    [[SPLIT10:%.*]] = shufflevector <9 x double> [[MERGE]], <9 x double> undef, <3 x i32> <i32 3, i32 4, i32 5>
+; CHECK-NEXT:    [[SPLIT11:%.*]] = shufflevector <9 x double> [[MERGE]], <9 x double> undef, <3 x i32> <i32 6, i32 7, i32 8>
+; CHECK-NEXT:    [[BLOCK:%.*]] = shufflevector <3 x double> [[SPLIT6]], <3 x double> undef, <1 x i32> zeroinitializer
+;
+entry:
+  br i1 %cond, label %if.then, label %if.else
+
+if.then:                    ; preds = %entry
+  %A.trans = tail call <9 x double> @llvm.matrix.transpose.v9f64(<9 x double> %A, i32 3, i32 3)
+  br label %if.end
+
+if.else:                                       ; preds = %entry
+  %B.trans = tail call <9 x double> @llvm.matrix.transpose.v9f64(<9 x double> %B, i32 3, i32 3)
+  br label %if.end
+
+if.end:                                        ; preds = %if.then, %if.else
+  %merge = phi <9 x double> [ %A.trans, %if.then], [ %B.trans, %if.else ]
+  %res = tail call <9 x double> @llvm.matrix.multiply.v9f64.v9f64.v9f64(<9 x double> %C, <9 x double> %merge, i32 3, i32 3, i32 3)
+  ret <9 x double> %res
+}
+
+; Make sure we use a flattened vector when calling @foo and the use its flat vector result properly.
+define <9 x double> @unsupported_call(i1 %cond, <9 x double> %A, <9 x double> %B) {
+; CHECK-LABEL: @unsupported_call(
+; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <9 x double> [[A:%.*]], <9 x double> undef, <3 x i32> <i32 0, i32 1, i32 2>
+; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <9 x double> [[A]], <9 x double> undef, <3 x i32> <i32 3, i32 4, i32 5>
+; CHECK-NEXT:    [[SPLIT2:%.*]] = shufflevector <9 x double> [[A]], <9 x double> undef, <3 x i32> <i32 6, i32 7, i32 8>
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <3 x double> [[SPLIT]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <3 x double> undef, double [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <3 x double> [[SPLIT1]], i64 0
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <3 x double> [[TMP2]], double [[TMP3]], i64 1
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <3 x double> [[SPLIT2]], i64 0
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <3 x double> [[TMP4]], double [[TMP5]], i64 2
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <3 x double> [[SPLIT]], i64 1
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <3 x double> undef, double [[TMP7]], i64 0
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <3 x double> [[SPLIT1]], i64 1
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <3 x double> [[TMP8]], double [[TMP9]], i64 1
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <3 x double> [[SPLIT2]], i64 1
+; CHECK-NEXT:    [[TMP12:%.*]] = insertelement <3 x double> [[TMP10]], double [[TMP11]], i64 2
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <3 x double> [[SPLIT]], i64 2
+; CHECK-NEXT:    [[TMP14:%.*]] = insertelement <3 x double> undef, double [[TMP13]], i64 0
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <3 x double> [[SPLIT1]], i64 2
+; CHECK-NEXT:    [[TMP16:%.*]] = insertelement <3 x double> [[TMP14]], double [[TMP15]], i64 1
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <3 x double> [[SPLIT2]], i64 2
+; CHECK-NEXT:    [[TMP18:%.*]] = insertelement <3 x double> [[TMP16]], double [[TMP17]], i64 2
+; CHECK-NEXT:    [[TMP19:%.*]] = shufflevector <3 x double> [[TMP6]], <3 x double> [[TMP12]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
+; CHECK-NEXT:    [[TMP20:%.*]] = shufflevector <3 x double> [[TMP18]], <3 x double> undef, <6 x i32> <i32 0, i32 1, i32 2, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP21:%.*]] = shufflevector <6 x double> [[TMP19]], <6 x double> [[TMP20]], <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
+; CHECK-NEXT:    [[A_FOO:%.*]] = call <9 x double> @foo(<9 x double> [[TMP21]])
+; CHECK-NEXT:    [[SPLIT3:%.*]] = shufflevector <9 x double> [[B:%.*]], <9 x double> undef, <3 x i32> <i32 0, i32 1, i32 2>
+; CHECK-NEXT:    [[SPLIT4:%.*]] = shufflevector <9 x double> [[B]], <9 x double> undef, <3 x i32> <i32 3, i32 4, i32 5>
+; CHECK-NEXT:    [[SPLIT5:%.*]] = shufflevector <9 x double> [[B]], <9 x double> undef, <3 x i32> <i32 6, i32 7, i32 8>
+; CHECK-NEXT:    [[SPLIT6:%.*]] = shufflevector <9 x double> [[A_FOO]], <9 x double> undef, <3 x i32> <i32 0, i32 1, i32 2>
+; CHECK-NEXT:    [[SPLIT7:%.*]] = shufflevector <9 x double> [[A_FOO]], <9 x double> undef, <3 x i32> <i32 3, i32 4, i32 5>
+; CHECK-NEXT:    [[SPLIT8:%.*]] = shufflevector <9 x double> [[A_FOO]], <9 x double> undef, <3 x i32> <i32 6, i32 7, i32 8>
+;
+  %A.trans = tail call <9 x double> @llvm.matrix.transpose.v9f64(<9 x double> %A, i32 3, i32 3)
+  %A.foo = call <9 x double> @foo(<9 x double> %A.trans)
+  %res = tail call <9 x double> @llvm.matrix.multiply.v9f64.v9f64.v9f64(<9 x double> %B, <9 x double> %A.foo, i32 3, i32 3, i32 3)
+  ret <9 x double> %res
+}
+
+declare <9 x double> @llvm.matrix.multiply.v9f64.v9f64.v9f64(<9 x double>, <9 x double>, i32 immarg, i32 immarg, i32 immarg)
+declare <9 x double> @llvm.matrix.transpose.v9f64(<9 x double>, i32 immarg, i32 immarg)
+declare <9 x double> @foo(<9 x double>)
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/propagate-forward.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/propagate-forward.ll
new file mode 100644
index 000000000..1092aa283
--- /dev/null
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/propagate-forward.ll
@@ -0,0 +1,116 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -lower-matrix-intrinsics -S < %s | FileCheck  %s
+; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s
+
+; Check that we do not emit shufflevectors to flatten the result of the
+; transpose and store the columns directly.
+define void @transpose_store(<8 x double> %a, <8 x double>* %Ptr) {
+; CHECK-LABEL: @transpose_store(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <8 x double> [[A:%.*]], <8 x double> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <8 x double> [[A]], <8 x double> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[SPLIT2:%.*]] = shufflevector <8 x double> [[A]], <8 x double> undef, <2 x i32> <i32 4, i32 5>
+; CHECK-NEXT:    [[SPLIT3:%.*]] = shufflevector <8 x double> [[A]], <8 x double> undef, <2 x i32> <i32 6, i32 7>
+; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <2 x double> [[SPLIT]], i64 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x double> undef, double [[TMP0]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x double> [[SPLIT1]], i64 0
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x double> [[TMP1]], double [[TMP2]], i64 1
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x double> [[SPLIT2]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <4 x double> [[TMP3]], double [[TMP4]], i64 2
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x double> [[SPLIT3]], i64 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <4 x double> [[TMP5]], double [[TMP6]], i64 3
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x double> [[SPLIT]], i64 1
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <4 x double> undef, double [[TMP8]], i64 0
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <2 x double> [[SPLIT1]], i64 1
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <4 x double> [[TMP9]], double [[TMP10]], i64 1
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <2 x double> [[SPLIT2]], i64 1
+; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <4 x double> [[TMP11]], double [[TMP12]], i64 2
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x double> [[SPLIT3]], i64 1
+; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <4 x double> [[TMP13]], double [[TMP14]], i64 3
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <8 x double>* [[PTR:%.*]] to double*
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast double* [[TMP16]] to <4 x double>*
+; CHECK-NEXT:    store <4 x double> [[TMP7]], <4 x double>* [[TMP17]], align 8
+; CHECK-NEXT:    [[TMP18:%.*]] = getelementptr double, double* [[TMP16]], i32 4
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast double* [[TMP18]] to <4 x double>*
+; CHECK-NEXT:    store <4 x double> [[TMP15]], <4 x double>* [[TMP19]], align 8
+; CHECK-NEXT:    ret void
+;
+entry:
+  %c  = call <8 x double> @llvm.matrix.transpose(<8 x double> %a, i32 2, i32 4)
+  store <8 x double> %c, <8 x double>* %Ptr
+  ret void
+}
+
+declare <8 x double> @llvm.matrix.transpose(<8 x double>, i32, i32)
+
+define <8 x double> @transpose_fadd(<8 x double> %a) {
+; CHECK-LABEL: @transpose_fadd(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <8 x double> [[A:%.*]], <8 x double> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <8 x double> [[A]], <8 x double> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[SPLIT2:%.*]] = shufflevector <8 x double> [[A]], <8 x double> undef, <2 x i32> <i32 4, i32 5>
+; CHECK-NEXT:    [[SPLIT3:%.*]] = shufflevector <8 x double> [[A]], <8 x double> undef, <2 x i32> <i32 6, i32 7>
+; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <2 x double> [[SPLIT]], i64 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x double> undef, double [[TMP0]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x double> [[SPLIT1]], i64 0
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x double> [[TMP1]], double [[TMP2]], i64 1
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x double> [[SPLIT2]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <4 x double> [[TMP3]], double [[TMP4]], i64 2
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x double> [[SPLIT3]], i64 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <4 x double> [[TMP5]], double [[TMP6]], i64 3
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x double> [[SPLIT]], i64 1
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <4 x double> undef, double [[TMP8]], i64 0
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <2 x double> [[SPLIT1]], i64 1
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <4 x double> [[TMP9]], double [[TMP10]], i64 1
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <2 x double> [[SPLIT2]], i64 1
+; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <4 x double> [[TMP11]], double [[TMP12]], i64 2
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x double> [[SPLIT3]], i64 1
+; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <4 x double> [[TMP13]], double [[TMP14]], i64 3
+; CHECK-NEXT:    [[SPLIT4:%.*]] = shufflevector <8 x double> [[A]], <8 x double> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[SPLIT5:%.*]] = shufflevector <8 x double> [[A]], <8 x double> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP16:%.*]] = fadd <4 x double> [[TMP7]], [[SPLIT4]]
+; CHECK-NEXT:    [[TMP17:%.*]] = fadd <4 x double> [[TMP15]], [[SPLIT5]]
+; CHECK-NEXT:    [[TMP18:%.*]] = shufflevector <4 x double> [[TMP16]], <4 x double> [[TMP17]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    ret <8 x double> [[TMP18]]
+;
+entry:
+  %c  = call <8 x double> @llvm.matrix.transpose(<8 x double> %a, i32 2, i32 4)
+  %res = fadd <8 x double> %c, %a
+  ret <8 x double> %res
+}
+
+define <8 x double> @transpose_fmul(<8 x double> %a) {
+; CHECK-LABEL: @transpose_fmul(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <8 x double> [[A:%.*]], <8 x double> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <8 x double> [[A]], <8 x double> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[SPLIT2:%.*]] = shufflevector <8 x double> [[A]], <8 x double> undef, <2 x i32> <i32 4, i32 5>
+; CHECK-NEXT:    [[SPLIT3:%.*]] = shufflevector <8 x double> [[A]], <8 x double> undef, <2 x i32> <i32 6, i32 7>
+; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <2 x double> [[SPLIT]], i64 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x double> undef, double [[TMP0]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x double> [[SPLIT1]], i64 0
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x double> [[TMP1]], double [[TMP2]], i64 1
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x double> [[SPLIT2]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <4 x double> [[TMP3]], double [[TMP4]], i64 2
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x double> [[SPLIT3]], i64 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <4 x double> [[TMP5]], double [[TMP6]], i64 3
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x double> [[SPLIT]], i64 1
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <4 x double> undef, double [[TMP8]], i64 0
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <2 x double> [[SPLIT1]], i64 1
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <4 x double> [[TMP9]], double [[TMP10]], i64 1
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <2 x double> [[SPLIT2]], i64 1
+; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <4 x double> [[TMP11]], double [[TMP12]], i64 2
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x double> [[SPLIT3]], i64 1
+; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <4 x double> [[TMP13]], double [[TMP14]], i64 3
+; CHECK-NEXT:    [[SPLIT4:%.*]] = shufflevector <8 x double> [[A]], <8 x double> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[SPLIT5:%.*]] = shufflevector <8 x double> [[A]], <8 x double> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP16:%.*]] = fmul <4 x double> [[TMP7]], [[SPLIT4]]
+; CHECK-NEXT:    [[TMP17:%.*]] = fmul <4 x double> [[TMP15]], [[SPLIT5]]
+; CHECK-NEXT:    [[TMP18:%.*]] = shufflevector <4 x double> [[TMP16]], <4 x double> [[TMP17]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    ret <8 x double> [[TMP18]]
+;
+entry:
+  %c  = call <8 x double> @llvm.matrix.transpose(<8 x double> %a, i32 2, i32 4)
+  %res = fmul <8 x double> %c, %a
+  ret <8 x double> %res
+}
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/propagate-mixed-users.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/propagate-mixed-users.ll
new file mode 100644
index 000000000..f2bb272ba
--- /dev/null
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/propagate-mixed-users.ll
@@ -0,0 +1,53 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -lower-matrix-intrinsics -S < %s | FileCheck %s
+; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s
+
+; Currently we only lower stores with shape information, but need to embed the
+; matrix in a flat vector for function calls and returns.
+define <8 x double> @strided_load_4x4(<8 x double> %in, <8 x double>* %Ptr) {
+; CHECK-LABEL: @strided_load_4x4(
+; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <8 x double> [[IN:%.*]], <8 x double> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <8 x double> [[IN]], <8 x double> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x double> [[SPLIT]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> undef, double [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x double> [[SPLIT1]], i64 0
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <2 x double> [[TMP2]], double [[TMP3]], i64 1
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x double> [[SPLIT]], i64 1
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x double> undef, double [[TMP5]], i64 0
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x double> [[SPLIT1]], i64 1
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <2 x double> [[TMP6]], double [[TMP7]], i64 1
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <4 x double> [[SPLIT]], i64 2
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <2 x double> undef, double [[TMP9]], i64 0
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <4 x double> [[SPLIT1]], i64 2
+; CHECK-NEXT:    [[TMP12:%.*]] = insertelement <2 x double> [[TMP10]], double [[TMP11]], i64 1
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <4 x double> [[SPLIT]], i64 3
+; CHECK-NEXT:    [[TMP14:%.*]] = insertelement <2 x double> undef, double [[TMP13]], i64 0
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <4 x double> [[SPLIT1]], i64 3
+; CHECK-NEXT:    [[TMP16:%.*]] = insertelement <2 x double> [[TMP14]], double [[TMP15]], i64 1
+; CHECK-NEXT:    [[TMP17:%.*]] = shufflevector <2 x double> [[TMP4]], <2 x double> [[TMP8]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP18:%.*]] = shufflevector <2 x double> [[TMP12]], <2 x double> [[TMP16]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP19:%.*]] = shufflevector <4 x double> [[TMP17]], <4 x double> [[TMP18]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast <8 x double>* [[PTR:%.*]] to double*
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast double* [[TMP20]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP4]], <2 x double>* [[TMP21]], align 8
+; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr double, double* [[TMP20]], i32 2
+; CHECK-NEXT:    [[TMP23:%.*]] = bitcast double* [[TMP22]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP8]], <2 x double>* [[TMP23]], align 8
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr double, double* [[TMP20]], i32 4
+; CHECK-NEXT:    [[TMP25:%.*]] = bitcast double* [[TMP24]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP12]], <2 x double>* [[TMP25]], align 8
+; CHECK-NEXT:    [[TMP26:%.*]] = getelementptr double, double* [[TMP20]], i32 6
+; CHECK-NEXT:    [[TMP27:%.*]] = bitcast double* [[TMP26]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP16]], <2 x double>* [[TMP27]], align 8
+; CHECK-NEXT:    call void @foo(<8 x double> [[TMP19]])
+; CHECK-NEXT:    ret <8 x double> [[TMP19]]
+;
+  %transposed = call <8 x double> @llvm.matrix.transpose(<8 x double> %in, i32 4, i32 2)
+  store <8 x double> %transposed, <8 x double>* %Ptr
+  call void @foo(<8 x double> %transposed)
+  ret <8 x double> %transposed
+}
+
+declare <8 x double> @llvm.matrix.transpose(<8 x double>, i32, i32)
+
+declare void @foo(<8 x double>)
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/propagate-multiple-iterations.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/propagate-multiple-iterations.ll
new file mode 100644
index 000000000..38200b388
--- /dev/null
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/propagate-multiple-iterations.ll
@@ -0,0 +1,84 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -lower-matrix-intrinsics -S < %s | FileCheck %s
+; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s
+
+
+; Make sure we propagate in multiple iterations. First, we back-propagate the
+; shape information from the transpose to %A, in the next iteration we
+; forward-propagate it to %Mul, and then back to %B.
+define <16 x double> @backpropagation_iterations(<16 x double>* %A.Ptr, <16 x double>* %B.Ptr) {
+; CHECK-LABEL: @backpropagation_iterations(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <16 x double>* [[A_PTR:%.*]] to double*
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast double* [[TMP1]] to <4 x double>*
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x double>, <4 x double>* [[TMP2]], align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr double, double* [[TMP1]], i32 4
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast double* [[TMP5]] to <4 x double>*
+; CHECK-NEXT:    [[TMP7:%.*]] = load <4 x double>, <4 x double>* [[TMP6]], align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr double, double* [[TMP1]], i32 8
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast double* [[TMP9]] to <4 x double>*
+; CHECK-NEXT:    [[TMP11:%.*]] = load <4 x double>, <4 x double>* [[TMP10]], align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr double, double* [[TMP1]], i32 12
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast double* [[TMP13]] to <4 x double>*
+; CHECK-NEXT:    [[TMP15:%.*]] = load <4 x double>, <4 x double>* [[TMP14]], align 8
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <4 x double> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP17:%.*]] = insertelement <4 x double> undef, double [[TMP16]], i64 0
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <4 x double> [[TMP7]], i64 0
+; CHECK-NEXT:    [[TMP19:%.*]] = insertelement <4 x double> [[TMP17]], double [[TMP18]], i64 1
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <4 x double> [[TMP11]], i64 0
+; CHECK-NEXT:    [[TMP21:%.*]] = insertelement <4 x double> [[TMP19]], double [[TMP20]], i64 2
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <4 x double> [[TMP15]], i64 0
+; CHECK-NEXT:    [[TMP23:%.*]] = insertelement <4 x double> [[TMP21]], double [[TMP22]], i64 3
+; CHECK-NEXT:    [[TMP24:%.*]] = extractelement <4 x double> [[TMP3]], i64 1
+; CHECK-NEXT:    [[TMP25:%.*]] = insertelement <4 x double> undef, double [[TMP24]], i64 0
+; CHECK-NEXT:    [[TMP26:%.*]] = extractelement <4 x double> [[TMP7]], i64 1
+; CHECK-NEXT:    [[TMP27:%.*]] = insertelement <4 x double> [[TMP25]], double [[TMP26]], i64 1
+; CHECK-NEXT:    [[TMP28:%.*]] = extractelement <4 x double> [[TMP11]], i64 1
+; CHECK-NEXT:    [[TMP29:%.*]] = insertelement <4 x double> [[TMP27]], double [[TMP28]], i64 2
+; CHECK-NEXT:    [[TMP30:%.*]] = extractelement <4 x double> [[TMP15]], i64 1
+; CHECK-NEXT:    [[TMP31:%.*]] = insertelement <4 x double> [[TMP29]], double [[TMP30]], i64 3
+; CHECK-NEXT:    [[TMP32:%.*]] = extractelement <4 x double> [[TMP3]], i64 2
+; CHECK-NEXT:    [[TMP33:%.*]] = insertelement <4 x double> undef, double [[TMP32]], i64 0
+; CHECK-NEXT:    [[TMP34:%.*]] = extractelement <4 x double> [[TMP7]], i64 2
+; CHECK-NEXT:    [[TMP35:%.*]] = insertelement <4 x double> [[TMP33]], double [[TMP34]], i64 1
+; CHECK-NEXT:    [[TMP36:%.*]] = extractelement <4 x double> [[TMP11]], i64 2
+; CHECK-NEXT:    [[TMP37:%.*]] = insertelement <4 x double> [[TMP35]], double [[TMP36]], i64 2
+; CHECK-NEXT:    [[TMP38:%.*]] = extractelement <4 x double> [[TMP15]], i64 2
+; CHECK-NEXT:    [[TMP39:%.*]] = insertelement <4 x double> [[TMP37]], double [[TMP38]], i64 3
+; CHECK-NEXT:    [[TMP40:%.*]] = extractelement <4 x double> [[TMP3]], i64 3
+; CHECK-NEXT:    [[TMP41:%.*]] = insertelement <4 x double> undef, double [[TMP40]], i64 0
+; CHECK-NEXT:    [[TMP42:%.*]] = extractelement <4 x double> [[TMP7]], i64 3
+; CHECK-NEXT:    [[TMP43:%.*]] = insertelement <4 x double> [[TMP41]], double [[TMP42]], i64 1
+; CHECK-NEXT:    [[TMP44:%.*]] = extractelement <4 x double> [[TMP11]], i64 3
+; CHECK-NEXT:    [[TMP45:%.*]] = insertelement <4 x double> [[TMP43]], double [[TMP44]], i64 2
+; CHECK-NEXT:    [[TMP46:%.*]] = extractelement <4 x double> [[TMP15]], i64 3
+; CHECK-NEXT:    [[TMP47:%.*]] = insertelement <4 x double> [[TMP45]], double [[TMP46]], i64 3
+; CHECK-NEXT:    [[TMP48:%.*]] = bitcast <16 x double>* [[B_PTR:%.*]] to double*
+; CHECK-NEXT:    [[TMP49:%.*]] = bitcast double* [[TMP48]] to <4 x double>*
+; CHECK-NEXT:    [[TMP50:%.*]] = load <4 x double>, <4 x double>* [[TMP49]], align 8
+; CHECK-NEXT:    [[TMP52:%.*]] = getelementptr double, double* [[TMP48]], i32 4
+; CHECK-NEXT:    [[TMP53:%.*]] = bitcast double* [[TMP52]] to <4 x double>*
+; CHECK-NEXT:    [[TMP54:%.*]] = load <4 x double>, <4 x double>* [[TMP53]], align 8
+; CHECK-NEXT:    [[TMP56:%.*]] = getelementptr double, double* [[TMP48]], i32 8
+; CHECK-NEXT:    [[TMP57:%.*]] = bitcast double* [[TMP56]] to <4 x double>*
+; CHECK-NEXT:    [[TMP58:%.*]] = load <4 x double>, <4 x double>* [[TMP57]], align 8
+; CHECK-NEXT:    [[TMP60:%.*]] = getelementptr double, double* [[TMP48]], i32 12
+; CHECK-NEXT:    [[TMP61:%.*]] = bitcast double* [[TMP60]] to <4 x double>*
+; CHECK-NEXT:    [[TMP62:%.*]] = load <4 x double>, <4 x double>* [[TMP61]], align 8
+; CHECK-NEXT:    [[TMP63:%.*]] = fmul <4 x double> [[TMP3]], [[TMP50]]
+; CHECK-NEXT:    [[TMP64:%.*]] = fmul <4 x double> [[TMP7]], [[TMP54]]
+; CHECK-NEXT:    [[TMP65:%.*]] = fmul <4 x double> [[TMP11]], [[TMP58]]
+; CHECK-NEXT:    [[TMP66:%.*]] = fmul <4 x double> [[TMP15]], [[TMP62]]
+; CHECK-NEXT:    [[TMP67:%.*]] = shufflevector <4 x double> [[TMP63]], <4 x double> [[TMP64]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP68:%.*]] = shufflevector <4 x double> [[TMP65]], <4 x double> [[TMP66]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP69:%.*]] = shufflevector <8 x double> [[TMP67]], <8 x double> [[TMP68]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    ret <16 x double> [[TMP69]]
+;
+  %A = load <16 x double>, <16 x double>* %A.Ptr
+  %A.trans = tail call <16 x double> @llvm.matrix.transpose.v16f64(<16 x double> %A, i32 4, i32 4)
+  %B = load <16 x double>, <16 x double>* %B.Ptr
+  %Mul = fmul <16 x double> %A, %B
+  ret <16 x double> %Mul
+}
+
+declare <16 x double> @llvm.matrix.multiply.v16f64.v16f64.v16f64(<16 x double>, <16 x double>, i32 immarg, i32 immarg, i32 immarg)
+declare <16 x double> @llvm.matrix.transpose.v16f64(<16 x double>, i32 immarg, i32 immarg)
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/strided-load-double.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/strided-load-double.ll
new file mode 100644
index 000000000..c5bdb4f32
--- /dev/null
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/strided-load-double.ll
@@ -0,0 +1,68 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -lower-matrix-intrinsics -S < %s | FileCheck %s
+; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s
+
+define <9 x double> @strided_load_3x3(<9 x double>* %in, i32 %stride) {
+; CHECK-LABEL: @strided_load_3x3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <9 x double>* [[IN:%.*]] to double*
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i32 0, [[STRIDE:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr double, double* [[TMP0]], i32 [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast double* [[TMP2]] to <3 x double>*
+; CHECK-NEXT:    [[TMP4:%.*]] = load <3 x double>, <3 x double>* [[TMP3]], align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = mul i32 1, [[STRIDE]]
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr double, double* [[TMP0]], i32 [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast double* [[TMP6]] to <3 x double>*
+; CHECK-NEXT:    [[TMP8:%.*]] = load <3 x double>, <3 x double>* [[TMP7]], align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = mul i32 2, [[STRIDE]]
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr double, double* [[TMP0]], i32 [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast double* [[TMP10]] to <3 x double>*
+; CHECK-NEXT:    [[TMP12:%.*]] = load <3 x double>, <3 x double>* [[TMP11]], align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <3 x double> [[TMP4]], <3 x double> [[TMP8]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <3 x double> [[TMP12]], <3 x double> undef, <6 x i32> <i32 0, i32 1, i32 2, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP15:%.*]] = shufflevector <6 x double> [[TMP13]], <6 x double> [[TMP14]], <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
+; CHECK-NEXT:    ret <9 x double> [[TMP15]]
+;
+entry:
+  %load = call <9 x double> @llvm.matrix.columnwise.load(<9 x double>* %in, i32 %stride, i32 3, i32 3)
+  ret <9 x double> %load
+}
+
+declare <9 x double> @llvm.matrix.columnwise.load(<9 x double>*, i32, i32, i32)
+
+define <9 x double> @strided_load_9x1(<9 x double>* %in, i32 %stride) {
+; CHECK-LABEL: @strided_load_9x1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <9 x double>* [[IN:%.*]] to double*
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i32 0, [[STRIDE:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr double, double* [[TMP0]], i32 [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast double* [[TMP2]] to <9 x double>*
+; CHECK-NEXT:    [[TMP4:%.*]] = load <9 x double>, <9 x double>* [[TMP3]], align 8
+; CHECK-NEXT:    ret <9 x double> [[TMP4]]
+;
+entry:
+  %load = call <9 x double> @llvm.matrix.columnwise.load(<9 x double>* %in, i32 %stride, i32 9, i32 1)
+  ret <9 x double> %load
+}
+
+declare <8 x double> @llvm.matrix.columnwise.load.v8f64(<8 x double>*, i32, i32, i32)
+
+define <8 x double> @strided_load_4x2(<8 x double>* %in, i32 %stride) {
+; CHECK-LABEL: @strided_load_4x2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x double>* [[IN:%.*]] to double*
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i32 0, [[STRIDE:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr double, double* [[TMP0]], i32 [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast double* [[TMP2]] to <4 x double>*
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x double>, <4 x double>* [[TMP3]], align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = mul i32 1, [[STRIDE]]
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr double, double* [[TMP0]], i32 [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast double* [[TMP6]] to <4 x double>*
+; CHECK-NEXT:    [[TMP8:%.*]] = load <4 x double>, <4 x double>* [[TMP7]], align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <4 x double> [[TMP4]], <4 x double> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    ret <8 x double> [[TMP9]]
+;
+entry:
+  %load = call <8 x double> @llvm.matrix.columnwise.load.v8f64(<8 x double>* %in, i32 %stride, i32 4, i32 2)
+  ret <8 x double> %load
+}
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/strided-load-float.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/strided-load-float.ll
new file mode 100644
index 000000000..a66c4671d
--- /dev/null
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/strided-load-float.ll
@@ -0,0 +1,68 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -lower-matrix-intrinsics -S < %s | FileCheck %s
+; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s
+
+define <9 x float> @strided_load_3x3(<9 x float>* %in, i32 %stride) {
+; CHECK-LABEL: @strided_load_3x3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <9 x float>* [[IN:%.*]] to float*
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i32 0, [[STRIDE:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr float, float* [[TMP0]], i32 [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float* [[TMP2]] to <3 x float>*
+; CHECK-NEXT:    [[TMP4:%.*]] = load <3 x float>, <3 x float>* [[TMP3]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = mul i32 1, [[STRIDE]]
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr float, float* [[TMP0]], i32 [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast float* [[TMP6]] to <3 x float>*
+; CHECK-NEXT:    [[TMP8:%.*]] = load <3 x float>, <3 x float>* [[TMP7]], align 4
+; CHECK-NEXT:    [[TMP9:%.*]] = mul i32 2, [[STRIDE]]
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr float, float* [[TMP0]], i32 [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast float* [[TMP10]] to <3 x float>*
+; CHECK-NEXT:    [[TMP12:%.*]] = load <3 x float>, <3 x float>* [[TMP11]], align 4
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <3 x float> [[TMP4]], <3 x float> [[TMP8]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <3 x float> [[TMP12]], <3 x float> undef, <6 x i32> <i32 0, i32 1, i32 2, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP15:%.*]] = shufflevector <6 x float> [[TMP13]], <6 x float> [[TMP14]], <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
+; CHECK-NEXT:    ret <9 x float> [[TMP15]]
+;
+entry:
+  %load = call <9 x float> @llvm.matrix.columnwise.load(<9 x float>* %in, i32 %stride, i32 3, i32 3)
+  ret <9 x float> %load
+}
+
+declare <9 x float> @llvm.matrix.columnwise.load(<9 x float>*, i32, i32, i32)
+
+define <9 x float> @strided_load_9x1(<9 x float>* %in, i32 %stride) {
+; CHECK-LABEL: @strided_load_9x1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <9 x float>* [[IN:%.*]] to float*
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i32 0, [[STRIDE:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr float, float* [[TMP0]], i32 [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float* [[TMP2]] to <9 x float>*
+; CHECK-NEXT:    [[TMP4:%.*]] = load <9 x float>, <9 x float>* [[TMP3]], align 4
+; CHECK-NEXT:    ret <9 x float> [[TMP4]]
+;
+entry:
+  %load = call <9 x float> @llvm.matrix.columnwise.load(<9 x float>* %in, i32 %stride, i32 9, i32 1)
+  ret <9 x float> %load
+}
+
+declare <8 x float> @llvm.matrix.columnwise.load.v8f32(<8 x float>*, i32, i32, i32)
+
+define <8 x float> @strided_load_4x2(<8 x float>* %in, i32 %stride) {
+; CHECK-LABEL: @strided_load_4x2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x float>* [[IN:%.*]] to float*
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i32 0, [[STRIDE:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr float, float* [[TMP0]], i32 [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float* [[TMP2]] to <4 x float>*
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x float>, <4 x float>* [[TMP3]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = mul i32 1, [[STRIDE]]
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr float, float* [[TMP0]], i32 [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast float* [[TMP6]] to <4 x float>*
+; CHECK-NEXT:    [[TMP8:%.*]] = load <4 x float>, <4 x float>* [[TMP7]], align 4
+; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <4 x float> [[TMP4]], <4 x float> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    ret <8 x float> [[TMP9]]
+;
+entry:
+  %load = call <8 x float> @llvm.matrix.columnwise.load.v8f32(<8 x float>* %in, i32 %stride, i32 4, i32 2)
+  ret <8 x float> %load
+}
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/strided-load-i32.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/strided-load-i32.ll
new file mode 100644
index 000000000..d2351abc5
--- /dev/null
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/strided-load-i32.ll
@@ -0,0 +1,68 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -lower-matrix-intrinsics -S < %s | FileCheck %s
+; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s
+
+define <9 x i32> @strided_load_3x3(<9 x i32>* %in, i32 %stride) {
+; CHECK-LABEL: @strided_load_3x3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <9 x i32>* [[IN:%.*]] to i32*
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i32 0, [[STRIDE:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr i32, i32* [[TMP0]], i32 [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i32* [[TMP2]] to <3 x i32>*
+; CHECK-NEXT:    [[TMP4:%.*]] = load <3 x i32>, <3 x i32>* [[TMP3]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = mul i32 1, [[STRIDE]]
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr i32, i32* [[TMP0]], i32 [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i32* [[TMP6]] to <3 x i32>*
+; CHECK-NEXT:    [[TMP8:%.*]] = load <3 x i32>, <3 x i32>* [[TMP7]], align 4
+; CHECK-NEXT:    [[TMP9:%.*]] = mul i32 2, [[STRIDE]]
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr i32, i32* [[TMP0]], i32 [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i32* [[TMP10]] to <3 x i32>*
+; CHECK-NEXT:    [[TMP12:%.*]] = load <3 x i32>, <3 x i32>* [[TMP11]], align 4
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <3 x i32> [[TMP4]], <3 x i32> [[TMP8]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <3 x i32> [[TMP12]], <3 x i32> undef, <6 x i32> <i32 0, i32 1, i32 2, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP15:%.*]] = shufflevector <6 x i32> [[TMP13]], <6 x i32> [[TMP14]], <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
+; CHECK-NEXT:    ret <9 x i32> [[TMP15]]
+;
+entry:
+  %load = call <9 x i32> @llvm.matrix.columnwise.load(<9 x i32>* %in, i32 %stride, i32 3, i32 3)
+  ret <9 x i32> %load
+}
+
+declare <9 x i32> @llvm.matrix.columnwise.load(<9 x i32>*, i32, i32, i32)
+
+define <9 x i32> @strided_load_9x1(<9 x i32>* %in, i32 %stride) {
+; CHECK-LABEL: @strided_load_9x1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <9 x i32>* [[IN:%.*]] to i32*
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i32 0, [[STRIDE:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr i32, i32* [[TMP0]], i32 [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i32* [[TMP2]] to <9 x i32>*
+; CHECK-NEXT:    [[TMP4:%.*]] = load <9 x i32>, <9 x i32>* [[TMP3]], align 4
+; CHECK-NEXT:    ret <9 x i32> [[TMP4]]
+;
+entry:
+  %load = call <9 x i32> @llvm.matrix.columnwise.load(<9 x i32>* %in, i32 %stride, i32 9, i32 1)
+  ret <9 x i32> %load
+}
+
+declare <8 x i32> @llvm.matrix.columnwise.load.v8i32(<8 x i32>*, i32, i32, i32)
+
+define <8 x i32> @strided_load_4x2(<8 x i32>* %in, i32 %stride) {
+; CHECK-LABEL: @strided_load_4x2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x i32>* [[IN:%.*]] to i32*
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i32 0, [[STRIDE:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr i32, i32* [[TMP0]], i32 [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i32* [[TMP2]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i32>, <4 x i32>* [[TMP3]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = mul i32 1, [[STRIDE]]
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr i32, i32* [[TMP0]], i32 [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i32* [[TMP6]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP8:%.*]] = load <4 x i32>, <4 x i32>* [[TMP7]], align 4
+; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    ret <8 x i32> [[TMP9]]
+;
+entry:
+  %load = call <8 x i32> @llvm.matrix.columnwise.load.v8i32(<8 x i32>* %in, i32 %stride, i32 4, i32 2)
+  ret <8 x i32> %load
+}
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/strided-store-double.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/strided-store-double.ll
new file mode 100644
index 000000000..377c7effb
--- /dev/null
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/strided-store-double.ll
@@ -0,0 +1,68 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -lower-matrix-intrinsics -S < %s | FileCheck %s
+; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s
+
+define void @strided_store_3x2(<6 x double> %in, double* %out) {
+; CHECK-LABEL: @strided_store_3x2(
+; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <6 x double> [[IN:%.*]], <6 x double> undef, <3 x i32> <i32 0, i32 1, i32 2>
+; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <6 x double> [[IN]], <6 x double> undef, <3 x i32> <i32 3, i32 4, i32 5>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast double* [[OUT:%.*]] to <3 x double>*
+; CHECK-NEXT:    store <3 x double> [[SPLIT]], <3 x double>* [[TMP1]], align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr double, double* [[OUT]], i32 5
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast double* [[TMP2]] to <3 x double>*
+; CHECK-NEXT:    store <3 x double> [[SPLIT1]], <3 x double>* [[TMP3]], align 8
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.matrix.columnwise.store(<6 x double> %in, double* %out, i32 5, i32 3, i32 2)
+  ret void
+}
+
+define void @strided_store_3x2_nonconst_stride(<6 x double> %in, i32 %stride, double* %out) {
+; CHECK-LABEL: @strided_store_3x2_nonconst_stride(
+; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <6 x double> [[IN:%.*]], <6 x double> undef, <3 x i32> <i32 0, i32 1, i32 2>
+; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <6 x double> [[IN]], <6 x double> undef, <3 x i32> <i32 3, i32 4, i32 5>
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i32 0, [[STRIDE:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr double, double* [[OUT:%.*]], i32 [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast double* [[TMP2]] to <3 x double>*
+; CHECK-NEXT:    store <3 x double> [[SPLIT]], <3 x double>* [[TMP3]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = mul i32 1, [[STRIDE]]
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr double, double* [[OUT]], i32 [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast double* [[TMP5]] to <3 x double>*
+; CHECK-NEXT:    store <3 x double> [[SPLIT1]], <3 x double>* [[TMP6]], align 8
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.matrix.columnwise.store(<6 x double> %in, double* %out, i32 %stride, i32 3, i32 2)
+  ret void
+}
+
+
+declare void @llvm.matrix.columnwise.store(<6 x double>, double*, i32, i32, i32)
+
+define void @strided_store_2x3(<10 x double> %in, double* %out) {
+; CHECK-LABEL: @strided_store_2x3(
+; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <10 x double> [[IN:%.*]], <10 x double> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <10 x double> [[IN]], <10 x double> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[SPLIT2:%.*]] = shufflevector <10 x double> [[IN]], <10 x double> undef, <2 x i32> <i32 4, i32 5>
+; CHECK-NEXT:    [[SPLIT3:%.*]] = shufflevector <10 x double> [[IN]], <10 x double> undef, <2 x i32> <i32 6, i32 7>
+; CHECK-NEXT:    [[SPLIT4:%.*]] = shufflevector <10 x double> [[IN]], <10 x double> undef, <2 x i32> <i32 8, i32 9>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast double* [[OUT:%.*]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[SPLIT]], <2 x double>* [[TMP1]], align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr double, double* [[OUT]], i32 4
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast double* [[TMP2]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[SPLIT1]], <2 x double>* [[TMP3]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr double, double* [[OUT]], i32 8
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast double* [[TMP4]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[SPLIT2]], <2 x double>* [[TMP5]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr double, double* [[OUT]], i32 12
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast double* [[TMP6]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[SPLIT3]], <2 x double>* [[TMP7]], align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr double, double* [[OUT]], i32 16
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast double* [[TMP8]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[SPLIT4]], <2 x double>* [[TMP9]], align 8
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.matrix.columnwise.store.v10f64(<10 x double> %in, double* %out, i32 4, i32 2, i32 5)
+  ret void
+}
+
+declare void @llvm.matrix.columnwise.store.v10f64(<10 x double>, double*, i32, i32, i32)
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/strided-store-float.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/strided-store-float.ll
new file mode 100644
index 000000000..dc9ff0496
--- /dev/null
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/strided-store-float.ll
@@ -0,0 +1,68 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -lower-matrix-intrinsics -S < %s | FileCheck %s
+; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s
+
+define void @strided_store_3x2(<6 x float> %in, float* %out) {
+; CHECK-LABEL: @strided_store_3x2(
+; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <6 x float> [[IN:%.*]], <6 x float> undef, <3 x i32> <i32 0, i32 1, i32 2>
+; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <6 x float> [[IN]], <6 x float> undef, <3 x i32> <i32 3, i32 4, i32 5>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float* [[OUT:%.*]] to <3 x float>*
+; CHECK-NEXT:    store <3 x float> [[SPLIT]], <3 x float>* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr float, float* [[OUT]], i32 5
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float* [[TMP2]] to <3 x float>*
+; CHECK-NEXT:    store <3 x float> [[SPLIT1]], <3 x float>* [[TMP3]], align 4
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.matrix.columnwise.store(<6 x float> %in, float* %out, i32 5, i32 3, i32 2)
+  ret void
+}
+
+define void @strided_store_3x2_nonconst_stride(<6 x float> %in, i32 %stride, float* %out) {
+; CHECK-LABEL: @strided_store_3x2_nonconst_stride(
+; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <6 x float> [[IN:%.*]], <6 x float> undef, <3 x i32> <i32 0, i32 1, i32 2>
+; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <6 x float> [[IN]], <6 x float> undef, <3 x i32> <i32 3, i32 4, i32 5>
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i32 0, [[STRIDE:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr float, float* [[OUT:%.*]], i32 [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float* [[TMP2]] to <3 x float>*
+; CHECK-NEXT:    store <3 x float> [[SPLIT]], <3 x float>* [[TMP3]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = mul i32 1, [[STRIDE]]
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr float, float* [[OUT]], i32 [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast float* [[TMP5]] to <3 x float>*
+; CHECK-NEXT:    store <3 x float> [[SPLIT1]], <3 x float>* [[TMP6]], align 4
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.matrix.columnwise.store(<6 x float> %in, float* %out, i32 %stride, i32 3, i32 2)
+  ret void
+}
+
+
+declare void @llvm.matrix.columnwise.store(<6 x float>, float*, i32, i32, i32)
+
+define void @strided_store_2x3(<10 x float> %in, float* %out) {
+; CHECK-LABEL: @strided_store_2x3(
+; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <10 x float> [[IN:%.*]], <10 x float> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <10 x float> [[IN]], <10 x float> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[SPLIT2:%.*]] = shufflevector <10 x float> [[IN]], <10 x float> undef, <2 x i32> <i32 4, i32 5>
+; CHECK-NEXT:    [[SPLIT3:%.*]] = shufflevector <10 x float> [[IN]], <10 x float> undef, <2 x i32> <i32 6, i32 7>
+; CHECK-NEXT:    [[SPLIT4:%.*]] = shufflevector <10 x float> [[IN]], <10 x float> undef, <2 x i32> <i32 8, i32 9>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float* [[OUT:%.*]] to <2 x float>*
+; CHECK-NEXT:    store <2 x float> [[SPLIT]], <2 x float>* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr float, float* [[OUT]], i32 4
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float* [[TMP2]] to <2 x float>*
+; CHECK-NEXT:    store <2 x float> [[SPLIT1]], <2 x float>* [[TMP3]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr float, float* [[OUT]], i32 8
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast float* [[TMP4]] to <2 x float>*
+; CHECK-NEXT:    store <2 x float> [[SPLIT2]], <2 x float>* [[TMP5]], align 4
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr float, float* [[OUT]], i32 12
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast float* [[TMP6]] to <2 x float>*
+; CHECK-NEXT:    store <2 x float> [[SPLIT3]], <2 x float>* [[TMP7]], align 4
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr float, float* [[OUT]], i32 16
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast float* [[TMP8]] to <2 x float>*
+; CHECK-NEXT:    store <2 x float> [[SPLIT4]], <2 x float>* [[TMP9]], align 4
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.matrix.columnwise.store.v10f32(<10 x float> %in, float* %out, i32 4, i32 2, i32 5)
+  ret void
+}
+
+declare void @llvm.matrix.columnwise.store.v10f32(<10 x float>, float*, i32, i32, i32)
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/strided-store-i32.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/strided-store-i32.ll
new file mode 100644
index 000000000..6c8af109c
--- /dev/null
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/strided-store-i32.ll
@@ -0,0 +1,68 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -lower-matrix-intrinsics -S < %s | FileCheck %s
+; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s
+
+define void @strided_store_3x2(<6 x i32> %in, i32* %out) {
+; CHECK-LABEL: @strided_store_3x2(
+; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <6 x i32> [[IN:%.*]], <6 x i32> undef, <3 x i32> <i32 0, i32 1, i32 2>
+; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <6 x i32> [[IN]], <6 x i32> undef, <3 x i32> <i32 3, i32 4, i32 5>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32* [[OUT:%.*]] to <3 x i32>*
+; CHECK-NEXT:    store <3 x i32> [[SPLIT]], <3 x i32>* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr i32, i32* [[OUT]], i32 5
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i32* [[TMP2]] to <3 x i32>*
+; CHECK-NEXT:    store <3 x i32> [[SPLIT1]], <3 x i32>* [[TMP3]], align 4
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.matrix.columnwise.store(<6 x i32> %in, i32* %out, i32 5, i32 3, i32 2)
+  ret void
+}
+
+define void @strided_store_3x2_nonconst_stride(<6 x i32> %in, i32 %stride, i32* %out) {
+; CHECK-LABEL: @strided_store_3x2_nonconst_stride(
+; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <6 x i32> [[IN:%.*]], <6 x i32> undef, <3 x i32> <i32 0, i32 1, i32 2>
+; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <6 x i32> [[IN]], <6 x i32> undef, <3 x i32> <i32 3, i32 4, i32 5>
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i32 0, [[STRIDE:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr i32, i32* [[OUT:%.*]], i32 [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i32* [[TMP2]] to <3 x i32>*
+; CHECK-NEXT:    store <3 x i32> [[SPLIT]], <3 x i32>* [[TMP3]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = mul i32 1, [[STRIDE]]
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr i32, i32* [[OUT]], i32 [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32* [[TMP5]] to <3 x i32>*
+; CHECK-NEXT:    store <3 x i32> [[SPLIT1]], <3 x i32>* [[TMP6]], align 4
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.matrix.columnwise.store(<6 x i32> %in, i32* %out, i32 %stride, i32 3, i32 2)
+  ret void
+}
+
+
+declare void @llvm.matrix.columnwise.store(<6 x i32>, i32*, i32, i32, i32)
+
+define void @strided_store_2x3(<10 x i32> %in, i32* %out) {
+; CHECK-LABEL: @strided_store_2x3(
+; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <10 x i32> [[IN:%.*]], <10 x i32> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <10 x i32> [[IN]], <10 x i32> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[SPLIT2:%.*]] = shufflevector <10 x i32> [[IN]], <10 x i32> undef, <2 x i32> <i32 4, i32 5>
+; CHECK-NEXT:    [[SPLIT3:%.*]] = shufflevector <10 x i32> [[IN]], <10 x i32> undef, <2 x i32> <i32 6, i32 7>
+; CHECK-NEXT:    [[SPLIT4:%.*]] = shufflevector <10 x i32> [[IN]], <10 x i32> undef, <2 x i32> <i32 8, i32 9>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32* [[OUT:%.*]] to <2 x i32>*
+; CHECK-NEXT:    store <2 x i32> [[SPLIT]], <2 x i32>* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr i32, i32* [[OUT]], i32 4
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i32* [[TMP2]] to <2 x i32>*
+; CHECK-NEXT:    store <2 x i32> [[SPLIT1]], <2 x i32>* [[TMP3]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr i32, i32* [[OUT]], i32 8
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32* [[TMP4]] to <2 x i32>*
+; CHECK-NEXT:    store <2 x i32> [[SPLIT2]], <2 x i32>* [[TMP5]], align 4
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr i32, i32* [[OUT]], i32 12
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i32* [[TMP6]] to <2 x i32>*
+; CHECK-NEXT:    store <2 x i32> [[SPLIT3]], <2 x i32>* [[TMP7]], align 4
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr i32, i32* [[OUT]], i32 16
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i32* [[TMP8]] to <2 x i32>*
+; CHECK-NEXT:    store <2 x i32> [[SPLIT4]], <2 x i32>* [[TMP9]], align 4
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.matrix.columnwise.store.v10i32(<10 x i32> %in, i32* %out, i32 4, i32 2, i32 5)
+  ret void
+}
+
+declare void @llvm.matrix.columnwise.store.v10i32(<10 x i32>, i32*, i32, i32, i32)
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/transpose-double.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/transpose-double.ll
new file mode 100644
index 000000000..6bea523bb
--- /dev/null
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/transpose-double.ll
@@ -0,0 +1,114 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -lower-matrix-intrinsics -S < %s | FileCheck  %s
+; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s
+
+
+define <8 x double> @transpose(<8 x double> %a) {
+; CHECK-LABEL: @transpose(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <8 x double> [[A:%.*]], <8 x double> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <8 x double> [[A]], <8 x double> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[SPLIT2:%.*]] = shufflevector <8 x double> [[A]], <8 x double> undef, <2 x i32> <i32 4, i32 5>
+; CHECK-NEXT:    [[SPLIT3:%.*]] = shufflevector <8 x double> [[A]], <8 x double> undef, <2 x i32> <i32 6, i32 7>
+; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <2 x double> [[SPLIT]], i64 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x double> undef, double [[TMP0]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x double> [[SPLIT1]], i64 0
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x double> [[TMP1]], double [[TMP2]], i64 1
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x double> [[SPLIT2]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <4 x double> [[TMP3]], double [[TMP4]], i64 2
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x double> [[SPLIT3]], i64 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <4 x double> [[TMP5]], double [[TMP6]], i64 3
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x double> [[SPLIT]], i64 1
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <4 x double> undef, double [[TMP8]], i64 0
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <2 x double> [[SPLIT1]], i64 1
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <4 x double> [[TMP9]], double [[TMP10]], i64 1
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <2 x double> [[SPLIT2]], i64 1
+; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <4 x double> [[TMP11]], double [[TMP12]], i64 2
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x double> [[SPLIT3]], i64 1
+; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <4 x double> [[TMP13]], double [[TMP14]], i64 3
+; CHECK-NEXT:    [[TMP16:%.*]] = shufflevector <4 x double> [[TMP7]], <4 x double> [[TMP15]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    ret <8 x double> [[TMP16]]
+;
+entry:
+  %c  = call <8 x double> @llvm.matrix.transpose(<8 x double> %a, i32 2, i32 4)
+  ret <8 x double> %c
+}
+
+declare <8 x double> @llvm.matrix.transpose(<8 x double>, i32, i32)
+
+define <8 x double> @transpose_single_column(<8 x double> %a) {
+; CHECK-LABEL: @transpose_single_column(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <8 x double> [[A:%.*]], <8 x double> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <8 x double> [[SPLIT]], i64 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <1 x double> undef, double [[TMP0]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <8 x double> [[SPLIT]], i64 1
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <1 x double> undef, double [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <8 x double> [[SPLIT]], i64 2
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <1 x double> undef, double [[TMP4]], i64 0
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <8 x double> [[SPLIT]], i64 3
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <1 x double> undef, double [[TMP6]], i64 0
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <8 x double> [[SPLIT]], i64 4
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <1 x double> undef, double [[TMP8]], i64 0
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <8 x double> [[SPLIT]], i64 5
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <1 x double> undef, double [[TMP10]], i64 0
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <8 x double> [[SPLIT]], i64 6
+; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <1 x double> undef, double [[TMP12]], i64 0
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <8 x double> [[SPLIT]], i64 7
+; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <1 x double> undef, double [[TMP14]], i64 0
+; CHECK-NEXT:    [[TMP16:%.*]] = shufflevector <1 x double> [[TMP1]], <1 x double> [[TMP3]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP17:%.*]] = shufflevector <1 x double> [[TMP5]], <1 x double> [[TMP7]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP18:%.*]] = shufflevector <1 x double> [[TMP9]], <1 x double> [[TMP11]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP19:%.*]] = shufflevector <1 x double> [[TMP13]], <1 x double> [[TMP15]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP20:%.*]] = shufflevector <2 x double> [[TMP16]], <2 x double> [[TMP17]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP21:%.*]] = shufflevector <2 x double> [[TMP18]], <2 x double> [[TMP19]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP22:%.*]] = shufflevector <4 x double> [[TMP20]], <4 x double> [[TMP21]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    ret <8 x double> [[TMP22]]
+;
+entry:
+  %c  = call <8 x double> @llvm.matrix.transpose(<8 x double> %a, i32 8, i32 1)
+  ret <8 x double> %c
+}
+
+declare <12 x double> @llvm.matrix.transpose.v12f64(<12 x double>, i32, i32)
+
+define <12 x double> @transpose_double_3x4(<12 x double> %a) {
+; CHECK-LABEL: @transpose_double_3x4(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <12 x double> [[A:%.*]], <12 x double> undef, <3 x i32> <i32 0, i32 1, i32 2>
+; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <12 x double> [[A]], <12 x double> undef, <3 x i32> <i32 3, i32 4, i32 5>
+; CHECK-NEXT:    [[SPLIT2:%.*]] = shufflevector <12 x double> [[A]], <12 x double> undef, <3 x i32> <i32 6, i32 7, i32 8>
+; CHECK-NEXT:    [[SPLIT3:%.*]] = shufflevector <12 x double> [[A]], <12 x double> undef, <3 x i32> <i32 9, i32 10, i32 11>
+; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <3 x double> [[SPLIT]], i64 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x double> undef, double [[TMP0]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <3 x double> [[SPLIT1]], i64 0
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x double> [[TMP1]], double [[TMP2]], i64 1
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <3 x double> [[SPLIT2]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <4 x double> [[TMP3]], double [[TMP4]], i64 2
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <3 x double> [[SPLIT3]], i64 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <4 x double> [[TMP5]], double [[TMP6]], i64 3
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <3 x double> [[SPLIT]], i64 1
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <4 x double> undef, double [[TMP8]], i64 0
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <3 x double> [[SPLIT1]], i64 1
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <4 x double> [[TMP9]], double [[TMP10]], i64 1
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <3 x double> [[SPLIT2]], i64 1
+; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <4 x double> [[TMP11]], double [[TMP12]], i64 2
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <3 x double> [[SPLIT3]], i64 1
+; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <4 x double> [[TMP13]], double [[TMP14]], i64 3
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <3 x double> [[SPLIT]], i64 2
+; CHECK-NEXT:    [[TMP17:%.*]] = insertelement <4 x double> undef, double [[TMP16]], i64 0
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <3 x double> [[SPLIT1]], i64 2
+; CHECK-NEXT:    [[TMP19:%.*]] = insertelement <4 x double> [[TMP17]], double [[TMP18]], i64 1
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <3 x double> [[SPLIT2]], i64 2
+; CHECK-NEXT:    [[TMP21:%.*]] = insertelement <4 x double> [[TMP19]], double [[TMP20]], i64 2
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <3 x double> [[SPLIT3]], i64 2
+; CHECK-NEXT:    [[TMP23:%.*]] = insertelement <4 x double> [[TMP21]], double [[TMP22]], i64 3
+; CHECK-NEXT:    [[TMP24:%.*]] = shufflevector <4 x double> [[TMP7]], <4 x double> [[TMP15]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP25:%.*]] = shufflevector <4 x double> [[TMP23]], <4 x double> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP26:%.*]] = shufflevector <8 x double> [[TMP24]], <8 x double> [[TMP25]], <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEXT:    ret <12 x double> [[TMP26]]
+;
+entry:
+  %c  = call <12 x double> @llvm.matrix.transpose.v12f64(<12 x double> %a, i32 3, i32 4)
+  ret <12 x double> %c
+}
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/transpose-float.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/transpose-float.ll
new file mode 100644
index 000000000..f54f5746b
--- /dev/null
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/transpose-float.ll
@@ -0,0 +1,114 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -lower-matrix-intrinsics -S < %s | FileCheck  %s
+; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s
+
+
+define <8 x float> @transpose(<8 x float> %a) {
+; CHECK-LABEL: @transpose(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <8 x float> [[A:%.*]], <8 x float> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <8 x float> [[A]], <8 x float> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[SPLIT2:%.*]] = shufflevector <8 x float> [[A]], <8 x float> undef, <2 x i32> <i32 4, i32 5>
+; CHECK-NEXT:    [[SPLIT3:%.*]] = shufflevector <8 x float> [[A]], <8 x float> undef, <2 x i32> <i32 6, i32 7>
+; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <2 x float> [[SPLIT]], i64 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> undef, float [[TMP0]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x float> [[SPLIT1]], i64 0
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x float> [[TMP1]], float [[TMP2]], i64 1
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x float> [[SPLIT2]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <4 x float> [[TMP3]], float [[TMP4]], i64 2
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x float> [[SPLIT3]], i64 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <4 x float> [[TMP5]], float [[TMP6]], i64 3
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x float> [[SPLIT]], i64 1
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <4 x float> undef, float [[TMP8]], i64 0
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <2 x float> [[SPLIT1]], i64 1
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <4 x float> [[TMP9]], float [[TMP10]], i64 1
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <2 x float> [[SPLIT2]], i64 1
+; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <4 x float> [[TMP11]], float [[TMP12]], i64 2
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x float> [[SPLIT3]], i64 1
+; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <4 x float> [[TMP13]], float [[TMP14]], i64 3
+; CHECK-NEXT:    [[TMP16:%.*]] = shufflevector <4 x float> [[TMP7]], <4 x float> [[TMP15]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    ret <8 x float> [[TMP16]]
+;
+entry:
+  %c  = call <8 x float> @llvm.matrix.transpose(<8 x float> %a, i32 2, i32 4)
+  ret <8 x float> %c
+}
+
+declare <8 x float> @llvm.matrix.transpose(<8 x float>, i32, i32)
+
+define <8 x float> @transpose_single_column(<8 x float> %a) {
+; CHECK-LABEL: @transpose_single_column(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <8 x float> [[A:%.*]], <8 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <8 x float> [[SPLIT]], i64 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <1 x float> undef, float [[TMP0]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <8 x float> [[SPLIT]], i64 1
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <1 x float> undef, float [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <8 x float> [[SPLIT]], i64 2
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <1 x float> undef, float [[TMP4]], i64 0
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <8 x float> [[SPLIT]], i64 3
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <1 x float> undef, float [[TMP6]], i64 0
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <8 x float> [[SPLIT]], i64 4
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <1 x float> undef, float [[TMP8]], i64 0
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <8 x float> [[SPLIT]], i64 5
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <1 x float> undef, float [[TMP10]], i64 0
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <8 x float> [[SPLIT]], i64 6
+; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <1 x float> undef, float [[TMP12]], i64 0
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <8 x float> [[SPLIT]], i64 7
+; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <1 x float> undef, float [[TMP14]], i64 0
+; CHECK-NEXT:    [[TMP16:%.*]] = shufflevector <1 x float> [[TMP1]], <1 x float> [[TMP3]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP17:%.*]] = shufflevector <1 x float> [[TMP5]], <1 x float> [[TMP7]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP18:%.*]] = shufflevector <1 x float> [[TMP9]], <1 x float> [[TMP11]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP19:%.*]] = shufflevector <1 x float> [[TMP13]], <1 x float> [[TMP15]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP20:%.*]] = shufflevector <2 x float> [[TMP16]], <2 x float> [[TMP17]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP21:%.*]] = shufflevector <2 x float> [[TMP18]], <2 x float> [[TMP19]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP22:%.*]] = shufflevector <4 x float> [[TMP20]], <4 x float> [[TMP21]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    ret <8 x float> [[TMP22]]
+;
+entry:
+  %c  = call <8 x float> @llvm.matrix.transpose(<8 x float> %a, i32 8, i32 1)
+  ret <8 x float> %c
+}
+
+declare <12 x float> @llvm.matrix.transpose.v12f32(<12 x float>, i32, i32)
+
+define <12 x float> @transpose_float_3x4(<12 x float> %a) {
+; CHECK-LABEL: @transpose_float_3x4(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <12 x float> [[A:%.*]], <12 x float> undef, <3 x i32> <i32 0, i32 1, i32 2>
+; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <12 x float> [[A]], <12 x float> undef, <3 x i32> <i32 3, i32 4, i32 5>
+; CHECK-NEXT:    [[SPLIT2:%.*]] = shufflevector <12 x float> [[A]], <12 x float> undef, <3 x i32> <i32 6, i32 7, i32 8>
+; CHECK-NEXT:    [[SPLIT3:%.*]] = shufflevector <12 x float> [[A]], <12 x float> undef, <3 x i32> <i32 9, i32 10, i32 11>
+; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <3 x float> [[SPLIT]], i64 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> undef, float [[TMP0]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <3 x float> [[SPLIT1]], i64 0
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x float> [[TMP1]], float [[TMP2]], i64 1
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <3 x float> [[SPLIT2]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <4 x float> [[TMP3]], float [[TMP4]], i64 2
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <3 x float> [[SPLIT3]], i64 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <4 x float> [[TMP5]], float [[TMP6]], i64 3
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <3 x float> [[SPLIT]], i64 1
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <4 x float> undef, float [[TMP8]], i64 0
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <3 x float> [[SPLIT1]], i64 1
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <4 x float> [[TMP9]], float [[TMP10]], i64 1
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <3 x float> [[SPLIT2]], i64 1
+; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <4 x float> [[TMP11]], float [[TMP12]], i64 2
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <3 x float> [[SPLIT3]], i64 1
+; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <4 x float> [[TMP13]], float [[TMP14]], i64 3
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <3 x float> [[SPLIT]], i64 2
+; CHECK-NEXT:    [[TMP17:%.*]] = insertelement <4 x float> undef, float [[TMP16]], i64 0
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <3 x float> [[SPLIT1]], i64 2
+; CHECK-NEXT:    [[TMP19:%.*]] = insertelement <4 x float> [[TMP17]], float [[TMP18]], i64 1
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <3 x float> [[SPLIT2]], i64 2
+; CHECK-NEXT:    [[TMP21:%.*]] = insertelement <4 x float> [[TMP19]], float [[TMP20]], i64 2
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <3 x float> [[SPLIT3]], i64 2
+; CHECK-NEXT:    [[TMP23:%.*]] = insertelement <4 x float> [[TMP21]], float [[TMP22]], i64 3
+; CHECK-NEXT:    [[TMP24:%.*]] = shufflevector <4 x float> [[TMP7]], <4 x float> [[TMP15]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP25:%.*]] = shufflevector <4 x float> [[TMP23]], <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP26:%.*]] = shufflevector <8 x float> [[TMP24]], <8 x float> [[TMP25]], <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEXT:    ret <12 x float> [[TMP26]]
+;
+entry:
+  %c  = call <12 x float> @llvm.matrix.transpose.v12f32(<12 x float> %a, i32 3, i32 4)
+  ret <12 x float> %c
+}
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/transpose-i32.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/transpose-i32.ll
new file mode 100644
index 000000000..f5ea365d7
--- /dev/null
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/transpose-i32.ll
@@ -0,0 +1,114 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -lower-matrix-intrinsics -S < %s | FileCheck  %s
+; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s
+
+
+define <8 x i32> @transpose(<8 x i32> %a) {
+; CHECK-LABEL: @transpose(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <8 x i32> [[A:%.*]], <8 x i32> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[SPLIT2:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> undef, <2 x i32> <i32 4, i32 5>
+; CHECK-NEXT:    [[SPLIT3:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> undef, <2 x i32> <i32 6, i32 7>
+; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <2 x i32> [[SPLIT]], i64 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i32> undef, i32 [[TMP0]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x i32> [[SPLIT1]], i64 0
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x i32> [[TMP1]], i32 [[TMP2]], i64 1
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i32> [[SPLIT2]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <4 x i32> [[TMP3]], i32 [[TMP4]], i64 2
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i32> [[SPLIT3]], i64 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <4 x i32> [[TMP5]], i32 [[TMP6]], i64 3
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x i32> [[SPLIT]], i64 1
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <4 x i32> undef, i32 [[TMP8]], i64 0
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <2 x i32> [[SPLIT1]], i64 1
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <4 x i32> [[TMP9]], i32 [[TMP10]], i64 1
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <2 x i32> [[SPLIT2]], i64 1
+; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP12]], i64 2
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x i32> [[SPLIT3]], i64 1
+; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <4 x i32> [[TMP13]], i32 [[TMP14]], i64 3
+; CHECK-NEXT:    [[TMP16:%.*]] = shufflevector <4 x i32> [[TMP7]], <4 x i32> [[TMP15]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    ret <8 x i32> [[TMP16]]
+;
+entry:
+  %c  = call <8 x i32> @llvm.matrix.transpose(<8 x i32> %a, i32 2, i32 4)
+  ret <8 x i32> %c
+}
+
+declare <8 x i32> @llvm.matrix.transpose(<8 x i32>, i32, i32)
+
+define <8 x i32> @transpose_single_column(<8 x i32> %a) {
+; CHECK-LABEL: @transpose_single_column(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <8 x i32> [[A:%.*]], <8 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <8 x i32> [[SPLIT]], i64 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <1 x i32> undef, i32 [[TMP0]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <8 x i32> [[SPLIT]], i64 1
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <1 x i32> undef, i32 [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <8 x i32> [[SPLIT]], i64 2
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <1 x i32> undef, i32 [[TMP4]], i64 0
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <8 x i32> [[SPLIT]], i64 3
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <1 x i32> undef, i32 [[TMP6]], i64 0
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <8 x i32> [[SPLIT]], i64 4
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <1 x i32> undef, i32 [[TMP8]], i64 0
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <8 x i32> [[SPLIT]], i64 5
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <1 x i32> undef, i32 [[TMP10]], i64 0
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <8 x i32> [[SPLIT]], i64 6
+; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <1 x i32> undef, i32 [[TMP12]], i64 0
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <8 x i32> [[SPLIT]], i64 7
+; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <1 x i32> undef, i32 [[TMP14]], i64 0
+; CHECK-NEXT:    [[TMP16:%.*]] = shufflevector <1 x i32> [[TMP1]], <1 x i32> [[TMP3]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP17:%.*]] = shufflevector <1 x i32> [[TMP5]], <1 x i32> [[TMP7]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP18:%.*]] = shufflevector <1 x i32> [[TMP9]], <1 x i32> [[TMP11]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP19:%.*]] = shufflevector <1 x i32> [[TMP13]], <1 x i32> [[TMP15]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP20:%.*]] = shufflevector <2 x i32> [[TMP16]], <2 x i32> [[TMP17]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP21:%.*]] = shufflevector <2 x i32> [[TMP18]], <2 x i32> [[TMP19]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP22:%.*]] = shufflevector <4 x i32> [[TMP20]], <4 x i32> [[TMP21]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    ret <8 x i32> [[TMP22]]
+;
+entry:
+  %c  = call <8 x i32> @llvm.matrix.transpose(<8 x i32> %a, i32 8, i32 1)
+  ret <8 x i32> %c
+}
+
+declare <12 x i32> @llvm.matrix.transpose.v12i32(<12 x i32>, i32, i32)
+
+define <12 x i32> @transpose_i32_3x4(<12 x i32> %a) {
+; CHECK-LABEL: @transpose_i32_3x4(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <12 x i32> [[A:%.*]], <12 x i32> undef, <3 x i32> <i32 0, i32 1, i32 2>
+; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <12 x i32> [[A]], <12 x i32> undef, <3 x i32> <i32 3, i32 4, i32 5>
+; CHECK-NEXT:    [[SPLIT2:%.*]] = shufflevector <12 x i32> [[A]], <12 x i32> undef, <3 x i32> <i32 6, i32 7, i32 8>
+; CHECK-NEXT:    [[SPLIT3:%.*]] = shufflevector <12 x i32> [[A]], <12 x i32> undef, <3 x i32> <i32 9, i32 10, i32 11>
+; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <3 x i32> [[SPLIT]], i64 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i32> undef, i32 [[TMP0]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <3 x i32> [[SPLIT1]], i64 0
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x i32> [[TMP1]], i32 [[TMP2]], i64 1
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <3 x i32> [[SPLIT2]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <4 x i32> [[TMP3]], i32 [[TMP4]], i64 2
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <3 x i32> [[SPLIT3]], i64 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <4 x i32> [[TMP5]], i32 [[TMP6]], i64 3
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <3 x i32> [[SPLIT]], i64 1
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <4 x i32> undef, i32 [[TMP8]], i64 0
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <3 x i32> [[SPLIT1]], i64 1
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <4 x i32> [[TMP9]], i32 [[TMP10]], i64 1
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <3 x i32> [[SPLIT2]], i64 1
+; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP12]], i64 2
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <3 x i32> [[SPLIT3]], i64 1
+; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <4 x i32> [[TMP13]], i32 [[TMP14]], i64 3
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <3 x i32> [[SPLIT]], i64 2
+; CHECK-NEXT:    [[TMP17:%.*]] = insertelement <4 x i32> undef, i32 [[TMP16]], i64 0
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <3 x i32> [[SPLIT1]], i64 2
+; CHECK-NEXT:    [[TMP19:%.*]] = insertelement <4 x i32> [[TMP17]], i32 [[TMP18]], i64 1
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <3 x i32> [[SPLIT2]], i64 2
+; CHECK-NEXT:    [[TMP21:%.*]] = insertelement <4 x i32> [[TMP19]], i32 [[TMP20]], i64 2
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <3 x i32> [[SPLIT3]], i64 2
+; CHECK-NEXT:    [[TMP23:%.*]] = insertelement <4 x i32> [[TMP21]], i32 [[TMP22]], i64 3
+; CHECK-NEXT:    [[TMP24:%.*]] = shufflevector <4 x i32> [[TMP7]], <4 x i32> [[TMP15]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP25:%.*]] = shufflevector <4 x i32> [[TMP23]], <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP26:%.*]] = shufflevector <8 x i32> [[TMP24]], <8 x i32> [[TMP25]], <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEXT:    ret <12 x i32> [[TMP26]]
+;
+entry:
+  %c  = call <12 x i32> @llvm.matrix.transpose.v12i32(<12 x i32> %a, i32 3, i32 4)
+  ret <12 x i32> %c
+}
diff --git a/llvm/test/Transforms/LowerTypeTests/align.ll b/llvm/test/Transforms/LowerTypeTests/align.ll
new file mode 100644
index 000000000..ab06c46d9
--- /dev/null
+++ b/llvm/test/Transforms/LowerTypeTests/align.ll
@@ -0,0 +1,16 @@
+; RUN: opt -S -lowertypetests < %s | FileCheck %s
+
+target datalayout = "e-p:32:32"
+
+; CHECK: private constant { i32, [4 x i8], i32 } { i32 1, [4 x i8] zeroinitializer, i32 2 }, align 8
+@a = constant i32 1, !type !0
+@b = constant i32 2, align 8, !type !0
+
+!0 = !{i32 0, !"typeid1"}
+
+declare i1 @llvm.type.test(i8* %ptr, metadata %bitset) nounwind readnone
+
+define i1 @foo(i8* %p) {
+  %x = call i1 @llvm.type.test(i8* %p, metadata !"typeid1")
+  ret i1 %x
+}
diff --git a/llvm/test/Transforms/LowerTypeTests/export-rename-local.ll b/llvm/test/Transforms/LowerTypeTests/export-rename-local.ll
new file mode 100644
index 000000000..9e7c54ee9
--- /dev/null
+++ b/llvm/test/Transforms/LowerTypeTests/export-rename-local.ll
@@ -0,0 +1,15 @@
+; RUN: opt -S %s -lowertypetests -lowertypetests-summary-action=export -lowertypetests-read-summary=%S/Inputs/exported-funcs.yaml | FileCheck %s
+
+; CHECK: define internal void @external_addrtaken.1()
+; CHECK: declare {{.*}} void @external_addrtaken.cfi()
+
+target triple = "x86_64-unknown-linux"
+
+define internal void @external_addrtaken() !type !1 {
+  ret void
+}
+
+!cfi.functions = !{!0}
+
+!0 = !{!"external_addrtaken", i8 0, !1}
+!1 = !{i64 0, !"typeid1"}
diff --git a/llvm/test/Transforms/LowerTypeTests/import-icall.ll b/llvm/test/Transforms/LowerTypeTests/import-icall.ll
index 0b151db00..67788a2be 100644
--- a/llvm/test/Transforms/LowerTypeTests/import-icall.ll
+++ b/llvm/test/Transforms/LowerTypeTests/import-icall.ll
@@ -3,6 +3,11 @@
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
+@llvm.used = appending global [1 x i8*] [i8* bitcast (i8* ()* @local_decl to i8*)], section "llvm.metadata"
+@llvm.compiler.used = appending global [1 x i8*] [i8* bitcast (i8* ()* @local_decl to i8*)], section "llvm.metadata"
+
+@local_decl_alias = alias i8* (), i8* ()* @local_decl
+
 define i8 @local_a() {
   call void @external()
   call void @external_weak()
@@ -19,14 +24,16 @@ define i8 @use_b() {
   ret i8 %x
 }
 
-define void @local_decl() {
-  call void @local_decl()
-  ret void
+define i8* @local_decl() {
+  call i8* @local_decl()
+  ret i8* bitcast (i8* ()* @local_decl to i8*)
 }
 
 declare void @external()
 declare extern_weak void @external_weak()
 
+; CHECK: @local_decl_alias = alias i8* (), i8* ()* @local_decl
+
 ; CHECK:      define hidden i8 @local_a.cfi() {
 ; CHECK-NEXT:   call void @external()
 ; CHECK-NEXT:   call void @external_weak()
@@ -37,8 +44,9 @@ declare extern_weak void @external_weak()
 ; CHECK:      define internal i8 @local_b() {
 ; CHECK-NEXT:   call i8 @local_a()
 
-; CHECK:      define void @local_decl()
-; CHECK-NEXT:   call void @local_decl()
+; CHECK:      define i8* @local_decl()
+; CHECK-NEXT:   call i8* @local_decl()
+; CHECK-NEXT:   ret i8* bitcast (i8* ()* @local_decl.cfi_jt to i8*)
 
 ; CHECK: declare void @external()
 ; CHECK: declare extern_weak void @external_weak()
diff --git a/llvm/test/Transforms/MemCpyOpt/aggregate-type-crash.ll b/llvm/test/Transforms/MemCpyOpt/aggregate-type-crash.ll
new file mode 100644
index 000000000..16d107730
--- /dev/null
+++ b/llvm/test/Transforms/MemCpyOpt/aggregate-type-crash.ll
@@ -0,0 +1,30 @@
+; RUN: opt -memcpyopt -S -o - < %s | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.14.0"
+
+%my_struct = type { i8, i32 }
+
+; Function Attrs: inaccessiblemem_or_argmemonly
+declare noalias i8* @my_malloc(%my_struct*) #0
+
+define void @my_func(%my_struct* %0) {
+entry:
+; CHECK: entry:
+  %1 = load %my_struct, %my_struct* %0
+  %2 = call i8* @my_malloc(%my_struct* %0)
+  %3 = bitcast i8* %2 to %my_struct*
+  store %my_struct %1, %my_struct* %3
+; CHECK-NOT: call void @llvm.memcpy.{{.*}}.{{.*}}.{{.*}}
+  ret void
+}
+
+attributes #0 = { inaccessiblemem_or_argmemonly }
+
+!llvm.module.flags = !{!0, !1, !2}
+!llvm.ident = !{!3}
+
+!0 = !{i32 2, !"SDK Version", [2 x i32] [i32 10, i32 14]}
+!1 = !{i32 1, !"wchar_size", i32 4}
+!2 = !{i32 7, !"PIC Level", i32 2}
+!3 = !{!"Apple LLVM version 10.0.1 (clang-1001.0.46.4)"}
diff --git a/llvm/test/Transforms/MemCpyOpt/lifetime.ll b/llvm/test/Transforms/MemCpyOpt/lifetime.ll
index 9ddf3f4f9..ad14bdd6d 100644
--- a/llvm/test/Transforms/MemCpyOpt/lifetime.ll
+++ b/llvm/test/Transforms/MemCpyOpt/lifetime.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -O1 -S | FileCheck %s
+; RUN: opt < %s -O2 -S | FileCheck %s
 
 ; performCallSlotOptzn in MemCpy should not exchange the calls to
 ; @llvm.lifetime.start and @llvm.memcpy.
diff --git a/llvm/test/Transforms/MemCpyOpt/memcpy-to-memset-with-lifetimes.ll b/llvm/test/Transforms/MemCpyOpt/memcpy-to-memset-with-lifetimes.ll
index 73d567a00..ee2f6adb4 100644
--- a/llvm/test/Transforms/MemCpyOpt/memcpy-to-memset-with-lifetimes.ll
+++ b/llvm/test/Transforms/MemCpyOpt/memcpy-to-memset-with-lifetimes.ll
@@ -1,9 +1,16 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -basicaa -memcpyopt -instcombine -S < %s | FileCheck %s
 
 target datalayout = "e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define void @foo([8 x i64]* noalias nocapture sret dereferenceable(64) %sret) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry-block:
+; CHECK-NEXT:    [[SRET1:%.*]] = bitcast [8 x i64]* [[SRET:%.*]] to i8*
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* nonnull align 8 dereferenceable(64) [[SRET1]], i8 0, i64 64, i1 false)
+; CHECK-NEXT:    ret void
+;
 entry-block:
   %a = alloca [8 x i64], align 8
   %a.cast = bitcast [8 x i64]* %a to i8*
@@ -14,14 +21,23 @@ entry-block:
   call void @llvm.lifetime.end.p0i8(i64 64, i8* %a.cast)
   ret void
 
-; CHECK-LABEL: @foo(
-; CHECK:         %[[sret_cast:[^=]+]] = bitcast [8 x i64]* %sret to i8*
-; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* nonnull align 8 %[[sret_cast]], i8 0, i64 64
-; CHECK-NOT: call void @llvm.memcpy
-; CHECK: ret void
 }
 
 define void @bar([8 x i64]* noalias nocapture sret dereferenceable(64) %sret, [8 x i64]* noalias nocapture dereferenceable(64) %out) {
+; CHECK-LABEL: @bar(
+; CHECK-NEXT:  entry-block:
+; CHECK-NEXT:    [[A:%.*]] = alloca [8 x i64], align 8
+; CHECK-NEXT:    [[A_CAST:%.*]] = bitcast [8 x i64]* [[A]] to i8*
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0i8(i64 64, i8* nonnull [[A_CAST]])
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* nonnull align 8 dereferenceable(64) [[A_CAST]], i8 0, i64 64, i1 false)
+; CHECK-NEXT:    [[SRET_CAST:%.*]] = bitcast [8 x i64]* [[SRET:%.*]] to i8*
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* nonnull align 8 dereferenceable(64) [[SRET_CAST]], i8 0, i64 64, i1 false)
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* nonnull align 8 dereferenceable(32) [[A_CAST]], i8 42, i64 32, i1 false)
+; CHECK-NEXT:    [[OUT_CAST:%.*]] = bitcast [8 x i64]* [[OUT:%.*]] to i8*
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 8 dereferenceable(64) [[OUT_CAST]], i8* nonnull align 8 dereferenceable(64) [[A_CAST]], i64 64, i1 false)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0i8(i64 64, i8* nonnull [[A_CAST]])
+; CHECK-NEXT:    ret void
+;
 entry-block:
   %a = alloca [8 x i64], align 8
   %a.cast = bitcast [8 x i64]* %a to i8*
@@ -35,17 +51,6 @@ entry-block:
   call void @llvm.lifetime.end.p0i8(i64 64, i8* %a.cast)
   ret void
 
-; CHECK-LABEL: @bar(
-; CHECK:         %[[a:[^=]+]] = alloca [8 x i64]
-; CHECK:         %[[a_cast:[^=]+]] = bitcast [8 x i64]* %[[a]] to i8*
-; CHECK:         call void @llvm.memset.p0i8.i64(i8* nonnull align 8 %[[a_cast]], i8 0, i64 64
-; CHECK:         %[[sret_cast:[^=]+]] = bitcast [8 x i64]* %sret to i8*
-; CHECK:         call void @llvm.memset.p0i8.i64(i8* nonnull align 8 %[[sret_cast]], i8 0, i64 64
-; CHECK:         call void @llvm.memset.p0i8.i64(i8* nonnull align 8 %[[a_cast]], i8 42, i64 32
-; CHECK:         %[[out_cast:[^=]+]] = bitcast [8 x i64]* %out to i8*
-; CHECK:         call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 8 %[[out_cast]], i8* nonnull align 8 %[[a_cast]], i64 64
-; CHECK-NOT: call void @llvm.memcpy
-; CHECK: ret void
 }
 
 declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture) nounwind
diff --git a/llvm/test/Transforms/MemCpyOpt/memcpy.ll b/llvm/test/Transforms/MemCpyOpt/memcpy.ll
index 4c5f6cbeb..23b137608 100644
--- a/llvm/test/Transforms/MemCpyOpt/memcpy.ll
+++ b/llvm/test/Transforms/MemCpyOpt/memcpy.ll
@@ -248,6 +248,6 @@ declare void @f1(%struct.big* nocapture sret)
 declare void @f2(%struct.big*)
 
 ; CHECK: attributes [[$NUW]] = { nounwind }
-; CHECK: attributes #1 = { argmemonly nounwind }
+; CHECK: attributes #1 = { argmemonly nounwind willreturn }
 ; CHECK: attributes #2 = { nounwind ssp }
 ; CHECK: attributes #3 = { nounwind ssp uwtable }
diff --git a/llvm/test/Transforms/MemCpyOpt/store-to-memset.ll b/llvm/test/Transforms/MemCpyOpt/store-to-memset.ll
new file mode 100644
index 000000000..59ed892b6
--- /dev/null
+++ b/llvm/test/Transforms/MemCpyOpt/store-to-memset.ll
@@ -0,0 +1,77 @@
+; RUN: opt < %s -memcpyopt -S | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-grtev4-linux-gnu"
+
+define i8* @foo(i8* returned %0, i32 %1, i64 %2) {
+entry:
+  %3 = getelementptr inbounds i8, i8* %0, i64 %2
+  %4 = getelementptr inbounds i8, i8* %3, i64 -32
+  %vv = trunc i32 %1 to i8
+  store i8 %vv, i8* %4, align 1
+  %5 = getelementptr inbounds i8, i8* %4, i64 1
+  store i8 %vv, i8* %5, align 1
+  %6= getelementptr inbounds i8, i8* %4, i64 2
+  store i8 %vv, i8* %6, align 1
+  %7= getelementptr inbounds i8, i8* %4, i64 3
+  store i8 %vv, i8* %7, align 1
+  %8= getelementptr inbounds i8, i8* %4, i64 4
+  store i8 %vv, i8* %8, align 1
+  %9= getelementptr inbounds i8, i8* %4, i64 5
+  store i8 %vv, i8* %9, align 1
+  %10= getelementptr inbounds i8, i8* %4, i64 6
+  store i8 %vv, i8* %10, align 1
+  %11= getelementptr inbounds i8, i8* %4, i64 7
+  store i8 %vv, i8* %11, align 1
+  %12= getelementptr inbounds i8, i8* %4, i64 8
+  store i8 %vv, i8* %12, align 1
+  %13= getelementptr inbounds i8, i8* %4, i64 9
+  store i8 %vv, i8* %13, align 1
+  %14= getelementptr inbounds i8, i8* %4, i64 10
+  store i8 %vv, i8* %14, align 1
+  %15= getelementptr inbounds i8, i8* %4, i64 11
+  store i8 %vv, i8* %15, align 1
+  %16= getelementptr inbounds i8, i8* %4, i64 12
+  store i8 %vv, i8* %16, align 1
+  %17= getelementptr inbounds i8, i8* %4, i64 13
+  store i8 %vv, i8* %17, align 1
+  %18= getelementptr inbounds i8, i8* %4, i64 14
+  store i8 %vv, i8* %18, align 1
+  %19= getelementptr inbounds i8, i8* %4, i64 15
+  store i8 %vv, i8* %19, align 1
+  %20= getelementptr inbounds i8, i8* %4, i64 16
+  store i8 %vv, i8* %20, align 1
+  %21= getelementptr inbounds i8, i8* %20, i64 1
+  store i8 %vv, i8* %21, align 1
+  %22= getelementptr inbounds i8, i8* %20, i64 2
+  store i8 %vv, i8* %22, align 1
+  %23= getelementptr inbounds i8, i8* %20, i64 3
+  store i8 %vv, i8* %23, align 1
+  %24= getelementptr inbounds i8, i8* %20, i64 4
+  store i8 %vv, i8* %24, align 1
+  %25= getelementptr inbounds i8, i8* %20, i64 5
+  store i8 %vv, i8* %25, align 1
+  %26= getelementptr inbounds i8, i8* %20, i64 6
+  store i8 %vv, i8* %26, align 1
+  %27= getelementptr inbounds i8, i8* %20, i64 7
+  store i8 %vv, i8* %27, align 1
+  %28= getelementptr inbounds i8, i8* %20, i64 8
+  store i8 %vv, i8* %28, align 1
+  %29= getelementptr inbounds i8, i8* %20, i64 9
+  store i8 %vv, i8* %29, align 1
+  %30= getelementptr inbounds i8, i8* %20, i64 10
+  store i8 %vv, i8* %30, align 1
+  %31 = getelementptr inbounds i8, i8* %20, i64 11
+  store i8 %vv, i8* %31, align 1
+  %32 = getelementptr inbounds i8, i8* %20, i64 12
+  store i8 %vv, i8* %32, align 1
+  %33 = getelementptr inbounds i8, i8* %20, i64 13
+  store i8 %vv, i8* %33, align 1
+  %34 = getelementptr inbounds i8, i8* %20, i64 14
+  store i8 %vv, i8* %34, align 1
+  %35 = getelementptr inbounds i8, i8* %20, i64 15
+  store i8 %vv, i8* %35, align 1
+  ret i8* %0
+; CHECK-LABEL: @foo
+; CHECK: call void @llvm.memset.p0i8.i64(i8* align 1 %4, i8 %vv, i64 32, i1 false)
+}
+
diff --git a/llvm/test/Transforms/MergeFunc/byval-attr-congruent-type.ll b/llvm/test/Transforms/MergeFunc/byval-attr-congruent-type.ll
new file mode 100644
index 000000000..7e7d772b9
--- /dev/null
+++ b/llvm/test/Transforms/MergeFunc/byval-attr-congruent-type.ll
@@ -0,0 +1,30 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -mergefunc %s | FileCheck %s
+
+%struct.c = type { i32 }
+%struct.a = type { i32 }
+
+@d = external dso_local global %struct.c
+
+define void @e(%struct.a* byval(%struct.a) %f) {
+; CHECK-LABEL: @e(
+; CHECK-NEXT:    ret void
+;
+  ret void
+}
+
+define void @g(%struct.c* byval(%struct.c) %f) {
+; CHECK-LABEL: @g(
+; CHECK-NEXT:    ret void
+;
+  ret void
+}
+
+define void @h() {
+; CHECK-LABEL: @h(
+; CHECK-NEXT:    call void bitcast (void (%struct.a*)* @e to void (%struct.c*)*)(%struct.c* byval(%struct.c) @d)
+; CHECK-NEXT:    ret void
+;
+  call void @g(%struct.c* byval(%struct.c) @d)
+  ret void
+}
diff --git a/llvm/test/Transforms/MergeFunc/comdat.ll b/llvm/test/Transforms/MergeFunc/comdat.ll
index ce289598f..6ef7c5439 100644
--- a/llvm/test/Transforms/MergeFunc/comdat.ll
+++ b/llvm/test/Transforms/MergeFunc/comdat.ll
@@ -20,5 +20,5 @@ define linkonce_odr hidden i32 @g(i32 %x, i32 %y) comdat {
 }
 
 ; CHECK-DAG: define linkonce_odr hidden i32 @f(i32 %x, i32 %y) comdat
-; CHECK-DAG: define linkonce_odr hidden i32 @g(i32, i32) comdat
+; CHECK-DAG: define linkonce_odr hidden i32 @g(i32 %0, i32 %1) comdat
 
diff --git a/llvm/test/Transforms/MergeFunc/merge-block-address.ll b/llvm/test/Transforms/MergeFunc/merge-block-address.ll
index 4ce13e5da..d2340a77c 100644
--- a/llvm/test/Transforms/MergeFunc/merge-block-address.ll
+++ b/llvm/test/Transforms/MergeFunc/merge-block-address.ll
@@ -1,4 +1,5 @@
 ; RUN: opt -S -mergefunc < %s | FileCheck %s
+; RUN: opt -S -passes=mergefunc < %s | FileCheck %s
 
 ; These two functions are identical. The basic block labels are the same, and
 ; induce the same CFG. We are testing that block addresses within different
diff --git a/llvm/test/Transforms/MergeFunc/phi-check-blocks.ll b/llvm/test/Transforms/MergeFunc/phi-check-blocks.ll
index b2de9a0c0..76f4491c2 100644
--- a/llvm/test/Transforms/MergeFunc/phi-check-blocks.ll
+++ b/llvm/test/Transforms/MergeFunc/phi-check-blocks.ll
@@ -3,8 +3,8 @@
 ; Ensure that we do not merge functions that are identical with the
 ; exception of the order of the incoming blocks to a phi.
 
-; CHECK-LABEL: define linkonce_odr hidden i1 @first(i2)
-define linkonce_odr hidden i1 @first(i2) {
+; CHECK-LABEL: define linkonce_odr hidden i1 @first(i2 %0)
+define linkonce_odr hidden i1 @first(i2 %0) {
 entry:
 ; CHECK: switch i2
   switch i2 %0, label %default [
@@ -26,8 +26,8 @@ done:
   ret i1 %result
 }
 
-; CHECK-LABEL: define linkonce_odr hidden i1 @second(i2)
-define linkonce_odr hidden i1 @second(i2) {
+; CHECK-LABEL: define linkonce_odr hidden i1 @second(i2 %0)
+define linkonce_odr hidden i1 @second(i2 %0) {
 entry:
 ; CHECK: switch i2
   switch i2 %0, label %default [
diff --git a/llvm/test/Transforms/MergeFunc/weak-small.ll b/llvm/test/Transforms/MergeFunc/weak-small.ll
index 64f108317..44ac7daee 100644
--- a/llvm/test/Transforms/MergeFunc/weak-small.ll
+++ b/llvm/test/Transforms/MergeFunc/weak-small.ll
@@ -2,15 +2,15 @@
 
 ; Weak functions too small for merging to be profitable
 
-; CHECK: define weak i32 @foo(i8*, i32)
+; CHECK: define weak i32 @foo(i8* %0, i32 %1)
 ; CHECK-NEXT: ret i32 %1
-; CHECK: define weak i32 @bar(i8*, i32)
+; CHECK: define weak i32 @bar(i8* %0, i32 %1)
 ; CHECK-NEXT: ret i32 %1
 
-define weak i32 @foo(i8*, i32) #0 {
+define weak i32 @foo(i8* %0, i32 %1) #0 {
     ret i32 %1
 }
 
-define weak i32 @bar(i8*, i32) #0 {
+define weak i32 @bar(i8* %0, i32 %1) #0 {
     ret i32 %1
 }
diff --git a/llvm/test/Transforms/MergeICmps/X86/int64-and-ptr.ll b/llvm/test/Transforms/MergeICmps/X86/int64-and-ptr.ll
index 2705659cc..ff525e96d 100644
--- a/llvm/test/Transforms/MergeICmps/X86/int64-and-ptr.ll
+++ b/llvm/test/Transforms/MergeICmps/X86/int64-and-ptr.ll
@@ -35,6 +35,6 @@ declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture) #1
 ; Function Attrs: argmemonly nounwind
 declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture) #1
 
-attributes #0 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { argmemonly nounwind }
 attributes #2 = { nounwind }
diff --git a/llvm/test/Transforms/MergeICmps/X86/pr41917.ll b/llvm/test/Transforms/MergeICmps/X86/pr41917.ll
index f889b4032..a697b4622 100644
--- a/llvm/test/Transforms/MergeICmps/X86/pr41917.ll
+++ b/llvm/test/Transforms/MergeICmps/X86/pr41917.ll
@@ -57,8 +57,8 @@ land.end6:                                        ; preds = %land.rhs3, %land.en
 
 declare dso_local zeroext i1 @f2() local_unnamed_addr #1
 
-attributes #0 = { nounwind optsize "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-features"="+cx8,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { optsize "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-features"="+cx8,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind optsize "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-features"="+cx8,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { optsize "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-features"="+cx8,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind optsize }
 
 !llvm.module.flags = !{!0, !1}
diff --git a/llvm/test/Transforms/NewGVN/basic-cyclic-opt.ll b/llvm/test/Transforms/NewGVN/basic-cyclic-opt.ll
index 7830d7ea7..a712a6954 100644
--- a/llvm/test/Transforms/NewGVN/basic-cyclic-opt.ll
+++ b/llvm/test/Transforms/NewGVN/basic-cyclic-opt.ll
@@ -308,7 +308,7 @@ bb3:                                              ; preds = %bb2
   %tmp3 = sub i32 %tmp, %phi2
   ret i32 %tmp3
 }
-attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.ident = !{!0, !0, !0}
 
diff --git a/llvm/test/Transforms/NewGVN/cond_br2.ll b/llvm/test/Transforms/NewGVN/cond_br2.ll
index ff7a76d14..cb537565a 100644
--- a/llvm/test/Transforms/NewGVN/cond_br2.ll
+++ b/llvm/test/Transforms/NewGVN/cond_br2.ll
@@ -128,10 +128,10 @@ declare void @_ZN4llvm15SmallVectorBase8grow_podEmm(%"class.llvm::SmallVectorBas
 ; Function Attrs: nounwind
 declare void @free(i8* nocapture) #3
 
-attributes #0 = { ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { ssp uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind }
-attributes #2 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #3 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !0 = !{!"any pointer", !1}
 !1 = !{!"omnipotent char", !2}
diff --git a/llvm/test/Transforms/NewGVN/equivalent-phi.ll b/llvm/test/Transforms/NewGVN/equivalent-phi.ll
index 2deeb760f..34d916b9c 100644
--- a/llvm/test/Transforms/NewGVN/equivalent-phi.ll
+++ b/llvm/test/Transforms/NewGVN/equivalent-phi.ll
@@ -59,7 +59,7 @@ bb20:                                             ; preds = %bb17
   ret i32 %tmp14
 }
 
-attributes #0 = { nounwind ssp uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+fxsr,+mmx,+sse,+sse2,+sse3,+sse4.1,+ssse3" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind ssp uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+fxsr,+mmx,+sse,+sse2,+sse3,+sse4.1,+ssse3" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.module.flags = !{!0}
 !llvm.ident = !{!1}
diff --git a/llvm/test/Transforms/NewGVN/memory-handling.ll b/llvm/test/Transforms/NewGVN/memory-handling.ll
index dfb52d617..79197a3db 100644
--- a/llvm/test/Transforms/NewGVN/memory-handling.ll
+++ b/llvm/test/Transforms/NewGVN/memory-handling.ll
@@ -167,10 +167,10 @@ declare void @llvm.memset.p0i8.i64(i8* nocapture writeonly, i8, i64, i1) #2
 ; Function Attrs: inlinehint nounwind readonly uwtable
 declare i32 @tolower(i32) local_unnamed_addr #3
 
-attributes #0 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { argmemonly nounwind }
-attributes #3 = { inlinehint nounwind readonly uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { inlinehint nounwind readonly uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #4 = { nounwind readnone }
 attributes #5 = { nounwind readonly }
 
diff --git a/llvm/test/Transforms/NewGVN/phi-edge-handling.ll b/llvm/test/Transforms/NewGVN/phi-edge-handling.ll
index 890958c97..4ac78a557 100644
--- a/llvm/test/Transforms/NewGVN/phi-edge-handling.ll
+++ b/llvm/test/Transforms/NewGVN/phi-edge-handling.ll
@@ -51,7 +51,7 @@ bb7:                                              ; preds = %bb
   unreachable
 }
 
-attributes #0 = { ssp uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+fxsr,+mmx,+sse,+sse2,+sse3,+sse4.1,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { ssp uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+fxsr,+mmx,+sse,+sse2,+sse3,+sse4.1,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.ident = !{!0}
 
diff --git a/llvm/test/Transforms/NewGVN/pr31483.ll b/llvm/test/Transforms/NewGVN/pr31483.ll
index 94b485a99..640d2a617 100644
--- a/llvm/test/Transforms/NewGVN/pr31483.ll
+++ b/llvm/test/Transforms/NewGVN/pr31483.ll
@@ -100,7 +100,7 @@ declare signext i32 @zot(i8*, ...) #1
 ; Function Attrs: nounwind
 declare void @llvm.va_end(i8*) #2
 
-attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64" "target-features"="+altivec,-bpermd,-crypto,-direct-move,-extdiv,-power8-vector,-qpx,-vsx" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64" "target-features"="+altivec,-bpermd,-crypto,-direct-move,-extdiv,-power8-vector,-qpx,-vsx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64" "target-features"="+altivec,-bpermd,-crypto,-direct-move,-extdiv,-power8-vector,-qpx,-vsx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64" "target-features"="+altivec,-bpermd,-crypto,-direct-move,-extdiv,-power8-vector,-qpx,-vsx" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind }
 
diff --git a/llvm/test/Transforms/NewGVN/pr31501.ll b/llvm/test/Transforms/NewGVN/pr31501.ll
index bc95969ea..fad10110c 100644
--- a/llvm/test/Transforms/NewGVN/pr31501.ll
+++ b/llvm/test/Transforms/NewGVN/pr31501.ll
@@ -117,7 +117,7 @@ bb21:                                             ; preds = %bb19, %bb
   ret %struct.foo* %tmp22
 }
 
-attributes #0 = { norecurse nounwind ssp uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+fxsr,+mmx,+sse,+sse2,+sse3,+sse4.1,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { norecurse nounwind ssp uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+fxsr,+mmx,+sse,+sse2,+sse3,+sse4.1,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.module.flags = !{!0}
 !llvm.ident = !{!1}
diff --git a/llvm/test/Transforms/NewGVN/pr33187.ll b/llvm/test/Transforms/NewGVN/pr33187.ll
index 61e767d36..1410edc22 100644
--- a/llvm/test/Transforms/NewGVN/pr33187.ll
+++ b/llvm/test/Transforms/NewGVN/pr33187.ll
@@ -108,7 +108,7 @@ bb1:                                              ; preds = %bb1, %bb
   br label %bb1
 }
 
-attributes #0 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 
 source_filename = "pr33187-c.ll"
diff --git a/llvm/test/Transforms/NewGVN/pr33305.ll b/llvm/test/Transforms/NewGVN/pr33305.ll
index 9cbc10c09..17dd597dd 100644
--- a/llvm/test/Transforms/NewGVN/pr33305.ll
+++ b/llvm/test/Transforms/NewGVN/pr33305.ll
@@ -166,8 +166,8 @@ declare void @abort() local_unnamed_addr #1
 ; Function Attrs: nounwind
 declare i32 @puts(i8* nocapture readonly) local_unnamed_addr #2
 
-attributes #0 = { nounwind optsize ssp uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+fxsr,+mmx,+sse,+sse2,+sse3,+sse4.1,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { noreturn nounwind optsize "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+fxsr,+mmx,+sse,+sse2,+sse3,+sse4.1,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind optsize ssp uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+fxsr,+mmx,+sse,+sse2,+sse3,+sse4.1,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { noreturn nounwind optsize "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+fxsr,+mmx,+sse,+sse2,+sse3,+sse4.1,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind }
 attributes #3 = { noreturn nounwind optsize }
 
diff --git a/llvm/test/Transforms/NewGVN/pr34430.ll b/llvm/test/Transforms/NewGVN/pr34430.ll
index 0b59dfc72..b6e70a16b 100644
--- a/llvm/test/Transforms/NewGVN/pr34430.ll
+++ b/llvm/test/Transforms/NewGVN/pr34430.ll
@@ -41,7 +41,7 @@ bb6:                                              ; preds = %bb4, %bb2, %bb1, %b
   br label %bb4
 }
 
-attributes #0 = { norecurse noreturn nounwind ssp uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+fxsr,+mmx,+sse,+sse2,+sse3,+sse4.1,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { norecurse noreturn nounwind ssp uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+fxsr,+mmx,+sse,+sse2,+sse3,+sse4.1,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.ident = !{!0}
 
diff --git a/llvm/test/Transforms/NewGVN/pr34452.ll b/llvm/test/Transforms/NewGVN/pr34452.ll
index 4e5b371a4..495fb3560 100644
--- a/llvm/test/Transforms/NewGVN/pr34452.ll
+++ b/llvm/test/Transforms/NewGVN/pr34452.ll
@@ -37,7 +37,7 @@ while.body.us:                                    ; preds = %while.body.us, %ent
 ; Function Attrs: nounwind readnone speculatable
 declare { i64, i1 } @llvm.sadd.with.overflow.i64(i64, i64) #1
 
-attributes #0 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "polly-optimized" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "polly-optimized" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone speculatable }
 
 !llvm.ident = !{!0}
diff --git a/llvm/test/Transforms/NewGVN/pr43441.ll b/llvm/test/Transforms/NewGVN/pr43441.ll
new file mode 100644
index 000000000..5b4fc3937
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/pr43441.ll
@@ -0,0 +1,42 @@
+; RUN: opt -newgvn -S < %s | FileCheck %s
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; CHECK-LABEL: @print_long_format()
+define dso_local void @print_long_format() #0 {
+entry:
+  switch i32 undef, label %sw.default [
+    i32 1, label %sw.bb
+    i32 0, label %sw.bb19
+    i32 2, label %sw.bb23
+  ]
+
+sw.bb:                                            ; preds = %entry
+  unreachable
+
+sw.bb19:                                          ; preds = %entry
+  br i1 undef, label %if.then37, label %if.end50
+
+sw.bb23:                                          ; preds = %entry
+  unreachable
+
+sw.default:                                       ; preds = %entry
+  unreachable
+
+if.then37:                                        ; preds = %sw.bb19
+  unreachable
+
+if.end50:                                         ; preds = %sw.bb19
+  %call180 = call i32 @timespec_cmp() #2
+  %cmp181 = icmp slt i32 %call180, 0
+  ret void
+}
+
+; Function Attrs: writeonly
+declare dso_local i32 @timespec_cmp() #1
+
+attributes #0 = { "use-soft-float"="false" }
+attributes #1 = { writeonly }
+attributes #2 = { nounwind readonly }
+
diff --git a/llvm/test/Transforms/ObjCARC/basic.ll b/llvm/test/Transforms/ObjCARC/basic.ll
index 6524dad4a..5c4f1d143 100644
--- a/llvm/test/Transforms/ObjCARC/basic.ll
+++ b/llvm/test/Transforms/ObjCARC/basic.ll
@@ -3070,5 +3070,5 @@ define void @test67(i8* %x) {
 !5 = !{i32 2, !"Debug Info Version", i32 3}
 
 ; CHECK: attributes [[NUW]] = { nounwind }
-; CHECK: attributes #1 = { nounwind readnone speculatable }
+; CHECK: attributes #1 = { nounwind readnone speculatable willreturn }
 ; CHECK: ![[RELEASE]] = !{}
diff --git a/llvm/test/Transforms/ObjCARC/code-motion.ll b/llvm/test/Transforms/ObjCARC/code-motion.ll
new file mode 100644
index 000000000..7c3242ce8
--- /dev/null
+++ b/llvm/test/Transforms/ObjCARC/code-motion.ll
@@ -0,0 +1,39 @@
+; RUN: opt -objc-arc -S < %s | FileCheck %s
+
+declare void @alterRefCount()
+declare void @use(i8*)
+
+; Check that ARC optimizer doesn't reverse the order of the retain call and the
+; release call when there are debug instructions.
+
+; CHECK: call i8* @llvm.objc.retain(i8* %x)
+; CHECK: call void @llvm.objc.release(i8* %x)
+
+define i32 @test(i8* %x, i8* %y, i8 %z, i32 %i) {
+  %i.addr = alloca i32, align 4
+  store i32 %i, i32* %i.addr, align 4
+  %v1 = tail call i8* @llvm.objc.retain(i8* %x)
+  store i8 %z, i8* %x
+  call void @llvm.dbg.declare(metadata i32* %i.addr, metadata !9, metadata !DIExpression()), !dbg !10
+  call void @alterRefCount()
+  tail call void @llvm.objc.release(i8* %x)
+  ret i32 %i
+}
+
+declare void @llvm.dbg.declare(metadata, metadata, metadata)
+declare i8* @llvm.objc.retain(i8*) local_unnamed_addr
+declare void @llvm.objc.release(i8*) local_unnamed_addr
+
+!llvm.module.flags = !{!0, !1}
+
+!0 = !{i32 2, !"Dwarf Version", i32 4}
+!1 = !{i32 2, !"Debug Info Version", i32 3}
+!2 = !DILocalVariable(name: "i", arg: 1, scope: !3, file: !4, line: 1, type: !7)
+!3 = distinct !DISubprogram(name: "test", scope: !4, file: !4, line: 1, type: !5, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !8, retainedNodes: !9)
+!4 = !DIFile(filename: "test.m", directory: "dir")
+!5 = !DISubroutineType(types: !6)
+!6 = !{!7, !7}
+!7 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!8 = distinct !DICompileUnit(language: DW_LANG_ObjC, file: !4, isOptimized: false, runtimeVersion: 2, emissionKind: FullDebug, enums: !9, nameTableKind: None)
+!9 = !{}
+!10 = !DILocation(line: 1, column: 14, scope: !3)
diff --git a/llvm/test/Transforms/ObjCARC/ensure-that-exception-unwind-path-is-visited.ll b/llvm/test/Transforms/ObjCARC/ensure-that-exception-unwind-path-is-visited.ll
index 589cb7b94..eec0f035a 100644
--- a/llvm/test/Transforms/ObjCARC/ensure-that-exception-unwind-path-is-visited.ll
+++ b/llvm/test/Transforms/ObjCARC/ensure-that-exception-unwind-path-is-visited.ll
@@ -105,7 +105,7 @@ declare void @NSLog(i8*, ...)
 declare void @llvm.dbg.value(metadata, metadata, metadata) nounwind readnone
 
 ; CHECK: attributes #0 = { ssp uwtable }
-; CHECK: attributes #1 = { nounwind readnone speculatable }
+; CHECK: attributes #1 = { nounwind readnone speculatable willreturn }
 ; CHECK: attributes #2 = { nonlazybind }
 ; CHECK: attributes [[NUW]] = { nounwind }
 ; CHECK: attributes #4 = { noinline ssp uwtable }
diff --git a/llvm/test/Transforms/ObjCARC/inlined-autorelease-return-value.ll b/llvm/test/Transforms/ObjCARC/inlined-autorelease-return-value.ll
new file mode 100644
index 000000000..84d33193e
--- /dev/null
+++ b/llvm/test/Transforms/ObjCARC/inlined-autorelease-return-value.ll
@@ -0,0 +1,292 @@
+; RUN: opt -basicaa -objc-arc -S < %s | FileCheck %s
+
+target datalayout = "e-p:64:64:64"
+
+declare i8* @llvm.objc.retain(i8*)
+declare i8* @llvm.objc.autoreleaseReturnValue(i8*)
+declare i8* @llvm.objc.retainAutoreleasedReturnValue(i8*)
+declare i8* @llvm.objc.unsafeClaimAutoreleasedReturnValue(i8*)
+declare void @opaque()
+declare void @llvm.lifetime.start(i64, i8* nocapture)
+declare void @llvm.lifetime.end(i64, i8* nocapture)
+
+; CHECK-LABEL: define i8* @elide_with_retainRV(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret i8* %x
+define i8* @elide_with_retainRV(i8* %x) nounwind {
+entry:
+  %b = call i8* @llvm.objc.autoreleaseReturnValue(i8* %x) nounwind
+  %c = call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %b) nounwind
+  ret i8* %c
+}
+
+; CHECK-LABEL: define i8* @elide_with_retainRV_bitcast(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %c = bitcast i32* %x to i8*
+; CHECK-NEXT:    ret i8* %c
+define i8* @elide_with_retainRV_bitcast(i32* %x) nounwind {
+entry:
+  %a = bitcast i32* %x to i8*
+  %b = call i8* @llvm.objc.autoreleaseReturnValue(i8* %a) nounwind
+  %c = bitcast i32* %x to i8*
+  %d = call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %c) nounwind
+  ret i8* %d
+}
+
+; CHECK-LABEL: define i8* @elide_with_retainRV_phi(
+; CHECK-NOT:   define
+; CHECK:       phis:
+; CHECK-NEXT:    phi i8*
+; CHECK-NEXT:    ret i8*
+define i8* @elide_with_retainRV_phi(i8* %x) nounwind {
+entry:
+  br label %phis
+
+phis:
+  %a = phi i8* [ %x, %entry ]
+  %c = phi i8* [ %x, %entry ]
+  %b = call i8* @llvm.objc.autoreleaseReturnValue(i8* %a) nounwind
+  %d = call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %c) nounwind
+  ret i8* %d
+}
+
+; CHECK-LABEL: define i8* @elide_with_retainRV_splitByRetain(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %b = call i8* @llvm.objc.autorelease(i8* %x)
+; CHECK-NEXT:    tail call i8* @llvm.objc.retain(i8* %x)
+; CHECK-NEXT:    tail call i8* @llvm.objc.retain(i8* %b)
+define i8* @elide_with_retainRV_splitByRetain(i8* %x) nounwind {
+entry:
+  ; Cleanup is blocked by other ARC intrinsics for ease of implementation; we
+  ; only delay processing AutoreleaseRV until the very next ARC intrinsic.  In
+  ; practice, it would be very strange for this to matter.
+  %b = call i8* @llvm.objc.autoreleaseReturnValue(i8* %x) nounwind
+  %c = call i8* @llvm.objc.retain(i8* %x) nounwind
+  %d = call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %b) nounwind
+  ret i8* %d
+}
+
+; CHECK-LABEL: define i8* @elide_with_retainRV_splitByOpaque(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %b = call i8* @llvm.objc.autorelease(i8* %x)
+; CHECK-NEXT:    call void @opaque()
+; CHECK-NEXT:    %d = tail call i8* @llvm.objc.retain(i8* %b)
+; CHECK-NEXT:    ret i8* %d
+define i8* @elide_with_retainRV_splitByOpaque(i8* %x) nounwind {
+entry:
+  ; Cleanup should get blocked by opaque calls.
+  %b = call i8* @llvm.objc.autoreleaseReturnValue(i8* %x) nounwind
+  call void @opaque() nounwind
+  %d = call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %b) nounwind
+  ret i8* %d
+}
+
+; CHECK-LABEL: define i8* @elide_with_retainRV_splitByLifetime(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0i8(i64 8, i8* %x)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0i8(i64 8, i8* %x)
+; CHECK-NEXT:    ret i8* %x
+define i8* @elide_with_retainRV_splitByLifetime(i8* %x) nounwind {
+entry:
+  ; Cleanup should skip over lifetime intrinsics.
+  call void @llvm.lifetime.start(i64 8, i8* %x)
+  %b = call i8* @llvm.objc.autoreleaseReturnValue(i8* %x) nounwind
+  call void @llvm.lifetime.end(i64 8, i8* %x)
+  %d = call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %b) nounwind
+  ret i8* %d
+}
+
+; CHECK-LABEL: define i8* @elide_with_retainRV_wrongArg(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    call void @llvm.objc.release(i8* %x)
+; CHECK-NEXT:    tail call i8* @llvm.objc.retain(i8* %y)
+define i8* @elide_with_retainRV_wrongArg(i8* %x, i8* %y) nounwind {
+entry:
+  %b = call i8* @llvm.objc.autoreleaseReturnValue(i8* %x) nounwind
+  %c = call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %y) nounwind
+  ret i8* %c
+}
+
+; CHECK-LABEL: define i8* @elide_with_retainRV_wrongBB(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    call i8* @llvm.objc.autorelease(i8* %x)
+; CHECK-NEXT:    br label %next
+; CHECK:       next:
+; CHECK-NEXT:    tail call i8* @llvm.objc.retain(
+; CHECK-NEXT:    ret i8*
+define i8* @elide_with_retainRV_wrongBB(i8* %x) nounwind {
+entry:
+  %b = call i8* @llvm.objc.autoreleaseReturnValue(i8* %x) nounwind
+  br label %next
+
+next:
+  %c = call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %b) nounwind
+  ret i8* %c
+}
+
+; CHECK-LABEL: define i8* @elide_with_retainRV_beforeAutoreleaseRV(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    tail call i8* @llvm.objc.autoreleaseReturnValue(i8* %x)
+; CHECK-NEXT:    ret i8* %x
+define i8* @elide_with_retainRV_beforeAutoreleaseRV(i8* %x) nounwind {
+entry:
+  %b = call i8* @llvm.objc.autoreleaseReturnValue(i8* %x) nounwind
+  %c = call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %b) nounwind
+  %d = call i8* @llvm.objc.autoreleaseReturnValue(i8* %c) nounwind
+  ret i8* %c
+}
+
+; CHECK-LABEL: define i8* @elide_with_retainRV_afterRetain(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    tail call i8* @llvm.objc.retain(i8* %x)
+; CHECK-NEXT:    ret i8* %a
+define i8* @elide_with_retainRV_afterRetain(i8* %x) nounwind {
+entry:
+  %a = call i8* @llvm.objc.retain(i8* %x) nounwind
+  %b = call i8* @llvm.objc.autoreleaseReturnValue(i8* %a) nounwind
+  %c = call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %b) nounwind
+  ret i8* %c
+}
+
+; CHECK-LABEL: define i8* @elide_with_claimRV(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    tail call void @llvm.objc.release(i8* %x)
+; CHECK-NEXT:    ret i8* %x
+define i8* @elide_with_claimRV(i8* %x) nounwind {
+entry:
+  %b = call i8* @llvm.objc.autoreleaseReturnValue(i8* %x) nounwind
+  %c = call i8* @llvm.objc.unsafeClaimAutoreleasedReturnValue(i8* %b) nounwind
+  ret i8* %c
+}
+
+; CHECK-LABEL: define i8* @elide_with_claimRV_bitcast(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %c = bitcast i32* %x to i8*
+; CHECK-NEXT:    tail call void @llvm.objc.release(i8* %c)
+; CHECK-NEXT:    ret i8* %c
+define i8* @elide_with_claimRV_bitcast(i32* %x) nounwind {
+entry:
+  %a = bitcast i32* %x to i8*
+  %b = call i8* @llvm.objc.autoreleaseReturnValue(i8* %a) nounwind
+  %c = bitcast i32* %x to i8*
+  %d = call i8* @llvm.objc.unsafeClaimAutoreleasedReturnValue(i8* %c) nounwind
+  ret i8* %d
+}
+
+; CHECK-LABEL: define i8* @elide_with_claimRV_phi(
+; CHECK-NOT:   define
+; CHECK:       phis:
+; CHECK-NEXT:    %c = phi i8*
+; CHECK-NEXT:    tail call void @llvm.objc.release(i8* %c)
+; CHECK-NEXT:    ret i8* %c
+define i8* @elide_with_claimRV_phi(i8* %x) nounwind {
+entry:
+  br label %phis
+
+phis:
+  %a = phi i8* [ %x, %entry ]
+  %c = phi i8* [ %x, %entry ]
+  %b = call i8* @llvm.objc.autoreleaseReturnValue(i8* %a) nounwind
+  %d = call i8* @llvm.objc.unsafeClaimAutoreleasedReturnValue(i8* %c) nounwind
+  ret i8* %d
+}
+
+; CHECK-LABEL: define i8* @elide_with_claimRV_splitByRetain(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %b = call i8* @llvm.objc.autorelease(i8* %x)
+; CHECK-NEXT:    tail call i8* @llvm.objc.retain(i8* %x)
+; CHECK-NEXT:    tail call i8* @llvm.objc.unsafeClaimAutoreleasedReturnValue(i8* %b)
+define i8* @elide_with_claimRV_splitByRetain(i8* %x) nounwind {
+entry:
+  ; Cleanup is blocked by other ARC intrinsics for ease of implementation; we
+  ; only delay processing AutoreleaseRV until the very next ARC intrinsic.  In
+  ; practice, it would be very strange for this to matter.
+  %b = call i8* @llvm.objc.autoreleaseReturnValue(i8* %x) nounwind
+  %c = call i8* @llvm.objc.retain(i8* %x) nounwind
+  %d = call i8* @llvm.objc.unsafeClaimAutoreleasedReturnValue(i8* %b) nounwind
+  ret i8* %d
+}
+
+; CHECK-LABEL: define i8* @elide_with_claimRV_splitByOpaque(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %b = call i8* @llvm.objc.autorelease(i8* %x)
+; CHECK-NEXT:    call void @opaque()
+; CHECK-NEXT:    %d = tail call i8* @llvm.objc.unsafeClaimAutoreleasedReturnValue(i8* %b)
+; CHECK-NEXT:    ret i8* %d
+define i8* @elide_with_claimRV_splitByOpaque(i8* %x) nounwind {
+entry:
+  ; Cleanup should get blocked by opaque calls.
+  %b = call i8* @llvm.objc.autoreleaseReturnValue(i8* %x) nounwind
+  call void @opaque() nounwind
+  %d = call i8* @llvm.objc.unsafeClaimAutoreleasedReturnValue(i8* %b) nounwind
+  ret i8* %d
+}
+
+; CHECK-LABEL: define i8* @elide_with_claimRV_splitByLifetime(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0i8(i64 8, i8* %x)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0i8(i64 8, i8* %x)
+; CHECK-NEXT:    tail call void @llvm.objc.release(i8* %x)
+; CHECK-NEXT:    ret i8* %x
+define i8* @elide_with_claimRV_splitByLifetime(i8* %x) nounwind {
+entry:
+  ; Cleanup should skip over lifetime intrinsics.
+  call void @llvm.lifetime.start(i64 8, i8* %x)
+  %b = call i8* @llvm.objc.autoreleaseReturnValue(i8* %x) nounwind
+  call void @llvm.lifetime.end(i64 8, i8* %x)
+  %d = call i8* @llvm.objc.unsafeClaimAutoreleasedReturnValue(i8* %b) nounwind
+  ret i8* %d
+}
+
+; CHECK-LABEL: define i8* @elide_with_claimRV_wrongArg(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    call void @llvm.objc.release(i8* %x)
+; CHECK-NEXT:    tail call i8* @llvm.objc.unsafeClaimAutoreleasedReturnValue(i8* %y)
+define i8* @elide_with_claimRV_wrongArg(i8* %x, i8* %y) nounwind {
+entry:
+  %b = call i8* @llvm.objc.autoreleaseReturnValue(i8* %x) nounwind
+  %c = call i8* @llvm.objc.unsafeClaimAutoreleasedReturnValue(i8* %y) nounwind
+  ret i8* %c
+}
+
+; CHECK-LABEL: define i8* @elide_with_claimRV_wrongBB(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    call i8* @llvm.objc.autorelease(i8* %x)
+; CHECK-NEXT:    br label %next
+; CHECK:       next:
+; CHECK-NEXT:    tail call i8* @llvm.objc.unsafeClaimAutoreleasedReturnValue(
+; CHECK-NEXT:    ret i8*
+define i8* @elide_with_claimRV_wrongBB(i8* %x) nounwind {
+entry:
+  %b = call i8* @llvm.objc.autoreleaseReturnValue(i8* %x) nounwind
+  br label %next
+
+next:
+  %c = call i8* @llvm.objc.unsafeClaimAutoreleasedReturnValue(i8* %b) nounwind
+  ret i8* %c
+}
+
+
+; CHECK-LABEL: define i8* @elide_with_claimRV_beforeAutoreleaseRV(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    tail call void @llvm.objc.release(i8* %x)
+; CHECK-NEXT:    tail call i8* @llvm.objc.autoreleaseReturnValue(i8* %x)
+; CHECK-NEXT:    ret i8* %x
+define i8* @elide_with_claimRV_beforeAutoreleaseRV(i8* %x) nounwind {
+entry:
+  %b = call i8* @llvm.objc.autoreleaseReturnValue(i8* %x) nounwind
+  %c = call i8* @llvm.objc.unsafeClaimAutoreleasedReturnValue(i8* %b) nounwind
+  %d = call i8* @llvm.objc.autoreleaseReturnValue(i8* %c) nounwind
+  ret i8* %c
+}
+
+; CHECK-LABEL: define i8* @elide_with_claimRV_afterRetain(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret i8* %x
+define i8* @elide_with_claimRV_afterRetain(i8* %x) nounwind {
+entry:
+  %a = call i8* @llvm.objc.retain(i8* %x) nounwind
+  %b = call i8* @llvm.objc.autoreleaseReturnValue(i8* %a) nounwind
+  %c = call i8* @llvm.objc.unsafeClaimAutoreleasedReturnValue(i8* %b) nounwind
+  ret i8* %c
+}
diff --git a/llvm/test/Transforms/ObjCARC/nested.ll b/llvm/test/Transforms/ObjCARC/nested.ll
index 8b7e673e1..841104854 100644
--- a/llvm/test/Transforms/ObjCARC/nested.ll
+++ b/llvm/test/Transforms/ObjCARC/nested.ll
@@ -821,5 +821,5 @@ entry:
 
 
 ; CHECK: attributes [[NUW]] = { nounwind }
-; CHECK: attributes #1 = { argmemonly nounwind }
+; CHECK: attributes #1 = { argmemonly nounwind willreturn }
 ; CHECK: attributes #2 = { nonlazybind }
diff --git a/llvm/test/Transforms/ObjCARC/post-inlining.ll b/llvm/test/Transforms/ObjCARC/post-inlining.ll
index 0304d59c8..8e4e762c6 100644
--- a/llvm/test/Transforms/ObjCARC/post-inlining.ll
+++ b/llvm/test/Transforms/ObjCARC/post-inlining.ll
@@ -46,3 +46,67 @@ entry:
   call void @use_pointer(i8* %p)
   ret void
 }
+
+; Check that we can delete the autoreleaseRV+retainAutoreleasedRV pair even in
+; presence of instructions added by the inliner as part of the return sequence.
+
+; 1) Noop instructions: bitcasts and zero-indices GEPs.
+
+; CHECK-LABEL: define i8* @testNoop(
+; CHECK: entry:
+; CHECK-NEXT: %noop0 = bitcast i8* %call.i to i64*
+; CHECK-NEXT: %noop1 = getelementptr i8, i8* %call.i, i32 0
+; CHECK-NEXT: ret i8* %call.i
+; CHECK-NEXT: }
+define i8* @testNoop(i8* %call.i) {
+entry:
+  %0 = tail call i8* @llvm.objc.autoreleaseReturnValue(i8* %call.i) nounwind
+  %noop0 = bitcast i8* %call.i to i64*
+  %noop1 = getelementptr i8, i8* %call.i, i32 0
+  %1 = tail call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %call.i) nounwind
+  ret i8* %call.i
+}
+
+; 2) Lifetime markers.
+
+declare void @llvm.lifetime.start.p0i8(i64, i8*)
+declare void @llvm.lifetime.end.p0i8(i64, i8*)
+
+; CHECK-LABEL: define i8* @testLifetime(
+; CHECK: entry:
+; CHECK-NEXT: %obj = alloca i8
+; CHECK-NEXT: call void @llvm.lifetime.start.p0i8(i64 8, i8* %obj)
+; CHECK-NEXT: call void @llvm.lifetime.end.p0i8(i64 8, i8* %obj)
+; CHECK-NEXT: ret i8* %call.i
+; CHECK-NEXT: }
+define i8* @testLifetime(i8* %call.i) {
+entry:
+  %obj = alloca i8
+  call void @llvm.lifetime.start.p0i8(i64 8, i8* %obj)
+  %0 = tail call i8* @llvm.objc.autoreleaseReturnValue(i8* %call.i) nounwind
+  call void @llvm.lifetime.end.p0i8(i64 8, i8* %obj)
+  %1 = tail call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %call.i) nounwind
+  ret i8* %call.i
+}
+
+; 3) Dynamic alloca markers.
+
+declare i8* @llvm.stacksave()
+declare void @llvm.stackrestore(i8*)
+
+; CHECK-LABEL: define i8* @testStack(
+; CHECK: entry:
+; CHECK-NEXT: %save = tail call i8* @llvm.stacksave()
+; CHECK-NEXT: %obj = alloca i8, i8 %arg
+; CHECK-NEXT: call void @llvm.stackrestore(i8* %save)
+; CHECK-NEXT: ret i8* %call.i
+; CHECK-NEXT: }
+define i8* @testStack(i8* %call.i, i8 %arg) {
+entry:
+  %save = tail call i8* @llvm.stacksave()
+  %obj = alloca i8, i8 %arg
+  %0 = tail call i8* @llvm.objc.autoreleaseReturnValue(i8* %call.i) nounwind
+  call void @llvm.stackrestore(i8* %save)
+  %1 = tail call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %call.i) nounwind
+  ret i8* %call.i
+}
diff --git a/llvm/test/Transforms/ObjCARC/unsafe-claim-rv.ll b/llvm/test/Transforms/ObjCARC/unsafe-claim-rv.ll
index 8b6480256..a76a79295 100644
--- a/llvm/test/Transforms/ObjCARC/unsafe-claim-rv.ll
+++ b/llvm/test/Transforms/ObjCARC/unsafe-claim-rv.ll
@@ -40,8 +40,7 @@ if.end:                                           ; preds = %if.then, %entry
 
 ; CHECK: if.then
 ; CHECK: tail call i8* @llvm.objc.retain
-; CHECK-NEXT: call i8* @llvm.objc.autorelease
 ; CHECK: %Y.0 = phi
-; CHECK-NEXT: tail call i8* @llvm.objc.unsafeClaimAutoreleasedReturnValue(i8* %Y.0)
+; CHECK-NEXT: tail call void @llvm.objc.release
 ; CHECK-NEXT: tail call void @llvm.objc.release
 
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/thinlto_cspgo_bar_gen.ll b/llvm/test/Transforms/PGOProfile/Inputs/thinlto_cspgo_bar_gen.ll
index bec04c6a3..42c645db7 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/thinlto_cspgo_bar_gen.ll
+++ b/llvm/test/Transforms/PGOProfile/Inputs/thinlto_cspgo_bar_gen.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 $__llvm_profile_filename = comdat any
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/thinlto_cspgo_bar_use.ll b/llvm/test/Transforms/PGOProfile/Inputs/thinlto_cspgo_bar_use.ll
index 016172215..ed1765f00 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/thinlto_cspgo_bar_use.ll
+++ b/llvm/test/Transforms/PGOProfile/Inputs/thinlto_cspgo_bar_use.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @odd = common dso_local global i32 0, align 4
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/thinlto_indirect_call_promotion.ll b/llvm/test/Transforms/PGOProfile/Inputs/thinlto_indirect_call_promotion.ll
index 8732e5407..7412120bb 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/thinlto_indirect_call_promotion.ll
+++ b/llvm/test/Transforms/PGOProfile/Inputs/thinlto_indirect_call_promotion.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 source_filename = "thinlto_indirect_call_promotion.c"
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/thinlto_samplepgo_icp.ll b/llvm/test/Transforms/PGOProfile/Inputs/thinlto_samplepgo_icp.ll
index 22860f52b..709c950ce 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/thinlto_samplepgo_icp.ll
+++ b/llvm/test/Transforms/PGOProfile/Inputs/thinlto_samplepgo_icp.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @fptr = external local_unnamed_addr global void ()*, align 8
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/thinlto_samplepgo_icp2a.ll b/llvm/test/Transforms/PGOProfile/Inputs/thinlto_samplepgo_icp2a.ll
index 545b36c11..fc3cc935c 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/thinlto_samplepgo_icp2a.ll
+++ b/llvm/test/Transforms/PGOProfile/Inputs/thinlto_samplepgo_icp2a.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 ; Function Attrs: norecurse nounwind readnone uwtable
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/thinlto_samplepgo_icp2b.ll b/llvm/test/Transforms/PGOProfile/Inputs/thinlto_samplepgo_icp2b.ll
index f8de10c4d..67e091c7f 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/thinlto_samplepgo_icp2b.ll
+++ b/llvm/test/Transforms/PGOProfile/Inputs/thinlto_samplepgo_icp2b.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define void @_ZL3barv() #1 {
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/thinlto_samplepgo_icp3.ll b/llvm/test/Transforms/PGOProfile/Inputs/thinlto_samplepgo_icp3.ll
index 10d633be5..d040f1fce 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/thinlto_samplepgo_icp3.ll
+++ b/llvm/test/Transforms/PGOProfile/Inputs/thinlto_samplepgo_icp3.ll
@@ -1,4 +1,4 @@
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @fptr = external local_unnamed_addr global void ()*, align 8
diff --git a/llvm/test/Transforms/PGOProfile/X86/macho.ll b/llvm/test/Transforms/PGOProfile/X86/macho.ll
index d2fe65f2f..1dcfbbe44 100644
--- a/llvm/test/Transforms/PGOProfile/X86/macho.ll
+++ b/llvm/test/Transforms/PGOProfile/X86/macho.ll
@@ -1,6 +1,6 @@
 ; RUN: opt < %s -pgo-instr-gen -instrprof -S | llc | FileCheck %s --check-prefix=MACHO-DIRECTIVE
 
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.11.0"
 
 ; MACHO-DIRECTIVE: .weak_definition        ___llvm_profile_raw_version
diff --git a/llvm/test/Transforms/PGOProfile/branch1.ll b/llvm/test/Transforms/PGOProfile/branch1.ll
index bbc7b7229..e3eb99228 100644
--- a/llvm/test/Transforms/PGOProfile/branch1.ll
+++ b/llvm/test/Transforms/PGOProfile/branch1.ll
@@ -18,7 +18,7 @@
 ; RUN: opt < %s -pgo-instr-use -pgo-test-profile-file=%t.profdata -pass-remarks=pgo-instrumentation -pgo-emit-branch-prob -S 2>&1| FileCheck %s --check-prefix=ANALYSIS
 ; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -pass-remarks=pgo-instrumentation -pgo-emit-branch-prob -S 2>&1| FileCheck %s --check-prefix=ANALYSIS
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 ; GEN-DARWIN-LINKONCE: target triple = "x86_64-apple-darwin"
 
diff --git a/llvm/test/Transforms/PGOProfile/branch2.ll b/llvm/test/Transforms/PGOProfile/branch2.ll
index f8df54b94..4582ab7f7 100644
--- a/llvm/test/Transforms/PGOProfile/branch2.ll
+++ b/llvm/test/Transforms/PGOProfile/branch2.ll
@@ -3,7 +3,7 @@
 ; RUN: llvm-profdata merge %S/Inputs/branch2.proftext -o %t.profdata
 ; RUN: opt < %s -pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
 ; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 ; GEN: $__llvm_profile_raw_version = comdat any
diff --git a/llvm/test/Transforms/PGOProfile/chr.ll b/llvm/test/Transforms/PGOProfile/chr.ll
index 7d6a2ee17..021ec2533 100644
--- a/llvm/test/Transforms/PGOProfile/chr.ll
+++ b/llvm/test/Transforms/PGOProfile/chr.ll
@@ -468,15 +468,14 @@ define i32 @test_chr_5(i32* %i, i32 %sum0) !prof !14 {
 ; CHECK-NEXT:    [[SUM2_NONCHR:%.*]] = select i1 [[TMP11]], i32 [[SUM1_NONCHR]], i32 [[TMP12]], !prof !16
 ; CHECK-NEXT:    [[TMP13:%.*]] = and i32 [[TMP0]], 4
 ; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i32 [[TMP13]], 0
-; CHECK-NEXT:    br i1 [[TMP14]], label [[BB3]], label [[BB1_NONCHR:%.*]], !prof !16
-; CHECK:       bb1.nonchr:
 ; CHECK-NEXT:    [[TMP15:%.*]] = and i32 [[TMP0]], 8
 ; CHECK-NEXT:    [[TMP16:%.*]] = icmp eq i32 [[TMP15]], 0
-; CHECK-NEXT:    [[SUM4_NONCHR_V:%.*]] = select i1 [[TMP16]], i32 44, i32 88, !prof !16
+; CHECK-NEXT:    [[SUM4_NONCHR_V:%.*]] = select i1 [[TMP16]], i32 44, i32 88
 ; CHECK-NEXT:    [[SUM4_NONCHR:%.*]] = add i32 [[SUM2_NONCHR]], [[SUM4_NONCHR_V]]
+; CHECK-NEXT:    [[SUM5_NONCHR:%.*]] = select i1 [[TMP14]], i32 [[SUM2_NONCHR]], i32 [[SUM4_NONCHR]], !prof !16
 ; CHECK-NEXT:    br label [[BB3]]
 ; CHECK:       bb3:
-; CHECK-NEXT:    [[SUM6:%.*]] = phi i32 [ [[TMP4]], [[BB0]] ], [ [[SUM0]], [[ENTRY_SPLIT_NONCHR]] ], [ [[SUM2_NONCHR]], [[BB0_NONCHR]] ], [ [[SUM4_NONCHR]], [[BB1_NONCHR]] ]
+; CHECK-NEXT:    [[SUM6:%.*]] = phi i32 [ [[TMP4]], [[BB0]] ], [ [[SUM0]], [[ENTRY_SPLIT_NONCHR]] ], [ [[SUM5_NONCHR]], [[BB0_NONCHR]] ]
 ; CHECK-NEXT:    ret i32 [[SUM6]]
 ;
 entry:
@@ -569,15 +568,14 @@ define i32 @test_chr_5_1(i32* %i, i32 %sum0) !prof !14 {
 ; CHECK-NEXT:    [[SUM2_NONCHR:%.*]] = select i1 [[TMP14]], i32 [[SUM1_NONCHR]], i32 [[TMP15]], !prof !16
 ; CHECK-NEXT:    [[TMP16:%.*]] = and i32 [[SUM0]], 4
 ; CHECK-NEXT:    [[TMP17:%.*]] = icmp eq i32 [[TMP16]], 0
-; CHECK-NEXT:    br i1 [[TMP17]], label [[BB3]], label [[BB1_NONCHR:%.*]], !prof !16
-; CHECK:       bb1.nonchr:
 ; CHECK-NEXT:    [[TMP18:%.*]] = and i32 [[TMP0]], 8
 ; CHECK-NEXT:    [[TMP19:%.*]] = icmp eq i32 [[TMP18]], 0
-; CHECK-NEXT:    [[SUM4_NONCHR_V:%.*]] = select i1 [[TMP19]], i32 44, i32 88, !prof !16
+; CHECK-NEXT:    [[SUM4_NONCHR_V:%.*]] = select i1 [[TMP19]], i32 44, i32 88
 ; CHECK-NEXT:    [[SUM4_NONCHR:%.*]] = add i32 [[SUM2_NONCHR]], [[SUM4_NONCHR_V]]
+; CHECK-NEXT:    [[SUM5_NONCHR:%.*]] = select i1 [[TMP17]], i32 [[SUM2_NONCHR]], i32 [[SUM4_NONCHR]], !prof !16
 ; CHECK-NEXT:    br label [[BB3]]
 ; CHECK:       bb3:
-; CHECK-NEXT:    [[SUM6:%.*]] = phi i32 [ [[TMP7]], [[BB0]] ], [ [[SUM0]], [[ENTRY_SPLIT_NONCHR]] ], [ [[SUM2_NONCHR]], [[BB0_NONCHR]] ], [ [[SUM4_NONCHR]], [[BB1_NONCHR]] ]
+; CHECK-NEXT:    [[SUM6:%.*]] = phi i32 [ [[TMP7]], [[BB0]] ], [ [[SUM0]], [[ENTRY_SPLIT_NONCHR]] ], [ [[SUM5_NONCHR]], [[BB0_NONCHR]] ]
 ; CHECK-NEXT:    ret i32 [[SUM6]]
 ;
 entry:
@@ -667,15 +665,14 @@ define i32 @test_chr_6(i32* %i, i32* %j, i32 %sum0) !prof !14 {
 ; CHECK-NEXT:    [[SUM2_NONCHR:%.*]] = select i1 [[V4_NONCHR]], i32 [[SUM0]], i32 [[V8_NONCHR]], !prof !16
 ; CHECK-NEXT:    [[V9_NONCHR:%.*]] = and i32 [[J0]], 4
 ; CHECK-NEXT:    [[V10_NONCHR:%.*]] = icmp eq i32 [[V9_NONCHR]], 0
-; CHECK-NEXT:    br i1 [[V10_NONCHR]], label [[BB3]], label [[BB1_NONCHR:%.*]], !prof !16
-; CHECK:       bb1.nonchr:
 ; CHECK-NEXT:    [[V11_NONCHR:%.*]] = and i32 [[I0]], 8
 ; CHECK-NEXT:    [[V12_NONCHR:%.*]] = icmp eq i32 [[V11_NONCHR]], 0
-; CHECK-NEXT:    [[SUM4_NONCHR_V:%.*]] = select i1 [[V12_NONCHR]], i32 44, i32 88, !prof !16
+; CHECK-NEXT:    [[SUM4_NONCHR_V:%.*]] = select i1 [[V12_NONCHR]], i32 44, i32 88
 ; CHECK-NEXT:    [[SUM4_NONCHR:%.*]] = add i32 [[SUM2_NONCHR]], [[SUM4_NONCHR_V]]
+; CHECK-NEXT:    [[SUM5_NONCHR:%.*]] = select i1 [[V10_NONCHR]], i32 [[SUM2_NONCHR]], i32 [[SUM4_NONCHR]], !prof !16
 ; CHECK-NEXT:    br label [[BB3]]
 ; CHECK:       bb3:
-; CHECK-NEXT:    [[SUM6:%.*]] = phi i32 [ [[V13]], [[BB0]] ], [ [[SUM0]], [[ENTRY_SPLIT_NONCHR]] ], [ [[SUM2_NONCHR]], [[BB0_NONCHR]] ], [ [[SUM4_NONCHR]], [[BB1_NONCHR]] ]
+; CHECK-NEXT:    [[SUM6:%.*]] = phi i32 [ [[V13]], [[BB0]] ], [ [[SUM0]], [[ENTRY_SPLIT_NONCHR]] ], [ [[SUM5_NONCHR]], [[BB0_NONCHR]] ]
 ; CHECK-NEXT:    ret i32 [[SUM6]]
 ;
 entry:
@@ -1754,15 +1751,14 @@ define i32 @test_chr_19(i32* %i, i32 %sum0) !prof !14 {
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i32 [[TMP7]], 0
 ; CHECK-NEXT:    [[TMP9:%.*]] = add i32 [[SUM0]], 85
 ; CHECK-NEXT:    [[SUM2_NONCHR:%.*]] = select i1 [[TMP8]], i32 [[SUM0]], i32 [[TMP9]], !prof !16
-; CHECK-NEXT:    br i1 [[TMP8]], label [[BB3]], label [[BB1_NONCHR:%.*]], !prof !16
-; CHECK:       bb1.nonchr:
 ; CHECK-NEXT:    [[TMP10:%.*]] = and i32 [[TMP0]], 8
 ; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i32 [[TMP10]], 0
-; CHECK-NEXT:    [[SUM4_NONCHR_V:%.*]] = select i1 [[TMP11]], i32 44, i32 88, !prof !16
+; CHECK-NEXT:    [[SUM4_NONCHR_V:%.*]] = select i1 [[TMP11]], i32 44, i32 88
 ; CHECK-NEXT:    [[SUM4_NONCHR:%.*]] = add i32 [[SUM2_NONCHR]], [[SUM4_NONCHR_V]]
+; CHECK-NEXT:    [[SUM5_NONCHR:%.*]] = select i1 [[TMP8]], i32 [[SUM2_NONCHR]], i32 [[SUM4_NONCHR]], !prof !16
 ; CHECK-NEXT:    br label [[BB3]]
 ; CHECK:       bb3:
-; CHECK-NEXT:    [[SUM6:%.*]] = phi i32 [ [[TMP4]], [[BB0]] ], [ [[SUM0]], [[ENTRY_SPLIT_NONCHR]] ], [ [[SUM2_NONCHR]], [[BB0_NONCHR]] ], [ [[SUM4_NONCHR]], [[BB1_NONCHR]] ]
+; CHECK-NEXT:    [[SUM6:%.*]] = phi i32 [ [[TMP4]], [[BB0]] ], [ [[SUM0]], [[ENTRY_SPLIT_NONCHR]] ], [ [[SUM5_NONCHR]], [[BB0_NONCHR]] ]
 ; CHECK-NEXT:    ret i32 [[SUM6]]
 ;
 entry:
@@ -2316,6 +2312,182 @@ bb0:
   ret i64 99
 }
 
+; Test a case with a really long use-def chains. This test checks that it's not
+; really slow and doesn't appear to be hanging. This is different from
+; test_chr_22 in that it has nested control structures (multiple scopes) and
+; covers additional code.
+define i64 @test_chr_23(i64 %v0) !prof !14 {
+entry:
+  %v1 = add i64 %v0, 3
+  %v2 = add i64 %v1, %v1
+  %v3 = add i64 %v2, %v1
+  %v4 = add i64 %v2, %v3
+  %v5 = add i64 %v4, %v2
+  %v6 = add i64 %v5, %v4
+  %v7 = add i64 %v6, %v5
+  %v8 = add i64 %v7, %v6
+  %v9 = add i64 %v8, %v7
+  %v10 = icmp eq i64 %v9, 100
+  br i1 %v10, label %body, label %end, !prof !15
+
+body:
+  %v1_0 = add i64 %v9, 3
+  %v2_0 = add i64 %v1_0, %v1_0
+  %v3_0 = add i64 %v2_0, %v1_0
+  %v4_0 = add i64 %v2_0, %v3_0
+  %v5_0 = add i64 %v4_0, %v2_0
+  %v6_0 = add i64 %v5_0, %v4_0
+  %v7_0 = add i64 %v6_0, %v5_0
+  %v8_0 = add i64 %v7_0, %v6_0
+  %v9_0 = add i64 %v8_0, %v7_0
+  %v10_0 = icmp eq i64 %v9_0, 100
+  br i1 %v10_0, label %body.1, label %end, !prof !15
+
+body.1:
+  %v1_1 = add i64 %v9_0, 3
+  %v2_1 = add i64 %v1_1, %v1_1
+  %v3_1 = add i64 %v2_1, %v1_1
+  %v4_1 = add i64 %v2_1, %v3_1
+  %v5_1 = add i64 %v4_1, %v2_1
+  %v6_1 = add i64 %v5_1, %v4_1
+  %v7_1 = add i64 %v6_1, %v5_1
+  %v8_1 = add i64 %v7_1, %v6_1
+  %v9_1 = add i64 %v8_1, %v7_1
+  %v10_1 = icmp eq i64 %v9_1, 100
+  br i1 %v10_1, label %body.2, label %end, !prof !15
+
+body.2:
+  %v1_2 = add i64 %v9_1, 3
+  %v2_2 = add i64 %v1_2, %v1_2
+  %v3_2 = add i64 %v2_2, %v1_2
+  %v4_2 = add i64 %v2_2, %v3_2
+  %v5_2 = add i64 %v4_2, %v2_2
+  %v6_2 = add i64 %v5_2, %v4_2
+  %v7_2 = add i64 %v6_2, %v5_2
+  %v8_2 = add i64 %v7_2, %v6_2
+  %v9_2 = add i64 %v8_2, %v7_2
+  %v10_2 = icmp eq i64 %v9_2, 100
+  br i1 %v10_2, label %body.3, label %end, !prof !15
+
+body.3:
+  %v1_3 = add i64 %v9_2, 3
+  %v2_3 = add i64 %v1_3, %v1_3
+  %v3_3 = add i64 %v2_3, %v1_3
+  %v4_3 = add i64 %v2_3, %v3_3
+  %v5_3 = add i64 %v4_3, %v2_3
+  %v6_3 = add i64 %v5_3, %v4_3
+  %v7_3 = add i64 %v6_3, %v5_3
+  %v8_3 = add i64 %v7_3, %v6_3
+  %v9_3 = add i64 %v8_3, %v7_3
+  %v10_3 = icmp eq i64 %v9_3, 100
+  br i1 %v10_3, label %body.4, label %end, !prof !15
+
+body.4:
+  %v1_4 = add i64 %v9_3, 3
+  %v2_4 = add i64 %v1_4, %v1_4
+  %v3_4 = add i64 %v2_4, %v1_4
+  %v4_4 = add i64 %v2_4, %v3_4
+  %v5_4 = add i64 %v4_4, %v2_4
+  %v6_4 = add i64 %v5_4, %v4_4
+  %v7_4 = add i64 %v6_4, %v5_4
+  %v8_4 = add i64 %v7_4, %v6_4
+  %v9_4 = add i64 %v8_4, %v7_4
+  %v10_4 = icmp eq i64 %v9_4, 100
+  br i1 %v10_4, label %body.5, label %end, !prof !15
+
+body.5:
+  %v1_5 = add i64 %v9_4, 3
+  %v2_5 = add i64 %v1_5, %v1_5
+  %v3_5 = add i64 %v2_5, %v1_5
+  %v4_5 = add i64 %v2_5, %v3_5
+  %v5_5 = add i64 %v4_5, %v2_5
+  %v6_5 = add i64 %v5_5, %v4_5
+  %v7_5 = add i64 %v6_5, %v5_5
+  %v8_5 = add i64 %v7_5, %v6_5
+  %v9_5 = add i64 %v8_5, %v7_5
+  %v10_5 = icmp eq i64 %v9_5, 100
+  br i1 %v10_5, label %body.6, label %end, !prof !15
+
+body.6:
+  %v1_6 = add i64 %v9_5, 3
+  %v2_6 = add i64 %v1_6, %v1_6
+  %v3_6 = add i64 %v2_6, %v1_6
+  %v4_6 = add i64 %v2_6, %v3_6
+  %v5_6 = add i64 %v4_6, %v2_6
+  %v6_6 = add i64 %v5_6, %v4_6
+  %v7_6 = add i64 %v6_6, %v5_6
+  %v8_6 = add i64 %v7_6, %v6_6
+  %v9_6 = add i64 %v8_6, %v7_6
+  %v10_6 = icmp eq i64 %v9_6, 100
+  br i1 %v10_6, label %body.7, label %end, !prof !15
+
+body.7:
+  %v1_7 = add i64 %v9_6, 3
+  %v2_7 = add i64 %v1_7, %v1_7
+  %v3_7 = add i64 %v2_7, %v1_7
+  %v4_7 = add i64 %v2_7, %v3_7
+  %v5_7 = add i64 %v4_7, %v2_7
+  %v6_7 = add i64 %v5_7, %v4_7
+  %v7_7 = add i64 %v6_7, %v5_7
+  %v8_7 = add i64 %v7_7, %v6_7
+  %v9_7 = add i64 %v8_7, %v7_7
+  %v10_7 = icmp eq i64 %v9_7, 100
+  br i1 %v10_7, label %body.8, label %end, !prof !15
+
+body.8:
+  %v1_8 = add i64 %v9_7, 3
+  %v2_8 = add i64 %v1_8, %v1_8
+  %v3_8 = add i64 %v2_8, %v1_8
+  %v4_8 = add i64 %v2_8, %v3_8
+  %v5_8 = add i64 %v4_8, %v2_8
+  %v6_8 = add i64 %v5_8, %v4_8
+  %v7_8 = add i64 %v6_8, %v5_8
+  %v8_8 = add i64 %v7_8, %v6_8
+  %v9_8 = add i64 %v8_8, %v7_8
+  %v10_8 = icmp eq i64 %v9_8, 100
+  br i1 %v10_8, label %body.9, label %end, !prof !15
+
+body.9:
+  %v1_9 = add i64 %v9_8, 3
+  %v2_9 = add i64 %v1_9, %v1_9
+  %v3_9 = add i64 %v2_9, %v1_9
+  %v4_9 = add i64 %v2_9, %v3_9
+  %v5_9 = add i64 %v4_9, %v2_9
+  %v6_9 = add i64 %v5_9, %v4_9
+  %v7_9 = add i64 %v6_9, %v5_9
+  %v8_9 = add i64 %v7_9, %v6_9
+  %v9_9 = add i64 %v8_9, %v7_9
+  br label %end
+
+end:
+  ret i64 99
+}
+
+; Test to not crash upon a 0:0 branch_weight metadata.
+define void @test_chr_24(i32* %i) !prof !14 {
+entry:
+  %0 = load i32, i32* %i
+  %1 = and i32 %0, 1
+  %2 = icmp eq i32 %1, 0
+  br i1 %2, label %bb1, label %bb0, !prof !17
+
+bb0:
+  call void @foo()
+  br label %bb1
+
+bb1:
+  %3 = and i32 %0, 2
+  %4 = icmp eq i32 %3, 0
+  br i1 %4, label %bb3, label %bb2, !prof !17
+
+bb2:
+  call void @foo()
+  br label %bb3
+
+bb3:
+  ret void
+}
+
 !llvm.module.flags = !{!0}
 !0 = !{i32 1, !"ProfileSummary", !1}
 !1 = !{!2, !3, !4, !5, !6, !7, !8, !9}
@@ -2335,6 +2507,7 @@ bb0:
 !14 = !{!"function_entry_count", i64 100}
 !15 = !{!"branch_weights", i32 0, i32 1}
 !16 = !{!"branch_weights", i32 1, i32 1}
+!17 = !{!"branch_weights", i32 0, i32 0}
 ; CHECK: !15 = !{!"branch_weights", i32 1000, i32 0}
 ; CHECK: !16 = !{!"branch_weights", i32 0, i32 1}
 ; CHECK: !17 = !{!"branch_weights", i32 1, i32 1}
diff --git a/llvm/test/Transforms/PGOProfile/comdat_internal.ll b/llvm/test/Transforms/PGOProfile/comdat_internal.ll
index 746301791..f0906bf2b 100644
--- a/llvm/test/Transforms/PGOProfile/comdat_internal.ll
+++ b/llvm/test/Transforms/PGOProfile/comdat_internal.ll
@@ -1,22 +1,21 @@
 ; RUN: opt < %s -pgo-instr-gen -instrprof -S | FileCheck %s
 ; RUN: opt < %s -passes=pgo-instr-gen,instrprof -S | FileCheck %s
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 $foo = comdat any
 ; CHECK: $foo = comdat any
 
 ; CHECK: $__llvm_profile_raw_version = comdat any
-; CHECK: $__profv__stdin__foo.[[FOO_HASH:[0-9]+]] = comdat any
 
 @bar = global i32 ()* @foo, align 8
 
 ; CHECK: @__llvm_profile_raw_version = constant i64 {{[0-9]+}}, comdat
 ; CHECK-NOT: __profn__stdin__foo
-; CHECK: @__profc__stdin__foo.[[FOO_HASH]] = private global [1 x i64] zeroinitializer, section "__llvm_prf_cnts", comdat($__profv__stdin__foo.[[FOO_HASH]]), align 8
+; CHECK: @__profc__stdin__foo.[[FOO_HASH:[0-9]+]] = private global [1 x i64] zeroinitializer, section "__llvm_prf_cnts", comdat, align 8
 ; CHECK: @__profd__stdin__foo.[[FOO_HASH]] = private global { i64, i64, i64*, i8*, i8*, i32, [2 x i16] } { i64 -5640069336071256030, i64 [[FOO_HASH]], i64* getelementptr inbounds ([1 x i64], [1 x i64]* @__profc__stdin__foo.[[FOO_HASH]], i32 0, i32 0), i8* null
 ; CHECK-NOT: bitcast (i32 ()* @foo to i8*)
-; CHECK-SAME: , i8* null, i32 1, [2 x i16] zeroinitializer }, section "__llvm_prf_data", comdat($__profv__stdin__foo.[[FOO_HASH]]), align 8
+; CHECK-SAME: , i8* null, i32 1, [2 x i16] zeroinitializer }, section "__llvm_prf_data", comdat, align 8
 ; CHECK: @__llvm_prf_nm
 ; CHECK: @llvm.used
 
diff --git a/llvm/test/Transforms/PGOProfile/criticaledge.ll b/llvm/test/Transforms/PGOProfile/criticaledge.ll
index 4b2ea6bec..93798b563 100644
--- a/llvm/test/Transforms/PGOProfile/criticaledge.ll
+++ b/llvm/test/Transforms/PGOProfile/criticaledge.ll
@@ -3,7 +3,7 @@
 ; RUN: llvm-profdata merge %S/Inputs/criticaledge.proftext -o %t.profdata
 ; RUN: opt < %s -pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
 ; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 ; GEN: $__llvm_profile_raw_version = comdat any
diff --git a/llvm/test/Transforms/PGOProfile/cspgo_profile_summary.ll b/llvm/test/Transforms/PGOProfile/cspgo_profile_summary.ll
index ad977658f..b3098336f 100644
--- a/llvm/test/Transforms/PGOProfile/cspgo_profile_summary.ll
+++ b/llvm/test/Transforms/PGOProfile/cspgo_profile_summary.ll
@@ -4,7 +4,7 @@
 ; RUN: opt < %s -O2 -disable-preinline -pgo-kind=pgo-instr-use-pipeline -profile-file=%t.profdata -S | FileCheck %s --check-prefix=PGOSUMMARY
 ; RUN: opt < %s -O2 -disable-preinline -pgo-kind=pgo-instr-use-pipeline -profile-file=%t.profdata -S -cspgo-kind=cspgo-instr-use-pipeline| FileCheck %s --check-prefix=CSPGOSUMMARY
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @odd = common dso_local global i32 0, align 4
diff --git a/llvm/test/Transforms/PGOProfile/diag_FE_profile.ll.x b/llvm/test/Transforms/PGOProfile/diag_FE_profile.ll.x
index cd3395428..e5f58ff13 100644
--- a/llvm/test/Transforms/PGOProfile/diag_FE_profile.ll.x
+++ b/llvm/test/Transforms/PGOProfile/diag_FE_profile.ll.x
@@ -4,7 +4,7 @@
 
 ; CHECK: Not an IR level instrumentation profile
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define i32 @foo() {
diff --git a/llvm/test/Transforms/PGOProfile/diag_mismatch.ll b/llvm/test/Transforms/PGOProfile/diag_mismatch.ll
index e2b7f8cdc..c1f6c6d59 100644
--- a/llvm/test/Transforms/PGOProfile/diag_mismatch.ll
+++ b/llvm/test/Transforms/PGOProfile/diag_mismatch.ll
@@ -4,7 +4,7 @@
 
 ; CHECK: Function control flow change detected (hash mismatch) foo
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define i32 @foo() {
diff --git a/llvm/test/Transforms/PGOProfile/diag_no_funcprofdata.ll b/llvm/test/Transforms/PGOProfile/diag_no_funcprofdata.ll
index 61c1f7ab6..88133e217 100644
--- a/llvm/test/Transforms/PGOProfile/diag_no_funcprofdata.ll
+++ b/llvm/test/Transforms/PGOProfile/diag_no_funcprofdata.ll
@@ -8,7 +8,7 @@
 ; CHECK: No profile data available for function bar
 ; DEFAULT-NOT: No profile data available for function bar
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define i32 @bar() {
diff --git a/llvm/test/Transforms/PGOProfile/diag_no_profile.ll.x b/llvm/test/Transforms/PGOProfile/diag_no_profile.ll.x
index 222d9bd09..7a60dc9c1 100644
--- a/llvm/test/Transforms/PGOProfile/diag_no_profile.ll.x
+++ b/llvm/test/Transforms/PGOProfile/diag_no_profile.ll.x
@@ -1,7 +1,7 @@
 ; RUN: not opt < %s -pgo-instr-use -pgo-test-profile-file=%t.profdata -S  2>&1
 ; RUN: not opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -S  2>&1
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define i32 @foo() {
diff --git a/llvm/test/Transforms/PGOProfile/diag_no_value_sites.ll b/llvm/test/Transforms/PGOProfile/diag_no_value_sites.ll
index 251e5ce7d..d5cf2abea 100644
--- a/llvm/test/Transforms/PGOProfile/diag_no_value_sites.ll
+++ b/llvm/test/Transforms/PGOProfile/diag_no_value_sites.ll
@@ -3,7 +3,7 @@
 
 ; CHECK: Inconsistent number of value sites for memory intrinsic functions size profiling
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define void @foo(i8* %dst, i8* %src, i64 %n) {
diff --git a/llvm/test/Transforms/PGOProfile/do-not-instrument.ll b/llvm/test/Transforms/PGOProfile/do-not-instrument.ll
index 616e94273..ca950a621 100644
--- a/llvm/test/Transforms/PGOProfile/do-not-instrument.ll
+++ b/llvm/test/Transforms/PGOProfile/do-not-instrument.ll
@@ -1,7 +1,7 @@
 ; RUN: opt < %s -pgo-instr-gen -S | FileCheck %s
 ; RUN: opt < %s -passes=pgo-instr-gen -S | FileCheck %s
 
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.12.0"
 
 define i32 @f1() {
diff --git a/llvm/test/Transforms/PGOProfile/func_entry.ll b/llvm/test/Transforms/PGOProfile/func_entry.ll
index dac996e35..fe1b44b0b 100644
--- a/llvm/test/Transforms/PGOProfile/func_entry.ll
+++ b/llvm/test/Transforms/PGOProfile/func_entry.ll
@@ -1,13 +1,14 @@
 ; RUN: llvm-profdata merge %S/Inputs/func_entry.proftext -o %t.profdata
 ; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @s = common dso_local local_unnamed_addr global i32 0, align 4
 
-define void @bar() {
-; CHECK-LABEL: @bar
+define void @cold() {
+; CHECK-LABEL: @cold()
+; CHECK-SAME: #[[COLD_ATTR:[0-1]+]]
 ; CHECK-SAME: !prof ![[FUNC_ENTRY_COUNT_ZERO:[0-9]+]]
 
 entry:
@@ -15,8 +16,9 @@ entry:
   ret void
 }
 
-define void @foo() {
-; CHECK-LABEL: @foo
+define void @hot() {
+; CHECK-LABEL: @hot()
+; CHECK-SAME: #[[HOT_ATTR:[0-1]+]]
 ; CHECK-SAME: !prof ![[FUNC_ENTRY_COUNT_NON_ZERO:[0-9]+]]
 entry:
   %0 = load i32, i32* @s, align 4
@@ -25,5 +27,18 @@ entry:
   ret void
 }
 
-; CHECK-DAG: ![[FUNC_ENTRY_COUNT_ZERO]] = !{!"function_entry_count", i64 0}
-; CHECK-DAG: ![[FUNC_ENTRY_COUNT_NON_ZERO]] = !{!"function_entry_count", i64 9999}
+define void @med() {
+; CHECK-LABEL: @med
+; CHECK-NOT: #
+; CHECK-SAME: !prof ![[FUNC_ENTRY_COUNT_MED:[0-9]+]]
+
+entry:
+  store i32 1, i32* @s, align 4
+  ret void
+}
+
+; CHECK-DAG: attributes #[[COLD_ATTR]] = { cold }
+; CHECK-DAG: attributes #[[HOT_ATTR]] = { inlinehint }
+; CHECK-DAG: ![[FUNC_ENTRY_COUNT_ZERO]] = !{!"function_entry_count", i64 10}
+; CHECK-DAG: ![[FUNC_ENTRY_COUNT_NON_ZERO]] = !{!"function_entry_count", i64 9000}
+; CHECK-DAG: ![[FUNC_ENTRY_COUNT_MED]] = !{!"function_entry_count", i64 50}
diff --git a/llvm/test/Transforms/PGOProfile/icp_covariant_call_return.ll b/llvm/test/Transforms/PGOProfile/icp_covariant_call_return.ll
index aba075461..c2101a98d 100644
--- a/llvm/test/Transforms/PGOProfile/icp_covariant_call_return.ll
+++ b/llvm/test/Transforms/PGOProfile/icp_covariant_call_return.ll
@@ -1,6 +1,6 @@
 ; RUN: opt < %s -pgo-icall-prom -S | FileCheck %s --check-prefix=ICALL-PROM
 ; RUN: opt < %s -passes=pgo-icall-prom -S | FileCheck %s --check-prefix=ICALL-PROM
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 %struct.D = type { %struct.B }
diff --git a/llvm/test/Transforms/PGOProfile/icp_covariant_invoke_return.ll b/llvm/test/Transforms/PGOProfile/icp_covariant_invoke_return.ll
index 0a4444783..d0f27ac73 100644
--- a/llvm/test/Transforms/PGOProfile/icp_covariant_invoke_return.ll
+++ b/llvm/test/Transforms/PGOProfile/icp_covariant_invoke_return.ll
@@ -1,6 +1,6 @@
 ; RUN: opt < %s -pgo-icall-prom -S | FileCheck %s --check-prefix=ICALL-PROM
 ; RUN: opt < %s -passes=pgo-icall-prom -S | FileCheck %s --check-prefix=ICALL-PROM
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 %struct.D = type { %struct.B }
 %struct.B = type { i32 (...)** }
diff --git a/llvm/test/Transforms/PGOProfile/icp_invoke.ll b/llvm/test/Transforms/PGOProfile/icp_invoke.ll
index 1cacc1bc1..c30aeb6e7 100644
--- a/llvm/test/Transforms/PGOProfile/icp_invoke.ll
+++ b/llvm/test/Transforms/PGOProfile/icp_invoke.ll
@@ -1,6 +1,6 @@
 ; RUN: opt < %s -icp-lto -pgo-icall-prom -S | FileCheck %s --check-prefix=ICP
 ; RUN: opt < %s -icp-lto -passes=pgo-icall-prom -S | FileCheck %s --check-prefix=ICP
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @foo1 = global void ()* null, align 8
diff --git a/llvm/test/Transforms/PGOProfile/icp_invoke_nouse.ll b/llvm/test/Transforms/PGOProfile/icp_invoke_nouse.ll
index 096d2e0f2..55eb85881 100644
--- a/llvm/test/Transforms/PGOProfile/icp_invoke_nouse.ll
+++ b/llvm/test/Transforms/PGOProfile/icp_invoke_nouse.ll
@@ -1,5 +1,5 @@
 ; RUN: opt < %s -icp-lto -pgo-icall-prom -S | FileCheck %s --check-prefix=ICP
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @_ZTISt9exception = external constant i8*
diff --git a/llvm/test/Transforms/PGOProfile/icp_mismatch_msg.ll b/llvm/test/Transforms/PGOProfile/icp_mismatch_msg.ll
index 5484d221a..8a606e323 100644
--- a/llvm/test/Transforms/PGOProfile/icp_mismatch_msg.ll
+++ b/llvm/test/Transforms/PGOProfile/icp_mismatch_msg.ll
@@ -5,7 +5,7 @@
 ; CHECK: remark: <unknown>:0:0: Cannot promote indirect call: target with md5sum{{.*}} not found
 ; CHECK: remark: <unknown>:0:0: Cannot promote indirect call to func2 with count of 7890: Return type mismatch
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @foo = common global i32 ()* null, align 8
diff --git a/llvm/test/Transforms/PGOProfile/icp_vararg.ll b/llvm/test/Transforms/PGOProfile/icp_vararg.ll
index ec2434702..ba803cf85 100644
--- a/llvm/test/Transforms/PGOProfile/icp_vararg.ll
+++ b/llvm/test/Transforms/PGOProfile/icp_vararg.ll
@@ -1,6 +1,6 @@
 ; RUN: opt < %s -pgo-icall-prom -S | FileCheck %s --check-prefix=ICALL-PROM
 ; RUN: opt < %s -passes=pgo-icall-prom -S | FileCheck %s --check-prefix=ICALL-PROM
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @foo = common global i32 (i32, ...)* null, align 8
diff --git a/llvm/test/Transforms/PGOProfile/indirect_call_annotation.ll b/llvm/test/Transforms/PGOProfile/indirect_call_annotation.ll
index 6f72a9987..7842df37e 100644
--- a/llvm/test/Transforms/PGOProfile/indirect_call_annotation.ll
+++ b/llvm/test/Transforms/PGOProfile/indirect_call_annotation.ll
@@ -1,7 +1,7 @@
 ; RUN: llvm-profdata merge %S/Inputs/indirect_call.proftext -o %t.profdata
 ; RUN: opt < %s -pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=VP-ANNOTATION
 ; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=VP-ANNOTATION
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @foo = common global i32 (i32)* null, align 8
diff --git a/llvm/test/Transforms/PGOProfile/indirect_call_profile.ll b/llvm/test/Transforms/PGOProfile/indirect_call_profile.ll
index e1f499c08..a3c05e5b4 100644
--- a/llvm/test/Transforms/PGOProfile/indirect_call_profile.ll
+++ b/llvm/test/Transforms/PGOProfile/indirect_call_profile.ll
@@ -2,7 +2,7 @@
 ; RUN: opt < %s -passes=pgo-instr-gen -S | FileCheck %s --check-prefix=GEN
 ; RUN: opt < %s -passes=pgo-instr-gen,instrprof -S | FileCheck %s --check-prefix=LOWER
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 $foo3 = comdat any
diff --git a/llvm/test/Transforms/PGOProfile/indirect_call_promotion.ll b/llvm/test/Transforms/PGOProfile/indirect_call_promotion.ll
index 85df5260f..6a9b65211 100644
--- a/llvm/test/Transforms/PGOProfile/indirect_call_promotion.ll
+++ b/llvm/test/Transforms/PGOProfile/indirect_call_promotion.ll
@@ -15,7 +15,7 @@
 ; PASS2-REMARK-NOT: remark: <unknown>:0:0: Promote indirect call to func3
 ; PASS2-REMARK-NOT: remark: <unknown>:0:0: Promote indirect call to func1
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @foo = common global i32 ()* null, align 8
diff --git a/llvm/test/Transforms/PGOProfile/indirect_call_promotion_vla.ll b/llvm/test/Transforms/PGOProfile/indirect_call_promotion_vla.ll
index 0065031d1..6a815275e 100644
--- a/llvm/test/Transforms/PGOProfile/indirect_call_promotion_vla.ll
+++ b/llvm/test/Transforms/PGOProfile/indirect_call_promotion_vla.ll
@@ -1,6 +1,6 @@
 ; RUN: opt < %s -pgo-icall-prom -S | FileCheck %s
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 %struct.A = type { i8 }
diff --git a/llvm/test/Transforms/PGOProfile/indirectbr.ll b/llvm/test/Transforms/PGOProfile/indirectbr.ll
index ce7261d39..8c961adbf 100644
--- a/llvm/test/Transforms/PGOProfile/indirectbr.ll
+++ b/llvm/test/Transforms/PGOProfile/indirectbr.ll
@@ -4,7 +4,7 @@
 ; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
 ; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -S | opt -S -analyze -branch-prob | FileCheck %s --check-prefix=BRANCHPROB
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @foo.table = internal unnamed_addr constant [3 x i8*] [i8* blockaddress(@foo, %return), i8* blockaddress(@foo, %label2), i8* blockaddress(@foo, %label3)], align 16
diff --git a/llvm/test/Transforms/PGOProfile/landingpad.ll b/llvm/test/Transforms/PGOProfile/landingpad.ll
index 3a1a3eb57..932d2299b 100644
--- a/llvm/test/Transforms/PGOProfile/landingpad.ll
+++ b/llvm/test/Transforms/PGOProfile/landingpad.ll
@@ -3,7 +3,7 @@
 ; RUN: llvm-profdata merge %S/Inputs/landingpad.proftext -o %t.profdata
 ; RUN: opt < %s -pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
 ; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @val = global i32 0, align 4
diff --git a/llvm/test/Transforms/PGOProfile/large_count_remarks.ll b/llvm/test/Transforms/PGOProfile/large_count_remarks.ll
index 15088c396..65ab2db72 100644
--- a/llvm/test/Transforms/PGOProfile/large_count_remarks.ll
+++ b/llvm/test/Transforms/PGOProfile/large_count_remarks.ll
@@ -2,7 +2,7 @@
 ; RUN: opt < %s -pgo-instr-use -pgo-test-profile-file=%t.profdata -pass-remarks=pgo-instrumentation -pgo-emit-branch-prob -S 2>&1| FileCheck %s --check-prefix=ANALYSIS
 ; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -pass-remarks=pgo-instrumentation -pgo-emit-branch-prob -S 2>&1| FileCheck %s --check-prefix=ANALYSIS
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define i32 @test(i32 %i) {
diff --git a/llvm/test/Transforms/PGOProfile/loop1.ll b/llvm/test/Transforms/PGOProfile/loop1.ll
index dbc728a49..dbc29547b 100644
--- a/llvm/test/Transforms/PGOProfile/loop1.ll
+++ b/llvm/test/Transforms/PGOProfile/loop1.ll
@@ -3,7 +3,7 @@
 ; RUN: llvm-profdata merge %S/Inputs/loop1.proftext -o %t.profdata
 ; RUN: opt < %s -pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
 ; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 ; GEN: $__llvm_profile_raw_version = comdat any
diff --git a/llvm/test/Transforms/PGOProfile/loop2.ll b/llvm/test/Transforms/PGOProfile/loop2.ll
index 5a86f0299..1334dd261 100644
--- a/llvm/test/Transforms/PGOProfile/loop2.ll
+++ b/llvm/test/Transforms/PGOProfile/loop2.ll
@@ -3,7 +3,7 @@
 ; RUN: llvm-profdata merge %S/Inputs/loop2.proftext -o %t.profdata
 ; RUN: opt < %s -pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
 ; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 ; GEN: $__llvm_profile_raw_version = comdat any
diff --git a/llvm/test/Transforms/PGOProfile/memcpy.ll b/llvm/test/Transforms/PGOProfile/memcpy.ll
index 0a6c44b43..6047c95e7 100644
--- a/llvm/test/Transforms/PGOProfile/memcpy.ll
+++ b/llvm/test/Transforms/PGOProfile/memcpy.ll
@@ -1,6 +1,6 @@
 ; RUN: opt < %s -pgo-instr-gen -instrprof -S | FileCheck %s
 ; RUN: opt <%s -passes=pgo-instr-gen,instrprof -S | FileCheck %s
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define void @foo(i8* %dst, i8* %src, i32* %a, i32 %n) {
diff --git a/llvm/test/Transforms/PGOProfile/memop_size_annotation.ll b/llvm/test/Transforms/PGOProfile/memop_size_annotation.ll
index 4310d8ec9..a59988462 100644
--- a/llvm/test/Transforms/PGOProfile/memop_size_annotation.ll
+++ b/llvm/test/Transforms/PGOProfile/memop_size_annotation.ll
@@ -4,7 +4,7 @@
 ; RUN: opt < %s -pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefixes=MEMOP_ANNOTATION,MEMOP_ANNOTATION4
 ; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefixes=MEMOP_ANNOTATION,MEMOP_ANNOTATION4
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define void @foo(i8* %dst, i8* %src, i32* %a, i32 %n) {
diff --git a/llvm/test/Transforms/PGOProfile/memop_size_opt.ll b/llvm/test/Transforms/PGOProfile/memop_size_opt.ll
index 69cdd81dd..8d6215cf9 100644
--- a/llvm/test/Transforms/PGOProfile/memop_size_opt.ll
+++ b/llvm/test/Transforms/PGOProfile/memop_size_opt.ll
@@ -6,7 +6,7 @@
 ; RUN: FileCheck %s -input-file=%t.opt.yaml --check-prefix=YAML
 
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define void @foo(i8* %dst, i8* %src, i8* %dst2, i8* %src2, i32* %a, i32 %n) !prof !27 {
diff --git a/llvm/test/Transforms/PGOProfile/memop_size_opt_zero.ll b/llvm/test/Transforms/PGOProfile/memop_size_opt_zero.ll
index 43ca71053..e46cadf41 100644
--- a/llvm/test/Transforms/PGOProfile/memop_size_opt_zero.ll
+++ b/llvm/test/Transforms/PGOProfile/memop_size_opt_zero.ll
@@ -3,7 +3,7 @@
 ; RUN: opt < %s -passes=pgo-memop-opt -verify-dom-info -pgo-memop-count-threshold=1 -S | FileCheck %s --check-prefix=MEMOP_OPT
 ; RUN: opt < %s -pgo-memop-opt -verify-dom-info -pgo-memop-count-threshold=1 -S | FileCheck %s --check-prefix=MEMOP_OPT
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define void @foo(i8* %dst, i8* %src, i64 %conv) !prof !0 {
diff --git a/llvm/test/Transforms/PGOProfile/misexpect-branch-correct.ll b/llvm/test/Transforms/PGOProfile/misexpect-branch-correct.ll
new file mode 100644
index 000000000..90fd4bd4f
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/misexpect-branch-correct.ll
@@ -0,0 +1,94 @@
+; RUN: llvm-profdata merge %S/Inputs/misexpect-branch-correct.proftext -o %t.profdata
+
+; RUN: opt < %s -lower-expect -pgo-instr-use -pgo-test-profile-file=%t.profdata -S -pgo-warn-misexpect -pass-remarks=misexpect 2>&1 | FileCheck %s
+
+; New PM
+; RUN: opt < %s -passes="function(lower-expect),pgo-instr-use" -pgo-test-profile-file=%t.profdata -pgo-warn-misexpect -pass-remarks=misexpect -S  2>&1 | FileCheck %s
+
+; CHECK-NOT: warning: {{.*}}
+; CHECK-NOT: remark: {{.*}}
+; CHECK: !{!"misexpect", i64 1, i64 2000, i64 1}
+
+
+; ModuleID = 'misexpect-branch-correct.c'
+source_filename = "misexpect-branch-correct.c"
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@inner_loop = constant i32 100, align 4
+@outer_loop = constant i32 2000, align 4
+
+; Function Attrs: nounwind
+define i32 @bar() #0 {
+entry:
+  %rando = alloca i32, align 4
+  %x = alloca i32, align 4
+  %0 = bitcast i32* %rando to i8*
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* %0) #4
+  %call = call i32 (...) @buzz()
+  store i32 %call, i32* %rando, align 4, !tbaa !3
+  %1 = bitcast i32* %x to i8*
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* %1) #4
+  store i32 0, i32* %x, align 4, !tbaa !3
+  %2 = load i32, i32* %rando, align 4, !tbaa !3
+  %rem = srem i32 %2, 200000
+  %cmp = icmp eq i32 %rem, 0
+  %lnot = xor i1 %cmp, true
+  %lnot1 = xor i1 %lnot, true
+  %lnot.ext = zext i1 %lnot1 to i32
+  %conv = sext i32 %lnot.ext to i64
+  %expval = call i64 @llvm.expect.i64(i64 %conv, i64 0)
+  %tobool = icmp ne i64 %expval, 0
+  br i1 %tobool, label %if.then, label %if.else
+
+if.then:                                          ; preds = %entry
+  %3 = load i32, i32* %rando, align 4, !tbaa !3
+  %call2 = call i32 @baz(i32 %3)
+  store i32 %call2, i32* %x, align 4, !tbaa !3
+  br label %if.end
+
+if.else:                                          ; preds = %entry
+  %call3 = call i32 @foo(i32 50)
+  store i32 %call3, i32* %x, align 4, !tbaa !3
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  %4 = load i32, i32* %x, align 4, !tbaa !3
+  %5 = bitcast i32* %x to i8*
+  call void @llvm.lifetime.end.p0i8(i64 4, i8* %5) #4
+  %6 = bitcast i32* %rando to i8*
+  call void @llvm.lifetime.end.p0i8(i64 4, i8* %6) #4
+  ret i32 %4
+}
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.lifetime.start.p0i8(i64 immarg, i8* nocapture) #1
+
+declare i32 @buzz(...) #2
+
+; Function Attrs: nounwind readnone willreturn
+declare i64 @llvm.expect.i64(i64, i64) #3
+
+declare i32 @baz(i32) #2
+
+declare i32 @foo(i32) #2
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.lifetime.end.p0i8(i64 immarg, i8* nocapture) #1
+
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-features"="+cx8,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { argmemonly nounwind willreturn }
+attributes #2 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-features"="+cx8,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { nounwind readnone willreturn }
+attributes #4 = { nounwind }
+
+!llvm.module.flags = !{!0, !1}
+!llvm.ident = !{!2}
+
+!0 = !{i32 2, !"Debug Info Version", i32 3}
+!1 = !{i32 1, !"wchar_size", i32 4}
+!2 = !{!"clang version 10.0.0 (c20270bfffc9d6965219de339d66c61e9fe7d82d)"}
+!3 = !{!4, !4, i64 0}
+!4 = !{!"int", !5, i64 0}
+!5 = !{!"omnipotent char", !6, i64 0}
+!6 = !{!"Simple C/C++ TBAA"}
diff --git a/llvm/test/Transforms/PGOProfile/misexpect-branch-stripped.ll b/llvm/test/Transforms/PGOProfile/misexpect-branch-stripped.ll
new file mode 100644
index 000000000..f629543c5
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/misexpect-branch-stripped.ll
@@ -0,0 +1,115 @@
+
+; RUN: llvm-profdata merge %S/Inputs/misexpect-branch.proftext -o %t.profdata
+
+
+; RUN: opt < %s -lower-expect -pgo-instr-use -pgo-test-profile-file=%t.profdata -S -pgo-warn-misexpect 2>&1 | FileCheck %s --check-prefix=WARNING
+; RUN: opt < %s -lower-expect -pgo-instr-use -pgo-test-profile-file=%t.profdata -S -pass-remarks=misexpect 2>&1 | FileCheck %s --check-prefix=REMARK
+; RUN: opt < %s -lower-expect -pgo-instr-use -pgo-test-profile-file=%t.profdata -S -pgo-warn-misexpect -pass-remarks=misexpect 2>&1 | FileCheck %s --check-prefix=BOTH
+; RUN: opt < %s -lower-expect -pgo-instr-use -pgo-test-profile-file=%t.profdata -S 2>&1 | FileCheck %s --check-prefix=DISABLED
+
+; New PM
+; RUN: opt < %s -passes="function(lower-expect),pgo-instr-use" -pgo-test-profile-file=%t.profdata -pgo-warn-misexpect -S 2>&1 | FileCheck %s --check-prefix=WARNING
+; RUN: opt < %s -passes="function(lower-expect),pgo-instr-use" -pgo-test-profile-file=%t.profdata -pass-remarks=misexpect -S 2>&1 | FileCheck %s --check-prefix=REMARK
+; RUN: opt < %s -passes="function(lower-expect),pgo-instr-use" -pgo-test-profile-file=%t.profdata -pgo-warn-misexpect -pass-remarks=misexpect -S 2>&1 | FileCheck %s --check-prefix=BOTH
+; RUN: opt < %s -passes="function(lower-expect),pgo-instr-use" -pgo-test-profile-file=%t.profdata -S 2>&1 | FileCheck %s --check-prefix=DISABLED
+
+
+
+; WARNING-DAG: warning: <unknown>:0:0: 19.98%
+; WARNING-NOT: remark: <unknown>:0:0: Potential performance regression from use of the llvm.expect intrinsic: Annotation was correct on 19.98% (399668 / 2000000) of profiled executions.
+
+; REMARK-NOT: warning: <unknown>:0:0: 19.98%
+; REMARK-DAG: remark: <unknown>:0:0: Potential performance regression from use of the llvm.expect intrinsic: Annotation was correct on 19.98% (399668 / 2000000) of profiled executions.
+
+; BOTH-DAG: warning: <unknown>:0:0: 19.98%
+; BOTH-DAG: remark: <unknown>:0:0: Potential performance regression from use of the llvm.expect intrinsic: Annotation was correct on 19.98% (399668 / 2000000) of profiled executions.
+
+; DISABLED-NOT: warning: <unknown>:0:0: 19.98%
+; DISABLED-NOT: remark: <unknown>:0:0: Potential performance regression from use of the llvm.expect intrinsic: Annotation was correct on 19.98% (399668 / 2000000) of profiled executions.
+
+; CHECK-DAG: !{!"misexpect", i64 1, i64 2000, i64 1}
+
+
+
+; ModuleID = 'misexpect-branch.c'
+source_filename = "misexpect-branch.c"
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@inner_loop = constant i32 100, align 4
+@outer_loop = constant i32 2000, align 4
+
+; Function Attrs: nounwind
+define i32 @bar() #0 {
+entry:
+  %rando = alloca i32, align 4
+  %x = alloca i32, align 4
+  %0 = bitcast i32* %rando to i8*
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* %0) #4
+  %call = call i32 (...) @buzz()
+  store i32 %call, i32* %rando, align 4, !tbaa !3
+  %1 = bitcast i32* %x to i8*
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* %1) #4
+  store i32 0, i32* %x, align 4, !tbaa !3
+  %2 = load i32, i32* %rando, align 4, !tbaa !3
+  %rem = srem i32 %2, 200000
+  %cmp = icmp eq i32 %rem, 0
+  %lnot = xor i1 %cmp, true
+  %lnot1 = xor i1 %lnot, true
+  %lnot.ext = zext i1 %lnot1 to i32
+  %conv = sext i32 %lnot.ext to i64
+  %expval = call i64 @llvm.expect.i64(i64 %conv, i64 1)
+  %tobool = icmp ne i64 %expval, 0
+  br i1 %tobool, label %if.then, label %if.else
+
+if.then:                                          ; preds = %entry
+  %3 = load i32, i32* %rando, align 4, !tbaa !3
+  %call2 = call i32 @baz(i32 %3)
+  store i32 %call2, i32* %x, align 4, !tbaa !3
+  br label %if.end
+
+if.else:                                          ; preds = %entry
+  %call3 = call i32 @foo(i32 50)
+  store i32 %call3, i32* %x, align 4, !tbaa !3
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  %4 = load i32, i32* %x, align 4, !tbaa !3
+  %5 = bitcast i32* %x to i8*
+  call void @llvm.lifetime.end.p0i8(i64 4, i8* %5) #4
+  %6 = bitcast i32* %rando to i8*
+  call void @llvm.lifetime.end.p0i8(i64 4, i8* %6) #4
+  ret i32 %4
+}
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.lifetime.start.p0i8(i64 immarg, i8* nocapture) #1
+
+declare i32 @buzz(...) #2
+
+; Function Attrs: nounwind readnone willreturn
+declare i64 @llvm.expect.i64(i64, i64) #3
+
+declare i32 @baz(i32) #2
+
+declare i32 @foo(i32) #2
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.lifetime.end.p0i8(i64 immarg, i8* nocapture) #1
+
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-features"="+cx8,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { argmemonly nounwind willreturn }
+attributes #2 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-features"="+cx8,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { nounwind readnone willreturn }
+attributes #4 = { nounwind }
+
+!llvm.module.flags = !{!0, !1}
+!llvm.ident = !{!2}
+
+!0 = !{i32 2, !"Debug Info Version", i32 3}
+!1 = !{i32 1, !"wchar_size", i32 4}
+!2 = !{!"clang version 10.0.0 (trunk c20270bfffc9d6965219de339d66c61e9fe7d82d)"}
+!3 = !{!4, !4, i64 0}
+!4 = !{!"int", !5, i64 0}
+!5 = !{!"omnipotent char", !6, i64 0}
+!6 = !{!"Simple C/C++ TBAA"}
diff --git a/llvm/test/Transforms/PGOProfile/misexpect-branch-unpredictable.ll b/llvm/test/Transforms/PGOProfile/misexpect-branch-unpredictable.ll
new file mode 100644
index 000000000..be62746da
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/misexpect-branch-unpredictable.ll
@@ -0,0 +1,89 @@
+; RUN: llvm-profdata merge %S/Inputs/misexpect-branch-correct.proftext -o %t.profdata
+
+; RUN: opt < %s -lower-expect -pgo-instr-use -pgo-test-profile-file=%t.profdata -S -pgo-warn-misexpect -pass-remarks=misexpect 2>&1 | FileCheck %s
+
+; New PM
+; RUN: opt < %s -passes="function(lower-expect),pgo-instr-use" -pgo-test-profile-file=%t.profdata -pgo-warn-misexpect -pass-remarks=misexpect -S  2>&1 | FileCheck %s
+
+; CHECK-NOT: warning: {{.*}}
+; CHECK-NOT: remark: {{.*}}
+; CHECK-NOT: !"misexpect"
+
+
+; ModuleID = 'misexpect-branch-unpredictable.c'
+source_filename = "clang/test/Profile/misexpect-branch-unpredictable.c"
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@inner_loop = constant i32 100, align 4
+@outer_loop = constant i32 2000, align 4
+
+; Function Attrs: nounwind
+define i32 @bar() #0 {
+entry:
+  %rando = alloca i32, align 4
+  %x = alloca i32, align 4
+  %0 = bitcast i32* %rando to i8*
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* %0) #3
+  %call = call i32 (...) @buzz()
+  store i32 %call, i32* %rando, align 4, !tbaa !2
+  %1 = bitcast i32* %x to i8*
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* %1) #3
+  store i32 0, i32* %x, align 4, !tbaa !2
+  %2 = load i32, i32* %rando, align 4, !tbaa !2
+  %rem = srem i32 %2, 200000
+  %cmp = icmp eq i32 %rem, 0
+  %lnot = xor i1 %cmp, true
+  %lnot1 = xor i1 %lnot, true
+  %lnot.ext = zext i1 %lnot1 to i32
+  %conv = sext i32 %lnot.ext to i64
+  %tobool = icmp ne i64 %conv, 0
+  br i1 %tobool, label %if.then, label %if.else, !unpredictable !6
+
+if.then:                                          ; preds = %entry
+  %3 = load i32, i32* %rando, align 4, !tbaa !2
+  %call2 = call i32 @baz(i32 %3)
+  store i32 %call2, i32* %x, align 4, !tbaa !2
+  br label %if.end
+
+if.else:                                          ; preds = %entry
+  %call3 = call i32 @foo(i32 50)
+  store i32 %call3, i32* %x, align 4, !tbaa !2
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  %4 = load i32, i32* %x, align 4, !tbaa !2
+  %5 = bitcast i32* %x to i8*
+  call void @llvm.lifetime.end.p0i8(i64 4, i8* %5) #3
+  %6 = bitcast i32* %rando to i8*
+  call void @llvm.lifetime.end.p0i8(i64 4, i8* %6) #3
+  ret i32 %4
+}
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.lifetime.start.p0i8(i64 immarg, i8* nocapture) #1
+
+declare i32 @buzz(...) #2
+
+declare i32 @baz(i32) #2
+
+declare i32 @foo(i32) #2
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.lifetime.end.p0i8(i64 immarg, i8* nocapture) #1
+
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-features"="+cx8,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { argmemonly nounwind willreturn }
+attributes #2 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-features"="+cx8,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { nounwind }
+
+!llvm.module.flags = !{!0}
+!llvm.ident = !{!1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{!"Fuchsia clang version 10.0.0 (153b453014c94291c8c6cf6320b2f46df40f26f3) (based on LLVM 10.0.0svn)"}
+!2 = !{!3, !3, i64 0}
+!3 = !{!"int", !4, i64 0}
+!4 = !{!"omnipotent char", !5, i64 0}
+!5 = !{!"Simple C/C++ TBAA"}
+!6 = !{}
diff --git a/llvm/test/Transforms/PGOProfile/misexpect-branch.ll b/llvm/test/Transforms/PGOProfile/misexpect-branch.ll
new file mode 100644
index 000000000..df4b53de2
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/misexpect-branch.ll
@@ -0,0 +1,130 @@
+
+; RUN: llvm-profdata merge %S/Inputs/misexpect-branch.proftext -o %t.profdata
+
+
+; RUN: opt < %s -lower-expect -pgo-instr-use -pgo-test-profile-file=%t.profdata -S -pgo-warn-misexpect 2>&1 | FileCheck %s --check-prefix=WARNING
+; RUN: opt < %s -lower-expect -pgo-instr-use -pgo-test-profile-file=%t.profdata -S -pass-remarks=misexpect 2>&1 | FileCheck %s --check-prefix=REMARK
+; RUN: opt < %s -lower-expect -pgo-instr-use -pgo-test-profile-file=%t.profdata -S -pgo-warn-misexpect -pass-remarks=misexpect 2>&1 | FileCheck %s --check-prefix=BOTH
+; RUN: opt < %s -lower-expect -pgo-instr-use -pgo-test-profile-file=%t.profdata -S 2>&1 | FileCheck %s --check-prefix=DISABLED
+
+; New PM
+; RUN: opt < %s -passes="function(lower-expect),pgo-instr-use" -pgo-test-profile-file=%t.profdata -pgo-warn-misexpect -S 2>&1 | FileCheck %s --check-prefix=WARNING
+; RUN: opt < %s -passes="function(lower-expect),pgo-instr-use" -pgo-test-profile-file=%t.profdata -pass-remarks=misexpect -S 2>&1 | FileCheck %s --check-prefix=REMARK
+; RUN: opt < %s -passes="function(lower-expect),pgo-instr-use" -pgo-test-profile-file=%t.profdata -pgo-warn-misexpect -pass-remarks=misexpect -S 2>&1 | FileCheck %s --check-prefix=BOTH
+; RUN: opt < %s -passes="function(lower-expect),pgo-instr-use" -pgo-test-profile-file=%t.profdata -S 2>&1 | FileCheck %s --check-prefix=DISABLED
+
+
+
+; WARNING-DAG: warning: misexpect-branch.c:22:0: 19.98%
+; WARNING-NOT: remark: misexpect-branch.c:22:0: Potential performance regression from use of the llvm.expect intrinsic: Annotation was correct on 19.98% (399668 / 2000000) of profiled executions.
+
+; REMARK-NOT: warning: misexpect-branch.c:22:0: 19.98%
+; REMARK-DAG: remark: misexpect-branch.c:22:0: Potential performance regression from use of the llvm.expect intrinsic: Annotation was correct on 19.98% (399668 / 2000000) of profiled executions.
+
+; BOTH-DAG: warning: misexpect-branch.c:22:0: 19.98%
+; BOTH-DAG: remark: misexpect-branch.c:22:0: Potential performance regression from use of the llvm.expect intrinsic: Annotation was correct on 19.98% (399668 / 2000000) of profiled executions.
+
+; DISABLED-NOT: warning: misexpect-branch.c:22:0: 19.98%
+; DISABLED-NOT: remark: misexpect-branch.c:22:0: Potential performance regression from use of the llvm.expect intrinsic: Annotation was correct on 19.98% (399668 / 2000000) of profiled executions.
+
+; CHECK-DAG: !{!"misexpect", i64 1, i64 2000, i64 1}
+
+
+
+; ModuleID = 'misexpect-branch.c'
+source_filename = "misexpect-branch.c"
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@inner_loop = constant i32 100, align 4
+@outer_loop = constant i32 2000, align 4
+
+; Function Attrs: nounwind
+define i32 @bar() #0 !dbg !6 {
+entry:
+  %rando = alloca i32, align 4
+  %x = alloca i32, align 4
+  %0 = bitcast i32* %rando to i8*, !dbg !9
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* %0) #4, !dbg !9
+  %call = call i32 (...) @buzz(), !dbg !9
+  store i32 %call, i32* %rando, align 4, !dbg !9, !tbaa !10
+  %1 = bitcast i32* %x to i8*, !dbg !14
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* %1) #4, !dbg !14
+  store i32 0, i32* %x, align 4, !dbg !14, !tbaa !10
+  %2 = load i32, i32* %rando, align 4, !dbg !15, !tbaa !10
+  %rem = srem i32 %2, 200000, !dbg !15
+  %cmp = icmp eq i32 %rem, 0, !dbg !15
+  %lnot = xor i1 %cmp, true, !dbg !15
+  %lnot1 = xor i1 %lnot, true, !dbg !15
+  %lnot.ext = zext i1 %lnot1 to i32, !dbg !15
+  %conv = sext i32 %lnot.ext to i64, !dbg !15
+  %expval = call i64 @llvm.expect.i64(i64 %conv, i64 1), !dbg !15
+  %tobool = icmp ne i64 %expval, 0, !dbg !15
+  br i1 %tobool, label %if.then, label %if.else, !dbg !15
+
+if.then:                                          ; preds = %entry
+  %3 = load i32, i32* %rando, align 4, !dbg !16, !tbaa !10
+  %call2 = call i32 @baz(i32 %3), !dbg !16
+  store i32 %call2, i32* %x, align 4, !dbg !16, !tbaa !10
+  br label %if.end, !dbg !17
+
+if.else:                                          ; preds = %entry
+  %call3 = call i32 @foo(i32 50), !dbg !18
+  store i32 %call3, i32* %x, align 4, !dbg !18, !tbaa !10
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  %4 = load i32, i32* %x, align 4, !dbg !19, !tbaa !10
+  %5 = bitcast i32* %x to i8*, !dbg !20
+  call void @llvm.lifetime.end.p0i8(i64 4, i8* %5) #4, !dbg !20
+  %6 = bitcast i32* %rando to i8*, !dbg !20
+  call void @llvm.lifetime.end.p0i8(i64 4, i8* %6) #4, !dbg !20
+  ret i32 %4, !dbg !19
+}
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.lifetime.start.p0i8(i64 immarg, i8* nocapture) #1
+
+declare i32 @buzz(...) #2
+
+; Function Attrs: nounwind readnone willreturn
+declare i64 @llvm.expect.i64(i64, i64) #3
+
+declare i32 @baz(i32) #2
+
+declare i32 @foo(i32) #2
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.lifetime.end.p0i8(i64 immarg, i8* nocapture) #1
+
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-features"="+cx8,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { argmemonly nounwind willreturn }
+attributes #2 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-features"="+cx8,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { nounwind readnone willreturn }
+attributes #4 = { nounwind }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4}
+!llvm.ident = !{!5}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 10.0.0 (trunk c20270bfffc9d6965219de339d66c61e9fe7d82d)", isOptimized: true, runtimeVersion: 0, emissionKind: LineTablesOnly, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "<stdin>", directory: ".")
+!2 = !{}
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!4 = !{i32 1, !"wchar_size", i32 4}
+!5 = !{!"clang version 10.0.0 (trunk c20270bfffc9d6965219de339d66c61e9fe7d82d)"}
+!6 = distinct !DISubprogram(name: "bar", scope: !7, file: !7, line: 19, type: !8, scopeLine: 19, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2)
+!7 = !DIFile(filename: "misexpect-branch.c", directory: ".")
+!8 = !DISubroutineType(types: !2)
+!9 = !DILocation(line: 20, scope: !6)
+!10 = !{!11, !11, i64 0}
+!11 = !{!"int", !12, i64 0}
+!12 = !{!"omnipotent char", !13, i64 0}
+!13 = !{!"Simple C/C++ TBAA"}
+!14 = !DILocation(line: 21, scope: !6)
+!15 = !DILocation(line: 22, scope: !6)
+!16 = !DILocation(line: 23, scope: !6)
+!17 = !DILocation(line: 24, scope: !6)
+!18 = !DILocation(line: 25, scope: !6)
+!19 = !DILocation(line: 27, scope: !6)
+!20 = !DILocation(line: 28, scope: !6)
diff --git a/llvm/test/Transforms/PGOProfile/misexpect-switch-default.ll b/llvm/test/Transforms/PGOProfile/misexpect-switch-default.ll
new file mode 100644
index 000000000..e6a9389c8
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/misexpect-switch-default.ll
@@ -0,0 +1,196 @@
+
+; RUN: llvm-profdata merge %S/Inputs/misexpect-switch.proftext -o %t.profdata
+
+; RUN: opt < %s -lower-expect -pgo-instr-use -pgo-test-profile-file=%t.profdata -S -pgo-warn-misexpect 2>&1 | FileCheck %s --check-prefix=WARNING
+; RUN: opt < %s -lower-expect -pgo-instr-use -pgo-test-profile-file=%t.profdata -S -pass-remarks=misexpect 2>&1 | FileCheck %s --check-prefix=REMARK
+; RUN: opt < %s -lower-expect -pgo-instr-use -pgo-test-profile-file=%t.profdata -S -pgo-warn-misexpect -pass-remarks=misexpect 2>&1 | FileCheck %s --check-prefix=BOTH
+; RUN: opt < %s -lower-expect -pgo-instr-use -pgo-test-profile-file=%t.profdata -S 2>&1 | FileCheck %s --check-prefix=DISABLED
+
+; New PM
+; RUN: opt < %s -passes="function(lower-expect),pgo-instr-use" -pgo-test-profile-file=%t.profdata -pgo-warn-misexpect -S 2>&1 | FileCheck %s --check-prefix=WARNING
+; RUN: opt < %s -passes="function(lower-expect),pgo-instr-use" -pgo-test-profile-file=%t.profdata -pass-remarks=misexpect -S 2>&1 | FileCheck %s --check-prefix=REMARK
+; RUN: opt < %s -passes="function(lower-expect),pgo-instr-use" -pgo-test-profile-file=%t.profdata -pgo-warn-misexpect -pass-remarks=misexpect -S 2>&1 | FileCheck %s --check-prefix=BOTH
+; RUN: opt < %s -passes="function(lower-expect),pgo-instr-use" -pgo-test-profile-file=%t.profdata -S 2>&1 | FileCheck %s --check-prefix=DISABLED
+
+; WARNING-DAG: warning: <unknown>:0:0: 0.00%
+; WARNING-NOT: remark: <unknown>:0:0: Potential performance regression from use of the llvm.expect intrinsic: Annotation was correct on 0.00% (0 / 27943) of profiled executions.
+
+; REMARK-NOT: warning: <unknown>:0:0: 0.00%
+; REMARK-DAG: remark: <unknown>:0:0: Potential performance regression from use of the llvm.expect intrinsic: Annotation was correct on 0.00% (0 / 27943) of profiled executions.
+
+; BOTH-DAG: warning: <unknown>:0:0: 0.00%
+; BOTH-DAG: remark: <unknown>:0:0: Potential performance regression from use of the llvm.expect intrinsic: Annotation was correct on 0.00% (0 / 27943) of profiled executions.
+
+; DISABLED-NOT: warning: <unknown>:0:0: 0.00%
+; DISABLED-NOT: remark: <unknown>:0:0: Potential performance regression from use of the llvm.expect intrinsic: Annotation was correct on 0.00% (0 / 27943) of profiled executions.
+
+; DISABLED-NOT: warning: <unknown>:0:0: 0.00%
+; DISABLED-NOT: remark: <unknown>:0:0: Potential performance regression from use of the llvm.expect intrinsic: Annotation was correct on 0.00% (0 / 27943) of profiled executions.
+
+; CORRECT-NOT: warning: {{.*}}
+; CORRECT-NOT: remark: {{.*}}
+; CHECK-DAG: !{!"misexpect", i64 0, i64 2000, i64 1}
+
+
+
+; ModuleID = 'misexpect-switch.c'
+source_filename = "misexpect-switch.c"
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@inner_loop = dso_local constant i32 1000, align 4
+@outer_loop = dso_local constant i32 20, align 4
+@arry_size = dso_local constant i32 25, align 4
+@arry = dso_local global [25 x i32] zeroinitializer, align 16
+
+; Function Attrs: nounwind uwtable
+define dso_local void @init_arry() #0 {
+entry:
+  %i = alloca i32, align 4
+  %0 = bitcast i32* %i to i8*
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* %0) #6
+  store i32 0, i32* %i, align 4, !tbaa !4
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %1 = load i32, i32* %i, align 4, !tbaa !4
+  %cmp = icmp slt i32 %1, 25
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %call = call i32 @rand() #6
+  %rem = srem i32 %call, 10
+  %2 = load i32, i32* %i, align 4, !tbaa !4
+  %idxprom = sext i32 %2 to i64
+  %arrayidx = getelementptr inbounds [25 x i32], [25 x i32]* @arry, i64 0, i64 %idxprom
+  store i32 %rem, i32* %arrayidx, align 4, !tbaa !4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body
+  %3 = load i32, i32* %i, align 4, !tbaa !4
+  %inc = add nsw i32 %3, 1
+  store i32 %inc, i32* %i, align 4, !tbaa !4
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  %4 = bitcast i32* %i to i8*
+  call void @llvm.lifetime.end.p0i8(i64 4, i8* %4) #6
+  ret void
+}
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.lifetime.start.p0i8(i64 immarg, i8* nocapture) #1
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #2
+
+; Function Attrs: nounwind
+declare dso_local i32 @rand() #3
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.lifetime.end.p0i8(i64 immarg, i8* nocapture) #1
+
+; Function Attrs: nounwind uwtable
+define dso_local i32 @main() #0 {
+entry:
+  %retval = alloca i32, align 4
+  %val = alloca i32, align 4
+  %j = alloca i32, align 4
+  %condition = alloca i32, align 4
+  store i32 0, i32* %retval, align 4
+  call void @init_arry()
+  %0 = bitcast i32* %val to i8*
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* %0) #6
+  store i32 0, i32* %val, align 4, !tbaa !4
+  %1 = bitcast i32* %j to i8*
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* %1) #6
+  store i32 0, i32* %j, align 4, !tbaa !4
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %2 = load i32, i32* %j, align 4, !tbaa !4
+  %cmp = icmp slt i32 %2, 20000
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %3 = bitcast i32* %condition to i8*
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* %3) #6
+  %call = call i32 @rand() #6
+  %rem = srem i32 %call, 5
+  store i32 %rem, i32* %condition, align 4, !tbaa !4
+  %4 = load i32, i32* %condition, align 4, !tbaa !4
+  %conv = zext i32 %4 to i64
+  %expval = call i64 @llvm.expect.i64(i64 %conv, i64 6)
+  switch i64 %expval, label %sw.default [
+    i64 0, label %sw.bb
+    i64 1, label %sw.bb2
+    i64 2, label %sw.bb2
+    i64 3, label %sw.bb2
+    i64 4, label %sw.bb3
+  ]
+
+sw.bb:                                            ; preds = %for.body
+  %call1 = call i32 @sum(i32* getelementptr inbounds ([25 x i32], [25 x i32]* @arry, i64 0, i64 0), i32 25)
+  %5 = load i32, i32* %val, align 4, !tbaa !4
+  %add = add nsw i32 %5, %call1
+  store i32 %add, i32* %val, align 4, !tbaa !4
+  br label %sw.epilog
+
+sw.bb2:                                           ; preds = %for.body, %for.body, %for.body
+  br label %sw.epilog
+
+sw.bb3:                                           ; preds = %for.body
+  %call4 = call i32 @random_sample(i32* getelementptr inbounds ([25 x i32], [25 x i32]* @arry, i64 0, i64 0), i32 25)
+  %6 = load i32, i32* %val, align 4, !tbaa !4
+  %add5 = add nsw i32 %6, %call4
+  store i32 %add5, i32* %val, align 4, !tbaa !4
+  br label %sw.epilog
+
+sw.default:                                       ; preds = %for.body
+  unreachable
+
+sw.epilog:                                        ; preds = %sw.bb3, %sw.bb2, %sw.bb
+  %7 = bitcast i32* %condition to i8*
+  call void @llvm.lifetime.end.p0i8(i64 4, i8* %7) #6
+  br label %for.inc
+
+for.inc:                                          ; preds = %sw.epilog
+  %8 = load i32, i32* %j, align 4, !tbaa !4
+  %inc = add nsw i32 %8, 1
+  store i32 %inc, i32* %j, align 4, !tbaa !4
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  %9 = bitcast i32* %j to i8*
+  call void @llvm.lifetime.end.p0i8(i64 4, i8* %9) #6
+  %10 = bitcast i32* %val to i8*
+  call void @llvm.lifetime.end.p0i8(i64 4, i8* %10) #6
+  ret i32 0
+}
+
+; Function Attrs: nounwind readnone willreturn
+declare i64 @llvm.expect.i64(i64, i64) #4
+
+declare dso_local i32 @sum(i32*, i32) #5
+
+declare dso_local i32 @random_sample(i32*, i32) #5
+
+attributes #0 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { argmemonly nounwind willreturn }
+attributes #2 = { nounwind readnone speculatable willreturn }
+attributes #3 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #4 = { nounwind readnone willreturn }
+attributes #5 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #6 = { nounwind }
+
+!llvm.module.flags = !{!0, !1, !2}
+!llvm.ident = !{!3}
+
+!0 = !{i32 2, !"Dwarf Version", i32 4}
+!1 = !{i32 2, !"Debug Info Version", i32 3}
+!2 = !{i32 1, !"wchar_size", i32 4}
+!3 = !{!"clang version 10.0.0 (60b79b85b1763d3d25630261e5cd1adb7f0835bc)"}
+!4 = !{!5, !5, i64 0}
+!5 = !{!"int", !6, i64 0}
+!6 = !{!"omnipotent char", !7, i64 0}
+!7 = !{!"Simple C/C++ TBAA"}
diff --git a/llvm/test/Transforms/PGOProfile/misexpect-switch.ll b/llvm/test/Transforms/PGOProfile/misexpect-switch.ll
new file mode 100644
index 000000000..7883c2354
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/misexpect-switch.ll
@@ -0,0 +1,293 @@
+
+; RUN: llvm-profdata merge %S/Inputs/misexpect-switch.proftext -o %t.profdata
+; RUN: llvm-profdata merge %S/Inputs/misexpect-switch-correct.proftext -o %t.c.profdata
+
+; RUN: opt < %s -lower-expect -pgo-instr-use -pgo-test-profile-file=%t.profdata -S -pgo-warn-misexpect 2>&1 | FileCheck %s --check-prefix=WARNING
+; RUN: opt < %s -lower-expect -pgo-instr-use -pgo-test-profile-file=%t.profdata -S -pass-remarks=misexpect 2>&1 | FileCheck %s --check-prefix=REMARK
+; RUN: opt < %s -lower-expect -pgo-instr-use -pgo-test-profile-file=%t.profdata -S -pgo-warn-misexpect -pass-remarks=misexpect 2>&1 | FileCheck %s --check-prefix=BOTH
+; RUN: opt < %s -lower-expect -pgo-instr-use -pgo-test-profile-file=%t.profdata -S 2>&1 | FileCheck %s --check-prefix=DISABLED
+
+; New PM
+; RUN: opt < %s -passes="function(lower-expect),pgo-instr-use" -pgo-test-profile-file=%t.profdata -pgo-warn-misexpect -S 2>&1 | FileCheck %s --check-prefix=WARNING
+; RUN: opt < %s -passes="function(lower-expect),pgo-instr-use" -pgo-test-profile-file=%t.profdata -pass-remarks=misexpect -S 2>&1 | FileCheck %s --check-prefix=REMARK
+; RUN: opt < %s -passes="function(lower-expect),pgo-instr-use" -pgo-test-profile-file=%t.profdata -pgo-warn-misexpect -pass-remarks=misexpect -S 2>&1 | FileCheck %s --check-prefix=BOTH
+; RUN: opt < %s -passes="function(lower-expect),pgo-instr-use" -pgo-test-profile-file=%t.profdata -S 2>&1 | FileCheck %s --check-prefix=DISABLED
+
+; RUN: opt < %s -lower-expect -pgo-instr-use -pgo-test-profile-file=%t.c.profdata -S -pgo-warn-misexpect -pass-remarks=misexpect 2>&1 | FileCheck %s --check-prefix=CORRECT
+; RUN: opt < %s -passes="function(lower-expect),pgo-instr-use" -pgo-test-profile-file=%t.c.profdata -pgo-warn-misexpect -pass-remarks=misexpect -S 2>&1 | FileCheck %s --check-prefix=CORRECT
+
+; WARNING-DAG: warning: misexpect-switch.c:26:5: 0.00%
+; WARNING-NOT: remark: misexpect-switch.c:26:5: Potential performance regression from use of the llvm.expect intrinsic: Annotation was correct on 0.00% (0 / 8112) of profiled executions.
+
+; REMARK-NOT: warning: misexpect-switch.c:26:5: 0.00%
+; REMARK-DAG: remark: misexpect-switch.c:26:5: Potential performance regression from use of the llvm.expect intrinsic: Annotation was correct on 0.00% (0 / 8112) of profiled executions.
+
+; BOTH-DAG: warning: misexpect-switch.c:26:5: 0.00%
+; BOTH-DAG: remark: misexpect-switch.c:26:5: Potential performance regression from use of the llvm.expect intrinsic: Annotation was correct on 0.00% (0 / 8112) of profiled executions.
+
+; DISABLED-NOT: warning: misexpect-switch.c:26:5: 0.00%
+; DISABLED-NOT: remark: misexpect-switch.c:26:5: Potential performance regression from use of the llvm.expect intrinsic: Annotation was correct on 0.00% (0 / 8112) of profiled executions.
+
+; DISABLED-NOT: warning: misexpect-switch.c:26:5: 0.00%
+; DISABLED-NOT: remark: misexpect-switch.c:26:5: Potential performance regression from use of the llvm.expect intrinsic: Annotation was correct on 0.00% (0 / 8112) of profiled executions.
+
+; CORRECT-NOT: warning: {{.*}}
+; CORRECT-NOT: remark: {{.*}}
+; CHECK-DAG: !{!"misexpect", i64 0, i64 2000, i64 1}
+
+
+
+; ModuleID = 'misexpect-switch.c'
+source_filename = "misexpect-switch.c"
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@inner_loop = dso_local constant i32 1000, align 4, !dbg !0
+@outer_loop = dso_local constant i32 20, align 4, !dbg !6
+@arry_size = dso_local constant i32 25, align 4, !dbg !10
+@arry = dso_local global [25 x i32] zeroinitializer, align 16, !dbg !12
+
+; Function Attrs: nounwind uwtable
+define dso_local void @init_arry() #0 !dbg !21 {
+entry:
+  %i = alloca i32, align 4
+  %0 = bitcast i32* %i to i8*, !dbg !26
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* %0) #6, !dbg !26
+  call void @llvm.dbg.declare(metadata i32* %i, metadata !25, metadata !DIExpression()), !dbg !27
+  store i32 0, i32* %i, align 4, !dbg !28, !tbaa !30
+  br label %for.cond, !dbg !34
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %1 = load i32, i32* %i, align 4, !dbg !35, !tbaa !30
+  %cmp = icmp slt i32 %1, 25, !dbg !37
+  br i1 %cmp, label %for.body, label %for.end, !dbg !38
+
+for.body:                                         ; preds = %for.cond
+  %call = call i32 @rand() #6, !dbg !39
+  %rem = srem i32 %call, 10, !dbg !41
+  %2 = load i32, i32* %i, align 4, !dbg !42, !tbaa !30
+  %idxprom = sext i32 %2 to i64, !dbg !43
+  %arrayidx = getelementptr inbounds [25 x i32], [25 x i32]* @arry, i64 0, i64 %idxprom, !dbg !43
+  store i32 %rem, i32* %arrayidx, align 4, !dbg !44, !tbaa !30
+  br label %for.inc, !dbg !45
+
+for.inc:                                          ; preds = %for.body
+  %3 = load i32, i32* %i, align 4, !dbg !46, !tbaa !30
+  %inc = add nsw i32 %3, 1, !dbg !46
+  store i32 %inc, i32* %i, align 4, !dbg !46, !tbaa !30
+  br label %for.cond, !dbg !47, !llvm.loop !48
+
+for.end:                                          ; preds = %for.cond
+  %4 = bitcast i32* %i to i8*, !dbg !50
+  call void @llvm.lifetime.end.p0i8(i64 4, i8* %4) #6, !dbg !50
+  ret void, !dbg !50
+}
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.lifetime.start.p0i8(i64 immarg, i8* nocapture) #1
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #2
+
+; Function Attrs: nounwind
+declare dso_local i32 @rand() #3
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.lifetime.end.p0i8(i64 immarg, i8* nocapture) #1
+
+; Function Attrs: nounwind uwtable
+define dso_local i32 @main() #0 !dbg !51 {
+entry:
+  %retval = alloca i32, align 4
+  %val = alloca i32, align 4
+  %j = alloca i32, align 4
+  %condition = alloca i32, align 4
+  store i32 0, i32* %retval, align 4
+  call void @init_arry(), !dbg !62
+  %0 = bitcast i32* %val to i8*, !dbg !63
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* %0) #6, !dbg !63
+  call void @llvm.dbg.declare(metadata i32* %val, metadata !55, metadata !DIExpression()), !dbg !64
+  store i32 0, i32* %val, align 4, !dbg !64, !tbaa !30
+  %1 = bitcast i32* %j to i8*, !dbg !65
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* %1) #6, !dbg !65
+  call void @llvm.dbg.declare(metadata i32* %j, metadata !56, metadata !DIExpression()), !dbg !66
+  store i32 0, i32* %j, align 4, !dbg !67, !tbaa !30
+  br label %for.cond, !dbg !68
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %2 = load i32, i32* %j, align 4, !dbg !69, !tbaa !30
+  %cmp = icmp slt i32 %2, 20000, !dbg !70
+  br i1 %cmp, label %for.body, label %for.end, !dbg !71
+
+for.body:                                         ; preds = %for.cond
+  %3 = bitcast i32* %condition to i8*, !dbg !72
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* %3) #6, !dbg !72
+  call void @llvm.dbg.declare(metadata i32* %condition, metadata !57, metadata !DIExpression()), !dbg !73
+  %call = call i32 @rand() #6, !dbg !74
+  %rem = srem i32 %call, 5, !dbg !75
+  store i32 %rem, i32* %condition, align 4, !dbg !73, !tbaa !30
+  %4 = load i32, i32* %condition, align 4, !dbg !76, !tbaa !30
+  %conv = zext i32 %4 to i64, !dbg !76
+  %expval = call i64 @llvm.expect.i64(i64 %conv, i64 0), !dbg !77
+  switch i64 %expval, label %sw.default [
+    i64 0, label %sw.bb
+    i64 1, label %sw.bb2
+    i64 2, label %sw.bb2
+    i64 3, label %sw.bb2
+    i64 4, label %sw.bb3
+  ], !dbg !78
+
+sw.bb:                                            ; preds = %for.body
+  %call1 = call i32 @sum(i32* getelementptr inbounds ([25 x i32], [25 x i32]* @arry, i64 0, i64 0), i32 25), !dbg !79
+  %5 = load i32, i32* %val, align 4, !dbg !81, !tbaa !30
+  %add = add nsw i32 %5, %call1, !dbg !81
+  store i32 %add, i32* %val, align 4, !dbg !81, !tbaa !30
+  br label %sw.epilog, !dbg !82
+
+sw.bb2:                                           ; preds = %for.body, %for.body, %for.body
+  br label %sw.epilog, !dbg !83
+
+sw.bb3:                                           ; preds = %for.body
+  %call4 = call i32 @random_sample(i32* getelementptr inbounds ([25 x i32], [25 x i32]* @arry, i64 0, i64 0), i32 25), !dbg !84
+  %6 = load i32, i32* %val, align 4, !dbg !85, !tbaa !30
+  %add5 = add nsw i32 %6, %call4, !dbg !85
+  store i32 %add5, i32* %val, align 4, !dbg !85, !tbaa !30
+  br label %sw.epilog, !dbg !86
+
+sw.default:                                       ; preds = %for.body
+  unreachable, !dbg !87
+
+sw.epilog:                                        ; preds = %sw.bb3, %sw.bb2, %sw.bb
+  %7 = bitcast i32* %condition to i8*, !dbg !88
+  call void @llvm.lifetime.end.p0i8(i64 4, i8* %7) #6, !dbg !88
+  br label %for.inc, !dbg !89
+
+for.inc:                                          ; preds = %sw.epilog
+  %8 = load i32, i32* %j, align 4, !dbg !90, !tbaa !30
+  %inc = add nsw i32 %8, 1, !dbg !90
+  store i32 %inc, i32* %j, align 4, !dbg !90, !tbaa !30
+  br label %for.cond, !dbg !91, !llvm.loop !92
+
+for.end:                                          ; preds = %for.cond
+  %9 = bitcast i32* %j to i8*, !dbg !94
+  call void @llvm.lifetime.end.p0i8(i64 4, i8* %9) #6, !dbg !94
+  %10 = bitcast i32* %val to i8*, !dbg !94
+  call void @llvm.lifetime.end.p0i8(i64 4, i8* %10) #6, !dbg !94
+  ret i32 0, !dbg !95
+}
+
+; Function Attrs: nounwind readnone willreturn
+declare i64 @llvm.expect.i64(i64, i64) #4
+
+declare dso_local i32 @sum(i32*, i32) #5
+
+declare dso_local i32 @random_sample(i32*, i32) #5
+
+attributes #0 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { argmemonly nounwind willreturn }
+attributes #2 = { nounwind readnone speculatable willreturn }
+attributes #3 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #4 = { nounwind readnone willreturn }
+attributes #5 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #6 = { nounwind }
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!17, !18, !19}
+!llvm.ident = !{!20}
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "inner_loop", scope: !2, file: !3, line: 7, type: !8, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 10.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5, nameTableKind: None)
+!3 = !DIFile(filename: "misexpect-switch.c", directory: ".")
+!4 = !{}
+!5 = !{!0, !6, !10, !12}
+!6 = !DIGlobalVariableExpression(var: !7, expr: !DIExpression())
+!7 = distinct !DIGlobalVariable(name: "outer_loop", scope: !2, file: !3, line: 8, type: !8, isLocal: false, isDefinition: true)
+!8 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !9)
+!9 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!10 = !DIGlobalVariableExpression(var: !11, expr: !DIExpression())
+!11 = distinct !DIGlobalVariable(name: "arry_size", scope: !2, file: !3, line: 9, type: !8, isLocal: false, isDefinition: true)
+!12 = !DIGlobalVariableExpression(var: !13, expr: !DIExpression())
+!13 = distinct !DIGlobalVariable(name: "arry", scope: !2, file: !3, line: 11, type: !14, isLocal: false, isDefinition: true)
+!14 = !DICompositeType(tag: DW_TAG_array_type, baseType: !9, size: 800, elements: !15)
+!15 = !{!16}
+!16 = !DISubrange(count: 25)
+!17 = !{i32 2, !"Dwarf Version", i32 4}
+!18 = !{i32 2, !"Debug Info Version", i32 3}
+!19 = !{i32 1, !"wchar_size", i32 4}
+!20 = !{!"clang version 10.0.0"}
+!21 = distinct !DISubprogram(name: "init_arry", scope: !3, file: !3, line: 13, type: !22, scopeLine: 13, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !2, retainedNodes: !24)
+!22 = !DISubroutineType(types: !23)
+!23 = !{null}
+!24 = !{!25}
+!25 = !DILocalVariable(name: "i", scope: !21, file: !3, line: 14, type: !9)
+!26 = !DILocation(line: 14, column: 3, scope: !21)
+!27 = !DILocation(line: 14, column: 7, scope: !21)
+!28 = !DILocation(line: 15, column: 10, scope: !29)
+!29 = distinct !DILexicalBlock(scope: !21, file: !3, line: 15, column: 3)
+!30 = !{!31, !31, i64 0}
+!31 = !{!"int", !32, i64 0}
+!32 = !{!"omnipotent char", !33, i64 0}
+!33 = !{!"Simple C/C++ TBAA"}
+!34 = !DILocation(line: 15, column: 8, scope: !29)
+!35 = !DILocation(line: 15, column: 15, scope: !36)
+!36 = distinct !DILexicalBlock(scope: !29, file: !3, line: 15, column: 3)
+!37 = !DILocation(line: 15, column: 17, scope: !36)
+!38 = !DILocation(line: 15, column: 3, scope: !29)
+!39 = !DILocation(line: 16, column: 15, scope: !40)
+!40 = distinct !DILexicalBlock(scope: !36, file: !3, line: 15, column: 35)
+!41 = !DILocation(line: 16, column: 22, scope: !40)
+!42 = !DILocation(line: 16, column: 10, scope: !40)
+!43 = !DILocation(line: 16, column: 5, scope: !40)
+!44 = !DILocation(line: 16, column: 13, scope: !40)
+!45 = !DILocation(line: 17, column: 3, scope: !40)
+!46 = !DILocation(line: 15, column: 30, scope: !36)
+!47 = !DILocation(line: 15, column: 3, scope: !36)
+!48 = distinct !{!48, !38, !49}
+!49 = !DILocation(line: 17, column: 3, scope: !29)
+!50 = !DILocation(line: 18, column: 1, scope: !21)
+!51 = distinct !DISubprogram(name: "main", scope: !3, file: !3, line: 20, type: !52, scopeLine: 20, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !2, retainedNodes: !54)
+!52 = !DISubroutineType(types: !53)
+!53 = !{!9}
+!54 = !{!55, !56, !57}
+!55 = !DILocalVariable(name: "val", scope: !51, file: !3, line: 22, type: !9)
+!56 = !DILocalVariable(name: "j", scope: !51, file: !3, line: 23, type: !9)
+!57 = !DILocalVariable(name: "condition", scope: !58, file: !3, line: 25, type: !61)
+!58 = distinct !DILexicalBlock(scope: !59, file: !3, line: 24, column: 49)
+!59 = distinct !DILexicalBlock(scope: !60, file: !3, line: 24, column: 3)
+!60 = distinct !DILexicalBlock(scope: !51, file: !3, line: 24, column: 3)
+!61 = !DIBasicType(name: "unsigned int", size: 32, encoding: DW_ATE_unsigned)
+!62 = !DILocation(line: 21, column: 3, scope: !51)
+!63 = !DILocation(line: 22, column: 3, scope: !51)
+!64 = !DILocation(line: 22, column: 7, scope: !51)
+!65 = !DILocation(line: 23, column: 3, scope: !51)
+!66 = !DILocation(line: 23, column: 7, scope: !51)
+!67 = !DILocation(line: 24, column: 10, scope: !60)
+!68 = !DILocation(line: 24, column: 8, scope: !60)
+!69 = !DILocation(line: 24, column: 15, scope: !59)
+!70 = !DILocation(line: 24, column: 17, scope: !59)
+!71 = !DILocation(line: 24, column: 3, scope: !60)
+!72 = !DILocation(line: 25, column: 5, scope: !58)
+!73 = !DILocation(line: 25, column: 14, scope: !58)
+!74 = !DILocation(line: 25, column: 26, scope: !58)
+!75 = !DILocation(line: 25, column: 33, scope: !58)
+!76 = !DILocation(line: 26, column: 30, scope: !58)
+!77 = !DILocation(line: 26, column: 13, scope: !58)
+!78 = !DILocation(line: 26, column: 5, scope: !58)
+!79 = !DILocation(line: 28, column: 14, scope: !80)
+!80 = distinct !DILexicalBlock(scope: !58, file: !3, line: 26, column: 45)
+!81 = !DILocation(line: 28, column: 11, scope: !80)
+!82 = !DILocation(line: 29, column: 7, scope: !80)
+!83 = !DILocation(line: 33, column: 7, scope: !80)
+!84 = !DILocation(line: 35, column: 14, scope: !80)
+!85 = !DILocation(line: 35, column: 11, scope: !80)
+!86 = !DILocation(line: 36, column: 7, scope: !80)
+!87 = !DILocation(line: 38, column: 7, scope: !80)
+!88 = !DILocation(line: 40, column: 3, scope: !59)
+!89 = !DILocation(line: 40, column: 3, scope: !58)
+!90 = !DILocation(line: 24, column: 44, scope: !59)
+!91 = !DILocation(line: 24, column: 3, scope: !59)
+!92 = distinct !{!92, !71, !93}
+!93 = !DILocation(line: 40, column: 3, scope: !60)
+!94 = !DILocation(line: 43, column: 1, scope: !51)
+!95 = !DILocation(line: 42, column: 3, scope: !51)
diff --git a/llvm/test/Transforms/PGOProfile/multiple_hash_profile.ll b/llvm/test/Transforms/PGOProfile/multiple_hash_profile.ll
index 6da94826a..fdfb16b9e 100644
--- a/llvm/test/Transforms/PGOProfile/multiple_hash_profile.ll
+++ b/llvm/test/Transforms/PGOProfile/multiple_hash_profile.ll
@@ -1,7 +1,7 @@
 ; RUN: llvm-profdata merge %S/Inputs/multiple_hash_profile.proftext -o %t.profdata
 ; RUN: opt < %s -pgo-instr-use -pgo-test-profile-file=%t.profdata  -S | FileCheck %s
 ; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 $_Z3fooi = comdat any
diff --git a/llvm/test/Transforms/PGOProfile/noreturncall.ll b/llvm/test/Transforms/PGOProfile/noreturncall.ll
index c108a82ab..4470285e1 100644
--- a/llvm/test/Transforms/PGOProfile/noreturncall.ll
+++ b/llvm/test/Transforms/PGOProfile/noreturncall.ll
@@ -3,7 +3,7 @@
 ; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
 ; REQUIRES: asserts
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 declare i32 @bar0(i32)
diff --git a/llvm/test/Transforms/PGOProfile/preinline.ll b/llvm/test/Transforms/PGOProfile/preinline.ll
index 277baf99c..9a42e97d1 100644
--- a/llvm/test/Transforms/PGOProfile/preinline.ll
+++ b/llvm/test/Transforms/PGOProfile/preinline.ll
@@ -1,6 +1,6 @@
 ; RUN: opt < %s -O2 -pgo-kind=pgo-instr-gen-pipeline -S | FileCheck %s --check-prefix=GEN
 ; RUN: opt < %s -O2 -pgo-kind=pgo-instr-gen-pipeline -profile-file=default.profraw -S | FileCheck %s --check-prefix=GEN
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define i32 @foo(i32 %i) {
diff --git a/llvm/test/Transforms/PGOProfile/remap.ll b/llvm/test/Transforms/PGOProfile/remap.ll
index 2fdca9e33..e84c3c40f 100644
--- a/llvm/test/Transforms/PGOProfile/remap.ll
+++ b/llvm/test/Transforms/PGOProfile/remap.ll
@@ -1,7 +1,7 @@
 ; RUN: llvm-profdata merge %S/Inputs/remap.proftext -o %t.profdata
 ; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -pgo-test-profile-remapping-file=%S/Inputs/remap.map -S | FileCheck %s --check-prefix=USE
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define i32 @_ZN3foo3barERKN1M1XINS_6detail3quxEEE(i32 %i) {
diff --git a/llvm/test/Transforms/PGOProfile/select1.ll b/llvm/test/Transforms/PGOProfile/select1.ll
index 34e30a23a..47d83c84a 100644
--- a/llvm/test/Transforms/PGOProfile/select1.ll
+++ b/llvm/test/Transforms/PGOProfile/select1.ll
@@ -5,7 +5,7 @@
 ; RUN: llvm-profdata merge %S/Inputs/select1.proftext -o %t.profdata
 ; RUN: opt < %s -pgo-instr-use -pgo-test-profile-file=%t.profdata -pgo-instr-select=true -S | FileCheck %s --check-prefix=USE
 ; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -pgo-instr-select=true -S | FileCheck %s --check-prefix=USE
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define i32 @test_br_2(i32 %i) {
diff --git a/llvm/test/Transforms/PGOProfile/select2.ll b/llvm/test/Transforms/PGOProfile/select2.ll
index b19fa1ff7..e95d00b24 100644
--- a/llvm/test/Transforms/PGOProfile/select2.ll
+++ b/llvm/test/Transforms/PGOProfile/select2.ll
@@ -2,7 +2,7 @@
 ; RUN: opt < %s -pgo-instr-use -pgo-test-profile-file=%t.profdata -pgo-instr-select=true -S | FileCheck %s --check-prefix=USE
 ; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -pgo-instr-select=true -S | FileCheck %s --check-prefix=USE
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define i32 @foo(i32 %n) {
diff --git a/llvm/test/Transforms/PGOProfile/select_hash_conflict.ll b/llvm/test/Transforms/PGOProfile/select_hash_conflict.ll
index e7c3cdb21..ab91db48c 100644
--- a/llvm/test/Transforms/PGOProfile/select_hash_conflict.ll
+++ b/llvm/test/Transforms/PGOProfile/select_hash_conflict.ll
@@ -2,7 +2,7 @@
 ; RUN: opt < %s -pgo-instr-use -pgo-test-profile-file=%t.profdata -pgo-instr-select=true -S | FileCheck %s
 ; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -pgo-instr-select=true -S | FileCheck %s
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @a = common dso_local local_unnamed_addr global [16 x i32] zeroinitializer, align 16
diff --git a/llvm/test/Transforms/PGOProfile/single_bb.ll b/llvm/test/Transforms/PGOProfile/single_bb.ll
index 874d8e4d2..646f798a6 100644
--- a/llvm/test/Transforms/PGOProfile/single_bb.ll
+++ b/llvm/test/Transforms/PGOProfile/single_bb.ll
@@ -1,6 +1,6 @@
 ; RUN: opt < %s -pgo-instr-gen -S | FileCheck %s --check-prefix=GEN
 ; RUN: opt < %s -passes=pgo-instr-gen -S | FileCheck %s --check-prefix=GEN
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 ; GEN: $__llvm_profile_raw_version = comdat any
diff --git a/llvm/test/Transforms/PGOProfile/statics_counter_naming.ll b/llvm/test/Transforms/PGOProfile/statics_counter_naming.ll
index c329ddba9..dc15151f3 100644
--- a/llvm/test/Transforms/PGOProfile/statics_counter_naming.ll
+++ b/llvm/test/Transforms/PGOProfile/statics_counter_naming.ll
@@ -4,7 +4,7 @@
 ; RUN: opt %s -passes=pgo-instr-gen -static-func-strip-dirname-prefix=1000 -S | FileCheck %s --check-prefix=NOPATH
 ; RUN: opt %s --pgo-instr-gen -static-func-strip-dirname-prefix=1 -S | FileCheck %s --check-prefix=HASPATH
 ; RUN: opt %s -passes=pgo-instr-gen -static-func-strip-dirname-prefix=1 -S | FileCheck %s --check-prefix=HASPATH
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 ; NOPATH: @__profn_statics_counter_naming.ll_func = private constant [30 x i8] c"statics_counter_naming.ll:func"
diff --git a/llvm/test/Transforms/PGOProfile/switch.ll b/llvm/test/Transforms/PGOProfile/switch.ll
index e590e2170..2cefd34d8 100644
--- a/llvm/test/Transforms/PGOProfile/switch.ll
+++ b/llvm/test/Transforms/PGOProfile/switch.ll
@@ -3,7 +3,7 @@
 ; RUN: llvm-profdata merge %S/Inputs/switch.proftext -o %t.profdata
 ; RUN: opt < %s -pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
 ; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 ; GEN: $__llvm_profile_raw_version = comdat any
diff --git a/llvm/test/Transforms/PGOProfile/thinlto_cspgo_gen.ll b/llvm/test/Transforms/PGOProfile/thinlto_cspgo_gen.ll
index 1f6264378..1a213e821 100644
--- a/llvm/test/Transforms/PGOProfile/thinlto_cspgo_gen.ll
+++ b/llvm/test/Transforms/PGOProfile/thinlto_cspgo_gen.ll
@@ -19,7 +19,7 @@
 ; CSGEN: @__profd_
 
 source_filename = "cspgo.c"
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 $__llvm_profile_filename = comdat any
diff --git a/llvm/test/Transforms/PGOProfile/thinlto_cspgo_use.ll b/llvm/test/Transforms/PGOProfile/thinlto_cspgo_use.ll
index 6a8b03431..1be8de4c0 100644
--- a/llvm/test/Transforms/PGOProfile/thinlto_cspgo_use.ll
+++ b/llvm/test/Transforms/PGOProfile/thinlto_cspgo_use.ll
@@ -18,7 +18,7 @@
 ; CSUSE-DAG: {{![0-9]+}} = !{!"branch_weights", i32 0, i32 100000}
 
 source_filename = "cspgo.c"
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define dso_local void @foo() #0 !prof !29 {
diff --git a/llvm/test/Transforms/PGOProfile/thinlto_indirect_call_promotion.ll b/llvm/test/Transforms/PGOProfile/thinlto_indirect_call_promotion.ll
index 6d928eaca..b4c9c6fc5 100644
--- a/llvm/test/Transforms/PGOProfile/thinlto_indirect_call_promotion.ll
+++ b/llvm/test/Transforms/PGOProfile/thinlto_indirect_call_promotion.ll
@@ -12,7 +12,7 @@
 ; PASS-REMARK: Promote indirect call to a with count 1 out of 1
 ; PASS-REMARK: Promote indirect call to c.llvm.0 with count 1 out of 1
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @foo = external local_unnamed_addr global void ()*, align 8
diff --git a/llvm/test/Transforms/PGOProfile/thinlto_samplepgo_icp.ll b/llvm/test/Transforms/PGOProfile/thinlto_samplepgo_icp.ll
index b6b668dbc..e89e43f44 100644
--- a/llvm/test/Transforms/PGOProfile/thinlto_samplepgo_icp.ll
+++ b/llvm/test/Transforms/PGOProfile/thinlto_samplepgo_icp.ll
@@ -9,7 +9,7 @@
 ; IMPORTS: Import _ZL3foov.llvm.0
 ; RUN: opt %t4.bc -icp-lto -pgo-icall-prom -S | FileCheck %s --check-prefix=ICALL-PROM
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @fptr = local_unnamed_addr global void ()* null, align 8
diff --git a/llvm/test/Transforms/PGOProfile/thinlto_samplepgo_icp2.ll b/llvm/test/Transforms/PGOProfile/thinlto_samplepgo_icp2.ll
index 7e3afb077..1271b7780 100644
--- a/llvm/test/Transforms/PGOProfile/thinlto_samplepgo_icp2.ll
+++ b/llvm/test/Transforms/PGOProfile/thinlto_samplepgo_icp2.ll
@@ -24,7 +24,7 @@
 ; RUN: llvm-dis < %t3.3.2.internalize.bc | FileCheck %s --check-prefix=INTERNALIZE
 ; INTERNALIZE: define dso_local void @_ZL3barv
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @fptr = local_unnamed_addr global void ()* null, align 8
diff --git a/llvm/test/Transforms/PGOProfile/thinlto_samplepgo_icp3.ll b/llvm/test/Transforms/PGOProfile/thinlto_samplepgo_icp3.ll
index 3044964f3..15fa0398b 100644
--- a/llvm/test/Transforms/PGOProfile/thinlto_samplepgo_icp3.ll
+++ b/llvm/test/Transforms/PGOProfile/thinlto_samplepgo_icp3.ll
@@ -10,7 +10,7 @@
 ; Make sure we import the promted indirectly called target
 ; IMPORTS: Import _ZL3foov.llvm.0
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @fptr = local_unnamed_addr global void ()* null, align 8
diff --git a/llvm/test/Transforms/PGOProfile/thinlto_samplepgo_icp_droppeddead.ll b/llvm/test/Transforms/PGOProfile/thinlto_samplepgo_icp_droppeddead.ll
index fcbc2d645..df93826a7 100644
--- a/llvm/test/Transforms/PGOProfile/thinlto_samplepgo_icp_droppeddead.ll
+++ b/llvm/test/Transforms/PGOProfile/thinlto_samplepgo_icp_droppeddead.ll
@@ -12,7 +12,7 @@
 ; RUN: 		-r %t.bc,main,plx -r %t.bc,_ZL3foov,l -compute-dead=false
 ; RUN: llvm-dis < %t2.1.4.opt.bc | FileCheck %s --check-prefix=OPT-NODEAD
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @fptr = local_unnamed_addr global void ()* null, align 8
diff --git a/llvm/test/Transforms/PGOProfile/unreachable_bb.ll b/llvm/test/Transforms/PGOProfile/unreachable_bb.ll
index a27fbe775..1b7fe4b05 100644
--- a/llvm/test/Transforms/PGOProfile/unreachable_bb.ll
+++ b/llvm/test/Transforms/PGOProfile/unreachable_bb.ll
@@ -2,7 +2,7 @@
 ; RUN: opt < %s -pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
 ; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define void @foo() {
diff --git a/llvm/test/Transforms/PhaseOrdering/lifetime-sanitizer.ll b/llvm/test/Transforms/PhaseOrdering/lifetime-sanitizer.ll
new file mode 100644
index 000000000..f0dea312f
--- /dev/null
+++ b/llvm/test/Transforms/PhaseOrdering/lifetime-sanitizer.ll
@@ -0,0 +1,71 @@
+; RUN: opt < %s -O0 -S | FileCheck %s
+; RUN: opt < %s -O1 -S | FileCheck %s
+; RUN: opt < %s -O2 -S | FileCheck %s
+; RUN: opt < %s -O3 -S | FileCheck %s
+; RUN: opt < %s -passes='default<O0>' -S | FileCheck %s
+; RUN: opt < %s -passes='default<O1>' -S | FileCheck %s
+; RUN: opt < %s -passes='default<O2>' -S | FileCheck %s
+; RUN: opt < %s -passes='default<O3>' -S | FileCheck %s
+
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture)
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture)
+declare void @foo(i8* nocapture)
+
+define void @asan() sanitize_address {
+entry:
+  ; CHECK-LABEL: @asan(
+  %text = alloca i8, align 1
+
+  call void @llvm.lifetime.start.p0i8(i64 1, i8* %text)
+  call void @llvm.lifetime.end.p0i8(i64 1, i8* %text)
+  ; CHECK: call void @llvm.lifetime.start
+  ; CHECK-NEXT: call void @llvm.lifetime.end
+
+  call void @foo(i8* %text) ; Keep alloca alive
+
+  ret void
+}
+
+define void @hwasan() sanitize_hwaddress {
+entry:
+  ; CHECK-LABEL: @hwasan(
+  %text = alloca i8, align 1
+
+  call void @llvm.lifetime.start.p0i8(i64 1, i8* %text)
+  call void @llvm.lifetime.end.p0i8(i64 1, i8* %text)
+  ; CHECK: call void @llvm.lifetime.start
+  ; CHECK-NEXT: call void @llvm.lifetime.end
+
+  call void @foo(i8* %text) ; Keep alloca alive
+
+  ret void
+}
+
+define void @msan() sanitize_memory {
+entry:
+  ; CHECK-LABEL: @msan(
+  %text = alloca i8, align 1
+
+  call void @llvm.lifetime.start.p0i8(i64 1, i8* %text)
+  call void @llvm.lifetime.end.p0i8(i64 1, i8* %text)
+  ; CHECK: call void @llvm.lifetime.start
+  ; CHECK-NEXT: call void @llvm.lifetime.end
+
+  call void @foo(i8* %text) ; Keep alloca alive
+
+  ret void
+}
+
+define void @no_asan() {
+entry:
+  ; CHECK-LABEL: @no_asan(
+  %text = alloca i8, align 1
+
+  call void @llvm.lifetime.start.p0i8(i64 1, i8* %text)
+  call void @llvm.lifetime.end.p0i8(i64 1, i8* %text)
+  ; CHECK-NO: call void @llvm.lifetime
+
+  call void @foo(i8* %text) ; Keep alloca alive
+
+  ret void
+}
diff --git a/llvm/test/Transforms/PhaseOrdering/min-max-abs-cse.ll b/llvm/test/Transforms/PhaseOrdering/min-max-abs-cse.ll
new file mode 100644
index 000000000..0cdb56748
--- /dev/null
+++ b/llvm/test/Transforms/PhaseOrdering/min-max-abs-cse.ll
@@ -0,0 +1,114 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -S -O1 | FileCheck %s
+; RUN: opt -passes='default<O1>' -S < %s | FileCheck %s
+
+; In all tests, expect instcombine to canonicalize the select patterns
+; for min/max/abs to allow CSE and subsequent simplification.
+
+; sub (smax a,b), (smax a,b) --> 0
+
+; FIXME: We should canonicalize min/max to a form
+; where the cmp operands match the select operands.
+
+define i8 @smax_nsw(i8 %a, i8 %b) {
+; CHECK-LABEL: @smax_nsw(
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i8 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i8 [[A]], [[B]]
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i8 [[SUB]], 0
+; CHECK-NEXT:    [[M1:%.*]] = select i1 [[CMP1]], i8 0, i8 [[SUB]]
+; CHECK-NEXT:    [[M2:%.*]] = select i1 [[CMP2]], i8 [[SUB]], i8 0
+; CHECK-NEXT:    [[R:%.*]] = sub nsw i8 [[M2]], [[M1]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %sub = sub nsw i8 %a, %b
+  %cmp1 = icmp slt i8 %a, %b
+  %cmp2 = icmp sgt i8 %sub, 0
+  %m1 = select i1 %cmp1, i8 0, i8 %sub
+  %m2 = select i1 %cmp2, i8 %sub, i8 0
+  %r = sub i8 %m2, %m1
+  ret i8 %r
+}
+
+; or (abs a), (abs a) --> abs a
+
+define i8 @abs_swapped(i8 %a) {
+; CHECK-LABEL: @abs_swapped(
+; CHECK-NEXT:    [[NEG:%.*]] = sub i8 0, [[A:%.*]]
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i8 [[A]], 0
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp slt i8 [[A]], 0
+; CHECK-NEXT:    [[M1:%.*]] = select i1 [[CMP1]], i8 [[A]], i8 [[NEG]]
+; CHECK-NEXT:    [[M2:%.*]] = select i1 [[CMP2]], i8 [[NEG]], i8 [[A]]
+; CHECK-NEXT:    [[R:%.*]] = or i8 [[M2]], [[M1]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %neg = sub i8 0, %a
+  %cmp1 = icmp sgt i8 %a, 0
+  %cmp2 = icmp slt i8 %a, 0
+  %m1 = select i1 %cmp1, i8 %a, i8 %neg
+  %m2 = select i1 %cmp2, i8 %neg, i8 %a
+  %r = or i8 %m2, %m1
+  ret i8 %r
+}
+
+; xor (nabs a), (nabs a) --> 0
+
+define i8 @nabs_swapped(i8 %a) {
+; CHECK-LABEL: @nabs_swapped(
+; CHECK-NEXT:    [[NEG:%.*]] = sub i8 0, [[A:%.*]]
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i8 [[A]], 0
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i8 [[A]], 0
+; CHECK-NEXT:    [[M1:%.*]] = select i1 [[CMP1]], i8 [[A]], i8 [[NEG]]
+; CHECK-NEXT:    [[M2:%.*]] = select i1 [[CMP2]], i8 [[NEG]], i8 [[A]]
+; CHECK-NEXT:    [[R:%.*]] = xor i8 [[M2]], [[M1]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %neg = sub i8 0, %a
+  %cmp1 = icmp slt i8 %a, 0
+  %cmp2 = icmp sgt i8 %a, 0
+  %m1 = select i1 %cmp1, i8 %a, i8 %neg
+  %m2 = select i1 %cmp2, i8 %neg, i8 %a
+  %r = xor i8 %m2, %m1
+  ret i8 %r
+}
+
+; xor (abs a), (abs a) --> 0
+
+define i8 @abs_different_constants(i8 %a) {
+; CHECK-LABEL: @abs_different_constants(
+; CHECK-NEXT:    [[NEG:%.*]] = sub i8 0, [[A:%.*]]
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i8 [[A]], -1
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp slt i8 [[A]], 0
+; CHECK-NEXT:    [[M1:%.*]] = select i1 [[CMP1]], i8 [[A]], i8 [[NEG]]
+; CHECK-NEXT:    [[M2:%.*]] = select i1 [[CMP2]], i8 [[NEG]], i8 [[A]]
+; CHECK-NEXT:    [[R:%.*]] = xor i8 [[M2]], [[M1]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %neg = sub i8 0, %a
+  %cmp1 = icmp sgt i8 %a, -1
+  %cmp2 = icmp slt i8 %a, 0
+  %m1 = select i1 %cmp1, i8 %a, i8 %neg
+  %m2 = select i1 %cmp2, i8 %neg, i8 %a
+  %r = xor i8 %m2, %m1
+  ret i8 %r
+}
+
+; or (nabs a), (nabs a) --> nabs a
+
+define i8 @nabs_different_constants(i8 %a) {
+; CHECK-LABEL: @nabs_different_constants(
+; CHECK-NEXT:    [[NEG:%.*]] = sub i8 0, [[A:%.*]]
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i8 [[A]], 0
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i8 [[A]], -1
+; CHECK-NEXT:    [[M1:%.*]] = select i1 [[CMP1]], i8 [[A]], i8 [[NEG]]
+; CHECK-NEXT:    [[M2:%.*]] = select i1 [[CMP2]], i8 [[NEG]], i8 [[A]]
+; CHECK-NEXT:    [[R:%.*]] = or i8 [[M2]], [[M1]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %neg = sub i8 0, %a
+  %cmp1 = icmp slt i8 %a, 0
+  %cmp2 = icmp sgt i8 %a, -1
+  %m1 = select i1 %cmp1, i8 %a, i8 %neg
+  %m2 = select i1 %cmp2, i8 %neg, i8 %a
+  %r = or i8 %m2, %m1
+  ret i8 %r
+}
diff --git a/llvm/test/Transforms/PhaseOrdering/scev-custom-dl.ll b/llvm/test/Transforms/PhaseOrdering/scev-custom-dl.ll
index ae822dd2e..f67b12138 100644
--- a/llvm/test/Transforms/PhaseOrdering/scev-custom-dl.ll
+++ b/llvm/test/Transforms/PhaseOrdering/scev-custom-dl.ll
@@ -65,3 +65,73 @@ for.inc:                                          ; preds = %for.body
 for.end:                                          ; preds = %for.cond
   ret void
 }
+
+@array = weak global [101 x i32] zeroinitializer, align 32		; <[100 x i32]*> [#uses=1]
+
+; CHECK: Loop %bb: backedge-taken count is 100
+
+define void @test_range_ref1a(i32 %x) {
+entry:
+	br label %bb
+
+bb:		; preds = %bb, %entry
+	%i.01.0 = phi i32 [ 100, %entry ], [ %tmp4, %bb ]		; <i32> [#uses=2]
+	%tmp1 = getelementptr [101 x i32], [101 x i32]* @array, i32 0, i32 %i.01.0		; <i32*> [#uses=1]
+	store i32 %x, i32* %tmp1
+	%tmp4 = add i32 %i.01.0, -1		; <i32> [#uses=2]
+	%tmp7 = icmp sgt i32 %tmp4, -1		; <i1> [#uses=1]
+	br i1 %tmp7, label %bb, label %return
+
+return:		; preds = %bb
+	ret void
+}
+
+define i32 @test_loop_idiom_recogize(i32 %x, i32 %y, i32* %lam, i32* %alp) nounwind {
+bb1.thread:
+	br label %bb1
+
+bb1:		; preds = %bb1, %bb1.thread
+	%indvar = phi i32 [ 0, %bb1.thread ], [ %indvar.next, %bb1 ]		; <i32> [#uses=4]
+	%i.0.reg2mem.0 = sub i32 255, %indvar		; <i32> [#uses=2]
+	%0 = getelementptr i32, i32* %alp, i32 %i.0.reg2mem.0		; <i32*> [#uses=1]
+	%1 = load i32, i32* %0, align 4		; <i32> [#uses=1]
+	%2 = getelementptr i32, i32* %lam, i32 %i.0.reg2mem.0		; <i32*> [#uses=1]
+	store i32 %1, i32* %2, align 4
+	%3 = sub i32 254, %indvar		; <i32> [#uses=1]
+	%4 = icmp slt i32 %3, 0		; <i1> [#uses=1]
+	%indvar.next = add i32 %indvar, 1		; <i32> [#uses=1]
+	br i1 %4, label %bb2, label %bb1
+
+bb2:		; preds = %bb1
+	%tmp10 = mul i32 %indvar, %x		; <i32> [#uses=1]
+	%z.0.reg2mem.0 = add i32 %tmp10, %y		; <i32> [#uses=1]
+	%5 = add i32 %z.0.reg2mem.0, %x		; <i32> [#uses=1]
+	ret i32 %5
+}
+
+declare void @use(i1)
+
+declare void @llvm.experimental.guard(i1, ...)
+
+; This tests getRangeRef acts as intended with different idx size.
+; CHECK: max backedge-taken count is 318
+define void @test_range_ref1(i8 %t) {
+ entry:
+  %t.ptr = inttoptr i8 %t to i8*
+  %p.42 = inttoptr i8 42 to i8*
+  %cmp1 = icmp slt i8* %t.ptr, %p.42
+  call void(i1, ...) @llvm.experimental.guard(i1 %cmp1) [ "deopt"() ]
+  br label %loop
+
+ loop:
+  %idx = phi i8* [ %t.ptr, %entry ], [ %snext, %loop ]
+  %snext = getelementptr inbounds i8, i8* %idx, i64 1
+  %c = icmp slt i8* %idx, %p.42
+  call void @use(i1 %c)
+  %be = icmp slt i8* %snext, %p.42
+  br i1 %be, label %loop, label %exit
+
+ exit:
+  ret void
+}
+
diff --git a/llvm/test/Transforms/PhaseOrdering/simplifycfg-options.ll b/llvm/test/Transforms/PhaseOrdering/simplifycfg-options.ll
index 693462346..6b3ba66c9 100644
--- a/llvm/test/Transforms/PhaseOrdering/simplifycfg-options.ll
+++ b/llvm/test/Transforms/PhaseOrdering/simplifycfg-options.ll
@@ -7,7 +7,7 @@
 
 define i1 @PR33605(i32 %a, i32 %b, i32* %c) {
 ; ALL-LABEL: @PR33605(
-; ALL-NEXT:  for.body:
+; ALL-NEXT:  entry:
 ; ALL-NEXT:    [[OR:%.*]] = or i32 [[B:%.*]], [[A:%.*]]
 ; ALL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[C:%.*]], i64 1
 ; ALL-NEXT:    [[TMP0:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
@@ -15,16 +15,16 @@ define i1 @PR33605(i32 %a, i32 %b, i32* %c) {
 ; ALL-NEXT:    br i1 [[CMP]], label [[IF_END:%.*]], label [[IF_THEN:%.*]]
 ; ALL:       if.then:
 ; ALL-NEXT:    store i32 [[OR]], i32* [[ARRAYIDX]], align 4
-; ALL-NEXT:    tail call void @foo()
+; ALL-NEXT:    call void @foo()
 ; ALL-NEXT:    br label [[IF_END]]
 ; ALL:       if.end:
-; ALL-NEXT:    [[CHANGED_1_OFF0:%.*]] = phi i1 [ true, [[IF_THEN]] ], [ false, [[FOR_BODY:%.*]] ]
+; ALL-NEXT:    [[CHANGED_1_OFF0:%.*]] = phi i1 [ true, [[IF_THEN]] ], [ false, [[ENTRY:%.*]] ]
 ; ALL-NEXT:    [[TMP1:%.*]] = load i32, i32* [[C]], align 4
 ; ALL-NEXT:    [[CMP_1:%.*]] = icmp eq i32 [[OR]], [[TMP1]]
 ; ALL-NEXT:    br i1 [[CMP_1]], label [[IF_END_1:%.*]], label [[IF_THEN_1:%.*]]
 ; ALL:       if.then.1:
 ; ALL-NEXT:    store i32 [[OR]], i32* [[C]], align 4
-; ALL-NEXT:    tail call void @foo()
+; ALL-NEXT:    call void @foo()
 ; ALL-NEXT:    br label [[IF_END_1]]
 ; ALL:       if.end.1:
 ; ALL-NEXT:    [[CHANGED_1_OFF0_1:%.*]] = phi i1 [ true, [[IF_THEN_1]] ], [ [[CHANGED_1_OFF0]], [[IF_END]] ]
diff --git a/llvm/test/Transforms/PhaseOrdering/two-shifts-by-sext.ll b/llvm/test/Transforms/PhaseOrdering/two-shifts-by-sext.ll
new file mode 100644
index 000000000..82f5cfbc9
--- /dev/null
+++ b/llvm/test/Transforms/PhaseOrdering/two-shifts-by-sext.ll
@@ -0,0 +1,125 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -O1 -S < %s | FileCheck %s --check-prefixes=CHECK,CHECK-01
+; RUN: opt -O2 -S < %s | FileCheck %s --check-prefixes=CHECK,CHECK-02
+; RUN: opt -O3 -S < %s | FileCheck %s --check-prefixes=CHECK,CHECK-03
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; int two_shifts_by_sext(int val, char len) {
+;   return (val << len) >> len;
+; }
+;
+; int two_shifts_by_same_sext(int val, char len) {
+;   int wide_len = len;
+;   return (val << wide_len) >> wide_len;
+; }
+;
+; void use_int32(int);
+; int two_shifts_by_sext_with_extra_use(int val, char len) {
+;   use_int32(len);
+;   return (val << len) >> len;
+; }
+;
+; int two_shifts_by_same_sext_with_extra_use(int val, char len) {
+;   int wide_len = len;
+;   use_int32(wide_len);
+;   return (val << wide_len) >> wide_len;
+; }
+
+define i32 @two_shifts_by_sext(i32 %val, i8 signext %len) {
+; CHECK-LABEL: @two_shifts_by_sext(
+; CHECK-NEXT:    [[CONV:%.*]] = sext i8 [[LEN:%.*]] to i32
+; CHECK-NEXT:    [[SHL:%.*]] = shl i32 [[VAL:%.*]], [[CONV]]
+; CHECK-NEXT:    [[SHR:%.*]] = ashr i32 [[SHL]], [[CONV]]
+; CHECK-NEXT:    ret i32 [[SHR]]
+;
+  %val.addr = alloca i32, align 4
+  %len.addr = alloca i8, align 1
+  store i32 %val, i32* %val.addr, align 4
+  store i8 %len, i8* %len.addr, align 1
+  %val.reloaded = load i32, i32* %val.addr, align 4
+  %len.reloaded.0 = load i8, i8* %len.addr, align 1
+  %conv = sext i8 %len.reloaded.0 to i32
+  %shl = shl i32 %val.reloaded, %conv
+  %len.reloaded.1 = load i8, i8* %len.addr, align 1
+  %conv1 = sext i8 %len.reloaded.1 to i32
+  %shr = ashr i32 %shl, %conv1
+  ret i32 %shr
+}
+
+define i32 @two_shifts_by_same_sext(i32 %val, i8 signext %len) {
+; CHECK-LABEL: @two_shifts_by_same_sext(
+; CHECK-NEXT:    [[CONV:%.*]] = sext i8 [[LEN:%.*]] to i32
+; CHECK-NEXT:    [[SHL:%.*]] = shl i32 [[VAL:%.*]], [[CONV]]
+; CHECK-NEXT:    [[SHR:%.*]] = ashr i32 [[SHL]], [[CONV]]
+; CHECK-NEXT:    ret i32 [[SHR]]
+;
+  %val.addr = alloca i32, align 4
+  %len.addr = alloca i8, align 1
+  %wide_len = alloca i32, align 4
+  store i32 %val, i32* %val.addr, align 4
+  store i8 %len, i8* %len.addr, align 1
+  %len.reloaded.0 = load i8, i8* %len.addr, align 1
+  %conv = sext i8 %len.reloaded.0 to i32
+  store i32 %conv, i32* %wide_len, align 4
+  %val.reloaded = load i32, i32* %val.addr, align 4
+  %len.reloaded.1 = load i32, i32* %wide_len, align 4
+  %shl = shl i32 %val.reloaded, %len.reloaded.1
+  %len.reloaded.2 = load i32, i32* %wide_len, align 4
+  %shr = ashr i32 %shl, %len.reloaded.2
+  ret i32 %shr
+}
+
+define i32 @two_shifts_by_sext_with_extra_use(i32 %val, i8 signext %len) {
+; CHECK-LABEL: @two_shifts_by_sext_with_extra_use(
+; CHECK-NEXT:    [[CONV:%.*]] = sext i8 [[LEN:%.*]] to i32
+; CHECK-NEXT:    call void @use_int32(i32 [[CONV]])
+; CHECK-NEXT:    [[SHL:%.*]] = shl i32 [[VAL:%.*]], [[CONV]]
+; CHECK-NEXT:    [[SHR:%.*]] = ashr i32 [[SHL]], [[CONV]]
+; CHECK-NEXT:    ret i32 [[SHR]]
+;
+  %val.addr = alloca i32, align 4
+  %len.addr = alloca i8, align 1
+  store i32 %val, i32* %val.addr, align 4
+  store i8 %len, i8* %len.addr, align 1
+  %len.reloaded.0 = load i8, i8* %len.addr, align 1
+  %conv = sext i8 %len.reloaded.0 to i32
+  call void @use_int32(i32 %conv)
+  %val.reloaded = load i32, i32* %val.addr, align 4
+  %len.reloaded.1 = load i8, i8* %len.addr, align 1
+  %conv1 = sext i8 %len.reloaded.1 to i32
+  %shl = shl i32 %val.reloaded, %conv1
+  %len.reloaded.2 = load i8, i8* %len.addr, align 1
+  %conv2 = sext i8 %len.reloaded.2 to i32
+  %shr = ashr i32 %shl, %conv2
+  ret i32 %shr
+}
+
+declare void @use_int32(i32)
+
+define i32 @two_shifts_by_same_sext_with_extra_use(i32 %val, i8 signext %len) {
+; CHECK-LABEL: @two_shifts_by_same_sext_with_extra_use(
+; CHECK-NEXT:    [[CONV:%.*]] = sext i8 [[LEN:%.*]] to i32
+; CHECK-NEXT:    call void @use_int32(i32 [[CONV]])
+; CHECK-NEXT:    [[SHL:%.*]] = shl i32 [[VAL:%.*]], [[CONV]]
+; CHECK-NEXT:    [[SHR:%.*]] = ashr i32 [[SHL]], [[CONV]]
+; CHECK-NEXT:    ret i32 [[SHR]]
+;
+  %val.addr = alloca i32, align 4
+  %len.addr = alloca i8, align 1
+  %wide_len = alloca i32, align 4
+  store i32 %val, i32* %val.addr, align 4
+  store i8 %len, i8* %len.addr, align 1
+  %len.reloaded.0 = load i8, i8* %len.addr, align 1
+  %conv = sext i8 %len.reloaded.0 to i32
+  store i32 %conv, i32* %wide_len, align 4
+  %val.reloaded = load i32, i32* %wide_len, align 4
+  call void @use_int32(i32 %val.reloaded)
+  %len.reloaded.1 = load i32, i32* %val.addr, align 4
+  %len.reloaded.2 = load i32, i32* %wide_len, align 4
+  %shl = shl i32 %len.reloaded.1, %len.reloaded.2
+  %wide_len.reloaded = load i32, i32* %wide_len, align 4
+  %shr = ashr i32 %shl, %wide_len.reloaded
+  ret i32 %shr
+}
diff --git a/llvm/test/Transforms/PhaseOrdering/unsigned-multiply-overflow-check.ll b/llvm/test/Transforms/PhaseOrdering/unsigned-multiply-overflow-check.ll
new file mode 100644
index 000000000..6b097da3f
--- /dev/null
+++ b/llvm/test/Transforms/PhaseOrdering/unsigned-multiply-overflow-check.ll
@@ -0,0 +1,161 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -simplifycfg -S < %s | FileCheck %s --check-prefixes=ALL,SIMPLIFYCFG
+; RUN: opt -instcombine -S < %s | FileCheck %s --check-prefixes=ALL,INSTCOMBINE,INSTCOMBINEONLY
+; RUN: opt -instcombine -simplifycfg -S < %s | FileCheck %s --check-prefixes=ALL,INSTCOMBINE,INSTCOMBINESIMPLIFYCFG,INSTCOMBINESIMPLIFYCFGDEFAULT,INSTCOMBINESIMPLIFYCFGONLY
+; RUN: opt -instcombine -simplifycfg -instcombine -S < %s | FileCheck %s --check-prefixes=ALL,INSTCOMBINE,INSTCOMBINESIMPLIFYCFG,INSTCOMBINESIMPLIFYCFGDEFAULT,INSTCOMBINESIMPLIFYCFGINSTCOMBINE
+; RUN: opt -instcombine -simplifycfg -phi-node-folding-threshold=3 -S < %s | FileCheck %s --check-prefixes=ALL,INSTCOMBINE,INSTCOMBINESIMPLIFYCFG,INSTCOMBINESIMPLIFYCFGCOSTLY,INSTCOMBINESIMPLIFYCFGCOSTLYONLY
+; RUN: opt -instcombine -simplifycfg -instcombine -phi-node-folding-threshold=3 -S < %s | FileCheck %s --check-prefixes=ALL,INSTCOMBINE,INSTCOMBINESIMPLIFYCFG,INSTCOMBINESIMPLIFYCFGCOSTLY,INSTCOMBINESIMPLIFYCFGCOSTLYINSTCOMBINE
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-linux-gnu"
+
+; #include <limits>
+; #include <cstdint>
+;
+; using size_type = std::size_t;
+; bool will_not_overflow(size_type size, size_type nmemb) {
+;   return (size != 0 && (nmemb > std::numeric_limits<size_type>::max() / size));
+; }
+
+define i1 @will_not_overflow(i64 %arg, i64 %arg1) {
+; SIMPLIFYCFG-LABEL: @will_not_overflow(
+; SIMPLIFYCFG-NEXT:  bb:
+; SIMPLIFYCFG-NEXT:    [[T0:%.*]] = icmp eq i64 [[ARG:%.*]], 0
+; SIMPLIFYCFG-NEXT:    br i1 [[T0]], label [[BB5:%.*]], label [[BB2:%.*]]
+; SIMPLIFYCFG:       bb2:
+; SIMPLIFYCFG-NEXT:    [[T3:%.*]] = udiv i64 -1, [[ARG]]
+; SIMPLIFYCFG-NEXT:    [[T4:%.*]] = icmp ult i64 [[T3]], [[ARG1:%.*]]
+; SIMPLIFYCFG-NEXT:    br label [[BB5]]
+; SIMPLIFYCFG:       bb5:
+; SIMPLIFYCFG-NEXT:    [[T6:%.*]] = phi i1 [ false, [[BB:%.*]] ], [ [[T4]], [[BB2]] ]
+; SIMPLIFYCFG-NEXT:    ret i1 [[T6]]
+;
+; INSTCOMBINEONLY-LABEL: @will_not_overflow(
+; INSTCOMBINEONLY-NEXT:  bb:
+; INSTCOMBINEONLY-NEXT:    [[T0:%.*]] = icmp eq i64 [[ARG:%.*]], 0
+; INSTCOMBINEONLY-NEXT:    br i1 [[T0]], label [[BB5:%.*]], label [[BB2:%.*]]
+; INSTCOMBINEONLY:       bb2:
+; INSTCOMBINEONLY-NEXT:    [[UMUL:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 [[ARG]], i64 [[ARG1:%.*]])
+; INSTCOMBINEONLY-NEXT:    [[UMUL_OV:%.*]] = extractvalue { i64, i1 } [[UMUL]], 1
+; INSTCOMBINEONLY-NEXT:    br label [[BB5]]
+; INSTCOMBINEONLY:       bb5:
+; INSTCOMBINEONLY-NEXT:    [[T6:%.*]] = phi i1 [ false, [[BB:%.*]] ], [ [[UMUL_OV]], [[BB2]] ]
+; INSTCOMBINEONLY-NEXT:    ret i1 [[T6]]
+;
+; INSTCOMBINESIMPLIFYCFGONLY-LABEL: @will_not_overflow(
+; INSTCOMBINESIMPLIFYCFGONLY-NEXT:  bb:
+; INSTCOMBINESIMPLIFYCFGONLY-NEXT:    [[T0:%.*]] = icmp eq i64 [[ARG:%.*]], 0
+; INSTCOMBINESIMPLIFYCFGONLY-NEXT:    [[UMUL:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 [[ARG]], i64 [[ARG1:%.*]])
+; INSTCOMBINESIMPLIFYCFGONLY-NEXT:    [[UMUL_OV:%.*]] = extractvalue { i64, i1 } [[UMUL]], 1
+; INSTCOMBINESIMPLIFYCFGONLY-NEXT:    [[T6:%.*]] = select i1 [[T0]], i1 false, i1 [[UMUL_OV]]
+; INSTCOMBINESIMPLIFYCFGONLY-NEXT:    ret i1 [[T6]]
+;
+; INSTCOMBINESIMPLIFYCFGINSTCOMBINE-LABEL: @will_not_overflow(
+; INSTCOMBINESIMPLIFYCFGINSTCOMBINE-NEXT:  bb:
+; INSTCOMBINESIMPLIFYCFGINSTCOMBINE-NEXT:    [[UMUL:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 [[ARG:%.*]], i64 [[ARG1:%.*]])
+; INSTCOMBINESIMPLIFYCFGINSTCOMBINE-NEXT:    [[UMUL_OV:%.*]] = extractvalue { i64, i1 } [[UMUL]], 1
+; INSTCOMBINESIMPLIFYCFGINSTCOMBINE-NEXT:    ret i1 [[UMUL_OV]]
+;
+; INSTCOMBINESIMPLIFYCFGCOSTLYONLY-LABEL: @will_not_overflow(
+; INSTCOMBINESIMPLIFYCFGCOSTLYONLY-NEXT:  bb:
+; INSTCOMBINESIMPLIFYCFGCOSTLYONLY-NEXT:    [[T0:%.*]] = icmp eq i64 [[ARG:%.*]], 0
+; INSTCOMBINESIMPLIFYCFGCOSTLYONLY-NEXT:    [[UMUL:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 [[ARG]], i64 [[ARG1:%.*]])
+; INSTCOMBINESIMPLIFYCFGCOSTLYONLY-NEXT:    [[UMUL_OV:%.*]] = extractvalue { i64, i1 } [[UMUL]], 1
+; INSTCOMBINESIMPLIFYCFGCOSTLYONLY-NEXT:    [[T6:%.*]] = select i1 [[T0]], i1 false, i1 [[UMUL_OV]]
+; INSTCOMBINESIMPLIFYCFGCOSTLYONLY-NEXT:    ret i1 [[T6]]
+;
+; INSTCOMBINESIMPLIFYCFGCOSTLYINSTCOMBINE-LABEL: @will_not_overflow(
+; INSTCOMBINESIMPLIFYCFGCOSTLYINSTCOMBINE-NEXT:  bb:
+; INSTCOMBINESIMPLIFYCFGCOSTLYINSTCOMBINE-NEXT:    [[UMUL:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 [[ARG:%.*]], i64 [[ARG1:%.*]])
+; INSTCOMBINESIMPLIFYCFGCOSTLYINSTCOMBINE-NEXT:    [[UMUL_OV:%.*]] = extractvalue { i64, i1 } [[UMUL]], 1
+; INSTCOMBINESIMPLIFYCFGCOSTLYINSTCOMBINE-NEXT:    ret i1 [[UMUL_OV]]
+;
+bb:
+  %t0 = icmp eq i64 %arg, 0
+  br i1 %t0, label %bb5, label %bb2
+
+bb2:                                              ; preds = %bb
+  %t3 = udiv i64 -1, %arg
+  %t4 = icmp ult i64 %t3, %arg1
+  br label %bb5
+
+bb5:                                              ; preds = %bb2, %bb
+  %t6 = phi i1 [ false, %bb ], [ %t4, %bb2 ]
+  ret i1 %t6
+}
+
+; Same as @will_not_overflow, but inverting return value.
+
+define i1 @will_overflow(i64 %arg, i64 %arg1) {
+; SIMPLIFYCFG-LABEL: @will_overflow(
+; SIMPLIFYCFG-NEXT:  bb:
+; SIMPLIFYCFG-NEXT:    [[T0:%.*]] = icmp eq i64 [[ARG:%.*]], 0
+; SIMPLIFYCFG-NEXT:    br i1 [[T0]], label [[BB5:%.*]], label [[BB2:%.*]]
+; SIMPLIFYCFG:       bb2:
+; SIMPLIFYCFG-NEXT:    [[T3:%.*]] = udiv i64 -1, [[ARG]]
+; SIMPLIFYCFG-NEXT:    [[T4:%.*]] = icmp ult i64 [[T3]], [[ARG1:%.*]]
+; SIMPLIFYCFG-NEXT:    br label [[BB5]]
+; SIMPLIFYCFG:       bb5:
+; SIMPLIFYCFG-NEXT:    [[T6:%.*]] = phi i1 [ false, [[BB:%.*]] ], [ [[T4]], [[BB2]] ]
+; SIMPLIFYCFG-NEXT:    [[T7:%.*]] = xor i1 [[T6]], true
+; SIMPLIFYCFG-NEXT:    ret i1 [[T7]]
+;
+; INSTCOMBINEONLY-LABEL: @will_overflow(
+; INSTCOMBINEONLY-NEXT:  bb:
+; INSTCOMBINEONLY-NEXT:    [[T0:%.*]] = icmp eq i64 [[ARG:%.*]], 0
+; INSTCOMBINEONLY-NEXT:    br i1 [[T0]], label [[BB5:%.*]], label [[BB2:%.*]]
+; INSTCOMBINEONLY:       bb2:
+; INSTCOMBINEONLY-NEXT:    [[UMUL:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 [[ARG]], i64 [[ARG1:%.*]])
+; INSTCOMBINEONLY-NEXT:    [[UMUL_OV:%.*]] = extractvalue { i64, i1 } [[UMUL]], 1
+; INSTCOMBINEONLY-NEXT:    [[PHITMP:%.*]] = xor i1 [[UMUL_OV]], true
+; INSTCOMBINEONLY-NEXT:    br label [[BB5]]
+; INSTCOMBINEONLY:       bb5:
+; INSTCOMBINEONLY-NEXT:    [[T6:%.*]] = phi i1 [ true, [[BB:%.*]] ], [ [[PHITMP]], [[BB2]] ]
+; INSTCOMBINEONLY-NEXT:    ret i1 [[T6]]
+;
+; INSTCOMBINESIMPLIFYCFGONLY-LABEL: @will_overflow(
+; INSTCOMBINESIMPLIFYCFGONLY-NEXT:  bb:
+; INSTCOMBINESIMPLIFYCFGONLY-NEXT:    [[T0:%.*]] = icmp eq i64 [[ARG:%.*]], 0
+; INSTCOMBINESIMPLIFYCFGONLY-NEXT:    [[UMUL:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 [[ARG]], i64 [[ARG1:%.*]])
+; INSTCOMBINESIMPLIFYCFGONLY-NEXT:    [[UMUL_OV:%.*]] = extractvalue { i64, i1 } [[UMUL]], 1
+; INSTCOMBINESIMPLIFYCFGONLY-NEXT:    [[PHITMP:%.*]] = xor i1 [[UMUL_OV]], true
+; INSTCOMBINESIMPLIFYCFGONLY-NEXT:    [[T6:%.*]] = select i1 [[T0]], i1 true, i1 [[PHITMP]]
+; INSTCOMBINESIMPLIFYCFGONLY-NEXT:    ret i1 [[T6]]
+;
+; INSTCOMBINESIMPLIFYCFGINSTCOMBINE-LABEL: @will_overflow(
+; INSTCOMBINESIMPLIFYCFGINSTCOMBINE-NEXT:  bb:
+; INSTCOMBINESIMPLIFYCFGINSTCOMBINE-NEXT:    [[UMUL:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 [[ARG:%.*]], i64 [[ARG1:%.*]])
+; INSTCOMBINESIMPLIFYCFGINSTCOMBINE-NEXT:    [[UMUL_OV:%.*]] = extractvalue { i64, i1 } [[UMUL]], 1
+; INSTCOMBINESIMPLIFYCFGINSTCOMBINE-NEXT:    [[PHITMP:%.*]] = xor i1 [[UMUL_OV]], true
+; INSTCOMBINESIMPLIFYCFGINSTCOMBINE-NEXT:    ret i1 [[PHITMP]]
+;
+; INSTCOMBINESIMPLIFYCFGCOSTLYONLY-LABEL: @will_overflow(
+; INSTCOMBINESIMPLIFYCFGCOSTLYONLY-NEXT:  bb:
+; INSTCOMBINESIMPLIFYCFGCOSTLYONLY-NEXT:    [[T0:%.*]] = icmp eq i64 [[ARG:%.*]], 0
+; INSTCOMBINESIMPLIFYCFGCOSTLYONLY-NEXT:    [[UMUL:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 [[ARG]], i64 [[ARG1:%.*]])
+; INSTCOMBINESIMPLIFYCFGCOSTLYONLY-NEXT:    [[UMUL_OV:%.*]] = extractvalue { i64, i1 } [[UMUL]], 1
+; INSTCOMBINESIMPLIFYCFGCOSTLYONLY-NEXT:    [[PHITMP:%.*]] = xor i1 [[UMUL_OV]], true
+; INSTCOMBINESIMPLIFYCFGCOSTLYONLY-NEXT:    [[T6:%.*]] = select i1 [[T0]], i1 true, i1 [[PHITMP]]
+; INSTCOMBINESIMPLIFYCFGCOSTLYONLY-NEXT:    ret i1 [[T6]]
+;
+; INSTCOMBINESIMPLIFYCFGCOSTLYINSTCOMBINE-LABEL: @will_overflow(
+; INSTCOMBINESIMPLIFYCFGCOSTLYINSTCOMBINE-NEXT:  bb:
+; INSTCOMBINESIMPLIFYCFGCOSTLYINSTCOMBINE-NEXT:    [[UMUL:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 [[ARG:%.*]], i64 [[ARG1:%.*]])
+; INSTCOMBINESIMPLIFYCFGCOSTLYINSTCOMBINE-NEXT:    [[UMUL_OV:%.*]] = extractvalue { i64, i1 } [[UMUL]], 1
+; INSTCOMBINESIMPLIFYCFGCOSTLYINSTCOMBINE-NEXT:    [[PHITMP:%.*]] = xor i1 [[UMUL_OV]], true
+; INSTCOMBINESIMPLIFYCFGCOSTLYINSTCOMBINE-NEXT:    ret i1 [[PHITMP]]
+;
+bb:
+  %t0 = icmp eq i64 %arg, 0
+  br i1 %t0, label %bb5, label %bb2
+
+bb2:                                              ; preds = %bb
+  %t3 = udiv i64 -1, %arg
+  %t4 = icmp ult i64 %t3, %arg1
+  br label %bb5
+
+bb5:                                              ; preds = %bb2, %bb
+  %t6 = phi i1 [ false, %bb ], [ %t4, %bb2 ]
+  %t7 = xor i1 %t6, true
+  ret i1 %t7
+}
diff --git a/llvm/test/Transforms/PreISelIntrinsicLowering/objc-arc.ll b/llvm/test/Transforms/PreISelIntrinsicLowering/objc-arc.ll
index 8b7d11ea7..5ff620a71 100644
--- a/llvm/test/Transforms/PreISelIntrinsicLowering/objc-arc.ll
+++ b/llvm/test/Transforms/PreISelIntrinsicLowering/objc-arc.ll
@@ -7,7 +7,7 @@
 define i8* @test_objc_autorelease(i8* %arg0) {
 ; CHECK-LABEL: test_objc_autorelease
 ; CHECK-NEXT: entry
-; CHECK-NEXT: %0 = call i8* @objc_autorelease(i8* %arg0)
+; CHECK-NEXT: %0 = notail call i8* @objc_autorelease(i8* %arg0)
 ; CHECK-NEXT: ret i8* %0
 entry:
   %0 = call i8* @llvm.objc.autorelease(i8* %arg0)
@@ -37,7 +37,7 @@ entry:
 define i8* @test_objc_autoreleaseReturnValue(i8* %arg0) {
 ; CHECK-LABEL: test_objc_autoreleaseReturnValue
 ; CHECK-NEXT: entry
-; CHECK-NEXT: %0 = call i8* @objc_autoreleaseReturnValue(i8* %arg0)
+; CHECK-NEXT: %0 = tail call i8* @objc_autoreleaseReturnValue(i8* %arg0)
 ; CHECK-NEXT: ret i8* %0
 entry:
   %0 = call i8* @llvm.objc.autoreleaseReturnValue(i8* %arg0)
@@ -117,7 +117,7 @@ entry:
 define i8* @test_objc_retain(i8* %arg0) {
 ; CHECK-LABEL: test_objc_retain
 ; CHECK-NEXT: entry
-; CHECK-NEXT: %0 = call i8* @objc_retain(i8* %arg0)
+; CHECK-NEXT: %0 = tail call i8* @objc_retain(i8* %arg0)
 ; CHECK-NEXT: ret i8* %0
 entry:
   %0 = call i8* @llvm.objc.retain(i8* %arg0)
@@ -147,7 +147,7 @@ entry:
 define i8* @test_objc_retainAutoreleasedReturnValue(i8* %arg0) {
 ; CHECK-LABEL: test_objc_retainAutoreleasedReturnValue
 ; CHECK-NEXT: entry
-; CHECK-NEXT: %0 = call i8* @objc_retainAutoreleasedReturnValue(i8* %arg0)
+; CHECK-NEXT: %0 = tail call i8* @objc_retainAutoreleasedReturnValue(i8* %arg0)
 ; CHECK-NEXT: ret i8* %0
 entry:
   %0 = call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %arg0)
@@ -187,7 +187,7 @@ entry:
 define i8* @test_objc_unsafeClaimAutoreleasedReturnValue(i8* %arg0) {
 ; CHECK-LABEL: test_objc_unsafeClaimAutoreleasedReturnValue
 ; CHECK-NEXT: entry
-; CHECK-NEXT: %0 = call i8* @objc_unsafeClaimAutoreleasedReturnValue(i8* %arg0)
+; CHECK-NEXT: %0 = tail call i8* @objc_unsafeClaimAutoreleasedReturnValue(i8* %arg0)
 ; CHECK-NEXT: ret i8* %0
 entry:
   %0 = call i8* @llvm.objc.unsafeClaimAutoreleasedReturnValue(i8* %arg0)
diff --git a/llvm/test/Transforms/Reassociate/2002-05-15-MissedTree.ll b/llvm/test/Transforms/Reassociate/2002-05-15-MissedTree.ll
index 5f3c9209a..099c394f9 100644
--- a/llvm/test/Transforms/Reassociate/2002-05-15-MissedTree.ll
+++ b/llvm/test/Transforms/Reassociate/2002-05-15-MissedTree.ll
@@ -1,11 +1,13 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -reassociate -instcombine -S | FileCheck %s
 
 define i32 @test1(i32 %A, i32 %B) {
-; CHECK-LABEL: test1
-; CHECK: %Z = add i32 %B, %A
-; CHECK: ret i32 %Z
-	%W = add i32 %B, -5
-	%Y = add i32 %A, 5
-	%Z = add i32 %W, %Y
-	ret i32 %Z
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[Z:%.*]] = add i32 [[B:%.*]], [[A:%.*]]
+; CHECK-NEXT:    ret i32 [[Z]]
+;
+  %W = add i32 %B, -5
+  %Y = add i32 %A, 5
+  %Z = add i32 %W, %Y
+  ret i32 %Z
 }
diff --git a/llvm/test/Transforms/Reassociate/2002-05-15-SubReassociate.ll b/llvm/test/Transforms/Reassociate/2002-05-15-SubReassociate.ll
index 8039ddef4..f3709c819 100644
--- a/llvm/test/Transforms/Reassociate/2002-05-15-SubReassociate.ll
+++ b/llvm/test/Transforms/Reassociate/2002-05-15-SubReassociate.ll
@@ -1,9 +1,10 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -reassociate -constprop -instcombine -dce -S | FileCheck %s
 
 ; With sub reassociation, constant folding can eliminate all of the constants.
 define i32 @test1(i32 %A, i32 %B) {
 ; CHECK-LABEL: @test1(
-; CHECK-NEXT:    [[Z:%.*]] = sub i32 %A, %B
+; CHECK-NEXT:    [[Z:%.*]] = sub i32 [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    ret i32 [[Z]]
 ;
   %W = add i32 5, %B
@@ -16,9 +17,9 @@ define i32 @test1(i32 %A, i32 %B) {
 ; With sub reassociation, constant folding can eliminate the two 12 constants.
 define i32 @test2(i32 %A, i32 %B, i32 %C, i32 %D) {
 ; CHECK-LABEL: @test2(
-; CHECK-NEXT:    [[SUM:%.*]] = add i32 %B, %A
-; CHECK-NEXT:    [[SUM1:%.*]] = add i32 [[SUM]], %C
-; CHECK-NEXT:    [[Q:%.*]] = sub i32 %D, [[SUM1]]
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[B:%.*]], [[A:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = add i32 [[TMP1]], [[C:%.*]]
+; CHECK-NEXT:    [[Q:%.*]] = sub i32 [[D:%.*]], [[TMP2]]
 ; CHECK-NEXT:    ret i32 [[Q]]
 ;
   %M = add i32 %A, 12
diff --git a/llvm/test/Transforms/Reassociate/2019-08-22-FNegAssert.ll b/llvm/test/Transforms/Reassociate/2019-08-22-FNegAssert.ll
new file mode 100644
index 000000000..345d4e339
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/2019-08-22-FNegAssert.ll
@@ -0,0 +1,24 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -reassociate -S | FileCheck %s
+; D61675
+
+define float @test1(float %arg, float %arg1, float %arg2, float %arg3) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[TMP5:%.*]] = fneg fast float [[ARG:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub fast float [[ARG3:%.*]], [[ARG2:%.*]]
+; CHECK-NEXT:    [[TMP2_NEG:%.*]] = fadd fast float [[TMP5]], 5.000000e+02
+; CHECK-NEXT:    [[TMP3:%.*]] = fadd fast float [[TMP2_NEG]], [[TMP1]]
+; CHECK-NEXT:    [[TMP4:%.*]] = fmul fast float [[TMP3]], [[TMP3]]
+; CHECK-NEXT:    [[TMP6:%.*]] = fdiv fast float [[TMP5]], [[ARG1:%.*]]
+; CHECK-NEXT:    [[TMP7:%.*]] = fadd fast float [[TMP4]], [[TMP6]]
+; CHECK-NEXT:    ret float [[TMP7]]
+;
+  %tmp1 = fsub fast float %arg3, %arg2
+  %tmp2 = fadd fast float %arg, -5.000000e+02
+  %tmp3 = fsub fast float %tmp1, %tmp2
+  %tmp4 = fmul fast float %tmp3, %tmp3
+  %tmp5 = fneg fast float %arg
+  %tmp6 = fdiv fast float %tmp5, %arg1
+  %tmp7 = fadd fast float %tmp6, %tmp4
+  ret float %tmp7
+}
diff --git a/llvm/test/Transforms/Reassociate/canonicalize-neg-const.ll b/llvm/test/Transforms/Reassociate/canonicalize-neg-const.ll
index 7afc66080..23ef14736 100644
--- a/llvm/test/Transforms/Reassociate/canonicalize-neg-const.ll
+++ b/llvm/test/Transforms/Reassociate/canonicalize-neg-const.ll
@@ -1,11 +1,12 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -reassociate -gvn -S < %s | FileCheck %s
 
 ; (x + 0.1234 * y) * (x + -0.1234 * y) -> (x + 0.1234 * y) * (x - 0.1234 * y)
 define double @test1(double %x, double %y) {
 ; CHECK-LABEL: @test1(
-; CHECK-NEXT:    [[MUL:%.*]] = fmul double %y, 1.234000e-01
-; CHECK-NEXT:    [[ADD:%.*]] = fadd double %x, [[MUL]]
-; CHECK-NEXT:    [[ADD21:%.*]] = fsub double %x, [[MUL]]
+; CHECK-NEXT:    [[MUL:%.*]] = fmul double [[Y:%.*]], 1.234000e-01
+; CHECK-NEXT:    [[ADD:%.*]] = fadd double [[X:%.*]], [[MUL]]
+; CHECK-NEXT:    [[ADD21:%.*]] = fsub double [[X]], [[MUL]]
 ; CHECK-NEXT:    [[MUL3:%.*]] = fmul double [[ADD]], [[ADD21]]
 ; CHECK-NEXT:    ret double [[MUL3]]
 ;
@@ -20,8 +21,8 @@ define double @test1(double %x, double %y) {
 ; (x + -0.1234 * y) * (x + -0.1234 * y) -> (x - 0.1234 * y) * (x - 0.1234 * y)
 define double @test2(double %x, double %y) {
 ; CHECK-LABEL: @test2(
-; CHECK-NEXT:    [[MUL:%.*]] = fmul double %y, 1.234000e-01
-; CHECK-NEXT:    [[ADD1:%.*]] = fsub double %x, [[MUL]]
+; CHECK-NEXT:    [[MUL:%.*]] = fmul double [[Y:%.*]], 1.234000e-01
+; CHECK-NEXT:    [[ADD1:%.*]] = fsub double [[X:%.*]], [[MUL]]
 ; CHECK-NEXT:    [[MUL3:%.*]] = fmul double [[ADD1]], [[ADD1]]
 ; CHECK-NEXT:    ret double [[MUL3]]
 ;
@@ -36,8 +37,8 @@ define double @test2(double %x, double %y) {
 ; (x + 0.1234 * y) * (x - -0.1234 * y) -> (x + 0.1234 * y) * (x + 0.1234 * y)
 define double @test3(double %x, double %y) {
 ; CHECK-LABEL: @test3(
-; CHECK-NEXT:    [[MUL:%.*]] = fmul double %y, 1.234000e-01
-; CHECK-NEXT:    [[ADD:%.*]] = fadd double %x, [[MUL]]
+; CHECK-NEXT:    [[MUL:%.*]] = fmul double [[Y:%.*]], 1.234000e-01
+; CHECK-NEXT:    [[ADD:%.*]] = fadd double [[X:%.*]], [[MUL]]
 ; CHECK-NEXT:    [[MUL3:%.*]] = fmul double [[ADD]], [[ADD]]
 ; CHECK-NEXT:    ret double [[MUL3]]
 ;
@@ -52,8 +53,8 @@ define double @test3(double %x, double %y) {
 ; Canonicalize (x - -0.1234 * y)
 define double @test5(double %x, double %y) {
 ; CHECK-LABEL: @test5(
-; CHECK-NEXT:    [[MUL:%.*]] = fmul double %y, 1.234000e-01
-; CHECK-NEXT:    [[SUB1:%.*]] = fadd double %x, [[MUL]]
+; CHECK-NEXT:    [[MUL:%.*]] = fmul double [[Y:%.*]], 1.234000e-01
+; CHECK-NEXT:    [[SUB1:%.*]] = fadd double [[X:%.*]], [[MUL]]
 ; CHECK-NEXT:    ret double [[SUB1]]
 ;
   %mul = fmul double -1.234000e-01, %y
@@ -64,8 +65,8 @@ define double @test5(double %x, double %y) {
 ; Don't modify (-0.1234 * y - x)
 define double @test6(double %x, double %y) {
 ; CHECK-LABEL: @test6(
-; CHECK-NEXT:    [[MUL:%.*]] = fmul double %y, -1.234000e-01
-; CHECK-NEXT:    [[SUB:%.*]] = fsub double [[MUL]], %x
+; CHECK-NEXT:    [[MUL:%.*]] = fmul double [[Y:%.*]], -1.234000e-01
+; CHECK-NEXT:    [[SUB:%.*]] = fsub double [[MUL]], [[X:%.*]]
 ; CHECK-NEXT:    ret double [[SUB]]
 ;
   %mul = fmul double -1.234000e-01, %y
@@ -76,8 +77,8 @@ define double @test6(double %x, double %y) {
 ; Canonicalize (-0.1234 * y + x) -> (x - 0.1234 * y)
 define double @test7(double %x, double %y) {
 ; CHECK-LABEL: @test7(
-; CHECK-NEXT:    [[MUL:%.*]] = fmul double %y, 1.234000e-01
-; CHECK-NEXT:    [[ADD1:%.*]] = fsub double %x, [[MUL]]
+; CHECK-NEXT:    [[MUL:%.*]] = fmul double [[Y:%.*]], 1.234000e-01
+; CHECK-NEXT:    [[ADD1:%.*]] = fsub double [[X:%.*]], [[MUL]]
 ; CHECK-NEXT:    ret double [[ADD1]]
 ;
   %mul = fmul double -1.234000e-01, %y
@@ -88,8 +89,8 @@ define double @test7(double %x, double %y) {
 ; Canonicalize (y * -0.1234 + x) -> (x - 0.1234 * y)
 define double @test8(double %x, double %y) {
 ; CHECK-LABEL: @test8(
-; CHECK-NEXT:    [[MUL:%.*]] = fmul double %y, 1.234000e-01
-; CHECK-NEXT:    [[ADD1:%.*]] = fsub double %x, [[MUL]]
+; CHECK-NEXT:    [[MUL:%.*]] = fmul double [[Y:%.*]], 1.234000e-01
+; CHECK-NEXT:    [[ADD1:%.*]] = fsub double [[X:%.*]], [[MUL]]
 ; CHECK-NEXT:    ret double [[ADD1]]
 ;
   %mul = fmul double %y, -1.234000e-01
@@ -100,8 +101,8 @@ define double @test8(double %x, double %y) {
 ; Canonicalize (x - -0.1234 / y)
 define double @test9(double %x, double %y) {
 ; CHECK-LABEL: @test9(
-; CHECK-NEXT:    [[DIV:%.*]] = fdiv double 1.234000e-01, %y
-; CHECK-NEXT:    [[SUB1:%.*]] = fadd double %x, [[DIV]]
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv double 1.234000e-01, [[Y:%.*]]
+; CHECK-NEXT:    [[SUB1:%.*]] = fadd double [[X:%.*]], [[DIV]]
 ; CHECK-NEXT:    ret double [[SUB1]]
 ;
   %div = fdiv double -1.234000e-01, %y
@@ -112,8 +113,8 @@ define double @test9(double %x, double %y) {
 ; Don't modify (-0.1234 / y - x)
 define double @test10(double %x, double %y) {
 ; CHECK-LABEL: @test10(
-; CHECK-NEXT:    [[DIV:%.*]] = fdiv double -1.234000e-01, %y
-; CHECK-NEXT:    [[SUB:%.*]] = fsub double [[DIV]], %x
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv double -1.234000e-01, [[Y:%.*]]
+; CHECK-NEXT:    [[SUB:%.*]] = fsub double [[DIV]], [[X:%.*]]
 ; CHECK-NEXT:    ret double [[SUB]]
 ;
   %div = fdiv double -1.234000e-01, %y
@@ -124,8 +125,8 @@ define double @test10(double %x, double %y) {
 ; Canonicalize (-0.1234 / y + x) -> (x - 0.1234 / y)
 define double @test11(double %x, double %y) {
 ; CHECK-LABEL: @test11(
-; CHECK-NEXT:    [[DIV:%.*]] = fdiv double 1.234000e-01, %y
-; CHECK-NEXT:    [[ADD1:%.*]] = fsub double %x, [[DIV]]
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv double 1.234000e-01, [[Y:%.*]]
+; CHECK-NEXT:    [[ADD1:%.*]] = fsub double [[X:%.*]], [[DIV]]
 ; CHECK-NEXT:    ret double [[ADD1]]
 ;
   %div = fdiv double -1.234000e-01, %y
@@ -136,8 +137,8 @@ define double @test11(double %x, double %y) {
 ; Canonicalize (y / -0.1234 + x) -> (x - y / 0.1234)
 define double @test12(double %x, double %y) {
 ; CHECK-LABEL: @test12(
-; CHECK-NEXT:    [[DIV:%.*]] = fdiv double %y, 1.234000e-01
-; CHECK-NEXT:    [[ADD1:%.*]] = fsub double %x, [[DIV]]
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv double [[Y:%.*]], 1.234000e-01
+; CHECK-NEXT:    [[ADD1:%.*]] = fsub double [[X:%.*]], [[DIV]]
 ; CHECK-NEXT:    ret double [[ADD1]]
 ;
   %div = fdiv double %y, -1.234000e-01
@@ -148,7 +149,7 @@ define double @test12(double %x, double %y) {
 ; Don't create an NSW violation
 define i4 @test13(i4 %x) {
 ; CHECK-LABEL: @test13(
-; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i4 %x, -2
+; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i4 [[X:%.*]], -2
 ; CHECK-NEXT:    [[ADD:%.*]] = add i4 [[MUL]], 3
 ; CHECK-NEXT:    ret i4 [[ADD]]
 ;
@@ -169,8 +170,8 @@ define i4 @test13(i4 %x) {
 
 define double @pr34078(double %A) {
 ; CHECK-LABEL: @pr34078(
-; CHECK-NEXT:    [[SUB:%.*]] = fsub fast double 1.000000e+00, %A
-; CHECK-NEXT:    [[POW2:%.*]] = fmul double %A, %A
+; CHECK-NEXT:    [[SUB:%.*]] = fsub fast double 1.000000e+00, [[A:%.*]]
+; CHECK-NEXT:    [[POW2:%.*]] = fmul double [[A]], [[A]]
 ; CHECK-NEXT:    [[MUL5_NEG:%.*]] = fmul fast double [[POW2]], -5.000000e-01
 ; CHECK-NEXT:    [[SUB1:%.*]] = fadd fast double [[MUL5_NEG]], [[SUB]]
 ; CHECK-NEXT:    [[FACTOR:%.*]] = fmul fast double [[SUB1]], 2.000000e+00
@@ -183,3 +184,335 @@ define double @pr34078(double %A) {
   %add = fadd fast double %sub1, %sub1
   ret double %add
 }
+
+define double @fadd_fmul_neg_const1(double %a, double %b, double %c) {
+; CHECK-LABEL: @fadd_fmul_neg_const1(
+; CHECK-NEXT:    [[MUL0:%.*]] = fmul double [[B:%.*]], 3.000000e+00
+; CHECK-NEXT:    [[MUL1:%.*]] = fmul double [[MUL0]], [[C:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub double [[A:%.*]], [[MUL1]]
+; CHECK-NEXT:    ret double [[TMP1]]
+;
+  %mul0 = fmul double %b, -3.0
+  %mul1 = fmul double %mul0, %c
+  %add = fadd double %mul1, %a
+  ret double %add
+}
+
+define double @fadd_fmul_neg_const2(double %a, double %b, double %c) {
+; CHECK-LABEL: @fadd_fmul_neg_const2(
+; CHECK-NEXT:    [[MUL0:%.*]] = fmul double [[A:%.*]], 3.000000e+00
+; CHECK-NEXT:    [[MUL1:%.*]] = fmul double [[MUL0]], [[B:%.*]]
+; CHECK-NEXT:    [[MUL2:%.*]] = fmul double [[MUL1]], 4.000000e+00
+; CHECK-NEXT:    [[ADD:%.*]] = fadd double [[A]], [[MUL2]]
+; CHECK-NEXT:    ret double [[ADD]]
+;
+  %mul0 = fmul double %a, -3.0
+  %mul1 = fmul double %mul0, %b
+  %mul2 = fmul double %mul1, -4.0
+  %add = fadd double %mul2, %a
+  ret double %add
+}
+
+define double @fadd_fmul_neg_const3(double %a, double %b, double %c) {
+; CHECK-LABEL: @fadd_fmul_neg_const3(
+; CHECK-NEXT:    [[MUL0:%.*]] = fmul double [[A:%.*]], 3.000000e+00
+; CHECK-NEXT:    [[MUL1:%.*]] = fmul double [[MUL0]], [[B:%.*]]
+; CHECK-NEXT:    [[MUL2:%.*]] = fmul double [[MUL1]], 4.000000e+00
+; CHECK-NEXT:    [[MUL3:%.*]] = fmul double [[MUL2]], 5.000000e+00
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub double [[C:%.*]], [[MUL3]]
+; CHECK-NEXT:    ret double [[TMP1]]
+;
+  %mul0 = fmul double %a, -3.0
+  %mul1 = fmul double %mul0, %b
+  %mul2 = fmul double %mul1, -4.0
+  %mul3 = fmul double %mul2, -5.0
+  %add = fadd double %mul3, %c
+  ret double %add
+}
+
+define double @fsub_fmul_neg_const1(double %a, double %b, double %c) {
+; CHECK-LABEL: @fsub_fmul_neg_const1(
+; CHECK-NEXT:    [[MUL0:%.*]] = fmul double [[B:%.*]], 3.000000e+00
+; CHECK-NEXT:    [[MUL1:%.*]] = fmul double [[MUL0]], [[C:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fadd double [[A:%.*]], [[MUL1]]
+; CHECK-NEXT:    ret double [[TMP1]]
+;
+  %mul0 = fmul double %b, -3.0
+  %mul1 = fmul double %mul0, %c
+  %sub = fsub double %a, %mul1
+  ret double %sub
+}
+
+define double @fsub_fmul_neg_const2(double %a, double %b, double %c) {
+; CHECK-LABEL: @fsub_fmul_neg_const2(
+; CHECK-NEXT:    [[MUL0:%.*]] = fmul double [[A:%.*]], 3.000000e+00
+; CHECK-NEXT:    [[MUL1:%.*]] = fmul double [[MUL0]], [[B:%.*]]
+; CHECK-NEXT:    [[MUL2:%.*]] = fmul double [[MUL1]], 4.000000e+00
+; CHECK-NEXT:    [[TMP1:%.*]] = fadd double [[A]], [[MUL2]]
+; CHECK-NEXT:    ret double [[TMP1]]
+;
+  %mul0 = fmul double %a, -3.0
+  %mul1 = fmul double %mul0, %b
+  %mul2 = fmul double %mul1, 4.0
+  %sub = fsub double %a, %mul2
+  ret double %sub
+}
+
+define double @fsub_fmul_neg_const3(double %a, double %b, double %c) {
+; CHECK-LABEL: @fsub_fmul_neg_const3(
+; CHECK-NEXT:    [[MUL0:%.*]] = fmul double [[A:%.*]], 3.000000e+00
+; CHECK-NEXT:    [[MUL1:%.*]] = fmul double [[MUL0]], [[B:%.*]]
+; CHECK-NEXT:    [[MUL2:%.*]] = fmul double [[MUL1]], 4.000000e+00
+; CHECK-NEXT:    [[MUL3:%.*]] = fmul double [[MUL2]], 5.000000e+00
+; CHECK-NEXT:    [[SUB:%.*]] = fsub double [[C:%.*]], [[MUL3]]
+; CHECK-NEXT:    ret double [[SUB]]
+;
+  %mul0 = fmul double %a, 3.0
+  %mul1 = fmul double %mul0, %b
+  %mul2 = fmul double %mul1, -4.0
+  %mul3 = fmul double %mul2, -5.0
+  %sub = fsub double %c, %mul3
+  ret double %sub
+}
+
+define double @fadd_fdiv_neg_const1(double %a, double %b, double %c) {
+; CHECK-LABEL: @fadd_fdiv_neg_const1(
+; CHECK-NEXT:    [[DIV0:%.*]] = fdiv double [[B:%.*]], 3.000000e+00
+; CHECK-NEXT:    [[DIV1:%.*]] = fdiv double [[DIV0]], [[C:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub double [[A:%.*]], [[DIV1]]
+; CHECK-NEXT:    ret double [[TMP1]]
+;
+  %div0 = fdiv double %b, -3.0
+  %div1 = fdiv double %div0, %c
+  %add = fadd double %div1, %a
+  ret double %add
+}
+
+define double @fadd_fdiv_neg_const2(double %a, double %b, double %c) {
+; CHECK-LABEL: @fadd_fdiv_neg_const2(
+; CHECK-NEXT:    [[DIV0:%.*]] = fdiv double 3.000000e+00, [[A:%.*]]
+; CHECK-NEXT:    [[DIV1:%.*]] = fdiv double [[DIV0]], [[B:%.*]]
+; CHECK-NEXT:    [[DIV2:%.*]] = fdiv double [[DIV1]], 7.000000e+00
+; CHECK-NEXT:    [[ADD:%.*]] = fadd double [[A]], [[DIV2]]
+; CHECK-NEXT:    ret double [[ADD]]
+;
+  %div0 = fdiv double -3.0, %a
+  %div1 = fdiv double %div0, %b
+  %div2 = fdiv double %div1, -7.0
+  %add = fadd double %div2, %a
+  ret double %add
+}
+
+define double @fadd_fdiv_neg_const3(double %a, double %b, double %c) {
+; CHECK-LABEL: @fadd_fdiv_neg_const3(
+; CHECK-NEXT:    [[DIV0:%.*]] = fdiv double [[A:%.*]], 3.000000e+00
+; CHECK-NEXT:    [[DIV1:%.*]] = fdiv double [[DIV0]], [[B:%.*]]
+; CHECK-NEXT:    [[DIV2:%.*]] = fdiv double 4.000000e+00, [[DIV1]]
+; CHECK-NEXT:    [[DIV3:%.*]] = fdiv double [[DIV2]], 5.000000e+00
+; CHECK-NEXT:    [[ADD:%.*]] = fadd double [[C:%.*]], [[DIV3]]
+; CHECK-NEXT:    ret double [[ADD]]
+;
+  %div0 = fdiv double %a, -3.0
+  %div1 = fdiv double %div0, %b
+  %div2 = fdiv double -4.0, %div1
+  %div3 = fdiv double %div2, 5.0
+  %add = fadd double %div3, %c
+  ret double %add
+}
+
+define double @fsub_fdiv_neg_const1(double %a, double %b, double %c) {
+; CHECK-LABEL: @fsub_fdiv_neg_const1(
+; CHECK-NEXT:    [[DIV0:%.*]] = fdiv double [[B:%.*]], 3.000000e+00
+; CHECK-NEXT:    [[DIV1:%.*]] = fdiv double [[DIV0]], [[C:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fadd double [[A:%.*]], [[DIV1]]
+; CHECK-NEXT:    ret double [[TMP1]]
+;
+  %div0 = fdiv double %b, -3.0
+  %div1 = fdiv double %div0, %c
+  %sub = fsub double %a, %div1
+  ret double %sub
+}
+
+define double @fsub_fdiv_neg_const2(double %a, double %b, double %c) {
+; CHECK-LABEL: @fsub_fdiv_neg_const2(
+; CHECK-NEXT:    [[DIV0:%.*]] = fdiv double 3.000000e+00, [[A:%.*]]
+; CHECK-NEXT:    [[DIV1:%.*]] = fdiv double [[DIV0]], [[B:%.*]]
+; CHECK-NEXT:    [[DIV2:%.*]] = fdiv double [[DIV1]], 7.000000e+00
+; CHECK-NEXT:    [[TMP1:%.*]] = fadd double [[A]], [[DIV2]]
+; CHECK-NEXT:    ret double [[TMP1]]
+;
+  %div0 = fdiv double -3.0, %a
+  %div1 = fdiv double %div0, %b
+  %div2 = fdiv double %div1, 7.0
+  %sub = fsub double %a, %div2
+  ret double %sub
+}
+
+define double @fsub_fdiv_neg_const3(double %a, double %b, double %c) {
+; CHECK-LABEL: @fsub_fdiv_neg_const3(
+; CHECK-NEXT:    [[DIV0:%.*]] = fdiv double 3.000000e+00, [[A:%.*]]
+; CHECK-NEXT:    [[DIV1:%.*]] = fdiv double [[DIV0]], [[B:%.*]]
+; CHECK-NEXT:    [[DIV2:%.*]] = fdiv double [[DIV1]], 7.000000e+00
+; CHECK-NEXT:    [[DIV3:%.*]] = fdiv double 5.000000e+00, [[DIV2]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fadd double [[C:%.*]], [[DIV3]]
+; CHECK-NEXT:    ret double [[TMP1]]
+;
+  %div0 = fdiv double -3.0, %a
+  %div1 = fdiv double %div0, %b
+  %div2 = fdiv double %div1, -7.0
+  %div3 = fdiv double -5.0, %div2
+  %sub = fsub double %c, %div3
+  ret double %sub
+}
+
+define double @fadd_mix_neg_const1(double %a, double %b, double %c) {
+; CHECK-LABEL: @fadd_mix_neg_const1(
+; CHECK-NEXT:    [[MUL0:%.*]] = fmul double [[B:%.*]], 3.000000e+00
+; CHECK-NEXT:    [[DIV1:%.*]] = fdiv double [[MUL0]], [[C:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub double [[A:%.*]], [[DIV1]]
+; CHECK-NEXT:    ret double [[TMP1]]
+;
+  %mul0 = fmul double %b, -3.0
+  %div1 = fdiv double %mul0, %c
+  %add = fadd double %div1, %a
+  ret double %add
+}
+
+define double @fadd_mix_neg_const2(double %a, double %b, double %c) {
+; CHECK-LABEL: @fadd_mix_neg_const2(
+; CHECK-NEXT:    [[DIV0:%.*]] = fdiv double 3.000000e+00, [[A:%.*]]
+; CHECK-NEXT:    [[MUL1:%.*]] = fmul double [[DIV0]], [[B:%.*]]
+; CHECK-NEXT:    [[DIV2:%.*]] = fdiv double [[MUL1]], 5.000000e+00
+; CHECK-NEXT:    [[ADD:%.*]] = fadd double [[A]], [[DIV2]]
+; CHECK-NEXT:    ret double [[ADD]]
+;
+  %div0 = fdiv double -3.0, %a
+  %mul1 = fmul double %div0, %b
+  %div2 = fdiv double %mul1, -5.0
+  %add = fadd double %div2, %a
+  ret double %add
+}
+
+define double @fadd_mix_neg_const3(double %a, double %b, double %c) {
+; CHECK-LABEL: @fadd_mix_neg_const3(
+; CHECK-NEXT:    [[MUL0:%.*]] = fmul double [[A:%.*]], 3.000000e+00
+; CHECK-NEXT:    [[DIV1:%.*]] = fdiv double [[MUL0]], [[B:%.*]]
+; CHECK-NEXT:    [[MUL2:%.*]] = fmul double [[DIV1]], 4.000000e+00
+; CHECK-NEXT:    [[DIV3:%.*]] = fdiv double [[MUL2]], 5.000000e+00
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub double [[C:%.*]], [[DIV3]]
+; CHECK-NEXT:    ret double [[TMP1]]
+;
+  %mul0 = fmul double %a, -3.0
+  %div1 = fdiv double %mul0, %b
+  %mul2 = fmul double -4.0, %div1
+  %div3 = fdiv double %mul2, -5.0
+  %add = fadd double %div3, %c
+  ret double %add
+}
+
+define double @fsub_mix_neg_const1(double %a, double %b, double %c) {
+; CHECK-LABEL: @fsub_mix_neg_const1(
+; CHECK-NEXT:    [[DIV0:%.*]] = fdiv double [[B:%.*]], 3.000000e+00
+; CHECK-NEXT:    [[MUL1:%.*]] = fmul double [[DIV0]], [[C:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fadd double [[A:%.*]], [[MUL1]]
+; CHECK-NEXT:    ret double [[TMP1]]
+;
+  %div0 = fdiv double %b, -3.0
+  %mul1 = fmul double %div0, %c
+  %sub = fsub double %a, %mul1
+  ret double %sub
+}
+define double @fsub_mix_neg_const2(double %a, double %b, double %c) {
+; CHECK-LABEL: @fsub_mix_neg_const2(
+; CHECK-NEXT:    [[MUL0:%.*]] = fmul double [[A:%.*]], 3.000000e+00
+; CHECK-NEXT:    [[DIV1:%.*]] = fdiv double [[MUL0]], [[B:%.*]]
+; CHECK-NEXT:    [[MUL2:%.*]] = fmul double [[DIV1]], 5.000000e+00
+; CHECK-NEXT:    [[SUB:%.*]] = fsub double [[A]], [[MUL2]]
+; CHECK-NEXT:    ret double [[SUB]]
+;
+  %mul0 = fmul double -3.0, %a
+  %div1 = fdiv double %mul0, %b
+  %mul2 = fmul double %div1, -5.0
+  %sub = fsub double %a, %mul2
+  ret double %sub
+}
+
+define double @fsub_mix_neg_const3(double %a, double %b, double %c) {
+; CHECK-LABEL: @fsub_mix_neg_const3(
+; CHECK-NEXT:    [[DIV0:%.*]] = fdiv double 3.000000e+00, [[A:%.*]]
+; CHECK-NEXT:    [[MUL1:%.*]] = fmul double [[DIV0]], [[B:%.*]]
+; CHECK-NEXT:    [[DIV2:%.*]] = fdiv double [[MUL1]], 7.000000e+00
+; CHECK-NEXT:    [[MUL3:%.*]] = fmul double [[DIV2]], 5.000000e+00
+; CHECK-NEXT:    [[TMP1:%.*]] = fadd double [[C:%.*]], [[MUL3]]
+; CHECK-NEXT:    ret double [[TMP1]]
+;
+  %div0 = fdiv double -3.0, %a
+  %mul1 = fmul double %div0, %b
+  %div2 = fdiv double %mul1, -7.0
+  %mul3 = fmul double -5.0, %div2
+  %sub = fsub double %c, %mul3
+  ret double %sub
+}
+
+define double @fadd_both_ops_mix_neg_const1(double %a, double %b, double %c) {
+; CHECK-LABEL: @fadd_both_ops_mix_neg_const1(
+; CHECK-NEXT:    [[MUL0:%.*]] = fmul double [[B:%.*]], -3.000000e+00
+; CHECK-NEXT:    [[DIV1:%.*]] = fdiv double [[MUL0]], [[C:%.*]]
+; CHECK-NEXT:    [[MUL2:%.*]] = fmul double [[A:%.*]], 4.000000e+00
+; CHECK-NEXT:    [[DIV3:%.*]] = fdiv double [[MUL2]], [[C]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub double [[DIV1]], [[DIV3]]
+; CHECK-NEXT:    ret double [[TMP1]]
+;
+  %mul0 = fmul double %b, -3.0
+  %div1 = fdiv double %mul0, %c
+  %mul2 = fmul double %a, -4.0
+  %div3 = fdiv double %mul2, %c
+  %add = fadd double %div1, %div3
+  ret double %add
+}
+
+define double @fadd_both_ops_mix_neg_const2(double %a, double %b, double %c) {
+; CHECK-LABEL: @fadd_both_ops_mix_neg_const2(
+; CHECK-NEXT:    [[DIV0:%.*]] = fdiv double 3.000000e+00, [[A:%.*]]
+; CHECK-NEXT:    [[MUL1:%.*]] = fmul double [[DIV0]], [[B:%.*]]
+; CHECK-NEXT:    [[DIV2:%.*]] = fdiv double [[MUL1]], 7.000000e+00
+; CHECK-NEXT:    [[DIV3:%.*]] = fdiv double 5.000000e+00, [[C:%.*]]
+; CHECK-NEXT:    [[MUL4:%.*]] = fmul double [[B]], [[DIV3]]
+; CHECK-NEXT:    [[DIV5:%.*]] = fdiv double [[MUL4]], 6.000000e+00
+; CHECK-NEXT:    [[ADD:%.*]] = fadd double [[DIV2]], [[DIV5]]
+; CHECK-NEXT:    ret double [[ADD]]
+;
+  %div0 = fdiv double -3.0, %a
+  %mul1 = fmul double %div0, %b
+  %div2 = fdiv double %mul1, -7.0
+  %div3 = fdiv double -5.0, %c
+  %mul4 = fmul double %div3, %b
+  %div5 = fdiv double %mul4, -6.0
+  %add = fadd double %div2, %div5
+  ret double %add
+}
+
+define double @fadd_both_opsmix_neg_const3(double %a, double %b, double %c) {
+; CHECK-LABEL: @fadd_both_opsmix_neg_const3(
+; CHECK-NEXT:    [[MUL0:%.*]] = fmul double [[A:%.*]], -3.000000e+00
+; CHECK-NEXT:    [[DIV1:%.*]] = fdiv double [[MUL0]], [[B:%.*]]
+; CHECK-NEXT:    [[MUL2:%.*]] = fmul double [[DIV1]], -4.000000e+00
+; CHECK-NEXT:    [[DIV3:%.*]] = fdiv double [[MUL2]], -5.000000e+00
+; CHECK-NEXT:    [[MUL4:%.*]] = fmul double [[C:%.*]], 6.000000e+00
+; CHECK-NEXT:    [[DIV5:%.*]] = fdiv double [[MUL4]], [[B]]
+; CHECK-NEXT:    [[MUL6:%.*]] = fmul double [[DIV5]], 7.000000e+00
+; CHECK-NEXT:    [[MUL7:%.*]] = fdiv double [[MUL6]], 9.000000e+00
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub double [[DIV3]], [[MUL7]]
+; CHECK-NEXT:    ret double [[TMP1]]
+;
+  %mul0 = fmul double %a, -3.0
+  %div1 = fdiv double %mul0, %b
+  %mul2 = fmul double -4.0, %div1
+  %div3 = fdiv double %mul2, -5.0
+  %mul4 = fmul double %c, -6.0
+  %div5 = fdiv double %mul4, %b
+  %mul6 = fmul double -7.0, %div5
+  %mul7 = fdiv double %mul6, -9.0
+  %add = fadd double %div3, %mul7
+  ret double %add
+}
diff --git a/llvm/test/Transforms/Reassociate/commute.ll b/llvm/test/Transforms/Reassociate/commute.ll
index 760e51b05..6009159d7 100644
--- a/llvm/test/Transforms/Reassociate/commute.ll
+++ b/llvm/test/Transforms/Reassociate/commute.ll
@@ -1,14 +1,17 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -reassociate -S < %s | FileCheck %s
 
 declare void @use(i32)
 
 define void @test1(i32 %x, i32 %y) {
-; CHECK-LABEL: test1
-; CHECK: mul i32 %y, %x
-; CHECK: mul i32 %y, %x
-; CHECK: sub i32 %1, %2
-; CHECK: call void @use(i32 %{{.*}})
-; CHECK: call void @use(i32 %{{.*}})
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i32 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = mul i32 [[Y]], [[X]]
+; CHECK-NEXT:    [[TMP3:%.*]] = sub i32 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    call void @use(i32 [[TMP1]])
+; CHECK-NEXT:    call void @use(i32 [[TMP3]])
+; CHECK-NEXT:    ret void
+;
 
   %1 = mul i32 %x, %y
   %2 = mul i32 %y, %x
diff --git a/llvm/test/Transforms/Reassociate/fast-ReassociateVector.ll b/llvm/test/Transforms/Reassociate/fast-ReassociateVector.ll
index a3d8f32c1..339884b1a 100644
--- a/llvm/test/Transforms/Reassociate/fast-ReassociateVector.ll
+++ b/llvm/test/Transforms/Reassociate/fast-ReassociateVector.ll
@@ -282,9 +282,10 @@ define <2 x double> @test9_reassoc_unary_fneg(<2 x double> %b, <2 x double> %a)
 define <2 x float> @test10(<2 x float> %a, <2 x float> %b, <2 x float> %z) {
 ; CHECK-LABEL: @test10(
 ; CHECK-NEXT:    [[TMP1:%.*]] = fsub fast <2 x float> zeroinitializer, zeroinitializer
-; CHECK-NEXT:    [[E:%.*]] = fmul fast <2 x float> [[A:%.*]], <float 4.000000e+01, float 4.000000e+01>
-; CHECK-NEXT:    [[F:%.*]] = fmul fast <2 x float> [[E]], [[Z:%.*]]
-; CHECK-NEXT:    ret <2 x float> [[F]]
+; CHECK-NEXT:    [[C:%.*]] = fmul fast <2 x float> [[A:%.*]], <float 4.000000e+01, float 4.000000e+01>
+; CHECK-NEXT:    [[E:%.*]] = fmul fast <2 x float> [[C]], [[Z:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fadd fast <2 x float> [[E]], zeroinitializer
+; CHECK-NEXT:    ret <2 x float> [[TMP2]]
 ;
   %d = fmul fast <2 x float> %z, <float 4.000000e+01, float 4.000000e+01>
   %c = fsub fast <2 x float> <float 0.000000e+00, float 0.000000e+00>, %d
@@ -343,7 +344,7 @@ define <2 x float> @test10_reassoc_unary_fneg(<2 x float> %a, <2 x float> %b, <2
 
 define <2 x double> @test11(<2 x double> %x, <2 x double> %y) {
 ; CHECK-LABEL: @test11(
-; CHECK-NEXT:    [[FACTOR:%.*]] = fmul fast <2 x double> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[FACTOR:%.*]] = fmul fast <2 x double> [[Y:%.*]], [[X:%.*]]
 ; CHECK-NEXT:    [[REASS_MUL:%.*]] = fmul fast <2 x double> [[FACTOR]], <double 2.000000e+00, double 2.000000e+00>
 ; CHECK-NEXT:    ret <2 x double> [[REASS_MUL]]
 ;
@@ -381,15 +382,13 @@ define <2 x i64> @test12(<2 x i64> %b, <2 x i64> %c) {
   ret <2 x i64> %shl
 }
 
-; FIXME: expressions with a negative const should be canonicalized to assist
-; further reassociation.
-; We would expect (-5*b)+a -> a-(5*b) but only the constant operand is commuted.
+; (-5*b)+a -> a-(5*b)
 
 define <4 x float> @test13(<4 x float> %a, <4 x float> %b) {
 ; CHECK-LABEL: @test13(
-; CHECK-NEXT:    [[MUL:%.*]] = fmul fast <4 x float> [[B:%.*]], <float -5.000000e+00, float -5.000000e+00, float -5.000000e+00, float -5.000000e+00>
-; CHECK-NEXT:    [[ADD:%.*]] = fadd fast <4 x float> [[MUL]], [[A:%.*]]
-; CHECK-NEXT:    ret <4 x float> [[ADD]]
+; CHECK-NEXT:    [[MUL:%.*]] = fmul fast <4 x float> [[B:%.*]], <float 5.000000e+00, float 5.000000e+00, float 5.000000e+00, float 5.000000e+00>
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub fast <4 x float> [[A:%.*]], [[MUL]]
+; CHECK-NEXT:    ret <4 x float> [[TMP1]]
 ;
   %mul = fmul fast <4 x float> <float -5.000000e+00, float -5.000000e+00, float -5.000000e+00, float -5.000000e+00>, %b
   %add = fadd fast <4 x float> %mul, %a
diff --git a/llvm/test/Transforms/Reassociate/fast-SubReassociate.ll b/llvm/test/Transforms/Reassociate/fast-SubReassociate.ll
index 9b2b55746..35c8dd416 100644
--- a/llvm/test/Transforms/Reassociate/fast-SubReassociate.ll
+++ b/llvm/test/Transforms/Reassociate/fast-SubReassociate.ll
@@ -1,9 +1,10 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -reassociate -constprop -instcombine -S | FileCheck %s
 
 define float @test1(float %A, float %B) {
 ; CHECK-LABEL: @test1(
-; CHECK-NEXT:    [[W:%.*]] = fadd float %B, 5.000000e+00
-; CHECK-NEXT:    [[X:%.*]] = fadd float %A, -7.000000e+00
+; CHECK-NEXT:    [[W:%.*]] = fadd float [[B:%.*]], 5.000000e+00
+; CHECK-NEXT:    [[X:%.*]] = fadd float [[A:%.*]], -7.000000e+00
 ; CHECK-NEXT:    [[Y:%.*]] = fsub float [[X]], [[W]]
 ; CHECK-NEXT:    [[Z:%.*]] = fadd float [[Y]], 1.200000e+01
 ; CHECK-NEXT:    ret float [[Z]]
@@ -18,7 +19,7 @@ define float @test1(float %A, float %B) {
 ; With sub reassociation, constant folding can eliminate all of the constants.
 define float @test2(float %A, float %B) {
 ; CHECK-LABEL: @test2(
-; CHECK-NEXT:    [[Z:%.*]] = fsub fast float %A, %B
+; CHECK-NEXT:    [[Z:%.*]] = fsub fast float [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    ret float [[Z]]
 ;
   %W = fadd fast float %B, 5.000000e+00
@@ -32,8 +33,8 @@ define float @test2(float %A, float %B) {
 ; Both 'reassoc' and 'nsz' are required.
 define float @test2_minimal(float %A, float %B) {
 ; CHECK-LABEL: @test2_minimal(
-; CHECK-NEXT:    [[Z:%.*]] = fsub reassoc nsz float %A, %B
-; CHECK-NEXT:    ret float [[Z]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub reassoc nsz float [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret float [[TMP1]]
 ;
   %W = fadd reassoc nsz float %B, 5.000000e+00
   %X = fadd reassoc nsz float %A, -7.000000e+00
@@ -45,8 +46,8 @@ define float @test2_minimal(float %A, float %B) {
 ; Verify the fold is not done with only 'reassoc' ('nsz' is required).
 define float @test2_reassoc(float %A, float %B) {
 ; CHECK-LABEL: @test2_reassoc(
-; CHECK-NEXT:    [[W:%.*]] = fadd reassoc float %B, 5.000000e+00
-; CHECK-NEXT:    [[X:%.*]] = fadd reassoc float %A, -7.000000e+00
+; CHECK-NEXT:    [[W:%.*]] = fadd reassoc float [[B:%.*]], 5.000000e+00
+; CHECK-NEXT:    [[X:%.*]] = fadd reassoc float [[A:%.*]], -7.000000e+00
 ; CHECK-NEXT:    [[Y:%.*]] = fsub reassoc float [[X]], [[W]]
 ; CHECK-NEXT:    [[Z:%.*]] = fadd reassoc float [[Y]], 1.200000e+01
 ; CHECK-NEXT:    ret float [[Z]]
@@ -60,10 +61,10 @@ define float @test2_reassoc(float %A, float %B) {
 
 define float @test3(float %A, float %B, float %C, float %D) {
 ; CHECK-LABEL: @test3(
-; CHECK-NEXT:    [[M:%.*]] = fadd float %A, 1.200000e+01
-; CHECK-NEXT:    [[N:%.*]] = fadd float [[M]], %B
-; CHECK-NEXT:    [[O:%.*]] = fadd float [[N]], %C
-; CHECK-NEXT:    [[P:%.*]] = fsub float %D, [[O]]
+; CHECK-NEXT:    [[M:%.*]] = fadd float [[A:%.*]], 1.200000e+01
+; CHECK-NEXT:    [[N:%.*]] = fadd float [[M]], [[B:%.*]]
+; CHECK-NEXT:    [[O:%.*]] = fadd float [[N]], [[C:%.*]]
+; CHECK-NEXT:    [[P:%.*]] = fsub float [[D:%.*]], [[O]]
 ; CHECK-NEXT:    [[Q:%.*]] = fadd float [[P]], 1.200000e+01
 ; CHECK-NEXT:    ret float [[Q]]
 ;
@@ -84,10 +85,10 @@ define float @test4(float %A, float %B, float %C, float %D) {
 ; %Q = fsub fast float %D, %sum1
 ; ret i32 %Q
 ; CHECK-LABEL: @test4(
-; CHECK-NEXT:    [[B_NEG:%.*]] = fsub fast float -0.000000e+00, %B
-; CHECK-NEXT:    [[O_NEG:%.*]] = fsub fast float [[B_NEG]], %A
-; CHECK-NEXT:    [[P:%.*]] = fsub fast float [[O_NEG]], %C
-; CHECK-NEXT:    [[Q:%.*]] = fadd fast float [[P]], %D
+; CHECK-NEXT:    [[B_NEG:%.*]] = fsub fast float -0.000000e+00, [[B:%.*]]
+; CHECK-NEXT:    [[O_NEG:%.*]] = fsub fast float [[B_NEG]], [[A:%.*]]
+; CHECK-NEXT:    [[P:%.*]] = fsub fast float [[O_NEG]], [[C:%.*]]
+; CHECK-NEXT:    [[Q:%.*]] = fadd fast float [[P]], [[D:%.*]]
 ; CHECK-NEXT:    ret float [[Q]]
 ;
   %M = fadd fast float 1.200000e+01, %A
@@ -102,10 +103,10 @@ define float @test4(float %A, float %B, float %C, float %D) {
 
 define float @test4_reassoc(float %A, float %B, float %C, float %D) {
 ; CHECK-LABEL: @test4_reassoc(
-; CHECK-NEXT:    [[M:%.*]] = fadd reassoc float %A, 1.200000e+01
-; CHECK-NEXT:    [[N:%.*]] = fadd reassoc float [[M]], %B
-; CHECK-NEXT:    [[O:%.*]] = fadd reassoc float [[N]], %C
-; CHECK-NEXT:    [[P:%.*]] = fsub reassoc float %D, [[O]]
+; CHECK-NEXT:    [[M:%.*]] = fadd reassoc float [[A:%.*]], 1.200000e+01
+; CHECK-NEXT:    [[N:%.*]] = fadd reassoc float [[M]], [[B:%.*]]
+; CHECK-NEXT:    [[O:%.*]] = fadd reassoc float [[N]], [[C:%.*]]
+; CHECK-NEXT:    [[P:%.*]] = fsub reassoc float [[D:%.*]], [[O]]
 ; CHECK-NEXT:    [[Q:%.*]] = fadd reassoc float [[P]], 1.200000e+01
 ; CHECK-NEXT:    ret float [[Q]]
 ;
diff --git a/llvm/test/Transforms/Reassociate/fast-basictest.ll b/llvm/test/Transforms/Reassociate/fast-basictest.ll
index a01b805a6..62b13f41b 100644
--- a/llvm/test/Transforms/Reassociate/fast-basictest.ll
+++ b/llvm/test/Transforms/Reassociate/fast-basictest.ll
@@ -614,12 +614,25 @@ define float @test18_reassoc(float %a, float %b, float %z) {
   ret float %f
 }
 
-; It is not safe to reassociate unary fneg without nnan.
+; fneg of fneg is an identity operation, so no FMF are needed to remove those instructions.
+
+define float @test18_unary_fneg_no_FMF(float %a, float %b, float %z) {
+; CHECK-LABEL: @test18_unary_fneg_no_FMF(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul float [[Z:%.*]], 4.000000e+01
+; CHECK-NEXT:    [[F:%.*]] = fmul float [[TMP1]], [[A:%.*]]
+; CHECK-NEXT:    ret float [[F]]
+;
+  %d = fmul float %z, 4.000000e+01
+  %c = fneg float %d
+  %e = fmul float %a, %c
+  %f = fneg float %e
+  ret float %f
+}
+
 define float @test18_reassoc_unary_fneg(float %a, float %b, float %z) {
 ; CHECK-LABEL: @test18_reassoc_unary_fneg(
-; CHECK-NEXT:    [[C:%.*]] = fmul reassoc float [[Z:%.*]], -4.000000e+01
-; CHECK-NEXT:    [[E:%.*]] = fmul reassoc float [[C]], [[A:%.*]]
-; CHECK-NEXT:    [[F:%.*]] = fneg reassoc float [[E]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul reassoc float [[Z:%.*]], 4.000000e+01
+; CHECK-NEXT:    [[F:%.*]] = fmul reassoc float [[TMP1]], [[A:%.*]]
 ; CHECK-NEXT:    ret float [[F]]
 ;
   %d = fmul reassoc float %z, 4.000000e+01
diff --git a/llvm/test/Transforms/Reassociate/fast-fp-commute.ll b/llvm/test/Transforms/Reassociate/fast-fp-commute.ll
index c623abdde..937c4c112 100644
--- a/llvm/test/Transforms/Reassociate/fast-fp-commute.ll
+++ b/llvm/test/Transforms/Reassociate/fast-fp-commute.ll
@@ -1,11 +1,12 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -reassociate -S < %s | FileCheck %s
 
 declare void @use(float)
 
 define void @test1(float %x, float %y) {
 ; CHECK-LABEL: @test1(
-; CHECK-NEXT:    [[TMP1:%.*]] = fmul fast float %y, %x
-; CHECK-NEXT:    [[TMP2:%.*]] = fmul fast float %y, %x
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul fast float [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fmul fast float [[Y]], [[X]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = fsub fast float [[TMP1]], [[TMP2]]
 ; CHECK-NEXT:    call void @use(float [[TMP1]])
 ; CHECK-NEXT:    call void @use(float [[TMP3]])
@@ -21,8 +22,8 @@ define void @test1(float %x, float %y) {
 
 define float @test2(float %x, float %y) {
 ; CHECK-LABEL: @test2(
-; CHECK-NEXT:    [[TMP1:%.*]] = fmul fast float %y, %x
-; CHECK-NEXT:    [[TMP2:%.*]] = fmul fast float %y, %x
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul fast float [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fmul fast float [[Y]], [[X]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = fsub fast float [[TMP1]], [[TMP2]]
 ; CHECK-NEXT:    ret float [[TMP3]]
 ;
@@ -34,7 +35,7 @@ define float @test2(float %x, float %y) {
 
 define float @test3(float %x, float %y) {
 ; CHECK-LABEL: @test3(
-; CHECK-NEXT:    [[FACTOR:%.*]] = fmul fast float %y, %x
+; CHECK-NEXT:    [[FACTOR:%.*]] = fmul fast float [[Y:%.*]], [[X:%.*]]
 ; CHECK-NEXT:    [[REASS_MUL:%.*]] = fmul fast float [[FACTOR]], 2.000000e+00
 ; CHECK-NEXT:    ret float [[REASS_MUL]]
 ;
diff --git a/llvm/test/Transforms/Reassociate/fast-multistep.ll b/llvm/test/Transforms/Reassociate/fast-multistep.ll
index 6f2290f1c..419e101cd 100644
--- a/llvm/test/Transforms/Reassociate/fast-multistep.ll
+++ b/llvm/test/Transforms/Reassociate/fast-multistep.ll
@@ -1,12 +1,13 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -reassociate -S | FileCheck %s
 
 ; Check that a*a*b+a*a*c is turned into a*(a*(b+c)).
 
 define float @fmultistep1(float %a, float %b, float %c) {
 ; CHECK-LABEL: @fmultistep1(
-; CHECK-NEXT:    [[REASS_ADD1:%.*]] = fadd fast float %c, %b
-; CHECK-NEXT:    [[REASS_MUL2:%.*]] = fmul fast float %a, %a
-; CHECK-NEXT:    [[REASS_MUL:%.*]] = fmul fast float [[REASS_MUL:%.*]]2, [[REASS_ADD1]]
+; CHECK-NEXT:    [[REASS_ADD1:%.*]] = fadd fast float [[C:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[REASS_MUL2:%.*]] = fmul fast float [[A:%.*]], [[A]]
+; CHECK-NEXT:    [[REASS_MUL:%.*]] = fmul fast float [[REASS_MUL2]], [[REASS_ADD1]]
 ; CHECK-NEXT:    ret float [[REASS_MUL]]
 ;
   %t0 = fmul fast float %a, %b
@@ -21,9 +22,9 @@ define float @fmultistep1(float %a, float %b, float %c) {
 
 define float @fmultistep2(float %a, float %b, float %c, float %d) {
 ; CHECK-LABEL: @fmultistep2(
-; CHECK-NEXT:    [[REASS_ADD:%.*]] = fadd fast float %c, %b
-; CHECK-NEXT:    [[REASS_MUL:%.*]] = fmul fast float [[REASS_ADD]], %a
-; CHECK-NEXT:    [[T3:%.*]] = fadd fast float [[REASS_MUL]], %d
+; CHECK-NEXT:    [[REASS_ADD:%.*]] = fadd fast float [[C:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[REASS_MUL:%.*]] = fmul fast float [[REASS_ADD]], [[A:%.*]]
+; CHECK-NEXT:    [[T3:%.*]] = fadd fast float [[REASS_MUL]], [[D:%.*]]
 ; CHECK-NEXT:    ret float [[T3]]
 ;
   %t0 = fmul fast float %a, %b
diff --git a/llvm/test/Transforms/Reassociate/looptest.ll b/llvm/test/Transforms/Reassociate/looptest.ll
index 2c3db3105..6130d4359 100644
--- a/llvm/test/Transforms/Reassociate/looptest.ll
+++ b/llvm/test/Transforms/Reassociate/looptest.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; This testcase comes from this C fragment:
 ;
 ; void test(unsigned Num, int *Array) {
@@ -19,34 +20,63 @@
 declare i32 @printf(i8*, ...)
 
 ; Check that (i+j) has been reassociated (i=reg115, j=reg116)
-; CHECK: %reg113 = add i32 %reg116, %reg115
 define void @test(i32 %Num, i32* %Array) {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:  bb0:
+; CHECK-NEXT:    [[COND221:%.*]] = icmp eq i32 0, [[NUM:%.*]]
+; CHECK-NEXT:    br i1 [[COND221]], label [[BB7:%.*]], label [[BB2:%.*]]
+; CHECK:       bb2:
+; CHECK-NEXT:    [[REG115:%.*]] = phi i32 [ [[REG120:%.*]], [[BB6:%.*]] ], [ 0, [[BB0:%.*]] ]
+; CHECK-NEXT:    br i1 [[COND221]], label [[BB6]], label [[BB3:%.*]]
+; CHECK:       bb3:
+; CHECK-NEXT:    [[REG116:%.*]] = phi i32 [ [[REG119:%.*]], [[BB5:%.*]] ], [ 0, [[BB2]] ]
+; CHECK-NEXT:    br i1 [[COND221]], label [[BB5]], label [[BB4:%.*]]
+; CHECK:       bb4:
+; CHECK-NEXT:    [[REG117:%.*]] = phi i32 [ [[REG118:%.*]], [[BB4]] ], [ 0, [[BB3]] ]
+; CHECK-NEXT:    [[REG113:%.*]] = add i32 [[REG116]], [[REG115]]
+; CHECK-NEXT:    [[REG114:%.*]] = add i32 [[REG113]], [[REG117]]
+; CHECK-NEXT:    [[CAST227:%.*]] = getelementptr [4 x i8], [4 x i8]* @.LC0, i64 0, i64 0
+; CHECK-NEXT:    [[TMP0:%.*]] = call i32 (i8*, ...) @printf(i8* [[CAST227]], i32 [[REG114]])
+; CHECK-NEXT:    [[REG118]] = add i32 [[REG117]], 1
+; CHECK-NEXT:    [[COND224:%.*]] = icmp ne i32 [[REG118]], [[NUM]]
+; CHECK-NEXT:    br i1 [[COND224]], label [[BB4]], label [[BB5]]
+; CHECK:       bb5:
+; CHECK-NEXT:    [[REG119]] = add i32 [[REG116]], 1
+; CHECK-NEXT:    [[COND225:%.*]] = icmp ne i32 [[REG119]], [[NUM]]
+; CHECK-NEXT:    br i1 [[COND225]], label [[BB3]], label [[BB6]]
+; CHECK:       bb6:
+; CHECK-NEXT:    [[REG120]] = add i32 [[REG115]], 1
+; CHECK-NEXT:    [[COND226:%.*]] = icmp ne i32 [[REG120]], [[NUM]]
+; CHECK-NEXT:    br i1 [[COND226]], label [[BB2]], label [[BB7]]
+; CHECK:       bb7:
+; CHECK-NEXT:    ret void
+;
 bb0:
-	%cond221 = icmp eq i32 0, %Num		; <i1> [#uses=3]
-	br i1 %cond221, label %bb7, label %bb2
+  %cond221 = icmp eq i32 0, %Num		; <i1> [#uses=3]
+  br i1 %cond221, label %bb7, label %bb2
 bb2:		; preds = %bb6, %bb0
-	%reg115 = phi i32 [ %reg120, %bb6 ], [ 0, %bb0 ]		; <i32> [#uses=2]
-	br i1 %cond221, label %bb6, label %bb3
+  %reg115 = phi i32 [ %reg120, %bb6 ], [ 0, %bb0 ]		; <i32> [#uses=2]
+  br i1 %cond221, label %bb6, label %bb3
 bb3:		; preds = %bb5, %bb2
-	%reg116 = phi i32 [ %reg119, %bb5 ], [ 0, %bb2 ]		; <i32> [#uses=2]
-	br i1 %cond221, label %bb5, label %bb4
+  %reg116 = phi i32 [ %reg119, %bb5 ], [ 0, %bb2 ]		; <i32> [#uses=2]
+  br i1 %cond221, label %bb5, label %bb4
 bb4:		; preds = %bb4, %bb3
-	%reg117 = phi i32 [ %reg118, %bb4 ], [ 0, %bb3 ]		; <i32> [#uses=2]
-	%reg113 = add i32 %reg115, %reg117		; <i32> [#uses=1]
-	%reg114 = add i32 %reg113, %reg116		; <i32> [#uses=1]
-	%cast227 = getelementptr [4 x i8], [4 x i8]* @.LC0, i64 0, i64 0		; <i8*> [#uses=1]
-	call i32 (i8*, ...) @printf( i8* %cast227, i32 %reg114 )		; <i32>:0 [#uses=0]
-	%reg118 = add i32 %reg117, 1		; <i32> [#uses=2]
-	%cond224 = icmp ne i32 %reg118, %Num		; <i1> [#uses=1]
-	br i1 %cond224, label %bb4, label %bb5
+  %reg117 = phi i32 [ %reg118, %bb4 ], [ 0, %bb3 ]		; <i32> [#uses=2]
+  %reg113 = add i32 %reg115, %reg117		; <i32> [#uses=1]
+  %reg114 = add i32 %reg113, %reg116		; <i32> [#uses=1]
+  %cast227 = getelementptr [4 x i8], [4 x i8]* @.LC0, i64 0, i64 0		; <i8*> [#uses=1]
+  call i32 (i8*, ...) @printf( i8* %cast227, i32 %reg114 )		; <i32>:0 [#uses=0]
+  %reg118 = add i32 %reg117, 1		; <i32> [#uses=2]
+  %cond224 = icmp ne i32 %reg118, %Num		; <i1> [#uses=1]
+  br i1 %cond224, label %bb4, label %bb5
 bb5:		; preds = %bb4, %bb3
-	%reg119 = add i32 %reg116, 1		; <i32> [#uses=2]
-	%cond225 = icmp ne i32 %reg119, %Num		; <i1> [#uses=1]
-	br i1 %cond225, label %bb3, label %bb6
+  %reg119 = add i32 %reg116, 1		; <i32> [#uses=2]
+  %cond225 = icmp ne i32 %reg119, %Num		; <i1> [#uses=1]
+  br i1 %cond225, label %bb3, label %bb6
 bb6:		; preds = %bb5, %bb2
-	%reg120 = add i32 %reg115, 1		; <i32> [#uses=2]
-	%cond226 = icmp ne i32 %reg120, %Num		; <i1> [#uses=1]
-	br i1 %cond226, label %bb2, label %bb7
+  %reg120 = add i32 %reg115, 1		; <i32> [#uses=2]
+  %cond226 = icmp ne i32 %reg120, %Num		; <i1> [#uses=1]
+  br i1 %cond226, label %bb2, label %bb7
 bb7:		; preds = %bb6, %bb0
-	ret void
+  ret void
 }
diff --git a/llvm/test/Transforms/Reassociate/mixed-fast-nonfast-fp.ll b/llvm/test/Transforms/Reassociate/mixed-fast-nonfast-fp.ll
index 14871f45b..d37bfbbdf 100644
--- a/llvm/test/Transforms/Reassociate/mixed-fast-nonfast-fp.ll
+++ b/llvm/test/Transforms/Reassociate/mixed-fast-nonfast-fp.ll
@@ -1,11 +1,12 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -reassociate %s -S | FileCheck %s
 
 define float @foo(float %a,float %b, float %c) {
 ; CHECK-LABEL: @foo(
-; CHECK-NEXT:    [[MUL3:%.*]] = fmul float %a, %b
-; CHECK-NEXT:    [[FACTOR:%.*]] = fmul fast float %c, 2.000000e+00
-; CHECK-NEXT:    [[REASS_ADD1:%.*]] = fadd fast float [[FACTOR]], %b
-; CHECK-NEXT:    [[REASS_MUL:%.*]] = fmul fast float [[REASS_ADD1]], %a
+; CHECK-NEXT:    [[MUL3:%.*]] = fmul float [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[FACTOR:%.*]] = fmul fast float [[C:%.*]], 2.000000e+00
+; CHECK-NEXT:    [[REASS_ADD1:%.*]] = fadd fast float [[FACTOR]], [[B]]
+; CHECK-NEXT:    [[REASS_MUL:%.*]] = fmul fast float [[REASS_ADD1]], [[A]]
 ; CHECK-NEXT:    [[ADD3:%.*]] = fadd fast float [[REASS_MUL]], [[MUL3]]
 ; CHECK-NEXT:    ret float [[ADD3]]
 ;
@@ -21,10 +22,10 @@ define float @foo(float %a,float %b, float %c) {
 
 define float @foo_reassoc(float %a,float %b, float %c) {
 ; CHECK-LABEL: @foo_reassoc(
-; CHECK-NEXT:    [[MUL1:%.*]] = fmul reassoc float %a, %c
-; CHECK-NEXT:    [[MUL2:%.*]] = fmul fast float %b, %a
-; CHECK-NEXT:    [[MUL3:%.*]] = fmul float %a, %b
-; CHECK-NEXT:    [[MUL4:%.*]] = fmul reassoc float %a, %c
+; CHECK-NEXT:    [[MUL1:%.*]] = fmul reassoc float [[A:%.*]], [[C:%.*]]
+; CHECK-NEXT:    [[MUL2:%.*]] = fmul fast float [[B:%.*]], [[A]]
+; CHECK-NEXT:    [[MUL3:%.*]] = fmul float [[A]], [[B]]
+; CHECK-NEXT:    [[MUL4:%.*]] = fmul reassoc float [[A]], [[C]]
 ; CHECK-NEXT:    [[ADD1:%.*]] = fadd fast float [[MUL1]], [[MUL3]]
 ; CHECK-NEXT:    [[ADD2:%.*]] = fadd reassoc float [[MUL2]], [[MUL4]]
 ; CHECK-NEXT:    [[ADD3:%.*]] = fadd fast float [[ADD1]], [[ADD2]]
diff --git a/llvm/test/Transforms/Reassociate/mulfactor.ll b/llvm/test/Transforms/Reassociate/mulfactor.ll
index ca0dbcde8..4cbeaad38 100644
--- a/llvm/test/Transforms/Reassociate/mulfactor.ll
+++ b/llvm/test/Transforms/Reassociate/mulfactor.ll
@@ -1,11 +1,12 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -reassociate -S | FileCheck %s
 
 define i32 @test1(i32 %a, i32 %b) {
 ; CHECK-LABEL: @test1(
-; CHECK-NEXT:    [[T2:%.*]] = mul i32 %a, %a
-; CHECK-NEXT:    [[T6:%.*]] = mul i32 %a, 2
-; CHECK-NEXT:    [[REASS_ADD:%.*]] = add i32 [[T6]], %b
-; CHECK-NEXT:    [[REASS_MUL:%.*]] = mul i32 [[REASS_ADD]], %b
+; CHECK-NEXT:    [[T2:%.*]] = mul i32 [[A:%.*]], [[A]]
+; CHECK-NEXT:    [[T6:%.*]] = mul i32 [[A]], 2
+; CHECK-NEXT:    [[REASS_ADD:%.*]] = add i32 [[T6]], [[B:%.*]]
+; CHECK-NEXT:    [[REASS_MUL:%.*]] = mul i32 [[REASS_ADD]], [[B]]
 ; CHECK-NEXT:    [[T11:%.*]] = add i32 [[REASS_MUL]], [[T2]]
 ; CHECK-NEXT:    ret i32 [[T11]]
 ;
@@ -20,7 +21,7 @@ define i32 @test1(i32 %a, i32 %b) {
 
 define i32 @test2(i32 %t) {
 ; CHECK-LABEL: @test2(
-; CHECK-NEXT:    [[REASS_MUL:%.*]] = mul i32 %t, 42
+; CHECK-NEXT:    [[REASS_MUL:%.*]] = mul i32 [[T:%.*]], 42
 ; CHECK-NEXT:    [[D:%.*]] = add i32 [[REASS_MUL]], 15
 ; CHECK-NEXT:    ret i32 [[D]]
 ;
@@ -34,7 +35,7 @@ define i32 @test2(i32 %t) {
 ; (x^8)
 define i32 @test3(i32 %x) {
 ; CHECK-LABEL: @test3(
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i32 %x, %x
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i32 [[X:%.*]], [[X]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = mul i32 [[TMP1]], [[TMP1]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = mul i32 [[TMP2]], [[TMP2]]
 ; CHECK-NEXT:    ret i32 [[TMP3]]
@@ -52,9 +53,9 @@ define i32 @test3(i32 %x) {
 ; (x^7)
 define i32 @test4(i32 %x) {
 ; CHECK-LABEL: @test4(
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i32 %x, %x
-; CHECK-NEXT:    [[TMP2:%.*]] = mul i32 [[TMP1]], %x
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i32 [[TMP2]], %x
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i32 [[X:%.*]], [[X]]
+; CHECK-NEXT:    [[TMP2:%.*]] = mul i32 [[TMP1]], [[X]]
+; CHECK-NEXT:    [[TMP3:%.*]] = mul i32 [[TMP2]], [[X]]
 ; CHECK-NEXT:    [[F:%.*]] = mul i32 [[TMP3]], [[TMP2]]
 ; CHECK-NEXT:    ret i32 [[F]]
 ;
@@ -70,8 +71,8 @@ define i32 @test4(i32 %x) {
 ; (x^4) * (y^2)
 define i32 @test5(i32 %x, i32 %y) {
 ; CHECK-LABEL: @test5(
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i32 %x, %x
-; CHECK-NEXT:    [[TMP2:%.*]] = mul i32 [[TMP1]], %y
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i32 [[X:%.*]], [[X]]
+; CHECK-NEXT:    [[TMP2:%.*]] = mul i32 [[TMP1]], [[Y:%.*]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = mul i32 [[TMP2]], [[TMP2]]
 ; CHECK-NEXT:    ret i32 [[TMP3]]
 ;
@@ -86,12 +87,12 @@ define i32 @test5(i32 %x, i32 %y) {
 ; (x^5) * (y^3) * z
 define i32 @test6(i32 %x, i32 %y, i32 %z) {
 ; CHECK-LABEL: @test6(
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i32 %x, %x
-; CHECK-NEXT:    [[TMP2:%.*]] = mul i32 [[TMP1]], %y
-; CHECK-NEXT:    [[F:%.*]] = mul i32 %y, %x
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i32 [[X:%.*]], [[X]]
+; CHECK-NEXT:    [[TMP2:%.*]] = mul i32 [[TMP1]], [[Y:%.*]]
+; CHECK-NEXT:    [[F:%.*]] = mul i32 [[Y]], [[X]]
 ; CHECK-NEXT:    [[G:%.*]] = mul i32 [[F]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = mul i32 [[G]], [[TMP2]]
-; CHECK-NEXT:    [[H:%.*]] = mul i32 [[TMP3]], %z
+; CHECK-NEXT:    [[H:%.*]] = mul i32 [[TMP3]], [[Z:%.*]]
 ; CHECK-NEXT:    ret i32 [[H]]
 ;
   %a = mul i32 %x, %y
@@ -108,10 +109,10 @@ define i32 @test6(i32 %x, i32 %y, i32 %z) {
 ; (x^4) * (y^3) * (z^2)
 define i32 @test7(i32 %x, i32 %y, i32 %z) {
 ; CHECK-LABEL: @test7(
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i32 %x, %x
-; CHECK-NEXT:    [[TMP2:%.*]] = mul i32 [[TMP1]], %y
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i32 [[TMP2]], %z
-; CHECK-NEXT:    [[TMP4:%.*]] = mul i32 [[TMP3]], %y
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i32 [[X:%.*]], [[X]]
+; CHECK-NEXT:    [[TMP2:%.*]] = mul i32 [[TMP1]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP3:%.*]] = mul i32 [[TMP2]], [[Z:%.*]]
+; CHECK-NEXT:    [[TMP4:%.*]] = mul i32 [[TMP3]], [[Y]]
 ; CHECK-NEXT:    [[H:%.*]] = mul i32 [[TMP4]], [[TMP3]]
 ; CHECK-NEXT:    ret i32 [[H]]
 ;
diff --git a/llvm/test/Transforms/Reassociate/multistep.ll b/llvm/test/Transforms/Reassociate/multistep.ll
index 239ffe082..35a853d82 100644
--- a/llvm/test/Transforms/Reassociate/multistep.ll
+++ b/llvm/test/Transforms/Reassociate/multistep.ll
@@ -1,12 +1,13 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -reassociate -S | FileCheck %s
 
 ; Check that a*a*b+a*a*c is turned into a*(a*(b+c)).
 
 define i64 @multistep1(i64 %a, i64 %b, i64 %c) {
 ; CHECK-LABEL: @multistep1(
-; CHECK-NEXT:    [[REASS_ADD1:%.*]] = add i64 %c, %b
-; CHECK-NEXT:    [[REASS_MUL2:%.*]] = mul i64 %a, %a
-; CHECK-NEXT:    [[REASS_MUL:%.*]] = mul i64 [[REASS_MUL:%.*]]2, [[REASS_ADD1]]
+; CHECK-NEXT:    [[REASS_ADD1:%.*]] = add i64 [[C:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[REASS_MUL2:%.*]] = mul i64 [[A:%.*]], [[A]]
+; CHECK-NEXT:    [[REASS_MUL:%.*]] = mul i64 [[REASS_MUL2]], [[REASS_ADD1]]
 ; CHECK-NEXT:    ret i64 [[REASS_MUL]]
 ;
   %t0 = mul i64 %a, %b
@@ -21,9 +22,9 @@ define i64 @multistep1(i64 %a, i64 %b, i64 %c) {
 
 define i64 @multistep2(i64 %a, i64 %b, i64 %c, i64 %d) {
 ; CHECK-LABEL: @multistep2(
-; CHECK-NEXT:    [[REASS_ADD:%.*]] = add i64 %c, %b
-; CHECK-NEXT:    [[REASS_MUL:%.*]] = mul i64 [[REASS_ADD]], %a
-; CHECK-NEXT:    [[T3:%.*]] = add i64 [[REASS_MUL]], %d
+; CHECK-NEXT:    [[REASS_ADD:%.*]] = add i64 [[C:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[REASS_MUL:%.*]] = mul i64 [[REASS_ADD]], [[A:%.*]]
+; CHECK-NEXT:    [[T3:%.*]] = add i64 [[REASS_MUL]], [[D:%.*]]
 ; CHECK-NEXT:    ret i64 [[T3]]
 ;
   %t0 = mul i64 %a, %b
diff --git a/llvm/test/Transforms/Reassociate/no-op.ll b/llvm/test/Transforms/Reassociate/no-op.ll
index 7b02df994..248766d39 100644
--- a/llvm/test/Transforms/Reassociate/no-op.ll
+++ b/llvm/test/Transforms/Reassociate/no-op.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -reassociate -S | FileCheck %s
 
 ; When there is nothing to do, or not much to do, check that reassociate leaves
@@ -9,16 +10,19 @@ define void @test1(i32 %a, i32 %b) {
 ; Shouldn't change or move any of the add instructions.  Should commute but
 ; otherwise not change or move any of the mul instructions.
 ; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[A0:%.*]] = add nsw i32 [[A:%.*]], 1
+; CHECK-NEXT:    [[M0:%.*]] = mul nsw i32 [[A]], 3
+; CHECK-NEXT:    [[A1:%.*]] = add nsw i32 [[A0]], [[B:%.*]]
+; CHECK-NEXT:    [[M1:%.*]] = mul nsw i32 [[M0]], [[B]]
+; CHECK-NEXT:    call void @use(i32 [[A1]])
+; CHECK-NEXT:    call void @use(i32 [[M1]])
+; CHECK-NEXT:    ret void
+;
   %a0 = add nsw i32 %a, 1
-; CHECK-NEXT: %a0 = add nsw i32 %a, 1
   %m0 = mul nsw i32 3, %a
-; CHECK-NEXT: %m0 = mul nsw i32 %a, 3
   %a1 = add nsw i32 %a0, %b
-; CHECK-NEXT: %a1 = add nsw i32 %a0, %b
   %m1 = mul nsw i32 %b, %m0
-; CHECK-NEXT: %m1 = mul nsw i32 %m0, %b
   call void @use(i32 %a1)
-; CHECK-NEXT: call void @use
   call void @use(i32 %m1)
   ret void
 }
@@ -26,13 +30,15 @@ define void @test1(i32 %a, i32 %b) {
 define void @test2(i32 %a, i32 %b, i32 %c, i32 %d) {
 ; The initial add doesn't change so should not lose the nsw flag.
 ; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[A0:%.*]] = add nsw i32 [[B:%.*]], [[A:%.*]]
+; CHECK-NEXT:    [[A1:%.*]] = add i32 [[A0]], [[C:%.*]]
+; CHECK-NEXT:    [[A2:%.*]] = add i32 [[A1]], [[D:%.*]]
+; CHECK-NEXT:    call void @use(i32 [[A2]])
+; CHECK-NEXT:    ret void
+;
   %a0 = add nsw i32 %b, %a
-; CHECK-NEXT: %a0 = add nsw i32 %b, %a
   %a1 = add nsw i32 %a0, %d
-; CHECK-NEXT: %a1 = add i32 %a0, %c
   %a2 = add nsw i32 %a1, %c
-; CHECK-NEXT: %a2 = add i32 %a1, %d
   call void @use(i32 %a2)
-; CHECK-NEXT: call void @use
   ret void
 }
diff --git a/llvm/test/Transforms/Reassociate/optional-flags.ll b/llvm/test/Transforms/Reassociate/optional-flags.ll
index bf599be78..00c2d0ac3 100644
--- a/llvm/test/Transforms/Reassociate/optional-flags.ll
+++ b/llvm/test/Transforms/Reassociate/optional-flags.ll
@@ -1,29 +1,40 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -S -reassociate < %s | FileCheck %s
 ; rdar://8944681
 
 ; Reassociate should clear optional flags like nsw when reassociating.
 
-; CHECK-LABEL: @test0(
-; CHECK: %y = add i64 %b, %a
-; CHECK: %z = add i64 %y, %c
 define i64 @test0(i64 %a, i64 %b, i64 %c) {
+;
+; CHECK-LABEL: @test0(
+; CHECK-NEXT:    [[Y:%.*]] = add i64 [[B:%.*]], [[A:%.*]]
+; CHECK-NEXT:    [[Z:%.*]] = add i64 [[Y]], [[C:%.*]]
+; CHECK-NEXT:    ret i64 [[Z]]
+;
   %y = add nsw i64 %c, %b
   %z = add i64 %y, %a
   ret i64 %z
 }
 
-; CHECK-LABEL: @test1(
-; CHECK: %y = add i64 %b, %a
-; CHECK: %z = add i64 %y, %c
 define i64 @test1(i64 %a, i64 %b, i64 %c) {
+;
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[Y:%.*]] = add i64 [[B:%.*]], [[A:%.*]]
+; CHECK-NEXT:    [[Z:%.*]] = add i64 [[Y]], [[C:%.*]]
+; CHECK-NEXT:    ret i64 [[Z]]
+;
   %y = add i64 %c, %b
   %z = add nsw i64 %y, %a
   ret i64 %z
 }
 
 ; PR9215
-; CHECK: %s = add nsw i32 %y, %x
 define i32 @test2(i32 %x, i32 %y) {
+;
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[S:%.*]] = add nsw i32 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[S]]
+;
   %s = add nsw i32 %x, %y
   ret i32 %s
 }
diff --git a/llvm/test/Transforms/Reassociate/propagate-flags.ll b/llvm/test/Transforms/Reassociate/propagate-flags.ll
index b6cc9bad5..fe477213c 100644
--- a/llvm/test/Transforms/Reassociate/propagate-flags.ll
+++ b/llvm/test/Transforms/Reassociate/propagate-flags.ll
@@ -1,8 +1,9 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -reassociate -S | FileCheck %s
 
 define double @func(double %a, double %b) {
 ; CHECK-LABEL: @func(
-; CHECK-NEXT:    [[TMP1:%.*]] = fmul fast double %b, %a
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul fast double [[B:%.*]], [[A:%.*]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = fmul fast double [[TMP1]], [[TMP1]]
 ; CHECK-NEXT:    ret double [[TMP2]]
 ;
diff --git a/llvm/test/Transforms/Reassociate/reassoc-intermediate-fnegs.ll b/llvm/test/Transforms/Reassociate/reassoc-intermediate-fnegs.ll
index 3bd839663..7f247de6c 100644
--- a/llvm/test/Transforms/Reassociate/reassoc-intermediate-fnegs.ll
+++ b/llvm/test/Transforms/Reassociate/reassoc-intermediate-fnegs.ll
@@ -1,15 +1,15 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -reassociate -S | FileCheck %s
 
-; Input is A op (B op C)
+; B * 5 + A * -5
+; TODO: (B - A) * 5
 
 define half @faddsubAssoc1(half %a, half %b) {
 ; CHECK-LABEL: @faddsubAssoc1(
-; CHECK-NEXT:    [[T2_NEG:%.*]] = fmul fast half [[A:%.*]], 0xH4500
+; CHECK-NEXT:    [[T2_NEG:%.*]] = fmul fast half [[A:%.*]], 0xHC500
 ; CHECK-NEXT:    [[REASS_MUL:%.*]] = fmul fast half [[B:%.*]], 0xH4500
-; CHECK-NEXT:    [[T51:%.*]] = fsub fast half [[REASS_MUL]], [[T2_NEG]]
 ; CHECK-NEXT:    [[T5:%.*]] = fadd fast half [[REASS_MUL]], [[T2_NEG]]
-; CHECK-NEXT:    ret half [[T51]]
+; CHECK-NEXT:    ret half [[T5]]
 ;
   %t1 = fmul fast half %b, 0xH4200 ; 3*b
   %t2 = fmul fast half %a, 0xH4500 ; 5*a
@@ -19,7 +19,7 @@ define half @faddsubAssoc1(half %a, half %b) {
   ret half %t5 ; = 5 * (b - a)
 }
 
-; Input is (A op B) op C
+; B + A * 5
 
 define half @faddsubAssoc2(half %a, half %b) {
 ; CHECK-LABEL: @faddsubAssoc2(
diff --git a/llvm/test/Transforms/Reassociate/reassociate-landingpad.ll b/llvm/test/Transforms/Reassociate/reassociate-landingpad.ll
index 432cbed94..6adfbb0a7 100644
--- a/llvm/test/Transforms/Reassociate/reassociate-landingpad.ll
+++ b/llvm/test/Transforms/Reassociate/reassociate-landingpad.ll
@@ -47,7 +47,7 @@ invoke.cont25:                                    ; preds = %", bb8"
 
 declare void @c() #0
 
-attributes #0 = { "no-frame-pointer-elim-non-leaf" }
+attributes #0 = { "frame-pointer"="non-leaf" }
 
 !llvm.module.flags = !{!0}
 
diff --git a/llvm/test/Transforms/Reassociate/reassociate_salvages_debug_info.ll b/llvm/test/Transforms/Reassociate/reassociate_salvages_debug_info.ll
new file mode 100644
index 000000000..34e0b9a04
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/reassociate_salvages_debug_info.ll
@@ -0,0 +1,50 @@
+; RUN: opt < %s -reassociate -S | FileCheck %s
+
+; Check that reassociate pass now salvages debug info when dropping instructions.
+
+define hidden i32 @main(i32 %argc, i8** %argv) {
+entry:
+  ; CHECK: call void @llvm.dbg.value(metadata i32 %argc, metadata [[VAR_B:![0-9]+]], metadata !DIExpression(DW_OP_plus_uconst, 1, DW_OP_stack_value))
+  %add = add nsw i32 %argc, 1, !dbg !26
+  call void @llvm.dbg.value(metadata i32 %add, metadata !22, metadata !DIExpression()), !dbg !25
+  %add1 = add nsw i32 %argc, %add, !dbg !27
+  ret i32 %add1, !dbg !28
+}
+
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
+declare void @llvm.dbg.value(metadata, metadata, metadata) #1
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5, !6}
+!llvm.ident = !{!7}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 10.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, debugInfoForProfiling: true, nameTableKind: None)
+!1 = !DIFile(filename: "test2.cpp", directory: "C:\")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 2}
+!6 = !{i32 7, !"PIC Level", i32 2}
+!7 = !{!"clang version 10.0.0"}
+!8 = distinct !DISubprogram(name: "main", scope: !9, file: !9, line: 1, type: !10, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !18)
+!9 = !DIFile(filename: "./test2.cpp", directory: "C:\")
+!10 = !DISubroutineType(types: !11)
+!11 = !{!12, !13, !14}
+!12 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!13 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !12)
+!14 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !15, size: 64)
+!15 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !16, size: 64)
+!16 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !17)
+!17 = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_signed_char)
+!18 = !{!19, !20, !21, !22, !23, !24}
+!19 = !DILocalVariable(name: "argc", arg: 1, scope: !8, file: !9, line: 1, type: !13)
+!20 = !DILocalVariable(name: "argv", arg: 2, scope: !8, file: !9, line: 1, type: !14)
+!21 = !DILocalVariable(name: "a", scope: !8, file: !9, line: 2, type: !12)
+; CHECK: [[VAR_B]] = !DILocalVariable(name: "b"
+!22 = !DILocalVariable(name: "b", scope: !8, file: !9, line: 3, type: !12)
+!23 = !DILocalVariable(name: "to_return", scope: !8, file: !9, line: 4, type: !12)
+!24 = !DILocalVariable(name: "result", scope: !8, file: !9, line: 5, type: !12)
+!25 = !DILocation(line: 0, scope: !8)
+!26 = !DILocation(line: 3, scope: !8)
+!27 = !DILocation(line: 4, scope: !8)
+!28 = !DILocation(line: 6, scope: !8)
diff --git a/llvm/test/Transforms/Reassociate/shift-factor.ll b/llvm/test/Transforms/Reassociate/shift-factor.ll
index 4b102b07d..d4ccf138f 100644
--- a/llvm/test/Transforms/Reassociate/shift-factor.ll
+++ b/llvm/test/Transforms/Reassociate/shift-factor.ll
@@ -1,10 +1,11 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -reassociate -instcombine -S | FileCheck %s
 
 ; There should be exactly one shift and one add left.
 
 define i32 @test1(i32 %X, i32 %Y) {
 ; CHECK-LABEL: @test1(
-; CHECK-NEXT:    [[REASS_ADD:%.*]] = add i32 %Y, %X
+; CHECK-NEXT:    [[REASS_ADD:%.*]] = add i32 [[Y:%.*]], [[X:%.*]]
 ; CHECK-NEXT:    [[REASS_MUL:%.*]] = shl i32 [[REASS_ADD]], 1
 ; CHECK-NEXT:    ret i32 [[REASS_MUL]]
 ;
diff --git a/llvm/test/Transforms/Reassociate/undef_intrinsics_when_deleting_instructions.ll b/llvm/test/Transforms/Reassociate/undef_intrinsics_when_deleting_instructions.ll
new file mode 100644
index 000000000..98c51c5cf
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/undef_intrinsics_when_deleting_instructions.ll
@@ -0,0 +1,95 @@
+; RUN: opt < %s -reassociate -S | FileCheck %s
+
+; Check that reassociate pass now undefs debug intrinsics that reference a value
+; that gets dropped and cannot be salvaged.
+
+define hidden i32 @main() local_unnamed_addr {
+entry:
+  %foo = alloca i32, align 4, !dbg !20
+  %foo.0.foo.0..sroa_cast = bitcast i32* %foo to i8*, !dbg !20
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* nonnull %foo.0.foo.0..sroa_cast), !dbg !20
+  store volatile i32 4, i32* %foo, align 4, !dbg !20, !tbaa !21
+  %foo.0.foo.0. = load volatile i32, i32* %foo, align 4, !dbg !25, !tbaa !21
+  %foo.0.foo.0.15 = load volatile i32, i32* %foo, align 4, !dbg !27, !tbaa !21
+  %foo.0.foo.0.16 = load volatile i32, i32* %foo, align 4, !dbg !28, !tbaa !21
+  ; CHECK-NOT: %add = add nsw i32 %foo.0.foo.0., %foo.0.foo.0.15
+  %add = add nsw i32 %foo.0.foo.0., %foo.0.foo.0.15, !dbg !29
+  ; CHECK: call void @llvm.dbg.value(metadata i32 undef, metadata [[VAR_A:![0-9]+]], metadata !DIExpression())
+  call void @llvm.dbg.value(metadata i32 %add, metadata !19, metadata !DIExpression()), !dbg !26
+  %foo.0.foo.0.17 = load volatile i32, i32* %foo, align 4, !dbg !30, !tbaa !21
+  %cmp = icmp eq i32 %foo.0.foo.0.17, 4, !dbg !30
+  br i1 %cmp, label %if.then, label %if.end, !dbg !32
+
+  ; CHECK-LABEL: if.then:
+if.then:
+  ; CHECK-NOT: %add1 = add nsw i32 %add, %foo.0.foo.0.16
+  %add1 = add nsw i32 %add, %foo.0.foo.0.16, !dbg !33
+  ; CHECK: call void @llvm.dbg.value(metadata i32 undef, metadata [[VAR_A]], metadata !DIExpression())
+  call void @llvm.dbg.value(metadata i32 %add1, metadata !19, metadata !DIExpression()), !dbg !26
+  ; CHECK: call void @llvm.dbg.value(metadata i32 undef, metadata [[VAR_CHEESE:![0-9]+]], metadata !DIExpression())
+  call void @llvm.dbg.value(metadata i32 %add, metadata !18, metadata !DIExpression()), !dbg !26
+  %sub = add nsw i32 %add, -12, !dbg !34
+  %sub3 = sub nsw i32 %add1, %sub, !dbg !34
+  %mul = mul nsw i32 %sub3, 20, !dbg !36
+  %div = sdiv i32 %mul, 3, !dbg !37
+  br label %if.end, !dbg !38
+
+if.end:
+  %a.0 = phi i32 [ %div, %if.then ], [ 0, %entry ], !dbg !39
+  call void @llvm.lifetime.end.p0i8(i64 4, i8* nonnull %foo.0.foo.0..sroa_cast), !dbg !40
+  ret i32 %a.0, !dbg !41
+}
+
+declare void @llvm.lifetime.start.p0i8(i64 immarg, i8* nocapture) #1
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #2
+declare void @llvm.lifetime.end.p0i8(i64 immarg, i8* nocapture) #1
+declare void @llvm.dbg.value(metadata, metadata, metadata) #2
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5, !6}
+!llvm.ident = !{!7}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 10.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, debugInfoForProfiling: true, nameTableKind: None)
+!1 = !DIFile(filename: "test.cpp", directory: "F:\")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 2}
+!6 = !{i32 7, !"PIC Level", i32 2}
+!7 = !{!"clang version 10.0.0"}
+!8 = distinct !DISubprogram(name: "main", scope: !9, file: !9, line: 1, type: !10, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !13)
+!9 = !DIFile(filename: "./test.cpp", directory: "F:\")
+!10 = !DISubroutineType(types: !11)
+!11 = !{!12}
+!12 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!13 = !{!14, !16, !17, !18, !19}
+!14 = !DILocalVariable(name: "foo", scope: !8, file: !9, line: 2, type: !15)
+!15 = !DIDerivedType(tag: DW_TAG_volatile_type, baseType: !12)
+!16 = !DILocalVariable(name: "read1", scope: !8, file: !9, line: 3, type: !12)
+!17 = !DILocalVariable(name: "read2", scope: !8, file: !9, line: 4, type: !12)
+; CHECK: [[VAR_CHEESE]] = !DILocalVariable(name: "cheese"
+!18 = !DILocalVariable(name: "cheese", scope: !8, file: !9, line: 6, type: !12)
+; CHECK: [[VAR_A]] = !DILocalVariable(name: "a"
+!19 = !DILocalVariable(name: "a", scope: !8, file: !9, line: 7, type: !12)
+!20 = !DILocation(line: 2, scope: !8)
+!21 = !{!22, !22, i64 0}
+!22 = !{!"int", !23, i64 0}
+!23 = !{!"omnipotent char", !24, i64 0}
+!24 = !{!"Simple C++ TBAA"}
+!25 = !DILocation(line: 3, scope: !8)
+!26 = !DILocation(line: 0, scope: !8)
+!27 = !DILocation(line: 4, scope: !8)
+!28 = !DILocation(line: 6, scope: !8)
+!29 = !DILocation(line: 7, scope: !8)
+!30 = !DILocation(line: 10, scope: !31)
+!31 = distinct !DILexicalBlock(scope: !8, file: !9, line: 10)
+!32 = !DILocation(line: 10, scope: !8)
+!33 = !DILocation(line: 8, scope: !8)
+!34 = !DILocation(line: 12, scope: !35)
+!35 = distinct !DILexicalBlock(scope: !31, file: !9, line: 10)
+!36 = !DILocation(line: 13, scope: !35)
+!37 = !DILocation(line: 14, scope: !35)
+!38 = !DILocation(line: 15, scope: !35)
+!39 = !DILocation(line: 0, scope: !31)
+!40 = !DILocation(line: 20, scope: !8)
+!41 = !DILocation(line: 19, scope: !8)
diff --git a/llvm/test/Transforms/Reassociate/vaarg_movable.ll b/llvm/test/Transforms/Reassociate/vaarg_movable.ll
index be4fe121f..625597ddc 100644
--- a/llvm/test/Transforms/Reassociate/vaarg_movable.ll
+++ b/llvm/test/Transforms/Reassociate/vaarg_movable.ll
@@ -1,18 +1,25 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -S -reassociate -die < %s | FileCheck %s
 
 ; The two va_arg instructions depend on the memory/context, are therfore not
 ; identical and the sub should not be optimized to 0 by reassociate.
 ;
-; CHECK-LABEL: @func(
 ; ...
-; CHECK: %v0 = va_arg i8** %varargs, i32
-; CHECK: %v1 = va_arg i8** %varargs, i32
-; CHECK: %v0.neg = sub i32 0, %v0
-; CHECK: %sub = add i32 %v0.neg, 1
-; CHECK: %add = add i32 %sub, %v1
 ; ...
-; CHECK: ret i32 %add
 define i32 @func(i32 %dummy, ...) {
+;
+; CHECK-LABEL: @func(
+; CHECK-NEXT:    [[VARARGS:%.*]] = alloca i8*, align 8
+; CHECK-NEXT:    [[VARARGS1:%.*]] = bitcast i8** [[VARARGS]] to i8*
+; CHECK-NEXT:    call void @llvm.va_start(i8* [[VARARGS1]])
+; CHECK-NEXT:    [[V0:%.*]] = va_arg i8** [[VARARGS]], i32
+; CHECK-NEXT:    [[V1:%.*]] = va_arg i8** [[VARARGS]], i32
+; CHECK-NEXT:    [[V0_NEG:%.*]] = sub i32 0, [[V0]]
+; CHECK-NEXT:    [[SUB:%.*]] = add i32 [[V0_NEG]], 1
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[SUB]], [[V1]]
+; CHECK-NEXT:    call void @llvm.va_end(i8* [[VARARGS1]])
+; CHECK-NEXT:    ret i32 [[ADD]]
+;
   %varargs = alloca i8*, align 8
   %varargs1 = bitcast i8** %varargs to i8*
   call void @llvm.va_start(i8* %varargs1)
diff --git a/llvm/test/Transforms/Reassociate/wrap-flags.ll b/llvm/test/Transforms/Reassociate/wrap-flags.ll
index f56719d32..893379bc5 100644
--- a/llvm/test/Transforms/Reassociate/wrap-flags.ll
+++ b/llvm/test/Transforms/Reassociate/wrap-flags.ll
@@ -1,44 +1,59 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -reassociate -dce -S | FileCheck %s
 ; PR12985
 
 ; Verify the nsw flags are preserved when converting shl to mul.
 
-; CHECK-LABEL: @shl_to_mul_nsw(
-; CHECK: %mul = mul i32 %i, -2147483648
-; CHECK: add i32 %mul, 1
 define i32 @shl_to_mul_nsw(i32 %i) {
+;
+; CHECK-LABEL: @shl_to_mul_nsw(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MUL:%.*]] = mul i32 [[I:%.*]], -2147483648
+; CHECK-NEXT:    [[MUL2:%.*]] = add i32 [[MUL]], 1
+; CHECK-NEXT:    ret i32 [[MUL2]]
+;
 entry:
   %mul = shl nsw i32 %i, 31
   %mul2 = add i32 %mul, 1
   ret i32 %mul2
 }
 
-; CHECK-LABEL: @shl_to_mul_nuw(
-; CHECK: %mul = mul nuw i32 %i, 4
-; CHECK: add i32 %mul, 1
 define i32 @shl_to_mul_nuw(i32 %i) {
+;
+; CHECK-LABEL: @shl_to_mul_nuw(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MUL:%.*]] = mul nuw i32 [[I:%.*]], 4
+; CHECK-NEXT:    [[MUL2:%.*]] = add i32 [[MUL]], 1
+; CHECK-NEXT:    ret i32 [[MUL2]]
+;
 entry:
   %mul = shl nuw i32 %i, 2
   %mul2 = add i32 %mul, 1
   ret i32 %mul2
 }
 
-; CHECK-LABEL: @shl_to_mul_nuw_nsw(
-; CHECK: %mul = mul nuw nsw i32 %i, 4
-; CHECK: add i32 %mul, 1
 define i32 @shl_to_mul_nuw_nsw(i32 %i) {
+;
+; CHECK-LABEL: @shl_to_mul_nuw_nsw(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MUL:%.*]] = mul nuw nsw i32 [[I:%.*]], 4
+; CHECK-NEXT:    [[MUL2:%.*]] = add i32 [[MUL]], 1
+; CHECK-NEXT:    ret i32 [[MUL2]]
+;
 entry:
   %mul = shl nuw nsw i32 %i, 2
   %mul2 = add i32 %mul, 1
   ret i32 %mul2
 }
 
-; CHECK-LABEL: @pr23926(
-; CHECK:       %[[X1_neg:.*]] = sub i2 0, %X1
-; CHECK-NEXT:  %[[sub_one:.*]] = add i2 %[[X1_neg]], -1
-; CHECK-NEXT:  %[[add:.*]] = add i2 %[[sub_one]], %X2
-; CHECK-NEXT:  ret i2 %[[add]]
 define i2 @pr23926(i2 %X1, i2 %X2) {
+;
+; CHECK-LABEL: @pr23926(
+; CHECK-NEXT:    [[X1_NEG:%.*]] = sub i2 0, [[X1:%.*]]
+; CHECK-NEXT:    [[ADD_NEG:%.*]] = add i2 [[X1_NEG]], -1
+; CHECK-NEXT:    [[SUB:%.*]] = add i2 [[ADD_NEG]], [[X2:%.*]]
+; CHECK-NEXT:    ret i2 [[SUB]]
+;
   %add = add nuw i2 %X1, 1
   %sub = sub nuw nsw i2 %X2, %add
   ret i2 %sub
diff --git a/llvm/test/Transforms/Reassociate/xor_reassoc.ll b/llvm/test/Transforms/Reassociate/xor_reassoc.ll
index 4d617ea03..81b0311bd 100644
--- a/llvm/test/Transforms/Reassociate/xor_reassoc.ll
+++ b/llvm/test/Transforms/Reassociate/xor_reassoc.ll
@@ -1,243 +1,275 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ;RUN: opt -S -reassociate < %s | FileCheck %s
 
 ; ==========================================================================
 ;
 ;   Xor reassociation general cases
-;  
+;
 ; ==========================================================================
 
 ; (x | c1) ^ (x | c2) => (x & c3) ^ c3, where c3 = c1^c2
-;   
+;
 define i32 @xor1(i32 %x) {
+; CHECK-LABEL: @xor1(
+; CHECK-NEXT:    [[AND_RA:%.*]] = and i32 [[X:%.*]], 435
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[AND_RA]], 435
+; CHECK-NEXT:    ret i32 [[XOR]]
+;
   %or = or i32 %x, 123
   %or1 = or i32 %x, 456
   %xor = xor i32 %or, %or1
   ret i32 %xor
-
-;CHECK-LABEL: @xor1(
-;CHECK: %and.ra = and i32 %x, 435
-;CHECK: %xor = xor i32 %and.ra, 435
 }
 
 ; (x | c1) ^ (x | c2) => (x & c3) ^ c3, where c3 = c1^c2
-;   
+;
 define <2 x i32> @xor1_vec(<2 x i32> %x) {
+; CHECK-LABEL: @xor1_vec(
+; CHECK-NEXT:    [[AND_RA:%.*]] = and <2 x i32> [[X:%.*]], <i32 435, i32 435>
+; CHECK-NEXT:    [[XOR:%.*]] = xor <2 x i32> [[AND_RA]], <i32 435, i32 435>
+; CHECK-NEXT:    ret <2 x i32> [[XOR]]
+;
   %or = or <2 x i32> %x, <i32 123, i32 123>
   %or1 = or <2 x i32> %x, <i32 456, i32 456>
   %xor = xor <2 x i32> %or, %or1
   ret <2 x i32> %xor
-
-;CHECK-LABEL: @xor1_vec(
-;CHECK: %and.ra = and <2 x i32> %x, <i32 435, i32 435>
-;CHECK: %xor = xor <2 x i32> %and.ra, <i32 435, i32 435>
 }
 
 ; Test rule : (x & c1) ^ (x & c2) = (x & (c1^c2))
 ; Real testing case : (x & 123) ^ y ^ (x & 345) => (x & 435) ^ y
 define i32 @xor2(i32 %x, i32 %y) {
+; CHECK-LABEL: @xor2(
+; CHECK-NEXT:    [[AND_RA:%.*]] = and i32 [[X:%.*]], 435
+; CHECK-NEXT:    [[XOR2:%.*]] = xor i32 [[AND_RA]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[XOR2]]
+;
   %and = and i32 %x, 123
   %xor = xor i32 %and, %y
   %and1 = and i32 %x, 456
   %xor2 = xor i32 %xor, %and1
   ret i32 %xor2
-
-;CHECK-LABEL: @xor2(
-;CHECK: %and.ra = and i32 %x, 435
-;CHECK: %xor2 = xor i32 %and.ra, %y
 }
 
 ; Test rule : (x & c1) ^ (x & c2) = (x & (c1^c2))
 ; Real testing case : (x & 123) ^ y ^ (x & 345) => (x & 435) ^ y
 define <2 x i32> @xor2_vec(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @xor2_vec(
+; CHECK-NEXT:    [[AND_RA:%.*]] = and <2 x i32> [[X:%.*]], <i32 435, i32 435>
+; CHECK-NEXT:    [[XOR2:%.*]] = xor <2 x i32> [[AND_RA]], [[Y:%.*]]
+; CHECK-NEXT:    ret <2 x i32> [[XOR2]]
+;
   %and = and <2 x i32> %x, <i32 123, i32 123>
   %xor = xor <2 x i32> %and, %y
   %and1 = and <2 x i32> %x, <i32 456, i32 456>
   %xor2 = xor <2 x i32> %xor, %and1
   ret <2 x i32> %xor2
-
-;CHECK-LABEL: @xor2_vec(
-;CHECK: %and.ra = and <2 x i32> %x, <i32 435, i32 435>
-;CHECK: %xor2 = xor <2 x i32> %and.ra, %y
 }
 
 ; Test rule: (x | c1) ^ (x & c2) = (x & c3) ^ c1, where c3 = ~c1 ^ c2
 ;  c3 = ~c1 ^ c2
 define i32 @xor3(i32 %x, i32 %y) {
+; CHECK-LABEL: @xor3(
+; CHECK-NEXT:    [[AND_RA:%.*]] = and i32 [[X:%.*]], -436
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[Y:%.*]], 123
+; CHECK-NEXT:    [[XOR1:%.*]] = xor i32 [[XOR]], [[AND_RA]]
+; CHECK-NEXT:    ret i32 [[XOR1]]
+;
   %or = or i32 %x, 123
   %xor = xor i32 %or, %y
   %and = and i32 %x, 456
   %xor1 = xor i32 %xor, %and
   ret i32 %xor1
-
-;CHECK-LABEL: @xor3(
-;CHECK: %and.ra = and i32 %x, -436
-;CHECK: %xor = xor i32 %y, 123
-;CHECK: %xor1 = xor i32 %xor, %and.ra
 }
 
 ; Test rule: (x | c1) ^ (x & c2) = (x & c3) ^ c1, where c3 = ~c1 ^ c2
 ;  c3 = ~c1 ^ c2
 define <2 x i32> @xor3_vec(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @xor3_vec(
+; CHECK-NEXT:    [[AND_RA:%.*]] = and <2 x i32> [[X:%.*]], <i32 -436, i32 -436>
+; CHECK-NEXT:    [[XOR:%.*]] = xor <2 x i32> [[Y:%.*]], <i32 123, i32 123>
+; CHECK-NEXT:    [[XOR1:%.*]] = xor <2 x i32> [[XOR]], [[AND_RA]]
+; CHECK-NEXT:    ret <2 x i32> [[XOR1]]
+;
   %or = or <2 x i32> %x, <i32 123, i32 123>
   %xor = xor <2 x i32> %or, %y
   %and = and <2 x i32> %x, <i32 456, i32 456>
   %xor1 = xor <2 x i32> %xor, %and
   ret <2 x i32> %xor1
-
-;CHECK-LABEL: @xor3_vec(
-;CHECK: %and.ra = and <2 x i32> %x, <i32 -436, i32 -436>
-;CHECK: %xor = xor <2 x i32> %y, <i32 123, i32 123>
-;CHECK: %xor1 = xor <2 x i32> %xor, %and.ra
 }
 
 ; Test rule: (x | c1) ^ c2 = (x & ~c1) ^ (c1 ^ c2)
 define i32 @xor4(i32 %x, i32 %y) {
+; CHECK-LABEL: @xor4(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], -124
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[Y:%.*]], 435
+; CHECK-NEXT:    [[XOR1:%.*]] = xor i32 [[XOR]], [[AND]]
+; CHECK-NEXT:    ret i32 [[XOR1]]
+;
   %and = and i32 %x, -124
   %xor = xor i32 %y, 435
   %xor1 = xor i32 %xor, %and
   ret i32 %xor1
-; CHECK-LABEL: @xor4(
-; CHECK: %and = and i32 %x, -124
-; CHECK: %xor = xor i32 %y, 435
-; CHECK: %xor1 = xor i32 %xor, %and
 }
 
 ; Test rule: (x | c1) ^ c2 = (x & ~c1) ^ (c1 ^ c2)
 define <2 x i32> @xor4_vec(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @xor4_vec(
+; CHECK-NEXT:    [[AND:%.*]] = and <2 x i32> [[X:%.*]], <i32 -124, i32 -124>
+; CHECK-NEXT:    [[XOR:%.*]] = xor <2 x i32> [[Y:%.*]], <i32 435, i32 435>
+; CHECK-NEXT:    [[XOR1:%.*]] = xor <2 x i32> [[XOR]], [[AND]]
+; CHECK-NEXT:    ret <2 x i32> [[XOR1]]
+;
   %and = and <2 x i32> %x, <i32 -124, i32 -124>
   %xor = xor <2 x i32> %y, <i32 435, i32 435>
   %xor1 = xor <2 x i32> %xor, %and
   ret <2 x i32> %xor1
-; CHECK-LABEL: @xor4_vec(
-; CHECK: %and = and <2 x i32> %x, <i32 -124, i32 -124>
-; CHECK: %xor = xor <2 x i32> %y, <i32 435, i32 435>
-; CHECK: %xor1 = xor <2 x i32> %xor, %and
 }
 
 ; ==========================================================================
 ;
 ;  Xor reassociation special cases
-;  
+;
 ; ==========================================================================
 
-; Special case1: 
+; Special case1:
 ;  (x | c1) ^ (x & ~c1) = c1
 define i32 @xor_special1(i32 %x, i32 %y) {
+; CHECK-LABEL: @xor_special1(
+; CHECK-NEXT:    [[XOR1:%.*]] = xor i32 [[Y:%.*]], 123
+; CHECK-NEXT:    ret i32 [[XOR1]]
+;
   %or = or i32 %x, 123
   %xor = xor i32 %or, %y
   %and = and i32 %x, -124
   %xor1 = xor i32 %xor, %and
   ret i32 %xor1
-; CHECK-LABEL: @xor_special1(
-; CHECK: %xor1 = xor i32 %y, 123
-; CHECK: ret i32 %xor1
 }
 
-; Special case1: 
+; Special case1:
 ;  (x | c1) ^ (x & ~c1) = c1
 define <2 x i32> @xor_special1_vec(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @xor_special1_vec(
+; CHECK-NEXT:    [[XOR1:%.*]] = xor <2 x i32> [[Y:%.*]], <i32 123, i32 123>
+; CHECK-NEXT:    ret <2 x i32> [[XOR1]]
+;
   %or = or <2 x i32> %x, <i32 123, i32 123>
   %xor = xor <2 x i32> %or, %y
   %and = and <2 x i32> %x, <i32 -124, i32 -124>
   %xor1 = xor <2 x i32> %xor, %and
   ret <2 x i32> %xor1
-; CHECK-LABEL: @xor_special1_vec(
-; CHECK: %xor1 = xor <2 x i32> %y, <i32 123, i32 123>
-; CHECK: ret <2 x i32> %xor1
 }
 
-; Special case1: 
+; Special case1:
 ;  (x | c1) ^ (x & c1) = x ^ c1
 define i32 @xor_special2(i32 %x, i32 %y) {
+; CHECK-LABEL: @xor_special2(
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[X:%.*]], 123
+; CHECK-NEXT:    [[XOR1:%.*]] = xor i32 [[XOR]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[XOR1]]
+;
   %or = or i32 %x, 123
   %xor = xor i32 %or, %y
   %and = and i32 %x, 123
   %xor1 = xor i32 %xor, %and
   ret i32 %xor1
-; CHECK-LABEL: @xor_special2(
-; CHECK: %xor = xor i32 %x, 123
-; CHECK: %xor1 = xor i32 %xor, %y
-; CHECK: ret i32 %xor1
 }
 
-; Special case1: 
+; Special case1:
 ;  (x | c1) ^ (x & c1) = x ^ c1
 define <2 x i32> @xor_special2_vec(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @xor_special2_vec(
+; CHECK-NEXT:    [[XOR:%.*]] = xor <2 x i32> [[X:%.*]], <i32 123, i32 123>
+; CHECK-NEXT:    [[XOR1:%.*]] = xor <2 x i32> [[XOR]], [[Y:%.*]]
+; CHECK-NEXT:    ret <2 x i32> [[XOR1]]
+;
   %or = or <2 x i32> %x, <i32 123, i32 123>
   %xor = xor <2 x i32> %or, %y
   %and = and <2 x i32> %x, <i32 123, i32 123>
   %xor1 = xor <2 x i32> %xor, %and
   ret <2 x i32> %xor1
-; CHECK-LABEL: @xor_special2_vec(
-; CHECK: %xor = xor <2 x i32> %x, <i32 123, i32 123>
-; CHECK: %xor1 = xor <2 x i32> %xor, %y
-; CHECK: ret <2 x i32> %xor1
 }
 
 ; (x | c1) ^ (x | c1) => 0
 define i32 @xor_special3(i32 %x) {
+; CHECK-LABEL: @xor_special3(
+; CHECK-NEXT:    ret i32 0
+;
   %or = or i32 %x, 123
   %or1 = or i32 %x, 123
   %xor = xor i32 %or, %or1
   ret i32 %xor
-;CHECK-LABEL: @xor_special3(
-;CHECK: ret i32 0
 }
 
 ; (x | c1) ^ (x | c1) => 0
 define <2 x i32> @xor_special3_vec(<2 x i32> %x) {
+; CHECK-LABEL: @xor_special3_vec(
+; CHECK-NEXT:    ret <2 x i32> zeroinitializer
+;
   %or = or <2 x i32> %x, <i32 123, i32 123>
   %or1 = or <2 x i32> %x, <i32 123, i32 123>
   %xor = xor <2 x i32> %or, %or1
   ret <2 x i32> %xor
-;CHECK-LABEL: @xor_special3_vec(
-;CHECK: ret <2 x i32> zeroinitializer
 }
 
 ; (x & c1) ^ (x & c1) => 0
 define i32 @xor_special4(i32 %x) {
+; CHECK-LABEL: @xor_special4(
+; CHECK-NEXT:    ret i32 0
+;
   %or = and i32 %x, 123
   %or1 = and i32 123, %x
   %xor = xor i32 %or, %or1
   ret i32 %xor
-;CHECK-LABEL: @xor_special4(
-;CHECK: ret i32 0
 }
 
 ; (x & c1) ^ (x & c1) => 0
 define <2 x i32> @xor_special4_vec(<2 x i32> %x) {
+; CHECK-LABEL: @xor_special4_vec(
+; CHECK-NEXT:    ret <2 x i32> zeroinitializer
+;
   %or = and <2 x i32> %x, <i32 123, i32 123>
   %or1 = and <2 x i32> <i32 123, i32 123>, %x
   %xor = xor <2 x i32> %or, %or1
   ret <2 x i32> %xor
-;CHECK-LABEL: @xor_special4_vec(
-;CHECK: ret <2 x i32> zeroinitializer
 }
 
 ; ==========================================================================
 ;
 ;  Xor reassociation curtail code size
-;  
+;
 ; ==========================================================================
 
 ; (x | c1) ^ (x | c2) => (x & c3) ^ c3
 ; is enabled if one of operands has multiple uses
-;   
+;
 define i32 @xor_ra_size1(i32 %x) {
+; CHECK-LABEL: @xor_ra_size1(
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[X:%.*]], 123
+; CHECK-NEXT:    [[AND_RA:%.*]] = and i32 [[X]], 435
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[AND_RA]], 435
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[XOR]], [[OR]]
+; CHECK-NEXT:    ret i32 [[ADD]]
+;
   %or = or i32 %x, 123
   %or1 = or i32 %x, 456
   %xor = xor i32 %or, %or1
 
   %add = add i32 %xor, %or
   ret i32 %add
-;CHECK-LABEL: @xor_ra_size1(
-;CHECK: %xor = xor i32 %and.ra, 435
 }
 
 ; (x | c1) ^ (x | c2) => (x & c3) ^ c3
 ; is disenabled if bothf operands has multiple uses.
-;   
+;
 define i32 @xor_ra_size2(i32 %x) {
+; CHECK-LABEL: @xor_ra_size2(
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[X:%.*]], 123
+; CHECK-NEXT:    [[OR1:%.*]] = or i32 [[X]], 456
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[OR]], [[OR1]]
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[OR1]], [[OR]]
+; CHECK-NEXT:    [[ADD2:%.*]] = add i32 [[ADD]], [[XOR]]
+; CHECK-NEXT:    ret i32 [[ADD2]]
+;
   %or = or i32 %x, 123
   %or1 = or i32 %x, 456
   %xor = xor i32 %or, %or1
@@ -246,20 +278,22 @@ define i32 @xor_ra_size2(i32 %x) {
   %add2 = add i32 %add, %or1
   ret i32 %add2
 
-;CHECK-LABEL: @xor_ra_size2(
-;CHECK: %or1 = or i32 %x, 456
-;CHECK: %xor = xor i32 %or, %or1
 }
 
 
 ; ==========================================================================
 ;
 ;  Xor reassociation bugs
-;  
+;
 ; ==========================================================================
 
 @xor_bug1_data = external global <{}>, align 4
 define void @xor_bug1() {
+; CHECK-LABEL: @xor_bug1(
+; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint i32* undef to i64
+; CHECK-NEXT:    [[TMP2:%.*]] = xor i64 [[TMP1]], ptrtoint (<{}>* @xor_bug1_data to i64)
+; CHECK-NEXT:    ret void
+;
   %1 = ptrtoint i32* undef to i64
   %2 = xor i64 %1, ptrtoint (<{}>* @xor_bug1_data to i64)
   %3 = and i64 undef, %2
@@ -272,6 +306,22 @@ define void @xor_bug1() {
 ; of c1 and c2 accordingly, hence cause the problem.
 ;
 define i32 @xor_bug2(i32, i32, i32, i32) {
+; CHECK-LABEL: @xor_bug2(
+; CHECK-NEXT:    [[TMP5:%.*]] = mul i32 [[TMP0:%.*]], 123
+; CHECK-NEXT:    [[TMP6:%.*]] = and i32 [[TMP1:%.*]], 3456789
+; CHECK-NEXT:    [[TMP7:%.*]] = or i32 [[TMP6]], 4567890
+; CHECK-NEXT:    [[TMP8:%.*]] = and i32 [[TMP2:%.*]], 255
+; CHECK-NEXT:    [[AND_RA:%.*]] = and i32 [[TMP1]], -360490541
+; CHECK-NEXT:    [[TMP9:%.*]] = xor i32 [[TMP5]], 891034567
+; CHECK-NEXT:    [[TMP10:%.*]] = xor i32 [[TMP9]], [[AND_RA]]
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i32 [[TMP10]], [[TMP7]]
+; CHECK-NEXT:    [[TMP12:%.*]] = and i32 [[TMP3:%.*]], 255
+; CHECK-NEXT:    [[TMP13:%.*]] = add i32 [[TMP1]], 32
+; CHECK-NEXT:    [[TMP14:%.*]] = add i32 [[TMP13]], [[TMP2]]
+; CHECK-NEXT:    [[TMP15:%.*]] = add i32 [[TMP14]], [[TMP8]]
+; CHECK-NEXT:    [[TMP16:%.*]] = add i32 [[TMP15]], [[TMP11]]
+; CHECK-NEXT:    ret i32 [[TMP16]]
+;
   %5 = mul i32 %0, 123
   %6 = add i32 %2, 24
   %7 = add i32 %1, 8
@@ -289,6 +339,4 @@ define i32 @xor_bug2(i32, i32, i32, i32) {
   %19 = add i32 %18, %12
   %20 = add i32 %19, %15
   ret i32 %20
-;CHECK-LABEL: @xor_bug2(
-;CHECK: xor i32 %5, 891034567
 }
diff --git a/llvm/test/Transforms/SCCP/apint-basictest3.ll b/llvm/test/Transforms/SCCP/apint-basictest3.ll
index b8fcca6fd..863902c07 100644
--- a/llvm/test/Transforms/SCCP/apint-basictest3.ll
+++ b/llvm/test/Transforms/SCCP/apint-basictest3.ll
@@ -1,5 +1,5 @@
 ; This is a basic sanity check for constant propagation.  It tests the basic
-; arithmatic operations.
+; arithmetic operations.
 
 
 ; RUN: opt < %s -sccp -S | not grep mul
diff --git a/llvm/test/Transforms/SCCP/indirectbr.ll b/llvm/test/Transforms/SCCP/indirectbr.ll
index b977961ca..f54da3fb5 100644
--- a/llvm/test/Transforms/SCCP/indirectbr.ll
+++ b/llvm/test/Transforms/SCCP/indirectbr.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -sccp < %s | FileCheck %s
+; RUN: opt -S -ipsccp < %s | FileCheck %s
 
 declare void @BB0_f()
 declare void @BB1_f()
@@ -74,3 +74,35 @@ BB1:
 }
 
 
+; CHECK-LABEL: define internal i32 @indbrtest5(
+; CHECK: ret i32 undef
+define internal i32 @indbrtest5(i1 %c) {
+entry:
+  br i1 %c, label %bb1, label %bb2
+
+bb1:
+  br label %branch.block
+
+
+bb2:
+  br label %branch.block
+
+branch.block:
+  %addr = phi i8* [blockaddress(@indbrtest5, %target1), %bb1], [blockaddress(@indbrtest5, %target2), %bb2]
+  indirectbr i8* %addr, [label %target1, label %target2]
+
+target1:
+  br label %target2
+
+target2:
+  ret i32 10
+}
+
+
+define i32 @indbrtest5_callee(i1 %c) {
+; CHECK-LABEL: define i32 @indbrtest5_callee(
+; CHECK-NEXT:    %r = call i32 @indbrtest5(i1 %c)
+; CHECK-NEXT:    ret i32 10
+  %r = call i32 @indbrtest5(i1 %c)
+  ret i32 %r
+}
diff --git a/llvm/test/Transforms/SCCP/ipsccp-predinfo-order.ll b/llvm/test/Transforms/SCCP/ipsccp-predinfo-order.ll
new file mode 100644
index 000000000..2474a19b0
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/ipsccp-predinfo-order.ll
@@ -0,0 +1,76 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -ipsccp -S %s | FileCheck %s
+
+declare i32 @hoge()
+
+define dso_local i32 @ham(i8* %arg, i8* %arg1) {
+; CHECK-LABEL: @ham(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP:%.*]] = alloca i32
+; CHECK-NEXT:    [[TMP2:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    br label [[BB19:%.*]]
+; CHECK:       bb4:
+; CHECK-NEXT:    br label [[BB6:%.*]]
+; CHECK:       bb6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call i32 @hoge()
+; CHECK-NEXT:    store i32 [[TMP7]], i32* [[TMP]]
+; CHECK-NEXT:    [[TMP8:%.*]] = load i32, i32* [[TMP]]
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[TMP8]], 912730082
+; CHECK-NEXT:    [[TMP10:%.*]] = load i32, i32* [[TMP]]
+; CHECK-NEXT:    br i1 [[TMP9]], label [[BB11:%.*]], label [[BB16:%.*]]
+; CHECK:       bb11:
+; CHECK-NEXT:    unreachable
+; CHECK:       bb13:
+; CHECK-NEXT:    br label [[BB14:%.*]]
+; CHECK:       bb14:
+; CHECK-NEXT:    [[TMP15:%.*]] = load i32, i32* [[TMP]]
+; CHECK-NEXT:    br label [[BB16]]
+; CHECK:       bb16:
+; CHECK-NEXT:    [[TMP17:%.*]] = phi i32 [ [[TMP10]], [[BB6]] ], [ 0, [[BB14]] ]
+; CHECK-NEXT:    br label [[BB19]]
+; CHECK:       bb18:
+; CHECK-NEXT:    unreachable
+; CHECK:       bb19:
+; CHECK-NEXT:    br label [[BB20:%.*]]
+; CHECK:       bb20:
+; CHECK-NEXT:    indirectbr i8* null, [label [[BB4:%.*]], label [[BB13:%.*]], label %bb18]
+;
+bb:
+  %tmp = alloca i32
+  %tmp2 = alloca i32, align 4
+  br label %bb19
+
+bb4:                                              ; preds = %bb20
+  br label %bb6
+
+bb6:                                              ; preds = %bb4
+  %tmp7 = call i32 @hoge()
+  store i32 %tmp7, i32* %tmp
+  %tmp8 = load i32, i32* %tmp
+  %tmp9 = icmp eq i32 %tmp8, 912730082
+  %tmp10 = load i32, i32* %tmp
+  br i1 %tmp9, label %bb11, label %bb16
+
+bb11:                                             ; preds = %bb6
+  unreachable
+
+bb13:                                             ; preds = %bb20
+  br label %bb14
+
+bb14:                                             ; preds = %bb13
+  %tmp15 = load i32, i32* %tmp
+  br label %bb16
+
+bb16:                                             ; preds = %bb14, %bb6
+  %tmp17 = phi i32 [ %tmp10, %bb6 ], [ 0, %bb14 ]
+  br label %bb19
+
+bb18:                                             ; preds = %bb20
+  unreachable
+
+bb19:                                             ; preds = %bb16, %bb
+  br label %bb20
+
+bb20:                                             ; preds = %bb19
+  indirectbr i8* null, [label %bb4, label %bb13, label %bb18]
+}
diff --git a/llvm/test/Transforms/SCCP/struct-arg-resolve-undefs.ll b/llvm/test/Transforms/SCCP/struct-arg-resolve-undefs.ll
new file mode 100644
index 000000000..9f62e71d9
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/struct-arg-resolve-undefs.ll
@@ -0,0 +1,49 @@
+; RUN: opt -ipsccp -S %s | FileCheck %s
+
+%struct.S = type { i32 }
+
+
+define void @main() {
+; CHECK-LABEL: void @main() {
+; CHECK-NEXT:    %r = call i32 @f(%struct.S { i32 100 })
+; CHECK-NEXT:    call void @do_report(i32 123)
+  %r = call i32 @f(%struct.S { i32 100 })
+  call void @do_report(i32 %r)
+  ret void
+}
+
+declare void @do_report(i32)
+
+define internal i32 @f(%struct.S %s.coerce) {
+; CHECK-LABEL: define internal i32 @f(%struct.S %s.coerce)
+; CHECK-LABEL: entry:
+; CHECK-NEXT:     %call = call i8 @lsh(i8 1, i32 100)
+; CHECK-LABEL: if.end:
+; CHECK-NEXT:     ret i32 undef
+entry:
+  %ev = extractvalue %struct.S %s.coerce, 0
+  %call = call i8 @lsh(i8 1, i32 %ev)
+  %tobool = icmp ne i8 %call, 0
+  br i1 %tobool, label %for.cond, label %if.end
+
+for.cond:                                         ; preds = %for.cond, %if.then
+  %i.0 = phi i32 [ 0, %entry], [ %inc, %for.cond ]
+  %cmp = icmp slt i32 %i.0, 1
+  %inc = add nsw i32 %i.0, 1
+  br i1 %cmp, label %for.cond, label %if.end
+
+if.end:                                           ; preds = %for.cond, %entry
+  ret i32 123
+}
+
+define internal i8 @lsh(i8 %l, i32 %r) {
+entry:
+  %conv = sext i8 %l to i32
+  %cmp = icmp slt i32 %conv, 0
+  %shr = ashr i32 127, %r
+  %cmp4 = icmp sgt i32 %conv, %shr
+  %or.cond13 = or i1 %cmp, %cmp4
+  %cond = select i1 %or.cond13, i32 %conv, i32 0
+  %conv7 = trunc i32 %cond to i8
+  ret i8 %conv7
+}
diff --git a/llvm/test/Transforms/SCCP/ubsan_overflow.ll b/llvm/test/Transforms/SCCP/ubsan_overflow.ll
new file mode 100644
index 000000000..a207e979a
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/ubsan_overflow.ll
@@ -0,0 +1,13 @@
+; RUN: opt -sccp -S %s | FileCheck %s
+
+@0 = private unnamed_addr constant [16 x i8] c"\01\00\00\00\01\01\00\00\01\01\01\00\01\01\01\01"
+
+; CHECK-LABEL: @foo
+define i8 @foo() {
+entry:
+  %0 = add nuw nsw i64 0, -1
+  %1 = lshr i64 %0, 1
+  %2 = getelementptr inbounds [4 x [4 x i8]], [4 x [4 x i8]]* bitcast ([16 x i8]* @0 to [4 x [4 x i8]]*), i64 0, i64 0, i64 %1
+  %3 = load i8, i8* %2, align 1
+  ret i8 %3
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/ext-trunc.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/ext-trunc.ll
index 8e36a921c..58da29925 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/ext-trunc.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/ext-trunc.ll
@@ -61,23 +61,25 @@ define void @test2(<4 x i16> %a, <4 x i16> %b, i64 %c0, i64 %c1, i64 %c2, i64 %c
 ; CHECK-NEXT:    [[Z0:%.*]] = zext <4 x i16> [[A:%.*]] to <4 x i32>
 ; CHECK-NEXT:    [[Z1:%.*]] = zext <4 x i16> [[B:%.*]] to <4 x i32>
 ; CHECK-NEXT:    [[SUB0:%.*]] = sub <4 x i32> [[Z0]], [[Z1]]
-; CHECK-NEXT:    [[TMP0:%.*]] = sext <4 x i32> [[SUB0]] to <4 x i64>
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i64> undef, i64 [[C0:%.*]], i32 0
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x i64> [[TMP1]], i64 [[C1:%.*]], i32 1
-; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x i64> [[TMP2]], i64 [[C2:%.*]], i32 2
-; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <4 x i64> [[TMP3]], i64 [[C3:%.*]], i32 3
-; CHECK-NEXT:    [[TMP5:%.*]] = add <4 x i64> [[TMP0]], [[TMP4]]
-; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x i64> [[TMP5]], i32 0
-; CHECK-NEXT:    [[GEP0:%.*]] = getelementptr inbounds i64, i64* [[P:%.*]], i64 [[TMP6]]
+; CHECK-NEXT:    [[E0:%.*]] = extractelement <4 x i32> [[SUB0]], i32 0
+; CHECK-NEXT:    [[S0:%.*]] = sext i32 [[E0]] to i64
+; CHECK-NEXT:    [[A0:%.*]] = add i64 [[S0]], [[C0:%.*]]
+; CHECK-NEXT:    [[GEP0:%.*]] = getelementptr inbounds i64, i64* [[P:%.*]], i64 [[A0]]
 ; CHECK-NEXT:    [[LOAD0:%.*]] = load i64, i64* [[GEP0]]
-; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x i64> [[TMP5]], i32 1
-; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr inbounds i64, i64* [[P]], i64 [[TMP7]]
+; CHECK-NEXT:    [[E1:%.*]] = extractelement <4 x i32> [[SUB0]], i32 1
+; CHECK-NEXT:    [[S1:%.*]] = sext i32 [[E1]] to i64
+; CHECK-NEXT:    [[A1:%.*]] = add i64 [[S1]], [[C1:%.*]]
+; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr inbounds i64, i64* [[P]], i64 [[A1]]
 ; CHECK-NEXT:    [[LOAD1:%.*]] = load i64, i64* [[GEP1]]
-; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <4 x i64> [[TMP5]], i32 2
-; CHECK-NEXT:    [[GEP2:%.*]] = getelementptr inbounds i64, i64* [[P]], i64 [[TMP8]]
+; CHECK-NEXT:    [[E2:%.*]] = extractelement <4 x i32> [[SUB0]], i32 2
+; CHECK-NEXT:    [[S2:%.*]] = sext i32 [[E2]] to i64
+; CHECK-NEXT:    [[A2:%.*]] = add i64 [[S2]], [[C2:%.*]]
+; CHECK-NEXT:    [[GEP2:%.*]] = getelementptr inbounds i64, i64* [[P]], i64 [[A2]]
 ; CHECK-NEXT:    [[LOAD2:%.*]] = load i64, i64* [[GEP2]]
-; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <4 x i64> [[TMP5]], i32 3
-; CHECK-NEXT:    [[GEP3:%.*]] = getelementptr inbounds i64, i64* [[P]], i64 [[TMP9]]
+; CHECK-NEXT:    [[E3:%.*]] = extractelement <4 x i32> [[SUB0]], i32 3
+; CHECK-NEXT:    [[S3:%.*]] = sext i32 [[E3]] to i64
+; CHECK-NEXT:    [[A3:%.*]] = add i64 [[S3]], [[C3:%.*]]
+; CHECK-NEXT:    [[GEP3:%.*]] = getelementptr inbounds i64, i64* [[P]], i64 [[A3]]
 ; CHECK-NEXT:    [[LOAD3:%.*]] = load i64, i64* [[GEP3]]
 ; CHECK-NEXT:    call void @foo(i64 [[LOAD0]], i64 [[LOAD1]], i64 [[LOAD2]], i64 [[LOAD3]])
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/gather-root.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/gather-root.ll
index 5e6db8b48..5f8cbd5c8 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/gather-root.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/gather-root.ll
@@ -17,16 +17,8 @@ define void @PR28330(i32 %n) {
 ; DEFAULT:       for.body:
 ; DEFAULT-NEXT:    [[P17:%.*]] = phi i32 [ [[OP_EXTRA:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
 ; DEFAULT-NEXT:    [[TMP2:%.*]] = select <8 x i1> [[TMP1]], <8 x i32> <i32 -720, i32 -720, i32 -720, i32 -720, i32 -720, i32 -720, i32 -720, i32 -720>, <8 x i32> <i32 -80, i32 -80, i32 -80, i32 -80, i32 -80, i32 -80, i32 -80, i32 -80>
-; DEFAULT-NEXT:    [[P20:%.*]] = add i32 [[P17]], undef
-; DEFAULT-NEXT:    [[P22:%.*]] = add i32 [[P20]], undef
-; DEFAULT-NEXT:    [[P24:%.*]] = add i32 [[P22]], undef
-; DEFAULT-NEXT:    [[P26:%.*]] = add i32 [[P24]], undef
-; DEFAULT-NEXT:    [[P28:%.*]] = add i32 [[P26]], undef
-; DEFAULT-NEXT:    [[P30:%.*]] = add i32 [[P28]], undef
-; DEFAULT-NEXT:    [[P32:%.*]] = add i32 [[P30]], undef
 ; DEFAULT-NEXT:    [[TMP3:%.*]] = call i32 @llvm.experimental.vector.reduce.add.v8i32(<8 x i32> [[TMP2]])
 ; DEFAULT-NEXT:    [[OP_EXTRA]] = add i32 [[TMP3]], [[P17]]
-; DEFAULT-NEXT:    [[P34:%.*]] = add i32 [[P32]], undef
 ; DEFAULT-NEXT:    br label [[FOR_BODY]]
 ;
 ; GATHER-LABEL: @PR28330(
@@ -36,37 +28,30 @@ define void @PR28330(i32 %n) {
 ; GATHER-NEXT:    br label [[FOR_BODY:%.*]]
 ; GATHER:       for.body:
 ; GATHER-NEXT:    [[P17:%.*]] = phi i32 [ [[OP_EXTRA:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
-; GATHER-NEXT:    [[TMP2:%.*]] = extractelement <8 x i1> [[TMP1]], i32 0
-; GATHER-NEXT:    [[TMP3:%.*]] = insertelement <8 x i1> undef, i1 [[TMP2]], i32 0
-; GATHER-NEXT:    [[TMP4:%.*]] = extractelement <8 x i1> [[TMP1]], i32 1
-; GATHER-NEXT:    [[TMP5:%.*]] = insertelement <8 x i1> [[TMP3]], i1 [[TMP4]], i32 1
-; GATHER-NEXT:    [[TMP6:%.*]] = extractelement <8 x i1> [[TMP1]], i32 2
-; GATHER-NEXT:    [[TMP7:%.*]] = insertelement <8 x i1> [[TMP5]], i1 [[TMP6]], i32 2
-; GATHER-NEXT:    [[TMP8:%.*]] = extractelement <8 x i1> [[TMP1]], i32 3
-; GATHER-NEXT:    [[TMP9:%.*]] = insertelement <8 x i1> [[TMP7]], i1 [[TMP8]], i32 3
-; GATHER-NEXT:    [[TMP10:%.*]] = extractelement <8 x i1> [[TMP1]], i32 4
-; GATHER-NEXT:    [[TMP11:%.*]] = insertelement <8 x i1> [[TMP9]], i1 [[TMP10]], i32 4
-; GATHER-NEXT:    [[TMP12:%.*]] = extractelement <8 x i1> [[TMP1]], i32 5
-; GATHER-NEXT:    [[TMP13:%.*]] = insertelement <8 x i1> [[TMP11]], i1 [[TMP12]], i32 5
-; GATHER-NEXT:    [[TMP14:%.*]] = extractelement <8 x i1> [[TMP1]], i32 6
-; GATHER-NEXT:    [[TMP15:%.*]] = insertelement <8 x i1> [[TMP13]], i1 [[TMP14]], i32 6
-; GATHER-NEXT:    [[TMP16:%.*]] = extractelement <8 x i1> [[TMP1]], i32 7
-; GATHER-NEXT:    [[TMP17:%.*]] = insertelement <8 x i1> [[TMP15]], i1 [[TMP16]], i32 7
+; GATHER-NEXT:    [[TMP2:%.*]] = extractelement <8 x i1> [[TMP1]], i32 7
+; GATHER-NEXT:    [[TMP3:%.*]] = extractelement <8 x i1> [[TMP1]], i32 0
+; GATHER-NEXT:    [[TMP4:%.*]] = insertelement <8 x i1> undef, i1 [[TMP3]], i32 0
+; GATHER-NEXT:    [[TMP5:%.*]] = extractelement <8 x i1> [[TMP1]], i32 1
+; GATHER-NEXT:    [[TMP6:%.*]] = insertelement <8 x i1> [[TMP4]], i1 [[TMP5]], i32 1
+; GATHER-NEXT:    [[TMP7:%.*]] = extractelement <8 x i1> [[TMP1]], i32 2
+; GATHER-NEXT:    [[TMP8:%.*]] = insertelement <8 x i1> [[TMP6]], i1 [[TMP7]], i32 2
+; GATHER-NEXT:    [[TMP9:%.*]] = extractelement <8 x i1> [[TMP1]], i32 3
+; GATHER-NEXT:    [[TMP10:%.*]] = insertelement <8 x i1> [[TMP8]], i1 [[TMP9]], i32 3
+; GATHER-NEXT:    [[TMP11:%.*]] = extractelement <8 x i1> [[TMP1]], i32 4
+; GATHER-NEXT:    [[TMP12:%.*]] = insertelement <8 x i1> [[TMP10]], i1 [[TMP11]], i32 4
+; GATHER-NEXT:    [[TMP13:%.*]] = extractelement <8 x i1> [[TMP1]], i32 5
+; GATHER-NEXT:    [[TMP14:%.*]] = insertelement <8 x i1> [[TMP12]], i1 [[TMP13]], i32 5
+; GATHER-NEXT:    [[TMP15:%.*]] = extractelement <8 x i1> [[TMP1]], i32 6
+; GATHER-NEXT:    [[TMP16:%.*]] = insertelement <8 x i1> [[TMP14]], i1 [[TMP15]], i32 6
+; GATHER-NEXT:    [[TMP17:%.*]] = insertelement <8 x i1> [[TMP16]], i1 [[TMP2]], i32 7
 ; GATHER-NEXT:    [[TMP18:%.*]] = select <8 x i1> [[TMP17]], <8 x i32> <i32 -720, i32 -720, i32 -720, i32 -720, i32 -720, i32 -720, i32 -720, i32 -720>, <8 x i32> <i32 -80, i32 -80, i32 -80, i32 -80, i32 -80, i32 -80, i32 -80, i32 -80>
 ; GATHER-NEXT:    [[TMP19:%.*]] = extractelement <8 x i32> [[TMP18]], i32 0
-; GATHER-NEXT:    [[P20:%.*]] = add i32 [[P17]], [[TMP19]]
 ; GATHER-NEXT:    [[TMP20:%.*]] = extractelement <8 x i32> [[TMP18]], i32 1
-; GATHER-NEXT:    [[P22:%.*]] = add i32 [[P20]], [[TMP20]]
 ; GATHER-NEXT:    [[TMP21:%.*]] = extractelement <8 x i32> [[TMP18]], i32 2
-; GATHER-NEXT:    [[P24:%.*]] = add i32 [[P22]], [[TMP21]]
 ; GATHER-NEXT:    [[TMP22:%.*]] = extractelement <8 x i32> [[TMP18]], i32 3
-; GATHER-NEXT:    [[P26:%.*]] = add i32 [[P24]], [[TMP22]]
 ; GATHER-NEXT:    [[TMP23:%.*]] = extractelement <8 x i32> [[TMP18]], i32 4
-; GATHER-NEXT:    [[P28:%.*]] = add i32 [[P26]], [[TMP23]]
 ; GATHER-NEXT:    [[TMP24:%.*]] = extractelement <8 x i32> [[TMP18]], i32 5
-; GATHER-NEXT:    [[P30:%.*]] = add i32 [[P28]], [[TMP24]]
 ; GATHER-NEXT:    [[TMP25:%.*]] = extractelement <8 x i32> [[TMP18]], i32 6
-; GATHER-NEXT:    [[P32:%.*]] = add i32 [[P30]], [[TMP25]]
 ; GATHER-NEXT:    [[TMP26:%.*]] = insertelement <8 x i32> undef, i32 [[TMP19]], i32 0
 ; GATHER-NEXT:    [[TMP27:%.*]] = insertelement <8 x i32> [[TMP26]], i32 [[TMP20]], i32 1
 ; GATHER-NEXT:    [[TMP28:%.*]] = insertelement <8 x i32> [[TMP27]], i32 [[TMP21]], i32 2
@@ -78,7 +63,6 @@ define void @PR28330(i32 %n) {
 ; GATHER-NEXT:    [[TMP34:%.*]] = insertelement <8 x i32> [[TMP32]], i32 [[TMP33]], i32 7
 ; GATHER-NEXT:    [[TMP35:%.*]] = call i32 @llvm.experimental.vector.reduce.add.v8i32(<8 x i32> [[TMP34]])
 ; GATHER-NEXT:    [[OP_EXTRA]] = add i32 [[TMP35]], [[P17]]
-; GATHER-NEXT:    [[P34:%.*]] = add i32 [[P32]], [[TMP33]]
 ; GATHER-NEXT:    br label [[FOR_BODY]]
 ;
 ; MAX-COST-LABEL: @PR28330(
@@ -169,16 +153,8 @@ define void @PR32038(i32 %n) {
 ; DEFAULT:       for.body:
 ; DEFAULT-NEXT:    [[P17:%.*]] = phi i32 [ [[OP_EXTRA:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
 ; DEFAULT-NEXT:    [[TMP2:%.*]] = select <8 x i1> [[TMP1]], <8 x i32> <i32 -720, i32 -720, i32 -720, i32 -720, i32 -720, i32 -720, i32 -720, i32 -720>, <8 x i32> <i32 -80, i32 -80, i32 -80, i32 -80, i32 -80, i32 -80, i32 -80, i32 -80>
-; DEFAULT-NEXT:    [[P20:%.*]] = add i32 -5, undef
-; DEFAULT-NEXT:    [[P22:%.*]] = add i32 [[P20]], undef
-; DEFAULT-NEXT:    [[P24:%.*]] = add i32 [[P22]], undef
-; DEFAULT-NEXT:    [[P26:%.*]] = add i32 [[P24]], undef
-; DEFAULT-NEXT:    [[P28:%.*]] = add i32 [[P26]], undef
-; DEFAULT-NEXT:    [[P30:%.*]] = add i32 [[P28]], undef
-; DEFAULT-NEXT:    [[P32:%.*]] = add i32 [[P30]], undef
 ; DEFAULT-NEXT:    [[TMP3:%.*]] = call i32 @llvm.experimental.vector.reduce.add.v8i32(<8 x i32> [[TMP2]])
 ; DEFAULT-NEXT:    [[OP_EXTRA]] = add i32 [[TMP3]], -5
-; DEFAULT-NEXT:    [[P34:%.*]] = add i32 [[P32]], undef
 ; DEFAULT-NEXT:    br label [[FOR_BODY]]
 ;
 ; GATHER-LABEL: @PR32038(
@@ -188,37 +164,30 @@ define void @PR32038(i32 %n) {
 ; GATHER-NEXT:    br label [[FOR_BODY:%.*]]
 ; GATHER:       for.body:
 ; GATHER-NEXT:    [[P17:%.*]] = phi i32 [ [[OP_EXTRA:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
-; GATHER-NEXT:    [[TMP2:%.*]] = extractelement <8 x i1> [[TMP1]], i32 0
-; GATHER-NEXT:    [[TMP3:%.*]] = insertelement <8 x i1> undef, i1 [[TMP2]], i32 0
-; GATHER-NEXT:    [[TMP4:%.*]] = extractelement <8 x i1> [[TMP1]], i32 1
-; GATHER-NEXT:    [[TMP5:%.*]] = insertelement <8 x i1> [[TMP3]], i1 [[TMP4]], i32 1
-; GATHER-NEXT:    [[TMP6:%.*]] = extractelement <8 x i1> [[TMP1]], i32 2
-; GATHER-NEXT:    [[TMP7:%.*]] = insertelement <8 x i1> [[TMP5]], i1 [[TMP6]], i32 2
-; GATHER-NEXT:    [[TMP8:%.*]] = extractelement <8 x i1> [[TMP1]], i32 3
-; GATHER-NEXT:    [[TMP9:%.*]] = insertelement <8 x i1> [[TMP7]], i1 [[TMP8]], i32 3
-; GATHER-NEXT:    [[TMP10:%.*]] = extractelement <8 x i1> [[TMP1]], i32 4
-; GATHER-NEXT:    [[TMP11:%.*]] = insertelement <8 x i1> [[TMP9]], i1 [[TMP10]], i32 4
-; GATHER-NEXT:    [[TMP12:%.*]] = extractelement <8 x i1> [[TMP1]], i32 5
-; GATHER-NEXT:    [[TMP13:%.*]] = insertelement <8 x i1> [[TMP11]], i1 [[TMP12]], i32 5
-; GATHER-NEXT:    [[TMP14:%.*]] = extractelement <8 x i1> [[TMP1]], i32 6
-; GATHER-NEXT:    [[TMP15:%.*]] = insertelement <8 x i1> [[TMP13]], i1 [[TMP14]], i32 6
-; GATHER-NEXT:    [[TMP16:%.*]] = extractelement <8 x i1> [[TMP1]], i32 7
-; GATHER-NEXT:    [[TMP17:%.*]] = insertelement <8 x i1> [[TMP15]], i1 [[TMP16]], i32 7
+; GATHER-NEXT:    [[TMP2:%.*]] = extractelement <8 x i1> [[TMP1]], i32 7
+; GATHER-NEXT:    [[TMP3:%.*]] = extractelement <8 x i1> [[TMP1]], i32 0
+; GATHER-NEXT:    [[TMP4:%.*]] = insertelement <8 x i1> undef, i1 [[TMP3]], i32 0
+; GATHER-NEXT:    [[TMP5:%.*]] = extractelement <8 x i1> [[TMP1]], i32 1
+; GATHER-NEXT:    [[TMP6:%.*]] = insertelement <8 x i1> [[TMP4]], i1 [[TMP5]], i32 1
+; GATHER-NEXT:    [[TMP7:%.*]] = extractelement <8 x i1> [[TMP1]], i32 2
+; GATHER-NEXT:    [[TMP8:%.*]] = insertelement <8 x i1> [[TMP6]], i1 [[TMP7]], i32 2
+; GATHER-NEXT:    [[TMP9:%.*]] = extractelement <8 x i1> [[TMP1]], i32 3
+; GATHER-NEXT:    [[TMP10:%.*]] = insertelement <8 x i1> [[TMP8]], i1 [[TMP9]], i32 3
+; GATHER-NEXT:    [[TMP11:%.*]] = extractelement <8 x i1> [[TMP1]], i32 4
+; GATHER-NEXT:    [[TMP12:%.*]] = insertelement <8 x i1> [[TMP10]], i1 [[TMP11]], i32 4
+; GATHER-NEXT:    [[TMP13:%.*]] = extractelement <8 x i1> [[TMP1]], i32 5
+; GATHER-NEXT:    [[TMP14:%.*]] = insertelement <8 x i1> [[TMP12]], i1 [[TMP13]], i32 5
+; GATHER-NEXT:    [[TMP15:%.*]] = extractelement <8 x i1> [[TMP1]], i32 6
+; GATHER-NEXT:    [[TMP16:%.*]] = insertelement <8 x i1> [[TMP14]], i1 [[TMP15]], i32 6
+; GATHER-NEXT:    [[TMP17:%.*]] = insertelement <8 x i1> [[TMP16]], i1 [[TMP2]], i32 7
 ; GATHER-NEXT:    [[TMP18:%.*]] = select <8 x i1> [[TMP17]], <8 x i32> <i32 -720, i32 -720, i32 -720, i32 -720, i32 -720, i32 -720, i32 -720, i32 -720>, <8 x i32> <i32 -80, i32 -80, i32 -80, i32 -80, i32 -80, i32 -80, i32 -80, i32 -80>
 ; GATHER-NEXT:    [[TMP19:%.*]] = extractelement <8 x i32> [[TMP18]], i32 0
-; GATHER-NEXT:    [[P20:%.*]] = add i32 -5, [[TMP19]]
 ; GATHER-NEXT:    [[TMP20:%.*]] = extractelement <8 x i32> [[TMP18]], i32 1
-; GATHER-NEXT:    [[P22:%.*]] = add i32 [[P20]], [[TMP20]]
 ; GATHER-NEXT:    [[TMP21:%.*]] = extractelement <8 x i32> [[TMP18]], i32 2
-; GATHER-NEXT:    [[P24:%.*]] = add i32 [[P22]], [[TMP21]]
 ; GATHER-NEXT:    [[TMP22:%.*]] = extractelement <8 x i32> [[TMP18]], i32 3
-; GATHER-NEXT:    [[P26:%.*]] = add i32 [[P24]], [[TMP22]]
 ; GATHER-NEXT:    [[TMP23:%.*]] = extractelement <8 x i32> [[TMP18]], i32 4
-; GATHER-NEXT:    [[P28:%.*]] = add i32 [[P26]], [[TMP23]]
 ; GATHER-NEXT:    [[TMP24:%.*]] = extractelement <8 x i32> [[TMP18]], i32 5
-; GATHER-NEXT:    [[P30:%.*]] = add i32 [[P28]], [[TMP24]]
 ; GATHER-NEXT:    [[TMP25:%.*]] = extractelement <8 x i32> [[TMP18]], i32 6
-; GATHER-NEXT:    [[P32:%.*]] = add i32 [[P30]], [[TMP25]]
 ; GATHER-NEXT:    [[TMP26:%.*]] = insertelement <8 x i32> undef, i32 [[TMP19]], i32 0
 ; GATHER-NEXT:    [[TMP27:%.*]] = insertelement <8 x i32> [[TMP26]], i32 [[TMP20]], i32 1
 ; GATHER-NEXT:    [[TMP28:%.*]] = insertelement <8 x i32> [[TMP27]], i32 [[TMP21]], i32 2
@@ -230,7 +199,6 @@ define void @PR32038(i32 %n) {
 ; GATHER-NEXT:    [[TMP34:%.*]] = insertelement <8 x i32> [[TMP32]], i32 [[TMP33]], i32 7
 ; GATHER-NEXT:    [[TMP35:%.*]] = call i32 @llvm.experimental.vector.reduce.add.v8i32(<8 x i32> [[TMP34]])
 ; GATHER-NEXT:    [[OP_EXTRA]] = add i32 [[TMP35]], -5
-; GATHER-NEXT:    [[P34:%.*]] = add i32 [[P32]], [[TMP33]]
 ; GATHER-NEXT:    br label [[FOR_BODY]]
 ;
 ; MAX-COST-LABEL: @PR32038(
@@ -259,18 +227,12 @@ define void @PR32038(i32 %n) {
 ; MAX-COST-NEXT:    [[TMP6:%.*]] = insertelement <4 x i1> [[TMP5]], i1 [[P5]], i32 2
 ; MAX-COST-NEXT:    [[TMP7:%.*]] = insertelement <4 x i1> [[TMP6]], i1 [[P7]], i32 3
 ; MAX-COST-NEXT:    [[TMP8:%.*]] = select <4 x i1> [[TMP7]], <4 x i32> <i32 -720, i32 -720, i32 -720, i32 -720>, <4 x i32> <i32 -80, i32 -80, i32 -80, i32 -80>
-; MAX-COST-NEXT:    [[P20:%.*]] = add i32 -5, undef
-; MAX-COST-NEXT:    [[P22:%.*]] = add i32 [[P20]], undef
-; MAX-COST-NEXT:    [[P24:%.*]] = add i32 [[P22]], undef
-; MAX-COST-NEXT:    [[P26:%.*]] = add i32 [[P24]], undef
 ; MAX-COST-NEXT:    [[P27:%.*]] = select i1 [[P9]], i32 -720, i32 -80
-; MAX-COST-NEXT:    [[P28:%.*]] = add i32 [[P26]], [[P27]]
 ; MAX-COST-NEXT:    [[P29:%.*]] = select i1 [[P11]], i32 -720, i32 -80
 ; MAX-COST-NEXT:    [[TMP9:%.*]] = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> [[TMP8]])
 ; MAX-COST-NEXT:    [[TMP10:%.*]] = add i32 [[TMP9]], [[P27]]
 ; MAX-COST-NEXT:    [[TMP11:%.*]] = add i32 [[TMP10]], [[P29]]
 ; MAX-COST-NEXT:    [[OP_EXTRA:%.*]] = add i32 [[TMP11]], -5
-; MAX-COST-NEXT:    [[P30:%.*]] = add i32 [[P28]], [[P29]]
 ; MAX-COST-NEXT:    [[P31:%.*]] = select i1 [[P13]], i32 -720, i32 -80
 ; MAX-COST-NEXT:    [[P32:%.*]] = add i32 [[OP_EXTRA]], [[P31]]
 ; MAX-COST-NEXT:    [[P33:%.*]] = select i1 [[P15]], i32 -720, i32 -80
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr.ll
index d3bbf3df8..8278d0607 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr.ll
@@ -23,10 +23,7 @@ target triple = "aarch64--linux-gnu"
 ; }
 ;
 
-; YAML:      --- !Passed
-; YAML-NEXT: Pass:            slp-vectorizer
-; YAML-NEXT: Name:            VectorizedList
-; YAML-NEXT: Function:        getelementptr_4x32
+; YAML-LABEL: Function:        getelementptr_4x32
 ; YAML-NEXT: Args:
 ; YAML-NEXT:   - String:          'SLP vectorized with cost '
 ; YAML-NEXT:   - Cost:            '11'
@@ -39,7 +36,7 @@ target triple = "aarch64--linux-gnu"
 ; YAML-NEXT: Function:        getelementptr_4x32
 ; YAML-NEXT: Args:
 ; YAML-NEXT:   - String:          'SLP vectorized with cost '
-; YAML-NEXT:   - Cost:            '16'
+; YAML-NEXT:   - Cost:            '6'
 ; YAML-NEXT:   - String:          ' and with tree size '
 ; YAML-NEXT:   - TreeSize:        '3'
 
@@ -49,49 +46,50 @@ define i32 @getelementptr_4x32(i32* nocapture readonly %g, i32 %n, i32 %x, i32 %
 ; CHECK-NEXT:    [[CMP31:%.*]] = icmp sgt i32 [[N:%.*]], 0
 ; CHECK-NEXT:    br i1 [[CMP31]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
 ; CHECK:       for.body.preheader:
-; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>, i32 [[X:%.*]], i32 1
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i32> [[TMP0]], i32 [[Y:%.*]], i32 2
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x i32> [[TMP1]], i32 [[Z:%.*]], i32 3
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x i32> <i32 0, i32 undef>, i32 [[X:%.*]], i32 1
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i32> undef, i32 [[Y:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x i32> [[TMP1]], i32 [[Z:%.*]], i32 1
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.cond.cleanup.loopexit:
-; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x i32> [[TMP21:%.*]], i32 1
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x i32> [[TMP22:%.*]], i32 1
 ; CHECK-NEXT:    br label [[FOR_COND_CLEANUP]]
 ; CHECK:       for.cond.cleanup:
 ; CHECK-NEXT:    [[SUM_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP3]], [[FOR_COND_CLEANUP_LOOPEXIT:%.*]] ]
 ; CHECK-NEXT:    ret i32 [[SUM_0_LCSSA]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[TMP4:%.*]] = phi <2 x i32> [ zeroinitializer, [[FOR_BODY_PREHEADER]] ], [ [[TMP21]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP4:%.*]] = phi <2 x i32> [ zeroinitializer, [[FOR_BODY_PREHEADER]] ], [ [[TMP22]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i32> [[TMP4]], i32 0
 ; CHECK-NEXT:    [[T4:%.*]] = shl nsw i32 [[TMP5]], 1
-; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <4 x i32> undef, i32 [[T4]], i32 0
-; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> undef, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP8:%.*]] = add nsw <4 x i32> [[TMP7]], [[TMP2]]
-; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <4 x i32> [[TMP8]], i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i32> undef, i32 [[T4]], i32 0
+; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <2 x i32> [[TMP6]], <2 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = add nsw <2 x i32> [[TMP7]], [[TMP0]]
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <2 x i32> [[TMP8]], i32 0
 ; CHECK-NEXT:    [[TMP10:%.*]] = sext i32 [[TMP9]] to i64
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[G:%.*]], i64 [[TMP10]]
 ; CHECK-NEXT:    [[T6:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <2 x i32> [[TMP4]], i32 1
 ; CHECK-NEXT:    [[ADD1:%.*]] = add nsw i32 [[T6]], [[TMP11]]
-; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <4 x i32> [[TMP8]], i32 1
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <2 x i32> [[TMP8]], i32 1
 ; CHECK-NEXT:    [[TMP13:%.*]] = sext i32 [[TMP12]] to i64
 ; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, i32* [[G]], i64 [[TMP13]]
 ; CHECK-NEXT:    [[T8:%.*]] = load i32, i32* [[ARRAYIDX5]], align 4
 ; CHECK-NEXT:    [[ADD6:%.*]] = add nsw i32 [[ADD1]], [[T8]]
-; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <4 x i32> [[TMP8]], i32 2
-; CHECK-NEXT:    [[TMP15:%.*]] = sext i32 [[TMP14]] to i64
-; CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds i32, i32* [[G]], i64 [[TMP15]]
+; CHECK-NEXT:    [[TMP14:%.*]] = add nsw <2 x i32> [[TMP7]], [[TMP2]]
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <2 x i32> [[TMP14]], i32 0
+; CHECK-NEXT:    [[TMP16:%.*]] = sext i32 [[TMP15]] to i64
+; CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds i32, i32* [[G]], i64 [[TMP16]]
 ; CHECK-NEXT:    [[T10:%.*]] = load i32, i32* [[ARRAYIDX10]], align 4
 ; CHECK-NEXT:    [[ADD11:%.*]] = add nsw i32 [[ADD6]], [[T10]]
-; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <4 x i32> [[TMP8]], i32 3
-; CHECK-NEXT:    [[TMP17:%.*]] = sext i32 [[TMP16]] to i64
-; CHECK-NEXT:    [[ARRAYIDX15:%.*]] = getelementptr inbounds i32, i32* [[G]], i64 [[TMP17]]
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <2 x i32> [[TMP14]], i32 1
+; CHECK-NEXT:    [[TMP18:%.*]] = sext i32 [[TMP17]] to i64
+; CHECK-NEXT:    [[ARRAYIDX15:%.*]] = getelementptr inbounds i32, i32* [[G]], i64 [[TMP18]]
 ; CHECK-NEXT:    [[T12:%.*]] = load i32, i32* [[ARRAYIDX15]], align 4
-; CHECK-NEXT:    [[TMP18:%.*]] = insertelement <2 x i32> undef, i32 [[TMP5]], i32 0
-; CHECK-NEXT:    [[TMP19:%.*]] = insertelement <2 x i32> [[TMP18]], i32 [[ADD11]], i32 1
-; CHECK-NEXT:    [[TMP20:%.*]] = insertelement <2 x i32> <i32 1, i32 undef>, i32 [[T12]], i32 1
-; CHECK-NEXT:    [[TMP21]] = add nsw <2 x i32> [[TMP19]], [[TMP20]]
-; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <2 x i32> [[TMP21]], i32 0
-; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[TMP22]], [[N]]
+; CHECK-NEXT:    [[TMP19:%.*]] = insertelement <2 x i32> undef, i32 [[TMP5]], i32 0
+; CHECK-NEXT:    [[TMP20:%.*]] = insertelement <2 x i32> [[TMP19]], i32 [[ADD11]], i32 1
+; CHECK-NEXT:    [[TMP21:%.*]] = insertelement <2 x i32> <i32 1, i32 undef>, i32 [[T12]], i32 1
+; CHECK-NEXT:    [[TMP22]] = add nsw <2 x i32> [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <2 x i32> [[TMP22]], i32 0
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[TMP23]], [[N]]
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]]
 ;
 entry:
@@ -133,10 +131,7 @@ for.body:
   br i1 %exitcond, label %for.cond.cleanup.loopexit, label %for.body
 }
 
-; YAML:      --- !Passed
-; YAML-NEXT: Pass:            slp-vectorizer
-; YAML-NEXT: Name:            VectorizedList
-; YAML-NEXT: Function:        getelementptr_2x32
+; YAML-LABEL: Function:        getelementptr_2x32
 ; YAML-NEXT: Args:
 ; YAML-NEXT:   - String:          'SLP vectorized with cost '
 ; YAML-NEXT:   - Cost:            '11'
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/horizontal.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/horizontal.ll
index 7e4b95d72..b3ce90dab 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/horizontal.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/horizontal.ll
@@ -46,12 +46,8 @@ define i32 @test_select(i32* noalias nocapture readonly %blk1, i32* noalias noca
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp slt <4 x i32> [[TMP4]], zeroinitializer
 ; CHECK-NEXT:    [[TMP6:%.*]] = sub nsw <4 x i32> zeroinitializer, [[TMP4]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = select <4 x i1> [[TMP5]], <4 x i32> [[TMP6]], <4 x i32> [[TMP4]]
-; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 undef, [[S_026]]
-; CHECK-NEXT:    [[ADD11:%.*]] = add nsw i32 [[ADD]], undef
-; CHECK-NEXT:    [[ADD19:%.*]] = add nsw i32 [[ADD11]], undef
 ; CHECK-NEXT:    [[TMP8:%.*]] = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> [[TMP7]])
 ; CHECK-NEXT:    [[OP_EXTRA]] = add nsw i32 [[TMP8]], [[S_026]]
-; CHECK-NEXT:    [[ADD27:%.*]] = add nsw i32 [[ADD19]], undef
 ; CHECK-NEXT:    [[ADD_PTR]] = getelementptr inbounds i32, i32* [[P1_023]], i64 [[IDX_EXT]]
 ; CHECK-NEXT:    [[ADD_PTR29]] = getelementptr inbounds i32, i32* [[P2_024]], i64 [[IDX_EXT]]
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[J_025]], 1
@@ -173,12 +169,8 @@ define i32 @reduction_with_br(i32* noalias nocapture readonly %blk1, i32* noalia
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i32* [[P2_018]] to <4 x i32>*
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, <4 x i32>* [[TMP2]], align 4
 ; CHECK-NEXT:    [[TMP4:%.*]] = mul nsw <4 x i32> [[TMP3]], [[TMP1]]
-; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 undef, [[S_020]]
-; CHECK-NEXT:    [[ADD5:%.*]] = add nsw i32 [[ADD]], undef
-; CHECK-NEXT:    [[ADD9:%.*]] = add nsw i32 [[ADD5]], undef
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> [[TMP4]])
 ; CHECK-NEXT:    [[OP_EXTRA]] = add nsw i32 [[TMP5]], [[S_020]]
-; CHECK-NEXT:    [[ADD13:%.*]] = add nsw i32 [[ADD9]], undef
 ; CHECK-NEXT:    [[CMP14:%.*]] = icmp slt i32 [[OP_EXTRA]], [[LIM:%.*]]
 ; CHECK-NEXT:    br i1 [[CMP14]], label [[IF_END]], label [[FOR_END_LOOPEXIT:%.*]]
 ; CHECK:       if.end:
@@ -293,16 +285,8 @@ define i32 @test_unrolled_select(i8* noalias nocapture readonly %blk1, i8* noali
 ; CHECK-NEXT:    [[TMP7:%.*]] = icmp slt <8 x i32> [[TMP6]], zeroinitializer
 ; CHECK-NEXT:    [[TMP8:%.*]] = sub nsw <8 x i32> zeroinitializer, [[TMP6]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = select <8 x i1> [[TMP7]], <8 x i32> [[TMP8]], <8 x i32> [[TMP6]]
-; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 undef, [[S_047]]
-; CHECK-NEXT:    [[ADD16:%.*]] = add nsw i32 [[ADD]], undef
-; CHECK-NEXT:    [[ADD27:%.*]] = add nsw i32 [[ADD16]], undef
-; CHECK-NEXT:    [[ADD38:%.*]] = add nsw i32 [[ADD27]], undef
-; CHECK-NEXT:    [[ADD49:%.*]] = add nsw i32 [[ADD38]], undef
-; CHECK-NEXT:    [[ADD60:%.*]] = add nsw i32 [[ADD49]], undef
-; CHECK-NEXT:    [[ADD71:%.*]] = add nsw i32 [[ADD60]], undef
 ; CHECK-NEXT:    [[TMP10:%.*]] = call i32 @llvm.experimental.vector.reduce.add.v8i32(<8 x i32> [[TMP9]])
 ; CHECK-NEXT:    [[OP_EXTRA]] = add nsw i32 [[TMP10]], [[S_047]]
-; CHECK-NEXT:    [[ADD82:%.*]] = add nsw i32 [[ADD71]], undef
 ; CHECK-NEXT:    [[CMP83:%.*]] = icmp slt i32 [[OP_EXTRA]], [[LIM:%.*]]
 ; CHECK-NEXT:    br i1 [[CMP83]], label [[IF_END_86]], label [[FOR_END_LOOPEXIT:%.*]]
 ; CHECK:       if.end.86:
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/spillcost-di.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/spillcost-di.ll
index 98a9fd482..90771e4eb 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/spillcost-di.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/spillcost-di.ll
@@ -13,11 +13,11 @@ define void @patatino(i64 %n, i64 %i, %struct.S* %p) !dbg !7 {
 ; CHECK-NEXT:    call void @llvm.dbg.value(metadata i64 [[I:%.*]], metadata !19, metadata !DIExpression()), !dbg !24
 ; CHECK-NEXT:    call void @llvm.dbg.value(metadata %struct.S* [[P:%.*]], metadata !20, metadata !DIExpression()), !dbg !25
 ; CHECK-NEXT:    [[X1:%.*]] = getelementptr inbounds [[STRUCT_S:%.*]], %struct.S* [[P]], i64 [[N]], i32 0, !dbg !26
-; CHECK-NEXT:    call void @llvm.dbg.value(metadata i64 undef, metadata !21, metadata !DIExpression()), !dbg !27
+; CHECK-NEXT:    call void @llvm.dbg.value(metadata !2, metadata !21, metadata !DIExpression()), !dbg !27
 ; CHECK-NEXT:    [[Y3:%.*]] = getelementptr inbounds [[STRUCT_S]], %struct.S* [[P]], i64 [[N]], i32 1, !dbg !28
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i64* [[X1]] to <2 x i64>*, !dbg !26
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, <2 x i64>* [[TMP0]], align 8, !dbg !26, !tbaa !29
-; CHECK-NEXT:    call void @llvm.dbg.value(metadata i64 undef, metadata !22, metadata !DIExpression()), !dbg !33
+; CHECK-NEXT:    call void @llvm.dbg.value(metadata !2, metadata !22, metadata !DIExpression()), !dbg !33
 ; CHECK-NEXT:    [[X5:%.*]] = getelementptr inbounds [[STRUCT_S]], %struct.S* [[P]], i64 [[I]], i32 0, !dbg !34
 ; CHECK-NEXT:    [[Y7:%.*]] = getelementptr inbounds [[STRUCT_S]], %struct.S* [[P]], i64 [[I]], i32 1, !dbg !35
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i64* [[X5]] to <2 x i64>*, !dbg !36
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/transpose.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/transpose.ll
index e4842af52..707d2a216 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/transpose.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/transpose.ll
@@ -6,19 +6,16 @@ target triple = "aarch64--linux-gnu"
 
 define <2 x i64> @build_vec_v2i64(<2 x i64> %v0, <2 x i64> %v1) {
 ; CHECK-LABEL: @build_vec_v2i64(
-; CHECK-NEXT:    [[V0_0:%.*]] = extractelement <2 x i64> [[V0:%.*]], i32 0
-; CHECK-NEXT:    [[V0_1:%.*]] = extractelement <2 x i64> [[V0]], i32 1
-; CHECK-NEXT:    [[V1_0:%.*]] = extractelement <2 x i64> [[V1:%.*]], i32 0
-; CHECK-NEXT:    [[V1_1:%.*]] = extractelement <2 x i64> [[V1]], i32 1
-; CHECK-NEXT:    [[TMP0_0:%.*]] = add i64 [[V0_0]], [[V1_0]]
-; CHECK-NEXT:    [[TMP0_1:%.*]] = add i64 [[V0_1]], [[V1_1]]
-; CHECK-NEXT:    [[TMP1_0:%.*]] = sub i64 [[V0_0]], [[V1_0]]
-; CHECK-NEXT:    [[TMP1_1:%.*]] = sub i64 [[V0_1]], [[V1_1]]
-; CHECK-NEXT:    [[TMP2_0:%.*]] = add i64 [[TMP0_0]], [[TMP0_1]]
-; CHECK-NEXT:    [[TMP2_1:%.*]] = add i64 [[TMP1_0]], [[TMP1_1]]
-; CHECK-NEXT:    [[TMP3_0:%.*]] = insertelement <2 x i64> undef, i64 [[TMP2_0]], i32 0
-; CHECK-NEXT:    [[TMP3_1:%.*]] = insertelement <2 x i64> [[TMP3_0]], i64 [[TMP2_1]], i32 1
-; CHECK-NEXT:    ret <2 x i64> [[TMP3_1]]
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x i64> [[V0:%.*]], <2 x i64> undef, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i64> [[V1:%.*]], <2 x i64> undef, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT:    [[TMP3:%.*]] = add <2 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = sub <2 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <2 x i64> [[TMP3]], <2 x i64> [[TMP4]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT:    [[TMP6:%.*]] = add <2 x i64> [[V0]], [[V1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = sub <2 x i64> [[V0]], [[V1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <2 x i64> [[TMP6]], <2 x i64> [[TMP7]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT:    [[TMP9:%.*]] = add <2 x i64> [[TMP8]], [[TMP5]]
+; CHECK-NEXT:    ret <2 x i64> [[TMP9]]
 ;
   %v0.0 = extractelement <2 x i64> %v0, i32 0
   %v0.1 = extractelement <2 x i64> %v0, i32 1
@@ -77,22 +74,16 @@ define void @store_chain_v2i64(i64* %a, i64* %b, i64* %c) {
 
 define <4 x i32> @build_vec_v4i32(<4 x i32> %v0, <4 x i32> %v1) {
 ; CHECK-LABEL: @build_vec_v4i32(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> undef, <2 x i32> <i32 0, i32 2>
-; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> undef, <4 x i32> <i32 0, i32 0, i32 1, i32 1>
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i32> [[V1:%.*]], <4 x i32> undef, <2 x i32> <i32 0, i32 2>
-; CHECK-NEXT:    [[SHUFFLE1:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> undef, <4 x i32> <i32 0, i32 0, i32 1, i32 1>
-; CHECK-NEXT:    [[TMP3:%.*]] = add <4 x i32> [[SHUFFLE]], [[SHUFFLE1]]
-; CHECK-NEXT:    [[TMP4:%.*]] = sub <4 x i32> [[SHUFFLE]], [[SHUFFLE1]]
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i32> [[V1:%.*]], <4 x i32> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+; CHECK-NEXT:    [[TMP3:%.*]] = add <4 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = sub <4 x i32> [[TMP1]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
-; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i32> [[V0]], <4 x i32> undef, <2 x i32> <i32 1, i32 3>
-; CHECK-NEXT:    [[SHUFFLE2:%.*]] = shufflevector <2 x i32> [[TMP6]], <2 x i32> undef, <4 x i32> <i32 0, i32 0, i32 1, i32 1>
-; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[V1]], <4 x i32> undef, <2 x i32> <i32 1, i32 3>
-; CHECK-NEXT:    [[SHUFFLE3:%.*]] = shufflevector <2 x i32> [[TMP7]], <2 x i32> undef, <4 x i32> <i32 0, i32 0, i32 1, i32 1>
-; CHECK-NEXT:    [[TMP8:%.*]] = add <4 x i32> [[SHUFFLE2]], [[SHUFFLE3]]
-; CHECK-NEXT:    [[TMP9:%.*]] = sub <4 x i32> [[SHUFFLE2]], [[SHUFFLE3]]
-; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> [[TMP9]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
-; CHECK-NEXT:    [[TMP11:%.*]] = add <4 x i32> [[TMP5]], [[TMP10]]
-; CHECK-NEXT:    ret <4 x i32> [[TMP11]]
+; CHECK-NEXT:    [[TMP6:%.*]] = add <4 x i32> [[V0]], [[V1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = sub <4 x i32> [[V0]], [[V1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> [[TMP7]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    [[TMP9:%.*]] = add <4 x i32> [[TMP8]], [[TMP5]]
+; CHECK-NEXT:    ret <4 x i32> [[TMP9]]
 ;
   %v0.0 = extractelement <4 x i32> %v0, i32 0
   %v0.1 = extractelement <4 x i32> %v0, i32 1
@@ -123,18 +114,16 @@ define <4 x i32> @build_vec_v4i32(<4 x i32> %v0, <4 x i32> %v1) {
 
 define <4 x i32> @build_vec_v4i32_reuse_0(<2 x i32> %v0, <2 x i32> %v1) {
 ; CHECK-LABEL: @build_vec_v4i32_reuse_0(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x i32> [[V0:%.*]], <2 x i32> undef, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i32> [[V1:%.*]], <2 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x i32> [[V0:%.*]], <2 x i32> undef, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i32> [[V1:%.*]], <2 x i32> undef, <2 x i32> <i32 1, i32 0>
 ; CHECK-NEXT:    [[TMP3:%.*]] = add <2 x i32> [[TMP1]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = sub <2 x i32> [[TMP1]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <2 x i32> [[TMP3]], <2 x i32> [[TMP4]], <2 x i32> <i32 0, i32 3>
-; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <2 x i32> [[V0]], <2 x i32> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <2 x i32> [[V1]], <2 x i32> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT:    [[TMP8:%.*]] = add <2 x i32> [[TMP6]], [[TMP7]]
-; CHECK-NEXT:    [[TMP9:%.*]] = sub <2 x i32> [[TMP6]], [[TMP7]]
-; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <2 x i32> [[TMP8]], <2 x i32> [[TMP9]], <2 x i32> <i32 0, i32 3>
-; CHECK-NEXT:    [[TMP11:%.*]] = add <2 x i32> [[TMP5]], [[TMP10]]
-; CHECK-NEXT:    [[TMP3_3:%.*]] = shufflevector <2 x i32> [[TMP11]], <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+; CHECK-NEXT:    [[TMP6:%.*]] = add <2 x i32> [[V0]], [[V1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = sub <2 x i32> [[V0]], [[V1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <2 x i32> [[TMP6]], <2 x i32> [[TMP7]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT:    [[TMP9:%.*]] = add <2 x i32> [[TMP8]], [[TMP5]]
+; CHECK-NEXT:    [[TMP3_3:%.*]] = shufflevector <2 x i32> [[TMP9]], <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
 ; CHECK-NEXT:    ret <4 x i32> [[TMP3_3]]
 ;
   %v0.0 = extractelement <2 x i32> %v0, i32 0
@@ -240,28 +229,22 @@ define <4 x i32> @build_vec_v4i32_3_binops(<2 x i32> %v0, <2 x i32> %v1) {
 
 define i32 @reduction_v4i32(<4 x i32> %v0, <4 x i32> %v1) {
 ; CHECK-LABEL: @reduction_v4i32(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> undef, <2 x i32> <i32 0, i32 2>
-; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> undef, <4 x i32> <i32 0, i32 0, i32 1, i32 1>
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i32> [[V1:%.*]], <4 x i32> undef, <2 x i32> <i32 0, i32 2>
-; CHECK-NEXT:    [[SHUFFLE1:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> undef, <4 x i32> <i32 0, i32 0, i32 1, i32 1>
-; CHECK-NEXT:    [[TMP3:%.*]] = sub <4 x i32> [[SHUFFLE]], [[SHUFFLE1]]
-; CHECK-NEXT:    [[TMP4:%.*]] = add <4 x i32> [[SHUFFLE]], [[SHUFFLE1]]
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i32> [[V1:%.*]], <4 x i32> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+; CHECK-NEXT:    [[TMP3:%.*]] = sub <4 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = add <4 x i32> [[TMP1]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <4 x i32> <i32 0, i32 5, i32 6, i32 3>
-; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i32> [[V0]], <4 x i32> undef, <2 x i32> <i32 1, i32 3>
-; CHECK-NEXT:    [[SHUFFLE2:%.*]] = shufflevector <2 x i32> [[TMP6]], <2 x i32> undef, <4 x i32> <i32 0, i32 0, i32 1, i32 1>
-; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[V1]], <4 x i32> undef, <2 x i32> <i32 1, i32 3>
-; CHECK-NEXT:    [[SHUFFLE3:%.*]] = shufflevector <2 x i32> [[TMP7]], <2 x i32> undef, <4 x i32> <i32 0, i32 0, i32 1, i32 1>
-; CHECK-NEXT:    [[TMP8:%.*]] = sub <4 x i32> [[SHUFFLE2]], [[SHUFFLE3]]
-; CHECK-NEXT:    [[TMP9:%.*]] = add <4 x i32> [[SHUFFLE2]], [[SHUFFLE3]]
-; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> [[TMP9]], <4 x i32> <i32 0, i32 5, i32 6, i32 3>
-; CHECK-NEXT:    [[TMP11:%.*]] = add <4 x i32> [[TMP5]], [[TMP10]]
-; CHECK-NEXT:    [[TMP12:%.*]] = lshr <4 x i32> [[TMP11]], <i32 15, i32 15, i32 15, i32 15>
-; CHECK-NEXT:    [[TMP13:%.*]] = and <4 x i32> [[TMP12]], <i32 65537, i32 65537, i32 65537, i32 65537>
-; CHECK-NEXT:    [[TMP14:%.*]] = mul nuw <4 x i32> [[TMP13]], <i32 65535, i32 65535, i32 65535, i32 65535>
-; CHECK-NEXT:    [[TMP15:%.*]] = add <4 x i32> [[TMP14]], [[TMP11]]
-; CHECK-NEXT:    [[TMP16:%.*]] = xor <4 x i32> [[TMP15]], [[TMP14]]
-; CHECK-NEXT:    [[TMP17:%.*]] = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> [[TMP16]])
-; CHECK-NEXT:    ret i32 [[TMP17]]
+; CHECK-NEXT:    [[TMP6:%.*]] = sub <4 x i32> [[V0]], [[V1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = add <4 x i32> [[V0]], [[V1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> [[TMP7]], <4 x i32> <i32 0, i32 5, i32 6, i32 3>
+; CHECK-NEXT:    [[TMP9:%.*]] = add <4 x i32> [[TMP8]], [[TMP5]]
+; CHECK-NEXT:    [[TMP10:%.*]] = lshr <4 x i32> [[TMP9]], <i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT:    [[TMP11:%.*]] = and <4 x i32> [[TMP10]], <i32 65537, i32 65537, i32 65537, i32 65537>
+; CHECK-NEXT:    [[TMP12:%.*]] = mul nuw <4 x i32> [[TMP11]], <i32 65535, i32 65535, i32 65535, i32 65535>
+; CHECK-NEXT:    [[TMP13:%.*]] = add <4 x i32> [[TMP12]], [[TMP9]]
+; CHECK-NEXT:    [[TMP14:%.*]] = xor <4 x i32> [[TMP13]], [[TMP12]]
+; CHECK-NEXT:    [[TMP15:%.*]] = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> [[TMP14]])
+; CHECK-NEXT:    ret i32 [[TMP15]]
 ;
   %v0.0 = extractelement <4 x i32> %v0, i32 0
   %v0.1 = extractelement <4 x i32> %v0, i32 1
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/address-space-ptr-sze-gep-index-assert.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/address-space-ptr-sze-gep-index-assert.ll
index 735ce651e..4c904b66f 100644
--- a/llvm/test/Transforms/SLPVectorizer/AMDGPU/address-space-ptr-sze-gep-index-assert.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/address-space-ptr-sze-gep-index-assert.ll
@@ -147,3 +147,16 @@ bb:
   store i32 %sub1, i32* undef
   ret void
 }
+
+; CHECK-LABEL: slp_crash_on_addrspacecast
+; CHECK: ret void
+define void @slp_crash_on_addrspacecast() {
+entry:
+  %0 = getelementptr inbounds i64, i64 addrspace(3)* undef, i32 undef
+  %p0 = addrspacecast i64 addrspace(3)* %0 to i64*
+  store i64 undef, i64* %p0, align 8
+  %1 = getelementptr inbounds i64, i64 addrspace(3)* undef, i32 undef
+  %p1 = addrspacecast i64 addrspace(3)* %1 to i64*
+  store i64 undef, i64* %p1, align 8
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/NVPTX/non-vectorizable-intrinsic.ll b/llvm/test/Transforms/SLPVectorizer/NVPTX/non-vectorizable-intrinsic.ll
index 3eae2d0b0..ff8778f31 100644
--- a/llvm/test/Transforms/SLPVectorizer/NVPTX/non-vectorizable-intrinsic.ll
+++ b/llvm/test/Transforms/SLPVectorizer/NVPTX/non-vectorizable-intrinsic.ll
@@ -53,5 +53,5 @@ entry:
 
 declare i8 @llvm.ctlz.i8(i8, i1) #3
 
-attributes #0 = { alwaysinline nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { alwaysinline nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/PR31847.ll b/llvm/test/Transforms/SLPVectorizer/X86/PR31847.ll
new file mode 100644
index 000000000..2fe342430
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/PR31847.ll
@@ -0,0 +1,153 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -slp-vectorizer -S -o - -mtriple=i386 -mcpu=haswell < %s | FileCheck %s
+target datalayout = "e-m:e-p:32:32-f64:32:64-f80:32-n8:16:32-S128"
+
+@shift = common local_unnamed_addr global [10 x i32] zeroinitializer, align 4
+@data = common local_unnamed_addr global [10 x i8*] zeroinitializer, align 4
+
+define void @flat(i32 %intensity) {
+; CHECK-LABEL: @flat(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* getelementptr inbounds ([10 x i32], [10 x i32]* @shift, i32 0, i32 0), align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* getelementptr inbounds ([10 x i32], [10 x i32]* @shift, i32 0, i32 1), align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8*, i8** getelementptr inbounds ([10 x i8*], [10 x i8*]* @data, i32 0, i32 0), align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8*, i8** getelementptr inbounds ([10 x i8*], [10 x i8*]* @data, i32 0, i32 1), align 4
+; CHECK-NEXT:    [[SHR:%.*]] = lshr i32 1, [[TMP0]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, i8* [[TMP2]], i32 [[SHR]]
+; CHECK-NEXT:    [[SHR1:%.*]] = lshr i32 1, [[TMP1]]
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i8, i8* [[TMP3]], i32 [[SHR1]]
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.cond.cleanup:
+; CHECK-NEXT:    ret void
+; CHECK:       for.body:
+; CHECK-NEXT:    [[D1_DATA_046:%.*]] = phi i8* [ [[TMP3]], [[ENTRY:%.*]] ], [ [[ADD_PTR23_1:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[Y_045:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[INC_1:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, i8* [[ARRAYIDX]], align 1
+; CHECK-NEXT:    [[CONV:%.*]] = zext i8 [[TMP4]] to i32
+; CHECK-NEXT:    [[SUB:%.*]] = add nsw i32 [[CONV]], -128
+; CHECK-NEXT:    [[TMP5:%.*]] = load i8, i8* [[ARRAYIDX2]], align 1
+; CHECK-NEXT:    [[CONV3:%.*]] = zext i8 [[TMP5]] to i32
+; CHECK-NEXT:    [[SUB4:%.*]] = add nsw i32 [[CONV3]], -128
+; CHECK-NEXT:    [[CMP5:%.*]] = icmp sgt i32 [[SUB]], -1
+; CHECK-NEXT:    [[SUB7:%.*]] = sub nsw i32 128, [[CONV]]
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP5]], i32 [[SUB]], i32 [[SUB7]]
+; CHECK-NEXT:    [[CMP8:%.*]] = icmp sgt i32 [[SUB4]], -1
+; CHECK-NEXT:    [[SUB12:%.*]] = sub nsw i32 128, [[CONV3]]
+; CHECK-NEXT:    [[COND14:%.*]] = select i1 [[CMP8]], i32 [[SUB4]], i32 [[SUB12]]
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[COND14]], [[COND]]
+; CHECK-NEXT:    [[IDX_NEG:%.*]] = sub nsw i32 0, [[ADD]]
+; CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds i8, i8* [[D1_DATA_046]], i32 [[IDX_NEG]]
+; CHECK-NEXT:    [[TMP6:%.*]] = load i8, i8* [[ADD_PTR]], align 1
+; CHECK-NEXT:    [[CONV15:%.*]] = zext i8 [[TMP6]] to i32
+; CHECK-NEXT:    [[ADD16:%.*]] = add nsw i32 [[CONV15]], [[INTENSITY:%.*]]
+; CHECK-NEXT:    [[CONV17:%.*]] = trunc i32 [[ADD16]] to i8
+; CHECK-NEXT:    store i8 [[CONV17]], i8* [[ADD_PTR]], align 1
+; CHECK-NEXT:    [[ADD_PTR18:%.*]] = getelementptr inbounds i8, i8* [[D1_DATA_046]], i32 [[ADD]]
+; CHECK-NEXT:    [[TMP7:%.*]] = load i8, i8* [[ADD_PTR18]], align 1
+; CHECK-NEXT:    [[NOT_TOBOOL:%.*]] = icmp eq i8 [[TMP7]], 0
+; CHECK-NEXT:    [[CONV21:%.*]] = zext i1 [[NOT_TOBOOL]] to i8
+; CHECK-NEXT:    store i8 [[CONV21]], i8* [[ADD_PTR18]], align 1
+; CHECK-NEXT:    [[ADD_PTR23:%.*]] = getelementptr inbounds i8, i8* [[D1_DATA_046]], i32 [[TMP1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = load i8, i8* [[ARRAYIDX]], align 1
+; CHECK-NEXT:    [[CONV_1:%.*]] = zext i8 [[TMP8]] to i32
+; CHECK-NEXT:    [[SUB_1:%.*]] = add nsw i32 [[CONV_1]], -128
+; CHECK-NEXT:    [[TMP9:%.*]] = load i8, i8* [[ARRAYIDX2]], align 1
+; CHECK-NEXT:    [[CONV3_1:%.*]] = zext i8 [[TMP9]] to i32
+; CHECK-NEXT:    [[SUB4_1:%.*]] = add nsw i32 [[CONV3_1]], -128
+; CHECK-NEXT:    [[CMP5_1:%.*]] = icmp sgt i32 [[SUB_1]], -1
+; CHECK-NEXT:    [[SUB7_1:%.*]] = sub nsw i32 128, [[CONV_1]]
+; CHECK-NEXT:    [[COND_1:%.*]] = select i1 [[CMP5_1]], i32 [[SUB_1]], i32 [[SUB7_1]]
+; CHECK-NEXT:    [[CMP8_1:%.*]] = icmp sgt i32 [[SUB4_1]], -1
+; CHECK-NEXT:    [[SUB12_1:%.*]] = sub nsw i32 128, [[CONV3_1]]
+; CHECK-NEXT:    [[COND14_1:%.*]] = select i1 [[CMP8_1]], i32 [[SUB4_1]], i32 [[SUB12_1]]
+; CHECK-NEXT:    [[ADD_1:%.*]] = add nsw i32 [[COND14_1]], [[COND_1]]
+; CHECK-NEXT:    [[IDX_NEG_1:%.*]] = sub nsw i32 0, [[ADD_1]]
+; CHECK-NEXT:    [[ADD_PTR_1:%.*]] = getelementptr inbounds i8, i8* [[ADD_PTR23]], i32 [[IDX_NEG_1]]
+; CHECK-NEXT:    [[TMP10:%.*]] = load i8, i8* [[ADD_PTR_1]], align 1
+; CHECK-NEXT:    [[CONV15_1:%.*]] = zext i8 [[TMP10]] to i32
+; CHECK-NEXT:    [[ADD16_1:%.*]] = add nsw i32 [[CONV15_1]], [[INTENSITY]]
+; CHECK-NEXT:    [[CONV17_1:%.*]] = trunc i32 [[ADD16_1]] to i8
+; CHECK-NEXT:    store i8 [[CONV17_1]], i8* [[ADD_PTR_1]], align 1
+; CHECK-NEXT:    [[ADD_PTR18_1:%.*]] = getelementptr inbounds i8, i8* [[ADD_PTR23]], i32 [[ADD_1]]
+; CHECK-NEXT:    [[TMP11:%.*]] = load i8, i8* [[ADD_PTR18_1]], align 1
+; CHECK-NEXT:    [[NOT_TOBOOL_1:%.*]] = icmp eq i8 [[TMP11]], 0
+; CHECK-NEXT:    [[CONV21_1:%.*]] = zext i1 [[NOT_TOBOOL_1]] to i8
+; CHECK-NEXT:    store i8 [[CONV21_1]], i8* [[ADD_PTR18_1]], align 1
+; CHECK-NEXT:    [[ADD_PTR23_1]] = getelementptr inbounds i8, i8* [[ADD_PTR23]], i32 [[TMP1]]
+; CHECK-NEXT:    [[INC_1]] = add nsw i32 [[Y_045]], 2
+; CHECK-NEXT:    [[EXITCOND_1:%.*]] = icmp eq i32 [[INC_1]], 128
+; CHECK-NEXT:    br i1 [[EXITCOND_1]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY]]
+;
+entry:
+  %0 = load i32, i32* getelementptr inbounds ([10 x i32], [10 x i32]* @shift, i32 0, i32 0), align 4
+  %1 = load i32, i32* getelementptr inbounds ([10 x i32], [10 x i32]* @shift, i32 0, i32 1), align 4
+  %2 = load i8*, i8** getelementptr inbounds ([10 x i8*], [10 x i8*]* @data, i32 0, i32 0), align 4
+  %3 = load i8*, i8** getelementptr inbounds ([10 x i8*], [10 x i8*]* @data, i32 0, i32 1), align 4
+  %shr = lshr i32 1, %0
+  %arrayidx = getelementptr inbounds i8, i8* %2, i32 %shr
+  %shr1 = lshr i32 1, %1
+  %arrayidx2 = getelementptr inbounds i8, i8* %3, i32 %shr1
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body
+  ret void
+
+for.body:                                         ; preds = %for.body, %entry
+  %d1_data.046 = phi i8* [ %3, %entry ], [ %add.ptr23.1, %for.body ]
+  %y.045 = phi i32 [ 0, %entry ], [ %inc.1, %for.body ]
+  %4 = load i8, i8* %arrayidx, align 1
+  %conv = zext i8 %4 to i32
+  %sub = add nsw i32 %conv, -128
+  %5 = load i8, i8* %arrayidx2, align 1
+  %conv3 = zext i8 %5 to i32
+  %sub4 = add nsw i32 %conv3, -128
+  %cmp5 = icmp sgt i32 %sub, -1
+  %sub7 = sub nsw i32 128, %conv
+  %cond = select i1 %cmp5, i32 %sub, i32 %sub7
+  %cmp8 = icmp sgt i32 %sub4, -1
+  %sub12 = sub nsw i32 128, %conv3
+  %cond14 = select i1 %cmp8, i32 %sub4, i32 %sub12
+  %add = add nsw i32 %cond14, %cond
+  %idx.neg = sub nsw i32 0, %add
+  %add.ptr = getelementptr inbounds i8, i8* %d1_data.046, i32 %idx.neg
+  %6 = load i8, i8* %add.ptr, align 1
+  %conv15 = zext i8 %6 to i32
+  %add16 = add nsw i32 %conv15, %intensity
+  %conv17 = trunc i32 %add16 to i8
+  store i8 %conv17, i8* %add.ptr, align 1
+  %add.ptr18 = getelementptr inbounds i8, i8* %d1_data.046, i32 %add
+  %7 = load i8, i8* %add.ptr18, align 1
+  %not.tobool = icmp eq i8 %7, 0
+  %conv21 = zext i1 %not.tobool to i8
+  store i8 %conv21, i8* %add.ptr18, align 1
+  %add.ptr23 = getelementptr inbounds i8, i8* %d1_data.046, i32 %1
+  %8 = load i8, i8* %arrayidx, align 1
+  %conv.1 = zext i8 %8 to i32
+  %sub.1 = add nsw i32 %conv.1, -128
+  %9 = load i8, i8* %arrayidx2, align 1
+  %conv3.1 = zext i8 %9 to i32
+  %sub4.1 = add nsw i32 %conv3.1, -128
+  %cmp5.1 = icmp sgt i32 %sub.1, -1
+  %sub7.1 = sub nsw i32 128, %conv.1
+  %cond.1 = select i1 %cmp5.1, i32 %sub.1, i32 %sub7.1
+  %cmp8.1 = icmp sgt i32 %sub4.1, -1
+  %sub12.1 = sub nsw i32 128, %conv3.1
+  %cond14.1 = select i1 %cmp8.1, i32 %sub4.1, i32 %sub12.1
+  %add.1 = add nsw i32 %cond14.1, %cond.1
+  %idx.neg.1 = sub nsw i32 0, %add.1
+  %add.ptr.1 = getelementptr inbounds i8, i8* %add.ptr23, i32 %idx.neg.1
+  %10 = load i8, i8* %add.ptr.1, align 1
+  %conv15.1 = zext i8 %10 to i32
+  %add16.1 = add nsw i32 %conv15.1, %intensity
+  %conv17.1 = trunc i32 %add16.1 to i8
+  store i8 %conv17.1, i8* %add.ptr.1, align 1
+  %add.ptr18.1 = getelementptr inbounds i8, i8* %add.ptr23, i32 %add.1
+  %11 = load i8, i8* %add.ptr18.1, align 1
+  %not.tobool.1 = icmp eq i8 %11, 0
+  %conv21.1 = zext i1 %not.tobool.1 to i8
+  store i8 %conv21.1, i8* %add.ptr18.1, align 1
+  %add.ptr23.1 = getelementptr inbounds i8, i8* %add.ptr23, i32 %1
+  %inc.1 = add nsw i32 %y.045, 2
+  %exitcond.1 = icmp eq i32 %inc.1, 128
+  br i1 %exitcond.1, label %for.cond.cleanup, label %for.body
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/PR34635.ll b/llvm/test/Transforms/SLPVectorizer/X86/PR34635.ll
index e36d6f3e4..daa68b12b 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/PR34635.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/PR34635.ll
@@ -2,97 +2,96 @@
 ; RUN: opt < %s -mtriple=x86_64-unknown-linux -slp-vectorizer -S -mcpu=corei7 | FileCheck %s
 
 define i32 @main() {
-; CHECK-LABEL: @main(
+; CHECK-LABEL: define {{[^@]+}}@main(
 ; CHECK-NEXT:  bb:
-; CHECK-NEXT:    [[TMP:%.*]] = alloca <8 x i32>, align 32
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <8 x i32>* [[TMP]] to [8 x i32]*
-; CHECK-NEXT:    [[TMP2:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i32>* [[TMP]] to i8*
-; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds <8 x i32>, <8 x i32>* [[TMP]], i64 0, i64 0
-; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds [8 x i32], [8 x i32]* [[TMP1]], i64 0, i64 1
-; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds [8 x i32], [8 x i32]* [[TMP1]], i64 0, i64 2
-; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds [8 x i32], [8 x i32]* [[TMP1]], i64 0, i64 3
-; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds [8 x i32], [8 x i32]* [[TMP1]], i64 0, i64 4
-; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds [8 x i32], [8 x i32]* [[TMP1]], i64 0, i64 6
-; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds [8 x i32], [8 x i32]* [[TMP1]], i64 0, i64 5
-; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds [8 x i32], [8 x i32]* [[TMP1]], i64 0, i64 7
-; CHECK-NEXT:    store <8 x i32> <i32 -221320154, i32 -756426931, i32 563883532, i32 382683935, i32 144890241, i32 -1052877364, i32 -1052877364, i32 -1016007675>, <8 x i32>* [[TMP]], align 32
-; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i32* [[TMP2]] to i8*
-; CHECK-NEXT:    [[TMP13:%.*]] = load i32, i32* [[TMP4]], align 32
-; CHECK-NEXT:    [[TMP14:%.*]] = load i32, i32* [[TMP5]], align 4
-; CHECK-NEXT:    [[TMP15:%.*]] = icmp slt i32 [[TMP14]], [[TMP13]]
-; CHECK-NEXT:    [[TMP16:%.*]] = select i1 [[TMP15]], i32 [[TMP14]], i32 [[TMP13]]
-; CHECK-NEXT:    [[TMP17:%.*]] = zext i1 [[TMP15]] to i32
-; CHECK-NEXT:    [[TMP18:%.*]] = load i32, i32* [[TMP6]], align 8
-; CHECK-NEXT:    [[TMP19:%.*]] = icmp slt i32 [[TMP18]], [[TMP16]]
-; CHECK-NEXT:    [[TMP20:%.*]] = select i1 [[TMP19]], i32 [[TMP18]], i32 [[TMP16]]
-; CHECK-NEXT:    [[TMP21:%.*]] = select i1 [[TMP19]], i32 2, i32 [[TMP16]]
-; CHECK-NEXT:    [[TMP22:%.*]] = load i32, i32* [[TMP7]], align 4
-; CHECK-NEXT:    [[TMP23:%.*]] = icmp slt i32 [[TMP22]], [[TMP20]]
-; CHECK-NEXT:    [[TMP24:%.*]] = select i1 [[TMP23]], i32 [[TMP22]], i32 [[TMP20]]
-; CHECK-NEXT:    [[TMP25:%.*]] = select i1 [[TMP23]], i32 3, i32 [[TMP21]]
-; CHECK-NEXT:    [[TMP26:%.*]] = load i32, i32* [[TMP8]], align 16
-; CHECK-NEXT:    [[TMP27:%.*]] = icmp slt i32 [[TMP26]], [[TMP24]]
-; CHECK-NEXT:    [[TMP28:%.*]] = select i1 [[TMP27]], i32 [[TMP26]], i32 [[TMP24]]
-; CHECK-NEXT:    [[TMP29:%.*]] = select i1 [[TMP27]], i32 4, i32 [[TMP25]]
-; CHECK-NEXT:    [[TMP30:%.*]] = load i32, i32* [[TMP10]], align 4
-; CHECK-NEXT:    [[TMP31:%.*]] = icmp slt i32 [[TMP30]], [[TMP28]]
-; CHECK-NEXT:    [[TMP32:%.*]] = select i1 [[TMP31]], i32 [[TMP30]], i32 [[TMP28]]
-; CHECK-NEXT:    [[TMP33:%.*]] = select i1 [[TMP31]], i32 5, i32 [[TMP29]]
-; CHECK-NEXT:    [[TMP34:%.*]] = load i32, i32* [[TMP9]], align 8
-; CHECK-NEXT:    [[TMP35:%.*]] = icmp slt i32 [[TMP34]], [[TMP32]]
-; CHECK-NEXT:    [[TMP36:%.*]] = select i1 [[TMP35]], i32 [[TMP34]], i32 [[TMP32]]
-; CHECK-NEXT:    [[TMP37:%.*]] = select i1 [[TMP35]], i32 6, i32 [[TMP33]]
-; CHECK-NEXT:    [[TMP38:%.*]] = load i32, i32* [[TMP11]], align 4
-; CHECK-NEXT:    [[TMP39:%.*]] = icmp slt i32 [[TMP38]], [[TMP36]]
-; CHECK-NEXT:    [[TMP40:%.*]] = select i1 [[TMP39]], i32 7, i32 [[TMP37]]
-; CHECK-NEXT:    store i32 [[TMP40]], i32* [[TMP2]], align 4
+; CHECK-NEXT:    [[T:%.*]] = alloca <8 x i32>, align 32
+; CHECK-NEXT:    [[T1:%.*]] = bitcast <8 x i32>* [[T]] to [8 x i32]*
+; CHECK-NEXT:    [[T2:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[T3:%.*]] = bitcast <8 x i32>* [[T]] to i8*
+; CHECK-NEXT:    [[T4:%.*]] = getelementptr inbounds <8 x i32>, <8 x i32>* [[T]], i64 0, i64 0
+; CHECK-NEXT:    [[T5:%.*]] = getelementptr inbounds [8 x i32], [8 x i32]* [[T1]], i64 0, i64 1
+; CHECK-NEXT:    [[T6:%.*]] = getelementptr inbounds [8 x i32], [8 x i32]* [[T1]], i64 0, i64 2
+; CHECK-NEXT:    [[T7:%.*]] = getelementptr inbounds [8 x i32], [8 x i32]* [[T1]], i64 0, i64 3
+; CHECK-NEXT:    [[T8:%.*]] = getelementptr inbounds [8 x i32], [8 x i32]* [[T1]], i64 0, i64 4
+; CHECK-NEXT:    [[T9:%.*]] = getelementptr inbounds [8 x i32], [8 x i32]* [[T1]], i64 0, i64 6
+; CHECK-NEXT:    [[T10:%.*]] = getelementptr inbounds [8 x i32], [8 x i32]* [[T1]], i64 0, i64 5
+; CHECK-NEXT:    [[T11:%.*]] = getelementptr inbounds [8 x i32], [8 x i32]* [[T1]], i64 0, i64 7
+; CHECK-NEXT:    store <8 x i32> <i32 -221320154, i32 -756426931, i32 563883532, i32 382683935, i32 144890241, i32 -1052877364, i32 -1052877364, i32 -1016007675>, <8 x i32>* [[T]], align 32
+; CHECK-NEXT:    [[T12:%.*]] = bitcast i32* [[T2]] to i8*
+; CHECK-NEXT:    [[T13:%.*]] = load i32, i32* [[T4]], align 32
+; CHECK-NEXT:    [[T14:%.*]] = load i32, i32* [[T5]], align 4
+; CHECK-NEXT:    [[T15:%.*]] = icmp slt i32 [[T14]], [[T13]]
+; CHECK-NEXT:    [[T16:%.*]] = select i1 [[T15]], i32 [[T14]], i32 [[T13]]
+; CHECK-NEXT:    [[T17:%.*]] = zext i1 [[T15]] to i32
+; CHECK-NEXT:    [[T18:%.*]] = load i32, i32* [[T6]], align 8
+; CHECK-NEXT:    [[T19:%.*]] = icmp slt i32 [[T18]], [[T16]]
+; CHECK-NEXT:    [[T20:%.*]] = select i1 [[T19]], i32 [[T18]], i32 [[T16]]
+; CHECK-NEXT:    [[T21:%.*]] = select i1 [[T19]], i32 2, i32 [[T16]]
+; CHECK-NEXT:    [[T22:%.*]] = load i32, i32* [[T7]], align 4
+; CHECK-NEXT:    [[T23:%.*]] = icmp slt i32 [[T22]], [[T20]]
+; CHECK-NEXT:    [[T24:%.*]] = select i1 [[T23]], i32 [[T22]], i32 [[T20]]
+; CHECK-NEXT:    [[T25:%.*]] = select i1 [[T23]], i32 3, i32 [[T21]]
+; CHECK-NEXT:    [[T26:%.*]] = load i32, i32* [[T8]], align 16
+; CHECK-NEXT:    [[T27:%.*]] = icmp slt i32 [[T26]], [[T24]]
+; CHECK-NEXT:    [[T28:%.*]] = select i1 [[T27]], i32 [[T26]], i32 [[T24]]
+; CHECK-NEXT:    [[T29:%.*]] = select i1 [[T27]], i32 4, i32 [[T25]]
+; CHECK-NEXT:    [[T30:%.*]] = load i32, i32* [[T10]], align 4
+; CHECK-NEXT:    [[T31:%.*]] = icmp slt i32 [[T30]], [[T28]]
+; CHECK-NEXT:    [[T32:%.*]] = select i1 [[T31]], i32 [[T30]], i32 [[T28]]
+; CHECK-NEXT:    [[T33:%.*]] = select i1 [[T31]], i32 5, i32 [[T29]]
+; CHECK-NEXT:    [[T34:%.*]] = load i32, i32* [[T9]], align 8
+; CHECK-NEXT:    [[T35:%.*]] = icmp slt i32 [[T34]], [[T32]]
+; CHECK-NEXT:    [[T36:%.*]] = select i1 [[T35]], i32 [[T34]], i32 [[T32]]
+; CHECK-NEXT:    [[T37:%.*]] = select i1 [[T35]], i32 6, i32 [[T33]]
+; CHECK-NEXT:    [[T38:%.*]] = load i32, i32* [[T11]], align 4
+; CHECK-NEXT:    [[T39:%.*]] = icmp slt i32 [[T38]], [[T36]]
+; CHECK-NEXT:    [[T40:%.*]] = select i1 [[T39]], i32 7, i32 [[T37]]
+; CHECK-NEXT:    store i32 [[T40]], i32* [[T2]], align 4
 ; CHECK-NEXT:    ret i32 0
 ;
 bb:
-  %tmp = alloca <8 x i32>, align 32
-  %tmp1 = bitcast <8 x i32>* %tmp to [8 x i32]*
-  %tmp2 = alloca i32, align 4
-  %tmp3 = bitcast <8 x i32>* %tmp to i8*
-  %tmp4 = getelementptr inbounds <8 x i32>, <8 x i32>* %tmp, i64 0, i64 0
-  %tmp5 = getelementptr inbounds [8 x i32], [8 x i32]* %tmp1, i64 0, i64 1
-  %tmp6 = getelementptr inbounds [8 x i32], [8 x i32]* %tmp1, i64 0, i64 2
-  %tmp7 = getelementptr inbounds [8 x i32], [8 x i32]* %tmp1, i64 0, i64 3
-  %tmp8 = getelementptr inbounds [8 x i32], [8 x i32]* %tmp1, i64 0, i64 4
-  %tmp9 = getelementptr inbounds [8 x i32], [8 x i32]* %tmp1, i64 0, i64 6
-  %tmp10 = getelementptr inbounds [8 x i32], [8 x i32]* %tmp1, i64 0, i64 5
-  %tmp11 = getelementptr inbounds [8 x i32], [8 x i32]* %tmp1, i64 0, i64 7
-  store <8 x i32> <i32 -221320154, i32 -756426931, i32 563883532, i32 382683935, i32 144890241, i32 -1052877364, i32 -1052877364, i32 -1016007675>, <8 x i32>* %tmp, align 32
-  %tmp12 = bitcast i32* %tmp2 to i8*
-  %tmp13 = load i32, i32* %tmp4, align 32
-  %tmp14 = load i32, i32* %tmp5, align 4
-  %tmp15 = icmp slt i32 %tmp14, %tmp13
-  %tmp16 = select i1 %tmp15, i32 %tmp14, i32 %tmp13
-  %tmp17 = zext i1 %tmp15 to i32
-  %tmp18 = load i32, i32* %tmp6, align 8
-  %tmp19 = icmp slt i32 %tmp18, %tmp16
-  %tmp20 = select i1 %tmp19, i32 %tmp18, i32 %tmp16
-  %tmp21 = select i1 %tmp19, i32 2, i32 %tmp16
-  %tmp22 = load i32, i32* %tmp7, align 4
-  %tmp23 = icmp slt i32 %tmp22, %tmp20
-  %tmp24 = select i1 %tmp23, i32 %tmp22, i32 %tmp20
-  %tmp25 = select i1 %tmp23, i32 3, i32 %tmp21
-  %tmp26 = load i32, i32* %tmp8, align 16
-  %tmp27 = icmp slt i32 %tmp26, %tmp24
-  %tmp28 = select i1 %tmp27, i32 %tmp26, i32 %tmp24
-  %tmp29 = select i1 %tmp27, i32 4, i32 %tmp25
-  %tmp30 = load i32, i32* %tmp10, align 4
-  %tmp31 = icmp slt i32 %tmp30, %tmp28
-  %tmp32 = select i1 %tmp31, i32 %tmp30, i32 %tmp28
-  %tmp33 = select i1 %tmp31, i32 5, i32 %tmp29
-  %tmp34 = load i32, i32* %tmp9, align 8
-  %tmp35 = icmp slt i32 %tmp34, %tmp32
-  %tmp36 = select i1 %tmp35, i32 %tmp34, i32 %tmp32
-  %tmp37 = select i1 %tmp35, i32 6, i32 %tmp33
-  %tmp38 = load i32, i32* %tmp11, align 4
-  %tmp39 = icmp slt i32 %tmp38, %tmp36
-  %tmp40 = select i1 %tmp39, i32 7, i32 %tmp37
-  store i32 %tmp40, i32* %tmp2, align 4
+  %t = alloca <8 x i32>, align 32
+  %t1 = bitcast <8 x i32>* %t to [8 x i32]*
+  %t2 = alloca i32, align 4
+  %t3 = bitcast <8 x i32>* %t to i8*
+  %t4 = getelementptr inbounds <8 x i32>, <8 x i32>* %t, i64 0, i64 0
+  %t5 = getelementptr inbounds [8 x i32], [8 x i32]* %t1, i64 0, i64 1
+  %t6 = getelementptr inbounds [8 x i32], [8 x i32]* %t1, i64 0, i64 2
+  %t7 = getelementptr inbounds [8 x i32], [8 x i32]* %t1, i64 0, i64 3
+  %t8 = getelementptr inbounds [8 x i32], [8 x i32]* %t1, i64 0, i64 4
+  %t9 = getelementptr inbounds [8 x i32], [8 x i32]* %t1, i64 0, i64 6
+  %t10 = getelementptr inbounds [8 x i32], [8 x i32]* %t1, i64 0, i64 5
+  %t11 = getelementptr inbounds [8 x i32], [8 x i32]* %t1, i64 0, i64 7
+  store <8 x i32> <i32 -221320154, i32 -756426931, i32 563883532, i32 382683935, i32 144890241, i32 -1052877364, i32 -1052877364, i32 -1016007675>, <8 x i32>* %t, align 32
+  %t12 = bitcast i32* %t2 to i8*
+  %t13 = load i32, i32* %t4, align 32
+  %t14 = load i32, i32* %t5, align 4
+  %t15 = icmp slt i32 %t14, %t13
+  %t16 = select i1 %t15, i32 %t14, i32 %t13
+  %t17 = zext i1 %t15 to i32
+  %t18 = load i32, i32* %t6, align 8
+  %t19 = icmp slt i32 %t18, %t16
+  %t20 = select i1 %t19, i32 %t18, i32 %t16
+  %t21 = select i1 %t19, i32 2, i32 %t16
+  %t22 = load i32, i32* %t7, align 4
+  %t23 = icmp slt i32 %t22, %t20
+  %t24 = select i1 %t23, i32 %t22, i32 %t20
+  %t25 = select i1 %t23, i32 3, i32 %t21
+  %t26 = load i32, i32* %t8, align 16
+  %t27 = icmp slt i32 %t26, %t24
+  %t28 = select i1 %t27, i32 %t26, i32 %t24
+  %t29 = select i1 %t27, i32 4, i32 %t25
+  %t30 = load i32, i32* %t10, align 4
+  %t31 = icmp slt i32 %t30, %t28
+  %t32 = select i1 %t31, i32 %t30, i32 %t28
+  %t33 = select i1 %t31, i32 5, i32 %t29
+  %t34 = load i32, i32* %t9, align 8
+  %t35 = icmp slt i32 %t34, %t32
+  %t36 = select i1 %t35, i32 %t34, i32 %t32
+  %t37 = select i1 %t35, i32 6, i32 %t33
+  %t38 = load i32, i32* %t11, align 4
+  %t39 = icmp slt i32 %t38, %t36
+  %t40 = select i1 %t39, i32 7, i32 %t37
+  store i32 %t40, i32* %t2, align 4
   ret i32 0
 }
-
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/PR35628_1.ll b/llvm/test/Transforms/SLPVectorizer/X86/PR35628_1.ll
index 625748f68..e033ed768 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/PR35628_1.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/PR35628_1.ll
@@ -18,23 +18,16 @@ define void @mainTest(i32* %ptr) #0  {
 ; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x i32> [[TMP4]], i32 2
 ; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x i32> [[TMP4]], i32 1
 ; CHECK-NEXT:    [[TMP8:%.*]] = mul <4 x i32> [[TMP4]], [[TMP4]]
-; CHECK-NEXT:    [[TMP9:%.*]] = add i32 1, undef
-; CHECK-NEXT:    [[TMP10:%.*]] = add i32 [[TMP9]], [[TMP7]]
-; CHECK-NEXT:    [[TMP11:%.*]] = add i32 [[TMP10]], undef
-; CHECK-NEXT:    [[TMP12:%.*]] = add i32 [[TMP11]], [[TMP6]]
-; CHECK-NEXT:    [[TMP13:%.*]] = add i32 [[TMP12]], undef
-; CHECK-NEXT:    [[TMP14:%.*]] = sext i32 [[TMP6]] to i64
-; CHECK-NEXT:    [[TMP15:%.*]] = add i32 [[TMP13]], [[TMP5]]
+; CHECK-NEXT:    [[TMP9:%.*]] = sext i32 [[TMP6]] to i64
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <4 x i32> [[TMP8]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x i32> [[BIN_RDX]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX2:%.*]] = add <4 x i32> [[BIN_RDX]], [[RDX_SHUF1]]
-; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <4 x i32> [[BIN_RDX2]], i32 0
-; CHECK-NEXT:    [[OP_EXTRA:%.*]] = add i32 [[TMP16]], 1
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <4 x i32> [[BIN_RDX2]], i32 0
+; CHECK-NEXT:    [[OP_EXTRA:%.*]] = add i32 [[TMP10]], 1
 ; CHECK-NEXT:    [[OP_EXTRA3:%.*]] = add i32 [[OP_EXTRA]], [[TMP7]]
 ; CHECK-NEXT:    [[OP_EXTRA4:%.*]] = add i32 [[OP_EXTRA3]], [[TMP6]]
 ; CHECK-NEXT:    [[OP_EXTRA5]] = add i32 [[OP_EXTRA4]], [[TMP5]]
-; CHECK-NEXT:    [[TMP17:%.*]] = add i32 [[TMP15]], undef
 ; CHECK-NEXT:    br label [[LOOP]]
 ; CHECK:       bail_out:
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/PR35628_2.ll b/llvm/test/Transforms/SLPVectorizer/X86/PR35628_2.ll
index 712ff040a..4916d283b 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/PR35628_2.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/PR35628_2.ll
@@ -20,10 +20,6 @@ define void @test() #0 {
 ; CHECK-NEXT:    [[DUMMY_SHL:%.*]] = shl i64 [[TMP7]], 32
 ; CHECK-NEXT:    [[TMP8:%.*]] = add <4 x i64> <i64 1, i64 1, i64 1, i64 1>, [[TMP5]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = ashr exact <4 x i64> [[TMP8]], <i64 32, i64 32, i64 32, i64 32>
-; CHECK-NEXT:    [[SUM1:%.*]] = add i64 undef, undef
-; CHECK-NEXT:    [[SUM2:%.*]] = add i64 [[SUM1]], undef
-; CHECK-NEXT:    [[ZSUM:%.*]] = add i64 [[SUM2]], 0
-; CHECK-NEXT:    [[JOIN:%.*]] = add i64 [[TMP6]], [[ZSUM]]
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i64> [[TMP9]], <4 x i64> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <4 x i64> [[TMP9]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x i64> [[BIN_RDX]], <4 x i64> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
@@ -31,7 +27,6 @@ define void @test() #0 {
 ; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <4 x i64> [[BIN_RDX2]], i32 0
 ; CHECK-NEXT:    [[OP_EXTRA:%.*]] = add i64 [[TMP10]], 0
 ; CHECK-NEXT:    [[OP_EXTRA3]] = add i64 [[OP_EXTRA]], [[TMP6]]
-; CHECK-NEXT:    [[LAST:%.*]] = add i64 [[JOIN]], undef
 ; CHECK-NEXT:    br label [[LOOP]]
 ;
 entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/PR35777.ll b/llvm/test/Transforms/SLPVectorizer/X86/PR35777.ll
index 4a403e7b9..53215d612 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/PR35777.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/PR35777.ll
@@ -4,7 +4,7 @@
 @global = local_unnamed_addr global [6 x double] zeroinitializer, align 16
 
 define { i64, i64 } @patatino(double %arg) {
-; CHECK-LABEL: @patatino(
+; CHECK-LABEL: define {{[^@]+}}@patatino(
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x double>, <2 x double>* bitcast ([6 x double]* @global to <2 x double>*), align 16
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* bitcast (double* getelementptr inbounds ([6 x double], [6 x double]* @global, i64 0, i64 2) to <2 x double>*), align 16
@@ -17,29 +17,29 @@ define { i64, i64 } @patatino(double %arg) {
 ; CHECK-NEXT:    [[TMP8:%.*]] = fptosi <2 x double> [[TMP7]] to <2 x i32>
 ; CHECK-NEXT:    [[TMP9:%.*]] = sext <2 x i32> [[TMP8]] to <2 x i64>
 ; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <2 x i64> [[TMP9]], i32 0
-; CHECK-NEXT:    [[TMP16:%.*]] = insertvalue { i64, i64 } undef, i64 [[TMP10]], 0
+; CHECK-NEXT:    [[T16:%.*]] = insertvalue { i64, i64 } undef, i64 [[TMP10]], 0
 ; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <2 x i64> [[TMP9]], i32 1
-; CHECK-NEXT:    [[TMP17:%.*]] = insertvalue { i64, i64 } [[TMP16]], i64 [[TMP11]], 1
-; CHECK-NEXT:    ret { i64, i64 } [[TMP17]]
+; CHECK-NEXT:    [[T17:%.*]] = insertvalue { i64, i64 } [[T16]], i64 [[TMP11]], 1
+; CHECK-NEXT:    ret { i64, i64 } [[T17]]
 ;
 bb:
-  %tmp = load double, double* getelementptr inbounds ([6 x double], [6 x double]* @global, i64 0, i64 0), align 16
-  %tmp1 = load double, double* getelementptr inbounds ([6 x double], [6 x double]* @global, i64 0, i64 2), align 16
-  %tmp2 = fmul double %tmp1, %arg
-  %tmp3 = fadd double %tmp, %tmp2
-  %tmp4 = load double, double* getelementptr inbounds ([6 x double], [6 x double]* @global, i64 0, i64 4), align 16
-  %tmp5 = fadd double %tmp4, %tmp3
-  %tmp6 = fptosi double %tmp5 to i32
-  %tmp7 = sext i32 %tmp6 to i64
-  %tmp8 = load double, double* getelementptr inbounds ([6 x double], [6 x double]* @global, i64 0, i64 1), align 8
-  %tmp9 = load double, double* getelementptr inbounds ([6 x double], [6 x double]* @global, i64 0, i64 3), align 8
-  %tmp10 = fmul double %tmp9, %arg
-  %tmp11 = fadd double %tmp8, %tmp10
-  %tmp12 = load double, double* getelementptr inbounds ([6 x double], [6 x double]* @global, i64 0, i64 5), align 8
-  %tmp13 = fadd double %tmp12, %tmp11
-  %tmp14 = fptosi double %tmp13 to i32
-  %tmp15 = sext i32 %tmp14 to i64
-  %tmp16 = insertvalue { i64, i64 } undef, i64 %tmp7, 0
-  %tmp17 = insertvalue { i64, i64 } %tmp16, i64 %tmp15, 1
-  ret { i64, i64 } %tmp17
+  %t = load double, double* getelementptr inbounds ([6 x double], [6 x double]* @global, i64 0, i64 0), align 16
+  %t1 = load double, double* getelementptr inbounds ([6 x double], [6 x double]* @global, i64 0, i64 2), align 16
+  %t2 = fmul double %t1, %arg
+  %t3 = fadd double %t, %t2
+  %t4 = load double, double* getelementptr inbounds ([6 x double], [6 x double]* @global, i64 0, i64 4), align 16
+  %t5 = fadd double %t4, %t3
+  %t6 = fptosi double %t5 to i32
+  %t7 = sext i32 %t6 to i64
+  %t8 = load double, double* getelementptr inbounds ([6 x double], [6 x double]* @global, i64 0, i64 1), align 8
+  %t9 = load double, double* getelementptr inbounds ([6 x double], [6 x double]* @global, i64 0, i64 3), align 8
+  %t10 = fmul double %t9, %arg
+  %t11 = fadd double %t8, %t10
+  %t12 = load double, double* getelementptr inbounds ([6 x double], [6 x double]* @global, i64 0, i64 5), align 8
+  %t13 = fadd double %t12, %t11
+  %t14 = fptosi double %t13 to i32
+  %t15 = sext i32 %t14 to i64
+  %t16 = insertvalue { i64, i64 } undef, i64 %t7, 0
+  %t17 = insertvalue { i64, i64 } %t16, i64 %t15, 1
+  ret { i64, i64 } %t17
 }
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/PR39774.ll b/llvm/test/Transforms/SLPVectorizer/X86/PR39774.ll
index 24f75b32c..db02d64ba 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/PR39774.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/PR39774.ll
@@ -11,40 +11,6 @@ define void @Test(i32) {
 ; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> undef, <8 x i32> <i32 0, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
 ; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <8 x i32> [[SHUFFLE]], i32 1
 ; CHECK-NEXT:    [[TMP3:%.*]] = add <8 x i32> [[SHUFFLE]], <i32 0, i32 55, i32 285, i32 1240, i32 1496, i32 8555, i32 12529, i32 13685>
-; CHECK-NEXT:    [[VAL_1:%.*]] = and i32 [[TMP2]], undef
-; CHECK-NEXT:    [[VAL_2:%.*]] = and i32 [[VAL_1]], [[TMP0:%.*]]
-; CHECK-NEXT:    [[VAL_3:%.*]] = and i32 [[VAL_2]], [[TMP0]]
-; CHECK-NEXT:    [[VAL_4:%.*]] = and i32 [[VAL_3]], [[TMP0]]
-; CHECK-NEXT:    [[VAL_5:%.*]] = and i32 [[VAL_4]], [[TMP0]]
-; CHECK-NEXT:    [[VAL_7:%.*]] = and i32 [[VAL_5]], undef
-; CHECK-NEXT:    [[VAL_8:%.*]] = and i32 [[VAL_7]], [[TMP0]]
-; CHECK-NEXT:    [[VAL_9:%.*]] = and i32 [[VAL_8]], [[TMP0]]
-; CHECK-NEXT:    [[VAL_10:%.*]] = and i32 [[VAL_9]], [[TMP0]]
-; CHECK-NEXT:    [[VAL_12:%.*]] = and i32 [[VAL_10]], undef
-; CHECK-NEXT:    [[VAL_13:%.*]] = and i32 [[VAL_12]], [[TMP0]]
-; CHECK-NEXT:    [[VAL_14:%.*]] = and i32 [[VAL_13]], [[TMP0]]
-; CHECK-NEXT:    [[VAL_15:%.*]] = and i32 [[VAL_14]], [[TMP0]]
-; CHECK-NEXT:    [[VAL_16:%.*]] = and i32 [[VAL_15]], [[TMP0]]
-; CHECK-NEXT:    [[VAL_17:%.*]] = and i32 [[VAL_16]], [[TMP0]]
-; CHECK-NEXT:    [[VAL_19:%.*]] = and i32 [[VAL_17]], undef
-; CHECK-NEXT:    [[VAL_21:%.*]] = and i32 [[VAL_19]], undef
-; CHECK-NEXT:    [[VAL_22:%.*]] = and i32 [[VAL_21]], [[TMP0]]
-; CHECK-NEXT:    [[VAL_23:%.*]] = and i32 [[VAL_22]], [[TMP0]]
-; CHECK-NEXT:    [[VAL_24:%.*]] = and i32 [[VAL_23]], [[TMP0]]
-; CHECK-NEXT:    [[VAL_25:%.*]] = and i32 [[VAL_24]], [[TMP0]]
-; CHECK-NEXT:    [[VAL_26:%.*]] = and i32 [[VAL_25]], [[TMP0]]
-; CHECK-NEXT:    [[VAL_27:%.*]] = and i32 [[VAL_26]], [[TMP0]]
-; CHECK-NEXT:    [[VAL_28:%.*]] = and i32 [[VAL_27]], [[TMP0]]
-; CHECK-NEXT:    [[VAL_29:%.*]] = and i32 [[VAL_28]], [[TMP0]]
-; CHECK-NEXT:    [[VAL_30:%.*]] = and i32 [[VAL_29]], [[TMP0]]
-; CHECK-NEXT:    [[VAL_31:%.*]] = and i32 [[VAL_30]], [[TMP0]]
-; CHECK-NEXT:    [[VAL_32:%.*]] = and i32 [[VAL_31]], [[TMP0]]
-; CHECK-NEXT:    [[VAL_33:%.*]] = and i32 [[VAL_32]], [[TMP0]]
-; CHECK-NEXT:    [[VAL_35:%.*]] = and i32 [[VAL_33]], undef
-; CHECK-NEXT:    [[VAL_36:%.*]] = and i32 [[VAL_35]], [[TMP0]]
-; CHECK-NEXT:    [[VAL_37:%.*]] = and i32 [[VAL_36]], [[TMP0]]
-; CHECK-NEXT:    [[VAL_38:%.*]] = and i32 [[VAL_37]], [[TMP0]]
-; CHECK-NEXT:    [[VAL_40:%.*]] = and i32 [[VAL_38]], undef
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x i32> [[TMP3]], <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = and <8 x i32> [[TMP3]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x i32> [[BIN_RDX]], <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -52,7 +18,7 @@ define void @Test(i32) {
 ; CHECK-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <8 x i32> [[BIN_RDX2]], <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX4:%.*]] = and <8 x i32> [[BIN_RDX2]], [[RDX_SHUF3]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <8 x i32> [[BIN_RDX4]], i32 0
-; CHECK-NEXT:    [[OP_EXTRA:%.*]] = and i32 [[TMP4]], [[TMP0]]
+; CHECK-NEXT:    [[OP_EXTRA:%.*]] = and i32 [[TMP4]], [[TMP0:%.*]]
 ; CHECK-NEXT:    [[OP_EXTRA5:%.*]] = and i32 [[OP_EXTRA]], [[TMP0]]
 ; CHECK-NEXT:    [[OP_EXTRA6:%.*]] = and i32 [[OP_EXTRA5]], [[TMP0]]
 ; CHECK-NEXT:    [[OP_EXTRA7:%.*]] = and i32 [[OP_EXTRA6]], [[TMP0]]
@@ -79,7 +45,6 @@ define void @Test(i32) {
 ; CHECK-NEXT:    [[OP_EXTRA28:%.*]] = and i32 [[OP_EXTRA27]], [[TMP0]]
 ; CHECK-NEXT:    [[OP_EXTRA29:%.*]] = and i32 [[OP_EXTRA28]], [[TMP0]]
 ; CHECK-NEXT:    [[OP_EXTRA30:%.*]] = and i32 [[OP_EXTRA29]], [[TMP0]]
-; CHECK-NEXT:    [[VAL_42:%.*]] = and i32 [[VAL_40]], undef
 ; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <2 x i32> undef, i32 [[OP_EXTRA30]], i32 0
 ; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i32> [[TMP5]], i32 14910, i32 1
 ; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <2 x i32> undef, i32 [[TMP2]], i32 0
@@ -101,40 +66,8 @@ define void @Test(i32) {
 ; FORCE_REDUCTION-NEXT:    [[SHUFFLE:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 1, i32 1>
 ; FORCE_REDUCTION-NEXT:    [[TMP2:%.*]] = extractelement <4 x i32> [[SHUFFLE]], i32 1
 ; FORCE_REDUCTION-NEXT:    [[TMP3:%.*]] = add <4 x i32> [[SHUFFLE]], <i32 0, i32 55, i32 285, i32 1240>
-; FORCE_REDUCTION-NEXT:    [[VAL_1:%.*]] = and i32 [[TMP2]], undef
-; FORCE_REDUCTION-NEXT:    [[VAL_2:%.*]] = and i32 [[VAL_1]], [[TMP0:%.*]]
-; FORCE_REDUCTION-NEXT:    [[VAL_3:%.*]] = and i32 [[VAL_2]], [[TMP0]]
-; FORCE_REDUCTION-NEXT:    [[VAL_4:%.*]] = and i32 [[VAL_3]], [[TMP0]]
-; FORCE_REDUCTION-NEXT:    [[VAL_5:%.*]] = and i32 [[VAL_4]], [[TMP0]]
-; FORCE_REDUCTION-NEXT:    [[VAL_7:%.*]] = and i32 [[VAL_5]], undef
-; FORCE_REDUCTION-NEXT:    [[VAL_8:%.*]] = and i32 [[VAL_7]], [[TMP0]]
-; FORCE_REDUCTION-NEXT:    [[VAL_9:%.*]] = and i32 [[VAL_8]], [[TMP0]]
-; FORCE_REDUCTION-NEXT:    [[VAL_10:%.*]] = and i32 [[VAL_9]], [[TMP0]]
-; FORCE_REDUCTION-NEXT:    [[VAL_12:%.*]] = and i32 [[VAL_10]], undef
-; FORCE_REDUCTION-NEXT:    [[VAL_13:%.*]] = and i32 [[VAL_12]], [[TMP0]]
-; FORCE_REDUCTION-NEXT:    [[VAL_14:%.*]] = and i32 [[VAL_13]], [[TMP0]]
-; FORCE_REDUCTION-NEXT:    [[VAL_15:%.*]] = and i32 [[VAL_14]], [[TMP0]]
-; FORCE_REDUCTION-NEXT:    [[VAL_16:%.*]] = and i32 [[VAL_15]], [[TMP0]]
-; FORCE_REDUCTION-NEXT:    [[VAL_17:%.*]] = and i32 [[VAL_16]], [[TMP0]]
-; FORCE_REDUCTION-NEXT:    [[VAL_19:%.*]] = and i32 [[VAL_17]], undef
 ; FORCE_REDUCTION-NEXT:    [[VAL_20:%.*]] = add i32 [[TMP2]], 1496
-; FORCE_REDUCTION-NEXT:    [[VAL_21:%.*]] = and i32 [[VAL_19]], [[VAL_20]]
-; FORCE_REDUCTION-NEXT:    [[VAL_22:%.*]] = and i32 [[VAL_21]], [[TMP0]]
-; FORCE_REDUCTION-NEXT:    [[VAL_23:%.*]] = and i32 [[VAL_22]], [[TMP0]]
-; FORCE_REDUCTION-NEXT:    [[VAL_24:%.*]] = and i32 [[VAL_23]], [[TMP0]]
-; FORCE_REDUCTION-NEXT:    [[VAL_25:%.*]] = and i32 [[VAL_24]], [[TMP0]]
-; FORCE_REDUCTION-NEXT:    [[VAL_26:%.*]] = and i32 [[VAL_25]], [[TMP0]]
-; FORCE_REDUCTION-NEXT:    [[VAL_27:%.*]] = and i32 [[VAL_26]], [[TMP0]]
-; FORCE_REDUCTION-NEXT:    [[VAL_28:%.*]] = and i32 [[VAL_27]], [[TMP0]]
-; FORCE_REDUCTION-NEXT:    [[VAL_29:%.*]] = and i32 [[VAL_28]], [[TMP0]]
-; FORCE_REDUCTION-NEXT:    [[VAL_30:%.*]] = and i32 [[VAL_29]], [[TMP0]]
-; FORCE_REDUCTION-NEXT:    [[VAL_31:%.*]] = and i32 [[VAL_30]], [[TMP0]]
-; FORCE_REDUCTION-NEXT:    [[VAL_32:%.*]] = and i32 [[VAL_31]], [[TMP0]]
-; FORCE_REDUCTION-NEXT:    [[VAL_33:%.*]] = and i32 [[VAL_32]], [[TMP0]]
 ; FORCE_REDUCTION-NEXT:    [[VAL_34:%.*]] = add i32 [[TMP2]], 8555
-; FORCE_REDUCTION-NEXT:    [[VAL_35:%.*]] = and i32 [[VAL_33]], [[VAL_34]]
-; FORCE_REDUCTION-NEXT:    [[VAL_36:%.*]] = and i32 [[VAL_35]], [[TMP0]]
-; FORCE_REDUCTION-NEXT:    [[VAL_37:%.*]] = and i32 [[VAL_36]], [[TMP0]]
 ; FORCE_REDUCTION-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
 ; FORCE_REDUCTION-NEXT:    [[BIN_RDX:%.*]] = and <4 x i32> [[TMP3]], [[RDX_SHUF]]
 ; FORCE_REDUCTION-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x i32> [[BIN_RDX]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
@@ -142,7 +75,7 @@ define void @Test(i32) {
 ; FORCE_REDUCTION-NEXT:    [[TMP4:%.*]] = extractelement <4 x i32> [[BIN_RDX2]], i32 0
 ; FORCE_REDUCTION-NEXT:    [[TMP5:%.*]] = and i32 [[TMP4]], [[VAL_20]]
 ; FORCE_REDUCTION-NEXT:    [[TMP6:%.*]] = and i32 [[TMP5]], [[VAL_34]]
-; FORCE_REDUCTION-NEXT:    [[OP_EXTRA:%.*]] = and i32 [[TMP6]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[OP_EXTRA:%.*]] = and i32 [[TMP6]], [[TMP0:%.*]]
 ; FORCE_REDUCTION-NEXT:    [[OP_EXTRA3:%.*]] = and i32 [[OP_EXTRA]], [[TMP0]]
 ; FORCE_REDUCTION-NEXT:    [[OP_EXTRA4:%.*]] = and i32 [[OP_EXTRA3]], [[TMP0]]
 ; FORCE_REDUCTION-NEXT:    [[OP_EXTRA5:%.*]] = and i32 [[OP_EXTRA4]], [[TMP0]]
@@ -170,7 +103,6 @@ define void @Test(i32) {
 ; FORCE_REDUCTION-NEXT:    [[OP_EXTRA27:%.*]] = and i32 [[OP_EXTRA26]], [[TMP0]]
 ; FORCE_REDUCTION-NEXT:    [[OP_EXTRA28:%.*]] = and i32 [[OP_EXTRA27]], [[TMP0]]
 ; FORCE_REDUCTION-NEXT:    [[OP_EXTRA29:%.*]] = and i32 [[OP_EXTRA28]], [[TMP2]]
-; FORCE_REDUCTION-NEXT:    [[VAL_38:%.*]] = and i32 [[VAL_37]], [[TMP0]]
 ; FORCE_REDUCTION-NEXT:    [[VAL_39:%.*]] = add i32 [[TMP2]], 12529
 ; FORCE_REDUCTION-NEXT:    [[VAL_40:%.*]] = and i32 [[OP_EXTRA29]], [[VAL_39]]
 ; FORCE_REDUCTION-NEXT:    [[VAL_41:%.*]] = add i32 [[TMP2]], 13685
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/PR40310.ll b/llvm/test/Transforms/SLPVectorizer/X86/PR40310.ll
index 2a0b66ee2..4622c77e8 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/PR40310.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/PR40310.ll
@@ -13,21 +13,6 @@ define void @mainTest(i32 %param, i32 * %vals, i32 %len) {
 ; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <16 x i32> [[SHUFFLE]], i32 15
 ; CHECK-NEXT:    store atomic i32 [[TMP3]], i32* [[VALS:%.*]] unordered, align 4
 ; CHECK-NEXT:    [[TMP4:%.*]] = add <16 x i32> [[SHUFFLE]], <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 -1>
-; CHECK-NEXT:    [[V14:%.*]] = and i32 [[TMP2]], undef
-; CHECK-NEXT:    [[V16:%.*]] = and i32 undef, [[V14]]
-; CHECK-NEXT:    [[V18:%.*]] = and i32 undef, [[V16]]
-; CHECK-NEXT:    [[V20:%.*]] = and i32 undef, [[V18]]
-; CHECK-NEXT:    [[V22:%.*]] = and i32 undef, [[V20]]
-; CHECK-NEXT:    [[V24:%.*]] = and i32 undef, [[V22]]
-; CHECK-NEXT:    [[V26:%.*]] = and i32 undef, [[V24]]
-; CHECK-NEXT:    [[V28:%.*]] = and i32 undef, [[V26]]
-; CHECK-NEXT:    [[V30:%.*]] = and i32 undef, [[V28]]
-; CHECK-NEXT:    [[V32:%.*]] = and i32 undef, [[V30]]
-; CHECK-NEXT:    [[V34:%.*]] = and i32 undef, [[V32]]
-; CHECK-NEXT:    [[V36:%.*]] = and i32 undef, [[V34]]
-; CHECK-NEXT:    [[V38:%.*]] = and i32 undef, [[V36]]
-; CHECK-NEXT:    [[V40:%.*]] = and i32 undef, [[V38]]
-; CHECK-NEXT:    [[V42:%.*]] = and i32 undef, [[V40]]
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <16 x i32> [[TMP4]], <16 x i32> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = and <16 x i32> [[TMP4]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <16 x i32> [[BIN_RDX]], <16 x i32> undef, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -38,7 +23,6 @@ define void @mainTest(i32 %param, i32 * %vals, i32 %len) {
 ; CHECK-NEXT:    [[BIN_RDX6:%.*]] = and <16 x i32> [[BIN_RDX4]], [[RDX_SHUF5]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <16 x i32> [[BIN_RDX6]], i32 0
 ; CHECK-NEXT:    [[OP_EXTRA:%.*]] = and i32 [[TMP5]], [[TMP2]]
-; CHECK-NEXT:    [[V43:%.*]] = and i32 undef, [[V42]]
 ; CHECK-NEXT:    [[V44:%.*]] = add i32 [[TMP2]], 16
 ; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i32> undef, i32 [[V44]], i32 0
 ; CHECK-NEXT:    [[TMP7]] = insertelement <2 x i32> [[TMP6]], i32 [[OP_EXTRA]], i32 1
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/align.ll b/llvm/test/Transforms/SLPVectorizer/X86/align.ll
index 5c7c4ceae..c27890ca1 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/align.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/align.ll
@@ -39,7 +39,7 @@ entry:
   ret void
 }
 
-; Float has 4 byte abi alignment on x86_64. We must use the alignmnet of the
+; Float has 4 byte abi alignment on x86_64. We must use the alignment of the
 ; value being loaded/stored not the alignment of the pointer type.
 
 define void @test2(float * %a, float * %b) {
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/alternate-cast.ll b/llvm/test/Transforms/SLPVectorizer/X86/alternate-cast.ll
index 8f8b1d443..9ee016e43 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/alternate-cast.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/alternate-cast.ll
@@ -35,30 +35,9 @@ define <8 x float> @sitofp_uitofp(<8 x i32> %a) {
 ; SSE-NEXT:    ret <8 x float> [[R7]]
 ;
 ; SLM-LABEL: @sitofp_uitofp(
-; SLM-NEXT:    [[A0:%.*]] = extractelement <8 x i32> [[A:%.*]], i32 0
-; SLM-NEXT:    [[A1:%.*]] = extractelement <8 x i32> [[A]], i32 1
-; SLM-NEXT:    [[A2:%.*]] = extractelement <8 x i32> [[A]], i32 2
-; SLM-NEXT:    [[A3:%.*]] = extractelement <8 x i32> [[A]], i32 3
-; SLM-NEXT:    [[A4:%.*]] = extractelement <8 x i32> [[A]], i32 4
-; SLM-NEXT:    [[A5:%.*]] = extractelement <8 x i32> [[A]], i32 5
-; SLM-NEXT:    [[A6:%.*]] = extractelement <8 x i32> [[A]], i32 6
-; SLM-NEXT:    [[A7:%.*]] = extractelement <8 x i32> [[A]], i32 7
-; SLM-NEXT:    [[AB0:%.*]] = sitofp i32 [[A0]] to float
-; SLM-NEXT:    [[AB1:%.*]] = sitofp i32 [[A1]] to float
-; SLM-NEXT:    [[AB2:%.*]] = sitofp i32 [[A2]] to float
-; SLM-NEXT:    [[AB3:%.*]] = sitofp i32 [[A3]] to float
-; SLM-NEXT:    [[AB4:%.*]] = uitofp i32 [[A4]] to float
-; SLM-NEXT:    [[AB5:%.*]] = uitofp i32 [[A5]] to float
-; SLM-NEXT:    [[AB6:%.*]] = uitofp i32 [[A6]] to float
-; SLM-NEXT:    [[AB7:%.*]] = uitofp i32 [[A7]] to float
-; SLM-NEXT:    [[R0:%.*]] = insertelement <8 x float> undef, float [[AB0]], i32 0
-; SLM-NEXT:    [[R1:%.*]] = insertelement <8 x float> [[R0]], float [[AB1]], i32 1
-; SLM-NEXT:    [[R2:%.*]] = insertelement <8 x float> [[R1]], float [[AB2]], i32 2
-; SLM-NEXT:    [[R3:%.*]] = insertelement <8 x float> [[R2]], float [[AB3]], i32 3
-; SLM-NEXT:    [[R4:%.*]] = insertelement <8 x float> [[R3]], float [[AB4]], i32 4
-; SLM-NEXT:    [[R5:%.*]] = insertelement <8 x float> [[R4]], float [[AB5]], i32 5
-; SLM-NEXT:    [[R6:%.*]] = insertelement <8 x float> [[R5]], float [[AB6]], i32 6
-; SLM-NEXT:    [[R7:%.*]] = insertelement <8 x float> [[R6]], float [[AB7]], i32 7
+; SLM-NEXT:    [[TMP1:%.*]] = sitofp <8 x i32> [[A:%.*]] to <8 x float>
+; SLM-NEXT:    [[TMP2:%.*]] = uitofp <8 x i32> [[A]] to <8 x float>
+; SLM-NEXT:    [[R7:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
 ; SLM-NEXT:    ret <8 x float> [[R7]]
 ;
 ; AVX-LABEL: @sitofp_uitofp(
@@ -268,11 +247,50 @@ define <8 x float> @fneg_fabs(<8 x float> %a) {
 }
 
 define <8 x i32> @sext_zext(<8 x i16> %a) {
-; CHECK-LABEL: @sext_zext(
-; CHECK-NEXT:    [[TMP1:%.*]] = sext <8 x i16> [[A:%.*]] to <8 x i32>
-; CHECK-NEXT:    [[TMP2:%.*]] = zext <8 x i16> [[A]] to <8 x i32>
-; CHECK-NEXT:    [[R7:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT:    ret <8 x i32> [[R7]]
+; SSE-LABEL: @sext_zext(
+; SSE-NEXT:    [[TMP1:%.*]] = sext <8 x i16> [[A:%.*]] to <8 x i32>
+; SSE-NEXT:    [[TMP2:%.*]] = zext <8 x i16> [[A]] to <8 x i32>
+; SSE-NEXT:    [[R7:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
+; SSE-NEXT:    ret <8 x i32> [[R7]]
+;
+; SLM-LABEL: @sext_zext(
+; SLM-NEXT:    [[A0:%.*]] = extractelement <8 x i16> [[A:%.*]], i32 0
+; SLM-NEXT:    [[A1:%.*]] = extractelement <8 x i16> [[A]], i32 1
+; SLM-NEXT:    [[A2:%.*]] = extractelement <8 x i16> [[A]], i32 2
+; SLM-NEXT:    [[A3:%.*]] = extractelement <8 x i16> [[A]], i32 3
+; SLM-NEXT:    [[A4:%.*]] = extractelement <8 x i16> [[A]], i32 4
+; SLM-NEXT:    [[A5:%.*]] = extractelement <8 x i16> [[A]], i32 5
+; SLM-NEXT:    [[A6:%.*]] = extractelement <8 x i16> [[A]], i32 6
+; SLM-NEXT:    [[A7:%.*]] = extractelement <8 x i16> [[A]], i32 7
+; SLM-NEXT:    [[AB0:%.*]] = sext i16 [[A0]] to i32
+; SLM-NEXT:    [[AB1:%.*]] = sext i16 [[A1]] to i32
+; SLM-NEXT:    [[AB2:%.*]] = sext i16 [[A2]] to i32
+; SLM-NEXT:    [[AB3:%.*]] = sext i16 [[A3]] to i32
+; SLM-NEXT:    [[AB4:%.*]] = zext i16 [[A4]] to i32
+; SLM-NEXT:    [[AB5:%.*]] = zext i16 [[A5]] to i32
+; SLM-NEXT:    [[AB6:%.*]] = zext i16 [[A6]] to i32
+; SLM-NEXT:    [[AB7:%.*]] = zext i16 [[A7]] to i32
+; SLM-NEXT:    [[R0:%.*]] = insertelement <8 x i32> undef, i32 [[AB0]], i32 0
+; SLM-NEXT:    [[R1:%.*]] = insertelement <8 x i32> [[R0]], i32 [[AB1]], i32 1
+; SLM-NEXT:    [[R2:%.*]] = insertelement <8 x i32> [[R1]], i32 [[AB2]], i32 2
+; SLM-NEXT:    [[R3:%.*]] = insertelement <8 x i32> [[R2]], i32 [[AB3]], i32 3
+; SLM-NEXT:    [[R4:%.*]] = insertelement <8 x i32> [[R3]], i32 [[AB4]], i32 4
+; SLM-NEXT:    [[R5:%.*]] = insertelement <8 x i32> [[R4]], i32 [[AB5]], i32 5
+; SLM-NEXT:    [[R6:%.*]] = insertelement <8 x i32> [[R5]], i32 [[AB6]], i32 6
+; SLM-NEXT:    [[R7:%.*]] = insertelement <8 x i32> [[R6]], i32 [[AB7]], i32 7
+; SLM-NEXT:    ret <8 x i32> [[R7]]
+;
+; AVX-LABEL: @sext_zext(
+; AVX-NEXT:    [[TMP1:%.*]] = sext <8 x i16> [[A:%.*]] to <8 x i32>
+; AVX-NEXT:    [[TMP2:%.*]] = zext <8 x i16> [[A]] to <8 x i32>
+; AVX-NEXT:    [[R7:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
+; AVX-NEXT:    ret <8 x i32> [[R7]]
+;
+; AVX512-LABEL: @sext_zext(
+; AVX512-NEXT:    [[TMP1:%.*]] = sext <8 x i16> [[A:%.*]] to <8 x i32>
+; AVX512-NEXT:    [[TMP2:%.*]] = zext <8 x i16> [[A]] to <8 x i32>
+; AVX512-NEXT:    [[R7:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
+; AVX512-NEXT:    ret <8 x i32> [[R7]]
 ;
   %a0 = extractelement <8 x i16> %a, i32 0
   %a1 = extractelement <8 x i16> %a, i32 1
@@ -383,26 +401,24 @@ define <8 x float> @sitofp_uitofp_4i32_8i16_16i8(<4 x i32> %a, <8 x i16> %b, <16
 ; SSE-NEXT:    ret <8 x float> [[R7]]
 ;
 ; SLM-LABEL: @sitofp_uitofp_4i32_8i16_16i8(
-; SLM-NEXT:    [[A0:%.*]] = extractelement <4 x i32> [[A:%.*]], i32 0
-; SLM-NEXT:    [[A1:%.*]] = extractelement <4 x i32> [[A]], i32 1
-; SLM-NEXT:    [[A2:%.*]] = extractelement <4 x i32> [[A]], i32 2
-; SLM-NEXT:    [[A3:%.*]] = extractelement <4 x i32> [[A]], i32 3
 ; SLM-NEXT:    [[B0:%.*]] = extractelement <8 x i16> [[B:%.*]], i32 0
 ; SLM-NEXT:    [[B1:%.*]] = extractelement <8 x i16> [[B]], i32 1
 ; SLM-NEXT:    [[C0:%.*]] = extractelement <16 x i8> [[C:%.*]], i32 0
 ; SLM-NEXT:    [[C1:%.*]] = extractelement <16 x i8> [[C]], i32 1
-; SLM-NEXT:    [[AB0:%.*]] = sitofp i32 [[A0]] to float
-; SLM-NEXT:    [[AB1:%.*]] = sitofp i32 [[A1]] to float
-; SLM-NEXT:    [[AB2:%.*]] = uitofp i32 [[A2]] to float
-; SLM-NEXT:    [[AB3:%.*]] = uitofp i32 [[A3]] to float
+; SLM-NEXT:    [[TMP1:%.*]] = sitofp <4 x i32> [[A:%.*]] to <4 x float>
+; SLM-NEXT:    [[TMP2:%.*]] = uitofp <4 x i32> [[A]] to <4 x float>
 ; SLM-NEXT:    [[AB4:%.*]] = sitofp i16 [[B0]] to float
 ; SLM-NEXT:    [[AB5:%.*]] = uitofp i16 [[B1]] to float
 ; SLM-NEXT:    [[AB6:%.*]] = sitofp i8 [[C0]] to float
 ; SLM-NEXT:    [[AB7:%.*]] = uitofp i8 [[C1]] to float
-; SLM-NEXT:    [[R0:%.*]] = insertelement <8 x float> undef, float [[AB0]], i32 0
-; SLM-NEXT:    [[R1:%.*]] = insertelement <8 x float> [[R0]], float [[AB1]], i32 1
-; SLM-NEXT:    [[R2:%.*]] = insertelement <8 x float> [[R1]], float [[AB2]], i32 2
-; SLM-NEXT:    [[R3:%.*]] = insertelement <8 x float> [[R2]], float [[AB3]], i32 3
+; SLM-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
+; SLM-NEXT:    [[R0:%.*]] = insertelement <8 x float> undef, float [[TMP3]], i32 0
+; SLM-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i32 1
+; SLM-NEXT:    [[R1:%.*]] = insertelement <8 x float> [[R0]], float [[TMP4]], i32 1
+; SLM-NEXT:    [[TMP5:%.*]] = extractelement <4 x float> [[TMP2]], i32 2
+; SLM-NEXT:    [[R2:%.*]] = insertelement <8 x float> [[R1]], float [[TMP5]], i32 2
+; SLM-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[TMP2]], i32 3
+; SLM-NEXT:    [[R3:%.*]] = insertelement <8 x float> [[R2]], float [[TMP6]], i32 3
 ; SLM-NEXT:    [[R4:%.*]] = insertelement <8 x float> [[R3]], float [[AB4]], i32 4
 ; SLM-NEXT:    [[R5:%.*]] = insertelement <8 x float> [[R4]], float [[AB5]], i32 5
 ; SLM-NEXT:    [[R6:%.*]] = insertelement <8 x float> [[R5]], float [[AB6]], i32 6
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/alternate-int.ll b/llvm/test/Transforms/SLPVectorizer/X86/alternate-int.ll
index 44729b4a8..23d1634fd 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/alternate-int.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/alternate-int.ll
@@ -75,42 +75,11 @@ define <4 x i32> @add_and_v4i32(<4 x i32> %a, <4 x i32> %b) {
 }
 
 define <4 x i32> @add_mul_v4i32(<4 x i32> %a, <4 x i32> %b) {
-; SSE-LABEL: @add_mul_v4i32(
-; SSE-NEXT:    [[TMP1:%.*]] = mul <4 x i32> [[A:%.*]], [[B:%.*]]
-; SSE-NEXT:    [[TMP2:%.*]] = add <4 x i32> [[A]], [[B]]
-; SSE-NEXT:    [[R3:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> <i32 0, i32 5, i32 6, i32 3>
-; SSE-NEXT:    ret <4 x i32> [[R3]]
-;
-; SLM-LABEL: @add_mul_v4i32(
-; SLM-NEXT:    [[A0:%.*]] = extractelement <4 x i32> [[A:%.*]], i32 0
-; SLM-NEXT:    [[A1:%.*]] = extractelement <4 x i32> [[A]], i32 1
-; SLM-NEXT:    [[A2:%.*]] = extractelement <4 x i32> [[A]], i32 2
-; SLM-NEXT:    [[A3:%.*]] = extractelement <4 x i32> [[A]], i32 3
-; SLM-NEXT:    [[B0:%.*]] = extractelement <4 x i32> [[B:%.*]], i32 0
-; SLM-NEXT:    [[B1:%.*]] = extractelement <4 x i32> [[B]], i32 1
-; SLM-NEXT:    [[B2:%.*]] = extractelement <4 x i32> [[B]], i32 2
-; SLM-NEXT:    [[B3:%.*]] = extractelement <4 x i32> [[B]], i32 3
-; SLM-NEXT:    [[AB0:%.*]] = mul i32 [[A0]], [[B0]]
-; SLM-NEXT:    [[AB1:%.*]] = add i32 [[A1]], [[B1]]
-; SLM-NEXT:    [[AB2:%.*]] = add i32 [[A2]], [[B2]]
-; SLM-NEXT:    [[AB3:%.*]] = mul i32 [[A3]], [[B3]]
-; SLM-NEXT:    [[R0:%.*]] = insertelement <4 x i32> undef, i32 [[AB0]], i32 0
-; SLM-NEXT:    [[R1:%.*]] = insertelement <4 x i32> [[R0]], i32 [[AB1]], i32 1
-; SLM-NEXT:    [[R2:%.*]] = insertelement <4 x i32> [[R1]], i32 [[AB2]], i32 2
-; SLM-NEXT:    [[R3:%.*]] = insertelement <4 x i32> [[R2]], i32 [[AB3]], i32 3
-; SLM-NEXT:    ret <4 x i32> [[R3]]
-;
-; AVX-LABEL: @add_mul_v4i32(
-; AVX-NEXT:    [[TMP1:%.*]] = mul <4 x i32> [[A:%.*]], [[B:%.*]]
-; AVX-NEXT:    [[TMP2:%.*]] = add <4 x i32> [[A]], [[B]]
-; AVX-NEXT:    [[R3:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> <i32 0, i32 5, i32 6, i32 3>
-; AVX-NEXT:    ret <4 x i32> [[R3]]
-;
-; AVX512-LABEL: @add_mul_v4i32(
-; AVX512-NEXT:    [[TMP1:%.*]] = mul <4 x i32> [[A:%.*]], [[B:%.*]]
-; AVX512-NEXT:    [[TMP2:%.*]] = add <4 x i32> [[A]], [[B]]
-; AVX512-NEXT:    [[R3:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> <i32 0, i32 5, i32 6, i32 3>
-; AVX512-NEXT:    ret <4 x i32> [[R3]]
+; CHECK-LABEL: @add_mul_v4i32(
+; CHECK-NEXT:    [[TMP1:%.*]] = mul <4 x i32> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = add <4 x i32> [[A]], [[B]]
+; CHECK-NEXT:    [[R3:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> <i32 0, i32 5, i32 6, i32 3>
+; CHECK-NEXT:    ret <4 x i32> [[R3]]
 ;
   %a0 = extractelement <4 x i32> %a, i32 0
   %a1 = extractelement <4 x i32> %a, i32 1
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/arith-add-ssat.ll b/llvm/test/Transforms/SLPVectorizer/X86/arith-add-ssat.ll
index 0bf3ea20d..4ec98a89c 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/arith-add-ssat.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/arith-add-ssat.ll
@@ -354,14 +354,10 @@ define void @add_v32i16() {
 ; AVX-NEXT:    ret void
 ;
 ; AVX512-LABEL: @add_v32i16(
-; AVX512-NEXT:    [[TMP1:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @a16 to <16 x i16>*), align 2
-; AVX512-NEXT:    [[TMP2:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <16 x i16>*), align 2
-; AVX512-NEXT:    [[TMP3:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @b16 to <16 x i16>*), align 2
-; AVX512-NEXT:    [[TMP4:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <16 x i16>*), align 2
-; AVX512-NEXT:    [[TMP5:%.*]] = call <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16> [[TMP1]], <16 x i16> [[TMP3]])
-; AVX512-NEXT:    [[TMP6:%.*]] = call <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16> [[TMP2]], <16 x i16> [[TMP4]])
-; AVX512-NEXT:    store <16 x i16> [[TMP5]], <16 x i16>* bitcast ([32 x i16]* @c16 to <16 x i16>*), align 2
-; AVX512-NEXT:    store <16 x i16> [[TMP6]], <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP1:%.*]] = load <32 x i16>, <32 x i16>* bitcast ([32 x i16]* @a16 to <32 x i16>*), align 2
+; AVX512-NEXT:    [[TMP2:%.*]] = load <32 x i16>, <32 x i16>* bitcast ([32 x i16]* @b16 to <32 x i16>*), align 2
+; AVX512-NEXT:    [[TMP3:%.*]] = call <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16> [[TMP1]], <32 x i16> [[TMP2]])
+; AVX512-NEXT:    store <32 x i16> [[TMP3]], <32 x i16>* bitcast ([32 x i16]* @c16 to <32 x i16>*), align 2
 ; AVX512-NEXT:    ret void
 ;
   %a0  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 0 ), align 2
@@ -496,24 +492,61 @@ define void @add_v32i16() {
 }
 
 define void @add_v64i8() {
-; CHECK-LABEL: @add_v64i8(
-; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @a8 to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP5:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @b8 to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP6:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP7:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP8:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> [[TMP1]], <16 x i8> [[TMP5]])
-; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> [[TMP2]], <16 x i8> [[TMP6]])
-; CHECK-NEXT:    [[TMP11:%.*]] = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> [[TMP3]], <16 x i8> [[TMP7]])
-; CHECK-NEXT:    [[TMP12:%.*]] = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> [[TMP4]], <16 x i8> [[TMP8]])
-; CHECK-NEXT:    store <16 x i8> [[TMP9]], <16 x i8>* bitcast ([64 x i8]* @c8 to <16 x i8>*), align 1
-; CHECK-NEXT:    store <16 x i8> [[TMP10]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16) to <16 x i8>*), align 1
-; CHECK-NEXT:    store <16 x i8> [[TMP11]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32) to <16 x i8>*), align 1
-; CHECK-NEXT:    store <16 x i8> [[TMP12]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48) to <16 x i8>*), align 1
-; CHECK-NEXT:    ret void
+; SSE-LABEL: @add_v64i8(
+; SSE-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @a8 to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP3:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP4:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP5:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @b8 to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP6:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP7:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP8:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP9:%.*]] = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> [[TMP1]], <16 x i8> [[TMP5]])
+; SSE-NEXT:    [[TMP10:%.*]] = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> [[TMP2]], <16 x i8> [[TMP6]])
+; SSE-NEXT:    [[TMP11:%.*]] = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> [[TMP3]], <16 x i8> [[TMP7]])
+; SSE-NEXT:    [[TMP12:%.*]] = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> [[TMP4]], <16 x i8> [[TMP8]])
+; SSE-NEXT:    store <16 x i8> [[TMP9]], <16 x i8>* bitcast ([64 x i8]* @c8 to <16 x i8>*), align 1
+; SSE-NEXT:    store <16 x i8> [[TMP10]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16) to <16 x i8>*), align 1
+; SSE-NEXT:    store <16 x i8> [[TMP11]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32) to <16 x i8>*), align 1
+; SSE-NEXT:    store <16 x i8> [[TMP12]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48) to <16 x i8>*), align 1
+; SSE-NEXT:    ret void
+;
+; SLM-LABEL: @add_v64i8(
+; SLM-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @a8 to <16 x i8>*), align 1
+; SLM-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16) to <16 x i8>*), align 1
+; SLM-NEXT:    [[TMP3:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32) to <16 x i8>*), align 1
+; SLM-NEXT:    [[TMP4:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48) to <16 x i8>*), align 1
+; SLM-NEXT:    [[TMP5:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @b8 to <16 x i8>*), align 1
+; SLM-NEXT:    [[TMP6:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16) to <16 x i8>*), align 1
+; SLM-NEXT:    [[TMP7:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32) to <16 x i8>*), align 1
+; SLM-NEXT:    [[TMP8:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48) to <16 x i8>*), align 1
+; SLM-NEXT:    [[TMP9:%.*]] = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> [[TMP1]], <16 x i8> [[TMP5]])
+; SLM-NEXT:    [[TMP10:%.*]] = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> [[TMP2]], <16 x i8> [[TMP6]])
+; SLM-NEXT:    [[TMP11:%.*]] = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> [[TMP3]], <16 x i8> [[TMP7]])
+; SLM-NEXT:    [[TMP12:%.*]] = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> [[TMP4]], <16 x i8> [[TMP8]])
+; SLM-NEXT:    store <16 x i8> [[TMP9]], <16 x i8>* bitcast ([64 x i8]* @c8 to <16 x i8>*), align 1
+; SLM-NEXT:    store <16 x i8> [[TMP10]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16) to <16 x i8>*), align 1
+; SLM-NEXT:    store <16 x i8> [[TMP11]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32) to <16 x i8>*), align 1
+; SLM-NEXT:    store <16 x i8> [[TMP12]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48) to <16 x i8>*), align 1
+; SLM-NEXT:    ret void
+;
+; AVX-LABEL: @add_v64i8(
+; AVX-NEXT:    [[TMP1:%.*]] = load <32 x i8>, <32 x i8>* bitcast ([64 x i8]* @a8 to <32 x i8>*), align 1
+; AVX-NEXT:    [[TMP2:%.*]] = load <32 x i8>, <32 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32) to <32 x i8>*), align 1
+; AVX-NEXT:    [[TMP3:%.*]] = load <32 x i8>, <32 x i8>* bitcast ([64 x i8]* @b8 to <32 x i8>*), align 1
+; AVX-NEXT:    [[TMP4:%.*]] = load <32 x i8>, <32 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32) to <32 x i8>*), align 1
+; AVX-NEXT:    [[TMP5:%.*]] = call <32 x i8> @llvm.sadd.sat.v32i8(<32 x i8> [[TMP1]], <32 x i8> [[TMP3]])
+; AVX-NEXT:    [[TMP6:%.*]] = call <32 x i8> @llvm.sadd.sat.v32i8(<32 x i8> [[TMP2]], <32 x i8> [[TMP4]])
+; AVX-NEXT:    store <32 x i8> [[TMP5]], <32 x i8>* bitcast ([64 x i8]* @c8 to <32 x i8>*), align 1
+; AVX-NEXT:    store <32 x i8> [[TMP6]], <32 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32) to <32 x i8>*), align 1
+; AVX-NEXT:    ret void
+;
+; AVX512-LABEL: @add_v64i8(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <64 x i8>, <64 x i8>* bitcast ([64 x i8]* @a8 to <64 x i8>*), align 1
+; AVX512-NEXT:    [[TMP2:%.*]] = load <64 x i8>, <64 x i8>* bitcast ([64 x i8]* @b8 to <64 x i8>*), align 1
+; AVX512-NEXT:    [[TMP3:%.*]] = call <64 x i8> @llvm.sadd.sat.v64i8(<64 x i8> [[TMP1]], <64 x i8> [[TMP2]])
+; AVX512-NEXT:    store <64 x i8> [[TMP3]], <64 x i8>* bitcast ([64 x i8]* @c8 to <64 x i8>*), align 1
+; AVX512-NEXT:    ret void
 ;
   %a0  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 0 ), align 1
   %a1  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 1 ), align 1
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/arith-add-usat.ll b/llvm/test/Transforms/SLPVectorizer/X86/arith-add-usat.ll
index 4bd84d846..65104d2a5 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/arith-add-usat.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/arith-add-usat.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE,SSE2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE,SLM
 ; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
 ; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
 ; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F
@@ -61,25 +61,6 @@ define void @add_v8i64() {
 ; SSE-NEXT:    store i64 [[R7]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 7), align 8
 ; SSE-NEXT:    ret void
 ;
-; SLM-LABEL: @add_v8i64(
-; SLM-NEXT:    [[TMP1:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([8 x i64]* @a64 to <2 x i64>*), align 8
-; SLM-NEXT:    [[TMP2:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2) to <2 x i64>*), align 8
-; SLM-NEXT:    [[TMP3:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4) to <2 x i64>*), align 8
-; SLM-NEXT:    [[TMP4:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6) to <2 x i64>*), align 8
-; SLM-NEXT:    [[TMP5:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([8 x i64]* @b64 to <2 x i64>*), align 8
-; SLM-NEXT:    [[TMP6:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2) to <2 x i64>*), align 8
-; SLM-NEXT:    [[TMP7:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4) to <2 x i64>*), align 8
-; SLM-NEXT:    [[TMP8:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6) to <2 x i64>*), align 8
-; SLM-NEXT:    [[TMP9:%.*]] = call <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64> [[TMP1]], <2 x i64> [[TMP5]])
-; SLM-NEXT:    [[TMP10:%.*]] = call <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64> [[TMP2]], <2 x i64> [[TMP6]])
-; SLM-NEXT:    [[TMP11:%.*]] = call <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64> [[TMP3]], <2 x i64> [[TMP7]])
-; SLM-NEXT:    [[TMP12:%.*]] = call <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64> [[TMP4]], <2 x i64> [[TMP8]])
-; SLM-NEXT:    store <2 x i64> [[TMP9]], <2 x i64>* bitcast ([8 x i64]* @c64 to <2 x i64>*), align 8
-; SLM-NEXT:    store <2 x i64> [[TMP10]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2) to <2 x i64>*), align 8
-; SLM-NEXT:    store <2 x i64> [[TMP11]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4) to <2 x i64>*), align 8
-; SLM-NEXT:    store <2 x i64> [[TMP12]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6) to <2 x i64>*), align 8
-; SLM-NEXT:    ret void
-;
 ; AVX-LABEL: @add_v8i64(
 ; AVX-NEXT:    [[TMP1:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([8 x i64]* @a64 to <4 x i64>*), align 8
 ; AVX-NEXT:    [[TMP2:%.*]] = load <4 x i64>, <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4) to <4 x i64>*), align 8
@@ -153,25 +134,6 @@ define void @add_v16i32() {
 ; SSE-NEXT:    store <4 x i32> [[TMP12]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12) to <4 x i32>*), align 4
 ; SSE-NEXT:    ret void
 ;
-; SLM-LABEL: @add_v16i32(
-; SLM-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @a32 to <4 x i32>*), align 4
-; SLM-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4) to <4 x i32>*), align 4
-; SLM-NEXT:    [[TMP3:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8) to <4 x i32>*), align 4
-; SLM-NEXT:    [[TMP4:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12) to <4 x i32>*), align 4
-; SLM-NEXT:    [[TMP5:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @b32 to <4 x i32>*), align 4
-; SLM-NEXT:    [[TMP6:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4) to <4 x i32>*), align 4
-; SLM-NEXT:    [[TMP7:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8) to <4 x i32>*), align 4
-; SLM-NEXT:    [[TMP8:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12) to <4 x i32>*), align 4
-; SLM-NEXT:    [[TMP9:%.*]] = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> [[TMP1]], <4 x i32> [[TMP5]])
-; SLM-NEXT:    [[TMP10:%.*]] = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> [[TMP2]], <4 x i32> [[TMP6]])
-; SLM-NEXT:    [[TMP11:%.*]] = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> [[TMP3]], <4 x i32> [[TMP7]])
-; SLM-NEXT:    [[TMP12:%.*]] = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> [[TMP4]], <4 x i32> [[TMP8]])
-; SLM-NEXT:    store <4 x i32> [[TMP9]], <4 x i32>* bitcast ([16 x i32]* @c32 to <4 x i32>*), align 4
-; SLM-NEXT:    store <4 x i32> [[TMP10]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4) to <4 x i32>*), align 4
-; SLM-NEXT:    store <4 x i32> [[TMP11]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8) to <4 x i32>*), align 4
-; SLM-NEXT:    store <4 x i32> [[TMP12]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12) to <4 x i32>*), align 4
-; SLM-NEXT:    ret void
-;
 ; AVX-LABEL: @add_v16i32(
 ; AVX-NEXT:    [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([16 x i32]* @a32 to <8 x i32>*), align 4
 ; AVX-NEXT:    [[TMP2:%.*]] = load <8 x i32>, <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8) to <8 x i32>*), align 4
@@ -277,25 +239,6 @@ define void @add_v32i16() {
 ; SSE-NEXT:    store <8 x i16> [[TMP12]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 24) to <8 x i16>*), align 2
 ; SSE-NEXT:    ret void
 ;
-; SLM-LABEL: @add_v32i16(
-; SLM-NEXT:    [[TMP1:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @a16 to <8 x i16>*), align 2
-; SLM-NEXT:    [[TMP2:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 8) to <8 x i16>*), align 2
-; SLM-NEXT:    [[TMP3:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <8 x i16>*), align 2
-; SLM-NEXT:    [[TMP4:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 24) to <8 x i16>*), align 2
-; SLM-NEXT:    [[TMP5:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @b16 to <8 x i16>*), align 2
-; SLM-NEXT:    [[TMP6:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 8) to <8 x i16>*), align 2
-; SLM-NEXT:    [[TMP7:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <8 x i16>*), align 2
-; SLM-NEXT:    [[TMP8:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 24) to <8 x i16>*), align 2
-; SLM-NEXT:    [[TMP9:%.*]] = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> [[TMP1]], <8 x i16> [[TMP5]])
-; SLM-NEXT:    [[TMP10:%.*]] = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> [[TMP2]], <8 x i16> [[TMP6]])
-; SLM-NEXT:    [[TMP11:%.*]] = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> [[TMP3]], <8 x i16> [[TMP7]])
-; SLM-NEXT:    [[TMP12:%.*]] = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> [[TMP4]], <8 x i16> [[TMP8]])
-; SLM-NEXT:    store <8 x i16> [[TMP9]], <8 x i16>* bitcast ([32 x i16]* @c16 to <8 x i16>*), align 2
-; SLM-NEXT:    store <8 x i16> [[TMP10]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 8) to <8 x i16>*), align 2
-; SLM-NEXT:    store <8 x i16> [[TMP11]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <8 x i16>*), align 2
-; SLM-NEXT:    store <8 x i16> [[TMP12]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 24) to <8 x i16>*), align 2
-; SLM-NEXT:    ret void
-;
 ; AVX-LABEL: @add_v32i16(
 ; AVX-NEXT:    [[TMP1:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @a16 to <16 x i16>*), align 2
 ; AVX-NEXT:    [[TMP2:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <16 x i16>*), align 2
@@ -308,14 +251,10 @@ define void @add_v32i16() {
 ; AVX-NEXT:    ret void
 ;
 ; AVX512-LABEL: @add_v32i16(
-; AVX512-NEXT:    [[TMP1:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @a16 to <16 x i16>*), align 2
-; AVX512-NEXT:    [[TMP2:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <16 x i16>*), align 2
-; AVX512-NEXT:    [[TMP3:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @b16 to <16 x i16>*), align 2
-; AVX512-NEXT:    [[TMP4:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <16 x i16>*), align 2
-; AVX512-NEXT:    [[TMP5:%.*]] = call <16 x i16> @llvm.uadd.sat.v16i16(<16 x i16> [[TMP1]], <16 x i16> [[TMP3]])
-; AVX512-NEXT:    [[TMP6:%.*]] = call <16 x i16> @llvm.uadd.sat.v16i16(<16 x i16> [[TMP2]], <16 x i16> [[TMP4]])
-; AVX512-NEXT:    store <16 x i16> [[TMP5]], <16 x i16>* bitcast ([32 x i16]* @c16 to <16 x i16>*), align 2
-; AVX512-NEXT:    store <16 x i16> [[TMP6]], <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP1:%.*]] = load <32 x i16>, <32 x i16>* bitcast ([32 x i16]* @a16 to <32 x i16>*), align 2
+; AVX512-NEXT:    [[TMP2:%.*]] = load <32 x i16>, <32 x i16>* bitcast ([32 x i16]* @b16 to <32 x i16>*), align 2
+; AVX512-NEXT:    [[TMP3:%.*]] = call <32 x i16> @llvm.uadd.sat.v32i16(<32 x i16> [[TMP1]], <32 x i16> [[TMP2]])
+; AVX512-NEXT:    store <32 x i16> [[TMP3]], <32 x i16>* bitcast ([32 x i16]* @c16 to <32 x i16>*), align 2
 ; AVX512-NEXT:    ret void
 ;
   %a0  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 0 ), align 2
@@ -450,24 +389,42 @@ define void @add_v32i16() {
 }
 
 define void @add_v64i8() {
-; CHECK-LABEL: @add_v64i8(
-; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @a8 to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP5:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @b8 to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP6:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP7:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP8:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> [[TMP1]], <16 x i8> [[TMP5]])
-; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> [[TMP2]], <16 x i8> [[TMP6]])
-; CHECK-NEXT:    [[TMP11:%.*]] = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> [[TMP3]], <16 x i8> [[TMP7]])
-; CHECK-NEXT:    [[TMP12:%.*]] = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> [[TMP4]], <16 x i8> [[TMP8]])
-; CHECK-NEXT:    store <16 x i8> [[TMP9]], <16 x i8>* bitcast ([64 x i8]* @c8 to <16 x i8>*), align 1
-; CHECK-NEXT:    store <16 x i8> [[TMP10]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16) to <16 x i8>*), align 1
-; CHECK-NEXT:    store <16 x i8> [[TMP11]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32) to <16 x i8>*), align 1
-; CHECK-NEXT:    store <16 x i8> [[TMP12]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48) to <16 x i8>*), align 1
-; CHECK-NEXT:    ret void
+; SSE-LABEL: @add_v64i8(
+; SSE-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @a8 to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP3:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP4:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP5:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @b8 to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP6:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP7:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP8:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP9:%.*]] = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> [[TMP1]], <16 x i8> [[TMP5]])
+; SSE-NEXT:    [[TMP10:%.*]] = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> [[TMP2]], <16 x i8> [[TMP6]])
+; SSE-NEXT:    [[TMP11:%.*]] = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> [[TMP3]], <16 x i8> [[TMP7]])
+; SSE-NEXT:    [[TMP12:%.*]] = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> [[TMP4]], <16 x i8> [[TMP8]])
+; SSE-NEXT:    store <16 x i8> [[TMP9]], <16 x i8>* bitcast ([64 x i8]* @c8 to <16 x i8>*), align 1
+; SSE-NEXT:    store <16 x i8> [[TMP10]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16) to <16 x i8>*), align 1
+; SSE-NEXT:    store <16 x i8> [[TMP11]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32) to <16 x i8>*), align 1
+; SSE-NEXT:    store <16 x i8> [[TMP12]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48) to <16 x i8>*), align 1
+; SSE-NEXT:    ret void
+;
+; AVX-LABEL: @add_v64i8(
+; AVX-NEXT:    [[TMP1:%.*]] = load <32 x i8>, <32 x i8>* bitcast ([64 x i8]* @a8 to <32 x i8>*), align 1
+; AVX-NEXT:    [[TMP2:%.*]] = load <32 x i8>, <32 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32) to <32 x i8>*), align 1
+; AVX-NEXT:    [[TMP3:%.*]] = load <32 x i8>, <32 x i8>* bitcast ([64 x i8]* @b8 to <32 x i8>*), align 1
+; AVX-NEXT:    [[TMP4:%.*]] = load <32 x i8>, <32 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32) to <32 x i8>*), align 1
+; AVX-NEXT:    [[TMP5:%.*]] = call <32 x i8> @llvm.uadd.sat.v32i8(<32 x i8> [[TMP1]], <32 x i8> [[TMP3]])
+; AVX-NEXT:    [[TMP6:%.*]] = call <32 x i8> @llvm.uadd.sat.v32i8(<32 x i8> [[TMP2]], <32 x i8> [[TMP4]])
+; AVX-NEXT:    store <32 x i8> [[TMP5]], <32 x i8>* bitcast ([64 x i8]* @c8 to <32 x i8>*), align 1
+; AVX-NEXT:    store <32 x i8> [[TMP6]], <32 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32) to <32 x i8>*), align 1
+; AVX-NEXT:    ret void
+;
+; AVX512-LABEL: @add_v64i8(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <64 x i8>, <64 x i8>* bitcast ([64 x i8]* @a8 to <64 x i8>*), align 1
+; AVX512-NEXT:    [[TMP2:%.*]] = load <64 x i8>, <64 x i8>* bitcast ([64 x i8]* @b8 to <64 x i8>*), align 1
+; AVX512-NEXT:    [[TMP3:%.*]] = call <64 x i8> @llvm.uadd.sat.v64i8(<64 x i8> [[TMP1]], <64 x i8> [[TMP2]])
+; AVX512-NEXT:    store <64 x i8> [[TMP3]], <64 x i8>* bitcast ([64 x i8]* @c8 to <64 x i8>*), align 1
+; AVX512-NEXT:    ret void
 ;
   %a0  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 0 ), align 1
   %a1  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 1 ), align 1
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/arith-add.ll b/llvm/test/Transforms/SLPVectorizer/X86/arith-add.ll
index 5c6430c68..087b27b52 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/arith-add.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/arith-add.ll
@@ -1,8 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
 ; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SLM
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX1
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -mattr=-prefer-128-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -mattr=+prefer-128-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -mattr=-prefer-128-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -mattr=+prefer-128-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
 ; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512F
 ; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=-prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512BW
 ; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=+prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
@@ -287,14 +289,10 @@ define void @add_v32i16() {
 ; AVX-NEXT:    ret void
 ;
 ; AVX512-LABEL: @add_v32i16(
-; AVX512-NEXT:    [[TMP1:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @a16 to <16 x i16>*), align 2
-; AVX512-NEXT:    [[TMP2:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <16 x i16>*), align 2
-; AVX512-NEXT:    [[TMP3:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @b16 to <16 x i16>*), align 2
-; AVX512-NEXT:    [[TMP4:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <16 x i16>*), align 2
-; AVX512-NEXT:    [[TMP5:%.*]] = add <16 x i16> [[TMP1]], [[TMP3]]
-; AVX512-NEXT:    [[TMP6:%.*]] = add <16 x i16> [[TMP2]], [[TMP4]]
-; AVX512-NEXT:    store <16 x i16> [[TMP5]], <16 x i16>* bitcast ([32 x i16]* @c16 to <16 x i16>*), align 2
-; AVX512-NEXT:    store <16 x i16> [[TMP6]], <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP1:%.*]] = load <32 x i16>, <32 x i16>* bitcast ([32 x i16]* @a16 to <32 x i16>*), align 2
+; AVX512-NEXT:    [[TMP2:%.*]] = load <32 x i16>, <32 x i16>* bitcast ([32 x i16]* @b16 to <32 x i16>*), align 2
+; AVX512-NEXT:    [[TMP3:%.*]] = add <32 x i16> [[TMP1]], [[TMP2]]
+; AVX512-NEXT:    store <32 x i16> [[TMP3]], <32 x i16>* bitcast ([32 x i16]* @c16 to <32 x i16>*), align 2
 ; AVX512-NEXT:    ret void
 ;
   %a0  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 0 ), align 2
@@ -429,24 +427,61 @@ define void @add_v32i16() {
 }
 
 define void @add_v64i8() {
-; CHECK-LABEL: @add_v64i8(
-; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @a8 to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP5:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @b8 to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP6:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP7:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP8:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP9:%.*]] = add <16 x i8> [[TMP1]], [[TMP5]]
-; CHECK-NEXT:    [[TMP10:%.*]] = add <16 x i8> [[TMP2]], [[TMP6]]
-; CHECK-NEXT:    [[TMP11:%.*]] = add <16 x i8> [[TMP3]], [[TMP7]]
-; CHECK-NEXT:    [[TMP12:%.*]] = add <16 x i8> [[TMP4]], [[TMP8]]
-; CHECK-NEXT:    store <16 x i8> [[TMP9]], <16 x i8>* bitcast ([64 x i8]* @c8 to <16 x i8>*), align 1
-; CHECK-NEXT:    store <16 x i8> [[TMP10]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16) to <16 x i8>*), align 1
-; CHECK-NEXT:    store <16 x i8> [[TMP11]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32) to <16 x i8>*), align 1
-; CHECK-NEXT:    store <16 x i8> [[TMP12]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48) to <16 x i8>*), align 1
-; CHECK-NEXT:    ret void
+; SSE-LABEL: @add_v64i8(
+; SSE-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @a8 to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP3:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP4:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP5:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @b8 to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP6:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP7:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP8:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP9:%.*]] = add <16 x i8> [[TMP1]], [[TMP5]]
+; SSE-NEXT:    [[TMP10:%.*]] = add <16 x i8> [[TMP2]], [[TMP6]]
+; SSE-NEXT:    [[TMP11:%.*]] = add <16 x i8> [[TMP3]], [[TMP7]]
+; SSE-NEXT:    [[TMP12:%.*]] = add <16 x i8> [[TMP4]], [[TMP8]]
+; SSE-NEXT:    store <16 x i8> [[TMP9]], <16 x i8>* bitcast ([64 x i8]* @c8 to <16 x i8>*), align 1
+; SSE-NEXT:    store <16 x i8> [[TMP10]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16) to <16 x i8>*), align 1
+; SSE-NEXT:    store <16 x i8> [[TMP11]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32) to <16 x i8>*), align 1
+; SSE-NEXT:    store <16 x i8> [[TMP12]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48) to <16 x i8>*), align 1
+; SSE-NEXT:    ret void
+;
+; SLM-LABEL: @add_v64i8(
+; SLM-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @a8 to <16 x i8>*), align 1
+; SLM-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16) to <16 x i8>*), align 1
+; SLM-NEXT:    [[TMP3:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32) to <16 x i8>*), align 1
+; SLM-NEXT:    [[TMP4:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48) to <16 x i8>*), align 1
+; SLM-NEXT:    [[TMP5:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @b8 to <16 x i8>*), align 1
+; SLM-NEXT:    [[TMP6:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16) to <16 x i8>*), align 1
+; SLM-NEXT:    [[TMP7:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32) to <16 x i8>*), align 1
+; SLM-NEXT:    [[TMP8:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48) to <16 x i8>*), align 1
+; SLM-NEXT:    [[TMP9:%.*]] = add <16 x i8> [[TMP1]], [[TMP5]]
+; SLM-NEXT:    [[TMP10:%.*]] = add <16 x i8> [[TMP2]], [[TMP6]]
+; SLM-NEXT:    [[TMP11:%.*]] = add <16 x i8> [[TMP3]], [[TMP7]]
+; SLM-NEXT:    [[TMP12:%.*]] = add <16 x i8> [[TMP4]], [[TMP8]]
+; SLM-NEXT:    store <16 x i8> [[TMP9]], <16 x i8>* bitcast ([64 x i8]* @c8 to <16 x i8>*), align 1
+; SLM-NEXT:    store <16 x i8> [[TMP10]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16) to <16 x i8>*), align 1
+; SLM-NEXT:    store <16 x i8> [[TMP11]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32) to <16 x i8>*), align 1
+; SLM-NEXT:    store <16 x i8> [[TMP12]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48) to <16 x i8>*), align 1
+; SLM-NEXT:    ret void
+;
+; AVX-LABEL: @add_v64i8(
+; AVX-NEXT:    [[TMP1:%.*]] = load <32 x i8>, <32 x i8>* bitcast ([64 x i8]* @a8 to <32 x i8>*), align 1
+; AVX-NEXT:    [[TMP2:%.*]] = load <32 x i8>, <32 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32) to <32 x i8>*), align 1
+; AVX-NEXT:    [[TMP3:%.*]] = load <32 x i8>, <32 x i8>* bitcast ([64 x i8]* @b8 to <32 x i8>*), align 1
+; AVX-NEXT:    [[TMP4:%.*]] = load <32 x i8>, <32 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32) to <32 x i8>*), align 1
+; AVX-NEXT:    [[TMP5:%.*]] = add <32 x i8> [[TMP1]], [[TMP3]]
+; AVX-NEXT:    [[TMP6:%.*]] = add <32 x i8> [[TMP2]], [[TMP4]]
+; AVX-NEXT:    store <32 x i8> [[TMP5]], <32 x i8>* bitcast ([64 x i8]* @c8 to <32 x i8>*), align 1
+; AVX-NEXT:    store <32 x i8> [[TMP6]], <32 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32) to <32 x i8>*), align 1
+; AVX-NEXT:    ret void
+;
+; AVX512-LABEL: @add_v64i8(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <64 x i8>, <64 x i8>* bitcast ([64 x i8]* @a8 to <64 x i8>*), align 1
+; AVX512-NEXT:    [[TMP2:%.*]] = load <64 x i8>, <64 x i8>* bitcast ([64 x i8]* @b8 to <64 x i8>*), align 1
+; AVX512-NEXT:    [[TMP3:%.*]] = add <64 x i8> [[TMP1]], [[TMP2]]
+; AVX512-NEXT:    store <64 x i8> [[TMP3]], <64 x i8>* bitcast ([64 x i8]* @c8 to <64 x i8>*), align 1
+; AVX512-NEXT:    ret void
 ;
   %a0  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 0 ), align 1
   %a1  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 1 ), align 1
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/arith-fix.ll b/llvm/test/Transforms/SLPVectorizer/X86/arith-fix.ll
index a8897aac6..af0b42f7a 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/arith-fix.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/arith-fix.ll
@@ -149,22 +149,70 @@ define void @smul_v8i64() {
 
 define void @smul_v16i32() {
 ; SSE-LABEL: @smul_v16i32(
-; SSE-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @a32 to <4 x i32>*), align 4
-; SSE-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4) to <4 x i32>*), align 4
-; SSE-NEXT:    [[TMP3:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8) to <4 x i32>*), align 4
-; SSE-NEXT:    [[TMP4:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12) to <4 x i32>*), align 4
-; SSE-NEXT:    [[TMP5:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @b32 to <4 x i32>*), align 4
-; SSE-NEXT:    [[TMP6:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4) to <4 x i32>*), align 4
-; SSE-NEXT:    [[TMP7:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8) to <4 x i32>*), align 4
-; SSE-NEXT:    [[TMP8:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12) to <4 x i32>*), align 4
-; SSE-NEXT:    [[TMP9:%.*]] = call <4 x i32> @llvm.smul.fix.v4i32(<4 x i32> [[TMP1]], <4 x i32> [[TMP5]], i32 3)
-; SSE-NEXT:    [[TMP10:%.*]] = call <4 x i32> @llvm.smul.fix.v4i32(<4 x i32> [[TMP2]], <4 x i32> [[TMP6]], i32 3)
-; SSE-NEXT:    [[TMP11:%.*]] = call <4 x i32> @llvm.smul.fix.v4i32(<4 x i32> [[TMP3]], <4 x i32> [[TMP7]], i32 3)
-; SSE-NEXT:    [[TMP12:%.*]] = call <4 x i32> @llvm.smul.fix.v4i32(<4 x i32> [[TMP4]], <4 x i32> [[TMP8]], i32 3)
-; SSE-NEXT:    store <4 x i32> [[TMP9]], <4 x i32>* bitcast ([16 x i32]* @c32 to <4 x i32>*), align 4
-; SSE-NEXT:    store <4 x i32> [[TMP10]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4) to <4 x i32>*), align 4
-; SSE-NEXT:    store <4 x i32> [[TMP11]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8) to <4 x i32>*), align 4
-; SSE-NEXT:    store <4 x i32> [[TMP12]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12) to <4 x i32>*), align 4
+; SSE-NEXT:    [[A0:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 0), align 4
+; SSE-NEXT:    [[A1:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 1), align 4
+; SSE-NEXT:    [[A2:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 2), align 4
+; SSE-NEXT:    [[A3:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 3), align 4
+; SSE-NEXT:    [[A4:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4), align 4
+; SSE-NEXT:    [[A5:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 5), align 4
+; SSE-NEXT:    [[A6:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 6), align 4
+; SSE-NEXT:    [[A7:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 7), align 4
+; SSE-NEXT:    [[A8:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8), align 4
+; SSE-NEXT:    [[A9:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 9), align 4
+; SSE-NEXT:    [[A10:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 10), align 4
+; SSE-NEXT:    [[A11:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 11), align 4
+; SSE-NEXT:    [[A12:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12), align 4
+; SSE-NEXT:    [[A13:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 13), align 4
+; SSE-NEXT:    [[A14:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 14), align 4
+; SSE-NEXT:    [[A15:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 15), align 4
+; SSE-NEXT:    [[B0:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 0), align 4
+; SSE-NEXT:    [[B1:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 1), align 4
+; SSE-NEXT:    [[B2:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 2), align 4
+; SSE-NEXT:    [[B3:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 3), align 4
+; SSE-NEXT:    [[B4:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4), align 4
+; SSE-NEXT:    [[B5:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 5), align 4
+; SSE-NEXT:    [[B6:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 6), align 4
+; SSE-NEXT:    [[B7:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 7), align 4
+; SSE-NEXT:    [[B8:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8), align 4
+; SSE-NEXT:    [[B9:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 9), align 4
+; SSE-NEXT:    [[B10:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 10), align 4
+; SSE-NEXT:    [[B11:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 11), align 4
+; SSE-NEXT:    [[B12:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12), align 4
+; SSE-NEXT:    [[B13:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 13), align 4
+; SSE-NEXT:    [[B14:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 14), align 4
+; SSE-NEXT:    [[B15:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 15), align 4
+; SSE-NEXT:    [[R0:%.*]] = call i32 @llvm.smul.fix.i32(i32 [[A0]], i32 [[B0]], i32 3)
+; SSE-NEXT:    [[R1:%.*]] = call i32 @llvm.smul.fix.i32(i32 [[A1]], i32 [[B1]], i32 3)
+; SSE-NEXT:    [[R2:%.*]] = call i32 @llvm.smul.fix.i32(i32 [[A2]], i32 [[B2]], i32 3)
+; SSE-NEXT:    [[R3:%.*]] = call i32 @llvm.smul.fix.i32(i32 [[A3]], i32 [[B3]], i32 3)
+; SSE-NEXT:    [[R4:%.*]] = call i32 @llvm.smul.fix.i32(i32 [[A4]], i32 [[B4]], i32 3)
+; SSE-NEXT:    [[R5:%.*]] = call i32 @llvm.smul.fix.i32(i32 [[A5]], i32 [[B5]], i32 3)
+; SSE-NEXT:    [[R6:%.*]] = call i32 @llvm.smul.fix.i32(i32 [[A6]], i32 [[B6]], i32 3)
+; SSE-NEXT:    [[R7:%.*]] = call i32 @llvm.smul.fix.i32(i32 [[A7]], i32 [[B7]], i32 3)
+; SSE-NEXT:    [[R8:%.*]] = call i32 @llvm.smul.fix.i32(i32 [[A8]], i32 [[B8]], i32 3)
+; SSE-NEXT:    [[R9:%.*]] = call i32 @llvm.smul.fix.i32(i32 [[A9]], i32 [[B9]], i32 3)
+; SSE-NEXT:    [[R10:%.*]] = call i32 @llvm.smul.fix.i32(i32 [[A10]], i32 [[B10]], i32 3)
+; SSE-NEXT:    [[R11:%.*]] = call i32 @llvm.smul.fix.i32(i32 [[A11]], i32 [[B11]], i32 3)
+; SSE-NEXT:    [[R12:%.*]] = call i32 @llvm.smul.fix.i32(i32 [[A12]], i32 [[B12]], i32 3)
+; SSE-NEXT:    [[R13:%.*]] = call i32 @llvm.smul.fix.i32(i32 [[A13]], i32 [[B13]], i32 3)
+; SSE-NEXT:    [[R14:%.*]] = call i32 @llvm.smul.fix.i32(i32 [[A14]], i32 [[B14]], i32 3)
+; SSE-NEXT:    [[R15:%.*]] = call i32 @llvm.smul.fix.i32(i32 [[A15]], i32 [[B15]], i32 3)
+; SSE-NEXT:    store i32 [[R0]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 0), align 4
+; SSE-NEXT:    store i32 [[R1]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 1), align 4
+; SSE-NEXT:    store i32 [[R2]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 2), align 4
+; SSE-NEXT:    store i32 [[R3]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 3), align 4
+; SSE-NEXT:    store i32 [[R4]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4), align 4
+; SSE-NEXT:    store i32 [[R5]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 5), align 4
+; SSE-NEXT:    store i32 [[R6]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 6), align 4
+; SSE-NEXT:    store i32 [[R7]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 7), align 4
+; SSE-NEXT:    store i32 [[R8]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8), align 4
+; SSE-NEXT:    store i32 [[R9]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 9), align 4
+; SSE-NEXT:    store i32 [[R10]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 10), align 4
+; SSE-NEXT:    store i32 [[R11]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 11), align 4
+; SSE-NEXT:    store i32 [[R12]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12), align 4
+; SSE-NEXT:    store i32 [[R13]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 13), align 4
+; SSE-NEXT:    store i32 [[R14]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 14), align 4
+; SSE-NEXT:    store i32 [[R15]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 15), align 4
 ; SSE-NEXT:    ret void
 ;
 ; SLM-LABEL: @smul_v16i32(
@@ -448,14 +496,10 @@ define void @smul_v32i16() {
 ; AVX-NEXT:    ret void
 ;
 ; AVX512-LABEL: @smul_v32i16(
-; AVX512-NEXT:    [[TMP1:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @a16 to <16 x i16>*), align 2
-; AVX512-NEXT:    [[TMP2:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <16 x i16>*), align 2
-; AVX512-NEXT:    [[TMP3:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @b16 to <16 x i16>*), align 2
-; AVX512-NEXT:    [[TMP4:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <16 x i16>*), align 2
-; AVX512-NEXT:    [[TMP5:%.*]] = call <16 x i16> @llvm.smul.fix.v16i16(<16 x i16> [[TMP1]], <16 x i16> [[TMP3]], i32 3)
-; AVX512-NEXT:    [[TMP6:%.*]] = call <16 x i16> @llvm.smul.fix.v16i16(<16 x i16> [[TMP2]], <16 x i16> [[TMP4]], i32 3)
-; AVX512-NEXT:    store <16 x i16> [[TMP5]], <16 x i16>* bitcast ([32 x i16]* @c16 to <16 x i16>*), align 2
-; AVX512-NEXT:    store <16 x i16> [[TMP6]], <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP1:%.*]] = load <32 x i16>, <32 x i16>* bitcast ([32 x i16]* @a16 to <32 x i16>*), align 2
+; AVX512-NEXT:    [[TMP2:%.*]] = load <32 x i16>, <32 x i16>* bitcast ([32 x i16]* @b16 to <32 x i16>*), align 2
+; AVX512-NEXT:    [[TMP3:%.*]] = call <32 x i16> @llvm.smul.fix.v32i16(<32 x i16> [[TMP1]], <32 x i16> [[TMP2]], i32 3)
+; AVX512-NEXT:    store <32 x i16> [[TMP3]], <32 x i16>* bitcast ([32 x i16]* @c16 to <32 x i16>*), align 2
 ; AVX512-NEXT:    ret void
 ;
   %a0  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 0 ), align 2
@@ -590,24 +634,61 @@ define void @smul_v32i16() {
 }
 
 define void @smul_v64i8() {
-; CHECK-LABEL: @smul_v64i8(
-; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @a8 to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP5:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @b8 to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP6:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP7:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP8:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x i8> @llvm.smul.fix.v16i8(<16 x i8> [[TMP1]], <16 x i8> [[TMP5]], i32 3)
-; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x i8> @llvm.smul.fix.v16i8(<16 x i8> [[TMP2]], <16 x i8> [[TMP6]], i32 3)
-; CHECK-NEXT:    [[TMP11:%.*]] = call <16 x i8> @llvm.smul.fix.v16i8(<16 x i8> [[TMP3]], <16 x i8> [[TMP7]], i32 3)
-; CHECK-NEXT:    [[TMP12:%.*]] = call <16 x i8> @llvm.smul.fix.v16i8(<16 x i8> [[TMP4]], <16 x i8> [[TMP8]], i32 3)
-; CHECK-NEXT:    store <16 x i8> [[TMP9]], <16 x i8>* bitcast ([64 x i8]* @c8 to <16 x i8>*), align 1
-; CHECK-NEXT:    store <16 x i8> [[TMP10]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16) to <16 x i8>*), align 1
-; CHECK-NEXT:    store <16 x i8> [[TMP11]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32) to <16 x i8>*), align 1
-; CHECK-NEXT:    store <16 x i8> [[TMP12]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48) to <16 x i8>*), align 1
-; CHECK-NEXT:    ret void
+; SSE-LABEL: @smul_v64i8(
+; SSE-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @a8 to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP3:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP4:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP5:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @b8 to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP6:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP7:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP8:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP9:%.*]] = call <16 x i8> @llvm.smul.fix.v16i8(<16 x i8> [[TMP1]], <16 x i8> [[TMP5]], i32 3)
+; SSE-NEXT:    [[TMP10:%.*]] = call <16 x i8> @llvm.smul.fix.v16i8(<16 x i8> [[TMP2]], <16 x i8> [[TMP6]], i32 3)
+; SSE-NEXT:    [[TMP11:%.*]] = call <16 x i8> @llvm.smul.fix.v16i8(<16 x i8> [[TMP3]], <16 x i8> [[TMP7]], i32 3)
+; SSE-NEXT:    [[TMP12:%.*]] = call <16 x i8> @llvm.smul.fix.v16i8(<16 x i8> [[TMP4]], <16 x i8> [[TMP8]], i32 3)
+; SSE-NEXT:    store <16 x i8> [[TMP9]], <16 x i8>* bitcast ([64 x i8]* @c8 to <16 x i8>*), align 1
+; SSE-NEXT:    store <16 x i8> [[TMP10]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16) to <16 x i8>*), align 1
+; SSE-NEXT:    store <16 x i8> [[TMP11]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32) to <16 x i8>*), align 1
+; SSE-NEXT:    store <16 x i8> [[TMP12]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48) to <16 x i8>*), align 1
+; SSE-NEXT:    ret void
+;
+; SLM-LABEL: @smul_v64i8(
+; SLM-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @a8 to <16 x i8>*), align 1
+; SLM-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16) to <16 x i8>*), align 1
+; SLM-NEXT:    [[TMP3:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32) to <16 x i8>*), align 1
+; SLM-NEXT:    [[TMP4:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48) to <16 x i8>*), align 1
+; SLM-NEXT:    [[TMP5:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @b8 to <16 x i8>*), align 1
+; SLM-NEXT:    [[TMP6:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16) to <16 x i8>*), align 1
+; SLM-NEXT:    [[TMP7:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32) to <16 x i8>*), align 1
+; SLM-NEXT:    [[TMP8:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48) to <16 x i8>*), align 1
+; SLM-NEXT:    [[TMP9:%.*]] = call <16 x i8> @llvm.smul.fix.v16i8(<16 x i8> [[TMP1]], <16 x i8> [[TMP5]], i32 3)
+; SLM-NEXT:    [[TMP10:%.*]] = call <16 x i8> @llvm.smul.fix.v16i8(<16 x i8> [[TMP2]], <16 x i8> [[TMP6]], i32 3)
+; SLM-NEXT:    [[TMP11:%.*]] = call <16 x i8> @llvm.smul.fix.v16i8(<16 x i8> [[TMP3]], <16 x i8> [[TMP7]], i32 3)
+; SLM-NEXT:    [[TMP12:%.*]] = call <16 x i8> @llvm.smul.fix.v16i8(<16 x i8> [[TMP4]], <16 x i8> [[TMP8]], i32 3)
+; SLM-NEXT:    store <16 x i8> [[TMP9]], <16 x i8>* bitcast ([64 x i8]* @c8 to <16 x i8>*), align 1
+; SLM-NEXT:    store <16 x i8> [[TMP10]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16) to <16 x i8>*), align 1
+; SLM-NEXT:    store <16 x i8> [[TMP11]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32) to <16 x i8>*), align 1
+; SLM-NEXT:    store <16 x i8> [[TMP12]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48) to <16 x i8>*), align 1
+; SLM-NEXT:    ret void
+;
+; AVX-LABEL: @smul_v64i8(
+; AVX-NEXT:    [[TMP1:%.*]] = load <32 x i8>, <32 x i8>* bitcast ([64 x i8]* @a8 to <32 x i8>*), align 1
+; AVX-NEXT:    [[TMP2:%.*]] = load <32 x i8>, <32 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32) to <32 x i8>*), align 1
+; AVX-NEXT:    [[TMP3:%.*]] = load <32 x i8>, <32 x i8>* bitcast ([64 x i8]* @b8 to <32 x i8>*), align 1
+; AVX-NEXT:    [[TMP4:%.*]] = load <32 x i8>, <32 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32) to <32 x i8>*), align 1
+; AVX-NEXT:    [[TMP5:%.*]] = call <32 x i8> @llvm.smul.fix.v32i8(<32 x i8> [[TMP1]], <32 x i8> [[TMP3]], i32 3)
+; AVX-NEXT:    [[TMP6:%.*]] = call <32 x i8> @llvm.smul.fix.v32i8(<32 x i8> [[TMP2]], <32 x i8> [[TMP4]], i32 3)
+; AVX-NEXT:    store <32 x i8> [[TMP5]], <32 x i8>* bitcast ([64 x i8]* @c8 to <32 x i8>*), align 1
+; AVX-NEXT:    store <32 x i8> [[TMP6]], <32 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32) to <32 x i8>*), align 1
+; AVX-NEXT:    ret void
+;
+; AVX512-LABEL: @smul_v64i8(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <64 x i8>, <64 x i8>* bitcast ([64 x i8]* @a8 to <64 x i8>*), align 1
+; AVX512-NEXT:    [[TMP2:%.*]] = load <64 x i8>, <64 x i8>* bitcast ([64 x i8]* @b8 to <64 x i8>*), align 1
+; AVX512-NEXT:    [[TMP3:%.*]] = call <64 x i8> @llvm.smul.fix.v64i8(<64 x i8> [[TMP1]], <64 x i8> [[TMP2]], i32 3)
+; AVX512-NEXT:    store <64 x i8> [[TMP3]], <64 x i8>* bitcast ([64 x i8]* @c8 to <64 x i8>*), align 1
+; AVX512-NEXT:    ret void
 ;
   %a0  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 0 ), align 1
   %a1  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 1 ), align 1
@@ -1344,14 +1425,10 @@ define void @umul_v32i16() {
 ; AVX-NEXT:    ret void
 ;
 ; AVX512-LABEL: @umul_v32i16(
-; AVX512-NEXT:    [[TMP1:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @a16 to <16 x i16>*), align 2
-; AVX512-NEXT:    [[TMP2:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <16 x i16>*), align 2
-; AVX512-NEXT:    [[TMP3:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @b16 to <16 x i16>*), align 2
-; AVX512-NEXT:    [[TMP4:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <16 x i16>*), align 2
-; AVX512-NEXT:    [[TMP5:%.*]] = call <16 x i16> @llvm.umul.fix.v16i16(<16 x i16> [[TMP1]], <16 x i16> [[TMP3]], i32 3)
-; AVX512-NEXT:    [[TMP6:%.*]] = call <16 x i16> @llvm.umul.fix.v16i16(<16 x i16> [[TMP2]], <16 x i16> [[TMP4]], i32 3)
-; AVX512-NEXT:    store <16 x i16> [[TMP5]], <16 x i16>* bitcast ([32 x i16]* @c16 to <16 x i16>*), align 2
-; AVX512-NEXT:    store <16 x i16> [[TMP6]], <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP1:%.*]] = load <32 x i16>, <32 x i16>* bitcast ([32 x i16]* @a16 to <32 x i16>*), align 2
+; AVX512-NEXT:    [[TMP2:%.*]] = load <32 x i16>, <32 x i16>* bitcast ([32 x i16]* @b16 to <32 x i16>*), align 2
+; AVX512-NEXT:    [[TMP3:%.*]] = call <32 x i16> @llvm.umul.fix.v32i16(<32 x i16> [[TMP1]], <32 x i16> [[TMP2]], i32 3)
+; AVX512-NEXT:    store <32 x i16> [[TMP3]], <32 x i16>* bitcast ([32 x i16]* @c16 to <32 x i16>*), align 2
 ; AVX512-NEXT:    ret void
 ;
   %a0  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 0 ), align 2
@@ -1486,24 +1563,61 @@ define void @umul_v32i16() {
 }
 
 define void @umul_v64i8() {
-; CHECK-LABEL: @umul_v64i8(
-; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @a8 to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP5:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @b8 to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP6:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP7:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP8:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x i8> @llvm.umul.fix.v16i8(<16 x i8> [[TMP1]], <16 x i8> [[TMP5]], i32 3)
-; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x i8> @llvm.umul.fix.v16i8(<16 x i8> [[TMP2]], <16 x i8> [[TMP6]], i32 3)
-; CHECK-NEXT:    [[TMP11:%.*]] = call <16 x i8> @llvm.umul.fix.v16i8(<16 x i8> [[TMP3]], <16 x i8> [[TMP7]], i32 3)
-; CHECK-NEXT:    [[TMP12:%.*]] = call <16 x i8> @llvm.umul.fix.v16i8(<16 x i8> [[TMP4]], <16 x i8> [[TMP8]], i32 3)
-; CHECK-NEXT:    store <16 x i8> [[TMP9]], <16 x i8>* bitcast ([64 x i8]* @c8 to <16 x i8>*), align 1
-; CHECK-NEXT:    store <16 x i8> [[TMP10]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16) to <16 x i8>*), align 1
-; CHECK-NEXT:    store <16 x i8> [[TMP11]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32) to <16 x i8>*), align 1
-; CHECK-NEXT:    store <16 x i8> [[TMP12]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48) to <16 x i8>*), align 1
-; CHECK-NEXT:    ret void
+; SSE-LABEL: @umul_v64i8(
+; SSE-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @a8 to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP3:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP4:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP5:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @b8 to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP6:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP7:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP8:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP9:%.*]] = call <16 x i8> @llvm.umul.fix.v16i8(<16 x i8> [[TMP1]], <16 x i8> [[TMP5]], i32 3)
+; SSE-NEXT:    [[TMP10:%.*]] = call <16 x i8> @llvm.umul.fix.v16i8(<16 x i8> [[TMP2]], <16 x i8> [[TMP6]], i32 3)
+; SSE-NEXT:    [[TMP11:%.*]] = call <16 x i8> @llvm.umul.fix.v16i8(<16 x i8> [[TMP3]], <16 x i8> [[TMP7]], i32 3)
+; SSE-NEXT:    [[TMP12:%.*]] = call <16 x i8> @llvm.umul.fix.v16i8(<16 x i8> [[TMP4]], <16 x i8> [[TMP8]], i32 3)
+; SSE-NEXT:    store <16 x i8> [[TMP9]], <16 x i8>* bitcast ([64 x i8]* @c8 to <16 x i8>*), align 1
+; SSE-NEXT:    store <16 x i8> [[TMP10]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16) to <16 x i8>*), align 1
+; SSE-NEXT:    store <16 x i8> [[TMP11]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32) to <16 x i8>*), align 1
+; SSE-NEXT:    store <16 x i8> [[TMP12]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48) to <16 x i8>*), align 1
+; SSE-NEXT:    ret void
+;
+; SLM-LABEL: @umul_v64i8(
+; SLM-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @a8 to <16 x i8>*), align 1
+; SLM-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16) to <16 x i8>*), align 1
+; SLM-NEXT:    [[TMP3:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32) to <16 x i8>*), align 1
+; SLM-NEXT:    [[TMP4:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48) to <16 x i8>*), align 1
+; SLM-NEXT:    [[TMP5:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @b8 to <16 x i8>*), align 1
+; SLM-NEXT:    [[TMP6:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16) to <16 x i8>*), align 1
+; SLM-NEXT:    [[TMP7:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32) to <16 x i8>*), align 1
+; SLM-NEXT:    [[TMP8:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48) to <16 x i8>*), align 1
+; SLM-NEXT:    [[TMP9:%.*]] = call <16 x i8> @llvm.umul.fix.v16i8(<16 x i8> [[TMP1]], <16 x i8> [[TMP5]], i32 3)
+; SLM-NEXT:    [[TMP10:%.*]] = call <16 x i8> @llvm.umul.fix.v16i8(<16 x i8> [[TMP2]], <16 x i8> [[TMP6]], i32 3)
+; SLM-NEXT:    [[TMP11:%.*]] = call <16 x i8> @llvm.umul.fix.v16i8(<16 x i8> [[TMP3]], <16 x i8> [[TMP7]], i32 3)
+; SLM-NEXT:    [[TMP12:%.*]] = call <16 x i8> @llvm.umul.fix.v16i8(<16 x i8> [[TMP4]], <16 x i8> [[TMP8]], i32 3)
+; SLM-NEXT:    store <16 x i8> [[TMP9]], <16 x i8>* bitcast ([64 x i8]* @c8 to <16 x i8>*), align 1
+; SLM-NEXT:    store <16 x i8> [[TMP10]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16) to <16 x i8>*), align 1
+; SLM-NEXT:    store <16 x i8> [[TMP11]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32) to <16 x i8>*), align 1
+; SLM-NEXT:    store <16 x i8> [[TMP12]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48) to <16 x i8>*), align 1
+; SLM-NEXT:    ret void
+;
+; AVX-LABEL: @umul_v64i8(
+; AVX-NEXT:    [[TMP1:%.*]] = load <32 x i8>, <32 x i8>* bitcast ([64 x i8]* @a8 to <32 x i8>*), align 1
+; AVX-NEXT:    [[TMP2:%.*]] = load <32 x i8>, <32 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32) to <32 x i8>*), align 1
+; AVX-NEXT:    [[TMP3:%.*]] = load <32 x i8>, <32 x i8>* bitcast ([64 x i8]* @b8 to <32 x i8>*), align 1
+; AVX-NEXT:    [[TMP4:%.*]] = load <32 x i8>, <32 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32) to <32 x i8>*), align 1
+; AVX-NEXT:    [[TMP5:%.*]] = call <32 x i8> @llvm.umul.fix.v32i8(<32 x i8> [[TMP1]], <32 x i8> [[TMP3]], i32 3)
+; AVX-NEXT:    [[TMP6:%.*]] = call <32 x i8> @llvm.umul.fix.v32i8(<32 x i8> [[TMP2]], <32 x i8> [[TMP4]], i32 3)
+; AVX-NEXT:    store <32 x i8> [[TMP5]], <32 x i8>* bitcast ([64 x i8]* @c8 to <32 x i8>*), align 1
+; AVX-NEXT:    store <32 x i8> [[TMP6]], <32 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32) to <32 x i8>*), align 1
+; AVX-NEXT:    ret void
+;
+; AVX512-LABEL: @umul_v64i8(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <64 x i8>, <64 x i8>* bitcast ([64 x i8]* @a8 to <64 x i8>*), align 1
+; AVX512-NEXT:    [[TMP2:%.*]] = load <64 x i8>, <64 x i8>* bitcast ([64 x i8]* @b8 to <64 x i8>*), align 1
+; AVX512-NEXT:    [[TMP3:%.*]] = call <64 x i8> @llvm.umul.fix.v64i8(<64 x i8> [[TMP1]], <64 x i8> [[TMP2]], i32 3)
+; AVX512-NEXT:    store <64 x i8> [[TMP3]], <64 x i8>* bitcast ([64 x i8]* @c8 to <64 x i8>*), align 1
+; AVX512-NEXT:    ret void
 ;
   %a0  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 0 ), align 1
   %a1  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 1 ), align 1
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/arith-fp.ll b/llvm/test/Transforms/SLPVectorizer/X86/arith-fp.ll
index 119cf594c..9de83db6a 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/arith-fp.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/arith-fp.ll
@@ -1,9 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
 ; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SLM
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -mattr=-prefer-128-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -mattr=+prefer-128-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -mattr=-prefer-128-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -mattr=+prefer-128-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512BW
 
 ;
 ; 128-bit Vectors
@@ -96,6 +98,14 @@ define <2 x double> @buildvector_div_2f64(<2 x double> %a, <2 x double> %b) {
 ; AVX-NEXT:    [[TMP3:%.*]] = extractelement <2 x double> [[TMP1]], i32 1
 ; AVX-NEXT:    [[R1:%.*]] = insertelement <2 x double> [[R0]], double [[TMP3]], i32 1
 ; AVX-NEXT:    ret <2 x double> [[R1]]
+;
+; AVX512-LABEL: @buildvector_div_2f64(
+; AVX512-NEXT:    [[TMP1:%.*]] = fdiv <2 x double> [[A:%.*]], [[B:%.*]]
+; AVX512-NEXT:    [[TMP2:%.*]] = extractelement <2 x double> [[TMP1]], i32 0
+; AVX512-NEXT:    [[R0:%.*]] = insertelement <2 x double> undef, double [[TMP2]], i32 0
+; AVX512-NEXT:    [[TMP3:%.*]] = extractelement <2 x double> [[TMP1]], i32 1
+; AVX512-NEXT:    [[R1:%.*]] = insertelement <2 x double> [[R0]], double [[TMP3]], i32 1
+; AVX512-NEXT:    ret <2 x double> [[R1]]
 ;
   %a0 = extractelement <2 x double> %a, i32 0
   %a1 = extractelement <2 x double> %a, i32 1
@@ -379,6 +389,18 @@ define <4 x double> @buildvector_div_4f64(<4 x double> %a, <4 x double> %b) {
 ; AVX-NEXT:    [[TMP5:%.*]] = extractelement <4 x double> [[TMP1]], i32 3
 ; AVX-NEXT:    [[R3:%.*]] = insertelement <4 x double> [[R2]], double [[TMP5]], i32 3
 ; AVX-NEXT:    ret <4 x double> [[R3]]
+;
+; AVX512-LABEL: @buildvector_div_4f64(
+; AVX512-NEXT:    [[TMP1:%.*]] = fdiv <4 x double> [[A:%.*]], [[B:%.*]]
+; AVX512-NEXT:    [[TMP2:%.*]] = extractelement <4 x double> [[TMP1]], i32 0
+; AVX512-NEXT:    [[R0:%.*]] = insertelement <4 x double> undef, double [[TMP2]], i32 0
+; AVX512-NEXT:    [[TMP3:%.*]] = extractelement <4 x double> [[TMP1]], i32 1
+; AVX512-NEXT:    [[R1:%.*]] = insertelement <4 x double> [[R0]], double [[TMP3]], i32 1
+; AVX512-NEXT:    [[TMP4:%.*]] = extractelement <4 x double> [[TMP1]], i32 2
+; AVX512-NEXT:    [[R2:%.*]] = insertelement <4 x double> [[R1]], double [[TMP4]], i32 2
+; AVX512-NEXT:    [[TMP5:%.*]] = extractelement <4 x double> [[TMP1]], i32 3
+; AVX512-NEXT:    [[R3:%.*]] = insertelement <4 x double> [[R2]], double [[TMP5]], i32 3
+; AVX512-NEXT:    ret <4 x double> [[R3]]
 ;
   %a0 = extractelement <4 x double> %a, i32 0
   %a1 = extractelement <4 x double> %a, i32 1
@@ -870,6 +892,26 @@ define <8 x double> @buildvector_div_8f64(<8 x double> %a, <8 x double> %b) {
 ; AVX-NEXT:    [[TMP9:%.*]] = extractelement <8 x double> [[TMP1]], i32 7
 ; AVX-NEXT:    [[R7:%.*]] = insertelement <8 x double> [[R6]], double [[TMP9]], i32 7
 ; AVX-NEXT:    ret <8 x double> [[R7]]
+;
+; AVX512-LABEL: @buildvector_div_8f64(
+; AVX512-NEXT:    [[TMP1:%.*]] = fdiv <8 x double> [[A:%.*]], [[B:%.*]]
+; AVX512-NEXT:    [[TMP2:%.*]] = extractelement <8 x double> [[TMP1]], i32 0
+; AVX512-NEXT:    [[R0:%.*]] = insertelement <8 x double> undef, double [[TMP2]], i32 0
+; AVX512-NEXT:    [[TMP3:%.*]] = extractelement <8 x double> [[TMP1]], i32 1
+; AVX512-NEXT:    [[R1:%.*]] = insertelement <8 x double> [[R0]], double [[TMP3]], i32 1
+; AVX512-NEXT:    [[TMP4:%.*]] = extractelement <8 x double> [[TMP1]], i32 2
+; AVX512-NEXT:    [[R2:%.*]] = insertelement <8 x double> [[R1]], double [[TMP4]], i32 2
+; AVX512-NEXT:    [[TMP5:%.*]] = extractelement <8 x double> [[TMP1]], i32 3
+; AVX512-NEXT:    [[R3:%.*]] = insertelement <8 x double> [[R2]], double [[TMP5]], i32 3
+; AVX512-NEXT:    [[TMP6:%.*]] = extractelement <8 x double> [[TMP1]], i32 4
+; AVX512-NEXT:    [[R4:%.*]] = insertelement <8 x double> [[R3]], double [[TMP6]], i32 4
+; AVX512-NEXT:    [[TMP7:%.*]] = extractelement <8 x double> [[TMP1]], i32 5
+; AVX512-NEXT:    [[R5:%.*]] = insertelement <8 x double> [[R4]], double [[TMP7]], i32 5
+; AVX512-NEXT:    [[TMP8:%.*]] = extractelement <8 x double> [[TMP1]], i32 6
+; AVX512-NEXT:    [[R6:%.*]] = insertelement <8 x double> [[R5]], double [[TMP8]], i32 6
+; AVX512-NEXT:    [[TMP9:%.*]] = extractelement <8 x double> [[TMP1]], i32 7
+; AVX512-NEXT:    [[R7:%.*]] = insertelement <8 x double> [[R6]], double [[TMP9]], i32 7
+; AVX512-NEXT:    ret <8 x double> [[R7]]
 ;
   %a0 = extractelement <8 x double> %a, i32 0
   %a1 = extractelement <8 x double> %a, i32 1
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/arith-mul.ll b/llvm/test/Transforms/SLPVectorizer/X86/arith-mul.ll
index 8a01c85bc..8dc59d3e8 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/arith-mul.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/arith-mul.ll
@@ -1,8 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
 ; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SLM
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX1
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -mattr=-prefer-128-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -mattr=+prefer-128-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -mattr=-prefer-128-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -mattr=+prefer-128-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
 ; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512F
 ; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=-prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512BW
 ; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=+prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
@@ -354,14 +356,10 @@ define void @mul_v32i16() {
 ; AVX-NEXT:    ret void
 ;
 ; AVX512-LABEL: @mul_v32i16(
-; AVX512-NEXT:    [[TMP1:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @a16 to <16 x i16>*), align 2
-; AVX512-NEXT:    [[TMP2:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <16 x i16>*), align 2
-; AVX512-NEXT:    [[TMP3:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @b16 to <16 x i16>*), align 2
-; AVX512-NEXT:    [[TMP4:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <16 x i16>*), align 2
-; AVX512-NEXT:    [[TMP5:%.*]] = mul <16 x i16> [[TMP1]], [[TMP3]]
-; AVX512-NEXT:    [[TMP6:%.*]] = mul <16 x i16> [[TMP2]], [[TMP4]]
-; AVX512-NEXT:    store <16 x i16> [[TMP5]], <16 x i16>* bitcast ([32 x i16]* @c16 to <16 x i16>*), align 2
-; AVX512-NEXT:    store <16 x i16> [[TMP6]], <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP1:%.*]] = load <32 x i16>, <32 x i16>* bitcast ([32 x i16]* @a16 to <32 x i16>*), align 2
+; AVX512-NEXT:    [[TMP2:%.*]] = load <32 x i16>, <32 x i16>* bitcast ([32 x i16]* @b16 to <32 x i16>*), align 2
+; AVX512-NEXT:    [[TMP3:%.*]] = mul <32 x i16> [[TMP1]], [[TMP2]]
+; AVX512-NEXT:    store <32 x i16> [[TMP3]], <32 x i16>* bitcast ([32 x i16]* @c16 to <32 x i16>*), align 2
 ; AVX512-NEXT:    ret void
 ;
   %a0  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 0 ), align 2
@@ -496,24 +494,61 @@ define void @mul_v32i16() {
 }
 
 define void @mul_v64i8() {
-; CHECK-LABEL: @mul_v64i8(
-; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @a8 to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP5:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @b8 to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP6:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP7:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP8:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP9:%.*]] = mul <16 x i8> [[TMP1]], [[TMP5]]
-; CHECK-NEXT:    [[TMP10:%.*]] = mul <16 x i8> [[TMP2]], [[TMP6]]
-; CHECK-NEXT:    [[TMP11:%.*]] = mul <16 x i8> [[TMP3]], [[TMP7]]
-; CHECK-NEXT:    [[TMP12:%.*]] = mul <16 x i8> [[TMP4]], [[TMP8]]
-; CHECK-NEXT:    store <16 x i8> [[TMP9]], <16 x i8>* bitcast ([64 x i8]* @c8 to <16 x i8>*), align 1
-; CHECK-NEXT:    store <16 x i8> [[TMP10]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16) to <16 x i8>*), align 1
-; CHECK-NEXT:    store <16 x i8> [[TMP11]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32) to <16 x i8>*), align 1
-; CHECK-NEXT:    store <16 x i8> [[TMP12]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48) to <16 x i8>*), align 1
-; CHECK-NEXT:    ret void
+; SSE-LABEL: @mul_v64i8(
+; SSE-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @a8 to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP3:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP4:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP5:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @b8 to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP6:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP7:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP8:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP9:%.*]] = mul <16 x i8> [[TMP1]], [[TMP5]]
+; SSE-NEXT:    [[TMP10:%.*]] = mul <16 x i8> [[TMP2]], [[TMP6]]
+; SSE-NEXT:    [[TMP11:%.*]] = mul <16 x i8> [[TMP3]], [[TMP7]]
+; SSE-NEXT:    [[TMP12:%.*]] = mul <16 x i8> [[TMP4]], [[TMP8]]
+; SSE-NEXT:    store <16 x i8> [[TMP9]], <16 x i8>* bitcast ([64 x i8]* @c8 to <16 x i8>*), align 1
+; SSE-NEXT:    store <16 x i8> [[TMP10]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16) to <16 x i8>*), align 1
+; SSE-NEXT:    store <16 x i8> [[TMP11]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32) to <16 x i8>*), align 1
+; SSE-NEXT:    store <16 x i8> [[TMP12]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48) to <16 x i8>*), align 1
+; SSE-NEXT:    ret void
+;
+; SLM-LABEL: @mul_v64i8(
+; SLM-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @a8 to <16 x i8>*), align 1
+; SLM-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16) to <16 x i8>*), align 1
+; SLM-NEXT:    [[TMP3:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32) to <16 x i8>*), align 1
+; SLM-NEXT:    [[TMP4:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48) to <16 x i8>*), align 1
+; SLM-NEXT:    [[TMP5:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @b8 to <16 x i8>*), align 1
+; SLM-NEXT:    [[TMP6:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16) to <16 x i8>*), align 1
+; SLM-NEXT:    [[TMP7:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32) to <16 x i8>*), align 1
+; SLM-NEXT:    [[TMP8:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48) to <16 x i8>*), align 1
+; SLM-NEXT:    [[TMP9:%.*]] = mul <16 x i8> [[TMP1]], [[TMP5]]
+; SLM-NEXT:    [[TMP10:%.*]] = mul <16 x i8> [[TMP2]], [[TMP6]]
+; SLM-NEXT:    [[TMP11:%.*]] = mul <16 x i8> [[TMP3]], [[TMP7]]
+; SLM-NEXT:    [[TMP12:%.*]] = mul <16 x i8> [[TMP4]], [[TMP8]]
+; SLM-NEXT:    store <16 x i8> [[TMP9]], <16 x i8>* bitcast ([64 x i8]* @c8 to <16 x i8>*), align 1
+; SLM-NEXT:    store <16 x i8> [[TMP10]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16) to <16 x i8>*), align 1
+; SLM-NEXT:    store <16 x i8> [[TMP11]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32) to <16 x i8>*), align 1
+; SLM-NEXT:    store <16 x i8> [[TMP12]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48) to <16 x i8>*), align 1
+; SLM-NEXT:    ret void
+;
+; AVX-LABEL: @mul_v64i8(
+; AVX-NEXT:    [[TMP1:%.*]] = load <32 x i8>, <32 x i8>* bitcast ([64 x i8]* @a8 to <32 x i8>*), align 1
+; AVX-NEXT:    [[TMP2:%.*]] = load <32 x i8>, <32 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32) to <32 x i8>*), align 1
+; AVX-NEXT:    [[TMP3:%.*]] = load <32 x i8>, <32 x i8>* bitcast ([64 x i8]* @b8 to <32 x i8>*), align 1
+; AVX-NEXT:    [[TMP4:%.*]] = load <32 x i8>, <32 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32) to <32 x i8>*), align 1
+; AVX-NEXT:    [[TMP5:%.*]] = mul <32 x i8> [[TMP1]], [[TMP3]]
+; AVX-NEXT:    [[TMP6:%.*]] = mul <32 x i8> [[TMP2]], [[TMP4]]
+; AVX-NEXT:    store <32 x i8> [[TMP5]], <32 x i8>* bitcast ([64 x i8]* @c8 to <32 x i8>*), align 1
+; AVX-NEXT:    store <32 x i8> [[TMP6]], <32 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32) to <32 x i8>*), align 1
+; AVX-NEXT:    ret void
+;
+; AVX512-LABEL: @mul_v64i8(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <64 x i8>, <64 x i8>* bitcast ([64 x i8]* @a8 to <64 x i8>*), align 1
+; AVX512-NEXT:    [[TMP2:%.*]] = load <64 x i8>, <64 x i8>* bitcast ([64 x i8]* @b8 to <64 x i8>*), align 1
+; AVX512-NEXT:    [[TMP3:%.*]] = mul <64 x i8> [[TMP1]], [[TMP2]]
+; AVX512-NEXT:    store <64 x i8> [[TMP3]], <64 x i8>* bitcast ([64 x i8]* @c8 to <64 x i8>*), align 1
+; AVX512-NEXT:    ret void
 ;
   %a0  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 0 ), align 1
   %a1  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 1 ), align 1
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/arith-sub-ssat.ll b/llvm/test/Transforms/SLPVectorizer/X86/arith-sub-ssat.ll
index 31c8c4243..dd9d4f62f 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/arith-sub-ssat.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/arith-sub-ssat.ll
@@ -354,14 +354,10 @@ define void @sub_v32i16() {
 ; AVX-NEXT:    ret void
 ;
 ; AVX512-LABEL: @sub_v32i16(
-; AVX512-NEXT:    [[TMP1:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @a16 to <16 x i16>*), align 2
-; AVX512-NEXT:    [[TMP2:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <16 x i16>*), align 2
-; AVX512-NEXT:    [[TMP3:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @b16 to <16 x i16>*), align 2
-; AVX512-NEXT:    [[TMP4:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <16 x i16>*), align 2
-; AVX512-NEXT:    [[TMP5:%.*]] = call <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16> [[TMP1]], <16 x i16> [[TMP3]])
-; AVX512-NEXT:    [[TMP6:%.*]] = call <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16> [[TMP2]], <16 x i16> [[TMP4]])
-; AVX512-NEXT:    store <16 x i16> [[TMP5]], <16 x i16>* bitcast ([32 x i16]* @c16 to <16 x i16>*), align 2
-; AVX512-NEXT:    store <16 x i16> [[TMP6]], <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP1:%.*]] = load <32 x i16>, <32 x i16>* bitcast ([32 x i16]* @a16 to <32 x i16>*), align 2
+; AVX512-NEXT:    [[TMP2:%.*]] = load <32 x i16>, <32 x i16>* bitcast ([32 x i16]* @b16 to <32 x i16>*), align 2
+; AVX512-NEXT:    [[TMP3:%.*]] = call <32 x i16> @llvm.ssub.sat.v32i16(<32 x i16> [[TMP1]], <32 x i16> [[TMP2]])
+; AVX512-NEXT:    store <32 x i16> [[TMP3]], <32 x i16>* bitcast ([32 x i16]* @c16 to <32 x i16>*), align 2
 ; AVX512-NEXT:    ret void
 ;
   %a0  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 0 ), align 2
@@ -496,24 +492,61 @@ define void @sub_v32i16() {
 }
 
 define void @sub_v64i8() {
-; CHECK-LABEL: @sub_v64i8(
-; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @a8 to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP5:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @b8 to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP6:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP7:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP8:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> [[TMP1]], <16 x i8> [[TMP5]])
-; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> [[TMP2]], <16 x i8> [[TMP6]])
-; CHECK-NEXT:    [[TMP11:%.*]] = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> [[TMP3]], <16 x i8> [[TMP7]])
-; CHECK-NEXT:    [[TMP12:%.*]] = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> [[TMP4]], <16 x i8> [[TMP8]])
-; CHECK-NEXT:    store <16 x i8> [[TMP9]], <16 x i8>* bitcast ([64 x i8]* @c8 to <16 x i8>*), align 1
-; CHECK-NEXT:    store <16 x i8> [[TMP10]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16) to <16 x i8>*), align 1
-; CHECK-NEXT:    store <16 x i8> [[TMP11]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32) to <16 x i8>*), align 1
-; CHECK-NEXT:    store <16 x i8> [[TMP12]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48) to <16 x i8>*), align 1
-; CHECK-NEXT:    ret void
+; SSE-LABEL: @sub_v64i8(
+; SSE-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @a8 to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP3:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP4:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP5:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @b8 to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP6:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP7:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP8:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP9:%.*]] = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> [[TMP1]], <16 x i8> [[TMP5]])
+; SSE-NEXT:    [[TMP10:%.*]] = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> [[TMP2]], <16 x i8> [[TMP6]])
+; SSE-NEXT:    [[TMP11:%.*]] = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> [[TMP3]], <16 x i8> [[TMP7]])
+; SSE-NEXT:    [[TMP12:%.*]] = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> [[TMP4]], <16 x i8> [[TMP8]])
+; SSE-NEXT:    store <16 x i8> [[TMP9]], <16 x i8>* bitcast ([64 x i8]* @c8 to <16 x i8>*), align 1
+; SSE-NEXT:    store <16 x i8> [[TMP10]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16) to <16 x i8>*), align 1
+; SSE-NEXT:    store <16 x i8> [[TMP11]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32) to <16 x i8>*), align 1
+; SSE-NEXT:    store <16 x i8> [[TMP12]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48) to <16 x i8>*), align 1
+; SSE-NEXT:    ret void
+;
+; SLM-LABEL: @sub_v64i8(
+; SLM-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @a8 to <16 x i8>*), align 1
+; SLM-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16) to <16 x i8>*), align 1
+; SLM-NEXT:    [[TMP3:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32) to <16 x i8>*), align 1
+; SLM-NEXT:    [[TMP4:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48) to <16 x i8>*), align 1
+; SLM-NEXT:    [[TMP5:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @b8 to <16 x i8>*), align 1
+; SLM-NEXT:    [[TMP6:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16) to <16 x i8>*), align 1
+; SLM-NEXT:    [[TMP7:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32) to <16 x i8>*), align 1
+; SLM-NEXT:    [[TMP8:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48) to <16 x i8>*), align 1
+; SLM-NEXT:    [[TMP9:%.*]] = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> [[TMP1]], <16 x i8> [[TMP5]])
+; SLM-NEXT:    [[TMP10:%.*]] = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> [[TMP2]], <16 x i8> [[TMP6]])
+; SLM-NEXT:    [[TMP11:%.*]] = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> [[TMP3]], <16 x i8> [[TMP7]])
+; SLM-NEXT:    [[TMP12:%.*]] = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> [[TMP4]], <16 x i8> [[TMP8]])
+; SLM-NEXT:    store <16 x i8> [[TMP9]], <16 x i8>* bitcast ([64 x i8]* @c8 to <16 x i8>*), align 1
+; SLM-NEXT:    store <16 x i8> [[TMP10]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16) to <16 x i8>*), align 1
+; SLM-NEXT:    store <16 x i8> [[TMP11]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32) to <16 x i8>*), align 1
+; SLM-NEXT:    store <16 x i8> [[TMP12]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48) to <16 x i8>*), align 1
+; SLM-NEXT:    ret void
+;
+; AVX-LABEL: @sub_v64i8(
+; AVX-NEXT:    [[TMP1:%.*]] = load <32 x i8>, <32 x i8>* bitcast ([64 x i8]* @a8 to <32 x i8>*), align 1
+; AVX-NEXT:    [[TMP2:%.*]] = load <32 x i8>, <32 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32) to <32 x i8>*), align 1
+; AVX-NEXT:    [[TMP3:%.*]] = load <32 x i8>, <32 x i8>* bitcast ([64 x i8]* @b8 to <32 x i8>*), align 1
+; AVX-NEXT:    [[TMP4:%.*]] = load <32 x i8>, <32 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32) to <32 x i8>*), align 1
+; AVX-NEXT:    [[TMP5:%.*]] = call <32 x i8> @llvm.ssub.sat.v32i8(<32 x i8> [[TMP1]], <32 x i8> [[TMP3]])
+; AVX-NEXT:    [[TMP6:%.*]] = call <32 x i8> @llvm.ssub.sat.v32i8(<32 x i8> [[TMP2]], <32 x i8> [[TMP4]])
+; AVX-NEXT:    store <32 x i8> [[TMP5]], <32 x i8>* bitcast ([64 x i8]* @c8 to <32 x i8>*), align 1
+; AVX-NEXT:    store <32 x i8> [[TMP6]], <32 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32) to <32 x i8>*), align 1
+; AVX-NEXT:    ret void
+;
+; AVX512-LABEL: @sub_v64i8(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <64 x i8>, <64 x i8>* bitcast ([64 x i8]* @a8 to <64 x i8>*), align 1
+; AVX512-NEXT:    [[TMP2:%.*]] = load <64 x i8>, <64 x i8>* bitcast ([64 x i8]* @b8 to <64 x i8>*), align 1
+; AVX512-NEXT:    [[TMP3:%.*]] = call <64 x i8> @llvm.ssub.sat.v64i8(<64 x i8> [[TMP1]], <64 x i8> [[TMP2]])
+; AVX512-NEXT:    store <64 x i8> [[TMP3]], <64 x i8>* bitcast ([64 x i8]* @c8 to <64 x i8>*), align 1
+; AVX512-NEXT:    ret void
 ;
   %a0  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 0 ), align 1
   %a1  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 1 ), align 1
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/arith-sub-usat.ll b/llvm/test/Transforms/SLPVectorizer/X86/arith-sub-usat.ll
index ec554ff4f..5080d05e3 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/arith-sub-usat.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/arith-sub-usat.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE,SSE2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE,SLM
 ; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
 ; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
 ; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F
@@ -61,25 +61,6 @@ define void @sub_v8i64() {
 ; SSE-NEXT:    store i64 [[R7]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 7), align 8
 ; SSE-NEXT:    ret void
 ;
-; SLM-LABEL: @sub_v8i64(
-; SLM-NEXT:    [[TMP1:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([8 x i64]* @a64 to <2 x i64>*), align 8
-; SLM-NEXT:    [[TMP2:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2) to <2 x i64>*), align 8
-; SLM-NEXT:    [[TMP3:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4) to <2 x i64>*), align 8
-; SLM-NEXT:    [[TMP4:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6) to <2 x i64>*), align 8
-; SLM-NEXT:    [[TMP5:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([8 x i64]* @b64 to <2 x i64>*), align 8
-; SLM-NEXT:    [[TMP6:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2) to <2 x i64>*), align 8
-; SLM-NEXT:    [[TMP7:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4) to <2 x i64>*), align 8
-; SLM-NEXT:    [[TMP8:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6) to <2 x i64>*), align 8
-; SLM-NEXT:    [[TMP9:%.*]] = call <2 x i64> @llvm.usub.sat.v2i64(<2 x i64> [[TMP1]], <2 x i64> [[TMP5]])
-; SLM-NEXT:    [[TMP10:%.*]] = call <2 x i64> @llvm.usub.sat.v2i64(<2 x i64> [[TMP2]], <2 x i64> [[TMP6]])
-; SLM-NEXT:    [[TMP11:%.*]] = call <2 x i64> @llvm.usub.sat.v2i64(<2 x i64> [[TMP3]], <2 x i64> [[TMP7]])
-; SLM-NEXT:    [[TMP12:%.*]] = call <2 x i64> @llvm.usub.sat.v2i64(<2 x i64> [[TMP4]], <2 x i64> [[TMP8]])
-; SLM-NEXT:    store <2 x i64> [[TMP9]], <2 x i64>* bitcast ([8 x i64]* @c64 to <2 x i64>*), align 8
-; SLM-NEXT:    store <2 x i64> [[TMP10]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2) to <2 x i64>*), align 8
-; SLM-NEXT:    store <2 x i64> [[TMP11]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4) to <2 x i64>*), align 8
-; SLM-NEXT:    store <2 x i64> [[TMP12]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6) to <2 x i64>*), align 8
-; SLM-NEXT:    ret void
-;
 ; AVX-LABEL: @sub_v8i64(
 ; AVX-NEXT:    [[TMP1:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([8 x i64]* @a64 to <4 x i64>*), align 8
 ; AVX-NEXT:    [[TMP2:%.*]] = load <4 x i64>, <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4) to <4 x i64>*), align 8
@@ -153,25 +134,6 @@ define void @sub_v16i32() {
 ; SSE-NEXT:    store <4 x i32> [[TMP12]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12) to <4 x i32>*), align 4
 ; SSE-NEXT:    ret void
 ;
-; SLM-LABEL: @sub_v16i32(
-; SLM-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @a32 to <4 x i32>*), align 4
-; SLM-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4) to <4 x i32>*), align 4
-; SLM-NEXT:    [[TMP3:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8) to <4 x i32>*), align 4
-; SLM-NEXT:    [[TMP4:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12) to <4 x i32>*), align 4
-; SLM-NEXT:    [[TMP5:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @b32 to <4 x i32>*), align 4
-; SLM-NEXT:    [[TMP6:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4) to <4 x i32>*), align 4
-; SLM-NEXT:    [[TMP7:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8) to <4 x i32>*), align 4
-; SLM-NEXT:    [[TMP8:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12) to <4 x i32>*), align 4
-; SLM-NEXT:    [[TMP9:%.*]] = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> [[TMP1]], <4 x i32> [[TMP5]])
-; SLM-NEXT:    [[TMP10:%.*]] = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> [[TMP2]], <4 x i32> [[TMP6]])
-; SLM-NEXT:    [[TMP11:%.*]] = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> [[TMP3]], <4 x i32> [[TMP7]])
-; SLM-NEXT:    [[TMP12:%.*]] = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> [[TMP4]], <4 x i32> [[TMP8]])
-; SLM-NEXT:    store <4 x i32> [[TMP9]], <4 x i32>* bitcast ([16 x i32]* @c32 to <4 x i32>*), align 4
-; SLM-NEXT:    store <4 x i32> [[TMP10]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4) to <4 x i32>*), align 4
-; SLM-NEXT:    store <4 x i32> [[TMP11]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8) to <4 x i32>*), align 4
-; SLM-NEXT:    store <4 x i32> [[TMP12]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12) to <4 x i32>*), align 4
-; SLM-NEXT:    ret void
-;
 ; AVX-LABEL: @sub_v16i32(
 ; AVX-NEXT:    [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([16 x i32]* @a32 to <8 x i32>*), align 4
 ; AVX-NEXT:    [[TMP2:%.*]] = load <8 x i32>, <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8) to <8 x i32>*), align 4
@@ -277,25 +239,6 @@ define void @sub_v32i16() {
 ; SSE-NEXT:    store <8 x i16> [[TMP12]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 24) to <8 x i16>*), align 2
 ; SSE-NEXT:    ret void
 ;
-; SLM-LABEL: @sub_v32i16(
-; SLM-NEXT:    [[TMP1:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @a16 to <8 x i16>*), align 2
-; SLM-NEXT:    [[TMP2:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 8) to <8 x i16>*), align 2
-; SLM-NEXT:    [[TMP3:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <8 x i16>*), align 2
-; SLM-NEXT:    [[TMP4:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 24) to <8 x i16>*), align 2
-; SLM-NEXT:    [[TMP5:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @b16 to <8 x i16>*), align 2
-; SLM-NEXT:    [[TMP6:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 8) to <8 x i16>*), align 2
-; SLM-NEXT:    [[TMP7:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <8 x i16>*), align 2
-; SLM-NEXT:    [[TMP8:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 24) to <8 x i16>*), align 2
-; SLM-NEXT:    [[TMP9:%.*]] = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> [[TMP1]], <8 x i16> [[TMP5]])
-; SLM-NEXT:    [[TMP10:%.*]] = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> [[TMP2]], <8 x i16> [[TMP6]])
-; SLM-NEXT:    [[TMP11:%.*]] = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> [[TMP3]], <8 x i16> [[TMP7]])
-; SLM-NEXT:    [[TMP12:%.*]] = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> [[TMP4]], <8 x i16> [[TMP8]])
-; SLM-NEXT:    store <8 x i16> [[TMP9]], <8 x i16>* bitcast ([32 x i16]* @c16 to <8 x i16>*), align 2
-; SLM-NEXT:    store <8 x i16> [[TMP10]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 8) to <8 x i16>*), align 2
-; SLM-NEXT:    store <8 x i16> [[TMP11]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <8 x i16>*), align 2
-; SLM-NEXT:    store <8 x i16> [[TMP12]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 24) to <8 x i16>*), align 2
-; SLM-NEXT:    ret void
-;
 ; AVX-LABEL: @sub_v32i16(
 ; AVX-NEXT:    [[TMP1:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @a16 to <16 x i16>*), align 2
 ; AVX-NEXT:    [[TMP2:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <16 x i16>*), align 2
@@ -308,14 +251,10 @@ define void @sub_v32i16() {
 ; AVX-NEXT:    ret void
 ;
 ; AVX512-LABEL: @sub_v32i16(
-; AVX512-NEXT:    [[TMP1:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @a16 to <16 x i16>*), align 2
-; AVX512-NEXT:    [[TMP2:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <16 x i16>*), align 2
-; AVX512-NEXT:    [[TMP3:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @b16 to <16 x i16>*), align 2
-; AVX512-NEXT:    [[TMP4:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <16 x i16>*), align 2
-; AVX512-NEXT:    [[TMP5:%.*]] = call <16 x i16> @llvm.usub.sat.v16i16(<16 x i16> [[TMP1]], <16 x i16> [[TMP3]])
-; AVX512-NEXT:    [[TMP6:%.*]] = call <16 x i16> @llvm.usub.sat.v16i16(<16 x i16> [[TMP2]], <16 x i16> [[TMP4]])
-; AVX512-NEXT:    store <16 x i16> [[TMP5]], <16 x i16>* bitcast ([32 x i16]* @c16 to <16 x i16>*), align 2
-; AVX512-NEXT:    store <16 x i16> [[TMP6]], <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP1:%.*]] = load <32 x i16>, <32 x i16>* bitcast ([32 x i16]* @a16 to <32 x i16>*), align 2
+; AVX512-NEXT:    [[TMP2:%.*]] = load <32 x i16>, <32 x i16>* bitcast ([32 x i16]* @b16 to <32 x i16>*), align 2
+; AVX512-NEXT:    [[TMP3:%.*]] = call <32 x i16> @llvm.usub.sat.v32i16(<32 x i16> [[TMP1]], <32 x i16> [[TMP2]])
+; AVX512-NEXT:    store <32 x i16> [[TMP3]], <32 x i16>* bitcast ([32 x i16]* @c16 to <32 x i16>*), align 2
 ; AVX512-NEXT:    ret void
 ;
   %a0  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 0 ), align 2
@@ -450,24 +389,42 @@ define void @sub_v32i16() {
 }
 
 define void @sub_v64i8() {
-; CHECK-LABEL: @sub_v64i8(
-; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @a8 to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP5:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @b8 to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP6:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP7:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP8:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> [[TMP1]], <16 x i8> [[TMP5]])
-; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> [[TMP2]], <16 x i8> [[TMP6]])
-; CHECK-NEXT:    [[TMP11:%.*]] = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> [[TMP3]], <16 x i8> [[TMP7]])
-; CHECK-NEXT:    [[TMP12:%.*]] = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> [[TMP4]], <16 x i8> [[TMP8]])
-; CHECK-NEXT:    store <16 x i8> [[TMP9]], <16 x i8>* bitcast ([64 x i8]* @c8 to <16 x i8>*), align 1
-; CHECK-NEXT:    store <16 x i8> [[TMP10]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16) to <16 x i8>*), align 1
-; CHECK-NEXT:    store <16 x i8> [[TMP11]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32) to <16 x i8>*), align 1
-; CHECK-NEXT:    store <16 x i8> [[TMP12]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48) to <16 x i8>*), align 1
-; CHECK-NEXT:    ret void
+; SSE-LABEL: @sub_v64i8(
+; SSE-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @a8 to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP3:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP4:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP5:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @b8 to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP6:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP7:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP8:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP9:%.*]] = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> [[TMP1]], <16 x i8> [[TMP5]])
+; SSE-NEXT:    [[TMP10:%.*]] = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> [[TMP2]], <16 x i8> [[TMP6]])
+; SSE-NEXT:    [[TMP11:%.*]] = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> [[TMP3]], <16 x i8> [[TMP7]])
+; SSE-NEXT:    [[TMP12:%.*]] = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> [[TMP4]], <16 x i8> [[TMP8]])
+; SSE-NEXT:    store <16 x i8> [[TMP9]], <16 x i8>* bitcast ([64 x i8]* @c8 to <16 x i8>*), align 1
+; SSE-NEXT:    store <16 x i8> [[TMP10]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16) to <16 x i8>*), align 1
+; SSE-NEXT:    store <16 x i8> [[TMP11]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32) to <16 x i8>*), align 1
+; SSE-NEXT:    store <16 x i8> [[TMP12]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48) to <16 x i8>*), align 1
+; SSE-NEXT:    ret void
+;
+; AVX-LABEL: @sub_v64i8(
+; AVX-NEXT:    [[TMP1:%.*]] = load <32 x i8>, <32 x i8>* bitcast ([64 x i8]* @a8 to <32 x i8>*), align 1
+; AVX-NEXT:    [[TMP2:%.*]] = load <32 x i8>, <32 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32) to <32 x i8>*), align 1
+; AVX-NEXT:    [[TMP3:%.*]] = load <32 x i8>, <32 x i8>* bitcast ([64 x i8]* @b8 to <32 x i8>*), align 1
+; AVX-NEXT:    [[TMP4:%.*]] = load <32 x i8>, <32 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32) to <32 x i8>*), align 1
+; AVX-NEXT:    [[TMP5:%.*]] = call <32 x i8> @llvm.usub.sat.v32i8(<32 x i8> [[TMP1]], <32 x i8> [[TMP3]])
+; AVX-NEXT:    [[TMP6:%.*]] = call <32 x i8> @llvm.usub.sat.v32i8(<32 x i8> [[TMP2]], <32 x i8> [[TMP4]])
+; AVX-NEXT:    store <32 x i8> [[TMP5]], <32 x i8>* bitcast ([64 x i8]* @c8 to <32 x i8>*), align 1
+; AVX-NEXT:    store <32 x i8> [[TMP6]], <32 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32) to <32 x i8>*), align 1
+; AVX-NEXT:    ret void
+;
+; AVX512-LABEL: @sub_v64i8(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <64 x i8>, <64 x i8>* bitcast ([64 x i8]* @a8 to <64 x i8>*), align 1
+; AVX512-NEXT:    [[TMP2:%.*]] = load <64 x i8>, <64 x i8>* bitcast ([64 x i8]* @b8 to <64 x i8>*), align 1
+; AVX512-NEXT:    [[TMP3:%.*]] = call <64 x i8> @llvm.usub.sat.v64i8(<64 x i8> [[TMP1]], <64 x i8> [[TMP2]])
+; AVX512-NEXT:    store <64 x i8> [[TMP3]], <64 x i8>* bitcast ([64 x i8]* @c8 to <64 x i8>*), align 1
+; AVX512-NEXT:    ret void
 ;
   %a0  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 0 ), align 1
   %a1  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 1 ), align 1
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/arith-sub.ll b/llvm/test/Transforms/SLPVectorizer/X86/arith-sub.ll
index 28c807408..eb78e39b0 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/arith-sub.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/arith-sub.ll
@@ -1,8 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
 ; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SLM
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX1
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -mattr=-prefer-128-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -mattr=+prefer-128-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -mattr=-prefer-128-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -mattr=+prefer-128-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
 ; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512F
 ; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=-prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512BW
 ; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=+prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
@@ -287,14 +289,10 @@ define void @sub_v32i16() {
 ; AVX-NEXT:    ret void
 ;
 ; AVX512-LABEL: @sub_v32i16(
-; AVX512-NEXT:    [[TMP1:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @a16 to <16 x i16>*), align 2
-; AVX512-NEXT:    [[TMP2:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <16 x i16>*), align 2
-; AVX512-NEXT:    [[TMP3:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @b16 to <16 x i16>*), align 2
-; AVX512-NEXT:    [[TMP4:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <16 x i16>*), align 2
-; AVX512-NEXT:    [[TMP5:%.*]] = sub <16 x i16> [[TMP1]], [[TMP3]]
-; AVX512-NEXT:    [[TMP6:%.*]] = sub <16 x i16> [[TMP2]], [[TMP4]]
-; AVX512-NEXT:    store <16 x i16> [[TMP5]], <16 x i16>* bitcast ([32 x i16]* @c16 to <16 x i16>*), align 2
-; AVX512-NEXT:    store <16 x i16> [[TMP6]], <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP1:%.*]] = load <32 x i16>, <32 x i16>* bitcast ([32 x i16]* @a16 to <32 x i16>*), align 2
+; AVX512-NEXT:    [[TMP2:%.*]] = load <32 x i16>, <32 x i16>* bitcast ([32 x i16]* @b16 to <32 x i16>*), align 2
+; AVX512-NEXT:    [[TMP3:%.*]] = sub <32 x i16> [[TMP1]], [[TMP2]]
+; AVX512-NEXT:    store <32 x i16> [[TMP3]], <32 x i16>* bitcast ([32 x i16]* @c16 to <32 x i16>*), align 2
 ; AVX512-NEXT:    ret void
 ;
   %a0  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 0 ), align 2
@@ -429,24 +427,61 @@ define void @sub_v32i16() {
 }
 
 define void @sub_v64i8() {
-; CHECK-LABEL: @sub_v64i8(
-; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @a8 to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP5:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @b8 to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP6:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP7:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP8:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP9:%.*]] = sub <16 x i8> [[TMP1]], [[TMP5]]
-; CHECK-NEXT:    [[TMP10:%.*]] = sub <16 x i8> [[TMP2]], [[TMP6]]
-; CHECK-NEXT:    [[TMP11:%.*]] = sub <16 x i8> [[TMP3]], [[TMP7]]
-; CHECK-NEXT:    [[TMP12:%.*]] = sub <16 x i8> [[TMP4]], [[TMP8]]
-; CHECK-NEXT:    store <16 x i8> [[TMP9]], <16 x i8>* bitcast ([64 x i8]* @c8 to <16 x i8>*), align 1
-; CHECK-NEXT:    store <16 x i8> [[TMP10]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16) to <16 x i8>*), align 1
-; CHECK-NEXT:    store <16 x i8> [[TMP11]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32) to <16 x i8>*), align 1
-; CHECK-NEXT:    store <16 x i8> [[TMP12]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48) to <16 x i8>*), align 1
-; CHECK-NEXT:    ret void
+; SSE-LABEL: @sub_v64i8(
+; SSE-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @a8 to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP3:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP4:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP5:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @b8 to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP6:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP7:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP8:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP9:%.*]] = sub <16 x i8> [[TMP1]], [[TMP5]]
+; SSE-NEXT:    [[TMP10:%.*]] = sub <16 x i8> [[TMP2]], [[TMP6]]
+; SSE-NEXT:    [[TMP11:%.*]] = sub <16 x i8> [[TMP3]], [[TMP7]]
+; SSE-NEXT:    [[TMP12:%.*]] = sub <16 x i8> [[TMP4]], [[TMP8]]
+; SSE-NEXT:    store <16 x i8> [[TMP9]], <16 x i8>* bitcast ([64 x i8]* @c8 to <16 x i8>*), align 1
+; SSE-NEXT:    store <16 x i8> [[TMP10]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16) to <16 x i8>*), align 1
+; SSE-NEXT:    store <16 x i8> [[TMP11]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32) to <16 x i8>*), align 1
+; SSE-NEXT:    store <16 x i8> [[TMP12]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48) to <16 x i8>*), align 1
+; SSE-NEXT:    ret void
+;
+; SLM-LABEL: @sub_v64i8(
+; SLM-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @a8 to <16 x i8>*), align 1
+; SLM-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16) to <16 x i8>*), align 1
+; SLM-NEXT:    [[TMP3:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32) to <16 x i8>*), align 1
+; SLM-NEXT:    [[TMP4:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48) to <16 x i8>*), align 1
+; SLM-NEXT:    [[TMP5:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @b8 to <16 x i8>*), align 1
+; SLM-NEXT:    [[TMP6:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16) to <16 x i8>*), align 1
+; SLM-NEXT:    [[TMP7:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32) to <16 x i8>*), align 1
+; SLM-NEXT:    [[TMP8:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48) to <16 x i8>*), align 1
+; SLM-NEXT:    [[TMP9:%.*]] = sub <16 x i8> [[TMP1]], [[TMP5]]
+; SLM-NEXT:    [[TMP10:%.*]] = sub <16 x i8> [[TMP2]], [[TMP6]]
+; SLM-NEXT:    [[TMP11:%.*]] = sub <16 x i8> [[TMP3]], [[TMP7]]
+; SLM-NEXT:    [[TMP12:%.*]] = sub <16 x i8> [[TMP4]], [[TMP8]]
+; SLM-NEXT:    store <16 x i8> [[TMP9]], <16 x i8>* bitcast ([64 x i8]* @c8 to <16 x i8>*), align 1
+; SLM-NEXT:    store <16 x i8> [[TMP10]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16) to <16 x i8>*), align 1
+; SLM-NEXT:    store <16 x i8> [[TMP11]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32) to <16 x i8>*), align 1
+; SLM-NEXT:    store <16 x i8> [[TMP12]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48) to <16 x i8>*), align 1
+; SLM-NEXT:    ret void
+;
+; AVX-LABEL: @sub_v64i8(
+; AVX-NEXT:    [[TMP1:%.*]] = load <32 x i8>, <32 x i8>* bitcast ([64 x i8]* @a8 to <32 x i8>*), align 1
+; AVX-NEXT:    [[TMP2:%.*]] = load <32 x i8>, <32 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32) to <32 x i8>*), align 1
+; AVX-NEXT:    [[TMP3:%.*]] = load <32 x i8>, <32 x i8>* bitcast ([64 x i8]* @b8 to <32 x i8>*), align 1
+; AVX-NEXT:    [[TMP4:%.*]] = load <32 x i8>, <32 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32) to <32 x i8>*), align 1
+; AVX-NEXT:    [[TMP5:%.*]] = sub <32 x i8> [[TMP1]], [[TMP3]]
+; AVX-NEXT:    [[TMP6:%.*]] = sub <32 x i8> [[TMP2]], [[TMP4]]
+; AVX-NEXT:    store <32 x i8> [[TMP5]], <32 x i8>* bitcast ([64 x i8]* @c8 to <32 x i8>*), align 1
+; AVX-NEXT:    store <32 x i8> [[TMP6]], <32 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32) to <32 x i8>*), align 1
+; AVX-NEXT:    ret void
+;
+; AVX512-LABEL: @sub_v64i8(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <64 x i8>, <64 x i8>* bitcast ([64 x i8]* @a8 to <64 x i8>*), align 1
+; AVX512-NEXT:    [[TMP2:%.*]] = load <64 x i8>, <64 x i8>* bitcast ([64 x i8]* @b8 to <64 x i8>*), align 1
+; AVX512-NEXT:    [[TMP3:%.*]] = sub <64 x i8> [[TMP1]], [[TMP2]]
+; AVX512-NEXT:    store <64 x i8> [[TMP3]], <64 x i8>* bitcast ([64 x i8]* @c8 to <64 x i8>*), align 1
+; AVX512-NEXT:    ret void
 ;
   %a0  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 0 ), align 1
   %a1  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 1 ), align 1
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/bad-reduction.ll b/llvm/test/Transforms/SLPVectorizer/X86/bad-reduction.ll
new file mode 100644
index 000000000..c44a8524e
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/bad-reduction.ll
@@ -0,0 +1,358 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -slp-vectorizer -S -mtriple=x86_64-- -mattr=avx2 | FileCheck %s
+
+%v8i8 = type { i8, i8, i8, i8, i8, i8, i8, i8 }
+
+; https://bugs.llvm.org/show_bug.cgi?id=43146
+
+define i64 @load_bswap(%v8i8* %p) {
+; CHECK-LABEL: @load_bswap(
+; CHECK-NEXT:    [[G0:%.*]] = getelementptr inbounds [[V8I8:%.*]], %v8i8* [[P:%.*]], i64 0, i32 0
+; CHECK-NEXT:    [[G1:%.*]] = getelementptr inbounds [[V8I8]], %v8i8* [[P]], i64 0, i32 1
+; CHECK-NEXT:    [[G2:%.*]] = getelementptr inbounds [[V8I8]], %v8i8* [[P]], i64 0, i32 2
+; CHECK-NEXT:    [[G3:%.*]] = getelementptr inbounds [[V8I8]], %v8i8* [[P]], i64 0, i32 3
+; CHECK-NEXT:    [[G4:%.*]] = getelementptr inbounds [[V8I8]], %v8i8* [[P]], i64 0, i32 4
+; CHECK-NEXT:    [[G5:%.*]] = getelementptr inbounds [[V8I8]], %v8i8* [[P]], i64 0, i32 5
+; CHECK-NEXT:    [[G6:%.*]] = getelementptr inbounds [[V8I8]], %v8i8* [[P]], i64 0, i32 6
+; CHECK-NEXT:    [[G7:%.*]] = getelementptr inbounds [[V8I8]], %v8i8* [[P]], i64 0, i32 7
+; CHECK-NEXT:    [[T0:%.*]] = load i8, i8* [[G0]]
+; CHECK-NEXT:    [[T1:%.*]] = load i8, i8* [[G1]]
+; CHECK-NEXT:    [[T2:%.*]] = load i8, i8* [[G2]]
+; CHECK-NEXT:    [[T3:%.*]] = load i8, i8* [[G3]]
+; CHECK-NEXT:    [[T4:%.*]] = load i8, i8* [[G4]]
+; CHECK-NEXT:    [[T5:%.*]] = load i8, i8* [[G5]]
+; CHECK-NEXT:    [[T6:%.*]] = load i8, i8* [[G6]]
+; CHECK-NEXT:    [[T7:%.*]] = load i8, i8* [[G7]]
+; CHECK-NEXT:    [[Z0:%.*]] = zext i8 [[T0]] to i64
+; CHECK-NEXT:    [[Z1:%.*]] = zext i8 [[T1]] to i64
+; CHECK-NEXT:    [[Z2:%.*]] = zext i8 [[T2]] to i64
+; CHECK-NEXT:    [[Z3:%.*]] = zext i8 [[T3]] to i64
+; CHECK-NEXT:    [[Z4:%.*]] = zext i8 [[T4]] to i64
+; CHECK-NEXT:    [[Z5:%.*]] = zext i8 [[T5]] to i64
+; CHECK-NEXT:    [[Z6:%.*]] = zext i8 [[T6]] to i64
+; CHECK-NEXT:    [[Z7:%.*]] = zext i8 [[T7]] to i64
+; CHECK-NEXT:    [[SH0:%.*]] = shl nuw i64 [[Z0]], 56
+; CHECK-NEXT:    [[SH1:%.*]] = shl nuw nsw i64 [[Z1]], 48
+; CHECK-NEXT:    [[SH2:%.*]] = shl nuw nsw i64 [[Z2]], 40
+; CHECK-NEXT:    [[SH3:%.*]] = shl nuw nsw i64 [[Z3]], 32
+; CHECK-NEXT:    [[SH4:%.*]] = shl nuw nsw i64 [[Z4]], 24
+; CHECK-NEXT:    [[SH5:%.*]] = shl nuw nsw i64 [[Z5]], 16
+; CHECK-NEXT:    [[SH6:%.*]] = shl nuw nsw i64 [[Z6]], 8
+; CHECK-NEXT:    [[OR01:%.*]] = or i64 [[SH0]], [[SH1]]
+; CHECK-NEXT:    [[OR012:%.*]] = or i64 [[OR01]], [[SH2]]
+; CHECK-NEXT:    [[OR0123:%.*]] = or i64 [[OR012]], [[SH3]]
+; CHECK-NEXT:    [[OR01234:%.*]] = or i64 [[OR0123]], [[SH4]]
+; CHECK-NEXT:    [[OR012345:%.*]] = or i64 [[OR01234]], [[SH5]]
+; CHECK-NEXT:    [[OR0123456:%.*]] = or i64 [[OR012345]], [[SH6]]
+; CHECK-NEXT:    [[OR01234567:%.*]] = or i64 [[OR0123456]], [[Z7]]
+; CHECK-NEXT:    ret i64 [[OR01234567]]
+;
+  %g0 = getelementptr inbounds %v8i8, %v8i8* %p, i64 0, i32 0
+  %g1 = getelementptr inbounds %v8i8, %v8i8* %p, i64 0, i32 1
+  %g2 = getelementptr inbounds %v8i8, %v8i8* %p, i64 0, i32 2
+  %g3 = getelementptr inbounds %v8i8, %v8i8* %p, i64 0, i32 3
+  %g4 = getelementptr inbounds %v8i8, %v8i8* %p, i64 0, i32 4
+  %g5 = getelementptr inbounds %v8i8, %v8i8* %p, i64 0, i32 5
+  %g6 = getelementptr inbounds %v8i8, %v8i8* %p, i64 0, i32 6
+  %g7 = getelementptr inbounds %v8i8, %v8i8* %p, i64 0, i32 7
+
+  %t0 = load i8, i8* %g0
+  %t1 = load i8, i8* %g1
+  %t2 = load i8, i8* %g2
+  %t3 = load i8, i8* %g3
+  %t4 = load i8, i8* %g4
+  %t5 = load i8, i8* %g5
+  %t6 = load i8, i8* %g6
+  %t7 = load i8, i8* %g7
+
+  %z0 = zext i8 %t0 to i64
+  %z1 = zext i8 %t1 to i64
+  %z2 = zext i8 %t2 to i64
+  %z3 = zext i8 %t3 to i64
+  %z4 = zext i8 %t4 to i64
+  %z5 = zext i8 %t5 to i64
+  %z6 = zext i8 %t6 to i64
+  %z7 = zext i8 %t7 to i64
+
+  %sh0 = shl nuw i64 %z0, 56
+  %sh1 = shl nuw nsw i64 %z1, 48
+  %sh2 = shl nuw nsw i64 %z2, 40
+  %sh3 = shl nuw nsw i64 %z3, 32
+  %sh4 = shl nuw nsw i64 %z4, 24
+  %sh5 = shl nuw nsw i64 %z5, 16
+  %sh6 = shl nuw nsw i64 %z6, 8
+;  %sh7 = shl nuw nsw i64 %z7, 0 <-- missing phantom shift
+
+  %or01 = or i64 %sh0, %sh1
+  %or012 = or i64 %or01, %sh2
+  %or0123 = or i64 %or012, %sh3
+  %or01234 = or i64 %or0123, %sh4
+  %or012345 = or i64 %or01234, %sh5
+  %or0123456 = or i64 %or012345, %sh6
+  %or01234567 = or i64 %or0123456, %z7
+  ret i64 %or01234567
+}
+
+define i64 @load_bswap_nop_shift(%v8i8* %p) {
+; CHECK-LABEL: @load_bswap_nop_shift(
+; CHECK-NEXT:    [[G0:%.*]] = getelementptr inbounds [[V8I8:%.*]], %v8i8* [[P:%.*]], i64 0, i32 0
+; CHECK-NEXT:    [[G1:%.*]] = getelementptr inbounds [[V8I8]], %v8i8* [[P]], i64 0, i32 1
+; CHECK-NEXT:    [[G2:%.*]] = getelementptr inbounds [[V8I8]], %v8i8* [[P]], i64 0, i32 2
+; CHECK-NEXT:    [[G3:%.*]] = getelementptr inbounds [[V8I8]], %v8i8* [[P]], i64 0, i32 3
+; CHECK-NEXT:    [[G4:%.*]] = getelementptr inbounds [[V8I8]], %v8i8* [[P]], i64 0, i32 4
+; CHECK-NEXT:    [[G5:%.*]] = getelementptr inbounds [[V8I8]], %v8i8* [[P]], i64 0, i32 5
+; CHECK-NEXT:    [[G6:%.*]] = getelementptr inbounds [[V8I8]], %v8i8* [[P]], i64 0, i32 6
+; CHECK-NEXT:    [[G7:%.*]] = getelementptr inbounds [[V8I8]], %v8i8* [[P]], i64 0, i32 7
+; CHECK-NEXT:    [[T0:%.*]] = load i8, i8* [[G0]]
+; CHECK-NEXT:    [[T1:%.*]] = load i8, i8* [[G1]]
+; CHECK-NEXT:    [[T2:%.*]] = load i8, i8* [[G2]]
+; CHECK-NEXT:    [[T3:%.*]] = load i8, i8* [[G3]]
+; CHECK-NEXT:    [[T4:%.*]] = load i8, i8* [[G4]]
+; CHECK-NEXT:    [[T5:%.*]] = load i8, i8* [[G5]]
+; CHECK-NEXT:    [[T6:%.*]] = load i8, i8* [[G6]]
+; CHECK-NEXT:    [[T7:%.*]] = load i8, i8* [[G7]]
+; CHECK-NEXT:    [[Z0:%.*]] = zext i8 [[T0]] to i64
+; CHECK-NEXT:    [[Z1:%.*]] = zext i8 [[T1]] to i64
+; CHECK-NEXT:    [[Z2:%.*]] = zext i8 [[T2]] to i64
+; CHECK-NEXT:    [[Z3:%.*]] = zext i8 [[T3]] to i64
+; CHECK-NEXT:    [[Z4:%.*]] = zext i8 [[T4]] to i64
+; CHECK-NEXT:    [[Z5:%.*]] = zext i8 [[T5]] to i64
+; CHECK-NEXT:    [[Z6:%.*]] = zext i8 [[T6]] to i64
+; CHECK-NEXT:    [[Z7:%.*]] = zext i8 [[T7]] to i64
+; CHECK-NEXT:    [[SH0:%.*]] = shl nuw i64 [[Z0]], 56
+; CHECK-NEXT:    [[SH1:%.*]] = shl nuw nsw i64 [[Z1]], 48
+; CHECK-NEXT:    [[SH2:%.*]] = shl nuw nsw i64 [[Z2]], 40
+; CHECK-NEXT:    [[SH3:%.*]] = shl nuw nsw i64 [[Z3]], 32
+; CHECK-NEXT:    [[SH4:%.*]] = shl nuw nsw i64 [[Z4]], 24
+; CHECK-NEXT:    [[SH5:%.*]] = shl nuw nsw i64 [[Z5]], 16
+; CHECK-NEXT:    [[SH6:%.*]] = shl nuw nsw i64 [[Z6]], 8
+; CHECK-NEXT:    [[SH7:%.*]] = shl nuw nsw i64 [[Z7]], 0
+; CHECK-NEXT:    [[OR01:%.*]] = or i64 [[SH0]], [[SH1]]
+; CHECK-NEXT:    [[OR012:%.*]] = or i64 [[OR01]], [[SH2]]
+; CHECK-NEXT:    [[OR0123:%.*]] = or i64 [[OR012]], [[SH3]]
+; CHECK-NEXT:    [[OR01234:%.*]] = or i64 [[OR0123]], [[SH4]]
+; CHECK-NEXT:    [[OR012345:%.*]] = or i64 [[OR01234]], [[SH5]]
+; CHECK-NEXT:    [[OR0123456:%.*]] = or i64 [[OR012345]], [[SH6]]
+; CHECK-NEXT:    [[OR01234567:%.*]] = or i64 [[OR0123456]], [[SH7]]
+; CHECK-NEXT:    ret i64 [[OR01234567]]
+;
+  %g0 = getelementptr inbounds %v8i8, %v8i8* %p, i64 0, i32 0
+  %g1 = getelementptr inbounds %v8i8, %v8i8* %p, i64 0, i32 1
+  %g2 = getelementptr inbounds %v8i8, %v8i8* %p, i64 0, i32 2
+  %g3 = getelementptr inbounds %v8i8, %v8i8* %p, i64 0, i32 3
+  %g4 = getelementptr inbounds %v8i8, %v8i8* %p, i64 0, i32 4
+  %g5 = getelementptr inbounds %v8i8, %v8i8* %p, i64 0, i32 5
+  %g6 = getelementptr inbounds %v8i8, %v8i8* %p, i64 0, i32 6
+  %g7 = getelementptr inbounds %v8i8, %v8i8* %p, i64 0, i32 7
+
+  %t0 = load i8, i8* %g0
+  %t1 = load i8, i8* %g1
+  %t2 = load i8, i8* %g2
+  %t3 = load i8, i8* %g3
+  %t4 = load i8, i8* %g4
+  %t5 = load i8, i8* %g5
+  %t6 = load i8, i8* %g6
+  %t7 = load i8, i8* %g7
+
+  %z0 = zext i8 %t0 to i64
+  %z1 = zext i8 %t1 to i64
+  %z2 = zext i8 %t2 to i64
+  %z3 = zext i8 %t3 to i64
+  %z4 = zext i8 %t4 to i64
+  %z5 = zext i8 %t5 to i64
+  %z6 = zext i8 %t6 to i64
+  %z7 = zext i8 %t7 to i64
+
+  %sh0 = shl nuw i64 %z0, 56
+  %sh1 = shl nuw nsw i64 %z1, 48
+  %sh2 = shl nuw nsw i64 %z2, 40
+  %sh3 = shl nuw nsw i64 %z3, 32
+  %sh4 = shl nuw nsw i64 %z4, 24
+  %sh5 = shl nuw nsw i64 %z5, 16
+  %sh6 = shl nuw nsw i64 %z6, 8
+  %sh7 = shl nuw nsw i64 %z7, 0
+
+  %or01 = or i64 %sh0, %sh1
+  %or012 = or i64 %or01, %sh2
+  %or0123 = or i64 %or012, %sh3
+  %or01234 = or i64 %or0123, %sh4
+  %or012345 = or i64 %or01234, %sh5
+  %or0123456 = or i64 %or012345, %sh6
+  %or01234567 = or i64 %or0123456, %sh7
+  ret i64 %or01234567
+}
+
+; https://bugs.llvm.org/show_bug.cgi?id=42708
+
+define i64 @load64le(i8* %arg) {
+; CHECK-LABEL: @load64le(
+; CHECK-NEXT:    [[G1:%.*]] = getelementptr inbounds i8, i8* [[ARG:%.*]], i64 1
+; CHECK-NEXT:    [[G2:%.*]] = getelementptr inbounds i8, i8* [[ARG]], i64 2
+; CHECK-NEXT:    [[G3:%.*]] = getelementptr inbounds i8, i8* [[ARG]], i64 3
+; CHECK-NEXT:    [[G4:%.*]] = getelementptr inbounds i8, i8* [[ARG]], i64 4
+; CHECK-NEXT:    [[G5:%.*]] = getelementptr inbounds i8, i8* [[ARG]], i64 5
+; CHECK-NEXT:    [[G6:%.*]] = getelementptr inbounds i8, i8* [[ARG]], i64 6
+; CHECK-NEXT:    [[G7:%.*]] = getelementptr inbounds i8, i8* [[ARG]], i64 7
+; CHECK-NEXT:    [[LD0:%.*]] = load i8, i8* [[ARG]], align 1
+; CHECK-NEXT:    [[LD1:%.*]] = load i8, i8* [[G1]], align 1
+; CHECK-NEXT:    [[LD2:%.*]] = load i8, i8* [[G2]], align 1
+; CHECK-NEXT:    [[LD3:%.*]] = load i8, i8* [[G3]], align 1
+; CHECK-NEXT:    [[LD4:%.*]] = load i8, i8* [[G4]], align 1
+; CHECK-NEXT:    [[LD5:%.*]] = load i8, i8* [[G5]], align 1
+; CHECK-NEXT:    [[LD6:%.*]] = load i8, i8* [[G6]], align 1
+; CHECK-NEXT:    [[LD7:%.*]] = load i8, i8* [[G7]], align 1
+; CHECK-NEXT:    [[Z0:%.*]] = zext i8 [[LD0]] to i64
+; CHECK-NEXT:    [[Z1:%.*]] = zext i8 [[LD1]] to i64
+; CHECK-NEXT:    [[Z2:%.*]] = zext i8 [[LD2]] to i64
+; CHECK-NEXT:    [[Z3:%.*]] = zext i8 [[LD3]] to i64
+; CHECK-NEXT:    [[Z4:%.*]] = zext i8 [[LD4]] to i64
+; CHECK-NEXT:    [[Z5:%.*]] = zext i8 [[LD5]] to i64
+; CHECK-NEXT:    [[Z6:%.*]] = zext i8 [[LD6]] to i64
+; CHECK-NEXT:    [[Z7:%.*]] = zext i8 [[LD7]] to i64
+; CHECK-NEXT:    [[S1:%.*]] = shl nuw nsw i64 [[Z1]], 8
+; CHECK-NEXT:    [[S2:%.*]] = shl nuw nsw i64 [[Z2]], 16
+; CHECK-NEXT:    [[S3:%.*]] = shl nuw nsw i64 [[Z3]], 24
+; CHECK-NEXT:    [[S4:%.*]] = shl nuw nsw i64 [[Z4]], 32
+; CHECK-NEXT:    [[S5:%.*]] = shl nuw nsw i64 [[Z5]], 40
+; CHECK-NEXT:    [[S6:%.*]] = shl nuw nsw i64 [[Z6]], 48
+; CHECK-NEXT:    [[S7:%.*]] = shl nuw i64 [[Z7]], 56
+; CHECK-NEXT:    [[O1:%.*]] = or i64 [[S1]], [[Z0]]
+; CHECK-NEXT:    [[O2:%.*]] = or i64 [[O1]], [[S2]]
+; CHECK-NEXT:    [[O3:%.*]] = or i64 [[O2]], [[S3]]
+; CHECK-NEXT:    [[O4:%.*]] = or i64 [[O3]], [[S4]]
+; CHECK-NEXT:    [[O5:%.*]] = or i64 [[O4]], [[S5]]
+; CHECK-NEXT:    [[O6:%.*]] = or i64 [[O5]], [[S6]]
+; CHECK-NEXT:    [[O7:%.*]] = or i64 [[O6]], [[S7]]
+; CHECK-NEXT:    ret i64 [[O7]]
+;
+  %g1 = getelementptr inbounds i8, i8* %arg, i64 1
+  %g2 = getelementptr inbounds i8, i8* %arg, i64 2
+  %g3 = getelementptr inbounds i8, i8* %arg, i64 3
+  %g4 = getelementptr inbounds i8, i8* %arg, i64 4
+  %g5 = getelementptr inbounds i8, i8* %arg, i64 5
+  %g6 = getelementptr inbounds i8, i8* %arg, i64 6
+  %g7 = getelementptr inbounds i8, i8* %arg, i64 7
+
+  %ld0 = load i8, i8* %arg, align 1
+  %ld1 = load i8, i8* %g1, align 1
+  %ld2 = load i8, i8* %g2, align 1
+  %ld3 = load i8, i8* %g3, align 1
+  %ld4 = load i8, i8* %g4, align 1
+  %ld5 = load i8, i8* %g5, align 1
+  %ld6 = load i8, i8* %g6, align 1
+  %ld7 = load i8, i8* %g7, align 1
+
+  %z0 = zext i8 %ld0 to i64
+  %z1 = zext i8 %ld1 to i64
+  %z2 = zext i8 %ld2 to i64
+  %z3 = zext i8 %ld3 to i64
+  %z4 = zext i8 %ld4 to i64
+  %z5 = zext i8 %ld5 to i64
+  %z6 = zext i8 %ld6 to i64
+  %z7 = zext i8 %ld7 to i64
+
+;  %s0 = shl nuw nsw i64 %z0, 0 <-- missing phantom shift
+  %s1 = shl nuw nsw i64 %z1, 8
+  %s2 = shl nuw nsw i64 %z2, 16
+  %s3 = shl nuw nsw i64 %z3, 24
+  %s4 = shl nuw nsw i64 %z4, 32
+  %s5 = shl nuw nsw i64 %z5, 40
+  %s6 = shl nuw nsw i64 %z6, 48
+  %s7 = shl nuw i64 %z7, 56
+
+  %o1 = or i64 %s1, %z0
+  %o2 = or i64 %o1, %s2
+  %o3 = or i64 %o2, %s3
+  %o4 = or i64 %o3, %s4
+  %o5 = or i64 %o4, %s5
+  %o6 = or i64 %o5, %s6
+  %o7 = or i64 %o6, %s7
+  ret i64 %o7
+}
+
+define i64 @load64le_nop_shift(i8* %arg) {
+; CHECK-LABEL: @load64le_nop_shift(
+; CHECK-NEXT:    [[G1:%.*]] = getelementptr inbounds i8, i8* [[ARG:%.*]], i64 1
+; CHECK-NEXT:    [[G2:%.*]] = getelementptr inbounds i8, i8* [[ARG]], i64 2
+; CHECK-NEXT:    [[G3:%.*]] = getelementptr inbounds i8, i8* [[ARG]], i64 3
+; CHECK-NEXT:    [[G4:%.*]] = getelementptr inbounds i8, i8* [[ARG]], i64 4
+; CHECK-NEXT:    [[G5:%.*]] = getelementptr inbounds i8, i8* [[ARG]], i64 5
+; CHECK-NEXT:    [[G6:%.*]] = getelementptr inbounds i8, i8* [[ARG]], i64 6
+; CHECK-NEXT:    [[G7:%.*]] = getelementptr inbounds i8, i8* [[ARG]], i64 7
+; CHECK-NEXT:    [[LD0:%.*]] = load i8, i8* [[ARG]], align 1
+; CHECK-NEXT:    [[LD1:%.*]] = load i8, i8* [[G1]], align 1
+; CHECK-NEXT:    [[LD2:%.*]] = load i8, i8* [[G2]], align 1
+; CHECK-NEXT:    [[LD3:%.*]] = load i8, i8* [[G3]], align 1
+; CHECK-NEXT:    [[LD4:%.*]] = load i8, i8* [[G4]], align 1
+; CHECK-NEXT:    [[LD5:%.*]] = load i8, i8* [[G5]], align 1
+; CHECK-NEXT:    [[LD6:%.*]] = load i8, i8* [[G6]], align 1
+; CHECK-NEXT:    [[LD7:%.*]] = load i8, i8* [[G7]], align 1
+; CHECK-NEXT:    [[Z0:%.*]] = zext i8 [[LD0]] to i64
+; CHECK-NEXT:    [[Z1:%.*]] = zext i8 [[LD1]] to i64
+; CHECK-NEXT:    [[Z2:%.*]] = zext i8 [[LD2]] to i64
+; CHECK-NEXT:    [[Z3:%.*]] = zext i8 [[LD3]] to i64
+; CHECK-NEXT:    [[Z4:%.*]] = zext i8 [[LD4]] to i64
+; CHECK-NEXT:    [[Z5:%.*]] = zext i8 [[LD5]] to i64
+; CHECK-NEXT:    [[Z6:%.*]] = zext i8 [[LD6]] to i64
+; CHECK-NEXT:    [[Z7:%.*]] = zext i8 [[LD7]] to i64
+; CHECK-NEXT:    [[S0:%.*]] = shl nuw nsw i64 [[Z0]], 0
+; CHECK-NEXT:    [[S1:%.*]] = shl nuw nsw i64 [[Z1]], 8
+; CHECK-NEXT:    [[S2:%.*]] = shl nuw nsw i64 [[Z2]], 16
+; CHECK-NEXT:    [[S3:%.*]] = shl nuw nsw i64 [[Z3]], 24
+; CHECK-NEXT:    [[S4:%.*]] = shl nuw nsw i64 [[Z4]], 32
+; CHECK-NEXT:    [[S5:%.*]] = shl nuw nsw i64 [[Z5]], 40
+; CHECK-NEXT:    [[S6:%.*]] = shl nuw nsw i64 [[Z6]], 48
+; CHECK-NEXT:    [[S7:%.*]] = shl nuw i64 [[Z7]], 56
+; CHECK-NEXT:    [[O1:%.*]] = or i64 [[S1]], [[S0]]
+; CHECK-NEXT:    [[O2:%.*]] = or i64 [[O1]], [[S2]]
+; CHECK-NEXT:    [[O3:%.*]] = or i64 [[O2]], [[S3]]
+; CHECK-NEXT:    [[O4:%.*]] = or i64 [[O3]], [[S4]]
+; CHECK-NEXT:    [[O5:%.*]] = or i64 [[O4]], [[S5]]
+; CHECK-NEXT:    [[O6:%.*]] = or i64 [[O5]], [[S6]]
+; CHECK-NEXT:    [[O7:%.*]] = or i64 [[O6]], [[S7]]
+; CHECK-NEXT:    ret i64 [[O7]]
+;
+  %g1 = getelementptr inbounds i8, i8* %arg, i64 1
+  %g2 = getelementptr inbounds i8, i8* %arg, i64 2
+  %g3 = getelementptr inbounds i8, i8* %arg, i64 3
+  %g4 = getelementptr inbounds i8, i8* %arg, i64 4
+  %g5 = getelementptr inbounds i8, i8* %arg, i64 5
+  %g6 = getelementptr inbounds i8, i8* %arg, i64 6
+  %g7 = getelementptr inbounds i8, i8* %arg, i64 7
+
+  %ld0 = load i8, i8* %arg, align 1
+  %ld1 = load i8, i8* %g1, align 1
+  %ld2 = load i8, i8* %g2, align 1
+  %ld3 = load i8, i8* %g3, align 1
+  %ld4 = load i8, i8* %g4, align 1
+  %ld5 = load i8, i8* %g5, align 1
+  %ld6 = load i8, i8* %g6, align 1
+  %ld7 = load i8, i8* %g7, align 1
+
+  %z0 = zext i8 %ld0 to i64
+  %z1 = zext i8 %ld1 to i64
+  %z2 = zext i8 %ld2 to i64
+  %z3 = zext i8 %ld3 to i64
+  %z4 = zext i8 %ld4 to i64
+  %z5 = zext i8 %ld5 to i64
+  %z6 = zext i8 %ld6 to i64
+  %z7 = zext i8 %ld7 to i64
+
+  %s0 = shl nuw nsw i64 %z0, 0
+  %s1 = shl nuw nsw i64 %z1, 8
+  %s2 = shl nuw nsw i64 %z2, 16
+  %s3 = shl nuw nsw i64 %z3, 24
+  %s4 = shl nuw nsw i64 %z4, 32
+  %s5 = shl nuw nsw i64 %z5, 40
+  %s6 = shl nuw nsw i64 %z6, 48
+  %s7 = shl nuw i64 %z7, 56
+
+  %o1 = or i64 %s1, %s0
+  %o2 = or i64 %o1, %s2
+  %o3 = or i64 %o2, %s3
+  %o4 = or i64 %o3, %s4
+  %o5 = or i64 %o4, %s5
+  %o6 = or i64 %o5, %s6
+  %o7 = or i64 %o6, %s7
+  ret i64 %o7
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/bitreverse.ll b/llvm/test/Transforms/SLPVectorizer/X86/bitreverse.ll
index 749e93b04..58b173202 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/bitreverse.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/bitreverse.ll
@@ -311,14 +311,26 @@ define void @bitreverse_16i8() #0 {
 }
 
 define void @bitreverse_32i8() #0 {
-; CHECK-LABEL: @bitreverse_32i8(
-; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([32 x i8]* @src8 to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 16) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP3:%.*]] = call <16 x i8> @llvm.bitreverse.v16i8(<16 x i8> [[TMP1]])
-; CHECK-NEXT:    [[TMP4:%.*]] = call <16 x i8> @llvm.bitreverse.v16i8(<16 x i8> [[TMP2]])
-; CHECK-NEXT:    store <16 x i8> [[TMP3]], <16 x i8>* bitcast ([32 x i8]* @dst8 to <16 x i8>*), align 1
-; CHECK-NEXT:    store <16 x i8> [[TMP4]], <16 x i8>* bitcast (i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 16) to <16 x i8>*), align 1
-; CHECK-NEXT:    ret void
+; SSE-LABEL: @bitreverse_32i8(
+; SSE-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([32 x i8]* @src8 to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 16) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP3:%.*]] = call <16 x i8> @llvm.bitreverse.v16i8(<16 x i8> [[TMP1]])
+; SSE-NEXT:    [[TMP4:%.*]] = call <16 x i8> @llvm.bitreverse.v16i8(<16 x i8> [[TMP2]])
+; SSE-NEXT:    store <16 x i8> [[TMP3]], <16 x i8>* bitcast ([32 x i8]* @dst8 to <16 x i8>*), align 1
+; SSE-NEXT:    store <16 x i8> [[TMP4]], <16 x i8>* bitcast (i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 16) to <16 x i8>*), align 1
+; SSE-NEXT:    ret void
+;
+; AVX-LABEL: @bitreverse_32i8(
+; AVX-NEXT:    [[TMP1:%.*]] = load <32 x i8>, <32 x i8>* bitcast ([32 x i8]* @src8 to <32 x i8>*), align 1
+; AVX-NEXT:    [[TMP2:%.*]] = call <32 x i8> @llvm.bitreverse.v32i8(<32 x i8> [[TMP1]])
+; AVX-NEXT:    store <32 x i8> [[TMP2]], <32 x i8>* bitcast ([32 x i8]* @dst8 to <32 x i8>*), align 1
+; AVX-NEXT:    ret void
+;
+; XOP-LABEL: @bitreverse_32i8(
+; XOP-NEXT:    [[TMP1:%.*]] = load <32 x i8>, <32 x i8>* bitcast ([32 x i8]* @src8 to <32 x i8>*), align 1
+; XOP-NEXT:    [[TMP2:%.*]] = call <32 x i8> @llvm.bitreverse.v32i8(<32 x i8> [[TMP1]])
+; XOP-NEXT:    store <32 x i8> [[TMP2]], <32 x i8>* bitcast ([32 x i8]* @dst8 to <32 x i8>*), align 1
+; XOP-NEXT:    ret void
 ;
   %ld0  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  0), align 1
   %ld1  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  1), align 1
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/blending-shuffle.ll b/llvm/test/Transforms/SLPVectorizer/X86/blending-shuffle.ll
index 9189fb4d1..776f32dee 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/blending-shuffle.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/blending-shuffle.ll
@@ -22,9 +22,19 @@ define <2 x i8> @g(<2 x i8> %x, <2 x i8> %y) {
 
 define <4 x i8> @h(<4 x i8> %x, <4 x i8> %y) {
 ; CHECK-LABEL: @h(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i8> [[X:%.*]], <4 x i8> [[Y:%.*]], <4 x i32> <i32 0, i32 3, i32 5, i32 6>
-; CHECK-NEXT:    [[TMP2:%.*]] = mul <4 x i8> [[TMP1]], [[TMP1]]
-; CHECK-NEXT:    ret <4 x i8> [[TMP2]]
+; CHECK-NEXT:    [[X0:%.*]] = extractelement <4 x i8> [[X:%.*]], i32 0
+; CHECK-NEXT:    [[X3:%.*]] = extractelement <4 x i8> [[X]], i32 3
+; CHECK-NEXT:    [[Y1:%.*]] = extractelement <4 x i8> [[Y:%.*]], i32 1
+; CHECK-NEXT:    [[Y2:%.*]] = extractelement <4 x i8> [[Y]], i32 2
+; CHECK-NEXT:    [[X0X0:%.*]] = mul i8 [[X0]], [[X0]]
+; CHECK-NEXT:    [[X3X3:%.*]] = mul i8 [[X3]], [[X3]]
+; CHECK-NEXT:    [[Y1Y1:%.*]] = mul i8 [[Y1]], [[Y1]]
+; CHECK-NEXT:    [[Y2Y2:%.*]] = mul i8 [[Y2]], [[Y2]]
+; CHECK-NEXT:    [[INS1:%.*]] = insertelement <4 x i8> undef, i8 [[X0X0]], i32 0
+; CHECK-NEXT:    [[INS2:%.*]] = insertelement <4 x i8> [[INS1]], i8 [[X3X3]], i32 1
+; CHECK-NEXT:    [[INS3:%.*]] = insertelement <4 x i8> [[INS2]], i8 [[Y1Y1]], i32 2
+; CHECK-NEXT:    [[INS4:%.*]] = insertelement <4 x i8> [[INS3]], i8 [[Y2Y2]], i32 3
+; CHECK-NEXT:    ret <4 x i8> [[INS4]]
 ;
   %x0 = extractelement <4 x i8> %x, i32 0
   %x3 = extractelement <4 x i8> %x, i32 3
@@ -43,9 +53,16 @@ define <4 x i8> @h(<4 x i8> %x, <4 x i8> %y) {
 
 define <4 x i8> @h_undef(<4 x i8> %x, <4 x i8> %y) {
 ; CHECK-LABEL: @h_undef(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i8> [[X:%.*]], <4 x i8> [[Y:%.*]], <4 x i32> <i32 undef, i32 3, i32 5, i32 6>
-; CHECK-NEXT:    [[TMP2:%.*]] = mul <4 x i8> [[TMP1]], [[TMP1]]
-; CHECK-NEXT:    ret <4 x i8> [[TMP2]]
+; CHECK-NEXT:    [[X3:%.*]] = extractelement <4 x i8> [[X:%.*]], i32 3
+; CHECK-NEXT:    [[Y1:%.*]] = extractelement <4 x i8> [[Y:%.*]], i32 1
+; CHECK-NEXT:    [[Y2:%.*]] = extractelement <4 x i8> [[Y]], i32 2
+; CHECK-NEXT:    [[X3X3:%.*]] = mul i8 [[X3]], [[X3]]
+; CHECK-NEXT:    [[Y1Y1:%.*]] = mul i8 [[Y1]], [[Y1]]
+; CHECK-NEXT:    [[Y2Y2:%.*]] = mul i8 [[Y2]], [[Y2]]
+; CHECK-NEXT:    [[INS2:%.*]] = insertelement <4 x i8> undef, i8 [[X3X3]], i32 1
+; CHECK-NEXT:    [[INS3:%.*]] = insertelement <4 x i8> [[INS2]], i8 [[Y1Y1]], i32 2
+; CHECK-NEXT:    [[INS4:%.*]] = insertelement <4 x i8> [[INS3]], i8 [[Y2Y2]], i32 3
+; CHECK-NEXT:    ret <4 x i8> [[INS4]]
 ;
   %x0 = extractelement <4 x i8> undef, i32 0
   %x3 = extractelement <4 x i8> %x, i32 3
@@ -64,13 +81,17 @@ define <4 x i8> @h_undef(<4 x i8> %x, <4 x i8> %y) {
 
 define i8 @i(<4 x i8> %x, <4 x i8> %y) {
 ; CHECK-LABEL: @i(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i8> [[X:%.*]], <4 x i8> [[Y:%.*]], <4 x i32> <i32 0, i32 3, i32 5, i32 6>
-; CHECK-NEXT:    [[TMP2:%.*]] = mul <4 x i8> [[TMP1]], [[TMP1]]
-; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i8> [[TMP2]], <4 x i8> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
-; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <4 x i8> [[TMP2]], [[RDX_SHUF]]
-; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x i8> [[BIN_RDX]], <4 x i8> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[BIN_RDX2:%.*]] = add <4 x i8> [[BIN_RDX]], [[RDX_SHUF1]]
-; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x i8> [[BIN_RDX2]], i32 0
+; CHECK-NEXT:    [[X0:%.*]] = extractelement <4 x i8> [[X:%.*]], i32 0
+; CHECK-NEXT:    [[X3:%.*]] = extractelement <4 x i8> [[X]], i32 3
+; CHECK-NEXT:    [[Y1:%.*]] = extractelement <4 x i8> [[Y:%.*]], i32 1
+; CHECK-NEXT:    [[Y2:%.*]] = extractelement <4 x i8> [[Y]], i32 2
+; CHECK-NEXT:    [[X0X0:%.*]] = mul i8 [[X0]], [[X0]]
+; CHECK-NEXT:    [[X3X3:%.*]] = mul i8 [[X3]], [[X3]]
+; CHECK-NEXT:    [[Y1Y1:%.*]] = mul i8 [[Y1]], [[Y1]]
+; CHECK-NEXT:    [[Y2Y2:%.*]] = mul i8 [[Y2]], [[Y2]]
+; CHECK-NEXT:    [[TMP1:%.*]] = add i8 [[X0X0]], [[X3X3]]
+; CHECK-NEXT:    [[TMP2:%.*]] = add i8 [[Y1Y1]], [[Y2Y2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = add i8 [[TMP1]], [[TMP2]]
 ; CHECK-NEXT:    ret i8 [[TMP3]]
 ;
   %x0 = extractelement <4 x i8> %x, i32 0
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/call.ll b/llvm/test/Transforms/SLPVectorizer/X86/call.ll
index c93397c03..093389752 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/call.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/call.ll
@@ -173,5 +173,5 @@ define void @round_custom(i64* %a, i64* %b) {
 ; CHECK: declare <2 x double> @llvm.pow.v2f64(<2 x double>, <2 x double>) [[ATTR0]]
 ; CHECK: declare <2 x double> @llvm.exp2.v2f64(<2 x double>) [[ATTR0]]
 
-; CHECK: attributes [[ATTR0]] = { nounwind readnone speculatable }
+; CHECK: attributes [[ATTR0]] = { nounwind readnone speculatable willreturn }
 
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/cast.ll b/llvm/test/Transforms/SLPVectorizer/X86/cast.ll
index 2f9f84948..5d023008e 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/cast.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/cast.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 -basicaa -slp-vectorizer -dce -S | FileCheck %s
-; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx -basicaa -slp-vectorizer -dce -S | FileCheck %s
+; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 -basicaa -slp-vectorizer -dce -S | FileCheck %s --check-prefixes=CHECK,SSE42
+; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx -basicaa -slp-vectorizer -dce -S | FileCheck %s --check-prefixes=CHECK,AVX
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
@@ -76,21 +76,30 @@ entry:
 }
 
 define i64 @test_sext_4i16_to_4i64(i64* noalias nocapture %A, i16* noalias nocapture %B) {
-; CHECK-LABEL: @test_sext_4i16_to_4i64(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i16* [[B:%.*]] to <2 x i16>*
-; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i16>, <2 x i16>* [[TMP0]], align 1
-; CHECK-NEXT:    [[TMP2:%.*]] = sext <2 x i16> [[TMP1]] to <2 x i64>
-; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i64* [[A:%.*]] to <2 x i64>*
-; CHECK-NEXT:    store <2 x i64> [[TMP2]], <2 x i64>* [[TMP3]], align 4
-; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds i16, i16* [[B]], i64 2
-; CHECK-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds i64, i64* [[A]], i64 2
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i16* [[ARRAYIDX5]] to <2 x i16>*
-; CHECK-NEXT:    [[TMP5:%.*]] = load <2 x i16>, <2 x i16>* [[TMP4]], align 1
-; CHECK-NEXT:    [[TMP6:%.*]] = sext <2 x i16> [[TMP5]] to <2 x i64>
-; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i64* [[ARRAYIDX7]] to <2 x i64>*
-; CHECK-NEXT:    store <2 x i64> [[TMP6]], <2 x i64>* [[TMP7]], align 4
-; CHECK-NEXT:    ret i64 undef
+; SSE42-LABEL: @test_sext_4i16_to_4i64(
+; SSE42-NEXT:  entry:
+; SSE42-NEXT:    [[TMP0:%.*]] = bitcast i16* [[B:%.*]] to <2 x i16>*
+; SSE42-NEXT:    [[TMP1:%.*]] = load <2 x i16>, <2 x i16>* [[TMP0]], align 1
+; SSE42-NEXT:    [[TMP2:%.*]] = sext <2 x i16> [[TMP1]] to <2 x i64>
+; SSE42-NEXT:    [[TMP3:%.*]] = bitcast i64* [[A:%.*]] to <2 x i64>*
+; SSE42-NEXT:    store <2 x i64> [[TMP2]], <2 x i64>* [[TMP3]], align 4
+; SSE42-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds i16, i16* [[B]], i64 2
+; SSE42-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds i64, i64* [[A]], i64 2
+; SSE42-NEXT:    [[TMP4:%.*]] = bitcast i16* [[ARRAYIDX5]] to <2 x i16>*
+; SSE42-NEXT:    [[TMP5:%.*]] = load <2 x i16>, <2 x i16>* [[TMP4]], align 1
+; SSE42-NEXT:    [[TMP6:%.*]] = sext <2 x i16> [[TMP5]] to <2 x i64>
+; SSE42-NEXT:    [[TMP7:%.*]] = bitcast i64* [[ARRAYIDX7]] to <2 x i64>*
+; SSE42-NEXT:    store <2 x i64> [[TMP6]], <2 x i64>* [[TMP7]], align 4
+; SSE42-NEXT:    ret i64 undef
+;
+; AVX-LABEL: @test_sext_4i16_to_4i64(
+; AVX-NEXT:  entry:
+; AVX-NEXT:    [[TMP0:%.*]] = bitcast i16* [[B:%.*]] to <4 x i16>*
+; AVX-NEXT:    [[TMP1:%.*]] = load <4 x i16>, <4 x i16>* [[TMP0]], align 1
+; AVX-NEXT:    [[TMP2:%.*]] = sext <4 x i16> [[TMP1]] to <4 x i64>
+; AVX-NEXT:    [[TMP3:%.*]] = bitcast i64* [[A:%.*]] to <4 x i64>*
+; AVX-NEXT:    store <4 x i64> [[TMP2]], <4 x i64>* [[TMP3]], align 4
+; AVX-NEXT:    ret i64 undef
 ;
 entry:
   %0 = load i16, i16* %B, align 1
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/consecutive-access.ll b/llvm/test/Transforms/SLPVectorizer/X86/consecutive-access.ll
index f394dc743..42c95805d 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/consecutive-access.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/consecutive-access.ll
@@ -549,7 +549,39 @@ for.body:                                         ; preds = %entry, %for.body
   br i1 %cmp, label %for.body, label %for.cond.cleanup
 }
 
-attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+; Globals/constant expressions are not normal constants.
+; They should not be treated as the usual vectorization candidates.
+
+@g1 = external global i32, align 4
+@g2 = external global i32, align 4
+
+define void @PR33958(i32** nocapture %p) {
+; CHECK-LABEL: @PR33958(
+; CHECK-NEXT:    store i32* @g1, i32** [[P:%.*]], align 8
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i32*, i32** [[P]], i64 1
+; CHECK-NEXT:    store i32* @g2, i32** [[ARRAYIDX1]], align 8
+; CHECK-NEXT:    ret void
+;
+  store i32* @g1, i32** %p, align 8
+  %arrayidx1 = getelementptr inbounds i32*, i32** %p, i64 1
+  store i32* @g2, i32** %arrayidx1, align 8
+  ret void
+}
+
+define void @store_constant_expression(i64* %p) {
+; CHECK-LABEL: @store_constant_expression(
+; CHECK-NEXT:    store i64 ptrtoint (i32* @g1 to i64), i64* [[P:%.*]], align 8
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i64, i64* [[P]], i64 1
+; CHECK-NEXT:    store i64 ptrtoint (i32* @g2 to i64), i64* [[ARRAYIDX1]], align 8
+; CHECK-NEXT:    ret void
+;
+  store i64 ptrtoint (i32* @g1 to i64), i64* %p, align 8
+  %arrayidx1 = getelementptr inbounds i64, i64* %p, i64 1
+  store i64 ptrtoint (i32* @g2 to i64), i64* %arrayidx1, align 8
+  ret void
+}
+
+attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.ident = !{!0}
 
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/crash_bullet3.ll b/llvm/test/Transforms/SLPVectorizer/X86/crash_bullet3.ll
index 7ec4fc185..0ee8943f5 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/crash_bullet3.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/crash_bullet3.ll
@@ -138,4 +138,4 @@ if.end22.2:                                       ; preds = %if.then17.2, %if.en
   br i1 undef, label %for.end36, label %for.body
 }
 
-attributes #0 = { ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { ssp uwtable "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/crash_flop7.ll b/llvm/test/Transforms/SLPVectorizer/X86/crash_flop7.ll
index d149c27b7..57dd780ec 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/crash_flop7.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/crash_flop7.ll
@@ -74,4 +74,4 @@ for.end300:                                       ; preds = %for.body267, %for.e
   unreachable
 }
 
-attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/crash_gep.ll b/llvm/test/Transforms/SLPVectorizer/X86/crash_gep.ll
index eca21e7d3..8fdc72396 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/crash_gep.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/crash_gep.ll
@@ -29,3 +29,26 @@ entry:
   store i64 %2, i64* %add.ptr, align 8
   ret i32 undef
 }
+
+define void @PR43799() {
+; CHECK-LABEL: @PR43799(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[BODY:%.*]]
+; CHECK:       body:
+; CHECK-NEXT:    br label [[BODY]]
+; CHECK:       epilog:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %body
+
+body:
+  %p.1.i19 = phi i8* [ undef, %entry ], [ %incdec.ptr.i.7, %body ]
+  %lsr.iv17 = phi i8* [ undef, %entry ], [ %scevgep113.7, %body ]
+  %incdec.ptr.i.7 = getelementptr inbounds i8, i8* undef, i32 1
+  %scevgep113.7 = getelementptr i8, i8* undef, i64 1
+  br label %body
+
+epilog:
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/crash_smallpt.ll b/llvm/test/Transforms/SLPVectorizer/X86/crash_smallpt.ll
index 5c753091f..b5dad94c6 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/crash_smallpt.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/crash_smallpt.ll
@@ -163,4 +163,4 @@ return:                                           ; preds = %if.then78, %if.then
   ret void
 }
 
-attributes #0 = { ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { ssp uwtable "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/ctlz.ll b/llvm/test/Transforms/SLPVectorizer/X86/ctlz.ll
index cd5a358fc..562d5d916 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/ctlz.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/ctlz.ll
@@ -75,20 +75,47 @@ define void @ctlz_4i64() #0 {
 }
 
 define void @ctlz_4i32() #0 {
-; CHECK-LABEL: @ctlz_4i32(
-; CHECK-NEXT:    [[LD0:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 0), align 4
-; CHECK-NEXT:    [[LD1:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 1), align 4
-; CHECK-NEXT:    [[LD2:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 2), align 4
-; CHECK-NEXT:    [[LD3:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 3), align 4
-; CHECK-NEXT:    [[CTLZ0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD0]], i1 false)
-; CHECK-NEXT:    [[CTLZ1:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD1]], i1 false)
-; CHECK-NEXT:    [[CTLZ2:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD2]], i1 false)
-; CHECK-NEXT:    [[CTLZ3:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD3]], i1 false)
-; CHECK-NEXT:    store i32 [[CTLZ0]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 0), align 4
-; CHECK-NEXT:    store i32 [[CTLZ1]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 1), align 4
-; CHECK-NEXT:    store i32 [[CTLZ2]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 2), align 4
-; CHECK-NEXT:    store i32 [[CTLZ3]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 3), align 4
-; CHECK-NEXT:    ret void
+; SSE2-LABEL: @ctlz_4i32(
+; SSE2-NEXT:    [[LD0:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 0), align 4
+; SSE2-NEXT:    [[LD1:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 1), align 4
+; SSE2-NEXT:    [[LD2:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 2), align 4
+; SSE2-NEXT:    [[LD3:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 3), align 4
+; SSE2-NEXT:    [[CTLZ0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD0]], i1 false)
+; SSE2-NEXT:    [[CTLZ1:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD1]], i1 false)
+; SSE2-NEXT:    [[CTLZ2:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD2]], i1 false)
+; SSE2-NEXT:    [[CTLZ3:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD3]], i1 false)
+; SSE2-NEXT:    store i32 [[CTLZ0]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 0), align 4
+; SSE2-NEXT:    store i32 [[CTLZ1]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 1), align 4
+; SSE2-NEXT:    store i32 [[CTLZ2]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 2), align 4
+; SSE2-NEXT:    store i32 [[CTLZ3]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 3), align 4
+; SSE2-NEXT:    ret void
+;
+; SSE42-LABEL: @ctlz_4i32(
+; SSE42-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([8 x i32]* @src32 to <4 x i32>*), align 4
+; SSE42-NEXT:    [[TMP2:%.*]] = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> [[TMP1]], i1 false)
+; SSE42-NEXT:    store <4 x i32> [[TMP2]], <4 x i32>* bitcast ([8 x i32]* @dst32 to <4 x i32>*), align 4
+; SSE42-NEXT:    ret void
+;
+; AVX1-LABEL: @ctlz_4i32(
+; AVX1-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([8 x i32]* @src32 to <4 x i32>*), align 4
+; AVX1-NEXT:    [[TMP2:%.*]] = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> [[TMP1]], i1 false)
+; AVX1-NEXT:    store <4 x i32> [[TMP2]], <4 x i32>* bitcast ([8 x i32]* @dst32 to <4 x i32>*), align 4
+; AVX1-NEXT:    ret void
+;
+; AVX2-LABEL: @ctlz_4i32(
+; AVX2-NEXT:    [[LD0:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 0), align 4
+; AVX2-NEXT:    [[LD1:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 1), align 4
+; AVX2-NEXT:    [[LD2:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 2), align 4
+; AVX2-NEXT:    [[LD3:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 3), align 4
+; AVX2-NEXT:    [[CTLZ0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD0]], i1 false)
+; AVX2-NEXT:    [[CTLZ1:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD1]], i1 false)
+; AVX2-NEXT:    [[CTLZ2:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD2]], i1 false)
+; AVX2-NEXT:    [[CTLZ3:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD3]], i1 false)
+; AVX2-NEXT:    store i32 [[CTLZ0]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 0), align 4
+; AVX2-NEXT:    store i32 [[CTLZ1]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 1), align 4
+; AVX2-NEXT:    store i32 [[CTLZ2]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 2), align 4
+; AVX2-NEXT:    store i32 [[CTLZ3]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 3), align 4
+; AVX2-NEXT:    ret void
 ;
   %ld0 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 0), align 4
   %ld1 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 1), align 4
@@ -106,65 +133,47 @@ define void @ctlz_4i32() #0 {
 }
 
 define void @ctlz_8i32() #0 {
-; SSE-LABEL: @ctlz_8i32(
-; SSE-NEXT:    [[LD0:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 0), align 2
-; SSE-NEXT:    [[LD1:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 1), align 2
-; SSE-NEXT:    [[LD2:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 2), align 2
-; SSE-NEXT:    [[LD3:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 3), align 2
-; SSE-NEXT:    [[LD4:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 4), align 2
-; SSE-NEXT:    [[LD5:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 5), align 2
-; SSE-NEXT:    [[LD6:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 6), align 2
-; SSE-NEXT:    [[LD7:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 7), align 2
-; SSE-NEXT:    [[CTLZ0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD0]], i1 false)
-; SSE-NEXT:    [[CTLZ1:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD1]], i1 false)
-; SSE-NEXT:    [[CTLZ2:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD2]], i1 false)
-; SSE-NEXT:    [[CTLZ3:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD3]], i1 false)
-; SSE-NEXT:    [[CTLZ4:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD4]], i1 false)
-; SSE-NEXT:    [[CTLZ5:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD5]], i1 false)
-; SSE-NEXT:    [[CTLZ6:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD6]], i1 false)
-; SSE-NEXT:    [[CTLZ7:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD7]], i1 false)
-; SSE-NEXT:    store i32 [[CTLZ0]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 0), align 2
-; SSE-NEXT:    store i32 [[CTLZ1]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 1), align 2
-; SSE-NEXT:    store i32 [[CTLZ2]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 2), align 2
-; SSE-NEXT:    store i32 [[CTLZ3]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 3), align 2
-; SSE-NEXT:    store i32 [[CTLZ4]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 4), align 2
-; SSE-NEXT:    store i32 [[CTLZ5]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 5), align 2
-; SSE-NEXT:    store i32 [[CTLZ6]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 6), align 2
-; SSE-NEXT:    store i32 [[CTLZ7]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 7), align 2
-; SSE-NEXT:    ret void
+; SSE2-LABEL: @ctlz_8i32(
+; SSE2-NEXT:    [[LD0:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 0), align 2
+; SSE2-NEXT:    [[LD1:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 1), align 2
+; SSE2-NEXT:    [[LD2:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 2), align 2
+; SSE2-NEXT:    [[LD3:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 3), align 2
+; SSE2-NEXT:    [[LD4:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 4), align 2
+; SSE2-NEXT:    [[LD5:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 5), align 2
+; SSE2-NEXT:    [[LD6:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 6), align 2
+; SSE2-NEXT:    [[LD7:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 7), align 2
+; SSE2-NEXT:    [[CTLZ0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD0]], i1 false)
+; SSE2-NEXT:    [[CTLZ1:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD1]], i1 false)
+; SSE2-NEXT:    [[CTLZ2:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD2]], i1 false)
+; SSE2-NEXT:    [[CTLZ3:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD3]], i1 false)
+; SSE2-NEXT:    [[CTLZ4:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD4]], i1 false)
+; SSE2-NEXT:    [[CTLZ5:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD5]], i1 false)
+; SSE2-NEXT:    [[CTLZ6:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD6]], i1 false)
+; SSE2-NEXT:    [[CTLZ7:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD7]], i1 false)
+; SSE2-NEXT:    store i32 [[CTLZ0]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 0), align 2
+; SSE2-NEXT:    store i32 [[CTLZ1]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 1), align 2
+; SSE2-NEXT:    store i32 [[CTLZ2]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 2), align 2
+; SSE2-NEXT:    store i32 [[CTLZ3]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 3), align 2
+; SSE2-NEXT:    store i32 [[CTLZ4]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 4), align 2
+; SSE2-NEXT:    store i32 [[CTLZ5]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 5), align 2
+; SSE2-NEXT:    store i32 [[CTLZ6]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 6), align 2
+; SSE2-NEXT:    store i32 [[CTLZ7]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 7), align 2
+; SSE2-NEXT:    ret void
 ;
-; AVX1-LABEL: @ctlz_8i32(
-; AVX1-NEXT:    [[LD0:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 0), align 2
-; AVX1-NEXT:    [[LD1:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 1), align 2
-; AVX1-NEXT:    [[LD2:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 2), align 2
-; AVX1-NEXT:    [[LD3:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 3), align 2
-; AVX1-NEXT:    [[LD4:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 4), align 2
-; AVX1-NEXT:    [[LD5:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 5), align 2
-; AVX1-NEXT:    [[LD6:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 6), align 2
-; AVX1-NEXT:    [[LD7:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 7), align 2
-; AVX1-NEXT:    [[CTLZ0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD0]], i1 false)
-; AVX1-NEXT:    [[CTLZ1:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD1]], i1 false)
-; AVX1-NEXT:    [[CTLZ2:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD2]], i1 false)
-; AVX1-NEXT:    [[CTLZ3:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD3]], i1 false)
-; AVX1-NEXT:    [[CTLZ4:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD4]], i1 false)
-; AVX1-NEXT:    [[CTLZ5:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD5]], i1 false)
-; AVX1-NEXT:    [[CTLZ6:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD6]], i1 false)
-; AVX1-NEXT:    [[CTLZ7:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD7]], i1 false)
-; AVX1-NEXT:    store i32 [[CTLZ0]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 0), align 2
-; AVX1-NEXT:    store i32 [[CTLZ1]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 1), align 2
-; AVX1-NEXT:    store i32 [[CTLZ2]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 2), align 2
-; AVX1-NEXT:    store i32 [[CTLZ3]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 3), align 2
-; AVX1-NEXT:    store i32 [[CTLZ4]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 4), align 2
-; AVX1-NEXT:    store i32 [[CTLZ5]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 5), align 2
-; AVX1-NEXT:    store i32 [[CTLZ6]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 6), align 2
-; AVX1-NEXT:    store i32 [[CTLZ7]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 7), align 2
-; AVX1-NEXT:    ret void
+; SSE42-LABEL: @ctlz_8i32(
+; SSE42-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([8 x i32]* @src32 to <4 x i32>*), align 2
+; SSE42-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 4) to <4 x i32>*), align 2
+; SSE42-NEXT:    [[TMP3:%.*]] = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> [[TMP1]], i1 false)
+; SSE42-NEXT:    [[TMP4:%.*]] = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> [[TMP2]], i1 false)
+; SSE42-NEXT:    store <4 x i32> [[TMP3]], <4 x i32>* bitcast ([8 x i32]* @dst32 to <4 x i32>*), align 2
+; SSE42-NEXT:    store <4 x i32> [[TMP4]], <4 x i32>* bitcast (i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 4) to <4 x i32>*), align 2
+; SSE42-NEXT:    ret void
 ;
-; AVX2-LABEL: @ctlz_8i32(
-; AVX2-NEXT:    [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([8 x i32]* @src32 to <8 x i32>*), align 2
-; AVX2-NEXT:    [[TMP2:%.*]] = call <8 x i32> @llvm.ctlz.v8i32(<8 x i32> [[TMP1]], i1 false)
-; AVX2-NEXT:    store <8 x i32> [[TMP2]], <8 x i32>* bitcast ([8 x i32]* @dst32 to <8 x i32>*), align 2
-; AVX2-NEXT:    ret void
+; AVX-LABEL: @ctlz_8i32(
+; AVX-NEXT:    [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([8 x i32]* @src32 to <8 x i32>*), align 2
+; AVX-NEXT:    [[TMP2:%.*]] = call <8 x i32> @llvm.ctlz.v8i32(<8 x i32> [[TMP1]], i1 false)
+; AVX-NEXT:    store <8 x i32> [[TMP2]], <8 x i32>* bitcast ([8 x i32]* @dst32 to <8 x i32>*), align 2
+; AVX-NEXT:    ret void
 ;
   %ld0 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 0), align 2
   %ld1 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 1), align 2
@@ -353,14 +362,20 @@ define void @ctlz_16i8() #0 {
 }
 
 define void @ctlz_32i8() #0 {
-; CHECK-LABEL: @ctlz_32i8(
-; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([32 x i8]* @src8 to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 16) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP3:%.*]] = call <16 x i8> @llvm.ctlz.v16i8(<16 x i8> [[TMP1]], i1 false)
-; CHECK-NEXT:    [[TMP4:%.*]] = call <16 x i8> @llvm.ctlz.v16i8(<16 x i8> [[TMP2]], i1 false)
-; CHECK-NEXT:    store <16 x i8> [[TMP3]], <16 x i8>* bitcast ([32 x i8]* @dst8 to <16 x i8>*), align 1
-; CHECK-NEXT:    store <16 x i8> [[TMP4]], <16 x i8>* bitcast (i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 16) to <16 x i8>*), align 1
-; CHECK-NEXT:    ret void
+; SSE-LABEL: @ctlz_32i8(
+; SSE-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([32 x i8]* @src8 to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 16) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP3:%.*]] = call <16 x i8> @llvm.ctlz.v16i8(<16 x i8> [[TMP1]], i1 false)
+; SSE-NEXT:    [[TMP4:%.*]] = call <16 x i8> @llvm.ctlz.v16i8(<16 x i8> [[TMP2]], i1 false)
+; SSE-NEXT:    store <16 x i8> [[TMP3]], <16 x i8>* bitcast ([32 x i8]* @dst8 to <16 x i8>*), align 1
+; SSE-NEXT:    store <16 x i8> [[TMP4]], <16 x i8>* bitcast (i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 16) to <16 x i8>*), align 1
+; SSE-NEXT:    ret void
+;
+; AVX-LABEL: @ctlz_32i8(
+; AVX-NEXT:    [[TMP1:%.*]] = load <32 x i8>, <32 x i8>* bitcast ([32 x i8]* @src8 to <32 x i8>*), align 1
+; AVX-NEXT:    [[TMP2:%.*]] = call <32 x i8> @llvm.ctlz.v32i8(<32 x i8> [[TMP1]], i1 false)
+; AVX-NEXT:    store <32 x i8> [[TMP2]], <32 x i8>* bitcast ([32 x i8]* @dst8 to <32 x i8>*), align 1
+; AVX-NEXT:    ret void
 ;
   %ld0  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  0), align 1
   %ld1  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  1), align 1
@@ -516,20 +531,47 @@ define void @ctlz_undef_4i64() #0 {
 }
 
 define void @ctlz_undef_4i32() #0 {
-; CHECK-LABEL: @ctlz_undef_4i32(
-; CHECK-NEXT:    [[LD0:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 0), align 4
-; CHECK-NEXT:    [[LD1:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 1), align 4
-; CHECK-NEXT:    [[LD2:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 2), align 4
-; CHECK-NEXT:    [[LD3:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 3), align 4
-; CHECK-NEXT:    [[CTLZ0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD0]], i1 true)
-; CHECK-NEXT:    [[CTLZ1:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD1]], i1 true)
-; CHECK-NEXT:    [[CTLZ2:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD2]], i1 true)
-; CHECK-NEXT:    [[CTLZ3:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD3]], i1 true)
-; CHECK-NEXT:    store i32 [[CTLZ0]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 0), align 4
-; CHECK-NEXT:    store i32 [[CTLZ1]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 1), align 4
-; CHECK-NEXT:    store i32 [[CTLZ2]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 2), align 4
-; CHECK-NEXT:    store i32 [[CTLZ3]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 3), align 4
-; CHECK-NEXT:    ret void
+; SSE2-LABEL: @ctlz_undef_4i32(
+; SSE2-NEXT:    [[LD0:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 0), align 4
+; SSE2-NEXT:    [[LD1:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 1), align 4
+; SSE2-NEXT:    [[LD2:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 2), align 4
+; SSE2-NEXT:    [[LD3:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 3), align 4
+; SSE2-NEXT:    [[CTLZ0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD0]], i1 true)
+; SSE2-NEXT:    [[CTLZ1:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD1]], i1 true)
+; SSE2-NEXT:    [[CTLZ2:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD2]], i1 true)
+; SSE2-NEXT:    [[CTLZ3:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD3]], i1 true)
+; SSE2-NEXT:    store i32 [[CTLZ0]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 0), align 4
+; SSE2-NEXT:    store i32 [[CTLZ1]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 1), align 4
+; SSE2-NEXT:    store i32 [[CTLZ2]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 2), align 4
+; SSE2-NEXT:    store i32 [[CTLZ3]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 3), align 4
+; SSE2-NEXT:    ret void
+;
+; SSE42-LABEL: @ctlz_undef_4i32(
+; SSE42-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([8 x i32]* @src32 to <4 x i32>*), align 4
+; SSE42-NEXT:    [[TMP2:%.*]] = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> [[TMP1]], i1 true)
+; SSE42-NEXT:    store <4 x i32> [[TMP2]], <4 x i32>* bitcast ([8 x i32]* @dst32 to <4 x i32>*), align 4
+; SSE42-NEXT:    ret void
+;
+; AVX1-LABEL: @ctlz_undef_4i32(
+; AVX1-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([8 x i32]* @src32 to <4 x i32>*), align 4
+; AVX1-NEXT:    [[TMP2:%.*]] = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> [[TMP1]], i1 true)
+; AVX1-NEXT:    store <4 x i32> [[TMP2]], <4 x i32>* bitcast ([8 x i32]* @dst32 to <4 x i32>*), align 4
+; AVX1-NEXT:    ret void
+;
+; AVX2-LABEL: @ctlz_undef_4i32(
+; AVX2-NEXT:    [[LD0:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 0), align 4
+; AVX2-NEXT:    [[LD1:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 1), align 4
+; AVX2-NEXT:    [[LD2:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 2), align 4
+; AVX2-NEXT:    [[LD3:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 3), align 4
+; AVX2-NEXT:    [[CTLZ0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD0]], i1 true)
+; AVX2-NEXT:    [[CTLZ1:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD1]], i1 true)
+; AVX2-NEXT:    [[CTLZ2:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD2]], i1 true)
+; AVX2-NEXT:    [[CTLZ3:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD3]], i1 true)
+; AVX2-NEXT:    store i32 [[CTLZ0]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 0), align 4
+; AVX2-NEXT:    store i32 [[CTLZ1]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 1), align 4
+; AVX2-NEXT:    store i32 [[CTLZ2]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 2), align 4
+; AVX2-NEXT:    store i32 [[CTLZ3]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 3), align 4
+; AVX2-NEXT:    ret void
 ;
   %ld0 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 0), align 4
   %ld1 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 1), align 4
@@ -547,65 +589,47 @@ define void @ctlz_undef_4i32() #0 {
 }
 
 define void @ctlz_undef_8i32() #0 {
-; SSE-LABEL: @ctlz_undef_8i32(
-; SSE-NEXT:    [[LD0:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 0), align 2
-; SSE-NEXT:    [[LD1:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 1), align 2
-; SSE-NEXT:    [[LD2:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 2), align 2
-; SSE-NEXT:    [[LD3:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 3), align 2
-; SSE-NEXT:    [[LD4:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 4), align 2
-; SSE-NEXT:    [[LD5:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 5), align 2
-; SSE-NEXT:    [[LD6:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 6), align 2
-; SSE-NEXT:    [[LD7:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 7), align 2
-; SSE-NEXT:    [[CTLZ0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD0]], i1 true)
-; SSE-NEXT:    [[CTLZ1:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD1]], i1 true)
-; SSE-NEXT:    [[CTLZ2:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD2]], i1 true)
-; SSE-NEXT:    [[CTLZ3:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD3]], i1 true)
-; SSE-NEXT:    [[CTLZ4:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD4]], i1 true)
-; SSE-NEXT:    [[CTLZ5:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD5]], i1 true)
-; SSE-NEXT:    [[CTLZ6:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD6]], i1 true)
-; SSE-NEXT:    [[CTLZ7:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD7]], i1 true)
-; SSE-NEXT:    store i32 [[CTLZ0]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 0), align 2
-; SSE-NEXT:    store i32 [[CTLZ1]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 1), align 2
-; SSE-NEXT:    store i32 [[CTLZ2]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 2), align 2
-; SSE-NEXT:    store i32 [[CTLZ3]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 3), align 2
-; SSE-NEXT:    store i32 [[CTLZ4]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 4), align 2
-; SSE-NEXT:    store i32 [[CTLZ5]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 5), align 2
-; SSE-NEXT:    store i32 [[CTLZ6]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 6), align 2
-; SSE-NEXT:    store i32 [[CTLZ7]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 7), align 2
-; SSE-NEXT:    ret void
+; SSE2-LABEL: @ctlz_undef_8i32(
+; SSE2-NEXT:    [[LD0:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 0), align 2
+; SSE2-NEXT:    [[LD1:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 1), align 2
+; SSE2-NEXT:    [[LD2:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 2), align 2
+; SSE2-NEXT:    [[LD3:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 3), align 2
+; SSE2-NEXT:    [[LD4:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 4), align 2
+; SSE2-NEXT:    [[LD5:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 5), align 2
+; SSE2-NEXT:    [[LD6:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 6), align 2
+; SSE2-NEXT:    [[LD7:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 7), align 2
+; SSE2-NEXT:    [[CTLZ0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD0]], i1 true)
+; SSE2-NEXT:    [[CTLZ1:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD1]], i1 true)
+; SSE2-NEXT:    [[CTLZ2:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD2]], i1 true)
+; SSE2-NEXT:    [[CTLZ3:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD3]], i1 true)
+; SSE2-NEXT:    [[CTLZ4:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD4]], i1 true)
+; SSE2-NEXT:    [[CTLZ5:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD5]], i1 true)
+; SSE2-NEXT:    [[CTLZ6:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD6]], i1 true)
+; SSE2-NEXT:    [[CTLZ7:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD7]], i1 true)
+; SSE2-NEXT:    store i32 [[CTLZ0]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 0), align 2
+; SSE2-NEXT:    store i32 [[CTLZ1]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 1), align 2
+; SSE2-NEXT:    store i32 [[CTLZ2]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 2), align 2
+; SSE2-NEXT:    store i32 [[CTLZ3]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 3), align 2
+; SSE2-NEXT:    store i32 [[CTLZ4]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 4), align 2
+; SSE2-NEXT:    store i32 [[CTLZ5]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 5), align 2
+; SSE2-NEXT:    store i32 [[CTLZ6]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 6), align 2
+; SSE2-NEXT:    store i32 [[CTLZ7]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 7), align 2
+; SSE2-NEXT:    ret void
 ;
-; AVX1-LABEL: @ctlz_undef_8i32(
-; AVX1-NEXT:    [[LD0:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 0), align 2
-; AVX1-NEXT:    [[LD1:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 1), align 2
-; AVX1-NEXT:    [[LD2:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 2), align 2
-; AVX1-NEXT:    [[LD3:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 3), align 2
-; AVX1-NEXT:    [[LD4:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 4), align 2
-; AVX1-NEXT:    [[LD5:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 5), align 2
-; AVX1-NEXT:    [[LD6:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 6), align 2
-; AVX1-NEXT:    [[LD7:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 7), align 2
-; AVX1-NEXT:    [[CTLZ0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD0]], i1 true)
-; AVX1-NEXT:    [[CTLZ1:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD1]], i1 true)
-; AVX1-NEXT:    [[CTLZ2:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD2]], i1 true)
-; AVX1-NEXT:    [[CTLZ3:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD3]], i1 true)
-; AVX1-NEXT:    [[CTLZ4:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD4]], i1 true)
-; AVX1-NEXT:    [[CTLZ5:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD5]], i1 true)
-; AVX1-NEXT:    [[CTLZ6:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD6]], i1 true)
-; AVX1-NEXT:    [[CTLZ7:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD7]], i1 true)
-; AVX1-NEXT:    store i32 [[CTLZ0]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 0), align 2
-; AVX1-NEXT:    store i32 [[CTLZ1]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 1), align 2
-; AVX1-NEXT:    store i32 [[CTLZ2]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 2), align 2
-; AVX1-NEXT:    store i32 [[CTLZ3]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 3), align 2
-; AVX1-NEXT:    store i32 [[CTLZ4]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 4), align 2
-; AVX1-NEXT:    store i32 [[CTLZ5]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 5), align 2
-; AVX1-NEXT:    store i32 [[CTLZ6]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 6), align 2
-; AVX1-NEXT:    store i32 [[CTLZ7]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 7), align 2
-; AVX1-NEXT:    ret void
+; SSE42-LABEL: @ctlz_undef_8i32(
+; SSE42-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([8 x i32]* @src32 to <4 x i32>*), align 2
+; SSE42-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 4) to <4 x i32>*), align 2
+; SSE42-NEXT:    [[TMP3:%.*]] = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> [[TMP1]], i1 true)
+; SSE42-NEXT:    [[TMP4:%.*]] = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> [[TMP2]], i1 true)
+; SSE42-NEXT:    store <4 x i32> [[TMP3]], <4 x i32>* bitcast ([8 x i32]* @dst32 to <4 x i32>*), align 2
+; SSE42-NEXT:    store <4 x i32> [[TMP4]], <4 x i32>* bitcast (i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 4) to <4 x i32>*), align 2
+; SSE42-NEXT:    ret void
 ;
-; AVX2-LABEL: @ctlz_undef_8i32(
-; AVX2-NEXT:    [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([8 x i32]* @src32 to <8 x i32>*), align 2
-; AVX2-NEXT:    [[TMP2:%.*]] = call <8 x i32> @llvm.ctlz.v8i32(<8 x i32> [[TMP1]], i1 true)
-; AVX2-NEXT:    store <8 x i32> [[TMP2]], <8 x i32>* bitcast ([8 x i32]* @dst32 to <8 x i32>*), align 2
-; AVX2-NEXT:    ret void
+; AVX-LABEL: @ctlz_undef_8i32(
+; AVX-NEXT:    [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([8 x i32]* @src32 to <8 x i32>*), align 2
+; AVX-NEXT:    [[TMP2:%.*]] = call <8 x i32> @llvm.ctlz.v8i32(<8 x i32> [[TMP1]], i1 true)
+; AVX-NEXT:    store <8 x i32> [[TMP2]], <8 x i32>* bitcast ([8 x i32]* @dst32 to <8 x i32>*), align 2
+; AVX-NEXT:    ret void
 ;
   %ld0 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 0), align 2
   %ld1 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 1), align 2
@@ -794,14 +818,20 @@ define void @ctlz_undef_16i8() #0 {
 }
 
 define void @ctlz_undef_32i8() #0 {
-; CHECK-LABEL: @ctlz_undef_32i8(
-; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([32 x i8]* @src8 to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 16) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP3:%.*]] = call <16 x i8> @llvm.ctlz.v16i8(<16 x i8> [[TMP1]], i1 true)
-; CHECK-NEXT:    [[TMP4:%.*]] = call <16 x i8> @llvm.ctlz.v16i8(<16 x i8> [[TMP2]], i1 true)
-; CHECK-NEXT:    store <16 x i8> [[TMP3]], <16 x i8>* bitcast ([32 x i8]* @dst8 to <16 x i8>*), align 1
-; CHECK-NEXT:    store <16 x i8> [[TMP4]], <16 x i8>* bitcast (i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 16) to <16 x i8>*), align 1
-; CHECK-NEXT:    ret void
+; SSE-LABEL: @ctlz_undef_32i8(
+; SSE-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([32 x i8]* @src8 to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 16) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP3:%.*]] = call <16 x i8> @llvm.ctlz.v16i8(<16 x i8> [[TMP1]], i1 true)
+; SSE-NEXT:    [[TMP4:%.*]] = call <16 x i8> @llvm.ctlz.v16i8(<16 x i8> [[TMP2]], i1 true)
+; SSE-NEXT:    store <16 x i8> [[TMP3]], <16 x i8>* bitcast ([32 x i8]* @dst8 to <16 x i8>*), align 1
+; SSE-NEXT:    store <16 x i8> [[TMP4]], <16 x i8>* bitcast (i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 16) to <16 x i8>*), align 1
+; SSE-NEXT:    ret void
+;
+; AVX-LABEL: @ctlz_undef_32i8(
+; AVX-NEXT:    [[TMP1:%.*]] = load <32 x i8>, <32 x i8>* bitcast ([32 x i8]* @src8 to <32 x i8>*), align 1
+; AVX-NEXT:    [[TMP2:%.*]] = call <32 x i8> @llvm.ctlz.v32i8(<32 x i8> [[TMP1]], i1 true)
+; AVX-NEXT:    store <32 x i8> [[TMP2]], <32 x i8>* bitcast ([32 x i8]* @dst8 to <32 x i8>*), align 1
+; AVX-NEXT:    ret void
 ;
   %ld0  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  0), align 1
   %ld1  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  1), align 1
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/ctpop.ll b/llvm/test/Transforms/SLPVectorizer/X86/ctpop.ll
index 42f19c8dd..8782e6768 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/ctpop.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/ctpop.ll
@@ -21,14 +21,29 @@ declare i16 @llvm.ctpop.i16(i16)
 declare  i8 @llvm.ctpop.i8(i8)
 
 define void @ctpop_2i64() #0 {
-; CHECK-LABEL: @ctpop_2i64(
-; CHECK-NEXT:    [[LD0:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i32 0, i64 0), align 8
-; CHECK-NEXT:    [[LD1:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i32 0, i64 1), align 8
-; CHECK-NEXT:    [[CTPOP0:%.*]] = call i64 @llvm.ctpop.i64(i64 [[LD0]])
-; CHECK-NEXT:    [[CTPOP1:%.*]] = call i64 @llvm.ctpop.i64(i64 [[LD1]])
-; CHECK-NEXT:    store i64 [[CTPOP0]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i32 0, i64 0), align 8
-; CHECK-NEXT:    store i64 [[CTPOP1]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i32 0, i64 1), align 8
-; CHECK-NEXT:    ret void
+; SSE2-LABEL: @ctpop_2i64(
+; SSE2-NEXT:    [[TMP1:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([4 x i64]* @src64 to <2 x i64>*), align 8
+; SSE2-NEXT:    [[TMP2:%.*]] = call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> [[TMP1]])
+; SSE2-NEXT:    store <2 x i64> [[TMP2]], <2 x i64>* bitcast ([4 x i64]* @dst64 to <2 x i64>*), align 8
+; SSE2-NEXT:    ret void
+;
+; SSE42-LABEL: @ctpop_2i64(
+; SSE42-NEXT:    [[LD0:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i32 0, i64 0), align 8
+; SSE42-NEXT:    [[LD1:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i32 0, i64 1), align 8
+; SSE42-NEXT:    [[CTPOP0:%.*]] = call i64 @llvm.ctpop.i64(i64 [[LD0]])
+; SSE42-NEXT:    [[CTPOP1:%.*]] = call i64 @llvm.ctpop.i64(i64 [[LD1]])
+; SSE42-NEXT:    store i64 [[CTPOP0]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i32 0, i64 0), align 8
+; SSE42-NEXT:    store i64 [[CTPOP1]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i32 0, i64 1), align 8
+; SSE42-NEXT:    ret void
+;
+; AVX-LABEL: @ctpop_2i64(
+; AVX-NEXT:    [[LD0:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i32 0, i64 0), align 8
+; AVX-NEXT:    [[LD1:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i32 0, i64 1), align 8
+; AVX-NEXT:    [[CTPOP0:%.*]] = call i64 @llvm.ctpop.i64(i64 [[LD0]])
+; AVX-NEXT:    [[CTPOP1:%.*]] = call i64 @llvm.ctpop.i64(i64 [[LD1]])
+; AVX-NEXT:    store i64 [[CTPOP0]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i32 0, i64 0), align 8
+; AVX-NEXT:    store i64 [[CTPOP1]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i32 0, i64 1), align 8
+; AVX-NEXT:    ret void
 ;
   %ld0 = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i32 0, i64 0), align 8
   %ld1 = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i32 0, i64 1), align 8
@@ -40,20 +55,29 @@ define void @ctpop_2i64() #0 {
 }
 
 define void @ctpop_4i64() #0 {
-; SSE-LABEL: @ctpop_4i64(
-; SSE-NEXT:    [[LD0:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 0), align 4
-; SSE-NEXT:    [[LD1:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 1), align 4
-; SSE-NEXT:    [[LD2:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 2), align 4
-; SSE-NEXT:    [[LD3:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 3), align 4
-; SSE-NEXT:    [[CTPOP0:%.*]] = call i64 @llvm.ctpop.i64(i64 [[LD0]])
-; SSE-NEXT:    [[CTPOP1:%.*]] = call i64 @llvm.ctpop.i64(i64 [[LD1]])
-; SSE-NEXT:    [[CTPOP2:%.*]] = call i64 @llvm.ctpop.i64(i64 [[LD2]])
-; SSE-NEXT:    [[CTPOP3:%.*]] = call i64 @llvm.ctpop.i64(i64 [[LD3]])
-; SSE-NEXT:    store i64 [[CTPOP0]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 0), align 4
-; SSE-NEXT:    store i64 [[CTPOP1]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 1), align 4
-; SSE-NEXT:    store i64 [[CTPOP2]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 2), align 4
-; SSE-NEXT:    store i64 [[CTPOP3]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 3), align 4
-; SSE-NEXT:    ret void
+; SSE2-LABEL: @ctpop_4i64(
+; SSE2-NEXT:    [[TMP1:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([4 x i64]* @src64 to <2 x i64>*), align 4
+; SSE2-NEXT:    [[TMP2:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 2) to <2 x i64>*), align 4
+; SSE2-NEXT:    [[TMP3:%.*]] = call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> [[TMP1]])
+; SSE2-NEXT:    [[TMP4:%.*]] = call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> [[TMP2]])
+; SSE2-NEXT:    store <2 x i64> [[TMP3]], <2 x i64>* bitcast ([4 x i64]* @dst64 to <2 x i64>*), align 4
+; SSE2-NEXT:    store <2 x i64> [[TMP4]], <2 x i64>* bitcast (i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 2) to <2 x i64>*), align 4
+; SSE2-NEXT:    ret void
+;
+; SSE42-LABEL: @ctpop_4i64(
+; SSE42-NEXT:    [[LD0:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 0), align 4
+; SSE42-NEXT:    [[LD1:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 1), align 4
+; SSE42-NEXT:    [[LD2:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 2), align 4
+; SSE42-NEXT:    [[LD3:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 3), align 4
+; SSE42-NEXT:    [[CTPOP0:%.*]] = call i64 @llvm.ctpop.i64(i64 [[LD0]])
+; SSE42-NEXT:    [[CTPOP1:%.*]] = call i64 @llvm.ctpop.i64(i64 [[LD1]])
+; SSE42-NEXT:    [[CTPOP2:%.*]] = call i64 @llvm.ctpop.i64(i64 [[LD2]])
+; SSE42-NEXT:    [[CTPOP3:%.*]] = call i64 @llvm.ctpop.i64(i64 [[LD3]])
+; SSE42-NEXT:    store i64 [[CTPOP0]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 0), align 4
+; SSE42-NEXT:    store i64 [[CTPOP1]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 1), align 4
+; SSE42-NEXT:    store i64 [[CTPOP2]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 2), align 4
+; SSE42-NEXT:    store i64 [[CTPOP3]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 3), align 4
+; SSE42-NEXT:    ret void
 ;
 ; AVX1-LABEL: @ctpop_4i64(
 ; AVX1-NEXT:    [[LD0:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 0), align 4
@@ -400,14 +424,20 @@ define void @ctpop_16i8() #0 {
 }
 
 define void @ctpop_32i8() #0 {
-; CHECK-LABEL: @ctpop_32i8(
-; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([32 x i8]* @src8 to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 16) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP3:%.*]] = call <16 x i8> @llvm.ctpop.v16i8(<16 x i8> [[TMP1]])
-; CHECK-NEXT:    [[TMP4:%.*]] = call <16 x i8> @llvm.ctpop.v16i8(<16 x i8> [[TMP2]])
-; CHECK-NEXT:    store <16 x i8> [[TMP3]], <16 x i8>* bitcast ([32 x i8]* @dst8 to <16 x i8>*), align 1
-; CHECK-NEXT:    store <16 x i8> [[TMP4]], <16 x i8>* bitcast (i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 16) to <16 x i8>*), align 1
-; CHECK-NEXT:    ret void
+; SSE-LABEL: @ctpop_32i8(
+; SSE-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([32 x i8]* @src8 to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 16) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP3:%.*]] = call <16 x i8> @llvm.ctpop.v16i8(<16 x i8> [[TMP1]])
+; SSE-NEXT:    [[TMP4:%.*]] = call <16 x i8> @llvm.ctpop.v16i8(<16 x i8> [[TMP2]])
+; SSE-NEXT:    store <16 x i8> [[TMP3]], <16 x i8>* bitcast ([32 x i8]* @dst8 to <16 x i8>*), align 1
+; SSE-NEXT:    store <16 x i8> [[TMP4]], <16 x i8>* bitcast (i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 16) to <16 x i8>*), align 1
+; SSE-NEXT:    ret void
+;
+; AVX-LABEL: @ctpop_32i8(
+; AVX-NEXT:    [[TMP1:%.*]] = load <32 x i8>, <32 x i8>* bitcast ([32 x i8]* @src8 to <32 x i8>*), align 1
+; AVX-NEXT:    [[TMP2:%.*]] = call <32 x i8> @llvm.ctpop.v32i8(<32 x i8> [[TMP1]])
+; AVX-NEXT:    store <32 x i8> [[TMP2]], <32 x i8>* bitcast ([32 x i8]* @dst8 to <32 x i8>*), align 1
+; AVX-NEXT:    ret void
 ;
   %ld0  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  0), align 1
   %ld1  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  1), align 1
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/cttz.ll b/llvm/test/Transforms/SLPVectorizer/X86/cttz.ll
index 751fea779..6a1afb96d 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/cttz.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/cttz.ll
@@ -353,14 +353,20 @@ define void @cttz_16i8() #0 {
 }
 
 define void @cttz_32i8() #0 {
-; CHECK-LABEL: @cttz_32i8(
-; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([32 x i8]* @src8 to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 16) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP3:%.*]] = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> [[TMP1]], i1 false)
-; CHECK-NEXT:    [[TMP4:%.*]] = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> [[TMP2]], i1 false)
-; CHECK-NEXT:    store <16 x i8> [[TMP3]], <16 x i8>* bitcast ([32 x i8]* @dst8 to <16 x i8>*), align 1
-; CHECK-NEXT:    store <16 x i8> [[TMP4]], <16 x i8>* bitcast (i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 16) to <16 x i8>*), align 1
-; CHECK-NEXT:    ret void
+; SSE-LABEL: @cttz_32i8(
+; SSE-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([32 x i8]* @src8 to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 16) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP3:%.*]] = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> [[TMP1]], i1 false)
+; SSE-NEXT:    [[TMP4:%.*]] = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> [[TMP2]], i1 false)
+; SSE-NEXT:    store <16 x i8> [[TMP3]], <16 x i8>* bitcast ([32 x i8]* @dst8 to <16 x i8>*), align 1
+; SSE-NEXT:    store <16 x i8> [[TMP4]], <16 x i8>* bitcast (i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 16) to <16 x i8>*), align 1
+; SSE-NEXT:    ret void
+;
+; AVX-LABEL: @cttz_32i8(
+; AVX-NEXT:    [[TMP1:%.*]] = load <32 x i8>, <32 x i8>* bitcast ([32 x i8]* @src8 to <32 x i8>*), align 1
+; AVX-NEXT:    [[TMP2:%.*]] = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> [[TMP1]], i1 false)
+; AVX-NEXT:    store <32 x i8> [[TMP2]], <32 x i8>* bitcast ([32 x i8]* @dst8 to <32 x i8>*), align 1
+; AVX-NEXT:    ret void
 ;
   %ld0  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  0), align 1
   %ld1  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  1), align 1
@@ -794,14 +800,20 @@ define void @cttz_undef_16i8() #0 {
 }
 
 define void @cttz_undef_32i8() #0 {
-; CHECK-LABEL: @cttz_undef_32i8(
-; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([32 x i8]* @src8 to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 16) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP3:%.*]] = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> [[TMP1]], i1 true)
-; CHECK-NEXT:    [[TMP4:%.*]] = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> [[TMP2]], i1 true)
-; CHECK-NEXT:    store <16 x i8> [[TMP3]], <16 x i8>* bitcast ([32 x i8]* @dst8 to <16 x i8>*), align 1
-; CHECK-NEXT:    store <16 x i8> [[TMP4]], <16 x i8>* bitcast (i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 16) to <16 x i8>*), align 1
-; CHECK-NEXT:    ret void
+; SSE-LABEL: @cttz_undef_32i8(
+; SSE-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([32 x i8]* @src8 to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 16) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP3:%.*]] = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> [[TMP1]], i1 true)
+; SSE-NEXT:    [[TMP4:%.*]] = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> [[TMP2]], i1 true)
+; SSE-NEXT:    store <16 x i8> [[TMP3]], <16 x i8>* bitcast ([32 x i8]* @dst8 to <16 x i8>*), align 1
+; SSE-NEXT:    store <16 x i8> [[TMP4]], <16 x i8>* bitcast (i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 16) to <16 x i8>*), align 1
+; SSE-NEXT:    ret void
+;
+; AVX-LABEL: @cttz_undef_32i8(
+; AVX-NEXT:    [[TMP1:%.*]] = load <32 x i8>, <32 x i8>* bitcast ([32 x i8]* @src8 to <32 x i8>*), align 1
+; AVX-NEXT:    [[TMP2:%.*]] = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> [[TMP1]], i1 true)
+; AVX-NEXT:    store <32 x i8> [[TMP2]], <32 x i8>* bitcast ([32 x i8]* @dst8 to <32 x i8>*), align 1
+; AVX-NEXT:    ret void
 ;
   %ld0  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  0), align 1
   %ld1  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  1), align 1
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/debug_info.ll b/llvm/test/Transforms/SLPVectorizer/X86/debug_info.ll
index 0fe399e72..de7c1206e 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/debug_info.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/debug_info.ll
@@ -64,7 +64,7 @@ for.end:                                          ; preds = %for.body.lr.ph, %en
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.value(metadata, i64, metadata, metadata) #1
 
-attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf"="true" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/different-vec-widths.ll b/llvm/test/Transforms/SLPVectorizer/X86/different-vec-widths.ll
new file mode 100644
index 000000000..782ea484d
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/different-vec-widths.ll
@@ -0,0 +1,104 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mattr=sse2 -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: opt < %s -mattr=avx2 -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX
+
+; TODO:
+; With AVX, we are able to vectorize the 1st 4 elements as 256-bit vector ops,
+; but the final 2 elements remain scalar. They should get vectorized using
+; 128-bit ops identically to what happens with SSE.
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @PR28457(double* noalias nocapture align 32 %q, double* noalias nocapture readonly align 32 %p) {
+; SSE-LABEL: @PR28457(
+; SSE-NEXT:    [[P0:%.*]] = getelementptr inbounds double, double* [[P:%.*]], i64 0
+; SSE-NEXT:    [[P1:%.*]] = getelementptr inbounds double, double* [[P]], i64 1
+; SSE-NEXT:    [[P2:%.*]] = getelementptr inbounds double, double* [[P]], i64 2
+; SSE-NEXT:    [[P3:%.*]] = getelementptr inbounds double, double* [[P]], i64 3
+; SSE-NEXT:    [[P4:%.*]] = getelementptr inbounds double, double* [[P]], i64 4
+; SSE-NEXT:    [[P5:%.*]] = getelementptr inbounds double, double* [[P]], i64 5
+; SSE-NEXT:    [[Q0:%.*]] = getelementptr inbounds double, double* [[Q:%.*]], i64 0
+; SSE-NEXT:    [[Q1:%.*]] = getelementptr inbounds double, double* [[Q]], i64 1
+; SSE-NEXT:    [[Q2:%.*]] = getelementptr inbounds double, double* [[Q]], i64 2
+; SSE-NEXT:    [[Q3:%.*]] = getelementptr inbounds double, double* [[Q]], i64 3
+; SSE-NEXT:    [[Q4:%.*]] = getelementptr inbounds double, double* [[Q]], i64 4
+; SSE-NEXT:    [[Q5:%.*]] = getelementptr inbounds double, double* [[Q]], i64 5
+; SSE-NEXT:    [[TMP1:%.*]] = bitcast double* [[P0]] to <2 x double>*
+; SSE-NEXT:    [[TMP2:%.*]] = load <2 x double>, <2 x double>* [[TMP1]], align 8
+; SSE-NEXT:    [[TMP3:%.*]] = bitcast double* [[P2]] to <2 x double>*
+; SSE-NEXT:    [[TMP4:%.*]] = load <2 x double>, <2 x double>* [[TMP3]], align 8
+; SSE-NEXT:    [[TMP5:%.*]] = bitcast double* [[P4]] to <2 x double>*
+; SSE-NEXT:    [[TMP6:%.*]] = load <2 x double>, <2 x double>* [[TMP5]], align 8
+; SSE-NEXT:    [[TMP7:%.*]] = fadd <2 x double> [[TMP2]], <double 1.000000e+00, double 1.000000e+00>
+; SSE-NEXT:    [[TMP8:%.*]] = fadd <2 x double> [[TMP4]], <double 1.000000e+00, double 1.000000e+00>
+; SSE-NEXT:    [[TMP9:%.*]] = fadd <2 x double> [[TMP6]], <double 1.000000e+00, double 1.000000e+00>
+; SSE-NEXT:    [[TMP10:%.*]] = bitcast double* [[Q0]] to <2 x double>*
+; SSE-NEXT:    store <2 x double> [[TMP7]], <2 x double>* [[TMP10]], align 8
+; SSE-NEXT:    [[TMP11:%.*]] = bitcast double* [[Q2]] to <2 x double>*
+; SSE-NEXT:    store <2 x double> [[TMP8]], <2 x double>* [[TMP11]], align 8
+; SSE-NEXT:    [[TMP12:%.*]] = bitcast double* [[Q4]] to <2 x double>*
+; SSE-NEXT:    store <2 x double> [[TMP9]], <2 x double>* [[TMP12]], align 8
+; SSE-NEXT:    ret void
+;
+; AVX-LABEL: @PR28457(
+; AVX-NEXT:    [[P0:%.*]] = getelementptr inbounds double, double* [[P:%.*]], i64 0
+; AVX-NEXT:    [[P1:%.*]] = getelementptr inbounds double, double* [[P]], i64 1
+; AVX-NEXT:    [[P2:%.*]] = getelementptr inbounds double, double* [[P]], i64 2
+; AVX-NEXT:    [[P3:%.*]] = getelementptr inbounds double, double* [[P]], i64 3
+; AVX-NEXT:    [[P4:%.*]] = getelementptr inbounds double, double* [[P]], i64 4
+; AVX-NEXT:    [[P5:%.*]] = getelementptr inbounds double, double* [[P]], i64 5
+; AVX-NEXT:    [[Q0:%.*]] = getelementptr inbounds double, double* [[Q:%.*]], i64 0
+; AVX-NEXT:    [[Q1:%.*]] = getelementptr inbounds double, double* [[Q]], i64 1
+; AVX-NEXT:    [[Q2:%.*]] = getelementptr inbounds double, double* [[Q]], i64 2
+; AVX-NEXT:    [[Q3:%.*]] = getelementptr inbounds double, double* [[Q]], i64 3
+; AVX-NEXT:    [[Q4:%.*]] = getelementptr inbounds double, double* [[Q]], i64 4
+; AVX-NEXT:    [[Q5:%.*]] = getelementptr inbounds double, double* [[Q]], i64 5
+; AVX-NEXT:    [[TMP1:%.*]] = bitcast double* [[P0]] to <4 x double>*
+; AVX-NEXT:    [[TMP2:%.*]] = load <4 x double>, <4 x double>* [[TMP1]], align 8
+; AVX-NEXT:    [[TMP3:%.*]] = bitcast double* [[P4]] to <2 x double>*
+; AVX-NEXT:    [[TMP4:%.*]] = load <2 x double>, <2 x double>* [[TMP3]], align 8
+; AVX-NEXT:    [[TMP5:%.*]] = fadd <4 x double> [[TMP2]], <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>
+; AVX-NEXT:    [[TMP6:%.*]] = fadd <2 x double> [[TMP4]], <double 1.000000e+00, double 1.000000e+00>
+; AVX-NEXT:    [[TMP7:%.*]] = bitcast double* [[Q0]] to <4 x double>*
+; AVX-NEXT:    store <4 x double> [[TMP5]], <4 x double>* [[TMP7]], align 8
+; AVX-NEXT:    [[TMP8:%.*]] = bitcast double* [[Q4]] to <2 x double>*
+; AVX-NEXT:    store <2 x double> [[TMP6]], <2 x double>* [[TMP8]], align 8
+; AVX-NEXT:    ret void
+;
+  %p0 = getelementptr inbounds double, double* %p, i64 0
+  %p1 = getelementptr inbounds double, double* %p, i64 1
+  %p2 = getelementptr inbounds double, double* %p, i64 2
+  %p3 = getelementptr inbounds double, double* %p, i64 3
+  %p4 = getelementptr inbounds double, double* %p, i64 4
+  %p5 = getelementptr inbounds double, double* %p, i64 5
+
+  %q0 = getelementptr inbounds double, double* %q, i64 0
+  %q1 = getelementptr inbounds double, double* %q, i64 1
+  %q2 = getelementptr inbounds double, double* %q, i64 2
+  %q3 = getelementptr inbounds double, double* %q, i64 3
+  %q4 = getelementptr inbounds double, double* %q, i64 4
+  %q5 = getelementptr inbounds double, double* %q, i64 5
+
+  %d0 = load double, double* %p0
+  %d1 = load double, double* %p1
+  %d2 = load double, double* %p2
+  %d3 = load double, double* %p3
+  %d4 = load double, double* %p4
+  %d5 = load double, double* %p5
+
+  %a0 = fadd double %d0, 1.0
+  %a1 = fadd double %d1, 1.0
+  %a2 = fadd double %d2, 1.0
+  %a3 = fadd double %d3, 1.0
+  %a4 = fadd double %d4, 1.0
+  %a5 = fadd double %d5, 1.0
+
+  store double %a0, double* %q0
+  store double %a1, double* %q1
+  store double %a2, double* %q2
+  store double %a3, double* %q3
+  store double %a4, double* %q4
+  store double %a5, double* %q5
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/fptosi.ll b/llvm/test/Transforms/SLPVectorizer/X86/fptosi.ll
index 69b9f32b7..cd8fd1a57 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/fptosi.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/fptosi.ll
@@ -160,38 +160,11 @@ define void @fptosi_8f64_8i32() #0 {
 }
 
 define void @fptosi_8f64_8i16() #0 {
-; SSE-LABEL: @fptosi_8f64_8i16(
-; SSE-NEXT:    [[A0:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
-; SSE-NEXT:    [[A1:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
-; SSE-NEXT:    [[A2:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2), align 8
-; SSE-NEXT:    [[A3:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 3), align 8
-; SSE-NEXT:    [[A4:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4), align 8
-; SSE-NEXT:    [[A5:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 5), align 8
-; SSE-NEXT:    [[A6:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 6), align 8
-; SSE-NEXT:    [[A7:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 7), align 8
-; SSE-NEXT:    [[CVT0:%.*]] = fptosi double [[A0]] to i16
-; SSE-NEXT:    [[CVT1:%.*]] = fptosi double [[A1]] to i16
-; SSE-NEXT:    [[CVT2:%.*]] = fptosi double [[A2]] to i16
-; SSE-NEXT:    [[CVT3:%.*]] = fptosi double [[A3]] to i16
-; SSE-NEXT:    [[CVT4:%.*]] = fptosi double [[A4]] to i16
-; SSE-NEXT:    [[CVT5:%.*]] = fptosi double [[A5]] to i16
-; SSE-NEXT:    [[CVT6:%.*]] = fptosi double [[A6]] to i16
-; SSE-NEXT:    [[CVT7:%.*]] = fptosi double [[A7]] to i16
-; SSE-NEXT:    store i16 [[CVT0]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 0), align 2
-; SSE-NEXT:    store i16 [[CVT1]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 1), align 2
-; SSE-NEXT:    store i16 [[CVT2]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 2), align 2
-; SSE-NEXT:    store i16 [[CVT3]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 3), align 2
-; SSE-NEXT:    store i16 [[CVT4]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 4), align 2
-; SSE-NEXT:    store i16 [[CVT5]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 5), align 2
-; SSE-NEXT:    store i16 [[CVT6]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 6), align 2
-; SSE-NEXT:    store i16 [[CVT7]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 7), align 2
-; SSE-NEXT:    ret void
-;
-; AVX-LABEL: @fptosi_8f64_8i16(
-; AVX-NEXT:    [[TMP1:%.*]] = load <8 x double>, <8 x double>* bitcast ([8 x double]* @src64 to <8 x double>*), align 8
-; AVX-NEXT:    [[TMP2:%.*]] = fptosi <8 x double> [[TMP1]] to <8 x i16>
-; AVX-NEXT:    store <8 x i16> [[TMP2]], <8 x i16>* bitcast ([32 x i16]* @dst16 to <8 x i16>*), align 2
-; AVX-NEXT:    ret void
+; CHECK-LABEL: @fptosi_8f64_8i16(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x double>, <8 x double>* bitcast ([8 x double]* @src64 to <8 x double>*), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = fptosi <8 x double> [[TMP1]] to <8 x i16>
+; CHECK-NEXT:    store <8 x i16> [[TMP2]], <8 x i16>* bitcast ([32 x i16]* @dst16 to <8 x i16>*), align 2
+; CHECK-NEXT:    ret void
 ;
   %a0 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
   %a1 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/fptoui.ll b/llvm/test/Transforms/SLPVectorizer/X86/fptoui.ll
index 33d998ffb..dafab8ee7 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/fptoui.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/fptoui.ll
@@ -238,44 +238,11 @@ define void @fptoui_8f64_8i16() #0 {
 ; SSE-NEXT:    store i16 [[CVT7]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 7), align 2
 ; SSE-NEXT:    ret void
 ;
-; AVX256NODQ-LABEL: @fptoui_8f64_8i16(
-; AVX256NODQ-NEXT:    [[A0:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
-; AVX256NODQ-NEXT:    [[A1:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
-; AVX256NODQ-NEXT:    [[A2:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2), align 8
-; AVX256NODQ-NEXT:    [[A3:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 3), align 8
-; AVX256NODQ-NEXT:    [[A4:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4), align 8
-; AVX256NODQ-NEXT:    [[A5:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 5), align 8
-; AVX256NODQ-NEXT:    [[A6:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 6), align 8
-; AVX256NODQ-NEXT:    [[A7:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 7), align 8
-; AVX256NODQ-NEXT:    [[CVT0:%.*]] = fptoui double [[A0]] to i16
-; AVX256NODQ-NEXT:    [[CVT1:%.*]] = fptoui double [[A1]] to i16
-; AVX256NODQ-NEXT:    [[CVT2:%.*]] = fptoui double [[A2]] to i16
-; AVX256NODQ-NEXT:    [[CVT3:%.*]] = fptoui double [[A3]] to i16
-; AVX256NODQ-NEXT:    [[CVT4:%.*]] = fptoui double [[A4]] to i16
-; AVX256NODQ-NEXT:    [[CVT5:%.*]] = fptoui double [[A5]] to i16
-; AVX256NODQ-NEXT:    [[CVT6:%.*]] = fptoui double [[A6]] to i16
-; AVX256NODQ-NEXT:    [[CVT7:%.*]] = fptoui double [[A7]] to i16
-; AVX256NODQ-NEXT:    store i16 [[CVT0]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 0), align 2
-; AVX256NODQ-NEXT:    store i16 [[CVT1]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 1), align 2
-; AVX256NODQ-NEXT:    store i16 [[CVT2]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 2), align 2
-; AVX256NODQ-NEXT:    store i16 [[CVT3]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 3), align 2
-; AVX256NODQ-NEXT:    store i16 [[CVT4]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 4), align 2
-; AVX256NODQ-NEXT:    store i16 [[CVT5]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 5), align 2
-; AVX256NODQ-NEXT:    store i16 [[CVT6]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 6), align 2
-; AVX256NODQ-NEXT:    store i16 [[CVT7]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 7), align 2
-; AVX256NODQ-NEXT:    ret void
-;
-; AVX512-LABEL: @fptoui_8f64_8i16(
-; AVX512-NEXT:    [[TMP1:%.*]] = load <8 x double>, <8 x double>* bitcast ([8 x double]* @src64 to <8 x double>*), align 8
-; AVX512-NEXT:    [[TMP2:%.*]] = fptoui <8 x double> [[TMP1]] to <8 x i16>
-; AVX512-NEXT:    store <8 x i16> [[TMP2]], <8 x i16>* bitcast ([32 x i16]* @dst16 to <8 x i16>*), align 2
-; AVX512-NEXT:    ret void
-;
-; AVX256DQ-LABEL: @fptoui_8f64_8i16(
-; AVX256DQ-NEXT:    [[TMP1:%.*]] = load <8 x double>, <8 x double>* bitcast ([8 x double]* @src64 to <8 x double>*), align 8
-; AVX256DQ-NEXT:    [[TMP2:%.*]] = fptoui <8 x double> [[TMP1]] to <8 x i16>
-; AVX256DQ-NEXT:    store <8 x i16> [[TMP2]], <8 x i16>* bitcast ([32 x i16]* @dst16 to <8 x i16>*), align 2
-; AVX256DQ-NEXT:    ret void
+; AVX-LABEL: @fptoui_8f64_8i16(
+; AVX-NEXT:    [[TMP1:%.*]] = load <8 x double>, <8 x double>* bitcast ([8 x double]* @src64 to <8 x double>*), align 8
+; AVX-NEXT:    [[TMP2:%.*]] = fptoui <8 x double> [[TMP1]] to <8 x i16>
+; AVX-NEXT:    store <8 x i16> [[TMP2]], <8 x i16>* bitcast ([32 x i16]* @dst16 to <8 x i16>*), align 2
+; AVX-NEXT:    ret void
 ;
   %a0 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
   %a1 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/hadd.ll b/llvm/test/Transforms/SLPVectorizer/X86/hadd.ll
index b02244f96..71f72a930 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/hadd.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/hadd.ll
@@ -78,34 +78,11 @@ define <4 x float> @test_v4f32(<4 x float> %a, <4 x float> %b) {
 }
 
 define <2 x i64> @test_v2i64(<2 x i64> %a, <2 x i64> %b) {
-; SSE-LABEL: @test_v2i64(
-; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], <2 x i32> <i32 0, i32 2>
-; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i64> [[A]], <2 x i64> [[B]], <2 x i32> <i32 1, i32 3>
-; SSE-NEXT:    [[TMP3:%.*]] = add <2 x i64> [[TMP1]], [[TMP2]]
-; SSE-NEXT:    ret <2 x i64> [[TMP3]]
-;
-; SLM-LABEL: @test_v2i64(
-; SLM-NEXT:    [[A0:%.*]] = extractelement <2 x i64> [[A:%.*]], i32 0
-; SLM-NEXT:    [[A1:%.*]] = extractelement <2 x i64> [[A]], i32 1
-; SLM-NEXT:    [[B0:%.*]] = extractelement <2 x i64> [[B:%.*]], i32 0
-; SLM-NEXT:    [[B1:%.*]] = extractelement <2 x i64> [[B]], i32 1
-; SLM-NEXT:    [[R0:%.*]] = add i64 [[A0]], [[A1]]
-; SLM-NEXT:    [[R1:%.*]] = add i64 [[B0]], [[B1]]
-; SLM-NEXT:    [[R00:%.*]] = insertelement <2 x i64> undef, i64 [[R0]], i32 0
-; SLM-NEXT:    [[R01:%.*]] = insertelement <2 x i64> [[R00]], i64 [[R1]], i32 1
-; SLM-NEXT:    ret <2 x i64> [[R01]]
-;
-; AVX-LABEL: @test_v2i64(
-; AVX-NEXT:    [[TMP1:%.*]] = shufflevector <2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], <2 x i32> <i32 0, i32 2>
-; AVX-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i64> [[A]], <2 x i64> [[B]], <2 x i32> <i32 1, i32 3>
-; AVX-NEXT:    [[TMP3:%.*]] = add <2 x i64> [[TMP1]], [[TMP2]]
-; AVX-NEXT:    ret <2 x i64> [[TMP3]]
-;
-; AVX512-LABEL: @test_v2i64(
-; AVX512-NEXT:    [[TMP1:%.*]] = shufflevector <2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], <2 x i32> <i32 0, i32 2>
-; AVX512-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i64> [[A]], <2 x i64> [[B]], <2 x i32> <i32 1, i32 3>
-; AVX512-NEXT:    [[TMP3:%.*]] = add <2 x i64> [[TMP1]], [[TMP2]]
-; AVX512-NEXT:    ret <2 x i64> [[TMP3]]
+; CHECK-LABEL: @test_v2i64(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i64> [[A]], <2 x i64> [[B]], <2 x i32> <i32 1, i32 3>
+; CHECK-NEXT:    [[TMP3:%.*]] = add <2 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret <2 x i64> [[TMP3]]
 ;
   %a0 = extractelement <2 x i64> %a, i32 0
   %a1 = extractelement <2 x i64> %a, i32 1
@@ -322,14 +299,10 @@ define <4 x i64> @test_v4i64(<4 x i64> %a, <4 x i64> %b) {
 ; SSE-NEXT:    ret <4 x i64> [[R03]]
 ;
 ; SLM-LABEL: @test_v4i64(
-; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i64> [[A:%.*]], <4 x i64> [[B:%.*]], <2 x i32> <i32 0, i32 4>
-; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <2 x i32> <i32 1, i32 5>
-; SLM-NEXT:    [[TMP3:%.*]] = add <2 x i64> [[TMP1]], [[TMP2]]
-; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <2 x i32> <i32 2, i32 6>
-; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <2 x i32> <i32 3, i32 7>
-; SLM-NEXT:    [[TMP6:%.*]] = add <2 x i64> [[TMP4]], [[TMP5]]
-; SLM-NEXT:    [[R03:%.*]] = shufflevector <2 x i64> [[TMP3]], <2 x i64> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; SLM-NEXT:    ret <4 x i64> [[R03]]
+; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i64> [[A:%.*]], <4 x i64> [[B:%.*]], <4 x i32> <i32 0, i32 4, i32 2, i32 6>
+; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <4 x i32> <i32 1, i32 5, i32 3, i32 7>
+; SLM-NEXT:    [[TMP3:%.*]] = add <4 x i64> [[TMP1]], [[TMP2]]
+; SLM-NEXT:    ret <4 x i64> [[TMP3]]
 ;
 ; AVX-LABEL: @test_v4i64(
 ; AVX-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i64> [[A:%.*]], <4 x i64> [[B:%.*]], <4 x i32> <i32 0, i32 4, i32 2, i32 6>
@@ -374,14 +347,10 @@ define <8 x i32> @test_v8i32(<8 x i32> %a, <8 x i32> %b) {
 ; SSE-NEXT:    ret <8 x i32> [[R07]]
 ;
 ; SLM-LABEL: @test_v8i32(
-; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A:%.*]], <8 x i32> [[B:%.*]], <4 x i32> <i32 0, i32 2, i32 8, i32 10>
-; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <4 x i32> <i32 1, i32 3, i32 9, i32 11>
-; SLM-NEXT:    [[TMP3:%.*]] = add <4 x i32> [[TMP1]], [[TMP2]]
-; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <4 x i32> <i32 4, i32 6, i32 12, i32 14>
-; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <4 x i32> <i32 5, i32 7, i32 13, i32 15>
-; SLM-NEXT:    [[TMP6:%.*]] = add <4 x i32> [[TMP4]], [[TMP5]]
-; SLM-NEXT:    [[R07:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SLM-NEXT:    ret <8 x i32> [[R07]]
+; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A:%.*]], <8 x i32> [[B:%.*]], <8 x i32> <i32 0, i32 2, i32 8, i32 10, i32 4, i32 6, i32 12, i32 14>
+; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <8 x i32> <i32 1, i32 3, i32 9, i32 11, i32 5, i32 7, i32 13, i32 15>
+; SLM-NEXT:    [[TMP3:%.*]] = add <8 x i32> [[TMP1]], [[TMP2]]
+; SLM-NEXT:    ret <8 x i32> [[TMP3]]
 ;
 ; AVX-LABEL: @test_v8i32(
 ; AVX-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A:%.*]], <8 x i32> [[B:%.*]], <8 x i32> <i32 0, i32 2, i32 8, i32 10, i32 4, i32 6, i32 12, i32 14>
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/horizontal-list.ll b/llvm/test/Transforms/SLPVectorizer/X86/horizontal-list.ll
index 6c3994b0a..1a2c66c68 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/horizontal-list.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/horizontal-list.ll
@@ -100,16 +100,8 @@ define float @bazz() {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x float>, <8 x float>* bitcast ([20 x float]* @arr to <8 x float>*), align 16
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x float>, <8 x float>* bitcast ([20 x float]* @arr1 to <8 x float>*), align 16
 ; CHECK-NEXT:    [[TMP3:%.*]] = fmul fast <8 x float> [[TMP2]], [[TMP1]]
-; CHECK-NEXT:    [[ADD:%.*]] = fadd fast float undef, [[CONV]]
-; CHECK-NEXT:    [[ADD_1:%.*]] = fadd fast float undef, [[ADD]]
-; CHECK-NEXT:    [[ADD_2:%.*]] = fadd fast float undef, [[ADD_1]]
-; CHECK-NEXT:    [[ADD_3:%.*]] = fadd fast float undef, [[ADD_2]]
 ; CHECK-NEXT:    [[MUL5:%.*]] = shl nsw i32 [[TMP0]], 2
 ; CHECK-NEXT:    [[CONV6:%.*]] = sitofp i32 [[MUL5]] to float
-; CHECK-NEXT:    [[ADD7:%.*]] = fadd fast float [[ADD_3]], [[CONV6]]
-; CHECK-NEXT:    [[ADD19:%.*]] = fadd fast float undef, [[ADD7]]
-; CHECK-NEXT:    [[ADD19_1:%.*]] = fadd fast float undef, [[ADD19]]
-; CHECK-NEXT:    [[ADD19_2:%.*]] = fadd fast float undef, [[ADD19_1]]
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x float> [[TMP3]], <8 x float> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = fadd fast <8 x float> [[TMP3]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x float> [[BIN_RDX]], <8 x float> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -119,7 +111,6 @@ define float @bazz() {
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <8 x float> [[BIN_RDX4]], i32 0
 ; CHECK-NEXT:    [[OP_EXTRA:%.*]] = fadd fast float [[TMP4]], [[CONV]]
 ; CHECK-NEXT:    [[OP_EXTRA5:%.*]] = fadd fast float [[OP_EXTRA]], [[CONV6]]
-; CHECK-NEXT:    [[ADD19_3:%.*]] = fadd fast float undef, [[ADD19_2]]
 ; CHECK-NEXT:    store float [[OP_EXTRA5]], float* @res, align 4
 ; CHECK-NEXT:    ret float [[OP_EXTRA5]]
 ;
@@ -131,16 +122,8 @@ define float @bazz() {
 ; THRESHOLD-NEXT:    [[TMP1:%.*]] = load <8 x float>, <8 x float>* bitcast ([20 x float]* @arr to <8 x float>*), align 16
 ; THRESHOLD-NEXT:    [[TMP2:%.*]] = load <8 x float>, <8 x float>* bitcast ([20 x float]* @arr1 to <8 x float>*), align 16
 ; THRESHOLD-NEXT:    [[TMP3:%.*]] = fmul fast <8 x float> [[TMP2]], [[TMP1]]
-; THRESHOLD-NEXT:    [[ADD:%.*]] = fadd fast float undef, [[CONV]]
-; THRESHOLD-NEXT:    [[ADD_1:%.*]] = fadd fast float undef, [[ADD]]
-; THRESHOLD-NEXT:    [[ADD_2:%.*]] = fadd fast float undef, [[ADD_1]]
-; THRESHOLD-NEXT:    [[ADD_3:%.*]] = fadd fast float undef, [[ADD_2]]
 ; THRESHOLD-NEXT:    [[MUL5:%.*]] = shl nsw i32 [[TMP0]], 2
 ; THRESHOLD-NEXT:    [[CONV6:%.*]] = sitofp i32 [[MUL5]] to float
-; THRESHOLD-NEXT:    [[ADD7:%.*]] = fadd fast float [[ADD_3]], [[CONV6]]
-; THRESHOLD-NEXT:    [[ADD19:%.*]] = fadd fast float undef, [[ADD7]]
-; THRESHOLD-NEXT:    [[ADD19_1:%.*]] = fadd fast float undef, [[ADD19]]
-; THRESHOLD-NEXT:    [[ADD19_2:%.*]] = fadd fast float undef, [[ADD19_1]]
 ; THRESHOLD-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x float> [[TMP3]], <8 x float> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
 ; THRESHOLD-NEXT:    [[BIN_RDX:%.*]] = fadd fast <8 x float> [[TMP3]], [[RDX_SHUF]]
 ; THRESHOLD-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x float> [[BIN_RDX]], <8 x float> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -150,7 +133,6 @@ define float @bazz() {
 ; THRESHOLD-NEXT:    [[TMP4:%.*]] = extractelement <8 x float> [[BIN_RDX4]], i32 0
 ; THRESHOLD-NEXT:    [[OP_EXTRA:%.*]] = fadd fast float [[TMP4]], [[CONV]]
 ; THRESHOLD-NEXT:    [[OP_EXTRA5:%.*]] = fadd fast float [[OP_EXTRA]], [[CONV6]]
-; THRESHOLD-NEXT:    [[ADD19_3:%.*]] = fadd fast float undef, [[ADD19_2]]
 ; THRESHOLD-NEXT:    store float [[OP_EXTRA5]], float* @res, align 4
 ; THRESHOLD-NEXT:    ret float [[OP_EXTRA5]]
 ;
@@ -205,17 +187,14 @@ define float @bazzz() {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* bitcast ([20 x float]* @arr to <4 x float>*), align 16
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x float>, <4 x float>* bitcast ([20 x float]* @arr1 to <4 x float>*), align 16
 ; CHECK-NEXT:    [[TMP3:%.*]] = fmul fast <4 x float> [[TMP2]], [[TMP1]]
-; CHECK-NEXT:    [[TMP4:%.*]] = fadd fast float undef, undef
-; CHECK-NEXT:    [[TMP5:%.*]] = fadd fast float undef, [[TMP4]]
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = fadd fast <4 x float> [[TMP3]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x float> [[BIN_RDX]], <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX2:%.*]] = fadd fast <4 x float> [[BIN_RDX]], [[RDX_SHUF1]]
-; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[BIN_RDX2]], i32 0
-; CHECK-NEXT:    [[TMP7:%.*]] = fadd fast float undef, [[TMP5]]
-; CHECK-NEXT:    [[TMP8:%.*]] = fmul fast float [[CONV]], [[TMP6]]
-; CHECK-NEXT:    store float [[TMP8]], float* @res, align 4
-; CHECK-NEXT:    ret float [[TMP8]]
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[BIN_RDX2]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = fmul fast float [[CONV]], [[TMP4]]
+; CHECK-NEXT:    store float [[TMP5]], float* @res, align 4
+; CHECK-NEXT:    ret float [[TMP5]]
 ;
 ; THRESHOLD-LABEL: @bazzz(
 ; THRESHOLD-NEXT:  entry:
@@ -224,17 +203,14 @@ define float @bazzz() {
 ; THRESHOLD-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* bitcast ([20 x float]* @arr to <4 x float>*), align 16
 ; THRESHOLD-NEXT:    [[TMP2:%.*]] = load <4 x float>, <4 x float>* bitcast ([20 x float]* @arr1 to <4 x float>*), align 16
 ; THRESHOLD-NEXT:    [[TMP3:%.*]] = fmul fast <4 x float> [[TMP2]], [[TMP1]]
-; THRESHOLD-NEXT:    [[TMP4:%.*]] = fadd fast float undef, undef
-; THRESHOLD-NEXT:    [[TMP5:%.*]] = fadd fast float undef, [[TMP4]]
 ; THRESHOLD-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
 ; THRESHOLD-NEXT:    [[BIN_RDX:%.*]] = fadd fast <4 x float> [[TMP3]], [[RDX_SHUF]]
 ; THRESHOLD-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x float> [[BIN_RDX]], <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
 ; THRESHOLD-NEXT:    [[BIN_RDX2:%.*]] = fadd fast <4 x float> [[BIN_RDX]], [[RDX_SHUF1]]
-; THRESHOLD-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[BIN_RDX2]], i32 0
-; THRESHOLD-NEXT:    [[TMP7:%.*]] = fadd fast float undef, [[TMP5]]
-; THRESHOLD-NEXT:    [[TMP8:%.*]] = fmul fast float [[CONV]], [[TMP6]]
-; THRESHOLD-NEXT:    store float [[TMP8]], float* @res, align 4
-; THRESHOLD-NEXT:    ret float [[TMP8]]
+; THRESHOLD-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[BIN_RDX2]], i32 0
+; THRESHOLD-NEXT:    [[TMP5:%.*]] = fmul fast float [[CONV]], [[TMP4]]
+; THRESHOLD-NEXT:    store float [[TMP5]], float* @res, align 4
+; THRESHOLD-NEXT:    ret float [[TMP5]]
 ;
 entry:
   %0 = load i32, i32* @n, align 4
@@ -267,16 +243,13 @@ define i32 @foo() {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* bitcast ([20 x float]* @arr to <4 x float>*), align 16
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x float>, <4 x float>* bitcast ([20 x float]* @arr1 to <4 x float>*), align 16
 ; CHECK-NEXT:    [[TMP3:%.*]] = fmul fast <4 x float> [[TMP2]], [[TMP1]]
-; CHECK-NEXT:    [[TMP4:%.*]] = fadd fast float undef, undef
-; CHECK-NEXT:    [[TMP5:%.*]] = fadd fast float undef, [[TMP4]]
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = fadd fast <4 x float> [[TMP3]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x float> [[BIN_RDX]], <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX2:%.*]] = fadd fast <4 x float> [[BIN_RDX]], [[RDX_SHUF1]]
-; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[BIN_RDX2]], i32 0
-; CHECK-NEXT:    [[TMP7:%.*]] = fadd fast float undef, [[TMP5]]
-; CHECK-NEXT:    [[TMP8:%.*]] = fmul fast float [[CONV]], [[TMP6]]
-; CHECK-NEXT:    [[CONV4:%.*]] = fptosi float [[TMP8]] to i32
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[BIN_RDX2]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = fmul fast float [[CONV]], [[TMP4]]
+; CHECK-NEXT:    [[CONV4:%.*]] = fptosi float [[TMP5]] to i32
 ; CHECK-NEXT:    store i32 [[CONV4]], i32* @n, align 4
 ; CHECK-NEXT:    ret i32 [[CONV4]]
 ;
@@ -287,16 +260,13 @@ define i32 @foo() {
 ; THRESHOLD-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* bitcast ([20 x float]* @arr to <4 x float>*), align 16
 ; THRESHOLD-NEXT:    [[TMP2:%.*]] = load <4 x float>, <4 x float>* bitcast ([20 x float]* @arr1 to <4 x float>*), align 16
 ; THRESHOLD-NEXT:    [[TMP3:%.*]] = fmul fast <4 x float> [[TMP2]], [[TMP1]]
-; THRESHOLD-NEXT:    [[TMP4:%.*]] = fadd fast float undef, undef
-; THRESHOLD-NEXT:    [[TMP5:%.*]] = fadd fast float undef, [[TMP4]]
 ; THRESHOLD-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
 ; THRESHOLD-NEXT:    [[BIN_RDX:%.*]] = fadd fast <4 x float> [[TMP3]], [[RDX_SHUF]]
 ; THRESHOLD-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x float> [[BIN_RDX]], <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
 ; THRESHOLD-NEXT:    [[BIN_RDX2:%.*]] = fadd fast <4 x float> [[BIN_RDX]], [[RDX_SHUF1]]
-; THRESHOLD-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[BIN_RDX2]], i32 0
-; THRESHOLD-NEXT:    [[TMP7:%.*]] = fadd fast float undef, [[TMP5]]
-; THRESHOLD-NEXT:    [[TMP8:%.*]] = fmul fast float [[CONV]], [[TMP6]]
-; THRESHOLD-NEXT:    [[CONV4:%.*]] = fptosi float [[TMP8]] to i32
+; THRESHOLD-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[BIN_RDX2]], i32 0
+; THRESHOLD-NEXT:    [[TMP5:%.*]] = fmul fast float [[CONV]], [[TMP4]]
+; THRESHOLD-NEXT:    [[CONV4:%.*]] = fptosi float [[TMP5]] to i32
 ; THRESHOLD-NEXT:    store i32 [[CONV4]], i32* @n, align 4
 ; THRESHOLD-NEXT:    ret i32 [[CONV4]]
 ;
@@ -330,11 +300,6 @@ define float @bar() {
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x float>, <4 x float>* bitcast ([20 x float]* @arr to <4 x float>*), align 16
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* bitcast ([20 x float]* @arr1 to <4 x float>*), align 16
 ; CHECK-NEXT:    [[TMP2:%.*]] = fmul fast <4 x float> [[TMP1]], [[TMP0]]
-; CHECK-NEXT:    [[CMP4:%.*]] = fcmp fast ogt float undef, undef
-; CHECK-NEXT:    [[MAX_0_MUL3:%.*]] = select i1 [[CMP4]], float undef, float undef
-; CHECK-NEXT:    [[CMP4_1:%.*]] = fcmp fast ogt float [[MAX_0_MUL3]], undef
-; CHECK-NEXT:    [[MAX_0_MUL3_1:%.*]] = select i1 [[CMP4_1]], float [[MAX_0_MUL3]], float undef
-; CHECK-NEXT:    [[CMP4_2:%.*]] = fcmp fast ogt float [[MAX_0_MUL3_1]], undef
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp fast ogt <4 x float> [[TMP2]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x float> [[TMP2]], <4 x float> [[RDX_SHUF]]
@@ -342,7 +307,6 @@ define float @bar() {
 ; CHECK-NEXT:    [[RDX_MINMAX_CMP2:%.*]] = fcmp fast ogt <4 x float> [[RDX_MINMAX_SELECT]], [[RDX_SHUF1]]
 ; CHECK-NEXT:    [[RDX_MINMAX_SELECT3:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP2]], <4 x float> [[RDX_MINMAX_SELECT]], <4 x float> [[RDX_SHUF1]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[RDX_MINMAX_SELECT3]], i32 0
-; CHECK-NEXT:    [[MAX_0_MUL3_2:%.*]] = select i1 [[CMP4_2]], float [[MAX_0_MUL3_1]], float undef
 ; CHECK-NEXT:    store float [[TMP3]], float* @res, align 4
 ; CHECK-NEXT:    ret float [[TMP3]]
 ;
@@ -351,11 +315,6 @@ define float @bar() {
 ; THRESHOLD-NEXT:    [[TMP0:%.*]] = load <4 x float>, <4 x float>* bitcast ([20 x float]* @arr to <4 x float>*), align 16
 ; THRESHOLD-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* bitcast ([20 x float]* @arr1 to <4 x float>*), align 16
 ; THRESHOLD-NEXT:    [[TMP2:%.*]] = fmul fast <4 x float> [[TMP1]], [[TMP0]]
-; THRESHOLD-NEXT:    [[CMP4:%.*]] = fcmp fast ogt float undef, undef
-; THRESHOLD-NEXT:    [[MAX_0_MUL3:%.*]] = select i1 [[CMP4]], float undef, float undef
-; THRESHOLD-NEXT:    [[CMP4_1:%.*]] = fcmp fast ogt float [[MAX_0_MUL3]], undef
-; THRESHOLD-NEXT:    [[MAX_0_MUL3_1:%.*]] = select i1 [[CMP4_1]], float [[MAX_0_MUL3]], float undef
-; THRESHOLD-NEXT:    [[CMP4_2:%.*]] = fcmp fast ogt float [[MAX_0_MUL3_1]], undef
 ; THRESHOLD-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
 ; THRESHOLD-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp fast ogt <4 x float> [[TMP2]], [[RDX_SHUF]]
 ; THRESHOLD-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x float> [[TMP2]], <4 x float> [[RDX_SHUF]]
@@ -363,7 +322,6 @@ define float @bar() {
 ; THRESHOLD-NEXT:    [[RDX_MINMAX_CMP2:%.*]] = fcmp fast ogt <4 x float> [[RDX_MINMAX_SELECT]], [[RDX_SHUF1]]
 ; THRESHOLD-NEXT:    [[RDX_MINMAX_SELECT3:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP2]], <4 x float> [[RDX_MINMAX_SELECT]], <4 x float> [[RDX_SHUF1]]
 ; THRESHOLD-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[RDX_MINMAX_SELECT3]], i32 0
-; THRESHOLD-NEXT:    [[MAX_0_MUL3_2:%.*]] = select i1 [[CMP4_2]], float [[MAX_0_MUL3_1]], float undef
 ; THRESHOLD-NEXT:    store float [[TMP3]], float* @res, align 4
 ; THRESHOLD-NEXT:    ret float [[TMP3]]
 ;
@@ -410,21 +368,6 @@ define float @f(float* nocapture readonly %x) {
 ; CHECK-NEXT:    [[ARRAYIDX_15:%.*]] = getelementptr inbounds float, float* [[X]], i64 15
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast float* [[X]] to <16 x float>*
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x float>, <16 x float>* [[TMP0]], align 4
-; CHECK-NEXT:    [[ADD_1:%.*]] = fadd fast float undef, undef
-; CHECK-NEXT:    [[ADD_2:%.*]] = fadd fast float undef, [[ADD_1]]
-; CHECK-NEXT:    [[ADD_3:%.*]] = fadd fast float undef, [[ADD_2]]
-; CHECK-NEXT:    [[ADD_4:%.*]] = fadd fast float undef, [[ADD_3]]
-; CHECK-NEXT:    [[ADD_5:%.*]] = fadd fast float undef, [[ADD_4]]
-; CHECK-NEXT:    [[ADD_6:%.*]] = fadd fast float undef, [[ADD_5]]
-; CHECK-NEXT:    [[ADD_7:%.*]] = fadd fast float undef, [[ADD_6]]
-; CHECK-NEXT:    [[ADD_8:%.*]] = fadd fast float undef, [[ADD_7]]
-; CHECK-NEXT:    [[ADD_9:%.*]] = fadd fast float undef, [[ADD_8]]
-; CHECK-NEXT:    [[ADD_10:%.*]] = fadd fast float undef, [[ADD_9]]
-; CHECK-NEXT:    [[ADD_11:%.*]] = fadd fast float undef, [[ADD_10]]
-; CHECK-NEXT:    [[ADD_12:%.*]] = fadd fast float undef, [[ADD_11]]
-; CHECK-NEXT:    [[ADD_13:%.*]] = fadd fast float undef, [[ADD_12]]
-; CHECK-NEXT:    [[ADD_14:%.*]] = fadd fast float undef, [[ADD_13]]
-; CHECK-NEXT:    [[ADD_15:%.*]] = fadd fast float undef, [[ADD_14]]
 ; CHECK-NEXT:    [[ARRAYIDX_16:%.*]] = getelementptr inbounds float, float* [[X]], i64 16
 ; CHECK-NEXT:    [[ARRAYIDX_17:%.*]] = getelementptr inbounds float, float* [[X]], i64 17
 ; CHECK-NEXT:    [[ARRAYIDX_18:%.*]] = getelementptr inbounds float, float* [[X]], i64 18
@@ -459,37 +402,6 @@ define float @f(float* nocapture readonly %x) {
 ; CHECK-NEXT:    [[ARRAYIDX_47:%.*]] = getelementptr inbounds float, float* [[X]], i64 47
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast float* [[ARRAYIDX_16]] to <32 x float>*
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <32 x float>, <32 x float>* [[TMP2]], align 4
-; CHECK-NEXT:    [[ADD_16:%.*]] = fadd fast float undef, [[ADD_15]]
-; CHECK-NEXT:    [[ADD_17:%.*]] = fadd fast float undef, [[ADD_16]]
-; CHECK-NEXT:    [[ADD_18:%.*]] = fadd fast float undef, [[ADD_17]]
-; CHECK-NEXT:    [[ADD_19:%.*]] = fadd fast float undef, [[ADD_18]]
-; CHECK-NEXT:    [[ADD_20:%.*]] = fadd fast float undef, [[ADD_19]]
-; CHECK-NEXT:    [[ADD_21:%.*]] = fadd fast float undef, [[ADD_20]]
-; CHECK-NEXT:    [[ADD_22:%.*]] = fadd fast float undef, [[ADD_21]]
-; CHECK-NEXT:    [[ADD_23:%.*]] = fadd fast float undef, [[ADD_22]]
-; CHECK-NEXT:    [[ADD_24:%.*]] = fadd fast float undef, [[ADD_23]]
-; CHECK-NEXT:    [[ADD_25:%.*]] = fadd fast float undef, [[ADD_24]]
-; CHECK-NEXT:    [[ADD_26:%.*]] = fadd fast float undef, [[ADD_25]]
-; CHECK-NEXT:    [[ADD_27:%.*]] = fadd fast float undef, [[ADD_26]]
-; CHECK-NEXT:    [[ADD_28:%.*]] = fadd fast float undef, [[ADD_27]]
-; CHECK-NEXT:    [[ADD_29:%.*]] = fadd fast float undef, [[ADD_28]]
-; CHECK-NEXT:    [[ADD_30:%.*]] = fadd fast float undef, [[ADD_29]]
-; CHECK-NEXT:    [[ADD_31:%.*]] = fadd fast float undef, [[ADD_30]]
-; CHECK-NEXT:    [[ADD_32:%.*]] = fadd fast float undef, [[ADD_31]]
-; CHECK-NEXT:    [[ADD_33:%.*]] = fadd fast float undef, [[ADD_32]]
-; CHECK-NEXT:    [[ADD_34:%.*]] = fadd fast float undef, [[ADD_33]]
-; CHECK-NEXT:    [[ADD_35:%.*]] = fadd fast float undef, [[ADD_34]]
-; CHECK-NEXT:    [[ADD_36:%.*]] = fadd fast float undef, [[ADD_35]]
-; CHECK-NEXT:    [[ADD_37:%.*]] = fadd fast float undef, [[ADD_36]]
-; CHECK-NEXT:    [[ADD_38:%.*]] = fadd fast float undef, [[ADD_37]]
-; CHECK-NEXT:    [[ADD_39:%.*]] = fadd fast float undef, [[ADD_38]]
-; CHECK-NEXT:    [[ADD_40:%.*]] = fadd fast float undef, [[ADD_39]]
-; CHECK-NEXT:    [[ADD_41:%.*]] = fadd fast float undef, [[ADD_40]]
-; CHECK-NEXT:    [[ADD_42:%.*]] = fadd fast float undef, [[ADD_41]]
-; CHECK-NEXT:    [[ADD_43:%.*]] = fadd fast float undef, [[ADD_42]]
-; CHECK-NEXT:    [[ADD_44:%.*]] = fadd fast float undef, [[ADD_43]]
-; CHECK-NEXT:    [[ADD_45:%.*]] = fadd fast float undef, [[ADD_44]]
-; CHECK-NEXT:    [[ADD_46:%.*]] = fadd fast float undef, [[ADD_45]]
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <32 x float> [[TMP3]], <32 x float> undef, <32 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = fadd fast <32 x float> [[TMP3]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <32 x float> [[BIN_RDX]], <32 x float> undef, <32 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -511,7 +423,6 @@ define float @f(float* nocapture readonly %x) {
 ; CHECK-NEXT:    [[BIN_RDX16:%.*]] = fadd fast <16 x float> [[BIN_RDX14]], [[RDX_SHUF15]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <16 x float> [[BIN_RDX16]], i32 0
 ; CHECK-NEXT:    [[OP_RDX:%.*]] = fadd fast float [[TMP4]], [[TMP5]]
-; CHECK-NEXT:    [[ADD_47:%.*]] = fadd fast float undef, [[ADD_46]]
 ; CHECK-NEXT:    ret float [[OP_RDX]]
 ;
 ; THRESHOLD-LABEL: @f(
@@ -533,21 +444,6 @@ define float @f(float* nocapture readonly %x) {
 ; THRESHOLD-NEXT:    [[ARRAYIDX_15:%.*]] = getelementptr inbounds float, float* [[X]], i64 15
 ; THRESHOLD-NEXT:    [[TMP0:%.*]] = bitcast float* [[X]] to <16 x float>*
 ; THRESHOLD-NEXT:    [[TMP1:%.*]] = load <16 x float>, <16 x float>* [[TMP0]], align 4
-; THRESHOLD-NEXT:    [[ADD_1:%.*]] = fadd fast float undef, undef
-; THRESHOLD-NEXT:    [[ADD_2:%.*]] = fadd fast float undef, [[ADD_1]]
-; THRESHOLD-NEXT:    [[ADD_3:%.*]] = fadd fast float undef, [[ADD_2]]
-; THRESHOLD-NEXT:    [[ADD_4:%.*]] = fadd fast float undef, [[ADD_3]]
-; THRESHOLD-NEXT:    [[ADD_5:%.*]] = fadd fast float undef, [[ADD_4]]
-; THRESHOLD-NEXT:    [[ADD_6:%.*]] = fadd fast float undef, [[ADD_5]]
-; THRESHOLD-NEXT:    [[ADD_7:%.*]] = fadd fast float undef, [[ADD_6]]
-; THRESHOLD-NEXT:    [[ADD_8:%.*]] = fadd fast float undef, [[ADD_7]]
-; THRESHOLD-NEXT:    [[ADD_9:%.*]] = fadd fast float undef, [[ADD_8]]
-; THRESHOLD-NEXT:    [[ADD_10:%.*]] = fadd fast float undef, [[ADD_9]]
-; THRESHOLD-NEXT:    [[ADD_11:%.*]] = fadd fast float undef, [[ADD_10]]
-; THRESHOLD-NEXT:    [[ADD_12:%.*]] = fadd fast float undef, [[ADD_11]]
-; THRESHOLD-NEXT:    [[ADD_13:%.*]] = fadd fast float undef, [[ADD_12]]
-; THRESHOLD-NEXT:    [[ADD_14:%.*]] = fadd fast float undef, [[ADD_13]]
-; THRESHOLD-NEXT:    [[ADD_15:%.*]] = fadd fast float undef, [[ADD_14]]
 ; THRESHOLD-NEXT:    [[ARRAYIDX_16:%.*]] = getelementptr inbounds float, float* [[X]], i64 16
 ; THRESHOLD-NEXT:    [[ARRAYIDX_17:%.*]] = getelementptr inbounds float, float* [[X]], i64 17
 ; THRESHOLD-NEXT:    [[ARRAYIDX_18:%.*]] = getelementptr inbounds float, float* [[X]], i64 18
@@ -582,37 +478,6 @@ define float @f(float* nocapture readonly %x) {
 ; THRESHOLD-NEXT:    [[ARRAYIDX_47:%.*]] = getelementptr inbounds float, float* [[X]], i64 47
 ; THRESHOLD-NEXT:    [[TMP2:%.*]] = bitcast float* [[ARRAYIDX_16]] to <32 x float>*
 ; THRESHOLD-NEXT:    [[TMP3:%.*]] = load <32 x float>, <32 x float>* [[TMP2]], align 4
-; THRESHOLD-NEXT:    [[ADD_16:%.*]] = fadd fast float undef, [[ADD_15]]
-; THRESHOLD-NEXT:    [[ADD_17:%.*]] = fadd fast float undef, [[ADD_16]]
-; THRESHOLD-NEXT:    [[ADD_18:%.*]] = fadd fast float undef, [[ADD_17]]
-; THRESHOLD-NEXT:    [[ADD_19:%.*]] = fadd fast float undef, [[ADD_18]]
-; THRESHOLD-NEXT:    [[ADD_20:%.*]] = fadd fast float undef, [[ADD_19]]
-; THRESHOLD-NEXT:    [[ADD_21:%.*]] = fadd fast float undef, [[ADD_20]]
-; THRESHOLD-NEXT:    [[ADD_22:%.*]] = fadd fast float undef, [[ADD_21]]
-; THRESHOLD-NEXT:    [[ADD_23:%.*]] = fadd fast float undef, [[ADD_22]]
-; THRESHOLD-NEXT:    [[ADD_24:%.*]] = fadd fast float undef, [[ADD_23]]
-; THRESHOLD-NEXT:    [[ADD_25:%.*]] = fadd fast float undef, [[ADD_24]]
-; THRESHOLD-NEXT:    [[ADD_26:%.*]] = fadd fast float undef, [[ADD_25]]
-; THRESHOLD-NEXT:    [[ADD_27:%.*]] = fadd fast float undef, [[ADD_26]]
-; THRESHOLD-NEXT:    [[ADD_28:%.*]] = fadd fast float undef, [[ADD_27]]
-; THRESHOLD-NEXT:    [[ADD_29:%.*]] = fadd fast float undef, [[ADD_28]]
-; THRESHOLD-NEXT:    [[ADD_30:%.*]] = fadd fast float undef, [[ADD_29]]
-; THRESHOLD-NEXT:    [[ADD_31:%.*]] = fadd fast float undef, [[ADD_30]]
-; THRESHOLD-NEXT:    [[ADD_32:%.*]] = fadd fast float undef, [[ADD_31]]
-; THRESHOLD-NEXT:    [[ADD_33:%.*]] = fadd fast float undef, [[ADD_32]]
-; THRESHOLD-NEXT:    [[ADD_34:%.*]] = fadd fast float undef, [[ADD_33]]
-; THRESHOLD-NEXT:    [[ADD_35:%.*]] = fadd fast float undef, [[ADD_34]]
-; THRESHOLD-NEXT:    [[ADD_36:%.*]] = fadd fast float undef, [[ADD_35]]
-; THRESHOLD-NEXT:    [[ADD_37:%.*]] = fadd fast float undef, [[ADD_36]]
-; THRESHOLD-NEXT:    [[ADD_38:%.*]] = fadd fast float undef, [[ADD_37]]
-; THRESHOLD-NEXT:    [[ADD_39:%.*]] = fadd fast float undef, [[ADD_38]]
-; THRESHOLD-NEXT:    [[ADD_40:%.*]] = fadd fast float undef, [[ADD_39]]
-; THRESHOLD-NEXT:    [[ADD_41:%.*]] = fadd fast float undef, [[ADD_40]]
-; THRESHOLD-NEXT:    [[ADD_42:%.*]] = fadd fast float undef, [[ADD_41]]
-; THRESHOLD-NEXT:    [[ADD_43:%.*]] = fadd fast float undef, [[ADD_42]]
-; THRESHOLD-NEXT:    [[ADD_44:%.*]] = fadd fast float undef, [[ADD_43]]
-; THRESHOLD-NEXT:    [[ADD_45:%.*]] = fadd fast float undef, [[ADD_44]]
-; THRESHOLD-NEXT:    [[ADD_46:%.*]] = fadd fast float undef, [[ADD_45]]
 ; THRESHOLD-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <32 x float> [[TMP3]], <32 x float> undef, <32 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; THRESHOLD-NEXT:    [[BIN_RDX:%.*]] = fadd fast <32 x float> [[TMP3]], [[RDX_SHUF]]
 ; THRESHOLD-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <32 x float> [[BIN_RDX]], <32 x float> undef, <32 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -634,7 +499,6 @@ define float @f(float* nocapture readonly %x) {
 ; THRESHOLD-NEXT:    [[BIN_RDX16:%.*]] = fadd fast <16 x float> [[BIN_RDX14]], [[RDX_SHUF15]]
 ; THRESHOLD-NEXT:    [[TMP5:%.*]] = extractelement <16 x float> [[BIN_RDX16]], i32 0
 ; THRESHOLD-NEXT:    [[OP_RDX:%.*]] = fadd fast float [[TMP4]], [[TMP5]]
-; THRESHOLD-NEXT:    [[ADD_47:%.*]] = fadd fast float undef, [[ADD_46]]
 ; THRESHOLD-NEXT:    ret float [[OP_RDX]]
 ;
   entry:
@@ -821,37 +685,6 @@ define float @f1(float* nocapture readonly %x, i32 %a, i32 %b) {
 ; CHECK-NEXT:    [[ARRAYIDX_31:%.*]] = getelementptr inbounds float, float* [[X]], i64 31
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast float* [[X]] to <32 x float>*
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x float>, <32 x float>* [[TMP0]], align 4
-; CHECK-NEXT:    [[ADD:%.*]] = fadd fast float undef, [[CONV]]
-; CHECK-NEXT:    [[ADD_1:%.*]] = fadd fast float undef, [[ADD]]
-; CHECK-NEXT:    [[ADD_2:%.*]] = fadd fast float undef, [[ADD_1]]
-; CHECK-NEXT:    [[ADD_3:%.*]] = fadd fast float undef, [[ADD_2]]
-; CHECK-NEXT:    [[ADD_4:%.*]] = fadd fast float undef, [[ADD_3]]
-; CHECK-NEXT:    [[ADD_5:%.*]] = fadd fast float undef, [[ADD_4]]
-; CHECK-NEXT:    [[ADD_6:%.*]] = fadd fast float undef, [[ADD_5]]
-; CHECK-NEXT:    [[ADD_7:%.*]] = fadd fast float undef, [[ADD_6]]
-; CHECK-NEXT:    [[ADD_8:%.*]] = fadd fast float undef, [[ADD_7]]
-; CHECK-NEXT:    [[ADD_9:%.*]] = fadd fast float undef, [[ADD_8]]
-; CHECK-NEXT:    [[ADD_10:%.*]] = fadd fast float undef, [[ADD_9]]
-; CHECK-NEXT:    [[ADD_11:%.*]] = fadd fast float undef, [[ADD_10]]
-; CHECK-NEXT:    [[ADD_12:%.*]] = fadd fast float undef, [[ADD_11]]
-; CHECK-NEXT:    [[ADD_13:%.*]] = fadd fast float undef, [[ADD_12]]
-; CHECK-NEXT:    [[ADD_14:%.*]] = fadd fast float undef, [[ADD_13]]
-; CHECK-NEXT:    [[ADD_15:%.*]] = fadd fast float undef, [[ADD_14]]
-; CHECK-NEXT:    [[ADD_16:%.*]] = fadd fast float undef, [[ADD_15]]
-; CHECK-NEXT:    [[ADD_17:%.*]] = fadd fast float undef, [[ADD_16]]
-; CHECK-NEXT:    [[ADD_18:%.*]] = fadd fast float undef, [[ADD_17]]
-; CHECK-NEXT:    [[ADD_19:%.*]] = fadd fast float undef, [[ADD_18]]
-; CHECK-NEXT:    [[ADD_20:%.*]] = fadd fast float undef, [[ADD_19]]
-; CHECK-NEXT:    [[ADD_21:%.*]] = fadd fast float undef, [[ADD_20]]
-; CHECK-NEXT:    [[ADD_22:%.*]] = fadd fast float undef, [[ADD_21]]
-; CHECK-NEXT:    [[ADD_23:%.*]] = fadd fast float undef, [[ADD_22]]
-; CHECK-NEXT:    [[ADD_24:%.*]] = fadd fast float undef, [[ADD_23]]
-; CHECK-NEXT:    [[ADD_25:%.*]] = fadd fast float undef, [[ADD_24]]
-; CHECK-NEXT:    [[ADD_26:%.*]] = fadd fast float undef, [[ADD_25]]
-; CHECK-NEXT:    [[ADD_27:%.*]] = fadd fast float undef, [[ADD_26]]
-; CHECK-NEXT:    [[ADD_28:%.*]] = fadd fast float undef, [[ADD_27]]
-; CHECK-NEXT:    [[ADD_29:%.*]] = fadd fast float undef, [[ADD_28]]
-; CHECK-NEXT:    [[ADD_30:%.*]] = fadd fast float undef, [[ADD_29]]
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <32 x float> [[TMP1]], <32 x float> undef, <32 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = fadd fast <32 x float> [[TMP1]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <32 x float> [[BIN_RDX]], <32 x float> undef, <32 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -864,7 +697,6 @@ define float @f1(float* nocapture readonly %x, i32 %a, i32 %b) {
 ; CHECK-NEXT:    [[BIN_RDX8:%.*]] = fadd fast <32 x float> [[BIN_RDX6]], [[RDX_SHUF7]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <32 x float> [[BIN_RDX8]], i32 0
 ; CHECK-NEXT:    [[OP_EXTRA:%.*]] = fadd fast float [[TMP2]], [[CONV]]
-; CHECK-NEXT:    [[ADD_31:%.*]] = fadd fast float undef, [[ADD_30]]
 ; CHECK-NEXT:    ret float [[OP_EXTRA]]
 ;
 ; THRESHOLD-LABEL: @f1(
@@ -904,37 +736,6 @@ define float @f1(float* nocapture readonly %x, i32 %a, i32 %b) {
 ; THRESHOLD-NEXT:    [[ARRAYIDX_31:%.*]] = getelementptr inbounds float, float* [[X]], i64 31
 ; THRESHOLD-NEXT:    [[TMP0:%.*]] = bitcast float* [[X]] to <32 x float>*
 ; THRESHOLD-NEXT:    [[TMP1:%.*]] = load <32 x float>, <32 x float>* [[TMP0]], align 4
-; THRESHOLD-NEXT:    [[ADD:%.*]] = fadd fast float undef, [[CONV]]
-; THRESHOLD-NEXT:    [[ADD_1:%.*]] = fadd fast float undef, [[ADD]]
-; THRESHOLD-NEXT:    [[ADD_2:%.*]] = fadd fast float undef, [[ADD_1]]
-; THRESHOLD-NEXT:    [[ADD_3:%.*]] = fadd fast float undef, [[ADD_2]]
-; THRESHOLD-NEXT:    [[ADD_4:%.*]] = fadd fast float undef, [[ADD_3]]
-; THRESHOLD-NEXT:    [[ADD_5:%.*]] = fadd fast float undef, [[ADD_4]]
-; THRESHOLD-NEXT:    [[ADD_6:%.*]] = fadd fast float undef, [[ADD_5]]
-; THRESHOLD-NEXT:    [[ADD_7:%.*]] = fadd fast float undef, [[ADD_6]]
-; THRESHOLD-NEXT:    [[ADD_8:%.*]] = fadd fast float undef, [[ADD_7]]
-; THRESHOLD-NEXT:    [[ADD_9:%.*]] = fadd fast float undef, [[ADD_8]]
-; THRESHOLD-NEXT:    [[ADD_10:%.*]] = fadd fast float undef, [[ADD_9]]
-; THRESHOLD-NEXT:    [[ADD_11:%.*]] = fadd fast float undef, [[ADD_10]]
-; THRESHOLD-NEXT:    [[ADD_12:%.*]] = fadd fast float undef, [[ADD_11]]
-; THRESHOLD-NEXT:    [[ADD_13:%.*]] = fadd fast float undef, [[ADD_12]]
-; THRESHOLD-NEXT:    [[ADD_14:%.*]] = fadd fast float undef, [[ADD_13]]
-; THRESHOLD-NEXT:    [[ADD_15:%.*]] = fadd fast float undef, [[ADD_14]]
-; THRESHOLD-NEXT:    [[ADD_16:%.*]] = fadd fast float undef, [[ADD_15]]
-; THRESHOLD-NEXT:    [[ADD_17:%.*]] = fadd fast float undef, [[ADD_16]]
-; THRESHOLD-NEXT:    [[ADD_18:%.*]] = fadd fast float undef, [[ADD_17]]
-; THRESHOLD-NEXT:    [[ADD_19:%.*]] = fadd fast float undef, [[ADD_18]]
-; THRESHOLD-NEXT:    [[ADD_20:%.*]] = fadd fast float undef, [[ADD_19]]
-; THRESHOLD-NEXT:    [[ADD_21:%.*]] = fadd fast float undef, [[ADD_20]]
-; THRESHOLD-NEXT:    [[ADD_22:%.*]] = fadd fast float undef, [[ADD_21]]
-; THRESHOLD-NEXT:    [[ADD_23:%.*]] = fadd fast float undef, [[ADD_22]]
-; THRESHOLD-NEXT:    [[ADD_24:%.*]] = fadd fast float undef, [[ADD_23]]
-; THRESHOLD-NEXT:    [[ADD_25:%.*]] = fadd fast float undef, [[ADD_24]]
-; THRESHOLD-NEXT:    [[ADD_26:%.*]] = fadd fast float undef, [[ADD_25]]
-; THRESHOLD-NEXT:    [[ADD_27:%.*]] = fadd fast float undef, [[ADD_26]]
-; THRESHOLD-NEXT:    [[ADD_28:%.*]] = fadd fast float undef, [[ADD_27]]
-; THRESHOLD-NEXT:    [[ADD_29:%.*]] = fadd fast float undef, [[ADD_28]]
-; THRESHOLD-NEXT:    [[ADD_30:%.*]] = fadd fast float undef, [[ADD_29]]
 ; THRESHOLD-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <32 x float> [[TMP1]], <32 x float> undef, <32 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; THRESHOLD-NEXT:    [[BIN_RDX:%.*]] = fadd fast <32 x float> [[TMP1]], [[RDX_SHUF]]
 ; THRESHOLD-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <32 x float> [[BIN_RDX]], <32 x float> undef, <32 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -947,7 +748,6 @@ define float @f1(float* nocapture readonly %x, i32 %a, i32 %b) {
 ; THRESHOLD-NEXT:    [[BIN_RDX8:%.*]] = fadd fast <32 x float> [[BIN_RDX6]], [[RDX_SHUF7]]
 ; THRESHOLD-NEXT:    [[TMP2:%.*]] = extractelement <32 x float> [[BIN_RDX8]], i32 0
 ; THRESHOLD-NEXT:    [[OP_EXTRA:%.*]] = fadd fast float [[TMP2]], [[CONV]]
-; THRESHOLD-NEXT:    [[ADD_31:%.*]] = fadd fast float undef, [[ADD_30]]
 ; THRESHOLD-NEXT:    ret float [[OP_EXTRA]]
 ;
   entry:
@@ -1058,17 +858,12 @@ define float @loadadd31(float* nocapture readonly %x) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = load float, float* [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr inbounds float, float* [[X]], i64 2
 ; CHECK-NEXT:    [[TMP1:%.*]] = load float, float* [[ARRAYIDX_1]], align 4
-; CHECK-NEXT:    [[ADD_1:%.*]] = fadd fast float [[TMP1]], [[TMP0]]
 ; CHECK-NEXT:    [[ARRAYIDX_2:%.*]] = getelementptr inbounds float, float* [[X]], i64 3
 ; CHECK-NEXT:    [[ARRAYIDX_3:%.*]] = getelementptr inbounds float, float* [[X]], i64 4
 ; CHECK-NEXT:    [[ARRAYIDX_4:%.*]] = getelementptr inbounds float, float* [[X]], i64 5
 ; CHECK-NEXT:    [[ARRAYIDX_5:%.*]] = getelementptr inbounds float, float* [[X]], i64 6
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast float* [[ARRAYIDX_2]] to <4 x float>*
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x float>, <4 x float>* [[TMP2]], align 4
-; CHECK-NEXT:    [[ADD_2:%.*]] = fadd fast float undef, [[ADD_1]]
-; CHECK-NEXT:    [[ADD_3:%.*]] = fadd fast float undef, [[ADD_2]]
-; CHECK-NEXT:    [[ADD_4:%.*]] = fadd fast float undef, [[ADD_3]]
-; CHECK-NEXT:    [[ADD_5:%.*]] = fadd fast float undef, [[ADD_4]]
 ; CHECK-NEXT:    [[ARRAYIDX_6:%.*]] = getelementptr inbounds float, float* [[X]], i64 7
 ; CHECK-NEXT:    [[ARRAYIDX_7:%.*]] = getelementptr inbounds float, float* [[X]], i64 8
 ; CHECK-NEXT:    [[ARRAYIDX_8:%.*]] = getelementptr inbounds float, float* [[X]], i64 9
@@ -1079,14 +874,6 @@ define float @loadadd31(float* nocapture readonly %x) {
 ; CHECK-NEXT:    [[ARRAYIDX_13:%.*]] = getelementptr inbounds float, float* [[X]], i64 14
 ; CHECK-NEXT:    [[TMP4:%.*]] = bitcast float* [[ARRAYIDX_6]] to <8 x float>*
 ; CHECK-NEXT:    [[TMP5:%.*]] = load <8 x float>, <8 x float>* [[TMP4]], align 4
-; CHECK-NEXT:    [[ADD_6:%.*]] = fadd fast float undef, [[ADD_5]]
-; CHECK-NEXT:    [[ADD_7:%.*]] = fadd fast float undef, [[ADD_6]]
-; CHECK-NEXT:    [[ADD_8:%.*]] = fadd fast float undef, [[ADD_7]]
-; CHECK-NEXT:    [[ADD_9:%.*]] = fadd fast float undef, [[ADD_8]]
-; CHECK-NEXT:    [[ADD_10:%.*]] = fadd fast float undef, [[ADD_9]]
-; CHECK-NEXT:    [[ADD_11:%.*]] = fadd fast float undef, [[ADD_10]]
-; CHECK-NEXT:    [[ADD_12:%.*]] = fadd fast float undef, [[ADD_11]]
-; CHECK-NEXT:    [[ADD_13:%.*]] = fadd fast float undef, [[ADD_12]]
 ; CHECK-NEXT:    [[ARRAYIDX_14:%.*]] = getelementptr inbounds float, float* [[X]], i64 15
 ; CHECK-NEXT:    [[ARRAYIDX_15:%.*]] = getelementptr inbounds float, float* [[X]], i64 16
 ; CHECK-NEXT:    [[ARRAYIDX_16:%.*]] = getelementptr inbounds float, float* [[X]], i64 17
@@ -1105,21 +892,6 @@ define float @loadadd31(float* nocapture readonly %x) {
 ; CHECK-NEXT:    [[ARRAYIDX_29:%.*]] = getelementptr inbounds float, float* [[X]], i64 30
 ; CHECK-NEXT:    [[TMP6:%.*]] = bitcast float* [[ARRAYIDX_14]] to <16 x float>*
 ; CHECK-NEXT:    [[TMP7:%.*]] = load <16 x float>, <16 x float>* [[TMP6]], align 4
-; CHECK-NEXT:    [[ADD_14:%.*]] = fadd fast float undef, [[ADD_13]]
-; CHECK-NEXT:    [[ADD_15:%.*]] = fadd fast float undef, [[ADD_14]]
-; CHECK-NEXT:    [[ADD_16:%.*]] = fadd fast float undef, [[ADD_15]]
-; CHECK-NEXT:    [[ADD_17:%.*]] = fadd fast float undef, [[ADD_16]]
-; CHECK-NEXT:    [[ADD_18:%.*]] = fadd fast float undef, [[ADD_17]]
-; CHECK-NEXT:    [[ADD_19:%.*]] = fadd fast float undef, [[ADD_18]]
-; CHECK-NEXT:    [[ADD_20:%.*]] = fadd fast float undef, [[ADD_19]]
-; CHECK-NEXT:    [[ADD_21:%.*]] = fadd fast float undef, [[ADD_20]]
-; CHECK-NEXT:    [[ADD_22:%.*]] = fadd fast float undef, [[ADD_21]]
-; CHECK-NEXT:    [[ADD_23:%.*]] = fadd fast float undef, [[ADD_22]]
-; CHECK-NEXT:    [[ADD_24:%.*]] = fadd fast float undef, [[ADD_23]]
-; CHECK-NEXT:    [[ADD_25:%.*]] = fadd fast float undef, [[ADD_24]]
-; CHECK-NEXT:    [[ADD_26:%.*]] = fadd fast float undef, [[ADD_25]]
-; CHECK-NEXT:    [[ADD_27:%.*]] = fadd fast float undef, [[ADD_26]]
-; CHECK-NEXT:    [[ADD_28:%.*]] = fadd fast float undef, [[ADD_27]]
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <16 x float> [[TMP7]], <16 x float> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = fadd fast <16 x float> [[TMP7]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <16 x float> [[BIN_RDX]], <16 x float> undef, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -1145,7 +917,6 @@ define float @loadadd31(float* nocapture readonly %x) {
 ; CHECK-NEXT:    [[OP_RDX17:%.*]] = fadd fast float [[OP_RDX]], [[TMP10]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = fadd fast float [[OP_RDX17]], [[TMP1]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = fadd fast float [[TMP11]], [[TMP0]]
-; CHECK-NEXT:    [[ADD_29:%.*]] = fadd fast float undef, [[ADD_28]]
 ; CHECK-NEXT:    ret float [[TMP12]]
 ;
 ; THRESHOLD-LABEL: @loadadd31(
@@ -1154,17 +925,12 @@ define float @loadadd31(float* nocapture readonly %x) {
 ; THRESHOLD-NEXT:    [[TMP0:%.*]] = load float, float* [[ARRAYIDX]], align 4
 ; THRESHOLD-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr inbounds float, float* [[X]], i64 2
 ; THRESHOLD-NEXT:    [[TMP1:%.*]] = load float, float* [[ARRAYIDX_1]], align 4
-; THRESHOLD-NEXT:    [[ADD_1:%.*]] = fadd fast float [[TMP1]], [[TMP0]]
 ; THRESHOLD-NEXT:    [[ARRAYIDX_2:%.*]] = getelementptr inbounds float, float* [[X]], i64 3
 ; THRESHOLD-NEXT:    [[ARRAYIDX_3:%.*]] = getelementptr inbounds float, float* [[X]], i64 4
 ; THRESHOLD-NEXT:    [[ARRAYIDX_4:%.*]] = getelementptr inbounds float, float* [[X]], i64 5
 ; THRESHOLD-NEXT:    [[ARRAYIDX_5:%.*]] = getelementptr inbounds float, float* [[X]], i64 6
 ; THRESHOLD-NEXT:    [[TMP2:%.*]] = bitcast float* [[ARRAYIDX_2]] to <4 x float>*
 ; THRESHOLD-NEXT:    [[TMP3:%.*]] = load <4 x float>, <4 x float>* [[TMP2]], align 4
-; THRESHOLD-NEXT:    [[ADD_2:%.*]] = fadd fast float undef, [[ADD_1]]
-; THRESHOLD-NEXT:    [[ADD_3:%.*]] = fadd fast float undef, [[ADD_2]]
-; THRESHOLD-NEXT:    [[ADD_4:%.*]] = fadd fast float undef, [[ADD_3]]
-; THRESHOLD-NEXT:    [[ADD_5:%.*]] = fadd fast float undef, [[ADD_4]]
 ; THRESHOLD-NEXT:    [[ARRAYIDX_6:%.*]] = getelementptr inbounds float, float* [[X]], i64 7
 ; THRESHOLD-NEXT:    [[ARRAYIDX_7:%.*]] = getelementptr inbounds float, float* [[X]], i64 8
 ; THRESHOLD-NEXT:    [[ARRAYIDX_8:%.*]] = getelementptr inbounds float, float* [[X]], i64 9
@@ -1175,14 +941,6 @@ define float @loadadd31(float* nocapture readonly %x) {
 ; THRESHOLD-NEXT:    [[ARRAYIDX_13:%.*]] = getelementptr inbounds float, float* [[X]], i64 14
 ; THRESHOLD-NEXT:    [[TMP4:%.*]] = bitcast float* [[ARRAYIDX_6]] to <8 x float>*
 ; THRESHOLD-NEXT:    [[TMP5:%.*]] = load <8 x float>, <8 x float>* [[TMP4]], align 4
-; THRESHOLD-NEXT:    [[ADD_6:%.*]] = fadd fast float undef, [[ADD_5]]
-; THRESHOLD-NEXT:    [[ADD_7:%.*]] = fadd fast float undef, [[ADD_6]]
-; THRESHOLD-NEXT:    [[ADD_8:%.*]] = fadd fast float undef, [[ADD_7]]
-; THRESHOLD-NEXT:    [[ADD_9:%.*]] = fadd fast float undef, [[ADD_8]]
-; THRESHOLD-NEXT:    [[ADD_10:%.*]] = fadd fast float undef, [[ADD_9]]
-; THRESHOLD-NEXT:    [[ADD_11:%.*]] = fadd fast float undef, [[ADD_10]]
-; THRESHOLD-NEXT:    [[ADD_12:%.*]] = fadd fast float undef, [[ADD_11]]
-; THRESHOLD-NEXT:    [[ADD_13:%.*]] = fadd fast float undef, [[ADD_12]]
 ; THRESHOLD-NEXT:    [[ARRAYIDX_14:%.*]] = getelementptr inbounds float, float* [[X]], i64 15
 ; THRESHOLD-NEXT:    [[ARRAYIDX_15:%.*]] = getelementptr inbounds float, float* [[X]], i64 16
 ; THRESHOLD-NEXT:    [[ARRAYIDX_16:%.*]] = getelementptr inbounds float, float* [[X]], i64 17
@@ -1201,21 +959,6 @@ define float @loadadd31(float* nocapture readonly %x) {
 ; THRESHOLD-NEXT:    [[ARRAYIDX_29:%.*]] = getelementptr inbounds float, float* [[X]], i64 30
 ; THRESHOLD-NEXT:    [[TMP6:%.*]] = bitcast float* [[ARRAYIDX_14]] to <16 x float>*
 ; THRESHOLD-NEXT:    [[TMP7:%.*]] = load <16 x float>, <16 x float>* [[TMP6]], align 4
-; THRESHOLD-NEXT:    [[ADD_14:%.*]] = fadd fast float undef, [[ADD_13]]
-; THRESHOLD-NEXT:    [[ADD_15:%.*]] = fadd fast float undef, [[ADD_14]]
-; THRESHOLD-NEXT:    [[ADD_16:%.*]] = fadd fast float undef, [[ADD_15]]
-; THRESHOLD-NEXT:    [[ADD_17:%.*]] = fadd fast float undef, [[ADD_16]]
-; THRESHOLD-NEXT:    [[ADD_18:%.*]] = fadd fast float undef, [[ADD_17]]
-; THRESHOLD-NEXT:    [[ADD_19:%.*]] = fadd fast float undef, [[ADD_18]]
-; THRESHOLD-NEXT:    [[ADD_20:%.*]] = fadd fast float undef, [[ADD_19]]
-; THRESHOLD-NEXT:    [[ADD_21:%.*]] = fadd fast float undef, [[ADD_20]]
-; THRESHOLD-NEXT:    [[ADD_22:%.*]] = fadd fast float undef, [[ADD_21]]
-; THRESHOLD-NEXT:    [[ADD_23:%.*]] = fadd fast float undef, [[ADD_22]]
-; THRESHOLD-NEXT:    [[ADD_24:%.*]] = fadd fast float undef, [[ADD_23]]
-; THRESHOLD-NEXT:    [[ADD_25:%.*]] = fadd fast float undef, [[ADD_24]]
-; THRESHOLD-NEXT:    [[ADD_26:%.*]] = fadd fast float undef, [[ADD_25]]
-; THRESHOLD-NEXT:    [[ADD_27:%.*]] = fadd fast float undef, [[ADD_26]]
-; THRESHOLD-NEXT:    [[ADD_28:%.*]] = fadd fast float undef, [[ADD_27]]
 ; THRESHOLD-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <16 x float> [[TMP7]], <16 x float> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; THRESHOLD-NEXT:    [[BIN_RDX:%.*]] = fadd fast <16 x float> [[TMP7]], [[RDX_SHUF]]
 ; THRESHOLD-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <16 x float> [[BIN_RDX]], <16 x float> undef, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -1241,7 +984,6 @@ define float @loadadd31(float* nocapture readonly %x) {
 ; THRESHOLD-NEXT:    [[OP_RDX17:%.*]] = fadd fast float [[OP_RDX]], [[TMP10]]
 ; THRESHOLD-NEXT:    [[TMP11:%.*]] = fadd fast float [[OP_RDX17]], [[TMP1]]
 ; THRESHOLD-NEXT:    [[TMP12:%.*]] = fadd fast float [[TMP11]], [[TMP0]]
-; THRESHOLD-NEXT:    [[ADD_29:%.*]] = fadd fast float undef, [[ADD_28]]
 ; THRESHOLD-NEXT:    ret float [[TMP12]]
 ;
   entry:
@@ -1352,14 +1094,6 @@ define float @extra_args(float* nocapture readonly %x, i32 %a, i32 %b) {
 ; CHECK-NEXT:    [[ARRAYIDX3_6:%.*]] = getelementptr inbounds float, float* [[X]], i64 7
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast float* [[X]] to <8 x float>*
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x float>, <8 x float>* [[TMP0]], align 4
-; CHECK-NEXT:    [[ADD1:%.*]] = fadd fast float undef, [[ADD]]
-; CHECK-NEXT:    [[ADD4:%.*]] = fadd fast float undef, [[ADD1]]
-; CHECK-NEXT:    [[ADD5:%.*]] = fadd fast float [[ADD4]], [[CONV]]
-; CHECK-NEXT:    [[ADD4_1:%.*]] = fadd fast float undef, [[ADD5]]
-; CHECK-NEXT:    [[ADD4_2:%.*]] = fadd fast float undef, [[ADD4_1]]
-; CHECK-NEXT:    [[ADD4_3:%.*]] = fadd fast float undef, [[ADD4_2]]
-; CHECK-NEXT:    [[ADD4_4:%.*]] = fadd fast float undef, [[ADD4_3]]
-; CHECK-NEXT:    [[ADD4_5:%.*]] = fadd fast float undef, [[ADD4_4]]
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = fadd fast <8 x float> [[TMP1]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x float> [[BIN_RDX]], <8 x float> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -1369,7 +1103,6 @@ define float @extra_args(float* nocapture readonly %x, i32 %a, i32 %b) {
 ; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <8 x float> [[BIN_RDX4]], i32 0
 ; CHECK-NEXT:    [[OP_EXTRA:%.*]] = fadd fast float [[TMP2]], [[ADD]]
 ; CHECK-NEXT:    [[OP_EXTRA5:%.*]] = fadd fast float [[OP_EXTRA]], [[CONV]]
-; CHECK-NEXT:    [[ADD4_6:%.*]] = fadd fast float undef, [[ADD4_5]]
 ; CHECK-NEXT:    ret float [[OP_EXTRA5]]
 ;
 ; THRESHOLD-LABEL: @extra_args(
@@ -1386,14 +1119,6 @@ define float @extra_args(float* nocapture readonly %x, i32 %a, i32 %b) {
 ; THRESHOLD-NEXT:    [[ARRAYIDX3_6:%.*]] = getelementptr inbounds float, float* [[X]], i64 7
 ; THRESHOLD-NEXT:    [[TMP0:%.*]] = bitcast float* [[X]] to <8 x float>*
 ; THRESHOLD-NEXT:    [[TMP1:%.*]] = load <8 x float>, <8 x float>* [[TMP0]], align 4
-; THRESHOLD-NEXT:    [[ADD1:%.*]] = fadd fast float undef, [[ADD]]
-; THRESHOLD-NEXT:    [[ADD4:%.*]] = fadd fast float undef, [[ADD1]]
-; THRESHOLD-NEXT:    [[ADD5:%.*]] = fadd fast float [[ADD4]], [[CONV]]
-; THRESHOLD-NEXT:    [[ADD4_1:%.*]] = fadd fast float undef, [[ADD5]]
-; THRESHOLD-NEXT:    [[ADD4_2:%.*]] = fadd fast float undef, [[ADD4_1]]
-; THRESHOLD-NEXT:    [[ADD4_3:%.*]] = fadd fast float undef, [[ADD4_2]]
-; THRESHOLD-NEXT:    [[ADD4_4:%.*]] = fadd fast float undef, [[ADD4_3]]
-; THRESHOLD-NEXT:    [[ADD4_5:%.*]] = fadd fast float undef, [[ADD4_4]]
 ; THRESHOLD-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
 ; THRESHOLD-NEXT:    [[BIN_RDX:%.*]] = fadd fast <8 x float> [[TMP1]], [[RDX_SHUF]]
 ; THRESHOLD-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x float> [[BIN_RDX]], <8 x float> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -1403,7 +1128,6 @@ define float @extra_args(float* nocapture readonly %x, i32 %a, i32 %b) {
 ; THRESHOLD-NEXT:    [[TMP2:%.*]] = extractelement <8 x float> [[BIN_RDX4]], i32 0
 ; THRESHOLD-NEXT:    [[OP_EXTRA:%.*]] = fadd fast float [[TMP2]], [[ADD]]
 ; THRESHOLD-NEXT:    [[OP_EXTRA5:%.*]] = fadd fast float [[OP_EXTRA]], [[CONV]]
-; THRESHOLD-NEXT:    [[ADD4_6:%.*]] = fadd fast float undef, [[ADD4_5]]
 ; THRESHOLD-NEXT:    ret float [[OP_EXTRA5]]
 ;
   entry:
@@ -1452,16 +1176,6 @@ define float @extra_args_same_several_times(float* nocapture readonly %x, i32 %a
 ; CHECK-NEXT:    [[ARRAYIDX3_6:%.*]] = getelementptr inbounds float, float* [[X]], i64 7
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast float* [[X]] to <8 x float>*
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x float>, <8 x float>* [[TMP0]], align 4
-; CHECK-NEXT:    [[ADD1:%.*]] = fadd fast float undef, [[ADD]]
-; CHECK-NEXT:    [[ADD4:%.*]] = fadd fast float undef, [[ADD1]]
-; CHECK-NEXT:    [[ADD41:%.*]] = fadd fast float [[ADD4]], 5.000000e+00
-; CHECK-NEXT:    [[ADD5:%.*]] = fadd fast float [[ADD41]], [[CONV]]
-; CHECK-NEXT:    [[ADD4_1:%.*]] = fadd fast float undef, [[ADD5]]
-; CHECK-NEXT:    [[ADD4_11:%.*]] = fadd fast float [[ADD4_1]], 5.000000e+00
-; CHECK-NEXT:    [[ADD4_2:%.*]] = fadd fast float undef, [[ADD4_11]]
-; CHECK-NEXT:    [[ADD4_3:%.*]] = fadd fast float undef, [[ADD4_2]]
-; CHECK-NEXT:    [[ADD4_4:%.*]] = fadd fast float undef, [[ADD4_3]]
-; CHECK-NEXT:    [[ADD4_5:%.*]] = fadd fast float undef, [[ADD4_4]]
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = fadd fast <8 x float> [[TMP1]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x float> [[BIN_RDX]], <8 x float> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -1473,7 +1187,6 @@ define float @extra_args_same_several_times(float* nocapture readonly %x, i32 %a
 ; CHECK-NEXT:    [[OP_EXTRA5:%.*]] = fadd fast float [[OP_EXTRA]], 5.000000e+00
 ; CHECK-NEXT:    [[OP_EXTRA6:%.*]] = fadd fast float [[OP_EXTRA5]], 5.000000e+00
 ; CHECK-NEXT:    [[OP_EXTRA7:%.*]] = fadd fast float [[OP_EXTRA6]], [[CONV]]
-; CHECK-NEXT:    [[ADD4_6:%.*]] = fadd fast float undef, [[ADD4_5]]
 ; CHECK-NEXT:    ret float [[OP_EXTRA7]]
 ;
 ; THRESHOLD-LABEL: @extra_args_same_several_times(
@@ -1490,16 +1203,6 @@ define float @extra_args_same_several_times(float* nocapture readonly %x, i32 %a
 ; THRESHOLD-NEXT:    [[ARRAYIDX3_6:%.*]] = getelementptr inbounds float, float* [[X]], i64 7
 ; THRESHOLD-NEXT:    [[TMP0:%.*]] = bitcast float* [[X]] to <8 x float>*
 ; THRESHOLD-NEXT:    [[TMP1:%.*]] = load <8 x float>, <8 x float>* [[TMP0]], align 4
-; THRESHOLD-NEXT:    [[ADD1:%.*]] = fadd fast float undef, [[ADD]]
-; THRESHOLD-NEXT:    [[ADD4:%.*]] = fadd fast float undef, [[ADD1]]
-; THRESHOLD-NEXT:    [[ADD41:%.*]] = fadd fast float [[ADD4]], 5.000000e+00
-; THRESHOLD-NEXT:    [[ADD5:%.*]] = fadd fast float [[ADD41]], [[CONV]]
-; THRESHOLD-NEXT:    [[ADD4_1:%.*]] = fadd fast float undef, [[ADD5]]
-; THRESHOLD-NEXT:    [[ADD4_11:%.*]] = fadd fast float [[ADD4_1]], 5.000000e+00
-; THRESHOLD-NEXT:    [[ADD4_2:%.*]] = fadd fast float undef, [[ADD4_11]]
-; THRESHOLD-NEXT:    [[ADD4_3:%.*]] = fadd fast float undef, [[ADD4_2]]
-; THRESHOLD-NEXT:    [[ADD4_4:%.*]] = fadd fast float undef, [[ADD4_3]]
-; THRESHOLD-NEXT:    [[ADD4_5:%.*]] = fadd fast float undef, [[ADD4_4]]
 ; THRESHOLD-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
 ; THRESHOLD-NEXT:    [[BIN_RDX:%.*]] = fadd fast <8 x float> [[TMP1]], [[RDX_SHUF]]
 ; THRESHOLD-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x float> [[BIN_RDX]], <8 x float> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -1511,7 +1214,6 @@ define float @extra_args_same_several_times(float* nocapture readonly %x, i32 %a
 ; THRESHOLD-NEXT:    [[OP_EXTRA5:%.*]] = fadd fast float [[OP_EXTRA]], 5.000000e+00
 ; THRESHOLD-NEXT:    [[OP_EXTRA6:%.*]] = fadd fast float [[OP_EXTRA5]], 5.000000e+00
 ; THRESHOLD-NEXT:    [[OP_EXTRA7:%.*]] = fadd fast float [[OP_EXTRA6]], [[CONV]]
-; THRESHOLD-NEXT:    [[ADD4_6:%.*]] = fadd fast float undef, [[ADD4_5]]
 ; THRESHOLD-NEXT:    ret float [[OP_EXTRA7]]
 ;
   entry:
@@ -1564,14 +1266,6 @@ define float @extra_args_no_replace(float* nocapture readonly %x, i32 %a, i32 %b
 ; CHECK-NEXT:    [[ARRAYIDX3_6:%.*]] = getelementptr inbounds float, float* [[X]], i64 7
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast float* [[X]] to <8 x float>*
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x float>, <8 x float>* [[TMP0]], align 4
-; CHECK-NEXT:    [[ADD1:%.*]] = fadd fast float undef, [[ADD]]
-; CHECK-NEXT:    [[ADD4:%.*]] = fadd fast float undef, [[ADD1]]
-; CHECK-NEXT:    [[ADD4_1:%.*]] = fadd fast float undef, [[ADD4]]
-; CHECK-NEXT:    [[ADD4_2:%.*]] = fadd fast float undef, [[ADD4_1]]
-; CHECK-NEXT:    [[ADD4_3:%.*]] = fadd fast float undef, [[ADD4_2]]
-; CHECK-NEXT:    [[ADD5:%.*]] = fadd fast float [[ADD4_3]], [[CONV]]
-; CHECK-NEXT:    [[ADD4_4:%.*]] = fadd fast float undef, [[ADD5]]
-; CHECK-NEXT:    [[ADD4_5:%.*]] = fadd fast float undef, [[ADD4_4]]
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = fadd fast <8 x float> [[TMP1]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x float> [[BIN_RDX]], <8 x float> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -1581,7 +1275,6 @@ define float @extra_args_no_replace(float* nocapture readonly %x, i32 %a, i32 %b
 ; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <8 x float> [[BIN_RDX4]], i32 0
 ; CHECK-NEXT:    [[OP_EXTRA:%.*]] = fadd fast float [[TMP2]], [[ADD]]
 ; CHECK-NEXT:    [[OP_EXTRA5:%.*]] = fadd fast float [[OP_EXTRA]], [[CONV]]
-; CHECK-NEXT:    [[ADD4_6:%.*]] = fadd fast float undef, [[ADD4_5]]
 ; CHECK-NEXT:    ret float [[OP_EXTRA5]]
 ;
 ; THRESHOLD-LABEL: @extra_args_no_replace(
@@ -1600,14 +1293,6 @@ define float @extra_args_no_replace(float* nocapture readonly %x, i32 %a, i32 %b
 ; THRESHOLD-NEXT:    [[ARRAYIDX3_6:%.*]] = getelementptr inbounds float, float* [[X]], i64 7
 ; THRESHOLD-NEXT:    [[TMP0:%.*]] = bitcast float* [[X]] to <8 x float>*
 ; THRESHOLD-NEXT:    [[TMP1:%.*]] = load <8 x float>, <8 x float>* [[TMP0]], align 4
-; THRESHOLD-NEXT:    [[ADD1:%.*]] = fadd fast float undef, [[ADD]]
-; THRESHOLD-NEXT:    [[ADD4:%.*]] = fadd fast float undef, [[ADD1]]
-; THRESHOLD-NEXT:    [[ADD4_1:%.*]] = fadd fast float undef, [[ADD4]]
-; THRESHOLD-NEXT:    [[ADD4_2:%.*]] = fadd fast float undef, [[ADD4_1]]
-; THRESHOLD-NEXT:    [[ADD4_3:%.*]] = fadd fast float undef, [[ADD4_2]]
-; THRESHOLD-NEXT:    [[ADD5:%.*]] = fadd fast float [[ADD4_3]], [[CONV]]
-; THRESHOLD-NEXT:    [[ADD4_4:%.*]] = fadd fast float undef, [[ADD5]]
-; THRESHOLD-NEXT:    [[ADD4_5:%.*]] = fadd fast float undef, [[ADD4_4]]
 ; THRESHOLD-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
 ; THRESHOLD-NEXT:    [[BIN_RDX:%.*]] = fadd fast <8 x float> [[TMP1]], [[RDX_SHUF]]
 ; THRESHOLD-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x float> [[BIN_RDX]], <8 x float> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -1617,7 +1302,6 @@ define float @extra_args_no_replace(float* nocapture readonly %x, i32 %a, i32 %b
 ; THRESHOLD-NEXT:    [[TMP2:%.*]] = extractelement <8 x float> [[BIN_RDX4]], i32 0
 ; THRESHOLD-NEXT:    [[OP_EXTRA:%.*]] = fadd fast float [[TMP2]], [[ADD]]
 ; THRESHOLD-NEXT:    [[OP_EXTRA5:%.*]] = fadd fast float [[OP_EXTRA]], [[CONV]]
-; THRESHOLD-NEXT:    [[ADD4_6:%.*]] = fadd fast float undef, [[ADD4_5]]
 ; THRESHOLD-NEXT:    ret float [[OP_EXTRA5]]
 ;
   entry:
@@ -1668,10 +1352,6 @@ define i32 @wobble(i32 %arg, i32 %bar) {
 ; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <4 x i32> [[TMP8]], i32 3
 ; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq <4 x i32> [[TMP8]], zeroinitializer
 ; CHECK-NEXT:    [[TMP11:%.*]] = sext <4 x i1> [[TMP10]] to <4 x i32>
-; CHECK-NEXT:    [[R1:%.*]] = add nuw i32 [[ARG]], undef
-; CHECK-NEXT:    [[R2:%.*]] = add nsw i32 [[R1]], undef
-; CHECK-NEXT:    [[R3:%.*]] = add nsw i32 [[R2]], undef
-; CHECK-NEXT:    [[R4:%.*]] = add nsw i32 [[R3]], undef
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[TMP11]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <4 x i32> [[TMP11]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x i32> [[BIN_RDX]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
@@ -1679,7 +1359,6 @@ define i32 @wobble(i32 %arg, i32 %bar) {
 ; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <4 x i32> [[BIN_RDX2]], i32 0
 ; CHECK-NEXT:    [[OP_EXTRA:%.*]] = add nuw i32 [[TMP12]], [[ARG]]
 ; CHECK-NEXT:    [[OP_EXTRA3:%.*]] = add nsw i32 [[OP_EXTRA]], [[TMP9]]
-; CHECK-NEXT:    [[R5:%.*]] = add nsw i32 [[R4]], [[TMP9]]
 ; CHECK-NEXT:    ret i32 [[OP_EXTRA3]]
 ;
 ; THRESHOLD-LABEL: @wobble(
@@ -1696,10 +1375,6 @@ define i32 @wobble(i32 %arg, i32 %bar) {
 ; THRESHOLD-NEXT:    [[TMP9:%.*]] = extractelement <4 x i32> [[TMP8]], i32 3
 ; THRESHOLD-NEXT:    [[TMP10:%.*]] = icmp eq <4 x i32> [[TMP8]], zeroinitializer
 ; THRESHOLD-NEXT:    [[TMP11:%.*]] = sext <4 x i1> [[TMP10]] to <4 x i32>
-; THRESHOLD-NEXT:    [[R1:%.*]] = add nuw i32 [[ARG]], undef
-; THRESHOLD-NEXT:    [[R2:%.*]] = add nsw i32 [[R1]], undef
-; THRESHOLD-NEXT:    [[R3:%.*]] = add nsw i32 [[R2]], undef
-; THRESHOLD-NEXT:    [[R4:%.*]] = add nsw i32 [[R3]], undef
 ; THRESHOLD-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[TMP11]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
 ; THRESHOLD-NEXT:    [[BIN_RDX:%.*]] = add <4 x i32> [[TMP11]], [[RDX_SHUF]]
 ; THRESHOLD-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x i32> [[BIN_RDX]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
@@ -1707,7 +1382,6 @@ define i32 @wobble(i32 %arg, i32 %bar) {
 ; THRESHOLD-NEXT:    [[TMP12:%.*]] = extractelement <4 x i32> [[BIN_RDX2]], i32 0
 ; THRESHOLD-NEXT:    [[OP_EXTRA:%.*]] = add nuw i32 [[TMP12]], [[ARG]]
 ; THRESHOLD-NEXT:    [[OP_EXTRA3:%.*]] = add nsw i32 [[OP_EXTRA]], [[TMP9]]
-; THRESHOLD-NEXT:    [[R5:%.*]] = add nsw i32 [[R4]], [[TMP9]]
 ; THRESHOLD-NEXT:    ret i32 [[OP_EXTRA3]]
 ;
   bb:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/horizontal-minmax.ll b/llvm/test/Transforms/SLPVectorizer/X86/horizontal-minmax.ll
index 8432b910d..40ea60bb5 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/horizontal-minmax.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/horizontal-minmax.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown-linux -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE
-; RUN: opt < %s -mtriple=x86_64-unknown-linux -mcpu=corei7-avx -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX
-; RUN: opt < %s -mtriple=x86_64-unknown-linux -mcpu=core-avx2 -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX2
-; RUN: opt < %s -mtriple=x86_64-unknown-linux -mcpu=skx -slp-vectorizer -S -slp-threshold=-100 | FileCheck %s --check-prefixes=CHECK,SKX
+; RUN: opt < %s -mtriple=x86_64-unknown-linux -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,DEFAULT,SSE
+; RUN: opt < %s -mtriple=x86_64-unknown-linux -mcpu=corei7-avx -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,DEFAULT,AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown-linux -mcpu=core-avx2 -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,DEFAULT,AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown-linux -mcpu=skx -slp-vectorizer -S -slp-threshold=-100 | FileCheck %s --check-prefixes=CHECK,THRESH
 
 @arr = local_unnamed_addr global [32 x i32] zeroinitializer, align 16
 @arr1 = local_unnamed_addr global [32 x float] zeroinitializer, align 16
@@ -12,19 +12,6 @@
 define i32 @maxi8(i32) {
 ; CHECK-LABEL: @maxi8(
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([32 x i32]* @arr to <8 x i32>*), align 16
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp sgt i32 undef, undef
-; CHECK-NEXT:    [[TMP4:%.*]] = select i1 [[TMP3]], i32 undef, i32 undef
-; CHECK-NEXT:    [[TMP5:%.*]] = icmp sgt i32 [[TMP4]], undef
-; CHECK-NEXT:    [[TMP6:%.*]] = select i1 [[TMP5]], i32 [[TMP4]], i32 undef
-; CHECK-NEXT:    [[TMP7:%.*]] = icmp sgt i32 [[TMP6]], undef
-; CHECK-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], i32 [[TMP6]], i32 undef
-; CHECK-NEXT:    [[TMP9:%.*]] = icmp sgt i32 [[TMP8]], undef
-; CHECK-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], i32 [[TMP8]], i32 undef
-; CHECK-NEXT:    [[TMP11:%.*]] = icmp sgt i32 [[TMP10]], undef
-; CHECK-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], i32 [[TMP10]], i32 undef
-; CHECK-NEXT:    [[TMP13:%.*]] = icmp sgt i32 [[TMP12]], undef
-; CHECK-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP12]], i32 undef
-; CHECK-NEXT:    [[TMP15:%.*]] = icmp sgt i32 [[TMP14]], undef
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x i32> [[TMP2]], <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = icmp sgt <8 x i32> [[TMP2]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <8 x i1> [[RDX_MINMAX_CMP]], <8 x i32> [[TMP2]], <8 x i32> [[RDX_SHUF]]
@@ -34,9 +21,8 @@ define i32 @maxi8(i32) {
 ; CHECK-NEXT:    [[RDX_SHUF4:%.*]] = shufflevector <8 x i32> [[RDX_MINMAX_SELECT3]], <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[RDX_MINMAX_CMP5:%.*]] = icmp sgt <8 x i32> [[RDX_MINMAX_SELECT3]], [[RDX_SHUF4]]
 ; CHECK-NEXT:    [[RDX_MINMAX_SELECT6:%.*]] = select <8 x i1> [[RDX_MINMAX_CMP5]], <8 x i32> [[RDX_MINMAX_SELECT3]], <8 x i32> [[RDX_SHUF4]]
-; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <8 x i32> [[RDX_MINMAX_SELECT6]], i32 0
-; CHECK-NEXT:    [[TMP17:%.*]] = select i1 [[TMP15]], i32 [[TMP14]], i32 undef
-; CHECK-NEXT:    ret i32 [[TMP16]]
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <8 x i32> [[RDX_MINMAX_SELECT6]], i32 0
+; CHECK-NEXT:    ret i32 [[TMP3]]
 ;
   %2 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 0), align 16
   %3 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 1), align 4
@@ -66,35 +52,6 @@ define i32 @maxi8(i32) {
 define i32 @maxi16(i32) {
 ; CHECK-LABEL: @maxi16(
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, <16 x i32>* bitcast ([32 x i32]* @arr to <16 x i32>*), align 16
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp sgt i32 undef, undef
-; CHECK-NEXT:    [[TMP4:%.*]] = select i1 [[TMP3]], i32 undef, i32 undef
-; CHECK-NEXT:    [[TMP5:%.*]] = icmp sgt i32 [[TMP4]], undef
-; CHECK-NEXT:    [[TMP6:%.*]] = select i1 [[TMP5]], i32 [[TMP4]], i32 undef
-; CHECK-NEXT:    [[TMP7:%.*]] = icmp sgt i32 [[TMP6]], undef
-; CHECK-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], i32 [[TMP6]], i32 undef
-; CHECK-NEXT:    [[TMP9:%.*]] = icmp sgt i32 [[TMP8]], undef
-; CHECK-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], i32 [[TMP8]], i32 undef
-; CHECK-NEXT:    [[TMP11:%.*]] = icmp sgt i32 [[TMP10]], undef
-; CHECK-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], i32 [[TMP10]], i32 undef
-; CHECK-NEXT:    [[TMP13:%.*]] = icmp sgt i32 [[TMP12]], undef
-; CHECK-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP12]], i32 undef
-; CHECK-NEXT:    [[TMP15:%.*]] = icmp sgt i32 [[TMP14]], undef
-; CHECK-NEXT:    [[TMP16:%.*]] = select i1 [[TMP15]], i32 [[TMP14]], i32 undef
-; CHECK-NEXT:    [[TMP17:%.*]] = icmp sgt i32 [[TMP16]], undef
-; CHECK-NEXT:    [[TMP18:%.*]] = select i1 [[TMP17]], i32 [[TMP16]], i32 undef
-; CHECK-NEXT:    [[TMP19:%.*]] = icmp sgt i32 [[TMP18]], undef
-; CHECK-NEXT:    [[TMP20:%.*]] = select i1 [[TMP19]], i32 [[TMP18]], i32 undef
-; CHECK-NEXT:    [[TMP21:%.*]] = icmp sgt i32 [[TMP20]], undef
-; CHECK-NEXT:    [[TMP22:%.*]] = select i1 [[TMP21]], i32 [[TMP20]], i32 undef
-; CHECK-NEXT:    [[TMP23:%.*]] = icmp sgt i32 [[TMP22]], undef
-; CHECK-NEXT:    [[TMP24:%.*]] = select i1 [[TMP23]], i32 [[TMP22]], i32 undef
-; CHECK-NEXT:    [[TMP25:%.*]] = icmp sgt i32 [[TMP24]], undef
-; CHECK-NEXT:    [[TMP26:%.*]] = select i1 [[TMP25]], i32 [[TMP24]], i32 undef
-; CHECK-NEXT:    [[TMP27:%.*]] = icmp sgt i32 [[TMP26]], undef
-; CHECK-NEXT:    [[TMP28:%.*]] = select i1 [[TMP27]], i32 [[TMP26]], i32 undef
-; CHECK-NEXT:    [[TMP29:%.*]] = icmp sgt i32 [[TMP28]], undef
-; CHECK-NEXT:    [[TMP30:%.*]] = select i1 [[TMP29]], i32 [[TMP28]], i32 undef
-; CHECK-NEXT:    [[TMP31:%.*]] = icmp sgt i32 [[TMP30]], undef
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <16 x i32> [[TMP2]], <16 x i32> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = icmp sgt <16 x i32> [[TMP2]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <16 x i1> [[RDX_MINMAX_CMP]], <16 x i32> [[TMP2]], <16 x i32> [[RDX_SHUF]]
@@ -107,9 +64,8 @@ define i32 @maxi16(i32) {
 ; CHECK-NEXT:    [[RDX_SHUF7:%.*]] = shufflevector <16 x i32> [[RDX_MINMAX_SELECT6]], <16 x i32> undef, <16 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[RDX_MINMAX_CMP8:%.*]] = icmp sgt <16 x i32> [[RDX_MINMAX_SELECT6]], [[RDX_SHUF7]]
 ; CHECK-NEXT:    [[RDX_MINMAX_SELECT9:%.*]] = select <16 x i1> [[RDX_MINMAX_CMP8]], <16 x i32> [[RDX_MINMAX_SELECT6]], <16 x i32> [[RDX_SHUF7]]
-; CHECK-NEXT:    [[TMP32:%.*]] = extractelement <16 x i32> [[RDX_MINMAX_SELECT9]], i32 0
-; CHECK-NEXT:    [[TMP33:%.*]] = select i1 [[TMP31]], i32 [[TMP30]], i32 undef
-; CHECK-NEXT:    ret i32 [[TMP32]]
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <16 x i32> [[RDX_MINMAX_SELECT9]], i32 0
+; CHECK-NEXT:    ret i32 [[TMP3]]
 ;
   %2 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 0), align 16
   %3 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 1), align 4
@@ -163,67 +119,6 @@ define i32 @maxi16(i32) {
 define i32 @maxi32(i32) {
 ; CHECK-LABEL: @maxi32(
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i32>, <32 x i32>* bitcast ([32 x i32]* @arr to <32 x i32>*), align 16
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp sgt i32 undef, undef
-; CHECK-NEXT:    [[TMP4:%.*]] = select i1 [[TMP3]], i32 undef, i32 undef
-; CHECK-NEXT:    [[TMP5:%.*]] = icmp sgt i32 [[TMP4]], undef
-; CHECK-NEXT:    [[TMP6:%.*]] = select i1 [[TMP5]], i32 [[TMP4]], i32 undef
-; CHECK-NEXT:    [[TMP7:%.*]] = icmp sgt i32 [[TMP6]], undef
-; CHECK-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], i32 [[TMP6]], i32 undef
-; CHECK-NEXT:    [[TMP9:%.*]] = icmp sgt i32 [[TMP8]], undef
-; CHECK-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], i32 [[TMP8]], i32 undef
-; CHECK-NEXT:    [[TMP11:%.*]] = icmp sgt i32 [[TMP10]], undef
-; CHECK-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], i32 [[TMP10]], i32 undef
-; CHECK-NEXT:    [[TMP13:%.*]] = icmp sgt i32 [[TMP12]], undef
-; CHECK-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP12]], i32 undef
-; CHECK-NEXT:    [[TMP15:%.*]] = icmp sgt i32 [[TMP14]], undef
-; CHECK-NEXT:    [[TMP16:%.*]] = select i1 [[TMP15]], i32 [[TMP14]], i32 undef
-; CHECK-NEXT:    [[TMP17:%.*]] = icmp sgt i32 [[TMP16]], undef
-; CHECK-NEXT:    [[TMP18:%.*]] = select i1 [[TMP17]], i32 [[TMP16]], i32 undef
-; CHECK-NEXT:    [[TMP19:%.*]] = icmp sgt i32 [[TMP18]], undef
-; CHECK-NEXT:    [[TMP20:%.*]] = select i1 [[TMP19]], i32 [[TMP18]], i32 undef
-; CHECK-NEXT:    [[TMP21:%.*]] = icmp sgt i32 [[TMP20]], undef
-; CHECK-NEXT:    [[TMP22:%.*]] = select i1 [[TMP21]], i32 [[TMP20]], i32 undef
-; CHECK-NEXT:    [[TMP23:%.*]] = icmp sgt i32 [[TMP22]], undef
-; CHECK-NEXT:    [[TMP24:%.*]] = select i1 [[TMP23]], i32 [[TMP22]], i32 undef
-; CHECK-NEXT:    [[TMP25:%.*]] = icmp sgt i32 [[TMP24]], undef
-; CHECK-NEXT:    [[TMP26:%.*]] = select i1 [[TMP25]], i32 [[TMP24]], i32 undef
-; CHECK-NEXT:    [[TMP27:%.*]] = icmp sgt i32 [[TMP26]], undef
-; CHECK-NEXT:    [[TMP28:%.*]] = select i1 [[TMP27]], i32 [[TMP26]], i32 undef
-; CHECK-NEXT:    [[TMP29:%.*]] = icmp sgt i32 [[TMP28]], undef
-; CHECK-NEXT:    [[TMP30:%.*]] = select i1 [[TMP29]], i32 [[TMP28]], i32 undef
-; CHECK-NEXT:    [[TMP31:%.*]] = icmp sgt i32 [[TMP30]], undef
-; CHECK-NEXT:    [[TMP32:%.*]] = select i1 [[TMP31]], i32 [[TMP30]], i32 undef
-; CHECK-NEXT:    [[TMP33:%.*]] = icmp sgt i32 [[TMP32]], undef
-; CHECK-NEXT:    [[TMP34:%.*]] = select i1 [[TMP33]], i32 [[TMP32]], i32 undef
-; CHECK-NEXT:    [[TMP35:%.*]] = icmp sgt i32 [[TMP34]], undef
-; CHECK-NEXT:    [[TMP36:%.*]] = select i1 [[TMP35]], i32 [[TMP34]], i32 undef
-; CHECK-NEXT:    [[TMP37:%.*]] = icmp sgt i32 [[TMP36]], undef
-; CHECK-NEXT:    [[TMP38:%.*]] = select i1 [[TMP37]], i32 [[TMP36]], i32 undef
-; CHECK-NEXT:    [[TMP39:%.*]] = icmp sgt i32 [[TMP38]], undef
-; CHECK-NEXT:    [[TMP40:%.*]] = select i1 [[TMP39]], i32 [[TMP38]], i32 undef
-; CHECK-NEXT:    [[TMP41:%.*]] = icmp sgt i32 [[TMP40]], undef
-; CHECK-NEXT:    [[TMP42:%.*]] = select i1 [[TMP41]], i32 [[TMP40]], i32 undef
-; CHECK-NEXT:    [[TMP43:%.*]] = icmp sgt i32 [[TMP42]], undef
-; CHECK-NEXT:    [[TMP44:%.*]] = select i1 [[TMP43]], i32 [[TMP42]], i32 undef
-; CHECK-NEXT:    [[TMP45:%.*]] = icmp sgt i32 [[TMP44]], undef
-; CHECK-NEXT:    [[TMP46:%.*]] = select i1 [[TMP45]], i32 [[TMP44]], i32 undef
-; CHECK-NEXT:    [[TMP47:%.*]] = icmp sgt i32 [[TMP46]], undef
-; CHECK-NEXT:    [[TMP48:%.*]] = select i1 [[TMP47]], i32 [[TMP46]], i32 undef
-; CHECK-NEXT:    [[TMP49:%.*]] = icmp sgt i32 [[TMP48]], undef
-; CHECK-NEXT:    [[TMP50:%.*]] = select i1 [[TMP49]], i32 [[TMP48]], i32 undef
-; CHECK-NEXT:    [[TMP51:%.*]] = icmp sgt i32 [[TMP50]], undef
-; CHECK-NEXT:    [[TMP52:%.*]] = select i1 [[TMP51]], i32 [[TMP50]], i32 undef
-; CHECK-NEXT:    [[TMP53:%.*]] = icmp sgt i32 [[TMP52]], undef
-; CHECK-NEXT:    [[TMP54:%.*]] = select i1 [[TMP53]], i32 [[TMP52]], i32 undef
-; CHECK-NEXT:    [[TMP55:%.*]] = icmp sgt i32 [[TMP54]], undef
-; CHECK-NEXT:    [[TMP56:%.*]] = select i1 [[TMP55]], i32 [[TMP54]], i32 undef
-; CHECK-NEXT:    [[TMP57:%.*]] = icmp sgt i32 [[TMP56]], undef
-; CHECK-NEXT:    [[TMP58:%.*]] = select i1 [[TMP57]], i32 [[TMP56]], i32 undef
-; CHECK-NEXT:    [[TMP59:%.*]] = icmp sgt i32 [[TMP58]], undef
-; CHECK-NEXT:    [[TMP60:%.*]] = select i1 [[TMP59]], i32 [[TMP58]], i32 undef
-; CHECK-NEXT:    [[TMP61:%.*]] = icmp sgt i32 [[TMP60]], undef
-; CHECK-NEXT:    [[TMP62:%.*]] = select i1 [[TMP61]], i32 [[TMP60]], i32 undef
-; CHECK-NEXT:    [[TMP63:%.*]] = icmp sgt i32 [[TMP62]], undef
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <32 x i32> [[TMP2]], <32 x i32> undef, <32 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = icmp sgt <32 x i32> [[TMP2]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <32 x i1> [[RDX_MINMAX_CMP]], <32 x i32> [[TMP2]], <32 x i32> [[RDX_SHUF]]
@@ -239,9 +134,8 @@ define i32 @maxi32(i32) {
 ; CHECK-NEXT:    [[RDX_SHUF10:%.*]] = shufflevector <32 x i32> [[RDX_MINMAX_SELECT9]], <32 x i32> undef, <32 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[RDX_MINMAX_CMP11:%.*]] = icmp sgt <32 x i32> [[RDX_MINMAX_SELECT9]], [[RDX_SHUF10]]
 ; CHECK-NEXT:    [[RDX_MINMAX_SELECT12:%.*]] = select <32 x i1> [[RDX_MINMAX_CMP11]], <32 x i32> [[RDX_MINMAX_SELECT9]], <32 x i32> [[RDX_SHUF10]]
-; CHECK-NEXT:    [[TMP64:%.*]] = extractelement <32 x i32> [[RDX_MINMAX_SELECT12]], i32 0
-; CHECK-NEXT:    [[TMP65:%.*]] = select i1 [[TMP63]], i32 [[TMP62]], i32 undef
-; CHECK-NEXT:    ret i32 [[TMP64]]
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <32 x i32> [[RDX_MINMAX_SELECT12]], i32 0
+; CHECK-NEXT:    ret i32 [[TMP3]]
 ;
   %2 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 0), align 16
   %3 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 1), align 4
@@ -343,19 +237,6 @@ define i32 @maxi32(i32) {
 define float @maxf8(float) {
 ; CHECK-LABEL: @maxf8(
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x float>, <8 x float>* bitcast ([32 x float]* @arr1 to <8 x float>*), align 16
-; CHECK-NEXT:    [[TMP3:%.*]] = fcmp fast ogt float undef, undef
-; CHECK-NEXT:    [[TMP4:%.*]] = select i1 [[TMP3]], float undef, float undef
-; CHECK-NEXT:    [[TMP5:%.*]] = fcmp fast ogt float [[TMP4]], undef
-; CHECK-NEXT:    [[TMP6:%.*]] = select i1 [[TMP5]], float [[TMP4]], float undef
-; CHECK-NEXT:    [[TMP7:%.*]] = fcmp fast ogt float [[TMP6]], undef
-; CHECK-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], float [[TMP6]], float undef
-; CHECK-NEXT:    [[TMP9:%.*]] = fcmp fast ogt float [[TMP8]], undef
-; CHECK-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], float [[TMP8]], float undef
-; CHECK-NEXT:    [[TMP11:%.*]] = fcmp fast ogt float [[TMP10]], undef
-; CHECK-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], float [[TMP10]], float undef
-; CHECK-NEXT:    [[TMP13:%.*]] = fcmp fast ogt float [[TMP12]], undef
-; CHECK-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], float [[TMP12]], float undef
-; CHECK-NEXT:    [[TMP15:%.*]] = fcmp fast ogt float [[TMP14]], undef
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x float> [[TMP2]], <8 x float> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp fast ogt <8 x float> [[TMP2]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <8 x i1> [[RDX_MINMAX_CMP]], <8 x float> [[TMP2]], <8 x float> [[RDX_SHUF]]
@@ -365,9 +246,8 @@ define float @maxf8(float) {
 ; CHECK-NEXT:    [[RDX_SHUF4:%.*]] = shufflevector <8 x float> [[RDX_MINMAX_SELECT3]], <8 x float> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[RDX_MINMAX_CMP5:%.*]] = fcmp fast ogt <8 x float> [[RDX_MINMAX_SELECT3]], [[RDX_SHUF4]]
 ; CHECK-NEXT:    [[RDX_MINMAX_SELECT6:%.*]] = select <8 x i1> [[RDX_MINMAX_CMP5]], <8 x float> [[RDX_MINMAX_SELECT3]], <8 x float> [[RDX_SHUF4]]
-; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <8 x float> [[RDX_MINMAX_SELECT6]], i32 0
-; CHECK-NEXT:    [[TMP17:%.*]] = select i1 [[TMP15]], float [[TMP14]], float undef
-; CHECK-NEXT:    ret float [[TMP16]]
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <8 x float> [[RDX_MINMAX_SELECT6]], i32 0
+; CHECK-NEXT:    ret float [[TMP3]]
 ;
   %2 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 0), align 16
   %3 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 1), align 4
@@ -397,35 +277,6 @@ define float @maxf8(float) {
 define float @maxf16(float) {
 ; CHECK-LABEL: @maxf16(
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x float>, <16 x float>* bitcast ([32 x float]* @arr1 to <16 x float>*), align 16
-; CHECK-NEXT:    [[TMP3:%.*]] = fcmp fast ogt float undef, undef
-; CHECK-NEXT:    [[TMP4:%.*]] = select i1 [[TMP3]], float undef, float undef
-; CHECK-NEXT:    [[TMP5:%.*]] = fcmp fast ogt float [[TMP4]], undef
-; CHECK-NEXT:    [[TMP6:%.*]] = select i1 [[TMP5]], float [[TMP4]], float undef
-; CHECK-NEXT:    [[TMP7:%.*]] = fcmp fast ogt float [[TMP6]], undef
-; CHECK-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], float [[TMP6]], float undef
-; CHECK-NEXT:    [[TMP9:%.*]] = fcmp fast ogt float [[TMP8]], undef
-; CHECK-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], float [[TMP8]], float undef
-; CHECK-NEXT:    [[TMP11:%.*]] = fcmp fast ogt float [[TMP10]], undef
-; CHECK-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], float [[TMP10]], float undef
-; CHECK-NEXT:    [[TMP13:%.*]] = fcmp fast ogt float [[TMP12]], undef
-; CHECK-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], float [[TMP12]], float undef
-; CHECK-NEXT:    [[TMP15:%.*]] = fcmp fast ogt float [[TMP14]], undef
-; CHECK-NEXT:    [[TMP16:%.*]] = select i1 [[TMP15]], float [[TMP14]], float undef
-; CHECK-NEXT:    [[TMP17:%.*]] = fcmp fast ogt float [[TMP16]], undef
-; CHECK-NEXT:    [[TMP18:%.*]] = select i1 [[TMP17]], float [[TMP16]], float undef
-; CHECK-NEXT:    [[TMP19:%.*]] = fcmp fast ogt float [[TMP18]], undef
-; CHECK-NEXT:    [[TMP20:%.*]] = select i1 [[TMP19]], float [[TMP18]], float undef
-; CHECK-NEXT:    [[TMP21:%.*]] = fcmp fast ogt float [[TMP20]], undef
-; CHECK-NEXT:    [[TMP22:%.*]] = select i1 [[TMP21]], float [[TMP20]], float undef
-; CHECK-NEXT:    [[TMP23:%.*]] = fcmp fast ogt float [[TMP22]], undef
-; CHECK-NEXT:    [[TMP24:%.*]] = select i1 [[TMP23]], float [[TMP22]], float undef
-; CHECK-NEXT:    [[TMP25:%.*]] = fcmp fast ogt float [[TMP24]], undef
-; CHECK-NEXT:    [[TMP26:%.*]] = select i1 [[TMP25]], float [[TMP24]], float undef
-; CHECK-NEXT:    [[TMP27:%.*]] = fcmp fast ogt float [[TMP26]], undef
-; CHECK-NEXT:    [[TMP28:%.*]] = select i1 [[TMP27]], float [[TMP26]], float undef
-; CHECK-NEXT:    [[TMP29:%.*]] = fcmp fast ogt float [[TMP28]], undef
-; CHECK-NEXT:    [[TMP30:%.*]] = select i1 [[TMP29]], float [[TMP28]], float undef
-; CHECK-NEXT:    [[TMP31:%.*]] = fcmp fast ogt float [[TMP30]], undef
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <16 x float> [[TMP2]], <16 x float> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp fast ogt <16 x float> [[TMP2]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <16 x i1> [[RDX_MINMAX_CMP]], <16 x float> [[TMP2]], <16 x float> [[RDX_SHUF]]
@@ -438,9 +289,8 @@ define float @maxf16(float) {
 ; CHECK-NEXT:    [[RDX_SHUF7:%.*]] = shufflevector <16 x float> [[RDX_MINMAX_SELECT6]], <16 x float> undef, <16 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[RDX_MINMAX_CMP8:%.*]] = fcmp fast ogt <16 x float> [[RDX_MINMAX_SELECT6]], [[RDX_SHUF7]]
 ; CHECK-NEXT:    [[RDX_MINMAX_SELECT9:%.*]] = select <16 x i1> [[RDX_MINMAX_CMP8]], <16 x float> [[RDX_MINMAX_SELECT6]], <16 x float> [[RDX_SHUF7]]
-; CHECK-NEXT:    [[TMP32:%.*]] = extractelement <16 x float> [[RDX_MINMAX_SELECT9]], i32 0
-; CHECK-NEXT:    [[TMP33:%.*]] = select i1 [[TMP31]], float [[TMP30]], float undef
-; CHECK-NEXT:    ret float [[TMP32]]
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <16 x float> [[RDX_MINMAX_SELECT9]], i32 0
+; CHECK-NEXT:    ret float [[TMP3]]
 ;
   %2 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 0), align 16
   %3 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 1), align 4
@@ -494,67 +344,6 @@ define float @maxf16(float) {
 define float @maxf32(float) {
 ; CHECK-LABEL: @maxf32(
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x float>, <32 x float>* bitcast ([32 x float]* @arr1 to <32 x float>*), align 16
-; CHECK-NEXT:    [[TMP3:%.*]] = fcmp fast ogt float undef, undef
-; CHECK-NEXT:    [[TMP4:%.*]] = select i1 [[TMP3]], float undef, float undef
-; CHECK-NEXT:    [[TMP5:%.*]] = fcmp fast ogt float [[TMP4]], undef
-; CHECK-NEXT:    [[TMP6:%.*]] = select i1 [[TMP5]], float [[TMP4]], float undef
-; CHECK-NEXT:    [[TMP7:%.*]] = fcmp fast ogt float [[TMP6]], undef
-; CHECK-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], float [[TMP6]], float undef
-; CHECK-NEXT:    [[TMP9:%.*]] = fcmp fast ogt float [[TMP8]], undef
-; CHECK-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], float [[TMP8]], float undef
-; CHECK-NEXT:    [[TMP11:%.*]] = fcmp fast ogt float [[TMP10]], undef
-; CHECK-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], float [[TMP10]], float undef
-; CHECK-NEXT:    [[TMP13:%.*]] = fcmp fast ogt float [[TMP12]], undef
-; CHECK-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], float [[TMP12]], float undef
-; CHECK-NEXT:    [[TMP15:%.*]] = fcmp fast ogt float [[TMP14]], undef
-; CHECK-NEXT:    [[TMP16:%.*]] = select i1 [[TMP15]], float [[TMP14]], float undef
-; CHECK-NEXT:    [[TMP17:%.*]] = fcmp fast ogt float [[TMP16]], undef
-; CHECK-NEXT:    [[TMP18:%.*]] = select i1 [[TMP17]], float [[TMP16]], float undef
-; CHECK-NEXT:    [[TMP19:%.*]] = fcmp fast ogt float [[TMP18]], undef
-; CHECK-NEXT:    [[TMP20:%.*]] = select i1 [[TMP19]], float [[TMP18]], float undef
-; CHECK-NEXT:    [[TMP21:%.*]] = fcmp fast ogt float [[TMP20]], undef
-; CHECK-NEXT:    [[TMP22:%.*]] = select i1 [[TMP21]], float [[TMP20]], float undef
-; CHECK-NEXT:    [[TMP23:%.*]] = fcmp fast ogt float [[TMP22]], undef
-; CHECK-NEXT:    [[TMP24:%.*]] = select i1 [[TMP23]], float [[TMP22]], float undef
-; CHECK-NEXT:    [[TMP25:%.*]] = fcmp fast ogt float [[TMP24]], undef
-; CHECK-NEXT:    [[TMP26:%.*]] = select i1 [[TMP25]], float [[TMP24]], float undef
-; CHECK-NEXT:    [[TMP27:%.*]] = fcmp fast ogt float [[TMP26]], undef
-; CHECK-NEXT:    [[TMP28:%.*]] = select i1 [[TMP27]], float [[TMP26]], float undef
-; CHECK-NEXT:    [[TMP29:%.*]] = fcmp fast ogt float [[TMP28]], undef
-; CHECK-NEXT:    [[TMP30:%.*]] = select i1 [[TMP29]], float [[TMP28]], float undef
-; CHECK-NEXT:    [[TMP31:%.*]] = fcmp fast ogt float [[TMP30]], undef
-; CHECK-NEXT:    [[TMP32:%.*]] = select i1 [[TMP31]], float [[TMP30]], float undef
-; CHECK-NEXT:    [[TMP33:%.*]] = fcmp fast ogt float [[TMP32]], undef
-; CHECK-NEXT:    [[TMP34:%.*]] = select i1 [[TMP33]], float [[TMP32]], float undef
-; CHECK-NEXT:    [[TMP35:%.*]] = fcmp fast ogt float [[TMP34]], undef
-; CHECK-NEXT:    [[TMP36:%.*]] = select i1 [[TMP35]], float [[TMP34]], float undef
-; CHECK-NEXT:    [[TMP37:%.*]] = fcmp fast ogt float [[TMP36]], undef
-; CHECK-NEXT:    [[TMP38:%.*]] = select i1 [[TMP37]], float [[TMP36]], float undef
-; CHECK-NEXT:    [[TMP39:%.*]] = fcmp fast ogt float [[TMP38]], undef
-; CHECK-NEXT:    [[TMP40:%.*]] = select i1 [[TMP39]], float [[TMP38]], float undef
-; CHECK-NEXT:    [[TMP41:%.*]] = fcmp fast ogt float [[TMP40]], undef
-; CHECK-NEXT:    [[TMP42:%.*]] = select i1 [[TMP41]], float [[TMP40]], float undef
-; CHECK-NEXT:    [[TMP43:%.*]] = fcmp fast ogt float [[TMP42]], undef
-; CHECK-NEXT:    [[TMP44:%.*]] = select i1 [[TMP43]], float [[TMP42]], float undef
-; CHECK-NEXT:    [[TMP45:%.*]] = fcmp fast ogt float [[TMP44]], undef
-; CHECK-NEXT:    [[TMP46:%.*]] = select i1 [[TMP45]], float [[TMP44]], float undef
-; CHECK-NEXT:    [[TMP47:%.*]] = fcmp fast ogt float [[TMP46]], undef
-; CHECK-NEXT:    [[TMP48:%.*]] = select i1 [[TMP47]], float [[TMP46]], float undef
-; CHECK-NEXT:    [[TMP49:%.*]] = fcmp fast ogt float [[TMP48]], undef
-; CHECK-NEXT:    [[TMP50:%.*]] = select i1 [[TMP49]], float [[TMP48]], float undef
-; CHECK-NEXT:    [[TMP51:%.*]] = fcmp fast ogt float [[TMP50]], undef
-; CHECK-NEXT:    [[TMP52:%.*]] = select i1 [[TMP51]], float [[TMP50]], float undef
-; CHECK-NEXT:    [[TMP53:%.*]] = fcmp fast ogt float [[TMP52]], undef
-; CHECK-NEXT:    [[TMP54:%.*]] = select i1 [[TMP53]], float [[TMP52]], float undef
-; CHECK-NEXT:    [[TMP55:%.*]] = fcmp fast ogt float [[TMP54]], undef
-; CHECK-NEXT:    [[TMP56:%.*]] = select i1 [[TMP55]], float [[TMP54]], float undef
-; CHECK-NEXT:    [[TMP57:%.*]] = fcmp fast ogt float [[TMP56]], undef
-; CHECK-NEXT:    [[TMP58:%.*]] = select i1 [[TMP57]], float [[TMP56]], float undef
-; CHECK-NEXT:    [[TMP59:%.*]] = fcmp fast ogt float [[TMP58]], undef
-; CHECK-NEXT:    [[TMP60:%.*]] = select i1 [[TMP59]], float [[TMP58]], float undef
-; CHECK-NEXT:    [[TMP61:%.*]] = fcmp fast ogt float [[TMP60]], undef
-; CHECK-NEXT:    [[TMP62:%.*]] = select i1 [[TMP61]], float [[TMP60]], float undef
-; CHECK-NEXT:    [[TMP63:%.*]] = fcmp fast ogt float [[TMP62]], undef
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <32 x float> [[TMP2]], <32 x float> undef, <32 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp fast ogt <32 x float> [[TMP2]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <32 x i1> [[RDX_MINMAX_CMP]], <32 x float> [[TMP2]], <32 x float> [[RDX_SHUF]]
@@ -570,9 +359,8 @@ define float @maxf32(float) {
 ; CHECK-NEXT:    [[RDX_SHUF10:%.*]] = shufflevector <32 x float> [[RDX_MINMAX_SELECT9]], <32 x float> undef, <32 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[RDX_MINMAX_CMP11:%.*]] = fcmp fast ogt <32 x float> [[RDX_MINMAX_SELECT9]], [[RDX_SHUF10]]
 ; CHECK-NEXT:    [[RDX_MINMAX_SELECT12:%.*]] = select <32 x i1> [[RDX_MINMAX_CMP11]], <32 x float> [[RDX_MINMAX_SELECT9]], <32 x float> [[RDX_SHUF10]]
-; CHECK-NEXT:    [[TMP64:%.*]] = extractelement <32 x float> [[RDX_MINMAX_SELECT12]], i32 0
-; CHECK-NEXT:    [[TMP65:%.*]] = select i1 [[TMP63]], float [[TMP62]], float undef
-; CHECK-NEXT:    ret float [[TMP64]]
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <32 x float> [[RDX_MINMAX_SELECT12]], i32 0
+; CHECK-NEXT:    ret float [[TMP3]]
 ;
   %2 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 0), align 16
   %3 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 1), align 4
@@ -672,151 +460,61 @@ define float @maxf32(float) {
 }
 
 define i32 @maxi8_mutiple_uses(i32) {
-; SSE-LABEL: @maxi8_mutiple_uses(
-; SSE-NEXT:    [[TMP2:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 0), align 16
-; SSE-NEXT:    [[TMP3:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 1), align 4
-; SSE-NEXT:    [[TMP4:%.*]] = icmp sgt i32 [[TMP2]], [[TMP3]]
-; SSE-NEXT:    [[TMP5:%.*]] = select i1 [[TMP4]], i32 [[TMP2]], i32 [[TMP3]]
-; SSE-NEXT:    [[TMP6:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 2) to <4 x i32>*), align 8
-; SSE-NEXT:    [[TMP7:%.*]] = icmp sgt i32 [[TMP5]], undef
-; SSE-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], i32 [[TMP5]], i32 undef
-; SSE-NEXT:    [[TMP9:%.*]] = icmp sgt i32 [[TMP8]], undef
-; SSE-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], i32 [[TMP8]], i32 undef
-; SSE-NEXT:    [[TMP11:%.*]] = icmp sgt i32 [[TMP10]], undef
-; SSE-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], i32 [[TMP10]], i32 undef
-; SSE-NEXT:    [[TMP13:%.*]] = icmp sgt i32 [[TMP12]], undef
-; SSE-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP12]], i32 undef
-; SSE-NEXT:    [[TMP15:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 6), align 8
-; SSE-NEXT:    [[TMP16:%.*]] = icmp sgt i32 [[TMP14]], [[TMP15]]
-; SSE-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
-; SSE-NEXT:    [[RDX_MINMAX_CMP:%.*]] = icmp sgt <4 x i32> [[TMP6]], [[RDX_SHUF]]
-; SSE-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x i32> [[TMP6]], <4 x i32> [[RDX_SHUF]]
-; SSE-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x i32> [[RDX_MINMAX_SELECT]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
-; SSE-NEXT:    [[RDX_MINMAX_CMP2:%.*]] = icmp sgt <4 x i32> [[RDX_MINMAX_SELECT]], [[RDX_SHUF1]]
-; SSE-NEXT:    [[RDX_MINMAX_SELECT3:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP2]], <4 x i32> [[RDX_MINMAX_SELECT]], <4 x i32> [[RDX_SHUF1]]
-; SSE-NEXT:    [[TMP17:%.*]] = extractelement <4 x i32> [[RDX_MINMAX_SELECT3]], i32 0
-; SSE-NEXT:    [[TMP18:%.*]] = icmp sgt i32 [[TMP17]], [[TMP15]]
-; SSE-NEXT:    [[TMP19:%.*]] = select i1 [[TMP18]], i32 [[TMP17]], i32 [[TMP15]]
-; SSE-NEXT:    [[TMP20:%.*]] = icmp sgt i32 [[TMP19]], [[TMP5]]
-; SSE-NEXT:    [[OP_EXTRA:%.*]] = select i1 [[TMP20]], i32 [[TMP19]], i32 [[TMP5]]
-; SSE-NEXT:    [[TMP21:%.*]] = select i1 [[TMP16]], i32 [[TMP14]], i32 [[TMP15]]
-; SSE-NEXT:    [[TMP22:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 7), align 4
-; SSE-NEXT:    [[TMP23:%.*]] = icmp sgt i32 [[OP_EXTRA]], [[TMP22]]
-; SSE-NEXT:    [[TMP24:%.*]] = select i1 [[TMP23]], i32 [[OP_EXTRA]], i32 [[TMP22]]
-; SSE-NEXT:    [[TMP25:%.*]] = select i1 [[TMP4]], i32 3, i32 4
-; SSE-NEXT:    store i32 [[TMP25]], i32* @var, align 8
-; SSE-NEXT:    ret i32 [[TMP24]]
+; DEFAULT-LABEL: @maxi8_mutiple_uses(
+; DEFAULT-NEXT:    [[TMP2:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 0), align 16
+; DEFAULT-NEXT:    [[TMP3:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 1), align 4
+; DEFAULT-NEXT:    [[TMP4:%.*]] = icmp sgt i32 [[TMP2]], [[TMP3]]
+; DEFAULT-NEXT:    [[TMP5:%.*]] = select i1 [[TMP4]], i32 [[TMP2]], i32 [[TMP3]]
+; DEFAULT-NEXT:    [[TMP6:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 2) to <4 x i32>*), align 8
+; DEFAULT-NEXT:    [[TMP7:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 6), align 8
+; DEFAULT-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; DEFAULT-NEXT:    [[RDX_MINMAX_CMP:%.*]] = icmp sgt <4 x i32> [[TMP6]], [[RDX_SHUF]]
+; DEFAULT-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x i32> [[TMP6]], <4 x i32> [[RDX_SHUF]]
+; DEFAULT-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x i32> [[RDX_MINMAX_SELECT]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; DEFAULT-NEXT:    [[RDX_MINMAX_CMP2:%.*]] = icmp sgt <4 x i32> [[RDX_MINMAX_SELECT]], [[RDX_SHUF1]]
+; DEFAULT-NEXT:    [[RDX_MINMAX_SELECT3:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP2]], <4 x i32> [[RDX_MINMAX_SELECT]], <4 x i32> [[RDX_SHUF1]]
+; DEFAULT-NEXT:    [[TMP8:%.*]] = extractelement <4 x i32> [[RDX_MINMAX_SELECT3]], i32 0
+; DEFAULT-NEXT:    [[TMP9:%.*]] = icmp sgt i32 [[TMP8]], [[TMP7]]
+; DEFAULT-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], i32 [[TMP8]], i32 [[TMP7]]
+; DEFAULT-NEXT:    [[TMP11:%.*]] = icmp sgt i32 [[TMP10]], [[TMP5]]
+; DEFAULT-NEXT:    [[OP_EXTRA:%.*]] = select i1 [[TMP11]], i32 [[TMP10]], i32 [[TMP5]]
+; DEFAULT-NEXT:    [[TMP12:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 7), align 4
+; DEFAULT-NEXT:    [[TMP13:%.*]] = icmp sgt i32 [[OP_EXTRA]], [[TMP12]]
+; DEFAULT-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[OP_EXTRA]], i32 [[TMP12]]
+; DEFAULT-NEXT:    [[TMP15:%.*]] = select i1 [[TMP4]], i32 3, i32 4
+; DEFAULT-NEXT:    store i32 [[TMP15]], i32* @var, align 8
+; DEFAULT-NEXT:    ret i32 [[TMP14]]
 ;
-; AVX-LABEL: @maxi8_mutiple_uses(
-; AVX-NEXT:    [[TMP2:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 0), align 16
-; AVX-NEXT:    [[TMP3:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 1), align 4
-; AVX-NEXT:    [[TMP4:%.*]] = icmp sgt i32 [[TMP2]], [[TMP3]]
-; AVX-NEXT:    [[TMP5:%.*]] = select i1 [[TMP4]], i32 [[TMP2]], i32 [[TMP3]]
-; AVX-NEXT:    [[TMP6:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 2) to <4 x i32>*), align 8
-; AVX-NEXT:    [[TMP7:%.*]] = icmp sgt i32 [[TMP5]], undef
-; AVX-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], i32 [[TMP5]], i32 undef
-; AVX-NEXT:    [[TMP9:%.*]] = icmp sgt i32 [[TMP8]], undef
-; AVX-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], i32 [[TMP8]], i32 undef
-; AVX-NEXT:    [[TMP11:%.*]] = icmp sgt i32 [[TMP10]], undef
-; AVX-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], i32 [[TMP10]], i32 undef
-; AVX-NEXT:    [[TMP13:%.*]] = icmp sgt i32 [[TMP12]], undef
-; AVX-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP12]], i32 undef
-; AVX-NEXT:    [[TMP15:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 6), align 8
-; AVX-NEXT:    [[TMP16:%.*]] = icmp sgt i32 [[TMP14]], [[TMP15]]
-; AVX-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
-; AVX-NEXT:    [[RDX_MINMAX_CMP:%.*]] = icmp sgt <4 x i32> [[TMP6]], [[RDX_SHUF]]
-; AVX-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x i32> [[TMP6]], <4 x i32> [[RDX_SHUF]]
-; AVX-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x i32> [[RDX_MINMAX_SELECT]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:    [[RDX_MINMAX_CMP2:%.*]] = icmp sgt <4 x i32> [[RDX_MINMAX_SELECT]], [[RDX_SHUF1]]
-; AVX-NEXT:    [[RDX_MINMAX_SELECT3:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP2]], <4 x i32> [[RDX_MINMAX_SELECT]], <4 x i32> [[RDX_SHUF1]]
-; AVX-NEXT:    [[TMP17:%.*]] = extractelement <4 x i32> [[RDX_MINMAX_SELECT3]], i32 0
-; AVX-NEXT:    [[TMP18:%.*]] = icmp sgt i32 [[TMP17]], [[TMP15]]
-; AVX-NEXT:    [[TMP19:%.*]] = select i1 [[TMP18]], i32 [[TMP17]], i32 [[TMP15]]
-; AVX-NEXT:    [[TMP20:%.*]] = icmp sgt i32 [[TMP19]], [[TMP5]]
-; AVX-NEXT:    [[OP_EXTRA:%.*]] = select i1 [[TMP20]], i32 [[TMP19]], i32 [[TMP5]]
-; AVX-NEXT:    [[TMP21:%.*]] = select i1 [[TMP16]], i32 [[TMP14]], i32 [[TMP15]]
-; AVX-NEXT:    [[TMP22:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 7), align 4
-; AVX-NEXT:    [[TMP23:%.*]] = icmp sgt i32 [[OP_EXTRA]], [[TMP22]]
-; AVX-NEXT:    [[TMP24:%.*]] = select i1 [[TMP23]], i32 [[OP_EXTRA]], i32 [[TMP22]]
-; AVX-NEXT:    [[TMP25:%.*]] = select i1 [[TMP4]], i32 3, i32 4
-; AVX-NEXT:    store i32 [[TMP25]], i32* @var, align 8
-; AVX-NEXT:    ret i32 [[TMP24]]
-;
-; AVX2-LABEL: @maxi8_mutiple_uses(
-; AVX2-NEXT:    [[TMP2:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 0), align 16
-; AVX2-NEXT:    [[TMP3:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 1), align 4
-; AVX2-NEXT:    [[TMP4:%.*]] = icmp sgt i32 [[TMP2]], [[TMP3]]
-; AVX2-NEXT:    [[TMP5:%.*]] = select i1 [[TMP4]], i32 [[TMP2]], i32 [[TMP3]]
-; AVX2-NEXT:    [[TMP6:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 2) to <4 x i32>*), align 8
-; AVX2-NEXT:    [[TMP7:%.*]] = icmp sgt i32 [[TMP5]], undef
-; AVX2-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], i32 [[TMP5]], i32 undef
-; AVX2-NEXT:    [[TMP9:%.*]] = icmp sgt i32 [[TMP8]], undef
-; AVX2-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], i32 [[TMP8]], i32 undef
-; AVX2-NEXT:    [[TMP11:%.*]] = icmp sgt i32 [[TMP10]], undef
-; AVX2-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], i32 [[TMP10]], i32 undef
-; AVX2-NEXT:    [[TMP13:%.*]] = icmp sgt i32 [[TMP12]], undef
-; AVX2-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP12]], i32 undef
-; AVX2-NEXT:    [[TMP15:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 6), align 8
-; AVX2-NEXT:    [[TMP16:%.*]] = icmp sgt i32 [[TMP14]], [[TMP15]]
-; AVX2-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
-; AVX2-NEXT:    [[RDX_MINMAX_CMP:%.*]] = icmp sgt <4 x i32> [[TMP6]], [[RDX_SHUF]]
-; AVX2-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x i32> [[TMP6]], <4 x i32> [[RDX_SHUF]]
-; AVX2-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x i32> [[RDX_MINMAX_SELECT]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:    [[RDX_MINMAX_CMP2:%.*]] = icmp sgt <4 x i32> [[RDX_MINMAX_SELECT]], [[RDX_SHUF1]]
-; AVX2-NEXT:    [[RDX_MINMAX_SELECT3:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP2]], <4 x i32> [[RDX_MINMAX_SELECT]], <4 x i32> [[RDX_SHUF1]]
-; AVX2-NEXT:    [[TMP17:%.*]] = extractelement <4 x i32> [[RDX_MINMAX_SELECT3]], i32 0
-; AVX2-NEXT:    [[TMP18:%.*]] = icmp sgt i32 [[TMP17]], [[TMP15]]
-; AVX2-NEXT:    [[TMP19:%.*]] = select i1 [[TMP18]], i32 [[TMP17]], i32 [[TMP15]]
-; AVX2-NEXT:    [[TMP20:%.*]] = icmp sgt i32 [[TMP19]], [[TMP5]]
-; AVX2-NEXT:    [[OP_EXTRA:%.*]] = select i1 [[TMP20]], i32 [[TMP19]], i32 [[TMP5]]
-; AVX2-NEXT:    [[TMP21:%.*]] = select i1 [[TMP16]], i32 [[TMP14]], i32 [[TMP15]]
-; AVX2-NEXT:    [[TMP22:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 7), align 4
-; AVX2-NEXT:    [[TMP23:%.*]] = icmp sgt i32 [[OP_EXTRA]], [[TMP22]]
-; AVX2-NEXT:    [[TMP24:%.*]] = select i1 [[TMP23]], i32 [[OP_EXTRA]], i32 [[TMP22]]
-; AVX2-NEXT:    [[TMP25:%.*]] = select i1 [[TMP4]], i32 3, i32 4
-; AVX2-NEXT:    store i32 [[TMP25]], i32* @var, align 8
-; AVX2-NEXT:    ret i32 [[TMP24]]
-;
-; SKX-LABEL: @maxi8_mutiple_uses(
-; SKX-NEXT:    [[TMP2:%.*]] = load <2 x i32>, <2 x i32>* bitcast ([32 x i32]* @arr to <2 x i32>*), align 16
-; SKX-NEXT:    [[TMP3:%.*]] = extractelement <2 x i32> [[TMP2]], i32 0
-; SKX-NEXT:    [[TMP4:%.*]] = extractelement <2 x i32> [[TMP2]], i32 1
-; SKX-NEXT:    [[TMP5:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 2) to <4 x i32>*), align 8
-; SKX-NEXT:    [[TMP6:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 6), align 8
-; SKX-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
-; SKX-NEXT:    [[RDX_MINMAX_CMP:%.*]] = icmp sgt <4 x i32> [[TMP5]], [[RDX_SHUF]]
-; SKX-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x i32> [[TMP5]], <4 x i32> [[RDX_SHUF]]
-; SKX-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x i32> [[RDX_MINMAX_SELECT]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
-; SKX-NEXT:    [[RDX_MINMAX_CMP2:%.*]] = icmp sgt <4 x i32> [[RDX_MINMAX_SELECT]], [[RDX_SHUF1]]
-; SKX-NEXT:    [[RDX_MINMAX_SELECT3:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP2]], <4 x i32> [[RDX_MINMAX_SELECT]], <4 x i32> [[RDX_SHUF1]]
-; SKX-NEXT:    [[TMP7:%.*]] = extractelement <4 x i32> [[RDX_MINMAX_SELECT3]], i32 0
-; SKX-NEXT:    [[TMP8:%.*]] = insertelement <2 x i32> undef, i32 [[TMP7]], i32 0
-; SKX-NEXT:    [[TMP9:%.*]] = insertelement <2 x i32> [[TMP8]], i32 [[TMP3]], i32 1
-; SKX-NEXT:    [[TMP10:%.*]] = insertelement <2 x i32> undef, i32 [[TMP6]], i32 0
-; SKX-NEXT:    [[TMP11:%.*]] = insertelement <2 x i32> [[TMP10]], i32 [[TMP4]], i32 1
-; SKX-NEXT:    [[TMP12:%.*]] = icmp sgt <2 x i32> [[TMP9]], [[TMP11]]
-; SKX-NEXT:    [[TMP13:%.*]] = select <2 x i1> [[TMP12]], <2 x i32> [[TMP9]], <2 x i32> [[TMP11]]
-; SKX-NEXT:    [[TMP14:%.*]] = extractelement <2 x i32> [[TMP13]], i32 1
-; SKX-NEXT:    [[TMP15:%.*]] = icmp sgt i32 [[TMP14]], undef
-; SKX-NEXT:    [[TMP16:%.*]] = select i1 [[TMP15]], i32 [[TMP14]], i32 undef
-; SKX-NEXT:    [[TMP17:%.*]] = icmp sgt i32 [[TMP16]], undef
-; SKX-NEXT:    [[TMP18:%.*]] = select i1 [[TMP17]], i32 [[TMP16]], i32 undef
-; SKX-NEXT:    [[TMP19:%.*]] = icmp sgt i32 [[TMP18]], undef
-; SKX-NEXT:    [[TMP20:%.*]] = select i1 [[TMP19]], i32 [[TMP18]], i32 undef
-; SKX-NEXT:    [[TMP21:%.*]] = icmp sgt i32 [[TMP20]], undef
-; SKX-NEXT:    [[TMP22:%.*]] = select i1 [[TMP21]], i32 [[TMP20]], i32 undef
-; SKX-NEXT:    [[TMP23:%.*]] = icmp sgt i32 [[TMP22]], [[TMP6]]
-; SKX-NEXT:    [[TMP24:%.*]] = extractelement <2 x i32> [[TMP13]], i32 0
-; SKX-NEXT:    [[TMP25:%.*]] = icmp sgt i32 [[TMP24]], [[TMP14]]
-; SKX-NEXT:    [[OP_EXTRA:%.*]] = select i1 [[TMP25]], i32 [[TMP24]], i32 [[TMP14]]
-; SKX-NEXT:    [[TMP26:%.*]] = select i1 [[TMP23]], i32 [[TMP22]], i32 [[TMP6]]
-; SKX-NEXT:    [[TMP27:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 7), align 4
-; SKX-NEXT:    [[TMP28:%.*]] = icmp sgt i32 [[OP_EXTRA]], [[TMP27]]
-; SKX-NEXT:    [[TMP29:%.*]] = select i1 [[TMP28]], i32 [[OP_EXTRA]], i32 [[TMP27]]
-; SKX-NEXT:    [[TMP30:%.*]] = extractelement <2 x i1> [[TMP12]], i32 1
-; SKX-NEXT:    [[TMP31:%.*]] = select i1 [[TMP30]], i32 3, i32 4
-; SKX-NEXT:    store i32 [[TMP31]], i32* @var, align 8
-; SKX-NEXT:    ret i32 [[TMP29]]
+; THRESH-LABEL: @maxi8_mutiple_uses(
+; THRESH-NEXT:    [[TMP2:%.*]] = load <2 x i32>, <2 x i32>* bitcast ([32 x i32]* @arr to <2 x i32>*), align 16
+; THRESH-NEXT:    [[TMP3:%.*]] = extractelement <2 x i32> [[TMP2]], i32 0
+; THRESH-NEXT:    [[TMP4:%.*]] = extractelement <2 x i32> [[TMP2]], i32 1
+; THRESH-NEXT:    [[TMP5:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 2) to <4 x i32>*), align 8
+; THRESH-NEXT:    [[TMP6:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 6), align 8
+; THRESH-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; THRESH-NEXT:    [[RDX_MINMAX_CMP:%.*]] = icmp sgt <4 x i32> [[TMP5]], [[RDX_SHUF]]
+; THRESH-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x i32> [[TMP5]], <4 x i32> [[RDX_SHUF]]
+; THRESH-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x i32> [[RDX_MINMAX_SELECT]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; THRESH-NEXT:    [[RDX_MINMAX_CMP2:%.*]] = icmp sgt <4 x i32> [[RDX_MINMAX_SELECT]], [[RDX_SHUF1]]
+; THRESH-NEXT:    [[RDX_MINMAX_SELECT3:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP2]], <4 x i32> [[RDX_MINMAX_SELECT]], <4 x i32> [[RDX_SHUF1]]
+; THRESH-NEXT:    [[TMP7:%.*]] = extractelement <4 x i32> [[RDX_MINMAX_SELECT3]], i32 0
+; THRESH-NEXT:    [[TMP8:%.*]] = insertelement <2 x i32> undef, i32 [[TMP7]], i32 0
+; THRESH-NEXT:    [[TMP9:%.*]] = insertelement <2 x i32> [[TMP8]], i32 [[TMP3]], i32 1
+; THRESH-NEXT:    [[TMP10:%.*]] = insertelement <2 x i32> undef, i32 [[TMP6]], i32 0
+; THRESH-NEXT:    [[TMP11:%.*]] = insertelement <2 x i32> [[TMP10]], i32 [[TMP4]], i32 1
+; THRESH-NEXT:    [[TMP12:%.*]] = icmp sgt <2 x i32> [[TMP9]], [[TMP11]]
+; THRESH-NEXT:    [[TMP13:%.*]] = select <2 x i1> [[TMP12]], <2 x i32> [[TMP9]], <2 x i32> [[TMP11]]
+; THRESH-NEXT:    [[TMP14:%.*]] = extractelement <2 x i32> [[TMP13]], i32 1
+; THRESH-NEXT:    [[TMP15:%.*]] = extractelement <2 x i32> [[TMP13]], i32 0
+; THRESH-NEXT:    [[TMP16:%.*]] = icmp sgt i32 [[TMP15]], [[TMP14]]
+; THRESH-NEXT:    [[OP_EXTRA:%.*]] = select i1 [[TMP16]], i32 [[TMP15]], i32 [[TMP14]]
+; THRESH-NEXT:    [[TMP17:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 7), align 4
+; THRESH-NEXT:    [[TMP18:%.*]] = icmp sgt i32 [[OP_EXTRA]], [[TMP17]]
+; THRESH-NEXT:    [[TMP19:%.*]] = select i1 [[TMP18]], i32 [[OP_EXTRA]], i32 [[TMP17]]
+; THRESH-NEXT:    [[TMP20:%.*]] = extractelement <2 x i1> [[TMP12]], i32 1
+; THRESH-NEXT:    [[TMP21:%.*]] = select i1 [[TMP20]], i32 3, i32 4
+; THRESH-NEXT:    store i32 [[TMP21]], i32* @var, align 8
+; THRESH-NEXT:    ret i32 [[TMP19]]
 ;
   %2 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 0), align 16
   %3 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 1), align 4
@@ -846,161 +544,63 @@ define i32 @maxi8_mutiple_uses(i32) {
 }
 
 define i32 @maxi8_wrong_parent(i32) {
-; SSE-LABEL: @maxi8_wrong_parent(
-; SSE-NEXT:    [[TMP2:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 0), align 16
-; SSE-NEXT:    [[TMP3:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 1), align 4
-; SSE-NEXT:    [[TMP4:%.*]] = icmp sgt i32 [[TMP2]], [[TMP3]]
-; SSE-NEXT:    br label [[PP:%.*]]
-; SSE:       pp:
-; SSE-NEXT:    [[TMP5:%.*]] = select i1 [[TMP4]], i32 [[TMP2]], i32 [[TMP3]]
-; SSE-NEXT:    [[TMP6:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 2) to <4 x i32>*), align 8
-; SSE-NEXT:    [[TMP7:%.*]] = icmp sgt i32 [[TMP5]], undef
-; SSE-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], i32 [[TMP5]], i32 undef
-; SSE-NEXT:    [[TMP9:%.*]] = icmp sgt i32 [[TMP8]], undef
-; SSE-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], i32 [[TMP8]], i32 undef
-; SSE-NEXT:    [[TMP11:%.*]] = icmp sgt i32 [[TMP10]], undef
-; SSE-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], i32 [[TMP10]], i32 undef
-; SSE-NEXT:    [[TMP13:%.*]] = icmp sgt i32 [[TMP12]], undef
-; SSE-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP12]], i32 undef
-; SSE-NEXT:    [[TMP15:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 6), align 8
-; SSE-NEXT:    [[TMP16:%.*]] = icmp sgt i32 [[TMP14]], [[TMP15]]
-; SSE-NEXT:    [[TMP17:%.*]] = select i1 [[TMP16]], i32 [[TMP14]], i32 [[TMP15]]
-; SSE-NEXT:    [[TMP18:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 7), align 4
-; SSE-NEXT:    [[TMP19:%.*]] = icmp sgt i32 [[TMP17]], [[TMP18]]
-; SSE-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
-; SSE-NEXT:    [[RDX_MINMAX_CMP:%.*]] = icmp sgt <4 x i32> [[TMP6]], [[RDX_SHUF]]
-; SSE-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x i32> [[TMP6]], <4 x i32> [[RDX_SHUF]]
-; SSE-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x i32> [[RDX_MINMAX_SELECT]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
-; SSE-NEXT:    [[RDX_MINMAX_CMP2:%.*]] = icmp sgt <4 x i32> [[RDX_MINMAX_SELECT]], [[RDX_SHUF1]]
-; SSE-NEXT:    [[RDX_MINMAX_SELECT3:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP2]], <4 x i32> [[RDX_MINMAX_SELECT]], <4 x i32> [[RDX_SHUF1]]
-; SSE-NEXT:    [[TMP20:%.*]] = extractelement <4 x i32> [[RDX_MINMAX_SELECT3]], i32 0
-; SSE-NEXT:    [[TMP21:%.*]] = icmp sgt i32 [[TMP20]], [[TMP15]]
-; SSE-NEXT:    [[TMP22:%.*]] = select i1 [[TMP21]], i32 [[TMP20]], i32 [[TMP15]]
-; SSE-NEXT:    [[TMP23:%.*]] = icmp sgt i32 [[TMP22]], [[TMP18]]
-; SSE-NEXT:    [[TMP24:%.*]] = select i1 [[TMP23]], i32 [[TMP22]], i32 [[TMP18]]
-; SSE-NEXT:    [[TMP25:%.*]] = icmp sgt i32 [[TMP24]], [[TMP5]]
-; SSE-NEXT:    [[OP_EXTRA:%.*]] = select i1 [[TMP25]], i32 [[TMP24]], i32 [[TMP5]]
-; SSE-NEXT:    [[TMP26:%.*]] = select i1 [[TMP19]], i32 [[TMP17]], i32 [[TMP18]]
-; SSE-NEXT:    ret i32 [[OP_EXTRA]]
-;
-; AVX-LABEL: @maxi8_wrong_parent(
-; AVX-NEXT:    [[TMP2:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 0), align 16
-; AVX-NEXT:    [[TMP3:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 1), align 4
-; AVX-NEXT:    [[TMP4:%.*]] = icmp sgt i32 [[TMP2]], [[TMP3]]
-; AVX-NEXT:    br label [[PP:%.*]]
-; AVX:       pp:
-; AVX-NEXT:    [[TMP5:%.*]] = select i1 [[TMP4]], i32 [[TMP2]], i32 [[TMP3]]
-; AVX-NEXT:    [[TMP6:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 2) to <4 x i32>*), align 8
-; AVX-NEXT:    [[TMP7:%.*]] = icmp sgt i32 [[TMP5]], undef
-; AVX-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], i32 [[TMP5]], i32 undef
-; AVX-NEXT:    [[TMP9:%.*]] = icmp sgt i32 [[TMP8]], undef
-; AVX-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], i32 [[TMP8]], i32 undef
-; AVX-NEXT:    [[TMP11:%.*]] = icmp sgt i32 [[TMP10]], undef
-; AVX-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], i32 [[TMP10]], i32 undef
-; AVX-NEXT:    [[TMP13:%.*]] = icmp sgt i32 [[TMP12]], undef
-; AVX-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP12]], i32 undef
-; AVX-NEXT:    [[TMP15:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 6), align 8
-; AVX-NEXT:    [[TMP16:%.*]] = icmp sgt i32 [[TMP14]], [[TMP15]]
-; AVX-NEXT:    [[TMP17:%.*]] = select i1 [[TMP16]], i32 [[TMP14]], i32 [[TMP15]]
-; AVX-NEXT:    [[TMP18:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 7), align 4
-; AVX-NEXT:    [[TMP19:%.*]] = icmp sgt i32 [[TMP17]], [[TMP18]]
-; AVX-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
-; AVX-NEXT:    [[RDX_MINMAX_CMP:%.*]] = icmp sgt <4 x i32> [[TMP6]], [[RDX_SHUF]]
-; AVX-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x i32> [[TMP6]], <4 x i32> [[RDX_SHUF]]
-; AVX-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x i32> [[RDX_MINMAX_SELECT]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:    [[RDX_MINMAX_CMP2:%.*]] = icmp sgt <4 x i32> [[RDX_MINMAX_SELECT]], [[RDX_SHUF1]]
-; AVX-NEXT:    [[RDX_MINMAX_SELECT3:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP2]], <4 x i32> [[RDX_MINMAX_SELECT]], <4 x i32> [[RDX_SHUF1]]
-; AVX-NEXT:    [[TMP20:%.*]] = extractelement <4 x i32> [[RDX_MINMAX_SELECT3]], i32 0
-; AVX-NEXT:    [[TMP21:%.*]] = icmp sgt i32 [[TMP20]], [[TMP15]]
-; AVX-NEXT:    [[TMP22:%.*]] = select i1 [[TMP21]], i32 [[TMP20]], i32 [[TMP15]]
-; AVX-NEXT:    [[TMP23:%.*]] = icmp sgt i32 [[TMP22]], [[TMP18]]
-; AVX-NEXT:    [[TMP24:%.*]] = select i1 [[TMP23]], i32 [[TMP22]], i32 [[TMP18]]
-; AVX-NEXT:    [[TMP25:%.*]] = icmp sgt i32 [[TMP24]], [[TMP5]]
-; AVX-NEXT:    [[OP_EXTRA:%.*]] = select i1 [[TMP25]], i32 [[TMP24]], i32 [[TMP5]]
-; AVX-NEXT:    [[TMP26:%.*]] = select i1 [[TMP19]], i32 [[TMP17]], i32 [[TMP18]]
-; AVX-NEXT:    ret i32 [[OP_EXTRA]]
+; DEFAULT-LABEL: @maxi8_wrong_parent(
+; DEFAULT-NEXT:    [[TMP2:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 0), align 16
+; DEFAULT-NEXT:    [[TMP3:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 1), align 4
+; DEFAULT-NEXT:    [[TMP4:%.*]] = icmp sgt i32 [[TMP2]], [[TMP3]]
+; DEFAULT-NEXT:    br label [[PP:%.*]]
+; DEFAULT:       pp:
+; DEFAULT-NEXT:    [[TMP5:%.*]] = select i1 [[TMP4]], i32 [[TMP2]], i32 [[TMP3]]
+; DEFAULT-NEXT:    [[TMP6:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 2) to <4 x i32>*), align 8
+; DEFAULT-NEXT:    [[TMP7:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 6), align 8
+; DEFAULT-NEXT:    [[TMP8:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 7), align 4
+; DEFAULT-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; DEFAULT-NEXT:    [[RDX_MINMAX_CMP:%.*]] = icmp sgt <4 x i32> [[TMP6]], [[RDX_SHUF]]
+; DEFAULT-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x i32> [[TMP6]], <4 x i32> [[RDX_SHUF]]
+; DEFAULT-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x i32> [[RDX_MINMAX_SELECT]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; DEFAULT-NEXT:    [[RDX_MINMAX_CMP2:%.*]] = icmp sgt <4 x i32> [[RDX_MINMAX_SELECT]], [[RDX_SHUF1]]
+; DEFAULT-NEXT:    [[RDX_MINMAX_SELECT3:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP2]], <4 x i32> [[RDX_MINMAX_SELECT]], <4 x i32> [[RDX_SHUF1]]
+; DEFAULT-NEXT:    [[TMP9:%.*]] = extractelement <4 x i32> [[RDX_MINMAX_SELECT3]], i32 0
+; DEFAULT-NEXT:    [[TMP10:%.*]] = icmp sgt i32 [[TMP9]], [[TMP7]]
+; DEFAULT-NEXT:    [[TMP11:%.*]] = select i1 [[TMP10]], i32 [[TMP9]], i32 [[TMP7]]
+; DEFAULT-NEXT:    [[TMP12:%.*]] = icmp sgt i32 [[TMP11]], [[TMP8]]
+; DEFAULT-NEXT:    [[TMP13:%.*]] = select i1 [[TMP12]], i32 [[TMP11]], i32 [[TMP8]]
+; DEFAULT-NEXT:    [[TMP14:%.*]] = icmp sgt i32 [[TMP13]], [[TMP5]]
+; DEFAULT-NEXT:    [[OP_EXTRA:%.*]] = select i1 [[TMP14]], i32 [[TMP13]], i32 [[TMP5]]
+; DEFAULT-NEXT:    ret i32 [[OP_EXTRA]]
 ;
-; AVX2-LABEL: @maxi8_wrong_parent(
-; AVX2-NEXT:    [[TMP2:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 0), align 16
-; AVX2-NEXT:    [[TMP3:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 1), align 4
-; AVX2-NEXT:    [[TMP4:%.*]] = icmp sgt i32 [[TMP2]], [[TMP3]]
-; AVX2-NEXT:    br label [[PP:%.*]]
-; AVX2:       pp:
-; AVX2-NEXT:    [[TMP5:%.*]] = select i1 [[TMP4]], i32 [[TMP2]], i32 [[TMP3]]
-; AVX2-NEXT:    [[TMP6:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 2) to <4 x i32>*), align 8
-; AVX2-NEXT:    [[TMP7:%.*]] = icmp sgt i32 [[TMP5]], undef
-; AVX2-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], i32 [[TMP5]], i32 undef
-; AVX2-NEXT:    [[TMP9:%.*]] = icmp sgt i32 [[TMP8]], undef
-; AVX2-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], i32 [[TMP8]], i32 undef
-; AVX2-NEXT:    [[TMP11:%.*]] = icmp sgt i32 [[TMP10]], undef
-; AVX2-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], i32 [[TMP10]], i32 undef
-; AVX2-NEXT:    [[TMP13:%.*]] = icmp sgt i32 [[TMP12]], undef
-; AVX2-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP12]], i32 undef
-; AVX2-NEXT:    [[TMP15:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 6), align 8
-; AVX2-NEXT:    [[TMP16:%.*]] = icmp sgt i32 [[TMP14]], [[TMP15]]
-; AVX2-NEXT:    [[TMP17:%.*]] = select i1 [[TMP16]], i32 [[TMP14]], i32 [[TMP15]]
-; AVX2-NEXT:    [[TMP18:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 7), align 4
-; AVX2-NEXT:    [[TMP19:%.*]] = icmp sgt i32 [[TMP17]], [[TMP18]]
-; AVX2-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
-; AVX2-NEXT:    [[RDX_MINMAX_CMP:%.*]] = icmp sgt <4 x i32> [[TMP6]], [[RDX_SHUF]]
-; AVX2-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x i32> [[TMP6]], <4 x i32> [[RDX_SHUF]]
-; AVX2-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x i32> [[RDX_MINMAX_SELECT]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:    [[RDX_MINMAX_CMP2:%.*]] = icmp sgt <4 x i32> [[RDX_MINMAX_SELECT]], [[RDX_SHUF1]]
-; AVX2-NEXT:    [[RDX_MINMAX_SELECT3:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP2]], <4 x i32> [[RDX_MINMAX_SELECT]], <4 x i32> [[RDX_SHUF1]]
-; AVX2-NEXT:    [[TMP20:%.*]] = extractelement <4 x i32> [[RDX_MINMAX_SELECT3]], i32 0
-; AVX2-NEXT:    [[TMP21:%.*]] = icmp sgt i32 [[TMP20]], [[TMP15]]
-; AVX2-NEXT:    [[TMP22:%.*]] = select i1 [[TMP21]], i32 [[TMP20]], i32 [[TMP15]]
-; AVX2-NEXT:    [[TMP23:%.*]] = icmp sgt i32 [[TMP22]], [[TMP18]]
-; AVX2-NEXT:    [[TMP24:%.*]] = select i1 [[TMP23]], i32 [[TMP22]], i32 [[TMP18]]
-; AVX2-NEXT:    [[TMP25:%.*]] = icmp sgt i32 [[TMP24]], [[TMP5]]
-; AVX2-NEXT:    [[OP_EXTRA:%.*]] = select i1 [[TMP25]], i32 [[TMP24]], i32 [[TMP5]]
-; AVX2-NEXT:    [[TMP26:%.*]] = select i1 [[TMP19]], i32 [[TMP17]], i32 [[TMP18]]
-; AVX2-NEXT:    ret i32 [[OP_EXTRA]]
-;
-; SKX-LABEL: @maxi8_wrong_parent(
-; SKX-NEXT:    [[TMP2:%.*]] = load <2 x i32>, <2 x i32>* bitcast ([32 x i32]* @arr to <2 x i32>*), align 16
-; SKX-NEXT:    [[TMP3:%.*]] = extractelement <2 x i32> [[TMP2]], i32 0
-; SKX-NEXT:    [[TMP4:%.*]] = extractelement <2 x i32> [[TMP2]], i32 1
-; SKX-NEXT:    [[TMP5:%.*]] = icmp sgt i32 [[TMP3]], [[TMP4]]
-; SKX-NEXT:    br label [[PP:%.*]]
-; SKX:       pp:
-; SKX-NEXT:    [[TMP6:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 2) to <4 x i32>*), align 8
-; SKX-NEXT:    [[TMP7:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 6), align 8
-; SKX-NEXT:    [[TMP8:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 7), align 4
-; SKX-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
-; SKX-NEXT:    [[RDX_MINMAX_CMP:%.*]] = icmp sgt <4 x i32> [[TMP6]], [[RDX_SHUF]]
-; SKX-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x i32> [[TMP6]], <4 x i32> [[RDX_SHUF]]
-; SKX-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x i32> [[RDX_MINMAX_SELECT]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
-; SKX-NEXT:    [[RDX_MINMAX_CMP2:%.*]] = icmp sgt <4 x i32> [[RDX_MINMAX_SELECT]], [[RDX_SHUF1]]
-; SKX-NEXT:    [[RDX_MINMAX_SELECT3:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP2]], <4 x i32> [[RDX_MINMAX_SELECT]], <4 x i32> [[RDX_SHUF1]]
-; SKX-NEXT:    [[TMP9:%.*]] = extractelement <4 x i32> [[RDX_MINMAX_SELECT3]], i32 0
-; SKX-NEXT:    [[TMP10:%.*]] = icmp sgt i32 [[TMP9]], [[TMP7]]
-; SKX-NEXT:    [[TMP11:%.*]] = select i1 [[TMP10]], i32 [[TMP9]], i32 [[TMP7]]
-; SKX-NEXT:    [[TMP12:%.*]] = icmp sgt i32 [[TMP11]], [[TMP8]]
-; SKX-NEXT:    [[TMP13:%.*]] = insertelement <2 x i1> undef, i1 [[TMP12]], i32 0
-; SKX-NEXT:    [[TMP14:%.*]] = insertelement <2 x i1> [[TMP13]], i1 [[TMP5]], i32 1
-; SKX-NEXT:    [[TMP15:%.*]] = insertelement <2 x i32> undef, i32 [[TMP11]], i32 0
-; SKX-NEXT:    [[TMP16:%.*]] = insertelement <2 x i32> [[TMP15]], i32 [[TMP3]], i32 1
-; SKX-NEXT:    [[TMP17:%.*]] = insertelement <2 x i32> undef, i32 [[TMP8]], i32 0
-; SKX-NEXT:    [[TMP18:%.*]] = insertelement <2 x i32> [[TMP17]], i32 [[TMP4]], i32 1
-; SKX-NEXT:    [[TMP19:%.*]] = select <2 x i1> [[TMP14]], <2 x i32> [[TMP16]], <2 x i32> [[TMP18]]
-; SKX-NEXT:    [[TMP20:%.*]] = extractelement <2 x i32> [[TMP19]], i32 1
-; SKX-NEXT:    [[TMP21:%.*]] = icmp sgt i32 [[TMP20]], undef
-; SKX-NEXT:    [[TMP22:%.*]] = select i1 [[TMP21]], i32 [[TMP20]], i32 undef
-; SKX-NEXT:    [[TMP23:%.*]] = icmp sgt i32 [[TMP22]], undef
-; SKX-NEXT:    [[TMP24:%.*]] = select i1 [[TMP23]], i32 [[TMP22]], i32 undef
-; SKX-NEXT:    [[TMP25:%.*]] = icmp sgt i32 [[TMP24]], undef
-; SKX-NEXT:    [[TMP26:%.*]] = select i1 [[TMP25]], i32 [[TMP24]], i32 undef
-; SKX-NEXT:    [[TMP27:%.*]] = icmp sgt i32 [[TMP26]], undef
-; SKX-NEXT:    [[TMP28:%.*]] = select i1 [[TMP27]], i32 [[TMP26]], i32 undef
-; SKX-NEXT:    [[TMP29:%.*]] = icmp sgt i32 [[TMP28]], [[TMP7]]
-; SKX-NEXT:    [[TMP30:%.*]] = select i1 [[TMP29]], i32 [[TMP28]], i32 [[TMP7]]
-; SKX-NEXT:    [[TMP31:%.*]] = icmp sgt i32 [[TMP30]], [[TMP8]]
-; SKX-NEXT:    [[TMP32:%.*]] = extractelement <2 x i32> [[TMP19]], i32 0
-; SKX-NEXT:    [[TMP33:%.*]] = icmp sgt i32 [[TMP32]], [[TMP20]]
-; SKX-NEXT:    [[OP_EXTRA:%.*]] = select i1 [[TMP33]], i32 [[TMP32]], i32 [[TMP20]]
-; SKX-NEXT:    [[TMP34:%.*]] = select i1 [[TMP31]], i32 [[TMP30]], i32 [[TMP8]]
-; SKX-NEXT:    ret i32 [[OP_EXTRA]]
+; THRESH-LABEL: @maxi8_wrong_parent(
+; THRESH-NEXT:    [[TMP2:%.*]] = load <2 x i32>, <2 x i32>* bitcast ([32 x i32]* @arr to <2 x i32>*), align 16
+; THRESH-NEXT:    [[TMP3:%.*]] = extractelement <2 x i32> [[TMP2]], i32 0
+; THRESH-NEXT:    [[TMP4:%.*]] = extractelement <2 x i32> [[TMP2]], i32 1
+; THRESH-NEXT:    [[TMP5:%.*]] = icmp sgt i32 [[TMP3]], [[TMP4]]
+; THRESH-NEXT:    br label [[PP:%.*]]
+; THRESH:       pp:
+; THRESH-NEXT:    [[TMP6:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 2) to <4 x i32>*), align 8
+; THRESH-NEXT:    [[TMP7:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 6), align 8
+; THRESH-NEXT:    [[TMP8:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 7), align 4
+; THRESH-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; THRESH-NEXT:    [[RDX_MINMAX_CMP:%.*]] = icmp sgt <4 x i32> [[TMP6]], [[RDX_SHUF]]
+; THRESH-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x i32> [[TMP6]], <4 x i32> [[RDX_SHUF]]
+; THRESH-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x i32> [[RDX_MINMAX_SELECT]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; THRESH-NEXT:    [[RDX_MINMAX_CMP2:%.*]] = icmp sgt <4 x i32> [[RDX_MINMAX_SELECT]], [[RDX_SHUF1]]
+; THRESH-NEXT:    [[RDX_MINMAX_SELECT3:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP2]], <4 x i32> [[RDX_MINMAX_SELECT]], <4 x i32> [[RDX_SHUF1]]
+; THRESH-NEXT:    [[TMP9:%.*]] = extractelement <4 x i32> [[RDX_MINMAX_SELECT3]], i32 0
+; THRESH-NEXT:    [[TMP10:%.*]] = icmp sgt i32 [[TMP9]], [[TMP7]]
+; THRESH-NEXT:    [[TMP11:%.*]] = select i1 [[TMP10]], i32 [[TMP9]], i32 [[TMP7]]
+; THRESH-NEXT:    [[TMP12:%.*]] = icmp sgt i32 [[TMP11]], [[TMP8]]
+; THRESH-NEXT:    [[TMP13:%.*]] = insertelement <2 x i1> undef, i1 [[TMP12]], i32 0
+; THRESH-NEXT:    [[TMP14:%.*]] = insertelement <2 x i1> [[TMP13]], i1 [[TMP5]], i32 1
+; THRESH-NEXT:    [[TMP15:%.*]] = insertelement <2 x i32> undef, i32 [[TMP11]], i32 0
+; THRESH-NEXT:    [[TMP16:%.*]] = insertelement <2 x i32> [[TMP15]], i32 [[TMP3]], i32 1
+; THRESH-NEXT:    [[TMP17:%.*]] = insertelement <2 x i32> undef, i32 [[TMP8]], i32 0
+; THRESH-NEXT:    [[TMP18:%.*]] = insertelement <2 x i32> [[TMP17]], i32 [[TMP4]], i32 1
+; THRESH-NEXT:    [[TMP19:%.*]] = select <2 x i1> [[TMP14]], <2 x i32> [[TMP16]], <2 x i32> [[TMP18]]
+; THRESH-NEXT:    [[TMP20:%.*]] = extractelement <2 x i32> [[TMP19]], i32 1
+; THRESH-NEXT:    [[TMP21:%.*]] = extractelement <2 x i32> [[TMP19]], i32 0
+; THRESH-NEXT:    [[TMP22:%.*]] = icmp sgt i32 [[TMP21]], [[TMP20]]
+; THRESH-NEXT:    [[OP_EXTRA:%.*]] = select i1 [[TMP22]], i32 [[TMP21]], i32 [[TMP20]]
+; THRESH-NEXT:    ret i32 [[OP_EXTRA]]
 ;
   %2 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 0), align 16
   %3 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 1), align 4
@@ -1032,106 +632,56 @@ pp:
 
 ; PR38191 - We don't handle array-of-pointer reductions.
 define i32* @maxp8(i32) {
-; SSE-LABEL: @maxp8(
-; SSE-NEXT:    [[TMP2:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 0), align 16
-; SSE-NEXT:    [[TMP3:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 1), align 4
-; SSE-NEXT:    [[TMP4:%.*]] = icmp ugt i32* [[TMP2]], [[TMP3]]
-; SSE-NEXT:    [[TMP5:%.*]] = select i1 [[TMP4]], i32* [[TMP2]], i32* [[TMP3]]
-; SSE-NEXT:    [[TMP6:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 2), align 8
-; SSE-NEXT:    [[TMP7:%.*]] = icmp ugt i32* [[TMP5]], [[TMP6]]
-; SSE-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], i32* [[TMP5]], i32* [[TMP6]]
-; SSE-NEXT:    [[TMP9:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 3), align 4
-; SSE-NEXT:    [[TMP10:%.*]] = icmp ugt i32* [[TMP8]], [[TMP9]]
-; SSE-NEXT:    [[TMP11:%.*]] = select i1 [[TMP10]], i32* [[TMP8]], i32* [[TMP9]]
-; SSE-NEXT:    [[TMP12:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 4), align 16
-; SSE-NEXT:    [[TMP13:%.*]] = icmp ugt i32* [[TMP11]], [[TMP12]]
-; SSE-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i32* [[TMP11]], i32* [[TMP12]]
-; SSE-NEXT:    [[TMP15:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 5), align 4
-; SSE-NEXT:    [[TMP16:%.*]] = icmp ugt i32* [[TMP14]], [[TMP15]]
-; SSE-NEXT:    [[TMP17:%.*]] = select i1 [[TMP16]], i32* [[TMP14]], i32* [[TMP15]]
-; SSE-NEXT:    [[TMP18:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 6), align 8
-; SSE-NEXT:    [[TMP19:%.*]] = icmp ugt i32* [[TMP17]], [[TMP18]]
-; SSE-NEXT:    [[TMP20:%.*]] = select i1 [[TMP19]], i32* [[TMP17]], i32* [[TMP18]]
-; SSE-NEXT:    [[TMP21:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 7), align 4
-; SSE-NEXT:    [[TMP22:%.*]] = icmp ugt i32* [[TMP20]], [[TMP21]]
-; SSE-NEXT:    [[TMP23:%.*]] = select i1 [[TMP22]], i32* [[TMP20]], i32* [[TMP21]]
-; SSE-NEXT:    ret i32* [[TMP23]]
-;
-; AVX-LABEL: @maxp8(
-; AVX-NEXT:    [[TMP2:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 0), align 16
-; AVX-NEXT:    [[TMP3:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 1), align 4
-; AVX-NEXT:    [[TMP4:%.*]] = icmp ugt i32* [[TMP2]], [[TMP3]]
-; AVX-NEXT:    [[TMP5:%.*]] = select i1 [[TMP4]], i32* [[TMP2]], i32* [[TMP3]]
-; AVX-NEXT:    [[TMP6:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 2), align 8
-; AVX-NEXT:    [[TMP7:%.*]] = icmp ugt i32* [[TMP5]], [[TMP6]]
-; AVX-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], i32* [[TMP5]], i32* [[TMP6]]
-; AVX-NEXT:    [[TMP9:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 3), align 4
-; AVX-NEXT:    [[TMP10:%.*]] = icmp ugt i32* [[TMP8]], [[TMP9]]
-; AVX-NEXT:    [[TMP11:%.*]] = select i1 [[TMP10]], i32* [[TMP8]], i32* [[TMP9]]
-; AVX-NEXT:    [[TMP12:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 4), align 16
-; AVX-NEXT:    [[TMP13:%.*]] = icmp ugt i32* [[TMP11]], [[TMP12]]
-; AVX-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i32* [[TMP11]], i32* [[TMP12]]
-; AVX-NEXT:    [[TMP15:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 5), align 4
-; AVX-NEXT:    [[TMP16:%.*]] = icmp ugt i32* [[TMP14]], [[TMP15]]
-; AVX-NEXT:    [[TMP17:%.*]] = select i1 [[TMP16]], i32* [[TMP14]], i32* [[TMP15]]
-; AVX-NEXT:    [[TMP18:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 6), align 8
-; AVX-NEXT:    [[TMP19:%.*]] = icmp ugt i32* [[TMP17]], [[TMP18]]
-; AVX-NEXT:    [[TMP20:%.*]] = select i1 [[TMP19]], i32* [[TMP17]], i32* [[TMP18]]
-; AVX-NEXT:    [[TMP21:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 7), align 4
-; AVX-NEXT:    [[TMP22:%.*]] = icmp ugt i32* [[TMP20]], [[TMP21]]
-; AVX-NEXT:    [[TMP23:%.*]] = select i1 [[TMP22]], i32* [[TMP20]], i32* [[TMP21]]
-; AVX-NEXT:    ret i32* [[TMP23]]
-;
-; AVX2-LABEL: @maxp8(
-; AVX2-NEXT:    [[TMP2:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 0), align 16
-; AVX2-NEXT:    [[TMP3:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 1), align 4
-; AVX2-NEXT:    [[TMP4:%.*]] = icmp ugt i32* [[TMP2]], [[TMP3]]
-; AVX2-NEXT:    [[TMP5:%.*]] = select i1 [[TMP4]], i32* [[TMP2]], i32* [[TMP3]]
-; AVX2-NEXT:    [[TMP6:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 2), align 8
-; AVX2-NEXT:    [[TMP7:%.*]] = icmp ugt i32* [[TMP5]], [[TMP6]]
-; AVX2-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], i32* [[TMP5]], i32* [[TMP6]]
-; AVX2-NEXT:    [[TMP9:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 3), align 4
-; AVX2-NEXT:    [[TMP10:%.*]] = icmp ugt i32* [[TMP8]], [[TMP9]]
-; AVX2-NEXT:    [[TMP11:%.*]] = select i1 [[TMP10]], i32* [[TMP8]], i32* [[TMP9]]
-; AVX2-NEXT:    [[TMP12:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 4), align 16
-; AVX2-NEXT:    [[TMP13:%.*]] = icmp ugt i32* [[TMP11]], [[TMP12]]
-; AVX2-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i32* [[TMP11]], i32* [[TMP12]]
-; AVX2-NEXT:    [[TMP15:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 5), align 4
-; AVX2-NEXT:    [[TMP16:%.*]] = icmp ugt i32* [[TMP14]], [[TMP15]]
-; AVX2-NEXT:    [[TMP17:%.*]] = select i1 [[TMP16]], i32* [[TMP14]], i32* [[TMP15]]
-; AVX2-NEXT:    [[TMP18:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 6), align 8
-; AVX2-NEXT:    [[TMP19:%.*]] = icmp ugt i32* [[TMP17]], [[TMP18]]
-; AVX2-NEXT:    [[TMP20:%.*]] = select i1 [[TMP19]], i32* [[TMP17]], i32* [[TMP18]]
-; AVX2-NEXT:    [[TMP21:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 7), align 4
-; AVX2-NEXT:    [[TMP22:%.*]] = icmp ugt i32* [[TMP20]], [[TMP21]]
-; AVX2-NEXT:    [[TMP23:%.*]] = select i1 [[TMP22]], i32* [[TMP20]], i32* [[TMP21]]
-; AVX2-NEXT:    ret i32* [[TMP23]]
+; DEFAULT-LABEL: @maxp8(
+; DEFAULT-NEXT:    [[TMP2:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 0), align 16
+; DEFAULT-NEXT:    [[TMP3:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 1), align 4
+; DEFAULT-NEXT:    [[TMP4:%.*]] = icmp ugt i32* [[TMP2]], [[TMP3]]
+; DEFAULT-NEXT:    [[TMP5:%.*]] = select i1 [[TMP4]], i32* [[TMP2]], i32* [[TMP3]]
+; DEFAULT-NEXT:    [[TMP6:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 2), align 8
+; DEFAULT-NEXT:    [[TMP7:%.*]] = icmp ugt i32* [[TMP5]], [[TMP6]]
+; DEFAULT-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], i32* [[TMP5]], i32* [[TMP6]]
+; DEFAULT-NEXT:    [[TMP9:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 3), align 4
+; DEFAULT-NEXT:    [[TMP10:%.*]] = icmp ugt i32* [[TMP8]], [[TMP9]]
+; DEFAULT-NEXT:    [[TMP11:%.*]] = select i1 [[TMP10]], i32* [[TMP8]], i32* [[TMP9]]
+; DEFAULT-NEXT:    [[TMP12:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 4), align 16
+; DEFAULT-NEXT:    [[TMP13:%.*]] = icmp ugt i32* [[TMP11]], [[TMP12]]
+; DEFAULT-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i32* [[TMP11]], i32* [[TMP12]]
+; DEFAULT-NEXT:    [[TMP15:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 5), align 4
+; DEFAULT-NEXT:    [[TMP16:%.*]] = icmp ugt i32* [[TMP14]], [[TMP15]]
+; DEFAULT-NEXT:    [[TMP17:%.*]] = select i1 [[TMP16]], i32* [[TMP14]], i32* [[TMP15]]
+; DEFAULT-NEXT:    [[TMP18:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 6), align 8
+; DEFAULT-NEXT:    [[TMP19:%.*]] = icmp ugt i32* [[TMP17]], [[TMP18]]
+; DEFAULT-NEXT:    [[TMP20:%.*]] = select i1 [[TMP19]], i32* [[TMP17]], i32* [[TMP18]]
+; DEFAULT-NEXT:    [[TMP21:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 7), align 4
+; DEFAULT-NEXT:    [[TMP22:%.*]] = icmp ugt i32* [[TMP20]], [[TMP21]]
+; DEFAULT-NEXT:    [[TMP23:%.*]] = select i1 [[TMP22]], i32* [[TMP20]], i32* [[TMP21]]
+; DEFAULT-NEXT:    ret i32* [[TMP23]]
 ;
-; SKX-LABEL: @maxp8(
-; SKX-NEXT:    [[TMP2:%.*]] = load <2 x i32*>, <2 x i32*>* bitcast ([32 x i32*]* @arrp to <2 x i32*>*), align 16
-; SKX-NEXT:    [[TMP3:%.*]] = extractelement <2 x i32*> [[TMP2]], i32 0
-; SKX-NEXT:    [[TMP4:%.*]] = extractelement <2 x i32*> [[TMP2]], i32 1
-; SKX-NEXT:    [[TMP5:%.*]] = icmp ugt i32* [[TMP3]], [[TMP4]]
-; SKX-NEXT:    [[TMP6:%.*]] = select i1 [[TMP5]], i32* [[TMP3]], i32* [[TMP4]]
-; SKX-NEXT:    [[TMP7:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 2), align 8
-; SKX-NEXT:    [[TMP8:%.*]] = icmp ugt i32* [[TMP6]], [[TMP7]]
-; SKX-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i32* [[TMP6]], i32* [[TMP7]]
-; SKX-NEXT:    [[TMP10:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 3), align 4
-; SKX-NEXT:    [[TMP11:%.*]] = icmp ugt i32* [[TMP9]], [[TMP10]]
-; SKX-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], i32* [[TMP9]], i32* [[TMP10]]
-; SKX-NEXT:    [[TMP13:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 4), align 16
-; SKX-NEXT:    [[TMP14:%.*]] = icmp ugt i32* [[TMP12]], [[TMP13]]
-; SKX-NEXT:    [[TMP15:%.*]] = select i1 [[TMP14]], i32* [[TMP12]], i32* [[TMP13]]
-; SKX-NEXT:    [[TMP16:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 5), align 4
-; SKX-NEXT:    [[TMP17:%.*]] = icmp ugt i32* [[TMP15]], [[TMP16]]
-; SKX-NEXT:    [[TMP18:%.*]] = select i1 [[TMP17]], i32* [[TMP15]], i32* [[TMP16]]
-; SKX-NEXT:    [[TMP19:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 6), align 8
-; SKX-NEXT:    [[TMP20:%.*]] = icmp ugt i32* [[TMP18]], [[TMP19]]
-; SKX-NEXT:    [[TMP21:%.*]] = select i1 [[TMP20]], i32* [[TMP18]], i32* [[TMP19]]
-; SKX-NEXT:    [[TMP22:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 7), align 4
-; SKX-NEXT:    [[TMP23:%.*]] = icmp ugt i32* [[TMP21]], [[TMP22]]
-; SKX-NEXT:    [[TMP24:%.*]] = select i1 [[TMP23]], i32* [[TMP21]], i32* [[TMP22]]
-; SKX-NEXT:    ret i32* [[TMP24]]
+; THRESH-LABEL: @maxp8(
+; THRESH-NEXT:    [[TMP2:%.*]] = load <2 x i32*>, <2 x i32*>* bitcast ([32 x i32*]* @arrp to <2 x i32*>*), align 16
+; THRESH-NEXT:    [[TMP3:%.*]] = extractelement <2 x i32*> [[TMP2]], i32 0
+; THRESH-NEXT:    [[TMP4:%.*]] = extractelement <2 x i32*> [[TMP2]], i32 1
+; THRESH-NEXT:    [[TMP5:%.*]] = icmp ugt i32* [[TMP3]], [[TMP4]]
+; THRESH-NEXT:    [[TMP6:%.*]] = select i1 [[TMP5]], i32* [[TMP3]], i32* [[TMP4]]
+; THRESH-NEXT:    [[TMP7:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 2), align 8
+; THRESH-NEXT:    [[TMP8:%.*]] = icmp ugt i32* [[TMP6]], [[TMP7]]
+; THRESH-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i32* [[TMP6]], i32* [[TMP7]]
+; THRESH-NEXT:    [[TMP10:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 3), align 4
+; THRESH-NEXT:    [[TMP11:%.*]] = icmp ugt i32* [[TMP9]], [[TMP10]]
+; THRESH-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], i32* [[TMP9]], i32* [[TMP10]]
+; THRESH-NEXT:    [[TMP13:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 4), align 16
+; THRESH-NEXT:    [[TMP14:%.*]] = icmp ugt i32* [[TMP12]], [[TMP13]]
+; THRESH-NEXT:    [[TMP15:%.*]] = select i1 [[TMP14]], i32* [[TMP12]], i32* [[TMP13]]
+; THRESH-NEXT:    [[TMP16:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 5), align 4
+; THRESH-NEXT:    [[TMP17:%.*]] = icmp ugt i32* [[TMP15]], [[TMP16]]
+; THRESH-NEXT:    [[TMP18:%.*]] = select i1 [[TMP17]], i32* [[TMP15]], i32* [[TMP16]]
+; THRESH-NEXT:    [[TMP19:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 6), align 8
+; THRESH-NEXT:    [[TMP20:%.*]] = icmp ugt i32* [[TMP18]], [[TMP19]]
+; THRESH-NEXT:    [[TMP21:%.*]] = select i1 [[TMP20]], i32* [[TMP18]], i32* [[TMP19]]
+; THRESH-NEXT:    [[TMP22:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 7), align 4
+; THRESH-NEXT:    [[TMP23:%.*]] = icmp ugt i32* [[TMP21]], [[TMP22]]
+; THRESH-NEXT:    [[TMP24:%.*]] = select i1 [[TMP23]], i32* [[TMP21]], i32* [[TMP22]]
+; THRESH-NEXT:    ret i32* [[TMP24]]
 ;
   %2 = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 0), align 16
   %3 = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 1), align 4
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/horizontal.ll b/llvm/test/Transforms/SLPVectorizer/X86/horizontal.ll
index 311d8a476..ed0e5784e 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/horizontal.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/horizontal.ll
@@ -37,14 +37,11 @@ define i32 @add_red(float* %A, i32 %n) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float* [[ARRAYIDX]] to <4 x float>*
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x float>, <4 x float>* [[TMP1]], align 4
 ; CHECK-NEXT:    [[TMP3:%.*]] = fmul <4 x float> [[TMP2]], <float 7.000000e+00, float 7.000000e+00, float 7.000000e+00, float 7.000000e+00>
-; CHECK-NEXT:    [[ADD6:%.*]] = fadd fast float undef, undef
-; CHECK-NEXT:    [[ADD11:%.*]] = fadd fast float [[ADD6]], undef
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = fadd fast <4 x float> [[TMP3]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x float> [[BIN_RDX]], <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX2:%.*]] = fadd fast <4 x float> [[BIN_RDX]], [[RDX_SHUF1]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[BIN_RDX2]], i32 0
-; CHECK-NEXT:    [[ADD16:%.*]] = fadd fast float [[ADD11]], undef
 ; CHECK-NEXT:    [[ADD17]] = fadd fast float [[SUM_032]], [[TMP4]]
 ; CHECK-NEXT:    [[INC]] = add nsw i64 [[I_033]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INC]], [[TMP0]]
@@ -77,14 +74,11 @@ define i32 @add_red(float* %A, i32 %n) {
 ; STORE-NEXT:    [[TMP1:%.*]] = bitcast float* [[ARRAYIDX]] to <4 x float>*
 ; STORE-NEXT:    [[TMP2:%.*]] = load <4 x float>, <4 x float>* [[TMP1]], align 4
 ; STORE-NEXT:    [[TMP3:%.*]] = fmul <4 x float> [[TMP2]], <float 7.000000e+00, float 7.000000e+00, float 7.000000e+00, float 7.000000e+00>
-; STORE-NEXT:    [[ADD6:%.*]] = fadd fast float undef, undef
-; STORE-NEXT:    [[ADD11:%.*]] = fadd fast float [[ADD6]], undef
 ; STORE-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
 ; STORE-NEXT:    [[BIN_RDX:%.*]] = fadd fast <4 x float> [[TMP3]], [[RDX_SHUF]]
 ; STORE-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x float> [[BIN_RDX]], <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
 ; STORE-NEXT:    [[BIN_RDX2:%.*]] = fadd fast <4 x float> [[BIN_RDX]], [[RDX_SHUF1]]
 ; STORE-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[BIN_RDX2]], i32 0
-; STORE-NEXT:    [[ADD16:%.*]] = fadd fast float [[ADD11]], undef
 ; STORE-NEXT:    [[ADD17]] = fadd fast float [[SUM_032]], [[TMP4]]
 ; STORE-NEXT:    [[INC]] = add nsw i64 [[I_033]], 1
 ; STORE-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INC]], [[TMP0]]
@@ -178,14 +172,11 @@ define i32 @mul_red(float* noalias %A, float* noalias %B, i32 %n) {
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float* [[ARRAYIDX2]] to <4 x float>*
 ; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x float>, <4 x float>* [[TMP3]], align 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = fmul <4 x float> [[TMP1]], [[TMP4]]
-; CHECK-NEXT:    [[ADD8:%.*]] = fadd fast float undef, undef
-; CHECK-NEXT:    [[ADD14:%.*]] = fadd fast float [[ADD8]], undef
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x float> [[TMP5]], <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = fadd fast <4 x float> [[TMP5]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x float> [[BIN_RDX]], <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX2:%.*]] = fadd fast <4 x float> [[BIN_RDX]], [[RDX_SHUF1]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[BIN_RDX2]], i32 0
-; CHECK-NEXT:    [[ADD20:%.*]] = fadd fast float [[ADD14]], undef
 ; CHECK-NEXT:    [[MUL21]] = fmul float [[SUM_039]], [[TMP6]]
 ; CHECK-NEXT:    [[INC]] = add nsw i64 [[I_040]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INC]], [[TMP2]]
@@ -223,14 +214,11 @@ define i32 @mul_red(float* noalias %A, float* noalias %B, i32 %n) {
 ; STORE-NEXT:    [[TMP3:%.*]] = bitcast float* [[ARRAYIDX2]] to <4 x float>*
 ; STORE-NEXT:    [[TMP4:%.*]] = load <4 x float>, <4 x float>* [[TMP3]], align 4
 ; STORE-NEXT:    [[TMP5:%.*]] = fmul <4 x float> [[TMP1]], [[TMP4]]
-; STORE-NEXT:    [[ADD8:%.*]] = fadd fast float undef, undef
-; STORE-NEXT:    [[ADD14:%.*]] = fadd fast float [[ADD8]], undef
 ; STORE-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x float> [[TMP5]], <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
 ; STORE-NEXT:    [[BIN_RDX:%.*]] = fadd fast <4 x float> [[TMP5]], [[RDX_SHUF]]
 ; STORE-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x float> [[BIN_RDX]], <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
 ; STORE-NEXT:    [[BIN_RDX2:%.*]] = fadd fast <4 x float> [[BIN_RDX]], [[RDX_SHUF1]]
 ; STORE-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[BIN_RDX2]], i32 0
-; STORE-NEXT:    [[ADD20:%.*]] = fadd fast float [[ADD14]], undef
 ; STORE-NEXT:    [[MUL21]] = fmul float [[SUM_039]], [[TMP6]]
 ; STORE-NEXT:    [[INC]] = add nsw i64 [[I_040]], 1
 ; STORE-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INC]], [[TMP2]]
@@ -350,13 +338,6 @@ define i32 @long_red(float* noalias %A, float* noalias %B, i32 %n) {
 ; CHECK-NEXT:    [[TMP4:%.*]] = bitcast float* [[ARRAYIDX2]] to <8 x float>*
 ; CHECK-NEXT:    [[TMP5:%.*]] = load <8 x float>, <8 x float>* [[TMP4]], align 4
 ; CHECK-NEXT:    [[TMP6:%.*]] = fmul fast <8 x float> [[TMP1]], [[TMP5]]
-; CHECK-NEXT:    [[ADD8:%.*]] = fadd fast float undef, undef
-; CHECK-NEXT:    [[ADD14:%.*]] = fadd fast float [[ADD8]], undef
-; CHECK-NEXT:    [[ADD20:%.*]] = fadd fast float [[ADD14]], undef
-; CHECK-NEXT:    [[ADD26:%.*]] = fadd fast float [[ADD20]], undef
-; CHECK-NEXT:    [[ADD32:%.*]] = fadd fast float [[ADD26]], undef
-; CHECK-NEXT:    [[ADD38:%.*]] = fadd fast float [[ADD32]], undef
-; CHECK-NEXT:    [[ADD44:%.*]] = fadd fast float [[ADD38]], undef
 ; CHECK-NEXT:    [[ADD47:%.*]] = add nsw i64 [[MUL]], 8
 ; CHECK-NEXT:    [[ARRAYIDX48:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[ADD47]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = load float, float* [[ARRAYIDX48]], align 4
@@ -369,7 +350,6 @@ define i32 @long_red(float* noalias %A, float* noalias %B, i32 %n) {
 ; CHECK-NEXT:    [[BIN_RDX4:%.*]] = fadd fast <8 x float> [[BIN_RDX2]], [[RDX_SHUF3]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <8 x float> [[BIN_RDX4]], i32 0
 ; CHECK-NEXT:    [[TMP9:%.*]] = fadd fast float [[TMP8]], [[MUL49]]
-; CHECK-NEXT:    [[ADD50:%.*]] = fadd fast float [[ADD44]], [[MUL49]]
 ; CHECK-NEXT:    [[ADD51]] = fadd fast float [[SUM_082]], [[TMP9]]
 ; CHECK-NEXT:    [[INC]] = add nsw i64 [[I_083]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INC]], [[TMP3]]
@@ -421,13 +401,6 @@ define i32 @long_red(float* noalias %A, float* noalias %B, i32 %n) {
 ; STORE-NEXT:    [[TMP4:%.*]] = bitcast float* [[ARRAYIDX2]] to <8 x float>*
 ; STORE-NEXT:    [[TMP5:%.*]] = load <8 x float>, <8 x float>* [[TMP4]], align 4
 ; STORE-NEXT:    [[TMP6:%.*]] = fmul fast <8 x float> [[TMP1]], [[TMP5]]
-; STORE-NEXT:    [[ADD8:%.*]] = fadd fast float undef, undef
-; STORE-NEXT:    [[ADD14:%.*]] = fadd fast float [[ADD8]], undef
-; STORE-NEXT:    [[ADD20:%.*]] = fadd fast float [[ADD14]], undef
-; STORE-NEXT:    [[ADD26:%.*]] = fadd fast float [[ADD20]], undef
-; STORE-NEXT:    [[ADD32:%.*]] = fadd fast float [[ADD26]], undef
-; STORE-NEXT:    [[ADD38:%.*]] = fadd fast float [[ADD32]], undef
-; STORE-NEXT:    [[ADD44:%.*]] = fadd fast float [[ADD38]], undef
 ; STORE-NEXT:    [[ADD47:%.*]] = add nsw i64 [[MUL]], 8
 ; STORE-NEXT:    [[ARRAYIDX48:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[ADD47]]
 ; STORE-NEXT:    [[TMP7:%.*]] = load float, float* [[ARRAYIDX48]], align 4
@@ -440,7 +413,6 @@ define i32 @long_red(float* noalias %A, float* noalias %B, i32 %n) {
 ; STORE-NEXT:    [[BIN_RDX4:%.*]] = fadd fast <8 x float> [[BIN_RDX2]], [[RDX_SHUF3]]
 ; STORE-NEXT:    [[TMP8:%.*]] = extractelement <8 x float> [[BIN_RDX4]], i32 0
 ; STORE-NEXT:    [[TMP9:%.*]] = fadd fast float [[TMP8]], [[MUL49]]
-; STORE-NEXT:    [[ADD50:%.*]] = fadd fast float [[ADD44]], [[MUL49]]
 ; STORE-NEXT:    [[ADD51]] = fadd fast float [[SUM_082]], [[TMP9]]
 ; STORE-NEXT:    [[INC]] = add nsw i64 [[I_083]], 1
 ; STORE-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INC]], [[TMP3]]
@@ -576,16 +548,12 @@ define i32 @chain_red(float* noalias %A, float* noalias %B, i32 %n) {
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float* [[ARRAYIDX2]] to <4 x float>*
 ; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x float>, <4 x float>* [[TMP3]], align 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = fmul fast <4 x float> [[TMP1]], [[TMP4]]
-; CHECK-NEXT:    [[ADD:%.*]] = fadd fast float [[SUM_042]], undef
-; CHECK-NEXT:    [[ADD9:%.*]] = fadd fast float [[ADD]], undef
-; CHECK-NEXT:    [[ADD15:%.*]] = fadd fast float [[ADD9]], undef
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x float> [[TMP5]], <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = fadd fast <4 x float> [[TMP5]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x float> [[BIN_RDX]], <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX2:%.*]] = fadd fast <4 x float> [[BIN_RDX]], [[RDX_SHUF1]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[BIN_RDX2]], i32 0
 ; CHECK-NEXT:    [[OP_EXTRA]] = fadd fast float [[TMP6]], [[SUM_042]]
-; CHECK-NEXT:    [[ADD21:%.*]] = fadd fast float [[ADD15]], undef
 ; CHECK-NEXT:    [[INC]] = add nsw i64 [[I_043]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INC]], [[TMP2]]
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_FOR_END_CRIT_EDGE:%.*]], label [[FOR_BODY]]
@@ -622,16 +590,12 @@ define i32 @chain_red(float* noalias %A, float* noalias %B, i32 %n) {
 ; STORE-NEXT:    [[TMP3:%.*]] = bitcast float* [[ARRAYIDX2]] to <4 x float>*
 ; STORE-NEXT:    [[TMP4:%.*]] = load <4 x float>, <4 x float>* [[TMP3]], align 4
 ; STORE-NEXT:    [[TMP5:%.*]] = fmul fast <4 x float> [[TMP1]], [[TMP4]]
-; STORE-NEXT:    [[ADD:%.*]] = fadd fast float [[SUM_042]], undef
-; STORE-NEXT:    [[ADD9:%.*]] = fadd fast float [[ADD]], undef
-; STORE-NEXT:    [[ADD15:%.*]] = fadd fast float [[ADD9]], undef
 ; STORE-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x float> [[TMP5]], <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
 ; STORE-NEXT:    [[BIN_RDX:%.*]] = fadd fast <4 x float> [[TMP5]], [[RDX_SHUF]]
 ; STORE-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x float> [[BIN_RDX]], <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
 ; STORE-NEXT:    [[BIN_RDX2:%.*]] = fadd fast <4 x float> [[BIN_RDX]], [[RDX_SHUF1]]
 ; STORE-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[BIN_RDX2]], i32 0
 ; STORE-NEXT:    [[OP_EXTRA]] = fadd fast float [[TMP6]], [[SUM_042]]
-; STORE-NEXT:    [[ADD21:%.*]] = fadd fast float [[ADD15]], undef
 ; STORE-NEXT:    [[INC]] = add nsw i64 [[I_043]], 1
 ; STORE-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INC]], [[TMP2]]
 ; STORE-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_FOR_END_CRIT_EDGE:%.*]], label [[FOR_BODY]]
@@ -1087,14 +1051,11 @@ define i32 @store_red(float* noalias %A, float* noalias %B, float* noalias %C, i
 ; STORE-NEXT:    [[TMP3:%.*]] = bitcast float* [[ARRAYIDX2]] to <4 x float>*
 ; STORE-NEXT:    [[TMP4:%.*]] = load <4 x float>, <4 x float>* [[TMP3]], align 4
 ; STORE-NEXT:    [[TMP5:%.*]] = fmul fast <4 x float> [[TMP2]], [[TMP4]]
-; STORE-NEXT:    [[ADD8:%.*]] = fadd fast float undef, undef
-; STORE-NEXT:    [[ADD14:%.*]] = fadd fast float [[ADD8]], undef
 ; STORE-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x float> [[TMP5]], <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
 ; STORE-NEXT:    [[BIN_RDX:%.*]] = fadd fast <4 x float> [[TMP5]], [[RDX_SHUF]]
 ; STORE-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x float> [[BIN_RDX]], <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
 ; STORE-NEXT:    [[BIN_RDX2:%.*]] = fadd fast <4 x float> [[BIN_RDX]], [[RDX_SHUF1]]
 ; STORE-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[BIN_RDX2]], i32 0
-; STORE-NEXT:    [[ADD20:%.*]] = fadd fast float [[ADD14]], undef
 ; STORE-NEXT:    store float [[TMP6]], float* [[C_ADDR_038]], align 4
 ; STORE-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds float, float* [[C_ADDR_038]], i64 1
 ; STORE-NEXT:    [[INC]] = add nsw i64 [[I_039]], 1
@@ -1169,14 +1130,11 @@ define void @float_red_example4(float* %res) {
 ; STORE-LABEL: @float_red_example4(
 ; STORE-NEXT:  entry:
 ; STORE-NEXT:    [[TMP0:%.*]] = load <4 x float>, <4 x float>* bitcast ([32 x float]* @arr_float to <4 x float>*), align 16
-; STORE-NEXT:    [[ADD:%.*]] = fadd fast float undef, undef
-; STORE-NEXT:    [[ADD_1:%.*]] = fadd fast float undef, [[ADD]]
 ; STORE-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
 ; STORE-NEXT:    [[BIN_RDX:%.*]] = fadd fast <4 x float> [[TMP0]], [[RDX_SHUF]]
 ; STORE-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x float> [[BIN_RDX]], <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
 ; STORE-NEXT:    [[BIN_RDX2:%.*]] = fadd fast <4 x float> [[BIN_RDX]], [[RDX_SHUF1]]
 ; STORE-NEXT:    [[TMP1:%.*]] = extractelement <4 x float> [[BIN_RDX2]], i32 0
-; STORE-NEXT:    [[ADD_2:%.*]] = fadd fast float undef, [[ADD_1]]
 ; STORE-NEXT:    store float [[TMP1]], float* [[RES:%.*]], align 16
 ; STORE-NEXT:    ret void
 ;
@@ -1216,12 +1174,6 @@ define void @float_red_example8(float* %res) {
 ; STORE-LABEL: @float_red_example8(
 ; STORE-NEXT:  entry:
 ; STORE-NEXT:    [[TMP0:%.*]] = load <8 x float>, <8 x float>* bitcast ([32 x float]* @arr_float to <8 x float>*), align 16
-; STORE-NEXT:    [[ADD:%.*]] = fadd fast float undef, undef
-; STORE-NEXT:    [[ADD_1:%.*]] = fadd fast float undef, [[ADD]]
-; STORE-NEXT:    [[ADD_2:%.*]] = fadd fast float undef, [[ADD_1]]
-; STORE-NEXT:    [[ADD_3:%.*]] = fadd fast float undef, [[ADD_2]]
-; STORE-NEXT:    [[ADD_4:%.*]] = fadd fast float undef, [[ADD_3]]
-; STORE-NEXT:    [[ADD_5:%.*]] = fadd fast float undef, [[ADD_4]]
 ; STORE-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x float> [[TMP0]], <8 x float> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
 ; STORE-NEXT:    [[BIN_RDX:%.*]] = fadd fast <8 x float> [[TMP0]], [[RDX_SHUF]]
 ; STORE-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x float> [[BIN_RDX]], <8 x float> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -1229,7 +1181,6 @@ define void @float_red_example8(float* %res) {
 ; STORE-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <8 x float> [[BIN_RDX2]], <8 x float> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; STORE-NEXT:    [[BIN_RDX4:%.*]] = fadd fast <8 x float> [[BIN_RDX2]], [[RDX_SHUF3]]
 ; STORE-NEXT:    [[TMP1:%.*]] = extractelement <8 x float> [[BIN_RDX4]], i32 0
-; STORE-NEXT:    [[ADD_6:%.*]] = fadd fast float undef, [[ADD_5]]
 ; STORE-NEXT:    store float [[TMP1]], float* [[RES:%.*]], align 16
 ; STORE-NEXT:    ret void
 ;
@@ -1293,20 +1244,6 @@ define void @float_red_example16(float* %res) {
 ; STORE-LABEL: @float_red_example16(
 ; STORE-NEXT:  entry:
 ; STORE-NEXT:    [[TMP0:%.*]] = load <16 x float>, <16 x float>* bitcast ([32 x float]* @arr_float to <16 x float>*), align 16
-; STORE-NEXT:    [[ADD:%.*]] = fadd fast float undef, undef
-; STORE-NEXT:    [[ADD_1:%.*]] = fadd fast float undef, [[ADD]]
-; STORE-NEXT:    [[ADD_2:%.*]] = fadd fast float undef, [[ADD_1]]
-; STORE-NEXT:    [[ADD_3:%.*]] = fadd fast float undef, [[ADD_2]]
-; STORE-NEXT:    [[ADD_4:%.*]] = fadd fast float undef, [[ADD_3]]
-; STORE-NEXT:    [[ADD_5:%.*]] = fadd fast float undef, [[ADD_4]]
-; STORE-NEXT:    [[ADD_6:%.*]] = fadd fast float undef, [[ADD_5]]
-; STORE-NEXT:    [[ADD_7:%.*]] = fadd fast float undef, [[ADD_6]]
-; STORE-NEXT:    [[ADD_8:%.*]] = fadd fast float undef, [[ADD_7]]
-; STORE-NEXT:    [[ADD_9:%.*]] = fadd fast float undef, [[ADD_8]]
-; STORE-NEXT:    [[ADD_10:%.*]] = fadd fast float undef, [[ADD_9]]
-; STORE-NEXT:    [[ADD_11:%.*]] = fadd fast float undef, [[ADD_10]]
-; STORE-NEXT:    [[ADD_12:%.*]] = fadd fast float undef, [[ADD_11]]
-; STORE-NEXT:    [[ADD_13:%.*]] = fadd fast float undef, [[ADD_12]]
 ; STORE-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <16 x float> [[TMP0]], <16 x float> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; STORE-NEXT:    [[BIN_RDX:%.*]] = fadd fast <16 x float> [[TMP0]], [[RDX_SHUF]]
 ; STORE-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <16 x float> [[BIN_RDX]], <16 x float> undef, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -1316,7 +1253,6 @@ define void @float_red_example16(float* %res) {
 ; STORE-NEXT:    [[RDX_SHUF5:%.*]] = shufflevector <16 x float> [[BIN_RDX4]], <16 x float> undef, <16 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; STORE-NEXT:    [[BIN_RDX6:%.*]] = fadd fast <16 x float> [[BIN_RDX4]], [[RDX_SHUF5]]
 ; STORE-NEXT:    [[TMP1:%.*]] = extractelement <16 x float> [[BIN_RDX6]], i32 0
-; STORE-NEXT:    [[ADD_14:%.*]] = fadd fast float undef, [[ADD_13]]
 ; STORE-NEXT:    store float [[TMP1]], float* [[RES:%.*]], align 16
 ; STORE-NEXT:    ret void
 ;
@@ -1372,14 +1308,11 @@ define void @i32_red_example4(i32* %res) {
 ; STORE-LABEL: @i32_red_example4(
 ; STORE-NEXT:  entry:
 ; STORE-NEXT:    [[TMP0:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([32 x i32]* @arr_i32 to <4 x i32>*), align 16
-; STORE-NEXT:    [[ADD:%.*]] = add nsw i32 undef, undef
-; STORE-NEXT:    [[ADD_1:%.*]] = add nsw i32 undef, [[ADD]]
 ; STORE-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
 ; STORE-NEXT:    [[BIN_RDX:%.*]] = add nsw <4 x i32> [[TMP0]], [[RDX_SHUF]]
 ; STORE-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x i32> [[BIN_RDX]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
 ; STORE-NEXT:    [[BIN_RDX2:%.*]] = add nsw <4 x i32> [[BIN_RDX]], [[RDX_SHUF1]]
 ; STORE-NEXT:    [[TMP1:%.*]] = extractelement <4 x i32> [[BIN_RDX2]], i32 0
-; STORE-NEXT:    [[ADD_2:%.*]] = add nsw i32 undef, [[ADD_1]]
 ; STORE-NEXT:    store i32 [[TMP1]], i32* [[RES:%.*]], align 16
 ; STORE-NEXT:    ret void
 ;
@@ -1419,12 +1352,6 @@ define void @i32_red_example8(i32* %res) {
 ; STORE-LABEL: @i32_red_example8(
 ; STORE-NEXT:  entry:
 ; STORE-NEXT:    [[TMP0:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([32 x i32]* @arr_i32 to <8 x i32>*), align 16
-; STORE-NEXT:    [[ADD:%.*]] = add nsw i32 undef, undef
-; STORE-NEXT:    [[ADD_1:%.*]] = add nsw i32 undef, [[ADD]]
-; STORE-NEXT:    [[ADD_2:%.*]] = add nsw i32 undef, [[ADD_1]]
-; STORE-NEXT:    [[ADD_3:%.*]] = add nsw i32 undef, [[ADD_2]]
-; STORE-NEXT:    [[ADD_4:%.*]] = add nsw i32 undef, [[ADD_3]]
-; STORE-NEXT:    [[ADD_5:%.*]] = add nsw i32 undef, [[ADD_4]]
 ; STORE-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x i32> [[TMP0]], <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
 ; STORE-NEXT:    [[BIN_RDX:%.*]] = add nsw <8 x i32> [[TMP0]], [[RDX_SHUF]]
 ; STORE-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x i32> [[BIN_RDX]], <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -1432,7 +1359,6 @@ define void @i32_red_example8(i32* %res) {
 ; STORE-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <8 x i32> [[BIN_RDX2]], <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; STORE-NEXT:    [[BIN_RDX4:%.*]] = add nsw <8 x i32> [[BIN_RDX2]], [[RDX_SHUF3]]
 ; STORE-NEXT:    [[TMP1:%.*]] = extractelement <8 x i32> [[BIN_RDX4]], i32 0
-; STORE-NEXT:    [[ADD_6:%.*]] = add nsw i32 undef, [[ADD_5]]
 ; STORE-NEXT:    store i32 [[TMP1]], i32* [[RES:%.*]], align 16
 ; STORE-NEXT:    ret void
 ;
@@ -1496,20 +1422,6 @@ define void @i32_red_example16(i32* %res) {
 ; STORE-LABEL: @i32_red_example16(
 ; STORE-NEXT:  entry:
 ; STORE-NEXT:    [[TMP0:%.*]] = load <16 x i32>, <16 x i32>* bitcast ([32 x i32]* @arr_i32 to <16 x i32>*), align 16
-; STORE-NEXT:    [[ADD:%.*]] = add nsw i32 undef, undef
-; STORE-NEXT:    [[ADD_1:%.*]] = add nsw i32 undef, [[ADD]]
-; STORE-NEXT:    [[ADD_2:%.*]] = add nsw i32 undef, [[ADD_1]]
-; STORE-NEXT:    [[ADD_3:%.*]] = add nsw i32 undef, [[ADD_2]]
-; STORE-NEXT:    [[ADD_4:%.*]] = add nsw i32 undef, [[ADD_3]]
-; STORE-NEXT:    [[ADD_5:%.*]] = add nsw i32 undef, [[ADD_4]]
-; STORE-NEXT:    [[ADD_6:%.*]] = add nsw i32 undef, [[ADD_5]]
-; STORE-NEXT:    [[ADD_7:%.*]] = add nsw i32 undef, [[ADD_6]]
-; STORE-NEXT:    [[ADD_8:%.*]] = add nsw i32 undef, [[ADD_7]]
-; STORE-NEXT:    [[ADD_9:%.*]] = add nsw i32 undef, [[ADD_8]]
-; STORE-NEXT:    [[ADD_10:%.*]] = add nsw i32 undef, [[ADD_9]]
-; STORE-NEXT:    [[ADD_11:%.*]] = add nsw i32 undef, [[ADD_10]]
-; STORE-NEXT:    [[ADD_12:%.*]] = add nsw i32 undef, [[ADD_11]]
-; STORE-NEXT:    [[ADD_13:%.*]] = add nsw i32 undef, [[ADD_12]]
 ; STORE-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <16 x i32> [[TMP0]], <16 x i32> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; STORE-NEXT:    [[BIN_RDX:%.*]] = add nsw <16 x i32> [[TMP0]], [[RDX_SHUF]]
 ; STORE-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <16 x i32> [[BIN_RDX]], <16 x i32> undef, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -1519,7 +1431,6 @@ define void @i32_red_example16(i32* %res) {
 ; STORE-NEXT:    [[RDX_SHUF5:%.*]] = shufflevector <16 x i32> [[BIN_RDX4]], <16 x i32> undef, <16 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; STORE-NEXT:    [[BIN_RDX6:%.*]] = add nsw <16 x i32> [[BIN_RDX4]], [[RDX_SHUF5]]
 ; STORE-NEXT:    [[TMP1:%.*]] = extractelement <16 x i32> [[BIN_RDX6]], i32 0
-; STORE-NEXT:    [[ADD_14:%.*]] = add nsw i32 undef, [[ADD_13]]
 ; STORE-NEXT:    store i32 [[TMP1]], i32* [[RES:%.*]], align 16
 ; STORE-NEXT:    ret void
 ;
@@ -1631,36 +1542,6 @@ define void @i32_red_example32(i32* %res) {
 ; STORE-LABEL: @i32_red_example32(
 ; STORE-NEXT:  entry:
 ; STORE-NEXT:    [[TMP0:%.*]] = load <32 x i32>, <32 x i32>* bitcast ([32 x i32]* @arr_i32 to <32 x i32>*), align 16
-; STORE-NEXT:    [[ADD:%.*]] = add nsw i32 undef, undef
-; STORE-NEXT:    [[ADD_1:%.*]] = add nsw i32 undef, [[ADD]]
-; STORE-NEXT:    [[ADD_2:%.*]] = add nsw i32 undef, [[ADD_1]]
-; STORE-NEXT:    [[ADD_3:%.*]] = add nsw i32 undef, [[ADD_2]]
-; STORE-NEXT:    [[ADD_4:%.*]] = add nsw i32 undef, [[ADD_3]]
-; STORE-NEXT:    [[ADD_5:%.*]] = add nsw i32 undef, [[ADD_4]]
-; STORE-NEXT:    [[ADD_6:%.*]] = add nsw i32 undef, [[ADD_5]]
-; STORE-NEXT:    [[ADD_7:%.*]] = add nsw i32 undef, [[ADD_6]]
-; STORE-NEXT:    [[ADD_8:%.*]] = add nsw i32 undef, [[ADD_7]]
-; STORE-NEXT:    [[ADD_9:%.*]] = add nsw i32 undef, [[ADD_8]]
-; STORE-NEXT:    [[ADD_10:%.*]] = add nsw i32 undef, [[ADD_9]]
-; STORE-NEXT:    [[ADD_11:%.*]] = add nsw i32 undef, [[ADD_10]]
-; STORE-NEXT:    [[ADD_12:%.*]] = add nsw i32 undef, [[ADD_11]]
-; STORE-NEXT:    [[ADD_13:%.*]] = add nsw i32 undef, [[ADD_12]]
-; STORE-NEXT:    [[ADD_14:%.*]] = add nsw i32 undef, [[ADD_13]]
-; STORE-NEXT:    [[ADD_15:%.*]] = add nsw i32 undef, [[ADD_14]]
-; STORE-NEXT:    [[ADD_16:%.*]] = add nsw i32 undef, [[ADD_15]]
-; STORE-NEXT:    [[ADD_17:%.*]] = add nsw i32 undef, [[ADD_16]]
-; STORE-NEXT:    [[ADD_18:%.*]] = add nsw i32 undef, [[ADD_17]]
-; STORE-NEXT:    [[ADD_19:%.*]] = add nsw i32 undef, [[ADD_18]]
-; STORE-NEXT:    [[ADD_20:%.*]] = add nsw i32 undef, [[ADD_19]]
-; STORE-NEXT:    [[ADD_21:%.*]] = add nsw i32 undef, [[ADD_20]]
-; STORE-NEXT:    [[ADD_22:%.*]] = add nsw i32 undef, [[ADD_21]]
-; STORE-NEXT:    [[ADD_23:%.*]] = add nsw i32 undef, [[ADD_22]]
-; STORE-NEXT:    [[ADD_24:%.*]] = add nsw i32 undef, [[ADD_23]]
-; STORE-NEXT:    [[ADD_25:%.*]] = add nsw i32 undef, [[ADD_24]]
-; STORE-NEXT:    [[ADD_26:%.*]] = add nsw i32 undef, [[ADD_25]]
-; STORE-NEXT:    [[ADD_27:%.*]] = add nsw i32 undef, [[ADD_26]]
-; STORE-NEXT:    [[ADD_28:%.*]] = add nsw i32 undef, [[ADD_27]]
-; STORE-NEXT:    [[ADD_29:%.*]] = add nsw i32 undef, [[ADD_28]]
 ; STORE-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <32 x i32> [[TMP0]], <32 x i32> undef, <32 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; STORE-NEXT:    [[BIN_RDX:%.*]] = add nsw <32 x i32> [[TMP0]], [[RDX_SHUF]]
 ; STORE-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <32 x i32> [[BIN_RDX]], <32 x i32> undef, <32 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -1672,7 +1553,6 @@ define void @i32_red_example32(i32* %res) {
 ; STORE-NEXT:    [[RDX_SHUF7:%.*]] = shufflevector <32 x i32> [[BIN_RDX6]], <32 x i32> undef, <32 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; STORE-NEXT:    [[BIN_RDX8:%.*]] = add nsw <32 x i32> [[BIN_RDX6]], [[RDX_SHUF7]]
 ; STORE-NEXT:    [[TMP1:%.*]] = extractelement <32 x i32> [[BIN_RDX8]], i32 0
-; STORE-NEXT:    [[ADD_30:%.*]] = add nsw i32 undef, [[ADD_29]]
 ; STORE-NEXT:    store i32 [[TMP1]], i32* [[RES:%.*]], align 16
 ; STORE-NEXT:    ret void
 ;
@@ -1750,12 +1630,6 @@ define void @i32_red_call(i32 %val) {
 ; CHECK-LABEL: @i32_red_call(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([32 x i32]* @arr_i32 to <8 x i32>*), align 16
-; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 undef, undef
-; CHECK-NEXT:    [[ADD_1:%.*]] = add nsw i32 undef, [[ADD]]
-; CHECK-NEXT:    [[ADD_2:%.*]] = add nsw i32 undef, [[ADD_1]]
-; CHECK-NEXT:    [[ADD_3:%.*]] = add nsw i32 undef, [[ADD_2]]
-; CHECK-NEXT:    [[ADD_4:%.*]] = add nsw i32 undef, [[ADD_3]]
-; CHECK-NEXT:    [[ADD_5:%.*]] = add nsw i32 undef, [[ADD_4]]
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x i32> [[TMP0]], <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = add nsw <8 x i32> [[TMP0]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x i32> [[BIN_RDX]], <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -1763,19 +1637,12 @@ define void @i32_red_call(i32 %val) {
 ; CHECK-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <8 x i32> [[BIN_RDX2]], <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX4:%.*]] = add nsw <8 x i32> [[BIN_RDX2]], [[RDX_SHUF3]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <8 x i32> [[BIN_RDX4]], i32 0
-; CHECK-NEXT:    [[ADD_6:%.*]] = add nsw i32 undef, [[ADD_5]]
 ; CHECK-NEXT:    [[RES:%.*]] = call i32 @foobar(i32 [[TMP1]])
 ; CHECK-NEXT:    ret void
 ;
 ; STORE-LABEL: @i32_red_call(
 ; STORE-NEXT:  entry:
 ; STORE-NEXT:    [[TMP0:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([32 x i32]* @arr_i32 to <8 x i32>*), align 16
-; STORE-NEXT:    [[ADD:%.*]] = add nsw i32 undef, undef
-; STORE-NEXT:    [[ADD_1:%.*]] = add nsw i32 undef, [[ADD]]
-; STORE-NEXT:    [[ADD_2:%.*]] = add nsw i32 undef, [[ADD_1]]
-; STORE-NEXT:    [[ADD_3:%.*]] = add nsw i32 undef, [[ADD_2]]
-; STORE-NEXT:    [[ADD_4:%.*]] = add nsw i32 undef, [[ADD_3]]
-; STORE-NEXT:    [[ADD_5:%.*]] = add nsw i32 undef, [[ADD_4]]
 ; STORE-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x i32> [[TMP0]], <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
 ; STORE-NEXT:    [[BIN_RDX:%.*]] = add nsw <8 x i32> [[TMP0]], [[RDX_SHUF]]
 ; STORE-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x i32> [[BIN_RDX]], <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -1783,7 +1650,6 @@ define void @i32_red_call(i32 %val) {
 ; STORE-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <8 x i32> [[BIN_RDX2]], <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; STORE-NEXT:    [[BIN_RDX4:%.*]] = add nsw <8 x i32> [[BIN_RDX2]], [[RDX_SHUF3]]
 ; STORE-NEXT:    [[TMP1:%.*]] = extractelement <8 x i32> [[BIN_RDX4]], i32 0
-; STORE-NEXT:    [[ADD_6:%.*]] = add nsw i32 undef, [[ADD_5]]
 ; STORE-NEXT:    [[RES:%.*]] = call i32 @foobar(i32 [[TMP1]])
 ; STORE-NEXT:    ret void
 ;
@@ -1811,12 +1677,6 @@ define void @i32_red_invoke(i32 %val) personality i32 (...)* @__gxx_personality_
 ; CHECK-LABEL: @i32_red_invoke(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([32 x i32]* @arr_i32 to <8 x i32>*), align 16
-; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 undef, undef
-; CHECK-NEXT:    [[ADD_1:%.*]] = add nsw i32 undef, [[ADD]]
-; CHECK-NEXT:    [[ADD_2:%.*]] = add nsw i32 undef, [[ADD_1]]
-; CHECK-NEXT:    [[ADD_3:%.*]] = add nsw i32 undef, [[ADD_2]]
-; CHECK-NEXT:    [[ADD_4:%.*]] = add nsw i32 undef, [[ADD_3]]
-; CHECK-NEXT:    [[ADD_5:%.*]] = add nsw i32 undef, [[ADD_4]]
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x i32> [[TMP0]], <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = add nsw <8 x i32> [[TMP0]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x i32> [[BIN_RDX]], <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -1824,7 +1684,6 @@ define void @i32_red_invoke(i32 %val) personality i32 (...)* @__gxx_personality_
 ; CHECK-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <8 x i32> [[BIN_RDX2]], <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX4:%.*]] = add nsw <8 x i32> [[BIN_RDX2]], [[RDX_SHUF3]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <8 x i32> [[BIN_RDX4]], i32 0
-; CHECK-NEXT:    [[ADD_6:%.*]] = add nsw i32 undef, [[ADD_5]]
 ; CHECK-NEXT:    [[RES:%.*]] = invoke i32 @foobar(i32 [[TMP1]])
 ; CHECK-NEXT:    to label [[NORMAL:%.*]] unwind label [[EXCEPTION:%.*]]
 ; CHECK:       exception:
@@ -1837,12 +1696,6 @@ define void @i32_red_invoke(i32 %val) personality i32 (...)* @__gxx_personality_
 ; STORE-LABEL: @i32_red_invoke(
 ; STORE-NEXT:  entry:
 ; STORE-NEXT:    [[TMP0:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([32 x i32]* @arr_i32 to <8 x i32>*), align 16
-; STORE-NEXT:    [[ADD:%.*]] = add nsw i32 undef, undef
-; STORE-NEXT:    [[ADD_1:%.*]] = add nsw i32 undef, [[ADD]]
-; STORE-NEXT:    [[ADD_2:%.*]] = add nsw i32 undef, [[ADD_1]]
-; STORE-NEXT:    [[ADD_3:%.*]] = add nsw i32 undef, [[ADD_2]]
-; STORE-NEXT:    [[ADD_4:%.*]] = add nsw i32 undef, [[ADD_3]]
-; STORE-NEXT:    [[ADD_5:%.*]] = add nsw i32 undef, [[ADD_4]]
 ; STORE-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x i32> [[TMP0]], <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
 ; STORE-NEXT:    [[BIN_RDX:%.*]] = add nsw <8 x i32> [[TMP0]], [[RDX_SHUF]]
 ; STORE-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x i32> [[BIN_RDX]], <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -1850,7 +1703,6 @@ define void @i32_red_invoke(i32 %val) personality i32 (...)* @__gxx_personality_
 ; STORE-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <8 x i32> [[BIN_RDX2]], <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; STORE-NEXT:    [[BIN_RDX4:%.*]] = add nsw <8 x i32> [[BIN_RDX2]], [[RDX_SHUF3]]
 ; STORE-NEXT:    [[TMP1:%.*]] = extractelement <8 x i32> [[BIN_RDX4]], i32 0
-; STORE-NEXT:    [[ADD_6:%.*]] = add nsw i32 undef, [[ADD_5]]
 ; STORE-NEXT:    [[RES:%.*]] = invoke i32 @foobar(i32 [[TMP1]])
 ; STORE-NEXT:    to label [[NORMAL:%.*]] unwind label [[EXCEPTION:%.*]]
 ; STORE:       exception:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/hsub.ll b/llvm/test/Transforms/SLPVectorizer/X86/hsub.ll
index d6e44aa1d..b7e487eed 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/hsub.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/hsub.ll
@@ -78,34 +78,11 @@ define <4 x float> @test_v4f32(<4 x float> %a, <4 x float> %b) {
 }
 
 define <2 x i64> @test_v2i64(<2 x i64> %a, <2 x i64> %b) {
-; SSE-LABEL: @test_v2i64(
-; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], <2 x i32> <i32 0, i32 2>
-; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i64> [[A]], <2 x i64> [[B]], <2 x i32> <i32 1, i32 3>
-; SSE-NEXT:    [[TMP3:%.*]] = sub <2 x i64> [[TMP1]], [[TMP2]]
-; SSE-NEXT:    ret <2 x i64> [[TMP3]]
-;
-; SLM-LABEL: @test_v2i64(
-; SLM-NEXT:    [[A0:%.*]] = extractelement <2 x i64> [[A:%.*]], i32 0
-; SLM-NEXT:    [[A1:%.*]] = extractelement <2 x i64> [[A]], i32 1
-; SLM-NEXT:    [[B0:%.*]] = extractelement <2 x i64> [[B:%.*]], i32 0
-; SLM-NEXT:    [[B1:%.*]] = extractelement <2 x i64> [[B]], i32 1
-; SLM-NEXT:    [[R0:%.*]] = sub i64 [[A0]], [[A1]]
-; SLM-NEXT:    [[R1:%.*]] = sub i64 [[B0]], [[B1]]
-; SLM-NEXT:    [[R00:%.*]] = insertelement <2 x i64> undef, i64 [[R0]], i32 0
-; SLM-NEXT:    [[R01:%.*]] = insertelement <2 x i64> [[R00]], i64 [[R1]], i32 1
-; SLM-NEXT:    ret <2 x i64> [[R01]]
-;
-; AVX-LABEL: @test_v2i64(
-; AVX-NEXT:    [[TMP1:%.*]] = shufflevector <2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], <2 x i32> <i32 0, i32 2>
-; AVX-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i64> [[A]], <2 x i64> [[B]], <2 x i32> <i32 1, i32 3>
-; AVX-NEXT:    [[TMP3:%.*]] = sub <2 x i64> [[TMP1]], [[TMP2]]
-; AVX-NEXT:    ret <2 x i64> [[TMP3]]
-;
-; AVX512-LABEL: @test_v2i64(
-; AVX512-NEXT:    [[TMP1:%.*]] = shufflevector <2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], <2 x i32> <i32 0, i32 2>
-; AVX512-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i64> [[A]], <2 x i64> [[B]], <2 x i32> <i32 1, i32 3>
-; AVX512-NEXT:    [[TMP3:%.*]] = sub <2 x i64> [[TMP1]], [[TMP2]]
-; AVX512-NEXT:    ret <2 x i64> [[TMP3]]
+; CHECK-LABEL: @test_v2i64(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i64> [[A]], <2 x i64> [[B]], <2 x i32> <i32 1, i32 3>
+; CHECK-NEXT:    [[TMP3:%.*]] = sub <2 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret <2 x i64> [[TMP3]]
 ;
   %a0 = extractelement <2 x i64> %a, i32 0
   %a1 = extractelement <2 x i64> %a, i32 1
@@ -322,14 +299,10 @@ define <4 x i64> @test_v4i64(<4 x i64> %a, <4 x i64> %b) {
 ; SSE-NEXT:    ret <4 x i64> [[R03]]
 ;
 ; SLM-LABEL: @test_v4i64(
-; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i64> [[A:%.*]], <4 x i64> [[B:%.*]], <2 x i32> <i32 0, i32 4>
-; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <2 x i32> <i32 1, i32 5>
-; SLM-NEXT:    [[TMP3:%.*]] = sub <2 x i64> [[TMP1]], [[TMP2]]
-; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <2 x i32> <i32 2, i32 6>
-; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <2 x i32> <i32 3, i32 7>
-; SLM-NEXT:    [[TMP6:%.*]] = sub <2 x i64> [[TMP4]], [[TMP5]]
-; SLM-NEXT:    [[R03:%.*]] = shufflevector <2 x i64> [[TMP3]], <2 x i64> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; SLM-NEXT:    ret <4 x i64> [[R03]]
+; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i64> [[A:%.*]], <4 x i64> [[B:%.*]], <4 x i32> <i32 0, i32 4, i32 2, i32 6>
+; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <4 x i32> <i32 1, i32 5, i32 3, i32 7>
+; SLM-NEXT:    [[TMP3:%.*]] = sub <4 x i64> [[TMP1]], [[TMP2]]
+; SLM-NEXT:    ret <4 x i64> [[TMP3]]
 ;
 ; AVX-LABEL: @test_v4i64(
 ; AVX-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i64> [[A:%.*]], <4 x i64> [[B:%.*]], <4 x i32> <i32 0, i32 4, i32 2, i32 6>
@@ -374,14 +347,10 @@ define <8 x i32> @test_v8i32(<8 x i32> %a, <8 x i32> %b) {
 ; SSE-NEXT:    ret <8 x i32> [[R07]]
 ;
 ; SLM-LABEL: @test_v8i32(
-; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A:%.*]], <8 x i32> [[B:%.*]], <4 x i32> <i32 0, i32 2, i32 8, i32 10>
-; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <4 x i32> <i32 1, i32 3, i32 9, i32 11>
-; SLM-NEXT:    [[TMP3:%.*]] = sub <4 x i32> [[TMP1]], [[TMP2]]
-; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <4 x i32> <i32 4, i32 6, i32 12, i32 14>
-; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <4 x i32> <i32 5, i32 7, i32 13, i32 15>
-; SLM-NEXT:    [[TMP6:%.*]] = sub <4 x i32> [[TMP4]], [[TMP5]]
-; SLM-NEXT:    [[R07:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SLM-NEXT:    ret <8 x i32> [[R07]]
+; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A:%.*]], <8 x i32> [[B:%.*]], <8 x i32> <i32 0, i32 2, i32 8, i32 10, i32 4, i32 6, i32 12, i32 14>
+; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <8 x i32> <i32 1, i32 3, i32 9, i32 11, i32 5, i32 7, i32 13, i32 15>
+; SLM-NEXT:    [[TMP3:%.*]] = sub <8 x i32> [[TMP1]], [[TMP2]]
+; SLM-NEXT:    ret <8 x i32> [[TMP3]]
 ;
 ; AVX-LABEL: @test_v8i32(
 ; AVX-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A:%.*]], <8 x i32> [[B:%.*]], <8 x i32> <i32 0, i32 2, i32 8, i32 10, i32 4, i32 6, i32 12, i32 14>
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/insert-element-build-vector.ll b/llvm/test/Transforms/SLPVectorizer/X86/insert-element-build-vector.ll
index 5f6e8f143..711347e62 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/insert-element-build-vector.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/insert-element-build-vector.ll
@@ -1,36 +1,23 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S -slp-vectorizer -slp-threshold=-10000 < %s | FileCheck %s
-; RUN: opt -S -slp-vectorizer -slp-threshold=0 < %s | FileCheck %s --check-prefix=ZEROTHRESH
-target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-n8:16:32:64-S128"
+; RUN: opt -S -slp-vectorizer -slp-threshold=-10000 < %s | FileCheck %s --check-prefixes=ANY,BIGTHRESH
+; RUN: opt -S -slp-vectorizer -slp-threshold=0 < %s | FileCheck %s --check-prefixes=ANY,ZEROTHRESH
 
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.8.0"
 
 define <4 x float> @simple_select(<4 x float> %a, <4 x float> %b, <4 x i32> %c) #0 {
-; CHECK-LABEL: @simple_select(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne <4 x i32> [[C:%.*]], zeroinitializer
-; CHECK-NEXT:    [[TMP2:%.*]] = select <4 x i1> [[TMP1]], <4 x float> [[A:%.*]], <4 x float> [[B:%.*]]
-; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[TMP2]], i32 0
-; CHECK-NEXT:    [[RA:%.*]] = insertelement <4 x float> undef, float [[TMP3]], i32 0
-; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[TMP2]], i32 1
-; CHECK-NEXT:    [[RB:%.*]] = insertelement <4 x float> [[RA]], float [[TMP4]], i32 1
-; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x float> [[TMP2]], i32 2
-; CHECK-NEXT:    [[RC:%.*]] = insertelement <4 x float> [[RB]], float [[TMP5]], i32 2
-; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[TMP2]], i32 3
-; CHECK-NEXT:    [[RD:%.*]] = insertelement <4 x float> [[RC]], float [[TMP6]], i32 3
-; CHECK-NEXT:    ret <4 x float> [[RD]]
-;
-; ZEROTHRESH-LABEL: @simple_select(
-; ZEROTHRESH-NEXT:    [[TMP1:%.*]] = icmp ne <4 x i32> [[C:%.*]], zeroinitializer
-; ZEROTHRESH-NEXT:    [[TMP2:%.*]] = select <4 x i1> [[TMP1]], <4 x float> [[A:%.*]], <4 x float> [[B:%.*]]
-; ZEROTHRESH-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[TMP2]], i32 0
-; ZEROTHRESH-NEXT:    [[RA:%.*]] = insertelement <4 x float> undef, float [[TMP3]], i32 0
-; ZEROTHRESH-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[TMP2]], i32 1
-; ZEROTHRESH-NEXT:    [[RB:%.*]] = insertelement <4 x float> [[RA]], float [[TMP4]], i32 1
-; ZEROTHRESH-NEXT:    [[TMP5:%.*]] = extractelement <4 x float> [[TMP2]], i32 2
-; ZEROTHRESH-NEXT:    [[RC:%.*]] = insertelement <4 x float> [[RB]], float [[TMP5]], i32 2
-; ZEROTHRESH-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[TMP2]], i32 3
-; ZEROTHRESH-NEXT:    [[RD:%.*]] = insertelement <4 x float> [[RC]], float [[TMP6]], i32 3
-; ZEROTHRESH-NEXT:    ret <4 x float> [[RD]]
+; ANY-LABEL: @simple_select(
+; ANY-NEXT:    [[TMP1:%.*]] = icmp ne <4 x i32> [[C:%.*]], zeroinitializer
+; ANY-NEXT:    [[TMP2:%.*]] = select <4 x i1> [[TMP1]], <4 x float> [[A:%.*]], <4 x float> [[B:%.*]]
+; ANY-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[TMP2]], i32 0
+; ANY-NEXT:    [[RA:%.*]] = insertelement <4 x float> undef, float [[TMP3]], i32 0
+; ANY-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[TMP2]], i32 1
+; ANY-NEXT:    [[RB:%.*]] = insertelement <4 x float> [[RA]], float [[TMP4]], i32 1
+; ANY-NEXT:    [[TMP5:%.*]] = extractelement <4 x float> [[TMP2]], i32 2
+; ANY-NEXT:    [[RC:%.*]] = insertelement <4 x float> [[RB]], float [[TMP5]], i32 2
+; ANY-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[TMP2]], i32 3
+; ANY-NEXT:    [[RD:%.*]] = insertelement <4 x float> [[RC]], float [[TMP6]], i32 3
+; ANY-NEXT:    ret <4 x float> [[RD]]
 ;
   %c0 = extractelement <4 x i32> %c, i32 0
   %c1 = extractelement <4 x i32> %c, i32 1
@@ -63,77 +50,41 @@ declare void @llvm.assume(i1) nounwind
 
 ; This entire tree is ephemeral, don't vectorize any of it.
 define <4 x float> @simple_select_eph(<4 x float> %a, <4 x float> %b, <4 x i32> %c) #0 {
-; CHECK-LABEL: @simple_select_eph(
-; CHECK-NEXT:    [[C0:%.*]] = extractelement <4 x i32> [[C:%.*]], i32 0
-; CHECK-NEXT:    [[C1:%.*]] = extractelement <4 x i32> [[C]], i32 1
-; CHECK-NEXT:    [[C2:%.*]] = extractelement <4 x i32> [[C]], i32 2
-; CHECK-NEXT:    [[C3:%.*]] = extractelement <4 x i32> [[C]], i32 3
-; CHECK-NEXT:    [[A0:%.*]] = extractelement <4 x float> [[A:%.*]], i32 0
-; CHECK-NEXT:    [[A1:%.*]] = extractelement <4 x float> [[A]], i32 1
-; CHECK-NEXT:    [[A2:%.*]] = extractelement <4 x float> [[A]], i32 2
-; CHECK-NEXT:    [[A3:%.*]] = extractelement <4 x float> [[A]], i32 3
-; CHECK-NEXT:    [[B0:%.*]] = extractelement <4 x float> [[B:%.*]], i32 0
-; CHECK-NEXT:    [[B1:%.*]] = extractelement <4 x float> [[B]], i32 1
-; CHECK-NEXT:    [[B2:%.*]] = extractelement <4 x float> [[B]], i32 2
-; CHECK-NEXT:    [[B3:%.*]] = extractelement <4 x float> [[B]], i32 3
-; CHECK-NEXT:    [[CMP0:%.*]] = icmp ne i32 [[C0]], 0
-; CHECK-NEXT:    [[CMP1:%.*]] = icmp ne i32 [[C1]], 0
-; CHECK-NEXT:    [[CMP2:%.*]] = icmp ne i32 [[C2]], 0
-; CHECK-NEXT:    [[CMP3:%.*]] = icmp ne i32 [[C3]], 0
-; CHECK-NEXT:    [[S0:%.*]] = select i1 [[CMP0]], float [[A0]], float [[B0]]
-; CHECK-NEXT:    [[S1:%.*]] = select i1 [[CMP1]], float [[A1]], float [[B1]]
-; CHECK-NEXT:    [[S2:%.*]] = select i1 [[CMP2]], float [[A2]], float [[B2]]
-; CHECK-NEXT:    [[S3:%.*]] = select i1 [[CMP3]], float [[A3]], float [[B3]]
-; CHECK-NEXT:    [[RA:%.*]] = insertelement <4 x float> undef, float [[S0]], i32 0
-; CHECK-NEXT:    [[RB:%.*]] = insertelement <4 x float> [[RA]], float [[S1]], i32 1
-; CHECK-NEXT:    [[RC:%.*]] = insertelement <4 x float> [[RB]], float [[S2]], i32 2
-; CHECK-NEXT:    [[RD:%.*]] = insertelement <4 x float> [[RC]], float [[S3]], i32 3
-; CHECK-NEXT:    [[Q0:%.*]] = extractelement <4 x float> [[RD]], i32 0
-; CHECK-NEXT:    [[Q1:%.*]] = extractelement <4 x float> [[RD]], i32 1
-; CHECK-NEXT:    [[Q2:%.*]] = extractelement <4 x float> [[RD]], i32 2
-; CHECK-NEXT:    [[Q3:%.*]] = extractelement <4 x float> [[RD]], i32 3
-; CHECK-NEXT:    [[Q4:%.*]] = fadd float [[Q0]], [[Q1]]
-; CHECK-NEXT:    [[Q5:%.*]] = fadd float [[Q2]], [[Q3]]
-; CHECK-NEXT:    [[Q6:%.*]] = fadd float [[Q4]], [[Q5]]
-; CHECK-NEXT:    [[QI:%.*]] = fcmp olt float [[Q6]], [[Q5]]
-; CHECK-NEXT:    call void @llvm.assume(i1 [[QI]])
-; CHECK-NEXT:    ret <4 x float> undef
-;
-; ZEROTHRESH-LABEL: @simple_select_eph(
-; ZEROTHRESH-NEXT:    [[C0:%.*]] = extractelement <4 x i32> [[C:%.*]], i32 0
-; ZEROTHRESH-NEXT:    [[C1:%.*]] = extractelement <4 x i32> [[C]], i32 1
-; ZEROTHRESH-NEXT:    [[C2:%.*]] = extractelement <4 x i32> [[C]], i32 2
-; ZEROTHRESH-NEXT:    [[C3:%.*]] = extractelement <4 x i32> [[C]], i32 3
-; ZEROTHRESH-NEXT:    [[A0:%.*]] = extractelement <4 x float> [[A:%.*]], i32 0
-; ZEROTHRESH-NEXT:    [[A1:%.*]] = extractelement <4 x float> [[A]], i32 1
-; ZEROTHRESH-NEXT:    [[A2:%.*]] = extractelement <4 x float> [[A]], i32 2
-; ZEROTHRESH-NEXT:    [[A3:%.*]] = extractelement <4 x float> [[A]], i32 3
-; ZEROTHRESH-NEXT:    [[B0:%.*]] = extractelement <4 x float> [[B:%.*]], i32 0
-; ZEROTHRESH-NEXT:    [[B1:%.*]] = extractelement <4 x float> [[B]], i32 1
-; ZEROTHRESH-NEXT:    [[B2:%.*]] = extractelement <4 x float> [[B]], i32 2
-; ZEROTHRESH-NEXT:    [[B3:%.*]] = extractelement <4 x float> [[B]], i32 3
-; ZEROTHRESH-NEXT:    [[CMP0:%.*]] = icmp ne i32 [[C0]], 0
-; ZEROTHRESH-NEXT:    [[CMP1:%.*]] = icmp ne i32 [[C1]], 0
-; ZEROTHRESH-NEXT:    [[CMP2:%.*]] = icmp ne i32 [[C2]], 0
-; ZEROTHRESH-NEXT:    [[CMP3:%.*]] = icmp ne i32 [[C3]], 0
-; ZEROTHRESH-NEXT:    [[S0:%.*]] = select i1 [[CMP0]], float [[A0]], float [[B0]]
-; ZEROTHRESH-NEXT:    [[S1:%.*]] = select i1 [[CMP1]], float [[A1]], float [[B1]]
-; ZEROTHRESH-NEXT:    [[S2:%.*]] = select i1 [[CMP2]], float [[A2]], float [[B2]]
-; ZEROTHRESH-NEXT:    [[S3:%.*]] = select i1 [[CMP3]], float [[A3]], float [[B3]]
-; ZEROTHRESH-NEXT:    [[RA:%.*]] = insertelement <4 x float> undef, float [[S0]], i32 0
-; ZEROTHRESH-NEXT:    [[RB:%.*]] = insertelement <4 x float> [[RA]], float [[S1]], i32 1
-; ZEROTHRESH-NEXT:    [[RC:%.*]] = insertelement <4 x float> [[RB]], float [[S2]], i32 2
-; ZEROTHRESH-NEXT:    [[RD:%.*]] = insertelement <4 x float> [[RC]], float [[S3]], i32 3
-; ZEROTHRESH-NEXT:    [[Q0:%.*]] = extractelement <4 x float> [[RD]], i32 0
-; ZEROTHRESH-NEXT:    [[Q1:%.*]] = extractelement <4 x float> [[RD]], i32 1
-; ZEROTHRESH-NEXT:    [[Q2:%.*]] = extractelement <4 x float> [[RD]], i32 2
-; ZEROTHRESH-NEXT:    [[Q3:%.*]] = extractelement <4 x float> [[RD]], i32 3
-; ZEROTHRESH-NEXT:    [[Q4:%.*]] = fadd float [[Q0]], [[Q1]]
-; ZEROTHRESH-NEXT:    [[Q5:%.*]] = fadd float [[Q2]], [[Q3]]
-; ZEROTHRESH-NEXT:    [[Q6:%.*]] = fadd float [[Q4]], [[Q5]]
-; ZEROTHRESH-NEXT:    [[QI:%.*]] = fcmp olt float [[Q6]], [[Q5]]
-; ZEROTHRESH-NEXT:    call void @llvm.assume(i1 [[QI]])
-; ZEROTHRESH-NEXT:    ret <4 x float> undef
+; ANY-LABEL: @simple_select_eph(
+; ANY-NEXT:    [[C0:%.*]] = extractelement <4 x i32> [[C:%.*]], i32 0
+; ANY-NEXT:    [[C1:%.*]] = extractelement <4 x i32> [[C]], i32 1
+; ANY-NEXT:    [[C2:%.*]] = extractelement <4 x i32> [[C]], i32 2
+; ANY-NEXT:    [[C3:%.*]] = extractelement <4 x i32> [[C]], i32 3
+; ANY-NEXT:    [[A0:%.*]] = extractelement <4 x float> [[A:%.*]], i32 0
+; ANY-NEXT:    [[A1:%.*]] = extractelement <4 x float> [[A]], i32 1
+; ANY-NEXT:    [[A2:%.*]] = extractelement <4 x float> [[A]], i32 2
+; ANY-NEXT:    [[A3:%.*]] = extractelement <4 x float> [[A]], i32 3
+; ANY-NEXT:    [[B0:%.*]] = extractelement <4 x float> [[B:%.*]], i32 0
+; ANY-NEXT:    [[B1:%.*]] = extractelement <4 x float> [[B]], i32 1
+; ANY-NEXT:    [[B2:%.*]] = extractelement <4 x float> [[B]], i32 2
+; ANY-NEXT:    [[B3:%.*]] = extractelement <4 x float> [[B]], i32 3
+; ANY-NEXT:    [[CMP0:%.*]] = icmp ne i32 [[C0]], 0
+; ANY-NEXT:    [[CMP1:%.*]] = icmp ne i32 [[C1]], 0
+; ANY-NEXT:    [[CMP2:%.*]] = icmp ne i32 [[C2]], 0
+; ANY-NEXT:    [[CMP3:%.*]] = icmp ne i32 [[C3]], 0
+; ANY-NEXT:    [[S0:%.*]] = select i1 [[CMP0]], float [[A0]], float [[B0]]
+; ANY-NEXT:    [[S1:%.*]] = select i1 [[CMP1]], float [[A1]], float [[B1]]
+; ANY-NEXT:    [[S2:%.*]] = select i1 [[CMP2]], float [[A2]], float [[B2]]
+; ANY-NEXT:    [[S3:%.*]] = select i1 [[CMP3]], float [[A3]], float [[B3]]
+; ANY-NEXT:    [[RA:%.*]] = insertelement <4 x float> undef, float [[S0]], i32 0
+; ANY-NEXT:    [[RB:%.*]] = insertelement <4 x float> [[RA]], float [[S1]], i32 1
+; ANY-NEXT:    [[RC:%.*]] = insertelement <4 x float> [[RB]], float [[S2]], i32 2
+; ANY-NEXT:    [[RD:%.*]] = insertelement <4 x float> [[RC]], float [[S3]], i32 3
+; ANY-NEXT:    [[Q0:%.*]] = extractelement <4 x float> [[RD]], i32 0
+; ANY-NEXT:    [[Q1:%.*]] = extractelement <4 x float> [[RD]], i32 1
+; ANY-NEXT:    [[Q2:%.*]] = extractelement <4 x float> [[RD]], i32 2
+; ANY-NEXT:    [[Q3:%.*]] = extractelement <4 x float> [[RD]], i32 3
+; ANY-NEXT:    [[Q4:%.*]] = fadd float [[Q0]], [[Q1]]
+; ANY-NEXT:    [[Q5:%.*]] = fadd float [[Q2]], [[Q3]]
+; ANY-NEXT:    [[Q6:%.*]] = fadd float [[Q4]], [[Q5]]
+; ANY-NEXT:    [[QI:%.*]] = fcmp olt float [[Q6]], [[Q5]]
+; ANY-NEXT:    call void @llvm.assume(i1 [[QI]])
+; ANY-NEXT:    ret <4 x float> undef
 ;
   %c0 = extractelement <4 x i32> %c, i32 0
   %c1 = extractelement <4 x i32> %c, i32 1
@@ -174,31 +125,18 @@ define <4 x float> @simple_select_eph(<4 x float> %a, <4 x float> %b, <4 x i32>
 ; Insert in an order different from the vector indices to make sure it
 ; doesn't matter
 define <4 x float> @simple_select_insert_out_of_order(<4 x float> %a, <4 x float> %b, <4 x i32> %c) #0 {
-; CHECK-LABEL: @simple_select_insert_out_of_order(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne <4 x i32> [[C:%.*]], zeroinitializer
-; CHECK-NEXT:    [[TMP2:%.*]] = select <4 x i1> [[TMP1]], <4 x float> [[A:%.*]], <4 x float> [[B:%.*]]
-; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[TMP2]], i32 0
-; CHECK-NEXT:    [[RA:%.*]] = insertelement <4 x float> undef, float [[TMP3]], i32 2
-; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[TMP2]], i32 1
-; CHECK-NEXT:    [[RB:%.*]] = insertelement <4 x float> [[RA]], float [[TMP4]], i32 1
-; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x float> [[TMP2]], i32 2
-; CHECK-NEXT:    [[RC:%.*]] = insertelement <4 x float> [[RB]], float [[TMP5]], i32 0
-; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[TMP2]], i32 3
-; CHECK-NEXT:    [[RD:%.*]] = insertelement <4 x float> [[RC]], float [[TMP6]], i32 3
-; CHECK-NEXT:    ret <4 x float> [[RD]]
-;
-; ZEROTHRESH-LABEL: @simple_select_insert_out_of_order(
-; ZEROTHRESH-NEXT:    [[TMP1:%.*]] = icmp ne <4 x i32> [[C:%.*]], zeroinitializer
-; ZEROTHRESH-NEXT:    [[TMP2:%.*]] = select <4 x i1> [[TMP1]], <4 x float> [[A:%.*]], <4 x float> [[B:%.*]]
-; ZEROTHRESH-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[TMP2]], i32 0
-; ZEROTHRESH-NEXT:    [[RA:%.*]] = insertelement <4 x float> undef, float [[TMP3]], i32 2
-; ZEROTHRESH-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[TMP2]], i32 1
-; ZEROTHRESH-NEXT:    [[RB:%.*]] = insertelement <4 x float> [[RA]], float [[TMP4]], i32 1
-; ZEROTHRESH-NEXT:    [[TMP5:%.*]] = extractelement <4 x float> [[TMP2]], i32 2
-; ZEROTHRESH-NEXT:    [[RC:%.*]] = insertelement <4 x float> [[RB]], float [[TMP5]], i32 0
-; ZEROTHRESH-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[TMP2]], i32 3
-; ZEROTHRESH-NEXT:    [[RD:%.*]] = insertelement <4 x float> [[RC]], float [[TMP6]], i32 3
-; ZEROTHRESH-NEXT:    ret <4 x float> [[RD]]
+; ANY-LABEL: @simple_select_insert_out_of_order(
+; ANY-NEXT:    [[TMP1:%.*]] = icmp ne <4 x i32> [[C:%.*]], zeroinitializer
+; ANY-NEXT:    [[TMP2:%.*]] = select <4 x i1> [[TMP1]], <4 x float> [[A:%.*]], <4 x float> [[B:%.*]]
+; ANY-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[TMP2]], i32 0
+; ANY-NEXT:    [[RA:%.*]] = insertelement <4 x float> undef, float [[TMP3]], i32 2
+; ANY-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[TMP2]], i32 1
+; ANY-NEXT:    [[RB:%.*]] = insertelement <4 x float> [[RA]], float [[TMP4]], i32 1
+; ANY-NEXT:    [[TMP5:%.*]] = extractelement <4 x float> [[TMP2]], i32 2
+; ANY-NEXT:    [[RC:%.*]] = insertelement <4 x float> [[RB]], float [[TMP5]], i32 0
+; ANY-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[TMP2]], i32 3
+; ANY-NEXT:    [[RD:%.*]] = insertelement <4 x float> [[RC]], float [[TMP6]], i32 3
+; ANY-NEXT:    ret <4 x float> [[RD]]
 ;
   %c0 = extractelement <4 x i32> %c, i32 0
   %c1 = extractelement <4 x i32> %c, i32 1
@@ -232,33 +170,19 @@ declare void @f32_user(float) #0
 
 ; Multiple users of the final constructed vector
 define <4 x float> @simple_select_users(<4 x float> %a, <4 x float> %b, <4 x i32> %c) #0 {
-; CHECK-LABEL: @simple_select_users(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne <4 x i32> [[C:%.*]], zeroinitializer
-; CHECK-NEXT:    [[TMP2:%.*]] = select <4 x i1> [[TMP1]], <4 x float> [[A:%.*]], <4 x float> [[B:%.*]]
-; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[TMP2]], i32 0
-; CHECK-NEXT:    [[RA:%.*]] = insertelement <4 x float> undef, float [[TMP3]], i32 0
-; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[TMP2]], i32 1
-; CHECK-NEXT:    [[RB:%.*]] = insertelement <4 x float> [[RA]], float [[TMP4]], i32 1
-; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x float> [[TMP2]], i32 2
-; CHECK-NEXT:    [[RC:%.*]] = insertelement <4 x float> [[RB]], float [[TMP5]], i32 2
-; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[TMP2]], i32 3
-; CHECK-NEXT:    [[RD:%.*]] = insertelement <4 x float> [[RC]], float [[TMP6]], i32 3
-; CHECK-NEXT:    call void @v4f32_user(<4 x float> [[RD]]) #0
-; CHECK-NEXT:    ret <4 x float> [[RD]]
-;
-; ZEROTHRESH-LABEL: @simple_select_users(
-; ZEROTHRESH-NEXT:    [[TMP1:%.*]] = icmp ne <4 x i32> [[C:%.*]], zeroinitializer
-; ZEROTHRESH-NEXT:    [[TMP2:%.*]] = select <4 x i1> [[TMP1]], <4 x float> [[A:%.*]], <4 x float> [[B:%.*]]
-; ZEROTHRESH-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[TMP2]], i32 0
-; ZEROTHRESH-NEXT:    [[RA:%.*]] = insertelement <4 x float> undef, float [[TMP3]], i32 0
-; ZEROTHRESH-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[TMP2]], i32 1
-; ZEROTHRESH-NEXT:    [[RB:%.*]] = insertelement <4 x float> [[RA]], float [[TMP4]], i32 1
-; ZEROTHRESH-NEXT:    [[TMP5:%.*]] = extractelement <4 x float> [[TMP2]], i32 2
-; ZEROTHRESH-NEXT:    [[RC:%.*]] = insertelement <4 x float> [[RB]], float [[TMP5]], i32 2
-; ZEROTHRESH-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[TMP2]], i32 3
-; ZEROTHRESH-NEXT:    [[RD:%.*]] = insertelement <4 x float> [[RC]], float [[TMP6]], i32 3
-; ZEROTHRESH-NEXT:    call void @v4f32_user(<4 x float> [[RD]]) #0
-; ZEROTHRESH-NEXT:    ret <4 x float> [[RD]]
+; ANY-LABEL: @simple_select_users(
+; ANY-NEXT:    [[TMP1:%.*]] = icmp ne <4 x i32> [[C:%.*]], zeroinitializer
+; ANY-NEXT:    [[TMP2:%.*]] = select <4 x i1> [[TMP1]], <4 x float> [[A:%.*]], <4 x float> [[B:%.*]]
+; ANY-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[TMP2]], i32 0
+; ANY-NEXT:    [[RA:%.*]] = insertelement <4 x float> undef, float [[TMP3]], i32 0
+; ANY-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[TMP2]], i32 1
+; ANY-NEXT:    [[RB:%.*]] = insertelement <4 x float> [[RA]], float [[TMP4]], i32 1
+; ANY-NEXT:    [[TMP5:%.*]] = extractelement <4 x float> [[TMP2]], i32 2
+; ANY-NEXT:    [[RC:%.*]] = insertelement <4 x float> [[RB]], float [[TMP5]], i32 2
+; ANY-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[TMP2]], i32 3
+; ANY-NEXT:    [[RD:%.*]] = insertelement <4 x float> [[RC]], float [[TMP6]], i32 3
+; ANY-NEXT:    call void @v4f32_user(<4 x float> [[RD]]) #0
+; ANY-NEXT:    ret <4 x float> [[RD]]
 ;
   %c0 = extractelement <4 x i32> %c, i32 0
   %c1 = extractelement <4 x i32> %c, i32 1
@@ -290,44 +214,44 @@ define <4 x float> @simple_select_users(<4 x float> %a, <4 x float> %b, <4 x i32
 
 ; Unused insertelement
 define <4 x float> @simple_select_no_users(<4 x float> %a, <4 x float> %b, <4 x i32> %c) #0 {
-; CHECK-LABEL: @simple_select_no_users(
-; CHECK-NEXT:    [[C0:%.*]] = extractelement <4 x i32> [[C:%.*]], i32 0
-; CHECK-NEXT:    [[C1:%.*]] = extractelement <4 x i32> [[C]], i32 1
-; CHECK-NEXT:    [[C2:%.*]] = extractelement <4 x i32> [[C]], i32 2
-; CHECK-NEXT:    [[C3:%.*]] = extractelement <4 x i32> [[C]], i32 3
-; CHECK-NEXT:    [[A0:%.*]] = extractelement <4 x float> [[A:%.*]], i32 0
-; CHECK-NEXT:    [[A1:%.*]] = extractelement <4 x float> [[A]], i32 1
-; CHECK-NEXT:    [[A2:%.*]] = extractelement <4 x float> [[A]], i32 2
-; CHECK-NEXT:    [[A3:%.*]] = extractelement <4 x float> [[A]], i32 3
-; CHECK-NEXT:    [[B0:%.*]] = extractelement <4 x float> [[B:%.*]], i32 0
-; CHECK-NEXT:    [[B1:%.*]] = extractelement <4 x float> [[B]], i32 1
-; CHECK-NEXT:    [[B2:%.*]] = extractelement <4 x float> [[B]], i32 2
-; CHECK-NEXT:    [[B3:%.*]] = extractelement <4 x float> [[B]], i32 3
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i32> undef, i32 [[C0]], i32 0
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x i32> [[TMP1]], i32 [[C1]], i32 1
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <2 x i32> [[TMP2]], zeroinitializer
-; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <2 x i32> undef, i32 [[C2]], i32 0
-; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <2 x i32> [[TMP4]], i32 [[C3]], i32 1
-; CHECK-NEXT:    [[TMP6:%.*]] = icmp ne <2 x i32> [[TMP5]], zeroinitializer
-; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <2 x float> undef, float [[A0]], i32 0
-; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <2 x float> [[TMP7]], float [[A1]], i32 1
-; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <2 x float> undef, float [[B0]], i32 0
-; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <2 x float> [[TMP9]], float [[B1]], i32 1
-; CHECK-NEXT:    [[TMP11:%.*]] = select <2 x i1> [[TMP3]], <2 x float> [[TMP8]], <2 x float> [[TMP10]]
-; CHECK-NEXT:    [[TMP12:%.*]] = insertelement <2 x float> undef, float [[A2]], i32 0
-; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <2 x float> [[TMP12]], float [[A3]], i32 1
-; CHECK-NEXT:    [[TMP14:%.*]] = insertelement <2 x float> undef, float [[B2]], i32 0
-; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <2 x float> [[TMP14]], float [[B3]], i32 1
-; CHECK-NEXT:    [[TMP16:%.*]] = select <2 x i1> [[TMP6]], <2 x float> [[TMP13]], <2 x float> [[TMP15]]
-; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <2 x float> [[TMP11]], i32 0
-; CHECK-NEXT:    [[RA:%.*]] = insertelement <4 x float> undef, float [[TMP17]], i32 0
-; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <2 x float> [[TMP11]], i32 1
-; CHECK-NEXT:    [[RB:%.*]] = insertelement <4 x float> [[RA]], float [[TMP18]], i32 1
-; CHECK-NEXT:    [[TMP19:%.*]] = extractelement <2 x float> [[TMP16]], i32 0
-; CHECK-NEXT:    [[RC:%.*]] = insertelement <4 x float> undef, float [[TMP19]], i32 2
-; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <2 x float> [[TMP16]], i32 1
-; CHECK-NEXT:    [[RD:%.*]] = insertelement <4 x float> [[RC]], float [[TMP20]], i32 3
-; CHECK-NEXT:    ret <4 x float> [[RD]]
+; BIGTHRESH-LABEL: @simple_select_no_users(
+; BIGTHRESH-NEXT:    [[C0:%.*]] = extractelement <4 x i32> [[C:%.*]], i32 0
+; BIGTHRESH-NEXT:    [[C1:%.*]] = extractelement <4 x i32> [[C]], i32 1
+; BIGTHRESH-NEXT:    [[C2:%.*]] = extractelement <4 x i32> [[C]], i32 2
+; BIGTHRESH-NEXT:    [[C3:%.*]] = extractelement <4 x i32> [[C]], i32 3
+; BIGTHRESH-NEXT:    [[A0:%.*]] = extractelement <4 x float> [[A:%.*]], i32 0
+; BIGTHRESH-NEXT:    [[A1:%.*]] = extractelement <4 x float> [[A]], i32 1
+; BIGTHRESH-NEXT:    [[A2:%.*]] = extractelement <4 x float> [[A]], i32 2
+; BIGTHRESH-NEXT:    [[A3:%.*]] = extractelement <4 x float> [[A]], i32 3
+; BIGTHRESH-NEXT:    [[B0:%.*]] = extractelement <4 x float> [[B:%.*]], i32 0
+; BIGTHRESH-NEXT:    [[B1:%.*]] = extractelement <4 x float> [[B]], i32 1
+; BIGTHRESH-NEXT:    [[B2:%.*]] = extractelement <4 x float> [[B]], i32 2
+; BIGTHRESH-NEXT:    [[B3:%.*]] = extractelement <4 x float> [[B]], i32 3
+; BIGTHRESH-NEXT:    [[TMP1:%.*]] = insertelement <2 x i32> undef, i32 [[C0]], i32 0
+; BIGTHRESH-NEXT:    [[TMP2:%.*]] = insertelement <2 x i32> [[TMP1]], i32 [[C1]], i32 1
+; BIGTHRESH-NEXT:    [[TMP3:%.*]] = icmp ne <2 x i32> [[TMP2]], zeroinitializer
+; BIGTHRESH-NEXT:    [[TMP4:%.*]] = insertelement <2 x i32> undef, i32 [[C2]], i32 0
+; BIGTHRESH-NEXT:    [[TMP5:%.*]] = insertelement <2 x i32> [[TMP4]], i32 [[C3]], i32 1
+; BIGTHRESH-NEXT:    [[TMP6:%.*]] = icmp ne <2 x i32> [[TMP5]], zeroinitializer
+; BIGTHRESH-NEXT:    [[TMP7:%.*]] = insertelement <2 x float> undef, float [[A0]], i32 0
+; BIGTHRESH-NEXT:    [[TMP8:%.*]] = insertelement <2 x float> [[TMP7]], float [[A1]], i32 1
+; BIGTHRESH-NEXT:    [[TMP9:%.*]] = insertelement <2 x float> undef, float [[B0]], i32 0
+; BIGTHRESH-NEXT:    [[TMP10:%.*]] = insertelement <2 x float> [[TMP9]], float [[B1]], i32 1
+; BIGTHRESH-NEXT:    [[TMP11:%.*]] = select <2 x i1> [[TMP3]], <2 x float> [[TMP8]], <2 x float> [[TMP10]]
+; BIGTHRESH-NEXT:    [[TMP12:%.*]] = insertelement <2 x float> undef, float [[A2]], i32 0
+; BIGTHRESH-NEXT:    [[TMP13:%.*]] = insertelement <2 x float> [[TMP12]], float [[A3]], i32 1
+; BIGTHRESH-NEXT:    [[TMP14:%.*]] = insertelement <2 x float> undef, float [[B2]], i32 0
+; BIGTHRESH-NEXT:    [[TMP15:%.*]] = insertelement <2 x float> [[TMP14]], float [[B3]], i32 1
+; BIGTHRESH-NEXT:    [[TMP16:%.*]] = select <2 x i1> [[TMP6]], <2 x float> [[TMP13]], <2 x float> [[TMP15]]
+; BIGTHRESH-NEXT:    [[TMP17:%.*]] = extractelement <2 x float> [[TMP11]], i32 0
+; BIGTHRESH-NEXT:    [[RA:%.*]] = insertelement <4 x float> undef, float [[TMP17]], i32 0
+; BIGTHRESH-NEXT:    [[TMP18:%.*]] = extractelement <2 x float> [[TMP11]], i32 1
+; BIGTHRESH-NEXT:    [[RB:%.*]] = insertelement <4 x float> [[RA]], float [[TMP18]], i32 1
+; BIGTHRESH-NEXT:    [[TMP19:%.*]] = extractelement <2 x float> [[TMP16]], i32 0
+; BIGTHRESH-NEXT:    [[RC:%.*]] = insertelement <4 x float> undef, float [[TMP19]], i32 2
+; BIGTHRESH-NEXT:    [[TMP20:%.*]] = extractelement <2 x float> [[TMP16]], i32 1
+; BIGTHRESH-NEXT:    [[RD:%.*]] = insertelement <4 x float> [[RC]], float [[TMP20]], i32 3
+; BIGTHRESH-NEXT:    ret <4 x float> [[RD]]
 ;
 ; ZEROTHRESH-LABEL: @simple_select_no_users(
 ; ZEROTHRESH-NEXT:    [[C0:%.*]] = extractelement <4 x i32> [[C:%.*]], i32 0
@@ -344,16 +268,22 @@ define <4 x float> @simple_select_no_users(<4 x float> %a, <4 x float> %b, <4 x
 ; ZEROTHRESH-NEXT:    [[B3:%.*]] = extractelement <4 x float> [[B]], i32 3
 ; ZEROTHRESH-NEXT:    [[CMP0:%.*]] = icmp ne i32 [[C0]], 0
 ; ZEROTHRESH-NEXT:    [[CMP1:%.*]] = icmp ne i32 [[C1]], 0
-; ZEROTHRESH-NEXT:    [[CMP2:%.*]] = icmp ne i32 [[C2]], 0
-; ZEROTHRESH-NEXT:    [[CMP3:%.*]] = icmp ne i32 [[C3]], 0
+; ZEROTHRESH-NEXT:    [[TMP1:%.*]] = insertelement <2 x i32> undef, i32 [[C2]], i32 0
+; ZEROTHRESH-NEXT:    [[TMP2:%.*]] = insertelement <2 x i32> [[TMP1]], i32 [[C3]], i32 1
+; ZEROTHRESH-NEXT:    [[TMP3:%.*]] = icmp ne <2 x i32> [[TMP2]], zeroinitializer
 ; ZEROTHRESH-NEXT:    [[S0:%.*]] = select i1 [[CMP0]], float [[A0]], float [[B0]]
 ; ZEROTHRESH-NEXT:    [[S1:%.*]] = select i1 [[CMP1]], float [[A1]], float [[B1]]
-; ZEROTHRESH-NEXT:    [[S2:%.*]] = select i1 [[CMP2]], float [[A2]], float [[B2]]
-; ZEROTHRESH-NEXT:    [[S3:%.*]] = select i1 [[CMP3]], float [[A3]], float [[B3]]
+; ZEROTHRESH-NEXT:    [[TMP4:%.*]] = insertelement <2 x float> undef, float [[A2]], i32 0
+; ZEROTHRESH-NEXT:    [[TMP5:%.*]] = insertelement <2 x float> [[TMP4]], float [[A3]], i32 1
+; ZEROTHRESH-NEXT:    [[TMP6:%.*]] = insertelement <2 x float> undef, float [[B2]], i32 0
+; ZEROTHRESH-NEXT:    [[TMP7:%.*]] = insertelement <2 x float> [[TMP6]], float [[B3]], i32 1
+; ZEROTHRESH-NEXT:    [[TMP8:%.*]] = select <2 x i1> [[TMP3]], <2 x float> [[TMP5]], <2 x float> [[TMP7]]
 ; ZEROTHRESH-NEXT:    [[RA:%.*]] = insertelement <4 x float> undef, float [[S0]], i32 0
 ; ZEROTHRESH-NEXT:    [[RB:%.*]] = insertelement <4 x float> [[RA]], float [[S1]], i32 1
-; ZEROTHRESH-NEXT:    [[RC:%.*]] = insertelement <4 x float> undef, float [[S2]], i32 2
-; ZEROTHRESH-NEXT:    [[RD:%.*]] = insertelement <4 x float> [[RC]], float [[S3]], i32 3
+; ZEROTHRESH-NEXT:    [[TMP9:%.*]] = extractelement <2 x float> [[TMP8]], i32 0
+; ZEROTHRESH-NEXT:    [[RC:%.*]] = insertelement <4 x float> undef, float [[TMP9]], i32 2
+; ZEROTHRESH-NEXT:    [[TMP10:%.*]] = extractelement <2 x float> [[TMP8]], i32 1
+; ZEROTHRESH-NEXT:    [[RD:%.*]] = insertelement <4 x float> [[RC]], float [[TMP10]], i32 3
 ; ZEROTHRESH-NEXT:    ret <4 x float> [[RD]]
 ;
   %c0 = extractelement <4 x i32> %c, i32 0
@@ -386,27 +316,16 @@ define <4 x float> @simple_select_no_users(<4 x float> %a, <4 x float> %b, <4 x
 ; Make sure infinite loop doesn't happen which I ran into when trying
 ; to do this backwards this backwards
 define <4 x i32> @reconstruct(<4 x i32> %c) #0 {
-; CHECK-LABEL: @reconstruct(
-; CHECK-NEXT:    [[C0:%.*]] = extractelement <4 x i32> [[C:%.*]], i32 0
-; CHECK-NEXT:    [[C1:%.*]] = extractelement <4 x i32> [[C]], i32 1
-; CHECK-NEXT:    [[C2:%.*]] = extractelement <4 x i32> [[C]], i32 2
-; CHECK-NEXT:    [[C3:%.*]] = extractelement <4 x i32> [[C]], i32 3
-; CHECK-NEXT:    [[RA:%.*]] = insertelement <4 x i32> undef, i32 [[C0]], i32 0
-; CHECK-NEXT:    [[RB:%.*]] = insertelement <4 x i32> [[RA]], i32 [[C1]], i32 1
-; CHECK-NEXT:    [[RC:%.*]] = insertelement <4 x i32> [[RB]], i32 [[C2]], i32 2
-; CHECK-NEXT:    [[RD:%.*]] = insertelement <4 x i32> [[RC]], i32 [[C3]], i32 3
-; CHECK-NEXT:    ret <4 x i32> [[RD]]
-;
-; ZEROTHRESH-LABEL: @reconstruct(
-; ZEROTHRESH-NEXT:    [[C0:%.*]] = extractelement <4 x i32> [[C:%.*]], i32 0
-; ZEROTHRESH-NEXT:    [[C1:%.*]] = extractelement <4 x i32> [[C]], i32 1
-; ZEROTHRESH-NEXT:    [[C2:%.*]] = extractelement <4 x i32> [[C]], i32 2
-; ZEROTHRESH-NEXT:    [[C3:%.*]] = extractelement <4 x i32> [[C]], i32 3
-; ZEROTHRESH-NEXT:    [[RA:%.*]] = insertelement <4 x i32> undef, i32 [[C0]], i32 0
-; ZEROTHRESH-NEXT:    [[RB:%.*]] = insertelement <4 x i32> [[RA]], i32 [[C1]], i32 1
-; ZEROTHRESH-NEXT:    [[RC:%.*]] = insertelement <4 x i32> [[RB]], i32 [[C2]], i32 2
-; ZEROTHRESH-NEXT:    [[RD:%.*]] = insertelement <4 x i32> [[RC]], i32 [[C3]], i32 3
-; ZEROTHRESH-NEXT:    ret <4 x i32> [[RD]]
+; ANY-LABEL: @reconstruct(
+; ANY-NEXT:    [[C0:%.*]] = extractelement <4 x i32> [[C:%.*]], i32 0
+; ANY-NEXT:    [[C1:%.*]] = extractelement <4 x i32> [[C]], i32 1
+; ANY-NEXT:    [[C2:%.*]] = extractelement <4 x i32> [[C]], i32 2
+; ANY-NEXT:    [[C3:%.*]] = extractelement <4 x i32> [[C]], i32 3
+; ANY-NEXT:    [[RA:%.*]] = insertelement <4 x i32> undef, i32 [[C0]], i32 0
+; ANY-NEXT:    [[RB:%.*]] = insertelement <4 x i32> [[RA]], i32 [[C1]], i32 1
+; ANY-NEXT:    [[RC:%.*]] = insertelement <4 x i32> [[RB]], i32 [[C2]], i32 2
+; ANY-NEXT:    [[RD:%.*]] = insertelement <4 x i32> [[RC]], i32 [[C3]], i32 3
+; ANY-NEXT:    ret <4 x i32> [[RD]]
 ;
   %c0 = extractelement <4 x i32> %c, i32 0
   %c1 = extractelement <4 x i32> %c, i32 1
@@ -420,29 +339,14 @@ define <4 x i32> @reconstruct(<4 x i32> %c) #0 {
 }
 
 define <2 x float> @simple_select_v2(<2 x float> %a, <2 x float> %b, <2 x i32> %c) #0 {
-; CHECK-LABEL: @simple_select_v2(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne <2 x i32> [[C:%.*]], zeroinitializer
-; CHECK-NEXT:    [[TMP2:%.*]] = select <2 x i1> [[TMP1]], <2 x float> [[A:%.*]], <2 x float> [[B:%.*]]
-; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x float> [[TMP2]], i32 0
-; CHECK-NEXT:    [[RA:%.*]] = insertelement <2 x float> undef, float [[TMP3]], i32 0
-; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x float> [[TMP2]], i32 1
-; CHECK-NEXT:    [[RB:%.*]] = insertelement <2 x float> [[RA]], float [[TMP4]], i32 1
-; CHECK-NEXT:    ret <2 x float> [[RB]]
-;
-; ZEROTHRESH-LABEL: @simple_select_v2(
-; ZEROTHRESH-NEXT:    [[C0:%.*]] = extractelement <2 x i32> [[C:%.*]], i32 0
-; ZEROTHRESH-NEXT:    [[C1:%.*]] = extractelement <2 x i32> [[C]], i32 1
-; ZEROTHRESH-NEXT:    [[A0:%.*]] = extractelement <2 x float> [[A:%.*]], i32 0
-; ZEROTHRESH-NEXT:    [[A1:%.*]] = extractelement <2 x float> [[A]], i32 1
-; ZEROTHRESH-NEXT:    [[B0:%.*]] = extractelement <2 x float> [[B:%.*]], i32 0
-; ZEROTHRESH-NEXT:    [[B1:%.*]] = extractelement <2 x float> [[B]], i32 1
-; ZEROTHRESH-NEXT:    [[CMP0:%.*]] = icmp ne i32 [[C0]], 0
-; ZEROTHRESH-NEXT:    [[CMP1:%.*]] = icmp ne i32 [[C1]], 0
-; ZEROTHRESH-NEXT:    [[S0:%.*]] = select i1 [[CMP0]], float [[A0]], float [[B0]]
-; ZEROTHRESH-NEXT:    [[S1:%.*]] = select i1 [[CMP1]], float [[A1]], float [[B1]]
-; ZEROTHRESH-NEXT:    [[RA:%.*]] = insertelement <2 x float> undef, float [[S0]], i32 0
-; ZEROTHRESH-NEXT:    [[RB:%.*]] = insertelement <2 x float> [[RA]], float [[S1]], i32 1
-; ZEROTHRESH-NEXT:    ret <2 x float> [[RB]]
+; ANY-LABEL: @simple_select_v2(
+; ANY-NEXT:    [[TMP1:%.*]] = icmp ne <2 x i32> [[C:%.*]], zeroinitializer
+; ANY-NEXT:    [[TMP2:%.*]] = select <2 x i1> [[TMP1]], <2 x float> [[A:%.*]], <2 x float> [[B:%.*]]
+; ANY-NEXT:    [[TMP3:%.*]] = extractelement <2 x float> [[TMP2]], i32 0
+; ANY-NEXT:    [[RA:%.*]] = insertelement <2 x float> undef, float [[TMP3]], i32 0
+; ANY-NEXT:    [[TMP4:%.*]] = extractelement <2 x float> [[TMP2]], i32 1
+; ANY-NEXT:    [[RB:%.*]] = insertelement <2 x float> [[RA]], float [[TMP4]], i32 1
+; ANY-NEXT:    ret <2 x float> [[RB]]
 ;
   %c0 = extractelement <2 x i32> %c, i32 0
   %c1 = extractelement <2 x i32> %c, i32 1
@@ -463,47 +367,26 @@ define <2 x float> @simple_select_v2(<2 x float> %a, <2 x float> %b, <2 x i32> %
 ; re-visiting the insertelement chains starting with undef
 ; (low cost threshold needed to force this to happen)
 define <4 x float> @simple_select_partial_vector(<4 x float> %a, <4 x float> %b, <4 x i32> %c) #0 {
-; CHECK-LABEL: @simple_select_partial_vector(
-; CHECK-NEXT:    [[C0:%.*]] = extractelement <4 x i32> [[C:%.*]], i32 0
-; CHECK-NEXT:    [[C1:%.*]] = extractelement <4 x i32> [[C]], i32 1
-; CHECK-NEXT:    [[A0:%.*]] = extractelement <4 x float> [[A:%.*]], i32 0
-; CHECK-NEXT:    [[A1:%.*]] = extractelement <4 x float> [[A]], i32 1
-; CHECK-NEXT:    [[B0:%.*]] = extractelement <4 x float> [[B:%.*]], i32 0
-; CHECK-NEXT:    [[B1:%.*]] = extractelement <4 x float> [[B]], i32 1
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i32> undef, i32 [[C0]], i32 0
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x i32> [[TMP1]], i32 [[C1]], i32 1
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <2 x i32> [[TMP2]], zeroinitializer
-; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <2 x float> undef, float [[A0]], i32 0
-; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <2 x float> [[TMP4]], float [[A1]], i32 1
-; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x float> undef, float [[B0]], i32 0
-; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <2 x float> [[TMP6]], float [[B1]], i32 1
-; CHECK-NEXT:    [[TMP8:%.*]] = select <2 x i1> [[TMP3]], <2 x float> [[TMP5]], <2 x float> [[TMP7]]
-; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <2 x float> [[TMP8]], i32 0
-; CHECK-NEXT:    [[RA:%.*]] = insertelement <4 x float> undef, float [[TMP9]], i32 0
-; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <2 x float> [[TMP8]], i32 1
-; CHECK-NEXT:    [[RB:%.*]] = insertelement <4 x float> [[RA]], float [[TMP10]], i32 1
-; CHECK-NEXT:    ret <4 x float> [[RB]]
-;
-; ZEROTHRESH-LABEL: @simple_select_partial_vector(
-; ZEROTHRESH-NEXT:    [[C0:%.*]] = extractelement <4 x i32> [[C:%.*]], i32 0
-; ZEROTHRESH-NEXT:    [[C1:%.*]] = extractelement <4 x i32> [[C]], i32 1
-; ZEROTHRESH-NEXT:    [[A0:%.*]] = extractelement <4 x float> [[A:%.*]], i32 0
-; ZEROTHRESH-NEXT:    [[A1:%.*]] = extractelement <4 x float> [[A]], i32 1
-; ZEROTHRESH-NEXT:    [[B0:%.*]] = extractelement <4 x float> [[B:%.*]], i32 0
-; ZEROTHRESH-NEXT:    [[B1:%.*]] = extractelement <4 x float> [[B]], i32 1
-; ZEROTHRESH-NEXT:    [[TMP1:%.*]] = insertelement <2 x i32> undef, i32 [[C0]], i32 0
-; ZEROTHRESH-NEXT:    [[TMP2:%.*]] = insertelement <2 x i32> [[TMP1]], i32 [[C1]], i32 1
-; ZEROTHRESH-NEXT:    [[TMP3:%.*]] = icmp ne <2 x i32> [[TMP2]], zeroinitializer
-; ZEROTHRESH-NEXT:    [[TMP4:%.*]] = insertelement <2 x float> undef, float [[A0]], i32 0
-; ZEROTHRESH-NEXT:    [[TMP5:%.*]] = insertelement <2 x float> [[TMP4]], float [[A1]], i32 1
-; ZEROTHRESH-NEXT:    [[TMP6:%.*]] = insertelement <2 x float> undef, float [[B0]], i32 0
-; ZEROTHRESH-NEXT:    [[TMP7:%.*]] = insertelement <2 x float> [[TMP6]], float [[B1]], i32 1
-; ZEROTHRESH-NEXT:    [[TMP8:%.*]] = select <2 x i1> [[TMP3]], <2 x float> [[TMP5]], <2 x float> [[TMP7]]
-; ZEROTHRESH-NEXT:    [[TMP9:%.*]] = extractelement <2 x float> [[TMP8]], i32 0
-; ZEROTHRESH-NEXT:    [[RA:%.*]] = insertelement <4 x float> undef, float [[TMP9]], i32 0
-; ZEROTHRESH-NEXT:    [[TMP10:%.*]] = extractelement <2 x float> [[TMP8]], i32 1
-; ZEROTHRESH-NEXT:    [[RB:%.*]] = insertelement <4 x float> [[RA]], float [[TMP10]], i32 1
-; ZEROTHRESH-NEXT:    ret <4 x float> [[RB]]
+; ANY-LABEL: @simple_select_partial_vector(
+; ANY-NEXT:    [[C0:%.*]] = extractelement <4 x i32> [[C:%.*]], i32 0
+; ANY-NEXT:    [[C1:%.*]] = extractelement <4 x i32> [[C]], i32 1
+; ANY-NEXT:    [[A0:%.*]] = extractelement <4 x float> [[A:%.*]], i32 0
+; ANY-NEXT:    [[A1:%.*]] = extractelement <4 x float> [[A]], i32 1
+; ANY-NEXT:    [[B0:%.*]] = extractelement <4 x float> [[B:%.*]], i32 0
+; ANY-NEXT:    [[B1:%.*]] = extractelement <4 x float> [[B]], i32 1
+; ANY-NEXT:    [[TMP1:%.*]] = insertelement <2 x i32> undef, i32 [[C0]], i32 0
+; ANY-NEXT:    [[TMP2:%.*]] = insertelement <2 x i32> [[TMP1]], i32 [[C1]], i32 1
+; ANY-NEXT:    [[TMP3:%.*]] = icmp ne <2 x i32> [[TMP2]], zeroinitializer
+; ANY-NEXT:    [[TMP4:%.*]] = insertelement <2 x float> undef, float [[A0]], i32 0
+; ANY-NEXT:    [[TMP5:%.*]] = insertelement <2 x float> [[TMP4]], float [[A1]], i32 1
+; ANY-NEXT:    [[TMP6:%.*]] = insertelement <2 x float> undef, float [[B0]], i32 0
+; ANY-NEXT:    [[TMP7:%.*]] = insertelement <2 x float> [[TMP6]], float [[B1]], i32 1
+; ANY-NEXT:    [[TMP8:%.*]] = select <2 x i1> [[TMP3]], <2 x float> [[TMP5]], <2 x float> [[TMP7]]
+; ANY-NEXT:    [[TMP9:%.*]] = extractelement <2 x float> [[TMP8]], i32 0
+; ANY-NEXT:    [[RA:%.*]] = insertelement <4 x float> undef, float [[TMP9]], i32 0
+; ANY-NEXT:    [[TMP10:%.*]] = extractelement <2 x float> [[TMP8]], i32 1
+; ANY-NEXT:    [[RB:%.*]] = insertelement <4 x float> [[RA]], float [[TMP10]], i32 1
+; ANY-NEXT:    ret <4 x float> [[RB]]
 ;
   %c0 = extractelement <4 x i32> %c, i32 0
   %c1 = extractelement <4 x i32> %c, i32 1
@@ -529,29 +412,17 @@ define <4 x float> @simple_select_partial_vector(<4 x float> %a, <4 x float> %b,
 ; Make sure that vectorization happens even if insertelements operations
 ; must be rescheduled. The case here is from compiling Julia.
 define <4 x float> @reschedule_extract(<4 x float> %a, <4 x float> %b) {
-; CHECK-LABEL: @reschedule_extract(
-; CHECK-NEXT:    [[TMP1:%.*]] = fadd <4 x float> [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
-; CHECK-NEXT:    [[V0:%.*]] = insertelement <4 x float> undef, float [[TMP2]], i32 0
-; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[TMP1]], i32 1
-; CHECK-NEXT:    [[V1:%.*]] = insertelement <4 x float> [[V0]], float [[TMP3]], i32 1
-; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i32 2
-; CHECK-NEXT:    [[V2:%.*]] = insertelement <4 x float> [[V1]], float [[TMP4]], i32 2
-; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x float> [[TMP1]], i32 3
-; CHECK-NEXT:    [[V3:%.*]] = insertelement <4 x float> [[V2]], float [[TMP5]], i32 3
-; CHECK-NEXT:    ret <4 x float> [[V3]]
-;
-; ZEROTHRESH-LABEL: @reschedule_extract(
-; ZEROTHRESH-NEXT:    [[TMP1:%.*]] = fadd <4 x float> [[A:%.*]], [[B:%.*]]
-; ZEROTHRESH-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
-; ZEROTHRESH-NEXT:    [[V0:%.*]] = insertelement <4 x float> undef, float [[TMP2]], i32 0
-; ZEROTHRESH-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[TMP1]], i32 1
-; ZEROTHRESH-NEXT:    [[V1:%.*]] = insertelement <4 x float> [[V0]], float [[TMP3]], i32 1
-; ZEROTHRESH-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i32 2
-; ZEROTHRESH-NEXT:    [[V2:%.*]] = insertelement <4 x float> [[V1]], float [[TMP4]], i32 2
-; ZEROTHRESH-NEXT:    [[TMP5:%.*]] = extractelement <4 x float> [[TMP1]], i32 3
-; ZEROTHRESH-NEXT:    [[V3:%.*]] = insertelement <4 x float> [[V2]], float [[TMP5]], i32 3
-; ZEROTHRESH-NEXT:    ret <4 x float> [[V3]]
+; ANY-LABEL: @reschedule_extract(
+; ANY-NEXT:    [[TMP1:%.*]] = fadd <4 x float> [[A:%.*]], [[B:%.*]]
+; ANY-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
+; ANY-NEXT:    [[V0:%.*]] = insertelement <4 x float> undef, float [[TMP2]], i32 0
+; ANY-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[TMP1]], i32 1
+; ANY-NEXT:    [[V1:%.*]] = insertelement <4 x float> [[V0]], float [[TMP3]], i32 1
+; ANY-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i32 2
+; ANY-NEXT:    [[V2:%.*]] = insertelement <4 x float> [[V1]], float [[TMP4]], i32 2
+; ANY-NEXT:    [[TMP5:%.*]] = extractelement <4 x float> [[TMP1]], i32 3
+; ANY-NEXT:    [[V3:%.*]] = insertelement <4 x float> [[V2]], float [[TMP5]], i32 3
+; ANY-NEXT:    ret <4 x float> [[V3]]
 ;
   %a0 = extractelement <4 x float> %a, i32 0
   %b0 = extractelement <4 x float> %b, i32 0
@@ -575,29 +446,17 @@ define <4 x float> @reschedule_extract(<4 x float> %a, <4 x float> %b) {
 ; Check that cost model for vectorization takes credit for
 ; instructions that are erased.
 define <4 x float> @take_credit(<4 x float> %a, <4 x float> %b) {
-; CHECK-LABEL: @take_credit(
-; CHECK-NEXT:    [[TMP1:%.*]] = fadd <4 x float> [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
-; CHECK-NEXT:    [[V0:%.*]] = insertelement <4 x float> undef, float [[TMP2]], i32 0
-; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[TMP1]], i32 1
-; CHECK-NEXT:    [[V1:%.*]] = insertelement <4 x float> [[V0]], float [[TMP3]], i32 1
-; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i32 2
-; CHECK-NEXT:    [[V2:%.*]] = insertelement <4 x float> [[V1]], float [[TMP4]], i32 2
-; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x float> [[TMP1]], i32 3
-; CHECK-NEXT:    [[V3:%.*]] = insertelement <4 x float> [[V2]], float [[TMP5]], i32 3
-; CHECK-NEXT:    ret <4 x float> [[V3]]
-;
-; ZEROTHRESH-LABEL: @take_credit(
-; ZEROTHRESH-NEXT:    [[TMP1:%.*]] = fadd <4 x float> [[A:%.*]], [[B:%.*]]
-; ZEROTHRESH-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
-; ZEROTHRESH-NEXT:    [[V0:%.*]] = insertelement <4 x float> undef, float [[TMP2]], i32 0
-; ZEROTHRESH-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[TMP1]], i32 1
-; ZEROTHRESH-NEXT:    [[V1:%.*]] = insertelement <4 x float> [[V0]], float [[TMP3]], i32 1
-; ZEROTHRESH-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i32 2
-; ZEROTHRESH-NEXT:    [[V2:%.*]] = insertelement <4 x float> [[V1]], float [[TMP4]], i32 2
-; ZEROTHRESH-NEXT:    [[TMP5:%.*]] = extractelement <4 x float> [[TMP1]], i32 3
-; ZEROTHRESH-NEXT:    [[V3:%.*]] = insertelement <4 x float> [[V2]], float [[TMP5]], i32 3
-; ZEROTHRESH-NEXT:    ret <4 x float> [[V3]]
+; ANY-LABEL: @take_credit(
+; ANY-NEXT:    [[TMP1:%.*]] = fadd <4 x float> [[A:%.*]], [[B:%.*]]
+; ANY-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
+; ANY-NEXT:    [[V0:%.*]] = insertelement <4 x float> undef, float [[TMP2]], i32 0
+; ANY-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[TMP1]], i32 1
+; ANY-NEXT:    [[V1:%.*]] = insertelement <4 x float> [[V0]], float [[TMP3]], i32 1
+; ANY-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i32 2
+; ANY-NEXT:    [[V2:%.*]] = insertelement <4 x float> [[V1]], float [[TMP4]], i32 2
+; ANY-NEXT:    [[TMP5:%.*]] = extractelement <4 x float> [[TMP1]], i32 3
+; ANY-NEXT:    [[V3:%.*]] = insertelement <4 x float> [[V2]], float [[TMP5]], i32 3
+; ANY-NEXT:    ret <4 x float> [[V3]]
 ;
   %a0 = extractelement <4 x float> %a, i32 0
   %b0 = extractelement <4 x float> %b, i32 0
@@ -620,43 +479,23 @@ define <4 x float> @take_credit(<4 x float> %a, <4 x float> %b) {
 
 ; Make sure we handle multiple trees that feed one build vector correctly.
 define <4 x double> @multi_tree(double %w, double %x, double %y, double %z) {
-; CHECK-LABEL: @multi_tree(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <4 x double> undef, double [[W:%.*]], i32 0
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x double> [[TMP0]], double [[X:%.*]], i32 1
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x double> [[TMP1]], double [[Y:%.*]], i32 2
-; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x double> [[TMP2]], double [[Z:%.*]], i32 3
-; CHECK-NEXT:    [[TMP4:%.*]] = fadd <4 x double> [[TMP3]], <double 0.000000e+00, double 1.000000e+00, double 2.000000e+00, double 3.000000e+00>
-; CHECK-NEXT:    [[TMP5:%.*]] = fmul <4 x double> [[TMP4]], <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>
-; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x double> [[TMP5]], i32 0
-; CHECK-NEXT:    [[I1:%.*]] = insertelement <4 x double> undef, double [[TMP6]], i32 3
-; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x double> [[TMP5]], i32 1
-; CHECK-NEXT:    [[I2:%.*]] = insertelement <4 x double> [[I1]], double [[TMP7]], i32 2
-; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <4 x double> [[TMP5]], i32 2
-; CHECK-NEXT:    [[I3:%.*]] = insertelement <4 x double> [[I2]], double [[TMP8]], i32 1
-; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <4 x double> [[TMP5]], i32 3
-; CHECK-NEXT:    [[I4:%.*]] = insertelement <4 x double> [[I3]], double [[TMP9]], i32 0
-; CHECK-NEXT:    ret <4 x double> [[I4]]
-;
-; ZEROTHRESH-LABEL: @multi_tree(
-; ZEROTHRESH-NEXT:  entry:
-; ZEROTHRESH-NEXT:    [[TMP0:%.*]] = insertelement <4 x double> undef, double [[W:%.*]], i32 0
-; ZEROTHRESH-NEXT:    [[TMP1:%.*]] = insertelement <4 x double> [[TMP0]], double [[X:%.*]], i32 1
-; ZEROTHRESH-NEXT:    [[TMP2:%.*]] = insertelement <4 x double> [[TMP1]], double [[Y:%.*]], i32 2
-; ZEROTHRESH-NEXT:    [[TMP3:%.*]] = insertelement <4 x double> [[TMP2]], double [[Z:%.*]], i32 3
-; ZEROTHRESH-NEXT:    [[TMP4:%.*]] = fadd <4 x double> [[TMP3]], <double 0.000000e+00, double 1.000000e+00, double 2.000000e+00, double 3.000000e+00>
-; ZEROTHRESH-NEXT:    [[TMP5:%.*]] = fmul <4 x double> [[TMP4]], <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>
-; ZEROTHRESH-NEXT:    [[TMP6:%.*]] = extractelement <4 x double> [[TMP5]], i32 0
-; ZEROTHRESH-NEXT:    [[I1:%.*]] = insertelement <4 x double> undef, double [[TMP6]], i32 3
-; ZEROTHRESH-NEXT:    [[TMP7:%.*]] = extractelement <4 x double> [[TMP5]], i32 1
-; ZEROTHRESH-NEXT:    [[I2:%.*]] = insertelement <4 x double> [[I1]], double [[TMP7]], i32 2
-; ZEROTHRESH-NEXT:    [[TMP8:%.*]] = extractelement <4 x double> [[TMP5]], i32 2
-; ZEROTHRESH-NEXT:    [[I3:%.*]] = insertelement <4 x double> [[I2]], double [[TMP8]], i32 1
-; ZEROTHRESH-NEXT:    [[TMP9:%.*]] = extractelement <4 x double> [[TMP5]], i32 3
-; ZEROTHRESH-NEXT:    [[I4:%.*]] = insertelement <4 x double> [[I3]], double [[TMP9]], i32 0
-; ZEROTHRESH-NEXT:    ret <4 x double> [[I4]]
+; ANY-LABEL: @multi_tree(
+; ANY-NEXT:    [[TMP1:%.*]] = insertelement <4 x double> undef, double [[W:%.*]], i32 0
+; ANY-NEXT:    [[TMP2:%.*]] = insertelement <4 x double> [[TMP1]], double [[X:%.*]], i32 1
+; ANY-NEXT:    [[TMP3:%.*]] = insertelement <4 x double> [[TMP2]], double [[Y:%.*]], i32 2
+; ANY-NEXT:    [[TMP4:%.*]] = insertelement <4 x double> [[TMP3]], double [[Z:%.*]], i32 3
+; ANY-NEXT:    [[TMP5:%.*]] = fadd <4 x double> [[TMP4]], <double 0.000000e+00, double 1.000000e+00, double 2.000000e+00, double 3.000000e+00>
+; ANY-NEXT:    [[TMP6:%.*]] = fmul <4 x double> [[TMP5]], <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>
+; ANY-NEXT:    [[TMP7:%.*]] = extractelement <4 x double> [[TMP6]], i32 0
+; ANY-NEXT:    [[I1:%.*]] = insertelement <4 x double> undef, double [[TMP7]], i32 3
+; ANY-NEXT:    [[TMP8:%.*]] = extractelement <4 x double> [[TMP6]], i32 1
+; ANY-NEXT:    [[I2:%.*]] = insertelement <4 x double> [[I1]], double [[TMP8]], i32 2
+; ANY-NEXT:    [[TMP9:%.*]] = extractelement <4 x double> [[TMP6]], i32 2
+; ANY-NEXT:    [[I3:%.*]] = insertelement <4 x double> [[I2]], double [[TMP9]], i32 1
+; ANY-NEXT:    [[TMP10:%.*]] = extractelement <4 x double> [[TMP6]], i32 3
+; ANY-NEXT:    [[I4:%.*]] = insertelement <4 x double> [[I3]], double [[TMP10]], i32 0
+; ANY-NEXT:    ret <4 x double> [[I4]]
 ;
-entry:
   %t0 = fadd double %w , 0.000000e+00
   %t1 = fadd double %x , 1.000000e+00
   %t2 = fadd double %y , 2.000000e+00
@@ -673,49 +512,26 @@ entry:
 }
 
 define <8 x float> @_vadd256(<8 x float> %a, <8 x float> %b) local_unnamed_addr #0 {
-; CHECK-LABEL: @_vadd256(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = fadd <8 x float> [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <8 x float> [[TMP0]], i32 0
-; CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <8 x float> undef, float [[TMP1]], i32 0
-; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <8 x float> [[TMP0]], i32 1
-; CHECK-NEXT:    [[VECINIT1_I:%.*]] = insertelement <8 x float> [[VECINIT_I]], float [[TMP2]], i32 1
-; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <8 x float> [[TMP0]], i32 2
-; CHECK-NEXT:    [[VECINIT2_I:%.*]] = insertelement <8 x float> [[VECINIT1_I]], float [[TMP3]], i32 2
-; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <8 x float> [[TMP0]], i32 3
-; CHECK-NEXT:    [[VECINIT3_I:%.*]] = insertelement <8 x float> [[VECINIT2_I]], float [[TMP4]], i32 3
-; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <8 x float> [[TMP0]], i32 4
-; CHECK-NEXT:    [[VECINIT4_I:%.*]] = insertelement <8 x float> [[VECINIT3_I]], float [[TMP5]], i32 4
-; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <8 x float> [[TMP0]], i32 5
-; CHECK-NEXT:    [[VECINIT5_I:%.*]] = insertelement <8 x float> [[VECINIT4_I]], float [[TMP6]], i32 5
-; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <8 x float> [[TMP0]], i32 6
-; CHECK-NEXT:    [[VECINIT6_I:%.*]] = insertelement <8 x float> [[VECINIT5_I]], float [[TMP7]], i32 6
-; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <8 x float> [[TMP0]], i32 7
-; CHECK-NEXT:    [[VECINIT7_I:%.*]] = insertelement <8 x float> [[VECINIT6_I]], float [[TMP8]], i32 7
-; CHECK-NEXT:    ret <8 x float> [[VECINIT7_I]]
-;
-; ZEROTHRESH-LABEL: @_vadd256(
-; ZEROTHRESH-NEXT:  entry:
-; ZEROTHRESH-NEXT:    [[TMP0:%.*]] = fadd <8 x float> [[A:%.*]], [[B:%.*]]
-; ZEROTHRESH-NEXT:    [[TMP1:%.*]] = extractelement <8 x float> [[TMP0]], i32 0
-; ZEROTHRESH-NEXT:    [[VECINIT_I:%.*]] = insertelement <8 x float> undef, float [[TMP1]], i32 0
-; ZEROTHRESH-NEXT:    [[TMP2:%.*]] = extractelement <8 x float> [[TMP0]], i32 1
-; ZEROTHRESH-NEXT:    [[VECINIT1_I:%.*]] = insertelement <8 x float> [[VECINIT_I]], float [[TMP2]], i32 1
-; ZEROTHRESH-NEXT:    [[TMP3:%.*]] = extractelement <8 x float> [[TMP0]], i32 2
-; ZEROTHRESH-NEXT:    [[VECINIT2_I:%.*]] = insertelement <8 x float> [[VECINIT1_I]], float [[TMP3]], i32 2
-; ZEROTHRESH-NEXT:    [[TMP4:%.*]] = extractelement <8 x float> [[TMP0]], i32 3
-; ZEROTHRESH-NEXT:    [[VECINIT3_I:%.*]] = insertelement <8 x float> [[VECINIT2_I]], float [[TMP4]], i32 3
-; ZEROTHRESH-NEXT:    [[TMP5:%.*]] = extractelement <8 x float> [[TMP0]], i32 4
-; ZEROTHRESH-NEXT:    [[VECINIT4_I:%.*]] = insertelement <8 x float> [[VECINIT3_I]], float [[TMP5]], i32 4
-; ZEROTHRESH-NEXT:    [[TMP6:%.*]] = extractelement <8 x float> [[TMP0]], i32 5
-; ZEROTHRESH-NEXT:    [[VECINIT5_I:%.*]] = insertelement <8 x float> [[VECINIT4_I]], float [[TMP6]], i32 5
-; ZEROTHRESH-NEXT:    [[TMP7:%.*]] = extractelement <8 x float> [[TMP0]], i32 6
-; ZEROTHRESH-NEXT:    [[VECINIT6_I:%.*]] = insertelement <8 x float> [[VECINIT5_I]], float [[TMP7]], i32 6
-; ZEROTHRESH-NEXT:    [[TMP8:%.*]] = extractelement <8 x float> [[TMP0]], i32 7
-; ZEROTHRESH-NEXT:    [[VECINIT7_I:%.*]] = insertelement <8 x float> [[VECINIT6_I]], float [[TMP8]], i32 7
-; ZEROTHRESH-NEXT:    ret <8 x float> [[VECINIT7_I]]
+; ANY-LABEL: @_vadd256(
+; ANY-NEXT:    [[TMP1:%.*]] = fadd <8 x float> [[A:%.*]], [[B:%.*]]
+; ANY-NEXT:    [[TMP2:%.*]] = extractelement <8 x float> [[TMP1]], i32 0
+; ANY-NEXT:    [[VECINIT_I:%.*]] = insertelement <8 x float> undef, float [[TMP2]], i32 0
+; ANY-NEXT:    [[TMP3:%.*]] = extractelement <8 x float> [[TMP1]], i32 1
+; ANY-NEXT:    [[VECINIT1_I:%.*]] = insertelement <8 x float> [[VECINIT_I]], float [[TMP3]], i32 1
+; ANY-NEXT:    [[TMP4:%.*]] = extractelement <8 x float> [[TMP1]], i32 2
+; ANY-NEXT:    [[VECINIT2_I:%.*]] = insertelement <8 x float> [[VECINIT1_I]], float [[TMP4]], i32 2
+; ANY-NEXT:    [[TMP5:%.*]] = extractelement <8 x float> [[TMP1]], i32 3
+; ANY-NEXT:    [[VECINIT3_I:%.*]] = insertelement <8 x float> [[VECINIT2_I]], float [[TMP5]], i32 3
+; ANY-NEXT:    [[TMP6:%.*]] = extractelement <8 x float> [[TMP1]], i32 4
+; ANY-NEXT:    [[VECINIT4_I:%.*]] = insertelement <8 x float> [[VECINIT3_I]], float [[TMP6]], i32 4
+; ANY-NEXT:    [[TMP7:%.*]] = extractelement <8 x float> [[TMP1]], i32 5
+; ANY-NEXT:    [[VECINIT5_I:%.*]] = insertelement <8 x float> [[VECINIT4_I]], float [[TMP7]], i32 5
+; ANY-NEXT:    [[TMP8:%.*]] = extractelement <8 x float> [[TMP1]], i32 6
+; ANY-NEXT:    [[VECINIT6_I:%.*]] = insertelement <8 x float> [[VECINIT5_I]], float [[TMP8]], i32 6
+; ANY-NEXT:    [[TMP9:%.*]] = extractelement <8 x float> [[TMP1]], i32 7
+; ANY-NEXT:    [[VECINIT7_I:%.*]] = insertelement <8 x float> [[VECINIT6_I]], float [[TMP9]], i32 7
+; ANY-NEXT:    ret <8 x float> [[VECINIT7_I]]
 ;
-  entry:
   %vecext = extractelement <8 x float> %a, i32 0
   %vecext1 = extractelement <8 x float> %b, i32 0
   %add = fadd float %vecext, %vecext1
@@ -751,4 +567,4 @@ define <8 x float> @_vadd256(<8 x float> %a, <8 x float> %b) local_unnamed_addr
   ret <8 x float> %vecinit7.i
 }
 
-attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf"="true" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/jumbled_store_crash.ll b/llvm/test/Transforms/SLPVectorizer/X86/jumbled_store_crash.ll
new file mode 100644
index 000000000..8b12b9272
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/jumbled_store_crash.ll
@@ -0,0 +1,104 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt --slp-vectorizer -mtriple=x86_64-unknown-linux-gnu -o - -S < %s | FileCheck %s
+
+@b = common dso_local global i32* null, align 8
+@e = common dso_local global float 0.000000e+00, align 4
+@c = common dso_local global float 0.000000e+00, align 4
+@g = common dso_local global float 0.000000e+00, align 4
+@d = common dso_local global float 0.000000e+00, align 4
+@f = common dso_local global float 0.000000e+00, align 4
+@a = common dso_local global i32 0, align 4
+@h = common dso_local global float 0.000000e+00, align 4
+
+define dso_local void @j() local_unnamed_addr {
+; CHECK-LABEL: define {{[^@]+}}@j(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32*, i32** @b, align 8
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 12
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 5
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32* [[ARRAYIDX]] to <2 x i32>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i32>, <2 x i32>* [[TMP1]], align 4
+; CHECK-NEXT:    [[REORDER_SHUFFLE1:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> undef, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 13
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i32* [[ARRAYIDX1]] to <2 x i32>*
+; CHECK-NEXT:    [[TMP4:%.*]] = load <2 x i32>, <2 x i32>* [[TMP3]], align 4
+; CHECK-NEXT:    [[REORDER_SHUFFLE:%.*]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> undef, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT:    [[TMP5:%.*]] = add nsw <2 x i32> [[REORDER_SHUFFLE]], [[REORDER_SHUFFLE1]]
+; CHECK-NEXT:    [[TMP6:%.*]] = sitofp <2 x i32> [[TMP5]] to <2 x float>
+; CHECK-NEXT:    [[TMP7:%.*]] = fmul <2 x float> [[TMP6]], <float 1.000000e+01, float 1.000000e+01>
+; CHECK-NEXT:    [[TMP8:%.*]] = fsub <2 x float> <float 0.000000e+00, float 1.000000e+00>, [[TMP7]]
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <2 x float> [[TMP8]], <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <4 x float> [[SHUFFLE]], i32 1
+; CHECK-NEXT:    store float [[TMP9]], float* @g, align 4
+; CHECK-NEXT:    [[TMP10:%.*]] = fadd <4 x float> [[SHUFFLE]], <float -1.000000e+00, float -1.000000e+00, float 1.000000e+00, float 1.000000e+00>
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <4 x float> [[TMP10]], i32 2
+; CHECK-NEXT:    store float [[TMP11]], float* @c, align 4
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <4 x float> [[TMP10]], i32 0
+; CHECK-NEXT:    store float [[TMP12]], float* @d, align 4
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <4 x float> [[TMP10]], i32 3
+; CHECK-NEXT:    store float [[TMP13]], float* @e, align 4
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <4 x float> [[TMP10]], i32 1
+; CHECK-NEXT:    store float [[TMP14]], float* @f, align 4
+; CHECK-NEXT:    [[ARRAYIDX15:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 14
+; CHECK-NEXT:    [[ARRAYIDX18:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 15
+; CHECK-NEXT:    [[TMP15:%.*]] = load i32, i32* @a, align 4
+; CHECK-NEXT:    [[CONV19:%.*]] = sitofp i32 [[TMP15]] to float
+; CHECK-NEXT:    [[TMP16:%.*]] = insertelement <4 x float> undef, float [[CONV19]], i32 0
+; CHECK-NEXT:    [[TMP17:%.*]] = insertelement <4 x float> [[TMP16]], float -1.000000e+00, i32 1
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <4 x float> [[SHUFFLE]], i32 0
+; CHECK-NEXT:    [[TMP19:%.*]] = insertelement <4 x float> [[TMP17]], float [[TMP18]], i32 2
+; CHECK-NEXT:    [[TMP20:%.*]] = insertelement <4 x float> [[TMP19]], float -1.000000e+00, i32 3
+; CHECK-NEXT:    [[TMP21:%.*]] = fsub <4 x float> [[TMP10]], [[TMP20]]
+; CHECK-NEXT:    [[TMP22:%.*]] = fadd <4 x float> [[TMP10]], [[TMP20]]
+; CHECK-NEXT:    [[TMP23:%.*]] = shufflevector <4 x float> [[TMP21]], <4 x float> [[TMP22]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    [[TMP24:%.*]] = fptosi <4 x float> [[TMP23]] to <4 x i32>
+; CHECK-NEXT:    [[TMP25:%.*]] = bitcast i32* [[ARRAYIDX1]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP24]], <4 x i32>* [[TMP25]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %0 = load i32*, i32** @b, align 8
+  %arrayidx = getelementptr inbounds i32, i32* %0, i64 4
+  %1 = load i32, i32* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds i32, i32* %0, i64 12
+  %2 = load i32, i32* %arrayidx1, align 4
+  %add = add nsw i32 %2, %1
+  %conv = sitofp i32 %add to float
+  %mul = fmul float %conv, 1.000000e+01
+  %arrayidx2 = getelementptr inbounds i32, i32* %0, i64 5
+  %3 = load i32, i32* %arrayidx2, align 4
+  %arrayidx3 = getelementptr inbounds i32, i32* %0, i64 13
+  %4 = load i32, i32* %arrayidx3, align 4
+  %add4 = add nsw i32 %4, %3
+  %conv5 = sitofp i32 %add4 to float
+  %mul6 = fmul float %conv5, 1.000000e+01
+  %sub = fsub float 0.000000e+00, %mul6
+  %sub7 = fsub float 1.000000e+00, %mul
+  store float %sub7, float* @g, align 4
+  %add9 = fadd float %sub, 1.000000e+00
+  store float %add9, float* @c, align 4
+  %sub10 = fadd float %sub, -1.000000e+00
+  store float %sub10, float* @d, align 4
+  %add11 = fadd float %sub7, 1.000000e+00
+  store float %add11, float* @e, align 4
+  %sub12 = fadd float %sub7, -1.000000e+00
+  store float %sub12, float* @f, align 4
+  %sub13 = fsub float %add9, %sub
+  %conv14 = fptosi float %sub13 to i32
+  %arrayidx15 = getelementptr inbounds i32, i32* %0, i64 14
+  store i32 %conv14, i32* %arrayidx15, align 4
+  %sub16 = fadd float %add11, -1.000000e+00
+  %conv17 = fptosi float %sub16 to i32
+  %arrayidx18 = getelementptr inbounds i32, i32* %0, i64 15
+  store i32 %conv17, i32* %arrayidx18, align 4
+  %5 = load i32, i32* @a, align 4
+  %conv19 = sitofp i32 %5 to float
+  %sub20 = fsub float %sub10, %conv19
+  %conv21 = fptosi float %sub20 to i32
+  store i32 %conv21, i32* %arrayidx1, align 4
+  %sub23 = fadd float %sub12, -1.000000e+00
+  %conv24 = fptosi float %sub23 to i32
+  store i32 %conv24, i32* %arrayidx3, align 4
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/load-merge.ll b/llvm/test/Transforms/SLPVectorizer/X86/load-merge.ll
index df990be07..50eeead88 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/load-merge.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/load-merge.ll
@@ -48,3 +48,161 @@ entry:
   %or11 = or i32 %or7, %shl10
   ret i32 %or11
 }
+
+define <4 x float> @PR16739_byref(<4 x float>* nocapture readonly dereferenceable(16) %x) {
+; CHECK-LABEL: @PR16739_byref(
+; CHECK-NEXT:    [[GEP0:%.*]] = getelementptr inbounds <4 x float>, <4 x float>* [[X:%.*]], i64 0, i64 0
+; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr inbounds <4 x float>, <4 x float>* [[X]], i64 0, i64 1
+; CHECK-NEXT:    [[GEP2:%.*]] = getelementptr inbounds <4 x float>, <4 x float>* [[X]], i64 0, i64 2
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float* [[GEP0]] to <2 x float>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x float>, <2 x float>* [[TMP1]], align 4
+; CHECK-NEXT:    [[X2:%.*]] = load float, float* [[GEP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x float> [[TMP2]], i32 0
+; CHECK-NEXT:    [[I0:%.*]] = insertelement <4 x float> undef, float [[TMP3]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x float> [[TMP2]], i32 1
+; CHECK-NEXT:    [[I1:%.*]] = insertelement <4 x float> [[I0]], float [[TMP4]], i32 1
+; CHECK-NEXT:    [[I2:%.*]] = insertelement <4 x float> [[I1]], float [[X2]], i32 2
+; CHECK-NEXT:    [[I3:%.*]] = insertelement <4 x float> [[I2]], float [[X2]], i32 3
+; CHECK-NEXT:    ret <4 x float> [[I3]]
+;
+  %gep0 = getelementptr inbounds <4 x float>, <4 x float>* %x, i64 0, i64 0
+  %gep1 = getelementptr inbounds <4 x float>, <4 x float>* %x, i64 0, i64 1
+  %gep2 = getelementptr inbounds <4 x float>, <4 x float>* %x, i64 0, i64 2
+  %x0 = load float, float* %gep0
+  %x1 = load float, float* %gep1
+  %x2 = load float, float* %gep2
+  %i0 = insertelement <4 x float> undef, float %x0, i32 0
+  %i1 = insertelement <4 x float> %i0, float %x1, i32 1
+  %i2 = insertelement <4 x float> %i1, float %x2, i32 2
+  %i3 = insertelement <4 x float> %i2, float %x2, i32 3
+  ret <4 x float> %i3
+}
+
+define <4 x float> @PR16739_byref_alt(<4 x float>* nocapture readonly dereferenceable(16) %x) {
+; CHECK-LABEL: @PR16739_byref_alt(
+; CHECK-NEXT:    [[GEP0:%.*]] = getelementptr inbounds <4 x float>, <4 x float>* [[X:%.*]], i64 0, i64 0
+; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr inbounds <4 x float>, <4 x float>* [[X]], i64 0, i64 1
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float* [[GEP0]] to <2 x float>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x float>, <2 x float>* [[TMP1]], align 4
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> undef, <4 x i32> <i32 0, i32 0, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[SHUFFLE]], i32 0
+; CHECK-NEXT:    [[I0:%.*]] = insertelement <4 x float> undef, float [[TMP3]], i32 0
+; CHECK-NEXT:    [[I1:%.*]] = insertelement <4 x float> [[I0]], float [[TMP3]], i32 1
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[SHUFFLE]], i32 2
+; CHECK-NEXT:    [[I2:%.*]] = insertelement <4 x float> [[I1]], float [[TMP4]], i32 2
+; CHECK-NEXT:    [[I3:%.*]] = insertelement <4 x float> [[I2]], float [[TMP4]], i32 3
+; CHECK-NEXT:    ret <4 x float> [[I3]]
+;
+  %gep0 = getelementptr inbounds <4 x float>, <4 x float>* %x, i64 0, i64 0
+  %gep1 = getelementptr inbounds <4 x float>, <4 x float>* %x, i64 0, i64 1
+  %x0 = load float, float* %gep0
+  %x1 = load float, float* %gep1
+  %i0 = insertelement <4 x float> undef, float %x0, i32 0
+  %i1 = insertelement <4 x float> %i0, float %x0, i32 1
+  %i2 = insertelement <4 x float> %i1, float %x1, i32 2
+  %i3 = insertelement <4 x float> %i2, float %x1, i32 3
+  ret <4 x float> %i3
+}
+
+define <4 x float> @PR16739_byval(<4 x float>* nocapture readonly dereferenceable(16) %x) {
+; CHECK-LABEL: @PR16739_byval(
+; CHECK-NEXT:    [[T0:%.*]] = bitcast <4 x float>* [[X:%.*]] to i64*
+; CHECK-NEXT:    [[T1:%.*]] = load i64, i64* [[T0]], align 16
+; CHECK-NEXT:    [[T2:%.*]] = getelementptr inbounds <4 x float>, <4 x float>* [[X]], i64 0, i64 2
+; CHECK-NEXT:    [[T3:%.*]] = bitcast float* [[T2]] to i64*
+; CHECK-NEXT:    [[T4:%.*]] = load i64, i64* [[T3]], align 8
+; CHECK-NEXT:    [[T5:%.*]] = trunc i64 [[T1]] to i32
+; CHECK-NEXT:    [[T6:%.*]] = bitcast i32 [[T5]] to float
+; CHECK-NEXT:    [[T7:%.*]] = insertelement <4 x float> undef, float [[T6]], i32 0
+; CHECK-NEXT:    [[T8:%.*]] = lshr i64 [[T1]], 32
+; CHECK-NEXT:    [[T9:%.*]] = trunc i64 [[T8]] to i32
+; CHECK-NEXT:    [[T10:%.*]] = bitcast i32 [[T9]] to float
+; CHECK-NEXT:    [[T11:%.*]] = insertelement <4 x float> [[T7]], float [[T10]], i32 1
+; CHECK-NEXT:    [[T12:%.*]] = trunc i64 [[T4]] to i32
+; CHECK-NEXT:    [[T13:%.*]] = bitcast i32 [[T12]] to float
+; CHECK-NEXT:    [[T14:%.*]] = insertelement <4 x float> [[T11]], float [[T13]], i32 2
+; CHECK-NEXT:    [[T15:%.*]] = insertelement <4 x float> [[T14]], float [[T13]], i32 3
+; CHECK-NEXT:    ret <4 x float> [[T15]]
+;
+  %t0 = bitcast <4 x float>* %x to i64*
+  %t1 = load i64, i64* %t0, align 16
+  %t2 = getelementptr inbounds <4 x float>, <4 x float>* %x, i64 0, i64 2
+  %t3 = bitcast float* %t2 to i64*
+  %t4 = load i64, i64* %t3, align 8
+  %t5 = trunc i64 %t1 to i32
+  %t6 = bitcast i32 %t5 to float
+  %t7 = insertelement <4 x float> undef, float %t6, i32 0
+  %t8 = lshr i64 %t1, 32
+  %t9 = trunc i64 %t8 to i32
+  %t10 = bitcast i32 %t9 to float
+  %t11 = insertelement <4 x float> %t7, float %t10, i32 1
+  %t12 = trunc i64 %t4 to i32
+  %t13 = bitcast i32 %t12 to float
+  %t14 = insertelement <4 x float> %t11, float %t13, i32 2
+  %t15 = insertelement <4 x float> %t14, float %t13, i32 3
+  ret <4 x float> %t15
+}
+
+define void @PR43578_prefer128(i32* %r, i64* %p, i64* %q) #0 {
+; CHECK-LABEL: @PR43578_prefer128(
+; CHECK-NEXT:    [[P0:%.*]] = getelementptr inbounds i64, i64* [[P:%.*]], i64 0
+; CHECK-NEXT:    [[P1:%.*]] = getelementptr inbounds i64, i64* [[P]], i64 1
+; CHECK-NEXT:    [[P2:%.*]] = getelementptr inbounds i64, i64* [[P]], i64 2
+; CHECK-NEXT:    [[P3:%.*]] = getelementptr inbounds i64, i64* [[P]], i64 3
+; CHECK-NEXT:    [[Q0:%.*]] = getelementptr inbounds i64, i64* [[Q:%.*]], i64 0
+; CHECK-NEXT:    [[Q1:%.*]] = getelementptr inbounds i64, i64* [[Q]], i64 1
+; CHECK-NEXT:    [[Q2:%.*]] = getelementptr inbounds i64, i64* [[Q]], i64 2
+; CHECK-NEXT:    [[Q3:%.*]] = getelementptr inbounds i64, i64* [[Q]], i64 3
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i64* [[P0]] to <2 x i64>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, <2 x i64>* [[TMP1]], align 2
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i64* [[P2]] to <2 x i64>*
+; CHECK-NEXT:    [[TMP4:%.*]] = load <2 x i64>, <2 x i64>* [[TMP3]], align 2
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i64* [[Q0]] to <2 x i64>*
+; CHECK-NEXT:    [[TMP6:%.*]] = load <2 x i64>, <2 x i64>* [[TMP5]], align 2
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i64* [[Q2]] to <2 x i64>*
+; CHECK-NEXT:    [[TMP8:%.*]] = load <2 x i64>, <2 x i64>* [[TMP7]], align 2
+; CHECK-NEXT:    [[TMP9:%.*]] = sub nsw <2 x i64> [[TMP2]], [[TMP6]]
+; CHECK-NEXT:    [[TMP10:%.*]] = sub nsw <2 x i64> [[TMP4]], [[TMP8]]
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <2 x i64> [[TMP9]], i32 0
+; CHECK-NEXT:    [[G0:%.*]] = getelementptr inbounds i32, i32* [[R:%.*]], i64 [[TMP11]]
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <2 x i64> [[TMP9]], i32 1
+; CHECK-NEXT:    [[G1:%.*]] = getelementptr inbounds i32, i32* [[R]], i64 [[TMP12]]
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <2 x i64> [[TMP10]], i32 0
+; CHECK-NEXT:    [[G2:%.*]] = getelementptr inbounds i32, i32* [[R]], i64 [[TMP13]]
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x i64> [[TMP10]], i32 1
+; CHECK-NEXT:    [[G3:%.*]] = getelementptr inbounds i32, i32* [[R]], i64 [[TMP14]]
+; CHECK-NEXT:    ret void
+;
+  %p0 = getelementptr inbounds i64, i64* %p, i64 0
+  %p1 = getelementptr inbounds i64, i64* %p, i64 1
+  %p2 = getelementptr inbounds i64, i64* %p, i64 2
+  %p3 = getelementptr inbounds i64, i64* %p, i64 3
+
+  %q0 = getelementptr inbounds i64, i64* %q, i64 0
+  %q1 = getelementptr inbounds i64, i64* %q, i64 1
+  %q2 = getelementptr inbounds i64, i64* %q, i64 2
+  %q3 = getelementptr inbounds i64, i64* %q, i64 3
+
+  %x0 = load i64, i64* %p0, align 2
+  %x1 = load i64, i64* %p1, align 2
+  %x2 = load i64, i64* %p2, align 2
+  %x3 = load i64, i64* %p3, align 2
+
+  %y0 = load i64, i64* %q0, align 2
+  %y1 = load i64, i64* %q1, align 2
+  %y2 = load i64, i64* %q2, align 2
+  %y3 = load i64, i64* %q3, align 2
+
+  %sub0 = sub nsw i64 %x0, %y0
+  %sub1 = sub nsw i64 %x1, %y1
+  %sub2 = sub nsw i64 %x2, %y2
+  %sub3 = sub nsw i64 %x3, %y3
+
+  %g0 = getelementptr inbounds i32, i32* %r, i64 %sub0
+  %g1 = getelementptr inbounds i32, i32* %r, i64 %sub1
+  %g2 = getelementptr inbounds i32, i32* %r, i64 %sub2
+  %g3 = getelementptr inbounds i32, i32* %r, i64 %sub3
+  ret void
+}
+
+attributes #0 = { "prefer-vector-width"="128" }
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/long_chains.ll b/llvm/test/Transforms/SLPVectorizer/X86/long_chains.ll
index ffbdd9f1d..52c1c6bee 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/long_chains.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/long_chains.ll
@@ -12,10 +12,10 @@ define i32 @test(double* nocapture %A, i8* nocapture %B) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i8* [[B:%.*]] to <2 x i8>*
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i8>, <2 x i8>* [[TMP0]], align 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = add <2 x i8> [[TMP1]], <i8 3, i8 3>
-; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x i8> [[TMP2]], i32 0
-; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <2 x i8> undef, i8 [[TMP3]], i32 0
-; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i8> [[TMP2]], i32 1
-; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i8> [[TMP4]], i8 [[TMP5]], i32 1
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x i8> [[TMP2]], i32 1
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i8> [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <2 x i8> undef, i8 [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i8> [[TMP5]], i8 [[TMP3]], i32 1
 ; CHECK-NEXT:    [[TMP7:%.*]] = sitofp <2 x i8> [[TMP6]] to <2 x double>
 ; CHECK-NEXT:    [[TMP8:%.*]] = fmul <2 x double> [[TMP7]], [[TMP7]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = fadd <2 x double> [[TMP8]], <double 1.000000e+00, double 1.000000e+00>
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/lookahead.ll b/llvm/test/Transforms/SLPVectorizer/X86/lookahead.ll
index f89cae88a..751653089 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/lookahead.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/lookahead.ll
@@ -27,22 +27,19 @@ define void @lookahead_basic(double* %array) {
 ; CHECK-NEXT:    [[IDX5:%.*]] = getelementptr inbounds double, double* [[ARRAY]], i64 5
 ; CHECK-NEXT:    [[IDX6:%.*]] = getelementptr inbounds double, double* [[ARRAY]], i64 6
 ; CHECK-NEXT:    [[IDX7:%.*]] = getelementptr inbounds double, double* [[ARRAY]], i64 7
-; CHECK-NEXT:    [[A_0:%.*]] = load double, double* [[IDX0]], align 8
-; CHECK-NEXT:    [[A_1:%.*]] = load double, double* [[IDX1]], align 8
-; CHECK-NEXT:    [[B_0:%.*]] = load double, double* [[IDX2]], align 8
-; CHECK-NEXT:    [[B_1:%.*]] = load double, double* [[IDX3]], align 8
-; CHECK-NEXT:    [[C_0:%.*]] = load double, double* [[IDX4]], align 8
-; CHECK-NEXT:    [[C_1:%.*]] = load double, double* [[IDX5]], align 8
-; CHECK-NEXT:    [[D_0:%.*]] = load double, double* [[IDX6]], align 8
-; CHECK-NEXT:    [[D_1:%.*]] = load double, double* [[IDX7]], align 8
-; CHECK-NEXT:    [[SUBAB_0:%.*]] = fsub fast double [[A_0]], [[B_0]]
-; CHECK-NEXT:    [[SUBCD_0:%.*]] = fsub fast double [[C_0]], [[D_0]]
-; CHECK-NEXT:    [[SUBAB_1:%.*]] = fsub fast double [[A_1]], [[B_1]]
-; CHECK-NEXT:    [[SUBCD_1:%.*]] = fsub fast double [[C_1]], [[D_1]]
-; CHECK-NEXT:    [[ADDABCD_0:%.*]] = fadd fast double [[SUBAB_0]], [[SUBCD_0]]
-; CHECK-NEXT:    [[ADDCDAB_1:%.*]] = fadd fast double [[SUBCD_1]], [[SUBAB_1]]
-; CHECK-NEXT:    store double [[ADDABCD_0]], double* [[IDX0]], align 8
-; CHECK-NEXT:    store double [[ADDCDAB_1]], double* [[IDX1]], align 8
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast double* [[IDX0]] to <2 x double>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* [[TMP0]], align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast double* [[IDX2]] to <2 x double>*
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x double>, <2 x double>* [[TMP2]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast double* [[IDX4]] to <2 x double>*
+; CHECK-NEXT:    [[TMP5:%.*]] = load <2 x double>, <2 x double>* [[TMP4]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast double* [[IDX6]] to <2 x double>*
+; CHECK-NEXT:    [[TMP7:%.*]] = load <2 x double>, <2 x double>* [[TMP6]], align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = fsub fast <2 x double> [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP9:%.*]] = fsub fast <2 x double> [[TMP5]], [[TMP7]]
+; CHECK-NEXT:    [[TMP10:%.*]] = fadd fast <2 x double> [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast double* [[IDX0]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP10]], <2 x double>* [[TMP11]], align 8
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -164,22 +161,23 @@ define void @lookahead_alt2(double* %array) {
 ; CHECK-NEXT:    [[IDX5:%.*]] = getelementptr inbounds double, double* [[ARRAY]], i64 5
 ; CHECK-NEXT:    [[IDX6:%.*]] = getelementptr inbounds double, double* [[ARRAY]], i64 6
 ; CHECK-NEXT:    [[IDX7:%.*]] = getelementptr inbounds double, double* [[ARRAY]], i64 7
-; CHECK-NEXT:    [[A_0:%.*]] = load double, double* [[IDX0]], align 8
-; CHECK-NEXT:    [[A_1:%.*]] = load double, double* [[IDX1]], align 8
-; CHECK-NEXT:    [[B_0:%.*]] = load double, double* [[IDX2]], align 8
-; CHECK-NEXT:    [[B_1:%.*]] = load double, double* [[IDX3]], align 8
-; CHECK-NEXT:    [[C_0:%.*]] = load double, double* [[IDX4]], align 8
-; CHECK-NEXT:    [[C_1:%.*]] = load double, double* [[IDX5]], align 8
-; CHECK-NEXT:    [[D_0:%.*]] = load double, double* [[IDX6]], align 8
-; CHECK-NEXT:    [[D_1:%.*]] = load double, double* [[IDX7]], align 8
-; CHECK-NEXT:    [[ADDAB_0:%.*]] = fadd fast double [[A_0]], [[B_0]]
-; CHECK-NEXT:    [[SUBCD_0:%.*]] = fsub fast double [[C_0]], [[D_0]]
-; CHECK-NEXT:    [[ADDCD_1:%.*]] = fadd fast double [[C_1]], [[D_1]]
-; CHECK-NEXT:    [[SUBAB_1:%.*]] = fsub fast double [[A_1]], [[B_1]]
-; CHECK-NEXT:    [[ADDABCD_0:%.*]] = fadd fast double [[ADDAB_0]], [[SUBCD_0]]
-; CHECK-NEXT:    [[ADDCDAB_1:%.*]] = fadd fast double [[ADDCD_1]], [[SUBAB_1]]
-; CHECK-NEXT:    store double [[ADDABCD_0]], double* [[IDX0]], align 8
-; CHECK-NEXT:    store double [[ADDCDAB_1]], double* [[IDX1]], align 8
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast double* [[IDX0]] to <2 x double>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* [[TMP0]], align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast double* [[IDX2]] to <2 x double>*
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x double>, <2 x double>* [[TMP2]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast double* [[IDX4]] to <2 x double>*
+; CHECK-NEXT:    [[TMP5:%.*]] = load <2 x double>, <2 x double>* [[TMP4]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast double* [[IDX6]] to <2 x double>*
+; CHECK-NEXT:    [[TMP7:%.*]] = load <2 x double>, <2 x double>* [[TMP6]], align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = fsub fast <2 x double> [[TMP5]], [[TMP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = fadd fast <2 x double> [[TMP5]], [[TMP7]]
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <2 x double> [[TMP8]], <2 x double> [[TMP9]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT:    [[TMP11:%.*]] = fadd fast <2 x double> [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP12:%.*]] = fsub fast <2 x double> [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <2 x double> [[TMP11]], <2 x double> [[TMP12]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT:    [[TMP14:%.*]] = fadd fast <2 x double> [[TMP13]], [[TMP10]]
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast double* [[IDX0]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP14]], <2 x double>* [[TMP15]], align 8
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -239,6 +237,97 @@ define void @lookahead_external_uses(double* %A, double *%B, double *%C, double
 ; CHECK-NEXT:    [[IDXB2:%.*]] = getelementptr inbounds double, double* [[B]], i64 2
 ; CHECK-NEXT:    [[IDXA2:%.*]] = getelementptr inbounds double, double* [[A]], i64 2
 ; CHECK-NEXT:    [[IDXB1:%.*]] = getelementptr inbounds double, double* [[B]], i64 1
+; CHECK-NEXT:    [[A0:%.*]] = load double, double* [[IDXA0]], align 8
+; CHECK-NEXT:    [[C0:%.*]] = load double, double* [[IDXC0]], align 8
+; CHECK-NEXT:    [[D0:%.*]] = load double, double* [[IDXD0]], align 8
+; CHECK-NEXT:    [[A1:%.*]] = load double, double* [[IDXA1]], align 8
+; CHECK-NEXT:    [[B2:%.*]] = load double, double* [[IDXB2]], align 8
+; CHECK-NEXT:    [[A2:%.*]] = load double, double* [[IDXA2]], align 8
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast double* [[IDXB0]] to <2 x double>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* [[TMP0]], align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> undef, double [[C0]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x double> [[TMP2]], double [[A1]], i32 1
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <2 x double> undef, double [[D0]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <2 x double> [[TMP4]], double [[B2]], i32 1
+; CHECK-NEXT:    [[TMP6:%.*]] = fsub fast <2 x double> [[TMP3]], [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <2 x double> undef, double [[A0]], i32 0
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <2 x double> [[TMP7]], double [[A2]], i32 1
+; CHECK-NEXT:    [[TMP9:%.*]] = fsub fast <2 x double> [[TMP8]], [[TMP1]]
+; CHECK-NEXT:    [[TMP10:%.*]] = fadd fast <2 x double> [[TMP9]], [[TMP6]]
+; CHECK-NEXT:    [[IDXS0:%.*]] = getelementptr inbounds double, double* [[S:%.*]], i64 0
+; CHECK-NEXT:    [[IDXS1:%.*]] = getelementptr inbounds double, double* [[S]], i64 1
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast double* [[IDXS0]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP10]], <2 x double>* [[TMP11]], align 8
+; CHECK-NEXT:    store double [[A1]], double* [[EXT1:%.*]], align 8
+; CHECK-NEXT:    ret void
+;
+entry:
+  %IdxA0 = getelementptr inbounds double, double* %A, i64 0
+  %IdxB0 = getelementptr inbounds double, double* %B, i64 0
+  %IdxC0 = getelementptr inbounds double, double* %C, i64 0
+  %IdxD0 = getelementptr inbounds double, double* %D, i64 0
+
+  %IdxA1 = getelementptr inbounds double, double* %A, i64 1
+  %IdxB2 = getelementptr inbounds double, double* %B, i64 2
+  %IdxA2 = getelementptr inbounds double, double* %A, i64 2
+  %IdxB1 = getelementptr inbounds double, double* %B, i64 1
+
+  %A0 = load double, double *%IdxA0, align 8
+  %B0 = load double, double *%IdxB0, align 8
+  %C0 = load double, double *%IdxC0, align 8
+  %D0 = load double, double *%IdxD0, align 8
+
+  %A1 = load double, double *%IdxA1, align 8
+  %B2 = load double, double *%IdxB2, align 8
+  %A2 = load double, double *%IdxA2, align 8
+  %B1 = load double, double *%IdxB1, align 8
+
+  %subA0B0 = fsub fast double %A0, %B0
+  %subC0D0 = fsub fast double %C0, %D0
+
+  %subA1B2 = fsub fast double %A1, %B2
+  %subA2B1 = fsub fast double %A2, %B1
+
+  %add0 = fadd fast double %subA0B0, %subC0D0
+  %add1 = fadd fast double %subA1B2, %subA2B1
+
+  %IdxS0 = getelementptr inbounds double, double* %S, i64 0
+  %IdxS1 = getelementptr inbounds double, double* %S, i64 1
+
+  store double %add0, double *%IdxS0, align 8
+  store double %add1, double *%IdxS1, align 8
+
+  ; External use
+  store double %A1, double *%Ext1, align 8
+  ret void
+}
+
+; A[0] B[0] C[0] D[0]  A[1] B[2] A[2] B[1]
+;     \  /   \  /       /  \  /   \  / \
+;       -     -    U1,U2,U3  -     -  U4,U5
+;        \   /                \   /
+;          +                    +
+;          |                    |
+;         S[0]                 S[1]
+;
+;
+; If we limit the users budget for the look-ahead heuristic to 2, then the
+; look-ahead heuristic has no way of choosing B[1] (with 2 external users)
+; over A[1] (with 3 external users).
+; The result is that the operands are of the Add not reordered and the loads
+; from A get vectorized instead of the loads from B.
+;
+define void @lookahead_limit_users_budget(double* %A, double *%B, double *%C, double *%D, double *%S, double *%Ext1, double *%Ext2, double *%Ext3, double *%Ext4, double *%Ext5) {
+; CHECK-LABEL: @lookahead_limit_users_budget(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[IDXA0:%.*]] = getelementptr inbounds double, double* [[A:%.*]], i64 0
+; CHECK-NEXT:    [[IDXB0:%.*]] = getelementptr inbounds double, double* [[B:%.*]], i64 0
+; CHECK-NEXT:    [[IDXC0:%.*]] = getelementptr inbounds double, double* [[C:%.*]], i64 0
+; CHECK-NEXT:    [[IDXD0:%.*]] = getelementptr inbounds double, double* [[D:%.*]], i64 0
+; CHECK-NEXT:    [[IDXA1:%.*]] = getelementptr inbounds double, double* [[A]], i64 1
+; CHECK-NEXT:    [[IDXB2:%.*]] = getelementptr inbounds double, double* [[B]], i64 2
+; CHECK-NEXT:    [[IDXA2:%.*]] = getelementptr inbounds double, double* [[A]], i64 2
+; CHECK-NEXT:    [[IDXB1:%.*]] = getelementptr inbounds double, double* [[B]], i64 1
 ; CHECK-NEXT:    [[B0:%.*]] = load double, double* [[IDXB0]], align 8
 ; CHECK-NEXT:    [[C0:%.*]] = load double, double* [[IDXC0]], align 8
 ; CHECK-NEXT:    [[D0:%.*]] = load double, double* [[IDXD0]], align 8
@@ -262,6 +351,10 @@ define void @lookahead_external_uses(double* %A, double *%B, double *%C, double
 ; CHECK-NEXT:    store <2 x double> [[TMP10]], <2 x double>* [[TMP11]], align 8
 ; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <2 x double> [[TMP1]], i32 1
 ; CHECK-NEXT:    store double [[TMP12]], double* [[EXT1:%.*]], align 8
+; CHECK-NEXT:    store double [[TMP12]], double* [[EXT2:%.*]], align 8
+; CHECK-NEXT:    store double [[TMP12]], double* [[EXT3:%.*]], align 8
+; CHECK-NEXT:    store double [[B1]], double* [[EXT4:%.*]], align 8
+; CHECK-NEXT:    store double [[B1]], double* [[EXT5:%.*]], align 8
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -300,7 +393,254 @@ entry:
   store double %add0, double *%IdxS0, align 8
   store double %add1, double *%IdxS1, align 8
 
-  ; External use
+  ; External uses of A1
   store double %A1, double *%Ext1, align 8
+  store double %A1, double *%Ext2, align 8
+  store double %A1, double *%Ext3, align 8
+
+  ; External uses of B1
+  store double %B1, double *%Ext4, align 8
+  store double %B1, double *%Ext5, align 8
+
+  ret void
+}
+
+; This checks that the lookahead code does not crash when instructions with the same opcodes have different numbers of operands (in this case the calls).
+
+%Class = type { i8 }
+declare double @_ZN1i2ayEv(%Class*)
+declare double @_ZN1i2axEv()
+
+define void @lookahead_crash(double* %A, double *%S, %Class *%Arg0) {
+; CHECK-LABEL: @lookahead_crash(
+; CHECK-NEXT:    [[IDXA0:%.*]] = getelementptr inbounds double, double* [[A:%.*]], i64 0
+; CHECK-NEXT:    [[IDXA1:%.*]] = getelementptr inbounds double, double* [[A]], i64 1
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast double* [[IDXA0]] to <2 x double>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x double>, <2 x double>* [[TMP1]], align 8
+; CHECK-NEXT:    [[C0:%.*]] = call double @_ZN1i2ayEv(%Class* [[ARG0:%.*]])
+; CHECK-NEXT:    [[C1:%.*]] = call double @_ZN1i2axEv()
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x double> undef, double [[C0]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <2 x double> [[TMP3]], double [[C1]], i32 1
+; CHECK-NEXT:    [[TMP5:%.*]] = fadd fast <2 x double> [[TMP2]], [[TMP4]]
+; CHECK-NEXT:    [[IDXS0:%.*]] = getelementptr inbounds double, double* [[S:%.*]], i64 0
+; CHECK-NEXT:    [[IDXS1:%.*]] = getelementptr inbounds double, double* [[S]], i64 1
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast double* [[IDXS0]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP5]], <2 x double>* [[TMP6]], align 8
+; CHECK-NEXT:    ret void
+;
+  %IdxA0 = getelementptr inbounds double, double* %A, i64 0
+  %IdxA1 = getelementptr inbounds double, double* %A, i64 1
+
+  %A0 = load double, double *%IdxA0, align 8
+  %A1 = load double, double *%IdxA1, align 8
+
+  %C0 = call double @_ZN1i2ayEv(%Class *%Arg0)
+  %C1 = call double @_ZN1i2axEv()
+
+  %add0 = fadd fast double %A0, %C0
+  %add1 = fadd fast double %A1, %C1
+
+  %IdxS0 = getelementptr inbounds double, double* %S, i64 0
+  %IdxS1 = getelementptr inbounds double, double* %S, i64 1
+  store double %add0, double *%IdxS0, align 8
+  store double %add1, double *%IdxS1, align 8
+  ret void
+}
+
+; This checks that we choose to group consecutive extracts from the same vectors.
+define void @ChecksExtractScores(double* %storeArray, double* %array, <2 x double> *%vecPtr1, <2 x double>* %vecPtr2) {
+; CHECK-LABEL: @ChecksExtractScores(
+; CHECK-NEXT:    [[IDX0:%.*]] = getelementptr inbounds double, double* [[ARRAY:%.*]], i64 0
+; CHECK-NEXT:    [[IDX1:%.*]] = getelementptr inbounds double, double* [[ARRAY]], i64 1
+; CHECK-NEXT:    [[LOADA0:%.*]] = load double, double* [[IDX0]], align 4
+; CHECK-NEXT:    [[LOADA1:%.*]] = load double, double* [[IDX1]], align 4
+; CHECK-NEXT:    [[LOADVEC:%.*]] = load <2 x double>, <2 x double>* [[VECPTR1:%.*]], align 4
+; CHECK-NEXT:    [[LOADVEC2:%.*]] = load <2 x double>, <2 x double>* [[VECPTR2:%.*]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> undef, double [[LOADA0]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> [[TMP1]], double [[LOADA0]], i32 1
+; CHECK-NEXT:    [[TMP3:%.*]] = fmul <2 x double> [[LOADVEC]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <2 x double> undef, double [[LOADA1]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <2 x double> [[TMP4]], double [[LOADA1]], i32 1
+; CHECK-NEXT:    [[TMP6:%.*]] = fmul <2 x double> [[LOADVEC2]], [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = fadd <2 x double> [[TMP3]], [[TMP6]]
+; CHECK-NEXT:    [[SIDX0:%.*]] = getelementptr inbounds double, double* [[STOREARRAY:%.*]], i64 0
+; CHECK-NEXT:    [[SIDX1:%.*]] = getelementptr inbounds double, double* [[STOREARRAY]], i64 1
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast double* [[SIDX0]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP7]], <2 x double>* [[TMP8]], align 8
+; CHECK-NEXT:    ret void
+;
+  %idx0 = getelementptr inbounds double, double* %array, i64 0
+  %idx1 = getelementptr inbounds double, double* %array, i64 1
+  %loadA0 = load double, double* %idx0, align 4
+  %loadA1 = load double, double* %idx1, align 4
+
+  %loadVec = load <2 x double>, <2 x double>* %vecPtr1, align 4
+  %extrA0 = extractelement <2 x double> %loadVec, i32 0
+  %extrA1 = extractelement <2 x double> %loadVec, i32 1
+  %loadVec2 = load <2 x double>, <2 x double>* %vecPtr2, align 4
+  %extrB0 = extractelement <2 x double> %loadVec2, i32 0
+  %extrB1 = extractelement <2 x double> %loadVec2, i32 1
+
+  %mul0 = fmul double %extrA0, %loadA0
+  %mul1 = fmul double %extrA1, %loadA0
+  %mul3 = fmul double %extrB0, %loadA1
+  %mul4 = fmul double %extrB1, %loadA1
+  %add0 = fadd double %mul0, %mul3
+  %add1 = fadd double %mul1, %mul4
+
+  %sidx0 = getelementptr inbounds double, double* %storeArray, i64 0
+  %sidx1 = getelementptr inbounds double, double* %storeArray, i64 1
+  store double %add0, double *%sidx0, align 8
+  store double %add1, double *%sidx1, align 8
+  ret void
+}
+
+
+define i1 @ExtractIdxNotConstantInt1(float %a, float %b, float %c, <4 x float> %vec, i64 %idx2) {
+; CHECK-LABEL: @ExtractIdxNotConstantInt1(
+; CHECK-NEXT:    [[VECEXT_I291_I166:%.*]] = extractelement <4 x float> [[VEC:%.*]], i64 undef
+; CHECK-NEXT:    [[SUB14_I167:%.*]] = fsub float undef, [[VECEXT_I291_I166]]
+; CHECK-NEXT:    [[FM:%.*]] = fmul float [[A:%.*]], [[SUB14_I167]]
+; CHECK-NEXT:    [[SUB25_I168:%.*]] = fsub float [[FM]], [[B:%.*]]
+; CHECK-NEXT:    [[VECEXT_I276_I169:%.*]] = extractelement <4 x float> [[VEC]], i64 [[IDX2:%.*]]
+; CHECK-NEXT:    [[ADD36_I173:%.*]] = fadd float [[SUB25_I168]], 1.000000e+01
+; CHECK-NEXT:    [[MUL72_I179:%.*]] = fmul float [[C:%.*]], [[VECEXT_I276_I169]]
+; CHECK-NEXT:    [[ADD78_I180:%.*]] = fsub float [[MUL72_I179]], 3.000000e+01
+; CHECK-NEXT:    [[ADD79_I181:%.*]] = fadd float 2.000000e+00, [[ADD78_I180]]
+; CHECK-NEXT:    [[MUL123_I184:%.*]] = fmul float [[ADD36_I173]], [[ADD79_I181]]
+; CHECK-NEXT:    [[CMP_I185:%.*]] = fcmp ogt float [[MUL123_I184]], 0.000000e+00
+; CHECK-NEXT:    ret i1 [[CMP_I185]]
+;
+  %vecext.i291.i166 = extractelement <4 x float> %vec, i64 undef
+  %sub14.i167 = fsub float undef, %vecext.i291.i166
+  %fm = fmul float %a, %sub14.i167
+  %sub25.i168 = fsub float %fm, %b
+  %vecext.i276.i169 = extractelement <4 x float> %vec, i64 %idx2
+  %add36.i173 = fadd float %sub25.i168, 10.0
+  %mul72.i179 = fmul float %c, %vecext.i276.i169
+  %add78.i180 = fsub float %mul72.i179, 30.0
+  %add79.i181 = fadd float 2.0, %add78.i180
+  %mul123.i184 = fmul float %add36.i173, %add79.i181
+  %cmp.i185 = fcmp ogt float %mul123.i184, 0.000000e+00
+  ret i1 %cmp.i185
+}
+
+
+define i1 @ExtractIdxNotConstantInt2(float %a, float %b, float %c, <4 x float> %vec, i64 %idx2) {
+; CHECK-LABEL: @ExtractIdxNotConstantInt2(
+; CHECK-NEXT:    [[VECEXT_I291_I166:%.*]] = extractelement <4 x float> [[VEC:%.*]], i64 1
+; CHECK-NEXT:    [[SUB14_I167:%.*]] = fsub float undef, [[VECEXT_I291_I166]]
+; CHECK-NEXT:    [[FM:%.*]] = fmul float [[A:%.*]], [[SUB14_I167]]
+; CHECK-NEXT:    [[SUB25_I168:%.*]] = fsub float [[FM]], [[B:%.*]]
+; CHECK-NEXT:    [[VECEXT_I276_I169:%.*]] = extractelement <4 x float> [[VEC]], i64 [[IDX2:%.*]]
+; CHECK-NEXT:    [[ADD36_I173:%.*]] = fadd float [[SUB25_I168]], 1.000000e+01
+; CHECK-NEXT:    [[MUL72_I179:%.*]] = fmul float [[C:%.*]], [[VECEXT_I276_I169]]
+; CHECK-NEXT:    [[ADD78_I180:%.*]] = fsub float [[MUL72_I179]], 3.000000e+01
+; CHECK-NEXT:    [[ADD79_I181:%.*]] = fadd float 2.000000e+00, [[ADD78_I180]]
+; CHECK-NEXT:    [[MUL123_I184:%.*]] = fmul float [[ADD36_I173]], [[ADD79_I181]]
+; CHECK-NEXT:    [[CMP_I185:%.*]] = fcmp ogt float [[MUL123_I184]], 0.000000e+00
+; CHECK-NEXT:    ret i1 [[CMP_I185]]
+;
+  %vecext.i291.i166 = extractelement <4 x float> %vec, i64 1
+  %sub14.i167 = fsub float undef, %vecext.i291.i166
+  %fm = fmul float %a, %sub14.i167
+  %sub25.i168 = fsub float %fm, %b
+  %vecext.i276.i169 = extractelement <4 x float> %vec, i64 %idx2
+  %add36.i173 = fadd float %sub25.i168, 10.0
+  %mul72.i179 = fmul float %c, %vecext.i276.i169
+  %add78.i180 = fsub float %mul72.i179, 30.0
+  %add79.i181 = fadd float 2.0, %add78.i180
+  %mul123.i184 = fmul float %add36.i173, %add79.i181
+  %cmp.i185 = fcmp ogt float %mul123.i184, 0.000000e+00
+  ret i1 %cmp.i185
+}
+
+
+define i1 @foo(float %a, float %b, float %c, <4 x float> %vec, i64 %idx2) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:    [[VECEXT_I291_I166:%.*]] = extractelement <4 x float> [[VEC:%.*]], i64 0
+; CHECK-NEXT:    [[SUB14_I167:%.*]] = fsub float undef, [[VECEXT_I291_I166]]
+; CHECK-NEXT:    [[FM:%.*]] = fmul float [[A:%.*]], [[SUB14_I167]]
+; CHECK-NEXT:    [[SUB25_I168:%.*]] = fsub float [[FM]], [[B:%.*]]
+; CHECK-NEXT:    [[VECEXT_I276_I169:%.*]] = extractelement <4 x float> [[VEC]], i64 1
+; CHECK-NEXT:    [[ADD36_I173:%.*]] = fadd float [[SUB25_I168]], 1.000000e+01
+; CHECK-NEXT:    [[MUL72_I179:%.*]] = fmul float [[C:%.*]], [[VECEXT_I276_I169]]
+; CHECK-NEXT:    [[ADD78_I180:%.*]] = fsub float [[MUL72_I179]], 3.000000e+01
+; CHECK-NEXT:    [[ADD79_I181:%.*]] = fadd float 2.000000e+00, [[ADD78_I180]]
+; CHECK-NEXT:    [[MUL123_I184:%.*]] = fmul float [[ADD36_I173]], [[ADD79_I181]]
+; CHECK-NEXT:    [[CMP_I185:%.*]] = fcmp ogt float [[MUL123_I184]], 0.000000e+00
+; CHECK-NEXT:    ret i1 [[CMP_I185]]
+;
+  %vecext.i291.i166 = extractelement <4 x float> %vec, i64 0
+  %sub14.i167 = fsub float undef, %vecext.i291.i166
+  %fm = fmul float %a, %sub14.i167
+  %sub25.i168 = fsub float %fm, %b
+  %vecext.i276.i169 = extractelement <4 x float> %vec, i64 1
+  %add36.i173 = fadd float %sub25.i168, 10.0
+  %mul72.i179 = fmul float %c, %vecext.i276.i169
+  %add78.i180 = fsub float %mul72.i179, 30.0
+  %add79.i181 = fadd float 2.0, %add78.i180
+  %mul123.i184 = fmul float %add36.i173, %add79.i181
+  %cmp.i185 = fcmp ogt float %mul123.i184, 0.000000e+00
+  ret i1 %cmp.i185
+}
+
+; Same as @ChecksExtractScores, but the extratelement vector operands do not match.
+define void @ChecksExtractScores_different_vectors(double* %storeArray, double* %array, <2 x double> *%vecPtr1, <2 x double>* %vecPtr2, <2 x double>* %vecPtr3, <2 x double>* %vecPtr4) {
+; CHECK-LABEL: @ChecksExtractScores_different_vectors(
+; CHECK-NEXT:    [[IDX0:%.*]] = getelementptr inbounds double, double* [[ARRAY:%.*]], i64 0
+; CHECK-NEXT:    [[IDX1:%.*]] = getelementptr inbounds double, double* [[ARRAY]], i64 1
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast double* [[IDX0]] to <2 x double>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x double>, <2 x double>* [[TMP1]], align 4
+; CHECK-NEXT:    [[LOADVEC:%.*]] = load <2 x double>, <2 x double>* [[VECPTR1:%.*]], align 4
+; CHECK-NEXT:    [[LOADVEC2:%.*]] = load <2 x double>, <2 x double>* [[VECPTR2:%.*]], align 4
+; CHECK-NEXT:    [[EXTRA0:%.*]] = extractelement <2 x double> [[LOADVEC]], i32 0
+; CHECK-NEXT:    [[EXTRA1:%.*]] = extractelement <2 x double> [[LOADVEC2]], i32 1
+; CHECK-NEXT:    [[LOADVEC3:%.*]] = load <2 x double>, <2 x double>* [[VECPTR3:%.*]], align 4
+; CHECK-NEXT:    [[LOADVEC4:%.*]] = load <2 x double>, <2 x double>* [[VECPTR4:%.*]], align 4
+; CHECK-NEXT:    [[EXTRB0:%.*]] = extractelement <2 x double> [[LOADVEC3]], i32 0
+; CHECK-NEXT:    [[EXTRB1:%.*]] = extractelement <2 x double> [[LOADVEC4]], i32 1
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x double> undef, double [[EXTRB0]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <2 x double> [[TMP3]], double [[EXTRA1]], i32 1
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x double> [[TMP2]], i32 1
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x double> undef, double [[TMP5]], i32 0
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x double> [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <2 x double> [[TMP6]], double [[TMP7]], i32 1
+; CHECK-NEXT:    [[TMP9:%.*]] = fmul <2 x double> [[TMP4]], [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <2 x double> undef, double [[EXTRA0]], i32 0
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <2 x double> [[TMP10]], double [[EXTRB1]], i32 1
+; CHECK-NEXT:    [[TMP12:%.*]] = fmul <2 x double> [[TMP11]], [[TMP2]]
+; CHECK-NEXT:    [[TMP13:%.*]] = fadd <2 x double> [[TMP12]], [[TMP9]]
+; CHECK-NEXT:    [[SIDX0:%.*]] = getelementptr inbounds double, double* [[STOREARRAY:%.*]], i64 0
+; CHECK-NEXT:    [[SIDX1:%.*]] = getelementptr inbounds double, double* [[STOREARRAY]], i64 1
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast double* [[SIDX0]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP13]], <2 x double>* [[TMP14]], align 8
+; CHECK-NEXT:    ret void
+;
+  %idx0 = getelementptr inbounds double, double* %array, i64 0
+  %idx1 = getelementptr inbounds double, double* %array, i64 1
+  %loadA0 = load double, double* %idx0, align 4
+  %loadA1 = load double, double* %idx1, align 4
+
+  %loadVec = load <2 x double>, <2 x double>* %vecPtr1, align 4
+  %loadVec2 = load <2 x double>, <2 x double>* %vecPtr2, align 4
+  %extrA0 = extractelement <2 x double> %loadVec, i32 0
+  %extrA1 = extractelement <2 x double> %loadVec2, i32 1
+  %loadVec3= load <2 x double>, <2 x double>* %vecPtr3, align 4
+  %loadVec4 = load <2 x double>, <2 x double>* %vecPtr4, align 4
+  %extrB0 = extractelement <2 x double> %loadVec3, i32 0
+  %extrB1 = extractelement <2 x double> %loadVec4, i32 1
+
+  %mul0 = fmul double %extrA0, %loadA0
+  %mul1 = fmul double %extrA1, %loadA0
+  %mul3 = fmul double %extrB0, %loadA1
+  %mul4 = fmul double %extrB1, %loadA1
+  %add0 = fadd double %mul0, %mul3
+  %add1 = fadd double %mul1, %mul4
+
+  %sidx0 = getelementptr inbounds double, double* %storeArray, i64 0
+  %sidx1 = getelementptr inbounds double, double* %storeArray, i64 1
+  store double %add0, double *%sidx0, align 8
+  store double %add1, double *%sidx1, align 8
   ret void
 }
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/no_alternate_divrem.ll b/llvm/test/Transforms/SLPVectorizer/X86/no_alternate_divrem.ll
new file mode 100644
index 000000000..468cabc59
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/no_alternate_divrem.ll
@@ -0,0 +1,129 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -slp-vectorizer -slp-threshold=-200 -mtriple=x86_64-unknown-linux -mcpu=core-avx2 -S | FileCheck %s
+
+define void @test_add_sdiv(i32 *%arr1, i32 *%arr2, i32 %a0, i32 %a1, i32 %a2, i32 %a3) {
+; CHECK-LABEL: @test_add_sdiv(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[GEP1_0:%.*]] = getelementptr i32, i32* [[ARR1:%.*]], i32 0
+; CHECK-NEXT:    [[GEP1_1:%.*]] = getelementptr i32, i32* [[ARR1]], i32 1
+; CHECK-NEXT:    [[GEP1_2:%.*]] = getelementptr i32, i32* [[ARR1]], i32 2
+; CHECK-NEXT:    [[GEP1_3:%.*]] = getelementptr i32, i32* [[ARR1]], i32 3
+; CHECK-NEXT:    [[GEP2_0:%.*]] = getelementptr i32, i32* [[ARR2:%.*]], i32 0
+; CHECK-NEXT:    [[GEP2_1:%.*]] = getelementptr i32, i32* [[ARR2]], i32 1
+; CHECK-NEXT:    [[GEP2_2:%.*]] = getelementptr i32, i32* [[ARR2]], i32 2
+; CHECK-NEXT:    [[GEP2_3:%.*]] = getelementptr i32, i32* [[ARR2]], i32 3
+; CHECK-NEXT:    [[V0:%.*]] = load i32, i32* [[GEP1_0]]
+; CHECK-NEXT:    [[V1:%.*]] = load i32, i32* [[GEP1_1]]
+; CHECK-NEXT:    [[V2:%.*]] = load i32, i32* [[GEP1_2]]
+; CHECK-NEXT:    [[V3:%.*]] = load i32, i32* [[GEP1_3]]
+; CHECK-NEXT:    [[Y0:%.*]] = add nsw i32 [[A0:%.*]], 1146
+; CHECK-NEXT:    [[Y1:%.*]] = add nsw i32 [[A1:%.*]], 146
+; CHECK-NEXT:    [[Y2:%.*]] = add nsw i32 [[A2:%.*]], 42
+; CHECK-NEXT:    [[Y3:%.*]] = add nsw i32 [[A3:%.*]], 0
+; CHECK-NEXT:    [[RES0:%.*]] = add nsw i32 [[V0]], [[Y0]]
+; CHECK-NEXT:    [[RES1:%.*]] = add nsw i32 [[V1]], [[Y1]]
+; CHECK-NEXT:    [[RES2:%.*]] = sdiv i32 [[V2]], [[Y2]]
+; CHECK-NEXT:    [[RES3:%.*]] = add nsw i32 [[V3]], [[Y3]]
+; CHECK-NEXT:    store i32 [[RES0]], i32* [[GEP2_0]]
+; CHECK-NEXT:    store i32 [[RES1]], i32* [[GEP2_1]]
+; CHECK-NEXT:    store i32 [[RES2]], i32* [[GEP2_2]]
+; CHECK-NEXT:    store i32 [[RES3]], i32* [[GEP2_3]]
+; CHECK-NEXT:    ret void
+;
+entry:
+  %gep1.0 = getelementptr i32, i32* %arr1, i32 0
+  %gep1.1 = getelementptr i32, i32* %arr1, i32 1
+  %gep1.2 = getelementptr i32, i32* %arr1, i32 2
+  %gep1.3 = getelementptr i32, i32* %arr1, i32 3
+  %gep2.0 = getelementptr i32, i32* %arr2, i32 0
+  %gep2.1 = getelementptr i32, i32* %arr2, i32 1
+  %gep2.2 = getelementptr i32, i32* %arr2, i32 2
+  %gep2.3 = getelementptr i32, i32* %arr2, i32 3
+  %v0 = load i32, i32* %gep1.0
+  %v1 = load i32, i32* %gep1.1
+  %v2 = load i32, i32* %gep1.2
+  %v3 = load i32, i32* %gep1.3
+  %y0 = add nsw i32 %a0, 1146
+  %y1 = add nsw i32 %a1, 146
+  %y2 = add nsw i32 %a2, 42
+  ;; %y3 is zero if %a3 is zero
+  %y3 = add nsw i32 %a3, 0
+  %res0 = add nsw i32 %v0, %y0
+  %res1 = add nsw i32 %v1, %y1
+  ;; As such, doing alternate shuffling would be incorrect:
+  ;;   %vadd = add nsw %v[0-3], %y[0-3]
+  ;;   %vsdiv = sdiv %v[0-3], %y[0-3]
+  ;;   %result = shuffle %vadd, %vsdiv, <mask>
+  ;; would be illegal.
+  %res2 = sdiv i32 %v2, %y2
+  %res3 = add nsw i32 %v3, %y3
+  store i32 %res0, i32* %gep2.0
+  store i32 %res1, i32* %gep2.1
+  store i32 %res2, i32* %gep2.2
+  store i32 %res3, i32* %gep2.3
+  ret void
+}
+
+;; Similar test, but now div/rem is main opcode and not the alternate one. Same issue.
+define void @test_urem_add(i32 *%arr1, i32 *%arr2, i32 %a0, i32 %a1, i32 %a2, i32 %a3) {
+; CHECK-LABEL: @test_urem_add(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[GEP1_0:%.*]] = getelementptr i32, i32* [[ARR1:%.*]], i32 0
+; CHECK-NEXT:    [[GEP1_1:%.*]] = getelementptr i32, i32* [[ARR1]], i32 1
+; CHECK-NEXT:    [[GEP1_2:%.*]] = getelementptr i32, i32* [[ARR1]], i32 2
+; CHECK-NEXT:    [[GEP1_3:%.*]] = getelementptr i32, i32* [[ARR1]], i32 3
+; CHECK-NEXT:    [[GEP2_0:%.*]] = getelementptr i32, i32* [[ARR2:%.*]], i32 0
+; CHECK-NEXT:    [[GEP2_1:%.*]] = getelementptr i32, i32* [[ARR2]], i32 1
+; CHECK-NEXT:    [[GEP2_2:%.*]] = getelementptr i32, i32* [[ARR2]], i32 2
+; CHECK-NEXT:    [[GEP2_3:%.*]] = getelementptr i32, i32* [[ARR2]], i32 3
+; CHECK-NEXT:    [[V0:%.*]] = load i32, i32* [[GEP1_0]]
+; CHECK-NEXT:    [[V1:%.*]] = load i32, i32* [[GEP1_1]]
+; CHECK-NEXT:    [[V2:%.*]] = load i32, i32* [[GEP1_2]]
+; CHECK-NEXT:    [[V3:%.*]] = load i32, i32* [[GEP1_3]]
+; CHECK-NEXT:    [[Y0:%.*]] = add nsw i32 [[A0:%.*]], 1146
+; CHECK-NEXT:    [[Y1:%.*]] = add nsw i32 [[A1:%.*]], 146
+; CHECK-NEXT:    [[Y2:%.*]] = add nsw i32 [[A2:%.*]], 42
+; CHECK-NEXT:    [[Y3:%.*]] = add nsw i32 [[A3:%.*]], 0
+; CHECK-NEXT:    [[RES0:%.*]] = urem i32 [[V0]], [[Y0]]
+; CHECK-NEXT:    [[RES1:%.*]] = urem i32 [[V1]], [[Y1]]
+; CHECK-NEXT:    [[RES2:%.*]] = urem i32 [[V2]], [[Y2]]
+; CHECK-NEXT:    [[RES3:%.*]] = add nsw i32 [[V3]], [[Y3]]
+; CHECK-NEXT:    store i32 [[RES0]], i32* [[GEP2_0]]
+; CHECK-NEXT:    store i32 [[RES1]], i32* [[GEP2_1]]
+; CHECK-NEXT:    store i32 [[RES2]], i32* [[GEP2_2]]
+; CHECK-NEXT:    store i32 [[RES3]], i32* [[GEP2_3]]
+; CHECK-NEXT:    ret void
+;
+entry:
+  %gep1.0 = getelementptr i32, i32* %arr1, i32 0
+  %gep1.1 = getelementptr i32, i32* %arr1, i32 1
+  %gep1.2 = getelementptr i32, i32* %arr1, i32 2
+  %gep1.3 = getelementptr i32, i32* %arr1, i32 3
+  %gep2.0 = getelementptr i32, i32* %arr2, i32 0
+  %gep2.1 = getelementptr i32, i32* %arr2, i32 1
+  %gep2.2 = getelementptr i32, i32* %arr2, i32 2
+  %gep2.3 = getelementptr i32, i32* %arr2, i32 3
+  %v0 = load i32, i32* %gep1.0
+  %v1 = load i32, i32* %gep1.1
+  %v2 = load i32, i32* %gep1.2
+  %v3 = load i32, i32* %gep1.3
+  %y0 = add nsw i32 %a0, 1146
+  %y1 = add nsw i32 %a1, 146
+  %y2 = add nsw i32 %a2, 42
+  ;; %y3 is zero if %a3 is zero
+  %y3 = add nsw i32 %a3, 0
+  %res0 = urem i32 %v0, %y0
+  %res1 = urem i32 %v1, %y1
+  %res2 = urem i32 %v2, %y2
+  ;; As such, doing alternate shuffling would be incorrect:
+  ;;   %vurem = urem %v[0-3], %y[0-3]
+  ;;   %vadd = add nsw %v[0-3], %y[0-3]
+  ;;   %result = shuffle %vurem, %vadd, <mask>
+  ;; would be illegal.
+  %res3 = add nsw i32 %v3, %y3
+  store i32 %res0, i32* %gep2.0
+  store i32 %res1, i32* %gep2.1
+  store i32 %res2, i32* %gep2.2
+  store i32 %res3, i32* %gep2.3
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/pr16899.ll b/llvm/test/Transforms/SLPVectorizer/X86/pr16899.ll
index 5b91c3072..ceb48d98b 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/pr16899.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/pr16899.ll
@@ -36,7 +36,7 @@ do.body:                                          ; preds = %do.body, %entry
   br label %do.body
 }
 
-attributes #0 = { noreturn nounwind readonly "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf"="true" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noreturn nounwind readonly "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !0 = !{!"any pointer", !1}
 !1 = !{!"omnipotent char", !2}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/pr19657.ll b/llvm/test/Transforms/SLPVectorizer/X86/pr19657.ll
index 2bde319c1..39db608fc 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/pr19657.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/pr19657.ll
@@ -1,40 +1,40 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -S -mcpu=corei7-avx | FileCheck %s
-; RUN: opt < %s -basicaa -slp-vectorizer -slp-max-reg-size=128 -S -mcpu=corei7-avx | FileCheck %s --check-prefix=V128
+; RUN: opt < %s -basicaa -slp-vectorizer -S -mcpu=corei7-avx | FileCheck %s --check-prefixes=ANY,AVX
+; RUN: opt < %s -basicaa -slp-vectorizer -slp-max-reg-size=128 -S -mcpu=corei7-avx | FileCheck %s --check-prefixes=ANY,MAX128
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
-define void @foo(double* %x) {
-; CHECK-LABEL: @foo(
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds double, double* [[X:%.*]], i64 1
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds double, double* [[X]], i64 2
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds double, double* [[X]], i64 3
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast double* [[X]] to <4 x double>*
-; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x double>, <4 x double>* [[TMP4]], align 8
-; CHECK-NEXT:    [[TMP6:%.*]] = fadd <4 x double> [[TMP5]], [[TMP5]]
-; CHECK-NEXT:    [[TMP7:%.*]] = fadd <4 x double> [[TMP6]], [[TMP5]]
-; CHECK-NEXT:    [[TMP8:%.*]] = bitcast double* [[X]] to <4 x double>*
-; CHECK-NEXT:    store <4 x double> [[TMP7]], <4 x double>* [[TMP8]], align 8
-; CHECK-NEXT:    ret void
+define void @store_chains(double* %x) {
+; AVX-LABEL: @store_chains(
+; AVX-NEXT:    [[TMP1:%.*]] = getelementptr inbounds double, double* [[X:%.*]], i64 1
+; AVX-NEXT:    [[TMP2:%.*]] = getelementptr inbounds double, double* [[X]], i64 2
+; AVX-NEXT:    [[TMP3:%.*]] = getelementptr inbounds double, double* [[X]], i64 3
+; AVX-NEXT:    [[TMP4:%.*]] = bitcast double* [[X]] to <4 x double>*
+; AVX-NEXT:    [[TMP5:%.*]] = load <4 x double>, <4 x double>* [[TMP4]], align 8
+; AVX-NEXT:    [[TMP6:%.*]] = fadd <4 x double> [[TMP5]], [[TMP5]]
+; AVX-NEXT:    [[TMP7:%.*]] = fadd <4 x double> [[TMP6]], [[TMP5]]
+; AVX-NEXT:    [[TMP8:%.*]] = bitcast double* [[X]] to <4 x double>*
+; AVX-NEXT:    store <4 x double> [[TMP7]], <4 x double>* [[TMP8]], align 8
+; AVX-NEXT:    ret void
 ;
-; V128-LABEL: @foo(
-; V128-NEXT:    [[TMP1:%.*]] = getelementptr inbounds double, double* [[X:%.*]], i64 1
-; V128-NEXT:    [[TMP2:%.*]] = bitcast double* [[X]] to <2 x double>*
-; V128-NEXT:    [[TMP3:%.*]] = load <2 x double>, <2 x double>* [[TMP2]], align 8
-; V128-NEXT:    [[TMP4:%.*]] = fadd <2 x double> [[TMP3]], [[TMP3]]
-; V128-NEXT:    [[TMP5:%.*]] = fadd <2 x double> [[TMP4]], [[TMP3]]
-; V128-NEXT:    [[TMP6:%.*]] = bitcast double* [[X]] to <2 x double>*
-; V128-NEXT:    store <2 x double> [[TMP5]], <2 x double>* [[TMP6]], align 8
-; V128-NEXT:    [[TMP7:%.*]] = getelementptr inbounds double, double* [[X]], i64 2
-; V128-NEXT:    [[TMP8:%.*]] = getelementptr inbounds double, double* [[X]], i64 3
-; V128-NEXT:    [[TMP9:%.*]] = bitcast double* [[TMP7]] to <2 x double>*
-; V128-NEXT:    [[TMP10:%.*]] = load <2 x double>, <2 x double>* [[TMP9]], align 8
-; V128-NEXT:    [[TMP11:%.*]] = fadd <2 x double> [[TMP10]], [[TMP10]]
-; V128-NEXT:    [[TMP12:%.*]] = fadd <2 x double> [[TMP11]], [[TMP10]]
-; V128-NEXT:    [[TMP13:%.*]] = bitcast double* [[TMP7]] to <2 x double>*
-; V128-NEXT:    store <2 x double> [[TMP12]], <2 x double>* [[TMP13]], align 8
-; V128-NEXT:    ret void
+; MAX128-LABEL: @store_chains(
+; MAX128-NEXT:    [[TMP1:%.*]] = getelementptr inbounds double, double* [[X:%.*]], i64 1
+; MAX128-NEXT:    [[TMP2:%.*]] = bitcast double* [[X]] to <2 x double>*
+; MAX128-NEXT:    [[TMP3:%.*]] = load <2 x double>, <2 x double>* [[TMP2]], align 8
+; MAX128-NEXT:    [[TMP4:%.*]] = fadd <2 x double> [[TMP3]], [[TMP3]]
+; MAX128-NEXT:    [[TMP5:%.*]] = fadd <2 x double> [[TMP4]], [[TMP3]]
+; MAX128-NEXT:    [[TMP6:%.*]] = bitcast double* [[X]] to <2 x double>*
+; MAX128-NEXT:    store <2 x double> [[TMP5]], <2 x double>* [[TMP6]], align 8
+; MAX128-NEXT:    [[TMP7:%.*]] = getelementptr inbounds double, double* [[X]], i64 2
+; MAX128-NEXT:    [[TMP8:%.*]] = getelementptr inbounds double, double* [[X]], i64 3
+; MAX128-NEXT:    [[TMP9:%.*]] = bitcast double* [[TMP7]] to <2 x double>*
+; MAX128-NEXT:    [[TMP10:%.*]] = load <2 x double>, <2 x double>* [[TMP9]], align 8
+; MAX128-NEXT:    [[TMP11:%.*]] = fadd <2 x double> [[TMP10]], [[TMP10]]
+; MAX128-NEXT:    [[TMP12:%.*]] = fadd <2 x double> [[TMP11]], [[TMP10]]
+; MAX128-NEXT:    [[TMP13:%.*]] = bitcast double* [[TMP7]] to <2 x double>*
+; MAX128-NEXT:    store <2 x double> [[TMP12]], <2 x double>* [[TMP13]], align 8
+; MAX128-NEXT:    ret void
 ;
   %1 = load double, double* %x, align 8
   %2 = fadd double %1, %1
@@ -58,3 +58,45 @@ define void @foo(double* %x) {
   ret void
 }
 
+define void @store_chains_prefer_width_attr(double* %x) #0 {
+; ANY-LABEL: @store_chains_prefer_width_attr(
+; ANY-NEXT:    [[TMP1:%.*]] = getelementptr inbounds double, double* [[X:%.*]], i64 1
+; ANY-NEXT:    [[TMP2:%.*]] = bitcast double* [[X]] to <2 x double>*
+; ANY-NEXT:    [[TMP3:%.*]] = load <2 x double>, <2 x double>* [[TMP2]], align 8
+; ANY-NEXT:    [[TMP4:%.*]] = fadd <2 x double> [[TMP3]], [[TMP3]]
+; ANY-NEXT:    [[TMP5:%.*]] = fadd <2 x double> [[TMP4]], [[TMP3]]
+; ANY-NEXT:    [[TMP6:%.*]] = bitcast double* [[X]] to <2 x double>*
+; ANY-NEXT:    store <2 x double> [[TMP5]], <2 x double>* [[TMP6]], align 8
+; ANY-NEXT:    [[TMP7:%.*]] = getelementptr inbounds double, double* [[X]], i64 2
+; ANY-NEXT:    [[TMP8:%.*]] = getelementptr inbounds double, double* [[X]], i64 3
+; ANY-NEXT:    [[TMP9:%.*]] = bitcast double* [[TMP7]] to <2 x double>*
+; ANY-NEXT:    [[TMP10:%.*]] = load <2 x double>, <2 x double>* [[TMP9]], align 8
+; ANY-NEXT:    [[TMP11:%.*]] = fadd <2 x double> [[TMP10]], [[TMP10]]
+; ANY-NEXT:    [[TMP12:%.*]] = fadd <2 x double> [[TMP11]], [[TMP10]]
+; ANY-NEXT:    [[TMP13:%.*]] = bitcast double* [[TMP7]] to <2 x double>*
+; ANY-NEXT:    store <2 x double> [[TMP12]], <2 x double>* [[TMP13]], align 8
+; ANY-NEXT:    ret void
+;
+  %1 = load double, double* %x, align 8
+  %2 = fadd double %1, %1
+  %3 = fadd double %2, %1
+  store double %3, double* %x, align 8
+  %4 = getelementptr inbounds double, double* %x, i64 1
+  %5 = load double, double* %4, align 8
+  %6 = fadd double %5, %5
+  %7 = fadd double %6, %5
+  store double %7, double* %4, align 8
+  %8 = getelementptr inbounds double, double* %x, i64 2
+  %9 = load double, double* %8, align 8
+  %10 = fadd double %9, %9
+  %11 = fadd double %10, %9
+  store double %11, double* %8, align 8
+  %12 = getelementptr inbounds double, double* %x, i64 3
+  %13 = load double, double* %12, align 8
+  %14 = fadd double %13, %13
+  %15 = fadd double %14, %13
+  store double %15, double* %12, align 8
+  ret void
+}
+
+attributes #0 = { "prefer-vector-width"="128" }
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/pr35497.ll b/llvm/test/Transforms/SLPVectorizer/X86/pr35497.ll
index bdb37b28d..2c53b1afb 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/pr35497.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/pr35497.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
+; RUN: opt -slp-vectorizer -slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
 
 %class.1 = type { %class.2 }
 %class.2 = type { %"class.3" }
@@ -67,10 +67,13 @@ define void @pr35497() local_unnamed_addr #0 {
 ; CHECK-NEXT:    [[ARRAYIDX2_6:%.*]] = getelementptr inbounds [0 x i64], [0 x i64]* undef, i64 0, i64 0
 ; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i64* [[ARRAYIDX2_6]] to <2 x i64>*
 ; CHECK-NEXT:    store <2 x i64> [[TMP4]], <2 x i64>* [[TMP10]], align 1
-; CHECK-NEXT:    [[TMP11:%.*]] = lshr <2 x i64> [[TMP4]], <i64 6, i64 6>
-; CHECK-NEXT:    [[TMP12:%.*]] = add nuw nsw <2 x i64> [[TMP9]], [[TMP11]]
-; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i64* [[ARRAYIDX2_2]] to <2 x i64>*
-; CHECK-NEXT:    store <2 x i64> [[TMP12]], <2 x i64>* [[TMP13]], align 1
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <2 x i64> [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP12:%.*]] = insertelement <2 x i64> undef, i64 [[TMP11]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <2 x i64> [[TMP12]], i64 [[TMP5]], i32 1
+; CHECK-NEXT:    [[TMP14:%.*]] = lshr <2 x i64> [[TMP13]], <i64 6, i64 6>
+; CHECK-NEXT:    [[TMP15:%.*]] = add nuw nsw <2 x i64> [[TMP9]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast i64* [[ARRAYIDX2_2]] to <2 x i64>*
+; CHECK-NEXT:    store <2 x i64> [[TMP15]], <2 x i64>* [[TMP16]], align 1
 ; CHECK-NEXT:    ret void
 ;
 entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/pr42022.ll b/llvm/test/Transforms/SLPVectorizer/X86/pr42022.ll
new file mode 100644
index 000000000..2a41de10f
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/pr42022.ll
@@ -0,0 +1,278 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
+
+; See https://reviews.llvm.org/D70068 and https://reviews.llvm.org/D70587 for context
+
+; Checks that vector insertvalues into the struct become SLP seeds.
+define { <2 x float>, <2 x float> } @StructOfVectors(float *%Ptr) {
+; CHECK-LABEL: @StructOfVectors(
+; CHECK-NEXT:    [[GEP0:%.*]] = getelementptr inbounds float, float* [[PTR:%.*]], i64 0
+; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr inbounds float, float* [[PTR]], i64 1
+; CHECK-NEXT:    [[GEP2:%.*]] = getelementptr inbounds float, float* [[PTR]], i64 2
+; CHECK-NEXT:    [[GEP3:%.*]] = getelementptr inbounds float, float* [[PTR]], i64 3
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float* [[GEP0]] to <4 x float>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x float>, <4 x float>* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = fadd fast <4 x float> [[TMP2]], <float 1.100000e+01, float 1.200000e+01, float 1.300000e+01, float 1.400000e+01>
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[TMP3]], i32 0
+; CHECK-NEXT:    [[VECIN0:%.*]] = insertelement <2 x float> undef, float [[TMP4]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x float> [[TMP3]], i32 1
+; CHECK-NEXT:    [[VECIN1:%.*]] = insertelement <2 x float> [[VECIN0]], float [[TMP5]], i64 1
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[TMP3]], i32 2
+; CHECK-NEXT:    [[VECIN2:%.*]] = insertelement <2 x float> undef, float [[TMP6]], i64 0
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x float> [[TMP3]], i32 3
+; CHECK-NEXT:    [[VECIN3:%.*]] = insertelement <2 x float> [[VECIN2]], float [[TMP7]], i64 1
+; CHECK-NEXT:    [[RET0:%.*]] = insertvalue { <2 x float>, <2 x float> } undef, <2 x float> [[VECIN1]], 0
+; CHECK-NEXT:    [[RET1:%.*]] = insertvalue { <2 x float>, <2 x float> } [[RET0]], <2 x float> [[VECIN3]], 1
+; CHECK-NEXT:    ret { <2 x float>, <2 x float> } [[RET1]]
+;
+  %GEP0 = getelementptr inbounds float, float* %Ptr, i64 0
+  %L0 = load float, float * %GEP0
+  %GEP1 = getelementptr inbounds float, float* %Ptr, i64 1
+  %L1 = load float, float * %GEP1
+  %GEP2 = getelementptr inbounds float, float* %Ptr, i64 2
+  %L2 = load float, float * %GEP2
+  %GEP3 = getelementptr inbounds float, float* %Ptr, i64 3
+  %L3 = load float, float * %GEP3
+
+  %Fadd0 = fadd fast float %L0, 1.1e+01
+  %Fadd1 = fadd fast float %L1, 1.2e+01
+  %Fadd2 = fadd fast float %L2, 1.3e+01
+  %Fadd3 = fadd fast float %L3, 1.4e+01
+
+  %VecIn0 = insertelement <2 x float> undef, float %Fadd0, i64 0
+  %VecIn1 = insertelement <2 x float> %VecIn0, float %Fadd1, i64 1
+
+  %VecIn2 = insertelement <2 x float> undef, float %Fadd2, i64 0
+  %VecIn3 = insertelement <2 x float> %VecIn2, float %Fadd3, i64 1
+
+  %Ret0 = insertvalue {<2 x float>, <2 x float>} undef, <2 x float> %VecIn1, 0
+  %Ret1 = insertvalue {<2 x float>, <2 x float>} %Ret0, <2 x float> %VecIn3, 1
+  ret {<2 x float>, <2 x float>} %Ret1
+}
+
+%StructTy = type { float, float}
+
+define [2 x %StructTy] @ArrayOfStruct(float *%Ptr) {
+; CHECK-LABEL: @ArrayOfStruct(
+; CHECK-NEXT:    [[GEP0:%.*]] = getelementptr inbounds float, float* [[PTR:%.*]], i64 0
+; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr inbounds float, float* [[PTR]], i64 1
+; CHECK-NEXT:    [[GEP2:%.*]] = getelementptr inbounds float, float* [[PTR]], i64 2
+; CHECK-NEXT:    [[GEP3:%.*]] = getelementptr inbounds float, float* [[PTR]], i64 3
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float* [[GEP0]] to <4 x float>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x float>, <4 x float>* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = fadd fast <4 x float> [[TMP2]], <float 1.100000e+01, float 1.200000e+01, float 1.300000e+01, float 1.400000e+01>
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[TMP3]], i32 0
+; CHECK-NEXT:    [[STRUCTIN0:%.*]] = insertvalue [[STRUCTTY:%.*]] undef, float [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x float> [[TMP3]], i32 1
+; CHECK-NEXT:    [[STRUCTIN1:%.*]] = insertvalue [[STRUCTTY]] %StructIn0, float [[TMP5]], 1
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[TMP3]], i32 2
+; CHECK-NEXT:    [[STRUCTIN2:%.*]] = insertvalue [[STRUCTTY]] undef, float [[TMP6]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x float> [[TMP3]], i32 3
+; CHECK-NEXT:    [[STRUCTIN3:%.*]] = insertvalue [[STRUCTTY]] %StructIn2, float [[TMP7]], 1
+; CHECK-NEXT:    [[RET0:%.*]] = insertvalue [2 x %StructTy] undef, [[STRUCTTY]] %StructIn1, 0
+; CHECK-NEXT:    [[RET1:%.*]] = insertvalue [2 x %StructTy] [[RET0]], [[STRUCTTY]] %StructIn3, 1
+; CHECK-NEXT:    ret [2 x %StructTy] [[RET1]]
+;
+  %GEP0 = getelementptr inbounds float, float* %Ptr, i64 0
+  %L0 = load float, float * %GEP0
+  %GEP1 = getelementptr inbounds float, float* %Ptr, i64 1
+  %L1 = load float, float * %GEP1
+  %GEP2 = getelementptr inbounds float, float* %Ptr, i64 2
+  %L2 = load float, float * %GEP2
+  %GEP3 = getelementptr inbounds float, float* %Ptr, i64 3
+  %L3 = load float, float * %GEP3
+
+  %Fadd0 = fadd fast float %L0, 1.1e+01
+  %Fadd1 = fadd fast float %L1, 1.2e+01
+  %Fadd2 = fadd fast float %L2, 1.3e+01
+  %Fadd3 = fadd fast float %L3, 1.4e+01
+
+  %StructIn0 = insertvalue %StructTy undef, float %Fadd0, 0
+  %StructIn1 = insertvalue %StructTy %StructIn0, float %Fadd1, 1
+
+  %StructIn2 = insertvalue %StructTy undef, float %Fadd2, 0
+  %StructIn3 = insertvalue %StructTy %StructIn2, float %Fadd3, 1
+
+  %Ret0 = insertvalue [2 x %StructTy] undef, %StructTy %StructIn1, 0
+  %Ret1 = insertvalue [2 x %StructTy] %Ret0, %StructTy %StructIn3, 1
+  ret [2 x %StructTy] %Ret1
+}
+
+define {%StructTy, %StructTy} @StructOfStruct(float *%Ptr) {
+; CHECK-LABEL: @StructOfStruct(
+; CHECK-NEXT:    [[GEP0:%.*]] = getelementptr inbounds float, float* [[PTR:%.*]], i64 0
+; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr inbounds float, float* [[PTR]], i64 1
+; CHECK-NEXT:    [[GEP2:%.*]] = getelementptr inbounds float, float* [[PTR]], i64 2
+; CHECK-NEXT:    [[GEP3:%.*]] = getelementptr inbounds float, float* [[PTR]], i64 3
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float* [[GEP0]] to <4 x float>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x float>, <4 x float>* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = fadd fast <4 x float> [[TMP2]], <float 1.100000e+01, float 1.200000e+01, float 1.300000e+01, float 1.400000e+01>
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[TMP3]], i32 0
+; CHECK-NEXT:    [[STRUCTIN0:%.*]] = insertvalue [[STRUCTTY:%.*]] undef, float [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x float> [[TMP3]], i32 1
+; CHECK-NEXT:    [[STRUCTIN1:%.*]] = insertvalue [[STRUCTTY]] %StructIn0, float [[TMP5]], 1
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[TMP3]], i32 2
+; CHECK-NEXT:    [[STRUCTIN2:%.*]] = insertvalue [[STRUCTTY]] undef, float [[TMP6]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x float> [[TMP3]], i32 3
+; CHECK-NEXT:    [[STRUCTIN3:%.*]] = insertvalue [[STRUCTTY]] %StructIn2, float [[TMP7]], 1
+; CHECK-NEXT:    [[RET0:%.*]] = insertvalue { [[STRUCTTY]], [[STRUCTTY]] } undef, [[STRUCTTY]] %StructIn1, 0
+; CHECK-NEXT:    [[RET1:%.*]] = insertvalue { [[STRUCTTY]], [[STRUCTTY]] } [[RET0]], [[STRUCTTY]] %StructIn3, 1
+; CHECK-NEXT:    ret { [[STRUCTTY]], [[STRUCTTY]] } [[RET1]]
+;
+  %GEP0 = getelementptr inbounds float, float* %Ptr, i64 0
+  %L0 = load float, float * %GEP0
+  %GEP1 = getelementptr inbounds float, float* %Ptr, i64 1
+  %L1 = load float, float * %GEP1
+  %GEP2 = getelementptr inbounds float, float* %Ptr, i64 2
+  %L2 = load float, float * %GEP2
+  %GEP3 = getelementptr inbounds float, float* %Ptr, i64 3
+  %L3 = load float, float * %GEP3
+
+  %Fadd0 = fadd fast float %L0, 1.1e+01
+  %Fadd1 = fadd fast float %L1, 1.2e+01
+  %Fadd2 = fadd fast float %L2, 1.3e+01
+  %Fadd3 = fadd fast float %L3, 1.4e+01
+
+  %StructIn0 = insertvalue %StructTy undef, float %Fadd0, 0
+  %StructIn1 = insertvalue %StructTy %StructIn0, float %Fadd1, 1
+
+  %StructIn2 = insertvalue %StructTy undef, float %Fadd2, 0
+  %StructIn3 = insertvalue %StructTy %StructIn2, float %Fadd3, 1
+
+  %Ret0 = insertvalue {%StructTy, %StructTy} undef, %StructTy %StructIn1, 0
+  %Ret1 = insertvalue {%StructTy, %StructTy} %Ret0, %StructTy %StructIn3, 1
+  ret {%StructTy, %StructTy} %Ret1
+}
+
+define {%StructTy, float, float} @NonHomogeneousStruct(float *%Ptr) {
+; CHECK-LABEL: @NonHomogeneousStruct(
+; CHECK-NEXT:    [[GEP0:%.*]] = getelementptr inbounds float, float* [[PTR:%.*]], i64 0
+; CHECK-NEXT:    [[L0:%.*]] = load float, float* [[GEP0]]
+; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr inbounds float, float* [[PTR]], i64 1
+; CHECK-NEXT:    [[L1:%.*]] = load float, float* [[GEP1]]
+; CHECK-NEXT:    [[GEP2:%.*]] = getelementptr inbounds float, float* [[PTR]], i64 2
+; CHECK-NEXT:    [[L2:%.*]] = load float, float* [[GEP2]]
+; CHECK-NEXT:    [[GEP3:%.*]] = getelementptr inbounds float, float* [[PTR]], i64 3
+; CHECK-NEXT:    [[L3:%.*]] = load float, float* [[GEP3]]
+; CHECK-NEXT:    [[FADD0:%.*]] = fadd fast float [[L0]], 1.100000e+01
+; CHECK-NEXT:    [[FADD1:%.*]] = fadd fast float [[L1]], 1.200000e+01
+; CHECK-NEXT:    [[FADD2:%.*]] = fadd fast float [[L2]], 1.300000e+01
+; CHECK-NEXT:    [[FADD3:%.*]] = fadd fast float [[L3]], 1.400000e+01
+; CHECK-NEXT:    [[STRUCTIN0:%.*]] = insertvalue [[STRUCTTY:%.*]] undef, float [[FADD0]], 0
+; CHECK-NEXT:    [[STRUCTIN1:%.*]] = insertvalue [[STRUCTTY]] %StructIn0, float [[FADD1]], 1
+; CHECK-NEXT:    [[RET0:%.*]] = insertvalue { [[STRUCTTY]], float, float } undef, [[STRUCTTY]] %StructIn1, 0
+; CHECK-NEXT:    [[RET1:%.*]] = insertvalue { [[STRUCTTY]], float, float } [[RET0]], float [[FADD2]], 1
+; CHECK-NEXT:    [[RET2:%.*]] = insertvalue { [[STRUCTTY]], float, float } [[RET1]], float [[FADD3]], 2
+; CHECK-NEXT:    ret { [[STRUCTTY]], float, float } [[RET2]]
+;
+  %GEP0 = getelementptr inbounds float, float* %Ptr, i64 0
+  %L0 = load float, float * %GEP0
+  %GEP1 = getelementptr inbounds float, float* %Ptr, i64 1
+  %L1 = load float, float * %GEP1
+  %GEP2 = getelementptr inbounds float, float* %Ptr, i64 2
+  %L2 = load float, float * %GEP2
+  %GEP3 = getelementptr inbounds float, float* %Ptr, i64 3
+  %L3 = load float, float * %GEP3
+
+  %Fadd0 = fadd fast float %L0, 1.1e+01
+  %Fadd1 = fadd fast float %L1, 1.2e+01
+  %Fadd2 = fadd fast float %L2, 1.3e+01
+  %Fadd3 = fadd fast float %L3, 1.4e+01
+
+  %StructIn0 = insertvalue %StructTy undef, float %Fadd0, 0
+  %StructIn1 = insertvalue %StructTy %StructIn0, float %Fadd1, 1
+
+  %Ret0 = insertvalue {%StructTy, float, float} undef, %StructTy %StructIn1, 0
+  %Ret1 = insertvalue {%StructTy, float, float} %Ret0, float %Fadd2, 1
+  %Ret2 = insertvalue {%StructTy, float, float} %Ret1, float %Fadd3, 2
+  ret {%StructTy, float, float} %Ret2
+}
+
+%Struct1Ty = type { i16, i16 }
+%Struct2Ty = type { %Struct1Ty, %Struct1Ty}
+
+define {%Struct2Ty, %Struct2Ty} @StructOfStructOfStruct(i16 *%Ptr) {
+; CHECK-LABEL: @StructOfStructOfStruct(
+; CHECK-NEXT:    [[GEP0:%.*]] = getelementptr inbounds i16, i16* [[PTR:%.*]], i64 0
+; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr inbounds i16, i16* [[PTR]], i64 1
+; CHECK-NEXT:    [[GEP2:%.*]] = getelementptr inbounds i16, i16* [[PTR]], i64 2
+; CHECK-NEXT:    [[GEP3:%.*]] = getelementptr inbounds i16, i16* [[PTR]], i64 3
+; CHECK-NEXT:    [[GEP4:%.*]] = getelementptr inbounds i16, i16* [[PTR]], i64 4
+; CHECK-NEXT:    [[GEP5:%.*]] = getelementptr inbounds i16, i16* [[PTR]], i64 5
+; CHECK-NEXT:    [[GEP6:%.*]] = getelementptr inbounds i16, i16* [[PTR]], i64 6
+; CHECK-NEXT:    [[GEP7:%.*]] = getelementptr inbounds i16, i16* [[PTR]], i64 7
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i16* [[GEP0]] to <8 x i16>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, <8 x i16>* [[TMP1]], align 2
+; CHECK-NEXT:    [[TMP3:%.*]] = add <8 x i16> [[TMP2]], <i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8>
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <8 x i16> [[TMP3]], i32 0
+; CHECK-NEXT:    [[STRUCTIN0:%.*]] = insertvalue [[STRUCT1TY:%.*]] undef, i16 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <8 x i16> [[TMP3]], i32 1
+; CHECK-NEXT:    [[STRUCTIN1:%.*]] = insertvalue [[STRUCT1TY]] %StructIn0, i16 [[TMP5]], 1
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <8 x i16> [[TMP3]], i32 2
+; CHECK-NEXT:    [[STRUCTIN2:%.*]] = insertvalue [[STRUCT1TY]] undef, i16 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <8 x i16> [[TMP3]], i32 3
+; CHECK-NEXT:    [[STRUCTIN3:%.*]] = insertvalue [[STRUCT1TY]] %StructIn2, i16 [[TMP7]], 1
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <8 x i16> [[TMP3]], i32 4
+; CHECK-NEXT:    [[STRUCTIN4:%.*]] = insertvalue [[STRUCT1TY]] undef, i16 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <8 x i16> [[TMP3]], i32 5
+; CHECK-NEXT:    [[STRUCTIN5:%.*]] = insertvalue [[STRUCT1TY]] %StructIn4, i16 [[TMP9]], 1
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <8 x i16> [[TMP3]], i32 6
+; CHECK-NEXT:    [[STRUCTIN6:%.*]] = insertvalue [[STRUCT1TY]] undef, i16 [[TMP10]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <8 x i16> [[TMP3]], i32 7
+; CHECK-NEXT:    [[STRUCTIN7:%.*]] = insertvalue [[STRUCT1TY]] %StructIn6, i16 [[TMP11]], 1
+; CHECK-NEXT:    [[STRUCT2IN0:%.*]] = insertvalue [[STRUCT2TY:%.*]] undef, [[STRUCT1TY]] %StructIn1, 0
+; CHECK-NEXT:    [[STRUCT2IN1:%.*]] = insertvalue [[STRUCT2TY]] %Struct2In0, [[STRUCT1TY]] %StructIn3, 1
+; CHECK-NEXT:    [[STRUCT2IN2:%.*]] = insertvalue [[STRUCT2TY]] undef, [[STRUCT1TY]] %StructIn5, 0
+; CHECK-NEXT:    [[STRUCT2IN3:%.*]] = insertvalue [[STRUCT2TY]] %Struct2In2, [[STRUCT1TY]] %StructIn7, 1
+; CHECK-NEXT:    [[RET0:%.*]] = insertvalue { [[STRUCT2TY]], [[STRUCT2TY]] } undef, [[STRUCT2TY]] %Struct2In1, 0
+; CHECK-NEXT:    [[RET1:%.*]] = insertvalue { [[STRUCT2TY]], [[STRUCT2TY]] } [[RET0]], [[STRUCT2TY]] %Struct2In3, 1
+; CHECK-NEXT:    ret { [[STRUCT2TY]], [[STRUCT2TY]] } [[RET1]]
+;
+  %GEP0 = getelementptr inbounds i16, i16* %Ptr, i64 0
+  %L0 = load i16, i16 * %GEP0
+  %GEP1 = getelementptr inbounds i16, i16* %Ptr, i64 1
+  %L1 = load i16, i16 * %GEP1
+  %GEP2 = getelementptr inbounds i16, i16* %Ptr, i64 2
+  %L2 = load i16, i16 * %GEP2
+  %GEP3 = getelementptr inbounds i16, i16* %Ptr, i64 3
+  %L3 = load i16, i16 * %GEP3
+  %GEP4 = getelementptr inbounds i16, i16* %Ptr, i64 4
+  %L4 = load i16, i16 * %GEP4
+  %GEP5 = getelementptr inbounds i16, i16* %Ptr, i64 5
+  %L5 = load i16, i16 * %GEP5
+  %GEP6 = getelementptr inbounds i16, i16* %Ptr, i64 6
+  %L6 = load i16, i16 * %GEP6
+  %GEP7 = getelementptr inbounds i16, i16* %Ptr, i64 7
+  %L7 = load i16, i16 * %GEP7
+
+  %Fadd0 = add i16 %L0, 1
+  %Fadd1 = add i16 %L1, 2
+  %Fadd2 = add i16 %L2, 3
+  %Fadd3 = add i16 %L3, 4
+  %Fadd4 = add i16 %L4, 5
+  %Fadd5 = add i16 %L5, 6
+  %Fadd6 = add i16 %L6, 7
+  %Fadd7 = add i16 %L7, 8
+
+  %StructIn0 = insertvalue %Struct1Ty undef, i16 %Fadd0, 0
+  %StructIn1 = insertvalue %Struct1Ty %StructIn0, i16 %Fadd1, 1
+
+  %StructIn2 = insertvalue %Struct1Ty undef, i16 %Fadd2, 0
+  %StructIn3 = insertvalue %Struct1Ty %StructIn2, i16 %Fadd3, 1
+
+  %StructIn4 = insertvalue %Struct1Ty undef, i16 %Fadd4, 0
+  %StructIn5 = insertvalue %Struct1Ty %StructIn4, i16 %Fadd5, 1
+
+  %StructIn6 = insertvalue %Struct1Ty undef, i16 %Fadd6, 0
+  %StructIn7 = insertvalue %Struct1Ty %StructIn6, i16 %Fadd7, 1
+
+  %Struct2In0 = insertvalue %Struct2Ty undef, %Struct1Ty %StructIn1, 0
+  %Struct2In1 = insertvalue %Struct2Ty %Struct2In0, %Struct1Ty %StructIn3, 1
+
+  %Struct2In2 = insertvalue %Struct2Ty undef, %Struct1Ty %StructIn5, 0
+  %Struct2In3 = insertvalue %Struct2Ty %Struct2In2, %Struct1Ty %StructIn7, 1
+
+  %Ret0 = insertvalue {%Struct2Ty, %Struct2Ty} undef, %Struct2Ty %Struct2In1, 0
+  %Ret1 = insertvalue {%Struct2Ty, %Struct2Ty} %Ret0, %Struct2Ty %Struct2In3, 1
+  ret {%Struct2Ty, %Struct2Ty} %Ret1
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reassociated-loads.ll b/llvm/test/Transforms/SLPVectorizer/X86/reassociated-loads.ll
index 13bbe5e57..0b8c72b35 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reassociated-loads.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reassociated-loads.ll
@@ -5,36 +5,6 @@ define signext i8 @Foo(<32 x i8>* %__v) {
 ; CHECK-LABEL: @Foo(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <32 x i8>, <32 x i8>* [[__V:%.*]], align 32
-; CHECK-NEXT:    [[ADD_I_1_I:%.*]] = add i8 undef, undef
-; CHECK-NEXT:    [[ADD_I_2_I:%.*]] = add i8 [[ADD_I_1_I]], undef
-; CHECK-NEXT:    [[ADD_I_3_I:%.*]] = add i8 [[ADD_I_2_I]], undef
-; CHECK-NEXT:    [[ADD_I_4_I:%.*]] = add i8 [[ADD_I_3_I]], undef
-; CHECK-NEXT:    [[ADD_I_5_I:%.*]] = add i8 [[ADD_I_4_I]], undef
-; CHECK-NEXT:    [[ADD_I_6_I:%.*]] = add i8 [[ADD_I_5_I]], undef
-; CHECK-NEXT:    [[ADD_I_7_I:%.*]] = add i8 [[ADD_I_6_I]], undef
-; CHECK-NEXT:    [[ADD_I_8_I:%.*]] = add i8 [[ADD_I_7_I]], undef
-; CHECK-NEXT:    [[ADD_I_9_I:%.*]] = add i8 [[ADD_I_8_I]], undef
-; CHECK-NEXT:    [[ADD_I_10_I:%.*]] = add i8 [[ADD_I_9_I]], undef
-; CHECK-NEXT:    [[ADD_I_11_I:%.*]] = add i8 [[ADD_I_10_I]], undef
-; CHECK-NEXT:    [[ADD_I_12_I:%.*]] = add i8 [[ADD_I_11_I]], undef
-; CHECK-NEXT:    [[ADD_I_13_I:%.*]] = add i8 [[ADD_I_12_I]], undef
-; CHECK-NEXT:    [[ADD_I_14_I:%.*]] = add i8 [[ADD_I_13_I]], undef
-; CHECK-NEXT:    [[ADD_I_15_I:%.*]] = add i8 [[ADD_I_14_I]], undef
-; CHECK-NEXT:    [[ADD_I_16_I:%.*]] = add i8 [[ADD_I_15_I]], undef
-; CHECK-NEXT:    [[ADD_I_17_I:%.*]] = add i8 [[ADD_I_16_I]], undef
-; CHECK-NEXT:    [[ADD_I_18_I:%.*]] = add i8 [[ADD_I_17_I]], undef
-; CHECK-NEXT:    [[ADD_I_19_I:%.*]] = add i8 [[ADD_I_18_I]], undef
-; CHECK-NEXT:    [[ADD_I_20_I:%.*]] = add i8 [[ADD_I_19_I]], undef
-; CHECK-NEXT:    [[ADD_I_21_I:%.*]] = add i8 [[ADD_I_20_I]], undef
-; CHECK-NEXT:    [[ADD_I_22_I:%.*]] = add i8 [[ADD_I_21_I]], undef
-; CHECK-NEXT:    [[ADD_I_23_I:%.*]] = add i8 [[ADD_I_22_I]], undef
-; CHECK-NEXT:    [[ADD_I_24_I:%.*]] = add i8 [[ADD_I_23_I]], undef
-; CHECK-NEXT:    [[ADD_I_25_I:%.*]] = add i8 [[ADD_I_24_I]], undef
-; CHECK-NEXT:    [[ADD_I_26_I:%.*]] = add i8 [[ADD_I_25_I]], undef
-; CHECK-NEXT:    [[ADD_I_27_I:%.*]] = add i8 [[ADD_I_26_I]], undef
-; CHECK-NEXT:    [[ADD_I_28_I:%.*]] = add i8 [[ADD_I_27_I]], undef
-; CHECK-NEXT:    [[ADD_I_29_I:%.*]] = add i8 [[ADD_I_28_I]], undef
-; CHECK-NEXT:    [[ADD_I_30_I:%.*]] = add i8 [[ADD_I_29_I]], undef
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <32 x i8> [[TMP0]], <32 x i8> undef, <32 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <32 x i8> [[TMP0]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <32 x i8> [[BIN_RDX]], <32 x i8> undef, <32 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -46,7 +16,6 @@ define signext i8 @Foo(<32 x i8>* %__v) {
 ; CHECK-NEXT:    [[RDX_SHUF7:%.*]] = shufflevector <32 x i8> [[BIN_RDX6]], <32 x i8> undef, <32 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX8:%.*]] = add <32 x i8> [[BIN_RDX6]], [[RDX_SHUF7]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <32 x i8> [[BIN_RDX8]], i32 0
-; CHECK-NEXT:    [[ADD_I_31_I:%.*]] = add i8 [[ADD_I_30_I]], undef
 ; CHECK-NEXT:    ret i8 [[TMP1]]
 ;
 entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reduction.ll b/llvm/test/Transforms/SLPVectorizer/X86/reduction.ll
index e9f8e7f7c..9060a1e32 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reduction.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reduction.ll
@@ -68,3 +68,87 @@ for.end:                                          ; preds = %for.cond.for.end_cr
   ret i32 %sum.0.lcssa
 }
 
+; FIXME: PR43948 - https://bugs.llvm.org/show_bug.cgi?id=43948
+; The extra use of a non-vectorized element of a reduction must not be killed.
+
+define i32 @horiz_max_multiple_uses([32 x i32]* %x, i32* %p) {
+; CHECK-LABEL: @horiz_max_multiple_uses(
+; CHECK-NEXT:    [[X0:%.*]] = getelementptr [32 x i32], [32 x i32]* [[X:%.*]], i64 0, i64 0
+; CHECK-NEXT:    [[X4:%.*]] = getelementptr [32 x i32], [32 x i32]* [[X]], i64 0, i64 4
+; CHECK-NEXT:    [[X5:%.*]] = getelementptr [32 x i32], [32 x i32]* [[X]], i64 0, i64 5
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32* [[X0]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP1]], align 4
+; CHECK-NEXT:    [[T4:%.*]] = load i32, i32* [[X4]]
+; CHECK-NEXT:    [[T5:%.*]] = load i32, i32* [[X5]]
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = icmp sgt <4 x i32> [[TMP2]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x i32> [[TMP2]], <4 x i32> [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x i32> [[RDX_MINMAX_SELECT]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_MINMAX_CMP2:%.*]] = icmp sgt <4 x i32> [[RDX_MINMAX_SELECT]], [[RDX_SHUF1]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT3:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP2]], <4 x i32> [[RDX_MINMAX_SELECT]], <4 x i32> [[RDX_SHUF1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x i32> [[RDX_MINMAX_SELECT3]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp sgt i32 [[TMP3]], [[T4]]
+; CHECK-NEXT:    [[TMP5:%.*]] = select i1 [[TMP4]], i32 [[TMP3]], i32 [[T4]]
+; CHECK-NEXT:    [[C012345:%.*]] = icmp sgt i32 [[TMP5]], [[T5]]
+; CHECK-NEXT:    [[T17:%.*]] = select i1 [[C012345]], i32 [[TMP5]], i32 [[T5]]
+; CHECK-NEXT:    [[THREE_OR_FOUR:%.*]] = select i1 [[TMP4]], i32 3, i32 4
+; CHECK-NEXT:    store i32 [[THREE_OR_FOUR]], i32* [[P:%.*]], align 8
+; CHECK-NEXT:    ret i32 [[T17]]
+;
+  %x0 = getelementptr [32 x i32], [32 x i32]* %x, i64 0, i64 0
+  %x1 = getelementptr [32 x i32], [32 x i32]* %x, i64 0, i64 1
+  %x2 = getelementptr [32 x i32], [32 x i32]* %x, i64 0, i64 2
+  %x3 = getelementptr [32 x i32], [32 x i32]* %x, i64 0, i64 3
+  %x4 = getelementptr [32 x i32], [32 x i32]* %x, i64 0, i64 4
+  %x5 = getelementptr [32 x i32], [32 x i32]* %x, i64 0, i64 5
+
+  %t0 = load i32, i32* %x0
+  %t1 = load i32, i32* %x1
+  %t2 = load i32, i32* %x2
+  %t3 = load i32, i32* %x3
+  %t4 = load i32, i32* %x4
+  %t5 = load i32, i32* %x5
+
+  %c01 = icmp sgt i32 %t0, %t1
+  %s5 = select i1 %c01, i32 %t0, i32 %t1
+  %c012 = icmp sgt i32 %s5, %t2
+  %t8 = select i1 %c012, i32 %s5, i32 %t2
+  %c0123 = icmp sgt i32 %t8, %t3
+  %rdx4 = select i1 %c0123, i32 %t8, i32 %t3
+  %MAX_ROOT_CMP = icmp sgt i32 %rdx4, %t4
+  %MAX_ROOT_SEL = select i1 %MAX_ROOT_CMP, i32 %rdx4, i32 %t4
+  %c012345 = icmp sgt i32 %MAX_ROOT_SEL, %t5
+  %t17 = select i1 %c012345, i32 %MAX_ROOT_SEL, i32 %t5
+  %three_or_four = select i1 %MAX_ROOT_CMP, i32 3, i32 4
+  store i32 %three_or_four, i32* %p, align 8
+  ret i32 %t17
+}
+
+; FIXME: This is a miscompile (see the undef operand) and/or test for invalid IR.
+
+define i1 @bad_insertpoint_rdx([8 x i32]* %p) #0 {
+; CHECK-LABEL: @bad_insertpoint_rdx(
+; CHECK-NEXT:    [[ARRAYIDX22:%.*]] = getelementptr inbounds [8 x i32], [8 x i32]* [[P:%.*]], i64 0, i64 0
+; CHECK-NEXT:    [[T0:%.*]] = load i32, i32* [[ARRAYIDX22]], align 16
+; CHECK-NEXT:    [[CMP23:%.*]] = icmp sgt i32 [[T0]], 0
+; CHECK-NEXT:    [[SPEC_SELECT:%.*]] = select i1 [[CMP23]], i32 [[T0]], i32 0
+; CHECK-NEXT:    [[ARRAYIDX22_1:%.*]] = getelementptr inbounds [8 x i32], [8 x i32]* [[P]], i64 0, i64 1
+; CHECK-NEXT:    [[T1:%.*]] = load i32, i32* [[ARRAYIDX22_1]], align 4
+; CHECK-NEXT:    [[CMP23_1:%.*]] = icmp sgt i32 [[T1]], [[SPEC_SELECT]]
+; CHECK-NEXT:    [[SPEC_STORE_SELECT87:%.*]] = zext i1 [[CMP23_1]] to i32
+; CHECK-NEXT:    [[SPEC_SELECT88:%.*]] = select i1 [[CMP23_1]], i32 [[T1]], i32 [[SPEC_SELECT]]
+; CHECK-NEXT:    [[CMP23_2:%.*]] = icmp sgt i32 [[SPEC_STORE_SELECT87]], [[SPEC_SELECT88]]
+; CHECK-NEXT:    ret i1 [[CMP23_2]]
+;
+  %arrayidx22 = getelementptr inbounds [8 x i32], [8 x i32]* %p, i64 0, i64 0
+  %t0 = load i32, i32* %arrayidx22, align 16
+  %cmp23 = icmp sgt i32 %t0, 0
+  %spec.select = select i1 %cmp23, i32 %t0, i32 0
+  %arrayidx22.1 = getelementptr inbounds [8 x i32], [8 x i32]* %p, i64 0, i64 1
+  %t1 = load i32, i32* %arrayidx22.1, align 4
+  %cmp23.1 = icmp sgt i32 %t1, %spec.select
+  %spec.store.select87 = zext i1 %cmp23.1 to i32
+  %spec.select88 = select i1 %cmp23.1, i32 %t1, i32 %spec.select
+  %cmp23.2 = icmp sgt i32 %spec.store.select87, %spec.select88
+  ret i1 %cmp23.2
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reduction2.ll b/llvm/test/Transforms/SLPVectorizer/X86/reduction2.ll
index 87a6af792..b5f433549 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reduction2.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reduction2.ll
@@ -7,7 +7,8 @@ target triple = "i386-apple-macosx10.8.0"
 define double @foo(double* nocapture %D) {
 ; CHECK-LABEL: @foo(
 ; CHECK-NEXT:    br label [[TMP1:%.*]]
-; CHECK:         [[I_02:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP12:%.*]], [[TMP1]] ]
+; CHECK:       1:
+; CHECK-NEXT:    [[I_02:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP12:%.*]], [[TMP1]] ]
 ; CHECK-NEXT:    [[SUM_01:%.*]] = phi double [ 0.000000e+00, [[TMP0]] ], [ [[TMP11:%.*]], [[TMP1]] ]
 ; CHECK-NEXT:    [[TMP2:%.*]] = shl nsw i32 [[I_02]], 1
 ; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds double, double* [[D:%.*]], i32 [[TMP2]]
@@ -22,7 +23,8 @@ define double @foo(double* nocapture %D) {
 ; CHECK-NEXT:    [[TMP12]] = add nsw i32 [[I_02]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[TMP12]], 100
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label [[TMP13:%.*]], label [[TMP1]]
-; CHECK:         ret double [[TMP11]]
+; CHECK:       13:
+; CHECK-NEXT:    ret double [[TMP11]]
 ;
   br label %1
 
@@ -49,3 +51,114 @@ define double @foo(double* nocapture %D) {
   ret double %9
 }
 
+define i1 @two_wide_fcmp_reduction(<2 x double> %a0) {
+; CHECK-LABEL: @two_wide_fcmp_reduction(
+; CHECK-NEXT:    [[A:%.*]] = fcmp ogt <2 x double> [[A0:%.*]], <double 1.000000e+00, double 1.000000e+00>
+; CHECK-NEXT:    [[B:%.*]] = extractelement <2 x i1> [[A]], i32 0
+; CHECK-NEXT:    [[C:%.*]] = extractelement <2 x i1> [[A]], i32 1
+; CHECK-NEXT:    [[D:%.*]] = and i1 [[B]], [[C]]
+; CHECK-NEXT:    ret i1 [[D]]
+;
+  %a = fcmp ogt <2 x double> %a0, <double 1.0, double 1.0>
+  %b = extractelement <2 x i1> %a, i32 0
+  %c = extractelement <2 x i1> %a, i32 1
+  %d = and i1 %b, %c
+  ret i1 %d
+}
+
+define double @fadd_reduction(<2 x double> %a0) {
+; CHECK-LABEL: @fadd_reduction(
+; CHECK-NEXT:    [[A:%.*]] = fadd fast <2 x double> [[A0:%.*]], <double 1.000000e+00, double 1.000000e+00>
+; CHECK-NEXT:    [[B:%.*]] = extractelement <2 x double> [[A]], i32 0
+; CHECK-NEXT:    [[C:%.*]] = extractelement <2 x double> [[A]], i32 1
+; CHECK-NEXT:    [[D:%.*]] = fadd fast double [[B]], [[C]]
+; CHECK-NEXT:    ret double [[D]]
+;
+  %a = fadd fast <2 x double> %a0, <double 1.000000e+00, double 1.000000e+00>
+  %b = extractelement <2 x double> %a, i32 0
+  %c = extractelement <2 x double> %a, i32 1
+  %d = fadd fast double %b, %c
+  ret double %d
+}
+
+; PR43745 https://bugs.llvm.org/show_bug.cgi?id=43745
+
+define i1 @fcmp_lt_gt(double %a, double %b, double %c) {
+; CHECK-LABEL: @fcmp_lt_gt(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[FNEG:%.*]] = fneg double [[B:%.*]]
+; CHECK-NEXT:    [[MUL:%.*]] = fmul double [[A:%.*]], 2.000000e+00
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x double> undef, double [[FNEG]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> [[TMP0]], double [[C:%.*]], i32 1
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> undef, double [[C]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x double> [[TMP2]], double [[B]], i32 1
+; CHECK-NEXT:    [[TMP4:%.*]] = fsub <2 x double> [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <2 x double> undef, double [[MUL]], i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x double> [[TMP5]], double [[MUL]], i32 1
+; CHECK-NEXT:    [[TMP7:%.*]] = fdiv <2 x double> [[TMP4]], [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x double> [[TMP7]], i32 1
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp olt double [[TMP8]], 0x3EB0C6F7A0B5ED8D
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <2 x double> [[TMP7]], i32 0
+; CHECK-NEXT:    [[CMP4:%.*]] = fcmp olt double [[TMP9]], 0x3EB0C6F7A0B5ED8D
+; CHECK-NEXT:    [[OR_COND:%.*]] = and i1 [[CMP]], [[CMP4]]
+; CHECK-NEXT:    br i1 [[OR_COND]], label [[CLEANUP:%.*]], label [[LOR_LHS_FALSE:%.*]]
+; CHECK:       lor.lhs.false:
+; CHECK-NEXT:    [[TMP10:%.*]] = fcmp ule <2 x double> [[TMP7]], <double 1.000000e+00, double 1.000000e+00>
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <2 x i1> [[TMP10]], i32 0
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <2 x i1> [[TMP10]], i32 1
+; CHECK-NEXT:    [[NOT_OR_COND9:%.*]] = or i1 [[TMP11]], [[TMP12]]
+; CHECK-NEXT:    ret i1 [[NOT_OR_COND9]]
+; CHECK:       cleanup:
+; CHECK-NEXT:    ret i1 false
+;
+entry:
+  %fneg = fneg double %b
+  %add = fsub double %c, %b
+  %mul = fmul double %a, 2.000000e+00
+  %div = fdiv double %add, %mul
+  %sub = fsub double %fneg, %c
+  %div3 = fdiv double %sub, %mul
+  %cmp = fcmp olt double %div, 0x3EB0C6F7A0B5ED8D
+  %cmp4 = fcmp olt double %div3, 0x3EB0C6F7A0B5ED8D
+  %or.cond = and i1 %cmp, %cmp4
+  br i1 %or.cond, label %cleanup, label %lor.lhs.false
+
+lor.lhs.false:
+  %cmp5 = fcmp ule double %div, 1.000000e+00
+  %cmp7 = fcmp ule double %div3, 1.000000e+00
+  %not.or.cond9 = or i1 %cmp7, %cmp5
+  ret i1 %not.or.cond9
+
+cleanup:
+  ret i1 false
+}
+
+define i1 @fcmp_lt(double %a, double %b, double %c) {
+; CHECK-LABEL: @fcmp_lt(
+; CHECK-NEXT:    [[FNEG:%.*]] = fneg double [[B:%.*]]
+; CHECK-NEXT:    [[MUL:%.*]] = fmul double [[A:%.*]], 2.000000e+00
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> undef, double [[FNEG]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> [[TMP1]], double [[C:%.*]], i32 1
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x double> undef, double [[C]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <2 x double> [[TMP3]], double [[B]], i32 1
+; CHECK-NEXT:    [[TMP5:%.*]] = fsub <2 x double> [[TMP2]], [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x double> undef, double [[MUL]], i32 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <2 x double> [[TMP6]], double [[MUL]], i32 1
+; CHECK-NEXT:    [[TMP8:%.*]] = fdiv <2 x double> [[TMP5]], [[TMP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = fcmp uge <2 x double> [[TMP8]], <double 0x3EB0C6F7A0B5ED8D, double 0x3EB0C6F7A0B5ED8D>
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <2 x i1> [[TMP9]], i32 0
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <2 x i1> [[TMP9]], i32 1
+; CHECK-NEXT:    [[NOT_OR_COND:%.*]] = or i1 [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    ret i1 [[NOT_OR_COND]]
+;
+  %fneg = fneg double %b
+  %add = fsub double %c, %b
+  %mul = fmul double %a, 2.000000e+00
+  %div = fdiv double %add, %mul
+  %sub = fsub double %fneg, %c
+  %div3 = fdiv double %sub, %mul
+  %cmp = fcmp uge double %div, 0x3EB0C6F7A0B5ED8D
+  %cmp4 = fcmp uge double %div3, 0x3EB0C6F7A0B5ED8D
+  %not.or.cond = or i1 %cmp4, %cmp
+  ret i1 %not.or.cond
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reduction_loads.ll b/llvm/test/Transforms/SLPVectorizer/X86/reduction_loads.ll
index 56539ab92..15ede8156 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reduction_loads.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reduction_loads.ll
@@ -35,13 +35,6 @@ define i32 @test(i32* nocapture readonly %p) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32* [[P]] to <8 x i32>*
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* [[TMP0]], align 4
 ; CHECK-NEXT:    [[TMP2:%.*]] = mul <8 x i32> [[TMP1]], <i32 42, i32 42, i32 42, i32 42, i32 42, i32 42, i32 42, i32 42>
-; CHECK-NEXT:    [[ADD:%.*]] = add i32 undef, [[SUM]]
-; CHECK-NEXT:    [[ADD_1:%.*]] = add i32 undef, [[ADD]]
-; CHECK-NEXT:    [[ADD_2:%.*]] = add i32 undef, [[ADD_1]]
-; CHECK-NEXT:    [[ADD_3:%.*]] = add i32 undef, [[ADD_2]]
-; CHECK-NEXT:    [[ADD_4:%.*]] = add i32 undef, [[ADD_3]]
-; CHECK-NEXT:    [[ADD_5:%.*]] = add i32 undef, [[ADD_4]]
-; CHECK-NEXT:    [[ADD_6:%.*]] = add i32 undef, [[ADD_5]]
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x i32> [[TMP2]], <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <8 x i32> [[TMP2]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x i32> [[BIN_RDX]], <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -50,7 +43,6 @@ define i32 @test(i32* nocapture readonly %p) {
 ; CHECK-NEXT:    [[BIN_RDX4:%.*]] = add <8 x i32> [[BIN_RDX2]], [[RDX_SHUF3]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <8 x i32> [[BIN_RDX4]], i32 0
 ; CHECK-NEXT:    [[OP_EXTRA]] = add i32 [[TMP3]], [[SUM]]
-; CHECK-NEXT:    [[ADD_7:%.*]] = add i32 undef, [[ADD_6]]
 ; CHECK-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[FOR_BODY]]
 ; CHECK:       for.end:
 ; CHECK-NEXT:    ret i32 [[OP_EXTRA]]
@@ -138,13 +130,6 @@ define i32 @test2(i32* nocapture readonly %p, i32* nocapture readonly %q) {
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i32* [[Q]] to <8 x i32>*
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, <8 x i32>* [[TMP2]], align 4
 ; CHECK-NEXT:    [[TMP4:%.*]] = mul <8 x i32> [[TMP1]], [[TMP3]]
-; CHECK-NEXT:    [[ADD:%.*]] = add i32 undef, [[SUM]]
-; CHECK-NEXT:    [[ADD_1:%.*]] = add i32 undef, [[ADD]]
-; CHECK-NEXT:    [[ADD_2:%.*]] = add i32 undef, [[ADD_1]]
-; CHECK-NEXT:    [[ADD_3:%.*]] = add i32 undef, [[ADD_2]]
-; CHECK-NEXT:    [[ADD_4:%.*]] = add i32 undef, [[ADD_3]]
-; CHECK-NEXT:    [[ADD_5:%.*]] = add i32 undef, [[ADD_4]]
-; CHECK-NEXT:    [[ADD_6:%.*]] = add i32 undef, [[ADD_5]]
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x i32> [[TMP4]], <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <8 x i32> [[TMP4]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x i32> [[BIN_RDX]], <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -153,7 +138,6 @@ define i32 @test2(i32* nocapture readonly %p, i32* nocapture readonly %q) {
 ; CHECK-NEXT:    [[BIN_RDX4:%.*]] = add <8 x i32> [[BIN_RDX2]], [[RDX_SHUF3]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <8 x i32> [[BIN_RDX4]], i32 0
 ; CHECK-NEXT:    [[OP_EXTRA]] = add i32 [[TMP5]], [[SUM]]
-; CHECK-NEXT:    [[ADD_7:%.*]] = add i32 undef, [[ADD_6]]
 ; CHECK-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[FOR_BODY]]
 ; CHECK:       for.end:
 ; CHECK-NEXT:    ret i32 [[OP_EXTRA]]
@@ -258,13 +242,6 @@ define i32 @test3(i32* nocapture readonly %p, i32* nocapture readonly %q) {
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i32* [[Q]] to <8 x i32>*
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, <8 x i32>* [[TMP2]], align 4
 ; CHECK-NEXT:    [[TMP4:%.*]] = mul <8 x i32> [[REORDER_SHUFFLE]], [[TMP3]]
-; CHECK-NEXT:    [[ADD:%.*]] = add i32 undef, [[SUM]]
-; CHECK-NEXT:    [[ADD_1:%.*]] = add i32 undef, [[ADD]]
-; CHECK-NEXT:    [[ADD_2:%.*]] = add i32 undef, [[ADD_1]]
-; CHECK-NEXT:    [[ADD_3:%.*]] = add i32 undef, [[ADD_2]]
-; CHECK-NEXT:    [[ADD_4:%.*]] = add i32 undef, [[ADD_3]]
-; CHECK-NEXT:    [[ADD_5:%.*]] = add i32 undef, [[ADD_4]]
-; CHECK-NEXT:    [[ADD_6:%.*]] = add i32 undef, [[ADD_5]]
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x i32> [[TMP4]], <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <8 x i32> [[TMP4]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x i32> [[BIN_RDX]], <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -273,7 +250,6 @@ define i32 @test3(i32* nocapture readonly %p, i32* nocapture readonly %q) {
 ; CHECK-NEXT:    [[BIN_RDX4:%.*]] = add <8 x i32> [[BIN_RDX2]], [[RDX_SHUF3]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <8 x i32> [[BIN_RDX4]], i32 0
 ; CHECK-NEXT:    [[OP_EXTRA]] = add i32 [[TMP5]], [[SUM]]
-; CHECK-NEXT:    [[ADD_7:%.*]] = add i32 undef, [[ADD_6]]
 ; CHECK-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[FOR_BODY]]
 ; CHECK:       for.end:
 ; CHECK-NEXT:    ret i32 [[OP_EXTRA]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reduction_unrolled.ll b/llvm/test/Transforms/SLPVectorizer/X86/reduction_unrolled.ll
index 4dd40876a..7fdc95392 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reduction_unrolled.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reduction_unrolled.ll
@@ -26,12 +26,6 @@ define i32 @test_add(i32* nocapture readonly %p) {
 ; CHECK-NEXT:    [[ARRAYIDX_7:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 7
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32* [[P]] to <8 x i32>*
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* [[TMP0]], align 4
-; CHECK-NEXT:    [[MUL_18:%.*]] = add i32 undef, undef
-; CHECK-NEXT:    [[MUL_29:%.*]] = add i32 undef, [[MUL_18]]
-; CHECK-NEXT:    [[MUL_310:%.*]] = add i32 undef, [[MUL_29]]
-; CHECK-NEXT:    [[MUL_411:%.*]] = add i32 undef, [[MUL_310]]
-; CHECK-NEXT:    [[MUL_512:%.*]] = add i32 undef, [[MUL_411]]
-; CHECK-NEXT:    [[MUL_613:%.*]] = add i32 undef, [[MUL_512]]
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <8 x i32> [[TMP1]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x i32> [[BIN_RDX]], <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -39,7 +33,6 @@ define i32 @test_add(i32* nocapture readonly %p) {
 ; CHECK-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <8 x i32> [[BIN_RDX2]], <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX4:%.*]] = add <8 x i32> [[BIN_RDX2]], [[RDX_SHUF3]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <8 x i32> [[BIN_RDX4]], i32 0
-; CHECK-NEXT:    [[MUL_714:%.*]] = add i32 undef, [[MUL_613]]
 ; CHECK-NEXT:    ret i32 [[TMP2]]
 ;
 entry:
@@ -147,12 +140,6 @@ define i32 @test_and(i32* nocapture readonly %p) {
 ; CHECK-NEXT:    [[ARRAYIDX_7:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 7
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32* [[P]] to <8 x i32>*
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* [[TMP0]], align 4
-; CHECK-NEXT:    [[MUL_18:%.*]] = and i32 undef, undef
-; CHECK-NEXT:    [[MUL_29:%.*]] = and i32 undef, [[MUL_18]]
-; CHECK-NEXT:    [[MUL_310:%.*]] = and i32 undef, [[MUL_29]]
-; CHECK-NEXT:    [[MUL_411:%.*]] = and i32 undef, [[MUL_310]]
-; CHECK-NEXT:    [[MUL_512:%.*]] = and i32 undef, [[MUL_411]]
-; CHECK-NEXT:    [[MUL_613:%.*]] = and i32 undef, [[MUL_512]]
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = and <8 x i32> [[TMP1]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x i32> [[BIN_RDX]], <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -160,7 +147,6 @@ define i32 @test_and(i32* nocapture readonly %p) {
 ; CHECK-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <8 x i32> [[BIN_RDX2]], <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX4:%.*]] = and <8 x i32> [[BIN_RDX2]], [[RDX_SHUF3]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <8 x i32> [[BIN_RDX4]], i32 0
-; CHECK-NEXT:    [[MUL_714:%.*]] = and i32 undef, [[MUL_613]]
 ; CHECK-NEXT:    ret i32 [[TMP2]]
 ;
 entry:
@@ -208,12 +194,6 @@ define i32 @test_or(i32* nocapture readonly %p) {
 ; CHECK-NEXT:    [[ARRAYIDX_7:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 7
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32* [[P]] to <8 x i32>*
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* [[TMP0]], align 4
-; CHECK-NEXT:    [[MUL_18:%.*]] = or i32 undef, undef
-; CHECK-NEXT:    [[MUL_29:%.*]] = or i32 undef, [[MUL_18]]
-; CHECK-NEXT:    [[MUL_310:%.*]] = or i32 undef, [[MUL_29]]
-; CHECK-NEXT:    [[MUL_411:%.*]] = or i32 undef, [[MUL_310]]
-; CHECK-NEXT:    [[MUL_512:%.*]] = or i32 undef, [[MUL_411]]
-; CHECK-NEXT:    [[MUL_613:%.*]] = or i32 undef, [[MUL_512]]
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = or <8 x i32> [[TMP1]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x i32> [[BIN_RDX]], <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -221,7 +201,6 @@ define i32 @test_or(i32* nocapture readonly %p) {
 ; CHECK-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <8 x i32> [[BIN_RDX2]], <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX4:%.*]] = or <8 x i32> [[BIN_RDX2]], [[RDX_SHUF3]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <8 x i32> [[BIN_RDX4]], i32 0
-; CHECK-NEXT:    [[MUL_714:%.*]] = or i32 undef, [[MUL_613]]
 ; CHECK-NEXT:    ret i32 [[TMP2]]
 ;
 entry:
@@ -269,12 +248,6 @@ define i32 @test_xor(i32* nocapture readonly %p) {
 ; CHECK-NEXT:    [[ARRAYIDX_7:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 7
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32* [[P]] to <8 x i32>*
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* [[TMP0]], align 4
-; CHECK-NEXT:    [[MUL_18:%.*]] = xor i32 undef, undef
-; CHECK-NEXT:    [[MUL_29:%.*]] = xor i32 undef, [[MUL_18]]
-; CHECK-NEXT:    [[MUL_310:%.*]] = xor i32 undef, [[MUL_29]]
-; CHECK-NEXT:    [[MUL_411:%.*]] = xor i32 undef, [[MUL_310]]
-; CHECK-NEXT:    [[MUL_512:%.*]] = xor i32 undef, [[MUL_411]]
-; CHECK-NEXT:    [[MUL_613:%.*]] = xor i32 undef, [[MUL_512]]
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = xor <8 x i32> [[TMP1]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x i32> [[BIN_RDX]], <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -282,7 +255,6 @@ define i32 @test_xor(i32* nocapture readonly %p) {
 ; CHECK-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <8 x i32> [[BIN_RDX2]], <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX4:%.*]] = xor <8 x i32> [[BIN_RDX2]], [[RDX_SHUF3]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <8 x i32> [[BIN_RDX4]], i32 0
-; CHECK-NEXT:    [[MUL_714:%.*]] = xor i32 undef, [[MUL_613]]
 ; CHECK-NEXT:    ret i32 [[TMP2]]
 ;
 entry:
@@ -322,15 +294,12 @@ define i32 @PR37731(<4 x i32>* noalias nocapture dereferenceable(16) %self) unna
 ; CHECK-NEXT:    [[TMP5:%.*]] = shl <4 x i32> [[TMP4]], <i32 18, i32 2, i32 7, i32 13>
 ; CHECK-NEXT:    [[TMP6:%.*]] = xor <4 x i32> [[TMP3]], [[TMP5]]
 ; CHECK-NEXT:    store <4 x i32> [[TMP6]], <4 x i32>* [[SELF]], align 16
-; CHECK-NEXT:    [[TMP7:%.*]] = xor i32 undef, undef
-; CHECK-NEXT:    [[TMP8:%.*]] = xor i32 [[TMP7]], undef
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = xor <4 x i32> [[TMP6]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x i32> [[BIN_RDX]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX2:%.*]] = xor <4 x i32> [[BIN_RDX]], [[RDX_SHUF1]]
-; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <4 x i32> [[BIN_RDX2]], i32 0
-; CHECK-NEXT:    [[TMP10:%.*]] = xor i32 [[TMP8]], undef
-; CHECK-NEXT:    ret i32 [[TMP9]]
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x i32> [[BIN_RDX2]], i32 0
+; CHECK-NEXT:    ret i32 [[TMP7]]
 ;
 entry:
   %0 = load <4 x i32>, <4 x i32>* %self, align 16
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/remark_horcost.ll b/llvm/test/Transforms/SLPVectorizer/X86/remark_horcost.ll
index 27997f6af..34ceacac3 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/remark_horcost.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/remark_horcost.ll
@@ -33,11 +33,8 @@ define i32 @foo(i32* %diff) #0 {
 ; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i32* [[ARRAYIDX2]] to <4 x i32>*
 ; CHECK-NEXT:    [[TMP12:%.*]] = load <4 x i32>, <4 x i32>* [[TMP11]], align 4
 ; CHECK-NEXT:    [[TMP13:%.*]] = add nsw <4 x i32> [[TMP12]], [[TMP9]]
-; CHECK-NEXT:    [[ADD10:%.*]] = add nsw i32 undef, [[A_088]]
 ; CHECK-NEXT:    [[ARRAYIDX20:%.*]] = getelementptr inbounds [8 x [8 x i32]], [8 x [8 x i32]]* [[M2]], i64 0, i64 [[INDVARS_IV]], i64 1
-; CHECK-NEXT:    [[ADD24:%.*]] = add nsw i32 [[ADD10]], undef
 ; CHECK-NEXT:    [[ARRAYIDX34:%.*]] = getelementptr inbounds [8 x [8 x i32]], [8 x [8 x i32]]* [[M2]], i64 0, i64 [[INDVARS_IV]], i64 2
-; CHECK-NEXT:    [[ADD38:%.*]] = add nsw i32 [[ADD24]], undef
 ; CHECK-NEXT:    [[ARRAYIDX48:%.*]] = getelementptr inbounds [8 x [8 x i32]], [8 x [8 x i32]]* [[M2]], i64 0, i64 [[INDVARS_IV]], i64 3
 ; CHECK-NEXT:    [[TMP14:%.*]] = bitcast i32* [[ARRAYIDX6]] to <4 x i32>*
 ; CHECK-NEXT:    store <4 x i32> [[TMP13]], <4 x i32>* [[TMP14]], align 16
@@ -47,7 +44,6 @@ define i32 @foo(i32* %diff) #0 {
 ; CHECK-NEXT:    [[BIN_RDX2:%.*]] = add nsw <4 x i32> [[BIN_RDX]], [[RDX_SHUF1]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <4 x i32> [[BIN_RDX2]], i32 0
 ; CHECK-NEXT:    [[OP_EXTRA]] = add nsw i32 [[TMP15]], [[A_088]]
-; CHECK-NEXT:    [[ADD52:%.*]] = add nsw i32 [[ADD38]], undef
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 8
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]]
@@ -124,7 +120,7 @@ for.body:                                         ; preds = %for.body, %entry
   ; YAML-NEXT: Function:        foo
   ; YAML-NEXT: Args:
   ; YAML-NEXT:   - String:          'Vectorized horizontal reduction with cost '
-  ; YAML-NEXT:   - Cost:            '-2'
+  ; YAML-NEXT:   - Cost:            '-4'
   ; YAML-NEXT:   - String:          ' and with tree size '
   ; YAML-NEXT:   - TreeSize:        '1'
 
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reorder_repeated_ops.ll b/llvm/test/Transforms/SLPVectorizer/X86/reorder_repeated_ops.ll
index 13884efd9..c015a50e1 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reorder_repeated_ops.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reorder_repeated_ops.ll
@@ -4,14 +4,14 @@
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
 define void @hoge() {
-; CHECK-LABEL: @hoge(
+; CHECK-LABEL: define {{[^@]+}}@hoge(
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    br i1 undef, label [[BB1:%.*]], label [[BB2:%.*]]
 ; CHECK:       bb1:
 ; CHECK-NEXT:    ret void
 ; CHECK:       bb2:
-; CHECK-NEXT:    [[TMP:%.*]] = select i1 undef, i16 undef, i16 15
-; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x i16> undef, i16 [[TMP]], i32 0
+; CHECK-NEXT:    [[T:%.*]] = select i1 undef, i16 undef, i16 15
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x i16> undef, i16 [[T]], i32 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i16> [[TMP0]], i16 undef, i32 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = sext <2 x i16> [[TMP1]] to <2 x i32>
 ; CHECK-NEXT:    [[REORDER_SHUFFLE:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> undef, <2 x i32> <i32 1, i32 0>
@@ -19,11 +19,6 @@ define void @hoge() {
 ; CHECK-NEXT:    [[TMP4:%.*]] = sub <2 x i32> [[TMP3]], undef
 ; CHECK-NEXT:    [[SHUFFLE8:%.*]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 1, i32 1>
 ; CHECK-NEXT:    [[TMP5:%.*]] = add <4 x i32> [[SHUFFLE8]], <i32 undef, i32 15, i32 31, i32 47>
-; CHECK-NEXT:    [[TMP11:%.*]] = icmp sgt i32 undef, undef
-; CHECK-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], i32 undef, i32 undef
-; CHECK-NEXT:    [[TMP14:%.*]] = icmp sgt i32 [[TMP12]], undef
-; CHECK-NEXT:    [[TMP15:%.*]] = select i1 [[TMP14]], i32 [[TMP12]], i32 undef
-; CHECK-NEXT:    [[TMP17:%.*]] = icmp sgt i32 [[TMP15]], undef
 ; CHECK-NEXT:    [[RDX_SHUF9:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[RDX_MINMAX_CMP10:%.*]] = icmp sgt <4 x i32> [[TMP5]], [[RDX_SHUF9]]
 ; CHECK-NEXT:    [[RDX_MINMAX_SELECT11:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP10]], <4 x i32> [[TMP5]], <4 x i32> [[RDX_SHUF9]]
@@ -31,28 +26,12 @@ define void @hoge() {
 ; CHECK-NEXT:    [[RDX_MINMAX_CMP13:%.*]] = icmp sgt <4 x i32> [[RDX_MINMAX_SELECT11]], [[RDX_SHUF12]]
 ; CHECK-NEXT:    [[RDX_MINMAX_SELECT14:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP13]], <4 x i32> [[RDX_MINMAX_SELECT11]], <4 x i32> [[RDX_SHUF12]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x i32> [[RDX_MINMAX_SELECT14]], i32 0
-; CHECK-NEXT:    [[TMP18:%.*]] = select i1 [[TMP17]], i32 [[TMP15]], i32 undef
-; CHECK-NEXT:    [[TMP19:%.*]] = select i1 undef, i32 [[TMP6]], i32 undef
-; CHECK-NEXT:    [[TMP20:%.*]] = icmp sgt i32 [[TMP19]], 63
+; CHECK-NEXT:    [[T19:%.*]] = select i1 undef, i32 [[TMP6]], i32 undef
+; CHECK-NEXT:    [[T20:%.*]] = icmp sgt i32 [[T19]], 63
 ; CHECK-NEXT:    [[TMP7:%.*]] = sub nsw <2 x i32> undef, [[TMP2]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = sub <2 x i32> [[TMP7]], undef
 ; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <2 x i32> [[TMP8]], <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
 ; CHECK-NEXT:    [[TMP9:%.*]] = add nsw <4 x i32> [[SHUFFLE]], <i32 -49, i32 -33, i32 -33, i32 -17>
-; CHECK-NEXT:    [[TMP26:%.*]] = icmp sgt i32 undef, undef
-; CHECK-NEXT:    [[TMP27:%.*]] = select i1 [[TMP26]], i32 undef, i32 undef
-; CHECK-NEXT:    [[TMP28:%.*]] = icmp sgt i32 [[TMP27]], undef
-; CHECK-NEXT:    [[TMP29:%.*]] = select i1 [[TMP28]], i32 undef, i32 [[TMP27]]
-; CHECK-NEXT:    [[TMP31:%.*]] = icmp sgt i32 undef, undef
-; CHECK-NEXT:    [[TMP32:%.*]] = select i1 [[TMP31]], i32 undef, i32 undef
-; CHECK-NEXT:    [[TMP33:%.*]] = icmp sgt i32 [[TMP32]], [[TMP29]]
-; CHECK-NEXT:    [[TMP34:%.*]] = select i1 [[TMP33]], i32 [[TMP29]], i32 [[TMP32]]
-; CHECK-NEXT:    [[TMP36:%.*]] = icmp sgt i32 undef, undef
-; CHECK-NEXT:    [[TMP37:%.*]] = select i1 [[TMP36]], i32 undef, i32 undef
-; CHECK-NEXT:    [[TMP38:%.*]] = icmp sgt i32 [[TMP37]], [[TMP34]]
-; CHECK-NEXT:    [[TMP39:%.*]] = select i1 [[TMP38]], i32 [[TMP34]], i32 [[TMP37]]
-; CHECK-NEXT:    [[TMP41:%.*]] = icmp sgt i32 undef, undef
-; CHECK-NEXT:    [[TMP42:%.*]] = select i1 [[TMP41]], i32 undef, i32 undef
-; CHECK-NEXT:    [[TMP43:%.*]] = icmp sgt i32 [[TMP42]], [[TMP39]]
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[TMP9]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = icmp slt <4 x i32> [[TMP9]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x i32> [[TMP9]], <4 x i32> [[RDX_SHUF]]
@@ -70,8 +49,7 @@ define void @hoge() {
 ; CHECK-NEXT:    [[OP_EXTRA6:%.*]] = select i1 [[TMP14]], i32 [[OP_EXTRA5]], i32 undef
 ; CHECK-NEXT:    [[TMP15:%.*]] = icmp slt i32 [[OP_EXTRA6]], undef
 ; CHECK-NEXT:    [[OP_EXTRA7:%.*]] = select i1 [[TMP15]], i32 [[OP_EXTRA6]], i32 undef
-; CHECK-NEXT:    [[TMP44:%.*]] = select i1 [[TMP43]], i32 [[TMP39]], i32 [[TMP42]]
-; CHECK-NEXT:    [[TMP45:%.*]] = icmp sgt i32 undef, [[OP_EXTRA7]]
+; CHECK-NEXT:    [[T45:%.*]] = icmp sgt i32 undef, [[OP_EXTRA7]]
 ; CHECK-NEXT:    unreachable
 ;
 bb:
@@ -81,50 +59,50 @@ bb1:                                              ; preds = %bb
   ret void
 
 bb2:                                              ; preds = %bb
-  %tmp = select i1 undef, i16 undef, i16 15
-  %tmp3 = sext i16 undef to i32
-  %tmp4 = sext i16 %tmp to i32
-  %tmp5 = sub nsw i32 undef, %tmp4
-  %tmp6 = sub i32 %tmp5, undef
-  %tmp7 = sub nsw i32 63, %tmp3
-  %tmp8 = sub i32 %tmp7, undef
-  %tmp9 = add i32 %tmp8, undef
-  %tmp10 = add nsw i32 %tmp6, 15
-  %tmp11 = icmp sgt i32 %tmp9, %tmp10
-  %tmp12 = select i1 %tmp11, i32 %tmp9, i32 %tmp10
-  %tmp13 = add nsw i32 %tmp6, 31
-  %tmp14 = icmp sgt i32 %tmp12, %tmp13
-  %tmp15 = select i1 %tmp14, i32 %tmp12, i32 %tmp13
-  %tmp16 = add nsw i32 %tmp6, 47
-  %tmp17 = icmp sgt i32 %tmp15, %tmp16
-  %tmp18 = select i1 %tmp17, i32 %tmp15, i32 %tmp16
-  %tmp19 = select i1 undef, i32 %tmp18, i32 undef
-  %tmp20 = icmp sgt i32 %tmp19, 63
-  %tmp21 = sub nsw i32 undef, %tmp3
-  %tmp22 = sub i32 %tmp21, undef
-  %tmp23 = sub nsw i32 undef, %tmp4
-  %tmp24 = sub i32 %tmp23, undef
-  %tmp25 = add nsw i32 %tmp24, -49
-  %tmp26 = icmp sgt i32 %tmp25, undef
-  %tmp27 = select i1 %tmp26, i32 undef, i32 %tmp25
-  %tmp28 = icmp sgt i32 %tmp27, undef
-  %tmp29 = select i1 %tmp28, i32 undef, i32 %tmp27
-  %tmp30 = add nsw i32 %tmp22, -33
-  %tmp31 = icmp sgt i32 %tmp30, undef
-  %tmp32 = select i1 %tmp31, i32 undef, i32 %tmp30
-  %tmp33 = icmp sgt i32 %tmp32, %tmp29
-  %tmp34 = select i1 %tmp33, i32 %tmp29, i32 %tmp32
-  %tmp35 = add nsw i32 %tmp24, -33
-  %tmp36 = icmp sgt i32 %tmp35, undef
-  %tmp37 = select i1 %tmp36, i32 undef, i32 %tmp35
-  %tmp38 = icmp sgt i32 %tmp37, %tmp34
-  %tmp39 = select i1 %tmp38, i32 %tmp34, i32 %tmp37
-  %tmp40 = add nsw i32 %tmp22, -17
-  %tmp41 = icmp sgt i32 %tmp40, undef
-  %tmp42 = select i1 %tmp41, i32 undef, i32 %tmp40
-  %tmp43 = icmp sgt i32 %tmp42, %tmp39
-  %tmp44 = select i1 %tmp43, i32 %tmp39, i32 %tmp42
-  %tmp45 = icmp sgt i32 undef, %tmp44
+  %t = select i1 undef, i16 undef, i16 15
+  %t3 = sext i16 undef to i32
+  %t4 = sext i16 %t to i32
+  %t5 = sub nsw i32 undef, %t4
+  %t6 = sub i32 %t5, undef
+  %t7 = sub nsw i32 63, %t3
+  %t8 = sub i32 %t7, undef
+  %t9 = add i32 %t8, undef
+  %t10 = add nsw i32 %t6, 15
+  %t11 = icmp sgt i32 %t9, %t10
+  %t12 = select i1 %t11, i32 %t9, i32 %t10
+  %t13 = add nsw i32 %t6, 31
+  %t14 = icmp sgt i32 %t12, %t13
+  %t15 = select i1 %t14, i32 %t12, i32 %t13
+  %t16 = add nsw i32 %t6, 47
+  %t17 = icmp sgt i32 %t15, %t16
+  %t18 = select i1 %t17, i32 %t15, i32 %t16
+  %t19 = select i1 undef, i32 %t18, i32 undef
+  %t20 = icmp sgt i32 %t19, 63
+  %t21 = sub nsw i32 undef, %t3
+  %t22 = sub i32 %t21, undef
+  %t23 = sub nsw i32 undef, %t4
+  %t24 = sub i32 %t23, undef
+  %t25 = add nsw i32 %t24, -49
+  %t26 = icmp sgt i32 %t25, undef
+  %t27 = select i1 %t26, i32 undef, i32 %t25
+  %t28 = icmp sgt i32 %t27, undef
+  %t29 = select i1 %t28, i32 undef, i32 %t27
+  %t30 = add nsw i32 %t22, -33
+  %t31 = icmp sgt i32 %t30, undef
+  %t32 = select i1 %t31, i32 undef, i32 %t30
+  %t33 = icmp sgt i32 %t32, %t29
+  %t34 = select i1 %t33, i32 %t29, i32 %t32
+  %t35 = add nsw i32 %t24, -33
+  %t36 = icmp sgt i32 %t35, undef
+  %t37 = select i1 %t36, i32 undef, i32 %t35
+  %t38 = icmp sgt i32 %t37, %t34
+  %t39 = select i1 %t38, i32 %t34, i32 %t37
+  %t40 = add nsw i32 %t22, -17
+  %t41 = icmp sgt i32 %t40, undef
+  %t42 = select i1 %t41, i32 undef, i32 %t40
+  %t43 = icmp sgt i32 %t42, %t39
+  %t44 = select i1 %t43, i32 %t39, i32 %t42
+  %t45 = icmp sgt i32 undef, %t44
   unreachable
 }
 
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reuse-extracts-in-wider-vect.ll b/llvm/test/Transforms/SLPVectorizer/X86/reuse-extracts-in-wider-vect.ll
new file mode 100644
index 000000000..62fa52c46
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reuse-extracts-in-wider-vect.ll
@@ -0,0 +1,67 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -slp-vectorizer -S -mtriple=x86_64-unknown-linux -mcpu=core-avx2 | FileCheck %s
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+
+%struct.S = type { [3 x float], [3 x float], [4 x float] }
+
+define i32 @foo(i32 %0, i32* %1, float* %2)  {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:    [[T4:%.*]] = alloca [[STRUCT_S:%.*]], align 8
+; CHECK-NEXT:    [[T8:%.*]] = getelementptr inbounds [[STRUCT_S]], %struct.S* [[T4]], i64 0, i32 1
+; CHECK-NEXT:    [[T9:%.*]] = getelementptr inbounds [3 x float], [3 x float]* [[T8]], i64 1, i64 0
+; CHECK-NEXT:    [[T14:%.*]] = getelementptr inbounds [[STRUCT_S]], %struct.S* [[T4]], i64 0, i32 1, i64 0
+; CHECK-NEXT:    [[T11:%.*]] = getelementptr inbounds [[STRUCT_S]], %struct.S* [[T4]], i64 0, i32 1, i64 1
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast float* [[T14]] to <2 x float>*
+; CHECK-NEXT:    [[TMP5:%.*]] = load <2 x float>, <2 x float>* [[TMP4]], align 4
+; CHECK-NEXT:    br label [[T37:%.*]]
+; CHECK:       t37:
+; CHECK-NEXT:    [[TMP6:%.*]] = phi <2 x float> [ [[TMP5]], [[TMP3:%.*]] ], [ [[T89:%.*]], [[T37]] ]
+; CHECK-NEXT:    [[TMP7:%.*]] = fdiv fast <2 x float> <float 1.000000e+00, float 1.000000e+00>, [[TMP6]]
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <2 x float> [[TMP7]], <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[T21:%.*]] = getelementptr inbounds [[STRUCT_S]], %struct.S* [[T4]], i64 0, i32 2, i64 0
+; CHECK-NEXT:    [[T25:%.*]] = getelementptr inbounds [[STRUCT_S]], %struct.S* [[T4]], i64 0, i32 2, i64 1
+; CHECK-NEXT:    [[T31:%.*]] = getelementptr inbounds [[STRUCT_S]], %struct.S* [[T4]], i64 0, i32 2, i64 2
+; CHECK-NEXT:    [[T33:%.*]] = getelementptr inbounds [[STRUCT_S]], %struct.S* [[T4]], i64 0, i32 2, i64 3
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast float* [[T21]] to <4 x float>*
+; CHECK-NEXT:    store <4 x float> [[SHUFFLE]], <4 x float>* [[TMP8]], align 4
+; CHECK-NEXT:    [[T88:%.*]] = bitcast float* [[T9]] to <2 x float>*
+; CHECK-NEXT:    [[T89]] = load <2 x float>, <2 x float>* [[T88]], align 4
+; CHECK-NEXT:    br i1 undef, label [[T37]], label [[T55:%.*]]
+; CHECK:       t55:
+; CHECK-NEXT:    ret i32 0
+;
+  %t4 = alloca %struct.S, align 8
+  %t8 = getelementptr inbounds %struct.S, %struct.S* %t4, i64 0, i32 1
+  %t9 = getelementptr inbounds [3 x float], [3 x float]* %t8, i64 1, i64 0
+  %t14 = getelementptr inbounds %struct.S, %struct.S* %t4, i64 0, i32 1, i64 0
+  %t11 = getelementptr inbounds %struct.S, %struct.S* %t4, i64 0, i32 1, i64 1
+  %t15 = load float, float* %t14, align 4
+  %t16 = load float, float* %t11, align 4
+  br label %t37
+
+t37:
+
+  %t18 = phi float [ %t16, %3 ], [ %x24, %t37 ]
+  %t19 = phi float [ %t15, %3 ], [ %x23, %t37 ]
+  %t20 = fdiv fast float 1.000000e+00, %t19
+  %t24 = fdiv fast float 1.000000e+00, %t18
+  %t21 = getelementptr inbounds %struct.S, %struct.S* %t4, i64 0, i32 2, i64 0
+  %t25 = getelementptr inbounds %struct.S, %struct.S* %t4, i64 0, i32 2, i64 1
+  %t31 = getelementptr inbounds %struct.S, %struct.S* %t4, i64 0, i32 2, i64 2
+  %t33 = getelementptr inbounds %struct.S, %struct.S* %t4, i64 0, i32 2, i64 3
+  store float %t20, float* %t21, align 4
+  store float %t24, float* %t25, align 4
+  store float %t24, float* %t31, align 4
+  store float %t24, float* %t33, align 4
+  %t88 = bitcast float* %t9 to <2 x float>*
+  %t89 = load <2 x float>, <2 x float>* %t88, align 4
+  %x23 = extractelement <2 x float> %t89, i32 0
+  %x24 = extractelement <2 x float> %t89, i32 1
+  br i1 undef, label %t37, label %t55
+
+t55:
+
+  ret i32 0
+
+}
+
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/sext.ll b/llvm/test/Transforms/SLPVectorizer/X86/sext.ll
index 75406c927..c3eba4701 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/sext.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/sext.ll
@@ -11,26 +11,15 @@
 ;
 
 define <2 x i64> @loadext_2i8_to_2i64(i8* %p0) {
-; SSE2-LABEL: @loadext_2i8_to_2i64(
-; SSE2-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
-; SSE2-NEXT:    [[I0:%.*]] = load i8, i8* [[P0]], align 1
-; SSE2-NEXT:    [[I1:%.*]] = load i8, i8* [[P1]], align 1
-; SSE2-NEXT:    [[X0:%.*]] = sext i8 [[I0]] to i64
-; SSE2-NEXT:    [[X1:%.*]] = sext i8 [[I1]] to i64
-; SSE2-NEXT:    [[V0:%.*]] = insertelement <2 x i64> undef, i64 [[X0]], i32 0
-; SSE2-NEXT:    [[V1:%.*]] = insertelement <2 x i64> [[V0]], i64 [[X1]], i32 1
-; SSE2-NEXT:    ret <2 x i64> [[V1]]
-;
-; SLM-LABEL: @loadext_2i8_to_2i64(
-; SLM-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
-; SLM-NEXT:    [[TMP1:%.*]] = bitcast i8* [[P0]] to <2 x i8>*
-; SLM-NEXT:    [[TMP2:%.*]] = load <2 x i8>, <2 x i8>* [[TMP1]], align 1
-; SLM-NEXT:    [[TMP3:%.*]] = sext <2 x i8> [[TMP2]] to <2 x i64>
-; SLM-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i32 0
-; SLM-NEXT:    [[V0:%.*]] = insertelement <2 x i64> undef, i64 [[TMP4]], i32 0
-; SLM-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[TMP3]], i32 1
-; SLM-NEXT:    [[V1:%.*]] = insertelement <2 x i64> [[V0]], i64 [[TMP5]], i32 1
-; SLM-NEXT:    ret <2 x i64> [[V1]]
+; SSE-LABEL: @loadext_2i8_to_2i64(
+; SSE-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; SSE-NEXT:    [[I0:%.*]] = load i8, i8* [[P0]], align 1
+; SSE-NEXT:    [[I1:%.*]] = load i8, i8* [[P1]], align 1
+; SSE-NEXT:    [[X0:%.*]] = sext i8 [[I0]] to i64
+; SSE-NEXT:    [[X1:%.*]] = sext i8 [[I1]] to i64
+; SSE-NEXT:    [[V0:%.*]] = insertelement <2 x i64> undef, i64 [[X0]], i32 0
+; SSE-NEXT:    [[V1:%.*]] = insertelement <2 x i64> [[V0]], i64 [[X1]], i32 1
+; SSE-NEXT:    ret <2 x i64> [[V1]]
 ;
 ; AVX-LABEL: @loadext_2i8_to_2i64(
 ; AVX-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
@@ -54,40 +43,23 @@ define <2 x i64> @loadext_2i8_to_2i64(i8* %p0) {
 }
 
 define <4 x i32> @loadext_4i8_to_4i32(i8* %p0) {
-; SSE2-LABEL: @loadext_4i8_to_4i32(
-; SSE2-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
-; SSE2-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
-; SSE2-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
-; SSE2-NEXT:    [[I0:%.*]] = load i8, i8* [[P0]], align 1
-; SSE2-NEXT:    [[I1:%.*]] = load i8, i8* [[P1]], align 1
-; SSE2-NEXT:    [[I2:%.*]] = load i8, i8* [[P2]], align 1
-; SSE2-NEXT:    [[I3:%.*]] = load i8, i8* [[P3]], align 1
-; SSE2-NEXT:    [[X0:%.*]] = sext i8 [[I0]] to i32
-; SSE2-NEXT:    [[X1:%.*]] = sext i8 [[I1]] to i32
-; SSE2-NEXT:    [[X2:%.*]] = sext i8 [[I2]] to i32
-; SSE2-NEXT:    [[X3:%.*]] = sext i8 [[I3]] to i32
-; SSE2-NEXT:    [[V0:%.*]] = insertelement <4 x i32> undef, i32 [[X0]], i32 0
-; SSE2-NEXT:    [[V1:%.*]] = insertelement <4 x i32> [[V0]], i32 [[X1]], i32 1
-; SSE2-NEXT:    [[V2:%.*]] = insertelement <4 x i32> [[V1]], i32 [[X2]], i32 2
-; SSE2-NEXT:    [[V3:%.*]] = insertelement <4 x i32> [[V2]], i32 [[X3]], i32 3
-; SSE2-NEXT:    ret <4 x i32> [[V3]]
-;
-; SLM-LABEL: @loadext_4i8_to_4i32(
-; SLM-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
-; SLM-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
-; SLM-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
-; SLM-NEXT:    [[TMP1:%.*]] = bitcast i8* [[P0]] to <4 x i8>*
-; SLM-NEXT:    [[TMP2:%.*]] = load <4 x i8>, <4 x i8>* [[TMP1]], align 1
-; SLM-NEXT:    [[TMP3:%.*]] = sext <4 x i8> [[TMP2]] to <4 x i32>
-; SLM-NEXT:    [[TMP4:%.*]] = extractelement <4 x i32> [[TMP3]], i32 0
-; SLM-NEXT:    [[V0:%.*]] = insertelement <4 x i32> undef, i32 [[TMP4]], i32 0
-; SLM-NEXT:    [[TMP5:%.*]] = extractelement <4 x i32> [[TMP3]], i32 1
-; SLM-NEXT:    [[V1:%.*]] = insertelement <4 x i32> [[V0]], i32 [[TMP5]], i32 1
-; SLM-NEXT:    [[TMP6:%.*]] = extractelement <4 x i32> [[TMP3]], i32 2
-; SLM-NEXT:    [[V2:%.*]] = insertelement <4 x i32> [[V1]], i32 [[TMP6]], i32 2
-; SLM-NEXT:    [[TMP7:%.*]] = extractelement <4 x i32> [[TMP3]], i32 3
-; SLM-NEXT:    [[V3:%.*]] = insertelement <4 x i32> [[V2]], i32 [[TMP7]], i32 3
-; SLM-NEXT:    ret <4 x i32> [[V3]]
+; SSE-LABEL: @loadext_4i8_to_4i32(
+; SSE-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; SSE-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; SSE-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; SSE-NEXT:    [[I0:%.*]] = load i8, i8* [[P0]], align 1
+; SSE-NEXT:    [[I1:%.*]] = load i8, i8* [[P1]], align 1
+; SSE-NEXT:    [[I2:%.*]] = load i8, i8* [[P2]], align 1
+; SSE-NEXT:    [[I3:%.*]] = load i8, i8* [[P3]], align 1
+; SSE-NEXT:    [[X0:%.*]] = sext i8 [[I0]] to i32
+; SSE-NEXT:    [[X1:%.*]] = sext i8 [[I1]] to i32
+; SSE-NEXT:    [[X2:%.*]] = sext i8 [[I2]] to i32
+; SSE-NEXT:    [[X3:%.*]] = sext i8 [[I3]] to i32
+; SSE-NEXT:    [[V0:%.*]] = insertelement <4 x i32> undef, i32 [[X0]], i32 0
+; SSE-NEXT:    [[V1:%.*]] = insertelement <4 x i32> [[V0]], i32 [[X1]], i32 1
+; SSE-NEXT:    [[V2:%.*]] = insertelement <4 x i32> [[V1]], i32 [[X2]], i32 2
+; SSE-NEXT:    [[V3:%.*]] = insertelement <4 x i32> [[V2]], i32 [[X3]], i32 3
+; SSE-NEXT:    ret <4 x i32> [[V3]]
 ;
 ; AVX-LABEL: @loadext_4i8_to_4i32(
 ; AVX-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
@@ -125,59 +97,76 @@ define <4 x i32> @loadext_4i8_to_4i32(i8* %p0) {
 }
 
 define <4 x i64> @loadext_4i8_to_4i64(i8* %p0) {
-; SSE2-LABEL: @loadext_4i8_to_4i64(
-; SSE2-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
-; SSE2-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
-; SSE2-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
-; SSE2-NEXT:    [[I0:%.*]] = load i8, i8* [[P0]], align 1
-; SSE2-NEXT:    [[I1:%.*]] = load i8, i8* [[P1]], align 1
-; SSE2-NEXT:    [[I2:%.*]] = load i8, i8* [[P2]], align 1
-; SSE2-NEXT:    [[I3:%.*]] = load i8, i8* [[P3]], align 1
-; SSE2-NEXT:    [[X0:%.*]] = sext i8 [[I0]] to i64
-; SSE2-NEXT:    [[X1:%.*]] = sext i8 [[I1]] to i64
-; SSE2-NEXT:    [[X2:%.*]] = sext i8 [[I2]] to i64
-; SSE2-NEXT:    [[X3:%.*]] = sext i8 [[I3]] to i64
-; SSE2-NEXT:    [[V0:%.*]] = insertelement <4 x i64> undef, i64 [[X0]], i32 0
-; SSE2-NEXT:    [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[X1]], i32 1
-; SSE2-NEXT:    [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[X2]], i32 2
-; SSE2-NEXT:    [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[X3]], i32 3
-; SSE2-NEXT:    ret <4 x i64> [[V3]]
-;
-; SLM-LABEL: @loadext_4i8_to_4i64(
-; SLM-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
-; SLM-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
-; SLM-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
-; SLM-NEXT:    [[TMP1:%.*]] = bitcast i8* [[P0]] to <4 x i8>*
-; SLM-NEXT:    [[TMP2:%.*]] = load <4 x i8>, <4 x i8>* [[TMP1]], align 1
-; SLM-NEXT:    [[TMP3:%.*]] = sext <4 x i8> [[TMP2]] to <4 x i64>
-; SLM-NEXT:    [[TMP4:%.*]] = extractelement <4 x i64> [[TMP3]], i32 0
-; SLM-NEXT:    [[V0:%.*]] = insertelement <4 x i64> undef, i64 [[TMP4]], i32 0
-; SLM-NEXT:    [[TMP5:%.*]] = extractelement <4 x i64> [[TMP3]], i32 1
-; SLM-NEXT:    [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[TMP5]], i32 1
-; SLM-NEXT:    [[TMP6:%.*]] = extractelement <4 x i64> [[TMP3]], i32 2
-; SLM-NEXT:    [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[TMP6]], i32 2
-; SLM-NEXT:    [[TMP7:%.*]] = extractelement <4 x i64> [[TMP3]], i32 3
-; SLM-NEXT:    [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[TMP7]], i32 3
-; SLM-NEXT:    ret <4 x i64> [[V3]]
-;
-; AVX-LABEL: @loadext_4i8_to_4i64(
-; AVX-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
-; AVX-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
-; AVX-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
-; AVX-NEXT:    [[TMP1:%.*]] = bitcast i8* [[P0]] to <2 x i8>*
-; AVX-NEXT:    [[TMP2:%.*]] = load <2 x i8>, <2 x i8>* [[TMP1]], align 1
-; AVX-NEXT:    [[I2:%.*]] = load i8, i8* [[P2]], align 1
-; AVX-NEXT:    [[I3:%.*]] = load i8, i8* [[P3]], align 1
-; AVX-NEXT:    [[TMP3:%.*]] = sext <2 x i8> [[TMP2]] to <2 x i64>
-; AVX-NEXT:    [[X2:%.*]] = sext i8 [[I2]] to i64
-; AVX-NEXT:    [[X3:%.*]] = sext i8 [[I3]] to i64
-; AVX-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i32 0
-; AVX-NEXT:    [[V0:%.*]] = insertelement <4 x i64> undef, i64 [[TMP4]], i32 0
-; AVX-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[TMP3]], i32 1
-; AVX-NEXT:    [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[TMP5]], i32 1
-; AVX-NEXT:    [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[X2]], i32 2
-; AVX-NEXT:    [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[X3]], i32 3
-; AVX-NEXT:    ret <4 x i64> [[V3]]
+; SSE-LABEL: @loadext_4i8_to_4i64(
+; SSE-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; SSE-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; SSE-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; SSE-NEXT:    [[I0:%.*]] = load i8, i8* [[P0]], align 1
+; SSE-NEXT:    [[I1:%.*]] = load i8, i8* [[P1]], align 1
+; SSE-NEXT:    [[I2:%.*]] = load i8, i8* [[P2]], align 1
+; SSE-NEXT:    [[I3:%.*]] = load i8, i8* [[P3]], align 1
+; SSE-NEXT:    [[X0:%.*]] = sext i8 [[I0]] to i64
+; SSE-NEXT:    [[X1:%.*]] = sext i8 [[I1]] to i64
+; SSE-NEXT:    [[X2:%.*]] = sext i8 [[I2]] to i64
+; SSE-NEXT:    [[X3:%.*]] = sext i8 [[I3]] to i64
+; SSE-NEXT:    [[V0:%.*]] = insertelement <4 x i64> undef, i64 [[X0]], i32 0
+; SSE-NEXT:    [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[X1]], i32 1
+; SSE-NEXT:    [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[X2]], i32 2
+; SSE-NEXT:    [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[X3]], i32 3
+; SSE-NEXT:    ret <4 x i64> [[V3]]
+;
+; AVX1-LABEL: @loadext_4i8_to_4i64(
+; AVX1-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; AVX1-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; AVX1-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; AVX1-NEXT:    [[TMP1:%.*]] = bitcast i8* [[P0]] to <2 x i8>*
+; AVX1-NEXT:    [[TMP2:%.*]] = load <2 x i8>, <2 x i8>* [[TMP1]], align 1
+; AVX1-NEXT:    [[I2:%.*]] = load i8, i8* [[P2]], align 1
+; AVX1-NEXT:    [[I3:%.*]] = load i8, i8* [[P3]], align 1
+; AVX1-NEXT:    [[TMP3:%.*]] = sext <2 x i8> [[TMP2]] to <2 x i64>
+; AVX1-NEXT:    [[X2:%.*]] = sext i8 [[I2]] to i64
+; AVX1-NEXT:    [[X3:%.*]] = sext i8 [[I3]] to i64
+; AVX1-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i32 0
+; AVX1-NEXT:    [[V0:%.*]] = insertelement <4 x i64> undef, i64 [[TMP4]], i32 0
+; AVX1-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[TMP3]], i32 1
+; AVX1-NEXT:    [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[TMP5]], i32 1
+; AVX1-NEXT:    [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[X2]], i32 2
+; AVX1-NEXT:    [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[X3]], i32 3
+; AVX1-NEXT:    ret <4 x i64> [[V3]]
+;
+; AVX2-LABEL: @loadext_4i8_to_4i64(
+; AVX2-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; AVX2-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; AVX2-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; AVX2-NEXT:    [[TMP1:%.*]] = bitcast i8* [[P0]] to <4 x i8>*
+; AVX2-NEXT:    [[TMP2:%.*]] = load <4 x i8>, <4 x i8>* [[TMP1]], align 1
+; AVX2-NEXT:    [[TMP3:%.*]] = sext <4 x i8> [[TMP2]] to <4 x i64>
+; AVX2-NEXT:    [[TMP4:%.*]] = extractelement <4 x i64> [[TMP3]], i32 0
+; AVX2-NEXT:    [[V0:%.*]] = insertelement <4 x i64> undef, i64 [[TMP4]], i32 0
+; AVX2-NEXT:    [[TMP5:%.*]] = extractelement <4 x i64> [[TMP3]], i32 1
+; AVX2-NEXT:    [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[TMP5]], i32 1
+; AVX2-NEXT:    [[TMP6:%.*]] = extractelement <4 x i64> [[TMP3]], i32 2
+; AVX2-NEXT:    [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[TMP6]], i32 2
+; AVX2-NEXT:    [[TMP7:%.*]] = extractelement <4 x i64> [[TMP3]], i32 3
+; AVX2-NEXT:    [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[TMP7]], i32 3
+; AVX2-NEXT:    ret <4 x i64> [[V3]]
+;
+; AVX512-LABEL: @loadext_4i8_to_4i64(
+; AVX512-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; AVX512-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; AVX512-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; AVX512-NEXT:    [[TMP1:%.*]] = bitcast i8* [[P0]] to <4 x i8>*
+; AVX512-NEXT:    [[TMP2:%.*]] = load <4 x i8>, <4 x i8>* [[TMP1]], align 1
+; AVX512-NEXT:    [[TMP3:%.*]] = sext <4 x i8> [[TMP2]] to <4 x i64>
+; AVX512-NEXT:    [[TMP4:%.*]] = extractelement <4 x i64> [[TMP3]], i32 0
+; AVX512-NEXT:    [[V0:%.*]] = insertelement <4 x i64> undef, i64 [[TMP4]], i32 0
+; AVX512-NEXT:    [[TMP5:%.*]] = extractelement <4 x i64> [[TMP3]], i32 1
+; AVX512-NEXT:    [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[TMP5]], i32 1
+; AVX512-NEXT:    [[TMP6:%.*]] = extractelement <4 x i64> [[TMP3]], i32 2
+; AVX512-NEXT:    [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[TMP6]], i32 2
+; AVX512-NEXT:    [[TMP7:%.*]] = extractelement <4 x i64> [[TMP3]], i32 3
+; AVX512-NEXT:    [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[TMP7]], i32 3
+; AVX512-NEXT:    ret <4 x i64> [[V3]]
 ;
   %p1 = getelementptr inbounds i8, i8* %p0, i64 1
   %p2 = getelementptr inbounds i8, i8* %p0, i64 2
@@ -198,34 +187,97 @@ define <4 x i64> @loadext_4i8_to_4i64(i8* %p0) {
 }
 
 define <8 x i16> @loadext_8i8_to_8i16(i8* %p0) {
-; CHECK-LABEL: @loadext_8i8_to_8i16(
-; CHECK-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
-; CHECK-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
-; CHECK-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
-; CHECK-NEXT:    [[P4:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 4
-; CHECK-NEXT:    [[P5:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 5
-; CHECK-NEXT:    [[P6:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 6
-; CHECK-NEXT:    [[P7:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 7
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8* [[P0]] to <8 x i8>*
-; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i8>, <8 x i8>* [[TMP1]], align 1
-; CHECK-NEXT:    [[TMP3:%.*]] = sext <8 x i8> [[TMP2]] to <8 x i16>
-; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <8 x i16> [[TMP3]], i32 0
-; CHECK-NEXT:    [[V0:%.*]] = insertelement <8 x i16> undef, i16 [[TMP4]], i32 0
-; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <8 x i16> [[TMP3]], i32 1
-; CHECK-NEXT:    [[V1:%.*]] = insertelement <8 x i16> [[V0]], i16 [[TMP5]], i32 1
-; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <8 x i16> [[TMP3]], i32 2
-; CHECK-NEXT:    [[V2:%.*]] = insertelement <8 x i16> [[V1]], i16 [[TMP6]], i32 2
-; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <8 x i16> [[TMP3]], i32 3
-; CHECK-NEXT:    [[V3:%.*]] = insertelement <8 x i16> [[V2]], i16 [[TMP7]], i32 3
-; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <8 x i16> [[TMP3]], i32 4
-; CHECK-NEXT:    [[V4:%.*]] = insertelement <8 x i16> [[V3]], i16 [[TMP8]], i32 4
-; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <8 x i16> [[TMP3]], i32 5
-; CHECK-NEXT:    [[V5:%.*]] = insertelement <8 x i16> [[V4]], i16 [[TMP9]], i32 5
-; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <8 x i16> [[TMP3]], i32 6
-; CHECK-NEXT:    [[V6:%.*]] = insertelement <8 x i16> [[V5]], i16 [[TMP10]], i32 6
-; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <8 x i16> [[TMP3]], i32 7
-; CHECK-NEXT:    [[V7:%.*]] = insertelement <8 x i16> [[V6]], i16 [[TMP11]], i32 7
-; CHECK-NEXT:    ret <8 x i16> [[V7]]
+; SSE2-LABEL: @loadext_8i8_to_8i16(
+; SSE2-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; SSE2-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; SSE2-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; SSE2-NEXT:    [[P4:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 4
+; SSE2-NEXT:    [[P5:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 5
+; SSE2-NEXT:    [[P6:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 6
+; SSE2-NEXT:    [[P7:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 7
+; SSE2-NEXT:    [[TMP1:%.*]] = bitcast i8* [[P0]] to <8 x i8>*
+; SSE2-NEXT:    [[TMP2:%.*]] = load <8 x i8>, <8 x i8>* [[TMP1]], align 1
+; SSE2-NEXT:    [[TMP3:%.*]] = sext <8 x i8> [[TMP2]] to <8 x i16>
+; SSE2-NEXT:    [[TMP4:%.*]] = extractelement <8 x i16> [[TMP3]], i32 0
+; SSE2-NEXT:    [[V0:%.*]] = insertelement <8 x i16> undef, i16 [[TMP4]], i32 0
+; SSE2-NEXT:    [[TMP5:%.*]] = extractelement <8 x i16> [[TMP3]], i32 1
+; SSE2-NEXT:    [[V1:%.*]] = insertelement <8 x i16> [[V0]], i16 [[TMP5]], i32 1
+; SSE2-NEXT:    [[TMP6:%.*]] = extractelement <8 x i16> [[TMP3]], i32 2
+; SSE2-NEXT:    [[V2:%.*]] = insertelement <8 x i16> [[V1]], i16 [[TMP6]], i32 2
+; SSE2-NEXT:    [[TMP7:%.*]] = extractelement <8 x i16> [[TMP3]], i32 3
+; SSE2-NEXT:    [[V3:%.*]] = insertelement <8 x i16> [[V2]], i16 [[TMP7]], i32 3
+; SSE2-NEXT:    [[TMP8:%.*]] = extractelement <8 x i16> [[TMP3]], i32 4
+; SSE2-NEXT:    [[V4:%.*]] = insertelement <8 x i16> [[V3]], i16 [[TMP8]], i32 4
+; SSE2-NEXT:    [[TMP9:%.*]] = extractelement <8 x i16> [[TMP3]], i32 5
+; SSE2-NEXT:    [[V5:%.*]] = insertelement <8 x i16> [[V4]], i16 [[TMP9]], i32 5
+; SSE2-NEXT:    [[TMP10:%.*]] = extractelement <8 x i16> [[TMP3]], i32 6
+; SSE2-NEXT:    [[V6:%.*]] = insertelement <8 x i16> [[V5]], i16 [[TMP10]], i32 6
+; SSE2-NEXT:    [[TMP11:%.*]] = extractelement <8 x i16> [[TMP3]], i32 7
+; SSE2-NEXT:    [[V7:%.*]] = insertelement <8 x i16> [[V6]], i16 [[TMP11]], i32 7
+; SSE2-NEXT:    ret <8 x i16> [[V7]]
+;
+; SLM-LABEL: @loadext_8i8_to_8i16(
+; SLM-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; SLM-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; SLM-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; SLM-NEXT:    [[P4:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 4
+; SLM-NEXT:    [[P5:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 5
+; SLM-NEXT:    [[P6:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 6
+; SLM-NEXT:    [[P7:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 7
+; SLM-NEXT:    [[I0:%.*]] = load i8, i8* [[P0]], align 1
+; SLM-NEXT:    [[I1:%.*]] = load i8, i8* [[P1]], align 1
+; SLM-NEXT:    [[I2:%.*]] = load i8, i8* [[P2]], align 1
+; SLM-NEXT:    [[I3:%.*]] = load i8, i8* [[P3]], align 1
+; SLM-NEXT:    [[I4:%.*]] = load i8, i8* [[P4]], align 1
+; SLM-NEXT:    [[I5:%.*]] = load i8, i8* [[P5]], align 1
+; SLM-NEXT:    [[I6:%.*]] = load i8, i8* [[P6]], align 1
+; SLM-NEXT:    [[I7:%.*]] = load i8, i8* [[P7]], align 1
+; SLM-NEXT:    [[X0:%.*]] = sext i8 [[I0]] to i16
+; SLM-NEXT:    [[X1:%.*]] = sext i8 [[I1]] to i16
+; SLM-NEXT:    [[X2:%.*]] = sext i8 [[I2]] to i16
+; SLM-NEXT:    [[X3:%.*]] = sext i8 [[I3]] to i16
+; SLM-NEXT:    [[X4:%.*]] = sext i8 [[I4]] to i16
+; SLM-NEXT:    [[X5:%.*]] = sext i8 [[I5]] to i16
+; SLM-NEXT:    [[X6:%.*]] = sext i8 [[I6]] to i16
+; SLM-NEXT:    [[X7:%.*]] = sext i8 [[I7]] to i16
+; SLM-NEXT:    [[V0:%.*]] = insertelement <8 x i16> undef, i16 [[X0]], i32 0
+; SLM-NEXT:    [[V1:%.*]] = insertelement <8 x i16> [[V0]], i16 [[X1]], i32 1
+; SLM-NEXT:    [[V2:%.*]] = insertelement <8 x i16> [[V1]], i16 [[X2]], i32 2
+; SLM-NEXT:    [[V3:%.*]] = insertelement <8 x i16> [[V2]], i16 [[X3]], i32 3
+; SLM-NEXT:    [[V4:%.*]] = insertelement <8 x i16> [[V3]], i16 [[X4]], i32 4
+; SLM-NEXT:    [[V5:%.*]] = insertelement <8 x i16> [[V4]], i16 [[X5]], i32 5
+; SLM-NEXT:    [[V6:%.*]] = insertelement <8 x i16> [[V5]], i16 [[X6]], i32 6
+; SLM-NEXT:    [[V7:%.*]] = insertelement <8 x i16> [[V6]], i16 [[X7]], i32 7
+; SLM-NEXT:    ret <8 x i16> [[V7]]
+;
+; AVX-LABEL: @loadext_8i8_to_8i16(
+; AVX-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; AVX-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; AVX-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; AVX-NEXT:    [[P4:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 4
+; AVX-NEXT:    [[P5:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 5
+; AVX-NEXT:    [[P6:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 6
+; AVX-NEXT:    [[P7:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 7
+; AVX-NEXT:    [[TMP1:%.*]] = bitcast i8* [[P0]] to <8 x i8>*
+; AVX-NEXT:    [[TMP2:%.*]] = load <8 x i8>, <8 x i8>* [[TMP1]], align 1
+; AVX-NEXT:    [[TMP3:%.*]] = sext <8 x i8> [[TMP2]] to <8 x i16>
+; AVX-NEXT:    [[TMP4:%.*]] = extractelement <8 x i16> [[TMP3]], i32 0
+; AVX-NEXT:    [[V0:%.*]] = insertelement <8 x i16> undef, i16 [[TMP4]], i32 0
+; AVX-NEXT:    [[TMP5:%.*]] = extractelement <8 x i16> [[TMP3]], i32 1
+; AVX-NEXT:    [[V1:%.*]] = insertelement <8 x i16> [[V0]], i16 [[TMP5]], i32 1
+; AVX-NEXT:    [[TMP6:%.*]] = extractelement <8 x i16> [[TMP3]], i32 2
+; AVX-NEXT:    [[V2:%.*]] = insertelement <8 x i16> [[V1]], i16 [[TMP6]], i32 2
+; AVX-NEXT:    [[TMP7:%.*]] = extractelement <8 x i16> [[TMP3]], i32 3
+; AVX-NEXT:    [[V3:%.*]] = insertelement <8 x i16> [[V2]], i16 [[TMP7]], i32 3
+; AVX-NEXT:    [[TMP8:%.*]] = extractelement <8 x i16> [[TMP3]], i32 4
+; AVX-NEXT:    [[V4:%.*]] = insertelement <8 x i16> [[V3]], i16 [[TMP8]], i32 4
+; AVX-NEXT:    [[TMP9:%.*]] = extractelement <8 x i16> [[TMP3]], i32 5
+; AVX-NEXT:    [[V5:%.*]] = insertelement <8 x i16> [[V4]], i16 [[TMP9]], i32 5
+; AVX-NEXT:    [[TMP10:%.*]] = extractelement <8 x i16> [[TMP3]], i32 6
+; AVX-NEXT:    [[V6:%.*]] = insertelement <8 x i16> [[V5]], i16 [[TMP10]], i32 6
+; AVX-NEXT:    [[TMP11:%.*]] = extractelement <8 x i16> [[TMP3]], i32 7
+; AVX-NEXT:    [[V7:%.*]] = insertelement <8 x i16> [[V6]], i16 [[TMP11]], i32 7
+; AVX-NEXT:    ret <8 x i16> [[V7]]
 ;
   %p1 = getelementptr inbounds i8, i8* %p0, i64 1
   %p2 = getelementptr inbounds i8, i8* %p0, i64 2
@@ -262,125 +314,97 @@ define <8 x i16> @loadext_8i8_to_8i16(i8* %p0) {
 }
 
 define <8 x i32> @loadext_8i8_to_8i32(i8* %p0) {
-; SSE-LABEL: @loadext_8i8_to_8i32(
-; SSE-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
-; SSE-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
-; SSE-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
-; SSE-NEXT:    [[P4:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 4
-; SSE-NEXT:    [[P5:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 5
-; SSE-NEXT:    [[P6:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 6
-; SSE-NEXT:    [[P7:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 7
-; SSE-NEXT:    [[TMP1:%.*]] = bitcast i8* [[P0]] to <8 x i8>*
-; SSE-NEXT:    [[TMP2:%.*]] = load <8 x i8>, <8 x i8>* [[TMP1]], align 1
-; SSE-NEXT:    [[TMP3:%.*]] = sext <8 x i8> [[TMP2]] to <8 x i32>
-; SSE-NEXT:    [[TMP4:%.*]] = extractelement <8 x i32> [[TMP3]], i32 0
-; SSE-NEXT:    [[V0:%.*]] = insertelement <8 x i32> undef, i32 [[TMP4]], i32 0
-; SSE-NEXT:    [[TMP5:%.*]] = extractelement <8 x i32> [[TMP3]], i32 1
-; SSE-NEXT:    [[V1:%.*]] = insertelement <8 x i32> [[V0]], i32 [[TMP5]], i32 1
-; SSE-NEXT:    [[TMP6:%.*]] = extractelement <8 x i32> [[TMP3]], i32 2
-; SSE-NEXT:    [[V2:%.*]] = insertelement <8 x i32> [[V1]], i32 [[TMP6]], i32 2
-; SSE-NEXT:    [[TMP7:%.*]] = extractelement <8 x i32> [[TMP3]], i32 3
-; SSE-NEXT:    [[V3:%.*]] = insertelement <8 x i32> [[V2]], i32 [[TMP7]], i32 3
-; SSE-NEXT:    [[TMP8:%.*]] = extractelement <8 x i32> [[TMP3]], i32 4
-; SSE-NEXT:    [[V4:%.*]] = insertelement <8 x i32> [[V3]], i32 [[TMP8]], i32 4
-; SSE-NEXT:    [[TMP9:%.*]] = extractelement <8 x i32> [[TMP3]], i32 5
-; SSE-NEXT:    [[V5:%.*]] = insertelement <8 x i32> [[V4]], i32 [[TMP9]], i32 5
-; SSE-NEXT:    [[TMP10:%.*]] = extractelement <8 x i32> [[TMP3]], i32 6
-; SSE-NEXT:    [[V6:%.*]] = insertelement <8 x i32> [[V5]], i32 [[TMP10]], i32 6
-; SSE-NEXT:    [[TMP11:%.*]] = extractelement <8 x i32> [[TMP3]], i32 7
-; SSE-NEXT:    [[V7:%.*]] = insertelement <8 x i32> [[V6]], i32 [[TMP11]], i32 7
-; SSE-NEXT:    ret <8 x i32> [[V7]]
-;
-; AVX1-LABEL: @loadext_8i8_to_8i32(
-; AVX1-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
-; AVX1-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
-; AVX1-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
-; AVX1-NEXT:    [[P4:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 4
-; AVX1-NEXT:    [[P5:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 5
-; AVX1-NEXT:    [[P6:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 6
-; AVX1-NEXT:    [[P7:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 7
-; AVX1-NEXT:    [[TMP1:%.*]] = bitcast i8* [[P0]] to <4 x i8>*
-; AVX1-NEXT:    [[TMP2:%.*]] = load <4 x i8>, <4 x i8>* [[TMP1]], align 1
-; AVX1-NEXT:    [[I4:%.*]] = load i8, i8* [[P4]], align 1
-; AVX1-NEXT:    [[I5:%.*]] = load i8, i8* [[P5]], align 1
-; AVX1-NEXT:    [[I6:%.*]] = load i8, i8* [[P6]], align 1
-; AVX1-NEXT:    [[I7:%.*]] = load i8, i8* [[P7]], align 1
-; AVX1-NEXT:    [[TMP3:%.*]] = sext <4 x i8> [[TMP2]] to <4 x i32>
-; AVX1-NEXT:    [[X4:%.*]] = sext i8 [[I4]] to i32
-; AVX1-NEXT:    [[X5:%.*]] = sext i8 [[I5]] to i32
-; AVX1-NEXT:    [[X6:%.*]] = sext i8 [[I6]] to i32
-; AVX1-NEXT:    [[X7:%.*]] = sext i8 [[I7]] to i32
-; AVX1-NEXT:    [[TMP4:%.*]] = extractelement <4 x i32> [[TMP3]], i32 0
-; AVX1-NEXT:    [[V0:%.*]] = insertelement <8 x i32> undef, i32 [[TMP4]], i32 0
-; AVX1-NEXT:    [[TMP5:%.*]] = extractelement <4 x i32> [[TMP3]], i32 1
-; AVX1-NEXT:    [[V1:%.*]] = insertelement <8 x i32> [[V0]], i32 [[TMP5]], i32 1
-; AVX1-NEXT:    [[TMP6:%.*]] = extractelement <4 x i32> [[TMP3]], i32 2
-; AVX1-NEXT:    [[V2:%.*]] = insertelement <8 x i32> [[V1]], i32 [[TMP6]], i32 2
-; AVX1-NEXT:    [[TMP7:%.*]] = extractelement <4 x i32> [[TMP3]], i32 3
-; AVX1-NEXT:    [[V3:%.*]] = insertelement <8 x i32> [[V2]], i32 [[TMP7]], i32 3
-; AVX1-NEXT:    [[V4:%.*]] = insertelement <8 x i32> [[V3]], i32 [[X4]], i32 4
-; AVX1-NEXT:    [[V5:%.*]] = insertelement <8 x i32> [[V4]], i32 [[X5]], i32 5
-; AVX1-NEXT:    [[V6:%.*]] = insertelement <8 x i32> [[V5]], i32 [[X6]], i32 6
-; AVX1-NEXT:    [[V7:%.*]] = insertelement <8 x i32> [[V6]], i32 [[X7]], i32 7
-; AVX1-NEXT:    ret <8 x i32> [[V7]]
-;
-; AVX2-LABEL: @loadext_8i8_to_8i32(
-; AVX2-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
-; AVX2-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
-; AVX2-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
-; AVX2-NEXT:    [[P4:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 4
-; AVX2-NEXT:    [[P5:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 5
-; AVX2-NEXT:    [[P6:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 6
-; AVX2-NEXT:    [[P7:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 7
-; AVX2-NEXT:    [[TMP1:%.*]] = bitcast i8* [[P0]] to <8 x i8>*
-; AVX2-NEXT:    [[TMP2:%.*]] = load <8 x i8>, <8 x i8>* [[TMP1]], align 1
-; AVX2-NEXT:    [[TMP3:%.*]] = sext <8 x i8> [[TMP2]] to <8 x i32>
-; AVX2-NEXT:    [[TMP4:%.*]] = extractelement <8 x i32> [[TMP3]], i32 0
-; AVX2-NEXT:    [[V0:%.*]] = insertelement <8 x i32> undef, i32 [[TMP4]], i32 0
-; AVX2-NEXT:    [[TMP5:%.*]] = extractelement <8 x i32> [[TMP3]], i32 1
-; AVX2-NEXT:    [[V1:%.*]] = insertelement <8 x i32> [[V0]], i32 [[TMP5]], i32 1
-; AVX2-NEXT:    [[TMP6:%.*]] = extractelement <8 x i32> [[TMP3]], i32 2
-; AVX2-NEXT:    [[V2:%.*]] = insertelement <8 x i32> [[V1]], i32 [[TMP6]], i32 2
-; AVX2-NEXT:    [[TMP7:%.*]] = extractelement <8 x i32> [[TMP3]], i32 3
-; AVX2-NEXT:    [[V3:%.*]] = insertelement <8 x i32> [[V2]], i32 [[TMP7]], i32 3
-; AVX2-NEXT:    [[TMP8:%.*]] = extractelement <8 x i32> [[TMP3]], i32 4
-; AVX2-NEXT:    [[V4:%.*]] = insertelement <8 x i32> [[V3]], i32 [[TMP8]], i32 4
-; AVX2-NEXT:    [[TMP9:%.*]] = extractelement <8 x i32> [[TMP3]], i32 5
-; AVX2-NEXT:    [[V5:%.*]] = insertelement <8 x i32> [[V4]], i32 [[TMP9]], i32 5
-; AVX2-NEXT:    [[TMP10:%.*]] = extractelement <8 x i32> [[TMP3]], i32 6
-; AVX2-NEXT:    [[V6:%.*]] = insertelement <8 x i32> [[V5]], i32 [[TMP10]], i32 6
-; AVX2-NEXT:    [[TMP11:%.*]] = extractelement <8 x i32> [[TMP3]], i32 7
-; AVX2-NEXT:    [[V7:%.*]] = insertelement <8 x i32> [[V6]], i32 [[TMP11]], i32 7
-; AVX2-NEXT:    ret <8 x i32> [[V7]]
-;
-; AVX512-LABEL: @loadext_8i8_to_8i32(
-; AVX512-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
-; AVX512-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
-; AVX512-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
-; AVX512-NEXT:    [[P4:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 4
-; AVX512-NEXT:    [[P5:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 5
-; AVX512-NEXT:    [[P6:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 6
-; AVX512-NEXT:    [[P7:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 7
-; AVX512-NEXT:    [[TMP1:%.*]] = bitcast i8* [[P0]] to <8 x i8>*
-; AVX512-NEXT:    [[TMP2:%.*]] = load <8 x i8>, <8 x i8>* [[TMP1]], align 1
-; AVX512-NEXT:    [[TMP3:%.*]] = sext <8 x i8> [[TMP2]] to <8 x i32>
-; AVX512-NEXT:    [[TMP4:%.*]] = extractelement <8 x i32> [[TMP3]], i32 0
-; AVX512-NEXT:    [[V0:%.*]] = insertelement <8 x i32> undef, i32 [[TMP4]], i32 0
-; AVX512-NEXT:    [[TMP5:%.*]] = extractelement <8 x i32> [[TMP3]], i32 1
-; AVX512-NEXT:    [[V1:%.*]] = insertelement <8 x i32> [[V0]], i32 [[TMP5]], i32 1
-; AVX512-NEXT:    [[TMP6:%.*]] = extractelement <8 x i32> [[TMP3]], i32 2
-; AVX512-NEXT:    [[V2:%.*]] = insertelement <8 x i32> [[V1]], i32 [[TMP6]], i32 2
-; AVX512-NEXT:    [[TMP7:%.*]] = extractelement <8 x i32> [[TMP3]], i32 3
-; AVX512-NEXT:    [[V3:%.*]] = insertelement <8 x i32> [[V2]], i32 [[TMP7]], i32 3
-; AVX512-NEXT:    [[TMP8:%.*]] = extractelement <8 x i32> [[TMP3]], i32 4
-; AVX512-NEXT:    [[V4:%.*]] = insertelement <8 x i32> [[V3]], i32 [[TMP8]], i32 4
-; AVX512-NEXT:    [[TMP9:%.*]] = extractelement <8 x i32> [[TMP3]], i32 5
-; AVX512-NEXT:    [[V5:%.*]] = insertelement <8 x i32> [[V4]], i32 [[TMP9]], i32 5
-; AVX512-NEXT:    [[TMP10:%.*]] = extractelement <8 x i32> [[TMP3]], i32 6
-; AVX512-NEXT:    [[V6:%.*]] = insertelement <8 x i32> [[V5]], i32 [[TMP10]], i32 6
-; AVX512-NEXT:    [[TMP11:%.*]] = extractelement <8 x i32> [[TMP3]], i32 7
-; AVX512-NEXT:    [[V7:%.*]] = insertelement <8 x i32> [[V6]], i32 [[TMP11]], i32 7
-; AVX512-NEXT:    ret <8 x i32> [[V7]]
+; SSE2-LABEL: @loadext_8i8_to_8i32(
+; SSE2-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; SSE2-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; SSE2-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; SSE2-NEXT:    [[P4:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 4
+; SSE2-NEXT:    [[P5:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 5
+; SSE2-NEXT:    [[P6:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 6
+; SSE2-NEXT:    [[P7:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 7
+; SSE2-NEXT:    [[TMP1:%.*]] = bitcast i8* [[P0]] to <8 x i8>*
+; SSE2-NEXT:    [[TMP2:%.*]] = load <8 x i8>, <8 x i8>* [[TMP1]], align 1
+; SSE2-NEXT:    [[TMP3:%.*]] = sext <8 x i8> [[TMP2]] to <8 x i32>
+; SSE2-NEXT:    [[TMP4:%.*]] = extractelement <8 x i32> [[TMP3]], i32 0
+; SSE2-NEXT:    [[V0:%.*]] = insertelement <8 x i32> undef, i32 [[TMP4]], i32 0
+; SSE2-NEXT:    [[TMP5:%.*]] = extractelement <8 x i32> [[TMP3]], i32 1
+; SSE2-NEXT:    [[V1:%.*]] = insertelement <8 x i32> [[V0]], i32 [[TMP5]], i32 1
+; SSE2-NEXT:    [[TMP6:%.*]] = extractelement <8 x i32> [[TMP3]], i32 2
+; SSE2-NEXT:    [[V2:%.*]] = insertelement <8 x i32> [[V1]], i32 [[TMP6]], i32 2
+; SSE2-NEXT:    [[TMP7:%.*]] = extractelement <8 x i32> [[TMP3]], i32 3
+; SSE2-NEXT:    [[V3:%.*]] = insertelement <8 x i32> [[V2]], i32 [[TMP7]], i32 3
+; SSE2-NEXT:    [[TMP8:%.*]] = extractelement <8 x i32> [[TMP3]], i32 4
+; SSE2-NEXT:    [[V4:%.*]] = insertelement <8 x i32> [[V3]], i32 [[TMP8]], i32 4
+; SSE2-NEXT:    [[TMP9:%.*]] = extractelement <8 x i32> [[TMP3]], i32 5
+; SSE2-NEXT:    [[V5:%.*]] = insertelement <8 x i32> [[V4]], i32 [[TMP9]], i32 5
+; SSE2-NEXT:    [[TMP10:%.*]] = extractelement <8 x i32> [[TMP3]], i32 6
+; SSE2-NEXT:    [[V6:%.*]] = insertelement <8 x i32> [[V5]], i32 [[TMP10]], i32 6
+; SSE2-NEXT:    [[TMP11:%.*]] = extractelement <8 x i32> [[TMP3]], i32 7
+; SSE2-NEXT:    [[V7:%.*]] = insertelement <8 x i32> [[V6]], i32 [[TMP11]], i32 7
+; SSE2-NEXT:    ret <8 x i32> [[V7]]
+;
+; SLM-LABEL: @loadext_8i8_to_8i32(
+; SLM-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; SLM-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; SLM-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; SLM-NEXT:    [[P4:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 4
+; SLM-NEXT:    [[P5:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 5
+; SLM-NEXT:    [[P6:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 6
+; SLM-NEXT:    [[P7:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 7
+; SLM-NEXT:    [[I0:%.*]] = load i8, i8* [[P0]], align 1
+; SLM-NEXT:    [[I1:%.*]] = load i8, i8* [[P1]], align 1
+; SLM-NEXT:    [[I2:%.*]] = load i8, i8* [[P2]], align 1
+; SLM-NEXT:    [[I3:%.*]] = load i8, i8* [[P3]], align 1
+; SLM-NEXT:    [[I4:%.*]] = load i8, i8* [[P4]], align 1
+; SLM-NEXT:    [[I5:%.*]] = load i8, i8* [[P5]], align 1
+; SLM-NEXT:    [[I6:%.*]] = load i8, i8* [[P6]], align 1
+; SLM-NEXT:    [[I7:%.*]] = load i8, i8* [[P7]], align 1
+; SLM-NEXT:    [[X0:%.*]] = sext i8 [[I0]] to i32
+; SLM-NEXT:    [[X1:%.*]] = sext i8 [[I1]] to i32
+; SLM-NEXT:    [[X2:%.*]] = sext i8 [[I2]] to i32
+; SLM-NEXT:    [[X3:%.*]] = sext i8 [[I3]] to i32
+; SLM-NEXT:    [[X4:%.*]] = sext i8 [[I4]] to i32
+; SLM-NEXT:    [[X5:%.*]] = sext i8 [[I5]] to i32
+; SLM-NEXT:    [[X6:%.*]] = sext i8 [[I6]] to i32
+; SLM-NEXT:    [[X7:%.*]] = sext i8 [[I7]] to i32
+; SLM-NEXT:    [[V0:%.*]] = insertelement <8 x i32> undef, i32 [[X0]], i32 0
+; SLM-NEXT:    [[V1:%.*]] = insertelement <8 x i32> [[V0]], i32 [[X1]], i32 1
+; SLM-NEXT:    [[V2:%.*]] = insertelement <8 x i32> [[V1]], i32 [[X2]], i32 2
+; SLM-NEXT:    [[V3:%.*]] = insertelement <8 x i32> [[V2]], i32 [[X3]], i32 3
+; SLM-NEXT:    [[V4:%.*]] = insertelement <8 x i32> [[V3]], i32 [[X4]], i32 4
+; SLM-NEXT:    [[V5:%.*]] = insertelement <8 x i32> [[V4]], i32 [[X5]], i32 5
+; SLM-NEXT:    [[V6:%.*]] = insertelement <8 x i32> [[V5]], i32 [[X6]], i32 6
+; SLM-NEXT:    [[V7:%.*]] = insertelement <8 x i32> [[V6]], i32 [[X7]], i32 7
+; SLM-NEXT:    ret <8 x i32> [[V7]]
+;
+; AVX-LABEL: @loadext_8i8_to_8i32(
+; AVX-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; AVX-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; AVX-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; AVX-NEXT:    [[P4:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 4
+; AVX-NEXT:    [[P5:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 5
+; AVX-NEXT:    [[P6:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 6
+; AVX-NEXT:    [[P7:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 7
+; AVX-NEXT:    [[TMP1:%.*]] = bitcast i8* [[P0]] to <8 x i8>*
+; AVX-NEXT:    [[TMP2:%.*]] = load <8 x i8>, <8 x i8>* [[TMP1]], align 1
+; AVX-NEXT:    [[TMP3:%.*]] = sext <8 x i8> [[TMP2]] to <8 x i32>
+; AVX-NEXT:    [[TMP4:%.*]] = extractelement <8 x i32> [[TMP3]], i32 0
+; AVX-NEXT:    [[V0:%.*]] = insertelement <8 x i32> undef, i32 [[TMP4]], i32 0
+; AVX-NEXT:    [[TMP5:%.*]] = extractelement <8 x i32> [[TMP3]], i32 1
+; AVX-NEXT:    [[V1:%.*]] = insertelement <8 x i32> [[V0]], i32 [[TMP5]], i32 1
+; AVX-NEXT:    [[TMP6:%.*]] = extractelement <8 x i32> [[TMP3]], i32 2
+; AVX-NEXT:    [[V2:%.*]] = insertelement <8 x i32> [[V1]], i32 [[TMP6]], i32 2
+; AVX-NEXT:    [[TMP7:%.*]] = extractelement <8 x i32> [[TMP3]], i32 3
+; AVX-NEXT:    [[V3:%.*]] = insertelement <8 x i32> [[V2]], i32 [[TMP7]], i32 3
+; AVX-NEXT:    [[TMP8:%.*]] = extractelement <8 x i32> [[TMP3]], i32 4
+; AVX-NEXT:    [[V4:%.*]] = insertelement <8 x i32> [[V3]], i32 [[TMP8]], i32 4
+; AVX-NEXT:    [[TMP9:%.*]] = extractelement <8 x i32> [[TMP3]], i32 5
+; AVX-NEXT:    [[V5:%.*]] = insertelement <8 x i32> [[V4]], i32 [[TMP9]], i32 5
+; AVX-NEXT:    [[TMP10:%.*]] = extractelement <8 x i32> [[TMP3]], i32 6
+; AVX-NEXT:    [[V6:%.*]] = insertelement <8 x i32> [[V5]], i32 [[TMP10]], i32 6
+; AVX-NEXT:    [[TMP11:%.*]] = extractelement <8 x i32> [[TMP3]], i32 7
+; AVX-NEXT:    [[V7:%.*]] = insertelement <8 x i32> [[V6]], i32 [[TMP11]], i32 7
+; AVX-NEXT:    ret <8 x i32> [[V7]]
 ;
   %p1 = getelementptr inbounds i8, i8* %p0, i64 1
   %p2 = getelementptr inbounds i8, i8* %p0, i64 2
@@ -417,58 +441,177 @@ define <8 x i32> @loadext_8i8_to_8i32(i8* %p0) {
 }
 
 define <16 x i16> @loadext_16i8_to_16i16(i8* %p0) {
-; CHECK-LABEL: @loadext_16i8_to_16i16(
-; CHECK-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
-; CHECK-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
-; CHECK-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
-; CHECK-NEXT:    [[P4:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 4
-; CHECK-NEXT:    [[P5:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 5
-; CHECK-NEXT:    [[P6:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 6
-; CHECK-NEXT:    [[P7:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 7
-; CHECK-NEXT:    [[P8:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 8
-; CHECK-NEXT:    [[P9:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 9
-; CHECK-NEXT:    [[P10:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 10
-; CHECK-NEXT:    [[P11:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 11
-; CHECK-NEXT:    [[P12:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 12
-; CHECK-NEXT:    [[P13:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 13
-; CHECK-NEXT:    [[P14:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 14
-; CHECK-NEXT:    [[P15:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 15
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8* [[P0]] to <16 x i8>*
-; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* [[TMP1]], align 1
-; CHECK-NEXT:    [[TMP3:%.*]] = sext <16 x i8> [[TMP2]] to <16 x i16>
-; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <16 x i16> [[TMP3]], i32 0
-; CHECK-NEXT:    [[V0:%.*]] = insertelement <16 x i16> undef, i16 [[TMP4]], i32 0
-; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <16 x i16> [[TMP3]], i32 1
-; CHECK-NEXT:    [[V1:%.*]] = insertelement <16 x i16> [[V0]], i16 [[TMP5]], i32 1
-; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <16 x i16> [[TMP3]], i32 2
-; CHECK-NEXT:    [[V2:%.*]] = insertelement <16 x i16> [[V1]], i16 [[TMP6]], i32 2
-; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <16 x i16> [[TMP3]], i32 3
-; CHECK-NEXT:    [[V3:%.*]] = insertelement <16 x i16> [[V2]], i16 [[TMP7]], i32 3
-; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <16 x i16> [[TMP3]], i32 4
-; CHECK-NEXT:    [[V4:%.*]] = insertelement <16 x i16> [[V3]], i16 [[TMP8]], i32 4
-; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <16 x i16> [[TMP3]], i32 5
-; CHECK-NEXT:    [[V5:%.*]] = insertelement <16 x i16> [[V4]], i16 [[TMP9]], i32 5
-; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <16 x i16> [[TMP3]], i32 6
-; CHECK-NEXT:    [[V6:%.*]] = insertelement <16 x i16> [[V5]], i16 [[TMP10]], i32 6
-; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <16 x i16> [[TMP3]], i32 7
-; CHECK-NEXT:    [[V7:%.*]] = insertelement <16 x i16> [[V6]], i16 [[TMP11]], i32 7
-; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <16 x i16> [[TMP3]], i32 8
-; CHECK-NEXT:    [[V8:%.*]] = insertelement <16 x i16> [[V7]], i16 [[TMP12]], i32 8
-; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <16 x i16> [[TMP3]], i32 9
-; CHECK-NEXT:    [[V9:%.*]] = insertelement <16 x i16> [[V8]], i16 [[TMP13]], i32 9
-; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <16 x i16> [[TMP3]], i32 10
-; CHECK-NEXT:    [[V10:%.*]] = insertelement <16 x i16> [[V9]], i16 [[TMP14]], i32 10
-; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <16 x i16> [[TMP3]], i32 11
-; CHECK-NEXT:    [[V11:%.*]] = insertelement <16 x i16> [[V10]], i16 [[TMP15]], i32 11
-; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <16 x i16> [[TMP3]], i32 12
-; CHECK-NEXT:    [[V12:%.*]] = insertelement <16 x i16> [[V11]], i16 [[TMP16]], i32 12
-; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <16 x i16> [[TMP3]], i32 13
-; CHECK-NEXT:    [[V13:%.*]] = insertelement <16 x i16> [[V12]], i16 [[TMP17]], i32 13
-; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <16 x i16> [[TMP3]], i32 14
-; CHECK-NEXT:    [[V14:%.*]] = insertelement <16 x i16> [[V13]], i16 [[TMP18]], i32 14
-; CHECK-NEXT:    [[TMP19:%.*]] = extractelement <16 x i16> [[TMP3]], i32 15
-; CHECK-NEXT:    [[V15:%.*]] = insertelement <16 x i16> [[V14]], i16 [[TMP19]], i32 15
-; CHECK-NEXT:    ret <16 x i16> [[V15]]
+; SSE2-LABEL: @loadext_16i8_to_16i16(
+; SSE2-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; SSE2-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; SSE2-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; SSE2-NEXT:    [[P4:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 4
+; SSE2-NEXT:    [[P5:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 5
+; SSE2-NEXT:    [[P6:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 6
+; SSE2-NEXT:    [[P7:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 7
+; SSE2-NEXT:    [[P8:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 8
+; SSE2-NEXT:    [[P9:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 9
+; SSE2-NEXT:    [[P10:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 10
+; SSE2-NEXT:    [[P11:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 11
+; SSE2-NEXT:    [[P12:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 12
+; SSE2-NEXT:    [[P13:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 13
+; SSE2-NEXT:    [[P14:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 14
+; SSE2-NEXT:    [[P15:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 15
+; SSE2-NEXT:    [[TMP1:%.*]] = bitcast i8* [[P0]] to <16 x i8>*
+; SSE2-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* [[TMP1]], align 1
+; SSE2-NEXT:    [[TMP3:%.*]] = sext <16 x i8> [[TMP2]] to <16 x i16>
+; SSE2-NEXT:    [[TMP4:%.*]] = extractelement <16 x i16> [[TMP3]], i32 0
+; SSE2-NEXT:    [[V0:%.*]] = insertelement <16 x i16> undef, i16 [[TMP4]], i32 0
+; SSE2-NEXT:    [[TMP5:%.*]] = extractelement <16 x i16> [[TMP3]], i32 1
+; SSE2-NEXT:    [[V1:%.*]] = insertelement <16 x i16> [[V0]], i16 [[TMP5]], i32 1
+; SSE2-NEXT:    [[TMP6:%.*]] = extractelement <16 x i16> [[TMP3]], i32 2
+; SSE2-NEXT:    [[V2:%.*]] = insertelement <16 x i16> [[V1]], i16 [[TMP6]], i32 2
+; SSE2-NEXT:    [[TMP7:%.*]] = extractelement <16 x i16> [[TMP3]], i32 3
+; SSE2-NEXT:    [[V3:%.*]] = insertelement <16 x i16> [[V2]], i16 [[TMP7]], i32 3
+; SSE2-NEXT:    [[TMP8:%.*]] = extractelement <16 x i16> [[TMP3]], i32 4
+; SSE2-NEXT:    [[V4:%.*]] = insertelement <16 x i16> [[V3]], i16 [[TMP8]], i32 4
+; SSE2-NEXT:    [[TMP9:%.*]] = extractelement <16 x i16> [[TMP3]], i32 5
+; SSE2-NEXT:    [[V5:%.*]] = insertelement <16 x i16> [[V4]], i16 [[TMP9]], i32 5
+; SSE2-NEXT:    [[TMP10:%.*]] = extractelement <16 x i16> [[TMP3]], i32 6
+; SSE2-NEXT:    [[V6:%.*]] = insertelement <16 x i16> [[V5]], i16 [[TMP10]], i32 6
+; SSE2-NEXT:    [[TMP11:%.*]] = extractelement <16 x i16> [[TMP3]], i32 7
+; SSE2-NEXT:    [[V7:%.*]] = insertelement <16 x i16> [[V6]], i16 [[TMP11]], i32 7
+; SSE2-NEXT:    [[TMP12:%.*]] = extractelement <16 x i16> [[TMP3]], i32 8
+; SSE2-NEXT:    [[V8:%.*]] = insertelement <16 x i16> [[V7]], i16 [[TMP12]], i32 8
+; SSE2-NEXT:    [[TMP13:%.*]] = extractelement <16 x i16> [[TMP3]], i32 9
+; SSE2-NEXT:    [[V9:%.*]] = insertelement <16 x i16> [[V8]], i16 [[TMP13]], i32 9
+; SSE2-NEXT:    [[TMP14:%.*]] = extractelement <16 x i16> [[TMP3]], i32 10
+; SSE2-NEXT:    [[V10:%.*]] = insertelement <16 x i16> [[V9]], i16 [[TMP14]], i32 10
+; SSE2-NEXT:    [[TMP15:%.*]] = extractelement <16 x i16> [[TMP3]], i32 11
+; SSE2-NEXT:    [[V11:%.*]] = insertelement <16 x i16> [[V10]], i16 [[TMP15]], i32 11
+; SSE2-NEXT:    [[TMP16:%.*]] = extractelement <16 x i16> [[TMP3]], i32 12
+; SSE2-NEXT:    [[V12:%.*]] = insertelement <16 x i16> [[V11]], i16 [[TMP16]], i32 12
+; SSE2-NEXT:    [[TMP17:%.*]] = extractelement <16 x i16> [[TMP3]], i32 13
+; SSE2-NEXT:    [[V13:%.*]] = insertelement <16 x i16> [[V12]], i16 [[TMP17]], i32 13
+; SSE2-NEXT:    [[TMP18:%.*]] = extractelement <16 x i16> [[TMP3]], i32 14
+; SSE2-NEXT:    [[V14:%.*]] = insertelement <16 x i16> [[V13]], i16 [[TMP18]], i32 14
+; SSE2-NEXT:    [[TMP19:%.*]] = extractelement <16 x i16> [[TMP3]], i32 15
+; SSE2-NEXT:    [[V15:%.*]] = insertelement <16 x i16> [[V14]], i16 [[TMP19]], i32 15
+; SSE2-NEXT:    ret <16 x i16> [[V15]]
+;
+; SLM-LABEL: @loadext_16i8_to_16i16(
+; SLM-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; SLM-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; SLM-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; SLM-NEXT:    [[P4:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 4
+; SLM-NEXT:    [[P5:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 5
+; SLM-NEXT:    [[P6:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 6
+; SLM-NEXT:    [[P7:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 7
+; SLM-NEXT:    [[P8:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 8
+; SLM-NEXT:    [[P9:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 9
+; SLM-NEXT:    [[P10:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 10
+; SLM-NEXT:    [[P11:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 11
+; SLM-NEXT:    [[P12:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 12
+; SLM-NEXT:    [[P13:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 13
+; SLM-NEXT:    [[P14:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 14
+; SLM-NEXT:    [[P15:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 15
+; SLM-NEXT:    [[I0:%.*]] = load i8, i8* [[P0]], align 1
+; SLM-NEXT:    [[I1:%.*]] = load i8, i8* [[P1]], align 1
+; SLM-NEXT:    [[I2:%.*]] = load i8, i8* [[P2]], align 1
+; SLM-NEXT:    [[I3:%.*]] = load i8, i8* [[P3]], align 1
+; SLM-NEXT:    [[I4:%.*]] = load i8, i8* [[P4]], align 1
+; SLM-NEXT:    [[I5:%.*]] = load i8, i8* [[P5]], align 1
+; SLM-NEXT:    [[I6:%.*]] = load i8, i8* [[P6]], align 1
+; SLM-NEXT:    [[I7:%.*]] = load i8, i8* [[P7]], align 1
+; SLM-NEXT:    [[I8:%.*]] = load i8, i8* [[P8]], align 1
+; SLM-NEXT:    [[I9:%.*]] = load i8, i8* [[P9]], align 1
+; SLM-NEXT:    [[I10:%.*]] = load i8, i8* [[P10]], align 1
+; SLM-NEXT:    [[I11:%.*]] = load i8, i8* [[P11]], align 1
+; SLM-NEXT:    [[I12:%.*]] = load i8, i8* [[P12]], align 1
+; SLM-NEXT:    [[I13:%.*]] = load i8, i8* [[P13]], align 1
+; SLM-NEXT:    [[I14:%.*]] = load i8, i8* [[P14]], align 1
+; SLM-NEXT:    [[I15:%.*]] = load i8, i8* [[P15]], align 1
+; SLM-NEXT:    [[X0:%.*]] = sext i8 [[I0]] to i16
+; SLM-NEXT:    [[X1:%.*]] = sext i8 [[I1]] to i16
+; SLM-NEXT:    [[X2:%.*]] = sext i8 [[I2]] to i16
+; SLM-NEXT:    [[X3:%.*]] = sext i8 [[I3]] to i16
+; SLM-NEXT:    [[X4:%.*]] = sext i8 [[I4]] to i16
+; SLM-NEXT:    [[X5:%.*]] = sext i8 [[I5]] to i16
+; SLM-NEXT:    [[X6:%.*]] = sext i8 [[I6]] to i16
+; SLM-NEXT:    [[X7:%.*]] = sext i8 [[I7]] to i16
+; SLM-NEXT:    [[X8:%.*]] = sext i8 [[I8]] to i16
+; SLM-NEXT:    [[X9:%.*]] = sext i8 [[I9]] to i16
+; SLM-NEXT:    [[X10:%.*]] = sext i8 [[I10]] to i16
+; SLM-NEXT:    [[X11:%.*]] = sext i8 [[I11]] to i16
+; SLM-NEXT:    [[X12:%.*]] = sext i8 [[I12]] to i16
+; SLM-NEXT:    [[X13:%.*]] = sext i8 [[I13]] to i16
+; SLM-NEXT:    [[X14:%.*]] = sext i8 [[I14]] to i16
+; SLM-NEXT:    [[X15:%.*]] = sext i8 [[I15]] to i16
+; SLM-NEXT:    [[V0:%.*]] = insertelement <16 x i16> undef, i16 [[X0]], i32 0
+; SLM-NEXT:    [[V1:%.*]] = insertelement <16 x i16> [[V0]], i16 [[X1]], i32 1
+; SLM-NEXT:    [[V2:%.*]] = insertelement <16 x i16> [[V1]], i16 [[X2]], i32 2
+; SLM-NEXT:    [[V3:%.*]] = insertelement <16 x i16> [[V2]], i16 [[X3]], i32 3
+; SLM-NEXT:    [[V4:%.*]] = insertelement <16 x i16> [[V3]], i16 [[X4]], i32 4
+; SLM-NEXT:    [[V5:%.*]] = insertelement <16 x i16> [[V4]], i16 [[X5]], i32 5
+; SLM-NEXT:    [[V6:%.*]] = insertelement <16 x i16> [[V5]], i16 [[X6]], i32 6
+; SLM-NEXT:    [[V7:%.*]] = insertelement <16 x i16> [[V6]], i16 [[X7]], i32 7
+; SLM-NEXT:    [[V8:%.*]] = insertelement <16 x i16> [[V7]], i16 [[X8]], i32 8
+; SLM-NEXT:    [[V9:%.*]] = insertelement <16 x i16> [[V8]], i16 [[X9]], i32 9
+; SLM-NEXT:    [[V10:%.*]] = insertelement <16 x i16> [[V9]], i16 [[X10]], i32 10
+; SLM-NEXT:    [[V11:%.*]] = insertelement <16 x i16> [[V10]], i16 [[X11]], i32 11
+; SLM-NEXT:    [[V12:%.*]] = insertelement <16 x i16> [[V11]], i16 [[X12]], i32 12
+; SLM-NEXT:    [[V13:%.*]] = insertelement <16 x i16> [[V12]], i16 [[X13]], i32 13
+; SLM-NEXT:    [[V14:%.*]] = insertelement <16 x i16> [[V13]], i16 [[X14]], i32 14
+; SLM-NEXT:    [[V15:%.*]] = insertelement <16 x i16> [[V14]], i16 [[X15]], i32 15
+; SLM-NEXT:    ret <16 x i16> [[V15]]
+;
+; AVX-LABEL: @loadext_16i8_to_16i16(
+; AVX-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; AVX-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; AVX-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; AVX-NEXT:    [[P4:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 4
+; AVX-NEXT:    [[P5:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 5
+; AVX-NEXT:    [[P6:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 6
+; AVX-NEXT:    [[P7:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 7
+; AVX-NEXT:    [[P8:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 8
+; AVX-NEXT:    [[P9:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 9
+; AVX-NEXT:    [[P10:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 10
+; AVX-NEXT:    [[P11:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 11
+; AVX-NEXT:    [[P12:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 12
+; AVX-NEXT:    [[P13:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 13
+; AVX-NEXT:    [[P14:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 14
+; AVX-NEXT:    [[P15:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 15
+; AVX-NEXT:    [[TMP1:%.*]] = bitcast i8* [[P0]] to <16 x i8>*
+; AVX-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* [[TMP1]], align 1
+; AVX-NEXT:    [[TMP3:%.*]] = sext <16 x i8> [[TMP2]] to <16 x i16>
+; AVX-NEXT:    [[TMP4:%.*]] = extractelement <16 x i16> [[TMP3]], i32 0
+; AVX-NEXT:    [[V0:%.*]] = insertelement <16 x i16> undef, i16 [[TMP4]], i32 0
+; AVX-NEXT:    [[TMP5:%.*]] = extractelement <16 x i16> [[TMP3]], i32 1
+; AVX-NEXT:    [[V1:%.*]] = insertelement <16 x i16> [[V0]], i16 [[TMP5]], i32 1
+; AVX-NEXT:    [[TMP6:%.*]] = extractelement <16 x i16> [[TMP3]], i32 2
+; AVX-NEXT:    [[V2:%.*]] = insertelement <16 x i16> [[V1]], i16 [[TMP6]], i32 2
+; AVX-NEXT:    [[TMP7:%.*]] = extractelement <16 x i16> [[TMP3]], i32 3
+; AVX-NEXT:    [[V3:%.*]] = insertelement <16 x i16> [[V2]], i16 [[TMP7]], i32 3
+; AVX-NEXT:    [[TMP8:%.*]] = extractelement <16 x i16> [[TMP3]], i32 4
+; AVX-NEXT:    [[V4:%.*]] = insertelement <16 x i16> [[V3]], i16 [[TMP8]], i32 4
+; AVX-NEXT:    [[TMP9:%.*]] = extractelement <16 x i16> [[TMP3]], i32 5
+; AVX-NEXT:    [[V5:%.*]] = insertelement <16 x i16> [[V4]], i16 [[TMP9]], i32 5
+; AVX-NEXT:    [[TMP10:%.*]] = extractelement <16 x i16> [[TMP3]], i32 6
+; AVX-NEXT:    [[V6:%.*]] = insertelement <16 x i16> [[V5]], i16 [[TMP10]], i32 6
+; AVX-NEXT:    [[TMP11:%.*]] = extractelement <16 x i16> [[TMP3]], i32 7
+; AVX-NEXT:    [[V7:%.*]] = insertelement <16 x i16> [[V6]], i16 [[TMP11]], i32 7
+; AVX-NEXT:    [[TMP12:%.*]] = extractelement <16 x i16> [[TMP3]], i32 8
+; AVX-NEXT:    [[V8:%.*]] = insertelement <16 x i16> [[V7]], i16 [[TMP12]], i32 8
+; AVX-NEXT:    [[TMP13:%.*]] = extractelement <16 x i16> [[TMP3]], i32 9
+; AVX-NEXT:    [[V9:%.*]] = insertelement <16 x i16> [[V8]], i16 [[TMP13]], i32 9
+; AVX-NEXT:    [[TMP14:%.*]] = extractelement <16 x i16> [[TMP3]], i32 10
+; AVX-NEXT:    [[V10:%.*]] = insertelement <16 x i16> [[V9]], i16 [[TMP14]], i32 10
+; AVX-NEXT:    [[TMP15:%.*]] = extractelement <16 x i16> [[TMP3]], i32 11
+; AVX-NEXT:    [[V11:%.*]] = insertelement <16 x i16> [[V10]], i16 [[TMP15]], i32 11
+; AVX-NEXT:    [[TMP16:%.*]] = extractelement <16 x i16> [[TMP3]], i32 12
+; AVX-NEXT:    [[V12:%.*]] = insertelement <16 x i16> [[V11]], i16 [[TMP16]], i32 12
+; AVX-NEXT:    [[TMP17:%.*]] = extractelement <16 x i16> [[TMP3]], i32 13
+; AVX-NEXT:    [[V13:%.*]] = insertelement <16 x i16> [[V12]], i16 [[TMP17]], i32 13
+; AVX-NEXT:    [[TMP18:%.*]] = extractelement <16 x i16> [[TMP3]], i32 14
+; AVX-NEXT:    [[V14:%.*]] = insertelement <16 x i16> [[V13]], i16 [[TMP18]], i32 14
+; AVX-NEXT:    [[TMP19:%.*]] = extractelement <16 x i16> [[TMP3]], i32 15
+; AVX-NEXT:    [[V15:%.*]] = insertelement <16 x i16> [[V14]], i16 [[TMP19]], i32 15
+; AVX-NEXT:    ret <16 x i16> [[V15]]
 ;
   %p1  = getelementptr inbounds i8, i8* %p0, i64 1
   %p2  = getelementptr inbounds i8, i8* %p0, i64 2
@@ -541,26 +684,15 @@ define <16 x i16> @loadext_16i8_to_16i16(i8* %p0) {
 ;
 
 define <2 x i64> @loadext_2i16_to_2i64(i16* %p0) {
-; SSE2-LABEL: @loadext_2i16_to_2i64(
-; SSE2-NEXT:    [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
-; SSE2-NEXT:    [[I0:%.*]] = load i16, i16* [[P0]], align 1
-; SSE2-NEXT:    [[I1:%.*]] = load i16, i16* [[P1]], align 1
-; SSE2-NEXT:    [[X0:%.*]] = sext i16 [[I0]] to i64
-; SSE2-NEXT:    [[X1:%.*]] = sext i16 [[I1]] to i64
-; SSE2-NEXT:    [[V0:%.*]] = insertelement <2 x i64> undef, i64 [[X0]], i32 0
-; SSE2-NEXT:    [[V1:%.*]] = insertelement <2 x i64> [[V0]], i64 [[X1]], i32 1
-; SSE2-NEXT:    ret <2 x i64> [[V1]]
-;
-; SLM-LABEL: @loadext_2i16_to_2i64(
-; SLM-NEXT:    [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
-; SLM-NEXT:    [[TMP1:%.*]] = bitcast i16* [[P0]] to <2 x i16>*
-; SLM-NEXT:    [[TMP2:%.*]] = load <2 x i16>, <2 x i16>* [[TMP1]], align 1
-; SLM-NEXT:    [[TMP3:%.*]] = sext <2 x i16> [[TMP2]] to <2 x i64>
-; SLM-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i32 0
-; SLM-NEXT:    [[V0:%.*]] = insertelement <2 x i64> undef, i64 [[TMP4]], i32 0
-; SLM-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[TMP3]], i32 1
-; SLM-NEXT:    [[V1:%.*]] = insertelement <2 x i64> [[V0]], i64 [[TMP5]], i32 1
-; SLM-NEXT:    ret <2 x i64> [[V1]]
+; SSE-LABEL: @loadext_2i16_to_2i64(
+; SSE-NEXT:    [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
+; SSE-NEXT:    [[I0:%.*]] = load i16, i16* [[P0]], align 1
+; SSE-NEXT:    [[I1:%.*]] = load i16, i16* [[P1]], align 1
+; SSE-NEXT:    [[X0:%.*]] = sext i16 [[I0]] to i64
+; SSE-NEXT:    [[X1:%.*]] = sext i16 [[I1]] to i64
+; SSE-NEXT:    [[V0:%.*]] = insertelement <2 x i64> undef, i64 [[X0]], i32 0
+; SSE-NEXT:    [[V1:%.*]] = insertelement <2 x i64> [[V0]], i64 [[X1]], i32 1
+; SSE-NEXT:    ret <2 x i64> [[V1]]
 ;
 ; AVX-LABEL: @loadext_2i16_to_2i64(
 ; AVX-NEXT:    [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
@@ -584,22 +716,57 @@ define <2 x i64> @loadext_2i16_to_2i64(i16* %p0) {
 }
 
 define <4 x i32> @loadext_4i16_to_4i32(i16* %p0) {
-; CHECK-LABEL: @loadext_4i16_to_4i32(
-; CHECK-NEXT:    [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
-; CHECK-NEXT:    [[P2:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 2
-; CHECK-NEXT:    [[P3:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 3
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i16* [[P0]] to <4 x i16>*
-; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i16>, <4 x i16>* [[TMP1]], align 1
-; CHECK-NEXT:    [[TMP3:%.*]] = sext <4 x i16> [[TMP2]] to <4 x i32>
-; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x i32> [[TMP3]], i32 0
-; CHECK-NEXT:    [[V0:%.*]] = insertelement <4 x i32> undef, i32 [[TMP4]], i32 0
-; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x i32> [[TMP3]], i32 1
-; CHECK-NEXT:    [[V1:%.*]] = insertelement <4 x i32> [[V0]], i32 [[TMP5]], i32 1
-; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x i32> [[TMP3]], i32 2
-; CHECK-NEXT:    [[V2:%.*]] = insertelement <4 x i32> [[V1]], i32 [[TMP6]], i32 2
-; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x i32> [[TMP3]], i32 3
-; CHECK-NEXT:    [[V3:%.*]] = insertelement <4 x i32> [[V2]], i32 [[TMP7]], i32 3
-; CHECK-NEXT:    ret <4 x i32> [[V3]]
+; SSE2-LABEL: @loadext_4i16_to_4i32(
+; SSE2-NEXT:    [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
+; SSE2-NEXT:    [[P2:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 2
+; SSE2-NEXT:    [[P3:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 3
+; SSE2-NEXT:    [[TMP1:%.*]] = bitcast i16* [[P0]] to <4 x i16>*
+; SSE2-NEXT:    [[TMP2:%.*]] = load <4 x i16>, <4 x i16>* [[TMP1]], align 1
+; SSE2-NEXT:    [[TMP3:%.*]] = sext <4 x i16> [[TMP2]] to <4 x i32>
+; SSE2-NEXT:    [[TMP4:%.*]] = extractelement <4 x i32> [[TMP3]], i32 0
+; SSE2-NEXT:    [[V0:%.*]] = insertelement <4 x i32> undef, i32 [[TMP4]], i32 0
+; SSE2-NEXT:    [[TMP5:%.*]] = extractelement <4 x i32> [[TMP3]], i32 1
+; SSE2-NEXT:    [[V1:%.*]] = insertelement <4 x i32> [[V0]], i32 [[TMP5]], i32 1
+; SSE2-NEXT:    [[TMP6:%.*]] = extractelement <4 x i32> [[TMP3]], i32 2
+; SSE2-NEXT:    [[V2:%.*]] = insertelement <4 x i32> [[V1]], i32 [[TMP6]], i32 2
+; SSE2-NEXT:    [[TMP7:%.*]] = extractelement <4 x i32> [[TMP3]], i32 3
+; SSE2-NEXT:    [[V3:%.*]] = insertelement <4 x i32> [[V2]], i32 [[TMP7]], i32 3
+; SSE2-NEXT:    ret <4 x i32> [[V3]]
+;
+; SLM-LABEL: @loadext_4i16_to_4i32(
+; SLM-NEXT:    [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
+; SLM-NEXT:    [[P2:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 2
+; SLM-NEXT:    [[P3:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 3
+; SLM-NEXT:    [[I0:%.*]] = load i16, i16* [[P0]], align 1
+; SLM-NEXT:    [[I1:%.*]] = load i16, i16* [[P1]], align 1
+; SLM-NEXT:    [[I2:%.*]] = load i16, i16* [[P2]], align 1
+; SLM-NEXT:    [[I3:%.*]] = load i16, i16* [[P3]], align 1
+; SLM-NEXT:    [[X0:%.*]] = sext i16 [[I0]] to i32
+; SLM-NEXT:    [[X1:%.*]] = sext i16 [[I1]] to i32
+; SLM-NEXT:    [[X2:%.*]] = sext i16 [[I2]] to i32
+; SLM-NEXT:    [[X3:%.*]] = sext i16 [[I3]] to i32
+; SLM-NEXT:    [[V0:%.*]] = insertelement <4 x i32> undef, i32 [[X0]], i32 0
+; SLM-NEXT:    [[V1:%.*]] = insertelement <4 x i32> [[V0]], i32 [[X1]], i32 1
+; SLM-NEXT:    [[V2:%.*]] = insertelement <4 x i32> [[V1]], i32 [[X2]], i32 2
+; SLM-NEXT:    [[V3:%.*]] = insertelement <4 x i32> [[V2]], i32 [[X3]], i32 3
+; SLM-NEXT:    ret <4 x i32> [[V3]]
+;
+; AVX-LABEL: @loadext_4i16_to_4i32(
+; AVX-NEXT:    [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
+; AVX-NEXT:    [[P2:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 2
+; AVX-NEXT:    [[P3:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 3
+; AVX-NEXT:    [[TMP1:%.*]] = bitcast i16* [[P0]] to <4 x i16>*
+; AVX-NEXT:    [[TMP2:%.*]] = load <4 x i16>, <4 x i16>* [[TMP1]], align 1
+; AVX-NEXT:    [[TMP3:%.*]] = sext <4 x i16> [[TMP2]] to <4 x i32>
+; AVX-NEXT:    [[TMP4:%.*]] = extractelement <4 x i32> [[TMP3]], i32 0
+; AVX-NEXT:    [[V0:%.*]] = insertelement <4 x i32> undef, i32 [[TMP4]], i32 0
+; AVX-NEXT:    [[TMP5:%.*]] = extractelement <4 x i32> [[TMP3]], i32 1
+; AVX-NEXT:    [[V1:%.*]] = insertelement <4 x i32> [[V0]], i32 [[TMP5]], i32 1
+; AVX-NEXT:    [[TMP6:%.*]] = extractelement <4 x i32> [[TMP3]], i32 2
+; AVX-NEXT:    [[V2:%.*]] = insertelement <4 x i32> [[V1]], i32 [[TMP6]], i32 2
+; AVX-NEXT:    [[TMP7:%.*]] = extractelement <4 x i32> [[TMP3]], i32 3
+; AVX-NEXT:    [[V3:%.*]] = insertelement <4 x i32> [[V2]], i32 [[TMP7]], i32 3
+; AVX-NEXT:    ret <4 x i32> [[V3]]
 ;
   %p1 = getelementptr inbounds i16, i16* %p0, i64 1
   %p2 = getelementptr inbounds i16, i16* %p0, i64 2
@@ -620,59 +787,76 @@ define <4 x i32> @loadext_4i16_to_4i32(i16* %p0) {
 }
 
 define <4 x i64> @loadext_4i16_to_4i64(i16* %p0) {
-; SSE2-LABEL: @loadext_4i16_to_4i64(
-; SSE2-NEXT:    [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
-; SSE2-NEXT:    [[P2:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 2
-; SSE2-NEXT:    [[P3:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 3
-; SSE2-NEXT:    [[I0:%.*]] = load i16, i16* [[P0]], align 1
-; SSE2-NEXT:    [[I1:%.*]] = load i16, i16* [[P1]], align 1
-; SSE2-NEXT:    [[I2:%.*]] = load i16, i16* [[P2]], align 1
-; SSE2-NEXT:    [[I3:%.*]] = load i16, i16* [[P3]], align 1
-; SSE2-NEXT:    [[X0:%.*]] = sext i16 [[I0]] to i64
-; SSE2-NEXT:    [[X1:%.*]] = sext i16 [[I1]] to i64
-; SSE2-NEXT:    [[X2:%.*]] = sext i16 [[I2]] to i64
-; SSE2-NEXT:    [[X3:%.*]] = sext i16 [[I3]] to i64
-; SSE2-NEXT:    [[V0:%.*]] = insertelement <4 x i64> undef, i64 [[X0]], i32 0
-; SSE2-NEXT:    [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[X1]], i32 1
-; SSE2-NEXT:    [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[X2]], i32 2
-; SSE2-NEXT:    [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[X3]], i32 3
-; SSE2-NEXT:    ret <4 x i64> [[V3]]
-;
-; SLM-LABEL: @loadext_4i16_to_4i64(
-; SLM-NEXT:    [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
-; SLM-NEXT:    [[P2:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 2
-; SLM-NEXT:    [[P3:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 3
-; SLM-NEXT:    [[TMP1:%.*]] = bitcast i16* [[P0]] to <4 x i16>*
-; SLM-NEXT:    [[TMP2:%.*]] = load <4 x i16>, <4 x i16>* [[TMP1]], align 1
-; SLM-NEXT:    [[TMP3:%.*]] = sext <4 x i16> [[TMP2]] to <4 x i64>
-; SLM-NEXT:    [[TMP4:%.*]] = extractelement <4 x i64> [[TMP3]], i32 0
-; SLM-NEXT:    [[V0:%.*]] = insertelement <4 x i64> undef, i64 [[TMP4]], i32 0
-; SLM-NEXT:    [[TMP5:%.*]] = extractelement <4 x i64> [[TMP3]], i32 1
-; SLM-NEXT:    [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[TMP5]], i32 1
-; SLM-NEXT:    [[TMP6:%.*]] = extractelement <4 x i64> [[TMP3]], i32 2
-; SLM-NEXT:    [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[TMP6]], i32 2
-; SLM-NEXT:    [[TMP7:%.*]] = extractelement <4 x i64> [[TMP3]], i32 3
-; SLM-NEXT:    [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[TMP7]], i32 3
-; SLM-NEXT:    ret <4 x i64> [[V3]]
-;
-; AVX-LABEL: @loadext_4i16_to_4i64(
-; AVX-NEXT:    [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
-; AVX-NEXT:    [[P2:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 2
-; AVX-NEXT:    [[P3:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 3
-; AVX-NEXT:    [[TMP1:%.*]] = bitcast i16* [[P0]] to <2 x i16>*
-; AVX-NEXT:    [[TMP2:%.*]] = load <2 x i16>, <2 x i16>* [[TMP1]], align 1
-; AVX-NEXT:    [[I2:%.*]] = load i16, i16* [[P2]], align 1
-; AVX-NEXT:    [[I3:%.*]] = load i16, i16* [[P3]], align 1
-; AVX-NEXT:    [[TMP3:%.*]] = sext <2 x i16> [[TMP2]] to <2 x i64>
-; AVX-NEXT:    [[X2:%.*]] = sext i16 [[I2]] to i64
-; AVX-NEXT:    [[X3:%.*]] = sext i16 [[I3]] to i64
-; AVX-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i32 0
-; AVX-NEXT:    [[V0:%.*]] = insertelement <4 x i64> undef, i64 [[TMP4]], i32 0
-; AVX-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[TMP3]], i32 1
-; AVX-NEXT:    [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[TMP5]], i32 1
-; AVX-NEXT:    [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[X2]], i32 2
-; AVX-NEXT:    [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[X3]], i32 3
-; AVX-NEXT:    ret <4 x i64> [[V3]]
+; SSE-LABEL: @loadext_4i16_to_4i64(
+; SSE-NEXT:    [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
+; SSE-NEXT:    [[P2:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 2
+; SSE-NEXT:    [[P3:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 3
+; SSE-NEXT:    [[I0:%.*]] = load i16, i16* [[P0]], align 1
+; SSE-NEXT:    [[I1:%.*]] = load i16, i16* [[P1]], align 1
+; SSE-NEXT:    [[I2:%.*]] = load i16, i16* [[P2]], align 1
+; SSE-NEXT:    [[I3:%.*]] = load i16, i16* [[P3]], align 1
+; SSE-NEXT:    [[X0:%.*]] = sext i16 [[I0]] to i64
+; SSE-NEXT:    [[X1:%.*]] = sext i16 [[I1]] to i64
+; SSE-NEXT:    [[X2:%.*]] = sext i16 [[I2]] to i64
+; SSE-NEXT:    [[X3:%.*]] = sext i16 [[I3]] to i64
+; SSE-NEXT:    [[V0:%.*]] = insertelement <4 x i64> undef, i64 [[X0]], i32 0
+; SSE-NEXT:    [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[X1]], i32 1
+; SSE-NEXT:    [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[X2]], i32 2
+; SSE-NEXT:    [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[X3]], i32 3
+; SSE-NEXT:    ret <4 x i64> [[V3]]
+;
+; AVX1-LABEL: @loadext_4i16_to_4i64(
+; AVX1-NEXT:    [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
+; AVX1-NEXT:    [[P2:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 2
+; AVX1-NEXT:    [[P3:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 3
+; AVX1-NEXT:    [[TMP1:%.*]] = bitcast i16* [[P0]] to <2 x i16>*
+; AVX1-NEXT:    [[TMP2:%.*]] = load <2 x i16>, <2 x i16>* [[TMP1]], align 1
+; AVX1-NEXT:    [[I2:%.*]] = load i16, i16* [[P2]], align 1
+; AVX1-NEXT:    [[I3:%.*]] = load i16, i16* [[P3]], align 1
+; AVX1-NEXT:    [[TMP3:%.*]] = sext <2 x i16> [[TMP2]] to <2 x i64>
+; AVX1-NEXT:    [[X2:%.*]] = sext i16 [[I2]] to i64
+; AVX1-NEXT:    [[X3:%.*]] = sext i16 [[I3]] to i64
+; AVX1-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i32 0
+; AVX1-NEXT:    [[V0:%.*]] = insertelement <4 x i64> undef, i64 [[TMP4]], i32 0
+; AVX1-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[TMP3]], i32 1
+; AVX1-NEXT:    [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[TMP5]], i32 1
+; AVX1-NEXT:    [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[X2]], i32 2
+; AVX1-NEXT:    [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[X3]], i32 3
+; AVX1-NEXT:    ret <4 x i64> [[V3]]
+;
+; AVX2-LABEL: @loadext_4i16_to_4i64(
+; AVX2-NEXT:    [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
+; AVX2-NEXT:    [[P2:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 2
+; AVX2-NEXT:    [[P3:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 3
+; AVX2-NEXT:    [[TMP1:%.*]] = bitcast i16* [[P0]] to <4 x i16>*
+; AVX2-NEXT:    [[TMP2:%.*]] = load <4 x i16>, <4 x i16>* [[TMP1]], align 1
+; AVX2-NEXT:    [[TMP3:%.*]] = sext <4 x i16> [[TMP2]] to <4 x i64>
+; AVX2-NEXT:    [[TMP4:%.*]] = extractelement <4 x i64> [[TMP3]], i32 0
+; AVX2-NEXT:    [[V0:%.*]] = insertelement <4 x i64> undef, i64 [[TMP4]], i32 0
+; AVX2-NEXT:    [[TMP5:%.*]] = extractelement <4 x i64> [[TMP3]], i32 1
+; AVX2-NEXT:    [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[TMP5]], i32 1
+; AVX2-NEXT:    [[TMP6:%.*]] = extractelement <4 x i64> [[TMP3]], i32 2
+; AVX2-NEXT:    [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[TMP6]], i32 2
+; AVX2-NEXT:    [[TMP7:%.*]] = extractelement <4 x i64> [[TMP3]], i32 3
+; AVX2-NEXT:    [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[TMP7]], i32 3
+; AVX2-NEXT:    ret <4 x i64> [[V3]]
+;
+; AVX512-LABEL: @loadext_4i16_to_4i64(
+; AVX512-NEXT:    [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
+; AVX512-NEXT:    [[P2:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 2
+; AVX512-NEXT:    [[P3:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 3
+; AVX512-NEXT:    [[TMP1:%.*]] = bitcast i16* [[P0]] to <4 x i16>*
+; AVX512-NEXT:    [[TMP2:%.*]] = load <4 x i16>, <4 x i16>* [[TMP1]], align 1
+; AVX512-NEXT:    [[TMP3:%.*]] = sext <4 x i16> [[TMP2]] to <4 x i64>
+; AVX512-NEXT:    [[TMP4:%.*]] = extractelement <4 x i64> [[TMP3]], i32 0
+; AVX512-NEXT:    [[V0:%.*]] = insertelement <4 x i64> undef, i64 [[TMP4]], i32 0
+; AVX512-NEXT:    [[TMP5:%.*]] = extractelement <4 x i64> [[TMP3]], i32 1
+; AVX512-NEXT:    [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[TMP5]], i32 1
+; AVX512-NEXT:    [[TMP6:%.*]] = extractelement <4 x i64> [[TMP3]], i32 2
+; AVX512-NEXT:    [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[TMP6]], i32 2
+; AVX512-NEXT:    [[TMP7:%.*]] = extractelement <4 x i64> [[TMP3]], i32 3
+; AVX512-NEXT:    [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[TMP7]], i32 3
+; AVX512-NEXT:    ret <4 x i64> [[V3]]
 ;
   %p1 = getelementptr inbounds i16, i16* %p0, i64 1
   %p2 = getelementptr inbounds i16, i16* %p0, i64 2
@@ -693,34 +877,97 @@ define <4 x i64> @loadext_4i16_to_4i64(i16* %p0) {
 }
 
 define <8 x i32> @loadext_8i16_to_8i32(i16* %p0) {
-; CHECK-LABEL: @loadext_8i16_to_8i32(
-; CHECK-NEXT:    [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
-; CHECK-NEXT:    [[P2:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 2
-; CHECK-NEXT:    [[P3:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 3
-; CHECK-NEXT:    [[P4:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 4
-; CHECK-NEXT:    [[P5:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 5
-; CHECK-NEXT:    [[P6:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 6
-; CHECK-NEXT:    [[P7:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 7
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i16* [[P0]] to <8 x i16>*
-; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, <8 x i16>* [[TMP1]], align 1
-; CHECK-NEXT:    [[TMP3:%.*]] = sext <8 x i16> [[TMP2]] to <8 x i32>
-; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <8 x i32> [[TMP3]], i32 0
-; CHECK-NEXT:    [[V0:%.*]] = insertelement <8 x i32> undef, i32 [[TMP4]], i32 0
-; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <8 x i32> [[TMP3]], i32 1
-; CHECK-NEXT:    [[V1:%.*]] = insertelement <8 x i32> [[V0]], i32 [[TMP5]], i32 1
-; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <8 x i32> [[TMP3]], i32 2
-; CHECK-NEXT:    [[V2:%.*]] = insertelement <8 x i32> [[V1]], i32 [[TMP6]], i32 2
-; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <8 x i32> [[TMP3]], i32 3
-; CHECK-NEXT:    [[V3:%.*]] = insertelement <8 x i32> [[V2]], i32 [[TMP7]], i32 3
-; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <8 x i32> [[TMP3]], i32 4
-; CHECK-NEXT:    [[V4:%.*]] = insertelement <8 x i32> [[V3]], i32 [[TMP8]], i32 4
-; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <8 x i32> [[TMP3]], i32 5
-; CHECK-NEXT:    [[V5:%.*]] = insertelement <8 x i32> [[V4]], i32 [[TMP9]], i32 5
-; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <8 x i32> [[TMP3]], i32 6
-; CHECK-NEXT:    [[V6:%.*]] = insertelement <8 x i32> [[V5]], i32 [[TMP10]], i32 6
-; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <8 x i32> [[TMP3]], i32 7
-; CHECK-NEXT:    [[V7:%.*]] = insertelement <8 x i32> [[V6]], i32 [[TMP11]], i32 7
-; CHECK-NEXT:    ret <8 x i32> [[V7]]
+; SSE2-LABEL: @loadext_8i16_to_8i32(
+; SSE2-NEXT:    [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
+; SSE2-NEXT:    [[P2:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 2
+; SSE2-NEXT:    [[P3:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 3
+; SSE2-NEXT:    [[P4:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 4
+; SSE2-NEXT:    [[P5:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 5
+; SSE2-NEXT:    [[P6:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 6
+; SSE2-NEXT:    [[P7:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 7
+; SSE2-NEXT:    [[TMP1:%.*]] = bitcast i16* [[P0]] to <8 x i16>*
+; SSE2-NEXT:    [[TMP2:%.*]] = load <8 x i16>, <8 x i16>* [[TMP1]], align 1
+; SSE2-NEXT:    [[TMP3:%.*]] = sext <8 x i16> [[TMP2]] to <8 x i32>
+; SSE2-NEXT:    [[TMP4:%.*]] = extractelement <8 x i32> [[TMP3]], i32 0
+; SSE2-NEXT:    [[V0:%.*]] = insertelement <8 x i32> undef, i32 [[TMP4]], i32 0
+; SSE2-NEXT:    [[TMP5:%.*]] = extractelement <8 x i32> [[TMP3]], i32 1
+; SSE2-NEXT:    [[V1:%.*]] = insertelement <8 x i32> [[V0]], i32 [[TMP5]], i32 1
+; SSE2-NEXT:    [[TMP6:%.*]] = extractelement <8 x i32> [[TMP3]], i32 2
+; SSE2-NEXT:    [[V2:%.*]] = insertelement <8 x i32> [[V1]], i32 [[TMP6]], i32 2
+; SSE2-NEXT:    [[TMP7:%.*]] = extractelement <8 x i32> [[TMP3]], i32 3
+; SSE2-NEXT:    [[V3:%.*]] = insertelement <8 x i32> [[V2]], i32 [[TMP7]], i32 3
+; SSE2-NEXT:    [[TMP8:%.*]] = extractelement <8 x i32> [[TMP3]], i32 4
+; SSE2-NEXT:    [[V4:%.*]] = insertelement <8 x i32> [[V3]], i32 [[TMP8]], i32 4
+; SSE2-NEXT:    [[TMP9:%.*]] = extractelement <8 x i32> [[TMP3]], i32 5
+; SSE2-NEXT:    [[V5:%.*]] = insertelement <8 x i32> [[V4]], i32 [[TMP9]], i32 5
+; SSE2-NEXT:    [[TMP10:%.*]] = extractelement <8 x i32> [[TMP3]], i32 6
+; SSE2-NEXT:    [[V6:%.*]] = insertelement <8 x i32> [[V5]], i32 [[TMP10]], i32 6
+; SSE2-NEXT:    [[TMP11:%.*]] = extractelement <8 x i32> [[TMP3]], i32 7
+; SSE2-NEXT:    [[V7:%.*]] = insertelement <8 x i32> [[V6]], i32 [[TMP11]], i32 7
+; SSE2-NEXT:    ret <8 x i32> [[V7]]
+;
+; SLM-LABEL: @loadext_8i16_to_8i32(
+; SLM-NEXT:    [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
+; SLM-NEXT:    [[P2:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 2
+; SLM-NEXT:    [[P3:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 3
+; SLM-NEXT:    [[P4:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 4
+; SLM-NEXT:    [[P5:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 5
+; SLM-NEXT:    [[P6:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 6
+; SLM-NEXT:    [[P7:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 7
+; SLM-NEXT:    [[I0:%.*]] = load i16, i16* [[P0]], align 1
+; SLM-NEXT:    [[I1:%.*]] = load i16, i16* [[P1]], align 1
+; SLM-NEXT:    [[I2:%.*]] = load i16, i16* [[P2]], align 1
+; SLM-NEXT:    [[I3:%.*]] = load i16, i16* [[P3]], align 1
+; SLM-NEXT:    [[I4:%.*]] = load i16, i16* [[P4]], align 1
+; SLM-NEXT:    [[I5:%.*]] = load i16, i16* [[P5]], align 1
+; SLM-NEXT:    [[I6:%.*]] = load i16, i16* [[P6]], align 1
+; SLM-NEXT:    [[I7:%.*]] = load i16, i16* [[P7]], align 1
+; SLM-NEXT:    [[X0:%.*]] = sext i16 [[I0]] to i32
+; SLM-NEXT:    [[X1:%.*]] = sext i16 [[I1]] to i32
+; SLM-NEXT:    [[X2:%.*]] = sext i16 [[I2]] to i32
+; SLM-NEXT:    [[X3:%.*]] = sext i16 [[I3]] to i32
+; SLM-NEXT:    [[X4:%.*]] = sext i16 [[I4]] to i32
+; SLM-NEXT:    [[X5:%.*]] = sext i16 [[I5]] to i32
+; SLM-NEXT:    [[X6:%.*]] = sext i16 [[I6]] to i32
+; SLM-NEXT:    [[X7:%.*]] = sext i16 [[I7]] to i32
+; SLM-NEXT:    [[V0:%.*]] = insertelement <8 x i32> undef, i32 [[X0]], i32 0
+; SLM-NEXT:    [[V1:%.*]] = insertelement <8 x i32> [[V0]], i32 [[X1]], i32 1
+; SLM-NEXT:    [[V2:%.*]] = insertelement <8 x i32> [[V1]], i32 [[X2]], i32 2
+; SLM-NEXT:    [[V3:%.*]] = insertelement <8 x i32> [[V2]], i32 [[X3]], i32 3
+; SLM-NEXT:    [[V4:%.*]] = insertelement <8 x i32> [[V3]], i32 [[X4]], i32 4
+; SLM-NEXT:    [[V5:%.*]] = insertelement <8 x i32> [[V4]], i32 [[X5]], i32 5
+; SLM-NEXT:    [[V6:%.*]] = insertelement <8 x i32> [[V5]], i32 [[X6]], i32 6
+; SLM-NEXT:    [[V7:%.*]] = insertelement <8 x i32> [[V6]], i32 [[X7]], i32 7
+; SLM-NEXT:    ret <8 x i32> [[V7]]
+;
+; AVX-LABEL: @loadext_8i16_to_8i32(
+; AVX-NEXT:    [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
+; AVX-NEXT:    [[P2:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 2
+; AVX-NEXT:    [[P3:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 3
+; AVX-NEXT:    [[P4:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 4
+; AVX-NEXT:    [[P5:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 5
+; AVX-NEXT:    [[P6:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 6
+; AVX-NEXT:    [[P7:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 7
+; AVX-NEXT:    [[TMP1:%.*]] = bitcast i16* [[P0]] to <8 x i16>*
+; AVX-NEXT:    [[TMP2:%.*]] = load <8 x i16>, <8 x i16>* [[TMP1]], align 1
+; AVX-NEXT:    [[TMP3:%.*]] = sext <8 x i16> [[TMP2]] to <8 x i32>
+; AVX-NEXT:    [[TMP4:%.*]] = extractelement <8 x i32> [[TMP3]], i32 0
+; AVX-NEXT:    [[V0:%.*]] = insertelement <8 x i32> undef, i32 [[TMP4]], i32 0
+; AVX-NEXT:    [[TMP5:%.*]] = extractelement <8 x i32> [[TMP3]], i32 1
+; AVX-NEXT:    [[V1:%.*]] = insertelement <8 x i32> [[V0]], i32 [[TMP5]], i32 1
+; AVX-NEXT:    [[TMP6:%.*]] = extractelement <8 x i32> [[TMP3]], i32 2
+; AVX-NEXT:    [[V2:%.*]] = insertelement <8 x i32> [[V1]], i32 [[TMP6]], i32 2
+; AVX-NEXT:    [[TMP7:%.*]] = extractelement <8 x i32> [[TMP3]], i32 3
+; AVX-NEXT:    [[V3:%.*]] = insertelement <8 x i32> [[V2]], i32 [[TMP7]], i32 3
+; AVX-NEXT:    [[TMP8:%.*]] = extractelement <8 x i32> [[TMP3]], i32 4
+; AVX-NEXT:    [[V4:%.*]] = insertelement <8 x i32> [[V3]], i32 [[TMP8]], i32 4
+; AVX-NEXT:    [[TMP9:%.*]] = extractelement <8 x i32> [[TMP3]], i32 5
+; AVX-NEXT:    [[V5:%.*]] = insertelement <8 x i32> [[V4]], i32 [[TMP9]], i32 5
+; AVX-NEXT:    [[TMP10:%.*]] = extractelement <8 x i32> [[TMP3]], i32 6
+; AVX-NEXT:    [[V6:%.*]] = insertelement <8 x i32> [[V5]], i32 [[TMP10]], i32 6
+; AVX-NEXT:    [[TMP11:%.*]] = extractelement <8 x i32> [[TMP3]], i32 7
+; AVX-NEXT:    [[V7:%.*]] = insertelement <8 x i32> [[V6]], i32 [[TMP11]], i32 7
+; AVX-NEXT:    ret <8 x i32> [[V7]]
 ;
   %p1 = getelementptr inbounds i16, i16* %p0, i64 1
   %p2 = getelementptr inbounds i16, i16* %p0, i64 2
@@ -761,26 +1008,15 @@ define <8 x i32> @loadext_8i16_to_8i32(i16* %p0) {
 ;
 
 define <2 x i64> @loadext_2i32_to_2i64(i32* %p0) {
-; SSE2-LABEL: @loadext_2i32_to_2i64(
-; SSE2-NEXT:    [[P1:%.*]] = getelementptr inbounds i32, i32* [[P0:%.*]], i64 1
-; SSE2-NEXT:    [[I0:%.*]] = load i32, i32* [[P0]], align 1
-; SSE2-NEXT:    [[I1:%.*]] = load i32, i32* [[P1]], align 1
-; SSE2-NEXT:    [[X0:%.*]] = sext i32 [[I0]] to i64
-; SSE2-NEXT:    [[X1:%.*]] = sext i32 [[I1]] to i64
-; SSE2-NEXT:    [[V0:%.*]] = insertelement <2 x i64> undef, i64 [[X0]], i32 0
-; SSE2-NEXT:    [[V1:%.*]] = insertelement <2 x i64> [[V0]], i64 [[X1]], i32 1
-; SSE2-NEXT:    ret <2 x i64> [[V1]]
-;
-; SLM-LABEL: @loadext_2i32_to_2i64(
-; SLM-NEXT:    [[P1:%.*]] = getelementptr inbounds i32, i32* [[P0:%.*]], i64 1
-; SLM-NEXT:    [[TMP1:%.*]] = bitcast i32* [[P0]] to <2 x i32>*
-; SLM-NEXT:    [[TMP2:%.*]] = load <2 x i32>, <2 x i32>* [[TMP1]], align 1
-; SLM-NEXT:    [[TMP3:%.*]] = sext <2 x i32> [[TMP2]] to <2 x i64>
-; SLM-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i32 0
-; SLM-NEXT:    [[V0:%.*]] = insertelement <2 x i64> undef, i64 [[TMP4]], i32 0
-; SLM-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[TMP3]], i32 1
-; SLM-NEXT:    [[V1:%.*]] = insertelement <2 x i64> [[V0]], i64 [[TMP5]], i32 1
-; SLM-NEXT:    ret <2 x i64> [[V1]]
+; SSE-LABEL: @loadext_2i32_to_2i64(
+; SSE-NEXT:    [[P1:%.*]] = getelementptr inbounds i32, i32* [[P0:%.*]], i64 1
+; SSE-NEXT:    [[I0:%.*]] = load i32, i32* [[P0]], align 1
+; SSE-NEXT:    [[I1:%.*]] = load i32, i32* [[P1]], align 1
+; SSE-NEXT:    [[X0:%.*]] = sext i32 [[I0]] to i64
+; SSE-NEXT:    [[X1:%.*]] = sext i32 [[I1]] to i64
+; SSE-NEXT:    [[V0:%.*]] = insertelement <2 x i64> undef, i64 [[X0]], i32 0
+; SSE-NEXT:    [[V1:%.*]] = insertelement <2 x i64> [[V0]], i64 [[X1]], i32 1
+; SSE-NEXT:    ret <2 x i64> [[V1]]
 ;
 ; AVX-LABEL: @loadext_2i32_to_2i64(
 ; AVX-NEXT:    [[P1:%.*]] = getelementptr inbounds i32, i32* [[P0:%.*]], i64 1
@@ -804,40 +1040,23 @@ define <2 x i64> @loadext_2i32_to_2i64(i32* %p0) {
 }
 
 define <4 x i64> @loadext_4i32_to_4i64(i32* %p0) {
-; SSE2-LABEL: @loadext_4i32_to_4i64(
-; SSE2-NEXT:    [[P1:%.*]] = getelementptr inbounds i32, i32* [[P0:%.*]], i64 1
-; SSE2-NEXT:    [[P2:%.*]] = getelementptr inbounds i32, i32* [[P0]], i64 2
-; SSE2-NEXT:    [[P3:%.*]] = getelementptr inbounds i32, i32* [[P0]], i64 3
-; SSE2-NEXT:    [[I0:%.*]] = load i32, i32* [[P0]], align 1
-; SSE2-NEXT:    [[I1:%.*]] = load i32, i32* [[P1]], align 1
-; SSE2-NEXT:    [[I2:%.*]] = load i32, i32* [[P2]], align 1
-; SSE2-NEXT:    [[I3:%.*]] = load i32, i32* [[P3]], align 1
-; SSE2-NEXT:    [[X0:%.*]] = sext i32 [[I0]] to i64
-; SSE2-NEXT:    [[X1:%.*]] = sext i32 [[I1]] to i64
-; SSE2-NEXT:    [[X2:%.*]] = sext i32 [[I2]] to i64
-; SSE2-NEXT:    [[X3:%.*]] = sext i32 [[I3]] to i64
-; SSE2-NEXT:    [[V0:%.*]] = insertelement <4 x i64> undef, i64 [[X0]], i32 0
-; SSE2-NEXT:    [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[X1]], i32 1
-; SSE2-NEXT:    [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[X2]], i32 2
-; SSE2-NEXT:    [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[X3]], i32 3
-; SSE2-NEXT:    ret <4 x i64> [[V3]]
-;
-; SLM-LABEL: @loadext_4i32_to_4i64(
-; SLM-NEXT:    [[P1:%.*]] = getelementptr inbounds i32, i32* [[P0:%.*]], i64 1
-; SLM-NEXT:    [[P2:%.*]] = getelementptr inbounds i32, i32* [[P0]], i64 2
-; SLM-NEXT:    [[P3:%.*]] = getelementptr inbounds i32, i32* [[P0]], i64 3
-; SLM-NEXT:    [[TMP1:%.*]] = bitcast i32* [[P0]] to <4 x i32>*
-; SLM-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP1]], align 1
-; SLM-NEXT:    [[TMP3:%.*]] = sext <4 x i32> [[TMP2]] to <4 x i64>
-; SLM-NEXT:    [[TMP4:%.*]] = extractelement <4 x i64> [[TMP3]], i32 0
-; SLM-NEXT:    [[V0:%.*]] = insertelement <4 x i64> undef, i64 [[TMP4]], i32 0
-; SLM-NEXT:    [[TMP5:%.*]] = extractelement <4 x i64> [[TMP3]], i32 1
-; SLM-NEXT:    [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[TMP5]], i32 1
-; SLM-NEXT:    [[TMP6:%.*]] = extractelement <4 x i64> [[TMP3]], i32 2
-; SLM-NEXT:    [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[TMP6]], i32 2
-; SLM-NEXT:    [[TMP7:%.*]] = extractelement <4 x i64> [[TMP3]], i32 3
-; SLM-NEXT:    [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[TMP7]], i32 3
-; SLM-NEXT:    ret <4 x i64> [[V3]]
+; SSE-LABEL: @loadext_4i32_to_4i64(
+; SSE-NEXT:    [[P1:%.*]] = getelementptr inbounds i32, i32* [[P0:%.*]], i64 1
+; SSE-NEXT:    [[P2:%.*]] = getelementptr inbounds i32, i32* [[P0]], i64 2
+; SSE-NEXT:    [[P3:%.*]] = getelementptr inbounds i32, i32* [[P0]], i64 3
+; SSE-NEXT:    [[I0:%.*]] = load i32, i32* [[P0]], align 1
+; SSE-NEXT:    [[I1:%.*]] = load i32, i32* [[P1]], align 1
+; SSE-NEXT:    [[I2:%.*]] = load i32, i32* [[P2]], align 1
+; SSE-NEXT:    [[I3:%.*]] = load i32, i32* [[P3]], align 1
+; SSE-NEXT:    [[X0:%.*]] = sext i32 [[I0]] to i64
+; SSE-NEXT:    [[X1:%.*]] = sext i32 [[I1]] to i64
+; SSE-NEXT:    [[X2:%.*]] = sext i32 [[I2]] to i64
+; SSE-NEXT:    [[X3:%.*]] = sext i32 [[I3]] to i64
+; SSE-NEXT:    [[V0:%.*]] = insertelement <4 x i64> undef, i64 [[X0]], i32 0
+; SSE-NEXT:    [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[X1]], i32 1
+; SSE-NEXT:    [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[X2]], i32 2
+; SSE-NEXT:    [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[X3]], i32 3
+; SSE-NEXT:    ret <4 x i64> [[V3]]
 ;
 ; AVX1-LABEL: @loadext_4i32_to_4i64(
 ; AVX1-NEXT:    [[P1:%.*]] = getelementptr inbounds i32, i32* [[P0:%.*]], i64 1
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/shift-ashr.ll b/llvm/test/Transforms/SLPVectorizer/X86/shift-ashr.ll
index e4b71ba97..9ced739e5 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/shift-ashr.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/shift-ashr.ll
@@ -482,14 +482,10 @@ define void @ashr_v32i16() {
 ; AVX-NEXT:    ret void
 ;
 ; AVX512-LABEL: @ashr_v32i16(
-; AVX512-NEXT:    [[TMP1:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @a16 to <16 x i16>*), align 2
-; AVX512-NEXT:    [[TMP2:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <16 x i16>*), align 2
-; AVX512-NEXT:    [[TMP3:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @b16 to <16 x i16>*), align 2
-; AVX512-NEXT:    [[TMP4:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <16 x i16>*), align 2
-; AVX512-NEXT:    [[TMP5:%.*]] = ashr <16 x i16> [[TMP1]], [[TMP3]]
-; AVX512-NEXT:    [[TMP6:%.*]] = ashr <16 x i16> [[TMP2]], [[TMP4]]
-; AVX512-NEXT:    store <16 x i16> [[TMP5]], <16 x i16>* bitcast ([32 x i16]* @c16 to <16 x i16>*), align 2
-; AVX512-NEXT:    store <16 x i16> [[TMP6]], <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP1:%.*]] = load <32 x i16>, <32 x i16>* bitcast ([32 x i16]* @a16 to <32 x i16>*), align 2
+; AVX512-NEXT:    [[TMP2:%.*]] = load <32 x i16>, <32 x i16>* bitcast ([32 x i16]* @b16 to <32 x i16>*), align 2
+; AVX512-NEXT:    [[TMP3:%.*]] = ashr <32 x i16> [[TMP1]], [[TMP2]]
+; AVX512-NEXT:    store <32 x i16> [[TMP3]], <32 x i16>* bitcast ([32 x i16]* @c16 to <32 x i16>*), align 2
 ; AVX512-NEXT:    ret void
 ;
 ; XOP-LABEL: @ashr_v32i16(
@@ -635,24 +631,53 @@ define void @ashr_v32i16() {
 }
 
 define void @ashr_v64i8() {
-; CHECK-LABEL: @ashr_v64i8(
-; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @a8 to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP5:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @b8 to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP6:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP7:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP8:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP9:%.*]] = ashr <16 x i8> [[TMP1]], [[TMP5]]
-; CHECK-NEXT:    [[TMP10:%.*]] = ashr <16 x i8> [[TMP2]], [[TMP6]]
-; CHECK-NEXT:    [[TMP11:%.*]] = ashr <16 x i8> [[TMP3]], [[TMP7]]
-; CHECK-NEXT:    [[TMP12:%.*]] = ashr <16 x i8> [[TMP4]], [[TMP8]]
-; CHECK-NEXT:    store <16 x i8> [[TMP9]], <16 x i8>* bitcast ([64 x i8]* @c8 to <16 x i8>*), align 1
-; CHECK-NEXT:    store <16 x i8> [[TMP10]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16) to <16 x i8>*), align 1
-; CHECK-NEXT:    store <16 x i8> [[TMP11]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32) to <16 x i8>*), align 1
-; CHECK-NEXT:    store <16 x i8> [[TMP12]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48) to <16 x i8>*), align 1
-; CHECK-NEXT:    ret void
+; SSE-LABEL: @ashr_v64i8(
+; SSE-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @a8 to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP3:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP4:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP5:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @b8 to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP6:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP7:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP8:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP9:%.*]] = ashr <16 x i8> [[TMP1]], [[TMP5]]
+; SSE-NEXT:    [[TMP10:%.*]] = ashr <16 x i8> [[TMP2]], [[TMP6]]
+; SSE-NEXT:    [[TMP11:%.*]] = ashr <16 x i8> [[TMP3]], [[TMP7]]
+; SSE-NEXT:    [[TMP12:%.*]] = ashr <16 x i8> [[TMP4]], [[TMP8]]
+; SSE-NEXT:    store <16 x i8> [[TMP9]], <16 x i8>* bitcast ([64 x i8]* @c8 to <16 x i8>*), align 1
+; SSE-NEXT:    store <16 x i8> [[TMP10]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16) to <16 x i8>*), align 1
+; SSE-NEXT:    store <16 x i8> [[TMP11]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32) to <16 x i8>*), align 1
+; SSE-NEXT:    store <16 x i8> [[TMP12]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48) to <16 x i8>*), align 1
+; SSE-NEXT:    ret void
+;
+; AVX-LABEL: @ashr_v64i8(
+; AVX-NEXT:    [[TMP1:%.*]] = load <32 x i8>, <32 x i8>* bitcast ([64 x i8]* @a8 to <32 x i8>*), align 1
+; AVX-NEXT:    [[TMP2:%.*]] = load <32 x i8>, <32 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32) to <32 x i8>*), align 1
+; AVX-NEXT:    [[TMP3:%.*]] = load <32 x i8>, <32 x i8>* bitcast ([64 x i8]* @b8 to <32 x i8>*), align 1
+; AVX-NEXT:    [[TMP4:%.*]] = load <32 x i8>, <32 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32) to <32 x i8>*), align 1
+; AVX-NEXT:    [[TMP5:%.*]] = ashr <32 x i8> [[TMP1]], [[TMP3]]
+; AVX-NEXT:    [[TMP6:%.*]] = ashr <32 x i8> [[TMP2]], [[TMP4]]
+; AVX-NEXT:    store <32 x i8> [[TMP5]], <32 x i8>* bitcast ([64 x i8]* @c8 to <32 x i8>*), align 1
+; AVX-NEXT:    store <32 x i8> [[TMP6]], <32 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32) to <32 x i8>*), align 1
+; AVX-NEXT:    ret void
+;
+; AVX512-LABEL: @ashr_v64i8(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <64 x i8>, <64 x i8>* bitcast ([64 x i8]* @a8 to <64 x i8>*), align 1
+; AVX512-NEXT:    [[TMP2:%.*]] = load <64 x i8>, <64 x i8>* bitcast ([64 x i8]* @b8 to <64 x i8>*), align 1
+; AVX512-NEXT:    [[TMP3:%.*]] = ashr <64 x i8> [[TMP1]], [[TMP2]]
+; AVX512-NEXT:    store <64 x i8> [[TMP3]], <64 x i8>* bitcast ([64 x i8]* @c8 to <64 x i8>*), align 1
+; AVX512-NEXT:    ret void
+;
+; XOP-LABEL: @ashr_v64i8(
+; XOP-NEXT:    [[TMP1:%.*]] = load <32 x i8>, <32 x i8>* bitcast ([64 x i8]* @a8 to <32 x i8>*), align 1
+; XOP-NEXT:    [[TMP2:%.*]] = load <32 x i8>, <32 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32) to <32 x i8>*), align 1
+; XOP-NEXT:    [[TMP3:%.*]] = load <32 x i8>, <32 x i8>* bitcast ([64 x i8]* @b8 to <32 x i8>*), align 1
+; XOP-NEXT:    [[TMP4:%.*]] = load <32 x i8>, <32 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32) to <32 x i8>*), align 1
+; XOP-NEXT:    [[TMP5:%.*]] = ashr <32 x i8> [[TMP1]], [[TMP3]]
+; XOP-NEXT:    [[TMP6:%.*]] = ashr <32 x i8> [[TMP2]], [[TMP4]]
+; XOP-NEXT:    store <32 x i8> [[TMP5]], <32 x i8>* bitcast ([64 x i8]* @c8 to <32 x i8>*), align 1
+; XOP-NEXT:    store <32 x i8> [[TMP6]], <32 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32) to <32 x i8>*), align 1
+; XOP-NEXT:    ret void
 ;
   %a0  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 0 ), align 1
   %a1  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 1 ), align 1
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/shift-lshr.ll b/llvm/test/Transforms/SLPVectorizer/X86/shift-lshr.ll
index 237673da5..3164d4bd3 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/shift-lshr.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/shift-lshr.ll
@@ -431,14 +431,10 @@ define void @lshr_v32i16() {
 ; AVX-NEXT:    ret void
 ;
 ; AVX512-LABEL: @lshr_v32i16(
-; AVX512-NEXT:    [[TMP1:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @a16 to <16 x i16>*), align 2
-; AVX512-NEXT:    [[TMP2:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <16 x i16>*), align 2
-; AVX512-NEXT:    [[TMP3:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @b16 to <16 x i16>*), align 2
-; AVX512-NEXT:    [[TMP4:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <16 x i16>*), align 2
-; AVX512-NEXT:    [[TMP5:%.*]] = lshr <16 x i16> [[TMP1]], [[TMP3]]
-; AVX512-NEXT:    [[TMP6:%.*]] = lshr <16 x i16> [[TMP2]], [[TMP4]]
-; AVX512-NEXT:    store <16 x i16> [[TMP5]], <16 x i16>* bitcast ([32 x i16]* @c16 to <16 x i16>*), align 2
-; AVX512-NEXT:    store <16 x i16> [[TMP6]], <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP1:%.*]] = load <32 x i16>, <32 x i16>* bitcast ([32 x i16]* @a16 to <32 x i16>*), align 2
+; AVX512-NEXT:    [[TMP2:%.*]] = load <32 x i16>, <32 x i16>* bitcast ([32 x i16]* @b16 to <32 x i16>*), align 2
+; AVX512-NEXT:    [[TMP3:%.*]] = lshr <32 x i16> [[TMP1]], [[TMP2]]
+; AVX512-NEXT:    store <32 x i16> [[TMP3]], <32 x i16>* bitcast ([32 x i16]* @c16 to <32 x i16>*), align 2
 ; AVX512-NEXT:    ret void
 ;
 ; XOP-LABEL: @lshr_v32i16(
@@ -584,24 +580,53 @@ define void @lshr_v32i16() {
 }
 
 define void @lshr_v64i8() {
-; CHECK-LABEL: @lshr_v64i8(
-; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @a8 to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP5:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @b8 to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP6:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP7:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP8:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP9:%.*]] = lshr <16 x i8> [[TMP1]], [[TMP5]]
-; CHECK-NEXT:    [[TMP10:%.*]] = lshr <16 x i8> [[TMP2]], [[TMP6]]
-; CHECK-NEXT:    [[TMP11:%.*]] = lshr <16 x i8> [[TMP3]], [[TMP7]]
-; CHECK-NEXT:    [[TMP12:%.*]] = lshr <16 x i8> [[TMP4]], [[TMP8]]
-; CHECK-NEXT:    store <16 x i8> [[TMP9]], <16 x i8>* bitcast ([64 x i8]* @c8 to <16 x i8>*), align 1
-; CHECK-NEXT:    store <16 x i8> [[TMP10]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16) to <16 x i8>*), align 1
-; CHECK-NEXT:    store <16 x i8> [[TMP11]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32) to <16 x i8>*), align 1
-; CHECK-NEXT:    store <16 x i8> [[TMP12]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48) to <16 x i8>*), align 1
-; CHECK-NEXT:    ret void
+; SSE-LABEL: @lshr_v64i8(
+; SSE-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @a8 to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP3:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP4:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP5:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @b8 to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP6:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP7:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP8:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP9:%.*]] = lshr <16 x i8> [[TMP1]], [[TMP5]]
+; SSE-NEXT:    [[TMP10:%.*]] = lshr <16 x i8> [[TMP2]], [[TMP6]]
+; SSE-NEXT:    [[TMP11:%.*]] = lshr <16 x i8> [[TMP3]], [[TMP7]]
+; SSE-NEXT:    [[TMP12:%.*]] = lshr <16 x i8> [[TMP4]], [[TMP8]]
+; SSE-NEXT:    store <16 x i8> [[TMP9]], <16 x i8>* bitcast ([64 x i8]* @c8 to <16 x i8>*), align 1
+; SSE-NEXT:    store <16 x i8> [[TMP10]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16) to <16 x i8>*), align 1
+; SSE-NEXT:    store <16 x i8> [[TMP11]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32) to <16 x i8>*), align 1
+; SSE-NEXT:    store <16 x i8> [[TMP12]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48) to <16 x i8>*), align 1
+; SSE-NEXT:    ret void
+;
+; AVX-LABEL: @lshr_v64i8(
+; AVX-NEXT:    [[TMP1:%.*]] = load <32 x i8>, <32 x i8>* bitcast ([64 x i8]* @a8 to <32 x i8>*), align 1
+; AVX-NEXT:    [[TMP2:%.*]] = load <32 x i8>, <32 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32) to <32 x i8>*), align 1
+; AVX-NEXT:    [[TMP3:%.*]] = load <32 x i8>, <32 x i8>* bitcast ([64 x i8]* @b8 to <32 x i8>*), align 1
+; AVX-NEXT:    [[TMP4:%.*]] = load <32 x i8>, <32 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32) to <32 x i8>*), align 1
+; AVX-NEXT:    [[TMP5:%.*]] = lshr <32 x i8> [[TMP1]], [[TMP3]]
+; AVX-NEXT:    [[TMP6:%.*]] = lshr <32 x i8> [[TMP2]], [[TMP4]]
+; AVX-NEXT:    store <32 x i8> [[TMP5]], <32 x i8>* bitcast ([64 x i8]* @c8 to <32 x i8>*), align 1
+; AVX-NEXT:    store <32 x i8> [[TMP6]], <32 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32) to <32 x i8>*), align 1
+; AVX-NEXT:    ret void
+;
+; AVX512-LABEL: @lshr_v64i8(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <64 x i8>, <64 x i8>* bitcast ([64 x i8]* @a8 to <64 x i8>*), align 1
+; AVX512-NEXT:    [[TMP2:%.*]] = load <64 x i8>, <64 x i8>* bitcast ([64 x i8]* @b8 to <64 x i8>*), align 1
+; AVX512-NEXT:    [[TMP3:%.*]] = lshr <64 x i8> [[TMP1]], [[TMP2]]
+; AVX512-NEXT:    store <64 x i8> [[TMP3]], <64 x i8>* bitcast ([64 x i8]* @c8 to <64 x i8>*), align 1
+; AVX512-NEXT:    ret void
+;
+; XOP-LABEL: @lshr_v64i8(
+; XOP-NEXT:    [[TMP1:%.*]] = load <32 x i8>, <32 x i8>* bitcast ([64 x i8]* @a8 to <32 x i8>*), align 1
+; XOP-NEXT:    [[TMP2:%.*]] = load <32 x i8>, <32 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32) to <32 x i8>*), align 1
+; XOP-NEXT:    [[TMP3:%.*]] = load <32 x i8>, <32 x i8>* bitcast ([64 x i8]* @b8 to <32 x i8>*), align 1
+; XOP-NEXT:    [[TMP4:%.*]] = load <32 x i8>, <32 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32) to <32 x i8>*), align 1
+; XOP-NEXT:    [[TMP5:%.*]] = lshr <32 x i8> [[TMP1]], [[TMP3]]
+; XOP-NEXT:    [[TMP6:%.*]] = lshr <32 x i8> [[TMP2]], [[TMP4]]
+; XOP-NEXT:    store <32 x i8> [[TMP5]], <32 x i8>* bitcast ([64 x i8]* @c8 to <32 x i8>*), align 1
+; XOP-NEXT:    store <32 x i8> [[TMP6]], <32 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32) to <32 x i8>*), align 1
+; XOP-NEXT:    ret void
 ;
   %a0  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 0 ), align 1
   %a1  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 1 ), align 1
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/shift-shl.ll b/llvm/test/Transforms/SLPVectorizer/X86/shift-shl.ll
index 8eadd04d1..82f973385 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/shift-shl.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/shift-shl.ll
@@ -383,14 +383,10 @@ define void @shl_v32i16() {
 ; AVX-NEXT:    ret void
 ;
 ; AVX512-LABEL: @shl_v32i16(
-; AVX512-NEXT:    [[TMP1:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @a16 to <16 x i16>*), align 2
-; AVX512-NEXT:    [[TMP2:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <16 x i16>*), align 2
-; AVX512-NEXT:    [[TMP3:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @b16 to <16 x i16>*), align 2
-; AVX512-NEXT:    [[TMP4:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <16 x i16>*), align 2
-; AVX512-NEXT:    [[TMP5:%.*]] = shl <16 x i16> [[TMP1]], [[TMP3]]
-; AVX512-NEXT:    [[TMP6:%.*]] = shl <16 x i16> [[TMP2]], [[TMP4]]
-; AVX512-NEXT:    store <16 x i16> [[TMP5]], <16 x i16>* bitcast ([32 x i16]* @c16 to <16 x i16>*), align 2
-; AVX512-NEXT:    store <16 x i16> [[TMP6]], <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP1:%.*]] = load <32 x i16>, <32 x i16>* bitcast ([32 x i16]* @a16 to <32 x i16>*), align 2
+; AVX512-NEXT:    [[TMP2:%.*]] = load <32 x i16>, <32 x i16>* bitcast ([32 x i16]* @b16 to <32 x i16>*), align 2
+; AVX512-NEXT:    [[TMP3:%.*]] = shl <32 x i16> [[TMP1]], [[TMP2]]
+; AVX512-NEXT:    store <32 x i16> [[TMP3]], <32 x i16>* bitcast ([32 x i16]* @c16 to <32 x i16>*), align 2
 ; AVX512-NEXT:    ret void
 ;
 ; XOP-LABEL: @shl_v32i16(
@@ -536,24 +532,53 @@ define void @shl_v32i16() {
 }
 
 define void @shl_v64i8() {
-; CHECK-LABEL: @shl_v64i8(
-; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @a8 to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP5:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @b8 to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP6:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP7:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP8:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48) to <16 x i8>*), align 1
-; CHECK-NEXT:    [[TMP9:%.*]] = shl <16 x i8> [[TMP1]], [[TMP5]]
-; CHECK-NEXT:    [[TMP10:%.*]] = shl <16 x i8> [[TMP2]], [[TMP6]]
-; CHECK-NEXT:    [[TMP11:%.*]] = shl <16 x i8> [[TMP3]], [[TMP7]]
-; CHECK-NEXT:    [[TMP12:%.*]] = shl <16 x i8> [[TMP4]], [[TMP8]]
-; CHECK-NEXT:    store <16 x i8> [[TMP9]], <16 x i8>* bitcast ([64 x i8]* @c8 to <16 x i8>*), align 1
-; CHECK-NEXT:    store <16 x i8> [[TMP10]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16) to <16 x i8>*), align 1
-; CHECK-NEXT:    store <16 x i8> [[TMP11]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32) to <16 x i8>*), align 1
-; CHECK-NEXT:    store <16 x i8> [[TMP12]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48) to <16 x i8>*), align 1
-; CHECK-NEXT:    ret void
+; SSE-LABEL: @shl_v64i8(
+; SSE-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @a8 to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP3:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP4:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP5:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @b8 to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP6:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP7:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP8:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48) to <16 x i8>*), align 1
+; SSE-NEXT:    [[TMP9:%.*]] = shl <16 x i8> [[TMP1]], [[TMP5]]
+; SSE-NEXT:    [[TMP10:%.*]] = shl <16 x i8> [[TMP2]], [[TMP6]]
+; SSE-NEXT:    [[TMP11:%.*]] = shl <16 x i8> [[TMP3]], [[TMP7]]
+; SSE-NEXT:    [[TMP12:%.*]] = shl <16 x i8> [[TMP4]], [[TMP8]]
+; SSE-NEXT:    store <16 x i8> [[TMP9]], <16 x i8>* bitcast ([64 x i8]* @c8 to <16 x i8>*), align 1
+; SSE-NEXT:    store <16 x i8> [[TMP10]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16) to <16 x i8>*), align 1
+; SSE-NEXT:    store <16 x i8> [[TMP11]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32) to <16 x i8>*), align 1
+; SSE-NEXT:    store <16 x i8> [[TMP12]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48) to <16 x i8>*), align 1
+; SSE-NEXT:    ret void
+;
+; AVX-LABEL: @shl_v64i8(
+; AVX-NEXT:    [[TMP1:%.*]] = load <32 x i8>, <32 x i8>* bitcast ([64 x i8]* @a8 to <32 x i8>*), align 1
+; AVX-NEXT:    [[TMP2:%.*]] = load <32 x i8>, <32 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32) to <32 x i8>*), align 1
+; AVX-NEXT:    [[TMP3:%.*]] = load <32 x i8>, <32 x i8>* bitcast ([64 x i8]* @b8 to <32 x i8>*), align 1
+; AVX-NEXT:    [[TMP4:%.*]] = load <32 x i8>, <32 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32) to <32 x i8>*), align 1
+; AVX-NEXT:    [[TMP5:%.*]] = shl <32 x i8> [[TMP1]], [[TMP3]]
+; AVX-NEXT:    [[TMP6:%.*]] = shl <32 x i8> [[TMP2]], [[TMP4]]
+; AVX-NEXT:    store <32 x i8> [[TMP5]], <32 x i8>* bitcast ([64 x i8]* @c8 to <32 x i8>*), align 1
+; AVX-NEXT:    store <32 x i8> [[TMP6]], <32 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32) to <32 x i8>*), align 1
+; AVX-NEXT:    ret void
+;
+; AVX512-LABEL: @shl_v64i8(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <64 x i8>, <64 x i8>* bitcast ([64 x i8]* @a8 to <64 x i8>*), align 1
+; AVX512-NEXT:    [[TMP2:%.*]] = load <64 x i8>, <64 x i8>* bitcast ([64 x i8]* @b8 to <64 x i8>*), align 1
+; AVX512-NEXT:    [[TMP3:%.*]] = shl <64 x i8> [[TMP1]], [[TMP2]]
+; AVX512-NEXT:    store <64 x i8> [[TMP3]], <64 x i8>* bitcast ([64 x i8]* @c8 to <64 x i8>*), align 1
+; AVX512-NEXT:    ret void
+;
+; XOP-LABEL: @shl_v64i8(
+; XOP-NEXT:    [[TMP1:%.*]] = load <32 x i8>, <32 x i8>* bitcast ([64 x i8]* @a8 to <32 x i8>*), align 1
+; XOP-NEXT:    [[TMP2:%.*]] = load <32 x i8>, <32 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32) to <32 x i8>*), align 1
+; XOP-NEXT:    [[TMP3:%.*]] = load <32 x i8>, <32 x i8>* bitcast ([64 x i8]* @b8 to <32 x i8>*), align 1
+; XOP-NEXT:    [[TMP4:%.*]] = load <32 x i8>, <32 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32) to <32 x i8>*), align 1
+; XOP-NEXT:    [[TMP5:%.*]] = shl <32 x i8> [[TMP1]], [[TMP3]]
+; XOP-NEXT:    [[TMP6:%.*]] = shl <32 x i8> [[TMP2]], [[TMP4]]
+; XOP-NEXT:    store <32 x i8> [[TMP5]], <32 x i8>* bitcast ([64 x i8]* @c8 to <32 x i8>*), align 1
+; XOP-NEXT:    store <32 x i8> [[TMP6]], <32 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32) to <32 x i8>*), align 1
+; XOP-NEXT:    ret void
 ;
   %a0  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 0 ), align 1
   %a1  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 1 ), align 1
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/sitofp.ll b/llvm/test/Transforms/SLPVectorizer/X86/sitofp.ll
index d4c4be5bb..435836a7c 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/sitofp.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/sitofp.ll
@@ -916,11 +916,26 @@ define void @sitofp_16i32_16f32() #0 {
 }
 
 define void @sitofp_4i16_4f32() #0 {
-; CHECK-LABEL: @sitofp_4i16_4f32(
-; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i16>, <4 x i16>* bitcast ([32 x i16]* @src16 to <4 x i16>*), align 64
-; CHECK-NEXT:    [[TMP2:%.*]] = sitofp <4 x i16> [[TMP1]] to <4 x float>
-; CHECK-NEXT:    store <4 x float> [[TMP2]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 64
-; CHECK-NEXT:    ret void
+; SSE-LABEL: @sitofp_4i16_4f32(
+; SSE-NEXT:    [[LD0:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 0), align 64
+; SSE-NEXT:    [[LD1:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 1), align 2
+; SSE-NEXT:    [[LD2:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 2), align 4
+; SSE-NEXT:    [[LD3:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 3), align 2
+; SSE-NEXT:    [[CVT0:%.*]] = sitofp i16 [[LD0]] to float
+; SSE-NEXT:    [[CVT1:%.*]] = sitofp i16 [[LD1]] to float
+; SSE-NEXT:    [[CVT2:%.*]] = sitofp i16 [[LD2]] to float
+; SSE-NEXT:    [[CVT3:%.*]] = sitofp i16 [[LD3]] to float
+; SSE-NEXT:    store float [[CVT0]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 64
+; SSE-NEXT:    store float [[CVT1]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+; SSE-NEXT:    store float [[CVT2]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 8
+; SSE-NEXT:    store float [[CVT3]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+; SSE-NEXT:    ret void
+;
+; AVX-LABEL: @sitofp_4i16_4f32(
+; AVX-NEXT:    [[TMP1:%.*]] = load <4 x i16>, <4 x i16>* bitcast ([32 x i16]* @src16 to <4 x i16>*), align 64
+; AVX-NEXT:    [[TMP2:%.*]] = sitofp <4 x i16> [[TMP1]] to <4 x float>
+; AVX-NEXT:    store <4 x float> [[TMP2]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 64
+; AVX-NEXT:    ret void
 ;
   %ld0 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 0), align 64
   %ld1 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 1), align 2
@@ -939,12 +954,30 @@ define void @sitofp_4i16_4f32() #0 {
 
 define void @sitofp_8i16_8f32() #0 {
 ; SSE-LABEL: @sitofp_8i16_8f32(
-; SSE-NEXT:    [[TMP1:%.*]] = load <4 x i16>, <4 x i16>* bitcast ([32 x i16]* @src16 to <4 x i16>*), align 64
-; SSE-NEXT:    [[TMP2:%.*]] = load <4 x i16>, <4 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 4) to <4 x i16>*), align 8
-; SSE-NEXT:    [[TMP3:%.*]] = sitofp <4 x i16> [[TMP1]] to <4 x float>
-; SSE-NEXT:    [[TMP4:%.*]] = sitofp <4 x i16> [[TMP2]] to <4 x float>
-; SSE-NEXT:    store <4 x float> [[TMP3]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 64
-; SSE-NEXT:    store <4 x float> [[TMP4]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4) to <4 x float>*), align 16
+; SSE-NEXT:    [[LD0:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 0), align 64
+; SSE-NEXT:    [[LD1:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 1), align 2
+; SSE-NEXT:    [[LD2:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 2), align 4
+; SSE-NEXT:    [[LD3:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 3), align 2
+; SSE-NEXT:    [[LD4:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 4), align 8
+; SSE-NEXT:    [[LD5:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 5), align 2
+; SSE-NEXT:    [[LD6:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 6), align 4
+; SSE-NEXT:    [[LD7:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 7), align 2
+; SSE-NEXT:    [[CVT0:%.*]] = sitofp i16 [[LD0]] to float
+; SSE-NEXT:    [[CVT1:%.*]] = sitofp i16 [[LD1]] to float
+; SSE-NEXT:    [[CVT2:%.*]] = sitofp i16 [[LD2]] to float
+; SSE-NEXT:    [[CVT3:%.*]] = sitofp i16 [[LD3]] to float
+; SSE-NEXT:    [[CVT4:%.*]] = sitofp i16 [[LD4]] to float
+; SSE-NEXT:    [[CVT5:%.*]] = sitofp i16 [[LD5]] to float
+; SSE-NEXT:    [[CVT6:%.*]] = sitofp i16 [[LD6]] to float
+; SSE-NEXT:    [[CVT7:%.*]] = sitofp i16 [[LD7]] to float
+; SSE-NEXT:    store float [[CVT0]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 64
+; SSE-NEXT:    store float [[CVT1]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+; SSE-NEXT:    store float [[CVT2]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 8
+; SSE-NEXT:    store float [[CVT3]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+; SSE-NEXT:    store float [[CVT4]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4), align 16
+; SSE-NEXT:    store float [[CVT5]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 5), align 4
+; SSE-NEXT:    store float [[CVT6]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 6), align 8
+; SSE-NEXT:    store float [[CVT7]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 7), align 4
 ; SSE-NEXT:    ret void
 ;
 ; AVX-LABEL: @sitofp_8i16_8f32(
@@ -982,18 +1015,54 @@ define void @sitofp_8i16_8f32() #0 {
 
 define void @sitofp_16i16_16f32() #0 {
 ; SSE-LABEL: @sitofp_16i16_16f32(
-; SSE-NEXT:    [[TMP1:%.*]] = load <4 x i16>, <4 x i16>* bitcast ([32 x i16]* @src16 to <4 x i16>*), align 64
-; SSE-NEXT:    [[TMP2:%.*]] = load <4 x i16>, <4 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 4) to <4 x i16>*), align 8
-; SSE-NEXT:    [[TMP3:%.*]] = load <4 x i16>, <4 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 8) to <4 x i16>*), align 16
-; SSE-NEXT:    [[TMP4:%.*]] = load <4 x i16>, <4 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 12) to <4 x i16>*), align 8
-; SSE-NEXT:    [[TMP5:%.*]] = sitofp <4 x i16> [[TMP1]] to <4 x float>
-; SSE-NEXT:    [[TMP6:%.*]] = sitofp <4 x i16> [[TMP2]] to <4 x float>
-; SSE-NEXT:    [[TMP7:%.*]] = sitofp <4 x i16> [[TMP3]] to <4 x float>
-; SSE-NEXT:    [[TMP8:%.*]] = sitofp <4 x i16> [[TMP4]] to <4 x float>
-; SSE-NEXT:    store <4 x float> [[TMP5]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 64
-; SSE-NEXT:    store <4 x float> [[TMP6]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4) to <4 x float>*), align 16
-; SSE-NEXT:    store <4 x float> [[TMP7]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 8) to <4 x float>*), align 32
-; SSE-NEXT:    store <4 x float> [[TMP8]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 12) to <4 x float>*), align 16
+; SSE-NEXT:    [[LD0:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 0), align 64
+; SSE-NEXT:    [[LD1:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 1), align 2
+; SSE-NEXT:    [[LD2:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 2), align 4
+; SSE-NEXT:    [[LD3:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 3), align 2
+; SSE-NEXT:    [[LD4:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 4), align 8
+; SSE-NEXT:    [[LD5:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 5), align 2
+; SSE-NEXT:    [[LD6:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 6), align 4
+; SSE-NEXT:    [[LD7:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 7), align 2
+; SSE-NEXT:    [[LD8:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 8), align 16
+; SSE-NEXT:    [[LD9:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 9), align 2
+; SSE-NEXT:    [[LD10:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 10), align 4
+; SSE-NEXT:    [[LD11:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 11), align 2
+; SSE-NEXT:    [[LD12:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 12), align 8
+; SSE-NEXT:    [[LD13:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 13), align 2
+; SSE-NEXT:    [[LD14:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 14), align 4
+; SSE-NEXT:    [[LD15:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 15), align 2
+; SSE-NEXT:    [[CVT0:%.*]] = sitofp i16 [[LD0]] to float
+; SSE-NEXT:    [[CVT1:%.*]] = sitofp i16 [[LD1]] to float
+; SSE-NEXT:    [[CVT2:%.*]] = sitofp i16 [[LD2]] to float
+; SSE-NEXT:    [[CVT3:%.*]] = sitofp i16 [[LD3]] to float
+; SSE-NEXT:    [[CVT4:%.*]] = sitofp i16 [[LD4]] to float
+; SSE-NEXT:    [[CVT5:%.*]] = sitofp i16 [[LD5]] to float
+; SSE-NEXT:    [[CVT6:%.*]] = sitofp i16 [[LD6]] to float
+; SSE-NEXT:    [[CVT7:%.*]] = sitofp i16 [[LD7]] to float
+; SSE-NEXT:    [[CVT8:%.*]] = sitofp i16 [[LD8]] to float
+; SSE-NEXT:    [[CVT9:%.*]] = sitofp i16 [[LD9]] to float
+; SSE-NEXT:    [[CVT10:%.*]] = sitofp i16 [[LD10]] to float
+; SSE-NEXT:    [[CVT11:%.*]] = sitofp i16 [[LD11]] to float
+; SSE-NEXT:    [[CVT12:%.*]] = sitofp i16 [[LD12]] to float
+; SSE-NEXT:    [[CVT13:%.*]] = sitofp i16 [[LD13]] to float
+; SSE-NEXT:    [[CVT14:%.*]] = sitofp i16 [[LD14]] to float
+; SSE-NEXT:    [[CVT15:%.*]] = sitofp i16 [[LD15]] to float
+; SSE-NEXT:    store float [[CVT0]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 64
+; SSE-NEXT:    store float [[CVT1]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+; SSE-NEXT:    store float [[CVT2]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 8
+; SSE-NEXT:    store float [[CVT3]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+; SSE-NEXT:    store float [[CVT4]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4), align 16
+; SSE-NEXT:    store float [[CVT5]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 5), align 4
+; SSE-NEXT:    store float [[CVT6]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 6), align 8
+; SSE-NEXT:    store float [[CVT7]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 7), align 4
+; SSE-NEXT:    store float [[CVT8]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 8), align 32
+; SSE-NEXT:    store float [[CVT9]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 9), align 4
+; SSE-NEXT:    store float [[CVT10]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 10), align 8
+; SSE-NEXT:    store float [[CVT11]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 11), align 4
+; SSE-NEXT:    store float [[CVT12]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 12), align 16
+; SSE-NEXT:    store float [[CVT13]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 13), align 4
+; SSE-NEXT:    store float [[CVT14]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 14), align 8
+; SSE-NEXT:    store float [[CVT15]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 15), align 4
 ; SSE-NEXT:    ret void
 ;
 ; AVX256-LABEL: @sitofp_16i16_16f32(
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/sqrt.ll b/llvm/test/Transforms/SLPVectorizer/X86/sqrt.ll
index 06d2134fb..a07da8a0f 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/sqrt.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/sqrt.ll
@@ -1,9 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=bdver1 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skylake-avx512 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown -mattr=avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256
+; RUN: opt < %s -mtriple=x86_64-unknown -mattr=avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256
+; RUN: opt < %s -mtriple=x86_64-unknown -mattr=avx512vl,avx512dq,avx512bw -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX512
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/store-jumbled.ll b/llvm/test/Transforms/SLPVectorizer/X86/store-jumbled.ll
index 2255a1234..e99864205 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/store-jumbled.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/store-jumbled.ll
@@ -11,21 +11,20 @@ define i32 @jumbled-load(i32* noalias nocapture %in, i32* noalias nocapture %inn
 ; CHECK-NEXT:    [[GEP_3:%.*]] = getelementptr inbounds i32, i32* [[IN_ADDR]], i64 3
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32* [[IN_ADDR]] to <4 x i32>*
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP1]], align 4
-; CHECK-NEXT:    [[REORDER_SHUFFLE:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> undef, <4 x i32> <i32 1, i32 3, i32 0, i32 2>
 ; CHECK-NEXT:    [[INN_ADDR:%.*]] = getelementptr inbounds i32, i32* [[INN:%.*]], i64 0
 ; CHECK-NEXT:    [[GEP_4:%.*]] = getelementptr inbounds i32, i32* [[INN_ADDR]], i64 1
 ; CHECK-NEXT:    [[GEP_5:%.*]] = getelementptr inbounds i32, i32* [[INN_ADDR]], i64 2
 ; CHECK-NEXT:    [[GEP_6:%.*]] = getelementptr inbounds i32, i32* [[INN_ADDR]], i64 3
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i32* [[INN_ADDR]] to <4 x i32>*
 ; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i32>, <4 x i32>* [[TMP3]], align 4
-; CHECK-NEXT:    [[REORDER_SHUFFLE1:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> undef, <4 x i32> <i32 1, i32 3, i32 0, i32 2>
-; CHECK-NEXT:    [[TMP5:%.*]] = mul <4 x i32> [[REORDER_SHUFFLE]], [[REORDER_SHUFFLE1]]
+; CHECK-NEXT:    [[TMP5:%.*]] = mul <4 x i32> [[TMP2]], [[TMP4]]
 ; CHECK-NEXT:    [[GEP_7:%.*]] = getelementptr inbounds i32, i32* [[OUT:%.*]], i64 0
 ; CHECK-NEXT:    [[GEP_8:%.*]] = getelementptr inbounds i32, i32* [[OUT]], i64 1
 ; CHECK-NEXT:    [[GEP_9:%.*]] = getelementptr inbounds i32, i32* [[OUT]], i64 2
 ; CHECK-NEXT:    [[GEP_10:%.*]] = getelementptr inbounds i32, i32* [[OUT]], i64 3
+; CHECK-NEXT:    [[REORDER_SHUFFLE:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> undef, <4 x i32> <i32 1, i32 3, i32 0, i32 2>
 ; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32* [[GEP_7]] to <4 x i32>*
-; CHECK-NEXT:    store <4 x i32> [[TMP5]], <4 x i32>* [[TMP6]], align 4
+; CHECK-NEXT:    store <4 x i32> [[REORDER_SHUFFLE]], <4 x i32>* [[TMP6]], align 4
 ; CHECK-NEXT:    ret i32 undef
 ;
   %in.addr = getelementptr inbounds i32, i32* %in, i64 0
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/stores_vectorize.ll b/llvm/test/Transforms/SLPVectorizer/X86/stores_vectorize.ll
index 48f2687b3..fd697f1a7 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/stores_vectorize.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/stores_vectorize.ll
@@ -92,15 +92,14 @@ define void @store_reverse(i64* %p3) {
 ; CHECK-NEXT:    [[ARRAYIDX11:%.*]] = getelementptr inbounds i64, i64* [[P3]], i64 3
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i64* [[P3]] to <4 x i64>*
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, <4 x i64>* [[TMP0]], align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i64> [[TMP1]], <4 x i64> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    [[ARRAYIDX12:%.*]] = getelementptr inbounds i64, i64* [[P3]], i64 11
-; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i64* [[ARRAYIDX1]] to <4 x i64>*
-; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i64>, <4 x i64>* [[TMP3]], align 8
-; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i64> [[TMP4]], <4 x i64> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; CHECK-NEXT:    [[TMP6:%.*]] = shl <4 x i64> [[TMP2]], [[TMP5]]
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i64* [[ARRAYIDX1]] to <4 x i64>*
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i64>, <4 x i64>* [[TMP2]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = shl <4 x i64> [[TMP1]], [[TMP3]]
 ; CHECK-NEXT:    [[ARRAYIDX14:%.*]] = getelementptr inbounds i64, i64* [[P3]], i64 4
-; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i64* [[ARRAYIDX14]] to <4 x i64>*
-; CHECK-NEXT:    store <4 x i64> [[TMP6]], <4 x i64>* [[TMP7]], align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i64> [[TMP4]], <4 x i64> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i64* [[ARRAYIDX14]] to <4 x i64>*
+; CHECK-NEXT:    store <4 x i64> [[TMP5]], <4 x i64>* [[TMP6]], align 8
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -147,18 +146,11 @@ define void @store15(float* %p1, i32 %p2, i64* %p3, float* %p4) {
 ; CHECK-NEXT:    [[ADD:%.*]] = fadd float [[TMP0]], [[TMP1]]
 ; CHECK-NEXT:    store float [[ADD]], float* [[ARRAYIDX2]], align 4
 ; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds i64, i64* [[P3]], i64 1
-; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i64* [[P3]] to <2 x i64>*
-; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, <2 x i64>* [[TMP2]], align 8
-; CHECK-NEXT:    [[TMP4:%.*]] = lshr <2 x i64> [[TMP3]], <i64 5, i64 5>
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i64* [[P3]] to <2 x i64>*
-; CHECK-NEXT:    store <2 x i64> [[TMP4]], <2 x i64>* [[TMP5]], align 8
 ; CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds i64, i64* [[P3]], i64 2
-; CHECK-NEXT:    [[TMP6:%.*]] = load i64, i64* [[ARRAYIDX6]], align 8
-; CHECK-NEXT:    [[SHR7:%.*]] = lshr i64 [[TMP6]], 5
-; CHECK-NEXT:    store i64 [[SHR7]], i64* [[ARRAYIDX6]], align 8
 ; CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds i64, i64* [[P3]], i64 3
-; CHECK-NEXT:    [[TMP7:%.*]] = load i64, i64* [[ARRAYIDX8]], align 8
-; CHECK-NEXT:    [[SHR9:%.*]] = lshr i64 [[TMP7]], 5
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i64* [[P3]] to <4 x i64>*
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i64>, <4 x i64>* [[TMP2]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = lshr <4 x i64> [[TMP3]], <i64 5, i64 5, i64 5, i64 5>
 ; CHECK-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds i64, i64* [[P3]], i64 5
 ; CHECK-NEXT:    store i64 5, i64* [[ARRAYIDX9]], align 8
 ; CHECK-NEXT:    store i64 5, i64* [[ARRAYIDX9]], align 8
@@ -175,7 +167,8 @@ define void @store15(float* %p1, i32 %p2, i64* %p3, float* %p4) {
 ; CHECK-NEXT:    store i64 5, i64* [[ARRAYIDX9]], align 8
 ; CHECK-NEXT:    store i64 5, i64* [[ARRAYIDX9]], align 8
 ; CHECK-NEXT:    store i64 5, i64* [[ARRAYIDX9]], align 8
-; CHECK-NEXT:    store i64 [[SHR9]], i64* [[ARRAYIDX8]], align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i64* [[P3]] to <4 x i64>*
+; CHECK-NEXT:    store <4 x i64> [[TMP4]], <4 x i64>* [[TMP5]], align 8
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -235,18 +228,11 @@ define void @store16(float* %p1, i32 %p2, i64* %p3, float* %p4) {
 ; CHECK-NEXT:    [[ADD:%.*]] = fadd float [[TMP0]], [[TMP1]]
 ; CHECK-NEXT:    store float [[ADD]], float* [[ARRAYIDX2]], align 4
 ; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds i64, i64* [[P3]], i64 1
-; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i64* [[P3]] to <2 x i64>*
-; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, <2 x i64>* [[TMP2]], align 8
-; CHECK-NEXT:    [[TMP4:%.*]] = lshr <2 x i64> [[TMP3]], <i64 5, i64 5>
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i64* [[P3]] to <2 x i64>*
-; CHECK-NEXT:    store <2 x i64> [[TMP4]], <2 x i64>* [[TMP5]], align 8
 ; CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds i64, i64* [[P3]], i64 2
-; CHECK-NEXT:    [[TMP6:%.*]] = load i64, i64* [[ARRAYIDX6]], align 8
-; CHECK-NEXT:    [[SHR7:%.*]] = lshr i64 [[TMP6]], 5
-; CHECK-NEXT:    store i64 [[SHR7]], i64* [[ARRAYIDX6]], align 8
 ; CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds i64, i64* [[P3]], i64 3
-; CHECK-NEXT:    [[TMP7:%.*]] = load i64, i64* [[ARRAYIDX8]], align 8
-; CHECK-NEXT:    [[SHR9:%.*]] = lshr i64 [[TMP7]], 5
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i64* [[P3]] to <4 x i64>*
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i64>, <4 x i64>* [[TMP2]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = lshr <4 x i64> [[TMP3]], <i64 5, i64 5, i64 5, i64 5>
 ; CHECK-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds i64, i64* [[P3]], i64 5
 ; CHECK-NEXT:    store i64 5, i64* [[ARRAYIDX9]], align 8
 ; CHECK-NEXT:    store i64 5, i64* [[ARRAYIDX9]], align 8
@@ -264,7 +250,8 @@ define void @store16(float* %p1, i32 %p2, i64* %p3, float* %p4) {
 ; CHECK-NEXT:    store i64 5, i64* [[ARRAYIDX9]], align 8
 ; CHECK-NEXT:    store i64 5, i64* [[ARRAYIDX9]], align 8
 ; CHECK-NEXT:    store i64 5, i64* [[ARRAYIDX9]], align 8
-; CHECK-NEXT:    store i64 [[SHR9]], i64* [[ARRAYIDX8]], align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i64* [[P3]] to <4 x i64>*
+; CHECK-NEXT:    store <4 x i64> [[TMP4]], <4 x i64>* [[TMP5]], align 8
 ; CHECK-NEXT:    ret void
 ;
 entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/uitofp.ll b/llvm/test/Transforms/SLPVectorizer/X86/uitofp.ll
index 092918115..97b4cbefd 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/uitofp.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/uitofp.ll
@@ -868,11 +868,26 @@ define void @uitofp_16i32_16f32() #0 {
 }
 
 define void @uitofp_4i16_4f32() #0 {
-; CHECK-LABEL: @uitofp_4i16_4f32(
-; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i16>, <4 x i16>* bitcast ([32 x i16]* @src16 to <4 x i16>*), align 64
-; CHECK-NEXT:    [[TMP2:%.*]] = uitofp <4 x i16> [[TMP1]] to <4 x float>
-; CHECK-NEXT:    store <4 x float> [[TMP2]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 64
-; CHECK-NEXT:    ret void
+; SSE-LABEL: @uitofp_4i16_4f32(
+; SSE-NEXT:    [[LD0:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 0), align 64
+; SSE-NEXT:    [[LD1:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 1), align 2
+; SSE-NEXT:    [[LD2:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 2), align 4
+; SSE-NEXT:    [[LD3:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 3), align 2
+; SSE-NEXT:    [[CVT0:%.*]] = uitofp i16 [[LD0]] to float
+; SSE-NEXT:    [[CVT1:%.*]] = uitofp i16 [[LD1]] to float
+; SSE-NEXT:    [[CVT2:%.*]] = uitofp i16 [[LD2]] to float
+; SSE-NEXT:    [[CVT3:%.*]] = uitofp i16 [[LD3]] to float
+; SSE-NEXT:    store float [[CVT0]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 64
+; SSE-NEXT:    store float [[CVT1]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+; SSE-NEXT:    store float [[CVT2]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 8
+; SSE-NEXT:    store float [[CVT3]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+; SSE-NEXT:    ret void
+;
+; AVX-LABEL: @uitofp_4i16_4f32(
+; AVX-NEXT:    [[TMP1:%.*]] = load <4 x i16>, <4 x i16>* bitcast ([32 x i16]* @src16 to <4 x i16>*), align 64
+; AVX-NEXT:    [[TMP2:%.*]] = uitofp <4 x i16> [[TMP1]] to <4 x float>
+; AVX-NEXT:    store <4 x float> [[TMP2]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 64
+; AVX-NEXT:    ret void
 ;
   %ld0 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 0), align 64
   %ld1 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 1), align 2
@@ -891,12 +906,30 @@ define void @uitofp_4i16_4f32() #0 {
 
 define void @uitofp_8i16_8f32() #0 {
 ; SSE-LABEL: @uitofp_8i16_8f32(
-; SSE-NEXT:    [[TMP1:%.*]] = load <4 x i16>, <4 x i16>* bitcast ([32 x i16]* @src16 to <4 x i16>*), align 64
-; SSE-NEXT:    [[TMP2:%.*]] = load <4 x i16>, <4 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 4) to <4 x i16>*), align 8
-; SSE-NEXT:    [[TMP3:%.*]] = uitofp <4 x i16> [[TMP1]] to <4 x float>
-; SSE-NEXT:    [[TMP4:%.*]] = uitofp <4 x i16> [[TMP2]] to <4 x float>
-; SSE-NEXT:    store <4 x float> [[TMP3]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 64
-; SSE-NEXT:    store <4 x float> [[TMP4]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4) to <4 x float>*), align 16
+; SSE-NEXT:    [[LD0:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 0), align 64
+; SSE-NEXT:    [[LD1:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 1), align 2
+; SSE-NEXT:    [[LD2:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 2), align 4
+; SSE-NEXT:    [[LD3:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 3), align 2
+; SSE-NEXT:    [[LD4:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 4), align 8
+; SSE-NEXT:    [[LD5:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 5), align 2
+; SSE-NEXT:    [[LD6:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 6), align 4
+; SSE-NEXT:    [[LD7:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 7), align 2
+; SSE-NEXT:    [[CVT0:%.*]] = uitofp i16 [[LD0]] to float
+; SSE-NEXT:    [[CVT1:%.*]] = uitofp i16 [[LD1]] to float
+; SSE-NEXT:    [[CVT2:%.*]] = uitofp i16 [[LD2]] to float
+; SSE-NEXT:    [[CVT3:%.*]] = uitofp i16 [[LD3]] to float
+; SSE-NEXT:    [[CVT4:%.*]] = uitofp i16 [[LD4]] to float
+; SSE-NEXT:    [[CVT5:%.*]] = uitofp i16 [[LD5]] to float
+; SSE-NEXT:    [[CVT6:%.*]] = uitofp i16 [[LD6]] to float
+; SSE-NEXT:    [[CVT7:%.*]] = uitofp i16 [[LD7]] to float
+; SSE-NEXT:    store float [[CVT0]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 64
+; SSE-NEXT:    store float [[CVT1]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+; SSE-NEXT:    store float [[CVT2]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 8
+; SSE-NEXT:    store float [[CVT3]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+; SSE-NEXT:    store float [[CVT4]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4), align 16
+; SSE-NEXT:    store float [[CVT5]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 5), align 4
+; SSE-NEXT:    store float [[CVT6]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 6), align 8
+; SSE-NEXT:    store float [[CVT7]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 7), align 4
 ; SSE-NEXT:    ret void
 ;
 ; AVX-LABEL: @uitofp_8i16_8f32(
@@ -934,18 +967,54 @@ define void @uitofp_8i16_8f32() #0 {
 
 define void @uitofp_16i16_16f32() #0 {
 ; SSE-LABEL: @uitofp_16i16_16f32(
-; SSE-NEXT:    [[TMP1:%.*]] = load <4 x i16>, <4 x i16>* bitcast ([32 x i16]* @src16 to <4 x i16>*), align 64
-; SSE-NEXT:    [[TMP2:%.*]] = load <4 x i16>, <4 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 4) to <4 x i16>*), align 8
-; SSE-NEXT:    [[TMP3:%.*]] = load <4 x i16>, <4 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 8) to <4 x i16>*), align 16
-; SSE-NEXT:    [[TMP4:%.*]] = load <4 x i16>, <4 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 12) to <4 x i16>*), align 8
-; SSE-NEXT:    [[TMP5:%.*]] = uitofp <4 x i16> [[TMP1]] to <4 x float>
-; SSE-NEXT:    [[TMP6:%.*]] = uitofp <4 x i16> [[TMP2]] to <4 x float>
-; SSE-NEXT:    [[TMP7:%.*]] = uitofp <4 x i16> [[TMP3]] to <4 x float>
-; SSE-NEXT:    [[TMP8:%.*]] = uitofp <4 x i16> [[TMP4]] to <4 x float>
-; SSE-NEXT:    store <4 x float> [[TMP5]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 64
-; SSE-NEXT:    store <4 x float> [[TMP6]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4) to <4 x float>*), align 16
-; SSE-NEXT:    store <4 x float> [[TMP7]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 8) to <4 x float>*), align 32
-; SSE-NEXT:    store <4 x float> [[TMP8]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 12) to <4 x float>*), align 16
+; SSE-NEXT:    [[LD0:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 0), align 64
+; SSE-NEXT:    [[LD1:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 1), align 2
+; SSE-NEXT:    [[LD2:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 2), align 4
+; SSE-NEXT:    [[LD3:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 3), align 2
+; SSE-NEXT:    [[LD4:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 4), align 8
+; SSE-NEXT:    [[LD5:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 5), align 2
+; SSE-NEXT:    [[LD6:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 6), align 4
+; SSE-NEXT:    [[LD7:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 7), align 2
+; SSE-NEXT:    [[LD8:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 8), align 16
+; SSE-NEXT:    [[LD9:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 9), align 2
+; SSE-NEXT:    [[LD10:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 10), align 4
+; SSE-NEXT:    [[LD11:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 11), align 2
+; SSE-NEXT:    [[LD12:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 12), align 8
+; SSE-NEXT:    [[LD13:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 13), align 2
+; SSE-NEXT:    [[LD14:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 14), align 4
+; SSE-NEXT:    [[LD15:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 15), align 2
+; SSE-NEXT:    [[CVT0:%.*]] = uitofp i16 [[LD0]] to float
+; SSE-NEXT:    [[CVT1:%.*]] = uitofp i16 [[LD1]] to float
+; SSE-NEXT:    [[CVT2:%.*]] = uitofp i16 [[LD2]] to float
+; SSE-NEXT:    [[CVT3:%.*]] = uitofp i16 [[LD3]] to float
+; SSE-NEXT:    [[CVT4:%.*]] = uitofp i16 [[LD4]] to float
+; SSE-NEXT:    [[CVT5:%.*]] = uitofp i16 [[LD5]] to float
+; SSE-NEXT:    [[CVT6:%.*]] = uitofp i16 [[LD6]] to float
+; SSE-NEXT:    [[CVT7:%.*]] = uitofp i16 [[LD7]] to float
+; SSE-NEXT:    [[CVT8:%.*]] = uitofp i16 [[LD8]] to float
+; SSE-NEXT:    [[CVT9:%.*]] = uitofp i16 [[LD9]] to float
+; SSE-NEXT:    [[CVT10:%.*]] = uitofp i16 [[LD10]] to float
+; SSE-NEXT:    [[CVT11:%.*]] = uitofp i16 [[LD11]] to float
+; SSE-NEXT:    [[CVT12:%.*]] = uitofp i16 [[LD12]] to float
+; SSE-NEXT:    [[CVT13:%.*]] = uitofp i16 [[LD13]] to float
+; SSE-NEXT:    [[CVT14:%.*]] = uitofp i16 [[LD14]] to float
+; SSE-NEXT:    [[CVT15:%.*]] = uitofp i16 [[LD15]] to float
+; SSE-NEXT:    store float [[CVT0]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 64
+; SSE-NEXT:    store float [[CVT1]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+; SSE-NEXT:    store float [[CVT2]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 8
+; SSE-NEXT:    store float [[CVT3]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+; SSE-NEXT:    store float [[CVT4]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4), align 16
+; SSE-NEXT:    store float [[CVT5]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 5), align 4
+; SSE-NEXT:    store float [[CVT6]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 6), align 8
+; SSE-NEXT:    store float [[CVT7]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 7), align 4
+; SSE-NEXT:    store float [[CVT8]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 8), align 32
+; SSE-NEXT:    store float [[CVT9]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 9), align 4
+; SSE-NEXT:    store float [[CVT10]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 10), align 8
+; SSE-NEXT:    store float [[CVT11]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 11), align 4
+; SSE-NEXT:    store float [[CVT12]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 12), align 16
+; SSE-NEXT:    store float [[CVT13]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 13), align 4
+; SSE-NEXT:    store float [[CVT14]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 14), align 8
+; SSE-NEXT:    store float [[CVT15]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 15), align 4
 ; SSE-NEXT:    ret void
 ;
 ; AVX256-LABEL: @uitofp_16i16_16f32(
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/undef_vect.ll b/llvm/test/Transforms/SLPVectorizer/X86/undef_vect.ll
index 7ecd5805f..822c362ec 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/undef_vect.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/undef_vect.ll
@@ -16,15 +16,6 @@ define void @_Z2azv() local_unnamed_addr {
 ; CHECK-NEXT:    [[DOTSROA_RAW_IDX_7:%.*]] = getelementptr inbounds %"struct.std::h.0.4.8.12.16.20.24.28.248.0.1.2.3.76", %"struct.std::h.0.4.8.12.16.20.24.28.248.0.1.2.3.76"* undef, i64 7, i32 1
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32* [[DOTSROA_CAST_4]] to <8 x i32>*
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* [[TMP0]], align 4
-; CHECK-NEXT:    [[CMP_I1_4:%.*]] = icmp slt i32 undef, undef
-; CHECK-NEXT:    [[DOTSROA_SPECULATED_4:%.*]] = select i1 [[CMP_I1_4]], i32 undef, i32 undef
-; CHECK-NEXT:    [[CMP_I1_5:%.*]] = icmp slt i32 [[DOTSROA_SPECULATED_4]], undef
-; CHECK-NEXT:    [[DOTSROA_SPECULATED_5:%.*]] = select i1 [[CMP_I1_5]], i32 undef, i32 [[DOTSROA_SPECULATED_4]]
-; CHECK-NEXT:    [[CMP_I1_6:%.*]] = icmp slt i32 [[DOTSROA_SPECULATED_5]], undef
-; CHECK-NEXT:    [[DOTSROA_SPECULATED_6:%.*]] = select i1 [[CMP_I1_6]], i32 undef, i32 [[DOTSROA_SPECULATED_5]]
-; CHECK-NEXT:    [[CMP_I1_7:%.*]] = icmp slt i32 [[DOTSROA_SPECULATED_6]], undef
-; CHECK-NEXT:    [[DOTSROA_SPECULATED_7:%.*]] = select i1 [[CMP_I1_7]], i32 undef, i32 [[DOTSROA_SPECULATED_6]]
-; CHECK-NEXT:    [[CMP_I1_8:%.*]] = icmp slt i32 undef, undef
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = icmp sgt <8 x i32> [[TMP1]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <8 x i1> [[RDX_MINMAX_CMP]], <8 x i32> [[TMP1]], <8 x i32> [[RDX_SHUF]]
@@ -39,7 +30,6 @@ define void @_Z2azv() local_unnamed_addr {
 ; CHECK-NEXT:    [[OP_EXTRA:%.*]] = select i1 [[TMP3]], i32 [[TMP2]], i32 undef
 ; CHECK-NEXT:    [[TMP4:%.*]] = icmp sgt i32 [[OP_EXTRA]], undef
 ; CHECK-NEXT:    [[OP_EXTRA7:%.*]] = select i1 [[TMP4]], i32 [[OP_EXTRA]], i32 undef
-; CHECK-NEXT:    [[DOTSROA_SPECULATED_8:%.*]] = select i1 [[CMP_I1_8]], i32 undef, i32 undef
 ; CHECK-NEXT:    [[DOTSROA_SPECULATED_9:%.*]] = select i1 undef, i32 undef, i32 [[OP_EXTRA7]]
 ; CHECK-NEXT:    [[CMP_I1_10:%.*]] = icmp slt i32 [[DOTSROA_SPECULATED_9]], undef
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/used-reduced-op.ll b/llvm/test/Transforms/SLPVectorizer/X86/used-reduced-op.ll
new file mode 100644
index 000000000..468821130
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/used-reduced-op.ll
@@ -0,0 +1,529 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -slp-vectorizer -S -o - -mtriple=x86_64-unknown-linux-gnu -mcpu=haswell < %s | FileCheck %s
+
+@k = external dso_local constant [8 x [4 x i32]], align 16
+@l = external dso_local global [366 x i32], align 16
+
+; Function Attrs: nofree norecurse noreturn nounwind writeonly
+define void @n() local_unnamed_addr #0 {
+; CHECK-LABEL: @n(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 0, i64 0), align 16
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 0, i64 1) to <4 x i32>*), align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 1, i64 1), align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 1, i64 2), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 1, i64 3), align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 2, i64 0), align 16
+; CHECK-NEXT:    [[TMP6:%.*]] = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 2, i64 1), align 4
+; CHECK-NEXT:    [[TMP7:%.*]] = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 2, i64 2), align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 2, i64 3), align 4
+; CHECK-NEXT:    [[TMP9:%.*]] = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 3, i64 0), align 16
+; CHECK-NEXT:    [[TMP10:%.*]] = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 3, i64 1), align 4
+; CHECK-NEXT:    [[TMP11:%.*]] = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 3, i64 2), align 8
+; CHECK-NEXT:    [[TMP12:%.*]] = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 3, i64 3), align 4
+; CHECK-NEXT:    [[TMP13:%.*]] = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 4, i64 0), align 16
+; CHECK-NEXT:    [[TMP14:%.*]] = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 4, i64 1), align 4
+; CHECK-NEXT:    [[TMP15:%.*]] = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 4, i64 2), align 8
+; CHECK-NEXT:    [[TMP16:%.*]] = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 4, i64 3), align 4
+; CHECK-NEXT:    [[TMP17:%.*]] = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 5, i64 0), align 16
+; CHECK-NEXT:    [[TMP18:%.*]] = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 5, i64 1), align 4
+; CHECK-NEXT:    [[TMP19:%.*]] = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 5, i64 2), align 8
+; CHECK-NEXT:    [[TMP20:%.*]] = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 5, i64 3), align 4
+; CHECK-NEXT:    [[TMP21:%.*]] = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 6, i64 0), align 16
+; CHECK-NEXT:    [[TMP22:%.*]] = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 6, i64 1), align 4
+; CHECK-NEXT:    [[TMP23:%.*]] = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 6, i64 2), align 8
+; CHECK-NEXT:    [[TMP24:%.*]] = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 6, i64 3), align 4
+; CHECK-NEXT:    [[TMP25:%.*]] = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 7, i64 0), align 16
+; CHECK-NEXT:    [[TMP26:%.*]] = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 7, i64 1), align 4
+; CHECK-NEXT:    [[TMP27:%.*]] = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 7, i64 2), align 8
+; CHECK-NEXT:    [[TMP28:%.*]] = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 7, i64 3), align 4
+; CHECK-NEXT:    br label [[FOR_COND:%.*]]
+; CHECK:       for.cond:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_COND]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[B_0:%.*]] = phi i32 [ [[SPEC_SELECT8_3_7:%.*]], [[FOR_COND]] ], [ undef, [[ENTRY]] ]
+; CHECK-NEXT:    [[TMP29:%.*]] = trunc i64 [[INDVARS_IV]] to i32
+; CHECK-NEXT:    [[TMP30:%.*]] = add i32 [[TMP29]], -183
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 [[TMP30]], [[TMP0]]
+; CHECK-NEXT:    [[TMP31:%.*]] = icmp slt i32 [[SUB]], 0
+; CHECK-NEXT:    [[NEG:%.*]] = sub nsw i32 0, [[SUB]]
+; CHECK-NEXT:    [[TMP32:%.*]] = select i1 [[TMP31]], i32 [[NEG]], i32 [[SUB]]
+; CHECK-NEXT:    [[TMP33:%.*]] = insertelement <4 x i32> undef, i32 [[TMP30]], i32 0
+; CHECK-NEXT:    [[TMP34:%.*]] = insertelement <4 x i32> [[TMP33]], i32 [[TMP30]], i32 1
+; CHECK-NEXT:    [[TMP35:%.*]] = insertelement <4 x i32> [[TMP34]], i32 [[TMP30]], i32 2
+; CHECK-NEXT:    [[TMP36:%.*]] = insertelement <4 x i32> [[TMP35]], i32 [[TMP30]], i32 3
+; CHECK-NEXT:    [[TMP37:%.*]] = sub <4 x i32> [[TMP36]], [[TMP1]]
+; CHECK-NEXT:    [[TMP38:%.*]] = icmp slt <4 x i32> [[TMP37]], zeroinitializer
+; CHECK-NEXT:    [[TMP39:%.*]] = sub nsw <4 x i32> zeroinitializer, [[TMP37]]
+; CHECK-NEXT:    [[TMP40:%.*]] = select <4 x i1> [[TMP38]], <4 x i32> [[TMP39]], <4 x i32> [[TMP37]]
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[TMP40]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = icmp slt <4 x i32> [[TMP40]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x i32> [[TMP40]], <4 x i32> [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x i32> [[RDX_MINMAX_SELECT]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_MINMAX_CMP2:%.*]] = icmp slt <4 x i32> [[RDX_MINMAX_SELECT]], [[RDX_SHUF1]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT3:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP2]], <4 x i32> [[RDX_MINMAX_SELECT]], <4 x i32> [[RDX_SHUF1]]
+; CHECK-NEXT:    [[TMP41:%.*]] = extractelement <4 x i32> [[RDX_MINMAX_SELECT3]], i32 0
+; CHECK-NEXT:    [[TMP42:%.*]] = icmp slt i32 [[TMP41]], [[TMP32]]
+; CHECK-NEXT:    [[TMP43:%.*]] = select i1 [[TMP42]], i32 [[TMP41]], i32 [[TMP32]]
+; CHECK-NEXT:    [[TMP44:%.*]] = icmp slt i32 [[TMP43]], [[B_0]]
+; CHECK-NEXT:    [[OP_EXTRA:%.*]] = select i1 [[TMP44]], i32 [[TMP43]], i32 [[B_0]]
+; CHECK-NEXT:    [[SUB_1_1:%.*]] = sub i32 [[TMP30]], [[TMP2]]
+; CHECK-NEXT:    [[TMP45:%.*]] = icmp slt i32 [[SUB_1_1]], 0
+; CHECK-NEXT:    [[NEG_1_1:%.*]] = sub nsw i32 0, [[SUB_1_1]]
+; CHECK-NEXT:    [[TMP46:%.*]] = select i1 [[TMP45]], i32 [[NEG_1_1]], i32 [[SUB_1_1]]
+; CHECK-NEXT:    [[CMP12_1_1:%.*]] = icmp slt i32 [[TMP46]], [[OP_EXTRA]]
+; CHECK-NEXT:    [[NARROW:%.*]] = or i1 [[CMP12_1_1]], [[TMP44]]
+; CHECK-NEXT:    [[SPEC_SELECT8_1_1:%.*]] = select i1 [[CMP12_1_1]], i32 [[TMP46]], i32 [[OP_EXTRA]]
+; CHECK-NEXT:    [[SUB_2_1:%.*]] = sub i32 [[TMP30]], [[TMP3]]
+; CHECK-NEXT:    [[TMP47:%.*]] = icmp slt i32 [[SUB_2_1]], 0
+; CHECK-NEXT:    [[NEG_2_1:%.*]] = sub nsw i32 0, [[SUB_2_1]]
+; CHECK-NEXT:    [[TMP48:%.*]] = select i1 [[TMP47]], i32 [[NEG_2_1]], i32 [[SUB_2_1]]
+; CHECK-NEXT:    [[CMP12_2_1:%.*]] = icmp slt i32 [[TMP48]], [[SPEC_SELECT8_1_1]]
+; CHECK-NEXT:    [[NARROW34:%.*]] = or i1 [[CMP12_2_1]], [[NARROW]]
+; CHECK-NEXT:    [[SPEC_SELECT8_2_1:%.*]] = select i1 [[CMP12_2_1]], i32 [[TMP48]], i32 [[SPEC_SELECT8_1_1]]
+; CHECK-NEXT:    [[SUB_3_1:%.*]] = sub i32 [[TMP30]], [[TMP4]]
+; CHECK-NEXT:    [[TMP49:%.*]] = icmp slt i32 [[SUB_3_1]], 0
+; CHECK-NEXT:    [[NEG_3_1:%.*]] = sub nsw i32 0, [[SUB_3_1]]
+; CHECK-NEXT:    [[TMP50:%.*]] = select i1 [[TMP49]], i32 [[NEG_3_1]], i32 [[SUB_3_1]]
+; CHECK-NEXT:    [[CMP12_3_1:%.*]] = icmp slt i32 [[TMP50]], [[SPEC_SELECT8_2_1]]
+; CHECK-NEXT:    [[NARROW35:%.*]] = or i1 [[CMP12_3_1]], [[NARROW34]]
+; CHECK-NEXT:    [[SPEC_SELECT_3_1:%.*]] = zext i1 [[NARROW35]] to i32
+; CHECK-NEXT:    [[SPEC_SELECT8_3_1:%.*]] = select i1 [[CMP12_3_1]], i32 [[TMP50]], i32 [[SPEC_SELECT8_2_1]]
+; CHECK-NEXT:    [[SUB_222:%.*]] = sub i32 [[TMP30]], [[TMP5]]
+; CHECK-NEXT:    [[TMP51:%.*]] = icmp slt i32 [[SUB_222]], 0
+; CHECK-NEXT:    [[NEG_223:%.*]] = sub nsw i32 0, [[SUB_222]]
+; CHECK-NEXT:    [[TMP52:%.*]] = select i1 [[TMP51]], i32 [[NEG_223]], i32 [[SUB_222]]
+; CHECK-NEXT:    [[CMP12_224:%.*]] = icmp slt i32 [[TMP52]], [[SPEC_SELECT8_3_1]]
+; CHECK-NEXT:    [[SPEC_SELECT8_226:%.*]] = select i1 [[CMP12_224]], i32 [[TMP52]], i32 [[SPEC_SELECT8_3_1]]
+; CHECK-NEXT:    [[SUB_1_2:%.*]] = sub i32 [[TMP30]], [[TMP6]]
+; CHECK-NEXT:    [[TMP53:%.*]] = icmp slt i32 [[SUB_1_2]], 0
+; CHECK-NEXT:    [[NEG_1_2:%.*]] = sub nsw i32 0, [[SUB_1_2]]
+; CHECK-NEXT:    [[TMP54:%.*]] = select i1 [[TMP53]], i32 [[NEG_1_2]], i32 [[SUB_1_2]]
+; CHECK-NEXT:    [[CMP12_1_2:%.*]] = icmp slt i32 [[TMP54]], [[SPEC_SELECT8_226]]
+; CHECK-NEXT:    [[TMP55:%.*]] = or i1 [[CMP12_1_2]], [[CMP12_224]]
+; CHECK-NEXT:    [[SPEC_SELECT8_1_2:%.*]] = select i1 [[CMP12_1_2]], i32 [[TMP54]], i32 [[SPEC_SELECT8_226]]
+; CHECK-NEXT:    [[SUB_2_2:%.*]] = sub i32 [[TMP30]], [[TMP7]]
+; CHECK-NEXT:    [[TMP56:%.*]] = icmp slt i32 [[SUB_2_2]], 0
+; CHECK-NEXT:    [[NEG_2_2:%.*]] = sub nsw i32 0, [[SUB_2_2]]
+; CHECK-NEXT:    [[TMP57:%.*]] = select i1 [[TMP56]], i32 [[NEG_2_2]], i32 [[SUB_2_2]]
+; CHECK-NEXT:    [[CMP12_2_2:%.*]] = icmp slt i32 [[TMP57]], [[SPEC_SELECT8_1_2]]
+; CHECK-NEXT:    [[TMP58:%.*]] = or i1 [[CMP12_2_2]], [[TMP55]]
+; CHECK-NEXT:    [[SPEC_SELECT8_2_2:%.*]] = select i1 [[CMP12_2_2]], i32 [[TMP57]], i32 [[SPEC_SELECT8_1_2]]
+; CHECK-NEXT:    [[SUB_3_2:%.*]] = sub i32 [[TMP30]], [[TMP8]]
+; CHECK-NEXT:    [[TMP59:%.*]] = icmp slt i32 [[SUB_3_2]], 0
+; CHECK-NEXT:    [[NEG_3_2:%.*]] = sub nsw i32 0, [[SUB_3_2]]
+; CHECK-NEXT:    [[TMP60:%.*]] = select i1 [[TMP59]], i32 [[NEG_3_2]], i32 [[SUB_3_2]]
+; CHECK-NEXT:    [[CMP12_3_2:%.*]] = icmp slt i32 [[TMP60]], [[SPEC_SELECT8_2_2]]
+; CHECK-NEXT:    [[TMP61:%.*]] = or i1 [[CMP12_3_2]], [[TMP58]]
+; CHECK-NEXT:    [[SPEC_SELECT_3_2:%.*]] = select i1 [[TMP61]], i32 2, i32 [[SPEC_SELECT_3_1]]
+; CHECK-NEXT:    [[SPEC_SELECT8_3_2:%.*]] = select i1 [[CMP12_3_2]], i32 [[TMP60]], i32 [[SPEC_SELECT8_2_2]]
+; CHECK-NEXT:    [[SUB_328:%.*]] = sub i32 [[TMP30]], [[TMP9]]
+; CHECK-NEXT:    [[TMP62:%.*]] = icmp slt i32 [[SUB_328]], 0
+; CHECK-NEXT:    [[NEG_329:%.*]] = sub nsw i32 0, [[SUB_328]]
+; CHECK-NEXT:    [[TMP63:%.*]] = select i1 [[TMP62]], i32 [[NEG_329]], i32 [[SUB_328]]
+; CHECK-NEXT:    [[CMP12_330:%.*]] = icmp slt i32 [[TMP63]], [[SPEC_SELECT8_3_2]]
+; CHECK-NEXT:    [[SPEC_SELECT8_332:%.*]] = select i1 [[CMP12_330]], i32 [[TMP63]], i32 [[SPEC_SELECT8_3_2]]
+; CHECK-NEXT:    [[SUB_1_3:%.*]] = sub i32 [[TMP30]], [[TMP10]]
+; CHECK-NEXT:    [[TMP64:%.*]] = icmp slt i32 [[SUB_1_3]], 0
+; CHECK-NEXT:    [[NEG_1_3:%.*]] = sub nsw i32 0, [[SUB_1_3]]
+; CHECK-NEXT:    [[TMP65:%.*]] = select i1 [[TMP64]], i32 [[NEG_1_3]], i32 [[SUB_1_3]]
+; CHECK-NEXT:    [[CMP12_1_3:%.*]] = icmp slt i32 [[TMP65]], [[SPEC_SELECT8_332]]
+; CHECK-NEXT:    [[TMP66:%.*]] = or i1 [[CMP12_1_3]], [[CMP12_330]]
+; CHECK-NEXT:    [[SPEC_SELECT8_1_3:%.*]] = select i1 [[CMP12_1_3]], i32 [[TMP65]], i32 [[SPEC_SELECT8_332]]
+; CHECK-NEXT:    [[SUB_2_3:%.*]] = sub i32 [[TMP30]], [[TMP11]]
+; CHECK-NEXT:    [[TMP67:%.*]] = icmp slt i32 [[SUB_2_3]], 0
+; CHECK-NEXT:    [[NEG_2_3:%.*]] = sub nsw i32 0, [[SUB_2_3]]
+; CHECK-NEXT:    [[TMP68:%.*]] = select i1 [[TMP67]], i32 [[NEG_2_3]], i32 [[SUB_2_3]]
+; CHECK-NEXT:    [[CMP12_2_3:%.*]] = icmp slt i32 [[TMP68]], [[SPEC_SELECT8_1_3]]
+; CHECK-NEXT:    [[TMP69:%.*]] = or i1 [[CMP12_2_3]], [[TMP66]]
+; CHECK-NEXT:    [[SPEC_SELECT8_2_3:%.*]] = select i1 [[CMP12_2_3]], i32 [[TMP68]], i32 [[SPEC_SELECT8_1_3]]
+; CHECK-NEXT:    [[SUB_3_3:%.*]] = sub i32 [[TMP30]], [[TMP12]]
+; CHECK-NEXT:    [[TMP70:%.*]] = icmp slt i32 [[SUB_3_3]], 0
+; CHECK-NEXT:    [[NEG_3_3:%.*]] = sub nsw i32 0, [[SUB_3_3]]
+; CHECK-NEXT:    [[TMP71:%.*]] = select i1 [[TMP70]], i32 [[NEG_3_3]], i32 [[SUB_3_3]]
+; CHECK-NEXT:    [[CMP12_3_3:%.*]] = icmp slt i32 [[TMP71]], [[SPEC_SELECT8_2_3]]
+; CHECK-NEXT:    [[TMP72:%.*]] = or i1 [[CMP12_3_3]], [[TMP69]]
+; CHECK-NEXT:    [[SPEC_SELECT_3_3:%.*]] = select i1 [[TMP72]], i32 3, i32 [[SPEC_SELECT_3_2]]
+; CHECK-NEXT:    [[SPEC_SELECT8_3_3:%.*]] = select i1 [[CMP12_3_3]], i32 [[TMP71]], i32 [[SPEC_SELECT8_2_3]]
+; CHECK-NEXT:    [[SUB_4:%.*]] = sub i32 [[TMP30]], [[TMP13]]
+; CHECK-NEXT:    [[TMP73:%.*]] = icmp slt i32 [[SUB_4]], 0
+; CHECK-NEXT:    [[NEG_4:%.*]] = sub nsw i32 0, [[SUB_4]]
+; CHECK-NEXT:    [[TMP74:%.*]] = select i1 [[TMP73]], i32 [[NEG_4]], i32 [[SUB_4]]
+; CHECK-NEXT:    [[CMP12_4:%.*]] = icmp slt i32 [[TMP74]], [[SPEC_SELECT8_3_3]]
+; CHECK-NEXT:    [[SPEC_SELECT8_4:%.*]] = select i1 [[CMP12_4]], i32 [[TMP74]], i32 [[SPEC_SELECT8_3_3]]
+; CHECK-NEXT:    [[SUB_1_4:%.*]] = sub i32 [[TMP30]], [[TMP14]]
+; CHECK-NEXT:    [[TMP75:%.*]] = icmp slt i32 [[SUB_1_4]], 0
+; CHECK-NEXT:    [[NEG_1_4:%.*]] = sub nsw i32 0, [[SUB_1_4]]
+; CHECK-NEXT:    [[TMP76:%.*]] = select i1 [[TMP75]], i32 [[NEG_1_4]], i32 [[SUB_1_4]]
+; CHECK-NEXT:    [[CMP12_1_4:%.*]] = icmp slt i32 [[TMP76]], [[SPEC_SELECT8_4]]
+; CHECK-NEXT:    [[TMP77:%.*]] = or i1 [[CMP12_1_4]], [[CMP12_4]]
+; CHECK-NEXT:    [[SPEC_SELECT8_1_4:%.*]] = select i1 [[CMP12_1_4]], i32 [[TMP76]], i32 [[SPEC_SELECT8_4]]
+; CHECK-NEXT:    [[SUB_2_4:%.*]] = sub i32 [[TMP30]], [[TMP15]]
+; CHECK-NEXT:    [[TMP78:%.*]] = icmp slt i32 [[SUB_2_4]], 0
+; CHECK-NEXT:    [[NEG_2_4:%.*]] = sub nsw i32 0, [[SUB_2_4]]
+; CHECK-NEXT:    [[TMP79:%.*]] = select i1 [[TMP78]], i32 [[NEG_2_4]], i32 [[SUB_2_4]]
+; CHECK-NEXT:    [[CMP12_2_4:%.*]] = icmp slt i32 [[TMP79]], [[SPEC_SELECT8_1_4]]
+; CHECK-NEXT:    [[TMP80:%.*]] = or i1 [[CMP12_2_4]], [[TMP77]]
+; CHECK-NEXT:    [[SPEC_SELECT8_2_4:%.*]] = select i1 [[CMP12_2_4]], i32 [[TMP79]], i32 [[SPEC_SELECT8_1_4]]
+; CHECK-NEXT:    [[SUB_3_4:%.*]] = sub i32 [[TMP30]], [[TMP16]]
+; CHECK-NEXT:    [[TMP81:%.*]] = icmp slt i32 [[SUB_3_4]], 0
+; CHECK-NEXT:    [[NEG_3_4:%.*]] = sub nsw i32 0, [[SUB_3_4]]
+; CHECK-NEXT:    [[TMP82:%.*]] = select i1 [[TMP81]], i32 [[NEG_3_4]], i32 [[SUB_3_4]]
+; CHECK-NEXT:    [[CMP12_3_4:%.*]] = icmp slt i32 [[TMP82]], [[SPEC_SELECT8_2_4]]
+; CHECK-NEXT:    [[TMP83:%.*]] = or i1 [[CMP12_3_4]], [[TMP80]]
+; CHECK-NEXT:    [[SPEC_SELECT_3_4:%.*]] = select i1 [[TMP83]], i32 4, i32 [[SPEC_SELECT_3_3]]
+; CHECK-NEXT:    [[SPEC_SELECT8_3_4:%.*]] = select i1 [[CMP12_3_4]], i32 [[TMP82]], i32 [[SPEC_SELECT8_2_4]]
+; CHECK-NEXT:    [[SUB_5:%.*]] = sub i32 [[TMP30]], [[TMP17]]
+; CHECK-NEXT:    [[TMP84:%.*]] = icmp slt i32 [[SUB_5]], 0
+; CHECK-NEXT:    [[NEG_5:%.*]] = sub nsw i32 0, [[SUB_5]]
+; CHECK-NEXT:    [[TMP85:%.*]] = select i1 [[TMP84]], i32 [[NEG_5]], i32 [[SUB_5]]
+; CHECK-NEXT:    [[CMP12_5:%.*]] = icmp slt i32 [[TMP85]], [[SPEC_SELECT8_3_4]]
+; CHECK-NEXT:    [[SPEC_SELECT8_5:%.*]] = select i1 [[CMP12_5]], i32 [[TMP85]], i32 [[SPEC_SELECT8_3_4]]
+; CHECK-NEXT:    [[SUB_1_5:%.*]] = sub i32 [[TMP30]], [[TMP18]]
+; CHECK-NEXT:    [[TMP86:%.*]] = icmp slt i32 [[SUB_1_5]], 0
+; CHECK-NEXT:    [[NEG_1_5:%.*]] = sub nsw i32 0, [[SUB_1_5]]
+; CHECK-NEXT:    [[TMP87:%.*]] = select i1 [[TMP86]], i32 [[NEG_1_5]], i32 [[SUB_1_5]]
+; CHECK-NEXT:    [[CMP12_1_5:%.*]] = icmp slt i32 [[TMP87]], [[SPEC_SELECT8_5]]
+; CHECK-NEXT:    [[TMP88:%.*]] = or i1 [[CMP12_1_5]], [[CMP12_5]]
+; CHECK-NEXT:    [[SPEC_SELECT8_1_5:%.*]] = select i1 [[CMP12_1_5]], i32 [[TMP87]], i32 [[SPEC_SELECT8_5]]
+; CHECK-NEXT:    [[SUB_2_5:%.*]] = sub i32 [[TMP30]], [[TMP19]]
+; CHECK-NEXT:    [[TMP89:%.*]] = icmp slt i32 [[SUB_2_5]], 0
+; CHECK-NEXT:    [[NEG_2_5:%.*]] = sub nsw i32 0, [[SUB_2_5]]
+; CHECK-NEXT:    [[TMP90:%.*]] = select i1 [[TMP89]], i32 [[NEG_2_5]], i32 [[SUB_2_5]]
+; CHECK-NEXT:    [[CMP12_2_5:%.*]] = icmp slt i32 [[TMP90]], [[SPEC_SELECT8_1_5]]
+; CHECK-NEXT:    [[TMP91:%.*]] = or i1 [[CMP12_2_5]], [[TMP88]]
+; CHECK-NEXT:    [[SPEC_SELECT8_2_5:%.*]] = select i1 [[CMP12_2_5]], i32 [[TMP90]], i32 [[SPEC_SELECT8_1_5]]
+; CHECK-NEXT:    [[SUB_3_5:%.*]] = sub i32 [[TMP30]], [[TMP20]]
+; CHECK-NEXT:    [[TMP92:%.*]] = icmp slt i32 [[SUB_3_5]], 0
+; CHECK-NEXT:    [[NEG_3_5:%.*]] = sub nsw i32 0, [[SUB_3_5]]
+; CHECK-NEXT:    [[TMP93:%.*]] = select i1 [[TMP92]], i32 [[NEG_3_5]], i32 [[SUB_3_5]]
+; CHECK-NEXT:    [[CMP12_3_5:%.*]] = icmp slt i32 [[TMP93]], [[SPEC_SELECT8_2_5]]
+; CHECK-NEXT:    [[TMP94:%.*]] = or i1 [[CMP12_3_5]], [[TMP91]]
+; CHECK-NEXT:    [[SPEC_SELECT_3_5:%.*]] = select i1 [[TMP94]], i32 5, i32 [[SPEC_SELECT_3_4]]
+; CHECK-NEXT:    [[SPEC_SELECT8_3_5:%.*]] = select i1 [[CMP12_3_5]], i32 [[TMP93]], i32 [[SPEC_SELECT8_2_5]]
+; CHECK-NEXT:    [[SUB_6:%.*]] = sub i32 [[TMP30]], [[TMP21]]
+; CHECK-NEXT:    [[TMP95:%.*]] = icmp slt i32 [[SUB_6]], 0
+; CHECK-NEXT:    [[NEG_6:%.*]] = sub nsw i32 0, [[SUB_6]]
+; CHECK-NEXT:    [[TMP96:%.*]] = select i1 [[TMP95]], i32 [[NEG_6]], i32 [[SUB_6]]
+; CHECK-NEXT:    [[CMP12_6:%.*]] = icmp slt i32 [[TMP96]], [[SPEC_SELECT8_3_5]]
+; CHECK-NEXT:    [[SPEC_SELECT8_6:%.*]] = select i1 [[CMP12_6]], i32 [[TMP96]], i32 [[SPEC_SELECT8_3_5]]
+; CHECK-NEXT:    [[SUB_1_6:%.*]] = sub i32 [[TMP30]], [[TMP22]]
+; CHECK-NEXT:    [[TMP97:%.*]] = icmp slt i32 [[SUB_1_6]], 0
+; CHECK-NEXT:    [[NEG_1_6:%.*]] = sub nsw i32 0, [[SUB_1_6]]
+; CHECK-NEXT:    [[TMP98:%.*]] = select i1 [[TMP97]], i32 [[NEG_1_6]], i32 [[SUB_1_6]]
+; CHECK-NEXT:    [[CMP12_1_6:%.*]] = icmp slt i32 [[TMP98]], [[SPEC_SELECT8_6]]
+; CHECK-NEXT:    [[TMP99:%.*]] = or i1 [[CMP12_1_6]], [[CMP12_6]]
+; CHECK-NEXT:    [[SPEC_SELECT8_1_6:%.*]] = select i1 [[CMP12_1_6]], i32 [[TMP98]], i32 [[SPEC_SELECT8_6]]
+; CHECK-NEXT:    [[SUB_2_6:%.*]] = sub i32 [[TMP30]], [[TMP23]]
+; CHECK-NEXT:    [[TMP100:%.*]] = icmp slt i32 [[SUB_2_6]], 0
+; CHECK-NEXT:    [[NEG_2_6:%.*]] = sub nsw i32 0, [[SUB_2_6]]
+; CHECK-NEXT:    [[TMP101:%.*]] = select i1 [[TMP100]], i32 [[NEG_2_6]], i32 [[SUB_2_6]]
+; CHECK-NEXT:    [[CMP12_2_6:%.*]] = icmp slt i32 [[TMP101]], [[SPEC_SELECT8_1_6]]
+; CHECK-NEXT:    [[TMP102:%.*]] = or i1 [[CMP12_2_6]], [[TMP99]]
+; CHECK-NEXT:    [[SPEC_SELECT8_2_6:%.*]] = select i1 [[CMP12_2_6]], i32 [[TMP101]], i32 [[SPEC_SELECT8_1_6]]
+; CHECK-NEXT:    [[SUB_3_6:%.*]] = sub i32 [[TMP30]], [[TMP24]]
+; CHECK-NEXT:    [[TMP103:%.*]] = icmp slt i32 [[SUB_3_6]], 0
+; CHECK-NEXT:    [[NEG_3_6:%.*]] = sub nsw i32 0, [[SUB_3_6]]
+; CHECK-NEXT:    [[TMP104:%.*]] = select i1 [[TMP103]], i32 [[NEG_3_6]], i32 [[SUB_3_6]]
+; CHECK-NEXT:    [[CMP12_3_6:%.*]] = icmp slt i32 [[TMP104]], [[SPEC_SELECT8_2_6]]
+; CHECK-NEXT:    [[TMP105:%.*]] = or i1 [[CMP12_3_6]], [[TMP102]]
+; CHECK-NEXT:    [[SPEC_SELECT_3_6:%.*]] = select i1 [[TMP105]], i32 6, i32 [[SPEC_SELECT_3_5]]
+; CHECK-NEXT:    [[SPEC_SELECT8_3_6:%.*]] = select i1 [[CMP12_3_6]], i32 [[TMP104]], i32 [[SPEC_SELECT8_2_6]]
+; CHECK-NEXT:    [[SUB_7:%.*]] = sub i32 [[TMP30]], [[TMP25]]
+; CHECK-NEXT:    [[TMP106:%.*]] = icmp slt i32 [[SUB_7]], 0
+; CHECK-NEXT:    [[NEG_7:%.*]] = sub nsw i32 0, [[SUB_7]]
+; CHECK-NEXT:    [[TMP107:%.*]] = select i1 [[TMP106]], i32 [[NEG_7]], i32 [[SUB_7]]
+; CHECK-NEXT:    [[CMP12_7:%.*]] = icmp slt i32 [[TMP107]], [[SPEC_SELECT8_3_6]]
+; CHECK-NEXT:    [[SPEC_SELECT8_7:%.*]] = select i1 [[CMP12_7]], i32 [[TMP107]], i32 [[SPEC_SELECT8_3_6]]
+; CHECK-NEXT:    [[SUB_1_7:%.*]] = sub i32 [[TMP30]], [[TMP26]]
+; CHECK-NEXT:    [[TMP108:%.*]] = icmp slt i32 [[SUB_1_7]], 0
+; CHECK-NEXT:    [[NEG_1_7:%.*]] = sub nsw i32 0, [[SUB_1_7]]
+; CHECK-NEXT:    [[TMP109:%.*]] = select i1 [[TMP108]], i32 [[NEG_1_7]], i32 [[SUB_1_7]]
+; CHECK-NEXT:    [[CMP12_1_7:%.*]] = icmp slt i32 [[TMP109]], [[SPEC_SELECT8_7]]
+; CHECK-NEXT:    [[TMP110:%.*]] = or i1 [[CMP12_1_7]], [[CMP12_7]]
+; CHECK-NEXT:    [[SPEC_SELECT8_1_7:%.*]] = select i1 [[CMP12_1_7]], i32 [[TMP109]], i32 [[SPEC_SELECT8_7]]
+; CHECK-NEXT:    [[SUB_2_7:%.*]] = sub i32 [[TMP30]], [[TMP27]]
+; CHECK-NEXT:    [[TMP111:%.*]] = icmp slt i32 [[SUB_2_7]], 0
+; CHECK-NEXT:    [[NEG_2_7:%.*]] = sub nsw i32 0, [[SUB_2_7]]
+; CHECK-NEXT:    [[TMP112:%.*]] = select i1 [[TMP111]], i32 [[NEG_2_7]], i32 [[SUB_2_7]]
+; CHECK-NEXT:    [[CMP12_2_7:%.*]] = icmp slt i32 [[TMP112]], [[SPEC_SELECT8_1_7]]
+; CHECK-NEXT:    [[TMP113:%.*]] = or i1 [[CMP12_2_7]], [[TMP110]]
+; CHECK-NEXT:    [[SPEC_SELECT8_2_7:%.*]] = select i1 [[CMP12_2_7]], i32 [[TMP112]], i32 [[SPEC_SELECT8_1_7]]
+; CHECK-NEXT:    [[SUB_3_7:%.*]] = sub i32 [[TMP30]], [[TMP28]]
+; CHECK-NEXT:    [[TMP114:%.*]] = icmp slt i32 [[SUB_3_7]], 0
+; CHECK-NEXT:    [[NEG_3_7:%.*]] = sub nsw i32 0, [[SUB_3_7]]
+; CHECK-NEXT:    [[TMP115:%.*]] = select i1 [[TMP114]], i32 [[NEG_3_7]], i32 [[SUB_3_7]]
+; CHECK-NEXT:    [[CMP12_3_7:%.*]] = icmp slt i32 [[TMP115]], [[SPEC_SELECT8_2_7]]
+; CHECK-NEXT:    [[TMP116:%.*]] = or i1 [[CMP12_3_7]], [[TMP113]]
+; CHECK-NEXT:    [[SPEC_SELECT_3_7:%.*]] = select i1 [[TMP116]], i32 7, i32 [[SPEC_SELECT_3_6]]
+; CHECK-NEXT:    [[SPEC_SELECT8_3_7]] = select i1 [[CMP12_3_7]], i32 [[TMP115]], i32 [[SPEC_SELECT8_2_7]]
+; CHECK-NEXT:    [[K:%.*]] = getelementptr inbounds [366 x i32], [366 x i32]* @l, i64 0, i64 [[INDVARS_IV]]
+; CHECK-NEXT:    store i32 [[SPEC_SELECT_3_7]], i32* [[K]], align 4
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    br label [[FOR_COND]]
+;
+entry:
+  %0 = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 0, i64 0), align 16
+  %1 = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 0, i64 1), align 4
+  %2 = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 0, i64 2), align 8
+  %3 = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 0, i64 3), align 4
+  %4 = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 1, i64 0), align 16
+  %5 = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 1, i64 1), align 4
+  %6 = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 1, i64 2), align 8
+  %7 = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 1, i64 3), align 4
+  %8 = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 2, i64 0), align 16
+  %9 = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 2, i64 1), align 4
+  %10 = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 2, i64 2), align 8
+  %11 = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 2, i64 3), align 4
+  %12 = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 3, i64 0), align 16
+  %13 = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 3, i64 1), align 4
+  %14 = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 3, i64 2), align 8
+  %15 = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 3, i64 3), align 4
+  %16 = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 4, i64 0), align 16
+  %17 = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 4, i64 1), align 4
+  %18 = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 4, i64 2), align 8
+  %19 = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 4, i64 3), align 4
+  %20 = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 5, i64 0), align 16
+  %21 = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 5, i64 1), align 4
+  %22 = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 5, i64 2), align 8
+  %23 = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 5, i64 3), align 4
+  %24 = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 6, i64 0), align 16
+  %25 = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 6, i64 1), align 4
+  %26 = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 6, i64 2), align 8
+  %27 = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 6, i64 3), align 4
+  %28 = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 7, i64 0), align 16
+  %29 = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 7, i64 1), align 4
+  %30 = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 7, i64 2), align 8
+  %31 = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 7, i64 3), align 4
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.cond, %entry
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.cond ], [ 0, %entry ]
+  %b.0 = phi i32 [ %spec.select8.3.7, %for.cond ], [ undef, %entry ]
+  %32 = trunc i64 %indvars.iv to i32
+  %33 = add i32 %32, -183
+  %sub = sub i32 %33, %0
+  %34 = icmp slt i32 %sub, 0
+  %neg = sub nsw i32 0, %sub
+  %35 = select i1 %34, i32 %neg, i32 %sub
+  %cmp12 = icmp slt i32 %35, %b.0
+  %spec.select8 = select i1 %cmp12, i32 %35, i32 %b.0
+  %sub.1 = sub i32 %33, %1
+  %36 = icmp slt i32 %sub.1, 0
+  %neg.1 = sub nsw i32 0, %sub.1
+  %37 = select i1 %36, i32 %neg.1, i32 %sub.1
+  %cmp12.1 = icmp slt i32 %37, %spec.select8
+  %spec.select8.1 = select i1 %cmp12.1, i32 %37, i32 %spec.select8
+  %sub.2 = sub i32 %33, %2
+  %38 = icmp slt i32 %sub.2, 0
+  %neg.2 = sub nsw i32 0, %sub.2
+  %39 = select i1 %38, i32 %neg.2, i32 %sub.2
+  %cmp12.2 = icmp slt i32 %39, %spec.select8.1
+  %spec.select8.2 = select i1 %cmp12.2, i32 %39, i32 %spec.select8.1
+  %sub.3 = sub i32 %33, %3
+  %40 = icmp slt i32 %sub.3, 0
+  %neg.3 = sub nsw i32 0, %sub.3
+  %41 = select i1 %40, i32 %neg.3, i32 %sub.3
+  %cmp12.3 = icmp slt i32 %41, %spec.select8.2
+  %spec.select8.3 = select i1 %cmp12.3, i32 %41, i32 %spec.select8.2
+  %sub.116 = sub i32 %33, %4
+  %42 = icmp slt i32 %sub.116, 0
+  %neg.117 = sub nsw i32 0, %sub.116
+  %43 = select i1 %42, i32 %neg.117, i32 %sub.116
+  %cmp12.118 = icmp slt i32 %43, %spec.select8.3
+  %spec.select8.120 = select i1 %cmp12.118, i32 %43, i32 %spec.select8.3
+  %sub.1.1 = sub i32 %33, %5
+  %44 = icmp slt i32 %sub.1.1, 0
+  %neg.1.1 = sub nsw i32 0, %sub.1.1
+  %45 = select i1 %44, i32 %neg.1.1, i32 %sub.1.1
+  %cmp12.1.1 = icmp slt i32 %45, %spec.select8.120
+  %narrow = or i1 %cmp12.1.1, %cmp12.118
+  %spec.select8.1.1 = select i1 %cmp12.1.1, i32 %45, i32 %spec.select8.120
+  %sub.2.1 = sub i32 %33, %6
+  %46 = icmp slt i32 %sub.2.1, 0
+  %neg.2.1 = sub nsw i32 0, %sub.2.1
+  %47 = select i1 %46, i32 %neg.2.1, i32 %sub.2.1
+  %cmp12.2.1 = icmp slt i32 %47, %spec.select8.1.1
+  %narrow34 = or i1 %cmp12.2.1, %narrow
+  %spec.select8.2.1 = select i1 %cmp12.2.1, i32 %47, i32 %spec.select8.1.1
+  %sub.3.1 = sub i32 %33, %7
+  %48 = icmp slt i32 %sub.3.1, 0
+  %neg.3.1 = sub nsw i32 0, %sub.3.1
+  %49 = select i1 %48, i32 %neg.3.1, i32 %sub.3.1
+  %cmp12.3.1 = icmp slt i32 %49, %spec.select8.2.1
+  %narrow35 = or i1 %cmp12.3.1, %narrow34
+  %spec.select.3.1 = zext i1 %narrow35 to i32
+  %spec.select8.3.1 = select i1 %cmp12.3.1, i32 %49, i32 %spec.select8.2.1
+  %sub.222 = sub i32 %33, %8
+  %50 = icmp slt i32 %sub.222, 0
+  %neg.223 = sub nsw i32 0, %sub.222
+  %51 = select i1 %50, i32 %neg.223, i32 %sub.222
+  %cmp12.224 = icmp slt i32 %51, %spec.select8.3.1
+  %spec.select8.226 = select i1 %cmp12.224, i32 %51, i32 %spec.select8.3.1
+  %sub.1.2 = sub i32 %33, %9
+  %52 = icmp slt i32 %sub.1.2, 0
+  %neg.1.2 = sub nsw i32 0, %sub.1.2
+  %53 = select i1 %52, i32 %neg.1.2, i32 %sub.1.2
+  %cmp12.1.2 = icmp slt i32 %53, %spec.select8.226
+  %54 = or i1 %cmp12.1.2, %cmp12.224
+  %spec.select8.1.2 = select i1 %cmp12.1.2, i32 %53, i32 %spec.select8.226
+  %sub.2.2 = sub i32 %33, %10
+  %55 = icmp slt i32 %sub.2.2, 0
+  %neg.2.2 = sub nsw i32 0, %sub.2.2
+  %56 = select i1 %55, i32 %neg.2.2, i32 %sub.2.2
+  %cmp12.2.2 = icmp slt i32 %56, %spec.select8.1.2
+  %57 = or i1 %cmp12.2.2, %54
+  %spec.select8.2.2 = select i1 %cmp12.2.2, i32 %56, i32 %spec.select8.1.2
+  %sub.3.2 = sub i32 %33, %11
+  %58 = icmp slt i32 %sub.3.2, 0
+  %neg.3.2 = sub nsw i32 0, %sub.3.2
+  %59 = select i1 %58, i32 %neg.3.2, i32 %sub.3.2
+  %cmp12.3.2 = icmp slt i32 %59, %spec.select8.2.2
+  %60 = or i1 %cmp12.3.2, %57
+  %spec.select.3.2 = select i1 %60, i32 2, i32 %spec.select.3.1
+  %spec.select8.3.2 = select i1 %cmp12.3.2, i32 %59, i32 %spec.select8.2.2
+  %sub.328 = sub i32 %33, %12
+  %61 = icmp slt i32 %sub.328, 0
+  %neg.329 = sub nsw i32 0, %sub.328
+  %62 = select i1 %61, i32 %neg.329, i32 %sub.328
+  %cmp12.330 = icmp slt i32 %62, %spec.select8.3.2
+  %spec.select8.332 = select i1 %cmp12.330, i32 %62, i32 %spec.select8.3.2
+  %sub.1.3 = sub i32 %33, %13
+  %63 = icmp slt i32 %sub.1.3, 0
+  %neg.1.3 = sub nsw i32 0, %sub.1.3
+  %64 = select i1 %63, i32 %neg.1.3, i32 %sub.1.3
+  %cmp12.1.3 = icmp slt i32 %64, %spec.select8.332
+  %65 = or i1 %cmp12.1.3, %cmp12.330
+  %spec.select8.1.3 = select i1 %cmp12.1.3, i32 %64, i32 %spec.select8.332
+  %sub.2.3 = sub i32 %33, %14
+  %66 = icmp slt i32 %sub.2.3, 0
+  %neg.2.3 = sub nsw i32 0, %sub.2.3
+  %67 = select i1 %66, i32 %neg.2.3, i32 %sub.2.3
+  %cmp12.2.3 = icmp slt i32 %67, %spec.select8.1.3
+  %68 = or i1 %cmp12.2.3, %65
+  %spec.select8.2.3 = select i1 %cmp12.2.3, i32 %67, i32 %spec.select8.1.3
+  %sub.3.3 = sub i32 %33, %15
+  %69 = icmp slt i32 %sub.3.3, 0
+  %neg.3.3 = sub nsw i32 0, %sub.3.3
+  %70 = select i1 %69, i32 %neg.3.3, i32 %sub.3.3
+  %cmp12.3.3 = icmp slt i32 %70, %spec.select8.2.3
+  %71 = or i1 %cmp12.3.3, %68
+  %spec.select.3.3 = select i1 %71, i32 3, i32 %spec.select.3.2
+  %spec.select8.3.3 = select i1 %cmp12.3.3, i32 %70, i32 %spec.select8.2.3
+  %sub.4 = sub i32 %33, %16
+  %72 = icmp slt i32 %sub.4, 0
+  %neg.4 = sub nsw i32 0, %sub.4
+  %73 = select i1 %72, i32 %neg.4, i32 %sub.4
+  %cmp12.4 = icmp slt i32 %73, %spec.select8.3.3
+  %spec.select8.4 = select i1 %cmp12.4, i32 %73, i32 %spec.select8.3.3
+  %sub.1.4 = sub i32 %33, %17
+  %74 = icmp slt i32 %sub.1.4, 0
+  %neg.1.4 = sub nsw i32 0, %sub.1.4
+  %75 = select i1 %74, i32 %neg.1.4, i32 %sub.1.4
+  %cmp12.1.4 = icmp slt i32 %75, %spec.select8.4
+  %76 = or i1 %cmp12.1.4, %cmp12.4
+  %spec.select8.1.4 = select i1 %cmp12.1.4, i32 %75, i32 %spec.select8.4
+  %sub.2.4 = sub i32 %33, %18
+  %77 = icmp slt i32 %sub.2.4, 0
+  %neg.2.4 = sub nsw i32 0, %sub.2.4
+  %78 = select i1 %77, i32 %neg.2.4, i32 %sub.2.4
+  %cmp12.2.4 = icmp slt i32 %78, %spec.select8.1.4
+  %79 = or i1 %cmp12.2.4, %76
+  %spec.select8.2.4 = select i1 %cmp12.2.4, i32 %78, i32 %spec.select8.1.4
+  %sub.3.4 = sub i32 %33, %19
+  %80 = icmp slt i32 %sub.3.4, 0
+  %neg.3.4 = sub nsw i32 0, %sub.3.4
+  %81 = select i1 %80, i32 %neg.3.4, i32 %sub.3.4
+  %cmp12.3.4 = icmp slt i32 %81, %spec.select8.2.4
+  %82 = or i1 %cmp12.3.4, %79
+  %spec.select.3.4 = select i1 %82, i32 4, i32 %spec.select.3.3
+  %spec.select8.3.4 = select i1 %cmp12.3.4, i32 %81, i32 %spec.select8.2.4
+  %sub.5 = sub i32 %33, %20
+  %83 = icmp slt i32 %sub.5, 0
+  %neg.5 = sub nsw i32 0, %sub.5
+  %84 = select i1 %83, i32 %neg.5, i32 %sub.5
+  %cmp12.5 = icmp slt i32 %84, %spec.select8.3.4
+  %spec.select8.5 = select i1 %cmp12.5, i32 %84, i32 %spec.select8.3.4
+  %sub.1.5 = sub i32 %33, %21
+  %85 = icmp slt i32 %sub.1.5, 0
+  %neg.1.5 = sub nsw i32 0, %sub.1.5
+  %86 = select i1 %85, i32 %neg.1.5, i32 %sub.1.5
+  %cmp12.1.5 = icmp slt i32 %86, %spec.select8.5
+  %87 = or i1 %cmp12.1.5, %cmp12.5
+  %spec.select8.1.5 = select i1 %cmp12.1.5, i32 %86, i32 %spec.select8.5
+  %sub.2.5 = sub i32 %33, %22
+  %88 = icmp slt i32 %sub.2.5, 0
+  %neg.2.5 = sub nsw i32 0, %sub.2.5
+  %89 = select i1 %88, i32 %neg.2.5, i32 %sub.2.5
+  %cmp12.2.5 = icmp slt i32 %89, %spec.select8.1.5
+  %90 = or i1 %cmp12.2.5, %87
+  %spec.select8.2.5 = select i1 %cmp12.2.5, i32 %89, i32 %spec.select8.1.5
+  %sub.3.5 = sub i32 %33, %23
+  %91 = icmp slt i32 %sub.3.5, 0
+  %neg.3.5 = sub nsw i32 0, %sub.3.5
+  %92 = select i1 %91, i32 %neg.3.5, i32 %sub.3.5
+  %cmp12.3.5 = icmp slt i32 %92, %spec.select8.2.5
+  %93 = or i1 %cmp12.3.5, %90
+  %spec.select.3.5 = select i1 %93, i32 5, i32 %spec.select.3.4
+  %spec.select8.3.5 = select i1 %cmp12.3.5, i32 %92, i32 %spec.select8.2.5
+  %sub.6 = sub i32 %33, %24
+  %94 = icmp slt i32 %sub.6, 0
+  %neg.6 = sub nsw i32 0, %sub.6
+  %95 = select i1 %94, i32 %neg.6, i32 %sub.6
+  %cmp12.6 = icmp slt i32 %95, %spec.select8.3.5
+  %spec.select8.6 = select i1 %cmp12.6, i32 %95, i32 %spec.select8.3.5
+  %sub.1.6 = sub i32 %33, %25
+  %96 = icmp slt i32 %sub.1.6, 0
+  %neg.1.6 = sub nsw i32 0, %sub.1.6
+  %97 = select i1 %96, i32 %neg.1.6, i32 %sub.1.6
+  %cmp12.1.6 = icmp slt i32 %97, %spec.select8.6
+  %98 = or i1 %cmp12.1.6, %cmp12.6
+  %spec.select8.1.6 = select i1 %cmp12.1.6, i32 %97, i32 %spec.select8.6
+  %sub.2.6 = sub i32 %33, %26
+  %99 = icmp slt i32 %sub.2.6, 0
+  %neg.2.6 = sub nsw i32 0, %sub.2.6
+  %100 = select i1 %99, i32 %neg.2.6, i32 %sub.2.6
+  %cmp12.2.6 = icmp slt i32 %100, %spec.select8.1.6
+  %101 = or i1 %cmp12.2.6, %98
+  %spec.select8.2.6 = select i1 %cmp12.2.6, i32 %100, i32 %spec.select8.1.6
+  %sub.3.6 = sub i32 %33, %27
+  %102 = icmp slt i32 %sub.3.6, 0
+  %neg.3.6 = sub nsw i32 0, %sub.3.6
+  %103 = select i1 %102, i32 %neg.3.6, i32 %sub.3.6
+  %cmp12.3.6 = icmp slt i32 %103, %spec.select8.2.6
+  %104 = or i1 %cmp12.3.6, %101
+  %spec.select.3.6 = select i1 %104, i32 6, i32 %spec.select.3.5
+  %spec.select8.3.6 = select i1 %cmp12.3.6, i32 %103, i32 %spec.select8.2.6
+  %sub.7 = sub i32 %33, %28
+  %105 = icmp slt i32 %sub.7, 0
+  %neg.7 = sub nsw i32 0, %sub.7
+  %106 = select i1 %105, i32 %neg.7, i32 %sub.7
+  %cmp12.7 = icmp slt i32 %106, %spec.select8.3.6
+  %spec.select8.7 = select i1 %cmp12.7, i32 %106, i32 %spec.select8.3.6
+  %sub.1.7 = sub i32 %33, %29
+  %107 = icmp slt i32 %sub.1.7, 0
+  %neg.1.7 = sub nsw i32 0, %sub.1.7
+  %108 = select i1 %107, i32 %neg.1.7, i32 %sub.1.7
+  %cmp12.1.7 = icmp slt i32 %108, %spec.select8.7
+  %109 = or i1 %cmp12.1.7, %cmp12.7
+  %spec.select8.1.7 = select i1 %cmp12.1.7, i32 %108, i32 %spec.select8.7
+  %sub.2.7 = sub i32 %33, %30
+  %110 = icmp slt i32 %sub.2.7, 0
+  %neg.2.7 = sub nsw i32 0, %sub.2.7
+  %111 = select i1 %110, i32 %neg.2.7, i32 %sub.2.7
+  %cmp12.2.7 = icmp slt i32 %111, %spec.select8.1.7
+  %112 = or i1 %cmp12.2.7, %109
+  %spec.select8.2.7 = select i1 %cmp12.2.7, i32 %111, i32 %spec.select8.1.7
+  %sub.3.7 = sub i32 %33, %31
+  %113 = icmp slt i32 %sub.3.7, 0
+  %neg.3.7 = sub nsw i32 0, %sub.3.7
+  %114 = select i1 %113, i32 %neg.3.7, i32 %sub.3.7
+  %cmp12.3.7 = icmp slt i32 %114, %spec.select8.2.7
+  %115 = or i1 %cmp12.3.7, %112
+  %spec.select.3.7 = select i1 %115, i32 7, i32 %spec.select.3.6
+  %spec.select8.3.7 = select i1 %cmp12.3.7, i32 %114, i32 %spec.select8.2.7
+  %k = getelementptr inbounds [366 x i32], [366 x i32]* @l, i64 0, i64 %indvars.iv
+  store i32 %spec.select.3.7, i32* %k, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  br label %for.cond
+}
+
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/vector_gep.ll b/llvm/test/Transforms/SLPVectorizer/X86/vector_gep.ll
index 436f091d7..1c443de50 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/vector_gep.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/vector_gep.ll
@@ -26,5 +26,5 @@ entry:
   unreachable
 }
 
-attributes #0 = { noreturn readonly uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="knl" "target-features"="+adx,+aes,+avx,+avx2,+avx512cd,+avx512er,+avx512f,+avx512pf,+bmi,+bmi2,+cx16,+f16c,+fma,+fsgsbase,+fxsr,+lzcnt,+mmx,+movbe,+pclmul,+popcnt,+prefetchwt1,+rdrnd,+rdseed,+rtm,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87,+xsave,+xsaveopt" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noreturn readonly uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="knl" "target-features"="+adx,+aes,+avx,+avx2,+avx512cd,+avx512er,+avx512f,+avx512pf,+bmi,+bmi2,+cx16,+f16c,+fma,+fsgsbase,+fxsr,+lzcnt,+mmx,+movbe,+pclmul,+popcnt,+prefetchwt1,+rdrnd,+rdseed,+rtm,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87,+xsave,+xsaveopt" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/vectorize-reorder-reuse.ll b/llvm/test/Transforms/SLPVectorizer/X86/vectorize-reorder-reuse.ll
index 889bba80b..779d52605 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/vectorize-reorder-reuse.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/vectorize-reorder-reuse.ll
@@ -18,19 +18,6 @@ define i32 @foo(i32* nocapture readonly %arr, i32 %a1, i32 %a2, i32 %a3, i32 %a4
 ; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <8 x i32> [[TMP7]], i32 [[A7:%.*]], i32 6
 ; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <8 x i32> [[TMP8]], i32 [[A8:%.*]], i32 7
 ; CHECK-NEXT:    [[TMP10:%.*]] = add <8 x i32> [[SHUFFLE]], [[TMP9]]
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 undef, undef
-; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i32 undef, i32 undef
-; CHECK-NEXT:    [[CMP15:%.*]] = icmp ult i32 [[COND]], undef
-; CHECK-NEXT:    [[COND19:%.*]] = select i1 [[CMP15]], i32 [[COND]], i32 undef
-; CHECK-NEXT:    [[CMP20:%.*]] = icmp ult i32 [[COND19]], undef
-; CHECK-NEXT:    [[COND24:%.*]] = select i1 [[CMP20]], i32 [[COND19]], i32 undef
-; CHECK-NEXT:    [[CMP25:%.*]] = icmp ult i32 [[COND24]], undef
-; CHECK-NEXT:    [[COND29:%.*]] = select i1 [[CMP25]], i32 [[COND24]], i32 undef
-; CHECK-NEXT:    [[CMP30:%.*]] = icmp ult i32 [[COND29]], undef
-; CHECK-NEXT:    [[COND34:%.*]] = select i1 [[CMP30]], i32 [[COND29]], i32 undef
-; CHECK-NEXT:    [[CMP35:%.*]] = icmp ult i32 [[COND34]], undef
-; CHECK-NEXT:    [[COND39:%.*]] = select i1 [[CMP35]], i32 [[COND34]], i32 undef
-; CHECK-NEXT:    [[CMP40:%.*]] = icmp ult i32 [[COND39]], undef
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x i32> [[TMP10]], <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = icmp ult <8 x i32> [[TMP10]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <8 x i1> [[RDX_MINMAX_CMP]], <8 x i32> [[TMP10]], <8 x i32> [[RDX_SHUF]]
@@ -41,7 +28,6 @@ define i32 @foo(i32* nocapture readonly %arr, i32 %a1, i32 %a2, i32 %a3, i32 %a4
 ; CHECK-NEXT:    [[RDX_MINMAX_CMP5:%.*]] = icmp ult <8 x i32> [[RDX_MINMAX_SELECT3]], [[RDX_SHUF4]]
 ; CHECK-NEXT:    [[RDX_MINMAX_SELECT6:%.*]] = select <8 x i1> [[RDX_MINMAX_CMP5]], <8 x i32> [[RDX_MINMAX_SELECT3]], <8 x i32> [[RDX_SHUF4]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <8 x i32> [[RDX_MINMAX_SELECT6]], i32 0
-; CHECK-NEXT:    [[COND44:%.*]] = select i1 [[CMP40]], i32 [[COND39]], i32 undef
 ; CHECK-NEXT:    ret i32 [[TMP11]]
 ;
 entry:
@@ -92,19 +78,6 @@ define i32 @foo1(i32* nocapture readonly %arr, i32 %a1, i32 %a2, i32 %a3, i32 %a
 ; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <8 x i32> [[TMP7]], i32 [[A7:%.*]], i32 6
 ; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <8 x i32> [[TMP8]], i32 [[A8:%.*]], i32 7
 ; CHECK-NEXT:    [[TMP10:%.*]] = add <8 x i32> [[SHUFFLE]], [[TMP9]]
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 undef, undef
-; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i32 undef, i32 undef
-; CHECK-NEXT:    [[CMP15:%.*]] = icmp ult i32 [[COND]], undef
-; CHECK-NEXT:    [[COND19:%.*]] = select i1 [[CMP15]], i32 [[COND]], i32 undef
-; CHECK-NEXT:    [[CMP20:%.*]] = icmp ult i32 [[COND19]], undef
-; CHECK-NEXT:    [[COND24:%.*]] = select i1 [[CMP20]], i32 [[COND19]], i32 undef
-; CHECK-NEXT:    [[CMP25:%.*]] = icmp ult i32 [[COND24]], undef
-; CHECK-NEXT:    [[COND29:%.*]] = select i1 [[CMP25]], i32 [[COND24]], i32 undef
-; CHECK-NEXT:    [[CMP30:%.*]] = icmp ult i32 [[COND29]], undef
-; CHECK-NEXT:    [[COND34:%.*]] = select i1 [[CMP30]], i32 [[COND29]], i32 undef
-; CHECK-NEXT:    [[CMP35:%.*]] = icmp ult i32 [[COND34]], undef
-; CHECK-NEXT:    [[COND39:%.*]] = select i1 [[CMP35]], i32 [[COND34]], i32 undef
-; CHECK-NEXT:    [[CMP40:%.*]] = icmp ult i32 [[COND39]], undef
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x i32> [[TMP10]], <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = icmp ult <8 x i32> [[TMP10]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <8 x i1> [[RDX_MINMAX_CMP]], <8 x i32> [[TMP10]], <8 x i32> [[RDX_SHUF]]
@@ -115,7 +88,6 @@ define i32 @foo1(i32* nocapture readonly %arr, i32 %a1, i32 %a2, i32 %a3, i32 %a
 ; CHECK-NEXT:    [[RDX_MINMAX_CMP5:%.*]] = icmp ult <8 x i32> [[RDX_MINMAX_SELECT3]], [[RDX_SHUF4]]
 ; CHECK-NEXT:    [[RDX_MINMAX_SELECT6:%.*]] = select <8 x i1> [[RDX_MINMAX_CMP5]], <8 x i32> [[RDX_MINMAX_SELECT3]], <8 x i32> [[RDX_SHUF4]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <8 x i32> [[RDX_MINMAX_SELECT6]], i32 0
-; CHECK-NEXT:    [[COND44:%.*]] = select i1 [[CMP40]], i32 [[COND39]], i32 undef
 ; CHECK-NEXT:    ret i32 [[TMP11]]
 ;
 entry:
@@ -170,19 +142,6 @@ define i32 @foo2(i32* nocapture readonly %arr, i32 %a1, i32 %a2, i32 %a3, i32 %a
 ; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <8 x i32> [[TMP7]], i32 [[A7:%.*]], i32 6
 ; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <8 x i32> [[TMP8]], i32 [[A8:%.*]], i32 7
 ; CHECK-NEXT:    [[TMP10:%.*]] = add <8 x i32> [[SHUFFLE]], [[TMP9]]
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 undef, undef
-; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i32 undef, i32 undef
-; CHECK-NEXT:    [[CMP15:%.*]] = icmp ult i32 [[COND]], undef
-; CHECK-NEXT:    [[COND19:%.*]] = select i1 [[CMP15]], i32 [[COND]], i32 undef
-; CHECK-NEXT:    [[CMP20:%.*]] = icmp ult i32 [[COND19]], undef
-; CHECK-NEXT:    [[COND24:%.*]] = select i1 [[CMP20]], i32 [[COND19]], i32 undef
-; CHECK-NEXT:    [[CMP25:%.*]] = icmp ult i32 [[COND24]], undef
-; CHECK-NEXT:    [[COND29:%.*]] = select i1 [[CMP25]], i32 [[COND24]], i32 undef
-; CHECK-NEXT:    [[CMP30:%.*]] = icmp ult i32 [[COND29]], undef
-; CHECK-NEXT:    [[COND34:%.*]] = select i1 [[CMP30]], i32 [[COND29]], i32 undef
-; CHECK-NEXT:    [[CMP35:%.*]] = icmp ult i32 [[COND34]], undef
-; CHECK-NEXT:    [[COND39:%.*]] = select i1 [[CMP35]], i32 [[COND34]], i32 undef
-; CHECK-NEXT:    [[CMP40:%.*]] = icmp ult i32 [[COND39]], undef
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x i32> [[TMP10]], <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = icmp ult <8 x i32> [[TMP10]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <8 x i1> [[RDX_MINMAX_CMP]], <8 x i32> [[TMP10]], <8 x i32> [[RDX_SHUF]]
@@ -193,7 +152,6 @@ define i32 @foo2(i32* nocapture readonly %arr, i32 %a1, i32 %a2, i32 %a3, i32 %a
 ; CHECK-NEXT:    [[RDX_MINMAX_CMP5:%.*]] = icmp ult <8 x i32> [[RDX_MINMAX_SELECT3]], [[RDX_SHUF4]]
 ; CHECK-NEXT:    [[RDX_MINMAX_SELECT6:%.*]] = select <8 x i1> [[RDX_MINMAX_CMP5]], <8 x i32> [[RDX_MINMAX_SELECT3]], <8 x i32> [[RDX_SHUF4]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <8 x i32> [[RDX_MINMAX_SELECT6]], i32 0
-; CHECK-NEXT:    [[COND44:%.*]] = select i1 [[CMP40]], i32 [[COND39]], i32 undef
 ; CHECK-NEXT:    ret i32 [[TMP11]]
 ;
 entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/zext.ll b/llvm/test/Transforms/SLPVectorizer/X86/zext.ll
index a05e41861..ead4ffdeb 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/zext.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/zext.ll
@@ -11,26 +11,15 @@
 ;
 
 define <2 x i64> @loadext_2i8_to_2i64(i8* %p0) {
-; SSE2-LABEL: @loadext_2i8_to_2i64(
-; SSE2-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
-; SSE2-NEXT:    [[I0:%.*]] = load i8, i8* [[P0]], align 1
-; SSE2-NEXT:    [[I1:%.*]] = load i8, i8* [[P1]], align 1
-; SSE2-NEXT:    [[X0:%.*]] = zext i8 [[I0]] to i64
-; SSE2-NEXT:    [[X1:%.*]] = zext i8 [[I1]] to i64
-; SSE2-NEXT:    [[V0:%.*]] = insertelement <2 x i64> undef, i64 [[X0]], i32 0
-; SSE2-NEXT:    [[V1:%.*]] = insertelement <2 x i64> [[V0]], i64 [[X1]], i32 1
-; SSE2-NEXT:    ret <2 x i64> [[V1]]
-;
-; SLM-LABEL: @loadext_2i8_to_2i64(
-; SLM-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
-; SLM-NEXT:    [[TMP1:%.*]] = bitcast i8* [[P0]] to <2 x i8>*
-; SLM-NEXT:    [[TMP2:%.*]] = load <2 x i8>, <2 x i8>* [[TMP1]], align 1
-; SLM-NEXT:    [[TMP3:%.*]] = zext <2 x i8> [[TMP2]] to <2 x i64>
-; SLM-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i32 0
-; SLM-NEXT:    [[V0:%.*]] = insertelement <2 x i64> undef, i64 [[TMP4]], i32 0
-; SLM-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[TMP3]], i32 1
-; SLM-NEXT:    [[V1:%.*]] = insertelement <2 x i64> [[V0]], i64 [[TMP5]], i32 1
-; SLM-NEXT:    ret <2 x i64> [[V1]]
+; SSE-LABEL: @loadext_2i8_to_2i64(
+; SSE-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; SSE-NEXT:    [[I0:%.*]] = load i8, i8* [[P0]], align 1
+; SSE-NEXT:    [[I1:%.*]] = load i8, i8* [[P1]], align 1
+; SSE-NEXT:    [[X0:%.*]] = zext i8 [[I0]] to i64
+; SSE-NEXT:    [[X1:%.*]] = zext i8 [[I1]] to i64
+; SSE-NEXT:    [[V0:%.*]] = insertelement <2 x i64> undef, i64 [[X0]], i32 0
+; SSE-NEXT:    [[V1:%.*]] = insertelement <2 x i64> [[V0]], i64 [[X1]], i32 1
+; SSE-NEXT:    ret <2 x i64> [[V1]]
 ;
 ; AVX-LABEL: @loadext_2i8_to_2i64(
 ; AVX-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
@@ -54,22 +43,57 @@ define <2 x i64> @loadext_2i8_to_2i64(i8* %p0) {
 }
 
 define <4 x i32> @loadext_4i8_to_4i32(i8* %p0) {
-; CHECK-LABEL: @loadext_4i8_to_4i32(
-; CHECK-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
-; CHECK-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
-; CHECK-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8* [[P0]] to <4 x i8>*
-; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i8>, <4 x i8>* [[TMP1]], align 1
-; CHECK-NEXT:    [[TMP3:%.*]] = zext <4 x i8> [[TMP2]] to <4 x i32>
-; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x i32> [[TMP3]], i32 0
-; CHECK-NEXT:    [[V0:%.*]] = insertelement <4 x i32> undef, i32 [[TMP4]], i32 0
-; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x i32> [[TMP3]], i32 1
-; CHECK-NEXT:    [[V1:%.*]] = insertelement <4 x i32> [[V0]], i32 [[TMP5]], i32 1
-; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x i32> [[TMP3]], i32 2
-; CHECK-NEXT:    [[V2:%.*]] = insertelement <4 x i32> [[V1]], i32 [[TMP6]], i32 2
-; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x i32> [[TMP3]], i32 3
-; CHECK-NEXT:    [[V3:%.*]] = insertelement <4 x i32> [[V2]], i32 [[TMP7]], i32 3
-; CHECK-NEXT:    ret <4 x i32> [[V3]]
+; SSE2-LABEL: @loadext_4i8_to_4i32(
+; SSE2-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; SSE2-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; SSE2-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; SSE2-NEXT:    [[TMP1:%.*]] = bitcast i8* [[P0]] to <4 x i8>*
+; SSE2-NEXT:    [[TMP2:%.*]] = load <4 x i8>, <4 x i8>* [[TMP1]], align 1
+; SSE2-NEXT:    [[TMP3:%.*]] = zext <4 x i8> [[TMP2]] to <4 x i32>
+; SSE2-NEXT:    [[TMP4:%.*]] = extractelement <4 x i32> [[TMP3]], i32 0
+; SSE2-NEXT:    [[V0:%.*]] = insertelement <4 x i32> undef, i32 [[TMP4]], i32 0
+; SSE2-NEXT:    [[TMP5:%.*]] = extractelement <4 x i32> [[TMP3]], i32 1
+; SSE2-NEXT:    [[V1:%.*]] = insertelement <4 x i32> [[V0]], i32 [[TMP5]], i32 1
+; SSE2-NEXT:    [[TMP6:%.*]] = extractelement <4 x i32> [[TMP3]], i32 2
+; SSE2-NEXT:    [[V2:%.*]] = insertelement <4 x i32> [[V1]], i32 [[TMP6]], i32 2
+; SSE2-NEXT:    [[TMP7:%.*]] = extractelement <4 x i32> [[TMP3]], i32 3
+; SSE2-NEXT:    [[V3:%.*]] = insertelement <4 x i32> [[V2]], i32 [[TMP7]], i32 3
+; SSE2-NEXT:    ret <4 x i32> [[V3]]
+;
+; SLM-LABEL: @loadext_4i8_to_4i32(
+; SLM-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; SLM-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; SLM-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; SLM-NEXT:    [[I0:%.*]] = load i8, i8* [[P0]], align 1
+; SLM-NEXT:    [[I1:%.*]] = load i8, i8* [[P1]], align 1
+; SLM-NEXT:    [[I2:%.*]] = load i8, i8* [[P2]], align 1
+; SLM-NEXT:    [[I3:%.*]] = load i8, i8* [[P3]], align 1
+; SLM-NEXT:    [[X0:%.*]] = zext i8 [[I0]] to i32
+; SLM-NEXT:    [[X1:%.*]] = zext i8 [[I1]] to i32
+; SLM-NEXT:    [[X2:%.*]] = zext i8 [[I2]] to i32
+; SLM-NEXT:    [[X3:%.*]] = zext i8 [[I3]] to i32
+; SLM-NEXT:    [[V0:%.*]] = insertelement <4 x i32> undef, i32 [[X0]], i32 0
+; SLM-NEXT:    [[V1:%.*]] = insertelement <4 x i32> [[V0]], i32 [[X1]], i32 1
+; SLM-NEXT:    [[V2:%.*]] = insertelement <4 x i32> [[V1]], i32 [[X2]], i32 2
+; SLM-NEXT:    [[V3:%.*]] = insertelement <4 x i32> [[V2]], i32 [[X3]], i32 3
+; SLM-NEXT:    ret <4 x i32> [[V3]]
+;
+; AVX-LABEL: @loadext_4i8_to_4i32(
+; AVX-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; AVX-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; AVX-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; AVX-NEXT:    [[TMP1:%.*]] = bitcast i8* [[P0]] to <4 x i8>*
+; AVX-NEXT:    [[TMP2:%.*]] = load <4 x i8>, <4 x i8>* [[TMP1]], align 1
+; AVX-NEXT:    [[TMP3:%.*]] = zext <4 x i8> [[TMP2]] to <4 x i32>
+; AVX-NEXT:    [[TMP4:%.*]] = extractelement <4 x i32> [[TMP3]], i32 0
+; AVX-NEXT:    [[V0:%.*]] = insertelement <4 x i32> undef, i32 [[TMP4]], i32 0
+; AVX-NEXT:    [[TMP5:%.*]] = extractelement <4 x i32> [[TMP3]], i32 1
+; AVX-NEXT:    [[V1:%.*]] = insertelement <4 x i32> [[V0]], i32 [[TMP5]], i32 1
+; AVX-NEXT:    [[TMP6:%.*]] = extractelement <4 x i32> [[TMP3]], i32 2
+; AVX-NEXT:    [[V2:%.*]] = insertelement <4 x i32> [[V1]], i32 [[TMP6]], i32 2
+; AVX-NEXT:    [[TMP7:%.*]] = extractelement <4 x i32> [[TMP3]], i32 3
+; AVX-NEXT:    [[V3:%.*]] = insertelement <4 x i32> [[V2]], i32 [[TMP7]], i32 3
+; AVX-NEXT:    ret <4 x i32> [[V3]]
 ;
   %p1 = getelementptr inbounds i8, i8* %p0, i64 1
   %p2 = getelementptr inbounds i8, i8* %p0, i64 2
@@ -90,59 +114,76 @@ define <4 x i32> @loadext_4i8_to_4i32(i8* %p0) {
 }
 
 define <4 x i64> @loadext_4i8_to_4i64(i8* %p0) {
-; SSE2-LABEL: @loadext_4i8_to_4i64(
-; SSE2-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
-; SSE2-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
-; SSE2-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
-; SSE2-NEXT:    [[I0:%.*]] = load i8, i8* [[P0]], align 1
-; SSE2-NEXT:    [[I1:%.*]] = load i8, i8* [[P1]], align 1
-; SSE2-NEXT:    [[I2:%.*]] = load i8, i8* [[P2]], align 1
-; SSE2-NEXT:    [[I3:%.*]] = load i8, i8* [[P3]], align 1
-; SSE2-NEXT:    [[X0:%.*]] = zext i8 [[I0]] to i64
-; SSE2-NEXT:    [[X1:%.*]] = zext i8 [[I1]] to i64
-; SSE2-NEXT:    [[X2:%.*]] = zext i8 [[I2]] to i64
-; SSE2-NEXT:    [[X3:%.*]] = zext i8 [[I3]] to i64
-; SSE2-NEXT:    [[V0:%.*]] = insertelement <4 x i64> undef, i64 [[X0]], i32 0
-; SSE2-NEXT:    [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[X1]], i32 1
-; SSE2-NEXT:    [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[X2]], i32 2
-; SSE2-NEXT:    [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[X3]], i32 3
-; SSE2-NEXT:    ret <4 x i64> [[V3]]
-;
-; SLM-LABEL: @loadext_4i8_to_4i64(
-; SLM-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
-; SLM-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
-; SLM-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
-; SLM-NEXT:    [[TMP1:%.*]] = bitcast i8* [[P0]] to <4 x i8>*
-; SLM-NEXT:    [[TMP2:%.*]] = load <4 x i8>, <4 x i8>* [[TMP1]], align 1
-; SLM-NEXT:    [[TMP3:%.*]] = zext <4 x i8> [[TMP2]] to <4 x i64>
-; SLM-NEXT:    [[TMP4:%.*]] = extractelement <4 x i64> [[TMP3]], i32 0
-; SLM-NEXT:    [[V0:%.*]] = insertelement <4 x i64> undef, i64 [[TMP4]], i32 0
-; SLM-NEXT:    [[TMP5:%.*]] = extractelement <4 x i64> [[TMP3]], i32 1
-; SLM-NEXT:    [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[TMP5]], i32 1
-; SLM-NEXT:    [[TMP6:%.*]] = extractelement <4 x i64> [[TMP3]], i32 2
-; SLM-NEXT:    [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[TMP6]], i32 2
-; SLM-NEXT:    [[TMP7:%.*]] = extractelement <4 x i64> [[TMP3]], i32 3
-; SLM-NEXT:    [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[TMP7]], i32 3
-; SLM-NEXT:    ret <4 x i64> [[V3]]
-;
-; AVX-LABEL: @loadext_4i8_to_4i64(
-; AVX-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
-; AVX-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
-; AVX-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
-; AVX-NEXT:    [[TMP1:%.*]] = bitcast i8* [[P0]] to <2 x i8>*
-; AVX-NEXT:    [[TMP2:%.*]] = load <2 x i8>, <2 x i8>* [[TMP1]], align 1
-; AVX-NEXT:    [[I2:%.*]] = load i8, i8* [[P2]], align 1
-; AVX-NEXT:    [[I3:%.*]] = load i8, i8* [[P3]], align 1
-; AVX-NEXT:    [[TMP3:%.*]] = zext <2 x i8> [[TMP2]] to <2 x i64>
-; AVX-NEXT:    [[X2:%.*]] = zext i8 [[I2]] to i64
-; AVX-NEXT:    [[X3:%.*]] = zext i8 [[I3]] to i64
-; AVX-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i32 0
-; AVX-NEXT:    [[V0:%.*]] = insertelement <4 x i64> undef, i64 [[TMP4]], i32 0
-; AVX-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[TMP3]], i32 1
-; AVX-NEXT:    [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[TMP5]], i32 1
-; AVX-NEXT:    [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[X2]], i32 2
-; AVX-NEXT:    [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[X3]], i32 3
-; AVX-NEXT:    ret <4 x i64> [[V3]]
+; SSE-LABEL: @loadext_4i8_to_4i64(
+; SSE-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; SSE-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; SSE-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; SSE-NEXT:    [[I0:%.*]] = load i8, i8* [[P0]], align 1
+; SSE-NEXT:    [[I1:%.*]] = load i8, i8* [[P1]], align 1
+; SSE-NEXT:    [[I2:%.*]] = load i8, i8* [[P2]], align 1
+; SSE-NEXT:    [[I3:%.*]] = load i8, i8* [[P3]], align 1
+; SSE-NEXT:    [[X0:%.*]] = zext i8 [[I0]] to i64
+; SSE-NEXT:    [[X1:%.*]] = zext i8 [[I1]] to i64
+; SSE-NEXT:    [[X2:%.*]] = zext i8 [[I2]] to i64
+; SSE-NEXT:    [[X3:%.*]] = zext i8 [[I3]] to i64
+; SSE-NEXT:    [[V0:%.*]] = insertelement <4 x i64> undef, i64 [[X0]], i32 0
+; SSE-NEXT:    [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[X1]], i32 1
+; SSE-NEXT:    [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[X2]], i32 2
+; SSE-NEXT:    [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[X3]], i32 3
+; SSE-NEXT:    ret <4 x i64> [[V3]]
+;
+; AVX1-LABEL: @loadext_4i8_to_4i64(
+; AVX1-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; AVX1-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; AVX1-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; AVX1-NEXT:    [[TMP1:%.*]] = bitcast i8* [[P0]] to <2 x i8>*
+; AVX1-NEXT:    [[TMP2:%.*]] = load <2 x i8>, <2 x i8>* [[TMP1]], align 1
+; AVX1-NEXT:    [[I2:%.*]] = load i8, i8* [[P2]], align 1
+; AVX1-NEXT:    [[I3:%.*]] = load i8, i8* [[P3]], align 1
+; AVX1-NEXT:    [[TMP3:%.*]] = zext <2 x i8> [[TMP2]] to <2 x i64>
+; AVX1-NEXT:    [[X2:%.*]] = zext i8 [[I2]] to i64
+; AVX1-NEXT:    [[X3:%.*]] = zext i8 [[I3]] to i64
+; AVX1-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i32 0
+; AVX1-NEXT:    [[V0:%.*]] = insertelement <4 x i64> undef, i64 [[TMP4]], i32 0
+; AVX1-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[TMP3]], i32 1
+; AVX1-NEXT:    [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[TMP5]], i32 1
+; AVX1-NEXT:    [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[X2]], i32 2
+; AVX1-NEXT:    [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[X3]], i32 3
+; AVX1-NEXT:    ret <4 x i64> [[V3]]
+;
+; AVX2-LABEL: @loadext_4i8_to_4i64(
+; AVX2-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; AVX2-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; AVX2-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; AVX2-NEXT:    [[TMP1:%.*]] = bitcast i8* [[P0]] to <4 x i8>*
+; AVX2-NEXT:    [[TMP2:%.*]] = load <4 x i8>, <4 x i8>* [[TMP1]], align 1
+; AVX2-NEXT:    [[TMP3:%.*]] = zext <4 x i8> [[TMP2]] to <4 x i64>
+; AVX2-NEXT:    [[TMP4:%.*]] = extractelement <4 x i64> [[TMP3]], i32 0
+; AVX2-NEXT:    [[V0:%.*]] = insertelement <4 x i64> undef, i64 [[TMP4]], i32 0
+; AVX2-NEXT:    [[TMP5:%.*]] = extractelement <4 x i64> [[TMP3]], i32 1
+; AVX2-NEXT:    [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[TMP5]], i32 1
+; AVX2-NEXT:    [[TMP6:%.*]] = extractelement <4 x i64> [[TMP3]], i32 2
+; AVX2-NEXT:    [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[TMP6]], i32 2
+; AVX2-NEXT:    [[TMP7:%.*]] = extractelement <4 x i64> [[TMP3]], i32 3
+; AVX2-NEXT:    [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[TMP7]], i32 3
+; AVX2-NEXT:    ret <4 x i64> [[V3]]
+;
+; AVX512-LABEL: @loadext_4i8_to_4i64(
+; AVX512-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; AVX512-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; AVX512-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; AVX512-NEXT:    [[TMP1:%.*]] = bitcast i8* [[P0]] to <4 x i8>*
+; AVX512-NEXT:    [[TMP2:%.*]] = load <4 x i8>, <4 x i8>* [[TMP1]], align 1
+; AVX512-NEXT:    [[TMP3:%.*]] = zext <4 x i8> [[TMP2]] to <4 x i64>
+; AVX512-NEXT:    [[TMP4:%.*]] = extractelement <4 x i64> [[TMP3]], i32 0
+; AVX512-NEXT:    [[V0:%.*]] = insertelement <4 x i64> undef, i64 [[TMP4]], i32 0
+; AVX512-NEXT:    [[TMP5:%.*]] = extractelement <4 x i64> [[TMP3]], i32 1
+; AVX512-NEXT:    [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[TMP5]], i32 1
+; AVX512-NEXT:    [[TMP6:%.*]] = extractelement <4 x i64> [[TMP3]], i32 2
+; AVX512-NEXT:    [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[TMP6]], i32 2
+; AVX512-NEXT:    [[TMP7:%.*]] = extractelement <4 x i64> [[TMP3]], i32 3
+; AVX512-NEXT:    [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[TMP7]], i32 3
+; AVX512-NEXT:    ret <4 x i64> [[V3]]
 ;
   %p1 = getelementptr inbounds i8, i8* %p0, i64 1
   %p2 = getelementptr inbounds i8, i8* %p0, i64 2
@@ -163,34 +204,97 @@ define <4 x i64> @loadext_4i8_to_4i64(i8* %p0) {
 }
 
 define <8 x i16> @loadext_8i8_to_8i16(i8* %p0) {
-; CHECK-LABEL: @loadext_8i8_to_8i16(
-; CHECK-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
-; CHECK-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
-; CHECK-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
-; CHECK-NEXT:    [[P4:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 4
-; CHECK-NEXT:    [[P5:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 5
-; CHECK-NEXT:    [[P6:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 6
-; CHECK-NEXT:    [[P7:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 7
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8* [[P0]] to <8 x i8>*
-; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i8>, <8 x i8>* [[TMP1]], align 1
-; CHECK-NEXT:    [[TMP3:%.*]] = zext <8 x i8> [[TMP2]] to <8 x i16>
-; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <8 x i16> [[TMP3]], i32 0
-; CHECK-NEXT:    [[V0:%.*]] = insertelement <8 x i16> undef, i16 [[TMP4]], i32 0
-; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <8 x i16> [[TMP3]], i32 1
-; CHECK-NEXT:    [[V1:%.*]] = insertelement <8 x i16> [[V0]], i16 [[TMP5]], i32 1
-; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <8 x i16> [[TMP3]], i32 2
-; CHECK-NEXT:    [[V2:%.*]] = insertelement <8 x i16> [[V1]], i16 [[TMP6]], i32 2
-; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <8 x i16> [[TMP3]], i32 3
-; CHECK-NEXT:    [[V3:%.*]] = insertelement <8 x i16> [[V2]], i16 [[TMP7]], i32 3
-; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <8 x i16> [[TMP3]], i32 4
-; CHECK-NEXT:    [[V4:%.*]] = insertelement <8 x i16> [[V3]], i16 [[TMP8]], i32 4
-; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <8 x i16> [[TMP3]], i32 5
-; CHECK-NEXT:    [[V5:%.*]] = insertelement <8 x i16> [[V4]], i16 [[TMP9]], i32 5
-; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <8 x i16> [[TMP3]], i32 6
-; CHECK-NEXT:    [[V6:%.*]] = insertelement <8 x i16> [[V5]], i16 [[TMP10]], i32 6
-; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <8 x i16> [[TMP3]], i32 7
-; CHECK-NEXT:    [[V7:%.*]] = insertelement <8 x i16> [[V6]], i16 [[TMP11]], i32 7
-; CHECK-NEXT:    ret <8 x i16> [[V7]]
+; SSE2-LABEL: @loadext_8i8_to_8i16(
+; SSE2-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; SSE2-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; SSE2-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; SSE2-NEXT:    [[P4:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 4
+; SSE2-NEXT:    [[P5:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 5
+; SSE2-NEXT:    [[P6:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 6
+; SSE2-NEXT:    [[P7:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 7
+; SSE2-NEXT:    [[TMP1:%.*]] = bitcast i8* [[P0]] to <8 x i8>*
+; SSE2-NEXT:    [[TMP2:%.*]] = load <8 x i8>, <8 x i8>* [[TMP1]], align 1
+; SSE2-NEXT:    [[TMP3:%.*]] = zext <8 x i8> [[TMP2]] to <8 x i16>
+; SSE2-NEXT:    [[TMP4:%.*]] = extractelement <8 x i16> [[TMP3]], i32 0
+; SSE2-NEXT:    [[V0:%.*]] = insertelement <8 x i16> undef, i16 [[TMP4]], i32 0
+; SSE2-NEXT:    [[TMP5:%.*]] = extractelement <8 x i16> [[TMP3]], i32 1
+; SSE2-NEXT:    [[V1:%.*]] = insertelement <8 x i16> [[V0]], i16 [[TMP5]], i32 1
+; SSE2-NEXT:    [[TMP6:%.*]] = extractelement <8 x i16> [[TMP3]], i32 2
+; SSE2-NEXT:    [[V2:%.*]] = insertelement <8 x i16> [[V1]], i16 [[TMP6]], i32 2
+; SSE2-NEXT:    [[TMP7:%.*]] = extractelement <8 x i16> [[TMP3]], i32 3
+; SSE2-NEXT:    [[V3:%.*]] = insertelement <8 x i16> [[V2]], i16 [[TMP7]], i32 3
+; SSE2-NEXT:    [[TMP8:%.*]] = extractelement <8 x i16> [[TMP3]], i32 4
+; SSE2-NEXT:    [[V4:%.*]] = insertelement <8 x i16> [[V3]], i16 [[TMP8]], i32 4
+; SSE2-NEXT:    [[TMP9:%.*]] = extractelement <8 x i16> [[TMP3]], i32 5
+; SSE2-NEXT:    [[V5:%.*]] = insertelement <8 x i16> [[V4]], i16 [[TMP9]], i32 5
+; SSE2-NEXT:    [[TMP10:%.*]] = extractelement <8 x i16> [[TMP3]], i32 6
+; SSE2-NEXT:    [[V6:%.*]] = insertelement <8 x i16> [[V5]], i16 [[TMP10]], i32 6
+; SSE2-NEXT:    [[TMP11:%.*]] = extractelement <8 x i16> [[TMP3]], i32 7
+; SSE2-NEXT:    [[V7:%.*]] = insertelement <8 x i16> [[V6]], i16 [[TMP11]], i32 7
+; SSE2-NEXT:    ret <8 x i16> [[V7]]
+;
+; SLM-LABEL: @loadext_8i8_to_8i16(
+; SLM-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; SLM-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; SLM-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; SLM-NEXT:    [[P4:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 4
+; SLM-NEXT:    [[P5:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 5
+; SLM-NEXT:    [[P6:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 6
+; SLM-NEXT:    [[P7:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 7
+; SLM-NEXT:    [[I0:%.*]] = load i8, i8* [[P0]], align 1
+; SLM-NEXT:    [[I1:%.*]] = load i8, i8* [[P1]], align 1
+; SLM-NEXT:    [[I2:%.*]] = load i8, i8* [[P2]], align 1
+; SLM-NEXT:    [[I3:%.*]] = load i8, i8* [[P3]], align 1
+; SLM-NEXT:    [[I4:%.*]] = load i8, i8* [[P4]], align 1
+; SLM-NEXT:    [[I5:%.*]] = load i8, i8* [[P5]], align 1
+; SLM-NEXT:    [[I6:%.*]] = load i8, i8* [[P6]], align 1
+; SLM-NEXT:    [[I7:%.*]] = load i8, i8* [[P7]], align 1
+; SLM-NEXT:    [[X0:%.*]] = zext i8 [[I0]] to i16
+; SLM-NEXT:    [[X1:%.*]] = zext i8 [[I1]] to i16
+; SLM-NEXT:    [[X2:%.*]] = zext i8 [[I2]] to i16
+; SLM-NEXT:    [[X3:%.*]] = zext i8 [[I3]] to i16
+; SLM-NEXT:    [[X4:%.*]] = zext i8 [[I4]] to i16
+; SLM-NEXT:    [[X5:%.*]] = zext i8 [[I5]] to i16
+; SLM-NEXT:    [[X6:%.*]] = zext i8 [[I6]] to i16
+; SLM-NEXT:    [[X7:%.*]] = zext i8 [[I7]] to i16
+; SLM-NEXT:    [[V0:%.*]] = insertelement <8 x i16> undef, i16 [[X0]], i32 0
+; SLM-NEXT:    [[V1:%.*]] = insertelement <8 x i16> [[V0]], i16 [[X1]], i32 1
+; SLM-NEXT:    [[V2:%.*]] = insertelement <8 x i16> [[V1]], i16 [[X2]], i32 2
+; SLM-NEXT:    [[V3:%.*]] = insertelement <8 x i16> [[V2]], i16 [[X3]], i32 3
+; SLM-NEXT:    [[V4:%.*]] = insertelement <8 x i16> [[V3]], i16 [[X4]], i32 4
+; SLM-NEXT:    [[V5:%.*]] = insertelement <8 x i16> [[V4]], i16 [[X5]], i32 5
+; SLM-NEXT:    [[V6:%.*]] = insertelement <8 x i16> [[V5]], i16 [[X6]], i32 6
+; SLM-NEXT:    [[V7:%.*]] = insertelement <8 x i16> [[V6]], i16 [[X7]], i32 7
+; SLM-NEXT:    ret <8 x i16> [[V7]]
+;
+; AVX-LABEL: @loadext_8i8_to_8i16(
+; AVX-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; AVX-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; AVX-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; AVX-NEXT:    [[P4:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 4
+; AVX-NEXT:    [[P5:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 5
+; AVX-NEXT:    [[P6:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 6
+; AVX-NEXT:    [[P7:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 7
+; AVX-NEXT:    [[TMP1:%.*]] = bitcast i8* [[P0]] to <8 x i8>*
+; AVX-NEXT:    [[TMP2:%.*]] = load <8 x i8>, <8 x i8>* [[TMP1]], align 1
+; AVX-NEXT:    [[TMP3:%.*]] = zext <8 x i8> [[TMP2]] to <8 x i16>
+; AVX-NEXT:    [[TMP4:%.*]] = extractelement <8 x i16> [[TMP3]], i32 0
+; AVX-NEXT:    [[V0:%.*]] = insertelement <8 x i16> undef, i16 [[TMP4]], i32 0
+; AVX-NEXT:    [[TMP5:%.*]] = extractelement <8 x i16> [[TMP3]], i32 1
+; AVX-NEXT:    [[V1:%.*]] = insertelement <8 x i16> [[V0]], i16 [[TMP5]], i32 1
+; AVX-NEXT:    [[TMP6:%.*]] = extractelement <8 x i16> [[TMP3]], i32 2
+; AVX-NEXT:    [[V2:%.*]] = insertelement <8 x i16> [[V1]], i16 [[TMP6]], i32 2
+; AVX-NEXT:    [[TMP7:%.*]] = extractelement <8 x i16> [[TMP3]], i32 3
+; AVX-NEXT:    [[V3:%.*]] = insertelement <8 x i16> [[V2]], i16 [[TMP7]], i32 3
+; AVX-NEXT:    [[TMP8:%.*]] = extractelement <8 x i16> [[TMP3]], i32 4
+; AVX-NEXT:    [[V4:%.*]] = insertelement <8 x i16> [[V3]], i16 [[TMP8]], i32 4
+; AVX-NEXT:    [[TMP9:%.*]] = extractelement <8 x i16> [[TMP3]], i32 5
+; AVX-NEXT:    [[V5:%.*]] = insertelement <8 x i16> [[V4]], i16 [[TMP9]], i32 5
+; AVX-NEXT:    [[TMP10:%.*]] = extractelement <8 x i16> [[TMP3]], i32 6
+; AVX-NEXT:    [[V6:%.*]] = insertelement <8 x i16> [[V5]], i16 [[TMP10]], i32 6
+; AVX-NEXT:    [[TMP11:%.*]] = extractelement <8 x i16> [[TMP3]], i32 7
+; AVX-NEXT:    [[V7:%.*]] = insertelement <8 x i16> [[V6]], i16 [[TMP11]], i32 7
+; AVX-NEXT:    ret <8 x i16> [[V7]]
 ;
   %p1 = getelementptr inbounds i8, i8* %p0, i64 1
   %p2 = getelementptr inbounds i8, i8* %p0, i64 2
@@ -227,34 +331,97 @@ define <8 x i16> @loadext_8i8_to_8i16(i8* %p0) {
 }
 
 define <8 x i32> @loadext_8i8_to_8i32(i8* %p0) {
-; CHECK-LABEL: @loadext_8i8_to_8i32(
-; CHECK-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
-; CHECK-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
-; CHECK-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
-; CHECK-NEXT:    [[P4:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 4
-; CHECK-NEXT:    [[P5:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 5
-; CHECK-NEXT:    [[P6:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 6
-; CHECK-NEXT:    [[P7:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 7
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8* [[P0]] to <8 x i8>*
-; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i8>, <8 x i8>* [[TMP1]], align 1
-; CHECK-NEXT:    [[TMP3:%.*]] = zext <8 x i8> [[TMP2]] to <8 x i32>
-; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <8 x i32> [[TMP3]], i32 0
-; CHECK-NEXT:    [[V0:%.*]] = insertelement <8 x i32> undef, i32 [[TMP4]], i32 0
-; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <8 x i32> [[TMP3]], i32 1
-; CHECK-NEXT:    [[V1:%.*]] = insertelement <8 x i32> [[V0]], i32 [[TMP5]], i32 1
-; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <8 x i32> [[TMP3]], i32 2
-; CHECK-NEXT:    [[V2:%.*]] = insertelement <8 x i32> [[V1]], i32 [[TMP6]], i32 2
-; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <8 x i32> [[TMP3]], i32 3
-; CHECK-NEXT:    [[V3:%.*]] = insertelement <8 x i32> [[V2]], i32 [[TMP7]], i32 3
-; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <8 x i32> [[TMP3]], i32 4
-; CHECK-NEXT:    [[V4:%.*]] = insertelement <8 x i32> [[V3]], i32 [[TMP8]], i32 4
-; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <8 x i32> [[TMP3]], i32 5
-; CHECK-NEXT:    [[V5:%.*]] = insertelement <8 x i32> [[V4]], i32 [[TMP9]], i32 5
-; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <8 x i32> [[TMP3]], i32 6
-; CHECK-NEXT:    [[V6:%.*]] = insertelement <8 x i32> [[V5]], i32 [[TMP10]], i32 6
-; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <8 x i32> [[TMP3]], i32 7
-; CHECK-NEXT:    [[V7:%.*]] = insertelement <8 x i32> [[V6]], i32 [[TMP11]], i32 7
-; CHECK-NEXT:    ret <8 x i32> [[V7]]
+; SSE2-LABEL: @loadext_8i8_to_8i32(
+; SSE2-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; SSE2-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; SSE2-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; SSE2-NEXT:    [[P4:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 4
+; SSE2-NEXT:    [[P5:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 5
+; SSE2-NEXT:    [[P6:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 6
+; SSE2-NEXT:    [[P7:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 7
+; SSE2-NEXT:    [[TMP1:%.*]] = bitcast i8* [[P0]] to <8 x i8>*
+; SSE2-NEXT:    [[TMP2:%.*]] = load <8 x i8>, <8 x i8>* [[TMP1]], align 1
+; SSE2-NEXT:    [[TMP3:%.*]] = zext <8 x i8> [[TMP2]] to <8 x i32>
+; SSE2-NEXT:    [[TMP4:%.*]] = extractelement <8 x i32> [[TMP3]], i32 0
+; SSE2-NEXT:    [[V0:%.*]] = insertelement <8 x i32> undef, i32 [[TMP4]], i32 0
+; SSE2-NEXT:    [[TMP5:%.*]] = extractelement <8 x i32> [[TMP3]], i32 1
+; SSE2-NEXT:    [[V1:%.*]] = insertelement <8 x i32> [[V0]], i32 [[TMP5]], i32 1
+; SSE2-NEXT:    [[TMP6:%.*]] = extractelement <8 x i32> [[TMP3]], i32 2
+; SSE2-NEXT:    [[V2:%.*]] = insertelement <8 x i32> [[V1]], i32 [[TMP6]], i32 2
+; SSE2-NEXT:    [[TMP7:%.*]] = extractelement <8 x i32> [[TMP3]], i32 3
+; SSE2-NEXT:    [[V3:%.*]] = insertelement <8 x i32> [[V2]], i32 [[TMP7]], i32 3
+; SSE2-NEXT:    [[TMP8:%.*]] = extractelement <8 x i32> [[TMP3]], i32 4
+; SSE2-NEXT:    [[V4:%.*]] = insertelement <8 x i32> [[V3]], i32 [[TMP8]], i32 4
+; SSE2-NEXT:    [[TMP9:%.*]] = extractelement <8 x i32> [[TMP3]], i32 5
+; SSE2-NEXT:    [[V5:%.*]] = insertelement <8 x i32> [[V4]], i32 [[TMP9]], i32 5
+; SSE2-NEXT:    [[TMP10:%.*]] = extractelement <8 x i32> [[TMP3]], i32 6
+; SSE2-NEXT:    [[V6:%.*]] = insertelement <8 x i32> [[V5]], i32 [[TMP10]], i32 6
+; SSE2-NEXT:    [[TMP11:%.*]] = extractelement <8 x i32> [[TMP3]], i32 7
+; SSE2-NEXT:    [[V7:%.*]] = insertelement <8 x i32> [[V6]], i32 [[TMP11]], i32 7
+; SSE2-NEXT:    ret <8 x i32> [[V7]]
+;
+; SLM-LABEL: @loadext_8i8_to_8i32(
+; SLM-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; SLM-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; SLM-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; SLM-NEXT:    [[P4:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 4
+; SLM-NEXT:    [[P5:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 5
+; SLM-NEXT:    [[P6:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 6
+; SLM-NEXT:    [[P7:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 7
+; SLM-NEXT:    [[I0:%.*]] = load i8, i8* [[P0]], align 1
+; SLM-NEXT:    [[I1:%.*]] = load i8, i8* [[P1]], align 1
+; SLM-NEXT:    [[I2:%.*]] = load i8, i8* [[P2]], align 1
+; SLM-NEXT:    [[I3:%.*]] = load i8, i8* [[P3]], align 1
+; SLM-NEXT:    [[I4:%.*]] = load i8, i8* [[P4]], align 1
+; SLM-NEXT:    [[I5:%.*]] = load i8, i8* [[P5]], align 1
+; SLM-NEXT:    [[I6:%.*]] = load i8, i8* [[P6]], align 1
+; SLM-NEXT:    [[I7:%.*]] = load i8, i8* [[P7]], align 1
+; SLM-NEXT:    [[X0:%.*]] = zext i8 [[I0]] to i32
+; SLM-NEXT:    [[X1:%.*]] = zext i8 [[I1]] to i32
+; SLM-NEXT:    [[X2:%.*]] = zext i8 [[I2]] to i32
+; SLM-NEXT:    [[X3:%.*]] = zext i8 [[I3]] to i32
+; SLM-NEXT:    [[X4:%.*]] = zext i8 [[I4]] to i32
+; SLM-NEXT:    [[X5:%.*]] = zext i8 [[I5]] to i32
+; SLM-NEXT:    [[X6:%.*]] = zext i8 [[I6]] to i32
+; SLM-NEXT:    [[X7:%.*]] = zext i8 [[I7]] to i32
+; SLM-NEXT:    [[V0:%.*]] = insertelement <8 x i32> undef, i32 [[X0]], i32 0
+; SLM-NEXT:    [[V1:%.*]] = insertelement <8 x i32> [[V0]], i32 [[X1]], i32 1
+; SLM-NEXT:    [[V2:%.*]] = insertelement <8 x i32> [[V1]], i32 [[X2]], i32 2
+; SLM-NEXT:    [[V3:%.*]] = insertelement <8 x i32> [[V2]], i32 [[X3]], i32 3
+; SLM-NEXT:    [[V4:%.*]] = insertelement <8 x i32> [[V3]], i32 [[X4]], i32 4
+; SLM-NEXT:    [[V5:%.*]] = insertelement <8 x i32> [[V4]], i32 [[X5]], i32 5
+; SLM-NEXT:    [[V6:%.*]] = insertelement <8 x i32> [[V5]], i32 [[X6]], i32 6
+; SLM-NEXT:    [[V7:%.*]] = insertelement <8 x i32> [[V6]], i32 [[X7]], i32 7
+; SLM-NEXT:    ret <8 x i32> [[V7]]
+;
+; AVX-LABEL: @loadext_8i8_to_8i32(
+; AVX-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; AVX-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; AVX-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; AVX-NEXT:    [[P4:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 4
+; AVX-NEXT:    [[P5:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 5
+; AVX-NEXT:    [[P6:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 6
+; AVX-NEXT:    [[P7:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 7
+; AVX-NEXT:    [[TMP1:%.*]] = bitcast i8* [[P0]] to <8 x i8>*
+; AVX-NEXT:    [[TMP2:%.*]] = load <8 x i8>, <8 x i8>* [[TMP1]], align 1
+; AVX-NEXT:    [[TMP3:%.*]] = zext <8 x i8> [[TMP2]] to <8 x i32>
+; AVX-NEXT:    [[TMP4:%.*]] = extractelement <8 x i32> [[TMP3]], i32 0
+; AVX-NEXT:    [[V0:%.*]] = insertelement <8 x i32> undef, i32 [[TMP4]], i32 0
+; AVX-NEXT:    [[TMP5:%.*]] = extractelement <8 x i32> [[TMP3]], i32 1
+; AVX-NEXT:    [[V1:%.*]] = insertelement <8 x i32> [[V0]], i32 [[TMP5]], i32 1
+; AVX-NEXT:    [[TMP6:%.*]] = extractelement <8 x i32> [[TMP3]], i32 2
+; AVX-NEXT:    [[V2:%.*]] = insertelement <8 x i32> [[V1]], i32 [[TMP6]], i32 2
+; AVX-NEXT:    [[TMP7:%.*]] = extractelement <8 x i32> [[TMP3]], i32 3
+; AVX-NEXT:    [[V3:%.*]] = insertelement <8 x i32> [[V2]], i32 [[TMP7]], i32 3
+; AVX-NEXT:    [[TMP8:%.*]] = extractelement <8 x i32> [[TMP3]], i32 4
+; AVX-NEXT:    [[V4:%.*]] = insertelement <8 x i32> [[V3]], i32 [[TMP8]], i32 4
+; AVX-NEXT:    [[TMP9:%.*]] = extractelement <8 x i32> [[TMP3]], i32 5
+; AVX-NEXT:    [[V5:%.*]] = insertelement <8 x i32> [[V4]], i32 [[TMP9]], i32 5
+; AVX-NEXT:    [[TMP10:%.*]] = extractelement <8 x i32> [[TMP3]], i32 6
+; AVX-NEXT:    [[V6:%.*]] = insertelement <8 x i32> [[V5]], i32 [[TMP10]], i32 6
+; AVX-NEXT:    [[TMP11:%.*]] = extractelement <8 x i32> [[TMP3]], i32 7
+; AVX-NEXT:    [[V7:%.*]] = insertelement <8 x i32> [[V6]], i32 [[TMP11]], i32 7
+; AVX-NEXT:    ret <8 x i32> [[V7]]
 ;
   %p1 = getelementptr inbounds i8, i8* %p0, i64 1
   %p2 = getelementptr inbounds i8, i8* %p0, i64 2
@@ -291,58 +458,177 @@ define <8 x i32> @loadext_8i8_to_8i32(i8* %p0) {
 }
 
 define <16 x i16> @loadext_16i8_to_16i16(i8* %p0) {
-; CHECK-LABEL: @loadext_16i8_to_16i16(
-; CHECK-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
-; CHECK-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
-; CHECK-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
-; CHECK-NEXT:    [[P4:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 4
-; CHECK-NEXT:    [[P5:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 5
-; CHECK-NEXT:    [[P6:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 6
-; CHECK-NEXT:    [[P7:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 7
-; CHECK-NEXT:    [[P8:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 8
-; CHECK-NEXT:    [[P9:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 9
-; CHECK-NEXT:    [[P10:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 10
-; CHECK-NEXT:    [[P11:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 11
-; CHECK-NEXT:    [[P12:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 12
-; CHECK-NEXT:    [[P13:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 13
-; CHECK-NEXT:    [[P14:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 14
-; CHECK-NEXT:    [[P15:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 15
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8* [[P0]] to <16 x i8>*
-; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* [[TMP1]], align 1
-; CHECK-NEXT:    [[TMP3:%.*]] = zext <16 x i8> [[TMP2]] to <16 x i16>
-; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <16 x i16> [[TMP3]], i32 0
-; CHECK-NEXT:    [[V0:%.*]] = insertelement <16 x i16> undef, i16 [[TMP4]], i32 0
-; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <16 x i16> [[TMP3]], i32 1
-; CHECK-NEXT:    [[V1:%.*]] = insertelement <16 x i16> [[V0]], i16 [[TMP5]], i32 1
-; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <16 x i16> [[TMP3]], i32 2
-; CHECK-NEXT:    [[V2:%.*]] = insertelement <16 x i16> [[V1]], i16 [[TMP6]], i32 2
-; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <16 x i16> [[TMP3]], i32 3
-; CHECK-NEXT:    [[V3:%.*]] = insertelement <16 x i16> [[V2]], i16 [[TMP7]], i32 3
-; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <16 x i16> [[TMP3]], i32 4
-; CHECK-NEXT:    [[V4:%.*]] = insertelement <16 x i16> [[V3]], i16 [[TMP8]], i32 4
-; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <16 x i16> [[TMP3]], i32 5
-; CHECK-NEXT:    [[V5:%.*]] = insertelement <16 x i16> [[V4]], i16 [[TMP9]], i32 5
-; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <16 x i16> [[TMP3]], i32 6
-; CHECK-NEXT:    [[V6:%.*]] = insertelement <16 x i16> [[V5]], i16 [[TMP10]], i32 6
-; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <16 x i16> [[TMP3]], i32 7
-; CHECK-NEXT:    [[V7:%.*]] = insertelement <16 x i16> [[V6]], i16 [[TMP11]], i32 7
-; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <16 x i16> [[TMP3]], i32 8
-; CHECK-NEXT:    [[V8:%.*]] = insertelement <16 x i16> [[V7]], i16 [[TMP12]], i32 8
-; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <16 x i16> [[TMP3]], i32 9
-; CHECK-NEXT:    [[V9:%.*]] = insertelement <16 x i16> [[V8]], i16 [[TMP13]], i32 9
-; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <16 x i16> [[TMP3]], i32 10
-; CHECK-NEXT:    [[V10:%.*]] = insertelement <16 x i16> [[V9]], i16 [[TMP14]], i32 10
-; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <16 x i16> [[TMP3]], i32 11
-; CHECK-NEXT:    [[V11:%.*]] = insertelement <16 x i16> [[V10]], i16 [[TMP15]], i32 11
-; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <16 x i16> [[TMP3]], i32 12
-; CHECK-NEXT:    [[V12:%.*]] = insertelement <16 x i16> [[V11]], i16 [[TMP16]], i32 12
-; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <16 x i16> [[TMP3]], i32 13
-; CHECK-NEXT:    [[V13:%.*]] = insertelement <16 x i16> [[V12]], i16 [[TMP17]], i32 13
-; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <16 x i16> [[TMP3]], i32 14
-; CHECK-NEXT:    [[V14:%.*]] = insertelement <16 x i16> [[V13]], i16 [[TMP18]], i32 14
-; CHECK-NEXT:    [[TMP19:%.*]] = extractelement <16 x i16> [[TMP3]], i32 15
-; CHECK-NEXT:    [[V15:%.*]] = insertelement <16 x i16> [[V14]], i16 [[TMP19]], i32 15
-; CHECK-NEXT:    ret <16 x i16> [[V15]]
+; SSE2-LABEL: @loadext_16i8_to_16i16(
+; SSE2-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; SSE2-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; SSE2-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; SSE2-NEXT:    [[P4:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 4
+; SSE2-NEXT:    [[P5:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 5
+; SSE2-NEXT:    [[P6:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 6
+; SSE2-NEXT:    [[P7:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 7
+; SSE2-NEXT:    [[P8:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 8
+; SSE2-NEXT:    [[P9:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 9
+; SSE2-NEXT:    [[P10:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 10
+; SSE2-NEXT:    [[P11:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 11
+; SSE2-NEXT:    [[P12:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 12
+; SSE2-NEXT:    [[P13:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 13
+; SSE2-NEXT:    [[P14:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 14
+; SSE2-NEXT:    [[P15:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 15
+; SSE2-NEXT:    [[TMP1:%.*]] = bitcast i8* [[P0]] to <16 x i8>*
+; SSE2-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* [[TMP1]], align 1
+; SSE2-NEXT:    [[TMP3:%.*]] = zext <16 x i8> [[TMP2]] to <16 x i16>
+; SSE2-NEXT:    [[TMP4:%.*]] = extractelement <16 x i16> [[TMP3]], i32 0
+; SSE2-NEXT:    [[V0:%.*]] = insertelement <16 x i16> undef, i16 [[TMP4]], i32 0
+; SSE2-NEXT:    [[TMP5:%.*]] = extractelement <16 x i16> [[TMP3]], i32 1
+; SSE2-NEXT:    [[V1:%.*]] = insertelement <16 x i16> [[V0]], i16 [[TMP5]], i32 1
+; SSE2-NEXT:    [[TMP6:%.*]] = extractelement <16 x i16> [[TMP3]], i32 2
+; SSE2-NEXT:    [[V2:%.*]] = insertelement <16 x i16> [[V1]], i16 [[TMP6]], i32 2
+; SSE2-NEXT:    [[TMP7:%.*]] = extractelement <16 x i16> [[TMP3]], i32 3
+; SSE2-NEXT:    [[V3:%.*]] = insertelement <16 x i16> [[V2]], i16 [[TMP7]], i32 3
+; SSE2-NEXT:    [[TMP8:%.*]] = extractelement <16 x i16> [[TMP3]], i32 4
+; SSE2-NEXT:    [[V4:%.*]] = insertelement <16 x i16> [[V3]], i16 [[TMP8]], i32 4
+; SSE2-NEXT:    [[TMP9:%.*]] = extractelement <16 x i16> [[TMP3]], i32 5
+; SSE2-NEXT:    [[V5:%.*]] = insertelement <16 x i16> [[V4]], i16 [[TMP9]], i32 5
+; SSE2-NEXT:    [[TMP10:%.*]] = extractelement <16 x i16> [[TMP3]], i32 6
+; SSE2-NEXT:    [[V6:%.*]] = insertelement <16 x i16> [[V5]], i16 [[TMP10]], i32 6
+; SSE2-NEXT:    [[TMP11:%.*]] = extractelement <16 x i16> [[TMP3]], i32 7
+; SSE2-NEXT:    [[V7:%.*]] = insertelement <16 x i16> [[V6]], i16 [[TMP11]], i32 7
+; SSE2-NEXT:    [[TMP12:%.*]] = extractelement <16 x i16> [[TMP3]], i32 8
+; SSE2-NEXT:    [[V8:%.*]] = insertelement <16 x i16> [[V7]], i16 [[TMP12]], i32 8
+; SSE2-NEXT:    [[TMP13:%.*]] = extractelement <16 x i16> [[TMP3]], i32 9
+; SSE2-NEXT:    [[V9:%.*]] = insertelement <16 x i16> [[V8]], i16 [[TMP13]], i32 9
+; SSE2-NEXT:    [[TMP14:%.*]] = extractelement <16 x i16> [[TMP3]], i32 10
+; SSE2-NEXT:    [[V10:%.*]] = insertelement <16 x i16> [[V9]], i16 [[TMP14]], i32 10
+; SSE2-NEXT:    [[TMP15:%.*]] = extractelement <16 x i16> [[TMP3]], i32 11
+; SSE2-NEXT:    [[V11:%.*]] = insertelement <16 x i16> [[V10]], i16 [[TMP15]], i32 11
+; SSE2-NEXT:    [[TMP16:%.*]] = extractelement <16 x i16> [[TMP3]], i32 12
+; SSE2-NEXT:    [[V12:%.*]] = insertelement <16 x i16> [[V11]], i16 [[TMP16]], i32 12
+; SSE2-NEXT:    [[TMP17:%.*]] = extractelement <16 x i16> [[TMP3]], i32 13
+; SSE2-NEXT:    [[V13:%.*]] = insertelement <16 x i16> [[V12]], i16 [[TMP17]], i32 13
+; SSE2-NEXT:    [[TMP18:%.*]] = extractelement <16 x i16> [[TMP3]], i32 14
+; SSE2-NEXT:    [[V14:%.*]] = insertelement <16 x i16> [[V13]], i16 [[TMP18]], i32 14
+; SSE2-NEXT:    [[TMP19:%.*]] = extractelement <16 x i16> [[TMP3]], i32 15
+; SSE2-NEXT:    [[V15:%.*]] = insertelement <16 x i16> [[V14]], i16 [[TMP19]], i32 15
+; SSE2-NEXT:    ret <16 x i16> [[V15]]
+;
+; SLM-LABEL: @loadext_16i8_to_16i16(
+; SLM-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; SLM-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; SLM-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; SLM-NEXT:    [[P4:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 4
+; SLM-NEXT:    [[P5:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 5
+; SLM-NEXT:    [[P6:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 6
+; SLM-NEXT:    [[P7:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 7
+; SLM-NEXT:    [[P8:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 8
+; SLM-NEXT:    [[P9:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 9
+; SLM-NEXT:    [[P10:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 10
+; SLM-NEXT:    [[P11:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 11
+; SLM-NEXT:    [[P12:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 12
+; SLM-NEXT:    [[P13:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 13
+; SLM-NEXT:    [[P14:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 14
+; SLM-NEXT:    [[P15:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 15
+; SLM-NEXT:    [[I0:%.*]] = load i8, i8* [[P0]], align 1
+; SLM-NEXT:    [[I1:%.*]] = load i8, i8* [[P1]], align 1
+; SLM-NEXT:    [[I2:%.*]] = load i8, i8* [[P2]], align 1
+; SLM-NEXT:    [[I3:%.*]] = load i8, i8* [[P3]], align 1
+; SLM-NEXT:    [[I4:%.*]] = load i8, i8* [[P4]], align 1
+; SLM-NEXT:    [[I5:%.*]] = load i8, i8* [[P5]], align 1
+; SLM-NEXT:    [[I6:%.*]] = load i8, i8* [[P6]], align 1
+; SLM-NEXT:    [[I7:%.*]] = load i8, i8* [[P7]], align 1
+; SLM-NEXT:    [[I8:%.*]] = load i8, i8* [[P8]], align 1
+; SLM-NEXT:    [[I9:%.*]] = load i8, i8* [[P9]], align 1
+; SLM-NEXT:    [[I10:%.*]] = load i8, i8* [[P10]], align 1
+; SLM-NEXT:    [[I11:%.*]] = load i8, i8* [[P11]], align 1
+; SLM-NEXT:    [[I12:%.*]] = load i8, i8* [[P12]], align 1
+; SLM-NEXT:    [[I13:%.*]] = load i8, i8* [[P13]], align 1
+; SLM-NEXT:    [[I14:%.*]] = load i8, i8* [[P14]], align 1
+; SLM-NEXT:    [[I15:%.*]] = load i8, i8* [[P15]], align 1
+; SLM-NEXT:    [[X0:%.*]] = zext i8 [[I0]] to i16
+; SLM-NEXT:    [[X1:%.*]] = zext i8 [[I1]] to i16
+; SLM-NEXT:    [[X2:%.*]] = zext i8 [[I2]] to i16
+; SLM-NEXT:    [[X3:%.*]] = zext i8 [[I3]] to i16
+; SLM-NEXT:    [[X4:%.*]] = zext i8 [[I4]] to i16
+; SLM-NEXT:    [[X5:%.*]] = zext i8 [[I5]] to i16
+; SLM-NEXT:    [[X6:%.*]] = zext i8 [[I6]] to i16
+; SLM-NEXT:    [[X7:%.*]] = zext i8 [[I7]] to i16
+; SLM-NEXT:    [[X8:%.*]] = zext i8 [[I8]] to i16
+; SLM-NEXT:    [[X9:%.*]] = zext i8 [[I9]] to i16
+; SLM-NEXT:    [[X10:%.*]] = zext i8 [[I10]] to i16
+; SLM-NEXT:    [[X11:%.*]] = zext i8 [[I11]] to i16
+; SLM-NEXT:    [[X12:%.*]] = zext i8 [[I12]] to i16
+; SLM-NEXT:    [[X13:%.*]] = zext i8 [[I13]] to i16
+; SLM-NEXT:    [[X14:%.*]] = zext i8 [[I14]] to i16
+; SLM-NEXT:    [[X15:%.*]] = zext i8 [[I15]] to i16
+; SLM-NEXT:    [[V0:%.*]] = insertelement <16 x i16> undef, i16 [[X0]], i32 0
+; SLM-NEXT:    [[V1:%.*]] = insertelement <16 x i16> [[V0]], i16 [[X1]], i32 1
+; SLM-NEXT:    [[V2:%.*]] = insertelement <16 x i16> [[V1]], i16 [[X2]], i32 2
+; SLM-NEXT:    [[V3:%.*]] = insertelement <16 x i16> [[V2]], i16 [[X3]], i32 3
+; SLM-NEXT:    [[V4:%.*]] = insertelement <16 x i16> [[V3]], i16 [[X4]], i32 4
+; SLM-NEXT:    [[V5:%.*]] = insertelement <16 x i16> [[V4]], i16 [[X5]], i32 5
+; SLM-NEXT:    [[V6:%.*]] = insertelement <16 x i16> [[V5]], i16 [[X6]], i32 6
+; SLM-NEXT:    [[V7:%.*]] = insertelement <16 x i16> [[V6]], i16 [[X7]], i32 7
+; SLM-NEXT:    [[V8:%.*]] = insertelement <16 x i16> [[V7]], i16 [[X8]], i32 8
+; SLM-NEXT:    [[V9:%.*]] = insertelement <16 x i16> [[V8]], i16 [[X9]], i32 9
+; SLM-NEXT:    [[V10:%.*]] = insertelement <16 x i16> [[V9]], i16 [[X10]], i32 10
+; SLM-NEXT:    [[V11:%.*]] = insertelement <16 x i16> [[V10]], i16 [[X11]], i32 11
+; SLM-NEXT:    [[V12:%.*]] = insertelement <16 x i16> [[V11]], i16 [[X12]], i32 12
+; SLM-NEXT:    [[V13:%.*]] = insertelement <16 x i16> [[V12]], i16 [[X13]], i32 13
+; SLM-NEXT:    [[V14:%.*]] = insertelement <16 x i16> [[V13]], i16 [[X14]], i32 14
+; SLM-NEXT:    [[V15:%.*]] = insertelement <16 x i16> [[V14]], i16 [[X15]], i32 15
+; SLM-NEXT:    ret <16 x i16> [[V15]]
+;
+; AVX-LABEL: @loadext_16i8_to_16i16(
+; AVX-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; AVX-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; AVX-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; AVX-NEXT:    [[P4:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 4
+; AVX-NEXT:    [[P5:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 5
+; AVX-NEXT:    [[P6:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 6
+; AVX-NEXT:    [[P7:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 7
+; AVX-NEXT:    [[P8:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 8
+; AVX-NEXT:    [[P9:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 9
+; AVX-NEXT:    [[P10:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 10
+; AVX-NEXT:    [[P11:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 11
+; AVX-NEXT:    [[P12:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 12
+; AVX-NEXT:    [[P13:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 13
+; AVX-NEXT:    [[P14:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 14
+; AVX-NEXT:    [[P15:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 15
+; AVX-NEXT:    [[TMP1:%.*]] = bitcast i8* [[P0]] to <16 x i8>*
+; AVX-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* [[TMP1]], align 1
+; AVX-NEXT:    [[TMP3:%.*]] = zext <16 x i8> [[TMP2]] to <16 x i16>
+; AVX-NEXT:    [[TMP4:%.*]] = extractelement <16 x i16> [[TMP3]], i32 0
+; AVX-NEXT:    [[V0:%.*]] = insertelement <16 x i16> undef, i16 [[TMP4]], i32 0
+; AVX-NEXT:    [[TMP5:%.*]] = extractelement <16 x i16> [[TMP3]], i32 1
+; AVX-NEXT:    [[V1:%.*]] = insertelement <16 x i16> [[V0]], i16 [[TMP5]], i32 1
+; AVX-NEXT:    [[TMP6:%.*]] = extractelement <16 x i16> [[TMP3]], i32 2
+; AVX-NEXT:    [[V2:%.*]] = insertelement <16 x i16> [[V1]], i16 [[TMP6]], i32 2
+; AVX-NEXT:    [[TMP7:%.*]] = extractelement <16 x i16> [[TMP3]], i32 3
+; AVX-NEXT:    [[V3:%.*]] = insertelement <16 x i16> [[V2]], i16 [[TMP7]], i32 3
+; AVX-NEXT:    [[TMP8:%.*]] = extractelement <16 x i16> [[TMP3]], i32 4
+; AVX-NEXT:    [[V4:%.*]] = insertelement <16 x i16> [[V3]], i16 [[TMP8]], i32 4
+; AVX-NEXT:    [[TMP9:%.*]] = extractelement <16 x i16> [[TMP3]], i32 5
+; AVX-NEXT:    [[V5:%.*]] = insertelement <16 x i16> [[V4]], i16 [[TMP9]], i32 5
+; AVX-NEXT:    [[TMP10:%.*]] = extractelement <16 x i16> [[TMP3]], i32 6
+; AVX-NEXT:    [[V6:%.*]] = insertelement <16 x i16> [[V5]], i16 [[TMP10]], i32 6
+; AVX-NEXT:    [[TMP11:%.*]] = extractelement <16 x i16> [[TMP3]], i32 7
+; AVX-NEXT:    [[V7:%.*]] = insertelement <16 x i16> [[V6]], i16 [[TMP11]], i32 7
+; AVX-NEXT:    [[TMP12:%.*]] = extractelement <16 x i16> [[TMP3]], i32 8
+; AVX-NEXT:    [[V8:%.*]] = insertelement <16 x i16> [[V7]], i16 [[TMP12]], i32 8
+; AVX-NEXT:    [[TMP13:%.*]] = extractelement <16 x i16> [[TMP3]], i32 9
+; AVX-NEXT:    [[V9:%.*]] = insertelement <16 x i16> [[V8]], i16 [[TMP13]], i32 9
+; AVX-NEXT:    [[TMP14:%.*]] = extractelement <16 x i16> [[TMP3]], i32 10
+; AVX-NEXT:    [[V10:%.*]] = insertelement <16 x i16> [[V9]], i16 [[TMP14]], i32 10
+; AVX-NEXT:    [[TMP15:%.*]] = extractelement <16 x i16> [[TMP3]], i32 11
+; AVX-NEXT:    [[V11:%.*]] = insertelement <16 x i16> [[V10]], i16 [[TMP15]], i32 11
+; AVX-NEXT:    [[TMP16:%.*]] = extractelement <16 x i16> [[TMP3]], i32 12
+; AVX-NEXT:    [[V12:%.*]] = insertelement <16 x i16> [[V11]], i16 [[TMP16]], i32 12
+; AVX-NEXT:    [[TMP17:%.*]] = extractelement <16 x i16> [[TMP3]], i32 13
+; AVX-NEXT:    [[V13:%.*]] = insertelement <16 x i16> [[V12]], i16 [[TMP17]], i32 13
+; AVX-NEXT:    [[TMP18:%.*]] = extractelement <16 x i16> [[TMP3]], i32 14
+; AVX-NEXT:    [[V14:%.*]] = insertelement <16 x i16> [[V13]], i16 [[TMP18]], i32 14
+; AVX-NEXT:    [[TMP19:%.*]] = extractelement <16 x i16> [[TMP3]], i32 15
+; AVX-NEXT:    [[V15:%.*]] = insertelement <16 x i16> [[V14]], i16 [[TMP19]], i32 15
+; AVX-NEXT:    ret <16 x i16> [[V15]]
 ;
   %p1  = getelementptr inbounds i8, i8* %p0, i64 1
   %p2  = getelementptr inbounds i8, i8* %p0, i64 2
@@ -415,26 +701,15 @@ define <16 x i16> @loadext_16i8_to_16i16(i8* %p0) {
 ;
 
 define <2 x i64> @loadext_2i16_to_2i64(i16* %p0) {
-; SSE2-LABEL: @loadext_2i16_to_2i64(
-; SSE2-NEXT:    [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
-; SSE2-NEXT:    [[I0:%.*]] = load i16, i16* [[P0]], align 1
-; SSE2-NEXT:    [[I1:%.*]] = load i16, i16* [[P1]], align 1
-; SSE2-NEXT:    [[X0:%.*]] = zext i16 [[I0]] to i64
-; SSE2-NEXT:    [[X1:%.*]] = zext i16 [[I1]] to i64
-; SSE2-NEXT:    [[V0:%.*]] = insertelement <2 x i64> undef, i64 [[X0]], i32 0
-; SSE2-NEXT:    [[V1:%.*]] = insertelement <2 x i64> [[V0]], i64 [[X1]], i32 1
-; SSE2-NEXT:    ret <2 x i64> [[V1]]
-;
-; SLM-LABEL: @loadext_2i16_to_2i64(
-; SLM-NEXT:    [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
-; SLM-NEXT:    [[TMP1:%.*]] = bitcast i16* [[P0]] to <2 x i16>*
-; SLM-NEXT:    [[TMP2:%.*]] = load <2 x i16>, <2 x i16>* [[TMP1]], align 1
-; SLM-NEXT:    [[TMP3:%.*]] = zext <2 x i16> [[TMP2]] to <2 x i64>
-; SLM-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i32 0
-; SLM-NEXT:    [[V0:%.*]] = insertelement <2 x i64> undef, i64 [[TMP4]], i32 0
-; SLM-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[TMP3]], i32 1
-; SLM-NEXT:    [[V1:%.*]] = insertelement <2 x i64> [[V0]], i64 [[TMP5]], i32 1
-; SLM-NEXT:    ret <2 x i64> [[V1]]
+; SSE-LABEL: @loadext_2i16_to_2i64(
+; SSE-NEXT:    [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
+; SSE-NEXT:    [[I0:%.*]] = load i16, i16* [[P0]], align 1
+; SSE-NEXT:    [[I1:%.*]] = load i16, i16* [[P1]], align 1
+; SSE-NEXT:    [[X0:%.*]] = zext i16 [[I0]] to i64
+; SSE-NEXT:    [[X1:%.*]] = zext i16 [[I1]] to i64
+; SSE-NEXT:    [[V0:%.*]] = insertelement <2 x i64> undef, i64 [[X0]], i32 0
+; SSE-NEXT:    [[V1:%.*]] = insertelement <2 x i64> [[V0]], i64 [[X1]], i32 1
+; SSE-NEXT:    ret <2 x i64> [[V1]]
 ;
 ; AVX-LABEL: @loadext_2i16_to_2i64(
 ; AVX-NEXT:    [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
@@ -458,22 +733,57 @@ define <2 x i64> @loadext_2i16_to_2i64(i16* %p0) {
 }
 
 define <4 x i32> @loadext_4i16_to_4i32(i16* %p0) {
-; CHECK-LABEL: @loadext_4i16_to_4i32(
-; CHECK-NEXT:    [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
-; CHECK-NEXT:    [[P2:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 2
-; CHECK-NEXT:    [[P3:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 3
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i16* [[P0]] to <4 x i16>*
-; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i16>, <4 x i16>* [[TMP1]], align 1
-; CHECK-NEXT:    [[TMP3:%.*]] = zext <4 x i16> [[TMP2]] to <4 x i32>
-; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x i32> [[TMP3]], i32 0
-; CHECK-NEXT:    [[V0:%.*]] = insertelement <4 x i32> undef, i32 [[TMP4]], i32 0
-; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x i32> [[TMP3]], i32 1
-; CHECK-NEXT:    [[V1:%.*]] = insertelement <4 x i32> [[V0]], i32 [[TMP5]], i32 1
-; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x i32> [[TMP3]], i32 2
-; CHECK-NEXT:    [[V2:%.*]] = insertelement <4 x i32> [[V1]], i32 [[TMP6]], i32 2
-; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x i32> [[TMP3]], i32 3
-; CHECK-NEXT:    [[V3:%.*]] = insertelement <4 x i32> [[V2]], i32 [[TMP7]], i32 3
-; CHECK-NEXT:    ret <4 x i32> [[V3]]
+; SSE2-LABEL: @loadext_4i16_to_4i32(
+; SSE2-NEXT:    [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
+; SSE2-NEXT:    [[P2:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 2
+; SSE2-NEXT:    [[P3:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 3
+; SSE2-NEXT:    [[TMP1:%.*]] = bitcast i16* [[P0]] to <4 x i16>*
+; SSE2-NEXT:    [[TMP2:%.*]] = load <4 x i16>, <4 x i16>* [[TMP1]], align 1
+; SSE2-NEXT:    [[TMP3:%.*]] = zext <4 x i16> [[TMP2]] to <4 x i32>
+; SSE2-NEXT:    [[TMP4:%.*]] = extractelement <4 x i32> [[TMP3]], i32 0
+; SSE2-NEXT:    [[V0:%.*]] = insertelement <4 x i32> undef, i32 [[TMP4]], i32 0
+; SSE2-NEXT:    [[TMP5:%.*]] = extractelement <4 x i32> [[TMP3]], i32 1
+; SSE2-NEXT:    [[V1:%.*]] = insertelement <4 x i32> [[V0]], i32 [[TMP5]], i32 1
+; SSE2-NEXT:    [[TMP6:%.*]] = extractelement <4 x i32> [[TMP3]], i32 2
+; SSE2-NEXT:    [[V2:%.*]] = insertelement <4 x i32> [[V1]], i32 [[TMP6]], i32 2
+; SSE2-NEXT:    [[TMP7:%.*]] = extractelement <4 x i32> [[TMP3]], i32 3
+; SSE2-NEXT:    [[V3:%.*]] = insertelement <4 x i32> [[V2]], i32 [[TMP7]], i32 3
+; SSE2-NEXT:    ret <4 x i32> [[V3]]
+;
+; SLM-LABEL: @loadext_4i16_to_4i32(
+; SLM-NEXT:    [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
+; SLM-NEXT:    [[P2:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 2
+; SLM-NEXT:    [[P3:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 3
+; SLM-NEXT:    [[I0:%.*]] = load i16, i16* [[P0]], align 1
+; SLM-NEXT:    [[I1:%.*]] = load i16, i16* [[P1]], align 1
+; SLM-NEXT:    [[I2:%.*]] = load i16, i16* [[P2]], align 1
+; SLM-NEXT:    [[I3:%.*]] = load i16, i16* [[P3]], align 1
+; SLM-NEXT:    [[X0:%.*]] = zext i16 [[I0]] to i32
+; SLM-NEXT:    [[X1:%.*]] = zext i16 [[I1]] to i32
+; SLM-NEXT:    [[X2:%.*]] = zext i16 [[I2]] to i32
+; SLM-NEXT:    [[X3:%.*]] = zext i16 [[I3]] to i32
+; SLM-NEXT:    [[V0:%.*]] = insertelement <4 x i32> undef, i32 [[X0]], i32 0
+; SLM-NEXT:    [[V1:%.*]] = insertelement <4 x i32> [[V0]], i32 [[X1]], i32 1
+; SLM-NEXT:    [[V2:%.*]] = insertelement <4 x i32> [[V1]], i32 [[X2]], i32 2
+; SLM-NEXT:    [[V3:%.*]] = insertelement <4 x i32> [[V2]], i32 [[X3]], i32 3
+; SLM-NEXT:    ret <4 x i32> [[V3]]
+;
+; AVX-LABEL: @loadext_4i16_to_4i32(
+; AVX-NEXT:    [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
+; AVX-NEXT:    [[P2:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 2
+; AVX-NEXT:    [[P3:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 3
+; AVX-NEXT:    [[TMP1:%.*]] = bitcast i16* [[P0]] to <4 x i16>*
+; AVX-NEXT:    [[TMP2:%.*]] = load <4 x i16>, <4 x i16>* [[TMP1]], align 1
+; AVX-NEXT:    [[TMP3:%.*]] = zext <4 x i16> [[TMP2]] to <4 x i32>
+; AVX-NEXT:    [[TMP4:%.*]] = extractelement <4 x i32> [[TMP3]], i32 0
+; AVX-NEXT:    [[V0:%.*]] = insertelement <4 x i32> undef, i32 [[TMP4]], i32 0
+; AVX-NEXT:    [[TMP5:%.*]] = extractelement <4 x i32> [[TMP3]], i32 1
+; AVX-NEXT:    [[V1:%.*]] = insertelement <4 x i32> [[V0]], i32 [[TMP5]], i32 1
+; AVX-NEXT:    [[TMP6:%.*]] = extractelement <4 x i32> [[TMP3]], i32 2
+; AVX-NEXT:    [[V2:%.*]] = insertelement <4 x i32> [[V1]], i32 [[TMP6]], i32 2
+; AVX-NEXT:    [[TMP7:%.*]] = extractelement <4 x i32> [[TMP3]], i32 3
+; AVX-NEXT:    [[V3:%.*]] = insertelement <4 x i32> [[V2]], i32 [[TMP7]], i32 3
+; AVX-NEXT:    ret <4 x i32> [[V3]]
 ;
   %p1 = getelementptr inbounds i16, i16* %p0, i64 1
   %p2 = getelementptr inbounds i16, i16* %p0, i64 2
@@ -494,59 +804,76 @@ define <4 x i32> @loadext_4i16_to_4i32(i16* %p0) {
 }
 
 define <4 x i64> @loadext_4i16_to_4i64(i16* %p0) {
-; SSE2-LABEL: @loadext_4i16_to_4i64(
-; SSE2-NEXT:    [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
-; SSE2-NEXT:    [[P2:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 2
-; SSE2-NEXT:    [[P3:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 3
-; SSE2-NEXT:    [[I0:%.*]] = load i16, i16* [[P0]], align 1
-; SSE2-NEXT:    [[I1:%.*]] = load i16, i16* [[P1]], align 1
-; SSE2-NEXT:    [[I2:%.*]] = load i16, i16* [[P2]], align 1
-; SSE2-NEXT:    [[I3:%.*]] = load i16, i16* [[P3]], align 1
-; SSE2-NEXT:    [[X0:%.*]] = zext i16 [[I0]] to i64
-; SSE2-NEXT:    [[X1:%.*]] = zext i16 [[I1]] to i64
-; SSE2-NEXT:    [[X2:%.*]] = zext i16 [[I2]] to i64
-; SSE2-NEXT:    [[X3:%.*]] = zext i16 [[I3]] to i64
-; SSE2-NEXT:    [[V0:%.*]] = insertelement <4 x i64> undef, i64 [[X0]], i32 0
-; SSE2-NEXT:    [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[X1]], i32 1
-; SSE2-NEXT:    [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[X2]], i32 2
-; SSE2-NEXT:    [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[X3]], i32 3
-; SSE2-NEXT:    ret <4 x i64> [[V3]]
-;
-; SLM-LABEL: @loadext_4i16_to_4i64(
-; SLM-NEXT:    [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
-; SLM-NEXT:    [[P2:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 2
-; SLM-NEXT:    [[P3:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 3
-; SLM-NEXT:    [[TMP1:%.*]] = bitcast i16* [[P0]] to <4 x i16>*
-; SLM-NEXT:    [[TMP2:%.*]] = load <4 x i16>, <4 x i16>* [[TMP1]], align 1
-; SLM-NEXT:    [[TMP3:%.*]] = zext <4 x i16> [[TMP2]] to <4 x i64>
-; SLM-NEXT:    [[TMP4:%.*]] = extractelement <4 x i64> [[TMP3]], i32 0
-; SLM-NEXT:    [[V0:%.*]] = insertelement <4 x i64> undef, i64 [[TMP4]], i32 0
-; SLM-NEXT:    [[TMP5:%.*]] = extractelement <4 x i64> [[TMP3]], i32 1
-; SLM-NEXT:    [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[TMP5]], i32 1
-; SLM-NEXT:    [[TMP6:%.*]] = extractelement <4 x i64> [[TMP3]], i32 2
-; SLM-NEXT:    [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[TMP6]], i32 2
-; SLM-NEXT:    [[TMP7:%.*]] = extractelement <4 x i64> [[TMP3]], i32 3
-; SLM-NEXT:    [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[TMP7]], i32 3
-; SLM-NEXT:    ret <4 x i64> [[V3]]
-;
-; AVX-LABEL: @loadext_4i16_to_4i64(
-; AVX-NEXT:    [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
-; AVX-NEXT:    [[P2:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 2
-; AVX-NEXT:    [[P3:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 3
-; AVX-NEXT:    [[TMP1:%.*]] = bitcast i16* [[P0]] to <2 x i16>*
-; AVX-NEXT:    [[TMP2:%.*]] = load <2 x i16>, <2 x i16>* [[TMP1]], align 1
-; AVX-NEXT:    [[I2:%.*]] = load i16, i16* [[P2]], align 1
-; AVX-NEXT:    [[I3:%.*]] = load i16, i16* [[P3]], align 1
-; AVX-NEXT:    [[TMP3:%.*]] = zext <2 x i16> [[TMP2]] to <2 x i64>
-; AVX-NEXT:    [[X2:%.*]] = zext i16 [[I2]] to i64
-; AVX-NEXT:    [[X3:%.*]] = zext i16 [[I3]] to i64
-; AVX-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i32 0
-; AVX-NEXT:    [[V0:%.*]] = insertelement <4 x i64> undef, i64 [[TMP4]], i32 0
-; AVX-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[TMP3]], i32 1
-; AVX-NEXT:    [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[TMP5]], i32 1
-; AVX-NEXT:    [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[X2]], i32 2
-; AVX-NEXT:    [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[X3]], i32 3
-; AVX-NEXT:    ret <4 x i64> [[V3]]
+; SSE-LABEL: @loadext_4i16_to_4i64(
+; SSE-NEXT:    [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
+; SSE-NEXT:    [[P2:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 2
+; SSE-NEXT:    [[P3:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 3
+; SSE-NEXT:    [[I0:%.*]] = load i16, i16* [[P0]], align 1
+; SSE-NEXT:    [[I1:%.*]] = load i16, i16* [[P1]], align 1
+; SSE-NEXT:    [[I2:%.*]] = load i16, i16* [[P2]], align 1
+; SSE-NEXT:    [[I3:%.*]] = load i16, i16* [[P3]], align 1
+; SSE-NEXT:    [[X0:%.*]] = zext i16 [[I0]] to i64
+; SSE-NEXT:    [[X1:%.*]] = zext i16 [[I1]] to i64
+; SSE-NEXT:    [[X2:%.*]] = zext i16 [[I2]] to i64
+; SSE-NEXT:    [[X3:%.*]] = zext i16 [[I3]] to i64
+; SSE-NEXT:    [[V0:%.*]] = insertelement <4 x i64> undef, i64 [[X0]], i32 0
+; SSE-NEXT:    [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[X1]], i32 1
+; SSE-NEXT:    [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[X2]], i32 2
+; SSE-NEXT:    [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[X3]], i32 3
+; SSE-NEXT:    ret <4 x i64> [[V3]]
+;
+; AVX1-LABEL: @loadext_4i16_to_4i64(
+; AVX1-NEXT:    [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
+; AVX1-NEXT:    [[P2:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 2
+; AVX1-NEXT:    [[P3:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 3
+; AVX1-NEXT:    [[TMP1:%.*]] = bitcast i16* [[P0]] to <2 x i16>*
+; AVX1-NEXT:    [[TMP2:%.*]] = load <2 x i16>, <2 x i16>* [[TMP1]], align 1
+; AVX1-NEXT:    [[I2:%.*]] = load i16, i16* [[P2]], align 1
+; AVX1-NEXT:    [[I3:%.*]] = load i16, i16* [[P3]], align 1
+; AVX1-NEXT:    [[TMP3:%.*]] = zext <2 x i16> [[TMP2]] to <2 x i64>
+; AVX1-NEXT:    [[X2:%.*]] = zext i16 [[I2]] to i64
+; AVX1-NEXT:    [[X3:%.*]] = zext i16 [[I3]] to i64
+; AVX1-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i32 0
+; AVX1-NEXT:    [[V0:%.*]] = insertelement <4 x i64> undef, i64 [[TMP4]], i32 0
+; AVX1-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[TMP3]], i32 1
+; AVX1-NEXT:    [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[TMP5]], i32 1
+; AVX1-NEXT:    [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[X2]], i32 2
+; AVX1-NEXT:    [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[X3]], i32 3
+; AVX1-NEXT:    ret <4 x i64> [[V3]]
+;
+; AVX2-LABEL: @loadext_4i16_to_4i64(
+; AVX2-NEXT:    [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
+; AVX2-NEXT:    [[P2:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 2
+; AVX2-NEXT:    [[P3:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 3
+; AVX2-NEXT:    [[TMP1:%.*]] = bitcast i16* [[P0]] to <4 x i16>*
+; AVX2-NEXT:    [[TMP2:%.*]] = load <4 x i16>, <4 x i16>* [[TMP1]], align 1
+; AVX2-NEXT:    [[TMP3:%.*]] = zext <4 x i16> [[TMP2]] to <4 x i64>
+; AVX2-NEXT:    [[TMP4:%.*]] = extractelement <4 x i64> [[TMP3]], i32 0
+; AVX2-NEXT:    [[V0:%.*]] = insertelement <4 x i64> undef, i64 [[TMP4]], i32 0
+; AVX2-NEXT:    [[TMP5:%.*]] = extractelement <4 x i64> [[TMP3]], i32 1
+; AVX2-NEXT:    [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[TMP5]], i32 1
+; AVX2-NEXT:    [[TMP6:%.*]] = extractelement <4 x i64> [[TMP3]], i32 2
+; AVX2-NEXT:    [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[TMP6]], i32 2
+; AVX2-NEXT:    [[TMP7:%.*]] = extractelement <4 x i64> [[TMP3]], i32 3
+; AVX2-NEXT:    [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[TMP7]], i32 3
+; AVX2-NEXT:    ret <4 x i64> [[V3]]
+;
+; AVX512-LABEL: @loadext_4i16_to_4i64(
+; AVX512-NEXT:    [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
+; AVX512-NEXT:    [[P2:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 2
+; AVX512-NEXT:    [[P3:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 3
+; AVX512-NEXT:    [[TMP1:%.*]] = bitcast i16* [[P0]] to <4 x i16>*
+; AVX512-NEXT:    [[TMP2:%.*]] = load <4 x i16>, <4 x i16>* [[TMP1]], align 1
+; AVX512-NEXT:    [[TMP3:%.*]] = zext <4 x i16> [[TMP2]] to <4 x i64>
+; AVX512-NEXT:    [[TMP4:%.*]] = extractelement <4 x i64> [[TMP3]], i32 0
+; AVX512-NEXT:    [[V0:%.*]] = insertelement <4 x i64> undef, i64 [[TMP4]], i32 0
+; AVX512-NEXT:    [[TMP5:%.*]] = extractelement <4 x i64> [[TMP3]], i32 1
+; AVX512-NEXT:    [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[TMP5]], i32 1
+; AVX512-NEXT:    [[TMP6:%.*]] = extractelement <4 x i64> [[TMP3]], i32 2
+; AVX512-NEXT:    [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[TMP6]], i32 2
+; AVX512-NEXT:    [[TMP7:%.*]] = extractelement <4 x i64> [[TMP3]], i32 3
+; AVX512-NEXT:    [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[TMP7]], i32 3
+; AVX512-NEXT:    ret <4 x i64> [[V3]]
 ;
   %p1 = getelementptr inbounds i16, i16* %p0, i64 1
   %p2 = getelementptr inbounds i16, i16* %p0, i64 2
@@ -567,34 +894,97 @@ define <4 x i64> @loadext_4i16_to_4i64(i16* %p0) {
 }
 
 define <8 x i32> @loadext_8i16_to_8i32(i16* %p0) {
-; CHECK-LABEL: @loadext_8i16_to_8i32(
-; CHECK-NEXT:    [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
-; CHECK-NEXT:    [[P2:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 2
-; CHECK-NEXT:    [[P3:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 3
-; CHECK-NEXT:    [[P4:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 4
-; CHECK-NEXT:    [[P5:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 5
-; CHECK-NEXT:    [[P6:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 6
-; CHECK-NEXT:    [[P7:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 7
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i16* [[P0]] to <8 x i16>*
-; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, <8 x i16>* [[TMP1]], align 1
-; CHECK-NEXT:    [[TMP3:%.*]] = zext <8 x i16> [[TMP2]] to <8 x i32>
-; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <8 x i32> [[TMP3]], i32 0
-; CHECK-NEXT:    [[V0:%.*]] = insertelement <8 x i32> undef, i32 [[TMP4]], i32 0
-; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <8 x i32> [[TMP3]], i32 1
-; CHECK-NEXT:    [[V1:%.*]] = insertelement <8 x i32> [[V0]], i32 [[TMP5]], i32 1
-; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <8 x i32> [[TMP3]], i32 2
-; CHECK-NEXT:    [[V2:%.*]] = insertelement <8 x i32> [[V1]], i32 [[TMP6]], i32 2
-; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <8 x i32> [[TMP3]], i32 3
-; CHECK-NEXT:    [[V3:%.*]] = insertelement <8 x i32> [[V2]], i32 [[TMP7]], i32 3
-; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <8 x i32> [[TMP3]], i32 4
-; CHECK-NEXT:    [[V4:%.*]] = insertelement <8 x i32> [[V3]], i32 [[TMP8]], i32 4
-; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <8 x i32> [[TMP3]], i32 5
-; CHECK-NEXT:    [[V5:%.*]] = insertelement <8 x i32> [[V4]], i32 [[TMP9]], i32 5
-; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <8 x i32> [[TMP3]], i32 6
-; CHECK-NEXT:    [[V6:%.*]] = insertelement <8 x i32> [[V5]], i32 [[TMP10]], i32 6
-; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <8 x i32> [[TMP3]], i32 7
-; CHECK-NEXT:    [[V7:%.*]] = insertelement <8 x i32> [[V6]], i32 [[TMP11]], i32 7
-; CHECK-NEXT:    ret <8 x i32> [[V7]]
+; SSE2-LABEL: @loadext_8i16_to_8i32(
+; SSE2-NEXT:    [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
+; SSE2-NEXT:    [[P2:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 2
+; SSE2-NEXT:    [[P3:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 3
+; SSE2-NEXT:    [[P4:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 4
+; SSE2-NEXT:    [[P5:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 5
+; SSE2-NEXT:    [[P6:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 6
+; SSE2-NEXT:    [[P7:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 7
+; SSE2-NEXT:    [[TMP1:%.*]] = bitcast i16* [[P0]] to <8 x i16>*
+; SSE2-NEXT:    [[TMP2:%.*]] = load <8 x i16>, <8 x i16>* [[TMP1]], align 1
+; SSE2-NEXT:    [[TMP3:%.*]] = zext <8 x i16> [[TMP2]] to <8 x i32>
+; SSE2-NEXT:    [[TMP4:%.*]] = extractelement <8 x i32> [[TMP3]], i32 0
+; SSE2-NEXT:    [[V0:%.*]] = insertelement <8 x i32> undef, i32 [[TMP4]], i32 0
+; SSE2-NEXT:    [[TMP5:%.*]] = extractelement <8 x i32> [[TMP3]], i32 1
+; SSE2-NEXT:    [[V1:%.*]] = insertelement <8 x i32> [[V0]], i32 [[TMP5]], i32 1
+; SSE2-NEXT:    [[TMP6:%.*]] = extractelement <8 x i32> [[TMP3]], i32 2
+; SSE2-NEXT:    [[V2:%.*]] = insertelement <8 x i32> [[V1]], i32 [[TMP6]], i32 2
+; SSE2-NEXT:    [[TMP7:%.*]] = extractelement <8 x i32> [[TMP3]], i32 3
+; SSE2-NEXT:    [[V3:%.*]] = insertelement <8 x i32> [[V2]], i32 [[TMP7]], i32 3
+; SSE2-NEXT:    [[TMP8:%.*]] = extractelement <8 x i32> [[TMP3]], i32 4
+; SSE2-NEXT:    [[V4:%.*]] = insertelement <8 x i32> [[V3]], i32 [[TMP8]], i32 4
+; SSE2-NEXT:    [[TMP9:%.*]] = extractelement <8 x i32> [[TMP3]], i32 5
+; SSE2-NEXT:    [[V5:%.*]] = insertelement <8 x i32> [[V4]], i32 [[TMP9]], i32 5
+; SSE2-NEXT:    [[TMP10:%.*]] = extractelement <8 x i32> [[TMP3]], i32 6
+; SSE2-NEXT:    [[V6:%.*]] = insertelement <8 x i32> [[V5]], i32 [[TMP10]], i32 6
+; SSE2-NEXT:    [[TMP11:%.*]] = extractelement <8 x i32> [[TMP3]], i32 7
+; SSE2-NEXT:    [[V7:%.*]] = insertelement <8 x i32> [[V6]], i32 [[TMP11]], i32 7
+; SSE2-NEXT:    ret <8 x i32> [[V7]]
+;
+; SLM-LABEL: @loadext_8i16_to_8i32(
+; SLM-NEXT:    [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
+; SLM-NEXT:    [[P2:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 2
+; SLM-NEXT:    [[P3:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 3
+; SLM-NEXT:    [[P4:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 4
+; SLM-NEXT:    [[P5:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 5
+; SLM-NEXT:    [[P6:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 6
+; SLM-NEXT:    [[P7:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 7
+; SLM-NEXT:    [[I0:%.*]] = load i16, i16* [[P0]], align 1
+; SLM-NEXT:    [[I1:%.*]] = load i16, i16* [[P1]], align 1
+; SLM-NEXT:    [[I2:%.*]] = load i16, i16* [[P2]], align 1
+; SLM-NEXT:    [[I3:%.*]] = load i16, i16* [[P3]], align 1
+; SLM-NEXT:    [[I4:%.*]] = load i16, i16* [[P4]], align 1
+; SLM-NEXT:    [[I5:%.*]] = load i16, i16* [[P5]], align 1
+; SLM-NEXT:    [[I6:%.*]] = load i16, i16* [[P6]], align 1
+; SLM-NEXT:    [[I7:%.*]] = load i16, i16* [[P7]], align 1
+; SLM-NEXT:    [[X0:%.*]] = zext i16 [[I0]] to i32
+; SLM-NEXT:    [[X1:%.*]] = zext i16 [[I1]] to i32
+; SLM-NEXT:    [[X2:%.*]] = zext i16 [[I2]] to i32
+; SLM-NEXT:    [[X3:%.*]] = zext i16 [[I3]] to i32
+; SLM-NEXT:    [[X4:%.*]] = zext i16 [[I4]] to i32
+; SLM-NEXT:    [[X5:%.*]] = zext i16 [[I5]] to i32
+; SLM-NEXT:    [[X6:%.*]] = zext i16 [[I6]] to i32
+; SLM-NEXT:    [[X7:%.*]] = zext i16 [[I7]] to i32
+; SLM-NEXT:    [[V0:%.*]] = insertelement <8 x i32> undef, i32 [[X0]], i32 0
+; SLM-NEXT:    [[V1:%.*]] = insertelement <8 x i32> [[V0]], i32 [[X1]], i32 1
+; SLM-NEXT:    [[V2:%.*]] = insertelement <8 x i32> [[V1]], i32 [[X2]], i32 2
+; SLM-NEXT:    [[V3:%.*]] = insertelement <8 x i32> [[V2]], i32 [[X3]], i32 3
+; SLM-NEXT:    [[V4:%.*]] = insertelement <8 x i32> [[V3]], i32 [[X4]], i32 4
+; SLM-NEXT:    [[V5:%.*]] = insertelement <8 x i32> [[V4]], i32 [[X5]], i32 5
+; SLM-NEXT:    [[V6:%.*]] = insertelement <8 x i32> [[V5]], i32 [[X6]], i32 6
+; SLM-NEXT:    [[V7:%.*]] = insertelement <8 x i32> [[V6]], i32 [[X7]], i32 7
+; SLM-NEXT:    ret <8 x i32> [[V7]]
+;
+; AVX-LABEL: @loadext_8i16_to_8i32(
+; AVX-NEXT:    [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
+; AVX-NEXT:    [[P2:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 2
+; AVX-NEXT:    [[P3:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 3
+; AVX-NEXT:    [[P4:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 4
+; AVX-NEXT:    [[P5:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 5
+; AVX-NEXT:    [[P6:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 6
+; AVX-NEXT:    [[P7:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 7
+; AVX-NEXT:    [[TMP1:%.*]] = bitcast i16* [[P0]] to <8 x i16>*
+; AVX-NEXT:    [[TMP2:%.*]] = load <8 x i16>, <8 x i16>* [[TMP1]], align 1
+; AVX-NEXT:    [[TMP3:%.*]] = zext <8 x i16> [[TMP2]] to <8 x i32>
+; AVX-NEXT:    [[TMP4:%.*]] = extractelement <8 x i32> [[TMP3]], i32 0
+; AVX-NEXT:    [[V0:%.*]] = insertelement <8 x i32> undef, i32 [[TMP4]], i32 0
+; AVX-NEXT:    [[TMP5:%.*]] = extractelement <8 x i32> [[TMP3]], i32 1
+; AVX-NEXT:    [[V1:%.*]] = insertelement <8 x i32> [[V0]], i32 [[TMP5]], i32 1
+; AVX-NEXT:    [[TMP6:%.*]] = extractelement <8 x i32> [[TMP3]], i32 2
+; AVX-NEXT:    [[V2:%.*]] = insertelement <8 x i32> [[V1]], i32 [[TMP6]], i32 2
+; AVX-NEXT:    [[TMP7:%.*]] = extractelement <8 x i32> [[TMP3]], i32 3
+; AVX-NEXT:    [[V3:%.*]] = insertelement <8 x i32> [[V2]], i32 [[TMP7]], i32 3
+; AVX-NEXT:    [[TMP8:%.*]] = extractelement <8 x i32> [[TMP3]], i32 4
+; AVX-NEXT:    [[V4:%.*]] = insertelement <8 x i32> [[V3]], i32 [[TMP8]], i32 4
+; AVX-NEXT:    [[TMP9:%.*]] = extractelement <8 x i32> [[TMP3]], i32 5
+; AVX-NEXT:    [[V5:%.*]] = insertelement <8 x i32> [[V4]], i32 [[TMP9]], i32 5
+; AVX-NEXT:    [[TMP10:%.*]] = extractelement <8 x i32> [[TMP3]], i32 6
+; AVX-NEXT:    [[V6:%.*]] = insertelement <8 x i32> [[V5]], i32 [[TMP10]], i32 6
+; AVX-NEXT:    [[TMP11:%.*]] = extractelement <8 x i32> [[TMP3]], i32 7
+; AVX-NEXT:    [[V7:%.*]] = insertelement <8 x i32> [[V6]], i32 [[TMP11]], i32 7
+; AVX-NEXT:    ret <8 x i32> [[V7]]
 ;
   %p1 = getelementptr inbounds i16, i16* %p0, i64 1
   %p2 = getelementptr inbounds i16, i16* %p0, i64 2
@@ -635,26 +1025,15 @@ define <8 x i32> @loadext_8i16_to_8i32(i16* %p0) {
 ;
 
 define <2 x i64> @loadext_2i32_to_2i64(i32* %p0) {
-; SSE2-LABEL: @loadext_2i32_to_2i64(
-; SSE2-NEXT:    [[P1:%.*]] = getelementptr inbounds i32, i32* [[P0:%.*]], i64 1
-; SSE2-NEXT:    [[I0:%.*]] = load i32, i32* [[P0]], align 1
-; SSE2-NEXT:    [[I1:%.*]] = load i32, i32* [[P1]], align 1
-; SSE2-NEXT:    [[X0:%.*]] = zext i32 [[I0]] to i64
-; SSE2-NEXT:    [[X1:%.*]] = zext i32 [[I1]] to i64
-; SSE2-NEXT:    [[V0:%.*]] = insertelement <2 x i64> undef, i64 [[X0]], i32 0
-; SSE2-NEXT:    [[V1:%.*]] = insertelement <2 x i64> [[V0]], i64 [[X1]], i32 1
-; SSE2-NEXT:    ret <2 x i64> [[V1]]
-;
-; SLM-LABEL: @loadext_2i32_to_2i64(
-; SLM-NEXT:    [[P1:%.*]] = getelementptr inbounds i32, i32* [[P0:%.*]], i64 1
-; SLM-NEXT:    [[TMP1:%.*]] = bitcast i32* [[P0]] to <2 x i32>*
-; SLM-NEXT:    [[TMP2:%.*]] = load <2 x i32>, <2 x i32>* [[TMP1]], align 1
-; SLM-NEXT:    [[TMP3:%.*]] = zext <2 x i32> [[TMP2]] to <2 x i64>
-; SLM-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i32 0
-; SLM-NEXT:    [[V0:%.*]] = insertelement <2 x i64> undef, i64 [[TMP4]], i32 0
-; SLM-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[TMP3]], i32 1
-; SLM-NEXT:    [[V1:%.*]] = insertelement <2 x i64> [[V0]], i64 [[TMP5]], i32 1
-; SLM-NEXT:    ret <2 x i64> [[V1]]
+; SSE-LABEL: @loadext_2i32_to_2i64(
+; SSE-NEXT:    [[P1:%.*]] = getelementptr inbounds i32, i32* [[P0:%.*]], i64 1
+; SSE-NEXT:    [[I0:%.*]] = load i32, i32* [[P0]], align 1
+; SSE-NEXT:    [[I1:%.*]] = load i32, i32* [[P1]], align 1
+; SSE-NEXT:    [[X0:%.*]] = zext i32 [[I0]] to i64
+; SSE-NEXT:    [[X1:%.*]] = zext i32 [[I1]] to i64
+; SSE-NEXT:    [[V0:%.*]] = insertelement <2 x i64> undef, i64 [[X0]], i32 0
+; SSE-NEXT:    [[V1:%.*]] = insertelement <2 x i64> [[V0]], i64 [[X1]], i32 1
+; SSE-NEXT:    ret <2 x i64> [[V1]]
 ;
 ; AVX-LABEL: @loadext_2i32_to_2i64(
 ; AVX-NEXT:    [[P1:%.*]] = getelementptr inbounds i32, i32* [[P0:%.*]], i64 1
@@ -678,40 +1057,23 @@ define <2 x i64> @loadext_2i32_to_2i64(i32* %p0) {
 }
 
 define <4 x i64> @loadext_4i32_to_4i64(i32* %p0) {
-; SSE2-LABEL: @loadext_4i32_to_4i64(
-; SSE2-NEXT:    [[P1:%.*]] = getelementptr inbounds i32, i32* [[P0:%.*]], i64 1
-; SSE2-NEXT:    [[P2:%.*]] = getelementptr inbounds i32, i32* [[P0]], i64 2
-; SSE2-NEXT:    [[P3:%.*]] = getelementptr inbounds i32, i32* [[P0]], i64 3
-; SSE2-NEXT:    [[I0:%.*]] = load i32, i32* [[P0]], align 1
-; SSE2-NEXT:    [[I1:%.*]] = load i32, i32* [[P1]], align 1
-; SSE2-NEXT:    [[I2:%.*]] = load i32, i32* [[P2]], align 1
-; SSE2-NEXT:    [[I3:%.*]] = load i32, i32* [[P3]], align 1
-; SSE2-NEXT:    [[X0:%.*]] = zext i32 [[I0]] to i64
-; SSE2-NEXT:    [[X1:%.*]] = zext i32 [[I1]] to i64
-; SSE2-NEXT:    [[X2:%.*]] = zext i32 [[I2]] to i64
-; SSE2-NEXT:    [[X3:%.*]] = zext i32 [[I3]] to i64
-; SSE2-NEXT:    [[V0:%.*]] = insertelement <4 x i64> undef, i64 [[X0]], i32 0
-; SSE2-NEXT:    [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[X1]], i32 1
-; SSE2-NEXT:    [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[X2]], i32 2
-; SSE2-NEXT:    [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[X3]], i32 3
-; SSE2-NEXT:    ret <4 x i64> [[V3]]
-;
-; SLM-LABEL: @loadext_4i32_to_4i64(
-; SLM-NEXT:    [[P1:%.*]] = getelementptr inbounds i32, i32* [[P0:%.*]], i64 1
-; SLM-NEXT:    [[P2:%.*]] = getelementptr inbounds i32, i32* [[P0]], i64 2
-; SLM-NEXT:    [[P3:%.*]] = getelementptr inbounds i32, i32* [[P0]], i64 3
-; SLM-NEXT:    [[TMP1:%.*]] = bitcast i32* [[P0]] to <4 x i32>*
-; SLM-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP1]], align 1
-; SLM-NEXT:    [[TMP3:%.*]] = zext <4 x i32> [[TMP2]] to <4 x i64>
-; SLM-NEXT:    [[TMP4:%.*]] = extractelement <4 x i64> [[TMP3]], i32 0
-; SLM-NEXT:    [[V0:%.*]] = insertelement <4 x i64> undef, i64 [[TMP4]], i32 0
-; SLM-NEXT:    [[TMP5:%.*]] = extractelement <4 x i64> [[TMP3]], i32 1
-; SLM-NEXT:    [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[TMP5]], i32 1
-; SLM-NEXT:    [[TMP6:%.*]] = extractelement <4 x i64> [[TMP3]], i32 2
-; SLM-NEXT:    [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[TMP6]], i32 2
-; SLM-NEXT:    [[TMP7:%.*]] = extractelement <4 x i64> [[TMP3]], i32 3
-; SLM-NEXT:    [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[TMP7]], i32 3
-; SLM-NEXT:    ret <4 x i64> [[V3]]
+; SSE-LABEL: @loadext_4i32_to_4i64(
+; SSE-NEXT:    [[P1:%.*]] = getelementptr inbounds i32, i32* [[P0:%.*]], i64 1
+; SSE-NEXT:    [[P2:%.*]] = getelementptr inbounds i32, i32* [[P0]], i64 2
+; SSE-NEXT:    [[P3:%.*]] = getelementptr inbounds i32, i32* [[P0]], i64 3
+; SSE-NEXT:    [[I0:%.*]] = load i32, i32* [[P0]], align 1
+; SSE-NEXT:    [[I1:%.*]] = load i32, i32* [[P1]], align 1
+; SSE-NEXT:    [[I2:%.*]] = load i32, i32* [[P2]], align 1
+; SSE-NEXT:    [[I3:%.*]] = load i32, i32* [[P3]], align 1
+; SSE-NEXT:    [[X0:%.*]] = zext i32 [[I0]] to i64
+; SSE-NEXT:    [[X1:%.*]] = zext i32 [[I1]] to i64
+; SSE-NEXT:    [[X2:%.*]] = zext i32 [[I2]] to i64
+; SSE-NEXT:    [[X3:%.*]] = zext i32 [[I3]] to i64
+; SSE-NEXT:    [[V0:%.*]] = insertelement <4 x i64> undef, i64 [[X0]], i32 0
+; SSE-NEXT:    [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[X1]], i32 1
+; SSE-NEXT:    [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[X2]], i32 2
+; SSE-NEXT:    [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[X3]], i32 3
+; SSE-NEXT:    ret <4 x i64> [[V3]]
 ;
 ; AVX1-LABEL: @loadext_4i32_to_4i64(
 ; AVX1-NEXT:    [[P1:%.*]] = getelementptr inbounds i32, i32* [[P0:%.*]], i64 1
diff --git a/llvm/test/Transforms/SROA/addrspacecast.ll b/llvm/test/Transforms/SROA/addrspacecast.ll
index d2a1cef81..d60f92b8e 100644
--- a/llvm/test/Transforms/SROA/addrspacecast.ll
+++ b/llvm/test/Transforms/SROA/addrspacecast.ll
@@ -282,7 +282,9 @@ define void @select_addrspacecast_const_op(i1 %a, i1 %b) {
   ret void
 }
 
-@gv = external addrspace(1) global i64
+;; If this was external, we wouldn't be able to prove dereferenceability
+;; of the location.
+@gv = addrspace(1) global i64 zeroinitializer
 
 define void @select_addrspacecast_gv(i1 %a, i1 %b) {
 ; CHECK-LABEL: @select_addrspacecast_gv(
@@ -299,10 +301,11 @@ define void @select_addrspacecast_gv(i1 %a, i1 %b) {
   ret void
 }
 
-; CHECK-LABEL: @select_addrspacecast_i8(
-; CHECK: [[SEL:%.*]] = select i1 undef, i8 undef, i8 undef
-; CHECK-NEXT: ret i8 [[SEL]]
 define i8 @select_addrspacecast_i8() {
+; CHECK-LABEL: @select_addrspacecast_i8(
+; CHECK-NEXT:    [[RET_SROA_SPECULATED:%.*]] = select i1 undef, i8 undef, i8 undef
+; CHECK-NEXT:    ret i8 [[RET_SROA_SPECULATED]]
+;
   %a = alloca i8
   %b = alloca i8
 
diff --git a/llvm/test/Transforms/SROA/alignment.ll b/llvm/test/Transforms/SROA/alignment.ll
index 81f8f2a00..674f45fb1 100644
--- a/llvm/test/Transforms/SROA/alignment.ll
+++ b/llvm/test/Transforms/SROA/alignment.ll
@@ -228,4 +228,19 @@ define void @test10() {
   ret void
 }
 
+%struct = type { i32, i32 }
+define dso_local i32 @pr45010(%struct* %A) {
+; CHECK-LABEL: @pr45010
+; CHECK: load atomic volatile i32, {{.*}}, align 4
+
+  %B = alloca %struct, align 4
+  %A.i = getelementptr inbounds %struct, %struct* %A, i32 0, i32 0
+  %B.i = getelementptr inbounds %struct, %struct* %B, i32 0, i32 0
+  %1 = load i32, i32* %A.i, align 4
+  store atomic volatile i32 %1, i32* %B.i release, align 4
+  %2 = bitcast %struct* %B to i32*
+  %x = load atomic volatile i32, i32* %2 acquire, align 4
+  ret i32 %x
+}
+
 declare void @populate(i8*)
diff --git a/llvm/test/Transforms/SROA/mem-par-metadata-sroa.ll b/llvm/test/Transforms/SROA/mem-par-metadata-sroa.ll
index 577245cda..fb38208d4 100644
--- a/llvm/test/Transforms/SROA/mem-par-metadata-sroa.ll
+++ b/llvm/test/Transforms/SROA/mem-par-metadata-sroa.ll
@@ -97,7 +97,7 @@ for.end:                                          ; preds = %for.cond
 ; Function Attrs: argmemonly nounwind
 declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture writeonly, i8* nocapture readonly, i64, i1) #1
 
-attributes #0 = { norecurse nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { norecurse nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { argmemonly nounwind }
 
 !llvm.ident = !{!0}
diff --git a/llvm/test/Transforms/SROA/tbaa-struct.ll b/llvm/test/Transforms/SROA/tbaa-struct.ll
new file mode 100644
index 000000000..fe008d8a9
--- /dev/null
+++ b/llvm/test/Transforms/SROA/tbaa-struct.ll
@@ -0,0 +1,32 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -sroa %s | FileCheck %s
+
+; SROA should keep `!tbaa.struct` metadata
+
+%vector = type { float, float }
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* writeonly, i8* readonly, i64, i1 immarg)
+declare <2 x float> @foo(%vector* %0)
+
+define void @bar(%vector* %y2) {
+; CHECK-LABEL: @bar(
+; CHECK-NEXT:    [[X14:%.*]] = call <2 x float> @foo(%vector* [[Y2:%.*]])
+; CHECK-NEXT:    [[X7_SROA_0_0_X18_SROA_CAST:%.*]] = bitcast %vector* [[Y2]] to <2 x float>*
+; CHECK-NEXT:    store <2 x float> [[X14]], <2 x float>* [[X7_SROA_0_0_X18_SROA_CAST]], align 4, !tbaa.struct !0
+; CHECK-NEXT:    ret void
+;
+  %x7 = alloca %vector
+  %x14 = call <2 x float> @foo(%vector* %y2)
+  %x15 = bitcast %vector* %x7 to <2 x float>*
+  store <2 x float> %x14, <2 x float>* %x15
+  %x19 = bitcast %vector* %x7 to i8*
+  %x18 = bitcast %vector* %y2 to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %x18, i8* align 4 %x19, i64 8, i1 false), !tbaa.struct !10
+  ret void
+}
+
+!4 = !{!"omnipotent char", !5, i64 0}
+!5 = !{!"Simple C++ TBAA"}
+!7 = !{!"vector", !8, i64 0, !8, i64 4}
+!8 = !{!"float", !4, i64 0}
+!10 = !{i64 0, i64 4, !11, i64 4, i64 4, !11}
+!11 = !{!8, !8, i64 0}
diff --git a/llvm/test/Transforms/SROA/vector-promotion-different-size.ll b/llvm/test/Transforms/SROA/vector-promotion-different-size.ll
new file mode 100644
index 000000000..56e1f1f21
--- /dev/null
+++ b/llvm/test/Transforms/SROA/vector-promotion-different-size.ll
@@ -0,0 +1,24 @@
+; RUN: opt < %s -sroa -S | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-n8:16:32:64"
+
+define <4 x i1> @vector_bitcast() {
+  ; CHECK-LABEL: @vector_bitcast
+  ; CHECK: alloca i1
+
+    %a = alloca <3 x i1>
+    store <3 x i1> <i1 1,i1 0,i1 1>, <3 x i1>* %a
+    %cast = bitcast <3 x i1>* %a to <4 x i1>*
+    %vec = load <4 x i1>, <4 x i1>* %cast
+    ret <4 x i1> %vec
+}
+
+define void @vector_bitcast_2() {
+  ; CHECK-LABEL: @vector_bitcast_2
+  ; CHECK: alloca <32 x i16>
+
+    %"sum$1.host2" = alloca <32 x i16>
+    store <32 x i16> undef, <32 x i16>* %"sum$1.host2"
+    %bc = bitcast <32 x i16>* %"sum$1.host2" to <64 x i16>*
+    %bcl = load <64 x i16>, <64 x i16>* %bc
+    ret void
+}
diff --git a/llvm/test/Transforms/SafeStack/ARM/debug.ll b/llvm/test/Transforms/SafeStack/ARM/debug.ll
index 6728c2f25..af132ae1f 100644
--- a/llvm/test/Transforms/SafeStack/ARM/debug.ll
+++ b/llvm/test/Transforms/SafeStack/ARM/debug.ll
@@ -46,11 +46,11 @@ declare void @Capture(i8*) local_unnamed_addr #4
 ; Function Attrs: argmemonly nounwind
 declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture) #2
 
-attributes #0 = { norecurse nounwind readonly safestack "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="generic" "target-features"="+armv7-a,+dsp,+neon,+vfp3,-thumb-mode" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { nounwind safestack "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="generic" "target-features"="+armv7-a,+dsp,+neon,+vfp3,-thumb-mode" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { norecurse nounwind readonly safestack "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="generic" "target-features"="+armv7-a,+dsp,+neon,+vfp3,-thumb-mode" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind safestack "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="generic" "target-features"="+armv7-a,+dsp,+neon,+vfp3,-thumb-mode" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { argmemonly nounwind }
 attributes #3 = { nounwind readnone speculatable }
-attributes #4 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="generic" "target-features"="+armv7-a,+dsp,+neon,+vfp3,-thumb-mode" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #4 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="generic" "target-features"="+armv7-a,+dsp,+neon,+vfp3,-thumb-mode" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #5 = { nounwind }
 
 !llvm.dbg.cu = !{!2}
diff --git a/llvm/test/Transforms/SafeStack/X86/debug-loc.ll b/llvm/test/Transforms/SafeStack/X86/debug-loc.ll
index b8a7c0874..ffd76d492 100644
--- a/llvm/test/Transforms/SafeStack/X86/debug-loc.ll
+++ b/llvm/test/Transforms/SafeStack/X86/debug-loc.ll
@@ -45,9 +45,9 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
 
 declare void @Capture(%struct.S*) #2
 
-attributes #0 = { safestack uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { safestack uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
-attributes #2 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!15, !16}
diff --git a/llvm/test/Transforms/SafeStack/X86/debug-loc2.ll b/llvm/test/Transforms/SafeStack/X86/debug-loc2.ll
index afa143f9c..10873f4b4 100644
--- a/llvm/test/Transforms/SafeStack/X86/debug-loc2.ll
+++ b/llvm/test/Transforms/SafeStack/X86/debug-loc2.ll
@@ -25,7 +25,11 @@ entry:
   tail call void @llvm.dbg.value(metadata i32* %x1, metadata !10, metadata !24), !dbg !16
 
 ; Supported dbg.value: rewritted based on the [[USP]] value.
-; CHECK: call void @llvm.dbg.value(metadata i8* %[[USP]], metadata ![[X1:.*]], metadata !DIExpression(DW_OP_deref, DW_OP_constu, 4, DW_OP_minus))
+; CHECK: call void @llvm.dbg.value(metadata i8* %[[USP]], metadata ![[X1:.*]], metadata !DIExpression(DW_OP_constu, 4, DW_OP_minus, DW_OP_deref, DW_OP_LLVM_fragment, 0, 4))
+  tail call void @llvm.dbg.value(metadata i32* %x1, metadata !10, metadata !25), !dbg !16
+
+; Supported dbg.value: rewritted based on the [[USP]] value.
+; CHECK: call void @llvm.dbg.value(metadata i8* %[[USP]], metadata ![[X1:.*]], metadata !DIExpression(DW_OP_constu, 4, DW_OP_minus, DW_OP_deref))
   tail call void @llvm.dbg.value(metadata i32* %x1, metadata !10, metadata !15), !dbg !16
   call void @capture(i32* nonnull %x1), !dbg !17
 
@@ -33,7 +37,7 @@ entry:
 ; CHECK: call void @llvm.random.metadata.use(metadata ![[EMPTY]])
   call void @llvm.random.metadata.use(metadata i32* %x2)
 
-; CHECK: call void @llvm.dbg.value(metadata i8* %[[USP]], metadata ![[X2:.*]], metadata !DIExpression(DW_OP_deref, DW_OP_constu, 8, DW_OP_minus))
+; CHECK: call void @llvm.dbg.value(metadata i8* %[[USP]], metadata ![[X2:.*]], metadata !DIExpression(DW_OP_constu, 8, DW_OP_minus, DW_OP_deref))
   call void @llvm.dbg.value(metadata i32* %x2, metadata !12, metadata !15), !dbg !18
   call void @capture(i32* nonnull %x2), !dbg !19
   ret void, !dbg !20
@@ -52,9 +56,9 @@ declare void @llvm.dbg.value(metadata, metadata, metadata) #3
 
 declare void @llvm.random.metadata.use(metadata)
 
-attributes #0 = { noinline safestack uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noinline safestack uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { argmemonly nounwind }
-attributes #2 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #3 = { nounwind readnone }
 attributes #4 = { nounwind }
 
@@ -94,3 +98,4 @@ attributes #4 = { nounwind }
 !22 = !DILexicalBlockFile(scope: !6, file: !1, discriminator: 1)
 !23 = !DIExpression()
 !24 = !DIExpression(DW_OP_constu, 42, DW_OP_minus)
+!25 = !DIExpression(DW_OP_deref, DW_OP_LLVM_fragment, 0, 4)
diff --git a/llvm/test/Transforms/SampleProfile/Inputs/profile-symbol-list.ll b/llvm/test/Transforms/SampleProfile/Inputs/profile-symbol-list.ll
new file mode 100644
index 000000000..c4623fbf4
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/Inputs/profile-symbol-list.ll
@@ -0,0 +1,134 @@
+; Original C++ test case
+;
+; #include <stdio.h>
+;
+; __attribute__((noinline)) int goo() { return 3 };
+; __attribute__((noinline)) int hoo() { return 4 };
+;
+; int sum(int x, int y) {
+;   return x + y;
+; }
+;
+; int main() {
+;   int s, i = 0;
+;   while (i++ < 20000 * 20000)
+;     if (i != 100) s = sum(i, s); else s = 30;
+;   printf("sum is %d\n", s);
+;   return goo() + hoo() != 7;
+; }
+;
+; Both goo and hoo don't show up in the input profile.
+; Suppose function goo shows up in the binary generating the input profile
+; and function hoo doesn't show up. Then the profile symbol list in the input
+; profile will contain goo but not hoo. Verify the entry count of goo is
+; 0 and the entry count of hoo is -1.
+; CHECK: define {{.*}} i32 @_Z3goov() {{.*}} !prof ![[IDX1:[0-9]*]]
+; CHECK: define {{.*}} i32 @_Z3hoov() {{.*}} !prof ![[IDX2:[0-9]*]]
+; CHECK: ![[IDX1]] = !{!"function_entry_count", i64 0}
+; CHECK: ![[IDX2]] = !{!"function_entry_count", i64 -1}
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@.str = private unnamed_addr constant [11 x i8] c"sum is %d\0A\00", align 1
+
+; Function Attrs: noinline norecurse nounwind readnone uwtable
+define dso_local i32 @_Z3goov() local_unnamed_addr #0 !dbg !7 {
+entry:
+  ret i32 3, !dbg !9
+}
+
+; Function Attrs: noinline norecurse nounwind readnone uwtable
+define dso_local i32 @_Z3hoov() local_unnamed_addr #0 !dbg !10 {
+entry:
+  ret i32 4, !dbg !11
+}
+
+; Function Attrs: norecurse nounwind readnone uwtable
+define dso_local i32 @_Z3sumii(i32 %x, i32 %y) local_unnamed_addr #1 !dbg !12 {
+entry:
+  %add = add nsw i32 %y, %x, !dbg !13
+  ret i32 %add, !dbg !14
+}
+
+; Function Attrs: nofree norecurse nounwind uwtable
+define dso_local i32 @main() local_unnamed_addr #2 !dbg !15 {
+entry:
+  br label %while.body, !dbg !16
+
+while.body:                                       ; preds = %while.body, %entry
+  %inc12 = phi i32 [ 1, %entry ], [ %inc.4, %while.body ]
+  %s.011 = phi i32 [ undef, %entry ], [ %spec.select.4, %while.body ]
+  %cmp1 = icmp eq i32 %inc12, 100, !dbg !18
+  %add.i = add nsw i32 %inc12, %s.011, !dbg !20
+  %spec.select = select i1 %cmp1, i32 30, i32 %add.i, !dbg !23
+  %inc = add nuw nsw i32 %inc12, 1, !dbg !24
+  %cmp1.1 = icmp eq i32 %inc, 100, !dbg !18
+  %add.i.1 = add nsw i32 %inc, %spec.select, !dbg !20
+  %spec.select.1 = select i1 %cmp1.1, i32 30, i32 %add.i.1, !dbg !23
+  %inc.1 = add nuw nsw i32 %inc12, 2, !dbg !24
+  %cmp1.2 = icmp eq i32 %inc.1, 100, !dbg !18
+  %add.i.2 = add nsw i32 %inc.1, %spec.select.1, !dbg !20
+  %spec.select.2 = select i1 %cmp1.2, i32 30, i32 %add.i.2, !dbg !23
+  %inc.2 = add nuw nsw i32 %inc12, 3, !dbg !24
+  %cmp1.3 = icmp eq i32 %inc.2, 100, !dbg !18
+  %add.i.3 = add nsw i32 %inc.2, %spec.select.2, !dbg !20
+  %spec.select.3 = select i1 %cmp1.3, i32 30, i32 %add.i.3, !dbg !23
+  %inc.3 = add nuw nsw i32 %inc12, 4, !dbg !24
+  %cmp1.4 = icmp eq i32 %inc.3, 100, !dbg !18
+  %add.i.4 = add nsw i32 %inc.3, %spec.select.3, !dbg !20
+  %spec.select.4 = select i1 %cmp1.4, i32 30, i32 %add.i.4, !dbg !23
+  %inc.4 = add nuw nsw i32 %inc12, 5, !dbg !24
+  %exitcond.4 = icmp eq i32 %inc.4, 400000001, !dbg !26
+  br i1 %exitcond.4, label %while.end, label %while.body, !dbg !27, !llvm.loop !28
+
+while.end:                                        ; preds = %while.body
+  %call2 = tail call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([11 x i8], [11 x i8]* @.str, i64 0, i64 0), i32 %spec.select.4), !dbg !31
+  ret i32 0, !dbg !32
+}
+
+; Function Attrs: nofree nounwind
+declare dso_local i32 @printf(i8* nocapture readonly, ...) local_unnamed_addr #3
+
+attributes #0 = { noinline norecurse nounwind readnone uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { norecurse nounwind readnone uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { nofree norecurse nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { nofree nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 10.0.0 (trunk 369144)", isOptimized: true, runtimeVersion: 0, emissionKind: LineTablesOnly, enums: !2, debugInfoForProfiling: true, nameTableKind: None)
+!1 = !DIFile(filename: "1.cc", directory: "/usr/local/google/home/wmi/workarea/llvm-r369144/src")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{!"clang version 10.0.0 (trunk 369144)"}
+!7 = distinct !DISubprogram(name: "goo", linkageName: "_Z3goov", scope: !1, file: !1, line: 3, type: !8, scopeLine: 3, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2)
+!8 = !DISubroutineType(types: !2)
+!9 = !DILocation(line: 3, column: 39, scope: !7)
+!10 = distinct !DISubprogram(name: "hoo", linkageName: "_Z3hoov", scope: !1, file: !1, line: 4, type: !8, scopeLine: 4, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2)
+!11 = !DILocation(line: 4, column: 39, scope: !10)
+!12 = distinct !DISubprogram(name: "sum", linkageName: "_Z3sumii", scope: !1, file: !1, line: 6, type: !8, scopeLine: 6, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2)
+!13 = !DILocation(line: 7, column: 12, scope: !12)
+!14 = !DILocation(line: 7, column: 3, scope: !12)
+!15 = distinct !DISubprogram(name: "main", scope: !1, file: !1, line: 10, type: !8, scopeLine: 10, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2)
+!16 = !DILocation(line: 12, column: 3, scope: !17)
+!17 = !DILexicalBlockFile(scope: !15, file: !1, discriminator: 2)
+!18 = !DILocation(line: 13, column: 11, scope: !19)
+!19 = !DILexicalBlockFile(scope: !15, file: !1, discriminator: 21)
+!20 = !DILocation(line: 7, column: 12, scope: !21, inlinedAt: !22)
+!21 = !DILexicalBlockFile(scope: !12, file: !1, discriminator: 21)
+!22 = distinct !DILocation(line: 13, column: 23, scope: !17)
+!23 = !DILocation(line: 13, column: 9, scope: !19)
+!24 = !DILocation(line: 12, column: 11, scope: !25)
+!25 = !DILexicalBlockFile(scope: !15, file: !1, discriminator: 1282)
+!26 = !DILocation(line: 12, column: 14, scope: !25)
+!27 = !DILocation(line: 12, column: 3, scope: !25)
+!28 = distinct !{!28, !29, !30}
+!29 = !DILocation(line: 12, column: 3, scope: !15)
+!30 = !DILocation(line: 13, column: 43, scope: !15)
+!31 = !DILocation(line: 14, column: 3, scope: !15)
+!32 = !DILocation(line: 15, column: 3, scope: !15)
diff --git a/llvm/test/Transforms/SampleProfile/branch.ll b/llvm/test/Transforms/SampleProfile/branch.ll
index d204e64ef..0976e39fa 100644
--- a/llvm/test/Transforms/SampleProfile/branch.ll
+++ b/llvm/test/Transforms/SampleProfile/branch.ll
@@ -150,10 +150,10 @@ declare i32 @atoi(i8*) #2
 
 declare i32 @printf(i8*, ...) #3
 
-attributes #0 = { uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
-attributes #2 = { nounwind readonly "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #3 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { nounwind readonly "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #4 = { nounwind readonly }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/Transforms/SampleProfile/compressed-profile-symbol-list.ll b/llvm/test/Transforms/SampleProfile/compressed-profile-symbol-list.ll
new file mode 100644
index 000000000..6ac62c170
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/compressed-profile-symbol-list.ll
@@ -0,0 +1,5 @@
+; REQUIRES: zlib
+; Append inline.prof with profile symbol list and save it after compression.
+; RUN: llvm-profdata merge --sample --prof-sym-list=%S/Inputs/profile-symbol-list.text --compress-all-sections=true --extbinary %S/Inputs/inline.prof --output=%t.profdata
+; RUN: opt < %S/Inputs/profile-symbol-list.ll -sample-profile -profile-accurate-for-symsinlist -sample-profile-file=%t.profdata -S | FileCheck %S/Inputs/profile-symbol-list.ll
+; RUN: opt < %S/Inputs/profile-symbol-list.ll -passes=sample-profile -profile-accurate-for-symsinlist -sample-profile-file=%t.profdata -S | FileCheck %S/Inputs/profile-symbol-list.ll
diff --git a/llvm/test/Transforms/SampleProfile/entry_counts_cold.ll b/llvm/test/Transforms/SampleProfile/entry_counts_cold.ll
index da60ebec6..dbc52b1ef 100644
--- a/llvm/test/Transforms/SampleProfile/entry_counts_cold.ll
+++ b/llvm/test/Transforms/SampleProfile/entry_counts_cold.ll
@@ -95,10 +95,10 @@ declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture) #2
 
 declare void @baz(...) #3
 
-attributes #0 = { nounwind ssp uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+fxsr,+mmx,+sahf,+sse,+sse2,+sse3,+sse4.1,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind ssp uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+fxsr,+mmx,+sahf,+sse,+sse2,+sse3,+sse4.1,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone speculatable }
 attributes #2 = { argmemonly nounwind }
-attributes #3 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+fxsr,+mmx,+sahf,+sse,+sse2,+sse3,+sse4.1,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+fxsr,+mmx,+sahf,+sse,+sse2,+sse3,+sse4.1,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #4 = { nounwind }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/Transforms/SampleProfile/entry_counts_missing_dbginfo.ll b/llvm/test/Transforms/SampleProfile/entry_counts_missing_dbginfo.ll
index 9ec41d1bd..45b42e041 100644
--- a/llvm/test/Transforms/SampleProfile/entry_counts_missing_dbginfo.ll
+++ b/llvm/test/Transforms/SampleProfile/entry_counts_missing_dbginfo.ll
@@ -105,10 +105,10 @@ declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture) #2
 
 declare void @baz(...) #3
 
-attributes #0 = { nounwind ssp uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+fxsr,+mmx,+sahf,+sse,+sse2,+sse3,+sse4.1,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind ssp uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+fxsr,+mmx,+sahf,+sse,+sse2,+sse3,+sse4.1,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone speculatable }
 attributes #2 = { argmemonly nounwind }
-attributes #3 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+fxsr,+mmx,+sahf,+sse,+sse2,+sse3,+sse4.1,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+fxsr,+mmx,+sahf,+sse,+sse2,+sse3,+sse4.1,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #4 = { nounwind }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/Transforms/SampleProfile/gcc-simple.ll b/llvm/test/Transforms/SampleProfile/gcc-simple.ll
index 149b316db..1ade69403 100644
--- a/llvm/test/Transforms/SampleProfile/gcc-simple.ll
+++ b/llvm/test/Transforms/SampleProfile/gcc-simple.ll
@@ -136,9 +136,9 @@ for.end.6:                                        ; preds = %for.cond
 ; CHECK ![[PROF3]] = !{!"branch_weights", i32 1, i32 1}
 ; CHECK ![[PROF4]] = !{!"branch_weights", i32 1, i32 20238}
 
-attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
-attributes #2 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #3 = { nounwind }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/Transforms/SampleProfile/inline-act.ll b/llvm/test/Transforms/SampleProfile/inline-act.ll
index f54387011..65bae0a8d 100644
--- a/llvm/test/Transforms/SampleProfile/inline-act.ll
+++ b/llvm/test/Transforms/SampleProfile/inline-act.ll
@@ -55,7 +55,7 @@ define void @_Z3bari(i32) #0 !dbg !9 {
   ret void
 }
 
-attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!3}
diff --git a/llvm/test/Transforms/SampleProfile/inline-callee-update.ll b/llvm/test/Transforms/SampleProfile/inline-callee-update.ll
new file mode 100644
index 000000000..6001d7741
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/inline-callee-update.ll
@@ -0,0 +1,73 @@
+; Make sure Import GUID list for ThinLTO properly maintained while update function's entry count for inlining
+
+; RUN: opt < %s -passes='thinlto-pre-link<O2>' -pgo-kind=pgo-sample-use-pipeline -sample-profile-file=%S/Inputs/inline-callee-update.prof -S | FileCheck %s
+
+@y = global i32* ()* null, align 8
+@z = global i32* ()* null, align 8
+
+define i32* @sample_loader_inlinee() !dbg !3 {
+bb:
+  %tmp = call i32* @direct_leaf_func(i32* null), !dbg !4
+  %cmp = icmp ne i32* %tmp, null
+  br i1 %cmp, label %then, label %else
+
+then:                                             ; preds = %bb
+  %tmp1 = load i32* ()*, i32* ()** @z, align 8, !dbg !5
+  %tmp2 = call i32* %tmp1(), !dbg !5
+  ret i32* %tmp2
+
+else:                                             ; preds = %bb
+  ret i32* null
+}
+
+define i32* @cgscc_inlinee() !dbg !6 {
+bb:
+  %tmp = call i32* @direct_leaf_func(i32* null), !dbg !7
+  %cmp = icmp ne i32* %tmp, null
+  br i1 %cmp, label %then, label %else
+
+then:                                             ; preds = %bb
+  %tmp1 = load i32* ()*, i32* ()** @y, align 8, !dbg !8
+  %tmp2 = call i32* %tmp1(), !dbg !8
+  ret i32* %tmp2
+
+else:                                             ; preds = %bb
+  ret i32* null
+}
+
+define i32* @test_sample_loader_inline(void ()* %arg) !dbg !9 {
+bb:
+  %tmp = call i32* @sample_loader_inlinee(), !dbg !10
+  ret i32* %tmp
+}
+
+define i32* @test_cgscc_inline(void ()* %arg) !dbg !11 {
+bb:
+  %tmp = call i32* @cgscc_inlinee(), !dbg !12
+  ret i32* %tmp
+}
+
+declare i32* @direct_leaf_func(i32*)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!2}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, isOptimized: false, runtimeVersion: 0, emissionKind: NoDebug)
+!1 = !DIFile(filename: "test.cc", directory: "/")
+!2 = !{i32 2, !"Debug Info Version", i32 3}
+!3 = distinct !DISubprogram(name: "sample_loader_inlinee", scope: !1, file: !1, line: 11, spFlags: DISPFlagDefinition, unit: !0)
+!4 = !DILocation(line: 12, scope: !3)
+!5 = !DILocation(line: 13, scope: !3)
+!6 = distinct !DISubprogram(name: "cgscc_inlinee", scope: !1, file: !1, line: 31, spFlags: DISPFlagDefinition, unit: !0)
+!7 = !DILocation(line: 32, scope: !6)
+!8 = !DILocation(line: 33, scope: !6)
+!9 = distinct !DISubprogram(name: "test_sample_loader_inline", scope: !1, file: !1, line: 3, spFlags: DISPFlagDefinition, unit: !0)
+!10 = !DILocation(line: 4, scope: !9)
+!11 = distinct !DISubprogram(name: "test_cgscc_inline", scope: !1, file: !1, line: 20, spFlags: DISPFlagDefinition, unit: !0)
+!12 = !DILocation(line: 21, scope: !11)
+
+; Make sure the ImportGUID stays with entry count metadata for ThinLTO-PreLink
+; CHECK: distinct !DISubprogram(name: "sample_loader_inlinee"
+; CHECK-NEXT: {!"function_entry_count", i64 1, i64 -9171813444624716006}
+; CHECK: distinct !DISubprogram(name: "cgscc_inlinee"
+; CHECK-NEXT: !{!"function_entry_count", i64 0, i64 -9171813444624716006}
diff --git a/llvm/test/Transforms/SampleProfile/inline-cold.ll b/llvm/test/Transforms/SampleProfile/inline-cold.ll
new file mode 100644
index 000000000..abec880d2
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/inline-cold.ll
@@ -0,0 +1,102 @@
+; Let sample profile loader replay inlining of small/cold functions
+
+; Make sure we don't inline the cold call sites by default
+; RUN: opt < %s -sample-profile -sample-profile-file=%S/Inputs/inline-cold.prof -S | FileCheck -check-prefix=NOTINLINE %s
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline-cold.prof -S | FileCheck -check-prefix=NOTINLINE %s
+
+; Make sure we inline code call sites for size if requested
+; RUN: opt < %s -sample-profile -sample-profile-file=%S/Inputs/inline-cold.prof -sample-profile-inline-size -S | FileCheck -check-prefix=INLINE %s
+
+; Make sure we re-inline everything if requested 
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline-cold.prof -sample-profile-inline-size -sample-profile-cold-inline-threshold=9999999 -S | FileCheck -check-prefix=INLINE %s
+
+; Make sure the separate size threshold for sample profile loader inlining works
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline-cold.prof -sample-profile-inline-size -sample-profile-cold-inline-threshold=-500 -S | FileCheck -check-prefix=NOTINLINE %s
+
+@.str = private unnamed_addr constant [11 x i8] c"sum is %d\0A\00", align 1
+
+define i32 @_Z3sumii(i32 %x, i32 %y) !dbg !6 {
+entry:
+  %x.addr = alloca i32, align 4
+  %y.addr = alloca i32, align 4
+  store i32 %x, i32* %x.addr, align 4
+  store i32 %y, i32* %y.addr, align 4
+  %tmp = load i32, i32* %x.addr, align 4, !dbg !8
+  %tmp1 = load i32, i32* %y.addr, align 4, !dbg !8
+  %add = add nsw i32 %tmp, %tmp1, !dbg !8
+  ret i32 %add, !dbg !8
+}
+
+define i32 @main() !dbg !9 {
+entry:
+  %retval = alloca i32, align 4
+  %s = alloca i32, align 4
+  %i = alloca i32, align 4
+  store i32 0, i32* %retval
+  store i32 0, i32* %i, align 4, !dbg !10
+  br label %while.cond, !dbg !11
+
+while.cond:                                       ; preds = %if.end, %entry
+  %tmp = load i32, i32* %i, align 4, !dbg !12
+  %inc = add nsw i32 %tmp, 1, !dbg !12
+  store i32 %inc, i32* %i, align 4, !dbg !12
+  %cmp = icmp slt i32 %tmp, 400000000, !dbg !12
+  br i1 %cmp, label %while.body, label %while.end, !dbg !12
+
+while.body:                                       ; preds = %while.cond
+  %tmp1 = load i32, i32* %i, align 4, !dbg !14
+  %cmp1 = icmp ne i32 %tmp1, 100, !dbg !14
+  br i1 %cmp1, label %if.then, label %if.else, !dbg !14
+
+if.then:                                          ; preds = %while.body
+  %tmp2 = load i32, i32* %i, align 4, !dbg !16
+  %tmp3 = load i32, i32* %s, align 4, !dbg !16
+  %call = call i32 @_Z3sumii(i32 %tmp2, i32 %tmp3), !dbg !16
+; INLINE-NOT: call i32 @_Z3sumii
+; NOTINLINE: call i32 @_Z3sumii
+  store i32 %call, i32* %s, align 4, !dbg !16
+  br label %if.end, !dbg !16
+
+if.else:                                          ; preds = %while.body
+  store i32 30, i32* %s, align 4, !dbg !18
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  br label %while.cond, !dbg !20
+
+while.end:                                        ; preds = %while.cond
+  %tmp4 = load i32, i32* %s, align 4, !dbg !22
+  %call2 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([11 x i8], [11 x i8]* @.str, i32 0, i32 0), i32 %tmp4), !dbg !22
+  ret i32 0, !dbg !23
+}
+
+declare i32 @printf(i8*, ...)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4}
+!llvm.ident = !{!5}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 3.5 ", isOptimized: false, runtimeVersion: 0, emissionKind: NoDebug, enums: !2, retainedTypes: !2, globals: !2, imports: !2)
+!1 = !DIFile(filename: "calls.cc", directory: ".")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 1, !"Debug Info Version", i32 3}
+!5 = !{!"clang version 3.5 "}
+!6 = distinct !DISubprogram(name: "sum", scope: !1, file: !1, line: 3, type: !7, scopeLine: 3, virtualIndex: 6, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !2)
+!7 = !DISubroutineType(types: !2)
+!8 = !DILocation(line: 4, scope: !6)
+!9 = distinct !DISubprogram(name: "main", scope: !1, file: !1, line: 7, type: !7, scopeLine: 7, virtualIndex: 6, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !2)
+!10 = !DILocation(line: 8, scope: !9)
+!11 = !DILocation(line: 9, scope: !9)
+!12 = !DILocation(line: 9, scope: !13)
+!13 = !DILexicalBlockFile(scope: !9, file: !1, discriminator: 2)
+!14 = !DILocation(line: 10, scope: !15)
+!15 = distinct !DILexicalBlock(scope: !9, file: !1, line: 10)
+!16 = !DILocation(line: 10, scope: !17)
+!17 = !DILexicalBlockFile(scope: !15, file: !1, discriminator: 2)
+!18 = !DILocation(line: 10, scope: !19)
+!19 = !DILexicalBlockFile(scope: !15, file: !1, discriminator: 4)
+!20 = !DILocation(line: 10, scope: !21)
+!21 = !DILexicalBlockFile(scope: !15, file: !1, discriminator: 6)
+!22 = !DILocation(line: 11, scope: !9)
+!23 = !DILocation(line: 12, scope: !9)
diff --git a/llvm/test/Transforms/SampleProfile/inline-coverage.ll b/llvm/test/Transforms/SampleProfile/inline-coverage.ll
index 7e1895451..134bbe8b8 100644
--- a/llvm/test/Transforms/SampleProfile/inline-coverage.ll
+++ b/llvm/test/Transforms/SampleProfile/inline-coverage.ll
@@ -16,7 +16,7 @@
 ;    11      return sum > 0 ? 0 : 1;
 ;    12    }
 ;
-; CHECK: remark: coverage.cc:10:12: inlined hot callee '_Z3fool' into 'main'
+; CHECK: remark: coverage.cc:10:12: previous inlining reattempted for hotness: '_Z3fool' into 'main'
 ; CHECK: remark: coverage.cc:9:21: Applied 23478 samples from profile (offset: 2.1)
 ; CHECK: remark: coverage.cc:10:16: Applied 23478 samples from profile (offset: 3)
 ; CHECK: remark: coverage.cc:4:10: Applied 31878 samples from profile (offset: 1)
diff --git a/llvm/test/Transforms/SampleProfile/inline-mergeprof.ll b/llvm/test/Transforms/SampleProfile/inline-mergeprof.ll
new file mode 100644
index 000000000..8b5989f7a
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/inline-mergeprof.ll
@@ -0,0 +1,97 @@
+; Test we lose details of not inlined profile without '-sample-profile-merge-inlinee'
+; RUN: opt < %s -sample-profile -sample-profile-file=%S/Inputs/inline-mergeprof.prof -S | FileCheck -check-prefix=SCALE %s
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline-mergeprof.prof -S | FileCheck -check-prefix=SCALE %s
+
+; Test we properly merge not inlined profile properly with '-sample-profile-merge-inlinee'
+; RUN: opt < %s -sample-profile -sample-profile-file=%S/Inputs/inline-mergeprof.prof -sample-profile-merge-inlinee -S | FileCheck -check-prefix=MERGE %s
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline-mergeprof.prof -sample-profile-merge-inlinee -S | FileCheck -check-prefix=MERGE  %s
+
+@.str = private unnamed_addr constant [11 x i8] c"sum is %d\0A\00", align 1
+
+define i32 @main() !dbg !6 {
+entry:
+  %retval = alloca i32, align 4
+  %s = alloca i32, align 4
+  %i = alloca i32, align 4
+  %tmp = load i32, i32* %i, align 4, !dbg !8
+  %tmp1 = load i32, i32* %s, align 4, !dbg !8
+  %call = call i32 @_Z3sumii(i32 %tmp, i32 %tmp1), !dbg !8
+; SCALE: call i32 @_Z3sumii
+; MERGE: call i32 @_Z3sumii
+  store i32 %call, i32* %s, align 4, !dbg !8
+  ret i32 0, !dbg !11
+}
+
+define i32 @_Z3sumii(i32 %x, i32 %y) !dbg !12 {
+entry:
+  %x.addr = alloca i32, align 4
+  %y.addr = alloca i32, align 4
+  store i32 %x, i32* %x.addr, align 4
+  store i32 %y, i32* %y.addr, align 4
+  %tmp = load i32, i32* %x.addr, align 4, !dbg !13
+  %tmp1 = load i32, i32* %y.addr, align 4, !dbg !13
+  %add = add nsw i32 %tmp, %tmp1, !dbg !13
+  %tmp2 = load i32, i32* %x.addr, align 4, !dbg !13
+  %tmp3 = load i32, i32* %y.addr, align 4, !dbg !13
+  %cmp1 = icmp ne i32 %tmp3, 100, !dbg !13
+  br i1 %cmp1, label %if.then, label %if.else, !dbg !13
+
+if.then:                                          ; preds = %entry
+  %call = call i32 @_Z3subii(i32 %tmp2, i32 %tmp3), !dbg !14
+  ret i32 %add, !dbg !14
+
+if.else:                                          ; preds = %entry
+  ret i32 %add, !dbg !15
+}
+
+define i32 @_Z3subii(i32 %x, i32 %y) !dbg !16 {
+entry:
+  %x.addr = alloca i32, align 4
+  %y.addr = alloca i32, align 4
+  store i32 %x, i32* %x.addr, align 4
+  store i32 %y, i32* %y.addr, align 4
+  %tmp = load i32, i32* %x.addr, align 4, !dbg !17
+  %tmp1 = load i32, i32* %y.addr, align 4, !dbg !17
+  %add = sub nsw i32 %tmp, %tmp1, !dbg !17
+  ret i32 %add, !dbg !18
+}
+
+declare i32 @printf(i8*, ...)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4}
+!llvm.ident = !{!5}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 3.5 ", isOptimized: false, runtimeVersion: 0, emissionKind: NoDebug, enums: !2, retainedTypes: !2, globals: !2, imports: !2)
+!1 = !DIFile(filename: "calls.cc", directory: ".")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 1, !"Debug Info Version", i32 3}
+!5 = !{!"clang version 3.5 "}
+!6 = distinct !DISubprogram(name: "main", scope: !1, file: !1, line: 7, type: !7, scopeLine: 7, virtualIndex: 6, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !2)
+!7 = !DISubroutineType(types: !2)
+!8 = !DILocation(line: 10, scope: !9)
+!9 = !DILexicalBlockFile(scope: !10, file: !1, discriminator: 2)
+!10 = distinct !DILexicalBlock(scope: !6, file: !1, line: 10)
+!11 = !DILocation(line: 12, scope: !6)
+!12 = distinct !DISubprogram(name: "sum", scope: !1, file: !1, line: 3, type: !7, scopeLine: 3, virtualIndex: 6, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !2)
+!13 = !DILocation(line: 4, scope: !12)
+!14 = !DILocation(line: 5, scope: !12)
+!15 = !DILocation(line: 6, scope: !12)
+!16 = distinct !DISubprogram(name: "sub", scope: !1, file: !1, line: 20, type: !7, scopeLine: 20, virtualIndex: 6, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !2)
+!17 = !DILocation(line: 20, scope: !16)
+!18 = !DILocation(line: 21, scope: !16)
+
+; SCALE: name: "sum"
+; SCALE-NEXT: {!"function_entry_count", i64 46}
+; SCALE: !{!"branch_weights", i32 11, i32 2}
+; SCALE: !{!"branch_weights", i64 20}
+; SCALE: name: "sub"
+; SCALE-NEXT: {!"function_entry_count", i64 -1}
+
+; MERGE: name: "sum"
+; MERGE-NEXT: {!"function_entry_count", i64 46}
+; MERGE: !{!"branch_weights", i32 11, i32 23}
+; MERGE: !{!"branch_weights", i32 10}
+; MERGE: name: "sub"
+; MERGE-NEXT: {!"function_entry_count", i64 3}
\ No newline at end of file
diff --git a/llvm/test/Transforms/SampleProfile/inline-stats.ll b/llvm/test/Transforms/SampleProfile/inline-stats.ll
new file mode 100644
index 000000000..f928aed6a
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/inline-stats.ll
@@ -0,0 +1,104 @@
+; REQUIRES: asserts
+; RUN: opt < %s -sample-profile -sample-profile-file=%S/Inputs/inline.prof -stats -S 2>&1 | FileCheck %s
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline.prof -stats -S 2>&1 | FileCheck %s
+
+; Original C++ test case
+;
+; #include <stdio.h>
+;
+; int sum(int x, int y) {
+;   return x + y;
+; }
+;
+; int main() {
+;   int s, i = 0;
+;   while (i++ < 20000 * 20000)
+;     if (i != 100) s = sum(i, s); else s = 30;
+;   printf("sum is %d\n", s);
+;   return 0;
+; }
+;
+@.str = private unnamed_addr constant [11 x i8] c"sum is %d\0A\00", align 1
+define i32 @_Z3sumii(i32 %x, i32 %y) !dbg !6 {
+entry:
+  %x.addr = alloca i32, align 4
+  %y.addr = alloca i32, align 4
+  store i32 %x, i32* %x.addr, align 4
+  store i32 %y, i32* %y.addr, align 4
+  %tmp = load i32, i32* %x.addr, align 4, !dbg !8
+  %tmp1 = load i32, i32* %y.addr, align 4, !dbg !8
+  %add = add nsw i32 %tmp, %tmp1, !dbg !8
+  ret i32 %add, !dbg !8
+}
+define i32 @main() !dbg !9 {
+entry:
+  %retval = alloca i32, align 4
+  %s = alloca i32, align 4
+  %i = alloca i32, align 4
+  store i32 0, i32* %retval
+  store i32 0, i32* %i, align 4, !dbg !10
+  br label %while.cond, !dbg !11
+
+while.cond:                                       ; preds = %if.end, %entry
+  %tmp = load i32, i32* %i, align 4, !dbg !12
+  %inc = add nsw i32 %tmp, 1, !dbg !12
+  store i32 %inc, i32* %i, align 4, !dbg !12
+  %cmp = icmp slt i32 %tmp, 400000000, !dbg !12
+  br i1 %cmp, label %while.body, label %while.end, !dbg !12
+
+while.body:                                       ; preds = %while.cond
+  %tmp1 = load i32, i32* %i, align 4, !dbg !14
+  %cmp1 = icmp ne i32 %tmp1, 100, !dbg !14
+  br i1 %cmp1, label %if.then, label %if.else, !dbg !14
+
+if.then:                                          ; preds = %while.body
+  %tmp2 = load i32, i32* %i, align 4, !dbg !16
+  %tmp3 = load i32, i32* %s, align 4, !dbg !16
+  %call = call i32 @_Z3sumii(i32 %tmp2, i32 %tmp3), !dbg !16
+  store i32 %call, i32* %s, align 4, !dbg !16
+  br label %if.end, !dbg !16
+
+if.else:                                          ; preds = %while.body
+  store i32 30, i32* %s, align 4, !dbg !18
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  br label %while.cond, !dbg !20
+
+while.end:                                        ; preds = %while.cond
+  %tmp4 = load i32, i32* %s, align 4, !dbg !22
+  %call2 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([11 x i8], [11 x i8]* @.str, i32 0, i32 0), i32 %tmp4), !dbg !22
+  ret i32 0, !dbg !23
+}
+declare i32 @printf(i8*, ...)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4}
+!llvm.ident = !{!5}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 3.5 ", isOptimized: false, runtimeVersion: 0, emissionKind: NoDebug, enums: !2, retainedTypes: !2, globals: !2, imports: !2)
+!1 = !DIFile(filename: "calls.cc", directory: ".")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 1, !"Debug Info Version", i32 3}
+!5 = !{!"clang version 3.5 "}
+!6 = distinct !DISubprogram(name: "sum", scope: !1, file: !1, line: 3, type: !7, scopeLine: 3, virtualIndex: 6, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !2)
+!7 = !DISubroutineType(types: !2)
+!8 = !DILocation(line: 4, scope: !6)
+!9 = distinct !DISubprogram(name: "main", scope: !1, file: !1, line: 7, type: !7, scopeLine: 7, virtualIndex: 6, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !2)
+!10 = !DILocation(line: 8, scope: !9)
+!11 = !DILocation(line: 9, scope: !9)
+!12 = !DILocation(line: 9, scope: !13)
+!13 = !DILexicalBlockFile(scope: !9, file: !1, discriminator: 2)
+!14 = !DILocation(line: 10, scope: !15)
+!15 = distinct !DILexicalBlock(scope: !9, file: !1, line: 10)
+!16 = !DILocation(line: 10, scope: !17)
+!17 = !DILexicalBlockFile(scope: !15, file: !1, discriminator: 2)
+!18 = !DILocation(line: 10, scope: !19)
+!19 = !DILexicalBlockFile(scope: !15, file: !1, discriminator: 4)
+!20 = !DILocation(line: 10, scope: !21)
+!21 = !DILexicalBlockFile(scope: !15, file: !1, discriminator: 6)
+!22 = !DILocation(line: 11, scope: !9)
+!23 = !DILocation(line: 12, scope: !9)
+
+; CHECK: 1 sample-profile - Number of functions inlined with context sensitive profile
\ No newline at end of file
diff --git a/llvm/test/Transforms/SampleProfile/inline-topdown.ll b/llvm/test/Transforms/SampleProfile/inline-topdown.ll
new file mode 100644
index 000000000..711440f00
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/inline-topdown.ll
@@ -0,0 +1,123 @@
+; Note that this needs new pass manager for now. Passing `-sample-profile-top-down-load` to legacy pass manager is a no-op.
+
+; Test we aren't doing specialization for inlining with default source order
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline-topdown.prof -S | FileCheck -check-prefix=DEFAULT %s
+
+; Test we specialize based on call path with context-sensitive profile while inlining with '-sample-profile-top-down-load'
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline-topdown.prof -sample-profile-merge-inlinee -sample-profile-top-down-load -S | FileCheck -check-prefix=TOPDOWN  %s
+
+
+@.str = private unnamed_addr constant [11 x i8] c"sum is %d\0A\00", align 1
+
+define i32 @_Z3sumii(i32 %x, i32 %y) !dbg !6 {
+entry:
+  %x.addr = alloca i32, align 4
+  %y.addr = alloca i32, align 4
+  store i32 %x, i32* %x.addr, align 4
+  store i32 %y, i32* %y.addr, align 4
+  %tmp = load i32, i32* %x.addr, align 4, !dbg !8
+  %tmp1 = load i32, i32* %y.addr, align 4, !dbg !8
+  %add = add nsw i32 %tmp, %tmp1, !dbg !8
+  %tmp2 = load i32, i32* %x.addr, align 4, !dbg !8
+  %tmp3 = load i32, i32* %y.addr, align 4, !dbg !8
+  %call = call i32 @_Z3subii(i32 %tmp2, i32 %tmp3), !dbg !8
+  ret i32 %add, !dbg !8
+}
+
+define i32 @_Z3subii(i32 %x, i32 %y) !dbg !9 {
+entry:
+  %x.addr = alloca i32, align 4
+  %y.addr = alloca i32, align 4
+  store i32 %x, i32* %x.addr, align 4
+  store i32 %y, i32* %y.addr, align 4
+  %tmp = load i32, i32* %x.addr, align 4, !dbg !10
+  %tmp1 = load i32, i32* %y.addr, align 4, !dbg !10
+  %add = sub nsw i32 %tmp, %tmp1, !dbg !10
+  ret i32 %add, !dbg !11
+}
+
+define i32 @main() !dbg !12 {
+entry:
+  %retval = alloca i32, align 4
+  %s = alloca i32, align 4
+  %i = alloca i32, align 4
+  store i32 0, i32* %retval
+  store i32 0, i32* %i, align 4, !dbg !13
+  br label %while.cond, !dbg !14
+
+while.cond:                                       ; preds = %if.end, %entry
+  %tmp = load i32, i32* %i, align 4, !dbg !15
+  %inc = add nsw i32 %tmp, 1, !dbg !15
+  store i32 %inc, i32* %i, align 4, !dbg !15
+  %cmp = icmp slt i32 %tmp, 400000000, !dbg !15
+  br i1 %cmp, label %while.body, label %while.end, !dbg !15
+
+while.body:                                       ; preds = %while.cond
+  %tmp1 = load i32, i32* %i, align 4, !dbg !17
+  %cmp1 = icmp ne i32 %tmp1, 100, !dbg !17
+  br i1 %cmp1, label %if.then, label %if.else, !dbg !17
+
+if.then:                                          ; preds = %while.body
+  %tmp2 = load i32, i32* %i, align 4, !dbg !19
+  %tmp3 = load i32, i32* %s, align 4, !dbg !19
+  %call = call i32 @_Z3sumii(i32 %tmp2, i32 %tmp3), !dbg !19
+  store i32 %call, i32* %s, align 4, !dbg !19
+  br label %if.end, !dbg !19
+
+if.else:                                          ; preds = %while.body
+  store i32 30, i32* %s, align 4, !dbg !21
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  br label %while.cond, !dbg !23
+
+while.end:                                        ; preds = %while.cond
+  %tmp4 = load i32, i32* %s, align 4, !dbg !25
+  %call2 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([11 x i8], [11 x i8]* @.str, i32 0, i32 0), i32 %tmp4), !dbg !25
+  ret i32 0, !dbg !26
+}
+
+declare i32 @printf(i8*, ...)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4}
+!llvm.ident = !{!5}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 3.5 ", isOptimized: false, runtimeVersion: 0, emissionKind: NoDebug, enums: !2, retainedTypes: !2, globals: !2, imports: !2)
+!1 = !DIFile(filename: "calls.cc", directory: ".")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 1, !"Debug Info Version", i32 3}
+!5 = !{!"clang version 3.5 "}
+!6 = distinct !DISubprogram(name: "sum", scope: !1, file: !1, line: 3, type: !7, scopeLine: 3, virtualIndex: 6, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !2)
+!7 = !DISubroutineType(types: !2)
+!8 = !DILocation(line: 4, scope: !6)
+!9 = distinct !DISubprogram(name: "sub", scope: !1, file: !1, line: 20, type: !7, scopeLine: 20, virtualIndex: 6, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !2)
+!10 = !DILocation(line: 20, scope: !9)
+!11 = !DILocation(line: 21, scope: !9)
+!12 = distinct !DISubprogram(name: "main", scope: !1, file: !1, line: 7, type: !7, scopeLine: 7, virtualIndex: 6, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !2)
+!13 = !DILocation(line: 8, scope: !12)
+!14 = !DILocation(line: 9, scope: !12)
+!15 = !DILocation(line: 9, scope: !16)
+!16 = !DILexicalBlockFile(scope: !12, file: !1, discriminator: 2)
+!17 = !DILocation(line: 10, scope: !18)
+!18 = distinct !DILexicalBlock(scope: !12, file: !1, line: 10)
+!19 = !DILocation(line: 10, scope: !20)
+!20 = !DILexicalBlockFile(scope: !18, file: !1, discriminator: 2)
+!21 = !DILocation(line: 10, scope: !22)
+!22 = !DILexicalBlockFile(scope: !18, file: !1, discriminator: 4)
+!23 = !DILocation(line: 10, scope: !24)
+!24 = !DILexicalBlockFile(scope: !18, file: !1, discriminator: 6)
+!25 = !DILocation(line: 11, scope: !12)
+!26 = !DILocation(line: 12, scope: !12)
+
+
+; DEFAULT: @_Z3sumii
+; DEFAULT-NOT: call i32 @_Z3subii
+; DEFAULT: @main()
+; DEFAULT-NOT: call i32 @_Z3subii
+
+; TOPDOWN: @_Z3sumii
+; TOPDOWN-NOT: call i32 @_Z3subii
+; TOPDOWN: @main()
+; TOPDOWN: call i32 @_Z3subii
\ No newline at end of file
diff --git a/llvm/test/Transforms/SampleProfile/offset.ll b/llvm/test/Transforms/SampleProfile/offset.ll
index 145763dc9..ba998b6e7 100644
--- a/llvm/test/Transforms/SampleProfile/offset.ll
+++ b/llvm/test/Transforms/SampleProfile/offset.ll
@@ -48,7 +48,7 @@ return:                                           ; preds = %if.else, %if.then
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
 
-attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/Transforms/SampleProfile/profile-format-compress.ll b/llvm/test/Transforms/SampleProfile/profile-format-compress.ll
new file mode 100644
index 000000000..be4eae3cd
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/profile-format-compress.ll
@@ -0,0 +1,123 @@
+; REQUIRES: zlib
+; RUN: opt < %s -sample-profile -sample-profile-file=%S/Inputs/inline.prof -S | FileCheck %s
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline.prof -S | FileCheck %s
+; RUN: llvm-profdata merge -sample -extbinary -compress-all-sections %S/Inputs/inline.prof -o %t.compress.extbinary.afdo
+; RUN: opt < %s -sample-profile -sample-profile-file=%t.compress.extbinary.afdo -S | FileCheck %s
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%t.compress.extbinary.afdo -S | FileCheck %s
+
+; Original C++ test case
+;
+; #include <stdio.h>
+;
+; int sum(int x, int y) {
+;   return x + y;
+; }
+;
+; int main() {
+;   int s, i = 0;
+;   while (i++ < 20000 * 20000)
+;     if (i != 100) s = sum(i, s); else s = 30;
+;   printf("sum is %d\n", s);
+;   return 0;
+; }
+;
+@.str = private unnamed_addr constant [11 x i8] c"sum is %d\0A\00", align 1
+
+; Check sample-profile phase using compressed extbinary format profile
+; will annotate the IR with exactly the same result as using text format.
+; CHECK: br i1 %cmp, label %while.body, label %while.end{{.*}} !prof ![[IDX1:[0-9]*]]
+; CHECK: br i1 %cmp1, label %if.then, label %if.else{{.*}} !prof ![[IDX2:[0-9]*]]
+; CHECK: call i32 (i8*, ...) @printf{{.*}} !prof ![[IDX3:[0-9]*]]
+; CHECK: = !{!"TotalCount", i64 26781}
+; CHECK: = !{!"MaxCount", i64 5553}
+; CHECK: ![[IDX1]] = !{!"branch_weights", i32 5392, i32 163}
+; CHECK: ![[IDX2]] = !{!"branch_weights", i32 5280, i32 113}
+; CHECK: ![[IDX3]] = !{!"branch_weights", i32 1}
+
+; Function Attrs: nounwind uwtable
+define i32 @_Z3sumii(i32 %x, i32 %y) !dbg !4 {
+entry:
+  %x.addr = alloca i32, align 4
+  %y.addr = alloca i32, align 4
+  store i32 %x, i32* %x.addr, align 4
+  store i32 %y, i32* %y.addr, align 4
+  %0 = load i32, i32* %x.addr, align 4, !dbg !11
+  %1 = load i32, i32* %y.addr, align 4, !dbg !11
+  %add = add nsw i32 %0, %1, !dbg !11
+  ret i32 %add, !dbg !11
+}
+
+; Function Attrs: uwtable
+define i32 @main() !dbg !7 {
+entry:
+  %retval = alloca i32, align 4
+  %s = alloca i32, align 4
+  %i = alloca i32, align 4
+  store i32 0, i32* %retval
+  store i32 0, i32* %i, align 4, !dbg !12
+  br label %while.cond, !dbg !13
+
+while.cond:                                       ; preds = %if.end, %entry
+  %0 = load i32, i32* %i, align 4, !dbg !14
+  %inc = add nsw i32 %0, 1, !dbg !14
+  store i32 %inc, i32* %i, align 4, !dbg !14
+  %cmp = icmp slt i32 %0, 400000000, !dbg !14
+  br i1 %cmp, label %while.body, label %while.end, !dbg !14
+
+while.body:                                       ; preds = %while.cond
+  %1 = load i32, i32* %i, align 4, !dbg !16
+  %cmp1 = icmp ne i32 %1, 100, !dbg !16
+  br i1 %cmp1, label %if.then, label %if.else, !dbg !16
+
+
+if.then:                                          ; preds = %while.body
+  %2 = load i32, i32* %i, align 4, !dbg !18
+  %3 = load i32, i32* %s, align 4, !dbg !18
+  %call = call i32 @_Z3sumii(i32 %2, i32 %3), !dbg !18
+  store i32 %call, i32* %s, align 4, !dbg !18
+  br label %if.end, !dbg !18
+
+if.else:                                          ; preds = %while.body
+  store i32 30, i32* %s, align 4, !dbg !20
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  br label %while.cond, !dbg !22
+
+while.end:                                        ; preds = %while.cond
+  %4 = load i32, i32* %s, align 4, !dbg !24
+  %call2 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([11 x i8], [11 x i8]* @.str, i32 0, i32 0), i32 %4), !dbg !24
+  ret i32 0, !dbg !25
+}
+
+declare i32 @printf(i8*, ...) #2
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!8, !9}
+!llvm.ident = !{!10}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5 ", isOptimized: false, emissionKind: NoDebug, file: !1, enums: !2, retainedTypes: !2, globals: !2, imports: !2)
+!1 = !DIFile(filename: "calls.cc", directory: ".")
+!2 = !{}
+!4 = distinct !DISubprogram(name: "sum", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, scopeLine: 3, file: !1, scope: !5, type: !6, retainedNodes: !2)
+!5 = !DIFile(filename: "calls.cc", directory: ".")
+!6 = !DISubroutineType(types: !2)
+!7 = distinct !DISubprogram(name: "main", line: 7, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, scopeLine: 7, file: !1, scope: !5, type: !6, retainedNodes: !2)
+!8 = !{i32 2, !"Dwarf Version", i32 4}
+!9 = !{i32 1, !"Debug Info Version", i32 3}
+!10 = !{!"clang version 3.5 "}
+!11 = !DILocation(line: 4, scope: !4)
+!12 = !DILocation(line: 8, scope: !7)
+!13 = !DILocation(line: 9, scope: !7)
+!14 = !DILocation(line: 9, scope: !15)
+!15 = !DILexicalBlockFile(discriminator: 2, file: !1, scope: !7)
+!16 = !DILocation(line: 10, scope: !17)
+!17 = distinct !DILexicalBlock(line: 10, column: 0, file: !1, scope: !7)
+!18 = !DILocation(line: 10, scope: !19)
+!19 = !DILexicalBlockFile(discriminator: 2, file: !1, scope: !17)
+!20 = !DILocation(line: 10, scope: !21)
+!21 = !DILexicalBlockFile(discriminator: 4, file: !1, scope: !17)
+!22 = !DILocation(line: 10, scope: !23)
+!23 = !DILexicalBlockFile(discriminator: 6, file: !1, scope: !17)
+!24 = !DILocation(line: 11, scope: !7)
+!25 = !DILocation(line: 12, scope: !7)
diff --git a/llvm/test/Transforms/SampleProfile/profile-format.ll b/llvm/test/Transforms/SampleProfile/profile-format.ll
new file mode 100644
index 000000000..4f7ae0a06
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/profile-format.ll
@@ -0,0 +1,123 @@
+; RUN: opt < %s -sample-profile -sample-profile-file=%S/Inputs/inline.prof -S | FileCheck %s
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline.prof -S | FileCheck %s
+; RUN: opt < %s -sample-profile -sample-profile-file=%S/Inputs/inline.compactbinary.afdo -S | FileCheck %s
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline.compactbinary.afdo -S | FileCheck %s
+; RUN: opt < %s -sample-profile -sample-profile-file=%S/Inputs/inline.extbinary.afdo -S | FileCheck %s
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline.extbinary.afdo -S | FileCheck %s
+
+; Original C++ test case
+;
+; #include <stdio.h>
+;
+; int sum(int x, int y) {
+;   return x + y;
+; }
+;
+; int main() {
+;   int s, i = 0;
+;   while (i++ < 20000 * 20000)
+;     if (i != 100) s = sum(i, s); else s = 30;
+;   printf("sum is %d\n", s);
+;   return 0;
+; }
+;
+@.str = private unnamed_addr constant [11 x i8] c"sum is %d\0A\00", align 1
+
+; Check sample-profile phase using compactbinary or extbinary format profile
+; will annotate the IR with exactly the same result as using text format.
+; CHECK: br i1 %cmp, label %while.body, label %while.end{{.*}} !prof ![[IDX1:[0-9]*]]
+; CHECK: br i1 %cmp1, label %if.then, label %if.else{{.*}} !prof ![[IDX2:[0-9]*]]
+; CHECK: call i32 (i8*, ...) @printf{{.*}} !prof ![[IDX3:[0-9]*]]
+; CHECK: = !{!"TotalCount", i64 26781}
+; CHECK: = !{!"MaxCount", i64 5553}
+; CHECK: ![[IDX1]] = !{!"branch_weights", i32 5392, i32 163}
+; CHECK: ![[IDX2]] = !{!"branch_weights", i32 5280, i32 113}
+; CHECK: ![[IDX3]] = !{!"branch_weights", i32 1}
+
+; Function Attrs: nounwind uwtable
+define i32 @_Z3sumii(i32 %x, i32 %y) !dbg !4 {
+entry:
+  %x.addr = alloca i32, align 4
+  %y.addr = alloca i32, align 4
+  store i32 %x, i32* %x.addr, align 4
+  store i32 %y, i32* %y.addr, align 4
+  %0 = load i32, i32* %x.addr, align 4, !dbg !11
+  %1 = load i32, i32* %y.addr, align 4, !dbg !11
+  %add = add nsw i32 %0, %1, !dbg !11
+  ret i32 %add, !dbg !11
+}
+
+; Function Attrs: uwtable
+define i32 @main() !dbg !7 {
+entry:
+  %retval = alloca i32, align 4
+  %s = alloca i32, align 4
+  %i = alloca i32, align 4
+  store i32 0, i32* %retval
+  store i32 0, i32* %i, align 4, !dbg !12
+  br label %while.cond, !dbg !13
+
+while.cond:                                       ; preds = %if.end, %entry
+  %0 = load i32, i32* %i, align 4, !dbg !14
+  %inc = add nsw i32 %0, 1, !dbg !14
+  store i32 %inc, i32* %i, align 4, !dbg !14
+  %cmp = icmp slt i32 %0, 400000000, !dbg !14
+  br i1 %cmp, label %while.body, label %while.end, !dbg !14
+
+while.body:                                       ; preds = %while.cond
+  %1 = load i32, i32* %i, align 4, !dbg !16
+  %cmp1 = icmp ne i32 %1, 100, !dbg !16
+  br i1 %cmp1, label %if.then, label %if.else, !dbg !16
+
+
+if.then:                                          ; preds = %while.body
+  %2 = load i32, i32* %i, align 4, !dbg !18
+  %3 = load i32, i32* %s, align 4, !dbg !18
+  %call = call i32 @_Z3sumii(i32 %2, i32 %3), !dbg !18
+  store i32 %call, i32* %s, align 4, !dbg !18
+  br label %if.end, !dbg !18
+
+if.else:                                          ; preds = %while.body
+  store i32 30, i32* %s, align 4, !dbg !20
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  br label %while.cond, !dbg !22
+
+while.end:                                        ; preds = %while.cond
+  %4 = load i32, i32* %s, align 4, !dbg !24
+  %call2 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([11 x i8], [11 x i8]* @.str, i32 0, i32 0), i32 %4), !dbg !24
+  ret i32 0, !dbg !25
+}
+
+declare i32 @printf(i8*, ...) #2
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!8, !9}
+!llvm.ident = !{!10}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5 ", isOptimized: false, emissionKind: NoDebug, file: !1, enums: !2, retainedTypes: !2, globals: !2, imports: !2)
+!1 = !DIFile(filename: "calls.cc", directory: ".")
+!2 = !{}
+!4 = distinct !DISubprogram(name: "sum", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, scopeLine: 3, file: !1, scope: !5, type: !6, retainedNodes: !2)
+!5 = !DIFile(filename: "calls.cc", directory: ".")
+!6 = !DISubroutineType(types: !2)
+!7 = distinct !DISubprogram(name: "main", line: 7, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, scopeLine: 7, file: !1, scope: !5, type: !6, retainedNodes: !2)
+!8 = !{i32 2, !"Dwarf Version", i32 4}
+!9 = !{i32 1, !"Debug Info Version", i32 3}
+!10 = !{!"clang version 3.5 "}
+!11 = !DILocation(line: 4, scope: !4)
+!12 = !DILocation(line: 8, scope: !7)
+!13 = !DILocation(line: 9, scope: !7)
+!14 = !DILocation(line: 9, scope: !15)
+!15 = !DILexicalBlockFile(discriminator: 2, file: !1, scope: !7)
+!16 = !DILocation(line: 10, scope: !17)
+!17 = distinct !DILexicalBlock(line: 10, column: 0, file: !1, scope: !7)
+!18 = !DILocation(line: 10, scope: !19)
+!19 = !DILexicalBlockFile(discriminator: 2, file: !1, scope: !17)
+!20 = !DILocation(line: 10, scope: !21)
+!21 = !DILexicalBlockFile(discriminator: 4, file: !1, scope: !17)
+!22 = !DILocation(line: 10, scope: !23)
+!23 = !DILexicalBlockFile(discriminator: 6, file: !1, scope: !17)
+!24 = !DILocation(line: 11, scope: !7)
+!25 = !DILocation(line: 12, scope: !7)
diff --git a/llvm/test/Transforms/SampleProfile/profile-sample-accurate.ll b/llvm/test/Transforms/SampleProfile/profile-sample-accurate.ll
new file mode 100644
index 000000000..bfade9d90
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/profile-sample-accurate.ll
@@ -0,0 +1,157 @@
+; RUN: opt < %s -sample-profile -sample-profile-file=%S/Inputs/profsampleacc.extbinary.afdo -profile-summary-cutoff-hot=600000 -profile-sample-accurate -S | FileCheck %s --check-prefix=CALL_SUM_IS_WARM
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/profsampleacc.extbinary.afdo -profile-summary-cutoff-hot=900000 -profile-sample-accurate -S | FileCheck %s --check-prefix=CALL_SUM_IS_HOT
+; RUN: opt < %s -sample-profile -sample-profile-file=%S/Inputs/profsampleacc.extbinary.afdo -profile-summary-cutoff-hot=600000 -profile-sample-accurate -S | FileCheck %s --check-prefix=CALL_SUM_IS_WARM
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/profsampleacc.extbinary.afdo -profile-summary-cutoff-hot=900000 -profile-sample-accurate -S | FileCheck %s --check-prefix=CALL_SUM_IS_HOT
+
+; RUN: llvm-profdata merge -sample -extbinary -prof-sym-list=%S/Inputs/profile-symbol-list.text %S/Inputs/profsampleacc.extbinary.afdo -o %t.symlist.afdo
+; RUN: opt < %s -sample-profile -sample-profile-file=%t.symlist.afdo -profile-summary-cutoff-hot=600000 -profile-accurate-for-symsinlist -S | FileCheck %s --check-prefix=PROFSYMLIST
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%t.symlist.afdo -profile-summary-cutoff-hot=600000 -profile-accurate-for-symsinlist -S | FileCheck %s --check-prefix=PROFSYMLIST
+;
+; If -profile-accurate-for-symsinlist and -profile-sample-accurate both present,
+; -profile-sample-accurate will override -profile-accurate-for-symsinlist.
+; RUN: opt < %s -sample-profile -sample-profile-file=%S/Inputs/profsampleacc.extbinary.afdo -profile-summary-cutoff-hot=600000 -profile-sample-accurate -profile-accurate-for-symsinlist -S | FileCheck %s --check-prefix=CALL_SUM_IS_WARM
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/profsampleacc.extbinary.afdo -profile-summary-cutoff-hot=900000 -profile-sample-accurate -profile-accurate-for-symsinlist -S | FileCheck %s --check-prefix=CALL_SUM_IS_HOT
+; RUN: opt < %s -sample-profile -sample-profile-file=%S/Inputs/profsampleacc.extbinary.afdo -profile-summary-cutoff-hot=600000 -profile-sample-accurate -profile-accurate-for-symsinlist -S | FileCheck %s --check-prefix=CALL_SUM_IS_WARM
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/profsampleacc.extbinary.afdo -profile-summary-cutoff-hot=900000 -profile-sample-accurate -profile-accurate-for-symsinlist -S | FileCheck %s --check-prefix=CALL_SUM_IS_HOT
+;
+; Original C++ test case
+;
+; #include <stdio.h>
+;
+; int sum(int x, int y) {
+;   return x + y;
+; }
+;
+; int main() {
+;   int s, i = 0;
+;   while (i++ < 20000 * 20000)
+;     if (i != 100) s = sum(i, s); else s = 30;
+;   printf("sum is %d\n", s);
+;   return 0;
+; }
+;
+@.str = private unnamed_addr constant [11 x i8] c"sum is %d\0A\00", align 1
+
+; Check _Z3sumii's function entry count will be 0 when
+; profile-sample-accurate is enabled.
+; CALL_SUM_IS_HOT: define i32 @_Z3sumii{{.*}}!prof ![[ZERO_ID:[0-9]+]]
+;
+; Check _Z3sumii's function entry count will be nonzero when
+; profile-sample-accurate is enabled because the callsite is warm and not
+; inlined so its function entry count is adjusted to nonzero.
+; CALL_SUM_IS_WARM: define i32 @_Z3sumii{{.*}}!prof ![[NONZERO_ID:[0-9]+]]
+;
+; Check _Z3sumii's function entry count will be initialized to -1 when
+; profile-accurate-for-profsymlist is enabled and _Z3sumii exists in the
+; profile symbol list because it also shows up in the profile as inline
+; instance.
+; PROFSYMLIST: define i32 @_Z3sumii{{.*}}!prof ![[UNKNOWN_ID:[0-9]+]]
+;
+; Function Attrs: nounwind uwtable
+define i32 @_Z3sumii(i32 %x, i32 %y) !dbg !4 {
+entry:
+  %x.addr = alloca i32, align 4
+  %y.addr = alloca i32, align 4
+  store i32 %x, i32* %x.addr, align 4
+  store i32 %y, i32* %y.addr, align 4
+  %0 = load i32, i32* %x.addr, align 4, !dbg !11
+  %1 = load i32, i32* %y.addr, align 4, !dbg !11
+  %add = add nsw i32 %0, %1, !dbg !11
+  ret i32 %add, !dbg !11
+}
+
+; Function Attrs: uwtable
+define i32 @main() !dbg !7 {
+entry:
+  %retval = alloca i32, align 4
+  %s = alloca i32, align 4
+  %i = alloca i32, align 4
+  store i32 0, i32* %retval
+  store i32 0, i32* %i, align 4, !dbg !12
+  br label %while.cond, !dbg !13
+
+while.cond:                                       ; preds = %if.end, %entry
+  %0 = load i32, i32* %i, align 4, !dbg !14
+  %inc = add nsw i32 %0, 1, !dbg !14
+  store i32 %inc, i32* %i, align 4, !dbg !14
+  %cmp = icmp slt i32 %0, 400000000, !dbg !14
+  br i1 %cmp, label %while.body, label %while.end, !dbg !14
+
+while.body:                                       ; preds = %while.cond
+  %1 = load i32, i32* %i, align 4, !dbg !16
+  %cmp1 = icmp ne i32 %1, 100, !dbg !16
+  br i1 %cmp1, label %if.then, label %if.else, !dbg !16
+
+; With the hot cutoff being set to 600000, the inline instance of _Z3sumii
+; in main is neither hot nor cold. Check it won't be inlined when
+; profile-sample-accurate is enabled.
+; CALL_SUM_IS_WARM: if.then:
+; CALL_SUM_IS_WARM: call i32 @_Z3sumii
+; CALL_SUM_IS_WARM: if.else:
+;
+; With the hot cutoff being set to 900000, the inline instance of _Z3sumii
+; in main is hot. Check the callsite of _Z3sumii will be inlined when
+; profile-sample-accurate is enabled.
+; CALL_SUM_IS_HOT: if.then:
+; CALL_SUM_IS_HOT-NOT: call i32 @_Z3sumii
+; CALL_SUM_IS_HOT: if.else:
+;
+; Check _Z3sumii will be inlined when profile-accurate-for-profsymlist is
+; enabled
+; PROFSYMLIST: if.then:
+; PROFSYMLIST-NOT: call i32 @_Z3sumii
+; PROFSYMLIST: if.else:
+if.then:                                          ; preds = %while.body
+  %2 = load i32, i32* %i, align 4, !dbg !18
+  %3 = load i32, i32* %s, align 4, !dbg !18
+  %call = call i32 @_Z3sumii(i32 %2, i32 %3), !dbg !18
+  store i32 %call, i32* %s, align 4, !dbg !18
+  br label %if.end, !dbg !18
+
+if.else:                                          ; preds = %while.body
+  store i32 30, i32* %s, align 4, !dbg !20
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  br label %while.cond, !dbg !22
+
+while.end:                                        ; preds = %while.cond
+  %4 = load i32, i32* %s, align 4, !dbg !24
+  %call2 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([11 x i8], [11 x i8]* @.str, i32 0, i32 0), i32 %4), !dbg !24
+  ret i32 0, !dbg !25
+}
+
+declare i32 @printf(i8*, ...) #2
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!8, !9}
+!llvm.ident = !{!10}
+
+; CALL_SUM_IS_HOT: ![[ZERO_ID]] = !{!"function_entry_count", i64 0}
+; CALL_SUM_IS_WARM: ![[NONZERO_ID]] = !{!"function_entry_count", i64 5179}
+; PROFSYMLIST: ![[UNKNOWN_ID]] = !{!"function_entry_count", i64 -1}
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5 ", isOptimized: false, emissionKind: NoDebug, file: !1, enums: !2, retainedTypes: !2, globals: !2, imports: !2)
+!1 = !DIFile(filename: "calls.cc", directory: ".")
+!2 = !{}
+!4 = distinct !DISubprogram(name: "sum", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, scopeLine: 3, file: !1, scope: !5, type: !6, retainedNodes: !2)
+!5 = !DIFile(filename: "calls.cc", directory: ".")
+!6 = !DISubroutineType(types: !2)
+!7 = distinct !DISubprogram(name: "main", line: 7, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, scopeLine: 7, file: !1, scope: !5, type: !6, retainedNodes: !2)
+!8 = !{i32 2, !"Dwarf Version", i32 4}
+!9 = !{i32 1, !"Debug Info Version", i32 3}
+!10 = !{!"clang version 3.5 "}
+!11 = !DILocation(line: 4, scope: !4)
+!12 = !DILocation(line: 8, scope: !7)
+!13 = !DILocation(line: 9, scope: !7)
+!14 = !DILocation(line: 9, scope: !15)
+!15 = !DILexicalBlockFile(discriminator: 2, file: !1, scope: !7)
+!16 = !DILocation(line: 10, scope: !17)
+!17 = distinct !DILexicalBlock(line: 10, column: 0, file: !1, scope: !7)
+!18 = !DILocation(line: 10, scope: !19)
+!19 = !DILexicalBlockFile(discriminator: 2, file: !1, scope: !17)
+!20 = !DILocation(line: 10, scope: !21)
+!21 = !DILexicalBlockFile(discriminator: 4, file: !1, scope: !17)
+!22 = !DILocation(line: 10, scope: !23)
+!23 = !DILexicalBlockFile(discriminator: 6, file: !1, scope: !17)
+!24 = !DILocation(line: 11, scope: !7)
+!25 = !DILocation(line: 12, scope: !7)
diff --git a/llvm/test/Transforms/SampleProfile/propagate.ll b/llvm/test/Transforms/SampleProfile/propagate.ll
index 303e59a2d..bd0279377 100644
--- a/llvm/test/Transforms/SampleProfile/propagate.ll
+++ b/llvm/test/Transforms/SampleProfile/propagate.ll
@@ -201,10 +201,10 @@ entry:
 
 declare i32 @printf(i8*, ...) #3
 
-attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
-attributes #2 = { norecurse uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #3 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { norecurse uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!3, !4}
diff --git a/llvm/test/Transforms/SampleProfile/remap.ll b/llvm/test/Transforms/SampleProfile/remap.ll
index 206962a3b..af37ddae1 100644
--- a/llvm/test/Transforms/SampleProfile/remap.ll
+++ b/llvm/test/Transforms/SampleProfile/remap.ll
@@ -1,5 +1,9 @@
 ; RUN: opt %s -passes=sample-profile -sample-profile-file=%S/Inputs/remap.prof -sample-profile-remapping-file=%S/Inputs/remap.map | opt -analyze -branch-prob | FileCheck %s
-
+;
+; Check whether profile remapping work with loading profile on demand used by extbinary format profile.
+; RUN: llvm-profdata merge -sample -extbinary %S/Inputs/remap.prof -o %t.extbinary.afdo
+; RUN: opt %s -passes=sample-profile -sample-profile-file=%S/Inputs/remap.prof -sample-profile-remapping-file=%S/Inputs/remap.map | opt -analyze -branch-prob | FileCheck %s
+;
 ; Reduced from branch.ll
 
 declare i1 @foo()
diff --git a/llvm/test/Transforms/SampleProfile/remarks.ll b/llvm/test/Transforms/SampleProfile/remarks.ll
index 3ecaa5342..a5681fb9d 100644
--- a/llvm/test/Transforms/SampleProfile/remarks.ll
+++ b/llvm/test/Transforms/SampleProfile/remarks.ll
@@ -21,7 +21,7 @@
 
 ; We are expecting foo() to be inlined in main() (almost all the cycles are
 ; spent inside foo).
-; CHECK: remark: remarks.cc:13:21: inlined hot callee '_Z3foov' into 'main'
+; CHECK: remark: remarks.cc:13:21: inlined callee '_Z3foov' into 'main'
 
 ; The back edge for the loop is the hottest edge in the loop subgraph.
 ; CHECK: remark: remarks.cc:6:9: most popular destination for conditional branches at remarks.cc:5:3
@@ -31,12 +31,12 @@
 
 ; Checking to see if YAML file is generated and contains remarks
 ;YAML:       --- !Passed
-;YAML-NEXT:  Pass:            sample-profile
-;YAML-NEXT:  Name:            HotInline
+;YAML-NEXT:  Pass:            sample-profile-inline
+;YAML-NEXT:  Name:            InlineSuccess
 ;YAML-NEXT:  DebugLoc:        { File: remarks.cc, Line: 13, Column: 21 }
 ;YAML-NEXT:  Function:        main
 ;YAML-NEXT:  Args:
-;YAML-NEXT:    - String:          'inlined hot callee '''
+;YAML-NEXT:    - String:          'inlined callee '''
 ;YAML-NEXT:    - Callee:          _Z3foov
 ;YAML-NEXT:      DebugLoc:        { File: remarks.cc, Line: 3, Column: 0 }
 ;YAML-NEXT:    - String:          ''' into '''
@@ -155,10 +155,10 @@ entry:
   ret i32 %conv, !dbg !58
 }
 
-attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind argmemonly }
 attributes #2 = { nounwind readnone }
-attributes #3 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #4 = { nounwind }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/Transforms/SampleProfile/uncompressed-profile-symbol-list.ll b/llvm/test/Transforms/SampleProfile/uncompressed-profile-symbol-list.ll
new file mode 100644
index 000000000..5eaf4b279
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/uncompressed-profile-symbol-list.ll
@@ -0,0 +1,4 @@
+; Append inline.prof with profile symbol list and save it without compression.
+; RUN: llvm-profdata merge --sample --prof-sym-list=%S/Inputs/profile-symbol-list.text --compress-all-sections=false --extbinary %S/Inputs/inline.prof --output=%t.profdata
+; RUN: opt < %S/Inputs/profile-symbol-list.ll -sample-profile -profile-accurate-for-symsinlist -sample-profile-file=%t.profdata -S | FileCheck %S/Inputs/profile-symbol-list.ll
+; RUN: opt < %S/Inputs/profile-symbol-list.ll -passes=sample-profile -profile-accurate-for-symsinlist -sample-profile-file=%t.profdata -S | FileCheck %S/Inputs/profile-symbol-list.ll
diff --git a/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-compressstore.ll b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-compressstore.ll
index cc53a2c85..4aa13b2dd 100644
--- a/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-compressstore.ll
+++ b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-compressstore.ll
@@ -3,20 +3,23 @@
 
 define void @scalarize_v2i64(i64* %p, <2 x i1> %mask, <2 x i64> %data) {
 ; CHECK-LABEL: @scalarize_v2i64(
-; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x i1> [[MASK:%.*]], i64 0
-; CHECK-NEXT:    br i1 [[TMP1]], label [[COND_STORE:%.*]], label [[ELSE:%.*]]
+; CHECK-NEXT:    [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK:%.*]] to i2
+; CHECK-NEXT:    [[TMP1:%.*]] = and i2 [[SCALAR_MASK]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i2 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[TMP2]], label [[COND_STORE:%.*]], label [[ELSE:%.*]]
 ; CHECK:       cond.store:
-; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x i64> [[DATA:%.*]], i64 0
-; CHECK-NEXT:    store i64 [[TMP2]], i64* [[P:%.*]], align 1
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i64, i64* [[P]], i32 1
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x i64> [[DATA:%.*]], i64 0
+; CHECK-NEXT:    store i64 [[TMP3]], i64* [[P:%.*]], align 1
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, i64* [[P]], i32 1
 ; CHECK-NEXT:    br label [[ELSE]]
 ; CHECK:       else:
-; CHECK-NEXT:    [[PTR_PHI_ELSE:%.*]] = phi i64* [ [[TMP3]], [[COND_STORE]] ], [ [[P]], [[TMP0:%.*]] ]
-; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i1> [[MASK]], i64 1
-; CHECK-NEXT:    br i1 [[TMP4]], label [[COND_STORE1:%.*]], label [[ELSE2:%.*]]
+; CHECK-NEXT:    [[PTR_PHI_ELSE:%.*]] = phi i64* [ [[TMP4]], [[COND_STORE]] ], [ [[P]], [[TMP0:%.*]] ]
+; CHECK-NEXT:    [[TMP5:%.*]] = and i2 [[SCALAR_MASK]], -2
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ne i2 [[TMP5]], 0
+; CHECK-NEXT:    br i1 [[TMP6]], label [[COND_STORE1:%.*]], label [[ELSE2:%.*]]
 ; CHECK:       cond.store1:
-; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[DATA]], i64 1
-; CHECK-NEXT:    store i64 [[TMP5]], i64* [[PTR_PHI_ELSE]], align 1
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x i64> [[DATA]], i64 1
+; CHECK-NEXT:    store i64 [[TMP7]], i64* [[PTR_PHI_ELSE]], align 1
 ; CHECK-NEXT:    br label [[ELSE2]]
 ; CHECK:       else2:
 ; CHECK-NEXT:    ret void
@@ -27,20 +30,12 @@ define void @scalarize_v2i64(i64* %p, <2 x i1> %mask, <2 x i64> %data) {
 
 define void @scalarize_v2i64_ones_mask(i64* %p, <2 x i64> %data) {
 ; CHECK-LABEL: @scalarize_v2i64_ones_mask(
-; CHECK-NEXT:    br i1 true, label [[COND_STORE:%.*]], label [[ELSE:%.*]]
-; CHECK:       cond.store:
-; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x i64> [[DATA:%.*]], i64 0
-; CHECK-NEXT:    store i64 [[TMP1]], i64* [[P:%.*]], align 1
+; CHECK-NEXT:    [[ELT0:%.*]] = extractelement <2 x i64> [[DATA:%.*]], i64 0
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i64, i64* [[P:%.*]], i32 0
+; CHECK-NEXT:    store i64 [[ELT0]], i64* [[TMP1]], align 1
+; CHECK-NEXT:    [[ELT1:%.*]] = extractelement <2 x i64> [[DATA]], i64 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, i64* [[P]], i32 1
-; CHECK-NEXT:    br label [[ELSE]]
-; CHECK:       else:
-; CHECK-NEXT:    [[PTR_PHI_ELSE:%.*]] = phi i64* [ [[TMP2]], [[COND_STORE]] ], [ [[P]], [[TMP0:%.*]] ]
-; CHECK-NEXT:    br i1 true, label [[COND_STORE1:%.*]], label [[ELSE2:%.*]]
-; CHECK:       cond.store1:
-; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x i64> [[DATA]], i64 1
-; CHECK-NEXT:    store i64 [[TMP3]], i64* [[PTR_PHI_ELSE]], align 1
-; CHECK-NEXT:    br label [[ELSE2]]
-; CHECK:       else2:
+; CHECK-NEXT:    store i64 [[ELT1]], i64* [[TMP2]], align 1
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.masked.compressstore.v2i64.p0v2i64(<2 x i64> %data, i64* %p, <2 x i1> <i1 true, i1 true>)
@@ -49,20 +44,6 @@ define void @scalarize_v2i64_ones_mask(i64* %p, <2 x i64> %data) {
 
 define void @scalarize_v2i64_zero_mask(i64* %p, <2 x i64> %data) {
 ; CHECK-LABEL: @scalarize_v2i64_zero_mask(
-; CHECK-NEXT:    br i1 false, label [[COND_STORE:%.*]], label [[ELSE:%.*]]
-; CHECK:       cond.store:
-; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x i64> [[DATA:%.*]], i64 0
-; CHECK-NEXT:    store i64 [[TMP1]], i64* [[P:%.*]], align 1
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, i64* [[P]], i32 1
-; CHECK-NEXT:    br label [[ELSE]]
-; CHECK:       else:
-; CHECK-NEXT:    [[PTR_PHI_ELSE:%.*]] = phi i64* [ [[TMP2]], [[COND_STORE]] ], [ [[P]], [[TMP0:%.*]] ]
-; CHECK-NEXT:    br i1 false, label [[COND_STORE1:%.*]], label [[ELSE2:%.*]]
-; CHECK:       cond.store1:
-; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x i64> [[DATA]], i64 1
-; CHECK-NEXT:    store i64 [[TMP3]], i64* [[PTR_PHI_ELSE]], align 1
-; CHECK-NEXT:    br label [[ELSE2]]
-; CHECK:       else2:
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.masked.compressstore.v2i64.p0v2i64(<2 x i64> %data, i64* %p, <2 x i1> <i1 false, i1 false>)
@@ -71,20 +52,9 @@ define void @scalarize_v2i64_zero_mask(i64* %p, <2 x i64> %data) {
 
 define void @scalarize_v2i64_const_mask(i64* %p, <2 x i64> %data) {
 ; CHECK-LABEL: @scalarize_v2i64_const_mask(
-; CHECK-NEXT:    br i1 false, label [[COND_STORE:%.*]], label [[ELSE:%.*]]
-; CHECK:       cond.store:
-; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x i64> [[DATA:%.*]], i64 0
-; CHECK-NEXT:    store i64 [[TMP1]], i64* [[P:%.*]], align 1
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, i64* [[P]], i32 1
-; CHECK-NEXT:    br label [[ELSE]]
-; CHECK:       else:
-; CHECK-NEXT:    [[PTR_PHI_ELSE:%.*]] = phi i64* [ [[TMP2]], [[COND_STORE]] ], [ [[P]], [[TMP0:%.*]] ]
-; CHECK-NEXT:    br i1 true, label [[COND_STORE1:%.*]], label [[ELSE2:%.*]]
-; CHECK:       cond.store1:
-; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x i64> [[DATA]], i64 1
-; CHECK-NEXT:    store i64 [[TMP3]], i64* [[PTR_PHI_ELSE]], align 1
-; CHECK-NEXT:    br label [[ELSE2]]
-; CHECK:       else2:
+; CHECK-NEXT:    [[ELT1:%.*]] = extractelement <2 x i64> [[DATA:%.*]], i64 1
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i64, i64* [[P:%.*]], i32 0
+; CHECK-NEXT:    store i64 [[ELT1]], i64* [[TMP1]], align 1
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.masked.compressstore.v2i64.p0v2i64(<2 x i64> %data, i64* %p, <2 x i1> <i1 false, i1 true>)
diff --git a/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-expandload.ll b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-expandload.ll
index 3b8c7aa18..f095adaeb 100644
--- a/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-expandload.ll
+++ b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-expandload.ll
@@ -3,24 +3,27 @@
 
 define <2 x i64> @scalarize_v2i64(i64* %p, <2 x i1> %mask, <2 x i64> %passthru) {
 ; CHECK-LABEL: @scalarize_v2i64(
-; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x i1> [[MASK:%.*]], i64 0
-; CHECK-NEXT:    br i1 [[TMP1]], label [[COND_LOAD:%.*]], label [[ELSE:%.*]]
+; CHECK-NEXT:    [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK:%.*]] to i2
+; CHECK-NEXT:    [[TMP1:%.*]] = and i2 [[SCALAR_MASK]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i2 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[TMP2]], label [[COND_LOAD:%.*]], label [[ELSE:%.*]]
 ; CHECK:       cond.load:
-; CHECK-NEXT:    [[TMP2:%.*]] = load i64, i64* [[P:%.*]], align 1
-; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x i64> [[PASSTHRU:%.*]], i64 [[TMP2]], i64 0
-; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, i64* [[P]], i32 1
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, i64* [[P:%.*]], align 1
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <2 x i64> [[PASSTHRU:%.*]], i64 [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, i64* [[P]], i32 1
 ; CHECK-NEXT:    br label [[ELSE]]
 ; CHECK:       else:
-; CHECK-NEXT:    [[RES_PHI_ELSE:%.*]] = phi <2 x i64> [ [[TMP3]], [[COND_LOAD]] ], [ [[PASSTHRU]], [[TMP0:%.*]] ]
-; CHECK-NEXT:    [[PTR_PHI_ELSE:%.*]] = phi i64* [ [[TMP4]], [[COND_LOAD]] ], [ [[P]], [[TMP0]] ]
-; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i1> [[MASK]], i64 1
-; CHECK-NEXT:    br i1 [[TMP5]], label [[COND_LOAD1:%.*]], label [[ELSE2:%.*]]
+; CHECK-NEXT:    [[RES_PHI_ELSE:%.*]] = phi <2 x i64> [ [[TMP4]], [[COND_LOAD]] ], [ [[PASSTHRU]], [[TMP0:%.*]] ]
+; CHECK-NEXT:    [[PTR_PHI_ELSE:%.*]] = phi i64* [ [[TMP5]], [[COND_LOAD]] ], [ [[P]], [[TMP0]] ]
+; CHECK-NEXT:    [[TMP6:%.*]] = and i2 [[SCALAR_MASK]], -2
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp ne i2 [[TMP6]], 0
+; CHECK-NEXT:    br i1 [[TMP7]], label [[COND_LOAD1:%.*]], label [[ELSE2:%.*]]
 ; CHECK:       cond.load1:
-; CHECK-NEXT:    [[TMP6:%.*]] = load i64, i64* [[PTR_PHI_ELSE]], align 1
-; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <2 x i64> [[RES_PHI_ELSE]], i64 [[TMP6]], i64 1
+; CHECK-NEXT:    [[TMP8:%.*]] = load i64, i64* [[PTR_PHI_ELSE]], align 1
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <2 x i64> [[RES_PHI_ELSE]], i64 [[TMP8]], i64 1
 ; CHECK-NEXT:    br label [[ELSE2]]
 ; CHECK:       else2:
-; CHECK-NEXT:    [[RES_PHI_ELSE3:%.*]] = phi <2 x i64> [ [[TMP7]], [[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], [[ELSE]] ]
+; CHECK-NEXT:    [[RES_PHI_ELSE3:%.*]] = phi <2 x i64> [ [[TMP9]], [[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], [[ELSE]] ]
 ; CHECK-NEXT:    ret <2 x i64> [[RES_PHI_ELSE3]]
 ;
   %ret = call <2 x i64> @llvm.masked.expandload.v2i64.p0v2i64(i64* %p, <2 x i1> %mask, <2 x i64> %passthru)
@@ -29,23 +32,13 @@ define <2 x i64> @scalarize_v2i64(i64* %p, <2 x i1> %mask, <2 x i64> %passthru)
 
 define <2 x i64> @scalarize_v2i64_ones_mask(i64* %p, <2 x i64> %passthru) {
 ; CHECK-LABEL: @scalarize_v2i64_ones_mask(
-; CHECK-NEXT:    br i1 true, label [[COND_LOAD:%.*]], label [[ELSE:%.*]]
-; CHECK:       cond.load:
-; CHECK-NEXT:    [[TMP1:%.*]] = load i64, i64* [[P:%.*]], align 1
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x i64> [[PASSTHRU:%.*]], i64 [[TMP1]], i64 0
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i64, i64* [[P]], i32 1
-; CHECK-NEXT:    br label [[ELSE]]
-; CHECK:       else:
-; CHECK-NEXT:    [[RES_PHI_ELSE:%.*]] = phi <2 x i64> [ [[TMP2]], [[COND_LOAD]] ], [ [[PASSTHRU]], [[TMP0:%.*]] ]
-; CHECK-NEXT:    [[PTR_PHI_ELSE:%.*]] = phi i64* [ [[TMP3]], [[COND_LOAD]] ], [ [[P]], [[TMP0]] ]
-; CHECK-NEXT:    br i1 true, label [[COND_LOAD1:%.*]], label [[ELSE2:%.*]]
-; CHECK:       cond.load1:
-; CHECK-NEXT:    [[TMP4:%.*]] = load i64, i64* [[PTR_PHI_ELSE]], align 1
-; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <2 x i64> [[RES_PHI_ELSE]], i64 [[TMP4]], i64 1
-; CHECK-NEXT:    br label [[ELSE2]]
-; CHECK:       else2:
-; CHECK-NEXT:    [[RES_PHI_ELSE3:%.*]] = phi <2 x i64> [ [[TMP5]], [[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], [[ELSE]] ]
-; CHECK-NEXT:    ret <2 x i64> [[RES_PHI_ELSE3]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i64, i64* [[P:%.*]], i32 0
+; CHECK-NEXT:    [[LOAD0:%.*]] = load i64, i64* [[TMP1]], align 1
+; CHECK-NEXT:    [[RES0:%.*]] = insertelement <2 x i64> [[PASSTHRU:%.*]], i64 [[LOAD0]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, i64* [[P]], i32 1
+; CHECK-NEXT:    [[LOAD1:%.*]] = load i64, i64* [[TMP2]], align 1
+; CHECK-NEXT:    [[RES1:%.*]] = insertelement <2 x i64> [[RES0]], i64 [[LOAD1]], i64 1
+; CHECK-NEXT:    ret <2 x i64> [[RES1]]
 ;
   %ret = call <2 x i64> @llvm.masked.expandload.v2i64.p0v2i64(i64* %p, <2 x i1> <i1 true, i1 true>, <2 x i64> %passthru)
   ret <2 x i64> %ret
@@ -53,23 +46,7 @@ define <2 x i64> @scalarize_v2i64_ones_mask(i64* %p, <2 x i64> %passthru) {
 
 define <2 x i64> @scalarize_v2i64_zero_mask(i64* %p, <2 x i64> %passthru) {
 ; CHECK-LABEL: @scalarize_v2i64_zero_mask(
-; CHECK-NEXT:    br i1 false, label [[COND_LOAD:%.*]], label [[ELSE:%.*]]
-; CHECK:       cond.load:
-; CHECK-NEXT:    [[TMP1:%.*]] = load i64, i64* [[P:%.*]], align 1
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x i64> [[PASSTHRU:%.*]], i64 [[TMP1]], i64 0
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i64, i64* [[P]], i32 1
-; CHECK-NEXT:    br label [[ELSE]]
-; CHECK:       else:
-; CHECK-NEXT:    [[RES_PHI_ELSE:%.*]] = phi <2 x i64> [ [[TMP2]], [[COND_LOAD]] ], [ [[PASSTHRU]], [[TMP0:%.*]] ]
-; CHECK-NEXT:    [[PTR_PHI_ELSE:%.*]] = phi i64* [ [[TMP3]], [[COND_LOAD]] ], [ [[P]], [[TMP0]] ]
-; CHECK-NEXT:    br i1 false, label [[COND_LOAD1:%.*]], label [[ELSE2:%.*]]
-; CHECK:       cond.load1:
-; CHECK-NEXT:    [[TMP4:%.*]] = load i64, i64* [[PTR_PHI_ELSE]], align 1
-; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <2 x i64> [[RES_PHI_ELSE]], i64 [[TMP4]], i64 1
-; CHECK-NEXT:    br label [[ELSE2]]
-; CHECK:       else2:
-; CHECK-NEXT:    [[RES_PHI_ELSE3:%.*]] = phi <2 x i64> [ [[TMP5]], [[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], [[ELSE]] ]
-; CHECK-NEXT:    ret <2 x i64> [[RES_PHI_ELSE3]]
+; CHECK-NEXT:    ret <2 x i64> [[PASSTHRU:%.*]]
 ;
   %ret = call <2 x i64> @llvm.masked.expandload.v2i64.p0v2i64(i64* %p, <2 x i1> <i1 false, i1 false>, <2 x i64> %passthru)
   ret <2 x i64> %ret
@@ -77,23 +54,10 @@ define <2 x i64> @scalarize_v2i64_zero_mask(i64* %p, <2 x i64> %passthru) {
 
 define <2 x i64> @scalarize_v2i64_const_mask(i64* %p, <2 x i64> %passthru) {
 ; CHECK-LABEL: @scalarize_v2i64_const_mask(
-; CHECK-NEXT:    br i1 false, label [[COND_LOAD:%.*]], label [[ELSE:%.*]]
-; CHECK:       cond.load:
-; CHECK-NEXT:    [[TMP1:%.*]] = load i64, i64* [[P:%.*]], align 1
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x i64> [[PASSTHRU:%.*]], i64 [[TMP1]], i64 0
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i64, i64* [[P]], i32 1
-; CHECK-NEXT:    br label [[ELSE]]
-; CHECK:       else:
-; CHECK-NEXT:    [[RES_PHI_ELSE:%.*]] = phi <2 x i64> [ [[TMP2]], [[COND_LOAD]] ], [ [[PASSTHRU]], [[TMP0:%.*]] ]
-; CHECK-NEXT:    [[PTR_PHI_ELSE:%.*]] = phi i64* [ [[TMP3]], [[COND_LOAD]] ], [ [[P]], [[TMP0]] ]
-; CHECK-NEXT:    br i1 true, label [[COND_LOAD1:%.*]], label [[ELSE2:%.*]]
-; CHECK:       cond.load1:
-; CHECK-NEXT:    [[TMP4:%.*]] = load i64, i64* [[PTR_PHI_ELSE]], align 1
-; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <2 x i64> [[RES_PHI_ELSE]], i64 [[TMP4]], i64 1
-; CHECK-NEXT:    br label [[ELSE2]]
-; CHECK:       else2:
-; CHECK-NEXT:    [[RES_PHI_ELSE3:%.*]] = phi <2 x i64> [ [[TMP5]], [[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], [[ELSE]] ]
-; CHECK-NEXT:    ret <2 x i64> [[RES_PHI_ELSE3]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i64, i64* [[P:%.*]], i32 0
+; CHECK-NEXT:    [[LOAD1:%.*]] = load i64, i64* [[TMP1]], align 1
+; CHECK-NEXT:    [[RES1:%.*]] = insertelement <2 x i64> [[PASSTHRU:%.*]], i64 [[LOAD1]], i64 1
+; CHECK-NEXT:    ret <2 x i64> [[RES1]]
 ;
   %ret = call <2 x i64> @llvm.masked.expandload.v2i64.p0v2i64(i64* %p, <2 x i1> <i1 false, i1 true>, <2 x i64> %passthru)
   ret <2 x i64> %ret
diff --git a/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-gather.ll b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-gather.ll
index c294cbea1..20791e7ee 100644
--- a/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-gather.ll
+++ b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-gather.ll
@@ -3,8 +3,10 @@
 
 define <2 x i64> @scalarize_v2i64(<2 x i64*> %p, <2 x i1> %mask, <2 x i64> %passthru) {
 ; CHECK-LABEL: @scalarize_v2i64(
-; CHECK-NEXT:    [[MASK0:%.*]] = extractelement <2 x i1> [[MASK:%.*]], i64 0
-; CHECK-NEXT:    br i1 [[MASK0]], label [[COND_LOAD:%.*]], label [[ELSE:%.*]]
+; CHECK-NEXT:    [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK:%.*]] to i2
+; CHECK-NEXT:    [[TMP1:%.*]] = and i2 [[SCALAR_MASK]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i2 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[TMP2]], label [[COND_LOAD:%.*]], label [[ELSE:%.*]]
 ; CHECK:       cond.load:
 ; CHECK-NEXT:    [[PTR0:%.*]] = extractelement <2 x i64*> [[P:%.*]], i64 0
 ; CHECK-NEXT:    [[LOAD0:%.*]] = load i64, i64* [[PTR0]], align 8
@@ -12,8 +14,9 @@ define <2 x i64> @scalarize_v2i64(<2 x i64*> %p, <2 x i1> %mask, <2 x i64> %pass
 ; CHECK-NEXT:    br label [[ELSE]]
 ; CHECK:       else:
 ; CHECK-NEXT:    [[RES_PHI_ELSE:%.*]] = phi <2 x i64> [ [[RES0]], [[COND_LOAD]] ], [ [[PASSTHRU]], [[TMP0:%.*]] ]
-; CHECK-NEXT:    [[MASK1:%.*]] = extractelement <2 x i1> [[MASK]], i64 1
-; CHECK-NEXT:    br i1 [[MASK1]], label [[COND_LOAD1:%.*]], label [[ELSE2:%.*]]
+; CHECK-NEXT:    [[TMP3:%.*]] = and i2 [[SCALAR_MASK]], -2
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne i2 [[TMP3]], 0
+; CHECK-NEXT:    br i1 [[TMP4]], label [[COND_LOAD1:%.*]], label [[ELSE2:%.*]]
 ; CHECK:       cond.load1:
 ; CHECK-NEXT:    [[PTR1:%.*]] = extractelement <2 x i64*> [[P]], i64 1
 ; CHECK-NEXT:    [[LOAD1:%.*]] = load i64, i64* [[PTR1]], align 8
diff --git a/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-load.ll b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-load.ll
index a96ff8475..c3820928a 100644
--- a/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-load.ll
+++ b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-load.ll
@@ -4,24 +4,27 @@
 define <2 x i64> @scalarize_v2i64(<2 x i64>* %p, <2 x i1> %mask, <2 x i64> %passthru) {
 ; CHECK-LABEL: @scalarize_v2i64(
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64>* [[P:%.*]] to i64*
-; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x i1> [[MASK:%.*]], i64 0
-; CHECK-NEXT:    br i1 [[TMP2]], label [[COND_LOAD:%.*]], label [[ELSE:%.*]]
+; CHECK-NEXT:    [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK:%.*]] to i2
+; CHECK-NEXT:    [[TMP2:%.*]] = and i2 [[SCALAR_MASK]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne i2 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[TMP3]], label [[COND_LOAD:%.*]], label [[ELSE:%.*]]
 ; CHECK:       cond.load:
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i64, i64* [[TMP1]], i32 0
-; CHECK-NEXT:    [[TMP4:%.*]] = load i64, i64* [[TMP3]], align 8
-; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <2 x i64> [[PASSTHRU:%.*]], i64 [[TMP4]], i64 0
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, i64* [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = load i64, i64* [[TMP4]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i64> [[PASSTHRU:%.*]], i64 [[TMP5]], i64 0
 ; CHECK-NEXT:    br label [[ELSE]]
 ; CHECK:       else:
-; CHECK-NEXT:    [[RES_PHI_ELSE:%.*]] = phi <2 x i64> [ [[TMP5]], [[COND_LOAD]] ], [ [[PASSTHRU]], [[TMP0:%.*]] ]
-; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i1> [[MASK]], i64 1
-; CHECK-NEXT:    br i1 [[TMP6]], label [[COND_LOAD1:%.*]], label [[ELSE2:%.*]]
+; CHECK-NEXT:    [[RES_PHI_ELSE:%.*]] = phi <2 x i64> [ [[TMP6]], [[COND_LOAD]] ], [ [[PASSTHRU]], [[TMP0:%.*]] ]
+; CHECK-NEXT:    [[TMP7:%.*]] = and i2 [[SCALAR_MASK]], -2
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne i2 [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[TMP8]], label [[COND_LOAD1:%.*]], label [[ELSE2:%.*]]
 ; CHECK:       cond.load1:
-; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, i64* [[TMP1]], i32 1
-; CHECK-NEXT:    [[TMP8:%.*]] = load i64, i64* [[TMP7]], align 8
-; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <2 x i64> [[RES_PHI_ELSE]], i64 [[TMP8]], i64 1
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, i64* [[TMP1]], i32 1
+; CHECK-NEXT:    [[TMP10:%.*]] = load i64, i64* [[TMP9]], align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <2 x i64> [[RES_PHI_ELSE]], i64 [[TMP10]], i64 1
 ; CHECK-NEXT:    br label [[ELSE2]]
 ; CHECK:       else2:
-; CHECK-NEXT:    [[RES_PHI_ELSE3:%.*]] = phi <2 x i64> [ [[TMP9]], [[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], [[ELSE]] ]
+; CHECK-NEXT:    [[RES_PHI_ELSE3:%.*]] = phi <2 x i64> [ [[TMP11]], [[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], [[ELSE]] ]
 ; CHECK-NEXT:    ret <2 x i64> [[RES_PHI_ELSE3]]
 ;
   %ret = call <2 x i64> @llvm.masked.load.v2i64.p0v2i64(<2 x i64>* %p, i32 128, <2 x i1> %mask, <2 x i64> %passthru)
@@ -62,24 +65,27 @@ define <2 x i64> @scalarize_v2i64_const_mask(<2 x i64>* %p, <2 x i64> %passthru)
 define <2 x i24> @scalarize_v2i24(<2 x i24>* %p, <2 x i1> %mask, <2 x i24> %passthru) {
 ; CHECK-LABEL: @scalarize_v2i24(
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i24>* [[P:%.*]] to i24*
-; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x i1> [[MASK:%.*]], i64 0
-; CHECK-NEXT:    br i1 [[TMP2]], label [[COND_LOAD:%.*]], label [[ELSE:%.*]]
+; CHECK-NEXT:    [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK:%.*]] to i2
+; CHECK-NEXT:    [[TMP2:%.*]] = and i2 [[SCALAR_MASK]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne i2 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[TMP3]], label [[COND_LOAD:%.*]], label [[ELSE:%.*]]
 ; CHECK:       cond.load:
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i24, i24* [[TMP1]], i32 0
-; CHECK-NEXT:    [[TMP4:%.*]] = load i24, i24* [[TMP3]], align 1
-; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <2 x i24> [[PASSTHRU:%.*]], i24 [[TMP4]], i64 0
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i24, i24* [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = load i24, i24* [[TMP4]], align 1
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i24> [[PASSTHRU:%.*]], i24 [[TMP5]], i64 0
 ; CHECK-NEXT:    br label [[ELSE]]
 ; CHECK:       else:
-; CHECK-NEXT:    [[RES_PHI_ELSE:%.*]] = phi <2 x i24> [ [[TMP5]], [[COND_LOAD]] ], [ [[PASSTHRU]], [[TMP0:%.*]] ]
-; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i1> [[MASK]], i64 1
-; CHECK-NEXT:    br i1 [[TMP6]], label [[COND_LOAD1:%.*]], label [[ELSE2:%.*]]
+; CHECK-NEXT:    [[RES_PHI_ELSE:%.*]] = phi <2 x i24> [ [[TMP6]], [[COND_LOAD]] ], [ [[PASSTHRU]], [[TMP0:%.*]] ]
+; CHECK-NEXT:    [[TMP7:%.*]] = and i2 [[SCALAR_MASK]], -2
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne i2 [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[TMP8]], label [[COND_LOAD1:%.*]], label [[ELSE2:%.*]]
 ; CHECK:       cond.load1:
-; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i24, i24* [[TMP1]], i32 1
-; CHECK-NEXT:    [[TMP8:%.*]] = load i24, i24* [[TMP7]], align 1
-; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <2 x i24> [[RES_PHI_ELSE]], i24 [[TMP8]], i64 1
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i24, i24* [[TMP1]], i32 1
+; CHECK-NEXT:    [[TMP10:%.*]] = load i24, i24* [[TMP9]], align 1
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <2 x i24> [[RES_PHI_ELSE]], i24 [[TMP10]], i64 1
 ; CHECK-NEXT:    br label [[ELSE2]]
 ; CHECK:       else2:
-; CHECK-NEXT:    [[RES_PHI_ELSE3:%.*]] = phi <2 x i24> [ [[TMP9]], [[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], [[ELSE]] ]
+; CHECK-NEXT:    [[RES_PHI_ELSE3:%.*]] = phi <2 x i24> [ [[TMP11]], [[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], [[ELSE]] ]
 ; CHECK-NEXT:    ret <2 x i24> [[RES_PHI_ELSE3]]
 ;
   %ret = call <2 x i24> @llvm.masked.load.v2i24.p0v2i24(<2 x i24>* %p, i32 8, <2 x i1> %mask, <2 x i24> %passthru)
@@ -90,24 +96,27 @@ define <2 x i24> @scalarize_v2i24(<2 x i24>* %p, <2 x i1> %mask, <2 x i24> %pass
 define <2 x i48> @scalarize_v2i48(<2 x i48>* %p, <2 x i1> %mask, <2 x i48> %passthru) {
 ; CHECK-LABEL: @scalarize_v2i48(
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i48>* [[P:%.*]] to i48*
-; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x i1> [[MASK:%.*]], i64 0
-; CHECK-NEXT:    br i1 [[TMP2]], label [[COND_LOAD:%.*]], label [[ELSE:%.*]]
+; CHECK-NEXT:    [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK:%.*]] to i2
+; CHECK-NEXT:    [[TMP2:%.*]] = and i2 [[SCALAR_MASK]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne i2 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[TMP3]], label [[COND_LOAD:%.*]], label [[ELSE:%.*]]
 ; CHECK:       cond.load:
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i48, i48* [[TMP1]], i32 0
-; CHECK-NEXT:    [[TMP4:%.*]] = load i48, i48* [[TMP3]], align 2
-; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <2 x i48> [[PASSTHRU:%.*]], i48 [[TMP4]], i64 0
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i48, i48* [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = load i48, i48* [[TMP4]], align 2
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i48> [[PASSTHRU:%.*]], i48 [[TMP5]], i64 0
 ; CHECK-NEXT:    br label [[ELSE]]
 ; CHECK:       else:
-; CHECK-NEXT:    [[RES_PHI_ELSE:%.*]] = phi <2 x i48> [ [[TMP5]], [[COND_LOAD]] ], [ [[PASSTHRU]], [[TMP0:%.*]] ]
-; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i1> [[MASK]], i64 1
-; CHECK-NEXT:    br i1 [[TMP6]], label [[COND_LOAD1:%.*]], label [[ELSE2:%.*]]
+; CHECK-NEXT:    [[RES_PHI_ELSE:%.*]] = phi <2 x i48> [ [[TMP6]], [[COND_LOAD]] ], [ [[PASSTHRU]], [[TMP0:%.*]] ]
+; CHECK-NEXT:    [[TMP7:%.*]] = and i2 [[SCALAR_MASK]], -2
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne i2 [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[TMP8]], label [[COND_LOAD1:%.*]], label [[ELSE2:%.*]]
 ; CHECK:       cond.load1:
-; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i48, i48* [[TMP1]], i32 1
-; CHECK-NEXT:    [[TMP8:%.*]] = load i48, i48* [[TMP7]], align 2
-; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <2 x i48> [[RES_PHI_ELSE]], i48 [[TMP8]], i64 1
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i48, i48* [[TMP1]], i32 1
+; CHECK-NEXT:    [[TMP10:%.*]] = load i48, i48* [[TMP9]], align 2
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <2 x i48> [[RES_PHI_ELSE]], i48 [[TMP10]], i64 1
 ; CHECK-NEXT:    br label [[ELSE2]]
 ; CHECK:       else2:
-; CHECK-NEXT:    [[RES_PHI_ELSE3:%.*]] = phi <2 x i48> [ [[TMP9]], [[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], [[ELSE]] ]
+; CHECK-NEXT:    [[RES_PHI_ELSE3:%.*]] = phi <2 x i48> [ [[TMP11]], [[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], [[ELSE]] ]
 ; CHECK-NEXT:    ret <2 x i48> [[RES_PHI_ELSE3]]
 ;
   %ret = call <2 x i48> @llvm.masked.load.v2i48.p0v2i48(<2 x i48>* %p, i32 16, <2 x i1> %mask, <2 x i48> %passthru)
diff --git a/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-scatter.ll b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-scatter.ll
new file mode 100644
index 000000000..8b0dad8cf
--- /dev/null
+++ b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-scatter.ll
@@ -0,0 +1,64 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S %s -scalarize-masked-mem-intrin -mtriple=x86_64-linux-gnu | FileCheck %s
+
+define void @scalarize_v2i64(<2 x i64*> %p, <2 x i1> %mask, <2 x i64> %value) {
+; CHECK-LABEL: @scalarize_v2i64(
+; CHECK-NEXT:    [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK:%.*]] to i2
+; CHECK-NEXT:    [[TMP1:%.*]] = and i2 [[SCALAR_MASK]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i2 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[TMP2]], label [[COND_STORE:%.*]], label [[ELSE:%.*]]
+; CHECK:       cond.store:
+; CHECK-NEXT:    [[ELT0:%.*]] = extractelement <2 x i64> [[VALUE:%.*]], i64 0
+; CHECK-NEXT:    [[PTR0:%.*]] = extractelement <2 x i64*> [[P:%.*]], i64 0
+; CHECK-NEXT:    store i64 [[ELT0]], i64* [[PTR0]], align 8
+; CHECK-NEXT:    br label [[ELSE]]
+; CHECK:       else:
+; CHECK-NEXT:    [[TMP3:%.*]] = and i2 [[SCALAR_MASK]], -2
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne i2 [[TMP3]], 0
+; CHECK-NEXT:    br i1 [[TMP4]], label [[COND_STORE1:%.*]], label [[ELSE2:%.*]]
+; CHECK:       cond.store1:
+; CHECK-NEXT:    [[ELT1:%.*]] = extractelement <2 x i64> [[VALUE]], i64 1
+; CHECK-NEXT:    [[PTR1:%.*]] = extractelement <2 x i64*> [[P]], i64 1
+; CHECK-NEXT:    store i64 [[ELT1]], i64* [[PTR1]], align 8
+; CHECK-NEXT:    br label [[ELSE2]]
+; CHECK:       else2:
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.masked.scatter.v2i64.v2p0i64(<2 x i64> %value, <2 x i64*> %p, i32 8, <2 x i1> %mask)
+  ret void
+}
+
+define void @scalarize_v2i64_ones_mask(<2 x i64*> %p, <2 x i64> %value) {
+; CHECK-LABEL: @scalarize_v2i64_ones_mask(
+; CHECK-NEXT:    [[ELT0:%.*]] = extractelement <2 x i64> [[VALUE:%.*]], i64 0
+; CHECK-NEXT:    [[PTR0:%.*]] = extractelement <2 x i64*> [[P:%.*]], i64 0
+; CHECK-NEXT:    store i64 [[ELT0]], i64* [[PTR0]], align 8
+; CHECK-NEXT:    [[ELT1:%.*]] = extractelement <2 x i64> [[VALUE]], i64 1
+; CHECK-NEXT:    [[PTR1:%.*]] = extractelement <2 x i64*> [[P]], i64 1
+; CHECK-NEXT:    store i64 [[ELT1]], i64* [[PTR1]], align 8
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.masked.scatter.v2i64.v2p0i64(<2 x i64> %value, <2 x i64*> %p, i32 8, <2 x i1> <i1 true, i1 true>)
+  ret void
+}
+
+define void @scalarize_v2i64_zero_mask(<2 x i64*> %p, <2 x i64> %value) {
+; CHECK-LABEL: @scalarize_v2i64_zero_mask(
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.masked.scatter.v2i64.v2p0i64(<2 x i64> %value, <2 x i64*> %p, i32 8, <2 x i1> <i1 false, i1 false>)
+  ret void
+}
+
+define void @scalarize_v2i64_const_mask(<2 x i64*> %p, <2 x i64> %value) {
+; CHECK-LABEL: @scalarize_v2i64_const_mask(
+; CHECK-NEXT:    [[ELT1:%.*]] = extractelement <2 x i64> [[VALUE:%.*]], i64 1
+; CHECK-NEXT:    [[PTR1:%.*]] = extractelement <2 x i64*> [[P:%.*]], i64 1
+; CHECK-NEXT:    store i64 [[ELT1]], i64* [[PTR1]], align 8
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.masked.scatter.v2i64.v2p0i64(<2 x i64> %value, <2 x i64*> %p, i32 8, <2 x i1> <i1 false, i1 true>)
+  ret void
+}
+
+declare void @llvm.masked.scatter.v2i64.v2p0i64(<2 x i64>, <2 x i64*>, i32, <2 x i1>)
diff --git a/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-store.ll b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-store.ll
index 9c8404573..b9fbeeabf 100644
--- a/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-store.ll
+++ b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-store.ll
@@ -4,20 +4,23 @@
 define void @scalarize_v2i64(<2 x i64>* %p, <2 x i1> %mask, <2 x i64> %data) {
 ; CHECK-LABEL: @scalarize_v2i64(
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64>* [[P:%.*]] to i64*
-; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x i1> [[MASK:%.*]], i64 0
-; CHECK-NEXT:    br i1 [[TMP2]], label [[COND_STORE:%.*]], label [[ELSE:%.*]]
+; CHECK-NEXT:    [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK:%.*]] to i2
+; CHECK-NEXT:    [[TMP2:%.*]] = and i2 [[SCALAR_MASK]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne i2 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[TMP3]], label [[COND_STORE:%.*]], label [[ELSE:%.*]]
 ; CHECK:       cond.store:
-; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x i64> [[DATA:%.*]], i64 0
-; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, i64* [[TMP1]], i32 0
-; CHECK-NEXT:    store i64 [[TMP3]], i64* [[TMP4]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[DATA:%.*]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, i64* [[TMP1]], i32 0
+; CHECK-NEXT:    store i64 [[TMP4]], i64* [[TMP5]], align 8
 ; CHECK-NEXT:    br label [[ELSE]]
 ; CHECK:       else:
-; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i1> [[MASK]], i64 1
-; CHECK-NEXT:    br i1 [[TMP5]], label [[COND_STORE1:%.*]], label [[ELSE2:%.*]]
+; CHECK-NEXT:    [[TMP6:%.*]] = and i2 [[SCALAR_MASK]], -2
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp ne i2 [[TMP6]], 0
+; CHECK-NEXT:    br i1 [[TMP7]], label [[COND_STORE1:%.*]], label [[ELSE2:%.*]]
 ; CHECK:       cond.store1:
-; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i64> [[DATA]], i64 1
-; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, i64* [[TMP1]], i32 1
-; CHECK-NEXT:    store i64 [[TMP6]], i64* [[TMP7]], align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x i64> [[DATA]], i64 1
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, i64* [[TMP1]], i32 1
+; CHECK-NEXT:    store i64 [[TMP8]], i64* [[TMP9]], align 8
 ; CHECK-NEXT:    br label [[ELSE2]]
 ; CHECK:       else2:
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/Scalarizer/dbginfo.ll b/llvm/test/Transforms/Scalarizer/dbginfo.ll
index 37452ec70..a5ae12bd8 100644
--- a/llvm/test/Transforms/Scalarizer/dbginfo.ll
+++ b/llvm/test/Transforms/Scalarizer/dbginfo.ll
@@ -51,7 +51,7 @@ entry:
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.value(metadata, metadata, metadata) #1
 
-attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/Transforms/Scalarizer/intrinsics.ll b/llvm/test/Transforms/Scalarizer/intrinsics.ll
index dcd44fa37..b277a5bf3 100644
--- a/llvm/test/Transforms/Scalarizer/intrinsics.ll
+++ b/llvm/test/Transforms/Scalarizer/intrinsics.ll
@@ -23,6 +23,7 @@ declare <2 x float> @llvm.powi.v2f32(<2 x float>, i32)
 
 ; Binary int plus constant scalar operand
 declare <2 x i32> @llvm.smul.fix.sat.v2i32(<2 x i32>, <2 x i32>, i32)
+declare <2 x i32> @llvm.umul.fix.sat.v2i32(<2 x i32>, <2 x i32>, i32)
 
 
 ; CHECK-LABEL: @scalarize_sqrt_v2f32(
@@ -123,3 +124,14 @@ define <2 x i32> @scalarize_smul_fix_sat_v2i32(<2 x i32> %x) #0 {
   %smulfixsat = call <2 x i32> @llvm.smul.fix.sat.v2i32(<2 x i32> %x, <2 x i32> <i32 5, i32 19>, i32 31)
   ret <2 x i32> %smulfixsat
 }
+
+; CHECK-LABEL: @scalarize_umul_fix_sat_v2i32(
+; CHECK: %umulfixsat.i0 = call i32 @llvm.umul.fix.sat.i32(i32 %x.i0, i32 5, i32 31)
+; CHECK: %umulfixsat.i1 = call i32 @llvm.umul.fix.sat.i32(i32 %x.i1, i32 19, i32 31)
+; CHECK: %umulfixsat.upto0 = insertelement <2 x i32> undef, i32 %umulfixsat.i0, i32 0
+; CHECK: %umulfixsat = insertelement <2 x i32> %umulfixsat.upto0, i32 %umulfixsat.i1, i32 1
+; CHECK: ret <2 x i32> %umulfixsat
+define <2 x i32> @scalarize_umul_fix_sat_v2i32(<2 x i32> %x) #0 {
+  %umulfixsat = call <2 x i32> @llvm.umul.fix.sat.v2i32(<2 x i32> %x, <2 x i32> <i32 5, i32 19>, i32 31)
+  ret <2 x i32> %umulfixsat
+}
diff --git a/llvm/test/Transforms/Scalarizer/phi-unreachable-pred.ll b/llvm/test/Transforms/Scalarizer/phi-unreachable-pred.ll
new file mode 100644
index 000000000..1de1f6509
--- /dev/null
+++ b/llvm/test/Transforms/Scalarizer/phi-unreachable-pred.ll
@@ -0,0 +1,98 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -scalarizer -S -o - | FileCheck %s
+
+define i16 @f1() {
+; CHECK-LABEL: @f1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_END:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INSERT:%.*]] = insertelement <4 x i16> [[INSERT]], i16 ptrtoint (i16 ()* @f1 to i16), i32 0
+; CHECK-NEXT:    br label [[FOR_COND:%.*]]
+; CHECK:       for.cond:
+; CHECK-NEXT:    br i1 undef, label [[FOR_BODY:%.*]], label [[FOR_END]]
+; CHECK:       for.end:
+; CHECK-NEXT:    [[PHI_I0:%.*]] = phi i16 [ 1, [[ENTRY:%.*]] ], [ undef, [[FOR_COND]] ]
+; CHECK-NEXT:    [[PHI_I1:%.*]] = phi i16 [ 1, [[ENTRY]] ], [ undef, [[FOR_COND]] ]
+; CHECK-NEXT:    [[PHI_I2:%.*]] = phi i16 [ 1, [[ENTRY]] ], [ undef, [[FOR_COND]] ]
+; CHECK-NEXT:    [[PHI_I3:%.*]] = phi i16 [ 1, [[ENTRY]] ], [ undef, [[FOR_COND]] ]
+; CHECK-NEXT:    [[PHI_UPTO0:%.*]] = insertelement <4 x i16> undef, i16 [[PHI_I0]], i32 0
+; CHECK-NEXT:    [[PHI_UPTO1:%.*]] = insertelement <4 x i16> [[PHI_UPTO0]], i16 [[PHI_I1]], i32 1
+; CHECK-NEXT:    [[PHI_UPTO2:%.*]] = insertelement <4 x i16> [[PHI_UPTO1]], i16 [[PHI_I2]], i32 2
+; CHECK-NEXT:    [[PHI:%.*]] = insertelement <4 x i16> [[PHI_UPTO2]], i16 [[PHI_I3]], i32 3
+; CHECK-NEXT:    [[EXTRACT:%.*]] = extractelement <4 x i16> [[PHI]], i32 0
+; CHECK-NEXT:    ret i16 [[EXTRACT]]
+;
+entry:
+  br label %for.end
+
+for.body:
+  %insert = insertelement <4 x i16> %insert, i16 ptrtoint (i16 () * @f1 to i16), i32 0
+  br label %for.cond
+
+for.cond:
+  br i1 undef, label %for.body, label %for.end
+
+for.end:
+  ; opt used to hang when scalarizing this code. When scattering %insert we
+  ; need to analyze the insertelement in the unreachable-from-entry block
+  ; for.body. Note that the insertelement instruction depends on itself, and
+  ; this kind of IR is not allowed in reachable-from-entry blocks.
+  %phi = phi <4 x i16> [ <i16 1, i16 1, i16 1, i16 1>, %entry ], [ %insert, %for.cond ]
+  %extract = extractelement <4 x i16> %phi, i32 0
+  ret i16 %extract
+}
+
+define void @f2() {
+; CHECK-LABEL: @f2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    br i1 undef, label [[IF_THEN:%.*]], label [[IF_END8:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    br label [[IF_END8]]
+; CHECK:       for.body2:
+; CHECK-NEXT:    br i1 undef, label [[FOR_END:%.*]], label [[FOR_INC:%.*]]
+; CHECK:       for.end:
+; CHECK-NEXT:    br label [[FOR_INC]]
+; CHECK:       for.inc:
+; CHECK-NEXT:    [[E_SROA_3_2:%.*]] = phi <2 x i64> [ <i64 1, i64 1>, [[FOR_END]] ], [ [[E_SROA_3_2]], [[FOR_BODY2:%.*]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = phi i32 [ 6, [[FOR_END]] ], [ [[TMP0]], [[FOR_BODY2]] ]
+; CHECK-NEXT:    br i1 undef, label [[FOR_BODY2]], label [[FOR_COND1_FOR_END7_CRIT_EDGE:%.*]]
+; CHECK:       for.cond1.for.end7_crit_edge:
+; CHECK-NEXT:    br label [[IF_END8]]
+; CHECK:       if.end8:
+; CHECK-NEXT:    [[E_SROA_3_4_I0:%.*]] = phi i64 [ undef, [[FOR_BODY]] ], [ undef, [[FOR_COND1_FOR_END7_CRIT_EDGE]] ], [ undef, [[IF_THEN]] ]
+; CHECK-NEXT:    [[E_SROA_3_4_I1:%.*]] = phi i64 [ undef, [[FOR_BODY]] ], [ undef, [[FOR_COND1_FOR_END7_CRIT_EDGE]] ], [ undef, [[IF_THEN]] ]
+; CHECK-NEXT:    br label [[FOR_BODY]]
+;
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %if.end8, %entry
+  br i1 undef, label %if.then, label %if.end8
+
+if.then:                                          ; preds = %for.body
+  br label %if.end8
+
+for.body2:                                        ; preds = %for.inc
+  br i1 undef, label %for.end, label %for.inc
+
+for.end:                                          ; preds = %for.body2
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.end, %for.body2
+  %e.sroa.3.2 = phi <2 x i64> [ <i64 1, i64 1>, %for.end ], [ %e.sroa.3.2, %for.body2 ]
+  %0 = phi i32 [ 6, %for.end ], [ %0, %for.body2 ]
+  br i1 undef, label %for.body2, label %for.cond1.for.end7_crit_edge
+
+for.cond1.for.end7_crit_edge:                     ; preds = %for.inc
+  br label %if.end8
+
+if.end8:                                          ; preds = %for.cond1.for.end7_crit_edge, %if.then, %for.body
+  ; This used to lead to inserted extractelement instructions between the phis
+  ; in %for.inc.
+  ; %e.sroa.3.2 is defined in a block that is unreachable from entry so we can
+  ; safely replace it with undef in the phi defining e.sroa.3.4.
+  %e.sroa.3.4 = phi <2 x i64> [ undef, %for.body ], [ %e.sroa.3.2, %for.cond1.for.end7_crit_edge ], [ undef, %if.then ]
+  br label %for.body
+}
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/basictest-profmd.ll b/llvm/test/Transforms/SimpleLoopUnswitch/basictest-profmd.ll
index 416fd4655..858088bee 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/basictest-profmd.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/basictest-profmd.ll
@@ -1,5 +1,5 @@
 ; RUN: opt -passes='loop(unswitch),verify<loops>' -S < %s | FileCheck %s
-; RUN: opt -enable-mssa-loop-dependency=true -verify-memoryssa -passes='loop(unswitch),verify<loops>' -S < %s | FileCheck %s
+; RUN: opt -verify-memoryssa -passes='loop-mssa(unswitch),verify<loops>' -S < %s | FileCheck %s
 
 declare void @incf()
 declare void @decf()
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/basictest.ll b/llvm/test/Transforms/SimpleLoopUnswitch/basictest.ll
index 240f433a8..95e6f671a 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/basictest.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/basictest.ll
@@ -1,5 +1,5 @@
 ; RUN: opt -passes='loop(unswitch),verify<loops>' -S < %s | FileCheck %s
-; RUN: opt -enable-mssa-loop-dependency=true -verify-memoryssa -passes='loop(unswitch),verify<loops>' -S < %s | FileCheck %s
+; RUN: opt -verify-memoryssa -passes='loop-mssa(unswitch),verify<loops>' -S < %s | FileCheck %s
 
 define i32 @test(i32* %A, i1 %C) {
 entry:
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/delete-dead-blocks.ll b/llvm/test/Transforms/SimpleLoopUnswitch/delete-dead-blocks.ll
index fa737ed3a..565925074 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/delete-dead-blocks.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/delete-dead-blocks.ll
@@ -1,5 +1,6 @@
 ; RUN: opt < %s -simple-loop-unswitch -enable-nontrivial-unswitch -S 2>&1 | FileCheck %s
 ; RUN: opt < %s -passes='unswitch<nontrivial>' -S 2>&1 | FileCheck %s
+; RUN: opt < %s -passes='loop-mssa(unswitch<nontrivial>)' -S 2>&1 | FileCheck %s
 ;
 ; Checking that (dead) blocks from inner loop are deleted after unswitch.
 ;
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/exponential-nontrivial-unswitch-nested.ll b/llvm/test/Transforms/SimpleLoopUnswitch/exponential-nontrivial-unswitch-nested.ll
index 28f726126..d2ca06357 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/exponential-nontrivial-unswitch-nested.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/exponential-nontrivial-unswitch-nested.ll
@@ -10,6 +10,13 @@
 ; RUN:     -unswitch-num-initial-unscaled-candidates=0 -unswitch-siblings-toplevel-div=16 \
 ; RUN:     -passes='loop(unswitch<nontrivial>),print<loops>' -disable-output 2>&1 | FileCheck %s --check-prefixes=LOOP1
 ;
+; RUN: opt < %s -enable-unswitch-cost-multiplier=true \
+; RUN:     -unswitch-num-initial-unscaled-candidates=0 -unswitch-siblings-toplevel-div=1 \
+; RUN:     -passes='loop-mssa(unswitch<nontrivial>),print<loops>' -disable-output 2>&1 | FileCheck %s --check-prefixes=LOOP1
+;
+; RUN: opt < %s -enable-unswitch-cost-multiplier=true \
+; RUN:     -unswitch-num-initial-unscaled-candidates=0 -unswitch-siblings-toplevel-div=16 \
+; RUN:     -passes='loop-mssa(unswitch<nontrivial>),print<loops>' -disable-output 2>&1 | FileCheck %s --check-prefixes=LOOP1
 ;
 ; When we relax the candidates part of a multiplier formula
 ; (unscaled candidates == 4) we start getting  some unswitches,
@@ -20,6 +27,11 @@
 ; RUN:     -passes='loop(unswitch<nontrivial>),print<loops>' -disable-output 2>&1 | \
 ; RUN:     sort -b -k 1 | FileCheck %s --check-prefixes=LOOP-UNSCALE4-DIV1
 ;
+; RUN: opt < %s -enable-unswitch-cost-multiplier=true \
+; RUN:     -unswitch-num-initial-unscaled-candidates=4 -unswitch-siblings-toplevel-div=1 \
+; RUN:     -passes='loop-mssa(unswitch<nontrivial>),print<loops>' -disable-output 2>&1 | \
+; RUN:     sort -b -k 1 | FileCheck %s --check-prefixes=LOOP-UNSCALE4-DIV1
+;
 ; NB: sort -b is essential here and below, otherwise blanks might lead to different
 ; order depending on locale.
 ;
@@ -28,6 +40,10 @@
 ; RUN:     -passes='loop(unswitch<nontrivial>),print<loops>' -disable-output 2>&1 | \
 ; RUN:     sort -b -k 1 | FileCheck %s --check-prefixes=LOOP-UNSCALE4-DIV2
 ;
+; RUN: opt < %s -enable-unswitch-cost-multiplier=true \
+; RUN:     -unswitch-num-initial-unscaled-candidates=4 -unswitch-siblings-toplevel-div=2 \
+; RUN:     -passes='loop-mssa(unswitch<nontrivial>),print<loops>' -disable-output 2>&1 | \
+; RUN:     sort -b -k 1 | FileCheck %s --check-prefixes=LOOP-UNSCALE4-DIV2
 ;
 ; Get
 ;    2^(num conds) == 2^5 = 32
@@ -37,6 +53,10 @@
 ; RUN:     -passes='loop(unswitch<nontrivial>),print<loops>' -disable-output 2>&1 | \
 ; RUN:	   sort -b -k 1 | FileCheck %s --check-prefixes=LOOP32
 ;
+; RUN: opt < %s -enable-unswitch-cost-multiplier=false \
+; RUN:     -passes='loop-mssa(unswitch<nontrivial>),print<loops>' -disable-output 2>&1 | \
+; RUN:	   sort -b -k 1 | FileCheck %s --check-prefixes=LOOP32
+;
 ; Single loop nest, not unswitched
 ; LOOP1:     Loop at depth 1 containing:
 ; LOOP1:     Loop at depth 2 containing:
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/exponential-nontrivial-unswitch-nested2.ll b/llvm/test/Transforms/SimpleLoopUnswitch/exponential-nontrivial-unswitch-nested2.ll
index 7e669f1b7..e8879e79c 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/exponential-nontrivial-unswitch-nested2.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/exponential-nontrivial-unswitch-nested2.ll
@@ -15,8 +15,16 @@
 ; RUN:     -unswitch-num-initial-unscaled-candidates=0 -unswitch-siblings-toplevel-div=16 \
 ; RUN:     -passes='loop(unswitch<nontrivial>),print<loops>' -disable-output 2>&1 | FileCheck %s --check-prefixes=LOOP1
 ;
+; RUN: opt < %s -enable-unswitch-cost-multiplier=true \
+; RUN:     -unswitch-num-initial-unscaled-candidates=0 -unswitch-siblings-toplevel-div=1 \
+; RUN:     -passes='loop-mssa(unswitch<nontrivial>),print<loops>' -disable-output 2>&1 | FileCheck %s --check-prefixes=LOOP1
+;
+; RUN: opt < %s -enable-unswitch-cost-multiplier=true \
+; RUN:     -unswitch-num-initial-unscaled-candidates=0 -unswitch-siblings-toplevel-div=16 \
+; RUN:     -passes='loop-mssa(unswitch<nontrivial>),print<loops>' -disable-output 2>&1 | FileCheck %s --check-prefixes=LOOP1
 ;
 ; When we relax the candidates part of a multiplier formula
+; When we relax the candidates part of a multiplier formula
 ; (unscaled candidates == 2) we start getting some unswitches in outer loops,
 ; which leads to siblings multiplier kicking in.
 ;
@@ -25,6 +33,11 @@
 ; RUN:     -passes='loop(unswitch<nontrivial>),print<loops>' -disable-output 2>&1 | \
 ; RUN:     sort -b -k 1 | FileCheck %s --check-prefixes=LOOP-UNSCALE3-DIV1
 ;
+; RUN: opt < %s -enable-unswitch-cost-multiplier=true \
+; RUN:     -unswitch-num-initial-unscaled-candidates=3 -unswitch-siblings-toplevel-div=1 \
+; RUN:     -passes='loop-mssa(unswitch<nontrivial>),print<loops>' -disable-output 2>&1 | \
+; RUN:     sort -b -k 1 | FileCheck %s --check-prefixes=LOOP-UNSCALE3-DIV1
+;
 ; NB: sort -b is essential here and below, otherwise blanks might lead to different
 ; order depending on locale.
 ;
@@ -33,12 +46,21 @@
 ; RUN:     -passes='loop(unswitch<nontrivial>),print<loops>' -disable-output 2>&1 | \
 ; RUN:     sort -b -k 1 | FileCheck %s --check-prefixes=LOOP-UNSCALE3-DIV2
 ;
+; RUN: opt < %s -enable-unswitch-cost-multiplier=true \
+; RUN:     -unswitch-num-initial-unscaled-candidates=3 -unswitch-siblings-toplevel-div=2 \
+; RUN:     -passes='loop-mssa(unswitch<nontrivial>),print<loops>' -disable-output 2>&1 | \
+; RUN:     sort -b -k 1 | FileCheck %s --check-prefixes=LOOP-UNSCALE3-DIV2
+;
 ; With disabled cost-multiplier we get maximal possible amount of unswitches.
 ;
 ; RUN: opt < %s -enable-unswitch-cost-multiplier=false \
 ; RUN:     -passes='loop(unswitch<nontrivial>),print<loops>' -disable-output 2>&1 | \
 ; RUN:	   sort -b -k 1 | FileCheck %s --check-prefixes=LOOP-MAX
 ;
+; RUN: opt < %s -enable-unswitch-cost-multiplier=false \
+; RUN:     -passes='loop-mssa(unswitch<nontrivial>),print<loops>' -disable-output 2>&1 | \
+; RUN:	   sort -b -k 1 | FileCheck %s --check-prefixes=LOOP-MAX
+;
 ; Single loop nest, not unswitched
 ; LOOP1:     Loop at depth 1 containing:
 ; LOOP1-NOT:  Loop at depth 1 containing:
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/exponential-nontrivial-unswitch.ll b/llvm/test/Transforms/SimpleLoopUnswitch/exponential-nontrivial-unswitch.ll
index 142974c5b..264479633 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/exponential-nontrivial-unswitch.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/exponential-nontrivial-unswitch.ll
@@ -10,6 +10,16 @@
 ; RUN:     -unswitch-num-initial-unscaled-candidates=0 -unswitch-siblings-toplevel-div=8 \
 ; RUN:     -passes='loop(unswitch<nontrivial>),print<loops>' -disable-output 2>&1 | FileCheck %s --check-prefixes=LOOP1
 ;
+; RUN: opt < %s -enable-unswitch-cost-multiplier=true \
+; RUN:     -unswitch-num-initial-unscaled-candidates=0 -unswitch-siblings-toplevel-div=1 \
+; RUN:     -passes='loop-mssa(unswitch<nontrivial>),print<loops>' -disable-output 2>&1 | FileCheck %s --check-prefixes=LOOP1
+;
+; RUN: opt < %s -enable-unswitch-cost-multiplier=true \
+; RUN:     -unswitch-num-initial-unscaled-candidates=0 -unswitch-siblings-toplevel-div=8 \
+; RUN:     -passes='loop-mssa(unswitch<nontrivial>),print<loops>' -disable-output 2>&1 | FileCheck %s --check-prefixes=LOOP1
+;
+; With relaxed candidates multiplier (unscaled candidates == 8) we should allow
+; some unswitches to happen until siblings multiplier starts kicking in:
 ; With relaxed candidates multiplier (unscaled candidates == 8) we should allow
 ; some unswitches to happen until siblings multiplier starts kicking in:
 ;
@@ -17,6 +27,10 @@
 ; RUN:     -unswitch-num-initial-unscaled-candidates=8 -unswitch-siblings-toplevel-div=1 \
 ; RUN:     -passes='loop(unswitch<nontrivial>),print<loops>' -disable-output 2>&1 | FileCheck %s --check-prefixes=LOOP5
 ;
+; RUN: opt < %s -enable-unswitch-cost-multiplier=true \
+; RUN:     -unswitch-num-initial-unscaled-candidates=8 -unswitch-siblings-toplevel-div=1 \
+; RUN:     -passes='loop-mssa(unswitch<nontrivial>),print<loops>' -disable-output 2>&1 | FileCheck %s --check-prefixes=LOOP5
+;
 ; With relaxed candidates multiplier (unscaled candidates == 8) and with relaxed
 ; siblings multiplier for top-level loops (toplevel-div == 8) we should get
 ;    2^(num conds) == 2^5 == 32
@@ -26,6 +40,10 @@
 ; RUN:     -unswitch-num-initial-unscaled-candidates=8 -unswitch-siblings-toplevel-div=8 \
 ; RUN:     -passes='loop(unswitch<nontrivial>),print<loops>' -disable-output 2>&1 | FileCheck %s --check-prefixes=LOOP32
 ;
+; RUN: opt < %s -enable-unswitch-cost-multiplier=true \
+; RUN:     -unswitch-num-initial-unscaled-candidates=8 -unswitch-siblings-toplevel-div=8 \
+; RUN:     -passes='loop-mssa(unswitch<nontrivial>),print<loops>' -disable-output 2>&1 | FileCheck %s --check-prefixes=LOOP32
+;
 ; Similarly get
 ;    2^(num conds) == 2^5 == 32
 ; copies of the loop when cost multiplier is disabled:
@@ -33,6 +51,8 @@
 ; RUN: opt < %s -enable-unswitch-cost-multiplier=false \
 ; RUN:     -passes='loop(unswitch<nontrivial>),print<loops>' -disable-output 2>&1 | FileCheck %s --check-prefixes=LOOP32
 ;
+; RUN: opt < %s -enable-unswitch-cost-multiplier=false \
+; RUN:     -passes='loop-mssa(unswitch<nontrivial>),print<loops>' -disable-output 2>&1 | FileCheck %s --check-prefixes=LOOP32
 ;
 ; Single loop, not unswitched
 ; LOOP1:     Loop at depth 1 containing:
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/exponential-nontrivial-unswitch2.ll b/llvm/test/Transforms/SimpleLoopUnswitch/exponential-nontrivial-unswitch2.ll
index 5cc594543..636039b45 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/exponential-nontrivial-unswitch2.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/exponential-nontrivial-unswitch2.ll
@@ -24,6 +24,26 @@
 ; RUN:     -passes='loop(unswitch<nontrivial>),print<loops>' -disable-output 2>&1 | FileCheck %s --check-prefixes=LOOP1
 ;
 ;
+; RUN: opt < %s -enable-unswitch-cost-multiplier=true \
+; RUN:     -unswitch-num-initial-unscaled-candidates=0 -unswitch-siblings-toplevel-div=1 \
+; RUN:     -passes='loop-mssa(unswitch<nontrivial>),print<loops>' -disable-output 2>&1 | FileCheck %s --check-prefixes=LOOP1
+;
+; RUN: opt < %s -enable-unswitch-cost-multiplier=true \
+; RUN:     -unswitch-num-initial-unscaled-candidates=0 -unswitch-siblings-toplevel-div=8 \
+; RUN:     -passes='loop-mssa(unswitch<nontrivial>),print<loops>' -disable-output 2>&1 | FileCheck %s --check-prefixes=LOOP1
+;
+; RUN: opt < %s -enable-unswitch-cost-multiplier=true \
+; RUN:     -unswitch-num-initial-unscaled-candidates=8 -unswitch-siblings-toplevel-div=1 \
+; RUN:     -passes='loop-mssa(unswitch<nontrivial>),print<loops>' -disable-output 2>&1 | FileCheck %s --check-prefixes=LOOP1
+;
+; RUN: opt < %s -enable-unswitch-cost-multiplier=true \
+; RUN:     -unswitch-num-initial-unscaled-candidates=8 -unswitch-siblings-toplevel-div=8 \
+; RUN:     -passes='loop-mssa(unswitch<nontrivial>),print<loops>' -disable-output 2>&1 | FileCheck %s --check-prefixes=LOOP1
+;
+; RUN: opt < %s -enable-unswitch-cost-multiplier=false \
+; RUN:     -passes='loop-mssa(unswitch<nontrivial>),print<loops>' -disable-output 2>&1 | FileCheck %s --check-prefixes=LOOP1
+;
+;
 ; Single loop, not unswitched
 ; LOOP1:     Loop at depth 1 containing:
 ; LOOP1-NOT: Loop at depth 1 containing:
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/exponential-switch-unswitch.ll b/llvm/test/Transforms/SimpleLoopUnswitch/exponential-switch-unswitch.ll
index 312c741ae..35d431e5e 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/exponential-switch-unswitch.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/exponential-switch-unswitch.ll
@@ -17,6 +17,13 @@
 ; RUN:     -unswitch-num-initial-unscaled-candidates=0 -unswitch-siblings-toplevel-div=16 \
 ; RUN:     -passes='loop(unswitch<nontrivial>),print<loops>' -disable-output 2>&1 | FileCheck %s --check-prefixes=LOOP1
 ;
+; RUN: opt < %s -enable-unswitch-cost-multiplier=true \
+; RUN:     -unswitch-num-initial-unscaled-candidates=0 -unswitch-siblings-toplevel-div=1 \
+; RUN:     -passes='loop-mssa(unswitch<nontrivial>),print<loops>' -disable-output 2>&1 | FileCheck %s --check-prefixes=LOOP1
+;
+; RUN: opt < %s -enable-unswitch-cost-multiplier=true \
+; RUN:     -unswitch-num-initial-unscaled-candidates=0 -unswitch-siblings-toplevel-div=16 \
+; RUN:     -passes='loop-mssa(unswitch<nontrivial>),print<loops>' -disable-output 2>&1 | FileCheck %s --check-prefixes=LOOP1
 ;
 ; With relaxed candidates multiplier (unscaled candidates == 8) we should allow
 ; some unswitches to happen until siblings multiplier starts kicking in:
@@ -26,6 +33,11 @@
 ; RUN:     -passes='loop(unswitch<nontrivial>),print<loops>' -disable-output 2>&1 | \
 ; RUN:     sort -b -k 1 | FileCheck %s --check-prefixes=LOOP-RELAX
 ;
+; RUN: opt < %s -enable-unswitch-cost-multiplier=true \
+; RUN:     -unswitch-num-initial-unscaled-candidates=8 -unswitch-siblings-toplevel-div=1 \
+; RUN:     -passes='loop-mssa(unswitch<nontrivial>),print<loops>' -disable-output 2>&1 | \
+; RUN:     sort -b -k 1 | FileCheck %s --check-prefixes=LOOP-RELAX
+;
 ; With relaxed candidates multiplier (unscaled candidates == 8) and with relaxed
 ; siblings multiplier for top-level loops (toplevel-div == 8) we should get
 ; considerably more copies of the loop (especially top-level ones).
@@ -35,12 +47,20 @@
 ; RUN:     -passes='loop(unswitch<nontrivial>),print<loops>' -disable-output 2>&1 | \
 ; RUN:     sort -b -k 1 | FileCheck %s --check-prefixes=LOOP-RELAX2
 ;
+; RUN: opt < %s -enable-unswitch-cost-multiplier=true \
+; RUN:     -unswitch-num-initial-unscaled-candidates=8 -unswitch-siblings-toplevel-div=8 \
+; RUN:     -passes='loop-mssa(unswitch<nontrivial>),print<loops>' -disable-output 2>&1 | \
+; RUN:     sort -b -k 1 | FileCheck %s --check-prefixes=LOOP-RELAX2
+;
 ; We get hundreds of copies of the loop when cost multiplier is disabled:
 ;
 ; RUN: opt < %s -enable-unswitch-cost-multiplier=false \
 ; RUN:     -passes='loop(unswitch<nontrivial>),print<loops>' -disable-output 2>&1 | \
 ; RUN:     sort -b -k 1 | FileCheck %s --check-prefixes=LOOP-MAX
 ;
+; RUN: opt < %s -enable-unswitch-cost-multiplier=false \
+; RUN:     -passes='loop-mssa(unswitch<nontrivial>),print<loops>' -disable-output 2>&1 | \
+; RUN:     sort -b -k 1 | FileCheck %s --check-prefixes=LOOP-MAX
 
 ; Single loop nest, not unswitched
 ; LOOP1:     Loop at depth 1 containing:
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/guards.ll b/llvm/test/Transforms/SimpleLoopUnswitch/guards.ll
index fef5582d9..de57075b6 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/guards.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/guards.ll
@@ -1,6 +1,6 @@
 ; RUN: opt -passes='loop(unswitch<nontrivial>),verify<loops>' -simple-loop-unswitch-guards -S < %s | FileCheck %s
 ; RUN: opt -simple-loop-unswitch -enable-nontrivial-unswitch -simple-loop-unswitch-guards -S < %s | FileCheck %s
-; RUN: opt -passes='loop(unswitch<nontrivial>),verify<loops>' -simple-loop-unswitch-guards -enable-mssa-loop-dependency=true -verify-memoryssa -S < %s | FileCheck %s
+; RUN: opt -passes='loop-mssa(unswitch<nontrivial>),verify<loops>' -simple-loop-unswitch-guards  -verify-memoryssa -S < %s | FileCheck %s
 
 declare void @llvm.experimental.guard(i1, ...)
 
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/msan.ll b/llvm/test/Transforms/SimpleLoopUnswitch/msan.ll
index 8a296bcd2..0022b33cd 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/msan.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/msan.ll
@@ -1,5 +1,5 @@
 ; RUN: opt -passes='loop(unswitch),verify<loops>' -S < %s | FileCheck %s
-; RUN: opt -enable-mssa-loop-dependency=true -verify-memoryssa -passes='loop(unswitch),verify<loops>' -S < %s | FileCheck %s
+; RUN: opt -verify-memoryssa -passes='loop-mssa(unswitch),verify<loops>' -S < %s | FileCheck %s
 
 declare void @unknown()
 declare void @unknown2()
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch-cost.ll b/llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch-cost.ll
index 8862d0132..692799db1 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch-cost.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch-cost.ll
@@ -1,6 +1,7 @@
 ; Specifically exercise the cost modeling for non-trivial loop unswitching.
 ;
 ; RUN: opt -passes='loop(unswitch<nontrivial>),verify<loops>' -unswitch-threshold=5 -S < %s | FileCheck %s
+; RUN: opt -passes='loop-mssa(unswitch<nontrivial>),verify<loops>' -unswitch-threshold=5 -S < %s | FileCheck %s
 ; RUN: opt -simple-loop-unswitch -enable-nontrivial-unswitch -unswitch-threshold=5 -S < %s | FileCheck %s
 ; RUN: opt -simple-loop-unswitch -enable-nontrivial-unswitch -unswitch-threshold=5 -enable-mssa-loop-dependency=true -verify-memoryssa -S < %s | FileCheck %s
 
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch-redundant-switch.ll b/llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch-redundant-switch.ll
index 8f743cac5..6dd2cdeef 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch-redundant-switch.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch-redundant-switch.ll
@@ -1,5 +1,6 @@
 ; REQUIRES: asserts
 ; RUN: opt -passes='unswitch<nontrivial>' -disable-output -S < %s
+; RUN: opt -passes='loop-mssa(unswitch<nontrivial>)' -disable-output -S < %s
 ; RUN: opt -simple-loop-unswitch -enable-nontrivial-unswitch -disable-output -S < %s
 
 ; This loop shouldn't trigger asserts in SimpleLoopUnswitch.
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch.ll b/llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch.ll
index 9f2e8a4b5..0aec52d44 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch.ll
@@ -1,4 +1,5 @@
 ; RUN: opt -passes='loop(unswitch<nontrivial>),verify<loops>' -S < %s | FileCheck %s
+; RUN: opt -passes='loop-mssa(unswitch<nontrivial>),verify<loops>' -S < %s | FileCheck %s
 ; RUN: opt -simple-loop-unswitch -enable-nontrivial-unswitch -S < %s | FileCheck %s
 ; RUN: opt -simple-loop-unswitch -enable-nontrivial-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa -S < %s | FileCheck %s
 
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/preserve-scev-exiting-multiple-loops.ll b/llvm/test/Transforms/SimpleLoopUnswitch/preserve-scev-exiting-multiple-loops.ll
new file mode 100644
index 000000000..badb623ff
--- /dev/null
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/preserve-scev-exiting-multiple-loops.ll
@@ -0,0 +1,63 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+
+; We run -indvars before -simple-loop-unswitch to compute SCEV exit counts before
+; running -simple-loop-unswitch.
+; RUN:  opt -indvars -simple-loop-unswitch -S %s -verify-scev | FileCheck %s
+
+; Test for PR43972.
+
+; We have a 3 nested loops (l1 <- l2 <- l3). %for.cond.5 is the exit block of
+; l3 and the loop for it is l2. But it is also the exiting block of l1. That
+; means we have to invalidate l1 to preserve SCEV.
+
+define void @f() {
+; CHECK-LABEL: @f(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LNOT:%.*]] = xor i1 undef, true
+; CHECK-NEXT:    br label [[FOR_COND:%.*]]
+; CHECK:       for.cond.loopexit:
+; CHECK-NEXT:    br label [[FOR_COND]]
+; CHECK:       for.cond:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.cond1:
+; CHECK-NEXT:    br i1 true, label [[FOR_BODY]], label [[FOR_COND_LOOPEXIT:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    br i1 [[LNOT]], label [[FOR_BODY_SPLIT:%.*]], label [[FOR_COND5_SPLIT:%.*]]
+; CHECK:       for.body.split:
+; CHECK-NEXT:    br label [[LAND_RHS:%.*]]
+; CHECK:       for.cond2:
+; CHECK-NEXT:    br i1 true, label [[LAND_RHS]], label [[FOR_COND5:%.*]]
+; CHECK:       land.rhs:
+; CHECK-NEXT:    br label [[FOR_COND2:%.*]]
+; CHECK:       for.cond5:
+; CHECK-NEXT:    br label [[FOR_COND5_SPLIT]]
+; CHECK:       for.cond5.split:
+; CHECK-NEXT:    br i1 true, label [[FOR_BODY7:%.*]], label [[FOR_COND1:%.*]]
+; CHECK:       for.body7:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %lnot = xor i1 undef, true
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.cond1, %entry
+  br label %for.body
+
+for.cond1:                                        ; preds = %for.cond5
+  br i1 true, label %for.body, label %for.cond
+
+for.body:                                         ; preds = %for.cond1, %for.cond
+  br label %land.rhs
+
+for.cond2:                                        ; preds = %land.rhs
+  br i1 true, label %land.rhs, label %for.cond5
+
+land.rhs:                                         ; preds = %for.cond2, %for.body
+  br i1 %lnot, label %for.cond2, label %for.cond5
+
+for.cond5:                                        ; preds = %land.rhs, %for.cond2
+  br i1 true, label %for.body7, label %for.cond1
+
+for.body7:                                        ; preds = %for.cond5
+  ret void
+}
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/trivial-unswitch-iteration.ll b/llvm/test/Transforms/SimpleLoopUnswitch/trivial-unswitch-iteration.ll
index 18b39ca80..bd14d5c72 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/trivial-unswitch-iteration.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/trivial-unswitch-iteration.ll
@@ -1,5 +1,5 @@
 ; RUN: opt -passes='loop(loop-instsimplify,simplify-cfg,unswitch),verify<loops>' -S < %s | FileCheck %s
-; RUN: opt -enable-mssa-loop-dependency=true -verify-memoryssa -passes='loop(loop-instsimplify,simplify-cfg,unswitch),verify<loops>' -S < %s | FileCheck %s
+; RUN: opt -verify-memoryssa -passes='loop-mssa(loop-instsimplify,simplify-cfg,unswitch),verify<loops>' -S < %s | FileCheck %s
 
 declare void @some_func() noreturn
 
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/trivial-unswitch-profmd.ll b/llvm/test/Transforms/SimpleLoopUnswitch/trivial-unswitch-profmd.ll
index 735caf37b..2deb9ce92 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/trivial-unswitch-profmd.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/trivial-unswitch-profmd.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ;       then metadata checks MDn were added manually.
 ; RUN: opt -passes='loop(unswitch),verify<loops>' -S < %s | FileCheck %s
-; RUN: opt -enable-mssa-loop-dependency=true -verify-memoryssa -passes='loop(unswitch),verify<loops>' -S < %s | FileCheck %s
+; RUN: opt -verify-memoryssa -passes='loop-mssa(unswitch),verify<loops>' -S < %s | FileCheck %s
 
 declare void @some_func()
 
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/trivial-unswitch.ll b/llvm/test/Transforms/SimpleLoopUnswitch/trivial-unswitch.ll
index 56a9bac89..4f01fd517 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/trivial-unswitch.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/trivial-unswitch.ll
@@ -1,5 +1,5 @@
 ; RUN: opt -passes='loop(unswitch),verify<loops>' -S < %s | FileCheck %s
-; RUN: opt -enable-mssa-loop-dependency=true -verify-memoryssa -passes='loop(unswitch),verify<loops>' -S < %s | FileCheck %s
+; RUN: opt -verify-memoryssa -passes='loop-mssa(unswitch),verify<loops>' -S < %s | FileCheck %s
 
 declare void @some_func() noreturn
 declare void @sink(i32)
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/update-scev.ll b/llvm/test/Transforms/SimpleLoopUnswitch/update-scev.ll
index 12d629480..2d59d758c 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/update-scev.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/update-scev.ll
@@ -1,5 +1,5 @@
 ; RUN: opt -passes='print<scalar-evolution>,loop(unswitch<nontrivial>,loop-instsimplify),print<scalar-evolution>' -S < %s 2>%t.scev | FileCheck %s
-; RUN: opt -enable-mssa-loop-dependency=true -verify-memoryssa -passes='print<scalar-evolution>,loop(unswitch<nontrivial>,loop-instsimplify),print<scalar-evolution>' -S < %s 2>%t.scev | FileCheck %s
+; RUN: opt -verify-memoryssa -passes='print<scalar-evolution>,loop-mssa(unswitch<nontrivial>,loop-instsimplify),print<scalar-evolution>' -S < %s 2>%t.scev | FileCheck %s
 ; RUN: FileCheck %s --check-prefix=SCEV < %t.scev
 
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/Transforms/SimplifyCFG/Hexagon/switch-to-lookup-table.ll b/llvm/test/Transforms/SimplifyCFG/Hexagon/switch-to-lookup-table.ll
index a568bba1d..65d110a77 100644
--- a/llvm/test/Transforms/SimplifyCFG/Hexagon/switch-to-lookup-table.ll
+++ b/llvm/test/Transforms/SimplifyCFG/Hexagon/switch-to-lookup-table.ll
@@ -59,4 +59,4 @@ return:                                           ; preds = %sw.default, %sw.bb5
   ret i32 %1
 }
 
-attributes #0 = { noinline nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="hexagonv60" "target-features"="-hvx,-long-calls" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noinline nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="hexagonv60" "target-features"="-hvx,-long-calls" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/Transforms/SimplifyCFG/HoistCode.ll b/llvm/test/Transforms/SimplifyCFG/HoistCode.ll
index 9697e56a7..575cb4f19 100644
--- a/llvm/test/Transforms/SimplifyCFG/HoistCode.ll
+++ b/llvm/test/Transforms/SimplifyCFG/HoistCode.ll
@@ -1,11 +1,38 @@
-; RUN: opt < %s -simplifycfg -S | not grep br
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -simplifycfg -S | FileCheck %s
 
 define void @foo(i1 %C, i32* %P) {
-        br i1 %C, label %T, label %F
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:    store i32 7, i32* [[P:%.*]]
+; CHECK-NEXT:    ret void
+;
+  br i1 %C, label %T, label %F
 T:              ; preds = %0
-        store i32 7, i32* %P
-        ret void
+  store i32 7, i32* %P
+  ret void
 F:              ; preds = %0
-        store i32 7, i32* %P
-        ret void
+  store i32 7, i32* %P
+  ret void
+}
+
+define float @PR39535min(float %x) {
+; CHECK-LABEL: @PR39535min(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TOBOOL:%.*]] = fcmp une float [[X:%.*]], 0.000000e+00
+; CHECK-NEXT:    [[DOTX:%.*]] = select fast i1 [[TOBOOL]], float 0.000000e+00, float [[X]]
+; CHECK-NEXT:    ret float [[DOTX]]
+;
+entry:
+  %tobool = fcmp une float %x, 0.0
+  br i1 %tobool, label %cond.true, label %cond.false
+
+cond.true:
+  br label %cond.end
+
+cond.false:
+  br label %cond.end
+
+cond.end:
+  %cond = phi fast float [ 0.0, %cond.true ], [ %x, %cond.false ]
+  ret float %cond
 }
diff --git a/llvm/test/Transforms/SimplifyCFG/PhiEliminate3.ll b/llvm/test/Transforms/SimplifyCFG/PhiEliminate3.ll
index 3566b874c..4d7435ce3 100644
--- a/llvm/test/Transforms/SimplifyCFG/PhiEliminate3.ll
+++ b/llvm/test/Transforms/SimplifyCFG/PhiEliminate3.ll
@@ -1,34 +1,49 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -simplifycfg -S -phi-node-folding-threshold=1 | FileCheck %s --check-prefix=ALL --check-prefix=CHECK-ONE
+; RUN: opt < %s -simplifycfg -S -phi-node-folding-threshold=2 | FileCheck %s --check-prefix=ALL --check-prefix=CHECK-TWO
+; RUN: opt < %s -simplifycfg -S -phi-node-folding-threshold=7 | FileCheck %s --check-prefix=ALL --check-prefix=CHECK-SEVEN
+
 ; Test merging of blocks containing complex expressions,
 ; with various folding thresholds
-;
-; RUN: opt < %s -simplifycfg -S -phi-node-folding-threshold=1 | grep N:
-; RUN: opt < %s -simplifycfg -S -phi-node-folding-threshold=2 | not grep N:
-; RUN: opt < %s -simplifycfg -S -phi-node-folding-threshold=2 | grep M:
-; RUN: opt < %s -simplifycfg -S -phi-node-folding-threshold=7 | not grep M:
-;
 
 define i32 @test(i1 %a, i1 %b, i32 %i, i32 %j, i32 %k) {
+; ALL-LABEL: @test(
+; ALL-NEXT:  entry:
+; ALL-NEXT:    br i1 [[A:%.*]], label [[M:%.*]], label [[O:%.*]]
+; ALL:       O:
+; ALL-NEXT:    [[IAJ:%.*]] = add i32 [[I:%.*]], [[J:%.*]]
+; ALL-NEXT:    [[IAJAK:%.*]] = add i32 [[IAJ]], [[K:%.*]]
+; ALL-NEXT:    [[IXJ:%.*]] = xor i32 [[I]], [[J]]
+; ALL-NEXT:    [[IXJXK:%.*]] = xor i32 [[IXJ]], [[K]]
+; ALL-NEXT:    [[WP:%.*]] = select i1 [[B:%.*]], i32 [[IAJAK]], i32 [[IXJXK]]
+; ALL-NEXT:    [[WP2:%.*]] = add i32 [[WP]], [[WP]]
+; ALL-NEXT:    br label [[M]]
+; ALL:       M:
+; ALL-NEXT:    [[W:%.*]] = phi i32 [ [[WP2]], [[O]] ], [ 2, [[ENTRY:%.*]] ]
+; ALL-NEXT:    [[R:%.*]] = add i32 [[W]], 1
+; ALL-NEXT:    ret i32 [[R]]
+;
 entry:
-        br i1 %a, label %M, label %O
+  br i1 %a, label %M, label %O
 O:
-        br i1 %b, label %P, label %Q
+  br i1 %b, label %P, label %Q
 P:
-        %iaj = add i32 %i, %j
-        %iajak = add i32 %iaj, %k
-        br label %N
+  %iaj = add i32 %i, %j
+  %iajak = add i32 %iaj, %k
+  br label %N
 Q:
-        %ixj = xor i32 %i, %j
-        %ixjxk = xor i32 %ixj, %k
-        br label %N
+  %ixj = xor i32 %i, %j
+  %ixjxk = xor i32 %ixj, %k
+  br label %N
 N:
-        ; This phi should be foldable if threshold >= 2
-        %Wp = phi i32 [ %iajak, %P ], [ %ixjxk, %Q ]
-        %Wp2 = add i32 %Wp, %Wp
-        br label %M
+  ; This phi should be foldable if threshold >= 2
+  %Wp = phi i32 [ %iajak, %P ], [ %ixjxk, %Q ]
+  %Wp2 = add i32 %Wp, %Wp
+  br label %M
 M:
-        ; This phi should be foldable if threshold >= 7
-        %W = phi i32 [ %Wp2, %N ], [ 2, %entry ]
-        %R = add i32 %W, 1
-        ret i32 %R
+  ; This phi should be foldable if threshold >= 7
+  %W = phi i32 [ %Wp2, %N ], [ 2, %entry ]
+  %R = add i32 %W, 1
+  ret i32 %R
 }
 
diff --git a/llvm/test/Transforms/SimplifyCFG/SpeculativeExec.ll b/llvm/test/Transforms/SimplifyCFG/SpeculativeExec.ll
index 19a7a7ec5..b33fac7b3 100644
--- a/llvm/test/Transforms/SimplifyCFG/SpeculativeExec.ll
+++ b/llvm/test/Transforms/SimplifyCFG/SpeculativeExec.ll
@@ -4,37 +4,67 @@
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
-define i32 @test1(i32 %a, i32 %b, i32 %c) nounwind  {
+define i32 @test1(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: @test1(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[B:%.*]], 0
-; CHECK-NEXT:    br i1 [[TMP1]], label [[BB1:%.*]], label [[BB3:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = icmp eq i32 [[B:%.*]], 0
+; CHECK-NEXT:    [[T2:%.*]] = icmp sgt i32 [[C:%.*]], 1
+; CHECK-NEXT:    [[T3:%.*]] = add i32 [[A:%.*]], 1
+; CHECK-NEXT:    [[SPEC_SELECT:%.*]] = select i1 [[T2]], i32 [[T3]], i32 [[A]]
+; CHECK-NEXT:    [[T4:%.*]] = select i1 [[T1]], i32 [[SPEC_SELECT]], i32 [[B]]
+; CHECK-NEXT:    [[T5:%.*]] = sub i32 [[T4]], 1
+; CHECK-NEXT:    ret i32 [[T5]]
+;
+entry:
+  %t1 = icmp eq i32 %b, 0
+  br i1 %t1, label %bb1, label %bb3
+
+bb1:
+  %t2 = icmp sgt i32 %c, 1
+  br i1 %t2, label %bb2, label %bb3
+
+bb2:
+  %t3 = add i32 %a, 1
+  br label %bb3
+
+bb3:
+  %t4 = phi i32 [ %b, %entry ], [ %a, %bb1 ], [ %t3, %bb2 ]
+  %t5 = sub i32 %t4, 1
+  ret i32 %t5
+}
+
+define float @spec_select_fp1(float %a, float %b, float %c) {
+; CHECK-LABEL: @spec_select_fp1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[T1:%.*]] = fcmp oeq float [[B:%.*]], 0.000000e+00
+; CHECK-NEXT:    br i1 [[T1]], label [[BB1:%.*]], label [[BB3:%.*]]
 ; CHECK:       bb1:
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp sgt i32 [[C:%.*]], 1
-; CHECK-NEXT:    [[TMP3:%.*]] = add i32 [[A:%.*]], 1
-; CHECK-NEXT:    [[TMP3_A:%.*]] = select i1 [[TMP2]], i32 [[TMP3]], i32 [[A]]
+; CHECK-NEXT:    [[T2:%.*]] = fcmp ogt float [[C:%.*]], 1.000000e+00
+; CHECK-NEXT:    br i1 [[T2]], label [[BB2:%.*]], label [[BB3]]
+; CHECK:       bb2:
+; CHECK-NEXT:    [[T3:%.*]] = fadd float [[A:%.*]], 1.000000e+00
 ; CHECK-NEXT:    br label [[BB3]]
 ; CHECK:       bb3:
-; CHECK-NEXT:    [[TMP4:%.*]] = phi i32 [ [[B]], [[ENTRY:%.*]] ], [ [[TMP3_A]], [[BB1]] ]
-; CHECK-NEXT:    [[TMP5:%.*]] = sub i32 [[TMP4]], 1
-; CHECK-NEXT:    ret i32 [[TMP5]]
+; CHECK-NEXT:    [[T4:%.*]] = phi ninf float [ [[B]], [[ENTRY:%.*]] ], [ [[A]], [[BB1]] ], [ [[T3]], [[BB2]] ]
+; CHECK-NEXT:    [[T5:%.*]] = fsub float [[T4]], 1.000000e+00
+; CHECK-NEXT:    ret float [[T5]]
 ;
 entry:
-  %tmp1 = icmp eq i32 %b, 0
-  br i1 %tmp1, label %bb1, label %bb3
+  %t1 = fcmp oeq float %b, 0.0
+  br i1 %t1, label %bb1, label %bb3
 
-bb1:            ; preds = %entry
-  %tmp2 = icmp sgt i32 %c, 1
-  br i1 %tmp2, label %bb2, label %bb3
+bb1:
+  %t2 = fcmp ogt float %c, 1.0
+  br i1 %t2, label %bb2, label %bb3
 
-bb2:		; preds = bb1
-  %tmp3 = add i32 %a, 1
+bb2:
+  %t3 = fadd float %a, 1.0
   br label %bb3
 
-bb3:		; preds = %bb2, %entry
-  %tmp4 = phi i32 [ %b, %entry ], [ %a, %bb1 ], [ %tmp3, %bb2 ]
-  %tmp5 = sub i32 %tmp4, 1
-  ret i32 %tmp5
+bb3:
+  %t4 = phi ninf float [ %b, %entry ], [ %a, %bb1 ], [ %t3, %bb2 ]
+  %t5 = fsub float %t4, 1.0
+  ret float %t5
 }
 
 define i8* @test4(i1* %dummy, i8* %a, i8* %b) {
@@ -89,33 +119,62 @@ end:
   ret i8* %x10
 }
 
-define i32* @test5(i32 %a, i32 %b, i32 %c, i32* dereferenceable(10) %ptr1,
+define i32* @test5(i32 %a, i32 %b, i32 %c, i32* dereferenceable(10) %ptr1, i32* dereferenceable(10) %ptr2, i32** dereferenceable(10) %ptr3) {
 ; CHECK-LABEL: @test5(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[B:%.*]], 0
-; CHECK-NEXT:    br i1 [[TMP1]], label [[BB1:%.*]], label [[BB3:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = icmp eq i32 [[B:%.*]], 0
+; CHECK-NEXT:    br i1 [[T1]], label [[BB1:%.*]], label [[BB3:%.*]]
 ; CHECK:       bb1:
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp sgt i32 [[C:%.*]], 1
-; CHECK-NEXT:    [[TMP3:%.*]] = load i32*, i32** [[PTR3:%.*]]
-; CHECK-NEXT:    [[TMP3_PTR2:%.*]] = select i1 [[TMP2]], i32* [[TMP3]], i32* [[PTR2:%.*]]
-; CHECK-NEXT:    ret i32* [[TMP3_PTR2]]
+; CHECK-NEXT:    [[T2:%.*]] = icmp sgt i32 [[C:%.*]], 1
+; CHECK-NEXT:    [[T3:%.*]] = load i32*, i32** [[PTR3:%.*]]
+; CHECK-NEXT:    [[SPEC_SELECT:%.*]] = select i1 [[T2]], i32* [[T3]], i32* [[PTR2:%.*]]
+; CHECK-NEXT:    ret i32* [[SPEC_SELECT]]
 ; CHECK:       bb3:
 ; CHECK-NEXT:    ret i32* [[PTR1:%.*]]
 ;
-  i32* dereferenceable(10) %ptr2, i32** dereferenceable(10) %ptr3) nounwind {
 entry:
-  %tmp1 = icmp eq i32 %b, 0
-  br i1 %tmp1, label %bb1, label %bb3
+  %t1 = icmp eq i32 %b, 0
+  br i1 %t1, label %bb1, label %bb3
+
+bb1:
+  %t2 = icmp sgt i32 %c, 1
+  br i1 %t2, label %bb2, label %bb3
+
+bb2:
+  %t3 = load i32*, i32** %ptr3, !dereferenceable !{i64 10}
+  br label %bb3
+
+bb3:
+  %t4 = phi i32* [ %ptr1, %entry ], [ %ptr2, %bb1 ], [ %t3, %bb2 ]
+  ret i32* %t4
+}
+
+define float @spec_select_fp5(float %a, float %b, float %c) {
+; CHECK-LABEL: @spec_select_fp5(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[T1:%.*]] = fcmp oeq float [[B:%.*]], 0.000000e+00
+; CHECK-NEXT:    br i1 [[T1]], label [[BB1:%.*]], label [[BB3:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    [[T2:%.*]] = fcmp ogt float [[C:%.*]], 1.000000e+00
+; CHECK-NEXT:    br i1 [[T2]], label [[BB2:%.*]], label [[BB3]]
+; CHECK:       bb2:
+; CHECK-NEXT:    br label [[BB3]]
+; CHECK:       bb3:
+; CHECK-NEXT:    [[T4:%.*]] = phi nsz float [ [[A:%.*]], [[ENTRY:%.*]] ], [ [[B]], [[BB1]] ], [ [[C]], [[BB2]] ]
+; CHECK-NEXT:    ret float [[T4]]
+;
+entry:
+  %t1 = fcmp oeq float %b, 0.0
+  br i1 %t1, label %bb1, label %bb3
 
-bb1:            ; preds = %entry
-  %tmp2 = icmp sgt i32 %c, 1
-  br i1 %tmp2, label %bb2, label %bb3
+bb1:
+  %t2 = fcmp ogt float %c, 1.0
+  br i1 %t2, label %bb2, label %bb3
 
-bb2:		; preds = bb1
-  %tmp3 = load i32*, i32** %ptr3, !dereferenceable !{i64 10}
+bb2:
   br label %bb3
 
-bb3:		; preds = %bb2, %entry
-  %tmp4 = phi i32* [ %ptr1, %entry ], [ %ptr2, %bb1 ], [ %tmp3, %bb2 ]
-  ret i32* %tmp4
+bb3:
+  %t4 = phi nsz float [ %a, %entry ], [ %b, %bb1 ], [ %c, %bb2 ]
+  ret float %t4
 }
diff --git a/llvm/test/Transforms/SimplifyCFG/X86/merge-cond-stores-cost.ll b/llvm/test/Transforms/SimplifyCFG/X86/merge-cond-stores-cost.ll
new file mode 100644
index 000000000..4a7f7d791
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/X86/merge-cond-stores-cost.ll
@@ -0,0 +1,43 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -simplifycfg -mtriple=x86_64-unknown-unknown < %s -simplifycfg-merge-cond-stores=true -simplifycfg-merge-cond-stores-aggressively=false -phi-node-folding-threshold=2 -S | FileCheck %s
+
+define void @test_costly(i32* %p, i32 %a, i32 %b, i32 %c, i32 %d) {
+; CHECK-LABEL: @test_costly(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[X1:%.*]] = icmp eq i32 [[A:%.*]], 0
+; CHECK-NEXT:    br i1 [[X1]], label [[FALLTHROUGH:%.*]], label [[YES1:%.*]]
+; CHECK:       yes1:
+; CHECK-NEXT:    [[VAL0:%.*]] = sdiv i32 [[D:%.*]], [[C:%.*]]
+; CHECK-NEXT:    store i32 [[VAL0]], i32* [[P:%.*]]
+; CHECK-NEXT:    br label [[FALLTHROUGH]]
+; CHECK:       fallthrough:
+; CHECK-NEXT:    [[X2:%.*]] = icmp eq i32 [[B:%.*]], 0
+; CHECK-NEXT:    br i1 [[X2]], label [[END:%.*]], label [[YES2:%.*]]
+; CHECK:       yes2:
+; CHECK-NEXT:    [[VAL1:%.*]] = sdiv i32 [[C]], [[D]]
+; CHECK-NEXT:    store i32 [[VAL1]], i32* [[P]]
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %x1 = icmp eq i32 %a, 0
+  br i1 %x1, label %fallthrough, label %yes1
+
+yes1:
+  %val0 = sdiv i32 %d, %c
+  store i32 %val0, i32* %p
+  br label %fallthrough
+
+fallthrough:
+  %x2 = icmp eq i32 %b, 0
+  br i1 %x2, label %end, label %yes2
+
+yes2:
+  %val1 = sdiv i32 %c, %d
+  store i32 %val1, i32* %p
+  br label %end
+
+end:
+  ret void
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/X86/speculate-cttz-ctlz.ll b/llvm/test/Transforms/SimplifyCFG/X86/speculate-cttz-ctlz.ll
index 11ba3984f..3a2f067a2 100644
--- a/llvm/test/Transforms/SimplifyCFG/X86/speculate-cttz-ctlz.ll
+++ b/llvm/test/Transforms/SimplifyCFG/X86/speculate-cttz-ctlz.ll
@@ -223,37 +223,13 @@ cond.end:                                         ; preds = %entry, %cond.true
 ; for the target.
 
 define i64 @test1e(i32 %x) {
-; BMI-LABEL: @test1e(
-; BMI-NEXT:  entry:
-; BMI-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[X:%.*]], 0
-; BMI-NEXT:    [[TMP0:%.*]] = tail call i32 @llvm.cttz.i32(i32 [[X]], i1 true)
-; BMI-NEXT:    [[PHITMP2:%.*]] = zext i32 [[TMP0]] to i64
-; BMI-NEXT:    [[COND:%.*]] = select i1 [[TOBOOL]], i64 32, i64 [[PHITMP2]]
-; BMI-NEXT:    ret i64 [[COND]]
-;
-; LZCNT-LABEL: @test1e(
-; LZCNT-NEXT:  entry:
-; LZCNT-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[X:%.*]], 0
-; LZCNT-NEXT:    br i1 [[TOBOOL]], label [[COND_END:%.*]], label [[COND_TRUE:%.*]]
-; LZCNT:       cond.true:
-; LZCNT-NEXT:    [[TMP0:%.*]] = tail call i32 @llvm.cttz.i32(i32 [[X]], i1 true)
-; LZCNT-NEXT:    [[PHITMP2:%.*]] = zext i32 [[TMP0]] to i64
-; LZCNT-NEXT:    br label [[COND_END]]
-; LZCNT:       cond.end:
-; LZCNT-NEXT:    [[COND:%.*]] = phi i64 [ [[PHITMP2]], [[COND_TRUE]] ], [ 32, [[ENTRY:%.*]] ]
-; LZCNT-NEXT:    ret i64 [[COND]]
-;
-; GENERIC-LABEL: @test1e(
-; GENERIC-NEXT:  entry:
-; GENERIC-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[X:%.*]], 0
-; GENERIC-NEXT:    br i1 [[TOBOOL]], label [[COND_END:%.*]], label [[COND_TRUE:%.*]]
-; GENERIC:       cond.true:
-; GENERIC-NEXT:    [[TMP0:%.*]] = tail call i32 @llvm.cttz.i32(i32 [[X]], i1 true)
-; GENERIC-NEXT:    [[PHITMP2:%.*]] = zext i32 [[TMP0]] to i64
-; GENERIC-NEXT:    br label [[COND_END]]
-; GENERIC:       cond.end:
-; GENERIC-NEXT:    [[COND:%.*]] = phi i64 [ [[PHITMP2]], [[COND_TRUE]] ], [ 32, [[ENTRY:%.*]] ]
-; GENERIC-NEXT:    ret i64 [[COND]]
+; ALL-LABEL: @test1e(
+; ALL-NEXT:  entry:
+; ALL-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[X:%.*]], 0
+; ALL-NEXT:    [[TMP0:%.*]] = tail call i32 @llvm.cttz.i32(i32 [[X]], i1 true)
+; ALL-NEXT:    [[PHITMP2:%.*]] = zext i32 [[TMP0]] to i64
+; ALL-NEXT:    [[COND:%.*]] = select i1 [[TOBOOL]], i64 32, i64 [[PHITMP2]]
+; ALL-NEXT:    ret i64 [[COND]]
 ;
 entry:
   %tobool = icmp eq i32 %x, 0
@@ -270,37 +246,13 @@ cond.end:                                         ; preds = %entry, %cond.true
 }
 
 define i32 @test2e(i64 %x) {
-; BMI-LABEL: @test2e(
-; BMI-NEXT:  entry:
-; BMI-NEXT:    [[TOBOOL:%.*]] = icmp eq i64 [[X:%.*]], 0
-; BMI-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.cttz.i64(i64 [[X]], i1 true)
-; BMI-NEXT:    [[CAST:%.*]] = trunc i64 [[TMP0]] to i32
-; BMI-NEXT:    [[COND:%.*]] = select i1 [[TOBOOL]], i32 64, i32 [[CAST]]
-; BMI-NEXT:    ret i32 [[COND]]
-;
-; LZCNT-LABEL: @test2e(
-; LZCNT-NEXT:  entry:
-; LZCNT-NEXT:    [[TOBOOL:%.*]] = icmp eq i64 [[X:%.*]], 0
-; LZCNT-NEXT:    br i1 [[TOBOOL]], label [[COND_END:%.*]], label [[COND_TRUE:%.*]]
-; LZCNT:       cond.true:
-; LZCNT-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.cttz.i64(i64 [[X]], i1 true)
-; LZCNT-NEXT:    [[CAST:%.*]] = trunc i64 [[TMP0]] to i32
-; LZCNT-NEXT:    br label [[COND_END]]
-; LZCNT:       cond.end:
-; LZCNT-NEXT:    [[COND:%.*]] = phi i32 [ [[CAST]], [[COND_TRUE]] ], [ 64, [[ENTRY:%.*]] ]
-; LZCNT-NEXT:    ret i32 [[COND]]
-;
-; GENERIC-LABEL: @test2e(
-; GENERIC-NEXT:  entry:
-; GENERIC-NEXT:    [[TOBOOL:%.*]] = icmp eq i64 [[X:%.*]], 0
-; GENERIC-NEXT:    br i1 [[TOBOOL]], label [[COND_END:%.*]], label [[COND_TRUE:%.*]]
-; GENERIC:       cond.true:
-; GENERIC-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.cttz.i64(i64 [[X]], i1 true)
-; GENERIC-NEXT:    [[CAST:%.*]] = trunc i64 [[TMP0]] to i32
-; GENERIC-NEXT:    br label [[COND_END]]
-; GENERIC:       cond.end:
-; GENERIC-NEXT:    [[COND:%.*]] = phi i32 [ [[CAST]], [[COND_TRUE]] ], [ 64, [[ENTRY:%.*]] ]
-; GENERIC-NEXT:    ret i32 [[COND]]
+; ALL-LABEL: @test2e(
+; ALL-NEXT:  entry:
+; ALL-NEXT:    [[TOBOOL:%.*]] = icmp eq i64 [[X:%.*]], 0
+; ALL-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.cttz.i64(i64 [[X]], i1 true)
+; ALL-NEXT:    [[CAST:%.*]] = trunc i64 [[TMP0]] to i32
+; ALL-NEXT:    [[COND:%.*]] = select i1 [[TOBOOL]], i32 64, i32 [[CAST]]
+; ALL-NEXT:    ret i32 [[COND]]
 ;
 entry:
   %tobool = icmp eq i64 %x, 0
@@ -317,37 +269,13 @@ cond.end:                                         ; preds = %entry, %cond.true
 }
 
 define i64 @test3e(i32 %x) {
-; BMI-LABEL: @test3e(
-; BMI-NEXT:  entry:
-; BMI-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[X:%.*]], 0
-; BMI-NEXT:    br i1 [[TOBOOL]], label [[COND_END:%.*]], label [[COND_TRUE:%.*]]
-; BMI:       cond.true:
-; BMI-NEXT:    [[TMP0:%.*]] = tail call i32 @llvm.ctlz.i32(i32 [[X]], i1 true)
-; BMI-NEXT:    [[PHITMP2:%.*]] = zext i32 [[TMP0]] to i64
-; BMI-NEXT:    br label [[COND_END]]
-; BMI:       cond.end:
-; BMI-NEXT:    [[COND:%.*]] = phi i64 [ [[PHITMP2]], [[COND_TRUE]] ], [ 32, [[ENTRY:%.*]] ]
-; BMI-NEXT:    ret i64 [[COND]]
-;
-; LZCNT-LABEL: @test3e(
-; LZCNT-NEXT:  entry:
-; LZCNT-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[X:%.*]], 0
-; LZCNT-NEXT:    [[TMP0:%.*]] = tail call i32 @llvm.ctlz.i32(i32 [[X]], i1 true)
-; LZCNT-NEXT:    [[PHITMP2:%.*]] = zext i32 [[TMP0]] to i64
-; LZCNT-NEXT:    [[COND:%.*]] = select i1 [[TOBOOL]], i64 32, i64 [[PHITMP2]]
-; LZCNT-NEXT:    ret i64 [[COND]]
-;
-; GENERIC-LABEL: @test3e(
-; GENERIC-NEXT:  entry:
-; GENERIC-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[X:%.*]], 0
-; GENERIC-NEXT:    br i1 [[TOBOOL]], label [[COND_END:%.*]], label [[COND_TRUE:%.*]]
-; GENERIC:       cond.true:
-; GENERIC-NEXT:    [[TMP0:%.*]] = tail call i32 @llvm.ctlz.i32(i32 [[X]], i1 true)
-; GENERIC-NEXT:    [[PHITMP2:%.*]] = zext i32 [[TMP0]] to i64
-; GENERIC-NEXT:    br label [[COND_END]]
-; GENERIC:       cond.end:
-; GENERIC-NEXT:    [[COND:%.*]] = phi i64 [ [[PHITMP2]], [[COND_TRUE]] ], [ 32, [[ENTRY:%.*]] ]
-; GENERIC-NEXT:    ret i64 [[COND]]
+; ALL-LABEL: @test3e(
+; ALL-NEXT:  entry:
+; ALL-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[X:%.*]], 0
+; ALL-NEXT:    [[TMP0:%.*]] = tail call i32 @llvm.ctlz.i32(i32 [[X]], i1 true)
+; ALL-NEXT:    [[PHITMP2:%.*]] = zext i32 [[TMP0]] to i64
+; ALL-NEXT:    [[COND:%.*]] = select i1 [[TOBOOL]], i64 32, i64 [[PHITMP2]]
+; ALL-NEXT:    ret i64 [[COND]]
 ;
 entry:
   %tobool = icmp eq i32 %x, 0
@@ -364,37 +292,13 @@ cond.end:                                         ; preds = %entry, %cond.true
 }
 
 define i32 @test4e(i64 %x) {
-; BMI-LABEL: @test4e(
-; BMI-NEXT:  entry:
-; BMI-NEXT:    [[TOBOOL:%.*]] = icmp eq i64 [[X:%.*]], 0
-; BMI-NEXT:    br i1 [[TOBOOL]], label [[COND_END:%.*]], label [[COND_TRUE:%.*]]
-; BMI:       cond.true:
-; BMI-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.ctlz.i64(i64 [[X]], i1 true)
-; BMI-NEXT:    [[CAST:%.*]] = trunc i64 [[TMP0]] to i32
-; BMI-NEXT:    br label [[COND_END]]
-; BMI:       cond.end:
-; BMI-NEXT:    [[COND:%.*]] = phi i32 [ [[CAST]], [[COND_TRUE]] ], [ 64, [[ENTRY:%.*]] ]
-; BMI-NEXT:    ret i32 [[COND]]
-;
-; LZCNT-LABEL: @test4e(
-; LZCNT-NEXT:  entry:
-; LZCNT-NEXT:    [[TOBOOL:%.*]] = icmp eq i64 [[X:%.*]], 0
-; LZCNT-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.ctlz.i64(i64 [[X]], i1 true)
-; LZCNT-NEXT:    [[CAST:%.*]] = trunc i64 [[TMP0]] to i32
-; LZCNT-NEXT:    [[COND:%.*]] = select i1 [[TOBOOL]], i32 64, i32 [[CAST]]
-; LZCNT-NEXT:    ret i32 [[COND]]
-;
-; GENERIC-LABEL: @test4e(
-; GENERIC-NEXT:  entry:
-; GENERIC-NEXT:    [[TOBOOL:%.*]] = icmp eq i64 [[X:%.*]], 0
-; GENERIC-NEXT:    br i1 [[TOBOOL]], label [[COND_END:%.*]], label [[COND_TRUE:%.*]]
-; GENERIC:       cond.true:
-; GENERIC-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.ctlz.i64(i64 [[X]], i1 true)
-; GENERIC-NEXT:    [[CAST:%.*]] = trunc i64 [[TMP0]] to i32
-; GENERIC-NEXT:    br label [[COND_END]]
-; GENERIC:       cond.end:
-; GENERIC-NEXT:    [[COND:%.*]] = phi i32 [ [[CAST]], [[COND_TRUE]] ], [ 64, [[ENTRY:%.*]] ]
-; GENERIC-NEXT:    ret i32 [[COND]]
+; ALL-LABEL: @test4e(
+; ALL-NEXT:  entry:
+; ALL-NEXT:    [[TOBOOL:%.*]] = icmp eq i64 [[X:%.*]], 0
+; ALL-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.ctlz.i64(i64 [[X]], i1 true)
+; ALL-NEXT:    [[CAST:%.*]] = trunc i64 [[TMP0]] to i32
+; ALL-NEXT:    [[COND:%.*]] = select i1 [[TOBOOL]], i32 64, i32 [[CAST]]
+; ALL-NEXT:    ret i32 [[COND]]
 ;
 entry:
   %tobool = icmp eq i64 %x, 0
@@ -411,37 +315,13 @@ cond.end:                                         ; preds = %entry, %cond.true
 }
 
 define i16 @test5e(i64 %x) {
-; BMI-LABEL: @test5e(
-; BMI-NEXT:  entry:
-; BMI-NEXT:    [[TOBOOL:%.*]] = icmp eq i64 [[X:%.*]], 0
-; BMI-NEXT:    br i1 [[TOBOOL]], label [[COND_END:%.*]], label [[COND_TRUE:%.*]]
-; BMI:       cond.true:
-; BMI-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.ctlz.i64(i64 [[X]], i1 true)
-; BMI-NEXT:    [[CAST:%.*]] = trunc i64 [[TMP0]] to i16
-; BMI-NEXT:    br label [[COND_END]]
-; BMI:       cond.end:
-; BMI-NEXT:    [[COND:%.*]] = phi i16 [ [[CAST]], [[COND_TRUE]] ], [ 64, [[ENTRY:%.*]] ]
-; BMI-NEXT:    ret i16 [[COND]]
-;
-; LZCNT-LABEL: @test5e(
-; LZCNT-NEXT:  entry:
-; LZCNT-NEXT:    [[TOBOOL:%.*]] = icmp eq i64 [[X:%.*]], 0
-; LZCNT-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.ctlz.i64(i64 [[X]], i1 true)
-; LZCNT-NEXT:    [[CAST:%.*]] = trunc i64 [[TMP0]] to i16
-; LZCNT-NEXT:    [[COND:%.*]] = select i1 [[TOBOOL]], i16 64, i16 [[CAST]]
-; LZCNT-NEXT:    ret i16 [[COND]]
-;
-; GENERIC-LABEL: @test5e(
-; GENERIC-NEXT:  entry:
-; GENERIC-NEXT:    [[TOBOOL:%.*]] = icmp eq i64 [[X:%.*]], 0
-; GENERIC-NEXT:    br i1 [[TOBOOL]], label [[COND_END:%.*]], label [[COND_TRUE:%.*]]
-; GENERIC:       cond.true:
-; GENERIC-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.ctlz.i64(i64 [[X]], i1 true)
-; GENERIC-NEXT:    [[CAST:%.*]] = trunc i64 [[TMP0]] to i16
-; GENERIC-NEXT:    br label [[COND_END]]
-; GENERIC:       cond.end:
-; GENERIC-NEXT:    [[COND:%.*]] = phi i16 [ [[CAST]], [[COND_TRUE]] ], [ 64, [[ENTRY:%.*]] ]
-; GENERIC-NEXT:    ret i16 [[COND]]
+; ALL-LABEL: @test5e(
+; ALL-NEXT:  entry:
+; ALL-NEXT:    [[TOBOOL:%.*]] = icmp eq i64 [[X:%.*]], 0
+; ALL-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.ctlz.i64(i64 [[X]], i1 true)
+; ALL-NEXT:    [[CAST:%.*]] = trunc i64 [[TMP0]] to i16
+; ALL-NEXT:    [[COND:%.*]] = select i1 [[TOBOOL]], i16 64, i16 [[CAST]]
+; ALL-NEXT:    ret i16 [[COND]]
 ;
 entry:
   %tobool = icmp eq i64 %x, 0
@@ -458,37 +338,13 @@ cond.end:                                         ; preds = %entry, %cond.true
 }
 
 define i16 @test6e(i32 %x) {
-; BMI-LABEL: @test6e(
-; BMI-NEXT:  entry:
-; BMI-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[X:%.*]], 0
-; BMI-NEXT:    br i1 [[TOBOOL]], label [[COND_END:%.*]], label [[COND_TRUE:%.*]]
-; BMI:       cond.true:
-; BMI-NEXT:    [[TMP0:%.*]] = tail call i32 @llvm.ctlz.i32(i32 [[X]], i1 true)
-; BMI-NEXT:    [[CAST:%.*]] = trunc i32 [[TMP0]] to i16
-; BMI-NEXT:    br label [[COND_END]]
-; BMI:       cond.end:
-; BMI-NEXT:    [[COND:%.*]] = phi i16 [ [[CAST]], [[COND_TRUE]] ], [ 32, [[ENTRY:%.*]] ]
-; BMI-NEXT:    ret i16 [[COND]]
-;
-; LZCNT-LABEL: @test6e(
-; LZCNT-NEXT:  entry:
-; LZCNT-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[X:%.*]], 0
-; LZCNT-NEXT:    [[TMP0:%.*]] = tail call i32 @llvm.ctlz.i32(i32 [[X]], i1 true)
-; LZCNT-NEXT:    [[CAST:%.*]] = trunc i32 [[TMP0]] to i16
-; LZCNT-NEXT:    [[COND:%.*]] = select i1 [[TOBOOL]], i16 32, i16 [[CAST]]
-; LZCNT-NEXT:    ret i16 [[COND]]
-;
-; GENERIC-LABEL: @test6e(
-; GENERIC-NEXT:  entry:
-; GENERIC-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[X:%.*]], 0
-; GENERIC-NEXT:    br i1 [[TOBOOL]], label [[COND_END:%.*]], label [[COND_TRUE:%.*]]
-; GENERIC:       cond.true:
-; GENERIC-NEXT:    [[TMP0:%.*]] = tail call i32 @llvm.ctlz.i32(i32 [[X]], i1 true)
-; GENERIC-NEXT:    [[CAST:%.*]] = trunc i32 [[TMP0]] to i16
-; GENERIC-NEXT:    br label [[COND_END]]
-; GENERIC:       cond.end:
-; GENERIC-NEXT:    [[COND:%.*]] = phi i16 [ [[CAST]], [[COND_TRUE]] ], [ 32, [[ENTRY:%.*]] ]
-; GENERIC-NEXT:    ret i16 [[COND]]
+; ALL-LABEL: @test6e(
+; ALL-NEXT:  entry:
+; ALL-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[X:%.*]], 0
+; ALL-NEXT:    [[TMP0:%.*]] = tail call i32 @llvm.ctlz.i32(i32 [[X]], i1 true)
+; ALL-NEXT:    [[CAST:%.*]] = trunc i32 [[TMP0]] to i16
+; ALL-NEXT:    [[COND:%.*]] = select i1 [[TOBOOL]], i16 32, i16 [[CAST]]
+; ALL-NEXT:    ret i16 [[COND]]
 ;
 entry:
   %tobool = icmp eq i32 %x, 0
@@ -505,37 +361,13 @@ cond.end:                                         ; preds = %entry, %cond.true
 }
 
 define i16 @test7e(i64 %x) {
-; BMI-LABEL: @test7e(
-; BMI-NEXT:  entry:
-; BMI-NEXT:    [[TOBOOL:%.*]] = icmp eq i64 [[X:%.*]], 0
-; BMI-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.cttz.i64(i64 [[X]], i1 true)
-; BMI-NEXT:    [[CAST:%.*]] = trunc i64 [[TMP0]] to i16
-; BMI-NEXT:    [[COND:%.*]] = select i1 [[TOBOOL]], i16 64, i16 [[CAST]]
-; BMI-NEXT:    ret i16 [[COND]]
-;
-; LZCNT-LABEL: @test7e(
-; LZCNT-NEXT:  entry:
-; LZCNT-NEXT:    [[TOBOOL:%.*]] = icmp eq i64 [[X:%.*]], 0
-; LZCNT-NEXT:    br i1 [[TOBOOL]], label [[COND_END:%.*]], label [[COND_TRUE:%.*]]
-; LZCNT:       cond.true:
-; LZCNT-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.cttz.i64(i64 [[X]], i1 true)
-; LZCNT-NEXT:    [[CAST:%.*]] = trunc i64 [[TMP0]] to i16
-; LZCNT-NEXT:    br label [[COND_END]]
-; LZCNT:       cond.end:
-; LZCNT-NEXT:    [[COND:%.*]] = phi i16 [ [[CAST]], [[COND_TRUE]] ], [ 64, [[ENTRY:%.*]] ]
-; LZCNT-NEXT:    ret i16 [[COND]]
-;
-; GENERIC-LABEL: @test7e(
-; GENERIC-NEXT:  entry:
-; GENERIC-NEXT:    [[TOBOOL:%.*]] = icmp eq i64 [[X:%.*]], 0
-; GENERIC-NEXT:    br i1 [[TOBOOL]], label [[COND_END:%.*]], label [[COND_TRUE:%.*]]
-; GENERIC:       cond.true:
-; GENERIC-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.cttz.i64(i64 [[X]], i1 true)
-; GENERIC-NEXT:    [[CAST:%.*]] = trunc i64 [[TMP0]] to i16
-; GENERIC-NEXT:    br label [[COND_END]]
-; GENERIC:       cond.end:
-; GENERIC-NEXT:    [[COND:%.*]] = phi i16 [ [[CAST]], [[COND_TRUE]] ], [ 64, [[ENTRY:%.*]] ]
-; GENERIC-NEXT:    ret i16 [[COND]]
+; ALL-LABEL: @test7e(
+; ALL-NEXT:  entry:
+; ALL-NEXT:    [[TOBOOL:%.*]] = icmp eq i64 [[X:%.*]], 0
+; ALL-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.cttz.i64(i64 [[X]], i1 true)
+; ALL-NEXT:    [[CAST:%.*]] = trunc i64 [[TMP0]] to i16
+; ALL-NEXT:    [[COND:%.*]] = select i1 [[TOBOOL]], i16 64, i16 [[CAST]]
+; ALL-NEXT:    ret i16 [[COND]]
 ;
 entry:
   %tobool = icmp eq i64 %x, 0
@@ -552,37 +384,13 @@ cond.end:                                         ; preds = %entry, %cond.true
 }
 
 define i16 @test8e(i32 %x) {
-; BMI-LABEL: @test8e(
-; BMI-NEXT:  entry:
-; BMI-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[X:%.*]], 0
-; BMI-NEXT:    [[TMP0:%.*]] = tail call i32 @llvm.cttz.i32(i32 [[X]], i1 true)
-; BMI-NEXT:    [[CAST:%.*]] = trunc i32 [[TMP0]] to i16
-; BMI-NEXT:    [[COND:%.*]] = select i1 [[TOBOOL]], i16 32, i16 [[CAST]]
-; BMI-NEXT:    ret i16 [[COND]]
-;
-; LZCNT-LABEL: @test8e(
-; LZCNT-NEXT:  entry:
-; LZCNT-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[X:%.*]], 0
-; LZCNT-NEXT:    br i1 [[TOBOOL]], label [[COND_END:%.*]], label [[COND_TRUE:%.*]]
-; LZCNT:       cond.true:
-; LZCNT-NEXT:    [[TMP0:%.*]] = tail call i32 @llvm.cttz.i32(i32 [[X]], i1 true)
-; LZCNT-NEXT:    [[CAST:%.*]] = trunc i32 [[TMP0]] to i16
-; LZCNT-NEXT:    br label [[COND_END]]
-; LZCNT:       cond.end:
-; LZCNT-NEXT:    [[COND:%.*]] = phi i16 [ [[CAST]], [[COND_TRUE]] ], [ 32, [[ENTRY:%.*]] ]
-; LZCNT-NEXT:    ret i16 [[COND]]
-;
-; GENERIC-LABEL: @test8e(
-; GENERIC-NEXT:  entry:
-; GENERIC-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[X:%.*]], 0
-; GENERIC-NEXT:    br i1 [[TOBOOL]], label [[COND_END:%.*]], label [[COND_TRUE:%.*]]
-; GENERIC:       cond.true:
-; GENERIC-NEXT:    [[TMP0:%.*]] = tail call i32 @llvm.cttz.i32(i32 [[X]], i1 true)
-; GENERIC-NEXT:    [[CAST:%.*]] = trunc i32 [[TMP0]] to i16
-; GENERIC-NEXT:    br label [[COND_END]]
-; GENERIC:       cond.end:
-; GENERIC-NEXT:    [[COND:%.*]] = phi i16 [ [[CAST]], [[COND_TRUE]] ], [ 32, [[ENTRY:%.*]] ]
-; GENERIC-NEXT:    ret i16 [[COND]]
+; ALL-LABEL: @test8e(
+; ALL-NEXT:  entry:
+; ALL-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[X:%.*]], 0
+; ALL-NEXT:    [[TMP0:%.*]] = tail call i32 @llvm.cttz.i32(i32 [[X]], i1 true)
+; ALL-NEXT:    [[CAST:%.*]] = trunc i32 [[TMP0]] to i16
+; ALL-NEXT:    [[COND:%.*]] = select i1 [[TOBOOL]], i16 32, i16 [[CAST]]
+; ALL-NEXT:    ret i16 [[COND]]
 ;
 entry:
   %tobool = icmp eq i32 %x, 0
diff --git a/llvm/test/Transforms/SimplifyCFG/X86/switch_to_lookup_table.ll b/llvm/test/Transforms/SimplifyCFG/X86/switch_to_lookup_table.ll
index c77438974..597b5b969 100644
--- a/llvm/test/Transforms/SimplifyCFG/X86/switch_to_lookup_table.ll
+++ b/llvm/test/Transforms/SimplifyCFG/X86/switch_to_lookup_table.ll
@@ -1437,14 +1437,10 @@ define i32 @no_reuse_cmp2(i32 %x, i32 %y) {
 ; CHECK-LABEL: @no_reuse_cmp2(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[EC:%.*]] = icmp ne i32 [[Y:%.*]], 0
-; CHECK-NEXT:    br i1 [[EC]], label [[SWITCH_ENTRY:%.*]], label [[SW_EPILOG:%.*]]
-; CHECK:       switch.entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[X:%.*]], 4
 ; CHECK-NEXT:    [[SWITCH_OFFSET:%.*]] = add i32 [[X]], 10
 ; CHECK-NEXT:    [[SPEC_SELECT:%.*]] = select i1 [[TMP0]], i32 [[SWITCH_OFFSET]], i32 0
-; CHECK-NEXT:    br label [[SW_EPILOG]]
-; CHECK:       sw.epilog:
-; CHECK-NEXT:    [[R_0:%.*]] = phi i32 [ 100, [[ENTRY:%.*]] ], [ [[SPEC_SELECT]], [[SWITCH_ENTRY]] ]
+; CHECK-NEXT:    [[R_0:%.*]] = select i1 [[EC]], i32 [[SPEC_SELECT]], i32 100
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[R_0]], 0
 ; CHECK-NEXT:    [[DOTR_0:%.*]] = select i1 [[CMP]], i32 100, i32 [[R_0]]
 ; CHECK-NEXT:    ret i32 [[DOTR_0]]
diff --git a/llvm/test/Transforms/SimplifyCFG/dce-cond-after-folding-terminator.ll b/llvm/test/Transforms/SimplifyCFG/dce-cond-after-folding-terminator.ll
index 036a615e7..9c414b488 100644
--- a/llvm/test/Transforms/SimplifyCFG/dce-cond-after-folding-terminator.ll
+++ b/llvm/test/Transforms/SimplifyCFG/dce-cond-after-folding-terminator.ll
@@ -37,10 +37,7 @@ define void @test_indirectbr(i32 %x) {
 entry:
 ; CHECK-LABEL: @test_indirectbr(
 ; CHECK-NEXT: entry:
-; Ideally this should now check:
-;   CHK-NEXT: ret void
-; But that doesn't happen yet. Instead:
-; CHECK-NEXT: br label %L1
+; CHECK-NEXT:   ret void
 
   %label = bitcast i8* blockaddress(@test_indirectbr, %L1) to i8*
   indirectbr i8* %label, [label %L1, label %L2]
diff --git a/llvm/test/Transforms/SimplifyCFG/hoist-dbgvalue-inlined.ll b/llvm/test/Transforms/SimplifyCFG/hoist-dbgvalue-inlined.ll
index ec8184164..968c6fd01 100644
--- a/llvm/test/Transforms/SimplifyCFG/hoist-dbgvalue-inlined.ll
+++ b/llvm/test/Transforms/SimplifyCFG/hoist-dbgvalue-inlined.ll
@@ -8,7 +8,6 @@ init:
 
 ; CHECK:  %vala = load i64, i64* %ptr
 ; CHECK-NEXT:  call void @llvm.dbg.value(metadata i64 %vala, metadata [[MD:![0-9]*]]
-; CHECK-NEXT:  call void @llvm.dbg.value(metadata i64 %vala, metadata [[MD]]
 ; CHECK-NEXT:  %valbmasked = and i64 %vala, 1
 
 a:                                              ; preds = %init
diff --git a/llvm/test/Transforms/SimplifyCFG/invalidate-dom.ll b/llvm/test/Transforms/SimplifyCFG/invalidate-dom.ll
new file mode 100644
index 000000000..ec8044503
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/invalidate-dom.ll
@@ -0,0 +1,90 @@
+; RUN: opt -aa-pipeline=default -passes="default<O2>" -disable-output %s
+; REQUIRES: asserts
+; PR42272
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@b = external global i32, align 4
+@f = external global i32, align 4
+@g = external global i32, align 4
+
+define i32* @a(i32 %h) #0 {
+entry:
+  %h.addr = alloca i32, align 4
+  store i32 %h, i32* %h.addr, align 4
+  %tmp0 = load i32, i32* %h.addr, align 4
+  switch i32 %tmp0, label %sw.default [
+    i32 4, label %sw.bb
+    i32 3, label %sw.bb1
+    i32 2, label %sw.bb3
+  ]
+
+sw.bb:                                            ; preds = %entry
+  %call = call i32 (...) @c()
+  unreachable
+
+sw.bb1:                                           ; preds = %entry
+  %call2 = call i32 (...) @c()
+  unreachable
+
+sw.bb3:                                           ; preds = %entry
+  %call4 = call i32 (...) @c()
+  %conv = sext i32 %call4 to i64
+  %tmp1 = inttoptr i64 %conv to i32*
+  ret i32* %tmp1
+
+sw.default:                                       ; preds = %entry
+  unreachable
+}
+
+define i32 @m() #1 {
+entry:
+  %call = call i32 @j()
+  %call1 = call i32 @j()
+  ret i32 undef
+}
+
+define internal i32 @j() #0 {
+entry:
+  %i = alloca i32, align 4
+  store i32 0, i32* %i, align 4
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.body, %entry
+  %tmp0 = load i32, i32* %i, align 4
+  %tmp1 = load i32, i32* @f, align 4
+  %cmp = icmp ult i32 %tmp0, %tmp1
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.cond
+  %tmp2 = load i32, i32* @f, align 4
+  %call3 = call i32* @a(i32 %tmp2)
+  ret i32 undef
+
+for.body:                                         ; preds = %for.cond
+  %call = call i32 (...) @c()
+  %call1 = call i32 (...) @c()
+  %call2 = call i32 (...) @c()
+  %tmp3 = load i32, i32* @b, align 4
+  %tmp4 = load i32, i32* @g, align 4
+  %sub = sub nsw i32 %tmp4, %tmp3
+  store i32 %sub, i32* @g, align 4
+  %tmp5 = load i32, i32* %i, align 4
+  %inc = add i32 %tmp5, 1
+  store i32 %inc, i32* %i, align 4
+  br label %for.cond
+}
+
+declare i32 @c(...) #0
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.lifetime.start.p0i8(i64 immarg, i8* nocapture) #2
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.lifetime.end.p0i8(i64 immarg, i8* nocapture) #2
+
+attributes #0 = { "use-soft-float"="false" }
+attributes #1 = { "target-cpu"="x86-64" }
+attributes #2 = { argmemonly nounwind }
+
diff --git a/llvm/test/Transforms/SimplifyCFG/merge-cleanuppads.ll b/llvm/test/Transforms/SimplifyCFG/merge-cleanuppads.ll
index 23bbbca72..94ef8d71d 100644
--- a/llvm/test/Transforms/SimplifyCFG/merge-cleanuppads.ll
+++ b/llvm/test/Transforms/SimplifyCFG/merge-cleanuppads.ll
@@ -34,6 +34,6 @@ declare void @may_throw(i32) #1
 
 declare i32 @__CxxFrameHandler3(...)
 
-attributes #0 = { uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind }
diff --git a/llvm/test/Transforms/SimplifyCFG/merge-cond-stores-2.ll b/llvm/test/Transforms/SimplifyCFG/merge-cond-stores-2.ll
index a2ca63d0a..300f78987 100644
--- a/llvm/test/Transforms/SimplifyCFG/merge-cond-stores-2.ll
+++ b/llvm/test/Transforms/SimplifyCFG/merge-cond-stores-2.ll
@@ -19,126 +19,128 @@ define i32 @f(i32* %b) {
 ; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[TMP1]], 2
 ; CHECK-NEXT:    [[TOBOOL2:%.*]] = icmp eq i32 [[AND1]], 0
 ; CHECK-NEXT:    [[OR4:%.*]] = or i32 [[TMP1]], 1073741824
-; CHECK-NEXT:    [[DOTOR4:%.*]] = select i1 [[TOBOOL2]], i32 [[TMP1]], i32 [[OR4]]
+; CHECK-NEXT:    [[SPEC_SELECT:%.*]] = select i1 [[TOBOOL2]], i32 [[TMP1]], i32 [[OR4]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = xor i1 [[TOBOOL]], true
 ; CHECK-NEXT:    [[TMP3:%.*]] = xor i1 [[TOBOOL2]], true
 ; CHECK-NEXT:    [[TMP4:%.*]] = or i1 [[TMP2]], [[TMP3]]
-; CHECK-NEXT:    [[AND6:%.*]] = and i32 [[DOTOR4]], 4
+; CHECK-NEXT:    [[AND6:%.*]] = and i32 [[SPEC_SELECT]], 4
 ; CHECK-NEXT:    [[TOBOOL7:%.*]] = icmp eq i32 [[AND6]], 0
-; CHECK-NEXT:    [[OR9:%.*]] = or i32 [[DOTOR4]], 536870912
-; CHECK-NEXT:    [[DOTOR4_OR9:%.*]] = select i1 [[TOBOOL7]], i32 [[DOTOR4]], i32 [[OR9]]
+; CHECK-NEXT:    [[OR9:%.*]] = or i32 [[SPEC_SELECT]], 536870912
+; CHECK-NEXT:    [[SPEC_SELECT1:%.*]] = select i1 [[TOBOOL7]], i32 [[SPEC_SELECT]], i32 [[OR9]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = xor i1 [[TMP4]], true
 ; CHECK-NEXT:    [[TMP6:%.*]] = xor i1 [[TOBOOL7]], true
 ; CHECK-NEXT:    [[TMP7:%.*]] = xor i1 [[TMP5]], true
 ; CHECK-NEXT:    [[TMP8:%.*]] = or i1 [[TMP7]], [[TMP6]]
-; CHECK-NEXT:    [[AND11:%.*]] = and i32 [[DOTOR4_OR9]], 8
+; CHECK-NEXT:    [[AND11:%.*]] = and i32 [[SPEC_SELECT1]], 8
 ; CHECK-NEXT:    [[TOBOOL12:%.*]] = icmp eq i32 [[AND11]], 0
-; CHECK-NEXT:    [[OR14:%.*]] = or i32 [[DOTOR4_OR9]], 268435456
-; CHECK-NEXT:    [[DOTOR4_OR9_OR14:%.*]] = select i1 [[TOBOOL12]], i32 [[DOTOR4_OR9]], i32 [[OR14]]
+; CHECK-NEXT:    [[OR14:%.*]] = or i32 [[SPEC_SELECT1]], 268435456
+; CHECK-NEXT:    [[SPEC_SELECT2:%.*]] = select i1 [[TOBOOL12]], i32 [[SPEC_SELECT1]], i32 [[OR14]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = xor i1 [[TMP8]], true
 ; CHECK-NEXT:    [[TMP10:%.*]] = xor i1 [[TOBOOL12]], true
 ; CHECK-NEXT:    [[TMP11:%.*]] = xor i1 [[TMP9]], true
 ; CHECK-NEXT:    [[TMP12:%.*]] = or i1 [[TMP11]], [[TMP10]]
-; CHECK-NEXT:    [[AND16:%.*]] = and i32 [[DOTOR4_OR9_OR14]], 16
+; CHECK-NEXT:    [[AND16:%.*]] = and i32 [[SPEC_SELECT2]], 16
 ; CHECK-NEXT:    [[TOBOOL17:%.*]] = icmp eq i32 [[AND16]], 0
-; CHECK-NEXT:    [[OR19:%.*]] = or i32 [[DOTOR4_OR9_OR14]], 134217728
-; CHECK-NEXT:    [[DOTOR4_OR9_OR14_OR19:%.*]] = select i1 [[TOBOOL17]], i32 [[DOTOR4_OR9_OR14]], i32 [[OR19]]
+; CHECK-NEXT:    [[OR19:%.*]] = or i32 [[SPEC_SELECT2]], 134217728
+; CHECK-NEXT:    [[SPEC_SELECT3:%.*]] = select i1 [[TOBOOL17]], i32 [[SPEC_SELECT2]], i32 [[OR19]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = xor i1 [[TMP12]], true
 ; CHECK-NEXT:    [[TMP14:%.*]] = xor i1 [[TOBOOL17]], true
 ; CHECK-NEXT:    [[TMP15:%.*]] = xor i1 [[TMP13]], true
 ; CHECK-NEXT:    [[TMP16:%.*]] = or i1 [[TMP15]], [[TMP14]]
-; CHECK-NEXT:    [[AND21:%.*]] = and i32 [[DOTOR4_OR9_OR14_OR19]], 32
+; CHECK-NEXT:    [[AND21:%.*]] = and i32 [[SPEC_SELECT3]], 32
 ; CHECK-NEXT:    [[TOBOOL22:%.*]] = icmp eq i32 [[AND21]], 0
-; CHECK-NEXT:    [[OR24:%.*]] = or i32 [[DOTOR4_OR9_OR14_OR19]], 67108864
-; CHECK-NEXT:    [[DOTOR4_OR9_OR14_OR19_OR24:%.*]] = select i1 [[TOBOOL22]], i32 [[DOTOR4_OR9_OR14_OR19]], i32 [[OR24]]
+; CHECK-NEXT:    [[OR24:%.*]] = or i32 [[SPEC_SELECT3]], 67108864
+; CHECK-NEXT:    [[SPEC_SELECT4:%.*]] = select i1 [[TOBOOL22]], i32 [[SPEC_SELECT3]], i32 [[OR24]]
 ; CHECK-NEXT:    [[TMP17:%.*]] = xor i1 [[TMP16]], true
 ; CHECK-NEXT:    [[TMP18:%.*]] = xor i1 [[TOBOOL22]], true
 ; CHECK-NEXT:    [[TMP19:%.*]] = xor i1 [[TMP17]], true
 ; CHECK-NEXT:    [[TMP20:%.*]] = or i1 [[TMP19]], [[TMP18]]
-; CHECK-NEXT:    [[AND26:%.*]] = and i32 [[DOTOR4_OR9_OR14_OR19_OR24]], 64
+; CHECK-NEXT:    [[AND26:%.*]] = and i32 [[SPEC_SELECT4]], 64
 ; CHECK-NEXT:    [[TOBOOL27:%.*]] = icmp eq i32 [[AND26]], 0
-; CHECK-NEXT:    [[OR29:%.*]] = or i32 [[DOTOR4_OR9_OR14_OR19_OR24]], 33554432
-; CHECK-NEXT:    [[DOTOR4_OR9_OR14_OR19_OR24_OR29:%.*]] = select i1 [[TOBOOL27]], i32 [[DOTOR4_OR9_OR14_OR19_OR24]], i32 [[OR29]]
+; CHECK-NEXT:    [[OR29:%.*]] = or i32 [[SPEC_SELECT4]], 33554432
+; CHECK-NEXT:    [[SPEC_SELECT5:%.*]] = select i1 [[TOBOOL27]], i32 [[SPEC_SELECT4]], i32 [[OR29]]
 ; CHECK-NEXT:    [[TMP21:%.*]] = xor i1 [[TMP20]], true
 ; CHECK-NEXT:    [[TMP22:%.*]] = xor i1 [[TOBOOL27]], true
 ; CHECK-NEXT:    [[TMP23:%.*]] = xor i1 [[TMP21]], true
 ; CHECK-NEXT:    [[TMP24:%.*]] = or i1 [[TMP23]], [[TMP22]]
-; CHECK-NEXT:    [[AND31:%.*]] = and i32 [[DOTOR4_OR9_OR14_OR19_OR24_OR29]], 256
+; CHECK-NEXT:    [[AND31:%.*]] = and i32 [[SPEC_SELECT5]], 256
 ; CHECK-NEXT:    [[TOBOOL32:%.*]] = icmp eq i32 [[AND31]], 0
-; CHECK-NEXT:    [[OR34:%.*]] = or i32 [[DOTOR4_OR9_OR14_OR19_OR24_OR29]], 8388608
-; CHECK-NEXT:    [[DOTOR4_OR9_OR14_OR19_OR24_OR29_OR34:%.*]] = select i1 [[TOBOOL32]], i32 [[DOTOR4_OR9_OR14_OR19_OR24_OR29]], i32 [[OR34]]
+; CHECK-NEXT:    [[OR34:%.*]] = or i32 [[SPEC_SELECT5]], 8388608
+; CHECK-NEXT:    [[SPEC_SELECT6:%.*]] = select i1 [[TOBOOL32]], i32 [[SPEC_SELECT5]], i32 [[OR34]]
 ; CHECK-NEXT:    [[TMP25:%.*]] = xor i1 [[TMP24]], true
 ; CHECK-NEXT:    [[TMP26:%.*]] = xor i1 [[TOBOOL32]], true
 ; CHECK-NEXT:    [[TMP27:%.*]] = xor i1 [[TMP25]], true
 ; CHECK-NEXT:    [[TMP28:%.*]] = or i1 [[TMP27]], [[TMP26]]
-; CHECK-NEXT:    [[AND36:%.*]] = and i32 [[DOTOR4_OR9_OR14_OR19_OR24_OR29_OR34]], 512
+; CHECK-NEXT:    [[AND36:%.*]] = and i32 [[SPEC_SELECT6]], 512
 ; CHECK-NEXT:    [[TOBOOL37:%.*]] = icmp eq i32 [[AND36]], 0
-; CHECK-NEXT:    [[OR39:%.*]] = or i32 [[DOTOR4_OR9_OR14_OR19_OR24_OR29_OR34]], 4194304
-; CHECK-NEXT:    [[DOTOR4_OR9_OR14_OR19_OR24_OR29_OR34_OR39:%.*]] = select i1 [[TOBOOL37]], i32 [[DOTOR4_OR9_OR14_OR19_OR24_OR29_OR34]], i32 [[OR39]]
+; CHECK-NEXT:    [[OR39:%.*]] = or i32 [[SPEC_SELECT6]], 4194304
+; CHECK-NEXT:    [[SPEC_SELECT7:%.*]] = select i1 [[TOBOOL37]], i32 [[SPEC_SELECT6]], i32 [[OR39]]
 ; CHECK-NEXT:    [[TMP29:%.*]] = xor i1 [[TMP28]], true
 ; CHECK-NEXT:    [[TMP30:%.*]] = xor i1 [[TOBOOL37]], true
 ; CHECK-NEXT:    [[TMP31:%.*]] = xor i1 [[TMP29]], true
 ; CHECK-NEXT:    [[TMP32:%.*]] = or i1 [[TMP31]], [[TMP30]]
-; CHECK-NEXT:    [[AND41:%.*]] = and i32 [[DOTOR4_OR9_OR14_OR19_OR24_OR29_OR34_OR39]], 1024
+; CHECK-NEXT:    [[AND41:%.*]] = and i32 [[SPEC_SELECT7]], 1024
 ; CHECK-NEXT:    [[TOBOOL42:%.*]] = icmp eq i32 [[AND41]], 0
-; CHECK-NEXT:    [[OR44:%.*]] = or i32 [[DOTOR4_OR9_OR14_OR19_OR24_OR29_OR34_OR39]], 2097152
-; CHECK-NEXT:    [[DOTOR4_OR9_OR14_OR19_OR24_OR29_OR34_OR39_OR44:%.*]] = select i1 [[TOBOOL42]], i32 [[DOTOR4_OR9_OR14_OR19_OR24_OR29_OR34_OR39]], i32 [[OR44]]
+; CHECK-NEXT:    [[OR44:%.*]] = or i32 [[SPEC_SELECT7]], 2097152
+; CHECK-NEXT:    [[SPEC_SELECT8:%.*]] = select i1 [[TOBOOL42]], i32 [[SPEC_SELECT7]], i32 [[OR44]]
 ; CHECK-NEXT:    [[TMP33:%.*]] = xor i1 [[TMP32]], true
 ; CHECK-NEXT:    [[TMP34:%.*]] = xor i1 [[TOBOOL42]], true
 ; CHECK-NEXT:    [[TMP35:%.*]] = xor i1 [[TMP33]], true
 ; CHECK-NEXT:    [[TMP36:%.*]] = or i1 [[TMP35]], [[TMP34]]
-; CHECK-NEXT:    [[AND46:%.*]] = and i32 [[DOTOR4_OR9_OR14_OR19_OR24_OR29_OR34_OR39_OR44]], 2048
+; CHECK-NEXT:    [[AND46:%.*]] = and i32 [[SPEC_SELECT8]], 2048
 ; CHECK-NEXT:    [[TOBOOL47:%.*]] = icmp eq i32 [[AND46]], 0
-; CHECK-NEXT:    [[OR49:%.*]] = or i32 [[DOTOR4_OR9_OR14_OR19_OR24_OR29_OR34_OR39_OR44]], 1048576
-; CHECK-NEXT:    [[DOTOR4_OR9_OR14_OR19_OR24_OR29_OR34_OR39_OR44_OR49:%.*]] = select i1 [[TOBOOL47]], i32 [[DOTOR4_OR9_OR14_OR19_OR24_OR29_OR34_OR39_OR44]], i32 [[OR49]]
+; CHECK-NEXT:    [[OR49:%.*]] = or i32 [[SPEC_SELECT8]], 1048576
+; CHECK-NEXT:    [[SPEC_SELECT9:%.*]] = select i1 [[TOBOOL47]], i32 [[SPEC_SELECT8]], i32 [[OR49]]
 ; CHECK-NEXT:    [[TMP37:%.*]] = xor i1 [[TMP36]], true
 ; CHECK-NEXT:    [[TMP38:%.*]] = xor i1 [[TOBOOL47]], true
 ; CHECK-NEXT:    [[TMP39:%.*]] = xor i1 [[TMP37]], true
 ; CHECK-NEXT:    [[TMP40:%.*]] = or i1 [[TMP39]], [[TMP38]]
-; CHECK-NEXT:    [[AND51:%.*]] = and i32 [[DOTOR4_OR9_OR14_OR19_OR24_OR29_OR34_OR39_OR44_OR49]], 4096
+; CHECK-NEXT:    [[AND51:%.*]] = and i32 [[SPEC_SELECT9]], 4096
 ; CHECK-NEXT:    [[TOBOOL52:%.*]] = icmp eq i32 [[AND51]], 0
-; CHECK-NEXT:    [[OR54:%.*]] = or i32 [[DOTOR4_OR9_OR14_OR19_OR24_OR29_OR34_OR39_OR44_OR49]], 524288
-; CHECK-NEXT:    [[DOTOR4_OR9_OR14_OR19_OR24_OR29_OR34_OR39_OR44_OR49_OR54:%.*]] = select i1 [[TOBOOL52]], i32 [[DOTOR4_OR9_OR14_OR19_OR24_OR29_OR34_OR39_OR44_OR49]], i32 [[OR54]]
+; CHECK-NEXT:    [[OR54:%.*]] = or i32 [[SPEC_SELECT9]], 524288
+; CHECK-NEXT:    [[SPEC_SELECT10:%.*]] = select i1 [[TOBOOL52]], i32 [[SPEC_SELECT9]], i32 [[OR54]]
 ; CHECK-NEXT:    [[TMP41:%.*]] = xor i1 [[TMP40]], true
 ; CHECK-NEXT:    [[TMP42:%.*]] = xor i1 [[TOBOOL52]], true
 ; CHECK-NEXT:    [[TMP43:%.*]] = xor i1 [[TMP41]], true
 ; CHECK-NEXT:    [[TMP44:%.*]] = or i1 [[TMP43]], [[TMP42]]
-; CHECK-NEXT:    [[AND56:%.*]] = and i32 [[DOTOR4_OR9_OR14_OR19_OR24_OR29_OR34_OR39_OR44_OR49_OR54]], 8192
+; CHECK-NEXT:    [[AND56:%.*]] = and i32 [[SPEC_SELECT10]], 8192
 ; CHECK-NEXT:    [[TOBOOL57:%.*]] = icmp eq i32 [[AND56]], 0
-; CHECK-NEXT:    [[OR59:%.*]] = or i32 [[DOTOR4_OR9_OR14_OR19_OR24_OR29_OR34_OR39_OR44_OR49_OR54]], 262144
-; CHECK-NEXT:    [[DOTOR4_OR9_OR14_OR19_OR24_OR29_OR34_OR39_OR44_OR49_OR54_OR59:%.*]] = select i1 [[TOBOOL57]], i32 [[DOTOR4_OR9_OR14_OR19_OR24_OR29_OR34_OR39_OR44_OR49_OR54]], i32 [[OR59]]
+; CHECK-NEXT:    [[OR59:%.*]] = or i32 [[SPEC_SELECT10]], 262144
+; CHECK-NEXT:    [[SPEC_SELECT11:%.*]] = select i1 [[TOBOOL57]], i32 [[SPEC_SELECT10]], i32 [[OR59]]
 ; CHECK-NEXT:    [[TMP45:%.*]] = xor i1 [[TMP44]], true
 ; CHECK-NEXT:    [[TMP46:%.*]] = xor i1 [[TOBOOL57]], true
 ; CHECK-NEXT:    [[TMP47:%.*]] = xor i1 [[TMP45]], true
 ; CHECK-NEXT:    [[TMP48:%.*]] = or i1 [[TMP47]], [[TMP46]]
-; CHECK-NEXT:    [[AND61:%.*]] = and i32 [[DOTOR4_OR9_OR14_OR19_OR24_OR29_OR34_OR39_OR44_OR49_OR54_OR59]], 16384
+; CHECK-NEXT:    [[AND61:%.*]] = and i32 [[SPEC_SELECT11]], 16384
 ; CHECK-NEXT:    [[TOBOOL62:%.*]] = icmp eq i32 [[AND61]], 0
-; CHECK-NEXT:    [[OR64:%.*]] = or i32 [[DOTOR4_OR9_OR14_OR19_OR24_OR29_OR34_OR39_OR44_OR49_OR54_OR59]], 131072
-; CHECK-NEXT:    [[DOTOR4_OR9_OR14_OR19_OR24_OR29_OR34_OR39_OR44_OR49_OR54_OR59_OR64:%.*]] = select i1 [[TOBOOL62]], i32 [[DOTOR4_OR9_OR14_OR19_OR24_OR29_OR34_OR39_OR44_OR49_OR54_OR59]], i32 [[OR64]]
+; CHECK-NEXT:    [[OR64:%.*]] = or i32 [[SPEC_SELECT11]], 131072
+; CHECK-NEXT:    [[SPEC_SELECT12:%.*]] = select i1 [[TOBOOL62]], i32 [[SPEC_SELECT11]], i32 [[OR64]]
 ; CHECK-NEXT:    [[TMP49:%.*]] = xor i1 [[TMP48]], true
 ; CHECK-NEXT:    [[TMP50:%.*]] = xor i1 [[TOBOOL62]], true
 ; CHECK-NEXT:    [[TMP51:%.*]] = xor i1 [[TMP49]], true
 ; CHECK-NEXT:    [[TMP52:%.*]] = or i1 [[TMP51]], [[TMP50]]
-; CHECK-NEXT:    [[AND66:%.*]] = and i32 [[DOTOR4_OR9_OR14_OR19_OR24_OR29_OR34_OR39_OR44_OR49_OR54_OR59_OR64]], 32768
+; CHECK-NEXT:    [[AND66:%.*]] = and i32 [[SPEC_SELECT12]], 32768
 ; CHECK-NEXT:    [[TOBOOL67:%.*]] = icmp eq i32 [[AND66]], 0
-; CHECK-NEXT:    [[OR69:%.*]] = or i32 [[DOTOR4_OR9_OR14_OR19_OR24_OR29_OR34_OR39_OR44_OR49_OR54_OR59_OR64]], 65536
-; CHECK-NEXT:    [[DOTOR4_OR9_OR14_OR19_OR24_OR29_OR34_OR39_OR44_OR49_OR54_OR59_OR64_OR69:%.*]] = select i1 [[TOBOOL67]], i32 [[DOTOR4_OR9_OR14_OR19_OR24_OR29_OR34_OR39_OR44_OR49_OR54_OR59_OR64]], i32 [[OR69]]
+; CHECK-NEXT:    [[OR69:%.*]] = or i32 [[SPEC_SELECT12]], 65536
+; CHECK-NEXT:    [[SPEC_SELECT13:%.*]] = select i1 [[TOBOOL67]], i32 [[SPEC_SELECT12]], i32 [[OR69]]
 ; CHECK-NEXT:    [[TMP53:%.*]] = xor i1 [[TMP52]], true
 ; CHECK-NEXT:    [[TMP54:%.*]] = xor i1 [[TOBOOL67]], true
 ; CHECK-NEXT:    [[TMP55:%.*]] = xor i1 [[TMP53]], true
 ; CHECK-NEXT:    [[TMP56:%.*]] = or i1 [[TMP55]], [[TMP54]]
-; CHECK-NEXT:    [[AND71:%.*]] = and i32 [[DOTOR4_OR9_OR14_OR19_OR24_OR29_OR34_OR39_OR44_OR49_OR54_OR59_OR64_OR69]], 128
+; CHECK-NEXT:    [[AND71:%.*]] = and i32 [[SPEC_SELECT13]], 128
 ; CHECK-NEXT:    [[TOBOOL72:%.*]] = icmp eq i32 [[AND71]], 0
-; CHECK-NEXT:    [[OR74:%.*]] = or i32 [[DOTOR4_OR9_OR14_OR19_OR24_OR29_OR34_OR39_OR44_OR49_OR54_OR59_OR64_OR69]], 16777216
-; CHECK-NEXT:    [[DOTOR4_OR9_OR14_OR19_OR24_OR29_OR34_OR39_OR44_OR49_OR54_OR59_OR64_OR69_OR74:%.*]] = select i1 [[TOBOOL72]], i32 [[DOTOR4_OR9_OR14_OR19_OR24_OR29_OR34_OR39_OR44_OR49_OR54_OR59_OR64_OR69]], i32 [[OR74]]
+; CHECK-NEXT:    [[OR74:%.*]] = or i32 [[SPEC_SELECT13]], 16777216
+; CHECK-NEXT:    [[SPEC_SELECT14:%.*]] = select i1 [[TOBOOL72]], i32 [[SPEC_SELECT13]], i32 [[OR74]]
 ; CHECK-NEXT:    [[TMP57:%.*]] = xor i1 [[TMP56]], true
 ; CHECK-NEXT:    [[TMP58:%.*]] = xor i1 [[TOBOOL72]], true
 ; CHECK-NEXT:    [[TMP59:%.*]] = xor i1 [[TMP57]], true
 ; CHECK-NEXT:    [[TMP60:%.*]] = or i1 [[TMP59]], [[TMP58]]
 ; CHECK-NEXT:    br i1 [[TMP60]], label [[TMP61:%.*]], label [[TMP62:%.*]]
-; CHECK:         store i32 [[DOTOR4_OR9_OR14_OR19_OR24_OR29_OR34_OR39_OR44_OR49_OR54_OR59_OR64_OR69_OR74]], i32* [[B]], align 4
+; CHECK:       61:
+; CHECK-NEXT:    store i32 [[SPEC_SELECT14]], i32* [[B]], align 4
 ; CHECK-NEXT:    br label [[TMP62]]
-; CHECK:         ret i32 0
+; CHECK:       62:
+; CHECK-NEXT:    ret i32 0
 ;
 entry:
   %0 = load i32, i32* %b, align 4
diff --git a/llvm/test/Transforms/SimplifyCFG/merge-cond-stores.ll b/llvm/test/Transforms/SimplifyCFG/merge-cond-stores.ll
index fd629c8ce..e9a3e608e 100644
--- a/llvm/test/Transforms/SimplifyCFG/merge-cond-stores.ll
+++ b/llvm/test/Transforms/SimplifyCFG/merge-cond-stores.ll
@@ -10,11 +10,13 @@ define void @test_simple(i32* %p, i32 %a, i32 %b) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = xor i1 [[X2]], true
 ; CHECK-NEXT:    [[TMP1:%.*]] = or i1 [[X1]], [[TMP0]]
 ; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP3:%.*]]
-; CHECK:         [[NOT_X2:%.*]] = xor i1 [[X2]], true
+; CHECK:       2:
+; CHECK-NEXT:    [[NOT_X2:%.*]] = xor i1 [[X2]], true
 ; CHECK-NEXT:    [[SPEC_SELECT:%.*]] = zext i1 [[NOT_X2]] to i32
 ; CHECK-NEXT:    store i32 [[SPEC_SELECT]], i32* [[P:%.*]], align 4
 ; CHECK-NEXT:    br label [[TMP3]]
-; CHECK:         ret void
+; CHECK:       3:
+; CHECK-NEXT:    ret void
 ;
 entry:
   %x1 = icmp eq i32 %a, 0
@@ -44,10 +46,12 @@ define void @test_simple_commuted(i32* %p, i32 %a, i32 %b) {
 ; CHECK-NEXT:    [[X2:%.*]] = icmp eq i32 [[B:%.*]], 0
 ; CHECK-NEXT:    [[TMP0:%.*]] = or i1 [[X1]], [[X2]]
 ; CHECK-NEXT:    br i1 [[TMP0]], label [[TMP1:%.*]], label [[TMP2:%.*]]
-; CHECK:         [[SPEC_SELECT:%.*]] = zext i1 [[X2]] to i32
+; CHECK:       1:
+; CHECK-NEXT:    [[SPEC_SELECT:%.*]] = zext i1 [[X2]] to i32
 ; CHECK-NEXT:    store i32 [[SPEC_SELECT]], i32* [[P:%.*]], align 4
 ; CHECK-NEXT:    br label [[TMP2]]
-; CHECK:         ret void
+; CHECK:       2:
+; CHECK-NEXT:    ret void
 ;
 entry:
   %x1 = icmp eq i32 %a, 0
@@ -80,14 +84,16 @@ define void @test_recursive(i32* %p, i32 %a, i32 %b, i32 %c, i32 %d) {
 ; CHECK-NEXT:    [[TMP3:%.*]] = xor i1 [[X4]], true
 ; CHECK-NEXT:    [[TMP4:%.*]] = or i1 [[TMP2]], [[TMP3]]
 ; CHECK-NEXT:    br i1 [[TMP4]], label [[TMP5:%.*]], label [[TMP6:%.*]]
-; CHECK:         [[X3:%.*]] = icmp eq i32 [[C]], 0
+; CHECK:       5:
+; CHECK-NEXT:    [[X3:%.*]] = icmp eq i32 [[C]], 0
 ; CHECK-NEXT:    [[X2:%.*]] = icmp ne i32 [[B]], 0
 ; CHECK-NEXT:    [[SPEC_SELECT:%.*]] = zext i1 [[X2]] to i32
 ; CHECK-NEXT:    [[SPEC_SELECT1:%.*]] = select i1 [[X3]], i32 [[SPEC_SELECT]], i32 2
 ; CHECK-NEXT:    [[SPEC_SELECT2:%.*]] = select i1 [[X4]], i32 [[SPEC_SELECT1]], i32 3
 ; CHECK-NEXT:    store i32 [[SPEC_SELECT2]], i32* [[P:%.*]], align 4
 ; CHECK-NEXT:    br label [[TMP6]]
-; CHECK:         ret void
+; CHECK:       6:
+; CHECK-NEXT:    ret void
 ;
 entry:
   %x1 = icmp eq i32 %a, 0
@@ -271,10 +277,12 @@ define i32 @test_diamond_simple(i32* %p, i32* %q, i32 %a, i32 %b) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = or i32 [[A]], [[B]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[TMP0]], 0
 ; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP3:%.*]], label [[TMP2:%.*]]
-; CHECK:         [[SIMPLIFYCFG_MERGE:%.*]] = select i1 [[X2]], i32 [[Z2]], i32 1
+; CHECK:       2:
+; CHECK-NEXT:    [[SIMPLIFYCFG_MERGE:%.*]] = select i1 [[X2]], i32 [[Z2]], i32 1
 ; CHECK-NEXT:    store i32 [[SIMPLIFYCFG_MERGE]], i32* [[P:%.*]], align 4
 ; CHECK-NEXT:    br label [[TMP3]]
-; CHECK:         ret i32 [[Z4]]
+; CHECK:       3:
+; CHECK-NEXT:    ret i32 [[Z4]]
 ;
 entry:
   %x1 = icmp eq i32 %a, 0
@@ -380,7 +388,8 @@ define void @test_outer_if(i32* %p, i32 %a, i32 %b, i32 %c) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = xor i1 [[X2]], true
 ; CHECK-NEXT:    [[TMP1:%.*]] = or i1 [[X1]], [[TMP0]]
 ; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP2:%.*]], label [[END]]
-; CHECK:         [[NOT_X2:%.*]] = xor i1 [[X2]], true
+; CHECK:       2:
+; CHECK-NEXT:    [[NOT_X2:%.*]] = xor i1 [[X2]], true
 ; CHECK-NEXT:    [[SPEC_SELECT:%.*]] = zext i1 [[NOT_X2]] to i32
 ; CHECK-NEXT:    store i32 [[SPEC_SELECT]], i32* [[P:%.*]], align 4
 ; CHECK-NEXT:    br label [[END]]
@@ -405,4 +414,3 @@ yes2:
 end:
   ret void
 }
-
diff --git a/llvm/test/Transforms/SimplifyCFG/safe-abs.ll b/llvm/test/Transforms/SimplifyCFG/safe-abs.ll
new file mode 100644
index 000000000..6d8028f8d
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/safe-abs.ll
@@ -0,0 +1,30 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -simplifycfg -S | FileCheck %s
+
+; Reduced from arm_abs_q31() from CMSIS DSP suite.
+; https://reviews.llvm.org/D65148#1629010
+
+define i32 @abs_with_clamp(i32 %arg) {
+; CHECK-LABEL: @abs_with_clamp(
+; CHECK-NEXT:  begin:
+; CHECK-NEXT:    [[IS_POSITIVE:%.*]] = icmp sgt i32 [[ARG:%.*]], 0
+; CHECK-NEXT:    [[IS_INT_MIN:%.*]] = icmp eq i32 [[ARG]], -2147483648
+; CHECK-NEXT:    [[NEGATED:%.*]] = sub nsw i32 0, [[ARG]]
+; CHECK-NEXT:    [[ABS:%.*]] = select i1 [[IS_INT_MIN]], i32 2147483647, i32 [[NEGATED]]
+; CHECK-NEXT:    [[TMP6:%.*]] = select i1 [[IS_POSITIVE]], i32 [[ARG]], i32 [[ABS]]
+; CHECK-NEXT:    ret i32 [[TMP6]]
+;
+begin:
+  %is_positive = icmp sgt i32 %arg, 0
+  br i1 %is_positive, label %end, label %negative
+
+negative: ; preds = %begin
+  %is_int_min = icmp eq i32 %arg, -2147483648
+  %negated = sub nsw i32 0, %arg
+  %abs = select i1 %is_int_min, i32 2147483647, i32 %negated
+  br label %end
+
+end:      ; preds = %negative, %begin
+  %tmp6 = phi i32 [ %arg, %begin ], [ %abs, %negative ]
+  ret i32 %tmp6
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/safe-low-bit-extract.ll b/llvm/test/Transforms/SimplifyCFG/safe-low-bit-extract.ll
new file mode 100644
index 000000000..19a6313aa
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/safe-low-bit-extract.ll
@@ -0,0 +1,31 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -simplifycfg -S | FileCheck %s
+
+; This is the naive implementation of x86 BZHI/BEXTR instruction:
+; it takes input and bit count, and extracts low nbits up to bit width.
+; I.e. unlike shift it does not have any UB when nbits >= bitwidth.
+; Which means we don't need a while PHI here, simple select will do.
+define i32 @extract_low_bits(i32 %input, i32 %nbits) {
+; CHECK-LABEL: @extract_low_bits(
+; CHECK-NEXT:  begin:
+; CHECK-NEXT:    [[SHOULD_MASK:%.*]] = icmp ult i32 [[NBITS:%.*]], 32
+; CHECK-NEXT:    [[MASK_NOT:%.*]] = shl nsw i32 -1, [[NBITS]]
+; CHECK-NEXT:    [[MASK:%.*]] = xor i32 [[MASK_NOT]], -1
+; CHECK-NEXT:    [[MASKED:%.*]] = and i32 [[MASK]], [[INPUT:%.*]]
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[SHOULD_MASK]], i32 [[MASKED]], i32 [[INPUT]]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+begin:
+  %should_mask = icmp ult i32 %nbits, 32
+  br i1 %should_mask, label %perform_masking, label %end
+
+perform_masking: ; preds = %begin
+  %mask.not = shl nsw i32 -1, %nbits
+  %mask = xor i32 %mask.not, -1
+  %masked = and i32 %mask, %input
+  br label %end
+
+end:             ; preds = %perform_masking, %begin
+  %res = phi i32 [ %masked, %perform_masking ], [ %input, %begin ]
+  ret i32 %res
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/signbit-like-value-extension.ll b/llvm/test/Transforms/SimplifyCFG/signbit-like-value-extension.ll
new file mode 100644
index 000000000..e955e0679
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/signbit-like-value-extension.ll
@@ -0,0 +1,35 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -simplifycfg -S | FileCheck %s
+
+; This looks like a normal sign extension, but it's not.
+; This comes up in JPEG decoding. This check is really unpredictable,
+; so it is likely beneficial to avoid the branch.
+
+define i32 @extend_value(i32 %storage, i32 %nbits) {
+; CHECK-LABEL: @extend_value(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[SKIPNBITS:%.*]] = sub i32 32, [[NBITS:%.*]]
+; CHECK-NEXT:    [[VALUE:%.*]] = lshr i32 [[STORAGE:%.*]], [[SKIPNBITS]]
+; CHECK-NEXT:    [[SHOULDEXTEND:%.*]] = icmp sgt i32 [[STORAGE]], -1
+; CHECK-NEXT:    [[HIGHBITMASK:%.*]] = shl nsw i32 -1, [[NBITS]]
+; CHECK-NEXT:    [[HIGHBITMASKPLUSONE:%.*]] = add nsw i32 [[HIGHBITMASK]], 1
+; CHECK-NEXT:    [[EXTENDED:%.*]] = add i32 [[HIGHBITMASKPLUSONE]], [[VALUE]]
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[SHOULDEXTEND]], i32 [[EXTENDED]], i32 [[VALUE]]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+bb:
+  %skipnbits = sub i32 32, %nbits
+  %value = lshr i32 %storage, %skipnbits
+  %shouldextend = icmp sgt i32 %storage, -1
+  br i1 %shouldextend, label %extend, label %end
+
+extend:                                              ; preds = %bb
+  %highbitmask = shl nsw i32 -1, %nbits
+  %highbitmaskplusone = add nsw i32 %highbitmask, 1
+  %extended = add i32 %highbitmaskplusone, %value
+  br label %end
+
+end:                                              ; preds = %extend, %bb
+  %res = phi i32 [ %extended, %extend ], [ %value, %bb ]
+  ret i32 %res
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/sink-common-code.ll b/llvm/test/Transforms/SimplifyCFG/sink-common-code.ll
index 12a3e59cd..8254a49e2 100644
--- a/llvm/test/Transforms/SimplifyCFG/sink-common-code.ll
+++ b/llvm/test/Transforms/SimplifyCFG/sink-common-code.ll
@@ -886,6 +886,33 @@ return:
 ; CHECK: ret
 }
 
+; CHECK-LABEL: @test_not_sink_lifetime_marker
+; CHECK-NOT: select
+; CHECK: call void @llvm.lifetime.end
+; CHECK: call void @llvm.lifetime.end
+define i32 @test_not_sink_lifetime_marker(i1 zeroext %flag, i32 %x) {
+entry:
+  %y = alloca i32
+  %z = alloca i32
+  br i1 %flag, label %if.then, label %if.else
+
+if.then:
+  %y.cast = bitcast i32* %y to i8*
+  call void @llvm.lifetime.end.p0i8(i64 4, i8* %y.cast)
+  br label %if.end
+
+if.else:
+  %z.cast = bitcast i32* %z to i8*
+  call void @llvm.lifetime.end.p0i8(i64 4, i8* %z.cast)
+  br label %if.end
+
+if.end:
+  ret i32 1
+}
+
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture)
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture)
+
 
 ; CHECK: ![[$TBAA]] = !{![[TYPE:[0-9]]], ![[TYPE]], i64 0}
 ; CHECK: ![[TYPE]] = !{!"float", ![[TEXT:[0-9]]]}
diff --git a/llvm/test/Transforms/SimplifyCFG/speculate-math.ll b/llvm/test/Transforms/SimplifyCFG/speculate-math.ll
index 87e01663e..a99b86ae5 100644
--- a/llvm/test/Transforms/SimplifyCFG/speculate-math.ll
+++ b/llvm/test/Transforms/SimplifyCFG/speculate-math.ll
@@ -1,5 +1,6 @@
-; RUN: opt -S -simplifycfg < %s | FileCheck %s --check-prefix=EXPENSIVE --check-prefix=ALL
-; RUN: opt -S -simplifycfg -speculate-one-expensive-inst=false < %s | FileCheck %s --check-prefix=CHEAP --check-prefix=ALL
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -simplifycfg < %s | FileCheck %s --check-prefix=ALL --check-prefix=EXPENSIVE
+; RUN: opt -S -simplifycfg -speculate-one-expensive-inst=false < %s | FileCheck %s --check-prefix=ALL --check-prefix=CHEAP
 
 declare float @llvm.sqrt.f32(float) nounwind readonly
 declare float @llvm.fma.f32(float, float, float) nounwind readonly
@@ -10,11 +11,14 @@ declare float @llvm.maxnum.f32(float, float) nounwind readonly
 declare float @llvm.minimum.f32(float, float) nounwind readonly
 declare float @llvm.maximum.f32(float, float) nounwind readonly
 
-; ALL-LABEL: @fdiv_test(
-; EXPENSIVE: select i1 %cmp, double %div, double 0.0
-; CHEAP-NOT: select
-
 define double @fdiv_test(double %a, double %b) {
+; ALL-LABEL: @fdiv_test(
+; ALL-NEXT:  entry:
+; ALL-NEXT:    [[CMP:%.*]] = fcmp ogt double [[A:%.*]], 0.000000e+00
+; ALL-NEXT:    [[DIV:%.*]] = fdiv double [[B:%.*]], [[A]]
+; ALL-NEXT:    [[COND:%.*]] = select nsz i1 [[CMP]], double [[DIV]], double 0.000000e+00
+; ALL-NEXT:    ret double [[COND]]
+;
 entry:
   %cmp = fcmp ogt double %a, 0.0
   br i1 %cmp, label %cond.true, label %cond.end
@@ -24,13 +28,19 @@ cond.true:
   br label %cond.end
 
 cond.end:
-  %cond = phi double [ %div, %cond.true ], [ 0.0, %entry ]
+  %cond = phi nsz double [ %div, %cond.true ], [ 0.0, %entry ]
   ret double %cond
 }
 
-; ALL-LABEL: @sqrt_test(
-; ALL: select
 define void @sqrt_test(float addrspace(1)* noalias nocapture %out, float %a) nounwind {
+; ALL-LABEL: @sqrt_test(
+; ALL-NEXT:  entry:
+; ALL-NEXT:    [[CMP_I:%.*]] = fcmp olt float [[A:%.*]], 0.000000e+00
+; ALL-NEXT:    [[TMP0:%.*]] = tail call float @llvm.sqrt.f32(float [[A]]) #2
+; ALL-NEXT:    [[COND_I:%.*]] = select afn i1 [[CMP_I]], float 0x7FF8000000000000, float [[TMP0]]
+; ALL-NEXT:    store float [[COND_I]], float addrspace(1)* [[OUT:%.*]], align 4
+; ALL-NEXT:    ret void
+;
 entry:
   %cmp.i = fcmp olt float %a, 0.000000e+00
   br i1 %cmp.i, label %test_sqrt.exit, label %cond.else.i
@@ -40,14 +50,20 @@ cond.else.i:                                      ; preds = %entry
   br label %test_sqrt.exit
 
 test_sqrt.exit:                                   ; preds = %cond.else.i, %entry
-  %cond.i = phi float [ %0, %cond.else.i ], [ 0x7FF8000000000000, %entry ]
+  %cond.i = phi afn float [ %0, %cond.else.i ], [ 0x7FF8000000000000, %entry ]
   store float %cond.i, float addrspace(1)* %out, align 4
   ret void
 }
 
-; ALL-LABEL: @fabs_test(
-; ALL: select
 define void @fabs_test(float addrspace(1)* noalias nocapture %out, float %a) nounwind {
+; ALL-LABEL: @fabs_test(
+; ALL-NEXT:  entry:
+; ALL-NEXT:    [[CMP_I:%.*]] = fcmp olt float [[A:%.*]], 0.000000e+00
+; ALL-NEXT:    [[TMP0:%.*]] = tail call float @llvm.fabs.f32(float [[A]]) #2
+; ALL-NEXT:    [[COND_I:%.*]] = select reassoc i1 [[CMP_I]], float 0x7FF8000000000000, float [[TMP0]]
+; ALL-NEXT:    store float [[COND_I]], float addrspace(1)* [[OUT:%.*]], align 4
+; ALL-NEXT:    ret void
+;
 entry:
   %cmp.i = fcmp olt float %a, 0.000000e+00
   br i1 %cmp.i, label %test_fabs.exit, label %cond.else.i
@@ -57,14 +73,20 @@ cond.else.i:                                      ; preds = %entry
   br label %test_fabs.exit
 
 test_fabs.exit:                                   ; preds = %cond.else.i, %entry
-  %cond.i = phi float [ %0, %cond.else.i ], [ 0x7FF8000000000000, %entry ]
+  %cond.i = phi reassoc float [ %0, %cond.else.i ], [ 0x7FF8000000000000, %entry ]
   store float %cond.i, float addrspace(1)* %out, align 4
   ret void
 }
 
-; ALL-LABEL: @fma_test(
-; ALL: select
 define void @fma_test(float addrspace(1)* noalias nocapture %out, float %a, float %b, float %c) nounwind {
+; ALL-LABEL: @fma_test(
+; ALL-NEXT:  entry:
+; ALL-NEXT:    [[CMP_I:%.*]] = fcmp olt float [[A:%.*]], 0.000000e+00
+; ALL-NEXT:    [[TMP0:%.*]] = tail call float @llvm.fma.f32(float [[A]], float [[B:%.*]], float [[C:%.*]]) #2
+; ALL-NEXT:    [[COND_I:%.*]] = select reassoc nsz i1 [[CMP_I]], float 0x7FF8000000000000, float [[TMP0]]
+; ALL-NEXT:    store float [[COND_I]], float addrspace(1)* [[OUT:%.*]], align 4
+; ALL-NEXT:    ret void
+;
 entry:
   %cmp.i = fcmp olt float %a, 0.000000e+00
   br i1 %cmp.i, label %test_fma.exit, label %cond.else.i
@@ -74,14 +96,20 @@ cond.else.i:                                      ; preds = %entry
   br label %test_fma.exit
 
 test_fma.exit:                                   ; preds = %cond.else.i, %entry
-  %cond.i = phi float [ %0, %cond.else.i ], [ 0x7FF8000000000000, %entry ]
+  %cond.i = phi nsz reassoc float [ %0, %cond.else.i ], [ 0x7FF8000000000000, %entry ]
   store float %cond.i, float addrspace(1)* %out, align 4
   ret void
 }
 
-; ALL-LABEL: @fmuladd_test(
-; ALL: select
 define void @fmuladd_test(float addrspace(1)* noalias nocapture %out, float %a, float %b, float %c) nounwind {
+; ALL-LABEL: @fmuladd_test(
+; ALL-NEXT:  entry:
+; ALL-NEXT:    [[CMP_I:%.*]] = fcmp olt float [[A:%.*]], 0.000000e+00
+; ALL-NEXT:    [[TMP0:%.*]] = tail call float @llvm.fmuladd.f32(float [[A]], float [[B:%.*]], float [[C:%.*]]) #2
+; ALL-NEXT:    [[COND_I:%.*]] = select ninf i1 [[CMP_I]], float 0x7FF8000000000000, float [[TMP0]]
+; ALL-NEXT:    store float [[COND_I]], float addrspace(1)* [[OUT:%.*]], align 4
+; ALL-NEXT:    ret void
+;
 entry:
   %cmp.i = fcmp olt float %a, 0.000000e+00
   br i1 %cmp.i, label %test_fmuladd.exit, label %cond.else.i
@@ -91,14 +119,20 @@ cond.else.i:                                      ; preds = %entry
   br label %test_fmuladd.exit
 
 test_fmuladd.exit:                                   ; preds = %cond.else.i, %entry
-  %cond.i = phi float [ %0, %cond.else.i ], [ 0x7FF8000000000000, %entry ]
+  %cond.i = phi ninf float [ %0, %cond.else.i ], [ 0x7FF8000000000000, %entry ]
   store float %cond.i, float addrspace(1)* %out, align 4
   ret void
 }
 
-; ALL-LABEL: @minnum_test(
-; ALL: select
 define void @minnum_test(float addrspace(1)* noalias nocapture %out, float %a, float %b) nounwind {
+; ALL-LABEL: @minnum_test(
+; ALL-NEXT:  entry:
+; ALL-NEXT:    [[CMP_I:%.*]] = fcmp olt float [[A:%.*]], 0.000000e+00
+; ALL-NEXT:    [[TMP0:%.*]] = tail call float @llvm.minnum.f32(float [[A]], float [[B:%.*]]) #2
+; ALL-NEXT:    [[COND_I:%.*]] = select i1 [[CMP_I]], float 0x7FF8000000000000, float [[TMP0]]
+; ALL-NEXT:    store float [[COND_I]], float addrspace(1)* [[OUT:%.*]], align 4
+; ALL-NEXT:    ret void
+;
 entry:
   %cmp.i = fcmp olt float %a, 0.000000e+00
   br i1 %cmp.i, label %test_minnum.exit, label %cond.else.i
@@ -113,9 +147,15 @@ test_minnum.exit:                                   ; preds = %cond.else.i, %ent
   ret void
 }
 
-; ALL-LABEL: @maxnum_test(
-; ALL: select
 define void @maxnum_test(float addrspace(1)* noalias nocapture %out, float %a, float %b) nounwind {
+; ALL-LABEL: @maxnum_test(
+; ALL-NEXT:  entry:
+; ALL-NEXT:    [[CMP_I:%.*]] = fcmp olt float [[A:%.*]], 0.000000e+00
+; ALL-NEXT:    [[TMP0:%.*]] = tail call float @llvm.maxnum.f32(float [[A]], float [[B:%.*]]) #2
+; ALL-NEXT:    [[COND_I:%.*]] = select ninf nsz i1 [[CMP_I]], float 0x7FF8000000000000, float [[TMP0]]
+; ALL-NEXT:    store float [[COND_I]], float addrspace(1)* [[OUT:%.*]], align 4
+; ALL-NEXT:    ret void
+;
 entry:
   %cmp.i = fcmp olt float %a, 0.000000e+00
   br i1 %cmp.i, label %test_maxnum.exit, label %cond.else.i
@@ -125,14 +165,20 @@ cond.else.i:                                      ; preds = %entry
   br label %test_maxnum.exit
 
 test_maxnum.exit:                                   ; preds = %cond.else.i, %entry
-  %cond.i = phi float [ %0, %cond.else.i ], [ 0x7FF8000000000000, %entry ]
+  %cond.i = phi ninf nsz float [ %0, %cond.else.i ], [ 0x7FF8000000000000, %entry ]
   store float %cond.i, float addrspace(1)* %out, align 4
   ret void
 }
 
-; ALL-LABEL: @minimum_test(
-; ALL: select
 define void @minimum_test(float addrspace(1)* noalias nocapture %out, float %a, float %b) nounwind {
+; ALL-LABEL: @minimum_test(
+; ALL-NEXT:  entry:
+; ALL-NEXT:    [[CMP_I:%.*]] = fcmp olt float [[A:%.*]], 0.000000e+00
+; ALL-NEXT:    [[TMP0:%.*]] = tail call float @llvm.minimum.f32(float [[A]], float [[B:%.*]]) #2
+; ALL-NEXT:    [[COND_I:%.*]] = select reassoc i1 [[CMP_I]], float 0x7FF8000000000000, float [[TMP0]]
+; ALL-NEXT:    store float [[COND_I]], float addrspace(1)* [[OUT:%.*]], align 4
+; ALL-NEXT:    ret void
+;
 entry:
   %cmp.i = fcmp olt float %a, 0.000000e+00
   br i1 %cmp.i, label %test_minimum.exit, label %cond.else.i
@@ -142,14 +188,20 @@ cond.else.i:                                      ; preds = %entry
   br label %test_minimum.exit
 
 test_minimum.exit:                                   ; preds = %cond.else.i, %entry
-  %cond.i = phi float [ %0, %cond.else.i ], [ 0x7FF8000000000000, %entry ]
+  %cond.i = phi reassoc float [ %0, %cond.else.i ], [ 0x7FF8000000000000, %entry ]
   store float %cond.i, float addrspace(1)* %out, align 4
   ret void
 }
 
-; ALL-LABEL: @maximum_test(
-; ALL: select
 define void @maximum_test(float addrspace(1)* noalias nocapture %out, float %a, float %b) nounwind {
+; ALL-LABEL: @maximum_test(
+; ALL-NEXT:  entry:
+; ALL-NEXT:    [[CMP_I:%.*]] = fcmp olt float [[A:%.*]], 0.000000e+00
+; ALL-NEXT:    [[TMP0:%.*]] = tail call float @llvm.maximum.f32(float [[A]], float [[B:%.*]]) #2
+; ALL-NEXT:    [[COND_I:%.*]] = select nsz i1 [[CMP_I]], float 0x7FF8000000000000, float [[TMP0]]
+; ALL-NEXT:    store float [[COND_I]], float addrspace(1)* [[OUT:%.*]], align 4
+; ALL-NEXT:    ret void
+;
 entry:
   %cmp.i = fcmp olt float %a, 0.000000e+00
   br i1 %cmp.i, label %test_maximum.exit, label %cond.else.i
@@ -159,7 +211,7 @@ cond.else.i:                                      ; preds = %entry
   br label %test_maximum.exit
 
 test_maximum.exit:                                   ; preds = %cond.else.i, %entry
-  %cond.i = phi float [ %0, %cond.else.i ], [ 0x7FF8000000000000, %entry ]
+  %cond.i = phi nsz float [ %0, %cond.else.i ], [ 0x7FF8000000000000, %entry ]
   store float %cond.i, float addrspace(1)* %out, align 4
   ret void
 }
diff --git a/llvm/test/Transforms/SimplifyCFG/switch-range-to-icmp.ll b/llvm/test/Transforms/SimplifyCFG/switch-range-to-icmp.ll
index c0f6a43fd..d99e84b6b 100644
--- a/llvm/test/Transforms/SimplifyCFG/switch-range-to-icmp.ll
+++ b/llvm/test/Transforms/SimplifyCFG/switch-range-to-icmp.ll
@@ -100,3 +100,41 @@ b:
   %1 = call i32 @f(i32 1)
   ret i32 %1
 }
+
+; This would crash because we did not clean up the
+; default block of the switch before removing the switch.
+
+define void @PR42737(i32* %a, i1 %c) {
+; CHECK-LABEL: @PR42737(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = xor i1 [[C:%.*]], true
+; CHECK-NEXT:    call void @llvm.assume(i1 [[TMP0]])
+; CHECK-NEXT:    unreachable
+;
+entry:
+  br i1 %c, label %switch, label %else
+
+else:
+  store i32 2, i32* %a
+  br label %switch
+
+switch:
+  %cleanup.dest1 = phi i32 [ 0, %else ], [ 3, %entry ]
+  switch i32 %cleanup.dest1, label %unreach1 [
+  i32 0, label %cleanup1
+  i32 3, label %cleanup2
+  ]
+
+cleanup1:
+  br label %unreach2
+
+cleanup2:
+  br label %unreach2
+
+unreach1:
+  %phi2 = phi i32 [ %cleanup.dest1, %switch ]
+  unreachable
+
+unreach2:
+  unreachable
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/switch_create-custom-dl.ll b/llvm/test/Transforms/SimplifyCFG/switch_create-custom-dl.ll
index 083cfe1ee..dc07eb205 100644
--- a/llvm/test/Transforms/SimplifyCFG/switch_create-custom-dl.ll
+++ b/llvm/test/Transforms/SimplifyCFG/switch_create-custom-dl.ll
@@ -33,10 +33,10 @@ F:              ; preds = %0
 
 define void @test1_ptr(i32* %V) {
 ; CHECK-LABEL: @test1_ptr(
-; CHECK-NEXT:    [[MAGICPTR:%.*]] = ptrtoint i32* [[V:%.*]] to i32
-; CHECK-NEXT:    switch i32 [[MAGICPTR]], label [[F:%.*]] [
-; CHECK-NEXT:    i32 17, label [[T:%.*]]
-; CHECK-NEXT:    i32 4, label [[T]]
+; CHECK-NEXT:    [[MAGICPTR:%.*]] = ptrtoint i32* [[V:%.*]] to i40
+; CHECK-NEXT:    switch i40 [[MAGICPTR]], label [[F:%.*]] [
+; CHECK-NEXT:    i40 17, label [[T:%.*]]
+; CHECK-NEXT:    i40 4, label [[T]]
 ; CHECK-NEXT:    ]
 ; CHECK:       T:
 ; CHECK-NEXT:    call void @foo1()
@@ -59,10 +59,10 @@ F:              ; preds = %0
 
 define void @test1_ptr_as1(i32 addrspace(1)* %V) {
 ; CHECK-LABEL: @test1_ptr_as1(
-; CHECK-NEXT:    [[MAGICPTR:%.*]] = ptrtoint i32 addrspace(1)* [[V:%.*]] to i32
-; CHECK-NEXT:    switch i32 [[MAGICPTR]], label [[F:%.*]] [
-; CHECK-NEXT:    i32 17, label [[T:%.*]]
-; CHECK-NEXT:    i32 4, label [[T]]
+; CHECK-NEXT:    [[MAGICPTR:%.*]] = ptrtoint i32 addrspace(1)* [[V:%.*]] to i40
+; CHECK-NEXT:    switch i40 [[MAGICPTR]], label [[F:%.*]] [
+; CHECK-NEXT:    i40 17, label [[T:%.*]]
+; CHECK-NEXT:    i40 4, label [[T]]
 ; CHECK-NEXT:    ]
 ; CHECK:       T:
 ; CHECK-NEXT:    call void @foo1()
diff --git a/llvm/test/Transforms/SimplifyCFG/switch_msan.ll b/llvm/test/Transforms/SimplifyCFG/switch_msan.ll
new file mode 100644
index 000000000..96e798289
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/switch_msan.ll
@@ -0,0 +1,102 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -simplifycfg < %s | FileCheck %s
+
+declare i8 @next_char();
+
+define void @test_no_msan() {
+; CHECK-LABEL: @test_no_msan(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[WHILE_BODY:%.*]]
+; CHECK:       while.body:
+; CHECK-NEXT:    br label [[WHILE_BODY_I:%.*]]
+; CHECK:       while.body.i:
+; CHECK-NEXT:    [[MAYBE_UNDEF:%.*]] = phi i1 [ undef, [[WHILE_BODY]] ], [ [[NEXT_MAYBE_UNDEF:%.*]], [[WHILE_BODY_I]] ]
+; CHECK-NEXT:    [[C:%.*]] = call fastcc signext i8 @next_char()
+; CHECK-NEXT:    [[C_10:%.*]] = icmp eq i8 [[C]], 10
+; CHECK-NEXT:    [[C_13:%.*]] = icmp eq i8 [[C]], 13
+; CHECK-NEXT:    [[C_10_OR_13:%.*]] = or i1 [[C_10]], [[C_13]]
+; CHECK-NEXT:    [[NEXT_MAYBE_UNDEF]] = or i1 [[MAYBE_UNDEF]], [[C_10_OR_13]]
+; CHECK-NEXT:    [[C_NOT_10_OR_13:%.*]] = xor i1 [[C_10_OR_13]], true
+; CHECK-NEXT:    br i1 [[C_NOT_10_OR_13]], label [[WHILE_BODY_I]], label [[WHILE_BODY_I_BREAK:%.*]]
+; CHECK:       while.body.i.break:
+; CHECK-NEXT:    br i1 [[MAYBE_UNDEF]], label [[WHILE_BODY]], label [[SWITCH_EARLY_TEST:%.*]]
+; CHECK:       switch.early.test:
+; CHECK-NEXT:    switch i8 [[C]], label [[RETURN:%.*]] [
+; CHECK-NEXT:    i8 13, label [[WHILE_BODY]]
+; CHECK-NEXT:    i8 10, label [[WHILE_BODY]]
+; CHECK-NEXT:    ]
+; CHECK:       return:
+; CHECK-NEXT:    ret void
+;
+
+entry:
+  br label %while.body
+
+while.body:
+  br label %while.body.i
+
+while.body.i:
+  %maybe_undef = phi i1 [ undef, %while.body ], [ %next_maybe_undef, %while.body.i ]
+  %c = call fastcc signext i8 @next_char()
+  %c_10 = icmp eq i8 %c, 10
+  %c_13 = icmp eq i8 %c, 13
+  %c_10_or_13 = or i1 %c_10, %c_13
+  %next_maybe_undef = or i1 %maybe_undef, %c_10_or_13
+  %c_not_10_or_13 = xor i1 %c_10_or_13, true
+  br i1 %c_not_10_or_13, label %while.body.i, label %while.body.i.break
+
+while.body.i.break:
+  ; NEXT_MAYBE_UNDEF is never undef if here
+  br i1 %next_maybe_undef, label %while.body, label %return
+
+
+return:
+  ret void
+}
+
+
+define void @test_msan() sanitize_memory {
+; CHECK-LABEL: @test_msan(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[WHILE_BODY:%.*]]
+; CHECK:       while.body:
+; CHECK-NEXT:    br label [[WHILE_BODY_I:%.*]]
+; CHECK:       while.body.i:
+; CHECK-NEXT:    [[MAYBE_UNDEF:%.*]] = phi i1 [ undef, [[WHILE_BODY]] ], [ [[NEXT_MAYBE_UNDEF:%.*]], [[WHILE_BODY_I]] ]
+; CHECK-NEXT:    [[C:%.*]] = call fastcc signext i8 @next_char()
+; CHECK-NEXT:    [[C_10:%.*]] = icmp eq i8 [[C]], 10
+; CHECK-NEXT:    [[C_13:%.*]] = icmp eq i8 [[C]], 13
+; CHECK-NEXT:    [[C_10_OR_13:%.*]] = or i1 [[C_10]], [[C_13]]
+; CHECK-NEXT:    [[NEXT_MAYBE_UNDEF]] = or i1 [[MAYBE_UNDEF]], [[C_10_OR_13]]
+; CHECK-NEXT:    [[C_NOT_10_OR_13:%.*]] = xor i1 [[C_10_OR_13]], true
+; CHECK-NEXT:    br i1 [[C_NOT_10_OR_13]], label [[WHILE_BODY_I]], label [[WHILE_BODY_I_BREAK:%.*]]
+; CHECK:       while.body.i.break:
+; CHECK-NEXT:    br i1 [[NEXT_MAYBE_UNDEF]], label [[WHILE_BODY]], label [[RETURN:%.*]]
+; CHECK:       return:
+; CHECK-NEXT:    ret void
+;
+
+entry:
+  br label %while.body
+
+while.body:
+  br label %while.body.i
+
+while.body.i:
+  %maybe_undef = phi i1 [ undef, %while.body ], [ %next_maybe_undef, %while.body.i ]
+  %c = call fastcc signext i8 @next_char()
+  %c_10 = icmp eq i8 %c, 10
+  %c_13 = icmp eq i8 %c, 13
+  %c_10_or_13 = or i1 %c_10, %c_13
+  %next_maybe_undef = or i1 %maybe_undef, %c_10_or_13
+  %c_not_10_or_13 = xor i1 %c_10_or_13, true
+  br i1 %c_not_10_or_13, label %while.body.i, label %while.body.i.break
+
+while.body.i.break:
+  ; NEXT_MAYBE_UNDEF is never undef if here
+  br i1 %next_maybe_undef, label %while.body, label %return
+
+
+return:
+  ret void
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/unsigned-multiplication-will-overflow.ll b/llvm/test/Transforms/SimplifyCFG/unsigned-multiplication-will-overflow.ll
new file mode 100644
index 000000000..01a949916
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/unsigned-multiplication-will-overflow.ll
@@ -0,0 +1,36 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -simplifycfg -S | FileCheck %s --check-prefixes=ALL,DEFAULT,FALLBACK0
+; RUN: opt < %s -simplifycfg -phi-node-folding-threshold=2 -S | FileCheck %s --check-prefixes=ALL,DEFAULT,FALLBACK1
+; RUN: opt < %s -simplifycfg -phi-node-folding-threshold=3 -S | FileCheck %s --check-prefixes=ALL,COSTLY
+
+; This is checking that the multiplication does overflow, with a leftover
+; guard against division-by-zero that was needed before InstCombine
+; produced llvm.umul.with.overflow.
+
+define i1 @will_overflow(i64 %size, i64 %nmemb) {
+; ALL-LABEL: @will_overflow(
+; ALL-NEXT:  entry:
+; ALL-NEXT:    [[CMP:%.*]] = icmp eq i64 [[SIZE:%.*]], 0
+; ALL-NEXT:    [[UMUL:%.*]] = tail call { i64, i1 } @llvm.umul.with.overflow.i64(i64 [[SIZE]], i64 [[NMEMB:%.*]])
+; ALL-NEXT:    [[UMUL_OV:%.*]] = extractvalue { i64, i1 } [[UMUL]], 1
+; ALL-NEXT:    [[UMUL_NOT_OV:%.*]] = xor i1 [[UMUL_OV]], true
+; ALL-NEXT:    [[TMP0:%.*]] = select i1 [[CMP]], i1 true, i1 [[UMUL_NOT_OV]]
+; ALL-NEXT:    ret i1 [[TMP0]]
+;
+entry:
+  %cmp = icmp eq i64 %size, 0
+  br i1 %cmp, label %land.end, label %land.rhs
+
+land.rhs:                                         ; preds = %entry
+  %umul = tail call { i64, i1 } @llvm.umul.with.overflow.i64(i64 %size, i64 %nmemb)
+  %umul.ov = extractvalue { i64, i1 } %umul, 1
+  %umul.not.ov = xor i1 %umul.ov, true
+  br label %land.end
+
+land.end:                                         ; preds = %land.rhs, %entry
+  %0 = phi i1 [ true, %entry ], [ %umul.not.ov, %land.rhs ]
+  ret i1 %0
+}
+
+; Function Attrs: nounwind readnone speculatable
+declare { i64, i1 } @llvm.umul.with.overflow.i64(i64, i64) #0
diff --git a/llvm/test/Transforms/SimplifyCFG/volatile-phioper.ll b/llvm/test/Transforms/SimplifyCFG/volatile-phioper.ll
index c366d059d..bfa06269e 100644
--- a/llvm/test/Transforms/SimplifyCFG/volatile-phioper.ll
+++ b/llvm/test/Transforms/SimplifyCFG/volatile-phioper.ll
@@ -41,8 +41,8 @@ end:
 }
 declare i32 @Trace(...) #1
 
-attributes #0 = { nounwind ssp uwtable "fp-contract-model"="standard" "no-frame-pointer-elim" "no-frame-pointer-elim-non-leaf" "relocation-model"="pic" "ssp-buffers-size"="8" }
-attributes #1 = { "fp-contract-model"="standard" "no-frame-pointer-elim" "no-frame-pointer-elim-non-leaf" "relocation-model"="pic" "ssp-buffers-size"="8" }
+attributes #0 = { nounwind ssp uwtable "fp-contract-model"="standard" "frame-pointer"="non-leaf" "relocation-model"="pic" "ssp-buffers-size"="8" }
+attributes #1 = { "fp-contract-model"="standard" "frame-pointer"="non-leaf" "relocation-model"="pic" "ssp-buffers-size"="8" }
 attributes #2 = { nounwind }
 
 !0 = !{i32 1039}
diff --git a/llvm/test/Transforms/SimplifyCFG/wc-widen-block.ll b/llvm/test/Transforms/SimplifyCFG/wc-widen-block.ll
new file mode 100644
index 000000000..e889d02a2
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/wc-widen-block.ll
@@ -0,0 +1,451 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -passes=simplify-cfg -S < %s | FileCheck %s
+
+define i32 @basic(i1 %cond_0, i32* %p) {
+; CHECK-LABEL: @basic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_0:%.*]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    [[DEOPTRET:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTRET]]
+; CHECK:       guarded:
+; CHECK-NEXT:    [[V:%.*]] = load i32, i32* [[P:%.*]]
+; CHECK-NEXT:    [[COND_1:%.*]] = icmp eq i32 [[V]], 0
+; CHECK-NEXT:    br i1 [[COND_1]], label [[RETURN:%.*]], label [[DEOPT]], !prof !0
+; CHECK:       return:
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %cond_0, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:
+  %deoptret = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+  ret i32 %deoptret
+
+guarded:
+  %v = load i32, i32* %p
+  %cond_1 = icmp eq i32 %v, 0
+  br i1 %cond_1, label %return, label %deopt2, !prof !0
+
+deopt2:
+  %deoptret2 = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+  ret i32 %deoptret2
+
+return:
+  ret i32 0
+}
+
+define i32 @mergeable(i1 %cond_0, i1 %cond_1) {
+; CHECK-LABEL: @mergeable(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_0:%.*]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND_NOT:%.*]] = xor i1 [[EXIPLICIT_GUARD_COND]], true
+; CHECK-NEXT:    [[COND_1_NOT:%.*]] = xor i1 [[COND_1:%.*]], true
+; CHECK-NEXT:    [[BRMERGE:%.*]] = or i1 [[EXIPLICIT_GUARD_COND_NOT]], [[COND_1_NOT]]
+; CHECK-NEXT:    br i1 [[BRMERGE]], label [[DEOPT:%.*]], label [[RETURN:%.*]], !prof !1
+; CHECK:       deopt:
+; CHECK-NEXT:    [[DEOPTRET:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTRET]]
+; CHECK:       return:
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %cond_0, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:
+  %deoptret = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+  ret i32 %deoptret
+
+guarded:
+  br i1 %cond_1, label %return, label %deopt2, !prof !0
+
+deopt2:
+  %deoptret2 = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+  ret i32 %deoptret2
+
+return:
+  ret i32 0
+}
+
+define i32 @basic_swapped_branch(i1 %cond_0, i32* %p) {
+; CHECK-LABEL: @basic_swapped_branch(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_0:%.*]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    [[DEOPTRET:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTRET]]
+; CHECK:       guarded:
+; CHECK-NEXT:    [[V:%.*]] = load i32, i32* [[P:%.*]]
+; CHECK-NEXT:    [[COND_1:%.*]] = icmp eq i32 [[V]], 0
+; CHECK-NEXT:    br i1 [[COND_1]], label [[DEOPT]], label [[RETURN:%.*]], !prof !0
+; CHECK:       return:
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %cond_0, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:
+  %deoptret = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+  ret i32 %deoptret
+
+guarded:
+  %v = load i32, i32* %p
+  %cond_1 = icmp eq i32 %v, 0
+  br i1 %cond_1, label %deopt2, label %return, !prof !0
+
+deopt2:
+  %deoptret2 = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+  ret i32 %deoptret2
+
+return:
+  ret i32 0
+}
+
+define i32 @todo_sink_side_effect(i1 %cond_0, i1 %cond_1) {
+; CHECK-LABEL: @todo_sink_side_effect(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_0:%.*]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    [[DEOPTRET:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTRET]]
+; CHECK:       guarded:
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    br i1 [[COND_1:%.*]], label [[RETURN:%.*]], label [[DEOPT2:%.*]], !prof !0
+; CHECK:       deopt2:
+; CHECK-NEXT:    [[DEOPTRET2:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTRET2]]
+; CHECK:       return:
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %cond_0, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:
+  %deoptret = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+  ret i32 %deoptret
+
+guarded:
+  call void @unknown()
+  br i1 %cond_1, label %return, label %deopt2, !prof !0
+
+deopt2:
+  %deoptret2 = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+  ret i32 %deoptret2
+
+return:
+  ret i32 0
+}
+
+define i32 @neg_unsinkable_side_effect(i1 %cond_0) {
+; CHECK-LABEL: @neg_unsinkable_side_effect(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_0:%.*]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    [[DEOPTRET:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTRET]]
+; CHECK:       guarded:
+; CHECK-NEXT:    [[V:%.*]] = call i32 @unknown_i32()
+; CHECK-NEXT:    [[COND_1:%.*]] = icmp eq i32 [[V]], 0
+; CHECK-NEXT:    br i1 [[COND_1]], label [[RETURN:%.*]], label [[DEOPT2:%.*]], !prof !0
+; CHECK:       deopt2:
+; CHECK-NEXT:    [[DEOPTRET2:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTRET2]]
+; CHECK:       return:
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %cond_0, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:
+  %deoptret = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+  ret i32 %deoptret
+
+guarded:
+  %v = call i32 @unknown_i32()
+  %cond_1 = icmp eq i32 %v, 0
+  br i1 %cond_1, label %return, label %deopt2, !prof !0
+
+deopt2:
+  %deoptret2 = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+  ret i32 %deoptret2
+
+return:
+  ret i32 0
+}
+
+
+define i32 @neg_inf_loop(i1 %cond_0, i1 %cond_1) {
+; CHECK-LABEL: @neg_inf_loop(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_0:%.*]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    [[DEOPTRET:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTRET]]
+; CHECK:       guarded:
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    br i1 [[COND_1:%.*]], label [[RETURN:%.*]], label [[DEOPT]], !prof !0
+; CHECK:       return:
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %cond_0, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:
+  %deoptret = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+  ret i32 %deoptret
+
+guarded:
+  call void @unknown()
+  br i1 %cond_1, label %return, label %deopt, !prof !0
+
+return:
+  ret i32 0
+}
+
+
+define i32 @todo_phi(i1 %cond_0, i1 %cond_1) {
+; CHECK-LABEL: @todo_phi(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_0:%.*]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    [[PHI:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[DEOPTRET:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"(i32 [[PHI]]) ]
+; CHECK-NEXT:    ret i32 [[DEOPTRET]]
+; CHECK:       guarded:
+; CHECK-NEXT:    br i1 [[COND_1:%.*]], label [[RETURN:%.*]], label [[DEOPT2:%.*]], !prof !0
+; CHECK:       deopt2:
+; CHECK-NEXT:    [[DEOPTRET2:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTRET2]]
+; CHECK:       return:
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %cond_0, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:
+  %phi = phi i32 [0, %entry]
+  %deoptret = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"(i32 %phi) ]
+  ret i32 %deoptret
+
+guarded:
+  br i1 %cond_1, label %return, label %deopt2, !prof !0
+
+deopt2:
+  %deoptret2 = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+  ret i32 %deoptret2
+
+return:
+  ret i32 0
+}
+
+
+define i32 @neg_loop(i1 %cond_0, i1 %cond_1) {
+; CHECK-LABEL: @neg_loop(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[GUARDED:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_0:%.*]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    [[DEOPTRET:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTRET]]
+; CHECK:       guarded:
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    br i1 [[COND_1:%.*]], label [[LOOP:%.*]], label [[DEOPT2:%.*]], !prof !0
+; CHECK:       deopt2:
+; CHECK-NEXT:    [[DEOPTRET2:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTRET2]]
+;
+entry:
+  br label %guarded
+
+loop:
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %cond_0, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:
+  %deoptret = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+  ret i32 %deoptret
+
+guarded:
+  call void @unknown()
+  br i1 %cond_1, label %loop, label %deopt2, !prof !0
+
+deopt2:
+  %deoptret2 = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+  ret i32 %deoptret2
+}
+
+; This one is subtle - We can't widen only one branch use of the
+; widenable condition as two branches are correlated.  We'd have to
+; widen them *both*.
+define i32 @neg_correlated(i1 %cond_0, i1 %cond_1, i32* %p) {
+; CHECK-LABEL: @neg_correlated(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_0:%.*]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    [[DEOPTRET:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTRET]]
+; CHECK:       guarded:
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND2:%.*]] = and i1 [[COND_1:%.*]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND2]], label [[GUARDED2:%.*]], label [[DEOPT2:%.*]], !prof !0
+; CHECK:       deopt2:
+; CHECK-NEXT:    [[DEOPTRET2:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTRET2]]
+; CHECK:       guarded2:
+; CHECK-NEXT:    [[V:%.*]] = load i32, i32* [[P:%.*]]
+; CHECK-NEXT:    [[COND_2:%.*]] = icmp eq i32 [[V]], 0
+; CHECK-NEXT:    br i1 [[COND_2]], label [[RETURN:%.*]], label [[DEOPT3:%.*]], !prof !0
+; CHECK:       deopt3:
+; CHECK-NEXT:    [[DEOPTRET3:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTRET3]]
+; CHECK:       return:
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %cond_0, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:
+  %deoptret = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+  ret i32 %deoptret
+
+guarded:
+  %exiplicit_guard_cond2 = and i1 %cond_1, %widenable_cond
+  br i1 %exiplicit_guard_cond2, label %guarded2, label %deopt2, !prof !0
+
+deopt2:
+  %deoptret2 = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+  ret i32 %deoptret2
+
+guarded2:
+  %v = load i32, i32* %p
+  %cond_2 = icmp eq i32 %v, 0
+  br i1 %cond_2, label %return, label %deopt3, !prof !0
+
+deopt3:
+  %deoptret3 = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+  ret i32 %deoptret3
+
+return:
+  ret i32 0
+}
+
+define i32 @trivial_wb(i1 %cond_0, i32* %p) {
+; CHECK-LABEL: @trivial_wb(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    br i1 [[WIDENABLE_COND]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    [[DEOPTRET:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTRET]]
+; CHECK:       guarded:
+; CHECK-NEXT:    [[V:%.*]] = load i32, i32* [[P:%.*]]
+; CHECK-NEXT:    [[COND_1:%.*]] = icmp eq i32 [[V]], 0
+; CHECK-NEXT:    br i1 [[COND_1]], label [[RETURN:%.*]], label [[DEOPT]], !prof !0
+; CHECK:       return:
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  br i1 %widenable_cond, label %guarded, label %deopt, !prof !0
+
+deopt:
+  %deoptret = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+  ret i32 %deoptret
+
+guarded:
+  %v = load i32, i32* %p
+  %cond_1 = icmp eq i32 %v, 0
+  br i1 %cond_1, label %return, label %deopt2, !prof !0
+
+deopt2:
+  %deoptret2 = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+  ret i32 %deoptret2
+
+return:
+  ret i32 0
+}
+
+
+define i32 @swapped_wb(i1 %cond_0, i32* %p) {
+; CHECK-LABEL: @swapped_wb(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[WIDENABLE_COND]], [[COND_0:%.*]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    [[DEOPTRET:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTRET]]
+; CHECK:       guarded:
+; CHECK-NEXT:    [[V:%.*]] = load i32, i32* [[P:%.*]]
+; CHECK-NEXT:    [[COND_1:%.*]] = icmp eq i32 [[V]], 0
+; CHECK-NEXT:    br i1 [[COND_1]], label [[RETURN:%.*]], label [[DEOPT]], !prof !0
+; CHECK:       return:
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %widenable_cond, %cond_0
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:
+  %deoptret = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+  ret i32 %deoptret
+
+guarded:
+  %v = load i32, i32* %p
+  %cond_1 = icmp eq i32 %v, 0
+  br i1 %cond_1, label %return, label %deopt2, !prof !0
+
+deopt2:
+  %deoptret2 = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
+  ret i32 %deoptret2
+
+return:
+  ret i32 0
+}
+
+
+
+
+declare void @unknown()
+declare i32 @unknown_i32()
+
+declare i1 @llvm.experimental.widenable.condition()
+declare i32 @llvm.experimental.deoptimize.i32(...)
+
+!0 = !{!"branch_weights", i32 1048576, i32 1}
+!1 = !{i32 1, i32 -2147483648}
diff --git a/llvm/test/Transforms/StraightLineStrengthReduce/AMDGPU/pr23975.ll b/llvm/test/Transforms/StraightLineStrengthReduce/AMDGPU/pr23975.ll
index f587a93bf..afe37a991 100644
--- a/llvm/test/Transforms/StraightLineStrengthReduce/AMDGPU/pr23975.ll
+++ b/llvm/test/Transforms/StraightLineStrengthReduce/AMDGPU/pr23975.ll
@@ -17,4 +17,4 @@ entry:
   ret void
 }
 
-attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "target-cpu"="tahiti" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "target-cpu"="tahiti" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/Transforms/ThinLTOBitcodeWriter/associated.ll b/llvm/test/Transforms/ThinLTOBitcodeWriter/associated.ll
new file mode 100644
index 000000000..600791d76
--- /dev/null
+++ b/llvm/test/Transforms/ThinLTOBitcodeWriter/associated.ll
@@ -0,0 +1,14 @@
+; RUN: opt -thinlto-bc -thinlto-split-lto-unit -o %t %s
+; RUN: llvm-modextract -b -n 0 -o - %t | llvm-dis | FileCheck --check-prefix=M0 %s
+; RUN: llvm-modextract -b -n 1 -o - %t | llvm-dis | FileCheck --check-prefix=M1 %s
+
+; M0: @g = external constant
+; M0-NOT: @assoc
+; M1: @g = constant i8 1
+; M1: @assoc = private constant i8 2
+
+@g = constant i8 1, !type !0
+@assoc = private constant i8 2, !associated !1
+
+!0 = !{i32 0, !"typeid"}
+!1 = !{i8* @g}
diff --git a/llvm/test/Transforms/ThinLTOBitcodeWriter/cfi-functions-canonical-jump-tables.ll b/llvm/test/Transforms/ThinLTOBitcodeWriter/cfi-functions-canonical-jump-tables.ll
new file mode 100644
index 000000000..ad775ee12
--- /dev/null
+++ b/llvm/test/Transforms/ThinLTOBitcodeWriter/cfi-functions-canonical-jump-tables.ll
@@ -0,0 +1,23 @@
+; RUN: opt -thinlto-bc -thinlto-split-lto-unit -o %t %s
+; RUN: llvm-modextract -b -n 1 -o - %t | llvm-dis | FileCheck %s
+
+; Check that cfi.functions metadata has the expected contents.
+
+; CHECK: !"f1", i8 1
+; CHECK: !"f2", i8 1
+; CHECK: !"f3", i8 0
+
+declare !type !1 void @f1()
+
+define void @f2() !type !1 {
+  ret void
+}
+
+define void @f3() "cfi-canonical-jump-table" !type !1 {
+  ret void
+}
+
+!llvm.module.flags = !{!0}
+
+!0 = !{i32 4, !"CFI Canonical Jump Tables", i32 0}
+!1 = !{i32 0, !"typeid1"}
diff --git a/llvm/test/Transforms/TypePromotion/ARM/calls.ll b/llvm/test/Transforms/TypePromotion/ARM/calls.ll
new file mode 100644
index 000000000..cd273c061
--- /dev/null
+++ b/llvm/test/Transforms/TypePromotion/ARM/calls.ll
@@ -0,0 +1,342 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -mtriple=arm -type-promotion -verify -disable-type-promotion=false -S %s -o - | FileCheck %s
+
+define i8 @call_with_imms(i8* %arg) {
+; CHECK-LABEL: @call_with_imms(
+; CHECK-NEXT:    [[CALL:%.*]] = tail call arm_aapcs_vfpcc zeroext i8 @dummy2(i8* nonnull [[ARG:%.*]], i8 zeroext 0, i8 zeroext 0)
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[CALL]], 0
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[CMP]], i8 [[CALL]], i8 1
+; CHECK-NEXT:    ret i8 [[RES]]
+;
+  %call = tail call arm_aapcs_vfpcc zeroext i8 @dummy2(i8* nonnull %arg, i8 zeroext 0, i8 zeroext 0)
+  %cmp = icmp eq i8 %call, 0
+  %res = select i1 %cmp, i8 %call, i8 1
+  ret i8 %res
+}
+
+define i16 @test_call(i8 zeroext %arg) {
+; CHECK-LABEL: @test_call(
+; CHECK-NEXT:    [[CALL:%.*]] = call i8 @dummy_i8(i8 [[ARG:%.*]])
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i8 [[CALL]], -128
+; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i16
+; CHECK-NEXT:    ret i16 [[CONV]]
+;
+  %call = call i8 @dummy_i8(i8 %arg)
+  %cmp = icmp ult i8 %call, 128
+  %conv = zext i1 %cmp to i16
+  ret i16 %conv
+}
+
+define i16 @promote_i8_sink_i16_1(i8 zeroext %arg0, i16 zeroext %arg1, i16 zeroext %arg2) {
+; CHECK-LABEL: @promote_i8_sink_i16_1(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[ARG2:%.*]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i16 [[ARG1:%.*]] to i32
+; CHECK-NEXT:    [[CALL:%.*]] = tail call zeroext i8 @dummy_i8(i8 [[ARG0:%.*]])
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i8 [[CALL]] to i32
+; CHECK-NEXT:    [[ADD:%.*]] = add nuw i32 [[TMP3]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[ADD]], [[TMP2]]
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 [[TMP2]], i32 [[TMP1]]
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc i32 [[SEL]] to i16
+; CHECK-NEXT:    [[RES:%.*]] = tail call zeroext i16 @dummy3(i16 [[TMP4]])
+; CHECK-NEXT:    [[TMP5:%.*]] = zext i16 [[RES]] to i32
+; CHECK-NEXT:    [[TMP6:%.*]] = trunc i32 [[TMP5]] to i16
+; CHECK-NEXT:    ret i16 [[TMP6]]
+;
+  %call = tail call zeroext i8 @dummy_i8(i8 %arg0)
+  %add = add nuw i8 %call, 1
+  %conv = zext i8 %add to i16
+  %cmp = icmp ne i16 %conv, %arg1
+  %sel = select i1 %cmp, i16 %arg1, i16 %arg2
+  %res = tail call zeroext i16 @dummy3(i16 %sel)
+  ret i16 %res
+}
+
+define i16 @promote_i8_sink_i16_2(i8 zeroext %arg0, i8 zeroext %arg1, i16 zeroext %arg2) {
+; CHECK-LABEL: @promote_i8_sink_i16_2(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[ARG1:%.*]] to i32
+; CHECK-NEXT:    [[CALL:%.*]] = tail call zeroext i8 @dummy_i8(i8 [[ARG0:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i8 [[CALL]] to i32
+; CHECK-NEXT:    [[ADD:%.*]] = add nuw i32 [[TMP2]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[ADD]], [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = trunc i32 [[TMP1]] to i8
+; CHECK-NEXT:    [[CONV:%.*]] = zext i8 [[TMP3]] to i16
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i16 [[CONV]], i16 [[ARG2:%.*]]
+; CHECK-NEXT:    [[RES:%.*]] = tail call zeroext i16 @dummy3(i16 [[SEL]])
+; CHECK-NEXT:    ret i16 [[RES]]
+;
+  %call = tail call zeroext i8 @dummy_i8(i8 %arg0)
+  %add = add nuw i8 %call, 1
+  %cmp = icmp ne i8 %add, %arg1
+  %conv = zext i8 %arg1 to i16
+  %sel = select i1 %cmp, i16 %conv, i16 %arg2
+  %res = tail call zeroext i16 @dummy3(i16 %sel)
+  ret i16 %res
+}
+
+@uc = global i8 42, align 1
+@LL = global i64 0, align 8
+
+define void @zext_i64() {
+; CHECK-LABEL: @zext_i64(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, i8* @uc, align 1
+; CHECK-NEXT:    [[CONV:%.*]] = zext i8 [[TMP0]] to i64
+; CHECK-NEXT:    store i64 [[CONV]], i64* @LL, align 8
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[TMP0]], 42
+; CHECK-NEXT:    [[CONV1:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 bitcast (i32 (...)* @assert to i32 (i32)*)(i32 [[CONV1]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %0 = load i8, i8* @uc, align 1
+  %conv = zext i8 %0 to i64
+  store i64 %conv, i64* @LL, align 8
+  %cmp = icmp eq i8 %0, 42
+  %conv1 = zext i1 %cmp to i32
+  %call = tail call i32 bitcast (i32 (...)* @assert to i32 (i32)*)(i32 %conv1)
+  ret void
+}
+
+@a = global i16* null, align 4
+@b = global i32 0, align 4
+
+define i32 @constexpr() {
+; CHECK-LABEL: @constexpr(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    store i32 ptrtoint (i32* @b to i32), i32* @b, align 4
+; CHECK-NEXT:    [[TMP0:%.*]] = load i16*, i16** @a, align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = load i16, i16* [[TMP0]], align 2
+; CHECK-NEXT:    [[OR:%.*]] = or i16 [[TMP1]], ptrtoint (i32* @b to i16)
+; CHECK-NEXT:    store i16 [[OR]], i16* [[TMP0]], align 2
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i16 [[OR]], 4
+; CHECK-NEXT:    [[CONV3:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 bitcast (i32 (...)* @e to i32 (i32)*)(i32 [[CONV3]])
+; CHECK-NEXT:    ret i32 undef
+;
+entry:
+  store i32 ptrtoint (i32* @b to i32), i32* @b, align 4
+  %0 = load i16*, i16** @a, align 4
+  %1 = load i16, i16* %0, align 2
+  %or = or i16 %1, ptrtoint (i32* @b to i16)
+  store i16 %or, i16* %0, align 2
+  %cmp = icmp ne i16 %or, 4
+  %conv3 = zext i1 %cmp to i32
+  %call = tail call i32 bitcast (i32 (...)* @e to i32 (i32)*)(i32 %conv3) #2
+  ret i32 undef
+}
+
+define fastcc i32 @call_zext_i8_i32(i32 %p_45, i8 zeroext %p_46) {
+; CHECK-LABEL: @call_zext_i8_i32(
+; CHECK-NEXT:  for.cond8.preheader:
+; CHECK-NEXT:    [[CALL217:%.*]] = call fastcc zeroext i8 @safe_mul_func_uint8_t_u_u(i8 zeroext undef)
+; CHECK-NEXT:    [[TOBOOL219:%.*]] = icmp eq i8 [[CALL217]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL219]], label [[FOR_END411:%.*]], label [[FOR_COND273_PREHEADER:%.*]]
+; CHECK:       for.cond273.preheader:
+; CHECK-NEXT:    [[CALL217_LCSSA:%.*]] = phi i8 [ [[CALL217]], [[FOR_COND8_PREHEADER:%.*]] ]
+; CHECK-NEXT:    [[CONV218_LE:%.*]] = zext i8 [[CALL217_LCSSA]] to i32
+; CHECK-NEXT:    [[CALL346:%.*]] = call fastcc zeroext i8 @safe_lshift_func(i8 zeroext [[CALL217_LCSSA]], i32 [[CONV218_LE]])
+; CHECK-NEXT:    unreachable
+; CHECK:       for.end411:
+; CHECK-NEXT:    [[CALL452:%.*]] = call fastcc i64 @safe_sub_func_int64_t_s_s(i64 undef, i64 4)
+; CHECK-NEXT:    unreachable
+;
+for.cond8.preheader:
+  %call217 = call fastcc zeroext i8 @safe_mul_func_uint8_t_u_u(i8 zeroext undef)
+  %tobool219 = icmp eq i8 %call217, 0
+  br i1 %tobool219, label %for.end411, label %for.cond273.preheader
+
+for.cond273.preheader:                            ; preds = %for.cond8.preheader
+  %call217.lcssa = phi i8 [ %call217, %for.cond8.preheader ]
+  %conv218.le = zext i8 %call217.lcssa to i32
+  %call346 = call fastcc zeroext i8 @safe_lshift_func(i8 zeroext %call217.lcssa, i32 %conv218.le)
+  unreachable
+
+for.end411:                                       ; preds = %for.cond8.preheader
+  %call452 = call fastcc i64 @safe_sub_func_int64_t_s_s(i64 undef, i64 4)
+  unreachable
+}
+
+%struct.anon = type { i32 }
+
+@g_57 = hidden local_unnamed_addr global %struct.anon zeroinitializer, align 4
+@g_893 = hidden local_unnamed_addr global %struct.anon zeroinitializer, align 4
+@g_82 = hidden local_unnamed_addr global i32 0, align 4
+
+define hidden i32 @call_return_pointer(i8 zeroext %p_13) local_unnamed_addr #0 {
+; CHECK-LABEL: @call_return_pointer(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = zext i8 [[P_13:%.*]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[TMP0]] to i8
+; CHECK-NEXT:    [[CONV1:%.*]] = zext i8 [[TMP1]] to i16
+; CHECK-NEXT:    [[CALL:%.*]] = tail call i16** @func_62(i8 zeroext undef, i32 undef, i16 signext [[CONV1]], i32* undef)
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* getelementptr inbounds (%struct.anon, %struct.anon* @g_893, i32 0, i32 0), align 4
+; CHECK-NEXT:    [[CONV2:%.*]] = trunc i32 [[TMP2]] to i16
+; CHECK-NEXT:    br label [[FOR_COND:%.*]]
+; CHECK:       for.cond:
+; CHECK-NEXT:    [[P_13_ADDR_0:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[P_13_ADDR_0_BE:%.*]], [[FOR_COND_BACKEDGE:%.*]] ]
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[P_13_ADDR_0]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL]], label [[FOR_COND_BACKEDGE]], label [[IF_THEN:%.*]]
+; CHECK:       for.cond.backedge:
+; CHECK-NEXT:    [[P_13_ADDR_0_BE]] = phi i32 [ [[TMP3:%.*]], [[IF_THEN]] ], [ 0, [[FOR_COND]] ]
+; CHECK-NEXT:    br label [[FOR_COND]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[CALL3:%.*]] = tail call fastcc signext i16 @safe_sub_func_int16_t_s_s(i16 signext [[CONV2]])
+; CHECK-NEXT:    [[CONV4:%.*]] = trunc i16 [[CALL3]] to i8
+; CHECK-NEXT:    [[TMP3]] = zext i8 [[CONV4]] to i32
+; CHECK-NEXT:    br label [[FOR_COND_BACKEDGE]]
+;
+entry:
+  %conv1 = zext i8 %p_13 to i16
+  %call = tail call i16** @func_62(i8 zeroext undef, i32 undef, i16 signext %conv1, i32* undef)
+  %0 = load i32, i32* getelementptr inbounds (%struct.anon, %struct.anon* @g_893, i32 0, i32 0), align 4
+  %conv2 = trunc i32 %0 to i16
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.cond.backedge, %entry
+  %p_13.addr.0 = phi i8 [ %p_13, %entry ], [ %p_13.addr.0.be, %for.cond.backedge ]
+  %tobool = icmp eq i8 %p_13.addr.0, 0
+  br i1 %tobool, label %for.cond.backedge, label %if.then
+
+for.cond.backedge:                                ; preds = %for.cond, %if.then
+  %p_13.addr.0.be = phi i8 [ %conv4, %if.then ], [ 0, %for.cond ]
+  br label %for.cond
+
+if.then:                                          ; preds = %for.cond
+  %call3 = tail call fastcc signext i16 @safe_sub_func_int16_t_s_s(i16 signext %conv2)
+  %conv4 = trunc i16 %call3 to i8
+  br label %for.cond.backedge
+}
+
+define i32 @check_zext_phi_call_arg() {
+; CHECK-LABEL: @check_zext_phi_call_arg(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_COND:%.*]]
+; CHECK:       for.cond:
+; CHECK-NEXT:    [[D_SROA_0_0:%.*]] = phi i32 [ 30, [[ENTRY:%.*]] ], [ [[D_SROA_0_0_BE:%.*]], [[FOR_COND_BACKEDGE:%.*]] ]
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[D_SROA_0_0]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL]], label [[FOR_COND_BACKEDGE]], label [[IF_THEN:%.*]]
+; CHECK:       for.cond.backedge:
+; CHECK-NEXT:    [[D_SROA_0_0_BE]] = phi i32 [ [[TMP1:%.*]], [[IF_THEN]] ], [ 0, [[FOR_COND]] ]
+; CHECK-NEXT:    br label [[FOR_COND]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[TMP0:%.*]] = trunc i32 [[D_SROA_0_0]] to i16
+; CHECK-NEXT:    [[CALL:%.*]] = tail call zeroext i16 bitcast (i16 (...)* @f to i16 (i32)*)(i32 [[D_SROA_0_0]])
+; CHECK-NEXT:    [[TMP1]] = zext i16 [[CALL]] to i32
+; CHECK-NEXT:    br label [[FOR_COND_BACKEDGE]]
+;
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.cond.backedge, %entry
+  %d.sroa.0.0 = phi i16 [ 30, %entry ], [ %d.sroa.0.0.be, %for.cond.backedge ]
+  %tobool = icmp eq i16 %d.sroa.0.0, 0
+  br i1 %tobool, label %for.cond.backedge, label %if.then
+
+for.cond.backedge:                                ; preds = %for.cond, %if.then
+  %d.sroa.0.0.be = phi i16 [ %call, %if.then ], [ 0, %for.cond ]
+  br label %for.cond
+
+if.then:                                          ; preds = %for.cond
+  %d.sroa.0.0.insert.ext = zext i16 %d.sroa.0.0 to i32
+  %call = tail call zeroext i16 bitcast (i16 (...)* @f to i16 (i32)*)(i32 %d.sroa.0.0.insert.ext) #2
+  br label %for.cond.backedge
+}
+
+%struct.atomic_flag = type { i8 }
+
+define zeroext i1 @atomic_flag_test_and_set(%struct.atomic_flag* %object) {
+; CHECK-LABEL: @atomic_flag_test_and_set(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[_VALUE:%.*]] = getelementptr inbounds [[STRUCT_ATOMIC_FLAG:%.*]], %struct.atomic_flag* [[OBJECT:%.*]], i32 0, i32 0
+; CHECK-NEXT:    [[CALL:%.*]] = tail call arm_aapcscc zeroext i8 @__atomic_exchange_1(i8* [[_VALUE]], i8 zeroext 1, i32 5)
+; CHECK-NEXT:    [[TMP0:%.*]] = zext i8 [[CALL]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[TMP0]], 1
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne i32 [[TMP1]], 0
+; CHECK-NEXT:    ret i1 [[TOBOOL]]
+;
+entry:
+  %_Value = getelementptr inbounds %struct.atomic_flag, %struct.atomic_flag* %object, i32 0, i32 0
+  %call = tail call arm_aapcscc zeroext i8 @__atomic_exchange_1(i8* %_Value, i8 zeroext 1, i32 5) #1
+  %0 = and i8 %call, 1
+  %tobool = icmp ne i8 %0, 0
+  ret i1 %tobool
+}
+
+define i1 @i1_zeroext_call(i16* %ts, i32 %a, i16* %b, i8* %c) {
+; CHECK-LABEL: @i1_zeroext_call(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i16, i16* [[TS:%.*]], align 2
+; CHECK-NEXT:    [[CONV_I860:%.*]] = trunc i32 [[A:%.*]] to i16
+; CHECK-NEXT:    store i16 [[CONV_I860]], i16* [[B:%.*]], align 2
+; CHECK-NEXT:    [[CALL_I848:%.*]] = call zeroext i1 @i1_zeroext(i8* [[C:%.*]], i32 64, i16 zeroext [[CONV_I860]])
+; CHECK-NEXT:    br i1 [[CALL_I848]], label [[IF_THEN223:%.*]], label [[IF_ELSE227:%.*]]
+; CHECK:       if.then223:
+; CHECK-NEXT:    [[CMP235:%.*]] = icmp eq i16 [[TMP0]], [[CONV_I860]]
+; CHECK-NEXT:    br label [[EXIT:%.*]]
+; CHECK:       if.else227:
+; CHECK-NEXT:    [[CMP236:%.*]] = icmp ult i16 [[TMP0]], [[CONV_I860]]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RETVAL:%.*]] = phi i1 [ [[CMP235]], [[IF_THEN223]] ], [ [[CMP236]], [[IF_ELSE227]] ]
+; CHECK-NEXT:    ret i1 [[RETVAL]]
+;
+entry:
+  %0 = load i16, i16* %ts, align 2
+  %conv.i860 = trunc i32 %a to i16
+  store i16 %conv.i860, i16* %b, align 2
+  %call.i848 = call zeroext i1 @i1_zeroext(i8* %c, i32 64, i16 zeroext %conv.i860)
+  br i1 %call.i848, label %if.then223, label %if.else227
+
+if.then223:
+  %cmp235 = icmp eq i16 %0, %conv.i860
+  br label %exit
+
+if.else227:
+  %cmp236 = icmp ult i16 %0, %conv.i860
+  br label %exit
+
+exit:
+  %retval = phi i1 [ %cmp235, %if.then223 ], [ %cmp236, %if.else227 ]
+  ret i1 %retval
+}
+
+define i16 @promote_arg_pass_to_call(i16 zeroext %arg1, i16 zeroext %arg2) {
+; CHECK-LABEL: @promote_arg_pass_to_call(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[ARG1:%.*]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i16 [[ARG2:%.*]] to i32
+; CHECK-NEXT:    [[CONV:%.*]] = add nuw i32 [[TMP1]], 15
+; CHECK-NEXT:    [[MUL:%.*]] = mul nuw nsw i32 [[CONV]], 3
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[MUL]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[TMP1]], 255
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc i32 [[TMP3]] to i8
+; CHECK-NEXT:    [[TMP5:%.*]] = trunc i32 [[TMP1]] to i16
+; CHECK-NEXT:    [[RES:%.*]] = call zeroext i16 @dummy4(i1 [[CMP]], i8 [[TMP4]], i16 [[TMP5]])
+; CHECK-NEXT:    [[TMP6:%.*]] = zext i16 [[RES]] to i32
+; CHECK-NEXT:    [[TMP7:%.*]] = trunc i32 [[TMP6]] to i16
+; CHECK-NEXT:    ret i16 [[TMP7]]
+;
+  %conv = add nuw i16 %arg1, 15
+  %mul = mul nuw nsw i16 %conv, 3
+  %cmp = icmp ult i16 %mul, %arg2
+  %trunc = trunc i16 %arg1 to i8
+  %res = call zeroext i16 @dummy4(i1 %cmp, i8 %trunc, i16 %arg1)
+  ret i16 %res
+}
+
+
+declare i32 @assert(...)
+declare i8 @dummy_i8(i8)
+declare i8 @dummy2(i8*, i8, i8)
+declare i16 @dummy3(i16)
+declare i16 @dummy4(i1, i8, i16)
+
+declare dso_local i32 @e(...) local_unnamed_addr #1
+declare dso_local zeroext i16 @f(...) local_unnamed_addr #1
+declare dso_local arm_aapcscc i8 @__atomic_exchange_1(i8*, i8, i32) local_unnamed_addr
+
+declare noalias i16** @func_62(i8 zeroext %p_63, i32 %p_64, i16 signext %p_65, i32* nocapture readnone %p_66)
+declare fastcc signext i16 @safe_sub_func_int16_t_s_s(i16 signext %si2)
+declare dso_local fastcc i64 @safe_sub_func_int64_t_s_s(i64, i64)
+declare dso_local fastcc zeroext i8 @safe_lshift_func(i8 zeroext, i32)
+declare dso_local fastcc zeroext i8 @safe_mul_func_uint8_t_u_u(i8 returned zeroext)
+declare i1 @i1_zeroext(i8*, i32, i16 zeroext)
diff --git a/llvm/test/Transforms/TypePromotion/ARM/casts.ll b/llvm/test/Transforms/TypePromotion/ARM/casts.ll
new file mode 100644
index 000000000..70fa61711
--- /dev/null
+++ b/llvm/test/Transforms/TypePromotion/ARM/casts.ll
@@ -0,0 +1,1072 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -mtriple=arm -type-promotion -verify -disable-type-promotion=false -S %s -o - | FileCheck %s
+
+define i16 @dsp_trunc(i32 %arg0, i32 %arg1, i16* %gep0, i16* %gep1) {
+; CHECK-LABEL: @dsp_trunc(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ADD0:%.*]] = add i32 [[ARG0:%.*]], [[ARG1:%.*]]
+; CHECK-NEXT:    [[CONV0:%.*]] = trunc i32 [[ADD0]] to i16
+; CHECK-NEXT:    [[SUB0:%.*]] = sub i16 0, [[CONV0]]
+; CHECK-NEXT:    [[LOAD0:%.*]] = load i16, i16* [[GEP0:%.*]], align 2
+; CHECK-NEXT:    [[LOAD1:%.*]] = load i16, i16* [[GEP1:%.*]], align 2
+; CHECK-NEXT:    [[SUB1:%.*]] = sub i16 [[LOAD0]], [[SUB0]]
+; CHECK-NEXT:    [[ADD1:%.*]] = add i16 [[LOAD1]], [[SUB0]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i16 [[SUB1]], [[ADD1]]
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[CMP]], i16 [[ADD1]], i16 [[SUB1]]
+; CHECK-NEXT:    ret i16 [[RES]]
+;
+entry:
+  %add0 = add i32 %arg0, %arg1
+  %conv0 = trunc i32 %add0 to i16
+  %sub0 = sub i16 0, %conv0
+  %load0 = load i16, i16* %gep0, align 2
+  %load1 = load i16, i16* %gep1, align 2
+  %sub1 = sub i16 %load0, %sub0
+  %add1 = add i16 %load1, %sub0
+  %cmp = icmp ult i16 %sub1, %add1
+  %res = select i1 %cmp, i16 %add1, i16 %sub1
+  ret i16 %res
+}
+
+define i8 @trunc_i16_i8(i16* %ptr, i16 zeroext %arg0, i8 zeroext %arg1) {
+; CHECK-LABEL: @trunc_i16_i8(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = zext i8 [[ARG1:%.*]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = load i16, i16* [[PTR:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = add i16 [[TMP1]], [[ARG0:%.*]]
+; CHECK-NEXT:    [[TMP3:%.*]] = trunc i16 [[TMP2]] to i8
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i8 [[TMP3]] to i32
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ugt i32 [[TMP4]], [[TMP0]]
+; CHECK-NEXT:    [[TMP6:%.*]] = select i1 [[TMP5]], i32 [[TMP4]], i32 [[TMP0]]
+; CHECK-NEXT:    [[TMP7:%.*]] = trunc i32 [[TMP6]] to i8
+; CHECK-NEXT:    ret i8 [[TMP7]]
+;
+entry:
+  %0 = load i16, i16* %ptr
+  %1 = add i16 %0, %arg0
+  %2 = trunc i16 %1 to i8
+  %3 = icmp ugt i8 %2, %arg1
+  %4 = select i1 %3, i8 %2, i8 %arg1
+  ret i8 %4
+}
+
+; The pass perform the transform, but a uxtb will still be inserted to handle
+; the zext to the icmp.
+define i8 @icmp_i32_zext(i8* %ptr) {
+; CHECK-LABEL: @icmp_i32_zext(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i8, i8* [[PTR:%.*]], i32 0
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, i8* [[GEP]], align 1
+; CHECK-NEXT:    [[TMP1:%.*]] = sub nuw nsw i8 [[TMP0]], 1
+; CHECK-NEXT:    [[CONV44:%.*]] = zext i8 [[TMP0]] to i32
+; CHECK-NEXT:    br label [[PREHEADER:%.*]]
+; CHECK:       preheader:
+; CHECK-NEXT:    br label [[BODY:%.*]]
+; CHECK:       body:
+; CHECK-NEXT:    [[TMP2:%.*]] = phi i8 [ [[TMP1]], [[PREHEADER]] ], [ [[TMP3:%.*]], [[IF_END:%.*]] ]
+; CHECK-NEXT:    [[SI_0274:%.*]] = phi i32 [ [[CONV44]], [[PREHEADER]] ], [ [[INC:%.*]], [[IF_END]] ]
+; CHECK-NEXT:    [[CONV51266:%.*]] = zext i8 [[TMP2]] to i32
+; CHECK-NEXT:    [[CMP52267:%.*]] = icmp eq i32 [[SI_0274]], [[CONV51266]]
+; CHECK-NEXT:    br i1 [[CMP52267]], label [[IF_END]], label [[EXIT:%.*]]
+; CHECK:       if.end:
+; CHECK-NEXT:    [[INC]] = add i32 [[SI_0274]], 1
+; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr inbounds i8, i8* [[PTR]], i32 [[INC]]
+; CHECK-NEXT:    [[TMP3]] = load i8, i8* [[GEP1]], align 1
+; CHECK-NEXT:    br label [[BODY]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i8 [[TMP2]]
+;
+entry:
+  %gep = getelementptr inbounds i8, i8* %ptr, i32 0
+  %0 = load i8, i8* %gep, align 1
+  %1 = sub nuw nsw i8 %0, 1
+  %conv44 = zext i8 %0 to i32
+  br label %preheader
+
+preheader:
+  br label %body
+
+body:
+  %2 = phi i8 [ %1, %preheader ], [ %3, %if.end ]
+  %si.0274 = phi i32 [ %conv44, %preheader ], [ %inc, %if.end ]
+  %conv51266 = zext i8 %2 to i32
+  %cmp52267 = icmp eq i32 %si.0274, %conv51266
+  br i1 %cmp52267, label %if.end, label %exit
+
+if.end:
+  %inc = add i32 %si.0274, 1
+  %gep1 = getelementptr inbounds i8, i8* %ptr, i32 %inc
+  %3 = load i8, i8* %gep1, align 1
+  br label %body
+
+exit:
+  ret i8 %2
+}
+
+; Won't don't handle sext
+define i32 @icmp_sext_zext_store_i8_i16() {
+; CHECK-LABEL: @icmp_sext_zext_store_i8_i16(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, i8* getelementptr inbounds ([16 x i8], [16 x i8]* @d_uch, i32 0, i32 2), align 1
+; CHECK-NEXT:    [[CONV:%.*]] = zext i8 [[TMP0]] to i16
+; CHECK-NEXT:    store i16 [[CONV]], i16* @sh1, align 2
+; CHECK-NEXT:    [[CONV1:%.*]] = zext i8 [[TMP0]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @d_sh, i32 0, i32 2), align 2
+; CHECK-NEXT:    [[CONV2:%.*]] = sext i16 [[TMP1]] to i32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[CONV1]], [[CONV2]]
+; CHECK-NEXT:    [[CONV3:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    ret i32 [[CONV3]]
+;
+entry:
+  %0 = load i8, i8* getelementptr inbounds ([16 x i8], [16 x i8]* @d_uch, i32 0, i32 2), align 1
+  %conv = zext i8 %0 to i16
+  store i16 %conv, i16* @sh1, align 2
+  %conv1 = zext i8 %0 to i32
+  %1 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @d_sh, i32 0, i32 2), align 2
+  %conv2 = sext i16 %1 to i32
+  %cmp = icmp eq i32 %conv1, %conv2
+  %conv3 = zext i1 %cmp to i32
+  ret i32 %conv3
+}
+
+define i1 @or_icmp_ugt(i32 %arg, i8* %ptr) {
+; CHECK-LABEL: @or_icmp_ugt(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, i8* [[PTR:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[TMP0]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i8 [[TMP0]] to i32
+; CHECK-NEXT:    [[MUL:%.*]] = shl nuw nsw i32 [[TMP2]], 1
+; CHECK-NEXT:    [[ADD0:%.*]] = add nuw nsw i32 [[MUL]], 6
+; CHECK-NEXT:    [[CMP0:%.*]] = icmp ne i32 [[ARG:%.*]], [[ADD0]]
+; CHECK-NEXT:    [[TMP3:%.*]] = sub i32 [[TMP1]], 1
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ugt i32 [[TMP3]], 3
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[CMP0]], [[CMP1]]
+; CHECK-NEXT:    ret i1 [[OR]]
+;
+entry:
+  %0 = load i8, i8* %ptr
+  %1 = zext i8 %0 to i32
+  %mul = shl nuw nsw i32 %1, 1
+  %add0 = add nuw nsw i32 %mul, 6
+  %cmp0 = icmp ne i32 %arg, %add0
+  %add1 = add i8 %0, -1
+  %cmp1 = icmp ugt i8 %add1, 3
+  %or = or i1 %cmp0, %cmp1
+  ret i1 %or
+}
+
+; We currently only handle truncs as sinks, so a uxt will still be needed for
+; the icmp ugt instruction.
+define void @urem_trunc_icmps(i16** %in, i32* %g, i32* %k) {
+; CHECK-LABEL: @urem_trunc_icmps(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[PTR:%.*]] = load i16*, i16** [[IN:%.*]], align 4
+; CHECK-NEXT:    [[LD:%.*]] = load i16, i16* [[PTR]], align 2
+; CHECK-NEXT:    [[CMP_I:%.*]] = icmp eq i16 [[LD]], 0
+; CHECK-NEXT:    br i1 [[CMP_I]], label [[EXIT:%.*]], label [[COND_FALSE_I:%.*]]
+; CHECK:       cond.false.i:
+; CHECK-NEXT:    [[REM:%.*]] = urem i16 5, [[LD]]
+; CHECK-NEXT:    [[EXTRACT_T:%.*]] = trunc i16 [[REM]] to i8
+; CHECK-NEXT:    br label [[BODY:%.*]]
+; CHECK:       body:
+; CHECK-NEXT:    [[COND_IN_I_OFF0:%.*]] = phi i8 [ [[EXTRACT_T]], [[COND_FALSE_I]] ], [ [[ADD:%.*]], [[FOR_INC:%.*]] ]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i8 [[COND_IN_I_OFF0]], 7
+; CHECK-NEXT:    [[CONV5:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    store i32 [[CONV5]], i32* [[G:%.*]], align 4
+; CHECK-NEXT:    [[DOTPR:%.*]] = load i32, i32* [[K:%.*]], align 4
+; CHECK-NEXT:    [[TOBOOL13150:%.*]] = icmp eq i32 [[DOTPR]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL13150]], label [[FOR_INC]], label [[EXIT]]
+; CHECK:       for.inc:
+; CHECK-NEXT:    [[ADD]] = add nuw i8 [[COND_IN_I_OFF0]], 1
+; CHECK-NEXT:    br label [[BODY]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %ptr = load i16*, i16** %in, align 4
+  %ld = load i16, i16* %ptr, align 2
+  %cmp.i = icmp eq i16 %ld, 0
+  br i1 %cmp.i, label %exit, label %cond.false.i
+
+cond.false.i:
+  %rem = urem i16 5, %ld
+  %extract.t = trunc i16 %rem to i8
+  br label %body
+
+body:
+  %cond.in.i.off0 = phi i8 [ %extract.t, %cond.false.i ], [ %add, %for.inc ]
+  %cmp = icmp ugt i8 %cond.in.i.off0, 7
+  %conv5 = zext i1 %cmp to i32
+  store i32 %conv5, i32* %g, align 4
+  %.pr = load i32, i32* %k, align 4
+  %tobool13150 = icmp eq i32 %.pr, 0
+  br i1 %tobool13150, label %for.inc, label %exit
+
+for.inc:
+  %add = add nuw i8 %cond.in.i.off0, 1
+  br label %body
+
+exit:
+  ret void
+}
+
+; Check that %exp requires uxth in all cases, and will also be required to
+; promote %1 for the call - unless we can generate a uadd16.
+define i32 @zext_load_sink_call(i16* %ptr, i16 %exp) {
+; CHECK-LABEL: @zext_load_sink_call(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i16, i16* [[PTR:%.*]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = add i16 [[EXP:%.*]], 3
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i16 [[TMP0]], [[EXP]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[EXIT:%.*]], label [[IF_THEN:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[CONV0:%.*]] = zext i16 [[TMP0]] to i32
+; CHECK-NEXT:    [[CONV1:%.*]] = zext i16 [[TMP1]] to i32
+; CHECK-NEXT:    [[CALL:%.*]] = tail call arm_aapcs_vfpcc i32 @dummy(i32 [[CONV0]], i32 [[CONV1]])
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[EXITVAL:%.*]] = phi i32 [ [[CALL]], [[IF_THEN]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    ret i32 [[EXITVAL]]
+;
+entry:
+  %0 = load i16, i16* %ptr, align 4
+  %1 = add i16 %exp, 3
+  %cmp = icmp eq i16 %0, %exp
+  br i1 %cmp, label %exit, label %if.then
+
+if.then:
+  %conv0 = zext i16 %0 to i32
+  %conv1 = zext i16 %1 to i32
+  %call = tail call arm_aapcs_vfpcc i32 @dummy(i32 %conv0, i32 %conv1)
+  br label %exit
+
+exit:
+  %exitval = phi i32 [ %call, %if.then ], [ 0, %entry  ]
+  ret i32 %exitval
+}
+
+define i16 @bitcast_i16(i16 zeroext %arg0, i16 zeroext %arg1) {
+entry:
+  %cast = bitcast i16 12345 to i16
+  %add = add nuw i16 %arg0, 1
+  %cmp = icmp ule i16 %add, %cast
+  %res = select i1 %cmp, i16 %arg1, i16 32657
+  ret i16 %res
+}
+
+define i8 @bitcast_i8(i8 zeroext %arg0, i8 zeroext %arg1) {
+; CHECK-LABEL: @bitcast_i8(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = zext i8 [[ARG0:%.*]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[ARG1:%.*]] to i32
+; CHECK-NEXT:    [[CAST:%.*]] = bitcast i8 127 to i8
+; CHECK-NEXT:    [[MUL:%.*]] = shl nuw i32 [[TMP0]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp uge i32 [[MUL]], [[TMP1]]
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[CMP]], i8 [[CAST]], i8 -128
+; CHECK-NEXT:    ret i8 [[RES]]
+;
+entry:
+  %cast = bitcast i8 127 to i8
+  %mul = shl nuw i8 %arg0, 1
+  %cmp = icmp uge i8 %mul, %arg1
+  %res = select i1 %cmp, i8 %cast, i8 128
+  ret i8 %res
+}
+
+define i16 @bitcast_i16_minus(i16 zeroext %arg0, i16 zeroext %arg1) {
+; CHECK-LABEL: @bitcast_i16_minus(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[ARG0:%.*]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[ARG1:%.*]] to i32
+; CHECK-NEXT:    [[CAST:%.*]] = bitcast i16 -12345 to i16
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[TMP0]], 7
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[XOR]], [[TMP1]]
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[CMP]], i16 [[CAST]], i16 32657
+; CHECK-NEXT:    ret i16 [[RES]]
+;
+entry:
+  %cast = bitcast i16 -12345 to i16
+  %xor = xor i16 %arg0, 7
+  %cmp = icmp eq i16 %xor, %arg1
+  %res = select i1 %cmp, i16 %cast, i16 32657
+  ret i16 %res
+}
+
+define i8 @bitcast_i8_minus(i8 zeroext %arg0, i8 zeroext %arg1) {
+; CHECK-LABEL: @bitcast_i8_minus(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = zext i8 [[ARG0:%.*]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[ARG1:%.*]] to i32
+; CHECK-NEXT:    [[CAST:%.*]] = bitcast i8 -127 to i8
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[TMP0]], 3
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[AND]], [[TMP1]]
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[CMP]], i8 [[CAST]], i8 -128
+; CHECK-NEXT:    ret i8 [[RES]]
+;
+entry:
+  %cast = bitcast i8 -127 to i8
+  %and = and i8 %arg0, 3
+  %cmp = icmp ne i8 %and, %arg1
+  %res = select i1 %cmp, i8 %cast, i8 128
+  ret i8 %res
+}
+
+declare i32 @dummy(i32, i32)
+
+@d_uch = hidden local_unnamed_addr global [16 x i8] zeroinitializer, align 1
+@sh1 = hidden local_unnamed_addr global i16 0, align 2
+@d_sh = hidden local_unnamed_addr global [16 x i16] zeroinitializer, align 2
+
+define i8* @two_stage_zext_trunc_mix(i32* %this, i32 %__pos1, i32 %__n1, i32** %__str, i32 %__pos2, i32 %__n2) {
+; CHECK-LABEL: @two_stage_zext_trunc_mix(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[__SIZE__I_I_I_I:%.*]] = bitcast i32** [[__STR:%.*]] to i8*
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, i8* [[__SIZE__I_I_I_I]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[TMP0]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], 1
+; CHECK-NEXT:    [[TOBOOL_I_I_I_I:%.*]] = icmp eq i32 [[TMP2]], 0
+; CHECK-NEXT:    [[__SIZE__I5_I_I:%.*]] = getelementptr inbounds i32*, i32** [[__STR]], i32 [[__N1:%.*]]
+; CHECK-NEXT:    [[CAST:%.*]] = bitcast i32** [[__SIZE__I5_I_I]] to i32*
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, i32* [[CAST]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = lshr i32 [[TMP1]], 1
+; CHECK-NEXT:    [[TMP5:%.*]] = trunc i32 [[TMP4]] to i8
+; CHECK-NEXT:    [[COND_I_I:%.*]] = select i1 [[TOBOOL_I_I_I_I]], i32 [[TMP4]], i32 [[TMP3]]
+; CHECK-NEXT:    [[__SIZE__I_I_I_I_I:%.*]] = bitcast i32* [[THIS:%.*]] to i8*
+; CHECK-NEXT:    [[TMP6:%.*]] = load i8, i8* [[__SIZE__I_I_I_I_I]], align 4
+; CHECK-NEXT:    [[TMP7:%.*]] = zext i8 [[TMP6]] to i32
+; CHECK-NEXT:    [[TMP8:%.*]] = and i32 [[TMP7]], 1
+; CHECK-NEXT:    [[TOBOOL_I_I_I_I_I:%.*]] = icmp eq i32 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, i8* [[__SIZE__I_I_I_I]], i32 [[__POS1:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i8, i8* [[__SIZE__I_I_I_I]], i32 [[__POS2:%.*]]
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[TOBOOL_I_I_I_I_I]], i8* [[TMP9]], i8* [[TMP10]]
+; CHECK-NEXT:    ret i8* [[RES]]
+;
+entry:
+  %__size_.i.i.i.i = bitcast i32** %__str to i8*
+  %0 = load i8, i8* %__size_.i.i.i.i, align 4
+  %1 = and i8 %0, 1
+  %tobool.i.i.i.i = icmp eq i8 %1, 0
+  %__size_.i5.i.i = getelementptr inbounds i32*, i32** %__str, i32 %__n1
+  %cast = bitcast i32** %__size_.i5.i.i to i32*
+  %2 = load i32, i32* %cast, align 4
+  %3 = lshr i8 %0, 1
+  %4 = zext i8 %3 to i32
+  %cond.i.i = select i1 %tobool.i.i.i.i, i32 %4, i32 %2
+  %__size_.i.i.i.i.i = bitcast i32* %this to i8*
+  %5 = load i8, i8* %__size_.i.i.i.i.i, align 4
+  %6 = and i8 %5, 1
+  %tobool.i.i.i.i.i = icmp eq i8 %6, 0
+  %7 = getelementptr inbounds i8, i8* %__size_.i.i.i.i, i32 %__pos1
+  %8 = getelementptr inbounds i8, i8* %__size_.i.i.i.i, i32 %__pos2
+  %res = select i1 %tobool.i.i.i.i.i,  i8* %7, i8* %8
+  ret i8* %res
+}
+
+define i8 @search_through_zext_1(i8 zeroext %a, i8 zeroext %b, i16 zeroext %c) {
+; CHECK-LABEL: @search_through_zext_1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = zext i8 [[B:%.*]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i16 [[C:%.*]] to i32
+; CHECK-NEXT:    [[ADD:%.*]] = add nuw i32 [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[ADD]], [[TMP2]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[SUB:%.*]] = sub nuw i32 [[TMP0]], [[TMP1]]
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ugt i32 [[SUB]], [[TMP2]]
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[CMP2]], i32 [[TMP1]], i32 [[TMP0]]
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    [[RETVAL:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[RES]], [[IF_THEN]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = trunc i32 [[RETVAL]] to i8
+; CHECK-NEXT:    ret i8 [[TMP3]]
+;
+entry:
+  %add = add nuw i8 %a, %b
+  %conv = zext i8 %add to i16
+  %cmp = icmp ult i16 %conv, %c
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  %sub = sub nuw i8 %b, %a
+  %conv2 = zext i8 %sub to i16
+  %cmp2 = icmp ugt i16 %conv2, %c
+  %res = select i1 %cmp2, i8 %a, i8 %b
+  br label %if.end
+
+if.end:
+  %retval = phi i8 [ 0, %entry ], [ %res, %if.then ]
+  ret i8 %retval
+}
+
+; TODO: We should be able to remove the uxtb here. The transform fails because
+; the icmp ugt uses an i32, which is too large... but this doesn't matter
+; because it won't be writing a large value to a register as a result.
+define i8 @search_through_zext_2(i8 zeroext %a, i8 zeroext %b, i16 zeroext %c, i32 %d) {
+; CHECK-LABEL: @search_through_zext_2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ADD:%.*]] = add nuw i8 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[CONV:%.*]] = zext i8 [[ADD]] to i16
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i16 [[CONV]], [[C:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[SUB:%.*]] = sub nuw i8 [[B]], [[A]]
+; CHECK-NEXT:    [[CONV2:%.*]] = zext i8 [[SUB]] to i32
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ugt i32 [[CONV2]], [[D:%.*]]
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[CMP2]], i8 [[A]], i8 [[B]]
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    [[RETVAL:%.*]] = phi i8 [ 0, [[ENTRY:%.*]] ], [ [[RES]], [[IF_THEN]] ]
+; CHECK-NEXT:    ret i8 [[RETVAL]]
+;
+entry:
+  %add = add nuw i8 %a, %b
+  %conv = zext i8 %add to i16
+  %cmp = icmp ult i16 %conv, %c
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  %sub = sub nuw i8 %b, %a
+  %conv2 = zext i8 %sub to i32
+  %cmp2 = icmp ugt i32 %conv2, %d
+  %res = select i1 %cmp2, i8 %a, i8 %b
+  br label %if.end
+
+if.end:
+  %retval = phi i8 [ 0, %entry ], [ %res, %if.then ]
+  ret i8 %retval
+}
+
+; TODO: We should be able to remove the uxtb here as all the calculations are
+; performed on i8s. The promotion of i8 to i16 and then the later truncation
+; results in the uxtb.
+define i8 @search_through_zext_3(i8 zeroext %a, i8 zeroext %b, i16 zeroext %c, i32 %d) {
+; CHECK-LABEL: @search_through_zext_3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ADD:%.*]] = add nuw i8 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[CONV:%.*]] = zext i8 [[ADD]] to i16
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i16 [[CONV]], [[C:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc i16 [[CONV]] to i8
+; CHECK-NEXT:    [[SUB:%.*]] = sub nuw i8 [[B]], [[TRUNC]]
+; CHECK-NEXT:    [[CONV2:%.*]] = zext i8 [[SUB]] to i32
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ugt i32 [[CONV2]], [[D:%.*]]
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[CMP2]], i8 [[A]], i8 [[B]]
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    [[RETVAL:%.*]] = phi i8 [ 0, [[ENTRY:%.*]] ], [ [[RES]], [[IF_THEN]] ]
+; CHECK-NEXT:    ret i8 [[RETVAL]]
+;
+entry:
+  %add = add nuw i8 %a, %b
+  %conv = zext i8 %add to i16
+  %cmp = icmp ult i16 %conv, %c
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  %trunc = trunc i16 %conv to i8
+  %sub = sub nuw i8 %b, %trunc
+  %conv2 = zext i8 %sub to i32
+  %cmp2 = icmp ugt i32 %conv2, %d
+  %res = select i1 %cmp2, i8 %a, i8 %b
+  br label %if.end
+
+if.end:
+  %retval = phi i8 [ 0, %entry ], [ %res, %if.then ]
+  ret i8 %retval
+}
+
+; TODO: We should be able to remove the uxt that gets introduced for %conv2
+define i8 @search_through_zext_cmp(i8 zeroext %a, i8 zeroext %b, i16 zeroext %c) {
+; CHECK-LABEL: @search_through_zext_cmp(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i8 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i16
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult i16 [[CONV]], [[C:%.*]]
+; CHECK-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[SUB:%.*]] = sub nuw i8 [[B]], [[A]]
+; CHECK-NEXT:    [[CONV2:%.*]] = zext i8 [[SUB]] to i16
+; CHECK-NEXT:    [[CMP3:%.*]] = icmp ugt i16 [[CONV2]], [[C]]
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[CMP3]], i8 [[A]], i8 [[B]]
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    [[RETVAL:%.*]] = phi i8 [ 0, [[ENTRY:%.*]] ], [ [[RES]], [[IF_THEN]] ]
+; CHECK-NEXT:    ret i8 [[RETVAL]]
+;
+entry:
+  %cmp = icmp ne i8 %a, %b
+  %conv = zext i1 %cmp to i16
+  %cmp1 = icmp ult i16 %conv, %c
+  br i1 %cmp1, label %if.then, label %if.end
+
+if.then:
+  %sub = sub nuw i8 %b, %a
+  %conv2 = zext i8 %sub to i16
+  %cmp3 = icmp ugt i16 %conv2, %c
+  %res = select i1 %cmp3, i8 %a, i8 %b
+  br label %if.end
+
+if.end:
+  %retval = phi i8 [ 0, %entry ], [ %res, %if.then ]
+  ret i8 %retval
+}
+
+define i8 @search_through_zext_load(i8* %a, i8 zeroext %b, i16 zeroext %c) {
+; CHECK-LABEL: @search_through_zext_load(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = zext i8 [[B:%.*]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[C:%.*]] to i32
+; CHECK-NEXT:    [[LOAD:%.*]] = load i8, i8* [[A:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i8 [[LOAD]] to i32
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult i32 [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[SUB:%.*]] = sub nuw i32 [[TMP0]], [[TMP2]]
+; CHECK-NEXT:    [[CMP3:%.*]] = icmp ugt i32 [[SUB]], [[TMP1]]
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[CMP3]], i32 [[TMP2]], i32 [[TMP0]]
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    [[RETVAL:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[RES]], [[IF_THEN]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = trunc i32 [[RETVAL]] to i8
+; CHECK-NEXT:    ret i8 [[TMP3]]
+;
+entry:
+  %load = load i8, i8* %a
+  %conv = zext i8 %load to i16
+  %cmp1 = icmp ult i16 %conv, %c
+  br i1 %cmp1, label %if.then, label %if.end
+
+if.then:
+  %sub = sub nuw i8 %b, %load
+  %conv2 = zext i8 %sub to i16
+  %cmp3 = icmp ugt i16 %conv2, %c
+  %res = select i1 %cmp3, i8 %load, i8 %b
+  br label %if.end
+
+if.end:
+  %retval = phi i8 [ 0, %entry ], [ %res, %if.then ]
+  ret i8 %retval
+}
+
+define i16 @trunc_sink_less_than_cmp(i16 zeroext %a, i16 zeroext %b, i16 zeroext %c, i8 zeroext %d) {
+; CHECK-LABEL: @trunc_sink_less_than_cmp(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SUB:%.*]] = sub nuw i16 [[B:%.*]], [[A:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i16 [[SUB]], [[C:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc i16 [[SUB]] to i8
+; CHECK-NEXT:    [[ADD:%.*]] = add nuw i8 [[D:%.*]], 1
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ugt i8 [[TRUNC]], [[ADD]]
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[CMP2]], i16 [[A]], i16 [[B]]
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    [[RETVAL:%.*]] = phi i16 [ 0, [[ENTRY:%.*]] ], [ [[RES]], [[IF_THEN]] ]
+; CHECK-NEXT:    ret i16 [[RETVAL]]
+;
+entry:
+  %sub = sub nuw i16 %b, %a
+  %cmp = icmp ult i16 %sub, %c
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  %trunc = trunc i16 %sub to i8
+  %add = add nuw i8 %d, 1
+  %cmp2 = icmp ugt i8 %trunc, %add
+  %res = select i1 %cmp2, i16 %a, i16 %b
+  br label %if.end
+
+if.end:
+  %retval = phi i16 [ 0, %entry ], [ %res, %if.then ]
+  ret i16 %retval
+}
+
+; TODO: We should be able to remove the uxth introduced to handle %sub
+define i16 @trunc_sink_less_than_arith(i16 zeroext %a, i16 zeroext %b, i16 zeroext %c, i8 zeroext %d, i8 zeroext %e) {
+; CHECK-LABEL: @trunc_sink_less_than_arith(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SUB:%.*]] = sub nuw i16 [[B:%.*]], [[A:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i16 [[SUB]], [[C:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc i16 [[SUB]] to i8
+; CHECK-NEXT:    [[ADD:%.*]] = add nuw i8 [[D:%.*]], [[TRUNC]]
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ugt i8 [[E:%.*]], [[ADD]]
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[CMP2]], i16 [[A]], i16 [[B]]
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    [[RETVAL:%.*]] = phi i16 [ 0, [[ENTRY:%.*]] ], [ [[RES]], [[IF_THEN]] ]
+; CHECK-NEXT:    ret i16 [[RETVAL]]
+;
+entry:
+  %sub = sub nuw i16 %b, %a
+  %cmp = icmp ult i16 %sub, %c
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  %trunc = trunc i16 %sub to i8
+  %add = add nuw i8 %d, %trunc
+  %cmp2 = icmp ugt i8 %e, %add
+  %res = select i1 %cmp2, i16 %a, i16 %b
+  br label %if.end
+
+if.end:
+  %retval = phi i16 [ 0, %entry ], [ %res, %if.then ]
+  ret i16 %retval
+}
+
+define i16 @trunc_sink_less_than_store(i16 zeroext %a, i16 zeroext %b, i16 zeroext %c, i8 zeroext %d, i8* %e) {
+; CHECK-LABEL: @trunc_sink_less_than_store(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[B:%.*]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[A:%.*]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i8 [[D:%.*]] to i32
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i16 [[C:%.*]] to i32
+; CHECK-NEXT:    [[SUB:%.*]] = sub nuw i32 [[TMP0]], [[TMP1]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[SUB]], [[TMP3]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[TMP4:%.*]] = and i32 [[SUB]], 255
+; CHECK-NEXT:    [[ADD:%.*]] = add nuw i32 [[TMP2]], [[TMP4]]
+; CHECK-NEXT:    [[TMP5:%.*]] = trunc i32 [[ADD]] to i8
+; CHECK-NEXT:    store i8 [[TMP5]], i8* [[E:%.*]]
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    [[RETVAL:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[SUB]], [[IF_THEN]] ]
+; CHECK-NEXT:    [[TMP6:%.*]] = trunc i32 [[RETVAL]] to i16
+; CHECK-NEXT:    ret i16 [[TMP6]]
+;
+entry:
+  %sub = sub nuw i16 %b, %a
+  %cmp = icmp ult i16 %sub, %c
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  %trunc = trunc i16 %sub to i8
+  %add = add nuw i8 %d, %trunc
+  store i8 %add, i8* %e
+  br label %if.end
+
+if.end:
+  %retval = phi i16 [ 0, %entry ], [ %sub, %if.then ]
+  ret i16 %retval
+}
+
+define i8 @trunc_sink_less_than_ret(i16 zeroext %a, i16 zeroext %b, i16 zeroext %c, i8 zeroext %d, i8 zeroext %e) {
+; CHECK-LABEL: @trunc_sink_less_than_ret(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[B:%.*]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[A:%.*]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i8 [[D:%.*]] to i32
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i16 [[C:%.*]] to i32
+; CHECK-NEXT:    [[SUB:%.*]] = sub nuw i32 [[TMP0]], [[TMP1]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[SUB]], [[TMP3]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[TMP4:%.*]] = and i32 [[SUB]], 255
+; CHECK-NEXT:    [[ADD:%.*]] = add nuw i32 [[TMP2]], [[TMP4]]
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    [[RETVAL:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[ADD]], [[IF_THEN]] ]
+; CHECK-NEXT:    [[TMP5:%.*]] = trunc i32 [[RETVAL]] to i8
+; CHECK-NEXT:    ret i8 [[TMP5]]
+;
+entry:
+  %sub = sub nuw i16 %b, %a
+  %cmp = icmp ult i16 %sub, %c
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  %trunc = trunc i16 %sub to i8
+  %add = add nuw i8 %d, %trunc
+  br label %if.end
+
+if.end:
+  %retval = phi i8 [ 0, %entry ], [ %add, %if.then ]
+  ret i8 %retval
+}
+
+define zeroext i8 @trunc_sink_less_than_zext_ret(i16 zeroext %a, i16 zeroext %b, i16 zeroext %c, i8 zeroext %d, i8 zeroext %e) {
+; CHECK-LABEL: @trunc_sink_less_than_zext_ret(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[B:%.*]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[A:%.*]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i8 [[D:%.*]] to i32
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i16 [[C:%.*]] to i32
+; CHECK-NEXT:    [[SUB:%.*]] = sub nuw i32 [[TMP0]], [[TMP1]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[SUB]], [[TMP3]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[TMP4:%.*]] = and i32 [[SUB]], 255
+; CHECK-NEXT:    [[ADD:%.*]] = add nuw i32 [[TMP2]], [[TMP4]]
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    [[RETVAL:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[ADD]], [[IF_THEN]] ]
+; CHECK-NEXT:    [[TMP5:%.*]] = trunc i32 [[RETVAL]] to i8
+; CHECK-NEXT:    ret i8 [[TMP5]]
+;
+entry:
+  %sub = sub nuw i16 %b, %a
+  %cmp = icmp ult i16 %sub, %c
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  %trunc = trunc i16 %sub to i8
+  %add = add nuw i8 %d, %trunc
+  br label %if.end
+
+if.end:
+  %retval = phi i8 [ 0, %entry ], [ %add, %if.then ]
+  ret i8 %retval
+}
+
+define i32 @bitcast_i1(i16 zeroext %a, i32 %b, i32 %c) {
+; CHECK-LABEL: @bitcast_i1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i1 true to i1
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i16 [[A:%.*]] to i1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i1 [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[EXIT:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[TMP0]] to i16
+; CHECK-NEXT:    [[CONV1:%.*]] = zext i1 [[TMP1]] to i16
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp uge i16 [[CONV]], [[CONV1]]
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP1]], i32 [[B:%.*]], i32 [[C:%.*]]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RETVAL:%.*]] = phi i32 [ [[SELECT]], [[IF_THEN]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    ret i32 [[RETVAL]]
+;
+entry:
+  %0 = bitcast i1 1 to i1
+  %1 = trunc i16 %a to i1
+  %cmp = icmp eq i1 %1, %0
+  br i1 %cmp, label %if.then, label %exit
+
+if.then:
+  %conv = zext i1 %0 to i16
+  %conv1 = zext i1 %1 to i16
+  %cmp1 = icmp uge i16 %conv, %conv1
+  %select = select i1 %cmp1, i32 %b, i32 %c
+  br label %exit
+
+exit:
+  %retval = phi i32 [ %select, %if.then ], [ 0, %entry ]
+  ret i32 %retval
+}
+
+define void @search_back_through_trunc(i8* %a, i8* %b, i8* %c, i8* %d, i16* %e) {
+; CHECK-LABEL: @search_back_through_trunc(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, i8* [[A:%.*]], align 1
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[TMP0]] to i32
+; CHECK-NEXT:    [[SHL:%.*]] = shl nuw i32 [[TMP1]], 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, i8* [[B:%.*]], align 1
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i8 [[TMP2]] to i32
+; CHECK-NEXT:    [[OR109:%.*]] = or i32 [[SHL]], [[TMP3]]
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, i8* [[C:%.*]], align 1
+; CHECK-NEXT:    [[TMP5:%.*]] = zext i8 [[TMP4]] to i32
+; CHECK-NEXT:    [[SHL120:%.*]] = shl nuw i32 [[TMP5]], 8
+; CHECK-NEXT:    [[TMP6:%.*]] = load i8, i8* [[D:%.*]], align 1
+; CHECK-NEXT:    [[TMP7:%.*]] = zext i8 [[TMP6]] to i32
+; CHECK-NEXT:    [[OR123:%.*]] = or i32 [[SHL120]], [[TMP7]]
+; CHECK-NEXT:    [[CMP133:%.*]] = icmp eq i32 [[OR109]], [[OR123]]
+; CHECK-NEXT:    br i1 [[CMP133]], label [[IF_END183:%.*]], label [[IF_ELSE136:%.*]]
+; CHECK:       if.else136:
+; CHECK-NEXT:    [[TMP8:%.*]] = load i16, i16* [[E:%.*]], align 2
+; CHECK-NEXT:    [[TMP9:%.*]] = zext i16 [[TMP8]] to i32
+; CHECK-NEXT:    [[TMP10:%.*]] = and i32 [[TMP9]], 255
+; CHECK-NEXT:    [[EXTRACT856:%.*]] = lshr i32 [[TMP9]], 8
+; CHECK-NEXT:    [[TMP11:%.*]] = and i32 [[EXTRACT856]], 255
+; CHECK-NEXT:    br label [[IF_END183]]
+; CHECK:       if.end183:
+; CHECK-NEXT:    [[W_0_OFF0:%.*]] = phi i32 [ [[TMP10]], [[IF_ELSE136]] ], [ [[TMP3]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[W_0_OFF8:%.*]] = phi i32 [ [[TMP11]], [[IF_ELSE136]] ], [ [[TMP5]], [[ENTRY]] ]
+; CHECK-NEXT:    [[TMP12:%.*]] = trunc i32 [[W_0_OFF8]] to i8
+; CHECK-NEXT:    store i8 [[TMP12]], i8* [[C]], align 1
+; CHECK-NEXT:    [[TMP13:%.*]] = trunc i32 [[W_0_OFF0]] to i8
+; CHECK-NEXT:    store i8 [[TMP13]], i8* [[D]], align 1
+; CHECK-NEXT:    ret void
+;
+entry:
+  %0 = load i8, i8* %a, align 1
+  %conv106 = zext i8 %0 to i16
+  %shl = shl nuw i16 %conv106, 8
+  %1 = load i8, i8* %b, align 1
+  %conv108 = zext i8 %1 to i16
+  %or109 = or i16 %shl, %conv108
+  %2 = load i8, i8* %c, align 1
+  %conv119 = zext i8 %2 to i16
+  %shl120 = shl nuw i16 %conv119, 8
+  %3 = load i8, i8* %d, align 1
+  %conv122 = zext i8 %3 to i16
+  %or123 = or i16 %shl120, %conv122
+  %cmp133 = icmp eq i16 %or109, %or123
+  br i1 %cmp133, label %if.end183, label %if.else136
+
+if.else136:
+  %4 = load i16, i16* %e, align 2
+  %extract.t854 = trunc i16 %4 to i8
+  %extract856 = lshr i16 %4, 8
+  %extract.t857 = trunc i16 %extract856 to i8
+  br label %if.end183
+
+if.end183:
+  %w.0.off0 = phi i8 [ %extract.t854, %if.else136 ], [ %1, %entry ]
+  %w.0.off8 = phi i8 [ %extract.t857, %if.else136 ], [ %2, %entry ]
+  store i8 %w.0.off8, i8* %c, align 1
+  store i8 %w.0.off0, i8* %d, align 1
+  ret void
+}
+
+@c = common dso_local local_unnamed_addr global i16 0, align 2
+@b = common dso_local local_unnamed_addr global i16 0, align 2
+@f = common dso_local local_unnamed_addr global i32 0, align 4
+@e = common dso_local local_unnamed_addr global i8 0, align 1
+@a = common dso_local local_unnamed_addr global i8 0, align 1
+@d = common dso_local local_unnamed_addr global i32 0, align 4
+
+define void @and_trunc_two_zext() {
+; CHECK-LABEL: @and_trunc_two_zext(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i16, i16* @c, align 2
+; CHECK-NEXT:    [[TMP1:%.*]] = load i16, i16* @b, align 2
+; CHECK-NEXT:    [[CONV:%.*]] = sext i16 [[TMP1]] to i32
+; CHECK-NEXT:    store i32 [[CONV]], i32* @f, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc i16 [[TMP1]] to i8
+; CHECK-NEXT:    [[CONV1:%.*]] = and i8 [[TMP2]], 1
+; CHECK-NEXT:    store i8 [[CONV1]], i8* @e, align 1
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, i8* @a, align 1
+; CHECK-NEXT:    [[NARROW:%.*]] = mul nuw i8 [[TMP3]], [[CONV1]]
+; CHECK-NEXT:    [[MUL:%.*]] = zext i8 [[NARROW]] to i32
+; CHECK-NEXT:    store i32 [[MUL]], i32* @d, align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i8 [[NARROW]] to i16
+; CHECK-NEXT:    [[CONV5:%.*]] = or i16 [[TMP0]], [[TMP4]]
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i16 [[CONV5]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL]], label [[IF_END:%.*]], label [[FOR_COND:%.*]]
+; CHECK:       for.cond:
+; CHECK-NEXT:    br label [[FOR_COND]]
+; CHECK:       if.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %0 = load i16, i16* @c, align 2
+  %1 = load i16, i16* @b, align 2
+  %conv = sext i16 %1 to i32
+  store i32 %conv, i32* @f, align 4
+  %2 = trunc i16 %1 to i8
+  %conv1 = and i8 %2, 1
+  store i8 %conv1, i8* @e, align 1
+  %3 = load i8, i8* @a, align 1
+  %narrow = mul nuw i8 %3, %conv1
+  %mul = zext i8 %narrow to i32
+  store i32 %mul, i32* @d, align 4
+  %4 = zext i8 %narrow to i16
+  %conv5 = or i16 %0, %4
+  %tobool = icmp eq i16 %conv5, 0
+  br i1 %tobool, label %if.end, label %for.cond
+
+for.cond:
+  br label %for.cond
+
+if.end:
+  ret void
+}
+
+define void @zext_urem_trunc() {
+; CHECK-LABEL: @zext_urem_trunc(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i16, i16* @c, align 2
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[TMP0]] to i32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, i8* @e, align 1
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i8 [[TMP2]] to i32
+; CHECK-NEXT:    br i1 [[CMP]], label [[COND_END:%.*]], label [[COND_FALSE:%.*]]
+; CHECK:       cond.false:
+; CHECK-NEXT:    [[REM7:%.*]] = urem i32 [[TMP3]], [[TMP1]]
+; CHECK-NEXT:    [[TMP4:%.*]] = and i32 [[REM7]], 255
+; CHECK-NEXT:    br label [[COND_END]]
+; CHECK:       cond.end:
+; CHECK-NEXT:    [[COND:%.*]] = phi i32 [ [[TMP4]], [[COND_FALSE]] ], [ [[TMP3]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[TMP5:%.*]] = trunc i32 [[COND]] to i8
+; CHECK-NEXT:    store i8 [[TMP5]], i8* @a, align 1
+; CHECK-NEXT:    ret void
+;
+entry:
+  %0 = load i16, i16* @c, align 2
+  %cmp = icmp eq i16 %0, 0
+  %1 = load i8, i8* @e, align 1
+  br i1 %cmp, label %cond.end, label %cond.false
+
+cond.false:
+  %rem.lhs.trunc = zext i8 %1 to i16
+  %rem7 = urem i16 %rem.lhs.trunc, %0
+  %rem.zext = trunc i16 %rem7 to i8
+  br label %cond.end
+
+cond.end:
+  %cond = phi i8 [ %rem.zext, %cond.false ], [ %1, %entry ]
+  store i8 %cond, i8* @a, align 1
+  ret void
+}
+
+define i1 @dont_replace_trunc_1(i8* %a, i16* %b, i16* %c, i32* %d, i8* %e, i32* %f) {
+; CHECK-LABEL: @dont_replace_trunc_1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i16, i16* [[C:%.*]], align 2
+; CHECK-NEXT:    [[TMP1:%.*]] = load i16, i16* [[B:%.*]], align 2
+; CHECK-NEXT:    [[CONV:%.*]] = sext i16 [[TMP1]] to i32
+; CHECK-NEXT:    store i32 [[CONV]], i32* [[F:%.*]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc i16 [[TMP1]] to i8
+; CHECK-NEXT:    [[CONV1:%.*]] = and i8 [[TMP2]], 1
+; CHECK-NEXT:    store i8 [[CONV1]], i8* [[E:%.*]], align 1
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, i8* [[A:%.*]], align 1
+; CHECK-NEXT:    [[NARROW:%.*]] = mul nuw i8 [[TMP3]], [[CONV1]]
+; CHECK-NEXT:    [[MUL:%.*]] = zext i8 [[NARROW]] to i32
+; CHECK-NEXT:    store i32 [[MUL]], i32* [[D:%.*]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i8 [[NARROW]] to i16
+; CHECK-NEXT:    [[CONV5:%.*]] = or i16 [[TMP0]], [[TMP4]]
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i16 [[CONV5]], 0
+; CHECK-NEXT:    ret i1 [[TOBOOL]]
+;
+entry:
+  %0 = load i16, i16* %c, align 2
+  %1 = load i16, i16* %b, align 2
+  %conv = sext i16 %1 to i32
+  store i32 %conv, i32* %f, align 4
+  %2 = trunc i16 %1 to i8
+  %conv1 = and i8 %2, 1
+  store i8 %conv1, i8* %e, align 1
+  %3 = load i8, i8* %a, align 1
+  %narrow = mul nuw i8 %3, %conv1
+  %mul = zext i8 %narrow to i32
+  store i32 %mul, i32* %d, align 4
+  %4 = zext i8 %narrow to i16
+  %conv5 = or i16 %0, %4
+  %tobool = icmp eq i16 %conv5, 0
+  ret i1 %tobool
+}
+
+define i32 @dont_replace_trunc_2(i16* %a, i8* %b) {
+; CHECK-LABEL: @dont_replace_trunc_2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i16, i16* [[A:%.*]], align 2
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[TMP0]] to i32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[TMP1]], 8
+; CHECK-NEXT:    [[NARROW:%.*]] = select i1 [[CMP]], i32 [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[NARROW]], 255
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, i8* [[B:%.*]], align 1
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i8 [[TMP3]] to i32
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[TMP4]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = trunc i32 [[OR]] to i8
+; CHECK-NEXT:    store i8 [[TMP5]], i8* [[B]], align 1
+; CHECK-NEXT:    [[TMP6:%.*]] = trunc i32 [[OR]] to i8
+; CHECK-NEXT:    [[CONV5:%.*]] = zext i8 [[TMP6]] to i32
+; CHECK-NEXT:    ret i32 [[CONV5]]
+;
+entry:
+  %0 = load i16, i16* %a, align 2
+  %cmp = icmp ugt i16 %0, 8
+  %narrow = select i1 %cmp, i16 %0, i16 0
+  %cond = trunc i16 %narrow to i8
+  %1 = load i8, i8* %b, align 1
+  %or = or i8 %1, %cond
+  store i8 %or, i8* %b, align 1
+  %conv5 = zext i8 %or to i32
+  ret i32 %conv5
+}
+
+define i32 @replace_trunk_with_mask(i16* %a) {
+; CHECK-LABEL: @replace_trunk_with_mask(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i16, i16* [[A:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[TMP0]] to i32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[COND_END:%.*]], label [[COND_FALSE:%.*]]
+; CHECK:       cond.false:
+; CHECK-NEXT:    [[TMP2:%.*]] = urem i32 535, [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[TMP2]], 255
+; CHECK-NEXT:    [[TMP4:%.*]] = udiv i32 [[TMP3]], 3
+; CHECK-NEXT:    [[TMP5:%.*]] = trunc i32 [[TMP4]] to i8
+; CHECK-NEXT:    [[PHITMP:%.*]] = zext i8 [[TMP5]] to i32
+; CHECK-NEXT:    br label [[COND_END]]
+; CHECK:       cond.end:
+; CHECK-NEXT:    [[COND:%.*]] = phi i32 [ [[PHITMP]], [[COND_FALSE]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+entry:
+  %0 = load i16, i16* %a
+  %cmp = icmp eq i16 %0, 0
+  br i1 %cmp, label %cond.end, label %cond.false
+
+cond.false:
+  %1 = urem i16 535, %0
+  %.lhs.trunc = trunc i16 %1 to i8
+  %2 = udiv i8 %.lhs.trunc, 3
+  %phitmp = zext i8 %2 to i32
+  br label %cond.end
+
+cond.end:
+  %cond = phi i32 [ %phitmp, %cond.false ], [ 0, %entry ]
+  ret i32 %cond
+}
+
+define float @test_i8_sitofp(i8* %ptr, i8 %arg) {
+; CHECK-LABEL: @test_i8_sitofp(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, i8* [[PTR:%.*]], align 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[TMP0]], [[ARG:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[EXIT:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.end:
+; CHECK-NEXT:    [[CONV:%.*]] = sitofp i8 [[ARG]] to float
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv float [[CONV]], 2.000000e+01
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RES:%.*]] = phi float [ 0.000000e+00, [[ENTRY:%.*]] ], [ [[DIV]], [[IF_END]] ]
+; CHECK-NEXT:    ret float [[RES]]
+;
+entry:
+  %0 = load i8, i8* %ptr, align 1
+  %cmp = icmp eq i8 %0, %arg
+  br i1 %cmp, label %exit, label %if.end
+
+if.end:
+  %conv = sitofp i8 %arg to float
+  %div = fdiv float %conv, 2.000000e+01
+  br label %exit
+
+exit:
+  %res = phi float [ 0.0, %entry ], [ %div, %if.end ]
+  ret float %res
+}
+
+define float @test_i16_sitofp(i16* %ptr, i16 %arg) {
+; CHECK-LABEL: @test_i16_sitofp(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i16, i16* [[PTR:%.*]], align 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i16 [[TMP0]], [[ARG:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[EXIT:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.end:
+; CHECK-NEXT:    [[CONV:%.*]] = sitofp i16 [[ARG]] to float
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv float [[CONV]], 2.000000e+01
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RES:%.*]] = phi float [ 0.000000e+00, [[ENTRY:%.*]] ], [ [[DIV]], [[IF_END]] ]
+; CHECK-NEXT:    ret float [[RES]]
+;
+entry:
+  %0 = load i16, i16* %ptr, align 1
+  %cmp = icmp eq i16 %0, %arg
+  br i1 %cmp, label %exit, label %if.end
+
+if.end:
+  %conv = sitofp i16 %arg to float
+  %div = fdiv float %conv, 2.000000e+01
+  br label %exit
+
+exit:
+  %res = phi float [ 0.0, %entry ], [ %div, %if.end ]
+  ret float %res
+}
diff --git a/llvm/test/Transforms/TypePromotion/ARM/clear-structures.ll b/llvm/test/Transforms/TypePromotion/ARM/clear-structures.ll
new file mode 100644
index 000000000..117c4c0d5
--- /dev/null
+++ b/llvm/test/Transforms/TypePromotion/ARM/clear-structures.ll
@@ -0,0 +1,124 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -mtriple=arm -type-promotion -verify -disable-type-promotion=false -S %s -o - | FileCheck %s
+
+define i32 @clear_structures(i8* nocapture readonly %fmt, [1 x i32] %ap.coerce, i8* %out, void (i32, i8*)* nocapture %write) {
+; CHECK-LABEL: @clear_structures(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[WHILE_COND_OUTER:%.*]]
+; CHECK:       while.cond.outer:
+; CHECK-NEXT:    [[FMT_ADDR_0_PH:%.*]] = phi i8* [ [[FMT:%.*]], [[ENTRY:%.*]] ], [ [[FMT_ADDR_3:%.*]], [[WHILE_COND_OUTER_BACKEDGE:%.*]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, i8* [[FMT_ADDR_0_PH]], align 1
+; CHECK-NEXT:    br label [[WHILE_COND:%.*]]
+; CHECK:       while.cond:
+; CHECK-NEXT:    switch i8 [[TMP0]], label [[WHILE_COND]] [
+; CHECK-NEXT:    i8 0, label [[WHILE_END48:%.*]]
+; CHECK-NEXT:    i8 37, label [[WHILE_COND2:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       while.cond2:
+; CHECK-NEXT:    [[FLAGS_0:%.*]] = phi i32 [ [[OR:%.*]], [[WHILE_COND2]] ], [ 0, [[WHILE_COND]] ]
+; CHECK-NEXT:    [[FMT_ADDR_0_PN:%.*]] = phi i8* [ [[FMT_ADDR_1:%.*]], [[WHILE_COND2]] ], [ [[FMT_ADDR_0_PH]], [[WHILE_COND]] ]
+; CHECK-NEXT:    [[FMT_ADDR_1]] = getelementptr inbounds i8, i8* [[FMT_ADDR_0_PN]], i32 1
+; CHECK-NEXT:    [[TMP1:%.*]] = load i8, i8* [[FMT_ADDR_1]], align 1
+; CHECK-NEXT:    [[SUB:%.*]] = add i8 [[TMP1]], -32
+; CHECK-NEXT:    [[CONV6:%.*]] = zext i8 [[SUB]] to i32
+; CHECK-NEXT:    [[SHL:%.*]] = shl i32 1, [[CONV6]]
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[SHL]], 75785
+; CHECK-NEXT:    [[TOBOOL7:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    [[OR]] = or i32 [[SHL]], [[FLAGS_0]]
+; CHECK-NEXT:    br i1 [[TOBOOL7]], label [[WHILE_COND10_PREHEADER:%.*]], label [[WHILE_COND2]]
+; CHECK:       while.cond10.preheader:
+; CHECK-NEXT:    [[DOTOFF:%.*]] = add i8 [[TMP1]], -48
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i8 [[DOTOFF]], 10
+; CHECK-NEXT:    br i1 [[TMP2]], label [[WHILE_COND10:%.*]], label [[WHILE_END18_SPLIT:%.*]]
+; CHECK:       while.cond10:
+; CHECK-NEXT:    br label [[WHILE_COND10]]
+; CHECK:       while.end18.split:
+; CHECK-NEXT:    [[CMP20:%.*]] = icmp eq i8 [[TMP1]], 46
+; CHECK-NEXT:    br i1 [[CMP20]], label [[IF_THEN22:%.*]], label [[COND_END:%.*]]
+; CHECK:       if.then22:
+; CHECK-NEXT:    [[INCDEC_PTR23:%.*]] = getelementptr inbounds i8, i8* [[FMT_ADDR_0_PN]], i32 2
+; CHECK-NEXT:    [[DOTPR74:%.*]] = load i8, i8* [[INCDEC_PTR23]], align 1
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i8 [[DOTPR74]] to i32
+; CHECK-NEXT:    [[TMP4:%.*]] = sub i32 [[TMP3]], 48
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ult i32 [[TMP4]], 10
+; CHECK-NEXT:    br i1 [[TMP5]], label [[WHILE_COND24:%.*]], label [[COND_END]]
+; CHECK:       while.cond24:
+; CHECK-NEXT:    br label [[WHILE_COND24]]
+; CHECK:       cond.end:
+; CHECK-NEXT:    [[FMT_ADDR_3]] = phi i8* [ [[FMT_ADDR_1]], [[WHILE_END18_SPLIT]] ], [ [[INCDEC_PTR23]], [[IF_THEN22]] ]
+; CHECK-NEXT:    [[AND39:%.*]] = and i32 [[FLAGS_0]], 2048
+; CHECK-NEXT:    [[TOBOOL40:%.*]] = icmp eq i32 [[AND39]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL40]], label [[WHILE_COND_OUTER_BACKEDGE]], label [[IF_THEN43:%.*]]
+; CHECK:       while.cond.outer.backedge:
+; CHECK-NEXT:    br label [[WHILE_COND_OUTER]]
+; CHECK:       if.then43:
+; CHECK-NEXT:    tail call void [[WRITE:%.*]](i32 43, i8* [[OUT:%.*]])
+; CHECK-NEXT:    br label [[WHILE_COND_OUTER_BACKEDGE]]
+; CHECK:       while.end48:
+; CHECK-NEXT:    ret i32 undef
+;
+entry:
+  br label %while.cond.outer
+
+while.cond.outer:
+  %fmt.addr.0.ph = phi i8* [ %fmt, %entry ], [ %fmt.addr.3, %while.cond.outer.backedge ]
+  %0 = load i8, i8* %fmt.addr.0.ph, align 1
+  br label %while.cond
+
+while.cond:
+  switch i8 %0, label %while.cond [
+  i8 0, label %while.end48
+  i8 37, label %while.cond2
+  ]
+
+while.cond2:
+  %flags.0 = phi i32 [ %or, %while.cond2 ], [ 0, %while.cond ]
+  %fmt.addr.0.pn = phi i8* [ %fmt.addr.1, %while.cond2 ], [ %fmt.addr.0.ph, %while.cond ]
+  %fmt.addr.1 = getelementptr inbounds i8, i8* %fmt.addr.0.pn, i32 1
+  %1 = load i8, i8* %fmt.addr.1, align 1
+  %sub = add i8 %1, -32
+  %conv6 = zext i8 %sub to i32
+  %shl = shl i32 1, %conv6
+  %and = and i32 %shl, 75785
+  %tobool7 = icmp eq i32 %and, 0
+  %or = or i32 %shl, %flags.0
+  br i1 %tobool7, label %while.cond10.preheader, label %while.cond2
+
+while.cond10.preheader:
+  %.off = add i8 %1, -48
+  %2 = icmp ult i8 %.off, 10
+  br i1 %2, label %while.cond10, label %while.end18.split
+
+while.cond10:
+  br label %while.cond10
+
+while.end18.split:
+  %cmp20 = icmp eq i8 %1, 46
+  br i1 %cmp20, label %if.then22, label %cond.end
+
+if.then22:
+  %incdec.ptr23 = getelementptr inbounds i8, i8* %fmt.addr.0.pn, i32 2
+  %.pr74 = load i8, i8* %incdec.ptr23, align 1
+  %.pr74.off = add i8 %.pr74, -48
+  %3 = icmp ult i8 %.pr74.off, 10
+  br i1 %3, label %while.cond24, label %cond.end
+
+while.cond24:
+  br label %while.cond24
+
+cond.end:
+  %fmt.addr.3 = phi i8* [ %fmt.addr.1, %while.end18.split ], [ %incdec.ptr23, %if.then22 ]
+  %and39 = and i32 %flags.0, 2048
+  %tobool40 = icmp eq i32 %and39, 0
+  br i1 %tobool40, label %while.cond.outer.backedge, label %if.then43
+
+while.cond.outer.backedge:
+  br label %while.cond.outer
+
+if.then43:
+  tail call void %write(i32 43, i8* %out) #1
+  br label %while.cond.outer.backedge
+
+while.end48:
+  ret i32 undef
+}
diff --git a/llvm/test/Transforms/TypePromotion/ARM/icmps.ll b/llvm/test/Transforms/TypePromotion/ARM/icmps.ll
new file mode 100644
index 000000000..6dda15c30
--- /dev/null
+++ b/llvm/test/Transforms/TypePromotion/ARM/icmps.ll
@@ -0,0 +1,349 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -mtriple=arm -type-promotion -verify -disable-type-promotion=false -S %s -o - | FileCheck %s
+
+define i32 @test_ult_254_inc_imm(i8 zeroext %x) {
+; CHECK-LABEL: @test_ult_254_inc_imm(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ADD:%.*]] = add i8 [[X:%.*]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i8 [[ADD]], -2
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[CMP]], i32 35, i32 47
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+entry:
+  %add = add i8 %x, 1
+  %cmp = icmp ult i8 %add, 254
+  %res = select i1 %cmp, i32 35, i32 47
+  ret i32 %res
+}
+
+define i32 @test_slt_254_inc_imm(i8 signext %x) {
+; CHECK-LABEL: @test_slt_254_inc_imm(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ADD:%.*]] = add i8 [[X:%.*]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[ADD]], -2
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[CMP]], i32 35, i32 47
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+entry:
+  %add = add i8 %x, 1
+  %cmp = icmp slt i8 %add, 254
+  %res = select i1 %cmp, i32 35, i32 47
+  ret i32 %res
+}
+
+define i32 @test_ult_254_inc_var(i8 zeroext %x, i8 zeroext %y) {
+; CHECK-LABEL: @test_ult_254_inc_var(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ADD:%.*]] = add i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i8 [[ADD]], -2
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[CMP]], i32 35, i32 47
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+entry:
+  %add = add i8 %x, %y
+  %cmp = icmp ult i8 %add, 254
+  %res = select i1 %cmp, i32 35, i32 47
+  ret i32 %res
+}
+
+define i32 @test_sle_254_inc_var(i8 %x, i8 %y) {
+; CHECK-LABEL: @test_sle_254_inc_var(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ADD:%.*]] = add i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i8 [[ADD]], -2
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[CMP]], i32 35, i32 47
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+entry:
+  %add = add i8 %x, %y
+  %cmp = icmp sle i8 %add, 254
+  %res = select i1 %cmp, i32 35, i32 47
+  ret i32 %res
+}
+
+define i32 @test_ugt_1_dec_imm(i8 zeroext %x) {
+; CHECK-LABEL: @test_ugt_1_dec_imm(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = zext i8 [[X:%.*]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i32 [[TMP0]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[TMP1]], 1
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[CMP]], i32 35, i32 47
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+entry:
+  %add = add i8 %x, -1
+  %cmp = icmp ugt i8 %add, 1
+  %res = select i1 %cmp, i32 35, i32 47
+  ret i32 %res
+}
+
+define i32 @test_sgt_1_dec_imm(i8 %x) {
+; CHECK-LABEL: @test_sgt_1_dec_imm(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ADD:%.*]] = add i8 [[X:%.*]], -1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i8 [[ADD]], 1
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[CMP]], i32 35, i32 47
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+entry:
+  %add = add i8 %x, -1
+  %cmp = icmp sgt i8 %add, 1
+  %res = select i1 %cmp, i32 35, i32 47
+  ret i32 %res
+}
+
+define i32 @test_ugt_1_dec_var(i8 zeroext %x, i8 zeroext %y) {
+; CHECK-LABEL: @test_ugt_1_dec_var(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SUB:%.*]] = sub i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i8 [[SUB]], 1
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[CMP]], i32 35, i32 47
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+entry:
+  %sub = sub i8 %x, %y
+  %cmp = icmp ugt i8 %sub, 1
+  %res = select i1 %cmp, i32 35, i32 47
+  ret i32 %res
+}
+
+define i32 @test_sge_1_dec_var(i8 %x, i8 %y) {
+; CHECK-LABEL: @test_sge_1_dec_var(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SUB:%.*]] = sub i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i8 [[SUB]], 1
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[CMP]], i32 35, i32 47
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+entry:
+  %sub = sub i8 %x, %y
+  %cmp = icmp sge i8 %sub, 1
+  %res = select i1 %cmp, i32 35, i32 47
+  ret i32 %res
+}
+
+define i32 @dsp_imm1(i8 zeroext %x, i8 zeroext %y) {
+; CHECK-LABEL: @dsp_imm1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[XOR:%.*]] = xor i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[AND:%.*]] = and i8 [[X]], 7
+; CHECK-NEXT:    [[SUB:%.*]] = sub i8 [[AND]], [[XOR]]
+; CHECK-NEXT:    [[ADD:%.*]] = add i8 [[SUB]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i8 [[ADD]], -2
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[CMP]], i32 35, i32 47
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+entry:
+  %xor = xor i8 %x, %y
+  %and = and i8 %x, 7
+  %sub = sub i8 %and, %xor
+  %add = add i8 %sub, 1
+  %cmp = icmp ult i8 %add, 254
+  %res = select i1 %cmp, i32 35, i32 47
+  ret i32 %res
+}
+
+define i32 @dsp_var(i8 zeroext %x, i8 zeroext %y) {
+; CHECK-LABEL: @dsp_var(
+; CHECK-NEXT:    [[XOR:%.*]] = xor i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[AND:%.*]] = and i8 [[X]], 7
+; CHECK-NEXT:    [[SUB:%.*]] = sub i8 [[AND]], [[XOR]]
+; CHECK-NEXT:    [[MUL:%.*]] = shl nuw i8 [[X]], 1
+; CHECK-NEXT:    [[ADD:%.*]] = add i8 [[SUB]], [[MUL]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i8 [[ADD]], -2
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[CMP]], i32 35, i32 47
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %xor = xor i8 %x, %y
+  %and = and i8 %x, 7
+  %sub = sub i8 %and, %xor
+  %mul = shl nuw i8 %x, 1
+  %add = add i8 %sub, %mul
+  %cmp = icmp ult i8 %add, 254
+  %res = select i1 %cmp, i32 35, i32 47
+  ret i32 %res
+}
+
+define void @store_dsp_res(i8* %in, i8* %out, i8 %compare) {
+; CHECK-LABEL: @store_dsp_res(
+; CHECK-NEXT:    [[FIRST:%.*]] = getelementptr inbounds i8, i8* [[IN:%.*]], i32 0
+; CHECK-NEXT:    [[SECOND:%.*]] = getelementptr inbounds i8, i8* [[IN]], i32 1
+; CHECK-NEXT:    [[LD0:%.*]] = load i8, i8* [[FIRST]]
+; CHECK-NEXT:    [[LD1:%.*]] = load i8, i8* [[SECOND]]
+; CHECK-NEXT:    [[XOR:%.*]] = xor i8 [[LD0]], -1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i8 [[COMPARE:%.*]], [[LD1]]
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i8 [[COMPARE]], i8 [[XOR]]
+; CHECK-NEXT:    [[SUB:%.*]] = sub i8 [[LD0]], [[SELECT]]
+; CHECK-NEXT:    store i8 [[SUB]], i8* [[OUT:%.*]], align 1
+; CHECK-NEXT:    ret void
+;
+  %first = getelementptr inbounds i8, i8* %in, i32 0
+  %second = getelementptr inbounds i8, i8* %in, i32 1
+  %ld0 = load i8, i8* %first
+  %ld1 = load i8, i8* %second
+  %xor = xor i8 %ld0, -1
+  %cmp = icmp ult i8 %compare, %ld1
+  %select = select i1 %cmp, i8 %compare, i8 %xor
+  %sub = sub i8 %ld0, %select
+  store i8 %sub, i8* %out, align 1
+  ret void
+}
+
+define i32 @ugt_1_dec_imm(i8 zeroext %x) {
+; CHECK-LABEL: @ugt_1_dec_imm(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = zext i8 [[X:%.*]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i32 [[TMP0]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[TMP1]], 1
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[CMP]], i32 35, i32 47
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+entry:
+  %add = add i8 %x, -1
+  %cmp = icmp ugt i8 %add, 1
+  %res = select i1 %cmp, i32 35, i32 47
+  ret i32 %res
+}
+
+define i32 @ugt_1_dec_var(i8 zeroext %x, i8 zeroext %y) {
+; CHECK-LABEL: @ugt_1_dec_var(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SUB:%.*]] = sub i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i8 [[SUB]], 1
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[CMP]], i32 35, i32 47
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+entry:
+  %sub = sub i8 %x, %y
+  %cmp = icmp ugt i8 %sub, 1
+  %res = select i1 %cmp, i32 35, i32 47
+  ret i32 %res
+}
+
+define i32 @icmp_eq_minus_one(i8* %ptr) {
+; CHECK-LABEL: @icmp_eq_minus_one(
+; CHECK-NEXT:    [[LOAD:%.*]] = load i8, i8* [[PTR:%.*]], align 1
+; CHECK-NEXT:    [[CONV:%.*]] = zext i8 [[LOAD]] to i32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[LOAD]], -1
+; CHECK-NEXT:    [[RET:%.*]] = select i1 [[CMP]], i32 [[CONV]], i32 -1
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %load = load i8, i8* %ptr, align 1
+  %conv = zext i8 %load to i32
+  %cmp = icmp eq i8 %load, -1
+  %ret = select i1 %cmp, i32 %conv, i32 -1
+  ret i32 %ret
+}
+
+define i32 @icmp_not(i16 zeroext %arg0, i16 zeroext %arg1) {
+; CHECK-LABEL: @icmp_not(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[ARG0:%.*]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i16 [[ARG1:%.*]] to i32
+; CHECK-NEXT:    [[NOT:%.*]] = xor i32 [[TMP1]], 65535
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[NOT]], [[TMP2]]
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[CMP]], i32 16, i32 32
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %not = xor i16 %arg0, -1
+  %cmp = icmp eq i16 %not, %arg1
+  %res = select i1 %cmp, i32 16, i32 32
+  ret i32 %res
+}
+
+define i32 @icmp_i1(i1* %arg0, i1 zeroext %arg1, i32 %a, i32 %b) {
+; CHECK-LABEL: @icmp_i1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LOAD:%.*]] = load i1, i1* [[ARG0:%.*]]
+; CHECK-NEXT:    [[NOT:%.*]] = xor i1 [[LOAD]], true
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i1 [[ARG1:%.*]], [[NOT]]
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[CMP]], i32 [[A:%.*]], i32 [[B:%.*]]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+entry:
+  %load = load i1, i1* %arg0
+  %not = xor i1 %load, 1
+  %cmp = icmp eq i1 %arg1, %not
+  %res = select i1 %cmp, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @icmp_i7(i7* %arg0, i7 zeroext %arg1, i32 %a, i32 %b) {
+; CHECK-LABEL: @icmp_i7(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = zext i7 [[ARG1:%.*]] to i32
+; CHECK-NEXT:    [[LOAD:%.*]] = load i7, i7* [[ARG0:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i7 [[LOAD]] to i32
+; CHECK-NEXT:    [[ADD:%.*]] = add nuw i32 [[TMP1]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[TMP0]], [[ADD]]
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[CMP]], i32 [[A:%.*]], i32 [[B:%.*]]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+entry:
+  %load = load i7, i7* %arg0
+  %add = add nuw i7 %load, 1
+  %cmp = icmp ult i7 %arg1, %add
+  %res = select i1 %cmp, i32 %a, i32 %b
+  ret i32 %res
+}
+
+define i32 @icmp_i15(i15 zeroext %arg0, i15 zeroext %arg1) {
+; CHECK-LABEL: @icmp_i15(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i15 [[ARG0:%.*]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i15 [[ARG1:%.*]] to i32
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[TMP1]], 32767
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[XOR]], [[TMP2]]
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[CMP]], i32 21, i32 42
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %xor = xor i15 %arg0, -1
+  %cmp = icmp eq i15 %xor, %arg1
+  %res = select i1 %cmp, i32 21, i32 42
+  ret i32 %res
+}
+
+define i32 @icmp_minus_imm(i8* %a) {
+; CHECK-LABEL: @icmp_minus_imm(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, i8* [[A:%.*]], align 1
+; CHECK-NEXT:    [[ADD_I:%.*]] = add i8 [[TMP0]], -7
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i8 [[ADD_I]], -5
+; CHECK-NEXT:    [[CONV1:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    ret i32 [[CONV1]]
+;
+entry:
+  %0 = load i8, i8* %a, align 1
+  %add.i = add i8 %0, -7
+  %cmp = icmp ugt i8 %add.i, -5
+  %conv1 = zext i1 %cmp to i32
+  ret i32 %conv1
+}
+
+define void @mul_with_neg_imm(i32, i32* %b) {
+; CHECK-LABEL: @mul_with_neg_imm(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[TMP0:%.*]] to i8
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i8 [[TMP1]] to i32
+; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[TMP2]], 1
+; CHECK-NEXT:    [[CONV_I:%.*]] = mul nuw i32 [[TMP3]], 132
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[CONV_I]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL]], label [[IF_END:%.*]], label [[IF_THEN:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    store i32 0, i32* [[B:%.*]], align 4
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %1 = trunc i32 %0 to i8
+  %2 = and i8 %1, 1
+  %conv.i = mul nuw i8 %2, -124
+  %tobool = icmp eq i8 %conv.i, 0
+  br i1 %tobool, label %if.end, label %if.then
+
+if.then:
+  store i32 0, i32* %b, align 4
+  br label %if.end
+
+if.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/TypePromotion/ARM/large-int.ll b/llvm/test/Transforms/TypePromotion/ARM/large-int.ll
new file mode 100644
index 000000000..156eda086
--- /dev/null
+++ b/llvm/test/Transforms/TypePromotion/ARM/large-int.ll
@@ -0,0 +1,66 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -mtriple=thumbv7 -type-promotion -disable-type-promotion=false -S %s -o - | FileCheck %s
+
+define hidden void @dont_promote_large_int(i8* %in, i64* %out) {
+; CHECK-LABEL: @dont_promote_large_int(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CAST:%.*]] = bitcast i8* [[IN:%.*]] to i144*
+; CHECK-NEXT:    [[LOAD:%.*]] = load i144, i144* [[CAST]], align 4
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc i144 [[LOAD]] to i98
+; CHECK-NEXT:    [[AND:%.*]] = and i98 [[TRUNC]], -79228162514264337593543950336
+; CHECK-NEXT:    [[TRUNC_1:%.*]] = trunc i98 [[TRUNC]] to i32
+; CHECK-NEXT:    switch i98 [[AND]], label [[BB_3:%.*]] [
+; CHECK-NEXT:    i98 79228162514264337593543950336, label [[BB_1:%.*]]
+; CHECK-NEXT:    i98 0, label [[BB_2:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       bb.1:
+; CHECK-NEXT:    [[AND_1:%.*]] = and i144 [[LOAD]], 237684487542793012780631851008
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i144 [[AND_1]], 0
+; CHECK-NEXT:    [[INDEX:%.*]] = select i1 [[CMP]], i32 0, i32 4
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, i64* [[OUT:%.*]], i32 [[INDEX]]
+; CHECK-NEXT:    [[MUL:%.*]] = mul nuw i32 [[TRUNC_1]], 27
+; CHECK-NEXT:    [[SUB:%.*]] = sub nuw i32 0, [[MUL]]
+; CHECK-NEXT:    [[CAST_1:%.*]] = zext i32 [[SUB]] to i64
+; CHECK-NEXT:    store i64 [[CAST_1]], i64* [[OUT]], align 4
+; CHECK-NEXT:    br label [[BB_3]]
+; CHECK:       bb.2:
+; CHECK-NEXT:    [[CAST_2:%.*]] = zext i32 [[TRUNC_1]] to i64
+; CHECK-NEXT:    [[SHL:%.*]] = shl nuw i64 [[CAST_2]], 32
+; CHECK-NEXT:    [[OR:%.*]] = or i64 [[CAST_2]], [[SHL]]
+; CHECK-NEXT:    store i64 [[OR]], i64* [[OUT]], align 4
+; CHECK-NEXT:    br label [[BB_3]]
+; CHECK:       bb.3:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cast = bitcast i8* %in to i144*
+  %load = load i144, i144* %cast, align 4
+  %trunc = trunc i144 %load to i98
+  %and = and i98 %trunc, -79228162514264337593543950336
+  %trunc.1 = trunc i98 %trunc to i32
+  switch i98 %and, label %bb.3 [
+  i98 79228162514264337593543950336, label %bb.1
+  i98 0, label %bb.2
+  ]
+
+bb.1:
+  %and.1 = and i144 %load, 237684487542793012780631851008
+  %cmp = icmp eq i144 %and.1, 0
+  %index = select i1 %cmp, i32 0, i32 4
+  %gep = getelementptr i64, i64* %out, i32 %index
+  %mul = mul nuw i32 %trunc.1, 27
+  %sub = sub nuw i32 0, %mul
+  %cast.1 = zext i32 %sub to i64
+  store i64 %cast.1, i64* %out, align 4
+  br label %bb.3
+
+bb.2:
+  %cast.2 = zext i32 %trunc.1 to i64
+  %shl = shl nuw i64 %cast.2, 32
+  %or = or i64 %cast.2, %shl
+  store i64 %or, i64* %out, align 4
+  br label %bb.3
+
+bb.3:
+  ret void
+}
diff --git a/llvm/test/Transforms/TypePromotion/ARM/phis-ret.ll b/llvm/test/Transforms/TypePromotion/ARM/phis-ret.ll
new file mode 100644
index 000000000..e79e4ff1b
--- /dev/null
+++ b/llvm/test/Transforms/TypePromotion/ARM/phis-ret.ll
@@ -0,0 +1,344 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -mtriple=arm -type-promotion -verify -disable-type-promotion=false -S %s -o - | FileCheck %s
+
+; Check that the arguments are extended but then nothing else is.
+; This also ensures that the pass can handle loops.
+define void @phi_feeding_phi_args(i8 %a, i8 %b) {
+; CHECK-LABEL: @phi_feeding_phi_args(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = zext i8 [[A:%.*]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[B:%.*]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ugt i32 [[TMP0]], [[TMP1]]
+; CHECK-NEXT:    br i1 [[TMP2]], label [[PREHEADER:%.*]], label [[EMPTY:%.*]]
+; CHECK:       empty:
+; CHECK-NEXT:    br label [[PREHEADER]]
+; CHECK:       preheader:
+; CHECK-NEXT:    [[TMP3:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[TMP1]], [[EMPTY]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[VAL:%.*]] = phi i32 [ [[TMP3]], [[PREHEADER]] ], [ [[INC2:%.*]], [[IF_END:%.*]] ]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[VAL]], 254
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[INC:%.*]] = sub nuw i32 [[VAL]], 2
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.else:
+; CHECK-NEXT:    [[INC1:%.*]] = shl nuw i32 [[VAL]], 1
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    [[INC2]] = phi i32 [ [[INC]], [[IF_THEN]] ], [ [[INC1]], [[IF_ELSE]] ]
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i32 [[INC2]], 255
+; CHECK-NEXT:    br i1 [[CMP1]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %0 = icmp ugt i8 %a, %b
+  br i1 %0, label %preheader, label %empty
+
+empty:
+  br label %preheader
+
+preheader:
+  %1 = phi i8 [ %a, %entry ], [ %b, %empty ]
+  br label %loop
+
+loop:
+  %val = phi i8 [ %1, %preheader ], [ %inc2, %if.end ]
+  %cmp = icmp ult i8 %val, 254
+  br i1 %cmp, label %if.then, label %if.else
+
+if.then:
+  %inc = sub nuw i8 %val, 2
+  br label %if.end
+
+if.else:
+  %inc1 = shl nuw i8 %val, 1
+  br label %if.end
+
+if.end:
+  %inc2 = phi i8 [ %inc, %if.then], [ %inc1, %if.else ]
+  %cmp1 = icmp eq i8 %inc2, 255
+  br i1 %cmp1, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+; Same as above, but as the args are zeroext, we shouldn't see any uxts.
+define void @phi_feeding_phi_zeroext_args(i8 zeroext %a, i8 zeroext %b) {
+; CHECK-LABEL: @phi_feeding_phi_zeroext_args(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = zext i8 [[A:%.*]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[B:%.*]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ugt i32 [[TMP0]], [[TMP1]]
+; CHECK-NEXT:    br i1 [[TMP2]], label [[PREHEADER:%.*]], label [[EMPTY:%.*]]
+; CHECK:       empty:
+; CHECK-NEXT:    br label [[PREHEADER]]
+; CHECK:       preheader:
+; CHECK-NEXT:    [[TMP3:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[TMP1]], [[EMPTY]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[VAL:%.*]] = phi i32 [ [[TMP3]], [[PREHEADER]] ], [ [[INC2:%.*]], [[IF_END:%.*]] ]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[VAL]], 254
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[INC:%.*]] = sub nuw i32 [[VAL]], 2
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.else:
+; CHECK-NEXT:    [[INC1:%.*]] = shl nuw i32 [[VAL]], 1
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    [[INC2]] = phi i32 [ [[INC]], [[IF_THEN]] ], [ [[INC1]], [[IF_ELSE]] ]
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i32 [[INC2]], 255
+; CHECK-NEXT:    br i1 [[CMP1]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %0 = icmp ugt i8 %a, %b
+  br i1 %0, label %preheader, label %empty
+
+empty:
+  br label %preheader
+
+preheader:
+  %1 = phi i8 [ %a, %entry ], [ %b, %empty ]
+  br label %loop
+
+loop:
+  %val = phi i8 [ %1, %preheader ], [ %inc2, %if.end ]
+  %cmp = icmp ult i8 %val, 254
+  br i1 %cmp, label %if.then, label %if.else
+
+if.then:
+  %inc = sub nuw i8 %val, 2
+  br label %if.end
+
+if.else:
+  %inc1 = shl nuw i8 %val, 1
+  br label %if.end
+
+if.end:
+  %inc2 = phi i8 [ %inc, %if.then], [ %inc1, %if.else ]
+  %cmp1 = icmp eq i8 %inc2, 255
+  br i1 %cmp1, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+; Just check that phis also work with i16s.
+define void @phi_i16() {
+; CHECK-LABEL: @phi_i16(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[VAL:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC2:%.*]], [[IF_END:%.*]] ]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[VAL]], 128
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[INC:%.*]] = add nuw i32 [[VAL]], 2
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.else:
+; CHECK-NEXT:    [[INC1:%.*]] = add nuw i32 [[VAL]], 1
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    [[INC2]] = phi i32 [ [[INC]], [[IF_THEN]] ], [ [[INC1]], [[IF_ELSE]] ]
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult i32 [[INC2]], 253
+; CHECK-NEXT:    br i1 [[CMP1]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %val = phi i16 [ 0, %entry ], [ %inc2, %if.end ]
+  %cmp = icmp ult i16 %val, 128
+  br i1 %cmp, label %if.then, label %if.else
+
+if.then:
+  %inc = add nuw i16 %val, 2
+  br label %if.end
+
+if.else:
+  %inc1 = add nuw i16 %val, 1
+  br label %if.end
+
+if.end:
+  %inc2 = phi i16 [ %inc, %if.then], [ %inc1, %if.else ]
+  %cmp1 = icmp ult i16 %inc2, 253
+  br i1 %cmp1, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+define i8 @ret_i8() {
+; CHECK-LABEL: @ret_i8(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[VAL:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC2:%.*]], [[IF_END:%.*]] ]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[VAL]], 128
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[INC:%.*]] = add nuw i32 [[VAL]], 2
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.else:
+; CHECK-NEXT:    [[INC1:%.*]] = add nuw i32 [[VAL]], 1
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    [[INC2]] = phi i32 [ [[INC]], [[IF_THEN]] ], [ [[INC1]], [[IF_ELSE]] ]
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult i32 [[INC2]], 253
+; CHECK-NEXT:    br i1 [[CMP1]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[TMP0:%.*]] = trunc i32 [[INC2]] to i8
+; CHECK-NEXT:    ret i8 [[TMP0]]
+;
+entry:
+  br label %loop
+
+loop:
+  %val = phi i8 [ 0, %entry ], [ %inc2, %if.end ]
+  %cmp = icmp ult i8 %val, 128
+  br i1 %cmp, label %if.then, label %if.else
+
+if.then:
+  %inc = add nuw i8 %val, 2
+  br label %if.end
+
+if.else:
+  %inc1 = add nuw i8 %val, 1
+  br label %if.end
+
+if.end:
+  %inc2 = phi i8 [ %inc, %if.then], [ %inc1, %if.else ]
+  %cmp1 = icmp ult i8 %inc2, 253
+  br i1 %cmp1, label %exit, label %loop
+
+exit:
+  ret i8 %inc2
+}
+
+define i16 @phi_multiple_undefs(i16 zeroext %arg) {
+; CHECK-LABEL: @phi_multiple_undefs(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[VAL:%.*]] = phi i32 [ undef, [[ENTRY:%.*]] ], [ [[INC2:%.*]], [[IF_END:%.*]] ]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[VAL]], 128
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[INC:%.*]] = add nuw i32 [[VAL]], 2
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.else:
+; CHECK-NEXT:    [[INC1:%.*]] = add nuw i32 [[VAL]], 1
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    [[INC2]] = phi i32 [ [[INC]], [[IF_THEN]] ], [ [[INC1]], [[IF_ELSE]] ]
+; CHECK-NEXT:    [[UNRELATED:%.*]] = phi i16 [ undef, [[IF_THEN]] ], [ [[ARG:%.*]], [[IF_ELSE]] ]
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult i32 [[INC2]], 253
+; CHECK-NEXT:    br i1 [[CMP1]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i16 [[UNRELATED]]
+;
+entry:
+  br label %loop
+
+loop:
+  %val = phi i16 [ undef, %entry ], [ %inc2, %if.end ]
+  %cmp = icmp ult i16 %val, 128
+  br i1 %cmp, label %if.then, label %if.else
+
+if.then:
+  %inc = add nuw i16 %val, 2
+  br label %if.end
+
+if.else:
+  %inc1 = add nuw i16 %val, 1
+  br label %if.end
+
+if.end:
+  %inc2 = phi i16 [ %inc, %if.then], [ %inc1, %if.else ]
+  %unrelated = phi i16 [ undef, %if.then ], [ %arg, %if.else ]
+  %cmp1 = icmp ult i16 %inc2, 253
+  br i1 %cmp1, label %loop, label %exit
+
+exit:
+  ret i16 %unrelated
+}
+
+define i16 @promote_arg_return(i16 zeroext %arg1, i16 zeroext %arg2, i8* %res) {
+; CHECK-LABEL: @promote_arg_return(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[ARG1:%.*]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i16 [[ARG2:%.*]] to i32
+; CHECK-NEXT:    [[ADD:%.*]] = add nuw i32 [[TMP1]], 15
+; CHECK-NEXT:    [[MUL:%.*]] = mul nuw nsw i32 [[ADD]], 3
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[MUL]], [[TMP2]]
+; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i8
+; CHECK-NEXT:    store i8 [[CONV]], i8* [[RES:%.*]]
+; CHECK-NEXT:    [[TMP3:%.*]] = trunc i32 [[TMP1]] to i16
+; CHECK-NEXT:    ret i16 [[TMP3]]
+;
+  %add = add nuw i16 %arg1, 15
+  %mul = mul nuw nsw i16 %add, 3
+  %cmp = icmp ult i16 %mul, %arg2
+  %conv = zext i1 %cmp to i8
+  store i8 %conv, i8* %res
+  ret i16 %arg1
+}
+
+define i16 @signext_bitcast_phi_select(i16 signext %start, i16* %in) {
+; CHECK-LABEL: @signext_bitcast_phi_select(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[START:%.*]] to i32
+; CHECK-NEXT:    [[CONST:%.*]] = bitcast i16 -1 to i16
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[IDX:%.*]] = phi i32 [ [[SELECT:%.*]], [[IF_ELSE:%.*]] ], [ [[TMP0]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[IDX]] to i16
+; CHECK-NEXT:    [[CMP_I:%.*]] = icmp sgt i16 [[TMP1]], [[CONST]]
+; CHECK-NEXT:    br i1 [[CMP_I]], label [[EXIT:%.*]], label [[IF_THEN:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[IDX_NEXT:%.*]] = getelementptr i16, i16* [[IN:%.*]], i32 [[IDX]]
+; CHECK-NEXT:    [[LD:%.*]] = load i16, i16* [[IDX_NEXT]], align 2
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i16 [[LD]] to i32
+; CHECK-NEXT:    [[CMP1_I:%.*]] = icmp eq i32 [[TMP2]], [[IDX]]
+; CHECK-NEXT:    br i1 [[CMP1_I]], label [[EXIT]], label [[IF_ELSE]]
+; CHECK:       if.else:
+; CHECK-NEXT:    [[LOBIT:%.*]] = lshr i32 [[IDX]], 15
+; CHECK-NEXT:    [[LOBIT_NOT:%.*]] = xor i32 [[LOBIT]], 1
+; CHECK-NEXT:    [[SELECT]] = add nuw i32 [[LOBIT_NOT]], [[IDX]]
+; CHECK-NEXT:    br label [[FOR_BODY]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RES:%.*]] = phi i32 [ [[TMP2]], [[IF_THEN]] ], [ 0, [[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = trunc i32 [[RES]] to i16
+; CHECK-NEXT:    ret i16 [[TMP3]]
+;
+entry:
+  %const = bitcast i16 -1 to i16
+  br label %for.body
+
+for.body:
+  %idx = phi i16 [ %select, %if.else ], [ %start, %entry ]
+  %cmp.i = icmp sgt i16 %idx, %const
+  br i1 %cmp.i, label %exit, label %if.then
+
+if.then:
+  %idx.next = getelementptr i16, i16* %in, i16 %idx
+  %ld = load i16, i16* %idx.next, align 2
+  %cmp1.i = icmp eq i16 %ld, %idx
+  br i1 %cmp1.i, label %exit, label %if.else
+
+if.else:
+  %lobit = lshr i16 %idx, 15
+  %lobit.not = xor i16 %lobit, 1
+  %select = add nuw i16 %lobit.not, %idx
+  br label %for.body
+
+exit:
+  %res = phi i16 [ %ld, %if.then ], [ 0, %for.body ]
+  ret i16 %res
+}
diff --git a/llvm/test/Transforms/TypePromotion/ARM/pointers.ll b/llvm/test/Transforms/TypePromotion/ARM/pointers.ll
new file mode 100644
index 000000000..3c5f097b1
--- /dev/null
+++ b/llvm/test/Transforms/TypePromotion/ARM/pointers.ll
@@ -0,0 +1,240 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -mtriple=arm -type-promotion -verify -disable-type-promotion=false -S %s -o - | FileCheck %s
+
+define void @phi_pointers(i16* %a, i16* %b, i8 zeroext %M, i8 zeroext %N) {
+; CHECK-LABEL: @phi_pointers(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = zext i8 [[M:%.*]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[N:%.*]] to i32
+; CHECK-NEXT:    [[ADD:%.*]] = add nuw i32 [[TMP0]], 1
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[ADD]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[ADD]], [[TMP1]]
+; CHECK-NEXT:    [[BASE:%.*]] = select i1 [[CMP]], i16* [[A:%.*]], i16* [[B:%.*]]
+; CHECK-NEXT:    [[OTHER:%.*]] = select i1 [[CMP]], i16* [[B]], i16* [[B]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[PTR:%.*]] = phi i16* [ [[BASE]], [[ENTRY:%.*]] ], [ [[GEP:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IDX:%.*]] = phi i32 [ [[AND]], [[ENTRY]] ], [ [[INC:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[LOAD:%.*]] = load i16, i16* [[PTR]], align 2
+; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[IDX]], 1
+; CHECK-NEXT:    [[GEP]] = getelementptr inbounds i16, i16* [[PTR]], i32 [[INC]]
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i16* [[GEP]], [[OTHER]]
+; CHECK-NEXT:    br i1 [[COND]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %add = add nuw i8 %M, 1
+  %and = and i8 %add, 1
+  %cmp = icmp ugt i8 %add, %N
+  %base = select i1 %cmp, i16* %a, i16* %b
+  %other = select i1 %cmp, i16* %b, i16* %b
+  br label %loop
+
+loop:
+  %ptr = phi i16* [ %base, %entry ], [ %gep, %loop ]
+  %idx = phi i8 [ %and, %entry ], [ %inc, %loop ]
+  %load = load i16, i16* %ptr, align 2
+  %inc = add nuw nsw i8 %idx, 1
+  %gep = getelementptr inbounds i16, i16* %ptr, i8 %inc
+  %cond = icmp eq i16* %gep, %other
+  br i1 %cond, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @phi_pointers_null(i16* %a, i16* %b, i8 zeroext %M, i8 zeroext %N) {
+; CHECK-LABEL: @phi_pointers_null(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = zext i8 [[M:%.*]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[N:%.*]] to i32
+; CHECK-NEXT:    [[ADD:%.*]] = add nuw i32 [[TMP0]], 1
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[ADD]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[ADD]], [[TMP1]]
+; CHECK-NEXT:    [[BASE:%.*]] = select i1 [[CMP]], i16* [[A:%.*]], i16* [[B:%.*]]
+; CHECK-NEXT:    [[OTHER:%.*]] = select i1 [[CMP]], i16* [[B]], i16* [[B]]
+; CHECK-NEXT:    [[CMP_1:%.*]] = icmp eq i16* [[BASE]], [[OTHER]]
+; CHECK-NEXT:    br i1 [[CMP_1]], label [[FAIL:%.*]], label [[LOOP:%.*]]
+; CHECK:       fail:
+; CHECK-NEXT:    br label [[LOOP]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[PTR:%.*]] = phi i16* [ [[BASE]], [[ENTRY:%.*]] ], [ null, [[FAIL]] ], [ [[GEP:%.*]], [[IF_THEN:%.*]] ]
+; CHECK-NEXT:    [[IDX:%.*]] = phi i32 [ [[AND]], [[ENTRY]] ], [ 0, [[FAIL]] ], [ [[INC:%.*]], [[IF_THEN]] ]
+; CHECK-NEXT:    [[UNDEF:%.*]] = icmp eq i16* [[PTR]], undef
+; CHECK-NEXT:    br i1 [[UNDEF]], label [[EXIT:%.*]], label [[IF_THEN]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[LOAD:%.*]] = load i16, i16* [[PTR]], align 2
+; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[IDX]], 1
+; CHECK-NEXT:    [[GEP]] = getelementptr inbounds i16, i16* [[PTR]], i32 [[INC]]
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i16* [[GEP]], [[OTHER]]
+; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %add = add nuw i8 %M, 1
+  %and = and i8 %add, 1
+  %cmp = icmp ugt i8 %add, %N
+  %base = select i1 %cmp, i16* %a, i16* %b
+  %other = select i1 %cmp, i16* %b, i16* %b
+  %cmp.1 = icmp eq i16* %base, %other
+  br i1 %cmp.1, label %fail, label %loop
+
+fail:
+  br label %loop
+
+loop:
+  %ptr = phi i16* [ %base, %entry ], [ null, %fail ], [ %gep, %if.then ]
+  %idx = phi i8 [ %and, %entry ], [ 0, %fail ], [ %inc, %if.then ]
+  %undef = icmp eq i16* %ptr, undef
+  br i1 %undef, label %exit, label %if.then
+
+if.then:
+  %load = load i16, i16* %ptr, align 2
+  %inc = add nuw nsw i8 %idx, 1
+  %gep = getelementptr inbounds i16, i16* %ptr, i8 %inc
+  %cond = icmp eq i16* %gep, %other
+  br i1 %cond, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+declare i8 @do_something_with_ptr(i8, i16*)
+
+define i8 @call_pointer(i8 zeroext %x, i8 zeroext %y, i16* %a, i16* %b) {
+; CHECK-LABEL: @call_pointer(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[X:%.*]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i8 [[Y:%.*]] to i32
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[SHR:%.*]] = lshr i32 [[OR]], 1
+; CHECK-NEXT:    [[ADD:%.*]] = add nuw i32 [[SHR]], 2
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[ADD]], 0
+; CHECK-NEXT:    [[PTR:%.*]] = select i1 [[CMP]], i16* [[A:%.*]], i16* [[B:%.*]]
+; CHECK-NEXT:    [[TMP3:%.*]] = trunc i32 [[SHR]] to i8
+; CHECK-NEXT:    [[CALL:%.*]] = tail call zeroext i8 @do_something_with_ptr(i8 [[TMP3]], i16* [[PTR]])
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i8 [[CALL]] to i32
+; CHECK-NEXT:    [[TMP5:%.*]] = trunc i32 [[TMP4]] to i8
+; CHECK-NEXT:    ret i8 [[TMP5]]
+;
+  %or = or i8 %x, %y
+  %shr = lshr i8 %or, 1
+  %add = add nuw i8 %shr, 2
+  %cmp = icmp ne i8 %add, 0
+  %ptr = select i1 %cmp, i16* %a, i16* %b
+  %call = tail call zeroext i8 @do_something_with_ptr(i8 %shr, i16* %ptr)
+  ret i8 %call
+}
+
+define i16 @pointer_to_pointer(i16** %arg, i16 zeroext %limit) {
+; CHECK-LABEL: @pointer_to_pointer(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ADDR:%.*]] = load i16*, i16** [[ARG:%.*]]
+; CHECK-NEXT:    [[VAL:%.*]] = load i16, i16* [[ADDR]]
+; CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[VAL]] to i32
+; CHECK-NEXT:    [[ADD:%.*]] = add nuw i32 [[TMP0]], 7
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[ADD]], 256
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[CMP]], i16 128, i16 255
+; CHECK-NEXT:    ret i16 [[RES]]
+;
+entry:
+  %addr = load i16*, i16** %arg
+  %val = load i16, i16* %addr
+  %add = add nuw i16 %val, 7
+  %cmp = icmp ult i16 %add, 256
+  %res = select i1 %cmp, i16 128, i16 255
+  ret i16 %res
+}
+
+define i8 @gep_2d_array(i8** %a, i8 zeroext %arg) {
+; CHECK-LABEL: @gep_2d_array(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = zext i8 [[ARG:%.*]] to i32
+; CHECK-NEXT:    [[ARRAYIDX_US:%.*]] = getelementptr inbounds i8*, i8** [[A:%.*]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = load i8*, i8** [[ARRAYIDX_US]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, i8* [[TMP1]], align 1
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i8 [[TMP2]] to i32
+; CHECK-NEXT:    [[SUB:%.*]] = sub nuw i32 [[TMP3]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[SUB]], [[TMP0]]
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[CMP]], i8 27, i8 54
+; CHECK-NEXT:    ret i8 [[RES]]
+;
+entry:
+  %arrayidx.us = getelementptr inbounds i8*, i8** %a, i32 0
+  %0 = load i8*, i8** %arrayidx.us, align 4
+  %1 = load i8, i8* %0, align 1
+  %sub = sub nuw i8 %1, 1
+  %cmp = icmp ult i8 %sub, %arg
+  %res = select i1 %cmp, i8 27, i8 54
+  ret i8 %res
+}
+
+define void @gep_2d_array_loop(i16** nocapture readonly %a, i16** nocapture readonly %b, i32 %N) {
+; CHECK-LABEL: @gep_2d_array_loop(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP30:%.*]] = icmp eq i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP30]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_COND1_PREHEADER_US:%.*]]
+; CHECK:       for.cond1.preheader.us:
+; CHECK-NEXT:    [[Y_031_US:%.*]] = phi i32 [ [[INC13_US:%.*]], [[FOR_COND1_FOR_COND_CLEANUP3_CRIT_EDGE_US:%.*]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[FOR_BODY4_US:%.*]]
+; CHECK:       for.body4.us:
+; CHECK-NEXT:    [[X_029_US:%.*]] = phi i32 [ 0, [[FOR_COND1_PREHEADER_US]] ], [ [[INC_US:%.*]], [[FOR_BODY4_US]] ]
+; CHECK-NEXT:    [[ARRAYIDX_US:%.*]] = getelementptr inbounds i16*, i16** [[A:%.*]], i32 [[X_029_US]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load i16*, i16** [[ARRAYIDX_US]], align 4
+; CHECK-NEXT:    [[ARRAYIDX5_US:%.*]] = getelementptr inbounds i16, i16* [[TMP0]], i32 [[Y_031_US]]
+; CHECK-NEXT:    [[TMP1:%.*]] = load i16, i16* [[ARRAYIDX5_US]], align 2
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i16 [[TMP1]] to i32
+; CHECK-NEXT:    [[DEC_US:%.*]] = add nuw i32 [[TMP2]], 65535
+; CHECK-NEXT:    [[CMP6_US:%.*]] = icmp ult i32 [[DEC_US]], 16383
+; CHECK-NEXT:    [[SHL_US:%.*]] = shl nuw i32 [[DEC_US]], 2
+; CHECK-NEXT:    [[SPEC_SELECT_US:%.*]] = select i1 [[CMP6_US]], i32 [[SHL_US]], i32 [[DEC_US]]
+; CHECK-NEXT:    [[ARRAYIDX10_US:%.*]] = getelementptr inbounds i16*, i16** [[B:%.*]], i32 [[X_029_US]]
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16*, i16** [[ARRAYIDX10_US]], align 4
+; CHECK-NEXT:    [[ARRAYIDX11_US:%.*]] = getelementptr inbounds i16, i16* [[TMP3]], i32 [[Y_031_US]]
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc i32 [[SPEC_SELECT_US]] to i16
+; CHECK-NEXT:    store i16 [[TMP4]], i16* [[ARRAYIDX11_US]], align 2
+; CHECK-NEXT:    [[INC_US]] = add nuw i32 [[X_029_US]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC_US]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND1_FOR_COND_CLEANUP3_CRIT_EDGE_US]], label [[FOR_BODY4_US]]
+; CHECK:       for.cond1.for.cond.cleanup3_crit_edge.us:
+; CHECK-NEXT:    [[INC13_US]] = add nuw i32 [[Y_031_US]], 1
+; CHECK-NEXT:    [[EXITCOND32:%.*]] = icmp eq i32 [[INC13_US]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND32]], label [[FOR_COND_CLEANUP]], label [[FOR_COND1_PREHEADER_US]]
+; CHECK:       for.cond.cleanup:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cmp30 = icmp eq i32 %N, 0
+  br i1 %cmp30, label %for.cond.cleanup, label %for.cond1.preheader.us
+
+for.cond1.preheader.us:
+  %y.031.us = phi i32 [ %inc13.us, %for.cond1.for.cond.cleanup3_crit_edge.us ], [ 0, %entry ]
+  br label %for.body4.us
+
+for.body4.us:
+  %x.029.us = phi i32 [ 0, %for.cond1.preheader.us ], [ %inc.us, %for.body4.us ]
+  %arrayidx.us = getelementptr inbounds i16*, i16** %a, i32 %x.029.us
+  %0 = load i16*, i16** %arrayidx.us, align 4
+  %arrayidx5.us = getelementptr inbounds i16, i16* %0, i32 %y.031.us
+  %1 = load i16, i16* %arrayidx5.us, align 2
+  %dec.us = add nuw i16 %1, -1
+  %cmp6.us = icmp ult i16 %dec.us, 16383
+  %shl.us = shl nuw i16 %dec.us, 2
+  %spec.select.us = select i1 %cmp6.us, i16 %shl.us, i16 %dec.us
+  %arrayidx10.us = getelementptr inbounds i16*, i16** %b, i32 %x.029.us
+  %2 = load i16*, i16** %arrayidx10.us, align 4
+  %arrayidx11.us = getelementptr inbounds i16, i16* %2, i32 %y.031.us
+  store i16 %spec.select.us, i16* %arrayidx11.us, align 2
+  %inc.us = add nuw i32 %x.029.us, 1
+  %exitcond = icmp eq i32 %inc.us, %N
+  br i1 %exitcond, label %for.cond1.for.cond.cleanup3_crit_edge.us, label %for.body4.us
+
+for.cond1.for.cond.cleanup3_crit_edge.us:
+  %inc13.us = add nuw i32 %y.031.us, 1
+  %exitcond32 = icmp eq i32 %inc13.us, %N
+  br i1 %exitcond32, label %for.cond.cleanup, label %for.cond1.preheader.us
+
+for.cond.cleanup:
+  ret void
+}
diff --git a/llvm/test/Transforms/TypePromotion/ARM/signed-icmps.ll b/llvm/test/Transforms/TypePromotion/ARM/signed-icmps.ll
new file mode 100644
index 000000000..dfdd4c10a
--- /dev/null
+++ b/llvm/test/Transforms/TypePromotion/ARM/signed-icmps.ll
@@ -0,0 +1,103 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -mtriple=arm -type-promotion -verify -disable-type-promotion=false -S %s -o - | FileCheck %s
+
+define i8 @eq_sgt(i8* %x, i8 *%y, i8 zeroext %z) {
+; CHECK-LABEL: @eq_sgt(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LOAD0:%.*]] = load i8, i8* [[X:%.*]], align 1
+; CHECK-NEXT:    [[LOAD1:%.*]] = load i8, i8* [[Y:%.*]], align 1
+; CHECK-NEXT:    [[ADD:%.*]] = add i8 [[LOAD0]], [[Z:%.*]]
+; CHECK-NEXT:    [[SUB:%.*]] = sub i8 [[LOAD1]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[ADD]], -56
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i8 [[SUB]], [[ADD]]
+; CHECK-NEXT:    [[RES0:%.*]] = select i1 [[CMP]], i8 35, i8 47
+; CHECK-NEXT:    [[RES1:%.*]] = select i1 [[CMP1]], i8 [[RES0]], i8 [[SUB]]
+; CHECK-NEXT:    ret i8 [[RES1]]
+;
+entry:
+  %load0 = load i8, i8* %x, align 1
+  %load1 = load i8, i8* %y, align 1
+  %add = add i8 %load0, %z
+  %sub = sub i8 %load1, 1
+  %cmp = icmp eq i8 %add, 200
+  %cmp1 = icmp sgt i8 %sub, %add
+  %res0 = select i1 %cmp, i8 35, i8 47
+  %res1 = select i1 %cmp1, i8 %res0, i8 %sub
+  ret i8 %res1
+}
+
+define i16 @ugt_slt(i16 *%x, i16 zeroext %y, i16 zeroext %z) {
+; CHECK-LABEL: @ugt_slt(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LOAD0:%.*]] = load i16, i16* [[X:%.*]], align 1
+; CHECK-NEXT:    [[ADD:%.*]] = add i16 [[LOAD0]], [[Z:%.*]]
+; CHECK-NEXT:    [[SUB:%.*]] = sub i16 [[Y:%.*]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i16 [[ADD]], [[Z]]
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ugt i16 [[SUB]], [[Z]]
+; CHECK-NEXT:    [[RES0:%.*]] = select i1 [[CMP]], i16 35, i16 -1
+; CHECK-NEXT:    [[RES1:%.*]] = select i1 [[CMP1]], i16 [[RES0]], i16 0
+; CHECK-NEXT:    ret i16 [[RES1]]
+;
+entry:
+  %load0 = load i16, i16* %x, align 1
+  %add = add i16 %load0, %z
+  %sub = sub i16 %y, 1
+  %cmp = icmp slt i16 %add, %z
+  %cmp1 = icmp ugt i16 %sub, %z
+  %res0 = select i1 %cmp, i16 35, i16 -1
+  %res1 = select i1 %cmp1, i16 %res0, i16 0
+  ret i16 %res1
+}
+
+define void @urem_trunc_icmps(i16** %in, i32* %g, i32* %k) {
+; CHECK-LABEL: @urem_trunc_icmps(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[PTR:%.*]] = load i16*, i16** [[IN:%.*]], align 4
+; CHECK-NEXT:    [[LD:%.*]] = load i16, i16* [[PTR]], align 2
+; CHECK-NEXT:    [[CMP_I:%.*]] = icmp eq i16 [[LD]], 0
+; CHECK-NEXT:    br i1 [[CMP_I]], label [[EXIT:%.*]], label [[COND_FALSE_I:%.*]]
+; CHECK:       cond.false.i:
+; CHECK-NEXT:    [[REM:%.*]] = urem i16 5, [[LD]]
+; CHECK-NEXT:    [[EXTRACT_T:%.*]] = trunc i16 [[REM]] to i8
+; CHECK-NEXT:    br label [[BODY:%.*]]
+; CHECK:       body:
+; CHECK-NEXT:    [[COND_IN_I_OFF0:%.*]] = phi i8 [ [[EXTRACT_T]], [[COND_FALSE_I]] ], [ [[ADD:%.*]], [[FOR_INC:%.*]] ]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i8 [[COND_IN_I_OFF0]], 7
+; CHECK-NEXT:    [[CONV5:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    store i32 [[CONV5]], i32* [[G:%.*]], align 4
+; CHECK-NEXT:    [[DOTPR:%.*]] = load i32, i32* [[K:%.*]], align 4
+; CHECK-NEXT:    [[TOBOOL13150:%.*]] = icmp eq i32 [[DOTPR]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL13150]], label [[FOR_INC]], label [[EXIT]]
+; CHECK:       for.inc:
+; CHECK-NEXT:    [[ADD]] = add nuw i8 [[COND_IN_I_OFF0]], 1
+; CHECK-NEXT:    br label [[BODY]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %ptr = load i16*, i16** %in, align 4
+  %ld = load i16, i16* %ptr, align 2
+  %cmp.i = icmp eq i16 %ld, 0
+  br i1 %cmp.i, label %exit, label %cond.false.i
+
+cond.false.i:
+  %rem = urem i16 5, %ld
+  %extract.t = trunc i16 %rem to i8
+  br label %body
+
+body:
+  %cond.in.i.off0 = phi i8 [ %extract.t, %cond.false.i ], [ %add, %for.inc ]
+  %cmp = icmp sgt i8 %cond.in.i.off0, 7
+  %conv5 = zext i1 %cmp to i32
+  store i32 %conv5, i32* %g, align 4
+  %.pr = load i32, i32* %k, align 4
+  %tobool13150 = icmp eq i32 %.pr, 0
+  br i1 %tobool13150, label %for.inc, label %exit
+
+for.inc:
+  %add = add nuw i8 %cond.in.i.off0, 1
+  br label %body
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/TypePromotion/ARM/signed.ll b/llvm/test/Transforms/TypePromotion/ARM/signed.ll
new file mode 100644
index 000000000..143220a53
--- /dev/null
+++ b/llvm/test/Transforms/TypePromotion/ARM/signed.ll
@@ -0,0 +1,123 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -mtriple=arm -type-promotion -verify -disable-type-promotion=false -S %s -o - | FileCheck %s
+
+; Test to check that ARMCodeGenPrepare doesn't optimised away sign extends.
+define i16 @test_signed_load(i16* %ptr) {
+; CHECK-LABEL: @test_signed_load(
+; CHECK-NEXT:    [[LOAD:%.*]] = load i16, i16* [[PTR:%.*]]
+; CHECK-NEXT:    [[CONV0:%.*]] = zext i16 [[LOAD]] to i32
+; CHECK-NEXT:    [[CONV1:%.*]] = sext i16 [[LOAD]] to i32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[CONV0]], [[CONV1]]
+; CHECK-NEXT:    [[CONV2:%.*]] = zext i1 [[CMP]] to i16
+; CHECK-NEXT:    ret i16 [[CONV2]]
+;
+  %load = load i16, i16* %ptr
+  %conv0 = zext i16 %load to i32
+  %conv1 = sext i16 %load to i32
+  %cmp = icmp eq i32 %conv0, %conv1
+  %conv2 = zext i1 %cmp to i16
+  ret i16 %conv2
+}
+
+; Don't allow sign bit generating opcodes.
+define i16 @test_ashr(i16 zeroext %arg) {
+; CHECK-LABEL: @test_ashr(
+; CHECK-NEXT:    [[ASHR:%.*]] = ashr i16 [[ARG:%.*]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i16 [[ASHR]], 0
+; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i16
+; CHECK-NEXT:    ret i16 [[CONV]]
+;
+  %ashr = ashr i16 %arg, 1
+  %cmp = icmp eq i16 %ashr, 0
+  %conv = zext i1 %cmp to i16
+  ret i16 %conv
+}
+
+define i16 @test_sdiv(i16 zeroext %arg) {
+; CHECK-LABEL: @test_sdiv(
+; CHECK-NEXT:    [[SDIV:%.*]] = sdiv i16 [[ARG:%.*]], 2
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i16 [[SDIV]], 0
+; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i16
+; CHECK-NEXT:    ret i16 [[CONV]]
+;
+  %sdiv = sdiv i16 %arg, 2
+  %cmp = icmp ne i16 %sdiv, 0
+  %conv = zext i1 %cmp to i16
+  ret i16 %conv
+}
+
+define i16 @test_srem(i16 zeroext %arg) {
+; CHECK-LABEL: @test_srem(
+; CHECK-NEXT:    [[SREM:%.*]] = srem i16 [[ARG:%.*]], 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i16 [[SREM]], 0
+; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i16
+; CHECK-NEXT:    ret i16 [[CONV]]
+;
+  %srem = srem i16 %arg, 4
+  %cmp = icmp ne i16 %srem, 0
+  %conv = zext i1 %cmp to i16
+  ret i16 %conv
+}
+
+define i32 @test_signext_b(i8* %ptr, i8 signext %arg) {
+; CHECK-LABEL: @test_signext_b(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = zext i8 [[ARG:%.*]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = load i8, i8* [[PTR:%.*]], align 1
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i8 [[TMP1]] to i32
+; CHECK-NEXT:    [[TMP3:%.*]] = add nuw nsw i32 [[TMP2]], [[TMP0]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[TMP3]], 128
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[CMP]], i32 42, i32 20894
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+entry:
+  %0 = load i8, i8* %ptr, align 1
+  %1 = add nuw nsw i8 %0, %arg
+  %cmp = icmp ult i8 %1, 128
+  %res = select i1 %cmp, i32 42, i32 20894
+  ret i32 %res
+}
+
+define i32 @test_signext_b_ult_slt(i8* %ptr, i8 signext %arg) {
+; CHECK-LABEL: @test_signext_b_ult_slt(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = zext i8 [[ARG:%.*]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = load i8, i8* [[PTR:%.*]], align 1
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i8 [[TMP1]] to i32
+; CHECK-NEXT:    [[TMP3:%.*]] = add nuw nsw i32 [[TMP2]], [[TMP0]]
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc i32 [[TMP3]] to i8
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i8 [[TMP4]], 126
+; CHECK-NEXT:    [[CMP_1:%.*]] = icmp ule i32 [[TMP3]], [[TMP0]]
+; CHECK-NEXT:    [[OR:%.*]] = and i1 [[CMP]], [[CMP_1]]
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[OR]], i32 42, i32 57
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+entry:
+  %0 = load i8, i8* %ptr, align 1
+  %1 = add nuw nsw i8 %0, %arg
+  %cmp = icmp sle i8 %1, 126
+  %cmp.1 = icmp ule i8 %1, %arg
+  %or = and i1 %cmp, %cmp.1
+  %res = select i1 %or, i32 42, i32 57
+  ret i32 %res
+}
+
+define i32 @test_signext_h(i16* %ptr, i16 signext %arg) {
+; CHECK-LABEL: @test_signext_h(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[ARG:%.*]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = load i16, i16* [[PTR:%.*]], align 1
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i16 [[TMP1]] to i32
+; CHECK-NEXT:    [[TMP3:%.*]] = add nuw nsw i32 [[TMP2]], [[TMP0]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[TMP3]], 32768
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[CMP]], i32 42, i32 20894
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+entry:
+  %0 = load i16, i16* %ptr, align 1
+  %1 = add nuw nsw i16 %0, %arg
+  %cmp = icmp ult i16 %1, 32768
+  %res = select i1 %cmp, i32 42, i32 20894
+  ret i32 %res
+}
+
diff --git a/llvm/test/Transforms/TypePromotion/ARM/switch.ll b/llvm/test/Transforms/TypePromotion/ARM/switch.ll
new file mode 100644
index 000000000..6736ebeea
--- /dev/null
+++ b/llvm/test/Transforms/TypePromotion/ARM/switch.ll
@@ -0,0 +1,291 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -mtriple=arm -type-promotion -verify -disable-type-promotion=false -S %s -o - | FileCheck %s
+
+define void @truncate_source_phi_switch(i8* %memblock, i8* %store, i16 %arg) {
+; CHECK-LABEL: @truncate_source_phi_switch(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[PRE:%.*]] = load i8, i8* [[MEMBLOCK:%.*]], align 1
+; CHECK-NEXT:    [[TMP0:%.*]] = zext i8 [[PRE]] to i32
+; CHECK-NEXT:    [[CONV:%.*]] = trunc i16 [[ARG:%.*]] to i8
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[CONV]] to i32
+; CHECK-NEXT:    br label [[HEADER:%.*]]
+; CHECK:       header:
+; CHECK-NEXT:    [[PHI_0:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[COUNT:%.*]], [[LATCH:%.*]] ]
+; CHECK-NEXT:    [[PHI_1:%.*]] = phi i32 [ [[TMP1]], [[ENTRY]] ], [ [[PHI_3:%.*]], [[LATCH]] ]
+; CHECK-NEXT:    [[PHI_2:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[COUNT]], [[LATCH]] ]
+; CHECK-NEXT:    switch i32 [[PHI_0]], label [[DEFAULT:%.*]] [
+; CHECK-NEXT:    i32 43, label [[FOR_INC_I:%.*]]
+; CHECK-NEXT:    i32 45, label [[FOR_INC_I_I:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       for.inc.i:
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[PHI_1]], 1
+; CHECK-NEXT:    br label [[LATCH]]
+; CHECK:       for.inc.i.i:
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[PHI_1]], 3
+; CHECK-NEXT:    br label [[LATCH]]
+; CHECK:       default:
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 [[PHI_0]], 1
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ugt i32 [[SUB]], 4
+; CHECK-NEXT:    br i1 [[CMP2]], label [[LATCH]], label [[EXIT:%.*]]
+; CHECK:       latch:
+; CHECK-NEXT:    [[PHI_3]] = phi i32 [ [[XOR]], [[FOR_INC_I]] ], [ [[AND]], [[FOR_INC_I_I]] ], [ [[PHI_2]], [[DEFAULT]] ]
+; CHECK-NEXT:    [[COUNT]] = add nuw i32 [[PHI_2]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc i32 [[COUNT]] to i8
+; CHECK-NEXT:    store i8 [[TMP2]], i8* [[STORE:%.*]], align 1
+; CHECK-NEXT:    br label [[HEADER]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %pre = load i8, i8* %memblock, align 1
+  %conv = trunc i16 %arg to i8
+  br label %header
+
+header:
+  %phi.0 = phi i8 [ %pre, %entry ], [ %count, %latch ]
+  %phi.1 = phi i8 [ %conv, %entry ], [ %phi.3, %latch ]
+  %phi.2 = phi i8 [ 0, %entry], [ %count, %latch ]
+  switch i8 %phi.0, label %default [
+  i8 43, label %for.inc.i
+  i8 45, label %for.inc.i.i
+  ]
+
+for.inc.i:
+  %xor = xor i8 %phi.1, 1
+  br label %latch
+
+for.inc.i.i:
+  %and = and i8 %phi.1, 3
+  br label %latch
+
+default:
+  %sub = sub i8 %phi.0, 1
+  %cmp2 = icmp ugt i8 %sub, 4
+  br i1 %cmp2, label %latch, label %exit
+
+latch:
+  %phi.3 = phi i8 [ %xor, %for.inc.i ], [ %and, %for.inc.i.i ], [ %phi.2, %default ]
+  %count = add nuw i8 %phi.2, 1
+  store i8 %count, i8* %store, align 1
+  br label %header
+
+exit:
+  ret void
+}
+
+define i16 @icmp_switch_source(i16 zeroext %arg) {
+; CHECK-LABEL: @icmp_switch_source(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[ARG:%.*]] to i32
+; CHECK-NEXT:    [[CONV:%.*]] = add nuw i32 [[TMP0]], 15
+; CHECK-NEXT:    [[MUL:%.*]] = mul nuw nsw i32 [[CONV]], 3
+; CHECK-NEXT:    switch i32 [[TMP0]], label [[DEFAULT:%.*]] [
+; CHECK-NEXT:    i32 0, label [[SW_BB:%.*]]
+; CHECK-NEXT:    i32 1, label [[SW_BB_I:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       sw.bb:
+; CHECK-NEXT:    [[CMP0:%.*]] = icmp ult i32 [[MUL]], 127
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP0]], i32 [[MUL]], i32 127
+; CHECK-NEXT:    br label [[EXIT:%.*]]
+; CHECK:       sw.bb.i:
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ugt i32 [[MUL]], 34
+; CHECK-NEXT:    [[SELECT_I:%.*]] = select i1 [[CMP1]], i32 [[MUL]], i32 34
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       default:
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RES:%.*]] = phi i32 [ [[SELECT]], [[SW_BB]] ], [ [[SELECT_I]], [[SW_BB_I]] ], [ [[MUL]], [[DEFAULT]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[RES]] to i16
+; CHECK-NEXT:    ret i16 [[TMP1]]
+;
+entry:
+  %conv = add nuw i16 %arg, 15
+  %mul = mul nuw nsw i16 %conv, 3
+  switch i16 %arg, label %default [
+  i16 0, label %sw.bb
+  i16 1, label %sw.bb.i
+  ]
+
+sw.bb:
+  %cmp0 = icmp ult i16 %mul, 127
+  %select = select i1 %cmp0, i16 %mul, i16 127
+  br label %exit
+
+sw.bb.i:
+  %cmp1 = icmp ugt i16 %mul, 34
+  %select.i = select i1 %cmp1, i16 %mul, i16 34
+  br label %exit
+
+default:
+  br label %exit
+
+exit:
+  %res = phi i16 [ %select, %sw.bb ], [ %select.i, %sw.bb.i ], [ %mul, %default ]
+  ret i16 %res
+}
+
+define i16 @icmp_switch_narrow_source(i8 zeroext %arg) {
+; CHECK-LABEL: @icmp_switch_narrow_source(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = zext i8 [[ARG:%.*]] to i32
+; CHECK-NEXT:    [[ADD:%.*]] = add nuw i32 [[TMP0]], 15
+; CHECK-NEXT:    [[MUL:%.*]] = mul nuw nsw i32 [[ADD]], 3
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[TMP0]] to i8
+; CHECK-NEXT:    switch i8 [[TMP1]], label [[DEFAULT:%.*]] [
+; CHECK-NEXT:    i8 0, label [[SW_BB:%.*]]
+; CHECK-NEXT:    i8 1, label [[SW_BB_I:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       sw.bb:
+; CHECK-NEXT:    [[CMP0:%.*]] = icmp ult i32 [[MUL]], 127
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP0]], i32 [[MUL]], i32 127
+; CHECK-NEXT:    br label [[EXIT:%.*]]
+; CHECK:       sw.bb.i:
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ugt i32 [[MUL]], 34
+; CHECK-NEXT:    [[SELECT_I:%.*]] = select i1 [[CMP1]], i32 [[MUL]], i32 34
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       default:
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RES:%.*]] = phi i32 [ [[SELECT]], [[SW_BB]] ], [ [[SELECT_I]], [[SW_BB_I]] ], [ [[MUL]], [[DEFAULT]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc i32 [[RES]] to i16
+; CHECK-NEXT:    ret i16 [[TMP2]]
+;
+entry:
+  %conv = zext i8 %arg to i16
+  %add = add nuw i16 %conv, 15
+  %mul = mul nuw nsw i16 %add, 3
+  switch i8 %arg, label %default [
+  i8 0, label %sw.bb
+  i8 1, label %sw.bb.i
+  ]
+
+sw.bb:
+  %cmp0 = icmp ult i16 %mul, 127
+  %select = select i1 %cmp0, i16 %mul, i16 127
+  br label %exit
+
+sw.bb.i:
+  %cmp1 = icmp ugt i16 %mul, 34
+  %select.i = select i1 %cmp1, i16 %mul, i16 34
+  br label %exit
+
+default:
+  br label %exit
+
+exit:
+  %res = phi i16 [ %select, %sw.bb ], [ %select.i, %sw.bb.i ], [ %mul, %default ]
+  ret i16 %res
+}
+
+define i16 @icmp_switch_trunc(i16 zeroext %arg) {
+; CHECK-LABEL: @icmp_switch_trunc(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[ARG:%.*]] to i32
+; CHECK-NEXT:    [[CONV:%.*]] = add nuw i32 [[TMP0]], 15
+; CHECK-NEXT:    [[MUL:%.*]] = mul nuw nsw i32 [[CONV]], 3
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[TMP0]], 7
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc i32 [[TMP1]] to i3
+; CHECK-NEXT:    switch i3 [[TMP2]], label [[DEFAULT:%.*]] [
+; CHECK-NEXT:    i3 0, label [[SW_BB:%.*]]
+; CHECK-NEXT:    i3 1, label [[SW_BB_I:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       sw.bb:
+; CHECK-NEXT:    [[CMP0:%.*]] = icmp ult i32 [[MUL]], 127
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP0]], i32 [[MUL]], i32 127
+; CHECK-NEXT:    br label [[EXIT:%.*]]
+; CHECK:       sw.bb.i:
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ugt i32 [[MUL]], 34
+; CHECK-NEXT:    [[SELECT_I:%.*]] = select i1 [[CMP1]], i32 [[MUL]], i32 34
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       default:
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RES:%.*]] = phi i32 [ [[SELECT]], [[SW_BB]] ], [ [[SELECT_I]], [[SW_BB_I]] ], [ [[MUL]], [[DEFAULT]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = trunc i32 [[RES]] to i16
+; CHECK-NEXT:    ret i16 [[TMP3]]
+;
+entry:
+  %conv = add nuw i16 %arg, 15
+  %mul = mul nuw nsw i16 %conv, 3
+  %trunc = trunc i16 %arg to i3
+  switch i3 %trunc, label %default [
+  i3 0, label %sw.bb
+  i3 1, label %sw.bb.i
+  ]
+
+sw.bb:
+  %cmp0 = icmp ult i16 %mul, 127
+  %select = select i1 %cmp0, i16 %mul, i16 127
+  br label %exit
+
+sw.bb.i:
+  %cmp1 = icmp ugt i16 %mul, 34
+  %select.i = select i1 %cmp1, i16 %mul, i16 34
+  br label %exit
+
+default:
+  br label %exit
+
+exit:
+  %res = phi i16 [ %select, %sw.bb ], [ %select.i, %sw.bb.i ], [ %mul, %default ]
+  ret i16 %res
+}
+
+%class.ae = type { i8 }
+%class.x = type { i8 }
+%class.v = type { %class.q }
+%class.q = type { i16 }
+declare %class.x* @_ZNK2ae2afEv(%class.ae*) local_unnamed_addr
+declare %class.v* @_ZN1x2acEv(%class.x*) local_unnamed_addr
+
+define i32 @trunc_i16_i9_switch(%class.ae* %this) {
+; CHECK-LABEL: @trunc_i16_i9_switch(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CALL:%.*]] = tail call %class.x* @_ZNK2ae2afEv(%class.ae* [[THIS:%.*]])
+; CHECK-NEXT:    [[CALL2:%.*]] = tail call %class.v* @_ZN1x2acEv(%class.x* [[CALL]])
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds [[CLASS_V:%.*]], %class.v* [[CALL2]], i32 0, i32 0, i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = load i16, i16* [[TMP0]], align 2
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i16 [[TMP1]] to i32
+; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[TMP2]], 511
+; CHECK-NEXT:    [[TRUNC:%.*]] = and i32 [[TMP3]], 448
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc i32 [[TRUNC]] to i9
+; CHECK-NEXT:    switch i9 [[TMP4]], label [[CLEANUP_FOLD_SPLIT:%.*]] [
+; CHECK-NEXT:    i9 0, label [[CLEANUP:%.*]]
+; CHECK-NEXT:    i9 -256, label [[IF_THEN7:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       if.then7:
+; CHECK-NEXT:    [[TMP5:%.*]] = and i32 [[TMP2]], 7
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[TMP5]], 0
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[TOBOOL]], i32 2, i32 1
+; CHECK-NEXT:    br label [[CLEANUP]]
+; CHECK:       cleanup.fold.split:
+; CHECK-NEXT:    br label [[CLEANUP]]
+; CHECK:       cleanup:
+; CHECK-NEXT:    [[RETVAL_0:%.*]] = phi i32 [ [[COND]], [[IF_THEN7]] ], [ 0, [[ENTRY:%.*]] ], [ 2, [[CLEANUP_FOLD_SPLIT]] ]
+; CHECK-NEXT:    ret i32 [[RETVAL_0]]
+;
+entry:
+  %call = tail call %class.x* @_ZNK2ae2afEv(%class.ae* %this)
+  %call2 = tail call %class.v* @_ZN1x2acEv(%class.x* %call)
+  %0 = getelementptr inbounds %class.v, %class.v* %call2, i32 0, i32 0, i32 0
+  %1 = load i16, i16* %0, align 2
+  %2 = trunc i16 %1 to i9
+  %trunc = and i9 %2, -64
+  switch i9 %trunc, label %cleanup.fold.split [
+  i9 0, label %cleanup
+  i9 -256, label %if.then7
+  ]
+
+if.then7:
+  %3 = and i16 %1, 7
+  %tobool = icmp eq i16 %3, 0
+  %cond = select i1 %tobool, i32 2, i32 1
+  br label %cleanup
+
+cleanup.fold.split:
+  br label %cleanup
+
+cleanup:
+  %retval.0 = phi i32 [ %cond, %if.then7 ], [ 0, %entry ], [ 2, %cleanup.fold.split ]
+  ret i32 %retval.0
+}
diff --git a/llvm/test/Transforms/TypePromotion/ARM/wrapping.ll b/llvm/test/Transforms/TypePromotion/ARM/wrapping.ll
new file mode 100644
index 000000000..23e50dec0
--- /dev/null
+++ b/llvm/test/Transforms/TypePromotion/ARM/wrapping.ll
@@ -0,0 +1,356 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -mtriple=arm -type-promotion -verify -disable-type-promotion=false -S %s -o - | FileCheck %s
+
+define zeroext i16 @overflow_add(i16 zeroext %a, i16 zeroext %b) {
+; CHECK-LABEL: @overflow_add(
+; CHECK-NEXT:    [[ADD:%.*]] = add i16 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[OR:%.*]] = or i16 [[ADD]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i16 [[OR]], 1024
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[CMP]], i16 2, i16 5
+; CHECK-NEXT:    ret i16 [[RES]]
+;
+  %add = add i16 %a, %b
+  %or = or i16 %add, 1
+  %cmp = icmp ugt i16 %or, 1024
+  %res = select i1 %cmp, i16 2, i16 5
+  ret i16 %res
+}
+
+define zeroext i16 @overflow_sub(i16 zeroext %a, i16 zeroext %b) {
+; CHECK-LABEL: @overflow_sub(
+; CHECK-NEXT:    [[ADD:%.*]] = sub i16 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[OR:%.*]] = or i16 [[ADD]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i16 [[OR]], 1024
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[CMP]], i16 2, i16 5
+; CHECK-NEXT:    ret i16 [[RES]]
+;
+  %add = sub i16 %a, %b
+  %or = or i16 %add, 1
+  %cmp = icmp ugt i16 %or, 1024
+  %res = select i1 %cmp, i16 2, i16 5
+  ret i16 %res
+}
+
+define zeroext i16 @overflow_mul(i16 zeroext %a, i16 zeroext %b) {
+; CHECK-LABEL: @overflow_mul(
+; CHECK-NEXT:    [[ADD:%.*]] = mul i16 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[OR:%.*]] = or i16 [[ADD]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i16 [[OR]], 1024
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[CMP]], i16 2, i16 5
+; CHECK-NEXT:    ret i16 [[RES]]
+;
+  %add = mul i16 %a, %b
+  %or = or i16 %add, 1
+  %cmp = icmp ugt i16 %or, 1024
+  %res = select i1 %cmp, i16 2, i16 5
+  ret i16 %res
+}
+
+define zeroext i16 @overflow_shl(i16 zeroext %a, i16 zeroext %b) {
+; CHECK-LABEL: @overflow_shl(
+; CHECK-NEXT:    [[ADD:%.*]] = shl i16 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[OR:%.*]] = or i16 [[ADD]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i16 [[OR]], 1024
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[CMP]], i16 2, i16 5
+; CHECK-NEXT:    ret i16 [[RES]]
+;
+  %add = shl i16 %a, %b
+  %or = or i16 %add, 1
+  %cmp = icmp ugt i16 %or, 1024
+  %res = select i1 %cmp, i16 2, i16 5
+  ret i16 %res
+}
+
+define i32 @overflow_add_no_consts(i8 zeroext %a, i8 zeroext %b, i8 zeroext %limit) {
+; CHECK-LABEL: @overflow_add_no_consts(
+; CHECK-NEXT:    [[ADD:%.*]] = add i8 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i8 [[ADD]], [[LIMIT:%.*]]
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[CMP]], i32 8, i32 16
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %add = add i8 %a, %b
+  %cmp = icmp ugt i8 %add, %limit
+  %res = select i1 %cmp, i32 8, i32 16
+  ret i32 %res
+}
+
+define i32 @overflow_add_const_limit(i8 zeroext %a, i8 zeroext %b) {
+; CHECK-LABEL: @overflow_add_const_limit(
+; CHECK-NEXT:    [[ADD:%.*]] = add i8 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i8 [[ADD]], -128
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[CMP]], i32 8, i32 16
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %add = add i8 %a, %b
+  %cmp = icmp ugt i8 %add, 128
+  %res = select i1 %cmp, i32 8, i32 16
+  ret i32 %res
+}
+
+define i32 @overflow_add_positive_const_limit(i8 zeroext %a) {
+; CHECK-LABEL: @overflow_add_positive_const_limit(
+; CHECK-NEXT:    [[ADD:%.*]] = add i8 [[A:%.*]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i8 [[ADD]], -128
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[CMP]], i32 8, i32 16
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %add = add i8 %a, 1
+  %cmp = icmp ugt i8 %add, 128
+  %res = select i1 %cmp, i32 8, i32 16
+  ret i32 %res
+}
+
+define i32 @unsafe_add_underflow(i8 zeroext %a) {
+; CHECK-LABEL: @unsafe_add_underflow(
+; CHECK-NEXT:    [[ADD:%.*]] = add i8 [[A:%.*]], -2
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i8 [[ADD]], -2
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[CMP]], i32 8, i32 16
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %add = add i8 %a, -2
+  %cmp = icmp ugt i8 %add, 254
+  %res = select i1 %cmp, i32 8, i32 16
+  ret i32 %res
+}
+
+define i32 @safe_add_underflow(i8 zeroext %a) {
+; CHECK-LABEL: @safe_add_underflow(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = sub i32 [[TMP1]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[TMP2]], 254
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[CMP]], i32 8, i32 16
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %add = add i8 %a, -1
+  %cmp = icmp ugt i8 %add, 254
+  %res = select i1 %cmp, i32 8, i32 16
+  ret i32 %res
+}
+
+define i32 @safe_add_underflow_neg(i8 zeroext %a) {
+; CHECK-LABEL: @safe_add_underflow_neg(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = sub i32 [[TMP1]], 2
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ule i32 [[TMP2]], 250
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[CMP]], i32 8, i32 16
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %add = add i8 %a, -2
+  %cmp = icmp ule i8 %add, -6
+  %res = select i1 %cmp, i32 8, i32 16
+  ret i32 %res
+}
+
+define i32 @overflow_sub_negative_const_limit(i8 zeroext %a) {
+; CHECK-LABEL: @overflow_sub_negative_const_limit(
+; CHECK-NEXT:    [[SUB:%.*]] = sub i8 [[A:%.*]], -1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i8 [[SUB]], -128
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[CMP]], i32 8, i32 16
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %sub = sub i8 %a, -1
+  %cmp = icmp ugt i8 %sub, 128
+  %res = select i1 %cmp, i32 8, i32 16
+  ret i32 %res
+}
+
+define i32 @unsafe_sub_underflow(i8 zeroext %a) {
+; CHECK-LABEL: @unsafe_sub_underflow(
+; CHECK-NEXT:    [[SUB:%.*]] = sub i8 [[A:%.*]], 6
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i8 [[SUB]], -6
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[CMP]], i32 8, i32 16
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %sub = sub i8 %a, 6
+  %cmp = icmp ugt i8 %sub, 250
+  %res = select i1 %cmp, i32 8, i32 16
+  ret i32 %res
+}
+
+define i32 @safe_sub_underflow(i8 zeroext %a) {
+; CHECK-LABEL: @safe_sub_underflow(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i32
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 [[TMP1]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ule i32 [[SUB]], 254
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[CMP]], i32 8, i32 16
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %sub = sub i8 %a, 1
+  %cmp = icmp ule i8 %sub, 254
+  %res = select i1 %cmp, i32 8, i32 16
+  ret i32 %res
+}
+
+define i32 @safe_sub_underflow_neg(i8 zeroext %a) {
+; CHECK-LABEL: @safe_sub_underflow_neg(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i32
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 [[TMP1]], 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp uge i32 [[SUB]], 251
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[CMP]], i32 8, i32 16
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %sub = sub i8 %a, 4
+  %cmp = icmp uge i8 %sub, -5
+  %res = select i1 %cmp, i32 8, i32 16
+  ret i32 %res
+}
+
+define i32 @unsafe_sub_underflow_neg(i8 zeroext %a) {
+; CHECK-LABEL: @unsafe_sub_underflow_neg(
+; CHECK-NEXT:    [[SUB:%.*]] = sub i8 [[A:%.*]], 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i8 [[SUB]], -3
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[CMP]], i32 8, i32 16
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %sub = sub i8 %a, 4
+  %cmp = icmp ult i8 %sub, -3
+  %res = select i1 %cmp, i32 8, i32 16
+  ret i32 %res
+}
+
+define i32 @safe_sub_imm_var(i8* %b) {
+; CHECK-LABEL: @safe_sub_imm_var(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, i8* [[B:%.*]], align 1
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[TMP0]] to i32
+; CHECK-NEXT:    [[SUB:%.*]] = sub nuw nsw i32 248, [[TMP1]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[SUB]], 252
+; CHECK-NEXT:    [[CONV4:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    ret i32 [[CONV4]]
+;
+entry:
+  %0 = load i8, i8* %b, align 1
+  %sub = sub nuw nsw i8 -8, %0
+  %cmp = icmp ugt i8 %sub, 252
+  %conv4 = zext i1 %cmp to i32
+  ret i32 %conv4
+}
+
+define i32 @safe_sub_var_imm(i8* %b) {
+; CHECK-LABEL: @safe_sub_var_imm(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, i8* [[B:%.*]], align 1
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[TMP0]] to i32
+; CHECK-NEXT:    [[SUB:%.*]] = sub nuw nsw i32 [[TMP1]], 248
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[SUB]], 252
+; CHECK-NEXT:    [[CONV4:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    ret i32 [[CONV4]]
+;
+entry:
+  %0 = load i8, i8* %b, align 1
+  %sub = sub nuw nsw i8 %0, -8
+  %cmp = icmp ugt i8 %sub, 252
+  %conv4 = zext i1 %cmp to i32
+  ret i32 %conv4
+}
+
+define i32 @safe_add_imm_var(i8* %b) {
+; CHECK-LABEL: @safe_add_imm_var(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, i8* [[B:%.*]], align 1
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[TMP0]] to i32
+; CHECK-NEXT:    [[ADD:%.*]] = add nuw nsw i32 129, [[TMP1]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[ADD]], 127
+; CHECK-NEXT:    [[CONV4:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    ret i32 [[CONV4]]
+;
+entry:
+  %0 = load i8, i8* %b, align 1
+  %add = add nuw nsw i8 -127, %0
+  %cmp = icmp ugt i8 %add, 127
+  %conv4 = zext i1 %cmp to i32
+  ret i32 %conv4
+}
+
+define i32 @safe_add_var_imm(i8* %b) {
+; CHECK-LABEL: @safe_add_var_imm(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, i8* [[B:%.*]], align 1
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[TMP0]] to i32
+; CHECK-NEXT:    [[ADD:%.*]] = add nuw nsw i32 [[TMP1]], 129
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[ADD]], 127
+; CHECK-NEXT:    [[CONV4:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    ret i32 [[CONV4]]
+;
+entry:
+  %0 = load i8, i8* %b, align 1
+  %add = add nuw nsw i8 %0, -127
+  %cmp = icmp ugt i8 %add, 127
+  %conv4 = zext i1 %cmp to i32
+  ret i32 %conv4
+}
+
+define i8 @convert_add_order(i8 zeroext %arg) {
+; CHECK-LABEL: @convert_add_order(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[ARG:%.*]] to i32
+; CHECK-NEXT:    [[MASK_0:%.*]] = and i32 [[TMP1]], 1
+; CHECK-NEXT:    [[MASK_1:%.*]] = and i32 [[TMP1]], 2
+; CHECK-NEXT:    [[SHL:%.*]] = or i32 [[TMP1]], 1
+; CHECK-NEXT:    [[ADD:%.*]] = add nuw i32 [[SHL]], 10
+; CHECK-NEXT:    [[CMP_0:%.*]] = icmp ult i32 [[ADD]], 60
+; CHECK-NEXT:    [[TMP2:%.*]] = sub nsw i32 [[SHL]], 40
+; CHECK-NEXT:    [[CMP_1:%.*]] = icmp ult i32 [[TMP2]], 20
+; CHECK-NEXT:    [[MASK_SEL:%.*]] = select i1 [[CMP_1]], i32 [[MASK_0]], i32 [[MASK_1]]
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[CMP_0]], i32 [[MASK_SEL]], i32 [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = trunc i32 [[RES]] to i8
+; CHECK-NEXT:    ret i8 [[TMP3]]
+;
+  %mask.0 = and i8 %arg, 1
+  %mask.1 = and i8 %arg, 2
+  %shl = or i8 %arg, 1
+  %add = add nuw i8 %shl, 10
+  %cmp.0 = icmp ult i8 %add, 60
+  %sub = add nsw i8 %shl, -40
+  %cmp.1 = icmp ult i8 %sub, 20
+  %mask.sel = select i1 %cmp.1, i8 %mask.0, i8 %mask.1
+  %res = select i1 %cmp.0, i8 %mask.sel, i8 %arg
+  ret i8 %res
+}
+
+define i8 @underflow_if_sub(i32 %arg, i8 zeroext %arg1) {
+; CHECK-LABEL: @underflow_if_sub(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[ARG1:%.*]] to i32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[ARG:%.*]], 0
+; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[ARG]], [[CONV]]
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc i32 [[AND]] to i8
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i8 [[TRUNC]] to i32
+; CHECK-NEXT:    [[CONV1:%.*]] = add nuw nsw i32 [[TMP2]], 245
+; CHECK-NEXT:    [[CMP_1:%.*]] = icmp ult i32 [[CONV1]], [[TMP1]]
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[CMP_1]], i32 [[CONV1]], i32 100
+; CHECK-NEXT:    [[TMP3:%.*]] = trunc i32 [[RES]] to i8
+; CHECK-NEXT:    ret i8 [[TMP3]]
+;
+  %cmp = icmp sgt i32 %arg, 0
+  %conv = zext i1 %cmp to i32
+  %and = and i32 %arg, %conv
+  %trunc = trunc i32 %and to i8
+  %conv1 = add nuw nsw i8 %trunc, -11
+  %cmp.1 = icmp ult i8 %conv1, %arg1
+  %res = select i1 %cmp.1, i8 %conv1, i8 100
+  ret i8 %res
+}
+
+define i8 @underflow_if_sub_signext(i32 %arg, i8 signext %arg1) {
+; CHECK-LABEL: @underflow_if_sub_signext(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[ARG1:%.*]] to i32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[ARG:%.*]], 0
+; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[ARG]], [[CONV]]
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc i32 [[AND]] to i8
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i8 [[TRUNC]] to i32
+; CHECK-NEXT:    [[CONV1:%.*]] = add nuw nsw i32 [[TMP2]], 245
+; CHECK-NEXT:    [[CMP_1:%.*]] = icmp ugt i32 [[TMP1]], [[CONV1]]
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[CMP_1]], i32 [[CONV1]], i32 100
+; CHECK-NEXT:    [[TMP3:%.*]] = trunc i32 [[RES]] to i8
+; CHECK-NEXT:    ret i8 [[TMP3]]
+;
+  %cmp = icmp sgt i32 %arg, 0
+  %conv = zext i1 %cmp to i32
+  %and = and i32 %arg, %conv
+  %trunc = trunc i32 %and to i8
+  %conv1 = add nuw nsw i8 %trunc, -11
+  %cmp.1 = icmp ugt i8 %arg1, %conv1
+  %res = select i1 %cmp.1, i8 %conv1, i8 100
+  ret i8 %res
+}
diff --git a/llvm/test/Transforms/Util/PredicateInfo/condprop.ll b/llvm/test/Transforms/Util/PredicateInfo/condprop.ll
index 0cd6ef34e..daf6bb8b4 100644
--- a/llvm/test/Transforms/Util/PredicateInfo/condprop.ll
+++ b/llvm/test/Transforms/Util/PredicateInfo/condprop.ll
@@ -98,10 +98,10 @@ define void @test3(i32 %x, i32 %y) {
 ; CHECK-NEXT:    [[XZ:%.*]] = icmp eq i32 [[X:%.*]], 0
 ; CHECK-NEXT:    [[YZ:%.*]] = icmp eq i32 [[Y:%.*]], 0
 ; CHECK-NEXT:    [[Z:%.*]] = and i1 [[XZ]], [[YZ]]
-; CHECK:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[X]])
-; CHECK:         [[Y_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[Y]])
 ; CHECK:         [[XZ_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[XZ]])
+; CHECK:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[X]])
 ; CHECK:         [[YZ_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[YZ]])
+; CHECK:         [[Y_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[Y]])
 ; CHECK:         [[Z_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[Z]])
 ; CHECK-NEXT:    br i1 [[Z]], label [[BOTH_ZERO:%.*]], label [[NOPE:%.*]]
 ; CHECK:       both_zero:
@@ -382,8 +382,8 @@ ret:
 define i32 @test10(i32 %j, i32 %i) {
 ; CHECK-LABEL: @test10(
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I:%.*]], [[J:%.*]]
-; CHECK:         [[J_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[J]])
 ; CHECK:         [[I_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[I]])
+; CHECK:         [[J_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[J]])
 ; CHECK-NEXT:    br i1 [[CMP]], label [[COND_TRUE:%.*]], label [[RET:%.*]]
 ; CHECK:       cond_true:
 ; CHECK-NEXT:    [[DIFF:%.*]] = sub i32 [[I_0]], [[J_0]]
diff --git a/llvm/test/Transforms/Util/PredicateInfo/testandor.ll b/llvm/test/Transforms/Util/PredicateInfo/testandor.ll
index 86d1a0d24..c84562cc5 100644
--- a/llvm/test/Transforms/Util/PredicateInfo/testandor.ll
+++ b/llvm/test/Transforms/Util/PredicateInfo/testandor.ll
@@ -10,10 +10,10 @@ define void @testor(i32 %x, i32 %y) {
 ; CHECK-NEXT:    [[XZ:%.*]] = icmp eq i32 [[X:%.*]], 0
 ; CHECK-NEXT:    [[YZ:%.*]] = icmp eq i32 [[Y:%.*]], 0
 ; CHECK-NEXT:    [[Z:%.*]] = or i1 [[XZ]], [[YZ]]
-; CHECK:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[X]])
-; CHECK:         [[Y_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[Y]])
 ; CHECK:         [[XZ_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[XZ]])
+; CHECK:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[X]])
 ; CHECK:         [[YZ_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[YZ]])
+; CHECK:         [[Y_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[Y]])
 ; CHECK:         [[Z_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[Z]])
 ; CHECK-NEXT:    br i1 [[Z]], label [[ONEOF:%.*]], label [[NEITHER:%.*]]
 ; CHECK:       oneof:
@@ -54,10 +54,10 @@ define void @testand(i32 %x, i32 %y) {
 ; CHECK-NEXT:    [[XZ:%.*]] = icmp eq i32 [[X:%.*]], 0
 ; CHECK-NEXT:    [[YZ:%.*]] = icmp eq i32 [[Y:%.*]], 0
 ; CHECK-NEXT:    [[Z:%.*]] = and i1 [[XZ]], [[YZ]]
-; CHECK:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[X]])
-; CHECK:         [[Y_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[Y]])
 ; CHECK:         [[XZ_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[XZ]])
+; CHECK:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[X]])
 ; CHECK:         [[YZ_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[YZ]])
+; CHECK:         [[Y_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[Y]])
 ; CHECK:         [[Z_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[Z]])
 ; CHECK-NEXT:    br i1 [[Z]], label [[BOTH:%.*]], label [[NOPE:%.*]]
 ; CHECK:       both:
@@ -98,9 +98,9 @@ define void @testandsame(i32 %x, i32 %y) {
 ; CHECK-NEXT:    [[XGT:%.*]] = icmp sgt i32 [[X:%.*]], 0
 ; CHECK-NEXT:    [[XLT:%.*]] = icmp slt i32 [[X]], 100
 ; CHECK-NEXT:    [[Z:%.*]] = and i1 [[XGT]], [[XLT]]
+; CHECK:         [[XGT_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[XGT]])
 ; CHECK:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[X]])
 ; CHECK:         [[X_0_1:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[X_0]])
-; CHECK:         [[XGT_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[XGT]])
 ; CHECK:         [[XLT_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[XLT]])
 ; CHECK:         [[Z_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[Z]])
 ; CHECK-NEXT:    br i1 [[Z]], label [[BOTH:%.*]], label [[NOPE:%.*]]
@@ -136,23 +136,23 @@ define void @testandassume(i32 %x, i32 %y) {
 ; CHECK-NEXT:    [[XZ:%.*]] = icmp eq i32 [[X:%.*]], 0
 ; CHECK-NEXT:    [[YZ:%.*]] = icmp eq i32 [[Y:%.*]], 0
 ; CHECK-NEXT:    [[Z:%.*]] = and i1 [[XZ]], [[YZ]]
-; CHECK:         [[TMP1:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[X]])
-; CHECK:         [[TMP2:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[Y]])
-; CHECK:         [[TMP3:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[XZ]])
-; CHECK:         [[TMP4:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[YZ]])
+; CHECK:         [[TMP1:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[XZ]])
+; CHECK:         [[TMP2:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[X]])
+; CHECK:         [[TMP3:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[YZ]])
+; CHECK:         [[TMP4:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[Y]])
 ; CHECK:         [[TMP5:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[Z]])
 ; CHECK-NEXT:    call void @llvm.assume(i1 [[TMP5]])
-; CHECK:         [[DOT0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[TMP1]])
+; CHECK:         [[DOT0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[TMP1]])
 ; CHECK:         [[DOT01:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[TMP2]])
 ; CHECK:         [[DOT02:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[TMP3]])
-; CHECK:         [[DOT03:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[TMP4]])
+; CHECK:         [[DOT03:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[TMP4]])
 ; CHECK:         [[DOT04:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[TMP5]])
 ; CHECK-NEXT:    br i1 [[TMP5]], label [[BOTH:%.*]], label [[NOPE:%.*]]
 ; CHECK:       both:
+; CHECK-NEXT:    call void @foo(i1 [[DOT0]])
 ; CHECK-NEXT:    call void @foo(i1 [[DOT02]])
-; CHECK-NEXT:    call void @foo(i1 [[DOT03]])
-; CHECK-NEXT:    call void @bar(i32 [[DOT0]])
 ; CHECK-NEXT:    call void @bar(i32 [[DOT01]])
+; CHECK-NEXT:    call void @bar(i32 [[DOT03]])
 ; CHECK-NEXT:    ret void
 ; CHECK:       nope:
 ; CHECK-NEXT:    call void @foo(i1 [[DOT04]])
diff --git a/llvm/test/Transforms/Util/add-TLI-mappings.ll b/llvm/test/Transforms/Util/add-TLI-mappings.ll
new file mode 100644
index 000000000..c68a9c9a7
--- /dev/null
+++ b/llvm/test/Transforms/Util/add-TLI-mappings.ll
@@ -0,0 +1,61 @@
+; RUN: opt -vector-library=SVML       -inject-tli-mappings        -S < %s | FileCheck %s  --check-prefixes=COMMON,SVML
+; RUN: opt -vector-library=SVML       -passes=inject-tli-mappings -S < %s | FileCheck %s  --check-prefixes=COMMON,SVML
+; RUN: opt -vector-library=MASSV      -inject-tli-mappings        -S < %s | FileCheck %s  --check-prefixes=COMMON,MASSV
+; RUN: opt -vector-library=MASSV      -passes=inject-tli-mappings -S < %s | FileCheck %s  --check-prefixes=COMMON,MASSV
+; RUN: opt -vector-library=Accelerate -inject-tli-mappings        -S < %s | FileCheck %s  --check-prefixes=COMMON,ACCELERATE
+; RUN: opt -vector-library=Accelerate -passes=inject-tli-mappings -S < %s | FileCheck %s  --check-prefixes=COMMON,ACCELERATE
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; COMMON-LABEL: @llvm.compiler.used = appending global
+; SVML-SAME:        [3 x i8*] [
+; SVML-SAME:          i8* bitcast (<2 x double> (<2 x double>)* @__svml_sin2 to i8*),
+; SVML-SAME:          i8* bitcast (<4 x double> (<4 x double>)* @__svml_sin4 to i8*),
+; SVML-SAME:          i8* bitcast (<8 x double> (<8 x double>)* @__svml_sin8 to i8*)
+; MASSV-SAME:       [2 x i8*] [
+; MASSV-SAME:         i8* bitcast (<2 x double> (<2 x double>)* @__sind2_massv to i8*),
+; MASSV-SAME:         i8* bitcast (<4 x float> (<4 x float>)* @__log10f4_massv to i8*)
+; ACCELERATE-SAME:  [1 x i8*] [
+; ACCELERATE-SAME:    i8* bitcast (<4 x float> (<4 x float>)* @vlog10f to i8*)
+; COMMON-SAME:      ], section "llvm.metadata"
+
+define double @sin_f64(double %in) {
+; COMMON-LABEL: @sin_f64(
+; SVML:         call double @sin(double %{{.*}}) #[[SIN:[0-9]+]]
+; MASSV:        call double @sin(double %{{.*}}) #[[SIN:[0-9]+]]
+; ACCELERATE:   call double @sin(double %{{.*}})
+; No mapping of "sin" to a vector function for Accelerate.
+; ACCELERATE-NOT: _ZGV_LLVM_{{.*}}_sin({{.*}})
+  %call = tail call double @sin(double %in)
+  ret double %call
+}
+
+declare double @sin(double) #0
+
+define float @call_llvm.log10.f32(float %in) {
+; COMMON-LABEL: @call_llvm.log10.f32(
+; SVML:         call float @llvm.log10.f32(float %{{.*}})
+; MASSV:        call float @llvm.log10.f32(float %{{.*}}) #[[LOG10:[0-9]+]]
+; ACCELERATE:   call float @llvm.log10.f32(float %{{.*}}) #[[LOG10:[0-9]+]]
+; No mapping of "llvm.log10.f32" to a vector function for SVML.
+; SVML-NOT:     _ZGV_LLVM_{{.*}}_llvm.log10.f32({{.*}})
+  %call = tail call float @llvm.log10.f32(float %in)
+  ret float %call
+}
+
+declare float @llvm.log10.f32(float) #0
+attributes #0 = { nounwind readnone }
+
+; SVML:      attributes #[[SIN]] = { "vector-function-abi-variant"=
+; SVML-SAME:   "_ZGV_LLVM_N2v_sin(__svml_sin2),
+; SVML-SAME:   _ZGV_LLVM_N4v_sin(__svml_sin4),
+; SVML-SAME:   _ZGV_LLVM_N8v_sin(__svml_sin8)" }
+
+; MASSV:      attributes #[[SIN]] = { "vector-function-abi-variant"=
+; MASSV-SAME:   "_ZGV_LLVM_N2v_sin(__sind2_massv)" }
+; MASSV:      attributes #[[LOG10]] = { "vector-function-abi-variant"=
+; MASSV-SAME:   "_ZGV_LLVM_N4v_llvm.log10.f32(__log10f4_massv)" }
+
+; ACCELERATE:      attributes #[[LOG10]] = { "vector-function-abi-variant"=
+; ACCELERATE-SAME:   "_ZGV_LLVM_N4v_llvm.log10.f32(vlog10f)" }
diff --git a/llvm/test/Transforms/Util/dbg-call-bitcast.ll b/llvm/test/Transforms/Util/dbg-call-bitcast.ll
new file mode 100644
index 000000000..6625b469b
--- /dev/null
+++ b/llvm/test/Transforms/Util/dbg-call-bitcast.ll
@@ -0,0 +1,48 @@
+; RUN: opt -instcombine -S %s | FileCheck %s
+
+define dso_local void @_Z1fv() {
+  %1 = alloca i32, align 4
+  %2 = bitcast i32* %1 to i8*
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* nonnull %2)
+  call void @llvm.dbg.declare(metadata i32* %1, metadata !16, metadata !DIExpression()), !dbg !19
+; CHECK: %[[A:.*]] = alloca i32, align 4
+; CHECK: call void @llvm.dbg.value(metadata i32* %[[A]], {{.*}}, metadata !DIExpression(DW_OP_deref)
+; CHECK: call void @_Z1gPv
+  call void @_Z1gPv(i8* nonnull %2)
+  %3 = bitcast i32* %1 to i8*
+; CHECK: call void @llvm.dbg.value(metadata i32* %[[A]], {{.*}}, metadata !DIExpression(DW_OP_deref)
+; CHECK: call void @_Z1gPv
+  call void @_Z1gPv(i8* nonnull %3)
+  call void @llvm.lifetime.end.p0i8(i64 4, i8* nonnull %2)
+  ret void, !dbg !21
+}
+
+declare void @llvm.lifetime.start.p0i8(i64 immarg, i8* nocapture)
+declare void @llvm.dbg.declare(metadata, metadata, metadata)
+declare dso_local void @_Z1gPv(i8*)
+declare void @llvm.lifetime.end.p0i8(i64 immarg, i8* nocapture)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!8, !9, !10}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 10.0.0 (git@github.com:llvm/llvm-project.git cab64b5708f614c71d275ec9d134e68b8c3baedd)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !3, nameTableKind: None)
+!1 = !DIFile(filename: "dbg.cc", directory: "/tmp")
+!2 = !{}
+!3 = !{!4, !5}
+!4 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: null, size: 64)
+!5 = !DISubprogram(name: "g", linkageName: "_Z1gPv", scope: !1, file: !1, line: 1, type: !6, flags: DIFlagPrototyped, spFlags: DISPFlagOptimized, retainedNodes: !2)
+!6 = !DISubroutineType(types: !7)
+!7 = !{null, !4}
+!8 = !{i32 7, !"Dwarf Version", i32 4}
+!9 = !{i32 2, !"Debug Info Version", i32 3}
+!10 = !{i32 1, !"wchar_size", i32 4}
+!12 = distinct !DISubprogram(name: "f", linkageName: "_Z1fv", scope: !1, file: !1, line: 2, type: !13, scopeLine: 2, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !15)
+!13 = !DISubroutineType(types: !14)
+!14 = !{null}
+!15 = !{!16}
+!16 = !DILocalVariable(name: "x", scope: !12, file: !1, line: 3, type: !17)
+!17 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!18 = !DILocation(line: 3, column: 3, scope: !12)
+!19 = !DILocation(line: 3, column: 7, scope: !12)
+!20 = !DILocation(line: 4, column: 3, scope: !12)
+!21 = !DILocation(line: 5, column: 1, scope: !12)
diff --git a/llvm/test/Transforms/Util/dbg-user-of-aext.ll b/llvm/test/Transforms/Util/dbg-user-of-aext.ll
index 9a3106693..f99151837 100644
--- a/llvm/test/Transforms/Util/dbg-user-of-aext.ll
+++ b/llvm/test/Transforms/Util/dbg-user-of-aext.ll
@@ -54,7 +54,7 @@ entry:
 ; Function Attrs: nounwind readnone speculatable
 declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
 
-attributes #0 = { noinline nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noinline nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone speculatable }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/Transforms/Util/flattencfg.ll b/llvm/test/Transforms/Util/flattencfg.ll
index 4fcb77ab0..3b31476df 100644
--- a/llvm/test/Transforms/Util/flattencfg.ll
+++ b/llvm/test/Transforms/Util/flattencfg.ll
@@ -24,3 +24,63 @@ b1:                                       ; preds = %entry, %b0
 exit:                               ; preds = %entry, %b0, %b1
   ret void
 }
+
+; CHECK-LABEL: @test_not_crash2
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %0 = fcmp ult float %a
+; CHECK-NEXT:    %1 = fcmp ult float %b
+; CHECK-NEXT:    [[COND:%[a-z0-9]+]] = or i1 %0, %1
+; CHECK-NEXT:    br i1 [[COND]], label %bb4, label %bb3
+; CHECK:       bb3:
+; CHECK-NEXT:    br label %bb4
+; CHECK:       bb4:
+; CHECK-NEXT:    ret void
+define void @test_not_crash2(float %a, float %b) #0 {
+entry:
+  %0 = fcmp ult float %a, 1.000000e+00
+  br i1 %0, label %bb0, label %bb1
+
+bb3:                                               ; preds = %bb0
+  br label %bb4
+
+bb4:                                               ; preds = %bb0, %bb3
+  ret void
+
+bb1:                                               ; preds = %entry
+  br label %bb0
+
+bb0:                                               ; preds = %bb1, %entry
+  %1 = fcmp ult float %b, 1.000000e+00
+  br i1 %1, label %bb4, label %bb3
+}
+
+; CHECK-LABEL: @test_not_crash3
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %a_eq_0 = icmp eq i32 %a, 0
+; CHECK-NEXT:    %a_eq_1 = icmp eq i32 %a, 1
+; CHECK-NEXT:    [[COND:%[a-z0-9]+]] = or i1 %a_eq_0, %a_eq_1
+; CHECK-NEXT:    br i1 [[COND]], label %bb2, label %bb3
+; CHECK:       bb2:
+; CHECK-NEXT:    br label %bb3
+; CHECK:       bb3:
+; CHECK-NEXT:    %check_badref = phi i32 [ 17, %entry ], [ 11, %bb2 ]
+; CHECK-NEXT:    ret void
+define void @test_not_crash3(i32 %a) #0 {
+entry:
+  %a_eq_0 = icmp eq i32 %a, 0
+  br i1 %a_eq_0, label %bb0, label %bb1
+
+bb0:                                              ; preds = %entry
+  br label %bb1
+
+bb1:                                              ; preds = %bb0, %entry
+  %a_eq_1 = icmp eq i32 %a, 1
+  br i1 %a_eq_1, label %bb2, label %bb3
+
+bb2:                                              ; preds = %bb1
+  br label %bb3
+
+bb3:                                              ; preds = %bb2, %bb1
+  %check_badref = phi i32 [ 17, %bb1 ], [ 11, %bb2 ]
+  ret void
+}
diff --git a/llvm/test/Transforms/Util/libcalls-fast-math-inf-loop.ll b/llvm/test/Transforms/Util/libcalls-fast-math-inf-loop.ll
index a351fe821..abbc7b7da 100644
--- a/llvm/test/Transforms/Util/libcalls-fast-math-inf-loop.ll
+++ b/llvm/test/Transforms/Util/libcalls-fast-math-inf-loop.ll
@@ -47,8 +47,8 @@ define available_externally float @expf(float %x) #1 {
 ; Function Attrs: nounwind readnone
 declare double @exp(double) #2
 
-attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-features"="+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { inlinehint nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-features"="+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-features"="+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { inlinehint nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-features"="+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind readnone }
 
 !llvm.ident = !{!0}
diff --git a/llvm/test/Transforms/Util/simplify-dbg-declare-load.ll b/llvm/test/Transforms/Util/simplify-dbg-declare-load.ll
index a62e2891c..483cf2f6d 100644
--- a/llvm/test/Transforms/Util/simplify-dbg-declare-load.ll
+++ b/llvm/test/Transforms/Util/simplify-dbg-declare-load.ll
@@ -1,5 +1,6 @@
 ; RUN: opt -instcombine -instcombine-lower-dbg-declare=1 -S < %s | FileCheck %s
 ; RUN: opt -instcombine -instcombine-lower-dbg-declare=0 -S < %s | FileCheck %s --check-prefix=DECLARE
+; XFAIL: *
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-pc-linux-gnu"
 
diff --git a/llvm/test/Transforms/WholeProgramDevirt/branch-funnel.ll b/llvm/test/Transforms/WholeProgramDevirt/branch-funnel.ll
index 49d9caaf7..db7608072 100644
--- a/llvm/test/Transforms/WholeProgramDevirt/branch-funnel.ll
+++ b/llvm/test/Transforms/WholeProgramDevirt/branch-funnel.ll
@@ -148,7 +148,7 @@ define i32 @fn3(i8* %obj) #0 {
 }
 
 ; CHECK-LABEL: define internal void @branch_funnel(i8*
-; CHECK: define hidden void @__typeid_typeid1_0_branch_funnel(i8* nest, ...)
+; CHECK: define hidden void @__typeid_typeid1_0_branch_funnel(i8* nest %0, ...)
 ; CHECK-NEXT: musttail call void (...) @llvm.icall.branch.funnel(i8* %0, i8* bitcast ([1 x i8*]* {{(nonnull )?}}@vt1_1 to i8*), i32 (i8*, i32)* {{(nonnull )?}}@vf1_1, i8* bitcast ([1 x i8*]* {{(nonnull )?}}@vt1_2 to i8*), i32 (i8*, i32)* {{(nonnull )?}}@vf1_2, ...)
 
 declare i1 @llvm.type.test(i8*, metadata)
diff --git a/llvm/test/Transforms/WholeProgramDevirt/export-single-impl.ll b/llvm/test/Transforms/WholeProgramDevirt/export-single-impl.ll
index e03933ccb..cbba1a4d1 100644
--- a/llvm/test/Transforms/WholeProgramDevirt/export-single-impl.ll
+++ b/llvm/test/Transforms/WholeProgramDevirt/export-single-impl.ll
@@ -76,17 +76,17 @@ $vf4 = comdat largest
 ; CHECK: declare void @vf1(i8*)
 declare void @vf1(i8*)
 
-; CHECK: define void @vf2(i8*)
+; CHECK: define void @vf2(i8* %0)
 define void @vf2(i8*) {
   ret void
 }
 
-; CHECK: define hidden void @"vf3$merged"(i8*) {
+; CHECK: define hidden void @"vf3$merged"(i8* %0) {
 define internal void @vf3(i8*) {
   ret void
 }
 
-; CHECK: define hidden void @"vf4$merged"(i8*) comdat {
+; CHECK: define hidden void @"vf4$merged"(i8* %0) comdat {
 define internal void @vf4(i8*) comdat {
   ret void
 }
diff --git a/llvm/test/Transforms/WholeProgramDevirt/virtual-const-prop-begin.ll b/llvm/test/Transforms/WholeProgramDevirt/virtual-const-prop-begin.ll
index 63549e854..6e55235eb 100644
--- a/llvm/test/Transforms/WholeProgramDevirt/virtual-const-prop-begin.ll
+++ b/llvm/test/Transforms/WholeProgramDevirt/virtual-const-prop-begin.ll
@@ -17,19 +17,19 @@ i8* bitcast (i1 (i8*)* @vf0i1 to i8*),
 i8* bitcast (i32 (i8*)* @vf2i32 to i8*)
 ], !type !0
 
-; CHECK: [[VT3DATA:@[^ ]*]] = private constant { [8 x i8], [3 x i8*], [0 x i8] } { [8 x i8] c"\00\00\00\01\03\00\00\00", [3 x i8*] [i8* bitcast (i1 (i8*)* @vf0i1 to i8*), i8* bitcast (i1 (i8*)* @vf1i1 to i8*), i8* bitcast (i32 (i8*)* @vf3i32 to i8*)], [0 x i8] zeroinitializer }, !type [[T8]]
+; CHECK: [[VT3DATA:@[^ ]*]] = private constant { [5 x i8], [3 x i8*], [0 x i8] } { [5 x i8] c"\01\03\00\00\00", [3 x i8*] [i8* bitcast (i1 (i8*)* @vf0i1 to i8*), i8* bitcast (i1 (i8*)* @vf1i1 to i8*), i8* bitcast (i32 (i8*)* @vf3i32 to i8*)], [0 x i8] zeroinitializer }, align 1, !type [[T5:![0-9]+]]
 @vt3 = constant [3 x i8*] [
 i8* bitcast (i1 (i8*)* @vf0i1 to i8*),
 i8* bitcast (i1 (i8*)* @vf1i1 to i8*),
 i8* bitcast (i32 (i8*)* @vf3i32 to i8*)
-], !type !0
+], align 1, !type !0
 
-; CHECK: [[VT4DATA:@[^ ]*]] = private constant { [8 x i8], [3 x i8*], [0 x i8] } { [8 x i8] c"\00\00\00\02\04\00\00\00", [3 x i8*] [i8* bitcast (i1 (i8*)* @vf1i1 to i8*), i8* bitcast (i1 (i8*)* @vf0i1 to i8*), i8* bitcast (i32 (i8*)* @vf4i32 to i8*)], [0 x i8] zeroinitializer }, !type [[T8]]
+; CHECK: [[VT4DATA:@[^ ]*]] = private constant { [16 x i8], [3 x i8*], [0 x i8] } { [16 x i8] c"\00\00\00\00\00\00\00\00\00\00\00\02\04\00\00\00", [3 x i8*] [i8* bitcast (i1 (i8*)* @vf1i1 to i8*), i8* bitcast (i1 (i8*)* @vf0i1 to i8*), i8* bitcast (i32 (i8*)* @vf4i32 to i8*)], [0 x i8] zeroinitializer },  align 16, !type [[T16:![0-9]+]]
 @vt4 = constant [3 x i8*] [
 i8* bitcast (i1 (i8*)* @vf1i1 to i8*),
 i8* bitcast (i1 (i8*)* @vf0i1 to i8*),
 i8* bitcast (i32 (i8*)* @vf4i32 to i8*)
-], !type !0
+], align 16, !type !0
 
 ; CHECK: @vt5 = {{.*}}, !type [[T0:![0-9]+]]
 @vt5 = constant [3 x i8*] [
@@ -40,8 +40,8 @@ i8* bitcast (void ()* @__cxa_pure_virtual to i8*)
 
 ; CHECK: @vt1 = alias [3 x i8*], getelementptr inbounds ({ [8 x i8], [3 x i8*], [0 x i8] }, { [8 x i8], [3 x i8*], [0 x i8] }* [[VT1DATA]], i32 0, i32 1)
 ; CHECK: @vt2 = alias [3 x i8*], getelementptr inbounds ({ [8 x i8], [3 x i8*], [0 x i8] }, { [8 x i8], [3 x i8*], [0 x i8] }* [[VT2DATA]], i32 0, i32 1)
-; CHECK: @vt3 = alias [3 x i8*], getelementptr inbounds ({ [8 x i8], [3 x i8*], [0 x i8] }, { [8 x i8], [3 x i8*], [0 x i8] }* [[VT3DATA]], i32 0, i32 1)
-; CHECK: @vt4 = alias [3 x i8*], getelementptr inbounds ({ [8 x i8], [3 x i8*], [0 x i8] }, { [8 x i8], [3 x i8*], [0 x i8] }* [[VT4DATA]], i32 0, i32 1)
+; CHECK: @vt3 = alias [3 x i8*], getelementptr inbounds ({ [5 x i8], [3 x i8*], [0 x i8] }, { [5 x i8], [3 x i8*], [0 x i8] }* [[VT3DATA]], i32 0, i32 1)
+; CHECK: @vt4 = alias [3 x i8*], getelementptr inbounds ({ [16 x i8], [3 x i8*], [0 x i8] }, { [16 x i8], [3 x i8*], [0 x i8] }* [[VT4DATA]], i32 0, i32 1)
 
 define i1 @vf0i1(i8* %this) readnone {
   ret i1 0
@@ -134,6 +134,8 @@ declare void @llvm.assume(i1)
 declare void @__cxa_pure_virtual()
 
 ; CHECK: [[T8]] = !{i32 8, !"typeid"}
+; CHECK: [[T5]] = !{i32 5, !"typeid"}
+; CHECK: [[T16]] = !{i32 16, !"typeid"}
 ; CHECK: [[T0]] = !{i32 0, !"typeid"}
 
 !0 = !{i32 0, !"typeid"}
diff --git a/llvm/test/Transforms/WholeProgramDevirt/virtual-const-prop-end.ll b/llvm/test/Transforms/WholeProgramDevirt/virtual-const-prop-end.ll
index 3d61e1e20..5ab9571a0 100644
--- a/llvm/test/Transforms/WholeProgramDevirt/virtual-const-prop-end.ll
+++ b/llvm/test/Transforms/WholeProgramDevirt/virtual-const-prop-end.ll
@@ -3,7 +3,7 @@
 target datalayout = "e-p:64:64"
 target triple = "x86_64-unknown-linux-gnu"
 
-; CHECK: [[VT1DATA:@[^ ]*]] = private constant { [0 x i8], [4 x i8*], [8 x i8] } { [0 x i8] zeroinitializer, [4 x i8*] [i8* null, i8* bitcast (i1 (i8*)* @vf0i1 to i8*), i8* bitcast (i1 (i8*)* @vf1i1 to i8*), i8* bitcast (i32 (i8*)* @vf1i32 to i8*)], [8 x i8] c"\01\00\00\00\01\00\00\00" }, !type [[T8:![0-9]+]]
+; CHECK: [[VT1DATA:@[^ ]*]] = private constant { [0 x i8], [4 x i8*], [5 x i8] } { [0 x i8] zeroinitializer, [4 x i8*] [i8* null, i8* bitcast (i1 (i8*)* @vf0i1 to i8*), i8* bitcast (i1 (i8*)* @vf1i1 to i8*), i8* bitcast (i32 (i8*)* @vf1i32 to i8*)], [5 x i8] c"\01\00\00\00\01" }, !type [[T8:![0-9]+]]
 @vt1 = constant [4 x i8*] [
 i8* null,
 i8* bitcast (i1 (i8*)* @vf0i1 to i8*),
@@ -11,14 +11,14 @@ i8* bitcast (i1 (i8*)* @vf1i1 to i8*),
 i8* bitcast (i32 (i8*)* @vf1i32 to i8*)
 ], !type !1
 
-; CHECK: [[VT2DATA:@[^ ]*]] = private constant { [0 x i8], [3 x i8*], [8 x i8] } { [0 x i8] zeroinitializer, [3 x i8*] [i8* bitcast (i1 (i8*)* @vf1i1 to i8*), i8* bitcast (i1 (i8*)* @vf0i1 to i8*), i8* bitcast (i32 (i8*)* @vf2i32 to i8*)], [8 x i8] c"\02\00\00\00\02\00\00\00" }, !type [[T0:![0-9]+]]
+; CHECK: [[VT2DATA:@[^ ]*]] = private constant { [0 x i8], [3 x i8*], [5 x i8] } { [0 x i8] zeroinitializer, [3 x i8*] [i8* bitcast (i1 (i8*)* @vf1i1 to i8*), i8* bitcast (i1 (i8*)* @vf0i1 to i8*), i8* bitcast (i32 (i8*)* @vf2i32 to i8*)], [5 x i8] c"\02\00\00\00\02" }, !type [[T0:![0-9]+]]
 @vt2 = constant [3 x i8*] [
 i8* bitcast (i1 (i8*)* @vf1i1 to i8*),
 i8* bitcast (i1 (i8*)* @vf0i1 to i8*),
 i8* bitcast (i32 (i8*)* @vf2i32 to i8*)
 ], !type !0
 
-; CHECK: [[VT3DATA:@[^ ]*]] = private constant { [0 x i8], [4 x i8*], [8 x i8] } { [0 x i8] zeroinitializer, [4 x i8*] [i8* null, i8* bitcast (i1 (i8*)* @vf0i1 to i8*), i8* bitcast (i1 (i8*)* @vf1i1 to i8*), i8* bitcast (i32 (i8*)* @vf3i32 to i8*)], [8 x i8] c"\03\00\00\00\01\00\00\00" }, !type [[T8]]
+; CHECK: [[VT3DATA:@[^ ]*]] = private constant { [0 x i8], [4 x i8*], [5 x i8] } { [0 x i8] zeroinitializer, [4 x i8*] [i8* null, i8* bitcast (i1 (i8*)* @vf0i1 to i8*), i8* bitcast (i1 (i8*)* @vf1i1 to i8*), i8* bitcast (i32 (i8*)* @vf3i32 to i8*)], [5 x i8] c"\03\00\00\00\01" }, !type [[T8]]
 @vt3 = constant [4 x i8*] [
 i8* null,
 i8* bitcast (i1 (i8*)* @vf0i1 to i8*),
@@ -26,17 +26,17 @@ i8* bitcast (i1 (i8*)* @vf1i1 to i8*),
 i8* bitcast (i32 (i8*)* @vf3i32 to i8*)
 ], !type !1
 
-; CHECK: [[VT4DATA:@[^ ]*]] = private constant { [0 x i8], [3 x i8*], [8 x i8] } { [0 x i8] zeroinitializer, [3 x i8*] [i8* bitcast (i1 (i8*)* @vf1i1 to i8*), i8* bitcast (i1 (i8*)* @vf0i1 to i8*), i8* bitcast (i32 (i8*)* @vf4i32 to i8*)], [8 x i8] c"\04\00\00\00\02\00\00\00" }, !type [[T0]]
+; CHECK: [[VT4DATA:@[^ ]*]] = private constant { [0 x i8], [3 x i8*], [5 x i8] } { [0 x i8] zeroinitializer, [3 x i8*] [i8* bitcast (i1 (i8*)* @vf1i1 to i8*), i8* bitcast (i1 (i8*)* @vf0i1 to i8*), i8* bitcast (i32 (i8*)* @vf4i32 to i8*)], [5 x i8] c"\04\00\00\00\02" }, !type [[T0]]
 @vt4 = constant [3 x i8*] [
 i8* bitcast (i1 (i8*)* @vf1i1 to i8*),
 i8* bitcast (i1 (i8*)* @vf0i1 to i8*),
 i8* bitcast (i32 (i8*)* @vf4i32 to i8*)
 ], !type !0
 
-; CHECK: @vt1 = alias [4 x i8*], getelementptr inbounds ({ [0 x i8], [4 x i8*], [8 x i8] }, { [0 x i8], [4 x i8*], [8 x i8] }* [[VT1DATA]], i32 0, i32 1)
-; CHECK: @vt2 = alias [3 x i8*], getelementptr inbounds ({ [0 x i8], [3 x i8*], [8 x i8] }, { [0 x i8], [3 x i8*], [8 x i8] }* [[VT2DATA]], i32 0, i32 1)
-; CHECK: @vt3 = alias [4 x i8*], getelementptr inbounds ({ [0 x i8], [4 x i8*], [8 x i8] }, { [0 x i8], [4 x i8*], [8 x i8] }* [[VT3DATA]], i32 0, i32 1)
-; CHECK: @vt4 = alias [3 x i8*], getelementptr inbounds ({ [0 x i8], [3 x i8*], [8 x i8] }, { [0 x i8], [3 x i8*], [8 x i8] }* [[VT4DATA]], i32 0, i32 1)
+; CHECK: @vt1 = alias [4 x i8*], getelementptr inbounds ({ [0 x i8], [4 x i8*], [5 x i8] }, { [0 x i8], [4 x i8*], [5 x i8] }* [[VT1DATA]], i32 0, i32 1)
+; CHECK: @vt2 = alias [3 x i8*], getelementptr inbounds ({ [0 x i8], [3 x i8*], [5 x i8] }, { [0 x i8], [3 x i8*], [5 x i8] }* [[VT2DATA]], i32 0, i32 1)
+; CHECK: @vt3 = alias [4 x i8*], getelementptr inbounds ({ [0 x i8], [4 x i8*], [5 x i8] }, { [0 x i8], [4 x i8*], [5 x i8] }* [[VT3DATA]], i32 0, i32 1)
+; CHECK: @vt4 = alias [3 x i8*], getelementptr inbounds ({ [0 x i8], [3 x i8*], [5 x i8] }, { [0 x i8], [3 x i8*], [5 x i8] }* [[VT4DATA]], i32 0, i32 1)
 
 define i1 @vf0i1(i8* %this) readnone {
   ret i1 0
diff --git a/llvm/test/Verifier/blockbyref.ll b/llvm/test/Verifier/blockbyref.ll
index 037da7f09..14d705647 100644
--- a/llvm/test/Verifier/blockbyref.ll
+++ b/llvm/test/Verifier/blockbyref.ll
@@ -1,6 +1,6 @@
 ; RUN: llvm-as -disable-output <%s 2>&1| FileCheck %s
 
-; CHECK: BlockByRef variable without complex expression
+; CHECK: DIBlockByRefStruct on DICompositeType is no longer supported
 ; CHECK: warning: ignoring invalid debug info
 
 define void @foo() {
@@ -16,4 +16,4 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata)
 !0 = !{i32 2, !"Debug Info Version", i32 3}
 !1 = distinct !DISubprogram()
 !2 = !DILocalVariable(scope: !1, type: !3)
-!3 = !DICompositeType(tag: DW_TAG_structure_type, flags: DIFlagBlockByrefStruct)
+!3 = !DICompositeType(tag: DW_TAG_structure_type, flags: DIFlagReservedBit4)
diff --git a/llvm/test/Verifier/callbr.ll b/llvm/test/Verifier/callbr.ll
new file mode 100644
index 000000000..403cc573d
--- /dev/null
+++ b/llvm/test/Verifier/callbr.ll
@@ -0,0 +1,50 @@
+; RUN: not opt -S %s -verify 2>&1 | FileCheck %s
+
+; CHECK: Indirect label missing from arglist.
+define void @foo() {
+  ; The %4 in the indirect label list is not found in the blockaddresses in the
+  ; arg list (bad).
+  callbr void asm sideeffect "${0:l} {1:l}", "X,X"(i8* blockaddress(@foo, %3), i8* blockaddress(@foo, %2))
+  to label %1 [label %4, label %2]
+1:
+  ret void
+2:
+  ret void
+3:
+  ret void
+4:
+  ret void
+}
+
+; CHECK-NOT: Indirect label missing from arglist.
+define void @bar() {
+  ; %4 and %2 are both in the indirect label list and the arg list (good).
+  callbr void asm sideeffect "${0:l} ${1:l}", "X,X"(i8* blockaddress(@bar, %4), i8* blockaddress(@bar, %2))
+  to label %1 [label %4, label %2]
+1:
+  ret void
+2:
+  ret void
+3:
+  ret void
+4:
+  ret void
+}
+
+; CHECK-NOT: Indirect label missing from arglist.
+define void @baz() {
+  ; note %2 blockaddress. Such a case is possible when passing the address of
+  ; a label as an input to the inline asm (both address of label and asm goto
+  ; use blockaddress constants; we're testing that the indirect label list from
+  ; the asm goto is in the arg list to the asm).
+  callbr void asm sideeffect "${0:l} ${1:l} ${2:l}", "X,X,X"(i8* blockaddress(@baz, %4), i8* blockaddress(@baz, %2), i8* blockaddress(@baz, %3))
+  to label %1 [label %3, label %4]
+1:
+  ret void
+2:
+  ret void
+3:
+  ret void
+4:
+  ret void
+}
diff --git a/llvm/test/Verifier/comdat2.ll.x b/llvm/test/Verifier/comdat2.ll.x
index 9d892b974..b461d5846 100644
--- a/llvm/test/Verifier/comdat2.ll.x
+++ b/llvm/test/Verifier/comdat2.ll.x
@@ -1,4 +1,6 @@
-; RUN: not llvm-as %s -o /dev/null 2>&1 | FileCheck %s
+; RUN: llvm-as %s -o /dev/null
+; RUN: opt -mtriple=x86_64-unknown-linux -o /dev/null
+; RUN: not opt -mtriple=x86_64-pc-win32 %s -o /dev/null 2>&1 | FileCheck %s
 
 $v = comdat any
 @v = private global i32 0, comdat($v)
diff --git a/llvm/test/Verifier/dereferenceable-md-inttoptr.ll b/llvm/test/Verifier/dereferenceable-md-inttoptr.ll
new file mode 100644
index 000000000..8c3c57e7d
--- /dev/null
+++ b/llvm/test/Verifier/dereferenceable-md-inttoptr.ll
@@ -0,0 +1,6 @@
+; RUN: llvm-as < %s -o /dev/null
+
+define i8* @f_0(i8 %val) {
+  %ptr = inttoptr i8 %val to i8*, !dereferenceable_or_null !{i64 2}
+  ret i8* %ptr 
+}
diff --git a/llvm/test/Verifier/dereferenceable-md.ll.x b/llvm/test/Verifier/dereferenceable-md.ll.x
index 94c89c332..d1b38abd8 100644
--- a/llvm/test/Verifier/dereferenceable-md.ll.x
+++ b/llvm/test/Verifier/dereferenceable-md.ll.x
@@ -7,7 +7,7 @@ entry:
   call i8* @foo(), !dereferenceable !{i64 2}
   ret void
 }
-; CHECK: dereferenceable, dereferenceable_or_null apply only to load instructions, use attributes for calls or invokes
+; CHECK: dereferenceable, dereferenceable_or_null apply only to load and inttoptr instructions, use attributes for calls or invokes
 ; CHECK-NEXT: call i8* @foo()
 
 define void @f2() {
@@ -15,7 +15,7 @@ entry:
   call i8* @foo(), !dereferenceable_or_null !{i64 2}
   ret void
 }
-; CHECK: dereferenceable, dereferenceable_or_null apply only to load instructions, use attributes for calls or invokes
+; CHECK: dereferenceable, dereferenceable_or_null apply only to load and inttoptr instructions, use attributes for calls or invokes
 ; CHECK-NEXT: call i8* @foo()
 
 define i8 @f3(i8* %x) {
@@ -83,4 +83,46 @@ entry:
   ret i8* %y
 }
 ; CHECK: dereferenceable, dereferenceable_or_null metadata value must be an i64!
-; CHECK-NEXT: load i8*, i8** %x
\ No newline at end of file
+; CHECK-NEXT: load i8*, i8** %x
+
+define i8* @f_11(i8 %val) {
+  %ptr = inttoptr i8 %val to i8*, !dereferenceable !{i32 2}
+  ret i8* %ptr
+}
+; CHECK: dereferenceable, dereferenceable_or_null metadata value must be an i64!
+; CHECK-NEXT: %ptr = inttoptr i8 %val to i8*, !dereferenceable !3
+
+define i8* @f_12(i8 %val) {
+  %ptr = inttoptr i8 %val to i8*, !dereferenceable_or_null !{i32 2}
+  ret i8* %ptr
+}
+; CHECK: dereferenceable, dereferenceable_or_null metadata value must be an i64!
+; CHECK-NEXT: %ptr = inttoptr i8 %val to i8*, !dereferenceable_or_null !3
+
+define i8* @f_13(i8 %val) {
+  %ptr = inttoptr i8 %val to i8*, !dereferenceable !{}
+  ret i8* %ptr
+}
+; CHECK: dereferenceable, dereferenceable_or_null take one operand
+; CHECK-NEXT: %ptr = inttoptr i8 %val to i8*, !dereferenceable !1
+
+define i8* @f_14(i8 %val) {
+  %ptr = inttoptr i8 %val to i8*, !dereferenceable_or_null !{}
+  ret i8* %ptr
+}
+; CHECK: dereferenceable, dereferenceable_or_null take one operand
+; CHECK-NEXT: %ptr = inttoptr i8 %val to i8*, !dereferenceable_or_null !1
+
+define i8* @f_15(i8 %val) {
+  %ptr = inttoptr i8 %val to i8*, !dereferenceable !{!"str"}
+  ret i8* %ptr
+}
+; CHECK: dereferenceable, dereferenceable_or_null metadata value must be an i64!
+; CHECK-NEXT: %ptr = inttoptr i8 %val to i8*, !dereferenceable !2
+
+define i8* @f_16(i8 %val) {
+  %ptr = inttoptr i8 %val to i8*, !dereferenceable_or_null !{!"str"}
+  ret i8* %ptr
+}
+; CHECK: dereferenceable, dereferenceable_or_null metadata value must be an i64!
+; CHECK-NEXT: %ptr = inttoptr i8 %val to i8*, !dereferenceable_or_null !2
diff --git a/llvm/test/Verifier/diexpression-dwarf-entry-value.ll b/llvm/test/Verifier/diexpression-dwarf-entry-value.ll
new file mode 100644
index 000000000..cc7b41da4
--- /dev/null
+++ b/llvm/test/Verifier/diexpression-dwarf-entry-value.ll
@@ -0,0 +1,8 @@
+; RUN: not opt -S < %s 2>&1 | FileCheck %s
+
+; We can only use the internal variant of the entry value operation,
+; DW_OP_LLVM_entry_value, in DIExpressions.
+
+!named = !{!0}
+; CHECK: invalid expression
+!0 = !DIExpression(DW_OP_entry_value, 1)
diff --git a/llvm/test/Verifier/diexpression-entry-value-llvm-ir.ll b/llvm/test/Verifier/diexpression-entry-value-llvm-ir.ll
new file mode 100644
index 000000000..99143297a
--- /dev/null
+++ b/llvm/test/Verifier/diexpression-entry-value-llvm-ir.ll
@@ -0,0 +1,31 @@
+; RUN: llvm-as -disable-output <%s 2>&1| FileCheck %s
+
+; The DW_OP_LLVM_entry_value operation can only be used in MIR.
+
+; CHECK: Entry values are only allowed in MIR
+; CHECK: call void @llvm.dbg.value(metadata i32 %param, metadata !{{.*}}, metadata !DIExpression(DW_OP_LLVM_entry_value, 1))
+; CHECK: warning: ignoring invalid debug info
+
+define void @foo(i32 %param) !dbg !4 {
+entry:
+  call void @llvm.dbg.value(metadata i32 %param, metadata !8, metadata !DIExpression(DW_OP_LLVM_entry_value, 1)), !dbg !9
+  ret void
+}
+
+declare void @llvm.dbg.value(metadata, metadata, metadata) #0
+
+attributes #0 = { nounwind readnone speculatable willreturn }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!2, !3}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, emissionKind: FullDebug)
+!1 = !DIFile(filename: "a.c", directory: "/")
+!2 = !{i32 2, !"Dwarf Version", i32 4}
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!4 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, type: !5, unit: !0)
+!5 = !DISubroutineType(types: !6)
+!6 = !{null, !7}
+!7 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!8 = !DILocalVariable(name: "param", arg: 1, scope: !4, file: !1, type: !7)
+!9 = !DILocation(line: 0, scope: !4)
diff --git a/llvm/test/Verifier/diexpression-entry-value.ll.x b/llvm/test/Verifier/diexpression-entry-value.ll.x
index 477b20079..34b1bceb0 100644
--- a/llvm/test/Verifier/diexpression-entry-value.ll.x
+++ b/llvm/test/Verifier/diexpression-entry-value.ll.x
@@ -2,6 +2,11 @@
 
 !named = !{!0, !1, !2}
 ; CHECK: invalid expression
-!0 = !DIExpression(DW_OP_entry_value, 4, DW_OP_constu, 0, DW_OP_stack_value)
-!1 = !DIExpression(DW_OP_constu, 0, DW_OP_entry_value, 1, DW_OP_constu, 0)
-!2 = !DIExpression(DW_OP_entry_value, 100, DW_OP_constu, 0)
+; CHECK-NEXT: !DIExpression
+; CHECK: invalid expression
+; CHECK-NEXT: !DIExpression
+; CHECK: invalid expression
+; CHECK-NEXT: !DIExpression
+!0 = !DIExpression(DW_OP_LLVM_entry_value, 4, DW_OP_constu, 0, DW_OP_stack_value)
+!1 = !DIExpression(DW_OP_constu, 0, DW_OP_LLVM_entry_value, 1, DW_OP_constu, 0)
+!2 = !DIExpression(DW_OP_LLVM_entry_value, 100, DW_OP_constu, 0)
diff --git a/llvm/test/Verifier/diexpression-valid-entry-value.ll b/llvm/test/Verifier/diexpression-valid-entry-value.ll
index e162b8a50..9a4b64ddb 100644
--- a/llvm/test/Verifier/diexpression-valid-entry-value.ll
+++ b/llvm/test/Verifier/diexpression-valid-entry-value.ll
@@ -2,4 +2,4 @@
 
 !named = !{!0}
 ; CHECK-NOT: invalid expression
-!0 = !DIExpression(DW_OP_entry_value, 1)
+!0 = !DIExpression(DW_OP_LLVM_entry_value, 1)
diff --git a/llvm/test/Verifier/fp-intrinsics.ll b/llvm/test/Verifier/fp-intrinsics.ll
index cc3c3bc2e..12a3c9d65 100644
--- a/llvm/test/Verifier/fp-intrinsics.ll
+++ b/llvm/test/Verifier/fp-intrinsics.ll
@@ -12,67 +12,69 @@ declare double @llvm.experimental.constrained.sqrt.f64(double, metadata, metadat
 ; attached to the FP intrinsic.
 ; CHECK1: declare double @llvm.experimental.constrained.fadd.f64(double, double, metadata, metadata) #[[ATTR:[0-9]+]]
 ; CHECK1: declare double @llvm.experimental.constrained.sqrt.f64(double, metadata, metadata) #[[ATTR]]
-; CHECK1: attributes #[[ATTR]] = { inaccessiblememonly nounwind }
+; CHECK1: attributes #[[ATTR]] = { inaccessiblememonly nounwind willreturn }
 ; Note: FP exceptions aren't usually caught through normal unwind mechanisms,
 ;       but we may want to revisit this for asynchronous exception handling.
-define double @f1(double %a, double %b) {
+define double @f1(double %a, double %b) #0 {
 entry:
   %fadd = call double @llvm.experimental.constrained.fadd.f64(
                                                double %a, double %b,
                                                metadata !"round.dynamic",
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
   ret double %fadd
 }
 
-define double @f1u(double %a) {
+define double @f1u(double %a) #0 {
 entry:
   %fsqrt = call double @llvm.experimental.constrained.sqrt.f64(
                                                double %a,
                                                metadata !"round.dynamic",
-                                               metadata !"fpexcept.strict")
+                                               metadata !"fpexcept.strict") #0
   ret double %fsqrt
 }
 
 ; Test an illegal value for the rounding mode argument.
 ; CHECK2: invalid rounding mode argument
-;T2: define double @f2(double %a, double %b) {
+;T2: define double @f2(double %a, double %b) #0 {
 ;T2: entry:
 ;T2:   %fadd = call double @llvm.experimental.constrained.fadd.f64(
 ;T2:                                           double %a, double %b,
 ;T2:                                           metadata !"round.dynomite",
-;T2:                                           metadata !"fpexcept.strict")
+;T2:                                           metadata !"fpexcept.strict") #0
 ;T2:   ret double %fadd
 ;T2: }
 
 ; Test an illegal value for the exception behavior argument.
 ; CHECK3: invalid exception behavior argument
-;T3: define double @f3(double %a, double %b) {
+;T3: define double @f3(double %a, double %b) #0 {
 ;T3: entry:
 ;T3:   %fadd = call double @llvm.experimental.constrained.fadd.f64(
 ;T3:                                         double %a, double %b,
 ;T3:                                         metadata !"round.dynamic",
-;T3:                                         metadata !"fpexcept.restrict")
+;T3:                                         metadata !"fpexcept.restrict") #0
 ;T3:   ret double %fadd
 ;T3: }
 
 ; Test an illegal value for the rounding mode argument.
 ; CHECK4: invalid rounding mode argument
-;T4: define double @f4(double %a) {
+;T4: define double @f4(double %a) #0 {
 ;T4: entry:
 ;T4:   %fadd = call double @llvm.experimental.constrained.sqrt.f64(
 ;T4:                                           double %a,
 ;T4:                                           metadata !"round.dynomite",
-;T4:                                           metadata !"fpexcept.strict")
+;T4:                                           metadata !"fpexcept.strict") #0
 ;T4:   ret double %fadd
 ;T4: }
 
 ; Test an illegal value for the exception behavior argument.
 ; CHECK5: invalid exception behavior argument
-;T5: define double @f5(double %a) {
+;T5: define double @f5(double %a) #0 {
 ;T5: entry:
 ;T5:   %fadd = call double @llvm.experimental.constrained.sqrt.f64(
 ;T5:                                         double %a,
 ;T5:                                         metadata !"round.dynamic",
-;T5:                                         metadata !"fpexcept.restrict")
+;T5:                                         metadata !"fpexcept.restrict") #0
 ;T5:   ret double %fadd
 ;T5: }
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/Verifier/intrinsic-bad-arg-type.ll b/llvm/test/Verifier/intrinsic-bad-arg-type.ll
new file mode 100644
index 000000000..1a6739e67
--- /dev/null
+++ b/llvm/test/Verifier/intrinsic-bad-arg-type.ll
@@ -0,0 +1,10 @@
+; RUN: not opt -S -verify 2>&1 < %s | FileCheck %s
+
+; CHECK: Intrinsic has incorrect argument type!
+; CHECK-NEXT: <vscale x 4 x i32> (<vscale x 4 x i32>*, i32, <4 x i1>, <vscale x 4 x i32>)* @llvm.masked.load.nxv4i32.p0nxv4i32
+
+define <vscale x 4 x i32> @masked_load(<vscale x 4 x i32>* %addr, <4 x i1> %mask, <vscale x 4 x i32> %dst) {
+  %res = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0nxv4i32(<vscale x 4 x i32>* %addr, i32 4, <4 x i1> %mask, <vscale x 4 x i32> %dst)
+  ret <vscale x 4 x i32> %res
+}
+declare <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0nxv4i32(<vscale x 4 x i32>*, i32, <4 x i1>, <vscale x 4 x i32>)
diff --git a/llvm/test/Verifier/intrinsic-immarg.ll.x b/llvm/test/Verifier/intrinsic-immarg.ll.x
index 0be3999f7..915432a93 100644
--- a/llvm/test/Verifier/intrinsic-immarg.ll.x
+++ b/llvm/test/Verifier/intrinsic-immarg.ll.x
@@ -13,7 +13,7 @@ declare i8* @llvm.frameaddress(i32)
 define void @frame_address(i32 %var) {
   ; CHECK: immarg operand has non-immediate parameter
   ; CHECK-NEXT: i32 %var
-  ; CHECK-NEXT: %result = call i8* @llvm.frameaddress(i32 %var)
+  ; CHECK-NEXT: %result = call i8* @llvm.frameaddress.p0i8(i32 %var)
   %result = call i8* @llvm.frameaddress(i32 %var)
   ret void
 }
@@ -74,6 +74,15 @@ define i64 @smul_fix(i64 %arg0, i64 %arg1, i32 %arg2) {
   ret i64 %ret
 }
 
+declare i64 @llvm.smul.fix.sat.i64(i64, i64, i32)
+define i64 @smul_fix_sat(i64 %arg0, i64 %arg1, i32 %arg2) {
+  ; CHECK: immarg operand has non-immediate parameter
+  ; CHECK-NEXT: i32 %arg2
+  ; CHECK-NEXT: %ret = call i64 @llvm.smul.fix.sat.i64(i64 %arg0, i64 %arg1, i32 %arg2)
+  %ret = call i64 @llvm.smul.fix.sat.i64(i64 %arg0, i64 %arg1, i32 %arg2)
+  ret i64 %ret
+}
+
 declare i64 @llvm.umul.fix.i64(i64, i64, i32)
 define i64 @umul_fix(i64 %arg0, i64 %arg1, i32 %arg2) {
   ; CHECK: immarg operand has non-immediate parameter
@@ -83,6 +92,15 @@ define i64 @umul_fix(i64 %arg0, i64 %arg1, i32 %arg2) {
   ret i64 %ret
 }
 
+declare i64 @llvm.umul.fix.sat.i64(i64, i64, i32)
+define i64 @umul_fix_sat(i64 %arg0, i64 %arg1, i32 %arg2) {
+  ; CHECK: immarg operand has non-immediate parameter
+  ; CHECK-NEXT: i32 %arg2
+  ; CHECK-NEXT: %ret = call i64 @llvm.umul.fix.sat.i64(i64 %arg0, i64 %arg1, i32 %arg2)
+  %ret = call i64 @llvm.umul.fix.sat.i64(i64 %arg0, i64 %arg1, i32 %arg2)
+  ret i64 %ret
+}
+
 declare <2 x double> @llvm.masked.load.v2f64.p0v2f64(<2 x double>*, i32, <2 x i1>, <2 x double>)
 define <2 x double> @masked_load(<2 x i1> %mask, <2 x double>* %addr, <2 x double> %dst, i32 %align) {
   ; CHECK: immarg operand has non-immediate parameter
@@ -159,7 +177,7 @@ declare void @llvm.prefetch(i8*, i32, i32, i32)
 define void @test_prefetch(i8* %ptr, i32 %arg0, i32 %arg1) {
   ; CHECK: immarg operand has non-immediate parameter
   ; CHECK-NEXT: i32 %arg0
-  ; CHECK-NEXT: call void @llvm.prefetch(i8* %ptr, i32 %arg0, i32 0, i32 0)
+  ; CHECK-NEXT: call void @llvm.prefetch.p0i8(i8* %ptr, i32 %arg0, i32 0, i32 0)
   ; CHECK: immarg operand has non-immediate parameter
   ; CHECK-NEXT:  i32 %arg1
   call void @llvm.prefetch(i8* %ptr, i32 %arg0, i32 0, i32 0)
diff --git a/llvm/test/Verifier/invalid-frame-pointer-attr-empty.ll b/llvm/test/Verifier/invalid-frame-pointer-attr-empty.ll
new file mode 100644
index 000000000..b1cd2a3a6
--- /dev/null
+++ b/llvm/test/Verifier/invalid-frame-pointer-attr-empty.ll
@@ -0,0 +1,9 @@
+; RUN: not llvm-as < %s -o /dev/null 2>&1 | FileCheck %s
+
+; CHECK: invalid value for 'frame-pointer' attribute:
+
+define void @func() #0 {
+  ret void
+}
+
+attributes #0 = { "frame-pointer"="" }
diff --git a/llvm/test/Verifier/invalid-frame-pointer-attr-no-value.ll b/llvm/test/Verifier/invalid-frame-pointer-attr-no-value.ll
new file mode 100644
index 000000000..8ba4d4eec
--- /dev/null
+++ b/llvm/test/Verifier/invalid-frame-pointer-attr-no-value.ll
@@ -0,0 +1,9 @@
+; RUN: not llvm-as < %s -o /dev/null 2>&1 | FileCheck %s
+
+; CHECK: invalid value for 'frame-pointer' attribute:
+
+define void @func() #0 {
+  ret void
+}
+
+attributes #0 = { "frame-pointer" }
diff --git a/llvm/test/Verifier/invalid-frame-pointer-attr.ll b/llvm/test/Verifier/invalid-frame-pointer-attr.ll
new file mode 100644
index 000000000..d589b96ac
--- /dev/null
+++ b/llvm/test/Verifier/invalid-frame-pointer-attr.ll
@@ -0,0 +1,9 @@
+; RUN: not llvm-as < %s -o /dev/null 2>&1 | FileCheck %s
+
+; CHECK: invalid value for 'frame-pointer' attribute: arst
+
+define void @func() #0 {
+  ret void
+}
+
+attributes #0 = { "frame-pointer"="arst" }
diff --git a/llvm/test/Verifier/invalid-patchable-function-entry.ll b/llvm/test/Verifier/invalid-patchable-function-entry.ll
new file mode 100644
index 000000000..e74037a28
--- /dev/null
+++ b/llvm/test/Verifier/invalid-patchable-function-entry.ll
@@ -0,0 +1,21 @@
+; RUN: not llvm-as %s -o /dev/null 2>&1 | FileCheck %s
+
+; CHECK: "patchable-function-entry" takes an unsigned integer:
+; CHECK: "patchable-function-entry" takes an unsigned integer: a
+; CHECK: "patchable-function-entry" takes an unsigned integer: -1
+; CHECK: "patchable-function-entry" takes an unsigned integer: 3,
+
+define void @f() "patchable-function-entry" { ret void }
+define void @fa() "patchable-function-entry"="a" { ret void }
+define void @f_1() "patchable-function-entry"="-1" { ret void }
+define void @f3comma() "patchable-function-entry"="3," { ret void }
+
+; CHECK: "patchable-function-prefix" takes an unsigned integer:
+; CHECK: "patchable-function-prefix" takes an unsigned integer: a
+; CHECK: "patchable-function-prefix" takes an unsigned integer: -1
+; CHECK: "patchable-function-prefix" takes an unsigned integer: 3,
+
+define void @g() "patchable-function-prefix" { ret void }
+define void @ga() "patchable-function-prefix"="a" { ret void }
+define void @g_1() "patchable-function-prefix"="-1" { ret void }
+define void @g3comma() "patchable-function-prefix"="3," { ret void }
diff --git a/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/basic.ll b/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/basic.ll
new file mode 100644
index 000000000..d95d94823
--- /dev/null
+++ b/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/basic.ll
@@ -0,0 +1,32 @@
+; Example input for update_llc_test_checks (taken from CodeGen/X86/iabs.ll)
+; RUN: llc < %s -mtriple=i686-unknown-unknown | FileCheck %s --check-prefix=X86 --check-prefix=X86-NO-CMOV
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+cmov | FileCheck %s --check-prefix=X86 --check-prefix=X86-CMOV
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s --check-prefix=X64
+
+define i8 @test_i8(i8 %a) nounwind {
+  %tmp1neg = sub i8 0, %a
+  %b = icmp sgt i8 %a, -1
+  %abs = select i1 %b, i8 %a, i8 %tmp1neg
+  ret i8 %abs
+}
+
+define i16 @test_i16(i16 %a) nounwind {
+  %tmp1neg = sub i16 0, %a
+  %b = icmp sgt i16 %a, -1
+  %abs = select i1 %b, i16 %a, i16 %tmp1neg
+  ret i16 %abs
+}
+
+define i32 @test_i32(i32 %a) nounwind {
+  %tmp1neg = sub i32 0, %a
+  %b = icmp sgt i32 %a, -1
+  %abs = select i1 %b, i32 %a, i32 %tmp1neg
+  ret i32 %abs
+}
+
+define i64 @test_i64(i64 %a) nounwind {
+  %tmp1neg = sub i64 0, %a
+  %b = icmp sgt i64 %a, -1
+  %abs = select i1 %b, i64 %a, i64 %tmp1neg
+  ret i64 %abs
+}
diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/argument_name_reuse.ll b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/argument_name_reuse.ll
new file mode 100644
index 000000000..c5bcee5ba
--- /dev/null
+++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/argument_name_reuse.ll
@@ -0,0 +1,7 @@
+; RUN: opt < %s -S | FileCheck %s
+
+define i32 @reuse_arg_names(i32 %X, i32 %Y) {
+  %Z = sub i32 %X, %Y
+  %Q = add i32 %Z, %Y
+  ret i32 %Q
+}
diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/basic.ll b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/basic.ll
new file mode 100644
index 000000000..7189eb485
--- /dev/null
+++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/basic.ll
@@ -0,0 +1,49 @@
+; Example input for update_llc_test_checks (taken from test/Transforms/InstSimplify/add.ll)
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+define i32 @common_sub_operand(i32 %X, i32 %Y) {
+; CHECK-LABEL: @common_sub_operand(
+; CHECK-NEXT:    ret i32 [[X:%.*]]
+;
+  %Z = sub i32 %X, %Y
+  %Q = add i32 %Z, %Y
+  ret i32 %Q
+}
+
+define i32 @negated_operand(i32 %x) {
+; CHECK-LABEL: @negated_operand(
+; CHECK-NEXT:    ret i32 0
+;
+  %negx = sub i32 0, %x
+  %r = add i32 %negx, %x
+  ret i32 %r
+}
+
+define <2 x i32> @negated_operand_commute_vec(<2 x i32> %x) {
+; CHECK-LABEL: @negated_operand_commute_vec(
+; CHECK-NEXT:    ret <2 x i32> zeroinitializer
+;
+  %negx = sub <2 x i32> zeroinitializer, %x
+  %r = add <2 x i32> %x, %negx
+  ret <2 x i32> %r
+}
+
+define i8 @knownnegation(i8 %x, i8 %y) {
+; CHECK-LABEL: @knownnegation(
+; CHECK-NEXT:    ret i8 0
+;
+  %xy = sub i8 %x, %y
+  %yx = sub i8 %y, %x
+  %r = add i8 %xy, %yx
+  ret i8 %r
+}
+
+define <2 x i8> @knownnegation_commute_vec(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @knownnegation_commute_vec(
+; CHECK-NEXT:    ret <2 x i8> zeroinitializer
+;
+  %xy = sub <2 x i8> %x, %y
+  %yx = sub <2 x i8> %y, %x
+  %r = add <2 x i8> %yx, %xy
+  ret <2 x i8> %r
+}
diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/scrub_attrs.ll b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/scrub_attrs.ll
new file mode 100644
index 000000000..2733cfbce
--- /dev/null
+++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/scrub_attrs.ll
@@ -0,0 +1,8 @@
+; RUN: opt -S < %s | FileCheck %s
+
+declare void @foo()
+
+define internal void @bar() {
+  call void @foo() readnone
+  ret void
+}
diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/sometimes_deleted_function.ll b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/sometimes_deleted_function.ll
new file mode 100644
index 000000000..6b82c49f0
--- /dev/null
+++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/sometimes_deleted_function.ll
@@ -0,0 +1,12 @@
+; RUN: opt -S < %s | FileCheck %s --check-prefixes=ALL,FIRST
+; RUN: opt -S -globalopt < %s | FileCheck %s --check-prefixes=ALL,SECOND
+;
+; Make sure we use FIRST to check for @sometimes_here as ALL does not work.
+
+define internal void @sometimes_here() {
+  ret void
+}
+
+define void @always_here() {
+  ret void
+}
diff --git a/llvm/test/tools/dsymutil/Inputs/frame-dw2.ll b/llvm/test/tools/dsymutil/Inputs/frame-dw2.ll
index 0e77cc584..2ed26e998 100644
--- a/llvm/test/tools/dsymutil/Inputs/frame-dw2.ll
+++ b/llvm/test/tools/dsymutil/Inputs/frame-dw2.ll
@@ -34,9 +34,9 @@ entry:
   ret i32 %call, !dbg !26
 }
 
-attributes #0 = { nounwind ssp "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="yonah" "target-features"="+cx16,+sse,+sse2,+sse3" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind ssp "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="yonah" "target-features"="+cx16,+sse,+sse2,+sse3" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
-attributes #2 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="yonah" "target-features"="+cx16,+sse,+sse2,+sse3" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="yonah" "target-features"="+cx16,+sse,+sse2,+sse3" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!9, !10, !11}
diff --git a/llvm/test/tools/dsymutil/Inputs/frame-dw4.ll b/llvm/test/tools/dsymutil/Inputs/frame-dw4.ll
index 070ed8844..376137bf0 100644
--- a/llvm/test/tools/dsymutil/Inputs/frame-dw4.ll
+++ b/llvm/test/tools/dsymutil/Inputs/frame-dw4.ll
@@ -34,9 +34,9 @@ entry:
   ret i32 %call, !dbg !26
 }
 
-attributes #0 = { nounwind ssp "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="yonah" "target-features"="+cx16,+sse,+sse2,+sse3" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind ssp "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="yonah" "target-features"="+cx16,+sse,+sse2,+sse3" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
-attributes #2 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="yonah" "target-features"="+cx16,+sse,+sse2,+sse3" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="yonah" "target-features"="+cx16,+sse,+sse2,+sse3" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!9, !10, !11}
diff --git a/llvm/test/tools/gold/X86/linkonce_odr_unnamed_addr.ll b/llvm/test/tools/gold/X86/linkonce_odr_unnamed_addr.ll
index 525bf2d68..f8df9430e 100644
--- a/llvm/test/tools/gold/X86/linkonce_odr_unnamed_addr.ll
+++ b/llvm/test/tools/gold/X86/linkonce_odr_unnamed_addr.ll
@@ -4,6 +4,7 @@
 ; RUN: opt -module-summary %s -o %t.o
 ; RUN: opt -module-summary %p/Inputs/linkonce_odr_unnamed_addr.ll -o %t2.o
 ; RUN: %gold -plugin %llvmshlibdir/LLVMgold%shlibext \
+; RUN:    -m elf_x86_64 \
 ; RUN:    --plugin-opt=save-temps \
 ; RUN:    %t.o %t2.o -o %t3.o
 ; RUN: llvm-dis %t.o.1.promote.bc -o - | FileCheck %s
@@ -12,6 +13,7 @@
 ; conservative and not auto hide.
 ; RUN: llc %p/Inputs/linkonce_odr_unnamed_addr.ll -o %t2native.o -filetype=obj
 ; RUN: %gold -plugin %llvmshlibdir/LLVMgold%shlibext \
+; RUN:    -m elf_x86_64 \
 ; RUN:    --plugin-opt=save-temps \
 ; RUN:    %t.o %t2native.o -o %t3.o
 ; RUN: llvm-dis %t.o.1.promote.bc -o - | FileCheck %s --check-prefix=NOSUMMARY
diff --git a/llvm/test/tools/gold/X86/slp-vectorize-pm.ll b/llvm/test/tools/gold/X86/slp-vectorize-pm.ll
new file mode 100644
index 000000000..26d11fe8a
--- /dev/null
+++ b/llvm/test/tools/gold/X86/slp-vectorize-pm.ll
@@ -0,0 +1,79 @@
+; RUN: opt -module-summary %s -o %t.o
+
+; Test SLP and Loop Vectorization are enabled by default at O2 and O3.
+; RUN: %gold -m elf_x86_64 -plugin %llvmshlibdir/LLVMgold%shlibext \
+; RUN:     --plugin-opt=thinlto \
+; RUN:     --plugin-opt=new-pass-manager \
+; RUN:     --plugin-opt=debug-pass-manager \
+; RUN:     --plugin-opt=cache-dir=%t.cache \
+; RUN:     --plugin-opt=O0 \
+; RUN:     --plugin-opt=save-temps \
+; RUN:     -shared \
+; RUN:     -o %t2.o %t.o 2>&1 | FileCheck %s --check-prefix=CHECK-O0-SLP
+; RUN: llvm-dis %t.o.4.opt.bc -o - | FileCheck %s --check-prefix=CHECK-O0-LPV
+
+; RUN: %gold -m elf_x86_64 -plugin %llvmshlibdir/LLVMgold%shlibext \
+; RUN:     --plugin-opt=thinlto \
+; RUN:     --plugin-opt=new-pass-manager \
+; RUN:     --plugin-opt=debug-pass-manager \
+; RUN:     --plugin-opt=cache-dir=%t.cache \
+; RUN:     --plugin-opt=O1 \
+; RUN:     --plugin-opt=save-temps \
+; RUN:     -shared \
+; RUN:     -o %t3.o %t.o 2>&1 | FileCheck %s --check-prefix=CHECK-O1-SLP
+; RUN: llvm-dis %t.o.4.opt.bc -o - | FileCheck %s --check-prefix=CHECK-O1-LPV
+
+; RUN: %gold -m elf_x86_64 -plugin %llvmshlibdir/LLVMgold%shlibext \
+; RUN:     --plugin-opt=thinlto \
+; RUN:     --plugin-opt=new-pass-manager \
+; RUN:     --plugin-opt=debug-pass-manager \
+; RUN:     --plugin-opt=cache-dir=%t.cache \
+; RUN:     --plugin-opt=O2 \
+; RUN:     --plugin-opt=save-temps \
+; RUN:     -shared \
+; RUN:     -o %t4.o %t.o 2>&1 | FileCheck %s --check-prefix=CHECK-O2-SLP
+; RUN: llvm-dis %t.o.4.opt.bc -o - | FileCheck %s --check-prefix=CHECK-O2-LPV
+
+; RUN: %gold -m elf_x86_64 -plugin %llvmshlibdir/LLVMgold%shlibext \
+; RUN:     --plugin-opt=thinlto \
+; RUN:     --plugin-opt=new-pass-manager \
+; RUN:     --plugin-opt=debug-pass-manager \
+; RUN:     --plugin-opt=cache-dir=%t.cache \
+; RUN:     --plugin-opt=O3 \
+; RUN:     --plugin-opt=save-temps \
+; RUN:     -shared \
+; RUN:     -o %t5.o %t.o 2>&1 | FileCheck %s --check-prefix=CHECK-O3-SLP
+; RUN: llvm-dis %t.o.4.opt.bc -o - | FileCheck %s --check-prefix=CHECK-O3-LPV
+
+; CHECK-O0-SLP-NOT: Running pass: SLPVectorizerPass
+; CHECK-O1-SLP-NOT: Running pass: SLPVectorizerPass
+; CHECK-O2-SLP: Running pass: SLPVectorizerPass
+; CHECK-O3-SLP: Running pass: SLPVectorizerPass
+; CHECK-O0-LPV-NOT: = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK-O1-LPV-NOT: = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK-O2-LPV: = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK-O3-LPV: = !{!"llvm.loop.isvectorized", i32 1}
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define i32 @foo(i32* %a) {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %red.05 = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add nsw i32 %0, %red.05
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 255
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !0
+
+for.end:
+  ret i32 %add
+}
+
+!0 = distinct !{!0, !1}
+!1 = !{!"llvm.loop.unroll.disable", i1 true}
diff --git a/llvm/test/tools/gold/X86/split-dwarf.ll b/llvm/test/tools/gold/X86/split-dwarf.ll
index 6746dfd0b..2da2712d3 100644
--- a/llvm/test/tools/gold/X86/split-dwarf.ll
+++ b/llvm/test/tools/gold/X86/split-dwarf.ll
@@ -41,7 +41,7 @@ entry:
   ret i32 0, !dbg !11
 }
 
-attributes #0 = { noinline nounwind optnone uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noinline nounwind optnone uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!3, !4, !5}
diff --git a/llvm/test/tools/gold/X86/strip_names.ll b/llvm/test/tools/gold/X86/strip_names.ll
index 1ac6bd776..150754ef7 100644
--- a/llvm/test/tools/gold/X86/strip_names.ll
+++ b/llvm/test/tools/gold/X86/strip_names.ll
@@ -20,7 +20,7 @@
 ; CHECK:  ret i32 %add
 
 ; NONAME: @GlobalValueName
-; NONAME: @foo(i32)
+; NONAME: @foo(i32 %0)
 ; NONAME-NOT: somelabel:
 ; NONAME:  %2 = load i32, i32* @GlobalValueName
 ; NONAME:  %3 = add i32 %0, %2
diff --git a/llvm/test/tools/gold/X86/v1.12/thinlto_emit_linked_objects.ll b/llvm/test/tools/gold/X86/v1.12/thinlto_emit_linked_objects.ll
index b110eb0de..1d585762a 100644
--- a/llvm/test/tools/gold/X86/v1.12/thinlto_emit_linked_objects.ll
+++ b/llvm/test/tools/gold/X86/v1.12/thinlto_emit_linked_objects.ll
@@ -35,7 +35,7 @@
 ; should not be set.
 ; RUN: llvm-bcanalyzer --dump %t1.o.thinlto.bc | FileCheck %s -check-prefixes=CHECK-BC1
 ; CHECK-BC1: <GLOBALVAL_SUMMARY_BLOCK
-; CHECK-BC1: <FLAGS op0=1/>
+; CHECK-BC1: <FLAGS op0=33/>
 ; CHECK-BC1: </GLOBALVAL_SUMMARY_BLOCK
 
 ; Nothing interesting in the corresponding object file, so
diff --git a/llvm/test/tools/llvm-dwarfdump/AArch64/arch.ll b/llvm/test/tools/llvm-dwarfdump/AArch64/arch.ll
index d74b19977..86505a3f9 100644
--- a/llvm/test/tools/llvm-dwarfdump/AArch64/arch.ll
+++ b/llvm/test/tools/llvm-dwarfdump/AArch64/arch.ll
@@ -1,6 +1,5 @@
 ; RUN: llc -O0 %s -filetype=obj -o %t.o
 ; RUN: llvm-dwarfdump -arch arm64   %t.o | FileCheck %s
-; RUN: llvm-dwarfdump -arch aarch64 %t.o | FileCheck %s
 ; RUN: llvm-dwarfdump -arch 0x0100000c %t.o | FileCheck %s
 ; CHECK: file format Mach-O arm64
 ;
diff --git a/llvm/test/tools/llvm-dwarfdump/X86/locstats.ll b/llvm/test/tools/llvm-dwarfdump/X86/locstats.ll
new file mode 100644
index 000000000..7521806c6
--- /dev/null
+++ b/llvm/test/tools/llvm-dwarfdump/X86/locstats.ll
@@ -0,0 +1,211 @@
+; RUN: llc -debug-entry-values %s -o - -filetype=obj \
+; RUN:   | llvm-dwarfdump -statistics - | FileCheck %s
+;
+; CHECK: "entry value scope bytes covered":5
+; CHECK: "formal params scope bytes total":20
+; CHECK: "formal params scope bytes covered":20
+; CHECK: "formal params entry value scope bytes covered":5
+; CHECK: "vars scope bytes total":90
+; CHECK: "vars scope bytes covered":60
+; CHECK: "vars entry value scope bytes covered":0
+; CHECK: "total variables procesed by location statistics":6
+; CHECK: "variables with 0% of its scope covered":1
+; CHECK: "variables with (0%,10%) of its scope covered":0
+; CHECK: "variables with [10%,20%) of its scope covered":0
+; CHECK: "variables with [20%,30%) of its scope covered":0
+; CHECK: "variables with [30%,40%) of its scope covered":0
+; CHECK: "variables with [40%,50%) of its scope covered":0
+; CHECK: "variables with [50%,60%) of its scope covered":1
+; CHECK: "variables with [60%,70%) of its scope covered":0
+; CHECK: "variables with [70%,80%) of its scope covered":0
+; CHECK: "variables with [80%,90%) of its scope covered":1
+; CHECK: "variables with [90%,100%) of its scope covered":0
+; CHECK: "variables with 100% of its scope covered":3
+; CHECK: "variables (excluding the debug entry values) with 0% of its scope covered":1
+; CHECK: "variables (excluding the debug entry values) with (0%,10%) of its scope covered":0
+; CHECK: "variables (excluding the debug entry values) with [10%,20%) of its scope covered":0
+; CHECK: "variables (excluding the debug entry values) with [20%,30%) of its scope covered":0
+; CHECK: "variables (excluding the debug entry values) with [30%,40%) of its scope covered":0
+; CHECK: "variables (excluding the debug entry values) with [40%,50%) of its scope covered":0
+; CHECK: "variables (excluding the debug entry values) with [50%,60%) of its scope covered":2
+; CHECK: "variables (excluding the debug entry values) with [60%,70%) of its scope covered":0
+; CHECK: "variables (excluding the debug entry values) with [70%,80%) of its scope covered":0
+; CHECK: "variables (excluding the debug entry values) with [80%,90%) of its scope covered":1
+; CHECK: "variables (excluding the debug entry values) with [90%,100%) of its scope covered":0
+; CHECK: "variables (excluding the debug entry values) with 100% of its scope covered":2
+; CHECK: "total params procesed by location statistics":2
+; CHECK: "params with 0% of its scope covered":0
+; CHECK: "params with (0%,10%) of its scope covered":0
+; CHECK: "params with [10%,20%) of its scope covered":0
+; CHECK: "params with [20%,30%) of its scope covered":0
+; CHECK: "params with [30%,40%) of its scope covered":0
+; CHECK: "params with [40%,50%) of its scope covered":0
+; CHECK: "params with [50%,60%) of its scope covered":0
+; CHECK: "params with [60%,70%) of its scope covered":0
+; CHECK: "params with [70%,80%) of its scope covered":0
+; CHECK: "params with [80%,90%) of its scope covered":0
+; CHECK: "params with [90%,100%) of its scope covered":0
+; CHECK: "params with 100% of its scope covered":2
+; CHECK: "params (excluding the debug entry values) with 0% of its scope covered":0
+; CHECK: "params (excluding the debug entry values) with (0%,10%) of its scope covered":0
+; CHECK: "params (excluding the debug entry values) with [10%,20%) of its scope covered":0
+; CHECK: "params (excluding the debug entry values) with [20%,30%) of its scope covered":0
+; CHECK: "params (excluding the debug entry values) with [30%,40%) of its scope covered":0
+; CHECK: "params (excluding the debug entry values) with [40%,50%) of its scope covered":0
+; CHECK: "params (excluding the debug entry values) with [50%,60%) of its scope covered":1
+; CHECK: "params (excluding the debug entry values) with [60%,70%) of its scope covered":0
+; CHECK: "params (excluding the debug entry values) with [70%,80%) of its scope covered":0
+; CHECK: "params (excluding the debug entry values) with [80%,90%) of its scope covered":0
+; CHECK: "params (excluding the debug entry values) with [90%,100%) of its scope covered":0
+; CHECK: "params (excluding the debug entry values) with 100% of its scope covered":1
+; CHECK: "total vars procesed by location statistics":4
+; CHECK: "vars with 0% of its scope covered":1
+; CHECK: "vars with (0%,10%) of its scope covered":0
+; CHECK: "vars with [10%,20%) of its scope covered":0
+; CHECK: "vars with [20%,30%) of its scope covered":0
+; CHECK: "vars with [30%,40%) of its scope covered":0
+; CHECK: "vars with [40%,50%) of its scope covered":0
+; CHECK: "vars with [50%,60%) of its scope covered":1
+; CHECK: "vars with [60%,70%) of its scope covered":0
+; CHECK: "vars with [70%,80%) of its scope covered":0
+; CHECK: "vars with [80%,90%) of its scope covered":1
+; CHECK: "vars with [90%,100%) of its scope covered":0
+; CHECK: "vars with 100% of its scope covered":1
+; CHECK: "vars (excluding the debug entry values) with 0% of its scope covered":1
+; CHECK: "vars (excluding the debug entry values) with (0%,10%) of its scope covered":0
+; CHECK: "vars (excluding the debug entry values) with [10%,20%) of its scope covered":0
+; CHECK: "vars (excluding the debug entry values) with [20%,30%) of its scope covered":0
+; CHECK: "vars (excluding the debug entry values) with [30%,40%) of its scope covered":0
+; CHECK: "vars (excluding the debug entry values) with [40%,50%) of its scope covered":0
+; CHECK: "vars (excluding the debug entry values) with [50%,60%) of its scope covered":1
+; CHECK: "vars (excluding the debug entry values) with [60%,70%) of its scope covered":0
+; CHECK: "vars (excluding the debug entry values) with [70%,80%) of its scope covered":0
+; CHECK: "vars (excluding the debug entry values) with [80%,90%) of its scope covered":1
+; CHECK: "vars (excluding the debug entry values) with [90%,100%) of its scope covered":0
+; CHECK: "vars (excluding the debug entry values) with 100% of its scope covered":1
+;
+; The source code of the test case:
+; extern void fn3(int *);
+; extern void fn2 (int);
+; __attribute__((noinline))
+; void
+; fn1 (int x, int y)
+; {
+;   int u = x + y;
+;   if (x > 1)
+;     u += 1;
+;   else
+;     u += 2;
+;   if (y > 4)
+;     u += x;
+;   int a = 7;
+;   fn2 (a);
+;   u --;
+; }
+;
+; __attribute__((noinline))
+; int f()
+; {
+;   int l, k;
+;   fn3(&l);
+;   fn3(&k);
+;   fn1 (l, k);
+;   return 0;
+; }
+;
+; ModuleID = 'test.c'
+source_filename = "test.c"
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Function Attrs: noinline nounwind uwtable
+define dso_local void @fn1(i32 %x, i32 %y) local_unnamed_addr !dbg !16 {
+entry:
+  call void @llvm.dbg.value(metadata i32 %x, metadata !20, metadata !DIExpression()), !dbg !24
+  call void @llvm.dbg.value(metadata i32 %y, metadata !21, metadata !DIExpression()), !dbg !24
+  call void @llvm.dbg.value(metadata i32 undef, metadata !22, metadata !DIExpression()), !dbg !24
+  call void @llvm.dbg.value(metadata i32 undef, metadata !22, metadata !DIExpression()), !dbg !24
+  call void @llvm.dbg.value(metadata i32 7, metadata !23, metadata !DIExpression()), !dbg !24
+  tail call void @fn2(i32 7), !dbg !25
+  call void @llvm.dbg.value(metadata i32 undef, metadata !22, metadata !DIExpression(DW_OP_constu, 1, DW_OP_minus, DW_OP_stack_value)), !dbg !24
+  ret void, !dbg !26
+}
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.lifetime.start.p0i8(i64 immarg, i8* nocapture)
+
+declare !dbg !4 dso_local void @fn2(i32) local_unnamed_addr
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.lifetime.end.p0i8(i64 immarg, i8* nocapture)
+
+; Function Attrs: noinline nounwind uwtable
+define dso_local i32 @f() local_unnamed_addr !dbg !27 {
+entry:
+  %l = alloca i32, align 4
+  %k = alloca i32, align 4
+  %0 = bitcast i32* %l to i8*, !dbg !33
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* nonnull %0), !dbg !33
+  %1 = bitcast i32* %k to i8*, !dbg !33
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* nonnull %1), !dbg !33
+  call void @llvm.dbg.value(metadata i32* %l, metadata !31, metadata !DIExpression(DW_OP_deref)), !dbg !34
+  call void @fn3(i32* nonnull %l), !dbg !35
+  call void @llvm.dbg.value(metadata i32* %k, metadata !32, metadata !DIExpression(DW_OP_deref)), !dbg !34
+  call void @fn3(i32* nonnull %k), !dbg !36
+  %2 = load i32, i32* %l, align 4, !dbg !37
+  call void @llvm.dbg.value(metadata i32 %2, metadata !31, metadata !DIExpression()), !dbg !34
+  %3 = load i32, i32* %k, align 4, !dbg !37
+  call void @llvm.dbg.value(metadata i32 %3, metadata !32, metadata !DIExpression()), !dbg !34
+  call void @fn1(i32 %2, i32 %3), !dbg !37
+  call void @llvm.lifetime.end.p0i8(i64 4, i8* nonnull %1), !dbg !37
+  call void @llvm.lifetime.end.p0i8(i64 4, i8* nonnull %0), !dbg !37
+  ret i32 0, !dbg !37
+}
+
+declare !dbg !8 dso_local void @fn3(i32*) local_unnamed_addr
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!12, !13, !14}
+!llvm.ident = !{!15}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 10.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !3, nameTableKind: None)
+!1 = !DIFile(filename: "test.c", directory: "/")
+!2 = !{}
+!3 = !{!4, !8}
+!4 = !DISubprogram(name: "fn2", scope: !1, file: !1, line: 2, type: !5, flags: DIFlagPrototyped, spFlags: DISPFlagOptimized, retainedNodes: !2)
+!5 = !DISubroutineType(types: !6)
+!6 = !{null, !7}
+!7 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!8 = !DISubprogram(name: "fn3", scope: !1, file: !1, line: 1, type: !9, flags: DIFlagPrototyped, spFlags: DISPFlagOptimized, retainedNodes: !2)
+!9 = !DISubroutineType(types: !10)
+!10 = !{null, !11}
+!11 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !7, size: 64)
+!12 = !{i32 2, !"Dwarf Version", i32 4}
+!13 = !{i32 2, !"Debug Info Version", i32 3}
+!14 = !{i32 1, !"wchar_size", i32 4}
+!15 = !{!"clang version 10.0.0"}
+!16 = distinct !DISubprogram(name: "fn1", scope: !1, file: !1, line: 6, type: !17, scopeLine: 7, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !19)
+!17 = !DISubroutineType(types: !18)
+!18 = !{null, !7, !7}
+!19 = !{!20, !21, !22, !23}
+!20 = !DILocalVariable(name: "x", arg: 1, scope: !16, file: !1, line: 6, type: !7)
+!21 = !DILocalVariable(name: "y", arg: 2, scope: !16, file: !1, line: 6, type: !7)
+!22 = !DILocalVariable(name: "u", scope: !16, file: !1, line: 8, type: !7)
+!23 = !DILocalVariable(name: "a", scope: !16, file: !1, line: 18, type: !7)
+!24 = !DILocation(line: 0, scope: !16)
+!25 = !DILocation(line: 20, column: 3, scope: !16)
+!26 = !DILocation(line: 22, column: 1, scope: !16)
+!27 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 25, type: !28, scopeLine: 26, flags: DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !30)
+!28 = !DISubroutineType(types: !29)
+!29 = !{!7}
+!30 = !{!31, !32}
+!31 = !DILocalVariable(name: "l", scope: !27, file: !1, line: 27, type: !7)
+!32 = !DILocalVariable(name: "k", scope: !27, file: !1, line: 27, type: !7)
+!33 = !DILocation(line: 27, column: 3, scope: !27)
+!34 = !DILocation(line: 0, scope: !27)
+!35 = !DILocation(line: 29, column: 3, scope: !27)
+!36 = !DILocation(line: 30, column: 3, scope: !27)
+!37 = !DILocation(line: 32, column: 8, scope: !27)
diff --git a/llvm/test/tools/llvm-dwarfdump/X86/statistics.ll b/llvm/test/tools/llvm-dwarfdump/X86/statistics.ll
index 221ef6022..f8d9326ff 100644
--- a/llvm/test/tools/llvm-dwarfdump/X86/statistics.ll
+++ b/llvm/test/tools/llvm-dwarfdump/X86/statistics.ll
@@ -1,6 +1,6 @@
 ; RUN: llc -O0 %s -o - -filetype=obj \
 ; RUN:   | llvm-dwarfdump -statistics - | FileCheck %s
-; CHECK: "version":3
+; CHECK: "version":4
 
 ; int GlobalConst = 42;
 ; int Global;
diff --git a/llvm/test/tools/llvm-dwarfdump/X86/stats-dbg-callsite-info.ll b/llvm/test/tools/llvm-dwarfdump/X86/stats-dbg-callsite-info.ll
new file mode 100644
index 000000000..c304e9d76
--- /dev/null
+++ b/llvm/test/tools/llvm-dwarfdump/X86/stats-dbg-callsite-info.ll
@@ -0,0 +1,76 @@
+; RUN: llc -debug-entry-values %s -o - -filetype=obj \
+; RUN:   | llvm-dwarfdump -statistics - | FileCheck %s
+;
+; The LLVM IR file was generated on this source code by using
+; option '-femit-debug-entry-values'.
+;
+; extern void foo(int *a, int b, int c, int d, int e, int f);
+; extern int getVal();
+;
+; void baa(int arg1, int arg2, int arg3) {
+;   int local1 = getVal();
+;   foo(&local1, arg2, 10, 15, arg3 + 3, arg1 + arg2);
+; }
+;
+; CHECK: "call site DIEs":2
+; CHECK-SAME: "call site parameter DIEs":6
+;
+; ModuleID = 'test.c'
+source_filename = "test.c"
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Function Attrs: nounwind uwtable
+define dso_local void @baa(i32 %arg1, i32 %arg2, i32 %arg3) local_unnamed_addr #0 !dbg !10 {
+entry:
+  %local1 = alloca i32, align 4
+  call void @llvm.dbg.value(metadata i32 %arg1, metadata !15, metadata !DIExpression()), !dbg !19
+  call void @llvm.dbg.value(metadata i32 %arg2, metadata !16, metadata !DIExpression()), !dbg !20
+  call void @llvm.dbg.value(metadata i32 %arg3, metadata !17, metadata !DIExpression()), !dbg !21
+  %0 = bitcast i32* %local1 to i8*, !dbg !22
+  %call = tail call i32 (...) @getVal(), !dbg !23
+  call void @llvm.dbg.value(metadata i32 %call, metadata !18, metadata !DIExpression()), !dbg !24
+  store i32 %call, i32* %local1, align 4, !dbg !24
+  %add = add nsw i32 %arg3, 3, !dbg !24
+  %add1 = add nsw i32 %arg2, %arg1, !dbg !24
+  call void @llvm.dbg.value(metadata i32* %local1, metadata !18, metadata !DIExpression(DW_OP_deref)), !dbg !24
+  call void @foo(i32* nonnull %local1, i32 %arg2, i32 10, i32 15, i32 %add, i32 %add1), !dbg !24
+  ret void, !dbg !24
+}
+
+declare !dbg !4 dso_local i32 @getVal(...) local_unnamed_addr
+
+declare !dbg !5 dso_local void @foo(i32*, i32, i32, i32, i32, i32) local_unnamed_addr
+
+; Function Attrs: nounwind readnone speculatable
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!6, !7, !8}
+!llvm.ident = !{!9}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 9.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !3, nameTableKind: None)
+!1 = !DIFile(filename: "test.c", directory: "/dir")
+!2 = !{}
+!3 = !{!4, !5}
+!4 = !DISubprogram(name: "getVal", scope: !1, file: !1, line: 2, spFlags: DISPFlagOptimized, retainedNodes: !2)
+!5 = !DISubprogram(name: "foo", scope: !1, file: !1, line: 1, flags: DIFlagPrototyped, spFlags: DISPFlagOptimized, retainedNodes: !2)
+!6 = !{i32 2, !"Dwarf Version", i32 4}
+!7 = !{i32 2, !"Debug Info Version", i32 3}
+!8 = !{i32 1, !"wchar_size", i32 4}
+!9 = !{!"clang version 9.0.0"}
+!10 = distinct !DISubprogram(name: "baa", scope: !1, file: !1, line: 4, type: !11, scopeLine: 4, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !14)
+!11 = !DISubroutineType(types: !12)
+!12 = !{null, !13, !13, !13}
+!13 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!14 = !{!15, !16, !17, !18}
+!15 = !DILocalVariable(name: "arg1", arg: 1, scope: !10, file: !1, line: 4, type: !13)
+!16 = !DILocalVariable(name: "arg2", arg: 2, scope: !10, file: !1, line: 4, type: !13)
+!17 = !DILocalVariable(name: "arg3", arg: 3, scope: !10, file: !1, line: 4, type: !13)
+!18 = !DILocalVariable(name: "local1", scope: !10, file: !1, line: 5, type: !13)
+!19 = !DILocation(line: 4, column: 14, scope: !10)
+!20 = !DILocation(line: 4, column: 24, scope: !10)
+!21 = !DILocation(line: 4, column: 34, scope: !10)
+!22 = !DILocation(line: 5, column: 3, scope: !10)
+!23 = !DILocation(line: 5, column: 16, scope: !10)
+!24 = !DILocation(line: 5, column: 7, scope: !10)
diff --git a/llvm/test/tools/llvm-dwarfdump/X86/valid-call-site-GNU-extensions.ll b/llvm/test/tools/llvm-dwarfdump/X86/valid-call-site-GNU-extensions.ll
new file mode 100644
index 000000000..935f3a2d1
--- /dev/null
+++ b/llvm/test/tools/llvm-dwarfdump/X86/valid-call-site-GNU-extensions.ll
@@ -0,0 +1,103 @@
+; RUN: llc -debug-entry-values %s -o - -filetype=obj \
+; RUN:   | llvm-dwarfdump -verify - | FileCheck %s
+;
+; CHECK: No errors.
+;
+; The source code of the test case:
+;
+;extern int fn1();
+;extern int fn3(int);
+;
+;__attribute__((noinline))
+;void
+;fn2 (int *arg) {
+;  int a = ++(*arg);
+;  fn3 (a);
+;}
+;
+;__attribute__((noinline))
+;int f() {
+;  int x = fn1();
+;  fn2 (&x);
+;  return 0;
+;}
+;
+; ModuleID = 'test.c'
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Function Attrs: noinline nounwind uwtable
+define dso_local void @fn2(i32* nocapture %arg) local_unnamed_addr !dbg !15 {
+entry:
+  call void @llvm.dbg.value(metadata i32* %arg, metadata !20, metadata !DIExpression()), !dbg !22
+  %0 = load i32, i32* %arg, align 4, !dbg !23
+  %inc = add nsw i32 %0, 1, !dbg !23
+  store i32 %inc, i32* %arg, align 4, !dbg !23
+  call void @llvm.dbg.value(metadata i32 %inc, metadata !21, metadata !DIExpression()), !dbg !22
+  %call = tail call i32 @fn3(i32 %inc), !dbg !23
+  ret void, !dbg !23
+}
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.lifetime.start.p0i8(i64 immarg, i8* nocapture)
+
+declare !dbg !4 dso_local i32 @fn3(i32) local_unnamed_addr
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.lifetime.end.p0i8(i64 immarg, i8* nocapture)
+
+; Function Attrs: noinline nounwind uwtable
+define dso_local i32 @f() local_unnamed_addr !dbg !24 {
+entry:
+  %x = alloca i32, align 4
+  %0 = bitcast i32* %x to i8*, !dbg !29
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* nonnull %0), !dbg !29
+  %call = tail call i32 (...) @fn1() #4, !dbg !29
+  call void @llvm.dbg.value(metadata i32 %call, metadata !28, metadata !DIExpression()), !dbg !30
+  store i32 %call, i32* %x, align 4, !dbg !29
+  call void @llvm.dbg.value(metadata i32* %x, metadata !28, metadata !DIExpression(DW_OP_deref)), !dbg !30
+  call void @fn2(i32* nonnull %x), !dbg !29
+  call void @llvm.lifetime.end.p0i8(i64 4, i8* nonnull %0), !dbg !29
+  ret i32 0, !dbg !29
+}
+
+declare !dbg !8 dso_local i32 @fn1(...) local_unnamed_addr
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!11, !12, !13}
+!llvm.ident = !{!14}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 10.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !3, nameTableKind: None)
+!1 = !DIFile(filename: "test.c", directory: "dir")
+!2 = !{}
+!3 = !{!4, !8}
+!4 = !DISubprogram(name: "fn3", scope: !1, file: !1, line: 2, type: !5, flags: DIFlagPrototyped, spFlags: DISPFlagOptimized, retainedNodes: !2)
+!5 = !DISubroutineType(types: !6)
+!6 = !{!7, !7}
+!7 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!8 = !DISubprogram(name: "fn1", scope: !1, file: !1, line: 1, type: !9, spFlags: DISPFlagOptimized, retainedNodes: !2)
+!9 = !DISubroutineType(types: !10)
+!10 = !{!7, null}
+!11 = !{i32 2, !"Dwarf Version", i32 4}
+!12 = !{i32 2, !"Debug Info Version", i32 3}
+!13 = !{i32 1, !"wchar_size", i32 4}
+!14 = !{!"clang version 10.0.0"}
+!15 = distinct !DISubprogram(name: "fn2", scope: !1, file: !1, line: 6, type: !16, scopeLine: 6, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !19)
+!16 = !DISubroutineType(types: !17)
+!17 = !{null, !18}
+!18 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !7, size: 64)
+!19 = !{!20, !21}
+!20 = !DILocalVariable(name: "arg", arg: 1, scope: !15, file: !1, line: 6, type: !18)
+!21 = !DILocalVariable(name: "a", scope: !15, file: !1, line: 7, type: !7)
+!22 = !DILocation(line: 0, scope: !15)
+!23 = !DILocation(line: 7, column: 11, scope: !15)
+!24 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 12, type: !25, scopeLine: 12, flags: DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !27)
+!25 = !DISubroutineType(types: !17)
+!26 = !{!7}
+!27 = !{!28}
+!28 = !DILocalVariable(name: "x", scope: !24, file: !1, line: 13, type: !7)
+!29 = !DILocation(line: 13, column: 3, scope: !24)
+!30 = !DILocation(line: 0, scope: !24)
diff --git a/llvm/test/tools/llvm-lib/Inputs/arm64.ll b/llvm/test/tools/llvm-lib/Inputs/arm64.ll
index db0820952..f0a591370 100644
--- a/llvm/test/tools/llvm-lib/Inputs/arm64.ll
+++ b/llvm/test/tools/llvm-lib/Inputs/arm64.ll
@@ -9,7 +9,7 @@ entry:
   ret void
 }
 
-attributes #0 = { noinline nounwind optnone uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="generic" "target-features"="+neon" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noinline nounwind optnone uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="generic" "target-features"="+neon" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.module.flags = !{!0}
 !llvm.ident = !{!1}
diff --git a/llvm/test/tools/llvm-lib/Inputs/i386.ll b/llvm/test/tools/llvm-lib/Inputs/i386.ll
index 268022262..901e903d2 100644
--- a/llvm/test/tools/llvm-lib/Inputs/i386.ll
+++ b/llvm/test/tools/llvm-lib/Inputs/i386.ll
@@ -9,7 +9,7 @@ entry:
   ret void
 }
 
-attributes #0 = { noinline nounwind optnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noinline nounwind optnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.module.flags = !{!0, !1}
 !llvm.ident = !{!2}
diff --git a/llvm/test/tools/llvm-lib/Inputs/x86_64.ll b/llvm/test/tools/llvm-lib/Inputs/x86_64.ll
index 6a94068a9..dbfe30434 100644
--- a/llvm/test/tools/llvm-lib/Inputs/x86_64.ll
+++ b/llvm/test/tools/llvm-lib/Inputs/x86_64.ll
@@ -9,7 +9,7 @@ entry:
   ret void
 }
 
-attributes #0 = { noinline nounwind optnone uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noinline nounwind optnone uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.module.flags = !{!0, !1}
 !llvm.ident = !{!2}
diff --git a/llvm/test/tools/llvm-locstats/locstats.ll b/llvm/test/tools/llvm-locstats/locstats.ll
new file mode 100644
index 000000000..e267b7eeb
--- /dev/null
+++ b/llvm/test/tools/llvm-locstats/locstats.ll
@@ -0,0 +1,175 @@
+; UNSUPPORTED: system-windows
+; REQUIRES: x86-registered-target
+; RUN: llc %s -o %t0.o -filetype=obj
+; RUN: %llvm-locstats %t0.o | FileCheck %s --check-prefix=LOCSTATS
+;
+; Test the llvm-locstats output.
+; LOCSTATS: 0% 0 0%
+; LOCSTATS: (0%,10%) 0 0%
+; LOCSTATS: [10%,20%) 0 0%
+; LOCSTATS: [20%,30%) 1 11%
+; LOCSTATS: [30%,40%) 0 0%
+; LOCSTATS: [40%,50%) 1 11%
+; LOCSTATS: [50%,60%) 1 11%
+; LOCSTATS: [60%,70%) 0 0%
+; LOCSTATS: [70%,80%) 0 0%
+; LOCSTATS: [80%,90%) 3 33%
+; LOCSTATS: [90%,100%) 1 11%
+; LOCSTATS: 100% 2 22%
+;
+; The source code of the test case:
+;extern int fn2 (int);
+;
+;__attribute__((noinline))
+;int
+;fn1 (int *x, int *y)
+;{
+;  int a = *x;
+;  int b = *y;
+;  int local = a + b;
+;  if (a > 1) {
+;    local += 2;
+;    ++local;
+;    if (local > 200)
+;      local -= fn2(a);
+;  } else {
+;    local += 3;
+;    ++local;
+;    local += fn2(a);
+; }
+;  if (b > 4)
+;   local += a;
+;  int local2 = 7;
+;  local -= fn2 (local2);
+;  return local;
+;}
+;
+;__attribute__((noinline))
+;int f()
+;{
+;  int l, k;
+;  int res = 0;
+;  res += fn1 (&l, &k);
+;  return res;
+;}
+;
+; ModuleID = 'locstats.c'
+source_filename = "locstats.c"
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Function Attrs: noinline nounwind uwtable
+define dso_local i32 @fn1(i32* nocapture readonly %0, i32* nocapture readonly %1) local_unnamed_addr !dbg !7 {
+  call void @llvm.dbg.value(metadata i32* %0, metadata !13, metadata !DIExpression()), !dbg !19
+  call void @llvm.dbg.value(metadata i32* %1, metadata !14, metadata !DIExpression()), !dbg !19
+  %3 = load i32, i32* %0, align 4, !dbg !20
+  call void @llvm.dbg.value(metadata i32 %3, metadata !15, metadata !DIExpression()), !dbg !19
+  %4 = load i32, i32* %1, align 4, !dbg !20
+  call void @llvm.dbg.value(metadata i32 %4, metadata !16, metadata !DIExpression()), !dbg !19
+  %5 = add nsw i32 %4, %3, !dbg !20
+  call void @llvm.dbg.value(metadata i32 %5, metadata !17, metadata !DIExpression()), !dbg !19
+  %6 = icmp sgt i32 %3, 1, !dbg !20
+  br i1 %6, label %7, label %13, !dbg !22
+
+7:                                                ; preds = %2
+  call void @llvm.dbg.value(metadata i32 %5, metadata !17, metadata !DIExpression(DW_OP_plus_uconst, 2, DW_OP_stack_value)), !dbg !19
+  %8 = add nsw i32 %5, 3, !dbg !23
+  call void @llvm.dbg.value(metadata i32 %8, metadata !17, metadata !DIExpression()), !dbg !19
+  %9 = icmp sgt i32 %8, 200, !dbg !25
+  br i1 %9, label %10, label %17, !dbg !27
+
+10:                                               ; preds = %7
+  %11 = tail call i32 @fn2(i32 %3), !dbg !27
+  %12 = sub nsw i32 %8, %11, !dbg !27
+  call void @llvm.dbg.value(metadata i32 %12, metadata !17, metadata !DIExpression()), !dbg !19
+  br label %17, !dbg !27
+
+13:                                               ; preds = %2
+  call void @llvm.dbg.value(metadata i32 %5, metadata !17, metadata !DIExpression(DW_OP_plus_uconst, 3, DW_OP_stack_value)), !dbg !19
+  %14 = add nsw i32 %5, 4, !dbg !28
+  call void @llvm.dbg.value(metadata i32 %14, metadata !17, metadata !DIExpression()), !dbg !19
+  %15 = tail call i32 @fn2(i32 %3), !dbg !30
+  %16 = add nsw i32 %14, %15, !dbg !30
+  call void @llvm.dbg.value(metadata i32 %16, metadata !17, metadata !DIExpression()), !dbg !19
+  br label %17
+
+17:                                               ; preds = %7, %10, %13
+  %18 = phi i32 [ %12, %10 ], [ %8, %7 ], [ %16, %13 ], !dbg !31
+  call void @llvm.dbg.value(metadata i32 %18, metadata !17, metadata !DIExpression()), !dbg !19
+  %19 = icmp sgt i32 %4, 4, !dbg !32
+  %20 = select i1 %19, i32 %3, i32 0, !dbg !34
+  %21 = add nsw i32 %18, %20, !dbg !34
+  call void @llvm.dbg.value(metadata i32 %21, metadata !17, metadata !DIExpression()), !dbg !19
+  call void @llvm.dbg.value(metadata i32 7, metadata !18, metadata !DIExpression()), !dbg !19
+  %22 = tail call i32 @fn2(i32 7), !dbg !34
+  %23 = sub i32 %21, %22, !dbg !34
+  call void @llvm.dbg.value(metadata i32 %23, metadata !17, metadata !DIExpression()), !dbg !19
+  ret i32 %23, !dbg !34
+}
+
+declare dso_local i32 @fn2(i32) local_unnamed_addr
+
+; Function Attrs: noinline nounwind uwtable
+define dso_local i32 @f() local_unnamed_addr !dbg !35 {
+  %1 = alloca i32, align 4
+  %2 = alloca i32, align 4
+  %3 = bitcast i32* %1 to i8*, !dbg !42
+  %4 = bitcast i32* %2 to i8*, !dbg !42
+  call void @llvm.dbg.value(metadata i32 0, metadata !41, metadata !DIExpression()), !dbg !42
+  call void @llvm.dbg.value(metadata i32* %1, metadata !39, metadata !DIExpression(DW_OP_deref)), !dbg !42
+  call void @llvm.dbg.value(metadata i32* %2, metadata !40, metadata !DIExpression(DW_OP_deref)), !dbg !42
+  %5 = call i32 @fn1(i32* nonnull %1, i32* nonnull %2), !dbg !42
+  call void @llvm.dbg.value(metadata i32 %5, metadata !41, metadata !DIExpression()), !dbg !42
+  ret i32 %5, !dbg !42
+}
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 10.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "locstats.c", directory: "/dir")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{!"clang version 10.0.0"}
+!7 = distinct !DISubprogram(name: "fn1", scope: !1, file: !1, line: 5, type: !8, scopeLine: 6, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !12)
+!8 = !DISubroutineType(types: !9)
+!9 = !{!10, !11, !11}
+!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!11 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !10, size: 64)
+!12 = !{!13, !14, !15, !16, !17, !18}
+!13 = !DILocalVariable(name: "x", arg: 1, scope: !7, file: !1, line: 5, type: !11)
+!14 = !DILocalVariable(name: "y", arg: 2, scope: !7, file: !1, line: 5, type: !11)
+!15 = !DILocalVariable(name: "a", scope: !7, file: !1, line: 7, type: !10)
+!16 = !DILocalVariable(name: "b", scope: !7, file: !1, line: 8, type: !10)
+!17 = !DILocalVariable(name: "local", scope: !7, file: !1, line: 9, type: !10)
+!18 = !DILocalVariable(name: "local2", scope: !7, file: !1, line: 22, type: !10)
+!19 = !DILocation(line: 0, scope: !7)
+!20 = !DILocation(line: 7, column: 11, scope: !7)
+!21 = distinct !DILexicalBlock(scope: !7, file: !1, line: 10, column: 7)
+!22 = !DILocation(line: 10, column: 7, scope: !7)
+!23 = !DILocation(line: 12, column: 5, scope: !24)
+!24 = distinct !DILexicalBlock(scope: !21, file: !1, line: 10, column: 14)
+!25 = !DILocation(line: 13, column: 15, scope: !26)
+!26 = distinct !DILexicalBlock(scope: !24, file: !1, line: 13, column: 9)
+!27 = !DILocation(line: 13, column: 9, scope: !24)
+!28 = !DILocation(line: 17, column: 5, scope: !26)
+!29 = distinct !DILexicalBlock(scope: !21, file: !1, line: 15, column: 10)
+!30 = !DILocation(line: 18, column: 14, scope: !29)
+!31 = !DILocation(line: 0, scope: !21)
+!32 = !DILocation(line: 20, column: 9, scope: !33)
+!33 = distinct !DILexicalBlock(scope: !7, file: !1, line: 20, column: 7)
+!34 = !DILocation(line: 20, column: 7, scope: !7)
+!35 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 28, type: !36, scopeLine: 29, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !38)
+!36 = !DISubroutineType(types: !37)
+!37 = !{!10}
+!38 = !{!39, !40, !41}
+!39 = !DILocalVariable(name: "l", scope: !35, file: !1, line: 30, type: !10)
+!40 = !DILocalVariable(name: "k", scope: !35, file: !1, line: 30, type: !10)
+!41 = !DILocalVariable(name: "res", scope: !35, file: !1, line: 31, type: !10)
+!42 = !DILocation(line: 30, column: 3, scope: !35)
diff --git a/llvm/test/tools/llvm-locstats/no_scope_bytes.ll b/llvm/test/tools/llvm-locstats/no_scope_bytes.ll
new file mode 100644
index 000000000..9c36b65e8
--- /dev/null
+++ b/llvm/test/tools/llvm-locstats/no_scope_bytes.ll
@@ -0,0 +1,39 @@
+; UNSUPPORTED: system-windows
+; REQUIRES: x86-registered-target
+; RUN: llc %s -o %t0.o -filetype=obj
+; RUN: %llvm-locstats %t0.o | FileCheck %s --check-prefix=LOCSTATS
+;
+; LOCSTATS: No scope bytes found.
+;
+; This is based on the following reproducer:
+;
+; int fn() {
+;  return 0;
+; }
+;
+; ModuleID = 'test.c'
+source_filename = "test.c"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+; Function Attrs: norecurse nounwind readnone uwtable
+define dso_local i32 @fn() local_unnamed_addr !dbg !7 {
+entry:
+  ret i32 0, !dbg !11
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 10.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "test.c", directory: "/")
+!2 = !{}
+!3 = !{i32 7, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{!"clang version 10.0.0"}
+!7 = distinct !DISubprogram(name: "fn", scope: !1, file: !1, line: 1, type: !8, scopeLine: 1, flags: DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2)
+!8 = !DISubroutineType(types: !9)
+!9 = !{!10}
+!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!11 = !DILocation(line: 2, column: 3, scope: !7)
diff --git a/llvm/test/tools/llvm-lto2/X86/pipeline.ll b/llvm/test/tools/llvm-lto2/X86/pipeline.ll
index 9ab81ac70..f9759b590 100644
--- a/llvm/test/tools/llvm-lto2/X86/pipeline.ll
+++ b/llvm/test/tools/llvm-lto2/X86/pipeline.ll
@@ -15,7 +15,7 @@
 ; is accepted).
 ; RUN: llvm-lto2 run %t1.bc -o %t.o -use-new-pm -r %t1.bc,patatino,px
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define void @patatino() {
diff --git a/llvm/test/tools/llvm-lto2/X86/slp-vectorize-pm.ll b/llvm/test/tools/llvm-lto2/X86/slp-vectorize-pm.ll
new file mode 100644
index 000000000..2d9ef9d1b
--- /dev/null
+++ b/llvm/test/tools/llvm-lto2/X86/slp-vectorize-pm.ll
@@ -0,0 +1,51 @@
+; RUN: opt -module-summary %s -o %t1.bc
+
+; Test SLP and Loop Vectorization are enabled by default at O2 and O3.
+; RUN: llvm-lto2 run %t1.bc -o %t2.o -O0 -r %t1.bc,foo,plx -debug-pass-manager \
+; RUN:  -use-new-pm -save-temps 2>&1 | FileCheck %s --check-prefix=CHECK-O0-SLP
+; RUN: llvm-dis %t2.o.1.4.opt.bc -o - | FileCheck %s --check-prefix=CHECK-O0-LPV
+
+; RUN: llvm-lto2 run %t1.bc -o %t3.o -O1 -r %t1.bc,foo,plx -debug-pass-manager \
+; RUN:  -use-new-pm -save-temps 2>&1 | FileCheck %s --check-prefix=CHECK-O1-SLP
+; RUN: llvm-dis %t3.o.1.4.opt.bc -o - | FileCheck %s --check-prefix=CHECK-O1-LPV
+
+; RUN: llvm-lto2 run %t1.bc -o %t4.o -O2 -r %t1.bc,foo,plx -debug-pass-manager \
+; RUN:  -use-new-pm -save-temps 2>&1 | FileCheck %s --check-prefix=CHECK-O2-SLP
+; RUN: llvm-dis %t4.o.1.4.opt.bc -o - | FileCheck %s --check-prefix=CHECK-O2-LPV
+
+; RUN: llvm-lto2 run %t1.bc -o %t5.o -O3 -r %t1.bc,foo,plx -debug-pass-manager \
+; RUN:  -use-new-pm -save-temps 2>&1 | FileCheck %s --check-prefix=CHECK-O3-SLP
+; RUN: llvm-dis %t5.o.1.4.opt.bc -o - | FileCheck %s --check-prefix=CHECK-O3-LPV
+
+; CHECK-O0-SLP-NOT: Running pass: SLPVectorizerPass
+; CHECK-O1-SLP-NOT: Running pass: SLPVectorizerPass
+; CHECK-O2-SLP: Running pass: SLPVectorizerPass
+; CHECK-O3-SLP: Running pass: SLPVectorizerPass
+; CHECK-O0-LPV-NOT: = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK-O1-LPV-NOT: = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK-O2-LPV: = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK-O3-LPV: = !{!"llvm.loop.isvectorized", i32 1}
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define i32 @foo(i32* %a) {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %red.05 = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add nsw i32 %0, %red.05
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 255
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !0
+
+for.end:
+  ret i32 %add
+}
+
+!0 = distinct !{!0, !1}
+!1 = !{!"llvm.loop.unroll.disable", i1 true}
diff --git a/llvm/test/tools/llvm-lto2/X86/stats-file-option.ll b/llvm/test/tools/llvm-lto2/X86/stats-file-option.ll
index c73c5021f..948c52202 100644
--- a/llvm/test/tools/llvm-lto2/X86/stats-file-option.ll
+++ b/llvm/test/tools/llvm-lto2/X86/stats-file-option.ll
@@ -6,7 +6,7 @@
 ; RUN: llvm-lto2 run %t1.bc -o %t.o -r %t1.bc,patatino,px -stats-file=%t2.stats
 ; RUN: FileCheck --input-file=%t2.stats %s
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define void @patatino() {
diff --git a/llvm/test/tools/llvm-objdump/AMDGPU/source-lines.ll b/llvm/test/tools/llvm-objdump/AMDGPU/source-lines.ll
index 4a4203d2a..2e31f6612 100644
--- a/llvm/test/tools/llvm-objdump/AMDGPU/source-lines.ll
+++ b/llvm/test/tools/llvm-objdump/AMDGPU/source-lines.ll
@@ -67,7 +67,7 @@ entry:
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
 
-attributes #0 = { noinline nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="gfx700" "target-features"="+fp64-fp16-denormals,-fp32-denormals" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noinline nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-features"="+fp64-fp16-denormals,-fp32-denormals" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/tools/llvm-objdump/Hexagon/source-interleave-hexagon.ll b/llvm/test/tools/llvm-objdump/Hexagon/source-interleave-hexagon.ll
index 02cef6764..35ca7065e 100644
--- a/llvm/test/tools/llvm-objdump/Hexagon/source-interleave-hexagon.ll
+++ b/llvm/test/tools/llvm-objdump/Hexagon/source-interleave-hexagon.ll
@@ -34,7 +34,7 @@ entry:
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
 
-attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="hexagonv60" "target-features"="-hvx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="hexagonv60" "target-features"="-hvx" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/tools/llvm-objdump/X86/Inputs/source-interleave-same-line-different-file.ll b/llvm/test/tools/llvm-objdump/X86/Inputs/source-interleave-same-line-different-file.ll
index 4d4ec3557..8b8b90152 100644
--- a/llvm/test/tools/llvm-objdump/X86/Inputs/source-interleave-same-line-different-file.ll
+++ b/llvm/test/tools/llvm-objdump/X86/Inputs/source-interleave-same-line-different-file.ll
@@ -30,7 +30,7 @@ entry:
   ret i32 4, !dbg !21
 }
 
-attributes #0 = { noinline nounwind optnone uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noinline nounwind optnone uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone speculatable }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/tools/llvm-objdump/X86/Inputs/source-interleave.ll b/llvm/test/tools/llvm-objdump/X86/Inputs/source-interleave.ll
index 8d49d01f0..96fb72e08 100644
--- a/llvm/test/tools/llvm-objdump/X86/Inputs/source-interleave.ll
+++ b/llvm/test/tools/llvm-objdump/X86/Inputs/source-interleave.ll
@@ -33,7 +33,7 @@ entry:
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
 
-attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/tools/llvm-objdump/X86/disassemble-archive-with-source.ll b/llvm/test/tools/llvm-objdump/X86/disassemble-archive-with-source.ll
index 151254978..2fbbe12a1 100644
--- a/llvm/test/tools/llvm-objdump/X86/disassemble-archive-with-source.ll
+++ b/llvm/test/tools/llvm-objdump/X86/disassemble-archive-with-source.ll
@@ -25,7 +25,7 @@ entry:
 
 declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
 
-attributes #0 = { noinline nounwind optnone sspstrong uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noinline nounwind optnone sspstrong uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone speculatable }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/tools/llvm-objdump/X86/macho-data-in-code.ll b/llvm/test/tools/llvm-objdump/X86/macho-data-in-code.ll
new file mode 100644
index 000000000..57877da09
--- /dev/null
+++ b/llvm/test/tools/llvm-objdump/X86/macho-data-in-code.ll
@@ -0,0 +1,45 @@
+; RUN: llc -mtriple x86_64-apple-darwin -filetype=obj -O0 %s -o %t.o
+; RUN: llvm-objdump -macho -disassemble -no-show-raw-insn %t.o | FileCheck %s
+
+; CHECK: .long {{[0-9]+}}	@ KIND_JUMP_TABLE32
+; CHECK: .long {{[0-9]+}}	@ KIND_JUMP_TABLE32
+; CHECK: .long {{[0-9]+}}	@ KIND_JUMP_TABLE32
+; CHECK: .long {{[0-9]+}}	@ KIND_JUMP_TABLE32
+; CHECK-NOT: invalid instruction encoding
+; CHECK-NOT: <unknown>
+
+; ModuleID = '-'
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.12.0"
+
+; Function Attrs: noinline nounwind optnone ssp uwtable
+define void @switchfunc(i32 %i) {
+  switch i32 %i, label %out [
+    i32 0, label %case1
+    i32 1, label %case2
+    i32 2, label %case3
+    i32 3, label %case4
+  ]
+
+case1:
+  call void @foo()
+  br label %out
+
+case2:
+  call void @bar()
+  br label %out
+
+case3:
+  call void @foo()
+  br label %out
+
+case4:
+  call void @bar()
+  br label %out
+
+out:
+  ret void
+}
+
+declare void @foo()
+declare void @bar()
diff --git a/llvm/test/tools/llvm-readobj/ELF/Inputs/trivial.ll b/llvm/test/tools/llvm-readobj/ELF/Inputs/trivial.ll
new file mode 100644
index 000000000..d3fdb40ae
--- /dev/null
+++ b/llvm/test/tools/llvm-readobj/ELF/Inputs/trivial.ll
@@ -0,0 +1,20 @@
+; Input used for generating checked-in binaries (trivial.obj.*)
+; llc -mtriple=i386-pc-win32 trivial.ll -filetype=obj -o trivial.obj.coff-i386
+; llc -mtriple=x86_64-pc-win32 trivial.ll -filetype=obj -o trivial.obj.coff-x86-64
+; llc -mtriple=i386-linux-gnu trivial.ll -filetype=obj -o trivial.obj.elf-i386 -relocation-model=pic
+; llc -mtriple=i386-apple-darwin10 trivial.ll -filetype=obj -o trivial.obj.macho-i386 -relocation-model=pic
+; llc -mtriple=x86_64-apple-darwin10 trivial.ll -filetype=obj -o trivial.obj.macho-x86-64 -relocation-model=pic
+; llc -mtriple=wasm32-unknown-unknown trivial.ll -filetype=obj -o trivial.obj.wasm
+
+@.str = private unnamed_addr constant [13 x i8] c"Hello World\0A\00", align 1
+
+define i32 @main() nounwind {
+entry:
+  %call = tail call i32 @puts(i8* getelementptr inbounds ([13 x i8], [13 x i8]* @.str, i32 0, i32 0)) nounwind
+  tail call void bitcast (void (...)* @SomeOtherFunction to void ()*)() nounwind
+  ret i32 0
+}
+
+declare i32 @puts(i8* nocapture) nounwind
+
+declare void @SomeOtherFunction(...)